From 9caf68cf7b47fb94ac4f574772bf5efdb5785555 Mon Sep 17 00:00:00 2001 From: Steve Nesae Date: Tue, 6 Nov 2018 12:47:02 -0600 Subject: [PATCH] Ambiq squashed commits --- .../ISSUE_TEMPLATE/40-tflite-op-request.md | 24 + README.md | 6 +- WORKSPACE | 22 +- configure.py | 22 +- tensorflow/BUILD | 38 +- tensorflow/api_template.__init__.py | 12 +- tensorflow/c/BUILD | 62 +- tensorflow/c/c_api_experimental.cc | 130 +- tensorflow/c/c_api_experimental.h | 37 + tensorflow/c/c_api_experimental_test.cc | 134 + tensorflow/c/c_api_function.cc | 22 +- tensorflow/c/eager/BUILD | 2 + tensorflow/c/eager/c_api.cc | 49 +- tensorflow/c/eager/c_api.h | 23 + tensorflow/c/eager/c_api_internal.h | 7 +- tensorflow/c/eager/c_api_test.cc | 80 +- tensorflow/c/eager/c_api_test_util.cc | 13 + tensorflow/c/eager/c_api_test_util.h | 3 + tensorflow/c/eager/tape.h | 23 +- tensorflow/c/kernels.cc | 143 + tensorflow/c/kernels.h | 110 + tensorflow/c/kernels_test.cc | 194 + tensorflow/c/python_api.cc | 13 + tensorflow/c/python_api.h | 8 + tensorflow/cc/BUILD | 1 + tensorflow/cc/saved_model/BUILD | 1 + tensorflow/cc/saved_model/constants.h | 9 +- tensorflow/cc/saved_model/loader.cc | 88 +- tensorflow/cc/saved_model/loader_test.cc | 14 + .../half_plus_two_v2/00000123/assets/foo.txt | 1 + .../half_plus_two_v2/00000123/saved_model.pb | Bin 0 -> 10774 bytes .../variables/variables.data-00000-of-00001 | Bin 0 -> 12 bytes .../00000123/variables/variables.index | Bin 0 -> 151 bytes tensorflow/compiler/aot/codegen.cc | 30 +- tensorflow/compiler/aot/codegen.h | 2 +- tensorflow/compiler/aot/codegen_test.cc | 16 +- tensorflow/compiler/aot/codegen_test_h.golden | 16 +- tensorflow/compiler/aot/codegen_test_o.golden | Bin 712 -> 720 bytes tensorflow/compiler/aot/compile.cc | 20 +- tensorflow/compiler/aot/compile.h | 6 +- .../compiler/aot/tests/tfcompile_test.cc | 10 +- tensorflow/compiler/jit/BUILD | 28 +- tensorflow/compiler/jit/build_xla_ops_pass.cc | 10 +- .../compiler/jit/build_xla_ops_pass_test.cc | 8 +- .../compiler/jit/create_xla_launch_op_test.cc | 6 +- tensorflow/compiler/jit/encapsulate_util.cc | 325 +- tensorflow/compiler/jit/encapsulate_util.h | 97 +- .../compiler/jit/encapsulate_util_test.cc | 67 +- .../jit/encapsulate_xla_computations_pass.cc | 7 +- .../jit/extract_outside_compilation_pass.cc | 16 +- .../extract_outside_compilation_pass_test.cc | 15 +- tensorflow/compiler/jit/flags.cc | 152 + ...k_for_compilation_pass_flags.h => flags.h} | 67 +- .../increase_dynamism_for_auto_jit_pass.cc | 56 +- ...ncrease_dynamism_for_auto_jit_pass_test.cc | 130 +- tensorflow/compiler/jit/kernels/BUILD | 2 +- tensorflow/compiler/jit/kernels/xla_ops.cc | 4 +- tensorflow/compiler/jit/legacy_flags/BUILD | 65 - .../legacy_flags/build_xla_ops_pass_flags.cc | 47 - .../legacy_flags/build_xla_ops_pass_flags.h | 37 - .../mark_for_compilation_pass_flags.cc | 98 - .../jit/legacy_flags/xla_device_flags.cc | 56 - .../jit/legacy_flags/xla_device_flags.h | 47 - .../jit/legacy_flags/xla_ops_common_flags.cc | 52 - .../jit/legacy_flags/xla_ops_common_flags.h | 36 - .../compiler/jit/mark_for_compilation_pass.cc | 53 +- .../jit/mark_for_compilation_pass_test.cc | 76 + .../mark_for_compilation_pass_test_helper.cc | 8 +- tensorflow/compiler/jit/ops/BUILD | 6 + .../jit/ops/xla_ops_grad.py} | 19 +- .../compiler/jit/partially_decluster_pass.cc | 19 +- .../jit/partially_decluster_pass_test.cc | 6 +- tensorflow/compiler/jit/xla_cpu_device.cc | 15 +- tensorflow/compiler/jit/xla_device.cc | 82 +- tensorflow/compiler/jit/xla_device.h | 29 + tensorflow/compiler/jit/xla_gpu_device.cc | 10 +- .../compiler/jit/xla_interpreter_device.cc | 7 +- tensorflow/compiler/tests/BUILD | 22 - tensorflow/compiler/tests/adagrad_da_test.py | 32 +- tensorflow/compiler/tests/adagrad_test.py | 30 +- tensorflow/compiler/tests/adam_test.py | 39 +- tensorflow/compiler/tests/adamax_test.py | 24 +- tensorflow/compiler/tests/addsign_test.py | 8 +- tensorflow/compiler/tests/binary_ops_test.py | 15 + .../compiler/tests/categorical_op_test.py | 68 +- tensorflow/compiler/tests/clustering_test.py | 4 +- tensorflow/compiler/tests/concat_ops_test.py | 32 +- tensorflow/compiler/tests/conv3d_test.py | 6 +- tensorflow/compiler/tests/dense_layer_test.py | 8 +- .../compiler/tests/dynamic_stitch_test.py | 9 + tensorflow/compiler/tests/eager_test.py | 4 +- tensorflow/compiler/tests/fft_test.py | 68 +- tensorflow/compiler/tests/fifo_queue_test.py | 6 +- tensorflow/compiler/tests/ftrl_test.py | 84 +- tensorflow/compiler/tests/function_test.py | 12 +- tensorflow/compiler/tests/jit_test.py | 2 +- tensorflow/compiler/tests/listdiff_op_test.py | 4 +- tensorflow/compiler/tests/lrn_ops_test.py | 4 +- tensorflow/compiler/tests/lstm_test.py | 8 +- tensorflow/compiler/tests/momentum_test.py | 64 +- tensorflow/compiler/tests/placeholder_test.py | 2 +- tensorflow/compiler/tests/powersign_test.py | 8 +- .../compiler/tests/proximal_adagrad_test.py | 40 +- .../tests/proximal_gradient_descent_test.py | 38 +- tensorflow/compiler/tests/qr_op_test.py | 2 +- tensorflow/compiler/tests/random_ops_test.py | 16 +- tensorflow/compiler/tests/randomized_tests.cc | 4 +- tensorflow/compiler/tests/reduce_ops_test.py | 6 + tensorflow/compiler/tests/rmsprop_test.py | 24 +- tensorflow/compiler/tests/scan_ops_test.py | 4 +- .../tests/stateless_random_ops_test.py | 2 +- .../compiler/tests/tensor_array_ops_test.py | 27 +- tensorflow/compiler/tests/unary_ops_test.py | 66 + .../compiler/tests/variable_ops_test.py | 42 +- tensorflow/compiler/tests/xla_device_test.py | 2 +- tensorflow/compiler/tf2xla/BUILD | 16 +- tensorflow/compiler/tf2xla/dump_graph.cc | 77 +- .../compiler/tf2xla/dump_graph_flags.cc | 63 - tensorflow/compiler/tf2xla/dump_graph_flags.h | 48 - .../tf2xla/functionalize_control_flow.cc | 19 + .../tf2xla/functionalize_control_flow.h | 6 + .../tf2xla/functionalize_control_flow_test.cc | 915 +- tensorflow/compiler/tf2xla/kernels/BUILD | 5 +- tensorflow/compiler/tf2xla/kernels/arg_op.cc | 2 +- .../tf2xla/kernels/batch_matmul_op.cc | 12 +- .../compiler/tf2xla/kernels/batch_norm_op.cc | 15 +- .../compiler/tf2xla/kernels/bias_ops.cc | 4 +- .../compiler/tf2xla/kernels/binary_ops.cc | 19 +- .../compiler/tf2xla/kernels/categorical_op.cc | 68 +- .../tf2xla/kernels/conv_op_helpers.cc | 76 +- .../compiler/tf2xla/kernels/conv_ops.cc | 2 +- tensorflow/compiler/tf2xla/kernels/diag_op.cc | 2 +- .../tf2xla/kernels/dynamic_stitch_op.cc | 25 +- .../kernels/extract_image_patches_op.cc | 1 - .../tf2xla/kernels/fake_quantize_ops.cc | 8 +- tensorflow/compiler/tf2xla/kernels/fft_ops.cc | 28 +- tensorflow/compiler/tf2xla/kernels/if_op.cc | 2 +- .../compiler/tf2xla/kernels/image_ops.cc | 5 +- .../tf2xla/kernels/image_resize_ops.cc | 1 - .../compiler/tf2xla/kernels/index_ops_cpu.cc | 74 +- .../compiler/tf2xla/kernels/l2loss_op.cc | 5 +- tensorflow/compiler/tf2xla/kernels/lrn_ops.cc | 17 +- .../tf2xla/kernels/matrix_band_part_op.cc | 10 +- .../tf2xla/kernels/matrix_set_diag_op.cc | 1 - .../compiler/tf2xla/kernels/permute_op.cc | 3 +- .../compiler/tf2xla/kernels/pooling_ops.cc | 31 +- .../kernels/quantize_and_dequantize_op.cc | 43 +- .../compiler/tf2xla/kernels/random_ops.cc | 1 - .../compiler/tf2xla/kernels/reduction_ops.cc | 21 +- .../compiler/tf2xla/kernels/reduction_ops.h | 15 +- .../tf2xla/kernels/reduction_ops_common.cc | 18 +- .../compiler/tf2xla/kernels/resampler_ops.cc | 83 +- .../compiler/tf2xla/kernels/retval_op.cc | 3 +- .../tf2xla/kernels/reverse_sequence_op.cc | 1 - .../compiler/tf2xla/kernels/scan_ops.cc | 11 +- .../compiler/tf2xla/kernels/sendrecv_ops.cc | 4 +- .../compiler/tf2xla/kernels/sequence_ops.cc | 2 +- .../compiler/tf2xla/kernels/softmax_op.cc | 15 +- .../compiler/tf2xla/kernels/stack_ops.cc | 19 +- .../tf2xla/kernels/stateless_random_ops.cc | 1 - .../tf2xla/kernels/tensor_array_ops.cc | 33 +- tensorflow/compiler/tf2xla/kernels/topk_op.cc | 1 - .../compiler/tf2xla/kernels/training_ops.cc | 3 +- .../compiler/tf2xla/kernels/while_op.cc | 2 +- .../tf2xla/kernels/xla_broadcast_helper_op.cc | 7 +- tensorflow/compiler/tf2xla/lib/BUILD | 49 +- tensorflow/compiler/tf2xla/lib/batch_dot.cc | 115 - tensorflow/compiler/tf2xla/lib/batch_dot.h | 54 - tensorflow/compiler/tf2xla/lib/broadcast.cc | 6 +- tensorflow/compiler/tf2xla/lib/cholesky.cc | 17 +- tensorflow/compiler/tf2xla/lib/qr.cc | 35 +- .../compiler/tf2xla/lib/triangular_solve.cc | 28 +- tensorflow/compiler/tf2xla/lib/util.cc | 126 - tensorflow/compiler/tf2xla/lib/util.h | 34 - tensorflow/compiler/tf2xla/python/BUILD | 10 +- tensorflow/compiler/tf2xla/shape_util.h | 1 + .../tf2xla/xla_compiled_cpu_function.h | 25 +- tensorflow/compiler/tf2xla/xla_compiler.cc | 81 +- tensorflow/compiler/tf2xla/xla_compiler.h | 2 +- .../compiler/tf2xla/xla_compiler_test.cc | 83 +- tensorflow/compiler/tf2xla/xla_context.cc | 35 +- tensorflow/compiler/tf2xla/xla_context.h | 32 +- tensorflow/compiler/tf2xla/xla_helpers.cc | 4 +- tensorflow/compiler/tf2xla/xla_helpers.h | 3 +- .../tf2xla/xla_jit_compiled_cpu_function.cc | 3 +- .../tf2xla/xla_jit_compiled_cpu_function.h | 6 +- .../xla_jit_compiled_cpu_function_test.cc | 12 +- tensorflow/compiler/tf2xla/xla_op_kernel.cc | 55 +- tensorflow/compiler/tf2xla/xla_op_kernel.h | 2 + tensorflow/compiler/tf2xla/xla_op_registry.cc | 5 +- tensorflow/compiler/tf2xla/xla_resource.cc | 35 +- tensorflow/compiler/tf2xla/xla_resource.h | 19 +- tensorflow/compiler/xla/BUILD | 21 + tensorflow/compiler/xla/array2d.h | 2 +- tensorflow/compiler/xla/client/BUILD | 5 +- tensorflow/compiler/xla/client/client.cc | 18 +- .../xla/client/executable_build_options.cc | 68 +- .../xla/client/executable_build_options.h | 50 +- tensorflow/compiler/xla/client/lib/BUILD | 57 +- tensorflow/compiler/xla/client/lib/math.cc | 26 +- tensorflow/compiler/xla/client/lib/math.h | 4 + tensorflow/compiler/xla/client/lib/matrix.cc | 185 + .../xla/client/lib/{numeric.h => matrix.h} | 37 +- .../lib/{numeric_test.cc => matrix_test.cc} | 51 +- tensorflow/compiler/xla/client/lib/numeric.cc | 89 - tensorflow/compiler/xla/client/lib/prng.cc | 1 - tensorflow/compiler/xla/client/lib/slicing.cc | 134 + tensorflow/compiler/xla/client/lib/slicing.h | 48 + .../client/lib/slicing_test.cc} | 48 +- tensorflow/compiler/xla/client/lib/sorting.cc | 13 +- .../compiler/xla/client/lib/sorting_test.cc | 33 + tensorflow/compiler/xla/client/lib/testing.cc | 6 +- .../compiler/xla/client/local_client.cc | 24 + tensorflow/compiler/xla/client/local_client.h | 4 + .../compiler/xla/client/sharding_builder.cc | 4 +- tensorflow/compiler/xla/client/xla_builder.cc | 353 +- tensorflow/compiler/xla/client/xla_builder.h | 320 +- .../compiler/xla/client/xla_builder_test.cc | 13 +- .../compiler/xla/client/xla_computation.cc | 2 +- .../compiler/xla/client/xla_computation.h | 1 + .../compiler/xla/debug_options_flags.cc | 14 +- .../experimental/xla_sharding/xla_sharding.py | 30 +- tensorflow/compiler/xla/g3doc/_book.yaml | 12 +- tensorflow/compiler/xla/g3doc/_index.yaml | 4 +- .../g3doc/images/xla_array_layout_figure1.png | Bin 0 -> 20398 bytes .../g3doc/images/xla_array_layout_figure2.png | Bin 0 -> 7913 bytes tensorflow/compiler/xla/g3doc/jit.md | 4 +- .../compiler/xla/g3doc/layout_with_tiling.md | 159 + .../compiler/xla/g3doc/operation_semantics.md | 55 +- .../xla/g3doc/tutorials/xla_compile.ipynb | 301 +- tensorflow/compiler/xla/index_util.h | 1 + tensorflow/compiler/xla/layout_util.cc | 7 + tensorflow/compiler/xla/layout_util.h | 1 + tensorflow/compiler/xla/literal.cc | 326 +- tensorflow/compiler/xla/literal.h | 2 +- tensorflow/compiler/xla/literal_test.cc | 201 +- .../compiler/xla/parse_flags_from_env.cc | 116 +- .../compiler/xla/parse_flags_from_env.h | 56 +- .../compiler/xla/parse_flags_from_env_test.cc | 20 +- tensorflow/compiler/xla/protobuf_util.cc | 10 - .../xla/python/local_computation_builder.cc | 176 +- .../xla/python/local_computation_builder.h | 12 +- .../xla/python/local_computation_builder.i | 43 +- tensorflow/compiler/xla/python/xla_client.py | 138 +- .../compiler/xla/python_api/xla_shape.py | 7 +- tensorflow/compiler/xla/rpc/BUILD | 1 - tensorflow/compiler/xla/rpc/xla_service.proto | 1 - tensorflow/compiler/xla/service/BUILD | 148 +- .../xla/service/algebraic_simplifier.cc | 360 +- .../xla/service/algebraic_simplifier.h | 79 +- ..._simplifier_proof_distributive_property.py | 82 + .../xla/service/algebraic_simplifier_test.cc | 1176 +- .../compiler/xla/service/ar_crs_combiner.cc | 286 + .../compiler/xla/service/ar_crs_combiner.h | 88 + .../xla/service/ar_crs_combiner_test.cc | 415 + .../xla/service/batchnorm_expander.cc | 65 +- .../xla/service/batchnorm_expander_test.cc | 18 +- .../compiler/xla/service/buffer_assignment.cc | 56 +- .../compiler/xla/service/buffer_assignment.h | 10 +- .../xla/service/buffer_assignment_test.cc | 18 +- .../xla/service/buffer_liveness_test.cc | 28 +- tensorflow/compiler/xla/service/call_graph.cc | 9 + tensorflow/compiler/xla/service/call_graph.h | 4 + .../xla/service/compile_only_service.cc | 8 +- .../compiler/xla/service/computation_placer.h | 2 - .../convolution_feature_group_converter.cc | 253 +- ...onvolution_feature_group_converter_test.cc | 20 +- .../compiler/xla/service/copy_insertion.cc | 2 - .../xla/service/copy_insertion_test.cc | 44 +- tensorflow/compiler/xla/service/cpu/BUILD | 1 + .../xla/service/cpu/compiler_functor.cc | 16 +- .../compiler/xla/service/cpu/cpu_compiler.cc | 27 +- .../xla/service/cpu/cpu_executable.cc | 3 +- .../compiler/xla/service/cpu/cpu_executable.h | 2 +- .../xla/service/cpu/cpu_instruction_fusion.cc | 7 +- .../cpu/cpu_instruction_fusion_test.cc | 50 +- .../service/cpu/cpu_layout_assignment_test.cc | 22 +- .../compiler/xla/service/cpu/cpu_options.cc | 1 - .../compiler/xla/service/cpu/ir_emitter.cc | 52 +- .../compiler/xla/service/cpu/ir_emitter.h | 13 +- .../xla/service/cpu/runtime_key_value_sort.cc | 82 +- .../xla/service/cpu/simple_orc_jit.cc | 12 +- .../cpu/tests/cpu_eigen_dot_operation_test.cc | 2 +- .../cpu/tests/cpu_external_constants_test.cc | 2 +- .../service/cpu/tests/cpu_intrinsic_test.cc | 2 +- .../cpu/tests/cpu_literal_caching_test.cc | 8 +- .../xla/service/cpu/tests/cpu_noalias_test.cc | 2 +- .../compiler/xla/service/cpu/xfeed_manager.h | 1 + .../compiler/xla/service/dfs_hlo_visitor.h | 1 + .../service/dfs_hlo_visitor_with_default.h | 3 + .../xla/service/dynamic_parameter_binding.cc | 138 + .../xla/service/dynamic_parameter_binding.h | 125 + .../service/dynamic_parameter_binding_test.cc | 153 + .../xla/service/elemental_ir_emitter.cc | 71 +- tensorflow/compiler/xla/service/executable.h | 6 +- tensorflow/compiler/xla/service/gpu/BUILD | 1 + .../xla/service/gpu/cudnn_conv_rewriter.cc | 6 +- .../xla/service/gpu/cudnn_conv_runner.cc | 15 +- .../xla/service/gpu/elemental_ir_emitter.cc | 10 + .../compiler/xla/service/gpu/fusion_merger.cc | 2 +- .../xla/service/gpu/gpu_executable.cc | 2 +- .../compiler/xla/service/gpu/gpu_executable.h | 2 +- .../compiler/xla/service/gpu/gpu_fusible.cc | 71 +- .../compiler/xla/service/gpu/gpu_fusible.h | 21 +- .../xla/service/gpu/gpu_fusible_test.cc | 319 +- .../xla/service/gpu/gpu_hlo_schedule.cc | 18 +- .../xla/service/gpu/gpu_hlo_schedule.h | 4 +- .../xla/service/gpu/gpu_hlo_schedule_test.cc | 12 +- .../xla/service/gpu/gpu_layout_assignment.cc | 20 +- .../service/gpu/gpu_layout_assignment_test.cc | 8 +- .../xla/service/gpu/instruction_fusion.cc | 5 +- .../service/gpu/instruction_fusion_test.cc | 29 +- .../compiler/xla/service/gpu/ir_emitter.cc | 15 +- .../compiler/xla/service/gpu/ir_emitter.h | 1 + .../xla/service/gpu/ir_emitter_unnested.cc | 2513 +- .../xla/service/gpu/ir_emitter_unnested.h | 226 +- .../gpu/llvm_gpu_backend/nvptx_backend_lib.cc | 16 +- .../xla/service/gpu/multi_output_fusion.cc | 47 +- .../service/gpu/multi_output_fusion_test.cc | 2 +- .../xla/service/gpu/nvptx_compiler.cc | 16 +- .../xla/service/gpu/stream_assignment_test.cc | 8 +- .../xla/service/gpu/tests/gpu_codegen_test.h | 2 +- .../xla/service/gpu/tests/gpu_copy_test.cc | 2 +- .../xla/service/gpu/tests/gpu_ftz_test.cc | 12 +- .../xla/service/gpu/tests/gpu_index_test.cc | 2 +- .../xla/service/gpu/tests/gpu_ldg_test.cc | 6 +- .../xla/service/gpu/tests/gpu_noalias_test.cc | 2 +- .../xla/service/gpu/thunk_schedule.cc | 4 +- .../compiler/xla/service/gpu/thunk_schedule.h | 2 +- .../xla/service/gpu/while_transformer_test.cc | 2 +- .../xla/service/heap_simulator_test.cc | 6 +- tensorflow/compiler/xla/service/hlo.proto | 48 +- .../compiler/xla/service/hlo_computation.cc | 10 +- .../compiler/xla/service/hlo_computation.h | 8 +- .../xla/service/hlo_computation_test.cc | 34 +- .../xla/service/hlo_constant_folding_test.cc | 33 +- .../compiler/xla/service/hlo_cost_analysis.cc | 15 + .../compiler/xla/service/hlo_cost_analysis.h | 1 + .../xla/service/hlo_cost_analysis_test.cc | 6 +- .../xla/service/hlo_dataflow_analysis.cc | 18 + .../xla/service/hlo_dataflow_analysis.h | 1 + .../xla/service/hlo_dataflow_analysis_test.cc | 26 +- .../compiler/xla/service/hlo_dce_test.cc | 10 +- .../compiler/xla/service/hlo_evaluator.cc | 30 +- .../compiler/xla/service/hlo_evaluator.h | 17 +- .../xla/service/hlo_evaluator_test.cc | 29 + .../xla/service/hlo_evaluator_typed_visitor.h | 57 +- .../hlo_get_dimension_size_rewriter.cc | 61 + .../service/hlo_get_dimension_size_rewriter.h | 36 + .../hlo_get_dimension_size_rewriter_test.cc | 83 + .../compiler/xla/service/hlo_graph_dumper.cc | 141 +- .../compiler/xla/service/hlo_graph_dumper.h | 16 +- .../compiler/xla/service/hlo_instruction.cc | 233 +- .../compiler/xla/service/hlo_instruction.h | 34 +- .../compiler/xla/service/hlo_instructions.cc | 13 +- .../compiler/xla/service/hlo_instructions.h | 5 + tensorflow/compiler/xla/service/hlo_lexer.h | 1 + .../compiler/xla/service/hlo_matchers.cc | 4 - .../compiler/xla/service/hlo_matchers.h | 1 - .../xla/service/hlo_memory_scheduler.cc | 82 +- .../xla/service/hlo_memory_scheduler.h | 14 +- .../xla/service/hlo_memory_scheduler_test.cc | 41 +- tensorflow/compiler/xla/service/hlo_module.cc | 14 +- tensorflow/compiler/xla/service/hlo_module.h | 26 +- .../compiler/xla/service/hlo_module_test.cc | 16 +- tensorflow/compiler/xla/service/hlo_opcode.h | 3 +- .../compiler/xla/service/hlo_ordering.cc | 3 +- .../compiler/xla/service/hlo_ordering_test.cc | 12 +- tensorflow/compiler/xla/service/hlo_parser.cc | 15 +- .../compiler/xla/service/hlo_parser_test.cc | 69 +- .../compiler/xla/service/hlo_proto_util.cc | 9 +- .../compiler/xla/service/hlo_proto_util.h | 5 +- .../xla/service/hlo_rematerialization.cc | 19 +- .../xla/service/hlo_rematerialization.h | 5 +- tensorflow/compiler/xla/service/hlo_runner.cc | 34 + tensorflow/compiler/xla/service/hlo_runner.h | 20 +- .../compiler/xla/service/hlo_schedule.cc | 25 +- .../compiler/xla/service/hlo_schedule.h | 12 +- .../compiler/xla/service/hlo_schedule_test.cc | 14 +- .../xla/service/hlo_sharding_metadata.cc | 2 +- .../hlo_subcomputation_unification_test.cc | 6 +- tensorflow/compiler/xla/service/hlo_value.h | 3 - .../compiler/xla/service/hlo_verifier.cc | 21 +- .../compiler/xla/service/hlo_verifier.h | 1 + .../compiler/xla/service/hlo_verifier_test.cc | 16 +- .../service/indexed_array_analysis_test.cc | 8 +- .../xla/service/instruction_fusion.cc | 19 +- .../xla/service/instruction_fusion_test.cc | 56 +- .../xla/service/interpreter/executable.cc | 2 +- .../xla/service/interpreter/executable.h | 2 +- .../xla/service/interpreter/executor.cc | 11 +- .../xla/service/interpreter/executor.h | 3 +- .../compiler/xla/service/layout_assignment.cc | 1 + .../xla/service/layout_assignment_test.cc | 63 +- .../compiler/xla/service/llvm_ir/ir_array.h | 9 + .../xla/service/llvm_ir/kernel_tiling.cc | 189 +- .../xla/service/llvm_ir/kernel_tiling.h | 168 +- .../compiler/xla/service/llvm_ir/llvm_util.cc | 7 +- .../compiler/xla/service/llvm_ir/sort_util.cc | 85 +- .../compiler/xla/service/llvm_ir/sort_util.h | 5 +- .../compiler/xla/service/local_service.cc | 46 +- .../compiler/xla/service/local_service.h | 5 + .../xla/service/logical_buffer_analysis.cc | 7 + .../xla/service/logical_buffer_analysis.h | 1 + .../compiler/xla/service/pattern_matcher.h | 1352 +- .../xla/service/pattern_matcher_gmock.h | 92 + .../xla/service/pattern_matcher_gmock_test.cc | 76 + .../xla/service/pattern_matcher_test.cc | 528 +- .../reduce_precision_insertion_test.cc | 28 +- tensorflow/compiler/xla/service/service.cc | 69 +- .../compiler/xla/service/shape_inference.cc | 24 +- .../compiler/xla/service/shape_inference.h | 7 - .../xla/service/transpose_folding_test.cc | 10 +- .../xla/service/tuple_points_to_analysis.cc | 7 + .../xla/service/tuple_points_to_analysis.h | 1 + .../service/tuple_points_to_analysis_test.cc | 16 + .../while_loop_invariant_code_motion.cc | 32 + .../while_loop_invariant_code_motion.h | 11 +- .../while_loop_invariant_code_motion_test.cc | 54 + .../xla/service/while_loop_simplifier.cc | 447 +- .../xla/service/while_loop_simplifier_test.cc | 190 +- tensorflow/compiler/xla/shape.cc | 107 + tensorflow/compiler/xla/shape.h | 204 + tensorflow/compiler/xla/shape_test.cc | 149 + tensorflow/compiler/xla/shape_tree.h | 11 +- tensorflow/compiler/xla/shape_tree_test.cc | 4 +- tensorflow/compiler/xla/shape_util.cc | 43 +- tensorflow/compiler/xla/shape_util.h | 31 +- tensorflow/compiler/xla/shape_util_test.cc | 72 +- tensorflow/compiler/xla/tests/BUILD | 54 + .../xla/tests/array_elementwise_ops_test.cc | 50 +- .../xla/tests/broadcast_simple_test.cc | 17 +- .../xla/tests/client_library_test_base.cc | 4 +- .../xla/tests/client_library_test_base.h | 2 +- tensorflow/compiler/xla/tests/client_test.cc | 6 +- tensorflow/compiler/xla/tests/concat_test.cc | 26 + .../compiler/xla/tests/conv_depthwise_test.cc | 234 + .../compiler/xla/tests/convolution_test.cc | 770 +- .../compiler/xla/tests/dot_operation_test.cc | 70 + .../xla/tests/grouped_convolution_test.cc | 245 + .../compiler/xla/tests/hlo_test_base.cc | 10 +- tensorflow/compiler/xla/tests/hlo_test_base.h | 8 +- tensorflow/compiler/xla/tests/iota_test.cc | 21 + tensorflow/compiler/xla/tests/replay_test.cc | 9 +- tensorflow/compiler/xla/tests/reshape_test.cc | 6 +- tensorflow/compiler/xla/tests/scatter_test.cc | 36 + tensorflow/compiler/xla/tests/test_utils.cc | 155 +- .../compiler/xla/tests/test_utils_test.cc | 23 + .../compiler/xla/tests/token_hlo_test.cc | 111 +- .../xla/tests/xla_hlo_profile_test.cc | 6 +- .../compiler/xla/tools/replay_computation.cc | 21 +- tensorflow/compiler/xla/util.h | 20 + tensorflow/compiler/xla/window_util.cc | 11 + tensorflow/compiler/xla/window_util.h | 1 + tensorflow/compiler/xla/xla.proto | 20 +- tensorflow/compiler/xla/xla_data.proto | 38 +- tensorflow/compiler/xrt/BUILD | 6 + .../compiler/xrt/kernels/xrt_compile_ops.cc | 27 +- .../compiler/xrt/kernels/xrt_execute_op.cc | 33 +- .../compiler/xrt/kernels/xrt_state_ops.cc | 13 + .../compiler/xrt/kernels/xrt_state_ops.h | 50 + tensorflow/compiler/xrt/ops/xrt_state_ops.cc | 14 + tensorflow/compiler/xrt/tests/raw_api_test.cc | 203 +- tensorflow/compiler/xrt/xrt.proto | 11 +- tensorflow/compiler/xrt/xrt_state.cc | 14 + tensorflow/compiler/xrt/xrt_state.h | 3 + tensorflow/compiler/xrt/xrt_util.cc | 76 + tensorflow/compiler/xrt/xrt_util.h | 34 + tensorflow/contrib/all_reduce/BUILD | 27 +- .../contrib/all_reduce/python/all_reduce.py | 841 +- .../autograph/examples/benchmarks/BUILD | 36 + .../examples/benchmarks/benchmark_base.py | 62 + .../examples/benchmarks/cartpole_benchmark.py | 492 + .../contrib/batching/python/ops/batch_ops.py | 12 +- .../batching/python/ops/batch_ops_test.py | 1 + .../python/kernel_tests/monte_carlo_test.py | 2 +- .../bayesflow/python/ops/monte_carlo_impl.py | 4 +- tensorflow/contrib/bigtable/README.md | 23 +- .../test_kernels/bigtable_test_client.cc | 33 + .../test_kernels/bigtable_test_client.h | 19 + .../python/kernel_tests/bigtable_ops_test.py | 18 +- .../bigtable/python/ops/bigtable_api.py | 16 +- .../boosted_trees/estimator_batch/BUILD | 1 + .../estimator_batch/custom_export_strategy.py | 9 +- .../dnn_tree_combined_estimator.py | 2 +- .../estimator_batch/estimator.py | 240 +- .../estimator_batch/estimator_test.py | 316 +- .../contrib/boosted_trees/examples/boston.py | 4 +- .../boosted_trees/examples/boston_combined.py | 4 +- .../kernels/split_handler_ops.cc | 11 +- .../batch/categorical_split_handler.py | 2 +- .../batch/categorical_split_handler_test.py | 8 +- .../learner/batch/ordinal_split_handler.py | 12 +- .../batch/ordinal_split_handler_test.py | 13 +- .../python/training/functions/gbdt_batch.py | 11 +- .../boosted_trees/python/utils/losses.py | 44 +- .../contrib/checkpoint/python/containers.py | 4 + tensorflow/contrib/cluster_resolver/BUILD | 162 +- .../contrib/cluster_resolver/__init__.py | 16 +- .../cluster_resolver_initialization_test.py | 53 + .../python/training/__init__.py | 41 +- .../python/training/cluster_resolver.py | 338 +- .../python/training/gce_cluster_resolver.py | 199 +- .../training/kubernetes_cluster_resolver.py | 120 +- .../python/training/slurm_cluster_resolver.py | 185 +- .../training/tfconfig_cluster_resolver.py | 80 +- .../python/training/tpu_cluster_resolver.py | 343 +- tensorflow/contrib/cmake/CMakeLists.txt | 33 +- tensorflow/contrib/cmake/README.md | 162 +- .../contrib/cmake/TensorflowConfig.cmake.in | 16 + .../cmake/TensorflowConfigVersion.cmake.in | 11 + .../contrib/cmake/external/abseil_cpp.cmake | 22 +- tensorflow/contrib/cmake/external/png.cmake | 1 + .../contrib/cmake/modules/FindAbseilCpp.cmake | 6 +- tensorflow/contrib/cmake/tf_c.cmake | 1 + tensorflow/contrib/cmake/tf_core_cpu.cmake | 2 + .../contrib/cmake/tf_core_eager_runtime.cmake | 57 + .../contrib/cmake/tf_core_framework.cmake | 8 +- tensorflow/contrib/cmake/tf_core_ops.cmake | 9 +- tensorflow/contrib/cmake/tf_python.cmake | 41 +- tensorflow/contrib/cmake/tf_shared_lib.cmake | 86 +- tensorflow/contrib/compiler/BUILD | 1 + tensorflow/contrib/compiler/xla.py | 1 + .../constrained_minimization_problem.py | 4 +- tensorflow/contrib/crf/python/ops/crf.py | 10 +- tensorflow/contrib/cudnn_rnn/BUILD | 5 +- .../python/kernel_tests/cudnn_rnn_ops_test.py | 1637 +- .../python/kernel_tests/cudnn_rnn_test.py | 16 +- .../cudnn_rnn/python/layers/cudnn_rnn.py | 3 +- .../cudnn_rnn/python/ops/cudnn_rnn_ops.py | 2 +- .../kernel_tests/assert_element_shape_test.py | 18 +- .../kernel_tests/lmdb_dataset_op_test.py | 3 +- .../kernel_tests/slide_dataset_op_test.py | 34 +- tensorflow/contrib/data/python/ops/BUILD | 3 +- tensorflow/contrib/data/python/ops/readers.py | 14 +- tensorflow/contrib/data/python/ops/sliding.py | 25 +- tensorflow/contrib/distribute/BUILD | 2 +- tensorflow/contrib/distribute/README.md | 6 +- tensorflow/contrib/distribute/__init__.py | 4 +- tensorflow/contrib/distribute/python/BUILD | 250 +- .../python/checkpoint_utils_test.py | 4 +- .../python/collective_all_reduce_strategy.py | 124 +- .../collective_all_reduce_strategy_test.py | 152 +- .../contrib/distribute/python/combinations.py | 42 +- ...r_ops_test.py => cross_device_ops_test.py} | 173 +- ...ils_test.py => cross_device_utils_test.py} | 28 +- .../python/estimator_integration_test.py | 10 +- .../python/estimator_training_test.py | 198 +- .../distribute/python/examples/keras_mnist.py | 13 +- .../python/keras_optimizer_v2_test.py | 102 +- .../contrib/distribute/python/keras_test.py | 668 +- .../distribute/python/metrics_v1_test.py | 13 +- .../distribute/python/minimize_loss_test.py | 73 +- .../distribute/python/mirrored_strategy.py | 794 +- .../python/mirrored_strategy_multigpu_test.py | 1404 +- .../python/mirrored_strategy_test.py | 107 - .../distribute/python/moving_averages_test.py | 3 +- .../python/multi_worker_test_base.py | 102 +- .../distribute/python/one_device_strategy.py | 92 +- .../python/one_device_strategy_test.py | 18 +- .../python/parameter_server_strategy.py | 150 +- .../python/parameter_server_strategy_test.py | 159 +- .../contrib/distribute/python/step_fn.py | 2 +- .../distribute/python/strategy_test_lib.py | 122 +- .../contrib/distribute/python/tpu_strategy.py | 222 +- .../contrib/distribute/python/values_test.py | 380 +- .../python/warm_starting_util_test.py | 4 +- tensorflow/contrib/distributions/BUILD | 2 +- .../normal_conjugate_posteriors_test.py | 2 +- .../python/kernel_tests/wishart_test.py | 5 +- .../python/ops/bijectors/softmax_centered.py | 2 +- .../distributions/python/ops/sample_stats.py | 6 +- tensorflow/contrib/eager/python/datasets.py | 6 - .../contrib/eager/python/datasets_test.py | 13 - tensorflow/contrib/eager/python/evaluator.py | 5 +- .../eager/python/examples/densenet/BUILD | 1 + .../examples/densenet/densenet_graph_test.py | 3 +- .../python/examples/gan/mnist_graph_test.py | 3 +- .../examples/generative_examples/cvae.ipynb | 2 +- .../image_captioning_with_attention.ipynb | 2286 +- .../linear_regression_graph_test.py | 2 +- .../nmt_with_attention.ipynb | 8 +- .../examples/resnet50/resnet50_graph_test.py | 3 +- .../eager/python/examples/revnet/main.py | 13 +- .../examples/rnn_ptb/rnn_ptb_graph_test.py | 5 +- .../contrib/eager/python/metrics_impl.py | 6 +- .../contrib/eager/python/metrics_test.py | 12 - tensorflow/contrib/eager/python/network.py | 4 +- tensorflow/contrib/eager/python/saver.py | 5 + tensorflow/contrib/eager/python/tfe_test.py | 4 +- tensorflow/contrib/estimator/BUILD | 25 - tensorflow/contrib/estimator/__init__.py | 2 - .../python/estimator/dnn_linear_combined.py | 34 - .../factorization/python/ops/kmeans.py | 2 +- .../factorization/python/ops/kmeans_test.py | 2 +- tensorflow/contrib/feature_column/BUILD | 11 +- .../feature_column/sequence_feature_column.py | 16 +- ...equence_feature_column_integration_test.py | 17 +- .../sequence_feature_column_test.py | 62 +- .../sequence_feature_column_v2.py | 20 +- .../sequence_feature_column_v2_test.py | 59 +- tensorflow/contrib/framework/BUILD | 20 +- .../contrib/framework/python/ops/sort_ops.py | 172 +- .../estimator/python/gan_estimator_impl.py | 15 +- .../estimator/python/gan_estimator_test.py | 35 +- .../gan/python/losses/python/losses_impl.py | 15 +- tensorflow/contrib/gan/python/namedtuples.py | 12 +- tensorflow/contrib/gan/python/train.py | 37 +- tensorflow/contrib/gan/python/train_test.py | 84 +- tensorflow/contrib/gdr/gdr_rendezvous_mgr.cc | 8 + .../hadoop/python/kernel_tests/hadoop_test.py | 3 +- .../hadoop/python/ops/hadoop_dataset_ops.py | 11 +- tensorflow/contrib/ignite/README.md | 51 +- .../python/tests/ignite_dataset_test.py | 3 +- .../image/kernels/adjust_hsv_in_yiq_op.cc | 2 +- .../image/python/ops/dense_image_warp.py | 77 +- .../keras/api/keras/layers/__init__.py | 2 +- .../contrib/keras/api/keras/utils/__init__.py | 1 + .../python/kernel_estimators.py | 2 +- .../kinesis/python/ops/kinesis_dataset_ops.py | 11 +- tensorflow/contrib/layers/BUILD | 9 +- .../python/layers/embedding_ops_test.py | 52 +- .../contrib/layers/python/layers/encoders.py | 3 +- .../layers/python/layers/feature_column.py | 3 +- .../python/layers/feature_column_ops_test.py | 2 +- .../python/layers/feature_column_test.py | 2 +- .../contrib/layers/python/layers/layers.py | 5 +- .../layers/python/layers/layers_test.py | 4 +- .../layers/python/layers/regularizers_test.py | 2 +- tensorflow/contrib/learn/BUILD | 6 + .../learn/python/learn/estimators/dnn.py | 8 +- .../learn/estimators/dnn_linear_combined.py | 10 +- .../estimators/dnn_linear_combined_test.py | 2 +- .../learn/python/learn/estimators/dnn_test.py | 2 +- .../estimators/dynamic_rnn_estimator_test.py | 6 +- .../python/learn/estimators/estimator.py | 6 +- .../learn/python/learn/estimators/linear.py | 6 +- .../python/learn/estimators/linear_test.py | 4 +- .../learn/python/learn/learn_io/numpy_io.py | 2 +- .../learn/python/learn/learn_io/pandas_io.py | 2 +- .../python/sdca_estimator_test.py | 2 +- tensorflow/contrib/lookup/lookup_ops_test.py | 5 +- .../contrib/losses/python/losses/loss_ops.py | 61 +- .../contrib/makefile/download_dependencies.sh | 2 +- tensorflow/contrib/makefile/tf_op_files.txt | 2 + .../metrics/python/metrics/classification.py | 2 +- .../python/metrics/classification_test.py | 11 +- .../contrib/metrics/python/ops/metric_ops.py | 54 +- .../python/loss_scale_manager_test.py | 2 +- .../python/loss_scale_optimizer_test.py | 4 +- .../python/layers/core_layers.py | 9 +- .../opt/python/training/lars_optimizer.py | 12 + .../opt/python/training/nadam_optimizer.py | 9 +- .../python/training/nadam_optimizer_test.py | 37 +- tensorflow/contrib/optimizer_v2/BUILD | 3 +- .../contrib/optimizer_v2/optimizer_v2.py | 44 +- tensorflow/contrib/predictor/BUILD | 1 + tensorflow/contrib/quantize/README.md | 4 +- .../contrib/quantize/python/quant_ops.py | 8 +- .../contrib/quantize/python/quantize.py | 4 +- tensorflow/contrib/resampler/BUILD | 32 +- .../resampler/xla/resampler_ops_xla_test.py} | 53 +- .../python/kernel_tests/core_rnn_cell_test.py | 6 +- .../rnn/python/kernel_tests/core_rnn_test.py | 2 +- tensorflow/contrib/rnn/python/ops/gru_ops.py | 4 +- tensorflow/contrib/rnn/python/ops/lstm_ops.py | 5 +- tensorflow/contrib/rnn/python/ops/rnn_cell.py | 14 +- tensorflow/contrib/saved_model/BUILD | 5 +- .../python/saved_model/keras_saved_model.py | 35 +- .../saved_model/keras_saved_model_test.py | 39 +- .../kernel_tests/attention_wrapper_test.py | 4 +- .../contrib/summary/summary_ops_test.py | 30 +- .../tensorboard/db/summary_file_writer.cc | 11 +- .../db/summary_file_writer_test.cc | 18 + tensorflow/contrib/tensorrt/BUILD | 29 + .../contrib/tensorrt/convert/convert_graph.cc | 201 +- .../contrib/tensorrt/convert/convert_graph.h | 10 +- .../tensorrt/convert/convert_graph_test.cc | 43 +- .../contrib/tensorrt/convert/convert_nodes.cc | 1136 +- .../contrib/tensorrt/convert/convert_nodes.h | 81 +- .../tensorrt/convert/convert_nodes_test.cc | 1295 +- .../tensorrt/convert/trt_optimization_pass.cc | 10 + .../tensorrt/convert/trt_optimization_pass.h | 4 +- .../contrib/tensorrt/kernels/trt_engine_op.cc | 27 +- .../contrib/tensorrt/kernels/trt_engine_op.h | 4 + .../contrib/tensorrt/ops/trt_engine_op.cc | 22 +- .../contrib/tensorrt/python/trt_convert.py | 79 +- .../tensorrt/python/trt_convert_test.py | 13 +- .../tensorrt/resources/trt_resources.h | 3 +- .../contrib/tensorrt/segment/segment.cc | 35 +- tensorflow/contrib/tensorrt/test/base_test.py | 77 +- .../tensorrt/test/batch_matmul_test.py | 6 +- .../tensorrt/test/biasadd_matmul_test.py | 34 +- .../binary_tensor_weight_broadcast_test.py | 7 +- .../tensorrt/test/concatenation_test.py | 2 +- .../tensorrt/test/const_broadcast_test.py | 2 +- .../tensorrt/test/memory_alignment_test.py | 2 +- .../multi_connection_neighbor_engine_test.py | 10 +- .../tensorrt/test/neighboring_engine_test.py | 4 +- .../tensorrt/test/quantization_mnist_test.py | 290 + .../tensorrt/test/quantization_test.py | 144 + .../contrib/tensorrt/test/rank_two_test.py | 4 +- .../tensorrt/test/reshape_transpose_test.py | 8 +- .../contrib/tensorrt/test/testdata/checkpoint | 3 + .../model.ckpt-46900.data-00000-of-00001 | Bin 0 -> 686728 bytes .../test/testdata/model.ckpt-46900.index | Bin 0 -> 652 bytes .../test/tf_trt_integration_test_base.py | 112 +- .../contrib/tensorrt/test/unary_test.py | 4 +- .../tensorrt/test/vgg_block_nchw_test.py | 2 +- .../contrib/tensorrt/test/vgg_block_test.py | 2 +- .../timeseries/python/timeseries/BUILD | 2 + .../python/timeseries/estimators.py | 2 +- .../python/timeseries/estimators_test.py | 2 +- .../timeseries/python/timeseries/head_test.py | 2 +- .../python/timeseries/math_utils.py | 14 +- .../timeseries/python/timeseries/model.py | 2 +- .../timeseries/state_space_models/BUILD | 5 +- tensorflow/contrib/tpu/BUILD | 5 +- .../pip_package/cloud_tpu_profiler/main.py | 6 +- .../tpu/python/tpu/async_checkpoint.py | 7 +- tensorflow/contrib/tpu/python/tpu/datasets.py | 4 +- .../contrib/tpu/python/tpu/datasets_test.py | 10 +- .../contrib/tpu/python/tpu/keras_support.py | 54 +- .../tpu/python/tpu/keras_tpu_variables.py | 4 + tensorflow/contrib/tpu/python/tpu/tpu.py | 6 +- .../contrib/tpu/python/tpu/tpu_context.py | 2 +- .../contrib/tpu/python/tpu/tpu_embedding.py | 11 +- .../contrib/tpu/python/tpu/tpu_estimator.py | 17 +- .../python/tpu/tpu_estimator_signals_test.py | 2 +- tensorflow/contrib/tpu/python/tpu/tpu_feed.py | 69 +- .../contrib/tpu/python/tpu/training_loop.py | 4 +- tensorflow/contrib/tpu/tpu_estimator.md | 9 +- tensorflow/contrib/training/BUILD | 23 - tensorflow/contrib/training/__init__.py | 5 +- .../python/training/tensor_queue_dataset.py | 201 - .../training/tensor_queue_dataset_test.py | 355 - tensorflow/contrib/verbs/rdma.cc | 6 +- tensorflow/core/BUILD | 108 +- tensorflow/core/api_def/api_test.cc | 13 +- .../base_api/api_def_BatchDataset.pbtxt | 1 + .../base_api/api_def_CacheDataset.pbtxt | 1 + .../base_api/api_def_ConcatenateDataset.pbtxt | 1 + .../api_def_DatasetToSingleElement.pbtxt | 1 + .../api_def_EnqueueInQueueDataset.pbtxt | 3 - ...perimentalBytesProducedStatsDataset.pbtxt} | 3 +- ...i_def_ExperimentalDatasetCardinality.pbtxt | 21 + ...i_def_ExperimentalDatasetToTFRecord.pbtxt} | 2 +- ...perimentalDenseToSparseBatchDataset.pbtxt} | 3 +- ...xperimentalFunctionBufferingResource.pbtxt | 58 - ...ntalFunctionBufferingResourceGetNext.pbtxt | 25 - ...mentalFunctionBufferingResourceReset.pbtxt | 13 - ...f_ExperimentalGroupByReducerDataset.pbtxt} | 2 +- ...ef_ExperimentalGroupByWindowDataset.pbtxt} | 3 +- ...def_ExperimentalLatencyStatsDataset.pbtxt} | 3 +- ..._def_ExperimentalMapAndBatchDataset.pbtxt} | 2 +- ...def_ExperimentalMatchingFilesDataset.pbtxt | 4 + ...rimentalMaxIntraOpParallelismDataset.pbtxt | 13 + ...perimentalParallelInterleaveDataset.pbtxt} | 3 +- ...def_ExperimentalParseExampleDataset.pbtxt} | 3 +- ...ExperimentalPrivateThreadPoolDataset.pbtxt | 13 + ...> api_def_ExperimentalRandomDataset.pbtxt} | 3 +- ... => api_def_ExperimentalScanDataset.pbtxt} | 3 +- ...xperimentalSetStatsAggregatorDataset.pbtxt | 4 + ...ef_ExperimentalSlidingWindowDataset.pbtxt} | 3 +- ...t => api_def_ExperimentalSqlDataset.pbtxt} | 3 +- ...ef_ExperimentalStatsAggregatorHandle.pbtxt | 5 + ..._ExperimentalStatsAggregatorSummary.pbtxt} | 3 +- ... api_def_ExperimentalUnbatchDataset.pbtxt} | 3 +- .../core/api_def/base_api/api_def_FFT.pbtxt | 4 +- .../core/api_def/base_api/api_def_FFT2D.pbtxt | 4 +- .../base_api/api_def_FilterDataset.pbtxt | 1 + .../api_def_FixedLengthRecordDataset.pbtxt | 1 + .../api_def_FixedLengthRecordDatasetV2.pbtxt | 1 + .../base_api/api_def_FlatMapDataset.pbtxt | 1 + .../base_api/api_def_GeneratorDataset.pbtxt | 1 + .../core/api_def/base_api/api_def_IFFT.pbtxt | 4 +- .../api_def/base_api/api_def_IFFT2D.pbtxt | 4 +- .../base_api/api_def_InterleaveDataset.pbtxt | 1 + .../base_api/api_def_MapAndBatchDataset.pbtxt | 53 - .../api_def/base_api/api_def_MapDataset.pbtxt | 1 + .../api_def_MatchingFilesDataset.pbtxt | 4 - .../base_api/api_def_PaddedBatchDataset.pbtxt | 1 + .../base_api/api_def_ParallelMapDataset.pbtxt | 1 + .../base_api/api_def_PrefetchDataset.pbtxt | 1 + ...rependFromQueueAndPaddedBatchDataset.pbtxt | 3 - .../api_def_QuantizeAndDequantizeV2.pbtxt | 15 +- .../base_api/api_def_RangeDataset.pbtxt | 1 + .../base_api/api_def_RepeatDataset.pbtxt | 1 + ...api_def_ResourceApplyAdamWithAmsgrad.pbtxt | 85 + .../api_def_ResourceApplyKerasMomentum.pbtxt | 56 + ...def_ResourceSparseApplyKerasMomentum.pbtxt | 64 + .../api_def/base_api/api_def_ScatterNd.pbtxt | 4 + .../api_def_SetStatsAggregatorDataset.pbtxt | 3 - .../api_def_ShuffleAndRepeatDataset.pbtxt | 1 + .../base_api/api_def_ShuffleDataset.pbtxt | 1 + .../base_api/api_def_SkipDataset.pbtxt | 1 + .../api_def_SparseTensorSliceDataset.pbtxt | 1 + .../api_def_StatsAggregatorHandle.pbtxt | 4 - .../base_api/api_def_TFRecordDataset.pbtxt | 1 + .../base_api/api_def_TakeDataset.pbtxt | 1 + .../base_api/api_def_TensorDataset.pbtxt | 1 + ...i_def_TensorForestCreateTreeVariable.pbtxt | 17 + .../api_def_TensorForestTreeDeserialize.pbtxt | 17 + ..._def_TensorForestTreeIsInitializedOp.pbtxt | 17 + .../api_def_TensorForestTreePredict.pbtxt | 29 + ...def_TensorForestTreeResourceHandleOp.pbtxt | 5 + .../api_def_TensorForestTreeSerialize.pbtxt | 17 + .../api_def_TensorForestTreeSize.pbtxt | 17 + .../base_api/api_def_TensorListConcat.pbtxt | 12 + .../base_api/api_def_TensorListSplit.pbtxt | 13 + .../base_api/api_def_TensorScatterAdd.pbtxt | 94 + .../base_api/api_def_TensorScatterSub.pbtxt | 94 + .../api_def_TensorScatterUpdate.pbtxt | 106 + .../base_api/api_def_TensorSliceDataset.pbtxt | 1 + .../base_api/api_def_TextLineDataset.pbtxt | 1 + .../api_def_UnicodeDecodeWithOffsets.pbtxt | 87 + .../base_api/api_def_UnicodeEncode.pbtxt | 73 + .../api_def/base_api/api_def_ZipDataset.pbtxt | 1 + .../python_api/api_def_BatchDataset.pbtxt | 4 - .../python_api/api_def_BatchToSpaceND.pbtxt | 2 + .../python_api/api_def_BesselI0e.pbtxt | 4 +- .../python_api/api_def_BesselI1e.pbtxt | 4 +- .../api_def_BytesProducedStatsDataset.pbtxt | 4 - .../python_api/api_def_CacheDataset.pbtxt | 4 - .../python_api/api_def_CheckNumerics.pbtxt | 2 + .../api_def_ConcatenateDataset.pbtxt | 4 - .../api_def/python_api/api_def_Conv2D.pbtxt | 4 +- .../api_def_Conv2DBackpropFilter.pbtxt | 4 +- .../api_def_Conv2DBackpropInput.pbtxt | 4 +- .../api_def/python_api/api_def_Conv3D.pbtxt | 4 +- .../api_def_Conv3DBackpropFilterV2.pbtxt | 4 + .../python_api/api_def_CropAndResize.pbtxt | 4 +- .../api_def_DatasetToSingleElement.pbtxt | 4 - .../api_def_DecodeAndCropJpeg.pbtxt | 4 +- .../python_api/api_def_DecodeBmp.pbtxt | 4 +- .../python_api/api_def_DecodeGif.pbtxt | 4 +- .../python_api/api_def_DecodeJpeg.pbtxt | 4 +- .../python_api/api_def_DecodePng.pbtxt | 4 +- .../api_def_DenseToSparseBatchDataset.pbtxt | 4 - .../api_def_DepthwiseConv2dNative.pbtxt | 2 + ..._DepthwiseConv2dNativeBackpropFilter.pbtxt | 5 + ...f_DepthwiseConv2dNativeBackpropInput.pbtxt | 5 + .../python_api/api_def_Dilation2D.pbtxt | 1 + .../python_api/api_def_EncodeJpeg.pbtxt | 4 +- .../api_def_EnqueueInQueueDataset.pbtxt | 4 - .../core/api_def/python_api/api_def_Erf.pbtxt | 8 +- .../api_def_ExtractImagePatches.pbtxt | 8 +- .../python_api/api_def_ExtractJpegShape.pbtxt | 4 +- .../python_api/api_def_FilterDataset.pbtxt | 4 - .../api_def_FixedLengthRecordDataset.pbtxt | 4 - .../api_def_FixedLengthRecordDatasetV2.pbtxt | 4 - .../python_api/api_def_FlatMapDataset.pbtxt | 4 - .../api_def_FractionalAvgPool.pbtxt | 4 +- .../api_def_FractionalMaxPool.pbtxt | 4 +- .../python_api/api_def_GeneratorDataset.pbtxt | 4 - .../api_def_GroupByWindowDataset.pbtxt | 4 - .../api_def_InterleaveDataset.pbtxt | 4 - .../api_def/python_api/api_def_IsFinite.pbtxt | 4 + .../api_def/python_api/api_def_IsInf.pbtxt | 4 + .../api_def/python_api/api_def_IsNan.pbtxt | 4 + .../api_def_LatencyStatsDataset.pbtxt | 4 - .../api_def/python_api/api_def_LinSpace.pbtxt | 1 + .../core/api_def/python_api/api_def_Log.pbtxt | 1 + .../api_def/python_api/api_def_Log1p.pbtxt | 1 + .../api_def_MapAndBatchDataset.pbtxt | 4 - .../python_api/api_def_MapDataset.pbtxt | 4 - .../api_def_MaxPoolWithArgmax.pbtxt | 1 + .../core/api_def/python_api/api_def_Neg.pbtxt | 7 +- .../api_def_PaddedBatchDataset.pbtxt | 4 - .../api_def_ParallelInterleaveDataset.pbtxt | 4 - .../api_def_ParallelMapDataset.pbtxt | 4 - .../api_def_ParseExampleDataset.pbtxt | 4 - .../python_api/api_def_PrefetchDataset.pbtxt | 4 - ...rependFromQueueAndPaddedBatchDataset.pbtxt | 4 - .../python_api/api_def_QuantizedAvgPool.pbtxt | 2 + .../python_api/api_def_QuantizedConv2D.pbtxt | 2 + .../python_api/api_def_QuantizedMaxPool.pbtxt | 2 + .../python_api/api_def_QuantizedReluX.pbtxt | 2 + .../python_api/api_def_RandomDataset.pbtxt | 4 - .../python_api/api_def_RangeDataset.pbtxt | 4 - .../python_api/api_def_RepeatDataset.pbtxt | 4 - .../python_api/api_def_ResizeArea.pbtxt | 4 +- .../python_api/api_def_ResizeBicubic.pbtxt | 4 +- .../python_api/api_def_ResizeBilinear.pbtxt | 4 +- .../api_def_ResizeNearestNeighbor.pbtxt | 4 +- ...api_def_ResourceApplyAdamWithAmsgrad.pbtxt | 4 + .../api_def_ResourceApplyKerasMomentum.pbtxt | 4 + ...def_ResourceSparseApplyKerasMomentum.pbtxt | 4 + .../python_api/api_def_ScanDataset.pbtxt | 4 - .../api_def_SetStatsAggregatorDataset.pbtxt | 4 - .../api_def_ShuffleAndRepeatDataset.pbtxt | 4 - .../python_api/api_def_ShuffleDataset.pbtxt | 4 - .../api_def/python_api/api_def_Sign.pbtxt | 7 +- .../python_api/api_def_SkipDataset.pbtxt | 4 - .../python_api/api_def_SlideDataset.pbtxt | 4 - .../api_def_SparseTensorSliceDataset.pbtxt | 4 - .../python_api/api_def_SqlDataset.pbtxt | 4 - .../api_def/python_api/api_def_Sqrt.pbtxt | 7 +- .../api_def/python_api/api_def_Square.pbtxt | 7 +- .../api_def_StatsAggregatorHandle.pbtxt | 4 - .../api_def_StatsAggregatorSummary.pbtxt | 4 - .../api_def_StringToHashBucket.pbtxt | 8 +- .../python_api/api_def_StringToNumber.pbtxt | 8 +- .../python_api/api_def_TFRecordDataset.pbtxt | 4 - .../python_api/api_def_TakeDataset.pbtxt | 4 - .../api_def/python_api/api_def_Tanh.pbtxt | 10 +- .../python_api/api_def_TensorDataset.pbtxt | 4 - .../python_api/api_def_TensorListConcat.pbtxt | 4 + ... => api_def_TensorListPushBackBatch.pbtxt} | 0 .../python_api/api_def_TensorListSplit.pbtxt | 4 + .../api_def_TensorSliceDataset.pbtxt | 4 - .../python_api/api_def_TextLineDataset.pbtxt | 4 - .../python_api/api_def_UnbatchDataset.pbtxt | 4 - .../python_api/api_def_ZipDataset.pbtxt | 4 - .../common_runtime/accumulate_n_optimizer.cc | 5 +- .../collective_executor_mgr_test.cc | 6 +- .../collective_param_resolver_local.cc | 4 +- .../collective_param_resolver_local_test.cc | 6 +- .../common_runtime/collective_rma_local.cc | 2 +- .../collective_rma_local_test.cc | 6 +- .../core/common_runtime/constant_folding.cc | 26 +- .../common_runtime/constant_folding_test.cc | 46 + tensorflow/core/common_runtime/device.cc | 2 + tensorflow/core/common_runtime/device.h | 10 + .../core/common_runtime/device_factory.cc | 24 +- .../core/common_runtime/device_factory.h | 13 +- tensorflow/core/common_runtime/device_mgr.cc | 37 +- tensorflow/core/common_runtime/device_mgr.h | 15 +- .../device_resolver_local_test.cc | 6 +- .../core/common_runtime/device_set_test.cc | 2 +- .../core/common_runtime/direct_session.cc | 28 +- .../core/common_runtime/direct_session.h | 6 +- tensorflow/core/common_runtime/eager/BUILD | 1 + .../core/common_runtime/eager/attr_builder.cc | 32 +- .../core/common_runtime/eager/attr_builder.h | 16 +- .../common_runtime/eager/attr_builder_test.cc | 17 +- .../core/common_runtime/eager/context.cc | 14 +- .../core/common_runtime/eager/context.h | 2 + .../common_runtime/eager/eager_operation.h | 14 +- .../core/common_runtime/eager/execute.cc | 15 +- .../eager/kernel_and_device_test.cc | 14 +- tensorflow/core/common_runtime/executor.cc | 44 +- .../core/common_runtime/executor_test.cc | 8 +- .../core/common_runtime/function_test.cc | 11 +- .../function_threadpool_test.cc | 12 +- .../core/common_runtime/gpu/gpu_device.cc | 20 +- .../core/common_runtime/gpu/gpu_device.h | 16 +- .../common_runtime/gpu/gpu_device_factory.cc | 21 +- .../gpu/gpu_device_on_non_gpu_machine_test.cc | 2 +- .../common_runtime/gpu/gpu_device_test.cc | 32 +- .../common_runtime/gpu/gpu_process_state.cc | 27 +- .../common_runtime/gpu/gpu_process_state.h | 22 +- .../common_runtime/graph_execution_state.cc | 96 +- .../hierarchical_tree_broadcaster_test.cc | 11 +- .../kernel_benchmark_testlib.cc | 8 +- .../common_runtime/kernel_benchmark_testlib.h | 2 +- .../core/common_runtime/local_device.cc | 73 +- tensorflow/core/common_runtime/local_device.h | 7 + tensorflow/core/common_runtime/lower_if_op.cc | 10 +- tensorflow/core/common_runtime/metrics.cc | 16 +- tensorflow/core/common_runtime/metrics.h | 4 +- .../common_runtime/optimization_registry.cc | 14 + tensorflow/core/common_runtime/placer.cc | 2 +- tensorflow/core/common_runtime/placer_test.cc | 2 +- .../core/common_runtime/pool_allocator.cc | 14 +- .../core/common_runtime/pool_allocator.h | 1 - .../process_function_library_runtime_test.cc | 18 +- .../core/common_runtime/process_state.cc | 28 +- .../core/common_runtime/process_state.h | 8 +- .../core/common_runtime/renamed_device.cc | 14 +- .../core/common_runtime/renamed_device.h | 7 +- .../core/common_runtime/ring_reducer.cc | 2 +- .../core/common_runtime/ring_reducer_test.cc | 11 +- .../core/common_runtime/threadpool_device.cc | 2 +- .../threadpool_device_factory.cc | 36 +- tensorflow/core/distributed_runtime/BUILD | 13 +- ...lective_param_resolver_distributed_test.cc | 11 +- .../collective_rma_distributed_test.cc | 11 +- .../device_resolver_distributed_test.cc | 13 +- .../core/distributed_runtime/eager/BUILD | 1 + .../eager/eager_service_impl.cc | 35 +- .../eager/eager_service_impl_test.cc | 9 +- .../core/distributed_runtime/graph_mgr.cc | 8 +- tensorflow/core/distributed_runtime/rpc/BUILD | 1 + .../rpc/eager/grpc_eager_client.cc | 2 +- .../distributed_runtime/rpc/grpc_channel.cc | 44 +- .../distributed_runtime/rpc/grpc_channel.h | 8 +- .../rpc/grpc_channel_test.cc | 43 +- .../rpc/grpc_remote_worker.cc | 14 +- .../rpc/grpc_remote_worker.h | 3 +- .../rpc/grpc_rpc_factory.cc | 10 +- .../rpc/grpc_server_lib.cc | 11 +- .../distributed_runtime/rpc/grpc_session.cc | 20 +- .../distributed_runtime/rpc/grpc_session.h | 3 + .../rpc/grpc_session_test.cc | 27 + .../core/distributed_runtime/rpc/grpc_state.h | 36 +- .../rpc/grpc_worker_cache.cc | 16 +- .../rpc/rpc_rendezvous_mgr.cc | 9 + .../rpc_collective_executor_mgr_test.cc | 6 +- .../core/distributed_runtime/server_lib.cc | 10 +- .../distributed_runtime/server_lib_test.cc | 56 + .../core/distributed_runtime/session_mgr.cc | 8 +- .../distributed_runtime/session_mgr_test.cc | 8 +- tensorflow/core/distributed_runtime/worker.cc | 4 +- tensorflow/core/framework/allocator.cc | 2 + tensorflow/core/framework/allocator.h | 2 + tensorflow/core/framework/common_shape_fns.cc | 45 + tensorflow/core/framework/common_shape_fns.h | 3 + tensorflow/core/framework/dataset.cc | 3 + tensorflow/core/framework/dataset.h | 32 +- tensorflow/core/framework/device_base.cc | 6 +- tensorflow/core/framework/function.cc | 187 +- tensorflow/core/framework/function.h | 8 + .../core/framework/function_handle_cache.cc | 66 + .../core/framework/function_handle_cache.h | 53 + tensorflow/core/framework/function_test.cc | 84 + tensorflow/core/framework/node_def_util.cc | 4 + tensorflow/core/framework/node_def_util.h | 1 + tensorflow/core/framework/op_kernel.cc | 65 +- tensorflow/core/framework/resource_mgr.cc | 9 +- tensorflow/core/framework/tensor.h | 93 +- tensorflow/core/framework/tensor_test.cc | 11 +- tensorflow/core/graph/edgeset.cc | 2 +- tensorflow/core/graph/edgeset.h | 20 +- tensorflow/core/graph/graph.cc | 24 + tensorflow/core/graph/graph.h | 21 +- tensorflow/core/graph/graph_partition.cc | 4 +- tensorflow/core/graph/graph_partition_test.cc | 10 +- tensorflow/core/graph/graph_test.cc | 39 - tensorflow/core/graph/mkl_layout_pass.cc | 490 +- tensorflow/core/graph/mkl_layout_pass_test.cc | 311 + tensorflow/core/graph/node_builder.cc | 2 + tensorflow/core/graph/node_builder.h | 1 + tensorflow/core/graph/optimizer_cse_test.cc | 10 +- tensorflow/core/graph/testlib.cc | 11 + tensorflow/core/graph/testlib.h | 4 + tensorflow/core/grappler/BUILD | 4 + .../core/grappler/costs/graph_properties.cc | 5 + .../costs/op_level_cost_estimator_test.cc | 2 +- .../core/grappler/costs/virtual_scheduler.cc | 20 +- .../core/grappler/costs/virtual_scheduler.h | 8 +- .../core/grappler/graph_analyzer/sig_node.h | 2 +- tensorflow/core/grappler/graph_view.cc | 6 + tensorflow/core/grappler/graph_view.h | 5 +- tensorflow/core/grappler/grappler_item.cc | 64 +- tensorflow/core/grappler/grappler_item.h | 35 +- .../core/grappler/grappler_item_builder.cc | 11 +- .../core/grappler/grappler_item_test.cc | 28 + tensorflow/core/grappler/op_types.cc | 17 +- tensorflow/core/grappler/op_types.h | 6 +- tensorflow/core/grappler/optimizers/BUILD | 29 +- .../optimizers/arithmetic_optimizer.cc | 24 +- .../optimizers/arithmetic_optimizer_test.cc | 27 + .../optimizers/constant_folding_test.cc | 326 +- .../core/grappler/optimizers/data/BUILD | 1 + .../optimizers/data/graph_test_utils.cc | 2 +- .../optimizers/data/hoist_random_uniform.cc | 2 +- .../data/hoist_random_uniform_test.cc | 2 +- .../optimizers/data/latency_all_edges.cc | 2 +- .../optimizers/data/latency_all_edges_test.cc | 7 +- .../optimizers/data/make_numa_aware.cc | 2 +- .../optimizers/data/make_numa_aware_test.cc | 6 +- .../optimizers/data/map_and_batch_fusion.cc | 2 +- .../data/map_and_batch_fusion_test.cc | 21 +- .../experimental_implementation_selector.cc | 114 +- ...perimental_implementation_selector_test.cc | 95 + .../grappler/optimizers/function_api_info.cc | 180 +- .../grappler/optimizers/function_api_info.h | 38 +- .../optimizers/function_api_info_test.cc | 122 +- .../grappler/optimizers/function_optimizer.cc | 850 +- .../optimizers/function_optimizer_test.cc | 235 + .../optimizers/graph_optimizer_stage.h | 3 +- .../grappler/optimizers/graph_rewriter.cc | 214 - .../core/grappler/optimizers/graph_rewriter.h | 102 - .../grappler/optimizers/layout_optimizer.cc | 4 + .../grappler/optimizers/memory_optimizer.cc | 1 - .../grappler/optimizers/meta_optimizer.cc | 66 +- .../core/grappler/optimizers/meta_optimizer.h | 10 +- .../optimizers/meta_optimizer_test.cc | 65 +- .../core/grappler/optimizers/model_pruner.cc | 164 +- .../core/grappler/optimizers/remapper.cc | 445 +- .../core/grappler/optimizers/remapper_test.cc | 244 +- tensorflow/core/grappler/utils.cc | 4 +- tensorflow/core/grappler/utils/BUILD | 1 - tensorflow/core/grappler/utils/functions.cc | 124 +- tensorflow/core/grappler/utils/functions.h | 30 +- .../core/grappler/utils/functions_test.cc | 86 +- .../core/grappler/utils/grappler_test.cc | 6 +- tensorflow/core/grappler/utils_test.cc | 23 + tensorflow/core/kernels/BUILD | 90 +- tensorflow/core/kernels/adjust_contrast_op.cc | 4 +- tensorflow/core/kernels/adjust_hue_op.cc | 4 +- tensorflow/core/kernels/barrier_ops.cc | 2 +- .../kernels/boosted_trees/boosted_trees.proto | 14 + tensorflow/core/kernels/control_flow_ops.cc | 4 + tensorflow/core/kernels/conv_2d.h | 2 +- .../{conv_ops_gpu_3.cu.cc => conv_2d_gpu.h} | 118 +- .../core/kernels/conv_2d_gpu_double.cu.cc | 50 + .../core/kernels/conv_2d_gpu_float.cu.cc | 63 + .../core/kernels/conv_2d_gpu_half.cu.cc | 57 + .../kernels/conv_2d_gpu_int.cu.cc} | 28 +- .../core/kernels/conv_2d_gpu_uint16.cu.cc | 38 + .../core/kernels/conv_2d_gpu_uint32.cu.cc | 38 + .../core/kernels/conv_2d_gpu_uint64.cu.cc | 38 + .../core/kernels/conv_2d_gpu_uint8.cu.cc | 38 + tensorflow/core/kernels/conv_grad_ops_3d.cc | 2 + tensorflow/core/kernels/conv_ops_3d.cc | 14 +- tensorflow/core/kernels/conv_ops_fused.cc | 1185 +- .../kernels/conv_ops_fused_image_transform.cc | 902 + tensorflow/core/kernels/conv_ops_test.cc | 743 +- tensorflow/core/kernels/cwise_ops.h | 21 + tensorflow/core/kernels/data/BUILD | 215 +- .../core/kernels/data/batch_dataset_op.cc | 9 + .../core/kernels/data/cache_dataset_ops.cc | 4 + .../core/kernels/data/captured_function.cc | 210 +- .../core/kernels/data/captured_function.h | 147 +- .../kernels/data/concatenate_dataset_op.cc | 12 + tensorflow/core/kernels/data/dataset_ops.cc | 17 + tensorflow/core/kernels/data/dataset_utils.cc | 8 +- tensorflow/core/kernels/data/dataset_utils.h | 4 +- .../core/kernels/data/experimental/BUILD | 253 +- .../experimental/assert_next_dataset_op.cc | 2 + .../dense_to_sparse_batch_dataset_op.cc | 13 +- .../group_by_reducer_dataset_op.cc | 36 +- .../group_by_window_dataset_op.cc | 30 +- .../experimental/identity_indexed_dataset.cc | 163 - .../experimental/ignore_errors_dataset_op.cc | 2 + .../data/experimental/indexed_dataset.h | 119 - ...dexed_dataset.cc => indexed_dataset_op.cc} | 251 +- .../map_and_batch_dataset_op.cc | 90 +- .../matching_files_dataset_op.cc | 5 +- .../non_serializable_dataset_op.cc | 2 + .../numa_map_and_batch_dataset_op.cc | 17 +- .../parallel_interleave_dataset_op.cc | 1085 + .../parse_example_dataset_op.cc | 206 +- .../data/experimental/prefetching_kernels.cc | 428 - .../{ => experimental}/random_dataset_op.cc | 4 +- .../{ => experimental}/scan_dataset_op.cc | 15 +- .../set_stats_aggregator_dataset_op.cc} | 7 +- .../data/experimental/sleep_dataset_op.cc | 2 + .../sliding_window_dataset_op.cc} | 19 +- .../kernels/data/{ => experimental}/sql/BUILD | 0 .../{ => experimental}/sql/driver_manager.cc | 4 +- .../{ => experimental}/sql/driver_manager.h | 8 +- .../{ => experimental}/sql/query_connection.h | 6 +- .../sql/sqlite_query_connection.cc | 4 +- .../sql/sqlite_query_connection.h | 8 +- .../sql_dataset_op.cc} | 7 +- .../stats_aggregator_ops.cc | 10 +- .../{ => experimental}/stats_dataset_ops.cc | 14 +- .../experimental/threadpool_dataset_op.cc | 238 +- .../to_tf_record_op.cc} | 21 +- .../{ => experimental}/unbatch_dataset_op.cc | 4 +- .../core/kernels/data/filter_dataset_op.cc | 20 +- .../core/kernels/data/flat_map_dataset_op.cc | 7 +- .../core/kernels/data/generator_dataset_op.cc | 33 +- .../kernels/data/interleave_dataset_op.cc | 8 +- tensorflow/core/kernels/data/iterator_ops.cc | 311 +- .../core/kernels/data/map_dataset_op.cc | 29 +- .../core/kernels/data/model_dataset_op.cc | 2 + .../kernels/data/multi_device_iterator_ops.cc | 50 +- .../core/kernels/data/optimize_dataset_op.cc | 81 +- tensorflow/core/kernels/data/optional_ops.cc | 134 +- .../core/kernels/data/optional_ops.cu.cc | 37 + tensorflow/core/kernels/data/optional_ops.h | 119 + .../kernels/data/padded_batch_dataset_op.cc | 9 + .../data/parallel_interleave_dataset_op.cc | 1058 +- .../kernels/data/parallel_map_dataset_op.cc | 135 +- .../kernels/data/parallel_map_iterator.cc | 35 +- .../core/kernels/data/parallel_map_iterator.h | 41 +- .../core/kernels/data/prefetch_dataset_op.cc | 7 +- .../core/kernels/data/range_dataset_op.cc | 8 + .../core/kernels/data/repeat_dataset_op.cc | 17 + .../core/kernels/data/shuffle_dataset_op.cc | 2 + .../kernels/data/single_threaded_executor.cc | 15 +- .../data/single_threaded_executor_test.cc | 8 +- .../core/kernels/data/skip_dataset_op.cc | 8 + .../data/sparse_tensor_slice_dataset_op.cc | 2 + .../core/kernels/data/take_dataset_op.cc | 11 + .../core/kernels/data/tensor_dataset_op.cc | 2 + .../kernels/data/tensor_queue_dataset_op.cc | 657 - .../kernels/data/tensor_slice_dataset_op.cc | 2 + .../core/kernels/data/window_dataset.cc | 2 + .../core/kernels/data/window_dataset_op.cc | 9 + .../core/kernels/data/zip_dataset_op.cc | 15 + tensorflow/core/kernels/data_format_ops.cc | 10 + tensorflow/core/kernels/deep_conv2d.cc | 21 +- .../core/kernels/depthwise_conv_op_gpu.cu.cc | 8 +- .../core/kernels/dynamic_partition_op.cc | 2 +- .../core/kernels/eigen_spatial_convolutions.h | 194 +- .../eigen_spatial_convolutions_test.cc | 101 +- .../core/kernels/fractional_avg_pool_op.cc | 2 +- tensorflow/core/kernels/functional_ops.cc | 31 +- .../core/kernels/fused_batch_norm_op.cc | 4 +- tensorflow/core/kernels/fuzzing/BUILD | 22 + .../012e3ad384a4a1165f8498b5c94ba0d32a73e187 | Bin 0 -> 420 bytes .../055d77f7810048caa28323f6eb552a53d156040b | Bin 0 -> 594 bytes .../131e251bfb82c681cb075d32b99f18fceaca115d | Bin 0 -> 287 bytes .../1399ab0bd9f2c91d270cb43251bdc5729bef3526 | Bin 0 -> 1684 bytes .../16a6ce88f66d2e9686c8354cad8ba915cf0c11de | Bin 0 -> 207 bytes .../185097ed0588195164619ea930ddd8274a5f32ad | Bin 0 -> 3648 bytes .../27711a87e06a50c81571c27c3aa403a6ad5dc55c | Bin 0 -> 2218 bytes .../298c3787ad1722b22569cbc405c464d2 | Bin 0 -> 4234 bytes .../2b95ba6d8141ce0d29ff279770903922 | Bin 0 -> 4234 bytes .../321fb3d758b86e37fc340ae2b09b8ed9fa73a4cb | Bin 0 -> 4234 bytes .../331a98b4e4c87840efea69223766ebd0e1736542 | Bin 0 -> 420 bytes .../352d73f841223ecb630b5836585d2ba7b0f9d883 | Bin 0 -> 2249 bytes .../3a84f409d4c117edfdebc508cd23e8fc | Bin 0 -> 3194 bytes .../3ef5cc982c0b45f69a26fd0f7d376415fdebabd1 | Bin 0 -> 220 bytes .../401c7de8e122018a0e17f57c93db7ee49ab0e906 | Bin 0 -> 865 bytes .../52fee71bb8c9c79068e1fe580677ad739a2d0415 | Bin 0 -> 361 bytes .../57b11507813d5727b7789354d888eda83d5f3d86 | Bin 0 -> 2846 bytes .../57dff0fa53ee0ef24a43cca6ab0523bfdc1f720d | Bin 0 -> 864 bytes .../5c42d3df0dc400a7a4175b8d4eec6cc8ee2437b2 | Bin 0 -> 2246 bytes .../5cca20637ae75fddad9370ee930837baef8aeb43 | Bin 0 -> 4234 bytes .../5d34bc9cef0c844b9c5ebe948145c4ca11b5ca09 | Bin 0 -> 2639 bytes .../5e162fe883bd12fb1c4131d4e0c979a12bd15eac | Bin 0 -> 76 bytes .../5e83f8faab9c1a51a33d5e29edbb9dcec23c6092 | Bin 0 -> 1412 bytes .../61b29dc2fcef7b6fbe3e0cc88769a7ef | Bin 0 -> 3194 bytes .../6361eca190157ece389665ee523ccc3aefcd957f | Bin 0 -> 3708 bytes .../65150515ab3b11d657519b22bb887d74e94b2d7f | Bin 0 -> 4234 bytes .../656f38ef6dcd58c6a909d61db11f777def69c394 | Bin 0 -> 1406 bytes .../66e0d2cafd592bf9d61ad900fade8ee530d5f3d7 | Bin 0 -> 203 bytes .../6b5b42cb105a2c4c5fd6034e9885cbe457f1b50c | Bin 0 -> 361 bytes .../722ed0197cb92ecbf9745edb38275e7a9aaf322f | Bin 0 -> 454 bytes .../77bdd2efdf328366cbbf3c5688768dc0a88d02b1 | Bin 0 -> 16 bytes .../7841bfa002c05c61d5a5d9241f214cc17a336166 | Bin 0 -> 864 bytes .../7899e22fc83f6be28e9130c4a1c91a48 | Bin 0 -> 4234 bytes .../7dddccaebd16ae0c26daeffc42df50f529891119 | Bin 0 -> 2249 bytes .../8157442eee4bbfdd9716e264b11085d61a9955b7 | Bin 0 -> 48 bytes .../81ff28ed63d5435ddc4c8771dd5d40aa658cbbe0 | Bin 0 -> 2052 bytes .../820c8c0d33c18f6c4d9edd314e91289186931ad0 | Bin 0 -> 1318 bytes .../849e9d7cee1c52105242327086997296e452b981 | Bin 0 -> 1237 bytes .../84ddb92c63e0fad7018f6069daf8779ce11501e2 | Bin 0 -> 1167 bytes .../86bc3d5dbb9313137502080e58551edd2e649c70 | Bin 0 -> 929 bytes .../87d94d88fe29d277c76e1a52042b02c092d5ae14 | Bin 0 -> 420 bytes .../8c4646f3357945c4e19a59ff79fffe3c874dbf16 | Bin 0 -> 189 bytes .../90632bc6dee4eb836f3d7db1d16446a9c8510080 | Bin 0 -> 3895 bytes .../94d06016aa949e8e7203217e4cc6625ded7f4244 | Bin 0 -> 3102 bytes .../9875819b9e5783e7489c29a81cc9d4279209956a | Bin 0 -> 22 bytes .../9c1cc734114b29aac6c51782d5c17e9dbe1faca2 | Bin 0 -> 3194 bytes .../9d2961871eeb201ef8a6f5503d8a8b62 | Bin 0 -> 3194 bytes .../9f39e11cdd88344a4894b678e5a04a810880064d | Bin 0 -> 972 bytes .../a350588a6dabe4376a066aed44ef8786d8e752e7 | Bin 0 -> 861 bytes .../a6101a79919d444e1fc50aefab5837c39e3f4a19 | Bin 0 -> 1237 bytes .../a9c8793f8fb063bec839ee1280406fe5396545e5 | Bin 0 -> 420 bytes .../ad4e9d2234e8599bdf12607c6b8cab4edae82c4e | Bin 0 -> 420 bytes .../b90b6830917919e94186d312f06481bd | Bin 0 -> 3194 bytes .../b98fd4cb1d7031240414301c19b03097c0035c6b | Bin 0 -> 857 bytes .../ba976fcdb4daf092ef17ce43bf2b78d9d8bc2aeb | Bin 0 -> 2262 bytes .../bc112b571eafee0f5a031f3c9cce6244216d128d | Bin 0 -> 4234 bytes .../c42b981c28a1715c375050f6fcf53f1d | Bin 0 -> 3194 bytes .../c6049874b33eadb016fccf0c5fa66e556ae069b9 | Bin 0 -> 3194 bytes .../c8697bf2369f6ab85f501376c4d93bb8a56974a3 | Bin 0 -> 420 bytes .../c8daf283e0aef2fd7b630c0430e05dc28f24ecf6 | Bin 0 -> 220 bytes .../cacff56e1af4b8fde912822da06b10fb8c545a19 | Bin 0 -> 1033 bytes .../ce4dcc22b1d595c49a25121c0b580104 | Bin 0 -> 4234 bytes .../d0cd71dbf039fd64cf42eff30da92a71a919226a | Bin 0 -> 587 bytes .../d5ce626ac3264bed6af5580e341a89406857cbb9 | Bin 0 -> 1760 bytes .../d77ada02e9bc8c24b2711eca6a8f52ae356bfc21 | Bin 0 -> 361 bytes .../d7eb9c5a0f9803df4c00390793b8ab57bd7c9484 | Bin 0 -> 3792 bytes .../dc1efccdeec17e151a1ec8228c09ab61c3040b33 | Bin 0 -> 35 bytes .../dcea22c66c60088165a2f1772036473f | Bin 0 -> 4234 bytes .../de539ae7442fa05dafcfe1a021f0186ef74a2b0e | Bin 0 -> 103 bytes .../e2306b1d6b88d0ccc4e2c3a9edb07462a5a32215 | Bin 0 -> 110 bytes .../e2778da0240fdd15ef5844905d81c4e05f34a8bd | Bin 0 -> 3194 bytes .../e6642e9266875f9d908942e534bf898103a2c794 | Bin 0 -> 2186 bytes .../ec6cdb929c08d8daf2bd7fc185fbf4d787b45120 | Bin 0 -> 1942 bytes .../ed8636357f79439b6a03eb14469b686cc401a1c9 | Bin 0 -> 3895 bytes .../ee313e9acecb5c688ce8c9bb10e70e136fbb9c6d | Bin 0 -> 3616 bytes .../ef689af320e7d9e22231109faae2e8149cb86e1c | Bin 0 -> 834 bytes .../fda6b9a9f6ffdf4765c00465619c7ceb3f7db2e4 | Bin 0 -> 1136 bytes .../ffe829bb0adac20d9c0756f68a22d1255e4fdb54 | Bin 0 -> 3194 bytes .../013a29ea098a178f8a36741c9fd91144 | 48 + .../0875575fb76d630ccb19c5da8aab66b2 | 1 + .../7e7f58fc443a11a0a2c5d9b643b7e99b | 1 + .../849a23936269a261c0370b5e9abe2416 | 1 + .../85282c1696d98b9843ce3e8bd1cd899f | 1 + .../90388b9c8093d8adedad0644b618da87 | 33 + .../9fa2f86ea6d3ade36e961247c3026f8d | 33 + .../c4f18ca60a84e9869a28faf6f65dc758 | 32 + .../d456ee029700adef5d28438593010223 | 1 + .../e9f0ff6ee8d691ae69d2ecb4710030a2 | 36 + .../010dc3d4b05288fcc40de2721052b3dc699f1cb3 | Bin 0 -> 233 bytes .../0555cd5e9d99629819cc985285f80da0f00be1e9 | Bin 0 -> 474 bytes .../0a0352aa168803ff65455792d9f6ee555c3e7c3f | Bin 0 -> 232 bytes .../0ed54162df93ef8d00f993ce6b59ba422903d381 | Bin 0 -> 221 bytes .../1547b448171c700613c3946d730de496c9b9863f | Bin 0 -> 260 bytes .../17859046cbe4ac598a645173d679ce2a52c6afba | Bin 0 -> 525 bytes .../1df76c07817fbc3653a26f34d97658e9973627c2 | Bin 0 -> 666 bytes .../1f0717f8856d7782e3ab7992d3a72d783a018443 | Bin 0 -> 122 bytes .../23b911e4ce936def88bc9a46b8b433c0e83fba2a | Bin 0 -> 375 bytes .../25592201c3edff0578dbdac6b0e4f2be109ce151 | Bin 0 -> 329 bytes .../266fd8495e0b8eb64387c1a62264185e061fee73 | Bin 0 -> 360 bytes .../27f178cf415b4ff8671131ddf1d042dafac2fb3e | Bin 0 -> 677 bytes .../2a0bdc4d9cc5ea5bb21dd256d6ac96075376a94f | Bin 0 -> 246 bytes .../2e5d25add6adc68e0457b358c7a34abf3d41c938 | Bin 0 -> 165 bytes .../2e6c5b6a766dd5e9bd41eacfd0a36572bd2f7544 | Bin 0 -> 320 bytes .../2e9c935cf82f6ca640e9a9abc3c30a578ad46176 | Bin 0 -> 666 bytes .../2fcf1ed4477f7eaee028f5b3f9edeb5f1a737826 | Bin 0 -> 507 bytes .../3480713774f590908ca5dba16d121cdfb8fba62b | Bin 0 -> 665 bytes .../39289afcec60d98802b333e0fbb1da4d7aed4ce5 | Bin 0 -> 247 bytes .../3adc488e21d4aca7bed9422f0241a42d0f93e7d9 | Bin 0 -> 366 bytes .../3cbf274da522483dc991fad9df43a22ac4fb3173 | Bin 0 -> 474 bytes .../3d840cdff7f5ad16fe8bcb985ed4946c03459432 | Bin 0 -> 666 bytes .../3f1e6753c1fca958e859189857449746592158ea | Bin 0 -> 94 bytes .../3fa4075993cb0f9bfa8eea785174a2038a69aa1b | Bin 0 -> 343 bytes .../4023a373e977be58413e55350380310c5dd1fd6a | Bin 0 -> 241 bytes .../40caba69dce1cfc48e0e43184d2bfbc6daa4399a | Bin 0 -> 470 bytes .../41841e9561d8135945c1c1e55ab9e9a1e933653b | Bin 0 -> 247 bytes .../41d40f2d66fa43e34537385594ee9911e65deadf | Bin 0 -> 677 bytes .../421bd39810b50309a71adb2dadc3b19f01a52312 | Bin 0 -> 221 bytes .../446c305b2c0665736f94fb2b62dbdef445eff0cf | Bin 0 -> 174 bytes .../449cee952bb645f6f4241a6665d3c6028c073c7a | Bin 0 -> 374 bytes .../45520b07609978c5aa3516d803527438b93fbadb | Bin 0 -> 158 bytes .../4da74a34bcede234b0415f77fbd87d70bf9a777e | Bin 0 -> 94 bytes .../51db5d31d2c5300d34831d9f23bcdd0aff9a998b | Bin 0 -> 677 bytes .../5cde2a9167798cb77f10abbfb2640a5c357f99fc | Bin 0 -> 507 bytes .../5e352fc10ac476cfbe1d755f092e069820223249 | Bin 0 -> 169 bytes .../63661677dd1306cec4b5a565190e65adf2446e52 | Bin 0 -> 374 bytes .../65887ed3db382aab1d9485c500f4401318d303b9 | Bin 0 -> 224 bytes .../67b5181f8f0644597e9bde539e8f083b5cacd0e7 | Bin 0 -> 165 bytes .../74c9dcf7afee2a6cb1ab3a2c0de744d1b03c1466 | Bin 0 -> 307 bytes .../792181ca19e6ded261434e588bb7fc2a4816d4ce | Bin 0 -> 544 bytes .../79f0e2a475487f8fa69e68c1cc947c5851bda741 | Bin 0 -> 355 bytes .../7e5fcdfeb557ce379ed96925c68505eaac0112db | Bin 0 -> 315 bytes .../7eec7530acf34b3a96fa9189783453999f7b6838 | Bin 0 -> 671 bytes .../80114bf9781bffc9db411413d83541d8deaaf7c1 | Bin 0 -> 343 bytes .../80425fb92bb86627e854892f23823fa804e5fdc3 | Bin 0 -> 441 bytes .../821cdd6eeb919a8dd7f35289abbd583828dd4945 | Bin 0 -> 677 bytes .../83e1a31785285338b0ddb3334b0ed098e63dedde | Bin 0 -> 285 bytes .../8a4c8100dedd0fb5f2a8b468c678f7ad8269deeb | Bin 0 -> 666 bytes .../8ae8268c24dc866c1edb3826b93a1c75dbf74ff4 | Bin 0 -> 677 bytes .../90f72038cc627f34f074ea72eadbba87a5e3e288 | Bin 0 -> 677 bytes .../92b67faee4a49df2cdbed785e27b4a1cddcfffa3 | Bin 0 -> 507 bytes .../9463810467aacdc9923b2b20a2236116b760d75b | Bin 0 -> 258 bytes .../94d7c96aea32ad41ce643d35b951a6d8990b81d6 | Bin 0 -> 546 bytes .../98cc7e9fe87df914d89a0aef008930f27b3c26f5 | Bin 0 -> 311 bytes .../99172dfdb4f59aaced29c7681ac6e6ce8356e814 | Bin 0 -> 104 bytes .../9ae3b647d895af97fe872c0b1442df7b5b767160 | Bin 0 -> 332 bytes .../9d2b1d2121b0508a4fa8d1508adb9d05633fdac3 | Bin 0 -> 222 bytes .../a335af37917ccf0c8b11bb884a3a74f3f1d2a7c6 | Bin 0 -> 677 bytes .../a738609112d3a6772c50a71e2c3504ebc515b709 | Bin 0 -> 337 bytes .../a8cecab5d917da5a4729632a7a18c564d7e1607d | Bin 0 -> 258 bytes .../ade919ab2b4a458e806575c941dfe50ae3fd3621 | Bin 0 -> 677 bytes .../b1251621a5eb5e7fda9cac9baead1c993a285c36 | Bin 0 -> 277 bytes .../b1516b78c3dfe77eeb554985fd7344c0478fbbcb | Bin 0 -> 258 bytes .../b41241740f5f8ad2c1d408f7bb6a313bd863c158 | Bin 0 -> 133 bytes .../b799c8596523a7ebeb8e11ada08818c10f7eabfc | Bin 0 -> 582 bytes .../ba48d0521a111222dc95a3a997c7c92dea5f4443 | Bin 0 -> 249 bytes .../c01457c6889fb1b597d308363a36412c0b7f90e7 | Bin 0 -> 497 bytes .../c82ebc0d6688d104af04fd20d6d3da591dc391f7 | Bin 0 -> 198 bytes .../c9a03eb758dd84e954e3d70916e2311e8fd21f3c | Bin 0 -> 249 bytes .../cf892756b33578a54ab20044514e573328d2f1d7 | Bin 0 -> 249 bytes .../d3bc3f158a63f1d50b474addd3f7b3d17f23e8e9 | Bin 0 -> 507 bytes .../d4906950aa9d60ad09dc0f5413c3d88080c3bc37 | Bin 0 -> 262 bytes .../da31578a8068bad65e1c7a3d06e8f543a2a0bc65 | Bin 0 -> 332 bytes .../dd4a9b5d0740679c249fc884efc499433b29436b | Bin 0 -> 666 bytes .../deea4ecc6f0b2a6d89fd25ff76762299f21602fb | Bin 0 -> 677 bytes .../e1040c7ffcb39915e0f539018c81f9798924cba6 | Bin 0 -> 94 bytes .../e381dc85682cc33ad99f622b89d145b47f7d6392 | Bin 0 -> 485 bytes .../ea24498fc7a144fccc6f1665ebf7020df803dd1a | Bin 0 -> 677 bytes .../eaa5d677e797c07bac98c3c7051abad91852e7c6 | Bin 0 -> 507 bytes .../ed7871269315725535d8bffec7836c45a3fc5c26 | Bin 0 -> 527 bytes .../ee8460f4077064c5a2137075b48eba7d3db5c570 | Bin 0 -> 94 bytes .../ef09f26e0ee61329f84a9f589629a865ae9ee0a6 | Bin 0 -> 276 bytes .../f477da4d7d8ff2066041e1dd5ee4e833b7111a1a | Bin 0 -> 527 bytes .../f8a379b2498a4eb452a85791a49adf065dab59ae | Bin 0 -> 666 bytes .../fe67bccb06f2174523943cc684518fcf1f7f8046 | Bin 0 -> 80 bytes .../ff1e67d17c1c27ef0d97900d0ea276b563a64628 | Bin 0 -> 271 bytes .../02cc44cdfec1d9d0d0c66c5a5f40d3d20e4c4c3a | Bin 0 -> 11564 bytes .../087e1d7fae1c1ddcbaa3b5f822a171ad15498186 | Bin 0 -> 146 bytes .../0f61c33027394a0f14d29dcd22f405cad943b7cf | Bin 0 -> 29936 bytes .../10cdebea1659c21a0248f88654ae41f62786abf1 | Bin 0 -> 58657 bytes .../126e68def9fd973a100e0f66cadf09448a716b57 | Bin 0 -> 9209 bytes .../1275d41ebf8788ce3a949352e4bc654b04012da3 | Bin 0 -> 70 bytes .../1a7f1c407fb3864ddb559f88f373a21d1be51584 | Bin 0 -> 35640 bytes .../1c3e1c91f187f6bcea86f172ff5bbbd955a9654d | Bin 0 -> 57 bytes .../300fe1e0a47543037cbf0243b6756c9aa48799c4 | Bin 0 -> 72238 bytes .../31ec5b0134bedcfe283f4978e6e65b7d35d5d4ad | Bin 0 -> 26123 bytes .../4e7cbb27667bcfca92838aa8020749990013a9b1 | Bin 0 -> 66423 bytes .../585e469231d202812bfba8285fb30c8e31c857b9 | Bin 0 -> 70833 bytes .../58eab6bc2386e2ef43fe4f55cb6ad3611399d5de | Bin 0 -> 22 bytes .../63448c6a9feb8c72b3e82af4d735ec2e62ddd328 | Bin 0 -> 58203 bytes .../6874d5b1c7a64b596c61f24877d422e89bebe58b | Bin 0 -> 25086 bytes .../7501f79cb067da108020579ed654349c7933d22f | Bin 0 -> 54715 bytes .../782051f8120182b860c7fe1b265179cfa2fe03fd | Bin 0 -> 66439 bytes .../793feab2deb35e284a975f6527d76a8be5540fe6 | Bin 0 -> 61 bytes .../7f41ec3a9805c6b8f3656c4f9f6d0ff7dbf8a329 | Bin 0 -> 6150 bytes .../8210dc595a2652f2f812093b01e239e7918ea065 | Bin 0 -> 55329 bytes .../8dffe4c5c26d891b578fd2ea4b9adfc0c96ad5f7 | Bin 0 -> 71409 bytes .../91d787a9298ddc015efa783a92c4bdba8af0d7de | Bin 0 -> 44 bytes .../92c065286f956f086e977556358f6b54b12bcacc | Bin 0 -> 315 bytes .../a35c9bb71792b60a13dea23a41b41847ad4b93d6 | Bin 0 -> 44 bytes .../a6ea960c7b4d42772888280277b26e645ceee904 | Bin 0 -> 22470 bytes .../aa526aa853333f0bb11804b5243df411452cecd2 | Bin 0 -> 21331 bytes .../ca533cd26c7ca6bf69e62351b265ded496fdf1d9 | Bin 0 -> 52741 bytes .../f38c61da15f2cb7a39ff02e69f0b00e99f37ec86 | Bin 0 -> 66555 bytes .../f88f1012473e6cfcc9b39b2552f682b2f73eff8c | Bin 0 -> 54757 bytes .../fa79819c5de04bc06c69bec3fa7f2e982826ea2f | Bin 0 -> 20280 bytes .../fce08de222896ac3a20657a3b4f42d5b6c54a96a | Bin 0 -> 69522 bytes .../4c01a1504da9de2216894743ecc44424 | 1 + .../5bf16424630b5afbcffe711fb9834440 | 1 + .../a7185605aef0a8fd682fcb4656e4a736 | 1 + .../d5606def44fdbb9385dd764612069db0 | Bin 0 -> 42 bytes .../dbac766f3160de65894bf5153f478146 | 1 + .../e85ff62f6d457666f54a37a19a115a24 | 1 + .../00fd47bf73afcb72e7ed51bffd5f5fec | 1 + .../14908973e6720513a5f37676cb9fcc29 | 1 + .../2779ba7c4d23eee9f79efa3660084c5d | 1 + .../5bf16424630b5afbcffe711fb9834440 | 1 + .../89734a96b93275e495a9498b806fafe1 | 1 + .../d5606def44fdbb9385dd764612069db0 | Bin 0 -> 42 bytes .../2db83ea58639b6d7d585fa12e3947a82 | 1 + .../36b4a931886b941dc41180050d12ca94 | 1 + .../50a2fabfdd276f573ff97ace8b11c5f4 | 1 + .../62edb2a1eee34b001652cd86584becf2 | 1 + .../90013d1ec28c46a5c00574e60c70b6fc | 1 + .../94f3e3cee6957ce5815326d6788c85f4 | 1 + .../96f547bc04bb913da0bc08915238ebd8 | 1 + .../d3a903d18fc11e1f35c572ad4da690ed | 1 + .../e3b629c92af44260c189deb32d6f06f3 | 1 + .../f03eecf3bcfe4967a1888156a3115c8d | 1 + .../fa54ca9186f77122ae2a82684a062e16 | 1 + .../dictionaries/decode_json_example.dict | 6 + .../fuzzing/dictionaries/decode_png.dict | 50 + .../fuzzing/dictionaries/decode_wav.dict | 4 + .../kernels/fuzzing/encode_base64_fuzz.cc | 2 +- .../core/kernels/fuzzing/fuzz_session.h | 4 +- .../core/kernels/fuzzing/string_split_fuzz.cc | 3 +- .../fuzzing/tf_ops_fuzz_target_lib.bzl | 32 +- tensorflow/core/kernels/list_kernels.cc | 44 +- tensorflow/core/kernels/list_kernels.cu.cc | 15 +- tensorflow/core/kernels/list_kernels.h | 384 +- .../core/kernels/maxpooling_op_gpu.cu.cc | 12 + tensorflow/core/kernels/mkl_avgpooling_op.cc | 129 +- tensorflow/core/kernels/mkl_conv_ops.cc | 64 +- tensorflow/core/kernels/mkl_lrn_op.cc | 722 +- tensorflow/core/kernels/mkl_maxpooling_op.cc | 161 +- .../core/kernels/mkl_pooling_ops_common.cc | 30 +- .../core/kernels/mkl_pooling_ops_common.h | 29 +- .../kernels/mkl_quantized_pooling_ops_test.cc | 201 + tensorflow/core/kernels/mkl_softmax_op.cc | 4 +- .../core/kernels/partitioned_function_ops.cc | 43 +- .../kernels/quantize_and_dequantize_op.cc | 29 +- .../core/kernels/quantize_and_dequantize_op.h | 81 +- .../quantize_and_dequantize_op_gpu.cu.cc | 4 +- .../quantize_and_dequantize_op_test.cc | 136 +- .../quantized_resize_bilinear_op_test.cc | 2 +- tensorflow/core/kernels/ragged_gather_op.cc | 6 +- tensorflow/core/kernels/scan_ops_gpu.cu.cc | 266 +- tensorflow/core/kernels/scan_ops_test.cc | 146 + tensorflow/core/kernels/scatter_nd_op.cc | 164 + tensorflow/core/kernels/stage_op.cc | 2 +- tensorflow/core/kernels/tensor_array_ops.cc | 6 +- tensorflow/core/kernels/tensor_forest/BUILD | 53 + .../kernels/tensor_forest/prediction_ops.cc | 93 + .../kernels/tensor_forest/resource_ops.cc | 135 + .../core/kernels/tensor_forest/resources.cc | 59 + .../core/kernels/tensor_forest/resources.h | 63 + tensorflow/core/kernels/training_ops.cc | 390 + tensorflow/core/kernels/training_ops.h | 23 + .../core/kernels/training_ops_gpu.cu.cc | 62 + tensorflow/core/kernels/training_ops_test.cc | 78 + tensorflow/core/kernels/unicode_ops.cc | 355 +- tensorflow/core/lib/core/threadpool.cc | 21 +- tensorflow/core/lib/core/threadpool.h | 6 +- tensorflow/core/lib/png/png_io.cc | 6 +- tensorflow/core/nccl/BUILD | 38 +- tensorflow/core/nccl/nccl_manager.cc | 66 +- tensorflow/core/nccl/nccl_manager.h | 8 +- tensorflow/core/nccl/nccl_manager_test.cc | 49 +- tensorflow/core/ops/array_ops.cc | 61 +- .../core/ops/compat/ops_history.v1.pbtxt | 2596 +- tensorflow/core/ops/dataset_ops.cc | 300 - .../core/ops/experimental_dataset_ops.cc | 292 +- tensorflow/core/ops/functional_ops.cc | 4 +- tensorflow/core/ops/list_ops.cc | 249 +- tensorflow/core/ops/nn_ops.cc | 11 +- tensorflow/core/ops/ops.pbtxt | 2137 +- tensorflow/core/ops/sparse_ops.cc | 4 +- tensorflow/core/ops/sparse_ops_test.cc | 7 + tensorflow/core/ops/string_ops.cc | 60 + tensorflow/core/ops/tensor_forest_ops.cc | 79 + tensorflow/core/ops/training_ops.cc | 66 + tensorflow/core/platform/cpu_feature_guard.cc | 2 +- .../core/platform/default/build_config.bzl | 3 + .../core/platform/default/device_tracer.cc | 55 +- tensorflow/core/platform/default/logger.cc | 34 + tensorflow/core/platform/default/logging.cc | 165 +- tensorflow/core/platform/default/logging.h | 44 +- tensorflow/core/platform/env.h | 2 + tensorflow/core/platform/logger.h | 51 + tensorflow/core/platform/numa_test.cc | 2 +- tensorflow/core/platform/platform_strings.cc | 64 + tensorflow/core/platform/platform_strings.h | 364 + .../core/platform/platform_strings_computed.h | 735 + .../core/platform/platform_strings_test.cc | 146 + .../core/platform/posix/posix_file_system.cc | 11 +- tensorflow/core/platform/regexp.h | 3 +- .../platform/windows/windows_file_system.cc | 3 + .../core/profiler/internal/tfprof_code.cc | 2 +- .../core/profiler/internal/tfprof_node.cc | 2 +- tensorflow/core/protobuf/config.proto | 12 + tensorflow/core/protobuf/master.proto | 2 +- .../core/protobuf/rewriter_config.proto | 2 +- tensorflow/core/util/dump_graph.cc | 131 + tensorflow/core/util/dump_graph.h | 52 + tensorflow/core/util/dump_graph_test.cc | 62 + tensorflow/core/util/mkl_util.h | 3 + .../core/util/permutation_input_iterator.h | 6 +- .../core/util/permutation_output_iterator.h | 129 + tensorflow/core/util/sparse/sparse_tensor.h | 50 +- tensorflow/core/util/stats_calculator.cc | 4 +- tensorflow/core/util/strided_slice_op.cc | 21 +- .../core/util/tensor_bundle/tensor_bundle.cc | 4 +- tensorflow/core/util/tensor_format.h | 3 +- tensorflow/core/util/tensor_ops_util.h | 128 + tensorflow/examples/adding_an_op/fact_test.py | 2 + .../examples/adding_an_op/zero_out_1_test.py | 2 + .../examples/adding_an_op/zero_out_2_test.py | 5 + .../examples/adding_an_op/zero_out_3_test.py | 9 +- .../autograph/integration_tests/BUILD | 2 - .../autograph/integration_tests/keras_test.py | 6 +- .../integration_tests/list_literals_test.py | 2 +- .../regression/custom_regression.py | 5 +- .../get_started/regression/dnn_regression.py | 5 +- .../linear_regression_categorical.py | 5 +- .../reading_data/fully_connected_reader.py | 2 +- .../examples/learn/iris_custom_decay_dnn.py | 4 +- .../examples/learn/iris_custom_model.py | 4 +- .../examples/speech_commands/freeze_test.py | 4 + .../speech_commands/input_data_test.py | 11 +- .../speech_commands/label_wav_test.py | 2 +- .../examples/speech_commands/models_test.py | 6 + .../speech_commands/wav_to_features_test.py | 4 +- .../examples/tutorials/layers/cnn_mnist.py | 9 +- tensorflow/go/README.md | 2 +- tensorflow/go/graph.go | 76 +- tensorflow/go/graph_test.go | 258 + tensorflow/go/op/gradients.go | 50 + tensorflow/go/op/gradients_test.go | 246 + tensorflow/go/op/wrappers.go | 2797 +- tensorflow/java/README.md | 2 +- .../test/java/org/tensorflow/TensorTest.java | 46 +- tensorflow/lite/BUILD | 2 + tensorflow/lite/build_def.bzl | 10 + tensorflow/lite/builtin_ops.h | 5 + tensorflow/lite/c/builtin_op_data.h | 18 + tensorflow/lite/c/builtin_op_data_test.cc | 1 + tensorflow/lite/c/c_api_internal.c | 15 +- tensorflow/lite/c/c_api_internal.h | 21 +- tensorflow/lite/c/c_api_internal_test.cc | 1 + .../lite/core/api/flatbuffer_conversions.cc | 35 + tensorflow/lite/core/subgraph.cc | 970 + tensorflow/lite/core/subgraph.h | 501 + tensorflow/lite/delegates/flex/BUILD | 3 + tensorflow/lite/delegates/flex/buffer_map.cc | 5 + tensorflow/lite/delegates/flex/delegate.cc | 54 +- .../lite/delegates/flex/delegate_data.cc | 7 +- .../lite/delegates/flex/delegate_test.cc | 19 + tensorflow/lite/delegates/flex/kernel.cc | 24 +- tensorflow/lite/delegates/flex/kernel_test.cc | 4 +- tensorflow/lite/delegates/flex/test_util.cc | 45 +- tensorflow/lite/delegates/flex/test_util.h | 2 + tensorflow/lite/delegates/flex/util.cc | 4 + .../lite/delegates/nnapi/nnapi_delegate.cc | 2 +- .../delegates/nnapi/nnapi_delegate_test.cc | 16 + .../ios/camera/CameraExampleViewController.h | 4 +- .../ios/camera/CameraExampleViewController.mm | 8 +- tensorflow/lite/examples/ios/camera/Podfile | 2 +- tensorflow/lite/examples/ios/simple/Podfile | 2 +- .../ios/simple/RunModelViewController.mm | 8 +- .../lstm/unidirectional_sequence_lstm_test.py | 2 +- tensorflow/lite/experimental/micro/README.md | 42 + .../micro/examples/micro_speech/.gitignore | 1 + .../micro/examples/micro_speech/BUILD | 219 +- .../examples/micro_speech/CMSIS/README.md | 10 + .../CMSIS/arm_cmplx_mag_squared_q10p6.c | 141 + .../CMSIS/arm_cmplx_mag_squared_q10p6.h | 33 + .../micro_speech/CMSIS/create_constants.py | 62 + .../micro/examples/micro_speech/CMSIS/hann.h | 1 - .../examples/micro_speech/CMSIS/hanning.cc | 67 + .../examples/micro_speech/CMSIS/hanning.h | 24 + .../CMSIS/no_power_spectrum_data.h | 29 - .../micro_speech/CMSIS/preprocessor.cc | 44 +- .../micro_speech/CMSIS/preprocessor_test.cc | 63 - .../examples/micro_speech/CMSIS/sin_1k.cc | 67 + .../examples/micro_speech/CMSIS/sin_1k.h | 24 + .../CMSIS/yes_power_spectrum_data.h | 29 - .../examples/micro_speech/apollo3/.gitignore | 4 + .../examples/micro_speech/apollo3/README.md | 72 + .../micro_speech/apollo3}/_main.c | 0 .../examples/micro_speech/apollo3/apollo3.h | 23332 ++++++++++++++++ .../apollo3/captured_data_to_wav.py | 40 + .../micro_speech/apollo3/compare_1k.py | 153 + .../micro_speech/apollo3/preprocessor_1k.cc | 57 + .../apollo3/preprocessor_1k_cmsis_test.cmd | 44 + .../apollo3/preprocessor_1k_micro_test.cmd | 32 + .../apollo3/preprocessor_test.cmd | 18 + .../apollo3/pushbutton_cmsis_scores.cmd | 33 + .../apollo3/pushbutton_cmsis_voice.cmd | 24 + .../micro_speech/apollo3/pushbutton_main.c | 339 + .../micro_speech/apollo3/pushbutton_test.cc | 120 + .../micro_speech/apollo3/system_apollo3.c | 116 + .../micro_speech/apollo3/system_apollo3.h | 72 + .../examples/micro_speech/audio_provider.cc | 33 + .../examples/micro_speech/audio_provider.h | 36 + .../micro_speech/audio_provider_test.cc | 44 + .../examples/micro_speech/feature_provider.cc | 121 + .../examples/micro_speech/feature_provider.h | 48 + .../micro_speech/feature_provider_test.cc | 38 + .../micro_speech/fixed_point/preprocessor.cc | 44 +- .../micro/examples/micro_speech/main.cc | 112 + ...wer_spectrum_data.cc => model_settings.cc} | 11 +- .../examples/micro_speech/model_settings.h | 42 + .../examples/micro_speech/preprocessor.cc | 46 +- .../examples/micro_speech/preprocessor.h | 8 + .../micro_speech/preprocessor_test.cc | 1 + .../yes_power_spectrum_data.cc => timer.cc} | 10 +- .../micro/examples/micro_speech/timer.h | 31 + .../micro/examples/micro_speech/timer_test.cc | 49 + .../experimental/micro/testing/micro_test.h | 18 + .../experimental/micro/tools/make/Makefile | 63 +- .../tools/make/targets/apollo3evb/README.md | 13 - .../targets/apollo3evb/get_yesno_data.cmd | 10 - .../targets/apollo3evb/prep_apollo3_files.sh | 21 +- .../targets/apollo3evb/preprocessor_test.cmd | 4 - .../apollo3evb/replace_calculated_data.py | 33 - .../make/targets/apollo3evb_makefile.inc | 76 +- .../tools/make/targets/cmsis-dsp_makefile.inc | 9 - .../audio_microfrontend_op_test.py | 4 +- tensorflow/lite/experimental/writer/BUILD | 9 +- .../writer/option_writer_generator.cc | 4 + tensorflow/lite/g3doc/_book.yaml | 6 +- tensorflow/lite/g3doc/_index.yaml | 6 +- tensorflow/lite/g3doc/convert/index.md | 14 +- tensorflow/lite/g3doc/convert/python_api.md | 29 +- tensorflow/lite/g3doc/devguide.md | 169 +- tensorflow/lite/g3doc/models.md | 7 +- tensorflow/lite/g3doc/tf_ops_compatibility.md | 270 +- tensorflow/lite/g3doc/using_select_tf_ops.md | 249 + tensorflow/lite/interpreter.cc | 935 +- tensorflow/lite/interpreter.h | 330 +- tensorflow/lite/interpreter_test.cc | 16 +- .../java/org/tensorflow/lite/DataType.java | 9 +- .../main/java/org/tensorflow/lite/Tensor.java | 2 + tensorflow/lite/java/src/main/native/BUILD | 1 + .../native/nativeinterpreterwrapper_jni.cc | 2 + .../lite/java/src/main/native/tensor_jni.cc | 115 +- .../org/tensorflow/lite/DataTypeTest.java | 1 + .../lite/NativeInterpreterWrapperTest.java | 47 + tensorflow/lite/java/src/testdata/string.bin | Bin 0 -> 584 bytes tensorflow/lite/kernels/BUILD | 57 + tensorflow/lite/kernels/activations.cc | 138 +- tensorflow/lite/kernels/activations_test.cc | 143 +- tensorflow/lite/kernels/conv.cc | 16 +- tensorflow/lite/kernels/eigen_support.cc | 13 +- tensorflow/lite/kernels/elementwise.cc | 13 + tensorflow/lite/kernels/elementwise_test.cc | 13 + tensorflow/lite/kernels/fill.cc | 141 + tensorflow/lite/kernels/fill_test.cc | 94 + tensorflow/lite/kernels/fully_connected.cc | 2 +- tensorflow/lite/kernels/gather.cc | 2 +- tensorflow/lite/kernels/hashtable_lookup.cc | 2 +- tensorflow/lite/kernels/internal/BUILD | 2 - .../kernels/internal/optimized/cblas_conv.h | 109 - .../internal/optimized/cblas_reference.h | 69 - .../internal/optimized/depthwiseconv_float.h | 28 +- .../internal/optimized/depthwiseconv_uint8.h | 28 +- .../depthwiseconv_uint8_3x3_filter.h | 168 +- .../internal/optimized/optimized_ops.h | 54 +- .../internal/reference/reference_ops.h | 79 +- .../lite/kernels/internal/tensor_ctypes.h | 10 + tensorflow/lite/kernels/internal/types.h | 12 + tensorflow/lite/kernels/mirror_pad.cc | 374 + tensorflow/lite/kernels/mirror_pad_test.cc | 189 + tensorflow/lite/kernels/pooling_test.cc | 43 + tensorflow/lite/kernels/reduce.cc | 36 +- tensorflow/lite/kernels/register.cc | 12 + tensorflow/lite/kernels/register.h | 1 - tensorflow/lite/kernels/skip_gram.cc | 4 +- tensorflow/lite/kernels/split_v.cc | 207 + tensorflow/lite/kernels/split_v_test.cc | 175 + tensorflow/lite/kernels/squared_difference.cc | 129 + .../lite/kernels/squared_difference_test.cc | 157 + tensorflow/lite/kernels/test_util.h | 3 +- .../lite/lib_package/create_ios_frameworks.sh | 6 +- .../android/smartreply/SmartReplyClient.java | 21 +- .../lite/models/smartreply/ops/normalize.cc | 2 +- .../lite/models/smartreply/predictor.cc | 2 +- tensorflow/lite/nnapi_delegate.cc | 92 +- tensorflow/lite/nnapi_delegate.h | 5 +- tensorflow/lite/nnapi_delegate_disabled.cc | 6 +- tensorflow/lite/optional_debug_tools.cc | 2 + tensorflow/lite/python/BUILD | 2 + tensorflow/lite/python/convert.py | 60 +- tensorflow/lite/python/convert_saved_model.py | 21 +- .../lite/python/convert_saved_model_test.py | 24 +- tensorflow/lite/python/convert_test.py | 58 + .../interpreter_wrapper.cc | 4 + tensorflow/lite/python/lite.py | 17 +- tensorflow/lite/python/lite_constants.py | 25 +- tensorflow/lite/python/lite_test.py | 30 +- tensorflow/lite/python/op_hint.py | 6 +- tensorflow/lite/python/tflite_convert.py | 33 +- tensorflow/lite/schema/schema.fbs | 36 + tensorflow/lite/schema/schema_generated.h | 652 +- tensorflow/lite/string_util.cc | 7 +- tensorflow/lite/string_util.h | 12 +- tensorflow/lite/string_util_test.cc | 37 +- tensorflow/lite/testing/generate_examples.py | 427 +- .../testing/generated_examples_zip_test.cc | 4 + tensorflow/lite/testing/tflite_driver.cc | 11 +- tensorflow/lite/testing/tflite_driver.h | 14 +- tensorflow/lite/toco/BUILD | 52 +- tensorflow/lite/toco/README.md | 8 +- tensorflow/lite/toco/export_tensorflow.cc | 48 +- .../fuse_binary_into_following_affine.cc | 10 +- .../propagate_array_data_types.cc | 7 + .../propagate_fixed_sizes.cc | 147 + .../toco/graph_transformations/quantize.cc | 5 +- .../resolve_constant_gather.cc | 4 + .../resolve_reduce_attributes.cc | 5 + .../unroll_batch_matmul.cc | 3 +- tensorflow/lite/toco/import_tensorflow.cc | 72 +- tensorflow/lite/toco/model.h | 65 +- tensorflow/lite/toco/tflite/export.cc | 31 +- tensorflow/lite/toco/tflite/export_test.cc | 40 +- tensorflow/lite/toco/tflite/operator.cc | 95 +- tensorflow/lite/toco/tflite/operator_test.cc | 32 + .../lite/toco/tflite/whitelisted_flex_ops.cc | 1 + tensorflow/lite/toco/toco.cc | 82 +- tensorflow/lite/toco/toco_convert.cc | 108 + tensorflow/lite/toco/toco_convert.h | 34 + tensorflow/lite/toco/toco_convert_test.cc | 173 + tensorflow/lite/toco/toco_tooling.cc | 7 +- tensorflow/lite/toco/tooling_util.cc | 52 +- tensorflow/lite/toco/types.proto | 3 + .../lite/tools/benchmark/benchmark_model.cc | 49 +- .../lite/tools/benchmark/benchmark_model.h | 3 +- .../lite/tools/benchmark/benchmark_test.cc | 2 + .../tools/benchmark/benchmark_tflite_model.cc | 11 +- .../tools/benchmark/benchmark_tflite_model.h | 7 +- .../TFLiteBenchmark.xcodeproj/project.pbxproj | 30 +- tensorflow/lite/tools/make/Makefile | 8 + .../lite/tools/make/targets/ios_makefile.inc | 2 +- .../tools/optimize/g3doc/quantize_weights.md | 2 +- tensorflow/lite/tools/pip_package/MANIFEST.in | 1 + tensorflow/lite/tools/pip_package/README.md | 33 + .../tools/pip_package/build_pip_package.sh | 54 + tensorflow/lite/tools/pip_package/setup.py | 150 + tensorflow/lite/tutorials/mnist_tflite.py | 4 +- tensorflow/lite/util.h | 6 + tensorflow/opensource_only.files | 17 + tensorflow/python/BUILD | 199 +- tensorflow/python/__init__.py | 6 +- tensorflow/python/autograph/converters/BUILD | 2 - .../autograph/converters/asserts_test.py | 4 +- .../converters/builtin_functions_test.py | 4 + .../python/autograph/converters/call_trees.py | 22 +- .../autograph/converters/call_trees_test.py | 34 +- .../converters/continue_statements.py | 91 +- .../autograph/converters/control_flow.py | 42 +- .../autograph/converters/control_flow_test.py | 13 + .../converters/function_scopes_test.py | 5 + .../python/autograph/converters/lists_test.py | 8 +- .../converters/logical_expressions_test.py | 4 + .../converters/side_effect_guards_test.py | 33 +- .../autograph/converters/slices_test.py | 2 +- tensorflow/python/autograph/core/converter.py | 13 +- .../autograph/core/converter_testing.py | 6 +- .../python/autograph/core/errors_test.py | 10 +- .../autograph/core/function_wrapping_test.py | 2 + tensorflow/python/autograph/core/naming.py | 5 +- tensorflow/python/autograph/impl/BUILD | 2 - tensorflow/python/autograph/impl/api.py | 11 + tensorflow/python/autograph/impl/api_test.py | 77 +- .../python/autograph/impl/conversion.py | 35 +- .../autograph/lang/special_functions_test.py | 12 +- .../autograph/operators/control_flow.py | 4 +- .../autograph/operators/control_flow_test.py | 20 +- .../operators/data_structures_test.py | 27 +- .../autograph/operators/exceptions_test.py | 9 +- .../autograph/operators/logical_test.py | 17 +- .../autograph/operators/py_builtins_test.py | 32 +- .../python/autograph/operators/slices_test.py | 8 +- tensorflow/python/autograph/pyct/BUILD | 2 - .../python/autograph/pyct/inspect_utils.py | 22 + .../autograph/pyct/inspect_utils_test.py | 37 +- .../autograph/pyct/static_analysis/BUILD | 2 - .../pyct/static_analysis/liveness.py | 14 + .../python/autograph/utils/misc_test.py | 7 +- .../python/autograph/utils/py_func_test.py | 18 +- .../autograph/utils/tensor_list_test.py | 33 +- .../python/autograph/utils/type_check.py | 2 +- .../python/autograph/utils/type_check_test.py | 1 + tensorflow/python/client/device_lib.i | 7 +- tensorflow/python/client/session.py | 4 +- .../client/session_clusterspec_prop_test.py | 6 +- .../python/client/session_partial_run_test.py | 26 +- tensorflow/python/client/timeline_test.py | 5 +- tensorflow/python/client/virtual_gpu_test.py | 2 +- tensorflow/python/compat/BUILD | 5 +- tensorflow/python/compat/compat.py | 45 +- tensorflow/python/data/__init__.py | 2 + tensorflow/python/data/benchmarks/BUILD | 55 + .../python/data/benchmarks/batch_benchmark.py | 85 + .../data/benchmarks/filter_benchmark.py | 69 + .../from_tensor_slices_benchmark.py | 188 + .../python/data/benchmarks/map_benchmark.py | 135 + .../python/data/benchmarks/range_benchmark.py | 2 +- .../python/data/experimental/__init__.py | 15 +- .../python/data/experimental/benchmarks/BUILD | 104 +- .../benchmarks/autotune_benchmark.py | 187 + .../benchmarks/csv_dataset_benchmark.py | 130 + .../benchmarks/map_and_batch_benchmark.py | 131 +- .../experimental/benchmarks/map_benchmark.py | 245 - .../benchmarks/map_vectorization_benchmark.py | 194 + .../benchmarks/matching_files_benchmark.py | 102 + .../benchmarks/optimize_benchmark.py | 120 + .../benchmarks/unbatch_benchmark.py | 107 + .../data/experimental/kernel_tests/BUILD | 60 +- .../kernel_tests/batch_dataset_op_test.py | 688 - .../bucket_by_sequence_length_test.py | 22 +- .../kernel_tests/cardinality_test.py | 158 + .../kernel_tests/copy_to_device_test.py | 238 +- .../experimental/kernel_tests/counter_test.py | 23 +- .../kernel_tests/csv_dataset_test.py | 99 - .../dense_to_sparse_batch_test.py | 41 +- .../directed_interleave_dataset_test.py | 24 +- .../kernel_tests/enumerate_dataset_test.py | 15 +- .../kernel_tests/filter_dataset_op_test.py | 6 +- .../function_buffering_resource_test.py | 248 - .../kernel_tests/get_single_element_test.py | 13 + .../kernel_tests/group_by_reducer_test.py | 22 +- .../kernel_tests/group_by_window_test.py | 102 +- .../kernel_tests/ignore_errors_test.py | 34 +- .../kernel_tests/indexed_dataset_ops_test.py | 14 +- .../make_batched_features_dataset_test.py | 103 +- .../kernel_tests/make_csv_dataset_test.py | 32 +- .../make_tf_record_dataset_test.py | 32 +- .../kernel_tests/map_and_batch_test.py | 109 +- .../kernel_tests/map_defun_op_test.py | 6 +- .../kernel_tests/matching_files_test.py} | 125 +- .../kernel_tests/optimization/BUILD | 25 +- .../optimization/assert_next_dataset_test.py | 36 +- .../optimization/filter_fusion_test.py | 41 +- .../optimization/hoist_random_uniform_test.py | 41 +- .../optimization/latency_all_edges_test.py | 58 +- .../optimization/make_numa_aware_test.py | 12 +- .../optimization/map_and_batch_fusion_test.py | 16 +- .../map_and_filter_fusion_test.py | 34 +- .../optimization/map_fusion_test.py | 32 +- .../optimization/map_parallelization_test.py | 25 +- .../optimization/map_vectorization_test.py | 118 +- .../optimization/model_dataset_test.py | 171 +- .../optimization/noop_elimination_test.py | 18 +- .../optimization/optimize_dataset_test.py | 197 +- .../shuffle_and_repeat_fusion_test.py | 26 +- .../kernel_tests/override_threadpool_test.py | 108 +- .../kernel_tests/parallel_interleave_test.py | 85 +- .../parse_example_dataset_test.py | 4 + .../kernel_tests/prefetch_to_device_test.py | 116 +- .../kernel_tests/rejection_resample_test.py | 14 +- .../kernel_tests/restructured_dataset_test.py | 2 + .../experimental/kernel_tests/scan_test.py | 32 +- .../kernel_tests/serialization/BUILD | 11 +- .../checkpoint_input_pipeline_hook_test.py | 8 +- .../dataset_serialization_test_base.py | 3 +- .../filter_dataset_serialization_test.py | 6 +- ...tching_files_dataset_serialization_test.py | 4 +- .../range_dataset_serialization_test.py | 34 +- .../serialization_integration_test.py | 6 +- .../shuffle_dataset_serialization_test.py | 4 +- .../kernel_tests/shuffle_and_repeat_test.py | 15 +- .../experimental/kernel_tests/sleep_test.py | 10 +- .../kernel_tests/sql_dataset_test.py | 169 +- .../kernel_tests/sql_dataset_test_base.py | 3 +- .../kernel_tests/stats_dataset_ops_test.py | 200 +- .../kernel_tests/stats_dataset_test_base.py | 3 +- .../experimental/kernel_tests/unbatch_test.py | 137 +- .../experimental/kernel_tests/unique_test.py | 11 +- tensorflow/python/data/experimental/ops/BUILD | 90 +- .../python/data/experimental/ops/batching.py | 54 +- .../data/experimental/ops/cardinality.py | 50 + .../python/data/experimental/ops/counter.py | 14 +- .../data/experimental/ops/enumerate_ops.py | 4 +- .../python/data/experimental/ops/error_ops.py | 14 +- .../experimental/ops/filter_for_shard_ops.py | 106 + .../experimental/ops/get_single_element.py | 2 +- .../python/data/experimental/ops/grouping.py | 142 +- .../experimental/ops/indexed_dataset_ops.py | 2 + .../data/experimental/ops/interleave_ops.py | 30 +- .../data/experimental/ops/matching_files.py | 51 + .../data/experimental/ops/optimization.py | 28 +- .../experimental/ops/optimization_options.py | 83 + .../data/experimental/ops/parsing_ops.py | 8 +- .../data/experimental/ops/prefetching_ops.py | 342 +- .../data/experimental/ops/random_ops.py | 27 +- .../python/data/experimental/ops/readers.py | 176 +- .../python/data/experimental/ops/scan_ops.py | 15 +- .../data/experimental/ops/shuffle_ops.py | 14 +- .../python/data/experimental/ops/sleep.py | 2 +- .../data/experimental/ops/stats_aggregator.py | 7 +- .../python/data/experimental/ops/stats_ops.py | 26 +- .../data/experimental/ops/stats_options.py | 78 +- .../experimental/ops/threading_options.py | 50 + .../data/experimental/ops/threadpool.py | 14 +- .../python/data/experimental/ops/unique.py | 14 +- .../python/data/experimental/ops/writers.py | 6 +- tensorflow/python/data/kernel_tests/BUILD | 540 +- .../kernel_tests/batch_dataset_op_test.py | 515 - .../python/data/kernel_tests/batch_test.py | 173 + .../kernel_tests/cache_dataset_op_test.py | 318 - .../python/data/kernel_tests/cache_test.py | 253 + ...dataset_op_test.py => concatenate_test.py} | 64 +- ..._op_test.py => dataset_checkpoint_test.py} | 71 +- .../dataset_constructor_op_test.py | 650 - .../{dataset_ops_test.py => dataset_test.py} | 87 +- .../kernel_tests/filter_dataset_op_test.py | 220 - .../python/data/kernel_tests/filter_test.py | 128 + .../fixed_length_record_dataset_test.py | 171 + ...ap_dataset_op_test.py => flat_map_test.py} | 102 +- ...ator_op_test.py => from_generator_test.py} | 103 +- .../from_sparse_tensor_slices_test.py | 86 + .../kernel_tests/from_tensor_slices_test.py | 177 + .../data/kernel_tests/from_tensors_test.py | 259 + .../python/data/kernel_tests/inputs_test.py | 149 - ..._dataset_op_test.py => interleave_test.py} | 77 +- .../kernel_tests/iterator_checkpoint_test.py | 129 + ...uster_test.py => iterator_cluster_test.py} | 8 +- ...{iterator_ops_test.py => iterator_test.py} | 208 +- .../list_files_dataset_op_test.py | 291 - .../data/kernel_tests/list_files_test.py | 214 + .../{map_dataset_op_test.py => map_test.py} | 272 +- .../multi_device_iterator_test.py | 122 +- ...{optional_ops_test.py => optional_test.py} | 137 +- .../data/kernel_tests/padded_batch_test.py | 243 + ...ch_dataset_op_test.py => prefetch_test.py} | 34 +- .../python/data/kernel_tests/range_test.py | 72 + .../kernel_tests/reader_dataset_ops_test.py | 846 - ...duce_dataset_op_test.py => reduce_test.py} | 43 +- .../python/data/kernel_tests/repeat_test.py | 84 + .../kernel_tests/sequence_dataset_op_test.py | 210 - ...shard_dataset_op_test.py => shard_test.py} | 58 +- .../kernel_tests/shuffle_dataset_op_test.py | 278 - .../python/data/kernel_tests/shuffle_test.py | 249 + .../python/data/kernel_tests/skip_test.py | 62 + .../python/data/kernel_tests/take_test.py | 55 + .../python/data/kernel_tests/test_base.py | 102 +- .../kernel_tests/text_line_dataset_test.py | 165 + .../kernel_tests/tf_record_dataset_test.py | 170 + .../kernel_tests/window_dataset_op_test.py | 291 - .../python/data/kernel_tests/window_test.py | 231 + .../data/kernel_tests/zip_dataset_op_test.py | 115 - .../python/data/kernel_tests/zip_test.py | 101 + tensorflow/python/data/ops/BUILD | 5 + tensorflow/python/data/ops/dataset_ops.py | 1495 +- tensorflow/python/data/ops/iterator_ops.py | 2 +- tensorflow/python/data/ops/optional_ops.py | 16 +- tensorflow/python/data/ops/readers.py | 126 +- tensorflow/python/data/util/BUILD | 17 + tensorflow/python/data/util/convert_test.py | 144 +- tensorflow/python/data/util/options.py | 131 + tensorflow/python/data/util/options_test.py | 96 + tensorflow/python/data/util/sparse.py | 2 +- tensorflow/python/data/util/sparse_test.py | 9 +- tensorflow/python/data/util/structure.py | 69 +- tensorflow/python/data/util/structure_test.py | 71 +- tensorflow/python/debug/BUILD | 2 + .../python/debug/cli/analyzer_cli_test.py | 35 +- .../python/debug/cli/cli_shared_test.py | 11 + .../debug/cli/profile_analyzer_cli_test.py | 5 + tensorflow/python/debug/lib/common_test.py | 3 + .../python/debug/lib/debug_gradients_test.py | 14 + .../lib/debug_graph_reconstruction_test.py | 14 +- .../python/debug/lib/debug_utils_test.py | 10 + .../debug/lib/session_debug_file_test.py | 2 + .../debug/lib/session_debug_multi_gpu_test.py | 2 +- .../python/debug/lib/source_utils_test.py | 5 +- .../python/debug/wrappers/framework_test.py | 2 +- tensorflow/python/distribute/BUILD | 240 +- tensorflow/python/distribute/all_reduce.py | 860 + .../distribute}/all_reduce_test.py | 11 +- .../python/distribute/cluster_resolver/BUILD | 180 + .../distribute}/cluster_resolver/README.md | 0 .../distribute/cluster_resolver}/README.slurm | 0 .../distribute/cluster_resolver/__init__.py | 57 + .../cluster_resolver/cluster_resolver.py | 374 + .../cluster_resolver_test.py | 4 +- .../cluster_resolver/gce_cluster_resolver.py | 206 + .../gce_cluster_resolver_test.py | 4 +- .../kubernetes_cluster_resolver.py | 173 + .../kubernetes_cluster_resolver_test.py | 19 +- .../slurm_cluster_resolver.py | 226 + .../slurm_cluster_resolver_test.py | 27 +- .../tfconfig_cluster_resolver.py | 171 + .../tfconfig_cluster_resolver_test.py | 54 +- .../cluster_resolver/tpu_cluster_resolver.py | 423 + .../tpu_cluster_resolver_test.py | 64 +- .../distribute/cross_device_ops.py} | 161 +- .../distribute/cross_device_utils.py} | 8 +- .../{training => distribute}/device_util.py | 2 +- .../device_util_test.py | 5 +- .../distribute/distribute_coordinator.py | 27 +- .../distribute/distribute_coordinator_test.py | 78 +- .../python/distribute/distribute_lib.py | 1682 ++ .../distribute_lib_test.py} | 47 +- .../distribution_strategy_context.py | 236 + .../python/distribute/estimator_training.py | 4 +- .../python => python/distribute}/input_ops.py | 20 +- .../distribute}/input_ops_test.py | 35 +- .../python/distribute/mirrored_strategy.py | 908 + .../python/distribute/multi_worker_util.py | 98 +- .../distribute/multi_worker_util_test.py | 91 +- tensorflow/python/distribute/reduce_util.py | 53 + .../distribute}/shared_variable_creator.py | 0 .../shared_variable_creator_test.py | 2 +- .../python => python/distribute}/values.py | 454 +- tensorflow/python/eager/BUILD | 32 +- tensorflow/python/eager/backprop.py | 231 +- tensorflow/python/eager/backprop_test.py | 253 +- tensorflow/python/eager/benchmarks_test.py | 58 +- tensorflow/python/eager/context.py | 55 +- tensorflow/python/eager/def_function.py | 8 +- tensorflow/python/eager/def_function_test.py | 21 +- .../python/eager/execution_callbacks.py | 45 +- .../python/eager/execution_callbacks_test.py | 55 + tensorflow/python/eager/function.py | 149 +- .../python/eager/function_gradients_test.py | 756 + tensorflow/python/eager/function_test.py | 745 +- .../python/eager/graph_only_ops_test.py | 4 +- tensorflow/python/eager/pywrap_tensor.cc | 30 +- tensorflow/python/eager/pywrap_tfe_src.cc | 32 +- tensorflow/python/eager/tape.py | 4 +- tensorflow/python/eager/tape_test.py | 8 +- tensorflow/python/eager/tensor_test.py | 47 +- tensorflow/python/eager/test.py | 2 +- tensorflow/python/eager/wrap_function.py | 18 + .../python/feature_column/feature_column.py | 242 +- .../feature_column/feature_column_lib.py | 1 + .../feature_column/feature_column_test.py | 1881 +- .../feature_column/feature_column_v2.py | 491 +- .../feature_column/feature_column_v2_test.py | 3175 ++- .../python/framework/auto_control_deps.py | 13 + tensorflow/python/framework/constant_op.py | 81 +- tensorflow/python/framework/device.py | 2 +- tensorflow/python/framework/dtypes.py | 10 +- tensorflow/python/framework/dtypes_test.py | 4 +- .../python/framework/file_system_test.py | 4 +- tensorflow/python/framework/func_graph.py | 72 +- tensorflow/python/framework/function.py | 4 +- .../framework/function_def_to_graph_test.py | 3 + tensorflow/python/framework/function_test.py | 92 +- .../python/framework/graph_util_test.py | 14 +- tensorflow/python/framework/importer.py | 24 +- tensorflow/python/framework/importer_test.py | 20 +- tensorflow/python/framework/load_library.py | 4 +- .../python/framework/meta_graph_test.py | 39 +- tensorflow/python/framework/op_def_library.py | 2 +- tensorflow/python/framework/ops.py | 355 +- tensorflow/python/framework/ops_test.py | 252 +- tensorflow/python/framework/python_op_gen.cc | 28 +- .../framework/python_op_gen_internal.cc | 6 +- tensorflow/python/framework/random_seed.py | 108 +- tensorflow/python/framework/registry.py | 11 +- tensorflow/python/framework/registry_test.py | 4 +- .../python/framework/smart_cond_test.py | 13 +- .../python/framework/sparse_tensor_test.py | 8 +- tensorflow/python/framework/subscribe_test.py | 38 +- tensorflow/python/framework/tensor_shape.py | 2 +- tensorflow/python/framework/tensor_spec.py | 42 +- .../python/framework/tensor_spec_test.py | 25 +- tensorflow/python/framework/tensor_util.py | 28 +- .../python/framework/tensor_util_test.py | 14 +- tensorflow/python/framework/test_util.py | 301 +- tensorflow/python/framework/test_util_test.py | 7 +- .../python/grappler/constant_folding_test.py | 2 +- .../python/grappler/cost_analyzer_test.py | 8 +- .../python/grappler/cost_analyzer_tool.py | 9 +- tensorflow/python/grappler/datasets_test.py | 20 +- tensorflow/python/grappler/graph_placer.py | 6 +- tensorflow/python/grappler/item_test.py | 2 + .../python/grappler/layout_optimizer_test.py | 71 +- .../python/grappler/memory_optimizer_test.py | 80 +- .../python/grappler/model_analyzer_test.py | 3 + tensorflow/python/grappler/tf_optimizer.i | 19 +- tensorflow/python/grappler/tf_optimizer.py | 8 +- .../python/grappler/tf_optimizer_test.py | 34 +- tensorflow/python/keras/BUILD | 73 +- tensorflow/python/keras/activations_test.py | 4 + tensorflow/python/keras/backend.py | 135 +- tensorflow/python/keras/backend_test.py | 139 +- tensorflow/python/keras/callbacks.py | 80 +- tensorflow/python/keras/callbacks_test.py | 84 +- tensorflow/python/keras/engine/__init__.py | 2 +- tensorflow/python/keras/engine/base_layer.py | 1979 +- .../python/keras/engine/base_layer_test.py | 8 +- .../python/keras/engine/base_layer_utils.py | 236 + .../engine/distributed_training_utils.py | 269 +- .../feature_columns_integration_test.py | 29 +- tensorflow/python/keras/engine/input_layer.py | 29 +- tensorflow/python/keras/engine/input_spec.py | 170 + tensorflow/python/keras/engine/network.py | 317 +- tensorflow/python/keras/engine/saving.py | 10 +- tensorflow/python/keras/engine/saving_test.py | 62 +- tensorflow/python/keras/engine/sequential.py | 65 +- .../python/keras/engine/sequential_test.py | 28 +- .../python/keras/engine/topology_test.py | 10 + tensorflow/python/keras/engine/training.py | 962 +- .../python/keras/engine/training_arrays.py | 215 +- .../keras/engine/training_dataset_test.py | 344 + .../keras/engine/training_distributed.py | 633 +- .../python/keras/engine/training_eager.py | 696 +- .../keras/engine/training_eager_test.py | 63 +- .../python/keras/engine/training_generator.py | 759 +- .../keras/engine/training_generator_test.py | 436 +- .../python/keras/engine/training_gpu_test.py | 2 +- .../python/keras/engine/training_test.py | 787 +- .../python/keras/engine/training_utils.py | 389 +- .../keras/engine/training_utils_test.py | 178 +- tensorflow/python/keras/estimator/__init__.py | 59 +- tensorflow/python/keras/initializers_test.py | 11 + tensorflow/python/keras/integration_test.py | 17 + tensorflow/python/keras/layers/__init__.py | 2 +- .../keras/layers/advanced_activations.py | 8 +- .../python/keras/layers/convolutional.py | 12 +- .../keras/layers/convolutional_recurrent.py | 2 +- tensorflow/python/keras/layers/core.py | 11 +- .../python/keras/layers/cudnn_recurrent.py | 16 +- tensorflow/python/keras/layers/embeddings.py | 7 +- tensorflow/python/keras/layers/local.py | 4 +- tensorflow/python/keras/layers/local_test.py | 262 +- tensorflow/python/keras/layers/lstm_test.py | 23 +- tensorflow/python/keras/layers/merge.py | 9 +- tensorflow/python/keras/layers/merge_test.py | 1 + tensorflow/python/keras/layers/noise.py | 3 - .../python/keras/layers/normalization.py | 126 +- .../python/keras/layers/normalization_test.py | 114 +- tensorflow/python/keras/layers/pooling.py | 8 +- tensorflow/python/keras/layers/recurrent.py | 482 +- .../python/keras/layers/recurrent_test.py | 30 +- .../python/keras/layers/simplernn_test.py | 3 + .../python/keras/layers/unified_lstm_test.py | 724 + tensorflow/python/keras/layers/wrappers.py | 6 +- .../python/keras/layers/wrappers_test.py | 36 +- tensorflow/python/keras/losses.py | 420 +- tensorflow/python/keras/losses_test.py | 633 + tensorflow/python/keras/metrics.py | 1101 +- tensorflow/python/keras/metrics_test.py | 993 +- .../python/keras/model_subclassing_test.py | 132 +- tensorflow/python/keras/models.py | 36 +- tensorflow/python/keras/models_test.py | 55 +- tensorflow/python/keras/optimizer_v2/BUILD | 150 +- .../python/keras/optimizer_v2/adadelta.py | 148 + .../keras/optimizer_v2/adadelta_test.py | 170 + .../python/keras/optimizer_v2/adagrad.py | 171 + .../python/keras/optimizer_v2/adagrad_test.py | 400 + tensorflow/python/keras/optimizer_v2/adam.py | 182 +- .../python/keras/optimizer_v2/adam_test.py | 508 + .../python/keras/optimizer_v2/adamax.py | 159 + .../python/keras/optimizer_v2/adamax_test.py | 367 + tensorflow/python/keras/optimizer_v2/ftrl.py | 210 + .../python/keras/optimizer_v2/ftrl_test.py | 440 + .../keras/optimizer_v2/gradient_descent.py | 39 +- .../optimizer_v2/gradient_descent_test.py | 212 +- tensorflow/python/keras/optimizer_v2/nadam.py | 143 + .../python/keras/optimizer_v2/nadam_test.py | 213 + .../python/keras/optimizer_v2/optimizer_v2.py | 82 +- .../keras/optimizer_v2/optimizer_v2_test.py | 259 +- .../python/keras/optimizer_v2/rmsprop.py | 196 + .../python/keras/optimizer_v2/rmsprop_test.py | 410 + tensorflow/python/keras/optimizers.py | 54 +- tensorflow/python/keras/optimizers_test.py | 47 + tensorflow/python/keras/regularizers_test.py | 2 + tensorflow/python/keras/testing_utils.py | 4 + tensorflow/python/keras/utils/__init__.py | 1 + .../python/keras/utils/generic_utils.py | 12 +- tensorflow/python/keras/utils/layer_utils.py | 2 +- tensorflow/python/keras/utils/losses_utils.py | 189 + .../keras/utils/multi_gpu_utils_test.py | 2 +- tensorflow/python/keras/utils/tf_utils.py | 44 +- .../python/keras/utils/tf_utils_test.py | 134 + tensorflow/python/kernel_tests/BUILD | 82 +- .../python/kernel_tests/accumulate_n_test.py | 10 +- .../python/kernel_tests/ackermann_test.py | 2 + .../python/kernel_tests/aggregate_ops_test.py | 5 +- .../python/kernel_tests/argmax_op_test.py | 11 +- .../python/kernel_tests/array_ops_test.py | 151 +- .../python/kernel_tests/as_string_op_test.py | 7 + .../python/kernel_tests/atrous_conv2d_test.py | 18 +- .../python/kernel_tests/attention_ops_test.py | 5 +- .../python/kernel_tests/barrier_ops_test.py | 24 +- .../python/kernel_tests/base64_ops_test.py | 2 +- .../python/kernel_tests/basic_gpu_test.py | 13 +- .../kernel_tests/batch_gather_op_test.py | 10 +- .../kernel_tests/batch_matmul_op_test.py | 2 +- .../kernel_tests/batch_scatter_ops_test.py | 6 +- .../kernel_tests/batchtospace_op_test.py | 21 + .../python/kernel_tests/bcast_ops_test.py | 6 + .../python/kernel_tests/benchmark_test.py | 15 +- .../python/kernel_tests/betainc_op_test.py | 10 +- .../python/kernel_tests/bias_op_test.py | 12 + .../python/kernel_tests/bincount_op_test.py | 7 + .../python/kernel_tests/bitcast_op_test.py | 7 +- .../boosted_trees/prediction_ops_test.py | 15 + .../boosted_trees/quantile_ops_test.py | 18 +- .../boosted_trees/resource_ops_test.py | 45 +- .../boosted_trees/stats_ops_test.py | 65 +- .../boosted_trees/training_ops_test.py | 10 + .../kernel_tests/broadcast_to_ops_test.py | 11 + .../python/kernel_tests/bucketize_op_test.py | 10 +- .../candidate_sampler_ops_test.py | 13 +- .../python/kernel_tests/cast_op_test.py | 17 +- .../python/kernel_tests/check_ops_test.py | 109 + .../kernel_tests/checkpoint_ops_test.py | 53 +- .../python/kernel_tests/cholesky_op_test.py | 15 +- .../python/kernel_tests/clip_ops_test.py | 74 +- .../compare_and_bitpack_op_test.py | 7 +- .../python/kernel_tests/concat_op_test.py | 170 +- .../python/kernel_tests/cond_v2_test.py | 130 +- .../conditional_accumulator_test.py | 66 +- .../kernel_tests/confusion_matrix_test.py | 57 +- .../python/kernel_tests/constant_op_test.py | 114 +- .../kernel_tests/control_flow_ops_py_test.py | 555 +- tensorflow/python/kernel_tests/conv1d_test.py | 4 +- .../conv2d_backprop_filter_grad_test.py | 2 + .../kernel_tests/conv2d_transpose_test.py | 14 +- .../conv3d_backprop_filter_v2_grad_test.py | 2 + .../kernel_tests/conv3d_transpose_test.py | 11 +- .../python/kernel_tests/conv_ops_3d_test.py | 31 +- .../python/kernel_tests/conv_ops_test.py | 18 +- .../python/kernel_tests/cross_grad_test.py | 2 + .../kernel_tests/ctc_decoder_ops_test.py | 3 + .../python/kernel_tests/ctc_loss_op_test.py | 584 +- .../kernel_tests/cwise_ops_binary_test.py | 123 +- .../python/kernel_tests/cwise_ops_test.py | 133 +- .../kernel_tests/cwise_ops_unary_test.py | 30 +- .../python/kernel_tests/decode_bmp_op_test.py | 4 +- .../kernel_tests/decode_compressed_op_test.py | 3 + .../kernel_tests/decode_image_op_test.py | 18 +- .../kernel_tests/decode_jpeg_op_test.py | 6 +- .../python/kernel_tests/decode_png_op_test.py | 2 +- .../python/kernel_tests/decode_raw_op_test.py | 7 + .../python/kernel_tests/denormal_test.py | 3 + .../dense_update_ops_no_tsan_test.py | 21 +- .../kernel_tests/dense_update_ops_test.py | 21 +- .../kernel_tests/depthtospace_op_test.py | 31 +- .../kernel_tests/depthwise_conv_op_test.py | 14 +- .../kernel_tests/determinant_op_test.py | 11 +- .../python/kernel_tests/diag_op_test.py | 44 +- .../distributions/bernoulli_test.py | 5 + .../distributions/bijector_test.py | 2 + .../distributions/categorical_test.py | 39 +- .../dirichlet_multinomial_test.py | 43 +- .../distributions/identity_bijector_test.py | 2 + .../distributions/kullback_leibler_test.py | 10 +- .../distributions/multinomial_test.py | 12 +- .../kernel_tests/distributions/normal_test.py | 1 + .../distributions/special_math_test.py | 15 +- .../kernel_tests/distributions/util_test.py | 52 +- .../kernel_tests/division_future_test.py | 2 +- .../python/kernel_tests/division_past_test.py | 2 +- .../kernel_tests/draw_bounding_box_op_test.py | 2 +- .../python/kernel_tests/duplicate_op_test.py | 2 + .../kernel_tests/dynamic_partition_op_test.py | 40 +- .../kernel_tests/dynamic_stitch_op_test.py | 49 +- .../kernel_tests/edit_distance_op_test.py | 4 +- .../python/kernel_tests/embedding_ops_test.py | 64 +- .../extract_image_patches_grad_test.py | 3 + .../extract_image_patches_op_test.py | 5 +- .../extract_volume_patches_op_test.py | 5 +- .../python/kernel_tests/fifo_queue_test.py | 190 +- .../fractional_avg_pool_op_test.py | 95 +- .../fractional_max_pool_op_test.py | 67 +- .../kernel_tests/functional_ops_test.py | 90 +- .../python/kernel_tests/gather_nd_op_test.py | 57 +- .../python/kernel_tests/gather_op_test.py | 21 +- .../kernel_tests/gradient_correctness_test.py | 19 +- .../python/kernel_tests/huge_slice_op_test.py | 4 +- .../kernel_tests/identity_n_op_py_test.py | 4 + .../kernel_tests/identity_op_py_test.py | 5 + .../python/kernel_tests/in_topk_op_test.py | 4 +- .../python/kernel_tests/init_ops_test.py | 102 +- .../python/kernel_tests/inplace_ops_test.py | 9 +- tensorflow/python/kernel_tests/io_ops_test.py | 7 +- .../kernel_tests/large_concat_op_test.py | 2 +- tensorflow/python/kernel_tests/linalg/BUILD | 63 +- .../linalg/linear_operator_addition_test.py | 13 + .../linalg/linear_operator_adjoint_test.py | 118 + .../linalg/linear_operator_algebra_test.py | 133 + .../linalg/linear_operator_block_diag_test.py | 45 +- .../linalg/linear_operator_circulant_test.py | 116 +- .../linear_operator_composition_test.py | 10 +- .../linalg/linear_operator_diag_test.py | 61 +- .../linear_operator_full_matrix_test.py | 27 +- .../linalg/linear_operator_identity_test.py | 97 +- .../linalg/linear_operator_inversion_test.py | 130 + .../linalg/linear_operator_kronecker_test.py | 54 +- .../linear_operator_low_rank_update_test.py | 13 +- .../linear_operator_lower_triangular_test.py | 30 + .../linalg/linear_operator_test.py | 86 +- .../linalg/linear_operator_util_test.py | 53 +- .../linalg/linear_operator_zeros_test.py | 25 +- .../python/kernel_tests/linalg_grad_test.py | 6 +- .../python/kernel_tests/linalg_ops_test.py | 21 +- .../python/kernel_tests/list_ops_test.py | 408 +- .../python/kernel_tests/listdiff_op_test.py | 2 +- .../python/kernel_tests/logging_ops_test.py | 12 +- .../python/kernel_tests/lookup_ops_test.py | 216 +- tensorflow/python/kernel_tests/losses_test.py | 219 +- tensorflow/python/kernel_tests/lrn_op_test.py | 4 + .../python/kernel_tests/manip_ops_test.py | 12 + .../python/kernel_tests/map_stage_op_test.py | 13 + .../python/kernel_tests/matmul_op_test.py | 10 +- .../kernel_tests/matrix_band_part_op_test.py | 2 +- .../matrix_exponential_op_test.py | 11 +- .../kernel_tests/matrix_inverse_op_test.py | 4 +- .../kernel_tests/matrix_logarithm_op_test.py | 7 +- .../kernel_tests/matrix_solve_ls_op_test.py | 4 + .../kernel_tests/matrix_solve_op_test.py | 10 +- .../matrix_square_root_op_test.py | 20 +- .../matrix_triangular_solve_op_test.py | 9 +- .../python/kernel_tests/metrics_test.py | 683 +- .../kernel_tests/morphological_ops_test.py | 11 +- .../neon_depthwise_conv_op_test.py | 9 +- .../python/kernel_tests/norm_op_test.py | 2 +- .../kernel_tests/nth_element_op_test.py | 10 +- .../python/kernel_tests/numerics_test.py | 26 +- .../python/kernel_tests/one_hot_op_test.py | 4 +- tensorflow/python/kernel_tests/pad_op_test.py | 41 +- .../kernel_tests/padding_fifo_queue_test.py | 182 +- .../parameterized_truncated_normal_op_test.py | 11 + .../parse_single_example_op_test.py | 9 +- .../python/kernel_tests/parsing_ops_test.py | 28 +- .../partitioned_variables_test.py | 29 +- tensorflow/python/kernel_tests/pool_test.py | 6 +- .../kernel_tests/pooling_ops_3d_test.py | 22 +- .../python/kernel_tests/pooling_ops_test.py | 44 +- .../kernel_tests/priority_queue_test.py | 22 +- .../python/kernel_tests/py_func_test.py | 22 +- tensorflow/python/kernel_tests/qr_op_test.py | 8 +- .../random/multinomial_op_big_test.py | 8 +- .../random/multinomial_op_test.py | 40 +- .../kernel_tests/random/random_crop_test.py | 7 +- .../kernel_tests/random/random_gamma_test.py | 8 +- .../kernel_tests/random/random_grad_test.py | 11 + .../kernel_tests/random/random_ops_test.py | 34 +- .../random/random_poisson_test.py | 10 +- .../random/random_shuffle_queue_test.py | 150 +- .../random/stateless_random_ops_test.py | 11 +- .../python/kernel_tests/reader_ops_test.py | 599 +- .../python/kernel_tests/record_input_test.py | 20 +- .../kernel_tests/reduce_benchmark_test.py | 4 +- .../kernel_tests/reduce_join_op_test.py | 20 +- .../python/kernel_tests/reduction_ops_test.py | 87 +- .../kernel_tests/regex_full_match_op_test.py | 9 +- .../kernel_tests/regex_replace_op_test.py | 11 +- .../python/kernel_tests/relu_op_test.py | 282 +- .../python/kernel_tests/reshape_op_test.py | 8 +- .../resource_variable_ops_test.py | 54 +- .../kernel_tests/reverse_sequence_op_test.py | 7 +- tensorflow/python/kernel_tests/rnn_test.py | 10 + .../kernel_tests/save_restore_ops_test.py | 19 + .../python/kernel_tests/scan_ops_test.py | 29 + .../kernel_tests/scatter_nd_ops_test.py | 130 +- .../python/kernel_tests/scatter_ops_test.py | 39 +- .../segment_reduction_ops_test.py | 103 +- .../kernel_tests/self_adjoint_eig_op_test.py | 8 +- .../python/kernel_tests/session_ops_test.py | 46 +- tensorflow/python/kernel_tests/sets_test.py | 24 +- .../python/kernel_tests/shape_ops_test.py | 77 +- tensorflow/python/kernel_tests/signal/BUILD | 31 +- .../kernel_tests/{ => signal}/dct_ops_test.py | 70 +- .../kernel_tests/{ => signal}/fft_ops_test.py | 42 +- .../kernel_tests/signal/mel_ops_test.py | 6 +- .../kernel_tests/signal/mfcc_ops_test.py | 4 + .../signal/reconstruction_ops_test.py | 82 +- .../kernel_tests/signal/shape_ops_test.py | 20 +- .../kernel_tests/signal/spectral_ops_test.py | 18 +- .../python/kernel_tests/signal/test_util.py | 14 +- .../kernel_tests/signal/window_ops_test.py | 3 + .../python/kernel_tests/slice_op_test.py | 54 +- .../python/kernel_tests/softmax_op_test.py | 7 +- .../python/kernel_tests/softplus_op_test.py | 7 +- .../python/kernel_tests/softsign_op_test.py | 5 +- .../kernel_tests/spacetobatch_op_test.py | 36 + .../kernel_tests/spacetodepth_op_test.py | 43 +- .../python/kernel_tests/sparse_add_op_test.py | 34 +- .../kernel_tests/sparse_concat_op_test.py | 21 +- .../sparse_conditional_accumulator_test.py | 60 +- .../kernel_tests/sparse_cross_op_test.py | 54 +- .../kernel_tests/sparse_matmul_op_test.py | 9 +- .../python/kernel_tests/sparse_ops_test.py | 121 +- .../kernel_tests/sparse_reorder_op_test.py | 8 +- .../kernel_tests/sparse_reshape_op_test.py | 22 +- .../sparse_serialization_ops_test.py | 27 +- .../kernel_tests/sparse_slice_op_test.py | 8 + .../kernel_tests/sparse_split_op_test.py | 8 + .../sparse_tensor_dense_matmul_grad_test.py | 2 + .../sparse_tensor_dense_matmul_op_test.py | 6 +- .../sparse_tensors_map_ops_test.py | 21 +- .../sparse_to_dense_op_py_test.py | 36 +- .../kernel_tests/sparse_xent_op_test.py | 22 +- .../python/kernel_tests/sparsemask_op_test.py | 2 + .../python/kernel_tests/split_op_test.py | 9 +- .../python/kernel_tests/stack_op_test.py | 29 +- .../python/kernel_tests/stack_ops_test.py | 43 +- .../python/kernel_tests/stage_op_test.py | 8 + .../kernel_tests/string_join_op_test.py | 2 + .../kernel_tests/string_length_op_test.py | 9 +- .../kernel_tests/string_split_op_test.py | 33 +- .../kernel_tests/string_strip_op_test.py | 6 +- .../string_to_hash_bucket_op_test.py | 9 +- .../kernel_tests/string_to_number_op_test.py | 5 + .../python/kernel_tests/substr_op_test.py | 45 +- .../kernel_tests/summary_v1_audio_op_test.py | 2 +- .../kernel_tests/summary_v1_image_op_test.py | 7 +- .../kernel_tests/summary_v1_ops_test.py | 8 +- .../kernel_tests/summary_v1_tensor_op_test.py | 14 +- tensorflow/python/kernel_tests/svd_op_test.py | 8 +- .../python/kernel_tests/template_test.py | 13 +- .../kernel_tests/tensor_array_ops_test.py | 308 +- .../python/kernel_tests/topk_op_test.py | 6 +- .../python/kernel_tests/trace_op_test.py | 2 + .../python/kernel_tests/transpose_op_test.py | 18 +- .../kernel_tests/unicode_decode_op_test.py | 153 + .../kernel_tests/unicode_encode_op_test.py | 301 + .../kernel_tests/unicode_script_op_test.py | 3 + .../kernel_tests/unicode_transcode_op_test.py | 119 +- .../python/kernel_tests/unique_op_test.py | 24 +- .../python/kernel_tests/unstack_op_test.py | 30 +- .../python/kernel_tests/variable_ops_test.py | 57 +- .../kernel_tests/variable_scope_test.py | 85 +- .../python/kernel_tests/variables_test.py | 253 +- .../kernel_tests/weights_broadcast_test.py | 33 +- .../python/kernel_tests/where_op_test.py | 21 +- .../python/kernel_tests/while_v2_test.py | 175 +- .../python/kernel_tests/xent_op_test.py | 17 +- .../python/kernel_tests/zero_division_test.py | 6 +- tensorflow/python/layers/base.py | 15 +- tensorflow/python/layers/base_test.py | 17 +- .../python/layers/convolutional_test.py | 72 +- tensorflow/python/layers/core_test.py | 21 +- tensorflow/python/layers/layers.py | 2 +- .../python/layers/normalization_test.py | 176 +- tensorflow/python/layers/pooling_test.py | 4 + tensorflow/python/lib/io/file_io.py | 258 +- tensorflow/python/lib/io/tf_record.py | 9 +- tensorflow/python/ops/array_grad.py | 32 +- tensorflow/python/ops/array_ops.py | 518 +- tensorflow/python/ops/bitwise_ops_test.py | 13 +- .../python/ops/candidate_sampling_ops.py | 7 +- tensorflow/python/ops/check_ops.py | 543 +- tensorflow/python/ops/clip_ops.py | 7 +- tensorflow/python/ops/clip_ops_test.py | 6 +- tensorflow/python/ops/collective_ops_test.py | 5 + tensorflow/python/ops/cond_v2.py | 326 +- tensorflow/python/ops/confusion_matrix.py | 77 +- tensorflow/python/ops/control_flow_ops.py | 307 +- .../python/ops/control_flow_ops_test.py | 30 +- tensorflow/python/ops/control_flow_util.py | 5 + tensorflow/python/ops/control_flow_util_v2.py | 30 + tensorflow/python/ops/ctc_ops.py | 795 +- tensorflow/python/ops/data_flow_ops.py | 15 +- tensorflow/python/ops/dequantize_op_test.py | 2 +- tensorflow/python/ops/distributions/util.py | 2 +- tensorflow/python/ops/embedding_ops.py | 5 +- tensorflow/python/ops/functional_ops.py | 19 +- tensorflow/python/ops/gradient_checker.py | 78 +- .../python/ops/gradient_checker_test.py | 14 + tensorflow/python/ops/gradient_checker_v2.py | 318 + .../python/ops/gradient_checker_v2_test.py | 300 + tensorflow/python/ops/gradients_impl.py | 140 +- tensorflow/python/ops/gradients_test.py | 46 +- tensorflow/python/ops/histogram_ops_test.py | 22 +- tensorflow/python/ops/image_grad_test.py | 20 +- tensorflow/python/ops/image_ops_impl.py | 274 +- tensorflow/python/ops/image_ops_test.py | 344 +- tensorflow/python/ops/init_ops.py | 189 +- tensorflow/python/ops/init_ops_test.py | 4 +- tensorflow/python/ops/linalg/BUILD | 1 + .../ops/linalg/cholesky_registrations.py | 101 + tensorflow/python/ops/linalg/linalg.py | 3 + tensorflow/python/ops/linalg/linalg_impl.py | 2 +- .../python/ops/linalg/linear_operator.py | 67 +- .../ops/linalg/linear_operator_adjoint.py | 207 + .../ops/linalg/linear_operator_algebra.py | 191 + .../ops/linalg/linear_operator_block_diag.py | 4 +- .../ops/linalg/linear_operator_circulant.py | 15 +- .../ops/linalg/linear_operator_composition.py | 3 - .../python/ops/linalg/linear_operator_diag.py | 4 +- .../ops/linalg/linear_operator_inversion.py | 207 + .../ops/linalg/linear_operator_kronecker.py | 4 +- .../linalg/linear_operator_low_rank_update.py | 2 +- .../linear_operator_lower_triangular.py | 4 +- .../ops/linalg/linear_operator_test_util.py | 26 +- .../python/ops/linalg/matmul_registrations.py | 252 + tensorflow/python/ops/linalg_ops.py | 73 +- tensorflow/python/ops/list_ops.py | 98 +- tensorflow/python/ops/lookup_ops.py | 11 +- tensorflow/python/ops/losses/losses_impl.py | 111 +- tensorflow/python/ops/losses/util_test.py | 2 + tensorflow/python/ops/math_grad.py | 7 +- tensorflow/python/ops/math_grad_test.py | 49 + tensorflow/python/ops/math_ops.py | 1120 +- tensorflow/python/ops/math_ops_test.py | 45 +- tensorflow/python/ops/metrics_impl.py | 136 +- tensorflow/python/ops/nccl_ops_test.py | 2 +- tensorflow/python/ops/nn_batchnorm_test.py | 31 +- .../python/ops/nn_fused_batchnorm_test.py | 12 +- tensorflow/python/ops/nn_grad.py | 14 +- tensorflow/python/ops/nn_grad_test.py | 2 + tensorflow/python/ops/nn_impl.py | 408 +- tensorflow/python/ops/nn_ops.py | 1201 +- tensorflow/python/ops/nn_test.py | 749 +- tensorflow/python/ops/nn_xent_test.py | 14 +- tensorflow/python/ops/numerics.py | 29 +- .../ops/optional_grad.py} | 23 +- .../ops/parallel_for/control_flow_ops.py | 119 +- .../ops/parallel_for/control_flow_ops_test.py | 264 +- .../python/ops/parallel_for/gradients.py | 25 +- .../python/ops/parallel_for/gradients_test.py | 23 +- tensorflow/python/ops/parallel_for/pfor.py | 10 +- tensorflow/python/ops/parsing_ops.py | 314 +- .../python/ops/partitioned_variables.py | 24 +- .../python/ops/quantized_conv_ops_test.py | 2 +- tensorflow/python/ops/quantized_ops_test.py | 4 +- tensorflow/python/ops/ragged/BUILD | 59 + tensorflow/python/ops/ragged/__init__.py | 10 + ...vert_to_tensor_or_ragged_tensor_op_test.py | 4 +- .../python/ops/ragged/ragged_array_ops.py | 100 +- .../ops/ragged/ragged_batch_gather_op_test.py | 3 + .../ops/ragged/ragged_boolean_mask_op_test.py | 2 + .../ops/ragged/ragged_concat_op_test.py | 10 + .../python/ops/ragged/ragged_const_op_test.py | 8 +- .../ops/ragged/ragged_conversion_ops.py | 29 +- .../ops/ragged/ragged_elementwise_ops.py | 62 +- .../ops/ragged/ragged_elementwise_ops_test.py | 56 +- .../ops/ragged/ragged_expand_dims_op_test.py | 1 + .../python/ops/ragged/ragged_factory_ops.py | 30 + .../ops/ragged/ragged_from_sparse_op_test.py | 19 + .../ops/ragged/ragged_from_tensor_op_test.py | 7 +- .../ops/ragged/ragged_gather_nd_op_test.py | 5 +- .../ops/ragged/ragged_gather_op_test.py | 10 + .../ops/ragged/ragged_map_fn_op_test.py | 11 +- .../ragged/ragged_map_inner_values_op_test.py | 13 + .../ops/ragged/ragged_operators_test.py | 3 + .../python/ops/ragged/ragged_range_op_test.py | 9 + .../ops/ragged/ragged_reduce_op_test.py | 4 + .../ops/ragged/ragged_row_lengths_op_test.py | 1 + ...agged_row_splits_to_segment_ids_op_test.py | 3 + ...agged_segment_ids_to_row_splits_op_test.py | 4 + .../ops/ragged/ragged_segment_op_test.py | 24 +- .../python/ops/ragged/ragged_stack_op_test.py | 2 + .../python/ops/ragged/ragged_string_ops.py | 119 + tensorflow/python/ops/ragged/ragged_tensor.py | 3 +- .../ragged_tensor_bounding_shape_op_test.py | 32 +- .../python/ops/ragged/ragged_tensor_shape.py | 570 + .../ops/ragged/ragged_tensor_shape_test.py | 487 + .../python/ops/ragged/ragged_tensor_test.py | 460 +- .../python/ops/ragged/ragged_tile_op_test.py | 11 + .../ops/ragged/ragged_to_sparse_op_test.py | 10 +- .../ops/ragged/ragged_to_tensor_op_test.py | 50 +- tensorflow/python/ops/ragged/ragged_util.py | 49 + .../python/ops/ragged/ragged_where_op_test.py | 3 +- tensorflow/python/ops/random_ops.py | 94 +- .../python/ops/resource_variable_ops.py | 57 +- tensorflow/python/ops/resources.py | 4 +- tensorflow/python/ops/rnn.py | 12 +- tensorflow/python/ops/rnn_cell_impl.py | 16 +- tensorflow/python/ops/sets_impl.py | 11 +- tensorflow/python/ops/signal/BUILD | 17 +- tensorflow/python/ops/signal/__init__.py | 37 - tensorflow/python/ops/signal/dct_ops.py | 192 + .../{spectral_ops.py => signal/fft_ops.py} | 302 +- tensorflow/python/ops/signal/mfcc_ops.py | 4 +- .../python/ops/signal/reconstruction_ops.py | 163 +- tensorflow/python/ops/signal/shape_ops.py | 2 +- tensorflow/python/ops/signal/signal.py | 65 + tensorflow/python/ops/signal/spectral_ops.py | 8 +- tensorflow/python/ops/sort_ops.py | 197 + .../python/ops/sort_ops_test.py | 16 +- tensorflow/python/ops/sparse_grad.py | 2 +- tensorflow/python/ops/sparse_ops.py | 410 +- tensorflow/python/ops/sparse_ops_test.py | 22 +- tensorflow/python/ops/special_math_ops.py | 7 +- .../python/ops/special_math_ops_test.py | 7 + tensorflow/python/ops/spectral_grad.py | 185 - tensorflow/python/ops/standard_ops.py | 3 +- tensorflow/python/ops/stateless_random_ops.py | 58 +- tensorflow/python/ops/string_ops.py | 77 +- tensorflow/python/ops/summary_op_util.py | 21 +- tensorflow/python/ops/summary_ops_v2.py | 99 +- tensorflow/python/ops/tensor_array_ops.py | 295 +- tensorflow/python/ops/tensor_forest_ops.py | 110 + tensorflow/python/ops/variable_scope.py | 183 +- tensorflow/python/ops/variables.py | 149 +- tensorflow/python/ops/while_v2.py | 79 +- tensorflow/python/platform/__init__.py | 0 tensorflow/python/platform/app.py | 2 +- tensorflow/python/platform/benchmark.py | 13 + tensorflow/python/platform/gfile.py | 4 +- tensorflow/python/platform/googletest.py | 11 +- tensorflow/python/platform/test.py | 4 +- tensorflow/python/platform/tf_logging.py | 65 +- .../profiler/internal/run_metadata_test.py | 3 + tensorflow/python/profiler/model_analyzer.py | 6 +- .../python/profiler/model_analyzer_test.py | 30 +- tensorflow/python/profiler/option_builder.py | 2 +- .../python/profiler/pprof_profiler_test.py | 2 + .../python/profiler/profile_context_test.py | 24 +- tensorflow/python/profiler/profiler.py | 8 +- tensorflow/python/profiler/profiler_test.py | 3 + tensorflow/python/profiler/tfprof_logger.py | 2 +- tensorflow/python/saved_model/BUILD | 63 +- tensorflow/python/saved_model/builder.py | 1 + tensorflow/python/saved_model/builder_impl.py | 567 +- tensorflow/python/saved_model/constants.py | 12 +- tensorflow/python/saved_model/load.py | 61 + tensorflow/python/saved_model/load_test.py | 51 + tensorflow/python/saved_model/loader_impl.py | 113 +- tensorflow/python/saved_model/loader_test.py | 124 +- tensorflow/python/saved_model/save.py | 364 +- tensorflow/python/saved_model/save_test.py | 156 +- .../python/saved_model/saved_model_test.py | 740 +- .../saved_model/saved_object_graph.proto | 38 + .../python/saved_model/signature_constants.py | 2 +- .../python/saved_model/signature_def_utils.py | 2 + .../saved_model/signature_def_utils_impl.py | 51 +- .../saved_model/signature_def_utils_test.py | 27 + .../python/saved_model/simple_save_test.py | 6 +- tensorflow/python/saved_model/utils_impl.py | 21 + tensorflow/python/summary/summary.py | 18 +- tensorflow/python/summary/summary_test.py | 13 + .../python/summary/writer/writer_test.py | 22 +- tensorflow/python/tf2.py | 17 +- tensorflow/python/tools/BUILD | 2 + .../python/tools/api/generator/api_gen.bzl | 11 +- .../tools/api/generator/api_init_files.bzl | 7 +- .../tools/api/generator/api_init_files_v1.bzl | 2 + .../tools/api/generator/create_python_api.py | 9 +- .../python/tools/api/generator/doc_srcs.py | 6 +- tensorflow/python/tools/freeze_graph_test.py | 2 + tensorflow/python/tools/inspect_checkpoint.py | 2 +- .../tools/optimize_for_inference_test.py | 5 + tensorflow/python/tools/strip_unused_test.py | 4 +- tensorflow/python/training/adadelta.py | 2 +- tensorflow/python/training/adadelta_test.py | 6 +- tensorflow/python/training/adagrad.py | 2 +- tensorflow/python/training/adagrad_da.py | 2 +- tensorflow/python/training/adagrad_da_test.py | 30 +- tensorflow/python/training/adagrad_test.py | 58 +- tensorflow/python/training/adam.py | 2 +- tensorflow/python/training/adam_test.py | 49 +- .../python/training/basic_loops_test.py | 4 + .../training/basic_session_run_hooks.py | 8 +- .../training/basic_session_run_hooks_test.py | 209 +- .../training/checkpoint_management_test.py | 9 +- .../python/training/checkpoint_ops_test.py | 5 +- .../python/training/checkpoint_utils.py | 4 +- .../python/training/checkpointable/BUILD | 2 +- .../checkpointable/data_structures.py | 50 +- .../checkpointable/data_structures_test.py | 21 + .../training/checkpointable/tracking.py | 37 + .../python/training/checkpointable/util.py | 85 +- .../training/checkpointable/util_test.py | 16 +- tensorflow/python/training/coordinator.py | 2 +- tensorflow/python/training/device_setter.py | 2 +- .../python/training/device_setter_test.py | 11 + tensorflow/python/training/distribute.py | 1229 +- .../training/distribution_strategy_context.py | 190 +- tensorflow/python/training/evaluation.py | 2 +- tensorflow/python/training/ftrl.py | 2 +- tensorflow/python/training/ftrl_test.py | 56 +- .../python/training/gradient_descent.py | 2 +- .../python/training/gradient_descent_test.py | 78 +- tensorflow/python/training/input_test.py | 264 +- .../python/training/learning_rate_decay.py | 2 +- .../training/learning_rate_decay_test.py | 35 +- .../python/training/learning_rate_decay_v2.py | 2 +- .../training/learning_rate_decay_v2_test.py | 34 +- tensorflow/python/training/momentum.py | 2 +- tensorflow/python/training/momentum_test.py | 126 +- .../python/training/monitored_session.py | 20 +- .../python/training/monitored_session_test.py | 75 +- tensorflow/python/training/moving_averages.py | 7 +- .../python/training/moving_averages_test.py | 73 +- tensorflow/python/training/optimizer.py | 47 +- tensorflow/python/training/optimizer_test.py | 28 +- .../python/training/proximal_adagrad.py | 2 +- .../python/training/proximal_adagrad_test.py | 36 +- .../proximal_gradient_descent_test.py | 31 +- .../python/training/quantize_training_test.py | 6 +- .../python/training/queue_runner_test.py | 34 +- tensorflow/python/training/rmsprop.py | 2 +- tensorflow/python/training/rmsprop_test.py | 151 +- tensorflow/python/training/saver.py | 58 +- .../saver_large_partitioned_variable_test.py | 9 +- tensorflow/python/training/saver_test.py | 177 +- .../server_lib_multiple_containers_test.py | 2 + ...lib_same_variables_clear_container_test.py | 10 +- .../server_lib_same_variables_clear_test.py | 2 + ...server_lib_same_variables_no_clear_test.py | 2 + .../training/server_lib_sparse_job_test.py | 4 +- tensorflow/python/training/server_lib_test.py | 2 +- tensorflow/python/training/session_manager.py | 2 +- .../python/training/session_manager_test.py | 15 + .../python/training/session_run_hook.py | 6 +- .../python/training/slot_creator_test.py | 20 +- tensorflow/python/training/supervisor.py | 2 +- tensorflow/python/training/supervisor_test.py | 26 +- .../training/sync_replicas_optimizer.py | 9 +- .../python/training/training_ops_test.py | 57 +- .../python/training/training_util_test.py | 3 + .../python/training/warm_starting_util.py | 10 +- .../training/warm_starting_util_test.py | 82 +- tensorflow/python/util/deprecation.py | 19 +- tensorflow/python/util/deprecation_test.py | 16 + tensorflow/python/util/dispatch.py | 192 + tensorflow/python/util/dispatch_test.py | 120 + tensorflow/python/util/nest_test.py | 1 + tensorflow/python/util/py_checkpoint_reader.i | 1 - tensorflow/python/util/tf_export.py | 40 + tensorflow/python/util/tf_export_test.py | 20 + tensorflow/python/util/tf_should_use_test.py | 9 +- tensorflow/stream_executor/BUILD | 15 +- tensorflow/stream_executor/cuda/cuda_blas.cc | 122 +- tensorflow/stream_executor/cuda/cuda_dnn.cc | 247 +- tensorflow/stream_executor/cuda/cuda_dnn.h | 13 +- tensorflow/stream_executor/cuda/cuda_fft.cc | 81 +- .../stream_executor/cuda/cuda_gpu_executor.cc | 9 +- .../stream_executor/cuda/cuda_gpu_executor.h | 3 +- tensorflow/stream_executor/cuda/cuda_rng.cc | 36 + .../stream_executor/device_description.cc | 12 +- tensorflow/stream_executor/dnn.cc | 133 +- tensorflow/stream_executor/dnn.h | 272 +- tensorflow/stream_executor/dnn.proto | 103 + .../stream_executor/host/host_gpu_executor.cc | 9 +- .../stream_executor/host/host_gpu_executor.h | 3 +- tensorflow/stream_executor/stream.cc | 5 +- tensorflow/stream_executor/stream.h | 16 +- .../stream_executor_internal.cc | 19 +- .../stream_executor_internal.h | 4 +- tensorflow/tensorflow.bzl | 27 +- ...nsorflow.-config-proto.-experimental.pbtxt | 8 +- .../golden/v1/tensorflow.-config-proto.pbtxt | 6 + .../golden/v1/tensorflow.-gradient-tape.pbtxt | 8 + .../golden/v1/tensorflow.-tensor-spec.pbtxt | 33 + .../golden/v1/tensorflow.data.-dataset.pbtxt | 3 +- ...ow.data.-fixed-length-record-dataset.pbtxt | 7 +- .../golden/v1/tensorflow.data.-options.pbtxt | 39 +- .../tensorflow.data.-t-f-record-dataset.pbtxt | 6 +- .../tensorflow.data.-text-line-dataset.pbtxt | 7 +- ...rflow.data.experimental.-csv-dataset.pbtxt | 7 +- ...a.experimental.-optimization-options.pbtxt | 46 + ...ow.data.experimental.-random-dataset.pbtxt | 7 +- ...rflow.data.experimental.-sql-dataset.pbtxt | 7 +- ...low.data.experimental.-stats-options.pbtxt | 3 +- ...data.experimental.-threading-options.pbtxt | 18 + .../v1/tensorflow.data.experimental.pbtxt | 26 +- .../tools/api/golden/v1/tensorflow.data.pbtxt | 8 + .../api/golden/v1/tensorflow.debugging.pbtxt | 2 +- ...tensorflow.distribute.-input-context.pbtxt | 25 + ...w.distribute.-input-replication-mode.pbtxt | 8 + .../v1/tensorflow.distribute.-reduce-op.pbtxt | 12 + ...nsorflow.distribute.-replica-context.pbtxt | 33 + ...orflow.distribute.-strategy-extended.pbtxt | 81 + .../v1/tensorflow.distribute.-strategy.pbtxt | 137 + .../api/golden/v1/tensorflow.distribute.pbtxt | 47 + ...rflow.estimator.-baseline-classifier.pbtxt | 7 +- ...orflow.estimator.-baseline-estimator.pbtxt | 7 +- ...orflow.estimator.-baseline-regressor.pbtxt | 7 +- ....estimator.-boosted-trees-classifier.pbtxt | 7 +- ...w.estimator.-boosted-trees-regressor.pbtxt | 7 +- ...nsorflow.estimator.-d-n-n-classifier.pbtxt | 7 +- ...ensorflow.estimator.-d-n-n-estimator.pbtxt | 7 +- ...or.-d-n-n-linear-combined-classifier.pbtxt | 7 +- ...tor.-d-n-n-linear-combined-estimator.pbtxt | 7 +- ...tor.-d-n-n-linear-combined-regressor.pbtxt | 7 +- ...ensorflow.estimator.-d-n-n-regressor.pbtxt | 7 +- .../v1/tensorflow.estimator.-estimator.pbtxt | 7 +- ...sorflow.estimator.-linear-classifier.pbtxt | 7 +- ...nsorflow.estimator.-linear-estimator.pbtxt | 7 +- ...nsorflow.estimator.-linear-regressor.pbtxt | 7 +- ...erimental.-in-memory-evaluator-hook.pbtxt} | 6 +- .../tensorflow.estimator.experimental.pbtxt | 12 + .../api/golden/v1/tensorflow.image.pbtxt | 8 + .../api/golden/v1/tensorflow.io.gfile.pbtxt | 51 + .../tools/api/golden/v1/tensorflow.io.pbtxt | 40 + .../golden/v1/tensorflow.keras.-model.pbtxt | 26 +- .../v1/tensorflow.keras.-sequential.pbtxt | 26 +- .../golden/v1/tensorflow.keras.backend.pbtxt | 6 +- ....experimental.-peephole-l-s-t-m-cell.pbtxt | 4 + .../tensorflow.keras.layers.-activation.pbtxt | 4 + ...eras.layers.-activity-regularization.pbtxt | 4 + .../v1/tensorflow.keras.layers.-add.pbtxt | 4 + ...nsorflow.keras.layers.-alpha-dropout.pbtxt | 4 + ...low.keras.layers.-average-pooling1-d.pbtxt | 4 + ...low.keras.layers.-average-pooling2-d.pbtxt | 4 + ...low.keras.layers.-average-pooling3-d.pbtxt | 4 + .../v1/tensorflow.keras.layers.-average.pbtxt | 4 + ...tensorflow.keras.layers.-avg-pool1-d.pbtxt | 4 + ...tensorflow.keras.layers.-avg-pool2-d.pbtxt | 4 + ...tensorflow.keras.layers.-avg-pool3-d.pbtxt | 4 + ...ow.keras.layers.-batch-normalization.pbtxt | 7 +- ...nsorflow.keras.layers.-bidirectional.pbtxt | 4 + ...tensorflow.keras.layers.-concatenate.pbtxt | 4 + ...orflow.keras.layers.-conv-l-s-t-m2-d.pbtxt | 4 + .../v1/tensorflow.keras.layers.-conv1-d.pbtxt | 4 + ...flow.keras.layers.-conv2-d-transpose.pbtxt | 4 + .../v1/tensorflow.keras.layers.-conv2-d.pbtxt | 4 + ...flow.keras.layers.-conv3-d-transpose.pbtxt | 4 + .../v1/tensorflow.keras.layers.-conv3-d.pbtxt | 4 + ...sorflow.keras.layers.-convolution1-d.pbtxt | 4 + ...ras.layers.-convolution2-d-transpose.pbtxt | 4 + ...sorflow.keras.layers.-convolution2-d.pbtxt | 4 + ...ras.layers.-convolution3-d-transpose.pbtxt | 4 + ...sorflow.keras.layers.-convolution3-d.pbtxt | 4 + ...tensorflow.keras.layers.-cropping1-d.pbtxt | 4 + ...tensorflow.keras.layers.-cropping2-d.pbtxt | 4 + ...tensorflow.keras.layers.-cropping3-d.pbtxt | 4 + ...sorflow.keras.layers.-cu-d-n-n-g-r-u.pbtxt | 4 + ...rflow.keras.layers.-cu-d-n-n-l-s-t-m.pbtxt | 4 + .../v1/tensorflow.keras.layers.-dense.pbtxt | 4 + ...flow.keras.layers.-depthwise-conv2-d.pbtxt | 4 + .../v1/tensorflow.keras.layers.-dot.pbtxt | 4 + .../v1/tensorflow.keras.layers.-dropout.pbtxt | 4 + .../v1/tensorflow.keras.layers.-e-l-u.pbtxt | 4 + .../tensorflow.keras.layers.-embedding.pbtxt | 4 + .../v1/tensorflow.keras.layers.-flatten.pbtxt | 4 + .../tensorflow.keras.layers.-g-r-u-cell.pbtxt | 4 + .../v1/tensorflow.keras.layers.-g-r-u.pbtxt | 4 + ...rflow.keras.layers.-gaussian-dropout.pbtxt | 4 + ...sorflow.keras.layers.-gaussian-noise.pbtxt | 4 + ...as.layers.-global-average-pooling1-d.pbtxt | 4 + ...as.layers.-global-average-pooling2-d.pbtxt | 4 + ...as.layers.-global-average-pooling3-d.pbtxt | 4 + ...low.keras.layers.-global-avg-pool1-d.pbtxt | 4 + ...low.keras.layers.-global-avg-pool2-d.pbtxt | 4 + ...low.keras.layers.-global-avg-pool3-d.pbtxt | 4 + ...low.keras.layers.-global-max-pool1-d.pbtxt | 4 + ...low.keras.layers.-global-max-pool2-d.pbtxt | 4 + ...low.keras.layers.-global-max-pool3-d.pbtxt | 4 + ....keras.layers.-global-max-pooling1-d.pbtxt | 4 + ....keras.layers.-global-max-pooling2-d.pbtxt | 4 + ....keras.layers.-global-max-pooling3-d.pbtxt | 4 + ...tensorflow.keras.layers.-input-layer.pbtxt | 4 + .../tensorflow.keras.layers.-input-spec.pbtxt | 2 +- ...ensorflow.keras.layers.-l-s-t-m-cell.pbtxt | 4 + .../v1/tensorflow.keras.layers.-l-s-t-m.pbtxt | 4 + .../v1/tensorflow.keras.layers.-lambda.pbtxt | 4 + .../v1/tensorflow.keras.layers.-layer.pbtxt | 4 + ...ensorflow.keras.layers.-leaky-re-l-u.pbtxt | 4 + ...w.keras.layers.-locally-connected1-d.pbtxt | 4 + ...w.keras.layers.-locally-connected2-d.pbtxt | 4 + .../v1/tensorflow.keras.layers.-masking.pbtxt | 4 + ...tensorflow.keras.layers.-max-pool1-d.pbtxt | 4 + ...tensorflow.keras.layers.-max-pool2-d.pbtxt | 4 + ...tensorflow.keras.layers.-max-pool3-d.pbtxt | 4 + ...sorflow.keras.layers.-max-pooling1-d.pbtxt | 4 + ...sorflow.keras.layers.-max-pooling2-d.pbtxt | 4 + ...sorflow.keras.layers.-max-pooling3-d.pbtxt | 4 + .../v1/tensorflow.keras.layers.-maximum.pbtxt | 4 + .../v1/tensorflow.keras.layers.-minimum.pbtxt | 4 + .../tensorflow.keras.layers.-multiply.pbtxt | 4 + .../tensorflow.keras.layers.-p-re-l-u.pbtxt | 4 + .../v1/tensorflow.keras.layers.-permute.pbtxt | 4 + .../v1/tensorflow.keras.layers.-r-n-n.pbtxt | 4 + .../v1/tensorflow.keras.layers.-re-l-u.pbtxt | 4 + ...nsorflow.keras.layers.-repeat-vector.pbtxt | 4 + .../v1/tensorflow.keras.layers.-reshape.pbtxt | 4 + ...flow.keras.layers.-separable-conv1-d.pbtxt | 4 + ...flow.keras.layers.-separable-conv2-d.pbtxt | 4 + ...ras.layers.-separable-convolution1-d.pbtxt | 4 + ...ras.layers.-separable-convolution2-d.pbtxt | 4 + ...flow.keras.layers.-simple-r-n-n-cell.pbtxt | 4 + ...ensorflow.keras.layers.-simple-r-n-n.pbtxt | 4 + .../v1/tensorflow.keras.layers.-softmax.pbtxt | 4 + ...low.keras.layers.-spatial-dropout1-d.pbtxt | 4 + ...low.keras.layers.-spatial-dropout2-d.pbtxt | 4 + ...low.keras.layers.-spatial-dropout3-d.pbtxt | 4 + ...ow.keras.layers.-stacked-r-n-n-cells.pbtxt | 4 + .../tensorflow.keras.layers.-subtract.pbtxt | 4 + ...low.keras.layers.-thresholded-re-l-u.pbtxt | 4 + ...rflow.keras.layers.-time-distributed.pbtxt | 4 + ...sorflow.keras.layers.-up-sampling1-d.pbtxt | 4 + ...sorflow.keras.layers.-up-sampling2-d.pbtxt | 4 + ...sorflow.keras.layers.-up-sampling3-d.pbtxt | 4 + .../v1/tensorflow.keras.layers.-wrapper.pbtxt | 4 + ...orflow.keras.layers.-zero-padding1-d.pbtxt | 4 + ...orflow.keras.layers.-zero-padding2-d.pbtxt | 4 + ...orflow.keras.layers.-zero-padding3-d.pbtxt | 4 + ...ow.keras.losses.-binary-crossentropy.pbtxt | 22 + ...ras.losses.-categorical-crossentropy.pbtxt | 22 + ...ow.keras.losses.-mean-absolute-error.pbtxt | 22 + ...sses.-mean-absolute-percentage-error.pbtxt | 22 + ...low.keras.losses.-mean-squared-error.pbtxt | 22 + ...sses.-mean-squared-logarithmic-error.pbtxt | 22 + .../golden/v1/tensorflow.keras.losses.pbtxt | 30 +- .../tensorflow.keras.metrics.-accuracy.pbtxt | 194 + ...rflow.keras.metrics.-binary-accuracy.pbtxt | 194 + ....keras.metrics.-categorical-accuracy.pbtxt | 194 + ...rflow.keras.metrics.-false-negatives.pbtxt | 193 + ...rflow.keras.metrics.-false-positives.pbtxt | 193 + .../v1/tensorflow.keras.metrics.-mean.pbtxt | 192 + .../tensorflow.keras.metrics.-precision.pbtxt | 192 + .../v1/tensorflow.keras.metrics.-recall.pbtxt | 192 + ...metrics.-sparse-categorical-accuracy.pbtxt | 194 + ...orflow.keras.metrics.-true-negatives.pbtxt | 193 + ...orflow.keras.metrics.-true-positives.pbtxt | 193 + .../golden/v1/tensorflow.keras.metrics.pbtxt | 50 +- .../v1/tensorflow.keras.models.-model.pbtxt | 26 +- .../tensorflow.keras.models.-sequential.pbtxt | 26 +- .../v1/tensorflow.keras.utils.-progbar.pbtxt | 2 +- ...ensorflow.layers.-average-pooling1-d.pbtxt | 4 + ...ensorflow.layers.-average-pooling2-d.pbtxt | 4 + ...ensorflow.layers.-average-pooling3-d.pbtxt | 4 + ...nsorflow.layers.-batch-normalization.pbtxt | 7 +- .../v1/tensorflow.layers.-conv1-d.pbtxt | 4 + ...tensorflow.layers.-conv2-d-transpose.pbtxt | 4 + .../v1/tensorflow.layers.-conv2-d.pbtxt | 4 + ...tensorflow.layers.-conv3-d-transpose.pbtxt | 4 + .../v1/tensorflow.layers.-conv3-d.pbtxt | 4 + .../golden/v1/tensorflow.layers.-dense.pbtxt | 4 + .../v1/tensorflow.layers.-dropout.pbtxt | 4 + .../v1/tensorflow.layers.-flatten.pbtxt | 4 + .../v1/tensorflow.layers.-input-spec.pbtxt | 2 +- .../golden/v1/tensorflow.layers.-layer.pbtxt | 4 + .../tensorflow.layers.-max-pooling1-d.pbtxt | 4 + .../tensorflow.layers.-max-pooling2-d.pbtxt | 4 + .../tensorflow.layers.-max-pooling3-d.pbtxt | 4 + ...tensorflow.layers.-separable-conv1-d.pbtxt | 4 + ...tensorflow.layers.-separable-conv2-d.pbtxt | 4 + ...w.linalg.-linear-operator-block-diag.pbtxt | 4 + ...ow.linalg.-linear-operator-circulant.pbtxt | 4 + ...linalg.-linear-operator-circulant2-d.pbtxt | 4 + ...linalg.-linear-operator-circulant3-d.pbtxt | 4 + ....linalg.-linear-operator-composition.pbtxt | 4 + ...sorflow.linalg.-linear-operator-diag.pbtxt | 4 + ....linalg.-linear-operator-full-matrix.pbtxt | 4 + ...low.linalg.-linear-operator-identity.pbtxt | 4 + ...ow.linalg.-linear-operator-kronecker.pbtxt | 4 + ...alg.-linear-operator-low-rank-update.pbtxt | 4 + ...lg.-linear-operator-lower-triangular.pbtxt | 4 + ...alg.-linear-operator-scaled-identity.pbtxt | 4 + ...orflow.linalg.-linear-operator-zeros.pbtxt | 4 + .../tensorflow.linalg.-linear-operator.pbtxt | 4 + .../golden/v1/tensorflow.lite.constants.pbtxt | 10 +- .../tools/api/golden/v1/tensorflow.math.pbtxt | 26 +- .../v1/tensorflow.metrics.-accuracy.pbtxt | 194 + .../tensorflow.metrics.-binary-accuracy.pbtxt | 194 + ...orflow.metrics.-categorical-accuracy.pbtxt | 194 + .../tensorflow.metrics.-false-negatives.pbtxt | 193 + .../tensorflow.metrics.-false-positives.pbtxt | 193 + .../golden/v1/tensorflow.metrics.-mean.pbtxt | 192 + .../v1/tensorflow.metrics.-precision.pbtxt | 192 + .../v1/tensorflow.metrics.-recall.pbtxt | 192 + ...metrics.-sparse-categorical-accuracy.pbtxt | 194 + .../tensorflow.metrics.-true-negatives.pbtxt | 193 + .../tensorflow.metrics.-true-positives.pbtxt | 193 + .../api/golden/v1/tensorflow.metrics.pbtxt | 44 + .../tools/api/golden/v1/tensorflow.nn.pbtxt | 28 +- ...flow.nn.rnn_cell.-basic-l-s-t-m-cell.pbtxt | 4 + ...orflow.nn.rnn_cell.-basic-r-n-n-cell.pbtxt | 4 + ...nsorflow.nn.rnn_cell.-device-wrapper.pbtxt | 4 + ...sorflow.nn.rnn_cell.-dropout-wrapper.pbtxt | 4 + .../tensorflow.nn.rnn_cell.-g-r-u-cell.pbtxt | 4 + ...tensorflow.nn.rnn_cell.-l-s-t-m-cell.pbtxt | 4 + ...orflow.nn.rnn_cell.-multi-r-n-n-cell.pbtxt | 4 + .../tensorflow.nn.rnn_cell.-r-n-n-cell.pbtxt | 4 + ...orflow.nn.rnn_cell.-residual-wrapper.pbtxt | 4 + .../tools/api/golden/v1/tensorflow.pbtxt | 56 +- .../golden/v1/tensorflow.quantization.pbtxt | 2 +- .../api/golden/v1/tensorflow.random.pbtxt | 8 + .../v1/tensorflow.saved_model.-builder.pbtxt | 1 + ...d_model.builder.-saved-model-builder.pbtxt | 1 + .../golden/v1/tensorflow.saved_model.pbtxt | 4 + .../tools/api/golden/v1/tensorflow.sets.pbtxt | 16 + .../api/golden/v1/tensorflow.signal.pbtxt | 32 + .../api/golden/v1/tensorflow.sparse.pbtxt | 6 +- .../api/golden/v1/tensorflow.strings.pbtxt | 4 + .../v1/tensorflow.test.-benchmark.pbtxt | 4 + .../api/golden/v1/tensorflow.train.pbtxt | 4 + ...orflow.-conditional-accumulator-base.pbtxt | 29 - .../tensorflow.-conditional-accumulator.pbtxt | 38 - ...nsorflow.-config-proto.-experimental.pbtxt | 6 + .../golden/v2/tensorflow.-config-proto.pbtxt | 6 + .../golden/v2/tensorflow.-device-spec.pbtxt | 37 - .../api/golden/v2/tensorflow.-dimension.pbtxt | 25 - .../golden/v2/tensorflow.-gradient-tape.pbtxt | 8 + .../golden/v2/tensorflow.-graph-keys.pbtxt | 140 - .../tensorflow.-tensor-info.-coo-sparse.pbtxt | 24 - .../golden/v2/tensorflow.-tensor-info.pbtxt | 59 - .../golden/v2/tensorflow.-tensor-spec.pbtxt | 33 + .../tools/api/golden/v2/tensorflow.app.pbtxt | 7 - .../golden/v2/tensorflow.data.-dataset.pbtxt | 19 +- ...ow.data.-fixed-length-record-dataset.pbtxt | 20 +- .../golden/v2/tensorflow.data.-iterator.pbtxt | 46 - .../golden/v2/tensorflow.data.-options.pbtxt | 39 +- .../tensorflow.data.-t-f-record-dataset.pbtxt | 20 +- .../tensorflow.data.-text-line-dataset.pbtxt | 20 +- ...rflow.data.experimental.-csv-dataset.pbtxt | 20 +- ...a.experimental.-optimization-options.pbtxt | 46 + ...ow.data.experimental.-random-dataset.pbtxt | 20 +- ...rflow.data.experimental.-sql-dataset.pbtxt | 20 +- ...low.data.experimental.-stats-options.pbtxt | 3 +- ...data.experimental.-threading-options.pbtxt | 18 + .../v2/tensorflow.data.experimental.pbtxt | 26 +- .../tools/api/golden/v2/tensorflow.data.pbtxt | 4 - .../api/golden/v2/tensorflow.debugging.pbtxt | 52 +- ...tensorflow.distribute.-input-context.pbtxt | 25 + ...w.distribute.-input-replication-mode.pbtxt | 8 + .../v2/tensorflow.distribute.-reduce-op.pbtxt | 12 + ...nsorflow.distribute.-replica-context.pbtxt | 33 + ...orflow.distribute.-strategy-extended.pbtxt | 81 + .../v2/tensorflow.distribute.-strategy.pbtxt | 137 + .../api/golden/v2/tensorflow.distribute.pbtxt | 47 + ...rflow.estimator.-baseline-classifier.pbtxt | 14 +- ...orflow.estimator.-baseline-estimator.pbtxt | 12 +- ...orflow.estimator.-baseline-regressor.pbtxt | 14 +- ....estimator.-boosted-trees-classifier.pbtxt | 7 +- ...w.estimator.-boosted-trees-regressor.pbtxt | 7 +- ...nsorflow.estimator.-d-n-n-classifier.pbtxt | 14 +- ...ensorflow.estimator.-d-n-n-estimator.pbtxt | 12 +- ...or.-d-n-n-linear-combined-classifier.pbtxt | 14 +- ...tor.-d-n-n-linear-combined-estimator.pbtxt | 12 +- ...tor.-d-n-n-linear-combined-regressor.pbtxt | 14 +- ...ensorflow.estimator.-d-n-n-regressor.pbtxt | 14 +- .../v2/tensorflow.estimator.-estimator.pbtxt | 10 +- ...sorflow.estimator.-linear-classifier.pbtxt | 10 +- ...nsorflow.estimator.-linear-estimator.pbtxt | 12 +- ...nsorflow.estimator.-linear-regressor.pbtxt | 14 +- ...perimental.-in-memory-evaluator-hook.pbtxt | 30 + .../tensorflow.estimator.experimental.pbtxt | 12 + .../golden/v2/tensorflow.feature_column.pbtxt | 10 +- .../v2/tensorflow.gfile.-fast-g-file.pbtxt | 58 - .../golden/v2/tensorflow.gfile.-g-file.pbtxt | 58 - .../golden/v2/tensorflow.gfile.-open.pbtxt | 58 - .../api/golden/v2/tensorflow.gfile.pbtxt | 63 - .../api/golden/v2/tensorflow.image.pbtxt | 30 +- .../api/golden/v2/tensorflow.io.gfile.pbtxt | 51 + .../tools/api/golden/v2/tensorflow.io.pbtxt | 54 +- .../golden/v2/tensorflow.keras.-model.pbtxt | 26 +- .../v2/tensorflow.keras.-sequential.pbtxt | 26 +- .../golden/v2/tensorflow.keras.backend.pbtxt | 6 +- ....experimental.-peephole-l-s-t-m-cell.pbtxt | 4 + .../tensorflow.keras.layers.-activation.pbtxt | 4 + ...eras.layers.-activity-regularization.pbtxt | 4 + .../v2/tensorflow.keras.layers.-add.pbtxt | 4 + ...nsorflow.keras.layers.-alpha-dropout.pbtxt | 4 + ...low.keras.layers.-average-pooling1-d.pbtxt | 4 + ...low.keras.layers.-average-pooling2-d.pbtxt | 4 + ...low.keras.layers.-average-pooling3-d.pbtxt | 4 + .../v2/tensorflow.keras.layers.-average.pbtxt | 4 + ...tensorflow.keras.layers.-avg-pool1-d.pbtxt | 4 + ...tensorflow.keras.layers.-avg-pool2-d.pbtxt | 4 + ...tensorflow.keras.layers.-avg-pool3-d.pbtxt | 4 + ...ow.keras.layers.-batch-normalization.pbtxt | 6 +- ...nsorflow.keras.layers.-bidirectional.pbtxt | 4 + ...tensorflow.keras.layers.-concatenate.pbtxt | 4 + ...orflow.keras.layers.-conv-l-s-t-m2-d.pbtxt | 4 + .../v2/tensorflow.keras.layers.-conv1-d.pbtxt | 4 + ...flow.keras.layers.-conv2-d-transpose.pbtxt | 4 + .../v2/tensorflow.keras.layers.-conv2-d.pbtxt | 4 + ...flow.keras.layers.-conv3-d-transpose.pbtxt | 4 + .../v2/tensorflow.keras.layers.-conv3-d.pbtxt | 4 + ...sorflow.keras.layers.-convolution1-d.pbtxt | 4 + ...ras.layers.-convolution2-d-transpose.pbtxt | 4 + ...sorflow.keras.layers.-convolution2-d.pbtxt | 4 + ...ras.layers.-convolution3-d-transpose.pbtxt | 4 + ...sorflow.keras.layers.-convolution3-d.pbtxt | 4 + ...tensorflow.keras.layers.-cropping1-d.pbtxt | 4 + ...tensorflow.keras.layers.-cropping2-d.pbtxt | 4 + ...tensorflow.keras.layers.-cropping3-d.pbtxt | 4 + ...sorflow.keras.layers.-cu-d-n-n-g-r-u.pbtxt | 4 + ...rflow.keras.layers.-cu-d-n-n-l-s-t-m.pbtxt | 4 + ...orflow.keras.layers.-dense-features.pbtxt} | 40 +- .../v2/tensorflow.keras.layers.-dense.pbtxt | 4 + ...flow.keras.layers.-depthwise-conv2-d.pbtxt | 4 + .../v2/tensorflow.keras.layers.-dot.pbtxt | 4 + .../v2/tensorflow.keras.layers.-dropout.pbtxt | 4 + .../v2/tensorflow.keras.layers.-e-l-u.pbtxt | 4 + .../tensorflow.keras.layers.-embedding.pbtxt | 4 + .../v2/tensorflow.keras.layers.-flatten.pbtxt | 4 + .../tensorflow.keras.layers.-g-r-u-cell.pbtxt | 4 + .../v2/tensorflow.keras.layers.-g-r-u.pbtxt | 4 + ...rflow.keras.layers.-gaussian-dropout.pbtxt | 4 + ...sorflow.keras.layers.-gaussian-noise.pbtxt | 4 + ...as.layers.-global-average-pooling1-d.pbtxt | 4 + ...as.layers.-global-average-pooling2-d.pbtxt | 4 + ...as.layers.-global-average-pooling3-d.pbtxt | 4 + ...low.keras.layers.-global-avg-pool1-d.pbtxt | 4 + ...low.keras.layers.-global-avg-pool2-d.pbtxt | 4 + ...low.keras.layers.-global-avg-pool3-d.pbtxt | 4 + ...low.keras.layers.-global-max-pool1-d.pbtxt | 4 + ...low.keras.layers.-global-max-pool2-d.pbtxt | 4 + ...low.keras.layers.-global-max-pool3-d.pbtxt | 4 + ....keras.layers.-global-max-pooling1-d.pbtxt | 4 + ....keras.layers.-global-max-pooling2-d.pbtxt | 4 + ....keras.layers.-global-max-pooling3-d.pbtxt | 4 + ...tensorflow.keras.layers.-input-layer.pbtxt | 4 + .../tensorflow.keras.layers.-input-spec.pbtxt | 2 +- ...ensorflow.keras.layers.-l-s-t-m-cell.pbtxt | 4 + .../v2/tensorflow.keras.layers.-l-s-t-m.pbtxt | 4 + .../v2/tensorflow.keras.layers.-lambda.pbtxt | 4 + .../v2/tensorflow.keras.layers.-layer.pbtxt | 4 + ...ensorflow.keras.layers.-leaky-re-l-u.pbtxt | 4 + ...ensorflow.keras.layers.-linear-model.pbtxt | 289 + ...w.keras.layers.-locally-connected1-d.pbtxt | 4 + ...w.keras.layers.-locally-connected2-d.pbtxt | 4 + .../v2/tensorflow.keras.layers.-masking.pbtxt | 4 + ...tensorflow.keras.layers.-max-pool1-d.pbtxt | 4 + ...tensorflow.keras.layers.-max-pool2-d.pbtxt | 4 + ...tensorflow.keras.layers.-max-pool3-d.pbtxt | 4 + ...sorflow.keras.layers.-max-pooling1-d.pbtxt | 4 + ...sorflow.keras.layers.-max-pooling2-d.pbtxt | 4 + ...sorflow.keras.layers.-max-pooling3-d.pbtxt | 4 + .../v2/tensorflow.keras.layers.-maximum.pbtxt | 4 + .../v2/tensorflow.keras.layers.-minimum.pbtxt | 4 + .../tensorflow.keras.layers.-multiply.pbtxt | 4 + .../tensorflow.keras.layers.-p-re-l-u.pbtxt | 4 + .../v2/tensorflow.keras.layers.-permute.pbtxt | 4 + .../v2/tensorflow.keras.layers.-r-n-n.pbtxt | 4 + .../v2/tensorflow.keras.layers.-re-l-u.pbtxt | 4 + ...nsorflow.keras.layers.-repeat-vector.pbtxt | 4 + .../v2/tensorflow.keras.layers.-reshape.pbtxt | 4 + ...flow.keras.layers.-separable-conv1-d.pbtxt | 4 + ...flow.keras.layers.-separable-conv2-d.pbtxt | 4 + ...ras.layers.-separable-convolution1-d.pbtxt | 4 + ...ras.layers.-separable-convolution2-d.pbtxt | 4 + ...flow.keras.layers.-simple-r-n-n-cell.pbtxt | 4 + ...ensorflow.keras.layers.-simple-r-n-n.pbtxt | 4 + .../v2/tensorflow.keras.layers.-softmax.pbtxt | 4 + ...low.keras.layers.-spatial-dropout1-d.pbtxt | 4 + ...low.keras.layers.-spatial-dropout2-d.pbtxt | 4 + ...low.keras.layers.-spatial-dropout3-d.pbtxt | 4 + ...ow.keras.layers.-stacked-r-n-n-cells.pbtxt | 4 + .../tensorflow.keras.layers.-subtract.pbtxt | 4 + ...low.keras.layers.-thresholded-re-l-u.pbtxt | 4 + ...rflow.keras.layers.-time-distributed.pbtxt | 4 + ...sorflow.keras.layers.-up-sampling1-d.pbtxt | 4 + ...sorflow.keras.layers.-up-sampling2-d.pbtxt | 4 + ...sorflow.keras.layers.-up-sampling3-d.pbtxt | 4 + .../v2/tensorflow.keras.layers.-wrapper.pbtxt | 4 + ...orflow.keras.layers.-zero-padding1-d.pbtxt | 4 + ...orflow.keras.layers.-zero-padding2-d.pbtxt | 4 + ...orflow.keras.layers.-zero-padding3-d.pbtxt | 4 + .../golden/v2/tensorflow.keras.layers.pbtxt | 8 + ...ow.keras.losses.-binary-crossentropy.pbtxt | 22 + ...ras.losses.-categorical-crossentropy.pbtxt | 22 + ...ow.keras.losses.-mean-absolute-error.pbtxt | 22 + ...sses.-mean-absolute-percentage-error.pbtxt | 22 + ...low.keras.losses.-mean-squared-error.pbtxt | 22 + ...sses.-mean-squared-logarithmic-error.pbtxt | 22 + .../tensorflow.keras.losses.-reduction.pbtxt | 28 + .../golden/v2/tensorflow.keras.losses.pbtxt | 34 +- .../tensorflow.keras.metrics.-accuracy.pbtxt | 194 + ...rflow.keras.metrics.-binary-accuracy.pbtxt | 194 + ....keras.metrics.-categorical-accuracy.pbtxt | 194 + ...rflow.keras.metrics.-false-negatives.pbtxt | 193 + ...rflow.keras.metrics.-false-positives.pbtxt | 193 + .../v2/tensorflow.keras.metrics.-mean.pbtxt | 192 + .../tensorflow.keras.metrics.-precision.pbtxt | 192 + .../v2/tensorflow.keras.metrics.-recall.pbtxt | 192 + ...metrics.-sparse-categorical-accuracy.pbtxt | 194 + ...orflow.keras.metrics.-true-negatives.pbtxt | 193 + ...orflow.keras.metrics.-true-positives.pbtxt | 193 + .../golden/v2/tensorflow.keras.metrics.pbtxt | 50 +- .../v2/tensorflow.keras.models.-model.pbtxt | 26 +- .../tensorflow.keras.models.-sequential.pbtxt | 26 +- .../v2/tensorflow.keras.utils.-progbar.pbtxt | 2 +- ...w.linalg.-linear-operator-block-diag.pbtxt | 4 + ...ow.linalg.-linear-operator-circulant.pbtxt | 4 + ...linalg.-linear-operator-circulant2-d.pbtxt | 4 + ...linalg.-linear-operator-circulant3-d.pbtxt | 4 + ....linalg.-linear-operator-composition.pbtxt | 4 + ...sorflow.linalg.-linear-operator-diag.pbtxt | 4 + ....linalg.-linear-operator-full-matrix.pbtxt | 4 + ...low.linalg.-linear-operator-identity.pbtxt | 4 + ...ow.linalg.-linear-operator-kronecker.pbtxt | 4 + ...alg.-linear-operator-low-rank-update.pbtxt | 4 + ...lg.-linear-operator-lower-triangular.pbtxt | 4 + ...alg.-linear-operator-scaled-identity.pbtxt | 4 + ...orflow.linalg.-linear-operator-zeros.pbtxt | 4 + .../tensorflow.linalg.-linear-operator.pbtxt | 4 + .../api/golden/v2/tensorflow.linalg.pbtxt | 4 +- .../golden/v2/tensorflow.lite.constants.pbtxt | 20 - .../api/golden/v2/tensorflow.logging.pbtxt | 83 - .../v2/tensorflow.losses.-reduction.pbtxt | 14 +- .../api/golden/v2/tensorflow.losses.pbtxt | 44 - .../tools/api/golden/v2/tensorflow.math.pbtxt | 54 +- .../v2/tensorflow.metrics.-accuracy.pbtxt | 194 + .../tensorflow.metrics.-binary-accuracy.pbtxt | 194 + ...orflow.metrics.-categorical-accuracy.pbtxt | 194 + .../tensorflow.metrics.-false-negatives.pbtxt | 193 + .../tensorflow.metrics.-false-positives.pbtxt | 193 + .../golden/v2/tensorflow.metrics.-mean.pbtxt | 192 + .../v2/tensorflow.metrics.-precision.pbtxt | 192 + .../v2/tensorflow.metrics.-recall.pbtxt | 192 + ...metrics.-sparse-categorical-accuracy.pbtxt | 194 + .../tensorflow.metrics.-true-negatives.pbtxt | 193 + .../tensorflow.metrics.-true-positives.pbtxt | 193 + .../api/golden/v2/tensorflow.metrics.pbtxt | 154 +- .../tools/api/golden/v2/tensorflow.nn.pbtxt | 102 +- ...nsorflow.nn.rnn_cell.-device-wrapper.pbtxt | 4 + ...sorflow.nn.rnn_cell.-dropout-wrapper.pbtxt | 4 + .../tensorflow.nn.rnn_cell.-r-n-n-cell.pbtxt | 4 + ...orflow.nn.rnn_cell.-residual-wrapper.pbtxt | 4 + .../golden/v2/tensorflow.nn.rnn_cell.pbtxt | 4 - .../tools/api/golden/v2/tensorflow.pbtxt | 258 +- ...flow.profiler.-advice-proto.-checker.pbtxt | 12 - ...ofiler.-advice-proto.-checkers-entry.pbtxt | 22 - .../tensorflow.profiler.-advice-proto.pbtxt | 41 - ...graph-node-proto.-input-shapes-entry.pbtxt | 22 - ...ensorflow.profiler.-graph-node-proto.pbtxt | 191 - ...low.profiler.-multi-graph-node-proto.pbtxt | 134 - ...er.-op-log-proto.-id-to-string-entry.pbtxt | 21 - .../tensorflow.profiler.-op-log-proto.pbtxt | 38 - ...low.profiler.-profile-option-builder.pbtxt | 93 - .../v2/tensorflow.profiler.-profiler.pbtxt | 37 - .../api/golden/v2/tensorflow.profiler.pbtxt | 39 - .../golden/v2/tensorflow.quantization.pbtxt | 2 +- .../api/golden/v2/tensorflow.random.pbtxt | 28 +- .../golden/v2/tensorflow.saved_model.pbtxt | 16 +- .../tools/api/golden/v2/tensorflow.sets.pbtxt | 8 +- .../api/golden/v2/tensorflow.signal.pbtxt | 32 + .../api/golden/v2/tensorflow.sparse.pbtxt | 36 +- .../api/golden/v2/tensorflow.spectral.pbtxt | 35 - .../api/golden/v2/tensorflow.strings.pbtxt | 14 +- .../tensorflow.summary.-summary-writer.pbtxt | 29 + .../api/golden/v2/tensorflow.summary.pbtxt | 40 +- .../v2/tensorflow.test.-benchmark.pbtxt | 4 + .../tools/api/golden/v2/tensorflow.test.pbtxt | 14 +- ...tensorflow.train.-adadelta-optimizer.pbtxt | 51 - ...sorflow.train.-adagrad-d-a-optimizer.pbtxt | 51 - .../tensorflow.train.-adagrad-optimizer.pbtxt | 51 - .../v2/tensorflow.train.-adam-optimizer.pbtxt | 51 - ...low.train.-checkpoint-saver-listener.pbtxt | 24 - ...sorflow.train.-chief-session-creator.pbtxt | 14 - .../v2/tensorflow.train.-ftrl-optimizer.pbtxt | 51 - ...ow.train.-gradient-descent-optimizer.pbtxt | 51 - .../v2/tensorflow.train.-looper-thread.pbtxt | 73 - ...tensorflow.train.-momentum-optimizer.pbtxt | 51 - ...ain.-monitored-session.-step-context.pbtxt | 21 - .../tensorflow.train.-monitored-session.pbtxt | 34 - ...rain.-nan-loss-during-training-error.pbtxt | 12 - .../v2/tensorflow.train.-optimizer.pbtxt | 50 - ...ow.train.-proximal-adagrad-optimizer.pbtxt | 51 - ...nsorflow.train.-r-m-s-prop-optimizer.pbtxt | 51 - .../v2/tensorflow.train.-scaffold.pbtxt | 53 - ...nsorflow.train.-second-or-step-timer.pbtxt | 26 - .../tensorflow.train.-session-creator.pbtxt | 12 - .../tensorflow.train.-session-manager.pbtxt | 21 - .../tensorflow.train.-session-run-args.pbtxt | 27 - ...ensorflow.train.-session-run-context.pbtxt | 25 - ...tensorflow.train.-session-run-values.pbtxt | 27 - ...ular-monitored-session.-step-context.pbtxt | 21 - ...ow.train.-singular-monitored-session.pbtxt | 38 - .../v2/tensorflow.train.-supervisor.pbtxt | 153 - .../v2/tensorflow.train.-vocab-info.pbtxt | 43 - ...orflow.train.-worker-session-creator.pbtxt | 14 - .../api/golden/v2/tensorflow.train.pbtxt | 126 +- .../tools/api/tests/api_compatibility_test.py | 9 +- ...Dockerfile.rbe.cuda10.0-cudnn7-ubuntu14.04 | 75 + .../tools/ci_build/builds/libtensorflow.sh | 1 + .../windows/cpu/pip/build_tf_windows.sh | 31 +- .../windows/gpu/pip/build_tf_windows.sh | 26 +- tensorflow/tools/compatibility/BUILD | 46 +- tensorflow/tools/compatibility/README.md | 63 +- tensorflow/tools/compatibility/ast_edits.py | 60 +- .../tools/compatibility/ast_edits_test.py | 420 + tensorflow/tools/compatibility/renames_v2.py | 303 +- .../compatibility/testdata/test_file_v1_12.py | 71 + .../tools/compatibility/tf_upgrade_v2.py | 893 +- .../tools/compatibility/tf_upgrade_v2_main.py | 104 + .../tools/compatibility/tf_upgrade_v2_test.py | 369 +- tensorflow/tools/compatibility/update/BUILD | 1 + .../update/generate_v2_renames_map.py | 92 +- tensorflow/tools/docker/Dockerfile | 4 +- tensorflow/tools/docker/Dockerfile.devel | 4 +- tensorflow/tools/docker/Dockerfile.devel-mkl | 4 +- .../tools/docker/Dockerfile.devel-mkl-horovod | 4 +- tensorflow/tools/docker/Dockerfile.mkl | 4 +- .../tools/docker/Dockerfile.mkl-horovod | 4 +- tensorflow/tools/dockerfiles/.gitignore | 1 + tensorflow/tools/dockerfiles/README.md | 49 +- tensorflow/tools/dockerfiles/assembler.py | 919 +- .../dockerfiles/cpu-devel-jupyter.Dockerfile | 64 +- .../dockerfiles/cpu-devel.Dockerfile | 46 +- .../dockerfiles/cpu-jupyter.Dockerfile | 53 +- .../dockerfiles/dockerfiles/cpu.Dockerfile | 35 +- ...ockerfile => gpu-devel-jupyter.Dockerfile} | 76 +- ...-devel.Dockerfile => gpu-devel.Dockerfile} | 58 +- ...yter.Dockerfile => gpu-jupyter.Dockerfile} | 62 +- .../{nvidia.Dockerfile => gpu.Dockerfile} | 44 +- .../partials/jupyter.partial.Dockerfile | 16 +- .../partials/tensorflow.partial.Dockerfile | 7 +- .../partials/test-import.partial.Dockerfile | 0 .../partials/ubuntu.partial.Dockerfile | 2 - .../{ => ubuntu}/bazel.partial.Dockerfile | 14 + .../cpu-devel.partial.Dockerfile} | 7 +- .../partials/ubuntu/cpu.partial.Dockerfile | 1 + .../nvidia-devel.partial.Dockerfile | 18 +- .../{ => ubuntu}/nvidia.partial.Dockerfile | 8 +- .../{ => ubuntu}/python.partial.Dockerfile | 5 +- .../ubuntu/test-devel.partial.Dockerfile | 0 .../ubuntu/version.partial.Dockerfile | 1 + .../tools/dockerfiles/readme-for-jupyter.md | 3 + tensorflow/tools/dockerfiles/spec.yml | 306 +- .../tools/dockerfiles/tests/build-cpu.sh | 37 + .../tools/dockerfiles/tests/build-gpu.sh | 36 + .../tools/dockerfiles/tests/import-gpu.sh | 18 + tensorflow/tools/dockerfiles/tests/import.sh | 19 + ...{assembler.Dockerfile => tools.Dockerfile} | 5 +- tensorflow/tools/docs/BUILD | 27 +- tensorflow/tools/docs/generate2.py | 82 + .../docs/generate2_test.py} | 29 +- tensorflow/tools/docs/generate_1_0.py | 92 - tensorflow/tools/docs/parser.py | 2 +- tensorflow/tools/lib_package/BUILD | 2 + tensorflow/tools/pip_package/BUILD | 12 +- tensorflow/tools/pip_package/setup.py | 4 +- tensorflow/workspace.bzl | 73 +- .../clang_toolchain/download_clang.bzl | 8 +- third_party/eigen_reshaped.patch | 48 - third_party/googleapis.BUILD | 14 +- third_party/gpus/crosstool/CROSSTOOL.tpl | 3 +- third_party/gpus/cuda_configure.bzl | 9 + third_party/gpus/rocm_configure.bzl | 2 +- third_party/icu/data/BUILD.bazel | 46 + third_party/icu/data/LICENSE | 414 + .../icu/data/icu_conversion_data.c.gz.aa | Bin 0 -> 177696 bytes .../icu/data/icu_conversion_data.c.gz.ab | Bin 0 -> 177696 bytes .../icu/data/icu_conversion_data.c.gz.ac | Bin 0 -> 177696 bytes .../icu/data/icu_conversion_data.c.gz.ad | Bin 0 -> 177696 bytes .../icu/data/icu_conversion_data.c.gz.ae | Bin 0 -> 177696 bytes .../icu/data/icu_conversion_data.c.gz.af | Bin 0 -> 177696 bytes .../icu/data/icu_conversion_data.c.gz.ag | Bin 0 -> 177696 bytes .../icu/data/icu_conversion_data.c.gz.ah | Bin 0 -> 177696 bytes .../icu/data/icu_conversion_data.c.gz.ai | Bin 0 -> 177696 bytes .../icu/data/icu_conversion_data.c.gz.aj | Bin 0 -> 177689 bytes third_party/icu/udata.patch | 53 + third_party/icu/workspace.bzl | 6 + third_party/libxsmm.BUILD | 2 +- third_party/llvm/llvm.autogenerated.BUILD | 2 + third_party/llvm/llvm.bzl | 1 + third_party/mkl_dnn/mkldnn.BUILD | 2 +- third_party/nccl/archive.BUILD | 26 +- third_party/ngraph/ngraph.BUILD | 26 +- third_party/ngraph/ngraph_tf.BUILD | 40 +- third_party/ngraph/tbb.BUILD | 10 +- third_party/png.BUILD | 2 +- third_party/repo.bzl | 4 +- third_party/toolchains/BUILD | 13 + .../preconfig/generate/containers.bzl | 2 +- .../ubuntu14.04/cuda10.0-cudnn7/WORKSPACE | 2 + .../ubuntu14.04/cuda10.0-cudnn7/cuda/BUILD | 1275 + .../cuda10.0-cudnn7/cuda/build_defs.bzl | 31 + .../cuda10.0-cudnn7/cuda/cuda/cuda_config.h | 26 + .../ubuntu14.04/cuda9.0-cudnn7/cuda/BUILD | 10 +- .../cuda9.0-cudnn7/cuda/build_defs.bzl | 6 +- .../ubuntu14.04/gcc-nvcc-cuda10.0/BUILD | 87 + .../ubuntu14.04/gcc-nvcc-cuda10.0/CROSSTOOL | 1431 + .../bin/crosstool_wrapper_driver_is_not_gcc | 264 + .../windows/msvc_wrapper_for_nvcc.bat | 20 + .../windows/msvc_wrapper_for_nvcc.py | 192 + .../ubuntu14.04/gcc-nvcc-cuda9.0/BUILD | 87 + .../ubuntu14.04/gcc-nvcc-cuda9.0/CROSSTOOL | 1431 + .../bin/crosstool_wrapper_driver_is_not_gcc | 264 + .../windows/msvc_wrapper_for_nvcc.bat | 20 + .../windows/msvc_wrapper_for_nvcc.py | 192 + .../preconfig/ubuntu14.04/py3/BUILD | 4 +- .../toolchains/preconfig/win_1803/BUILD | 2 +- tools/bazel.rc | 1 + 3376 files changed, 170761 insertions(+), 65412 deletions(-) create mode 100644 .github/ISSUE_TEMPLATE/40-tflite-op-request.md create mode 100644 tensorflow/c/kernels.cc create mode 100644 tensorflow/c/kernels.h create mode 100644 tensorflow/c/kernels_test.cc create mode 100644 tensorflow/cc/saved_model/testdata/half_plus_two_v2/00000123/assets/foo.txt create mode 100644 tensorflow/cc/saved_model/testdata/half_plus_two_v2/00000123/saved_model.pb create mode 100644 tensorflow/cc/saved_model/testdata/half_plus_two_v2/00000123/variables/variables.data-00000-of-00001 create mode 100644 tensorflow/cc/saved_model/testdata/half_plus_two_v2/00000123/variables/variables.index create mode 100644 tensorflow/compiler/jit/flags.cc rename tensorflow/compiler/jit/{legacy_flags/mark_for_compilation_pass_flags.h => flags.h} (57%) delete mode 100644 tensorflow/compiler/jit/legacy_flags/BUILD delete mode 100644 tensorflow/compiler/jit/legacy_flags/build_xla_ops_pass_flags.cc delete mode 100644 tensorflow/compiler/jit/legacy_flags/build_xla_ops_pass_flags.h delete mode 100644 tensorflow/compiler/jit/legacy_flags/mark_for_compilation_pass_flags.cc delete mode 100644 tensorflow/compiler/jit/legacy_flags/xla_device_flags.cc delete mode 100644 tensorflow/compiler/jit/legacy_flags/xla_device_flags.h delete mode 100644 tensorflow/compiler/jit/legacy_flags/xla_ops_common_flags.cc delete mode 100644 tensorflow/compiler/jit/legacy_flags/xla_ops_common_flags.h rename tensorflow/{contrib/estimator/python/estimator/dnn.py => compiler/jit/ops/xla_ops_grad.py} (62%) delete mode 100644 tensorflow/compiler/tf2xla/dump_graph_flags.cc delete mode 100644 tensorflow/compiler/tf2xla/dump_graph_flags.h delete mode 100644 tensorflow/compiler/tf2xla/lib/batch_dot.cc delete mode 100644 tensorflow/compiler/tf2xla/lib/batch_dot.h create mode 100644 tensorflow/compiler/xla/client/lib/matrix.cc rename tensorflow/compiler/xla/client/lib/{numeric.h => matrix.h} (56%) rename tensorflow/compiler/xla/client/lib/{numeric_test.cc => matrix_test.cc} (53%) delete mode 100644 tensorflow/compiler/xla/client/lib/numeric.cc create mode 100644 tensorflow/compiler/xla/client/lib/slicing.cc create mode 100644 tensorflow/compiler/xla/client/lib/slicing.h rename tensorflow/compiler/{tf2xla/lib/util_test.cc => xla/client/lib/slicing_test.cc} (67%) create mode 100644 tensorflow/compiler/xla/g3doc/images/xla_array_layout_figure1.png create mode 100644 tensorflow/compiler/xla/g3doc/images/xla_array_layout_figure2.png create mode 100644 tensorflow/compiler/xla/g3doc/layout_with_tiling.md create mode 100644 tensorflow/compiler/xla/service/algebraic_simplifier_proof_distributive_property.py create mode 100644 tensorflow/compiler/xla/service/ar_crs_combiner.cc create mode 100644 tensorflow/compiler/xla/service/ar_crs_combiner.h create mode 100644 tensorflow/compiler/xla/service/ar_crs_combiner_test.cc create mode 100644 tensorflow/compiler/xla/service/dynamic_parameter_binding.cc create mode 100644 tensorflow/compiler/xla/service/dynamic_parameter_binding.h create mode 100644 tensorflow/compiler/xla/service/dynamic_parameter_binding_test.cc create mode 100644 tensorflow/compiler/xla/service/hlo_get_dimension_size_rewriter.cc create mode 100644 tensorflow/compiler/xla/service/hlo_get_dimension_size_rewriter.h create mode 100644 tensorflow/compiler/xla/service/hlo_get_dimension_size_rewriter_test.cc create mode 100644 tensorflow/compiler/xla/service/pattern_matcher_gmock.h create mode 100644 tensorflow/compiler/xla/service/pattern_matcher_gmock_test.cc create mode 100644 tensorflow/compiler/xla/shape.cc create mode 100644 tensorflow/compiler/xla/shape.h create mode 100644 tensorflow/compiler/xla/shape_test.cc create mode 100644 tensorflow/compiler/xla/tests/conv_depthwise_test.cc create mode 100644 tensorflow/compiler/xla/tests/grouped_convolution_test.cc create mode 100644 tensorflow/compiler/xrt/xrt_util.cc create mode 100644 tensorflow/compiler/xrt/xrt_util.h create mode 100644 tensorflow/contrib/autograph/examples/benchmarks/BUILD create mode 100644 tensorflow/contrib/autograph/examples/benchmarks/benchmark_base.py create mode 100644 tensorflow/contrib/autograph/examples/benchmarks/cartpole_benchmark.py create mode 100644 tensorflow/contrib/cluster_resolver/cluster_resolver_initialization_test.py create mode 100644 tensorflow/contrib/cmake/TensorflowConfig.cmake.in create mode 100644 tensorflow/contrib/cmake/TensorflowConfigVersion.cmake.in create mode 100644 tensorflow/contrib/cmake/tf_core_eager_runtime.cmake rename tensorflow/contrib/distribute/python/{cross_tower_ops_test.py => cross_device_ops_test.py} (79%) rename tensorflow/contrib/distribute/python/{cross_tower_utils_test.py => cross_device_utils_test.py} (83%) delete mode 100644 tensorflow/contrib/distribute/python/mirrored_strategy_test.py delete mode 100644 tensorflow/contrib/estimator/python/estimator/dnn_linear_combined.py rename tensorflow/{compiler/tests/resampler_ops_test.py => contrib/resampler/xla/resampler_ops_xla_test.py} (76%) create mode 100644 tensorflow/contrib/tensorrt/test/quantization_mnist_test.py create mode 100644 tensorflow/contrib/tensorrt/test/quantization_test.py create mode 100644 tensorflow/contrib/tensorrt/test/testdata/checkpoint create mode 100644 tensorflow/contrib/tensorrt/test/testdata/model.ckpt-46900.data-00000-of-00001 create mode 100644 tensorflow/contrib/tensorrt/test/testdata/model.ckpt-46900.index delete mode 100644 tensorflow/contrib/training/python/training/tensor_queue_dataset.py delete mode 100644 tensorflow/contrib/training/python/training/tensor_queue_dataset_test.py delete mode 100644 tensorflow/core/api_def/base_api/api_def_EnqueueInQueueDataset.pbtxt rename tensorflow/core/api_def/base_api/{api_def_BytesProducedStatsDataset.pbtxt => api_def_ExperimentalBytesProducedStatsDataset.pbtxt} (56%) create mode 100644 tensorflow/core/api_def/base_api/api_def_ExperimentalDatasetCardinality.pbtxt rename tensorflow/core/api_def/base_api/{api_def_DatasetToTFRecord.pbtxt => api_def_ExperimentalDatasetToTFRecord.pbtxt} (91%) rename tensorflow/core/api_def/base_api/{api_def_DenseToSparseBatchDataset.pbtxt => api_def_ExperimentalDenseToSparseBatchDataset.pbtxt} (89%) delete mode 100644 tensorflow/core/api_def/base_api/api_def_ExperimentalFunctionBufferingResource.pbtxt delete mode 100644 tensorflow/core/api_def/base_api/api_def_ExperimentalFunctionBufferingResourceGetNext.pbtxt delete mode 100644 tensorflow/core/api_def/base_api/api_def_ExperimentalFunctionBufferingResourceReset.pbtxt rename tensorflow/core/api_def/base_api/{api_def_GroupByReducerDataset.pbtxt => api_def_ExperimentalGroupByReducerDataset.pbtxt} (97%) rename tensorflow/core/api_def/base_api/{api_def_GroupByWindowDataset.pbtxt => api_def_ExperimentalGroupByWindowDataset.pbtxt} (82%) rename tensorflow/core/api_def/base_api/{api_def_LatencyStatsDataset.pbtxt => api_def_ExperimentalLatencyStatsDataset.pbtxt} (58%) rename tensorflow/core/api_def/base_api/{api_def_MapAndBatchDatasetV2.pbtxt => api_def_ExperimentalMapAndBatchDataset.pbtxt} (96%) create mode 100644 tensorflow/core/api_def/base_api/api_def_ExperimentalMatchingFilesDataset.pbtxt create mode 100644 tensorflow/core/api_def/base_api/api_def_ExperimentalMaxIntraOpParallelismDataset.pbtxt rename tensorflow/core/api_def/base_api/{api_def_ParallelInterleaveDataset.pbtxt => api_def_ExperimentalParallelInterleaveDataset.pbtxt} (90%) rename tensorflow/core/api_def/base_api/{api_def_ParseExampleDataset.pbtxt => api_def_ExperimentalParseExampleDataset.pbtxt} (96%) create mode 100644 tensorflow/core/api_def/base_api/api_def_ExperimentalPrivateThreadPoolDataset.pbtxt rename tensorflow/core/api_def/base_api/{api_def_RandomDataset.pbtxt => api_def_ExperimentalRandomDataset.pbtxt} (86%) rename tensorflow/core/api_def/base_api/{api_def_ScanDataset.pbtxt => api_def_ExperimentalScanDataset.pbtxt} (61%) create mode 100644 tensorflow/core/api_def/base_api/api_def_ExperimentalSetStatsAggregatorDataset.pbtxt rename tensorflow/core/api_def/base_api/{api_def_SlideDataset.pbtxt => api_def_ExperimentalSlidingWindowDataset.pbtxt} (88%) rename tensorflow/core/api_def/base_api/{api_def_SqlDataset.pbtxt => api_def_ExperimentalSqlDataset.pbtxt} (87%) create mode 100644 tensorflow/core/api_def/base_api/api_def_ExperimentalStatsAggregatorHandle.pbtxt rename tensorflow/core/api_def/base_api/{api_def_StatsAggregatorSummary.pbtxt => api_def_ExperimentalStatsAggregatorSummary.pbtxt} (56%) rename tensorflow/core/api_def/base_api/{api_def_UnbatchDataset.pbtxt => api_def_ExperimentalUnbatchDataset.pbtxt} (57%) delete mode 100644 tensorflow/core/api_def/base_api/api_def_MapAndBatchDataset.pbtxt delete mode 100644 tensorflow/core/api_def/base_api/api_def_MatchingFilesDataset.pbtxt delete mode 100644 tensorflow/core/api_def/base_api/api_def_PrependFromQueueAndPaddedBatchDataset.pbtxt create mode 100644 tensorflow/core/api_def/base_api/api_def_ResourceApplyAdamWithAmsgrad.pbtxt create mode 100644 tensorflow/core/api_def/base_api/api_def_ResourceApplyKerasMomentum.pbtxt create mode 100644 tensorflow/core/api_def/base_api/api_def_ResourceSparseApplyKerasMomentum.pbtxt delete mode 100644 tensorflow/core/api_def/base_api/api_def_SetStatsAggregatorDataset.pbtxt delete mode 100644 tensorflow/core/api_def/base_api/api_def_StatsAggregatorHandle.pbtxt create mode 100644 tensorflow/core/api_def/base_api/api_def_TensorForestCreateTreeVariable.pbtxt create mode 100644 tensorflow/core/api_def/base_api/api_def_TensorForestTreeDeserialize.pbtxt create mode 100644 tensorflow/core/api_def/base_api/api_def_TensorForestTreeIsInitializedOp.pbtxt create mode 100644 tensorflow/core/api_def/base_api/api_def_TensorForestTreePredict.pbtxt create mode 100644 tensorflow/core/api_def/base_api/api_def_TensorForestTreeResourceHandleOp.pbtxt create mode 100644 tensorflow/core/api_def/base_api/api_def_TensorForestTreeSerialize.pbtxt create mode 100644 tensorflow/core/api_def/base_api/api_def_TensorForestTreeSize.pbtxt create mode 100644 tensorflow/core/api_def/base_api/api_def_TensorListConcat.pbtxt create mode 100644 tensorflow/core/api_def/base_api/api_def_TensorListSplit.pbtxt create mode 100644 tensorflow/core/api_def/base_api/api_def_TensorScatterAdd.pbtxt create mode 100644 tensorflow/core/api_def/base_api/api_def_TensorScatterSub.pbtxt create mode 100644 tensorflow/core/api_def/base_api/api_def_TensorScatterUpdate.pbtxt create mode 100644 tensorflow/core/api_def/base_api/api_def_UnicodeDecodeWithOffsets.pbtxt create mode 100644 tensorflow/core/api_def/base_api/api_def_UnicodeEncode.pbtxt delete mode 100644 tensorflow/core/api_def/python_api/api_def_BatchDataset.pbtxt delete mode 100644 tensorflow/core/api_def/python_api/api_def_BytesProducedStatsDataset.pbtxt delete mode 100644 tensorflow/core/api_def/python_api/api_def_CacheDataset.pbtxt delete mode 100644 tensorflow/core/api_def/python_api/api_def_ConcatenateDataset.pbtxt delete mode 100644 tensorflow/core/api_def/python_api/api_def_DatasetToSingleElement.pbtxt delete mode 100644 tensorflow/core/api_def/python_api/api_def_DenseToSparseBatchDataset.pbtxt delete mode 100644 tensorflow/core/api_def/python_api/api_def_EnqueueInQueueDataset.pbtxt delete mode 100644 tensorflow/core/api_def/python_api/api_def_FilterDataset.pbtxt delete mode 100644 tensorflow/core/api_def/python_api/api_def_FixedLengthRecordDataset.pbtxt delete mode 100644 tensorflow/core/api_def/python_api/api_def_FixedLengthRecordDatasetV2.pbtxt delete mode 100644 tensorflow/core/api_def/python_api/api_def_FlatMapDataset.pbtxt delete mode 100644 tensorflow/core/api_def/python_api/api_def_GeneratorDataset.pbtxt delete mode 100644 tensorflow/core/api_def/python_api/api_def_GroupByWindowDataset.pbtxt delete mode 100644 tensorflow/core/api_def/python_api/api_def_InterleaveDataset.pbtxt delete mode 100644 tensorflow/core/api_def/python_api/api_def_LatencyStatsDataset.pbtxt delete mode 100644 tensorflow/core/api_def/python_api/api_def_MapAndBatchDataset.pbtxt delete mode 100644 tensorflow/core/api_def/python_api/api_def_MapDataset.pbtxt delete mode 100644 tensorflow/core/api_def/python_api/api_def_PaddedBatchDataset.pbtxt delete mode 100644 tensorflow/core/api_def/python_api/api_def_ParallelInterleaveDataset.pbtxt delete mode 100644 tensorflow/core/api_def/python_api/api_def_ParallelMapDataset.pbtxt delete mode 100644 tensorflow/core/api_def/python_api/api_def_ParseExampleDataset.pbtxt delete mode 100644 tensorflow/core/api_def/python_api/api_def_PrefetchDataset.pbtxt delete mode 100644 tensorflow/core/api_def/python_api/api_def_PrependFromQueueAndPaddedBatchDataset.pbtxt delete mode 100644 tensorflow/core/api_def/python_api/api_def_RandomDataset.pbtxt delete mode 100644 tensorflow/core/api_def/python_api/api_def_RangeDataset.pbtxt delete mode 100644 tensorflow/core/api_def/python_api/api_def_RepeatDataset.pbtxt create mode 100644 tensorflow/core/api_def/python_api/api_def_ResourceApplyAdamWithAmsgrad.pbtxt create mode 100644 tensorflow/core/api_def/python_api/api_def_ResourceApplyKerasMomentum.pbtxt create mode 100644 tensorflow/core/api_def/python_api/api_def_ResourceSparseApplyKerasMomentum.pbtxt delete mode 100644 tensorflow/core/api_def/python_api/api_def_ScanDataset.pbtxt delete mode 100644 tensorflow/core/api_def/python_api/api_def_SetStatsAggregatorDataset.pbtxt delete mode 100644 tensorflow/core/api_def/python_api/api_def_ShuffleAndRepeatDataset.pbtxt delete mode 100644 tensorflow/core/api_def/python_api/api_def_ShuffleDataset.pbtxt delete mode 100644 tensorflow/core/api_def/python_api/api_def_SkipDataset.pbtxt delete mode 100644 tensorflow/core/api_def/python_api/api_def_SlideDataset.pbtxt delete mode 100644 tensorflow/core/api_def/python_api/api_def_SparseTensorSliceDataset.pbtxt delete mode 100644 tensorflow/core/api_def/python_api/api_def_SqlDataset.pbtxt delete mode 100644 tensorflow/core/api_def/python_api/api_def_StatsAggregatorHandle.pbtxt delete mode 100644 tensorflow/core/api_def/python_api/api_def_StatsAggregatorSummary.pbtxt delete mode 100644 tensorflow/core/api_def/python_api/api_def_TFRecordDataset.pbtxt delete mode 100644 tensorflow/core/api_def/python_api/api_def_TakeDataset.pbtxt delete mode 100644 tensorflow/core/api_def/python_api/api_def_TensorDataset.pbtxt create mode 100644 tensorflow/core/api_def/python_api/api_def_TensorListConcat.pbtxt rename tensorflow/core/api_def/python_api/{api_defTensorListPushBackBatch.pbtxt => api_def_TensorListPushBackBatch.pbtxt} (100%) create mode 100644 tensorflow/core/api_def/python_api/api_def_TensorListSplit.pbtxt delete mode 100644 tensorflow/core/api_def/python_api/api_def_TensorSliceDataset.pbtxt delete mode 100644 tensorflow/core/api_def/python_api/api_def_TextLineDataset.pbtxt delete mode 100644 tensorflow/core/api_def/python_api/api_def_UnbatchDataset.pbtxt delete mode 100644 tensorflow/core/api_def/python_api/api_def_ZipDataset.pbtxt create mode 100644 tensorflow/core/distributed_runtime/server_lib_test.cc create mode 100644 tensorflow/core/framework/function_handle_cache.cc create mode 100644 tensorflow/core/framework/function_handle_cache.h delete mode 100644 tensorflow/core/grappler/optimizers/graph_rewriter.cc delete mode 100644 tensorflow/core/grappler/optimizers/graph_rewriter.h rename tensorflow/core/kernels/{conv_ops_gpu_3.cu.cc => conv_2d_gpu.h} (91%) create mode 100644 tensorflow/core/kernels/conv_2d_gpu_double.cu.cc create mode 100644 tensorflow/core/kernels/conv_2d_gpu_float.cu.cc create mode 100644 tensorflow/core/kernels/conv_2d_gpu_half.cu.cc rename tensorflow/{lite/experimental/micro/examples/micro_speech/CMSIS/preprocessor.h => core/kernels/conv_2d_gpu_int.cu.cc} (54%) create mode 100644 tensorflow/core/kernels/conv_2d_gpu_uint16.cu.cc create mode 100644 tensorflow/core/kernels/conv_2d_gpu_uint32.cu.cc create mode 100644 tensorflow/core/kernels/conv_2d_gpu_uint64.cu.cc create mode 100644 tensorflow/core/kernels/conv_2d_gpu_uint8.cu.cc create mode 100644 tensorflow/core/kernels/conv_ops_fused_image_transform.cc rename tensorflow/core/kernels/data/{ => experimental}/dense_to_sparse_batch_dataset_op.cc (97%) rename tensorflow/core/kernels/data/{ => experimental}/group_by_reducer_dataset_op.cc (93%) rename tensorflow/core/kernels/data/{ => experimental}/group_by_window_dataset_op.cc (95%) delete mode 100644 tensorflow/core/kernels/data/experimental/identity_indexed_dataset.cc delete mode 100644 tensorflow/core/kernels/data/experimental/indexed_dataset.h rename tensorflow/core/kernels/data/experimental/{indexed_dataset.cc => indexed_dataset_op.cc} (62%) rename tensorflow/core/kernels/data/{ => experimental}/map_and_batch_dataset_op.cc (90%) rename tensorflow/core/kernels/data/{ => experimental}/matching_files_dataset_op.cc (99%) create mode 100644 tensorflow/core/kernels/data/experimental/parallel_interleave_dataset_op.cc rename tensorflow/core/kernels/data/{ => experimental}/parse_example_dataset_op.cc (85%) rename tensorflow/core/kernels/data/{ => experimental}/random_dataset_op.cc (97%) rename tensorflow/core/kernels/data/{ => experimental}/scan_dataset_op.cc (94%) rename tensorflow/core/kernels/data/{stats_aggregator_dataset_op.cc => experimental/set_stats_aggregator_dataset_op.cc} (97%) rename tensorflow/core/kernels/data/{slide_dataset_op.cc => experimental/sliding_window_dataset_op.cc} (95%) rename tensorflow/core/kernels/data/{ => experimental}/sql/BUILD (100%) rename tensorflow/core/kernels/data/{ => experimental}/sql/driver_manager.cc (88%) rename tensorflow/core/kernels/data/{ => experimental}/sql/driver_manager.h (81%) rename tensorflow/core/kernels/data/{ => experimental}/sql/query_connection.h (92%) rename tensorflow/core/kernels/data/{ => experimental}/sql/sqlite_query_connection.cc (97%) rename tensorflow/core/kernels/data/{ => experimental}/sql/sqlite_query_connection.h (84%) rename tensorflow/core/kernels/data/{sql_dataset_ops.cc => experimental/sql_dataset_op.cc} (96%) rename tensorflow/core/kernels/data/{ => experimental}/stats_aggregator_ops.cc (95%) rename tensorflow/core/kernels/data/{ => experimental}/stats_dataset_ops.cc (95%) rename tensorflow/core/kernels/data/{writer_ops.cc => experimental/to_tf_record_op.cc} (84%) rename tensorflow/core/kernels/data/{ => experimental}/unbatch_dataset_op.cc (98%) create mode 100644 tensorflow/core/kernels/data/optional_ops.cu.cc delete mode 100644 tensorflow/core/kernels/data/tensor_queue_dataset_op.cc create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_bmp/012e3ad384a4a1165f8498b5c94ba0d32a73e187 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_bmp/055d77f7810048caa28323f6eb552a53d156040b create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_bmp/131e251bfb82c681cb075d32b99f18fceaca115d create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_bmp/1399ab0bd9f2c91d270cb43251bdc5729bef3526 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_bmp/16a6ce88f66d2e9686c8354cad8ba915cf0c11de create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_bmp/185097ed0588195164619ea930ddd8274a5f32ad create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_bmp/27711a87e06a50c81571c27c3aa403a6ad5dc55c create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_bmp/298c3787ad1722b22569cbc405c464d2 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_bmp/2b95ba6d8141ce0d29ff279770903922 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_bmp/321fb3d758b86e37fc340ae2b09b8ed9fa73a4cb create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_bmp/331a98b4e4c87840efea69223766ebd0e1736542 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_bmp/352d73f841223ecb630b5836585d2ba7b0f9d883 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_bmp/3a84f409d4c117edfdebc508cd23e8fc create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_bmp/3ef5cc982c0b45f69a26fd0f7d376415fdebabd1 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_bmp/401c7de8e122018a0e17f57c93db7ee49ab0e906 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_bmp/52fee71bb8c9c79068e1fe580677ad739a2d0415 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_bmp/57b11507813d5727b7789354d888eda83d5f3d86 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_bmp/57dff0fa53ee0ef24a43cca6ab0523bfdc1f720d create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_bmp/5c42d3df0dc400a7a4175b8d4eec6cc8ee2437b2 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_bmp/5cca20637ae75fddad9370ee930837baef8aeb43 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_bmp/5d34bc9cef0c844b9c5ebe948145c4ca11b5ca09 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_bmp/5e162fe883bd12fb1c4131d4e0c979a12bd15eac create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_bmp/5e83f8faab9c1a51a33d5e29edbb9dcec23c6092 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_bmp/61b29dc2fcef7b6fbe3e0cc88769a7ef create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_bmp/6361eca190157ece389665ee523ccc3aefcd957f create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_bmp/65150515ab3b11d657519b22bb887d74e94b2d7f create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_bmp/656f38ef6dcd58c6a909d61db11f777def69c394 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_bmp/66e0d2cafd592bf9d61ad900fade8ee530d5f3d7 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_bmp/6b5b42cb105a2c4c5fd6034e9885cbe457f1b50c create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_bmp/722ed0197cb92ecbf9745edb38275e7a9aaf322f create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_bmp/77bdd2efdf328366cbbf3c5688768dc0a88d02b1 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_bmp/7841bfa002c05c61d5a5d9241f214cc17a336166 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_bmp/7899e22fc83f6be28e9130c4a1c91a48 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_bmp/7dddccaebd16ae0c26daeffc42df50f529891119 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_bmp/8157442eee4bbfdd9716e264b11085d61a9955b7 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_bmp/81ff28ed63d5435ddc4c8771dd5d40aa658cbbe0 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_bmp/820c8c0d33c18f6c4d9edd314e91289186931ad0 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_bmp/849e9d7cee1c52105242327086997296e452b981 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_bmp/84ddb92c63e0fad7018f6069daf8779ce11501e2 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_bmp/86bc3d5dbb9313137502080e58551edd2e649c70 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_bmp/87d94d88fe29d277c76e1a52042b02c092d5ae14 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_bmp/8c4646f3357945c4e19a59ff79fffe3c874dbf16 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_bmp/90632bc6dee4eb836f3d7db1d16446a9c8510080 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_bmp/94d06016aa949e8e7203217e4cc6625ded7f4244 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_bmp/9875819b9e5783e7489c29a81cc9d4279209956a create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_bmp/9c1cc734114b29aac6c51782d5c17e9dbe1faca2 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_bmp/9d2961871eeb201ef8a6f5503d8a8b62 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_bmp/9f39e11cdd88344a4894b678e5a04a810880064d create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_bmp/a350588a6dabe4376a066aed44ef8786d8e752e7 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_bmp/a6101a79919d444e1fc50aefab5837c39e3f4a19 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_bmp/a9c8793f8fb063bec839ee1280406fe5396545e5 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_bmp/ad4e9d2234e8599bdf12607c6b8cab4edae82c4e create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_bmp/b90b6830917919e94186d312f06481bd create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_bmp/b98fd4cb1d7031240414301c19b03097c0035c6b create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_bmp/ba976fcdb4daf092ef17ce43bf2b78d9d8bc2aeb create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_bmp/bc112b571eafee0f5a031f3c9cce6244216d128d create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_bmp/c42b981c28a1715c375050f6fcf53f1d create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_bmp/c6049874b33eadb016fccf0c5fa66e556ae069b9 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_bmp/c8697bf2369f6ab85f501376c4d93bb8a56974a3 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_bmp/c8daf283e0aef2fd7b630c0430e05dc28f24ecf6 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_bmp/cacff56e1af4b8fde912822da06b10fb8c545a19 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_bmp/ce4dcc22b1d595c49a25121c0b580104 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_bmp/d0cd71dbf039fd64cf42eff30da92a71a919226a create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_bmp/d5ce626ac3264bed6af5580e341a89406857cbb9 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_bmp/d77ada02e9bc8c24b2711eca6a8f52ae356bfc21 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_bmp/d7eb9c5a0f9803df4c00390793b8ab57bd7c9484 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_bmp/dc1efccdeec17e151a1ec8228c09ab61c3040b33 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_bmp/dcea22c66c60088165a2f1772036473f create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_bmp/de539ae7442fa05dafcfe1a021f0186ef74a2b0e create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_bmp/e2306b1d6b88d0ccc4e2c3a9edb07462a5a32215 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_bmp/e2778da0240fdd15ef5844905d81c4e05f34a8bd create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_bmp/e6642e9266875f9d908942e534bf898103a2c794 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_bmp/ec6cdb929c08d8daf2bd7fc185fbf4d787b45120 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_bmp/ed8636357f79439b6a03eb14469b686cc401a1c9 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_bmp/ee313e9acecb5c688ce8c9bb10e70e136fbb9c6d create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_bmp/ef689af320e7d9e22231109faae2e8149cb86e1c create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_bmp/fda6b9a9f6ffdf4765c00465619c7ceb3f7db2e4 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_bmp/ffe829bb0adac20d9c0756f68a22d1255e4fdb54 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_json_example/013a29ea098a178f8a36741c9fd91144 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_json_example/0875575fb76d630ccb19c5da8aab66b2 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_json_example/7e7f58fc443a11a0a2c5d9b643b7e99b create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_json_example/849a23936269a261c0370b5e9abe2416 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_json_example/85282c1696d98b9843ce3e8bd1cd899f create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_json_example/90388b9c8093d8adedad0644b618da87 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_json_example/9fa2f86ea6d3ade36e961247c3026f8d create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_json_example/c4f18ca60a84e9869a28faf6f65dc758 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_json_example/d456ee029700adef5d28438593010223 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_json_example/e9f0ff6ee8d691ae69d2ecb4710030a2 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_png/010dc3d4b05288fcc40de2721052b3dc699f1cb3 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_png/0555cd5e9d99629819cc985285f80da0f00be1e9 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_png/0a0352aa168803ff65455792d9f6ee555c3e7c3f create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_png/0ed54162df93ef8d00f993ce6b59ba422903d381 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_png/1547b448171c700613c3946d730de496c9b9863f create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_png/17859046cbe4ac598a645173d679ce2a52c6afba create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_png/1df76c07817fbc3653a26f34d97658e9973627c2 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_png/1f0717f8856d7782e3ab7992d3a72d783a018443 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_png/23b911e4ce936def88bc9a46b8b433c0e83fba2a create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_png/25592201c3edff0578dbdac6b0e4f2be109ce151 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_png/266fd8495e0b8eb64387c1a62264185e061fee73 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_png/27f178cf415b4ff8671131ddf1d042dafac2fb3e create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_png/2a0bdc4d9cc5ea5bb21dd256d6ac96075376a94f create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_png/2e5d25add6adc68e0457b358c7a34abf3d41c938 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_png/2e6c5b6a766dd5e9bd41eacfd0a36572bd2f7544 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_png/2e9c935cf82f6ca640e9a9abc3c30a578ad46176 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_png/2fcf1ed4477f7eaee028f5b3f9edeb5f1a737826 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_png/3480713774f590908ca5dba16d121cdfb8fba62b create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_png/39289afcec60d98802b333e0fbb1da4d7aed4ce5 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_png/3adc488e21d4aca7bed9422f0241a42d0f93e7d9 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_png/3cbf274da522483dc991fad9df43a22ac4fb3173 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_png/3d840cdff7f5ad16fe8bcb985ed4946c03459432 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_png/3f1e6753c1fca958e859189857449746592158ea create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_png/3fa4075993cb0f9bfa8eea785174a2038a69aa1b create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_png/4023a373e977be58413e55350380310c5dd1fd6a create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_png/40caba69dce1cfc48e0e43184d2bfbc6daa4399a create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_png/41841e9561d8135945c1c1e55ab9e9a1e933653b create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_png/41d40f2d66fa43e34537385594ee9911e65deadf create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_png/421bd39810b50309a71adb2dadc3b19f01a52312 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_png/446c305b2c0665736f94fb2b62dbdef445eff0cf create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_png/449cee952bb645f6f4241a6665d3c6028c073c7a create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_png/45520b07609978c5aa3516d803527438b93fbadb create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_png/4da74a34bcede234b0415f77fbd87d70bf9a777e create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_png/51db5d31d2c5300d34831d9f23bcdd0aff9a998b create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_png/5cde2a9167798cb77f10abbfb2640a5c357f99fc create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_png/5e352fc10ac476cfbe1d755f092e069820223249 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_png/63661677dd1306cec4b5a565190e65adf2446e52 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_png/65887ed3db382aab1d9485c500f4401318d303b9 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_png/67b5181f8f0644597e9bde539e8f083b5cacd0e7 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_png/74c9dcf7afee2a6cb1ab3a2c0de744d1b03c1466 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_png/792181ca19e6ded261434e588bb7fc2a4816d4ce create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_png/79f0e2a475487f8fa69e68c1cc947c5851bda741 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_png/7e5fcdfeb557ce379ed96925c68505eaac0112db create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_png/7eec7530acf34b3a96fa9189783453999f7b6838 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_png/80114bf9781bffc9db411413d83541d8deaaf7c1 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_png/80425fb92bb86627e854892f23823fa804e5fdc3 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_png/821cdd6eeb919a8dd7f35289abbd583828dd4945 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_png/83e1a31785285338b0ddb3334b0ed098e63dedde create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_png/8a4c8100dedd0fb5f2a8b468c678f7ad8269deeb create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_png/8ae8268c24dc866c1edb3826b93a1c75dbf74ff4 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_png/90f72038cc627f34f074ea72eadbba87a5e3e288 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_png/92b67faee4a49df2cdbed785e27b4a1cddcfffa3 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_png/9463810467aacdc9923b2b20a2236116b760d75b create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_png/94d7c96aea32ad41ce643d35b951a6d8990b81d6 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_png/98cc7e9fe87df914d89a0aef008930f27b3c26f5 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_png/99172dfdb4f59aaced29c7681ac6e6ce8356e814 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_png/9ae3b647d895af97fe872c0b1442df7b5b767160 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_png/9d2b1d2121b0508a4fa8d1508adb9d05633fdac3 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_png/a335af37917ccf0c8b11bb884a3a74f3f1d2a7c6 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_png/a738609112d3a6772c50a71e2c3504ebc515b709 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_png/a8cecab5d917da5a4729632a7a18c564d7e1607d create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_png/ade919ab2b4a458e806575c941dfe50ae3fd3621 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_png/b1251621a5eb5e7fda9cac9baead1c993a285c36 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_png/b1516b78c3dfe77eeb554985fd7344c0478fbbcb create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_png/b41241740f5f8ad2c1d408f7bb6a313bd863c158 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_png/b799c8596523a7ebeb8e11ada08818c10f7eabfc create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_png/ba48d0521a111222dc95a3a997c7c92dea5f4443 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_png/c01457c6889fb1b597d308363a36412c0b7f90e7 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_png/c82ebc0d6688d104af04fd20d6d3da591dc391f7 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_png/c9a03eb758dd84e954e3d70916e2311e8fd21f3c create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_png/cf892756b33578a54ab20044514e573328d2f1d7 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_png/d3bc3f158a63f1d50b474addd3f7b3d17f23e8e9 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_png/d4906950aa9d60ad09dc0f5413c3d88080c3bc37 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_png/da31578a8068bad65e1c7a3d06e8f543a2a0bc65 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_png/dd4a9b5d0740679c249fc884efc499433b29436b create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_png/deea4ecc6f0b2a6d89fd25ff76762299f21602fb create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_png/e1040c7ffcb39915e0f539018c81f9798924cba6 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_png/e381dc85682cc33ad99f622b89d145b47f7d6392 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_png/ea24498fc7a144fccc6f1665ebf7020df803dd1a create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_png/eaa5d677e797c07bac98c3c7051abad91852e7c6 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_png/ed7871269315725535d8bffec7836c45a3fc5c26 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_png/ee8460f4077064c5a2137075b48eba7d3db5c570 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_png/ef09f26e0ee61329f84a9f589629a865ae9ee0a6 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_png/f477da4d7d8ff2066041e1dd5ee4e833b7111a1a create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_png/f8a379b2498a4eb452a85791a49adf065dab59ae create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_png/fe67bccb06f2174523943cc684518fcf1f7f8046 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_png/ff1e67d17c1c27ef0d97900d0ea276b563a64628 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_wav/02cc44cdfec1d9d0d0c66c5a5f40d3d20e4c4c3a create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_wav/087e1d7fae1c1ddcbaa3b5f822a171ad15498186 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_wav/0f61c33027394a0f14d29dcd22f405cad943b7cf create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_wav/10cdebea1659c21a0248f88654ae41f62786abf1 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_wav/126e68def9fd973a100e0f66cadf09448a716b57 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_wav/1275d41ebf8788ce3a949352e4bc654b04012da3 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_wav/1a7f1c407fb3864ddb559f88f373a21d1be51584 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_wav/1c3e1c91f187f6bcea86f172ff5bbbd955a9654d create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_wav/300fe1e0a47543037cbf0243b6756c9aa48799c4 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_wav/31ec5b0134bedcfe283f4978e6e65b7d35d5d4ad create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_wav/4e7cbb27667bcfca92838aa8020749990013a9b1 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_wav/585e469231d202812bfba8285fb30c8e31c857b9 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_wav/58eab6bc2386e2ef43fe4f55cb6ad3611399d5de create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_wav/63448c6a9feb8c72b3e82af4d735ec2e62ddd328 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_wav/6874d5b1c7a64b596c61f24877d422e89bebe58b create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_wav/7501f79cb067da108020579ed654349c7933d22f create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_wav/782051f8120182b860c7fe1b265179cfa2fe03fd create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_wav/793feab2deb35e284a975f6527d76a8be5540fe6 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_wav/7f41ec3a9805c6b8f3656c4f9f6d0ff7dbf8a329 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_wav/8210dc595a2652f2f812093b01e239e7918ea065 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_wav/8dffe4c5c26d891b578fd2ea4b9adfc0c96ad5f7 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_wav/91d787a9298ddc015efa783a92c4bdba8af0d7de create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_wav/92c065286f956f086e977556358f6b54b12bcacc create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_wav/a35c9bb71792b60a13dea23a41b41847ad4b93d6 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_wav/a6ea960c7b4d42772888280277b26e645ceee904 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_wav/aa526aa853333f0bb11804b5243df411452cecd2 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_wav/ca533cd26c7ca6bf69e62351b265ded496fdf1d9 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_wav/f38c61da15f2cb7a39ff02e69f0b00e99f37ec86 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_wav/f88f1012473e6cfcc9b39b2552f682b2f73eff8c create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_wav/fa79819c5de04bc06c69bec3fa7f2e982826ea2f create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_wav/fce08de222896ac3a20657a3b4f42d5b6c54a96a create mode 100644 tensorflow/core/kernels/fuzzing/corpus/string_split/4c01a1504da9de2216894743ecc44424 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/string_split/5bf16424630b5afbcffe711fb9834440 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/string_split/a7185605aef0a8fd682fcb4656e4a736 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/string_split/d5606def44fdbb9385dd764612069db0 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/string_split/dbac766f3160de65894bf5153f478146 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/string_split/e85ff62f6d457666f54a37a19a115a24 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/string_split_v2/00fd47bf73afcb72e7ed51bffd5f5fec create mode 100644 tensorflow/core/kernels/fuzzing/corpus/string_split_v2/14908973e6720513a5f37676cb9fcc29 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/string_split_v2/2779ba7c4d23eee9f79efa3660084c5d create mode 100644 tensorflow/core/kernels/fuzzing/corpus/string_split_v2/5bf16424630b5afbcffe711fb9834440 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/string_split_v2/89734a96b93275e495a9498b806fafe1 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/string_split_v2/d5606def44fdbb9385dd764612069db0 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/string_to_number/2db83ea58639b6d7d585fa12e3947a82 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/string_to_number/36b4a931886b941dc41180050d12ca94 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/string_to_number/50a2fabfdd276f573ff97ace8b11c5f4 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/string_to_number/62edb2a1eee34b001652cd86584becf2 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/string_to_number/90013d1ec28c46a5c00574e60c70b6fc create mode 100644 tensorflow/core/kernels/fuzzing/corpus/string_to_number/94f3e3cee6957ce5815326d6788c85f4 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/string_to_number/96f547bc04bb913da0bc08915238ebd8 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/string_to_number/d3a903d18fc11e1f35c572ad4da690ed create mode 100644 tensorflow/core/kernels/fuzzing/corpus/string_to_number/e3b629c92af44260c189deb32d6f06f3 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/string_to_number/f03eecf3bcfe4967a1888156a3115c8d create mode 100644 tensorflow/core/kernels/fuzzing/corpus/string_to_number/fa54ca9186f77122ae2a82684a062e16 create mode 100644 tensorflow/core/kernels/fuzzing/dictionaries/decode_json_example.dict create mode 100644 tensorflow/core/kernels/fuzzing/dictionaries/decode_png.dict create mode 100644 tensorflow/core/kernels/fuzzing/dictionaries/decode_wav.dict create mode 100644 tensorflow/core/kernels/mkl_quantized_pooling_ops_test.cc create mode 100644 tensorflow/core/kernels/scan_ops_test.cc create mode 100644 tensorflow/core/kernels/tensor_forest/BUILD create mode 100644 tensorflow/core/kernels/tensor_forest/prediction_ops.cc create mode 100644 tensorflow/core/kernels/tensor_forest/resource_ops.cc create mode 100644 tensorflow/core/kernels/tensor_forest/resources.cc create mode 100644 tensorflow/core/kernels/tensor_forest/resources.h create mode 100644 tensorflow/core/ops/tensor_forest_ops.cc create mode 100644 tensorflow/core/platform/default/logger.cc create mode 100644 tensorflow/core/platform/logger.h create mode 100644 tensorflow/core/platform/platform_strings.cc create mode 100644 tensorflow/core/platform/platform_strings.h create mode 100644 tensorflow/core/platform/platform_strings_computed.h create mode 100644 tensorflow/core/platform/platform_strings_test.cc create mode 100644 tensorflow/core/util/dump_graph.cc create mode 100644 tensorflow/core/util/dump_graph.h create mode 100644 tensorflow/core/util/dump_graph_test.cc create mode 100644 tensorflow/core/util/permutation_output_iterator.h create mode 100644 tensorflow/core/util/tensor_ops_util.h create mode 100644 tensorflow/go/op/gradients.go create mode 100644 tensorflow/go/op/gradients_test.go create mode 100644 tensorflow/lite/core/subgraph.cc create mode 100644 tensorflow/lite/core/subgraph.h create mode 100644 tensorflow/lite/experimental/micro/examples/micro_speech/.gitignore create mode 100644 tensorflow/lite/experimental/micro/examples/micro_speech/CMSIS/README.md create mode 100644 tensorflow/lite/experimental/micro/examples/micro_speech/CMSIS/arm_cmplx_mag_squared_q10p6.c create mode 100644 tensorflow/lite/experimental/micro/examples/micro_speech/CMSIS/arm_cmplx_mag_squared_q10p6.h create mode 100755 tensorflow/lite/experimental/micro/examples/micro_speech/CMSIS/create_constants.py delete mode 100644 tensorflow/lite/experimental/micro/examples/micro_speech/CMSIS/hann.h create mode 100644 tensorflow/lite/experimental/micro/examples/micro_speech/CMSIS/hanning.cc create mode 100644 tensorflow/lite/experimental/micro/examples/micro_speech/CMSIS/hanning.h delete mode 100644 tensorflow/lite/experimental/micro/examples/micro_speech/CMSIS/no_power_spectrum_data.h delete mode 100644 tensorflow/lite/experimental/micro/examples/micro_speech/CMSIS/preprocessor_test.cc create mode 100644 tensorflow/lite/experimental/micro/examples/micro_speech/CMSIS/sin_1k.cc create mode 100644 tensorflow/lite/experimental/micro/examples/micro_speech/CMSIS/sin_1k.h delete mode 100644 tensorflow/lite/experimental/micro/examples/micro_speech/CMSIS/yes_power_spectrum_data.h create mode 100644 tensorflow/lite/experimental/micro/examples/micro_speech/apollo3/.gitignore create mode 100644 tensorflow/lite/experimental/micro/examples/micro_speech/apollo3/README.md rename tensorflow/lite/experimental/micro/{tools/make/targets/apollo3evb => examples/micro_speech/apollo3}/_main.c (100%) create mode 100755 tensorflow/lite/experimental/micro/examples/micro_speech/apollo3/apollo3.h create mode 100644 tensorflow/lite/experimental/micro/examples/micro_speech/apollo3/captured_data_to_wav.py create mode 100644 tensorflow/lite/experimental/micro/examples/micro_speech/apollo3/compare_1k.py create mode 100644 tensorflow/lite/experimental/micro/examples/micro_speech/apollo3/preprocessor_1k.cc create mode 100644 tensorflow/lite/experimental/micro/examples/micro_speech/apollo3/preprocessor_1k_cmsis_test.cmd create mode 100644 tensorflow/lite/experimental/micro/examples/micro_speech/apollo3/preprocessor_1k_micro_test.cmd create mode 100644 tensorflow/lite/experimental/micro/examples/micro_speech/apollo3/preprocessor_test.cmd create mode 100644 tensorflow/lite/experimental/micro/examples/micro_speech/apollo3/pushbutton_cmsis_scores.cmd create mode 100644 tensorflow/lite/experimental/micro/examples/micro_speech/apollo3/pushbutton_cmsis_voice.cmd create mode 100644 tensorflow/lite/experimental/micro/examples/micro_speech/apollo3/pushbutton_main.c create mode 100644 tensorflow/lite/experimental/micro/examples/micro_speech/apollo3/pushbutton_test.cc create mode 100755 tensorflow/lite/experimental/micro/examples/micro_speech/apollo3/system_apollo3.c create mode 100755 tensorflow/lite/experimental/micro/examples/micro_speech/apollo3/system_apollo3.h create mode 100644 tensorflow/lite/experimental/micro/examples/micro_speech/audio_provider.cc create mode 100644 tensorflow/lite/experimental/micro/examples/micro_speech/audio_provider.h create mode 100644 tensorflow/lite/experimental/micro/examples/micro_speech/audio_provider_test.cc create mode 100644 tensorflow/lite/experimental/micro/examples/micro_speech/feature_provider.cc create mode 100644 tensorflow/lite/experimental/micro/examples/micro_speech/feature_provider.h create mode 100644 tensorflow/lite/experimental/micro/examples/micro_speech/feature_provider_test.cc create mode 100644 tensorflow/lite/experimental/micro/examples/micro_speech/main.cc rename tensorflow/lite/experimental/micro/examples/micro_speech/{CMSIS/no_power_spectrum_data.cc => model_settings.cc} (73%) create mode 100644 tensorflow/lite/experimental/micro/examples/micro_speech/model_settings.h rename tensorflow/lite/experimental/micro/examples/micro_speech/{CMSIS/yes_power_spectrum_data.cc => timer.cc} (73%) create mode 100644 tensorflow/lite/experimental/micro/examples/micro_speech/timer.h create mode 100644 tensorflow/lite/experimental/micro/examples/micro_speech/timer_test.cc delete mode 100644 tensorflow/lite/experimental/micro/tools/make/targets/apollo3evb/README.md delete mode 100644 tensorflow/lite/experimental/micro/tools/make/targets/apollo3evb/get_yesno_data.cmd delete mode 100644 tensorflow/lite/experimental/micro/tools/make/targets/apollo3evb/preprocessor_test.cmd delete mode 100644 tensorflow/lite/experimental/micro/tools/make/targets/apollo3evb/replace_calculated_data.py delete mode 100644 tensorflow/lite/experimental/micro/tools/make/targets/cmsis-dsp_makefile.inc create mode 100644 tensorflow/lite/g3doc/using_select_tf_ops.md create mode 100644 tensorflow/lite/java/src/testdata/string.bin create mode 100644 tensorflow/lite/kernels/fill.cc create mode 100644 tensorflow/lite/kernels/fill_test.cc delete mode 100644 tensorflow/lite/kernels/internal/optimized/cblas_conv.h delete mode 100644 tensorflow/lite/kernels/internal/optimized/cblas_reference.h create mode 100644 tensorflow/lite/kernels/mirror_pad.cc create mode 100644 tensorflow/lite/kernels/mirror_pad_test.cc create mode 100644 tensorflow/lite/kernels/split_v.cc create mode 100644 tensorflow/lite/kernels/split_v_test.cc create mode 100644 tensorflow/lite/kernels/squared_difference.cc create mode 100644 tensorflow/lite/kernels/squared_difference_test.cc create mode 100644 tensorflow/lite/toco/toco_convert.cc create mode 100644 tensorflow/lite/toco/toco_convert.h create mode 100644 tensorflow/lite/toco/toco_convert_test.cc create mode 100644 tensorflow/lite/tools/pip_package/MANIFEST.in create mode 100644 tensorflow/lite/tools/pip_package/README.md create mode 100644 tensorflow/lite/tools/pip_package/build_pip_package.sh create mode 100644 tensorflow/lite/tools/pip_package/setup.py create mode 100644 tensorflow/opensource_only.files create mode 100644 tensorflow/python/data/benchmarks/batch_benchmark.py create mode 100644 tensorflow/python/data/benchmarks/filter_benchmark.py create mode 100644 tensorflow/python/data/benchmarks/from_tensor_slices_benchmark.py create mode 100644 tensorflow/python/data/benchmarks/map_benchmark.py create mode 100644 tensorflow/python/data/experimental/benchmarks/autotune_benchmark.py create mode 100644 tensorflow/python/data/experimental/benchmarks/csv_dataset_benchmark.py delete mode 100644 tensorflow/python/data/experimental/benchmarks/map_benchmark.py create mode 100644 tensorflow/python/data/experimental/benchmarks/map_vectorization_benchmark.py create mode 100644 tensorflow/python/data/experimental/benchmarks/matching_files_benchmark.py create mode 100644 tensorflow/python/data/experimental/benchmarks/optimize_benchmark.py create mode 100644 tensorflow/python/data/experimental/benchmarks/unbatch_benchmark.py delete mode 100644 tensorflow/python/data/experimental/kernel_tests/batch_dataset_op_test.py create mode 100644 tensorflow/python/data/experimental/kernel_tests/cardinality_test.py delete mode 100644 tensorflow/python/data/experimental/kernel_tests/function_buffering_resource_test.py rename tensorflow/python/data/{kernel_tests/matching_files_dataset_op_test.py => experimental/kernel_tests/matching_files_test.py} (57%) create mode 100644 tensorflow/python/data/experimental/ops/cardinality.py create mode 100644 tensorflow/python/data/experimental/ops/filter_for_shard_ops.py create mode 100644 tensorflow/python/data/experimental/ops/matching_files.py create mode 100644 tensorflow/python/data/experimental/ops/optimization_options.py create mode 100644 tensorflow/python/data/experimental/ops/threading_options.py delete mode 100644 tensorflow/python/data/kernel_tests/batch_dataset_op_test.py create mode 100644 tensorflow/python/data/kernel_tests/batch_test.py delete mode 100644 tensorflow/python/data/kernel_tests/cache_dataset_op_test.py create mode 100644 tensorflow/python/data/kernel_tests/cache_test.py rename tensorflow/python/data/kernel_tests/{concatenate_dataset_op_test.py => concatenate_test.py} (75%) rename tensorflow/python/data/kernel_tests/{range_dataset_op_test.py => dataset_checkpoint_test.py} (84%) delete mode 100644 tensorflow/python/data/kernel_tests/dataset_constructor_op_test.py rename tensorflow/python/data/kernel_tests/{dataset_ops_test.py => dataset_test.py} (69%) delete mode 100644 tensorflow/python/data/kernel_tests/filter_dataset_op_test.py create mode 100644 tensorflow/python/data/kernel_tests/filter_test.py create mode 100644 tensorflow/python/data/kernel_tests/fixed_length_record_dataset_test.py rename tensorflow/python/data/kernel_tests/{flat_map_dataset_op_test.py => flat_map_test.py} (57%) rename tensorflow/python/data/kernel_tests/{dataset_from_generator_op_test.py => from_generator_test.py} (85%) create mode 100644 tensorflow/python/data/kernel_tests/from_sparse_tensor_slices_test.py create mode 100644 tensorflow/python/data/kernel_tests/from_tensor_slices_test.py create mode 100644 tensorflow/python/data/kernel_tests/from_tensors_test.py delete mode 100644 tensorflow/python/data/kernel_tests/inputs_test.py rename tensorflow/python/data/kernel_tests/{interleave_dataset_op_test.py => interleave_test.py} (85%) create mode 100644 tensorflow/python/data/kernel_tests/iterator_checkpoint_test.py rename tensorflow/python/data/kernel_tests/{iterator_ops_cluster_test.py => iterator_cluster_test.py} (96%) rename tensorflow/python/data/kernel_tests/{iterator_ops_test.py => iterator_test.py} (83%) delete mode 100644 tensorflow/python/data/kernel_tests/list_files_dataset_op_test.py create mode 100644 tensorflow/python/data/kernel_tests/list_files_test.py rename tensorflow/python/data/kernel_tests/{map_dataset_op_test.py => map_test.py} (80%) rename tensorflow/python/data/kernel_tests/{optional_ops_test.py => optional_test.py} (64%) create mode 100644 tensorflow/python/data/kernel_tests/padded_batch_test.py rename tensorflow/python/data/kernel_tests/{prefetch_dataset_op_test.py => prefetch_test.py} (52%) create mode 100644 tensorflow/python/data/kernel_tests/range_test.py delete mode 100644 tensorflow/python/data/kernel_tests/reader_dataset_ops_test.py rename tensorflow/python/data/kernel_tests/{reduce_dataset_op_test.py => reduce_test.py} (72%) create mode 100644 tensorflow/python/data/kernel_tests/repeat_test.py delete mode 100644 tensorflow/python/data/kernel_tests/sequence_dataset_op_test.py rename tensorflow/python/data/kernel_tests/{shard_dataset_op_test.py => shard_test.py} (52%) delete mode 100644 tensorflow/python/data/kernel_tests/shuffle_dataset_op_test.py create mode 100644 tensorflow/python/data/kernel_tests/shuffle_test.py create mode 100644 tensorflow/python/data/kernel_tests/skip_test.py create mode 100644 tensorflow/python/data/kernel_tests/take_test.py create mode 100644 tensorflow/python/data/kernel_tests/text_line_dataset_test.py create mode 100644 tensorflow/python/data/kernel_tests/tf_record_dataset_test.py delete mode 100644 tensorflow/python/data/kernel_tests/window_dataset_op_test.py create mode 100644 tensorflow/python/data/kernel_tests/window_test.py delete mode 100644 tensorflow/python/data/kernel_tests/zip_dataset_op_test.py create mode 100644 tensorflow/python/data/kernel_tests/zip_test.py create mode 100644 tensorflow/python/data/util/options.py create mode 100644 tensorflow/python/data/util/options_test.py create mode 100644 tensorflow/python/distribute/all_reduce.py rename tensorflow/{contrib/all_reduce/python => python/distribute}/all_reduce_test.py (97%) create mode 100644 tensorflow/python/distribute/cluster_resolver/BUILD rename tensorflow/{contrib => python/distribute}/cluster_resolver/README.md (100%) rename tensorflow/{contrib/cluster_resolver/python/training => python/distribute/cluster_resolver}/README.slurm (100%) create mode 100644 tensorflow/python/distribute/cluster_resolver/__init__.py create mode 100644 tensorflow/python/distribute/cluster_resolver/cluster_resolver.py rename tensorflow/{contrib/cluster_resolver/python/training => python/distribute/cluster_resolver}/cluster_resolver_test.py (98%) create mode 100644 tensorflow/python/distribute/cluster_resolver/gce_cluster_resolver.py rename tensorflow/{contrib/cluster_resolver/python/training => python/distribute/cluster_resolver}/gce_cluster_resolver_test.py (98%) create mode 100644 tensorflow/python/distribute/cluster_resolver/kubernetes_cluster_resolver.py rename tensorflow/{contrib/cluster_resolver/python/training => python/distribute/cluster_resolver}/kubernetes_cluster_resolver_test.py (87%) create mode 100644 tensorflow/python/distribute/cluster_resolver/slurm_cluster_resolver.py rename tensorflow/{contrib/cluster_resolver/python/training => python/distribute/cluster_resolver}/slurm_cluster_resolver_test.py (85%) create mode 100644 tensorflow/python/distribute/cluster_resolver/tfconfig_cluster_resolver.py rename tensorflow/{contrib/cluster_resolver/python/training => python/distribute/cluster_resolver}/tfconfig_cluster_resolver_test.py (72%) create mode 100644 tensorflow/python/distribute/cluster_resolver/tpu_cluster_resolver.py rename tensorflow/{contrib/cluster_resolver/python/training => python/distribute/cluster_resolver}/tpu_cluster_resolver_test.py (86%) rename tensorflow/{contrib/distribute/python/cross_tower_ops.py => python/distribute/cross_device_ops.py} (86%) rename tensorflow/{contrib/distribute/python/cross_tower_utils.py => python/distribute/cross_device_utils.py} (99%) rename tensorflow/python/{training => distribute}/device_util.py (98%) rename tensorflow/python/{training => distribute}/device_util_test.py (95%) create mode 100644 tensorflow/python/distribute/distribute_lib.py rename tensorflow/python/{training/distribute_test.py => distribute/distribute_lib_test.py} (75%) create mode 100644 tensorflow/python/distribute/distribution_strategy_context.py rename tensorflow/{contrib/distribute/python => python/distribute}/input_ops.py (89%) rename tensorflow/{contrib/distribute/python => python/distribute}/input_ops_test.py (89%) create mode 100644 tensorflow/python/distribute/mirrored_strategy.py create mode 100644 tensorflow/python/distribute/reduce_util.py rename tensorflow/{contrib/distribute/python => python/distribute}/shared_variable_creator.py (100%) rename tensorflow/{contrib/distribute/python => python/distribute}/shared_variable_creator_test.py (97%) rename tensorflow/{contrib/distribute/python => python/distribute}/values.py (77%) create mode 100644 tensorflow/python/eager/execution_callbacks_test.py create mode 100644 tensorflow/python/eager/function_gradients_test.py create mode 100644 tensorflow/python/keras/engine/base_layer_utils.py create mode 100644 tensorflow/python/keras/engine/input_spec.py create mode 100644 tensorflow/python/keras/engine/training_dataset_test.py create mode 100644 tensorflow/python/keras/layers/unified_lstm_test.py create mode 100644 tensorflow/python/keras/optimizer_v2/adadelta.py create mode 100644 tensorflow/python/keras/optimizer_v2/adadelta_test.py create mode 100644 tensorflow/python/keras/optimizer_v2/adagrad.py create mode 100644 tensorflow/python/keras/optimizer_v2/adagrad_test.py create mode 100644 tensorflow/python/keras/optimizer_v2/adam_test.py create mode 100644 tensorflow/python/keras/optimizer_v2/adamax.py create mode 100644 tensorflow/python/keras/optimizer_v2/adamax_test.py create mode 100644 tensorflow/python/keras/optimizer_v2/ftrl.py create mode 100644 tensorflow/python/keras/optimizer_v2/ftrl_test.py create mode 100644 tensorflow/python/keras/optimizer_v2/nadam.py create mode 100644 tensorflow/python/keras/optimizer_v2/nadam_test.py create mode 100644 tensorflow/python/keras/optimizer_v2/rmsprop.py create mode 100644 tensorflow/python/keras/optimizer_v2/rmsprop_test.py create mode 100644 tensorflow/python/keras/utils/losses_utils.py create mode 100644 tensorflow/python/keras/utils/tf_utils_test.py create mode 100644 tensorflow/python/kernel_tests/linalg/linear_operator_adjoint_test.py create mode 100644 tensorflow/python/kernel_tests/linalg/linear_operator_algebra_test.py create mode 100644 tensorflow/python/kernel_tests/linalg/linear_operator_inversion_test.py rename tensorflow/python/kernel_tests/{ => signal}/dct_ops_test.py (67%) rename tensorflow/python/kernel_tests/{ => signal}/fft_ops_test.py (96%) create mode 100644 tensorflow/python/kernel_tests/unicode_decode_op_test.py create mode 100644 tensorflow/python/kernel_tests/unicode_encode_op_test.py create mode 100644 tensorflow/python/ops/gradient_checker_v2.py create mode 100644 tensorflow/python/ops/gradient_checker_v2_test.py create mode 100644 tensorflow/python/ops/linalg/cholesky_registrations.py create mode 100644 tensorflow/python/ops/linalg/linear_operator_adjoint.py create mode 100644 tensorflow/python/ops/linalg/linear_operator_algebra.py create mode 100644 tensorflow/python/ops/linalg/linear_operator_inversion.py create mode 100644 tensorflow/python/ops/linalg/matmul_registrations.py rename tensorflow/{tools/compatibility/testdata/test_file_v1_10.py => python/ops/optional_grad.py} (62%) create mode 100644 tensorflow/python/ops/ragged/ragged_string_ops.py create mode 100644 tensorflow/python/ops/ragged/ragged_tensor_shape.py create mode 100644 tensorflow/python/ops/ragged/ragged_tensor_shape_test.py delete mode 100644 tensorflow/python/ops/signal/__init__.py create mode 100644 tensorflow/python/ops/signal/dct_ops.py rename tensorflow/python/ops/{spectral_ops.py => signal/fft_ops.py} (51%) create mode 100644 tensorflow/python/ops/signal/signal.py create mode 100644 tensorflow/python/ops/sort_ops.py rename tensorflow/{contrib/framework => }/python/ops/sort_ops_test.py (90%) delete mode 100644 tensorflow/python/ops/spectral_grad.py create mode 100644 tensorflow/python/ops/tensor_forest_ops.py create mode 100644 tensorflow/python/platform/__init__.py create mode 100644 tensorflow/python/saved_model/load.py create mode 100644 tensorflow/python/saved_model/load_test.py create mode 100644 tensorflow/python/saved_model/saved_object_graph.proto create mode 100644 tensorflow/python/util/dispatch.py create mode 100644 tensorflow/python/util/dispatch_test.py create mode 100644 tensorflow/stream_executor/dnn.proto create mode 100644 tensorflow/tools/api/golden/v1/tensorflow.-tensor-spec.pbtxt create mode 100644 tensorflow/tools/api/golden/v1/tensorflow.data.experimental.-optimization-options.pbtxt create mode 100644 tensorflow/tools/api/golden/v1/tensorflow.data.experimental.-threading-options.pbtxt create mode 100644 tensorflow/tools/api/golden/v1/tensorflow.distribute.-input-context.pbtxt create mode 100644 tensorflow/tools/api/golden/v1/tensorflow.distribute.-input-replication-mode.pbtxt create mode 100644 tensorflow/tools/api/golden/v1/tensorflow.distribute.-reduce-op.pbtxt create mode 100644 tensorflow/tools/api/golden/v1/tensorflow.distribute.-replica-context.pbtxt create mode 100644 tensorflow/tools/api/golden/v1/tensorflow.distribute.-strategy-extended.pbtxt create mode 100644 tensorflow/tools/api/golden/v1/tensorflow.distribute.-strategy.pbtxt create mode 100644 tensorflow/tools/api/golden/v1/tensorflow.distribute.pbtxt rename tensorflow/tools/api/golden/{v2/tensorflow.train.-profiler-hook.pbtxt => v1/tensorflow.estimator.experimental.-in-memory-evaluator-hook.pbtxt} (71%) create mode 100644 tensorflow/tools/api/golden/v1/tensorflow.io.gfile.pbtxt create mode 100644 tensorflow/tools/api/golden/v1/tensorflow.keras.losses.-binary-crossentropy.pbtxt create mode 100644 tensorflow/tools/api/golden/v1/tensorflow.keras.losses.-categorical-crossentropy.pbtxt create mode 100644 tensorflow/tools/api/golden/v1/tensorflow.keras.losses.-mean-absolute-error.pbtxt create mode 100644 tensorflow/tools/api/golden/v1/tensorflow.keras.losses.-mean-absolute-percentage-error.pbtxt create mode 100644 tensorflow/tools/api/golden/v1/tensorflow.keras.losses.-mean-squared-error.pbtxt create mode 100644 tensorflow/tools/api/golden/v1/tensorflow.keras.losses.-mean-squared-logarithmic-error.pbtxt create mode 100644 tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-accuracy.pbtxt create mode 100644 tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-binary-accuracy.pbtxt create mode 100644 tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-categorical-accuracy.pbtxt create mode 100644 tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-false-negatives.pbtxt create mode 100644 tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-false-positives.pbtxt create mode 100644 tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-mean.pbtxt create mode 100644 tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-precision.pbtxt create mode 100644 tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-recall.pbtxt create mode 100644 tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-sparse-categorical-accuracy.pbtxt create mode 100644 tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-true-negatives.pbtxt create mode 100644 tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-true-positives.pbtxt create mode 100644 tensorflow/tools/api/golden/v1/tensorflow.metrics.-accuracy.pbtxt create mode 100644 tensorflow/tools/api/golden/v1/tensorflow.metrics.-binary-accuracy.pbtxt create mode 100644 tensorflow/tools/api/golden/v1/tensorflow.metrics.-categorical-accuracy.pbtxt create mode 100644 tensorflow/tools/api/golden/v1/tensorflow.metrics.-false-negatives.pbtxt create mode 100644 tensorflow/tools/api/golden/v1/tensorflow.metrics.-false-positives.pbtxt create mode 100644 tensorflow/tools/api/golden/v1/tensorflow.metrics.-mean.pbtxt create mode 100644 tensorflow/tools/api/golden/v1/tensorflow.metrics.-precision.pbtxt create mode 100644 tensorflow/tools/api/golden/v1/tensorflow.metrics.-recall.pbtxt create mode 100644 tensorflow/tools/api/golden/v1/tensorflow.metrics.-sparse-categorical-accuracy.pbtxt create mode 100644 tensorflow/tools/api/golden/v1/tensorflow.metrics.-true-negatives.pbtxt create mode 100644 tensorflow/tools/api/golden/v1/tensorflow.metrics.-true-positives.pbtxt delete mode 100644 tensorflow/tools/api/golden/v2/tensorflow.-conditional-accumulator-base.pbtxt delete mode 100644 tensorflow/tools/api/golden/v2/tensorflow.-conditional-accumulator.pbtxt delete mode 100644 tensorflow/tools/api/golden/v2/tensorflow.-device-spec.pbtxt delete mode 100644 tensorflow/tools/api/golden/v2/tensorflow.-dimension.pbtxt delete mode 100644 tensorflow/tools/api/golden/v2/tensorflow.-graph-keys.pbtxt delete mode 100644 tensorflow/tools/api/golden/v2/tensorflow.-tensor-info.-coo-sparse.pbtxt delete mode 100644 tensorflow/tools/api/golden/v2/tensorflow.-tensor-info.pbtxt create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.-tensor-spec.pbtxt delete mode 100644 tensorflow/tools/api/golden/v2/tensorflow.app.pbtxt delete mode 100644 tensorflow/tools/api/golden/v2/tensorflow.data.-iterator.pbtxt create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.data.experimental.-optimization-options.pbtxt create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.data.experimental.-threading-options.pbtxt create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.distribute.-input-context.pbtxt create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.distribute.-input-replication-mode.pbtxt create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.distribute.-reduce-op.pbtxt create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.distribute.-replica-context.pbtxt create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.distribute.-strategy-extended.pbtxt create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.distribute.-strategy.pbtxt create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.distribute.pbtxt create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.estimator.experimental.-in-memory-evaluator-hook.pbtxt delete mode 100644 tensorflow/tools/api/golden/v2/tensorflow.gfile.-fast-g-file.pbtxt delete mode 100644 tensorflow/tools/api/golden/v2/tensorflow.gfile.-g-file.pbtxt delete mode 100644 tensorflow/tools/api/golden/v2/tensorflow.gfile.-open.pbtxt delete mode 100644 tensorflow/tools/api/golden/v2/tensorflow.gfile.pbtxt create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.io.gfile.pbtxt rename tensorflow/tools/api/golden/v2/{tensorflow.nn.rnn_cell.-multi-r-n-n-cell.pbtxt => tensorflow.keras.layers.-dense-features.pbtxt} (77%) create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-linear-model.pbtxt create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.keras.losses.-binary-crossentropy.pbtxt create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.keras.losses.-categorical-crossentropy.pbtxt create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.keras.losses.-mean-absolute-error.pbtxt create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.keras.losses.-mean-absolute-percentage-error.pbtxt create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.keras.losses.-mean-squared-error.pbtxt create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.keras.losses.-mean-squared-logarithmic-error.pbtxt create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.keras.losses.-reduction.pbtxt create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-accuracy.pbtxt create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-binary-accuracy.pbtxt create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-categorical-accuracy.pbtxt create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-false-negatives.pbtxt create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-false-positives.pbtxt create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-mean.pbtxt create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-precision.pbtxt create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-recall.pbtxt create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-sparse-categorical-accuracy.pbtxt create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-true-negatives.pbtxt create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-true-positives.pbtxt delete mode 100644 tensorflow/tools/api/golden/v2/tensorflow.logging.pbtxt create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.metrics.-accuracy.pbtxt create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.metrics.-binary-accuracy.pbtxt create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.metrics.-categorical-accuracy.pbtxt create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.metrics.-false-negatives.pbtxt create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.metrics.-false-positives.pbtxt create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.metrics.-mean.pbtxt create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.metrics.-precision.pbtxt create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.metrics.-recall.pbtxt create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.metrics.-sparse-categorical-accuracy.pbtxt create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.metrics.-true-negatives.pbtxt create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.metrics.-true-positives.pbtxt delete mode 100644 tensorflow/tools/api/golden/v2/tensorflow.profiler.-advice-proto.-checker.pbtxt delete mode 100644 tensorflow/tools/api/golden/v2/tensorflow.profiler.-advice-proto.-checkers-entry.pbtxt delete mode 100644 tensorflow/tools/api/golden/v2/tensorflow.profiler.-advice-proto.pbtxt delete mode 100644 tensorflow/tools/api/golden/v2/tensorflow.profiler.-graph-node-proto.-input-shapes-entry.pbtxt delete mode 100644 tensorflow/tools/api/golden/v2/tensorflow.profiler.-graph-node-proto.pbtxt delete mode 100644 tensorflow/tools/api/golden/v2/tensorflow.profiler.-multi-graph-node-proto.pbtxt delete mode 100644 tensorflow/tools/api/golden/v2/tensorflow.profiler.-op-log-proto.-id-to-string-entry.pbtxt delete mode 100644 tensorflow/tools/api/golden/v2/tensorflow.profiler.-op-log-proto.pbtxt delete mode 100644 tensorflow/tools/api/golden/v2/tensorflow.profiler.-profile-option-builder.pbtxt delete mode 100644 tensorflow/tools/api/golden/v2/tensorflow.profiler.-profiler.pbtxt delete mode 100644 tensorflow/tools/api/golden/v2/tensorflow.profiler.pbtxt delete mode 100644 tensorflow/tools/api/golden/v2/tensorflow.spectral.pbtxt create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.summary.-summary-writer.pbtxt delete mode 100644 tensorflow/tools/api/golden/v2/tensorflow.train.-adadelta-optimizer.pbtxt delete mode 100644 tensorflow/tools/api/golden/v2/tensorflow.train.-adagrad-d-a-optimizer.pbtxt delete mode 100644 tensorflow/tools/api/golden/v2/tensorflow.train.-adagrad-optimizer.pbtxt delete mode 100644 tensorflow/tools/api/golden/v2/tensorflow.train.-adam-optimizer.pbtxt delete mode 100644 tensorflow/tools/api/golden/v2/tensorflow.train.-checkpoint-saver-listener.pbtxt delete mode 100644 tensorflow/tools/api/golden/v2/tensorflow.train.-chief-session-creator.pbtxt delete mode 100644 tensorflow/tools/api/golden/v2/tensorflow.train.-ftrl-optimizer.pbtxt delete mode 100644 tensorflow/tools/api/golden/v2/tensorflow.train.-gradient-descent-optimizer.pbtxt delete mode 100644 tensorflow/tools/api/golden/v2/tensorflow.train.-looper-thread.pbtxt delete mode 100644 tensorflow/tools/api/golden/v2/tensorflow.train.-momentum-optimizer.pbtxt delete mode 100644 tensorflow/tools/api/golden/v2/tensorflow.train.-monitored-session.-step-context.pbtxt delete mode 100644 tensorflow/tools/api/golden/v2/tensorflow.train.-monitored-session.pbtxt delete mode 100644 tensorflow/tools/api/golden/v2/tensorflow.train.-nan-loss-during-training-error.pbtxt delete mode 100644 tensorflow/tools/api/golden/v2/tensorflow.train.-optimizer.pbtxt delete mode 100644 tensorflow/tools/api/golden/v2/tensorflow.train.-proximal-adagrad-optimizer.pbtxt delete mode 100644 tensorflow/tools/api/golden/v2/tensorflow.train.-r-m-s-prop-optimizer.pbtxt delete mode 100644 tensorflow/tools/api/golden/v2/tensorflow.train.-scaffold.pbtxt delete mode 100644 tensorflow/tools/api/golden/v2/tensorflow.train.-second-or-step-timer.pbtxt delete mode 100644 tensorflow/tools/api/golden/v2/tensorflow.train.-session-creator.pbtxt delete mode 100644 tensorflow/tools/api/golden/v2/tensorflow.train.-session-manager.pbtxt delete mode 100644 tensorflow/tools/api/golden/v2/tensorflow.train.-session-run-args.pbtxt delete mode 100644 tensorflow/tools/api/golden/v2/tensorflow.train.-session-run-context.pbtxt delete mode 100644 tensorflow/tools/api/golden/v2/tensorflow.train.-session-run-values.pbtxt delete mode 100644 tensorflow/tools/api/golden/v2/tensorflow.train.-singular-monitored-session.-step-context.pbtxt delete mode 100644 tensorflow/tools/api/golden/v2/tensorflow.train.-singular-monitored-session.pbtxt delete mode 100644 tensorflow/tools/api/golden/v2/tensorflow.train.-supervisor.pbtxt delete mode 100644 tensorflow/tools/api/golden/v2/tensorflow.train.-vocab-info.pbtxt delete mode 100644 tensorflow/tools/api/golden/v2/tensorflow.train.-worker-session-creator.pbtxt create mode 100644 tensorflow/tools/ci_build/Dockerfile.rbe.cuda10.0-cudnn7-ubuntu14.04 create mode 100644 tensorflow/tools/compatibility/ast_edits_test.py create mode 100644 tensorflow/tools/compatibility/testdata/test_file_v1_12.py create mode 100644 tensorflow/tools/compatibility/tf_upgrade_v2_main.py create mode 100644 tensorflow/tools/dockerfiles/.gitignore rename tensorflow/tools/dockerfiles/dockerfiles/{nvidia-devel-jupyter.Dockerfile => gpu-devel-jupyter.Dockerfile} (66%) rename tensorflow/tools/dockerfiles/dockerfiles/{nvidia-devel.Dockerfile => gpu-devel.Dockerfile} (76%) rename tensorflow/tools/dockerfiles/dockerfiles/{nvidia-jupyter.Dockerfile => gpu-jupyter.Dockerfile} (62%) rename tensorflow/tools/dockerfiles/dockerfiles/{nvidia.Dockerfile => gpu.Dockerfile} (69%) create mode 100644 tensorflow/tools/dockerfiles/partials/test-import.partial.Dockerfile delete mode 100644 tensorflow/tools/dockerfiles/partials/ubuntu.partial.Dockerfile rename tensorflow/tools/dockerfiles/partials/{ => ubuntu}/bazel.partial.Dockerfile (58%) rename tensorflow/tools/dockerfiles/partials/{ubuntu-devel.partial.Dockerfile => ubuntu/cpu-devel.partial.Dockerfile} (86%) create mode 100644 tensorflow/tools/dockerfiles/partials/ubuntu/cpu.partial.Dockerfile rename tensorflow/tools/dockerfiles/partials/{ => ubuntu}/nvidia-devel.partial.Dockerfile (78%) rename tensorflow/tools/dockerfiles/partials/{ => ubuntu}/nvidia.partial.Dockerfile (78%) rename tensorflow/tools/dockerfiles/partials/{ => ubuntu}/python.partial.Dockerfile (66%) create mode 100644 tensorflow/tools/dockerfiles/partials/ubuntu/test-devel.partial.Dockerfile create mode 100644 tensorflow/tools/dockerfiles/partials/ubuntu/version.partial.Dockerfile create mode 100644 tensorflow/tools/dockerfiles/readme-for-jupyter.md create mode 100755 tensorflow/tools/dockerfiles/tests/build-cpu.sh create mode 100755 tensorflow/tools/dockerfiles/tests/build-gpu.sh create mode 100755 tensorflow/tools/dockerfiles/tests/import-gpu.sh create mode 100755 tensorflow/tools/dockerfiles/tests/import.sh rename tensorflow/tools/dockerfiles/{assembler.Dockerfile => tools.Dockerfile} (95%) create mode 100644 tensorflow/tools/docs/generate2.py rename tensorflow/{contrib/estimator/python/estimator/linear.py => tools/docs/generate2_test.py} (60%) delete mode 100644 tensorflow/tools/docs/generate_1_0.py delete mode 100644 third_party/eigen_reshaped.patch create mode 100644 third_party/icu/data/BUILD.bazel create mode 100644 third_party/icu/data/LICENSE create mode 100644 third_party/icu/data/icu_conversion_data.c.gz.aa create mode 100644 third_party/icu/data/icu_conversion_data.c.gz.ab create mode 100644 third_party/icu/data/icu_conversion_data.c.gz.ac create mode 100644 third_party/icu/data/icu_conversion_data.c.gz.ad create mode 100644 third_party/icu/data/icu_conversion_data.c.gz.ae create mode 100644 third_party/icu/data/icu_conversion_data.c.gz.af create mode 100644 third_party/icu/data/icu_conversion_data.c.gz.ag create mode 100644 third_party/icu/data/icu_conversion_data.c.gz.ah create mode 100644 third_party/icu/data/icu_conversion_data.c.gz.ai create mode 100644 third_party/icu/data/icu_conversion_data.c.gz.aj create mode 100644 third_party/icu/udata.patch create mode 100644 third_party/toolchains/preconfig/ubuntu14.04/cuda10.0-cudnn7/WORKSPACE create mode 100755 third_party/toolchains/preconfig/ubuntu14.04/cuda10.0-cudnn7/cuda/BUILD create mode 100755 third_party/toolchains/preconfig/ubuntu14.04/cuda10.0-cudnn7/cuda/build_defs.bzl create mode 100755 third_party/toolchains/preconfig/ubuntu14.04/cuda10.0-cudnn7/cuda/cuda/cuda_config.h create mode 100755 third_party/toolchains/preconfig/ubuntu14.04/gcc-nvcc-cuda10.0/BUILD create mode 100755 third_party/toolchains/preconfig/ubuntu14.04/gcc-nvcc-cuda10.0/CROSSTOOL create mode 100755 third_party/toolchains/preconfig/ubuntu14.04/gcc-nvcc-cuda10.0/clang/bin/crosstool_wrapper_driver_is_not_gcc create mode 100755 third_party/toolchains/preconfig/ubuntu14.04/gcc-nvcc-cuda10.0/windows/msvc_wrapper_for_nvcc.bat create mode 100755 third_party/toolchains/preconfig/ubuntu14.04/gcc-nvcc-cuda10.0/windows/msvc_wrapper_for_nvcc.py create mode 100755 third_party/toolchains/preconfig/ubuntu14.04/gcc-nvcc-cuda9.0/BUILD create mode 100755 third_party/toolchains/preconfig/ubuntu14.04/gcc-nvcc-cuda9.0/CROSSTOOL create mode 100755 third_party/toolchains/preconfig/ubuntu14.04/gcc-nvcc-cuda9.0/clang/bin/crosstool_wrapper_driver_is_not_gcc create mode 100755 third_party/toolchains/preconfig/ubuntu14.04/gcc-nvcc-cuda9.0/windows/msvc_wrapper_for_nvcc.bat create mode 100755 third_party/toolchains/preconfig/ubuntu14.04/gcc-nvcc-cuda9.0/windows/msvc_wrapper_for_nvcc.py diff --git a/.github/ISSUE_TEMPLATE/40-tflite-op-request.md b/.github/ISSUE_TEMPLATE/40-tflite-op-request.md new file mode 100644 index 00000000000..7b391279e47 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/40-tflite-op-request.md @@ -0,0 +1,24 @@ +--- +name: TensorFlow Lite Op Request +about: Use this template for reporting ops you are using or missing. + +--- + + +**System information** +- OS Platform and Distribution (e.g., Linux Ubuntu 16.04): +- TensorFlow installed from (source or binary): +- TensorFlow version (or github SHA if from source): + + +**Provide the text output from tflite_convert** + +``` +# Copy and paste here +``` + +Also, please include a link to a GraphDef or the model if possible. + +**Any other info / logs** + +Include any logs or source code that would be helpful to diagnose the problem. If including tracebacks, please include the full traceback. Large logs and files should be attached. diff --git a/README.md b/README.md index 8af5370befb..6fefdd32244 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,8 @@ data flow graphs. The graph nodes represent mathematical operations, while the graph edges represent the multidimensional data arrays (tensors) that flow between them. This flexible architecture enables you to deploy computation to one or more CPUs or GPUs in a desktop, server, or mobile device without rewriting -code. TensorFlow also includes [TensorBoard](https://www.tensorflow.org/guide/summaries_and_tensorboard), a data visualization toolkit. +code. TensorFlow also includes [TensorBoard](https://github.com/tensorflow/tensorboard), +a data visualization toolkit. TensorFlow was originally developed by researchers and engineers working on the Google Brain team within Google's Machine Intelligence Research @@ -111,7 +112,7 @@ The TensorFlow project strives to abide by generally accepted best practices in Build Type | Status | Artifacts ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------- **IBM s390x** | [![Build Status](http://ibmz-ci.osuosl.org/job/TensorFlow_IBMZ_CI/badge/icon)](http://ibmz-ci.osuosl.org/job/TensorFlow_IBMZ_CI/) | TBA -**IBM ppc64le CPU** | [![Build Status](http://powerci.osuosl.org/job/TensorFlow_Ubuntu_16.04_CPU/badge/icon)](http://powerci.osuosl.org/job/TensorFlow_Ubuntu_16.04_CPU/) | TBA +**IBM ppc64le CPU** | [![Build Status](http://powerci.osuosl.org/job/TensorFlow_PPC64LE_CPU_Build/badge/icon)](http://powerci.osuosl.org/job/TensorFlow_PPC64LE_CPU_Build/) | TBA **IBM ppc64le GPU** Nightly | [![Build Status](https://powerci.osuosl.org/job/TensorFlow_PPC64LE_GPU_Nightly_Artifact/badge/icon)](https://powerci.osuosl.org/job/TensorFlow_PPC64LE_GPU_Nightly_Artifact/) | [Nightly](https://powerci.osuosl.org/job/TensorFlow_PPC64LE_GPU_Nightly_Artifact/) **IBM ppc64le GPU** Stable Release | [![Build Status](https://powerci.osuosl.org/job/TensorFlow_PPC64LE_GPU_Release_Build/badge/icon)](https://powerci.osuosl.org/job/TensorFlow_PPC64LE_GPU_Release_Build/) | [Release](https://powerci.osuosl.org/job/TensorFlow_PPC64LE_GPU_Release_Build/) **Linux CPU with Intel® MKL-DNN** Nightly | [![Build Status](https://tensorflow-ci.intel.com/job/tensorflow-mkl-linux-cpu/badge/icon)](https://tensorflow-ci.intel.com/job/tensorflow-mkl-linux-cpu/) | [Nightly](https://tensorflow-ci.intel.com/job/tensorflow-mkl-build-whl-nightly/) @@ -127,6 +128,7 @@ Build Type * [TensorFlow Roadmap](https://www.tensorflow.org/community/roadmap) * [TensorFlow White Papers](https://www.tensorflow.org/about/bib) * [TensorFlow YouTube Channel](https://www.youtube.com/channel/UC0rqucBdTuFTjJiefW5t-IQ) +* [TensorFlow Visualization Toolkit](https://github.com/tensorflow/tensorboard) Learn more about the TensorFlow community at the [community page of tensorflow.org](https://www.tensorflow.org/community) for a few ways to participate. diff --git a/WORKSPACE b/WORKSPACE index 0c7bc085b51..7cc08e0164a 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -1,5 +1,7 @@ workspace(name = "org_tensorflow") +load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") + http_archive( name = "io_bazel_rules_closure", sha256 = "a38539c5b5c358548e75b44141b4ab637bba7c4dc02b46b1f62a96d6433f56ae", @@ -57,9 +59,9 @@ android_workspace() # Please add all new TensorFlow dependencies in workspace.bzl. tf_workspace() -new_http_archive( +http_archive( name = "inception_v1", - build_file = "models.BUILD", + build_file = "//:models.BUILD", sha256 = "7efe12a8363f09bc24d7b7a450304a15655a57a7751929b2c1593a71183bb105", urls = [ "http://storage.googleapis.com/download.tensorflow.org/models/inception_v1.zip", @@ -67,9 +69,9 @@ new_http_archive( ], ) -new_http_archive( +http_archive( name = "mobile_ssd", - build_file = "models.BUILD", + build_file = "//:models.BUILD", sha256 = "bddd81ea5c80a97adfac1c9f770e6f55cbafd7cce4d3bbe15fbeb041e6b8f3e8", urls = [ "http://storage.googleapis.com/download.tensorflow.org/models/object_detection/ssd_mobilenet_v1_android_export.zip", @@ -77,9 +79,9 @@ new_http_archive( ], ) -new_http_archive( +http_archive( name = "mobile_multibox", - build_file = "models.BUILD", + build_file = "//:models.BUILD", sha256 = "859edcddf84dddb974c36c36cfc1f74555148e9c9213dedacf1d6b613ad52b96", urls = [ "http://storage.googleapis.com/download.tensorflow.org/models/mobile_multibox_v1a.zip", @@ -87,9 +89,9 @@ new_http_archive( ], ) -new_http_archive( +http_archive( name = "stylize", - build_file = "models.BUILD", + build_file = "//:models.BUILD", sha256 = "3d374a730aef330424a356a8d4f04d8a54277c425e274ecb7d9c83aa912c6bfa", urls = [ "http://storage.googleapis.com/download.tensorflow.org/models/stylize_v1.zip", @@ -97,9 +99,9 @@ new_http_archive( ], ) -new_http_archive( +http_archive( name = "speech_commands", - build_file = "models.BUILD", + build_file = "//:models.BUILD", sha256 = "c3ec4fea3158eb111f1d932336351edfe8bd515bb6e87aad4f25dbad0a600d0c", urls = [ "http://storage.googleapis.com/download.tensorflow.org/models/speech_commands_v0.01.zip", diff --git a/configure.py b/configure.py index 234561d94a4..5f429c3de89 100644 --- a/configure.py +++ b/configure.py @@ -238,6 +238,13 @@ def setup_python(environ_cp): write_to_bazelrc('build --python_path=\"%s"' % python_bin_path) environ_cp['PYTHON_BIN_PATH'] = python_bin_path + # If choosen python_lib_path is from a path specified in the PYTHONPATH + # variable, need to tell bazel to include PYTHONPATH + if environ_cp.get('PYTHONPATH'): + python_paths = environ_cp.get('PYTHONPATH').split(':') + if python_lib_path in python_paths: + write_action_env_to_bazelrc('PYTHONPATH', environ_cp.get('PYTHONPATH')) + # Write tools/python_bin_path.sh with open( os.path.join(_TF_WORKSPACE_ROOT, 'tools', 'python_bin_path.sh'), @@ -445,11 +452,12 @@ def convert_version_to_int(version): return int(version_str) -def check_bazel_version(min_version): - """Check installed bazel version is at least min_version. +def check_bazel_version(min_version, max_version): + """Check installed bazel version is between min_version and max_version. Args: min_version: string for minimum bazel version. + max_version: string for maximum bazel version. Returns: The bazel version detected. @@ -467,6 +475,7 @@ def check_bazel_version(min_version): min_version_int = convert_version_to_int(min_version) curr_version_int = convert_version_to_int(curr_version) + max_version_int = convert_version_to_int(max_version) # Check if current bazel version can be detected properly. if not curr_version_int: @@ -480,6 +489,10 @@ def check_bazel_version(min_version): print('Please upgrade your bazel installation to version %s or higher to ' 'build TensorFlow!' % min_version) sys.exit(0) + if curr_version_int > max_version_int: + print('Please downgrade your bazel installation to version %s or lower to ' + 'build TensorFlow!' % max_version) + sys.exit(0) return curr_version @@ -859,7 +872,7 @@ def set_tf_cuda_version(environ_cp): cuda_toolkit_paths_full = [ os.path.join(cuda_toolkit_path, x) for x in cuda_rt_lib_paths ] - if any([os.path.exists(x) for x in cuda_toolkit_paths_full]): + if any(os.path.exists(x) for x in cuda_toolkit_paths_full): break # Reset and retry @@ -1552,7 +1565,7 @@ def main(): # environment variables. environ_cp = dict(os.environ) - check_bazel_version('0.15.0') + check_bazel_version('0.15.0', '0.20.0') reset_tf_configure_bazelrc() # Explicitly import tools/bazel.rc, this is needed for Bazel 0.19.0 or later @@ -1694,6 +1707,7 @@ def main(): config_info_line('nohdfs', 'Disable HDFS support.') config_info_line('noignite', 'Disable Apacha Ignite support.') config_info_line('nokafka', 'Disable Apache Kafka support.') + config_info_line('nonccl', 'Disable NVIDIA NCCL support.') if __name__ == '__main__': diff --git a/tensorflow/BUILD b/tensorflow/BUILD index 859dc3b8d77..fd4b94202aa 100644 --- a/tensorflow/BUILD +++ b/tensorflow/BUILD @@ -43,6 +43,11 @@ TENSORFLOW_API_INIT_FILES_V2 = ( TENSORFLOW_API_INIT_FILES + get_compat_files(TENSORFLOW_API_INIT_FILES_V1, 1) ) +# @unused +TENSORFLOW_API_INIT_FILES_V1_WITH_COMPAT = ( + TENSORFLOW_API_INIT_FILES_V1 + get_compat_files(TENSORFLOW_API_INIT_FILES_V1, 1) +) + # Config setting used when building for products # which requires restricted licenses to be avoided. config_setting( @@ -213,31 +218,37 @@ config_setting( # config_setting( name = "no_aws_support", - define_values = {"no_aws_support": "false"}, + define_values = {"no_aws_support": "true"}, visibility = ["//visibility:public"], ) config_setting( name = "no_gcp_support", - define_values = {"no_gcp_support": "false"}, + define_values = {"no_gcp_support": "true"}, visibility = ["//visibility:public"], ) config_setting( name = "no_hdfs_support", - define_values = {"no_hdfs_support": "false"}, + define_values = {"no_hdfs_support": "true"}, visibility = ["//visibility:public"], ) config_setting( name = "no_ignite_support", - define_values = {"no_ignite_support": "false"}, + define_values = {"no_ignite_support": "true"}, visibility = ["//visibility:public"], ) config_setting( name = "no_kafka_support", - define_values = {"no_kafka_support": "false"}, + define_values = {"no_kafka_support": "true"}, + visibility = ["//visibility:public"], +) + +config_setting( + name = "no_nccl_support", + define_values = {"no_nccl_support": "true"}, visibility = ["//visibility:public"], ) @@ -350,7 +361,7 @@ package_group( "-//third_party/tensorflow/python/estimator", "//learning/meta_rank/...", "//tensorflow/...", - "//tensorflow_estimator/...", + "//tensorflow_estimator/contrib/...", "//tensorflow_fold/llgtm/...", "//tensorflow_text/...", "//third_party/py/tensor2tensor/...", @@ -554,18 +565,24 @@ genrule( }), outs = ["__init__.py"], cmd = select({ - "api_version_2": "cp $(@D)/_api/v2/__init__.py $(OUTS)", - "//conditions:default": "cp $(@D)/_api/v1/__init__.py $(OUTS)", + "api_version_2": "cp $(@D)/_api/v2/v2.py $(OUTS)", + "//conditions:default": "cp $(@D)/_api/v1/v1.py $(OUTS)", }), ) gen_api_init_files( name = "tf_python_api_gen_v1", - srcs = ["api_template_v1.__init__.py"], + srcs = [ + "api_template_v1.__init__.py", + "compat_template_v1.__init__.py", + ], api_version = 1, + compat_api_versions = [1], + compat_init_templates = ["compat_template_v1.__init__.py"], output_dir = "_api/v1/", - output_files = TENSORFLOW_API_INIT_FILES_V1, + output_files = TENSORFLOW_API_INIT_FILES_V1_WITH_COMPAT, output_package = "tensorflow._api.v1", + root_file_name = "v1.py", root_init_template = "api_template_v1.__init__.py", ) @@ -581,6 +598,7 @@ gen_api_init_files( output_dir = "_api/v2/", output_files = TENSORFLOW_API_INIT_FILES_V2, output_package = "tensorflow._api.v2", + root_file_name = "v2.py", root_init_template = "api_template.__init__.py", ) diff --git a/tensorflow/api_template.__init__.py b/tensorflow/api_template.__init__.py index 0d497568385..d81cf067eb0 100644 --- a/tensorflow/api_template.__init__.py +++ b/tensorflow/api_template.__init__.py @@ -21,8 +21,6 @@ from __future__ import print_function as _print_function import os as _os # pylint: disable=g-bad-import-order -from tensorflow.python import pywrap_tensorflow # pylint: disable=unused-import - from tensorflow.python.tools import component_api_helper as _component_api_helper _component_api_helper.package_hook( parent_package_str=__name__, @@ -30,16 +28,16 @@ _component_api_helper.package_hook( # API IMPORTS PLACEHOLDER -from tensorflow.python.platform import flags # pylint: disable=g-import-not-at-top - # Make sure directory containing top level submodules is in # the __path__ so that "from tensorflow.foo import bar" works. -_tf_api_dir = _os.path.dirname(_os.path.dirname(app.__file__)) # pylint: disable=undefined-variable +# We're using bitwise, but there's nothing special about that. +_tf_api_dir = _os.path.dirname(_os.path.dirname(bitwise.__file__)) # pylint: disable=undefined-variable if _tf_api_dir not in __path__: __path__.append(_tf_api_dir) -# Calls to enable and disable features. -enable_eager_execution() # pylint: disable=undefined-variable +# Enable TF2 behaviors +from tensorflow.python.compat import compat as _compat # pylint: disable=g-import-not-at-top +_compat.enable_v2_behavior() # These symbols appear because we import the python package which # in turn imports from tensorflow.core and tensorflow.python. They diff --git a/tensorflow/c/BUILD b/tensorflow/c/BUILD index b8db1b21449..59c23e7c184 100644 --- a/tensorflow/c/BUILD +++ b/tensorflow/c/BUILD @@ -60,6 +60,7 @@ tf_cuda_library( "//tensorflow/core:framework", "//tensorflow/core:lib", "//tensorflow/core:op_gen_lib", + "//tensorflow/core/distributed_runtime:server_lib", ], }), ) @@ -120,7 +121,8 @@ tf_cuda_library( ":c_api", ":c_api_internal", "//tensorflow/c/eager:c_api", - "//tensorflow/compiler/jit/legacy_flags:mark_for_compilation_pass_flags", + "//tensorflow/c/eager:c_api_internal", + "//tensorflow/compiler/jit:flags", "//tensorflow/contrib/tpu:all_ops", "//tensorflow/core:core_cpu", "//tensorflow/core:framework", @@ -173,6 +175,30 @@ tf_cuda_library( ], ) +tf_cuda_library( + name = "kernels", + srcs = [ + "kernels.cc", + ], + hdrs = [ + "kernels.h", + ], + copts = tf_copts(), + visibility = ["//visibility:public"], + deps = select({ + "//tensorflow:android": [ + ":c_api", + ":c_api_internal", + "//tensorflow/core:android_tensorflow_lib_lite", + ], + "//conditions:default": [ + ":c_api", + ":c_api_internal", + "//tensorflow/core:framework", + ], + }), +) + # ----------------------------------------------------------------------------- # Tests @@ -208,7 +234,10 @@ tf_cuda_cc_test( "//tensorflow:darwin": ["-headerpad_max_install_names"], "//conditions:default": [], }), - tags = ["noasan"], + tags = [ + "no_oss", # http://b/119522529 + "noasan", + ], # We must ensure that the dependencies can be dynamically linked since # the shared library must be able to use core:framework. # linkstatic = tf_kernel_tests_linkstatic(), @@ -237,7 +266,7 @@ tf_cuda_cc_test( tf_cc_test( name = "c_api_experimental_test", - size = "small", + size = "medium", srcs = ["c_api_experimental_test.cc"], data = ["testdata/tf_record"], linkopts = select({ @@ -248,8 +277,11 @@ tf_cc_test( # the shared library must be able to use core:framework. # linkstatic = tf_kernel_tests_linkstatic(), deps = [ + ":c_api", ":c_api_experimental", ":c_test_util", + "//tensorflow/c/eager:c_api", + "//tensorflow/c/eager:c_api_test_util", "//tensorflow/core:lib", "//tensorflow/core:protos_all_cc", "//tensorflow/core:test", @@ -300,6 +332,30 @@ tf_kernel_library( alwayslink = 1, ) +tf_cuda_cc_test( + name = "kernels_test", + size = "small", + srcs = ["kernels_test.cc"], + linkopts = select({ + "//tensorflow:darwin": ["-headerpad_max_install_names"], + "//conditions:default": [], + }), + tags = ["noasan"], + # We must ensure that the dependencies can be dynamically linked since + # the shared library must be able to use core:framework. + # linkstatic = tf_kernel_tests_linkstatic(), + deps = [ + ":c_api", + ":kernels", + "//tensorflow/core:framework", + "//tensorflow/core:lib", + "//tensorflow/core:proto_text", + "//tensorflow/core:protos_all_cc", + "//tensorflow/core:test", + "//tensorflow/core:test_main", + ], +) + # ----------------------------------------------------------------------------- # Python API target diff --git a/tensorflow/c/c_api_experimental.cc b/tensorflow/c/c_api_experimental.cc index fabe2fa0f60..38e29aa74a9 100644 --- a/tensorflow/c/c_api_experimental.cc +++ b/tensorflow/c/c_api_experimental.cc @@ -15,13 +15,18 @@ limitations under the License. #include "tensorflow/c/c_api_experimental.h" +#include "tensorflow/c/c_api.h" #include "tensorflow/c/c_api_internal.h" -#include "tensorflow/compiler/jit/legacy_flags/mark_for_compilation_pass_flags.h" +#include "tensorflow/c/eager/c_api.h" +#include "tensorflow/c/eager/c_api_internal.h" +#include "tensorflow/compiler/jit/flags.h" #include "tensorflow/core/common_runtime/eager/attr_builder.h" #include "tensorflow/core/framework/tensor.pb.h" #include "tensorflow/core/graph/graph.h" #include "tensorflow/core/graph/node_builder.h" #include "tensorflow/core/lib/strings/strcat.h" +#include "tensorflow/core/platform/init_main.h" +#include "tensorflow/core/platform/net.h" #include "tensorflow/core/platform/platform.h" #include "tensorflow/core/protobuf/config.pb.h" #include "tensorflow/core/protobuf/tensorflow_server.pb.h" @@ -51,8 +56,8 @@ void TF_EnableXLACompilation(TF_SessionOptions* options, unsigned char enable) { // These XLA flags are needed to trigger XLA properly from C (more generally // non-Python) clients. If this API is called again with `enable` set to // false, it is safe to keep these flag values as is. - tensorflow::legacy_flags::MarkForCompilationPassFlags* flags = - tensorflow::legacy_flags::GetMarkForCompilationPassFlags(); + tensorflow::MarkForCompilationPassFlags* flags = + tensorflow::GetMarkForCompilationPassFlags(); flags->tf_xla_cpu_global_jit = true; flags->tf_xla_min_cluster_size = 1; } else { @@ -71,8 +76,8 @@ TF_Buffer* TF_CreateConfig(unsigned char enable_xla_compilation, // These XLA flags are needed to trigger XLA properly from C (more generally // non-Python) clients. If this API is called again with `enable` set to // false, it is safe to keep these flag values as is. - tensorflow::legacy_flags::MarkForCompilationPassFlags* flags = - tensorflow::legacy_flags::GetMarkForCompilationPassFlags(); + tensorflow::MarkForCompilationPassFlags* flags = + tensorflow::GetMarkForCompilationPassFlags(); flags->tf_xla_cpu_global_jit = true; flags->tf_xla_min_cluster_size = 1; } else { @@ -6525,7 +6530,7 @@ library { } } node_def { - name: "ParallelInterleaveDataset/cycle_length" + name: "ExperimentalParallelInterleaveDataset/cycle_length" op: "Const" attr { key: "dtype" @@ -6546,7 +6551,7 @@ library { } } node_def { - name: "ParallelInterleaveDataset/block_length" + name: "ExperimentalParallelInterleaveDataset/block_length" op: "Const" attr { key: "dtype" @@ -6567,7 +6572,7 @@ library { } } node_def { - name: "ParallelInterleaveDataset/sloppy" + name: "ExperimentalParallelInterleaveDataset/sloppy" op: "Const" attr { key: "dtype" @@ -6588,7 +6593,7 @@ library { } } node_def { - name: "ParallelInterleaveDataset/buffer_output_elements" + name: "ExperimentalParallelInterleaveDataset/buffer_output_elements" op: "Const" attr { key: "dtype" @@ -6609,7 +6614,7 @@ library { } } node_def { - name: "ParallelInterleaveDataset/prefetch_input_elements" + name: "ExperimentalParallelInterleaveDataset/prefetch_input_elements" op: "Const" attr { key: "dtype" @@ -6630,14 +6635,14 @@ library { } } node_def { - name: "ParallelInterleaveDataset" - op: "ParallelInterleaveDataset" + name: "ExperimentalParallelInterleaveDataset" + op: "ExperimentalParallelInterleaveDataset" input: "RepeatDataset:handle:0" - input: "ParallelInterleaveDataset/cycle_length:output:0" - input: "ParallelInterleaveDataset/block_length:output:0" - input: "ParallelInterleaveDataset/sloppy:output:0" - input: "ParallelInterleaveDataset/buffer_output_elements:output:0" - input: "ParallelInterleaveDataset/prefetch_input_elements:output:0" + input: "ExperimentalParallelInterleaveDataset/cycle_length:output:0" + input: "ExperimentalParallelInterleaveDataset/block_length:output:0" + input: "ExperimentalParallelInterleaveDataset/sloppy:output:0" + input: "ExperimentalParallelInterleaveDataset/buffer_output_elements:output:0" + input: "ExperimentalParallelInterleaveDataset/prefetch_input_elements:output:0" attr { key: "Targuments" value { @@ -6737,7 +6742,7 @@ library { node_def { name: "ShuffleDataset_2" op: "ShuffleDataset" - input: "ParallelInterleaveDataset:handle:0" + input: "ExperimentalParallelInterleaveDataset:handle:0" input: "ShuffleDataset_2/buffer_size_1:output:0" input: "ShuffleDataset_2/seed_2:output:0" input: "ShuffleDataset_2/seed2_2:output:0" @@ -8739,14 +8744,65 @@ void TFE_TensorHandlePrintDebugString(TFE_TensorHandle* handle) { TF_DeleteStatus(status); } -TF_CAPI_EXPORT extern void TF_MakeInternalErrorStatus(TF_Status* status, - const char* errMsg) { +struct TFE_ExecuteOpNotification { + TFE_ExecuteOpNotification() : status(TF_NewStatus(), TF_DeleteStatus) {} + tensorflow::Notification n; + std::unique_ptr thread; + std::unique_ptr status; +}; + +TFE_ExecuteOpNotification* TFE_ExecuteOpInNewThread(TFE_Op* op, + TFE_TensorHandle** retvals, + int* num_retvals, + TF_Status* status) { + TFE_ExecuteOpNotification* n = new TFE_ExecuteOpNotification; + + n->thread.reset(op->operation.EagerContext()->TFEnv()->StartThread( + tensorflow::ThreadOptions(), "ExecuteOpThread", + [op, retvals, num_retvals, n]() { + TFE_Execute(op, retvals, num_retvals, n->status.get()); + n->n.Notify(); + })); + + return n; +} + +void TFE_ExecuteOpNotificationWaitAndDelete( + TFE_ExecuteOpNotification* notification, TF_Status* status) { + if (notification == nullptr) { + status->status = tensorflow::errors::InvalidArgument( + "Passed in notification is a nullptr."); + + return; + } + if (notification->thread == nullptr) { + status->status = tensorflow::errors::InvalidArgument( + "Passed in notification didn't start a thread correctly. Cleaning up " + "this notification. Please re-execute the operation to get a new " + "notification."); + + delete notification; + return; + } + + notification->n.WaitForNotification(); + + status->status = notification->status->status; + + delete notification; +} + +void TF_MakeInternalErrorStatus(TF_Status* status, const char* errMsg) { status->status = tensorflow::errors::Internal(errMsg); } // This builder is used in the eager API to build a NodeDef. struct TF_AttrBuilder : public tensorflow::AttrBuilder { using tensorflow::AttrBuilder::AttrBuilder; + // The string buffers to make sure that any `attr_name` we pass into + // `builder->Set()` will outlive the subsequent + // `TF_AttrBuilderCheckCanRunOnDevice()` call(s) on the same `builder`. + std::set attr_names; }; TF_AttrBuilder* TF_NewAttrBuilder(const char* op_name) { @@ -8757,13 +8813,15 @@ void TF_DeleteAttrBuilder(TF_AttrBuilder* builder) { delete builder; } void TF_AttrBuilderSetType(TF_AttrBuilder* builder, const char* attr_name, TF_DataType value) { - builder->Set(attr_name, static_cast(value)); + auto iter = builder->attr_names.insert(attr_name).first; + builder->Set((*iter).c_str(), static_cast(value)); } void TF_AttrBuilderSetTypeList(TF_AttrBuilder* builder, const char* attr_name, const TF_DataType* values, int num_values) { + auto iter = builder->attr_names.insert(attr_name).first; builder->Set( - attr_name, + (*iter).c_str(), tensorflow::gtl::ArraySlice( reinterpret_cast(values), num_values)); } @@ -8800,3 +8858,31 @@ const char* TF_GetNumberAttrForOpListInput(const char* op_name, int input_index, // The returned string is owned by OpRegistry, so liveness is not a concern. return input_arg.number_attr().c_str(); } + +int TF_OpIsStateful(const char* op_type, TF_Status* status) { + const tensorflow::OpRegistrationData* op_reg_data; + status->status = + tensorflow::OpRegistry::Global()->LookUp(op_type, &op_reg_data); + if (!status->status.ok()) { + return 0; + } + return op_reg_data->op_def.is_stateful(); +} + +void TF_InitMain(const char* usage, int* argc, char*** argv) { + tensorflow::port::InitMain(usage, argc, argv); +} + +int TF_PickUnusedPortOrDie() { + return tensorflow::internal::PickUnusedPortOrDie(); +} + +TFE_TensorHandle* TFE_NewTensorHandleFromScalar(TF_DataType dtype_arg, + void* data, size_t len) { + auto dtype = static_cast(dtype_arg); + DCHECK(tensorflow::DataTypeCanUseMemcpy(dtype)); + + tensorflow::Tensor tensor(dtype, tensorflow::TensorShape({})); + std::memcpy(tensorflow::TensorCApi::Buffer(tensor)->data(), data, len); + return new TFE_TensorHandle(tensor, nullptr, nullptr); +} diff --git a/tensorflow/c/c_api_experimental.h b/tensorflow/c/c_api_experimental.h index 6639b0be72b..80c8bfe594c 100644 --- a/tensorflow/c/c_api_experimental.h +++ b/tensorflow/c/c_api_experimental.h @@ -180,6 +180,25 @@ TF_CAPI_EXPORT extern TFE_TensorHandle* TFE_DequeueVariantTensor( TF_CAPI_EXPORT extern void TFE_TensorHandlePrintDebugString( TFE_TensorHandle* handle); +typedef struct TFE_ExecuteOpNotification TFE_ExecuteOpNotification; + +// Allows invoking a kernel asynchronously, and explicitly returns a +// notification that can be waited upon. This always executes the kernel in a +// new thread. +// 1. `retvals` and `num_retvals` can only be consumed after +// `TFE_ExecuteOp` returns successfully. They shouldn't be used +// if the return is unsuccessful +// 2. These new APIs cannot be used together with the TFE context level async +// support. +TF_CAPI_EXPORT extern TFE_ExecuteOpNotification* TFE_ExecuteOpInNewThread( + TFE_Op* op, TFE_TensorHandle** retvals, int* num_retvals, + TF_Status* status); + +// Waits to complete the op execution, and cleans up the notification. +// Errors reported by op execution are set in `status`. +TF_CAPI_EXPORT extern void TFE_ExecuteOpNotificationWaitAndDelete( + TFE_ExecuteOpNotification* notification, TF_Status* status); + TF_CAPI_EXPORT extern void TF_MakeInternalErrorStatus(TF_Status* status, const char* errMsg); @@ -209,6 +228,24 @@ TF_CAPI_EXPORT extern void TF_AttrBuilderCheckCanRunOnDevice( TF_CAPI_EXPORT extern const char* TF_GetNumberAttrForOpListInput( const char* op_name, int input_index, TF_Status* status); +// Returns 1 if the op is stateful, 0 otherwise. The return value is undefined +// if the status is not ok. +TF_CAPI_EXPORT extern int TF_OpIsStateful(const char* op_type, + TF_Status* status); + +// Platform specific initialization routine. Very few platforms actually require +// this to be called. +TF_CAPI_EXPORT void TF_InitMain(const char* usage, int* argc, char*** argv); + +// Platform-specific implementation to return an unused port. (This should used +// in tests only.) +TF_CAPI_EXPORT int TF_PickUnusedPortOrDie(); + +// Fast path method that makes constructing a single scalar tensor require less +// overhead and copies. +TF_CAPI_EXPORT extern TFE_TensorHandle* TFE_NewTensorHandleFromScalar( + TF_DataType dtype, void* scalar, size_t len); + #ifdef __cplusplus } /* end extern "C" */ #endif diff --git a/tensorflow/c/c_api_experimental_test.cc b/tensorflow/c/c_api_experimental_test.cc index c6effd39697..daa7701b7fe 100644 --- a/tensorflow/c/c_api_experimental_test.cc +++ b/tensorflow/c/c_api_experimental_test.cc @@ -15,6 +15,8 @@ limitations under the License. #include "tensorflow/c/c_api_experimental.h" #include "tensorflow/c/c_test_util.h" +#include "tensorflow/c/eager/c_api.h" +#include "tensorflow/c/eager/c_api_test_util.h" #include "tensorflow/core/lib/io/path.h" #include "tensorflow/core/platform/env.h" #include "tensorflow/core/platform/logging.h" @@ -162,5 +164,137 @@ protocol: "grpc" TF_DeleteStatus(status); } +TEST(CAPI_EXPERIMENTAL, IsStateful) { + std::unique_ptr status( + TF_NewStatus(), TF_DeleteStatus); + int assign = TF_OpIsStateful("AssignAddVariableOp", status.get()); + ASSERT_EQ(TF_OK, TF_GetCode(status.get())) << TF_Message(status.get()); + EXPECT_EQ(assign, 1); + int id = TF_OpIsStateful("Identity", status.get()); + ASSERT_EQ(TF_OK, TF_GetCode(status.get())) << TF_Message(status.get()); + EXPECT_EQ(id, 0); +} + +TEST(CAPI_EXPERIMENTAL, TFE_ExecuteOpInNewThreadTest_Simple) { + TF_Status* status = TF_NewStatus(); + TFE_ContextOptions* opts = TFE_NewContextOptions(); + TFE_Context* ctx = TFE_NewContext(opts, status); + CHECK_EQ(TF_OK, TF_GetCode(status)) << TF_Message(status); + TFE_DeleteContextOptions(opts); + + TFE_TensorHandle* m = TestMatrixTensorHandle(); + + TFE_Op* matmul_op = MatMulOp(ctx, m, m); + + TFE_TensorHandle* retvals[1] = {nullptr}; + int num_retvals = 1; + + auto* r = + TFE_ExecuteOpInNewThread(matmul_op, &retvals[0], &num_retvals, status); + + TFE_ExecuteOpNotificationWaitAndDelete(r, status); + CHECK_EQ(TF_OK, TF_GetCode(status)) << TF_Message(status); + + TF_Tensor* t = TFE_TensorHandleResolve(retvals[0], status); + ASSERT_EQ(TF_OK, TF_GetCode(status)) << TF_Message(status); + float product[4] = {0}; + EXPECT_EQ(sizeof(product), TF_TensorByteSize(t)); + memcpy(&product[0], TF_TensorData(t), TF_TensorByteSize(t)); + TF_DeleteTensor(t); + EXPECT_EQ(7, product[0]); + EXPECT_EQ(10, product[1]); + EXPECT_EQ(15, product[2]); + EXPECT_EQ(22, product[3]); + + TFE_DeleteOp(matmul_op); + TFE_DeleteTensorHandle(m); + + TFE_DeleteTensorHandle(retvals[0]); + TFE_DeleteContext(ctx); + TF_DeleteStatus(status); +} + +// Perform a send/recv test. Recv blocks, so they need to be executed +// asynchronously. +TEST(CAPI_EXPERIMENTAL, TFE_ExecuteOpInNewThreadTest_Blocking) { + TF_Status* status = TF_NewStatus(); + TFE_ContextOptions* opts = TFE_NewContextOptions(); + CHECK_EQ(TF_OK, TF_GetCode(status)) << TF_Message(status); + TFE_Context* ctx = TFE_NewContext(opts, status); + CHECK_EQ(TF_OK, TF_GetCode(status)) << TF_Message(status); + TFE_DeleteContextOptions(opts); + + // Returns a 2x2 float32 Tensor on the CPU, with data 1., 2., 3., 4. + TFE_TensorHandle* m = TestMatrixTensorHandle(); + + // Build a send op. + TFE_Op* send_op = TFE_NewOp(ctx, "_Send", status); + CHECK_EQ(TF_OK, TF_GetCode(status)) << TF_Message(status); + TFE_OpAddInput(send_op, m, status); + CHECK_EQ(TF_OK, TF_GetCode(status)) << TF_Message(status); + + string tensor_name = "Tensor"; + TFE_OpSetAttrType(send_op, "T", TF_FLOAT); + TFE_OpSetAttrString(send_op, "tensor_name", tensor_name.c_str(), + tensor_name.size()); + string send_device = "/job:localhost/replica:0/task:0/device:CPU:0"; + TFE_OpSetAttrString(send_op, "send_device", send_device.c_str(), + send_device.size()); + TFE_OpSetAttrInt(send_op, "send_device_incarnation", 1234); + string recv_device = "/job:localhost/replica:0/task:0/device:CPU:0"; + TFE_OpSetAttrString(send_op, "recv_device", recv_device.c_str(), + recv_device.size()); + TFE_OpSetAttrBool(send_op, "client_terminated", true); + + // Build a recv op. + TFE_Op* recv_op = TFE_NewOp(ctx, "_Recv", status); + CHECK_EQ(TF_OK, TF_GetCode(status)) << TF_Message(status); + + TFE_OpSetAttrType(recv_op, "tensor_type", TF_FLOAT); + TFE_OpSetAttrString(recv_op, "tensor_name", tensor_name.c_str(), + tensor_name.size()); + TFE_OpSetAttrString(recv_op, "send_device", send_device.c_str(), + send_device.size()); + TFE_OpSetAttrInt(recv_op, "send_device_incarnation", 1234); + TFE_OpSetAttrString(recv_op, "recv_device", recv_device.c_str(), + recv_device.size()); + TFE_OpSetAttrBool(recv_op, "client_terminated", true); + + TFE_TensorHandle* send_retvals; + int send_num_retvals = 0; + auto* send_result = TFE_ExecuteOpInNewThread(send_op, &send_retvals, + &send_num_retvals, status); + + TFE_TensorHandle* recv_retvals[1] = {nullptr}; + int recv_num_retvals = 1; + auto* recv_result = TFE_ExecuteOpInNewThread(recv_op, &recv_retvals[0], + &recv_num_retvals, status); + + TFE_ExecuteOpNotificationWaitAndDelete(send_result, status); + CHECK_EQ(TF_OK, TF_GetCode(status)) << TF_Message(status); + TFE_ExecuteOpNotificationWaitAndDelete(recv_result, status); + CHECK_EQ(TF_OK, TF_GetCode(status)) << TF_Message(status); + + TF_Tensor* t = TFE_TensorHandleResolve(recv_retvals[0], status); + ASSERT_EQ(TF_OK, TF_GetCode(status)) << TF_Message(status); + + float product[4] = {0}; + EXPECT_EQ(sizeof(product), TF_TensorByteSize(t)); + memcpy(&product[0], TF_TensorData(t), TF_TensorByteSize(t)); + TF_DeleteTensor(t); + EXPECT_EQ(1, product[0]); + EXPECT_EQ(2, product[1]); + EXPECT_EQ(3, product[2]); + EXPECT_EQ(4, product[3]); + + TFE_DeleteOp(send_op); + TFE_DeleteOp(recv_op); + TFE_DeleteTensorHandle(m); + + TFE_DeleteTensorHandle(recv_retvals[0]); + TFE_DeleteContext(ctx); + TF_DeleteStatus(status); +} + } // namespace } // namespace tensorflow diff --git a/tensorflow/c/c_api_function.cc b/tensorflow/c/c_api_function.cc index f68f8a3e90a..28b9f8df9c8 100644 --- a/tensorflow/c/c_api_function.cc +++ b/tensorflow/c/c_api_function.cc @@ -392,26 +392,26 @@ Status ProcessInputs( EXCLUSIVE_LOCKS_REQUIRED(fn_body->mu) { input_tensors->reserve(ninputs); for (int i = 0; i < ninputs; ++i) { - const Node& node = inputs[i].oper->node; + Node* node = &inputs[i].oper->node; int idx = inputs[i].index; TF_RETURN_WITH_CONTEXT_IF_ERROR( - fn_body->graph.IsValidOutputTensor(&node, idx), + fn_body->graph.IsValidOutputTensor(node, idx), "Encountered while processing input ", i, " into function '", fn_name, "'"); - TF_RETURN_WITH_CONTEXT_IF_ERROR(ValidateNonRefOutput(&node, idx), + TF_RETURN_WITH_CONTEXT_IF_ERROR(ValidateNonRefOutput(node, idx), "Encountered while processing input ", i, " into function '", fn_name, "'"); - input_tensors->emplace_back(&node, idx); + input_tensors->emplace_back(node, idx); - const auto& iter = input_nodes->find(&node); + const auto& iter = input_nodes->find(node); if (iter == input_nodes->end()) { - input_nodes->insert({&node, {idx}}); + input_nodes->insert({node, {idx}}); } else { auto& indices = iter->second; if (std::find(indices.begin(), indices.end(), idx) != indices.end()) { - return InvalidArgument("TF_Output ", node.name(), ":", idx, + return InvalidArgument("TF_Output ", node->name(), ":", idx, " appears more than once in the input list"); } indices.push_back(idx); @@ -428,16 +428,16 @@ Status ProcessOutputs(const TF_Graph* fn_body, const char* fn_name, EXCLUSIVE_LOCKS_REQUIRED(fn_body->mu) { output_tensors->reserve(noutputs); for (int i = 0; i < noutputs; ++i) { - const Node& node = outputs[i].oper->node; + Node* node = &outputs[i].oper->node; int idx = outputs[i].index; TF_RETURN_WITH_CONTEXT_IF_ERROR( - fn_body->graph.IsValidOutputTensor(&node, idx), + fn_body->graph.IsValidOutputTensor(node, idx), "Encountered while processing output ", i, " from function '", fn_name, "'"); - TF_RETURN_WITH_CONTEXT_IF_ERROR(ValidateNonRefOutput(&node, idx), + TF_RETURN_WITH_CONTEXT_IF_ERROR(ValidateNonRefOutput(node, idx), "Encountered while creating function '", fn_name, "'"); - output_tensors->emplace_back(&node, idx); + output_tensors->emplace_back(node, idx); } return Status::OK(); } diff --git a/tensorflow/c/eager/BUILD b/tensorflow/c/eager/BUILD index ba3d8533db7..c34a84fcfee 100644 --- a/tensorflow/c/eager/BUILD +++ b/tensorflow/c/eager/BUILD @@ -50,6 +50,7 @@ tf_cuda_library( ], "//conditions:default": [], }) + [ + "@com_google_absl//absl/memory", "//tensorflow/core/common_runtime/eager:eager_operation", "//tensorflow/core/distributed_runtime/eager:eager_client", "//tensorflow/core/distributed_runtime/rpc/eager:grpc_eager_client", @@ -143,6 +144,7 @@ tf_cuda_cc_test( "//tensorflow/core:test", "//tensorflow/core:test_main", "//tensorflow/core/distributed_runtime/rpc:grpc_server_lib", + "@com_google_absl//absl/strings", ], ) diff --git a/tensorflow/c/eager/c_api.cc b/tensorflow/c/eager/c_api.cc index 408277468d7..027d752f420 100755 --- a/tensorflow/c/eager/c_api.cc +++ b/tensorflow/c/eager/c_api.cc @@ -21,9 +21,11 @@ limitations under the License. #include #include +#include "absl/memory/memory.h" #include "tensorflow/c/c_api.h" #include "tensorflow/c/c_api_internal.h" #include "tensorflow/c/eager/c_api_internal.h" +#include "tensorflow/core/platform/host_info.h" #ifdef TENSORFLOW_EAGER_USE_XLA #include "tensorflow/compiler/tf2xla/xla_op_registry.h" #endif // TENSORFLOW_EAGER_USE_XLA @@ -79,7 +81,7 @@ tensorflow::Status GetAllRemoteDevices( const std::vector& remote_workers, tensorflow::WorkerCacheInterface* worker_cache, std::unique_ptr* device_mgr) { - std::vector remote_devices; + std::vector> remote_devices; tensorflow::Status status; // TODO(nareshmodi) do this in parallel instead of serially. for (const string& remote_worker : remote_workers) { @@ -92,7 +94,7 @@ tensorflow::Status GetAllRemoteDevices( status = s; if (s.ok()) { for (tensorflow::Device* d : *devices) { - remote_devices.push_back(d); + remote_devices.emplace_back(d); } } n.Notify(); @@ -100,7 +102,7 @@ tensorflow::Status GetAllRemoteDevices( n.WaitForNotification(); } std::unique_ptr remote_device_mgr( - new tensorflow::DeviceMgr(remote_devices)); + new tensorflow::DeviceMgr(std::move(remote_devices))); TF_RETURN_IF_ERROR(status); @@ -261,13 +263,13 @@ TF_CAPI_EXPORT extern void TFE_ContextSetAsyncForThread(TFE_Context* ctx, void TFE_DeleteContextOptions(TFE_ContextOptions* options) { delete options; } TFE_Context* TFE_NewContext(const TFE_ContextOptions* opts, TF_Status* status) { - std::vector devices; + std::vector> devices; status->status = tensorflow::DeviceFactory::AddDevices( opts->session_options.options, "/job:localhost/replica:0/task:0", &devices); if (!status->status.ok()) return nullptr; std::unique_ptr device_mgr( - new tensorflow::DeviceMgr(devices)); + new tensorflow::DeviceMgr(std::move(devices))); tensorflow::Rendezvous* r = new tensorflow::IntraProcessRendezvous(device_mgr.get()); @@ -409,6 +411,18 @@ const char* TFE_TensorHandleDeviceName(TFE_TensorHandle* h, TF_Status* status) { : d->name().c_str(); } +const char* TFE_TensorHandleBackingDeviceName(TFE_TensorHandle* h, + TF_Status* status) { + if (h == nullptr || h->handle == nullptr) { + status->status = tensorflow::errors::InvalidArgument( + "The passed in handle is a nullptr"); + return nullptr; + } + tensorflow::Device* d = h->handle->device(); + return (d == nullptr) ? "/job:localhost/replica:0/task:0/device:CPU:0" + : d->name().c_str(); +} + TF_CAPI_EXPORT extern TFE_TensorHandle* TFE_TensorHandleCopySharingTensor( TFE_TensorHandle* h, TF_Status* status) { if (h == nullptr || h->handle == nullptr) { @@ -458,13 +472,20 @@ TFE_Op* TFE_NewOp(TFE_Context* ctx, const char* op_or_function_name, TF_Status* status) { const char* name = op_or_function_name; // Shorthand const tensorflow::AttrTypeMap* types; - status->status = tensorflow::AttrTypeMapForOp(name, &types); - if (status->status.ok()) return new TFE_Op(ctx, name, types); - if (TF_GetCode(status) == TF_NOT_FOUND) { - if (ctx->context.FindFunctionByName(name)) { - status->status = tensorflow::Status::OK(); - return new TFE_Op(ctx, name, nullptr); + bool is_function = false; + status->status = tensorflow::AttrTypeMapForOp(name, &types, &is_function); + if (status->status.ok()) { + if (is_function && !ctx->context.FindFunctionByName(name)) { + status->status = tensorflow::errors::NotFound( + "'", name, + "' is neither a type of a primitive operation nor a name " + "of a function registered in binary running on ", + tensorflow::port::Hostname(), + ". Make sure the operation or function is " + "registered in the binary running in this process."); + return nullptr; } + return new TFE_Op(ctx, name, is_function, types); } return nullptr; } @@ -497,12 +518,6 @@ void TFE_OpAddInput(TFE_Op* op, TFE_TensorHandle* h, TF_Status* status) { TF_AttrType TFE_OpGetAttrType(TFE_Op* op, const char* attr_name, unsigned char* is_list, TF_Status* status) { TF_AttrType ret; - if (op->operation.is_function()) { - status->status = tensorflow::errors::Unimplemented( - "TODO(apassos): Support for attributes for TensorFlow functions is not " - "ready yet."); - return TF_ATTR_INT; // The compiler requires that we return something. - } status->status = tensorflow::AttrTypeByName(*op->operation.AttrTypes(), attr_name, &ret, is_list); return ret; diff --git a/tensorflow/c/eager/c_api.h b/tensorflow/c/eager/c_api.h index b2454d87220..8d6c8d958d5 100755 --- a/tensorflow/c/eager/c_api.h +++ b/tensorflow/c/eager/c_api.h @@ -169,10 +169,33 @@ TF_CAPI_EXPORT extern int64_t TFE_TensorHandleNumElements(TFE_TensorHandle* h, TF_CAPI_EXPORT extern int64_t TFE_TensorHandleDim(TFE_TensorHandle* h, int dim_index, TF_Status* status); + +// Returns the device of the operation that produced `h`. +// If `h` was produced by a copy, returns the destination device of +// the copy. Note that returned device name is not always the device +// holding the tensor handle's memory. If you want the latter, use +// TFE_TensorHandleBackingDeviceName. +// This function will block till the operation that produces `h` has completed. +// +// Device on which the kernel of the operation that produced `h` ran. +// +// If `h` was produced by a copy, returns the destination device of +// the copy. +// +// Note that returned device name is not always the device that owns the memory +// that backs the tensor handle. For the latter see +// TFE_TensorHandleBackingDeviceName. +// // This function will block till the operation that produces `h` has completed. TF_CAPI_EXPORT extern const char* TFE_TensorHandleDeviceName( TFE_TensorHandle* h, TF_Status* status); +// Returns the name of the device in whose memory `h` resides. +// +// This function will block till the operation that produces `h` has completed. +TF_CAPI_EXPORT extern const char* TFE_TensorHandleBackingDeviceName( + TFE_TensorHandle* h, TF_Status* status); + // Return a pointer to a new TFE_TensorHandle that shares the underlying tensor // with `h`. On success, `status` is set to OK. On failure, `status` reflects // the error and a nullptr is returned. diff --git a/tensorflow/c/eager/c_api_internal.h b/tensorflow/c/eager/c_api_internal.h index fa1b22e3af4..67bc1bcd246 100644 --- a/tensorflow/c/eager/c_api_internal.h +++ b/tensorflow/c/eager/c_api_internal.h @@ -93,10 +93,9 @@ struct TFE_TensorDebugInfo { }; struct TFE_Op { - // t is NULL iff the TFE_Op corresponds to a TensorFlow function instead of a - // primitive operation. - TFE_Op(TFE_Context* ctx, const char* op, const tensorflow::AttrTypeMap* t) - : operation(&ctx->context, op, t) {} + TFE_Op(TFE_Context* ctx, const char* op, bool is_function, + const tensorflow::AttrTypeMap* t) + : operation(&ctx->context, op, is_function, t) {} tensorflow::EagerOperation operation; }; diff --git a/tensorflow/c/eager/c_api_test.cc b/tensorflow/c/eager/c_api_test.cc index 55331022b9d..6b39b79ee82 100644 --- a/tensorflow/c/eager/c_api_test.cc +++ b/tensorflow/c/eager/c_api_test.cc @@ -16,6 +16,7 @@ limitations under the License. #include "tensorflow/c/eager/c_api.h" #include +#include "absl/strings/match.h" #include "tensorflow/c/eager/c_api_test_util.h" #include "tensorflow/core/distributed_runtime/rpc/grpc_server_lib.h" #include "tensorflow/core/framework/function.pb.h" @@ -589,9 +590,22 @@ void TensorHandleCopyBetweenTwoGPUDevices(bool async) { TF_DeviceList* devices = TFE_ContextListDevices(ctx, status.get()); ASSERT_EQ(TF_OK, TF_GetCode(status.get())) << TF_Message(status.get()); const int num_devices = TF_DeviceListCount(devices); + bool has_gpu0 = false; + bool has_gpu1 = false; + for (int i = 0; i < num_devices; ++i) { + const char* dev = TF_DeviceListName(devices, i, status.get()); + ASSERT_EQ(TF_OK, TF_GetCode(status.get())) << TF_Message(status.get()); + string device_name(dev); + if (device_name.find("GPU:0") != string::npos) { + has_gpu0 = true; + } + if (device_name.find("GPU:1") != string::npos) { + has_gpu1 = true; + } + } const char* kCPUDevice = "CPU:0"; - if (num_devices < 3) { + if (!has_gpu0 || !has_gpu1) { TF_DeleteDeviceList(devices); TF_DeleteTensor(t); TFE_DeleteTensorHandle(hcpu); @@ -781,6 +795,14 @@ TEST(CAPI, TensorHandleNullptr) { TF_SetStatus(status.get(), TF_OK, ""); + device_name = TFE_TensorHandleBackingDeviceName(h, status.get()); + ASSERT_EQ(TF_INVALID_ARGUMENT, TF_GetCode(status.get())); + ASSERT_EQ(device_name, nullptr); + ASSERT_EQ("The passed in handle is a nullptr", + string(TF_Message(status.get()))); + + TF_SetStatus(status.get(), TF_OK, ""); + int num_dims = TFE_TensorHandleNumDims(h, status.get()); ASSERT_EQ(TF_INVALID_ARGUMENT, TF_GetCode(status.get())); ASSERT_EQ(num_dims, -1); @@ -796,6 +818,62 @@ TEST(CAPI, TensorHandleNullptr) { string(TF_Message(status.get()))); } +TEST(CAPI, TensorHandleDevices) { + std::unique_ptr status( + TF_NewStatus(), TF_DeleteStatus); + TFE_ContextOptions* opts = TFE_NewContextOptions(); + TFE_Context* ctx = TFE_NewContext(opts, status.get()); + TFE_DeleteContextOptions(opts); + ASSERT_EQ(TF_OK, TF_GetCode(status.get())) << TF_Message(status.get()); + + TFE_TensorHandle* hcpu = TestMatrixTensorHandle(); + const char* device_name = TFE_TensorHandleDeviceName(hcpu, status.get()); + ASSERT_EQ(TF_OK, TF_GetCode(status.get())) << TF_Message(status.get()); + ASSERT_TRUE(absl::StrContains(device_name, "CPU:0")) << device_name; + const char* backing_device_name = + TFE_TensorHandleBackingDeviceName(hcpu, status.get()); + ASSERT_EQ(TF_OK, TF_GetCode(status.get())) << TF_Message(status.get()); + ASSERT_TRUE(absl::StrContains(backing_device_name, "CPU:0")) + << backing_device_name; + + // Disable the test if no GPU is present. + string gpu_device_name; + if (GetDeviceName(ctx, &gpu_device_name, "GPU")) { + TFE_TensorHandle* hgpu = TFE_TensorHandleCopyToDevice( + hcpu, ctx, gpu_device_name.c_str(), status.get()); + ASSERT_TRUE(TF_GetCode(status.get()) == TF_OK) << TF_Message(status.get()); + + TFE_Op* shape_op = ShapeOp(ctx, hgpu); + TFE_OpSetDevice(shape_op, gpu_device_name.c_str(), status.get()); + ASSERT_TRUE(TF_GetCode(status.get()) == TF_OK) << TF_Message(status.get()); + TFE_TensorHandle* retvals[1]; + int num_retvals = 1; + TFE_Execute(shape_op, &retvals[0], &num_retvals, status.get()); + ASSERT_TRUE(TF_GetCode(status.get()) == TF_OK) << TF_Message(status.get()); + + // .device of shape is GPU since the op is executed on GPU + device_name = TFE_TensorHandleDeviceName(retvals[0], status.get()); + ASSERT_EQ(TF_OK, TF_GetCode(status.get())) << TF_Message(status.get()); + ASSERT_TRUE(absl::StrContains(device_name, "GPU:0")) << device_name; + + // .backing_device of shape is CPU since the tensor is backed by CPU + backing_device_name = + TFE_TensorHandleBackingDeviceName(retvals[0], status.get()); + ASSERT_EQ(TF_OK, TF_GetCode(status.get())) << TF_Message(status.get()); + ASSERT_TRUE(absl::StrContains(backing_device_name, "CPU:0")) + << backing_device_name; + + TFE_DeleteOp(shape_op); + TFE_DeleteTensorHandle(retvals[0]); + TFE_DeleteTensorHandle(hgpu); + } + + TFE_DeleteTensorHandle(hcpu); + TFE_ContextAsyncWait(ctx, status.get()); + EXPECT_EQ(TF_OK, TF_GetCode(status.get())) << TF_Message(status.get()); + TFE_DeleteContext(ctx); +} + void Execute_MatMul_CPU(bool async) { TF_Status* status = TF_NewStatus(); TFE_ContextOptions* opts = TFE_NewContextOptions(); diff --git a/tensorflow/c/eager/c_api_test_util.cc b/tensorflow/c/eager/c_api_test_util.cc index 008f088c2dc..bd38127d50c 100644 --- a/tensorflow/c/eager/c_api_test_util.cc +++ b/tensorflow/c/eager/c_api_test_util.cc @@ -104,6 +104,19 @@ TFE_Op* MatMulOp(TFE_Context* ctx, TFE_TensorHandle* a, TFE_TensorHandle* b) { return op; } +TFE_Op* ShapeOp(TFE_Context* ctx, TFE_TensorHandle* a) { + TF_Status* status = TF_NewStatus(); + + TFE_Op* op = TFE_NewOp(ctx, "Shape", status); + CHECK_EQ(TF_OK, TF_GetCode(status)) << TF_Message(status); + TFE_OpAddInput(op, a, status); + CHECK_EQ(TF_OK, TF_GetCode(status)) << TF_Message(status); + TF_DeleteStatus(status); + TFE_OpSetAttrType(op, "T", TFE_TensorHandleDataType(a)); + + return op; +} + TFE_TensorHandle* TestAxisTensorHandle() { int64_t dims[] = {1}; int data[] = {1}; diff --git a/tensorflow/c/eager/c_api_test_util.h b/tensorflow/c/eager/c_api_test_util.h index 474cae67c89..75ef9459e93 100644 --- a/tensorflow/c/eager/c_api_test_util.h +++ b/tensorflow/c/eager/c_api_test_util.h @@ -37,6 +37,9 @@ TFE_TensorHandle* TestMatrixTensorHandle3X2(); // Return a matmul op multiplying `a` by `b`. TFE_Op* MatMulOp(TFE_Context* ctx, TFE_TensorHandle* a, TFE_TensorHandle* b); +// Return a shape op fetching the shape of `a`. +TFE_Op* ShapeOp(TFE_Context* ctx, TFE_TensorHandle* a); + // Return an 1-D INT32 tensor containing a single value 1. TFE_TensorHandle* TestAxisTensorHandle(); diff --git a/tensorflow/c/eager/tape.h b/tensorflow/c/eager/tape.h index 5ba55a203ff..5c11f51e874 100644 --- a/tensorflow/c/eager/tape.h +++ b/tensorflow/c/eager/tape.h @@ -141,8 +141,9 @@ class GradientTape { // null. The result is populated with one tensor per target element. Status ComputeGradient( const VSpace& vspace, - gtl::ArraySlice target_tensor_ids, - gtl::ArraySlice source_tensor_id, + const gtl::ArraySlice target_tensor_ids, + const gtl::ArraySlice source_tensor_ids, + const gtl::FlatMap sources_that_are_targets, gtl::ArraySlice output_gradients, std::vector* result); @@ -396,6 +397,7 @@ template Status InitialGradients( const VSpace& vspace, gtl::ArraySlice target_tensor_ids, + gtl::FlatMap sources_that_are_targets, gtl::ArraySlice output_gradients, const TensorTape& tensor_tape, const OpTape& op_tape, gtl::FlatMap>* result) { @@ -425,8 +427,13 @@ Status InitialGradients( "none of operations outputs match expected tensor"); } } else { - // No record of the target tensor found on the tape, so no gradient - // needs to be computed from it. Do nothing. + // This target tensor was not generated by any operation recorded on + // the tape, so no gradient needs to be computed from it unless this + // target is also a source. + auto source_tensor = sources_that_are_targets.find(id); + if (source_tensor != sources_that_are_targets.end()) { + (*result)[id].push_back(vspace.Ones(source_tensor->second)); + } } } else { (*result)[id].push_back(output_gradients[i]); @@ -467,8 +474,9 @@ constexpr int kMinAggregateBytes = 128 * 1024 * 1024; template Status GradientTape::ComputeGradient( const VSpace& vspace, - gtl::ArraySlice target_tensor_ids, - gtl::ArraySlice source_tensor_ids, + const gtl::ArraySlice target_tensor_ids, + const gtl::ArraySlice source_tensor_ids, + const gtl::FlatMap sources_that_are_targets, gtl::ArraySlice output_gradients, std::vector* result) { gtl::FlatSet sources_set(source_tensor_ids.begin(), @@ -478,7 +486,8 @@ Status GradientTape::ComputeGradient( std::vector op_stack = InitialStack(state.op_tape, state.op_missing_tensor); gtl::FlatMap> gradients; - Status s = InitialGradients(vspace, target_tensor_ids, output_gradients, + Status s = InitialGradients(vspace, target_tensor_ids, + sources_that_are_targets, output_gradients, tensor_tape_, state.op_tape, &gradients); auto cleanup = [this, &state]() { if (!persistent_) { diff --git a/tensorflow/c/kernels.cc b/tensorflow/c/kernels.cc new file mode 100644 index 00000000000..3caa5bcb038 --- /dev/null +++ b/tensorflow/c/kernels.cc @@ -0,0 +1,143 @@ +/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include + +#include "tensorflow/c/c_api_internal.h" +#include "tensorflow/c/kernels.h" +#include "tensorflow/core/framework/kernel_def_builder.h" +#include "tensorflow/core/framework/op_kernel.h" + +// This file forms the basis of a stable ABI for third-party kernel +// implementations. It is crucial that changes to this file are made cautiously +// and with a focus on maintaining both source and binary compatibility. + +struct TF_KernelBuilder { + ::tensorflow::KernelDefBuilder* cc_builder; + + void* (*create_function)(TF_OpKernelConstruction*); + void (*compute_function)(void*, TF_OpKernelContext*); + void (*delete_function)(void*); +}; + +TF_KernelBuilder* TF_NewKernelBuilder( + const char* op_name, const char* device_name, + void* (*create_func)(TF_OpKernelConstruction*), + void (*compute_func)(void*, TF_OpKernelContext*), + void (*delete_func)(void*)) { + TF_KernelBuilder* result = new TF_KernelBuilder; + result->cc_builder = new ::tensorflow::KernelDefBuilder(op_name); + result->cc_builder->Device(device_name); + result->create_function = create_func; + result->compute_function = compute_func; + result->delete_function = delete_func; + return result; +} + +void TF_DeleteKernelBuilder(TF_KernelBuilder* builder) { + DCHECK_NE(builder, nullptr); + delete builder->cc_builder; + delete builder; +} + +namespace tensorflow { +namespace { + +// An OpKernel whose methods delegate to C function pointers. +class COpKernel : public OpKernel { + public: + explicit COpKernel(OpKernelConstruction* ctx, + void* (*create_func)(TF_OpKernelConstruction*), + void (*compute_func)(void*, TF_OpKernelContext*), + void (*delete_func)(void*)) + : OpKernel(ctx), compute_func_(compute_func), delete_func_(delete_func) { + if (create_func != nullptr) { + c_kernel_ = + (*create_func)(reinterpret_cast(ctx)); + } else { + c_kernel_ = nullptr; + } + } + + void Compute(OpKernelContext* ctx) override { + (*compute_func_)(c_kernel_, reinterpret_cast(ctx)); + } + + ~COpKernel() override { + if (delete_func_ != nullptr) { + (*delete_func_)(c_kernel_); + } + } + + private: + void (*compute_func_)(void*, TF_OpKernelContext* context); + void (*delete_func_)(void*); + void* c_kernel_; +}; + +// A KernelFactory that returns COpKernel instances. +class KernelBuilderFactory + : public ::tensorflow::kernel_factory::OpKernelFactory { + public: + explicit KernelBuilderFactory(TF_KernelBuilder* builder) + : builder_(builder) {} + ::tensorflow::OpKernel* Create( + ::tensorflow::OpKernelConstruction* context) override { + return new ::tensorflow::COpKernel(context, builder_->create_function, + builder_->compute_function, + builder_->delete_function); + } + ~KernelBuilderFactory() override { TF_DeleteKernelBuilder(builder_); } + + private: + TF_KernelBuilder* builder_; +}; +} // namespace +} // namespace tensorflow + +void TF_RegisterKernelBuilder(const char* name, TF_KernelBuilder* builder, + TF_Status* status) { + using tensorflow::register_kernel::Name; + + tensorflow::kernel_factory::OpKernelRegistrar( + builder->cc_builder->Build(), name, + absl::make_unique(builder)); + + TF_SetStatus(status, TF_OK, ""); +} + +int TF_NumInputs(TF_OpKernelContext* ctx) { + auto* cc_ctx = reinterpret_cast<::tensorflow::OpKernelContext*>(ctx); + return cc_ctx->num_inputs(); +} + +int TF_NumOutputs(TF_OpKernelContext* ctx) { + auto* cc_ctx = reinterpret_cast<::tensorflow::OpKernelContext*>(ctx); + return cc_ctx->num_outputs(); +} + +void TF_GetInput(TF_OpKernelContext* ctx, int i, TF_Tensor** tensor, + TF_Status* status) { + auto* cc_ctx = reinterpret_cast<::tensorflow::OpKernelContext*>(ctx); + if (i < 0 || i >= cc_ctx->num_inputs()) { + TF_SetStatus(status, TF_OUT_OF_RANGE, "input index out of range"); + return; + } + const ::tensorflow::Tensor& cc_tensor(cc_ctx->input(i)); + TF_Tensor* result = ::tensorflow::TF_TensorFromTensor(cc_tensor, status); + if (TF_GetCode(status) == TF_OK) { + *tensor = result; + } +} diff --git a/tensorflow/c/kernels.h b/tensorflow/c/kernels.h new file mode 100644 index 00000000000..d7778829bca --- /dev/null +++ b/tensorflow/c/kernels.h @@ -0,0 +1,110 @@ +/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_C_KERNELS_H_ +#define TENSORFLOW_C_KERNELS_H_ + +#include "tensorflow/c/c_api.h" + +#ifdef __cplusplus +extern "C" { +#endif + +// -------------------------------------------------------------------------- +// C API for TensorFlow Kernels. +// +// This API allows developers to register custom kernel implementations for +// TensorFlow. +// +// See c_api.h header comments for a discussion about API conventions. +// +// Users wishing to extend TensorFlow with new kernels will call +// `TF_NewKernelBuilder`. The resulting kernel builder can be registered with +// `TF_RegisterKernelBuilder`, which will allow TF to construct user-provided +// kernels when necessary. + +struct TF_KernelBuilder; +struct TF_OpKernelConstruction; +struct TF_OpKernelContext; + +// Allocates a new kernel builder and returns a pointer to it. +// +// If non-null, TensorFlow will call create_func when it needs to instantiate +// the kernel. The pointer returned by create_func will be passed to +// compute_func and delete_func, thereby functioning as a "this" pointer for +// referring to kernel instances. +// +// The TF_OpKernelConstruction pointer passed to create_func is owned by +// TensorFlow and will be deleted once create_func returns. It must not be used +// after this. +// +// When TensorFlow needs to perform a computation with this kernel, it will +// call compute_func. This function will receive the pointer returned by +// create_func (or null if no create_func was provided), along with the inputs +// to the computation. +// +// The TF_OpKernelContext pointer received by compute_func is owned by +// TensorFlow and will be deleted once compute_func returns. It must not be used +// after this. +// +// Finally, when TensorFlow no longer needs the kernel, it will call +// delete_func if one is provided. This function will receive the pointer +// returned in `create_func` or nullptr if no `create_func` was provided. +// +// The caller should pass the result of this function to +// TF_RegisterKernelBuilder, which will take ownership of the pointer. If, for +// some reason, the kernel builder will not be registered, the caller should +// delete it with TF_DeleteKernelBuilder. +TF_CAPI_EXPORT extern TF_KernelBuilder* TF_NewKernelBuilder( + const char* op_name, const char* device_name, + void* (*create_func)(TF_OpKernelConstruction*), + void (*compute_func)(void*, TF_OpKernelContext*), + void (*delete_func)(void*)); + +// Register the given kernel builder with the TensorFlow runtime. If +// registration fails, the given status will be populated. +// +// This call takes ownership of the `builder` pointer. +TF_CAPI_EXPORT extern void TF_RegisterKernelBuilder(const char* kernel_name, + TF_KernelBuilder* builder, + TF_Status* status); + +// Deletes the given TF_KernelBuilder. This should be called only if the kernel +// builder is not registered with TensorFlow via TF_RegisterKernelBuilder. +TF_CAPI_EXPORT extern void TF_DeleteKernelBuilder(TF_KernelBuilder* builder); + +// -------------------------------------------------------------------------- +// OpKernelContext routines + +// TF_NumInputs returns the number of inputs available in ctx. +TF_CAPI_EXPORT extern int TF_NumInputs(TF_OpKernelContext* ctx); + +// TF_NumOutputs returns the number of outputs to be placed in *ctx by the +// kernel. +TF_CAPI_EXPORT extern int TF_NumOutputs(TF_OpKernelContext* ctx); + +// Retrieves the ith input from ctx. If TF_GetCode(status) is TF_OK, *tensor is +// populated and its ownership is passed to the caller. In any other case, +// *tensor is not modified. +// +// If i < 0 or i >= TF_NumInputs(ctx), *status is set to TF_OUT_OF_RANGE. +TF_CAPI_EXPORT extern void TF_GetInput(TF_OpKernelContext* ctx, int i, + TF_Tensor** tensor, TF_Status* status); + +#ifdef __cplusplus +} /* end extern "C" */ +#endif + +#endif // TENSORFLOW_C_KERNELS_H_ diff --git a/tensorflow/c/kernels_test.cc b/tensorflow/c/kernels_test.cc new file mode 100644 index 00000000000..80bf12c0969 --- /dev/null +++ b/tensorflow/c/kernels_test.cc @@ -0,0 +1,194 @@ +/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/c/kernels.h" + +#include "tensorflow/c/c_api.h" +#include "tensorflow/core/framework/kernel_def.pb.h" +#include "tensorflow/core/framework/node_def.pb_text.h" +#include "tensorflow/core/framework/op.h" +#include "tensorflow/core/framework/op_kernel.h" +#include "tensorflow/core/framework/types.h" +#include "tensorflow/core/lib/core/status_test_util.h" +#include "tensorflow/core/platform/test.h" + +struct MyCustomKernel { + bool created; + bool compute_called; +}; + +static bool delete_called = false; + +static void* MyCreateFunc(TF_OpKernelConstruction* ctx) { + struct MyCustomKernel* s = new struct MyCustomKernel; + s->created = true; + s->compute_called = false; + return s; +} + +static void MyComputeFunc(void* kernel, TF_OpKernelContext* ctx) { + struct MyCustomKernel* s = static_cast(kernel); + s->compute_called = true; +} + +static void MyDeleteFunc(void* kernel) { + struct MyCustomKernel* s = static_cast(kernel); + EXPECT_TRUE(s->created); + EXPECT_TRUE(s->compute_called); + delete_called = true; + delete s; +} + +namespace tensorflow { + +static std::unique_ptr GetFakeKernel(const char* device_name, + const char* op_name, + Status* status) { + NodeDef def; + def.set_op(op_name); + def.set_device(device_name); + def.add_input("input1"); + def.add_input("input2"); + return CreateOpKernel(DeviceType(device_name), nullptr, nullptr, def, 1, + status); +} + +// Tests registration of a single C kernel and checks that calls through the +// C/C++ boundary are being made. +TEST(TestKernel, TestRegisterKernelBuilder) { + const char* kernel_name = "SomeKernelName"; + const char* op_name = "FooOp"; + const char* device_name = "FakeDeviceName1"; + + REGISTER_OP(op_name) + .Input("input1: double") + .Input("input2: uint8") + .Output("output1: uint8"); + + TF_KernelBuilder* builder = TF_NewKernelBuilder( + op_name, device_name, &MyCreateFunc, &MyComputeFunc, &MyDeleteFunc); + + { + TF_Status* status = TF_NewStatus(); + TF_RegisterKernelBuilder(kernel_name, builder, status); + EXPECT_EQ(TF_OK, TF_GetCode(status)); + TF_Buffer* buf = TF_GetRegisteredKernelsForOp(op_name, status); + EXPECT_EQ(TF_OK, TF_GetCode(status)); + KernelList list; + list.ParseFromArray(buf->data, buf->length); + ASSERT_EQ(1, list.kernel_size()); + ASSERT_EQ(device_name, list.kernel(0).device_type()); + TF_DeleteBuffer(buf); + TF_DeleteStatus(status); + } + + { + Status status; + std::unique_ptr kernel = + GetFakeKernel(device_name, op_name, &status); + TF_EXPECT_OK(status); + ASSERT_NE(nullptr, kernel.get()); + kernel->Compute(nullptr); + } + + ASSERT_TRUE(delete_called); +} + +class DummyDevice : public DeviceBase { + public: + DummyDevice(Env* env, bool save) : DeviceBase(env), save_(save) {} + bool RequiresRecordingAccessedTensors() const override { return save_; } + Allocator* GetAllocator(AllocatorAttributes /*attr*/) override { + return cpu_allocator(); + } + + private: + bool save_; +}; + +TEST(TestKernel, TestInputAndOutputCount) { + const char* kernel_name = "InputOutputCounterKernel"; + const char* op_name = "BarOp"; + const char* device_name = "FakeDeviceName2"; + + REGISTER_OP(op_name) + .Input("input1: double") + .Input("input2: uint8") + .Output("output1: uint8"); + + static int num_inputs = 0; + static int num_outputs = 0; + + // A kernel whose Compute function has a side-effect of updating num_inputs + // and num_outputs. Various functions on TF_OpKernelContext are also + // exercised. + auto my_compute_func = [](void* kernel, TF_OpKernelContext* ctx) { + num_inputs = TF_NumInputs(ctx); + num_outputs = TF_NumOutputs(ctx); + + TF_Tensor* input = nullptr; + TF_Status* s = TF_NewStatus(); + TF_GetInput(ctx, 0, &input, s); + EXPECT_EQ(TF_OK, TF_GetCode(s)) << "Failed to get input: " << TF_Message(s); + EXPECT_EQ(123, *static_cast(TF_TensorData(input))); + TF_GetInput(ctx, -1, &input, s); + EXPECT_EQ(TF_OUT_OF_RANGE, TF_GetCode(s)); + TF_GetInput(ctx, 3, &input, s); + EXPECT_EQ(TF_OUT_OF_RANGE, TF_GetCode(s)); + TF_DeleteStatus(s); + if (input != nullptr) { + TF_DeleteTensor(input); + } + }; + + TF_KernelBuilder* builder = TF_NewKernelBuilder(op_name, device_name, nullptr, + my_compute_func, nullptr); + + { + TF_Status* status = TF_NewStatus(); + TF_RegisterKernelBuilder(kernel_name, builder, status); + EXPECT_EQ(TF_OK, TF_GetCode(status)); + TF_DeleteStatus(status); + } + + { + OpKernelContext::Params p; + DummyDevice dummy_device(nullptr, false); + p.device = &dummy_device; + + Tensor t(tensorflow::uint8(123)); + + gtl::InlinedVector inputs; + // Simulate 2 inputs + inputs.emplace_back(&t); + inputs.emplace_back(); + p.inputs = &inputs; + + Status status; + std::unique_ptr kernel = + GetFakeKernel(device_name, op_name, &status); + TF_EXPECT_OK(status); + ASSERT_NE(nullptr, kernel.get()); + + p.op_kernel = kernel.get(); + OpKernelContext ctx(&p); + kernel->Compute(&ctx); + + ASSERT_EQ(2, num_inputs); + ASSERT_EQ(1, num_outputs); + } +} + +} // namespace tensorflow diff --git a/tensorflow/c/python_api.cc b/tensorflow/c/python_api.cc index 247236b760d..98d83933322 100644 --- a/tensorflow/c/python_api.cc +++ b/tensorflow/c/python_api.cc @@ -160,4 +160,17 @@ void SetHandleShapeAndType(TF_Graph* graph, TF_Output output, const void* proto, ic->set_output_handle_shapes_and_types(output.index, shapes_and_types); } +void AddWhileInputHack(TF_Graph* graph, TF_Output new_src, TF_Operation* dst, + TF_Status* status) { + mutex_lock l(graph->mu); + status->status = graph->graph.AddWhileInputHack(&new_src.oper->node, + new_src.index, &dst->node); + if (status->status.ok()) { + // This modification only updates the destination node for + // the purposes of running this graph in a session. Thus, we don't + // record the source node as being modified. + RecordMutation(graph, *dst, "adding input tensor"); + } +} + } // namespace tensorflow diff --git a/tensorflow/c/python_api.h b/tensorflow/c/python_api.h index 5cce84020bc..44779ca6561 100644 --- a/tensorflow/c/python_api.h +++ b/tensorflow/c/python_api.h @@ -34,6 +34,7 @@ void SetAttr(TF_Graph* graph, TF_Operation* op, const char* attr_name, void SetRequestedDevice(TF_Graph* graph, TF_Operation* op, const char* device); +// Updates 'dst' to consume 'new_src'. void UpdateEdge(TF_Graph* graph, TF_Output new_src, TF_Input dst, TF_Status* status); @@ -65,6 +66,13 @@ std::string GetHandleShapeAndType(TF_Graph* graph, TF_Output output); // because I couldn't get SWIG to work otherwise. void SetHandleShapeAndType(TF_Graph* graph, TF_Output output, const void* proto, size_t proto_len, TF_Status* status); + +// This method is used to add a new input edge to 'dst', which must be a While +// op. The While op's "T" attribute must have already been updated to include +// the new edge. This is used to construct tf.while_loop gradients. +void AddWhileInputHack(TF_Graph* graph, TF_Output new_src, TF_Operation* dst, + TF_Status* status); + } // namespace tensorflow #endif // TENSORFLOW_C_PYTHON_API_H_ diff --git a/tensorflow/cc/BUILD b/tensorflow/cc/BUILD index 83353b79f72..a09becc49b1 100644 --- a/tensorflow/cc/BUILD +++ b/tensorflow/cc/BUILD @@ -489,6 +489,7 @@ tf_gen_op_wrappers_cc( "image_ops", "io_ops", "linalg_ops", + "list_ops", "logging_ops", "lookup_ops", "manip_ops", diff --git a/tensorflow/cc/saved_model/BUILD b/tensorflow/cc/saved_model/BUILD index 3d3895c8fa8..52345a376cc 100644 --- a/tensorflow/cc/saved_model/BUILD +++ b/tensorflow/cc/saved_model/BUILD @@ -133,5 +133,6 @@ filegroup( "testdata/half_plus_two_pbtxt/**", "testdata/half_plus_two_main_op/**", "testdata/half_plus_two/**", + "testdata/half_plus_two_v2/**", ]), ) diff --git a/tensorflow/cc/saved_model/constants.h b/tensorflow/cc/saved_model/constants.h index 645a3f101d1..6f00dc324bd 100644 --- a/tensorflow/cc/saved_model/constants.h +++ b/tensorflow/cc/saved_model/constants.h @@ -33,10 +33,10 @@ constexpr char kSavedModelFilenamePb[] = "saved_model.pb"; /// SavedModel text format proto filename. constexpr char kSavedModelFilenamePbTxt[] = "saved_model.pbtxt"; -/// SavedModel legacy init op key. +/// SavedModel legacy init op collection key. Used in v1 SavedModels. constexpr char kSavedModelLegacyInitOpKey[] = "legacy_init_op"; -/// SavedModel main op key. +/// SavedModel main op collection key. Used in v1 SavedModels. constexpr char kSavedModelMainOpKey[] = "saved_model_main_op"; /// Directory in which to save the SavedModel variables. @@ -45,6 +45,11 @@ constexpr char kSavedModelVariablesDirectory[] = "variables"; /// SavedModel variables filename. constexpr char kSavedModelVariablesFilename[] = "variables"; +/// SavedModel SignatureDef keys for the initialization and train ops. Used in +/// V2 SavedModels. +constexpr char kSavedModelInitOpSignatureKey[] = "__saved_model_init_op"; +constexpr char kSavedModelTrainOpSignatureKey[] = "__saved_model_train_op"; + } // namespace tensorflow #endif // TENSORFLOW_CC_SAVED_MODEL_CONSTANTS_H_ diff --git a/tensorflow/cc/saved_model/loader.cc b/tensorflow/cc/saved_model/loader.cc index c6abe2f41b9..85d3dd01fa5 100644 --- a/tensorflow/cc/saved_model/loader.cc +++ b/tensorflow/cc/saved_model/loader.cc @@ -122,38 +122,58 @@ Status RunOnce(const RunOptions& run_options, return run_status; } -bool HasMainOp(const MetaGraphDef& meta_graph_def) { - const auto& collection_def_map = meta_graph_def.collection_def(); - if (collection_def_map.find(kSavedModelMainOpKey) != - collection_def_map.end()) { - return true; - } - return false; -} - -Status RunMainOp(const RunOptions& run_options, const string& export_dir, +// RunInitOp will return OK if the initialization op was run successfully. +// An empty init_op_name indicates that there are no init ops to run. +Status RunInitOp(const RunOptions& run_options, const string& export_dir, const MetaGraphDef& meta_graph_def, const std::vector& asset_file_defs, - Session* session, const string& main_op_key) { - LOG(INFO) << "Running MainOp with key " << main_op_key - << " on SavedModel bundle."; - const auto& collection_def_map = meta_graph_def.collection_def(); - const auto main_op_it = collection_def_map.find(main_op_key); - if (main_op_it != collection_def_map.end()) { - if (main_op_it->second.node_list().value_size() != 1) { - return errors::FailedPrecondition( - strings::StrCat("Expected exactly one main op in : ", export_dir)); - } + Session* session, const string& init_op_name) { + if (!init_op_name.empty()) { + LOG(INFO) << "Running initialization op on SavedModel bundle."; std::vector> inputs; AddAssetsTensorsToInputs(export_dir, asset_file_defs, &inputs); RunMetadata run_metadata; - const StringPiece main_op_name = main_op_it->second.node_list().value(0); - return RunOnce(run_options, inputs, {}, {string(main_op_name)}, + return RunOnce(run_options, inputs, {}, {init_op_name}, nullptr /* outputs */, &run_metadata, session); } return Status::OK(); } +// A SavedModel may store the name of the initialization op to run in the +// in the SignatureDef (v2) or a collection (v1). If an init_op collection +// exists, then the collection must contain exactly one op. +Status GetInitOp(const string& export_dir, const MetaGraphDef& meta_graph_def, + string* init_op_name) { + const auto& sig_def_map = meta_graph_def.signature_def(); + const auto& init_op_sig_it = + meta_graph_def.signature_def().find(kSavedModelInitOpSignatureKey); + if (init_op_sig_it != sig_def_map.end()) { + *init_op_name = init_op_sig_it->second.outputs() + .find(kSavedModelInitOpSignatureKey) + ->second.name(); + return Status::OK(); + } + + const auto& collection_def_map = meta_graph_def.collection_def(); + string init_op_collection_key; + if (collection_def_map.find(kSavedModelMainOpKey) != + collection_def_map.end()) { + init_op_collection_key = kSavedModelMainOpKey; + } else { + init_op_collection_key = kSavedModelLegacyInitOpKey; + } + + const auto init_op_it = collection_def_map.find(init_op_collection_key); + if (init_op_it != collection_def_map.end()) { + if (init_op_it->second.node_list().value_size() != 1) { + return errors::FailedPrecondition( + strings::StrCat("Expected exactly one main op in : ", export_dir)); + } + *init_op_name = init_op_it->second.node_list().value(0); + } + return Status::OK(); +} + Status RunRestore(const RunOptions& run_options, const string& export_dir, const StringPiece restore_op_name, const StringPiece variable_filename_const_op_name, @@ -193,6 +213,15 @@ Status RunRestore(const RunOptions& run_options, const string& export_dir, Status GetAssetFileDefs(const MetaGraphDef& meta_graph_def, std::vector* asset_file_defs) { + // With SavedModel v2, we write asset file def into metagraph instead of + // collection, so read from metagraph first. + if (meta_graph_def.asset_file_def_size() > 0) { + for (const auto& asset : meta_graph_def.asset_file_def()) { + asset_file_defs->push_back(asset); + } + return Status::OK(); + } + // Fall back to read from collection to be backward compatible with v1. const auto& collection_def_map = meta_graph_def.collection_def(); const auto assets_it = collection_def_map.find(kSavedModelAssetsKey); if (assets_it == collection_def_map.end()) { @@ -227,15 +256,12 @@ Status LoadSavedModelInternal(const SessionOptions& session_options, bundle->meta_graph_def.saver_def().restore_op_name(), bundle->meta_graph_def.saver_def().filename_tensor_name(), asset_file_defs, bundle->session.get())); - if (HasMainOp(bundle->meta_graph_def)) { - TF_RETURN_IF_ERROR(RunMainOp(run_options, export_dir, - bundle->meta_graph_def, asset_file_defs, - bundle->session.get(), kSavedModelMainOpKey)); - } else { - TF_RETURN_IF_ERROR(RunMainOp( - run_options, export_dir, bundle->meta_graph_def, asset_file_defs, - bundle->session.get(), kSavedModelLegacyInitOpKey)); - } + string init_op_name; + TF_RETURN_IF_ERROR( + GetInitOp(export_dir, bundle->meta_graph_def, &init_op_name)); + TF_RETURN_IF_ERROR(RunInitOp(run_options, export_dir, bundle->meta_graph_def, + asset_file_defs, bundle->session.get(), + init_op_name)); return Status::OK(); } diff --git a/tensorflow/cc/saved_model/loader_test.cc b/tensorflow/cc/saved_model/loader_test.cc index 72b8bc18710..597e42bb65a 100644 --- a/tensorflow/cc/saved_model/loader_test.cc +++ b/tensorflow/cc/saved_model/loader_test.cc @@ -36,6 +36,8 @@ constexpr char kTestDataMainOp[] = "cc/saved_model/testdata/half_plus_two_main_op/00000123"; constexpr char kTestDataSharded[] = "cc/saved_model/testdata/half_plus_two/00000123"; +constexpr char kTestDataInitOpV2[] = + "cc/saved_model/testdata/half_plus_two_v2/00000123"; class LoaderTest : public ::testing::Test { protected: @@ -227,5 +229,17 @@ TEST_F(LoaderTest, MaybeSavedModelDirectory) { EXPECT_FALSE(MaybeSavedModelDirectory(invalid_export_dir)); } +TEST_F(LoaderTest, SavedModelInitOpV2Format) { + SavedModelBundle bundle; + SessionOptions session_options; + RunOptions run_options; + + const string export_dir = + io::JoinPath(testing::TensorFlowSrcRoot(), kTestDataInitOpV2); + TF_ASSERT_OK(LoadSavedModel(session_options, run_options, export_dir, + {kSavedModelTagServe}, &bundle)); + CheckSavedModelBundle(export_dir, bundle); +} + } // namespace } // namespace tensorflow diff --git a/tensorflow/cc/saved_model/testdata/half_plus_two_v2/00000123/assets/foo.txt b/tensorflow/cc/saved_model/testdata/half_plus_two_v2/00000123/assets/foo.txt new file mode 100644 index 00000000000..f9ff0366880 --- /dev/null +++ b/tensorflow/cc/saved_model/testdata/half_plus_two_v2/00000123/assets/foo.txt @@ -0,0 +1 @@ +asset-file-contents \ No newline at end of file diff --git a/tensorflow/cc/saved_model/testdata/half_plus_two_v2/00000123/saved_model.pb b/tensorflow/cc/saved_model/testdata/half_plus_two_v2/00000123/saved_model.pb new file mode 100644 index 0000000000000000000000000000000000000000..a10bbf8fb6bca0fcee6414b2927d2f706de85ebc GIT binary patch literal 10774 zcmd5CU2hv%)#I=)+(ffKif?21qf=eCC^ligCG>10H zcU`wnv)fAjDlDCNw$XQcN6z42^5Szvf%{N+><&DiR-q{1*9#V-vR)04lcH}AJa<%I z2JuF40r#Nv9m^j0j(^4obp}WqBXlVOgj`Q31RKaxdvsvG)E^((y`!P)41AAVh4o%^ zYYdS$XKZ^lm23jBY|nQFrti3e;7v_MIaV#ZZzG#--!d#`p#?olRy z5}&*%?($nwgB9`IviHrCzVGS97yJR;f|UT4Gq9WKQu>{Z9Q@NHsgd;pwu-zel45{nHpKO9P_im$uSFw zBg?DhC@$O&Hq}(v1$gd{FerW(qY+t#B9CIs=g`$ra3UgN)Jb~!RVaHlx~PdFCPj}! zpjnau58$PVUfu1ZEYK!+EEg7n@w_e>ffP|OPM3S`z&D+Nod8?Eh7kc7ur02(I7SqH zy?}w_)YhPNuchB>zB}qQb@gO$G;m)H$fqAb52|Lv88|)$8-u$m5h1_6+_3CZ)Oh>x zvmdsb+EN(bvrR?YgcTz=ZxA(x1CobqJEL6*PlZYcN(#wn`AjDB4e&uRNsRYP!Cp(U zi2~KCagneGNF=9+ATjWJ1xjWkI5~q8lqx2b8J4L*MDYU=&&X7MRd&QOfox12ux#4~ z6pdcr^gML45-H*l+kRxCa~I9V$Tlq!9g#?uOhuZAV<=|`v*kD^oF7Z{0KtWB%D&NS|r;GRq|&JxVBn z>9aRap9}f^w8j?j&)$?idsJpvmgM64>|Iu$y(xY6WGd$BvzMyRJpg~-u!B4=o`lE< zFm01!EFR!2@6a$g5m8Ru#{&yCxCD@GLkVLUYV-8Kv(+yovk_)bD(HY)OXBw?+dN-v4X$306 z4FhLfp44hsqH1WVK{=VNWwrK{7jMIz_8M6#{U`i`@uZ~8c%%`j!plNNTaC&&UZkM! zIoK$P&@Zz0i#6>?p8bmG~zey)|+Gd7`R;Yr1X)3HwPVvA3;ar*VDv*akF9K--(Hd zi~kFt+6h{ZL1<4qBeeIz7=O3(4v+C)Y52#(hC6Z&ushgq*yEu)@*7x$*}m7`@!On(?BRc#TJ zDRmRPfQo1xG~!*6H?@I3VGmjANXX>V1l|cVv&)`ESezNRa{JVLJQWUJrSM3E;)?j_yZq;xej?w6k0iw!6G)Uw_F-*I07m-VU6%frcvd3dC0zmjuE1UXVtP#7m@%U!d_bHgQ9<%F zvoiTnN)2kERAwoD9;qB&!xmR$Gz9$;eg3C#OTzsny8T}jocA-gk2oijNpM`Lbe@3d zc^t?lG41c#TT@LM)_%g?cK8qe5QZozTgIRw@gSoMeC9$kxfz8=>RI{6(3Aybm~lk2 z#V%b<%6Gama0%)zGL-_EI(&0gRmAY4Er@%@1qya$PO5XXZI zc}2K;C}WzumkGzZMMypbjCp?mc3l7pU+|1ZGMRd|v_&6cTax((PhHzhB65!~T(z5l z@Ip9<;H|>8@_y#g^(XhU^}i{w?vHRb&#Dxhql+oOM}DV}&lLO^DrX|dXc|rNiVc9< zO?^$8zPFqA`T8zfAC+8q6|R@2yW#pCTOZbC6tBYd()5t6uiwO>KkJ7WtmGJ){-K9o zmX$#bc(4COf%5-3584T=jO+tchi8lp-!;xi4{nB^Xpjy3lqDLowVSId#o$72gDSpS z@Cb$689rC&bkmwrkpu4c-B%6v^#pzag5u=3w4cFhgjVNxcgPXkju3A_eypbeCU++I zaoyQY%DTA(e1h4(pb3*C{yi7jgIlayjx3E+AQD_~c440J!yx=B%-2qF2F<==aQ1P^ zx2UYY3PrEyqJHM0-I^il5SR7@t{4V$O3UCiufc13cSznPchg?sbRk}NGqr24pc?6n aldlu&Tw9TJm8R2qy3#jdPDR3I;eP?Lb}^0RR91 literal 0 HcmV?d00001 diff --git a/tensorflow/cc/saved_model/testdata/half_plus_two_v2/00000123/variables/variables.index b/tensorflow/cc/saved_model/testdata/half_plus_two_v2/00000123/variables/variables.index new file mode 100644 index 0000000000000000000000000000000000000000..7ec9fb4fe2dd21d0a6c324aecd7658fc37cf2326 GIT binary patch literal 151 zcmZQzVB=tvV&Y(AVB}8ZU=(7|U@>L0P?u+5> rewrites; - TF_RETURN_IF_ERROR(AddRewritesForShape(i, ps.parameters(i), &rewrites)); + TF_RETURN_IF_ERROR( + AddRewritesForShape(i, xla::Shape(ps.parameters(i)), &rewrites)); const string code = R"( - void set_arg{{NAME}}_data(void* data) { + void set_arg{{NAME}}_data(const void* data) { set_arg_data({{I}}, data); } {{TYPE}}* arg{{NAME}}_data() { @@ -204,7 +206,7 @@ Status GenArgMethods(const tf2xla::Config& config, const xla::ProgramShape& ps, // Generate methods for results (outputs). Status GenResultMethods(const tf2xla::Config& config, - const xla::ProgramShape& ps, string* methods) { + const xla::ProgramShapeProto& ps, string* methods) { if (ps.result().element_type() != xla::TUPLE) { // The XlaCompiler we use to build the xla computation always generates a // tuple result, and we rely on this to simplify code generation. @@ -217,8 +219,8 @@ Status GenResultMethods(const tf2xla::Config& config, } for (int i = 0; i < ps.result().tuple_shapes_size(); ++i) { std::vector> rewrites; - TF_RETURN_IF_ERROR( - AddRewritesForShape(i, ps.result().tuple_shapes(i), &rewrites)); + TF_RETURN_IF_ERROR(AddRewritesForShape( + i, xla::Shape(ps.result().tuple_shapes(i)), &rewrites)); string code = R"( {{TYPE}}* result{{NAME}}_data() { return static_cast<{{TYPE}}*>(result_data({{I}})); @@ -336,7 +338,7 @@ Status GenerateHeader(const CodegenOpts& opts, const tf2xla::Config& config, ExtractEntryParamBufferInfos(buffer_infos); std::vector buffer_infos_for_temps = ExtractTempBufferInfos(buffer_infos); - const xla::ProgramShape& ps = compile_result.program_shape; + const xla::ProgramShapeProto& ps = compile_result.program_shape; string methods_arg, methods_result; TF_RETURN_IF_ERROR(GenArgMethods(config, ps, compile_result, &methods_arg)); TF_RETURN_IF_ERROR(GenResultMethods(config, ps, &methods_result)); @@ -548,8 +550,8 @@ class {{CLASS}} : public tensorflow::XlaCompiledCpuFunction { static const char** StaticResultNames() {{RESULT_NAMES_CODE}} // Shape of the args and results. - static const xla::ProgramShape* StaticProgramShape() { - static const xla::ProgramShape* kShape = {{PROGRAM_SHAPE_SHIM_EXPRESSION}}; + static const xla::ProgramShapeProto* StaticProgramShape() { + static const xla::ProgramShapeProto* kShape = {{PROGRAM_SHAPE_SHIM_EXPRESSION}}; return kShape; } @@ -587,7 +589,7 @@ class {{CLASS}} : public tensorflow::XlaCompiledCpuFunction { {"{{METHODS_RESULT}}\n", methods_result}, {"{{NS_END}}\n", ns_end}, {"{{NS_START}}\n", ns_start}, - {"{{PROGRAM_SHAPE}}", xla::ShapeUtil::HumanString(ps)}, + {"{{PROGRAM_SHAPE}}", xla::ShapeUtil::HumanString(xla::ProgramShape(ps))}, {"{{PROGRAM_SHAPE_SHIM_EXPRESSION}}", metadata_result.program_shape_access_shim}, {"{{RESULT_INDEX}}", absl::StrCat(result_index)}, @@ -615,11 +617,11 @@ static string CreateUniqueIdentifier(const CodegenOpts& opts, Status GenerateMetadata(const CodegenOpts& opts, const CompileResult& compile_result, MetadataResult* metadata_result) { - std::unique_ptr program_shape; + std::unique_ptr program_shape; if (opts.gen_program_shape) { program_shape = - absl::make_unique(compile_result.program_shape); + absl::make_unique(compile_result.program_shape); // The parameter names are currently meaningless, and redundant with the // rest of our metadata, so clear them out to avoid confusion and save @@ -631,8 +633,8 @@ Status GenerateMetadata(const CodegenOpts& opts, // a shim that evaluates to nullptr, which is what we want. ProtobufToEmbed program_shape_protobuf{ - CreateUniqueIdentifier(opts, "ProgramShape"), "xla::ProgramShape", - program_shape.get()}; + CreateUniqueIdentifier(opts, "ProgramShapeProto"), + "xla::ProgramShapeProto", program_shape.get()}; ProtobufToEmbed hlo_profile_printer_data_protobuf{ CreateUniqueIdentifier(opts, "HloProfilePrinterData"), diff --git a/tensorflow/compiler/aot/codegen.h b/tensorflow/compiler/aot/codegen.h index 90410c46a8e..9485e86b10e 100644 --- a/tensorflow/compiler/aot/codegen.h +++ b/tensorflow/compiler/aot/codegen.h @@ -57,7 +57,7 @@ struct MetadataResult { std::vector header_variable_decls; // program_shape_access_shim is a C++ expression that constructs the - // xla::ProgramShape instance for the CompileResult passed to + // xla::ProgramShapeProto instance for the CompileResult passed to // GenerateMetadata. string program_shape_access_shim; diff --git a/tensorflow/compiler/aot/codegen_test.cc b/tensorflow/compiler/aot/codegen_test.cc index bb288d23000..c1788ca32a1 100644 --- a/tensorflow/compiler/aot/codegen_test.cc +++ b/tensorflow/compiler/aot/codegen_test.cc @@ -181,13 +181,15 @@ TEST(CodegenTest, Golden) { BufferInfo::MakeEntryParameter(/*size=*/96, /*param_number=*/1), BufferInfo::MakeTempBuffer(3), BufferInfo::MakeTempBuffer(120)}, 5, {})); - compile_result.program_shape = xla::ShapeUtil::MakeProgramShape( - { - xla::ShapeUtil::MakeShape(xla::F32, {1, 2}), - xla::ShapeUtil::MakeShape(xla::S64, {3, 4}), - }, - xla::ShapeUtil::MakeTupleShape( - {xla::ShapeUtil::MakeShape(xla::U32, {5, 6})})); + compile_result.program_shape = + xla::ShapeUtil::MakeProgramShape( + { + xla::ShapeUtil::MakeShape(xla::F32, {1, 2}), + xla::ShapeUtil::MakeShape(xla::S64, {3, 4}), + }, + xla::ShapeUtil::MakeTupleShape( + {xla::ShapeUtil::MakeShape(xla::U32, {5, 6})})) + .ToProto(); compile_result.entry_point = "entry_point"; compile_result.pointer_size = 8; diff --git a/tensorflow/compiler/aot/codegen_test_h.golden b/tensorflow/compiler/aot/codegen_test_h.golden index e4d8a02877c..968afad65ed 100644 --- a/tensorflow/compiler/aot/codegen_test_h.golden +++ b/tensorflow/compiler/aot/codegen_test_h.golden @@ -22,7 +22,7 @@ extern "C" void entry_point( void* result, const xla::ExecutableRunOptions* run_options, const void** args, void** temps, tensorflow::int64* profile_counters); -extern "C" char __tfcompile_foo_bar_MyClass_ProgramShape_protobuf_array_contents[]; +extern "C" char __tfcompile_foo_bar_MyClass_ProgramShapeProto_protobuf_array_contents[]; namespace foo { @@ -114,7 +114,7 @@ class MyClass : public tensorflow::XlaCompiledCpuFunction { // with dim indices specifying which value. No bounds checking is performed // on dim indices. - void set_arg0_data(void* data) { + void set_arg0_data(const void* data) { set_arg_data(0, data); } float* arg0_data() { @@ -132,7 +132,7 @@ class MyClass : public tensorflow::XlaCompiledCpuFunction { arg_data(0)))[dim0][dim1]; } - void set_arg_myfeed_data(void* data) { + void set_arg_myfeed_data(const void* data) { set_arg_data(0, data); } float* arg_myfeed_data() { @@ -150,7 +150,7 @@ class MyClass : public tensorflow::XlaCompiledCpuFunction { arg_data(0)))[dim0][dim1]; } - void set_arg1_data(void* data) { + void set_arg1_data(const void* data) { set_arg_data(1, data); } tensorflow::int64* arg1_data() { @@ -253,10 +253,10 @@ class MyClass : public tensorflow::XlaCompiledCpuFunction { } // Shape of the args and results. - static const xla::ProgramShape* StaticProgramShape() { - static const xla::ProgramShape* kShape = []() { - xla::ProgramShape* proto = new xla::ProgramShape; - proto->ParseFromArray(&__tfcompile_foo_bar_MyClass_ProgramShape_protobuf_array_contents[0], 52); + static const xla::ProgramShapeProto* StaticProgramShape() { + static const xla::ProgramShapeProto* kShape = []() { + xla::ProgramShapeProto* proto = new xla::ProgramShapeProto; + proto->ParseFromArray(&__tfcompile_foo_bar_MyClass_ProgramShapeProto_protobuf_array_contents[0], 52); return proto; }(); return kShape; diff --git a/tensorflow/compiler/aot/codegen_test_o.golden b/tensorflow/compiler/aot/codegen_test_o.golden index eb001c5d45bdfefc76629d7303d89f5480432235..ce8e5ec8c96a2c3696f14b8eea206d648182ecb5 100644 GIT binary patch delta 82 zcmX@XdVzI<24lcP&2+}ti4)^k1B&uX@+ZEZ$-%(DPz1!xlLHxrCof>+VC`Y7)G=uQ0OJr9Qvd(} delta 49 zcmcb>dV+O=2BXJB&2+|yi4)@{ewoRbJ9#3bJY(zRjg0Y(wUY&z>=`{K2Lj2`$rG9E E0kohHL;wH) diff --git a/tensorflow/compiler/aot/compile.cc b/tensorflow/compiler/aot/compile.cc index 2b5f97b34cd..9fc223bdc7c 100644 --- a/tensorflow/compiler/aot/compile.cc +++ b/tensorflow/compiler/aot/compile.cc @@ -56,17 +56,23 @@ Status CompileXla(xla::CompileOnlyClient* client, return errors::Unknown("Couldn't get XLA program shape: ", pshape_or.status().error_message()); } - compile_result->program_shape = *pshape_or.ValueOrDie(); - xla::ProgramShape* pshape = &compile_result->program_shape; - std::vector arg_layouts; - arg_layouts.reserve(pshape->parameters_size()); + compile_result->program_shape = pshape_or.ValueOrDie()->ToProto(); + xla::ProgramShapeProto* pshape = &compile_result->program_shape; + + // AotXlaComputationInstance::argument_layouts is a vector of Shape + // pointers. Accumulate the Shape objects themselves in a separate vector + // while building the vector of pointers. + std::vector arg_layout_ptrs(pshape->parameters_size()); + std::vector arg_layouts(pshape->parameters_size()); for (int i = 0; i < pshape->parameters_size(); ++i) { - arg_layouts.push_back(pshape->mutable_parameters(i)); + arg_layouts[i] = xla::Shape(*pshape->mutable_parameters(i)); + arg_layout_ptrs[i] = &arg_layouts[i]; } xla::CompileOnlyClient::AotXlaComputationInstance instance; instance.computation = &computation; - instance.argument_layouts = std::move(arg_layouts); - instance.result_layout = &pshape->result(); + instance.argument_layouts = std::move(arg_layout_ptrs); + xla::Shape result_shape(pshape->result()); + instance.result_layout = &result_shape; xla::StatusOr>> aot_or = client->CompileAheadOfTime({instance}, aot_opts); if (!aot_or.ok()) { diff --git a/tensorflow/compiler/aot/compile.h b/tensorflow/compiler/aot/compile.h index e03c5b1aa77..ee7bb26fabd 100644 --- a/tensorflow/compiler/aot/compile.h +++ b/tensorflow/compiler/aot/compile.h @@ -33,9 +33,9 @@ namespace tfcompile { struct CompileResult { // Contains object file and meta-info. std::unique_ptr aot; - xla::ProgramShape program_shape; // Static shape of args and results. - string entry_point; // Name of generated function. - int pointer_size = 0; // Size of a pointer in bytes. + xla::ProgramShapeProto program_shape; // Static shape of args and results. + string entry_point; // Name of generated function. + int pointer_size = 0; // Size of a pointer in bytes. }; // CompileGraph compiles the graph_def into an object file containing a function diff --git a/tensorflow/compiler/aot/tests/tfcompile_test.cc b/tensorflow/compiler/aot/tests/tfcompile_test.cc index f10852c7850..4dd79e5882d 100644 --- a/tensorflow/compiler/aot/tests/tfcompile_test.cc +++ b/tensorflow/compiler/aot/tests/tfcompile_test.cc @@ -526,13 +526,15 @@ TEST(TFCompileTest, ProgramShape) { // muladd has the program shape defined. MatMulAndAddComp muladd; - const xla::ProgramShape* muladd_shape = muladd.ProgramShape(); + const xla::ProgramShapeProto* muladd_shape = muladd.ProgramShape(); ASSERT_TRUE(muladd_shape != nullptr); ASSERT_EQ(muladd_shape->parameters_size(), 2); - EXPECT_TRUE(ShapeUtil::Compatible(muladd_shape->parameters(0), f32_2x2)); - EXPECT_TRUE(ShapeUtil::Compatible(muladd_shape->parameters(1), f32_2x2)); + EXPECT_TRUE( + ShapeUtil::Compatible(xla::Shape(muladd_shape->parameters(0)), f32_2x2)); + EXPECT_TRUE( + ShapeUtil::Compatible(xla::Shape(muladd_shape->parameters(1)), f32_2x2)); - const xla::Shape& muladd_result = muladd_shape->result(); + const xla::Shape muladd_result(muladd_shape->result()); ASSERT_EQ(muladd_result.element_type(), xla::TUPLE); ASSERT_EQ(ShapeUtil::TupleElementCount(muladd_result), 2); const xla::Shape& muladd_result0 = diff --git a/tensorflow/compiler/jit/BUILD b/tensorflow/compiler/jit/BUILD index 5f25e4626ad..be91ed4f432 100644 --- a/tensorflow/compiler/jit/BUILD +++ b/tensorflow/compiler/jit/BUILD @@ -23,7 +23,6 @@ package( load("//tensorflow:tensorflow.bzl", "cc_header_only_library") load("//tensorflow:tensorflow.bzl", "tf_cc_test") load("@local_config_cuda//cuda:build_defs.bzl", "if_cuda") -load("@local_config_cuda//cuda:build_defs.bzl", "if_cuda_is_configured") load("//tensorflow:tensorflow.bzl", "tf_cuda_cc_test") load("//tensorflow:tensorflow.bzl", "tf_custom_op_py_library") @@ -38,7 +37,7 @@ cc_library( ":xla_cpu_device", ":xla_cpu_jit", "//tensorflow/compiler/plugin", - ] + if_cuda_is_configured([ + ] + if_cuda([ ":xla_gpu_device", ":xla_gpu_jit", ]), @@ -51,6 +50,7 @@ cc_library( deps = [ ":jit_compilation_passes", "//tensorflow/compiler/jit/kernels:xla_ops", + "//tensorflow/compiler/tf2xla/kernels:xla_cpu_only_ops", "//tensorflow/compiler/tf2xla/kernels:xla_dummy_ops", "//tensorflow/compiler/tf2xla/kernels:xla_ops", "//tensorflow/compiler/xla/service:cpu_plugin", @@ -76,10 +76,10 @@ cc_library( srcs = ["xla_cpu_device.cc"], visibility = [":friends"], deps = [ + ":flags", ":jit_compilation_passes", ":xla_device", "//tensorflow/compiler/jit/kernels:xla_ops", - "//tensorflow/compiler/jit/legacy_flags:xla_device_flags", "//tensorflow/compiler/tf2xla:xla_compiler", "//tensorflow/compiler/tf2xla/kernels:xla_ops", "//tensorflow/compiler/xla/service:cpu_plugin", # buildcleaner: keep @@ -210,6 +210,18 @@ cc_library( # Internal targets below this point. +cc_library( + name = "flags", + srcs = ["flags.cc"], + hdrs = ["flags.h"], + visibility = [":friends"], + deps = [ + "//tensorflow/compiler/xla:parse_flags_from_env", + "//tensorflow/core:framework_internal", + "//tensorflow/core:lib", + ], +) + cc_library( name = "common", srcs = [ @@ -256,6 +268,7 @@ cc_library( "//tensorflow/compiler/tf2xla:common", "//tensorflow/compiler/tf2xla:dump_graph", "//tensorflow/compiler/tf2xla:xla_compiler", + "//tensorflow/compiler/xla:debug_options_flags", "//tensorflow/compiler/xla:statusor", "//tensorflow/compiler/xla/client:client_library", "//tensorflow/compiler/xla/client:local_client", @@ -268,6 +281,7 @@ cc_library( "//tensorflow/core/kernels:variable_ops", "@com_google_absl//absl/container:flat_hash_map", "@com_google_absl//absl/strings", + "@com_google_absl//absl/types:optional", "@com_google_absl//absl/types:span", ], ) @@ -487,6 +501,7 @@ cc_library( deps = [ ":common", ":encapsulate_util", + ":flags", ":shape_inference_helpers", ":union_find", ":xla_cluster_util", @@ -494,8 +509,6 @@ cc_library( "//tensorflow/cc:ops", "//tensorflow/cc:scope_internal", "//tensorflow/compiler/jit/graphcycles", - "//tensorflow/compiler/jit/legacy_flags:build_xla_ops_pass_flags", - "//tensorflow/compiler/jit/legacy_flags:mark_for_compilation_pass_flags", "//tensorflow/compiler/jit/ops:xla_ops", "//tensorflow/compiler/tf2xla:dump_graph", "//tensorflow/compiler/tf2xla:resource_operation_table", @@ -724,7 +737,10 @@ tf_custom_op_py_library( visibility = [ ":friends", ], - deps = ["//tensorflow/compiler/jit/ops:xla_ops_wrapper_py"], + deps = [ + "//tensorflow/compiler/jit/ops:xla_ops_grad", + "//tensorflow/compiler/jit/ops:xla_ops_wrapper_py", + ], ) # This target can be used by XLA device plugins to prevent circular dependencies, and provides access to all of the required headers for building a device library. diff --git a/tensorflow/compiler/jit/build_xla_ops_pass.cc b/tensorflow/compiler/jit/build_xla_ops_pass.cc index 93637a69d5d..9f4042630ed 100644 --- a/tensorflow/compiler/jit/build_xla_ops_pass.cc +++ b/tensorflow/compiler/jit/build_xla_ops_pass.cc @@ -23,7 +23,7 @@ limitations under the License. #include "tensorflow/cc/ops/control_flow_ops.h" #include "tensorflow/compiler/jit/defs.h" #include "tensorflow/compiler/jit/encapsulate_subgraphs_pass.h" -#include "tensorflow/compiler/jit/legacy_flags/build_xla_ops_pass_flags.h" +#include "tensorflow/compiler/jit/flags.h" #include "tensorflow/compiler/jit/xla_cluster_util.h" #include "tensorflow/compiler/tf2xla/cc/ops/xla_jit_ops.h" #include "tensorflow/compiler/tf2xla/dump_graph.h" @@ -320,10 +320,10 @@ Status BuildXlaOpsPass::Run(const GraphOptimizationPassOptions& options) { return IsXlaCompiledKernel(*n); }); - bool lazy_compilation_enabled = enable_lazy_compilation_ - ? *enable_lazy_compilation_ - : legacy_flags::GetBuildXlaOpsPassFlags() - .tf_xla_enable_lazy_compilation; + bool lazy_compilation_enabled = + enable_lazy_compilation_ + ? *enable_lazy_compilation_ + : GetBuildXlaOpsPassFlags().tf_xla_enable_lazy_compilation; for (Node* n : xla_compiled_kernels) { TF_RETURN_IF_ERROR(ReplaceNodeWithXlaCompileAndXlaRun( diff --git a/tensorflow/compiler/jit/build_xla_ops_pass_test.cc b/tensorflow/compiler/jit/build_xla_ops_pass_test.cc index 11df946cc18..48a23a4c171 100644 --- a/tensorflow/compiler/jit/build_xla_ops_pass_test.cc +++ b/tensorflow/compiler/jit/build_xla_ops_pass_test.cc @@ -42,14 +42,8 @@ class BuildXlaOpsTest : public ::testing::Test { .ok()); } - void TearDown() override { - for (Device* device : devices_) { - delete device; - } - } - private: - std::vector devices_; + std::vector> devices_; }; using ::tensorflow::testing::FindNodeByName; diff --git a/tensorflow/compiler/jit/create_xla_launch_op_test.cc b/tensorflow/compiler/jit/create_xla_launch_op_test.cc index 73866607621..0f872a480f4 100644 --- a/tensorflow/compiler/jit/create_xla_launch_op_test.cc +++ b/tensorflow/compiler/jit/create_xla_launch_op_test.cc @@ -59,8 +59,9 @@ class CreateXlaLaunchOpTest : public ::testing::Test { SessionOptions options; auto* device_count = options.config.mutable_device_count(); device_count->insert({"CPU", 1}); + std::vector> devices; TF_CHECK_OK(DeviceFactory::AddDevices( - options, "/job:localhost/replica:0/task:0", &devices_)); + options, "/job:localhost/replica:0/task:0", &devices)); FunctionDefLibrary proto; for (const auto& fdef : flib) { @@ -69,7 +70,7 @@ class CreateXlaLaunchOpTest : public ::testing::Test { lib_def_ = absl::make_unique( OpRegistry::Global(), proto); OptimizerOptions opts; - device_mgr_ = absl::make_unique(devices_); + device_mgr_ = absl::make_unique(std::move(devices)); pflr_ = absl::make_unique( device_mgr_.get(), Env::Default(), TF_GRAPH_DEF_VERSION, lib_def_.get(), opts, /*default_thread_pool=*/nullptr, /*cluster_flr=*/nullptr); @@ -77,7 +78,6 @@ class CreateXlaLaunchOpTest : public ::testing::Test { } FunctionLibraryRuntime* flr_; - std::vector devices_; std::unique_ptr device_mgr_; std::unique_ptr lib_def_; std::unique_ptr pflr_; diff --git a/tensorflow/compiler/jit/encapsulate_util.cc b/tensorflow/compiler/jit/encapsulate_util.cc index 28ec37b1b9c..1f4b9c90a4f 100644 --- a/tensorflow/compiler/jit/encapsulate_util.cc +++ b/tensorflow/compiler/jit/encapsulate_util.cc @@ -86,7 +86,7 @@ Status ProcessControlEdges(Graph* g, const string& xla_computation_attr_name, continue; } else if (src_xla_computation && !dst_xla_computation) { if (src_outside_compilation) { - // Case 1d: outside compilation to host computation control edge. + // Case 1c: outside compilation to host computation control edge. edges_to_remove.push_back(e); TF_RETURN_IF_ERROR(AppendToListAttr( @@ -94,7 +94,7 @@ Status ProcessControlEdges(Graph* g, const string& xla_computation_attr_name, } } else if (!src_xla_computation && dst_xla_computation) { if (dst_outside_compilation) { - // Case 1d: host computation control to outside compilation edge. + // Case 1c: host computation control to outside compilation edge. edges_to_remove.push_back(e); TF_RETURN_IF_ERROR(AppendToListAttr( @@ -103,40 +103,24 @@ Status ProcessControlEdges(Graph* g, const string& xla_computation_attr_name, } else { // src_xla_computation && dst_xla_computation if (*src_xla_computation != *dst_xla_computation) { if (src_outside_compilation && dst_outside_compilation) { - // Case 1c: outside compilation to outside compilation control edge. + // Case 1b: outside compilation to outside compilation control edge. edges_to_remove.push_back(e); TF_RETURN_IF_ERROR(AppendToListAttr( e->dst(), kXlaControlDependenciesAttrName, e->src()->name())); } else if (src_outside_compilation && !dst_outside_compilation) { - // Case 1b: outside compilation to another XLA computaition control + // Case 1a: outside compilation to another XLA computaition control // edge. TF_RETURN_IF_ERROR(AppendToListAttr( e->src(), kXlaConnectedToOtherXlaComputationAttrName, *dst_xla_computation)); } else if (!src_outside_compilation && dst_outside_compilation) { - // Case 1b: another XLA computaition to outside compilation control + // Case 1a: another XLA computaition to outside compilation control // edge. TF_RETURN_IF_ERROR(AppendToListAttr( e->dst(), kXlaConnectedFromOtherXlaComputationAttrName, *src_xla_computation)); } - } else { // *src_xla_computation == *dst_xla_computation - if (src_outside_compilation && dst_outside_compilation) { - if (*src_outside_compilation != *dst_outside_compilation) { - // Case 1c: outside compilation to outside compilation control edge. - edges_to_remove.push_back(e); - - TF_RETURN_IF_ERROR(AppendToListAttr( - e->dst(), kXlaControlDependenciesAttrName, e->src()->name())); - } - } else if (src_outside_compilation && !dst_outside_compilation) { - // Case 1a: outside compilation to its XLA computation control edge. - ReplaceAttr(e->src(), kXlaConnectedToXlaComputationAttrName, true); - } else if (!src_outside_compilation && dst_outside_compilation) { - // Case 1a: XLA computation to outside compilation in it control edge. - ReplaceAttr(e->dst(), kXlaConnectedFromXlaComputationAttrName, true); - } } } } @@ -181,12 +165,6 @@ Status ProcessXlaToXlaDataEdges(Graph* g, edges.push_back(EdgeInfo{e->dst_input(), e->dst()->id()}); VLOG(4) << "XLA -> XLA edge: " << e->DebugString(); } - } else { // *src_xla_computation == *dst_xla_computation - if (src_outside_compilation && dst_outside_compilation && - *src_outside_compilation != *dst_outside_compilation) { - edges.push_back(EdgeInfo{e->dst_input(), e->dst()->id()}); - VLOG(4) << "XLA -> XLA edge: " << e->DebugString(); - } } } @@ -263,7 +241,7 @@ Status ProcessDataEdgeBetweenOutsideCompilationAndHostComputation( // Remove the edge from host to outside compilation. Add a placeholder as // outside compilation node input. - std::map placeholders; + std::map, Node*> placeholders; for (int i = 0; i < edges.size(); i++) { Node* dst = g->FindNodeId(edges[i].dst_node_id); const Edge* e; @@ -275,9 +253,10 @@ Status ProcessDataEdgeBetweenOutsideCompilationAndHostComputation( // Find or create placeholder node. string new_name = edges[i].is_host_to_outside_compilation - ? absl::StrCat(src->name(), "_host_to_oc_placeholder") - : absl::StrCat(src->name(), "_oc_to_host_placeholder"); - auto iter = placeholders.find(new_name); + ? absl::StrCat(src->name(), "_host_to_oc_placeholder_", src_output) + : absl::StrCat(src->name(), "_oc_to_host_placeholder_", src_output); + auto placeholder_index = std::make_pair(src->name(), src_output); + auto iter = placeholders.find(placeholder_index); Node* placeholder_node; if (iter == placeholders.end()) { NodeDefBuilder placeholder_builder(new_name, "Placeholder"); @@ -310,7 +289,7 @@ Status ProcessDataEdgeBetweenOutsideCompilationAndHostComputation( Status s; placeholder_node = g->AddNode(placeholder_def, &s); TF_RETURN_IF_ERROR(s); - placeholders[new_name] = placeholder_node; + placeholders[placeholder_index] = placeholder_node; } else { placeholder_node = iter->second; } @@ -594,14 +573,244 @@ Status AddControlDependencies( return Status::OK(); } +// Step 1 for `PreprocessEdgesBetweenOutsideCompilations`. See comments of +// `PreprocessEdgesBetweenOutsideCompilations` for details. +Status PreprocessControlEdgesBetweenOutsideCompilations( + Graph* g, const string& outside_compilation_attr_name) { + // Gather edges to remove. We should not remove the edge while iterating. + std::vector edges_to_remove; + for (const Edge* e : g->edges()) { + if (!e->IsControlEdge()) { + continue; + } + + auto src_outside_compilation = + GetStringAttr(*e->src(), outside_compilation_attr_name); + auto dst_outside_compilation = + GetStringAttr(*e->dst(), outside_compilation_attr_name); + + if (src_outside_compilation && dst_outside_compilation) { + if (*src_outside_compilation != *dst_outside_compilation) { + // Case 1a: outside compilation to outside compilation control edge. + edges_to_remove.push_back(e); + + TF_RETURN_IF_ERROR(AppendToListAttr( + e->dst(), kXlaControlDependenciesWithinXlaClusterAttrName, + e->src()->name())); + } + } else if (src_outside_compilation && !dst_outside_compilation) { + // Case 1b: outside compilation to its XLA computation control edge. + ReplaceAttr(e->src(), kXlaConnectedToXlaComputationAttrName, true); + } else if (!src_outside_compilation && dst_outside_compilation) { + // Case 1b: XLA computation to outside compilation in it control edge. + ReplaceAttr(e->dst(), kXlaConnectedFromXlaComputationAttrName, true); + } + } + + for (auto e : edges_to_remove) { + g->RemoveEdge(e); + } + return Status::OK(); +} + +// Step 2 for `PreprocessEdgesBetweenOutsideCompilations`. See comments of +// `PreprocessEdgesBetweenOutsideCompilations` for details. +Status PreprocessDataEdgesBetweenOutsideCompilations( + Graph* g, const string& outside_compilation_attr_name) { + // Gather edges between outside compilation and host computation. Notice that + // we do not store `Edge*` directly because we remove some nodes while adding + // Identity nodes, and those Edge pointers might be invalidated. + struct EdgeInfo { + int dst_input, dst_node_id; + }; + std::vector edges; + for (const Edge* e : g->edges()) { + if (e->IsControlEdge()) { + continue; + } + + auto src_outside_compilation = + GetStringAttr(*e->src(), outside_compilation_attr_name); + auto dst_outside_compilation = + GetStringAttr(*e->dst(), outside_compilation_attr_name); + + if (src_outside_compilation && dst_outside_compilation && + *src_outside_compilation != *dst_outside_compilation) { + edges.push_back(EdgeInfo{e->dst_input(), e->dst()->id()}); + VLOG(4) << "Oc -> oc edge: " << e->DebugString(); + } + } + + // Remove the edge from host to outside compilation. Add a placeholder as + // outside compilation node input. + std::map, Node*> placeholders; + for (int i = 0; i < edges.size(); i++) { + Node* dst = g->FindNodeId(edges[i].dst_node_id); + const Edge* e; + TF_RETURN_IF_ERROR(dst->input_edge(edges[i].dst_input, &e)); + Node* src = e->src(); + int src_output = e->src_output(), dst_input = e->dst_input(); + g->RemoveEdge(e); + + // Find or create placeholder node. + string new_name = + absl::StrCat(src->name(), "_oc_to_oc_placeholder_", src_output); + auto placeholder_index = std::make_pair(src->name(), src_output); + auto iter = placeholders.find(placeholder_index); + Node* placeholder_node; + if (iter == placeholders.end()) { + NodeDefBuilder placeholder_builder(new_name, "Placeholder"); + placeholder_builder.Attr("dtype", src->output_type(src_output)); + string outside_compilation_attr; + TF_RETURN_IF_ERROR(GetNodeAttr(dst->attrs(), + outside_compilation_attr_name, + &outside_compilation_attr)); + placeholder_builder.Attr(outside_compilation_attr_name, + outside_compilation_attr); + placeholder_builder.Attr(kOutsideCompilationOriginalNodeAttrName, + src->name()); + placeholder_builder.Attr(kOutsideCompilationSrcOutputAttrName, + src_output); + NodeDef placeholder_def; + TF_RETURN_IF_ERROR(placeholder_builder.Finalize(&placeholder_def)); + Status s; + placeholder_node = g->AddNode(placeholder_def, &s); + TF_RETURN_IF_ERROR(s); + placeholders[placeholder_index] = placeholder_node; + } else { + placeholder_node = iter->second; + } + g->AddEdge(placeholder_node, 0, dst, dst_input); + + // Replace `e->dst()` because its input node changed. + NodeDef new_def = dst->def(); + *new_def.mutable_input(dst_input) = placeholder_node->name(); + TF_ASSIGN_OR_RETURN(Node * dst_replace_node, ReplaceNode(g, dst, new_def)); + + // Other edge in `edges` might have `e->dst()` as src or dst + // node. Before removing `e->dst()`, replace those edges with + // corresponding edges for `dst_replace_node`. + for (int j = i + 1; j < edges.size(); j++) { + if (edges[j].dst_node_id == edges[i].dst_node_id) { + edges[j].dst_node_id = dst_replace_node->id(); + } + } + } + return Status::OK(); +} + +// Step 1 for `PostprocessEdgesBetweenOutsideCompilations`. See comments of +// `PostprocessEdgesBetweenOutsideCompilations` for details. +Status PostprocessDataEdgesBetweenOutsideCompilations( + Graph* g, const string& outside_compilation_attr_name) { + // Gather all outside compilation to outside compilation nodes. + std::vector placeholder_nodes; + for (Node* n : g->nodes()) { + if (n->type_string() == "Placeholder" && + HasNodeAttr(n->def(), kOutsideCompilationOriginalNodeAttrName)) { + placeholder_nodes.push_back(n); + } + } + + // Remove the placeholder nodes, and reconnect original edge. + auto node_name_index = g->BuildNodeNameIndex(); + for (auto n : placeholder_nodes) { + string node_name; + int node_src_output; + TF_RETURN_IF_ERROR(GetNodeAttr( + n->attrs(), kOutsideCompilationOriginalNodeAttrName, &node_name)); + TF_RETURN_IF_ERROR(GetNodeAttr( + n->attrs(), kOutsideCompilationSrcOutputAttrName, &node_src_output)); + auto iter = node_name_index.find(node_name); + if (iter == node_name_index.end()) { + return errors::Internal( + "Cannot find original node for oc -> host placeholder node ", + node_name); + } + + // Change all usage node to use the original node instead. + Node* original_node = iter->second; + std::vector control_edges; + std::vector data_edges; + for (auto e : n->out_edges()) { + if (e->IsControlEdge()) { + control_edges.push_back(e); + } else { + data_edges.push_back({e->dst(), e->src_output(), e->dst_input()}); + } + } + for (const Edge* e : control_edges) { + g->AddControlEdge(original_node, e->dst()); + g->RemoveEdge(e); + } + for (int i = 0; i < data_edges.size(); i++) { + Node* dst = data_edges[i].dst; + NodeDef new_def = dst->def(); + int dst_input = data_edges[i].dst_input; + *new_def.mutable_input(dst_input) = + absl::StrCat(original_node->name(), ":", node_src_output); + TF_ASSIGN_OR_RETURN(Node * replace_node, ReplaceNode(g, dst, new_def)); + + const Edge* edge_to_replace = nullptr; + TF_RETURN_IF_ERROR(replace_node->input_edge(dst_input, &edge_to_replace)); + g->RemoveEdge(edge_to_replace); + g->AddEdge(original_node, node_src_output, replace_node, dst_input); + + // Other edges might have `dst` as dst node. Update those edges with + // `replace_node`. + for (int j = i + 1; j < data_edges.size(); j++) { + if (data_edges[j].dst == dst) { + data_edges[j].dst = replace_node; + } + } + + // Other placeholder node might have `dst` as original node. Update + // `node_name_index` with `replace_node`. + node_name_index[replace_node->name()] = replace_node; + } + + // Remove placeholder node. + g->RemoveNode(n); + } + return Status::OK(); +} + +// Step 2 for `PostprocessEdgesBetweenOutsideCompilations`. See comments of +// `PostprocessEdgesBetweenOutsideCompilations` for details. +Status PostprocessControlEdgesBetweenOutsideCompilations( + Graph* g, const string& outside_compilation_attr_name) { + auto node_name_index = g->BuildNodeNameIndex(); + + // Reconnect outside compilation to outside compilation control edge. + for (Node* n : g->nodes()) { + std::vector control_deps; + Status s = + GetNodeAttr(n->attrs(), kXlaControlDependenciesWithinXlaClusterAttrName, + &control_deps); + if (!s.ok()) { + if (s.code() != error::NOT_FOUND) { + return s; + } else { + continue; + } + } else { + n->ClearAttr(kXlaControlDependenciesWithinXlaClusterAttrName); + for (const string& control_input : control_deps) { + auto iter = node_name_index.find(control_input); + if (iter == node_name_index.end()) { + return errors::Internal("Cannot find original node for ", + control_input); + } + g->AddControlEdge(iter->second, n); + } + } + } + return Status::OK(); +} } // namespace const char kXlaInferredShapesAttrName[] = "_xla_inferred_shapes"; -const char kXlaConnectedToXlaComputationAttrName[] = - "_xla_connected_to_xla_computation"; -const char kXlaConnectedFromXlaComputationAttrName[] = - "_xla_connected_from_xla_computation"; const char kXlaConnectedToOtherXlaComputationAttrName[] = "_xla_connected_to_other_xla_computation"; const char kXlaConnectedFromOtherXlaComputationAttrName[] = @@ -616,6 +825,15 @@ const char kHostToOutsideCompilationOriginalNodeAttrName[] = "_xla_host_to_oc_node_name"; const char kHostToOutsideCompilationSrcOutputAttrName[] = "_xla_host_to_oc_src_output"; +const char kXlaConnectedToXlaComputationAttrName[] = + "_xla_connected_to_xla_computation"; +const char kXlaConnectedFromXlaComputationAttrName[] = + "_xla_connected_from_xla_computation"; +const char kOutsideCompilationOriginalNodeAttrName[] = + "_xla_oc_to_oc_node_name"; +const char kOutsideCompilationSrcOutputAttrName[] = "_xla_oc_to_oc_src_output"; +const char kXlaControlDependenciesWithinXlaClusterAttrName[] = + "_xla_control_dependencies_within_xla_cluster"; Status PerformStaticShapeInferenceBeforeEncapsulation( Graph* g, const string& xla_computation_attr_name, @@ -699,4 +917,39 @@ Status PostprocessForEncapsulation( return Status::OK(); } +Status PreprocessEdgesBetweenOutsideCompilations( + Graph* g, const string& outside_compilation_attr_name) { + // Remove edges from source node to outside compilation nodes, and edges + // from outside compilation nodes to sink node. + std::vector edges_to_remove; + for (const Edge* e : g->source_node()->out_edges()) { + if (HasNodeAttr(e->dst()->def(), outside_compilation_attr_name)) { + edges_to_remove.push_back(e); + } + } + for (const Edge* e : g->sink_node()->in_edges()) { + if (HasNodeAttr(e->src()->def(), outside_compilation_attr_name)) { + edges_to_remove.push_back(e); + } + } + for (auto e : edges_to_remove) { + g->RemoveEdge(e); + } + + TF_RETURN_IF_ERROR(PreprocessControlEdgesBetweenOutsideCompilations( + g, outside_compilation_attr_name)); + TF_RETURN_IF_ERROR(PreprocessDataEdgesBetweenOutsideCompilations( + g, outside_compilation_attr_name)); + return Status::OK(); +} + +Status PostprocessEdgesBetweenOutsideCompilations( + Graph* g, const string& outside_compilation_attr_name) { + TF_RETURN_IF_ERROR(PostprocessDataEdgesBetweenOutsideCompilations( + g, outside_compilation_attr_name)); + TF_RETURN_IF_ERROR(PostprocessControlEdgesBetweenOutsideCompilations( + g, outside_compilation_attr_name)); + return Status::OK(); +} + } // namespace tensorflow diff --git a/tensorflow/compiler/jit/encapsulate_util.h b/tensorflow/compiler/jit/encapsulate_util.h index 5e0c4bf6a0c..e363bc5754a 100644 --- a/tensorflow/compiler/jit/encapsulate_util.h +++ b/tensorflow/compiler/jit/encapsulate_util.h @@ -44,14 +44,6 @@ Status PerformStaticShapeInferenceBeforeEncapsulation( Graph* g, const string& xla_computation_attr_name, const string& outside_compilation_attr_name); -// Attribute indicating that some ops in this node's XLA computation has control -// dependency on this node. Attribute value will always be "true". -extern const char kXlaConnectedToXlaComputationAttrName[]; - -// Attribute indicating that this node has control dependency on some ops in -// this node's XLA computation. Attribute value will always be "true". -extern const char kXlaConnectedFromXlaComputationAttrName[]; - // Attribute indicating that some ops in other XLA computation has control // dependency on this node. Attribute value will be a list of string (XLA // computation names). @@ -81,6 +73,14 @@ extern const char kOutsideCompilationToHostOriginalNodeAttrName[]; // int (src_output for original edge). extern const char kOutsideCompilationToHostSrcOutputAttrName[]; +// Attribute indicating that some ops in this node's XLA computation has control +// dependency on this node. Attribute value will always be "true". +extern const char kXlaConnectedToXlaComputationAttrName[]; + +// Attribute indicating that this node has control dependency on some ops in +// this node's XLA computation. Attribute value will always be "true". +extern const char kXlaConnectedFromXlaComputationAttrName[]; + // Attribute indicating that this is an Placeholder node added to act as a // temporary input node for an host node. Attribute value will be string // (original input node name). @@ -91,19 +91,31 @@ extern const char kHostToOutsideCompilationOriginalNodeAttrName[]; // for original edge). extern const char kHostToOutsideCompilationSrcOutputAttrName[]; -// Preprocesses the graph for encapsulation. It will perform the following -// operations in order: +// Attribute indicating that this is an Placeholder node added to act as a +// temporary input node for an outside compilation node. Attribute value will be +// string (original input node name). +extern const char kOutsideCompilationOriginalNodeAttrName[]; + +// Attribute indicating that this is an Placeholder node added to act as a +// temporary input node for an outside compilation node. Attribute value will be +// int (src_output for original edge). +extern const char kOutsideCompilationSrcOutputAttrName[]; + +// Attribute indicating that this node has control dependencies on some other +// nodes within the same XLA cluster. Attribute value will be a list of string +// (node names). +extern const char kXlaControlDependenciesWithinXlaClusterAttrName[]; + +// Preprocesses edges between different XLA clusters for encapsulation. It will +// perform the following operations in order: // -// 1a. For control edges between outside compilation and its XLA computation, -// add attr "kXlaConnected{From, To}XlaComputationAttrName = true" to the -// outside compilation node. -// 1b. For control edges between outside compilation and another XLA +// 1a. For control edges between outside compilation and another XLA // computation, add attr "kXlaConnected{From, To}OtherXlaComputationAttrName // = XLA computation node name" to the outside compilation node. -// 1c. For control edges between different outside compilations, remove the edge -// and add attr "kXlaControlDependenciesAttrName = src node name" to dst -// node. -// 1d. For control edges between outside compilation and host computation, +// 1b. For control edges between different outside compilations (in different +// XLA computations), remove the edge and add attr +// "kXlaControlDependenciesAttrName = src node name" to dst node. +// 1c. For control edges between outside compilation and host computation, // remove the edge and add attr "kXlaControlDependenciesAttrName = src node // name" to dst node. // 2. For data edges between different XLA computations, if either src or dst @@ -146,26 +158,53 @@ struct XlaClusterInfo { const std::map host_compute_core; }; -// Postprocesses the graph for encapsulation. This function reverts what -// `PreprocessForEncapsulation` did. It will perform the following operations in -// order: +// Postprocesses edges between different XLA clusters for encapsulation. This +// function reverts what `PreprocessForEncapsulation` did. It will perform the +// following operations in order: // // 1. Remove Placeholder nodes between outside compilation and host computation // (created in `PreprocessForEncapsulation` step 3). // 2. Remove Identity nodes created in `PreprocessForEncapsulation` step 2. -// 3a. Reconnect control edges between different outside compilations (marked by -// `PreprocessForEncapsulation` step 1c) and control edges between outside -// compilation and host computation (marked by `PreprocessForEncapsulation` -// step 1d). -// 3b. Reconnect control edges between outside compilation and another XLA -// computation (marked by `PreprocessForEncapsulation` step 1b). -// Notice that control edges marked by `PreprocessForEncapsulation` step 1a are -// not handled here. They are handled in `RewriteOutsideCompilationSubgraphFn`. +// 3a. Reconnect control edges between outside compilation and another XLA +// computation (marked by `PreprocessForEncapsulation` step 1a). +// 3b. Reconnect control edges between different outside compilations (marked by +// `PreprocessForEncapsulation` step 1b). +// 3c. Reconnect control edges between outside compilation and host computation +// (marked by `PreprocessForEncapsulation` step 1c). Status PostprocessForEncapsulation( Graph* g, const string& xla_computation_attr_name, const string& outside_compilation_attr_name, const std::unordered_map& clusters); +// Preprocesses edges within the same XLA cluster. It will perform the following +// operations in order: +// +// 0. Remove edges from source node to outside compilation nodes, and edges +// from outside compilation nodes to sink node. +// 1a. For edges between different outside compilation clusters, remove the edge +// and add attr "kXlaControlDependenciesWithinXlaClusterAttrName = src node +// name" to dst node. +// 1b. For control edges between outside compilation and its XLA computation, +// add attr "kXlaConnected{From, To}XlaComputationAttrName = true" to the +// outside compilation node. +// 2. For data edges between different outside compilations, remove the edge +// and create a Placeholder node as dst node's input. +Status PreprocessEdgesBetweenOutsideCompilations( + Graph* g, const string& outside_compilation_attr_name); + +// Postprocesses edges within the same XLA cluster. This function reverts what +// `PreprocessEdgesBetweenOutsideCompilations` did. It will perform the +// following operations in order: +// +// 1. Remove Placeholder nodes between different outside compilations (created +// in `PreprocessEdgesBetweenOutsideCompilations` step 2). +// 2a. Reconnect control edges between different outside compilations (marked by +// `PreprocessEdgesBetweenOutsideCompilations` step 1a). +// Notice that control edges marked by +// `PreprocessEdgesBetweenOutsideCompilations` step 1b are not handled here. +// They are handled in `RewriteOutsideCompilationSubgraphFn`. +Status PostprocessEdgesBetweenOutsideCompilations( + Graph* g, const string& outside_compilation_attr_name); } // namespace tensorflow #endif // TENSORFLOW_COMPILER_JIT_ENCAPSULATE_UTIL_H_ diff --git a/tensorflow/compiler/jit/encapsulate_util_test.cc b/tensorflow/compiler/jit/encapsulate_util_test.cc index 7255df31129..3b8b49cb92f 100644 --- a/tensorflow/compiler/jit/encapsulate_util_test.cc +++ b/tensorflow/compiler/jit/encapsulate_util_test.cc @@ -107,28 +107,19 @@ TEST(PreprocessForEncapsulationTest, ControlEdges) { identity4_node->AddAttr("_xla", "1"); identity4_node->AddAttr("_oc", "0"); identity5_node->AddAttr("_xla", "1"); - // Case 1a: control edges between outside compilation and its XLA computation. - g.AddControlEdge(add_node, identity0_node); - g.AddControlEdge(identity0_node, identity1_node); - // Case 1b: control edges between outside compilation and another XLA + // Case 1a: control edges between outside compilation and another XLA // computation. g.AddControlEdge(identity0_node, identity3_node); g.AddControlEdge(identity1_node, identity4_node); - // Case 1c: control edges between different outside compilations. + // Case 1b: control edges between different outside compilations. g.AddControlEdge(identity0_node, identity4_node); - // Case 1d: control edges between outside compilation and host computation. + // Case 1c: control edges between outside compilation and host computation. g.AddControlEdge(const0_node, identity0_node); g.AddControlEdge(identity0_node, identity2_node); TF_CHECK_OK(PreprocessForEncapsulation(&g, "_xla", "_oc")); - // Case 1a: add attr "_xla_connected_{from/to}_xla_computation = true" to the - // outside compilation node. - EXPECT_TRUE(HasNodeAttr(identity0_node->def(), - kXlaConnectedFromXlaComputationAttrName)); - EXPECT_TRUE(HasNodeAttr(identity0_node->def(), - kXlaConnectedToXlaComputationAttrName)); - // Case 1b: add attr "_xla_control_deps_{from/to} = XLA computation node name" + // Case 1a: add attr "_xla_control_deps_{from/to} = XLA computation node name" // to the outside compilation node. std::vector attr; TF_CHECK_OK(GetNodeAttr(identity0_node->def(), @@ -140,13 +131,13 @@ TEST(PreprocessForEncapsulationTest, ControlEdges) { kXlaConnectedFromOtherXlaComputationAttrName, &attr)); EXPECT_EQ(attr.size(), 1); EXPECT_EQ(attr[0], "0"); - // Case 1c: add attr "_xla_control_deps = src node name" to dst node. + // Case 1b: add attr "_xla_control_deps = src node name" to dst node. attr.clear(); TF_CHECK_OK(GetNodeAttr(identity4_node->def(), kXlaControlDependenciesAttrName, &attr)); EXPECT_EQ(attr.size(), 1); EXPECT_EQ(attr[0], "identity0"); - // Case 1d: add attr "_xla_control_deps = src node name" to dst node. + // Case 1c: add attr "_xla_control_deps = src node name" to dst node. attr.clear(); TF_CHECK_OK(GetNodeAttr(identity0_node->def(), kXlaControlDependenciesAttrName, &attr)); @@ -162,23 +153,33 @@ TEST(PreprocessForEncapsulationTest, ControlEdges) { TEST(PreprocessForEncapsulationTest, DataEdges) { // Build the graph: // "const_0" and "const_1" in host computation + // "identityn0" = ("const_0", "const_1") in host computation 0 // "add0" = "const_0" + "const_1" in XLA computation 0 // "add1" = "add0" + "const_0" in XLA computation 0 & outside compilation 0 // "identity0" = "add1" in XLA computation 0 // "add2" = "add1" + "identity0" in host computation // "add3" = "add1" + "add2" in XLA computation 1 - // "add4" = "identity0" + "add2" in XLA computation 1 & outside compilation 1 + // "add4" = "identity0" + "add2" in XLA computation 1 & outside compilation 0 + // "add5" = "identityn0"[0] + "identityn0"[1] in XLA computation 1 & + // outside compilation 0 + // "identityn1" = ("identityn0"[0], "identityn0"[1]) in XLA computation 1 & + // outside compilation 0 // "identity1" = "add4" in XLA computation 1 // "identity2" = "identity1" in host computation tensorflow::Scope s = tensorflow::Scope::NewRootScope(); Output const_0 = ops::Const(s.WithOpName("const_0"), 1, {}); Output const_1 = ops::Const(s.WithOpName("const_1"), 2, {}); + auto identityn0 = + ops::IdentityN(s.WithOpName("identityn_0"), {const_0, const_1}); Output add0 = ops::Add(s.WithOpName("add0"), const_0, const_1); Output add1 = ops::Add(s.WithOpName("add1"), add0, const_0); Output identity0 = ops::Identity(s.WithOpName("identity0"), add1); Output add2 = ops::Add(s.WithOpName("add2"), add1, identity0); Output add3 = ops::Add(s.WithOpName("add3"), add1, add2); Output add4 = ops::Add(s.WithOpName("add4"), identity0, add2); + Output add5 = ops::Add(s.WithOpName("add5"), identityn0[0], identityn0[1]); + auto identityn1 = ops::IdentityN(s.WithOpName("identityn_1"), + {identityn0[0], identityn0[1]}); Output identity1 = ops::Identity(s.WithOpName("identity1"), add4); Output identity2 = ops::Identity(s.WithOpName("identity2"), add4); Graph g(OpRegistry::Global()); @@ -189,6 +190,8 @@ TEST(PreprocessForEncapsulationTest, DataEdges) { Node *add0_node = node_index["add0"], *add1_node = node_index["add1"], *identity0_node = node_index["identity0"], *add3_node = node_index["add3"], *add4_node = node_index["add4"], + *add5_node = node_index["add5"], + *identityn1_node = node_index["identityn_1"], *identity1_node = node_index["identity1"]; add0_node->AddAttr("_xla", "0"); add1_node->AddAttr("_xla", "0"); @@ -197,6 +200,10 @@ TEST(PreprocessForEncapsulationTest, DataEdges) { add3_node->AddAttr("_xla", "1"); add4_node->AddAttr("_xla", "1"); add4_node->AddAttr("_oc", "0"); + add5_node->AddAttr("_xla", "1"); + add5_node->AddAttr("_oc", "0"); + identityn1_node->AddAttr("_xla", "1"); + identityn1_node->AddAttr("_oc", "0"); identity1_node->AddAttr("_xla", "1"); TF_CHECK_OK(PreprocessForEncapsulation(&g, "_xla", "_oc")); @@ -214,8 +221,9 @@ TEST(PreprocessForEncapsulationTest, DataEdges) { EXPECT_NE(bridge_identity0_add4, nullptr); // Step 3: add placeholder for edges between host computation and outside // compilation. - EXPECT_EQ(bridge_add1_add3->def().input(0), "add1_oc_to_host_placeholder"); - Node *add1_oc_to_host_placeholder = node_index["add1_oc_to_host_placeholder"]; + EXPECT_EQ(bridge_add1_add3->def().input(0), "add1_oc_to_host_placeholder_0"); + Node *add1_oc_to_host_placeholder = + node_index["add1_oc_to_host_placeholder_0"]; TF_CHECK_OK(GetNodeAttr(add1_oc_to_host_placeholder->attrs(), kOutsideCompilationToHostOriginalNodeAttrName, &str)); EXPECT_EQ(str, "add1"); @@ -226,15 +234,34 @@ TEST(PreprocessForEncapsulationTest, DataEdges) { add4_node = node_index["add4"]; ASSERT_NE(add4_node, nullptr); EXPECT_EQ(add4_node->def().input(0), - "bridge_identity0_add4_host_to_oc_placeholder"); + "bridge_identity0_add4_host_to_oc_placeholder_0"); Node *identity0_host_to_oc_placeholder = - node_index["bridge_identity0_add4_host_to_oc_placeholder"]; + node_index["bridge_identity0_add4_host_to_oc_placeholder_0"]; TF_CHECK_OK(GetNodeAttr(identity0_host_to_oc_placeholder->attrs(), kHostToOutsideCompilationOriginalNodeAttrName, &str)); EXPECT_EQ(str, "bridge_identity0_add4"); TF_CHECK_OK(GetNodeAttr(identity0_host_to_oc_placeholder->attrs(), kHostToOutsideCompilationSrcOutputAttrName, &i)); EXPECT_EQ(i, 0); + + // Check different placeholder nodes are created for different src_output. + Node *placeholder0 = node_index["identityn_0_host_to_oc_placeholder_0"], + *placeholder1 = node_index["identityn_0_host_to_oc_placeholder_1"]; + EXPECT_NE(placeholder0, nullptr); + EXPECT_NE(placeholder1, nullptr); + // Check we only have 2 placeholder nodes created for "identityn_0". + int placeholder_count = 0; + for (Node *n : g.nodes()) { + if (HasNodeAttr(n->def(), kHostToOutsideCompilationOriginalNodeAttrName)) { + string attr; + TF_CHECK_OK(GetNodeAttr( + n->attrs(), kHostToOutsideCompilationOriginalNodeAttrName, &attr)); + if (attr == "identityn_0") { + ++placeholder_count; + } + } + } + EXPECT_EQ(placeholder_count, 2); } TEST(PostprocessForEncapsulationTest, ControlEdges) { diff --git a/tensorflow/compiler/jit/encapsulate_xla_computations_pass.cc b/tensorflow/compiler/jit/encapsulate_xla_computations_pass.cc index 2ce6fa73fc4..d334100aa4a 100644 --- a/tensorflow/compiler/jit/encapsulate_xla_computations_pass.cc +++ b/tensorflow/compiler/jit/encapsulate_xla_computations_pass.cc @@ -195,8 +195,11 @@ Status RewriteSubgraph(const std::vector& arg_source_tensors, e->dst()->attrs().Find(kXlaClusterAttr) == nullptr && e->dst()->type_string() != kXlaClusterOutput) { return errors::InvalidArgument( - "Undeclared output of XLA computation. A common cause of this error " - "is variable initializers that depend on the XLA computation. Edge: ", + "Undeclared output of XLA computation. Some common causes of this " + "error are: 1) variable initializers that depend on the XLA " + "computation; 2) gradient computations that depend on the XLA " + "computation, which can be mitigated by moving gradient computations " + "inside XLA computation. Offending edge: ", e->src()->name(), ":", e->src_output(), " -> ", e->dst()->name(), ":", e->dst_input()); } diff --git a/tensorflow/compiler/jit/extract_outside_compilation_pass.cc b/tensorflow/compiler/jit/extract_outside_compilation_pass.cc index 8b3587c5087..e3c7e2f89be 100644 --- a/tensorflow/compiler/jit/extract_outside_compilation_pass.cc +++ b/tensorflow/compiler/jit/extract_outside_compilation_pass.cc @@ -366,7 +366,7 @@ Status ReplaceOrRemoveOutsideCompilationCallNode( // replace this node with compilation result node. // 3) all outside compilation graphs. Status ConstructHostGraph( - const string& xla_cluster_name, + const string& xla_cluster_name, const string& outside_compilation_attr_name, const std::vector& outside_compilation_host_graphs, FunctionLibraryDefinition* fld, std::unique_ptr* host_graph) { host_graph->reset(new Graph(fld)); @@ -476,6 +476,10 @@ Status ConstructHostGraph( host_graph->get(), std::unordered_set{(*host_graph)->sink_node()}); + // Postprocess edges between different outside compilations. + TF_RETURN_IF_ERROR(PostprocessEdgesBetweenOutsideCompilations( + host_graph->get(), outside_compilation_attr_name)); + if (VLOG_IS_ON(4)) { dump_graph::DumpGraphToFile( absl::StrCat("extract_outside_compilation_host_graph_for_", @@ -801,6 +805,11 @@ Status ExtractOutsideCompilationForFunction( }, &fbody)); std::unique_ptr fbody_deleter(fbody); + + // Preprocess edges between different outside compilations. They will be + // restored in `ConstructHostGraph()`. + TF_RETURN_IF_ERROR(PreprocessEdgesBetweenOutsideCompilations( + fbody->graph, outside_compilation_attr_name)); if (VLOG_IS_ON(4)) { dump_graph::DumpGraphToFile( absl::StrCat("extract_outside_compilation_for_func_before_", func_name), @@ -860,8 +869,9 @@ Status ExtractOutsideCompilationForFunction( // Construct host graph. if (!outside_compilation_host_graphs.empty()) { - TF_RETURN_IF_ERROR(ConstructHostGraph( - xla_cluster_name, outside_compilation_host_graphs, fld, host_graph)); + TF_RETURN_IF_ERROR( + ConstructHostGraph(xla_cluster_name, outside_compilation_attr_name, + outside_compilation_host_graphs, fld, host_graph)); } // Remove the outside compilation graphs from function library. diff --git a/tensorflow/compiler/jit/extract_outside_compilation_pass_test.cc b/tensorflow/compiler/jit/extract_outside_compilation_pass_test.cc index c5bd64f004e..bff956100da 100644 --- a/tensorflow/compiler/jit/extract_outside_compilation_pass_test.cc +++ b/tensorflow/compiler/jit/extract_outside_compilation_pass_test.cc @@ -290,21 +290,18 @@ TEST(ExtractOutsideCompilationForFunctionTest, Basic) { TF_CHECK_OK(GetNodeAttr(host_compute_1->attrs(), "shapes", &shapes)); EXPECT_EQ(shapes.size(), 1); EXPECT_EQ(shapes[0].dim_size(), 1); - // Check XlaHostCompute nodes' "shape_inference_graph" attr. "0" should have a - // non-empty value, and "1" should have an empty value. + // Check XlaHostCompute nodes' "shape_inference_graph" attr. Both should have + // empty values. string shape_inference_graph; TF_CHECK_OK(GetNodeAttr(host_compute_0->attrs(), "shape_inference_graph", &shape_inference_graph)); - EXPECT_EQ(shape_inference_graph, - "_outside_compilation_shape_inference_cluster_0"); + EXPECT_EQ(shape_inference_graph, ""); TF_CHECK_OK(GetNodeAttr(host_compute_1->attrs(), "shape_inference_graph", &shape_inference_graph)); EXPECT_EQ(shape_inference_graph, ""); // Check `shape_inference_graphs`. - EXPECT_EQ(shape_inference_graphs.size(), 1); - EXPECT_EQ(shape_inference_graphs[0], - "_outside_compilation_shape_inference_cluster_0"); + EXPECT_EQ(shape_inference_graphs.size(), 0); // Check `host_graph`: verify we have key placeholder and sequencer. Node *key_placeholder = nullptr, *sequencer = nullptr; @@ -333,8 +330,8 @@ TEST(ExtractOutsideCompilationForFunctionTest, Basic) { send_recv_nodes.push_back(n); } } - EXPECT_EQ(num_send_from_host, 2); - EXPECT_EQ(num_recv_at_host, 2); + EXPECT_EQ(num_send_from_host, 1); + EXPECT_EQ(num_recv_at_host, 1); for (Node *n : send_recv_nodes) { Node *input_node; TF_CHECK_OK(n->input_node(n->num_inputs() - 1, &input_node)); diff --git a/tensorflow/compiler/jit/flags.cc b/tensorflow/compiler/jit/flags.cc new file mode 100644 index 00000000000..98e344b3a08 --- /dev/null +++ b/tensorflow/compiler/jit/flags.cc @@ -0,0 +1,152 @@ +/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include // NOLINT + +#include "tensorflow/compiler/jit/flags.h" +#include "tensorflow/compiler/xla/parse_flags_from_env.h" +#include "tensorflow/core/util/command_line_flags.h" + +namespace tensorflow { +namespace { + +BuildXlaOpsPassFlags* build_ops_flags; +DumpGraphFlags* dump_graph_flags; +MarkForCompilationPassFlags* mark_for_compilation_flags; +XlaDeviceFlags* device_flags; +XlaOpsCommonFlags* ops_flags; + +std::vector* flag_list; +std::once_flag flags_init; + +void AppendDumpGraphFlagsInternal(std::vector* flag_list) { + std::vector new_flags = { + Flag("tf_dump_graph_prefix", &dump_graph_flags->tf_dump_graph_prefix, + "Path prefix to which graphs dumped during debugging should be " + "written."), + }; + flag_list->insert(flag_list->end(), new_flags.begin(), new_flags.end()); +} + +void AppendMarkForCompilationPassFlagsInternal(std::vector* flag_list) { + std::vector new_flags = { + Flag("tf_xla_auto_jit", &mark_for_compilation_flags->tf_xla_auto_jit, + "Control compilation of operators into XLA computations on CPU and " + "GPU devices. 0 = use ConfigProto setting; -1 = off; 1 = on for " + "things very likely to be improved; 2 = on for everything. " + "Experimental."), + Flag("tf_xla_min_cluster_size", + &mark_for_compilation_flags->tf_xla_min_cluster_size, + "Minimum number of operators in an XLA compilation. Ignored for " + "operators placed on an XLA device or operators explicitly marked " + "for compilation."), + Flag("tf_xla_max_cluster_size", + &mark_for_compilation_flags->tf_xla_max_cluster_size, + "Maximum number of operators in an XLA compilation."), + Flag("tf_xla_clustering_debug", + &mark_for_compilation_flags->tf_xla_clustering_debug, + "Dump graphs during XLA compilation."), + Flag("tf_xla_cpu_global_jit", + &mark_for_compilation_flags->tf_xla_cpu_global_jit, + "Enables global JIT compilation for CPU via SessionOptions."), + Flag("tf_xla_clustering_fuel", + &mark_for_compilation_flags->tf_xla_clustering_fuel, + "Places an artificial limit on the number of ops marked as " + "eligible for clustering."), + Flag("tf_xla_fusion_only", + &mark_for_compilation_flags->tf_xla_fusion_only, + "enable fusion of element-wise operations only using XLA when " + "global_jit_level is ON*.")}; + flag_list->insert(flag_list->end(), new_flags.begin(), new_flags.end()); +} + +void AllocateAndParseFlags() { + build_ops_flags = new BuildXlaOpsPassFlags; + build_ops_flags->tf_xla_enable_lazy_compilation = true; + + dump_graph_flags = new DumpGraphFlags; + dump_graph_flags->tf_dump_graph_prefix = "/tmp/"; + + mark_for_compilation_flags = new MarkForCompilationPassFlags; + mark_for_compilation_flags->tf_xla_auto_jit = 0; + mark_for_compilation_flags->tf_xla_min_cluster_size = 2; + mark_for_compilation_flags->tf_xla_max_cluster_size = + std::numeric_limits::max(); + mark_for_compilation_flags->tf_xla_clustering_debug = false; + mark_for_compilation_flags->tf_xla_cpu_global_jit = false; + mark_for_compilation_flags->tf_xla_clustering_fuel = + std::numeric_limits::max(); + mark_for_compilation_flags->tf_xla_fusion_only = false; + + device_flags = new XlaDeviceFlags; + device_flags->tf_xla_compile_on_demand = false; + + ops_flags = new XlaOpsCommonFlags; + ops_flags->tf_xla_always_defer_compilation = false; + + flag_list = new std::vector({ + Flag("tf_xla_enable_lazy_compilation", + &build_ops_flags->tf_xla_enable_lazy_compilation, ""), + + Flag("tf_xla_compile_on_demand", &device_flags->tf_xla_compile_on_demand, + "Switch a device into 'on-demand' mode, where instead of " + "autoclustering ops are compiled one by one just-in-time."), + + Flag("tf_xla_always_defer_compilation", + &ops_flags->tf_xla_always_defer_compilation, ""), + }); + AppendDumpGraphFlagsInternal(flag_list); + AppendMarkForCompilationPassFlagsInternal(flag_list); + xla::ParseFlagsFromEnvAndDieIfUnknown("TF_XLA_FLAGS", *flag_list); +} + +} // namespace + +const BuildXlaOpsPassFlags& GetBuildXlaOpsPassFlags() { + std::call_once(flags_init, &AllocateAndParseFlags); + return *build_ops_flags; +} + +DumpGraphFlags* GetDumpGraphFlags() { + std::call_once(flags_init, &AllocateAndParseFlags); + return dump_graph_flags; +} + +MarkForCompilationPassFlags* GetMarkForCompilationPassFlags() { + std::call_once(flags_init, &AllocateAndParseFlags); + return mark_for_compilation_flags; +} + +XlaDeviceFlags* GetXlaDeviceFlags() { + std::call_once(flags_init, &AllocateAndParseFlags); + return device_flags; +} + +const XlaOpsCommonFlags& GetXlaOpsCommonFlags() { + std::call_once(flags_init, &AllocateAndParseFlags); + return *ops_flags; +} + +void AppendMarkForCompilationPassFlags(std::vector* flag_list) { + std::call_once(flags_init, &AllocateAndParseFlags); + AppendMarkForCompilationPassFlagsInternal(flag_list); +} + +void AppendDumpGraphFlags(std::vector* flag_list) { + std::call_once(flags_init, &AllocateAndParseFlags); + AppendDumpGraphFlagsInternal(flag_list); +} + +} // namespace tensorflow diff --git a/tensorflow/compiler/jit/legacy_flags/mark_for_compilation_pass_flags.h b/tensorflow/compiler/jit/flags.h similarity index 57% rename from tensorflow/compiler/jit/legacy_flags/mark_for_compilation_pass_flags.h rename to tensorflow/compiler/jit/flags.h index 79b47357a17..5ddea588eef 100644 --- a/tensorflow/compiler/jit/legacy_flags/mark_for_compilation_pass_flags.h +++ b/tensorflow/compiler/jit/flags.h @@ -13,10 +13,8 @@ See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ -#ifndef TENSORFLOW_COMPILER_JIT_LEGACY_FLAGS_MARK_FOR_COMPILATION_PASS_FLAGS_H_ -#define TENSORFLOW_COMPILER_JIT_LEGACY_FLAGS_MARK_FOR_COMPILATION_PASS_FLAGS_H_ - -// Legacy flags for the XLA bridge's mark_for_compilation_pass module. +#ifndef TENSORFLOW_COMPILER_JIT_FLAGS_H_ +#define TENSORFLOW_COMPILER_JIT_FLAGS_H_ #include @@ -24,15 +22,8 @@ limitations under the License. #include "tensorflow/core/util/command_line_flags.h" namespace tensorflow { -namespace legacy_flags { -// Append to *flag_list flag definitions associated with the XLA bridge's -// mark_for_compilation_pass module. -void AppendMarkForCompilationPassFlags( - std::vector* flag_list); - -// The values of flags associated with the XLA bridge's -// mark_for_compilation_pass module. +// Flags associated with the XLA bridge's mark_for_compilation_pass module. struct MarkForCompilationPassFlags { int32 tf_xla_auto_jit; // Control compilation of operators into XLA // computations on CPU and GPU devices. 0 = use @@ -57,12 +48,56 @@ struct MarkForCompilationPassFlags { // only using XLA. }; -// Return a pointer to the MarkForCompilationPassFlags struct; +// Flags associated with the XLA bridge's xla_device module. +struct XlaDeviceFlags { + // Switch the CPU device into "on-demand" mode, where instead of + // autoclustering ops are compiled one by one just-in-time. + // Enabling this mode by a legacy flag is a temporary mechanism. When this + // feature is battle-tested, we will switch this to be a session option. + bool tf_xla_compile_on_demand; +}; + +// Flags common to the _Xla* ops and their kernels. +struct XlaOpsCommonFlags { + // If true, _XlaCompile always refuses to compile the cluster, which means the + // XLA clusters always run in the TF executor. Defaults to false. + bool tf_xla_always_defer_compilation; +}; + +// Flags for the build_xla_ops pass. +struct BuildXlaOpsPassFlags { + // Enables lazy compilation for TF/XLA (only when auto-clustering) if true. + // Defaults to true. + bool tf_xla_enable_lazy_compilation; +}; + +// Flags for the XLA bridge's dump_graph module. +struct DumpGraphFlags { + // Path prefix to which graphs dumped during debugging should be written. + string tf_dump_graph_prefix; +}; + +// Return a pointer to the DumpGraphFlags struct; // repeated calls return the same pointer. // This should be called only after Flags::Parse() has returned. -MarkForCompilationPassFlags* GetMarkForCompilationPassFlags(); -} // namespace legacy_flags +// Getters for flags structs defined above. The first call to any of these +// parses TF_XLA_FLAGS for all of them. Those functions which return a pointer +// always return the same pointer. +MarkForCompilationPassFlags* GetMarkForCompilationPassFlags(); +const BuildXlaOpsPassFlags& GetBuildXlaOpsPassFlags(); +XlaDeviceFlags* GetXlaDeviceFlags(); +const XlaOpsCommonFlags& GetXlaOpsCommonFlags(); +DumpGraphFlags* GetDumpGraphFlags(); + +// Appends the flag definitions associated with +// MarkForCompilationPassFlags/DumpGraphFlags to `flag_list`. +// +// Has the side-effect of parsing TF_XLA_FLAGS if that hasn't happened yet. +void AppendMarkForCompilationPassFlags( + std::vector* flag_list); +void AppendDumpGraphFlags(std::vector* flag_list); + } // namespace tensorflow -#endif // TENSORFLOW_COMPILER_JIT_LEGACY_FLAGS_MARK_FOR_COMPILATION_PASS_FLAGS_H_ +#endif // TENSORFLOW_COMPILER_JIT_FLAGS_H_ diff --git a/tensorflow/compiler/jit/increase_dynamism_for_auto_jit_pass.cc b/tensorflow/compiler/jit/increase_dynamism_for_auto_jit_pass.cc index d984ca15cb7..ce53f70b79d 100644 --- a/tensorflow/compiler/jit/increase_dynamism_for_auto_jit_pass.cc +++ b/tensorflow/compiler/jit/increase_dynamism_for_auto_jit_pass.cc @@ -23,7 +23,7 @@ limitations under the License. #include "tensorflow/cc/ops/array_ops.h" #include "tensorflow/cc/ops/const_op.h" #include "tensorflow/cc/ops/math_ops.h" -#include "tensorflow/compiler/jit/legacy_flags/mark_for_compilation_pass_flags.h" +#include "tensorflow/compiler/jit/flags.h" #include "tensorflow/compiler/jit/xla_cluster_util.h" #include "tensorflow/compiler/tf2xla/cc/ops/xla_ops.h" #include "tensorflow/compiler/tf2xla/dump_graph.h" @@ -208,8 +208,12 @@ Status ComputeSliceSize(const Scope& host_scope, DCHECK_EQ(slice_size.back().type(), DT_INT64); } - *size = ops::Concat(host_scope.WithOpName("slice_size"), slice_size, - ops::Const(host_scope.WithOpName("concat_axis"), 0)); + // Trivial ConcatV2 nodes (with exactly one input) are disallowed. + *size = + slice_size.size() == 1 + ? slice_size[0] + : ops::Concat(host_scope.WithOpName("slice_size"), slice_size, + ops::Const(host_scope.WithOpName("concat_axis"), 0)); return Status::OK(); } @@ -242,6 +246,9 @@ Status ConvertTensorFlowSliceToStaticShapedSlice( .WithOpName("static_shaped_slice"), slice_inputs_int64.input, slice_inputs_int64.begin, slice_size) .node(); + + TF_RETURN_IF_ERROR(main_scope.status()); + std::vector compile_time_const_inputs; compile_time_const_inputs.push_back("size"); (*result)->AddAttr(kXlaCompileTimeConstantInputsAttr, @@ -284,49 +291,45 @@ Status RewriteSlice(Graph* g, Node* slice, const SliceInputs& slice_inputs, return Status::OK(); } -// If `n` is a slice we can rewrite to have a static shape (i.e. have the output -// shape only depend on the "size" input) then returns the a SliceInputs -// representing the inputs to `n`. Otherwise returns nullopt. -StatusOrOptional IsRewritableSlice(Node* n) { +// Return true if `n` is a slice we can rewrite to have a static shape +// (i.e. have the output shape only depend on the "size" input). +xla::StatusOr IsRewritableSlice(Node* n) { if (n->type_string() != "Slice") { - return {absl::nullopt}; + return false; } if (!GetXlaClusterForNode(*n).has_value()) { // There is no need to change slice ops outside XLA clusters. - return {absl::nullopt}; + return false; } TF_ASSIGN_OR_RETURN(absl::optional slice_inputs, GetSliceInputs(n)); if (!slice_inputs.has_value()) { - return {absl::nullopt}; + return false; } // If slice_size[i] < -1 for any i then executing the slice will throw an // error, and we don't do anything here. - bool slice_is_ok = absl::c_all_of(slice_inputs->size_as_vector, - [](int64 size_i) { return size_i >= -1; }); - if (!slice_is_ok) { - return {absl::nullopt}; - } - - return slice_inputs; + return absl::c_all_of(slice_inputs->size_as_vector, + [](int64 size_i) { return size_i >= -1; }); } Status FindAndRewriteSlices(Graph* g, bool* changed) { - std::vector> slices_to_rewrite; + std::vector slices_to_rewrite; for (Node* n : g->nodes()) { - TF_ASSIGN_OR_RETURN(absl::optional slice_inputs, - IsRewritableSlice(n)); - if (slice_inputs.has_value()) { - slices_to_rewrite.push_back({n, std::move(*slice_inputs)}); + TF_ASSIGN_OR_RETURN(bool is_rewritable, IsRewritableSlice(n)); + if (is_rewritable) { + slices_to_rewrite.push_back(n); } } - for (const auto& pair : slices_to_rewrite) { - TF_RETURN_IF_ERROR(RewriteSlice(g, pair.first, pair.second, - *GetXlaClusterForNode(*pair.first))); + for (Node* n : slices_to_rewrite) { + TF_ASSIGN_OR_RETURN(absl::optional slice_inputs, + GetSliceInputs(n)); + TF_RET_CHECK(slice_inputs.has_value()); + TF_RETURN_IF_ERROR( + RewriteSlice(g, n, *slice_inputs, *GetXlaClusterForNode(*n))); } if (!slices_to_rewrite.empty()) { @@ -342,8 +345,7 @@ Status FindAndRewriteSlices(Graph* g, bool* changed) { Status IncreaseDynamismForAutoJitPass::Run( const GraphOptimizationPassOptions& options) { - legacy_flags::MarkForCompilationPassFlags* flags = - legacy_flags::GetMarkForCompilationPassFlags(); + MarkForCompilationPassFlags* flags = GetMarkForCompilationPassFlags(); if (flags->tf_xla_clustering_debug) { dump_graph::DumpGraphToFile("before_increase_dynamism_for_auto_jit_pass", **options.graph, options.flib_def); diff --git a/tensorflow/compiler/jit/increase_dynamism_for_auto_jit_pass_test.cc b/tensorflow/compiler/jit/increase_dynamism_for_auto_jit_pass_test.cc index 0f6f612e967..a2f1b831ad7 100644 --- a/tensorflow/compiler/jit/increase_dynamism_for_auto_jit_pass_test.cc +++ b/tensorflow/compiler/jit/increase_dynamism_for_auto_jit_pass_test.cc @@ -27,6 +27,7 @@ limitations under the License. namespace tensorflow { namespace { +using ::testing::_; using testing::matchers::AssignedDevice; using testing::matchers::Attr; using testing::matchers::Const; @@ -142,6 +143,26 @@ TEST(SliceToDynamicSliceRewriteTest, Basic) { EXPECT_THAT(static_shaped_slice, m_dynamic_slice); } +TEST(SliceToDynamicSliceRewriteTest, SliceFromVector) { + Scope root = Scope::NewRootScope() + .ExitOnError() + .WithAssignedDevice(kDeviceName) + .WithXlaCluster("cluster_0"); + + Output input = ops::Placeholder(root.WithOpName("input"), DT_FLOAT); + Output begin = ops::Placeholder(root.WithOpName("begin"), DT_INT32); + Output size = ops::Const(root.WithOpName("size"), {-1}); + Output slice = ops::Slice(root.WithOpName("slice"), input, begin, size); + + std::unique_ptr result; + TF_ASSERT_OK(IncreaseDynamismForAutoJit(root, &result)); + + Node* static_shaped_slice = testing::FindNodeByName( + result.get(), "slice/static_shaped_slice/static_shaped_slice"); + EXPECT_NE(static_shaped_slice, nullptr); + EXPECT_THAT(result->nodes(), Not(Contains(NodeWith(Op("ConcatV2"))))); +} + TEST(SliceToDynamicSliceRewriteTest, ControlDependencePreserved) { Scope root = Scope::NewRootScope() .ExitOnError() @@ -166,18 +187,18 @@ TEST(SliceToDynamicSliceRewriteTest, ControlDependencePreserved) { CtrlDeps(NodeWith(Op("Placeholder"), Name("control"))))); } +int64 ToInt64(int v) { return static_cast(v); } + TEST(SliceToDynamicSliceRewriteTest, Int64Indices) { Scope root = Scope::NewRootScope() .ExitOnError() .WithAssignedDevice(kDeviceName) .WithXlaCluster("cluster_0"); - auto to_int64 = [](int v) { return static_cast(v); }; - Output input = ops::Placeholder(root.WithOpName("input"), DT_FLOAT); Output begin = ops::Placeholder(root.WithOpName("begin"), DT_INT64); Output size = - ops::Const(root.WithOpName("size"), {to_int64(-1), to_int64(500)}); + ops::Const(root.WithOpName("size"), {ToInt64(-1), ToInt64(500)}); Output slice = ops::Slice(root.WithOpName("slice"), input, begin, size); std::unique_ptr result; @@ -252,13 +273,35 @@ TEST(SliceToDynamicSliceRewriteTest, DontRewriteSliceWithNonConstSize) { Attr(kXlaCompileTimeConstantInputsAttr))))); } +TEST(SliceToDynamicSliceRewriteTest, ScalarSlice) { + Scope root = Scope::NewRootScope() + .ExitOnError() + .WithAssignedDevice(kDeviceName) + .WithXlaCluster("cluster_0"); + + Output input = ops::Placeholder(root.WithOpName("input"), DT_FLOAT); + Output begin = ops::Placeholder(root.WithOpName("begin"), DT_INT64); + Output size = ops::Const(root.WithOpName("size"), {}); + Output slice = ops::Slice(root.WithOpName("slice"), input, begin, size); + + std::unique_ptr result; + TF_ASSERT_OK(IncreaseDynamismForAutoJit(root, &result)); + + Node* static_shaped_slice = testing::FindNodeByName( + result.get(), "slice/static_shaped_slice/static_shaped_slice"); + ASSERT_NE(static_shaped_slice, nullptr); + EXPECT_THAT(static_shaped_slice, + NodeWith(Op("Slice"), Attr(kXlaCompileTimeConstantInputsAttr), + Inputs(_, _, Out(NodeWith(Name(size.node()->name())))))); +} + TEST(SliceToDynamicSliceRewriteTest, IndicesNotVector) { Scope root = Scope::NewRootScope() .ExitOnError() .WithAssignedDevice(kDeviceName) .WithXlaCluster("cluster_0"); - auto to_int64 = [](int v) { return static_cast(v); }; + auto ToInt64 = [](int v) { return static_cast(v); }; Output input = ops::Placeholder(root.WithOpName("input"), DT_FLOAT); Output begin = ops::Placeholder(root.WithOpName("begin"), DT_INT64); @@ -271,7 +314,7 @@ TEST(SliceToDynamicSliceRewriteTest, IndicesNotVector) { ops::Slice(root.WithOpName("slice"), input, begin, size_placeholder); Output size = - ops::Const(root.WithOpName("size"), {{to_int64(-1)}, {to_int64(500)}}); + ops::Const(root.WithOpName("size"), {{ToInt64(-1)}, {ToInt64(500)}}); TF_ASSERT_OK(root.graph()->UpdateEdge(size.node(), 0, slice.node(), 2)); std::unique_ptr result; @@ -281,5 +324,82 @@ TEST(SliceToDynamicSliceRewriteTest, IndicesNotVector) { Not(Contains(NodeWith(Op("Slice"), Attr(kXlaCompileTimeConstantInputsAttr))))); } + +TEST(SliceToDynamicSliceRewriteTest, SliceWithSliceInput) { + Scope root = Scope::NewRootScope() + .ExitOnError() + .WithAssignedDevice(kDeviceName) + .WithXlaCluster("cluster_0"); + + Output input = ops::Placeholder(root.WithOpName("input"), DT_FLOAT); + Output begin = ops::Placeholder(root.WithOpName("begin"), DT_INT32); + Output size_a = ops::Const(root.WithOpName("size_a"), {-1, 500}); + Output slice = ops::Slice(root.WithOpName("slice"), input, begin, size_a); + + Output size_b = ops::Const(root.WithOpName("size_a"), {-1, 200}); + Output slice_with_slice_input = ops::Slice( + root.WithOpName("slice_with_slice_input"), slice, begin, size_b); + + std::unique_ptr result; + TF_ASSERT_OK(IncreaseDynamismForAutoJit(root, &result)); + + Node* static_shaped_slice = testing::FindNodeByName( + result.get(), + "slice_with_slice_input/static_shaped_slice/static_shaped_slice"); + ASSERT_NE(static_shaped_slice, nullptr); + EXPECT_EQ(static_shaped_slice->output_type(0), DT_FLOAT) + << "Expected DT_FLOAT, was " + << DataType_Name(static_shaped_slice->output_type(0)); + EXPECT_THAT( + static_shaped_slice, + NodeWith( + Op("Slice"), + Inputs(Out(NodeWith( + Op("Slice"), + Name("slice/static_shaped_slice/static_shaped_slice"))), + _, _))); +} + +TEST(SliceToDynamicSliceRewriteTest, SliceWithSliceBegin) { + Scope root = Scope::NewRootScope() + .ExitOnError() + .WithAssignedDevice(kDeviceName) + .WithXlaCluster("cluster_0"); + + Output input_float = + ops::Placeholder(root.WithOpName("input_float"), DT_FLOAT); + Output input_i64 = ops::Placeholder(root.WithOpName("input_i64"), DT_INT64); + + Output begin_begin = + ops::Placeholder(root.WithOpName("begin_begin"), DT_INT32); + Output begin_size = ops::Const(root.WithOpName("begin_size"), {-1}); + Output begin = + ops::Slice(root.WithOpName("begin"), input_i64, begin_begin, begin_size); + + Output size = + ops::Const(root.WithOpName("size"), {ToInt64(-1), ToInt64(200)}); + Output slice_with_slice_begin = ops::Slice( + root.WithOpName("slice_with_slice_begin"), input_float, begin, size); + + std::unique_ptr result; + TF_ASSERT_OK(IncreaseDynamismForAutoJit(root, &result)); + + Node* static_shaped_slice = testing::FindNodeByName( + result.get(), + "slice_with_slice_begin/static_shaped_slice/static_shaped_slice"); + ASSERT_NE(static_shaped_slice, nullptr); + EXPECT_EQ(static_shaped_slice->output_type(0), DT_FLOAT) + << "Expected DT_FLOAT, was " + << DataType_Name(static_shaped_slice->output_type(0)); + EXPECT_THAT( + static_shaped_slice, + NodeWith( + Op("Slice"), + Inputs(_, + Out(NodeWith( + Op("Slice"), + Name("begin/static_shaped_slice/static_shaped_slice"))), + _))); +} } // namespace } // namespace tensorflow diff --git a/tensorflow/compiler/jit/kernels/BUILD b/tensorflow/compiler/jit/kernels/BUILD index 830db9ebdd9..0583774714c 100644 --- a/tensorflow/compiler/jit/kernels/BUILD +++ b/tensorflow/compiler/jit/kernels/BUILD @@ -12,10 +12,10 @@ cc_library( hdrs = ["xla_ops.h"], deps = [ "//tensorflow/compiler/jit:common", + "//tensorflow/compiler/jit:flags", "//tensorflow/compiler/jit:xla_compilation_cache", "//tensorflow/compiler/jit:xla_device", "//tensorflow/compiler/jit:xla_launch_util", - "//tensorflow/compiler/jit/legacy_flags:xla_ops_common_flags", "//tensorflow/compiler/tf2xla:common", "//tensorflow/compiler/tf2xla:tf2xla_util", "//tensorflow/compiler/tf2xla:xla_compiler", diff --git a/tensorflow/compiler/jit/kernels/xla_ops.cc b/tensorflow/compiler/jit/kernels/xla_ops.cc index 055de7afcc5..ad71df5a694 100644 --- a/tensorflow/compiler/jit/kernels/xla_ops.cc +++ b/tensorflow/compiler/jit/kernels/xla_ops.cc @@ -18,7 +18,7 @@ limitations under the License. #include "absl/container/flat_hash_map.h" #include "absl/memory/memory.h" #include "tensorflow/compiler/jit/defs.h" -#include "tensorflow/compiler/jit/legacy_flags/xla_ops_common_flags.h" +#include "tensorflow/compiler/jit/flags.h" #include "tensorflow/compiler/tf2xla/shape_util.h" #include "tensorflow/compiler/tf2xla/tf2xla_util.h" #include "tensorflow/compiler/tf2xla/xla_compiler.h" @@ -418,7 +418,7 @@ void XlaCompileOp::Compute(OpKernelContext* ctx) { cannot_compile_cluster = cannot_compile_cluster_; } - if (legacy_flags::GetXlaOpsCommonFlags().tf_xla_always_defer_compilation || + if (GetXlaOpsCommonFlags().tf_xla_always_defer_compilation || cannot_compile_cluster) { executable = nullptr; } else { diff --git a/tensorflow/compiler/jit/legacy_flags/BUILD b/tensorflow/compiler/jit/legacy_flags/BUILD deleted file mode 100644 index 5fa6c85f06f..00000000000 --- a/tensorflow/compiler/jit/legacy_flags/BUILD +++ /dev/null @@ -1,65 +0,0 @@ -# Legacy command line flags for the XLA bridge libraries. - -# Please do not add more flags to this package. - -# The XLA bridge libraries were written in an environment that allowed -# command-line flags to be scattered freely throughout the libraries. This -# model, while initially convenient, leads to a proliferation in unused command -# line flags in tests and binaries, and serious problems in servers, where one -# might wish parameters to be different in independent RPC calls to the same -# routine. -# -# Please don't add more flags. If you're a library author, pass options and -# parameters explicitly through the library's interface. - -licenses(["notice"]) # Apache 2.0 - -package(default_visibility = ["//tensorflow:internal"]) - -cc_library( - name = "mark_for_compilation_pass_flags", - srcs = ["mark_for_compilation_pass_flags.cc"], - hdrs = ["mark_for_compilation_pass_flags.h"], - deps = - [ - "//tensorflow/compiler/xla:parse_flags_from_env", - "//tensorflow/core:framework_internal", - "//tensorflow/core:lib", - ], -) - -cc_library( - name = "xla_device_flags", - srcs = ["xla_device_flags.cc"], - hdrs = ["xla_device_flags.h"], - deps = - [ - "//tensorflow/compiler/xla:parse_flags_from_env", - "//tensorflow/core:framework_internal", - "//tensorflow/core:lib", - ], -) - -cc_library( - name = "build_xla_ops_pass_flags", - srcs = ["build_xla_ops_pass_flags.cc"], - hdrs = ["build_xla_ops_pass_flags.h"], - deps = - [ - "//tensorflow/compiler/xla:parse_flags_from_env", - "//tensorflow/core:framework_internal", - "//tensorflow/core:lib", - ], -) - -cc_library( - name = "xla_ops_common_flags", - srcs = ["xla_ops_common_flags.cc"], - hdrs = ["xla_ops_common_flags.h"], - deps = - [ - "//tensorflow/compiler/xla:parse_flags_from_env", - "//tensorflow/core:framework_internal", - "//tensorflow/core:lib", - ], -) diff --git a/tensorflow/compiler/jit/legacy_flags/build_xla_ops_pass_flags.cc b/tensorflow/compiler/jit/legacy_flags/build_xla_ops_pass_flags.cc deleted file mode 100644 index 961c17c17ea..00000000000 --- a/tensorflow/compiler/jit/legacy_flags/build_xla_ops_pass_flags.cc +++ /dev/null @@ -1,47 +0,0 @@ -/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -#include // NOLINT - -#include "tensorflow/compiler/jit/legacy_flags/build_xla_ops_pass_flags.h" -#include "tensorflow/compiler/xla/parse_flags_from_env.h" -#include "tensorflow/core/util/command_line_flags.h" - -namespace tensorflow { -namespace legacy_flags { -namespace { - -BuildXlaOpsPassFlags* flags; -std::vector* flag_list; -std::once_flag flags_init; - -void AllocateAndParseFlags() { - flags = new BuildXlaOpsPassFlags; - flags->tf_xla_enable_lazy_compilation = true; - flag_list = new std::vector({ - Flag("tf_xla_enable_lazy_compilation", - &flags->tf_xla_enable_lazy_compilation, ""), - }); - xla::ParseFlagsFromEnv(*flag_list); -} - -} // namespace - -const BuildXlaOpsPassFlags& GetBuildXlaOpsPassFlags() { - std::call_once(flags_init, &AllocateAndParseFlags); - return *flags; -} -} // namespace legacy_flags -} // namespace tensorflow diff --git a/tensorflow/compiler/jit/legacy_flags/build_xla_ops_pass_flags.h b/tensorflow/compiler/jit/legacy_flags/build_xla_ops_pass_flags.h deleted file mode 100644 index 9aa5cf64d6d..00000000000 --- a/tensorflow/compiler/jit/legacy_flags/build_xla_ops_pass_flags.h +++ /dev/null @@ -1,37 +0,0 @@ -/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -#ifndef TENSORFLOW_COMPILER_JIT_LEGACY_FLAGS_BUILD_XLA_OPS_PASS_FLAGS_H_ -#define TENSORFLOW_COMPILER_JIT_LEGACY_FLAGS_BUILD_XLA_OPS_PASS_FLAGS_H_ - -namespace tensorflow { -namespace legacy_flags { - -// Flags for the build_xla_ops pass. -struct BuildXlaOpsPassFlags { - // Enables lazy compilation for TF/XLA (only when auto-clustering) if true. - // Defaults to true. - bool tf_xla_enable_lazy_compilation; -}; - -// Parses the flags in BuildXlaOpsPassFlags from the TF_XLA_FLAGS environment -// variable and returns a reference to the parsed copy. Parses TF_XLA_FLAGS -// only the first time this routine is called. -const BuildXlaOpsPassFlags& GetBuildXlaOpsPassFlags(); - -} // namespace legacy_flags -} // namespace tensorflow - -#endif // TENSORFLOW_COMPILER_JIT_LEGACY_FLAGS_BUILD_XLA_OPS_PASS_FLAGS_H_ diff --git a/tensorflow/compiler/jit/legacy_flags/mark_for_compilation_pass_flags.cc b/tensorflow/compiler/jit/legacy_flags/mark_for_compilation_pass_flags.cc deleted file mode 100644 index bad306e0b0a..00000000000 --- a/tensorflow/compiler/jit/legacy_flags/mark_for_compilation_pass_flags.cc +++ /dev/null @@ -1,98 +0,0 @@ -/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -// Legacy flags for the XLA bridge's mark_for_compilation_pass module. - -#include -#include - -#include "tensorflow/compiler/jit/legacy_flags/mark_for_compilation_pass_flags.h" -#include "tensorflow/compiler/xla/parse_flags_from_env.h" -#include "tensorflow/core/platform/logging.h" -#include "tensorflow/core/platform/types.h" -#include "tensorflow/core/util/command_line_flags.h" - -namespace tensorflow { -namespace legacy_flags { - -// Pointers to the parsed value of the flags and flag descriptors, initialized -// via flags_init. -static MarkForCompilationPassFlags* flags; -static std::vector* flag_list; -static std::once_flag flags_init; - -// Allocate *flags. Called via call_once(&flags_init,...). -static void AllocateFlags() { - flags = new MarkForCompilationPassFlags; - flags->tf_xla_auto_jit = 0; - flags->tf_xla_min_cluster_size = 2; - flags->tf_xla_max_cluster_size = std::numeric_limits::max(); - flags->tf_xla_clustering_debug = false; - flags->tf_xla_cpu_global_jit = false; - flags->tf_xla_clustering_fuel = std::numeric_limits::max(); - flags->tf_xla_fusion_only = false; - flag_list = new std::vector( - {Flag("tf_xla_auto_jit", &flags->tf_xla_auto_jit, - "Control compilation of operators into XLA computations on CPU and " - "GPU devices. 0 = use ConfigProto setting; -1 = off; 1 = on for " - "things very likely to be improved; 2 = on for everything. " - "Experimental."), - Flag("tf_xla_min_cluster_size", &flags->tf_xla_min_cluster_size, - "Minimum number of operators in an XLA compilation. Ignored for " - "operators placed on an XLA device or operators explicitly marked " - "for compilation."), - Flag("tf_xla_max_cluster_size", &flags->tf_xla_max_cluster_size, - "Maximum number of operators in an XLA compilation."), - Flag("tf_xla_clustering_debug", &flags->tf_xla_clustering_debug, - "Dump graphs during XLA compilation."), - Flag("tf_xla_cpu_global_jit", &flags->tf_xla_cpu_global_jit, - "Enables global JIT compilation for CPU via SessionOptions."), - Flag("tf_xla_clustering_fuel", &flags->tf_xla_clustering_fuel, - "Places an artificial limit on the number of ops marked as " - "eligible for clustering."), - Flag("tf_xla_fusion_only", &flags->tf_xla_fusion_only, - "enable fusion of element-wise operations only using XLA when " - "global_jit_level is ON*.")}); - xla::ParseFlagsFromEnv(*flag_list); - - if (VLOG_IS_ON(1)) { - VLOG(1) << "Parsed MarkForCompilationPassFlags:"; - VLOG(1) << " tf_xla_auto_jit = " << flags->tf_xla_auto_jit; - VLOG(1) << " tf_xla_min_cluster_size = " << flags->tf_xla_min_cluster_size; - VLOG(1) << " tf_xla_max_cluster_size = " << flags->tf_xla_max_cluster_size; - VLOG(1) << " tf_xla_clustering_debug = " << flags->tf_xla_clustering_debug; - VLOG(1) << " tf_xla_cpu_global_jit = " << flags->tf_xla_cpu_global_jit; - VLOG(1) << " tf_xla_clustering_fuel = " << flags->tf_xla_clustering_fuel; - VLOG(1) << " tf_xla_fusion_only = " << flags->tf_xla_fusion_only; - } -} - -// Append to *append_to flag definitions associated with the XLA bridge's -// mark_for_compilation_pass module. -void AppendMarkForCompilationPassFlags(std::vector* append_to) { - std::call_once(flags_init, &AllocateFlags); - append_to->insert(append_to->end(), flag_list->begin(), flag_list->end()); -} - -// Return a pointer to the MarkForCompilationPassFlags struct; -// repeated calls return the same pointer. -// This should be called only after Flags::Parse() has returned. -MarkForCompilationPassFlags* GetMarkForCompilationPassFlags() { - std::call_once(flags_init, &AllocateFlags); - return flags; -} - -} // namespace legacy_flags -} // namespace tensorflow diff --git a/tensorflow/compiler/jit/legacy_flags/xla_device_flags.cc b/tensorflow/compiler/jit/legacy_flags/xla_device_flags.cc deleted file mode 100644 index 76b80d3034c..00000000000 --- a/tensorflow/compiler/jit/legacy_flags/xla_device_flags.cc +++ /dev/null @@ -1,56 +0,0 @@ -/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -// Legacy flags for the XLA bridge's xla_device module. - -#include -#include - -#include "tensorflow/compiler/jit/legacy_flags/xla_device_flags.h" -#include "tensorflow/compiler/xla/parse_flags_from_env.h" -#include "tensorflow/core/platform/types.h" -#include "tensorflow/core/util/command_line_flags.h" - -namespace tensorflow { -namespace legacy_flags { - -// Pointers to the parsed value of the flags and flag descriptors, initialized -// via flags_init. -static XlaDeviceFlags* flags; -static std::vector* flag_list; -static std::once_flag flags_init; - -// Allocate *flags. Called via call_once(&flags_init,...). -static void AllocateFlags() { - flags = new XlaDeviceFlags; - flags->tf_xla_compile_on_demand = false; - flag_list = new std::vector({ - Flag("tf_xla_compile_on_demand", &flags->tf_xla_compile_on_demand, - "Switch a device into 'on-demand' mode, where instead of " - "autoclustering ops are compiled one by one just-in-time."), - }); - xla::ParseFlagsFromEnv(*flag_list); -} - -// Return a pointer to the XlaDeviceFlags struct; -// repeated calls return the same pointer. -// This should be called only after Flags::Parse() has returned. -XlaDeviceFlags* GetXlaDeviceFlags() { - std::call_once(flags_init, &AllocateFlags); - return flags; -} - -} // namespace legacy_flags -} // namespace tensorflow diff --git a/tensorflow/compiler/jit/legacy_flags/xla_device_flags.h b/tensorflow/compiler/jit/legacy_flags/xla_device_flags.h deleted file mode 100644 index 27b22121ac1..00000000000 --- a/tensorflow/compiler/jit/legacy_flags/xla_device_flags.h +++ /dev/null @@ -1,47 +0,0 @@ -/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -#ifndef TENSORFLOW_COMPILER_JIT_LEGACY_FLAGS_XLA_DEVICE_FLAGS_H_ -#define TENSORFLOW_COMPILER_JIT_LEGACY_FLAGS_XLA_DEVICE_FLAGS_H_ - -// Legacy flags for the XLA bridge's xla_device module. - -#include - -#include "tensorflow/core/platform/types.h" -#include "tensorflow/core/util/command_line_flags.h" - -namespace tensorflow { -namespace legacy_flags { - -// The values of flags associated with the XLA bridge's -// xla_device module. -typedef struct { - // Switch the CPU device into "on-demand" mode, where instead of - // autoclustering ops are compiled one by one just-in-time. - // Enabling this mode by a legacy flag is a temporary mechanism. When this - // feature is battle-tested, we will switch this to be a session option. - bool tf_xla_compile_on_demand; -} XlaDeviceFlags; - -// Return a pointer to the XlaDeviceFlags struct; -// repeated calls return the same pointer. -// This should be called only after Flags::Parse() has returned. -XlaDeviceFlags* GetXlaDeviceFlags(); - -} // namespace legacy_flags -} // namespace tensorflow - -#endif // TENSORFLOW_COMPILER_JIT_LEGACY_FLAGS_XLA_DEVICE_FLAGS_H_ diff --git a/tensorflow/compiler/jit/legacy_flags/xla_ops_common_flags.cc b/tensorflow/compiler/jit/legacy_flags/xla_ops_common_flags.cc deleted file mode 100644 index 1443d48a734..00000000000 --- a/tensorflow/compiler/jit/legacy_flags/xla_ops_common_flags.cc +++ /dev/null @@ -1,52 +0,0 @@ -/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -#include // NOLINT -#include - -#include "tensorflow/compiler/jit/legacy_flags/xla_ops_common_flags.h" -#include "tensorflow/compiler/xla/parse_flags_from_env.h" -#include "tensorflow/core/platform/logging.h" -#include "tensorflow/core/util/command_line_flags.h" - -namespace tensorflow { -namespace legacy_flags { - -XlaOpsCommonFlags* flags; -std::vector* flag_list; -std::once_flag flags_init; - -void AllocateAndParseFlags() { - flags = new XlaOpsCommonFlags; - flags->tf_xla_always_defer_compilation = false; - flag_list = new std::vector({ - Flag("tf_xla_always_defer_compilation", - &flags->tf_xla_always_defer_compilation, ""), - }); - xla::ParseFlagsFromEnv(*flag_list); - - if (VLOG_IS_ON(1)) { - VLOG(1) << "Parsed XlaOpsCommonFlags:"; - VLOG(1) << " tf_xla_always_defer_compilation = " - << flags->tf_xla_always_defer_compilation; - } -} - -const XlaOpsCommonFlags& GetXlaOpsCommonFlags() { - std::call_once(flags_init, &AllocateAndParseFlags); - return *flags; -} -} // namespace legacy_flags -} // namespace tensorflow diff --git a/tensorflow/compiler/jit/legacy_flags/xla_ops_common_flags.h b/tensorflow/compiler/jit/legacy_flags/xla_ops_common_flags.h deleted file mode 100644 index 7c5c1818ef2..00000000000 --- a/tensorflow/compiler/jit/legacy_flags/xla_ops_common_flags.h +++ /dev/null @@ -1,36 +0,0 @@ -/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ -#ifndef TENSORFLOW_COMPILER_JIT_LEGACY_FLAGS_XLA_OPS_COMMON_FLAGS_H_ -#define TENSORFLOW_COMPILER_JIT_LEGACY_FLAGS_XLA_OPS_COMMON_FLAGS_H_ - -namespace tensorflow { -namespace legacy_flags { - -// Flags common to the _Xla* ops and their kernels. -struct XlaOpsCommonFlags { - // If true, _XlaCompile always refuses to compile the cluster, which means the - // XLA clusters always run in the TF executor. Defaults to false. - bool tf_xla_always_defer_compilation; -}; - -// Parses the flags in XlaOpsCommonFlags from the TF_XLA_FLAGS environment -// variable and returns a reference to the parsed copy. Parses TF_XLA_FLAGS -// only the first time this routine is called. -const XlaOpsCommonFlags& GetXlaOpsCommonFlags(); - -} // namespace legacy_flags -} // namespace tensorflow - -#endif // TENSORFLOW_COMPILER_JIT_LEGACY_FLAGS_XLA_OPS_COMMON_FLAGS_H_ diff --git a/tensorflow/compiler/jit/mark_for_compilation_pass.cc b/tensorflow/compiler/jit/mark_for_compilation_pass.cc index 70033cae0af..6618e3a58ab 100644 --- a/tensorflow/compiler/jit/mark_for_compilation_pass.cc +++ b/tensorflow/compiler/jit/mark_for_compilation_pass.cc @@ -24,8 +24,8 @@ limitations under the License. #include "absl/container/flat_hash_set.h" #include "tensorflow/compiler/jit/deadness_analysis.h" #include "tensorflow/compiler/jit/defs.h" +#include "tensorflow/compiler/jit/flags.h" #include "tensorflow/compiler/jit/graphcycles/graphcycles.h" -#include "tensorflow/compiler/jit/legacy_flags/mark_for_compilation_pass_flags.h" #include "tensorflow/compiler/jit/union_find.h" #include "tensorflow/compiler/jit/xla_cluster_util.h" #include "tensorflow/compiler/tf2xla/const_analysis.h" @@ -72,6 +72,11 @@ struct OperationFilter { // to resort to a dummy implementation. Currently Assert and CheckNumerics ops // have dummy XLA implementations. bool allow_dummy_ops; + + // Whether ops that produce or consume DT_VARIANT values are allowed. We + // don't auto-cluster these ops because we don't yet support live-in or + // live-out DT_VARIANT values. + bool allow_ops_producing_or_consuming_variant; }; bool IsDummyImplOp(absl::string_view op_name) { @@ -81,7 +86,13 @@ bool IsDummyImplOp(absl::string_view op_name) { bool IsStatefulRandomOp(absl::string_view op_name) { return op_name == "RandomUniform" || op_name == "RandomShuffle" || op_name == "RandomUniformInt" || op_name == "RandomStandardNormal" || - op_name == "TruncatedNormal"; + op_name == "TruncatedNormal" || op_name == "Multinomial"; +} + +bool OpProducesOrConsumesVariant(const Node& node) { + auto is_variant = [](DataType dtype) { return dtype == DT_VARIANT; }; + return absl::c_any_of(node.input_types(), is_variant) || + absl::c_any_of(node.output_types(), is_variant); } bool HasXLAKernel(const Node& node, const DeviceType& jit_device_type) { @@ -246,6 +257,10 @@ bool IsCompilableCall(const NodeDef& call_def, if (!op_filter.allow_dummy_ops && IsDummyImplOp(node->type_string())) { return false; } + if (!op_filter.allow_ops_producing_or_consuming_variant && + OpProducesOrConsumesVariant(*node)) { + return false; + } if (!HasXLAKernel(*node, jit_device_type) && !IsCompilableCall(node->def(), jit_device_type, op_filter, depth + 1, lib_runtime)) { @@ -427,8 +442,7 @@ Status FindCompilationCandidates( BackwardsConstAnalysis(graph, /*compile_time_const_arg_indices=*/nullptr, &compile_time_const_nodes)); - int64& fuel = - legacy_flags::GetMarkForCompilationPassFlags()->tf_xla_clustering_fuel; + int64& fuel = GetMarkForCompilationPassFlags()->tf_xla_clustering_fuel; // Iterate over nodes in sorted order so that compiler fuel is deterministic. // We can't simply pass op_nodes().begin() and op_nodes().end to the @@ -471,16 +485,15 @@ Status FindCompilationCandidates( XlaOpRegistry::GetCompilationDevice(device_type.type(), ®istration)); DeviceType jit_device_type(registration->compilation_device_name); + bool always_auto_cluster = registration->autoclustering_policy == + XlaOpRegistry::AutoclusteringPolicy::kAlways; + OperationFilter op_filter; op_filter.allow_resource_ops = registration->compile_resource_ops; - op_filter.allow_stateful_rng_ops = - (registration->autoclustering_policy == - XlaOpRegistry::AutoclusteringPolicy::kAlways); - op_filter.allow_control_trigger = - (registration->autoclustering_policy == - XlaOpRegistry::AutoclusteringPolicy::kAlways); - op_filter.allow_dummy_ops = (registration->autoclustering_policy == - XlaOpRegistry::AutoclusteringPolicy::kAlways); + op_filter.allow_stateful_rng_ops = always_auto_cluster; + op_filter.allow_control_trigger = always_auto_cluster; + op_filter.allow_dummy_ops = always_auto_cluster; + op_filter.allow_ops_producing_or_consuming_variant = always_auto_cluster; if (!HasXLAKernel(*node, jit_device_type) && !IsCompilableCall(node->def(), jit_device_type, op_filter, 0, @@ -504,6 +517,12 @@ Status FindCompilationCandidates( << node->type_string() << ")"; continue; } + if (!op_filter.allow_ops_producing_or_consuming_variant && + OpProducesOrConsumesVariant(*node)) { + VLOG(2) << "Rejecting " << node->name() + << ": produces or consumes DT_VARIANT"; + continue; + } if (!op_filter.allow_resource_ops && (HasResourceOutput(*node) || IsNonResourceVarResourceOp(*node))) { @@ -607,8 +626,7 @@ OptimizerOptions::GlobalJitLevel GetGlobalJitLevel( // To set compilation to be on by default, change the following line. global_jit_level = OptimizerOptions::OFF; } - legacy_flags::MarkForCompilationPassFlags* flags = - legacy_flags::GetMarkForCompilationPassFlags(); + MarkForCompilationPassFlags* flags = GetMarkForCompilationPassFlags(); if (flags->tf_xla_auto_jit == -1 || (1 <= flags->tf_xla_auto_jit && flags->tf_xla_auto_jit <= 2)) { // If the flag tf_xla_auto_jit is a valid, non-zero setting, it overrides @@ -641,6 +659,7 @@ bool IsCompilable(FunctionLibraryRuntime* flr, const NodeDef& ndef) { op_filter.allow_stateful_rng_ops = true; op_filter.allow_control_trigger = true; op_filter.allow_dummy_ops = true; + op_filter.allow_ops_producing_or_consuming_variant = true; return IsCompilableCall(ndef, jit_device_type, op_filter, 0, flr); } @@ -651,8 +670,7 @@ Status MarkForCompilationPass::Run( // device ahead of time. OptimizerOptions::GlobalJitLevel global_jit_level = GetGlobalJitLevel(options); - legacy_flags::MarkForCompilationPassFlags* flags = - legacy_flags::GetMarkForCompilationPassFlags(); + MarkForCompilationPassFlags* flags = GetMarkForCompilationPassFlags(); bool fusion_only = flags->tf_xla_fusion_only; VLOG(1) << "flags->tf_xla_fusion_only = " << flags->tf_xla_fusion_only; @@ -953,8 +971,7 @@ Status MarkForCompilationPass::RunImpl( OptimizerOptions::GlobalJitLevel global_jit_level = GetGlobalJitLevel(options); - legacy_flags::MarkForCompilationPassFlags* flags = - legacy_flags::GetMarkForCompilationPassFlags(); + MarkForCompilationPassFlags* flags = GetMarkForCompilationPassFlags(); // Repeatedly contract edges between clusters that are on the same device, // provided the contraction would not create a cycle. diff --git a/tensorflow/compiler/jit/mark_for_compilation_pass_test.cc b/tensorflow/compiler/jit/mark_for_compilation_pass_test.cc index 24d78c07726..bf2c5508ea9 100644 --- a/tensorflow/compiler/jit/mark_for_compilation_pass_test.cc +++ b/tensorflow/compiler/jit/mark_for_compilation_pass_test.cc @@ -22,6 +22,7 @@ limitations under the License. #include "tensorflow/cc/ops/array_ops.h" #include "tensorflow/cc/ops/control_flow_ops_internal.h" #include "tensorflow/cc/ops/function_ops.h" +#include "tensorflow/cc/ops/list_ops.h" #include "tensorflow/cc/ops/resource_variable_ops.h" #include "tensorflow/cc/ops/sendrecv_ops.h" #include "tensorflow/cc/ops/standard_ops.h" @@ -1147,5 +1148,80 @@ TEST(XlaCompilationTest, DontAutoClusterDummyOps) { EXPECT_EQ(clusters["test/check"], ""); } +TEST(XlaCompilationTest, DontAutoClusterOpsProducingVariant) { + Scope root = Scope::NewRootScope().ExitOnError(); + Output a = ops::Placeholder(root.WithOpName("test/a"), DT_INT64); + Output b = ops::Placeholder(root.WithOpName("test/b"), DT_INT64); + + Output cast_a = ops::Cast(root.WithOpName("test/cast_a"), a, DT_INT32); + Output cast_b = ops::Cast(root.WithOpName("test/cast_b"), b, DT_INT32); + + Output tensor_list_reserve = ops::TensorListReserve( + root.WithOpName("test/tensor_list_reserve"), cast_a, cast_b, DT_FLOAT); + + std::unique_ptr graph(new Graph(OpRegistry::Global())); + TF_ASSERT_OK(root.ToGraph(graph.get())); + + TF_ASSERT_OK(MarkForCompilationPassTestHelper::MarkForCompilation(&graph)); + + std::unordered_map clusters = GetClusters(*graph); + EXPECT_EQ(clusters["test/tensor_list_reserve"], ""); +} + +TEST(XlaCompilationTest, DontAutoClusterOpsConsumingVariant) { + Scope root = Scope::NewRootScope().ExitOnError(); + Output dummy_input = + ops::Placeholder(root.WithOpName("test/dummy_input"), DT_INT64); + Output variant_input = + ops::Placeholder(root.WithOpName("test/variant_input"), DT_VARIANT); + + // Create one more node so that we don't avoid creating a cluster solely + // because it would be trivial. + Output dummy_cast = + ops::Cast(root.WithOpName("test/dummy_cast"), dummy_input, DT_INT32); + + Output tensor_list_element_shape = ops::TensorListElementShape( + root.WithOpName("test/tensor_list_element_shape"), variant_input, + DT_INT32); + + root.graph()->AddControlEdge(dummy_cast.node(), + tensor_list_element_shape.node()); + + std::unique_ptr graph(new Graph(OpRegistry::Global())); + TF_ASSERT_OK(root.ToGraph(graph.get())); + + TF_ASSERT_OK(MarkForCompilationPassTestHelper::MarkForCompilation(&graph)); + + std::unordered_map clusters = GetClusters(*graph); + EXPECT_EQ(clusters["test/tensor_list_element_shape"], ""); +} + +TEST(XlaCompilationTest, ClusterOpsProducingVariantIfOnXlaDevice) { + Scope root = Scope::NewRootScope().ExitOnError(); + Output a = ops::Placeholder(root.WithOpName("test/a"), DT_INT64); + Output b = ops::Placeholder(root.WithOpName("test/b"), DT_INT64); + + Output cast_a = ops::Cast(root.WithOpName("test/cast_a"), a, DT_INT32); + Output cast_b = ops::Cast(root.WithOpName("test/cast_b"), b, DT_INT32); + + Output tensor_list_reserve = ops::TensorListReserve( + root.WithOpName("test/tensor_list_reserve"), cast_a, cast_b, DT_FLOAT); + + std::unique_ptr graph(new Graph(OpRegistry::Global())); + TF_ASSERT_OK(root.ToGraph(graph.get())); + + string xla_cpu_device = "/job:worker/replica:0/task:0/device:XLA_CPU:0"; + for (Node* n : graph->nodes()) { + if (absl::StartsWith(n->name(), /*prefix=*/"test/")) { + n->set_assigned_device_name(xla_cpu_device); + } + } + + TF_ASSERT_OK(MarkForCompilationPassTestHelper::MarkForCompilation(&graph)); + + std::unordered_map clusters = GetClusters(*graph); + EXPECT_NE(clusters["test/tensor_list_reserve"], ""); +} + } // namespace } // namespace tensorflow diff --git a/tensorflow/compiler/jit/mark_for_compilation_pass_test_helper.cc b/tensorflow/compiler/jit/mark_for_compilation_pass_test_helper.cc index d56d0f8ccfc..64a33017457 100644 --- a/tensorflow/compiler/jit/mark_for_compilation_pass_test_helper.cc +++ b/tensorflow/compiler/jit/mark_for_compilation_pass_test_helper.cc @@ -34,15 +34,9 @@ namespace tensorflow { // // It may be worth refactoring out XlaOpRegistry::RegisterCompilationDevice to // make this more direct, but probably not worth it solely for this test. - std::vector devices; + std::vector> devices; TF_RETURN_IF_ERROR(DeviceFactory::AddDevices(*session_options, "", &devices)); - auto delete_devices = gtl::MakeCleanup([&] { - for (Device* d : devices) { - delete d; - } - }); - GraphOptimizationPassOptions opt_options; opt_options.graph = graph; opt_options.session_options = session_options; diff --git a/tensorflow/compiler/jit/ops/BUILD b/tensorflow/compiler/jit/ops/BUILD index f72224545b2..64409d93347 100644 --- a/tensorflow/compiler/jit/ops/BUILD +++ b/tensorflow/compiler/jit/ops/BUILD @@ -18,3 +18,9 @@ tf_gen_op_wrapper_py( out = "xla_ops.py", deps = ["//tensorflow/compiler/jit/ops:xla_ops"], ) + +py_library( + name = "xla_ops_grad", + srcs = ["xla_ops_grad.py"], + deps = ["//tensorflow/python:framework_ops"], +) diff --git a/tensorflow/contrib/estimator/python/estimator/dnn.py b/tensorflow/compiler/jit/ops/xla_ops_grad.py similarity index 62% rename from tensorflow/contrib/estimator/python/estimator/dnn.py rename to tensorflow/compiler/jit/ops/xla_ops_grad.py index 10f657df8de..2d31d8dc714 100644 --- a/tensorflow/contrib/estimator/python/estimator/dnn.py +++ b/tensorflow/compiler/jit/ops/xla_ops_grad.py @@ -1,3 +1,4 @@ +"""Gradients for XLA ops.""" # Copyright 2018 The TensorFlow Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -12,21 +13,17 @@ # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================== -"""dnn python module. - -Importing from tensorflow.python.estimator is unsupported -and will soon break! -""" -# pylint: disable=unused-import,g-bad-import-order,g-import-not-at-top,wildcard-import from __future__ import absolute_import from __future__ import division from __future__ import print_function -from tensorflow_estimator.contrib.estimator.python.estimator import dnn +from tensorflow.python.framework import ops -# Include attrs that start with single underscore. -_HAS_DYNAMIC_ATTRIBUTES = True -dnn.__all__ = [s for s in dir(dnn) if not s.startswith('__')] -from tensorflow_estimator.contrib.estimator.python.estimator.dnn import * +@ops.RegisterGradient("XlaClusterOutput") +def _XlaClusterOutputGrad(_, grad): + del grad # unused + raise RuntimeError("Gradient computation of graph in xla.compile() is " + "prohibited because it can cause performance degradation." + "Please move gradient computation inside xla.compile().") diff --git a/tensorflow/compiler/jit/partially_decluster_pass.cc b/tensorflow/compiler/jit/partially_decluster_pass.cc index 36b345ecbff..42ea3926e16 100644 --- a/tensorflow/compiler/jit/partially_decluster_pass.cc +++ b/tensorflow/compiler/jit/partially_decluster_pass.cc @@ -26,6 +26,10 @@ limitations under the License. namespace tensorflow { namespace { + +bool NotBackedge(const Edge& edge) { return !edge.src()->IsNextIteration(); } + +namespace reduce_device_to_host_copies { Status FindNodesToDecluster(const Graph& graph, absl::flat_hash_set* result, absl::Span post_order) { @@ -140,8 +144,6 @@ Status PartiallyDeclusterNode(Graph* graph, Node* n) { return Status::OK(); } -bool NotBackedge(const Edge& edge) { return !edge.src()->IsNextIteration(); } - // Clones nodes to outside their cluster to avoid device-to-host copies. For // instance, converts this: // @@ -168,7 +170,7 @@ bool NotBackedge(const Edge& edge) { return !edge.src()->IsNextIteration(); } // where the ===> arrow has a hostmem source and destination and would entail a // device to host copy if the source and destination were not in the same XLA // cluster. -Status PartiallyDeclusterToRemoveDeviceToHostCopies(Graph* graph) { +Status PartiallyDeclusterGraph(Graph* graph) { // When deciding whether to decluster a particular node, we base our decision // on if we've decided that some of its consumers have to be declustered too. // Iterating the graph in post-order guarantees that consumers have been @@ -206,7 +208,9 @@ Status PartiallyDeclusterToRemoveDeviceToHostCopies(Graph* graph) { return Status::OK(); } +} // namespace reduce_device_to_host_copies +namespace reduce_recompilation { bool IsIntraClusterEdge(const Edge& edge) { absl::optional src_cluster_name = GetXlaClusterForNode(*edge.src()); @@ -269,7 +273,7 @@ Status MustCompileNode(const Node* n, bool* must_compile) { // regress performance in any significant manner. We will have to revisit this // algorith with a more complex cost model if this assumption turns out to be // incorrect. -Status DeclusterNodesToReduceRecompilations(Graph* graph) { +Status PartiallyDeclusterGraph(Graph* graph) { std::vector compile_time_const_nodes(graph->num_node_ids()); TF_RETURN_IF_ERROR(BackwardsConstAnalysis( *graph, nullptr, &compile_time_const_nodes, IsIntraClusterEdge)); @@ -322,7 +326,7 @@ Status DeclusterNodesToReduceRecompilations(Graph* graph) { return Status::OK(); } - +} // namespace reduce_recompilation } // namespace Status PartiallyDeclusterPass::Run( @@ -334,8 +338,9 @@ Status PartiallyDeclusterPass::Run( Graph* graph = options.graph->get(); - TF_RETURN_IF_ERROR(PartiallyDeclusterToRemoveDeviceToHostCopies(graph)); - TF_RETURN_IF_ERROR(DeclusterNodesToReduceRecompilations(graph)); + TF_RETURN_IF_ERROR( + reduce_device_to_host_copies::PartiallyDeclusterGraph(graph)); + TF_RETURN_IF_ERROR(reduce_recompilation::PartiallyDeclusterGraph(graph)); return Status::OK(); } diff --git a/tensorflow/compiler/jit/partially_decluster_pass_test.cc b/tensorflow/compiler/jit/partially_decluster_pass_test.cc index 1fc5da5071f..38a54cc5efa 100644 --- a/tensorflow/compiler/jit/partially_decluster_pass_test.cc +++ b/tensorflow/compiler/jit/partially_decluster_pass_test.cc @@ -386,7 +386,7 @@ TEST(PartiallyDeclusterPassTest, DontDeclusterXlaDeviceOps) { TF_ASSERT_OK(s.ToGraph(graph.get())); // This is needed to register the XLA_GPU device. - std::vector devices; + std::vector> devices; TF_ASSERT_OK(DeviceFactory::AddDevices( SessionOptions(), "/job:localhost/replica:0/task:0", &devices)); @@ -400,10 +400,6 @@ TEST(PartiallyDeclusterPassTest, DontDeclusterXlaDeviceOps) { TF_ASSERT_OK(PartiallyDecluster(&graph)); EXPECT_EQ(GetXlaClusterForNode(*n), "cluster_0"); - - for (Device* d : devices) { - delete d; - } } TEST(PartiallyDeclusterPassTest, DontDeclusterNonTensorFlowOps) { diff --git a/tensorflow/compiler/jit/xla_cpu_device.cc b/tensorflow/compiler/jit/xla_cpu_device.cc index 116e0756036..7df898ad12a 100644 --- a/tensorflow/compiler/jit/xla_cpu_device.cc +++ b/tensorflow/compiler/jit/xla_cpu_device.cc @@ -17,8 +17,8 @@ limitations under the License. // operators using XLA via the XLA "Host" (CPU) backend. #include "absl/memory/memory.h" +#include "tensorflow/compiler/jit/flags.h" #include "tensorflow/compiler/jit/kernels/xla_ops.h" -#include "tensorflow/compiler/jit/legacy_flags/xla_device_flags.h" #include "tensorflow/compiler/jit/xla_compile_on_demand_op.h" #include "tensorflow/compiler/jit/xla_device.h" #include "tensorflow/compiler/jit/xla_device_ops.h" @@ -31,13 +31,13 @@ namespace tensorflow { class XlaCpuDeviceFactory : public DeviceFactory { public: Status CreateDevices(const SessionOptions& options, const string& name_prefix, - std::vector* devices) override; + std::vector>* devices) override; }; -Status XlaCpuDeviceFactory::CreateDevices(const SessionOptions& session_options, - const string& name_prefix, - std::vector* devices) { - legacy_flags::XlaDeviceFlags* flags = legacy_flags::GetXlaDeviceFlags(); +Status XlaCpuDeviceFactory::CreateDevices( + const SessionOptions& session_options, const string& name_prefix, + std::vector>* devices) { + XlaDeviceFlags* flags = GetXlaDeviceFlags(); bool compile_on_demand = flags->tf_xla_compile_on_demand; XlaOpRegistry::DeviceRegistration registration; @@ -63,8 +63,7 @@ Status XlaCpuDeviceFactory::CreateDevices(const SessionOptions& session_options, options.device_ordinal = 0; options.compilation_device_name = DEVICE_CPU_XLA_JIT; options.use_multiple_streams = false; - auto device = absl::make_unique(session_options, options); - devices->push_back(device.release()); + devices->push_back(absl::make_unique(session_options, options)); return Status::OK(); } diff --git a/tensorflow/compiler/jit/xla_device.cc b/tensorflow/compiler/jit/xla_device.cc index 5c1b55cb57f..4201ff91a89 100644 --- a/tensorflow/compiler/jit/xla_device.cc +++ b/tensorflow/compiler/jit/xla_device.cc @@ -218,6 +218,9 @@ XlaDevice::XlaDevice(const SessionOptions& session_options, XlaDevice::~XlaDevice() { VLOG(1) << "Destroying XLA device " << jit_device_name_ << " " << this; mutex_lock lock(mu_); + while (outstanding_asynchronous_operations_ > 0) { + outstanding_asynchronous_operations_cv_.wait(lock); + } if (device_context_) { device_context_->Unref(); } @@ -384,6 +387,7 @@ void XlaDevice::ComputeAsync(AsyncOpKernel* op_kernel, OpKernelContext* context, Status XlaDevice::Sync() { VLOG(1) << "XlaDevice::Sync"; + tracing::ScopedActivity activity("XlaDevice::Sync", /*is_expensive=*/true); std::shared_ptr stream; { mutex_lock lock(mu_); @@ -391,13 +395,46 @@ Status XlaDevice::Sync() { } if (!stream) return Status::OK(); - if (!stream->parent()->SynchronizeAllActivity() || !stream->ok()) { + Status status = stream->BlockHostUntilDone(); + { + mutex_lock lock(mu_); + while (outstanding_asynchronous_operations_ > 0) { + outstanding_asynchronous_operations_cv_.wait(lock); + } + } + TF_RETURN_IF_ERROR(status); + if (!stream->ok()) { return errors::Internal("XlaDevice::Sync() failed."); } VLOG(1) << "XlaDevice::Sync completed"; return Status::OK(); } +void XlaDevice::Sync(const DoneCallback& done) { + VLOG(1) << "XlaDevice::Sync (asynchronous)"; + std::shared_ptr stream; + { + mutex_lock lock(mu_); + stream = stream_; + } + if (!stream) { + done(Status::OK()); + return; + } + + stream->ThenEnqueueOnBackgroundThread( + [this, stream, done](se::StreamExecutor*) { + tracing::ScopedActivity activity("XlaDevice::Sync::Callback", + /*is_expensive=*/true); + mutex_lock lock(mu_); + while (outstanding_asynchronous_operations_ > 0) { + outstanding_asynchronous_operations_cv_.wait(lock); + } + done(stream->ok() ? Status::OK() + : errors::Internal("XlaDevice::Sync() failed.")); + }); +} + Status XlaDevice::MakeTensorFromProto(const TensorProto& tensor_proto, const AllocatorAttributes alloc_attrs, Tensor* tensor) { @@ -441,6 +478,49 @@ bool XlaDevice::RequiresSyncOnCompletion() const { return sync_on_completion_; } +XlaDevice::AsynchronousOperationHandle::AsynchronousOperationHandle( + XlaDevice* device) + : device_(device) { + mutex_lock lock(device_->mu_); + ++device_->outstanding_asynchronous_operations_; +} + +XlaDevice::AsynchronousOperationHandle::~AsynchronousOperationHandle() { + if (device_) { + mutex_lock lock(device_->mu_); + --device_->outstanding_asynchronous_operations_; + device_->outstanding_asynchronous_operations_cv_.notify_all(); + } +} + +XlaDevice::AsynchronousOperationHandle::AsynchronousOperationHandle( + const XlaDevice::AsynchronousOperationHandle& other) + : device_(other.device_) { + mutex_lock lock(device_->mu_); + ++device_->outstanding_asynchronous_operations_; +} + +XlaDevice::AsynchronousOperationHandle::AsynchronousOperationHandle( + XlaDevice::AsynchronousOperationHandle&& other) + : device_(other.device_) { + other.device_ = nullptr; +} + +XlaDevice::AsynchronousOperationHandle& XlaDevice::AsynchronousOperationHandle:: +operator=(const XlaDevice::AsynchronousOperationHandle& other) { + device_ = other.device_; + mutex_lock lock(device_->mu_); + ++device_->outstanding_asynchronous_operations_; + return *this; +} + +XlaDevice::AsynchronousOperationHandle& XlaDevice::AsynchronousOperationHandle:: +operator=(XlaDevice::AsynchronousOperationHandle&& other) { + device_ = other.device_; + other.device_ = nullptr; + return *this; +} + XlaDeviceOpRegistrations* RegisterXlaDeviceKernels(const char* device, const char* jit_device) { // Any op assigned to the device that isn't rewritten by the graph rewriter diff --git a/tensorflow/compiler/jit/xla_device.h b/tensorflow/compiler/jit/xla_device.h index 49f53b477ef..c8bb276cdb9 100644 --- a/tensorflow/compiler/jit/xla_device.h +++ b/tensorflow/compiler/jit/xla_device.h @@ -135,6 +135,7 @@ class XlaDevice : public LocalDevice { void ComputeAsync(AsyncOpKernel* op_kernel, OpKernelContext* context, AsyncOpKernel::DoneCallback done) override; Status Sync() override; + void Sync(const DoneCallback& done) override; Status FillContextMap(const Graph* graph, DeviceContextMap* device_context_map) override @@ -164,7 +165,30 @@ class XlaDevice : public LocalDevice { bool RequiresSyncOnCompletion() const override LOCKS_EXCLUDED(mu_); + // A simple RAII handle. On construction the device's + // outstanding_asynchronous_operations_ field is incremented; on destruction + // it is decremented. + class AsynchronousOperationHandle { + public: + AsynchronousOperationHandle(XlaDevice* device); + ~AsynchronousOperationHandle(); + AsynchronousOperationHandle(const AsynchronousOperationHandle& other); + AsynchronousOperationHandle(AsynchronousOperationHandle&& other); + AsynchronousOperationHandle& operator=( + const AsynchronousOperationHandle& other); + AsynchronousOperationHandle& operator=(AsynchronousOperationHandle&& other); + + private: + XlaDevice* device_ = nullptr; + }; + + AsynchronousOperationHandle CreateAsynchronousOperationHandle() { + return AsynchronousOperationHandle(this); + } + private: + friend class AsynchronousOperationHandle; + xla::LocalClient* client() const; Allocator* GetAllocatorLocked(AllocatorAttributes attr) EXCLUSIVE_LOCKS_REQUIRED(mu_); @@ -227,6 +251,11 @@ class XlaDevice : public LocalDevice { // True if the device requires XlaDevice::Sync to be called on completion // regardless of status. bool sync_on_completion_ GUARDED_BY(mu_) = false; + + // Count of outstanding asynchronous operations which must be zero on Sync() + // completion. + int64 outstanding_asynchronous_operations_ GUARDED_BY(mu_) = 0; + condition_variable outstanding_asynchronous_operations_cv_; }; // Builds OpKernel registrations on 'device' for the JIT operators diff --git a/tensorflow/compiler/jit/xla_gpu_device.cc b/tensorflow/compiler/jit/xla_gpu_device.cc index 44197016958..944f732b99c 100644 --- a/tensorflow/compiler/jit/xla_gpu_device.cc +++ b/tensorflow/compiler/jit/xla_gpu_device.cc @@ -29,12 +29,12 @@ namespace tensorflow { class XlaGpuDeviceFactory : public DeviceFactory { public: Status CreateDevices(const SessionOptions& options, const string& name_prefix, - std::vector* devices) override; + std::vector>* devices) override; }; -Status XlaGpuDeviceFactory::CreateDevices(const SessionOptions& session_options, - const string& name_prefix, - std::vector* devices) { +Status XlaGpuDeviceFactory::CreateDevices( + const SessionOptions& session_options, const string& name_prefix, + std::vector>* devices) { XlaOpRegistry::DeviceRegistration registration; registration.compilation_device_name = DEVICE_GPU_XLA_JIT; registration.autoclustering_policy = @@ -70,7 +70,7 @@ Status XlaGpuDeviceFactory::CreateDevices(const SessionOptions& session_options, return status; } - devices->push_back(device.release()); + devices->push_back(std::move(device)); } return Status::OK(); } diff --git a/tensorflow/compiler/jit/xla_interpreter_device.cc b/tensorflow/compiler/jit/xla_interpreter_device.cc index e828bae865d..4007309ed1c 100644 --- a/tensorflow/compiler/jit/xla_interpreter_device.cc +++ b/tensorflow/compiler/jit/xla_interpreter_device.cc @@ -33,12 +33,12 @@ constexpr std::array kExecAllTypes = { class XlaInterpreterDeviceFactory : public DeviceFactory { public: Status CreateDevices(const SessionOptions& options, const string& name_prefix, - std::vector* devices) override; + std::vector>* devices) override; }; Status XlaInterpreterDeviceFactory::CreateDevices( const SessionOptions& session_options, const string& name_prefix, - std::vector* devices) { + std::vector>* devices) { static XlaDeviceOpRegistrations* registrations = RegisterXlaDeviceKernels( DEVICE_XLA_INTERPRETER, DEVICE_INTERPRETER_XLA_JIT); (void)registrations; @@ -61,8 +61,7 @@ Status XlaInterpreterDeviceFactory::CreateDevices( options.device_ordinal = 0; options.compilation_device_name = DEVICE_INTERPRETER_XLA_JIT; options.use_multiple_streams = false; - auto device = absl::make_unique(session_options, options); - devices->push_back(device.release()); + devices->push_back(absl::make_unique(session_options, options)); return Status::OK(); } diff --git a/tensorflow/compiler/tests/BUILD b/tensorflow/compiler/tests/BUILD index 6b8e6bba1e1..bc3d60b90e5 100644 --- a/tensorflow/compiler/tests/BUILD +++ b/tensorflow/compiler/tests/BUILD @@ -375,27 +375,6 @@ tf_xla_py_test( ], ) -tf_xla_py_test( - name = "resampler_ops_test", - size = "small", - srcs = ["resampler_ops_test.py"], - disabled_backends = [ - # TODO(b/74459949) Support BatchDot in CPU backend. - "cpu", - "cpu_ondemand", - ], - # TODO(b/112295522): figure out how to make OSS build pass. - tags = ["no_oss"], - deps = [ - ":xla_test", - "//tensorflow/contrib/resampler:resampler_ops", - "//tensorflow/contrib/resampler:resampler_py", - "//tensorflow/python:array_ops", - "//tensorflow/python:client_testlib", - "//tensorflow/python:platform_test", - ], -) - tf_xla_py_test( name = "dynamic_stitch_test", size = "small", @@ -474,7 +453,6 @@ tf_xla_py_test( "//tensorflow/python:extra_py_tests_deps", "//tensorflow/python:framework", "//tensorflow/python:platform_test", - "//tensorflow/python:spectral_ops", "//tensorflow/python/ops/signal", ], ) diff --git a/tensorflow/compiler/tests/adagrad_da_test.py b/tensorflow/compiler/tests/adagrad_da_test.py index 69fb3ec2964..e9c2d363aca 100644 --- a/tensorflow/compiler/tests/adagrad_da_test.py +++ b/tensorflow/compiler/tests/adagrad_da_test.py @@ -50,8 +50,8 @@ class AdagradDAOptimizerTest(xla_test.XLATestCase): zip([grads0, grads1], [var0, var1]), global_step=global_step) variables.global_variables_initializer().run() - self.assertAllClose([0.0, 0.0], var0.eval()) - self.assertAllClose([0.0, 0.0], var1.eval()) + self.assertAllClose([0.0, 0.0], self.evaluate(var0)) + self.assertAllClose([0.0, 0.0], self.evaluate(var1)) # Run a step of AdagradDA update.run() @@ -63,9 +63,9 @@ class AdagradDAOptimizerTest(xla_test.XLATestCase): # For -0.1*3.0*(0.1 - 0)/(0 + sqrt(0.1 + 0.1*0.1)) = -0.904534 # similarly for others. self.assertAllCloseAccordingToType( - np.array([-0.904534, -1.603567]), var0.eval()) + np.array([-0.904534, -1.603567]), self.evaluate(var0)) self.assertAllCloseAccordingToType( - np.array([-0.094821, -0.189358]), var1.eval()) + np.array([-0.094821, -0.189358]), self.evaluate(var1)) def testAdagradDAwithoutRegularizationBasic2(self): for dtype in self.float_types: @@ -87,16 +87,16 @@ class AdagradDAOptimizerTest(xla_test.XLATestCase): zip([grads0, grads1], [var0, var1]), global_step=global_step) variables.global_variables_initializer().run() - self.assertAllCloseAccordingToType([1.0, 2.0], var0.eval()) - self.assertAllCloseAccordingToType([4.0, 3.0], var1.eval()) + self.assertAllCloseAccordingToType([1.0, 2.0], self.evaluate(var0)) + self.assertAllCloseAccordingToType([4.0, 3.0], self.evaluate(var1)) # Run a step of AdagradDA update.run() self.assertAllCloseAccordingToType( - np.array([-0.904534, -1.603567]), var0.eval()) + np.array([-0.904534, -1.603567]), self.evaluate(var0)) self.assertAllCloseAccordingToType( - np.array([-0.094821, -0.189358]), var1.eval()) + np.array([-0.094821, -0.189358]), self.evaluate(var1)) def testAdagradDAWithL1(self): for dtype in self.float_types: @@ -118,16 +118,16 @@ class AdagradDAOptimizerTest(xla_test.XLATestCase): zip([grads0, grads1], [var0, var1]), global_step=global_step) variables.global_variables_initializer().run() - self.assertAllCloseAccordingToType([1.0, 2.0], var0.eval()) - self.assertAllCloseAccordingToType([4.0, 3.0], var1.eval()) + self.assertAllCloseAccordingToType([1.0, 2.0], self.evaluate(var0)) + self.assertAllCloseAccordingToType([4.0, 3.0], self.evaluate(var1)) # Run a step of AdagradDA update.run() self.assertAllCloseAccordingToType( - np.array([-0.895489, -1.59555]), var0.eval()) + np.array([-0.895489, -1.59555]), self.evaluate(var0)) self.assertAllCloseAccordingToType( - np.array([-0.085339, -0.17989]), var1.eval()) + np.array([-0.085339, -0.17989]), self.evaluate(var1)) def testAdagradDAWithL1_L2(self): for dtype in self.float_types: @@ -149,16 +149,16 @@ class AdagradDAOptimizerTest(xla_test.XLATestCase): zip([grads0, grads1], [var0, var1]), global_step=global_step) variables.global_variables_initializer().run() - self.assertAllCloseAccordingToType([1.0, 2.0], var0.eval()) - self.assertAllCloseAccordingToType([4.0, 3.0], var1.eval()) + self.assertAllCloseAccordingToType([1.0, 2.0], self.evaluate(var0)) + self.assertAllCloseAccordingToType([4.0, 3.0], self.evaluate(var1)) # Run a step of AdagradDA update.run() self.assertAllCloseAccordingToType( - np.array([-0.046907, -0.093659]), var0.eval()) + np.array([-0.046907, -0.093659]), self.evaluate(var0)) self.assertAllCloseAccordingToType( - np.array([-0.004275, -0.009023]), var1.eval()) + np.array([-0.004275, -0.009023]), self.evaluate(var1)) if __name__ == "__main__": diff --git a/tensorflow/compiler/tests/adagrad_test.py b/tensorflow/compiler/tests/adagrad_test.py index ab69319c59f..e26483303c3 100644 --- a/tensorflow/compiler/tests/adagrad_test.py +++ b/tensorflow/compiler/tests/adagrad_test.py @@ -42,17 +42,19 @@ class AdagradOptimizerTest(xla_test.XLATestCase): zip([grads0, grads1], [var0, var1])) variables.global_variables_initializer().run() # Fetch params to validate initial values - self.assertAllClose([1.0, 2.0], var0.eval()) - self.assertAllClose([3.0, 4.0], var1.eval()) + self.assertAllClose([1.0, 2.0], self.evaluate(var0)) + self.assertAllClose([3.0, 4.0], self.evaluate(var1)) # Run 3 steps of adagrad for _ in range(3): ada_update.run() # Validate updated params self.assertAllCloseAccordingToType( - np.array([-1.6026098728179932, -0.6026098728179932]), var0.eval(), + np.array([-1.6026098728179932, -0.6026098728179932]), + self.evaluate(var0), float_rtol=1e-5) self.assertAllCloseAccordingToType( - np.array([2.715679168701172, 3.715679168701172]), var1.eval(), + np.array([2.715679168701172, 3.715679168701172]), + self.evaluate(var1), float_rtol=1e-5) def testTensorLearningRate(self): @@ -68,17 +70,19 @@ class AdagradOptimizerTest(xla_test.XLATestCase): zip([grads0, grads1], [var0, var1])) variables.global_variables_initializer().run() # Fetch params to validate initial values - self.assertAllClose([1.0, 2.0], var0.eval()) - self.assertAllClose([3.0, 4.0], var1.eval()) + self.assertAllClose([1.0, 2.0], self.evaluate(var0)) + self.assertAllClose([3.0, 4.0], self.evaluate(var1)) # Run 3 steps of adagrad for _ in range(3): ada_update.run() # Validate updated params self.assertAllCloseAccordingToType( - np.array([-1.6026098728179932, -0.6026098728179932]), var0.eval(), + np.array([-1.6026098728179932, -0.6026098728179932]), + self.evaluate(var0), float_rtol=1e-5) self.assertAllCloseAccordingToType( - np.array([2.715679168701172, 3.715679168701172]), var1.eval(), + np.array([2.715679168701172, 3.715679168701172]), + self.evaluate(var1), float_rtol=1e-5) def testSharing(self): @@ -103,18 +107,20 @@ class AdagradOptimizerTest(xla_test.XLATestCase): variables.global_variables_initializer().run() # Fetch params to validate initial values. - self.assertAllClose([1.0, 2.0], var0.eval()) - self.assertAllClose([3.0, 4.0], var1.eval()) + self.assertAllClose([1.0, 2.0], self.evaluate(var0)) + self.assertAllClose([3.0, 4.0], self.evaluate(var1)) # Mix the first and the second adagrad for 3 steps. ada_update1.run() ada_update2.run() ada_update1.run() # Validate updated params (the same as with only 1 Adagrad). self.assertAllCloseAccordingToType( - np.array([-1.6026098728179932, -0.6026098728179932]), var0.eval(), + np.array([-1.6026098728179932, -0.6026098728179932]), + self.evaluate(var0), float_rtol=1e-5) self.assertAllCloseAccordingToType( - np.array([2.715679168701172, 3.715679168701172]), var1.eval(), + np.array([2.715679168701172, 3.715679168701172]), + self.evaluate(var1), float_rtol=1e-5) diff --git a/tensorflow/compiler/tests/adam_test.py b/tensorflow/compiler/tests/adam_test.py index 058576b3d4b..8bcff9d379d 100644 --- a/tensorflow/compiler/tests/adam_test.py +++ b/tensorflow/compiler/tests/adam_test.py @@ -75,23 +75,24 @@ class AdamOptimizerTest(xla_test.XLATestCase): variables.global_variables_initializer().run() # Fetch params to validate initial values - self.assertAllClose([1.0, 2.0], var0.eval()) - self.assertAllClose([3.0, 4.0], var1.eval()) + self.assertAllClose([1.0, 2.0], self.evaluate(var0)) + self.assertAllClose([3.0, 4.0], self.evaluate(var1)) beta1_power, beta2_power = opt._get_beta_accumulators() # Run 3 steps of Adam for t in range(1, 4): - self.assertAllCloseAccordingToType(0.9**t, beta1_power.eval()) - self.assertAllCloseAccordingToType(0.999**t, beta2_power.eval()) + self.assertAllCloseAccordingToType(0.9**t, self.evaluate(beta1_power)) + self.assertAllCloseAccordingToType(0.999**t, + self.evaluate(beta2_power)) update.run(feed_dict={grads0: grads0_np, grads1: grads1_np}) var0_np, m0, v0 = adam_update_numpy(var0_np, grads0_np, t, m0, v0) var1_np, m1, v1 = adam_update_numpy(var1_np, grads1_np, t, m1, v1) # Validate updated params - self.assertAllCloseAccordingToType(var0_np, var0.eval()) - self.assertAllCloseAccordingToType(var1_np, var1.eval()) + self.assertAllCloseAccordingToType(var0_np, self.evaluate(var0)) + self.assertAllCloseAccordingToType(var1_np, self.evaluate(var1)) def testTensorLearningRate(self): for dtype in self.float_types: @@ -117,23 +118,24 @@ class AdamOptimizerTest(xla_test.XLATestCase): variables.global_variables_initializer().run() # Fetch params to validate initial values - self.assertAllClose([1.0, 2.0], var0.eval()) - self.assertAllClose([3.0, 4.0], var1.eval()) + self.assertAllClose([1.0, 2.0], self.evaluate(var0)) + self.assertAllClose([3.0, 4.0], self.evaluate(var1)) beta1_power, beta2_power = opt._get_beta_accumulators() # Run 3 steps of Adam for t in range(1, 4): - self.assertAllCloseAccordingToType(0.9**t, beta1_power.eval()) - self.assertAllCloseAccordingToType(0.999**t, beta2_power.eval()) + self.assertAllCloseAccordingToType(0.9**t, self.evaluate(beta1_power)) + self.assertAllCloseAccordingToType(0.999**t, + self.evaluate(beta2_power)) update.run(feed_dict={grads0: grads0_np, grads1: grads1_np}) var0_np, m0, v0 = adam_update_numpy(var0_np, grads0_np, t, m0, v0) var1_np, m1, v1 = adam_update_numpy(var1_np, grads1_np, t, m1, v1) # Validate updated params - self.assertAllCloseAccordingToType(var0_np, var0.eval()) - self.assertAllCloseAccordingToType(var1_np, var1.eval()) + self.assertAllCloseAccordingToType(var0_np, self.evaluate(var0)) + self.assertAllCloseAccordingToType(var1_np, self.evaluate(var1)) def testSharing(self): for dtype in self.float_types: @@ -162,13 +164,14 @@ class AdamOptimizerTest(xla_test.XLATestCase): beta1_power, beta2_power = opt._get_beta_accumulators() # Fetch params to validate initial values - self.assertAllClose([1.0, 2.0], var0.eval()) - self.assertAllClose([3.0, 4.0], var1.eval()) + self.assertAllClose([1.0, 2.0], self.evaluate(var0)) + self.assertAllClose([3.0, 4.0], self.evaluate(var1)) # Run 3 steps of intertwined Adam1 and Adam2. for t in range(1, 4): - self.assertAllCloseAccordingToType(0.9**t, beta1_power.eval()) - self.assertAllCloseAccordingToType(0.999**t, beta2_power.eval()) + self.assertAllCloseAccordingToType(0.9**t, self.evaluate(beta1_power)) + self.assertAllCloseAccordingToType(0.999**t, + self.evaluate(beta2_power)) if t % 2 == 0: update1.run(feed_dict={grads0: grads0_np, grads1: grads1_np}) else: @@ -178,8 +181,8 @@ class AdamOptimizerTest(xla_test.XLATestCase): var1_np, m1, v1 = adam_update_numpy(var1_np, grads1_np, t, m1, v1) # Validate updated params - self.assertAllCloseAccordingToType(var0_np, var0.eval()) - self.assertAllCloseAccordingToType(var1_np, var1.eval()) + self.assertAllCloseAccordingToType(var0_np, self.evaluate(var0)) + self.assertAllCloseAccordingToType(var1_np, self.evaluate(var1)) if __name__ == "__main__": diff --git a/tensorflow/compiler/tests/adamax_test.py b/tensorflow/compiler/tests/adamax_test.py index 3ed1d41b712..961b46375c9 100644 --- a/tensorflow/compiler/tests/adamax_test.py +++ b/tensorflow/compiler/tests/adamax_test.py @@ -78,8 +78,8 @@ class AdaMaxOptimizerTest(xla_test.XLATestCase): variables.global_variables_initializer().run() # Fetch params to validate initial values - self.assertAllClose([1.0, 2.0], var0.eval()) - self.assertAllClose([3.0, 4.0], var1.eval()) + self.assertAllClose([1.0, 2.0], self.evaluate(var0)) + self.assertAllClose([3.0, 4.0], self.evaluate(var1)) beta1_power = opt._get_beta_accumulators() @@ -87,14 +87,17 @@ class AdaMaxOptimizerTest(xla_test.XLATestCase): for t in range(1, 4): update.run() - self.assertAllCloseAccordingToType(0.9**(t + 1), beta1_power.eval()) + self.assertAllCloseAccordingToType(0.9**(t + 1), + self.evaluate(beta1_power)) var0_np, m0, v0 = adamax_update_numpy(var0_np, grads0_np, t, m0, v0) var1_np, m1, v1 = adamax_update_numpy(var1_np, grads1_np, t, m1, v1) # Validate updated params - self.assertAllCloseAccordingToType(var0_np, var0.eval(), rtol=1e-2) - self.assertAllCloseAccordingToType(var1_np, var1.eval(), rtol=1e-2) + self.assertAllCloseAccordingToType( + var0_np, self.evaluate(var0), rtol=1e-2) + self.assertAllCloseAccordingToType( + var1_np, self.evaluate(var1), rtol=1e-2) self.assertEqual("var0_%d/AdaMax:0" % (i,), opt.get_slot(var=var0, name="m").name) @@ -118,22 +121,23 @@ class AdaMaxOptimizerTest(xla_test.XLATestCase): variables.global_variables_initializer().run() # Fetch params to validate initial values - self.assertAllClose([1.0, 2.0], var0.eval()) - self.assertAllClose([3.0, 4.0], var1.eval()) + self.assertAllClose([1.0, 2.0], self.evaluate(var0)) + self.assertAllClose([3.0, 4.0], self.evaluate(var1)) beta1_power = opt._get_beta_accumulators() # Run 3 steps of AdaMax for t in range(1, 4): - self.assertAllCloseAccordingToType(0.9**t, beta1_power.eval()) + self.assertAllCloseAccordingToType(0.9**t, self.evaluate(beta1_power)) update.run() var0_np, m0, v0 = adamax_update_numpy(var0_np, grads0_np, t, m0, v0) var1_np, m1, v1 = adamax_update_numpy(var1_np, grads1_np, t, m1, v1) # Validate updated params - self.assertAllCloseAccordingToType(var0_np, var0.eval()) - self.assertAllCloseAccordingToType(var1_np, var1.eval()) + self.assertAllCloseAccordingToType(var0_np, self.evaluate(var0)) + self.assertAllCloseAccordingToType(var1_np, self.evaluate(var1)) + if __name__ == "__main__": test.main() diff --git a/tensorflow/compiler/tests/addsign_test.py b/tensorflow/compiler/tests/addsign_test.py index 1bc07ace23c..a37c97e6d37 100644 --- a/tensorflow/compiler/tests/addsign_test.py +++ b/tensorflow/compiler/tests/addsign_test.py @@ -90,8 +90,8 @@ class AddSignTest(xla_test.XLATestCase): variables.global_variables_initializer().run() # Fetch params to validate initial values - self.assertAllClose([1.0, 2.0], var0.eval()) - self.assertAllClose([3.0, 4.0], var1.eval()) + self.assertAllClose([1.0, 2.0], self.evaluate(var0)) + self.assertAllClose([3.0, 4.0], self.evaluate(var1)) # Run 7 steps of AddSign # first 4 steps with positive gradient @@ -125,8 +125,8 @@ class AddSignTest(xla_test.XLATestCase): # Validate updated params self.assertAllCloseAccordingToType( - var0_np, var0.eval(), half_rtol=1e-2) - self.assertAllCloseAccordingToType(var1_np, var1.eval()) + var0_np, self.evaluate(var0), half_rtol=1e-2) + self.assertAllCloseAccordingToType(var1_np, self.evaluate(var1)) def testDense(self): decay_steps = 10 diff --git a/tensorflow/compiler/tests/binary_ops_test.py b/tensorflow/compiler/tests/binary_ops_test.py index 332381c59ee..9a5423c1b2a 100644 --- a/tensorflow/compiler/tests/binary_ops_test.py +++ b/tensorflow/compiler/tests/binary_ops_test.py @@ -218,6 +218,21 @@ class BinaryOpsTest(xla_test.XLATestCase): ], equality_test=self.ListsAreClose) + # TF doesn't define these for bf16. + if dtype != dtypes.bfloat16.as_numpy_dtype: + self._testBinary( + gen_math_ops.xdivy, + np.array([0, 4, 3, 2, 1, 0], dtype=dtype), + np.array([0, 5, 6, 7, 8, float("NaN")], dtype=dtype), + expected=np.array([0, 0.8, 0.5, 0.285714, 0.125, 0], dtype=dtype)) + + self._testBinary( + gen_math_ops.xlogy, + np.array([0, 4, 3, 2, 1, 0], dtype=dtype), + np.array([0, 5, 6, 7, 8, float("NaN")], dtype=dtype), + expected=np.array([0, 6.437752, 5.375278, 3.89182, 2.079442, 0], + dtype=dtype)) + def testIntOps(self): for dtype in self.signed_int_types: self._testBinary( diff --git a/tensorflow/compiler/tests/categorical_op_test.py b/tensorflow/compiler/tests/categorical_op_test.py index a57d1dc81ea..5d5e486f616 100644 --- a/tensorflow/compiler/tests/categorical_op_test.py +++ b/tensorflow/compiler/tests/categorical_op_test.py @@ -27,6 +27,7 @@ from tensorflow.python.framework import dtypes from tensorflow.python.framework import random_seed from tensorflow.python.ops import array_ops from tensorflow.python.ops import random_ops +from tensorflow.python.ops import stateless_random_ops from tensorflow.python.platform import googletest @@ -56,11 +57,11 @@ class CategoricalTest(xla_test.XLATestCase): Returns: Frequencies from sampled classes; shape [batch_size, num_classes]. """ - with self.cached_session() as sess, self.test_scope(): + with self.cached_session(), self.test_scope(): random_seed.set_random_seed(1618) op = random_ops.multinomial(logits, num_samples, output_dtype=dtypes.int32) - d = sess.run(op) + d = self.evaluate(op) batch_size, num_classes = logits.shape freqs_mat = [] @@ -79,15 +80,15 @@ class CategoricalTest(xla_test.XLATestCase): def _testRngIsNotConstant(self, rng, dtype, output_dtype): # Tests that 'rng' does not always return the same value. - with self.cached_session() as sess: + with self.cached_session(): with self.test_scope(): x = rng(dtype, output_dtype) # The random-number generator, if working correctly, should produce the # same output multiple times with low probability. - y = sess.run(x) - z = sess.run(x) - w = sess.run(x) + y = self.evaluate(x) + z = self.evaluate(x) + w = self.evaluate(x) # We use exact equality here. If the random-number generator is producing # deterministic output, all three outputs will be bitwise identical. @@ -107,12 +108,12 @@ class CategoricalTest(xla_test.XLATestCase): def testCategoricalIsInRange(self): for dtype in self.float_types: for output_dtype in self.output_dtypes(): - with self.cached_session() as sess: + with self.cached_session(): with self.test_scope(): x = random_ops.multinomial( array_ops.ones(shape=[1, 20], dtype=dtype), 1000, output_dtype=output_dtype) - y = sess.run(x) + y = self.evaluate(x) self.assertTrue((y >= 0).sum() == 1000) self.assertTrue((y < 20).sum() == 1000) @@ -138,6 +139,57 @@ class CategoricalTest(xla_test.XLATestCase): chi2 = self._chi2(probs, freqs) self.assertLess(chi2, 1e-3) + def testStatelessMultinomialIsInRange(self): + for dtype in self.float_types: + for output_dtype in self.output_dtypes(): + with self.cached_session() as sess: + with self.test_scope(): + seed_t = array_ops.placeholder(dtypes.int32, shape=[2]) + x = stateless_random_ops.stateless_multinomial( + array_ops.ones(shape=[1, 20], dtype=dtype), + 1000, + seed_t, + output_dtype=output_dtype) + y = sess.run(x, {seed_t: [0x12345678, 0xabcdef12]}) + self.assertTrue((y >= 0).sum() == 1000) + self.assertTrue((y < 20).sum() == 1000) + + def testDeterminismMultinomial(self): + # Stateless values should be equal iff the seeds are equal (roughly) + num_samples = 10 + with self.cached_session(), self.test_scope(): + seed_t = array_ops.placeholder(dtypes.int32, shape=[2]) + seeds = [(x, y) for x in range(5) for y in range(5)] * 3 + for logits in ([[0.1, 0.25, 0.5, 0.15]], [[0.5, 0.5], [0.8, 0.2], + [0.25, 0.75]]): + pure = stateless_random_ops.stateless_multinomial( + logits, num_samples, seed=seed_t) + values = [(seed, pure.eval(feed_dict={seed_t: seed})) for seed in seeds] + for s0, v0 in values: + for s1, v1 in values: + self.assertEqual(s0 == s1, np.all(v0 == v1)) + + def testEmpty(self): + with self.cached_session(): + with self.test_scope(): + x = random_ops.multinomial( + array_ops.zeros([42, 40]), 0, output_dtype=dtypes.int32) + y = self.evaluate(x) + self.assertEqual(y.shape, (42, 0)) + + def testEmptyStateless(self): + with self.cached_session() as sess: + with self.test_scope(): + seed_t = array_ops.placeholder(dtypes.int32, shape=[2]) + x = stateless_random_ops.stateless_multinomial( + array_ops.zeros([42, 40]), + 0, + seed=seed_t, + output_dtype=dtypes.int32) + y = sess.run(x, {seed_t: [0x12345678, 0xabcdef12]}) + self.assertEqual(y.shape, (42, 0)) + + if __name__ == '__main__': googletest.main() diff --git a/tensorflow/compiler/tests/clustering_test.py b/tensorflow/compiler/tests/clustering_test.py index 88bd58b2da6..ef2d7af69de 100644 --- a/tensorflow/compiler/tests/clustering_test.py +++ b/tensorflow/compiler/tests/clustering_test.py @@ -43,7 +43,7 @@ class ClusteringTest(xla_test.XLATestCase): input1 = constant_op.constant(val1, name="const1") input2 = constant_op.constant(val2, name="const2") output = math_ops.add(input1, input2) - result = output.eval() + result = self.evaluate(output) self.assertAllClose(result, expected, rtol=1e-3) def testAddFromCpuMultiple(self): @@ -57,7 +57,7 @@ class ClusteringTest(xla_test.XLATestCase): with self.test_scope(): output = math_ops.add(input1, input2) for _ in xrange(10): - result = output.eval() + result = self.evaluate(output) self.assertAllClose(result, expected, rtol=1e-3) def testDeadlock(self): diff --git a/tensorflow/compiler/tests/concat_ops_test.py b/tensorflow/compiler/tests/concat_ops_test.py index 2d225ad226c..2187f57960f 100644 --- a/tensorflow/compiler/tests/concat_ops_test.py +++ b/tensorflow/compiler/tests/concat_ops_test.py @@ -72,7 +72,7 @@ class ConcatTest(xla_test.XLATestCase): x2 = constant_op.constant(p2) with self.test_scope(): c = array_ops.concat([x1, x2], 0) - result = c.eval() + result = self.evaluate(c) self.assertAllEqual(result[:2, :], p1) self.assertAllEqual(result[2:, :], p2) @@ -150,7 +150,7 @@ class ConcatTest(xla_test.XLATestCase): [float(x) for x in grad_inp.flatten()], shape=output_shape) grad = gradients_impl.gradients([c], inp_tensors, [grad_tensor]) concated_grad = array_ops.concat(grad, 1) - result = concated_grad.eval() + result = self.evaluate(concated_grad) self.assertAllEqual(result, grad_inp) def testGradientsSimpleAll(self): @@ -177,7 +177,7 @@ class ConcatTest(xla_test.XLATestCase): [float(x) for x in grad_inp.flatten()], shape=output_shape) grad = gradients_impl.gradients([c], inp_tensors, [grad_tensor]) concated_grad = array_ops.concat(grad, 0) - result = concated_grad.eval() + result = self.evaluate(concated_grad) self.assertAllEqual(result, grad_inp) @@ -205,7 +205,7 @@ class ConcatTest(xla_test.XLATestCase): [float(x) for x in grad_inp.flatten()], shape=output_shape) grad = gradients_impl.gradients([c], inp_tensors, [grad_tensor]) concated_grad = array_ops.concat(grad, 2) - result = concated_grad.eval() + result = self.evaluate(concated_grad) self.assertAllEqual(result, grad_inp) @@ -242,7 +242,7 @@ class ConcatTest(xla_test.XLATestCase): [float(x) for x in grad_inp.flatten()], shape=output_shape) grad = gradients_impl.gradients([c], inp_tensors, [grad_tensor]) concated_grad = array_ops.concat(grad, concat_dim) - result = concated_grad.eval() + result = self.evaluate(concated_grad) self.assertAllEqual(result, grad_inp) @@ -254,7 +254,7 @@ class ConcatTest(xla_test.XLATestCase): def DISABLED_testZeroSize(self): # Verify that concat doesn't crash and burn for zero size inputs np.random.seed(7) - with self.cached_session() as sess: + with self.cached_session(): with self.test_scope(): for shape0 in (), (2,): axis = len(shape0) @@ -270,7 +270,7 @@ class ConcatTest(xla_test.XLATestCase): self.assertAllEqual(c.eval(), correct) # Check gradients dc = np.random.randn(*c.get_shape().as_list()) - dxs = sess.run(gradients_impl.gradients(c, xs, dc)) + dxs = self.evaluate(gradients_impl.gradients(c, xs, dc)) self.assertAllEqual(dc, np.concatenate(dxs, axis=axis)) def testConcatTuple(self): @@ -280,7 +280,7 @@ class ConcatTest(xla_test.XLATestCase): with self.test_scope(): concat_list_t = array_ops.concat([c1, c2], 0) concat_tuple_t = array_ops.concat((c1, c2), 0) - self.assertAllEqual(concat_list_t.eval(), concat_tuple_t.eval()) + self.assertAllEqual(concat_list_t.eval(), self.evaluate(concat_tuple_t)) def testConcatNoScalars(self): with self.cached_session(): @@ -330,47 +330,47 @@ class ConcatTest(xla_test.XLATestCase): class ConcatOffsetTest(xla_test.XLATestCase): def testBasic(self): - with self.cached_session() as sess: + with self.cached_session(): with self.test_scope(): cdim = constant_op.constant(1, dtypes.int32) s0 = constant_op.constant([2, 3, 5], dtypes.int32) s1 = constant_op.constant([2, 7, 5], dtypes.int32) s2 = constant_op.constant([2, 20, 5], dtypes.int32) off = gen_array_ops.concat_offset(cdim, [s0, s1, s2]) - ans = sess.run(off) + ans = self.evaluate(off) self.assertAllEqual(ans, [[0, 0, 0], [0, 3, 0], [0, 10, 0]]) class PackTest(xla_test.XLATestCase): def testBasic(self): - with self.cached_session() as sess: + with self.cached_session(): with self.test_scope(): s0 = constant_op.constant([2, 3, 5], dtypes.int32) s1 = constant_op.constant([2, 7, 5], dtypes.int32) s2 = constant_op.constant([2, 20, 5], dtypes.int32) packed = array_ops.stack([s0, s1, s2]) - ans = sess.run(packed) + ans = self.evaluate(packed) self.assertAllEqual(ans, [[2, 3, 5], [2, 7, 5], [2, 20, 5]]) def testScalars(self): - with self.cached_session() as sess: + with self.cached_session(): with self.test_scope(): s0 = constant_op.constant(2, dtypes.int32) s1 = constant_op.constant(3, dtypes.int32) s2 = constant_op.constant(5, dtypes.int32) packed = array_ops.stack([s0, s1, s2]) - ans = sess.run(packed) + ans = self.evaluate(packed) self.assertAllEqual(ans, [2, 3, 5]) def testEmpty(self): - with self.cached_session() as sess: + with self.cached_session(): with self.test_scope(): s0 = constant_op.constant([[]], dtypes.int32) s1 = constant_op.constant([[]], dtypes.int32) s2 = constant_op.constant([[]], dtypes.int32) packed = array_ops.stack([s0, s1, s2]) - ans = sess.run(packed) + ans = self.evaluate(packed) self.assertAllEqual(ans, [[[]], [[]], [[]]]) diff --git a/tensorflow/compiler/tests/conv3d_test.py b/tensorflow/compiler/tests/conv3d_test.py index d59fd0236f4..01cc1b63928 100644 --- a/tensorflow/compiler/tests/conv3d_test.py +++ b/tensorflow/compiler/tests/conv3d_test.py @@ -85,7 +85,7 @@ class Conv3DTransposeTest(xla_test.XLATestCase): 1.0, shape=f_shape, name="filter", dtype=dtypes.float32) output = nn_ops.conv3d_transpose( x, f, y_shape, strides=strides, padding="SAME") - value = output.eval() + value = self.evaluate(output) # We count the number of cells being added at the locations in the output. # At the center, #cells = kernel_depth * kernel_height * kernel_width @@ -135,7 +135,7 @@ class Conv3DTransposeTest(xla_test.XLATestCase): 1.0, shape=f_shape, name="filter", dtype=dtypes.float32) output = nn_ops.conv3d_transpose( x, f, y_shape, strides=strides, padding="SAME") - value = output.eval() + value = self.evaluate(output) for n in xrange(x_shape[0]): for k in xrange(f_shape[3]): @@ -173,7 +173,7 @@ class Conv3DTransposeTest(xla_test.XLATestCase): 1.0, shape=f_shape, name="filter", dtype=dtypes.float32) output = nn_ops.conv3d_transpose( x, f, y_shape, strides=strides, padding="VALID") - value = output.eval() + value = self.evaluate(output) cache_values = np.zeros(y_shape, dtype=np.float32) diff --git a/tensorflow/compiler/tests/dense_layer_test.py b/tensorflow/compiler/tests/dense_layer_test.py index d1b90f098d7..bf5ea7b1fb6 100644 --- a/tensorflow/compiler/tests/dense_layer_test.py +++ b/tensorflow/compiler/tests/dense_layer_test.py @@ -42,7 +42,7 @@ def GetRunMetadataLabels(run_metadata): def InLabels(labels, substr): """Returns true iff one of the labels contains substr.""" - return any([substr in x for x in labels]) + return any(substr in x for x in labels) class DenseLayerTest(test.TestCase): @@ -72,7 +72,7 @@ class DenseLayerTest(test.TestCase): x = array_ops.placeholder(shape=[None, None, 3], dtype=np.float32) y = layers.dense(x, 3) - sess.run(variables.initialize_all_variables()) + self.evaluate(variables.initialize_all_variables()) run_metadata = config_pb2.RunMetadata() test_utils.RunWithWarmup( sess, @@ -97,7 +97,7 @@ class DenseLayerTest(test.TestCase): with jit_scope(): y = layers.dense(x, 3) - sess.run(variables.initialize_all_variables()) + self.evaluate(variables.initialize_all_variables()) run_metadata = config_pb2.RunMetadata() test_utils.RunWithWarmup( sess, @@ -126,7 +126,7 @@ class DenseLayerTest(test.TestCase): with jit_scope(): y = layers.dense(x, 3) - sess.run(variables.initialize_all_variables()) + self.evaluate(variables.initialize_all_variables()) run_metadata = config_pb2.RunMetadata() test_utils.RunWithWarmup( sess, diff --git a/tensorflow/compiler/tests/dynamic_stitch_test.py b/tensorflow/compiler/tests/dynamic_stitch_test.py index 50b04daa6b9..e89cf975f5d 100644 --- a/tensorflow/compiler/tests/dynamic_stitch_test.py +++ b/tensorflow/compiler/tests/dynamic_stitch_test.py @@ -58,6 +58,15 @@ class DynamicStitchTest(xla_test.XLATestCase): [idx1, idx2], [val1, val2], expected=np.array([[], [], [], []], np.int32)) + def testEmptyIndex(self): + idx1 = np.array([], dtype=np.int32) + idx2 = np.array([[], []], dtype=np.int32) + val1 = np.ndarray(shape=(0, 9), dtype=np.int32) + val2 = np.ndarray(shape=(2, 0, 9), dtype=np.int32) + self._AssertDynamicStitchResultIs([idx1, idx2], [val1, val2], + expected=np.ndarray( + shape=(0, 9), dtype=np.int32)) + def testSimple1D(self): val1 = np.array([0, 4, 7], dtype=np.int32) val2 = np.array([1, 6, 2, 3, 5], dtype=np.int32) diff --git a/tensorflow/compiler/tests/eager_test.py b/tensorflow/compiler/tests/eager_test.py index 63cee550fde..2af32b537ba 100644 --- a/tensorflow/compiler/tests/eager_test.py +++ b/tensorflow/compiler/tests/eager_test.py @@ -101,12 +101,12 @@ class EagerTest(xla_test.XLATestCase): self.assertAllEqual(15, product) # Run some ops graphly - with context.graph_mode(), self.cached_session() as sess: + with context.graph_mode(), self.cached_session(): with self.test_scope(): three = constant_op.constant(3) five = constant_op.constant(5) product = three * five - self.assertAllEqual(15, sess.run(product)) + self.assertAllEqual(15, self.evaluate(product)) def testDegenerateSlices(self): with self.test_scope(): diff --git a/tensorflow/compiler/tests/fft_test.py b/tensorflow/compiler/tests/fft_test.py index e92afd5d6fe..0edd0c35aa2 100644 --- a/tensorflow/compiler/tests/fft_test.py +++ b/tensorflow/compiler/tests/fft_test.py @@ -27,8 +27,7 @@ from tensorflow.compiler.tests import xla_test from tensorflow.python.framework import dtypes from tensorflow.python.ops import array_ops from tensorflow.python.ops import gradients_impl -from tensorflow.python.ops import signal -from tensorflow.python.ops import spectral_ops +from tensorflow.python.ops.signal import signal from tensorflow.python.platform import googletest BATCH_DIMS = (3, 5) @@ -107,39 +106,39 @@ class FFTTest(xla_test.XLATestCase): def testFFT(self): self._VerifyFftMethod(INNER_DIMS_1D, lambda x: x, np.fft.fft, - spectral_ops.fft) + signal.fft) def testFFT2D(self): self._VerifyFftMethod(INNER_DIMS_2D, lambda x: x, np.fft.fft2, - spectral_ops.fft2d) + signal.fft2d) def testFFT3D(self): self._VerifyFftMethod(INNER_DIMS_3D, lambda x: x, lambda x: np.fft.fftn(x, axes=(-3, -2, -1)), - spectral_ops.fft3d) + signal.fft3d) def testIFFT(self): self._VerifyFftMethod(INNER_DIMS_1D, lambda x: x, np.fft.ifft, - spectral_ops.ifft) + signal.ifft) def testIFFT2D(self): self._VerifyFftMethod(INNER_DIMS_2D, lambda x: x, np.fft.ifft2, - spectral_ops.ifft2d) + signal.ifft2d) def testIFFT3D(self): self._VerifyFftMethod(INNER_DIMS_3D, lambda x: x, lambda x: np.fft.ifftn(x, axes=(-3, -2, -1)), - spectral_ops.ifft3d) + signal.ifft3d) def testRFFT(self): self._VerifyFftMethod( INNER_DIMS_1D, np.real, lambda x: np.fft.rfft(x, n=x.shape[-1]), - lambda x: spectral_ops.rfft(x, fft_length=[x.shape[-1].value])) + lambda x: signal.rfft(x, fft_length=[x.shape[-1].value])) def testRFFT2D(self): def _tf_fn(x): - return spectral_ops.rfft2d( + return signal.rfft2d( x, fft_length=[x.shape[-2].value, x.shape[-1].value]) self._VerifyFftMethod( @@ -153,16 +152,33 @@ class FFTTest(xla_test.XLATestCase): x, axes=(-3, -2, -1), s=[x.shape[-3], x.shape[-2], x.shape[-1]]) def _tf_fn(x): - return spectral_ops.rfft3d( + return signal.rfft3d( x, fft_length=[x.shape[-3].value, x.shape[-2].value, x.shape[-1].value]) self._VerifyFftMethod(INNER_DIMS_3D, np.real, _to_expected, _tf_fn) + def testRFFT3DMismatchedSize(self): + + def _to_expected(x): + return np.fft.rfftn( + x, + axes=(-3, -2, -1), + s=[x.shape[-3] // 2, x.shape[-2], x.shape[-1] * 2]) + + def _tf_fn(x): + return signal.rfft3d( + x, + fft_length=[ + x.shape[-3].value // 2, x.shape[-2].value, x.shape[-1].value * 2 + ]) + + self._VerifyFftMethod(INNER_DIMS_3D, np.real, _to_expected, _tf_fn) + def testIRFFT(self): def _tf_fn(x): - return spectral_ops.irfft(x, fft_length=[2 * (x.shape[-1].value - 1)]) + return signal.irfft(x, fft_length=[2 * (x.shape[-1].value - 1)]) self._VerifyFftMethod( INNER_DIMS_1D, lambda x: np.fft.rfft(np.real(x), n=x.shape[-1]), @@ -171,7 +187,7 @@ class FFTTest(xla_test.XLATestCase): def testIRFFT2D(self): def _tf_fn(x): - return spectral_ops.irfft2d( + return signal.irfft2d( x, fft_length=[x.shape[-2].value, 2 * (x.shape[-1].value - 1)]) self._VerifyFftMethod( @@ -195,7 +211,7 @@ class FFTTest(xla_test.XLATestCase): s=[x.shape[-3], x.shape[-2], 2 * (x.shape[-1] - 1)]) def _tf_fn(x): - return spectral_ops.irfft3d( + return signal.irfft3d( x, fft_length=[ x.shape[-3].value, x.shape[-2].value, 2 * (x.shape[-1].value - 1) @@ -203,6 +219,30 @@ class FFTTest(xla_test.XLATestCase): self._VerifyFftMethod(INNER_DIMS_3D, _to_input, _to_expected, _tf_fn) + def testIRFFT3DMismatchedSize(self): + + def _to_input(x): + return np.fft.rfftn( + np.real(x), + axes=(-3, -2, -1), + s=[x.shape[-3] // 2, x.shape[-2], x.shape[-1] * 2]) + + def _to_expected(x): + return np.fft.irfftn( + x, + axes=(-3, -2, -1), + s=[x.shape[-3] // 2, x.shape[-2], x.shape[-1] * 2]) + + def _tf_fn(x): + return signal.irfft3d( + x, + fft_length=[ + x.shape[-3].value // 2, x.shape[-2].value, x.shape[-1].value * 2 + ]) + + self._VerifyFftMethod(INNER_DIMS_3D, _to_input, _to_expected, _tf_fn) + + if __name__ == "__main__": googletest.main() diff --git a/tensorflow/compiler/tests/fifo_queue_test.py b/tensorflow/compiler/tests/fifo_queue_test.py index 8c7edfd277c..91d77d2f791 100644 --- a/tensorflow/compiler/tests/fifo_queue_test.py +++ b/tensorflow/compiler/tests/fifo_queue_test.py @@ -129,7 +129,7 @@ class FIFOQueueTest(xla_test.XLATestCase): enqueue_op.run() for i in xrange(len(elems)): - vals = dequeued_t.eval() + vals = self.evaluate(dequeued_t) self.assertEqual([elems[i]], vals) def testEnqueueAndBlockingDequeue(self): @@ -192,9 +192,9 @@ class FIFOQueueTest(xla_test.XLATestCase): self.assertEqual([], size.get_shape()) enqueue_op.run() - self.assertEqual(1, size.eval()) + self.assertEqual(1, self.evaluate(size)) dequeued_t.op.run() - self.assertEqual(0, size.eval()) + self.assertEqual(0, self.evaluate(size)) if __name__ == "__main__": diff --git a/tensorflow/compiler/tests/ftrl_test.py b/tensorflow/compiler/tests/ftrl_test.py index 5b197afd655..b078053cdbd 100644 --- a/tensorflow/compiler/tests/ftrl_test.py +++ b/tensorflow/compiler/tests/ftrl_test.py @@ -50,14 +50,14 @@ class FtrlOptimizerTest(xla_test.XLATestCase): ftrl_update = opt.apply_gradients(zip([grads0, grads1], [var0, var1])) variables.global_variables_initializer().run() # Fetch params to validate initial values - self.assertAllClose([0.0, 0.0], var0.eval()) - self.assertAllClose([0.0, 0.0], var1.eval()) + self.assertAllClose([0.0, 0.0], self.evaluate(var0)) + self.assertAllClose([0.0, 0.0], self.evaluate(var1)) # Run Ftrl for a few steps for _ in range(steps): ftrl_update.run() - return var0.eval(), var1.eval() + return self.evaluate(var0), self.evaluate(var1) def equivAdagradTest_AdagradPart(self, steps, dtype): var0, var1, grads0, grads1 = self.initVariableAndGradient(dtype) @@ -65,14 +65,14 @@ class FtrlOptimizerTest(xla_test.XLATestCase): adagrad_update = opt.apply_gradients(zip([grads0, grads1], [var0, var1])) variables.global_variables_initializer().run() # Fetch params to validate initial values - self.assertAllClose([0.0, 0.0], var0.eval()) - self.assertAllClose([0.0, 0.0], var1.eval()) + self.assertAllClose([0.0, 0.0], self.evaluate(var0)) + self.assertAllClose([0.0, 0.0], self.evaluate(var1)) # Run Adagrad for a few steps for _ in range(steps): adagrad_update.run() - return var0.eval(), var1.eval() + return self.evaluate(var0), self.evaluate(var1) def equivGradientDescentTest_FtrlPart(self, steps, dtype): var0, var1, grads0, grads1 = self.initVariableAndGradient(dtype) @@ -85,14 +85,14 @@ class FtrlOptimizerTest(xla_test.XLATestCase): ftrl_update = opt.apply_gradients(zip([grads0, grads1], [var0, var1])) variables.global_variables_initializer().run() # Fetch params to validate initial values - self.assertAllClose([0.0, 0.0], var0.eval()) - self.assertAllClose([0.0, 0.0], var1.eval()) + self.assertAllClose([0.0, 0.0], self.evaluate(var0)) + self.assertAllClose([0.0, 0.0], self.evaluate(var1)) # Run Ftrl for a few steps for _ in range(steps): ftrl_update.run() - return var0.eval(), var1.eval() + return self.evaluate(var0), self.evaluate(var1) def equivGradientDescentTest_GradientDescentPart(self, steps, dtype): var0, var1, grads0, grads1 = self.initVariableAndGradient(dtype) @@ -100,14 +100,14 @@ class FtrlOptimizerTest(xla_test.XLATestCase): sgd_update = opt.apply_gradients(zip([grads0, grads1], [var0, var1])) variables.global_variables_initializer().run() # Fetch params to validate initial values - self.assertAllClose([0.0, 0.0], var0.eval()) - self.assertAllClose([0.0, 0.0], var1.eval()) + self.assertAllClose([0.0, 0.0], self.evaluate(var0)) + self.assertAllClose([0.0, 0.0], self.evaluate(var1)) # Run GradientDescent for a few steps for _ in range(steps): sgd_update.run() - return var0.eval(), var1.eval() + return self.evaluate(var0), self.evaluate(var1) def testFtrlwithoutRegularization(self): for dtype in self.float_types: @@ -124,8 +124,8 @@ class FtrlOptimizerTest(xla_test.XLATestCase): ftrl_update = opt.apply_gradients(zip([grads0, grads1], [var0, var1])) variables.global_variables_initializer().run() # Fetch params to validate initial values - self.assertAllClose([0.0, 0.0], var0.eval()) - self.assertAllClose([0.0, 0.0], var1.eval()) + self.assertAllClose([0.0, 0.0], self.evaluate(var0)) + self.assertAllClose([0.0, 0.0], self.evaluate(var1)) # Run 3 steps FTRL for _ in range(3): @@ -134,12 +134,12 @@ class FtrlOptimizerTest(xla_test.XLATestCase): # Validate updated params self.assertAllCloseAccordingToType( np.array([-2.60260963, -4.29698515]), - var0.eval(), + self.evaluate(var0), float_rtol=1e-4, half_rtol=1e-2) self.assertAllCloseAccordingToType( np.array([-0.28432083, -0.56694895]), - var1.eval(), + self.evaluate(var1), float_rtol=1e-5, half_rtol=1e-2) @@ -158,8 +158,8 @@ class FtrlOptimizerTest(xla_test.XLATestCase): ftrl_update = opt.apply_gradients(zip([grads0, grads1], [var0, var1])) variables.global_variables_initializer().run() # Fetch params to validate initial values - self.assertAllClose([1.0, 2.0], var0.eval()) - self.assertAllClose([4.0, 3.0], var1.eval()) + self.assertAllClose([1.0, 2.0], self.evaluate(var0)) + self.assertAllClose([4.0, 3.0], self.evaluate(var1)) # Run 3 steps FTRL for _ in range(3): @@ -167,10 +167,14 @@ class FtrlOptimizerTest(xla_test.XLATestCase): # Validate updated params self.assertAllCloseAccordingToType( - np.array([-2.55607247, -3.98729396]), var0.eval(), 1e-5, 1e-5, + np.array([-2.55607247, -3.98729396]), + self.evaluate(var0), + 1e-5, + 1e-5, float_rtol=1e-4) self.assertAllCloseAccordingToType( - np.array([-0.28232238, -0.56096673]), var1.eval(), 1e-5, 1e-5) + np.array([-0.28232238, -0.56096673]), self.evaluate(var1), 1e-5, + 1e-5) def testFtrlWithL1(self): for dtype in self.float_types: @@ -187,8 +191,8 @@ class FtrlOptimizerTest(xla_test.XLATestCase): ftrl_update = opt.apply_gradients(zip([grads0, grads1], [var0, var1])) variables.global_variables_initializer().run() # Fetch params to validate initial values - self.assertAllClose([1.0, 2.0], var0.eval()) - self.assertAllClose([4.0, 3.0], var1.eval()) + self.assertAllClose([1.0, 2.0], self.evaluate(var0)) + self.assertAllClose([4.0, 3.0], self.evaluate(var1)) # Run 10 steps FTRL for _ in range(10): @@ -197,12 +201,14 @@ class FtrlOptimizerTest(xla_test.XLATestCase): # Validate updated params self.assertAllCloseAccordingToType( np.array([-7.66718769, -10.91273689]), - var0.eval(), + self.evaluate(var0), rtol=1e-4, bfloat16_rtol=1e-1, bfloat16_atol=1e-1) self.assertAllCloseAccordingToType( - np.array([-0.93460727, -1.86147261]), var1.eval(), rtol=1e-4) + np.array([-0.93460727, -1.86147261]), + self.evaluate(var1), + rtol=1e-4) def testFtrlWithL1_L2(self): for dtype in self.float_types: @@ -219,8 +225,8 @@ class FtrlOptimizerTest(xla_test.XLATestCase): ftrl_update = opt.apply_gradients(zip([grads0, grads1], [var0, var1])) variables.global_variables_initializer().run() # Fetch params to validate initial values - self.assertAllClose([1.0, 2.0], var0.eval()) - self.assertAllClose([4.0, 3.0], var1.eval()) + self.assertAllClose([1.0, 2.0], self.evaluate(var0)) + self.assertAllClose([4.0, 3.0], self.evaluate(var1)) # Run 10 steps FTRL for _ in range(10): @@ -228,9 +234,13 @@ class FtrlOptimizerTest(xla_test.XLATestCase): # Validate updated params self.assertAllCloseAccordingToType( - np.array([-0.24059935, -0.46829352]), var0.eval(), rtol=1e-5) + np.array([-0.24059935, -0.46829352]), + self.evaluate(var0), + rtol=1e-5) self.assertAllCloseAccordingToType( - np.array([-0.02406147, -0.04830509]), var1.eval(), rtol=1e-5) + np.array([-0.02406147, -0.04830509]), + self.evaluate(var1), + rtol=1e-5) def testFtrlWithL1_L2_L2Shrinkage(self): """Test the new FTRL op with support for l2 shrinkage. @@ -254,8 +264,8 @@ class FtrlOptimizerTest(xla_test.XLATestCase): ftrl_update = opt.apply_gradients(zip([grads0, grads1], [var0, var1])) variables.global_variables_initializer().run() # Fetch params to validate initial values - self.assertAllCloseAccordingToType([1.0, 2.0], var0.eval()) - self.assertAllCloseAccordingToType([4.0, 3.0], var1.eval()) + self.assertAllCloseAccordingToType([1.0, 2.0], self.evaluate(var0)) + self.assertAllCloseAccordingToType([4.0, 3.0], self.evaluate(var1)) # Run 10 steps FTRL for _ in range(10): @@ -263,9 +273,13 @@ class FtrlOptimizerTest(xla_test.XLATestCase): # Validate updated params self.assertAllCloseAccordingToType( - np.array([-0.22578996, -0.44345799]), var0.eval(), rtol=1e-4) + np.array([-0.22578996, -0.44345799]), + self.evaluate(var0), + rtol=1e-4) self.assertAllCloseAccordingToType( - np.array([-0.14378493, -0.13229476]), var1.eval(), rtol=1e-4) + np.array([-0.14378493, -0.13229476]), + self.evaluate(var1), + rtol=1e-4) def testFtrlWithL2ShrinkageDoesNotChangeLrSchedule(self): """Verifies that l2 shrinkage in FTRL does not change lr schedule.""" @@ -291,8 +305,8 @@ class FtrlOptimizerTest(xla_test.XLATestCase): update1 = opt1.apply_gradients([(grads1, var1)]) variables.global_variables_initializer().run() - self.assertAllCloseAccordingToType([1.0, 2.0], var0.eval()) - self.assertAllCloseAccordingToType([1.0, 2.0], var1.eval()) + self.assertAllCloseAccordingToType([1.0, 2.0], self.evaluate(var0)) + self.assertAllCloseAccordingToType([1.0, 2.0], self.evaluate(var1)) # Run 10 steps FTRL for _ in range(10): @@ -301,7 +315,7 @@ class FtrlOptimizerTest(xla_test.XLATestCase): # var0 is experiencing L2 shrinkage so it should be smaller than var1 # in magnitude. - self.assertTrue((var0.eval()**2 < var1.eval()**2).all()) + self.assertTrue((var0.eval()**2 < self.evaluate(var1)**2).all()) accum0 = list(opt0._slots["accum"].values())[0].eval() accum1 = list(opt1._slots["accum"].values())[0].eval() # L2 shrinkage should not change how we update grad accumulator. diff --git a/tensorflow/compiler/tests/function_test.py b/tensorflow/compiler/tests/function_test.py index b1891b918c6..a61827c2ae4 100644 --- a/tensorflow/compiler/tests/function_test.py +++ b/tensorflow/compiler/tests/function_test.py @@ -40,7 +40,7 @@ class FunctionTest(xla_test.XLATestCase): bval = np.array([5, 6, 7, 8]).reshape([2, 2]).astype(np.float32) expected = APlus2B(aval, bval) - with self.cached_session() as sess: + with self.cached_session(): @function.Defun(dtypes.float32, dtypes.float32) def Foo(a, b): @@ -50,7 +50,7 @@ class FunctionTest(xla_test.XLATestCase): b = constant_op.constant(bval, name="b") with self.test_scope(): call_f = Foo(a, b) - result = sess.run(call_f) + result = self.evaluate(call_f) self.assertAllClose(result, expected, rtol=1e-3) def testNestedFunctions(self): @@ -66,7 +66,7 @@ class FunctionTest(xla_test.XLATestCase): bval = np.array([4, 3, 2, 1]).reshape([2, 2]).astype(np.float32) expected = APlus2B(aval, bval) - with self.cached_session() as sess: + with self.cached_session(): @function.Defun(dtypes.float32, dtypes.float32) def Foo(a, b): @@ -76,7 +76,7 @@ class FunctionTest(xla_test.XLATestCase): b = constant_op.constant(bval, name="b") with self.test_scope(): call_g = Foo(a, b) - result = sess.run(call_g) + result = self.evaluate(call_g) self.assertAllClose(result, expected, rtol=1e-3) def testFunctionMultipleRetvals(self): @@ -90,7 +90,7 @@ class FunctionTest(xla_test.XLATestCase): bval = np.array([5, 6, 7, 8]).reshape([2, 2]).astype(np.float32) expected = Func(aval, bval) - with self.cached_session() as sess: + with self.cached_session(): @function.Defun(dtypes.float32, dtypes.float32) def Foo(a, b): @@ -100,7 +100,7 @@ class FunctionTest(xla_test.XLATestCase): b = constant_op.constant(bval, name="b") with self.test_scope(): call_f = Foo(a, b) - result = sess.run(call_f) + result = self.evaluate(call_f) self.assertAllClose(result, expected, rtol=1e-3) def testCompileTimeConstantsInDefun(self): diff --git a/tensorflow/compiler/tests/jit_test.py b/tensorflow/compiler/tests/jit_test.py index 6f51ae33a1b..dbea9849e21 100644 --- a/tensorflow/compiler/tests/jit_test.py +++ b/tensorflow/compiler/tests/jit_test.py @@ -75,7 +75,7 @@ def RunMetadataLabels(run_metadata): def InLabels(labels, substr): """Returns true iff one of the labels contains substr.""" - return any([substr in x for x in labels]) + return any(substr in x for x in labels) def MetadataHasXlaRunOp(run_metadata): diff --git a/tensorflow/compiler/tests/listdiff_op_test.py b/tensorflow/compiler/tests/listdiff_op_test.py index 58622114e4f..0210201fa71 100644 --- a/tensorflow/compiler/tests/listdiff_op_test.py +++ b/tensorflow/compiler/tests/listdiff_op_test.py @@ -33,13 +33,13 @@ class ListDiffTest(xla_test.XLATestCase): def _testListDiff(self, x, y, out, idx): for dtype in [dtypes.int32, dtypes.int64]: for index_dtype in [dtypes.int32, dtypes.int64]: - with self.cached_session() as sess: + with self.cached_session(): x_tensor = ops.convert_to_tensor(x, dtype=dtype) y_tensor = ops.convert_to_tensor(y, dtype=dtype) with self.test_scope(): out_tensor, idx_tensor = array_ops.listdiff( x_tensor, y_tensor, out_idx=index_dtype) - tf_out, tf_idx = sess.run([out_tensor, idx_tensor]) + tf_out, tf_idx = self.evaluate([out_tensor, idx_tensor]) self.assertAllEqual(out, tf_out) self.assertAllEqual(idx, tf_idx) self.assertEqual(1, out_tensor.get_shape().ndims) diff --git a/tensorflow/compiler/tests/lrn_ops_test.py b/tensorflow/compiler/tests/lrn_ops_test.py index c6ad67993e8..5dddf6ae4e8 100644 --- a/tensorflow/compiler/tests/lrn_ops_test.py +++ b/tensorflow/compiler/tests/lrn_ops_test.py @@ -120,8 +120,8 @@ class LRNTest(xla_test.XLATestCase): with self.test_scope(): actual = gen_nn_ops.lrn_grad(out_grads, in_image, out_image, depth_radius, bias, alpha, beta) - expected_val = expected.eval() - actual_val = actual.eval() + expected_val = self.evaluate(expected) + actual_val = self.evaluate(actual) self.assertAllClose(actual_val, expected_val, rtol=1e-3) diff --git a/tensorflow/compiler/tests/lstm_test.py b/tensorflow/compiler/tests/lstm_test.py index 265c0b6d141..776ed899e68 100644 --- a/tensorflow/compiler/tests/lstm_test.py +++ b/tensorflow/compiler/tests/lstm_test.py @@ -88,8 +88,8 @@ class LSTMTest(test.TestCase): (basename, m_prev_scalar, c_prev_scalar, pad_scalar)) # Initialize variables and run the unrolled LSTM step. - sess.run(variables.global_variables_initializer()) - return sess.run([m, c]) + self.evaluate(variables.global_variables_initializer()) + return self.evaluate([m, c]) def testLSTMCell(self): # Run with all-0 weights, no padding. @@ -173,8 +173,8 @@ class LSTMTest(test.TestCase): (basename, m_init_scalar, c_init_scalar, pad_scalar)) # Initialize variables and run the unrolled LSTM layer. - sess.run(variables.global_variables_initializer()) - return sess.run(out_seq) + self.evaluate(variables.global_variables_initializer()) + return self.evaluate(out_seq) def testLSTMLayer(self): # Run with all-0 weights, no padding. diff --git a/tensorflow/compiler/tests/momentum_test.py b/tensorflow/compiler/tests/momentum_test.py index f77521a7c49..3416f7dbd6b 100644 --- a/tensorflow/compiler/tests/momentum_test.py +++ b/tensorflow/compiler/tests/momentum_test.py @@ -61,37 +61,43 @@ class MomentumOptimizerTest(xla_test.XLATestCase): self.assertFalse(slot1 in variables.trainable_variables()) # Fetch params to validate initial values - self.assertAllClose([1.0, 2.0], var0.eval()) - self.assertAllClose([3.0, 4.0], var1.eval()) + self.assertAllClose([1.0, 2.0], self.evaluate(var0)) + self.assertAllClose([3.0, 4.0], self.evaluate(var1)) # Step 1: the momentum accumulators where 0. So we should see a normal # update: v -= grad * learning_rate mom_update.run() # Check that the momentum accumulators have been updated. - self.assertAllCloseAccordingToType(np.array([0.1, 0.1]), slot0.eval()) - self.assertAllCloseAccordingToType(np.array([0.01, 0.01]), slot1.eval()) + self.assertAllCloseAccordingToType( + np.array([0.1, 0.1]), self.evaluate(slot0)) + self.assertAllCloseAccordingToType( + np.array([0.01, 0.01]), self.evaluate(slot1)) # Check that the parameters have been updated. self.assertAllCloseAccordingToType( - np.array([1.0 - (0.1 * 2.0), 2.0 - (0.1 * 2.0)]), var0.eval()) + np.array([1.0 - (0.1 * 2.0), 2.0 - (0.1 * 2.0)]), + self.evaluate(var0)) self.assertAllCloseAccordingToType( - np.array([3.0 - (0.01 * 2.0), 4.0 - (0.01 * 2.0)]), var1.eval()) + np.array([3.0 - (0.01 * 2.0), 4.0 - (0.01 * 2.0)]), + self.evaluate(var1)) # Step 2: the momentum accumulators contain the previous update. mom_update.run() # Check that the momentum accumulators have been updated. self.assertAllCloseAccordingToType( - np.array([(0.9 * 0.1 + 0.1), (0.9 * 0.1 + 0.1)]), slot0.eval()) + np.array([(0.9 * 0.1 + 0.1), (0.9 * 0.1 + 0.1)]), + self.evaluate(slot0)) self.assertAllCloseAccordingToType( - np.array([(0.9 * 0.01 + 0.01), (0.9 * 0.01 + 0.01)]), slot1.eval()) + np.array([(0.9 * 0.01 + 0.01), (0.9 * 0.01 + 0.01)]), + self.evaluate(slot1)) # Check that the parameters have been updated. self.assertAllCloseAccordingToType( np.array([ 1.0 - (0.1 * 2.0) - ((0.9 * 0.1 + 0.1) * 2.0), 2.0 - (0.1 * 2.0) - ((0.9 * 0.1 + 0.1) * 2.0) - ]), var0.eval()) + ]), self.evaluate(var0)) self.assertAllCloseAccordingToType( np.array([ - 2.98 - ((0.9 * 0.01 + 0.01) * 2.0), 3.98 - ( - (0.9 * 0.01 + 0.01) * 2.0) - ]), var1.eval()) + 2.98 - ((0.9 * 0.01 + 0.01) * 2.0), + 3.98 - ((0.9 * 0.01 + 0.01) * 2.0) + ]), self.evaluate(var1)) def testNesterovMomentum(self): for dtype in self.float_types: @@ -115,8 +121,8 @@ class MomentumOptimizerTest(xla_test.XLATestCase): var0_np, accum0_np, var0_np * 0.8, 0.1, 0.9) var1_np, accum1_np = self._update_nesterov_momentum_numpy( var1_np, accum1_np, 0.9, 0.1, 0.9) - self.assertAllCloseAccordingToType(var0_np, var0.eval()) - self.assertAllCloseAccordingToType(var1_np, var1.eval()) + self.assertAllCloseAccordingToType(var0_np, self.evaluate(var0)) + self.assertAllCloseAccordingToType(var1_np, self.evaluate(var1)) def testTensorLearningRateAndMomentum(self): for dtype in self.float_types: @@ -141,37 +147,43 @@ class MomentumOptimizerTest(xla_test.XLATestCase): self.assertFalse(slot1 in variables.trainable_variables()) # Fetch params to validate initial values - self.assertAllClose([1.0, 2.0], var0.eval()) - self.assertAllClose([3.0, 4.0], var1.eval()) + self.assertAllClose([1.0, 2.0], self.evaluate(var0)) + self.assertAllClose([3.0, 4.0], self.evaluate(var1)) # Step 1: the momentum accumulators where 0. So we should see a normal # update: v -= grad * learning_rate mom_update.run() # Check that the momentum accumulators have been updated. - self.assertAllCloseAccordingToType(np.array([0.1, 0.1]), slot0.eval()) - self.assertAllCloseAccordingToType(np.array([0.01, 0.01]), slot1.eval()) + self.assertAllCloseAccordingToType( + np.array([0.1, 0.1]), self.evaluate(slot0)) + self.assertAllCloseAccordingToType( + np.array([0.01, 0.01]), self.evaluate(slot1)) # Check that the parameters have been updated. self.assertAllCloseAccordingToType( - np.array([1.0 - (0.1 * 2.0), 2.0 - (0.1 * 2.0)]), var0.eval()) + np.array([1.0 - (0.1 * 2.0), 2.0 - (0.1 * 2.0)]), + self.evaluate(var0)) self.assertAllCloseAccordingToType( - np.array([3.0 - (0.01 * 2.0), 4.0 - (0.01 * 2.0)]), var1.eval()) + np.array([3.0 - (0.01 * 2.0), 4.0 - (0.01 * 2.0)]), + self.evaluate(var1)) # Step 2: the momentum accumulators contain the previous update. mom_update.run() # Check that the momentum accumulators have been updated. self.assertAllCloseAccordingToType( - np.array([(0.9 * 0.1 + 0.1), (0.9 * 0.1 + 0.1)]), slot0.eval()) + np.array([(0.9 * 0.1 + 0.1), (0.9 * 0.1 + 0.1)]), + self.evaluate(slot0)) self.assertAllCloseAccordingToType( - np.array([(0.9 * 0.01 + 0.01), (0.9 * 0.01 + 0.01)]), slot1.eval()) + np.array([(0.9 * 0.01 + 0.01), (0.9 * 0.01 + 0.01)]), + self.evaluate(slot1)) # Check that the parameters have been updated. self.assertAllCloseAccordingToType( np.array([ 1.0 - (0.1 * 2.0) - ((0.9 * 0.1 + 0.1) * 2.0), 2.0 - (0.1 * 2.0) - ((0.9 * 0.1 + 0.1) * 2.0) - ]), var0.eval()) + ]), self.evaluate(var0)) self.assertAllCloseAccordingToType( np.array([ - 2.98 - ((0.9 * 0.01 + 0.01) * 2.0), 3.98 - ( - (0.9 * 0.01 + 0.01) * 2.0) - ]), var1.eval()) + 2.98 - ((0.9 * 0.01 + 0.01) * 2.0), + 3.98 - ((0.9 * 0.01 + 0.01) * 2.0) + ]), self.evaluate(var1)) if __name__ == "__main__": diff --git a/tensorflow/compiler/tests/placeholder_test.py b/tensorflow/compiler/tests/placeholder_test.py index 77bb839409f..9671ae0ae97 100644 --- a/tensorflow/compiler/tests/placeholder_test.py +++ b/tensorflow/compiler/tests/placeholder_test.py @@ -33,7 +33,7 @@ class PlaceholderTest(xla_test.XLATestCase): ph = array_ops.placeholder_with_default(v, shape=[]) out = ph * 2 sess.run(variables.variables_initializer([v])) - self.assertEqual(8.0, sess.run(out)) + self.assertEqual(8.0, self.evaluate(out)) def test_placeholder_with_default_fed(self): with self.cached_session() as sess, self.test_scope(): diff --git a/tensorflow/compiler/tests/powersign_test.py b/tensorflow/compiler/tests/powersign_test.py index 86536da7fed..5b35c200277 100644 --- a/tensorflow/compiler/tests/powersign_test.py +++ b/tensorflow/compiler/tests/powersign_test.py @@ -91,8 +91,8 @@ class PowerSignTest(xla_test.XLATestCase): variables.global_variables_initializer().run() # Fetch params to validate initial values - self.assertAllClose([1.0, 2.0], var0.eval()) - self.assertAllClose([3.0, 4.0], var1.eval()) + self.assertAllClose([1.0, 2.0], self.evaluate(var0)) + self.assertAllClose([3.0, 4.0], self.evaluate(var1)) # Run 7 steps of powersign # first 4 steps with positive gradient @@ -125,8 +125,8 @@ class PowerSignTest(xla_test.XLATestCase): ) # Validate updated params - self.assertAllCloseAccordingToType(var0_np, var0.eval()) - self.assertAllCloseAccordingToType(var1_np, var1.eval()) + self.assertAllCloseAccordingToType(var0_np, self.evaluate(var0)) + self.assertAllCloseAccordingToType(var1_np, self.evaluate(var1)) def testDense(self): decay_steps = 10 diff --git a/tensorflow/compiler/tests/proximal_adagrad_test.py b/tensorflow/compiler/tests/proximal_adagrad_test.py index c41b4171e26..63cc51a4701 100644 --- a/tensorflow/compiler/tests/proximal_adagrad_test.py +++ b/tensorflow/compiler/tests/proximal_adagrad_test.py @@ -45,15 +45,17 @@ class ProximalAdagradOptimizerTest(xla_test.XLATestCase): update = opt.apply_gradients(zip([grads0, grads1], [var0, var1])) variables.global_variables_initializer().run() - self.assertAllClose([0.0, 0.0], var0.eval()) - self.assertAllClose([0.0, 0.0], var1.eval()) + self.assertAllClose([0.0, 0.0], self.evaluate(var0)) + self.assertAllClose([0.0, 0.0], self.evaluate(var1)) # Run 3 steps Proximal Adagrad. for _ in range(3): update.run() - self.assertAllClose(np.array([-2.60260963, -4.29698515]), var0.eval()) - self.assertAllClose(np.array([-0.28432083, -0.56694895]), var1.eval()) + self.assertAllClose( + np.array([-2.60260963, -4.29698515]), self.evaluate(var0)) + self.assertAllClose( + np.array([-0.28432083, -0.56694895]), self.evaluate(var1)) opt_vars = opt.variables() self.assertStartsWith(opt_vars[0].name, var0._shared_name) self.assertStartsWith(opt_vars[1].name, var1._shared_name) @@ -74,14 +76,14 @@ class ProximalAdagradOptimizerTest(xla_test.XLATestCase): update = opt.apply_gradients(zip([grads0, grads1], [var0, var1])) variables.global_variables_initializer().run() - self.assertAllClose([1.0, 2.0], var0.eval()) - self.assertAllClose([4.0, 3.0], var1.eval()) + self.assertAllClose([1.0, 2.0], self.evaluate(var0)) + self.assertAllClose([4.0, 3.0], self.evaluate(var1)) # Run 3 steps Proximal Adagrad. for _ in range(3): update.run() - self.assertAllClose(np.array([-1.60261, -2.296985]), var0.eval()) - self.assertAllClose(np.array([3.715679, 2.433051]), var1.eval()) + self.assertAllClose(np.array([-1.60261, -2.296985]), self.evaluate(var0)) + self.assertAllClose(np.array([3.715679, 2.433051]), self.evaluate(var1)) def testProximalAdagradWithL1(self): with self.cached_session(), self.test_scope(): @@ -98,14 +100,14 @@ class ProximalAdagradOptimizerTest(xla_test.XLATestCase): update = opt.apply_gradients(zip([grads0, grads1], [var0, var1])) variables.global_variables_initializer().run() - self.assertAllClose([1.0, 2.0], var0.eval()) - self.assertAllClose([4.0, 3.0], var1.eval()) + self.assertAllClose([1.0, 2.0], self.evaluate(var0)) + self.assertAllClose([4.0, 3.0], self.evaluate(var1)) # Run 10 steps Proximal Adagrad for _ in range(10): update.run() - self.assertAllClose(np.array([-6.663634, -9.190331]), var0.eval()) - self.assertAllClose(np.array([2.959304, 1.029232]), var1.eval()) + self.assertAllClose(np.array([-6.663634, -9.190331]), self.evaluate(var0)) + self.assertAllClose(np.array([2.959304, 1.029232]), self.evaluate(var1)) def testProximalAdagradWithL1_L2(self): with self.cached_session(), self.test_scope(): @@ -122,15 +124,15 @@ class ProximalAdagradOptimizerTest(xla_test.XLATestCase): update = opt.apply_gradients(zip([grads0, grads1], [var0, var1])) variables.global_variables_initializer().run() - self.assertAllClose([1.0, 2.0], var0.eval()) - self.assertAllClose([4.0, 3.0], var1.eval()) + self.assertAllClose([1.0, 2.0], self.evaluate(var0)) + self.assertAllClose([4.0, 3.0], self.evaluate(var1)) # Run 10 steps Proximal Adagrad. for _ in range(10): update.run() - self.assertAllClose(np.array([-0.0495, -0.0995]), var0.eval()) - self.assertAllClose(np.array([-0.0045, -0.0095]), var1.eval()) + self.assertAllClose(np.array([-0.0495, -0.0995]), self.evaluate(var0)) + self.assertAllClose(np.array([-0.0045, -0.0095]), self.evaluate(var1)) def applyOptimizer(self, opt, steps=5): var0 = resource_variable_ops.ResourceVariable([1.0, 2.0]) @@ -141,14 +143,14 @@ class ProximalAdagradOptimizerTest(xla_test.XLATestCase): update = opt.apply_gradients(zip([grads0, grads1], [var0, var1])) variables.global_variables_initializer().run() - self.assertAllClose([1.0, 2.0], var0.eval()) - self.assertAllClose([3.0, 4.0], var1.eval()) + self.assertAllClose([1.0, 2.0], self.evaluate(var0)) + self.assertAllClose([3.0, 4.0], self.evaluate(var1)) # Run ProximalAdagrad for a few steps for _ in range(steps): update.run() - return var0.eval(), var1.eval() + return self.evaluate(var0), self.evaluate(var1) def testEquivAdagradwithoutRegularization(self): with self.cached_session(), self.test_scope(): diff --git a/tensorflow/compiler/tests/proximal_gradient_descent_test.py b/tensorflow/compiler/tests/proximal_gradient_descent_test.py index 3d808e6b8a7..5aec433be76 100644 --- a/tensorflow/compiler/tests/proximal_gradient_descent_test.py +++ b/tensorflow/compiler/tests/proximal_gradient_descent_test.py @@ -42,15 +42,15 @@ class ProximalGradientDescentOptimizerTest(xla_test.XLATestCase): update = opt.apply_gradients(zip([grads0, grads1], [var0, var1])) variables.global_variables_initializer().run() - self.assertAllClose([0.0, 0.0], var0.eval()) - self.assertAllClose([0.0, 0.0], var1.eval()) + self.assertAllClose([0.0, 0.0], self.evaluate(var0)) + self.assertAllClose([0.0, 0.0], self.evaluate(var1)) # Run 3 steps Proximal Gradient Descent. for _ in range(3): update.run() - self.assertAllClose(np.array([-0.9, -1.8]), var0.eval()) - self.assertAllClose(np.array([-0.09, -0.18]), var1.eval()) + self.assertAllClose(np.array([-0.9, -1.8]), self.evaluate(var0)) + self.assertAllClose(np.array([-0.09, -0.18]), self.evaluate(var1)) def testProximalGradientDescentwithoutRegularization2(self): with self.cached_session(), self.test_scope(): @@ -64,15 +64,15 @@ class ProximalGradientDescentOptimizerTest(xla_test.XLATestCase): update = opt.apply_gradients(zip([grads0, grads1], [var0, var1])) variables.global_variables_initializer().run() - self.assertAllClose([1.0, 2.0], var0.eval()) - self.assertAllClose([4.0, 3.0], var1.eval()) + self.assertAllClose([1.0, 2.0], self.evaluate(var0)) + self.assertAllClose([4.0, 3.0], self.evaluate(var1)) # Run 3 steps Proximal Gradient Descent for _ in range(3): update.run() - self.assertAllClose(np.array([0.1, 0.2]), var0.eval()) - self.assertAllClose(np.array([3.91, 2.82]), var1.eval()) + self.assertAllClose(np.array([0.1, 0.2]), self.evaluate(var0)) + self.assertAllClose(np.array([3.91, 2.82]), self.evaluate(var1)) def testProximalGradientDescentWithL1(self): with self.cached_session(), self.test_scope(): @@ -86,15 +86,15 @@ class ProximalGradientDescentOptimizerTest(xla_test.XLATestCase): update = opt.apply_gradients(zip([grads0, grads1], [var0, var1])) variables.global_variables_initializer().run() - self.assertAllClose([1.0, 2.0], var0.eval()) - self.assertAllClose([4.0, 3.0], var1.eval()) + self.assertAllClose([1.0, 2.0], self.evaluate(var0)) + self.assertAllClose([4.0, 3.0], self.evaluate(var1)) # Run 10 steps proximal gradient descent. for _ in range(10): update.run() - self.assertAllClose(np.array([-1.988, -3.988001]), var0.eval()) - self.assertAllClose(np.array([3.67, 2.37]), var1.eval()) + self.assertAllClose(np.array([-1.988, -3.988001]), self.evaluate(var0)) + self.assertAllClose(np.array([3.67, 2.37]), self.evaluate(var1)) def testProximalGradientDescentWithL1_L2(self): with self.cached_session(), self.test_scope(): @@ -108,15 +108,15 @@ class ProximalGradientDescentOptimizerTest(xla_test.XLATestCase): update = opt.apply_gradients(zip([grads0, grads1], [var0, var1])) variables.global_variables_initializer().run() - self.assertAllClose([1.0, 2.0], var0.eval()) - self.assertAllClose([4.0, 3.0], var1.eval()) + self.assertAllClose([1.0, 2.0], self.evaluate(var0)) + self.assertAllClose([4.0, 3.0], self.evaluate(var1)) # Run 10 steps Proximal Gradient Descent for _ in range(10): update.run() - self.assertAllClose(np.array([-0.0495, -0.0995]), var0.eval()) - self.assertAllClose(np.array([-0.0045, -0.0095]), var1.eval()) + self.assertAllClose(np.array([-0.0495, -0.0995]), self.evaluate(var0)) + self.assertAllClose(np.array([-0.0045, -0.0095]), self.evaluate(var1)) def applyOptimizer(self, opt, steps=5): var0 = resource_variable_ops.ResourceVariable([1.0, 2.0]) @@ -127,14 +127,14 @@ class ProximalGradientDescentOptimizerTest(xla_test.XLATestCase): update = opt.apply_gradients(zip([grads0, grads1], [var0, var1])) variables.global_variables_initializer().run() - self.assertAllClose([1.0, 2.0], var0.eval()) - self.assertAllClose([3.0, 4.0], var1.eval()) + self.assertAllClose([1.0, 2.0], self.evaluate(var0)) + self.assertAllClose([3.0, 4.0], self.evaluate(var1)) # Run ProximalAdagrad for a few steps for _ in range(steps): update.run() - return var0.eval(), var1.eval() + return self.evaluate(var0), self.evaluate(var1) def testEquivGradientDescentwithoutRegularization(self): with self.cached_session(), self.test_scope(): diff --git a/tensorflow/compiler/tests/qr_op_test.py b/tensorflow/compiler/tests/qr_op_test.py index 236b1b881dc..b4d4193e35f 100644 --- a/tensorflow/compiler/tests/qr_op_test.py +++ b/tensorflow/compiler/tests/qr_op_test.py @@ -63,7 +63,7 @@ class QrOpTest(xla_test.XLATestCase, parameterized.TestCase): # Tests that x[...,:,:]^H * x[...,:,:] is close to the identity. xx = math_ops.matmul(x, x, adjoint_a=True) identity = array_ops.matrix_band_part(array_ops.ones_like(xx), 0, 0) - precision = self.AdjustedNorm(xx.eval() - identity.eval()) + precision = self.AdjustedNorm(xx.eval() - self.evaluate(identity)) self.assertTrue(np.all(precision < 5.0)) def _test(self, dtype, shape, full_matrices): diff --git a/tensorflow/compiler/tests/random_ops_test.py b/tensorflow/compiler/tests/random_ops_test.py index 36ef6ed5fee..97ffad34c00 100644 --- a/tensorflow/compiler/tests/random_ops_test.py +++ b/tensorflow/compiler/tests/random_ops_test.py @@ -46,9 +46,9 @@ class RandomOpsTest(xla_test.XLATestCase): # The random-number generator, if working correctly, should produce the # same output multiple times with low probability. - y = sess.run(x) - z = sess.run(x) - w = sess.run(x) + y = self.evaluate(x) + z = self.evaluate(x) + w = self.evaluate(x) # We use exact equality here. If the random-number generator is producing # deterministic output, all three outputs will be bitwise identical. @@ -83,7 +83,7 @@ class RandomOpsTest(xla_test.XLATestCase): with self.test_scope(): x = random_ops.random_uniform( shape=[1000], dtype=dtype, minval=-2, maxval=33) - y = sess.run(x) + y = self.evaluate(x) self.assertTrue((y >= -2).sum() == 1000) self.assertTrue((y < 33).sum() == 1000) @@ -102,7 +102,7 @@ class RandomOpsTest(xla_test.XLATestCase): with self.cached_session() as sess: with self.test_scope(): x = random_ops.truncated_normal(shape=[count], dtype=dtype) - y = sess.run(x) + y = self.evaluate(x) def normal_cdf(x): return .5 * math.erfc(-x / math.sqrt(2)) @@ -111,7 +111,7 @@ class RandomOpsTest(xla_test.XLATestCase): return math.exp(-(x**2) / 2.) / math.sqrt(2 * math.pi) def probit(x, sess=sess): - return sess.run(special_math.ndtri(x)) + return self.evaluate(special_math.ndtri(x)) a = -2. b = 2. @@ -148,7 +148,7 @@ class RandomOpsTest(xla_test.XLATestCase): with self.test_scope(): x = math_ops.range(1 << 16) shuffle = random_ops.random_shuffle(x) - result = sess.run(shuffle) + result = self.evaluate(shuffle) expected = range(1 << 16) # Compare sets to avoid randomness behavior changes but make sure still # have all the values. @@ -159,7 +159,7 @@ class RandomOpsTest(xla_test.XLATestCase): with self.test_scope(): x = array_ops.diag(math_ops.range(20)) shuffle = random_ops.random_shuffle(x) - result = sess.run(shuffle) + result = self.evaluate(shuffle) expected = np.diag(range(20)).flatten() # Compare sets to avoid randomness behavior changes but make sure still # have all the values. diff --git a/tensorflow/compiler/tests/randomized_tests.cc b/tensorflow/compiler/tests/randomized_tests.cc index a6b58020126..d23fd125163 100644 --- a/tensorflow/compiler/tests/randomized_tests.cc +++ b/tensorflow/compiler/tests/randomized_tests.cc @@ -3382,10 +3382,10 @@ int main(int argc, char** argv) { } // XLA devices register kernels at construction time; create all known devices // to make sure the kernels are registered. - std::vector devices; + std::vector> devices; TF_CHECK_OK(tensorflow::DeviceFactory::AddDevices( tensorflow::SessionOptions(), "", &devices)); - tensorflow::DeviceMgr device_mgr(devices); + tensorflow::DeviceMgr device_mgr(std::move(devices)); tensorflow::Device* ignored; TF_QCHECK_OK( diff --git a/tensorflow/compiler/tests/reduce_ops_test.py b/tensorflow/compiler/tests/reduce_ops_test.py index 132c59c32c9..e8fc81bbb54 100644 --- a/tensorflow/compiler/tests/reduce_ops_test.py +++ b/tensorflow/compiler/tests/reduce_ops_test.py @@ -91,6 +91,7 @@ class ReduceOpsTest(xla_test.XLATestCase, parameterized.TestCase): np.array([], dtype=np.bool).reshape(0, 3), np.array([[False, True, False], [True, True, False]]), ] + ONES = [np.ones([34000, 2])] def testReduceSumF32(self, index_dtype): self._testReduction(math_ops.reduce_sum, np.sum, np.float32, self.REAL_DATA, @@ -149,6 +150,11 @@ class ReduceOpsTest(xla_test.XLATestCase, parameterized.TestCase): self._testReduction(math_ops.reduce_mean, np.mean, np.float32, self.NONEMPTY_REAL_DATA, index_dtype) + def testReduceMeanF16(self, index_dtype): + if np.float16 in self.all_types: + self._testReduction(math_ops.reduce_mean, np.mean, np.float16, self.ONES, + index_dtype) + def testReduceMeanC64(self, index_dtype): self._testReduction(math_ops.reduce_mean, np.mean, np.complex64, self.NONEMPTY_COMPLEX_DATA, index_dtype) diff --git a/tensorflow/compiler/tests/rmsprop_test.py b/tensorflow/compiler/tests/rmsprop_test.py index 8840a1329a9..dc3e90b4afa 100644 --- a/tensorflow/compiler/tests/rmsprop_test.py +++ b/tensorflow/compiler/tests/rmsprop_test.py @@ -76,7 +76,7 @@ class RmspropTest(xla_test.XLATestCase): rms_opt = rmsprop.RMSPropOptimizer(learning_rate, centered=centered) rms_update = rms_opt.apply_gradients( zip([grads0, grads1], [var0, var1])) - variables.global_variables_initializer().run() + self.evaluate(variables.global_variables_initializer()) mg0 = rms_opt.get_slot(var0, "mg") self.assertEqual(mg0 is not None, centered) @@ -92,12 +92,12 @@ class RmspropTest(xla_test.XLATestCase): self.assertTrue(mom1 is not None) # Fetch params to validate initial values - self.assertAllClose([1.0, 2.0], var0.eval()) - self.assertAllClose([3.0, 4.0], var1.eval()) + self.assertAllClose([1.0, 2.0], self.evaluate(var0)) + self.assertAllClose([3.0, 4.0], self.evaluate(var1)) # Run 3 steps of RMSProp for _ in range(3): - rms_update.run() + self.evaluate(rms_update) var0_np, mg0_np, rms0_np, mom0_np = self._rmsprop_update_numpy( var0_np, @@ -118,14 +118,14 @@ class RmspropTest(xla_test.XLATestCase): # Validate updated params if centered: - self.assertAllCloseAccordingToType(mg0_np, mg0.eval()) - self.assertAllCloseAccordingToType(mg1_np, mg1.eval()) - self.assertAllCloseAccordingToType(rms0_np, rms0.eval()) - self.assertAllCloseAccordingToType(rms1_np, rms1.eval()) - self.assertAllCloseAccordingToType(mom0_np, mom0.eval()) - self.assertAllCloseAccordingToType(mom1_np, mom1.eval()) - self.assertAllCloseAccordingToType(var0_np, var0.eval()) - self.assertAllCloseAccordingToType(var1_np, var1.eval()) + self.assertAllCloseAccordingToType(mg0_np, self.evaluate(mg0)) + self.assertAllCloseAccordingToType(mg1_np, self.evaluate(mg1)) + self.assertAllCloseAccordingToType(rms0_np, self.evaluate(rms0)) + self.assertAllCloseAccordingToType(rms1_np, self.evaluate(rms1)) + self.assertAllCloseAccordingToType(mom0_np, self.evaluate(mom0)) + self.assertAllCloseAccordingToType(mom1_np, self.evaluate(mom1)) + self.assertAllCloseAccordingToType(var0_np, self.evaluate(var0)) + self.assertAllCloseAccordingToType(var1_np, self.evaluate(var1)) if __name__ == "__main__": diff --git a/tensorflow/compiler/tests/scan_ops_test.py b/tensorflow/compiler/tests/scan_ops_test.py index 897db384b7e..17639bd8a75 100644 --- a/tensorflow/compiler/tests/scan_ops_test.py +++ b/tensorflow/compiler/tests/scan_ops_test.py @@ -71,7 +71,7 @@ def handle_options(func, x, axis, exclusive, reverse): class CumsumTest(xla_test.XLATestCase): - valid_dtypes = [np.float32] + valid_dtypes = [np.float32, np.int32] def axis_dtypes(self): return set(self.int_types).intersection([np.int32, np.int64]) @@ -149,7 +149,7 @@ class CumsumTest(xla_test.XLATestCase): class CumprodTest(xla_test.XLATestCase): - valid_dtypes = [np.float32] + valid_dtypes = [np.float32, np.int32] def axis_dtypes(self): return set(self.int_types).intersection([np.int32, np.int64]) diff --git a/tensorflow/compiler/tests/stateless_random_ops_test.py b/tensorflow/compiler/tests/stateless_random_ops_test.py index 21708aa1587..ee7ca7e6f19 100644 --- a/tensorflow/compiler/tests/stateless_random_ops_test.py +++ b/tensorflow/compiler/tests/stateless_random_ops_test.py @@ -156,7 +156,7 @@ class StatelessRandomOpsTest(xla_test.XLATestCase): return math.exp(-(x**2) / 2.) / math.sqrt(2 * math.pi) def probit(x, sess=sess): - return sess.run(special_math.ndtri(x)) + return self.evaluate(special_math.ndtri(x)) a = -2. b = 2. diff --git a/tensorflow/compiler/tests/tensor_array_ops_test.py b/tensorflow/compiler/tests/tensor_array_ops_test.py index 46ca371c8ab..d7e26d79c4c 100644 --- a/tensorflow/compiler/tests/tensor_array_ops_test.py +++ b/tensorflow/compiler/tests/tensor_array_ops_test.py @@ -79,7 +79,8 @@ class TensorArrayTest(xla_test.XLATestCase): c0 = w2.stack() self.assertAllEqual( - convert([[[4.0, 5.0]], [[6.0, 7.0]], [[8.0, 9.0]]]), c0.eval()) + convert([[[4.0, 5.0]], [[6.0, 7.0]], [[8.0, 9.0]]]), + self.evaluate(c0)) def testTensorArrayWritePack(self): for dtype in self.numeric_tf_types: @@ -97,7 +98,7 @@ class TensorArrayTest(xla_test.XLATestCase): c0 = w2.stack() - self.assertAllEqual([3, 0, 1], c0.eval().shape) + self.assertAllEqual([3, 0, 1], self.evaluate(c0).shape) def _testTensorArrayWriteConcat(self, tf_dtype): with self.cached_session(), self.test_scope(): @@ -113,8 +114,8 @@ class TensorArrayTest(xla_test.XLATestCase): c0 = w2.concat() self.assertAllEqual( - convert([[4.0, 5.0], [104.0, 105.0], [6.0, 7.0], - [106.0, 107.0], [8.0, 9.0], [204.0, 205.0]]), c0.eval()) + convert([[4.0, 5.0], [104.0, 105.0], [6.0, 7.0], [106.0, 107.0], + [8.0, 9.0], [204.0, 205.0]]), self.evaluate(c0)) def testTensorArrayWriteConcat(self): for dtype in self.numeric_tf_types: @@ -341,7 +342,7 @@ class TensorArrayTest(xla_test.XLATestCase): r0_bad = gen_data_flow_ops.tensor_array_read_v3( handle=w0.handle, index=0, dtype=dtype2, flow_in=w0.flow) with self.assertRaisesOpError("TensorArray dtype is "): - r0_bad.eval() + self.evaluate(r0_bad) # Test reading from a different index than the one we wrote to w0.read(1) @@ -422,7 +423,7 @@ class TensorArrayTest(xla_test.XLATestCase): w2 = h2.write(0, 5.0) r2 = w2.read(0) r = r1 + r2 - self.assertAllClose(9.0, r.eval()) + self.assertAllClose(9.0, self.evaluate(r)) def _testTensorArrayGradientWriteReadType(self, dtype): with self.cached_session() as session, self.test_scope(): @@ -504,7 +505,7 @@ class TensorArrayTest(xla_test.XLATestCase): [-0.5, 1.5], # read(0) gradient [20.0, 30.0, 40.0, 50.0], # concat gradient ]) - grad_vals = sess.run(grad_r) # 2 + 2 entries + grad_vals = self.evaluate(grad_r) # 2 + 2 entries self.assertAllClose([2.0 - 0.5 + 20.0, 3.0 + 1.5 + 30.0], grad_vals[0]) self.assertAllEqual([4.0 + 40.0, 5.0 + 50.0], grad_vals[1]) @@ -526,7 +527,7 @@ class TensorArrayTest(xla_test.XLATestCase): with ops.control_dependencies([r0_readtwice]): r1_readtwice = w_readtwice.read(0) - self.assertAllEqual([1.0, -1.0], r1_readtwice.eval()) + self.assertAllEqual([1.0, -1.0], self.evaluate(r1_readtwice)) def _testTensorArrayGradientUnpackRead(self): with self.cached_session() as session, self.test_scope(): @@ -592,7 +593,7 @@ class TensorArrayTest(xla_test.XLATestCase): ta = tensor_array_ops.TensorArray( dtype=dtypes.float32, tensor_array_name="foo", size=3) s = ta.size() - self.assertAllEqual(3, s.eval()) + self.assertAllEqual(3, self.evaluate(s)) def testWriteCloseTensorArray(self): with self.cached_session(), self.test_scope(): @@ -722,7 +723,7 @@ class TensorArrayTest(xla_test.XLATestCase): # r = acc2.stack() # grad = gradients_impl.gradients(r, [x])[0] - # self.assertAllClose(31.0, grad.eval()) + # self.assertAllClose(31.0, self.evaluate(grad)) def testSumOfTwoReadVariablesWithoutRepeatGrad(self): with self.cached_session() as session, self.test_scope(): @@ -912,7 +913,7 @@ class TensorArrayTest(xla_test.XLATestCase): self.assertEqual(0, ta.size().eval()) ta = ta.unstack(array_ops.zeros([0, 3, 5])) packed = ta.stack() - self.assertAllEqual([0, 3, 5], packed.eval().shape) + self.assertAllEqual([0, 3, 5], self.evaluate(packed).shape) # Concatenating zero tensors along their first dimension gives a # first dimension of zero self.assertAllEqual([0, 5], ta.concat().eval().shape) @@ -1041,8 +1042,8 @@ class TensorArrayTest(xla_test.XLATestCase): (read0, read1, size0, size1)) # Tests that the control dependencies was added and executed. - self.assertEqual(1, v0.eval()) - self.assertEqual(1, v1.eval()) + self.assertEqual(1, self.evaluate(v0)) + self.assertEqual(1, self.evaluate(v1)) # Tests correct TensorArray. self.assertEqual(read0_v, 0) diff --git a/tensorflow/compiler/tests/unary_ops_test.py b/tensorflow/compiler/tests/unary_ops_test.py index d612d3b32dd..95c9e7ffd46 100644 --- a/tensorflow/compiler/tests/unary_ops_test.py +++ b/tensorflow/compiler/tests/unary_ops_test.py @@ -481,6 +481,72 @@ class UnaryOpsTest(xla_test.XLATestCase): np.array([-1, -0.5, 0, 0.3], dtype=dtype), expected=np.array([-1., -0.5, 0., 0.296875], dtype=dtype)) + def quantize_and_dequantize_v2_round_half_up(x): + return array_ops.quantize_and_dequantize_v2( + x, + -1, + 1.0, + signed_input=True, + num_bits=8, + range_given=True, + round_mode="HALF_UP") + + self._assertOpOutputMatchesExpected( + quantize_and_dequantize_v2_round_half_up, + np.array([-0.8, -0.5, 0, 0.3, 0.8, -2, 33], dtype=dtype), + expected=np.array([ + -102.0 / 127, + -63.0 / 127, + 0, + 38.0 / 127, + 102.0 / 127, + -128.0 / 127, + 1, + ], + dtype=dtype)) + + def quantize_and_dequantize_v2_round_half_to_even(x): + return array_ops.quantize_and_dequantize_v2( + x, + -1.0, + 1.0, + signed_input=True, + num_bits=8, + range_given=True, + round_mode="HALF_TO_EVEN") + + self._assertOpOutputMatchesExpected( + quantize_and_dequantize_v2_round_half_to_even, + np.array( + [ + -0.8, + # The -0.5 should become -63.5 after scaling and with + # rounding this should become -64. But with the test + # unary_ops_test_cpu_ondemand, this fails as the result + # before scaling becomes -63.499996 and gets rounded to -63. + # TODO(sreenik): Some one more familiar with this test needs + # to take a look and resolve this. This works on all other + # variations of the platform like cpu, and gpu. + # -0.5, + 0, + 0.3, + 0.8, + -2, + 33 + ], + dtype=dtype), + expected=np.array( + [ + -102.0 / 127, + # -64.0 / 127, + 0, + 38.0 / 127, + 102.0 / 127, + -128.0 / 127, + 1, + ], + dtype=dtype)) + def quantize_and_dequantize_v3(x): return array_ops.quantize_and_dequantize_v3( x, -127, 127, num_bits=8, signed_input=True, range_given=False) diff --git a/tensorflow/compiler/tests/variable_ops_test.py b/tensorflow/compiler/tests/variable_ops_test.py index 77cdeac8168..fcd7ac5ba1c 100644 --- a/tensorflow/compiler/tests/variable_ops_test.py +++ b/tensorflow/compiler/tests/variable_ops_test.py @@ -77,7 +77,7 @@ class VariableOpsTest(xla_test.XLATestCase): sess.run(variables.variables_initializer([v])) x = v.sparse_read(2) self.assertAllClose( - np.array([8j, 9, 10, 11]).astype(dtype), sess.run(x)) + np.array([8j, 9, 10, 11]).astype(dtype), self.evaluate(x)) def testSparseRead1DIndices(self): for dtype in self.numeric_types: @@ -89,7 +89,7 @@ class VariableOpsTest(xla_test.XLATestCase): x = v.sparse_read([2, 1]) self.assertAllClose( np.array([[8, 9, 10, 11], [4, 5, 6j, 7]]).astype(dtype), - sess.run(x)) + self.evaluate(x)) def testSparseRead2DIndices(self): for dtype in self.numeric_types: @@ -102,7 +102,7 @@ class VariableOpsTest(xla_test.XLATestCase): self.assertAllClose( np.array([[[8, 9, 10, 11], [4, 5, 6, 7]], [[0, 1, 2j, 3], [8, 9, 10, 11]]]).astype(dtype), - sess.run(x)) + self.evaluate(x)) def testSparseRead2DIndices3DTensor(self): for dtype in self.numeric_types: @@ -115,9 +115,9 @@ class VariableOpsTest(xla_test.XLATestCase): x = v.sparse_read([[2, 1], [3, 0]]) self.assertAllClose( np.array( - [[[[20, 21, 22], [23, 24j, 25]], [[10, 11, 12], [13, 14, 15]] - ], [[[30, 31, 32], [33, 34, 35]], [[0, 1, 2], [3, 4, 5]]] - ],).astype(dtype), sess.run(x)) + [[[[20, 21, 22], [23, 24j, 25]], [[10, 11, 12], [13, 14, 15]]], + [[[30, 31, 32], [33, 34, 35]], [[0, 1, 2], [3, 4, 5]]] + ],).astype(dtype), self.evaluate(x)) def testShape(self): for dtype in self.numeric_types: @@ -229,7 +229,7 @@ class VariableOpsTest(xla_test.XLATestCase): resource_variable_ops.resource_scatter_add( handle, [0], constant_op.constant([[2]], dtype=dtypes.int32))) read = resource_variable_ops.read_variable_op(handle, dtype=dtypes.int32) - self.assertAllEqual(sess.run(read), [[3], [7]]) + self.assertAllEqual(self.evaluate(read), [[3], [7]]) def testScatterSub(self): with self.test_session() as sess, self.test_scope(): @@ -242,7 +242,7 @@ class VariableOpsTest(xla_test.XLATestCase): resource_variable_ops.resource_scatter_sub( handle, [1], constant_op.constant([[2]], dtype=dtypes.int32))) read = resource_variable_ops.read_variable_op(handle, dtype=dtypes.int32) - self.assertAllEqual(sess.run(read), [[4], [-1]]) + self.assertAllEqual(self.evaluate(read), [[4], [-1]]) def testScatterMul(self): with self.test_session() as sess, self.test_scope(): @@ -255,7 +255,7 @@ class VariableOpsTest(xla_test.XLATestCase): resource_variable_ops.resource_scatter_mul( handle, [0], constant_op.constant([[5]], dtype=dtypes.int32))) read = resource_variable_ops.read_variable_op(handle, dtype=dtypes.int32) - self.assertEqual(sess.run(read), [[5]]) + self.assertEqual(self.evaluate(read), [[5]]) def testScatterDiv(self): with self.test_session() as sess, self.test_scope(): @@ -268,7 +268,7 @@ class VariableOpsTest(xla_test.XLATestCase): resource_variable_ops.resource_scatter_div( handle, [0], constant_op.constant([[3]], dtype=dtypes.int32))) read = resource_variable_ops.read_variable_op(handle, dtype=dtypes.int32) - self.assertAllEqual(sess.run(read), [[2]]) + self.assertAllEqual(self.evaluate(read), [[2]]) def testScatterMin(self): with self.test_session() as sess, self.test_scope(): @@ -281,7 +281,7 @@ class VariableOpsTest(xla_test.XLATestCase): resource_variable_ops.resource_scatter_min( handle, [0], constant_op.constant([[3]], dtype=dtypes.int32))) read = resource_variable_ops.read_variable_op(handle, dtype=dtypes.int32) - self.assertEqual(sess.run(read), [[3]]) + self.assertEqual(self.evaluate(read), [[3]]) def testScatterMax(self): with self.test_session() as sess, self.test_scope(): @@ -294,7 +294,7 @@ class VariableOpsTest(xla_test.XLATestCase): resource_variable_ops.resource_scatter_max( handle, [0], constant_op.constant([[3]], dtype=dtypes.int32))) read = resource_variable_ops.read_variable_op(handle, dtype=dtypes.int32) - self.assertEqual(sess.run(read), [[6]]) + self.assertEqual(self.evaluate(read), [[6]]) def testScatterUpdate(self): with self.test_session() as sess, self.test_scope(): @@ -307,7 +307,7 @@ class VariableOpsTest(xla_test.XLATestCase): resource_variable_ops.resource_scatter_update( handle, [0], constant_op.constant([[3]], dtype=dtypes.int32))) read = resource_variable_ops.read_variable_op(handle, dtype=dtypes.int32) - self.assertEqual(sess.run(read), [[3]]) + self.assertEqual(self.evaluate(read), [[3]]) def testScatterAddScalar(self): with self.test_session() as sess, self.test_scope(): @@ -320,7 +320,7 @@ class VariableOpsTest(xla_test.XLATestCase): resource_variable_ops.resource_scatter_add( handle, [0], constant_op.constant(2, dtype=dtypes.int32))) read = resource_variable_ops.read_variable_op(handle, dtype=dtypes.int32) - self.assertEqual(sess.run(read), [[3]]) + self.assertEqual(self.evaluate(read), [[3]]) def testScatterSubScalar(self): with self.test_session() as sess, self.test_scope(): @@ -333,7 +333,7 @@ class VariableOpsTest(xla_test.XLATestCase): resource_variable_ops.resource_scatter_sub( handle, [0], constant_op.constant(2, dtype=dtypes.int32))) read = resource_variable_ops.read_variable_op(handle, dtype=dtypes.int32) - self.assertEqual(sess.run(read), [[-1]]) + self.assertEqual(self.evaluate(read), [[-1]]) def testScatterMulScalar(self): with self.test_session() as sess, self.test_scope(): @@ -346,7 +346,7 @@ class VariableOpsTest(xla_test.XLATestCase): resource_variable_ops.resource_scatter_mul( handle, [0], constant_op.constant(5, dtype=dtypes.int32))) read = resource_variable_ops.read_variable_op(handle, dtype=dtypes.int32) - self.assertEqual(sess.run(read), [[5]]) + self.assertEqual(self.evaluate(read), [[5]]) def testScatterDivScalar(self): with self.test_session() as sess, self.test_scope(): @@ -359,7 +359,7 @@ class VariableOpsTest(xla_test.XLATestCase): resource_variable_ops.resource_scatter_div( handle, [0], constant_op.constant(3, dtype=dtypes.int32))) read = resource_variable_ops.read_variable_op(handle, dtype=dtypes.int32) - self.assertEqual(sess.run(read), [[2]]) + self.assertEqual(self.evaluate(read), [[2]]) def testScatterMinScalar(self): with self.test_session() as sess, self.test_scope(): @@ -372,7 +372,7 @@ class VariableOpsTest(xla_test.XLATestCase): resource_variable_ops.resource_scatter_min( handle, [0], constant_op.constant(3, dtype=dtypes.int32))) read = resource_variable_ops.read_variable_op(handle, dtype=dtypes.int32) - self.assertEqual(sess.run(read), [[3]]) + self.assertEqual(self.evaluate(read), [[3]]) def testScatterMaxScalar(self): with self.test_session() as sess, self.test_scope(): @@ -385,7 +385,7 @@ class VariableOpsTest(xla_test.XLATestCase): resource_variable_ops.resource_scatter_max( handle, [0], constant_op.constant(3, dtype=dtypes.int32))) read = resource_variable_ops.read_variable_op(handle, dtype=dtypes.int32) - self.assertEqual(sess.run(read), [[6]]) + self.assertEqual(self.evaluate(read), [[6]]) def testScatterNdAddOps(self): with self.test_session() as sess, self.test_scope(): @@ -400,7 +400,7 @@ class VariableOpsTest(xla_test.XLATestCase): sess.run(gen_state_ops.resource_scatter_nd_add(handle, indices, updates)) read = resource_variable_ops.read_variable_op( handle, dtype=dtypes.float32) - self.assertAllClose(expected, sess.run(read)) + self.assertAllClose(expected, self.evaluate(read)) def testScatterNdUpdateAddOps(self): with self.test_session() as sess, self.test_scope(): @@ -416,7 +416,7 @@ class VariableOpsTest(xla_test.XLATestCase): gen_state_ops.resource_scatter_nd_update(handle, indices, updates)) read = resource_variable_ops.read_variable_op( handle, dtype=dtypes.float32) - self.assertAllClose(expected, sess.run(read)) + self.assertAllClose(expected, self.evaluate(read)) class StridedSliceAssignChecker(object): diff --git a/tensorflow/compiler/tests/xla_device_test.py b/tensorflow/compiler/tests/xla_device_test.py index 28d61fb07dc..ef55292b1be 100644 --- a/tensorflow/compiler/tests/xla_device_test.py +++ b/tensorflow/compiler/tests/xla_device_test.py @@ -81,7 +81,7 @@ class XlaDeviceTest(xla_test.XLATestCase): with self.cached_session() as sess: with self.test_scope(): x = gen_control_flow_ops.control_trigger() - sess.run(x) + self.evaluate(x) if __name__ == "__main__": diff --git a/tensorflow/compiler/tf2xla/BUILD b/tensorflow/compiler/tf2xla/BUILD index e0171415492..5a0d9b9af9d 100644 --- a/tensorflow/compiler/tf2xla/BUILD +++ b/tensorflow/compiler/tf2xla/BUILD @@ -9,6 +9,7 @@ package_group( "//tensorflow/compiler/jit/...", "//tensorflow/compiler/tests/...", "//tensorflow/compiler/tf2xla/...", + "//tensorflow/contrib/compiler/...", ], ) @@ -195,8 +196,8 @@ cc_library( ":sharding_util", ":side_effect_util", ":tf2xla_util", + "//tensorflow/compiler/jit:flags", "//tensorflow/compiler/jit:xla_cluster_util", - "//tensorflow/compiler/jit/legacy_flags:mark_for_compilation_pass_flags", "//tensorflow/compiler/tf2xla/lib:util", "//tensorflow/compiler/xla:literal", "//tensorflow/compiler/xla:shape_util", @@ -204,13 +205,13 @@ cc_library( "//tensorflow/compiler/xla:statusor", "//tensorflow/compiler/xla:types", "//tensorflow/compiler/xla:xla_data_proto", + "//tensorflow/compiler/xla/client", "//tensorflow/compiler/xla/client:client_library", "//tensorflow/compiler/xla/client:local_client", "//tensorflow/compiler/xla/client:xla_builder", "//tensorflow/compiler/xla/client:xla_computation", "//tensorflow/compiler/xla/client/lib:arithmetic", "//tensorflow/compiler/xla/client/lib:constants", - "//tensorflow/compiler/xla/client/lib:numeric", "//tensorflow/core:core_cpu", "//tensorflow/core:core_cpu_internal", "//tensorflow/core:framework", @@ -221,6 +222,7 @@ cc_library( "@com_google_absl//absl/algorithm:container", "@com_google_absl//absl/memory", "@com_google_absl//absl/strings", + "@com_google_absl//absl/types:optional", "@com_google_absl//absl/types:span", ], alwayslink = 1, @@ -437,21 +439,15 @@ cc_library( name = "dump_graph", srcs = [ "dump_graph.cc", - "dump_graph_flags.cc", - "dump_graph_flags.h", ], hdrs = [ "dump_graph.h", ], deps = [ - "//tensorflow/compiler/xla:parse_flags_from_env", - "//tensorflow/core:core_cpu", - "//tensorflow/core:core_cpu_internal", + "//tensorflow/compiler/jit:flags", "//tensorflow/core:framework", - "//tensorflow/core:framework_internal", - "//tensorflow/core:lib", + "//tensorflow/core:graph", "//tensorflow/core:protos_all_cc", - "@com_google_absl//absl/strings", ], ) diff --git a/tensorflow/compiler/tf2xla/dump_graph.cc b/tensorflow/compiler/tf2xla/dump_graph.cc index 380c6a7e23d..64fdbbebc65 100644 --- a/tensorflow/compiler/tf2xla/dump_graph.cc +++ b/tensorflow/compiler/tf2xla/dump_graph.cc @@ -18,87 +18,26 @@ limitations under the License. #include "tensorflow/compiler/tf2xla/dump_graph.h" -#include "absl/strings/str_cat.h" -#include "tensorflow/compiler/tf2xla/dump_graph_flags.h" -#include "tensorflow/core/platform/env.h" -#include "tensorflow/core/platform/mutex.h" +#include "tensorflow/compiler/jit/flags.h" +#include "tensorflow/core/util/dump_graph.h" namespace tensorflow { namespace dump_graph { -namespace { - -struct NameCounts { - mutex counts_mutex; - std::unordered_map counts; -}; - -string MakeUniqueFilename(string name) { - static NameCounts& instance = *new NameCounts; - - // Remove illegal characters from `name`. - for (int i = 0; i < name.size(); ++i) { - char ch = name[i]; - if (ch == '/' || ch == '[' || ch == ']' || ch == '*' || ch == '?') { - name[i] = '_'; - } - } - - int count; - { - mutex_lock lock(instance.counts_mutex); - count = instance.counts[name]++; - } - - string filename = name; - if (count > 0) { - absl::StrAppend(&filename, "_", count); - } - absl::StrAppend(&filename, ".pbtxt"); - return filename; -} - -string WriteTextProtoToUniqueFile( - Env* env, const string& name, const char* proto_type, - const ::tensorflow::protobuf::Message& proto) { - const string& dirname = - legacy_flags::GetDumpGraphFlags()->tf_dump_graph_prefix; - Status status = env->RecursivelyCreateDir(dirname); - if (!status.ok()) { - LOG(WARNING) << "Failed to create " << dirname << " for dumping " - << proto_type << ": " << status; - return "(unavailable)"; - } - string filepath = absl::StrCat(dirname, "/", MakeUniqueFilename(name)); - status = WriteTextProto(Env::Default(), filepath, proto); - if (!status.ok()) { - LOG(WARNING) << "Failed to dump " << proto_type << " to file: " << filepath - << " : " << status; - return "(unavailable)"; - } - LOG(INFO) << "Dumped " << proto_type << " to " << filepath; - return filepath; -} - -} // anonymous namespace - string DumpGraphDefToFile(const string& name, GraphDef const& graph_def) { - return WriteTextProtoToUniqueFile(Env::Default(), name, "GraphDef", - graph_def); + return tensorflow::DumpGraphDefToFile( + name, graph_def, GetDumpGraphFlags()->tf_dump_graph_prefix); } string DumpGraphToFile(const string& name, Graph const& graph, const FunctionLibraryDefinition* flib_def) { - GraphDef graph_def; - graph.ToGraphDef(&graph_def); - if (flib_def) { - *graph_def.mutable_library() = flib_def->ToProto(); - } - return DumpGraphDefToFile(name, graph_def); + return tensorflow::DumpGraphToFile(name, graph, flib_def, + GetDumpGraphFlags()->tf_dump_graph_prefix); } string DumpFunctionDefToFile(const string& name, FunctionDef const& fdef) { - return WriteTextProtoToUniqueFile(Env::Default(), name, "FunctionDef", fdef); + return tensorflow::DumpFunctionDefToFile( + name, fdef, GetDumpGraphFlags()->tf_dump_graph_prefix); } } // namespace dump_graph diff --git a/tensorflow/compiler/tf2xla/dump_graph_flags.cc b/tensorflow/compiler/tf2xla/dump_graph_flags.cc deleted file mode 100644 index 2eb1f8cd849..00000000000 --- a/tensorflow/compiler/tf2xla/dump_graph_flags.cc +++ /dev/null @@ -1,63 +0,0 @@ -/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -// Legacy flags for the XLA bridge's dump_graph module. - -#include -#include - -#include "tensorflow/compiler/tf2xla/dump_graph_flags.h" -#include "tensorflow/compiler/xla/parse_flags_from_env.h" -#include "tensorflow/core/platform/types.h" -#include "tensorflow/core/util/command_line_flags.h" - -namespace tensorflow { -namespace legacy_flags { - -// Pointers to the parsed value of the flags and flag descriptors, initialized -// via flags_init. -static DumpGraphFlags* flags; -static std::vector* flag_list; -static std::once_flag flags_init; - -// Allocate *flags. Called via call_once(&flags_init,...). -static void AllocateFlags() { - flags = new DumpGraphFlags; - flags->tf_dump_graph_prefix = "/tmp/"; - flag_list = new std::vector({ - Flag("tf_dump_graph_prefix", &flags->tf_dump_graph_prefix, - "Path prefix to which graphs dumped during debugging should be " - "written."), - }); - xla::ParseFlagsFromEnv(*flag_list); -} - -// Append to *append_to flag definitions associated with the XLA bridge's -// dump_graph module. -void AppendDumpGraphFlags(std::vector* append_to) { - std::call_once(flags_init, &AllocateFlags); - append_to->insert(append_to->end(), flag_list->begin(), flag_list->end()); -} - -// Return a pointer to the DumpGraphFlags struct; -// repeated calls return the same pointer. -// This should be called only after Flags::Parse() has returned. -DumpGraphFlags* GetDumpGraphFlags() { - std::call_once(flags_init, &AllocateFlags); - return flags; -} - -} // namespace legacy_flags -} // namespace tensorflow diff --git a/tensorflow/compiler/tf2xla/dump_graph_flags.h b/tensorflow/compiler/tf2xla/dump_graph_flags.h deleted file mode 100644 index 80a3307d920..00000000000 --- a/tensorflow/compiler/tf2xla/dump_graph_flags.h +++ /dev/null @@ -1,48 +0,0 @@ -/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -#ifndef TENSORFLOW_COMPILER_TF2XLA_DUMP_GRAPH_FLAGS_H_ -#define TENSORFLOW_COMPILER_TF2XLA_DUMP_GRAPH_FLAGS_H_ - -// Legacy flags for the XLA bridge's dump_graph module. - -#include - -#include "tensorflow/core/platform/types.h" -#include "tensorflow/core/util/command_line_flags.h" - -namespace tensorflow { -namespace legacy_flags { - -// Append to *flag_list flag definitions associated with the XLA bridge's -// dump_graph module. -void AppendDumpGraphFlags(std::vector* flag_list); - -// The values of flags associated with the XLA bridge's -// dump_graph module. -typedef struct { - string tf_dump_graph_prefix; // Path prefix to which graphs dumped during - // debugging should be written. -} DumpGraphFlags; - -// Return a pointer to the DumpGraphFlags struct; -// repeated calls return the same pointer. -// This should be called only after Flags::Parse() has returned. -DumpGraphFlags* GetDumpGraphFlags(); - -} // namespace legacy_flags -} // namespace tensorflow - -#endif // TENSORFLOW_COMPILER_TF2XLA_DUMP_GRAPH_FLAGS_H_ diff --git a/tensorflow/compiler/tf2xla/functionalize_control_flow.cc b/tensorflow/compiler/tf2xla/functionalize_control_flow.cc index 9ef9f49f422..3dfd3f854c8 100644 --- a/tensorflow/compiler/tf2xla/functionalize_control_flow.cc +++ b/tensorflow/compiler/tf2xla/functionalize_control_flow.cc @@ -75,6 +75,25 @@ Status FunctionalizeControlFlow(Graph* graph, return FunctionalizeControlFlow(/*lookup_library=*/nullptr, graph, library); } +Status FunctionalizeControlFlowForGraphDef(GraphDef* graph_def, + FunctionLibraryDefinition* library) { + return FunctionalizeControlFlowForGraphDef(/*lookup_library=*/nullptr, + graph_def, library); +} + +Status FunctionalizeControlFlowForGraphDef( + const FunctionLibraryDefinition* lookup_library, GraphDef* graph_def, + FunctionLibraryDefinition* library) { + FunctionDefLibrary function_lib = graph_def->library(); + Graph graph(OpRegistry::Global()); + + TF_RETURN_IF_ERROR(ConvertGraphDefToGraph({}, *graph_def, &graph)); + TF_RETURN_IF_ERROR(FunctionalizeControlFlow(lookup_library, &graph, library)); + graph.ToGraphDef(graph_def); + std::swap(*graph_def->mutable_library(), function_lib); + return Status::OK(); +} + Status FunctionalizeControlFlowForFunction( const string& func_name, const string& new_func_name, const protobuf::Map& attrs, diff --git a/tensorflow/compiler/tf2xla/functionalize_control_flow.h b/tensorflow/compiler/tf2xla/functionalize_control_flow.h index ba99205640c..91d33fa4058 100644 --- a/tensorflow/compiler/tf2xla/functionalize_control_flow.h +++ b/tensorflow/compiler/tf2xla/functionalize_control_flow.h @@ -33,6 +33,12 @@ Status FunctionalizeControlFlow(const FunctionLibraryDefinition* lookup_library, Graph* graph, FunctionLibraryDefinition* library); +Status FunctionalizeControlFlowForGraphDef(GraphDef* graph_def, + FunctionLibraryDefinition* library); +Status FunctionalizeControlFlowForGraphDef( + const FunctionLibraryDefinition* lookup_library, GraphDef* graph_def, + FunctionLibraryDefinition* library); + // This pass looks at the graph and all associated FunctionDefs, and turns // traditional control flow structure (Switch/Merge/etc.) into functional // control flow structure (If/While). diff --git a/tensorflow/compiler/tf2xla/functionalize_control_flow_test.cc b/tensorflow/compiler/tf2xla/functionalize_control_flow_test.cc index c3841f996f8..9784985af83 100644 --- a/tensorflow/compiler/tf2xla/functionalize_control_flow_test.cc +++ b/tensorflow/compiler/tf2xla/functionalize_control_flow_test.cc @@ -95,77 +95,87 @@ TEST(FunctionalizeControlFlow, Conditional) { } FunctionLibraryDefinition library(OpRegistry::Global(), {}); + GraphDef optimized_graph_def; + graph.ToGraphDef(&optimized_graph_def); + TF_ASSERT_OK( + FunctionalizeControlFlowForGraphDef(&optimized_graph_def, &library)); TF_ASSERT_OK(FunctionalizeControlFlow(&graph, &library)); + GraphDef converted_graph_def; + graph.ToGraphDef(&converted_graph_def); - GraphDef graph_def; - graph.ToGraphDef(&graph_def); - string op_name; - NameAttrList then_fn; - NameAttrList else_fn; - TF_EXPECT_OK(FindIfThenAndElse(graph_def, &op_name, &then_fn, &else_fn)); - InstantiationResultForTest else_result; - TF_EXPECT_OK( - InstantiateFunctionForTest(else_fn.name(), library, &else_result)); + for (const GraphDef& graph_def : {optimized_graph_def, converted_graph_def}) { + string op_name; + NameAttrList then_fn; + NameAttrList else_fn; + TF_EXPECT_OK(FindIfThenAndElse(graph_def, &op_name, &then_fn, &else_fn)); + InstantiationResultForTest else_result; + TF_EXPECT_OK( + InstantiateFunctionForTest(else_fn.name(), library, &else_result)); - // Outer graph - { - Scope scope = Scope::NewRootScope().ExitOnError(); - auto y = ops::Placeholder(scope.WithOpName("y"), DT_INT32); - auto x = ops::Placeholder(scope.WithOpName("x"), DT_INT32); - auto less = ops::Less(scope.WithOpName("cond/Less"), y, x); - auto if_op = ops::If(scope.WithOpName(op_name), less, - std::initializer_list{less, y, x}, {DT_INT32}, - then_fn, else_fn); - auto id = ops::Identity(scope.WithOpName("cond/Merge"), if_op.output[0]); - GraphDef expected; - TF_EXPECT_OK(scope.ToGraphDef(&expected)); - TF_EXPECT_GRAPH_EQ(expected, graph_def); - } + // Outer graph + { + Scope scope = Scope::NewRootScope().ExitOnError(); + auto y = ops::Placeholder(scope.WithOpName("y"), DT_INT32); + auto x = ops::Placeholder(scope.WithOpName("x"), DT_INT32); + auto less = ops::Less(scope.WithOpName("cond/Less"), y, x); + auto if_op = ops::If(scope.WithOpName(op_name), less, + std::initializer_list{less, y, x}, {DT_INT32}, + then_fn, else_fn); + auto id = ops::Identity(scope.WithOpName("cond/Merge"), if_op.output[0]); + GraphDef expected; + TF_EXPECT_OK(scope.ToGraphDef(&expected)); + TF_EXPECT_GRAPH_EQ(expected, graph_def); + } - // then body. - { - Scope scope = Scope::NewRootScope().ExitOnError(); - auto arg_0 = ops::_Arg(scope.WithOpName("_arg0"), DT_BOOL, 0); - auto arg_1 = ops::_Arg(scope.WithOpName("_arg1"), DT_INT32, 1); - auto arg_2 = ops::_Arg(scope.WithOpName("_arg2"), DT_INT32, 2); - auto identity = ops::Identity(scope.WithOpName("cond/Identity"), arg_0); - auto cond = ops::Const( - scope.WithOpName("cond").WithControlDependencies(identity), 17); - auto mul = ops::Mul(scope.WithOpName("cond/Mul"), arg_1, cond); - auto retval0 = ops::_Retval(scope.WithOpName("_retval0_RetVal"), mul, 0); + // then body. + { + Scope scope = Scope::NewRootScope().ExitOnError(); + auto arg_0 = ops::_Arg(scope.WithOpName("_arg0"), DT_BOOL, 0); + auto arg_1 = ops::_Arg(scope.WithOpName("_arg1"), DT_INT32, 1); + auto arg_2 = ops::_Arg(scope.WithOpName("_arg2"), DT_INT32, 2); + auto identity = ops::Identity(scope.WithOpName("cond/Identity"), arg_0); + auto cond = ops::Const( + scope.WithOpName("cond").WithControlDependencies(identity), 17); + auto mul = ops::Mul(scope.WithOpName("cond/Mul"), arg_1, cond); + auto retval0 = ops::_Retval(scope.WithOpName("_retval0_RetVal"), mul, 0); - GraphDef expected; - TF_EXPECT_OK(scope.ToGraphDef(&expected)); + GraphDef expected; + TF_EXPECT_OK(scope.ToGraphDef(&expected)); - InstantiationResultForTest result; - TF_EXPECT_OK(InstantiateFunctionForTest(then_fn.name(), library, &result)); + InstantiationResultForTest result; + TF_EXPECT_OK( + InstantiateFunctionForTest(then_fn.name(), library, &result)); - EXPECT_EQ(DataTypeVector{DT_INT32}, result.ret_types); - EXPECT_EQ((DataTypeVector{DT_BOOL, DT_INT32, DT_INT32}), result.arg_types); - TF_EXPECT_GRAPH_EQ(expected, result.gdef); - } + EXPECT_EQ(DataTypeVector{DT_INT32}, result.ret_types); + EXPECT_EQ((DataTypeVector{DT_BOOL, DT_INT32, DT_INT32}), + result.arg_types); + TF_EXPECT_GRAPH_EQ(expected, result.gdef); + } - // else body. - { - Scope scope = Scope::NewRootScope().ExitOnError(); - auto arg_0 = ops::_Arg(scope.WithOpName("_arg0"), DT_BOOL, 0); - auto arg_1 = ops::_Arg(scope.WithOpName("_arg1"), DT_INT32, 1); - auto arg_2 = ops::_Arg(scope.WithOpName("_arg2"), DT_INT32, 2); - auto identity = ops::Identity(scope.WithOpName("cond/Identity_1"), arg_0); - auto cond_1 = ops::Const( - scope.WithOpName("cond_1").WithControlDependencies(identity), 23); - auto add = ops::Add(scope.WithOpName("cond/false/add"), arg_2, cond_1); - auto retval0 = ops::_Retval(scope.WithOpName("_retval0_RetVal"), add, 0); + // else body. + { + Scope scope = Scope::NewRootScope().ExitOnError(); + auto arg_0 = ops::_Arg(scope.WithOpName("_arg0"), DT_BOOL, 0); + auto arg_1 = ops::_Arg(scope.WithOpName("_arg1"), DT_INT32, 1); + auto arg_2 = ops::_Arg(scope.WithOpName("_arg2"), DT_INT32, 2); + auto identity = ops::Identity(scope.WithOpName("cond/Identity_1"), arg_0); + auto cond_1 = ops::Const( + scope.WithOpName("cond_1").WithControlDependencies(identity), 23); + auto add = ops::Add(scope.WithOpName("cond/false/add"), arg_2, cond_1); + auto retval0 = ops::_Retval(scope.WithOpName("_retval0_RetVal"), add, 0); - GraphDef expected; - TF_EXPECT_OK(scope.ToGraphDef(&expected)); + GraphDef expected; + TF_EXPECT_OK(scope.ToGraphDef(&expected)); - InstantiationResultForTest result; - TF_EXPECT_OK(InstantiateFunctionForTest(else_fn.name(), library, &result)); + InstantiationResultForTest result; + TF_EXPECT_OK( + InstantiateFunctionForTest(else_fn.name(), library, &result)); - EXPECT_EQ(DataTypeVector{DT_INT32}, result.ret_types); - EXPECT_EQ((DataTypeVector{DT_BOOL, DT_INT32, DT_INT32}), result.arg_types); - TF_EXPECT_GRAPH_EQ(expected, result.gdef); + EXPECT_EQ(DataTypeVector{DT_INT32}, result.ret_types); + EXPECT_EQ((DataTypeVector{DT_BOOL, DT_INT32, DT_INT32}), + result.arg_types); + TF_EXPECT_GRAPH_EQ(expected, result.gdef); + } } } @@ -239,75 +249,77 @@ TEST(FunctionalizeControlFlow, OneLoopVar) { } FunctionLibraryDefinition library(OpRegistry::Global(), {}); + GraphDef optimized_graph_def; + graph.ToGraphDef(&optimized_graph_def); + TF_ASSERT_OK( + FunctionalizeControlFlowForGraphDef(&optimized_graph_def, &library)); TF_ASSERT_OK(FunctionalizeControlFlow(&graph, &library)); + GraphDef converted_graph_def; + graph.ToGraphDef(&converted_graph_def); - GraphDef graph_def; - graph.ToGraphDef(&graph_def); + for (const GraphDef& graph_def : {optimized_graph_def, converted_graph_def}) { + NameAttrList cond_fn, body_fn; + TF_EXPECT_OK(FindWhileCondAndBody(graph_def, &cond_fn, &body_fn)); - NameAttrList cond_fn, body_fn; - TF_EXPECT_OK(FindWhileCondAndBody(graph_def, &cond_fn, &body_fn)); + // Outer graph + { + Scope scope = Scope::NewRootScope().ExitOnError(); + auto source = ops::Placeholder(scope.WithOpName("source"), DT_INT32); + auto while_op = + ops::While(scope.WithOpName("while/LoopCond"), + std::initializer_list{source}, cond_fn, body_fn); + auto sink = ops::Identity(scope.WithOpName("sink"), while_op[0]); + GraphDef expected; + TF_EXPECT_OK(scope.ToGraphDef(&expected)); + TF_EXPECT_GRAPH_EQ(expected, graph_def); + } - // Outer graph - { - Scope scope = Scope::NewRootScope().ExitOnError(); - auto source = ops::Placeholder(scope.WithOpName("source"), DT_INT32); - auto while_op = - ops::While(scope.WithOpName("while/LoopCond"), - std::initializer_list{source}, cond_fn, body_fn); - auto sink = ops::Identity(scope.WithOpName("sink"), while_op[0]); - GraphDef expected; - TF_EXPECT_OK(scope.ToGraphDef(&expected)); - TF_EXPECT_GRAPH_EQ(expected, graph_def); - } + // Condition graph + { + Scope scope = Scope::NewRootScope().ExitOnError(); + auto arg = ops::_Arg(scope.WithOpName("_arg0"), DT_INT32, 0); + auto ten = ops::Const( + scope.WithOpName("while/Less/y").WithControlDependencies(arg), 10); + auto less = ops::Less(scope.WithOpName("while/Less"), arg, ten); + auto retval = ops::_Retval(scope.WithOpName("_retval0_RetVal"), less, 0); - // Condition graph - { - Scope scope = Scope::NewRootScope().ExitOnError(); - auto arg = ops::_Arg(scope.WithOpName("_arg0"), DT_INT32, 0); - auto ten = ops::Const( - scope.WithOpName("while/Less/y").WithControlDependencies(arg), 10); - auto less = ops::Less(scope.WithOpName("while/Less"), arg, ten); - auto retval = ops::_Retval(scope.WithOpName("_retval0_RetVal"), less, 0); + GraphDef expected; + TF_EXPECT_OK(scope.ToGraphDef(&expected)); - GraphDef expected; - TF_EXPECT_OK(scope.ToGraphDef(&expected)); + InstantiationResultForTest result; + TF_EXPECT_OK( + InstantiateFunctionForTest(cond_fn.name(), library, &result)); - InstantiationResultForTest result; - TF_EXPECT_OK(InstantiateFunctionForTest(cond_fn.name(), library, &result)); + EXPECT_EQ(DataTypeVector{DT_INT32}, result.arg_types); + EXPECT_EQ(DataTypeVector{DT_BOOL}, result.ret_types); + TF_EXPECT_GRAPH_EQ(expected, result.gdef); + } - EXPECT_EQ(DataTypeVector{DT_INT32}, result.arg_types); - EXPECT_EQ(DataTypeVector{DT_BOOL}, result.ret_types); - TF_EXPECT_GRAPH_EQ(expected, result.gdef); - } + // Body graph. + { + Scope scope = Scope::NewRootScope().ExitOnError(); + auto arg = ops::_Arg(scope.WithOpName("_arg0"), DT_INT32, 0); + auto identity = ops::Identity(scope.WithOpName("while/Identity"), arg); + auto one = ops::Const( + scope.WithOpName("while/add/y").WithControlDependencies(identity), 1); + auto add = ops::Add(scope.WithOpName("while/add"), identity, one); + auto retval = ops::_Retval(scope.WithOpName("_retval0_RetVal"), add, 0); - // Body graph. - { - Scope scope = Scope::NewRootScope().ExitOnError(); - auto arg = ops::_Arg(scope.WithOpName("_arg0"), DT_INT32, 0); - auto identity = ops::Identity(scope.WithOpName("while/Identity"), arg); - auto one = ops::Const( - scope.WithOpName("while/add/y").WithControlDependencies(identity), 1); - auto add = ops::Add(scope.WithOpName("while/add"), identity, one); - auto retval = ops::_Retval(scope.WithOpName("_retval0_RetVal"), add, 0); + GraphDef expected; + TF_EXPECT_OK(scope.ToGraphDef(&expected)); - GraphDef expected; - TF_EXPECT_OK(scope.ToGraphDef(&expected)); + InstantiationResultForTest result; + TF_EXPECT_OK( + InstantiateFunctionForTest(body_fn.name(), library, &result)); - InstantiationResultForTest result; - TF_EXPECT_OK(InstantiateFunctionForTest(body_fn.name(), library, &result)); - - EXPECT_EQ(DataTypeVector{DT_INT32}, result.arg_types); - EXPECT_EQ(DataTypeVector{DT_INT32}, result.ret_types); - TF_EXPECT_GRAPH_EQ(expected, result.gdef); + EXPECT_EQ(DataTypeVector{DT_INT32}, result.arg_types); + EXPECT_EQ(DataTypeVector{DT_INT32}, result.ret_types); + TF_EXPECT_GRAPH_EQ(expected, result.gdef); + } } } -// @function.Defun(noinline=True) -// def increment_fn(x): -// return [x + 1] -// Define the above function, and add it to the given graph. It's used as the -// while loop body in NoinlineLoopBody test. -Status AddNoinlineFunctionToGraph(const string& node_name, Graph* graph) { +FunctionDef GetNoinlineFunctionDef() { FunctionDef fdef = FunctionDefHelper::Create( "increment_fn", {"x:int32"}, {"add:int32"}, {}, { @@ -316,8 +328,17 @@ Status AddNoinlineFunctionToGraph(const string& node_name, Graph* graph) { }, {{"add", "add_0:z:0"}}); (*fdef.mutable_attr())["_noinline"].set_b(true); + return fdef; +} + +// @function.Defun(noinline=True) +// def increment_fn(x): +// return [x + 1] +// Define the above function, and add it to the given graph. It's used as the +// while loop body in NoinlineLoopBody test. +Status AddNoinlineFunctionToGraph(const string& node_name, Graph* graph) { FunctionDefLibrary fdef_lib; - *(fdef_lib.add_function()) = fdef; + *(fdef_lib.add_function()) = GetNoinlineFunctionDef(); TF_RETURN_IF_ERROR(graph->AddFunctionLibrary(fdef_lib)); NodeDef increment_fn; increment_fn.set_name(node_name); @@ -376,55 +397,88 @@ TEST(FunctionalizeControlFlow, NoinlineLoopBody) { FunctionLibraryDefinition lookup_lib(graph.flib_def()); FunctionLibraryDefinition library(OpRegistry::Global(), {}); // Function increment_fn will be copied from lookup_lib to library. + GraphDef optimized_graph_def; + graph.ToGraphDef(&optimized_graph_def); + + *(optimized_graph_def.mutable_library()->add_function()) = + GetNoinlineFunctionDef(); + + TF_ASSERT_OK(FunctionalizeControlFlowForGraphDef( + &lookup_lib, &optimized_graph_def, &library)); TF_ASSERT_OK(FunctionalizeControlFlow(&lookup_lib, &graph, &library)); + GraphDef converted_graph_def; + graph.ToGraphDef(&converted_graph_def); - GraphDef graph_def; - graph.ToGraphDef(&graph_def); + for (const GraphDef& graph_def : {optimized_graph_def, converted_graph_def}) { + NameAttrList cond_fn, body_fn; + TF_ASSERT_OK(FindWhileCondAndBody(graph_def, &cond_fn, &body_fn)); - NameAttrList cond_fn, body_fn; - TF_ASSERT_OK(FindWhileCondAndBody(graph_def, &cond_fn, &body_fn)); + // Outer graph + { + Scope scope = Scope::NewRootScope().ExitOnError(); + auto source = ops::Placeholder(scope.WithOpName("source"), DT_INT32); + auto while_op = + ops::While(scope.WithOpName("while/LoopCond"), + std::initializer_list{source}, cond_fn, body_fn); + GraphDef expected; + TF_ASSERT_OK(scope.ToGraphDef(&expected)); + TF_EXPECT_GRAPH_EQ(expected, graph_def); + } - // Outer graph + // Body graph. + { + Scope scope = Scope::NewRootScope().ExitOnError(); + auto arg = ops::_Arg(scope.WithOpName("_arg0"), DT_INT32, 0); + TF_ASSERT_OK( + AddNoinlineFunctionToGraph(noinline_node_name, scope.graph())); + auto identity = ops::Identity(scope.WithOpName("while/Identity"), arg); + NodeDef retval; + retval.set_name("_retval0_RetVal"); + retval.set_op(FunctionLibraryDefinition::kRetOp); + *retval.add_input() = noinline_node_name; + (*retval.mutable_attr())["T"].set_type(DT_INT32); + (*retval.mutable_attr())["index"].set_i(0); + Status status; + scope.graph()->AddNode(retval, &status); + TF_ASSERT_OK(status); + + GraphDef expected; + TF_ASSERT_OK(scope.ToGraphDef(&expected)); + + InstantiationResultForTest result; + // Verify that increment_fn has been copied to library. + TF_EXPECT_OK( + InstantiateFunctionForTest(body_fn.name(), library, &result)); + + EXPECT_EQ(DataTypeVector{DT_INT32}, result.arg_types); + EXPECT_EQ(DataTypeVector{DT_INT32}, result.ret_types); + // Ignore the function library when comparing the graphs. + expected.clear_library(); + TF_EXPECT_GRAPH_EQ(expected, result.gdef); + } + } +} + +TEST(FunctionalizeControlFlow, MissingFunctionDefInLibrary) { + const string& noinline_node_name = "while/increment_fn"; + Graph graph(OpRegistry::Global()); { Scope scope = Scope::NewRootScope().ExitOnError(); auto source = ops::Placeholder(scope.WithOpName("source"), DT_INT32); - auto while_op = - ops::While(scope.WithOpName("while/LoopCond"), - std::initializer_list{source}, cond_fn, body_fn); - GraphDef expected; - TF_ASSERT_OK(scope.ToGraphDef(&expected)); - TF_EXPECT_GRAPH_EQ(expected, graph_def); - } - - // Body graph. - { - Scope scope = Scope::NewRootScope().ExitOnError(); - auto arg = ops::_Arg(scope.WithOpName("_arg0"), DT_INT32, 0); + auto identity = ops::Identity(scope.WithOpName("while/Identity"), source); TF_ASSERT_OK(AddNoinlineFunctionToGraph(noinline_node_name, scope.graph())); - auto identity = ops::Identity(scope.WithOpName("while/Identity"), arg); - NodeDef retval; - retval.set_name("_retval0_RetVal"); - retval.set_op(FunctionLibraryDefinition::kRetOp); - *retval.add_input() = noinline_node_name; - (*retval.mutable_attr())["T"].set_type(DT_INT32); - (*retval.mutable_attr())["index"].set_i(0); - Status status; - scope.graph()->AddNode(retval, &status); - TF_ASSERT_OK(status); - - GraphDef expected; - TF_ASSERT_OK(scope.ToGraphDef(&expected)); - - InstantiationResultForTest result; - // Verify that increment_fn has been copied to library. - TF_EXPECT_OK(InstantiateFunctionForTest(body_fn.name(), library, &result)); - - EXPECT_EQ(DataTypeVector{DT_INT32}, result.arg_types); - EXPECT_EQ(DataTypeVector{DT_INT32}, result.ret_types); - // Ignore the function library when comparing the graphs. - expected.clear_library(); - TF_EXPECT_GRAPH_EQ(expected, result.gdef); + TF_ASSERT_OK(scope.ToGraph(&graph)); } + + FunctionLibraryDefinition lookup_lib(graph.flib_def()); + FunctionLibraryDefinition library(OpRegistry::Global(), {}); + GraphDef graph_def; + graph.ToGraphDef(&graph_def); + graph_def.clear_library(); + + Status status = + FunctionalizeControlFlowForGraphDef(&lookup_lib, &graph_def, &library); + EXPECT_EQ(tensorflow::error::NOT_FOUND, status.code()); } // Tests functionalizing OneLoopVar where the loop value is not used post the @@ -467,65 +521,72 @@ TEST(FunctionalizeControlFlow, OneLoopVarWithoutExit) { } FunctionLibraryDefinition library(OpRegistry::Global(), {}); + GraphDef optimized_graph_def; + graph.ToGraphDef(&optimized_graph_def); + TF_ASSERT_OK( + FunctionalizeControlFlowForGraphDef(&optimized_graph_def, &library)); TF_ASSERT_OK(FunctionalizeControlFlow(&graph, &library)); + GraphDef converted_graph_def; + graph.ToGraphDef(&converted_graph_def); - GraphDef graph_def; - graph.ToGraphDef(&graph_def); + for (const GraphDef& graph_def : {optimized_graph_def, converted_graph_def}) { + NameAttrList cond_fn, body_fn; + TF_EXPECT_OK(FindWhileCondAndBody(graph_def, &cond_fn, &body_fn)); - NameAttrList cond_fn, body_fn; - TF_EXPECT_OK(FindWhileCondAndBody(graph_def, &cond_fn, &body_fn)); + // Outer graph + { + Scope scope = Scope::NewRootScope().ExitOnError(); + auto source = ops::Placeholder(scope.WithOpName("source"), DT_INT32); + auto while_op = + ops::While(scope.WithOpName("while/LoopCond"), + std::initializer_list{source}, cond_fn, body_fn); + GraphDef expected; + TF_EXPECT_OK(scope.ToGraphDef(&expected)); + TF_EXPECT_GRAPH_EQ(expected, graph_def); + } - // Outer graph - { - Scope scope = Scope::NewRootScope().ExitOnError(); - auto source = ops::Placeholder(scope.WithOpName("source"), DT_INT32); - auto while_op = - ops::While(scope.WithOpName("while/LoopCond"), - std::initializer_list{source}, cond_fn, body_fn); - GraphDef expected; - TF_EXPECT_OK(scope.ToGraphDef(&expected)); - TF_EXPECT_GRAPH_EQ(expected, graph_def); - } + // Condition graph + { + Scope scope = Scope::NewRootScope().ExitOnError(); + auto arg = ops::_Arg(scope.WithOpName("_arg0"), DT_INT32, 0); + auto ten = ops::Const( + scope.WithOpName("while/Less/y").WithControlDependencies(arg), 10); + auto less = ops::Less(scope.WithOpName("while/Less"), arg, ten); + auto retval = ops::_Retval(scope.WithOpName("_retval0_RetVal"), less, 0); - // Condition graph - { - Scope scope = Scope::NewRootScope().ExitOnError(); - auto arg = ops::_Arg(scope.WithOpName("_arg0"), DT_INT32, 0); - auto ten = ops::Const( - scope.WithOpName("while/Less/y").WithControlDependencies(arg), 10); - auto less = ops::Less(scope.WithOpName("while/Less"), arg, ten); - auto retval = ops::_Retval(scope.WithOpName("_retval0_RetVal"), less, 0); + GraphDef expected; + TF_EXPECT_OK(scope.ToGraphDef(&expected)); - GraphDef expected; - TF_EXPECT_OK(scope.ToGraphDef(&expected)); + InstantiationResultForTest result; + TF_EXPECT_OK( + InstantiateFunctionForTest(cond_fn.name(), library, &result)); - InstantiationResultForTest result; - TF_EXPECT_OK(InstantiateFunctionForTest(cond_fn.name(), library, &result)); + EXPECT_EQ(DataTypeVector{DT_INT32}, result.arg_types); + EXPECT_EQ(DataTypeVector{DT_BOOL}, result.ret_types); + TF_EXPECT_GRAPH_EQ(expected, result.gdef); + } - EXPECT_EQ(DataTypeVector{DT_INT32}, result.arg_types); - EXPECT_EQ(DataTypeVector{DT_BOOL}, result.ret_types); - TF_EXPECT_GRAPH_EQ(expected, result.gdef); - } + // Body graph. + { + Scope scope = Scope::NewRootScope().ExitOnError(); + auto arg = ops::_Arg(scope.WithOpName("_arg0"), DT_INT32, 0); + auto identity = ops::Identity(scope.WithOpName("while/Identity"), arg); + auto one = ops::Const( + scope.WithOpName("while/add/y").WithControlDependencies(identity), 1); + auto add = ops::Add(scope.WithOpName("while/add"), identity, one); + auto retval = ops::_Retval(scope.WithOpName("_retval0_RetVal"), add, 0); - // Body graph. - { - Scope scope = Scope::NewRootScope().ExitOnError(); - auto arg = ops::_Arg(scope.WithOpName("_arg0"), DT_INT32, 0); - auto identity = ops::Identity(scope.WithOpName("while/Identity"), arg); - auto one = ops::Const( - scope.WithOpName("while/add/y").WithControlDependencies(identity), 1); - auto add = ops::Add(scope.WithOpName("while/add"), identity, one); - auto retval = ops::_Retval(scope.WithOpName("_retval0_RetVal"), add, 0); + GraphDef expected; + TF_EXPECT_OK(scope.ToGraphDef(&expected)); - GraphDef expected; - TF_EXPECT_OK(scope.ToGraphDef(&expected)); + InstantiationResultForTest result; + TF_EXPECT_OK( + InstantiateFunctionForTest(body_fn.name(), library, &result)); - InstantiationResultForTest result; - TF_EXPECT_OK(InstantiateFunctionForTest(body_fn.name(), library, &result)); - - EXPECT_EQ(DataTypeVector{DT_INT32}, result.arg_types); - EXPECT_EQ(DataTypeVector{DT_INT32}, result.ret_types); - TF_EXPECT_GRAPH_EQ(expected, result.gdef); + EXPECT_EQ(DataTypeVector{DT_INT32}, result.arg_types); + EXPECT_EQ(DataTypeVector{DT_INT32}, result.ret_types); + TF_EXPECT_GRAPH_EQ(expected, result.gdef); + } } } @@ -608,86 +669,95 @@ TEST(FunctionalizeControlFlow, TwoLoopVars) { } FunctionLibraryDefinition library(OpRegistry::Global(), {}); + GraphDef optimized_graph_def; + graph.ToGraphDef(&optimized_graph_def); + TF_ASSERT_OK( + FunctionalizeControlFlowForGraphDef(&optimized_graph_def, &library)); TF_ASSERT_OK(FunctionalizeControlFlow(&graph, &library)); + GraphDef converted_graph_def; + graph.ToGraphDef(&converted_graph_def); - GraphDef graph_def; - graph.ToGraphDef(&graph_def); + for (const GraphDef& graph_def : {optimized_graph_def, converted_graph_def}) { + NameAttrList cond_fn, body_fn; + TF_EXPECT_OK(FindWhileCondAndBody(graph_def, &cond_fn, &body_fn)); - NameAttrList cond_fn, body_fn; - TF_EXPECT_OK(FindWhileCondAndBody(graph_def, &cond_fn, &body_fn)); + // Outer graph. + { + Scope scope = Scope::NewRootScope().ExitOnError(); + auto x = ops::Placeholder(scope.WithOpName("Placeholder/x"), DT_INT32); + auto y = ops::Placeholder(scope.WithOpName("Placeholder/y"), DT_INT32); + auto while_op = + ops::While(scope.WithOpName("while/LoopCond"), + std::initializer_list{x, y}, cond_fn, body_fn); + auto sink_x = ops::Identity(scope.WithOpName("sink_x"), while_op[0]); + auto sink_y = ops::Identity(scope.WithOpName("sink_y"), while_op[1]); + GraphDef expected; + TF_EXPECT_OK(scope.ToGraphDef(&expected)); + TF_EXPECT_GRAPH_EQ(expected, graph_def); + } - // Outer graph. - { - Scope scope = Scope::NewRootScope().ExitOnError(); - auto x = ops::Placeholder(scope.WithOpName("Placeholder/x"), DT_INT32); - auto y = ops::Placeholder(scope.WithOpName("Placeholder/y"), DT_INT32); - auto while_op = - ops::While(scope.WithOpName("while/LoopCond"), - std::initializer_list{x, y}, cond_fn, body_fn); - auto sink_x = ops::Identity(scope.WithOpName("sink_x"), while_op[0]); - auto sink_y = ops::Identity(scope.WithOpName("sink_y"), while_op[1]); - GraphDef expected; - TF_EXPECT_OK(scope.ToGraphDef(&expected)); - TF_EXPECT_GRAPH_EQ(expected, graph_def); - } - - // Condition graph. - { - Scope scope = Scope::NewRootScope().ExitOnError(); - auto arg0 = ops::_Arg(scope.WithOpName("_arg0"), DT_INT32, 0); - auto arg1 = ops::_Arg(scope.WithOpName("_arg1"), DT_INT32, 1); - auto three = ops::Const(scope.WithOpName("while/cond/three") + // Condition graph. + { + Scope scope = Scope::NewRootScope().ExitOnError(); + auto arg0 = ops::_Arg(scope.WithOpName("_arg0"), DT_INT32, 0); + auto arg1 = ops::_Arg(scope.WithOpName("_arg1"), DT_INT32, 1); + auto three = ops::Const(scope.WithOpName("while/cond/three") + .WithControlDependencies(arg0.output), + 3); + auto cond_add = + ops::Add(scope.WithOpName("while/cond/Add"), arg0.output, three); + auto ten = ops::Const(scope.WithOpName("while/cond/ten") .WithControlDependencies(arg0.output), - 3); - auto cond_add = - ops::Add(scope.WithOpName("while/cond/Add"), arg0.output, three); - auto ten = ops::Const( - scope.WithOpName("while/cond/ten").WithControlDependencies(arg0.output), - 10); - auto less = ops::Less(scope.WithOpName("while/cond/Less"), cond_add, ten); - auto retval = ops::_Retval(scope.WithOpName("_retval0_RetVal"), less, 0); + 10); + auto less = ops::Less(scope.WithOpName("while/cond/Less"), cond_add, ten); + auto retval = ops::_Retval(scope.WithOpName("_retval0_RetVal"), less, 0); - GraphDef expected; - TF_EXPECT_OK(scope.ToGraphDef(&expected)); + GraphDef expected; + TF_EXPECT_OK(scope.ToGraphDef(&expected)); - InstantiationResultForTest result; - TF_EXPECT_OK(InstantiateFunctionForTest(cond_fn.name(), library, &result)); + InstantiationResultForTest result; + TF_EXPECT_OK( + InstantiateFunctionForTest(cond_fn.name(), library, &result)); - EXPECT_EQ((DataTypeVector{DT_INT32, DT_INT32}), result.arg_types); - EXPECT_EQ(DataTypeVector{DT_BOOL}, result.ret_types); - TF_EXPECT_GRAPH_EQ(expected, result.gdef); - } + EXPECT_EQ((DataTypeVector{DT_INT32, DT_INT32}), result.arg_types); + EXPECT_EQ(DataTypeVector{DT_BOOL}, result.ret_types); + TF_EXPECT_GRAPH_EQ(expected, result.gdef); + } - // Body graph. - { - Scope scope = Scope::NewRootScope().ExitOnError(); - auto arg0 = ops::_Arg(scope.WithOpName("_arg0"), DT_INT32, 0); - auto arg1 = ops::_Arg(scope.WithOpName("_arg1"), DT_INT32, 1); + // Body graph. + { + Scope scope = Scope::NewRootScope().ExitOnError(); + auto arg0 = ops::_Arg(scope.WithOpName("_arg0"), DT_INT32, 0); + auto arg1 = ops::_Arg(scope.WithOpName("_arg1"), DT_INT32, 1); - auto identity_x = ops::Identity(scope.WithOpName("while/Identity/x"), arg0); - auto identity_y = ops::Identity(scope.WithOpName("while/Identity/y"), arg1); + auto identity_x = + ops::Identity(scope.WithOpName("while/Identity/x"), arg0); + auto identity_y = + ops::Identity(scope.WithOpName("while/Identity/y"), arg1); - auto one = ops::Const( - scope.WithOpName("while/add/one").WithControlDependencies(identity_x), - 1); - auto two = ops::Const( - scope.WithOpName("while/mul/two").WithControlDependencies(identity_x), - 2); + auto one = ops::Const( + scope.WithOpName("while/add/one").WithControlDependencies(identity_x), + 1); + auto two = ops::Const( + scope.WithOpName("while/mul/two").WithControlDependencies(identity_x), + 2); - auto add = ops::Add(scope.WithOpName("while/add"), identity_x, one); - auto mul = ops::Add(scope.WithOpName("while/mul"), identity_y, two); - auto retval0 = ops::_Retval(scope.WithOpName("_retval0_RetVal"), add, 0); - auto retval1 = ops::_Retval(scope.WithOpName("_retval1_RetVal"), mul, 1); + auto add = ops::Add(scope.WithOpName("while/add"), identity_x, one); + auto mul = ops::Add(scope.WithOpName("while/mul"), identity_y, two); + auto retval0 = ops::_Retval(scope.WithOpName("_retval0_RetVal"), add, 0); + auto retval1 = ops::_Retval(scope.WithOpName("_retval1_RetVal"), mul, 1); - GraphDef expected; - TF_EXPECT_OK(scope.ToGraphDef(&expected)); + GraphDef expected; + TF_EXPECT_OK(scope.ToGraphDef(&expected)); - InstantiationResultForTest result; - TF_EXPECT_OK(InstantiateFunctionForTest(body_fn.name(), library, &result)); + InstantiationResultForTest result; + TF_EXPECT_OK( + InstantiateFunctionForTest(body_fn.name(), library, &result)); - EXPECT_EQ((DataTypeVector{DT_INT32, DT_INT32}), result.arg_types); - EXPECT_EQ((DataTypeVector{DT_INT32, DT_INT32}), result.ret_types); - TF_EXPECT_GRAPH_EQ(expected, result.gdef); + EXPECT_EQ((DataTypeVector{DT_INT32, DT_INT32}), result.arg_types); + EXPECT_EQ((DataTypeVector{DT_INT32, DT_INT32}), result.ret_types); + TF_EXPECT_GRAPH_EQ(expected, result.gdef); + } } } @@ -841,177 +911,192 @@ TEST(FunctionalizeControlFlow, Complex) { } FunctionLibraryDefinition library(OpRegistry::Global(), {}); + GraphDef optimized_graph_def; + graph.ToGraphDef(&optimized_graph_def); + TF_ASSERT_OK( + FunctionalizeControlFlowForGraphDef(&optimized_graph_def, &library)); TF_ASSERT_OK(FunctionalizeControlFlow(&graph, &library)); + GraphDef converted_graph_def; + graph.ToGraphDef(&converted_graph_def); - GraphDef graph_def; - graph.ToGraphDef(&graph_def); - - NameAttrList outer_cond_fn, outer_body_fn; - TF_EXPECT_OK(FindWhileCondAndBody(graph_def, &outer_cond_fn, &outer_body_fn)); - - // Outer graph. - { - Scope scope = Scope::NewRootScope().ExitOnError(); - auto x = ops::Placeholder(scope.WithOpName("x"), DT_INT32); - auto three = ops::Const(scope.WithOpName("three"), 3); - auto y = ops::Add(scope.WithOpName("y"), x, three); - - auto var = ops::VarHandleOp(scope.WithOpName("Variable"), DT_INT32, - TensorShape({})); - - auto zero = ops::Const(scope.WithOpName("outer/Const"), 0); - - auto while_op = ops::While(scope.WithOpName("outer/LoopCond"), - std::initializer_list{zero, y, x, var}, - outer_cond_fn, outer_body_fn); - auto sink = ops::Identity(scope.WithOpName("sink"), while_op[0]); - GraphDef expected; - TF_EXPECT_OK(scope.ToGraphDef(&expected)); - TF_EXPECT_GRAPH_EQ(expected, graph_def); - } - - // Outer condition graph. - { - Scope scope = Scope::NewRootScope().ExitOnError(); - auto arg0 = ops::_Arg(scope.WithOpName("_arg0"), DT_INT32, 0); - auto arg1 = ops::_Arg(scope.WithOpName("_arg1"), DT_INT32, 1); - auto arg2 = ops::_Arg(scope.WithOpName("_arg2"), DT_INT32, 2); - auto arg3 = ops::_Arg(scope.WithOpName("_arg3"), DT_RESOURCE, 3); - - auto ten = ops::Const( - scope.WithOpName("outer/Less/y").WithControlDependencies(arg0.output), - 10); - auto less = ops::Less(scope.WithOpName("outer/Less_i"), arg0, ten); - auto retval = ops::_Retval(scope.WithOpName("_retval0_RetVal"), less, 0); - - GraphDef expected; - TF_EXPECT_OK(scope.ToGraphDef(&expected)); - - InstantiationResultForTest result; + for (const GraphDef& graph_def : {optimized_graph_def, converted_graph_def}) { + NameAttrList outer_cond_fn, outer_body_fn; TF_EXPECT_OK( - InstantiateFunctionForTest(outer_cond_fn.name(), library, &result)); + FindWhileCondAndBody(graph_def, &outer_cond_fn, &outer_body_fn)); - EXPECT_EQ((DataTypeVector{DT_INT32, DT_INT32, DT_INT32, DT_RESOURCE}), - result.arg_types); - EXPECT_EQ(DataTypeVector{DT_BOOL}, result.ret_types); - TF_EXPECT_GRAPH_EQ(expected, result.gdef); - } + // Outer graph. + { + Scope scope = Scope::NewRootScope().ExitOnError(); + auto x = ops::Placeholder(scope.WithOpName("x"), DT_INT32); + auto three = ops::Const(scope.WithOpName("three"), 3); + auto y = ops::Add(scope.WithOpName("y"), x, three); - // Outer body graph. - NameAttrList inner_cond_fn, inner_body_fn; - { - InstantiationResultForTest result; - TF_EXPECT_OK( - InstantiateFunctionForTest(outer_body_fn.name(), library, &result)); + auto var = ops::VarHandleOp(scope.WithOpName("Variable"), DT_INT32, + TensorShape({})); - // Find the inner condition and body names. - TF_EXPECT_OK( - FindWhileCondAndBody(result.gdef, &inner_cond_fn, &inner_body_fn)); + auto zero = ops::Const(scope.WithOpName("outer/Const"), 0); - Scope scope = Scope::NewRootScope().ExitOnError(); - auto arg0 = ops::_Arg(scope.WithOpName("_arg0"), DT_INT32, 0); - auto arg1 = ops::_Arg(scope.WithOpName("_arg1"), DT_INT32, 1); - auto arg2 = ops::_Arg(scope.WithOpName("_arg2"), DT_INT32, 2); - auto arg3 = ops::_Arg(scope.WithOpName("_arg3"), DT_RESOURCE, 3); + auto while_op = ops::While(scope.WithOpName("outer/LoopCond"), + std::initializer_list{zero, y, x, var}, + outer_cond_fn, outer_body_fn); + auto sink = ops::Identity(scope.WithOpName("sink"), while_op[0]); + GraphDef expected; + TF_EXPECT_OK(scope.ToGraphDef(&expected)); + TF_EXPECT_GRAPH_EQ(expected, graph_def); + } - auto identity_i = ops::Identity(scope.WithOpName("outer/Identity"), arg0); - auto one_j = ops::Const( - scope.WithOpName("outer/j").WithControlDependencies(identity_i), 1); - auto while_op = - ops::While(scope.WithOpName("outer/LoopCond_1"), - std::initializer_list{one_j, arg1, arg2, arg3}, - inner_cond_fn, inner_body_fn); + // Outer condition graph. + { + Scope scope = Scope::NewRootScope().ExitOnError(); + auto arg0 = ops::_Arg(scope.WithOpName("_arg0"), DT_INT32, 0); + auto arg1 = ops::_Arg(scope.WithOpName("_arg1"), DT_INT32, 1); + auto arg2 = ops::_Arg(scope.WithOpName("_arg2"), DT_INT32, 2); + auto arg3 = ops::_Arg(scope.WithOpName("_arg3"), DT_RESOURCE, 3); - auto one_outer = ops::Const( - scope.WithOpName("outer/add/y").WithControlDependencies(identity_i), 1); - auto add_i = - ops::Add(scope.WithOpName("outer/add") - .WithControlDependencies(absl::Span{ - while_op[0].op(), while_op[1].op()}), - identity_i, one_outer); + auto ten = ops::Const( + scope.WithOpName("outer/Less/y").WithControlDependencies(arg0.output), + 10); + auto less = ops::Less(scope.WithOpName("outer/Less_i"), arg0, ten); + auto retval = ops::_Retval(scope.WithOpName("_retval0_RetVal"), less, 0); - auto retval0 = ops::_Retval(scope.WithOpName("_retval0_RetVal"), add_i, 0); - auto retval1 = ops::_Retval(scope.WithOpName("_retval1_RetVal"), arg1, 1); - auto retval2 = ops::_Retval(scope.WithOpName("_retval2_RetVal"), arg2, 2); + GraphDef expected; + TF_EXPECT_OK(scope.ToGraphDef(&expected)); - GraphDef expected; - TF_EXPECT_OK(scope.ToGraphDef(&expected)); + InstantiationResultForTest result; + TF_EXPECT_OK( + InstantiateFunctionForTest(outer_cond_fn.name(), library, &result)); - EXPECT_EQ((DataTypeVector{DT_INT32, DT_INT32, DT_INT32, DT_RESOURCE}), - result.arg_types); - EXPECT_EQ((DataTypeVector{DT_INT32, DT_INT32, DT_INT32}), result.ret_types); - TF_EXPECT_GRAPH_EQ(expected, result.gdef); - } + EXPECT_EQ((DataTypeVector{DT_INT32, DT_INT32, DT_INT32, DT_RESOURCE}), + result.arg_types); + EXPECT_EQ(DataTypeVector{DT_BOOL}, result.ret_types); + TF_EXPECT_GRAPH_EQ(expected, result.gdef); + } - // Inner condition graph. - { - Scope scope = Scope::NewRootScope().ExitOnError(); - auto arg0 = ops::_Arg(scope.WithOpName("_arg0"), DT_INT32, 0); - auto arg1 = ops::_Arg(scope.WithOpName("_arg1"), DT_INT32, 1); - auto arg2 = ops::_Arg(scope.WithOpName("_arg2"), DT_INT32, 2); - auto arg3 = ops::_Arg(scope.WithOpName("_arg3"), DT_RESOURCE, 3); + // Outer body graph. + NameAttrList inner_cond_fn, inner_body_fn; + { + InstantiationResultForTest result; + TF_EXPECT_OK( + InstantiateFunctionForTest(outer_body_fn.name(), library, &result)); - auto five = ops::Const( - scope.WithOpName("outer/inner/Five").WithControlDependencies(arg0), 5); - auto less_j = ops::Less(scope.WithOpName("outer/inner/Less_j"), arg0, five); - auto retval = ops::_Retval(scope.WithOpName("_retval0_RetVal"), less_j, 0); + // Find the inner condition and body names. + TF_EXPECT_OK( + FindWhileCondAndBody(result.gdef, &inner_cond_fn, &inner_body_fn)); - GraphDef expected; - TF_EXPECT_OK(scope.ToGraphDef(&expected)); + Scope scope = Scope::NewRootScope().ExitOnError(); + auto arg0 = ops::_Arg(scope.WithOpName("_arg0"), DT_INT32, 0); + auto arg1 = ops::_Arg(scope.WithOpName("_arg1"), DT_INT32, 1); + auto arg2 = ops::_Arg(scope.WithOpName("_arg2"), DT_INT32, 2); + auto arg3 = ops::_Arg(scope.WithOpName("_arg3"), DT_RESOURCE, 3); - InstantiationResultForTest result; - TF_EXPECT_OK( - InstantiateFunctionForTest(inner_cond_fn.name(), library, &result)); + auto identity_i = ops::Identity(scope.WithOpName("outer/Identity"), arg0); + auto one_j = ops::Const( + scope.WithOpName("outer/j").WithControlDependencies(identity_i), 1); + auto while_op = + ops::While(scope.WithOpName("outer/LoopCond_1"), + std::initializer_list{one_j, arg1, arg2, arg3}, + inner_cond_fn, inner_body_fn); - EXPECT_EQ((DataTypeVector{DT_INT32, DT_INT32, DT_INT32, DT_RESOURCE}), - result.arg_types); - EXPECT_EQ(DataTypeVector{DT_BOOL}, result.ret_types); - TF_EXPECT_GRAPH_EQ(expected, result.gdef); - } + auto one_outer = ops::Const( + scope.WithOpName("outer/add/y").WithControlDependencies(identity_i), + 1); + auto add_i = + ops::Add(scope.WithOpName("outer/add") + .WithControlDependencies(absl::Span{ + while_op[0].op(), while_op[1].op()}), + identity_i, one_outer); - // Inner body graph. - { - Scope scope = Scope::NewRootScope().ExitOnError(); - auto arg0 = ops::_Arg(scope.WithOpName("_arg0"), DT_INT32, 0); - auto arg1 = ops::_Arg(scope.WithOpName("_arg1"), DT_INT32, 1); - auto arg2 = ops::_Arg(scope.WithOpName("_arg2"), DT_INT32, 2); - auto arg3 = ops::_Arg(scope.WithOpName("_arg3"), DT_RESOURCE, 3); + auto retval0 = + ops::_Retval(scope.WithOpName("_retval0_RetVal"), add_i, 0); + auto retval1 = ops::_Retval(scope.WithOpName("_retval1_RetVal"), arg1, 1); + auto retval2 = ops::_Retval(scope.WithOpName("_retval2_RetVal"), arg2, 2); - auto identity_j = - ops::Identity(scope.WithOpName("outer/inner/Identity_j"), arg0); - auto identity_k = - ops::Identity(scope.WithOpName("outer/inner/Identity_k"), arg1); + GraphDef expected; + TF_EXPECT_OK(scope.ToGraphDef(&expected)); - auto mul_jk = - ops::Mul(scope.WithOpName("outer/inner/mul"), identity_j, identity_k); - auto add_jkx = ops::Add(scope.WithOpName("outer/inner/add"), mul_jk, arg2); - auto assign = ops::AssignAddVariableOp( - scope.WithOpName("outer/inner/assign_add"), arg3, add_jkx); + EXPECT_EQ((DataTypeVector{DT_INT32, DT_INT32, DT_INT32, DT_RESOURCE}), + result.arg_types); + EXPECT_EQ((DataTypeVector{DT_INT32, DT_INT32, DT_INT32}), + result.ret_types); + TF_EXPECT_GRAPH_EQ(expected, result.gdef); + } - auto one = ops::Const( - scope.WithOpName("outer/inner/One") - .WithControlDependencies( - absl::Span{assign.operation}), - 1); - auto add_j = - ops::Add(scope.WithOpName("outer/inner/add_j"), identity_j, one); + // Inner condition graph. + { + Scope scope = Scope::NewRootScope().ExitOnError(); + auto arg0 = ops::_Arg(scope.WithOpName("_arg0"), DT_INT32, 0); + auto arg1 = ops::_Arg(scope.WithOpName("_arg1"), DT_INT32, 1); + auto arg2 = ops::_Arg(scope.WithOpName("_arg2"), DT_INT32, 2); + auto arg3 = ops::_Arg(scope.WithOpName("_arg3"), DT_RESOURCE, 3); - auto retval0 = ops::_Retval(scope.WithOpName("_retval0_RetVal"), add_j, 0); - auto retval1 = - ops::_Retval(scope.WithOpName("_retval1_RetVal"), identity_k, 1); - auto retval2 = ops::_Retval(scope.WithOpName("_retval2_RetVal"), arg2, 2); + auto five = ops::Const( + scope.WithOpName("outer/inner/Five").WithControlDependencies(arg0), + 5); + auto less_j = + ops::Less(scope.WithOpName("outer/inner/Less_j"), arg0, five); + auto retval = + ops::_Retval(scope.WithOpName("_retval0_RetVal"), less_j, 0); - GraphDef expected; - TF_EXPECT_OK(scope.ToGraphDef(&expected)); + GraphDef expected; + TF_EXPECT_OK(scope.ToGraphDef(&expected)); - InstantiationResultForTest result; - TF_EXPECT_OK( - InstantiateFunctionForTest(inner_body_fn.name(), library, &result)); + InstantiationResultForTest result; + TF_EXPECT_OK( + InstantiateFunctionForTest(inner_cond_fn.name(), library, &result)); - EXPECT_EQ((DataTypeVector{DT_INT32, DT_INT32, DT_INT32, DT_RESOURCE}), - result.arg_types); - EXPECT_EQ((DataTypeVector{DT_INT32, DT_INT32, DT_INT32}), result.ret_types); - TF_EXPECT_GRAPH_EQ(expected, result.gdef); + EXPECT_EQ((DataTypeVector{DT_INT32, DT_INT32, DT_INT32, DT_RESOURCE}), + result.arg_types); + EXPECT_EQ(DataTypeVector{DT_BOOL}, result.ret_types); + TF_EXPECT_GRAPH_EQ(expected, result.gdef); + } + + // Inner body graph. + { + Scope scope = Scope::NewRootScope().ExitOnError(); + auto arg0 = ops::_Arg(scope.WithOpName("_arg0"), DT_INT32, 0); + auto arg1 = ops::_Arg(scope.WithOpName("_arg1"), DT_INT32, 1); + auto arg2 = ops::_Arg(scope.WithOpName("_arg2"), DT_INT32, 2); + auto arg3 = ops::_Arg(scope.WithOpName("_arg3"), DT_RESOURCE, 3); + + auto identity_j = + ops::Identity(scope.WithOpName("outer/inner/Identity_j"), arg0); + auto identity_k = + ops::Identity(scope.WithOpName("outer/inner/Identity_k"), arg1); + + auto mul_jk = + ops::Mul(scope.WithOpName("outer/inner/mul"), identity_j, identity_k); + auto add_jkx = + ops::Add(scope.WithOpName("outer/inner/add"), mul_jk, arg2); + auto assign = ops::AssignAddVariableOp( + scope.WithOpName("outer/inner/assign_add"), arg3, add_jkx); + + auto one = ops::Const( + scope.WithOpName("outer/inner/One") + .WithControlDependencies( + absl::Span{assign.operation}), + 1); + auto add_j = + ops::Add(scope.WithOpName("outer/inner/add_j"), identity_j, one); + + auto retval0 = + ops::_Retval(scope.WithOpName("_retval0_RetVal"), add_j, 0); + auto retval1 = + ops::_Retval(scope.WithOpName("_retval1_RetVal"), identity_k, 1); + auto retval2 = ops::_Retval(scope.WithOpName("_retval2_RetVal"), arg2, 2); + + GraphDef expected; + TF_EXPECT_OK(scope.ToGraphDef(&expected)); + + InstantiationResultForTest result; + TF_EXPECT_OK( + InstantiateFunctionForTest(inner_body_fn.name(), library, &result)); + + EXPECT_EQ((DataTypeVector{DT_INT32, DT_INT32, DT_INT32, DT_RESOURCE}), + result.arg_types); + EXPECT_EQ((DataTypeVector{DT_INT32, DT_INT32, DT_INT32}), + result.ret_types); + TF_EXPECT_GRAPH_EQ(expected, result.gdef); + } } } diff --git a/tensorflow/compiler/tf2xla/kernels/BUILD b/tensorflow/compiler/tf2xla/kernels/BUILD index d85b4f5ae0c..fa51a72aea4 100644 --- a/tensorflow/compiler/tf2xla/kernels/BUILD +++ b/tensorflow/compiler/tf2xla/kernels/BUILD @@ -121,7 +121,6 @@ tf_kernel_library( ":while_op", "//tensorflow/compiler/tf2xla:common", "//tensorflow/compiler/tf2xla:xla_compiler", - "//tensorflow/compiler/tf2xla/lib:batch_dot", "//tensorflow/compiler/tf2xla/lib:broadcast", "//tensorflow/compiler/tf2xla/lib:cholesky", "//tensorflow/compiler/tf2xla/lib:qr", @@ -144,7 +143,7 @@ tf_kernel_library( "//tensorflow/compiler/xla/client/lib:arithmetic", "//tensorflow/compiler/xla/client/lib:constants", "//tensorflow/compiler/xla/client/lib:math", - "//tensorflow/compiler/xla/client/lib:numeric", + "//tensorflow/compiler/xla/client/lib:matrix", "//tensorflow/compiler/xla/client/lib:pooling", "//tensorflow/compiler/xla/client/lib:prng", "//tensorflow/compiler/xla/client/lib:sorting", @@ -196,7 +195,6 @@ cc_library( "//tensorflow/compiler/xla/client:xla_builder", "//tensorflow/compiler/xla/client/lib:arithmetic", "//tensorflow/compiler/xla/client/lib:constants", - "//tensorflow/compiler/xla/client/lib:numeric", "//tensorflow/core:framework", "//tensorflow/core:lib", "//tensorflow/core:protos_all_cc", @@ -216,7 +214,6 @@ cc_library( "//tensorflow/compiler/xla/client:xla_builder", "//tensorflow/compiler/xla/client/lib:arithmetic", "//tensorflow/compiler/xla/client/lib:constants", - "//tensorflow/compiler/xla/client/lib:numeric", "//tensorflow/core:framework", "//tensorflow/core/kernels:bounds_check", "//tensorflow/core/kernels:conv_ops", diff --git a/tensorflow/compiler/tf2xla/kernels/arg_op.cc b/tensorflow/compiler/tf2xla/kernels/arg_op.cc index 2db2514397d..795ea09831e 100644 --- a/tensorflow/compiler/tf2xla/kernels/arg_op.cc +++ b/tensorflow/compiler/tf2xla/kernels/arg_op.cc @@ -50,7 +50,7 @@ class XlaArgOp : public XlaOpKernel { return; } - const XlaExpression& arg = XlaContext::Get(ctx).args()[index_]; + const XlaExpression& arg = ctx->xla_context()->args()[index_]; OP_REQUIRES(ctx, arg.kind() != XlaExpression::Kind::kInvalid, errors::InvalidArgument("Invalid/missing argument expression")); ctx->SetOutputExpression(0, arg); diff --git a/tensorflow/compiler/tf2xla/kernels/batch_matmul_op.cc b/tensorflow/compiler/tf2xla/kernels/batch_matmul_op.cc index 4cfe946b2e6..1b254e328a8 100644 --- a/tensorflow/compiler/tf2xla/kernels/batch_matmul_op.cc +++ b/tensorflow/compiler/tf2xla/kernels/batch_matmul_op.cc @@ -13,9 +13,11 @@ See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ -#include "tensorflow/compiler/tf2xla/lib/batch_dot.h" +#include "tensorflow/compiler/tf2xla/lib/util.h" #include "tensorflow/compiler/tf2xla/xla_op_kernel.h" #include "tensorflow/compiler/tf2xla/xla_op_registry.h" +#include "tensorflow/compiler/xla/client/lib/math.h" +#include "tensorflow/compiler/xla/client/lib/matrix.h" namespace tensorflow { namespace { @@ -28,9 +30,11 @@ class BatchMatMulOp : public XlaOpKernel { } void Compile(XlaOpKernelContext* ctx) override { - auto result = BatchDot(ctx->Input(0), ctx->Input(1), - /*transpose_x=*/adj_x_, /*transpose_y=*/adj_y_, - /*conjugate_x=*/adj_x_, /*conjugate_y=*/adj_y_); + auto result = + xla::BatchDot(MaybeTransposeInMinorDims( + MaybeConjugate(ctx->Input(0), adj_x_), adj_x_), + MaybeTransposeInMinorDims( + MaybeConjugate(ctx->Input(1), adj_y_), adj_y_)); ctx->SetOutput(0, result); } diff --git a/tensorflow/compiler/tf2xla/kernels/batch_norm_op.cc b/tensorflow/compiler/tf2xla/kernels/batch_norm_op.cc index a267c0c72fc..0e2f335f335 100644 --- a/tensorflow/compiler/tf2xla/kernels/batch_norm_op.cc +++ b/tensorflow/compiler/tf2xla/kernels/batch_norm_op.cc @@ -115,9 +115,9 @@ class FusedBatchNormGradOp : public XlaOpKernel { // operators. For now, cast everything to the statistics type (which // may be more precise than the input type). auto grad_backprop = - XlaHelpers::ConvertElementType(b, ctx->Input(0), scale_dtype); + XlaHelpers::ConvertElementType(ctx->Input(0), scale_dtype); auto activations = - XlaHelpers::ConvertElementType(b, ctx->Input(1), scale_dtype); + XlaHelpers::ConvertElementType(ctx->Input(1), scale_dtype); auto scale = ctx->Input(2); auto mean = ctx->Input(3); auto var = ctx->Input(4); @@ -151,11 +151,11 @@ class FusedBatchNormGradOp : public XlaOpKernel { const DataType accumulation_type = XlaHelpers::SumAccumulationType(scale_dtype); auto converted = - XlaHelpers::ConvertElementType(b, grad_backprop, accumulation_type); + XlaHelpers::ConvertElementType(grad_backprop, accumulation_type); auto reduce = xla::Reduce(converted, XlaHelpers::Zero(b, accumulation_type), *ctx->GetOrCreateAdd(accumulation_type), reduction_dims); - offset_backprop = XlaHelpers::ConvertElementType(b, reduce, scale_dtype); + offset_backprop = XlaHelpers::ConvertElementType(reduce, scale_dtype); // scratch1 = rsqrt(pop_var + epsilon) auto neg_half = XlaHelpers::FloatLiteral(b, scale_dtype, -0.5); @@ -165,19 +165,18 @@ class FusedBatchNormGradOp : public XlaOpKernel { // scratch2 = sum(y_backprop * (x - mean)) auto mul = xla::Mul(grad_backprop, xla::Sub(activations, mean, {feature_index})); - converted = XlaHelpers::ConvertElementType(b, mul, accumulation_type); + converted = XlaHelpers::ConvertElementType(mul, accumulation_type); reduce = xla::Reduce(converted, XlaHelpers::Zero(b, accumulation_type), *ctx->GetOrCreateAdd(accumulation_type), reduction_dims); - auto scratch2 = XlaHelpers::ConvertElementType(b, reduce, scale_dtype); + auto scratch2 = XlaHelpers::ConvertElementType(reduce, scale_dtype); x_backprop = xla::Mul(grad_backprop, xla::Mul(scratch1, scale), {feature_index}); scale_backprop = xla::Mul(scratch1, scratch2); } - ctx->SetOutput(0, - XlaHelpers::ConvertElementType(b, x_backprop, input_dtype)); + ctx->SetOutput(0, XlaHelpers::ConvertElementType(x_backprop, input_dtype)); ctx->SetOutput(1, scale_backprop); ctx->SetOutput(2, offset_backprop); ctx->SetConstantOutput(3, Tensor()); diff --git a/tensorflow/compiler/tf2xla/kernels/bias_ops.cc b/tensorflow/compiler/tf2xla/kernels/bias_ops.cc index 41f540506ba..e7f369b761f 100644 --- a/tensorflow/compiler/tf2xla/kernels/bias_ops.cc +++ b/tensorflow/compiler/tf2xla/kernels/bias_ops.cc @@ -107,11 +107,11 @@ class BiasAddGradOp : public XlaOpKernel { const DataType accumulation_type = XlaHelpers::SumAccumulationType(input_type(0)); auto converted = - XlaHelpers::ConvertElementType(b, ctx->Input(0), accumulation_type); + XlaHelpers::ConvertElementType(ctx->Input(0), accumulation_type); auto reduce = xla::Reduce(converted, XlaHelpers::Zero(b, accumulation_type), *ctx->GetOrCreateAdd(accumulation_type), reduce_dims); - ctx->SetOutput(0, XlaHelpers::ConvertElementType(b, reduce, input_type(0))); + ctx->SetOutput(0, XlaHelpers::ConvertElementType(reduce, input_type(0))); } private: diff --git a/tensorflow/compiler/tf2xla/kernels/binary_ops.cc b/tensorflow/compiler/tf2xla/kernels/binary_ops.cc index 47e517a6576..5e9280c1fe6 100644 --- a/tensorflow/compiler/tf2xla/kernels/binary_ops.cc +++ b/tensorflow/compiler/tf2xla/kernels/binary_ops.cc @@ -43,6 +43,9 @@ namespace { const std::vector& extend_dimensions) override { \ xla::XlaBuilder* b = ctx->builder(); \ (void)b; \ + (void)lhs_shape; \ + (void)rhs_shape; \ + (void)extend_dimensions; \ return HLO; \ } \ }; \ @@ -103,23 +106,23 @@ static xla::XlaOp FloorDivImpl(xla::XlaBuilder* b, DataType dtype, xla::XlaOp x, XLA_MAKE_BINARY(FloorDiv, FloorDivImpl(b, input_type(0), lhs, rhs, broadcast_helper)); -static xla::XlaOp XlogyImpl(xla::XlaBuilder* b, DataType dtype, xla::XlaOp x, - xla::XlaOp y, const BCast& broadcast_helper) { +xla::XlaOp XlogyImpl(xla::XlaOp x, xla::XlaOp y, + const BCast& broadcast_helper) { std::tie(x, y) = XlaBinaryOp::Broadcast(x, y, broadcast_helper); - auto zero = XlaHelpers::Zero(b, dtype); + auto zero = xla::ZerosLike(x); auto is_zero = xla::Eq(x, zero); return xla::Select(is_zero, zero, xla::Mul(x, xla::Log(y))); } -XLA_MAKE_BINARY(Xlogy, XlogyImpl(b, input_type(0), lhs, rhs, broadcast_helper)); +XLA_MAKE_BINARY(Xlogy, XlogyImpl(lhs, rhs, broadcast_helper)); -static xla::XlaOp XdivyImpl(xla::XlaBuilder* b, DataType dtype, xla::XlaOp x, - xla::XlaOp y, const BCast& broadcast_helper) { +xla::XlaOp XdivyImpl(xla::XlaOp x, xla::XlaOp y, + const BCast& broadcast_helper) { std::tie(x, y) = XlaBinaryOp::Broadcast(x, y, broadcast_helper); - auto zero = XlaHelpers::Zero(b, dtype); + auto zero = xla::ZerosLike(x); auto is_zero = xla::Eq(x, zero); return xla::Select(is_zero, zero, xla::Div(x, y)); } -XLA_MAKE_BINARY(Xdivy, XdivyImpl(b, input_type(0), lhs, rhs, broadcast_helper)); +XLA_MAKE_BINARY(Xdivy, XdivyImpl(lhs, rhs, broadcast_helper)); // Implementation of FloorMod. Pseudo-code: // T trunc_mod = std::fmod(x, y); diff --git a/tensorflow/compiler/tf2xla/kernels/categorical_op.cc b/tensorflow/compiler/tf2xla/kernels/categorical_op.cc index ad85940920e..7199b9b6feb 100644 --- a/tensorflow/compiler/tf2xla/kernels/categorical_op.cc +++ b/tensorflow/compiler/tf2xla/kernels/categorical_op.cc @@ -21,10 +21,13 @@ limitations under the License. #include "tensorflow/compiler/tf2xla/xla_op_kernel.h" #include "tensorflow/compiler/tf2xla/xla_op_registry.h" #include "tensorflow/compiler/xla/client/lib/arithmetic.h" +#include "tensorflow/compiler/xla/client/lib/prng.h" #include "tensorflow/compiler/xla/client/xla_builder.h" +#include "tensorflow/compiler/xla/xla_data.pb.h" #include "tensorflow/core/framework/op_kernel.h" #include "tensorflow/core/framework/tensor.h" #include "tensorflow/core/framework/tensor_shape.h" +#include "tensorflow/core/framework/types.pb.h" namespace tensorflow { namespace { @@ -57,11 +60,9 @@ class CategoricalOp : public XlaOpKernel { const int64 batch_size = logits_shape.dim_size(0); const int64 num_classes = logits_shape.dim_size(1); - xla::XlaBuilder* builder = ctx->builder(); - xla::Shape uniform_shape; int class_dimension; - if (num_samples > 1) { + if (num_samples != 1) { std::array uniform_shape_array = { {batch_size, num_samples, num_classes}}; xla::PrimitiveType uniform_xla_type; @@ -83,16 +84,16 @@ class CategoricalOp : public XlaOpKernel { xla::ShapeUtil::MakeShape(uniform_xla_type, uniform_shape_array); class_dimension = 1; } - xla::XlaOp uniforms = - xla::RngUniform(XlaHelpers::Zero(builder, input_type(0)), - XlaHelpers::One(builder, input_type(0)), uniform_shape); + xla::PrimitiveType type; + OP_REQUIRES_OK(ctx, DataTypeToPrimitiveType(input_type(0), &type)); + xla::XlaOp log_uniforms = GetLogUniforms(uniform_shape, type, ctx); // Use Gumbel softmax trick to generate categorical samples. // See: // https://hips.seas.harvard.edu/blog/2013/04/06/the-gumbel-max-trick-for-discrete-distributions/ // TODO(b/68769470): Switch to using a cumulative sum approach. auto softmax_entries = - xla::Sub(logits, xla::Log(-xla::Log(uniforms)), + xla::Sub(logits, log_uniforms, /*broadcast_dimensions=*/{0, class_dimension}); xla::PrimitiveType xla_output_type; @@ -107,6 +108,16 @@ class CategoricalOp : public XlaOpKernel { ctx->SetOutput(0, argmax); } + virtual xla::XlaOp GetLogUniforms(xla::Shape uniform_shape, + xla::PrimitiveType type, + XlaOpKernelContext* ctx) { + xla::XlaBuilder* builder = ctx->builder(); + auto uniforms = + xla::RngUniform(XlaHelpers::Zero(builder, input_type(0)), + XlaHelpers::One(builder, input_type(0)), uniform_shape); + return xla::Log(-xla::Log(uniforms)); + } + private: TF_DISALLOW_COPY_AND_ASSIGN(CategoricalOp); }; @@ -115,5 +126,48 @@ class CategoricalOp : public XlaOpKernel { REGISTER_XLA_OP(Name("Multinomial").CompileTimeConstantInput("num_samples"), CategoricalOp); +class StatelessCategoricalOp : public CategoricalOp { + public: + explicit StatelessCategoricalOp(OpKernelConstruction* ctx) + : CategoricalOp(ctx) { + OP_REQUIRES_OK(ctx, ctx->GetAttr("T", &dtype_)); + } + + xla::XlaOp GetLogUniforms(xla::Shape uniform_shape, xla::PrimitiveType type, + XlaOpKernelContext* ctx) override { + xla::XlaOp seed = ctx->Input(2); + auto seed0 = xla::Reshape(xla::Slice(seed, {0}, {1}, {1}), {}); + auto seed1 = xla::Reshape(xla::Slice(seed, {1}, {2}, {1}), {}); + + xla::XlaBuilder* builder = ctx->builder(); + if (uniform_shape.element_type() == xla::BF16) { + uniform_shape.set_element_type(xla::F32); + } + auto uniforms = xla::StatelessRngUniform( + {seed0, seed1}, uniform_shape, XlaHelpers::Zero(builder, DT_FLOAT), + XlaHelpers::One(builder, DT_FLOAT)); + return xla::ConvertElementType(xla::Log(-xla::Log(uniforms)), type); + } + + void Compile(XlaOpKernelContext* ctx) override { + TensorShape seed_shape = ctx->InputShape(2); + OP_REQUIRES(ctx, seed_shape.dims() == 1 && seed_shape.dim_size(0) == 2, + errors::InvalidArgument("seed must have shape [2], not ", + seed_shape.DebugString())); + CategoricalOp::Compile(ctx); + } + + private: + DataType dtype_; + + TF_DISALLOW_COPY_AND_ASSIGN(StatelessCategoricalOp); +}; + +REGISTER_XLA_OP(Name("StatelessMultinomial") + .CompileTimeConstantInput("num_samples") + .TypeConstraint("T", {DT_FLOAT, DT_BFLOAT16}) + .TypeConstraint("Tseed", DT_INT32), + StatelessCategoricalOp); + } // anonymous namespace } // namespace tensorflow diff --git a/tensorflow/compiler/tf2xla/kernels/conv_op_helpers.cc b/tensorflow/compiler/tf2xla/kernels/conv_op_helpers.cc index c9a1be49406..641fefafb35 100644 --- a/tensorflow/compiler/tf2xla/kernels/conv_op_helpers.cc +++ b/tensorflow/compiler/tf2xla/kernels/conv_op_helpers.cc @@ -24,7 +24,6 @@ limitations under the License. #include "tensorflow/compiler/tf2xla/xla_op_registry.h" #include "tensorflow/compiler/xla/client/lib/arithmetic.h" #include "tensorflow/compiler/xla/client/lib/constants.h" -#include "tensorflow/compiler/xla/client/lib/numeric.h" #include "tensorflow/compiler/xla/client/xla_builder.h" #include "tensorflow/compiler/xla/literal_util.h" #include "tensorflow/core/framework/node_def_util.h" @@ -65,60 +64,63 @@ xla::Shape ExpandedFilterShapeForDepthwiseConvolution(const xla::Shape& shape) { // 0 0 1 1 0 0 0 0 1 1 0 0 // 0 0 0 0 1 1 0 0 0 0 1 1 // -// The first step is to create a one tensor, A, that is [3] -// 0 1 2 +// The first step is to create a iota A with iota_dimension = 2 +// 0 0 0 0 0 0 0 0 0 0 0 0 +// 1 1 1 1 1 1 1 1 1 1 1 1 +// 2 2 2 2 2 2 2 2 2 2 2 2 // -// and another tensor, B, that is [3 * 2] -// 0 1 2 3 4 5 +// 0 0 0 0 0 0 0 0 0 0 0 0 +// 1 1 1 1 1 1 1 1 1 1 1 1 +// 2 2 2 2 2 2 2 2 2 2 2 2 // -// and divide B it by 2 to get -// 0 0 1 1 2 2 +// and another iota B with iota_dimension = 3 +// 0 1 2 3 4 5 0 1 2 3 4 5 +// 0 1 2 3 4 5 0 1 2 3 4 5 +// 0 1 2 3 4 5 0 1 2 3 4 5 // -// then we broadcast the B to [2, 2, 3, 3 * 2] -// 0 0 1 1 2 2 0 0 1 1 2 2 -// 0 0 1 1 2 2 0 0 1 1 2 2 -// 0 0 1 1 2 2 0 0 1 1 2 2 +// 0 1 2 3 4 5 0 1 2 3 4 5 +// 0 1 2 3 4 5 0 1 2 3 4 5 +// 0 1 2 3 4 5 0 1 2 3 4 5 // -// 0 0 1 1 2 2 0 0 1 1 2 2 -// 0 0 1 1 2 2 0 0 1 1 2 2 -// 0 0 1 1 2 2 0 0 1 1 2 2 +// and divide B by 2 to get +// 0 0 1 1 2 2 0 0 1 1 2 2 +// 0 0 1 1 2 2 0 0 1 1 2 2 +// 0 0 1 1 2 2 0 0 1 1 2 2 // -// Finally compare A and broadcasted B in dimension 2 amd return the result at -// the beginning of the comment. +// 0 0 1 1 2 2 0 0 1 1 2 2 +// 0 0 1 1 2 2 0 0 1 1 2 2 +// 0 0 1 1 2 2 0 0 1 1 2 2 +// +// Finally compare A and B and return the result at the beginning of the +// comment. xla::XlaOp CreateExpandedFilterMask(const xla::Shape& filter_shape, xla::XlaBuilder* builder) { xla::Shape expanded_filter_shape = ExpandedFilterShapeForDepthwiseConvolution(filter_shape); int64 depthwise_multiplier = filter_shape.dimensions(filter_shape.dimensions_size() - 1); - int64 input_feature = - filter_shape.dimensions(filter_shape.dimensions_size() - 2); - // Create a M sized linspace and an M*N sized linspace that will be - // broadcasted into perpendicular dimensions and compared. - xla::XlaOp input_feature_iota = xla::Iota(builder, xla::S32, input_feature); - xla::XlaOp expanded_feature_iota = - xla::Iota(builder, xla::S32, input_feature * depthwise_multiplier); + // Create two iotas with the shape of the expanded filter, one of them with + // the iota dimension chosen as the feature dimension, and the other a iota + // with the iota dimension chosen as the expanded output feature dimension. + std::vector iota_dimensions(expanded_filter_shape.dimensions().begin(), + expanded_filter_shape.dimensions().end()); + xla::Shape iota_shape = xla::ShapeUtil::MakeShape(xla::S32, iota_dimensions); + xla::XlaOp input_feature_iota = xla::Iota( + builder, iota_shape, /*iota_dimension=*/iota_dimensions.size() - 2); + xla::XlaOp expanded_feature_iota = xla::Iota( + builder, iota_shape, /*iota_dimension=*/iota_dimensions.size() - 1); - // Divide the M*N sized linspace by the depthwise_multiplier to create - // [0 0 1 1 2 2] in the example in the function comment. + // Divide 'expanded_feature_iota' by the depthwise_multiplier to create + // [0 0 1 1 2 2] ... in the example in the function comment. expanded_feature_iota = xla::Div(expanded_feature_iota, XlaHelpers::IntegerLiteral(builder, DataType::DT_INT32, depthwise_multiplier)); - // Broadcast the N*M linspace to [H, W, ..., M, M*N]. - std::vector expanded_feature_broadcast_dims( - expanded_filter_shape.dimensions().begin(), - expanded_filter_shape.dimensions().end()); - expanded_feature_broadcast_dims.pop_back(); - auto broadcasted_expanded_feature_iota = - xla::Broadcast(expanded_feature_iota, expanded_feature_broadcast_dims); - - // Compare the broadcasted linspace to the input feature linspace in the - // input feature dimension to create a diagonal predicate. - return xla::Eq(broadcasted_expanded_feature_iota, input_feature_iota, - {expanded_filter_shape.dimensions_size() - 2}); + // Compare 'input_feature_iota' with 'expanded_feature_iota' to create a + // diagonal predicate. + return xla::Eq(expanded_feature_iota, input_feature_iota); } // Reshapes a filter of shape [H, W, ..., M, N] to [H, W, ..., 1, M*N]. Used to diff --git a/tensorflow/compiler/tf2xla/kernels/conv_ops.cc b/tensorflow/compiler/tf2xla/kernels/conv_ops.cc index d820528a430..eafdba876ae 100644 --- a/tensorflow/compiler/tf2xla/kernels/conv_ops.cc +++ b/tensorflow/compiler/tf2xla/kernels/conv_ops.cc @@ -22,7 +22,7 @@ limitations under the License. #include "tensorflow/compiler/tf2xla/xla_op_kernel.h" #include "tensorflow/compiler/tf2xla/xla_op_registry.h" #include "tensorflow/compiler/xla/client/lib/constants.h" -#include "tensorflow/compiler/xla/client/lib/numeric.h" +#include "tensorflow/compiler/xla/client/lib/matrix.h" #include "tensorflow/compiler/xla/client/xla_builder.h" #include "tensorflow/compiler/xla/literal_util.h" #include "tensorflow/core/framework/node_def_util.h" diff --git a/tensorflow/compiler/tf2xla/kernels/diag_op.cc b/tensorflow/compiler/tf2xla/kernels/diag_op.cc index 49c12fc2320..ee79cbc70da 100644 --- a/tensorflow/compiler/tf2xla/kernels/diag_op.cc +++ b/tensorflow/compiler/tf2xla/kernels/diag_op.cc @@ -19,7 +19,7 @@ limitations under the License. #include "tensorflow/compiler/tf2xla/xla_op_kernel.h" #include "tensorflow/compiler/tf2xla/xla_op_registry.h" #include "tensorflow/compiler/xla/client/lib/constants.h" -#include "tensorflow/compiler/xla/client/lib/numeric.h" +#include "tensorflow/compiler/xla/client/lib/matrix.h" #include "tensorflow/compiler/xla/client/xla_builder.h" #include "tensorflow/compiler/xla/util.h" #include "tensorflow/core/framework/op_kernel.h" diff --git a/tensorflow/compiler/tf2xla/kernels/dynamic_stitch_op.cc b/tensorflow/compiler/tf2xla/kernels/dynamic_stitch_op.cc index b2f6ef43fa9..6e6ba21daf5 100644 --- a/tensorflow/compiler/tf2xla/kernels/dynamic_stitch_op.cc +++ b/tensorflow/compiler/tf2xla/kernels/dynamic_stitch_op.cc @@ -113,8 +113,20 @@ class DynamicStitchOp : public XlaOpKernel { } } int number_of_indices = max_index + 1; - OP_REQUIRES(ctx, number_of_indices > 0, - errors::InvalidArgument("no indices supplied")); + int64 result_rank = 1 + data0_shape.dims() - indices0_shape.dims(); + if (number_of_indices == 0) { + std::vector result_shape(result_rank); + for (int d = indices0_shape.dims(); d < data0_shape.dims(); d++) { + result_shape[d - indices0_shape.dims() + 1] = data0_shape.dim_size(d); + } + xla::PrimitiveType element_type = + ctx->input_xla_type(ctx->num_inputs() - 1); + xla::Literal empty_literal = xla::Literal::CreateFromShape( + xla::ShapeUtil::MakeShape(element_type, result_shape)); + ctx->SetOutput(0, xla::ConstantLiteral(ctx->builder(), empty_literal)); + return; + } + // Construct the reverse mapping, for each index, of which slice of which // input it comes from. std::vector src_input_vector(number_of_indices); @@ -157,12 +169,9 @@ class DynamicStitchOp : public XlaOpKernel { // Set up the vectors for slicing: the first dimension will vary // slice by slice, and the rest take the full common extra shape. - std::vector slice_start(1 + data0_shape.dims() - - indices0_shape.dims()); - std::vector slice_limit(1 + data0_shape.dims() - - indices0_shape.dims()); - std::vector stride(1 + data0_shape.dims() - indices0_shape.dims(), - 1); + std::vector slice_start(result_rank); + std::vector slice_limit(result_rank); + std::vector stride(result_rank, 1); for (int d = indices0_shape.dims(); d < data0_shape.dims(); d++) { slice_limit[1 + d - indices0_shape.dims()] = data0_shape.dim_size(d); } diff --git a/tensorflow/compiler/tf2xla/kernels/extract_image_patches_op.cc b/tensorflow/compiler/tf2xla/kernels/extract_image_patches_op.cc index c68b0bfd796..29687c7b82f 100644 --- a/tensorflow/compiler/tf2xla/kernels/extract_image_patches_op.cc +++ b/tensorflow/compiler/tf2xla/kernels/extract_image_patches_op.cc @@ -17,7 +17,6 @@ limitations under the License. #include "tensorflow/compiler/tf2xla/xla_helpers.h" #include "tensorflow/compiler/tf2xla/xla_op_kernel.h" #include "tensorflow/compiler/tf2xla/xla_op_registry.h" -#include "tensorflow/compiler/xla/client/lib/numeric.h" #include "tensorflow/compiler/xla/client/xla_builder.h" #include "tensorflow/core/util/tensor_format.h" diff --git a/tensorflow/compiler/tf2xla/kernels/fake_quantize_ops.cc b/tensorflow/compiler/tf2xla/kernels/fake_quantize_ops.cc index cdba6680dee..142be030f73 100644 --- a/tensorflow/compiler/tf2xla/kernels/fake_quantize_ops.cc +++ b/tensorflow/compiler/tf2xla/kernels/fake_quantize_ops.cc @@ -260,19 +260,19 @@ class FakeQuantWithMinMaxVarsGradOp : public XlaOpKernel { xla::XlaOp below_min = xla::Lt(input, nudged_input_min); xla::XlaOp select1 = xla::Select(below_min, gradient, zeroes); xla::XlaOp reduce1 = xla::ReduceAll( - XlaHelpers::ConvertElementType(b, select1, accumulation_type), + XlaHelpers::ConvertElementType(select1, accumulation_type), XlaHelpers::Zero(b, accumulation_type), *ctx->GetOrCreateAdd(accumulation_type)); - xla::XlaOp output1 = XlaHelpers::ConvertElementType(b, reduce1, data_type); + xla::XlaOp output1 = XlaHelpers::ConvertElementType(reduce1, data_type); ctx->SetOutput(1, output1); xla::XlaOp above_max = xla::Gt(input, nudged_input_max); xla::XlaOp select2 = xla::Select(above_max, gradient, zeroes); xla::XlaOp reduce2 = xla::ReduceAll( - XlaHelpers::ConvertElementType(b, select2, accumulation_type), + XlaHelpers::ConvertElementType(select2, accumulation_type), XlaHelpers::Zero(b, accumulation_type), *ctx->GetOrCreateAdd(accumulation_type)); - xla::XlaOp output2 = XlaHelpers::ConvertElementType(b, reduce2, data_type); + xla::XlaOp output2 = XlaHelpers::ConvertElementType(reduce2, data_type); ctx->SetOutput(2, output2); } diff --git a/tensorflow/compiler/tf2xla/kernels/fft_ops.cc b/tensorflow/compiler/tf2xla/kernels/fft_ops.cc index 9b06357d9b7..6df8b5367d2 100644 --- a/tensorflow/compiler/tf2xla/kernels/fft_ops.cc +++ b/tensorflow/compiler/tf2xla/kernels/fft_ops.cc @@ -20,6 +20,7 @@ limitations under the License. #include "tensorflow/compiler/tf2xla/xla_op_registry.h" #include "tensorflow/compiler/xla/client/xla_builder.h" #include "tensorflow/compiler/xla/literal_util.h" +#include "tensorflow/compiler/xla/util.h" #include "tensorflow/core/framework/numeric_op.h" #include "tensorflow/core/framework/op_kernel.h" #include "tensorflow/core/framework/tensor.h" @@ -50,11 +51,36 @@ class GenericFftOp : public XlaOpKernel { errors::InvalidArgument("input must be at least 1 dimensional")); std::vector fft_length; + xla::XlaOp input = ctx->Input(0); if (fft_type_ == FftType::RFFT || fft_type_ == FftType::IRFFT) { OP_REQUIRES_OK(ctx, ctx->ConstantInputAsIntVector(1, &fft_length)); OP_REQUIRES(ctx, fft_length.size() == fft_rank_, errors::InvalidArgument("fft_length must be length ", fft_rank_, " vector")); + + // Zero pad or truncate the axes we're doing FFT on. + absl::InlinedVector slice_sizes = input_shape.dim_sizes(); + std::vector> padding_sizes(slice_sizes.size()); + std::vector expected_sizes = fft_length; + // IRFFT wants the innermost axis to be n / 2 + 1. + if (fft_type_ == FftType::IRFFT) { + expected_sizes[fft_rank_ - 1] = fft_length[fft_rank_ - 1] / 2 + 1; + } + for (int i = 0; i < fft_rank_; i++) { + int index = input_shape.dims() - fft_rank_ + i; + if (input_shape.dim_size(index) > expected_sizes[i]) { + slice_sizes[index] = expected_sizes[i]; + } else { + padding_sizes[index].second = + expected_sizes[i] - input_shape.dim_size(index); + } + } + + std::vector start_indices(input_shape.dims(), 0); + std::vector strides(input_shape.dims(), 1); + input = xla::Pad(xla::Slice(input, start_indices, slice_sizes, strides), + XlaHelpers::Zero(ctx->builder(), ctx->input_type(0)), + xla::MakeEdgePaddingConfig(padding_sizes)); } else { // Innermost axis provides the FFT length. for (int i = 0; i < fft_rank_; i++) { @@ -63,7 +89,7 @@ class GenericFftOp : public XlaOpKernel { } } - xla::XlaOp fft = xla::Fft(ctx->Input(0), fft_type_, fft_length); + xla::XlaOp fft = xla::Fft(input, fft_type_, fft_length); ctx->SetOutput(0, fft); } diff --git a/tensorflow/compiler/tf2xla/kernels/if_op.cc b/tensorflow/compiler/tf2xla/kernels/if_op.cc index 56da50f1408..b5e08391255 100644 --- a/tensorflow/compiler/tf2xla/kernels/if_op.cc +++ b/tensorflow/compiler/tf2xla/kernels/if_op.cc @@ -72,7 +72,7 @@ void XlaIfOp::Compile(XlaOpKernelContext* ctx) { arg.shape = resource->shape(); OP_REQUIRES(ctx, arg.initialized, errors::Unimplemented("Uninitialized arguments: ", arg.name)); - arg.tensor_array_size = resource->tensor_array_size(); + arg.max_array_size = resource->max_array_size(); for (const auto& gradient : resource->tensor_array_gradients()) { arg.tensor_array_gradients.insert(gradient.first); } diff --git a/tensorflow/compiler/tf2xla/kernels/image_ops.cc b/tensorflow/compiler/tf2xla/kernels/image_ops.cc index b49b2516d8b..e9bb0a77e99 100644 --- a/tensorflow/compiler/tf2xla/kernels/image_ops.cc +++ b/tensorflow/compiler/tf2xla/kernels/image_ops.cc @@ -191,12 +191,11 @@ class AdjustContrastOpV2 : public XlaOpKernel { DataType type = context->input_type(0); const DataType accumulation_type = XlaHelpers::SumAccumulationType(type); - auto converted = - XlaHelpers::ConvertElementType(b, input, accumulation_type); + auto converted = XlaHelpers::ConvertElementType(input, accumulation_type); auto reduce = xla::Reduce(converted, XlaHelpers::Zero(b, accumulation_type), *context->GetOrCreateAdd(accumulation_type), {height_dim, width_dim}); - auto output = XlaHelpers::ConvertElementType(b, reduce, type); + auto output = XlaHelpers::ConvertElementType(reduce, type); output = xla::Div(output, XlaHelpers::FloatLiteral(b, type, height * width)); diff --git a/tensorflow/compiler/tf2xla/kernels/image_resize_ops.cc b/tensorflow/compiler/tf2xla/kernels/image_resize_ops.cc index 0c7ca602bfa..5a10c52ba8b 100644 --- a/tensorflow/compiler/tf2xla/kernels/image_resize_ops.cc +++ b/tensorflow/compiler/tf2xla/kernels/image_resize_ops.cc @@ -19,7 +19,6 @@ limitations under the License. #include "tensorflow/compiler/tf2xla/xla_op_registry.h" #include "tensorflow/compiler/xla/array4d.h" #include "tensorflow/compiler/xla/client/lib/constants.h" -#include "tensorflow/compiler/xla/client/lib/numeric.h" #include "tensorflow/compiler/xla/client/xla_builder.h" #include "tensorflow/core/framework/kernel_def_builder.h" #include "tensorflow/core/framework/register_types.h" diff --git a/tensorflow/compiler/tf2xla/kernels/index_ops_cpu.cc b/tensorflow/compiler/tf2xla/kernels/index_ops_cpu.cc index e310db2162d..e2c05b648bb 100644 --- a/tensorflow/compiler/tf2xla/kernels/index_ops_cpu.cc +++ b/tensorflow/compiler/tf2xla/kernels/index_ops_cpu.cc @@ -30,7 +30,9 @@ limitations under the License. namespace tensorflow { namespace { -// The logic below uses a custom-call to implement argmax. +// The logic below uses a custom-call to implement argmax when possible. When +// custom-call is not allowed or input shapes are not supported, this kernel +// falls back to using XLA HLO native ArgMax. // // Also see b/29507024 for first-class XLA support for indexing ops. class ArgMaxCustomCallOp : public XlaOpKernel { @@ -50,27 +52,40 @@ class ArgMaxCustomCallOp : public XlaOpKernel { // overhead, when compiling ahead-of-time. int64 dim; OP_REQUIRES_OK(ctx, ctx->ConstantInputAsIntScalar(1, &dim)); - OP_REQUIRES(ctx, dim >= 0, errors::InvalidArgument("dim must be >= 0")); - OP_REQUIRES( - ctx, dim < input_shape.dims(), - errors::InvalidArgument("dim must be < input rank (", - input_shape.dims(), "), but got: ", dim)); - const int64 dim_size = input_shape.dim_size(dim); - OP_REQUIRES(ctx, dim_size > 0, + + const int input_dims = input_shape.dims(); + const int axis = dim < 0 ? dim + input_dims : dim; + OP_REQUIRES(ctx, axis >= 0 && axis < input_dims, + errors::InvalidArgument("Expected dimension in the range [", + -input_dims, ", ", input_dims, + "), but got ", dim)); + + const int64 axis_size = input_shape.dim_size(axis); + OP_REQUIRES(ctx, axis_size > 0, errors::InvalidArgument( "Reduction axis ", dim, " is empty in shape: ", input_shape.DebugString())); - // The output shape is the input shape contracted along dim. - TensorShape output_shape; - for (int d = 0; d < input_shape.dims() - 1; ++d) { - output_shape.AddDim(input_shape.dim_size((d < dim) ? d : d + 1)); + const DataType dtype = output_type(0); + xla::PrimitiveType output_type; + OP_REQUIRES_OK(ctx, DataTypeToPrimitiveType(dtype, &output_type)); + + // Fall back to XLA ArgMax HLO when CustomCall is not allowed or when input + // shape isn't supported. + if (!ctx->compiler()->options().allow_cpu_custom_calls || + (input_dims != 1 && input_dims != 2)) { + xla::XlaOp output = XlaHelpers::ArgMax(ctx->Input(0), output_type, axis); + ctx->SetOutput(0, output); + return; + } + + xla::XlaOp output; + // The output shape is the input shape contracted along axis. + TensorShape output_shape; + for (int d = 0; d < input_shape.dims() - 1; ++d) { + output_shape.AddDim(input_shape.dim_size((d < axis) ? d : d + 1)); } - // For now we use a custom-call, only for the 1d and 2d cases. - OP_REQUIRES(ctx, XlaContext::Get(ctx).allow_cpu_custom_calls(), - errors::InvalidArgument( - "ArgMax implementation requires a CustomCall on CPU")); xla::XlaBuilder& b = *ctx->builder(); // XLA passes to the function, so it is not included here. @@ -84,7 +99,7 @@ class ArgMaxCustomCallOp : public XlaOpKernel { args.push_back(xla::ConstantLiteral( &b, xla::LiteralUtil::CreateR1(output_shape.dim_sizes()))); args.push_back( - xla::ConstantLiteral(&b, xla::LiteralUtil::CreateR0(dim))); + xla::ConstantLiteral(&b, xla::LiteralUtil::CreateR0(axis))); } // The argmax function expects row-major layout. @@ -101,24 +116,15 @@ class ArgMaxCustomCallOp : public XlaOpKernel { } // Tell XLA to call the custom code, defined in - // index_ops_kernel_argmax_float_1d.cc. - xla::XlaOp output; - switch (input_shape.dims()) { - case 1: - output = xla::CustomCallWithLayout(&b, "argmax_float_1d_xla_impl", args, - xla_shape, arg_shapes); - break; - case 2: - output = xla::CustomCallWithLayout(&b, "argmax_float_2d_xla_impl", args, - xla_shape, arg_shapes); - break; - default: - OP_REQUIRES(ctx, false, - errors::Unimplemented( - "Argmax is only implemented for 1d and 2d tensors" - ", but got shape: ", - input_shape.DebugString())); + // index_ops_kernel_argmax_float_{1, 2}d.cc. + if (input_dims == 1) { + output = xla::CustomCallWithLayout(&b, "argmax_float_1d_xla_impl", args, + xla_shape, arg_shapes); + } else { + output = xla::CustomCallWithLayout(&b, "argmax_float_2d_xla_impl", args, + xla_shape, arg_shapes); } + output = xla::ConvertElementType(output, output_type); ctx->SetOutput(0, output); } diff --git a/tensorflow/compiler/tf2xla/kernels/l2loss_op.cc b/tensorflow/compiler/tf2xla/kernels/l2loss_op.cc index f028e361bcc..93f029731c3 100644 --- a/tensorflow/compiler/tf2xla/kernels/l2loss_op.cc +++ b/tensorflow/compiler/tf2xla/kernels/l2loss_op.cc @@ -37,12 +37,11 @@ class L2LossOp : public XlaOpKernel { // output = sum(t ** 2) / 2 const DataType accumulation_type = XlaHelpers::SumAccumulationType(dtype); - auto t = - XlaHelpers::ConvertElementType(b, ctx->Input(0), accumulation_type); + auto t = XlaHelpers::ConvertElementType(ctx->Input(0), accumulation_type); auto square = xla::Mul(t, t); auto reduce = xla::Reduce(square, XlaHelpers::Zero(b, accumulation_type), *ctx->GetOrCreateAdd(accumulation_type), dims); - auto deconverted = XlaHelpers::ConvertElementType(b, reduce, dtype); + auto deconverted = XlaHelpers::ConvertElementType(reduce, dtype); auto two = XlaHelpers::IntegerLiteral(b, dtype, 2); ctx->SetOutput(0, xla::Div(deconverted, two)); } diff --git a/tensorflow/compiler/tf2xla/kernels/lrn_ops.cc b/tensorflow/compiler/tf2xla/kernels/lrn_ops.cc index 87ee2d3aede..987901d82b3 100644 --- a/tensorflow/compiler/tf2xla/kernels/lrn_ops.cc +++ b/tensorflow/compiler/tf2xla/kernels/lrn_ops.cc @@ -49,16 +49,14 @@ class LRNOp : public XlaOpKernel { // We use a window of depth_radius_ * 2 + 1, to account for the current // element and a depth_radius_ on either side. auto accumulation_type = XlaHelpers::SumAccumulationType(input_type(0)); - auto converted = - XlaHelpers::ConvertElementType(builder, input, accumulation_type); + auto converted = XlaHelpers::ConvertElementType(input, accumulation_type); auto squared = xla::Mul(converted, converted); auto reduce = xla::ReduceWindow( squared, XlaHelpers::Zero(builder, accumulation_type), *ctx->GetOrCreateAdd(accumulation_type), /* window_dimensions = */ {1, 1, 1, depth_radius_ * 2 + 1}, /* window_strides = */ {1, 1, 1, 1}, xla::Padding::kSame); - auto sqr_sum = - XlaHelpers::ConvertElementType(builder, reduce, input_type(0)); + auto sqr_sum = XlaHelpers::ConvertElementType(reduce, input_type(0)); auto scale = xla::Pow( xla::Add(xla::ConstantR0(builder, bias_), @@ -138,15 +136,14 @@ class LRNGradOp : public XlaOpKernel { auto accumulation_type = XlaHelpers::SumAccumulationType(input_type(0)); auto converted = - XlaHelpers::ConvertElementType(builder, in_image, accumulation_type); + XlaHelpers::ConvertElementType(in_image, accumulation_type); auto squared = xla::Mul(converted, converted); auto reduce = xla::ReduceWindow( squared, XlaHelpers::Zero(builder, accumulation_type), *ctx->GetOrCreateAdd(accumulation_type), /* window_dimensions = */ {1, 1, 1, depth_radius_ * 2 + 1}, /* window_strides = */ {1, 1, 1, 1}, xla::Padding::kSame); - auto sqr_sum = - XlaHelpers::ConvertElementType(builder, reduce, input_type(0)); + auto sqr_sum = XlaHelpers::ConvertElementType(reduce, input_type(0)); auto norm = xla::Add(xla::ConstantR0(builder, bias_), @@ -157,15 +154,13 @@ class LRNGradOp : public XlaOpKernel { xla::Div(out_image, norm)), in_grads); - auto converted_dy = - XlaHelpers::ConvertElementType(builder, dy, accumulation_type); + auto converted_dy = XlaHelpers::ConvertElementType(dy, accumulation_type); auto dy_reduce = xla::ReduceWindow( converted_dy, XlaHelpers::Zero(builder, accumulation_type), *ctx->GetOrCreateAdd(accumulation_type), /* window_dimensions = */ {1, 1, 1, depth_radius_ * 2 + 1}, /* window_strides = */ {1, 1, 1, 1}, xla::Padding::kSame); - auto dy_reduced = - XlaHelpers::ConvertElementType(builder, dy_reduce, input_type(0)); + auto dy_reduced = XlaHelpers::ConvertElementType(dy_reduce, input_type(0)); xla::XlaOp gradients = xla::Add( xla::Mul(in_image, dy_reduced), diff --git a/tensorflow/compiler/tf2xla/kernels/matrix_band_part_op.cc b/tensorflow/compiler/tf2xla/kernels/matrix_band_part_op.cc index 8dfd7de591c..2dd0a710e47 100644 --- a/tensorflow/compiler/tf2xla/kernels/matrix_band_part_op.cc +++ b/tensorflow/compiler/tf2xla/kernels/matrix_band_part_op.cc @@ -16,8 +16,8 @@ limitations under the License. #include "tensorflow/compiler/tf2xla/xla_helpers.h" #include "tensorflow/compiler/tf2xla/xla_op_kernel.h" #include "tensorflow/compiler/tf2xla/xla_op_registry.h" -#include "tensorflow/compiler/xla/client/lib/numeric.h" #include "tensorflow/compiler/xla/client/xla_builder.h" +#include "tensorflow/compiler/xla/shape_util.h" #include "tensorflow/core/framework/tensor_shape.h" namespace tensorflow { @@ -61,11 +61,11 @@ class MatrixBandPartOp : public XlaOpKernel { // Compute 'offset', which is how many diagonals we are above/below the // diagonal. - xla::XlaOp iota_m = xla::Iota(builder, index_xla_type, m); - xla::XlaOp iota_n = xla::Iota(builder, index_xla_type, n); + xla::Shape iota_shape = xla::ShapeUtil::MakeShape(index_xla_type, {m, n}); + xla::XlaOp iota_m = xla::Iota(builder, iota_shape, /*iota_dimension=*/0); + xla::XlaOp iota_n = xla::Iota(builder, iota_shape, /*iota_dimension=*/1); - auto offset = xla::Sub(xla::Broadcast(iota_n, {m}), iota_m, - /*broadcast_dimensions=*/{0}); + auto offset = xla::Sub(iota_n, iota_m); // If num_lower or num_upper are negative, include all lower/upper // diagonals. diff --git a/tensorflow/compiler/tf2xla/kernels/matrix_set_diag_op.cc b/tensorflow/compiler/tf2xla/kernels/matrix_set_diag_op.cc index c0ca881ff82..4f980b6d14e 100644 --- a/tensorflow/compiler/tf2xla/kernels/matrix_set_diag_op.cc +++ b/tensorflow/compiler/tf2xla/kernels/matrix_set_diag_op.cc @@ -16,7 +16,6 @@ limitations under the License. #include "tensorflow/compiler/tf2xla/xla_helpers.h" #include "tensorflow/compiler/tf2xla/xla_op_kernel.h" #include "tensorflow/compiler/tf2xla/xla_op_registry.h" -#include "tensorflow/compiler/xla/client/lib/numeric.h" #include "tensorflow/compiler/xla/client/xla_builder.h" namespace tensorflow { diff --git a/tensorflow/compiler/tf2xla/kernels/permute_op.cc b/tensorflow/compiler/tf2xla/kernels/permute_op.cc index 94b51e1a586..71920bf5c1e 100644 --- a/tensorflow/compiler/tf2xla/kernels/permute_op.cc +++ b/tensorflow/compiler/tf2xla/kernels/permute_op.cc @@ -75,8 +75,7 @@ class DataFormatVecPermuteOp : public XlaOpKernel { } auto keys = xla::ConstantR1(builder, absl::Span(dst_indices)); if (input_rank == 2) { - keys = xla::BroadcastInDim( - keys, xla::ShapeUtil::MakeShape(xla::S32, {4, 2}), {0}); + keys = xla::BroadcastInDim(keys, {4, 2}, {0}); } auto sorted = xla::Sort(keys, {ctx->Input(0)}, 0); auto output = xla::GetTupleElement(sorted, 1); diff --git a/tensorflow/compiler/tf2xla/kernels/pooling_ops.cc b/tensorflow/compiler/tf2xla/kernels/pooling_ops.cc index a259da6383d..06c6cc37ec9 100644 --- a/tensorflow/compiler/tf2xla/kernels/pooling_ops.cc +++ b/tensorflow/compiler/tf2xla/kernels/pooling_ops.cc @@ -152,7 +152,12 @@ class MaxPoolOp : public PoolingOp { public: MaxPoolOp(OpKernelConstruction* ctx, int num_spatial_dims) : PoolingOp(ctx, /*num_spatial_dims=*/num_spatial_dims, - /*reduction_type=*/ctx->input_type(0)) {} + /*reduction_type=*/ctx->input_type(0)) { + string data_format_str; + OP_REQUIRES_OK(ctx, ctx->GetAttr("data_format", &data_format_str)); + OP_REQUIRES(ctx, FormatFromString(data_format_str, &data_format_), + errors::InvalidArgument("Invalid data format")); + } void Compile(XlaOpKernelContext* ctx) override { auto ksize_or_error = GetKernelSize(ctx); @@ -180,10 +185,6 @@ class MaxPool2DOp : public MaxPoolOp { public: explicit MaxPool2DOp(OpKernelConstruction* ctx) : MaxPoolOp(ctx, /*num_spatial_dims=*/2) { - string data_format_str; - OP_REQUIRES_OK(ctx, ctx->GetAttr("data_format", &data_format_str)); - OP_REQUIRES(ctx, FormatFromString(data_format_str, &data_format_), - errors::InvalidArgument("Invalid data format")); } }; REGISTER_XLA_OP(Name("MaxPool"), MaxPool2DOp); @@ -204,7 +205,12 @@ class AvgPoolOp : public PoolingOp { AvgPoolOp(OpKernelConstruction* ctx, int num_spatial_dims) : PoolingOp(ctx, /*num_spatial_dims=*/num_spatial_dims, /*reduction_type=*/ - XlaHelpers::SumAccumulationType(ctx->input_type(0))) {} + XlaHelpers::SumAccumulationType(ctx->input_type(0))) { + string data_format_str; + OP_REQUIRES_OK(ctx, ctx->GetAttr("data_format", &data_format_str)); + OP_REQUIRES(ctx, FormatFromString(data_format_str, &data_format_), + errors::InvalidArgument("Invalid data format")); + } void Compile(XlaOpKernelContext* ctx) override { auto ksize_or_error = GetKernelSize(ctx); @@ -241,10 +247,6 @@ class AvgPool2DOp : public AvgPoolOp { public: explicit AvgPool2DOp(OpKernelConstruction* ctx) : AvgPoolOp(ctx, /*num_spatial_dims=*/2) { - string data_format_str; - OP_REQUIRES_OK(ctx, ctx->GetAttr("data_format", &data_format_str)); - OP_REQUIRES(ctx, FormatFromString(data_format_str, &data_format_), - errors::InvalidArgument("Invalid data format")); } }; REGISTER_XLA_OP(Name("AvgPool"), AvgPool2DOp); @@ -390,6 +392,11 @@ class AvgPoolGradOp : public XlaOpKernel { OP_REQUIRES(ctx, ksize_[0] == 1 && stride_[0] == 1, errors::Unimplemented( "Pooling is not yet supported on the batch dimension.")); + + string data_format; + OP_REQUIRES_OK(ctx, ctx->GetAttr("data_format", &data_format)); + OP_REQUIRES(ctx, FormatFromString(data_format, &data_format_), + errors::InvalidArgument("Invalid data format")); } int num_dims() const { return num_spatial_dims_ + 2; } @@ -449,10 +456,6 @@ class AvgPool2DGradOp : public AvgPoolGradOp { public: explicit AvgPool2DGradOp(OpKernelConstruction* ctx) : AvgPoolGradOp(ctx, /*num_spatial_dims=*/2) { - string data_format; - OP_REQUIRES_OK(ctx, ctx->GetAttr("data_format", &data_format)); - OP_REQUIRES(ctx, FormatFromString(data_format, &data_format_), - errors::InvalidArgument("Invalid data format")); } }; REGISTER_XLA_OP( diff --git a/tensorflow/compiler/tf2xla/kernels/quantize_and_dequantize_op.cc b/tensorflow/compiler/tf2xla/kernels/quantize_and_dequantize_op.cc index 6f4ed496a17..7fe102428db 100644 --- a/tensorflow/compiler/tf2xla/kernels/quantize_and_dequantize_op.cc +++ b/tensorflow/compiler/tf2xla/kernels/quantize_and_dequantize_op.cc @@ -19,6 +19,7 @@ limitations under the License. #include "tensorflow/compiler/tf2xla/xla_op_registry.h" #include "tensorflow/compiler/xla/client/lib/arithmetic.h" #include "tensorflow/compiler/xla/client/lib/constants.h" +#include "tensorflow/compiler/xla/client/lib/math.h" #include "tensorflow/compiler/xla/client/xla_builder.h" #include "tensorflow/compiler/xla/client/xla_computation.h" #include "tensorflow/core/platform/macros.h" @@ -26,12 +27,26 @@ limitations under the License. namespace tensorflow { namespace { +enum QuantizerRoundMode { + // Round half up: if the fraction of y is exactly 0.5, then + // round(y) = y + 0.5 + // E.g., -5.5 gets rounded to -5, -5.4 goes to -5, + // 5.4 goes to 5, and 5.5 goes to 6. + ROUND_HALF_UP, + // Round half to even: if the fraction of y is exactly 0.5, then round(y) is + // the nearest even integer to y. + // E.g., 23.5 gets rounded to 24, 24.5 gets rounded to 24, while -23.5 becomes + // -24, and -24.5 gets rounded to 24. + ROUND_HALF_TO_EVEN, +}; + class QuantizeAndDequantizeOp : public XlaOpKernel { public: explicit QuantizeAndDequantizeOp(OpKernelConstruction* ctx) : XlaOpKernel(ctx) { OP_REQUIRES_OK(ctx, ctx->GetAttr("signed_input", &signed_input_)); OP_REQUIRES_OK(ctx, ctx->GetAttr("range_given", &range_given_)); + round_mode_ = ROUND_HALF_TO_EVEN; } void Compile(XlaOpKernelContext* ctx) override { @@ -117,8 +132,17 @@ class QuantizeAndDequantizeOp : public XlaOpKernel { // in that case they were measured from the tensor. input = Clamp(min_range, input, max_range); } - xla::XlaOp result = - Floor((input - min_range) * scale + half) * inverse_scale + min_range; + xla::XlaOp result; + switch (round_mode_) { + case ROUND_HALF_TO_EVEN: { + result = xla::RoundToEven(input * scale) * inverse_scale; + break; + } + case ROUND_HALF_UP: { + result = Floor(input * scale + half) * inverse_scale; + break; + } + } ctx->SetOutput(0, result); } @@ -126,6 +150,7 @@ class QuantizeAndDequantizeOp : public XlaOpKernel { int64 num_bits_ = -1; bool signed_input_; bool range_given_; + QuantizerRoundMode round_mode_; }; class QuantizeAndDequantizeV2Op : public QuantizeAndDequantizeOp { @@ -136,6 +161,20 @@ class QuantizeAndDequantizeV2Op : public QuantizeAndDequantizeOp { OP_REQUIRES(ctx, num_bits_ > 0 && num_bits_ < (signed_input_ ? 62 : 63), errors::InvalidArgument("num_bits is out of range: ", num_bits_, " with signed_input_ ", signed_input_)); + string round_mode_string; + OP_REQUIRES_OK(ctx, ctx->GetAttr("round_mode", &round_mode_string)); + OP_REQUIRES( + ctx, + (round_mode_string == "HALF_UP" || round_mode_string == "HALF_TO_EVEN"), + errors::InvalidArgument("Round mode string must be " + "'HALF_UP' or " + "'HALF_TO_EVEN', is '" + + round_mode_string + "'")); + if (round_mode_string == "HALF_UP") { + round_mode_ = ROUND_HALF_UP; + } else if (round_mode_string == "HALF_TO_EVEN") { + round_mode_ = ROUND_HALF_TO_EVEN; + } } }; diff --git a/tensorflow/compiler/tf2xla/kernels/random_ops.cc b/tensorflow/compiler/tf2xla/kernels/random_ops.cc index 415ce9b77ff..8822e29f7e7 100644 --- a/tensorflow/compiler/tf2xla/kernels/random_ops.cc +++ b/tensorflow/compiler/tf2xla/kernels/random_ops.cc @@ -26,7 +26,6 @@ limitations under the License. #include "tensorflow/compiler/tf2xla/xla_op_kernel.h" #include "tensorflow/compiler/tf2xla/xla_op_registry.h" #include "tensorflow/compiler/xla/client/lib/arithmetic.h" -#include "tensorflow/compiler/xla/client/lib/numeric.h" #include "tensorflow/compiler/xla/client/xla_builder.h" #include "tensorflow/core/framework/op_kernel.h" #include "tensorflow/core/framework/tensor.h" diff --git a/tensorflow/compiler/tf2xla/kernels/reduction_ops.cc b/tensorflow/compiler/tf2xla/kernels/reduction_ops.cc index 107fa62967a..65e158d64fd 100644 --- a/tensorflow/compiler/tf2xla/kernels/reduction_ops.cc +++ b/tensorflow/compiler/tf2xla/kernels/reduction_ops.cc @@ -113,12 +113,21 @@ class MeanOp : public XlaReductionOp { xla::Add(scalar_lhs, scalar_rhs); } - xla::XlaOp BuildFinalizer(xla::XlaBuilder* builder, - const xla::XlaOp& reduce_output, - int64 num_elements_reduced) override { - auto divisor = XlaHelpers::IntegerLiteral(builder, input_type(0), - num_elements_reduced); - return reduce_output / divisor; + xla::XlaOp BuildFinalizer( + xla::XlaBuilder* /*builder*/, const xla::XlaOp& input, + const xla::XlaOp& reduce_output, + const std::vector& dimensions_to_reduce) override { + if (dimensions_to_reduce.empty()) { + return reduce_output; + } + auto divisor = xla::GetDimensionSize(input, dimensions_to_reduce[0]); + for (int i = 1; i < dimensions_to_reduce.size(); i++) { + auto size = xla::GetDimensionSize(input, dimensions_to_reduce[i]); + divisor = xla::Mul(divisor, size); + } + divisor = xla::ConvertElementType(divisor, xla_reduction_type_); + return XlaHelpers::ConvertElementType(reduce_output / divisor, + input_type(0)); } }; diff --git a/tensorflow/compiler/tf2xla/kernels/reduction_ops.h b/tensorflow/compiler/tf2xla/kernels/reduction_ops.h index 466e79828d1..af716eab798 100644 --- a/tensorflow/compiler/tf2xla/kernels/reduction_ops.h +++ b/tensorflow/compiler/tf2xla/kernels/reduction_ops.h @@ -48,13 +48,14 @@ class XlaReductionOp : public XlaOpKernel { const xla::XlaOp& scalar_rhs) = 0; // Applies a transformation to the output of the reduction. The desired - // computation should be added to 'builder'. Argument 'reduce_output' is the - // output of the reduction. 'num_elements_reduced' is the number of elements - // that contributed to the reduction. Returns the transformed reduction - // output, Defaults to returning 'reduce_output' unchanged. - virtual xla::XlaOp BuildFinalizer(xla::XlaBuilder* builder, - const xla::XlaOp& reduce_output, - int64 num_elements_reduced); + // computation should be added to 'builder'. Argument 'input' is the original + // input of the reduction; 'reduce_output' is the output of the reduction. + // Returns the transformed reduction output. Defaults to returning + // 'reduce_output' converted to the input type. + virtual xla::XlaOp BuildFinalizer( + xla::XlaBuilder* builder, const xla::XlaOp& input, + const xla::XlaOp& reduce_output, + const std::vector& dimensions_to_reduce); void Compile(XlaOpKernelContext* ctx) override; diff --git a/tensorflow/compiler/tf2xla/kernels/reduction_ops_common.cc b/tensorflow/compiler/tf2xla/kernels/reduction_ops_common.cc index 118f2798d55..2ca2a85244b 100644 --- a/tensorflow/compiler/tf2xla/kernels/reduction_ops_common.cc +++ b/tensorflow/compiler/tf2xla/kernels/reduction_ops_common.cc @@ -35,12 +35,13 @@ XlaReductionOp::XlaReductionOp(OpKernelConstruction* ctx, ctx, DataTypeToPrimitiveType(reduction_type_, &xla_reduction_type_)); } -// Unless BuildFinalizer is overridden the reduction has no -// finalizer. -xla::XlaOp XlaReductionOp::BuildFinalizer(xla::XlaBuilder* builder, - const xla::XlaOp& reduce_output, - int64 num_elements_reduced) { - return reduce_output; +// The default finalizer converts the results back into the input type. This can +// be overridden. +xla::XlaOp XlaReductionOp::BuildFinalizer( + xla::XlaBuilder* /*builder*/, const xla::XlaOp& /*input*/, + const xla::XlaOp& reduce_output, + const std::vector& /*dimensions_to_reduce*/) { + return XlaHelpers::ConvertElementType(reduce_output, input_type(0)); } void XlaReductionOp::Compile(XlaOpKernelContext* ctx) { @@ -71,7 +72,6 @@ void XlaReductionOp::Compile(XlaOpKernelContext* ctx) { absl::InlinedVector bitmap(data_shape.dims(), false); std::vector xla_axes; - int64 num_elements_reduced = 1LL; for (int64 i = 0; i < axes_tensor_shape.num_elements(); ++i) { int64 index = axes[i]; OP_REQUIRES(ctx, @@ -82,7 +82,6 @@ void XlaReductionOp::Compile(XlaOpKernelContext* ctx) { index = (index + data_shape.dims()) % data_shape.dims(); bitmap[index] = true; xla_axes.push_back(index); - num_elements_reduced *= data_shape.dim_size(index); } std::vector final_shape; @@ -118,8 +117,7 @@ void XlaReductionOp::Compile(XlaOpKernelContext* ctx) { xla::XlaComputation reduction_computation = r.Build().ConsumeValueOrDie(); auto reduce = xla::Reduce(data, initial, reduction_computation, xla_axes); - auto deconverted = XlaHelpers::ConvertElementType(b, reduce, input_type(0)); - auto finalized = BuildFinalizer(b, deconverted, num_elements_reduced); + auto finalized = BuildFinalizer(b, data, reduce, xla_axes); auto result = keep_dims_ ? xla::Reshape(finalized, final_shape) : finalized; ctx->SetOutput(0, result); } diff --git a/tensorflow/compiler/tf2xla/kernels/resampler_ops.cc b/tensorflow/compiler/tf2xla/kernels/resampler_ops.cc index 847704608fb..54d34a38abc 100644 --- a/tensorflow/compiler/tf2xla/kernels/resampler_ops.cc +++ b/tensorflow/compiler/tf2xla/kernels/resampler_ops.cc @@ -24,7 +24,6 @@ limitations under the License. #include "tensorflow/compiler/xla/array4d.h" #include "tensorflow/compiler/xla/client/lib/arithmetic.h" #include "tensorflow/compiler/xla/client/lib/constants.h" -#include "tensorflow/compiler/xla/client/lib/numeric.h" #include "tensorflow/compiler/xla/client/xla_builder.h" #include "tensorflow/compiler/xla/literal.h" #include "tensorflow/compiler/xla/shape_util.h" @@ -44,9 +43,6 @@ namespace { using xla::XlaOp; -// TODO(b/112295522): note that sampling from image boundary is not currently -// being handled properly. - // Calculates the bilinear weight tensor, given basis ratio (px, py) of the // sampling position: // W = [(1-px)*(1-py), px*(1-py), (1-px)*py, px*py] @@ -70,11 +66,8 @@ XlaOp BilinearWeights(XlaOpKernelContext* ctx, XlaOp ratio, std::vector last_two_dims_indices = {(broadcast_dims_size - 2), (broadcast_dims_size - 1)}; - xla::Shape broadcast_shape = - xla::ShapeUtil::MakeShape(xla_type, broadcast_dims); - auto broadcast_first_term = - xla::BroadcastInDim(first_term, broadcast_shape, last_two_dims_indices); + xla::BroadcastInDim(first_term, broadcast_dims, last_two_dims_indices); // Ratio is of the same dimension as warp, which is [batch, dim_0,... dim_n, // 2], we broadcast ratio tensor to 'broadcast_dim' by keeping the @@ -85,7 +78,7 @@ XlaOp BilinearWeights(XlaOpKernelContext* ctx, XlaOp ratio, ratio_broadcast_indices.erase(ratio_broadcast_indices.end() - 2); auto broadcast_ratio = - xla::BroadcastInDim(ratio, broadcast_shape, ratio_broadcast_indices); + xla::BroadcastInDim(ratio, broadcast_dims, ratio_broadcast_indices); auto first_term_subtract_weights = broadcast_first_term - broadcast_ratio; @@ -96,7 +89,7 @@ XlaOp BilinearWeights(XlaOpKernelContext* ctx, XlaOp ratio, sign_change = xla::ConvertElementType(sign_change, xla_type); auto broadcast_sign_change = - xla::BroadcastInDim(sign_change, broadcast_shape, last_two_dims_indices); + xla::BroadcastInDim(sign_change, broadcast_dims, last_two_dims_indices); auto flipped = first_term_subtract_weights * broadcast_sign_change; @@ -232,21 +225,19 @@ XlaOp CalculateGradData(XlaOpKernelContext* ctx, XlaOp grad_output, XlaOp ratio, std::vector weights_with_channels_dims = reshaped_weights_dims; weights_with_channels_dims.push_back(data_channels); - auto weights_with_channels_shape = - xla::ShapeUtil::MakeShape(warp_type, weights_with_channels_dims); std::vector reshaped_weights_indices(reshaped_weights_dims.size()); std::iota(reshaped_weights_indices.begin(), reshaped_weights_indices.end(), 0); // The dimension is [batch, dim_0, ..., dim_n, 2, 2, data_channel]. auto broadcast_reshaped_weights = xla::BroadcastInDim( - reshaped_weights, weights_with_channels_shape, reshaped_weights_indices); + reshaped_weights, weights_with_channels_dims, reshaped_weights_indices); std::vector grad_output_indices(warp_dims_without_last_dims.size()); std::iota(grad_output_indices.begin(), grad_output_indices.end(), 0); grad_output_indices.push_back(weights_with_channels_dims.size() - 1); XlaOp broadcast_grad_output = xla::BroadcastInDim( - grad_output, weights_with_channels_shape, grad_output_indices); + grad_output, weights_with_channels_dims, grad_output_indices); auto grad_output_multiply_weights = broadcast_grad_output * broadcast_reshaped_weights; @@ -294,13 +285,10 @@ XlaOp CalculateGradWarp(XlaOpKernelContext* ctx, XlaOp grad_output, XlaOp ratio, std::vector warp_dims_without_last_dims(warp_dims.begin(), warp_dims.end() - 1); + // With dimension [batch, dim_0, ...dim_n, 4] std::vector neighbor_broadcast_dims = warp_dims_without_last_dims; neighbor_broadcast_dims.push_back(4); - // With dimension [batch, dim_0, ...dim_n, 4] - auto neighbor_broadcast_shape = - xla::ShapeUtil::MakeShape(data_type, neighbor_broadcast_dims); - // The dimension is [batch, dim_0, ... dim_n, 4, data_channels] auto neighbors_data = Gather2by2Neighbors( ctx->builder(), data, gather_indices, data_channels, warp_shape.dims()); @@ -326,7 +314,7 @@ XlaOp CalculateGradWarp(XlaOpKernelContext* ctx, XlaOp grad_output, XlaOp ratio, xla::BroadcastInDim( xla::ConvertElementType( xla::ConstantR1(ctx->builder(), {0, 0, -1, 1}), data_type), - neighbor_broadcast_shape, {last_warp_dim}), + neighbor_broadcast_dims, {last_warp_dim}), neighbors_data, dot_dims, /*precision_config=*/nullptr); // img_cxfy - img_fxfy @@ -334,7 +322,7 @@ XlaOp CalculateGradWarp(XlaOpKernelContext* ctx, XlaOp grad_output, XlaOp ratio, xla::BroadcastInDim( xla::ConvertElementType( xla::ConstantR1(ctx->builder(), {-1, 1, 0, 0}), data_type), - neighbor_broadcast_shape, {last_warp_dim}), + neighbor_broadcast_dims, {last_warp_dim}), neighbors_data, dot_dims, /*precision_config=*/nullptr); // img_cxcy - img_cxfy @@ -342,7 +330,7 @@ XlaOp CalculateGradWarp(XlaOpKernelContext* ctx, XlaOp grad_output, XlaOp ratio, xla::BroadcastInDim( xla::ConvertElementType( xla::ConstantR1(ctx->builder(), {0, -1, 0, 1}), data_type), - neighbor_broadcast_shape, {last_warp_dim}), + neighbor_broadcast_dims, {last_warp_dim}), neighbors_data, dot_dims, /*precision_config=*/nullptr); // img_fxcy - img_fxfy @@ -350,7 +338,7 @@ XlaOp CalculateGradWarp(XlaOpKernelContext* ctx, XlaOp grad_output, XlaOp ratio, xla::BroadcastInDim( xla::ConvertElementType( xla::ConstantR1(ctx->builder(), {-1, 0, 1, 0}), data_type), - neighbor_broadcast_shape, {last_warp_dim}), + neighbor_broadcast_dims, {last_warp_dim}), neighbors_data, dot_dims, /*precision_config=*/nullptr); // Slice out x and y. @@ -421,12 +409,13 @@ class ResamplerOp : public XlaOpKernel { OP_REQUIRES(ctx, warp_shape.dim_size(last_warp_dim) == 2, errors::InvalidArgument( "the last dimension of warp must be exactly size 2.")); + xla::PrimitiveType warp_type = ctx->input_xla_type(1); XlaOp data = ctx->Input("data"); XlaOp warp = ctx->Input("warp"); // Find the coordinates of the top left corner for the 2x2 region to be - // sampled from. The dimensions are (batch, dim_0, ... dim_n, 2) where the + // sampled from. The dimensions are [batch, dim_0, ... dim_n, 2] where the // last dimension of size 2 in turn is [x, y]. XlaOp top_left = xla::ConvertElementType(warp, xla::U32); @@ -457,10 +446,54 @@ class ResamplerOp : public XlaOpKernel { dot_dims.add_lhs_contracting_dimensions(warp_shape.dims() - 1); dot_dims.add_rhs_contracting_dimensions(warp_shape.dims() - 1); + // The dimension is [batch, dim_0, ...dim_n, data_channels]. auto blended_pixels = xla::DotGeneral(weights, neighbors_data, dot_dims, /*precision_config=*/nullptr); - ctx->SetOutput(0, blended_pixels); + // Handle out of boundary cases by constructing a predicate mask array based + // on the in-bound condition, and output 0 for the blended pixel value if + // out-bound. The dimension is the same as top_left: [batch, dim_0, + // ...dim_n, 2] where the last dimension of size 2 is the [x, y] coordinate. + + auto is_ge_zero = xla::Ge(warp, xla::ZerosLike(warp)); + + auto is_lt_image_size = xla::Lt( + warp, + xla::ConvertElementType( + xla::ConstantR1( + ctx->builder(), + {/*width=*/static_cast(data_shape.dim_size(2) - 1), + /*height=*/static_cast(data_shape.dim_size(1) - 1)}), + warp_type), + /*broadcast_dimensions=*/{warp_shape.dims() - 1}); + + auto is_in_bound_x_y = xla::And(is_ge_zero, is_lt_image_size); + // Reduce along last dimension. The resulting dimension is: + // [batch, dim_0, ...dim_n]. + auto is_in_bound = xla::Reduce( + is_in_bound_x_y, xla::ConstantR0(ctx->builder(), true), + xla::CreateScalarAndComputation(xla::PrimitiveType::PRED, + ctx->builder()), + {last_warp_dim}); + + // Broadcast 'is_in_bound' to the same dimension as 'blended_pixels', which + // is the dimension of the result: + // [batch, dim_0, ...dim_n, data_channels]. + auto warp_dims = warp_shape.dim_sizes(); + std::vector result_dims(warp_dims.begin(), warp_dims.end() - 1); + result_dims.push_back(data_channels); + + std::vector broadcasted_dims(warp_dims.size() - 1); + std::iota(broadcasted_dims.begin(), broadcasted_dims.end(), 0); + auto broadcasted_is_in_bound = + xla::BroadcastInDim(is_in_bound, result_dims, broadcasted_dims); + + // Set out of bound samples to zero. + auto zeros = + xla::Broadcast(xla::Zero(ctx->builder(), data_type), result_dims); + auto result = xla::Select(broadcasted_is_in_bound, blended_pixels, zeros); + + ctx->SetOutput(0, result); } }; @@ -473,6 +506,8 @@ class ResamplerGradOp : public XlaOpKernel { OP_REQUIRES_OK(ctx, ctx->GetAttr("T", &output_dtype)); } + // TODO(b/112295522): note that sampling from image boundary is not currently + // being handled properly. void Compile(XlaOpKernelContext* ctx) override { TensorShape data_shape_tf = ctx->InputShape("data"); OP_REQUIRES(ctx, data_shape_tf.dims() == 4, diff --git a/tensorflow/compiler/tf2xla/kernels/retval_op.cc b/tensorflow/compiler/tf2xla/kernels/retval_op.cc index 6970dd0a006..e4046c79557 100644 --- a/tensorflow/compiler/tf2xla/kernels/retval_op.cc +++ b/tensorflow/compiler/tf2xla/kernels/retval_op.cc @@ -47,8 +47,7 @@ class RetvalOp : public XlaOpKernel { // compilation. OP_REQUIRES_OK(ctx, frame->SetRetval(index_, input)); } else { - XlaContext& xla_context = XlaContext::Get(ctx); - xla_context.SetRetval(index_, ctx->InputExpression(0)); + ctx->xla_context()->SetRetval(index_, ctx->InputExpression(0)); } } diff --git a/tensorflow/compiler/tf2xla/kernels/reverse_sequence_op.cc b/tensorflow/compiler/tf2xla/kernels/reverse_sequence_op.cc index 7ff3e916381..d7b38e86cc9 100644 --- a/tensorflow/compiler/tf2xla/kernels/reverse_sequence_op.cc +++ b/tensorflow/compiler/tf2xla/kernels/reverse_sequence_op.cc @@ -18,7 +18,6 @@ limitations under the License. #include "tensorflow/compiler/tf2xla/xla_op_kernel.h" #include "tensorflow/compiler/tf2xla/xla_op_registry.h" #include "tensorflow/compiler/xla/client/lib/constants.h" -#include "tensorflow/compiler/xla/client/lib/numeric.h" #include "tensorflow/compiler/xla/client/xla_builder.h" #include "tensorflow/compiler/xla/xla_data.pb.h" #include "tensorflow/core/framework/tensor_shape.h" diff --git a/tensorflow/compiler/tf2xla/kernels/scan_ops.cc b/tensorflow/compiler/tf2xla/kernels/scan_ops.cc index b5fd7850bfc..4b9e1a578be 100644 --- a/tensorflow/compiler/tf2xla/kernels/scan_ops.cc +++ b/tensorflow/compiler/tf2xla/kernels/scan_ops.cc @@ -39,8 +39,8 @@ namespace { // TODO(phawkins): implement double-sized windowed reductions in XLA and remove // the type constraint. -constexpr std::array kScanOpTypes = { - {DT_HALF, DT_BFLOAT16, DT_FLOAT}}; +constexpr std::array kScanOpTypes = { + {DT_HALF, DT_BFLOAT16, DT_FLOAT, DT_INT32}}; class ScanOp : public XlaOpKernel { public: @@ -103,11 +103,10 @@ class ScanOp : public XlaOpKernel { reducer = ctx->GetOrCreateMul(dtype); } auto output = xla::ReduceWindowWithGeneralPadding( - XlaHelpers::ConvertElementType(builder, ctx->Input(0), dtype), init, - *reducer, window_dims, window_strides, + XlaHelpers::ConvertElementType(ctx->Input(0), dtype), init, *reducer, + window_dims, window_strides, /*base_dilations=*/{}, /*window_dilations=*/{}, padding); - output = - XlaHelpers::ConvertElementType(builder, output, ctx->input_type(0)); + output = XlaHelpers::ConvertElementType(output, ctx->input_type(0)); // In exclusive mode, we have computed an extra element containing the sum // of all the input elements. Slice off this extra "last" element. diff --git a/tensorflow/compiler/tf2xla/kernels/sendrecv_ops.cc b/tensorflow/compiler/tf2xla/kernels/sendrecv_ops.cc index a7f5a8f1698..84470b230d4 100644 --- a/tensorflow/compiler/tf2xla/kernels/sendrecv_ops.cc +++ b/tensorflow/compiler/tf2xla/kernels/sendrecv_ops.cc @@ -42,7 +42,7 @@ SendOp::SendOp(OpKernelConstruction* ctx) : XlaOpKernel(ctx) { } void SendOp::Compile(XlaOpKernelContext* ctx) { - XlaCompiler* compiler = XlaContext::Get(ctx).compiler(); + XlaCompiler* compiler = ctx->compiler(); xla::ChannelHandle channel; OP_REQUIRES_OK(ctx, compiler->GetChannelHandle(tensor_name_, &channel)); xla::Send(ctx->Input(0), channel); @@ -73,7 +73,7 @@ RecvOp::RecvOp(OpKernelConstruction* ctx) : XlaOpKernel(ctx) { } void RecvOp::Compile(XlaOpKernelContext* ctx) { - XlaCompiler* compiler = XlaContext::Get(ctx).compiler(); + XlaCompiler* compiler = ctx->compiler(); xla::ChannelHandle channel; OP_REQUIRES_OK(ctx, compiler->GetChannelHandle(tensor_name_, &channel)); ctx->SetOutput(0, xla::Recv(ctx->builder(), shape_, channel)); diff --git a/tensorflow/compiler/tf2xla/kernels/sequence_ops.cc b/tensorflow/compiler/tf2xla/kernels/sequence_ops.cc index 60b011ba6d9..b1fa2915d59 100644 --- a/tensorflow/compiler/tf2xla/kernels/sequence_ops.cc +++ b/tensorflow/compiler/tf2xla/kernels/sequence_ops.cc @@ -18,7 +18,7 @@ limitations under the License. #include "tensorflow/compiler/tf2xla/xla_helpers.h" #include "tensorflow/compiler/tf2xla/xla_op_kernel.h" #include "tensorflow/compiler/tf2xla/xla_op_registry.h" -#include "tensorflow/compiler/xla/client/lib/numeric.h" +#include "tensorflow/compiler/xla/client/xla_builder.h" #include "tensorflow/compiler/xla/literal.h" #include "tensorflow/compiler/xla/primitive_util.h" #include "tensorflow/core/framework/op_kernel.h" diff --git a/tensorflow/compiler/tf2xla/kernels/softmax_op.cc b/tensorflow/compiler/tf2xla/kernels/softmax_op.cc index d6bd927135c..20da8033536 100644 --- a/tensorflow/compiler/tf2xla/kernels/softmax_op.cc +++ b/tensorflow/compiler/tf2xla/kernels/softmax_op.cc @@ -71,7 +71,7 @@ class SoftmaxOp : public XlaOpKernel { auto reduce = xla::Reduce(converted, xla::Zero(b, xla_accumulation_type), *ctx->GetOrCreateAdd(accumulation_type), {kClassDim}); - auto sum = XlaHelpers::ConvertElementType(b, reduce, type); + auto sum = XlaHelpers::ConvertElementType(reduce, type); auto softmax = log_ // softmax = shifted_logits - log(sum(exp(shifted_logits))) @@ -111,11 +111,11 @@ std::pair CrossEntropyWithLogits( // sum_{class} (exp(logits - max_logits)) const DataType accumulation_type = XlaHelpers::SumAccumulationType(type); auto converted = - XlaHelpers::ConvertElementType(b, exp_shifted_logits, accumulation_type); + XlaHelpers::ConvertElementType(exp_shifted_logits, accumulation_type); auto reduce = xla::Reduce(converted, XlaHelpers::Zero(b, accumulation_type), *ctx->GetOrCreateAdd(accumulation_type), {kClassDim}); - auto sum_exp = XlaHelpers::ConvertElementType(b, reduce, type); + auto sum_exp = XlaHelpers::ConvertElementType(reduce, type); // log(sum(exp(logits - max_logits))) auto log_sum_exp = xla::Log(sum_exp); @@ -126,11 +126,10 @@ std::pair CrossEntropyWithLogits( // (The subtraction broadcasts along the batch dimension.) auto sub = xla::Sub(shifted_logits, log_sum_exp, {kBatchDim}); auto mul = xla::Mul(xla::Neg(labels), sub); - auto sum = - xla::Reduce(XlaHelpers::ConvertElementType(b, mul, accumulation_type), - XlaHelpers::Zero(b, accumulation_type), - *ctx->GetOrCreateAdd(accumulation_type), {kClassDim}); - auto loss = XlaHelpers::ConvertElementType(b, sum, type); + auto sum = xla::Reduce(XlaHelpers::ConvertElementType(mul, accumulation_type), + XlaHelpers::Zero(b, accumulation_type), + *ctx->GetOrCreateAdd(accumulation_type), {kClassDim}); + auto loss = XlaHelpers::ConvertElementType(sum, type); // backprop: prob - labels, where // prob = exp(logits - max_logits) / sum(exp(logits - max_logits)) diff --git a/tensorflow/compiler/tf2xla/kernels/stack_ops.cc b/tensorflow/compiler/tf2xla/kernels/stack_ops.cc index 7b96b43ad83..8e9e4daf99d 100644 --- a/tensorflow/compiler/tf2xla/kernels/stack_ops.cc +++ b/tensorflow/compiler/tf2xla/kernels/stack_ops.cc @@ -69,7 +69,7 @@ Status MaybeInitializeStack(xla::XlaBuilder* builder, XlaResource* resource, } TensorShape stack_shape; - stack_shape.AddDim(resource->tensor_array_size()); + stack_shape.AddDim(resource->max_array_size()); stack_shape.AppendShape(elem_shape); if (!resource->initialized()) { @@ -97,10 +97,10 @@ class StackOp : public XlaOpKernel { } void Compile(XlaOpKernelContext* ctx) override { - int64 size; - OP_REQUIRES_OK(ctx, ctx->ConstantInputAsIntScalar(0, &size)); + int64 max_size; + OP_REQUIRES_OK(ctx, ctx->ConstantInputAsIntScalar(0, &max_size)); OP_REQUIRES( - ctx, size >= 0, + ctx, max_size >= 0, errors::InvalidArgument( "XLA compilation requires a fixed stack size upper bound. If " "you are using tf.while_loop, set the maximum_iterations parameter " @@ -108,14 +108,9 @@ class StackOp : public XlaOpKernel { // We defer initializing the Stack resource until we see the first push. // Otherwise we do not know the shape of the stack elements. - xla::XlaOp value; - XlaContext& xc = XlaContext::Get(ctx); - XlaResource* resource; - string name = absl::StrCat("Stack: ", stack_name_); - OP_REQUIRES_OK( - ctx, xc.CreateResource(XlaResource::kStack, -1, std::move(name), dtype_, - TensorShape(), value, /*tensor_array_size=*/size, - /*tensor_array_gradients=*/{}, &resource)); + XlaResource* resource = + ctx->xla_context()->AddResource(XlaResource::CreateStack( + /*name=*/absl::StrCat("Stack: ", stack_name_), dtype_, max_size)); ctx->SetResourceOutput(0, resource); } diff --git a/tensorflow/compiler/tf2xla/kernels/stateless_random_ops.cc b/tensorflow/compiler/tf2xla/kernels/stateless_random_ops.cc index 5db52781be4..50653d7b397 100644 --- a/tensorflow/compiler/tf2xla/kernels/stateless_random_ops.cc +++ b/tensorflow/compiler/tf2xla/kernels/stateless_random_ops.cc @@ -23,7 +23,6 @@ limitations under the License. #include "tensorflow/compiler/tf2xla/xla_op_registry.h" #include "tensorflow/compiler/xla/client/lib/constants.h" #include "tensorflow/compiler/xla/client/lib/math.h" -#include "tensorflow/compiler/xla/client/lib/numeric.h" #include "tensorflow/compiler/xla/client/lib/prng.h" #include "tensorflow/compiler/xla/client/xla_builder.h" #include "tensorflow/core/framework/op_kernel.h" diff --git a/tensorflow/compiler/tf2xla/kernels/tensor_array_ops.cc b/tensorflow/compiler/tf2xla/kernels/tensor_array_ops.cc index 252967a7464..939d7e19515 100644 --- a/tensorflow/compiler/tf2xla/kernels/tensor_array_ops.cc +++ b/tensorflow/compiler/tf2xla/kernels/tensor_array_ops.cc @@ -61,8 +61,8 @@ Status MaybeInitializeTensorArray(xla::XlaBuilder* builder, " but op has dtype ", DataTypeString(dtype), "."); } - TF_RET_CHECK(resource->tensor_array_size() >= 0) - << resource->name() << " size " << resource->tensor_array_size(); + TF_RET_CHECK(resource->max_array_size() >= 0) + << resource->name() << " size " << resource->max_array_size(); if (!resource->initialized()) { TF_RETURN_IF_ERROR(resource->SetTypeAndShape(dtype, elem_shape)); @@ -78,7 +78,7 @@ Status MaybeInitializeTensorArray(xla::XlaBuilder* builder, XLAShapeToTensorShape(shape_or_status.ValueOrDie(), &shape)); TensorShape ta_shape; - ta_shape.AddDim(resource->tensor_array_size()); + ta_shape.AddDim(resource->max_array_size()); ta_shape.AppendShape(elem_shape); if (ta_shape != shape) { return errors::InvalidArgument( @@ -114,7 +114,7 @@ Status CheckTensorArrayIsInitialized(const string& op_name, Status GetTensorArrayShape(const XlaResource* resource, xla::XlaBuilder* builder, TensorShape* shape) { *shape = resource->shape(); - shape->InsertDim(0, resource->tensor_array_size()); + shape->InsertDim(0, resource->max_array_size()); return Status::OK(); } @@ -166,13 +166,10 @@ class TensorArrayOp : public XlaOpKernel { value = xla::Broadcast(zero, ta_shape.dim_sizes()); } - XlaContext& xc = XlaContext::Get(ctx); - XlaResource* var; - string name = absl::StrCat("TensorArray: ", tensor_array_name_); - OP_REQUIRES_OK( - ctx, xc.CreateResource(XlaResource::kTensorArray, -1, std::move(name), - dtype_, shape, value, /*tensor_array_size=*/size, - /*tensor_array_gradients=*/{}, &var)); + XlaResource* var = + ctx->xla_context()->AddResource(XlaResource::CreateTensorArray( + /*name=*/absl::StrCat("TensorArray: ", tensor_array_name_), dtype_, + shape, /*initial_value=*/value, /*max_array_size=*/size)); ctx->SetResourceOutput(0, var); Tensor flow(DT_FLOAT, TensorShape({})); @@ -517,14 +514,13 @@ class TensorArraySplitOp : public XlaOpKernel { xla::XlaOp ta = resource->value(); TensorShape ta_shape; - ta_shape.AddDim(resource->tensor_array_size()); + ta_shape.AddDim(resource->max_array_size()); ta_shape.AppendShape(elem_shape); - OP_REQUIRES( - ctx, lengths.size() == resource->tensor_array_size(), - errors::InvalidArgument( - "TensorArray's size is not equal to the size of lengths (", - lengths.size(), " vs. ", resource->tensor_array_size(), ")")); + OP_REQUIRES(ctx, lengths.size() == resource->max_array_size(), + errors::InvalidArgument( + "TensorArray's size is not equal to the size of lengths (", + lengths.size(), " vs. ", resource->max_array_size(), ")")); const xla::XlaOp value = ctx->Input(1); const xla::XlaOp flow = ctx->Input(3); @@ -562,8 +558,7 @@ class TensorArraySizeOp : public XlaOpKernel { XlaResource* var; OP_REQUIRES_OK(ctx, ctx->GetResourceInput(0, &var)); Tensor size_tensor(DT_INT32, {}); - size_tensor.scalar()() = - static_cast(var->tensor_array_size()); + size_tensor.scalar()() = static_cast(var->max_array_size()); ctx->SetConstantOutput(0, size_tensor); } diff --git a/tensorflow/compiler/tf2xla/kernels/topk_op.cc b/tensorflow/compiler/tf2xla/kernels/topk_op.cc index 8a0c94cfae1..ee3bdf3394e 100644 --- a/tensorflow/compiler/tf2xla/kernels/topk_op.cc +++ b/tensorflow/compiler/tf2xla/kernels/topk_op.cc @@ -15,7 +15,6 @@ limitations under the License. #include "tensorflow/compiler/tf2xla/xla_op_kernel.h" #include "tensorflow/compiler/tf2xla/xla_op_registry.h" -#include "tensorflow/compiler/xla/client/lib/numeric.h" #include "tensorflow/compiler/xla/client/lib/sorting.h" #include "tensorflow/compiler/xla/client/xla_builder.h" #include "tensorflow/compiler/xla/literal.h" diff --git a/tensorflow/compiler/tf2xla/kernels/training_ops.cc b/tensorflow/compiler/tf2xla/kernels/training_ops.cc index 7077c2e3a54..960c1462ceb 100644 --- a/tensorflow/compiler/tf2xla/kernels/training_ops.cc +++ b/tensorflow/compiler/tf2xla/kernels/training_ops.cc @@ -320,9 +320,8 @@ class ResourceApplyAdagradDA : public XlaOpKernel { xla::XlaOp lr = ctx->Input(4); xla::XlaOp l1 = ctx->Input(5); xla::XlaOp l2 = ctx->Input(6); - xla::XlaBuilder* const b = ctx->builder(); xla::XlaOp global_step = - XlaHelpers::ConvertElementType(b, ctx->Input(7), dtype_); + XlaHelpers::ConvertElementType(ctx->Input(7), dtype_); accum = accum + grad; squared_accum = squared_accum + xla::Square(grad); diff --git a/tensorflow/compiler/tf2xla/kernels/while_op.cc b/tensorflow/compiler/tf2xla/kernels/while_op.cc index 559414eeaa5..ce007fc04a8 100644 --- a/tensorflow/compiler/tf2xla/kernels/while_op.cc +++ b/tensorflow/compiler/tf2xla/kernels/while_op.cc @@ -64,7 +64,7 @@ Status MakeXlaCompilerArgumentsFromInputs( if (!arg.initialized) { *has_uninitialized_vars = true; } - arg.tensor_array_size = resource->tensor_array_size(); + arg.max_array_size = resource->max_array_size(); for (const auto& gradient : resource->tensor_array_gradients()) { arg.tensor_array_gradients.insert(gradient.first); } diff --git a/tensorflow/compiler/tf2xla/kernels/xla_broadcast_helper_op.cc b/tensorflow/compiler/tf2xla/kernels/xla_broadcast_helper_op.cc index a9f88a6df25..ad8e707e111 100644 --- a/tensorflow/compiler/tf2xla/kernels/xla_broadcast_helper_op.cc +++ b/tensorflow/compiler/tf2xla/kernels/xla_broadcast_helper_op.cc @@ -89,13 +89,10 @@ class XlaBroadcastHelperOp : public XlaOpKernel { lhs_shape.DebugString(), " and ", rhs_shape.DebugString())); broadcast_shape[dim] = min_rank_shape->dim_size(i); } - xla::PrimitiveType type = context->input_xla_type(0); - xla::Shape broadcast_xla_shape = - xla::ShapeUtil::MakeShape(type, broadcast_shape); if (broadcast_lhs) { - lhs = xla::BroadcastInDim(lhs, broadcast_xla_shape, broadcast_dims); + lhs = xla::BroadcastInDim(lhs, broadcast_shape, broadcast_dims); } else { - rhs = xla::BroadcastInDim(rhs, broadcast_xla_shape, broadcast_dims); + rhs = xla::BroadcastInDim(rhs, broadcast_shape, broadcast_dims); } context->SetOutput(0, lhs); context->SetOutput(1, rhs); diff --git a/tensorflow/compiler/tf2xla/lib/BUILD b/tensorflow/compiler/tf2xla/lib/BUILD index 1ce3930fd1c..422781d536a 100644 --- a/tensorflow/compiler/tf2xla/lib/BUILD +++ b/tensorflow/compiler/tf2xla/lib/BUILD @@ -17,20 +17,6 @@ filegroup( load("//tensorflow/compiler/xla/tests:build_defs.bzl", "xla_test") -cc_library( - name = "batch_dot", - srcs = ["batch_dot.cc"], - hdrs = ["batch_dot.h"], - deps = [ - "//tensorflow/compiler/xla:shape_util", - "//tensorflow/compiler/xla:status_macros", - "//tensorflow/compiler/xla:statusor", - "//tensorflow/compiler/xla:xla_data_proto", - "//tensorflow/compiler/xla/client:xla_builder", - "//tensorflow/core:lib", - ], -) - cc_library( name = "broadcast", srcs = ["broadcast.cc"], @@ -52,7 +38,6 @@ cc_library( srcs = ["cholesky.cc"], hdrs = ["cholesky.h"], deps = [ - ":batch_dot", ":triangular_solve", ":util", ":while_loop", @@ -63,6 +48,8 @@ cc_library( "//tensorflow/compiler/xla:xla_data_proto", "//tensorflow/compiler/xla/client:xla_builder", "//tensorflow/compiler/xla/client/lib:constants", + "//tensorflow/compiler/xla/client/lib:matrix", + "//tensorflow/compiler/xla/client/lib:slicing", "//tensorflow/core:lib", ], ) @@ -87,7 +74,6 @@ cc_library( srcs = ["qr.cc"], hdrs = ["qr.h"], deps = [ - ":batch_dot", ":util", ":while_loop", "//tensorflow/compiler/xla:literal_util", @@ -99,7 +85,8 @@ cc_library( "//tensorflow/compiler/xla/client/lib:arithmetic", "//tensorflow/compiler/xla/client/lib:constants", "//tensorflow/compiler/xla/client/lib:math", - "//tensorflow/compiler/xla/client/lib:numeric", + "//tensorflow/compiler/xla/client/lib:matrix", + "//tensorflow/compiler/xla/client/lib:slicing", "//tensorflow/core:lib", ], ) @@ -129,7 +116,6 @@ cc_library( srcs = ["triangular_solve.cc"], hdrs = ["triangular_solve.h"], deps = [ - ":batch_dot", ":util", "//tensorflow/compiler/xla:literal", "//tensorflow/compiler/xla:shape_util", @@ -140,7 +126,9 @@ cc_library( "//tensorflow/compiler/xla/client:xla_builder", "//tensorflow/compiler/xla/client:xla_computation", "//tensorflow/compiler/xla/client/lib:constants", - "//tensorflow/compiler/xla/client/lib:numeric", + "//tensorflow/compiler/xla/client/lib:math", + "//tensorflow/compiler/xla/client/lib:matrix", + "//tensorflow/compiler/xla/client/lib:slicing", "//tensorflow/core:lib", ], ) @@ -187,29 +175,6 @@ cc_library( ], ) -xla_test( - name = "util_test", - srcs = ["util_test.cc"], - deps = [ - ":batch_dot", - ":util", - "//tensorflow/compiler/xla:array2d", - "//tensorflow/compiler/xla:literal", - "//tensorflow/compiler/xla:shape_util", - "//tensorflow/compiler/xla:statusor", - "//tensorflow/compiler/xla:test", - "//tensorflow/compiler/xla:types", - "//tensorflow/compiler/xla:xla_data_proto", - "//tensorflow/compiler/xla/client:global_data", - "//tensorflow/compiler/xla/client:local_client", - "//tensorflow/compiler/xla/tests:client_library_test_base", - "//tensorflow/compiler/xla/tests:literal_test_util", - "//tensorflow/compiler/xla/tests:xla_internal_test_main", - "//tensorflow/core:lib", - "//tensorflow/core:test", - ], -) - cc_library( name = "while_loop", srcs = ["while_loop.cc"], diff --git a/tensorflow/compiler/tf2xla/lib/batch_dot.cc b/tensorflow/compiler/tf2xla/lib/batch_dot.cc deleted file mode 100644 index 5400e8834cb..00000000000 --- a/tensorflow/compiler/tf2xla/lib/batch_dot.cc +++ /dev/null @@ -1,115 +0,0 @@ -/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -#include "tensorflow/compiler/tf2xla/lib/batch_dot.h" - -#include -#include - -#include "tensorflow/compiler/xla/client/xla_builder.h" -#include "tensorflow/compiler/xla/shape_util.h" -#include "tensorflow/compiler/xla/status_macros.h" -#include "tensorflow/compiler/xla/statusor.h" -#include "tensorflow/core/lib/core/errors.h" - -namespace tensorflow { - -xla::XlaOp BatchDot(xla::XlaOp x, xla::XlaOp y, bool transpose_x, - bool transpose_y, bool conjugate_x, bool conjugate_y, - xla::PrecisionConfig::Precision precision) { - xla::XlaBuilder* builder = x.builder(); - return builder->ReportErrorOrReturn([&]() -> xla::StatusOr { - TF_ASSIGN_OR_RETURN(xla::Shape x_shape, builder->GetShape(x)); - TF_ASSIGN_OR_RETURN(xla::Shape y_shape, builder->GetShape(y)); - - // Check that both tensors have the same number of dimensions. There must be - // at least two (the batch dimensions can be empty). - if (xla::ShapeUtil::Rank(x_shape) != xla::ShapeUtil::Rank(y_shape)) { - return errors::InvalidArgument( - "Arguments to BatchedDot have different ranks: ", - xla::ShapeUtil::HumanString(x_shape), " vs. ", - xla::ShapeUtil::HumanString(y_shape)); - } - const int ndims = xla::ShapeUtil::Rank(x_shape); - if (ndims < 2) { - return errors::InvalidArgument( - "Arguments to BatchedDot must have rank >= 2: ", ndims); - } - - // The batch dimensions must be equal and the matrix dimensions must be - // valid. - std::vector batch_dimension_numbers; - for (int i = 0; i < ndims - 2; ++i) { - if (x_shape.dimensions(i) != y_shape.dimensions(i)) { - return errors::InvalidArgument( - "Dimension ", i, " of inputs to BatchedDot must be equal: ", - xla::ShapeUtil::HumanString(x_shape), " vs ", - xla::ShapeUtil::HumanString(y_shape)); - } - batch_dimension_numbers.push_back(i); - } - - int x_inner_dim = transpose_x ? (ndims - 2) : (ndims - 1); - int y_inner_dim = transpose_y ? (ndims - 1) : (ndims - 2); - if (x_shape.dimensions(x_inner_dim) != y_shape.dimensions(y_inner_dim)) { - return errors::InvalidArgument( - "Dimensions ", x_inner_dim, " and ", y_inner_dim, - " of arguments to BatchedDot must be equal: ", - xla::ShapeUtil::HumanString(x_shape), " transpose: ", transpose_x, - " vs. ", xla::ShapeUtil::HumanString(y_shape), - " transpose: ", transpose_y); - } - - // Check for zero lhs/rhs dim size. - if (xla::ShapeUtil::IsZeroElementArray(x_shape) || - xla::ShapeUtil::IsZeroElementArray(y_shape)) { - std::vector dimensions(batch_dimension_numbers.size()); - for (int i = 0; i < batch_dimension_numbers.size(); ++i) { - dimensions[i] = x_shape.dimensions(batch_dimension_numbers[i]); - } - int x_outer_dim = transpose_x ? (ndims - 1) : (ndims - 2); - int y_outer_dim = transpose_y ? (ndims - 2) : (ndims - 1); - dimensions.push_back(x_shape.dimensions(x_outer_dim)); - dimensions.push_back(y_shape.dimensions(y_outer_dim)); - return xla::Broadcast( - xla::ConstantLiteral(builder, - xla::LiteralUtil::Zero(x_shape.element_type())), - dimensions); - } - - if (x_shape.element_type() == xla::C64 && conjugate_x) { - x = xla::Conj(x); - } - if (y_shape.element_type() == xla::C64 && conjugate_y) { - y = xla::Conj(y); - } - - xla::PrecisionConfig precision_proto; - precision_proto.add_operand_precision(precision); - precision_proto.add_operand_precision(precision); - - xla::DotDimensionNumbers dot_dnums; - dot_dnums.add_lhs_contracting_dimensions(x_inner_dim); - dot_dnums.add_rhs_contracting_dimensions(y_inner_dim); - for (auto batch_dimension_number : batch_dimension_numbers) { - dot_dnums.add_lhs_batch_dimensions(batch_dimension_number); - dot_dnums.add_rhs_batch_dimensions(batch_dimension_number); - } - - return xla::DotGeneral(x, y, dot_dnums, &precision_proto); - }); -} - -} // namespace tensorflow diff --git a/tensorflow/compiler/tf2xla/lib/batch_dot.h b/tensorflow/compiler/tf2xla/lib/batch_dot.h deleted file mode 100644 index 6edd63a4d3b..00000000000 --- a/tensorflow/compiler/tf2xla/lib/batch_dot.h +++ /dev/null @@ -1,54 +0,0 @@ -/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -#ifndef TENSORFLOW_COMPILER_TF2XLA_LIB_BATCH_DOT_H_ -#define TENSORFLOW_COMPILER_TF2XLA_LIB_BATCH_DOT_H_ - -#include "tensorflow/compiler/xla/client/xla_builder.h" -#include "tensorflow/compiler/xla/xla_data.pb.h" - -namespace tensorflow { - -// Multiplies slices of two tensors in batches. - -// Multiplies all slices of `Tensor` `x` and `y` (each slice can be -// viewed as an element of a batch), and arranges the individual results -// in a single output tensor of the same batch size. Each of the -// individual slices can optionally be transposed before multiplication by -// setting the `transpose_x` or `transpose_y` flag to `true`. Similarly, each -// can be elementwise-complex-conjugated by setting the `conjugate_x` or -// `conjugate_y` flag to `true`. To apply a Hermitian adjoint to `x`, set both -// `transpose_x` and `conjugate_x` to `true`, and analogously for `y`. -// -// The input tensors `x` and `y` are 2-D or higher with shape `[..., r_x, c_x]` -// and `[..., r_y, c_y]`. -// -// The output tensor is 2-D or higher with shape `[..., r_o, c_o]`, where: -// -// r_o = c_x if transpose_x else r_x -// c_o = r_y if transpose_y else c_y -// -// It is computed as: -// -// output[..., :, :] = matrix(x[..., :, :]) * matrix(y[..., :, :]) -xla::XlaOp BatchDot( - xla::XlaOp x, xla::XlaOp y, bool transpose_x = false, - bool transpose_y = false, bool conjugate_x = false, - bool conjugate_y = false, - xla::PrecisionConfig::Precision precision = xla::PrecisionConfig::DEFAULT); - -} // namespace tensorflow - -#endif // TENSORFLOW_COMPILER_TF2XLA_LIB_BATCH_DOT_H_ diff --git a/tensorflow/compiler/tf2xla/lib/broadcast.cc b/tensorflow/compiler/tf2xla/lib/broadcast.cc index 3e402ef855c..be31f116686 100644 --- a/tensorflow/compiler/tf2xla/lib/broadcast.cc +++ b/tensorflow/compiler/tf2xla/lib/broadcast.cc @@ -80,10 +80,8 @@ xla::StatusOr BroadcastTo(xla::XlaOp input, broadcast_dim = broadcast_shape_size - broadcast_dim - 1; } absl::c_reverse(broadcast_shape); - xla::XlaOp output = xla::BroadcastInDim( - input, - xla::ShapeUtil::MakeShape(input_shape.element_type(), broadcast_shape), - broadcast_dims); + xla::XlaOp output = + xla::BroadcastInDim(input, broadcast_shape, broadcast_dims); if (broadcast_shape != output_dims) { output = xla::Reshape(output, output_dims); } diff --git a/tensorflow/compiler/tf2xla/lib/cholesky.cc b/tensorflow/compiler/tf2xla/lib/cholesky.cc index ab3d0a56683..7ef8659992f 100644 --- a/tensorflow/compiler/tf2xla/lib/cholesky.cc +++ b/tensorflow/compiler/tf2xla/lib/cholesky.cc @@ -18,11 +18,12 @@ limitations under the License. #include #include -#include "tensorflow/compiler/tf2xla/lib/batch_dot.h" #include "tensorflow/compiler/tf2xla/lib/triangular_solve.h" #include "tensorflow/compiler/tf2xla/lib/util.h" #include "tensorflow/compiler/tf2xla/lib/while_loop.h" #include "tensorflow/compiler/xla/client/lib/constants.h" +#include "tensorflow/compiler/xla/client/lib/matrix.h" +#include "tensorflow/compiler/xla/client/lib/slicing.h" #include "tensorflow/compiler/xla/client/xla_builder.h" #include "tensorflow/compiler/xla/literal.h" #include "tensorflow/compiler/xla/shape_util.h" @@ -101,10 +102,7 @@ xla::XlaOp CholeskyUnblocked(xla::XlaOp a, // a[..., i, i] auto a_ii = DynamicSliceInMinorDims(body_a, {i, i}, {1, 1}); // np.dot(row, np.swapaxes(row, -1, -2)) - auto diag_dot = BatchDot(row, row, - /*transpose_x=*/false, - /*transpose_y=*/true, /*conjugate_x=*/false, - /*conjugate_y=*/false, precision); + auto diag_dot = BatchDot(row, TransposeInMinorDims(row), precision); // l[..., i, i] = np.sqrt(a[..., i, i] - np.dot(row, // np.swapaxes(row, -1, -2))) auto l_ii = @@ -122,10 +120,7 @@ xla::XlaOp CholeskyUnblocked(xla::XlaOp a, // The columns in [i, n] are zeroed out in `row`, so we just have to // zero out rows above i+1 after the BatchDot. np.dot(l[..., :, :i], // r.T) - auto dot = BatchDot(body_l, row, - /*transpose_x=*/false, - /*transpose_y=*/true, /*conjugate_x=*/false, - /*conjugate_y=*/false, precision); + auto dot = BatchDot(body_l, TransposeInMinorDims(row), precision); // np.dot(l[..., i+1:, :i], r.T) auto dot_ip1 = xla::Select(xla::Le(mask_range_col, i), mask_zeros_col, dot); @@ -185,9 +180,7 @@ xla::XlaOp Cholesky(xla::XlaOp a, int64 block_size, // a[i:, i:i+k] -= np.dot(l[i:, :i], np.transpose(l[i:i+k, :i])) auto lhs = SliceInMinorDims(l, {i, 0}, {n, i}); auto rhs = SliceInMinorDims(l, {i, 0}, {i + k, i}); - auto delta = BatchDot(lhs, rhs, /*transpose_x=*/false, - /*transpose_y=*/true, /*conjugate_x=*/false, - /*conjugate_y=*/false, precision); + auto delta = BatchDot(lhs, TransposeInMinorDims(rhs), precision); auto before = SliceInMinorDims(a, {i, i}, {n, i + k}); a = UpdateSliceInMinorDims(a, before - delta, {i, i}); } diff --git a/tensorflow/compiler/tf2xla/lib/qr.cc b/tensorflow/compiler/tf2xla/lib/qr.cc index 6b3f2b6e065..d6007748609 100644 --- a/tensorflow/compiler/tf2xla/lib/qr.cc +++ b/tensorflow/compiler/tf2xla/lib/qr.cc @@ -18,13 +18,13 @@ limitations under the License. #include #include -#include "tensorflow/compiler/tf2xla/lib/batch_dot.h" #include "tensorflow/compiler/tf2xla/lib/util.h" #include "tensorflow/compiler/tf2xla/lib/while_loop.h" #include "tensorflow/compiler/xla/client/lib/arithmetic.h" #include "tensorflow/compiler/xla/client/lib/constants.h" #include "tensorflow/compiler/xla/client/lib/math.h" -#include "tensorflow/compiler/xla/client/lib/numeric.h" +#include "tensorflow/compiler/xla/client/lib/matrix.h" +#include "tensorflow/compiler/xla/client/lib/slicing.h" #include "tensorflow/compiler/xla/client/xla_builder.h" #include "tensorflow/compiler/xla/literal_util.h" #include "tensorflow/compiler/xla/shape_util.h" @@ -191,12 +191,8 @@ xla::StatusOr QRBlock( auto v_broadcast = xla::Reshape(v, shape); // a[:, :] -= tau * np.dot(v[:, np.newaxis], // np.dot(v[np.newaxis, :], a[:, :])) - auto vva = - BatchDot(v_broadcast, a, /*transpose_x=*/false, /*transpose_y=*/false, - /*conjugate_x=*/false, /*conjugate_y=*/false, precision); - vva = - BatchDot(v_broadcast, vva, /*transpose_x=*/true, /*transpose_y=*/false, - /*conjugate_x=*/false, /*conjugate_y=*/false, precision); + auto vva = BatchDot(v_broadcast, a, precision); + vva = BatchDot(TransposeInMinorDims(v_broadcast), vva, precision); a = a - xla::Mul(tau, vva, /*broadcast_dimensions=*/batch_dim_indices); @@ -278,12 +274,9 @@ xla::StatusOr ComputeWYRepresentation( auto beta = DynamicSliceInMinorDims(taus, {j}, {1}); // yv has shape [..., n, 1] - auto yv = BatchDot(y, v, /*transpose_x=*/true, /*transpose_y=*/false, - /*conjugate_x=*/false, /*conjugate_y=*/false, precision); + auto yv = BatchDot(TransposeInMinorDims(y), v, precision); // wyv has shape [..., m, 1] - auto wyv = - BatchDot(w, yv, /*transpose_x=*/false, /*transpose_y=*/false, - /*conjugate_x=*/false, /*conjugate_y=*/false, precision); + auto wyv = BatchDot(w, yv, precision); auto z = xla::Mul( -beta, v + wyv, @@ -375,23 +368,15 @@ xla::StatusOr QRDecomposition( // a[i:, i+k:] += np.dot(Y, np.dot(W.T, a[i:, i+k:])) auto a_panel = SliceInMinorDims(a, {i, i + k}, {m, n}); - auto a_update = - BatchDot(w, a_panel, /*transpose_x=*/true, /*transpose_y=*/false, - /*conjugate_x=*/false, /*conjugate_y=*/false, precision); - a_update = - BatchDot(y, a_update, /*transpose_x=*/false, /*transpose_y=*/false, - /*conjugate_x=*/false, /*conjugate_y=*/false, precision); + auto a_update = BatchDot(TransposeInMinorDims(w), a_panel, precision); + a_update = BatchDot(y, a_update, precision); a_panel = a_panel + a_update; a = UpdateSliceInMinorDims(a, a_panel, {i, i + k}); // q[:, i:] += np.dot(np.dot(q[:, i:], W), Y.T)) auto q_panel = SliceInMinorDims(q, {0, i}, {m, m}); - auto q_update = - BatchDot(q_panel, w, /*transpose_x=*/false, /*transpose_y=*/false, - /*conjugate_x=*/false, /*conjugate_y=*/false, precision); - q_update = BatchDot(q_update, y, /*transpose_x=*/false, - /*transpose_y=*/true, /*conjugate_x=*/false, - /*conjugate_y=*/false, precision); + auto q_update = BatchDot(q_panel, w, precision); + q_update = BatchDot(q_update, TransposeInMinorDims(y), precision); q_panel = q_panel + q_update; q = UpdateSliceInMinorDims(q, q_panel, {0, i}); } diff --git a/tensorflow/compiler/tf2xla/lib/triangular_solve.cc b/tensorflow/compiler/tf2xla/lib/triangular_solve.cc index 6524c2a9b1a..192a61dca26 100644 --- a/tensorflow/compiler/tf2xla/lib/triangular_solve.cc +++ b/tensorflow/compiler/tf2xla/lib/triangular_solve.cc @@ -18,10 +18,11 @@ limitations under the License. #include #include -#include "tensorflow/compiler/tf2xla/lib/batch_dot.h" #include "tensorflow/compiler/tf2xla/lib/util.h" #include "tensorflow/compiler/xla/client/lib/constants.h" -#include "tensorflow/compiler/xla/client/lib/numeric.h" +#include "tensorflow/compiler/xla/client/lib/math.h" +#include "tensorflow/compiler/xla/client/lib/matrix.h" +#include "tensorflow/compiler/xla/client/lib/slicing.h" #include "tensorflow/compiler/xla/client/xla_builder.h" #include "tensorflow/compiler/xla/client/xla_computation.h" #include "tensorflow/compiler/xla/literal.h" @@ -311,13 +312,13 @@ xla::XlaOp SolveWithInvertedDiagonalBlocks( auto a_row = MaybeConjugate(SliceInMinorDims(a, start, end), conjugate_a); if (left_side) { - remainder = b_row - BatchDot(a_row, x, transpose_a, false, - /*conjugate_x=*/false, - /*conjugate_y=*/false, precision); + remainder = + b_row - BatchDot(MaybeTransposeInMinorDims(a_row, transpose_a), x, + precision); } else { - remainder = b_row - BatchDot(x, a_row, false, transpose_a, - /*conjugate_x=*/false, - /*conjugate_y=*/false, precision); + remainder = + b_row - BatchDot(x, MaybeTransposeInMinorDims(a_row, transpose_a), + precision); } } @@ -327,13 +328,12 @@ xla::XlaOp SolveWithInvertedDiagonalBlocks( xla::ConstantR0WithType(builder, xla::S32, j * block_size); std::vector update_starts = {start_index, zero}; if (left_side) { - x_update = - BatchDot(inv_block, remainder, transpose_a, false, - /*conjugate_x=*/false, /*conjugate_y=*/false, precision); + x_update = BatchDot(MaybeTransposeInMinorDims(inv_block, transpose_a), + remainder, precision); } else { - x_update = - BatchDot(remainder, inv_block, false, transpose_a, - /*conjugate_x=*/false, /*conjugate_y=*/false, precision); + x_update = BatchDot(remainder, + MaybeTransposeInMinorDims(inv_block, transpose_a), + precision); std::swap(update_starts[0], update_starts[1]); } x = DynamicUpdateSliceInMinorDims(x, x_update, /*starts=*/update_starts); diff --git a/tensorflow/compiler/tf2xla/lib/util.cc b/tensorflow/compiler/tf2xla/lib/util.cc index 804671fbc75..c0bd172d17c 100644 --- a/tensorflow/compiler/tf2xla/lib/util.cc +++ b/tensorflow/compiler/tf2xla/lib/util.cc @@ -113,36 +113,6 @@ xla::XlaOp IntegerLiteral(xla::XlaBuilder* builder, xla::PrimitiveType type, return xla::ConstantLiteral(builder, literal); } -xla::XlaOp SliceInMinorDims(xla::XlaOp x, absl::Span start, - absl::Span end) { - xla::XlaBuilder* builder = x.builder(); - return builder->ReportErrorOrReturn([&]() -> xla::StatusOr { - TF_RET_CHECK(start.size() == end.size()); - int64 n_minor_dims = start.size(); - - TF_ASSIGN_OR_RETURN(xla::Shape shape, builder->GetShape(x)); - - const int64 n_dims = xla::ShapeUtil::Rank(shape); - TF_RET_CHECK(n_minor_dims <= n_dims); - auto major_dims = xla::AsInt64Slice(shape.dimensions()) - .subspan( - /*pos=*/0, - /*len=*/n_dims - n_minor_dims); - - // Prepends 0s in the major dim - std::vector padded_start(n_dims, 0); - std::copy(start.begin(), start.end(), - padded_start.begin() + major_dims.size()); - - // Prepends the shape of the major dims. - std::vector padded_end(n_dims); - std::copy(major_dims.begin(), major_dims.end(), padded_end.begin()); - std::copy(end.begin(), end.end(), padded_end.begin() + major_dims.size()); - - std::vector strides(n_dims, 1); - return xla::Slice(x, padded_start, padded_end, strides); - }); -} std::vector ConcatVectors(absl::Span xs, absl::Span ys) { @@ -152,100 +122,4 @@ std::vector ConcatVectors(absl::Span xs, return output; } -xla::XlaOp DynamicSliceInMinorDims(xla::XlaOp x, - absl::Span starts, - absl::Span sizes) { - xla::XlaBuilder* builder = x.builder(); - return builder->ReportErrorOrReturn([&]() -> xla::StatusOr { - TF_ASSIGN_OR_RETURN(xla::Shape shape, builder->GetShape(x)); - const int64 n_dims = xla::ShapeUtil::Rank(shape); - int64 n_minor_dims = starts.size(); - TF_RET_CHECK(n_minor_dims == sizes.size()); - TF_RET_CHECK(n_minor_dims <= n_dims); - auto major_dims = xla::AsInt64Slice(shape.dimensions()) - .subspan( - /*pos=*/0, - /*len=*/n_dims - sizes.size()); - auto padded_starts = PrependZerosInMajorDims(x, starts); - auto padded_sizes = ConcatVectors(major_dims, sizes); - return xla::DynamicSlice(x, padded_starts, padded_sizes); - }); -} - -xla::XlaOp UpdateSlice(xla::XlaOp x, xla::XlaOp update, - absl::Span start) { - xla::XlaBuilder* builder = x.builder(); - return builder->ReportErrorOrReturn([&]() -> xla::StatusOr { - // TODO(phawkins): make int64 work on all backends, remove the int32 cast. - std::vector start_as_int32(start.begin(), start.end()); - auto start_constant = xla::ConstantR1(builder, start_as_int32); - TF_ASSIGN_OR_RETURN(xla::Shape shape, builder->GetShape(x)); - const int64 n_dims = xla::ShapeUtil::Rank(shape); - TF_ASSIGN_OR_RETURN(xla::Shape start_constant_shape, - builder->GetShape(start_constant)); - const int64 start_length = - xla::ShapeUtil::GetDimension(start_constant_shape, -1); - TF_RET_CHECK(start_length == n_dims); - return xla::DynamicUpdateSlice(x, update, start_constant); - }); -} - -xla::XlaOp UpdateSliceInMinorDims(xla::XlaOp x, xla::XlaOp update, - absl::Span start) { - xla::XlaBuilder* builder = x.builder(); - return builder->ReportErrorOrReturn([&]() -> xla::StatusOr { - TF_ASSIGN_OR_RETURN(xla::Shape shape, builder->GetShape(x)); - const int64 n_dims = xla::ShapeUtil::Rank(shape); - const int64 n_minor_dims = start.size(); - TF_RET_CHECK(n_minor_dims <= n_dims); - std::vector padded_start(n_dims, 0); - std::copy(start.begin(), start.end(), - padded_start.begin() + (n_dims - n_minor_dims)); - return UpdateSlice(x, update, padded_start); - }); -} - -xla::XlaOp DynamicUpdateSliceInMinorDims(xla::XlaOp x, xla::XlaOp update, - absl::Span starts) { - auto padded_starts = PrependZerosInMajorDims(x, starts); - return xla::DynamicUpdateSlice(x, update, padded_starts); -} - -xla::XlaOp PrependZerosInMajorDims(xla::XlaOp x, - absl::Span starts) { - xla::XlaBuilder* builder = x.builder(); - return builder->ReportErrorOrReturn([&]() -> xla::StatusOr { - TF_ASSIGN_OR_RETURN(xla::Shape shape, builder->GetShape(x)); - const int64 n_dims = xla::ShapeUtil::Rank(shape); - auto zero = xla::Reshape(xla::ConstantR0(builder, 0), {1}); - std::vector padded_starts(n_dims, zero); - for (int i = 0; i < starts.size(); ++i) { - padded_starts[n_dims - starts.size() + i] = xla::Reshape(starts[i], {1}); - } - return xla::ConcatInDim(builder, padded_starts, 0); - }); -} - -xla::XlaOp TransposeInMinorDims(xla::XlaOp x) { - xla::XlaBuilder* builder = x.builder(); - return builder->ReportErrorOrReturn([&]() -> xla::StatusOr { - TF_ASSIGN_OR_RETURN(xla::Shape shape, builder->GetShape(x)); - const int64 n_dims = xla::ShapeUtil::Rank(shape); - TF_RET_CHECK(n_dims >= 2); - std::vector permutation(n_dims); - std::iota(permutation.begin(), permutation.end(), 0); - std::swap(permutation[n_dims - 1], permutation[n_dims - 2]); - return xla::Transpose(x, permutation); - }); -} - -xla::XlaOp MaybeConjugate(xla::XlaOp x, bool conjugate) { - xla::XlaBuilder* builder = x.builder(); - return builder->ReportErrorOrReturn([&]() -> xla::StatusOr { - TF_ASSIGN_OR_RETURN(xla::Shape shape, builder->GetShape(x)); - auto perform_conj = shape.element_type() == xla::C64 && conjugate; - return perform_conj ? xla::Conj(x) : x; - }); -} - } // namespace tensorflow diff --git a/tensorflow/compiler/tf2xla/lib/util.h b/tensorflow/compiler/tf2xla/lib/util.h index 80e9e5b002d..aec8061cb43 100644 --- a/tensorflow/compiler/tf2xla/lib/util.h +++ b/tensorflow/compiler/tf2xla/lib/util.h @@ -38,44 +38,10 @@ xla::XlaOp PrependZerosInMajorDims(xla::XlaOp x, xla::XlaOp IntegerLiteral(xla::XlaBuilder* builder, xla::PrimitiveType type, int64 value); -// Builds a vector of zeros of length rank(x) with the last values being -// those in `starts`. -xla::XlaOp PrependZerosInMajorDims(xla::XlaOp x, - absl::Span starts); - -// Performs a slice in the minor dimensions of a Tensor. -xla::XlaOp SliceInMinorDims(xla::XlaOp x, absl::Span start, - absl::Span end); - // Returns the concatenation of `xs` and `ys`. std::vector ConcatVectors(absl::Span xs, absl::Span ys); -// Performs a dynamic slice in the minor dimensions of a Tensor. -xla::XlaOp DynamicSliceInMinorDims(xla::XlaOp x, - absl::Span starts, - absl::Span sizes); - -// Updates a slice of 'x', i.e., -// x[start[0], ..., start[n]] = update -xla::XlaOp UpdateSlice(xla::XlaOp x, xla::XlaOp update, - absl::Span start); - -// Updates a slice of 'x', where 'start' contains a list of minor dimensions: -// x[..., start[0], ..., start[n]] = update -xla::XlaOp UpdateSliceInMinorDims(xla::XlaOp x, xla::XlaOp update, - absl::Span start); - -xla::XlaOp DynamicUpdateSliceInMinorDims(xla::XlaOp x, xla::XlaOp update, - absl::Span starts); - -// Transposes a stack of matrices `x` by swapping the last two dimensions. -xla::XlaOp TransposeInMinorDims(xla::XlaOp x); - -// Applies a complex conjugation operation if `a` is complex and `conjugate_a` -// is true, otherwise returns its argument. -xla::XlaOp MaybeConjugate(xla::XlaOp x, bool conjugate); - } // namespace tensorflow #endif // TENSORFLOW_COMPILER_TF2XLA_LIB_UTIL_H_ diff --git a/tensorflow/compiler/tf2xla/python/BUILD b/tensorflow/compiler/tf2xla/python/BUILD index c9f486edc8d..fef97b98c37 100644 --- a/tensorflow/compiler/tf2xla/python/BUILD +++ b/tensorflow/compiler/tf2xla/python/BUILD @@ -1,11 +1,13 @@ licenses(["notice"]) # Apache 2.0 +package_group( + name = "friends", + includes = ["//tensorflow:internal"], +) + package( default_visibility = [ - "//learning/deepmind/public/wavenet/python:__subpackages__", - "//learning/deepmind/research/alphastar:__subpackages__", - "//learning/tfx:__subpackages__", - "//tensorflow:internal", + ":friends", ], ) diff --git a/tensorflow/compiler/tf2xla/shape_util.h b/tensorflow/compiler/tf2xla/shape_util.h index f7e34a5b40c..0b231ea8e7a 100644 --- a/tensorflow/compiler/tf2xla/shape_util.h +++ b/tensorflow/compiler/tf2xla/shape_util.h @@ -18,6 +18,7 @@ limitations under the License. #ifndef TENSORFLOW_COMPILER_TF2XLA_SHAPE_UTIL_H_ #define TENSORFLOW_COMPILER_TF2XLA_SHAPE_UTIL_H_ +#include "tensorflow/compiler/xla/shape.h" #include "tensorflow/compiler/xla/xla_data.pb.h" #include "tensorflow/core/framework/tensor_shape.h" #include "tensorflow/core/framework/types.pb.h" diff --git a/tensorflow/compiler/tf2xla/xla_compiled_cpu_function.h b/tensorflow/compiler/tf2xla/xla_compiled_cpu_function.h index 425e769346f..c7341cf8b9e 100644 --- a/tensorflow/compiler/tf2xla/xla_compiled_cpu_function.h +++ b/tensorflow/compiler/tf2xla/xla_compiled_cpu_function.h @@ -26,7 +26,7 @@ limitations under the License. // Forward-declare, rather than include, to reduce code size for users that // never use this functionality. namespace xla { -class ProgramShape; +class ProgramShapeProto; class HloProfilePrinterData; } @@ -84,7 +84,7 @@ class XlaCompiledCpuFunction { void set_result_names(const char** result_names) { result_names_ = result_names; } - void set_program_shape(const xla::ProgramShape* program_shape) { + void set_program_shape(const xla::ProgramShapeProto* program_shape) { program_shape_ = program_shape; } const xla::HloProfilePrinterData* hlo_profile_printer_data() const { @@ -122,7 +122,7 @@ class XlaCompiledCpuFunction { const char** result_names_ = nullptr; // [Optional] Arg and result shapes. - const xla::ProgramShape* program_shape_ = nullptr; + const xla::ProgramShapeProto* program_shape_ = nullptr; // [Optional] Profile printer data. Null if profiling is disabled. const xla::HloProfilePrinterData* hlo_profile_printer_data_ = nullptr; @@ -206,8 +206,14 @@ class XlaCompiledCpuFunction { // // Aliasing of argument and result buffers is not allowed, and results in // undefined behavior. - void set_arg_data(size_t index, void* data) { - buffer_table_[arg_index_table_[index]] = data; + void set_arg_data(size_t index, const void* data) { + // The const_cast is safe because the generated code does not write to arg + // buffers. + // + // buffer_table_ contains pointers to buffers that _will_ be written to by + // generated code so it would be misleading to make buffer_table_ a `const + // void**`. + buffer_table_[arg_index_table_[index]] = const_cast(data); } // ------------------------------ @@ -264,7 +270,7 @@ class XlaCompiledCpuFunction { // Returns the shape of the args and results. May return nullptr if the // program shape isn't available. - const xla::ProgramShape* ProgramShape() const { return program_shape_; } + const xla::ProgramShapeProto* ProgramShape() const { return program_shape_; } bool hlo_profiling_enabled() const { return hlo_profile_printer_data_ != nullptr; @@ -287,11 +293,6 @@ class XlaCompiledCpuFunction { // Argument i needs to be placed in buffer_table_[arg_index_to_temp_index_[i]] // for XLA generated code to be able to find it. - // - // For now we need to keep around the args_ array because there is code that - // depends on args() returning a void**. However, in the future we may remove - // args_ in favor of using buffer_table_ as the sole storage for the - // arguments. const int32* const arg_index_table_; // The number of incoming arguments. @@ -310,7 +311,7 @@ class XlaCompiledCpuFunction { // Optional metadata. const char** arg_names_ = nullptr; const char** result_names_ = nullptr; - const xla::ProgramShape* program_shape_ = nullptr; + const xla::ProgramShapeProto* program_shape_ = nullptr; const xla::HloProfilePrinterData* hlo_profile_printer_data_ = nullptr; }; diff --git a/tensorflow/compiler/tf2xla/xla_compiler.cc b/tensorflow/compiler/tf2xla/xla_compiler.cc index a08d030ce71..ee461a3c07d 100644 --- a/tensorflow/compiler/tf2xla/xla_compiler.cc +++ b/tensorflow/compiler/tf2xla/xla_compiler.cc @@ -158,7 +158,8 @@ Status BuildComputation( xla::XlaBuilder* builder, xla::XlaComputation* computation, int* num_computation_outputs, int* num_nonconst_outputs, std::vector* outputs, - std::vector* resource_updates) { + std::vector* resource_updates, + xla::Shape* output_shape) { // Attach a common operator name as metadata. This has no semantic effect — it // merely makes the HLO graph more readable when visualized via TensorBoard, // since TensorBoard forms groups out of operators with similar names. @@ -176,6 +177,10 @@ Status BuildComputation( std::vector elems; elems.reserve(retvals.size()); + + // Keeps track of which retvals have layout to update. The first element is + // the output index, second element is the new layout. + std::vector> retval_to_update_layout; for (int i = 0; i < retvals.size(); ++i) { XlaCompiler::OutputDescription& output = (*outputs)[i]; const XlaExpression& retval = retvals[i]; @@ -202,10 +207,12 @@ Status BuildComputation( TF_ASSIGN_OR_RETURN(xla::Shape shape, shape_representation_fn( output.shape, output.type)); value = xla::Reshape(value, xla::AsInt64Slice(shape.dimensions())); + retval_to_update_layout.emplace_back(elems.size(), shape.layout()); } else if (it != retval_cores.end()) { // Apply the sharding to the output, if there is a core assignment. value = identity_op(value); } + elems.push_back(value); break; } @@ -297,6 +304,21 @@ Status BuildComputation( return computation_status.status(); } *computation = computation_status.ConsumeValueOrDie(); + + TF_ASSIGN_OR_RETURN(const auto& program_shape, + computation->GetProgramShape()); + *output_shape = program_shape.result(); + // Update the output layout to the layout of retval. + for (auto& update : retval_to_update_layout) { + if (!always_return_tuple && elems.size() == 1) { + *output_shape->mutable_layout() = update.second; + continue; + } + + xla::Shape* output_sub_shape = + xla::ShapeUtil::GetMutableSubshape(output_shape, {update.first}); + *output_sub_shape->mutable_layout() = update.second; + } return Status::OK(); } @@ -304,10 +326,10 @@ Status BuildComputation( bool XlaCompiler::Argument::operator==( const XlaCompiler::Argument& other) const { - if (std::tie(kind, resource_kind, type, name, initialized, tensor_array_size, + if (std::tie(kind, resource_kind, type, name, initialized, max_array_size, tensor_array_gradients) != std::tie(other.kind, other.resource_kind, other.type, other.name, - other.initialized, other.tensor_array_size, + other.initialized, other.max_array_size, other.tensor_array_gradients)) { return false; } @@ -337,8 +359,8 @@ string XlaCompiler::Argument::HumanString() const { string output = absl::StrCat("kind=resource", common, " resource_kind=", XlaResource::KindToString(resource_kind), " initialized=", initialized); - if (tensor_array_size >= 0) { - absl::StrAppend(&output, " tensor_array_size=", tensor_array_size); + if (max_array_size >= 0) { + absl::StrAppend(&output, " max_array_size=", max_array_size); } if (!tensor_array_gradients.empty()) { absl::StrAppend(&output, " tensor_array_gradients=", @@ -358,7 +380,7 @@ XlaCompiler::XlaCompiler(XlaCompiler::Options options) initialization_status_(Status::OK()), next_step_id_(1), device_(new XlaCompilationDevice(SessionOptions(), options_.device_type)), - device_mgr_({device_}) { + device_mgr_(absl::WrapUnique(device_)) { CHECK(!options_.device_type.type_string().empty()); if (options_.populate_resource_manager) { initialization_status_ = @@ -545,12 +567,12 @@ Status XlaCompiler::XLAShapeForArgument(const XlaCompiler::Argument& arg, return Status::OK(); } case XlaResource::kTensorArray: { - if (arg.tensor_array_size < 0) { + if (arg.max_array_size < 0) { return errors::InvalidArgument( - "Negative tensor_array_size in XLAShapeForArgument"); + "Negative max_array_size in XLAShapeForArgument"); } TensorShape shape; - shape.AddDim(arg.tensor_array_size); + shape.AddDim(arg.max_array_size); shape.AppendShape(arg.shape); TF_RETURN_IF_ERROR(TensorShapeToXLAShape(arg.type, shape, xla_shape)); @@ -562,12 +584,12 @@ Status XlaCompiler::XLAShapeForArgument(const XlaCompiler::Argument& arg, return Status::OK(); } case XlaResource::kStack: { - if (arg.tensor_array_size < 0) { + if (arg.max_array_size < 0) { return errors::InvalidArgument( - "Negative tensor_array_size in XLAShapeForArgument"); + "Negative max_array_size in XLAShapeForArgument"); } TensorShape shape; - shape.AddDim(arg.tensor_array_size); + shape.AddDim(arg.max_array_size); shape.AppendShape(arg.shape); xla::Shape buffer_shape; TF_RETURN_IF_ERROR( @@ -613,21 +635,23 @@ Status XlaCompiler::BuildArguments( const XlaCompiler::Argument& arg = args[i]; XlaExpression& arg_expression = (*arg_expressions)[i]; switch (arg.kind) { - case XlaCompiler::Argument::kResource: + case XlaCompiler::Argument::kResource: { TF_RET_CHECK(arg.resource_kind != XlaResource::kInvalid); // TODO(phawkins): this code assumes that resource arguments do not // alias. - XlaResource* resource; - TF_RETURN_IF_ERROR(context->CreateResource( - arg.resource_kind, i, arg.name, arg.type, arg.shape, xla::XlaOp(), - /*tensor_array_size=*/arg.tensor_array_size, - /*tensor_array_gradients=*/arg.tensor_array_gradients, &resource)); + XlaResource* resource = + context->AddResource(absl::make_unique( + arg.resource_kind, i, arg.name, arg.type, arg.shape, + xla::XlaOp(), + /*max_array_size=*/arg.max_array_size, + /*tensor_array_gradients=*/arg.tensor_array_gradients, + /*tensor_array_multiple_writes_aggregate=*/true)); arg_expression = XlaExpression::Resource(resource); if (arg.initialized) { input_mapping->push_back(i); } - break; + } case XlaCompiler::Argument::kParameter: case XlaCompiler::Argument::kToken: { input_mapping->push_back(i); @@ -901,9 +925,7 @@ Status XlaCompiler::CompileGraph(const XlaCompiler::CompileOptions& options, options_.device_type, name)); xla::XlaBuilder builder(name); - XlaContext* context = - new XlaContext(this, &builder, options_.allow_cpu_custom_calls, - &options_.shape_representation_fn); + XlaContext* context = new XlaContext(this, &builder); core::ScopedUnref context_unref(context); std::vector real_args(args.begin(), args.end()); @@ -988,23 +1010,12 @@ Status XlaCompiler::CompileGraph(const XlaCompiler::CompileOptions& options, options.return_updated_values_for_all_resources, options.always_return_tuple, &builder, result->computation.get(), &num_computation_outputs, &num_nonconst_outputs, &result->outputs, - &result->resource_updates)); + &result->resource_updates, &result->xla_output_shape)); VLOG(2) << "Outputs: total: " << context->retvals().size() << " nonconstant: " << num_nonconst_outputs; - - // Compute the XLA output shape, if there is a computation with non-constant - // outputs. - TF_ASSIGN_OR_RETURN(std::unique_ptr computation_shape, - client()->GetComputationShape(*result->computation)); - - result->xla_output_shape.Swap(computation_shape->mutable_result()); VLOG(2) << "XLA output shape: " - << xla::ShapeUtil::HumanString(result->xla_output_shape); - - // Tensorflow expects a major-to-minor order of results. - xla::LayoutUtil::SetToDefaultLayout(&result->xla_output_shape); - + << xla::ShapeUtil::HumanStringWithLayout(result->xla_output_shape); return Status::OK(); } diff --git a/tensorflow/compiler/tf2xla/xla_compiler.h b/tensorflow/compiler/tf2xla/xla_compiler.h index 63426124686..0d801b73a8c 100644 --- a/tensorflow/compiler/tf2xla/xla_compiler.h +++ b/tensorflow/compiler/tf2xla/xla_compiler.h @@ -150,7 +150,7 @@ class XlaCompiler { // For a TensorArray or Stack resource, what is the array's declared size? // (Used for lazy initialization.) - int64 tensor_array_size = -1; + int64 max_array_size = -1; // TensorArray resource parameters are passed as (array, gradient array 0, // ..., gradient array k), where the gradient arrays are in the same order diff --git a/tensorflow/compiler/tf2xla/xla_compiler_test.cc b/tensorflow/compiler/tf2xla/xla_compiler_test.cc index aaee208f634..fe2a5f5b0c9 100644 --- a/tensorflow/compiler/tf2xla/xla_compiler_test.cc +++ b/tensorflow/compiler/tf2xla/xla_compiler_test.cc @@ -20,6 +20,7 @@ limitations under the License. #include "tensorflow/cc/ops/function_ops.h" #include "tensorflow/cc/ops/resource_variable_ops.h" #include "tensorflow/cc/ops/standard_ops.h" +#include "tensorflow/compiler/tf2xla/shape_util.h" #include "tensorflow/compiler/tf2xla/side_effect_util.h" #include "tensorflow/compiler/tf2xla/type_util.h" #include "tensorflow/compiler/tf2xla/xla_op_kernel.h" @@ -649,7 +650,7 @@ TEST_F(XlaCompilerTest, CanPassTensorArraysToAndFromComputation) { args[0].initialized = true; args[0].type = DT_INT32; args[0].shape = TensorShape({}); - args[0].tensor_array_size = 2; + args[0].max_array_size = 2; args[0].tensor_array_gradients = {"grad2"}; // Compiles the graph. @@ -708,7 +709,7 @@ TEST_F(XlaCompilerTest, UnwrittenTensorArrayGradientsAreNotComputationOutputs) { args[0].initialized = true; args[0].type = DT_INT32; args[0].shape = TensorShape({}); - args[0].tensor_array_size = 2; + args[0].max_array_size = 2; args[0].tensor_array_gradients = {"grad1"}; // Compiles the graph. @@ -740,7 +741,7 @@ TEST_F(XlaCompilerTest, NewTensorArrayGradientsAreComputationOutputs) { args[0].initialized = true; args[0].type = DT_INT32; args[0].shape = TensorShape({}); - args[0].tensor_array_size = 2; + args[0].max_array_size = 2; args[0].tensor_array_gradients = {"grad1"}; // Compiles the graph. @@ -910,6 +911,82 @@ TEST_F(XlaCompilerTest, Variables) { RunAndCheckVariablesComputation(client_, result); } +TEST_F(XlaCompilerTest, ResultLayoutSingle) { + Scope scope = Scope::NewRootScope().ExitOnError(); + auto a = ops::_Arg(scope.WithOpName("A"), DT_INT32, 0); + auto b = ops::_Retval(scope.WithOpName("RET"), a, 0); + + std::unique_ptr graph(new Graph(OpRegistry::Global())); + TF_ASSERT_OK(scope.ToGraph(graph.get())); + + // Builds a description of the arguments. + std::vector args(1); + args[0].kind = XlaCompiler::Argument::kParameter; + args[0].type = DT_INT32; + args[0].shape = TensorShape({2, 3}); + + auto options = DefaultOptions(); + // Sets the representation function to return a non-default layout. + options.shape_representation_fn = + [](const TensorShape& shape, DataType type) -> xla::StatusOr { + xla::Shape xla_shape; + TF_RETURN_IF_ERROR(TensorShapeToXLAShape(type, shape, &xla_shape)); + *xla_shape.mutable_layout() = xla::LayoutUtil::MakeLayout({0, 1}); + return xla_shape; + }; + + // Compiles the graph. + XlaCompiler compiler(options); + + XlaCompiler::CompilationResult result; + auto compile_options = XlaCompiler::CompileOptions(); + compile_options.always_return_tuple = false; + TF_ASSERT_OK(compiler.CompileGraph(compile_options, "id", std::move(graph), + args, &result)); + EXPECT_TRUE(xla::ShapeUtil::Equal( + result.xla_output_shape, + xla::ShapeUtil::MakeShapeWithLayout(xla::S32, {2, 3}, {0, 1}))); +} + +TEST_F(XlaCompilerTest, ResultLayoutMultiple) { + Scope scope = Scope::NewRootScope().ExitOnError(); + auto a = ops::_Arg(scope.WithOpName("A"), DT_INT32, 0); + auto b = ops::_Retval(scope.WithOpName("RET1"), a, 0); + auto c = ops::_Retval(scope.WithOpName("RET2"), a, 1); + + std::unique_ptr graph(new Graph(OpRegistry::Global())); + TF_ASSERT_OK(scope.ToGraph(graph.get())); + + // Builds a description of the arguments. + std::vector args(1); + args[0].kind = XlaCompiler::Argument::kParameter; + args[0].type = DT_INT32; + args[0].shape = TensorShape({2, 3}); + + auto options = DefaultOptions(); + // Sets the representation function to return a non-default layout. + options.shape_representation_fn = + [](const TensorShape& shape, DataType type) -> xla::StatusOr { + xla::Shape xla_shape; + TF_RETURN_IF_ERROR(TensorShapeToXLAShape(type, shape, &xla_shape)); + *xla_shape.mutable_layout() = xla::LayoutUtil::MakeLayout({0, 1}); + return xla_shape; + }; + + // Compiles the graph. + XlaCompiler compiler(options); + + XlaCompiler::CompilationResult result; + TF_ASSERT_OK(compiler.CompileGraph(XlaCompiler::CompileOptions(), "id", + std::move(graph), args, &result)); + xla::Shape result_shape = + xla::ShapeUtil::MakeShapeWithLayout(xla::S32, {2, 3}, {0, 1}); + + EXPECT_TRUE(xla::ShapeUtil::Equal( + result.xla_output_shape, + xla::ShapeUtil::MakeTupleShape({result_shape, result_shape}))); +} + // Tests a simple graph that reads and writes a variable. TEST_F(XlaCompilerTest, ReturnResourceHandleOnly) { Scope scope = Scope::NewRootScope().ExitOnError(); diff --git a/tensorflow/compiler/tf2xla/xla_context.cc b/tensorflow/compiler/tf2xla/xla_context.cc index 43095fbb473..a69af705033 100644 --- a/tensorflow/compiler/tf2xla/xla_context.cc +++ b/tensorflow/compiler/tf2xla/xla_context.cc @@ -54,25 +54,14 @@ const char XlaContext::kXlaContextResourceName[] = "_xla_context"; return *context; } -/* static */ XlaContext& XlaContext::Get(const XlaOpKernelContext* ctx) { - return Get(ctx->op_kernel_context()); -} - void XlaContext::set_args(std::vector args) { args_ = std::move(args); } -XlaContext::XlaContext( - XlaCompiler* compiler, xla::XlaBuilder* builder, - bool allow_cpu_custom_calls, - const std::function( - const TensorShape&, DataType)>* shape_representation_fn) - : compiler_(compiler), - builder_(builder), - allow_cpu_custom_calls_(allow_cpu_custom_calls), - shape_representation_fn_(shape_representation_fn) {} +XlaContext::XlaContext(XlaCompiler* compiler, xla::XlaBuilder* builder) + : compiler_(compiler), builder_(builder) {} -string XlaContext::DebugString() { return "TLA JIT context"; } +string XlaContext::DebugString() { return "XLA JIT context"; } void XlaContext::SetRetval(int index, const XlaExpression& expression) { if (retvals_.size() <= index) { @@ -81,21 +70,9 @@ void XlaContext::SetRetval(int index, const XlaExpression& expression) { retvals_[index] = expression; } -Status XlaContext::CreateResource( - XlaResource::Kind kind, int arg_num, string name, DataType type, - TensorShape shape, const xla::XlaOp& handle, int64 tensor_array_size, - const std::set& tensor_array_gradients, XlaResource** resource) { - resources_.emplace_back( - new XlaResource(kind, arg_num, std::move(name), type, std::move(shape), - handle, tensor_array_size, tensor_array_gradients, - /*tensor_array_multiple_writes_aggregate=*/false)); - *resource = resources_.back().get(); - return Status::OK(); -} - -xla::StatusOr XlaContext::RepresentationShape( - const TensorShape& shape, DataType type) const { - return (*shape_representation_fn_)(shape, type); +XlaResource* XlaContext::AddResource(std::unique_ptr resource) { + resources_.push_back(std::move(resource)); + return resources_.back().get(); } const xla::XlaComputation* XlaContext::GetOrCreateMax(const DataType type) { diff --git a/tensorflow/compiler/tf2xla/xla_context.h b/tensorflow/compiler/tf2xla/xla_context.h index dbfd344c9ba..0767d1faac1 100644 --- a/tensorflow/compiler/tf2xla/xla_context.h +++ b/tensorflow/compiler/tf2xla/xla_context.h @@ -41,14 +41,10 @@ class XlaContext : public ResourceBase { public: // Retrieves the XlaContext of the current compilation. static XlaContext& Get(const OpKernelContext* ctx); - static XlaContext& Get(const XlaOpKernelContext* ctx); // Creates a new XlaContext. See the documentation on the class data fields // for descriptions of the arguments. - XlaContext(XlaCompiler* compiler, xla::XlaBuilder* builder, - bool allow_cpu_custom_calls, - const std::function( - const TensorShape&, DataType)>* shape_representation_fn); + XlaContext(XlaCompiler* compiler, xla::XlaBuilder* builder); // Virtual method defined by ResourceBase. string DebugString() override; @@ -58,8 +54,6 @@ class XlaContext : public ResourceBase { // Returns the XlaBuilder that Ops use for compiling new expressions. xla::XlaBuilder* builder() { return builder_; } - bool allow_cpu_custom_calls() const { return allow_cpu_custom_calls_; } - const std::vector& args() const { return args_; } void set_args(std::vector args); @@ -70,25 +64,13 @@ class XlaContext : public ResourceBase { // grows the return values vector to size index+1 if it is smaller. void SetRetval(int index, const XlaExpression& expression); - // Creates a resource with resource `kind` and initial value `handle`. `name` - // is a descriptive name for use in error messages. See the `XlaResource` - // constructor for a description of the remaining arguments. - // Fails if the resource already exists. - Status CreateResource(XlaResource::Kind kind, int arg_num, string name, - DataType type, TensorShape shape, - const xla::XlaOp& handle, int64 tensor_array_size, - const std::set& tensor_array_gradients, - XlaResource** resource); + // Adds 'resource' to the set of resources owned by the context. + XlaResource* AddResource(std::unique_ptr resource); const std::vector>& resources() { return resources_; } - // Returns the XLA shape to be used to represent a variable of TF `shape` - // and `type`, or of an argument or return value of a top-level computation. - xla::StatusOr RepresentationShape(const TensorShape& shape, - DataType type) const; - // Get an XLA lambda to compute Max. This is cached in the // XlaContext since it may be used by multiple Ops. There is a // separate specialization of the computation for each DataType. @@ -118,9 +100,6 @@ class XlaContext : public ResourceBase { // The XlaBuilder used to construct the subgraph's compiled representation. xla::XlaBuilder* builder_; - // Allow ops to emit CustomCall operations for CPU. - const bool allow_cpu_custom_calls_; - // Arguments to the Tensorflow graph, indexed by _Arg index. // Includes both compile-time constant arguments and runtime parameters. std::vector args_; @@ -131,11 +110,6 @@ class XlaContext : public ResourceBase { // Holds ownership of resources. The resources are not ordered. std::vector> resources_; - // Describes the on-host shapes of parameters and return values. Also see: - // XlaDevice::Options::shape_representation_fn. - const std::function(const TensorShape&, DataType)>* - shape_representation_fn_; - // Cache of prebuilt computations indexed by their type. using ComputationMap = std::map; diff --git a/tensorflow/compiler/tf2xla/xla_helpers.cc b/tensorflow/compiler/tf2xla/xla_helpers.cc index 9a34cd8c6ae..c2c07512111 100644 --- a/tensorflow/compiler/tf2xla/xla_helpers.cc +++ b/tensorflow/compiler/tf2xla/xla_helpers.cc @@ -26,7 +26,6 @@ limitations under the License. #include "tensorflow/compiler/tf2xla/xla_op_kernel.h" #include "tensorflow/compiler/xla/client/lib/arithmetic.h" #include "tensorflow/compiler/xla/client/lib/constants.h" -#include "tensorflow/compiler/xla/client/lib/numeric.h" #include "tensorflow/compiler/xla/client/xla_builder.h" #include "tensorflow/compiler/xla/client/xla_computation.h" #include "tensorflow/compiler/xla/types.h" @@ -216,8 +215,7 @@ DataType XlaHelpers::SumAccumulationType(const DataType& dtype) { return dtype; } -xla::XlaOp XlaHelpers::ConvertElementType(xla::XlaBuilder* const builder, - const xla::XlaOp& operand, +xla::XlaOp XlaHelpers::ConvertElementType(const xla::XlaOp& operand, const DataType new_element_type) { xla::PrimitiveType convert_to; TF_CHECK_OK(DataTypeToPrimitiveType(new_element_type, &convert_to)); diff --git a/tensorflow/compiler/tf2xla/xla_helpers.h b/tensorflow/compiler/tf2xla/xla_helpers.h index 39578144caa..4858dfee55a 100644 --- a/tensorflow/compiler/tf2xla/xla_helpers.h +++ b/tensorflow/compiler/tf2xla/xla_helpers.h @@ -80,8 +80,7 @@ class XlaHelpers { // A helper for creating a ConvertElementType xla op given a DataType rather // than the xla::PrimitiveType. - static xla::XlaOp ConvertElementType(xla::XlaBuilder* const builder, - const xla::XlaOp& operand, + static xla::XlaOp ConvertElementType(const xla::XlaOp& operand, const DataType new_element_type); }; diff --git a/tensorflow/compiler/tf2xla/xla_jit_compiled_cpu_function.cc b/tensorflow/compiler/tf2xla/xla_jit_compiled_cpu_function.cc index 86a78ee429e..fabbcd04fed 100644 --- a/tensorflow/compiler/tf2xla/xla_jit_compiled_cpu_function.cc +++ b/tensorflow/compiler/tf2xla/xla_jit_compiled_cpu_function.cc @@ -133,7 +133,8 @@ XlaJitCompiledCpuFunction::Compile( jit->executable_ = std::move(executable); jit->buffer_infos_ = std::move(buffer_infos); jit->arg_index_table_ = std::move(arg_index_table); - jit->program_shape_ = std::move(program_shape); + jit->program_shape_ = + absl::make_unique(program_shape->ToProto()); jit->static_data_.set_raw_function(raw_function); jit->static_data_.set_buffer_infos(jit->buffer_infos_.data()); jit->static_data_.set_num_buffers(jit->buffer_infos_.size()); diff --git a/tensorflow/compiler/tf2xla/xla_jit_compiled_cpu_function.h b/tensorflow/compiler/tf2xla/xla_jit_compiled_cpu_function.h index d3c8f22a807..a5392057177 100644 --- a/tensorflow/compiler/tf2xla/xla_jit_compiled_cpu_function.h +++ b/tensorflow/compiler/tf2xla/xla_jit_compiled_cpu_function.h @@ -80,8 +80,10 @@ class XlaJitCompiledCpuFunction { std::vector arg_names_; std::vector result_names_; - // The backing data for the program shape. - std::unique_ptr program_shape_; + // The backing data for the program shape. The proto form of program shape is + // used because the program shape is serialized and embedded in the object + // file. + std::unique_ptr program_shape_; }; } // namespace tensorflow diff --git a/tensorflow/compiler/tf2xla/xla_jit_compiled_cpu_function_test.cc b/tensorflow/compiler/tf2xla/xla_jit_compiled_cpu_function_test.cc index 6d49298a6f3..8846088678b 100644 --- a/tensorflow/compiler/tf2xla/xla_jit_compiled_cpu_function_test.cc +++ b/tensorflow/compiler/tf2xla/xla_jit_compiled_cpu_function_test.cc @@ -116,13 +116,13 @@ TEST(XlaJitCompiledCpuFunction, Sum) { // Check program shape. using xla::ShapeUtil; const xla::Shape s32 = ShapeUtil::MakeShape(xla::S32, {}); - const xla::ProgramShape* program_shape = function.ProgramShape(); - ASSERT_TRUE(program_shape != nullptr); - ASSERT_EQ(program_shape->parameters_size(), 2); - EXPECT_TRUE(ShapeUtil::Compatible(program_shape->parameters(0), s32)); - EXPECT_TRUE(ShapeUtil::Compatible(program_shape->parameters(1), s32)); + ASSERT_TRUE(function.ProgramShape() != nullptr); + const xla::ProgramShape program_shape(*function.ProgramShape()); + ASSERT_EQ(program_shape.parameters_size(), 2); + EXPECT_TRUE(ShapeUtil::Compatible(program_shape.parameters(0), s32)); + EXPECT_TRUE(ShapeUtil::Compatible(program_shape.parameters(1), s32)); - const xla::Shape& result = program_shape->result(); + const xla::Shape& result = program_shape.result(); ASSERT_EQ(result.element_type(), xla::TUPLE); ASSERT_EQ(ShapeUtil::TupleElementCount(result), 1); const xla::Shape& result0 = ShapeUtil::GetTupleElementShape(result, 0); diff --git a/tensorflow/compiler/tf2xla/xla_op_kernel.cc b/tensorflow/compiler/tf2xla/xla_op_kernel.cc index 8dd8def0549..58808c76de6 100644 --- a/tensorflow/compiler/tf2xla/xla_op_kernel.cc +++ b/tensorflow/compiler/tf2xla/xla_op_kernel.cc @@ -36,8 +36,16 @@ bool XlaOpKernelContext::ValidateInputsAreSameShape(OpKernel* op) { return context_->ValidateInputsAreSameShape(op); } +XlaContext* XlaOpKernelContext::xla_context() const { + return &XlaContext::Get(context_); +} + xla::XlaBuilder* XlaOpKernelContext::builder() const { - return XlaContext::Get(this).builder(); + return xla_context()->builder(); +} + +XlaCompiler* XlaOpKernelContext::compiler() const { + return xla_context()->compiler(); } // Retrieves an XlaExpression that was allocated by a previous Op. @@ -338,8 +346,8 @@ Status XlaOpKernelContext::ConstantInputList( namespace { Status ReadVariableInputTensor(const Tensor& tensor, DataType type, - const OpKernelContext* ctx, TensorShape* shape, - xla::XlaOp* value) { + const XlaOpKernelContext* ctx, + TensorShape* shape, xla::XlaOp* value) { const XlaExpression* expression = CastExpressionFromTensor(tensor); XlaResource* variable = expression->resource(); TF_RET_CHECK(variable != nullptr); @@ -357,10 +365,9 @@ Status ReadVariableInputTensor(const Tensor& tensor, DataType type, *shape = variable->shape(); } - XlaContext& xla_context = XlaContext::Get(ctx); - TF_ASSIGN_OR_RETURN( - xla::Shape representation_shape, - xla_context.RepresentationShape(variable->shape(), variable->type())); + TF_ASSIGN_OR_RETURN(xla::Shape representation_shape, + ctx->compiler()->options().shape_representation_fn( + variable->shape(), variable->type())); xla::Shape xla_shape; TF_RETURN_IF_ERROR( TensorShapeToXLAShape(variable->type(), variable->shape(), &xla_shape)); @@ -377,15 +384,15 @@ Status ReadVariableInputTensor(const Tensor& tensor, DataType type, Status XlaOpKernelContext::ReadVariableInput(int index, DataType type, TensorShape* shape, xla::XlaOp* value) { - return ReadVariableInputTensor(context_->input(index), type, context_, shape, + return ReadVariableInputTensor(context_->input(index), type, this, shape, value); } Status XlaOpKernelContext::ReadVariableInput(absl::string_view name, DataType type, TensorShape* shape, xla::XlaOp* value) { - return ReadVariableInputTensor(GetInputTensorByName(name), type, context_, - shape, value); + return ReadVariableInputTensor(GetInputTensorByName(name), type, this, shape, + value); } Status XlaOpKernelContext::GetVariableTypeAndShape(int index, DataType* type, @@ -464,7 +471,7 @@ Status XlaOpKernelContext::GetResourceInput(int index, XlaResource** resource) { namespace { Status AssignVariableTensor(const Tensor& tensor, DataType type, - const OpKernelContext* ctx, xla::XlaOp handle, + const XlaOpKernelContext* ctx, xla::XlaOp handle, xla::XlaBuilder* builder) { const XlaExpression* expression = CastExpressionFromTensor(tensor); XlaResource* variable = expression->resource(); @@ -481,9 +488,9 @@ Status AssignVariableTensor(const Tensor& tensor, DataType type, TF_RETURN_IF_ERROR(variable->SetTypeAndShape(type, shape)); - XlaContext& xla_context = XlaContext::Get(ctx); - TF_ASSIGN_OR_RETURN(xla::Shape representation_shape, - xla_context.RepresentationShape(shape, type)); + TF_ASSIGN_OR_RETURN( + xla::Shape representation_shape, + ctx->compiler()->options().shape_representation_fn(shape, type)); xla::Shape xla_shape; TF_RETURN_IF_ERROR(TensorShapeToXLAShape(type, shape, &xla_shape)); if (!xla::ShapeUtil::Compatible(xla_shape, representation_shape)) { @@ -498,19 +505,15 @@ Status AssignVariableTensor(const Tensor& tensor, DataType type, Status XlaOpKernelContext::AssignVariable(int input_index, DataType type, xla::XlaOp handle) { TF_RET_CHECK(handle.valid()); - return AssignVariableTensor(context_->input(input_index), type, context_, - handle, builder()); + return AssignVariableTensor(context_->input(input_index), type, this, handle, + builder()); } Status XlaOpKernelContext::AssignVariable(absl::string_view name, DataType type, xla::XlaOp handle) { TF_RET_CHECK(handle.valid()); - return AssignVariableTensor(GetInputTensorByName(name), type, context_, - handle, builder()); -} - -XlaCompiler* XlaOpKernelContext::compiler() const { - return XlaContext::Get(context_).compiler(); + return AssignVariableTensor(GetInputTensorByName(name), type, this, handle, + builder()); } void XlaOpKernelContext::CtxFailure(const Status& s) { @@ -530,22 +533,22 @@ void XlaOpKernelContext::CtxFailureWithWarning(const char* file, int line, const xla::XlaComputation* XlaOpKernelContext::GetOrCreateMax( const DataType type) { - return XlaContext::Get(context_).GetOrCreateMax(type); + return xla_context()->GetOrCreateMax(type); } const xla::XlaComputation* XlaOpKernelContext::GetOrCreateMin( const DataType type) { - return XlaContext::Get(context_).GetOrCreateMin(type); + return xla_context()->GetOrCreateMin(type); } const xla::XlaComputation* XlaOpKernelContext::GetOrCreateAdd( const DataType type) { - return XlaContext::Get(context_).GetOrCreateAdd(type); + return xla_context()->GetOrCreateAdd(type); } const xla::XlaComputation* XlaOpKernelContext::GetOrCreateMul( const DataType type) { - return XlaContext::Get(context_).GetOrCreateMul(type); + return xla_context()->GetOrCreateMul(type); } const Tensor& XlaOpKernelContext::GetInputTensorByName(absl::string_view name) { diff --git a/tensorflow/compiler/tf2xla/xla_op_kernel.h b/tensorflow/compiler/tf2xla/xla_op_kernel.h index c06efa2c474..1858844bc05 100644 --- a/tensorflow/compiler/tf2xla/xla_op_kernel.h +++ b/tensorflow/compiler/tf2xla/xla_op_kernel.h @@ -60,6 +60,8 @@ class XlaOpKernelContext { public: explicit XlaOpKernelContext(OpKernelContext* context); + XlaContext* xla_context() const; + // Returns the XLA XlaBuilder containing the output of compilation. xla::XlaBuilder* builder() const; diff --git a/tensorflow/compiler/tf2xla/xla_op_registry.cc b/tensorflow/compiler/tf2xla/xla_op_registry.cc index dcd0e9c5c1f..14237df6908 100644 --- a/tensorflow/compiler/tf2xla/xla_op_registry.cc +++ b/tensorflow/compiler/tf2xla/xla_op_registry.cc @@ -18,7 +18,7 @@ limitations under the License. #include #include -#include "tensorflow/compiler/jit/legacy_flags/mark_for_compilation_pass_flags.h" +#include "tensorflow/compiler/jit/flags.h" #include "tensorflow/compiler/jit/xla_cluster_util.h" #include "tensorflow/compiler/tf2xla/type_util.h" #include "tensorflow/compiler/tf2xla/xla_context.h" @@ -130,8 +130,7 @@ XlaOpRegistry::~XlaOpRegistry() = default; // Lazily register the CPU and GPU JIT devices the first time // GetCompilationDevice is called. static void* registration_init = [®istry]() { - legacy_flags::MarkForCompilationPassFlags* flags = - legacy_flags::GetMarkForCompilationPassFlags(); + MarkForCompilationPassFlags* flags = GetMarkForCompilationPassFlags(); bool cpu_global_jit = flags->tf_xla_cpu_global_jit; mutex_lock lock(registry.mutex_); diff --git a/tensorflow/compiler/tf2xla/xla_resource.cc b/tensorflow/compiler/tf2xla/xla_resource.cc index a322eb9015e..48a3c012727 100644 --- a/tensorflow/compiler/tf2xla/xla_resource.cc +++ b/tensorflow/compiler/tf2xla/xla_resource.cc @@ -18,6 +18,7 @@ limitations under the License. #include #include +#include "absl/memory/memory.h" #include "tensorflow/compiler/tf2xla/shape_util.h" #include "tensorflow/compiler/tf2xla/sharding_util.h" #include "tensorflow/compiler/tf2xla/xla_context.h" @@ -39,9 +40,29 @@ namespace tensorflow { } } +/*static*/ std::unique_ptr XlaResource::CreateStack( + string name, DataType type, int64 max_size) { + return absl::make_unique( + XlaResource::kStack, /*arg_num=*/-1, std::move(name), type, TensorShape(), + /*initial_value=*/xla::XlaOp(), + /*max_array_size=*/max_size, + /*tensor_array_gradients=*/std::set{}, + /*tensor_array_multiple_writes_aggregate=*/false); +} + +/*static*/ std::unique_ptr XlaResource::CreateTensorArray( + string name, DataType type, TensorShape shape, xla::XlaOp initial_value, + int64 max_array_size) { + return absl::make_unique( + XlaResource::kTensorArray, /*arg_num=*/-1, std::move(name), type, shape, + initial_value, max_array_size, + /*tensor_array_gradients=*/std::set{}, + /*tensor_array_multiple_writes_aggregate=*/false); +} + XlaResource::XlaResource(Kind kind, int arg_num, string name, DataType type, TensorShape shape, const xla::XlaOp& initial_value, - int64 tensor_array_size, + int64 max_array_size, const std::set& tensor_array_gradients, bool tensor_array_multiple_writes_aggregate) : kind_(kind), @@ -51,7 +72,7 @@ XlaResource::XlaResource(Kind kind, int arg_num, string name, DataType type, shape_(std::move(shape)), value_(initial_value), initial_value_(initial_value), - tensor_array_size_(tensor_array_size), + max_array_size_(max_array_size), tensor_array_multiple_writes_aggregate_( tensor_array_multiple_writes_aggregate) { CHECK(kind_ != kInvalid); @@ -60,7 +81,7 @@ XlaResource::XlaResource(Kind kind, int arg_num, string name, DataType type, tensor_array_gradients_[gradient].reset(new XlaResource( /*kind=*/kTensorArray, /*arg_num=*/-1, /*name=*/absl::StrCat("TensorArrayGrad: ", name_), type_, shape_, - xla::XlaOp(), tensor_array_size_, /*tensor_array_gradients=*/{}, + xla::XlaOp(), max_array_size_, /*tensor_array_gradients=*/{}, /*tensor_array_multiple_writes_aggregate=*/true)); } } @@ -113,7 +134,7 @@ Status XlaResource::SetZeroValue(xla::XlaBuilder* builder) { } case kTensorArray: { TensorShape ta_shape; - ta_shape.AddDim(tensor_array_size_); + ta_shape.AddDim(max_array_size_); ta_shape.AppendShape(shape_); value_ = xla::Broadcast(XlaHelpers::Zero(builder, type_), ta_shape.dim_sizes()); @@ -121,7 +142,7 @@ Status XlaResource::SetZeroValue(xla::XlaBuilder* builder) { } case kStack: { TensorShape ta_shape; - ta_shape.AddDim(tensor_array_size_); + ta_shape.AddDim(max_array_size_); ta_shape.AppendShape(shape_); value_ = xla::Tuple(builder, {xla::Broadcast(XlaHelpers::Zero(builder, type_), @@ -146,14 +167,14 @@ Status XlaResource::GetOrCreateTensorArrayGradient(const string& source, std::unique_ptr& gradient = tensor_array_gradients_[source]; if (!gradient) { TensorShape ta_shape; - ta_shape.AddDim(tensor_array_size_); + ta_shape.AddDim(max_array_size_); ta_shape.AppendShape(shape_); xla::XlaOp gradient_value = xla::Broadcast(XlaHelpers::Zero(builder, type_), ta_shape.dim_sizes()); gradient.reset( new XlaResource(/*kind=*/kTensorArray, /*arg_num=*/-1, /*name=*/absl::StrCat("TensorArrayGrad: ", name_), - type_, shape_, gradient_value, tensor_array_size_, + type_, shape_, gradient_value, max_array_size_, /*tensor_array_gradients=*/{}, /*tensor_array_multiple_writes_aggregate=*/true)); } diff --git a/tensorflow/compiler/tf2xla/xla_resource.h b/tensorflow/compiler/tf2xla/xla_resource.h index 857b9a928bb..736588bb8b8 100644 --- a/tensorflow/compiler/tf2xla/xla_resource.h +++ b/tensorflow/compiler/tf2xla/xla_resource.h @@ -38,9 +38,18 @@ class XlaResource { }; static absl::string_view KindToString(Kind kind); + // Creates a new Stack resource. + static std::unique_ptr CreateStack(string name, DataType type, + int64 max_size); + + // Creates a new TensorArray resource. + static std::unique_ptr CreateTensorArray( + string name, DataType type, TensorShape shape, xla::XlaOp initial_value, + int64 max_array_size); + XlaResource(Kind kind, int arg_num, string name, DataType type, TensorShape shape, const xla::XlaOp& initial_value, - int64 tensor_array_size, + int64 max_array_size, const std::set& tensor_array_gradients, bool tensor_array_multiple_writes_aggregate); @@ -119,12 +128,12 @@ class XlaResource { // TODO(phawkins): refactor this code to use subclasses, rather than putting // kind-specific fields in XlaResource. - // 'tensor_array_size' stores the expected size of the TensorArray or Stack. + // 'max_array_size' stores the expected size of the TensorArray or Stack. // We need to store this since sometimes TensorArrays must be initialized // lazily since we do not know the element shape at construction time. // Used by both TensorArrays and Stacks. - int64 tensor_array_size() const { return tensor_array_size_; } - void set_tensor_array_size(int64 size) { tensor_array_size_ = size; } + int64 max_array_size() const { return max_array_size_; } + void set_max_array_size(int64 size) { max_array_size_ = size; } bool tensor_array_multiple_writes_aggregate() const { return tensor_array_multiple_writes_aggregate_; @@ -151,7 +160,7 @@ class XlaResource { xla::XlaOp value_; xla::XlaOp initial_value_; - int64 tensor_array_size_ = -1; + int64 max_array_size_ = -1; bool tensor_array_multiple_writes_aggregate_ = false; std::map> tensor_array_gradients_; diff --git a/tensorflow/compiler/xla/BUILD b/tensorflow/compiler/xla/BUILD index 91096cf1d04..4360e085796 100644 --- a/tensorflow/compiler/xla/BUILD +++ b/tensorflow/compiler/xla/BUILD @@ -226,12 +226,14 @@ cc_library( "index_util.cc", "layout_util.cc", "primitive_util.cc", + "shape.cc", "shape_util.cc", ], hdrs = [ "index_util.h", "layout_util.h", "primitive_util.h", + "shape.h", "shape_util.h", ], visibility = ["//visibility:public"], @@ -254,6 +256,23 @@ cc_library( ], ) +tf_cc_test( + name = "shape_test", + srcs = ["shape_test.cc"], + deps = [ + ":shape_util", + ":status_macros", + ":test", + ":test_helpers", + ":types", + ":util", + ":xla_data_proto", + "//tensorflow/core:lib", + "//tensorflow/core:test_main", + "@com_google_absl//absl/strings", + ], +) + tf_cc_test( name = "shape_util_test", srcs = ["shape_util_test.cc"], @@ -745,6 +764,8 @@ cc_library( "//tensorflow/core:framework_internal", "//tensorflow/core:lib", "@com_google_absl//absl/strings", + "@com_google_absl//absl/strings:str_format", + "@com_google_absl//absl/types:span", ], ) diff --git a/tensorflow/compiler/xla/array2d.h b/tensorflow/compiler/xla/array2d.h index 782c966b4c5..e4aca98f67d 100644 --- a/tensorflow/compiler/xla/array2d.h +++ b/tensorflow/compiler/xla/array2d.h @@ -104,7 +104,7 @@ std::unique_ptr> MakeLinspaceArray2D(double from, double to, int64 count = n1 * n2; NativeT step = static_cast((count > 1) ? (to - from) / (count - 1) : 0); - auto set = [&array, n1, n2](int64 index, NativeT value) { + auto set = [&array, n2](int64 index, NativeT value) { (*array)(index / n2, index % n2) = value; }; for (int64 i = 0; i < count - 1; ++i) { diff --git a/tensorflow/compiler/xla/client/BUILD b/tensorflow/compiler/xla/client/BUILD index 42da0ebf499..fe99564d3c6 100644 --- a/tensorflow/compiler/xla/client/BUILD +++ b/tensorflow/compiler/xla/client/BUILD @@ -81,6 +81,7 @@ cc_library( "//tensorflow/core:lib", "@com_google_absl//absl/memory", "@com_google_absl//absl/strings", + "@com_google_absl//absl/types:optional", "@com_google_absl//absl/types:span", ], ) @@ -90,11 +91,12 @@ cc_library( srcs = ["executable_build_options.cc"], hdrs = ["executable_build_options.h"], deps = [ + "//tensorflow/compiler/xla:debug_options_flags", "//tensorflow/compiler/xla:shape_util", "//tensorflow/compiler/xla:util", "//tensorflow/compiler/xla:xla_data_proto", + "//tensorflow/compiler/xla:xla_proto", "//tensorflow/compiler/xla/service:device_memory_allocator", - "//tensorflow/core:lib", "@com_google_absl//absl/strings", "@com_google_absl//absl/strings:str_format", "@com_google_absl//absl/types:optional", @@ -191,6 +193,7 @@ cc_library( hdrs = ["xla_computation.h"], visibility = ["//visibility:public"], deps = [ + "//tensorflow/compiler/xla:shape_util", "//tensorflow/compiler/xla:status_macros", "//tensorflow/compiler/xla:util", "//tensorflow/compiler/xla:xla_data_proto", diff --git a/tensorflow/compiler/xla/client/client.cc b/tensorflow/compiler/xla/client/client.cc index eef2844e0df..74b76f92994 100644 --- a/tensorflow/compiler/xla/client/client.cc +++ b/tensorflow/compiler/xla/client/client.cc @@ -20,6 +20,7 @@ limitations under the License. #include "absl/memory/memory.h" #include "absl/strings/str_cat.h" +#include "absl/types/optional.h" #include "tensorflow/compiler/xla/client/xla_computation.h" #include "tensorflow/compiler/xla/debug_options_flags.h" #include "tensorflow/compiler/xla/execution_options_util.h" @@ -42,7 +43,7 @@ StatusOr Client::Transfer(const GlobalData& data, TransferToClientRequest request; *request.mutable_data() = data.handle(); if (shape_with_layout != nullptr) { - *request.mutable_shape_with_layout() = *shape_with_layout; + *request.mutable_shape_with_layout() = shape_with_layout->ToProto(); } TransferToClientResponse response; @@ -123,7 +124,7 @@ StatusOr Client::TransferFromOutfeed( } request.set_replica_id(replica_id); if (shape_with_layout != nullptr) { - *request.mutable_shape_with_layout() = *shape_with_layout; + *request.mutable_shape_with_layout() = shape_with_layout->ToProto(); } TransferFromOutfeedResponse response; @@ -170,11 +171,14 @@ StatusOr Client::ExecuteAndTransfer( std::unique_ptr data, Execute(computation, arguments, execution_options, execution_profile)); - const Shape* shape_with_output_layout = nullptr; + absl::optional shape_with_output_layout; if (execution_options && execution_options->has_shape_with_output_layout()) { - shape_with_output_layout = &execution_options->shape_with_output_layout(); + shape_with_output_layout = + Shape(execution_options->shape_with_output_layout()); } - return Transfer(*data, shape_with_output_layout); + return Transfer(*data, shape_with_output_layout.has_value() + ? &(*shape_with_output_layout) + : nullptr); } StatusOr Client::ComputeConstant(const XlaComputation& computation, @@ -229,7 +233,7 @@ StatusOr Client::Compile( // The argument shapes affect how the computation is compiled. for (const auto& arg_shape : argument_shapes) { - *request.add_input_shape_with_layout() = arg_shape; + *request.add_input_shape_with_layout() = arg_shape.ToProto(); } CompileResponse response; @@ -458,7 +462,7 @@ StatusOr Client::GetShape(const GlobalData& data) { return s; } - return response.shape(); + return Shape(response.shape()); } StatusOr Client::ExecutionStatsAsString( diff --git a/tensorflow/compiler/xla/client/executable_build_options.cc b/tensorflow/compiler/xla/client/executable_build_options.cc index 0f1745366b7..1f594e551af 100644 --- a/tensorflow/compiler/xla/client/executable_build_options.cc +++ b/tensorflow/compiler/xla/client/executable_build_options.cc @@ -16,6 +16,7 @@ limitations under the License. #include "tensorflow/compiler/xla/client/executable_build_options.h" #include "absl/strings/str_format.h" +#include "tensorflow/compiler/xla/debug_options_flags.h" #include "tensorflow/compiler/xla/shape_util.h" namespace xla { @@ -39,6 +40,13 @@ ExecutableBuildOptions& ExecutableBuildOptions::set_device_ordinal( int ExecutableBuildOptions::device_ordinal() const { return device_ordinal_; } +DebugOptions* ExecutableBuildOptions::mutable_debug_options() { + if (!has_debug_options()) { + debug_options_ = GetDebugOptionsFromFlags(); + } + return &debug_options_.value(); +} + ExecutableBuildOptions& ExecutableBuildOptions::set_result_layout( const Shape& shape_with_layout) { result_layout_set_ = true; @@ -55,68 +63,10 @@ string ExecutableBuildOptions::ToString() const { if (result_layout_set_) { result_layout = ShapeUtil::HumanStringWithLayout(result_layout_); } - string generate_hlo_graph = "nullopt"; - if (generate_hlo_graph_.has_value()) { - generate_hlo_graph = generate_hlo_graph_.value(); - } return absl::StrFormat( "ExecutableBuildOptions{device_ordinal=%d, result_layout=%s, " "generate_hlo_graph=%s}", - device_ordinal_, result_layout, generate_hlo_graph); -} - -ExecutableBuildOptions& ExecutableBuildOptions::set_generate_hlo_graph( - string regex) { - generate_hlo_graph_ = std::move(regex); - return *this; -} - -const absl::optional& ExecutableBuildOptions::generate_hlo_graph() - const { - return generate_hlo_graph_; -} - -ExecutableBuildOptions& ExecutableBuildOptions::set_dump_optimized_hlo_proto_to( - absl::string_view dirpath) { - dump_optimized_hlo_proto_to_ = string(dirpath); - return *this; -} - -const absl::optional& -ExecutableBuildOptions::dump_optimized_hlo_proto_to() const { - return dump_optimized_hlo_proto_to_; -} - -ExecutableBuildOptions& -ExecutableBuildOptions::set_dump_unoptimized_hlo_proto_to( - absl::string_view dirpath) { - dump_unoptimized_hlo_proto_to_ = string(dirpath); - return *this; -} - -const absl::optional& -ExecutableBuildOptions::dump_unoptimized_hlo_proto_to() const { - return dump_unoptimized_hlo_proto_to_; -} - -ExecutableBuildOptions& ExecutableBuildOptions::set_dump_per_pass_hlo_proto_to( - absl::string_view dirpath) { - dump_per_pass_hlo_proto_to_ = string(dirpath); - return *this; -} - -const absl::optional& -ExecutableBuildOptions::dump_per_pass_hlo_proto_to() const { - return dump_per_pass_hlo_proto_to_; -} - -ExecutableBuildOptions& ExecutableBuildOptions::set_hlo_profile(bool enabled) { - hlo_profile_ = enabled; - return *this; -} - -absl::optional ExecutableBuildOptions::hlo_profile() const { - return hlo_profile_; + device_ordinal_, result_layout, debug_options().xla_generate_hlo_graph()); } } // namespace xla diff --git a/tensorflow/compiler/xla/client/executable_build_options.h b/tensorflow/compiler/xla/client/executable_build_options.h index 93334db88bc..a58090253bf 100644 --- a/tensorflow/compiler/xla/client/executable_build_options.h +++ b/tensorflow/compiler/xla/client/executable_build_options.h @@ -19,7 +19,9 @@ limitations under the License. #include "absl/strings/string_view.h" #include "absl/types/optional.h" #include "tensorflow/compiler/xla/service/device_memory_allocator.h" +#include "tensorflow/compiler/xla/shape.h" #include "tensorflow/compiler/xla/util.h" +#include "tensorflow/compiler/xla/xla.pb.h" #include "tensorflow/compiler/xla/xla_data.pb.h" namespace xla { @@ -44,6 +46,12 @@ class ExecutableBuildOptions { ExecutableBuildOptions& set_result_layout(const Shape& shape_with_layout); const Shape* result_layout() const; + // Expose access to the XLA debug options which will be passed to the + // compilation process. + bool has_debug_options() const { return debug_options_.has_value(); } + const DebugOptions& debug_options() const { return *debug_options_; } + DebugOptions* mutable_debug_options(); + // If set, this specifies an allocator that can be used to allocate temporary // space on the device during compilation. For example, the compiler might // want to run various algorithms on the device and pick the fastest one -- it @@ -55,56 +63,16 @@ class ExecutableBuildOptions { DeviceMemoryAllocator* allocator); DeviceMemoryAllocator* device_allocator() const; - // If set, specifies a regexp of HLO graphs to dump (as in DebugOptions). - ExecutableBuildOptions& set_generate_hlo_graph(string regex); - const absl::optional& generate_hlo_graph() const; - - // If set, specifies a dirpath to dump the end-of-optimization-pipeline HLO - // protobuf to (as in DebugOptions). - ExecutableBuildOptions& set_dump_optimized_hlo_proto_to( - absl::string_view dirpath); - const absl::optional& dump_optimized_hlo_proto_to() const; - - // If set, specifies a dirpath to dump the start-of-optimization-pipeline HLO - // protobuf to (as in DebugOptions). - ExecutableBuildOptions& set_dump_unoptimized_hlo_proto_to( - absl::string_view dirpath); - const absl::optional& dump_unoptimized_hlo_proto_to() const; - - // If set, specifies a dirpath to dump the per-pass-in-pipeline HLO protobufs - // to (as in DebugOptions). - ExecutableBuildOptions& set_dump_per_pass_hlo_proto_to( - absl::string_view dirpath); - const absl::optional& dump_per_pass_hlo_proto_to() const; - - // If true, specifies that we should record an HLO profile during execution - // and log it after execution (as in DebugOptions). If nullopt the default is - // used. - ExecutableBuildOptions& set_hlo_profile(bool enabled); - absl::optional hlo_profile() const; - - void add_disabled_hlo_pass(absl::string_view pass_name) { - disabled_hlo_passes_.push_back(std::string(pass_name)); - } - const absl::Span disabled_hlo_passes() const { - return disabled_hlo_passes_; - } - // Returns a string representation of the build options, suitable for // debugging. string ToString() const; private: - absl::optional hlo_profile_; int device_ordinal_ = -1; Shape result_layout_; bool result_layout_set_ = false; - absl::optional generate_hlo_graph_; - absl::optional dump_optimized_hlo_proto_to_; - absl::optional dump_unoptimized_hlo_proto_to_; - absl::optional dump_per_pass_hlo_proto_to_; + absl::optional debug_options_; DeviceMemoryAllocator* device_allocator_ = nullptr; - std::vector disabled_hlo_passes_; }; } // namespace xla diff --git a/tensorflow/compiler/xla/client/lib/BUILD b/tensorflow/compiler/xla/client/lib/BUILD index f833ddcd323..f0f530d7d77 100644 --- a/tensorflow/compiler/xla/client/lib/BUILD +++ b/tensorflow/compiler/xla/client/lib/BUILD @@ -104,13 +104,17 @@ xla_test( ) cc_library( - name = "numeric", - srcs = ["numeric.cc"], - hdrs = ["numeric.h"], + name = "matrix", + srcs = ["matrix.cc"], + hdrs = ["matrix.h"], deps = [ ":arithmetic", ":constants", + "//tensorflow/compiler/xla:shape_util", + "//tensorflow/compiler/xla:status_macros", + "//tensorflow/compiler/xla:statusor", "//tensorflow/compiler/xla:types", + "//tensorflow/compiler/xla:util", "//tensorflow/compiler/xla:xla_data_proto", "//tensorflow/compiler/xla/client:xla_builder", "@com_google_absl//absl/types:span", @@ -118,11 +122,12 @@ cc_library( ) xla_test( - name = "numeric_test", - srcs = ["numeric_test.cc"], + name = "matrix_test", + srcs = ["matrix_test.cc"], tags = ["enable_for_xla_interpreter"], deps = [ - ":numeric", + ":matrix", + ":slicing", "//tensorflow/compiler/xla:test", "//tensorflow/compiler/xla:types", "//tensorflow/compiler/xla:xla_data_proto", @@ -164,7 +169,6 @@ cc_library( deps = [ ":constants", ":math", - ":numeric", "//tensorflow/compiler/xla:util", "//tensorflow/compiler/xla:xla_data_proto", "//tensorflow/compiler/xla/client:xla_builder", @@ -173,13 +177,46 @@ cc_library( ], ) +cc_library( + name = "slicing", + srcs = ["slicing.cc"], + hdrs = ["slicing.h"], + deps = [ + "//tensorflow/compiler/xla:shape_util", + "//tensorflow/compiler/xla:status_macros", + "//tensorflow/compiler/xla:statusor", + "//tensorflow/compiler/xla:types", + "//tensorflow/compiler/xla:util", + "//tensorflow/compiler/xla:xla_data_proto", + "//tensorflow/compiler/xla/client:xla_builder", + "@com_google_absl//absl/types:span", + ], +) + +xla_test( + name = "slicing_test", + srcs = ["slicing_test.cc"], + tags = ["enable_for_xla_interpreter"], + deps = [ + ":slicing", + "//tensorflow/compiler/xla:literal_util", + "//tensorflow/compiler/xla:test", + "//tensorflow/compiler/xla:types", + "//tensorflow/compiler/xla:xla_data_proto", + "//tensorflow/compiler/xla/client:xla_builder", + "//tensorflow/compiler/xla/tests:client_library_test_base", + "//tensorflow/compiler/xla/tests:xla_internal_test_main", + ], +) + cc_library( name = "sorting", srcs = ["sorting.cc"], hdrs = ["sorting.h"], deps = [ - ":numeric", + "//tensorflow/compiler/xla:shape_util", "//tensorflow/compiler/xla:types", + "//tensorflow/compiler/xla:util", "//tensorflow/compiler/xla:xla_data_proto", "//tensorflow/compiler/xla/client:xla_builder", ], @@ -188,10 +225,6 @@ cc_library( xla_test( name = "sorting_test", srcs = ["sorting_test.cc"], - blacklisted_backends = [ - "cpu", - "gpu", - ], tags = ["enable_for_xla_interpreter"], deps = [ ":sorting", diff --git a/tensorflow/compiler/xla/client/lib/math.cc b/tensorflow/compiler/xla/client/lib/math.cc index 08a887a6e46..36fdda39b41 100644 --- a/tensorflow/compiler/xla/client/lib/math.cc +++ b/tensorflow/compiler/xla/client/lib/math.cc @@ -268,17 +268,16 @@ XlaOp Digamma(XlaOp input) { // Implements Banker's rounding: numbers that are equidistant between two // integers are rounded towards even. XlaOp RoundToEven(XlaOp x) { - auto half = xla::ScalarLike(x, 0.5); - auto one = xla::ScalarLike(x, 1.0); - auto two = xla::ScalarLike(x, 2.0); + auto half = ScalarLike(x, 0.5); + auto one = ScalarLike(x, 1.0); + auto two = ScalarLike(x, 2.0); - auto round_val = xla::Floor(x); + auto round_val = Floor(x); auto fraction = x - round_val; - auto nearest_even_int = round_val - two * xla::Floor(half * x); - auto is_odd = xla::Eq(nearest_even_int, one); - return xla::Select(xla::Or(xla::Gt(fraction, half), - xla::And(xla::Eq(fraction, half), is_odd)), - round_val + one, round_val); + auto nearest_even_int = round_val - two * Floor(half * x); + auto is_odd = Eq(nearest_even_int, one); + return Select(Or(Gt(fraction, half), And(Eq(fraction, half), is_odd)), + round_val + one, round_val); } // Trigonometric functions. @@ -320,4 +319,13 @@ XlaOp Cosh(XlaOp x) { return (Exp(x) + Exp(-x)) * ScalarLike(x, 0.5); } XlaOp Sinh(XlaOp x) { return (Exp(x) - Exp(-x)) * ScalarLike(x, 0.5); } +XlaOp MaybeConjugate(XlaOp x, bool conjugate) { + XlaBuilder* builder = x.builder(); + return builder->ReportErrorOrReturn([&]() -> StatusOr { + TF_ASSIGN_OR_RETURN(Shape shape, builder->GetShape(x)); + auto perform_conj = shape.element_type() == C64 && conjugate; + return perform_conj ? Conj(x) : x; + }); +} + } // namespace xla diff --git a/tensorflow/compiler/xla/client/lib/math.h b/tensorflow/compiler/xla/client/lib/math.h index 3f06d04b9ae..17612bf9fdc 100644 --- a/tensorflow/compiler/xla/client/lib/math.h +++ b/tensorflow/compiler/xla/client/lib/math.h @@ -86,6 +86,10 @@ XlaOp Cosh(XlaOp x); // Computes the hyperbolic sine of 'x'. XlaOp Sinh(XlaOp x); +// Applies a complex conjugation operation if `a` is complex and `conjugate` +// is true, otherwise returns its argument. +xla::XlaOp MaybeConjugate(xla::XlaOp x, bool conjugate); + } // namespace xla #endif // TENSORFLOW_COMPILER_XLA_CLIENT_LIB_MATH_H_ diff --git a/tensorflow/compiler/xla/client/lib/matrix.cc b/tensorflow/compiler/xla/client/lib/matrix.cc new file mode 100644 index 00000000000..ffd744d1908 --- /dev/null +++ b/tensorflow/compiler/xla/client/lib/matrix.cc @@ -0,0 +1,185 @@ +/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/compiler/xla/client/lib/matrix.h" + +#include +#include + +#include "absl/types/span.h" +#include "tensorflow/compiler/xla/client/lib/arithmetic.h" +#include "tensorflow/compiler/xla/client/lib/constants.h" +#include "tensorflow/compiler/xla/client/xla_builder.h" +#include "tensorflow/compiler/xla/shape_util.h" +#include "tensorflow/compiler/xla/status_macros.h" +#include "tensorflow/compiler/xla/statusor.h" +#include "tensorflow/compiler/xla/util.h" + +namespace xla { + +XlaOp IdentityMatrix(XlaBuilder* builder, PrimitiveType type, int64 m, + int64 n) { + auto a = Iota(builder, type, m); + auto b = Iota(builder, type, n); + auto indicator = Eq(a, Broadcast(b, {m}), /*broadcast_dimensions=*/{0}); + return ConvertElementType(indicator, type); +} + +XlaOp GetMatrixDiagonal(XlaOp x) { + XlaBuilder* builder = x.builder(); + return builder->ReportErrorOrReturn([&]() -> StatusOr { + TF_ASSIGN_OR_RETURN(Shape shape, builder->GetShape(x)); + const int64 n_dims = ShapeUtil::Rank(shape); + TF_RET_CHECK(n_dims >= 2); + const int64 m = shape.dimensions(n_dims - 2); + const int64 n = shape.dimensions(n_dims - 1); + absl::Span major_dims = + AsInt64Slice(shape.dimensions()).subspan(/*pos=*/0, /*len=*/n_dims - 2); + auto a = Iota(builder, U32, n); + auto b = Iota(builder, U32, m); + auto indicator = Eq(b, Broadcast(a, {m}), /*broadcast_dimensions=*/{0}); + auto mask = Broadcast(indicator, major_dims); + + // TPUs don't support S64 add reduction at the moment. But fortunately + // OR-reductions work just as well for integers. + XlaComputation reducer = + primitive_util::IsIntegralType(shape.element_type()) + ? CreateScalarOrComputation(shape.element_type(), builder) + : CreateScalarAddComputation(shape.element_type(), builder); + + return Reduce(Select(mask, x, Zeros(builder, shape)), ScalarLike(x, 0), + reducer, {m >= n ? n_dims - 2 : n_dims - 1}); + }); +} + +XlaOp Triangle(XlaOp x, bool lower) { + XlaBuilder* builder = x.builder(); + return builder->ReportErrorOrReturn([&]() -> StatusOr { + TF_ASSIGN_OR_RETURN(Shape shape, builder->GetShape(x)); + const int64 n_dims = ShapeUtil::Rank(shape); + TF_RET_CHECK(n_dims >= 2); + const int64 m = shape.dimensions(n_dims - 2); + const int64 n = shape.dimensions(n_dims - 1); + absl::Span major_dims = + AsInt64Slice(shape.dimensions()).subspan(/*pos=*/0, /*len=*/n_dims - 2); + auto a = Iota(builder, U32, n); + auto b = Iota(builder, U32, m); + XlaOp indicator; + if (lower) { + indicator = Ge(b, Broadcast(a, {m}), /*broadcast_dimensions=*/{0}); + } else { + indicator = Le(b, Broadcast(a, {m}), /*broadcast_dimensions=*/{0}); + } + auto mask = Broadcast(indicator, major_dims); + + return Select(mask, x, Zeros(builder, shape)); + }); +} + +XlaOp UpperTriangle(XlaOp x) { return Triangle(x, false); } + +XlaOp LowerTriangle(XlaOp x) { return Triangle(x, true); } + +XlaOp BatchDot(XlaOp x, XlaOp y, PrecisionConfig::Precision precision) { + XlaBuilder* builder = x.builder(); + return builder->ReportErrorOrReturn([&]() -> StatusOr { + TF_ASSIGN_OR_RETURN(Shape x_shape, builder->GetShape(x)); + TF_ASSIGN_OR_RETURN(Shape y_shape, builder->GetShape(y)); + + // Check that both tensors have the same number of dimensions. There must be + // at least two (the batch dimensions can be empty). + if (ShapeUtil::Rank(x_shape) != ShapeUtil::Rank(y_shape)) { + return InvalidArgument( + "Arguments to BatchDot have different ranks: %s vs. %s", + ShapeUtil::HumanString(x_shape), ShapeUtil::HumanString(y_shape)); + } + const int ndims = ShapeUtil::Rank(x_shape); + if (ndims < 2) { + return InvalidArgument( + "Arguments to BatchDot must have rank >= 2: got %d", ndims); + } + + // The batch dimensions must be equal and the matrix dimensions must be + // valid. + std::vector batch_dimension_numbers; + for (int i = 0; i < ndims - 2; ++i) { + if (x_shape.dimensions(i) != y_shape.dimensions(i)) { + return InvalidArgument( + "Dimension %d of inputs to BatchDot must be equal: shapes %s vs %s", + i, ShapeUtil::HumanString(x_shape), + ShapeUtil::HumanString(y_shape)); + } + batch_dimension_numbers.push_back(i); + } + + int x_inner_dim = ndims - 1; + int y_inner_dim = ndims - 2; + if (x_shape.dimensions(x_inner_dim) != y_shape.dimensions(y_inner_dim)) { + return InvalidArgument( + "Dimensions %d and %d of arguments to BatchDot must be equal: " + "shapes %s vs %s", + x_inner_dim, y_inner_dim, ShapeUtil::HumanString(x_shape), + ShapeUtil::HumanString(y_shape)); + } + + // Check for zero lhs/rhs dim size. + if (ShapeUtil::IsZeroElementArray(x_shape) || + ShapeUtil::IsZeroElementArray(y_shape)) { + std::vector dimensions(batch_dimension_numbers.size()); + for (int i = 0; i < batch_dimension_numbers.size(); ++i) { + dimensions[i] = x_shape.dimensions(batch_dimension_numbers[i]); + } + int x_outer_dim = ndims - 2; + int y_outer_dim = ndims - 1; + dimensions.push_back(x_shape.dimensions(x_outer_dim)); + dimensions.push_back(y_shape.dimensions(y_outer_dim)); + return Broadcast( + ConstantLiteral(builder, LiteralUtil::Zero(x_shape.element_type())), + dimensions); + } + + PrecisionConfig precision_proto; + precision_proto.add_operand_precision(precision); + precision_proto.add_operand_precision(precision); + + DotDimensionNumbers dot_dnums; + dot_dnums.add_lhs_contracting_dimensions(x_inner_dim); + dot_dnums.add_rhs_contracting_dimensions(y_inner_dim); + for (auto batch_dimension_number : batch_dimension_numbers) { + dot_dnums.add_lhs_batch_dimensions(batch_dimension_number); + dot_dnums.add_rhs_batch_dimensions(batch_dimension_number); + } + + return DotGeneral(x, y, dot_dnums, &precision_proto); + }); +} + +XlaOp TransposeInMinorDims(XlaOp x) { + XlaBuilder* builder = x.builder(); + return builder->ReportErrorOrReturn([&]() -> StatusOr { + TF_ASSIGN_OR_RETURN(Shape shape, builder->GetShape(x)); + const int64 n_dims = ShapeUtil::Rank(shape); + TF_RET_CHECK(n_dims >= 2); + std::vector permutation(n_dims); + std::iota(permutation.begin(), permutation.end(), 0); + std::swap(permutation[n_dims - 1], permutation[n_dims - 2]); + return Transpose(x, permutation); + }); +} + +XlaOp MaybeTransposeInMinorDims(XlaOp x, bool transpose) { + return transpose ? TransposeInMinorDims(x) : x; +} +} // namespace xla diff --git a/tensorflow/compiler/xla/client/lib/numeric.h b/tensorflow/compiler/xla/client/lib/matrix.h similarity index 56% rename from tensorflow/compiler/xla/client/lib/numeric.h rename to tensorflow/compiler/xla/client/lib/matrix.h index efd8cdc2572..8856f99c7a0 100644 --- a/tensorflow/compiler/xla/client/lib/numeric.h +++ b/tensorflow/compiler/xla/client/lib/matrix.h @@ -13,8 +13,8 @@ See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ -#ifndef TENSORFLOW_COMPILER_XLA_CLIENT_LIB_NUMERIC_H_ -#define TENSORFLOW_COMPILER_XLA_CLIENT_LIB_NUMERIC_H_ +#ifndef TENSORFLOW_COMPILER_XLA_CLIENT_LIB_MATRIX_H_ +#define TENSORFLOW_COMPILER_XLA_CLIENT_LIB_MATRIX_H_ #include "tensorflow/compiler/xla/client/xla_builder.h" #include "tensorflow/compiler/xla/types.h" @@ -22,9 +22,6 @@ limitations under the License. namespace xla { -// Returns a rank 1 tensor of `type` containing values [0, 1, 2, ...]. -XlaOp Iota(XlaBuilder* builder, PrimitiveType type, int64 size); - // Returns an m x n matrix with 1s on the diagonal elements, zeros everywhere // else. XlaOp IdentityMatrix(XlaBuilder* builder, PrimitiveType type, int64 m, int64 n); @@ -43,6 +40,34 @@ XlaOp UpperTriangle(XlaOp x); // Get the lower triangle part of the last two dimensions XlaOp LowerTriangle(XlaOp x); +// Multiplies slices of two tensors in batches. + +// Multiplies all slices of `Tensor` `x` and `y` (each slice can be +// viewed as an element of a batch), and arranges the individual results +// in a single output tensor of the same batch size. +// +// The input tensors `x` and `y` are 2-D or higher with shape `[..., r_x, c_x]` +// and `[..., r_y, c_y]`. +// +// The output tensor is 2-D or higher with shape `[..., r_o, c_o]`, where: +// +// r_o = c_x if transpose_x else r_x +// c_o = r_y if transpose_y else c_y +// +// It is computed as: +// +// output[..., :, :] = matrix(x[..., :, :]) * matrix(y[..., :, :]) +xla::XlaOp BatchDot( + xla::XlaOp x, xla::XlaOp y, + xla::PrecisionConfig::Precision precision = xla::PrecisionConfig::DEFAULT); + +// Transposes a stack of matrices `x` by swapping the last two dimensions. +xla::XlaOp TransposeInMinorDims(xla::XlaOp x); + +// Transposes `x` in its minor dimensions if `transpose` is true, otherwise +// returns `x` unchanged. +xla::XlaOp MaybeTransposeInMinorDims(xla::XlaOp x, bool transpose); + } // namespace xla -#endif // TENSORFLOW_COMPILER_XLA_CLIENT_LIB_NUMERIC_H_ +#endif // TENSORFLOW_COMPILER_XLA_CLIENT_LIB_MATRIX_H_ diff --git a/tensorflow/compiler/xla/client/lib/numeric_test.cc b/tensorflow/compiler/xla/client/lib/matrix_test.cc similarity index 53% rename from tensorflow/compiler/xla/client/lib/numeric_test.cc rename to tensorflow/compiler/xla/client/lib/matrix_test.cc index 7d6aedd4946..0593a7517ac 100644 --- a/tensorflow/compiler/xla/client/lib/numeric_test.cc +++ b/tensorflow/compiler/xla/client/lib/matrix_test.cc @@ -13,7 +13,9 @@ See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ -#include "tensorflow/compiler/xla/client/lib/numeric.h" +#include "tensorflow/compiler/xla/client/lib/matrix.h" + +#include "tensorflow/compiler/xla/client/lib/slicing.h" #include "tensorflow/compiler/xla/client/xla_builder.h" #include "tensorflow/compiler/xla/test.h" #include "tensorflow/compiler/xla/tests/client_library_test_base.h" @@ -24,13 +26,13 @@ limitations under the License. namespace xla { namespace { -class NumericTest : public ClientLibraryTestBase { +class MatrixTest : public ClientLibraryTestBase { protected: template void TestMatrixDiagonal(); }; -XLA_TEST_F(NumericTest, Triangle) { +XLA_TEST_F(MatrixTest, Triangle) { XlaBuilder builder(TestName()); Array3D input(2, 3, 4); input.FillIota(0); @@ -45,7 +47,7 @@ XLA_TEST_F(NumericTest, Triangle) { } template -void NumericTest::TestMatrixDiagonal() { +void MatrixTest::TestMatrixDiagonal() { XlaBuilder builder("GetMatrixDiagonal"); Array3D input(2, 3, 4); input.FillIota(0); @@ -58,11 +60,46 @@ void NumericTest::TestMatrixDiagonal() { ComputeAndCompareR2(&builder, expected, {a_data.get()}); } -XLA_TEST_F(NumericTest, GetMatrixDiagonal_S32) { TestMatrixDiagonal(); } +XLA_TEST_F(MatrixTest, GetMatrixDiagonal_S32) { TestMatrixDiagonal(); } -XLA_TEST_F(NumericTest, GetMatrixDiagonal_S64) { TestMatrixDiagonal(); } +XLA_TEST_F(MatrixTest, GetMatrixDiagonal_S64) { TestMatrixDiagonal(); } -XLA_TEST_F(NumericTest, GetMatrixDiagonal_F32) { TestMatrixDiagonal(); } +XLA_TEST_F(MatrixTest, GetMatrixDiagonal_F32) { TestMatrixDiagonal(); } +Array3D BatchedAValsFull() { + return {{ + {2, 0, 1, 2}, + {3, 6, 0, 1}, + {4, 7, 9, 0}, + {5, 8, 10, 11}, + }, + { + {16, 24, 8, 12}, + {24, 61, 82, 48}, + {8, 82, 456, 106}, + {12, 48, 106, 62}, + }}; +} + +XLA_TEST_F(MatrixTest, RowBatchDot) { + XlaBuilder builder(TestName()); + + int n = 4; + + XlaOp a, row, index; + auto a_data = + CreateR3Parameter(BatchedAValsFull(), 0, "a", &builder, &a); + auto row_data = CreateR3Parameter({{{9, 1, 0, 0}}, {{2, 4, 0, 0}}}, 1, + "row", &builder, &row); + // Select {{3, 6, 0, 1}, {24, 61, 82, 48}} out of BatchedAValsFull(). + auto index_data = CreateR0Parameter(1, 2, "index", &builder, &index); + + auto l_index = DynamicSliceInMinorDims( + a, {index, ConstantR0(&builder, 0)}, {1, n}); + BatchDot(l_index, TransposeInMinorDims(row)); + + ComputeAndCompareR3(&builder, {{{33}}, {{292}}}, + {a_data.get(), row_data.get(), index_data.get()}); +} } // namespace } // namespace xla diff --git a/tensorflow/compiler/xla/client/lib/numeric.cc b/tensorflow/compiler/xla/client/lib/numeric.cc deleted file mode 100644 index 377654220b5..00000000000 --- a/tensorflow/compiler/xla/client/lib/numeric.cc +++ /dev/null @@ -1,89 +0,0 @@ -/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -#include -#include - -#include "absl/types/span.h" -#include "tensorflow/compiler/xla/client/lib/arithmetic.h" -#include "tensorflow/compiler/xla/client/lib/constants.h" -#include "tensorflow/compiler/xla/client/lib/numeric.h" - -namespace xla { - -XlaOp IdentityMatrix(XlaBuilder* builder, PrimitiveType type, int64 m, - int64 n) { - auto a = Iota(builder, type, m); - auto b = Iota(builder, type, n); - auto indicator = Eq(a, Broadcast(b, {m}), /*broadcast_dimensions=*/{0}); - return ConvertElementType(indicator, type); -} - -XlaOp GetMatrixDiagonal(XlaOp x) { - XlaBuilder* builder = x.builder(); - return builder->ReportErrorOrReturn([&]() -> StatusOr { - TF_ASSIGN_OR_RETURN(Shape shape, builder->GetShape(x)); - const int64 n_dims = ShapeUtil::Rank(shape); - TF_RET_CHECK(n_dims >= 2); - const int64 m = shape.dimensions(n_dims - 2); - const int64 n = shape.dimensions(n_dims - 1); - absl::Span major_dims = - AsInt64Slice(shape.dimensions()).subspan(/*pos=*/0, /*len=*/n_dims - 2); - auto a = Iota(builder, U32, n); - auto b = Iota(builder, U32, m); - auto indicator = Eq(b, Broadcast(a, {m}), /*broadcast_dimensions=*/{0}); - auto mask = Broadcast(indicator, major_dims); - - // TPUs don't support S64 add reduction at the moment. But fortunately - // OR-reductions work just as well for integers. - XlaComputation reducer = - primitive_util::IsIntegralType(shape.element_type()) - ? CreateScalarOrComputation(shape.element_type(), builder) - : CreateScalarAddComputation(shape.element_type(), builder); - - return Reduce(Select(mask, x, Zeros(builder, shape)), ScalarLike(x, 0), - reducer, {m >= n ? n_dims - 2 : n_dims - 1}); - }); -} - -XlaOp Triangle(XlaOp x, bool lower) { - XlaBuilder* builder = x.builder(); - return builder->ReportErrorOrReturn([&]() -> StatusOr { - TF_ASSIGN_OR_RETURN(Shape shape, builder->GetShape(x)); - const int64 n_dims = ShapeUtil::Rank(shape); - TF_RET_CHECK(n_dims >= 2); - const int64 m = shape.dimensions(n_dims - 2); - const int64 n = shape.dimensions(n_dims - 1); - absl::Span major_dims = - AsInt64Slice(shape.dimensions()).subspan(/*pos=*/0, /*len=*/n_dims - 2); - auto a = Iota(builder, U32, n); - auto b = Iota(builder, U32, m); - xla::XlaOp indicator; - if (lower) { - indicator = Ge(b, Broadcast(a, {m}), /*broadcast_dimensions=*/{0}); - } else { - indicator = Le(b, Broadcast(a, {m}), /*broadcast_dimensions=*/{0}); - } - auto mask = Broadcast(indicator, major_dims); - - return Select(mask, x, Zeros(builder, shape)); - }); -} - -XlaOp UpperTriangle(XlaOp x) { return Triangle(x, false); } - -XlaOp LowerTriangle(XlaOp x) { return Triangle(x, true); } - -} // namespace xla diff --git a/tensorflow/compiler/xla/client/lib/prng.cc b/tensorflow/compiler/xla/client/lib/prng.cc index c6f68c8ee2f..85b9e1827dc 100644 --- a/tensorflow/compiler/xla/client/lib/prng.cc +++ b/tensorflow/compiler/xla/client/lib/prng.cc @@ -18,7 +18,6 @@ limitations under the License. #include "absl/base/casts.h" #include "tensorflow/compiler/xla/client/lib/constants.h" #include "tensorflow/compiler/xla/client/lib/math.h" -#include "tensorflow/compiler/xla/client/lib/numeric.h" #include "tensorflow/compiler/xla/client/xla_builder.h" #include "tensorflow/compiler/xla/util.h" diff --git a/tensorflow/compiler/xla/client/lib/slicing.cc b/tensorflow/compiler/xla/client/lib/slicing.cc new file mode 100644 index 00000000000..f8c7df3ff51 --- /dev/null +++ b/tensorflow/compiler/xla/client/lib/slicing.cc @@ -0,0 +1,134 @@ +/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/compiler/xla/client/lib/slicing.h" + +namespace xla { + +XlaOp SliceInMinorDims(XlaOp x, absl::Span start, + absl::Span end) { + XlaBuilder* builder = x.builder(); + return builder->ReportErrorOrReturn([&]() -> StatusOr { + TF_RET_CHECK(start.size() == end.size()); + int64 n_minor_dims = start.size(); + + TF_ASSIGN_OR_RETURN(Shape shape, builder->GetShape(x)); + + const int64 n_dims = ShapeUtil::Rank(shape); + TF_RET_CHECK(n_minor_dims <= n_dims); + auto major_dims = AsInt64Slice(shape.dimensions()) + .subspan( + /*pos=*/0, + /*len=*/n_dims - n_minor_dims); + + // Prepends 0s in the major dim + std::vector padded_start(n_dims, 0); + std::copy(start.begin(), start.end(), + padded_start.begin() + major_dims.size()); + + // Prepends the shape of the major dims. + std::vector padded_end(n_dims); + std::copy(major_dims.begin(), major_dims.end(), padded_end.begin()); + std::copy(end.begin(), end.end(), padded_end.begin() + major_dims.size()); + + std::vector strides(n_dims, 1); + return Slice(x, padded_start, padded_end, strides); + }); +} + +XlaOp UpdateSlice(XlaOp x, XlaOp update, absl::Span start) { + XlaBuilder* builder = x.builder(); + return builder->ReportErrorOrReturn([&]() -> StatusOr { + // TODO(phawkins): make int64 work on all backends, remove the int32 cast. + std::vector start_as_int32(start.begin(), start.end()); + auto start_constant = ConstantR1(builder, start_as_int32); + TF_ASSIGN_OR_RETURN(Shape shape, builder->GetShape(x)); + const int64 n_dims = ShapeUtil::Rank(shape); + TF_ASSIGN_OR_RETURN(Shape start_constant_shape, + builder->GetShape(start_constant)); + const int64 start_length = + ShapeUtil::GetDimension(start_constant_shape, -1); + TF_RET_CHECK(start_length == n_dims); + return DynamicUpdateSlice(x, update, start_constant); + }); +} + +XlaOp UpdateSliceInMinorDims(XlaOp x, XlaOp update, + absl::Span start) { + XlaBuilder* builder = x.builder(); + return builder->ReportErrorOrReturn([&]() -> StatusOr { + TF_ASSIGN_OR_RETURN(Shape shape, builder->GetShape(x)); + const int64 n_dims = ShapeUtil::Rank(shape); + const int64 n_minor_dims = start.size(); + TF_RET_CHECK(n_minor_dims <= n_dims); + std::vector padded_start(n_dims, 0); + std::copy(start.begin(), start.end(), + padded_start.begin() + (n_dims - n_minor_dims)); + return UpdateSlice(x, update, padded_start); + }); +} + +namespace { + +std::vector ConcatVectors(absl::Span xs, + absl::Span ys) { + std::vector output(xs.size() + ys.size()); + std::copy(xs.begin(), xs.end(), output.begin()); + std::copy(ys.begin(), ys.end(), output.begin() + xs.size()); + return output; +} + +XlaOp PrependZerosInMajorDims(XlaOp x, absl::Span starts) { + XlaBuilder* builder = x.builder(); + return builder->ReportErrorOrReturn([&]() -> StatusOr { + TF_ASSIGN_OR_RETURN(Shape shape, builder->GetShape(x)); + const int64 n_dims = ShapeUtil::Rank(shape); + auto zero = Reshape(ConstantR0(builder, 0), {1}); + std::vector padded_starts(n_dims, zero); + for (int i = 0; i < starts.size(); ++i) { + padded_starts[n_dims - starts.size() + i] = Reshape(starts[i], {1}); + } + return ConcatInDim(builder, padded_starts, 0); + }); +} + +} // namespace + +XlaOp DynamicSliceInMinorDims(XlaOp x, absl::Span starts, + absl::Span sizes) { + XlaBuilder* builder = x.builder(); + return builder->ReportErrorOrReturn([&]() -> StatusOr { + TF_ASSIGN_OR_RETURN(Shape shape, builder->GetShape(x)); + const int64 n_dims = ShapeUtil::Rank(shape); + int64 n_minor_dims = starts.size(); + TF_RET_CHECK(n_minor_dims == sizes.size()); + TF_RET_CHECK(n_minor_dims <= n_dims); + auto major_dims = AsInt64Slice(shape.dimensions()) + .subspan( + /*pos=*/0, + /*len=*/n_dims - sizes.size()); + auto padded_starts = PrependZerosInMajorDims(x, starts); + auto padded_sizes = ConcatVectors(major_dims, sizes); + return DynamicSlice(x, padded_starts, padded_sizes); + }); +} + +XlaOp DynamicUpdateSliceInMinorDims(XlaOp x, XlaOp update, + absl::Span starts) { + auto padded_starts = PrependZerosInMajorDims(x, starts); + return DynamicUpdateSlice(x, update, padded_starts); +} + +} // namespace xla diff --git a/tensorflow/compiler/xla/client/lib/slicing.h b/tensorflow/compiler/xla/client/lib/slicing.h new file mode 100644 index 00000000000..6c482a38b54 --- /dev/null +++ b/tensorflow/compiler/xla/client/lib/slicing.h @@ -0,0 +1,48 @@ +/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "absl/types/span.h" +#include "tensorflow/compiler/xla/client/xla_builder.h" +#include "tensorflow/compiler/xla/types.h" + +#ifndef TENSORFLOW_COMPILER_XLA_CLIENT_LIB_SLICING_H_ +#define TENSORFLOW_COMPILER_XLA_CLIENT_LIB_SLICING_H_ + +namespace xla { + +// Updates a slice of 'x', i.e., +// x[start[0], ..., start[n]] = update +XlaOp UpdateSlice(XlaOp x, XlaOp update, absl::Span start); + +// Performs a slice in the minor dimensions of a tensor. +// x[..., start[0]:end[0], ..., start[n]:end[n]] +XlaOp SliceInMinorDims(XlaOp x, absl::Span start, + absl::Span end); + +// Updates a slice of 'x', where 'start' contains a list of minor dimensions: +// x[..., start[0]:..., ..., start[n]:...] = update +XlaOp UpdateSliceInMinorDims(XlaOp x, XlaOp update, + absl::Span start); + +// Performs a dynamic slice in the minor dimensions of a tensor. +XlaOp DynamicSliceInMinorDims(XlaOp x, absl::Span starts, + absl::Span sizes); + +XlaOp DynamicUpdateSliceInMinorDims(XlaOp x, XlaOp update, + absl::Span starts); + +} // namespace xla + +#endif // TENSORFLOW_COMPILER_XLA_CLIENT_LIB_SLICING_H_ diff --git a/tensorflow/compiler/tf2xla/lib/util_test.cc b/tensorflow/compiler/xla/client/lib/slicing_test.cc similarity index 67% rename from tensorflow/compiler/tf2xla/lib/util_test.cc rename to tensorflow/compiler/xla/client/lib/slicing_test.cc index 442fe92c34c..8d362119e01 100644 --- a/tensorflow/compiler/tf2xla/lib/util_test.cc +++ b/tensorflow/compiler/xla/client/lib/slicing_test.cc @@ -13,28 +13,19 @@ See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ -#include "tensorflow/compiler/tf2xla/lib/util.h" +#include "tensorflow/compiler/xla/client/lib/slicing.h" -#include -#include -#include - -#include "tensorflow/compiler/tf2xla/lib/batch_dot.h" -#include "tensorflow/compiler/xla/array2d.h" -#include "tensorflow/compiler/xla/literal.h" -#include "tensorflow/compiler/xla/statusor.h" +#include "tensorflow/compiler/xla/client/xla_builder.h" +#include "tensorflow/compiler/xla/literal_util.h" #include "tensorflow/compiler/xla/test.h" #include "tensorflow/compiler/xla/tests/client_library_test_base.h" -#include "tensorflow/compiler/xla/tests/literal_test_util.h" #include "tensorflow/compiler/xla/tests/test_macros.h" #include "tensorflow/compiler/xla/types.h" -#include "tensorflow/core/lib/core/status_test_util.h" -namespace tensorflow { +namespace xla { namespace { -using UtilTest = xla::ClientLibraryTestBase; -using UtilLeftLookingTest = xla::ClientLibraryTestBase; +using SlicingTest = xla::ClientLibraryTestBase; xla::Array2D BValsRight() { return {{1, 2, 3, 4}, {5, 6, 7, 8}, {9, 10, 11, 12}}; @@ -63,7 +54,7 @@ xla::Array3D BatchedAValsFull() { }}; } -XLA_TEST_F(UtilTest, Simple2dLookup) { +XLA_TEST_F(SlicingTest, Simple2dLookup) { xla::XlaBuilder builder(TestName()); xla::XlaOp a, x, y; @@ -77,7 +68,7 @@ XLA_TEST_F(UtilTest, Simple2dLookup) { xla::ErrorSpec(1e-2, 1e-2)); } -XLA_TEST_F(UtilTest, Simple3dLookup) { +XLA_TEST_F(SlicingTest, Simple3dLookup) { xla::XlaBuilder builder(TestName()); xla::XlaOp a, index; @@ -92,7 +83,7 @@ XLA_TEST_F(UtilTest, Simple3dLookup) { {a_data.get(), index_data.get()}); } -XLA_TEST_F(UtilTest, SimpleSliceUpdate) { +XLA_TEST_F(SlicingTest, SimpleSliceUpdate) { xla::XlaBuilder builder(TestName()); xla::XlaOp a, b, x, y; @@ -111,26 +102,5 @@ XLA_TEST_F(UtilTest, SimpleSliceUpdate) { {a_data.get(), b_data.get(), x_data.get(), y_data.get()}); } -XLA_TEST_F(UtilTest, RowBatchDot) { - xla::XlaBuilder builder(TestName()); - - int n = 4; - - xla::XlaOp a, row, index; - auto a_data = - CreateR3Parameter(BatchedAValsFull(), 0, "a", &builder, &a); - auto row_data = CreateR3Parameter({{{9, 1, 0, 0}}, {{2, 4, 0, 0}}}, 1, - "row", &builder, &row); - // Select {{3, 6, 0, 1}, {24, 61, 82, 48}} out of BatchedAValsFull(). - auto index_data = CreateR0Parameter(1, 2, "index", &builder, &index); - - auto l_index = DynamicSliceInMinorDims( - a, {index, xla::ConstantR0(&builder, 0)}, {1, n}); - BatchDot(l_index, row, /*transpose_x=*/false, /*transpose_y=*/true); - - ComputeAndCompareR3(&builder, {{{33}}, {{292}}}, - {a_data.get(), row_data.get(), index_data.get()}); -} - } // namespace -} // namespace tensorflow +} // namespace xla diff --git a/tensorflow/compiler/xla/client/lib/sorting.cc b/tensorflow/compiler/xla/client/lib/sorting.cc index 0475fd9c94f..e8553a08bb0 100644 --- a/tensorflow/compiler/xla/client/lib/sorting.cc +++ b/tensorflow/compiler/xla/client/lib/sorting.cc @@ -14,7 +14,9 @@ limitations under the License. ==============================================================================*/ #include "tensorflow/compiler/xla/client/lib/sorting.h" -#include "tensorflow/compiler/xla/client/lib/numeric.h" +#include "tensorflow/compiler/xla/client/xla_builder.h" +#include "tensorflow/compiler/xla/shape_util.h" +#include "tensorflow/compiler/xla/util.h" namespace xla { @@ -23,13 +25,12 @@ XlaOp TopK(XlaOp input, int64 k) { return builder->ReportErrorOrReturn([&]() -> StatusOr { TF_ASSIGN_OR_RETURN(Shape input_shape, builder->GetShape(input)); int last_dim = input_shape.dimensions_size() - 1; - int last_dim_size = input_shape.dimensions(last_dim); - XlaOp iota_s32 = Iota(builder, S32, last_dim_size); + Shape iota_shape = + ShapeUtil::MakeShape(S32, AsInt64Slice(input_shape.dimensions())); + XlaOp iota_s32 = Iota(builder, iota_shape, last_dim); auto input_dims = input_shape.dimensions(); - std::vector broadcast_dims(input_dims.begin(), input_dims.end() - 1); - XlaOp broadcast_s32 = Broadcast(iota_s32, broadcast_dims); - XlaOp sort_result = Sort(Neg(input), {broadcast_s32}); + XlaOp sort_result = Sort(Neg(input), {iota_s32}); std::vector start_indices(input_shape.dimensions_size(), 0); std::vector limit_indices(input_dims.begin(), input_dims.end()); limit_indices[last_dim] = k; diff --git a/tensorflow/compiler/xla/client/lib/sorting_test.cc b/tensorflow/compiler/xla/client/lib/sorting_test.cc index fef98c99230..27ff36c7491 100644 --- a/tensorflow/compiler/xla/client/lib/sorting_test.cc +++ b/tensorflow/compiler/xla/client/lib/sorting_test.cc @@ -14,6 +14,9 @@ limitations under the License. ==============================================================================*/ #include "tensorflow/compiler/xla/client/lib/sorting.h" + +#include + #include "tensorflow/compiler/xla/client/xla_builder.h" #include "tensorflow/compiler/xla/test.h" #include "tensorflow/compiler/xla/tests/client_library_test_base.h" @@ -41,6 +44,28 @@ XLA_TEST_F(SortingTest, TopK3From8Indices) { ComputeAndCompareR1(&builder, {0, 1, 2}, {}); } +// TODO(b/119930279): enable this test. +XLA_TEST_F(SortingTest, DISABLED_TopKFullSortMinInt) { + XlaBuilder builder(TestName()); + auto x_rev = ConstantR1(&builder, {std::numeric_limits::min(), + std::numeric_limits::min() + 1, + std::numeric_limits::max()}); + xla::GetTupleElement(xla::TopK(x_rev, 3), 1); + ComputeAndCompareR1(&builder, {2, 1, 0}, {}); +} + +XLA_TEST_F(SortingTest, NOT_TopKFullSortMinInt) { + XlaBuilder builder(TestName()); + auto x_rev = ConstantR1(&builder, {std::numeric_limits::min(), + std::numeric_limits::min() + 1, + std::numeric_limits::max()}); + xla::GetTupleElement(xla::TopK(x_rev, 3), 1); + // TopK currently negates the keys, which doesn't work correctly for + // std::numeric_limits::min(). Therefore, it will sort this key to the + // front instead of to the back. + ComputeAndCompareR1(&builder, {0, 2, 1}, {}); +} + XLA_TEST_F(SortingTest, TopKFullSort) { XlaBuilder builder(TestName()); const int kSize = 16; @@ -56,5 +81,13 @@ XLA_TEST_F(SortingTest, TopKFullSort) { ComputeAndCompareR1(&builder, inputs, {}); } +XLA_TEST_F(SortingTest, TopKFullSortWithDuplicates) { + XlaBuilder builder(TestName()); + XlaOp a; + auto a_data = CreateR1Parameter({1, 1, 2, 2, 1}, 0, "a", &builder, &a); + xla::GetTupleElement(xla::TopK(a, 5), 1); + ComputeAndCompareR1(&builder, {2, 3, 0, 1, 4}, {a_data.get()}); +} + } // namespace } // namespace xla diff --git a/tensorflow/compiler/xla/client/lib/testing.cc b/tensorflow/compiler/xla/client/lib/testing.cc index a44681f5862..a95bbf2c8c8 100644 --- a/tensorflow/compiler/xla/client/lib/testing.cc +++ b/tensorflow/compiler/xla/client/lib/testing.cc @@ -66,7 +66,7 @@ std::unique_ptr MakeFakeDataViaDeviceOrDie(const Shape& shape, XlaComputation computation = b.Build().ConsumeValueOrDie(); auto execution_options = CreateDefaultExecutionOptions(); - *execution_options.mutable_shape_with_output_layout() = shape; + *execution_options.mutable_shape_with_output_layout() = shape.ToProto(); return client->Execute(computation, /*arguments=*/{}, &execution_options) .ConsumeValueOrDie(); } @@ -98,8 +98,8 @@ std::vector> MakeFakeArgumentsOrDie( auto program_shape = computation.proto().host_program_shape(); std::vector> results; - for (const Shape& shape : program_shape.parameters()) { - results.push_back(MakeFakeDataOrDie(shape, client)); + for (const ShapeProto& shape : program_shape.parameters()) { + results.push_back(MakeFakeDataOrDie(Shape(shape), client)); } return results; } diff --git a/tensorflow/compiler/xla/client/local_client.cc b/tensorflow/compiler/xla/client/local_client.cc index f96b6c9c261..aaa5d6989ee 100644 --- a/tensorflow/compiler/xla/client/local_client.cc +++ b/tensorflow/compiler/xla/client/local_client.cc @@ -310,4 +310,28 @@ StatusOr LocalClient::ReplicaNumberToDeviceOrdinal(int replica_number) { return local_service_->ReplicaNumberToDeviceOrdinal(replica_number); } +StatusOr LocalClient::TransferToLocalServer( + const ::xla::BorrowingLiteral& literal, int device_oridinal) { + const ::xla::Shape& shape = literal.shape(); + + TF_ASSIGN_OR_RETURN( + ::xla::ScopedShapedBuffer shaped_buffer, + backend().transfer_manager()->AllocateScopedShapedBuffer( + shape, backend().memory_allocator(), device_oridinal)); + TF_ASSIGN_OR_RETURN(auto stream, + mutable_backend()->BorrowStream(device_oridinal)); + TF_RETURN_IF_ERROR(backend().transfer_manager()->TransferLiteralToDevice( + stream.get(), literal, shaped_buffer)); + std::vector<::xla::ScopedShapedBuffer> replicated_buffer; + replicated_buffer.emplace_back(std::move(shaped_buffer)); + ::xla::TransferToServerResponse result; + TF_ASSIGN_OR_RETURN(*result.mutable_data(), + local_service_->RegisterReplicatedBuffers( + std::move(replicated_buffer), + absl::StrCat("TransferToServer literal of shape ", + ::xla::ShapeUtil::HumanString(shape)))); + + return result; +} + } // namespace xla diff --git a/tensorflow/compiler/xla/client/local_client.h b/tensorflow/compiler/xla/client/local_client.h index e49451ca970..ddb36680e8b 100644 --- a/tensorflow/compiler/xla/client/local_client.h +++ b/tensorflow/compiler/xla/client/local_client.h @@ -129,6 +129,10 @@ class LocalClient : public Client { const Literal& literal, int device_ordinal, DeviceMemoryAllocator* allocator = nullptr); + // Transfer the BorrowingLiteral to the device with the given ordinal. + StatusOr TransferToLocalServer( + const ::xla::BorrowingLiteral& literal, int device_oridinal); + // Copy the data from the device contained in the given ShapedBuffer and // return as a Literal. StatusOr ShapedBufferToLiteral(const ShapedBuffer& shaped_buffer); diff --git a/tensorflow/compiler/xla/client/sharding_builder.cc b/tensorflow/compiler/xla/client/sharding_builder.cc index 176802b33ef..fb9ea6ec3fc 100644 --- a/tensorflow/compiler/xla/client/sharding_builder.cc +++ b/tensorflow/compiler/xla/client/sharding_builder.cc @@ -36,7 +36,7 @@ OpSharding Tile(const Shape& tile_shape, const TileAssignment& tile_assignment) { OpSharding result; result.set_type(OpSharding::Type::OpSharding_Type_OTHER); - *result.mutable_tile_shape() = tile_shape; + *result.mutable_tile_shape() = tile_shape.ToProto(); for (int64 dim : tile_assignment.dimensions()) { result.add_tile_assignment_dimensions(dim); } @@ -52,7 +52,7 @@ OpSharding Tile1D(const Shape& tile_shape, int64 num_tiles) { CHECK_EQ(ShapeUtil::Rank(tile_shape), 1); std::vector dimensions(1, num_tiles); - *result.mutable_tile_shape() = tile_shape; + *result.mutable_tile_shape() = tile_shape.ToProto(); auto& tile_dimension = (*result.mutable_tile_shape()->mutable_dimensions())[0]; tile_dimension = CeilOfRatio(static_cast(tile_dimension), num_tiles); diff --git a/tensorflow/compiler/xla/client/xla_builder.cc b/tensorflow/compiler/xla/client/xla_builder.cc index 0a587725d20..60df2ec3959 100644 --- a/tensorflow/compiler/xla/client/xla_builder.cc +++ b/tensorflow/compiler/xla/client/xla_builder.cc @@ -102,7 +102,7 @@ StatusOr XlaBuilder::GetShape(const XlaOp& op) const { TF_RETURN_IF_ERROR(first_error_); TF_ASSIGN_OR_RETURN(auto instr, LookUpInstruction(op)); - return instr->shape(); + return Shape(instr->shape()); } StatusOr> XlaBuilder::GetOperandShapes( @@ -155,7 +155,7 @@ StatusOr XlaBuilder::GetProgramShape(int64 root_id) const { ProgramShape program_shape; - *program_shape.mutable_result() = root_proto->shape(); + *program_shape.mutable_result() = Shape(root_proto->shape()); // Check that the parameter numbers are continuous from 0, and add parameter // shapes and names to the program shape. @@ -172,7 +172,7 @@ StatusOr XlaBuilder::GetProgramShape(int64 root_id) const { const int64 index = instr.parameter_number(); TF_RET_CHECK(index >= 0 && index < param_count) << "invalid parameter number: " << index; - *program_shape.mutable_parameters(index) = instr.shape(); + *program_shape.mutable_parameters(index) = Shape(instr.shape()); *program_shape.mutable_parameter_names(index) = instr.name(); } } @@ -239,6 +239,19 @@ void XlaBuilder::IsConstantVisitor(const int64 op_handle, visited->insert(op_handle); } +Status XlaBuilder::SetDynamicBinding(int64 dynamic_size_param_num, + ShapeIndex dynamic_size_param_index, + int64 target_param_num, + ShapeIndex target_param_index, + int64 target_dim_num) { + TF_RETURN_IF_ERROR(dynamic_parameter_binding_.Bind( + DynamicParameterBinding::DynamicParameter{dynamic_size_param_num, + dynamic_size_param_index}, + DynamicParameterBinding::DynamicDimension{ + target_param_num, target_param_index, target_dim_num})); + return Status::OK(); +} + XlaComputation XlaBuilder::BuildAndNoteError() { DCHECK(parent_builder_ != nullptr); auto build_status = Build(); @@ -275,7 +288,8 @@ StatusOr XlaBuilder::Build(int64 root_id) { HloComputationProto entry; SetProtoIdAndName(&entry, name_, kNameSeparator, GetNextId()); - TF_ASSIGN_OR_RETURN(*entry.mutable_program_shape(), GetProgramShape(root_id)); + TF_ASSIGN_OR_RETURN(ProgramShape program_shape, GetProgramShape(root_id)); + *entry.mutable_program_shape() = program_shape.ToProto(); entry.set_root_id(root_id); for (auto& instruction : instructions_) { @@ -297,6 +311,9 @@ StatusOr XlaBuilder::Build(int64 root_id) { } module->add_computations()->Swap(&entry); + *(module->mutable_dynamic_parameter_binding()) = + dynamic_parameter_binding_.ToProto(); + // Clear data held by this builder. this->instructions_.clear(); this->handle_to_index_.clear(); @@ -312,7 +329,7 @@ StatusOr XlaBuilder::InDimBroadcast( TF_RETURN_IF_ERROR(first_error_); HloInstructionProto instr; - *instr.mutable_shape() = shape; + *instr.mutable_shape() = shape.ToProto(); for (int64 dim : broadcast_dimensions) { instr.add_dimensions(dim); } @@ -363,8 +380,9 @@ XlaOp XlaBuilder::UnaryOp(HloOpcode unop, const XlaOp& operand) { return ReportErrorOrReturn([&]() -> StatusOr { HloInstructionProto instr; TF_ASSIGN_OR_RETURN(const Shape& operand_shape, GetShape(operand)); - TF_ASSIGN_OR_RETURN(*instr.mutable_shape(), + TF_ASSIGN_OR_RETURN(Shape shape, ShapeInference::InferUnaryOpShape(unop, operand_shape)); + *instr.mutable_shape() = shape.ToProto(); return AddInstruction(std::move(instr), unop, {operand}); }); } @@ -375,9 +393,10 @@ XlaOp XlaBuilder::BinaryOp(HloOpcode binop, const XlaOp& lhs, const XlaOp& rhs, HloInstructionProto instr; TF_ASSIGN_OR_RETURN(const Shape& lhs_shape, GetShape(lhs)); TF_ASSIGN_OR_RETURN(const Shape& rhs_shape, GetShape(rhs)); - TF_ASSIGN_OR_RETURN(*instr.mutable_shape(), + TF_ASSIGN_OR_RETURN(Shape shape, ShapeInference::InferBinaryOpShape( binop, lhs_shape, rhs_shape, broadcast_dimensions)); + *instr.mutable_shape() = shape.ToProto(); const int64 lhs_rank = ShapeUtil::Rank(lhs_shape); const int64 rhs_rank = ShapeUtil::Rank(rhs_shape); @@ -391,7 +410,7 @@ XlaOp XlaBuilder::BinaryOp(HloOpcode binop, const XlaOp& lhs, const XlaOp& rhs, const Shape& from_shape = should_broadcast_lhs ? lhs_shape : rhs_shape; std::vector to_size; - for (int64 size : instr.shape().dimensions()) { + for (int64 size : shape.dimensions()) { to_size.push_back(size); } for (int64 from_dim = 0; from_dim < ShapeUtil::Rank(from_shape); @@ -411,14 +430,14 @@ XlaOp XlaBuilder::BinaryOp(HloOpcode binop, const XlaOp& lhs, const XlaOp& rhs, } TF_ASSIGN_OR_RETURN(Shape updated_lhs_shape, GetShape(updated_lhs)); - if (!ShapeUtil::SameDimensions(instr.shape(), updated_lhs_shape)) { + if (!ShapeUtil::SameDimensions(shape, updated_lhs_shape)) { TF_ASSIGN_OR_RETURN(updated_lhs, - AddBroadcastSequence(instr.shape(), updated_lhs)); + AddBroadcastSequence(shape, updated_lhs)); } TF_ASSIGN_OR_RETURN(Shape updated_rhs_shape, GetShape(updated_rhs)); - if (!ShapeUtil::SameDimensions(instr.shape(), updated_rhs_shape)) { + if (!ShapeUtil::SameDimensions(shape, updated_rhs_shape)) { TF_ASSIGN_OR_RETURN(updated_rhs, - AddBroadcastSequence(instr.shape(), updated_rhs)); + AddBroadcastSequence(shape, updated_rhs)); } return AddInstruction(std::move(instr), binop, {updated_lhs, updated_rhs}); @@ -432,30 +451,28 @@ XlaOp XlaBuilder::TernaryOp(HloOpcode triop, const XlaOp& lhs, const XlaOp& rhs, TF_ASSIGN_OR_RETURN(const Shape& lhs_shape, GetShape(lhs)); TF_ASSIGN_OR_RETURN(const Shape& rhs_shape, GetShape(rhs)); TF_ASSIGN_OR_RETURN(const Shape& ehs_shape, GetShape(ehs)); - TF_ASSIGN_OR_RETURN(*instr.mutable_shape(), - ShapeInference::InferTernaryOpShape( - triop, lhs_shape, rhs_shape, ehs_shape)); + TF_ASSIGN_OR_RETURN( + Shape shape, ShapeInference::InferTernaryOpShape(triop, lhs_shape, + rhs_shape, ehs_shape)); + *instr.mutable_shape() = shape.ToProto(); XlaOp updated_lhs = lhs; XlaOp updated_rhs = rhs; XlaOp updated_ehs = ehs; - if (!ShapeUtil::IsTuple(instr.shape())) { + if (!ShapeUtil::IsTuple(shape)) { if (!ShapeUtil::IsTuple(lhs_shape) && - !ShapeUtil::SameDimensions(instr.shape(), lhs_shape)) { + !ShapeUtil::SameDimensions(shape, lhs_shape)) { // lhs is being implicitly broadcasted. Change to explicit. - TF_ASSIGN_OR_RETURN(updated_lhs, - AddBroadcastSequence(instr.shape(), lhs)); + TF_ASSIGN_OR_RETURN(updated_lhs, AddBroadcastSequence(shape, lhs)); } if (!ShapeUtil::IsTuple(rhs_shape) && - !ShapeUtil::SameDimensions(instr.shape(), rhs_shape)) { + !ShapeUtil::SameDimensions(shape, rhs_shape)) { // rhs is being implicitly broadcasted. Change to explicit. - TF_ASSIGN_OR_RETURN(updated_rhs, - AddBroadcastSequence(instr.shape(), rhs)); + TF_ASSIGN_OR_RETURN(updated_rhs, AddBroadcastSequence(shape, rhs)); } if (!ShapeUtil::IsTuple(ehs_shape) && - !ShapeUtil::SameDimensions(instr.shape(), ehs_shape)) { + !ShapeUtil::SameDimensions(shape, ehs_shape)) { // ehs is being implicitly broadcasted. Change to explicit. - TF_ASSIGN_OR_RETURN(updated_ehs, - AddBroadcastSequence(instr.shape(), ehs)); + TF_ASSIGN_OR_RETURN(updated_ehs, AddBroadcastSequence(shape, ehs)); } } return AddInstruction(std::move(instr), triop, @@ -476,7 +493,7 @@ XlaOp XlaBuilder::Mul(const XlaOp& lhs, const XlaOp& rhs, XlaOp XlaBuilder::ConstantLiteral(const LiteralSlice& literal) { return ReportErrorOrReturn([&]() -> StatusOr { HloInstructionProto instr; - *instr.mutable_shape() = literal.shape(); + *instr.mutable_shape() = literal.shape().ToProto(); *instr.mutable_literal() = literal.ToProto(); return AddInstruction(std::move(instr), HloOpcode::kConstant); }); @@ -485,7 +502,7 @@ XlaOp XlaBuilder::ConstantLiteral(const LiteralSlice& literal) { XlaOp XlaBuilder::Iota(const Shape& shape, int64 iota_dimension) { return ReportErrorOrReturn([&]() -> StatusOr { HloInstructionProto instr; - *instr.mutable_shape() = shape; + *instr.mutable_shape() = shape.ToProto(); instr.add_dimensions(iota_dimension); return AddInstruction(std::move(instr), HloOpcode::kIota); }); @@ -505,10 +522,10 @@ XlaOp XlaBuilder::Call(const XlaComputation& computation, [](const Shape& shape) { return &shape; }); TF_ASSIGN_OR_RETURN(const ProgramShape& called_program_shape, computation.GetProgramShape()); - TF_ASSIGN_OR_RETURN( - *instr.mutable_shape(), - ShapeInference::InferCallShape(operand_shape_ptrs, - /*to_apply=*/called_program_shape)); + TF_ASSIGN_OR_RETURN(Shape shape, ShapeInference::InferCallShape( + operand_shape_ptrs, + /*to_apply=*/called_program_shape)); + *instr.mutable_shape() = shape.ToProto(); AddCalledComputation(computation, &instr); @@ -526,7 +543,7 @@ XlaOp XlaBuilder::Parameter(int64 parameter_number, const Shape& shape, } instr.set_parameter_number(parameter_number); instr.set_name(name); - *instr.mutable_shape() = shape; + *instr.mutable_shape() = shape.ToProto(); return AddInstruction(std::move(instr), HloOpcode::kParameter); }); } @@ -556,27 +573,35 @@ XlaOp XlaBuilder::Broadcast(const XlaOp& operand, } XlaOp XlaBuilder::BroadcastInDim( - const XlaOp& operand, const Shape& shape, + const XlaOp& operand, const absl::Span out_dim_size, const absl::Span broadcast_dimensions) { return ReportErrorOrReturn([&]() -> StatusOr { TF_ASSIGN_OR_RETURN(const Shape& operand_shape, GetShape(operand)); - TF_RETURN_IF_ERROR(ShapeInference::InferBroadcastShape(operand_shape, shape, - broadcast_dimensions) + // Output shape, in the case of degenerate broadcast, the out_dim_size is + // not necessarily the same as the dimension sizes of the output shape. + const auto& output_shape = + ShapeUtil::MakeShape(operand_shape.element_type(), out_dim_size); + + TF_RETURN_IF_ERROR(ShapeInference::InferBroadcastShape( + operand_shape, output_shape, broadcast_dimensions) .status()); - std::vector in_dim_size(ShapeUtil::Rank(shape)); - absl::c_copy(shape.dimensions(), in_dim_size.begin()); + std::vector in_dim_size(out_dim_size.begin(), out_dim_size.end()); for (int i = 0; i < broadcast_dimensions.size(); i++) { in_dim_size[broadcast_dimensions[i]] = operand_shape.dimensions(i); } const auto& in_dim_shape = - ShapeUtil::MakeShape(shape.element_type(), in_dim_size); + ShapeUtil::MakeShape(operand_shape.element_type(), in_dim_size); TF_ASSIGN_OR_RETURN( XlaOp in_dim_broadcast, InDimBroadcast(in_dim_shape, operand, broadcast_dimensions)); - if (ShapeUtil::Equal(in_dim_shape, shape)) { + + // If broadcast is not degenerate, return broadcasted result. + if (ShapeUtil::Equal(in_dim_shape, output_shape)) { return in_dim_broadcast; } - return AddBroadcastSequence(shape, in_dim_broadcast); + + // Otherwise handle degenerate broadcast case. + return AddBroadcastSequence(output_shape, in_dim_broadcast); }); } @@ -584,7 +609,7 @@ StatusOr XlaBuilder::Reshape(const Shape& shape, const XlaOp& operand) { TF_RETURN_IF_ERROR(first_error_); HloInstructionProto instr; - *instr.mutable_shape() = shape; + *instr.mutable_shape() = shape.ToProto(); return AddInstruction(std::move(instr), HloOpcode::kReshape, {operand}); } @@ -596,9 +621,9 @@ XlaOp XlaBuilder::Slice(const XlaOp& operand, HloInstructionProto instr; TF_ASSIGN_OR_RETURN(const Shape& operand_shape, GetShape(operand)); TF_ASSIGN_OR_RETURN( - *instr.mutable_shape(), - ShapeInference::InferSliceShape(operand_shape, start_indices, - limit_indices, strides)); + Shape shape, ShapeInference::InferSliceShape( + operand_shape, start_indices, limit_indices, strides)); + *instr.mutable_shape() = shape.ToProto(); for (int i = 0; i < start_indices.size(); i++) { auto* slice_config = instr.add_slice_dimensions(); slice_config->set_start(start_indices[i]); @@ -633,9 +658,10 @@ XlaOp XlaBuilder::DynamicSlice(const XlaOp& operand, const XlaOp& start_indices, TF_ASSIGN_OR_RETURN(const Shape& operand_shape, GetShape(operand)); TF_ASSIGN_OR_RETURN(const Shape& start_indices_shape, GetShape(start_indices)); - TF_ASSIGN_OR_RETURN(*instr.mutable_shape(), + TF_ASSIGN_OR_RETURN(Shape shape, ShapeInference::InferDynamicSliceShape( operand_shape, start_indices_shape, slice_sizes)); + *instr.mutable_shape() = shape.ToProto(); for (int64 size : slice_sizes) { instr.add_dynamic_slice_sizes(size); @@ -655,9 +681,10 @@ XlaOp XlaBuilder::DynamicUpdateSlice(const XlaOp& operand, const XlaOp& update, TF_ASSIGN_OR_RETURN(const Shape& update_shape, GetShape(update)); TF_ASSIGN_OR_RETURN(const Shape& start_indices_shape, GetShape(start_indices)); - TF_ASSIGN_OR_RETURN(*instr.mutable_shape(), + TF_ASSIGN_OR_RETURN(Shape shape, ShapeInference::InferDynamicUpdateSliceShape( operand_shape, update_shape, start_indices_shape)); + *instr.mutable_shape() = shape.ToProto(); return AddInstruction(std::move(instr), HloOpcode::kDynamicUpdateSlice, {operand, update, start_indices}); @@ -673,9 +700,9 @@ XlaOp XlaBuilder::ConcatInDim(absl::Span operands, TF_ASSIGN_OR_RETURN(const auto& operand_shapes, GetOperandShapes(operands)); absl::c_transform(operand_shapes, std::back_inserter(operand_shape_ptrs), [](const Shape& shape) { return &shape; }); - TF_ASSIGN_OR_RETURN( - *instr.mutable_shape(), - ShapeInference::InferConcatOpShape(operand_shape_ptrs, dimension)); + TF_ASSIGN_OR_RETURN(Shape shape, ShapeInference::InferConcatOpShape( + operand_shape_ptrs, dimension)); + *instr.mutable_shape() = shape.ToProto(); instr.add_dimensions(dimension); @@ -692,10 +719,9 @@ XlaOp XlaBuilder::Pad(const XlaOp& operand, const XlaOp& padding_value, TF_ASSIGN_OR_RETURN(const Shape& padding_value_shape, GetShape(padding_value)); TF_ASSIGN_OR_RETURN( - *instr.mutable_shape(), - ShapeInference::InferPadShape(operand_shape, padding_value_shape, - padding_config)); - + Shape shape, ShapeInference::InferPadShape( + operand_shape, padding_value_shape, padding_config)); + *instr.mutable_shape() = shape.ToProto(); *instr.mutable_padding_config() = padding_config; return AddInstruction(std::move(instr), HloOpcode::kPad, @@ -708,7 +734,7 @@ XlaOp XlaBuilder::Reshape(const XlaOp& operand, absl::Span new_sizes) { return ReportErrorOrReturn([&]() -> StatusOr { TF_ASSIGN_OR_RETURN(const Shape& operand_shape, GetShape(operand)); - TF_ASSIGN_OR_RETURN(const Shape& shape, + TF_ASSIGN_OR_RETURN(const Shape shape, ShapeInference::InferReshapeShape( operand_shape, dimensions, new_sizes)); XlaOp transposed = IsIdentityPermutation(dimensions) @@ -721,7 +747,7 @@ XlaOp XlaBuilder::Reshape(const XlaOp& operand, XlaOp XlaBuilder::Reshape(const XlaOp& operand, absl::Span new_sizes) { return ReportErrorOrReturn([&]() -> StatusOr { - TF_ASSIGN_OR_RETURN(auto shape, GetShape(operand)); + TF_ASSIGN_OR_RETURN(Shape shape, GetShape(operand)); std::vector dimensions(shape.dimensions_size()); std::iota(dimensions.begin(), dimensions.end(), 0); return Reshape(operand, dimensions, new_sizes); @@ -771,7 +797,7 @@ XlaOp XlaBuilder::Collapse(const XlaOp& operand, void XlaBuilder::Trace(const string& tag, const XlaOp& operand) { ReportErrorOrReturn([&]() -> StatusOr { HloInstructionProto instr; - *instr.mutable_shape() = ShapeUtil::MakeNil(); + *instr.mutable_shape() = ShapeUtil::MakeNil().ToProto(); *instr.mutable_literal() = LiteralUtil::CreateR1U8(tag).ToProto(); return AddInstruction(std::move(instr), HloOpcode::kTrace, {operand}); }); @@ -797,9 +823,10 @@ XlaOp XlaBuilder::Tuple(absl::Span elements) { TF_ASSIGN_OR_RETURN(const auto& operand_shapes, GetOperandShapes(elements)); absl::c_transform(operand_shapes, std::back_inserter(operand_shape_ptrs), [](const Shape& shape) { return &shape; }); - TF_ASSIGN_OR_RETURN(*instr.mutable_shape(), + TF_ASSIGN_OR_RETURN(const Shape shape, ShapeInference::InferVariadicOpShape( HloOpcode::kTuple, operand_shape_ptrs)); + *instr.mutable_shape() = shape.ToProto(); return AddInstruction(std::move(instr), HloOpcode::kTuple, elements); }); } @@ -814,7 +841,7 @@ XlaOp XlaBuilder::GetTupleElement(const XlaOp& tuple_data, int64 index) { ShapeUtil::HumanString(tuple_shape)); } *instr.mutable_shape() = - ShapeUtil::GetTupleElementShape(tuple_shape, index); + ShapeUtil::GetTupleElementShape(tuple_shape, index).ToProto(); instr.set_tuple_index(index); @@ -873,9 +900,10 @@ XlaOp XlaBuilder::DotGeneral(const XlaOp& lhs, const XlaOp& rhs, HloInstructionProto instr; TF_ASSIGN_OR_RETURN(const Shape& lhs_shape, GetShape(lhs)); TF_ASSIGN_OR_RETURN(const Shape& rhs_shape, GetShape(rhs)); - TF_ASSIGN_OR_RETURN(*instr.mutable_shape(), + TF_ASSIGN_OR_RETURN(Shape shape, ShapeInference::InferDotOpShape(lhs_shape, rhs_shape, dimension_numbers)); + *instr.mutable_shape() = shape.ToProto(); *instr.mutable_dot_dimension_numbers() = dimension_numbers; if (precision_config != nullptr) { *instr.mutable_precision_config() = *precision_config; @@ -1017,10 +1045,11 @@ XlaOp XlaBuilder::ConvGeneralDilated( MakeWindow(window_dimensions, window_strides, padding, lhs_dilation, rhs_dilation)); - TF_ASSIGN_OR_RETURN(*instr.mutable_shape(), + TF_ASSIGN_OR_RETURN(Shape shape, ShapeInference::InferConvolveShape( lhs_shape, rhs_shape, feature_group_count, instr.window(), dimension_numbers)); + *instr.mutable_shape() = shape.ToProto(); *instr.mutable_convolution_dimension_numbers() = dimension_numbers; instr.set_feature_group_count(feature_group_count); @@ -1093,10 +1122,9 @@ XlaOp XlaBuilder::Fft(const XlaOp& operand, const FftType fft_type, return ReportErrorOrReturn([&]() -> StatusOr { HloInstructionProto instr; TF_ASSIGN_OR_RETURN(const Shape& operand_shape, GetShape(operand)); - TF_ASSIGN_OR_RETURN( - *instr.mutable_shape(), - ShapeInference::InferFftShape(operand_shape, fft_type, fft_length)); - + TF_ASSIGN_OR_RETURN(Shape shape, ShapeInference::InferFftShape( + operand_shape, fft_type, fft_length)); + *instr.mutable_shape() = shape.ToProto(); instr.set_fft_type(fft_type); for (int64 i : fft_length) { instr.add_fft_length(i); @@ -1114,7 +1142,7 @@ XlaOp XlaBuilder::Infeed(const Shape& shape, const string& config) { } const Shape infeed_instruction_shape = ShapeUtil::MakeTupleShape({shape, ShapeUtil::MakeTokenShape()}); - *instr.mutable_shape() = infeed_instruction_shape; + *instr.mutable_shape() = infeed_instruction_shape.ToProto(); instr.set_infeed_config(config); if (ShapeUtil::IsArray(shape) && sharding() && @@ -1135,7 +1163,7 @@ XlaOp XlaBuilder::Infeed(const Shape& shape, const string& config) { XlaOp token; auto make_token = [&]() { HloInstructionProto token_instr; - *token_instr.mutable_shape() = ShapeUtil::MakeTokenShape(); + *token_instr.mutable_shape() = ShapeUtil::MakeTokenShape().ToProto(); return AddInstruction(std::move(token_instr), HloOpcode::kAfterAll, {}); }; if (sharding()) { @@ -1174,7 +1202,7 @@ XlaOp XlaBuilder::Infeed(const Shape& shape, const string& config) { // TODO(b/80000000): Remove this when clients have been updated to handle // tokens. HloInstructionProto infeed_data; - *infeed_data.mutable_shape() = shape; + *infeed_data.mutable_shape() = shape.ToProto(); infeed_data.set_tuple_index(0); return AddInstruction(std::move(infeed_data), HloOpcode::kGetTupleElement, {infeed}); @@ -1190,7 +1218,7 @@ XlaOp XlaBuilder::InfeedWithToken(const XlaOp& token, const Shape& shape, } const Shape infeed_instruction_shape = ShapeUtil::MakeTupleShape({shape, ShapeUtil::MakeTokenShape()}); - *instr.mutable_shape() = infeed_instruction_shape; + *instr.mutable_shape() = infeed_instruction_shape.ToProto(); instr.set_infeed_config(config); if (ShapeUtil::IsArray(shape) && sharding() && @@ -1215,7 +1243,7 @@ void XlaBuilder::Outfeed(const XlaOp& operand, const Shape& shape_with_layout, ReportErrorOrReturn([&]() -> StatusOr { HloInstructionProto instr; - *instr.mutable_shape() = ShapeUtil::MakeTokenShape(); + *instr.mutable_shape() = ShapeUtil::MakeTokenShape().ToProto(); // Check and set outfeed shape. if (!LayoutUtil::HasLayout(shape_with_layout)) { @@ -1228,14 +1256,14 @@ void XlaBuilder::Outfeed(const XlaOp& operand, const Shape& shape_with_layout, ShapeUtil::HumanStringWithLayout(shape_with_layout), ShapeUtil::HumanStringWithLayout(operand_shape)); } - *instr.mutable_outfeed_shape() = shape_with_layout; + *instr.mutable_outfeed_shape() = shape_with_layout.ToProto(); instr.set_outfeed_config(outfeed_config); // Outfeed takes a token as its second operand. Generate the token to pass // to the outfeed. HloInstructionProto token_instr; - *token_instr.mutable_shape() = ShapeUtil::MakeTokenShape(); + *token_instr.mutable_shape() = ShapeUtil::MakeTokenShape().ToProto(); TF_ASSIGN_OR_RETURN(XlaOp token, AddInstruction(std::move(token_instr), HloOpcode::kAfterAll, {})); @@ -1249,7 +1277,7 @@ void XlaBuilder::Outfeed(const XlaOp& operand, const Shape& shape_with_layout, // TODO(b/80000000): Remove this when clients have been updated to handle // tokens. HloInstructionProto tuple_instr; - *tuple_instr.mutable_shape() = ShapeUtil::MakeNil(); + *tuple_instr.mutable_shape() = ShapeUtil::MakeNil().ToProto(); // The dummy tuple should have no sharding. { @@ -1268,7 +1296,7 @@ XlaOp XlaBuilder::OutfeedWithToken(const XlaOp& operand, const XlaOp& token, return ReportErrorOrReturn([&]() -> StatusOr { HloInstructionProto instr; - *instr.mutable_shape() = ShapeUtil::MakeTokenShape(); + *instr.mutable_shape() = ShapeUtil::MakeTokenShape().ToProto(); // Check and set outfeed shape. if (!LayoutUtil::HasLayout(shape_with_layout)) { @@ -1281,7 +1309,7 @@ XlaOp XlaBuilder::OutfeedWithToken(const XlaOp& operand, const XlaOp& token, ShapeUtil::HumanStringWithLayout(shape_with_layout), ShapeUtil::HumanStringWithLayout(operand_shape)); } - *instr.mutable_outfeed_shape() = shape_with_layout; + *instr.mutable_outfeed_shape() = shape_with_layout.ToProto(); instr.set_outfeed_config(outfeed_config); @@ -1293,7 +1321,7 @@ XlaOp XlaBuilder::OutfeedWithToken(const XlaOp& operand, const XlaOp& token, XlaOp XlaBuilder::CreateToken() { return ReportErrorOrReturn([&]() -> StatusOr { HloInstructionProto instr; - *instr.mutable_shape() = ShapeUtil::MakeTokenShape(); + *instr.mutable_shape() = ShapeUtil::MakeTokenShape().ToProto(); return AddInstruction(std::move(instr), HloOpcode::kAfterAll); }); } @@ -1303,8 +1331,17 @@ XlaOp XlaBuilder::AfterAll(absl::Span tokens) { if (tokens.empty()) { return InvalidArgument("AfterAll requires at least one operand"); } + for (int i = 0; i < tokens.size(); ++i) { + const XlaOp& operand = tokens[i]; + TF_ASSIGN_OR_RETURN(const Shape& operand_shape, GetShape(operand)); + if (!ShapeUtil::IsToken(operand_shape)) { + return InvalidArgument( + "All operands to AfterAll must be tokens; operand %d has shape %s", + i, ShapeUtil::HumanString(operand_shape)); + } + } HloInstructionProto instr; - *instr.mutable_shape() = ShapeUtil::MakeTokenShape(); + *instr.mutable_shape() = ShapeUtil::MakeTokenShape().ToProto(); return AddInstruction(std::move(instr), HloOpcode::kAfterAll, tokens); }); } @@ -1321,7 +1358,7 @@ XlaOp XlaBuilder::CustomCall( "are reserved for internal use.", call_target_name); } - *instr.mutable_shape() = shape; + *instr.mutable_shape() = shape.ToProto(); instr.set_custom_call_target(call_target_name); instr.set_custom_call_opaque(opaque); if (operand_shapes_with_layout.has_value()) { @@ -1345,7 +1382,7 @@ XlaOp XlaBuilder::CustomCall( "constrained layout.", operand_num); } - *instr.add_operand_shapes_with_layout() = operand_shape; + *instr.add_operand_shapes_with_layout() = operand_shape.ToProto(); ++operand_num; } } @@ -1499,9 +1536,9 @@ XlaOp XlaBuilder::Transpose(const XlaOp& operand, return ReportErrorOrReturn([&]() -> StatusOr { HloInstructionProto instr; TF_ASSIGN_OR_RETURN(const Shape& operand_shape, GetShape(operand)); - TF_ASSIGN_OR_RETURN( - *instr.mutable_shape(), - ShapeInference::InferTransposeShape(operand_shape, permutation)); + TF_ASSIGN_OR_RETURN(Shape shape, ShapeInference::InferTransposeShape( + operand_shape, permutation)); + *instr.mutable_shape() = shape.ToProto(); for (int64 dim : permutation) { instr.add_dimensions(dim); } @@ -1514,9 +1551,9 @@ XlaOp XlaBuilder::Rev(const XlaOp& operand, return ReportErrorOrReturn([&]() -> StatusOr { HloInstructionProto instr; TF_ASSIGN_OR_RETURN(const Shape& operand_shape, GetShape(operand)); - TF_ASSIGN_OR_RETURN( - *instr.mutable_shape(), - ShapeInference::InferReverseShape(operand_shape, dimensions)); + TF_ASSIGN_OR_RETURN(Shape shape, ShapeInference::InferReverseShape( + operand_shape, dimensions)); + *instr.mutable_shape() = shape.ToProto(); for (int64 dim : dimensions) { instr.add_dimensions(dim); } @@ -1535,9 +1572,9 @@ XlaOp XlaBuilder::Sort(const XlaOp& keys, absl::Span values, GetOperandShapes(values)); absl::c_transform(values_shapes, std::back_inserter(operand_shape_ptrs), [](const Shape& shape) { return &shape; }); - TF_ASSIGN_OR_RETURN(*instr.mutable_shape(), - ShapeInference::InferVariadicOpShape( - HloOpcode::kSort, operand_shape_ptrs)); + TF_ASSIGN_OR_RETURN(Shape shape, ShapeInference::InferVariadicOpShape( + HloOpcode::kSort, operand_shape_ptrs)); + *instr.mutable_shape() = shape.ToProto(); if (dimension == -1) { TF_ASSIGN_OR_RETURN(const Shape& keys_shape, GetShape(keys)); dimension = ShapeUtil::Rank(keys_shape) - 1; @@ -1559,9 +1596,9 @@ XlaOp XlaBuilder::ConvertElementType(const XlaOp& operand, return ReportErrorOrReturn([&]() -> StatusOr { HloInstructionProto instr; TF_ASSIGN_OR_RETURN(const Shape& operand_shape, GetShape(operand)); - TF_ASSIGN_OR_RETURN( - *instr.mutable_shape(), - ShapeInference::InferConvertShape(operand_shape, new_element_type)); + TF_ASSIGN_OR_RETURN(Shape shape, ShapeInference::InferConvertShape( + operand_shape, new_element_type)); + *instr.mutable_shape() = shape.ToProto(); return AddInstruction(std::move(instr), HloOpcode::kConvert, {operand}); }); } @@ -1571,9 +1608,9 @@ XlaOp XlaBuilder::BitcastConvertType(const XlaOp& operand, return ReportErrorOrReturn([&]() -> StatusOr { HloInstructionProto instr; TF_ASSIGN_OR_RETURN(const Shape& operand_shape, GetShape(operand)); - TF_ASSIGN_OR_RETURN( - *instr.mutable_shape(), - ShapeInference::InferConvertShape(operand_shape, new_element_type)); + TF_ASSIGN_OR_RETURN(Shape shape, ShapeInference::InferConvertShape( + operand_shape, new_element_type)); + *instr.mutable_shape() = shape.ToProto(); return AddInstruction(std::move(instr), HloOpcode::kBitcastConvert, {operand}); }); @@ -1605,11 +1642,11 @@ XlaOp XlaBuilder::Map(absl::Span operands, TF_ASSIGN_OR_RETURN(const ProgramShape& called_program_shape, computation.GetProgramShape()); TF_ASSIGN_OR_RETURN( - *instr.mutable_shape(), - ShapeInference::InferMapShape(operand_shape_ptrs, called_program_shape, - dimensions)); + Shape shape, ShapeInference::InferMapShape( + operand_shape_ptrs, called_program_shape, dimensions)); + *instr.mutable_shape() = shape.ToProto(); - const Shape& output_shape = instr.shape(); + Shape output_shape(instr.shape()); const int64 output_rank = ShapeUtil::Rank(output_shape); AddCalledComputation(computation, &instr); std::vector new_operands(operands.begin(), operands.end()); @@ -1652,7 +1689,7 @@ XlaOp XlaBuilder::RngOp(RandomDistribution distribution, } TF_RETURN_IF_ERROR(ShapeUtil::ValidateShapeWithOptionalLayout(shape)); - *instr.mutable_shape() = shape; + *instr.mutable_shape() = shape.ToProto(); instr.set_distribution(distribution); @@ -1680,10 +1717,10 @@ XlaOp XlaBuilder::While(const XlaComputation& condition, TF_ASSIGN_OR_RETURN(const auto& condition_program_shape, condition.GetProgramShape()); TF_ASSIGN_OR_RETURN(const Shape& init_shape, GetShape(init)); - TF_ASSIGN_OR_RETURN( - *instr.mutable_shape(), - ShapeInference::InferWhileShape(condition_program_shape, - body_program_shape, init_shape)); + TF_ASSIGN_OR_RETURN(Shape shape, ShapeInference::InferWhileShape( + condition_program_shape, + body_program_shape, init_shape)); + *instr.mutable_shape() = shape.ToProto(); // Body comes before condition computation in the vector. AddCalledComputation(body, &instr); AddCalledComputation(condition, &instr); @@ -1700,10 +1737,10 @@ XlaOp XlaBuilder::Gather(const XlaOp& input, const XlaOp& start_indices, TF_ASSIGN_OR_RETURN(const Shape& input_shape, GetShape(input)); TF_ASSIGN_OR_RETURN(const Shape& start_indices_shape, GetShape(start_indices)); - TF_ASSIGN_OR_RETURN( - *instr.mutable_shape(), - ShapeInference::InferGatherShape(input_shape, start_indices_shape, + TF_ASSIGN_OR_RETURN(Shape shape, ShapeInference::InferGatherShape( + input_shape, start_indices_shape, dimension_numbers, slice_sizes)); + *instr.mutable_shape() = shape.ToProto(); *instr.mutable_gather_dimension_numbers() = dimension_numbers; for (int64 bound : slice_sizes) { @@ -1728,10 +1765,11 @@ XlaOp XlaBuilder::Scatter(const XlaOp& input, const XlaOp& scatter_indices, TF_ASSIGN_OR_RETURN(const Shape& updates_shape, GetShape(updates)); TF_ASSIGN_OR_RETURN(const ProgramShape& to_apply_shape, update_computation.GetProgramShape()); - TF_ASSIGN_OR_RETURN(*instr.mutable_shape(), + TF_ASSIGN_OR_RETURN(Shape shape, ShapeInference::InferScatterShape( input_shape, scatter_indices_shape, updates_shape, to_apply_shape, dimension_numbers)); + *instr.mutable_shape() = shape.ToProto(); *instr.mutable_scatter_dimension_numbers() = dimension_numbers; @@ -1758,10 +1796,11 @@ XlaOp XlaBuilder::Conditional(const XlaOp& predicate, const XlaOp& true_operand, TF_ASSIGN_OR_RETURN(const ProgramShape& false_computation_shape, false_computation.GetProgramShape()); TF_ASSIGN_OR_RETURN( - *instr.mutable_shape(), + Shape shape, ShapeInference::InferConditionalShape( predicate_shape, true_operand_shape, false_operand_shape, true_computation_shape, false_computation_shape)); + *instr.mutable_shape() = shape.ToProto(); // The index of true_computation must be 0 and that of false computation // must be 1. @@ -1803,9 +1842,10 @@ XlaOp XlaBuilder::Reduce(absl::Span operands, [](const Shape& shape) { return &shape; }); TF_ASSIGN_OR_RETURN( - *instr.mutable_shape(), + Shape shape, ShapeInference::InferReduceShape( operand_shape_ptrs, dimensions_to_reduce, called_program_shape)); + *instr.mutable_shape() = shape.ToProto(); for (int64 dim : dimensions_to_reduce) { instr.add_dimensions(dim); @@ -1868,10 +1908,10 @@ XlaOp XlaBuilder::ReduceWindowWithGeneralPadding( MakeWindow(window_dimensions, window_strides, padding, /*lhs_dilation=*/base_dilations, /*rhs_dilation=*/window_dilations)); - TF_ASSIGN_OR_RETURN( - *instr.mutable_shape(), - ShapeInference::InferReduceWindowShape(operand_shape, init_shape, - instr.window(), to_apply_shape)); + TF_ASSIGN_OR_RETURN(Shape shape, ShapeInference::InferReduceWindowShape( + operand_shape, init_shape, + instr.window(), to_apply_shape)); + *instr.mutable_shape() = shape.ToProto(); AddCalledComputation(computation, &instr); return AddInstruction(std::move(instr), HloOpcode::kReduceWindow, @@ -1889,9 +1929,10 @@ XlaOp XlaBuilder::BatchNormTraining(const XlaOp& operand, const XlaOp& scale, TF_ASSIGN_OR_RETURN(const Shape& scale_shape, GetShape(scale)); TF_ASSIGN_OR_RETURN(const Shape& offset_shape, GetShape(offset)); TF_ASSIGN_OR_RETURN( - *instr.mutable_shape(), + Shape shape, ShapeInference::InferBatchNormTrainingShape( operand_shape, scale_shape, offset_shape, feature_index)); + *instr.mutable_shape() = shape.ToProto(); instr.set_epsilon(epsilon); instr.set_feature_index(feature_index); @@ -1913,10 +1954,11 @@ XlaOp XlaBuilder::BatchNormInference(const XlaOp& operand, const XlaOp& scale, TF_ASSIGN_OR_RETURN(const Shape& offset_shape, GetShape(offset)); TF_ASSIGN_OR_RETURN(const Shape& mean_shape, GetShape(mean)); TF_ASSIGN_OR_RETURN(const Shape& variance_shape, GetShape(variance)); - TF_ASSIGN_OR_RETURN(*instr.mutable_shape(), - ShapeInference::InferBatchNormInferenceShape( - operand_shape, scale_shape, offset_shape, - mean_shape, variance_shape, feature_index)); + TF_ASSIGN_OR_RETURN( + Shape shape, ShapeInference::InferBatchNormInferenceShape( + operand_shape, scale_shape, offset_shape, mean_shape, + variance_shape, feature_index)); + *instr.mutable_shape() = shape.ToProto(); instr.set_epsilon(epsilon); instr.set_feature_index(feature_index); @@ -1938,10 +1980,11 @@ XlaOp XlaBuilder::BatchNormGrad(const XlaOp& operand, const XlaOp& scale, TF_ASSIGN_OR_RETURN(const Shape& batch_mean_shape, GetShape(batch_mean)); TF_ASSIGN_OR_RETURN(const Shape& batch_var_shape, GetShape(batch_var)); TF_ASSIGN_OR_RETURN(const Shape& grad_output_shape, GetShape(grad_output)); - TF_ASSIGN_OR_RETURN(*instr.mutable_shape(), + TF_ASSIGN_OR_RETURN(Shape shape, ShapeInference::InferBatchNormGradShape( operand_shape, scale_shape, batch_mean_shape, batch_var_shape, grad_output_shape, feature_index)); + *instr.mutable_shape() = shape.ToProto(); instr.set_epsilon(epsilon); instr.set_feature_index(feature_index); @@ -1972,9 +2015,9 @@ XlaOp XlaBuilder::CrossReplicaSum( return ReportErrorOrReturn([&]() -> StatusOr { HloInstructionProto instr; TF_ASSIGN_OR_RETURN(const Shape& operand_shape, GetShape(operand)); - TF_ASSIGN_OR_RETURN( - *instr.mutable_shape(), - ShapeInference::InferCrossReplicaSumShape({&operand_shape})); + TF_ASSIGN_OR_RETURN(Shape shape, ShapeInference::InferCrossReplicaSumShape( + {&operand_shape})); + *instr.mutable_shape() = shape.ToProto(); for (const ReplicaGroup& group : replica_groups) { *instr.add_replica_groups() = group; @@ -2027,8 +2070,8 @@ XlaOp XlaBuilder::AllToAll(const XlaOp& operand, int64 split_dimension, absl::c_transform(slice_shapes, std::back_inserter(slice_shape_ptrs), [](const Shape& shape) { return &shape; }); TF_ASSIGN_OR_RETURN( - *instr.mutable_shape(), - ShapeInference::InferAllToAllTupleShape(slice_shape_ptrs)); + Shape shape, ShapeInference::InferAllToAllTupleShape(slice_shape_ptrs)); + *instr.mutable_shape() = shape.ToProto(); for (const ReplicaGroup& group : replica_groups) { *instr.add_replica_groups() = group; } @@ -2053,8 +2096,9 @@ XlaOp XlaBuilder::CollectivePermute( TF_ASSIGN_OR_RETURN(const Shape& operand_shape, GetShape(operand)); HloInstructionProto instr; TF_ASSIGN_OR_RETURN( - *instr.mutable_shape(), + Shape shape, ShapeInference::InferCollectivePermuteShape(operand_shape)); + *instr.mutable_shape() = shape.ToProto(); for (const auto& pair : source_target_pairs) { auto* proto_pair = instr.add_source_target_pairs(); @@ -2103,10 +2147,11 @@ XlaOp XlaBuilder::SelectAndScatterWithGeneralPadding( TF_ASSIGN_OR_RETURN(*instr.mutable_window(), MakeWindow(window_dimensions, window_strides, padding, /*lhs_dilation=*/{}, /*rhs_dilation=*/{})); - TF_ASSIGN_OR_RETURN(*instr.mutable_shape(), + TF_ASSIGN_OR_RETURN(Shape shape, ShapeInference::InferSelectAndScatterShape( operand_shape, select_shape, instr.window(), source_shape, init_shape, scatter_shape)); + *instr.mutable_shape() = shape.ToProto(); AddCalledComputation(select, &instr); AddCalledComputation(scatter, &instr); @@ -2121,9 +2166,10 @@ XlaOp XlaBuilder::ReducePrecision(const XlaOp& operand, const int exponent_bits, return ReportErrorOrReturn([&]() -> StatusOr { HloInstructionProto instr; TF_ASSIGN_OR_RETURN(const Shape& operand_shape, GetShape(operand)); - TF_ASSIGN_OR_RETURN(*instr.mutable_shape(), + TF_ASSIGN_OR_RETURN(Shape shape, ShapeInference::InferReducePrecisionShape( operand_shape, exponent_bits, mantissa_bits)); + *instr.mutable_shape() = shape.ToProto(); instr.set_exponent_bits(exponent_bits); instr.set_mantissa_bits(mantissa_bits); return AddInstruction(std::move(instr), HloOpcode::kReducePrecision, @@ -2138,7 +2184,7 @@ void XlaBuilder::Send(const XlaOp& operand, const ChannelHandle& handle) { // TODO(b/80000000): Remove this when clients have been updated to handle // tokens. HloInstructionProto token_instr; - *token_instr.mutable_shape() = ShapeUtil::MakeTokenShape(); + *token_instr.mutable_shape() = ShapeUtil::MakeTokenShape().ToProto(); TF_ASSIGN_OR_RETURN(XlaOp token, AddInstruction(std::move(token_instr), HloOpcode::kAfterAll, {})); @@ -2157,15 +2203,17 @@ XlaOp XlaBuilder::SendWithToken(const XlaOp& operand, const XlaOp& token, // token}. HloInstructionProto send_instr; TF_ASSIGN_OR_RETURN(const Shape& shape, GetShape(operand)); - *send_instr.mutable_shape() = ShapeUtil::MakeTupleShape( - {shape, ShapeUtil::MakeShape(U32, {}), ShapeUtil::MakeTokenShape()}); + *send_instr.mutable_shape() = + ShapeUtil::MakeTupleShape( + {shape, ShapeUtil::MakeShape(U32, {}), ShapeUtil::MakeTokenShape()}) + .ToProto(); send_instr.set_channel_id(handle.handle()); TF_ASSIGN_OR_RETURN(XlaOp send, AddInstruction(std::move(send_instr), HloOpcode::kSend, {operand, token})); HloInstructionProto send_done_instr; - *send_done_instr.mutable_shape() = ShapeUtil::MakeTokenShape(); + *send_done_instr.mutable_shape() = ShapeUtil::MakeTokenShape().ToProto(); send_done_instr.set_channel_id(handle.handle()); return AddInstruction(std::move(send_done_instr), HloOpcode::kSendDone, {send}); @@ -2179,7 +2227,7 @@ XlaOp XlaBuilder::Recv(const Shape& shape, const ChannelHandle& handle) { // TODO(b/80000000): Remove this when clients have been updated to handle // tokens. HloInstructionProto token_instr; - *token_instr.mutable_shape() = ShapeUtil::MakeTokenShape(); + *token_instr.mutable_shape() = ShapeUtil::MakeTokenShape().ToProto(); TF_ASSIGN_OR_RETURN(XlaOp token, AddInstruction(std::move(token_instr), HloOpcode::kAfterAll, {})); @@ -2190,7 +2238,7 @@ XlaOp XlaBuilder::Recv(const Shape& shape, const ChannelHandle& handle) { // TODO(b/80000000): Remove this when clients have been updated to handle // tokens. HloInstructionProto recv_data; - *recv_data.mutable_shape() = shape; + *recv_data.mutable_shape() = shape.ToProto(); recv_data.set_tuple_index(0); return AddInstruction(std::move(recv_data), HloOpcode::kGetTupleElement, {recv}); @@ -2207,15 +2255,18 @@ XlaOp XlaBuilder::RecvWithToken(const XlaOp& token, const Shape& shape, // Recv instruction produces a tuple of {receive buffer, U32 context, // token}. HloInstructionProto recv_instr; - *recv_instr.mutable_shape() = ShapeUtil::MakeTupleShape( - {shape, ShapeUtil::MakeShape(U32, {}), ShapeUtil::MakeTokenShape()}); + *recv_instr.mutable_shape() = + ShapeUtil::MakeTupleShape( + {shape, ShapeUtil::MakeShape(U32, {}), ShapeUtil::MakeTokenShape()}) + .ToProto(); recv_instr.set_channel_id(handle.handle()); TF_ASSIGN_OR_RETURN(XlaOp recv, AddInstruction(std::move(recv_instr), HloOpcode::kRecv, {token})); HloInstructionProto recv_done_instr; *recv_done_instr.mutable_shape() = - ShapeUtil::MakeTupleShape({shape, ShapeUtil::MakeTokenShape()}); + ShapeUtil::MakeTupleShape({shape, ShapeUtil::MakeTokenShape()}) + .ToProto(); recv_done_instr.set_channel_id(handle.handle()); return AddInstruction(std::move(recv_done_instr), HloOpcode::kRecvDone, {recv}); @@ -2249,9 +2300,11 @@ XlaOp XlaBuilder::SendToHost(const XlaOp& operand, const XlaOp& token, // Send instruction produces a tuple of {aliased operand, U32 context, // token}. HloInstructionProto send_instr; - *send_instr.mutable_shape() = ShapeUtil::MakeTupleShape( - {shape_with_layout, ShapeUtil::MakeShape(U32, {}), - ShapeUtil::MakeTokenShape()}); + *send_instr.mutable_shape() = + ShapeUtil::MakeTupleShape({shape_with_layout, + ShapeUtil::MakeShape(U32, {}), + ShapeUtil::MakeTokenShape()}) + .ToProto(); send_instr.set_channel_id(handle.handle()); send_instr.set_is_host_transfer(true); TF_ASSIGN_OR_RETURN(XlaOp send, @@ -2259,7 +2312,7 @@ XlaOp XlaBuilder::SendToHost(const XlaOp& operand, const XlaOp& token, {operand, token})); HloInstructionProto send_done_instr; - *send_done_instr.mutable_shape() = ShapeUtil::MakeTokenShape(); + *send_done_instr.mutable_shape() = ShapeUtil::MakeTokenShape().ToProto(); send_done_instr.set_channel_id(handle.handle()); send_done_instr.set_is_host_transfer(true); return AddInstruction(std::move(send_done_instr), HloOpcode::kSendDone, @@ -2288,8 +2341,10 @@ XlaOp XlaBuilder::RecvFromHost(const XlaOp& token, const Shape& shape, // Recv instruction produces a tuple of {receive buffer, U32 context, // token}. HloInstructionProto recv_instr; - *recv_instr.mutable_shape() = ShapeUtil::MakeTupleShape( - {shape, ShapeUtil::MakeShape(U32, {}), ShapeUtil::MakeTokenShape()}); + *recv_instr.mutable_shape() = + ShapeUtil::MakeTupleShape( + {shape, ShapeUtil::MakeShape(U32, {}), ShapeUtil::MakeTokenShape()}) + .ToProto(); recv_instr.set_channel_id(handle.handle()); recv_instr.set_is_host_transfer(true); TF_ASSIGN_OR_RETURN(XlaOp recv, AddInstruction(std::move(recv_instr), @@ -2297,7 +2352,8 @@ XlaOp XlaBuilder::RecvFromHost(const XlaOp& token, const Shape& shape, HloInstructionProto recv_done_instr; *recv_done_instr.mutable_shape() = - ShapeUtil::MakeTupleShape({shape, ShapeUtil::MakeTokenShape()}); + ShapeUtil::MakeTupleShape({shape, ShapeUtil::MakeTokenShape()}) + .ToProto(); recv_done_instr.set_channel_id(handle.handle()); recv_done_instr.set_is_host_transfer(true); return AddInstruction(std::move(recv_done_instr), HloOpcode::kRecvDone, @@ -2309,9 +2365,9 @@ XlaOp XlaBuilder::GetDimensionSize(const XlaOp& operand, int64 dimension) { return ReportErrorOrReturn([&]() -> StatusOr { HloInstructionProto instr; TF_ASSIGN_OR_RETURN(const auto& operand_shape, GetShape(operand)); - TF_ASSIGN_OR_RETURN( - *instr.mutable_shape(), - ShapeInference::InferGetDimensionSizeShape(operand_shape, dimension)); + TF_ASSIGN_OR_RETURN(Shape shape, ShapeInference::InferGetDimensionSizeShape( + operand_shape, dimension)); + *instr.mutable_shape() = shape.ToProto(); instr.add_dimensions(dimension); return AddInstruction(std::move(instr), HloOpcode::kGetDimensionSize, {operand}); @@ -2356,7 +2412,7 @@ StatusOr XlaBuilder::BuildConstantSubGraph( SetProtoIdAndName(&entry, StrCat(name_, "_compute_constant"), kNameSeparator, GetNextId()); entry.set_root_id(root->id()); - ProgramShape* program_shape = entry.mutable_program_shape(); + ProgramShapeProto* program_shape = entry.mutable_program_shape(); *program_shape->mutable_result() = root->shape(); // We use std::set to keep the instruction ids in ascending order (which is @@ -2617,9 +2673,10 @@ XlaOp Broadcast(const XlaOp& operand, absl::Span broadcast_sizes) { return operand.builder()->Broadcast(operand, broadcast_sizes); } -XlaOp BroadcastInDim(const XlaOp& operand, const Shape& shape, +XlaOp BroadcastInDim(const XlaOp& operand, + const absl::Span out_dim_size, const absl::Span broadcast_dimensions) { - return operand.builder()->BroadcastInDim(operand, shape, + return operand.builder()->BroadcastInDim(operand, out_dim_size, broadcast_dimensions); } diff --git a/tensorflow/compiler/xla/client/xla_builder.h b/tensorflow/compiler/xla/client/xla_builder.h index 68314a026ea..098efb60f9b 100644 --- a/tensorflow/compiler/xla/client/xla_builder.h +++ b/tensorflow/compiler/xla/client/xla_builder.h @@ -29,6 +29,7 @@ limitations under the License. #include "tensorflow/compiler/xla/client/xla_computation.h" #include "tensorflow/compiler/xla/literal.h" #include "tensorflow/compiler/xla/literal_util.h" +#include "tensorflow/compiler/xla/service/dynamic_parameter_binding.h" #include "tensorflow/compiler/xla/service/hlo.pb.h" #include "tensorflow/compiler/xla/service/hlo_opcode.h" #include "tensorflow/compiler/xla/shape_util.h" @@ -263,35 +264,30 @@ class XlaBuilder { // evaluating the computation. StatusOr IsConstant(const XlaOp& operand) const; + // Sets up binding which indicates that the `target_dim_num` in the subshape + // `target_param_index` of parameter `target_param_num` is a dynamic dimension + // and its real dynamic size is represented by `dynamic_param_index` in + // parameter `dynamic_param_num`. + // + // TODO(b/119520625): Remove this API once we have more dynamic shape infra + // ready. + Status SetDynamicBinding(int64 dynamic_size_param_num, + ShapeIndex dynamic_size_param_index, + int64 target_param_num, + ShapeIndex target_param_index, int64 target_dim_num); + private: // Build helper which takes the id of the root operation.. StatusOr Build(int64 root_id); - // Enqueues a "retrieve parameter value" instruction for a parameter that was - // passed to the computation. + // Description for the methods below can be found in the corresponding public + // functions section in this file. + XlaOp Parameter(int64 parameter_number, const Shape& shape, const string& name); - // Enqueues a constant with the value of the given literal onto the - // computation. XlaOp ConstantLiteral(const LiteralSlice& literal); - // Enqueues a constant onto the computation. Methods are templated on the - // native host type (NativeT) which corresponds to a specific XLA - // PrimitiveType as given in the following table: - // - // Native Type PrimitiveType - // ----------------------------- - // bool PRED - // int32 S32 - // int64 S64 - // uint32 U32 - // uint64 U64 - // float F32 - // double F64 - // - // Note: not all primitive types defined in xla_data.proto have a - // corresponding native type yet. template XlaOp ConstantR0(NativeT value); template @@ -321,181 +317,79 @@ class XlaBuilder { template XlaOp ConstantR4FromArray4D(const Array4D& values); - // Enqueues a rank one constant (vector) onto the computation. The vector has - // size 'length' and every element has the value 'value'. template XlaOp ConstantR1(int64 length, NativeT value); - // Adds dimensions to an array by duplicating the data in the array. - // - // The new dimensions are inserted on the left, i.e. if - // broadcast_sizes has values {a0, ..., aN} and the operand shape - // has dimensions {b0, ..., bM} then the shape of the output has - // dimensions {a0, ..., aN, b0, ..., bM}. - // - // The new dimensions index into copies of the operand, i.e. - // - // output[i0, ..., iN, j0, ..., jM] = operand[j0, ..., jM] XlaOp Broadcast(const XlaOp& operand, absl::Span broadcast_sizes); - XlaOp BroadcastInDim(const XlaOp& operand, const Shape& shape, + XlaOp BroadcastInDim(const XlaOp& operand, + const absl::Span out_dim_size, const absl::Span broadcast_dimensions); - // Enqueues a pad operation onto the computation that pads the given value on - // the edges as well as between the elements of the input. padding_config - // specifies the padding amount for each dimension. XlaOp Pad(const XlaOp& operand, const XlaOp& padding_value, const PaddingConfig& padding_config); - // Enqueues an operation onto the computation that flattens the operand based - // on the dimension order (major/slowest-varying to minor/fastest-varying) - // given, followed by reshaping it into the shape with the given dimension - // sizes (also major to minor). Conceptually, this is a limited form of - // "shape casting". XlaOp Reshape(const XlaOp& operand, absl::Span dimensions, absl::Span new_sizes); - // Enqueues an operation onto the computation that collapses the operand, from - // first to last dimension (C order), then reshapes it to the given dimension - // sizes. Conceptually, this is a limited form of "shape casting". XlaOp Reshape(const XlaOp& operand, absl::Span new_sizes); - // Wrapper for Reshape. - // Enqueues an operation to collapse the provided dimensions; e.g. an - // operand with dimensions {x=256, y=2, z=2, p=32} can be collapsed to - // {x=1024, y=32} by collapsing dims {0, 1, 2}. Collapsing dimensions must - // be a consecutive, in-order subsequence of the operand dimensions. - // - // Note that collapsing a single dimension does nothing: - // - // {256} collapsing {0} => {256} - // {1} collapsing {0} => {1} - // - // Collapsing multiple dimensions produces a single result dimension: - // - // {256, 2} collapsing {0,1} => {512} - // {256, 2, 3} collapsing {0,1} => {512, 3} - // - // This could potentially cause data to be moved -- it provides a more - // structured form of reshaping than an arbitrary Reshape operation. XlaOp Collapse(const XlaOp& operand, absl::Span dimensions); - // Enqueues a slice operation onto the computation that slices the operand - // from the start indices to the limit indices; e.g. - // - // x - // [ 0 1 2 3 ] - // y [ 4 5 6 7 ] => slice(start={1, 1}, limit={2, 3}) => [ 5 6 ] - // [ 8 9 a b ] - // - // Note that "limit" means up-to-but-not-including; i.e. [start, limit) in 1D - // range notation. - // The strides parameter determines the stride over the slice XlaOp Slice(const XlaOp& operand, absl::Span start_indices, absl::Span limit_indices, absl::Span strides); - // Enqueues a slice operation in a given dimension, taking all other - // dimensions as they are; e.g. if dimno is 1 from start_index 2 to - // limit_index 4 by 1, and the shape is f32[7,8,9], this call is short-hand - // for: - // - // array[:, 2:4:1, :] XlaOp SliceInDim(const XlaOp& operand, int64 start_index, int64 limit_index, int64 stride, int64 dimno); - // Enqueues a slice operation onto the computation that slices the 'operand' - // from dynamic start indices which are passed in 'start_indices'. - // The size of the slice in each dimension is passed in 'slice_sizes', - // which specify the end point of exclusive slice intervals in each - // dimension [start, start + size). - // The shape of 'start_indices' must be rank == 1, with dimension size - // equal to the rank of the 'operand'. - // Slice index calculations are computed modulo input dimension sizes to - // prevent dynamic start indices from generating out-of-bound array accesses. XlaOp DynamicSlice(const XlaOp& operand, const XlaOp& start_indices, absl::Span slice_sizes); - // Enqueues a dynamic update slice operation onto the computation, which - // updates a slice of 'operand' with 'update' at dynamic 'start_indices'. - // The shape of 'update' determines the shape of the slice of 'operand' - // which is updated. - // The indices specified in 'start_indices' specify the offset of the slice - // of 'operand' which is updated. - // - // update = {10, 11} // calculated at runtime. - // [1 2 3] start = {1, 1} // calculated at runtime. [1 2 3 ] - // [4 5 6] => DynamicUpdateslice(data, update, start) => [4 10 11] - // [7 8 9] [7 8 9 ] - // - // The shape of 'start_indices' must be rank == 1, with dimension size - // equal to the rank of the 'operand'. - // Slice index calculations are computed modulo update dimension sizes to - // prevent dynamic start indices from generating out-of-bound array accesses. XlaOp DynamicUpdateSlice(const XlaOp& operand, const XlaOp& update, const XlaOp& start_indices); - // Enqueues a concatenate instruction onto the computation. 'operands' must - // have >= 1 entry. XlaOp ConcatInDim(absl::Span operands, int64 dimension); - // Enqueue a tracing operation onto the computation; the computation will emit - // a logging message with the operand. void Trace(const string& tag, const XlaOp& operand); - // Enqueues a conditional-move-like select operation onto the computation; - // predicated on pred, selects between on_true and on_false. XlaOp Select(const XlaOp& pred, const XlaOp& on_true, const XlaOp& on_false); - // Enqueues a tuple-creation instruction onto the computation. XlaOp Tuple(absl::Span elements); - // Enqueues a tuple-element-get instruction onto the computation. XlaOp GetTupleElement(const XlaOp& tuple_data, int64 index); - // Enqueues an equal-to comparison instruction onto the computation. XlaOp Eq(const XlaOp& lhs, const XlaOp& rhs, absl::Span broadcast_dimensions = {}); - // Enqueues a not-equal comparison instruction onto the computation. XlaOp Ne(const XlaOp& lhs, const XlaOp& rhs, absl::Span broadcast_dimensions = {}); - // Enqueues a greater-or-equal comparison instruction onto the computation. XlaOp Ge(const XlaOp& lhs, const XlaOp& rhs, absl::Span broadcast_dimensions = {}); - // Enqueues a greater-than comparison instruction onto the computation. XlaOp Gt(const XlaOp& lhs, const XlaOp& rhs, absl::Span broadcast_dimensions = {}); - // Enqueues a less-than comparison instruction onto the computation. XlaOp Lt(const XlaOp& lhs, const XlaOp& rhs, absl::Span broadcast_dimensions = {}); - // Enqueues a less-or-equal comparison instruction onto the computation. XlaOp Le(const XlaOp& lhs, const XlaOp& rhs, absl::Span broadcast_dimensions = {}); - // Enqueues a dot instruction onto the computation. XlaOp Dot(const XlaOp& lhs, const XlaOp& rhs, const PrecisionConfig* precision_config = nullptr); - // Enqueues a general dot instruction onto the computation. XlaOp DotGeneral(const XlaOp& lhs, const XlaOp& rhs, const DotDimensionNumbers& dimension_numbers, const PrecisionConfig* precision_config = nullptr); - // Enqueues a convolution instruction onto the computation, which uses the - // default convolution dimension numbers. XlaOp Conv(const XlaOp& lhs, const XlaOp& rhs, absl::Span window_strides, Padding padding, int64 feature_group_count = 1, const PrecisionConfig* precision_config = nullptr); - // Enqueues a convolution instruction onto the computation, with the caller - // provided padding configuration in the format returned by MakePadding(). XlaOp ConvWithGeneralPadding( const XlaOp& lhs, const XlaOp& rhs, absl::Span window_strides, @@ -503,8 +397,6 @@ class XlaBuilder { int64 feature_group_count = 1, const PrecisionConfig* precision_config = nullptr); - // Enqueues a convolution instruction onto the computation, with the caller - // provided dimension numbers configuration. XlaOp ConvWithGeneralDimensions( const XlaOp& lhs, const XlaOp& rhs, absl::Span window_strides, Padding padding, @@ -512,8 +404,6 @@ class XlaBuilder { int64 feature_group_count = 1, const PrecisionConfig* precision_config = nullptr); - // Enqueues a convolution instruction onto the computation, with the caller - // provided padding configuration as well as the dimension numbers. XlaOp ConvGeneral(const XlaOp& lhs, const XlaOp& rhs, absl::Span window_strides, absl::Span> padding, @@ -521,8 +411,6 @@ class XlaBuilder { int64 feature_group_count = 1, const PrecisionConfig* precision_config = nullptr); - // Enqueues a convolution instruction onto the computation, with the caller - // provided padding configuration, dilation factors and dimension numbers. XlaOp ConvGeneralDilated(const XlaOp& lhs, const XlaOp& rhs, absl::Span window_strides, absl::Span> padding, @@ -532,80 +420,53 @@ class XlaBuilder { int64 feature_group_count = 1, const PrecisionConfig* precision_config = nullptr); - // Enqueues an FFT instruction onto the computation, of the given type and - // with the given FFT length. XlaOp Fft(const XlaOp& operand, FftType fft_type, absl::Span fft_length); - // Enqueues an infeed instruction onto the computation, which writes data of - // the given shape to the infeed buffer of the device. XlaOp Infeed(const Shape& shape, const string& config = ""); XlaOp InfeedWithToken(const XlaOp& token, const Shape& shape, const string& config = ""); - // Enqueues an outfeed instruction onto the computation. This instruction - // generates outgoing data transfers for the given data. - // - // shape_with_layout communicates the laid out shape that we want to outfeed - // -- if !ShapeUtil::Compatible(GetShape(operand), shape_with_layout) an error - // will occur. void Outfeed(const XlaOp& operand, const Shape& shape_with_layout, const string& outfeed_config); XlaOp OutfeedWithToken(const XlaOp& operand, const XlaOp& token, const Shape& shape_with_layout, const string& outfeed_config); - // Enqueues a call instruction onto the computation. XlaOp Call(const XlaComputation& computation, absl::Span operands); - // Enqueues a custom call instruction onto the computation. XlaOp CustomCall( const string& call_target_name, absl::Span operands, const Shape& shape_with_layout, const string& opaque, absl::optional> operand_shapes_with_layout); - // The following methods enqueue element-wise binary arithmetic operations - // onto the computation. The shapes of the operands have to match unless one - // of the operands is a scalar, or an explicit broadcast dimension is given - // (see g3doc for more details). - - // Enqueues a complex compose instruction onto the computation. XlaOp Complex(const XlaOp& real, const XlaOp& imag, absl::Span broadcast_dimensions = {}); - // Enqueues a complex conjugate instruction onto the computation. XlaOp Conj(const XlaOp& operand); - // Enqueues an add instruction onto the computation. XlaOp Add(const XlaOp& lhs, const XlaOp& rhs, absl::Span broadcast_dimensions = {}); - // Enqueues a subtract instruction onto the computation. XlaOp Sub(const XlaOp& lhs, const XlaOp& rhs, absl::Span broadcast_dimensions = {}); - // Enqueues a multiply instruction onto the computation. XlaOp Mul(const XlaOp& lhs, const XlaOp& rhs, absl::Span broadcast_dimensions = {}); - // Enqueues a divide instruction onto the computation. XlaOp Div(const XlaOp& lhs, const XlaOp& rhs, absl::Span broadcast_dimensions = {}); - // Enqueues a remainder instruction onto the computation. XlaOp Rem(const XlaOp& lhs, const XlaOp& rhs, absl::Span broadcast_dimensions = {}); - // Enqueues a max instruction onto the computation. XlaOp Max(const XlaOp& lhs, const XlaOp& rhs, absl::Span broadcast_dimensions = {}); - // Enqueues a min instruction onto the computation. XlaOp Min(const XlaOp& lhs, const XlaOp& rhs, absl::Span broadcast_dimensions = {}); - // Element-wise logical operators XlaOp And(const XlaOp& lhs, const XlaOp& rhs, absl::Span broadcast_dimensions = {}); @@ -624,32 +485,23 @@ class XlaBuilder { XlaOp ShiftRightLogical(const XlaOp& lhs, const XlaOp& rhs, absl::Span broadcast_dimensions = {}); - // Reduces an array among the provided dimensions, given "computation" as a - // reduction operator. XlaOp Reduce(const XlaOp& operand, const XlaOp& init_value, const XlaComputation& computation, absl::Span dimensions_to_reduce); - // Reduces several arrays simultaneously among the provided dimensions, given - // "computation" as a reduction operator. XlaOp Reduce(absl::Span operands, absl::Span init_values, const XlaComputation& computation, absl::Span dimensions_to_reduce); - // Convenience wrapper around the above that reduces all the dimensions in the - // operand shape. XlaOp ReduceAll(const XlaOp& operand, const XlaOp& init_value, const XlaComputation& computation); - // Enqueues a windowed reduce instruction onto the computation. XlaOp ReduceWindow(const XlaOp& operand, const XlaOp& init_value, const XlaComputation& computation, absl::Span window_dimensions, absl::Span window_strides, Padding padding); - // As ReduceWindow(), but the padding is given in the format - // returned by MakePadding(). XlaOp ReduceWindowWithGeneralPadding( const XlaOp& operand, const XlaOp& init_value, const XlaComputation& computation, @@ -659,48 +511,22 @@ class XlaBuilder { absl::Span window_dilations, absl::Span> padding); - // Returns the sum of the operand value within each subgroup of replicas. All - // replicas supply one input to the sum and all replicas receive the resulting - // sum for each subgroup. XlaOp CrossReplicaSum(const XlaOp& operand, absl::Span replica_groups = {}); - // Enqueues an operation that do an AllReduce of the operand cross cores. Here - // AllReduce means doing a reduction on the input operand cross cores and then - // broadcasting the reduction result to those cores. The reduction function is - // defined by `computation`, which should be a commutative computation on - // scalars, e.g., add, min, or max. The way that AllReduce is applied is - // configured by: - // - // - `replica_groups`: each ReplicaGroup contains a list of replica id. If - // empty, all replicas belong to one group. Allreduce will be applied within - // subgroups. For example, we have 4 replicas, then - // replica_groups={{0,2},{1,3}} means, replica 0 and 2 are in subgroup 0, - // replica 1 and 3 are in subgroup 1. - // - // - `channel_id`: for Allreduce nodes from different modules, if they have - // the same channel_id, they will be 'Allreduce'd. If empty, Allreduce will - // not be applied cross modules. - // - // TODO(b/117564385): Rename this to AllReduce when it's ready to use. XlaOp CrossReplicaSum( const XlaOp& operand, const XlaComputation& computation, absl::Span replica_groups = {}, const absl::optional& channel_id = absl::nullopt); - // Enqueues an operation that do an Alltoall of the operand cross cores. XlaOp AllToAll(const XlaOp& operand, int64 split_dimension, int64 concat_dimension, int64 split_count, const std::vector& replica_groups); - // Enqueues an operation that do an CollectivePermute of the operand cross - // cores. XlaOp CollectivePermute( const XlaOp& operand, const std::vector>& source_target_pairs); - // Enqueues an operation that scatters the `source` array to the selected - // indices of each window. XlaOp SelectAndScatter(const XlaOp& operand, const XlaComputation& select, absl::Span window_dimensions, absl::Span window_strides, @@ -708,8 +534,6 @@ class XlaBuilder { const XlaOp& init_value, const XlaComputation& scatter); - // As SelectAndScatter(), but the padding is given in the format - // returned by MakePadding(). XlaOp SelectAndScatterWithGeneralPadding( const XlaOp& operand, const XlaComputation& select, absl::Span window_dimensions, @@ -717,217 +541,119 @@ class XlaBuilder { absl::Span> padding, const XlaOp& source, const XlaOp& init_value, const XlaComputation& scatter); - // Enqueues an abs instruction onto the computation. XlaOp Abs(const XlaOp& operand); - // Enqueues a atan2 instruction onto the computation. XlaOp Atan2(const XlaOp& y, const XlaOp& x, absl::Span broadcast_dimensions = {}); - // Enqueues an exp instruction onto the computation. XlaOp Exp(const XlaOp& operand); - // Enqueues an expm1 instruction onto the computation. XlaOp Expm1(const XlaOp& operand); - // Enqueues a floor instruction onto the computation. XlaOp Floor(const XlaOp& operand); - // Enqueues a ceil instruction onto the computation. XlaOp Ceil(const XlaOp& operand); - // Enqueues a round instruction onto the computation, rounding to nearest even - // with half-way cases rounding away from zero. XlaOp Round(const XlaOp& operand); - // Enqueues an log instruction (natural logarithm) onto the computation. XlaOp Log(const XlaOp& operand); - // Enqueues an log1p instruction (log(x+1)) onto the computation. XlaOp Log1p(const XlaOp& operand); - // Enqueues a sign instruction onto the computation. XlaOp Sign(const XlaOp& operand); - // Enqueues a count leading zeros instruction onto the computation. XlaOp Clz(const XlaOp& operand); - // Enqueues a cosine instruction onto the computation. XlaOp Cos(const XlaOp& operand); - // Enqueues a sine instruction onto the computation. XlaOp Sin(const XlaOp& operand); - // Enqueues a tanh instruction onto the computation. XlaOp Tanh(const XlaOp& operand); - // Enqueues a real-part instruction onto the computation. XlaOp Real(const XlaOp& operand); - // Enqueues an imaginary-part instruction onto the computation. XlaOp Imag(const XlaOp& operand); - // Enqueues a lhs^rhs computation onto the computation. XlaOp Pow(const XlaOp& lhs, const XlaOp& rhs, absl::Span broadcast_dimensions = {}); - // Enqueues an operator that tests if the operand's values are finite, i.e., - // not Inf or NaN. Defined only for floating-point types. Returns an array of - // booleans with the same shape where entries are true iff the corresponding - // entry was NaN. XlaOp IsFinite(const XlaOp& operand); - // Enqueues an iota operation onto the computation. XlaOp Iota(const Shape& shape, int64 iota_dimension); - // Enqueues a rank-1 iota operation onto the computation. XlaOp Iota(PrimitiveType type, int64 size); - // Enqueues a convert instruction onto the computation that changes the - // element type of the operand array to primitive_type. XlaOp ConvertElementType(const XlaOp& operand, PrimitiveType new_element_type); - // Enqueues a no-op instruction onto the computation that changes - // the element type of the operand array to primitive_type. The - // bit-widths of the source and destination element types must be - // identical. XlaOp BitcastConvertType(const XlaOp& operand, PrimitiveType new_element_type); - // Enqueues a negate instruction onto the computation. XlaOp Neg(const XlaOp& operand); - // Enqueues a transpose instruction onto the computation. XlaOp Transpose(const XlaOp& operand, absl::Span permutation); - // Enqueues a reverse instruction onto the computation. The order of the - // elements in the given dimensions is reversed (i.e., the element at index i - // is moved to index dimension_size - 1 - i). XlaOp Rev(const XlaOp& operand, absl::Span dimensions); - // Enqueues a sort (as increasing order) instruction onto the computation. - // If only keys are provided: - // * If the keys are an rank-1 tensor (an array), the result is a sorted array - // of keys, in ascending order. - // * If the keys have higher rank, the keys are sorted along the provided - // dimension. For example, for a rank-2 tensor (a matrix) of keys, a dimension - // value of 0 will indepenently sort every column, and a dimension value of 1 - // will independently sort each row. If no dimension number is provided, then - // the last dimension is chosen by default. - // - // If both keys and values are provided: - // * The keys and all values must be tensors with the same dimensions. The - // element types of the tensors may be different. - // * The result is a tuple that consists of a sorted tensor of keys (along the - // provided dimension, as above) as the first element, and tensors with their - // corresponding values as the other elements. XlaOp Sort(const XlaOp& keys, absl::Span values = {}, int64 dimension = -1); - // Enqueues a clamp instruction onto the computation. XlaOp Clamp(const XlaOp& min, const XlaOp& operand, const XlaOp& max); - // Enqueues a map instruction onto the computation. XlaOp Map(absl::Span operands, const XlaComputation& computation, absl::Span dimensions, absl::Span static_operands = {}); - // Enqueues a N(mu, sigma) random number generation instruction onto the - // computation. XlaOp RngNormal(const XlaOp& mu, const XlaOp& sigma, const Shape& shape); - // Enqueues a U(a, b) random number generation instruction onto the - // computation. Returns values in the semi-open interval [a, b). XlaOp RngUniform(const XlaOp& a, const XlaOp& b, const Shape& shape); - // Enqueues a while node onto the computation. XlaOp While(const XlaComputation& condition, const XlaComputation& body, const XlaOp& init); - // Enqueues a conditional node onto the computation. XlaOp Conditional(const XlaOp& predicate, const XlaOp& true_operand, const XlaComputation& true_computation, const XlaOp& false_operand, const XlaComputation& false_computation); - // Enqueues a ReducePrecision node onto the computation. XlaOp ReducePrecision(const XlaOp& operand, const int exponent_bits, const int mantissa_bits); - // Enqueues a Gather node onto the computation. XlaOp Gather(const XlaOp& input, const XlaOp& start_indices, const GatherDimensionNumbers& dimension_numbers, absl::Span slice_sizes); - // Enqueues a Scatter node onto the computation. XlaOp Scatter(const XlaOp& input, const XlaOp& scatter_indices, const XlaOp& updates, const XlaComputation& update_computation, const ScatterDimensionNumbers& dimension_numbers); - // Enqueues a Send node onto the computation for device-to-device - // communication, to send the given operand to a Recv instruction that shares - // the same channel handle. void Send(const XlaOp& operand, const ChannelHandle& handle); XlaOp SendWithToken(const XlaOp& operand, const XlaOp& token, const ChannelHandle& handle); - // Enqueues a Send node which sends data to the host. XlaOp SendToHost(const XlaOp& operand, const XlaOp& token, const Shape& shape_with_layout, const ChannelHandle& handle); - // Enqueues a Recv node which receives data from the host. XlaOp RecvFromHost(const XlaOp& token, const Shape& shape, const ChannelHandle& handle); - // Enqueues an AfterAll operation with no operands producing a token-shaped - // value. XlaOp CreateToken(); - // Enqueues an AfterAll operation with no operands producing a token-shaped - // value. XlaOp AfterAll(absl::Span tokens); - // Enqueues a Recv node onto the computation. The data comes from a Send - // instruction that shares the same channel handle and its shape must - // be the same as the given shape. XlaOp Recv(const Shape& shape, const ChannelHandle& handle); XlaOp RecvWithToken(const XlaOp& token, const Shape& shape, const ChannelHandle& handle); - // Normalizes operand across spatial and batch dimensions for each feature. - // - // Returns a tuple (normalized, batch_mean, batch_var) where `normalized` - // is the normalized result and batch_mean and batch_var are the mean and - // variance, respectively, across batch for the operand. XlaOp BatchNormTraining(const XlaOp& operand, const XlaOp& scale, const XlaOp& offset, float epsilon, int64 feature_index); - // Normalizes operand across spatial and batch dimensions for each feature. - // - // `BatchNormInference` is equivalent to calling `BatchNormTraining` without - // computing `mean` and `variance` for each batch inside the operation. It - // uses the input `mean` and `variance` instead as estimated values. The - // purpose of this op is to reduce latency in inference, hence the name - // `BatchNormInference`. - // - // The output has the same shape as `operand`, and contains the normalized - // values for each batch. XlaOp BatchNormInference(const XlaOp& operand, const XlaOp& scale, const XlaOp& offset, const XlaOp& mean, const XlaOp& variance, float epsilon, int64 feature_index); - // Calculates the gradients of a batch norm op. - // - // The inputs `batch_mean` and `batch_var` represent the mean and variance - // across the batch. - // - // Returns a tuple of three elements: - // - grad_operand: Gradient with respect to input `operand` - // - grad_offset: Gradient with respect to input `offset` - // - grad_scale: Gradient with respect to input `scale` XlaOp BatchNormGrad(const XlaOp& operand, const XlaOp& scale, const XlaOp& batch_mean, const XlaOp& batch_var, const XlaOp& grad_output, float epsilon, @@ -1019,6 +745,9 @@ class XlaBuilder { // The instructions of this computation. std::vector instructions_; + // Dynamic parameter configuration of this computation. + DynamicParameterBinding dynamic_parameter_binding_; + // A map from XlaOp::Handle to the index in the instructions_ vector where the // instruction is held. absl::flat_hash_map handle_to_index_; @@ -1096,7 +825,7 @@ class XlaBuilder { absl::Span broadcast_sizes); friend XlaOp BroadcastInDim( - const XlaOp& operand, const Shape& shape, + const XlaOp& operand, const absl::Span out_dim_size, const absl::Span broadcast_dimensions); friend XlaOp Pad(const XlaOp& operand, const XlaOp& padding_value, @@ -1393,6 +1122,7 @@ class XlaScopedShardingAssignment { // Free functions for building XlaOps. The intention is that these will // become the public API for building XlaOps rather than calling methods on // XlaBuilder directly. +// // Enqueues a "retrieve parameter value" instruction for a parameter that was // passed to the computation. @@ -1488,7 +1218,8 @@ XlaOp Broadcast(const XlaOp& operand, absl::Span broadcast_sizes); // will generate output // {{1 , 1}, // {2 , 2}} -XlaOp BroadcastInDim(const XlaOp& operand, const Shape& shape, +XlaOp BroadcastInDim(const XlaOp& operand, + const absl::Span out_dim_size, const absl::Span broadcast_dimensions); // Enqueues a pad operation onto the computation that pads the given value on @@ -2138,6 +1869,7 @@ XlaOp BatchNormGrad(const XlaOp& operand, const XlaOp& scale, XlaOp GetDimensionSize(const XlaOp& operand, int64 dimension); // Implementation details below this point. +// template XlaOp XlaBuilder::ConstantR0(NativeT value) { diff --git a/tensorflow/compiler/xla/client/xla_builder_test.cc b/tensorflow/compiler/xla/client/xla_builder_test.cc index 8aa85c3cd63..b3f5be300d3 100644 --- a/tensorflow/compiler/xla/client/xla_builder_test.cc +++ b/tensorflow/compiler/xla/client/xla_builder_test.cc @@ -267,7 +267,7 @@ TEST_F(XlaBuilderTest, BinopHasInDimAndDegenerateBroadcast) { TEST_F(XlaBuilderTest, BroadcastInDim) { XlaBuilder b(TestName()); auto x = Parameter(&b, 0, ShapeUtil::MakeShape(F32, {2, 3}), "x"); - BroadcastInDim(x, ShapeUtil::MakeShape(F32, {2, 4, 3}), + BroadcastInDim(x, {2, 4, 3}, /*broadcast_dimensions=*/{0, 2}); TF_ASSERT_OK_AND_ASSIGN(auto module, BuildHloModule(&b)); auto root = module->entry_computation()->root_instruction(); @@ -277,7 +277,7 @@ TEST_F(XlaBuilderTest, BroadcastInDim) { TEST_F(XlaBuilderTest, BroadcastInDimWithDegeneratedDim) { XlaBuilder b(TestName()); auto x = Parameter(&b, 0, ShapeUtil::MakeShape(F32, {2, 1, 4}), "x"); - BroadcastInDim(x, ShapeUtil::MakeShape(F32, {2, 3, 4}), + BroadcastInDim(x, {2, 3, 4}, /*broadcast_dimensions=*/{0, 1, 2}); TF_ASSERT_OK_AND_ASSIGN(auto module, BuildHloModule(&b)); EXPECT_THAT(module->entry_computation()->root_instruction(), @@ -446,5 +446,14 @@ TEST_F(XlaBuilderTest, ProtoMatches) { EXPECT_EQ(c0_string, c1_string); } +TEST_F(XlaBuilderTest, AfterAllWithNonTokenOperands) { + XlaBuilder b(TestName()); + AfterAll(&b, {CreateToken(&b), ConstantR0(&b, 1.0)}); + Status status = b.Build().status(); + ASSERT_IS_NOT_OK(status); + EXPECT_THAT(status.error_message(), + ::testing::HasSubstr("All operands to AfterAll must be tokens")); +} + } // namespace } // namespace xla diff --git a/tensorflow/compiler/xla/client/xla_computation.cc b/tensorflow/compiler/xla/client/xla_computation.cc index c9870b65b91..f317892c125 100644 --- a/tensorflow/compiler/xla/client/xla_computation.cc +++ b/tensorflow/compiler/xla/client/xla_computation.cc @@ -25,7 +25,7 @@ namespace xla { StatusOr XlaComputation::GetProgramShape() const { TF_RET_CHECK(proto_.has_host_program_shape()); - return proto_.host_program_shape(); + return ProgramShape(proto_.host_program_shape()); } StatusOr> XlaComputation::Snapshot() const { diff --git a/tensorflow/compiler/xla/client/xla_computation.h b/tensorflow/compiler/xla/client/xla_computation.h index 71598ef8b29..3ccbfb28bd0 100644 --- a/tensorflow/compiler/xla/client/xla_computation.h +++ b/tensorflow/compiler/xla/client/xla_computation.h @@ -19,6 +19,7 @@ limitations under the License. #include #include "tensorflow/compiler/xla/service/hlo.pb.h" +#include "tensorflow/compiler/xla/shape.h" #include "tensorflow/compiler/xla/status_macros.h" #include "tensorflow/compiler/xla/xla_data.pb.h" diff --git a/tensorflow/compiler/xla/debug_options_flags.cc b/tensorflow/compiler/xla/debug_options_flags.cc index 033887d7c11..d7e7b9e6218 100644 --- a/tensorflow/compiler/xla/debug_options_flags.cc +++ b/tensorflow/compiler/xla/debug_options_flags.cc @@ -54,7 +54,7 @@ void SetDebugOptionsDefaults(DebugOptions* flags) { // TODO(jlebar): Disable fastmath once doing so is not a performance // regression. flags->set_xla_cpu_enable_fast_math(true); - flags->set_xla_gpu_enable_fast_math(true); + flags->set_xla_gpu_enable_fast_min_max(true); flags->set_xla_force_host_platform_device_count(1); } @@ -160,11 +160,11 @@ void AllocateFlags() { "Enable unsafe fast-math optimizations in the CPU compiler; " "this may produce faster code at the expense of some accuracy."), tensorflow::Flag( - "xla_gpu_enable_fast_math", - bool_setter_for(&DebugOptions::set_xla_cpu_enable_fast_math), - flag_values->xla_cpu_enable_fast_math(), - "Enable unsafe fast-math optimizations in the GPU compiler; " - "this may produce faster code at the expense of some accuracy."), + "xla_gpu_enable_fast_min_max", + bool_setter_for(&DebugOptions::set_xla_gpu_enable_fast_min_max), + flag_values->xla_gpu_enable_fast_min_max(), + "Enable fast floating point min/max lowering that does not propagate " + "NaNs."), tensorflow::Flag( "xla_llvm_enable_alias_scope_metadata", bool_setter_for( @@ -335,7 +335,7 @@ void AllocateFlags() { "behavior to help run tests on the host that run models in parallel " "across multiple devices."), }); - ParseFlagsFromEnv(*flag_objects); + ParseFlagsFromEnvAndDieIfUnknown("XLA_FLAGS", *flag_objects); } } // namespace diff --git a/tensorflow/compiler/xla/experimental/xla_sharding/xla_sharding.py b/tensorflow/compiler/xla/experimental/xla_sharding/xla_sharding.py index fb135f5ceda..1fea816a803 100644 --- a/tensorflow/compiler/xla/experimental/xla_sharding/xla_sharding.py +++ b/tensorflow/compiler/xla/experimental/xla_sharding/xla_sharding.py @@ -18,12 +18,9 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function -import math - import numpy as _np # Avoids becoming a part of public Tensorflow API. from tensorflow.compiler.xla import xla_data_pb2 -from tensorflow.compiler.xla.python_api import xla_shape from tensorflow.core.framework import attr_value_pb2 @@ -64,22 +61,18 @@ class Sharding(object): tile_assignment_devices=[core])) @classmethod - def tile(cls, tile_shape, tile_assignment): + def tile(cls, tile_assignment): """Returns a Tiled sharding attribute. This causes an op to be partially computed on multiple cores in the XLA device. Args: - tile_shape: A xla_shape.Shape describing the tile shape that each core - will compute. - The tile shape does not need to be divisible by the tile assignment. tile_assignment: An np.ndarray describing the topology of the tiling and which device will compute which part of the topology. Raises: - TypeError: tile_assignment was not of np.array type or tile_shape was - not of xla_shape.Shape type. + TypeError: tile_assignment was not of np.array type. TODO(jmolloy): This concept is nefarious and is not something we really want to expose to users (especially as the @@ -87,14 +80,11 @@ class Sharding(object): """ if not isinstance(tile_assignment, _np.ndarray): raise TypeError('Tile assignment must be of type np.ndarray') - if not isinstance(tile_shape, xla_shape.Shape): - raise TypeError('Tile shape must be of type xla_shape.Shape') dims = list(tile_assignment.shape) flattened_devices = tile_assignment.reshape(-1, order='C') return Sharding( proto=xla_data_pb2.OpSharding( type=xla_data_pb2.OpSharding.OTHER, - tile_shape=tile_shape.message, tile_assignment_dimensions=dims, tile_assignment_devices=list(flattened_devices))) @@ -118,14 +108,8 @@ class Sharding(object): shape = tensor.shape.as_list() if shape[split_dimension] < num_devices: raise ValueError('Split dimension was smaller than the required number ' - 'of splits: shape=%r, dimension=%r, num_devices=%r', - shape, split_dimension, num_devices) - - tile_shape = shape - tile_shape[split_dimension] = int( - math.ceil(tile_shape[split_dimension] / num_devices)) - tile_shape_proto = xla_data_pb2.Shape( - element_type=xla_data_pb2.F32, dimensions=tile_shape) + 'of splits: shape=%r, dimension=%r, num_devices=%r' % + (shape, split_dimension, num_devices)) tile_assignment_dims = [1] * len(shape) tile_assignment_dims[split_dimension] = num_devices @@ -133,7 +117,6 @@ class Sharding(object): return Sharding( proto=xla_data_pb2.OpSharding( type=xla_data_pb2.OpSharding.OTHER, - tile_shape=tile_shape_proto, tile_assignment_dimensions=tile_assignment_dims, tile_assignment_devices=range(num_devices))) @@ -149,7 +132,6 @@ class Sharding(object): type=xla_data_pb2.OpSharding.TUPLE, tuple_shardings=tuple_shardings) else: proto = self._proto - attr_value = attr_value_pb2.AttrValue(s=proto.SerializeToString()) # TODO(jmolloy): This need to be seriously revisited before declaring this # API available for public use. @@ -194,8 +176,8 @@ def assign_device(tensor, device): return tensor -def tile(tensor, tile_shape, tile_assignment): - Sharding.tile(tile_shape, tile_assignment).apply_to_tensor(tensor) +def tile(tensor, tile_assignment): + Sharding.tile(tile_assignment).apply_to_tensor(tensor) return tensor diff --git a/tensorflow/compiler/xla/g3doc/_book.yaml b/tensorflow/compiler/xla/g3doc/_book.yaml index bcfbcc3a22f..12b7094705e 100644 --- a/tensorflow/compiler/xla/g3doc/_book.yaml +++ b/tensorflow/compiler/xla/g3doc/_book.yaml @@ -3,15 +3,15 @@ upper_tabs: - include: /_upper_tabs_left.yaml - include: /api_docs/_upper_tabs_api.yaml # Dropdown menu -- name: Ecosystem - path: /ecosystem +- name: Resources + path: /resources is_default: true menu: - - include: /ecosystem/_menu_toc.yaml + - include: /resources/_menu_toc.yaml lower_tabs: # Subsite tabs other: - - name: Guide + - name: Guide & Tutorials contents: - title: XLA overview path: /xla/overview @@ -27,3 +27,7 @@ upper_tabs: path: /xla/shapes - title: Using AOT compilation path: /xla/tfcompile + - heading: Tutorials + - title: XLA compile API + path: /xla/tutorials/xla_compile + status: experimental diff --git a/tensorflow/compiler/xla/g3doc/_index.yaml b/tensorflow/compiler/xla/g3doc/_index.yaml index 7934cd11ba2..858de427119 100644 --- a/tensorflow/compiler/xla/g3doc/_index.yaml +++ b/tensorflow/compiler/xla/g3doc/_index.yaml @@ -17,7 +17,7 @@ landing_page: - classname: devsite-landing-row-cards items: - heading: XLA - TensorFlow, compiled - image_path: /ecosystem/images/tf-logo-card-16x9.png + image_path: /resources/images/tf-logo-card-16x9.png path: https://developers.googleblog.com/2017/03/xla-tensorflow-compiled.html buttons: - label: Read on Google Developers blog @@ -28,7 +28,7 @@ landing_page: - label: Watch the video path: https://www.youtube.com/watch?v=kAOanJczHA0 - heading: XLA on GitHub - image_path: /ecosystem/images/github-card-16x9.png + image_path: /resources/images/github-card-16x9.png path: https://github.com/tensorflow/tensorflow/tree/master/tensorflow/compiler/xla buttons: - label: View on GitHub diff --git a/tensorflow/compiler/xla/g3doc/images/xla_array_layout_figure1.png b/tensorflow/compiler/xla/g3doc/images/xla_array_layout_figure1.png new file mode 100644 index 0000000000000000000000000000000000000000..00cefe4c7806c1c09dd51499375e720bfb0baac6 GIT binary patch literal 20398 zcmdqJc{r4P_%~d4tAt9Cy_L$AEir~ll59oBIwaW}WE)Fpu@oT*4aSm?eK#0mipoBg z!7vL6V;M7Joyq=O?=IlAK9~K z50Bp6+a`PV?1S&wv-i&7gFs7rw3F(dJ;r`|w{JcSAkry3&rc3wb{4m+8L}mNADE^c zae4UlVfN?GSqFJ8>^*t%%B3qVH^l$hf0!>JYdHGAfmrkVZa4cP1x}rmy>tokG$%LO zw%RpU7WFvI?Wu!^3O!ck-Wlx9rlhmCRN{z6!iZjFCY$DGJ2LO#@9(7A{+yjYrj~FR z(BuD~FW>$8#gO0lB{gNv_S{gpLQi$SA{2pSIxMZ8`_w^eK13YIg-vMgzH zqn%gbPThSz=gR-9*PMR2)!wMRoO_9=CI~%GKNPFSsa=o*fxVPw(73@B@gmb``JV9g9j8nKa=hhoauD|bH+iOH9BncH*ezd{n z)%IUg9;RQ5FG{SKF-Y6j<zz7f)UlaF{ui-b1)4k#C!8&_HCDetG$vg%y+Eb*#-2t zD6jG|rm8o*aZ!|)qnbv@c@;l1_j4*$cm9#>8*|$L#0RYW<*b@Q@`(aVe5|-&^}&)> z@?&A3>BVWi*nuu_M055H>HbkW=)uRQb{F_?2JcUq^TTJ)5MSqOE9?4P>K!^Na;@0Y z%8GcWD9OUG7-C;Tfpi)e8q!Z5+8Ary3JD5Yzv4cos=|JzsHnJZt!3`A_56|_amYRC z3yDCGZKH={G8ZKwW$t@=&{cyU0uSsB&#qEi#7*8mb7TGcO}+43rQS#jshQUXpMbL_ z!2k3}G34nh+bD69r*ymI!9oqf>|5Vw+;NVDYEIdk@_qcSle zlb=;gJqYb)T}*=*TKAXR^ZPn=6TnG=#j%L9UIqdgcVuQZio~@aF6@*Z(pNU>g(ve{Q8EIH2QF=lbO(y7pEO~&fDML%pNbsWn*m!BquqA4 zrUvjtU%NmMVXTBQnB6{uggj6Fcx`%ULq@*%7_M|e*=BQeUn$%8Lm)GE#sLvx;LkKQZSg)|R#YVs^mKhgl;8w24a>A?&Gk_V zl}fZ}y>?UUm?D;&TPT(5%+N~af&Bg zM6ITe_C7+Mi5qD3r{woHcoeg550W9jtHi{H%w__+Dl%FJATyyErSFp*p_;$zJzvrm zxwkx)c{>LZeQ9f@SGCP4YyvZ1S(S{qUkizP-5{yLx(dxx7ivhmqfNzN2C= zyM`OnI<)_t$|!7hqE0(j=_Fx3zp6w7JI$mRoBwUAYGN#k=0Iat7t}UfBikC{pH*;~ zxQB@2#k^GvirMoHxvEF&S&P?tLSk07m~#aJSoW8-tB+E8f*)l&gCg<cu>1Mu;b{WT+MNNmwH6ZulCR_}kn4${o4CR|P7CSIZN9>p$llgA@9XoI_Mc{^v zE4Y0>Fim%j_YZlU?!D6tjiX)g%R$_}SpRav*sD1`<5(&`jSF`{2DjDK{yF0jQM zFCfqrz>h*UBaeW}vNfIYpnPz^+UN_njCk>!6Fiy?V&OwJ zMsZVL5H*y96u)BJTcF6~+j?Q2v|Y z*yWIvnG#@JE^eYe%xjjK{ivPa-~=3l#NOA$RZhDr`~L*1D?iQz&T(RGX%JU+w#V z8n0nN**gJ7Ba@rww&YQ|+pRi&xcpas1A_&lrROg9(eX}HBeZjuvjp7js{bu)RXoE| zvpb4cSVo%6l+GvU?@Qd+qKC+K^#}&mn$CoZK{x8VHX0CIv2$P#?QtVJB_7*Qu(#yO zDOKk*Tu8O*a1~|$^7r;_YB-wuym#-On~&=DJT}~E7kGBMcj(hXqC=4GmGtq@AjKIT z?!4B5S~V#jqQhgO)9l&y(2p=>5E;R5<^0zf2?I%O;vDLXWU4}peqgkfM0ntuQY1a< zWSFT>3v$-J2!0Os>Ru{Zu#F2m`r>Kx?Pm_HLt=LuTf2^C7l(-Bo-Gu$y1Ui6&-D_4 zL*T4M!|;w~O&rx39CgSHY3!b#IMbsksmu~ZhWNr-*G4=vd&1s2gVq~6KOYEYM{!fN z-`~Tw#@6;|qTq_yeG}bHCxHP{*OGMuO9&17u8s5F2rf_7esLNf_Ek;vV6|f&;UX#h zA!9kcTxH;husWCTKM?L_s5gHtba?o!qkc0o5T_>ju-?7-UaANq{o6R*ZyEF!&Y+16 z4SlPi&$E|ftGirU&kk{P1;%PfZsp>kJMgza?W-py20pp+c{@n(ej?rLzMzck_Ubi^ z-IQA&BwtMXZ8l$kmSKC4j+VV^F!VgL5)8BXp5%21aI6A|#C{A<_tBYwlL*VBrs<<< zp7L*tE;bnb!(LDSDk*ATXFc|~^)7kq;9&#f$4-FS85_>zMzCjWI0)-N z>Rw(pitisj7-xA)_b{BQ;@YQqO@R#aJVSP@x5EQ=B5(OQa{NcE>_MiI489vbP5jeF!wFRCLl-rVY82XEerjg)j=iWxczr{1>Oocooo2TJ8N5+*D2`IEq2f9$qYE4Z?Y+I1-PQNZc$Z)Q8ziS`Ug5>w! zOn&U;b`d!}cE3(Z!sFqm%)YmN_dd(2^?lQHR$bVZW2reYU>-{zcp+O+M<+6r!M3U< z;0=0j)dAl0aE#Drv&T+JUCYJNL+ySi>VNuG=4Y4O3*M_C^-=`PU-*t?rx-Ox!|R#pMP8$mur_NBdqc&1q{CSkIZ{kY&;=8 z%^x=C>MP0aolvh`;}Nqa)ZQcWPs}{?cMA1gt*-6#a@O>^&xbzvzS zeRI_EHW2+CyXw<#^(VC>v<#6{Nv=6)@nQOQ)sOTU4;8R*@Rkt+S%X>mVgCA!(6-q) z54`k@`BS4?>`(lC6h@*4&$&`dFg(?AJAI0$_PP|Md%nK&nPCL>Cv%dlLfP`?zde*T zQ6=3WgG9uJ=ZsUYe{`)=-diApv(kzBU!px%*ozt;yrx6TT;!9~s3P4Jvd8wsMFxbBfW+P)9xmO+@o=QCA3{F8Zg$zO+ELJwW(V!lBesq{5UCP@ZJ zEfWNQ5c8k`GfGLHD$akhHg&a8eT_0U;6qoGsQL%I;pI!u?!~J{IN>6 zk^l`Jg%^c1M%L1Njs~mg%X|v4o#H)Q+G}zQc2K`Z!Fr0gJrh|w#TM<>3%+SODnGiV zEVXptLF65(uJut&#nws_F9R0Hen2~Ac(;_I%lx9!$LeFM-2o*_UVwQETvm0b`MJ*r zZZ!Hk@|3^`!d$fOXo5$G~pr#r<7!=KCgoYc=?RR`hP&Izyib4{w*J zF#4yeARF5sO-Dx)lUY$n_!#ueIRtTOc{y{eF>qOUdSj|H#gmGX0LET1)Z(TFA-UFs zP`!&R157hcV(cD}Mz<-4Os+kP7TqKgrgGesri?{0K5RvJwtKR}ET%(C2A#UvK4^!9 z6#0?1Q>J>QjXTFP$x?%a*yC!YX%BnqYJE_op?x(>Z?h!Xc@n z33C(Zj_H_R?K`1=EifB$HX6p~ApLzqC5<+a?ldXKw{I&$=N<~xJ4CIFVC_y^x#Gse zr_vjAVtDo3t5`RSr)5aBZ!I>?=;@t{5sb0e_`TYt)Fb-am>ZaZ$GK^PLqMkDMk~42 z?-o*$eC%Nz{#bstQ42D%@l3OgK{FC{3S+^)6m+pOn}E{gf#

cV>)c}Pws|5yBb!P}pfYq-bP;|V^GTq1?p$}#6yLO9tqOC%taP(*9-jLPz__K9iw~qUFbZbRdLi`fe-FB zbuZ!L$%Miyk&Zy-dq#3S?GZs0A~N7*uD&C18T-_r>oxTBxRLehd+wFHX}W>a=Y+Z zlr1eRdDFqwgB%VtdNdzX3+TSMv|*bceq#QXQ1a|)<+f-yS3SrHk z!>%eh3{tx?pyQ}o$715V<;bZKCFi7wLF$v9N>UeHizDixSN*8@96OgKAsNd-@Q)F_YlJrMxiBX!8v&X8$*fu(R&}sUKxfV98Dtyqx;wo0Xct@5f z9`@($>l2?HJC}GPQ0(J@jCwIMdj|FtYiNO^>oqIbz0)F^&j%hXrD>(K0`t4d(3~48 zS+9~0Q>*5GUk=Q#uUx_#u(gxLPUx+Ideeqc#I>0Yzj67_n%AHXMeQ)p|D4@1{b$1nI2}TIZLcrPjqPb1w2SncMD6@ z>6&n}a0;xy>#aB*_dgVjd+(T)IKIJ0R3Cm0So^E?Sc6a5vbrAhtt_b40^L-Qic z>Msg0KAvKnPS)F1i>v63?Cpr@B#MyKM-E44sDik-D?3ezC^c*+pUHl13lfzUZA8L8G?m7cZ)yYEn``{Qt** zIQjyEIV0CXdAEjYi_Zw{YkF?)b_YpX*-+iFkxPR&T}gD7J$!*I+m5(W<}r3A6U!b` zR1EP#k}|g1LlmF4MSKWcodR^eTqW6#m-k540!-_N(Ai#flxsP@3=~>#R@&XD{@zPf zI{4#!phMDgo9!=OoYSGxwaidv1Mxj-dpv!J4=y}^PyeFHgR=?GCg#RUzcW(ZiIj4K zX0iX)4A>5-pk}dSDdUXjBFQ#=PmH^r5nCoe^Y^Roey?ngjT`Ml9t2^Yw;z}us;6f% z=ab=fW8yA#=!HKwYf^f!Ga6$XGL{+6>^}#NGfWOqJ6-)?EV(RfHI0DVa*d z1m7Gas|YZBujjGIlxi27`DRK}$O(kH*}0!TRlq@xjsxVE|Lwf5^?(?!L)H2Ah<&D! z1NHPj)yG-oU9#xW_ac?v|8T*rjM=}IMciN87PlY|s*3p+v?xX$MSsD;@Y6;W!Sq+@ z_jRHi86mXLU<>M->ZCeHol5bqp;%v`o+hXMXseG;V zvY@j}c~4DwZf6?_i@M+`Rrx`quS>rj_=X9tZKUAeDrzM5$p|JKaGLXtDvSzsVB ztPL0mi)c96%*SlFIFNhHqi4tVtNz*IM%u51OGF7y(9qf4_G>9B=ghe|7XGN*1 zRKl5WZD>^)gBshytg#ejZbqVGQ4}fDvbx&Q)kIyWEP+SlyiZw^La!Gpm}XeH&MFG= z>_jRUu%7mC%aBWKs>rst?xcJSL{D}Tfdv?QFI^@ii?Ae+WWRc$)|+Gy8GZ#Aqpw3( zK?Lm1cjNhNMo^j&hWi-kz17iR81!MIkOabe<4#(8}~TU%!BNMcl5 zmXA?VbH2bq0meZ-!wPI^&?PL+L@{2Y0_PDF_~)bh)KAq6_f@a_s-vKqPIp3gL6y-A zx4axG&DKb4iTfO zpyiH7&B#bqsI-|pbNGSkkE9fW!Jk}0O^X*{z#cI*b$p~eJM@LLY^=rfo#U1b)gh5+ z7Zo@pi}poZxkj+ZZ3-gY*4B0xPg^VF`qxf$i;b7QjsP(grR}3Yu$#b-eIy9h*Nm*a z+CnM9N`D+r;Y(~q?OcCvkBE%`iRG1KqwYd-~Cj?jq_jP*G-*%V(`sQ$DYX;+taUEy3XduU}IHcD0? z)K~oj{cwsQUq`69c{^T5^TO|Wz7siXD?Kv8zN zK?#~0aQnl++NrK=OHizxEvU4eWqm!Y@7MX~KoBnZu+JWma|h{ZH*^-Eu8gmTP2{My z`}Bm3H39Q4yx&2H5xUr%=3nm4@ah^m0$<-EIxo{Z1-~Gj3BM1h%b2<|$t{X1E3O#I zCj}M5;3ahzj{bOY5Uil!I6}E#qRdD2>2P_Ib`&o(ujN|JtRt;Kb5rWTt}g*>-FPW7 z0)6P3M+x$N?nT-YaaWEFqVN~G&?z5jzXnFPC$-^+G_yP8L(4}pG<4*(wzpLLZXsZkm&TFJ>Bkc^fF-f0u4NggR+7Ku<+1)?uW*bt zXV=*y=x4|^HMz%>e}kL+bzJn%HZHZuvMxy5%SL3~^d!YQMbw^r#oP0Iq2kaj>m&9| zYc|2?%ZXnmhe4t1(PFG-t0QN3`f|O;l0hxjM{Wi)S`~JuI+avx91qk0;puvUp z+@EVusp#{EgH8bz-ekz#NiF$*N22mSE(pylz#y7-Lywh!31k(e89yUqV+mLU+E8)^ zRznOJ`cF2uvxSpNiz3(mgc^jtX6SH>BIsT>ROz+tt5frmpE5L{+MbwGnTSOHA=a=%{Fai_SlEO}5+8V_*KVy!`#+D$i3thYEl8=ERj;yaRo|Ij z($^ksn+UXVu@^s8?+uF_;rGvy9zkv}(-$mR9rlfzIvvU4t6SH%`>>|!7{%&w-rf}X zbU)%hOZsnAEO;7v8Vm-9#3J4}=KBQ}578Ixg#UHONhET8BI5tPQyA%Ff-G9aL1tN>RQd}d@`_xI2;6N9icFtuL_|eV_ z|AiNvPXpwdx9?GJQ#C~2o-CqsvVs32SJ>@8K#mYu@?G0+nwZia=39L`dGN@;iUaE# zZCO>hqW~Y>j4rog6EC8RPL{4T-Mh*y@GyK3LiXv=%6z`mu3C&` z9U!iF%V6yq1GcBR1#Ik&L(n_hgF8ojc2eY^LC(5=wetAOEXF~804@uXnzN^$K_xW@EGRx?4M3VELs&c)XB_rG~3G96~Er*~V@@H6KDz6L@< zLXcYhqPvEMhLbMU60{P?6qh&HppGQ;jRnzhOV*kp+cs0m!WvEc586CruJfQqr5)tN zCLY;%aFo#?RBhtfNY7 z#S$oJ>mi@rWp-)FE;RKpcTeC237ZXy~(dV;k{9d0^QF*SeM1?`zm1vI2 z`O}O>C0rpKv!Tut1cWMu?4f&e2gX+DebS*@xEn1<7I75!GG5~SH zYD}PA>DiyeUb9MLYlz}SW>&Dj$2F2!yl-4}Ahf|D|5`N_I43v>1z@V)ahxk)Jx7~! zKd3zm`mF*7B9=%kdOg~onxw!D*U{lbKY6Q6i3AwwYdwHZ(W8XC)oupQ6rVxa?L!U@ zS&zk6--0^F-EVy`dE|-u!{xgt&@7^>*jPXjEFRe2jucd; z%>&2hcO_j6>~v7fjs^lV&(UEK-I0{Aa=%H%MQ=-jGKk zr5Nuaf=ZrqX>-H#Z^ia0i3tgPndb+exiYOPx$M=mJNEMTTfJ_2=KCqX6h@GYbh;)* zp8aweH#G`2uE1(m2jtpheDD}!#Yj8_k_4#IdLyyZ%6ZyGF3=};5_ei4pe{|(6+)jn zoARV>)O;)<(O6vvr^UR5p6%^A1JtS}tC5*Oaa(>%&9wPmc)O!uM~jxVSYqz6p%Moq zK)>phd}aM=Flbis0OW3`sJgF9MnMwEHE80BVZ2Q?m;Z?RM((N-<$BQKWoZ-q>l8|K zd_lHHjBECx#$>q&FWQ&-Y`KVXMkyEYC#&j7Wc|rkxDV~>fCb&ZCu`qnv&B)7SNVUer=3ZjTggCN(~-K+=LF64 ze1)i`OTIak`XaBS!|id<@X7{;>My2f`3U?=k)D8W9UM=B=qr%&**p(iUy$Z}LQ?Bt zmU~FP0!GYJOL48vbwJe$M0TxOFZ%iju!C99{ZhuXDc%Pe z*NCSI=OU!xO4@@N- z_Lo^*@S1axj&_I*Z>Nl$!0cd**P$%Tp%-5+?adlYV|`FQcEyVLYdt){NtNEc$bB_d z#eJhlV{d*%Eneis=W1}q{YgTh#!Wrp!$vjTDl5HZu*Eg=BH3(o%gdN)9_7s)HimR%v)YQSuf~2+2oEUi4`a0d!23o-&_K z_p2)=RhVpcmHoLw-eRw(O)aGCm@+=8D{MDY^yS}v$y1c!?=$+upBB(eDndwA=2!>(|-a}#%Lq=LE%7z{C;^Kg2uRJ@>JCt3o zW_L9$jl-8G0L@&9E^A|W;}5O!hB4SwbgsUR2PVl&(BNvJ(#JOIsPTfIUMSJb#4<;o z^j=uJf2E7&h~@|W<;uMDnbfoT-D+PDJdOxM#eWR^}O;ge54N2U zXwtS-)+RO=mp1SpCd#O#o_2tl20+X_gd|bOR;5kK#ZmR@!*=NFik?$m_foHZ{Msfp z`S-vlGtHMz^!8+RgwkxLR1c%3ho;vyYANRo;!@0X$I zBXIS1trU?wl@msnd``7`vcs?D?`dj2RmTc=og8eD2><#a|6qprJR?eJ=7kGaZf|a>m<-adUv8o|#fdP)J1V0bI*K~ynMkkv zo3tE+>PU-{gV4fbA%<^cm$5PUK`8pM6Wx#X6k%FYu5yL7S3XY#fGDe5J=T_pMa_rd z!nMV}lkzYQc_>(lP=|$M5XoP?D$UgbzmYnscne?*-T<8ls>A1fXWUP5m}C-+61AQ_ zub2o&!x@4Ig<{EJjCU}7mEkqLUxWTVX7E_IMFjeDu+QZ34-eLR1NN$aT~YJ^=bH4~ z3l(@ogbLVSJt2BJIV%jM{0U(nXWeyMg18Sw>azX1*RyWu&Ehmb6n)`Em>V}rD-AS% zHWaw@lVJQYy6R8|)|e@s$-Y|~1v}WMIoGfAWqPq>EOO;k`#nK~VzuNzte8tg%BFgD z4*-bH9y2WYzR}1=P2#`1ngkB*Q427s6kE4KhmcoK2YV+DdF<5HVH?v{#nj$|A=`se zmK{BFvSlz^i2TZ+<+MlDpiwn(76QV%@z8<=_o%}BcRPV|TZyJj4yo_XxV`?u-llqR zVB12_R+Q+y8KP!0ls$3#gnG>LHLcqXp*oW@@-a4PCivj#**@#)t7xJK<6g>`d}d~7 zp6L9u56?&Q%N|p@alD|=`VV^rZ){#Y#C&h$|2Z-3Q|fhSymXYf8Zgdg zXjgZ4#(5tn`DzkO)gX~C9&BX0GlZxNm@6jIUO!cGe$w8}?R;Q}k*=T@-Lz`Bc6IT3 zeK2v=&%^qIsxB4Roshn!=H2;jb_s22oWRCz`&s%7T5L3Ar5)@%uw1#%9c7p%gVn^K zK8lB>%U~BHgHSy^-4*|kO^Ph{UaI`=W~LyG?riT;+x$?ZYbbwcDda&y+neJ{J)R-@ z{+`5#!5ca8n(MCao7SkzK#xEtT9OUBsW&iEx6R zQjyk7dCnscUisOKSWO`XKK}|{II-{!D^a^3pAd=8MBa^Zx4oDnzZzOK>?Aq$-wLAGqvkJ;833x!%-yB;f6)5T}s)b(~=y zvfeQ#&5SMb%BL^1x3)_`(GSZS*O&etfn$t5ynfpnDZywKPA>3u4WGr2xtENcl_neZ z>2xS5DjrYwo=ijqd?|uv|=T9D_9xgxg)|4D!;wSDTp&33jiboVE1;mO7a!^kO)m`59OfS zFTZpJ>KTb?NWb?oUumID7JkP#&gF(oMfpSyptn{BCoaOtYCt;a&NNK)lzY+IuNSW`4)2YY0%e5H!*-^{@#m2&vb_ zmJQY$zDDxz>FK)53zReOp|h^l@oYJHBS`{Bgsif9sJ4n&Q_sd2H!I3Yqg||q(kpw3U4f^3oQ>qUi>74L zcTiW2<*;lF6C*}tIpZ=G+gA|yj%K$i=O<=QoDG3al%VDA?8q3$&g6*ob@@8`@88Ss zM7RY1S(M_9Tw_kfffxa*T@zu`NO+TB-2njosQs=cPr8W%ttbkN<0!B5Z{k8IS{dnb z8xl=td%XbQ6$JbJwh0CGq6mGswww;&?TqwiB^2#x%=Y)il=cyDlU$knC(+^ZE=mGr zHE0H+3)!$W?uN#^w*4$rTi*#(07zfImw^w0c`WRQmW+@JW$MM_qeO=FDnL=Q(Omq& zFA-V4g{Z3HjZ`5+N$JJ23KB~UkN9Fci8AZlEH=OObNy`4MABCpnt5}b4tT$F!uC%_v+0!TT( zD1mP;WF}rn9Ex4w@pp#$TV<)hjVafS6xY+Qcv6iD;3NqTggu+=$R~d6Ze8%@r4R3- z-+hR_t3&tOIJfcS&XdTKy3vj{Ptrf?bbPIM-^xmXMLva8qp}2<;~pbyPrlX3B`K}7 z_Cxj_S|Dd9M~x`-CYljVBkbhqshUL!F<4EtD8;U=u$4aD9q}HLpxDg5ELape@u1T^ zRK_$B<7;FLO(|m+0N{PA+QdHOeVx}x08E!hj?r&Mp({G&YTf7ZV5cpawLW*)$sfBy zUmAV={M}>TGUUnFpIN`su<0R$BK=hry_1w5pF+dYNps^Kx=*6O^OaXj%%;~r-%XH- zm8r4bxf?bXq~XS}lAT%Y)#-zRgK)jw*Q|3m5Lc}L2I;y zqAL-ya0?D#nE8L|p`6}>e22|&=l(>L*04+05*OHwjYY=Y546WtrQk(Uu59)CM# zNU^Wh`96;zB&TIl{{LFBf>Q^#KwClc;5lldS}eyvgPVxwQ+0Qu+h8LAQgN^M%Z#R&l6Llc$IpFe9U$S8%jP&SxY zV8=)RHE*D@0>tL_o|*QD7wr~wS#kshha4+LRM;H(4sv%YPPx@ja2Zwy+cmQ)!np{%eB;I=f zUsC2?oRyNNFMkYp%K6=DDXp&MTtq4;%W(K{4HxG&fMVpvcZXZ&dh5B(MFu3dU z?8}};JpclY^AZ&2Cn5`P8xjqvzgi?dhB&JG{|2_dYTpo34VhGv#1RIX+ zKa~(2gc=jBH_M3D&;jZfoH+)oI@~bV>d+_(@5;fJG zM!-4Ro{^u9&!Ng?0hEf?FnwIZys{c#4ZXKg-w|o~8s3PNrgHR(#Gw8)!ERdzEK%U5goCaW;Nf< zJ{o=W15W}E;0vngN;3Qlszdu5)9-$ZN#oz0?(xIT83(}?#GCw^E1r<)yht=&4&e}H zXt#7|_8IOkk#=ahQAWznD<`BLZsmdbO|TmGV4ZRvv5N@}KpF`91EiZgTcUYn#-OQf zox@QT_}*}|tH_-WwVYs5N?G0NEzpb_Dz;(mpAB_P@i>DRjJE=lU0-z8Pd?x9{vO`x z_<(qz1_kz;xQ?EkiZP<=0pkGRP_u+9Li|GK51l*YRwf<)v2&p9A!xX!m2(uvdEexv z4Eo=36S!{oxTXoB&jIy06%2RnPzRvY>$Y68^2@bqdoUflZ>zM?9M||*ht77kTh0nW z;?eoCZT5&Yyve++2HOv?+zlGa@+HtnGz;RB5B5F3d~ogHH}S0|NlOuVK3@y!->RPX zt)8b%l54-yD#?S%s0c9D}sK-X{42o4(Orl&hg##Z7=n9ZZ6ioo-K}5|K#h1T$ zq6fM@*N+_yv+7Uf+kTKzI*50Yy?z}WY@b`}sx&PTJ;=7K^5N@AJx&-m`r~gw4c8g8 z-(FsWKmKFCT`j60_93v%jcHA{ERuaUFu((MH|mCtDZmLLL4i0wzE*&?QMY&w!A4ev zc)Gg+$F_79fU21$iW`II(Ef(w-W`Es?*V(x`_zI8LgA{T97Ev&(M;HefN@GpF!Dkf zuJa4EGzq}GhhyKod(UyN9@b)(vij%s>Ed7*F;TZ0iU&@?k9v6{4TS;TkK}aA%7F3o zPbhLaVo;bt?dnNZ<9A!mlu%21E=skRr|(@Y#Xj50^)F8=R|f;d-(M>rsCb>*Z48Ym0V=MQ zG7e_?C`j}MunMIe@ zII*Pu0#g^#4v_Wk2z5jzCej6ntd`NRNXK8dO(-}(7-D+!Mm(P>G zo6D1p&qC%i8h=bWxOt+39NmhdKs-$2D0UqLC|bPz??R3F!aFY8a?pM4-h${tv^281wX~(~MiB z(BTbZz*PUt0_w2UspvRipsV{xdDoB|r!;%$hl(4c`6j363D6NmfNL>K;4nj<|4VDS zVZ+yX^f(|@B5aq*A=rh0f=U4YRl~dMEqyis0RE2#AV|=N)_`>nMact-Hv>N9`mQK|gW0u*BU)@+?!vDBuMe(raP#|Mt|5396u;@3-oN2ty88t8} zVTM!Jdg}mU`AV}J-U+9r9L8D!1hBEF3j>Lr!AuN0-rgh3iS@20-`Tk2a8Y+hYUm~B zUz|Q-olTse_}f0d@b&dEKLV8fb9oH=>eVaf7L9E_&A-1vz@4-Gx$q^sV~2sIIx))9 zKDz7hfmE>f`+Xn5-K7H3$30f3IOhvkbl^QF4B-R;oNvZw$!1aIaBjF0`~^Vu;S3o_ z0JA368ue4^L=)GVlVy{Za^-*8bz4s3&Pip8j;X*m*ffQ|s@N^y-kJNiNNb!xZ+1rQpl4)Kl#QpieA5^Hd(;avm7GOwive{#Xluf#%#XU$DH_~!O)2! z>eC;G6Y$3I%v3blW^JPGnZzQAgFf<3O?1lf2mT!gQhR28zQo6DZ2(JGX>!B;imM4` z{Hdju7Iu`XZBBvNjXj=PfGX-A3DZ4-LMm69BA+BD=?(Vjn#UaM6bIiDN)@`-jW{SG zmON6G|Bx}f=xQ@Wh7F(jkXo2>WR1>A#iJf)F3@_rnw$S}!4oClK5^cuVtIRbJReAR z<%0z^wAI4Ym6v0V(Rf-$b^ze`%LV64a^pbNc#oJ2xu82uXMKe$_mDKTMbW&hbc7JL zMFZyx_GpfW%@xE!ee(R08jdzMg|JXzV(KeC_0-hA1@(A71rw9o$#ON=5Gul%cj3Q5 zm!L)q)z*+;be_4?F90YWH+qdcSpVlwgNmK-bKd}JgP&Lxb^hDS)EhR&@Ooe= zzh{*xujpjZJr!r-lZUAkb^#|+hk>t*+eGEE=pxGm4X2GVCUf!$_N@;On?x*Te z7oLpBvpoBWMY(6Qz@z7M3EL59Ld z0MZUuk+2JU-ZfwYVE32RvnFSs^@o%eKh0Eu6S)gEu+Ia#xA{&VldU_!Dc<*f=X~`| zdD1Z5!l%EC8|jWoX1%EPE*khfm>NG)V%@zwI9-A9q&b4dCN+vOWoFW?W(Kc#^uCZM4okM<@)Jo%_trNwq(r^0n{?g&cS|JkVsEH<=IwYic zn$R7*8UF+{I>7`d2fd?=wZ8H&Mv)+6?4f0S6{dcLRqlq$Vm$m01QZ>bd}S;Z&LH)Lk~7O-a58@6PP7*&n}Y(t!Xapy%yQF$#M z%O%CQ>_iGJ`9^lG9?1)NCDB_^LdjSQZ4{U8*%lOSXCb}jt0=d66y)A5ZrmFK{e=Au z@x*4XU~EzxCn13h7H^DtM^m7-LRMtBUdkh!;G@NcyB-qiny?s0Xa?< z6_t|ni4^_eA6uRPs(4+=zQEM9IR?}lC?|!O$REG{lgp0b0%qhi*ne@qpT8f@IeVRT z)?R0yoi|rP-xc~-PD>E5_6hA28 zyMQL@1yQf98_2T=*c?kTHZ_^(Bnge92h#z@m?lSL=(X*xHdfXsYfYWhws7aTUasId z6%5tTnml|~nd<&r?jq)-C!Z zcFrjI+Dt#~cs#^Sl6Uq|cL?DLSPNQJHy3Rl)AnC1jUYD6A(XxT{fwd!_k`Q}9rw}S zmMyK|ku<}hz5LML6{&dY?Vg-CY()>X4l*R($^?{*YM8l1x>=9C=qF(wV zfro8T3IU!(L3fcja5ZPJco{&}6E(mc1jUsPg^nLPPw$})Ot@59=+}bI?&!rBXA(J$~ zY2g9fU+{(IceQ=Eh;o&t^bVt-@9N4Go6p~YIk<~p(Is#R>XGC%nA{z{Qpg1i!ef*v zA=OPhm($%rbHXFt(IGXiz%DrZKMtTOAJBX+Ei|*R8dh3x)gleMF!Ee-IG&+tZ=^!B zB|bvH^mFjY0ybDl!$IFGBSqI_&dB*~Ce2^-ld!|Gu8Gq1{5{EkhET#e)Esa}fJbZDluk997Ulu#x zgg|HBl9O>|zK~|S)|%uGt*81q$zg!}W@-CG91l;7g|CxgVd0@rcg4VBA%<-&;2wg8 zDuEUR#BW)p`P2q+{2{+fQ900}W=4v zjm}4bZ9|z2+m<~JRN5<{%3i6eUJq-F;cN|#w+TVURvmYy$sMWtHS^C96*tC79%5Z! zEO9ukYVqzRV%6$T`+PSx7$y^e?keQMVFBw2qmi|raw6i8qKPae@DS?5-sQ7x*)&dWNHomTbJ#b2qj;M-EC}>O*H>EN zU{tcRGz*bTrUDUyeCtB4gyjm7&ywH!J%5dxPKv%C;+aB!%Ji2k%5~2}kWD z1k_CgYN!f9qMehQWsdv5-RY~ojtM;QGbAag?CX~Dx8WG*m^W8m&o<;ymF&TC00#NW?8$t zP|4pai}LVA*Csr6Qv#l$EysN360yY0*flqKRPtq%Q1VuO2^}5txx%y`+yyg1?YLSc`bNbA;Y$Wd`=kN3DT*Lg1+^i z1!+EMGSL$v45p=gbrH5qP9RK}KHiEnhu%l_x01xGW zT-b8QG_QOE8l!KWG+ne`8h=1}ctU_Z^Nw1bn|g4>QLInM@CxG3&0!V02=PyQ@s7D$i)yF4CQfeW0A4*{7(dAy&>(yQCY( z$fAr{)FV)1#JQ!@!<9_V>?Opz9epP={*$ppV$Z2yTJ-<5&xlLpmn%B* z3V!KB_XUvywY66V1N^mE8@XPXF}mU#dS>ah8>_8nJA~t$-1E|b)tJKA0HJoOZpQm@ zodATjT=;Xw;H&SPVUs9}Hz%o!2SS4eIZjZASz}tMMD4i2yEYwPD&gBs*>~rbdrmd| z`2ARSQ}Mwj4hj=d!!1Q4Gcp5e8#xOvo!vKn|FNE^X8mWR7ws1<8qV8@Jp6Bn6Rjc1 G*Z&WjMzdr9 literal 0 HcmV?d00001 diff --git a/tensorflow/compiler/xla/g3doc/images/xla_array_layout_figure2.png b/tensorflow/compiler/xla/g3doc/images/xla_array_layout_figure2.png new file mode 100644 index 0000000000000000000000000000000000000000..6439c6e40272ae6b2954e9d7f3de2df470a2b36d GIT binary patch literal 7913 zcmcJUc{rO}-|v&ExoE5AMylFUl%i^g3R}@qw2D%5(ZN_#%wj6LW@@Z?Zczzps31bk zN^6XnsyQTvm?FtZ_kQ-d-g7S?jD@Us8_ z0Ji)0?iv69KnU%94HEwD-pj~|+|B=ZO+71oaZ%JO@uHh{ zCyr!66&Y+9!}-x~Q@lE{vg0PPmt zBmZX0C*;C{0=jfY0#Tb0_0&u9U3PBJ)fbaq3B%|{4R%S>Pj%iWyAMpdL^L9o zx^WurbkQ}8s~Tu|WhPlSJEosH6(Uh?*?+QH9eZ9ticx zLL>_1x`-K)hRTeH7TANz!^v81t5Sg?yEO^S{u?QlcIfP&kzkUgg`|^Z`M!P06Zu7I z55sj|o9e?#$LlT6V@Ip3;hXQ*Dv!s@&2*-B5RYXG8w}dEoXYFm16E&ag;rE-cQlk< zFh73-3-WWzU}+^ID&)6v9WVpB)V*Xem0=IK&%k*_tC~8Tq;j71isPtx0LSdcnfYGc ztGgvpBmD`ikR2{u3@IM|70-eFvPL_9x*px`e}D);`hR%(zV}uXdyWkEYu>rNO(}GI z%wEcsYrQRA0&73dyNp>z27%|fmddDGFTKeJU+os@d1ff_dL66ipmo?wz*Hga5fZP8MDS9<g|Mad1 zj!B>`4k?Udg0w~aUBn@Ue%EM?PJb7_Yn54xwM!nqHuUGVT^EV{Z{vH17r}{x-oGGQ z=z!S~gDHr7Q(2QVd-!YS`$(i&RG(4MaziLl1Q*Kdk@hLE{ORYUMq*_GgyBr;C!y(K zJ_jV+laQ>ux9al>dnkscPQ76?5BY{K=}&TLCh@%i4Zy!Pl#U#YJH9TShT^1a2QT0N zss6{FkLw}!wGNe+e%-=R8a-3Lc627QIQR@^-!`WAe-}agg~M#d9nqDYtIK6vZNDlxEeqB z=)IN=6JVZcXO;eqHspl6WjNWn>*{=NcYv=t;7$~NVeTAd@)4)_2Loml- zMySPxV`UP?`yi>=Y`mvD`h^&i&|!Ay%wyZg`}KPA_iC=Oogk;OcB`kSh5D$n3W67- zIVtM+W(zXqLBOAwjzGfGlj+mc{p>p^QGy~6l10K&`F1$5LN(5C3mqa=4Xbz>6+{N8&JTaV0NUk77QZ@yh4Qp{ruo0m$fG) zUNiLl<6~LjRfFm#BuSA<+G;XVW8eLQHV#ND!%}9Tl|${i*y{AEDS2V=iLU2^50 zR78!IHdSb1X9TCAX%p$h?GY=_eFDBzNQcwi$5p->Gs)pS3Pxw?Xg^5!^ri12K7+n( zfW3a=ofd=va;GZGpeAa8ENfN9dm;f#(!(S z#h4<_8jIN?Xl;xZ+(qUrgo0!g?Cfts&vMe1;RkqkDE0kSUlgQ1?4&Q ze=s$>(QR!l`TAlB_sax~#_1CXjxJcYm7OmIB-0pvcW&Sc?q}A1*5{vmLS}_a1dx^C(D3*n&bq$42|~EEV1F&GS^}Sagr2-{nXLWeadKBz#*7 z@l}QKIT@QJ_Mi8NO|fp~n9Z-$WMuu*h_$pl<{F{;%-*N02)6284A1 zYDF@CS_YtLe;aWLYH-H{J3*#fxr*&`VM4jH-}(v^NH%0rvXeK-mm8HG9#30%3L)L3 zHzVD&=v*It-LIGzU%{TSgn!ZU@-)nE>0Abz-`$2zcD{kKvnt_9-VX`vQPimDsB?B8 z*R4waC9W?Y*KSoXLq(wn)fjoZp z7|S_L>^WGIMGS>@{v}_>oUm|^jTZJt?!VNG@>liT->tNQjdt;`n)=^`7WVAFuHF9Y z&Wv%Jc&%s8o>}l6+IsY5QlQ6Oot+K0SMfR2AKmw!!}=@IC!}j^pVNBz`vd=}Px!wy z_5V7WJ(}?T*4TfOT7k9!HLOICOQ@fv<9v{E5vqc`un%{@Aq}Q7JuFjkYUE#U6FI_{ z!KZPArs>iy=+3*{90jo{C=-OyhG+fVYWudRnlglIu(D$ndc}-oKlkQx{L1jprp-b# ze30fz8e-~@)Ah~+jEm!zOpn8;c?A^wYOE;44^}aZ{5_lBM(vb!DNF(ZfCU8{seO|J z47ky@zx(E>yy$+r-#Tzc$11%-P<-tIjzHLi_`?$Yyc|=MO8WQKd(Al7286KREb25v zI@z#fCI2Gx(Sj5|J&D$^aUa;H>s088;u9H%F=OPot=FuvFh#)%B^6if(k%W*&^=#K zjvhVT`^E`stwPxC1gF#J$4EZn_e*?!_)Qj3*oTFQ+CxFX$;~Uct+ioY47-XSSnzC% z&9`SzpPM3jn5`1nCnN3`k)UPNi!^o#+w6!Lf;8vo<5mCtO*{*Z5d2a(>!~Yr|JN-d z^5tWS?X^3?*^M*%(W16)TWkVWTs!K$VQfjXb&$rQ#3r2Vp@&(f#S%(hy*e8oN6Sy2 ztnlo84rSLf0EGU5)pcGadQ|mEVvh}q>~(`Hd__+{2TK!ey9?bnbKW<1b73dC9L->5 z{p@P4z2F2!+{4D_GuZnWDMPH`ujx5#9c_XwYKP3d!U}aiY@0mDw>o`RaYmLkJX=t& zePD~Tfg=zS%y88i8Hlpx5yepm7HIbtKk`vrfYH$9sVlgP>l0ttP_y1MrJ7|Bbo5&%f7paX! zmMX4g(pY+SyEM3kQZy1}Xj94~z9pb!PW&EfQN%P#{TA9>O&msO6A4XY%9p3)yeI^% z!MzSH#m*HbjV(Xt}Mcww_^2>!y__iAOGKo>ZDXmwdBR&3MWfkkVMhBrjZ8&`9Dg#796X4VB zl4DiDb+Tc>3 zRl`6;CGTSwCaa@Pn``tb+**OmPFsX@{&Em)3X-=^-NCzWY_7joA6|*~P%H>Vk)JC?$OH)Q0*WlP4Z`-X} z>bXAjmF__`RM#%mFC86T!CGC`KdXI zkwHHRe56YA^x`eBPsRRE<(`mT`znM=$i(S-U48LNPh>mjY_7haCV5-9@3jnJ-)U;` zKxyx%%=S~{lm+q$AWL>rsT*^(JodP&L!@Cn*Wl{t3x}<~9sP3S(I+y#pN|)x@I?%( z9RyV#lDYM|y3S|pYQ;NkKiHOYtWXWxoRk~3Aa*rZcAn5Jx9EL~q5VF|BVsLwhdx$_ zbl$mh@mg}a(oyZo1PHpOlhJr>L_$)}SJTw~bjvj}TVumyeEJezb`{Jiuvi8+56DVS z_uqt1@^5?^s_N08h9CM=MIh>Js3ob;08Y;HF^7xZ1<;r}4FuL?DK;@H{Z|=W+n`z+ zPOkSRc6>o(XSlTtVJhp0TkcNVH3nHZ*R10Cu4(NU_fpj9cP6&=+mzUGGncIfGR5ZX zYm+>l$R~!qasAluLZvvnmuS6Ew4%x!txvplf4SkSC4#UX+tmTpK z!o@0t->5^&uvNdzXv+Gnv@*cP2Qyz<$7HyJ$WC_hkZ$;*Al%)!d{lz_ZZS14IdR>p#eFe{I1Sh5lquGz7NjgR*~1itpsi4#^c9#fEB zc%*b8AH8f*gK@;E*;-@1SXI*VO+PKb zlQ9MIp2YQ@Zo^hbXUy1V>O`XL31!?A={1bbCXTXES-I^p$L{^+x;sLaarz)f3GCkC zbL`m(o8qy#*&po<_4`uQLw@pP4pWLtKbc48Mc*3-qovqYUd7$Rcb%ucaWt9k7GJ3? ztB8;F8ih(p4j5787Z)wIsR#3ncD@0BESbgL+1UPYzRR-}!^y){=;=Xs8+{tS$2gPj zPd|lI+X-7j=!gw3qCyV(IL77#$=^urH3pW30h7Iw(@eMbdcP~knW|9ElV=E>hh1B` zy>qduPXdeBrq=LD)VBvJl@$k_eip0^&v?SJX%+*;h2z~jXjs<$f0>H^ZB+jDE94>T z`i)gBnb)hq2Bg$a7MCsjJX2ko8+#{DM&lSrV>6j{t4JZrCK@cB^n+6{ML@|C5$C9t zQ2#V#9E?6lly3jkxbK|85vNX~;v2L^f&$3?329Zy?;u-U%zlqR`v~(80WkYxLIU2l9dN{ z_BE5(v`+cKr>R4pF9nYRxjX@&7M=&K^mM4EPl)SMr<-Qwk0#!-RD-wXNiVwJNKw9oiOQ2^W(>HC z*98kATXSp`CO2_8+ToWuj9x<_Hrm(c4Rt<=*$N6SFgdY1&lPV8jZ#1eS1Hz<`Fc`-7AN8k`du%e#pYRC>M8q8rkRTTm0E+%sg9o zK585rZ~L7j4c?rxI|o|`wTK*$FKexCcX_>k$=3yQhPuCD=6*43X7onpbtwog;!f%f zTanoPuzD|JaV=}Q52rpTW$s#L$+kXu;T!BWT%M`$lOuug4{&fzI~U*WJlcbBEuo)? zIr;|;dGcWoT!{P_w!@giZ|ZRv_M25C;hHe;rN{?e6pzTzrnfE1lx{45uQ~e;A;?QP zSa+<}`D8aC$5=f?WEv^nJu^SZzug(8AN8Xvgvv_x0Jz_zgj*;^fM9@2)q-|P*mzuf zj|?|o5v_p@5cYu4bRa6LIPRbyz-#x;2Xu~re_VD2vK4=3z_NEE#P|=a{+Kd*!@f=O_U*ApaJ!-IS=@xIWuYr005V!LoTmoA$0%3L=CoMLjKzR6MnCc7;_KXXmAhifptwAxYM%2!+@Vn47@_{g6(_SL(hRJ;D zIM|xb4*AeV>c!@+v=f0&D2ob@0BJvl+^ymb_0ST>w3viexNdX|GSh5fFh~Y8BcV0F z;IFhH4TQA5*czG$!UABc_KH7b)fxEG6&I#!q1yM7)Ad@}CuFM|tGOjT2Zyd>^F8TT zaUsQF59K#SV);u$;Z%Pxq*@i2UZBN;?PKl7g<2%ua2-5*Y3@8B!$VX=&gs$6spF6? zUUsaFOVy-9F6%|s0j~L)L3$q1wlfgYWk4tx7}r6!*N=Xa@pnzVBAsP5F)ZiicEY5!7-T^=-!IhfArGyJ7}Q>i(Uv3r5;`&tg@1Zd`C3*V|9 z4F^eyl|ZG$mNh#5D+K(k z&Quk_0rJ<8syK`!dY)~KmNeVx3PTdx8{95tH_jFbUWlWzmRhHw#k7>_@vCrNIl<`| zvX59G>9#2)gwX*4%uLy}GC-rfqEmOUu94-8cia26Vx?VsA84Iy``1)D7oKc#&~-WD zHYTfT-V2vnM>2Xj=EfqHQ;%o2*rrC?tC>+Z+t@?sKg{>+?gxsyrhF)^T01`Nn z{#RX#cNiWLRRAw(!7OMVz@*p85IlO*tPb|p{KtnDs{e3KuEcetq$OHByBKD%QDM9j znId&e-Vxb;3W9BNUjrGTfF;<9xYy3 zSxRxZL}+8nY|exlQZ$b6A8psXbi93Ovbt0p2yA!wXhiPf)uYnl1r3D7?P)7SyxVG~ z;p2`urR_339M@H~@M14hRlz3GW5XxV-)jqCQFzRz-OOwwkSBs`V}b! zPD!O#&@^D?`?jC&QHn;fwYM+1&v2};Ue?k6gDrDCRP}g);!N>~jg2MJ@wU=4Wru?R zFW_OXXd%qcITsHM!%Gr9Xo;m$KA7PeZF;cGnI_G7SBryG7K$}pa}yFc0^s()U2Id% zSgmUZ)jQ<*<&rRMUrwG z6ne;MRCAMObxdtLJ0FM44GRaqPac>-g$Y@KeUrLVS%GsgzK=m~vIFyCYSStjlQoP% zXI9nMCV(%ypQjGIN64_Lg6B~*vmiz~b(n?A00t1HP)&x{8F__Jm~Dlbn)aj7-fqLL zkeJ*4^JEG>zn`roMiM^Ap3rr13K$JMdmhGM6B!s1C_ziKJ%8g=NelIct->hJ^DqGa z37w7?t@&e={DmxW%c&ZNnZA*$&74TO)wADe!F$^l8GtM`?3A^gzOpu6LyH|i>xW{e zK8KisnzAkPRNtQYzV4yf`DStL#}W7FpeH{}0q(Prl!NpD!T{|2zpfbz)6rn~`q(nU z7yvk=X%${mlZ%P}F|u9pXH%llq)@%s0rg}G;o7sER=*ID&A{eo?+4Zv^N*;7ZM%<} zxu3e-{Q>a4np<^f*g=$_zzj*=3gIi>dQspS@ddCNW|g;{yfN*?7YzhM44ENeNOfib zC(Uz`*nE9Cn!BM*^Lu)S92b>TZ?LplalAvSk0~kmC(QYK{Fq{ay;^%kiQ%V&EO|9> zivpCAotY%gv~pN^-7obw-cRX$E%~sOF;IOIu@G^dJ5XIj>W-RbqzYfP*k-&0PqU0f zDpu!}9cm#0)NA~ic`sAX8WoK6I139=%}}syda7^}2-#aYySMLL@yDrP1<6v$J3j>X zZKx^(;8u_I6S`(16(|GQyCIE~N>3r|w6S9ZecncH87&NPnx78|F`Bd{Jzv_E+BnbI z%p3B6HLZa5?5)?ngS)~#Ixf=>fJjJ(jjzQb1Wdq&FT2DdP}LuwfLzq**jv%)X!?-; z&kh7_KjPo?@!xV~f9s*ep_h!zP>Sjl{DbO0jO>VkJtIjTqkeJVwV6@2mWY$ZpDTqK z7k}f^!5Dmh;S%)v)hx=2`M%ckh0f_IGMatIHls~W?~}=58Fh_^yT>=a)Zo~5i{q|_ zkqUOHqpK%P$ee8TW1E<6^A{%bE48LrM^Yul%e{uF$R&K|v?EnGm0GVh+ay@&w!d&L z6+JFT1uMZ)e^c^)JgfukfmT+@FQPG><8!v_dit{OZ7$9x6)I;w{kX>wyWY2bzc7}e zao_bE&q^6}^DQ%NzsvOipO%ahzh7>nj5?%&Q3!tKx=}v~ev%O&PY6<+yQb42yXk-`pZsG=T@RISH@O(Wigcq0in(BoAdgHhw5Gtj76Zotai=2S0F z-wHo5K0e-i6na&z<1b1^Sk}JB_b=*~L4)jg^%!S*n!N9lvU3y}b4{N0g9FQkYnFA~ z7E4%eVJkBd*UF$S>+(xrg@ald8QKO^(&HrkC--#~N2L|lrgR@YS{c;1d#LB9+)|jB zcy-}o>8Cm0?&C&DrQ=9PgQVmac6N5sEyneotA`D6*XSMY!g$+WsFKogXeOn~t%~PC zKY8$|tE03Zsi}=q5lhP+W~F> literal 0 HcmV?d00001 diff --git a/tensorflow/compiler/xla/g3doc/jit.md b/tensorflow/compiler/xla/g3doc/jit.md index ded1e582b24..85fa16ccc7f 100644 --- a/tensorflow/compiler/xla/g3doc/jit.md +++ b/tensorflow/compiler/xla/g3doc/jit.md @@ -86,7 +86,7 @@ on uncompilable operator, xla.compile() returns an explicit error. This is useful if you want more predictable behaviors from XLA compilation. Please see -[xla.compile() tutorial Colab](https://colab.sandbox.google.com/github/tensorflow/compiler/xla/g3doc/tutorials/xla_compile.ipynb) +[xla.compile() tutorial Colab](./tutorials/xla_compile.ipynb) for how to use it. ### Placing operators on XLA devices @@ -144,7 +144,7 @@ Execute the python script to train the model with XLA and turn on a debugging feature of XLA via an environmental variable that outputs the XLA graph. ```shell -TF_XLA_FLAGS="--xla_hlo_graph_path=/tmp --xla_generate_hlo_graph=.*" python mnist_softmax_xla.py +XLA_FLAGS="--xla_hlo_graph_path=/tmp --xla_generate_hlo_graph=.*" python mnist_softmax_xla.py ``` Open the timeline file created (`timeline.ctf.json`). The rendered timeline diff --git a/tensorflow/compiler/xla/g3doc/layout_with_tiling.md b/tensorflow/compiler/xla/g3doc/layout_with_tiling.md new file mode 100644 index 00000000000..5e990851af7 --- /dev/null +++ b/tensorflow/compiler/xla/g3doc/layout_with_tiling.md @@ -0,0 +1,159 @@ +# Tiled layout + +*Note: This doc describes how tiled layout is intended to work. Tiling is being +implemented, but this is an early effort and it is currently not even guaranteed +to get an Unimplemented error if one tries to use tiling - it may be just +silently ignored.* + +

![](images/xla_array_layout_figure1.png) + +Figure 1
+ +Figure 1 shows how an array F32[3,5] is laid out in memory with 2x2 tiling. A +shape with this layout is written as F32[3,5]{1,0:(2,2)}, where 1,0 relates to +the physical order of dimensions (minor_to_major field in Layout) while (2,2) +after the colon indicates tiling of the physical dimensions by a 2x2 tile. + +Intuitively tiles are laid out to cover the shape and then within each tile, +elements are then laid out without tiling, as in the example above, where the +right part of the example shows the layout in memory, including the white +padding elements that are added in order to have complete 2x2 tiles even though +the original array bounds are not even. + +The extra elements in the padding are not required to contain any particular +value. + +## Linear index formulas for tiling given a shape and a tile + +Without tiling, an element e=(en, en-1, ... , +e1) in an array with array bounds d=(dn, dn-1, +... , d1) (d1 is the most minor dimension) is laid out by major to +minor order at position: + +   linear_index(e, d) \ += linear_index((en, en-1, ... , e1), +(dn, dn-1, ... , d1)) \ += endn-1...d1 + +en-1dn-2...d1 + ... + e1 + +For simplicity of notation in this document we assume a tile has the same number +of dimensions as the array. In XLA's implementation of tiling, this is +generalized to tilings with fewer dimensions by leaving the initial most-major +dimensions unchanged and applying the tiling only to the most minor dimensions, +so that the tiling that is specified mentions a suffix of the physical +dimensions of the shape being tiled. + +When tiling of size (tn, tn-1, ... , t1) is +used, an element in the array with indices (en, en-1, ... +, e1) is mapped to this position in the final layout: + +   linear_index_with_tile(e, d, t) \ += linear_index((⌊e/t⌋, e mod t), (⌈d/t⌉, t))     (arithmetic is +elementwise, (a,b) is concatenation) \ += linear_index((⌊en/tn⌋, ... , +⌊e1/t1⌋, en mod tn, ... , +e1 mod t1), (⌈dn/tn⌉, ... , +⌈d1/t1⌉, tn, tn-1, ... , +t1)) \ += linear_index((⌊en/tn⌋, ... , +⌊e1/t1⌋), (⌈dn/tn⌉, ... , +⌈d1/t1⌉))∙tntn-1...t1 + +linear_index((en mod tn, ... , e1 mod +t1), (tn, tn-1, ... , t1)) + +The layout can be thought of as having two parts: +(⌊en/tn⌋, ... , ⌊e1/t1⌋), which +corresponds to a tile index in an array of tiles of size +(⌈dn/tn⌉, ... , ⌈d1/t1⌉), and +(en mod tn, ... , e1 mod t1), which +corresponds to a within-tile index. The ceil function appears in +⌈di/ti⌉ because if tiles overrun the bounds of the larger +array, padding is inserted as in Figure 1. Both the tiles and elements within +tiles are laid out recursively without tiling. + +For the example in Figure 1, element (2,3) has tile index (1,1), and within-tile +index (0,1), for a combined coordinate vector of (1, 1, 0, 1). The tile indices +have bounds (2, 3) and the tile itself is (2, 2) for a combined vector of (2, 3, +2, 2). The linear index with tile for the element with index (2, 3) in the +logical shape is then + +   linear_index_with_tile((2,3), (3,5), (2,2)) \ += linear_index((1,1,0,1), (2,3,2,2)) \ += linear_index((1,1), (2,3)) ∙ 2 ∙ 2 + linear_index((0,1), (2,2)) \ += (1 ∙ 3 + 1) ∙ 2 ∙ 2 + (0 ∙ 2 + 1) \ += 17. + +# Tiling as pad-reshape-transpose + +Tiling-based layout operates as follows: \ +Consider an array of dimensions (dn, dn-1, ... , d1) (d1 +is the most minor dimension). When it’s laid out with tiling of size +(tn, tn-1, ... , t1) (t1 is the most +minor dimension), that tiling can be described in terms of pad-reshape-transpose +in the following way. + +1. The array is padded to (⌈dn/tn⌉∙tn, ... , + ⌈d1/t1⌉∙t1). +2. Each dimension i is broken into (⌈di/ti⌉, + ti), i.e. the array is reshaped to \ +     (⌈dn/tn⌉, tn, ... , + ⌈d1/t1⌉, t1). \ + There is no physical layout change in this reshape by itself, so this + reshape is a bitcast. If one is not explicitly thinking of a tiling, this + reshape could express any shape with the same number of elements as the + padded shape - the example here is of how to express a tile in this way. +3. A transpose happens by moving tn, ... , t1 to the most + minor dimensions while keeping their relative order, so that the order of + dimensions from most major to most minor becomes \ +     (⌈dn/tn⌉, ... , + ⌈d1/t1⌉, tn, ... , t1). + +The final shape has the prefix \ +    (⌈dn/tn⌉, ... , +⌈d1/t1⌉), which describes the number of tiles in each +dimension. An element in the array (en, ... , e1) is +mapped to this element in the final shape: \ +    (⌊en/tn⌋, ... , +⌊e0/t0⌋, en mod tn, ... , +e1 mod t1). It is easy to see that the linear index of the +element follows the formula above as expected. + +# Repeated tiling + +XLA's tiling becomes even more flexible by applying it repeatedly. + +
![](images/xla_array_layout_figure2.png) + +Figure 2
+ +Figure 2 shows how an array of size 4x8 is tiled by two levels of tiling (first +2x4 then 2x1). We represent this repeated tiling as (2,4)(2,1). Each color +indicates a 2x4 tile and each red border box is a 2x1 tile. The numbers +indicates the linear index in memory of that element in the tiled format. This +format matches the format used for BF16 on TPU, except that the initial tile is +bigger, namely the tiling is (8,128)(2,1), where the purpose of the second +tiling by 2x1 is to collect together two 16 bit values to form one 32 bit value +in a way that aligns with the architecture of a TPU. + +Note that a second or later tile can refer to both the minor within-tile +dimensions, which just rearranges data within the tile, as in this example with +(8,128)(2,1), but can also refer to the major cross-tile dimensions from the +prior tiling. + +# Combining dimensions using tiles + +XLA's tiling also supports combining dimensions. For example, it can combine +dimensions in F32[2,7,8,11,10]{4,3,2,1,0} into F32[112,110]{1,0} first before +tiling it with (2,3). The tile used is (∗,∗,2,∗,3). Here an +asterisk in a tile implies taking that dimension and combining it with the next +more minor dimension. Multiple adjacent dimensions can be subsumed together into +one dimension. A subsumed dimension is represented by a tile value of -1 in that +dimension of the tile, which is not otherwise valid in a tile as a dimension +size. + +More precisely, if dimension i of the shape is eliminated via an asterisk in the +tile, then before the prior definition of tiling is applied, that dimension is +removed from both the shape being tiled and the tile vector, and what was +dimension i-1 of the shape has its array bound increased from di-1 to +didi-1. This step is repeated for each asterisk in the +tile vector. diff --git a/tensorflow/compiler/xla/g3doc/operation_semantics.md b/tensorflow/compiler/xla/g3doc/operation_semantics.md index 73a9db75f6b..d888b1f23f3 100644 --- a/tensorflow/compiler/xla/g3doc/operation_semantics.md +++ b/tensorflow/compiler/xla/g3doc/operation_semantics.md @@ -13,6 +13,22 @@ arbitrary-dimensional array. For convenience, special cases have more specific and familiar names; for example a *vector* is a 1-dimensional array and a *matrix* is a 2-dimensional array. +## AfterAll + +See also +[`XlaBuilder::AfterAll`](https://www.tensorflow.org/code/tensorflow/compiler/xla/client/xla_builder.h). + +AfterAll takes a variadic number of tokens and produces a single token. Tokens +are primitive types which can be threaded between side-effecting operations to +enforce ordering. `AfterAll` can be used as a join of tokens for ordering a +operation after a set operations. + + `AfterAll(operands)` + +Arguments | Type | Semantics +---------- | ------- | ------------------------- +`operands` | `XlaOp` | variadic number of tokens + ## AllToAll See also @@ -402,6 +418,33 @@ then v12 == f32[8x3] {{10, 11, 12}, ``` +## CollectivePermute + +See also +[`XlaBuilder::CollectivePermute`](https://www.tensorflow.org/code/tensorflow/compiler/xla/client/xla_builder.h). + +CollectivePermute is a collective operation that sends and receives data cross +replicas. + + `CollectivePermute(operand, source_target_pairs)` + +| Arguments | Type | Semantics | +| --------------------- | ----------------------- | -------------------------- | +| `operand` | `XlaOp` | n dimensional input array | +| `source_target_pairs` | `` vector | A list of | +: : : (source_replica_id, : +: : : target_replica_id) pairs. : +: : : For each pair, the operand : +: : : is sent from source : +: : : replica to target replica. : + +Note that there are the following restrictions on the `source_target_pair`: + +- Any two pairs should not have the same target replica id, and they should + not have the same source replica id. +- If a replica id is not a target in any pair, then the output on that replica + is a tensor consists of 0(s) with the same shape as the input. + ## Concatenate See also @@ -1423,10 +1466,11 @@ Builds a constant literal on device rather than a potentially large host transfer. Creates a rank 1 array of values starting at zero and incrementing by one. -Arguments | Type | Semantics ---------- | --------------- | ------------------------------------ -`type` | `PrimitiveType` | type U -`size` | `int64` | The number of elements in the array. +Arguments | Type | Semantics +---------------- | --------------- | ------------------------------------ +`type` | `PrimitiveType` | type U +`size` | `int64` | The number of elements in the array. +`iota_dimension` | `int64` | The dimension to increment along. ## Map @@ -1780,8 +1824,9 @@ XlaBuilder builder(client_, "reduce_window_2x3"); auto shape = ShapeUtil::MakeShape(F32, {4, 6}); auto input = builder.Parameter(0, shape, "input"); builder.ReduceWindow( - input, *max, + input, /*init_val=*/builder.ConstantLiteral(LiteralUtil::MinValue(F32)), + *max, /*window_dimensions=*/{2, 3}, /*window_stride_dimensions=*/{2, 3}, Padding::kValid); diff --git a/tensorflow/compiler/xla/g3doc/tutorials/xla_compile.ipynb b/tensorflow/compiler/xla/g3doc/tutorials/xla_compile.ipynb index a83e3f78598..2a83092805b 100644 --- a/tensorflow/compiler/xla/g3doc/tutorials/xla_compile.ipynb +++ b/tensorflow/compiler/xla/g3doc/tutorials/xla_compile.ipynb @@ -1,25 +1,38 @@ { + "nbformat": 4, + "nbformat_minor": 0, + "metadata": { + "colab": { + "name": "The XLA compile API", + "version": "0.3.2", + "provenance": [], + "collapsed_sections": [], + "toc_visible": true + }, + "kernelspec": { + "name": "python3", + "display_name": "Python 3" + } + }, "cells": [ { - "cell_type": "markdown", "metadata": { "colab_type": "text", "id": "f4TSNCvpENrW" }, + "cell_type": "markdown", "source": [ "##### Copyright 2018 The TensorFlow Authors." ] }, { - "cell_type": "code", - "execution_count": 0, "metadata": { "cellView": "form", - "colab": {}, "colab_type": "code", - "id": "vamNSA0vEP-m" + "id": "vamNSA0vEP-m", + "colab": {} }, - "outputs": [], + "cell_type": "code", "source": [ "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", "# you may not use this file except in compliance with the License.\n", @@ -32,139 +45,84 @@ "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", "# See the License for the specific language governing permissions and\n", "# limitations under the License." - ] - }, - { - "cell_type": "code", + ], "execution_count": 0, - "metadata": { - "cellView": "form", - "colab": {}, - "colab_type": "code", - "id": "xD_ydfejEV7H" - }, - "outputs": [], - "source": [ - "#@title MIT License\n", - "#\n", - "# Copyright (c) 2017 François Chollet\n", - "#\n", - "# Permission is hereby granted, free of charge, to any person obtaining a\n", - "# copy of this software and associated documentation files (the \"Software\"),\n", - "# to deal in the Software without restriction, including without limitation\n", - "# the rights to use, copy, modify, merge, publish, distribute, sublicense,\n", - "# and/or sell copies of the Software, and to permit persons to whom the\n", - "# Software is furnished to do so, subject to the following conditions:\n", - "#\n", - "# The above copyright notice and this permission notice shall be included in\n", - "# all copies or substantial portions of the Software.\n", - "#\n", - "# THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n", - "# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n", - "# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL\n", - "# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n", - "# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n", - "# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER\n", - "# DEALINGS IN THE SOFTWARE." - ] + "outputs": [] }, { - "cell_type": "markdown", "metadata": { "colab_type": "text", "id": "e1oSi4lHFt3z" }, + "cell_type": "markdown", "source": [ - "# Welcome to `xla.compile()` tutorial" + "# The XLA compile API" ] }, { - "cell_type": "markdown", "metadata": { "colab_type": "text", "id": "b7noD9NjFRL-" }, + "cell_type": "markdown", "source": [ - "\u003ctable class=\"tfo-notebook-buttons\" align=\"left\"\u003e\n", - " \u003ctd\u003e\n", - " \u003ca target=\"_blank\" href=\"https://www.tensorflow.org/xla/jit#turning_on_jit_compilation\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/tf_logo_32px.png\" /\u003eView on TensorFlow.org\u003c/a\u003e\n", - " \u003c/td\u003e\n", - " \u003ctd\u003e\n", - " \u003ca target=\"_blank\" href=\"https://colab.sandbox.google.com/github/tensorflow/compiler/xla/g3doc/tutorials/xla_compile.ipynb\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/colab_logo_32px.png\" /\u003eRun in Google Colab\u003c/a\u003e\n", - " \u003c/td\u003e\n", - " \u003ctd\u003e\n", - " \u003ca target=\"_blank\" href=\"https://github.com/tensorflow/compiler/xla/g3doc/tutorials/xla_compile.ipynb\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/GitHub-Mark-32px.png\" /\u003eView source on GitHub\u003c/a\u003e\n", - " \u003c/td\u003e\n", - "\u003c/table\u003e" + "\n", + " \n", + " \n", + " \n", + "
\n", + " View on TensorFlow.org\n", + " \n", + " Run in Google Colab\n", + " \n", + " View source on GitHub\n", + "
" ] }, { - "cell_type": "markdown", "metadata": { "colab_type": "text", "id": "v9YbsuLZaBXy" }, + "cell_type": "markdown", "source": [ - "xla.compile() is a new experimental API that compiles part or all of a model with [XLA](https://www.tensorflow.org/extend/xla/).\n", "\n", - "Please run all code blocks in order." + "\n", + "Import TensorFlow and the XLA library. XLA contains `xla.compile()`, an experimental API that compiles part or all of a model with [XLA](https://www.tensorflow.org/extend/xla/)." ] }, { - "cell_type": "code", - "execution_count": 0, "metadata": { - "colab": {}, "colab_type": "code", - "id": "45kUPj5ZFrRa" + "id": "45kUPj5ZFrRa", + "colab": {} }, - "outputs": [], - "source": [ - "import tensorflow as tf" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "9NMQFjroSMns" - }, - "source": [ - "Imports XLA library, which includes xla.compile() experimental API." - ] - }, - { "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "-Uggy03rSGJm" - }, - "outputs": [], "source": [ + "import tensorflow as tf\n", + "\n", "from tensorflow.contrib.compiler import xla" - ] + ], + "execution_count": 0, + "outputs": [] }, { - "cell_type": "markdown", "metadata": { "colab_type": "text", "id": "GZVNiRmTDV-5" }, + "cell_type": "markdown", "source": [ - "Define some necessary constants and prepare MNIST dataset." + "Define some necessary constants and prepare the MNIST dataset." ] }, { - "cell_type": "code", - "execution_count": 0, "metadata": { - "colab": {}, "colab_type": "code", - "id": "f37TSEGvGX4_" + "id": "f37TSEGvGX4_", + "colab": {} }, - "outputs": [], + "cell_type": "code", "source": [ "# Size of each input image, 28 x 28 pixels\n", "IMAGE_SIZE = 28 * 28\n", @@ -174,17 +132,17 @@ "TRAIN_BATCH_SIZE = 100\n", "# Number of training steps to run\n", "TRAIN_STEPS = 1000" - ] + ], + "execution_count": 0, + "outputs": [] }, { - "cell_type": "code", - "execution_count": 0, "metadata": { - "colab": {}, "colab_type": "code", - "id": "TiVXchblG5hK" + "id": "TiVXchblG5hK", + "colab": {} }, - "outputs": [], + "cell_type": "code", "source": [ "# Loads MNIST dataset.\n", "train, test = tf.keras.datasets.mnist.load_data()\n", @@ -195,16 +153,18 @@ "images, labels = iterator.get_next()\n", "images = tf.reshape(images, [-1, IMAGE_SIZE])\n", "images, labels = tf.cast(images, tf.float32), tf.cast(labels, tf.int64)" - ] + ], + "execution_count": 0, + "outputs": [] }, { - "cell_type": "markdown", "metadata": { "colab_type": "text", "id": "x_ZehpZP-SfS" }, + "cell_type": "markdown", "source": [ - "## Defines build_mnist_model function to construct model\n", + "# Define the model constructing function\n", "\n", "Following code block contains a function that constructs a simple model with one dense layer, including both forward and backward propagation.\n", "\n", @@ -212,14 +172,12 @@ ] }, { - "cell_type": "code", - "execution_count": 0, "metadata": { - "colab": {}, "colab_type": "code", - "id": "ZbhJl_WvGa3g" + "id": "ZbhJl_WvGa3g", + "colab": {} }, - "outputs": [], + "cell_type": "code", "source": [ "def build_mnist_model(x, y_):\n", " y = tf.keras.layers.Dense(NUM_CLASSES).apply(x)\n", @@ -228,47 +186,41 @@ " train_step = tf.train.GradientDescentOptimizer(0.5).minimize(cross_entropy)\n", "\n", " return y, train_step" - ] + ], + "execution_count": 0, + "outputs": [] }, { - "cell_type": "markdown", "metadata": { "colab_type": "text", "id": "7Jh3lyQHDfM9" }, - "source": [ - "## Uses xla.compile with build_mnist_model function to enable XLA" - ] - }, - { "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "EtDwez_1gjzv" - }, "source": [ - "Following code block wraps the model with xla.compile(), which allows the target function with provided inputs to be executed by XLA." + "# Enable XLA\n", + "\n", + "Use `xla.compile` with the `build_mnist_model` function to enable XLA. Following code block wraps the model with `xla.compile()`, which allows the target function with provided inputs to be executed by XLA." ] }, { - "cell_type": "code", - "execution_count": 0, "metadata": { - "colab": {}, "colab_type": "code", - "id": "kYpCXCdRHNuN" + "id": "kYpCXCdRHNuN", + "colab": {} }, - "outputs": [], + "cell_type": "code", "source": [ "[y] = xla.compile(build_mnist_model, inputs=[images, labels])" - ] + ], + "execution_count": 0, + "outputs": [] }, { - "cell_type": "markdown", "metadata": { "colab_type": "text", "id": "4giQh62IrZGF" }, + "cell_type": "markdown", "source": [ "When compiling the graph, XLA replaces all the graph nodes constructed in the target function with a few XLA ops.\n", "\n", @@ -293,62 +245,62 @@ ] }, { - "cell_type": "markdown", "metadata": { "colab_type": "text", "id": "TPGas4jjFLZl" }, + "cell_type": "markdown", "source": [ "If you were to print the constructed graph now, you will see that it is not much different from a normal Tensorflow graph and you won't be able to find XLA ops mentioned before. This is because the actual compilation happens later when you try to execute the graph with `sess.run()`. At that time, Tensorflow triggers a series of graph rewrite passes that actually generate XLA ops, which compiles and executes computation when all inputs are ready." ] }, { - "cell_type": "markdown", "metadata": { "colab_type": "text", "id": "EZD1m_n1DxAF" }, + "cell_type": "markdown", "source": [ - "## Trains and tests the model" + "# Train and test the model" ] }, { - "cell_type": "code", - "execution_count": 0, "metadata": { - "colab": {}, "colab_type": "code", - "id": "qe28bAHNHUG2" + "id": "qe28bAHNHUG2", + "colab": {} }, - "outputs": [], + "cell_type": "code", "source": [ "# Creates session and initialize all variables.\n", "# xla.compile() doesn't work with Keras model.fit() API or TF eager mode yet.\n", "sess = tf.Session()\n", "sess.run(tf.global_variables_initializer())" - ] + ], + "execution_count": 0, + "outputs": [] }, { - "cell_type": "markdown", "metadata": { "colab_type": "text", "id": "qgsKmz3n2UiW" }, + "cell_type": "markdown", "source": [ - "Following code block trains model.\n", - "\n", - "Note that evaluating `y` also triggers its control dependency node `train_step`, which updates model variables." + "Following code block trains model. Evaluating `y` also triggers its control dependency node `train_step`, which updates model variables." ] }, { - "cell_type": "code", - "execution_count": 0, "metadata": { - "colab": {}, "colab_type": "code", - "id": "_GxF6jTRHVuA" + "id": "_GxF6jTRHVuA", + "colab": { + "base_uri": "https://localhost:8080/", + "height": 34 + }, + "outputId": "fbf299ca-02d5-4e95-f9fe-8f3c0432d132" }, - "outputs": [], + "cell_type": "code", "source": [ "# Feeds training dataset\n", "sess.run(iterator.make_initializer(train_ds))\n", @@ -356,18 +308,31 @@ "# Runs TRAIN_STEPS steps\n", "for i in range(TRAIN_STEPS):\n", " sess.run(y)\n", + "\n", "print(\"Model trained for %s steps.\" % TRAIN_STEPS)" + ], + "execution_count": 21, + "outputs": [ + { + "output_type": "stream", + "text": [ + "Model trained for 1000 steps.\n" + ], + "name": "stdout" + } ] }, { - "cell_type": "code", - "execution_count": 0, "metadata": { - "colab": {}, "colab_type": "code", - "id": "dHlQlRSRHXD1" + "id": "dHlQlRSRHXD1", + "colab": { + "base_uri": "https://localhost:8080/", + "height": 34 + }, + "outputId": "9c3677a2-ec84-406f-9d2c-d722844f3093" }, - "outputs": [], + "cell_type": "code", "source": [ "# Tests trained model\n", "\n", @@ -378,35 +343,31 @@ "correct_prediction = tf.equal(tf.argmax(y, 1), labels)\n", "accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))\n", "print(\"Prediction accuracy after training: %s\" % sess.run(accuracy))" + ], + "execution_count": 22, + "outputs": [ + { + "output_type": "stream", + "text": [ + "Prediction accuracy after training: 0.91\n" + ], + "name": "stdout" + } ] }, { - "cell_type": "code", - "execution_count": 0, "metadata": { - "colab": {}, "colab_type": "code", - "id": "ynJQIuzjHYOb" + "id": "ynJQIuzjHYOb", + "colab": {} }, - "outputs": [], + "cell_type": "code", "source": [ "# Cleans up session\n", "sess.close()" - ] + ], + "execution_count": 0, + "outputs": [] } - ], - "metadata": { - "colab": { - "collapsed_sections": [], - "name": "xla.compile() Tutorial", - "provenance": [], - "version": "0.3.2" - }, - "kernelspec": { - "display_name": "Python 2", - "name": "python2" - } - }, - "nbformat": 4, - "nbformat_minor": 0 -} + ] +} \ No newline at end of file diff --git a/tensorflow/compiler/xla/index_util.h b/tensorflow/compiler/xla/index_util.h index 458bdaf2f89..d76f61eb62c 100644 --- a/tensorflow/compiler/xla/index_util.h +++ b/tensorflow/compiler/xla/index_util.h @@ -21,6 +21,7 @@ limitations under the License. #include #include "absl/types/span.h" +#include "tensorflow/compiler/xla/shape.h" #include "tensorflow/compiler/xla/types.h" #include "tensorflow/compiler/xla/xla_data.pb.h" #include "tensorflow/core/platform/macros.h" diff --git a/tensorflow/compiler/xla/layout_util.cc b/tensorflow/compiler/xla/layout_util.cc index 2398470dd49..dbb81381acd 100644 --- a/tensorflow/compiler/xla/layout_util.cc +++ b/tensorflow/compiler/xla/layout_util.cc @@ -460,6 +460,13 @@ std::ostream& operator<<(std::ostream& out, const Layout& layout) { } hash_value = Hash64Combine(hash_value, layout.max_sparse_elements()); + for (Tile tile : layout.tiles()) { + for (int64 tile_dim : tile.dimensions()) { + hash_value = Hash64Combine(hash_value, hash()(tile_dim)); + } + } + hash_value = Hash64Combine(hash_value, layout.element_size_in_bits()); + return hash_value; } diff --git a/tensorflow/compiler/xla/layout_util.h b/tensorflow/compiler/xla/layout_util.h index 6e0390763da..6c298e57252 100644 --- a/tensorflow/compiler/xla/layout_util.h +++ b/tensorflow/compiler/xla/layout_util.h @@ -21,6 +21,7 @@ limitations under the License. #include #include "absl/types/span.h" +#include "tensorflow/compiler/xla/shape.h" #include "tensorflow/compiler/xla/status.h" #include "tensorflow/compiler/xla/types.h" #include "tensorflow/compiler/xla/xla_data.pb.h" diff --git a/tensorflow/compiler/xla/literal.cc b/tensorflow/compiler/xla/literal.cc index cb00a0ab16d..8f480c1f107 100644 --- a/tensorflow/compiler/xla/literal.cc +++ b/tensorflow/compiler/xla/literal.cc @@ -27,6 +27,7 @@ limitations under the License. #include "absl/strings/str_cat.h" #include "absl/strings/str_format.h" #include "absl/strings/str_join.h" +#include "absl/types/span.h" #include "tensorflow/compiler/xla/index_util.h" #include "tensorflow/compiler/xla/shape_util.h" #include "tensorflow/compiler/xla/status_macros.h" @@ -62,6 +63,14 @@ void ConvertEndianShort(char* bytes, int64 size) { } } +// Since Eigen::half doesn't satisfy the absl::bit_cast contract, we need to be +// able to transparently access the raw 16-bit value contained within. +template +T GetRawValue(T val) { + return val; +} +uint16 GetRawValue(Eigen::half val) { return val.x; } + } // namespace LiteralBase::~LiteralBase() {} @@ -283,16 +292,17 @@ Status MutableLiteralBase::CopyElementFrom(const LiteralSlice& src_literal, if (!proto.has_shape()) { return InvalidArgument("LiteralProto has no shape"); } - if (ShapeUtil::HasPrimitiveType(proto.shape(), OPAQUE)) { + Shape shape(proto.shape()); + if (ShapeUtil::HasPrimitiveType(shape, OPAQUE)) { return InvalidArgument("Literal shape cannot include OPAQUE sub-shape"); } - if (!LayoutUtil::HasLayout(proto.shape())) { + if (!LayoutUtil::HasLayout(shape)) { return InvalidArgument("LiteralProto has no layout"); } - TF_RETURN_IF_ERROR(ShapeUtil::ValidateShapeWithOptionalLayout(proto.shape())); + TF_RETURN_IF_ERROR(ShapeUtil::ValidateShapeWithOptionalLayout(shape)); - Literal literal(proto.shape()); + Literal literal(shape); TF_RETURN_IF_ERROR(literal.root_piece_->ForEachMutableSubpieceWithStatus( [&](const ShapeIndex& index, Piece* piece) { @@ -1012,166 +1022,143 @@ void LiteralBase::Piece::SortSparseElementsInternal() { namespace { +string ShapeToString(bool print_layout, const Shape& shape) { + return print_layout ? ShapeUtil::HumanStringWithLayout(shape) + : ShapeUtil::HumanString(shape); +} + +void ToStringHelper(const LiteralBase& literal, const ShapeIndex& shape_index, + bool print_layout, std::vector* pieces); + +void TupleToStringHelper(const LiteralBase& literal, + const ShapeIndex& shape_index, bool print_layout, + std::vector* pieces) { + const Shape& subshape = ShapeUtil::GetSubshape(literal.shape(), shape_index); + pieces->push_back(ShapeToString(print_layout, subshape)); + pieces->push_back(" (\n"); + std::vector tuple_pieces; + for (int i = 0; i < ShapeUtil::TupleElementCount(subshape); ++i) { + ShapeIndex element_index = shape_index; + element_index.push_back(i); + std::vector element_pieces; + ToStringHelper(literal, element_index, print_layout, &element_pieces); + tuple_pieces.push_back(absl::StrJoin(element_pieces, "")); + } + pieces->push_back(absl::StrJoin(tuple_pieces, ",\n")); + pieces->push_back("\n)"); +} + +void SparseArrayToStringHelper(const LiteralBase& literal, + const Shape& subshape, bool print_layout, + std::vector* pieces) { + pieces->push_back(ShapeToString(print_layout, subshape)); + pieces->push_back("{"); + int64 rank = ShapeUtil::Rank(subshape); + int64 num_elements = literal.sparse_element_count(); + for (int64 i = 0; i < num_elements; ++i) { + if (i > 0) { + pieces->push_back(", "); + } + if (rank == 1) { + pieces->push_back(StrCat(literal.GetSparseIndex(i)[0])); + pieces->push_back(": "); + } else { + pieces->push_back("["); + pieces->push_back(absl::StrJoin(literal.GetSparseIndex(i), ", ")); + pieces->push_back("]: "); + } + pieces->push_back(literal.GetSparseElementAsString(i)); + } + pieces->push_back("}"); +} + +void DenseArrayToStringHelper(const LiteralBase& literal, + const ShapeIndex& shape_index, bool print_layout, + std::vector* pieces) { + const Shape& subshape = ShapeUtil::GetSubshape(literal.shape(), shape_index); + int64 rank = ShapeUtil::Rank(subshape); + + std::function dimensions, std::vector*)> + to_string_recursive = [&](absl::Span dimensions, + std::vector* accum_indices) { + // dimensions.size() decreases by 1 at each recursive call, + // and accum_indices->size() increases by 1. + // Their sum is equal to the rank of the tensor. + CHECK_EQ(rank, dimensions.size() + accum_indices->size()); + + auto brace_to_string = [&](string brace) -> string { + // Handle 1D tensor + if (rank == 1) { + return brace; + } + // Handle the innermost tensor of a 2D+ tensor. + if (dimensions.size() == 1 && brace == "{") { + return StrCat(" ", brace, dimensions[0] <= 1 ? "" : " "); + } + if (dimensions.size() == 1 && brace == "}") { + return StrCat(dimensions[0] <= 1 ? "" : " ", brace); + } + // Handle the non-innermost tensors of a 2D+ tensor. + if (brace == "{") { + if (rank > 3 && !accum_indices->empty() && + accum_indices->size() < rank) { + int index = accum_indices->size() - 1; + int value = accum_indices->back(); + return StrCat(brace, " /*i", index, "=", value, "*/\n"); + } + return StrCat(brace, "\n"); + } + return StrCat("\n", brace); + }; + + if (dimensions.empty()) { + // Display predicates as 0s and 1s so that the string is more dense. + string elem; + if (subshape.element_type() == PRED && rank > 0) { + elem = literal.Get(*accum_indices, shape_index) ? "1" : "0"; + } else { + elem = literal.GetAsString(*accum_indices, shape_index); + } + pieces->push_back(elem); + } else { + pieces->push_back(brace_to_string("{")); + for (int i = 0; i < dimensions[0]; ++i) { + std::vector cloned_indices(*accum_indices); + cloned_indices.push_back(i); + to_string_recursive(dimensions.subspan(1), &cloned_indices); + if (i < dimensions[0] - 1) { + pieces->push_back(","); + pieces->push_back(dimensions.size() > 1 ? "\n" : " "); + } + } + pieces->push_back(brace_to_string("}")); + } + }; + + if (rank > 1) { + pieces->push_back(ShapeToString(print_layout, subshape)); + pieces->push_back(" "); + } + std::vector indices = {}; + std::vector dimensions(subshape.dimensions().begin(), + subshape.dimensions().end()); + to_string_recursive(dimensions, &indices); +} + void ToStringHelper(const LiteralBase& literal, const ShapeIndex& shape_index, bool print_layout, std::vector* pieces) { const Shape& subshape = ShapeUtil::GetSubshape(literal.shape(), shape_index); CHECK(LayoutUtil::HasLayout(literal.shape())); CHECK(LayoutUtil::HasLayout(subshape)); - - auto shape_to_string = [print_layout](const Shape& shape) { - if (print_layout) { - return ShapeUtil::HumanStringWithLayout(shape); - } else { - return ShapeUtil::HumanString(shape); - } - }; - - // TODO(b/32894291): refactor this code to reduce code duplication. if (ShapeUtil::IsTuple(subshape)) { - pieces->push_back(shape_to_string(subshape)); - pieces->push_back(" (\n"); - std::vector tuple_pieces; - for (int i = 0; i < ShapeUtil::TupleElementCount(subshape); ++i) { - ShapeIndex element_index = shape_index; - element_index.push_back(i); - std::vector element_pieces; - ToStringHelper(literal, element_index, print_layout, &element_pieces); - tuple_pieces.push_back(absl::StrJoin(element_pieces, "")); - } - pieces->push_back(absl::StrJoin(tuple_pieces, ",\n")); - pieces->push_back("\n)"); - return; - } - - if (ShapeUtil::IsToken(subshape)) { + TupleToStringHelper(literal, shape_index, print_layout, pieces); + } else if (ShapeUtil::IsToken(subshape)) { pieces->push_back("token"); - return; - } - - if (LayoutUtil::IsSparseArray(subshape)) { - pieces->push_back(shape_to_string(subshape)); - pieces->push_back("{"); - int64 rank = ShapeUtil::Rank(subshape); - int64 num_elements = literal.sparse_element_count(); - for (int64 i = 0; i < num_elements; ++i) { - if (i > 0) { - pieces->push_back(", "); - } - if (rank == 1) { - pieces->push_back(StrCat(literal.GetSparseIndex(i)[0])); - pieces->push_back(": "); - } else { - pieces->push_back("["); - pieces->push_back(absl::StrJoin(literal.GetSparseIndex(i), ", ")); - pieces->push_back("]: "); - } - pieces->push_back(literal.GetSparseElementAsString(i)); - } - pieces->push_back("}"); - return; - } - - CHECK(LayoutUtil::IsDenseArray(subshape)); - - auto element_to_string = [&](absl::Span indices) -> string { - PrimitiveType element_type = subshape.element_type(); - // We display predicates as 0s and 1s so that the string is more dense. - string elem = element_type == PRED - ? literal.Get(indices, shape_index) ? "1" : "0" - : literal.GetAsString(indices, shape_index); - return ((!indices.empty() && indices.back() > 0) ? ", " : "") + elem; - }; - - if (ShapeUtil::Rank(subshape) == 0) { - pieces->push_back(literal.GetAsString({}, shape_index)); - } else if (ShapeUtil::Rank(subshape) == 1) { - pieces->push_back("{"); - for (int64 i0 = 0; i0 < subshape.dimensions(0); ++i0) { - pieces->push_back(element_to_string({i0})); - } - pieces->push_back("}"); - } else if (ShapeUtil::Rank(subshape) == 2) { - pieces->push_back(shape_to_string(subshape)); - pieces->push_back(" {\n"); - for (int64 i0 = 0; i0 < subshape.dimensions(0); ++i0) { - pieces->push_back(" { "); - for (int64 i1 = 0; i1 < subshape.dimensions(1); ++i1) { - pieces->push_back(element_to_string({i0, i1})); - } - pieces->push_back(" "); - pieces->push_back(i0 == subshape.dimensions(0) - 1 ? "}\n" : "},\n"); - } - pieces->push_back("}"); - } else if (ShapeUtil::Rank(subshape) == 3) { - pieces->push_back(shape_to_string(subshape)); - pieces->push_back(" {\n"); - for (int64 i0 = 0; i0 < subshape.dimensions(0); ++i0) { - pieces->push_back(i0 > 0 ? ",\n{" : "{"); - for (int64 i1 = 0; i1 < subshape.dimensions(1); ++i1) { - pieces->push_back(i1 > 0 ? ",\n { " : " { "); - for (int64 i2 = 0; i2 < subshape.dimensions(2); ++i2) { - pieces->push_back(element_to_string({i0, i1, i2})); - } - pieces->push_back(" }"); - } - pieces->push_back(" }"); - } - pieces->push_back("\n}"); - } else if (ShapeUtil::Rank(subshape) == 4) { - pieces->push_back(shape_to_string(subshape)); - pieces->push_back(" {\n"); - for (int64 i0 = 0; i0 < subshape.dimensions(0); ++i0) { - pieces->push_back(StrFormat(" { /*i0=%d*/\n", i0)); - for (int64 i1 = 0; i1 < subshape.dimensions(1); ++i1) { - pieces->push_back(StrFormat(" { /*i1=%d*/\n", i1)); - for (int64 i2 = 0; i2 < subshape.dimensions(2); ++i2) { - pieces->push_back(" {"); - for (int64 i3 = 0; i3 < subshape.dimensions(3); ++i3) { - pieces->push_back(element_to_string({i0, i1, i2, i3})); - } - pieces->push_back(i2 == subshape.dimensions(2) - 1 ? "}\n" : "},\n"); - } - pieces->push_back(i1 == subshape.dimensions(1) - 1 ? " }\n" - : " },\n"); - } - pieces->push_back(i0 == subshape.dimensions(0) - 1 ? " }\n" : " },\n"); - } - pieces->push_back("}"); - } else if (ShapeUtil::Rank(subshape) == 5) { - pieces->push_back(shape_to_string(subshape)); - pieces->push_back(" {\n"); - for (int64 i0 = 0; i0 < subshape.dimensions(0); ++i0) { - pieces->push_back(StrFormat(" { /*i0=%d*/\n", i0)); - for (int64 i1 = 0; i1 < subshape.dimensions(1); ++i1) { - pieces->push_back(StrFormat(" { /*i1=%d*/\n", i1)); - for (int64 i2 = 0; i2 < subshape.dimensions(2); ++i2) { - pieces->push_back(StrFormat(" { /*i2=%d*/\n", i2)); - for (int64 i3 = 0; i3 < subshape.dimensions(3); ++i3) { - pieces->push_back(" {"); - for (int64 i4 = 0; i4 < subshape.dimensions(4); ++i4) { - pieces->push_back(element_to_string({i0, i1, i2, i3, i4})); - } - pieces->push_back(i3 == subshape.dimensions(3) - 1 ? "}\n" - : "},\n"); - } - pieces->push_back(i2 == subshape.dimensions(2) - 1 ? " }\n" - : " },\n"); - } - pieces->push_back(i1 == subshape.dimensions(1) - 1 ? " }\n" - : " },\n"); - } - pieces->push_back(i0 == subshape.dimensions(0) - 1 ? " }\n" : " },\n"); - } - pieces->push_back("}"); + } else if (LayoutUtil::IsSparseArray(subshape)) { + SparseArrayToStringHelper(literal, subshape, print_layout, pieces); } else { - pieces->push_back(shape_to_string(subshape)); - pieces->push_back(" {"); - literal.EachCellAsString( - [&](absl::Span indices, const string& value) { - pieces->push_back(" "); - pieces->push_back(value); - }); - pieces->push_back("}"); + CHECK(LayoutUtil::IsDenseArray(subshape)); + DenseArrayToStringHelper(literal, shape_index, print_layout, pieces); } } @@ -1228,16 +1215,32 @@ Literal ConvertBetweenNativeTypes(const LiteralBase& src_literal) { } template -typename std::enable_if<(sizeof(NativeSrcT) == sizeof(NativeDestT)), +typename std::enable_if<(sizeof(NativeSrcT) == sizeof(NativeDestT) && + !std::is_same::value), Literal>::type BitcastBetweenNativeTypes(const LiteralBase& src_literal) { auto converter = [](NativeSrcT src) { - return absl::bit_cast(src); + return absl::bit_cast(GetRawValue(src)); }; return ConvertBetweenNativeTypesWithConverter( src_literal, converter); } +template +typename std::enable_if<(sizeof(NativeSrcT) == sizeof(Eigen::half) && + std::is_same::value), + Literal>::type +BitcastBetweenNativeTypes(const LiteralBase& src_literal) { + // Eigen::half doesn't satisfy the absl::bit_cast contract, so explicitly + // cast to unsigned short and then use raw_uint16_to_half. + auto converter = [](NativeSrcT src) { + return Eigen::half_impl::raw_uint16_to_half( + absl::bit_cast(GetRawValue(src))); + }; + return ConvertBetweenNativeTypesWithConverter( + src_literal, converter); +} + // This template specialization is here to make the compiler happy. bit_cast has // a static check that the types are the same size. This specialization should // never be used because the source and destination types are checked for @@ -1792,7 +1795,7 @@ void CopyToRepeatedField(RepeatedFieldT* dest, } // namespace void LiteralBase::Piece::WriteToProto(LiteralProto* proto) const { - *proto->mutable_shape() = subshape(); + *proto->mutable_shape() = subshape().ToProto(); switch (subshape().element_type()) { case PRED: CopyToRepeatedField(proto->mutable_preds(), data()); @@ -1898,8 +1901,9 @@ Status LiteralBase::Piece::CopyFromProto(const LiteralProto& proto) { // These conditions should have been checked in // MutableLiteralBase::CreateFromProto. TF_RET_CHECK(proto.has_shape()); - TF_RET_CHECK(LayoutUtil::HasLayout(proto.shape())); - TF_RET_CHECK(ShapeUtil::Equal(proto.shape(), subshape())); + Shape shape(proto.shape()); + TF_RET_CHECK(LayoutUtil::HasLayout(shape)); + TF_RET_CHECK(ShapeUtil::Equal(shape, subshape())); if (LayoutUtil::IsSparseArray(subshape())) { // Compute the number of elements (indices) in the sparse shape and reserve diff --git a/tensorflow/compiler/xla/literal.h b/tensorflow/compiler/xla/literal.h index e791048b4d9..fa9a71af4ce 100644 --- a/tensorflow/compiler/xla/literal.h +++ b/tensorflow/compiler/xla/literal.h @@ -301,7 +301,7 @@ class LiteralBase { // // Note: It's an antipattern to use this method then immediately call // MutableLiteralBase::Populate on the result (since that results in zero - // initialization, then reinitialization. Conside if a call to + // initialization, then reinitialization. Consider if a call to // absl::make_unique(shape), followed by the call to // MutableLiteralBase::Populate can be used instead. static Literal CreateFromShape(const Shape& shape); diff --git a/tensorflow/compiler/xla/literal_test.cc b/tensorflow/compiler/xla/literal_test.cc index 8cec37897a9..49363ad802d 100644 --- a/tensorflow/compiler/xla/literal_test.cc +++ b/tensorflow/compiler/xla/literal_test.cc @@ -150,12 +150,58 @@ TEST_F(LiteralUtilTest, R3ToString) { const auto literal = LiteralUtil::CreateR3({{{1}, {2}}, {{3}, {4}}, {{5}, {6}}}); const string expected = R"(s32[3,2,1] { -{ { 1 }, - { 2 } }, -{ { 3 }, - { 4 } }, -{ { 5 }, - { 6 } } +{ + {1}, + {2} +}, +{ + {3}, + {4} +}, +{ + {5}, + {6} +} +})"; + EXPECT_EQ(expected, literal.ToString()); +} + +TEST_F(LiteralUtilTest, R6ToString) { + const auto literal = + LiteralUtil::CreateFromDimensions(S32, {2, 2, 1, 1, 1, 2}); + const string expected = R"(s32[2,2,1,1,1,2] { +{ /*i0=0*/ +{ /*i1=0*/ +{ /*i2=0*/ +{ /*i3=0*/ + { 0, 0 } +} +} +}, +{ /*i1=1*/ +{ /*i2=0*/ +{ /*i3=0*/ + { 0, 0 } +} +} +} +}, +{ /*i0=1*/ +{ /*i1=0*/ +{ /*i2=0*/ +{ /*i3=0*/ + { 0, 0 } +} +} +}, +{ /*i1=1*/ +{ /*i2=0*/ +{ /*i3=0*/ + { 0, 0 } +} +} +} +} })"; EXPECT_EQ(expected, literal.ToString()); } @@ -190,12 +236,16 @@ TEST_F(LiteralUtilTest, CreateR3FromArray3d) { EXPECT_THAT(literal.shape().dimensions(), ElementsAre(2, 3, 2)); string result = literal.ToString(); const string expected = R"(f32[2,3,2] { -{ { 1, 2 }, +{ + { 1, 2 }, { 3, 4 }, - { 5, 6 } }, -{ { 7, 8 }, + { 5, 6 } +}, +{ + { 7, 8 }, { 9, 10 }, - { 11, 12 } } + { 11, 12 } +} })"; EXPECT_EQ(expected, result); } @@ -247,18 +297,18 @@ TEST_F(LiteralUtilTest, LiteralR4F32ProjectedStringifies) { EXPECT_THAT(literal.shape().dimensions(), ElementsAre(1, 2, 3, 2)); string result = literal.ToString(); const string expected = R"(f32[1,2,3,2] { - { /*i0=0*/ - { /*i1=0*/ - {1, 2}, - {1001, 1002}, - {2001, 2002} - }, - { /*i1=1*/ - {1, 2}, - {1001, 1002}, - {2001, 2002} - } - } +{ /*i0=0*/ +{ /*i1=0*/ + { 1, 2 }, + { 1001, 1002 }, + { 2001, 2002 } +}, +{ /*i1=1*/ + { 1, 2 }, + { 1001, 1002 }, + { 2001, 2002 } +} +} })"; EXPECT_EQ(expected, result); } @@ -268,30 +318,30 @@ TEST_F(LiteralUtilTest, LiteralR4F32Stringifies) { ElementsAre(2, 2, 3, 3)); string result = literal_r4_2x2x3x3_dim0major_.ToString(); const string expected = R"(f32[2,2,3,3] { - { /*i0=0*/ - { /*i1=0*/ - {1, 2, 3}, - {4, 5, 6}, - {7, 8, 9} - }, - { /*i1=1*/ - {11, 12, 13}, - {14, 15, 16}, - {17, 18, 19} - } - }, - { /*i0=1*/ - { /*i1=0*/ - {101, 102, 103}, - {104, 105, 106}, - {107, 108, 109} - }, - { /*i1=1*/ - {201, 202, 203}, - {204, 205, 206}, - {207, 208, 209} - } - } +{ /*i0=0*/ +{ /*i1=0*/ + { 1, 2, 3 }, + { 4, 5, 6 }, + { 7, 8, 9 } +}, +{ /*i1=1*/ + { 11, 12, 13 }, + { 14, 15, 16 }, + { 17, 18, 19 } +} +}, +{ /*i0=1*/ +{ /*i1=0*/ + { 101, 102, 103 }, + { 104, 105, 106 }, + { 107, 108, 109 } +}, +{ /*i1=1*/ + { 201, 202, 203 }, + { 204, 205, 206 }, + { 207, 208, 209 } +} +} })"; EXPECT_EQ(expected, result); } @@ -1327,13 +1377,26 @@ TEST_F(LiteralUtilTest, BitcastConvertBetweenInvalidTypes) { absl::StrContains(status.error_message(), "bit widths are different")); } +// Sets the layout of the given ShapeProto to the default. +void SetDefaultLayoutOnProto(ShapeProto* shape_proto) { + CHECK(ShapeUtil::IsArrayPrimitiveType(shape_proto->element_type())); + shape_proto->mutable_layout()->set_format(DENSE); + auto* minor_to_major = + shape_proto->mutable_layout()->mutable_minor_to_major(); + minor_to_major->Resize(shape_proto->dimensions_size(), 0); + const int64 size = minor_to_major->size(); + for (int64 i = 0; i < size; ++i) { + minor_to_major->Set(i, size - 1 - i); + } +} + TEST_F(LiteralUtilTest, CopyFromProto_Bool) { LiteralProto p; p.mutable_shape()->set_element_type(PRED); for (int len = 0; len < 25; ++len) { p.mutable_shape()->clear_dimensions(); p.mutable_shape()->add_dimensions(len); - LayoutUtil::SetToDefaultLayout(p.mutable_shape()); + SetDefaultLayoutOnProto(p.mutable_shape()); p.clear_preds(); for (int i = 0; i < len; ++i) { p.add_preds((i % 2) == (len % 2)); @@ -1359,7 +1422,7 @@ TEST_F(LiteralUtilTest, ToProto_f16) { EXPECT_EQ(4, m.data().size()); LiteralProto p = m.ToProto(); - EXPECT_EQ(4, ShapeUtil::ElementsIn(p.shape())); + EXPECT_EQ(4, ShapeUtil::ElementsIn(Shape(p.shape()))); EXPECT_EQ(8, p.f16s().size()); const char* d = p.f16s().data(); EXPECT_EQ(d[0], 0); @@ -1382,7 +1445,7 @@ TEST_F(LiteralUtilTest, CopyFromProto_f16) { p.mutable_shape()->set_element_type(F16); p.mutable_shape()->clear_dimensions(); p.mutable_shape()->add_dimensions(4); - LayoutUtil::SetToDefaultLayout(p.mutable_shape()); + SetDefaultLayoutOnProto(p.mutable_shape()); p.clear_f16s(); p.set_f16s(half_vals, 8); TF_ASSERT_OK_AND_ASSIGN(Literal literal, Literal::CreateFromProto(p)); @@ -1404,7 +1467,7 @@ TEST_F(LiteralUtilTest, CopyFromProto_u16) { p.mutable_shape()->set_element_type(U16); p.mutable_shape()->clear_dimensions(); p.mutable_shape()->add_dimensions(4); - LayoutUtil::SetToDefaultLayout(p.mutable_shape()); + SetDefaultLayoutOnProto(p.mutable_shape()); p.clear_u16s(); p.set_u16s(uint16_vals, 8); TF_ASSERT_OK_AND_ASSIGN(Literal literal, Literal::CreateFromProto(p)); @@ -1537,9 +1600,9 @@ TEST_F(LiteralUtilTest, DecomposeTuple) { Literal nested_tuple = LiteralUtil::MakeTuple( {&tuple_elements[0], &tuple_elements[1], &nil_literal}); - EXPECT_FALSE(ShapeUtil::IsNil(nested_tuple.shape())); + EXPECT_FALSE(ShapeUtil::IsEmptyTuple(nested_tuple.shape())); std::vector elements = nested_tuple.DecomposeTuple(); - EXPECT_TRUE(ShapeUtil::IsNil(nested_tuple.shape())); + EXPECT_TRUE(ShapeUtil::IsEmptyTuple(nested_tuple.shape())); ASSERT_EQ(elements.size(), 3); @@ -1590,7 +1653,7 @@ TEST_F(LiteralUtilTest, MoveIntoTuple) { EXPECT_EQ(literal.Get({1}, /*shape_index=*/{2, 1}), 44.0); for (const Literal& element : elements) { - EXPECT_TRUE(ShapeUtil::IsNil(element.shape())); + EXPECT_TRUE(ShapeUtil::IsEmptyTuple(element.shape())); } } @@ -1706,7 +1769,7 @@ TEST_F(LiteralUtilTest, ProtoRoundTrip) { TEST_F(LiteralUtilTest, InvalidProtoNoValues) { // Proto contains a shape, but no values. LiteralProto proto; - *proto.mutable_shape() = ShapeUtil::MakeShape(F32, {3}); + *proto.mutable_shape() = ShapeUtil::MakeShape(F32, {3}).ToProto(); Status status = Literal::CreateFromProto(proto).status(); ASSERT_FALSE(status.ok()); EXPECT_THAT(status.error_message(), @@ -1727,7 +1790,7 @@ TEST_F(LiteralUtilTest, InvalidProtoNoShape) { TEST_F(LiteralUtilTest, InvalidProtoWrongContainer) { // Proto contains values in wrong container. LiteralProto proto; - *proto.mutable_shape() = ShapeUtil::MakeShape(F32, {3}); + *proto.mutable_shape() = ShapeUtil::MakeShape(F32, {3}).ToProto(); proto.add_preds(false); proto.add_preds(true); proto.add_preds(false); @@ -1740,7 +1803,7 @@ TEST_F(LiteralUtilTest, InvalidProtoWrongContainer) { TEST_F(LiteralUtilTest, InvalidProtoTooFewValues) { // Proto contains too few values. LiteralProto proto; - *proto.mutable_shape() = ShapeUtil::MakeShape(F32, {42, 2}); + *proto.mutable_shape() = ShapeUtil::MakeShape(F32, {42, 2}).ToProto(); proto.add_f32s(1.0); proto.add_f32s(2.0); proto.add_f32s(3.0); @@ -1753,7 +1816,7 @@ TEST_F(LiteralUtilTest, InvalidProtoTooFewValues) { TEST_F(LiteralUtilTest, InvalidProtoTooManyValues) { // Proto contains too many values. LiteralProto proto; - *proto.mutable_shape() = ShapeUtil::MakeShape(S32, {2}); + *proto.mutable_shape() = ShapeUtil::MakeShape(S32, {2}).ToProto(); proto.add_s32s(42); proto.add_s32s(-10); proto.add_s32s(100); @@ -1766,8 +1829,8 @@ TEST_F(LiteralUtilTest, InvalidProtoTooManyValues) { TEST_F(LiteralUtilTest, InvalidProtoMissingLayout) { // Proto shape missing layout. LiteralProto proto; - *proto.mutable_shape() = ShapeUtil::MakeShape(PRED, {2, 2}); - LayoutUtil::ClearLayout(proto.mutable_shape()); + *proto.mutable_shape() = ShapeUtil::MakeShape(PRED, {2, 2}).ToProto(); + proto.mutable_shape()->clear_layout(); proto.add_preds(true); proto.add_preds(false); proto.add_preds(true); @@ -1780,11 +1843,13 @@ TEST_F(LiteralUtilTest, InvalidProtoMissingLayout) { TEST_F(LiteralUtilTest, InvalidProtoTooFewTupleElements) { // Proto has the too few tuple elements. LiteralProto proto; - *proto.mutable_shape() = ShapeUtil::MakeTupleShape( - {ShapeUtil::MakeShape(PRED, {2}), ShapeUtil::MakeShape(F32, {})}); + *proto.mutable_shape() = + ShapeUtil::MakeTupleShape( + {ShapeUtil::MakeShape(PRED, {2}), ShapeUtil::MakeShape(F32, {})}) + .ToProto(); LiteralProto* element0 = proto.add_tuple_literals(); *element0->mutable_shape() = - ShapeUtil::GetTupleElementShape(proto.shape(), 0); + ShapeUtil::GetTupleElementShape(Shape(proto.shape()), 0).ToProto(); element0->add_preds(false); element0->add_preds(true); @@ -1796,19 +1861,21 @@ TEST_F(LiteralUtilTest, InvalidProtoTooFewTupleElements) { TEST_F(LiteralUtilTest, InvalidProtoTooManyTupleElements) { // Proto has the too many tuple elements. LiteralProto proto; - *proto.mutable_shape() = ShapeUtil::MakeTupleShape( - {ShapeUtil::MakeShape(PRED, {2}), ShapeUtil::MakeShape(F32, {})}); + *proto.mutable_shape() = + ShapeUtil::MakeTupleShape( + {ShapeUtil::MakeShape(PRED, {2}), ShapeUtil::MakeShape(F32, {})}) + .ToProto(); LiteralProto* element0 = proto.add_tuple_literals(); *element0->mutable_shape() = - ShapeUtil::GetTupleElementShape(proto.shape(), 0); + ShapeUtil::GetTupleElementShape(Shape(proto.shape()), 0).ToProto(); element0->add_preds(false); element0->add_preds(true); LiteralProto* element1 = proto.add_tuple_literals(); *element1->mutable_shape() = - ShapeUtil::GetTupleElementShape(proto.shape(), 1); + ShapeUtil::GetTupleElementShape(Shape(proto.shape()), 1).ToProto(); element1->add_f32s(42.0); LiteralProto* element2 = proto.add_tuple_literals(); - *element2->mutable_shape() = ShapeUtil::MakeShape(F32, {}); + *element2->mutable_shape() = ShapeUtil::MakeShape(F32, {}).ToProto(); element2->add_f32s(123.0); Status status = Literal::CreateFromProto(proto).status(); diff --git a/tensorflow/compiler/xla/parse_flags_from_env.cc b/tensorflow/compiler/xla/parse_flags_from_env.cc index 40481331b69..5b568888d14 100644 --- a/tensorflow/compiler/xla/parse_flags_from_env.cc +++ b/tensorflow/compiler/xla/parse_flags_from_env.cc @@ -13,15 +13,20 @@ See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ -// This module exports ParseFlagsFromEnv(), which allows other modules to parse -// flags from an environtment variable, or a file named by the environment -// variable. +// This module exports ParseFlagsFromEnvAndDieIfUnknown(), which allows other +// modules to parse flags from an environtment variable, or a file named by the +// environment variable. #include #include #include +#include +#include #include +#include "absl/strings/str_format.h" +#include "absl/strings/str_join.h" +#include "absl/types/span.h" #include "tensorflow/compiler/xla/parse_flags_from_env.h" #include "tensorflow/compiler/xla/types.h" #include "tensorflow/core/platform/logging.h" @@ -32,7 +37,6 @@ limitations under the License. namespace xla { -static const char kEnvVar[] = "TF_XLA_FLAGS"; // environment variable queried static const char kWS[] = " \t\r\n"; // whitespace // The following struct represents an argv[]-style array, parsed @@ -42,12 +46,20 @@ static const char kWS[] = " \t\r\n"; // whitespace // constructor/destructor collisions with other "private" types // in the same named namespace. namespace { + +// Functor which deletes objects by calling `free`. Necessary to free strdup'ed +// strings created by AppendToEnvArgv. +struct FreeDeleter { + void operator()(char* ptr) { free(ptr); } +}; + struct EnvArgv { EnvArgv() : initialized(false), argc(0) {} bool initialized; // whether the other fields have been set. int argc; // elements used in argv[] std::vector argv; // flag arguments parsed from environment string. - std::vector argv_save; // saved values from argv[] to avoid leaks + // saved values from argv[] to avoid leaks + std::vector> argv_save; }; } // anonymous namespace @@ -63,7 +75,7 @@ static void AppendToEnvArgv(const char* s0, size_t s0len, const char* s1, string s = string(s0, s0len) + string(s1, s1len); char* str = strdup(s.c_str()); a->argv.push_back(str); - a->argv_save.push_back(str); + a->argv_save.emplace_back(str); a->argc++; } } @@ -127,14 +139,14 @@ static void ParseArgvFromString(const string& flag_str, EnvArgv* a) { } } -// Call ParseArgvFromString(..., a) on a string derived from the setting of an -// environment variable kEnvVar, or a file it points to. -static void SetArgvFromEnv(EnvArgv* a) { +// Call ParseArgvFromString(..., a) on a string derived from the setting of the +// environment variable `envvar`, or a file it points to. +static void SetArgvFromEnv(absl::string_view envvar, EnvArgv* a) { if (!a->initialized) { static const char kDummyArgv[] = ""; AppendToEnvArgv(kDummyArgv, strlen(kDummyArgv), nullptr, 0, a); // dummy argv[0] - const char* env = getenv(kEnvVar); + const char* env = getenv(string(envvar).c_str()); if (env == nullptr || env[0] == '\0') { // nothing } else if (env[strspn(env, kWS)] == '-') { // flags in env var value @@ -157,48 +169,66 @@ static void SetArgvFromEnv(EnvArgv* a) { } } -// The simulated argv[] parsed from the environment. -static EnvArgv* env_argv; +// The simulated argv[] parsed from the environment, one for each different +// environment variable we've seen. +static std::unordered_map& EnvArgvs() { + static auto* env_argvs = new std::unordered_map(); + return *env_argvs; +} -// Used to protect accesses to env_argv. +// Used to protect accesses to env_argvs. static tensorflow::mutex env_argv_mu(tensorflow::LINKER_INITIALIZED); -// Call Flags::Parse(argc, argv, flag_list) against any as yet unrecognized -// flags passed in from the environment. -bool ParseFlagsFromEnv(const std::vector& flag_list) { - env_argv_mu.lock(); - if (env_argv == nullptr) { - env_argv = new EnvArgv; - } - SetArgvFromEnv(env_argv); // a no-op if already initialized +bool ParseFlagsFromEnvAndDieIfUnknown( + absl::string_view envvar, const std::vector& flag_list) { + tensorflow::mutex_lock lock(env_argv_mu); + auto* env_argv = &EnvArgvs()[string(envvar)]; + SetArgvFromEnv(envvar, env_argv); // a no-op if already initialized bool result = tensorflow::Flags::Parse(&env_argv->argc, &env_argv->argv[0], flag_list); - env_argv_mu.unlock(); + + // There's always at least one unparsed argc, namely the fake argv[0]. + if (result && env_argv->argc != 1) { + // Skip the first argv, which is the fake argv[0]. + auto unknown_flags = absl::MakeSpan(env_argv->argv); + unknown_flags.remove_prefix(1); + + // Some flags are set on XLA_FLAGS, others on TF_XLA_FLAGS. If we find an + // unrecognized flag, suggest the alternative. + string alternate_envvar; + if (envvar == "TF_XLA_FLAGS") { + alternate_envvar = "XLA_FLAGS"; + } else if (envvar == "XLA_FLAGS") { + alternate_envvar = "TF_XLA_FLAGS"; + } + string did_you_mean; + if (!alternate_envvar.empty()) { + did_you_mean = absl::StrFormat( + "\nPerhaps you meant to specify these on the %s envvar?", + alternate_envvar); + } + + LOG(FATAL) << "Unknown flag" << (unknown_flags.size() > 1 ? "s" : "") + << " in " << envvar << ": " << absl::StrJoin(unknown_flags, " ") + << did_you_mean; + return false; + } return result; } // Testing only. -// Reset the env_argv struct so that subsequent calls to ParseFlagsFromEnv() -// will parse the environment variable (or the file it points to) anew, and set -// *pargc, and *pargv to point to the internal locations of the argc and argv -// constructed from the environment. -void ResetFlagsFromEnvForTesting(int** pargc, std::vector** pargv) { - env_argv_mu.lock(); - if (env_argv == nullptr) { - env_argv = new EnvArgv; - } - if (!env_argv->argv_save.empty()) { - for (int i = 0; env_argv->argv_save[i] != nullptr; i++) { - free(env_argv->argv_save[i]); - } - } - env_argv->initialized = false; - env_argv->argc = 0; - env_argv->argv.clear(); - env_argv->argv_save.clear(); - env_argv_mu.unlock(); - *pargc = &env_argv->argc; - *pargv = &env_argv->argv; +// +// Resets the env_argv struct so that subsequent calls to +// ParseFlagsFromEnvAndDieIfUnknown() will parse the environment variable (or +// the file it points to) anew, and set *pargc, and *pargv to point to the +// internal locations of the argc and argv constructed from the environment. +void ResetFlagsFromEnvForTesting(absl::string_view envvar, int** pargc, + std::vector** pargv) { + tensorflow::mutex_lock lock(env_argv_mu); + EnvArgvs().erase(string(envvar)); + auto& env_argv = EnvArgvs()[string(envvar)]; + *pargc = &env_argv.argc; + *pargv = &env_argv.argv; } } // namespace xla diff --git a/tensorflow/compiler/xla/parse_flags_from_env.h b/tensorflow/compiler/xla/parse_flags_from_env.h index fe86ee687f8..76940a4299a 100644 --- a/tensorflow/compiler/xla/parse_flags_from_env.h +++ b/tensorflow/compiler/xla/parse_flags_from_env.h @@ -16,48 +16,58 @@ limitations under the License. #ifndef TENSORFLOW_COMPILER_XLA_PARSE_FLAGS_FROM_ENV_H_ #define TENSORFLOW_COMPILER_XLA_PARSE_FLAGS_FROM_ENV_H_ -// This module exports ParseFlagsFromEnv(), which allows other modules to parse -// flags from the environtment variable TF_XLA_FLAGS, or (if the first +// This module exports ParseFlagsFromEnvAndDieIfUnknown(), which allows other +// modules to parse flags from an environtment variable, or (if the first // non-whitespace in the variable value is not '-'), a file named by that -// environment variable. The accepted syntax is that flags arguments are of -// the form --flag=value or (for boolean flags) --flag, and are whitespace -// separated. The may be one of: -// - -// in which case the effective value is the string itself -// - in which case the effective value is the -// string with the single-quotes removed -// - in which case the effective value if the -// string with the double-quotes removed, and escaped sequences of -// replaced by . +// environment variable. +// +// The accepted syntax is that flags arguments are of the form --flag=value or +// (for boolean flags) --flag, and are whitespace separated. The may be +// one of: +// +// - +// in which case the effective value is the string itself +// - in which case the effective value is the +// string with the single-quotes removed +// - in which case the effective value if the +// string with the double-quotes removed, and escaped sequences of +// replaced by . // // Flags values inconsistent with the type of the flag will be rejected by the // flag parser. // // Examples: -// TF_XLA_FLAGS="--foo=bar --wombat='value with a space'" // -// TF_XLA_FLAGS=/tmp/flagfile +// - TF_XLA_FLAGS="--foo=bar --wombat='value with a space'" +// - TF_XLA_FLAGS=/tmp/flagfile +// // where /tmp/flagfile might contain -// --some_flag="This is a string containing a \" and a '." -// --another_flag=wombats +// +// --some_flag="This is a string containing a \" and a '." +// --another_flag=wombats #include +#include "absl/strings/string_view.h" #include "tensorflow/compiler/xla/types.h" #include "tensorflow/core/platform/types.h" #include "tensorflow/core/util/command_line_flags.h" namespace xla { -// Call tensorflow::Flags::Parse(argc, argv, flag_list) against any as yet -// unrecognized flags passed in from the environment, and return its -// return value. -bool ParseFlagsFromEnv(const std::vector& flag_list); +// Calls tensorflow::Flags::Parse(argc, argv, flag_list) against any as yet +// unrecognized flags passed in the environment variable `envvar`, and returns +// its return value. +// +// Raises a fatal error if any flags in `envvar` were not recognized. +bool ParseFlagsFromEnvAndDieIfUnknown( + absl::string_view envvar, const std::vector& flag_list); // Used only for testing. Not to be used by clients. -void ResetFlagsFromEnvForTesting(int** pargc, std::vector** pargv); +void ResetFlagsFromEnvForTesting(absl::string_view envvar, int** pargc, + std::vector** pargv); } // namespace xla diff --git a/tensorflow/compiler/xla/parse_flags_from_env_test.cc b/tensorflow/compiler/xla/parse_flags_from_env_test.cc index edd6538402d..3465552ebbf 100644 --- a/tensorflow/compiler/xla/parse_flags_from_env_test.cc +++ b/tensorflow/compiler/xla/parse_flags_from_env_test.cc @@ -37,20 +37,7 @@ static void TestParseFlagsFromEnv(const char* msg) { // Initialize module under test. int* pargc; std::vector* pargv; - ResetFlagsFromEnvForTesting(&pargc, &pargv); - - // Ensure that environment variable can be parsed when - // no flags are expected. - std::vector empty_flag_list; - bool parsed_ok = ParseFlagsFromEnv(empty_flag_list); - CHECK(parsed_ok) << msg; - const std::vector& argv_first = *pargv; - CHECK_NE(argv_first[0], nullptr) << msg; - int i = 0; - while (argv_first[i] != nullptr) { - i++; - } - CHECK_EQ(i, *pargc) << msg; + ResetFlagsFromEnvForTesting("TF_XLA_FLAGS", &pargc, &pargv); // Check that actual flags can be parsed. bool simple = false; @@ -65,7 +52,7 @@ static void TestParseFlagsFromEnv(const char* msg) { tensorflow::Flag("single_quoted", &single_quoted, ""), tensorflow::Flag("double_quoted", &double_quoted, ""), }; - parsed_ok = ParseFlagsFromEnv(flag_list); + bool parsed_ok = ParseFlagsFromEnvAndDieIfUnknown("TF_XLA_FLAGS", flag_list); CHECK_EQ(*pargc, 1) << msg; const std::vector& argv_second = *pargv; CHECK_NE(argv_second[0], nullptr) << msg; @@ -171,7 +158,8 @@ int main(int argc, char* argv[]) { tensorflow::Flag("int_flag", &int_flag, "An integer flag to test with"), }; xla::string usage = tensorflow::Flags::Usage(argv[0], flag_list); - bool parse_ok = xla::ParseFlagsFromEnv(flag_list); + bool parse_ok = + xla::ParseFlagsFromEnvAndDieIfUnknown("TF_XLA_FLAGS", flag_list); if (!parse_ok) { LOG(QFATAL) << "can't parse from environment\n" << usage; } diff --git a/tensorflow/compiler/xla/protobuf_util.cc b/tensorflow/compiler/xla/protobuf_util.cc index b507a2ef79f..ac342bf40fb 100644 --- a/tensorflow/compiler/xla/protobuf_util.cc +++ b/tensorflow/compiler/xla/protobuf_util.cc @@ -40,16 +40,6 @@ bool ProtobufEquals(const tensorflow::protobuf::Message& m1, namespace { -string SanitizeFilename(const string& file_name) { - string safe_file_name = file_name; - for (char& c : safe_file_name) { - if (c == '/' || c == '\\') { - c = '_'; - } - } - return safe_file_name; -} - std::pair>*> GetDirectoryExpanders() { static auto* mutex = new tensorflow::mutex; diff --git a/tensorflow/compiler/xla/python/local_computation_builder.cc b/tensorflow/compiler/xla/python/local_computation_builder.cc index 4d2a37cfac3..6e2ee866321 100644 --- a/tensorflow/compiler/xla/python/local_computation_builder.cc +++ b/tensorflow/compiler/xla/python/local_computation_builder.cc @@ -148,14 +148,19 @@ static StatusOr ToBuffer(LocalClient* client, /* static */ StatusOr LocalShapedBuffer::FromLiteral( - const Literal& argument, const absl::optional& shape_with_layout) { + const Literal& argument, const absl::optional& shape_with_layout, + int replica_number) { LocalClient* client = GetOrCreateLocalClient(); + TF_ASSIGN_OR_RETURN(int device_ordinal, + client->ReplicaNumberToDeviceOrdinal(replica_number)); + VLOG(1) << "Creating shaped buffer from literal on replica/ordinal: " + << replica_number << "/" << device_ordinal; StatusOr buf = [&] { if (shape_with_layout) { Literal relaid = argument.Relayout(shape_with_layout.value()); - return ToBuffer(client, /*device_ordinal=*/0, relaid); + return ToBuffer(client, device_ordinal, relaid); } - return ToBuffer(client, /*device_ordinal=*/0, argument); + return ToBuffer(client, device_ordinal, argument); }(); TF_RETURN_IF_ERROR(buf.status()); return new LocalShapedBuffer(std::move(buf).ValueOrDie()); @@ -312,67 +317,127 @@ CompiledLocalComputation::CompiledLocalComputation( StatusOr CompiledLocalComputation::Execute( absl::Span argument_handles) { LocalClient* client = GetOrCreateLocalClient(); + StatusOr device_ordinal_status = client->ReplicaNumberToDeviceOrdinal(0); + StatusOr result_buffer_status; + if (!device_ordinal_status.ok()) { + result_buffer_status = device_ordinal_status.status(); + } else { + const int device_ordinal = device_ordinal_status.ValueOrDie(); + VLOG(3) << "Replica 0 mapped to device ordinal for execution: " + << device_ordinal; - VLOG(1) << "Execution requested with " << GetReplicaCount() << " replicas."; + std::vector argument_buffers; + argument_buffers.reserve(argument_handles.size()); + for (auto& handle : argument_handles) { + argument_buffers.push_back(handle->shaped_buffer()); + } + + DeviceAssignment device_assignment = + client->backend() + .computation_placer() + ->AssignDevices(1, /*computation_count=*/1) + .ConsumeValueOrDie(); + + ExecutableRunOptions options; + options.set_device_ordinal(device_ordinal); + options.set_allocator(client->backend().memory_allocator()); + options.set_intra_op_thread_pool( + client->backend().eigen_intra_op_thread_pool_device()); + options.set_device_assignment(&device_assignment); + + result_buffer_status = executable_->Run(argument_buffers, options); + } + + if (!result_buffer_status.ok()) { + return InternalError( + "Failed running replica 0 (other replicas may have failed as well): " + "%s.", + result_buffer_status.status().ToString()); + } + return new LocalShapedBuffer(std::move(result_buffer_status).ValueOrDie()); +} + +StatusOr CompiledLocalComputation::ExecutePerReplica( + absl::Span> argument_handles) { + LocalClient* client = GetOrCreateLocalClient(); + const int num_replicas = GetReplicaCount(); + + if (argument_handles.size() != num_replicas) { + return InvalidArgument( + "Attempted to execute with %d replicas when replica count is %d", + argument_handles.size(), num_replicas); + } + + VLOG(1) << "Executing with " << num_replicas << " replicas."; // Each replica populates a StatusOr result, but only the output value of // replica zero is returned. - std::vector> results(GetReplicaCount()); - { - tensorflow::thread::ThreadPool pool(tensorflow::Env::Default(), "xlarun", - GetReplicaCount()); - - for (int replica = 0; replica < GetReplicaCount(); ++replica) { - pool.Schedule( - [this, client, replica, &argument_handles, &results] { - StatusOr device_ordinal_status = - client->ReplicaNumberToDeviceOrdinal(replica); - if (!device_ordinal_status.ok()) { - results[replica] = device_ordinal_status.status(); - return; - } - const int device_ordinal = device_ordinal_status.ValueOrDie(); - VLOG(3) << "Replica " << replica - << " mapped to device ordinal for execution: " - << device_ordinal; - - std::vector argument_buffers; - argument_buffers.reserve(argument_handles.size()); - for (auto& handle : argument_handles) { - argument_buffers.push_back(handle->shaped_buffer()); - } - - DeviceAssignment device_assignment = - client->backend() - .computation_placer() - ->AssignDevices(GetReplicaCount(), /*computation_count=*/1) - .ConsumeValueOrDie(); - - ExecutableRunOptions options; - options.set_device_ordinal(device_ordinal); - options.set_allocator(client->backend().memory_allocator()); - options.set_intra_op_thread_pool( - client->backend().eigen_intra_op_thread_pool_device()); - options.set_device_assignment(&device_assignment); - StatusOr result_buffer_status = - executable_->Run(argument_buffers, options); - - results[replica] = std::move(result_buffer_status); - }); + std::vector> results(num_replicas); + auto execute = [this, client, num_replicas, &argument_handles, + &results](int replica) { + StatusOr device_ordinal_status = + client->ReplicaNumberToDeviceOrdinal(replica); + if (!device_ordinal_status.ok()) { + results[replica] = device_ordinal_status.status(); + return; } + const int device_ordinal = device_ordinal_status.ValueOrDie(); + VLOG(3) << "Replica " << replica + << " mapped to device ordinal for execution: " << device_ordinal; + + std::vector argument_buffers; + argument_buffers.reserve(argument_handles[replica].size()); + for (auto& handle : argument_handles[replica]) { + argument_buffers.push_back(handle->shaped_buffer()); + } + + DeviceAssignment device_assignment = + client->backend() + .computation_placer() + ->AssignDevices(num_replicas, /*computation_count=*/1) + .ConsumeValueOrDie(); + + ExecutableRunOptions options; + options.set_device_ordinal(device_ordinal); + options.set_allocator(client->backend().memory_allocator()); + options.set_intra_op_thread_pool( + client->backend().eigen_intra_op_thread_pool_device()); + options.set_device_assignment(&device_assignment); + StatusOr result_buffer_status = + executable_->Run(argument_buffers, options); + + results[replica] = std::move(result_buffer_status); + }; + + if (num_replicas == 1) { + // Fast-path if there is only one replica — run the computation on the + // current thread. + execute(0); + } else { + // TODO(phawkins): don't recreate the threadpool for each execution. + tensorflow::thread::ThreadPool pool(tensorflow::Env::Default(), "xlarun", + num_replicas - 1); + + for (int replica = 0; replica < num_replicas - 1; ++replica) { + pool.Schedule([&execute, replica] { execute(replica); }); + } + execute(num_replicas - 1); } - for (int replica = 0; replica < GetReplicaCount(); ++replica) { - const auto& statusor = results[replica]; + std::vector wrapped_results(num_replicas); + for (int replica = 0; replica < num_replicas; ++replica) { + auto& statusor = results[replica]; if (!statusor.ok()) { return InternalError( "Failed running replica %d (other replicas may have failed as well): " "%s.", replica, statusor.status().ToString()); } + wrapped_results[replica] = + new LocalShapedBuffer(std::move(statusor).ValueOrDie()); } - return new LocalShapedBuffer(std::move(results[0]).ValueOrDie()); + return new LocalShapedBufferTuple(std::move(wrapped_results)); } static StatusOr GetReturnValueShape(const XlaComputation& computation) { @@ -487,12 +552,13 @@ StatusOr LocalComputation::CompileForXrt( xrt::XLAComputation c; auto config = c.mutable_config(); - auto shapes = config->mutable_program_shape(); + ProgramShape shapes; for (auto& shape : argument_shapes) { - *shapes->add_parameters() = shape; + *shapes.add_parameters() = shape; } - TF_ASSIGN_OR_RETURN(*shapes->mutable_result(), GetReturnValueShape()); - LayoutUtil::SetToDefaultLayout(shapes); + TF_ASSIGN_OR_RETURN(*shapes.mutable_result(), GetReturnValueShape()); + LayoutUtil::SetToDefaultLayout(&shapes); + *config->mutable_program_shape() = shapes.ToProto(); auto snapshot = computation().Snapshot().ValueOrDie(); *c.mutable_hlo_snapshot() = *snapshot; @@ -584,9 +650,9 @@ LocalOp LocalComputationBuilder::Broadcast( } LocalOp LocalComputationBuilder::BroadcastInDim( - const LocalOp& operand, const Shape& shape, + const LocalOp& operand, absl::Span out_dim_sizes, absl::Span broadcast_dimensions) { - return xla::BroadcastInDim(operand.op(), shape, broadcast_dimensions); + return xla::BroadcastInDim(operand.op(), out_dim_sizes, broadcast_dimensions); } LocalOp LocalComputationBuilder::Pad(const LocalOp& operand, diff --git a/tensorflow/compiler/xla/python/local_computation_builder.h b/tensorflow/compiler/xla/python/local_computation_builder.h index 9e617c48bdc..149e44570df 100644 --- a/tensorflow/compiler/xla/python/local_computation_builder.h +++ b/tensorflow/compiler/xla/python/local_computation_builder.h @@ -71,7 +71,8 @@ StatusOr TransferFromOutfeedLocalReplica(const Shape& shape, class LocalShapedBuffer { public: static StatusOr FromLiteral( - const Literal& argument, const absl::optional& shape_with_layout); + const Literal& argument, const absl::optional& shape_with_layout, + int replica_number); LocalShapedBuffer(ScopedShapedBuffer shaped_buffer); StatusOr ToLiteral() const; @@ -175,6 +176,12 @@ class CompiledLocalComputation { StatusOr Execute( absl::Span argument_handles); + // Execute on many replicas. Takes a sequence of argument lists (one argument + // list per replica) and returns a tuple of results (one result per replica). + // The number of argument lists must be equal to the replica count. + StatusOr ExecutePerReplica( + absl::Span > argument_handles); + private: std::unique_ptr executable_; }; @@ -282,7 +289,8 @@ class LocalComputationBuilder { LocalOp Broadcast(const LocalOp& operand, absl::Span broadcast_sizes); - LocalOp BroadcastInDim(const LocalOp& operand, const Shape& shape, + LocalOp BroadcastInDim(const LocalOp& operand, + absl::Span out_dim_sizes, absl::Span broadcast_dimensions); LocalOp Pad(const LocalOp& operand, const LocalOp& padding_value, diff --git a/tensorflow/compiler/xla/python/local_computation_builder.i b/tensorflow/compiler/xla/python/local_computation_builder.i index feabfdb889c..d23d693c1e5 100644 --- a/tensorflow/compiler/xla/python/local_computation_builder.i +++ b/tensorflow/compiler/xla/python/local_computation_builder.i @@ -363,6 +363,37 @@ tensorflow::ImportNumpy(); $1 = temps; } +%typemap(in) absl::Span > + (std::vector > temps) { + if (!PySequence_Check($input)) { + PyErr_SetString(PyExc_TypeError, "Argument is not a sequence"); + SWIG_fail; + } + const int size = PySequence_Size($input); + temps.reserve(size); + for (int i = 0; i < size; ++i) { + PyObject* o = PySequence_GetItem($input, i); + std::vector vec; + const int vec_size = PySequence_Size(o); + vec.reserve(vec_size); + for (int j = 0; j < vec_size; ++j) { + PyObject* vec_elt = PySequence_GetItem(o, j); + LocalShapedBuffer* lsbp; + if ((SWIG_ConvertPtr(vec_elt, (void**) &lsbp, $descriptor(xla::swig::LocalShapedBuffer*), + SWIG_POINTER_EXCEPTION)) == -1) { + Py_DECREF(vec_elt); + Py_DECREF(o); + SWIG_fail; + } + vec.push_back(lsbp); + Py_DECREF(vec_elt); + } + temps.push_back(vec); + Py_DECREF(o); + } + $1 = temps; +} + %typemap(in) absl::Span (std::vector temps) { if (!PySequence_Check($input)) { @@ -921,22 +952,22 @@ tensorflow::ImportNumpy(); $1 = NULL; } else { if (!HandleStringAttribute($input, "generate_hlo_graph", [&](string s) { - build_options.set_generate_hlo_graph(std::move(s)); + build_options.mutable_debug_options()->set_xla_generate_hlo_graph(std::move(s)); })) { return nullptr; } if (!HandleStringAttribute($input, "dump_optimized_hlo_proto_to", [&](string s) { - build_options.set_dump_optimized_hlo_proto_to(std::move(s)); + build_options.mutable_debug_options()->set_xla_dump_optimized_hlo_proto_to(std::move(s)); })) { return nullptr; } if (!HandleStringAttribute($input, "dump_unoptimized_hlo_proto_to", [&](string s) { - build_options.set_dump_unoptimized_hlo_proto_to(std::move(s)); + build_options.mutable_debug_options()->set_xla_dump_unoptimized_hlo_proto_to(std::move(s)); })) { return nullptr; } if (!HandleStringAttribute($input, "dump_per_pass_hlo_proto_to", [&](string s) { - build_options.set_dump_per_pass_hlo_proto_to(std::move(s)); + build_options.mutable_debug_options()->set_xla_dump_per_pass_hlo_proto_to(std::move(s)); })) { return nullptr; } @@ -950,7 +981,7 @@ tensorflow::ImportNumpy(); PyErr_SetString(PyExc_TypeError, "ExecutableBuildOptions.hlo_profile must be a bool or None."); SWIG_fail; } - build_options.set_hlo_profile(o == Py_True); + build_options.mutable_debug_options()->set_xla_hlo_profile(o == Py_True); } Py_DECREF(o); @@ -992,11 +1023,13 @@ tensorflow::ImportNumpy(); %unignore xla::swig::XrtAllocation; %unignore xla::swig::XrtAllocation::FromLiteral; %unignore xla::swig::XrtAllocation::ToLiteral; +%unignore xla::swig::XrtAllocation::shape; %unignore xla::swig::XrtAllocationTuple; %unignore xla::swig::XrtAllocationTuple::Release; %unignore xla::swig::XrtAllocationTuple::size; %unignore xla::swig::CompiledLocalComputation; %unignore xla::swig::CompiledLocalComputation::Execute; +%unignore xla::swig::CompiledLocalComputation::ExecutePerReplica; %unignore xla::swig::CompiledXrtComputation; %unignore xla::swig::CompiledXrtComputation::Execute; %unignore xla::swig::LocalComputation; diff --git a/tensorflow/compiler/xla/python/xla_client.py b/tensorflow/compiler/xla/python/xla_client.py index 92b0685dbba..c91a2aaf56d 100644 --- a/tensorflow/compiler/xla/python/xla_client.py +++ b/tensorflow/compiler/xla/python/xla_client.py @@ -26,6 +26,9 @@ import os import numpy as np +import six +from six.moves import xrange + from tensorflow.compiler.xla import xla_data_pb2 from tensorflow.compiler.xla.python import pywrap_xla as c_api from tensorflow.compiler.xla.service import hlo_pb2 @@ -75,6 +78,13 @@ def CurrentSourceInfoMetadata(op_type=None, op_name=None, skip_frames=1): source_line=lineno) +def _maybe_encode_string(s): + if six.PY3: + return s.encode('utf-8') + else: + return s + + class PaddingType(enum.Enum): VALID = 1 SAME = 2 @@ -212,23 +222,33 @@ class LocalBuffer(object): means the referent is in device memory. """ - def __init__(self, c_buffer, backend): + def __init__(self, c_buffer, backend, replica): self.c_buffer = c_buffer self._backend = backend + self._replica = replica if backend.backend_type == BackendType.XRT: self._delete = c_api.DeleteXrtAllocation else: self._delete = c_api.DeleteLocalShapedBuffer @staticmethod - def from_pyval(pyval, backend=XLA_LOCAL_BACKEND): + def from_pyval(pyval, replica=0, backend=XLA_LOCAL_BACKEND): """Allocate and copy to XLA the given python value.""" pyval = require_numpy_array_layout(pyval) + num_replicas = get_replica_count() + if not 0 <= replica < num_replicas: + raise ValueError( + 'Attempt to place buffer on replica {} when the replica count is {}' + .format(replica, num_replicas)) if backend.backend_type == BackendType.XRT: - cbuf = c_api.XrtAllocation.FromLiteral(pyval, backend.target) + if replica != 0: + raise NotImplementedError( + 'Multi-replica execution is not yet supported via the XRT backend.') + cbuf = c_api.XrtAllocation.FromLiteral( + pyval, _maybe_encode_string(backend.target)) else: - cbuf = c_api.LocalShapedBuffer.FromLiteral(pyval, None) - return LocalBuffer(cbuf, backend) + cbuf = c_api.LocalShapedBuffer.FromLiteral(pyval, None, replica) + return LocalBuffer(cbuf, backend, replica) def to_py(self): return self.c_buffer.ToLiteral() @@ -236,6 +256,9 @@ class LocalBuffer(object): def shape(self): return _wrap_shape(self.c_buffer.shape()) + def replica(self): + return self._replica + def delete(self): if self.c_buffer is not None: self._delete(self.c_buffer) @@ -245,14 +268,15 @@ class LocalBuffer(object): """Assuming a tuple buffer, unpack it into constituent tuple elements.""" assert self.c_buffer is not None if self._backend.backend_type == BackendType.XRT: - result = c_api.DestructureXrtAllocationTuple(self.c_buffer, - self._backend.target) + result = c_api.DestructureXrtAllocationTuple( + self.c_buffer, _maybe_encode_string(self._backend.target)) else: result = c_api.DestructureLocalShapedBufferTuple(self.c_buffer) self.delete() size = result.size() destructured = tuple( - LocalBuffer(result.Release(i), backend=self._backend) + LocalBuffer( + result.Release(i), replica=self._replica, backend=self._backend) for i in xrange(size)) return destructured @@ -322,6 +346,9 @@ class Shape(object): def __ne__(self, other): return not self == other + def __hash__(self): + return hash((self._dtype, self._dimensions, self._minor_to_major)) + def __repr__(self): return ('xla_client.Shape(_dtype={!r}, _dimensions={!r}, ' '_is_tuple={!r}, _minor_to_major={!r})').format( @@ -541,10 +568,13 @@ class LocalComputation(object): ] result_shape = result_shape.map_leaves(layout_fn) + argument_shapes = list(argument_shapes) + compile_options = compile_options or CompileOptions() compile_options.result_shape = result_shape if self._backend.backend_type == BackendType.XRT: - c = self.computation.CompileForXrt(argument_shapes, self._backend.target) + c = self.computation.CompileForXrt( + argument_shapes, _maybe_encode_string(self._backend.target)) else: c = self.computation.Compile(argument_shapes, compile_options) return LocalComputation(c, is_compiled=True, backend=self._backend) @@ -558,23 +588,87 @@ class LocalComputation(object): compile_options=compile_options, layout_fn=layout_fn) - def Execute(self, arguments=()): - """Execute with LocalBuffer arguments and return value.""" + def GetReturnValueShape(self): + return _wrap_shape(self._c_computation.GetReturnValueShape()) + + def Execute(self, arguments=(), check_for_deleted_args=True): + """Execute on one replica with LocalBuffer arguments and return value.""" + if check_for_deleted_args and any(arg.is_deleted() for arg in arguments): + raise ValueError('Executing with deleted local buffer argument') + raw_args = [arg.c_buffer for arg in arguments] + output_buffer = self._c_computation.Execute(raw_args) + return LocalBuffer(output_buffer, backend=self._backend, replica=0) + + def ExecutePerReplica(self, arguments=None): + """Execute on many replicas with LocalBuffer arguments and return value. + + Args: + arguments: A sequence of sequences of LocalBuffers. The i'th inner + sequence comprises the arguments for execution on the i'th replica. + + Returns: + A list of the computation's outputs on each replica, as a LocalBuffer. If + a shallow sequence of arguments was passed in for `arguments`, then the + sole, zero'th replica's output is returned instead, as a LocalBuffer. + """ if not self._is_compiled: raise ValueError('Cannot execute an uncompiled local XLA computation.') - arguments = tuple(arguments) - if any(arg.is_deleted() for arg in arguments): - raise ValueError('Executing with deleted local buffer argument') - return LocalBuffer( - self._c_computation.Execute([arg.c_buffer for arg in arguments]), - backend=self._backend) + if arguments is None: + arguments = ((),) * get_replica_count() + else: + arguments = [list(replica_args) for replica_args in arguments] + + # Check arguments + for replica, replica_args in enumerate(arguments): + for arg in replica_args: + if arg.is_deleted(): + raise ValueError('Executing with deleted local buffer argument') + if arg.replica() != replica: + raise ValueError( + 'Executing on replica {} with argument from replica {}'.format( + replica, arg.replica())) + + # Pull out argument buffer handles + stripped_args = [ + [arg.c_buffer for arg in replica_args] for replica_args in arguments + ] + + # Execute + if self._backend.backend_type == BackendType.XRT: + if len(stripped_args) > 1: + raise NotImplementedError( + 'Multi-replica execution is not yet supported via the XRT backend.') + output_buffers = [self._c_computation.Execute(stripped_args[0])] + else: + output_buffer_tup = self._c_computation.ExecutePerReplica(stripped_args) + size = output_buffer_tup.size() + output_buffers = [output_buffer_tup.Release(i) for i in xrange(size)] + + # Wrap output handles in LocalBuffer instances + return tuple( + LocalBuffer(output_buffer, backend=self._backend, replica=replica) + for replica, output_buffer in enumerate(output_buffers)) def ExecuteWithPythonValues(self, arguments=()): - """Execute with Python values as arguments and return value.""" - arguments = tuple( - LocalBuffer.from_pyval(arg, backend=self._backend) for arg in arguments) + """Execute on one replica with Python values as arguments and output.""" + + def put(arg): + return LocalBuffer.from_pyval(arg, backend=self._backend) + + arguments = [put(arg) for arg in arguments] return self.Execute(arguments).to_py() + def ExecuteWithPythonValuesPerReplica(self, arguments): + """Execute on many replicas with Python values as arguments and output.""" + + def put(arg, replica): + return LocalBuffer.from_pyval(arg, replica, backend=self._backend) + + arguments = [[put(arg, replica) + for arg in replica_args] + for replica, replica_args in enumerate(arguments)] + return [out.to_py() for out in self.ExecutePerReplica(arguments)] + def __del__(self): self._delete(self._c_computation) @@ -761,8 +855,7 @@ class ComputationBuilder(object): Returns: A LocalOp representing the added broadcast-in-dimensions op. """ - xla_shape = Shape.array_shape(self.GetShape(operand).element_type(), shape) - return self._client.BroadcastInDim(operand, xla_shape, broadcast_dimensions) + return self._client.BroadcastInDim(operand, shape, broadcast_dimensions) def Concatenate(self, operands, dimension): """Enqueues a concatenate operation onto the computation. @@ -1380,6 +1473,7 @@ def initialize_platform_name(platform_name): Raises: A runtime exception if the XLA service has already been initialized. """ + platform_name = _maybe_encode_string(platform_name) c_api.InitializePlatformName(platform_name) diff --git a/tensorflow/compiler/xla/python_api/xla_shape.py b/tensorflow/compiler/xla/python_api/xla_shape.py index f158f6b2410..95b2bf300ec 100644 --- a/tensorflow/compiler/xla/python_api/xla_shape.py +++ b/tensorflow/compiler/xla/python_api/xla_shape.py @@ -25,9 +25,10 @@ from tensorflow.compiler.xla.python_api import types class Shape(object): - """Wraps a xla_data_pb2.Shape message with a convenient Python type. + """Wraps a xla_data_pb2.ShapeProto message with a convenient Python type. - Provides direct access to the underlying xla_data_pb2.Shape message in the + Provides direct access to the underlying xla_data_pb2.ShapeProto message in + the message attribute, along with accessor wrappers to the message's fields. Avoid direct access to .message unless interacting directly with protobuf APIs like CopyFrom. In other words, prefer hauling the shape around in a Shape, and @@ -48,7 +49,7 @@ class Shape(object): Raises: ValueError: if element_type is TUPLE but dimensions are not Shape objects. """ - self.message = xla_data_pb2.Shape() + self.message = xla_data_pb2.ShapeProto() self.message.element_type = element_type if element_type == xla_data_pb2.TUPLE: if not all(isinstance(subshape, Shape) for subshape in dimensions): diff --git a/tensorflow/compiler/xla/rpc/BUILD b/tensorflow/compiler/xla/rpc/BUILD index 3abb3855a42..26affbcceb3 100644 --- a/tensorflow/compiler/xla/rpc/BUILD +++ b/tensorflow/compiler/xla/rpc/BUILD @@ -16,7 +16,6 @@ xla_proto_library( use_grpc_plugin = True, visibility = ["//visibility:public"], deps = [ - "//tensorflow/compiler/xla:xla_data_proto", "//tensorflow/compiler/xla:xla_proto", ], ) diff --git a/tensorflow/compiler/xla/rpc/xla_service.proto b/tensorflow/compiler/xla/rpc/xla_service.proto index e4f332cda22..0ff8adc2acb 100644 --- a/tensorflow/compiler/xla/rpc/xla_service.proto +++ b/tensorflow/compiler/xla/rpc/xla_service.proto @@ -43,7 +43,6 @@ limitations under the License. syntax = "proto3"; import "tensorflow/compiler/xla/xla.proto"; -import "tensorflow/compiler/xla/xla_data.proto"; package xla; diff --git a/tensorflow/compiler/xla/service/BUILD b/tensorflow/compiler/xla/service/BUILD index 19b5c1ca25d..81e71eee520 100644 --- a/tensorflow/compiler/xla/service/BUILD +++ b/tensorflow/compiler/xla/service/BUILD @@ -281,10 +281,12 @@ tf_cc_test( "//tensorflow/compiler/xla/service:hlo_element_type_converter", "//tensorflow/compiler/xla/tests:hlo_test_base", "//tensorflow/compiler/xla/tests:literal_test_util", + "//tensorflow/compiler/xla/tests:test_utils", "//tensorflow/compiler/xla/tests:xla_internal_test_main", # fixdeps: keep "//tensorflow/core:lib", "//tensorflow/core:test", "@com_google_absl//absl/memory", + "@com_google_absl//absl/strings:str_format", ], ) @@ -292,6 +294,7 @@ cc_library( name = "hlo", srcs = [ "dfs_hlo_visitor.cc", + "dynamic_parameter_binding.cc", "hlo_computation.cc", "hlo_input_output_alias_config.cc", "hlo_instruction.cc", @@ -305,6 +308,7 @@ cc_library( hdrs = [ "dfs_hlo_visitor.h", "dfs_hlo_visitor_with_default.h", + "dynamic_parameter_binding.h", "hlo_clone_context.h", "hlo_computation.h", "hlo_domain_metadata.h", @@ -350,6 +354,25 @@ cc_library( ], ) +tf_cc_test( + name = "dynamic_parameter_binding_test", + srcs = ["dynamic_parameter_binding_test.cc"], + deps = [ + ":hlo", + ":hlo_dce", + ":hlo_memory_scheduler", + ":hlo_ordering", + ":hlo_parser", + "//tensorflow/compiler/xla:shape_util", + "//tensorflow/compiler/xla:types", + "//tensorflow/compiler/xla:xla_data_proto", + "//tensorflow/compiler/xla/tests:hlo_test_base", + "//tensorflow/compiler/xla/tests:xla_internal_test_main", + "//tensorflow/core:test", + "@com_google_absl//absl/algorithm:container", + ], +) + tf_cc_test( name = "dfs_hlo_visitor_with_default_test", srcs = ["dfs_hlo_visitor_with_default_test.cc"], @@ -387,9 +410,36 @@ tf_cc_test( ":hlo", ":pattern_matcher", "//tensorflow/compiler/xla:shape_util", + "//tensorflow/compiler/xla:test", "//tensorflow/compiler/xla/service:hlo_parser", "//tensorflow/compiler/xla/tests:xla_internal_test_main", "//tensorflow/core:test", + "@com_google_absl//absl/strings", + ], +) + +cc_library( + name = "pattern_matcher_gmock", + testonly = 1, + hdrs = ["pattern_matcher_gmock.h"], + deps = [ + ":pattern_matcher", + "//tensorflow/compiler/xla:test", + "//tensorflow/core:test", + ], +) + +tf_cc_test( + name = "pattern_matcher_gmock_test", + srcs = ["pattern_matcher_gmock_test.cc"], + deps = [ + ":hlo", + ":pattern_matcher", + ":pattern_matcher_gmock", + "//tensorflow/compiler/xla:shape_util", + "//tensorflow/compiler/xla:test", + "//tensorflow/compiler/xla/tests:xla_internal_test_main", + "//tensorflow/core:test", ], ) @@ -403,6 +453,7 @@ cc_library( "//tensorflow/compiler/xla:util", "//tensorflow/core:lib", "//tensorflow/core:lib_internal", + "@com_google_absl//absl/base", "@com_google_absl//absl/container:flat_hash_map", "@com_google_absl//absl/types:span", ], @@ -1336,6 +1387,7 @@ cc_library( ":hlo", "//tensorflow/compiler/xla:literal", "//tensorflow/compiler/xla:shape_util", + "@com_google_absl//absl/container:flat_hash_set", ], ) @@ -1539,7 +1591,10 @@ tf_cc_test( ":hlo", ":hlo_casting_utils", ":hlo_matchers", + ":hlo_parser", ":hlo_pass", + ":pattern_matcher", + ":pattern_matcher_gmock", "//tensorflow/compiler/xla:literal", "//tensorflow/compiler/xla:shape_util", "//tensorflow/compiler/xla:test", @@ -1707,7 +1762,9 @@ cc_library( ":hlo", ":hlo_pass", ":hlo_query", + ":pattern_matcher", ":while_loop_analysis", + "//tensorflow/compiler/xla:shape_util", "//tensorflow/compiler/xla:statusor", "@com_google_absl//absl/container:flat_hash_map", "@com_google_absl//absl/container:flat_hash_set", @@ -1720,9 +1777,14 @@ tf_cc_test( name = "while_loop_simplifier_test", srcs = ["while_loop_simplifier_test.cc"], deps = [ + ":algebraic_simplifier", ":hlo", + ":hlo_cse", ":hlo_dce", ":hlo_matchers", + ":hlo_pass", + ":hlo_pass_pipeline", + ":tuple_simplifier", ":while_loop_simplifier", "//tensorflow/compiler/xla:test", "//tensorflow/compiler/xla/tests:hlo_test_base", @@ -2347,6 +2409,7 @@ cc_library( "//tensorflow/compiler/xla:util", "//tensorflow/compiler/xla:xla_data_proto", "//tensorflow/core:lib", + "@com_google_absl//absl/container:flat_hash_map", ], ) @@ -2600,6 +2663,8 @@ tf_cc_test( ":hlo", ":hlo_matchers", ":layout_assignment", + ":pattern_matcher", + ":pattern_matcher_gmock", "//tensorflow/compiler/xla:literal", "//tensorflow/compiler/xla:shape_layout", "//tensorflow/compiler/xla:shape_util", @@ -2744,6 +2809,8 @@ tf_cc_test( ":hlo_matchers", ":hlo_parser", ":hlo_pass", + ":pattern_matcher", + ":pattern_matcher_gmock", "//tensorflow/compiler/xla:literal", "//tensorflow/compiler/xla:shape_util", "//tensorflow/compiler/xla:test", @@ -2855,6 +2922,46 @@ tf_cc_test( ], ) +cc_library( + name = "hlo_get_dimension_size_rewriter", + srcs = ["hlo_get_dimension_size_rewriter.cc"], + hdrs = ["hlo_get_dimension_size_rewriter.h"], + deps = [ + ":hlo", + ":hlo_pass", + ":shape_inference", + "//tensorflow/compiler/xla:literal", + "//tensorflow/compiler/xla:literal_util", + "//tensorflow/compiler/xla:shape_util", + "//tensorflow/compiler/xla:types", + "//tensorflow/compiler/xla:xla_data_proto", + "//tensorflow/core:lib", + "@com_google_absl//absl/algorithm:container", + ], +) + +tf_cc_test( + name = "hlo_get_dimension_size_rewriter_test", + srcs = ["hlo_get_dimension_size_rewriter_test.cc"], + deps = [ + ":hlo", + ":hlo_get_dimension_size_rewriter", + ":hlo_matchers", + ":hlo_parser", + "//tensorflow/compiler/xla:literal", + "//tensorflow/compiler/xla:shape_util", + "//tensorflow/compiler/xla:types", + "//tensorflow/compiler/xla:util", + "//tensorflow/compiler/xla:xla_data_proto", + "//tensorflow/compiler/xla/tests:hlo_test_base", + "//tensorflow/compiler/xla/tests:literal_test_util", + "//tensorflow/compiler/xla/tests:test_utils", + "//tensorflow/compiler/xla/tests:xla_internal_test_main", + "//tensorflow/core:lib", + "//tensorflow/core:test", + ], +) + cc_library( name = "device_memory_allocator", srcs = [ @@ -2913,6 +3020,7 @@ cc_library( "//tensorflow/core:lib", "//tensorflow/core:lib_internal", "@com_google_absl//absl/algorithm:container", + "@com_google_absl//absl/container:flat_hash_map", "@com_google_absl//absl/strings", "@llvm//:core", "@llvm//:transform_utils", @@ -3026,6 +3134,7 @@ cc_library( ":hlo_casting_utils", ":hlo_execution_profile", ":hlo_tfgraph_builder", + ":pattern_matcher", "//tensorflow/compiler/xla:literal", "//tensorflow/compiler/xla:shape_util", "//tensorflow/compiler/xla:types", @@ -3318,9 +3427,9 @@ cc_library( ":tuple_util", ":while_loop_analysis", ":while_util", + "//tensorflow/compiler/xla:shape_util", "//tensorflow/compiler/xla:statusor", "//tensorflow/compiler/xla:util", - "//tensorflow/core:lib", "@com_google_absl//absl/algorithm:container", "@com_google_absl//absl/container:flat_hash_map", "@com_google_absl//absl/container:flat_hash_set", @@ -3463,6 +3572,8 @@ tf_cc_test( ":hlo_casting_utils", ":hlo_matchers", ":hlo_parser", + ":pattern_matcher", + ":pattern_matcher_gmock", "//tensorflow/compiler/xla:test_helpers", "//tensorflow/compiler/xla:window_util", "//tensorflow/core:lib", @@ -3513,6 +3624,41 @@ cc_library( ], ) +cc_library( + name = "ar_crs_combiner", + srcs = ["ar_crs_combiner.cc"], + hdrs = ["ar_crs_combiner.h"], + deps = [ + ":call_graph", + ":pattern_matcher", + "//tensorflow/compiler/xla:literal", + "//tensorflow/compiler/xla:literal_util", + "//tensorflow/compiler/xla:status_macros", + "//tensorflow/compiler/xla:statusor", + "//tensorflow/compiler/xla:types", + "//tensorflow/compiler/xla:xla_data_proto", + "//tensorflow/compiler/xla/service:hlo", + "//tensorflow/compiler/xla/service:hlo_pass", + "@com_google_absl//absl/container:flat_hash_map", + "@com_google_absl//absl/strings", + ], +) + +tf_cc_test( + name = "ar_crs_combiner_test", + srcs = ["ar_crs_combiner_test.cc"], + deps = [ + ":ar_crs_combiner", + ":hlo", + ":hlo_matchers", + "//tensorflow/compiler/xla:statusor", + "//tensorflow/compiler/xla/tests:hlo_test_base", + "//tensorflow/compiler/xla/tests:xla_internal_test_main", + "//tensorflow/core:lib", + "//tensorflow/core:test", + ], +) + tf_cc_test( name = "map_inliner_test", srcs = ["map_inliner_test.cc"], diff --git a/tensorflow/compiler/xla/service/algebraic_simplifier.cc b/tensorflow/compiler/xla/service/algebraic_simplifier.cc index 89e62bd2f0d..985c5af1c4d 100644 --- a/tensorflow/compiler/xla/service/algebraic_simplifier.cc +++ b/tensorflow/compiler/xla/service/algebraic_simplifier.cc @@ -16,6 +16,7 @@ limitations under the License. #include "tensorflow/compiler/xla/service/algebraic_simplifier.h" #include +#include #include #include #include @@ -68,6 +69,45 @@ bool IsAll(const HloInstruction* op, int8 value) { } } +// Checks whether `op` is a floating-point constant or broadcast of a constant +// of the form +/- 2^k for some integer k positive, negative, or zero. Such +// values are interesting because multiplying by a power of 2 just moves the +// exponent. +bool IsAllFpConstantPowerOf2(const HloInstruction* op) { + // Unwrap the broadcast if necessary. + const HloInstruction* c; + if (!Match(op, m::ConstantEffectiveScalar(&c)) && + !Match(op, m::Broadcast(m::Constant(&c).WithShape( + m::Shape().IsEffectiveScalar())))) { + return false; + } + auto val = [&]() -> absl::optional { + switch (c->shape().element_type()) { + case BF16: + return static_cast(c->literal().GetFirstElement()); + case F16: + return static_cast(c->literal().GetFirstElement()); + case F32: + return c->literal().GetFirstElement(); + case F64: + return c->literal().GetFirstElement(); + default: + // Cowardly refuse to consider complex types. + return absl::nullopt; + } + }(); + if (!val) { + return false; + } + + int exp; + double mantissa = std::frexp(*val, &exp); + // frexp returns a value in the range (-1; -0.5] U [0.5, 1). A return value + // of +/-0.5 therefore indicates that the floating point value is a power of + // 2. + return mantissa == 0.5 || mantissa == -0.5; +} + // Returns whether the given transpose produces a result which is bit-wise // identical to its operand and thus may be replaced with a bitcast. bool TransposeIsBitcast(const HloInstruction* transpose) { @@ -84,7 +124,8 @@ bool TransposeIsBitcast(const HloInstruction* transpose) { // reshape may still be a bitcast. For example, a reshape from [28x28] to [784]. bool ReshapeOrCopyIsBitcast( const HloInstruction* instr, - const AlgebraicSimplifier::ValidBitcastCallback& valid_bitcast_callback) { + const AlgebraicSimplifierOptions::ValidBitcastCallback& + valid_bitcast_callback) { CHECK(HloOpcode::kReshape == instr->opcode() || HloOpcode::kCopy == instr->opcode()); @@ -95,6 +136,11 @@ bool ReshapeOrCopyIsBitcast( valid_bitcast_callback(operand->shape(), instr->shape()); } +bool IsUnstridedSlice(const HloInstruction* hlo) { + return absl::c_all_of(hlo->slice_strides(), + [](int64 stride) { return stride == 1; }); +} + // AlgebraicSimplifierVisitor traverses the HLO computation and reduces certain // algebraic expressions to simplified forms. Note: This only supports // simplifications that simply look at the operands of an instruction. For the @@ -180,21 +226,13 @@ class AlgebraicSimplifierVisitor : public DfsHloVisitorWithDefault { const bool changed() const { return changed_; } // Runs the visitor on a computation. - static bool Run( - HloComputation* computation, bool is_layout_sensitive, - AlgebraicSimplifier::ValidBitcastCallback valid_bitcast_callback, - bool enable_dot_strength_reduction, bool enable_conv_simplification); + static bool Run(HloComputation* computation, + const AlgebraicSimplifierOptions& options); private: - explicit AlgebraicSimplifierVisitor( - HloComputation* computation, bool is_layout_sensitive, - AlgebraicSimplifier::ValidBitcastCallback valid_bitcast_callback, - bool enable_dot_strength_reduction, bool enable_conv_simplification) - : computation_(computation), - is_layout_sensitive_(is_layout_sensitive), - valid_bitcast_callback_(std::move(valid_bitcast_callback)), - enable_dot_strength_reduction_(enable_dot_strength_reduction), - enable_conv_simplification_(enable_conv_simplification) {} + explicit AlgebraicSimplifierVisitor(HloComputation* computation, + const AlgebraicSimplifierOptions& options) + : computation_(computation), options_(options) {} // Transforms Dots where at least one input is a vector or has a degenerate // dimension and converts it into a multiply and reduce. This should enable @@ -233,10 +271,10 @@ class AlgebraicSimplifierVisitor : public DfsHloVisitorWithDefault { HloInstruction* new_instruction); // Returns whether the shape of the output of the given instructions are the - // same for the purposes of simplification. If is_layout_sensitive_ is true, - // then this tests shape equality including layout (ShapeUtil::Equal). If - // is_layout_sensitive_ is false, then the tests shape compatibility - // (ShapeUtil::Compatible). + // same for the purposes of simplification. If options_.is_layout_sensitive() + // is true, then this tests shape equality including layout + // (ShapeUtil::Equal). If options_.is_layout_sensitive() is false, then the + // tests shape compatibility (ShapeUtil::Compatible). bool SameShape(const HloInstruction* lhs, const HloInstruction* rhs) const; // Returns whether it was possible to transform `root` to a clamp instruction. @@ -325,22 +363,12 @@ class AlgebraicSimplifierVisitor : public DfsHloVisitorWithDefault { // traversing. HloComputation* computation_; + // The backend-specific options selected for the algebraic simplifier. + const AlgebraicSimplifierOptions& options_; + // Whether algebraic simplification has occurred. bool changed_ = false; - // Whether layout is considered during transformation. - bool is_layout_sensitive_; - - // Callback used to determine if a bitcast is possible. - AlgebraicSimplifier::ValidBitcastCallback valid_bitcast_callback_; - - // Disable dot strength reduction on platforms where it causes a slowdown. - bool enable_dot_strength_reduction_; - - // Disable convolution -> dot simplification on platforms where it causes a - // slowdown. - bool enable_conv_simplification_; - // Cached computation for adding two scalar F32. HloComputation* scalar_add_computation_ = nullptr; }; @@ -348,19 +376,15 @@ class AlgebraicSimplifierVisitor : public DfsHloVisitorWithDefault { } // namespace bool AlgebraicSimplifierVisitor::Run( - HloComputation* computation, bool is_layout_sensitive, - AlgebraicSimplifier::ValidBitcastCallback valid_bitcast_callback, - bool enable_dot_strength_reduction, bool enable_conv_simplification) { - AlgebraicSimplifierVisitor visitor( - computation, is_layout_sensitive, std::move(valid_bitcast_callback), - enable_dot_strength_reduction, enable_conv_simplification); + HloComputation* computation, const AlgebraicSimplifierOptions& options) { + AlgebraicSimplifierVisitor visitor(computation, options); TF_CHECK_OK(computation->Accept(&visitor)); return visitor.changed_; } bool AlgebraicSimplifierVisitor::SameShape(const HloInstruction* lhs, const HloInstruction* rhs) const { - if (is_layout_sensitive_) { + if (options_.is_layout_sensitive()) { return ShapeUtil::Equal(lhs->shape(), rhs->shape()); } else { return ShapeUtil::Compatible(lhs->shape(), rhs->shape()); @@ -431,6 +455,40 @@ Status AlgebraicSimplifierVisitor::HandleAdd(HloInstruction* add) { sum_of_constants)); } + // A*C + B*C => (A+B)*C + // + // - If A, B, and C are integers, do this unconditionally. Proof of + // correctness: https://rise4fun.com/Alive/u9X. + // + // - If A, B, and C are floating point, do this if C is a scalar constant or + // broadcast of scalar constant and is equal to +/- 2^k for some (possibly + // negative) integer k. + // + // Multiplying by a power of 2 just moves the exponent, so our answer is + // exact modulo rounding of intermediate results so long as + // + // - none of the three products has an exponent which underflows (so the + // result is 0 or denormal), and + // - none of the three products overflows to inf. + // + // Proof: See algebraic_simplifier_proof_distributive_property.py. + // + // We deem these differences in rounding, underflow, and overflow + // acceptable in the ML context. + HloInstruction *b, *c; + if (((Match(lhs, m::Multiply(m::Op(&a), m::Op(&c))) && + Match(rhs, m::MultiplyAnyOrder(m::Op().Is(c), m::Op(&b)))) || + (Match(lhs, m::Multiply(m::Op(&c), m::Op(&a))) && + Match(rhs, m::MultiplyAnyOrder(m::Op().Is(c), m::Op(&b))))) && + (ShapeUtil::ElementIsIntegral(add->shape()) || + IsAllFpConstantPowerOf2(c))) { + return ReplaceWithNewInstruction( + add, HloInstruction::CreateBinary( + add->shape(), HloOpcode::kMultiply, + computation_->AddInstruction(HloInstruction::CreateBinary( + add->shape(), HloOpcode::kAdd, a, b)), + c)); + } return Status::OK(); } @@ -504,8 +562,8 @@ Status AlgebraicSimplifierVisitor::HandleCopy(HloInstruction* copy) { return Status::OK(); } - if (is_layout_sensitive_ && - ReshapeOrCopyIsBitcast(copy, valid_bitcast_callback_)) { + if (options_.is_layout_sensitive() && + ReshapeOrCopyIsBitcast(copy, options_.valid_bitcast_callback())) { ReplaceWithBitcast(copy); } @@ -541,7 +599,74 @@ Status AlgebraicSimplifierVisitor::HandleConcatenate( VLOG(10) << "trying to replace " << concatenate->ToString() << " with " << replacement->ToString(); ReplaceInstructionIfSameShape(concatenate, replacement); - } else if (operands.size() == 2) { + return Status::OK(); + } + + // Check if we can merge "adjacent" slice operands which take slices from the + // same other op. For simplicity we only merge unstrided slices. + int64 concatenate_dimension = concatenate->concatenate_dimension(); + for (int64 i = 0; i < operands.size(); ++i) { + if (operands[i]->opcode() != HloOpcode::kSlice || + !IsUnstridedSlice(operands[i])) { + continue; + } + int64 slice_end = operands[i]->slice_limits(concatenate_dimension); + HloInstruction* slice_operand = operands[i]->mutable_operand(0); + int64 j = i + 1; + while (j < operands.size() && operands[j]->opcode() == HloOpcode::kSlice && + IsUnstridedSlice(operands[j]) && + operands[j]->operand(0) == slice_operand && + operands[j]->slice_starts(concatenate_dimension) == slice_end) { + // Check that all the slice_start values are the same in all other + // dimensions. This implies that the slice_limit values are also the same, + // because operands of concatenate need to have the same shape, and we + // already checked that the slices are unstrided. + bool same_other_starts = true; + for (int64 k = 0; k < operands[j]->slice_starts().size(); ++k) { + if (k == concatenate_dimension) { + continue; + } + if (operands[i]->slice_starts(k) != operands[j]->slice_starts(k)) { + same_other_starts = false; + break; + } + } + if (!same_other_starts) { + break; + } + slice_end = operands[j]->slice_limits(concatenate_dimension); + ++j; + } + if (j - i > 1) { + Shape new_slice_shape = operands[i]->shape(); + new_slice_shape.set_dimensions( + concatenate_dimension, + slice_end - operands[i]->slice_starts(concatenate_dimension)); + auto new_limit_indices = operands[i]->slice_limits(); + new_limit_indices[concatenate_dimension] = slice_end; + auto new_slice_op = + computation_->AddInstruction(HloInstruction::CreateSlice( + new_slice_shape, slice_operand, + /*start_indices=*/operands[i]->slice_starts(), + /*limit_indices=*/new_limit_indices, + /*strides=*/operands[i]->slice_strides())); + std::vector new_operands; + for (int64 k = 0; k < i; ++k) { + new_operands.push_back(operands[k]); + } + new_operands.push_back(new_slice_op); + for (int64 k = j; k < operands.size(); ++k) { + new_operands.push_back(operands[k]); + } + auto replacement = + computation_->AddInstruction(concatenate->CloneWithNewOperands( + concatenate->shape(), new_operands)); + ReplaceInstructionIfSameShape(concatenate, replacement); + return Status::OK(); + } + } + + if (operands.size() == 2) { // A binary concat with a broadcasted scalar as an operand can be converted // into a pad which is simpler to fold into other operations. bool is_effective_low_pad = Match( @@ -557,7 +682,7 @@ Status AlgebraicSimplifierVisitor::HandleConcatenate( padding_config_dim->set_edge_padding_high(0); padding_config_dim->set_edge_padding_low(0); padding_config_dim->set_interior_padding(0); - if (dim == concatenate->concatenate_dimension()) { + if (dim == concatenate_dimension) { if (is_effective_low_pad) { padding_config_dim->set_edge_padding_low( operands[0]->shape().dimensions(dim)); @@ -1215,7 +1340,8 @@ Status AlgebraicSimplifierVisitor::HandleDot(HloInstruction* dot) { return ReplaceInstruction(dot, dot_of_gather_optimized); } - if (enable_dot_strength_reduction_ && !is_layout_sensitive_) { + if (options_.enable_dot_strength_reduction() && + !options_.is_layout_sensitive()) { TF_ASSIGN_OR_RETURN(bool did_strength_reduction, HandleDotStrengthReduction(dot)); if (did_strength_reduction) { @@ -1619,6 +1745,27 @@ Status AlgebraicSimplifierVisitor::HandlePad(HloInstruction* pad) { pad, HloInstruction::CreateBroadcast(pad->shape(), pad->mutable_operand(1), {})); } + + // Interior padding on one sized dimensions have no effect. As a result it + // makes other simplifications possible if there is no interior padding. + if (HasInteriorPadding(pad->padding_config())) { + PaddingConfig padding_config = pad->padding_config(); + bool cleared_interior_padding = false; + for (int64 i = 0; i < ShapeUtil::Rank(pad->shape()); ++i) { + if (padding_config.dimensions(i).interior_padding() > 0 && + pad->operand(0)->shape().dimensions(i) == 1) { + cleared_interior_padding = true; + padding_config.mutable_dimensions(i)->set_interior_padding(0); + } + } + if (cleared_interior_padding) { + return ReplaceWithNewInstruction( + pad, + HloInstruction::CreatePad(pad->shape(), pad->mutable_operand(0), + pad->mutable_operand(1), padding_config)); + } + } + // Eliminate nop pads (padding all zero), and replace a pad with negative // padding with a pad with non-negative padding followed by a slice. bool all_zero = true; @@ -1910,8 +2057,8 @@ Status AlgebraicSimplifierVisitor::HandleReshape(HloInstruction* reshape) { } // Make this a bitcast if possible. - if (is_layout_sensitive_ && - ReshapeOrCopyIsBitcast(reshape, valid_bitcast_callback_)) { + if (options_.is_layout_sensitive() && + ReshapeOrCopyIsBitcast(reshape, options_.valid_bitcast_callback())) { ReplaceWithBitcast(reshape); return Status::OK(); } @@ -2030,11 +2177,6 @@ StatusOr AlgebraicSimplifierVisitor::TrySimplifyScalarSlice( return false; } -bool IsUnstridedSlice(const HloInstruction* hlo) { - return absl::c_all_of(hlo->slice_strides(), - [](int64 stride) { return stride == 1; }); -} - StatusOr AlgebraicSimplifierVisitor::TryToReorderSliceAndReshape( HloInstruction* slice) { CHECK_EQ(slice->opcode(), HloOpcode::kSlice); @@ -2501,6 +2643,108 @@ Status AlgebraicSimplifierVisitor::HandleSort(HloInstruction* sort) { return ReplaceWithNewInstruction( sort, HloInstruction::CreateTuple(sort->operands())); } + if (!options_.enable_permutation_sort_replacement()) { + return Status::OK(); + } + // Check if we are sorting a permutation. In that case, we know that the keys + // will be sorted to the identity permutation, and we can represent the + // changes to the 'values' parameter as a scatter. + if (sort->operand_count() == 2 && + operand->opcode() == HloOpcode::kGetTupleElement) { + const HloInstruction* other_sort = operand->operand(0); + // Check whether the 'values' parameter is the result of another sort with + // the same sort dimension. + if (other_sort->opcode() == HloOpcode::kSort && + other_sort->operand_count() >= 2 && + other_sort->dimensions(0) == dimension_to_sort && + other_sort->operand(operand->tuple_index())->opcode() == + HloOpcode::kIota) { + auto* iota = + Cast(other_sort->operand(operand->tuple_index())); + // The sort operand needs to be an integral iota, and the iota dimension + // needs to be the dimension that was sorted. + if (iota->iota_dimension() == dimension_to_sort && + ShapeUtil::ElementIsIntegral(iota->shape())) { + // We use the following construction method for a Scatter that applies + // the permutation from 'keys' to the 'values' parameter. + // - Take the "keys" parameter of the second sort and reshape it to have + // another "1" dimension at the end. + // - Concatenate it with iotas of the same extended shape with all + // different iota_dimensions except the dimension_to_sort in the order + // of iota_dimensions/dimension_to_sort, so e.g. with rank 3 and + // dimension_to_sort = 1, we would have concatenate of (iota with + // iota_dimension=0, keys, iota with iota_dimension = 2) + // - Use this as the indices parameter of scatter, and set updates + // of the scatter to be a reshaped 'values' parameter of sort (adding + // 'rank' many 1 dimensions at the end). + int64 rank = ShapeUtil::Rank(operand->shape()); + Shape extended_shape = operand->shape(); + extended_shape.add_dimensions(1); + extended_shape.mutable_layout()->add_minor_to_major(rank); + auto reshaped_permutation = computation_->AddInstruction( + HloInstruction::CreateReshape(extended_shape, operand)); + std::vector concat_operands; + for (int64 i = 0; i < rank; ++i) { + if (i == dimension_to_sort) { + concat_operands.push_back(reshaped_permutation); + } else { + concat_operands.push_back(computation_->AddInstruction( + HloInstruction::CreateIota(extended_shape, i))); + } + } + Shape concat_shape = operand->shape(); + concat_shape.add_dimensions(rank); + concat_shape.mutable_layout()->add_minor_to_major(rank); + auto scatter_indices = + rank > 1 ? computation_->AddInstruction( + HloInstruction::CreateConcatenate( + concat_shape, concat_operands, rank)) + : reshaped_permutation; + + // We don't care about the operand, it will be completely overridden by + // the updates. + auto scatter_operand = computation_->AddInstruction( + HloInstruction::CreateIota(sort->operand(1)->shape(), 0)); + + // Construct the updates operand of scatter. + Shape update_shape = sort->operand(1)->shape(); + for (int64 i = 0; i < rank; ++i) { + update_shape.add_dimensions(1); + update_shape.mutable_layout()->add_minor_to_major(rank + i); + } + auto scatter_updates = + computation_->AddInstruction(HloInstruction::CreateReshape( + update_shape, sort->mutable_operand(1))); + + // Construct the updates computation, which simply replaces the operand + // values with the update values. + HloComputation::Builder b("update_replace_computation"); + Shape scalar_shape = ShapeUtil::MakeShape(S32, {}); + b.AddInstruction( + HloInstruction::CreateParameter(0, scalar_shape, "scalar_lhs")); + auto scalar_rhs = b.AddInstruction( + HloInstruction::CreateParameter(1, scalar_shape, "scalar_rhs")); + auto update_replace_computation = + computation_->parent()->AddEmbeddedComputation(b.Build(scalar_rhs)); + + ScatterDimensionNumbers dim_numbers; + dim_numbers.set_index_vector_dim(rank); + for (int64 i = 0; i < rank; ++i) { + dim_numbers.add_update_window_dims(rank + i); + dim_numbers.add_scatter_dims_to_operand_dims(i); + } + auto scatter = + computation_->AddInstruction(HloInstruction::CreateScatter( + sort->operand(1)->shape(), scatter_operand, scatter_indices, + scatter_updates, update_replace_computation, dim_numbers)); + return ReplaceWithNewInstruction( + sort, HloInstruction::CreateTuple( + {computation_->AddInstruction(HloInstruction::CreateIota( + operand->shape(), dimension_to_sort)), + scatter})); + } + } + } return Status::OK(); } @@ -2525,7 +2769,7 @@ Status AlgebraicSimplifierVisitor::HandleTranspose(HloInstruction* transpose) { return ReplaceInstruction(transpose, operand); } - if (is_layout_sensitive_ && TransposeIsBitcast(transpose)) { + if (options_.is_layout_sensitive() && TransposeIsBitcast(transpose)) { ReplaceWithBitcast(transpose); return Status::OK(); } @@ -2674,13 +2918,13 @@ StatusOr AlgebraicSimplifierVisitor::SimplifyConvToDot( const ConvolutionDimensionNumbers& dnums = convolution->convolution_dimension_numbers(); - if (!enable_conv_simplification_) { + if (!options_.enable_conv_simplification()) { return false; } // TODO(b/31337498): For now, we cowardly refuse to do this optimization in // layout-insensitive mode, for fear of adding nontrivial reshapes. - if (!is_layout_sensitive_) { + if (!options_.is_layout_sensitive()) { return false; } @@ -2770,9 +3014,9 @@ StatusOr AlgebraicSimplifierVisitor::SimplifyConvToDot( // We cannot insert bitcasts if the layouts will not be compatible. // TODO(b/33178038): Consider inserting a transpose if a bitcast would be // invalid. - if (!valid_bitcast_callback_(input_shape, new_input_shape) || - !valid_bitcast_callback_(filter_shape, new_filter_shape) || - !valid_bitcast_callback_(dot_output_shape, convolution_shape)) { + if (!options_.valid_bitcast_callback()(input_shape, new_input_shape) || + !options_.valid_bitcast_callback()(filter_shape, new_filter_shape) || + !options_.valid_bitcast_callback()(dot_output_shape, convolution_shape)) { return false; } @@ -2878,9 +3122,7 @@ StatusOr AlgebraicSimplifier::Run(HloModule* module) { "AlgebraicSimplifier::Run(), before:\n" + module->ToString()); bool changed = false; for (auto* comp : module->MakeNonfusionComputations()) { - if (AlgebraicSimplifierVisitor::Run( - comp, is_layout_sensitive_, valid_bitcast_callback_, - enable_dot_strength_reduction_, enable_conv_simplification_)) { + if (AlgebraicSimplifierVisitor::Run(comp, options_)) { changed = true; } } diff --git a/tensorflow/compiler/xla/service/algebraic_simplifier.h b/tensorflow/compiler/xla/service/algebraic_simplifier.h index 9f8d0ee88bd..d2775b9fafa 100644 --- a/tensorflow/compiler/xla/service/algebraic_simplifier.h +++ b/tensorflow/compiler/xla/service/algebraic_simplifier.h @@ -23,8 +23,7 @@ limitations under the License. namespace xla { -// A pass which performs algebraic simplifications. -class AlgebraicSimplifier : public HloModulePass { +class AlgebraicSimplifierOptions { public: // Given shapes 'from_shape' and 'to_shape', determines if it is valid to // bitcast from 'from_shape' to 'to_shape' after considering platform @@ -34,18 +33,63 @@ class AlgebraicSimplifier : public HloModulePass { using ValidBitcastCallback = std::function; + explicit AlgebraicSimplifierOptions( + ValidBitcastCallback valid_bitcast_callback) + : valid_bitcast_callback_(std::move(valid_bitcast_callback)) {} + // If valid_bitcast_callback returns true, then the pass will replace reshapes + // and transposes with bitcasts. + const ValidBitcastCallback& valid_bitcast_callback() const { + return valid_bitcast_callback_; + } + // If is_layout_sensitive is true, then the simplifier preserves layout during - // transformation. Otherwise, layout is ignored. If valid_bitcast_callback - // returns true, then the pass will replace reshapes and transposes with - // bitcasts. - AlgebraicSimplifier(bool is_layout_sensitive, - ValidBitcastCallback valid_bitcast_callback, - bool enable_dot_strength_reduction = true, - bool enable_conv_simplification = true) - : is_layout_sensitive_(is_layout_sensitive), - valid_bitcast_callback_(std::move(valid_bitcast_callback)), - enable_dot_strength_reduction_(enable_dot_strength_reduction), - enable_conv_simplification_(enable_conv_simplification) {} + // transformation. Otherwise, layout is ignored. + void set_is_layout_sensitive(bool is_layout_sensitive) { + is_layout_sensitive_ = is_layout_sensitive; + } + bool is_layout_sensitive() const { return is_layout_sensitive_; } + + // Enable dot simplification on platforms where it is profitable. + void set_enable_dot_strength_reduction(bool enable_dot_strength_reduction) { + enable_dot_strength_reduction_ = enable_dot_strength_reduction; + } + bool enable_dot_strength_reduction() const { + return enable_dot_strength_reduction_; + } + + // Enable convolution simplification on platforms where it is profitable. + void set_enable_conv_simplification(bool enable_conv_simplification) { + enable_conv_simplification_ = enable_conv_simplification; + } + bool enable_conv_simplification() const { + return enable_conv_simplification_; + } + + // If enable_permutation_sort_replacement is true, a sort op that is known to + // sort a permutation will be replaced with a scatter op. + void set_enable_permutation_sort_replacement( + bool enable_permutation_sort_replacement) { + enable_permutation_sort_replacement_ = enable_permutation_sort_replacement; + } + bool enable_permutation_sort_replacement() const { + return enable_permutation_sort_replacement_; + } + + private: + ValidBitcastCallback valid_bitcast_callback_; + bool is_layout_sensitive_{false}; + bool enable_dot_strength_reduction_{true}; + bool enable_conv_simplification_{true}; + bool enable_permutation_sort_replacement_{false}; +}; + +// A pass which performs algebraic simplifications. +class AlgebraicSimplifier : public HloModulePass { + public: + // If is_layout_sensitive is true, then the simplifier preserves layout during + // transformation. Otherwise, layout is ignored. + explicit AlgebraicSimplifier(const AlgebraicSimplifierOptions& options) + : options_(options) {} ~AlgebraicSimplifier() override = default; absl::string_view name() const override { return "algsimp"; } @@ -54,14 +98,7 @@ class AlgebraicSimplifier : public HloModulePass { StatusOr Run(HloModule* module) override; private: - bool is_layout_sensitive_; - ValidBitcastCallback valid_bitcast_callback_; - - // Enable dot simplification on platforms where it is profitable. - bool enable_dot_strength_reduction_; - - // Enable convolution simplification on platforms where it is profitable. - bool enable_conv_simplification_; + AlgebraicSimplifierOptions options_; }; } // namespace xla diff --git a/tensorflow/compiler/xla/service/algebraic_simplifier_proof_distributive_property.py b/tensorflow/compiler/xla/service/algebraic_simplifier_proof_distributive_property.py new file mode 100644 index 00000000000..5da13da041b --- /dev/null +++ b/tensorflow/compiler/xla/service/algebraic_simplifier_proof_distributive_property.py @@ -0,0 +1,82 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Proof that transforming (A*C)+(B*C) <=> (A+B)*C is "safe" if C=2^k. + +Specifically, for all floating-point values A, B, and C, if + + - C is equal to +/- 2^k for some (possibly negative) integer k, and + - A, B, C, A*C, B*C, and A+B are not subnormal, zero, or inf, + +then there exists a rounding mode rm in [RTZ, RNE] such that + + (A*C) + (B*C) == (A+B) * C (computed with rounding mode rm). + +Informally, this means that the equivalence holds for powers of 2 C, modulo +flushing to zero or inf, and modulo rounding of intermediate results. + +Requires z3 python bindings; try `pip install z3-solver`. +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +import z3 + +# We do float16 because it lets the solver run much faster. These results +# should generalize to fp32 and fp64, and you can verify this by changing the +# value of FLOAT_TY (and then waiting a while). +FLOAT_TY = z3.Float16 + +a = z3.FP("a", FLOAT_TY()) +b = z3.FP("b", FLOAT_TY()) +c = z3.FP("c", FLOAT_TY()) + +s = z3.Solver() + +# C must be a power of 2, i.e. significand bits must all be 0. +s.add(z3.Extract(FLOAT_TY().sbits() - 1, 0, z3.fpToIEEEBV(c)) == 0) + +for rm in [z3.RTZ(), z3.RNE()]: + z3.set_default_rounding_mode(rm) + before = a * c + b * c + after = (a + b) * c + + # Check that before == after, allowing that 0 == -0. + s.add( + z3.Not( + z3.Or( + before == after, # + z3.And(z3.fpIsZero(before), z3.fpIsZero(after))))) + + for x in [ + (a * c), + (b * c), + (a + b), + ]: + s.add(z3.Not(z3.fpIsSubnormal(x))) + s.add(z3.Not(z3.fpIsZero(x))) + s.add(z3.Not(z3.fpIsInf(x))) + +if s.check() == z3.sat: + m = s.model() + print("Counterexample found!") + print(m) + print("a*c: ", z3.simplify(m[a] * m[c])) + print("b*c: ", z3.simplify(m[b] * m[c])) + print("a+b: ", z3.simplify(m[a] + m[b])) + print("a*c + b*c: ", z3.simplify(m[a] * m[c] + m[b] * m[c])) + print("(a+b) * c: ", z3.simplify((m[a] + m[b]) * m[c])) +else: + print("Proved!") diff --git a/tensorflow/compiler/xla/service/algebraic_simplifier_test.cc b/tensorflow/compiler/xla/service/algebraic_simplifier_test.cc index e4c4da1b0e7..14ce519b6a0 100644 --- a/tensorflow/compiler/xla/service/algebraic_simplifier_test.cc +++ b/tensorflow/compiler/xla/service/algebraic_simplifier_test.cc @@ -27,9 +27,11 @@ limitations under the License. #include "tensorflow/compiler/xla/service/hlo_computation.h" #include "tensorflow/compiler/xla/service/hlo_instruction.h" #include "tensorflow/compiler/xla/service/hlo_instructions.h" -#include "tensorflow/compiler/xla/service/hlo_matchers.h" #include "tensorflow/compiler/xla/service/hlo_opcode.h" +#include "tensorflow/compiler/xla/service/hlo_parser.h" #include "tensorflow/compiler/xla/service/hlo_pass_fix.h" +#include "tensorflow/compiler/xla/service/pattern_matcher.h" +#include "tensorflow/compiler/xla/service/pattern_matcher_gmock.h" #include "tensorflow/compiler/xla/shape_util.h" #include "tensorflow/compiler/xla/test.h" #include "tensorflow/compiler/xla/tests/hlo_test_base.h" @@ -42,18 +44,20 @@ namespace xla { namespace { using ::testing::ElementsAre; +namespace m = match; -namespace op = xla::testing::opcode_matchers; - -AlgebraicSimplifier::ValidBitcastCallback bitcasting_callback() { +AlgebraicSimplifierOptions::ValidBitcastCallback bitcasting_callback() { return [](const Shape&, const Shape&) { return true; }; } -AlgebraicSimplifier::ValidBitcastCallback non_bitcasting_callback() { +AlgebraicSimplifierOptions::ValidBitcastCallback non_bitcasting_callback() { return [](const Shape&, const Shape&) { return false; }; } -class AlgebraicSimplifierTest : public HloTestBase {}; +class AlgebraicSimplifierTest : public HloTestBase { + protected: + AlgebraicSimplifierOptions default_options_{non_bitcasting_callback()}; +}; // Test that A + 0 is simplified to A TEST_F(AlgebraicSimplifierTest, AddZero) { @@ -70,13 +74,134 @@ TEST_F(AlgebraicSimplifierTest, AddZero) { auto computation = m->AddEntryComputation(builder.Build()); HloInstruction* root = computation->root_instruction(); EXPECT_EQ(root->opcode(), HloOpcode::kAdd); - AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false, - non_bitcasting_callback()); + AlgebraicSimplifier simplifier(default_options_); ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie()); root = computation->root_instruction(); EXPECT_EQ(root, param0); } +TEST_F(AlgebraicSimplifierTest, FactorIntegerAddition) { + const char* kModuleStr = R"( + HloModule m + test { + p0 = s32[8] parameter(0) + p1 = s32[8] parameter(1) + p2 = s32[8] parameter(2) + x = s32[8] multiply(p0, p2) + y = s32[8] multiply(p1, p2) + ROOT sum = s32[8] add(x, y) + } + )"; + TF_ASSERT_OK_AND_ASSIGN(auto m, ParseAndReturnVerifiedModule(kModuleStr)); + AlgebraicSimplifier simplifier(default_options_); + ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie()); + EXPECT_THAT( + m->entry_computation()->root_instruction(), + GmockMatch(m::MultiplyAnyOrder( + m::AddAnyOrder(m::Parameter(0), m::Parameter(1)), m::Parameter(2)))); +} + +// A*C + B*C => (A+B)*C if C is a floating-point power of 2. +TEST_F(AlgebraicSimplifierTest, FactorFpAddition) { + const char* kModuleStr = R"( + HloModule m + test { + p0 = f32[] parameter(0) + p1 = f32[] parameter(1) + c = f32[] constant(0.125) + x = f32[] multiply(p0, c) + y = f32[] multiply(p1, c) + ROOT sum = f32[] add(x, y) + } + )"; + TF_ASSERT_OK_AND_ASSIGN(auto m, ParseAndReturnVerifiedModule(kModuleStr)); + ASSERT_TRUE(AlgebraicSimplifier(default_options_).Run(m.get()).ValueOrDie()); + EXPECT_THAT(m->entry_computation()->root_instruction(), + GmockMatch(m::MultiplyAnyOrder( + m::AddAnyOrder(m::Parameter(0), m::Parameter(1)), + m::ConstantScalar(0.125)))); +} + +// A*C + B*C => (A+B)*C if C is a broadcast of a floating-point power of 2. +TEST_F(AlgebraicSimplifierTest, FactorFpAdditionWithBroadcast) { + const char* kModuleStr = R"( + HloModule m + test { + p0 = f32[4] parameter(0) + p1 = f32[4] parameter(1) + c = f32[] constant(0.125) + b = f32[4] broadcast(c), dimensions={} + x = f32[4] multiply(p0, b) + y = f32[4] multiply(p1, b) + ROOT sum = f32[4] add(x, y) + } + )"; + TF_ASSERT_OK_AND_ASSIGN(auto m, ParseAndReturnVerifiedModule(kModuleStr)); + ASSERT_TRUE(AlgebraicSimplifier(default_options_).Run(m.get()).ValueOrDie()); + EXPECT_THAT(m->entry_computation()->root_instruction(), + GmockMatch(m::MultiplyAnyOrder( + m::AddAnyOrder(m::Parameter(0), m::Parameter(1)), + m::Broadcast(m::ConstantScalar(0.125))))); +} + +// A*C + B*C => (A+B)*C simplification should not happen if C is not a +// floating-point power of 2. +TEST_F(AlgebraicSimplifierTest, FactorFpAdditionNotPowerOf2) { + const char* kModuleStr = R"( + HloModule m + test { + p0 = f32[] parameter(0) + p1 = f32[] parameter(1) + c = f32[] constant(0.3) + x = f32[] multiply(p0, c) + y = f32[] multiply(p1, c) + ROOT sum = f32[] add(x, y) + } + )"; + TF_ASSERT_OK_AND_ASSIGN(auto m, ParseAndReturnVerifiedModule(kModuleStr)); + EXPECT_FALSE(AlgebraicSimplifier(default_options_).Run(m.get()).ValueOrDie()); +} + +// A*C + B*C => (A+B)*C simplification should not happen if A, B, and C are +// complex numbers. +TEST_F(AlgebraicSimplifierTest, FactorFpAdditionComplex) { + const char* kModuleStr = R"( + HloModule m + test { + p0 = c64[8] parameter(0) + p1 = c64[8] parameter(1) + p2 = c64[8] parameter(2) + x = c64[8] multiply(p0, p2) + y = c64[8] multiply(p1, p2) + ROOT sum = c64[8] add(x, y) + } + )"; + TF_ASSERT_OK_AND_ASSIGN(auto m, ParseAndReturnVerifiedModule(kModuleStr)); + EXPECT_FALSE(AlgebraicSimplifier(default_options_).Run(m.get()).ValueOrDie()); +} + +// A*C + B*C => (A+B)*C simplification is OK if A, B, and C are complex. +TEST_F(AlgebraicSimplifierTest, FactorFpAdditionBfloat16) { + const char* kModuleStr = R"( + HloModule m + test { + p0 = bf16[4] parameter(0) + p1 = bf16[4] parameter(1) + c = bf16[] constant(0.125) + b = bf16[4] broadcast(c), dimensions={} + x = bf16[4] multiply(p0, b) + y = bf16[4] multiply(p1, b) + ROOT sum = bf16[4] add(x, y) + } + )"; + TF_ASSERT_OK_AND_ASSIGN(auto m, ParseAndReturnVerifiedModule(kModuleStr)); + ASSERT_TRUE(AlgebraicSimplifier(default_options_).Run(m.get()).ValueOrDie()); + EXPECT_THAT(m->entry_computation()->root_instruction(), + GmockMatch(m::MultiplyAnyOrder( + m::AddAnyOrder(m::Parameter(0), m::Parameter(1)), + m::Broadcast(m::ConstantScalar(0.125))))); +} + // Test that A * 0 is simplified to 0 TEST_F(AlgebraicSimplifierTest, MulZero) { auto m = CreateNewVerifiedModule(); @@ -92,8 +217,7 @@ TEST_F(AlgebraicSimplifierTest, MulZero) { auto computation = m->AddEntryComputation(builder.Build()); HloInstruction* root = computation->root_instruction(); EXPECT_EQ(root->opcode(), HloOpcode::kMultiply); - AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false, - non_bitcasting_callback()); + AlgebraicSimplifier simplifier(default_options_); ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie()); EXPECT_EQ(computation->root_instruction(), zero); } @@ -115,8 +239,7 @@ TEST_F(AlgebraicSimplifierTest, SelectTrue) { auto computation = module->AddEntryComputation(builder.Build()); HloInstruction* root = computation->root_instruction(); EXPECT_EQ(root->opcode(), HloOpcode::kSelect); - AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false, - non_bitcasting_callback()); + AlgebraicSimplifier simplifier(default_options_); ASSERT_TRUE(simplifier.Run(module.get()).ValueOrDie()); EXPECT_EQ(computation->root_instruction(), param0); } @@ -138,8 +261,7 @@ TEST_F(AlgebraicSimplifierTest, SelectFalse) { auto computation = module->AddEntryComputation(builder.Build()); HloInstruction* root = computation->root_instruction(); EXPECT_EQ(root->opcode(), HloOpcode::kSelect); - AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false, - non_bitcasting_callback()); + AlgebraicSimplifier simplifier(default_options_); ASSERT_TRUE(simplifier.Run(module.get()).ValueOrDie()); EXPECT_EQ(computation->root_instruction(), param1); } @@ -159,8 +281,7 @@ TEST_F(AlgebraicSimplifierTest, SelectIdentical) { auto computation = module->AddEntryComputation(builder.Build()); HloInstruction* root = computation->root_instruction(); EXPECT_EQ(root->opcode(), HloOpcode::kSelect); - AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false, - non_bitcasting_callback()); + AlgebraicSimplifier simplifier(default_options_); ASSERT_TRUE(simplifier.Run(module.get()).ValueOrDie()); EXPECT_EQ(computation->root_instruction(), param1); } @@ -196,11 +317,10 @@ TEST_F(AlgebraicSimplifierTest, TwoReducesToOne) { builder.AddInstruction(HloInstruction::CreateReduce(r1f32, reduce0, zero, dims1, add_computation)); m->AddEntryComputation(builder.Build()); - AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false, - non_bitcasting_callback()); + AlgebraicSimplifier simplifier(default_options_); ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie()); HloInstruction* root = m->entry_computation()->root_instruction(); - EXPECT_THAT(root, op::Reduce(param, zero)); + EXPECT_THAT(root, GmockMatch(m::Reduce(m::Parameter(0), m::Op().Is(zero)))); EXPECT_EQ(root->dimensions(), std::vector({0, 2, 3})); } @@ -219,11 +339,10 @@ TEST_F(AlgebraicSimplifierTest, AddConstOnLHS) { auto computation = m->AddEntryComputation(builder.Build()); HloInstruction* root = computation->root_instruction(); EXPECT_EQ(root->opcode(), HloOpcode::kAdd); - AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false, - non_bitcasting_callback()); + AlgebraicSimplifier simplifier(default_options_); ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie()); root = computation->root_instruction(); - EXPECT_THAT(root, op::Add(param0, op::Constant())); + EXPECT_THAT(root, GmockMatch(m::Add(m::Parameter(0), m::Constant()))); } // Test that [(A + C1) + C2] => [A + (C1 + C2)] for constants C1 and C2. @@ -246,11 +365,12 @@ TEST_F(AlgebraicSimplifierTest, AddReassociateMergeConstants) { auto computation = m->AddEntryComputation(builder.Build()); HloInstruction* root = computation->root_instruction(); EXPECT_EQ(root->opcode(), HloOpcode::kAdd); - AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false, - non_bitcasting_callback()); + AlgebraicSimplifier simplifier(default_options_); ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie()); root = computation->root_instruction(); - EXPECT_THAT(root, op::Add(param0, op::Add(constant1, constant2))); + EXPECT_THAT(root, GmockMatch(m::Add( + m::Op().Is(param0), + m::Add(m::Op().Is(constant1), m::Op().Is(constant2))))); } TEST_F(AlgebraicSimplifierTest, AddBroadcastZeroR0Operand) { @@ -269,8 +389,7 @@ TEST_F(AlgebraicSimplifierTest, AddBroadcastZeroR0Operand) { auto computation = m->AddEntryComputation(builder.Build()); HloInstruction* root = computation->root_instruction(); EXPECT_EQ(root->opcode(), HloOpcode::kAdd); - AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false, - non_bitcasting_callback()); + AlgebraicSimplifier simplifier(default_options_); ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie()); root = computation->root_instruction(); EXPECT_EQ(root, param0); @@ -306,11 +425,11 @@ TEST_F(AlgebraicSimplifierTest, InlineTrivialMap) { auto computation = m->AddEntryComputation(builder.Build()); HloInstruction* root = computation->root_instruction(); EXPECT_EQ(root->opcode(), HloOpcode::kMap); - AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false, - non_bitcasting_callback()); + AlgebraicSimplifier simplifier(default_options_); ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie()); root = computation->root_instruction(); - EXPECT_THAT(root, op::Add(param0, op::Broadcast(zero))); + EXPECT_THAT(root, GmockMatch(m::Add(m::Parameter(0), + m::Broadcast(m::Op().Is(zero))))); } TEST_F(AlgebraicSimplifierTest, AddBroadcastZeroR1Operand) { @@ -329,8 +448,7 @@ TEST_F(AlgebraicSimplifierTest, AddBroadcastZeroR1Operand) { auto computation = m->AddEntryComputation(builder.Build()); HloInstruction* root = computation->root_instruction(); EXPECT_EQ(root->opcode(), HloOpcode::kAdd); - AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false, - non_bitcasting_callback()); + AlgebraicSimplifier simplifier(default_options_); ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie()); root = computation->root_instruction(); EXPECT_EQ(root, param0); @@ -344,12 +462,11 @@ TEST_F(AlgebraicSimplifierTest, ConstantToBroadcast) { auto computation = m->AddEntryComputation(builder.Build()); HloInstruction* root = computation->root_instruction(); - EXPECT_THAT(root, op::Constant()); - AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false, - non_bitcasting_callback()); + EXPECT_THAT(root, GmockMatch(m::Constant())); + AlgebraicSimplifier simplifier(default_options_); ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie()); root = computation->root_instruction(); - EXPECT_THAT(root, op::Broadcast(op::Constant())); + EXPECT_THAT(root, GmockMatch(m::Broadcast(m::Constant()))); EXPECT_EQ(3.14f, root->operand(0)->literal().GetFirstElement()); } @@ -361,12 +478,11 @@ TEST_F(AlgebraicSimplifierTest, ConstantNotToBroadcast) { auto computation = m->AddEntryComputation(builder.Build()); HloInstruction* root = computation->root_instruction(); - EXPECT_THAT(root, op::Constant()); - AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false, - non_bitcasting_callback()); + EXPECT_THAT(root, GmockMatch(m::Constant())); + AlgebraicSimplifier simplifier(default_options_); ASSERT_FALSE(simplifier.Run(m.get()).ValueOrDie()); root = computation->root_instruction(); - EXPECT_THAT(root, op::Constant()); + EXPECT_THAT(root, GmockMatch(m::Constant())); } TEST_F(AlgebraicSimplifierTest, IotaToBroadcast) { @@ -377,12 +493,11 @@ TEST_F(AlgebraicSimplifierTest, IotaToBroadcast) { auto computation = m->AddEntryComputation(builder.Build()); HloInstruction* root = computation->root_instruction(); - EXPECT_THAT(root, op::Constant()); - AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false, - non_bitcasting_callback()); + EXPECT_THAT(root, GmockMatch(m::Constant())); + AlgebraicSimplifier simplifier(default_options_); ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie()); root = computation->root_instruction(); - EXPECT_THAT(root, op::Iota()); + EXPECT_THAT(root, GmockMatch(m::Iota())); } // Test that A - 0 is simplified to A @@ -400,8 +515,7 @@ TEST_F(AlgebraicSimplifierTest, SubZero) { auto computation = m->AddEntryComputation(builder.Build()); HloInstruction* root = computation->root_instruction(); EXPECT_EQ(root->opcode(), HloOpcode::kSubtract); - AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false, - non_bitcasting_callback()); + AlgebraicSimplifier simplifier(default_options_); ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie()); root = computation->root_instruction(); EXPECT_EQ(root, param0); @@ -422,11 +536,11 @@ TEST_F(AlgebraicSimplifierTest, SubConstCanonicalization) { auto computation = m->AddEntryComputation(builder.Build()); HloInstruction* root = computation->root_instruction(); EXPECT_EQ(root->opcode(), HloOpcode::kSubtract); - AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false, - non_bitcasting_callback()); + AlgebraicSimplifier simplifier(default_options_); ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie()); root = computation->root_instruction(); - EXPECT_THAT(root, op::Add(param0, op::Negate(constant))); + EXPECT_THAT(root, GmockMatch(m::Add(m::Parameter(0), + m::Negate(m::Op().Is(constant))))); } // Test that (A/B)/C is simplified to A/(B*C). @@ -448,14 +562,16 @@ TEST_F(AlgebraicSimplifierTest, LhsDivOfDiv) { auto computation = m->AddEntryComputation(builder.Build()); EXPECT_THAT(computation->root_instruction(), - op::Divide(op::Divide(param0, param1), param2)); + GmockMatch(m::Divide(m::Divide(m::Parameter(0), m::Parameter(1)), + m::Parameter(2)))); - AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false, - non_bitcasting_callback()); + AlgebraicSimplifier simplifier(default_options_); ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie()); - EXPECT_THAT(computation->root_instruction(), - op::Divide(param0, op::Multiply(param1, param2))); + EXPECT_THAT( + computation->root_instruction(), + GmockMatch(m::Divide(m::Parameter(0), + m::Multiply(m::Parameter(1), m::Parameter(2))))); } // Test that A/(B/C) is simplified to (A*C)/B. @@ -476,15 +592,18 @@ TEST_F(AlgebraicSimplifierTest, RhsDivOfDiv) { auto computation = m->AddEntryComputation(builder.Build()); - EXPECT_THAT(computation->root_instruction(), - op::Divide(param0, op::Divide(param1, param2))); + EXPECT_THAT( + computation->root_instruction(), + GmockMatch(m::Divide(m::Parameter(0), + m::Divide(m::Parameter(1), m::Parameter(2))))); - AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false, - non_bitcasting_callback()); + AlgebraicSimplifier simplifier(default_options_); ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie()); - EXPECT_THAT(computation->root_instruction(), - op::Divide(op::Multiply(param0, param2), param1)); + EXPECT_THAT( + computation->root_instruction(), + GmockMatch(m::Divide(m::Multiply(m::Parameter(0), m::Parameter(2)), + m::Parameter(1)))); } // Test that (A/B)/(C/D) is simplified to (A*D)/(B*C). @@ -511,15 +630,16 @@ TEST_F(AlgebraicSimplifierTest, DivOfDivAndDiv) { EXPECT_THAT( computation->root_instruction(), - op::Divide(op::Divide(param0, param1), op::Divide(param2, param3))); + GmockMatch(m::Divide(m::Divide(m::Parameter(0), m::Parameter(1)), + m::Divide(m::Parameter(2), m::Parameter(3))))); - AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false, - non_bitcasting_callback()); + AlgebraicSimplifier simplifier(default_options_); ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie()); EXPECT_THAT( computation->root_instruction(), - op::Divide(op::Multiply(param0, param3), op::Multiply(param1, param2))); + GmockMatch(m::Divide(m::Multiply(m::Parameter(0), m::Parameter(3)), + m::Multiply(m::Parameter(1), m::Parameter(2))))); } // Test that A/exp(B) is simplified to A*exp(-B). @@ -539,14 +659,14 @@ TEST_F(AlgebraicSimplifierTest, DivOfExp) { auto computation = m->AddEntryComputation(builder.Build()); EXPECT_THAT(computation->root_instruction(), - op::Divide(param0, op::Exp(param1))); + GmockMatch(m::Divide(m::Parameter(0), m::Exp(m::Parameter(1))))); - AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false, - non_bitcasting_callback()); + AlgebraicSimplifier simplifier(default_options_); ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie()); EXPECT_THAT(computation->root_instruction(), - op::Multiply(param0, op::Exp(op::Negate(param1)))); + GmockMatch(m::Multiply(m::Parameter(0), + m::Exp(m::Negate(m::Parameter(1)))))); } // Test that A/pow(B,C) is simplified to A*pow(B,-C). @@ -567,15 +687,18 @@ TEST_F(AlgebraicSimplifierTest, DivOfPower) { auto computation = m->AddEntryComputation(builder.Build()); - EXPECT_THAT(computation->root_instruction(), - op::Divide(param0, op::Power(param1, param2))); + EXPECT_THAT( + computation->root_instruction(), + GmockMatch(m::Divide(m::Parameter(0), + m::Power(m::Parameter(1), m::Parameter(2))))); - AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false, - non_bitcasting_callback()); + AlgebraicSimplifier simplifier(default_options_); ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie()); EXPECT_THAT(computation->root_instruction(), - op::Multiply(param0, op::Power(param1, op::Negate(param2)))); + GmockMatch(m::Multiply( + m::Parameter(0), + m::Power(m::Parameter(1), m::Negate(m::Parameter(2)))))); } // Test that broadcasting is done on the right step when simplifying A/pow(B,C) @@ -597,15 +720,18 @@ TEST_F(AlgebraicSimplifierTest, DivOfBroadcastingPower) { auto computation = m->AddEntryComputation(builder.Build()); - EXPECT_THAT(computation->root_instruction(), - op::Divide(param0, op::Power(param1, param2))); + EXPECT_THAT( + computation->root_instruction(), + GmockMatch(m::Divide(m::Parameter(0), + m::Power(m::Parameter(1), m::Parameter(2))))); - AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false, - non_bitcasting_callback()); + AlgebraicSimplifier simplifier(default_options_); ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie()); ASSERT_THAT(computation->root_instruction(), - op::Multiply(param0, op::Power(param1, op::Negate(param2)))); + GmockMatch(m::Multiply( + m::Parameter(0), + m::Power(m::Parameter(1), m::Negate(m::Parameter(2)))))); } // A / Const => A * InvertedConst @@ -623,12 +749,11 @@ TEST_F(AlgebraicSimplifierTest, DivideByConstant) { auto computation = m->AddEntryComputation(builder.Build()); - AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false, - non_bitcasting_callback()); + AlgebraicSimplifier simplifier(default_options_); ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie()); EXPECT_THAT(computation->root_instruction(), - op::Multiply(param0, op::Constant())); + GmockMatch(m::Multiply(m::Parameter(0), m::Constant()))); } // pow(pow(A, X), Y) => pow(A, X*Y) @@ -648,11 +773,12 @@ TEST_F(AlgebraicSimplifierTest, PowerOfPower) { inner_power, exp2)); auto computation = m->AddEntryComputation(builder.Build()); - AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false, - non_bitcasting_callback()); + AlgebraicSimplifier simplifier(default_options_); ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie()); - EXPECT_THAT(computation->root_instruction(), - op::Power(base, op::Multiply(exp1, exp2))); + EXPECT_THAT( + computation->root_instruction(), + GmockMatch(m::Power(m::Op().Is(base), + m::Multiply(m::Op().Is(exp1), m::Op().Is(exp2))))); } // Don't simplify pow(pow(A, X), Y) => pow(A, X*Y) if X and Y are complex @@ -673,8 +799,7 @@ TEST_F(AlgebraicSimplifierTest, PowerOfPowerComplex) { inner_power, exp2)); m->AddEntryComputation(builder.Build()); - AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false, - non_bitcasting_callback()); + AlgebraicSimplifier simplifier(default_options_); ASSERT_FALSE(simplifier.Run(m.get()).ValueOrDie()); } @@ -693,8 +818,7 @@ TEST_F(AlgebraicSimplifierTest, DivOneScalar) { auto computation = m->AddEntryComputation(builder.Build()); HloInstruction* root = computation->root_instruction(); EXPECT_EQ(root, div); - AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false, - non_bitcasting_callback()); + AlgebraicSimplifier simplifier(default_options_); ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie()); root = computation->root_instruction(); EXPECT_EQ(root, param0); @@ -715,8 +839,7 @@ TEST_F(AlgebraicSimplifierTest, DivOneArray) { auto computation = m->AddEntryComputation(builder.Build()); HloInstruction* root = computation->root_instruction(); EXPECT_EQ(root, div); - AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false, - non_bitcasting_callback()); + AlgebraicSimplifier simplifier(default_options_); ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie()); root = computation->root_instruction(); EXPECT_EQ(root, param0); @@ -740,8 +863,7 @@ TEST_F(AlgebraicSimplifierTest, ComplexOfRealImagC) { auto computation = m->AddEntryComputation(builder.Build()); HloInstruction* root = computation->root_instruction(); EXPECT_EQ(root, cplx); - AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false, - non_bitcasting_callback()); + AlgebraicSimplifier simplifier(default_options_); ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie()); root = computation->root_instruction(); EXPECT_EQ(root, param0); @@ -765,8 +887,7 @@ TEST_F(AlgebraicSimplifierTest, RealOfComplex) { auto computation = m->AddEntryComputation(builder.Build()); HloInstruction* root = computation->root_instruction(); EXPECT_EQ(root, real); - AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false, - non_bitcasting_callback()); + AlgebraicSimplifier simplifier(default_options_); ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie()); root = computation->root_instruction(); EXPECT_EQ(root, param0); @@ -790,8 +911,7 @@ TEST_F(AlgebraicSimplifierTest, ImagOfComplex) { auto computation = m->AddEntryComputation(builder.Build()); HloInstruction* root = computation->root_instruction(); EXPECT_EQ(root, imag); - AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false, - non_bitcasting_callback()); + AlgebraicSimplifier simplifier(default_options_); ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie()); root = computation->root_instruction(); EXPECT_EQ(root, param1); @@ -818,11 +938,10 @@ TEST_F(AlgebraicSimplifierTest, SelectMakeTuple) { auto computation = m->AddEntryComputation(builder.Build()); HloInstruction* root = computation->root_instruction(); EXPECT_EQ(root, add); - AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false, - non_bitcasting_callback()); + AlgebraicSimplifier simplifier(default_options_); ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie()); root = computation->root_instruction(); - EXPECT_THAT(root, op::Add(param1, param2)); + EXPECT_THAT(root, GmockMatch(m::Add(m::Parameter(1), m::Parameter(2)))); } // Test that exp(A)/exp(B) is simplified to exp(A-B) @@ -843,15 +962,16 @@ TEST_F(AlgebraicSimplifierTest, ExpDiv) { auto computation = m->AddEntryComputation(builder.Build()); - EXPECT_THAT(computation->root_instruction(), - op::Divide(op::Exp(param0), op::Exp(param1))); + EXPECT_THAT( + computation->root_instruction(), + GmockMatch(m::Divide(m::Exp(m::Parameter(0)), m::Exp(m::Parameter(1))))); - AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false, - non_bitcasting_callback()); + AlgebraicSimplifier simplifier(default_options_); ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie()); - EXPECT_THAT(computation->root_instruction(), - op::Exp(op::Subtract(param0, param1))); + EXPECT_THAT( + computation->root_instruction(), + GmockMatch(m::Exp(m::Subtract(m::Parameter(0), m::Parameter(1))))); } // Test that exp(A)*exp(B) is simplified to exp(A+B) @@ -873,14 +993,14 @@ TEST_F(AlgebraicSimplifierTest, ExpMul) { auto computation = m->AddEntryComputation(builder.Build()); EXPECT_THAT(computation->root_instruction(), - op::Multiply(op::Exp(param0), op::Exp(param1))); + GmockMatch(m::Multiply(m::Exp(m::Parameter(0)), + m::Exp(m::Parameter(1))))); - AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false, - non_bitcasting_callback()); + AlgebraicSimplifier simplifier(default_options_); ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie()); EXPECT_THAT(computation->root_instruction(), - op::Exp(op::Add(param0, param1))); + GmockMatch(m::Exp(m::Add(m::Parameter(0), m::Parameter(1))))); } // Test that pow(exp(A), B) is simplified to exp(A*B) @@ -900,14 +1020,14 @@ TEST_F(AlgebraicSimplifierTest, PowExp) { auto computation = m->AddEntryComputation(builder.Build()); EXPECT_THAT(computation->root_instruction(), - op::Power(op::Exp(param0), param1)); + GmockMatch(m::Power(m::Exp(m::Parameter(0)), m::Parameter(1)))); - AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false, - non_bitcasting_callback()); + AlgebraicSimplifier simplifier(default_options_); ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie()); - EXPECT_THAT(computation->root_instruction(), - op::Exp(op::Multiply(param0, param1))); + EXPECT_THAT( + computation->root_instruction(), + GmockMatch(m::Exp(m::Multiply(m::Parameter(0), m::Parameter(1))))); } // Test that ln(pow(A, B)) is simplified to ln(A)*B @@ -927,14 +1047,14 @@ TEST_F(AlgebraicSimplifierTest, LnPow) { auto computation = m->AddEntryComputation(builder.Build()); EXPECT_THAT(computation->root_instruction(), - op::Log(op::Power(param0, param1))); + GmockMatch(m::Log(m::Power(m::Parameter(0), m::Parameter(1))))); - AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false, - non_bitcasting_callback()); + AlgebraicSimplifier simplifier(default_options_); ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie()); - EXPECT_THAT(computation->root_instruction(), - op::Multiply(op::Log(param0), param1)); + EXPECT_THAT( + computation->root_instruction(), + GmockMatch(m::Multiply(m::Log(m::Parameter(0)), m::Parameter(1)))); } // Test that ln(exp(A)) is simplified to A @@ -951,10 +1071,10 @@ TEST_F(AlgebraicSimplifierTest, LnExp) { auto computation = m->AddEntryComputation(builder.Build()); - EXPECT_THAT(computation->root_instruction(), op::Log(op::Exp(param0))); + EXPECT_THAT(computation->root_instruction(), + GmockMatch(m::Log(m::Exp(m::Parameter(0))))); - AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false, - non_bitcasting_callback()); + AlgebraicSimplifier simplifier(default_options_); ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie()); EXPECT_EQ(computation->root_instruction(), param0); @@ -981,13 +1101,14 @@ TEST_F(AlgebraicSimplifierTest, LnExpDiv) { auto computation = m->AddEntryComputation(builder.Build()); EXPECT_THAT(computation->root_instruction(), - op::Log(op::Divide(op::Exp(param0), op::Exp(param1)))); + GmockMatch(m::Log(m::Divide(m::Exp(m::Parameter(0)), + m::Exp(m::Parameter(1)))))); - AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false, - non_bitcasting_callback()); + AlgebraicSimplifier simplifier(default_options_); ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie()); - EXPECT_THAT(computation->root_instruction(), op::Subtract(param0, param1)); + EXPECT_THAT(computation->root_instruction(), + GmockMatch(m::Subtract(m::Parameter(0), m::Parameter(1)))); } // Test that pow(A, 0) where A is a scalar is simplified to the scalar @@ -1005,14 +1126,14 @@ TEST_F(AlgebraicSimplifierTest, Pow0Scalar) { auto computation = m->AddEntryComputation(builder.Build()); - EXPECT_THAT(computation->root_instruction(), op::Power(param0, zero)); + EXPECT_THAT(computation->root_instruction(), + GmockMatch(m::Power(m::Parameter(0), m::Op().Is(zero)))); - AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false, - non_bitcasting_callback()); + AlgebraicSimplifier simplifier(default_options_); ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie()); HloInstruction* root = computation->root_instruction(); - EXPECT_THAT(root, op::Constant()); + EXPECT_THAT(root, GmockMatch(m::Constant())); EXPECT_EQ(root->literal().GetFirstElement(), 1); } @@ -1030,14 +1151,14 @@ TEST_F(AlgebraicSimplifierTest, Pow0Vector) { auto computation = m->AddEntryComputation(builder.Build()); - EXPECT_THAT(computation->root_instruction(), op::Power(param0, zero)); + EXPECT_THAT(computation->root_instruction(), + GmockMatch(m::Power(m::Parameter(0), m::Op().Is(zero)))); - AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false, - non_bitcasting_callback()); + AlgebraicSimplifier simplifier(default_options_); ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie()); HloInstruction* root = computation->root_instruction(); - EXPECT_THAT(root, op::Broadcast()); + EXPECT_THAT(root, GmockMatch(m::Broadcast())); EXPECT_TRUE(ShapeUtil::Equal(root->shape(), r1f32)) << ShapeUtil::HumanString(root->shape()); EXPECT_EQ(root->dimensions().size(), 0); @@ -1059,10 +1180,10 @@ TEST_F(AlgebraicSimplifierTest, Pow1) { auto computation = m->AddEntryComputation(builder.Build()); - EXPECT_THAT(computation->root_instruction(), op::Power(param0, one)); + EXPECT_THAT(computation->root_instruction(), + GmockMatch(m::Power(m::Parameter(0), m::Op().Is(one)))); - AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false, - non_bitcasting_callback()); + AlgebraicSimplifier simplifier(default_options_); ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie()); EXPECT_EQ(computation->root_instruction(), param0); @@ -1082,13 +1203,14 @@ TEST_F(AlgebraicSimplifierTest, Pow2) { auto computation = m->AddEntryComputation(builder.Build()); - EXPECT_THAT(computation->root_instruction(), op::Power(param0, two)); + EXPECT_THAT(computation->root_instruction(), + GmockMatch(m::Power(m::Parameter(0), m::Op().Is(two)))); - AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false, - non_bitcasting_callback()); + AlgebraicSimplifier simplifier(default_options_); ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie()); - EXPECT_THAT(computation->root_instruction(), op::Multiply(param0, param0)); + EXPECT_THAT(computation->root_instruction(), + GmockMatch(m::Multiply(m::Parameter(0), m::Parameter(0)))); } // Test that pow(A, -1) is simplified to 1/A. @@ -1105,14 +1227,14 @@ TEST_F(AlgebraicSimplifierTest, PowNegative1) { auto computation = m->AddEntryComputation(builder.Build()); - EXPECT_THAT(computation->root_instruction(), op::Power(param0, negative_one)); + EXPECT_THAT(computation->root_instruction(), + GmockMatch(m::Power(m::Parameter(0), m::Op().Is(negative_one)))); - AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false, - non_bitcasting_callback()); + AlgebraicSimplifier simplifier(default_options_); ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie()); HloInstruction* root = computation->root_instruction(); - EXPECT_THAT(root, op::Divide(op::Broadcast(), param0)); + EXPECT_THAT(root, GmockMatch(m::Divide(m::Broadcast(), m::Parameter(0)))); EXPECT_EQ(root->operand(0)->opcode(), HloOpcode::kBroadcast); EXPECT_EQ(root->operand(0)->operand(0)->literal().GetFirstElement(), 1); @@ -1153,13 +1275,12 @@ TEST_F(AlgebraicSimplifierTest, ZeroSizedConvolution) { ShapeUtil::MakeShape(F32, {3, 3, 3}), lhs, rhs, /*feature_group_count=*/1, window, dnums, DefaultPrecisionConfig(2))); m->AddEntryComputation(builder.Build()); - HloPassFix simplifier(/*is_layout_sensitive=*/false, - non_bitcasting_callback()); + HloPassFix simplifier(default_options_); EXPECT_THAT(m->entry_computation()->root_instruction(), - op::Convolution(lhs, rhs)); + GmockMatch(m::Convolution(m::Op().Is(lhs), m::Op().Is(rhs)))); ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie()); EXPECT_THAT(m->entry_computation()->root_instruction(), - op::Broadcast(op::Constant())); + GmockMatch(m::Broadcast(m::Constant()))); } TEST_F(AlgebraicSimplifierTest, ZeroSizedReduceWindow) { @@ -1196,13 +1317,12 @@ TEST_F(AlgebraicSimplifierTest, ZeroSizedReduceWindow) { HloInstruction::CreateConstant(LiteralUtil::CreateR0(0.0f))), window, add_computation)); m->AddEntryComputation(builder.Build()); - HloPassFix simplifier(/*is_layout_sensitive=*/false, - non_bitcasting_callback()); + HloPassFix simplifier(default_options_); EXPECT_THAT(m->entry_computation()->root_instruction(), - op::ReduceWindow(param, op::Constant())); + GmockMatch(m::ReduceWindow(m::Parameter(0), m::Constant()))); ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie()); EXPECT_THAT(m->entry_computation()->root_instruction(), - op::Broadcast(op::Constant())); + GmockMatch(m::Broadcast(m::Constant()))); } TEST_F(AlgebraicSimplifierTest, ZeroSizedPad) { @@ -1225,12 +1345,11 @@ TEST_F(AlgebraicSimplifierTest, ZeroSizedPad) { padding)); m->AddEntryComputation(builder.Build()); EXPECT_THAT(m->entry_computation()->root_instruction(), - op::Pad(param, op::Constant())); - HloPassFix simplifier(/*is_layout_sensitive=*/false, - non_bitcasting_callback()); + GmockMatch(m::Pad(m::Parameter(0), m::Constant()))); + HloPassFix simplifier(default_options_); ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie()); EXPECT_THAT(m->entry_computation()->root_instruction(), - op::Broadcast(op::Constant())); + GmockMatch(m::Broadcast(m::Constant()))); } TEST_F(AlgebraicSimplifierTest, ReshapeBroadcast) { @@ -1251,10 +1370,9 @@ TEST_F(AlgebraicSimplifierTest, ReshapeBroadcast) { m->AddEntryComputation(std::move(computation)); EXPECT_THAT(m->entry_computation()->root_instruction(), - op::Reshape(op::Broadcast(op::Reshape(op)))); + GmockMatch(m::Reshape(m::Broadcast(m::Reshape(m::Op().Is(op)))))); - HloPassFix simplifier(/*is_layout_sensitive=*/false, - non_bitcasting_callback()); + HloPassFix simplifier(default_options_); ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie()); EXPECT_THAT(m->entry_computation()->root_instruction(), op); @@ -1271,10 +1389,10 @@ TEST_F(AlgebraicSimplifierTest, ConvertBetweenSameType) { auto computation = m->AddEntryComputation(builder.Build()); - EXPECT_THAT(computation->root_instruction(), op::Convert(input)); + EXPECT_THAT(computation->root_instruction(), + GmockMatch(m::Convert(m::Op().Is(input)))); - AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false, - non_bitcasting_callback()); + AlgebraicSimplifier simplifier(default_options_); ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie()); EXPECT_THAT(computation->root_instruction(), input); @@ -1292,10 +1410,10 @@ TEST_F(AlgebraicSimplifierTest, RemoveCopy) { auto computation = m->AddEntryComputation(builder.Build()); - EXPECT_THAT(computation->root_instruction(), op::Copy(param0)); + EXPECT_THAT(computation->root_instruction(), + GmockMatch(m::Copy(m::Parameter(0)))); - AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false, - non_bitcasting_callback()); + AlgebraicSimplifier simplifier(default_options_); ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie()); EXPECT_THAT(computation->root_instruction(), param0); @@ -1314,19 +1432,24 @@ TEST_F(AlgebraicSimplifierTest, CopyEqualsBitcast) { *copy->mutable_shape()->mutable_layout() = LayoutUtil::MakeLayout({1, 2, 0, 3}); auto computation = m->AddEntryComputation(builder.Build()); - EXPECT_THAT(computation->root_instruction(), op::Copy(param)); + EXPECT_THAT(computation->root_instruction(), + GmockMatch(m::Copy(m::Parameter(0)))); - AlgebraicSimplifier simplifier1(/*is_layout_sensitive=*/true, - non_bitcasting_callback()); + AlgebraicSimplifierOptions options(non_bitcasting_callback()); + options.set_is_layout_sensitive(true); + AlgebraicSimplifier simplifier1(options); ASSERT_FALSE(simplifier1.Run(m.get()).ValueOrDie()); // Verify that the copy is not replaced. - EXPECT_THAT(computation->root_instruction(), op::Copy(param)); + EXPECT_THAT(computation->root_instruction(), + GmockMatch(m::Copy(m::Parameter(0)))); - AlgebraicSimplifier simplifier2(/*is_layout_sensitive=*/true, - bitcasting_callback()); + AlgebraicSimplifierOptions options2(bitcasting_callback()); + options2.set_is_layout_sensitive(true); + AlgebraicSimplifier simplifier2(options2); ASSERT_TRUE(simplifier2.Run(m.get()).ValueOrDie()); // Verify that the copy is replaced. - EXPECT_THAT(computation->root_instruction(), op::Bitcast(param)); + EXPECT_THAT(computation->root_instruction(), + GmockMatch(m::Bitcast(m::Parameter(0)))); } // Test that unary concatenates are removed. @@ -1341,10 +1464,10 @@ TEST_F(AlgebraicSimplifierTest, RemoveUnaryConcatenate) { auto computation = m->AddEntryComputation(builder.Build()); - EXPECT_THAT(computation->root_instruction(), op::Concatenate(param0)); + EXPECT_THAT(computation->root_instruction(), + GmockMatch(m::Concatenate(m::Parameter(0)))); - AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false, - non_bitcasting_callback()); + AlgebraicSimplifier simplifier(default_options_); ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie()); EXPECT_THAT(computation->root_instruction(), param0); @@ -1371,16 +1494,17 @@ TEST_F(AlgebraicSimplifierTest, RemoveEmptyConcatenateOperands) { auto computation = m->AddEntryComputation(builder.Build()); - EXPECT_THAT( - computation->root_instruction(), - op::Concatenate(empty_literal, param0, param0, empty_slice, param1)); + EXPECT_THAT(computation->root_instruction(), + GmockMatch(m::Concatenate( + m::Op().Is(empty_literal), m::Parameter(0), m::Parameter(0), + m::Op().Is(empty_slice), m::Parameter(1)))); - AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false, - non_bitcasting_callback()); + AlgebraicSimplifier simplifier(default_options_); ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie()); EXPECT_THAT(computation->root_instruction(), - op::Concatenate(param0, param0, param1)); + GmockMatch(m::Concatenate(m::Parameter(0), m::Parameter(0), + m::Parameter(1)))); } // Test that reduce of concat is simplified. @@ -1423,14 +1547,14 @@ TEST_F(AlgebraicSimplifierTest, SimplifyReduceOfConcat) { auto computation = m->AddEntryComputation(builder.Build()); - AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false, - non_bitcasting_callback()); + AlgebraicSimplifier simplifier(default_options_); ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie()); EXPECT_THAT( computation->root_instruction(), - op::Map(op::Map(op::Reduce(param0, zero), op::Reduce(param1, zero)), - op::Reduce(param2, zero))); + GmockMatch(m::Map(m::Map(m::Reduce(m::Parameter(0), m::Op().Is(zero)), + m::Reduce(m::Parameter(1), m::Op().Is(zero))), + m::Reduce(m::Parameter(2), m::Op().Is(zero))))); } // Test a concatenate with only empty operands is removed. @@ -1453,10 +1577,10 @@ TEST_F(AlgebraicSimplifierTest, OnlyEmptyConcatenateOperands) { auto computation = m->AddEntryComputation(builder.Build()); EXPECT_THAT(computation->root_instruction(), - op::Concatenate(empty_literal, empty_slice)); + GmockMatch(m::Concatenate(m::Op().Is(empty_literal), + m::Op().Is(empty_slice)))); - AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false, - non_bitcasting_callback()); + AlgebraicSimplifier simplifier(default_options_); ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie()); EXPECT_EQ(computation->root_instruction(), empty_literal); @@ -1479,10 +1603,80 @@ TEST_F(AlgebraicSimplifierTest, ConcatenateOfBroadcastBecomesPad) { auto computation = m->AddEntryComputation(builder.Build()); - AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false, - non_bitcasting_callback()); + AlgebraicSimplifier simplifier(default_options_); ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie()); - EXPECT_THAT(computation->root_instruction(), op::Pad(param0, param1)); + EXPECT_THAT(computation->root_instruction(), + GmockMatch(m::Pad(m::Parameter(0), m::Parameter(1)))); +} + +TEST_F(AlgebraicSimplifierTest, SimplifyConcatenateOfSlices) { + auto m = CreateNewVerifiedModule(); + Shape r2f32 = ShapeUtil::MakeShape(F32, {100, 99}); + Shape concat_shape = ShapeUtil::MakeShape(F32, {50, 80}); + HloComputation::Builder builder(TestName()); + HloInstruction* param0 = builder.AddInstruction( + HloInstruction::CreateParameter(0, r2f32, "param0")); + HloInstruction* param1 = builder.AddInstruction( + HloInstruction::CreateParameter(1, r2f32, "param1")); + + HloInstruction* slice0 = builder.AddInstruction(HloInstruction::CreateSlice( + ShapeUtil::MakeShape(F32, {50, 10}), param0, /*start_indices=*/{0, 0}, + /*limit_indices=*/{50, 10}, /*strides=*/{1, 1})); + + // Cannot merge 'slice0' and 'slice1' because of different start indices in + // dimension 0. + HloInstruction* slice1 = builder.AddInstruction(HloInstruction::CreateSlice( + ShapeUtil::MakeShape(F32, {50, 10}), param0, /*start_indices=*/{50, 10}, + /*limit_indices=*/{100, 20}, /*strides=*/{1, 1})); + + // Cannot merge 'slice1' and 'slice2' because of stride in dimension 2. + HloInstruction* slice2 = builder.AddInstruction(HloInstruction::CreateSlice( + ShapeUtil::MakeShape(F32, {50, 10}), param0, /*start_indices=*/{50, 20}, + /*limit_indices=*/{100, 40}, /*strides=*/{1, 2})); + + // Cannot merge 'slice2' and 'slice3' because of stride in dimension 2. + HloInstruction* slice3 = builder.AddInstruction(HloInstruction::CreateSlice( + ShapeUtil::MakeShape(F32, {50, 10}), param0, /*start_indices=*/{50, 40}, + /*limit_indices=*/{100, 50}, /*strides=*/{1, 1})); + + // Can merge 'slice3' and 'slice4'. + HloInstruction* slice4 = builder.AddInstruction(HloInstruction::CreateSlice( + ShapeUtil::MakeShape(F32, {50, 10}), param0, /*start_indices=*/{50, 50}, + /*limit_indices=*/{100, 60}, /*strides=*/{1, 1})); + + // Can merge 'slice4' and 'slice5'. + HloInstruction* slice5 = builder.AddInstruction(HloInstruction::CreateSlice( + ShapeUtil::MakeShape(F32, {50, 10}), param0, /*start_indices=*/{50, 60}, + /*limit_indices=*/{100, 70}, /*strides=*/{1, 1})); + + // Cannot merge 'slice5' and 'slice6' because of overlap. + HloInstruction* slice6 = builder.AddInstruction(HloInstruction::CreateSlice( + ShapeUtil::MakeShape(F32, {50, 10}), param0, /*start_indices=*/{50, 69}, + /*limit_indices=*/{100, 79}, /*strides=*/{1, 1})); + + // Cannot merge 'slice6' and 'slice7' because of slicing from a different + // parameter. + HloInstruction* slice7 = builder.AddInstruction(HloInstruction::CreateSlice( + ShapeUtil::MakeShape(F32, {50, 10}), param1, /*start_indices=*/{50, 79}, + /*limit_indices=*/{100, 89}, /*strides=*/{1, 1})); + + builder.AddInstruction(HloInstruction::CreateConcatenate( + concat_shape, + {slice0, slice1, slice2, slice3, slice4, slice5, slice6, slice7}, 1)); + auto computation = m->AddEntryComputation(builder.Build()); + + AlgebraicSimplifier simplifier(default_options_); + ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie()); + auto s = m::Slice(m::Parameter(0)); + EXPECT_THAT( + computation->root_instruction(), + GmockMatch(m::Concatenate(s, s, s, s, s, m::Slice(m::Parameter(1))))); + // The operand 3 should be a merge of 'slice3', 'slice4' and 'slice5', so its + // shape should have dimensions {50, 30}. + EXPECT_TRUE( + ShapeUtil::Equal(computation->root_instruction()->operand(3)->shape(), + ShapeUtil::MakeShape(F32, {50, 30}))); + EXPECT_EQ(computation->root_instruction()->operand(3)->slice_starts(1), 40); } // Test that a simplification which changes layouts is not performed if layout @@ -1502,14 +1696,17 @@ TEST_F(AlgebraicSimplifierTest, CopyWithDifferentLayout) { *param0->mutable_shape()->mutable_layout() = LayoutUtil::MakeLayout({0, 1}); *copy->mutable_shape()->mutable_layout() = LayoutUtil::MakeLayout({1, 0}); - EXPECT_THAT(computation->root_instruction(), op::Copy(param0)); + EXPECT_THAT(computation->root_instruction(), + GmockMatch(m::Copy(m::Parameter(0)))); - AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/true, - non_bitcasting_callback()); + AlgebraicSimplifierOptions options(non_bitcasting_callback()); + options.set_is_layout_sensitive(true); + AlgebraicSimplifier simplifier(options); EXPECT_FALSE(simplifier.Run(m.get()).ValueOrDie()); // Copy has not been removed. - EXPECT_THAT(computation->root_instruction(), op::Copy(param0)); + EXPECT_THAT(computation->root_instruction(), + GmockMatch(m::Copy(m::Parameter(0)))); } // Test that a simplification which preserves layouts is performed if layout @@ -1529,10 +1726,12 @@ TEST_F(AlgebraicSimplifierTest, CopyWithSameLayout) { *param0->mutable_shape()->mutable_layout() = LayoutUtil::MakeLayout({0, 1}); *copy->mutable_shape()->mutable_layout() = LayoutUtil::MakeLayout({0, 1}); - EXPECT_THAT(computation->root_instruction(), op::Copy(param0)); + EXPECT_THAT(computation->root_instruction(), + GmockMatch(m::Copy(m::Parameter(0)))); - AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/true, - non_bitcasting_callback()); + AlgebraicSimplifierOptions options(non_bitcasting_callback()); + options.set_is_layout_sensitive(true); + AlgebraicSimplifier simplifier(options); ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie()); // Copy has been removed. @@ -1557,14 +1756,17 @@ TEST_F(AlgebraicSimplifierTest, NoBitcastAdded) { auto computation = m->AddEntryComputation(builder.Build()); - EXPECT_THAT(computation->root_instruction(), op::Reshape(param0)); + EXPECT_THAT(computation->root_instruction(), + GmockMatch(m::Reshape(m::Parameter(0)))); - AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/true, - non_bitcasting_callback()); + AlgebraicSimplifierOptions options(non_bitcasting_callback()); + options.set_is_layout_sensitive(true); + AlgebraicSimplifier simplifier(options); EXPECT_FALSE(simplifier.Run(m.get()).ValueOrDie()); // Reshape is not replaced with a bitcast. - EXPECT_THAT(computation->root_instruction(), op::Reshape(param0)); + EXPECT_THAT(computation->root_instruction(), + GmockMatch(m::Reshape(m::Parameter(0)))); } // Test transforming reshapes and transposes of rng. @@ -1588,13 +1790,13 @@ TEST_F(AlgebraicSimplifierTest, ReshapeOfTransposeOfRngToRng) { auto computation = m->AddEntryComputation(builder.Build()); - AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false, - bitcasting_callback()); + AlgebraicSimplifier simplifier( + (AlgebraicSimplifierOptions(bitcasting_callback()))); EXPECT_TRUE(simplifier.Run(m.get()).ValueOrDie()); - // Verify that that reshape(transpose(rng)) is replace by a single rng of the + // Verify that reshape(transpose(rng)) is replace by a single rng of the // same shape as the reshape. - EXPECT_THAT(computation->root_instruction(), op::Rng()); + EXPECT_THAT(computation->root_instruction(), GmockMatch(m::Rng())); EXPECT_TRUE(ShapeUtil::Equal(computation->root_instruction()->shape(), reshape_shape)); } @@ -1636,17 +1838,20 @@ TEST_F(AlgebraicSimplifierTest, ReshapeReplacedWithBitcast) { auto computation = m->AddEntryComputation(builder.Build()); EXPECT_THAT(computation->root_instruction(), - op::Tuple(transformable_reshape, dimensions_wrong_reshape, - layout_wrong_reshape)); + GmockMatch(m::Tuple(m::Op().Is(transformable_reshape), + m::Op().Is(dimensions_wrong_reshape), + m::Op().Is(layout_wrong_reshape)))); - AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/true, - bitcasting_callback()); + AlgebraicSimplifierOptions options(bitcasting_callback()); + options.set_is_layout_sensitive(true); + AlgebraicSimplifier simplifier(options); simplifier.Run(m.get()).ValueOrDie(); // Verify that only the first reshape is replaced. EXPECT_THAT( computation->root_instruction(), - op::Tuple(op::Bitcast(), dimensions_wrong_reshape, layout_wrong_reshape)); + GmockMatch(m::Tuple(m::Bitcast(), m::Op().Is(dimensions_wrong_reshape), + m::Op().Is(layout_wrong_reshape)))); } // Regression test for a bug where if we failed to sink a reshape, we'd set the @@ -1667,8 +1872,8 @@ TEST_F(AlgebraicSimplifierTest, FailureToSinkReshapeDoesntAffectChangedBit) { builder.AddInstruction( HloInstruction::CreateReshape(ShapeUtil::MakeShape(F32, {4}), add)); - AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false, - bitcasting_callback()); + AlgebraicSimplifier simplifier( + (AlgebraicSimplifierOptions(bitcasting_callback()))); m->AddEntryComputation(builder.Build()); EXPECT_TRUE(simplifier.Run(m.get()).ValueOrDie()); } @@ -1692,8 +1897,8 @@ TEST_F(AlgebraicSimplifierTest, FailureToSinkBroadcastDoesntAffectChangedBit) { HloInstruction::CreateBroadcast(ShapeUtil::MakeShape(F32, {2, 2, 2}), add, /*broadcast_dimensions=*/{0, 1})); - AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false, - bitcasting_callback()); + AlgebraicSimplifier simplifier( + (AlgebraicSimplifierOptions(bitcasting_callback()))); m->AddEntryComputation(builder.Build()); EXPECT_TRUE(simplifier.Run(m.get()).ValueOrDie()); } @@ -1715,14 +1920,17 @@ TEST_F(AlgebraicSimplifierTest, TransposeEqualsBitcast1) { auto computation = m->AddEntryComputation(builder.Build()); - EXPECT_THAT(computation->root_instruction(), op::Transpose(param)); + EXPECT_THAT(computation->root_instruction(), + GmockMatch(m::Transpose(m::Parameter(0)))); - AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/true, - bitcasting_callback()); + AlgebraicSimplifierOptions options(bitcasting_callback()); + options.set_is_layout_sensitive(true); + AlgebraicSimplifier simplifier(options); ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie()); // Verify that the reshape is replaced. - EXPECT_THAT(computation->root_instruction(), op::Bitcast(param)); + EXPECT_THAT(computation->root_instruction(), + GmockMatch(m::Bitcast(m::Parameter(0)))); } TEST_F(AlgebraicSimplifierTest, TransposeEqualsBitcast2) { @@ -1742,14 +1950,17 @@ TEST_F(AlgebraicSimplifierTest, TransposeEqualsBitcast2) { auto computation = m->AddEntryComputation(builder.Build()); - EXPECT_THAT(computation->root_instruction(), op::Transpose(param)); + EXPECT_THAT(computation->root_instruction(), + GmockMatch(m::Transpose(m::Parameter(0)))); - AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/true, - bitcasting_callback()); + AlgebraicSimplifierOptions options(bitcasting_callback()); + options.set_is_layout_sensitive(true); + AlgebraicSimplifier simplifier(options); ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie()); // Verify that the reshape is replaced. - EXPECT_THAT(computation->root_instruction(), op::Bitcast(param)); + EXPECT_THAT(computation->root_instruction(), + GmockMatch(m::Bitcast(m::Parameter(0)))); } TEST_F(AlgebraicSimplifierTest, ReshapesMerged) { @@ -1769,13 +1980,13 @@ TEST_F(AlgebraicSimplifierTest, ReshapesMerged) { auto computation = m->AddEntryComputation(builder.Build()); EXPECT_THAT(computation->root_instruction(), - op::Reshape(op::Reshape(param0))); + GmockMatch(m::Reshape(m::Reshape(m::Parameter(0))))); - AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false, - non_bitcasting_callback()); + AlgebraicSimplifier simplifier(default_options_); ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie()); - EXPECT_THAT(computation->root_instruction(), op::Reshape(param0)); + EXPECT_THAT(computation->root_instruction(), + GmockMatch(m::Reshape(m::Parameter(0)))); } TEST_F(AlgebraicSimplifierTest, CopiesMerged) { @@ -1796,13 +2007,16 @@ TEST_F(AlgebraicSimplifierTest, CopiesMerged) { auto computation = m->AddEntryComputation(builder.Build()); - EXPECT_THAT(computation->root_instruction(), op::Copy(op::Copy(param0))); + EXPECT_THAT(computation->root_instruction(), + GmockMatch(m::Copy(m::Copy(m::Parameter(0))))); - AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/true, - non_bitcasting_callback()); + AlgebraicSimplifierOptions options(non_bitcasting_callback()); + options.set_is_layout_sensitive(true); + AlgebraicSimplifier simplifier(options); ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie()); - EXPECT_THAT(computation->root_instruction(), op::Copy(param0)); + EXPECT_THAT(computation->root_instruction(), + GmockMatch(m::Copy(m::Parameter(0)))); } TEST_F(AlgebraicSimplifierTest, TransposesMerged) { @@ -1821,13 +2035,14 @@ TEST_F(AlgebraicSimplifierTest, TransposesMerged) { auto computation = m->AddEntryComputation(builder.Build()); - EXPECT_THAT(computation->root_instruction(), op::Transpose(transpose1)); + EXPECT_THAT(computation->root_instruction(), + GmockMatch(m::Transpose(m::Op().Is(transpose1)))); - AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false, - non_bitcasting_callback()); + AlgebraicSimplifier simplifier(default_options_); ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie()); - EXPECT_THAT(computation->root_instruction(), op::Transpose(param0)); + EXPECT_THAT(computation->root_instruction(), + GmockMatch(m::Transpose(m::Parameter(0)))); EXPECT_EQ(std::vector({2, 1, 0}), computation->root_instruction()->dimensions()); } @@ -1846,13 +2061,13 @@ TEST_F(AlgebraicSimplifierTest, ReshapeAndBroadcastMerged) { auto computation = m->AddEntryComputation(builder.Build()); EXPECT_THAT(computation->root_instruction(), - op::Broadcast(op::Reshape(param0))); + GmockMatch(m::Broadcast(m::Reshape(m::Parameter(0))))); - AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false, - non_bitcasting_callback()); + AlgebraicSimplifier simplifier(default_options_); ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie()); - EXPECT_THAT(computation->root_instruction(), op::Broadcast(param0)); + EXPECT_THAT(computation->root_instruction(), + GmockMatch(m::Broadcast(m::Parameter(0)))); } // Test merging broadcast and reshape. @@ -1869,13 +2084,13 @@ TEST_F(AlgebraicSimplifierTest, BroadcastAndReshapeMerged) { auto computation = m->AddEntryComputation(builder.Build()); EXPECT_THAT(computation->root_instruction(), - op::Reshape(op::Broadcast(param0))); + GmockMatch(m::Reshape(m::Broadcast(m::Parameter(0))))); - AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false, - non_bitcasting_callback()); + AlgebraicSimplifier simplifier(default_options_); ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie()); - EXPECT_THAT(computation->root_instruction(), op::Broadcast(param0)); + EXPECT_THAT(computation->root_instruction(), + GmockMatch(m::Broadcast(m::Parameter(0)))); } TEST_F(AlgebraicSimplifierTest, BroadcastAndReshape_1_3x1_3) { @@ -1891,14 +2106,13 @@ TEST_F(AlgebraicSimplifierTest, BroadcastAndReshape_1_3x1_3) { auto computation = m->AddEntryComputation(builder.Build()); EXPECT_THAT(computation->root_instruction(), - op::Reshape(op::Broadcast(param))); + GmockMatch(m::Reshape(m::Broadcast(m::Parameter(0))))); - AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false, - non_bitcasting_callback()); + AlgebraicSimplifier simplifier(default_options_); EXPECT_FALSE(simplifier.Run(m.get()).ValueOrDie()); EXPECT_THAT(computation->root_instruction(), - op::Reshape(op::Broadcast(param))); + GmockMatch(m::Reshape(m::Broadcast(m::Parameter(0))))); } TEST_F(AlgebraicSimplifierTest, BroadcastAndReshape_4_3x2x4_6x1x1x4) { @@ -1914,13 +2128,13 @@ TEST_F(AlgebraicSimplifierTest, BroadcastAndReshape_4_3x2x4_6x1x1x4) { HloComputation* computation = m->AddEntryComputation(builder.Build()); EXPECT_THAT(computation->root_instruction(), - op::Reshape(op::Broadcast(param))); + GmockMatch(m::Reshape(m::Broadcast(m::Parameter(0))))); - AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false, - non_bitcasting_callback()); + AlgebraicSimplifier simplifier(default_options_); ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie()); - EXPECT_THAT(computation->root_instruction(), op::Broadcast(param)); + EXPECT_THAT(computation->root_instruction(), + GmockMatch(m::Broadcast(m::Parameter(0)))); EXPECT_THAT(computation->root_instruction()->dimensions(), ::testing::ElementsAre(3)); } @@ -1938,13 +2152,13 @@ TEST_F(AlgebraicSimplifierTest, BroadcastAndReshape_1_3x2x1_6x1x1x1) { HloComputation* computation = m->AddEntryComputation(builder.Build()); EXPECT_THAT(computation->root_instruction(), - op::Reshape(op::Broadcast(param))); + GmockMatch(m::Reshape(m::Broadcast(m::Parameter(0))))); - AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false, - non_bitcasting_callback()); + AlgebraicSimplifier simplifier(default_options_); ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie()); - EXPECT_THAT(computation->root_instruction(), op::Broadcast(param)); + EXPECT_THAT(computation->root_instruction(), + GmockMatch(m::Broadcast(m::Parameter(0)))); const std::vector broadcast_dims = computation->root_instruction()->dimensions(); EXPECT_EQ(1, broadcast_dims.size()); @@ -1964,14 +2178,13 @@ TEST_F(AlgebraicSimplifierTest, BroadcastAndReshape_4_3x2x4x2_6x8) { HloComputation* computation = m->AddEntryComputation(builder.Build()); EXPECT_THAT(computation->root_instruction(), - op::Reshape(op::Broadcast(param))); + GmockMatch(m::Reshape(m::Broadcast(m::Parameter(0))))); - AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false, - non_bitcasting_callback()); + AlgebraicSimplifier simplifier(default_options_); EXPECT_FALSE(simplifier.Run(m.get()).ValueOrDie()); EXPECT_THAT(computation->root_instruction(), - op::Reshape(op::Broadcast(param))); + GmockMatch(m::Reshape(m::Broadcast(m::Parameter(0))))); } TEST_F(AlgebraicSimplifierTest, IotaAndReshapeMerged) { @@ -1984,13 +2197,13 @@ TEST_F(AlgebraicSimplifierTest, IotaAndReshapeMerged) { auto computation = m->AddEntryComputation(builder.Build()); - EXPECT_THAT(computation->root_instruction(), op::Reshape(op::Iota())); + EXPECT_THAT(computation->root_instruction(), + GmockMatch(m::Reshape(m::Iota()))); - AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false, - non_bitcasting_callback()); + AlgebraicSimplifier simplifier(default_options_); ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie()); - EXPECT_THAT(computation->root_instruction(), op::Iota()); + EXPECT_THAT(computation->root_instruction(), GmockMatch(m::Iota())); EXPECT_TRUE( ShapeUtil::Equal(computation->root_instruction()->shape(), result_shape)); } @@ -2004,14 +2217,13 @@ TEST_F(AlgebraicSimplifierTest, IotaEffectiveScalar) { auto computation = m->AddEntryComputation(builder.Build()); - EXPECT_THAT(computation->root_instruction(), op::Iota()); + EXPECT_THAT(computation->root_instruction(), GmockMatch(m::Iota())); - AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false, - non_bitcasting_callback()); + AlgebraicSimplifier simplifier(default_options_); ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie()); auto root = computation->root_instruction(); - EXPECT_THAT(root, op::Broadcast(op::Constant())); + EXPECT_THAT(root, GmockMatch(m::Broadcast(m::Constant()))); EXPECT_EQ(0.0f, root->operand(0)->literal().GetFirstElement()); EXPECT_TRUE( ShapeUtil::Equal(computation->root_instruction()->shape(), result_shape)); @@ -2027,13 +2239,14 @@ TEST_F(AlgebraicSimplifierTest, IotaAndReshape_1_3x2_6) { auto computation = m->AddEntryComputation(builder.Build()); - EXPECT_THAT(computation->root_instruction(), op::Reshape(op::Iota())); + EXPECT_THAT(computation->root_instruction(), + GmockMatch(m::Reshape(m::Iota()))); - AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false, - non_bitcasting_callback()); + AlgebraicSimplifier simplifier(default_options_); EXPECT_FALSE(simplifier.Run(m.get()).ValueOrDie()); - EXPECT_THAT(computation->root_instruction(), op::Reshape(op::Iota())); + EXPECT_THAT(computation->root_instruction(), + GmockMatch(m::Reshape(m::Iota()))); } TEST_F(AlgebraicSimplifierTest, IotaAndReshape_4_3x2x4_6x1x1x4) { @@ -2046,13 +2259,13 @@ TEST_F(AlgebraicSimplifierTest, IotaAndReshape_4_3x2x4_6x1x1x4) { HloComputation* computation = m->AddEntryComputation(builder.Build()); - EXPECT_THAT(computation->root_instruction(), op::Reshape(op::Iota())); + EXPECT_THAT(computation->root_instruction(), + GmockMatch(m::Reshape(m::Iota()))); - AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false, - non_bitcasting_callback()); + AlgebraicSimplifier simplifier(default_options_); ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie()); - EXPECT_THAT(computation->root_instruction(), op::Iota()); + EXPECT_THAT(computation->root_instruction(), GmockMatch(m::Iota())); EXPECT_EQ(Cast(computation->root_instruction()) ->iota_dimension(), 3); @@ -2068,13 +2281,13 @@ TEST_F(AlgebraicSimplifierTest, IotaAndReshape_1_3x2x2_6x1x1x2) { HloComputation* computation = m->AddEntryComputation(builder.Build()); - EXPECT_THAT(computation->root_instruction(), op::Reshape(op::Iota())); + EXPECT_THAT(computation->root_instruction(), + GmockMatch(m::Reshape(m::Iota()))); - AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false, - non_bitcasting_callback()); + AlgebraicSimplifier simplifier(default_options_); ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie()); - EXPECT_THAT(computation->root_instruction(), op::Iota()); + EXPECT_THAT(computation->root_instruction(), GmockMatch(m::Iota())); const int64 iota_dim = Cast(computation->root_instruction()) ->iota_dimension(); @@ -2091,13 +2304,14 @@ TEST_F(AlgebraicSimplifierTest, IotaAndReshape_4_3x2x4x2_6x8) { HloComputation* computation = m->AddEntryComputation(builder.Build()); - EXPECT_THAT(computation->root_instruction(), op::Reshape(op::Iota())); + EXPECT_THAT(computation->root_instruction(), + GmockMatch(m::Reshape(m::Iota()))); - AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false, - non_bitcasting_callback()); + AlgebraicSimplifier simplifier(default_options_); EXPECT_FALSE(simplifier.Run(m.get()).ValueOrDie()); - EXPECT_THAT(computation->root_instruction(), op::Reshape(op::Iota())); + EXPECT_THAT(computation->root_instruction(), + GmockMatch(m::Reshape(m::Iota()))); } TEST_F(AlgebraicSimplifierTest, RemoveNoopPad) { @@ -2120,10 +2334,10 @@ TEST_F(AlgebraicSimplifierTest, RemoveNoopPad) { auto module = CreateNewVerifiedModule(); HloComputation* computation = module->AddEntryComputation(builder.Build()); - EXPECT_THAT(computation->root_instruction(), op::Pad(param, zero)); + EXPECT_THAT(computation->root_instruction(), + GmockMatch(m::Pad(m::Parameter(0), m::Op().Is(zero)))); - AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false, - non_bitcasting_callback()); + AlgebraicSimplifier simplifier(default_options_); ASSERT_TRUE(simplifier.Run(module.get()).ValueOrDie()); EXPECT_THAT(computation->root_instruction(), param); @@ -2153,8 +2367,7 @@ TEST_F(AlgebraicSimplifierTest, NegativePadding) { auto module = CreateNewVerifiedModule(); HloComputation* computation = module->AddEntryComputation(builder.Build()); - AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false, - non_bitcasting_callback()); + AlgebraicSimplifier simplifier(default_options_); auto has_negative_padding = [](const HloInstruction* pad) { for (auto& padding_dimension : pad->padding_config().dimensions()) { @@ -2166,16 +2379,54 @@ TEST_F(AlgebraicSimplifierTest, NegativePadding) { return false; }; - EXPECT_THAT(computation->root_instruction(), op::Pad(param, zero)); + EXPECT_THAT(computation->root_instruction(), + GmockMatch(m::Pad(m::Parameter(0), m::Op().Is(zero)))); EXPECT_TRUE(has_negative_padding(pad)); ASSERT_TRUE(simplifier.Run(module.get()).ValueOrDie()); - EXPECT_THAT(computation->root_instruction(), op::Slice(op::Pad(param, zero))); + EXPECT_THAT(computation->root_instruction(), + GmockMatch(m::Slice(m::Pad(m::Parameter(0), m::Op().Is(zero))))); EXPECT_FALSE( has_negative_padding(computation->root_instruction()->operand(0))); } +TEST_F(AlgebraicSimplifierTest, TrivialInteriorPadding) { + // Verify that a pad instruction with interior padding on one-sized + // dimensions, removes the interior padding. + HloComputation::Builder builder(TestName()); + HloInstruction* param = + builder.AddInstruction(HloInstruction::CreateParameter( + 0, ShapeUtil::MakeShape(F32, {2, 1}), "param")); + HloInstruction* zero = builder.AddInstruction( + HloInstruction::CreateConstant(LiteralUtil::CreateR0(0.0f))); + PaddingConfig padding; + for (int i = 0; i < 2; ++i) { + auto dimension = padding.add_dimensions(); + dimension->set_edge_padding_low(3); + dimension->set_edge_padding_high(3); + dimension->set_interior_padding(i * 3); + } + HloInstruction* pad = builder.AddInstruction(HloInstruction::CreatePad( + ShapeUtil::MakeShape(F32, {8, 7}), param, zero, padding)); + + auto module = CreateNewVerifiedModule(); + HloComputation* computation = module->AddEntryComputation(builder.Build()); + + AlgebraicSimplifier simplifier(default_options_); + + ASSERT_THAT(computation->root_instruction(), + GmockMatch(m::Pad(m::Parameter(0), m::Op().Is(zero)))); + ASSERT_TRUE(HasInteriorPadding(pad->padding_config())); + + EXPECT_TRUE(simplifier.Run(module.get()).ValueOrDie()); + + EXPECT_THAT(computation->root_instruction(), + GmockMatch(m::Pad(m::Parameter(0), m::Op().Is(zero)))); + EXPECT_FALSE( + HasInteriorPadding(computation->root_instruction()->padding_config())); +} + TEST_F(AlgebraicSimplifierTest, RemoveNoopReshape) { HloComputation::Builder builder(TestName()); HloInstruction* param = @@ -2187,10 +2438,10 @@ TEST_F(AlgebraicSimplifierTest, RemoveNoopReshape) { auto module = CreateNewVerifiedModule(); HloComputation* computation = module->AddEntryComputation(builder.Build()); - EXPECT_THAT(computation->root_instruction(), op::Reshape(param)); + EXPECT_THAT(computation->root_instruction(), + GmockMatch(m::Reshape(m::Parameter(0)))); - AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false, - non_bitcasting_callback()); + AlgebraicSimplifier simplifier(default_options_); ASSERT_TRUE(simplifier.Run(module.get()).ValueOrDie()); EXPECT_THAT(computation->root_instruction(), param); @@ -2210,10 +2461,10 @@ TEST_F(AlgebraicSimplifierTest, RemoveNoopSlice) { auto module = CreateNewVerifiedModule(); HloComputation* computation = module->AddEntryComputation(builder.Build()); - EXPECT_THAT(computation->root_instruction(), op::Slice(param)); + EXPECT_THAT(computation->root_instruction(), + GmockMatch(m::Slice(m::Parameter(0)))); - AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false, - non_bitcasting_callback()); + AlgebraicSimplifier simplifier(default_options_); ASSERT_TRUE(simplifier.Run(module.get()).ValueOrDie()); EXPECT_THAT(computation->root_instruction(), param); @@ -2239,13 +2490,14 @@ TEST_F(AlgebraicSimplifierTest, SliceOfSliceToSlice) { auto module = CreateNewVerifiedModule(); HloComputation* computation = module->AddEntryComputation(builder.Build()); - EXPECT_THAT(computation->root_instruction(), op::Slice(op::Slice(param))); + EXPECT_THAT(computation->root_instruction(), + GmockMatch(m::Slice(m::Slice(m::Parameter(0))))); - AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false, - non_bitcasting_callback()); + AlgebraicSimplifier simplifier(default_options_); ASSERT_TRUE(simplifier.Run(module.get()).ValueOrDie()); - EXPECT_THAT(computation->root_instruction(), op::Slice(param)); + EXPECT_THAT(computation->root_instruction(), + GmockMatch(m::Slice(m::Parameter(0)))); EXPECT_EQ(computation->root_instruction()->slice_starts(0), 3); EXPECT_EQ(computation->root_instruction()->slice_starts(1), 5); EXPECT_EQ(computation->root_instruction()->slice_limits(0), dim0 - 2); @@ -2271,13 +2523,14 @@ TEST_F(AlgebraicSimplifierTest, SliceOfReshapeToReshapeOfSlice) { auto module = CreateNewVerifiedModule(); HloComputation* computation = module->AddEntryComputation(builder.Build()); - EXPECT_THAT(computation->root_instruction(), op::Slice(op::Reshape(param))); + EXPECT_THAT(computation->root_instruction(), + GmockMatch(m::Slice(m::Reshape(m::Parameter(0))))); - AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false, - non_bitcasting_callback()); + AlgebraicSimplifier simplifier(default_options_); ASSERT_TRUE(simplifier.Run(module.get()).ValueOrDie()); - EXPECT_THAT(computation->root_instruction(), op::Reshape(op::Slice(param))); + EXPECT_THAT(computation->root_instruction(), + GmockMatch(m::Reshape(m::Slice(m::Parameter(0))))); } TEST_F(AlgebraicSimplifierTest, SliceOfReshapeUnchanged) { @@ -2296,10 +2549,10 @@ TEST_F(AlgebraicSimplifierTest, SliceOfReshapeUnchanged) { auto module = CreateNewVerifiedModule(); HloComputation* computation = module->AddEntryComputation(builder.Build()); - EXPECT_THAT(computation->root_instruction(), op::Slice(op::Reshape(param))); + EXPECT_THAT(computation->root_instruction(), + GmockMatch(m::Slice(m::Reshape(m::Parameter(0))))); - AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false, - non_bitcasting_callback()); + AlgebraicSimplifier simplifier(default_options_); ASSERT_FALSE(simplifier.Run(module.get()).ValueOrDie()); } @@ -2312,12 +2565,84 @@ TEST_F(AlgebraicSimplifierTest, RemoveNoopSort) { builder.AddInstruction(HloInstruction::CreateSort(keys_shape, 0, keys)); auto module = CreateNewVerifiedModule(); HloComputation* computation = module->AddEntryComputation(builder.Build()); - AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false, - non_bitcasting_callback()); + AlgebraicSimplifier simplifier(default_options_); ASSERT_TRUE(simplifier.Run(module.get()).ValueOrDie()); EXPECT_THAT(computation->root_instruction(), keys); } +TEST_F(AlgebraicSimplifierTest, ReplacePermutationSortWithScatter) { + const char* hlo_string = R"( + HloModule permutation_sort + + ENTRY sort_computation { + keys = f32[64,8732]{1,0} parameter(0) + values = s32[64,8732]{1,0} iota(), iota_dimension=1 + sort = (f32[64,8732]{1,0}, s32[64,8732]{1,0}) sort(keys, values), dimensions={1} + gte = s32[64,8732]{1,0} get-tuple-element(sort), index=1 + ROOT sort2 = (s32[64,8732]{1,0}, s32[64,8732]{1,0}) sort(gte, values), dimensions={1} + } + )"; + TF_ASSERT_OK_AND_ASSIGN(auto module, + ParseAndReturnVerifiedModule(hlo_string)); + + AlgebraicSimplifierOptions options(non_bitcasting_callback()); + options.set_enable_permutation_sort_replacement(true); + AlgebraicSimplifier simplifier(options); + EXPECT_TRUE(simplifier.Run(module.get()).ValueOrDie()); + auto root = module->entry_computation()->root_instruction(); + EXPECT_THAT(root, + GmockMatch(m::Tuple( + m::Iota(), + m::Scatter(m::Iota(), m::Concatenate(m::Iota(), m::Reshape()), + m::Reshape())))); +} + +TEST_F(AlgebraicSimplifierTest, DontReplacePermutationSortIfNonIntegral) { + // Same as ReplacePermutationSortWithScatter except that the iota has F32 + // type. + const char* hlo_string = R"( + HloModule permutation_sort + + ENTRY sort_computation { + keys = f32[64,8732]{1,0} parameter(0) + values = f32[64,8732]{1,0} iota(), iota_dimension=1 + sort = (f32[64,8732]{1,0}, f32[64,8732]{1,0}) sort(keys, values), dimensions={1} + gte = f32[64,8732]{1,0} get-tuple-element(sort), index=1 + ROOT sort2 = (f32[64,8732]{1,0}, f32[64,8732]{1,0}) sort(gte, values), dimensions={1} + } + )"; + TF_ASSERT_OK_AND_ASSIGN(auto module, + ParseAndReturnVerifiedModule(hlo_string)); + + AlgebraicSimplifierOptions options(non_bitcasting_callback()); + options.set_enable_permutation_sort_replacement(true); + AlgebraicSimplifier simplifier(options); + EXPECT_FALSE(simplifier.Run(module.get()).ValueOrDie()); +} + +TEST_F(AlgebraicSimplifierTest, DontReplacePermutationSortWrongDimensions) { + // Same as ReplacePermutationSortWithScatter except that the sort dimensions + // don't match. + const char* hlo_string = R"( + HloModule permutation_sort + + ENTRY sort_computation { + keys = f32[64,8732]{1,0} parameter(0) + values = s32[64,8732]{1,0} iota(), iota_dimension=1 + sort = (f32[64,8732]{1,0}, s32[64,8732]{1,0}) sort(keys, values), dimensions={1} + gte = s32[64,8732]{1,0} get-tuple-element(sort), index=1 + ROOT sort2 = (s32[64,8732]{1,0}, s32[64,8732]{1,0}) sort(gte, values), dimensions={0} + } + )"; + TF_ASSERT_OK_AND_ASSIGN(auto module, + ParseAndReturnVerifiedModule(hlo_string)); + + AlgebraicSimplifierOptions options(non_bitcasting_callback()); + options.set_enable_permutation_sort_replacement(true); + AlgebraicSimplifier simplifier(options); + EXPECT_FALSE(simplifier.Run(module.get()).ValueOrDie()); +} + TEST_F(AlgebraicSimplifierTest, ReplaceEffectiveScalarKeyValueSortWithTuple) { auto builder = HloComputation::Builder(TestName()); @@ -2334,11 +2659,11 @@ TEST_F(AlgebraicSimplifierTest, ReplaceEffectiveScalarKeyValueSortWithTuple) { keys, {values0, values1})); auto module = CreateNewVerifiedModule(); HloComputation* computation = module->AddEntryComputation(builder.Build()); - AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false, - non_bitcasting_callback()); + AlgebraicSimplifier simplifier(default_options_); ASSERT_TRUE(simplifier.Run(module.get()).ValueOrDie()); EXPECT_THAT(computation->root_instruction(), - op::Tuple(keys, values0, values1)); + GmockMatch(m::Tuple(m::Op().Is(keys), m::Op().Is(values0), + m::Op().Is(values1)))); } // Test that A && True is simplified to A @@ -2356,8 +2681,7 @@ TEST_F(AlgebraicSimplifierTest, AndTrue) { auto computation = m->AddEntryComputation(builder.Build()); HloInstruction* root = computation->root_instruction(); EXPECT_EQ(root->opcode(), HloOpcode::kAnd); - AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false, - non_bitcasting_callback()); + AlgebraicSimplifier simplifier(default_options_); ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie()); root = computation->root_instruction(); EXPECT_EQ(root, param0); @@ -2378,8 +2702,7 @@ TEST_F(AlgebraicSimplifierTest, AndTrue2) { auto computation = m->AddEntryComputation(builder.Build()); HloInstruction* root = computation->root_instruction(); EXPECT_EQ(root->opcode(), HloOpcode::kAnd); - AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false, - non_bitcasting_callback()); + AlgebraicSimplifier simplifier(default_options_); ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie()); root = computation->root_instruction(); EXPECT_EQ(root, param0); @@ -2400,8 +2723,7 @@ TEST_F(AlgebraicSimplifierTest, AndFalse) { auto computation = m->AddEntryComputation(builder.Build()); HloInstruction* root = computation->root_instruction(); EXPECT_EQ(root->opcode(), HloOpcode::kAnd); - AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false, - non_bitcasting_callback()); + AlgebraicSimplifier simplifier(default_options_); ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie()); root = computation->root_instruction(); EXPECT_EQ(root, const_false); @@ -2422,8 +2744,7 @@ TEST_F(AlgebraicSimplifierTest, AndFalse2) { auto computation = m->AddEntryComputation(builder.Build()); HloInstruction* root = computation->root_instruction(); EXPECT_EQ(root->opcode(), HloOpcode::kAnd); - AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false, - non_bitcasting_callback()); + AlgebraicSimplifier simplifier(default_options_); ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie()); root = computation->root_instruction(); EXPECT_EQ(root, const_false); @@ -2444,8 +2765,7 @@ TEST_F(AlgebraicSimplifierTest, OrTrue) { auto computation = m->AddEntryComputation(builder.Build()); HloInstruction* root = computation->root_instruction(); EXPECT_EQ(root->opcode(), HloOpcode::kOr); - AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false, - non_bitcasting_callback()); + AlgebraicSimplifier simplifier(default_options_); ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie()); root = computation->root_instruction(); EXPECT_EQ(root, const_true); @@ -2466,8 +2786,7 @@ TEST_F(AlgebraicSimplifierTest, OrTrue2) { auto computation = m->AddEntryComputation(builder.Build()); HloInstruction* root = computation->root_instruction(); EXPECT_EQ(root->opcode(), HloOpcode::kOr); - AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false, - non_bitcasting_callback()); + AlgebraicSimplifier simplifier(default_options_); ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie()); root = computation->root_instruction(); EXPECT_EQ(root, const_true); @@ -2488,8 +2807,7 @@ TEST_F(AlgebraicSimplifierTest, OrFalse) { auto computation = m->AddEntryComputation(builder.Build()); HloInstruction* root = computation->root_instruction(); EXPECT_EQ(root->opcode(), HloOpcode::kOr); - AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false, - non_bitcasting_callback()); + AlgebraicSimplifier simplifier(default_options_); ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie()); root = computation->root_instruction(); EXPECT_EQ(root, param0); @@ -2510,8 +2828,7 @@ TEST_F(AlgebraicSimplifierTest, OrFalse2) { auto computation = m->AddEntryComputation(builder.Build()); HloInstruction* root = computation->root_instruction(); EXPECT_EQ(root->opcode(), HloOpcode::kOr); - AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false, - non_bitcasting_callback()); + AlgebraicSimplifier simplifier(default_options_); ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie()); root = computation->root_instruction(); EXPECT_EQ(root, param0); @@ -2641,15 +2958,15 @@ TEST_P(ConvInputPaddingTest, DoTest) { auto module = CreateNewVerifiedModule(); module->AddEntryComputation(builder.Build()); - AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false, - non_bitcasting_callback()); + AlgebraicSimplifier simplifier(default_options_); if (testcase.expected_conv_window.empty()) { ASSERT_FALSE(simplifier.Run(module.get()).ValueOrDie()); } else { ASSERT_TRUE(simplifier.Run(module.get()).ValueOrDie()); auto* conv = module->entry_computation()->root_instruction(); SCOPED_TRACE(module->ToString()); - ASSERT_THAT(conv, op::Convolution(op::Parameter(), op::Parameter())); + ASSERT_THAT(conv, + GmockMatch(m::Convolution(m::Parameter(), m::Parameter()))); EXPECT_EQ(window_util::ToString(conv->window()), absl::StrCat("size=3x3 ", testcase.expected_conv_window)); } @@ -2759,15 +3076,15 @@ TEST_P(ConvFilterPaddingTest, DoIt) { auto module = CreateNewVerifiedModule(); module->AddEntryComputation(builder.Build()); - AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false, - non_bitcasting_callback()); + AlgebraicSimplifier simplifier(default_options_); if (testcase.expected_conv_window.empty()) { ASSERT_FALSE(simplifier.Run(module.get()).ValueOrDie()); } else { ASSERT_TRUE(simplifier.Run(module.get()).ValueOrDie()); auto* conv = module->entry_computation()->root_instruction(); SCOPED_TRACE(module->ToString()); - ASSERT_THAT(conv, op::Convolution(op::Parameter(), op::Parameter())); + ASSERT_THAT(conv, + GmockMatch(m::Convolution(m::Parameter(), m::Parameter()))); EXPECT_EQ(window_util::ToString(conv->window()), absl::StrFormat("size=%dx%d %s", conv->operand(1)->shape().dimensions(2), @@ -2908,8 +3225,9 @@ TEST_F(AlgebraicSimplifierTest, ConvertConvToMatmul) { auto module = CreateNewUnverifiedModule(); auto* computation = module->AddEntryComputation(b.Build()); - AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/true, - bitcasting_callback()); + AlgebraicSimplifierOptions simplifier_options(bitcasting_callback()); + simplifier_options.set_is_layout_sensitive(true); + AlgebraicSimplifier simplifier(simplifier_options); if (!simplifier.Run(module.get()).ValueOrDie()) { return "NO_CHANGE"; } @@ -3032,17 +3350,15 @@ TEST_F(AlgebraicSimplifierTest, ScalarBroadcastToSlice) { EXPECT_EQ(root, slice); EXPECT_TRUE(ShapeUtil::Equal(root->shape(), slice_shape)); - AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false, - non_bitcasting_callback()); + AlgebraicSimplifier simplifier(default_options_); ASSERT_TRUE(simplifier.Run(module.get()).ValueOrDie()); // Running simplification again should not result in any further changes. ASSERT_FALSE(simplifier.Run(module.get()).ValueOrDie()); - - root = computation->root_instruction(); - EXPECT_THAT(root, op::Broadcast(scalar_param)); - EXPECT_TRUE(ShapeUtil::Equal(root->shape(), slice_shape)); + EXPECT_THAT(computation->root_instruction(), + GmockMatch(m::Broadcast(m::Op().Is(scalar_param)) + .WithShapeEqualTo(&slice_shape))); } // Test that reshape(transpose(broadcast(/*scalar value*/))) simplifies to a @@ -3071,13 +3387,11 @@ TEST_F(AlgebraicSimplifierTest, ScalarBroadcastToTransposeReshape) { EXPECT_EQ(root, reshape); EXPECT_TRUE(ShapeUtil::Equal(root->shape(), reshape_shape)); - AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false, - non_bitcasting_callback()); + AlgebraicSimplifier simplifier(default_options_); ASSERT_TRUE(simplifier.Run(module.get()).ValueOrDie()); - - root = computation->root_instruction(); - EXPECT_THAT(root, op::Broadcast(forty_two)); - EXPECT_TRUE(ShapeUtil::Equal(root->shape(), reshape_shape)); + EXPECT_THAT(computation->root_instruction(), + GmockMatch(m::Broadcast(m::Op().Is(forty_two)) + .WithShapeEqualTo(&reshape_shape))); } // Test that ReduceWindow(Pad(op, x), y) can simplify to ReduceWindow(op, x). @@ -3138,8 +3452,7 @@ TEST_F(AlgebraicSimplifierTest, FoldPadIntoReduceWindow) { auto computation = module->AddEntryComputation(builder.Build()); HloInstruction* root = computation->root_instruction(); EXPECT_EQ(root, reduce_window); - AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false, - non_bitcasting_callback()); + AlgebraicSimplifier simplifier(default_options_); ASSERT_TRUE(simplifier.Run(module.get()).ValueOrDie()); // Running simplification again should not result in any further changes. @@ -3147,7 +3460,8 @@ TEST_F(AlgebraicSimplifierTest, FoldPadIntoReduceWindow) { // Verify the result root = computation->root_instruction(); - EXPECT_THAT(root, op::ReduceWindow(operand, op::Constant())); + EXPECT_THAT(root, + GmockMatch(m::ReduceWindow(m::Op().Is(operand), m::Constant()))); EXPECT_TRUE(ShapeUtil::Equal(root->shape(), reduce_window_shape)) << ShapeUtil::HumanString(root->shape()) << " vs " << ShapeUtil::HumanString(reduce_window_shape); @@ -3224,8 +3538,7 @@ TEST_F(AlgebraicSimplifierTest, FoldConvertedPadIntoReduceWindow) { auto computation = module->AddEntryComputation(builder.Build()); HloInstruction* root = computation->root_instruction(); EXPECT_EQ(root, reduce_window); - AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false, - non_bitcasting_callback()); + AlgebraicSimplifier simplifier(default_options_); ASSERT_TRUE(simplifier.Run(module.get()).ValueOrDie()); // Running simplification again should not result in any further changes. @@ -3233,7 +3546,8 @@ TEST_F(AlgebraicSimplifierTest, FoldConvertedPadIntoReduceWindow) { // Verify the result root = computation->root_instruction(); - EXPECT_THAT(root, op::ReduceWindow(op::Convert(parameter), op::Constant())); + EXPECT_THAT(root, GmockMatch(m::ReduceWindow(m::Convert(m::Parameter(0)), + m::Constant()))); EXPECT_TRUE(ShapeUtil::Equal(root->shape(), reduce_window_shape)) << ShapeUtil::HumanString(root->shape()) << " vs " << ShapeUtil::HumanString(reduce_window_shape); @@ -3258,8 +3572,7 @@ TEST_F(AlgebraicSimplifierTest, ReversalOfTrivialDimensionsToBitcast) { auto module = CreateNewVerifiedModule(); auto computation = module->AddEntryComputation(builder.Build()); - AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false, - non_bitcasting_callback()); + AlgebraicSimplifier simplifier(default_options_); ASSERT_TRUE(simplifier.Run(module.get()).ValueOrDie()); HloInstruction* root = computation->root_instruction(); @@ -3295,8 +3608,7 @@ TEST_F(AlgebraicSimplifierTest, IteratorInvalidation) { m->AddEmbeddedComputation(std::move(dot_computation)); m->AddEntryComputation(call_builder.Build()); - AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false, - non_bitcasting_callback()); + AlgebraicSimplifier simplifier(default_options_); ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie()); } @@ -3313,11 +3625,10 @@ TEST_F(AlgebraicSimplifierTest, ConstantTupleBecomesTupleOfConstants) { auto computation = m->AddEntryComputation(builder.Build()); - AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false, - non_bitcasting_callback()); + AlgebraicSimplifier simplifier(default_options_); ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie()); EXPECT_THAT(computation->root_instruction(), - op::Tuple(op::Constant(), op::Constant())); + GmockMatch(m::Tuple(m::Constant(), m::Constant()))); } // A dynamic-slice is trivial if its start indices are all zeroes and the size @@ -3337,10 +3648,9 @@ TEST_F(AlgebraicSimplifierTest, TrivialDynamicSlice) { /*slice_sizes=*/{10, 100, 1000})); auto computation = m->AddEntryComputation(builder.Build()); - AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false, - non_bitcasting_callback()); + AlgebraicSimplifier simplifier(default_options_); ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie()); - EXPECT_THAT(computation->root_instruction(), op::Parameter()); + EXPECT_THAT(computation->root_instruction(), GmockMatch(m::Parameter())); } // A dynamic-update-slice is trivial if its start indices are all zeroes and the @@ -3371,11 +3681,10 @@ TEST_F(AlgebraicSimplifierTest, TrivialDynamicUpdateSlice) { 3, ShapeUtil::MakeShape(U32, {3}), "update_indices")))); auto computation = m->AddEntryComputation(builder.Build()); - AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false, - non_bitcasting_callback()); + AlgebraicSimplifier simplifier(default_options_); ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie()); EXPECT_THAT(computation->root_instruction(), - op::DynamicSlice(op::Parameter(), op::Parameter())); + GmockMatch(m::DynamicSlice(m::Parameter(), m::Parameter()))); } // Test that two consecutive broadcasts can be merged to one. @@ -3394,11 +3703,10 @@ TEST_F(AlgebraicSimplifierTest, MergeBroadcasts) { auto computation = m->AddEntryComputation(builder.Build()); HloInstruction* root = computation->root_instruction(); EXPECT_EQ(root->opcode(), HloOpcode::kBroadcast); - AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false, - non_bitcasting_callback()); + AlgebraicSimplifier simplifier(default_options_); ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie()); root = computation->root_instruction(); - EXPECT_THAT(root, op::Broadcast(op::Constant())); + EXPECT_THAT(root, GmockMatch(m::Broadcast(m::Constant()))); EXPECT_THAT(root->dimensions(), ElementsAre(2)); } @@ -3421,11 +3729,10 @@ TEST_F(AlgebraicSimplifierTest, MergeBroadcasts2) { auto computation = m->AddEntryComputation(builder.Build()); HloInstruction* root = computation->root_instruction(); EXPECT_EQ(root->opcode(), HloOpcode::kBroadcast); - AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false, - non_bitcasting_callback()); + AlgebraicSimplifier simplifier(default_options_); ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie()); root = computation->root_instruction(); - EXPECT_THAT(root, op::Broadcast(op::Parameter(0))); + EXPECT_THAT(root, GmockMatch(m::Broadcast(m::Parameter(0)))); EXPECT_THAT(root->dimensions(), ElementsAre(1, 3)); } @@ -3442,11 +3749,10 @@ TEST_F(AlgebraicSimplifierTest, MergeBroadcastAndIota) { auto computation = m->AddEntryComputation(builder.Build()); HloInstruction* root = computation->root_instruction(); EXPECT_EQ(root->opcode(), HloOpcode::kBroadcast); - AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false, - non_bitcasting_callback()); + AlgebraicSimplifier simplifier(default_options_); ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie()); root = computation->root_instruction(); - EXPECT_THAT(root, op::Iota()); + EXPECT_THAT(root, GmockMatch(m::Iota())); EXPECT_EQ(Cast(root)->iota_dimension(), 2); } @@ -3464,11 +3770,10 @@ TEST_F(AlgebraicSimplifierTest, MergeBroadcastAndIota2) { auto computation = m->AddEntryComputation(builder.Build()); HloInstruction* root = computation->root_instruction(); EXPECT_EQ(root->opcode(), HloOpcode::kBroadcast); - AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false, - non_bitcasting_callback()); + AlgebraicSimplifier simplifier(default_options_); ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie()); root = computation->root_instruction(); - EXPECT_THAT(root, op::Iota()); + EXPECT_THAT(root, GmockMatch(m::Iota())); EXPECT_EQ(Cast(root)->iota_dimension(), 2); } @@ -3486,11 +3791,11 @@ TEST_F(AlgebraicSimplifierTest, SliceOfPadLow) { TF_ASSERT_OK_AND_ASSIGN(auto module, ParseAndReturnVerifiedModule(hlo_string)); - AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false, - bitcasting_callback()); + AlgebraicSimplifierOptions options(bitcasting_callback()); + AlgebraicSimplifier simplifier(options); EXPECT_TRUE(simplifier.Run(module.get()).ValueOrDie()); auto root = module->entry_computation()->root_instruction(); - EXPECT_THAT(root, op::Reshape(op::Constant())); + EXPECT_THAT(root, GmockMatch(m::Reshape(m::Constant()))); } TEST_F(AlgebraicSimplifierTest, SliceOfPadHigh) { @@ -3507,11 +3812,11 @@ TEST_F(AlgebraicSimplifierTest, SliceOfPadHigh) { TF_ASSERT_OK_AND_ASSIGN(auto module, ParseAndReturnVerifiedModule(hlo_string)); - AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false, - bitcasting_callback()); + AlgebraicSimplifierOptions options(bitcasting_callback()); + AlgebraicSimplifier simplifier(options); EXPECT_TRUE(simplifier.Run(module.get()).ValueOrDie()); auto root = module->entry_computation()->root_instruction(); - EXPECT_THAT(root, op::Reshape(op::Constant())); + EXPECT_THAT(root, GmockMatch(m::Reshape(m::Constant()))); } TEST_F(AlgebraicSimplifierTest, SliceOfPadMidNonScalar) { @@ -3528,8 +3833,8 @@ TEST_F(AlgebraicSimplifierTest, SliceOfPadMidNonScalar) { TF_ASSERT_OK_AND_ASSIGN(auto module, ParseAndReturnVerifiedModule(hlo_string)); - AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false, - bitcasting_callback()); + AlgebraicSimplifierOptions options(bitcasting_callback()); + AlgebraicSimplifier simplifier(options); EXPECT_FALSE(simplifier.Run(module.get()).ValueOrDie()); } @@ -3547,11 +3852,11 @@ TEST_F(AlgebraicSimplifierTest, SliceOfPadMidScalar) { TF_ASSERT_OK_AND_ASSIGN(auto module, ParseAndReturnVerifiedModule(hlo_string)); - AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false, - bitcasting_callback()); + AlgebraicSimplifierOptions options(bitcasting_callback()); + AlgebraicSimplifier simplifier(options); EXPECT_TRUE(simplifier.Run(module.get()).ValueOrDie()); auto root = module->entry_computation()->root_instruction(); - EXPECT_THAT(root, op::Parameter()); + EXPECT_THAT(root, GmockMatch(m::Parameter())); } TEST_F(AlgebraicSimplifierTest, SliceOfConcatScalarInput) { @@ -3569,11 +3874,11 @@ TEST_F(AlgebraicSimplifierTest, SliceOfConcatScalarInput) { TF_ASSERT_OK_AND_ASSIGN(auto module, ParseAndReturnVerifiedModule(hlo_string)); - AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false, - bitcasting_callback()); + AlgebraicSimplifierOptions options(bitcasting_callback()); + AlgebraicSimplifier simplifier(options); EXPECT_TRUE(simplifier.Run(module.get()).ValueOrDie()); auto root = module->entry_computation()->root_instruction(); - EXPECT_THAT(root, op::Parameter(1)); + EXPECT_THAT(root, GmockMatch(m::Parameter(1))); } TEST_F(AlgebraicSimplifierTest, SliceOfConcatNonScalarInput) { @@ -3591,11 +3896,11 @@ TEST_F(AlgebraicSimplifierTest, SliceOfConcatNonScalarInput) { TF_ASSERT_OK_AND_ASSIGN(auto module, ParseAndReturnVerifiedModule(hlo_string)); - AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false, - bitcasting_callback()); + AlgebraicSimplifierOptions options(bitcasting_callback()); + AlgebraicSimplifier simplifier(options); EXPECT_TRUE(simplifier.Run(module.get()).ValueOrDie()); auto root = module->entry_computation()->root_instruction(); - EXPECT_THAT(root, op::Slice(op::Parameter(2))); + EXPECT_THAT(root, GmockMatch(m::Slice(m::Parameter(2)))); EXPECT_EQ(root->slice_starts(0), 1); EXPECT_EQ(root->slice_limits(0), 2); } @@ -3613,11 +3918,11 @@ TEST_F(AlgebraicSimplifierTest, NegateNegate) { TF_ASSERT_OK_AND_ASSIGN(auto module, ParseAndReturnVerifiedModule(hlo_string)); - AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false, - bitcasting_callback()); + AlgebraicSimplifierOptions options(bitcasting_callback()); + AlgebraicSimplifier simplifier(options); EXPECT_TRUE(simplifier.Run(module.get()).ValueOrDie()); auto root = module->entry_computation()->root_instruction(); - EXPECT_THAT(root, op::Parameter(0)); + EXPECT_THAT(root, GmockMatch(m::Parameter(0))); } TEST_F(AlgebraicSimplifierTest, NotNot) { @@ -3633,11 +3938,11 @@ TEST_F(AlgebraicSimplifierTest, NotNot) { TF_ASSERT_OK_AND_ASSIGN(auto module, ParseAndReturnVerifiedModule(hlo_string)); - AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false, - bitcasting_callback()); + AlgebraicSimplifierOptions options(bitcasting_callback()); + AlgebraicSimplifier simplifier(options); EXPECT_TRUE(simplifier.Run(module.get()).ValueOrDie()); auto root = module->entry_computation()->root_instruction(); - EXPECT_THAT(root, op::Parameter(0)); + EXPECT_THAT(root, GmockMatch(m::Parameter(0))); } struct PadReduceWindowEffectiveBroadcastCase { @@ -3733,8 +4038,7 @@ TEST_P(PadReduceWindowEffectiveBroadcastTest, DoIt) { output_shape, pad, zero, window, add_computation)); auto computation = m->AddEntryComputation(builder.Build()); - AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false, - non_bitcasting_callback()); + AlgebraicSimplifier simplifier(default_options_); TF_ASSERT_OK_AND_ASSIGN(bool run_successful, simplifier.Run(m.get())); ASSERT_TRUE(run_successful); @@ -3742,10 +4046,10 @@ TEST_P(PadReduceWindowEffectiveBroadcastTest, DoIt) { ShapeUtil::Equal(computation->root_instruction()->shape(), output_shape)); if (param.should_become_broadcast) { - EXPECT_THAT(computation->root_instruction(), op::Broadcast(::testing::_)); + EXPECT_THAT(computation->root_instruction(), GmockMatch(m::Broadcast())); } else { EXPECT_THAT(computation->root_instruction(), - op::ReduceWindow(::testing::_, zero)); + GmockMatch(m::ReduceWindow(m::Op(), m::Op().Is(zero)))); } } @@ -3815,8 +4119,7 @@ TEST_P(DotStrengthReductionTest, DotStrengthReduction) { builder.AddInstruction(HloInstruction::CreateDot( dot_shape, lhs, rhs, dot_dnums, DefaultPrecisionConfig(2))); auto computation = module->AddEntryComputation(builder.Build()); - AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false, - non_bitcasting_callback()); + AlgebraicSimplifier simplifier(default_options_); TF_ASSERT_OK_AND_ASSIGN(bool changed, simplifier.Run(module.get())); const bool dot_should_be_transformed = m == 1 || k == 1 || n == 1; const bool computation_should_be_modified = @@ -3845,7 +4148,7 @@ struct DotOfConcatTestSpec { }; class DotOfConcatSimplificationTest - : public HloTestBase, + : public AlgebraicSimplifierTest, public ::testing::WithParamInterface {}; // Test that we transform @@ -3893,19 +4196,19 @@ TEST_P(DotOfConcatSimplificationTest, ConstantLHS) { dot_shape, lhs, rhs, dot_dnums, DefaultPrecisionConfig(2))); auto computation = m->AddEntryComputation(builder.Build()); - AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false, - non_bitcasting_callback()); + AlgebraicSimplifier simplifier(default_options_); TF_ASSERT_OK_AND_ASSIGN(bool run_successful, simplifier.Run(m.get())); ASSERT_TRUE(run_successful); EXPECT_TRUE( ShapeUtil::Equal(computation->root_instruction()->shape(), dot_shape)); - auto match_dot_0 = op::Dot(op::Slice(op::Constant()), op::Parameter(0)); - auto match_dot_1 = op::Dot(op::Slice(op::Constant()), op::Parameter(1)); - auto match_dot_2 = op::Dot(op::Slice(op::Constant()), op::Parameter(2)); - EXPECT_THAT(computation->root_instruction(), - op::Add(op::Add(match_dot_0, match_dot_1), match_dot_2)); + auto match_dot_0 = m::Dot(m::Slice(m::Constant()), m::Parameter(0)); + auto match_dot_1 = m::Dot(m::Slice(m::Constant()), m::Parameter(1)); + auto match_dot_2 = m::Dot(m::Slice(m::Constant()), m::Parameter(2)); + EXPECT_THAT( + computation->root_instruction(), + GmockMatch(m::Add(m::Add(match_dot_0, match_dot_1), match_dot_2))); } // Test that we transform @@ -3958,20 +4261,20 @@ TEST_P(DotOfConcatSimplificationTest, ConstantRHS) { dot_shape, lhs, rhs, dot_dnums, DefaultPrecisionConfig(2))); auto computation = m->AddEntryComputation(builder.Build()); - AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false, - non_bitcasting_callback()); + AlgebraicSimplifier simplifier(default_options_); TF_ASSERT_OK_AND_ASSIGN(bool run_successful, simplifier.Run(m.get())); ASSERT_TRUE(run_successful); EXPECT_TRUE( ShapeUtil::Equal(computation->root_instruction()->shape(), dot_shape)); - auto match_dot_0 = op::Dot(op::Parameter(0), op::Slice(op::Constant())); - auto match_dot_1 = op::Dot(op::Parameter(1), op::Slice(op::Constant())); - auto match_dot_2 = op::Dot(op::Parameter(2), op::Slice(op::Constant())); - auto match_dot_3 = op::Dot(op::Parameter(3), op::Slice(op::Constant())); - EXPECT_THAT(computation->root_instruction(), - op::Add(op::Add(op::Add(match_dot_0, match_dot_1), match_dot_2), - match_dot_3)); + auto match_dot_0 = m::Dot(m::Parameter(0), m::Slice(m::Constant())); + auto match_dot_1 = m::Dot(m::Parameter(1), m::Slice(m::Constant())); + auto match_dot_2 = m::Dot(m::Parameter(2), m::Slice(m::Constant())); + auto match_dot_3 = m::Dot(m::Parameter(3), m::Slice(m::Constant())); + EXPECT_THAT( + computation->root_instruction(), + GmockMatch(m::Add(m::Add(m::Add(match_dot_0, match_dot_1), match_dot_2), + match_dot_3))); } DotOfConcatTestSpec kDotOfConcatTestSpecs[] = { @@ -4000,8 +4303,7 @@ TEST_F(AlgebraicSimplifierTest, DynamicUpdateSliceZeroUpdate) { const HloComputation* const computation = m->AddEntryComputation(builder.Build()); - AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false, - non_bitcasting_callback()); + AlgebraicSimplifier simplifier(default_options_); ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie()); EXPECT_THAT(computation->root_instruction(), operand); } @@ -4021,7 +4323,7 @@ struct DotOfGatherTestSpec { }; class DotOfGatherSimplificationTest - : public HloTestBase, + : public AlgebraicSimplifierTest, public ::testing::WithParamInterface {}; // input: dot(DS(ctA), ctB)) @@ -4078,8 +4380,7 @@ TEST_P(DotOfGatherSimplificationTest, ConstantRHS) { dot_shape, ds, rhs, dot_dnums, DefaultPrecisionConfig(2))); auto computation = m->AddEntryComputation(builder.Build()); - AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false, - non_bitcasting_callback()); + AlgebraicSimplifier simplifier(default_options_); TF_ASSERT_OK_AND_ASSIGN(bool run_successful, simplifier.Run(m.get())); ASSERT_TRUE(run_successful); EXPECT_TRUE( @@ -4090,8 +4391,8 @@ TEST_P(DotOfGatherSimplificationTest, ConstantRHS) { HloOpcode::kDynamicSlice); } else { EXPECT_THAT(computation->root_instruction(), - op::DynamicSlice(op::Dot(op::Constant(), op::Constant()), - op::Concatenate())); + GmockMatch(m::DynamicSlice(m::Dot(m::Constant(), m::Constant()), + m::Concatenate()))); } } @@ -4149,8 +4450,7 @@ TEST_P(DotOfGatherSimplificationTest, ConstantLHS) { dot_shape, lhs, ds, dot_dnums, DefaultPrecisionConfig(2))); auto computation = m->AddEntryComputation(builder.Build()); - AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false, - non_bitcasting_callback()); + AlgebraicSimplifier simplifier(default_options_); TF_ASSERT_OK_AND_ASSIGN(bool run_successful, simplifier.Run(m.get())); ASSERT_TRUE(run_successful); EXPECT_TRUE( @@ -4161,8 +4461,8 @@ TEST_P(DotOfGatherSimplificationTest, ConstantLHS) { HloOpcode::kDynamicSlice); } else { EXPECT_THAT(computation->root_instruction(), - op::DynamicSlice(op::Dot(op::Constant(), op::Constant()), - op::Concatenate())); + GmockMatch(m::DynamicSlice(m::Dot(m::Constant(), m::Constant()), + m::Concatenate()))); } } diff --git a/tensorflow/compiler/xla/service/ar_crs_combiner.cc b/tensorflow/compiler/xla/service/ar_crs_combiner.cc new file mode 100644 index 00000000000..c11452a6fbd --- /dev/null +++ b/tensorflow/compiler/xla/service/ar_crs_combiner.cc @@ -0,0 +1,286 @@ +/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/compiler/xla/service/ar_crs_combiner.h" + +#include +#include +#include + +#include "tensorflow/compiler/xla/literal.h" +#include "tensorflow/compiler/xla/literal_util.h" +#include "tensorflow/compiler/xla/service/call_graph.h" +#include "tensorflow/compiler/xla/service/hlo_computation.h" +#include "tensorflow/compiler/xla/service/hlo_instruction.h" +#include "tensorflow/compiler/xla/service/hlo_opcode.h" +#include "tensorflow/compiler/xla/service/pattern_matcher.h" +#include "tensorflow/compiler/xla/status_macros.h" +#include "tensorflow/compiler/xla/types.h" +#include "tensorflow/compiler/xla/xla_data.pb.h" + +namespace xla { + +namespace { + +namespace m = match; + +// If the argument instruction is a CRS in the sequence +// AR -> Convert -> Add -> CRS +// then return the AR in the sequence. +// TODO(b/117554291): Rewrite this to recognize more general patterns, +// not just the specific one of AR -> Add -> Convert -> CRS. +absl::optional MatchesArCrsPattern( + HloInstruction* instruction) { + HloInstruction *ar, *convert, *add, *crs; + if (Match(instruction, + m::CrossReplicaSum( + &crs, m::Add(&add, m::Op(), + m::Convert(&convert, + m::CrossReplicaSum(&ar, m::Op()))))) && + ar->users().size() == 1 && ar->shape().element_type() == BF16 && + convert->shape().element_type() == F32 && !crs->all_reduce_id()) { + return ar; + } + return absl::optional(); +} + +} // namespace + +absl::optional ArCrsCombiner::WhileFromBodyParameter( + HloInstruction* instruction) { + CHECK(HloOpcode::kParameter == instruction->opcode()); + HloComputation* computation = instruction->parent(); + auto caller_instructions = call_graph_->GetComputationCallers(computation); + if (caller_instructions.size() == 1) { + auto caller_instruction = caller_instructions[0]; + if (caller_instruction->opcode() == HloOpcode::kWhile) { + return caller_instruction; + } + } + return absl::optional(); +} + +std::vector ArCrsCombiner::GetAllTuples( + HloInstruction* instruction) { + if (instruction->opcode() == HloOpcode::kTuple) { + return {instruction}; + } + if (instruction->opcode() == HloOpcode::kDomain) { + return GetAllTuples(instruction->operands()[0]); + } + if (instruction->opcode() == HloOpcode::kParameter) { + auto maybe_while = WhileFromBodyParameter(instruction); + if (!maybe_while) { + return {}; + } + auto while_instr = *maybe_while; + auto init_tuples = GetAllTuples(while_instr->while_init()); + auto body_tuples = + GetAllTuples(while_instr->while_body()->root_instruction()); + if (init_tuples.empty() || body_tuples.empty()) { + return {}; + } + init_tuples.insert(init_tuples.end(), body_tuples.begin(), + body_tuples.end()); + return init_tuples; + } + if (instruction->opcode() == HloOpcode::kGetTupleElement) { + std::vector result_tuples; + for (auto tuple : GetAllTuples(instruction->operands()[0])) { + auto tmp_tuples = + GetAllTuples(tuple->mutable_operand(instruction->tuple_index())); + if (tmp_tuples.empty()) { + return {}; + } + result_tuples.insert(result_tuples.end(), tmp_tuples.begin(), + tmp_tuples.end()); + } + return result_tuples; + } + return {}; +} + +bool ArCrsCombiner::TupleElementsComputeSameValue( + HloInstruction* tuple_shaped_instruction, int64 i1, int64 i2, + absl::flat_hash_map* visited_pairs) { + auto tuples = GetAllTuples(tuple_shaped_instruction); + if (tuples.empty()) { + return false; + } + for (auto tuple : tuples) { + CHECK(tuple->opcode() == HloOpcode::kTuple); + if (!InstructionsComputeSameValue(tuple->mutable_operand(i1), + tuple->mutable_operand(i2), + visited_pairs)) { + return false; + } + } + return true; +} + +/* static */ +bool ArCrsCombiner::TestInstructionsComputeSameValue(HloInstruction* i1, + HloInstruction* i2) { + ArCrsCombiner combiner(/*num_spatial_partitions=*/2); + auto module = i1->parent()->parent(); + CHECK_EQ(module, i2->parent()->parent()); + combiner.call_graph_ = CallGraph::Build(module); + absl::flat_hash_map visited_pairs; + return combiner.InstructionsComputeSameValue(i1, i2, &visited_pairs); +} + +bool ArCrsCombiner::InstructionsComputeSameValue( + HloInstruction* i1, HloInstruction* i2, + absl::flat_hash_map* visited_pairs) { + if (i1 == i2) { + return true; + } + auto uid1 = i1->unique_id(); + auto uid2 = i2->unique_id(); + auto min_uid = std::min(uid1, uid2); + auto max_uid = std::max(uid1, uid2); + auto it = visited_pairs->find(min_uid); + if (it != visited_pairs->end() && max_uid == it->second) { + return true; + } + auto opcode1 = i1->opcode(); + auto operands1 = i1->operands(); + if (opcode1 != i2->opcode() || operands1.size() != i2->operands().size()) { + return false; + } + if (opcode1 == HloOpcode::kConstant || i1->IsCrossModuleAllReduce()) { + return i1->Identical( + *i2, + /*eq_operands=*/std::equal_to(), + /*eq_computations=*/std::equal_to(), + /*layout_sensitive=*/false); + } + visited_pairs->emplace(min_uid, max_uid); + for (int i = 0; i < operands1.size(); ++i) { + auto operand1 = operands1[i]; + auto operand2 = i2->operands()[i]; + if (!InstructionsComputeSameValue(operand1, operand2, visited_pairs)) { + return false; + } + } + if (opcode1 == HloOpcode::kGetTupleElement) { + if (i1->tuple_index() == i2->tuple_index()) { + return true; + } + return TupleElementsComputeSameValue(operands1[0], i1->tuple_index(), + i2->tuple_index(), visited_pairs); + } + return true; +} + +void ArCrsCombiner::GroupAllReducesById(HloModule* module) { + for (HloComputation* computation : module->MakeNonfusionComputations()) { + for (HloInstruction* instruction : computation->instructions()) { + auto ar = MatchesArCrsPattern(instruction); + if (ar) { + all_reduce_map_[*((*ar)->all_reduce_id())].push_back(*ar); + } + } + } +} + +void ArCrsCombiner::KeepProvablyEqualInstructionGroups() { + for (auto it : all_reduce_map_) { + auto instruction_vec = it.second; + CHECK_EQ(instruction_vec.size(), num_spatial_partitions_); + + auto instr_0 = instruction_vec[0]; + auto add_0 = instr_0->users()[0]->users()[0]; + CHECK(HloOpcode::kAdd == add_0->opcode()); + + for (int i = 1; i < instruction_vec.size(); ++i) { + auto instr_i = instruction_vec[i]; + auto add_i = instr_i->users()[0]->users()[0]; + CHECK(HloOpcode::kAdd == add_i->opcode()); + absl::flat_hash_map visited_pairs; + if (!InstructionsComputeSameValue(add_0, add_i, &visited_pairs)) { + all_reduce_map_.erase(it.first); + } + } + } +} + +StatusOr ArCrsCombiner::RewriteGraph() { + if (all_reduce_map_.empty()) { + return false; + } + + auto computation_is_addition = [](HloComputation* c) { + return c->instruction_count() == 3 && + Match(c->root_instruction(), m::Add(m::Parameter(), m::Parameter())); + }; + + for (auto it : all_reduce_map_) { + auto instruction_vec = it.second; + for (auto all_reduce : instruction_vec) { + auto parent_computation = all_reduce->parent(); + auto convert = all_reduce->users()[0]; + auto add = convert->users()[0]; + auto crs = add->users()[0]; + + if (!computation_is_addition(all_reduce->called_computations()[0]) || + !computation_is_addition(crs->called_computations()[0])) { + continue; + } + HloInstruction* other_summand = (add->operands()[0] == convert) + ? add->operands()[1] + : add->operands()[0]; + // Remove the AllReduce and replace the CRS with: + // AllReduce - (other_summand * (num_spatial_partitions_ - 1)) + TF_CHECK_OK( + all_reduce->ReplaceAllUsesWith(all_reduce->mutable_operand(0))); + crs->set_all_reduce_id(all_reduce->all_reduce_id()); + auto new_shape = crs->shape(); + HloInstruction* to_subtract; + if (num_spatial_partitions_ == 2) { + to_subtract = other_summand; + } else { + Literal partitions_minus_1_lit = Literal(new_shape); + partitions_minus_1_lit.PopulateWithValue( + num_spatial_partitions_ - 1); + auto partitions_minus_1_const = parent_computation->AddInstruction( + HloInstruction::CreateConstant(partitions_minus_1_lit.Clone())); + to_subtract = + parent_computation->AddInstruction(HloInstruction::CreateBinary( + new_shape, HloOpcode::kMultiply, other_summand, + partitions_minus_1_const)); + } + auto sub = + parent_computation->AddInstruction(HloInstruction::CreateBinary( + new_shape, HloOpcode::kSubtract, crs, to_subtract)); + TF_CHECK_OK(crs->ReplaceAllUsesWith(sub)); + TF_CHECK_OK(parent_computation->RemoveInstruction(all_reduce)); + } + } + + return true; +} + +StatusOr ArCrsCombiner::Run(HloModule* module) { + call_graph_ = CallGraph::Build(module); + + GroupAllReducesById(module); + + KeepProvablyEqualInstructionGroups(); + + return RewriteGraph(); +} + +} // namespace xla diff --git a/tensorflow/compiler/xla/service/ar_crs_combiner.h b/tensorflow/compiler/xla/service/ar_crs_combiner.h new file mode 100644 index 00000000000..f6a7ef76ec3 --- /dev/null +++ b/tensorflow/compiler/xla/service/ar_crs_combiner.h @@ -0,0 +1,88 @@ +/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_COMPILER_XLA_SERVICE_AR_CRS_COMBINER_H_ +#define TENSORFLOW_COMPILER_XLA_SERVICE_AR_CRS_COMBINER_H_ + +#include "absl/container/flat_hash_map.h" +#include "absl/strings/string_view.h" +#include "tensorflow/compiler/xla/service/call_graph.h" +#include "tensorflow/compiler/xla/service/hlo_module.h" +#include "tensorflow/compiler/xla/service/hlo_pass_interface.h" +#include "tensorflow/compiler/xla/statusor.h" + +namespace xla { + +// Combine an AllReduce and a CrossReplicaSum when they are close to each other +// in the graph, to use an efficient CrossReplicaSum implementation that +// fully utilizes the interconnect bandwidth. +class ArCrsCombiner : public HloModulePass { + public: + ArCrsCombiner(int num_spatial_partitions) + : num_spatial_partitions_(num_spatial_partitions) {} + absl::string_view name() const override { return "ar-crs-combiner"; } + StatusOr Run(HloModule* module) override; + + // Helper method to allow testing of InstructionsComputeSameValue. + static bool TestInstructionsComputeSameValue(HloInstruction* i1, + HloInstruction* i2); + + private: + // If the passed instruction is a while parameter, and the while body is only + // called by a single while instruction, return the while instruction. + absl::optional WhileFromBodyParameter( + HloInstruction* instruction); + + // Returns a vector of tuple instructions. + // If all instructions that flow to "instruction" are tuples, return them. + // Otherwise, return an empty vector. + std::vector GetAllTuples(HloInstruction* instruction); + + // Checks whether two different elements in the same tuple compute the same + // value. + bool TupleElementsComputeSameValue( + HloInstruction* tuple_shaped_instruction, int64 i1, int64 i2, + absl::flat_hash_map* visited_pairs); + + // Returns whether the instructions i1 and i2 can be shown to evaluate to the + // same value. Handling WHILE requires recursion, which may cause us to visit + // the same instruction again. To avoid infinite loops, we pass a cache of + // visited instruction pairs. + bool InstructionsComputeSameValue( + HloInstruction* i1, HloInstruction* i2, + absl::flat_hash_map* visited_pairs); + + // Populates all_reduce_map_. + void GroupAllReducesById(HloModule* module); + + // Looks at each AllReduce group in all_reduce_map_, and keeps only the + // groups for which it's safe to move the AllReduce later in the HLO graph. + void KeepProvablyEqualInstructionGroups(); + + // Performs the graph rewrite that eliminates the early AllReduce and turns + // the later CRS into an AllReduce. + StatusOr RewriteGraph(); + + int num_spatial_partitions_; + + // Map from all-reduce ids to the all reduce instructions. + absl::flat_hash_map> all_reduce_map_; + + std::unique_ptr call_graph_; +}; + +} // namespace xla + +#endif // TENSORFLOW_COMPILER_XLA_SERVICE_AR_CRS_COMBINER_H_ diff --git a/tensorflow/compiler/xla/service/ar_crs_combiner_test.cc b/tensorflow/compiler/xla/service/ar_crs_combiner_test.cc new file mode 100644 index 00000000000..9d5eaf63ccf --- /dev/null +++ b/tensorflow/compiler/xla/service/ar_crs_combiner_test.cc @@ -0,0 +1,415 @@ +/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/compiler/xla/service/ar_crs_combiner.h" +#include "tensorflow/compiler/xla/service/hlo_matchers.h" +#include "tensorflow/compiler/xla/statusor.h" +#include "tensorflow/compiler/xla/tests/hlo_test_base.h" +#include "tensorflow/core/lib/core/status_test_util.h" + +namespace xla { +namespace { + +namespace op = xla::testing::opcode_matchers; + +class ArCrsCombinerTest : public HloTestBase {}; + +TEST_F(ArCrsCombinerTest, SameValueTestBasecase) { + const char* module_str = R"( +HloModule foobar + +ENTRY %entrycomp (p: f32[2,2]) -> (f32[2,2], f32[2,2]) { + %p = f32[2,2] parameter(0) + %constant.f32.1 = f32[2,2] constant(f32[2,2] {{1, 2}, {3, 4}}) + %constant.f32.2 = f32[2,2] constant(f32[2,2] {{1, 2}, {3, 4}}) + ROOT %tuple = (f32[2,2], f32[2,2]) tuple(%constant.f32.1, %constant.f32.2) +} +)"; + + TF_ASSERT_OK_AND_ASSIGN(std::unique_ptr module, + ParseAndReturnVerifiedModule(module_str)); + auto root_tuple = module->entry_computation()->root_instruction(); + auto i1 = root_tuple->operands()[0]; + auto i2 = root_tuple->operands()[1]; + EXPECT_FALSE(ArCrsCombiner::TestInstructionsComputeSameValue( + i1, module->entry_computation()->parameter_instruction(0))); + EXPECT_TRUE(ArCrsCombiner::TestInstructionsComputeSameValue(i1, i2)); +} + +TEST_F(ArCrsCombinerTest, SameValueTestNumOperands) { + const char* module_str = R"( +HloModule foobar + +ENTRY %entrycomp (p: f32[2,2]) -> ((f32[2,2]), (f32[2,2], f32[2,2])) { + %p = f32[2,2] parameter(0) + %constant.f32 = f32[2,2] constant(f32[2,2] {{1, 2}, {3, 4}}) + %tuple1 = (f32[2,2]) tuple(%constant.f32) + %tuple2 = (f32[2,2], f32[2,2]) tuple(%constant.f32, %constant.f32) + ROOT %tuple = ((f32[2,2]), (f32[2,2], f32[2,2])) tuple(%tuple1, %tuple2) +} +)"; + + TF_ASSERT_OK_AND_ASSIGN(std::unique_ptr module, + ParseAndReturnVerifiedModule(module_str)); + auto root_tuple = module->entry_computation()->root_instruction(); + auto i1 = root_tuple->operands()[0]; + auto i2 = root_tuple->operands()[1]; + EXPECT_FALSE(ArCrsCombiner::TestInstructionsComputeSameValue(i1, i2)); +} + +TEST_F(ArCrsCombinerTest, SameValueTestTupleElementSameIndex) { + const char* module_str = R"( +HloModule foobar + +ENTRY %entrycomp (p: f32[2,2]) -> (f32[2,2], f32[2,2]) { + %p = f32[2,2] parameter(0) + %constant.f32 = f32[2,2] constant(f32[2,2] {{1, 2}, {3, 4}}) + %tuple.1 = (f32[2,2], f32[2,2]) tuple(%constant.f32, %constant.f32) + %get-tuple-element.1 = f32[2,2] get-tuple-element(%tuple.1), index=0 + %get-tuple-element.2 = f32[2,2] get-tuple-element(%tuple.1), index=0 + ROOT %tuple = (f32[2,2], f32[2,2]) tuple(%get-tuple-element.1, %get-tuple-element.2) +} +)"; + + TF_ASSERT_OK_AND_ASSIGN(std::unique_ptr module, + ParseAndReturnVerifiedModule(module_str)); + auto root_tuple = module->entry_computation()->root_instruction(); + auto i1 = root_tuple->operands()[0]; + auto i2 = root_tuple->operands()[1]; + EXPECT_TRUE(ArCrsCombiner::TestInstructionsComputeSameValue(i1, i2)); +} + +TEST_F(ArCrsCombinerTest, SameValueTestTupleElementDifferentIndex1) { + const char* module_str = R"( +HloModule foobar + +ENTRY %entrycomp (p: f32[2,2]) -> (f32[2,2], f32[2,2]) { + %p = f32[2,2] parameter(0) + %constant.f32 = f32[2,2] constant(f32[2,2] {{1, 2}, {3, 4}}) + %tuple.1 = (f32[2,2], f32[2,2]) tuple(%constant.f32, %constant.f32) + %get-tuple-element.1 = f32[2,2] get-tuple-element(%tuple.1), index=0 + %get-tuple-element.2 = f32[2,2] get-tuple-element(%tuple.1), index=1 + ROOT %tuple = (f32[2,2], f32[2,2]) tuple(%get-tuple-element.1, %get-tuple-element.2) +} +)"; + + TF_ASSERT_OK_AND_ASSIGN(std::unique_ptr module, + ParseAndReturnVerifiedModule(module_str)); + auto root_tuple = module->entry_computation()->root_instruction(); + auto i1 = root_tuple->operands()[0]; + auto i2 = root_tuple->operands()[1]; + EXPECT_TRUE(ArCrsCombiner::TestInstructionsComputeSameValue(i1, i2)); +} + +TEST_F(ArCrsCombinerTest, SameValueTestTupleElementDifferentIndex2) { + const char* module_str = R"( +HloModule foobar + +ENTRY %entrycomp (p: f32[2,2]) -> (f32[2,2], f32[2,2]) { + %p = f32[2,2] parameter(0) + %constant.f32.1 = f32[2,2] constant(f32[2,2] {{1, 2}, {3, 4}}) + %constant.f32.2 = f32[2,2] constant(f32[2,2] {{2, 3}, {4, 5}}) + %tuple.1 = (f32[2,2], f32[2,2]) tuple(%constant.f32.1, %constant.f32.2) + %get-tuple-element.1 = f32[2,2] get-tuple-element(%tuple.1), index=0 + %get-tuple-element.2 = f32[2,2] get-tuple-element(%tuple.1), index=1 + ROOT %tuple = (f32[2,2], f32[2,2]) tuple(%get-tuple-element.1, %get-tuple-element.2) +} +)"; + + TF_ASSERT_OK_AND_ASSIGN(std::unique_ptr module, + ParseAndReturnVerifiedModule(module_str)); + auto root_tuple = module->entry_computation()->root_instruction(); + auto i1 = root_tuple->operands()[0]; + auto i2 = root_tuple->operands()[1]; + EXPECT_FALSE(ArCrsCombiner::TestInstructionsComputeSameValue(i1, i2)); +} + +TEST_F(ArCrsCombinerTest, SameValueTestWhile1) { + const char* module_str = R"( +HloModule foobar + +%condition (x: (f32[2,2], f32[2,2])) -> pred[] { + %x = (f32[2,2], f32[2,2]) parameter(0) + %constant.0 = s32[] constant(0) + %constant.1 = s32[] constant(1) + ROOT %greater-than = pred[] greater-than(s32[] %constant.1, s32[] %constant.0) +} + +%body (x: (f32[2,2], f32[2,2])) -> (f32[2,2], f32[2,2]) { + %x = (f32[2,2], f32[2,2]) parameter(0) + %constant.f32 = f32[2,2] constant(f32[2,2] {{1, 2}, {3, 4}}) + %get-tuple-element.1 = f32[2,2] get-tuple-element(%x), index=0 + %get-tuple-element.2 = f32[2,2] get-tuple-element(%x), index=1 + %add.1 = f32[2,2] add(%get-tuple-element.1, %constant.f32) + %add.2 = f32[2,2] add(%get-tuple-element.2, %constant.f32) + ROOT %tuple = (f32[2,2], f32[2,2]) tuple(%add.1, %add.2) +} + +ENTRY %WhileLoop () -> (f32[2,2], f32[2,2]) { + %constant.f32 = f32[2,2] constant(f32[2,2] {{3, 4}, {5, 6}}) + %init.tuple = (f32[2,2], f32[2,2]) tuple(%constant.f32, %constant.f32) + ROOT %while = (f32[2,2], f32[2,2]) while(%init.tuple), condition=%condition, body=%body +} +)"; + + TF_ASSERT_OK_AND_ASSIGN(std::unique_ptr module, + ParseAndReturnVerifiedModule(module_str)); + auto root_while = module->entry_computation()->root_instruction(); + auto body_tuple = root_while->while_body()->root_instruction(); + auto i1 = body_tuple->operands()[0]; + auto i2 = body_tuple->operands()[1]; + EXPECT_TRUE(ArCrsCombiner::TestInstructionsComputeSameValue(i1, i2)); +} + +TEST_F(ArCrsCombinerTest, SameValueTestWhile2) { + const char* module_str = R"( +HloModule foobar + +%condition (x: (f32[2,2], f32[2,2])) -> pred[] { + %x = (f32[2,2], f32[2,2]) parameter(0) + %constant.0 = s32[] constant(0) + %constant.1 = s32[] constant(1) + ROOT %greater-than = pred[] greater-than(s32[] %constant.1, s32[] %constant.0) +} + +%body (x: (f32[2,2], f32[2,2])) -> (f32[2,2], f32[2,2]) { + %x = (f32[2,2], f32[2,2]) parameter(0) + %constant.f32 = f32[2,2] constant(f32[2,2] {{1, 2}, {3, 4}}) + %get-tuple-element.1 = f32[2,2] get-tuple-element(%x), index=0 + %get-tuple-element.2 = f32[2,2] get-tuple-element(%x), index=1 + %add.1 = f32[2,2] add(%get-tuple-element.1, %constant.f32) + %add.2 = f32[2,2] add(%get-tuple-element.2, %constant.f32) + ROOT %tuple = (f32[2,2], f32[2,2]) tuple(%add.1, %add.2) +} + +ENTRY %WhileLoop () -> (f32[2,2], f32[2,2]) { + %constant.f32.1 = f32[2,2] constant(f32[2,2] {{3, 4}, {5, 6}}) + %constant.f32.2 = f32[2,2] constant(f32[2,2] {{3, 4}, {7, 8}}) + %init.tuple = (f32[2,2], f32[2,2]) tuple(%constant.f32.1, %constant.f32.2) + ROOT %while = (f32[2,2], f32[2,2]) while(%init.tuple), condition=%condition, body=%body +} +)"; + + TF_ASSERT_OK_AND_ASSIGN(std::unique_ptr module, + ParseAndReturnVerifiedModule(module_str)); + auto root_while = module->entry_computation()->root_instruction(); + auto body_tuple = root_while->while_body()->root_instruction(); + auto i1 = body_tuple->operands()[0]; + auto i2 = body_tuple->operands()[1]; + EXPECT_FALSE(ArCrsCombiner::TestInstructionsComputeSameValue(i1, i2)); +} + +TEST_F(ArCrsCombinerTest, SameValueTestWhile3) { + const char* module_str = R"( +HloModule foobar + +%condition (x: (f32[2,2], f32[2,2])) -> pred[] { + %x = (f32[2,2], f32[2,2]) parameter(0) + %constant.0 = s32[] constant(0) + %constant.1 = s32[] constant(1) + ROOT %greater-than = pred[] greater-than(s32[] %constant.1, s32[] %constant.0) +} + +%body (x: (f32[2,2], f32[2,2])) -> (f32[2,2], f32[2,2]) { + %x = (f32[2,2], f32[2,2]) parameter(0) + %constant.f32.1 = f32[2,2] constant(f32[2,2] {{1, 2}, {3, 4}}) + %constant.f32.2 = f32[2,2] constant(f32[2,2] {{3, 4}, {1, 2}}) + %get-tuple-element.1 = f32[2,2] get-tuple-element(%x), index=0 + %get-tuple-element.2 = f32[2,2] get-tuple-element(%x), index=1 + %add.1 = f32[2,2] add(%get-tuple-element.1, %constant.f32.1) + %add.2 = f32[2,2] add(%get-tuple-element.2, %constant.f32.2) + ROOT %tuple = (f32[2,2], f32[2,2]) tuple(%add.1, %add.2) +} + +ENTRY %WhileLoop () -> (f32[2,2], f32[2,2]) { + %constant.f32 = f32[2,2] constant(f32[2,2] {{3, 4}, {5, 6}}) + %init.tuple = (f32[2,2], f32[2,2]) tuple(%constant.f32, %constant.f32) + ROOT %while = (f32[2,2], f32[2,2]) while(%init.tuple), condition=%condition, body=%body +} +)"; + + TF_ASSERT_OK_AND_ASSIGN(std::unique_ptr module, + ParseAndReturnVerifiedModule(module_str)); + auto root_while = module->entry_computation()->root_instruction(); + auto body_tuple = root_while->while_body()->root_instruction(); + auto i1 = body_tuple->operands()[0]->operands()[0]; // %get-tuple-element.1 + auto i2 = body_tuple->operands()[1]->operands()[0]; // %get-tuple-element.2 + EXPECT_FALSE(ArCrsCombiner::TestInstructionsComputeSameValue(i1, i2)); +} + +TEST_F(ArCrsCombinerTest, RewritePatternArConvertAddCrs) { + const char* module_str = R"( +HloModule foobar + +%binary_add (a: bf16[], b: bf16[]) -> bf16[] { + %a = bf16[] parameter(0) + %b = bf16[] parameter(1) + ROOT %add = bf16[] add(%a, %b) +} + +%sum.f32 (x: f32[], y: f32[]) -> f32[] { + %x = f32[] parameter(0) + %y = f32[] parameter(1) + ROOT %add = f32[] add(%x, %y) +} + +ENTRY %entrycomp (p: f32[2,2]) -> (f32[2,2], f32[2,2]) { + %p = f32[2,2] parameter(0) + %constant.bf16 = bf16[2,2] constant(bf16[2,2] {{1, 2}, {3, 4}}) + %constant.f32 = f32[2,2] constant(f32[2,2] {{1, 2}, {3, 4}}) + + %cross-replica-sum.ar.1 = bf16[2,2] + cross-replica-sum(%constant.bf16), + replica_groups={{0},{1}}, + all_reduce_id=1, + to_apply=%binary_add, + sharding={maximal device=0} + %convert.1 = f32[2,2] + convert(%cross-replica-sum.ar.1), + sharding={maximal device=0} + %add.1 = f32[2,2] + add(%constant.f32, %convert.1), + sharding={maximal device=0} + %cross-replica-sum.1 = f32[2,2] + cross-replica-sum(%add.1), + replica_groups={{0,1}}, + to_apply=%sum.f32, + sharding={maximal device=0} + + %cross-replica-sum.ar.2 = bf16[2,2] + cross-replica-sum(%constant.bf16), + replica_groups={{0},{1}}, + all_reduce_id=1, + to_apply=%binary_add, + sharding={maximal device=1} + %convert.2 = f32[2,2] + convert(%cross-replica-sum.ar.2), + sharding={maximal device=1} + %add.2 = f32[2,2] + add(%constant.f32, %convert.2), + sharding={maximal device=1} + %cross-replica-sum.2 = f32[2,2] + cross-replica-sum(%add.2), + replica_groups={{0,1}}, + to_apply=%sum.f32, + sharding={maximal device=1} + + ROOT %tuple = (f32[2,2], f32[2,2]) + tuple(%cross-replica-sum.1, %cross-replica-sum.2), + sharding={{maximal device=0}, {maximal device=1}} +} +)"; + + TF_ASSERT_OK_AND_ASSIGN(std::unique_ptr module, + ParseAndReturnVerifiedModule(module_str)); + auto crs_before = + module->entry_computation()->root_instruction()->operands()[0]; + auto replica_groups_before = crs_before->replica_groups(); + ArCrsCombiner combiner(2); + auto changed = combiner.Run(module.get()).ValueOrDie(); + EXPECT_TRUE(changed); + EXPECT_THAT(module->entry_computation()->root_instruction(), + op::Tuple(op::Subtract(op::CrossReplicaSum(), op::Constant()), + op::Subtract(op::CrossReplicaSum(), op::Constant()))); + auto sub = module->entry_computation()->root_instruction()->operands()[0]; + auto crs_after = sub->operands()[0]; + auto replica_groups_after = crs_after->replica_groups(); + ASSERT_EQ(replica_groups_before.size(), replica_groups_after.size()); + for (int i = 0; i < replica_groups_before.size(); ++i) { + // Somewhat verbose way to compare the replica_ids, because EqualsProto + // is not available in the open-source build. + auto group_before = replica_groups_before[i]; + std::vector ids_before(group_before.replica_ids().begin(), + group_before.replica_ids().end()); + auto group_after = replica_groups_after[i]; + std::vector ids_after(group_after.replica_ids().begin(), + group_after.replica_ids().end()); + EXPECT_EQ(ids_before, ids_after); + } +} + +TEST_F(ArCrsCombinerTest, OtherSummandNotTheSameDontRewrite) { + const char* module_str = R"( +HloModule foobar + +%binary_add (a: bf16[], b: bf16[]) -> bf16[] { + %a = bf16[] parameter(0) + %b = bf16[] parameter(1) + ROOT %add = bf16[] add(%a, %b) +} + +%sum.f32 (x: f32[], y: f32[]) -> f32[] { + %x = f32[] parameter(0) + %y = f32[] parameter(1) + ROOT %add = f32[] add(%x, %y) +} + +ENTRY %entrycomp (p: f32[2,2]) -> (f32[2,2], f32[2,2]) { + %p = f32[2,2] parameter(0) + %constant.bf16 = bf16[2,2] constant(bf16[2,2] {{1, 2}, {3, 4}}) + %constant.f32.1 = f32[2,2] constant(f32[2,2] {{1, 2}, {3, 4}}) + %constant.f32.2 = f32[2,2] constant(f32[2,2] {{3, 4}, {5, 6}}) + + %cross-replica-sum.ar.1 = bf16[2,2] + cross-replica-sum(%constant.bf16), + replica_groups={{0},{1}}, + all_reduce_id=1, + to_apply=%binary_add, + sharding={maximal device=0} + %convert.1 = f32[2,2] + convert(%cross-replica-sum.ar.1), + sharding={maximal device=0} + %add.1 = f32[2,2] + add(%constant.f32.1, %convert.1), + sharding={maximal device=0} + %cross-replica-sum.1 = f32[2,2] + cross-replica-sum(%add.1), + replica_groups={{0,1}}, + to_apply=%sum.f32, + sharding={maximal device=0} + + %cross-replica-sum.ar.2 = bf16[2,2] + cross-replica-sum(%constant.bf16), + replica_groups={{0},{1}}, + all_reduce_id=1, + to_apply=%binary_add, + sharding={maximal device=1} + %convert.2 = f32[2,2] + convert(%cross-replica-sum.ar.2), + sharding={maximal device=1} + %add.2 = f32[2,2] + add(%constant.f32.2, %convert.2), + sharding={maximal device=1} + %cross-replica-sum.2 = f32[2,2] + cross-replica-sum(%add.2), + replica_groups={{0,1}}, + to_apply=%sum.f32, + sharding={maximal device=1} + + ROOT %tuple = (f32[2,2], f32[2,2]) + tuple(%cross-replica-sum.1, %cross-replica-sum.2), + sharding={{maximal device=0}, {maximal device=1}} +} +)"; + + TF_ASSERT_OK_AND_ASSIGN(std::unique_ptr module, + ParseAndReturnVerifiedModule(module_str)); + ArCrsCombiner combiner(2); + auto changed = combiner.Run(module.get()).ValueOrDie(); + EXPECT_FALSE(changed); +} + +} // namespace +} // namespace xla diff --git a/tensorflow/compiler/xla/service/batchnorm_expander.cc b/tensorflow/compiler/xla/service/batchnorm_expander.cc index f70f6ddfec6..0e6ca1871b3 100644 --- a/tensorflow/compiler/xla/service/batchnorm_expander.cc +++ b/tensorflow/compiler/xla/service/batchnorm_expander.cc @@ -107,19 +107,37 @@ class BatchNormExpanderVisitor : public DfsHloVisitorWithDefault { } std::unique_ptr Mean( - int64 element_count, HloInstruction* operand, + HloInstruction* element_count, HloInstruction* operand, const std::function)>& add_instruction) { - HloInstruction* elem_count_recip = - add_instruction(HloInstruction::CreateBroadcast( - operand->shape(), - add_instruction(HloInstruction::CreateConvert( - ShapeUtil::MakeShape(operand->shape().element_type(), {}), - add_instruction(HloInstruction::CreateConstant( - LiteralUtil::CreateR0(1.0 / element_count))))), - {})); - return HloInstruction::CreateBinary(operand->shape(), HloOpcode::kMultiply, - operand, elem_count_recip); + auto broadcast = add_instruction( + HloInstruction::CreateBroadcast(operand->shape(), element_count, {})); + return HloInstruction::CreateBinary(operand->shape(), HloOpcode::kDivide, + operand, broadcast); + } + + std::unique_ptr DynamicElementCountPerFeature( + HloInstruction* operand, int64 feature_index, + const std::function)>& + add_instruction) { + auto elements_per_feature_u32 = add_instruction( + HloInstruction::CreateConstant(LiteralUtil::CreateR0(1))); + + for (int64 i = 0; i < ShapeUtil::Rank(operand->shape()); ++i) { + if (i == feature_index) { + continue; + } + auto dynamic_dimension_size = + add_instruction(HloInstruction::CreateGetDimensionSize( + ShapeUtil::MakeShape(U32, {}), operand, i)); + elements_per_feature_u32 = add_instruction(HloInstruction::CreateBinary( + ShapeUtil::MakeShape(U32, {}), HloOpcode::kMultiply, + dynamic_dimension_size, elements_per_feature_u32)); + } + + return HloInstruction::CreateConvert( + ShapeUtil::MakeShape(operand->shape().element_type(), {}), + elements_per_feature_u32); } // Replaces the existing HLO instruction old_instruction, with @@ -195,9 +213,6 @@ Status BatchNormExpanderVisitor::HandleBatchNormTraining( const Shape operand_shape = operand->shape(); PrimitiveType ptype = operand_shape.element_type(); int64 feature_index = batch_norm->feature_index(); - const int64 feature_count = operand_shape.dimensions(feature_index); - const int64 size_in_elements = ShapeUtil::ElementsIn(operand_shape); - int64 elements_per_feature_int64 = size_in_elements / feature_count; HloInstruction* scale = batch_norm->mutable_operand(1); HloInstruction* offset = batch_norm->mutable_operand(2); @@ -220,6 +235,9 @@ Status BatchNormExpanderVisitor::HandleBatchNormTraining( } } + auto elements_per_feature = + add(DynamicElementCountPerFeature(operand, feature_index, add)); + auto scale_broadcasted = add( HloInstruction::CreateBroadcast(operand_shape, scale, {feature_index})); @@ -243,13 +261,13 @@ Status BatchNormExpanderVisitor::HandleBatchNormTraining( add_reduce_computation)); // E[X]. - auto mean = add(Mean(elements_per_feature_int64, sum, add)); + auto mean = add(Mean(elements_per_feature, sum, add)); auto mean_broadcasted = add( HloInstruction::CreateBroadcast(operand_shape, mean, {feature_index})); // E[X^2]. - auto square_mean = add(Mean(elements_per_feature_int64, squared_sum, add)); + auto square_mean = add(Mean(elements_per_feature, squared_sum, add)); // E^2[X]. auto mean_square = @@ -458,9 +476,8 @@ Status BatchNormExpanderVisitor::HandleBatchNormGrad( int64 feature_index = batch_norm->feature_index(); - const int64 size_in_elements = ShapeUtil::ElementsIn(activation_shape); - const int64 feature_count = activation_shape.dimensions(feature_index); - const int64 elements_per_feature_int64 = size_in_elements / feature_count; + auto elements_per_feature = + add(DynamicElementCountPerFeature(activation, feature_index, add)); auto zero_literal = LiteralUtil::CreateR0(0.0f); TF_ASSIGN_OR_RETURN(zero_literal, zero_literal.Convert(ptype)); @@ -553,15 +570,9 @@ Status BatchNormExpanderVisitor::HandleBatchNormGrad( add_binary(activation_shape, HloOpcode::kMultiply, scale_broadcasted, rsqrt_var_add_epsilon_broadcasted); - scale_times_rsqrt_var_add_epsilon = add( - Mean(elements_per_feature_int64, scale_times_rsqrt_var_add_epsilon, add)); + scale_times_rsqrt_var_add_epsilon = + add(Mean(elements_per_feature, scale_times_rsqrt_var_add_epsilon, add)); - auto elements_per_feature_literal = - LiteralUtil::CreateR0(elements_per_feature_int64); - TF_ASSIGN_OR_RETURN(elements_per_feature_literal, - elements_per_feature_literal.Convert(ptype)); - auto elements_per_feature = add( - HloInstruction::CreateConstant(std::move(elements_per_feature_literal))); auto i1 = add_binary(activation_shape, HloOpcode::kMultiply, grad_output, add(HloInstruction::CreateBroadcast( activation_shape, elements_per_feature, {}))); diff --git a/tensorflow/compiler/xla/service/batchnorm_expander_test.cc b/tensorflow/compiler/xla/service/batchnorm_expander_test.cc index 08cf8026177..8e8fbbd935b 100644 --- a/tensorflow/compiler/xla/service/batchnorm_expander_test.cc +++ b/tensorflow/compiler/xla/service/batchnorm_expander_test.cc @@ -36,7 +36,21 @@ limitations under the License. namespace xla { namespace { -using BatchNormExpanderTest = HloTestBase; +class BatchNormExpanderTest : public HloTestBase { + protected: + // BatchNorm should have a dynamic sized dividor for mean operations. + int64 CountGetDimensionSize(const HloModule& module) { + int64 count = 0; + for (HloComputation* comp : module.computations()) { + for (HloInstruction* inst : comp->instructions()) { + if (inst->opcode() == HloOpcode::kGetDimensionSize) { + count++; + } + } + } + return count; + } +}; // Test that we expand BatchNormTraining. TEST_F(BatchNormExpanderTest, BatchNormTraining) { @@ -68,6 +82,7 @@ TEST_F(BatchNormExpanderTest, BatchNormTraining) { /*rewrite_grad_op=*/true); ASSERT_TRUE(rewriter.Run(module.get()).ValueOrDie()); root = computation->root_instruction(); + EXPECT_EQ(CountGetDimensionSize(*module), 3); // Make sure this operation is expanded. EXPECT_EQ(root->opcode(), HloOpcode::kTuple); } @@ -110,6 +125,7 @@ TEST_F(BatchNormExpanderTest, BatchNormGrad) { /*rewrite_grad_op=*/true); ASSERT_TRUE(rewriter.Run(module.get()).ValueOrDie()); root = computation->root_instruction(); + EXPECT_EQ(CountGetDimensionSize(*module), 3); // Make sure this operation is expanded. EXPECT_EQ(root->opcode(), HloOpcode::kTuple); } diff --git a/tensorflow/compiler/xla/service/buffer_assignment.cc b/tensorflow/compiler/xla/service/buffer_assignment.cc index 40c012a5e42..8d7c6244785 100644 --- a/tensorflow/compiler/xla/service/buffer_assignment.cc +++ b/tensorflow/compiler/xla/service/buffer_assignment.cc @@ -746,8 +746,7 @@ StatusOr> BufferAssigner::Run( LogicalBuffer::AlignmentFunction color_alignment, bool allow_input_output_aliasing, bool allocate_buffers_for_constants, BufferLiveness::Colorer colorer, ReuseAllocationFunction reuse_checker) { - BufferAssigner assigner(allow_input_output_aliasing, - allocate_buffers_for_constants, std::move(colorer), + BufferAssigner assigner(allocate_buffers_for_constants, std::move(colorer), std::move(reuse_checker)); return assigner.CreateAssignment(module, std::move(hlo_ordering), std::move(buffer_size), @@ -1434,33 +1433,40 @@ BufferAssigner::MergeColocatedBufferSets( computation == module->entry_computation(); }; + std::vector set_can_be_merged(colocated_buffer_sets.size(), true); + + // Do not merge if one of the sets includes live outs, entry parameters or + // constants. + // + // Buffer liveness does not report the correct live range for entry + // parameter and live out buffers so we have to special case them here. On + // backends that support constant buffer allocations, constant buffers are + // assigned globals in readonly storage so we can't merge colocated buffer + // sets containing constants with colocated buffer sets containing writing + // instructions or other constants. + // + // Moreover (on the CPU/GPU backends) the entry parameter buffers belong to + // the caller of the executable so we can't write to entry parameters + // either, and the argument for not merging constants also applies to entry + // parameters. + for (int64 i = 0; i < colocated_buffer_sets.size(); ++i) { + for (auto& buffer : colocated_buffer_sets[i]) { + if (buffer_liveness.MaybeLiveOut(*buffer) || + is_entry_parameter(*buffer) || + buffer->instruction()->opcode() == HloOpcode::kConstant) { + set_can_be_merged[i] = false; + break; + } + } + } + // Returns true if the two colocated buffer sets (specified by their indices // into the colocated_buffer_sets) can be merged into a single set. auto cannot_merge_buffer_sets = [&colocated_buffer_sets, &buffer_liveness, &buffer_size, - &is_entry_parameter](int64 i, int64 j) { - // Do not merge if one of the sets includes live outs, entry parameters or - // constants. - // - // Buffer liveness does not report the correct live range for entry - // parameter and live out buffers so we have to special case them here. On - // backends that support constant buffer allocations, constant buffers are - // assigned globals in readonly storage so we can't merge colocated buffer - // sets containing constants with colocated buffer sets containing writing - // instructions or other constants. - // - // Moreover (on the CPU/GPU backends) the entry parameter buffers belong to - // the caller of the executable so we can't write to entry parameters - // either, and the argument for not merging constants also applies to entry - // parameters. - for (int64 key : {i, j}) { - for (auto& buffer : colocated_buffer_sets[key]) { - if (buffer_liveness.MaybeLiveOut(*buffer) || - is_entry_parameter(*buffer) || - buffer->instruction()->opcode() == HloOpcode::kConstant) { - return true; - } - } + &set_can_be_merged](int64 i, int64 j) { + if (!set_can_be_merged[i] || !set_can_be_merged[j]) { + return true; } // Colocated sets satisfy the invariant that all buffers within a set have diff --git a/tensorflow/compiler/xla/service/buffer_assignment.h b/tensorflow/compiler/xla/service/buffer_assignment.h index d8e1612b899..0a9fdede803 100644 --- a/tensorflow/compiler/xla/service/buffer_assignment.h +++ b/tensorflow/compiler/xla/service/buffer_assignment.h @@ -545,12 +545,10 @@ class BufferAssigner { ReuseAllocationFunction reuse_checker = nullptr); private: - BufferAssigner(bool allow_input_output_aliasing, - bool allocate_buffers_for_constants, + BufferAssigner(bool allocate_buffers_for_constants, BufferLiveness::Colorer colorer, ReuseAllocationFunction reuse_checker) - : allow_input_output_aliasing_(allow_input_output_aliasing), - allocate_buffers_for_constants_(allocate_buffers_for_constants), + : allocate_buffers_for_constants_(allocate_buffers_for_constants), colorer_(colorer), reuse_checker_(reuse_checker) {} virtual ~BufferAssigner() = default; @@ -640,10 +638,6 @@ class BufferAssigner { LogicalBuffer::Color::Hasher> SplitBuffersByColor(const absl::flat_hash_set& buffers); - // If true, buffer assignments assumes that input parameter buffers and output - // buffers can be shared if their sizes match. - bool allow_input_output_aliasing_; - // If true, allocate buffers for constant instructions. bool allocate_buffers_for_constants_; diff --git a/tensorflow/compiler/xla/service/buffer_assignment_test.cc b/tensorflow/compiler/xla/service/buffer_assignment_test.cc index b1fc50cb188..8f482e6ba8c 100644 --- a/tensorflow/compiler/xla/service/buffer_assignment_test.cc +++ b/tensorflow/compiler/xla/service/buffer_assignment_test.cc @@ -137,8 +137,7 @@ class BufferAssignmentTest : public HloTestBase { } std::unique_ptr RunBufferAssignmentWithInstructionSequence( - HloModule* module, - absl::Span instruction_sequence, + HloModule* module, absl::Span instruction_sequence, int64 alignment = 1) { HloSchedule schedule(module); schedule.set_sequence(module->entry_computation(), instruction_sequence); @@ -1853,7 +1852,7 @@ class WhileBufferAssignmentTest : public HloTestBase { std::unique_ptr RunBufferAssignment(HloModule* module, int64 alignment = 1) { HloSchedule schedule = - ScheduleModule(*module, ByteSizeOf).ConsumeValueOrDie(); + ScheduleModule(module, ByteSizeOf).ConsumeValueOrDie(); return BufferAssigner::Run( module, absl::make_unique(schedule), ByteSizeOf, @@ -2162,7 +2161,7 @@ TEST_F(WhileBufferAssignmentTest, ColocatedBuffers) { // nodes are traversed during BufferAssignment. TF_ASSERT_OK_AND_ASSIGN( HloSchedule schedule, - ScheduleModule(*module, [](const BufferValue& buffer) { + ScheduleModule(module.get(), [](const BufferValue& buffer) { return ShapeUtil::ByteSizeOf(buffer.shape(), /*pointer_size=*/sizeof(void*)); })); @@ -2391,15 +2390,16 @@ TEST_F(WhileBufferAssignmentTest, WhileLoopsInterferingResultRange) { RunCopyInsertion(module.get()); HloSchedule schedule = - ScheduleModule(*module, ByteSizeOf).ConsumeValueOrDie(); + ScheduleModule(module.get(), ByteSizeOf).ConsumeValueOrDie(); // To trigger b/38494731, we want a specific Hlo schedule for the // root computation, so we overwrite that entry with a manually // crafted sequence. - schedule.set_sequence(module->entry_computation(), - {input1, weights1, one, output1, while1->operand(0), - while1, input0, weights0, zero, output0, - while0->operand(0), while0, gte0, gte1, root_add}); + schedule.set_sequence( + module->entry_computation(), + {input1, weights1, one, output1, while1->mutable_operand(0), while1, + input0, weights0, zero, output0, while0->mutable_operand(0), while0, + gte0, gte1, root_add}); // If this ASSERT fails, we constructed a bogus sequence above and this test // itself is buggy. diff --git a/tensorflow/compiler/xla/service/buffer_liveness_test.cc b/tensorflow/compiler/xla/service/buffer_liveness_test.cc index aeee543e843..40825a78716 100644 --- a/tensorflow/compiler/xla/service/buffer_liveness_test.cc +++ b/tensorflow/compiler/xla/service/buffer_liveness_test.cc @@ -117,7 +117,7 @@ TEST_F(BufferLivenessTest, ElementwiseChain) { auto log = builder.AddInstruction( HloInstruction::CreateUnary(vec_, HloOpcode::kLog, exp)); - auto module = CreateNewUnverifiedModule(); + auto module = CreateNewVerifiedModule(); module->AddEntryComputation(builder.Build()); auto liveness = @@ -164,7 +164,7 @@ TEST_F(BufferLivenessTest, MultipleEntryParameters_Sequential) { auto add = builder.AddInstruction( HloInstruction::CreateBinary(vec_, HloOpcode::kAdd, negate, exp)); - auto module = CreateNewUnverifiedModule(); + auto module = CreateNewVerifiedModule(); HloComputation* entry = module->AddEntryComputation(builder.Build()); HloSchedule schedule(module.get()); @@ -213,7 +213,7 @@ TEST_F(BufferLivenessTest, NonElementwiseOperand) { auto reverse = builder.AddInstruction(HloInstruction::CreateReverse(vec_, negate, {0})); - auto module = CreateNewUnverifiedModule(); + auto module = CreateNewVerifiedModule(); module->AddEntryComputation(builder.Build()); auto liveness = @@ -247,7 +247,7 @@ TEST_F(BufferLivenessTest, OverlappedBuffers) { auto add = builder.AddInstruction( HloInstruction::CreateBinary(vec_, HloOpcode::kAdd, negate, exp)); - auto module = CreateNewUnverifiedModule(); + auto module = CreateNewVerifiedModule(); module->AddEntryComputation(builder.Build()); auto liveness = @@ -289,7 +289,7 @@ TEST_F(BufferLivenessTest, OverlappedBuffersSequentialOrder) { auto add = builder.AddInstruction( HloInstruction::CreateBinary(vec_, HloOpcode::kAdd, negate, exp)); - auto module = CreateNewUnverifiedModule(); + auto module = CreateNewVerifiedModule(); auto computation = module->AddEntryComputation(builder.Build()); HloSchedule schedule(module.get()); @@ -336,7 +336,7 @@ TEST_F(BufferLivenessTest, RootInstructionIsNotLastInSequentialOrder) { HloInstruction::CreateSend(recv_done, token, /*channel_id=*/1)); auto send_done = builder.AddInstruction(HloInstruction::CreateSendDone(send)); - auto module = CreateNewUnverifiedModule(); + auto module = CreateNewVerifiedModule(); auto computation = module->AddEntryComputation(builder.Build(add)); HloSchedule schedule(module.get()); @@ -373,7 +373,7 @@ TEST_F(BufferLivenessTest, TupleLiveOut) { auto outer_tuple = builder.AddInstruction(HloInstruction::CreateTuple({inner_tuple, exp})); - auto module = CreateNewUnverifiedModule(); + auto module = CreateNewVerifiedModule(); module->AddEntryComputation(builder.Build()); auto liveness = @@ -393,7 +393,7 @@ TEST_F(BufferLivenessTest, TupleLiveOut) { TEST_F(BufferLivenessTest, EmbeddedComputation) { // Test MaybeLiveOut and MayInterfere for embedded computation. - auto module = CreateNewUnverifiedModule(); + auto module = CreateNewVerifiedModule(); auto embedded_builder = HloComputation::Builder(TestName() + "_embedded"); auto embedded_param = embedded_builder.AddInstruction( @@ -450,7 +450,7 @@ TEST_F(BufferLivenessTest, TupleConstantLiveOut) { builder.AddInstruction(HloInstruction::CreateGetTupleElement( inner_tuple0.shape(), tuple_constant, 0)); - auto module = CreateNewUnverifiedModule(); + auto module = CreateNewVerifiedModule(); module->AddEntryComputation(builder.Build()); auto liveness = @@ -576,7 +576,7 @@ TEST_F(BufferLivenessTest, DependentTupleElements) { auto tuple_root = builder.AddInstruction(HloInstruction::CreateTuple({add0, add1})); - auto module = CreateNewUnverifiedModule(); + auto module = CreateNewVerifiedModule(); module->AddEntryComputation(BuildDummyComputation()); module->AddEmbeddedComputation(builder.Build()); @@ -611,8 +611,8 @@ TEST_F(BufferLivenessTest, DependentTupleElements) { class FusedDynamicUpdateSliceLivenessTest : public BufferLivenessTest { protected: // Builds and runs a computation (see test case computation graphs below). - std::unique_ptr BuildModule(const bool update_uses_tuple_element1, - const bool fuse_gte0) { + std::unique_ptr BuildModule( + const bool update_uses_tuple_element1, const bool fuse_gte0) { auto builder = HloComputation::Builder(TestName()); // Create param0 Tuple. Shape data_shape = ShapeUtil::MakeShape(F32, {8}); @@ -646,7 +646,7 @@ class FusedDynamicUpdateSliceLivenessTest : public BufferLivenessTest { builder.AddInstruction( HloInstruction::CreateTuple({gte0, dynamic_update_slice})); // Build module and get reference to entry computation. - auto module = CreateNewUnverifiedModule(); + auto module = CreateNewVerifiedModule(); module->AddEntryComputation(builder.Build()); auto* computation = module->entry_computation(); // Create fusion instruction based on number of tuple element 1 users. @@ -802,7 +802,7 @@ class DynamicUpdateSliceLivenessTest : public BufferLivenessTest { auto tuple_root = builder.AddInstruction( HloInstruction::CreateTuple({gte0, dynamic_update_slice})); // Build module and get reference to entry computation. - auto module = CreateNewUnverifiedModule(); + auto module = CreateNewVerifiedModule(); module->AddEntryComputation(BuildDummyComputation()); module->AddEmbeddedComputation(builder.Build()); // Run BufferLiveness on 'module'. diff --git a/tensorflow/compiler/xla/service/call_graph.cc b/tensorflow/compiler/xla/service/call_graph.cc index bdd5069632e..7987343bfaf 100644 --- a/tensorflow/compiler/xla/service/call_graph.cc +++ b/tensorflow/compiler/xla/service/call_graph.cc @@ -325,6 +325,15 @@ bool CallGraph::IsFlattened() const { return true; } +std::vector CallGraph::GetComputationCallers( + HloComputation* c) { + std::vector callers; + for (auto callsite : GetNode(c).caller_callsites()) { + callers.push_back(callsite.instruction()); + } + return callers; +} + std::pair CallGraph::NearestAncestorsInSameComputation(HloInstruction* a, HloInstruction* b) const { diff --git a/tensorflow/compiler/xla/service/call_graph.h b/tensorflow/compiler/xla/service/call_graph.h index cb56f4789d0..05c7c998738 100644 --- a/tensorflow/compiler/xla/service/call_graph.h +++ b/tensorflow/compiler/xla/service/call_graph.h @@ -236,6 +236,10 @@ class CallGraph { // FlattenCallGraph. bool IsFlattened() const; + // Returns a vector of instructions calling the passed computation. + // (Often a vector of size 1.) + std::vector GetComputationCallers(HloComputation* c); + string ToString() const; private: diff --git a/tensorflow/compiler/xla/service/compile_only_service.cc b/tensorflow/compiler/xla/service/compile_only_service.cc index 67132274c0d..1965925fa7f 100644 --- a/tensorflow/compiler/xla/service/compile_only_service.cc +++ b/tensorflow/compiler/xla/service/compile_only_service.cc @@ -86,15 +86,15 @@ CompileOnlyService::CompileAheadOfTime( Executable::DumpToDirectory(per_host_path, filename, hlo_snapshot)); } - const auto& program_shape = instance.computation.host_program_shape(); ExecutionOptions execution_options; *execution_options.mutable_debug_options() = debug_options; *execution_options.mutable_shape_with_output_layout() = - *instance.result_layout; + instance.result_layout->ToProto(); TF_ASSIGN_OR_RETURN( std::unique_ptr module_config, - CreateModuleConfig(program_shape, instance.argument_layouts, - &execution_options)); + CreateModuleConfig( + ProgramShape(instance.computation.host_program_shape()), + instance.argument_layouts, &execution_options)); TF_ASSIGN_OR_RETURN( std::unique_ptr hlo_module, diff --git a/tensorflow/compiler/xla/service/computation_placer.h b/tensorflow/compiler/xla/service/computation_placer.h index c899ffb9dc5..844b42a38d7 100644 --- a/tensorflow/compiler/xla/service/computation_placer.h +++ b/tensorflow/compiler/xla/service/computation_placer.h @@ -105,8 +105,6 @@ class ComputationPlacer { // Map from platform kind to computation placer singleton. static std::map* GetPlatformComputationPlacers(); - se::Platform::Id platform_id_; - TF_DISALLOW_COPY_AND_ASSIGN(ComputationPlacer); }; diff --git a/tensorflow/compiler/xla/service/convolution_feature_group_converter.cc b/tensorflow/compiler/xla/service/convolution_feature_group_converter.cc index 7f7f1503a09..95c7724c3c9 100644 --- a/tensorflow/compiler/xla/service/convolution_feature_group_converter.cc +++ b/tensorflow/compiler/xla/service/convolution_feature_group_converter.cc @@ -142,16 +142,16 @@ std::vector GetMaskIds(int64 group_size, int64 group_count) { // Finally we use the Eq op of these two broadcasted constants and get the // desired mask. HloInstruction* GetExpandedFilterMask( - const Shape& filter_shape, int64 input_feature_dim, - int64 output_feature_dim, int64 group_count, + const Shape& filter_shape, int64 kernel_input_feature_dim, + int64 kernel_output_feature_dim, int64 group_count, const std::function)>& add_instruction) { Shape expanded_filter_shape = - ExpandedFilterShape(filter_shape, group_count, input_feature_dim); + ExpandedFilterShape(filter_shape, group_count, kernel_input_feature_dim); Shape mask_shape = ShapeUtil::MakeShape( S32, AsInt64Slice(expanded_filter_shape.dimensions())); - int64 output_feature = filter_shape.dimensions(output_feature_dim); - int64 group_size = filter_shape.dimensions(input_feature_dim); + int64 output_feature = filter_shape.dimensions(kernel_output_feature_dim); + int64 group_size = filter_shape.dimensions(kernel_input_feature_dim); // Create a 'input_feature' sized linspace and 'output_feature' sized linspace // that will be broadcasted into perpendicular dimensions and compared. @@ -159,15 +159,14 @@ HloInstruction* GetExpandedFilterMask( GetMaskIds(group_size, group_count); const std::vector output_feature_filter_mask = GetMaskIds(output_feature / group_count, group_count); - auto mask1 = add_instruction(HloInstruction::CreateConstant( LiteralUtil::CreateR1(input_feature_filter_mask))); - auto broadcasted_mask1 = add_instruction( - HloInstruction::CreateBroadcast(mask_shape, mask1, {input_feature_dim})); + auto broadcasted_mask1 = add_instruction(HloInstruction::CreateBroadcast( + mask_shape, mask1, {kernel_input_feature_dim})); auto mask2 = add_instruction(HloInstruction::CreateConstant( LiteralUtil::CreateR1(output_feature_filter_mask))); - auto broadcasted_mask2 = add_instruction( - HloInstruction::CreateBroadcast(mask_shape, mask2, {output_feature_dim})); + auto broadcasted_mask2 = add_instruction(HloInstruction::CreateBroadcast( + mask_shape, mask2, {kernel_output_feature_dim})); // Compare the broadcasted output feature linspace to the input feature // linspace to create a diagonal predicate. @@ -189,91 +188,203 @@ Status ConvolutionVisitor::HandleConvolution(HloInstruction* convolution) { }; auto dim_numbers = convolution->convolution_dimension_numbers(); - int64 input_feature_dim = dim_numbers.kernel_input_feature_dimension(); - int64 group_size = filter->shape().dimensions(input_feature_dim); - int64 output_feature_dim = dim_numbers.kernel_output_feature_dimension(); - auto expanded_filter_shape = - ExpandedFilterShape(filter->shape(), group_count, input_feature_dim); - HloInstruction* filter_mask = GetExpandedFilterMask( - filter->shape(), input_feature_dim, output_feature_dim, group_count, add); + int64 kernel_input_feature_dim = dim_numbers.kernel_input_feature_dimension(); + int64 group_size = filter->shape().dimensions(kernel_input_feature_dim); + int64 kernel_output_feature_dim = + dim_numbers.kernel_output_feature_dimension(); + auto expanded_filter_shape = ExpandedFilterShape(filter->shape(), group_count, + kernel_input_feature_dim); + HloInstruction* filter_mask = + GetExpandedFilterMask(filter->shape(), kernel_input_feature_dim, + kernel_output_feature_dim, group_count, add); HloInstruction* expanded_filter; if (group_size == 1) { bool depthwise_separable = - (group_count == filter->shape().dimensions(output_feature_dim)); + (group_count == filter->shape().dimensions(kernel_output_feature_dim)); // If the code generator handles depthwise separable convolutions // inherently, then no filter expansion is needed. if (!filter_expansion_ && depthwise_separable) { - const int64 old_kernel_input_feature_dimension = - dim_numbers.kernel_input_feature_dimension(); - const int64 old_kernel_output_feature_dimension = - dim_numbers.kernel_output_feature_dimension(); - - // For depthwise convolutions, we want the kernel input feature dimension - // to be smaller than the output feature dimension. If that's not the - // case, we swap the dimensions. - if (old_kernel_input_feature_dimension > - old_kernel_output_feature_dimension) { - Shape reshaped_filter_shape = filter->shape(); - auto& dimensions = *reshaped_filter_shape.mutable_dimensions(); - std::swap(dimensions[old_kernel_input_feature_dimension], - dimensions[old_kernel_output_feature_dimension]); - - auto reshaped_filter = - add(HloInstruction::CreateReshape(reshaped_filter_shape, filter)); - - dim_numbers.set_kernel_input_feature_dimension( - old_kernel_output_feature_dimension); - - dim_numbers.set_kernel_output_feature_dimension( - old_kernel_input_feature_dimension); - - auto new_convolution = HloInstruction::CreateConvolve( - convolution->shape(), convolution->mutable_operand(0), - reshaped_filter, group_count, convolution->window(), dim_numbers, - convolution->precision_config()); - - TF_RETURN_IF_ERROR(computation_->ReplaceWithNewInstruction( - convolution, std::move(new_convolution))); - } return Status::OK(); } // We want to repeat 'filter' in the 'input_feature_dim' dimension // 'group_count' times. Shape reshaped_filter_shape = - ShapeUtil::DeleteDimension(input_feature_dim, filter->shape()); + ShapeUtil::DeleteDimension(kernel_input_feature_dim, filter->shape()); auto reshaped_filter = add(HloInstruction::CreateReshape(reshaped_filter_shape, filter)); std::vector broadcast_dims; for (int64 i = 0; i < filter->shape().dimensions_size(); ++i) { - if (i == input_feature_dim) { + if (i == kernel_input_feature_dim) { continue; } broadcast_dims.push_back(i); } expanded_filter = add(HloInstruction::CreateBroadcast( expanded_filter_shape, reshaped_filter, broadcast_dims)); + + auto zero = add(HloInstruction::CreateConstant( + LiteralUtil::Zero(expanded_filter_shape.element_type()))); + auto zero_filter = + add(HloInstruction::CreateBroadcast(expanded_filter_shape, zero, {})); + auto new_filter = add(HloInstruction::CreateTernary( + expanded_filter_shape, HloOpcode::kSelect, filter_mask, expanded_filter, + zero_filter)); + + auto new_convolution = HloInstruction::CreateConvolve( + convolution->shape(), convolution->mutable_operand(0), new_filter, + /*feature_group_count=*/1, convolution->window(), dim_numbers, + convolution->precision_config()); + TF_RETURN_IF_ERROR(computation_->ReplaceWithNewInstruction( + convolution, std::move(new_convolution))); } else { - // We could possibly also use reshape, broadcast, reshape instead of concat - // here, but it would require more complex code, and for depthwise - // convolution we would never end up in this branch. - std::vector concat_operands(group_count, filter); - expanded_filter = add(HloInstruction::CreateConcatenate( - expanded_filter_shape, concat_operands, input_feature_dim)); + int64 activation_input_feature_dim = dim_numbers.input_feature_dimension(); + + int64 output_feature = + filter->shape().dimensions(kernel_output_feature_dim); + + // If group_count == output_feature, then we map those grouped convolutions + // onto depthwise convolution. This is done by adding an additional spatial + // dimension to the activations, kernel, and the output. + // E.g., we would turn + // [2, 12]{B, IF} conv [3, 4]{IF, OF} into + // [3, 2, 4]{S, B, IF} depth conv [3, 1, 4]{S, IF, OF}, where S is the + // additional spatial dimension. The generated convolution output will be + // [1, 2, 4]{S, B, OF} and then reshape the output back to [2, 4] {B, OF}. + + if (group_count == output_feature && !filter_expansion_) { + auto filter = convolution->mutable_operand(1); + auto activation = convolution->mutable_operand(0); + + // Add spatial dimension to the activation, and reshape. + Shape reshaped_activation_shape = activation->shape(); + ShapeUtil::AppendMajorDimension(group_size, &reshaped_activation_shape); + + int64 new_spatial_dim = reshaped_activation_shape.dimensions().size() - 1; + + reshaped_activation_shape.set_dimensions(activation_input_feature_dim, + group_count); + activation = add( + HloInstruction::CreateReshape(reshaped_activation_shape, activation)); + + // Add spatial dimension to the filter, and reshape. + Shape reshaped_filter_shape = filter->shape(); + ShapeUtil::AppendMajorDimension(1, &reshaped_filter_shape); + + filter = + add(HloInstruction::CreateReshape(reshaped_filter_shape, filter)); + + Shape new_output_shape = convolution->shape(); + ShapeUtil::AppendMajorDimension(1, &new_output_shape); + + // Edit convolution dimension numbers. Note that kernel_input_feature_dim + // now becomes a spatial dimension, and the newly added dimension of size + // 1 is the new kernel_input_feature_dim. + dim_numbers.add_input_spatial_dimensions(new_spatial_dim); + dim_numbers.add_kernel_spatial_dimensions(kernel_input_feature_dim); + dim_numbers.set_kernel_input_feature_dimension(new_spatial_dim); + dim_numbers.add_output_spatial_dimensions(new_spatial_dim); + + // Add window for the new spatial dimension. + Window new_window = convolution->window(); + auto* dim = new_window.add_dimensions(); + dim->set_window_dilation(1); + dim->set_base_dilation(1); + dim->set_stride(1); + dim->set_size(group_size); + + auto new_convolution = add(HloInstruction::CreateConvolve( + new_output_shape, activation, filter, group_count, new_window, + dim_numbers, convolution->precision_config())); + + // Delete the extra spatial dimension, and reshape. + Shape reshaped_convolution_shape = + ShapeUtil::DeleteDimension(new_spatial_dim, new_convolution->shape()); + auto reshaped_convolution = HloInstruction::CreateReshape( + reshaped_convolution_shape, new_convolution); + + TF_RETURN_IF_ERROR(computation_->ReplaceWithNewInstruction( + convolution, std::move(reshaped_convolution))); + + } else { + // The filter expansion mechanism adds zeroes in the kernel. + // For an OF = 12, IF = 6, and kernel IF = 2, the expanded filter mask + // would look like (IF on the Y-axis, OF on the X-axis) + // 1 1 1 1 0 0 0 0 0 0 0 0 + // 1 1 1 1 0 0 0 0 0 0 0 0 + // 0 0 0 0 1 1 1 1 0 0 0 0 + // 0 0 0 0 1 1 1 1 0 0 0 0 + // 0 0 0 0 0 0 0 0 1 1 1 1 + // 0 0 0 0 0 0 0 0 1 1 1 1 + // + // Instead of convolving the above with the input, we instead slice the + // kernel into three kernels, each containing islands of 1s from the + // filter above. We also slice the activations in the IF dimension with + // each slice of size = group_size. For each slice, we perform + // convolutions, and concatenate the generated outputs in the output OF + // dimension. + + std::vector sliced_convolutions; + auto activation = convolution->mutable_operand(0); + std::vector slice_strides(filter->shape().dimensions_size(), 1); + std::vector filter_slice_starts(filter->shape().dimensions_size(), + 0); + std::vector filter_slice_limits( + filter->shape().dimensions().begin(), + filter->shape().dimensions().end()); + std::vector activation_slice_starts( + activation->shape().dimensions_size(), 0); + std::vector activation_slice_limits( + activation->shape().dimensions().begin(), + activation->shape().dimensions().end()); + + int64 output_feature = + filter->shape().dimensions(kernel_output_feature_dim); + auto output_feature_dim = dim_numbers.output_feature_dimension(); + int64 filter_slice_width = output_feature / group_count; + + int64 activation_input_feature_dim = + dim_numbers.input_feature_dimension(); + + for (int64 i = 0; i < group_count; i++) { + filter_slice_starts[kernel_output_feature_dim] = i * filter_slice_width; + filter_slice_limits[kernel_output_feature_dim] = + (i + 1) * filter_slice_width; + auto filter_sliced_shape = filter->shape(); + filter_sliced_shape.set_dimensions(kernel_output_feature_dim, + filter_slice_width); + auto filter_slice = add(HloInstruction::CreateSlice( + filter_sliced_shape, filter, filter_slice_starts, + filter_slice_limits, slice_strides)); + + activation_slice_starts[activation_input_feature_dim] = i * group_size; + activation_slice_limits[activation_input_feature_dim] = + (i + 1) * group_size; + auto activation_sliced_shape = activation->shape(); + activation_sliced_shape.set_dimensions(activation_input_feature_dim, + group_size); + auto activation_slice = add(HloInstruction::CreateSlice( + activation_sliced_shape, activation, activation_slice_starts, + activation_slice_limits, slice_strides)); + + auto conv_slice_shape = convolution->shape(); + conv_slice_shape.set_dimensions(output_feature_dim, filter_slice_width); + + auto new_convolution = add(HloInstruction::CreateConvolve( + conv_slice_shape, activation_slice, filter_slice, + /*feature_group_count=*/1, convolution->window(), dim_numbers, + convolution->precision_config())); + + sliced_convolutions.push_back(new_convolution); + } + + auto new_conv = HloInstruction::CreateConcatenate( + convolution->shape(), sliced_convolutions, output_feature_dim); + TF_RETURN_IF_ERROR(computation_->ReplaceWithNewInstruction( + convolution, std::move(new_conv))); + } } - auto zero = add(HloInstruction::CreateConstant( - LiteralUtil::Zero(expanded_filter_shape.element_type()))); - auto zero_filter = - add(HloInstruction::CreateBroadcast(expanded_filter_shape, zero, {})); - auto new_filter = add( - HloInstruction::CreateTernary(expanded_filter_shape, HloOpcode::kSelect, - filter_mask, expanded_filter, zero_filter)); - auto new_convolution = HloInstruction::CreateConvolve( - convolution->shape(), convolution->mutable_operand(0), new_filter, - /*feature_group_count=*/1, convolution->window(), dim_numbers, - convolution->precision_config()); - TF_RETURN_IF_ERROR(computation_->ReplaceWithNewInstruction( - convolution, std::move(new_convolution))); + return Status::OK(); } diff --git a/tensorflow/compiler/xla/service/convolution_feature_group_converter_test.cc b/tensorflow/compiler/xla/service/convolution_feature_group_converter_test.cc index 28373ebf636..e6bf2143a21 100644 --- a/tensorflow/compiler/xla/service/convolution_feature_group_converter_test.cc +++ b/tensorflow/compiler/xla/service/convolution_feature_group_converter_test.cc @@ -82,18 +82,14 @@ ENTRY %Convolve1D1Window_0.v3 (input: f32[1,2,4], filter: f32[1,2,2]) -> f32[1,2 ConvolutionFeatureGroupConverter converter; ASSERT_TRUE(converter.Run(module.get()).ValueOrDie()); root = computation->root_instruction(); - // Make sure the convolution is converted to one with feature_group_count = 1. - EXPECT_EQ(root->opcode(), HloOpcode::kConvolution); - EXPECT_EQ(root->feature_group_count(), 1); - // Verify that the filter operand has been replaced. - EXPECT_THAT(root->operand(1), - op::Select(op::Eq(op::Broadcast(op::Constant()), - op::Broadcast(op::Constant())), - // We expect to see Concatenate here instead of - // Broadcast, because feature_group_count < input - // feature dimension. - op::Concatenate(op::Parameter(), op::Parameter()), - op::Broadcast(op::Constant()))); + // Make sure the convolution is replaced with a concatenate. + EXPECT_EQ(root->opcode(), HloOpcode::kConcatenate); + // And the operands of the concatenate are convolutions, each with a feature + // group count = 1. + EXPECT_EQ(root->operand(0)->opcode(), HloOpcode::kConvolution); + EXPECT_EQ(root->operand(1)->opcode(), HloOpcode::kConvolution); + EXPECT_EQ(root->operand(0)->feature_group_count(), 1); + EXPECT_EQ(root->operand(1)->feature_group_count(), 1); } } // namespace diff --git a/tensorflow/compiler/xla/service/copy_insertion.cc b/tensorflow/compiler/xla/service/copy_insertion.cc index 4e547d925f6..df605966387 100644 --- a/tensorflow/compiler/xla/service/copy_insertion.cc +++ b/tensorflow/compiler/xla/service/copy_insertion.cc @@ -442,7 +442,6 @@ class CopyRemover { const HloOrdering& ordering, HloModule* module) : module_(module), alias_analysis_(alias_analysis), - ordering_(ordering), buffer_value_tracker_(*module, alias_analysis, ordering) {} // Try to elide the given copy. The copy is elided if the instruction is not @@ -1003,7 +1002,6 @@ class CopyRemover { HloModule* module_; const HloAliasAnalysis& alias_analysis_; - const HloOrdering& ordering_; // Object tracking the HLO values contained in each HLO buffer. BufferValueTracker buffer_value_tracker_; diff --git a/tensorflow/compiler/xla/service/copy_insertion_test.cc b/tensorflow/compiler/xla/service/copy_insertion_test.cc index 7446bc7cc11..e4e9d7ba05c 100644 --- a/tensorflow/compiler/xla/service/copy_insertion_test.cc +++ b/tensorflow/compiler/xla/service/copy_insertion_test.cc @@ -94,7 +94,7 @@ TEST_F(CopyInsertionTest, SingleParameter) { EXPECT_THAT(x->users(), UnorderedElementsAre(tuple)); - auto module = CreateNewUnverifiedModule(); + auto module = CreateNewVerifiedModule(); module->AddEntryComputation(builder.Build()); InsertCopies(module.get()); @@ -114,7 +114,7 @@ TEST_F(CopyInsertionTest, SingleConstant) { EXPECT_THAT(constant->users(), UnorderedElementsAre(tuple)); - auto module = CreateNewUnverifiedModule(); + auto module = CreateNewVerifiedModule(); module->AddEntryComputation(builder.Build()); InsertCopies(module.get()); @@ -127,7 +127,7 @@ TEST_F(CopyInsertionTest, SingleConstant) { TEST_F(CopyInsertionTest, ExistingCopiesNotRemoved) { // Verify that kCopy instructions which change layout and exist before // copy-insertion remain in the graph after copy-insertion. - auto module = CreateNewUnverifiedModule(); + auto module = CreateNewVerifiedModule(); auto builder = HloComputation::Builder(TestName()); HloInstruction* constant = @@ -181,7 +181,7 @@ TEST_F(CopyInsertionTest, MultipleConstantsAndParameters) { builder.AddInstruction(HloInstruction::CreateTuple({constant2, x, add})); - auto module = CreateNewUnverifiedModule(); + auto module = CreateNewVerifiedModule(); module->AddEntryComputation(builder.Build()); InsertCopies(module.get()); @@ -217,7 +217,7 @@ TEST_F(CopyInsertionTest, AmbiguousPointsToSet) { EXPECT_THAT(constant2->users(), UnorderedElementsAre(tuple1, tuple2)); EXPECT_THAT(constant3->users(), UnorderedElementsAre(tuple2)); - auto module = CreateNewUnverifiedModule(); + auto module = CreateNewVerifiedModule(); module->AddEntryComputation(builder.Build()); HloInstruction* old_root = module->entry_computation()->root_instruction(); @@ -238,7 +238,7 @@ TEST_F(CopyInsertionTest, BitcastParameter) { HloInstruction* bitcast = builder.AddInstruction(HloInstruction::CreateUnary( ShapeUtil::MakeShape(F32, {2, 2}), HloOpcode::kBitcast, x)); - auto module = CreateNewUnverifiedModule(); + auto module = CreateNewVerifiedModule(); module->AddEntryComputation(builder.Build()); EXPECT_THAT(x->users(), UnorderedElementsAre(bitcast)); @@ -261,7 +261,7 @@ TEST_F(CopyInsertionTest, BitcastConstant) { HloInstruction* bitcast = builder.AddInstruction(HloInstruction::CreateUnary( ShapeUtil::MakeShape(F32, {2, 2}), HloOpcode::kBitcast, constant)); - auto module = CreateNewUnverifiedModule(); + auto module = CreateNewVerifiedModule(); module->AddEntryComputation(builder.Build()); EXPECT_THAT(constant->users(), UnorderedElementsAre(bitcast)); @@ -283,7 +283,7 @@ TEST_F(CopyInsertionTest, BitcastTupleElementParameter) { ShapeUtil::MakeShape(F32, {2, 2}), HloOpcode::kBitcast, x)); builder.AddInstruction(HloInstruction::CreateTuple({bitcast})); - auto module = CreateNewUnverifiedModule(); + auto module = CreateNewVerifiedModule(); module->AddEntryComputation(builder.Build()); EXPECT_THAT(x->users(), UnorderedElementsAre(bitcast)); @@ -310,7 +310,7 @@ TEST_F(CopyInsertionTest, NestedTupleParameter) { ShapeUtil::MakeShape(F32, {42})}), "param0")); - auto module = CreateNewUnverifiedModule(); + auto module = CreateNewVerifiedModule(); module->AddEntryComputation(builder.Build()); EXPECT_EQ(HloOpcode::kParameter, @@ -351,7 +351,7 @@ TEST_F(CopyInsertionTest, ElementOfNestedTupleParameter) { auto gte = builder.AddInstruction(HloInstruction::CreateGetTupleElement( ShapeUtil::GetSubshape(param->shape(), {0}), param, 0)); - auto module = CreateNewUnverifiedModule(); + auto module = CreateNewVerifiedModule(); module->AddEntryComputation(builder.Build()); EXPECT_EQ(gte, module->entry_computation()->root_instruction()); @@ -388,7 +388,7 @@ TEST_F(CopyInsertionTest, AmbiguousTopLevelRoot) { builder.AddInstruction(HloInstruction::CreateGetTupleElement( ShapeUtil::GetSubshape(select->shape(), {0}), select, 0)); - auto module = CreateNewUnverifiedModule(); + auto module = CreateNewVerifiedModule(); module->AddEntryComputation(builder.Build()); EXPECT_EQ(gte, module->entry_computation()->root_instruction()); @@ -1295,7 +1295,7 @@ TEST_F(WhileCopyInsertionTest, InitPointsToNonDistinctUsedByTwoWhileLoops) { TEST_F(CopyInsertionTest, SwizzlingWhile) { // Test a while instruction with a body which permutes its tuple parameter // elements. - auto module = CreateNewUnverifiedModule(); + auto module = CreateNewVerifiedModule(); const Shape loop_state_shape = ShapeUtil::MakeTupleShape({scalar_shape_, scalar_shape_}); @@ -1362,7 +1362,7 @@ TEST_F(CopyInsertionTest, CrossingParameters) { // | / \ | // | / \| // (p1 , p0) - auto module = CreateNewUnverifiedModule(); + auto module = CreateNewVerifiedModule(); const Shape tuple_shape = ShapeUtil::MakeTupleShape({scalar_shape_, scalar_shape_}); @@ -1395,7 +1395,7 @@ TEST_F(CopyInsertionTest, ParametersAliasing) { // | | // | | // (p0 , p1) - auto module = CreateNewUnverifiedModule(); + auto module = CreateNewVerifiedModule(); const Shape tuple_shape = ShapeUtil::MakeTupleShape({scalar_shape_, scalar_shape_}); @@ -1428,7 +1428,7 @@ TEST_F(CopyInsertionTest, ParameterWithNoAliasing) { // | | // | | // (p0 , p1) - auto module = CreateNewUnverifiedModule(); + auto module = CreateNewVerifiedModule(); const Shape tuple_shape = ShapeUtil::MakeTupleShape({scalar_shape_, scalar_shape_}); @@ -1461,7 +1461,7 @@ TEST_F(CopyInsertionTest, ParameterWithPartialAliasing) { // | | // | | // (p0 , p1) - auto module = CreateNewUnverifiedModule(); + auto module = CreateNewVerifiedModule(); const Shape tuple_shape = ShapeUtil::MakeTupleShape({scalar_shape_, scalar_shape_}); @@ -1496,7 +1496,7 @@ TEST_F(CopyInsertionTest, ParameterAndParallelOpsWithPartialAliasing) { // | | | // | | | // +-- (p0 , p1) - auto module = CreateNewUnverifiedModule(); + auto module = CreateNewVerifiedModule(); const Shape tuple_shape = ShapeUtil::MakeTupleShape({scalar_shape_, scalar_shape_}); @@ -1534,7 +1534,7 @@ TEST_F(CopyInsertionTest, ParameterAndOpsWithPartialAliasing) { // | Add----+ // | | | // +-- (p0 , p1) - auto module = CreateNewUnverifiedModule(); + auto module = CreateNewVerifiedModule(); const Shape tuple_shape = ShapeUtil::MakeTupleShape({scalar_shape_, scalar_shape_}); @@ -1569,7 +1569,7 @@ TEST_F(CopyInsertionTest, SwizzlingWhileWithOneOp) { // the operation (instruction) on the element makes the live range of the // respective input and output elements different than if the instruction were // not there (as in the SwizzlingWhile test above). - auto module = CreateNewUnverifiedModule(); + auto module = CreateNewVerifiedModule(); const Shape loop_state_shape = ShapeUtil::MakeTupleShape({scalar_shape_, scalar_shape_}); @@ -1632,7 +1632,7 @@ TEST_F(CopyInsertionTest, SwizzlingWhileSharedInput) { // the while body is a single constant (both loop state elements are the same // constant). This means no copies are necessary because both loop state // elements are the same so interchanging them is a no-op. - auto module = CreateNewUnverifiedModule(); + auto module = CreateNewVerifiedModule(); const Shape loop_state_shape = ShapeUtil::MakeTupleShape({scalar_shape_, scalar_shape_}); @@ -1693,7 +1693,7 @@ TEST_F(CopyInsertionTest, SequentialWhiles) { const Shape loop_state_shape = ShapeUtil::MakeTupleShape( {element_shape, element_shape, element_shape, element_shape}); - auto module = CreateNewUnverifiedModule(); + auto module = CreateNewVerifiedModule(); auto builder = HloComputation::Builder(TestName()); auto param_0 = builder.AddInstruction( HloInstruction::CreateParameter(0, element_shape, "param_0")); @@ -1783,7 +1783,7 @@ TEST_F(CopyInsertionTest, SequentialWhiles) { TEST_F(CopyInsertionTest, WhileBodyWithConstantRoot) { // Test a while body and condition which are each simply a constant (root of // computation is a constant). The body constant should be copied. - auto module = CreateNewUnverifiedModule(); + auto module = CreateNewVerifiedModule(); auto builder = HloComputation::Builder(TestName()); auto param_0 = builder.AddInstruction( HloInstruction::CreateParameter(0, scalar_shape_, "param_0")); diff --git a/tensorflow/compiler/xla/service/cpu/BUILD b/tensorflow/compiler/xla/service/cpu/BUILD index 2763d18121a..ce4c2a9cc69 100644 --- a/tensorflow/compiler/xla/service/cpu/BUILD +++ b/tensorflow/compiler/xla/service/cpu/BUILD @@ -96,6 +96,7 @@ cc_library( "@com_google_absl//absl/types:span", "//tensorflow/compiler/tf2xla:cpu_function_runtime", "//tensorflow/compiler/xla/service:map_inliner", + "//tensorflow/compiler/xla/service:hlo_get_dimension_size_rewriter", "//tensorflow/compiler/xla/service:scatter_expander", "//tensorflow/compiler/xla:literal", "//tensorflow/compiler/xla:protobuf_util", diff --git a/tensorflow/compiler/xla/service/cpu/compiler_functor.cc b/tensorflow/compiler/xla/service/cpu/compiler_functor.cc index 73b03440cbb..796a7cf94d0 100644 --- a/tensorflow/compiler/xla/service/cpu/compiler_functor.cc +++ b/tensorflow/compiler/xla/service/cpu/compiler_functor.cc @@ -61,19 +61,6 @@ Disabling these as a starting point. // TODO(b/64227304) Creating a custom pass pipeline will replace this. namespace { -class FilteredFunctionPassManager : public llvm::legacy::FunctionPassManager { - public: - FilteredFunctionPassManager(llvm::Module* m, bool disable_expensive_passes) - : llvm::legacy::FunctionPassManager(m), - disable_expensive_passes_(disable_expensive_passes) {} - void add(llvm::Pass* p) override { - llvm::legacy::FunctionPassManager::add(p); - } - - private: - bool disable_expensive_passes_; -}; - class FilteredPassManager : public llvm::legacy::PassManager { public: explicit FilteredPassManager(bool disable_expensive_passes) @@ -96,8 +83,7 @@ class FilteredPassManager : public llvm::legacy::PassManager { std::unique_ptr CompilerFunctor::operator()( llvm::Module& module) const { FilteredPassManager module_passes(disable_expensive_passes_); - FilteredFunctionPassManager function_passes(&module, - disable_expensive_passes_); + llvm::legacy::FunctionPassManager function_passes(&module); VLOG(2) << "IR before optimizations"; XLA_VLOG_LINES(2, llvm_ir::DumpModuleToString(module)); diff --git a/tensorflow/compiler/xla/service/cpu/cpu_compiler.cc b/tensorflow/compiler/xla/service/cpu/cpu_compiler.cc index 4ce5a8a2925..6374822c81b 100644 --- a/tensorflow/compiler/xla/service/cpu/cpu_compiler.cc +++ b/tensorflow/compiler/xla/service/cpu/cpu_compiler.cc @@ -76,6 +76,7 @@ limitations under the License. #include "tensorflow/compiler/xla/service/hlo_cse.h" #include "tensorflow/compiler/xla/service/hlo_dce.h" #include "tensorflow/compiler/xla/service/hlo_element_type_converter.h" +#include "tensorflow/compiler/xla/service/hlo_get_dimension_size_rewriter.h" #include "tensorflow/compiler/xla/service/hlo_instruction.h" #include "tensorflow/compiler/xla/service/hlo_memory_scheduler.h" #include "tensorflow/compiler/xla/service/hlo_opcode.h" @@ -268,10 +269,11 @@ Status CpuCompiler::RunHloPassesThroughLayoutAssn( /*rewrite_training_op=*/true, /*rewrite_inference_op=*/true, /*rewrite_grad_op=*/true); - pass.AddPass( - /*is_layout_sensitive=*/false, - [](const Shape&, const Shape&) { return false; }, - /*enable_dot_strength_reduction=*/false); + pipeline.AddPass(); + AlgebraicSimplifierOptions options( + [](const Shape&, const Shape&) { return false; }); + options.set_enable_dot_strength_reduction(false); + pass.AddPass(options); pass.AddPass(); // BatchNormExpander can create zero-sized ops, so zero-sized HLO @@ -334,10 +336,11 @@ Status CpuCompiler::RunHloPassesAfterLayoutAssn( pass.AddInvariantChecker( /*layout_sensitive=*/true, /*allow_mixed_precision=*/false); - pass.AddPass>( - /*is_layout_sensitive=*/true, - [](const Shape&, const Shape&) { return true; }, - /*enable_dot_strength_reduction=*/false); + AlgebraicSimplifierOptions options( + [](const Shape&, const Shape&) { return true; }); + options.set_is_layout_sensitive(true); + options.set_enable_dot_strength_reduction(false); + pass.AddPass>(options); pass.AddPass(); pass.AddPass(/*is_layout_sensitive=*/true); } @@ -587,9 +590,9 @@ StatusOr> CpuCompiler::RunBackend( // Select an order for emitting the HLO instructions for each // computation. Using this sequence enables tighter buffer liveness analysis // and reduced memory usage (as compared to using DependencyHloOrdering). - TF_ASSIGN_OR_RETURN( - HloSchedule schedule, - ScheduleModule(*module, BufferSizeBytesFunction(), DFSMemoryScheduler)); + TF_ASSIGN_OR_RETURN(HloSchedule schedule, + ScheduleModule(module.get(), BufferSizeBytesFunction(), + DFSMemoryScheduler)); // Run buffer allocation on the HLO graph. TF_ASSIGN_OR_RETURN( @@ -779,7 +782,7 @@ CpuCompiler::CompileAheadOfTime(std::unique_ptr module_group, XLA_VLOG_LINES(2, module->ToString()); TF_ASSIGN_OR_RETURN(HloSchedule schedule, - ScheduleModule(*module, BufferSizeBytesFunction())); + ScheduleModule(module, BufferSizeBytesFunction())); // Run buffer analysis on the HLO graph. This analysis figures out which // temporary buffers are required to run the computation. diff --git a/tensorflow/compiler/xla/service/cpu/cpu_executable.cc b/tensorflow/compiler/xla/service/cpu/cpu_executable.cc index 29abf38e439..818b2b0d0db 100644 --- a/tensorflow/compiler/xla/service/cpu/cpu_executable.cc +++ b/tensorflow/compiler/xla/service/cpu/cpu_executable.cc @@ -51,8 +51,7 @@ namespace cpu { CpuExecutable::CpuExecutable( std::unique_ptr jit, std::unique_ptr assignment, - std::unique_ptr hlo_module, - const string& entry_function_name, + std::unique_ptr hlo_module, const string& entry_function_name, std::unique_ptr hlo_profile_printer_data, std::unique_ptr hlo_profile_index_map) : Executable(std::move(hlo_module), std::move(hlo_profile_printer_data), diff --git a/tensorflow/compiler/xla/service/cpu/cpu_executable.h b/tensorflow/compiler/xla/service/cpu/cpu_executable.h index 3c3c047bfe8..3b91b15ba9b 100644 --- a/tensorflow/compiler/xla/service/cpu/cpu_executable.h +++ b/tensorflow/compiler/xla/service/cpu/cpu_executable.h @@ -49,7 +49,7 @@ class CpuExecutable : public Executable { public: CpuExecutable(std::unique_ptr jit, std::unique_ptr assignment, - std::unique_ptr hlo_module, + std::unique_ptr hlo_module, const string& entry_function_name, std::unique_ptr hlo_profile_printer_data, std::unique_ptr hlo_profile_index_map); diff --git a/tensorflow/compiler/xla/service/cpu/cpu_instruction_fusion.cc b/tensorflow/compiler/xla/service/cpu/cpu_instruction_fusion.cc index f9cd61bea3d..6f79ad7c146 100644 --- a/tensorflow/compiler/xla/service/cpu/cpu_instruction_fusion.cc +++ b/tensorflow/compiler/xla/service/cpu/cpu_instruction_fusion.cc @@ -48,10 +48,15 @@ bool IsMatrixVectorDot(const HloInstruction* hlo) { (hlo_shape.dimensions(0) == 1 || hlo_shape.dimensions(1) == 1); } +bool HasExactlyOneUse(const HloInstruction& hlo_instr) { + return hlo_instr.user_count() == 1 && + absl::c_count(hlo_instr.users().front()->operands(), &hlo_instr) == 1; +} + bool CanBeOutputFused(const HloInstruction* producer, const HloInstruction* consumer) { return consumer->opcode() == HloOpcode::kAdd && IsMatrixVectorDot(producer) && - producer->user_count() == 1; + HasExactlyOneUse(*producer) == 1; } bool CanBeOutputFusedIntoSomeOperand(const HloInstruction* consumer) { diff --git a/tensorflow/compiler/xla/service/cpu/cpu_instruction_fusion_test.cc b/tensorflow/compiler/xla/service/cpu/cpu_instruction_fusion_test.cc index c95a514ca04..527df0bd1c2 100644 --- a/tensorflow/compiler/xla/service/cpu/cpu_instruction_fusion_test.cc +++ b/tensorflow/compiler/xla/service/cpu/cpu_instruction_fusion_test.cc @@ -321,7 +321,7 @@ TEST_F(OpcodeFusionTest, Exponential_Reshape_Negate) { builder.AddInstruction( HloInstruction::CreateUnary(result_shape, HloOpcode::kNegate, reshape2)); - auto module = CreateNewUnverifiedModule(); + auto module = CreateNewVerifiedModule(); module->AddEntryComputation(builder.Build()); RunFusionAndCheckOpcodesWereFused( @@ -370,7 +370,7 @@ TEST_F(OpcodeFusionTest, Broadcast_Negate) { builder.AddInstruction(HloInstruction::CreateUnary( result_shape, HloOpcode::kNegate, broadcast1)); - auto module = CreateNewUnverifiedModule(); + auto module = CreateNewVerifiedModule(); module->AddEntryComputation(builder.Build()); RunFusionAndCheckOpcodesWereFused( @@ -410,7 +410,7 @@ TEST_F(OpcodeFusionTest, Exponential_Negate) { builder.AddInstruction( HloInstruction::CreateUnary(param_shape, HloOpcode::kNegate, exp1)); - auto module = CreateNewUnverifiedModule(); + auto module = CreateNewVerifiedModule(); module->AddEntryComputation(builder.Build()); RunFusionAndCheckOpcodesWereFused( @@ -429,7 +429,7 @@ TEST_F(OpcodeFusionTest, Reshape_Negate) { builder.AddInstruction( HloInstruction::CreateUnary(result_shape, HloOpcode::kNegate, reshape1)); - auto module = CreateNewUnverifiedModule(); + auto module = CreateNewVerifiedModule(); module->AddEntryComputation(builder.Build()); RunFusionAndCheckOpcodesWereFused( @@ -447,7 +447,7 @@ TEST_F(OpcodeFusionTest, Reverse_Negate) { builder.AddInstruction( HloInstruction::CreateUnary(param_shape, HloOpcode::kNegate, reverse1)); - auto module = CreateNewUnverifiedModule(); + auto module = CreateNewVerifiedModule(); module->AddEntryComputation(builder.Build()); RunFusionAndCheckOpcodesWereFused( @@ -489,7 +489,7 @@ TEST_F(OpcodeFusionTest, Exponential_Transpose_Negate) { builder.AddInstruction(HloInstruction::CreateUnary( result_shape, HloOpcode::kNegate, transpose2)); - auto module = CreateNewUnverifiedModule(); + auto module = CreateNewVerifiedModule(); module->AddEntryComputation(builder.Build()); RunFusionAndCheckOpcodesWereFused( @@ -498,7 +498,7 @@ TEST_F(OpcodeFusionTest, Exponential_Transpose_Negate) { } TEST_F(OpcodeFusionTest, UnaryMapOfExp) { - auto module = CreateNewUnverifiedModule(); + auto module = CreateNewVerifiedModule(); HloComputation::Builder builder(TestName()); Shape shape = ShapeUtil::MakeShape(F32, {3, 4}); @@ -517,7 +517,7 @@ TEST_F(OpcodeFusionTest, UnaryMapOfExp) { } TEST_F(OpcodeFusionTest, BinaryMapOfExps) { - auto module = CreateNewUnverifiedModule(); + auto module = CreateNewVerifiedModule(); HloComputation::Builder builder(TestName()); Shape shape = ShapeUtil::MakeShape(F32, {3, 4}); @@ -542,7 +542,7 @@ TEST_F(OpcodeFusionTest, BinaryMapOfExps) { } TEST_F(OpcodeFusionTest, DynamicSliceWithDynamicUpdateSlice) { - auto module = CreateNewUnverifiedModule(); + auto module = CreateNewVerifiedModule(); HloComputation::Builder builder(TestName()); Shape full_shape = ShapeUtil::MakeShape(F32, {10, 100, 1000}); @@ -573,7 +573,7 @@ TEST_F(OpcodeFusionTest, DynamicSliceWithDynamicUpdateSlice) { } TEST_F(OpcodeFusionTest, MessOfFusibleNodes) { - auto module = CreateNewUnverifiedModule(); + auto module = CreateNewVerifiedModule(); HloComputation::Builder builder(TestName()); Shape full_shape = ShapeUtil::MakeShape(F32, {4, 100, 10, 100, 50}); @@ -712,7 +712,7 @@ void CreateComputationForDotAddOutputFusionTest(const string& test_name, } TEST_F(OpcodeFusionTest, DotAddOutputFusion_1x50x19) { - auto module = CreateNewUnverifiedModule(); + auto module = CreateNewVerifiedModule(); CreateComputationForDotAddOutputFusionTest(TestName(), module.get(), /*m=*/1, /*k=*/50, /*n=*/19, /*add_extra_use_for_dot=*/false); @@ -725,7 +725,7 @@ TEST_F(OpcodeFusionTest, DotAddOutputFusion_1x50x19) { } TEST_F(OpcodeFusionTest, DotAddOutputFusion_19x50x1) { - auto module = CreateNewUnverifiedModule(); + auto module = CreateNewVerifiedModule(); CreateComputationForDotAddOutputFusionTest(TestName(), module.get(), /*m=*/19, /*k=*/50, /*n=*/1, /*add_extra_use_for_dot=*/false); @@ -738,7 +738,7 @@ TEST_F(OpcodeFusionTest, DotAddOutputFusion_19x50x1) { } TEST_F(OpcodeFusionTest, DotAddOutputFusion_19x50x19) { - auto module = CreateNewUnverifiedModule(); + auto module = CreateNewVerifiedModule(); CreateComputationForDotAddOutputFusionTest(TestName(), module.get(), /*m=*/19, /*k=*/50, /*n=*/19, /*add_extra_use_for_dot=*/false); @@ -751,7 +751,7 @@ TEST_F(OpcodeFusionTest, DotAddOutputFusion_19x50x19) { } TEST_F(OpcodeFusionTest, DotAddOutputFusion_19x50x1_multi_use) { - auto module = CreateNewUnverifiedModule(); + auto module = CreateNewVerifiedModule(); CreateComputationForDotAddOutputFusionTest(TestName(), module.get(), /*m=*/19, /*k=*/50, /*n=*/1, /*add_extra_use_for_dot=*/true); @@ -763,6 +763,28 @@ TEST_F(OpcodeFusionTest, DotAddOutputFusion_19x50x1_multi_use) { Not(op::Fusion())); } +TEST_F(InstructionFusionTest, + DotOperationFusion_DontOutputFuseDuplicateOperands) { + absl::string_view module_string = R"( +HloModule module + +ENTRY main { + a = f32[50,60]{1,0} parameter(0) + b = f32[60,1]{1,0} parameter(1) + c = f32[50,1]{1,0} dot(a, b), lhs_contracting_dims={1}, rhs_contracting_dims={0} + ROOT d = f32[50,1]{1,0} add(c, c) +} +)"; + + TF_ASSERT_OK_AND_ASSIGN(std::unique_ptr module, + ParseAndReturnVerifiedModule(module_string)); + TF_ASSERT_OK_AND_ASSIGN(bool fused_something, + CpuInstructionFusion().Run(module.get())); + EXPECT_FALSE(fused_something); + EXPECT_THAT(module->entry_computation()->root_instruction(), + Not(op::Fusion())); +} + struct GatherLoopFusionTestSpec { string test_name; string hlo_computation_text; diff --git a/tensorflow/compiler/xla/service/cpu/cpu_layout_assignment_test.cc b/tensorflow/compiler/xla/service/cpu/cpu_layout_assignment_test.cc index 2cd52e4a18a..6c61b64758e 100644 --- a/tensorflow/compiler/xla/service/cpu/cpu_layout_assignment_test.cc +++ b/tensorflow/compiler/xla/service/cpu/cpu_layout_assignment_test.cc @@ -73,7 +73,7 @@ TEST_F(CpuLayoutAssignmentTest, DotWithConstantRhsTensor) { auto result = builder.AddInstruction( CreateCanonicalDot(result_shape, dot_lhs, dot_rhs)); - auto module = CreateNewUnverifiedModule(); + auto module = CreateNewVerifiedModule(); HloComputation* computation = module->AddEntryComputation(builder.Build()); ComputationLayout computation_layout(computation->ComputeProgramShape()); @@ -114,7 +114,7 @@ TEST_F(CpuLayoutAssignmentTest, MultipleDotsWithSameConstantRhsTensor0) { builder.AddInstruction(HloInstruction::CreateBinary( result_shape, HloOpcode::kAdd, dot_a_result, dot_b_result)); - auto module = CreateNewUnverifiedModule(); + auto module = CreateNewVerifiedModule(); HloComputation* computation = module->AddEntryComputation(builder.Build()); ComputationLayout computation_layout(computation->ComputeProgramShape()); @@ -158,7 +158,7 @@ TEST_F(CpuLayoutAssignmentTest, MultipleDotsWithSameConstantRhsTensor1) { auto tuple_result = builder.AddInstruction( HloInstruction::CreateTuple({dot_a_result, dot_b_result})); - auto module = CreateNewUnverifiedModule(); + auto module = CreateNewVerifiedModule(); HloComputation* computation = module->AddEntryComputation(builder.Build()); ComputationLayout computation_layout(computation->ComputeProgramShape()); @@ -192,7 +192,7 @@ TEST_F(CpuLayoutAssignmentTest, DotWithConstantLhsTensor) { auto dot_result = builder.AddInstruction( CreateCanonicalDot(result_shape, dot_lhs, dot_rhs)); - auto module = CreateNewUnverifiedModule(); + auto module = CreateNewVerifiedModule(); HloComputation* computation = module->AddEntryComputation(builder.Build()); ComputationLayout computation_layout(computation->ComputeProgramShape()); @@ -232,7 +232,7 @@ TEST_F(CpuLayoutAssignmentTest, DotWithConstantRhsTensorThroughGTE) { auto dot_result = builder.AddInstruction( CreateCanonicalDot(result_shape, dot_lhs, dot_rhs)); - auto module = CreateNewUnverifiedModule(); + auto module = CreateNewVerifiedModule(); HloComputation* computation = module->AddEntryComputation(builder.Build()); ComputationLayout computation_layout(computation->ComputeProgramShape()); @@ -353,7 +353,7 @@ static void AssertCorrectLayoutForDotOutputFusion( } TEST_F(CpuLayoutAssignmentTest, DotOutputFusion_1x50x19_dot_idx_0) { - std::unique_ptr module = CreateNewUnverifiedModule(); + std::unique_ptr module = CreateNewVerifiedModule(); TF_ASSERT_OK_AND_ASSIGN( DotOutputFusionLayoutAssignmentResult layout_assignment_result, RunDotOutputFusion(module.get(), TestName(), /*m=*/1, /*k=*/50, /*n=*/19, @@ -365,7 +365,7 @@ TEST_F(CpuLayoutAssignmentTest, DotOutputFusion_1x50x19_dot_idx_0) { } TEST_F(CpuLayoutAssignmentTest, DotOutputFusion_1x50x19_dot_idx_1) { - std::unique_ptr module = CreateNewUnverifiedModule(); + std::unique_ptr module = CreateNewVerifiedModule(); TF_ASSERT_OK_AND_ASSIGN( DotOutputFusionLayoutAssignmentResult layout_assignment_result, RunDotOutputFusion(module.get(), TestName(), /*m=*/1, /*k=*/50, /*n=*/19, @@ -377,7 +377,7 @@ TEST_F(CpuLayoutAssignmentTest, DotOutputFusion_1x50x19_dot_idx_1) { } TEST_F(CpuLayoutAssignmentTest, DotOutputFusion_19x50x1_dot_idx_0) { - std::unique_ptr module = CreateNewUnverifiedModule(); + std::unique_ptr module = CreateNewVerifiedModule(); TF_ASSERT_OK_AND_ASSIGN( DotOutputFusionLayoutAssignmentResult layout_assignment_result, RunDotOutputFusion(module.get(), TestName(), /*m=*/19, /*k=*/50, /*n=*/1, @@ -389,7 +389,7 @@ TEST_F(CpuLayoutAssignmentTest, DotOutputFusion_19x50x1_dot_idx_0) { } TEST_F(CpuLayoutAssignmentTest, DotOutputFusion_19x50x1_dot_idx_1) { - std::unique_ptr module = CreateNewUnverifiedModule(); + std::unique_ptr module = CreateNewVerifiedModule(); TF_ASSERT_OK_AND_ASSIGN( DotOutputFusionLayoutAssignmentResult layout_assignment_result, RunDotOutputFusion(module.get(), TestName(), /*m=*/19, /*k=*/50, /*n=*/1, @@ -401,7 +401,7 @@ TEST_F(CpuLayoutAssignmentTest, DotOutputFusion_19x50x1_dot_idx_1) { } TEST_F(CpuLayoutAssignmentTest, DotOutputFusion_19x50x19_dot_idx_0) { - std::unique_ptr module = CreateNewUnverifiedModule(); + std::unique_ptr module = CreateNewVerifiedModule(); TF_ASSERT_OK_AND_ASSIGN( DotOutputFusionLayoutAssignmentResult layout_assignment_result, RunDotOutputFusion(module.get(), TestName(), /*m=*/19, /*k=*/50, /*n=*/19, @@ -413,7 +413,7 @@ TEST_F(CpuLayoutAssignmentTest, DotOutputFusion_19x50x19_dot_idx_0) { } TEST_F(CpuLayoutAssignmentTest, DotOutputFusion_19x50x19_dot_idx_1) { - std::unique_ptr module = CreateNewUnverifiedModule(); + std::unique_ptr module = CreateNewVerifiedModule(); TF_ASSERT_OK_AND_ASSIGN( DotOutputFusionLayoutAssignmentResult layout_assignment_result, RunDotOutputFusion(module.get(), TestName(), /*m=*/19, /*k=*/50, /*n=*/19, diff --git a/tensorflow/compiler/xla/service/cpu/cpu_options.cc b/tensorflow/compiler/xla/service/cpu/cpu_options.cc index b8ace570268..92debb83e33 100644 --- a/tensorflow/compiler/xla/service/cpu/cpu_options.cc +++ b/tensorflow/compiler/xla/service/cpu/cpu_options.cc @@ -22,7 +22,6 @@ limitations under the License. namespace { const char* const kXlaOptimizeForSizeCpuOption = "xla_cpu_optimize_for_size"; -const char* const kXlaDisableVectorizedReduce = "xla_disable_vectorized_reduce"; const char* const kLlvmIrDotTilingFactor = "xla_llvm_dot_tiling_factor"; const char* const kXlaEnableExperimentalLlvmIrGemm = "xla_enable_experimental_llvm_ir_gemm"; diff --git a/tensorflow/compiler/xla/service/cpu/ir_emitter.cc b/tensorflow/compiler/xla/service/cpu/ir_emitter.cc index 620c45fa391..4032c2da2f3 100644 --- a/tensorflow/compiler/xla/service/cpu/ir_emitter.cc +++ b/tensorflow/compiler/xla/service/cpu/ir_emitter.cc @@ -111,7 +111,7 @@ IrEmitter::IrEmitter( StatusOr IrEmitter::EmitComputation( HloComputation* computation, const string& function_name_prefix, bool is_top_level_computation, - const std::vector* instruction_order) { + const std::vector* instruction_order) { string function_name = name_uniquer_.GetUniqueName(function_name_prefix); VLOG(2) << "Emitting IR for CPU function [" << function_name_prefix << "]; ordered? " << (instruction_order != nullptr); @@ -140,7 +140,7 @@ StatusOr IrEmitter::EmitComputation( // readcyclecounter if it is unavailable. bool use_rdtscp = arch_type_ == llvm::Triple::ArchType::x86 || arch_type_ == llvm::Triple::ArchType::x86_64; - profiling_state_ = ProfilingState(use_rdtscp, GetProfileCountersArgument()); + profiling_state_ = ProfilingState(use_rdtscp); if (instruction_order == nullptr) { TF_RETURN_IF_ERROR(computation->Accept(this)); } else { @@ -1379,33 +1379,6 @@ Status IrEmitter::HandleCrossReplicaSum(HloInstruction* crs) { return Status::OK(); } -// Fills up the free variables in 'index_with_free_var' with values from -// 'filler_index'. The size of free variables must be the same as the -// size of 'filler_index'. -// -// This is often used after dimension reduction, where -// 'index_with_free_var' has one or more dimensions reduced, which serves as -// free variables (represented as nullptr). For example, if we have a 4 -// dimensional input and index for the dimension being reduced is -// 2 (third dimension), we will have an index like [i, j, NULL, k] -// after reduced dimension. -// -// Here we fill up that free variable by 'filler_index', which contains -// the value in the reduced dimension. -static llvm_ir::IrArray::Index FillReducedDimensionIndex( - llvm_ir::IrArray::Index index_with_free_var, - llvm_ir::IrArray::Index filler_index) { - llvm_ir::IrArray::Index::const_iterator it = filler_index.begin(); - - for (size_t i = 0; i < index_with_free_var.size(); ++i) { - if (index_with_free_var[i] == nullptr) { - index_with_free_var[i] = *it++; - } - } - CHECK(filler_index.end() == it); - return index_with_free_var; -} - Status IrEmitter::HandleParameter(HloInstruction* parameter) { VLOG(2) << "HandleParameter: " << parameter->ToString(); return EmitTargetAddressForOp(parameter); @@ -2194,14 +2167,6 @@ Status IrEmitter::HandlePad(HloInstruction* pad) { return Status::OK(); } -// If `hlo` is a Transpose, returns its operand; otherwise returns `hlo` itself. -static const HloInstruction* StripTranspose(const HloInstruction& hlo) { - if (hlo.IsRank2Transpose()) { - return hlo.operand(0); - } - return &hlo; -} - Status IrEmitter::HandleFusion(HloInstruction* fusion) { auto* root = fusion->fused_expression_root(); if (llvm_ir::CanEmitFusedDynamicUpdateSliceInPlace(fusion, assignment_)) { @@ -2600,10 +2565,17 @@ Status IrEmitter::HandleConditional(HloInstruction* conditional) { return Status::OK(); } -Status IrEmitter::HandleAfterAll(HloInstruction* gen_token) { - TF_RET_CHECK(ByteSizeOf(gen_token->shape()) == 0); +Status IrEmitter::HandleAfterAll(HloInstruction* after_all) { + TF_RET_CHECK(ByteSizeOf(after_all->shape()) == 0); // No code to generate, but we need to emit an address for book-keeping. - TF_RETURN_IF_ERROR(EmitTargetAddressForOp(gen_token)); + TF_RETURN_IF_ERROR(EmitTargetAddressForOp(after_all)); + return Status::OK(); +} + +Status IrEmitter::HandleAddDependency(HloInstruction* add_dependency) { + // AddDedendency just forwards its zero-th operand. + emitted_value_[add_dependency] = + GetEmittedValueFor(add_dependency->operand(0)); return Status::OK(); } diff --git a/tensorflow/compiler/xla/service/cpu/ir_emitter.h b/tensorflow/compiler/xla/service/cpu/ir_emitter.h index 136b88ff75e..559a8162a2d 100644 --- a/tensorflow/compiler/xla/service/cpu/ir_emitter.h +++ b/tensorflow/compiler/xla/service/cpu/ir_emitter.h @@ -101,7 +101,7 @@ class IrEmitter : public DfsHloVisitorWithDefault, StatusOr EmitComputation( HloComputation* computation, const string& function_name_prefix, bool is_top_level_computation, - const std::vector* instruction_order); + const std::vector* instruction_order); llvm::IRBuilder<>* b() { return &b_; } @@ -159,7 +159,8 @@ class IrEmitter : public DfsHloVisitorWithDefault, Status HandleConcatenate(HloInstruction* concatenate) override; Status HandleConditional(HloInstruction* conditional) override; Status HandleScatter(HloInstruction* scatter) override; - Status HandleAfterAll(HloInstruction* gen_token) override; + Status HandleAfterAll(HloInstruction* after_all) override; + Status HandleAddDependency(HloInstruction* add_dependency) override; Status HandleRng(HloInstruction* rng) override; Status FinishVisit(HloInstruction* root) override; @@ -467,9 +468,8 @@ class IrEmitter : public DfsHloVisitorWithDefault, // profiling a computation. class ProfilingState { public: - ProfilingState() : use_rdtscp_(false), prof_counters_(nullptr) {} - ProfilingState(bool use_rdtscp, llvm::Value* prof_counters) - : use_rdtscp_(use_rdtscp), prof_counters_(prof_counters) {} + ProfilingState() : use_rdtscp_(false) {} + explicit ProfilingState(bool use_rdtscp) : use_rdtscp_(use_rdtscp) {} // Record the cycle counter before an HLO executes. void RecordCycleStart(llvm::IRBuilder<>* b, HloInstruction* hlo); @@ -494,9 +494,6 @@ class IrEmitter : public DfsHloVisitorWithDefault, // intrinsic? bool use_rdtscp_; - // The argument which corresponds to the profile counter buffer. - llvm::Value* prof_counters_; - // The first read cycle counter in the program. llvm::Value* first_read_cycle_start_ = nullptr; diff --git a/tensorflow/compiler/xla/service/cpu/runtime_key_value_sort.cc b/tensorflow/compiler/xla/service/cpu/runtime_key_value_sort.cc index 669eeb95f32..722aa3120ef 100644 --- a/tensorflow/compiler/xla/service/cpu/runtime_key_value_sort.cc +++ b/tensorflow/compiler/xla/service/cpu/runtime_key_value_sort.cc @@ -17,6 +17,7 @@ limitations under the License. #include #include #include +#include #include #include #include @@ -41,61 +42,60 @@ void KeyValueSort(std::pair* row_to_sort, int64 num_elements) { std::sort(row_to_sort, row_to_sort + num_elements); } -// For floating point numbers, we want a total order comparator. -NaN and NaN -// should appear at the beginning and end of the ordering, and -0.0 should -// appear before 0.0. Also we want to have a stable sort, so if the keys are the -// same, we compare the index values. -template -bool LessThan(KeyType lhs, int64 lhs_index, KeyType rhs, int64 rhs_index) { - bool lhs_is_negative = std::signbit(lhs); - bool rhs_is_negative = std::signbit(rhs); - // If the signs are different, we can just compare the signs. - if (lhs_is_negative != rhs_is_negative) { - return lhs_is_negative && !rhs_is_negative; +// We would like a total order of floating point numbers so that the +// sort has a predictable behavior in the presence of NaNs. Rather +// than using floating point comparison, we use the following trick: +// If f is a float, and +// x = bit_cast(f); +// y = x < 0 ? 0x7FFFFFFF - x : x; +// then y is ordered as an int32 such that finite values have the +// obvious order, -0 is ordered before 0, and -NaN and NaN appear at +// the beginning and end of the ordering. +template +CastType Convert(KeyType value) { + CastType casted_value; + memcpy(&casted_value, &value, sizeof(CastType)); + if (casted_value < 0) { + return static_cast(std::numeric_limits::max()) - + casted_value; } - bool lhs_nan = std::isnan(lhs); - bool rhs_nan = std::isnan(rhs); - // Exactly one number is nan? - if (lhs_nan != rhs_nan) { - if (lhs_nan) { - return lhs_is_negative; - } - return !rhs_is_negative; - } - if (lhs != rhs) { - return lhs < rhs; - } - return lhs_index < rhs_index; + return casted_value; +} + +template +bool LessThan(KeyType lhs, KeyType rhs) { + return Convert(lhs) < + Convert(rhs); } template <> void KeyValueSort(std::pair* row_to_sort, int64 num_elements) { - std::sort(row_to_sort, row_to_sort + num_elements, - [](const std::pair& lhs, - const std::pair& rhs) -> bool { - return LessThan(lhs.first, lhs.second, rhs.first, rhs.second); - }); + std::stable_sort(row_to_sort, row_to_sort + num_elements, + [](const std::pair& lhs, + const std::pair& rhs) -> bool { + return LessThan(lhs.first, rhs.first); + }); } template <> void KeyValueSort(std::pair* row_to_sort, int64 num_elements) { - std::sort(row_to_sort, row_to_sort + num_elements, - [](const std::pair& lhs, - const std::pair& rhs) -> bool { - return LessThan(lhs.first, lhs.second, rhs.first, rhs.second); - }); + std::stable_sort(row_to_sort, row_to_sort + num_elements, + [](const std::pair& lhs, + const std::pair& rhs) -> bool { + return LessThan(lhs.first, rhs.first); + }); } template <> void KeyValueSort(std::pair* row_to_sort, int64 num_elements) { - std::sort(row_to_sort, row_to_sort + num_elements, - [](const std::pair& lhs, - const std::pair& rhs) -> bool { - return LessThan( - Eigen::half_impl::half_to_float(lhs.first), lhs.second, - Eigen::half_impl::half_to_float(rhs.first), rhs.second); - }); + std::stable_sort(row_to_sort, row_to_sort + num_elements, + [](const std::pair& lhs, + const std::pair& rhs) -> bool { + return LessThan( + Eigen::half_impl::half_to_float(lhs.first), + Eigen::half_impl::half_to_float(rhs.first)); + }); } template diff --git a/tensorflow/compiler/xla/service/cpu/simple_orc_jit.cc b/tensorflow/compiler/xla/service/cpu/simple_orc_jit.cc index f77641eb7da..efccadedf27 100644 --- a/tensorflow/compiler/xla/service/cpu/simple_orc_jit.cc +++ b/tensorflow/compiler/xla/service/cpu/simple_orc_jit.cc @@ -128,8 +128,18 @@ SimpleOrcJIT::SimpleOrcJIT(const llvm::TargetOptions& target_options, } llvm::JITSymbol SimpleOrcJIT::ResolveRuntimeSymbol(const std::string& name) { - void* func_addr = CustomCallTargetRegistry::Global()->Lookup(name); + void* func_addr = nullptr; + if (name.size() > 1 && name.front() == data_layout_.getGlobalPrefix()) { + // On Mac OS X, 'name' may have a leading underscore prefix, even though the + // registered name may not. + std::string stripped_name(name.begin() + 1, name.end()); + func_addr = CustomCallTargetRegistry::Global()->Lookup(stripped_name); + } else { + func_addr = CustomCallTargetRegistry::Global()->Lookup(name); + } + if (func_addr == nullptr) { + VLOG(2) << "Unable to resolve runtime symbol: " << name; return nullptr; } llvm::JITEvaluatedSymbol symbol_info(reinterpret_cast(func_addr), diff --git a/tensorflow/compiler/xla/service/cpu/tests/cpu_eigen_dot_operation_test.cc b/tensorflow/compiler/xla/service/cpu/tests/cpu_eigen_dot_operation_test.cc index 691b3c7bee2..f8f5f392da8 100644 --- a/tensorflow/compiler/xla/service/cpu/tests/cpu_eigen_dot_operation_test.cc +++ b/tensorflow/compiler/xla/service/cpu/tests/cpu_eigen_dot_operation_test.cc @@ -50,7 +50,7 @@ class CpuEigenDotOperationTest /*entry_point_name=*/"entry", /*relocation_model=*/CpuAotCompilationOptions::RelocationModel::Static}; - auto hlo_module = CreateNewUnverifiedModule(); + auto hlo_module = CreateNewVerifiedModule(); hlo_module->AddEntryComputation(std::move(entry_computation)); CompileAheadOfTimeAndVerifyIr(std::move(hlo_module), options, diff --git a/tensorflow/compiler/xla/service/cpu/tests/cpu_external_constants_test.cc b/tensorflow/compiler/xla/service/cpu/tests/cpu_external_constants_test.cc index d201a151d7a..e30f95311fc 100644 --- a/tensorflow/compiler/xla/service/cpu/tests/cpu_external_constants_test.cc +++ b/tensorflow/compiler/xla/service/cpu/tests/cpu_external_constants_test.cc @@ -46,7 +46,7 @@ class CpuExternalConstantsTest : public CpuCodegenTest { builder.AddInstruction( HloInstruction::CreateBinary(shape, HloOpcode::kAdd, param, constant)); - std::unique_ptr module = CreateNewUnverifiedModule(); + std::unique_ptr module = CreateNewVerifiedModule(); module->AddEntryComputation(builder.Build()); CompileAndVerifyIr(std::move(module), filecheck_pattern, diff --git a/tensorflow/compiler/xla/service/cpu/tests/cpu_intrinsic_test.cc b/tensorflow/compiler/xla/service/cpu/tests/cpu_intrinsic_test.cc index 773336c7a92..9b10c49f4f5 100644 --- a/tensorflow/compiler/xla/service/cpu/tests/cpu_intrinsic_test.cc +++ b/tensorflow/compiler/xla/service/cpu/tests/cpu_intrinsic_test.cc @@ -91,7 +91,7 @@ TEST_P(CpuUnaryIntrinsicTest, DoIt) { /*entry_point_name=*/"entry", /*relocation_model=*/CpuAotCompilationOptions::RelocationModel::Static}; - auto hlo_module = CreateNewUnverifiedModule(); + auto hlo_module = CreateNewVerifiedModule(); hlo_module->AddEntryComputation(std::move(computation)); string check_lines{spec.check_lines.data(), spec.check_lines.size()}; diff --git a/tensorflow/compiler/xla/service/cpu/tests/cpu_literal_caching_test.cc b/tensorflow/compiler/xla/service/cpu/tests/cpu_literal_caching_test.cc index 3b87683ffff..fa0e09ff6b5 100644 --- a/tensorflow/compiler/xla/service/cpu/tests/cpu_literal_caching_test.cc +++ b/tensorflow/compiler/xla/service/cpu/tests/cpu_literal_caching_test.cc @@ -63,7 +63,7 @@ CHECK-NOT: private constant [48 x i8] )"; TF_ASSERT_OK_AND_ASSIGN(std::unique_ptr module, - ParseHloString(hlo_text)); + ParseAndReturnVerifiedModule(hlo_text)); CpuAotCompilationOptions options{ /*triple=*/"x86_64-pc-linux", /*cpu_name=*/"", /*features=*/"", @@ -104,14 +104,14 @@ ENTRY main { )"; string filecheck_pattern = R"( -CHECK: private constant [4 x i8] -CHECK: private constant [8 x i8] +CHECK-DAG: private constant [4 x i8] +CHECK-DAG: private constant [8 x i8] CHECK-NOT: private constant [4 x i8] CHECK-NOT: private constant [8 x i8] )"; TF_ASSERT_OK_AND_ASSIGN(std::unique_ptr module, - ParseHloString(hlo_text)); + ParseAndReturnVerifiedModule(hlo_text)); CpuAotCompilationOptions options{ /*triple=*/"x86_64-pc-linux", /*cpu_name=*/"", /*features=*/"", diff --git a/tensorflow/compiler/xla/service/cpu/tests/cpu_noalias_test.cc b/tensorflow/compiler/xla/service/cpu/tests/cpu_noalias_test.cc index f5419b7063b..a7702c2aeea 100644 --- a/tensorflow/compiler/xla/service/cpu/tests/cpu_noalias_test.cc +++ b/tensorflow/compiler/xla/service/cpu/tests/cpu_noalias_test.cc @@ -56,7 +56,7 @@ TEST_F(CpuNoAliasTest, Concat) { std::unique_ptr computation = builder.Build(); - auto hlo_module = CreateNewUnverifiedModule(); + auto hlo_module = CreateNewVerifiedModule(); hlo_module->AddEntryComputation(std::move(computation)); // Now that we have an HLO module, build an llvm_ir::AliasAnalysis for it. diff --git a/tensorflow/compiler/xla/service/cpu/xfeed_manager.h b/tensorflow/compiler/xla/service/cpu/xfeed_manager.h index 990ff94ba23..70008947f37 100644 --- a/tensorflow/compiler/xla/service/cpu/xfeed_manager.h +++ b/tensorflow/compiler/xla/service/cpu/xfeed_manager.h @@ -23,6 +23,7 @@ limitations under the License. #include #include "absl/types/span.h" +#include "tensorflow/compiler/xla/shape.h" #include "tensorflow/compiler/xla/statusor.h" #include "tensorflow/compiler/xla/types.h" #include "tensorflow/compiler/xla/xla_data.pb.h" diff --git a/tensorflow/compiler/xla/service/dfs_hlo_visitor.h b/tensorflow/compiler/xla/service/dfs_hlo_visitor.h index d6371283221..e84bf00153a 100644 --- a/tensorflow/compiler/xla/service/dfs_hlo_visitor.h +++ b/tensorflow/compiler/xla/service/dfs_hlo_visitor.h @@ -251,6 +251,7 @@ class DfsHloVisitorBase { virtual Status HandleBatchNormGrad(HloInstructionPtr hlo) = 0; + virtual Status HandleAddDependency(HloInstructionPtr add_dependency) = 0; virtual Status HandleAfterAll(HloInstructionPtr token) = 0; // Invoked to inform the visitor that the traversal has completed, and that diff --git a/tensorflow/compiler/xla/service/dfs_hlo_visitor_with_default.h b/tensorflow/compiler/xla/service/dfs_hlo_visitor_with_default.h index e57184f639f..80ea5be298a 100644 --- a/tensorflow/compiler/xla/service/dfs_hlo_visitor_with_default.h +++ b/tensorflow/compiler/xla/service/dfs_hlo_visitor_with_default.h @@ -206,6 +206,9 @@ class DfsHloVisitorWithDefaultBase Status HandleGetDimensionSize(HloInstructionPtr get_size) override { return DefaultAction(get_size); } + Status HandleAddDependency(HloInstructionPtr add_dependency) override { + return DefaultAction(add_dependency); + } // Invoked to inform the visitor that the traversal has completed, and that // the root was "root". diff --git a/tensorflow/compiler/xla/service/dynamic_parameter_binding.cc b/tensorflow/compiler/xla/service/dynamic_parameter_binding.cc new file mode 100644 index 00000000000..c8bfc890506 --- /dev/null +++ b/tensorflow/compiler/xla/service/dynamic_parameter_binding.cc @@ -0,0 +1,138 @@ +/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/compiler/xla/service/dynamic_parameter_binding.h" +#include "tensorflow/compiler/xla/service/hlo_computation.h" +#include "tensorflow/compiler/xla/service/hlo_instruction.h" +#include "tensorflow/compiler/xla/service/hlo_module.h" + +namespace xla { + +Status DynamicParameterBinding::Bind( + const DynamicParameter& dynamic_parameter, + const DynamicDimension& dynamic_dimension) { + auto result = bindings_.emplace(dynamic_dimension, dynamic_parameter); + TF_RET_CHECK(result.second); + return Status::OK(); +} + +absl::optional +DynamicParameterBinding::GetBinding(const DynamicDimension& dynamic_dimension) { + auto param_iter = bindings_.find(dynamic_dimension); + if (param_iter == bindings_.end()) { + return absl::nullopt; + } + return param_iter->second; +} + +DynamicParameterBindingProto DynamicParameterBinding::ToProto() const { + DynamicParameterBindingProto result; + for (const auto& binding : bindings_) { + const DynamicDimension& dynamic_dimension = binding.first; + const DynamicParameter& dynamic_param = binding.second; + DynamicParameterBindingProto::Binding binding_proto; + binding_proto.set_dynamic_param_num(dynamic_param.parameter_num); + for (int64 i : dynamic_param.parameter_index) { + binding_proto.add_dynamic_param_index(i); + } + + binding_proto.set_target_param_num(dynamic_dimension.parameter_num); + + for (int64 i : dynamic_dimension.parameter_index) { + binding_proto.add_target_param_index(i); + } + + binding_proto.set_target_param_dim_num(dynamic_dimension.dimension); + result.add_entries()->Swap(&binding_proto); + } + return result; +} + +StatusOr DynamicParameterBinding::CreateFromProto( + const DynamicParameterBindingProto& proto) { + DynamicParameterBinding result; + for (const DynamicParameterBindingProto::Binding& binding : proto.entries()) { + int64 dynamic_param_num = binding.dynamic_param_num(); + ShapeIndex dynamic_param_index(binding.dynamic_param_index().begin(), + binding.dynamic_param_index().end()); + int64 target_param_num = binding.target_param_num(); + ShapeIndex target_param_index(binding.target_param_index().begin(), + binding.target_param_index().end()); + int64 target_dim_num = binding.target_param_num(); + + TF_RETURN_IF_ERROR( + result.Bind(DynamicParameter{dynamic_param_num, dynamic_param_index}, + DynamicDimension{target_param_num, target_param_index, + target_dim_num})); + } + + return result; +} + +string DynamicParameterBinding::ToString() const { + std::vector pieces; + pieces.push_back("DynamicParameterBinding: "); + for (const auto& binding : bindings_) { + const DynamicDimension& dynamic_dimension = binding.first; + const DynamicParameter& dynamic_param = binding.second; + pieces.push_back(absl::StrFormat( + " -- Input param number %lld at %s has dim %lld as dynamic" + " dimension, which is represented by param number %lld at " + "%s", + dynamic_dimension.parameter_num, + dynamic_dimension.parameter_index.ToString(), + dynamic_dimension.dimension, dynamic_param.parameter_num, + dynamic_param.parameter_index.ToString())); + } + return absl::StrJoin(pieces, "\n"); +} + +Status DynamicParameterBinding::ForEachBinding(BindingFn fn) const { + for (const auto& binding : bindings_) { + TF_RETURN_IF_ERROR(fn(binding.second, binding.first)); + } + return Status::OK(); +} + +Status DynamicParameterBinding::Verify(const HloModule& module) const { + const HloComputation* entry = module.entry_computation(); + return ForEachBinding([&](const DynamicParameter& dynamic_parameter, + const DynamicDimension& dynamic_dimension) + -> Status { + TF_RET_CHECK(dynamic_parameter.parameter_num < entry->num_parameters()); + TF_RET_CHECK(dynamic_dimension.parameter_num < entry->num_parameters()); + TF_RET_CHECK(ShapeUtil::IndexIsValid( + entry->parameter_instruction(dynamic_parameter.parameter_num)->shape(), + dynamic_parameter.parameter_index)); + TF_RET_CHECK(ShapeUtil::IndexIsValid( + entry->parameter_instruction(dynamic_dimension.parameter_num)->shape(), + dynamic_dimension.parameter_index)); + TF_RET_CHECK( + dynamic_dimension.dimension < + ShapeUtil::Rank(ShapeUtil::GetSubshape( + entry->parameter_instruction(dynamic_dimension.parameter_num) + ->shape(), + dynamic_dimension.parameter_index))); + return Status::OK(); + }); +} + +std::ostream& operator<<(std::ostream& out, + const DynamicParameterBinding& binding) { + out << binding.ToString(); + return out; +} + +} // namespace xla diff --git a/tensorflow/compiler/xla/service/dynamic_parameter_binding.h b/tensorflow/compiler/xla/service/dynamic_parameter_binding.h new file mode 100644 index 00000000000..dd474d8eed1 --- /dev/null +++ b/tensorflow/compiler/xla/service/dynamic_parameter_binding.h @@ -0,0 +1,125 @@ +/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_COMPILER_XLA_SERVICE_DYNAMIC_PARAMETER_BINDING_H_ +#define TENSORFLOW_COMPILER_XLA_SERVICE_DYNAMIC_PARAMETER_BINDING_H_ + +#include + +#include "absl/container/flat_hash_map.h" +#include "absl/types/optional.h" +#include "tensorflow/compiler/xla/service/hlo.pb.h" +#include "tensorflow/compiler/xla/shape_tree.h" +#include "tensorflow/compiler/xla/shape_util.h" + +namespace xla { + +class HloModule; +// We currently use an explicit API that takes an extra parameter to indicate +// the runtime size of a dynamic dimension. DynamicParameterBinding indicates +// the relationship between parameter: We can have a dynamic parameter that +// points to another target parameter to indicate that the target parameter is +// dynamic. +// +// +// TODO(b/119520625): Remove this API once we have more dynamic shape infra +// ready. +class DynamicParameterBinding { + public: + // DynamicParameter represents a special parameter that is used to represent + // the runtime size of a dimension of another parameter. A dynamic parameter + // has to be a scalar value. + struct DynamicParameter { + // The parameter number of dynamic parameter. + int64 parameter_num; + // The index of the parameter. + ShapeIndex parameter_index; + }; + + // DynamicDimension represents a dimension whose size is determined at + // runtime. A DynamicDimension's runtime size is determined by the binded + // DynamicParameter using `DynamicParameterBinding::Bind` method. + struct DynamicDimension { + // The parameter number of dynamic dimension. + int64 parameter_num; + // The subshape index of the parameter. + ShapeIndex parameter_index; + // The dimension number in the subshape. + int64 dimension; + + // "friend" keyword are added so these functions can be found by ADL. + template + friend H AbslHashValue(H h, const DynamicDimension& m) { + return H::combine(std::move(h), m.parameter_num, m.parameter_index, + m.dimension); + } + + friend bool operator==(const DynamicDimension& lhs, + const DynamicDimension& rhs) { + return lhs.parameter_num == rhs.parameter_num && + lhs.parameter_index == rhs.parameter_index && + lhs.dimension == rhs.dimension; + } + }; + + DynamicParameterBinding() = default; + + virtual ~DynamicParameterBinding() = default; + + // Adds binding which indicates that the dimension indicated by + // `dynamic_dimension` is dynamic, and its runtime size is represented by + // `dynamic_parameter`. + Status Bind(const DynamicParameter& dynamic_parameter, + const DynamicDimension& dynamic_dimension); + + // Returns the parameter and the index representing the runtime size of + // dimension `dim_num` of parameter `param_num` at `param_index`. + // + // Returns nullopt if the binding is not set. + absl::optional GetBinding( + const DynamicDimension& dynamic_dimension); + + using BindingFn = + std::function; + + // Iterate through each binding. + Status ForEachBinding(BindingFn fn) const; + + DynamicParameterBindingProto ToProto() const; + + static StatusOr CreateFromProto( + const DynamicParameterBindingProto& proto); + + string ToString() const; + + // Verifies that the given binding is valid for the given module. + // Specifically, the binding's parameter and parameter size should be valid. + Status Verify(const HloModule& module) const; + + private: + // Keeps track of mappings from DynamicDimension to DynamicParameter. The + // direction of is chosen so that we can easily query if a dimension is + // dynamic and which dynamic parameter represents the real size of that + // dimension. + absl::flat_hash_map bindings_; +}; + +std::ostream& operator<<(std::ostream& out, + const DynamicParameterBinding& binding); + +} // namespace xla + +#endif // TENSORFLOW_COMPILER_XLA_SERVICE_DYNAMIC_PARAMETER_BINDING_H_ diff --git a/tensorflow/compiler/xla/service/dynamic_parameter_binding_test.cc b/tensorflow/compiler/xla/service/dynamic_parameter_binding_test.cc new file mode 100644 index 00000000000..83a6d83dffd --- /dev/null +++ b/tensorflow/compiler/xla/service/dynamic_parameter_binding_test.cc @@ -0,0 +1,153 @@ +/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/compiler/xla/service/dynamic_parameter_binding.h" + +#include +#include + +#include "absl/algorithm/container.h" +#include "tensorflow/compiler/xla/service/hlo_computation.h" +#include "tensorflow/compiler/xla/service/hlo_dce.h" +#include "tensorflow/compiler/xla/service/hlo_instruction.h" +#include "tensorflow/compiler/xla/service/hlo_memory_scheduler.h" +#include "tensorflow/compiler/xla/service/hlo_opcode.h" +#include "tensorflow/compiler/xla/service/hlo_ordering.h" +#include "tensorflow/compiler/xla/service/hlo_parser.h" +#include "tensorflow/compiler/xla/shape_util.h" +#include "tensorflow/compiler/xla/tests/hlo_test_base.h" +#include "tensorflow/compiler/xla/types.h" +#include "tensorflow/core/lib/core/status_test_util.h" + +namespace xla { +namespace { +class DynamicParameterBindingTest : public HloTestBase {}; + +TEST_F(DynamicParameterBindingTest, SimpleBinding) { + // 'b' is a dynamic shape; 'a' represents the real size of b's first + // dimension. + const string module_str = R"( +HloModule TEST + +ENTRY main { + a = f32[] parameter(0) + b = f32[10] parameter(1) + ROOT root = (f32[], f32[10]) tuple(%a, %b) +} +)"; + TF_ASSERT_OK_AND_ASSIGN(std::unique_ptr module, + ParseHloString(module_str)); + + DynamicParameterBinding binding; + + TF_EXPECT_OK( + binding.Bind(DynamicParameterBinding::DynamicParameter{0, {}}, + DynamicParameterBinding::DynamicDimension{1, {}, 0})); + + absl::optional param = + binding.GetBinding( + DynamicParameterBinding::DynamicDimension{/*parameter_num=*/1, + /*parameter_index=*/{}, + /*dimension=*/0}); + EXPECT_TRUE(param); + EXPECT_EQ(param->parameter_num, 0); + EXPECT_EQ(param->parameter_index, ShapeIndex({})); + TF_EXPECT_OK(binding.Verify(*module)); +} + +TEST_F(DynamicParameterBindingTest, TupleBinding) { + // 'gte2' is a dynamic shape; 'gte1' represents the real size of gte2's first + // dimension. + const string module_str = R"( +HloModule TEST + +ENTRY main { + param = (f32[], f32[10]) parameter(0) + gte1 = f32[] get-tuple-element(%param), index=0 + gte2 = f32[10] get-tuple-element(%param), index=1 + ROOT root = (f32[], f32[10]) tuple(%gte1, %gte2) +} +)"; + TF_ASSERT_OK_AND_ASSIGN(std::unique_ptr module, + ParseHloString(module_str)); + + DynamicParameterBinding binding; + + TF_EXPECT_OK( + binding.Bind(DynamicParameterBinding::DynamicParameter{0, {0}}, + DynamicParameterBinding::DynamicDimension{0, {1}, 0})); + + absl::optional param = + binding.GetBinding( + DynamicParameterBinding::DynamicDimension{/*parameter_num=*/0, + /*parameter_index=*/{1}, + /*dimension=*/0}); + + EXPECT_TRUE(param); + EXPECT_EQ(param->parameter_num, 0); + EXPECT_EQ(param->parameter_index, ShapeIndex({0})); + TF_EXPECT_OK(binding.Verify(*module)); +} + +TEST_F(DynamicParameterBindingTest, TupleBindingWithMultiDimension) { + // 'gte2' is a dynamic shape; 'gte1' represents the real size of gte2's both + // dimensions. + const string module_str = R"( +HloModule TEST + +ENTRY main { + param = (f32[], f32[10, 10]) parameter(0) + gte1 = f32[] get-tuple-element(%param), index=0 + gte2 = f32[10, 10] get-tuple-element(%param), index=1 + ROOT root = (f32[], f32[10, 10]) tuple(%gte1, %gte2) +} +)"; + TF_ASSERT_OK_AND_ASSIGN(std::unique_ptr module, + ParseHloString(module_str)); + + DynamicParameterBinding binding; + + TF_EXPECT_OK( + binding.Bind(DynamicParameterBinding::DynamicParameter{0, {0}}, + DynamicParameterBinding::DynamicDimension{0, {1}, 0})); + + TF_EXPECT_OK( + binding.Bind(DynamicParameterBinding::DynamicParameter{0, {0}}, + DynamicParameterBinding::DynamicDimension{0, {1}, 1})); + + absl::optional param = + binding.GetBinding( + DynamicParameterBinding::DynamicDimension{/*parameter_num=*/0, + /*parameter_index=*/{1}, + /*dimension=*/0}); + + EXPECT_TRUE(param); + EXPECT_EQ(param->parameter_num, 0); + EXPECT_EQ(param->parameter_index, ShapeIndex({0})); + + absl::optional param2 = + binding.GetBinding( + DynamicParameterBinding::DynamicDimension{/*parameter_num=*/0, + /*parameter_index=*/{1}, + /*dimension=*/0}); + EXPECT_TRUE(param2); + EXPECT_EQ(param2->parameter_num, 0); + EXPECT_EQ(param2->parameter_index, ShapeIndex({0})); + + TF_EXPECT_OK(binding.Verify(*module)); +} + +} // namespace +} // namespace xla diff --git a/tensorflow/compiler/xla/service/elemental_ir_emitter.cc b/tensorflow/compiler/xla/service/elemental_ir_emitter.cc index f98c943669b..6f1f95f2e90 100644 --- a/tensorflow/compiler/xla/service/elemental_ir_emitter.cc +++ b/tensorflow/compiler/xla/service/elemental_ir_emitter.cc @@ -22,6 +22,7 @@ limitations under the License. // IWYU pragma: no_include "llvm/IR/Intrinsics.gen.inc" #include "absl/algorithm/container.h" +#include "absl/container/flat_hash_map.h" #include "absl/strings/str_cat.h" #include "llvm/IR/BasicBlock.h" #include "llvm/IR/Instructions.h" @@ -1671,26 +1672,66 @@ StatusOr ElementalIrEmitter::EmitElementalConcatenate( b_->SetInsertPoint(init_block); + // Assign a unique id for each *different* operand, and count how often each + // operand is used. If all operands are different, the usage count will be 1 + // for each operand. + absl::flat_hash_map to_unique_operand_id; + std::vector operand_usage_count; + for (const auto* operand : hlo->operands()) { + if (to_unique_operand_id.contains(operand)) { + ++operand_usage_count[to_unique_operand_id[operand]]; + } else { + int64 unique_operand_id = to_unique_operand_id.size(); + to_unique_operand_id[operand] = unique_operand_id; + operand_usage_count.push_back(1); + } + } + + // To avoid that we emit the same operand more than once, we create one basic + // block for each *different* operand with a PHI node for the different source + // index inputs. + std::vector emit_operand_blocks( + to_unique_operand_id.size(), nullptr); + std::vector source_index_phis(to_unique_operand_id.size(), + nullptr); + for (const auto* operand : hlo->operands()) { + int64 operand_id = to_unique_operand_id[operand]; + if (emit_operand_blocks[operand_id] != nullptr) { + continue; + } + + emit_operand_blocks[operand_id] = llvm_ir::CreateBasicBlock( + exit_block, StrCat("concat_index_from_operand_id", operand_id), b_); + auto saved_insert_point = b_->GetInsertPoint(); + llvm_ir::SetToFirstInsertPoint(emit_operand_blocks[operand_id], b_); + source_index_phis[operand_id] = + PHI(source_index.GetType(), operand_usage_count[operand_id]); + auto operand_index = source_index; + operand_index[concat_dim] = source_index_phis[operand_id]; + + // Create the terminator of the block before calling operand generators, + // because they require non-degenerate basic blocks. + b_->SetInsertPoint(llvm::BranchInst::Create( + exit_block, /*InsertAtEnd=*/emit_operand_blocks[operand_id])); + TF_ASSIGN_OR_RETURN(llvm::Value * value, + operand_to_generator.at(operand)(operand_index)); + output->addIncoming(value, b_->GetInsertBlock()); + b_->SetInsertPoint(init_block, saved_insert_point); + } + for (int64 operand_idx = 0; operand_idx < hlo->operand_count(); ++operand_idx) { const HloInstruction* operand = hlo->operand(operand_idx); - auto true_block = llvm_ir::CreateBasicBlock( - exit_block, StrCat("concat_index_from_operand", operand_idx), b_); auto false_block = llvm_ir::CreateBasicBlock( exit_block, StrCat("concat_index_not_from_operand", operand_idx), b_); auto concat_dim_size = llvm::ConstantInt::get(source_index[concat_dim]->getType(), operand->shape().dimensions(concat_dim)); - CondBr(ICmpULT(source_index[concat_dim], concat_dim_size), true_block, - false_block); - - // Create the terminator of the true block before calling operand - // generators, because they require non-degenerate basic blocks. - b_->SetInsertPoint( - llvm::BranchInst::Create(exit_block, /*InsertAtEnd=*/true_block)); - TF_ASSIGN_OR_RETURN(llvm::Value * value, - operand_to_generator.at(operand)(source_index)); - output->addIncoming(value, b_->GetInsertBlock()); + int64 operand_id = to_unique_operand_id[operand]; + source_index_phis[operand_id]->addIncoming(source_index[concat_dim], + b_->GetInsertBlock()); + CondBr(ICmpULT(source_index[concat_dim], concat_dim_size), + emit_operand_blocks[operand_id], false_block); // Subtract the size of the concat dimension of the current operand // from the source index. @@ -2204,13 +2245,15 @@ llvm_ir::ElementGenerator ElementalIrEmitter::MakeElementGenerator( : iota->shape(); PrimitiveType component_element_type = component_shape.element_type(); llvm::Value* iota_result; - if (ShapeUtil::ElementIsIntegral(component_shape)) { + if (primitive_util::IsIntegralType(component_element_type) || + component_element_type == PRED) { iota_result = b_->CreateIntCast( elem_index_linear, llvm_ir::PrimitiveTypeToIrType(component_element_type, module_), /*isSigned=*/false); } else { - TF_RET_CHECK(ShapeUtil::ElementIsFloating(component_shape)) + TF_RET_CHECK( + primitive_util::IsFloatingPointType(component_element_type)) << component_element_type; llvm::Type* float_ir_type; if (component_element_type == BF16) { diff --git a/tensorflow/compiler/xla/service/executable.h b/tensorflow/compiler/xla/service/executable.h index 45f620f3f33..b34bca55a48 100644 --- a/tensorflow/compiler/xla/service/executable.h +++ b/tensorflow/compiler/xla/service/executable.h @@ -61,7 +61,7 @@ struct ExecutionOutput { class Executable { public: explicit Executable( - std::unique_ptr hlo_module, + std::unique_ptr hlo_module, std::unique_ptr hlo_profile_printer_data, std::unique_ptr hlo_profile_index_map) : hlo_module_(std::move(hlo_module)), @@ -162,7 +162,7 @@ class Executable { return hlo_profile_printer_data_ != nullptr; } - const HloModule& module() const { return *hlo_module_; } + HloModule& module() const { return *hlo_module_; } const bool has_module() const { return hlo_module_ != nullptr; } @@ -199,7 +199,7 @@ class Executable { // HloModule this was compiled from. BufferAssignment keeps pointers to // HloInstructions owned by the HloModule so we need to keep the HloModule // around. - const std::unique_ptr hlo_module_; + const std::unique_ptr hlo_module_; // HloSnapshot this was compiled from. Null if not dumping executions. std::unique_ptr hlo_snapshot_; diff --git a/tensorflow/compiler/xla/service/gpu/BUILD b/tensorflow/compiler/xla/service/gpu/BUILD index b1629616acd..bfd1b6cb149 100644 --- a/tensorflow/compiler/xla/service/gpu/BUILD +++ b/tensorflow/compiler/xla/service/gpu/BUILD @@ -701,6 +701,7 @@ cc_library( "//tensorflow/compiler/xla/service:hlo_cse", "//tensorflow/compiler/xla/service:hlo_dce", "//tensorflow/compiler/xla/service:hlo_element_type_converter", + "//tensorflow/compiler/xla/service:hlo_get_dimension_size_rewriter", "//tensorflow/compiler/xla/service:hlo_pass", "//tensorflow/compiler/xla/service:hlo_pass_pipeline", "//tensorflow/compiler/xla/service:hlo_proto", diff --git a/tensorflow/compiler/xla/service/gpu/cudnn_conv_rewriter.cc b/tensorflow/compiler/xla/service/gpu/cudnn_conv_rewriter.cc index 4ce877f62a5..e81850db69e 100644 --- a/tensorflow/compiler/xla/service/gpu/cudnn_conv_rewriter.cc +++ b/tensorflow/compiler/xla/service/gpu/cudnn_conv_rewriter.cc @@ -77,7 +77,11 @@ bool CanImplementAsCudnnForwardConv(HloInstruction* conv) { return false; } - if (window_util::HasWindowReversal(conv->window())) { + // CuDNN can perform either cross correlation (no reversal), + // or convolution (all dimensions reversed). + if (dnums.input_spatial_dimensions_size() == 2 + ? !window_util::AllOrNoneReversed(conv->window()) + : window_util::HasWindowReversal(conv->window())) { return false; } return true; diff --git a/tensorflow/compiler/xla/service/gpu/cudnn_conv_runner.cc b/tensorflow/compiler/xla/service/gpu/cudnn_conv_runner.cc index 492d290bf4a..3425e1b4942 100644 --- a/tensorflow/compiler/xla/service/gpu/cudnn_conv_runner.cc +++ b/tensorflow/compiler/xla/service/gpu/cudnn_conv_runner.cc @@ -138,6 +138,7 @@ Status RunCudnnConvImpl(CudnnConvParams params, const int num_dimensions = window.dimensions_size(); CHECK_LE(num_dimensions, 3); + CHECK_GE(num_dimensions, 1); // cuDNN does not support 1D convolutions. We therefore express 1D // convolutions as 2D convolutions where the first spatial dimension is 1. // This matches the behavior of TF (see definition of conv1d in @@ -148,10 +149,15 @@ Status RunCudnnConvImpl(CudnnConvParams params, output_shape.element_type()) << ShapeUtil::HumanString(output_shape); + // If one dimension is reversed, we need to have all dimensions reversed (so + // we're doing convolution not cross correlation). + const bool dims_reversed = window.dimensions()[0].window_reversal(); + CHECK_EQ(num_dimensions, dnums.input_spatial_dimensions_size()); CHECK_EQ(num_dimensions, dnums.kernel_spatial_dimensions_size()); CHECK_EQ(num_dimensions, dnums.output_spatial_dimensions_size()); for (const WindowDimension& dim : window.dimensions()) { + CHECK_EQ(dims_reversed, dim.window_reversal()); CHECK_EQ(dim.padding_low(), dim.padding_high()); CHECK_EQ(dim.base_dilation(), 1) << "cudnn does not support base dilation; it " @@ -198,6 +204,7 @@ Status RunCudnnConvImpl(CudnnConvParams params, ConvolutionDescriptor convolution_descriptor(effective_num_dimensions); convolution_descriptor.set_group_count(feature_group_count); + convolution_descriptor.set_convolution_not_crosscorr(dims_reversed); for (int dim = 0; dim < num_dimensions; ++dim) { convolution_descriptor .set_zero_padding( @@ -363,14 +370,12 @@ StatusOr GetCudnnConvParams( params.output_shape = &conv_result_shape; params.fusion.emplace(); auto& fusion = *params.fusion; - if (backend_config.activation_mode() < - static_cast(se::dnn::ActivationMode::kNumActivationModes)) { - fusion.mode = static_cast( - backend_config.activation_mode()); - } else { + if (!se::dnn::ActivationMode_IsValid(backend_config.activation_mode())) { return InternalError("Bad activation mode: %s", backend_config.ShortDebugString()); } + fusion.mode = static_cast( + backend_config.activation_mode()); fusion.side_input_scale = backend_config.side_input_scale(); params.input_buf = operand_buffers[0]; params.filter_buf = operand_buffers[1]; diff --git a/tensorflow/compiler/xla/service/gpu/elemental_ir_emitter.cc b/tensorflow/compiler/xla/service/gpu/elemental_ir_emitter.cc index 6dcdaf1cfe0..2ab754a4710 100644 --- a/tensorflow/compiler/xla/service/gpu/elemental_ir_emitter.cc +++ b/tensorflow/compiler/xla/service/gpu/elemental_ir_emitter.cc @@ -161,6 +161,16 @@ StatusOr GpuElementalIrEmitter::EmitFloatBinaryOp( PrimitiveType lhs_input_type = op->operand(0)->shape().element_type(); PrimitiveType rhs_input_type = op->operand(1)->shape().element_type(); PrimitiveType output_type = op->shape().element_type(); + HloOpcode opcode = op->opcode(); + + if (hlo_module_config_.debug_options().xla_gpu_enable_fast_min_max() && + (opcode == HloOpcode::kMaximum || opcode == HloOpcode::kMinimum)) { + return llvm_ir::EmitCallToIntrinsic( + opcode == HloOpcode::kMaximum ? llvm::Intrinsic::maxnum + : llvm::Intrinsic::minnum, + {lhs_value, rhs_value}, {lhs_value->getType()}, b_); + } + switch (op->opcode()) { case HloOpcode::kRemainder: { return EmitLibdeviceMathCall("__nv_fmod", {lhs_value, rhs_value}, diff --git a/tensorflow/compiler/xla/service/gpu/fusion_merger.cc b/tensorflow/compiler/xla/service/gpu/fusion_merger.cc index 30c1f908896..470457935ac 100644 --- a/tensorflow/compiler/xla/service/gpu/fusion_merger.cc +++ b/tensorflow/compiler/xla/service/gpu/fusion_merger.cc @@ -229,7 +229,7 @@ Status FusionInstructionMerger::HandleFusion(HloInstruction* fusion) { if (!absl::c_all_of(fusion->users(), [&](const HloInstruction* user) { return user->opcode() == HloOpcode::kFusion && (user->fusion_kind() == HloInstruction::FusionKind::kLoop || - (user->fusion_kind() == HloInstruction::FusionKind::kInput && + (IsReduceInputFusion(*user) && LayoutsAreReduceInputFusionFriendly(*fusion, *user))); })) { VLOG(3) << "Not merging " << fusion->name() diff --git a/tensorflow/compiler/xla/service/gpu/gpu_executable.cc b/tensorflow/compiler/xla/service/gpu/gpu_executable.cc index 57426327822..ae2e718db29 100644 --- a/tensorflow/compiler/xla/service/gpu/gpu_executable.cc +++ b/tensorflow/compiler/xla/service/gpu/gpu_executable.cc @@ -51,7 +51,7 @@ GpuExecutable::GpuExecutable( const string& ptx, const std::vector& cubin, std::pair compute_capability, std::unique_ptr thunk_schedule, - std::unique_ptr hlo_module, + std::unique_ptr hlo_module, std::unique_ptr assignment, std::unique_ptr hlo_profile_printer_data, std::unique_ptr hlo_profile_index_map) diff --git a/tensorflow/compiler/xla/service/gpu/gpu_executable.h b/tensorflow/compiler/xla/service/gpu/gpu_executable.h index 0e276282e40..2b3c77f5b82 100644 --- a/tensorflow/compiler/xla/service/gpu/gpu_executable.h +++ b/tensorflow/compiler/xla/service/gpu/gpu_executable.h @@ -54,7 +54,7 @@ class GpuExecutable : public Executable { GpuExecutable(const string& ptx, const std::vector& cubin, std::pair compute_capability, std::unique_ptr thunk_schedule, - std::unique_ptr hlo_module, + std::unique_ptr hlo_module, std::unique_ptr assignment, std::unique_ptr hlo_profile_printer_data, std::unique_ptr hlo_profile_index_map); diff --git a/tensorflow/compiler/xla/service/gpu/gpu_fusible.cc b/tensorflow/compiler/xla/service/gpu/gpu_fusible.cc index 2d31fd5570c..452e763a8ea 100644 --- a/tensorflow/compiler/xla/service/gpu/gpu_fusible.cc +++ b/tensorflow/compiler/xla/service/gpu/gpu_fusible.cc @@ -55,7 +55,7 @@ bool LayoutsAreReduceInputFusionFriendly(const HloInstruction& producer, }); } -bool IsInputFusibleReduction(const HloInstruction& instr) { +bool IsReduceInputFusion(const HloInstruction& instr) { if (instr.IsMultiOutputFusion()) { for (const HloInstruction* operand : instr.fused_expression_root()->operands()) { @@ -67,17 +67,70 @@ bool IsInputFusibleReduction(const HloInstruction& instr) { return true; } } - return false; - } else if (instr.opcode() == HloOpcode::kFusion) { - if (IsReductionToVector(*instr.fused_expression_root())) { - CHECK(instr.fusion_kind() == HloInstruction::FusionKind::kInput) - << " Fusion rooted at reduction-to-vector op must be of kind kInput: " - << instr.ToString(); - return true; + } else if (instr.opcode() == HloOpcode::kFusion && + IsReductionToVector(*instr.fused_expression_root())) { + CHECK(instr.fusion_kind() == HloInstruction::FusionKind::kInput) + << " Fusion rooted at reduction-to-vector op must be of kind kInput: " + << instr.ToString(); + return true; + } + return false; +} + +bool IsInputFusibleReduction(const HloInstruction& instr) { + return IsReduceInputFusion(instr) || IsReductionToVector(instr); +} + +bool ShapesCompatibleForMultiOutputFusion(const HloInstruction& instr1, + const HloInstruction& instr2) { + // Returns the instructions that determines the emitter used for lowering, + // sometimes referred to as "the real hero". + auto get_real_hero = + [&](const HloInstruction* instr) -> const HloInstruction* { + if (instr->opcode() == HloOpcode::kFusion) { + auto fused_expression_root = instr->fused_expression_root(); + if (instr->IsMultiOutputFusion()) { + // If possible, we want to pick a reduction-to-vector operand of the + // fusion root, because it has the most constraints. + for (const auto* inst : fused_expression_root->operands()) { + if (IsReductionToVector(*inst)) { + return inst; + } + } + return fused_expression_root->operands()[0]; + } + return fused_expression_root; } + return instr; + }; + + // Multi-output fusion kernels share a common parallel loop. The loop + // dimenstions are determined by instruction shapes. + auto get_loop_shape = [&](const HloInstruction* element_instr) { + // Special-case reduction-to-vector ops: The loop dimensions are determined + // by the shape of the first operand. + if (IsReductionToVector(*element_instr)) { + return element_instr->operand(0)->shape(); + } + return element_instr->shape(); + }; + + // All shapes of the root tuple of multi-output fusions should agree, i.e. all + // root ops should have equal output shapes. An exception are + // reduction-to-vector ops. Here the input shapes of the reduction (first + // operand shape) and the reduction dimensions need to match. + auto* instr_1 = get_real_hero(&instr1); + auto* instr_2 = get_real_hero(&instr2); + // TODO(tjoerg): Relax the shape constraint. The datatype does not matter. + if (IsReductionToVector(*instr_1) && IsReductionToVector(*instr_2) && + (!ShapeUtil::Equal(instr_1->shape(), instr_2->shape()) || + instr_1->dimensions() != instr_2->dimensions())) { return false; } - return IsReductionToVector(instr); + // The elementwise output shapes must be the same (including layout). + // TODO(tjoerg): Further relax the constraint. The datatype does not matter. + return ShapeUtil::EqualIgnoringFpPrecision(get_loop_shape(instr_1), + get_loop_shape(instr_2)); } } // namespace gpu diff --git a/tensorflow/compiler/xla/service/gpu/gpu_fusible.h b/tensorflow/compiler/xla/service/gpu/gpu_fusible.h index f7c24a0d5bb..e9d7ba1c4cf 100644 --- a/tensorflow/compiler/xla/service/gpu/gpu_fusible.h +++ b/tensorflow/compiler/xla/service/gpu/gpu_fusible.h @@ -33,16 +33,29 @@ namespace gpu { bool LayoutsAreReduceInputFusionFriendly(const HloInstruction& producer, const HloInstruction& reduce); -// Whether `instr` is fusible as root of a reduce input fusions, i.e. `instr` -// is either an unfused reduction-to-vector op, an input fusion rooted at a -// reduction-to-vector op, or a multi-output input fusion with at least one -// reduction-to-vector op root. // Note that reduction ops are lowered in different ways. Reduce input fusions // are lowered by IrEmitterUnnested::EmitReductionToVector and must be rooted at // reduction-to-vector ops. Other reduction ops are lowered by // GpuElementalIrEmitter and fused like elementwise ops. + +// Whether `instr` is an input fusion rooted at a reduction-to-vector op or a +// multi-output input fusion with at least one reduction-to-vector op root. +bool IsReduceInputFusion(const HloInstruction& instr); + +// Whether `instr` is fusible as root of a reduce input fusions, i.e. `instr` +// is either an unfused reduction-to-vector op or a reduce input fusion. bool IsInputFusibleReduction(const HloInstruction& instr); +// Whether instruction shapes are compatible for multi-output fusion, i.e. +// whether the emitters support lowering the resulting fusion. +// This function works for both, sibling and producer-conumser multi-output +// fusion. +// So far, multi-output fusion is supported for loop fusions and reduce +// input fusions only. It is up to the caller to ensure the instructions +// themselves are fusible! +bool ShapesCompatibleForMultiOutputFusion(const HloInstruction& instr1, + const HloInstruction& instr2); + } // namespace gpu } // namespace xla diff --git a/tensorflow/compiler/xla/service/gpu/gpu_fusible_test.cc b/tensorflow/compiler/xla/service/gpu/gpu_fusible_test.cc index d91b7bc61fd..15d4ee206ce 100644 --- a/tensorflow/compiler/xla/service/gpu/gpu_fusible_test.cc +++ b/tensorflow/compiler/xla/service/gpu/gpu_fusible_test.cc @@ -178,7 +178,7 @@ TEST_F(GpuFusibleTest, EXPECT_TRUE(LayoutsAreReduceInputFusionFriendly(*loop_fusion, *reduce)); } -TEST_F(GpuFusibleTest, IsInputFusibleReduction_ReductionToVector) { +TEST_F(GpuFusibleTest, IsReduceInputFusion_ReductionToVector) { auto module = ParseHloString(absl::StrCat(kModulePrefix, R"( ENTRY entry { c0 = f32[] parameter(0) @@ -191,10 +191,11 @@ TEST_F(GpuFusibleTest, IsInputFusibleReduction_ReductionToVector) { const HloInstruction* reduce = module->entry_computation()->root_instruction(); ASSERT_EQ(reduce->opcode(), HloOpcode::kReduce); + EXPECT_FALSE(IsReduceInputFusion(*reduce)); EXPECT_TRUE(IsInputFusibleReduction(*reduce)); } -TEST_F(GpuFusibleTest, IsInputFusibleReduction_ElementalReduction) { +TEST_F(GpuFusibleTest, IsReduceInputFusion_ElementalReduction) { auto module = ParseHloString(absl::StrCat(kModulePrefix, R"( ENTRY entry { c0 = f32[] parameter(0) @@ -207,10 +208,11 @@ TEST_F(GpuFusibleTest, IsInputFusibleReduction_ElementalReduction) { const HloInstruction* reduce = module->entry_computation()->root_instruction(); ASSERT_EQ(reduce->opcode(), HloOpcode::kReduce); + EXPECT_FALSE(IsReduceInputFusion(*reduce)); EXPECT_FALSE(IsInputFusibleReduction(*reduce)); } -TEST_F(GpuFusibleTest, IsInputFusibleReduction_SingleOutputInputReduceFusion) { +TEST_F(GpuFusibleTest, IsReduceInputFusion_SingleOutputInputReduceFusion) { auto module = ParseHloString(absl::StrCat(kModulePrefix, R"( fused_reduction { c0 = f32[] parameter(0) @@ -225,10 +227,11 @@ TEST_F(GpuFusibleTest, IsInputFusibleReduction_SingleOutputInputReduceFusion) { const HloInstruction* reduce = module->entry_computation()->root_instruction(); ASSERT_EQ(reduce->opcode(), HloOpcode::kFusion); + EXPECT_TRUE(IsReduceInputFusion(*reduce)); EXPECT_TRUE(IsInputFusibleReduction(*reduce)); } -TEST_F(GpuFusibleTest, IsInputFusibleReduction_SingleOutputLoopReduceFusion) { +TEST_F(GpuFusibleTest, IsReduceInputFusion_SingleOutputLoopReduceFusion) { auto module = ParseHloString(absl::StrCat(kModulePrefix, R"( fused_reduction { c0 = f32[] parameter(0) @@ -243,10 +246,11 @@ TEST_F(GpuFusibleTest, IsInputFusibleReduction_SingleOutputLoopReduceFusion) { const HloInstruction* reduce = module->entry_computation()->root_instruction(); ASSERT_EQ(reduce->opcode(), HloOpcode::kFusion); + EXPECT_FALSE(IsReduceInputFusion(*reduce)); EXPECT_FALSE(IsInputFusibleReduction(*reduce)); } -TEST_F(GpuFusibleTest, IsInputFusibleReduction_MultiOutputInputReduceFusion) { +TEST_F(GpuFusibleTest, IsReduceInputFusion_MultiOutputInputReduceFusion) { auto module = ParseHloString(absl::StrCat(kModulePrefix, R"( fused_reduction { c0 = f32[] parameter(0) @@ -263,11 +267,12 @@ TEST_F(GpuFusibleTest, IsInputFusibleReduction_MultiOutputInputReduceFusion) { const HloInstruction* reduce = module->entry_computation()->root_instruction(); ASSERT_EQ(reduce->opcode(), HloOpcode::kFusion); + EXPECT_TRUE(IsReduceInputFusion(*reduce)); EXPECT_TRUE(IsInputFusibleReduction(*reduce)); } TEST_F(GpuFusibleTest, - IsInputFusibleReduction_MultiOutputInputReduceFusionWithExtraOutputs) { + IsReduceInputFusion_MultiOutputInputReduceFusionWithExtraOutputs) { auto module = ParseHloString(absl::StrCat(kModulePrefix, R"( fused_reduction { c0 = f32[] parameter(0) @@ -284,10 +289,11 @@ TEST_F(GpuFusibleTest, const HloInstruction* reduce = module->entry_computation()->root_instruction(); ASSERT_EQ(reduce->opcode(), HloOpcode::kFusion); + EXPECT_TRUE(IsReduceInputFusion(*reduce)); EXPECT_TRUE(IsInputFusibleReduction(*reduce)); } -TEST_F(GpuFusibleTest, IsInputFusibleReduction_MultiOutputLoopReduceFusion) { +TEST_F(GpuFusibleTest, IsReduceInputFusion_MultiOutputLoopReduceFusion) { auto module = ParseHloString(absl::StrCat(kModulePrefix, R"( fused_reduction { c0 = f32[] parameter(0) @@ -304,11 +310,12 @@ TEST_F(GpuFusibleTest, IsInputFusibleReduction_MultiOutputLoopReduceFusion) { const HloInstruction* reduce = module->entry_computation()->root_instruction(); ASSERT_EQ(reduce->opcode(), HloOpcode::kFusion); + EXPECT_FALSE(IsReduceInputFusion(*reduce)); EXPECT_FALSE(IsInputFusibleReduction(*reduce)); } TEST_F(GpuFusibleTest, - IsInputFusibleReduction_MultiOutputLoopFusionReduceAndElementwiseOp) { + IsReduceInputFusion_MultiOutputLoopFusionReduceAndElementwiseOp) { auto module = ParseHloString(absl::StrCat(kModulePrefix, R"( fused_reduction { c0 = f32[] parameter(0) @@ -325,8 +332,304 @@ TEST_F(GpuFusibleTest, const HloInstruction* reduce = module->entry_computation()->root_instruction(); ASSERT_EQ(reduce->opcode(), HloOpcode::kFusion); + EXPECT_FALSE(IsReduceInputFusion(*reduce)); EXPECT_FALSE(IsInputFusibleReduction(*reduce)); } +TEST_F(GpuFusibleTest, ShapesCompatibleForMultiOutputFusion_LoopFusions) { + auto module = ParseHloString(absl::StrCat(kModulePrefix, R"( + fused_computation_1 { + p0.1 = f32[6400]{0} parameter(0) + ROOT mul = f32[6400]{0} multiply(p0.1, p0.1) + } + + fused_computation_2 { + p0.2 = f32[6400]{0} parameter(0) + const.2 = f32[] constant(1) + ROOT div = f32[6400]{0} divide(p0.2, const.2) + } + + ENTRY entry { + p0 = f32[6400]{0} parameter(0) + fusion.1 = f32[6400]{0} fusion(p0), kind=kLoop, calls=fused_computation_1 + fusion.2 = f32[6400]{0} fusion(p0), kind=kLoop, calls=fused_computation_2 + ROOT root = (f32[6400]{0}, f32[6400]{0}) tuple(fusion.1, fusion.2) + })")) + .ValueOrDie(); + const HloInstruction* fusion_1 = + module->entry_computation()->root_instruction()->operand(0); + const HloInstruction* fusion_2 = + module->entry_computation()->root_instruction()->operand(1); + EXPECT_TRUE(ShapesCompatibleForMultiOutputFusion(*fusion_1, *fusion_2)); +} + +TEST_F(GpuFusibleTest, ShapesCompatibleForMultiOutputFusion_IgnoreFpPrecision) { + auto module = ParseHloString(absl::StrCat(kModulePrefix, R"( + fused_computation_1 { + p0.1 = f32[6400]{0} parameter(0) + ROOT mul = f32[6400]{0} multiply(p0.1, p0.1) + } + + fused_computation_2 { + p0.2 = f32[6400]{0} parameter(0) + ROOT convert = f16[6400]{0} convert(p0.2) + } + + ENTRY entry { + p0 = f32[6400]{0} parameter(0) + fusion.1 = f32[6400]{0} fusion(p0), kind=kLoop, calls=fused_computation_1 + fusion.2 = f32[6400]{0} fusion(p0), kind=kLoop, calls=fused_computation_2 + ROOT root = (f32[6400]{0}, f32[6400]{0}) tuple(fusion.1, fusion.2) + })")) + .ValueOrDie(); + const HloInstruction* fusion_1 = + module->entry_computation()->root_instruction()->operand(0); + const HloInstruction* fusion_2 = + module->entry_computation()->root_instruction()->operand(1); + EXPECT_TRUE(ShapesCompatibleForMultiOutputFusion(*fusion_1, *fusion_2)); +} + +TEST_F(GpuFusibleTest, ShapesCompatibleForMultiOutputFusion_Reduce) { + auto module = ParseHloString(absl::StrCat(kModulePrefix, R"( + fused_computation_1 { + p0.1 = f32[6400]{0} parameter(0) + ROOT mul = f32[6400]{0} multiply(p0.1, p0.1) + } + + ENTRY entry { + p0 = f32[6400]{0} parameter(0) + fusion.1 = f32[6400]{0} fusion(p0), kind=kLoop, calls=fused_computation_1 + const.2 = f32[] constant(0) + reduce = f32[] reduce(p0, const.2), dimensions={0}, to_apply=scalar_add + ROOT root = (f32[6400]{0}, f32[]) tuple(fusion.1, reduce) + })")) + .ValueOrDie(); + const HloInstruction* fusion = + module->entry_computation()->root_instruction()->operand(0); + const HloInstruction* reduce = + module->entry_computation()->root_instruction()->operand(1); + EXPECT_TRUE(ShapesCompatibleForMultiOutputFusion(*fusion, *reduce)); +} + +TEST_F(GpuFusibleTest, ShapesCompatibleForMultiOutputFusion_Elementwise) { + auto module = ParseHloString(absl::StrCat(kModulePrefix, R"( + fused_computation_1 { + p0.1 = f32[6400]{0} parameter(0) + ROOT mul = f32[6400]{0} multiply(p0.1, p0.1) + } + + ENTRY entry { + p0 = f32[6400]{0} parameter(0) + fusion.1 = f32[6400]{0} fusion(p0), kind=kLoop, calls=fused_computation_1 + const.2 = f32[] constant(1) + div = f32[6400]{0} divide(p0, const.2) + ROOT root = (f32[6400]{0}, f32[6400]{0}) tuple(fusion.1, div) + })")) + .ValueOrDie(); + const HloInstruction* fusion = + module->entry_computation()->root_instruction()->operand(0); + const HloInstruction* div = + module->entry_computation()->root_instruction()->operand(1); + EXPECT_TRUE(ShapesCompatibleForMultiOutputFusion(*fusion, *div)); +} + +TEST_F(GpuFusibleTest, + ShapesCompatibleForMultiOutputFusion_MultiOutputLoopFusion) { + auto module = ParseHloString(absl::StrCat(kModulePrefix, R"( + fused_computation_1 { + p0.1 = f32[8,1,5,16,1,1]{5,4,3,2,1,0} parameter(0) + mul = f32[8,1,5,16,1,1]{5,4,3,2,1,0} multiply(p0.1, p0.1) + exp = f32[8,1,5,16,1,1]{5,4,3,2,1,0} exponential(p0.1) + ROOT tuple = (f32[8,1,5,16,1,1]{5,4,3,2,1,0}, f32[8,1,5,16,1,1]{5,4,3,2,1,0}) tuple(mul, exp) + } + + fused_computation_2 { + p0.2 = f32[8,1,5,16,1,1]{5,4,3,2,1,0} parameter(0) + const.2 = f32[] constant(0) + ROOT add = f32[8,1,5,16,1,1]{5,4,3,2,1,0} add(p0.2, const.2) + } + + ENTRY entry { + p0 = f32[8,1,5,16,1,1]{5,4,3,2,1,0} parameter(0) + fusion.1 = (f32[8,1,5,16,1,1]{5,4,3,2,1,0}, f32[8,1,5,16,1,1]{5,4,3,2,1,0}) fusion(p0), kind=kLoop, calls=fused_computation_1 + fusion.2 = f32[8,1,5,16,1,1]{5,4,3,2,1,0} fusion(p0), kind=kLoop, calls=fused_computation_2 + gte0 = f32[8,1,5,16,1,1]{5,4,3,2,1,0} get-tuple-element(fusion.1), index=0 + gte1 = f32[8,1,5,16,1,1]{5,4,3,2,1,0} get-tuple-element(fusion.1), index=1 + ROOT root = (f32[8,1,5,16,1,1]{5,4,3,2,1,0}, f32[8,1,5,16,1,1]{5,4,3,2,1,0}, f32[8,1,5,16,1,1]{5,4,3,2,1,0}) tuple(gte0, gte1, fusion.2) + })")) + .ValueOrDie(); + const HloInstruction* fusion_1 = + module->entry_computation()->root_instruction()->operand(0)->operand(0); + const HloInstruction* fusion_2 = + module->entry_computation()->root_instruction()->operand(1)->operand(0); + EXPECT_TRUE(ShapesCompatibleForMultiOutputFusion(*fusion_1, *fusion_2)); +} + +TEST_F(GpuFusibleTest, ShapesCompatibleForMultiOutputFusion_UnfusedOps) { + auto module = ParseHloString(absl::StrCat(kModulePrefix, R"( + ENTRY reduce { + p0 = f32[2,2,2]{2,1,0} parameter(0) + c0 = f32[] constant(0) + exp = f32[2,2,2]{2,1,0} exponential(p0) + reduce = f32[2,2]{1,0} reduce(exp, c0), dimensions={2}, to_apply=scalar_add + ROOT root = (f32[2,2]{1,0}, f32[2,2,2]{2,1,0}) tuple(reduce, exp) + })")) + .ValueOrDie(); + const HloInstruction* reduce = + module->entry_computation()->root_instruction()->operand(0); + const HloInstruction* exp = + module->entry_computation()->root_instruction()->operand(1); + EXPECT_TRUE(ShapesCompatibleForMultiOutputFusion(*reduce, *exp)); +} + +TEST_F(GpuFusibleTest, ShapesCompatibleForMultiOutputFusion_DifferentLayouts) { + auto module = ParseHloString(absl::StrCat(kModulePrefix, R"( + ENTRY reduce { + p0 = f32[2,2,2]{2,1,0} parameter(0) + p1 = f32[2,2,2]{0,1,2} parameter(1) + c0 = f32[] constant(0) + exp = f32[2,2,2]{2,1,0} exponential(p0) + reduce = f32[2,2]{0,1} reduce(p1, c0), dimensions={2}, to_apply=scalar_add + ROOT root = (f32[2,2]{0,1}, f32[2,2,2]{2,1,0}) tuple(reduce, exp) + })")) + .ValueOrDie(); + const HloInstruction* reduce = + module->entry_computation()->root_instruction()->operand(0); + const HloInstruction* exp = + module->entry_computation()->root_instruction()->operand(1); + EXPECT_FALSE(ShapesCompatibleForMultiOutputFusion(*reduce, *exp)); +} + +TEST_F(GpuFusibleTest, + ShapesCompatibleForMultiOutputFusion_MultiOutputReduceFusion) { + auto module = ParseHloString(absl::StrCat(kModulePrefix, R"( + fused_select { + p1.1 = f32[2,2,2]{2,1,0} parameter(1) + c0 = f32[] constant(0) + broadcast = f32[2,2,2]{2,1,0} broadcast(f32[] c0), dimensions={} + greater-than = pred[2,2,2]{2,1,0} greater-than(f32[2,2,2]{2,1,0} p1.1, f32[2,2,2]{2,1,0} broadcast) + p0.1 = f32[2,2,2]{2,1,0} parameter(0) + ROOT select = f32[2,2,2]{2,1,0} select(pred[2,2,2]{2,1,0} greater-than, f32[2,2,2]{2,1,0} p0.1, f32[2,2,2]{2,1,0} broadcast) + } + + fused_reduce { + p0.2 = f32[2,2,2]{2,1,0} parameter(0) + c1 = f32[] constant(0) + r1 = f32[2,2]{1,0} reduce(p0.2, c1), dimensions={2}, to_apply=scalar_add + mul = f32[2,2,2]{2,1,0} multiply(p0.2, p0.2) + r2 = f32[2,2]{1,0} reduce(mul, c1), dimensions={2}, to_apply=scalar_add + ROOT tuple = (f32[2,2]{1,0}, f32[2,2]{1,0}) tuple(r1, r2) + } + + ENTRY reduce { + p0 = f32[2,2,2]{2,1,0} parameter(0) + p1 = f32[2,2,2]{2,1,0} parameter(1) + select = f32[2,2,2]{2,1,0} fusion(p0, p1), kind=kLoop, calls=fused_select + fusion = (f32[2,2]{1,0}, f32[2,2]{1,0}) fusion(select), kind=kInput, calls=fused_reduce + gte0 = f32[2,2]{1,0} get-tuple-element(fusion), index=0 + gte1 = f32[2,2]{1,0} get-tuple-element(fusion), index=1 + ROOT root = (f32[2,2]{1,0}, f32[2,2]{1,0}, f32[2,2,2]{2,1,0}) tuple(gte1, gte1, select) + })")) + .ValueOrDie(); + const HloInstruction* fusion_1 = + module->entry_computation()->root_instruction()->operand(0)->operand(0); + const HloInstruction* fusion_2 = + module->entry_computation()->root_instruction()->operand(1)->operand(0); + EXPECT_TRUE(ShapesCompatibleForMultiOutputFusion(*fusion_1, *fusion_2)); +} + +TEST_F(GpuFusibleTest, ShapesCompatibleForMultiOutputFusion_ReduceFusions) { + auto module = ParseHloString(absl::StrCat(kModulePrefix, R"( + fused_reduce_1 { + p0.1 = f32[2,2,2]{2,1,0} parameter(0) + c0 = f32[] constant(0) + ROOT reduce = f32[2,2]{1,0} reduce(f32[2,2,2]{2,1,0} p0.1, f32[] c0), dimensions={0}, to_apply=scalar_add + } + + fused_reduce_2 { + p0.2 = f32[2,2,2]{2,1,0} parameter(0) + mul = f32[2,2,2]{2,1,0} multiply(f32[2,2,2]{2,1,0} p0.2, f32[2,2,2]{2,1,0} p0.2) + c1 = f32[] constant(0) + ROOT reduce = f32[2,2]{1,0} reduce(f32[2,2,2]{2,1,0} mul, f32[] c1), dimensions={0}, to_apply=scalar_add + } + + ENTRY reduce { + p0 = f32[2,2,2]{2,1,0} parameter(0) + p1 = f32[2,2,2]{2,1,0} parameter(1) + reduce_1 = f32[2,2]{1,0} fusion(p0), kind=kLoop, calls=fused_reduce_1 + reduce_2 = f32[2,2]{1,0} fusion(p1), kind=kLoop, calls=fused_reduce_2 + ROOT root = (f32[2,2]{1,0}, f32[2,2,2]{2,1,0}) tuple(reduce_1, reduce_2) + })")) + .ValueOrDie(); + const HloInstruction* fusion_1 = + module->entry_computation()->root_instruction()->operand(0); + const HloInstruction* fusion_2 = + module->entry_computation()->root_instruction()->operand(1); + EXPECT_TRUE(ShapesCompatibleForMultiOutputFusion(*fusion_1, *fusion_2)); +} + +TEST_F(GpuFusibleTest, + ShapesCompatibleForMultiOutputFusion_DifferentReduceDimensions) { + auto module = ParseHloString(absl::StrCat(kModulePrefix, R"( + fused_reduce_1 { + p0.1 = f32[2,2,2]{2,1,0} parameter(0) + c0 = f32[] constant(0) + ROOT reduce = f32[2,2]{1,0} reduce(f32[2,2,2]{2,1,0} p0.1, f32[] c0), dimensions={0}, to_apply=scalar_add + } + + fused_reduce_2 { + p0.2 = f32[2,2,2]{2,1,0} parameter(0) + mul = f32[2,2,2]{2,1,0} multiply(f32[2,2,2]{2,1,0} p0.2, f32[2,2,2]{2,1,0} p0.2) + c1 = f32[] constant(0) + ROOT reduce = f32[2,2]{1,0} reduce(f32[2,2,2]{2,1,0} mul, f32[] c1), dimensions={2}, to_apply=scalar_add + } + + ENTRY reduce { + p0 = f32[2,2,2]{2,1,0} parameter(0) + p1 = f32[2,2,2]{2,1,0} parameter(1) + reduce_1 = f32[2,2]{1,0} fusion(p0), kind=kLoop, calls=fused_reduce_1 + reduce_2 = f32[2,2]{1,0} fusion(p1), kind=kLoop, calls=fused_reduce_2 + ROOT root = (f32[2,2]{1,0}, f32[2,2,2]{2,1,0}) tuple(reduce_1, reduce_2) + })")) + .ValueOrDie(); + const HloInstruction* fusion_1 = + module->entry_computation()->root_instruction()->operand(0); + const HloInstruction* fusion_2 = + module->entry_computation()->root_instruction()->operand(1); + EXPECT_FALSE(ShapesCompatibleForMultiOutputFusion(*fusion_1, *fusion_2)); +} + +TEST_F(GpuFusibleTest, + ShapesCompatibleForMultiOutputFusion_NoReductionToVector) { + auto module = ParseHloString(absl::StrCat(kModulePrefix, R"( + fused_element_wise { + p0.1 = f32[2,2,2]{2,1,0} parameter(0) + p1.1 = f32[2,2,2]{2,1,0} parameter(1) + ROOT add = f32[2,2,2]{2,1,0} add(p0.1, p1.1) + } + + fused_reduce { + p0.2 = f32[2,2,2]{2,1,0} parameter(0) + mul = f32[2,2,2]{2,1,0} multiply(f32[2,2,2]{2,1,0} p0.2, f32[2,2,2]{2,1,0} p0.2) + c1 = f32[] constant(0) + // Note that reduce is not a reduction-to-vector. + ROOT reduce = f32[2,2]{1,0} reduce(f32[2,2,2]{2,1,0} mul, f32[] c1), dimensions={1}, to_apply=scalar_add + } + + ENTRY reduce { + p0 = f32[2,2,2]{2,1,0} parameter(0) + p1 = f32[2,2,2]{2,1,0} parameter(1) + element_wise = f32[2,2,2]{2,1,0} fusion(p0, p1), kind=kLoop, calls=fused_element_wise + fusion = (f32[2,2]{1,0}, f32[2,2]{1,0}) fusion(element_wise), kind=kLoop, calls=fused_reduce + ROOT root = (f32[2,2]{1,0}, f32[2,2,2]{2,1,0}) tuple(fusion, element_wise) + })")) + .ValueOrDie(); + const HloInstruction* fusion_1 = + module->entry_computation()->root_instruction()->operand(0); + const HloInstruction* fusion_2 = + module->entry_computation()->root_instruction()->operand(1); + EXPECT_FALSE(ShapesCompatibleForMultiOutputFusion(*fusion_1, *fusion_2)); +} + } // namespace gpu } // namespace xla diff --git a/tensorflow/compiler/xla/service/gpu/gpu_hlo_schedule.cc b/tensorflow/compiler/xla/service/gpu/gpu_hlo_schedule.cc index 91609c730b6..1126943624a 100644 --- a/tensorflow/compiler/xla/service/gpu/gpu_hlo_schedule.cc +++ b/tensorflow/compiler/xla/service/gpu/gpu_hlo_schedule.cc @@ -37,7 +37,7 @@ class GpuHloOrdering : public PredecessorHloOrdering { public: GpuHloOrdering(const HloModule* module, const StreamAssignment& stream_assignment, - const std::vector& thunk_launch_order); + const std::vector& thunk_launch_order); ~GpuHloOrdering() override = default; // Only the entry computation can possibly be sequentially ordered, and only @@ -56,7 +56,7 @@ class GpuHloOrdering : public PredecessorHloOrdering { GpuHloOrdering::GpuHloOrdering( const HloModule* module, const StreamAssignment& stream_assignment, - const std::vector& thunk_launch_order) + const std::vector& thunk_launch_order) : PredecessorHloOrdering(module) { // The entry computation has a total order when there's only one stream. if (stream_assignment.StreamCount() == 1) { @@ -150,7 +150,7 @@ GpuHloOrdering::GpuHloOrdering( // However, if the total order is A,B,D,C,E, then C and E can run // concurrently. void BFSLaunchOrder(const HloComputation* computation, - std::vector* launch_order) { + std::vector* launch_order) { // This topological sort uses two data structures: // 1. `incoming_edge_count` which keeps track of the number of incoming // edges to each HLO; @@ -158,9 +158,9 @@ void BFSLaunchOrder(const HloComputation* computation, // // The sorting algorithm repeatedly pops the top from the queue and deletes // that HLO from the graph, making more HLOs incoming-edge free. - std::deque queue; + std::deque queue; std::unordered_map incoming_edge_count; - for (const auto& hlo : computation->instructions()) { + for (auto* hlo : computation->instructions()) { if (hlo->operand_count() == 0) { queue.push_back(hlo); } else { @@ -172,10 +172,10 @@ void BFSLaunchOrder(const HloComputation* computation, } while (!queue.empty()) { - const HloInstruction* x = queue.front(); + HloInstruction* x = queue.front(); queue.pop_front(); launch_order->push_back(x); - for (const HloInstruction* y : x->users()) { + for (HloInstruction* y : x->users()) { --incoming_edge_count[y]; if (incoming_edge_count[y] == 0) { queue.push_back(y); @@ -195,14 +195,14 @@ StatusOr> GpuHloSchedule::Build( std::unique_ptr schedule(new GpuHloSchedule); // Initialize thunk_launch_order_, the total order of thunk launches. - const HloComputation* entry_computation = module.entry_computation(); + HloComputation* entry_computation = module.entry_computation(); if (stream_assignment.StreamCount() == 1) { // All kernels are launched on a single stream, so there's no loss of // concurrency by optimizing for minimal memory usage. TF_ASSIGN_OR_RETURN( HloInstructionSequence sequence, ScheduleComputation( - *entry_computation, [pointer_size](const BufferValue& buffer) { + entry_computation, [pointer_size](const BufferValue& buffer) { return ShapeUtil::ByteSizeOf(buffer.shape(), pointer_size); })); schedule->thunk_launch_order_ = sequence.instructions(); diff --git a/tensorflow/compiler/xla/service/gpu/gpu_hlo_schedule.h b/tensorflow/compiler/xla/service/gpu/gpu_hlo_schedule.h index 07a7fc67aa5..7f224ffe4f0 100644 --- a/tensorflow/compiler/xla/service/gpu/gpu_hlo_schedule.h +++ b/tensorflow/compiler/xla/service/gpu/gpu_hlo_schedule.h @@ -46,7 +46,7 @@ class GpuHloSchedule { // Returns the total order of thunk launches, represented in terms of HLO // instructions. - const std::vector& ThunkLaunchOrder() const { + const std::vector& ThunkLaunchOrder() const { return thunk_launch_order_; } @@ -60,7 +60,7 @@ class GpuHloSchedule { private: GpuHloSchedule(); - std::vector thunk_launch_order_; + std::vector thunk_launch_order_; std::unique_ptr hlo_ordering_; }; diff --git a/tensorflow/compiler/xla/service/gpu/gpu_hlo_schedule_test.cc b/tensorflow/compiler/xla/service/gpu/gpu_hlo_schedule_test.cc index 6d3aed15ebe..91db7151f22 100644 --- a/tensorflow/compiler/xla/service/gpu/gpu_hlo_schedule_test.cc +++ b/tensorflow/compiler/xla/service/gpu/gpu_hlo_schedule_test.cc @@ -33,7 +33,7 @@ namespace gpu { class GpuHloScheduleTest : public HloTestBase { protected: - using HloVec = std::vector; + using HloVec = std::vector; // Pre-canned shapes. Shape f32_2x2_ = ShapeUtil::MakeShape(F32, {2, 2}); @@ -44,7 +44,7 @@ class GpuHloScheduleTest : public HloTestBase { .ConsumeValueOrDie(); } - std::unique_ptr CreateNewUnverifiedModule() { + std::unique_ptr CreateNewVerifiedModule() { HloModuleConfig config; auto debug_options = GetDebugOptionsForTest(); debug_options.set_xla_gpu_disable_multi_streaming(false); @@ -79,7 +79,7 @@ TEST_F(GpuHloScheduleTest, SequentialMatMul) { HloInstruction* dot2 = builder.AddInstruction(CreateCanonicalDot(f32_2x2_, dot1, z)); - auto module = CreateNewUnverifiedModule(); + auto module = CreateNewVerifiedModule(); module->AddEntryComputation(builder.Build(dot2)); std::unique_ptr streams = AssignStreams(*module); @@ -139,7 +139,7 @@ TEST_F(GpuHloScheduleTest, SequentialAdd) { HloInstruction* add3 = builder.AddInstruction( HloInstruction::CreateBinary(f32_2x2_, HloOpcode::kAdd, add1, add2)); - auto module = CreateNewUnverifiedModule(); + auto module = CreateNewVerifiedModule(); module->AddEntryComputation(builder.Build(add3)); std::unique_ptr streams = AssignStreams(*module); @@ -209,7 +209,7 @@ TEST_F(GpuHloScheduleTest, ConcurrentMatMul) { HloInstruction* add = builder.AddInstruction(CreateCanonicalDot(f32_2x2_, dot1, dot2)); - auto module = CreateNewUnverifiedModule(); + auto module = CreateNewVerifiedModule(); module->AddEntryComputation(builder.Build(add)); std::unique_ptr streams = AssignStreams(*module); @@ -288,7 +288,7 @@ TEST_F(GpuHloScheduleTest, LatticeMatMul) { HloInstruction* d40 = builder.AddInstruction(CreateCanonicalDot(f32_2x2_, d30, d31)); - auto module = CreateNewUnverifiedModule(); + auto module = CreateNewVerifiedModule(); module->AddEntryComputation(builder.Build(d40)); std::unique_ptr streams = AssignStreams(*module); diff --git a/tensorflow/compiler/xla/service/gpu/gpu_layout_assignment.cc b/tensorflow/compiler/xla/service/gpu/gpu_layout_assignment.cc index 1c0a23fa3eb..f59da2caa18 100644 --- a/tensorflow/compiler/xla/service/gpu/gpu_layout_assignment.cc +++ b/tensorflow/compiler/xla/service/gpu/gpu_layout_assignment.cc @@ -65,8 +65,8 @@ HeuristicLayoutAssignment(const HloInstruction* instr, VLOG(2) << "Using heuristic to figure out layouts for " << instr->ToString(); - // Empirically we've found with Volta and cudnn 7 that backward-input convs - // with stride are significantly faster with NCHW layouts. + // Empirically we've found with Volta and cudnn <= 7.3 that backward-input + // convs with stride are significantly faster with NCHW layouts. // // We could have used a mixed layout combination, e.g. (NHWC, NCHW, NCHW), // which on paper gives good performance. However, there are two observations: @@ -75,11 +75,17 @@ HeuristicLayoutAssignment(const HloInstruction* instr, // * we've also observed that for mixed layouts, cuDNN transposes data back // and forth from a different layout combination. If we end up with // transposes anyway, we prefer to have them in XLA, as they can be fused. - // TODO(timshen): Figure out the exact condition. This may be achieved by - // auto-tuning layouts offline. - if (instr->custom_call_target() == kCudnnConvBackwardInputCallTarget && - window_util::HasStride(instr->window())) { - return kAllNCHW; + if (auto* dnn = stream_executor->AsDnn()) { + auto version_status = dnn->GetVersion(); + if (version_status.ok()) { + auto version = version_status.ConsumeValueOrDie(); + if (std::make_tuple(version.major_version(), version.minor_version()) <= + std::make_tuple(7, 3) && + instr->custom_call_target() == kCudnnConvBackwardInputCallTarget && + window_util::HasStride(instr->window())) { + return kAllNCHW; + } + } } // For other Volta f16 convolutions, use NHWC. diff --git a/tensorflow/compiler/xla/service/gpu/gpu_layout_assignment_test.cc b/tensorflow/compiler/xla/service/gpu/gpu_layout_assignment_test.cc index 8cc76c872c6..2ffc8bfb49b 100644 --- a/tensorflow/compiler/xla/service/gpu/gpu_layout_assignment_test.cc +++ b/tensorflow/compiler/xla/service/gpu/gpu_layout_assignment_test.cc @@ -61,7 +61,7 @@ TEST_F(LayoutAssignmentTest, Elementwise) { HloInstruction::CreateParameter(1, ashape, "y")); auto add = builder.AddInstruction( HloInstruction::CreateBinary(ashape, HloOpcode::kAdd, x, y)); - auto module = CreateNewUnverifiedModule(); + auto module = CreateNewVerifiedModule(); HloComputation* computation = module->AddEntryComputation(builder.Build(add)); @@ -148,7 +148,7 @@ TEST_F(LayoutAssignmentTest, BatchNormInference) { {operand, scale, offset, mean, variance, epsilon, feature_index}, kCudnnBatchNormForwardInferenceCallTarget)); - auto module = CreateNewUnverifiedModule(); + auto module = CreateNewVerifiedModule(); HloComputation* computation = module->AddEntryComputation(builder.Build(batchnorm)); @@ -217,7 +217,7 @@ TEST_F(LayoutAssignmentTest, BatchNormTraining) { batchnorm_shape, {operand, scale, offset, epsilon, feature_index}, kCudnnBatchNormForwardTrainingCallTarget)); - auto module = CreateNewUnverifiedModule(); + auto module = CreateNewVerifiedModule(); HloComputation* computation = module->AddEntryComputation(builder.Build(batchnorm)); @@ -298,7 +298,7 @@ TEST_F(LayoutAssignmentTest, BatchNormGrad) { feature_index}, kCudnnBatchNormBackwardCallTarget)); - auto module = CreateNewUnverifiedModule(); + auto module = CreateNewVerifiedModule(); HloComputation* computation = module->AddEntryComputation(builder.Build(batchnorm)); diff --git a/tensorflow/compiler/xla/service/gpu/instruction_fusion.cc b/tensorflow/compiler/xla/service/gpu/instruction_fusion.cc index 43f43b50e4a..6151dd8ff4c 100644 --- a/tensorflow/compiler/xla/service/gpu/instruction_fusion.cc +++ b/tensorflow/compiler/xla/service/gpu/instruction_fusion.cc @@ -80,7 +80,7 @@ bool IsIEEEFloatingPointScalarConstant(const HloInstruction* constant) { // This function limits the maximum number of operands to a fusion. // // There's a cap on how many parameters we can pass to a CUDA kernel, but -// exactly what that limit is is hazy, as it depends on (among other things) how +// exactly what that limit is hazy, as it depends on (among other things) how // much GPU constant memory is in use for other purposes. // // Moreover, we don't even know at the point that we're running fusion how many @@ -181,7 +181,8 @@ bool GpuInstructionFusion::ShouldFuse(HloInstruction* consumer, return true; } } else if (consumer->operand_count() == 2 && - consumer->opcode() == HloOpcode::kAdd) { + consumer->opcode() == HloOpcode::kAdd && + consumer->operand(other_operand_index) != producer) { // Fuse a bias add into the output of the dot. return true; } diff --git a/tensorflow/compiler/xla/service/gpu/instruction_fusion_test.cc b/tensorflow/compiler/xla/service/gpu/instruction_fusion_test.cc index fb77bc4b8eb..688604cd36e 100644 --- a/tensorflow/compiler/xla/service/gpu/instruction_fusion_test.cc +++ b/tensorflow/compiler/xla/service/gpu/instruction_fusion_test.cc @@ -117,7 +117,7 @@ TEST_F(InstructionFusionTest, PotentialBitcastReshapeOfDotUnfused) { auto reshape2 = builder.AddInstruction(HloInstruction::CreateReshape( ShapeUtil::MakeShape(S32, {1, 1, 1}), dot1)); - auto module = CreateNewUnverifiedModule(); + auto module = CreateNewVerifiedModule(); auto computation = module->AddEntryComputation(builder.Build()); EXPECT_EQ(reshape2, computation->root_instruction()); EXPECT_FALSE(GpuInstructionFusion(/*may_duplicate=*/true) @@ -134,7 +134,7 @@ TEST_F(InstructionFusionTest, PotentialBitcastTransposeOfDotUnfused) { auto transpose2 = builder.AddInstruction(HloInstruction::CreateTranspose( ShapeUtil::MakeShape(S32, {1, 1}), dot1, {0, 1})); - auto module = CreateNewUnverifiedModule(); + auto module = CreateNewVerifiedModule(); auto computation = module->AddEntryComputation(builder.Build()); EXPECT_EQ(transpose2, computation->root_instruction()); EXPECT_FALSE(GpuInstructionFusion(/*may_duplicate=*/true) @@ -358,6 +358,29 @@ TEST_F(InstructionFusionTest, DotOutputFusionBiasAdd) { op::Parameter())); } +TEST_F(InstructionFusionTest, + DotOperationFusion_DontOutputFuseDuplicateOperands) { + absl::string_view module_string = R"( +HloModule module + +ENTRY main { + a = f32[50,60]{1,0} parameter(0) + b = f32[60,1]{1,0} parameter(1) + c = f32[50,1]{1,0} dot(a, b), lhs_contracting_dims={1}, rhs_contracting_dims={0} + ROOT d = f32[50,1]{1,0} add(c, c) +} +)"; + + TF_ASSERT_OK_AND_ASSIGN(std::unique_ptr module, + ParseAndReturnVerifiedModule(module_string)); + TF_ASSERT_OK_AND_ASSIGN( + bool fused_something, + GpuInstructionFusion(/*may_duplicate=*/false).Run(module.get())); + EXPECT_FALSE(fused_something); + EXPECT_THAT(module->entry_computation()->root_instruction(), + Not(op::Fusion())); +} + // Compute sum(1/p0), where p0 has type f32, twice. Check that the division is // duplicated and fused into both reduces. TEST_F(InstructionFusionTest, FloatingPointDivIsCheap) { @@ -723,7 +746,7 @@ TEST_F(InstructionFusionTest, AvoidsLargeFusion) { sum = b.AddInstruction( HloInstruction::CreateBinary(shape, HloOpcode::kAdd, sum, param)); } - auto module = CreateNewUnverifiedModule(); + auto module = CreateNewVerifiedModule(); auto computation = module->AddEntryComputation(b.Build()); EXPECT_TRUE(GpuInstructionFusion(/*may_duplicate=*/true) .Run(module.get()) diff --git a/tensorflow/compiler/xla/service/gpu/ir_emitter.cc b/tensorflow/compiler/xla/service/gpu/ir_emitter.cc index 7fcdd805ed3..6693f66d62d 100644 --- a/tensorflow/compiler/xla/service/gpu/ir_emitter.cc +++ b/tensorflow/compiler/xla/service/gpu/ir_emitter.cc @@ -63,9 +63,6 @@ IrEmitter::IrEmitter(const HloModuleConfig& hlo_module_config, &ir_emitter_context->buffer_assignment(), &b_, module_, is_nested), hlo_module_config_(hlo_module_config) { - b_.setFastMathFlags(llvm_ir::GetFastMathFlags( - /*fast_math_enabled=*/hlo_module_config.debug_options() - .xla_gpu_enable_fast_math())); } Status IrEmitter::DefaultAction(HloInstruction* hlo) { @@ -97,6 +94,18 @@ Status IrEmitter::HandleBitcast(HloInstruction* bitcast) { return Status::OK(); } +Status IrEmitter::HandleAddDependency(HloInstruction* add_dependency) { + VLOG(2) << "HandleAddDependency: " << add_dependency->ToString(); + const HloInstruction* operand = add_dependency->operand(0); + // Add_Dependency is a no-op, but we still want to bind it to an llvm::Value + // sometimes, e.g., when it's operand is a constant or a bitcast of a + // constant. + if (bindings_.BoundToIrValue(*operand)) { + bindings_.BindHloToIrValue(*add_dependency, GetBasePointer(*operand)); + } + return Status::OK(); +} + Status IrEmitter::HandleGetTupleElement(HloInstruction* get_tuple_element) { auto operand = get_tuple_element->operand(0); CHECK(bindings_.BoundToIrValue(*operand)); diff --git a/tensorflow/compiler/xla/service/gpu/ir_emitter.h b/tensorflow/compiler/xla/service/gpu/ir_emitter.h index 56c3f452006..2da46c01693 100644 --- a/tensorflow/compiler/xla/service/gpu/ir_emitter.h +++ b/tensorflow/compiler/xla/service/gpu/ir_emitter.h @@ -100,6 +100,7 @@ class IrEmitter : public DfsHloVisitorWithDefault, Status HandleBatchNormInference(HloInstruction* batch_norm) override; Status HandleBatchNormTraining(HloInstruction* batch_norm) override; Status HandleBatchNormGrad(HloInstruction* batch_norm) override; + Status HandleAddDependency(HloInstruction* add_dependency) override; Status FinishVisit(HloInstruction* root) override { return Status::OK(); } diff --git a/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.cc b/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.cc index 87b6cd640ac..bbe1583c011 100644 --- a/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.cc +++ b/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.cc @@ -22,7 +22,6 @@ limitations under the License. #include "tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.h" #include "absl/algorithm/container.h" -#include "absl/container/inlined_vector.h" #include "absl/memory/memory.h" #include "absl/strings/str_cat.h" #include "absl/types/optional.h" @@ -65,11 +64,11 @@ limitations under the License. #include "tensorflow/compiler/xla/service/hlo_casting_utils.h" #include "tensorflow/compiler/xla/service/hlo_computation.h" #include "tensorflow/compiler/xla/service/hlo_instruction.h" +#include "tensorflow/compiler/xla/service/hlo_instructions.h" #include "tensorflow/compiler/xla/service/hlo_opcode.h" #include "tensorflow/compiler/xla/service/llvm_ir/buffer_assignment_util.h" #include "tensorflow/compiler/xla/service/llvm_ir/dynamic_update_slice_util.h" #include "tensorflow/compiler/xla/service/llvm_ir/fused_ir_emitter.h" -#include "tensorflow/compiler/xla/service/llvm_ir/kernel_support_library.h" #include "tensorflow/compiler/xla/service/llvm_ir/llvm_util.h" #include "tensorflow/compiler/xla/service/llvm_ir/sort_util.h" #include "tensorflow/compiler/xla/service/llvm_ir/tuple_ops.h" @@ -88,6 +87,8 @@ limitations under the License. namespace xla { namespace gpu { +using llvm_ir::KernelMappingScheme; + namespace { using absl::InlinedVector; @@ -546,91 +547,7 @@ Status IrEmitterUnnested::HandleFusion(HloInstruction* fusion) { // TODO(b/112040122): Support variadic reduce. return Unimplemented("Variadic reduce is not supported on GPU"); } - VLOG(3) << "Emitting fused reduction to vector: " << fusion->ToString(); - std::vector> thunks; - absl::Span output_instructions = - root->opcode() == HloOpcode::kTuple - ? root->operands() - : absl::Span(&root, 1); - - // For multi-output fusion emit an initializer for each tuple element. - // Otherwise it's sufficient to just initialize the single output. - HloInstruction* first_reduce = nullptr; - for (int i = 0, e = output_instructions.size(); i != e; ++i) { - if (output_instructions[i]->opcode() == HloOpcode::kReduce) { - TF_ASSIGN_OR_RETURN( - std::unique_ptr initializer_thunk, - BuildInitializerThunk(fusion, output_instructions[i] == root - ? ShapeIndex() - : ShapeIndex({i}))); - thunks.push_back(std::move(initializer_thunk)); - first_reduce = - first_reduce == nullptr ? output_instructions[i] : first_reduce; - } - } - CHECK(first_reduce != nullptr); - std::unique_ptr kernel_thunk = - BuildKernelThunk(fusion, /*implements_whole_instruction=*/false); - GpuElementalIrEmitter elemental_emitter( - hlo_module_config_, ir_emitter_context_->llvm_module(), &b_, - GetNestedComputer()); - FusedIrEmitter fused_emitter(GetGeneratorForOperandIrArrays(fusion), - &elemental_emitter); - TF_RETURN_IF_ERROR(root->Accept(&fused_emitter)); - - // For multi-output fusion CHECK the constraints and feed all the - // reduces into a single loop code generator. Single-output reduce - // fusion is a special case of that. - InlinedVector input_gens; - InlinedVector init_value_gens; - std::vector> - extra_output_gens; - InlinedVector reducers; - InlinedVector reduce_output_shapes; - for (int i = 0, e = output_instructions.size(); i != e; ++i) { - const HloInstruction* inst = output_instructions[i]; - ShapeIndex output_shape_index; - if (root->opcode() == HloOpcode::kTuple) { - output_shape_index = {i}; - } - if (inst->opcode() == HloOpcode::kReduce) { - CHECK(IsReductionToVector(*inst)) - << "Only reductions to vector are supported"; - // Shapes, layouts and dimensions must be the same for all reduces - // inside of this fusion. - CHECK(ShapeUtil::Equal(first_reduce->shape(), inst->shape())); - CHECK(ShapeUtil::Equal(first_reduce->operand(0)->shape(), - inst->operand(0)->shape())); - CHECK(ShapeUtil::Equal(first_reduce->operand(1)->shape(), - inst->operand(1)->shape())); - CHECK(first_reduce->dimensions() == inst->dimensions()); - input_gens.push_back(fused_emitter.GetGenerator(inst->operand(0))); - init_value_gens.push_back( - fused_emitter.GetGenerator(inst->operand(1))); - reducers.push_back(inst->to_apply()); - reduce_output_shapes.push_back(std::move(output_shape_index)); - } else { - // For extra outputs we can relax shape equality to allow different - // types (with the same number of elements). Layouts still have to - // match. - CHECK(ShapeUtil::CompatibleIgnoringElementType( - first_reduce->operand(0)->shape(), inst->shape())); - CHECK(LayoutUtil::Equal(first_reduce->operand(0)->shape().layout(), - inst->shape().layout())); - extra_output_gens.emplace_back(fused_emitter.GetGenerator(inst), - std::move(output_shape_index)); - } - } - const Shape& input_shape = first_reduce->operand(0)->shape(); - TF_CHECK_OK(EmitReductionToVector( - kernel_thunk.get(), first_reduce, input_shape, input_gens, - init_value_gens, first_reduce->dimensions(), reducers, - reduce_output_shapes, extra_output_gens)); - thunks.push_back(std::move(kernel_thunk)); - std::unique_ptr sequential_thunk = - absl::make_unique(std::move(thunks), fusion); - AddThunkToThunkSequence(std::move(sequential_thunk)); - return Status::OK(); + return EmitReductionToVector(fusion); } default: LOG(FATAL) << "Bad opcode for input fusion: " @@ -700,13 +617,12 @@ Status IrEmitterUnnested::HandleCopy(HloInstruction* copy) { } Status IrEmitterUnnested::EmitExtraOutputsForReduce( - const HloInstruction* reduce, const IrArray::Index& index, + const HloInstruction* unnested_hlo, const IrArray::Index& index, absl::Span> extra_output_gens) { for (int i = 0; i != extra_output_gens.size(); ++i) { - const HloInstruction* output = reduce->parent()->FusionInstruction(); llvm::Value* extra_output_address = - GetIrArray(*output, *output, extra_output_gens[i].second) + GetIrArray(*unnested_hlo, *unnested_hlo, extra_output_gens[i].second) .EmitArrayElementAddress(index, &b_, "extra_output_element_address"); TF_ASSIGN_OR_RETURN(llvm::Value* const extra_output_ir_value, @@ -716,984 +632,13 @@ Status IrEmitterUnnested::EmitExtraOutputsForReduce( return Status::OK(); } -Status IrEmitterUnnested::EmitReductionToScalar( - KernelThunk* kernel_thunk, HloInstruction* reduce, const Shape& input_shape, - absl::Span input_gens, - absl::Span init_value_gens, - absl::Span reducers, - absl::Span reduce_output_shapes, - absl::Span> - extra_output_gens) { - // Number of elements processed by a single thread. - constexpr int64 kTileSize = 16; - int64 num_elems = ShapeUtil::ElementsIn(input_shape); - - // Round up the number of tiles to a multiple of the warp size. This is - // necessary for correctness. We launch one thread per tile, and if the - // number of threads isn't a multiple of the number of the warp size, our - // shuffles will read from inactive threads, producing undefined values. - int64 num_tiles = - RoundUpToNearest(CeilOfRatio(num_elems, kTileSize), kWarpSize); - - Shape tiled_input_shape = ShapeUtil::MakeShapeWithLayout( - reduce->shape().element_type(), {num_tiles}, {0}); - LaunchDimensions launch_dimensions = CalculateLaunchDimensions( - tiled_input_shape, ir_emitter_context_->device_description()); - - llvm::Type* index_ty = - GetIndexTypeForKernel(reduce, launch_dimensions.launch_bound(), &b_); - - auto index_typed_constant = [&](uint64 c) -> llvm::Constant* { - return llvm::ConstantInt::get(index_ty, c); - }; - - // Check whether every thread will process a full tile's worth of elements - // without reading outside the bounds of the input. If this is true, we can - // skip some bounds checks in the final algorithm. - bool all_threads_in_bounds = num_tiles * kTileSize == num_elems; - - // __global__ void full_reduce_kernel() { - // x_in_tiles = threadIdx.x + blockIdx.x * blockDim.x; - // x = x_in_tiles * kTileSize; - // - // partial_result = init_value; - // if (all_threads_in_bounds || x + kTileSize <= num_elems) { - // for (i = 0; i < kTileSize; ++i) { - // partial_result = Reducer(partial_result, input[x + i]); - // } - // } else { - // for (i = 0; i < kTileSize; ++i) { - // if (x + i < num_elems) { - // partial_result = Reducer(partial_result, input[x + i]); - // } - // } - // } - // for (i = warpSize / 2; i > 0; i /= 2) { - // partial_result = Reducer(partial_result, - // __shfl_down(partial_result, i)); - // } - // if (lane_id == 0) { - // AtomicReducer(&output[y], partial_result); - // } - // } - // - // // Choose num_blocks and threads_per_block such that: - // // - // // num_blocks * threads_per_block = - // // RoundUpToNextMultipleOf(Ceil(num_elems / kTileSize), warpSize), - // // - // // and threads_per_block is a multiple of warpSize. - // reduce_kernel // - auto loop_body_emitter = [=](const IrArray::Index& tile_index) -> Status { - const int num_reduces = reducers.size(); - llvm::Type* element_ir_type = - llvm_ir::PrimitiveTypeToIrType(input_shape.element_type(), module_); - std::vector partial_reduction_result_addresses; - for (int i = 0; i != num_reduces; ++i) { - llvm::Value* partial_reduction_result_address = - Alloca(element_ir_type, /*ArraySize=*/nullptr, - "partial_reduction_result." + llvm::Twine(i)); - TF_ASSIGN_OR_RETURN(llvm::Value* const init_ir_value, - init_value_gens[i](IrArray::Index(index_ty))); - Store(init_ir_value, partial_reduction_result_address); - partial_reduction_result_addresses.push_back( - partial_reduction_result_address); - } - - llvm::Value* x_in_tiles = tile_index[0]; - x_in_tiles = ZExtOrTrunc(x_in_tiles, index_ty); - - // Emit an inner for-loop that reduces the elements in the tile. - auto emit_tile_element_loop = [=](bool tile_in_bounds) -> Status { - std::unique_ptr tile_element_loop = - llvm_ir::ForLoop::EmitForLoop( - "element_id_in_tile", index_typed_constant(0), - index_typed_constant(kTileSize), index_typed_constant(1), &b_); - - // Emit the body of the partial reduction loop. - llvm_ir::SetToFirstInsertPoint(tile_element_loop->GetBodyBasicBlock(), - &b_); - llvm::Value* x = - NSWAdd(NSWMul(x_in_tiles, index_typed_constant(kTileSize)), - tile_element_loop->GetIndVarValue()); - // Unless we know the tile is entirely in bounds, we have to emit a - // x-in-bounds check before reading from the input. - if (!tile_in_bounds) { - llvm_ir::LlvmIfData if_data = llvm_ir::EmitIfThenElse( - ICmpULT(x, index_typed_constant(num_elems)), "x_in_bounds", &b_); - - // Emit code that reads the input element and accumulates it to - // the partial reduction result. - llvm_ir::SetToFirstInsertPoint(if_data.true_block, &b_); - } - - IrArray::Index input_index( - /*linear=*/x, input_shape, &b_); - llvm::Value* input_address = Alloca(element_ir_type); - for (int i = 0; i != num_reduces; ++i) { - TF_ASSIGN_OR_RETURN(llvm::Value* const input_ir_value, - input_gens[i](input_index)); - Store(input_ir_value, input_address); - TF_RETURN_IF_ERROR(EmitCallToNestedComputation( - *reducers[i], - {partial_reduction_result_addresses[i], input_address}, - partial_reduction_result_addresses[i])); - } - return EmitExtraOutputsForReduce(reduce, input_index, extra_output_gens); - }; - - // x_end = kTileSize + x_in_tiles * kTileSize, i.e., the location that's - // immediately beyond the tile. - llvm::Value* x_end = - NSWAdd(index_typed_constant(kTileSize), - NSWMul(x_in_tiles, index_typed_constant(kTileSize))); - // The tile is entirely in bound if all_threads_in_bounds or - // x_end <= num_elems. - llvm::Value* tile_in_bounds = - Or(ICmpULE(x_end, index_typed_constant(num_elems)), - b_.getInt1(all_threads_in_bounds)); - llvm_ir::LlvmIfData if_tile_in_bounds_data = - llvm_ir::EmitIfThenElse(tile_in_bounds, "tile_in_bounds", &b_); - llvm_ir::SetToFirstInsertPoint(if_tile_in_bounds_data.true_block, &b_); - TF_RETURN_IF_ERROR(emit_tile_element_loop(/*tile_in_bounds=*/true)); - llvm_ir::SetToFirstInsertPoint(if_tile_in_bounds_data.false_block, &b_); - TF_RETURN_IF_ERROR(emit_tile_element_loop(/*tile_in_bounds=*/false)); - - // After the if-then-else statement on tile_in_bounds, emit calls to - // shfl_down that accumulate the partial reduction results of all threads - // from the warp. - llvm_ir::SetToFirstInsertPoint(if_tile_in_bounds_data.after_block, &b_); - int bit_width = llvm_ir::GetSizeInBits(element_ir_type); - // bitcast cannot be applied to aggregate types (even packed ones), so we - // instead bitcast addresses of load/store to intN* of the same bit-width. - llvm::Type* shuffle_ir_type = element_ir_type->isStructTy() - ? b_.getIntNTy(bit_width) - : element_ir_type; - for (int shuffle_distance = kWarpSize / 2; shuffle_distance >= 1; - shuffle_distance /= 2) { - llvm::Value* result_from_other_lane = - Alloca(element_ir_type, nullptr, "result_from_other_lane"); - for (int i = 0; i != num_reduces; ++i) { - llvm::Value* partial_reduction_result = - Load(BitCast(partial_reduction_result_addresses[i], - shuffle_ir_type->getPointerTo()), - "partial_reduction_result"); - CHECK_EQ(launch_dimensions.threads_per_block() % kWarpSize, 0) - << "Requires block size a multiple of the warp size, otherwise we " - "will read undefined elements."; - Store(EmitFullWarpShuffleDown(partial_reduction_result, - b_.getInt32(shuffle_distance), &b_), - BitCast(result_from_other_lane, shuffle_ir_type->getPointerTo())); - TF_RETURN_IF_ERROR(EmitCallToNestedComputation( - *reducers[i], - {partial_reduction_result_addresses[i], result_from_other_lane}, - partial_reduction_result_addresses[i])); - } - } - - const HloInstruction* output = - reduce->IsFused() ? reduce->parent()->FusionInstruction() : reduce; - - // Emit an atomic operation that accumulates the partial reduction result of - // lane 0 (which holds the partially accumulated result for its warp) to the - // output element. - llvm::Value* lane_id = - URem(x_in_tiles, index_typed_constant(kWarpSize), "lane_id"); - llvm_ir::LlvmIfData if_lane_id_is_zero_data = llvm_ir::EmitIfThenElse( - ICmpEQ(lane_id, index_typed_constant(0)), "lane_id_is_zero", &b_); - llvm_ir::SetToFirstInsertPoint(if_lane_id_is_zero_data.true_block, &b_); - - for (int i = 0; i != num_reduces; ++i) { - llvm::Value* output_address = - GetIrArray(*output, *output, reduce_output_shapes[i]) - .EmitArrayElementAddress( - IrArray::Index( - /*linear=*/b_.getInt64(0), - ShapeUtil::GetSubshape(output->shape(), - reduce_output_shapes[i]), - &b_), - &b_, "output_element_address"); - TF_RETURN_IF_ERROR(EmitAtomicOperationForNestedComputation( - *reducers[i], output_address, partial_reduction_result_addresses[i])); - } - return Status::OK(); - }; - - // Emit a parallel loop that iterates through all input tiles, one per thread. - UpdateLaunchDimensions(launch_dimensions, kernel_thunk, - ir_emitter_context_->llvm_module()); - return ParallelLoopEmitter(loop_body_emitter, tiled_input_shape, - launch_dimensions, &b_) - .EmitLoop(IrName(reduce), index_ty); -} - -Status IrEmitterUnnested::EmitColumnReduction( - KernelThunk* kernel_thunk, int64 height, int64 width, - HloInstruction* reduce, const Shape& input_shape, - absl::Span input_gens, - absl::Span init_value_gens, - absl::Span reducers, - absl::Span reduce_output_shapes, - absl::Span> - extra_output_gens) { - // Divide the input matrix into tiles of size KxL. For example, when the - // input matrix is 4x4, K=2, and L=1 the tiled matrix looks like - // - // 0123 - // 0123 - // 4567 - // 4567 // Numbers indicate tile IDs. - // - // Each tile is first partially reduced to a scalar by a thread, and then the - // scalar is accumulated to the output vector using atomic operations. - // - // We choose 128 as the tile size based on empirical evidence. It's big enough - // to reduce the amount of atomic adds in the end, maximizing the memory - // bandwidth. A tile width of 2 allows for high memory bandwidth utilization - // on 16b input data. - constexpr int64 kTileHeight = 128; - constexpr int64 kTileWidth = 2; - - // If the height is not a multiple of kTileHeight, we pad the bottom of the - // input matrix. - const int64 height_in_tiles = CeilOfRatio(height, kTileHeight); - // If width is not a multiple of kTileWidth the rightmost thread will process - // fewer input elements. - const int64 width_in_tiles = CeilOfRatio(width, kTileWidth); - Shape tiled_input_shape = - ShapeUtil::MakeShapeWithLayout(reduce->shape().element_type(), - {height_in_tiles, width_in_tiles}, {1, 0}); - LaunchDimensions launch_dimensions = CalculateLaunchDimensions( - tiled_input_shape, ir_emitter_context_->device_description()); - - // TODO(b/110211620): Convert to use i32 index_type when it is possible. - llvm::Type* index_ty = b_.getInt64Ty(); - - auto index_typed_constant = [&](uint64 c) -> llvm::Constant* { - return llvm::ConstantInt::get(index_ty, c); - }; - - // for (linear_index = threadIdx.x + blockIdx.x * blockDim.x; - // linear_index < height_in_tiles * width_in_tiles; - // linear_index += blockDim.x * gridDim.x) { - // y_in_tiles = linear_index / width_in_tiles; - // x_in_tiles = linear_index % width_in_tiles; - // - // partial_results[kTileWidth] = init_values; - // tile_in_y_bounds = height % kTileHeight == 0 || - // y_in_tiles * kTileHeight + kTileHeight <= height; - // tile_in_x_bounds = width % kTileWidth == 0 || - // x_in_tiles * kTileWidth + kTileWidth <= width; - // // The implementation handles y and x bound checks separately. - // if (tile_in_y_bounds && tile_in_x_bounds) { - // for (y_offset : range(kTileHeight)) { - // y = y_in_tiles * kTileHeight + y_offset; - // for (x_offset : range(kTileWidth)) { - // x = x_in_tiles * kTileWidth + x_offset; - // partial_result = Reducer(partial_result[x_offset], input[y][x]); - // } - // } - // } else { - // for (y_offset : range(kTileHeight)) { - // y = y_in_tiles * kTileHeight + y_offset; - // for (y_offset : range(kTileHeight)) { - // x = x_in_tiles * kTileWidth + x_offset; - // if (y < height && x < width) { - // partial_result = Reducer(partial_result, input[y][x]); - // } - // } - // } - // } - // for (x_offset : range(kTileWidth)) { - // AtomicReducer(&output[x + x_offset], partial_result[x_offset]); - // } - // } - auto loop_body_emitter = [=](const IrArray::Index& tile_index) -> Status { - const int num_reduces = reducers.size(); - // Emit the loop body that reduces one tile. - llvm::Type* element_ir_type = - llvm_ir::PrimitiveTypeToIrType(input_shape.element_type(), module_); - std::vector partial_reduction_result_addresses; - for (int i = 0; i != num_reduces; ++i) { - for (int x_offset = 0; x_offset < kTileWidth; ++x_offset) { - llvm::Value* partial_reduction_result_address = - Alloca(element_ir_type, /*ArraySize=*/nullptr, - "partial_reduction_result." + - llvm::Twine(i * kTileWidth + x_offset)); - TF_ASSIGN_OR_RETURN(llvm::Value* const init_ir_value, - init_value_gens[i](IrArray::Index(index_ty))); - Store(init_ir_value, partial_reduction_result_address); - partial_reduction_result_addresses.push_back( - partial_reduction_result_address); - } - } - - // Emit an inner for-loop that partially reduces the elements in the given - // tile. - llvm::Value* y_in_tiles = tile_index[0]; - llvm::Value* x_in_tiles = tile_index[1]; - - y_in_tiles = ZExtOrTrunc(y_in_tiles, index_ty); - x_in_tiles = ZExtOrTrunc(x_in_tiles, index_ty); - - auto emit_tile_element_loop = [=](bool tile_in_y_bounds, - bool tile_in_x_bounds) -> Status { - std::unique_ptr tile_element_loop = - llvm_ir::ForLoop::EmitForLoop( - "element_id_in_tile", index_typed_constant(0), - index_typed_constant(kTileHeight), index_typed_constant(1), &b_); - - // Emit the body of the partial reduction loop. - llvm_ir::SetToFirstInsertPoint(tile_element_loop->GetBodyBasicBlock(), - &b_); - llvm::Value* y = - NSWAdd(NSWMul(y_in_tiles, index_typed_constant(kTileHeight)), - tile_element_loop->GetIndVarValue()); - - // Unless we know that y is in bounds, we have to emit a check before - // reading from the input. - if (!tile_in_y_bounds) { - llvm_ir::LlvmIfData if_data = llvm_ir::EmitIfThenElse( - ICmpULT(y, index_typed_constant(height)), "y_in_bounds", &b_); - - // Emit code that reads the input element and accumulates it to - // the partial reduction result. - llvm_ir::SetToFirstInsertPoint(if_data.true_block, &b_); - } - for (int x_offset = 0; x_offset < kTileWidth; ++x_offset) { - llvm::Value* x = - NSWAdd(NSWMul(x_in_tiles, index_typed_constant(kTileWidth)), - index_typed_constant(x_offset)); - // Unless we know that x is in bounds, we have to emit a check before - // reading from the input. - if (!tile_in_x_bounds) { - llvm_ir::LlvmIfData if_data = llvm_ir::EmitIfThenElse( - ICmpULT(x, index_typed_constant(width)), "x_in_bounds", &b_); - llvm_ir::SetToFirstInsertPoint(if_data.true_block, &b_); - } - llvm::Value* input_address = Alloca(element_ir_type); - // {y,x} is an index to input_matrix_shape [height,width]. We need to - // convert that to an index to input_shape (the shape of the operand of - // "reduce"). This conversion is composed of a transposition from - // input_shape to normalized_input_shape and a reshape from - // normalized_input_shape to input_matrix_shape. - const Shape normalized_input_shape = - ShapeUtil::MakeShapeWithDescendingLayoutAndSamePhysicalLayout( - input_shape); - auto input_shape_min2maj = LayoutUtil::MinorToMajor(input_shape); - const std::vector transpose_dimension_mapping( - input_shape_min2maj.rbegin(), input_shape_min2maj.rend()); - - const Shape input_matrix_shape = - ShapeUtil::MakeShapeWithDescendingLayout(input_shape.element_type(), - {height, width}); - const IrArray::Index input_matrix_index({y, x}, input_matrix_shape, - &b_); - const IrArray::Index input_index = - input_matrix_index - .SourceIndexOfReshape(input_matrix_shape, - normalized_input_shape, &b_) - .SourceIndexOfTranspose(normalized_input_shape, input_shape, - transpose_dimension_mapping, &b_); - for (int i = 0; i != num_reduces; ++i) { - TF_ASSIGN_OR_RETURN(llvm::Value* const input_ir_value, - input_gens[i](input_index)); - Store(input_ir_value, input_address); - TF_RETURN_IF_ERROR(EmitCallToNestedComputation( - *reducers[i], - {partial_reduction_result_addresses[i * kTileWidth + x_offset], - input_address}, - partial_reduction_result_addresses[i * kTileWidth + x_offset])); - TF_RETURN_IF_ERROR(EmitExtraOutputsForReduce(reduce, input_index, - extra_output_gens)); - } - } - return Status::OK(); - }; - - // y_end = kTileHeight + y_in_tiles * kTileHeight, i.e., the y location - // that's immediately beyond the tile. - llvm::Value* y_end = - NSWAdd(index_typed_constant(kTileHeight), - NSWMul(y_in_tiles, index_typed_constant(kTileHeight))); - // x_end = kTileWidth + x_in_tiles * kTileWidth, i.e., the x location - // that's immediately beyond the tile. - llvm::Value* x_end = - NSWAdd(index_typed_constant(kTileWidth), - NSWMul(x_in_tiles, index_typed_constant(kTileWidth))); - llvm::Value* tile_in_y_bounds = - Or(ICmpULE(y_end, index_typed_constant(height)), - b_.getInt1(height % kTileHeight == 0)); - llvm::Value* tile_in_x_bounds = - Or(ICmpULE(x_end, index_typed_constant(width)), - b_.getInt1(width % kTileWidth == 0)); - // The tile is in y bounds if "height" is a multiple of kTileHeight or - // y_end <= height. - llvm_ir::LlvmIfData if_tile_in_y_bounds_data = - llvm_ir::EmitIfThenElse(tile_in_y_bounds, "tile_in_y_bounds", &b_); - llvm_ir::SetToFirstInsertPoint(if_tile_in_y_bounds_data.true_block, &b_); - // The tile is in x bounds if "width" is a multiple of kTileWidth or - // x_end <= width. - llvm_ir::LlvmIfData if_tile_in_x_bounds_data = - llvm_ir::EmitIfThenElse(tile_in_x_bounds, "tile_in_x_bounds", &b_); - llvm_ir::SetToFirstInsertPoint(if_tile_in_x_bounds_data.true_block, &b_); - TF_RETURN_IF_ERROR(emit_tile_element_loop(/*tile_in_y_bounds=*/true, - /*tile_in_x_bounds=*/true)); - llvm_ir::SetToFirstInsertPoint(if_tile_in_x_bounds_data.false_block, &b_); - TF_RETURN_IF_ERROR(emit_tile_element_loop(/*tile_in_y_bounds=*/true, - /*tile_in_x_bounds=*/false)); - llvm_ir::SetToFirstInsertPoint(if_tile_in_y_bounds_data.false_block, &b_); - if_tile_in_x_bounds_data = - llvm_ir::EmitIfThenElse(tile_in_x_bounds, "tile_in_x_bounds", &b_); - llvm_ir::SetToFirstInsertPoint(if_tile_in_x_bounds_data.true_block, &b_); - TF_RETURN_IF_ERROR(emit_tile_element_loop(/*tile_in_y_bounds=*/false, - /*tile_in_x_bounds=*/true)); - llvm_ir::SetToFirstInsertPoint(if_tile_in_x_bounds_data.false_block, &b_); - TF_RETURN_IF_ERROR(emit_tile_element_loop(/*tile_in_y_bounds=*/false, - /*tile_in_x_bounds=*/false)); - - // After the nested if-then-else statement on tile_in_y_bounds and - // tile_in_x_bounds, emit atomic operations to accumulate the partial - // reduction result to the output element. - llvm_ir::SetToFirstInsertPoint(if_tile_in_y_bounds_data.after_block, &b_); - const HloInstruction* output = - reduce->IsFused() ? reduce->parent()->FusionInstruction() : reduce; - for (int i = 0; i != num_reduces; ++i) { - for (int x_offset = 0; x_offset < kTileWidth; ++x_offset) { - llvm::Value* x = - NSWAdd(NSWMul(x_in_tiles, index_typed_constant(kTileWidth)), - index_typed_constant(x_offset)); - llvm::Value* output_address = - GetIrArray(*output, *output, reduce_output_shapes[i]) - .EmitArrayElementAddress( - IrArray::Index( - x, - ShapeUtil::GetSubshape(output->shape(), - reduce_output_shapes[i]), - &b_), - &b_, "output_element_address"); - TF_RETURN_IF_ERROR(EmitAtomicOperationForNestedComputation( - *reducers[i], output_address, - partial_reduction_result_addresses[i * kTileWidth + x_offset])); - } - } - return Status::OK(); - }; - - // Emit a parallel loop that iterate through all input tiles. - UpdateLaunchDimensions(launch_dimensions, kernel_thunk, - ir_emitter_context_->llvm_module()); - return ParallelLoopEmitter(loop_body_emitter, tiled_input_shape, - launch_dimensions, &b_) - .EmitLoop(IrName(reduce), index_ty); -} - -static std::pair ComputeTilingSchemeForReduction( - int64 depth, int64 width, int64 kWarpSize) { - constexpr int64 kTargetNumElementsPerThread = 64; - int64 x_tile_size = kTargetNumElementsPerThread; - int64 z_tile_size = 1; - - // Only tile along the x dimension with tile size kTargetNumElementsPerThread - // if doing so doesn't require a slow version of loop with bound check on each - // dimension. A more sophisticated heuristics is to enable tile along the - // x dimension with tile size kTargetNumElementsPerThread when either width is - // a factor of (kWarpSize * kTargetNumElementsPerThread) or width is big - // enough so that only a small fraction of the threads execute the slow - // version of loop with bound check. - if (width % (kWarpSize * kTargetNumElementsPerThread) != 0) { - x_tile_size = 8; - z_tile_size = 8; - while (depth % z_tile_size != 0) { - z_tile_size -= 1; - } - } - - return std::pair(x_tile_size, z_tile_size); -} - -Status IrEmitterUnnested::EmitRowReduction( - KernelThunk* kernel_thunk, int64 depth, int64 height, int64 width, - HloInstruction* reduce, const Shape& input_shape, - absl::Span input_gens, - absl::Span init_value_gens, - absl::Span reducers, - absl::Span reduce_output_shapes, - absl::Span> - extra_output_gens) { - // A naive algorithm is: - // 1. Divide the x dimension of the input tensor into tiles of size 1x1xX. - // 2. Partially reduces each tile to a scalar using one thread. - // 3. Accumulates that scalar to the output vector using atomic operations. - // - // for (linear_index = threadIdx.x + blockIdx.x * blockDim.x; - // linear_index < depth * height * width_in_tiles; - // linear_index += blockDim.x * gridDim.x) { - // int x_in_tiles = linear_index % width_in_tiles; - // int y = linear_index / width_in_tiles % height; - // int z = linear_index / (height * width_in_tiles); - // float partial_result = 0; - // for (element_id_in_tile : range(x_tile_size)) { - // int x = x_in_tiles * x_tile_size + element_id_in_tile; - // if (x < width) - // partial_result = reducer(partial_result, input[z][y][x]); - // } - // AtomicReducer(&output[y], partial_result); - // } - // - // Four optimizations are performed. - // - // 1. To coalesce global memory accesses, dilate the tile with a factor of 32 - // (i.e. the warp size). For example, suppose the width is 8x32=256. Instead - // of making each tile consecutive, we let make tile 0 column - // [0,32,64,...,224], tile 1 column [1,33,65,...,225], and so on. This ensures - // that threads in a warp access consecutive memory in one iteration (i.e. - // coalesced). In the above example, the warp that contains thread 0-31 - // accesses column 0-31 in the first iteration, and 32-63 in the second - // iteration, and so on. - // - // 2. Partially accumulate partial reduced results computed by threads in the - // same warp using shfl_down. Using shfl_down is faster than directly using - // atomic operations because shfl_down transfers the data between threads - // using shared memory and threads in the same warp run in lock step (thus no - // extra synchronization needed). See - // https://devblogs.nvidia.com/parallelforall/faster-parallel-reductions-kepler/ - // for details. The downside is, to produce correct results when using - // shfl_down, we need to guarantee threads in the same warp work on input - // elements with the same y, so the number of tiles in each row must be a - // multiple of 32. - // - // 3. Specialize the case that the entire tile is in bounds. When that is - // true, we don't need to emit "if(x 0; shuffle_distance /= 2) - // partial_result = Reducer( - // partial_result, - // __shfl_down_sync(CUDA_WARP_ALL, partial_result, shuffle_distance)); - // if (lane_id == 0) - // AtomicReducer(&output[y], partial_result); - // } - // - - int64 x_tile_size; - int64 z_tile_size; - std::tie(x_tile_size, z_tile_size) = - ComputeTilingSchemeForReduction(depth, width, kWarpSize); - - // Round the width in tiles up to the nearest multiple of kWarpSize, so that - // the use of shfl_down is valid. - const int64 width_in_tiles = - RoundUpToNearest(CeilOfRatio(width, x_tile_size), kWarpSize); - Shape tiled_input_shape = ShapeUtil::MakeShapeWithLayout( - reduce->shape().element_type(), - {depth / z_tile_size, height, width_in_tiles}, {2, 1, 0}); - LaunchDimensions launch_dimensions = CalculateLaunchDimensions( - tiled_input_shape, ir_emitter_context_->device_description()); - llvm::Type* index_ty = - GetIndexTypeForKernel(reduce, launch_dimensions.launch_bound(), &b_); - - auto index_typed_constant = [&](uint64 c) -> llvm::Constant* { - return llvm::ConstantInt::get(index_ty, c); - }; - - auto loop_body_emitter = [=](const IrArray::Index& tile_index) { - const int num_reduces = reducers.size(); - llvm::Type* element_ir_type = llvm_ir::PrimitiveTypeToIrType( - input_shape.element_type(), ir_emitter_context_->llvm_module()); - std::vector partial_reduction_result_addresses; - for (int i = 0; i != num_reduces; ++i) { - llvm::Value* partial_reduction_result_address = - Alloca(element_ir_type, /*ArraySize=*/nullptr, - "partial_reduction_result." + llvm::Twine(i)); - TF_ASSIGN_OR_RETURN(llvm::Value* const init_ir_value, - init_value_gens[i](IrArray::Index(index_ty))); - Store(init_ir_value, partial_reduction_result_address); - partial_reduction_result_addresses.push_back( - partial_reduction_result_address); - } - - llvm::Value* z_tile = tile_index[0]; - llvm::Value* y = tile_index[1]; - llvm::Value* x_tile = tile_index[2]; - - x_tile = ZExtOrTrunc(x_tile, index_ty); - - llvm::Value* warp_id = - UDiv(x_tile, index_typed_constant(kWarpSize), "warp_id"); - llvm::Value* lane_id = - URem(x_tile, index_typed_constant(kWarpSize), "lane_id"); - - // The x-location of the last element in this z-x-tile. - // last_x = lane_id + warpSize * (x_tile_size - 1 + warp_id * x_tile_size); - llvm::Value* last_x = NSWAdd( - lane_id, - NSWMul(index_typed_constant(kWarpSize), - NSWAdd(index_typed_constant(x_tile_size - 1), - NSWMul(warp_id, index_typed_constant(x_tile_size))))); - - KernelSupportLibrary ksl( - &b_, - /*unroll_mode=*/xla::llvm_ir::UnrollMode::kFullyUnroll, - /*prevent_vectorization=*/false); - - // Emit a for-loop that partially reduces the elements in the given - // z-x-tile. - auto emit_z_x_tile_element_loop = [&](bool x_tile_in_bounds, - int64 x_tile_loop_bound) -> Status { - auto emit_z_tile_element_loop = [&](llvm::Value* z_indvar) -> Status { - llvm::Value* z = - NSWAdd(z_indvar, NSWMul(index_typed_constant(z_tile_size), z_tile)); - TF_RETURN_IF_ERROR(ksl.For( - "x_tile", - /*start=*/index_typed_constant(0), - /*end=*/index_typed_constant(x_tile_loop_bound), - /*step=*/1, [&](llvm::Value* x_indvar) -> Status { - // x = lane_id + - // warpSize * (element_id_in_x_tile + warp_id * x_tile_size); - llvm::Value* x = NSWAdd( - lane_id, - NSWMul(index_typed_constant(kWarpSize), - NSWAdd(x_indvar, - NSWMul(warp_id, llvm::ConstantInt::get( - index_ty, x_tile_size))))); - - // Unless we know the x-tile is entirely in bounds, we have to - // emit a x-in-bounds check before reading from the input. - if (!x_tile_in_bounds) { - llvm_ir::LlvmIfData if_x_in_bounds_data = - llvm_ir::EmitIfThenElse( - ICmpULT(x, index_typed_constant(width)), "x_in_bounds", - &b_); - // Points b_ to the then-block. - llvm_ir::SetToFirstInsertPoint(if_x_in_bounds_data.true_block, - &b_); - } - - // Emit code that reads the input element and accumulates it - // to the partial reduction result. - llvm::Value* input_address = Alloca(element_ir_type); - { - // {z,y,x} is an index to input_3d_tensor_shape - // [depth,height,width]. We need to convert that to an index - // to input_shape (the shape of the operand of "reduce"). - // This conversion is composed of a transposition from - // input_shape to normalized_input_shape and a reshape from - // normalized_input_shape to input_3d_tensor_shape. - const Shape normalized_input_shape = ShapeUtil:: - MakeShapeWithDescendingLayoutAndSamePhysicalLayout( - input_shape); - auto input_shape_min2maj = - LayoutUtil::MinorToMajor(input_shape); - const std::vector transpose_dimension_mapping( - input_shape_min2maj.rbegin(), input_shape_min2maj.rend()); - const Shape input_3d_tensor_shape = - ShapeUtil::MakeShapeWithDescendingLayout( - input_shape.element_type(), {depth, height, width}); - const IrArray::Index input_3d_tensor_index( - {z, y, x}, input_3d_tensor_shape, &b_); - const IrArray::Index input_index = - input_3d_tensor_index - .SourceIndexOfReshape(input_3d_tensor_shape, - normalized_input_shape, &b_) - .SourceIndexOfTranspose( - normalized_input_shape, input_shape, - transpose_dimension_mapping, &b_); - - for (int i = 0; i != num_reduces; ++i) { - TF_ASSIGN_OR_RETURN(llvm::Value* const input_ir_value, - input_gens[i](input_index)); - Store(input_ir_value, input_address); - TF_RETURN_IF_ERROR(EmitCallToNestedComputation( - *reducers[i], - {partial_reduction_result_addresses[i], input_address}, - partial_reduction_result_addresses[i])); - } - return EmitExtraOutputsForReduce(reduce, input_index, - extra_output_gens); - } - })); - return Status::OK(); - }; - - return ksl.For("z_tile", - /*start=*/index_typed_constant(0), - /*end=*/index_typed_constant(z_tile_size), - /*step=*/1, emit_z_tile_element_loop); - }; - - llvm::Value* tile_in_bounds = - Or(b_.getInt1(width % (x_tile_size * kWarpSize) == 0), - ICmpULT(last_x, index_typed_constant(width))); - - TF_RETURN_IF_ERROR( - ksl.If(tile_in_bounds, - /*true_block_generator=*/ - [&]() -> Status { - return emit_z_x_tile_element_loop(/*x_tile_in_bounds=*/true, - x_tile_size); - }, - /*false_block_generator=*/ - [&]() -> Status { - return emit_z_x_tile_element_loop( - /*x_tile_in_bounds=*/false, - CeilOfRatio(width % (x_tile_size * kWarpSize), kWarpSize)); - })); - - // After accumulating the elements of the z_x_tile, emit calls to - // shfl_down that accumulate the partial reduction results of all - // threads in a warp. - int bit_width = llvm_ir::GetSizeInBits(element_ir_type); - // bitcast cannot be applied to aggregate types (even packed ones), so we - // instead bitcast addresses of load/store to intN* of the same bit-width. - llvm::Type* shuffle_ir_type = element_ir_type->isStructTy() - ? b_.getIntNTy(bit_width) - : element_ir_type; - for (int shuffle_distance = 16; shuffle_distance >= 1; - shuffle_distance /= 2) { - llvm::Value* result_from_other_lane = - Alloca(element_ir_type, nullptr, "result_from_other_lane"); - for (int i = 0; i != num_reduces; ++i) { - llvm::Value* partial_reduction_result = - Load(BitCast(partial_reduction_result_addresses[i], - shuffle_ir_type->getPointerTo()), - "partial_reduction_result"); - CHECK_EQ(launch_dimensions.threads_per_block() % kWarpSize, 0) - << "Requires block size a multiple of the warp size, otherwise we " - "will read undefined elements."; - Store(EmitFullWarpShuffleDown(partial_reduction_result, - b_.getInt32(shuffle_distance), &b_), - BitCast(result_from_other_lane, shuffle_ir_type->getPointerTo())); - TF_RETURN_IF_ERROR(EmitCallToNestedComputation( - *reducers[i], - {partial_reduction_result_addresses[i], result_from_other_lane}, - partial_reduction_result_addresses[i])); - } - } - - const HloInstruction* output = - reduce->IsFused() ? reduce->parent()->FusionInstruction() : reduce; - - // Emit an atomic operation that accumulates the partial reduction result of - // lane 0 (which holds the partially accumulated result for its warp) to the - // output element. - llvm_ir::LlvmIfData if_lane_id_is_zero_data = llvm_ir::EmitIfThenElse( - ICmpEQ(lane_id, index_typed_constant(0)), "lane_id_is_zero", &b_); - llvm_ir::SetToFirstInsertPoint(if_lane_id_is_zero_data.true_block, &b_); - for (int i = 0; i != num_reduces; ++i) { - llvm::Value* output_address = - GetIrArray(*output, *output, reduce_output_shapes[i]) - .EmitArrayElementAddress( - IrArray::Index(y, - ShapeUtil::GetSubshape( - output->shape(), reduce_output_shapes[i]), - &b_), - &b_, "output_element_address"); - // We don't need to emit atomic operations if there is only one tile of - // results. 'depth' is the z dimension, 'width' is the x dimension. - if (z_tile_size >= depth && x_tile_size >= width) { - TF_RETURN_IF_ERROR(EmitCallToNestedComputation( - *reducers[i], - {output_address, partial_reduction_result_addresses[i]}, - output_address)); - } else { - TF_RETURN_IF_ERROR(EmitAtomicOperationForNestedComputation( - *reducers[i], output_address, - partial_reduction_result_addresses[i])); - } - } - return Status::OK(); - }; - - // Emit a parallel loop that iterates through every input tiles. - UpdateLaunchDimensions(launch_dimensions, kernel_thunk, - ir_emitter_context_->llvm_module()); - return ParallelLoopEmitter(loop_body_emitter, tiled_input_shape, - launch_dimensions, &b_) - .EmitLoop(IrName(reduce), index_ty); -} - -// Figures out whether `reduce` is a row or column reduction, and which -// dimensions to reduce, and calls either `EmitRowReduction` or -// `EmitColumnReduction` as appropriate. -// Prerequisite: all the dimensions to keep are contiguous in the input layout -// and, if `reduce` is fused, the fused subgraph is pure -// elementwise. -Status IrEmitterUnnested::EmitReductionToVector( - KernelThunk* kernel_thunk, HloInstruction* reduce, const Shape& input_shape, - absl::Span input_gens, - absl::Span init_value_gens, - absl::Span dimensions_to_reduce, - absl::Span reducers, - absl::Span reduce_output_shapes, - absl::Span> - extra_output_gens) { - // This emission requires "reduce" to have an input layout. It is either set - // by LayoutAssignment (for a top-level kReduce) or by InstructionFusion (for - // a fused kReduce). - CHECK(input_shape.has_layout()) << "LayoutAssignment or InstructionFusion " - "doesn't set the input layout of " - << reduce->ToString(); - - // Specialize multi-dimensional-array-to-vector reduction. - std::vector input_dims_to_keep; - for (int64 input_dim = 0; input_dim < ShapeUtil::Rank(input_shape); - ++input_dim) { - if (std::find(dimensions_to_reduce.begin(), dimensions_to_reduce.end(), - input_dim) == dimensions_to_reduce.end()) { - input_dims_to_keep.push_back(input_dim); - } - } - - // Sort the dimensions to keep from minor to major, to facilitate checking - // whether another dimension is major or minor of them. - std::sort(input_dims_to_keep.begin(), input_dims_to_keep.end(), - [&input_shape](int64 dim_a, int64 dim_b) { - return PositionInContainer(LayoutUtil::MinorToMajor(input_shape), - dim_a) < - PositionInContainer(LayoutUtil::MinorToMajor(input_shape), - dim_b); - }); - // Now, if output rank is at least 1, `input_dims_to_keep.front()` is - // minormost and `input_dims_to_keep.back()` is majormost. - - // If the dimensions to keep are minormost, emit a column reduction. As all - // the dimensions to keep are contiguous, by prerequisite of - // `EmitReductionToVector`, we only need to check whether the minormost - // dimension of the input is to keep. - if (ShapeUtil::IsEffectiveScalar(reduce->shape())) { - return EmitReductionToScalar(kernel_thunk, reduce, input_shape, input_gens, - init_value_gens, reducers, - reduce_output_shapes, extra_output_gens); - } else if (input_dims_to_keep.front() == - LayoutUtil::Minor(input_shape.layout(), 0)) { - // Column reduction. Treat the result of "input" as a matrix whose width - // is the most minor dimension and height the product of other dimensions, - // and treat "reduce" as a column reduction of the input matrix. - const int64 width = ShapeUtil::ElementsIn(reduce->shape()); - // "width" can be zero, so don't do - // height = ShapeUtil::ElementsIn(input_shape) / width; - int64 height = 1; - for (int64 input_dim = 0; input_dim < ShapeUtil::Rank(input_shape); - ++input_dim) { - if (!std::count(input_dims_to_keep.begin(), input_dims_to_keep.end(), - input_dim)) { - height *= input_shape.dimensions(input_dim); - } - } - return EmitColumnReduction(kernel_thunk, height, width, reduce, input_shape, - input_gens, init_value_gens, reducers, - reduce_output_shapes, extra_output_gens); - } else { - // Reduce the row dimension of a matrix or reduce dimension 0 and 2 in a - // 3D tensor. The size of dimension 1 (the height) is the size of the - // dimension to keep, the size of dimension 0 (the depth) is the product - // of dimensions that are more major than the dimension to keep, and the - // size of dimension 2 (the width) is the product of more minor - // dimensions. - int64 depth = 1; - int64 width = 1; - for (int64 input_dim = 0; input_dim < ShapeUtil::Rank(input_shape); - ++input_dim) { - if (PositionInContainer(LayoutUtil::MinorToMajor(input_shape), - input_dim) > - PositionInContainer(LayoutUtil::MinorToMajor(input_shape), - input_dims_to_keep.back())) { - depth *= input_shape.dimensions(input_dim); - } else if (PositionInContainer(LayoutUtil::MinorToMajor(input_shape), - input_dim) < - PositionInContainer(LayoutUtil::MinorToMajor(input_shape), - input_dims_to_keep.front())) { - width *= input_shape.dimensions(input_dim); - } - } - const int64 height = ShapeUtil::ElementsIn(reduce->shape()); - return EmitRowReduction(kernel_thunk, depth, height, width, reduce, - input_shape, input_gens, init_value_gens, reducers, - reduce_output_shapes, extra_output_gens); - } -} - Status IrEmitterUnnested::HandleReduce(HloInstruction* reduce) { // TODO(b/112040122): Support multi-output reduce. if (!ShapeUtil::IsArray(reduce->shape())) { return Unimplemented("Multi-output reduce is not supported on GPU"); } - auto input = reduce->operand(0); - auto init_value = reduce->operand(1); - absl::Span dimensions_to_reduce(reduce->dimensions()); - HloComputation* reducer = reduce->to_apply(); - // HandleReduce specializes reduction from a multi-dimensional array to a 1D - // array. The specialized version requires an initializer thunk that - // initializes the output array to the initial value of the reduce. if (IsReductionToVector(*reduce)) { - TF_ASSIGN_OR_RETURN(std::unique_ptr initializer_thunk, - BuildInitializerThunk(reduce)); - std::vector> thunks; - thunks.push_back(std::move(initializer_thunk)); - std::unique_ptr kernel_thunk = - BuildKernelThunk(reduce, /*implements_whole_instruction=*/false); - - TF_CHECK_OK(EmitReductionToVector( - kernel_thunk.get(), reduce, input->shape(), - {[&](const IrArray::Index& index) { - return GetIrArray(*input, *reduce).EmitReadArrayElement(index, &b_); - }}, - {[&](const IrArray::Index& index) { - return GetIrArray(*init_value, *reduce) - .EmitReadArrayElement(index, &b_); - }}, - dimensions_to_reduce, {reducer}, {{}}, {})); - - thunks.push_back(std::move(kernel_thunk)); - - std::unique_ptr sequential_thunk = - absl::make_unique(std::move(thunks), reduce); - AddThunkToThunkSequence(std::move(sequential_thunk)); - return Status::OK(); + return EmitReductionToVector(reduce); } return IrEmitter::HandleReduce(reduce); @@ -1818,7 +763,7 @@ Status IrEmitterUnnested::HandleSelectAndScatter( // Create the inner loop to iterate over the window. llvm_ir::ForLoopNest window_loops(IrName(select_and_scatter, "inner"), &b_, index_type); - std::vector window_size; + DimensionVector window_size; for (const auto& dim : window.dimensions()) { window_size.push_back(dim.size()); CHECK_GT(dim.size(), 0); @@ -2171,7 +1116,18 @@ Status IrEmitterUnnested::HandleSelect(HloInstruction* select) { Status IrEmitterUnnested::HandleSort(HloInstruction* sort) { std::vector> thunks; Shape keys_shape = sort->operand(0)->shape(); + int64 dimension_to_sort = sort->dimensions(0); + // In case there is a 'values' parameter that is a iota, we take note and use + // it later to ensure a stable sort. Otherwise, we don't guarantee a stable + // sort. + int64 iota_values_parameter_index = -1; for (int64 i = 0; i < sort->operand_count(); ++i) { + if (i > 0 && sort->operand(i)->opcode() == HloOpcode::kIota && + ShapeUtil::ElementIsIntegral(sort->operand(i)->shape()) && + Cast(sort->operand(i))->iota_dimension() == + dimension_to_sort) { + iota_values_parameter_index = i; + } ShapeIndex shape_index = sort->operand_count() > 1 ? ShapeIndex({i}) : ShapeIndex({}); // We assume that the layout of all involved operands and outputs is the @@ -2196,7 +1152,6 @@ Status IrEmitterUnnested::HandleSort(HloInstruction* sort) { } } - int64 dimension_to_sort = sort->dimensions(0); uint64 dimension_to_sort_bound = keys_shape.dimensions(dimension_to_sort); int64 num_stages = tensorflow::Log2Ceiling(dimension_to_sort_bound); CHECK_GE(1ULL << num_stages, dimension_to_sort_bound); @@ -2298,8 +1253,9 @@ Status IrEmitterUnnested::HandleSort(HloInstruction* sort) { } } return llvm_ir::EmitSortInPlace( - dimension_to_sort, keys_array, values_arrays, IrName(sort), xor_masks, - &b_, launch_dimensions, + dimension_to_sort, keys_array, values_arrays, + iota_values_parameter_index, IrName(sort), xor_masks, &b_, + launch_dimensions, xor_masks.size() > 1 ? num_iterations_in_sort_dim : standard_num_iterations_in_sort_dim, kTileSize); @@ -2385,7 +1341,7 @@ Status IrEmitterUnnested::HandleCrossReplicaSum(HloInstruction* crs) { return Status::OK(); } -Status IrEmitterUnnested::HandleAfterAll(HloInstruction* gen_token) { +Status IrEmitterUnnested::HandleAfterAll(HloInstruction* after_all) { return Status::OK(); } @@ -3146,31 +2102,6 @@ std::vector IrEmitterUnnested::ConstructIrArrayForInputs( return param_arrays; } -int IrEmitterUnnested::ConstructOutputReducedShapeAndCastOutputIrArrayToShape( - const HloInstruction& hlo, const std::vector& output_arrays, - absl::Span reduced_output_dims, - std::vector* output_reduced_shapes, - std::vector* output_in_reduced_shape_arrays) { - int64 num_outputs = 1; - if (hlo.IsMultiOutputFusion()) { - num_outputs = ShapeUtil::TupleElementCount(hlo.shape()); - output_in_reduced_shape_arrays->reserve(num_outputs); - output_reduced_shapes->reserve(num_outputs); - for (int64 i = 0; i < num_outputs; ++i) { - output_reduced_shapes->push_back(ShapeUtil::MakeShapeWithDescendingLayout( - ShapeUtil::GetSubshape(hlo.shape(), {i}).element_type(), - reduced_output_dims)); - output_in_reduced_shape_arrays->push_back( - output_arrays[i].CastToShape((*output_reduced_shapes)[i], &b_)); - } - } else { - output_reduced_shapes->push_back(ShapeUtil::MakeShapeWithDescendingLayout( - hlo.shape().element_type(), reduced_output_dims)); - output_in_reduced_shape_arrays->push_back( - output_arrays[0].CastToShape((*output_reduced_shapes)[0], &b_)); - } - return num_outputs; -} int IrEmitterUnnested::ConstructInputReducedShapeAndCastInputIrArrayToShape( const HloInstruction& hlo, const std::vector& param_arrays, @@ -3230,82 +2161,854 @@ llvm::Value* GetBlockIdx(llvm::IRBuilder<>* builder, llvm::Type* index_ty, "block.id.x"); } -// Emits code to process up to (tile_size/num_rows) elements in a tile, given -// `emit_elem_function` is the function to emit code to process one element, `y` -// and `x` are the coordinates for the first element to process, and `index` is -// the index for the origin of the tile. Emits bounds check to ensure that each -// processed element is within the boundary defined by `tile_width` and -// `tile_height`. -void EmitTiledElementalCodeWithBoundsCheck( - int64 tile_size, int64 num_rows, const IrArray::Index& index, - const string& loop_name, KernelSupportLibrary* ksl, - llvm::IRBuilder<>* builder, llvm::Value* y, llvm::Value* x, - llvm::Value* tile_width, llvm::Value* tile_height, - const std::function& - emit_elem_function) { - llvm::Type* index_ty = tile_width->getType(); - // Emits a constant value with index type. - auto index_typed_constant = [&](uint64 c) -> llvm::Constant* { - return llvm::ConstantInt::get(index_ty, c); - }; - // Adds `addend` to the given `dim` of `index`. - auto offset_dim = [&](IrArray::Index index, llvm::Value* addend, int64 dim) { - index[dim] = builder->CreateAdd(index[dim], addend); - return index; - }; - - auto emit_full_tile = [&] { - for (int64 i = 0; i < tile_size; i += num_rows) { - auto source_idx = offset_dim(index, index_typed_constant(i), /*dim=*/1); - auto y_loc = builder->CreateAdd(index_typed_constant(i), y); - emit_elem_function(source_idx, y_loc); +void EmitFullTile(const KernelMappingScheme* mapping_scheme, + const IrArray::Index& tile_origin_index, + llvm::IRBuilder<>* builder, llvm::Value* y, llvm::Value* x, + llvm::Type* index_ty, + const std::function& emit_elem_function) { + int64 num_threads_x = mapping_scheme->GetNumberOfThreadsForDimensionX(); + int64 num_threads_y = mapping_scheme->GetNumberOfThreadsForDimensionY(); + int64 tile_size_x = mapping_scheme->GetTileSizeForDimensionX(); + int64 tile_size_y = mapping_scheme->GetTileSizeForDimensionY(); + for (int64 i = 0; i < tile_size_y; i += num_threads_y) { + IrArray::Index source_idx_y = + tile_origin_index.AddOffsetToDim(llvm::ConstantInt::get(index_ty, i), + KernelMappingScheme::DimY, builder); + llvm::Value* y_loc = + builder->CreateAdd(llvm::ConstantInt::get(index_ty, i), y); + for (int64 j = 0; j < tile_size_x; j += num_threads_x) { + IrArray::Index source_idx = + source_idx_y.AddOffsetToDim(llvm::ConstantInt::get(index_ty, j), + KernelMappingScheme::DimX, builder); + llvm::Value* x_loc = + builder->CreateAdd(llvm::ConstantInt::get(index_ty, j), x); + emit_elem_function(source_idx, y_loc, x_loc); } - }; + } +} + +void EmitPartialTile( + const KernelMappingScheme* mapping_scheme, + const IrArray::Index& tile_origin_index, const string& loop_name, + KernelSupportLibrary* ksl, llvm::IRBuilder<>* builder, llvm::Value* y, + llvm::Value* x, llvm::Value* tile_height, llvm::Value* tile_width, + llvm::Type* index_ty, + const std::function& emit_elem_function) { + int64 num_threads_x = mapping_scheme->GetNumberOfThreadsForDimensionX(); + int64 num_threads_y = mapping_scheme->GetNumberOfThreadsForDimensionY(); + int64 tile_size_x = mapping_scheme->GetTileSizeForDimensionX(); + + for (int64 j = 0; j < tile_size_x; j += num_threads_x) { + IrArray::Index source_idx = + tile_origin_index.AddOffsetToDim(llvm::ConstantInt::get(index_ty, j), + KernelMappingScheme::DimX, builder); + llvm::Value* x_loc = + builder->CreateAdd(llvm::ConstantInt::get(index_ty, j), x); + + ksl->IfReturnVoid( + loop_name + "_x_in_tile", builder->CreateICmpULT(x_loc, tile_width), + [&] { + // tile_height_bound = + // ceil(tile_height / num_threads_y) * num_threads_y + llvm::Value* ceiling_of_ratio = builder->CreateUDiv( + builder->CreateAdd(tile_height, llvm::ConstantInt::get( + index_ty, num_threads_y - 1)), + llvm::ConstantInt::get(index_ty, num_threads_y)); + llvm::Value* tile_height_bound = builder->CreateMul( + ceiling_of_ratio, + llvm::ConstantInt::get(index_ty, num_threads_y)); + ksl->ForReturnVoid( + loop_name, /*start=*/llvm::ConstantInt::get(index_ty, 0), + /*end=*/tile_height_bound, + /*step=*/llvm::ConstantInt::get(index_ty, num_threads_y), + [&](llvm::Value* y_indvar) { + llvm::Value* y_loc = builder->CreateAdd(y_indvar, y); + ksl->IfReturnVoid( + loop_name + "_y_in_tile", + builder->CreateICmpULT(y_loc, tile_height), [&] { + emit_elem_function( + source_idx.AddOffsetToDim( + y_indvar, KernelMappingScheme::DimY, builder), + y_loc, x_loc); + }); + }); + }); + } +} + +// Emits code to process up to +// (tile_size_x/num_threads_x * tile_size_y/num_threads_y) elements in a tile, +// given `emit_elem_function` is the function to emit code to process one +// element, `y` and `x` are the intra-tile coordinates for the first element +// to process, and `index` is the index for the origin of the tile. Information +// about tile_size_x/y and num_threads_x/y are stored in `mapping_scheme`. Emits +// bounds check to ensure that each processed element is within the boundary +// defined by `tile_width` and `tile_height`. +void EmitTiledElementalCodeWithBoundsCheck( + const KernelMappingScheme* mapping_scheme, + const IrArray::Index& tile_origin_index, const string& loop_name, + KernelSupportLibrary* ksl, llvm::IRBuilder<>* builder, llvm::Value* y, + llvm::Value* x, llvm::Value* tile_height, llvm::Value* tile_width, + const std::function& emit_elem_function) { + int64 tile_size_x = mapping_scheme->GetTileSizeForDimensionX(); + int64 tile_size_y = mapping_scheme->GetTileSizeForDimensionY(); + llvm::Type* index_ty = tile_width->getType(); - auto emit_last_row = [&] { - ksl->IfReturnVoid("x_in_tile", builder->CreateICmpULT(x, tile_width), [&] { - // tile_height_upper_bound = - // ceil(tile_height / num_rows) * num_rows - auto tile_height_upper_bound = builder->CreateMul( - builder->CreateUDiv( - builder->CreateAdd(tile_height, - index_typed_constant(num_rows - 1)), - index_typed_constant(num_rows)), - index_typed_constant(num_rows)); - ksl->ForReturnVoid( - loop_name, /*start=*/index_typed_constant(0), - /*end=*/tile_height_upper_bound, - /*step=*/index_typed_constant(num_rows), [&](llvm::Value* y_indvar) { - auto y_loc = builder->CreateAdd(y_indvar, y); - ksl->IfReturnVoid( - "y_in_tile", builder->CreateICmpULT(y_loc, tile_height), [&] { - emit_elem_function(offset_dim(index, y_indvar, /*dim=*/1), - y_loc); - }); - }); - }); - }; ksl->IfReturnVoid( - "full_tile", + loop_name + "_full_tile", builder->CreateAnd( - builder->CreateICmpEQ(index_typed_constant(tile_size), tile_width), - builder->CreateICmpEQ(index_typed_constant(tile_size), tile_height)), - emit_full_tile, emit_last_row); + builder->CreateICmpEQ(llvm::ConstantInt::get(index_ty, tile_size_x), + tile_width), + builder->CreateICmpEQ(llvm::ConstantInt::get(index_ty, tile_size_y), + tile_height)), + [&] { + EmitFullTile(mapping_scheme, tile_origin_index, builder, y, x, index_ty, + emit_elem_function); + }, + [&] { + EmitPartialTile(mapping_scheme, tile_origin_index, loop_name, ksl, + builder, y, x, tile_height, tile_width, index_ty, + emit_elem_function); + }); } } // namespace +// Emits code to process a tensor element in a tile for the given kCopy HLO that +// performs a 0-2-1 transpose. +// +// index: The index for the first output element in the normalized tensor. The +// normalized tensor is the resulting tensor after collapsing contiguous +// dimensions that play the same role in the transpose. +// y_loc: The y coordinate within a tile. +// x_loc: The x coordinate within a tile. +// kernel_info: Other information to support the kernel code generation. +void IrEmitterUnnested::EmitTileElementForCopy( + HloInstruction* hlo, const llvm_ir::IrArray::Index& index, + const KernelCodegenInfo* kernel_info, llvm::Value* y_loc, + llvm::Value* x_loc) { + llvm_ir::TiledParameterInfo* tiled_param_info = + kernel_info->GetTiledParameterInfo(); + // TODO(jlebar): Add AA metadata to this load. + llvm::Instruction* load_from_shmem_buffer = + Load(GEP(tiled_param_info->GetBufferForParameter(0), + {b_.getInt64(0), x_loc, y_loc}), + "output_element"); + llvm_ir::IrArray output_array = GetIrArray(*hlo, *hlo); + Shape output_reduced_shape = ShapeUtil::MakeShapeWithDescendingLayout( + hlo->shape().element_type(), + kernel_info->GetKernelMappingScheme()->GetDimensionsInElements()); + // When the output_reduced_shape is a 0-2-1 transpose of the input shape, + // the 0-2-1 transpose is achieved through EmitWriteArrayElement. + output_array.CastToShape(output_reduced_shape, &b_) + .EmitWriteArrayElement(index, load_from_shmem_buffer, &b_); +} + +// Emits code to process a tensor element in a tile for the given kLoop fusion +// HLO containing parameters that are 0-2-1 transpose of its outputs. +// +// index: The index for the first output element in the normalized tensor, that +// is the resulting tensor after collapsing contiguous dimensions that play +// the same role in the transpose. +// kernel_info: Other information to support the kernel code generation. +// y_loc: The y coordinate within a tile. +// x_loc: The x coordinate within a tile. +void IrEmitterUnnested::EmitTileElementForFusion( + HloInstruction* hlo, const llvm_ir::IrArray::Index& index, + const KernelCodegenInfo* kernel_info, llvm::Value* y_loc, + llvm::Value* x_loc) { + llvm_ir::TiledParameterInfo* tiled_param_info = + kernel_info->GetTiledParameterInfo(); + std::vector output_arrays = ConstructIrArrayForOutputs(*hlo); + GpuElementalIrEmitter elem_emitter(hlo_module_config_, module_, &b_, + GetNestedComputer()); + FusedIrEmitter fused_emitter(GetGeneratorForOperandIrArrays(hlo), + &elem_emitter); + tiled_param_info->set_y(y_loc); + tiled_param_info->set_x(x_loc); + fused_emitter.SetTiledParameterInfo(tiled_param_info); + TF_CHECK_OK(hlo->fused_expression_root()->Accept(&fused_emitter)); + IrArray::Index untiled_index = + kernel_info->GetKernelMappingScheme()->GetUnnormalizedIndex( + index, output_arrays[0].GetShape()); + const llvm_ir::ElementGenerator& output_generator = + fused_emitter.GetRootGenerator(); + llvm::Value* output_value = output_generator(untiled_index).ValueOrDie(); + if (hlo->IsMultiOutputFusion()) { + DCHECK(output_value->getType()->isStructTy()); + DCHECK_EQ(output_value->getType()->getStructNumElements(), + output_arrays.size()); + for (int64 i = 0; i < output_arrays.size(); ++i) { + output_arrays[i].EmitWriteArrayElement( + untiled_index, ExtractValue(output_value, i), &b_); + } + } else { + output_arrays[0].EmitWriteArrayElement(untiled_index, output_value, &b_); + } +} + +// Information to support the code generation for a tiled reduction kernel. +using AddressVector = InlinedVector; +class ReductionCodegenInfo : public IrEmitterUnnested::KernelCodegenInfo { + public: + explicit ReductionCodegenInfo(llvm_ir::KernelMappingScheme* mapping_scheme, + bool is_row_reduction) + : KernelCodegenInfo(mapping_scheme), + current_output_linear_index_address_(nullptr), + current_output_inbound_address_(nullptr), + is_row_reduction_(is_row_reduction) {} + + void SetCurrentOutputLinearIndexAddress(llvm::AllocaInst* a) { + current_output_linear_index_address_ = a; + } + // Returns the address of the memory that stores the linear index of the + // current output. Since we are processing reduction to contiguous physical + // dimensions, this linear index is the linear index of the 1D output array. + llvm::AllocaInst* GetCurrentOutputLinearIndexAddress() const { + return current_output_linear_index_address_; + } + + void SetCurrentOutputInboundAddress(llvm::AllocaInst* a) { + current_output_inbound_address_ = a; + } + + llvm::AllocaInst* GetCurrentOutputInboundAddress() const { + return current_output_inbound_address_; + } + + AddressVector* GetMutablePartialResultAddresses() { + return &partial_result_addresses_; + } + const AddressVector& GetPartialResultAddresses() const { + return partial_result_addresses_; + } + + AddressVector* GetMutableReductionInputAddresses() { + return &reduction_input_addresses_; + } + const AddressVector& GetReductionInputAddresses() const { + return reduction_input_addresses_; + } + + InlinedVector* GetMutableReducers() { return &reducers_; } + const InlinedVector& GetReducers() const { + return reducers_; + } + int GetNumberOfReduces() const { return reducers_.size(); } + + InlinedVector* GetMutableReductionOutputShapeIndices() { + return &reduction_output_shape_indices_; + } + const InlinedVector& GetReductionOutputShapeIndices() const { + return reduction_output_shape_indices_; + } + + bool IsRowReduction() const { return is_row_reduction_; } + + // Return the dimension that is being reduced between DimX and DimY. + int GetReducedDimensionEnum() const { + return IsRowReduction() ? llvm_ir::KernelMappingScheme::DimX + : llvm_ir::KernelMappingScheme::DimY; + } + + // Return the dimension that is being ketp between DimX and DimY. + int GetKeptDimensionEnum() const { + return IsRowReduction() ? llvm_ir::KernelMappingScheme::DimY + : llvm_ir::KernelMappingScheme::DimX; + } + + private: + AddressVector partial_result_addresses_; + AddressVector reduction_input_addresses_; + InlinedVector reducers_; + InlinedVector reduction_output_shape_indices_; + llvm::AllocaInst* current_output_linear_index_address_; + llvm::AllocaInst* current_output_inbound_address_; + bool is_row_reduction_; +}; + +namespace { +// Returns a group of instructions that generate the output for the kernel +// containing the given HLO instruction. The result may be an unnested kReduce +// HLO, a nested kReduce HLO of a kInput fusion, or the operands of the tuple +// for a multiple output fusion. +absl::Span GetOutputInstructions( + HloInstruction* const* reduce_or_tuple_pointer) { + HloOpcode opcode = (*reduce_or_tuple_pointer)->opcode(); + CHECK(opcode == HloOpcode::kReduce || opcode == HloOpcode::kTuple); + return opcode == HloOpcode::kTuple + ? (*reduce_or_tuple_pointer)->operands() + : absl::Span(reduce_or_tuple_pointer, 1); +} + +const HloInstruction* GetFirstReduceInstruction( + absl::Span instructions) { + auto first_reduce_iter = + absl::c_find_if(instructions, [](const HloInstruction* inst) { + return inst->opcode() == HloOpcode::kReduce; + }); + CHECK_NE(first_reduce_iter, instructions.end()); + return *first_reduce_iter; +} + +}; // namespace + +void IrEmitterUnnested::EmitPrologueForOneReduction( + HloInstruction* unnested_hlo, HloInstruction* reduce_inst, int reduce_idx, + KernelCodegenInfo* kernel_info, GpuElementalIrEmitter* elemental_emitter, + ShapeIndex output_shape_index) { + ReductionCodegenInfo* reduction_info = + static_cast(kernel_info); + + InlinedVector* reducers = + reduction_info->GetMutableReducers(); + CHECK(IsReductionToVector(*reduce_inst)); + reducers->push_back(reduce_inst->to_apply()); + + InlinedVector* reduction_output_shape_indices = + reduction_info->GetMutableReductionOutputShapeIndices(); + reduction_output_shape_indices->push_back(std::move(output_shape_index)); + + AddressVector* reduction_input_addresses = + reduction_info->GetMutableReductionInputAddresses(); + llvm::Type* element_type = llvm_ir::PrimitiveTypeToIrType( + reduce_inst->shape().element_type(), ir_emitter_context_->llvm_module()); + llvm::AllocaInst* reduction_input_address = Alloca(element_type); + reduction_input_addresses->push_back(reduction_input_address); + + AddressVector* partial_result_addresses = + reduction_info->GetMutablePartialResultAddresses(); + llvm::AllocaInst* partial_result_address = + Alloca(element_type, /*ArraySize=*/nullptr, + "partial_reduction_result." + llvm::Twine(reduce_idx)); + partial_result_addresses->push_back(partial_result_address); + + // Initialize the partial result with the initial value of the reduction. + llvm::Value* init_ir_value; + if (unnested_hlo->opcode() == HloOpcode::kFusion) { + HloInstruction* init_value_operand = reduce_inst->mutable_operand(1); + FusedIrEmitter fused_emitter(GetGeneratorForOperandIrArrays(unnested_hlo), + elemental_emitter); + + TF_CHECK_OK(init_value_operand->Accept(&fused_emitter)); + init_ir_value = + fused_emitter + .GetGenerator(init_value_operand)(IrArray::Index(b_.getInt32Ty())) + .ValueOrDie(); + } else { + const HloInstruction* init_value = unnested_hlo->operand(1); + init_ir_value = + GetIrArray(*init_value, *unnested_hlo) + .EmitReadArrayElement(IrArray::Index(b_.getInt32Ty()), &b_); + } + + Store(init_ir_value, partial_result_address); +} + +void IrEmitterUnnested::EmitPrologueForReduction( + HloInstruction* unnested_hlo, KernelCodegenInfo* kernel_info) { + VLOG(10) << "Emit prologue for reduction " << unnested_hlo->ToString(); + // Find the unnested kReduce or the tuple that contains a list of kReduce. + HloInstruction* reduce_or_tuple = unnested_hlo->opcode() == HloOpcode::kFusion + ? unnested_hlo->fused_expression_root() + : unnested_hlo; + absl::Span output_instructions = + GetOutputInstructions(&reduce_or_tuple); + ReductionCodegenInfo* reduction_info = + static_cast(kernel_info); + GpuElementalIrEmitter elemental_emitter(hlo_module_config_, + ir_emitter_context_->llvm_module(), + &b_, GetNestedComputer()); + const HloInstruction* first_reduce = nullptr; + for (int i = 0, e = output_instructions.size(); i != e; ++i) { + if (output_instructions[i]->opcode() != HloOpcode::kReduce) { + continue; + } + HloInstruction* reduce_inst = output_instructions[i]; + if (first_reduce == nullptr) { + first_reduce = reduce_inst; + } else { + CHECK(first_reduce->dimensions() == reduce_inst->dimensions()); + } + ShapeIndex output_shape_index; + if (reduce_or_tuple->opcode() == HloOpcode::kTuple) { + output_shape_index = {i}; + } + + EmitPrologueForOneReduction(unnested_hlo, reduce_inst, i, kernel_info, + &elemental_emitter, + std::move(output_shape_index)); + } + + // Allocate stack storage to store the current output linear index and record + // the address of the storage. + reduction_info->SetCurrentOutputLinearIndexAddress( + Alloca(reduction_info->GetIndexType())); + + if (!reduction_info->IsRowReduction()) { + llvm::Type* bool_ty = b_.getInt1Ty(); + llvm::AllocaInst* output_inbound_addr = Alloca(bool_ty); + Store(llvm::ConstantInt::get(bool_ty, 0), output_inbound_addr); + reduction_info->SetCurrentOutputInboundAddress(output_inbound_addr); + } +} + +void IrEmitterUnnested::EmitFullWarpShuffleDownLoopForAllReduces( + const InlinedVector& reducers, + const AddressVector& partial_result_addresses) { + for (int distance = 16; distance >= 1; distance /= 2) { + for (int i = 0; i != reducers.size(); ++i) { + llvm::Type* element_type = + partial_result_addresses[i]->getType()->getElementType(); + int bit_width = llvm_ir::GetSizeInBits(element_type); + llvm::Value* result_from_other_lane = Alloca( + element_type, nullptr, "result_from_other_lane" + llvm::Twine(i)); + // Bitcast cannot be applied to aggregate types (even packed ones), so + // we bitcast addresses of load/store to intN* of the same bit-width. + llvm::Type* shuffled_value_type = + element_type->isStructTy() ? b_.getIntNTy(bit_width) : element_type; + auto convert_pointer_for_shuffle = [&](llvm::Value* ptr) { + return BitCast(ptr, shuffled_value_type->getPointerTo()); + }; + llvm::Value* partial_result = + Load(convert_pointer_for_shuffle(partial_result_addresses[i]), + "partial_reduction_result"); + Store(EmitFullWarpShuffleDown(partial_result, b_.getInt32(distance), &b_), + convert_pointer_for_shuffle(result_from_other_lane)); + TF_CHECK_OK(EmitCallToNestedComputation( + *reducers[i], {partial_result_addresses[i], result_from_other_lane}, + partial_result_addresses[i])); + } + } +} + +void IrEmitterUnnested::EmitEpilogueForReduction( + HloInstruction* unnested_hlo, KernelCodegenInfo* kernel_info) { + ReductionCodegenInfo* reduction_info = + static_cast(kernel_info); + int num_reduces = reduction_info->GetNumberOfReduces(); + const AddressVector& partial_result_addresses = + reduction_info->GetPartialResultAddresses(); + const InlinedVector& reducers = + reduction_info->GetReducers(); + const InlinedVector& reduction_output_shape_indices = + reduction_info->GetReductionOutputShapeIndices(); + + if (reduction_info->IsRowReduction()) { + EmitFullWarpShuffleDownLoopForAllReduces(reducers, + partial_result_addresses); + llvm::Value* lane_id = reduction_info->GetLaneId(); + llvm_ir::LlvmIfData if_lane_id_is_zero_data = llvm_ir::EmitIfThenElse( + ICmpEQ(lane_id, llvm::ConstantInt::get(lane_id->getType(), 0)), + "lane_id_is_zero", &b_); + llvm_ir::SetToFirstInsertPoint(if_lane_id_is_zero_data.true_block, &b_); + } else { + llvm::Value* output_inbound_addr = + reduction_info->GetCurrentOutputInboundAddress(); + llvm::Value* output_inbound = Load(output_inbound_addr); + llvm_ir::LlvmIfData if_output_inbound_data = llvm_ir::EmitIfThenElse( + ICmpEQ(output_inbound, + llvm::ConstantInt::get(output_inbound->getType(), 1)), + "output_inbound", &b_); + llvm_ir::SetToFirstInsertPoint(if_output_inbound_data.true_block, &b_); + } + + // Emit an atomic operation that accumulates the partial reduction to the + // output element. For row reduction, this is only for lane 0 due to the + // if-statement emitted above. + for (int i = 0; i != num_reduces; ++i) { + IrArray::Index element_index( + /*linear=*/Load(reduction_info->GetCurrentOutputLinearIndexAddress(), + "output_linear_addr"), + ShapeUtil::GetSubshape(unnested_hlo->shape(), + reduction_output_shape_indices[i]), + &b_); + llvm::Value* output_address = + GetIrArray(*unnested_hlo, *unnested_hlo, + reduction_output_shape_indices[i]) + .EmitArrayElementAddress(element_index, &b_, + "output_element_address"); + // Do not emit atomic operations if each element in the reduction result is + // computed by one block, that is the dimension being reduced has only one + // block. + const llvm_ir::KernelMappingScheme* mapping_scheme = + reduction_info->GetKernelMappingScheme(); + if (mapping_scheme->GetTileBlockSizeForDimension( + llvm_ir::KernelMappingScheme::DimZ) == 1 && + mapping_scheme->GetTileBlockSizeForDimension( + reduction_info->GetReducedDimensionEnum()) == 1) { + TF_CHECK_OK(EmitCallToNestedComputation( + *reducers[i], {output_address, partial_result_addresses[i]}, + output_address)); + } else { + TF_CHECK_OK(EmitAtomicOperationForNestedComputation( + *reducers[i], output_address, partial_result_addresses[i])); + } + } +} + +void IrEmitterUnnested::EmitTileElementForReduction( + HloInstruction* unnested_hlo, const llvm_ir::IrArray::Index& index, + const KernelCodegenInfo* kernel_info, llvm::Value* y_loc, + llvm::Value* x_loc) { + VLOG(10) << "Emit tile element for reduce " << unnested_hlo->ToString(); + HloInstruction* reduce_or_tuple = unnested_hlo->opcode() == HloOpcode::kFusion + ? unnested_hlo->fused_expression_root() + : unnested_hlo; + llvm_ir::TiledParameterInfo* tiled_param_info = + kernel_info->GetTiledParameterInfo(); + tiled_param_info->set_y(y_loc); + tiled_param_info->set_x(x_loc); + + // Record the linear address for the current reduction. + const ReductionCodegenInfo* reduction_info = + dynamic_cast(kernel_info); + Store(index[reduction_info->GetKeptDimensionEnum()], + reduction_info->GetCurrentOutputLinearIndexAddress()); + if (!reduction_info->IsRowReduction()) { + llvm::Type* bool_ty = b_.getInt1Ty(); + llvm::AllocaInst* output_inbound_addr = + reduction_info->GetCurrentOutputInboundAddress(); + Store(llvm::ConstantInt::get(bool_ty, 1), output_inbound_addr); + } + + InlinedVector input_gens; + std::vector> + extra_output_gens; + GpuElementalIrEmitter elem_emitter(hlo_module_config_, module_, &b_, + GetNestedComputer()); + FusedIrEmitter fused_emitter(GetGeneratorForOperandIrArrays(unnested_hlo), + &elem_emitter); + absl::Span output_instructions = + GetOutputInstructions(&reduce_or_tuple); + // Construct the ElementGenerator for each reduction and extra output in the + // the group of output instructions. + if (unnested_hlo->opcode() == HloOpcode::kFusion) { + fused_emitter.SetTiledParameterInfo(tiled_param_info); + TF_CHECK_OK(unnested_hlo->fused_expression_root()->Accept(&fused_emitter)); + + for (int i = 0, e = output_instructions.size(); i != e; ++i) { + const HloInstruction* inst = output_instructions[i]; + ShapeIndex output_shape_index; + if (reduce_or_tuple->opcode() == HloOpcode::kTuple) { + output_shape_index = {i}; + } + if (inst->opcode() == HloOpcode::kReduce) { + input_gens.push_back(fused_emitter.GetGenerator(inst->operand(0))); + } else { + extra_output_gens.emplace_back(fused_emitter.GetGenerator(inst), + std::move(output_shape_index)); + } + } + } else { + input_gens.push_back([&](const IrArray::Index& index) { + return GetIrArray(*unnested_hlo->operand(0), *unnested_hlo) + .EmitReadArrayElement(index, &b_); + }); + } + + IrArray::Index input_index = + reduction_info->GetKernelMappingScheme()->GetUnnormalizedIndex( + index, + GetFirstReduceInstruction(output_instructions)->operand(0)->shape()); + const AddressVector& partial_reduction_result_addresses = + reduction_info->GetPartialResultAddresses(); + const AddressVector& reduction_input_addresses = + reduction_info->GetReductionInputAddresses(); + const InlinedVector& reducers = + reduction_info->GetReducers(); + + // Emit code to generate the input and perform the reduction computation for + // each reduction instruction. + for (int i = 0; i != reducers.size(); ++i) { + llvm::Value* const input_ir_value = input_gens[i](input_index).ValueOrDie(); + Store(input_ir_value, reduction_input_addresses[i]); + TF_CHECK_OK(EmitCallToNestedComputation( + *reducers[i], + {partial_reduction_result_addresses[i], reduction_input_addresses[i]}, + partial_reduction_result_addresses[i])); + } + + // Emit code to generate the output for the non-reduction instructions in the + // fusion, if any. + TF_CHECK_OK( + EmitExtraOutputsForReduce(unnested_hlo, input_index, extra_output_gens)); +} + +// Emits a kernel for the hlo instruction using the given tiling scheme. +void IrEmitterUnnested::EmitBlock(const TileGenerator& emit_one_tile, + const KernelCodegenInfo* kernel_info, + KernelSupportLibrary& ksl, + llvm::Type* index_ty) { + KernelMappingScheme* mapping_scheme = kernel_info->GetKernelMappingScheme(); + absl::Span dims_in_tile = mapping_scheme->GetDimensionsInTiles(); + absl::Span dims_in_block = + mapping_scheme->GetDimensionsInBlocks(); + absl::Span block_sizes = mapping_scheme->GetBlockSizes(); + auto index_typed_constant = [&](uint64 c) -> llvm::Constant* { + return llvm::ConstantInt::get(index_ty, c); + }; + + // Emit all the tiles for a given dimension in a tile block. + auto emit_tiles_for_block_dim = + [&](const string& loop_name, const IrArray::Index& starting_tile, + int dim_id, + const std::function + emit_next_block_dim) { + if (block_sizes[dim_id] == 1) { + emit_next_block_dim(starting_tile); + } else { + llvm::Value* starting_tile_index_for_dim = starting_tile[dim_id]; + llvm::Value* block_size_for_dim = + index_typed_constant(block_sizes[dim_id]); + llvm::Value* block_id_for_dim = + b_.CreateUDiv(starting_tile_index_for_dim, block_size_for_dim); + llvm::Value* last_block_for_dim = + index_typed_constant(dims_in_block[dim_id] - 1); + llvm::Value* last_block_size_for_dim = index_typed_constant( + dims_in_tile[dim_id] - + (dims_in_block[dim_id] - 1) * block_sizes[dim_id]); + llvm::Value* num_tiles_in_block = + Select(ICmpEQ(last_block_for_dim, block_id_for_dim), + last_block_size_for_dim, block_size_for_dim); + + ksl.ForReturnVoid( + loop_name, + /*start=*/index_typed_constant(0), + /*end=*/num_tiles_in_block, + /*step=*/1, [&](llvm::Value* block_dim_induction_var) { + IrArray::Index tile_index = starting_tile.AddOffsetToDim( + block_dim_induction_var, dim_id, &b_); + emit_next_block_dim(tile_index); + }); + } + }; + + absl::Span reduced_dims = + mapping_scheme->GetDimensionsInElements(); + const bool block_contains_multi_tiles = + mapping_scheme->GetNumberOfTilesInOneBlock() > 1; + + // Emit the tile with a given tile_index, by calculating the tight bounds for + // each dimension of the tile and then calling emit_one_tile. + auto emit_one_tile_for_tile_index = [&](const IrArray::Index& tile_index) { + std::vector output_tile_bounds(3); + for (int i = KernelMappingScheme::DimY; i < KernelMappingScheme::DimTot; + ++i) { + int64 tile_size_for_dim = mapping_scheme->GetTileSizeForDimension(i); + // Only last row or column may not have full size. + llvm::Value* is_last_row = + ICmpEQ(tile_index[i], index_typed_constant(dims_in_tile[i] - 1)); + int64 partial_row_size = + reduced_dims[i] - (dims_in_tile[i] - 1) * tile_size_for_dim; + output_tile_bounds[i] = + Select(is_last_row, index_typed_constant(partial_row_size), + index_typed_constant(tile_size_for_dim), "tile_bound"); + } + + IrArray::Index tile_origin = + mapping_scheme->GetElementIndexForTileOrigin(tile_index); + emit_one_tile(tile_origin, output_tile_bounds, block_contains_multi_tiles); + }; + + const IrArray::Index starting_block = + mapping_scheme->EmitBlockIndex(index_ty); + const IrArray::Index starting_tile_for_dim_z = + mapping_scheme->GetTileIndexForBlockOrigin(starting_block); + + // Emit the three dimensional block of tiles. + emit_tiles_for_block_dim( + "block_dim_z", starting_tile_for_dim_z, KernelMappingScheme::DimZ, + [&](const IrArray::Index& starting_tile_for_dim_y) { + emit_tiles_for_block_dim( + "block_dim_y", starting_tile_for_dim_y, KernelMappingScheme::DimY, + [&](const IrArray::Index& starting_tile_for_dim_x) { + emit_tiles_for_block_dim("block_dim_x", starting_tile_for_dim_x, + KernelMappingScheme::DimX, + emit_one_tile_for_tile_index); + }); + }); +} + +// Emits a kernel for the hlo instruction using the given kernel mapping scheme. +// +// unnested_hlo: The unnested hlo instruction for which the kernel is generated. +// Currently, these hlo instructions are supported: kLoop fusion, kCopy. +// tiled_param_ids: The IDs for the parameters that are 0-2-1 transpose of +// other tensors with the same dimensions and need to be tiled and tranposed. +// mapping_scheme: The tiling scheme to use. +// kernel_generator: Contains function objects for code generation, such as +// element generator, block prologue and epilogue generators. +// kernel_info: Represent other information to support the code generation +// of the tiled kernel for the hlo. +LaunchDimensions IrEmitterUnnested::EmitKernel( + HloInstruction* unnested_hlo, absl::Span tiled_param_ids, + const KernelCodeGenerator& kernel_generator, + KernelCodegenInfo* kernel_info) { + KernelMappingScheme* mapping_scheme = kernel_info->GetKernelMappingScheme(); + + std::vector param_arrays = ConstructIrArrayForInputs(*unnested_hlo); + int64 num_params = param_arrays.size(); + // Allocate shared memory buffers to store the tiled inputs. + std::vector param_shmem_buffers(num_params, nullptr); + for (int64 id : tiled_param_ids) { + const HloInstruction* param = unnested_hlo->operand(id); + param_shmem_buffers[id] = + mapping_scheme->GetSharedMemoryBufferForElementType( + llvm_ir::PrimitiveTypeToIrType(param->shape().element_type(), + module_), + IrName(unnested_hlo, StrCat("tile", id))); + VLOG(3) << "Added shmem buffer for parameter " << id << ": " + << llvm_ir::DumpToString(*param_shmem_buffers[id]); + } + + LaunchDimensions launch_dimensions = LaunchDimensions( + mapping_scheme->GetNumberOfBlocks(), mapping_scheme->GetThreadsPerTile()); + llvm::Type* index_ty = GetIndexTypeForKernel( + unnested_hlo, launch_dimensions.launch_bound(), &b_); + auto index_typed_constant = [&](uint64 c) -> llvm::Constant* { + return llvm::ConstantInt::get(index_ty, c); + }; + + // For each tiled parameter, cast its input IrArray to the corresponding + // reduced shape and keep the reduced shape live during IR emission. + std::vector param_in_reduced_shape_arrays; + std::vector param_reduced_shapes; + absl::Span reduced_dims = + mapping_scheme->GetDimensionsInElements(); + int num_shapes = ConstructInputReducedShapeAndCastInputIrArrayToShape( + *unnested_hlo, param_arrays, param_shmem_buffers, reduced_dims, + ¶m_reduced_shapes, ¶m_in_reduced_shape_arrays); + DCHECK_EQ(num_shapes, num_params); + + // Calculate the starting element coordinate within a tile for the current + // thread, (y, x) from thread_id. + llvm::Value* x; + llvm::Value* y; + std::tie(y, x) = mapping_scheme->EmitThreadYXCoordinate(index_ty); + + kernel_info->SetLaneId( + mapping_scheme->GetNumberOfThreadsForDimensionX() == kWarpSize ? x + : nullptr); + kernel_info->SetIndexType(index_ty); + + KernelSupportLibrary ksl(&b_, llvm_ir::UnrollMode::kDefaultUnroll); + // Curry a few parameters to EmitTiledElementalCodeWithBoundsCheck. + auto emit_tiled_elemental_code_with_bounds_check = + [&](const IrArray::Index& index, const string& loop_name, + llvm::Value* tile_height, llvm::Value* tile_width, + const std::function& emit_elem_function) { + EmitTiledElementalCodeWithBoundsCheck(mapping_scheme, index, loop_name, + &ksl, &b_, y, x, tile_height, + tile_width, emit_elem_function); + }; + + auto emit_one_tile = [&](const IrArray::Index& output_tile_origin, + absl::Span output_tile_bounds, + bool block_contains_multi_tiles) { + // Calculate the input tile origin from the output tile origin. + const IrArray::Index input_tile_origin( + Permute({0, 2, 1}, output_tile_origin.multidim())); + + const IrArray::Index input_index = + input_tile_origin.AddOffsetToDim(x, KernelMappingScheme::DimX, &b_) + .AddOffsetToDim(y, KernelMappingScheme::DimY, &b_); + + // If shared memory transpose is needed, wait for all threads to reach this + // point, lest we copy a value from tile to output before the other thread + // copies it from input to tile. This is `__syncthreads` in CUDA. + if (!tiled_param_ids.empty()) { + // Copy input parameter values to shared memory buffers: + // tile[y, x] = input[index] + // Note that tile_width and tile_height are flipped here because we are + // reading a transposed tile. + emit_tiled_elemental_code_with_bounds_check( + input_index, "input", output_tile_bounds[2], output_tile_bounds[1], + [&](const IrArray::Index& index, llvm::Value* y_loc, + llvm::Value* x_loc) { + for (int64 id : tiled_param_ids) { + IrArray& input_in_logical_shape = + param_in_reduced_shape_arrays[id]; + llvm::Value* shmem_buffer = param_shmem_buffers[id]; + // TODO(jlebar): Add AA metadata to this store. Tile buffers are + // global variables, so LLVM can't infer much about it. + Store(input_in_logical_shape.EmitReadArrayElement( + index, &b_, "input_element"), + GEP(shmem_buffer, {index_typed_constant(0), y_loc, x_loc})); + } + }); + + // Wait for all threads to reach this point using `__syncthreads` in CUDA. + llvm_ir::EmitCallToIntrinsic(llvm::Intrinsic::nvvm_barrier0, {}, {}, &b_); + } + + llvm_ir::TiledParameterInfo tiled_param_info(param_shmem_buffers, y, x); + kernel_info->SetTiledParamInfo(&tiled_param_info); + + const IrArray::Index output_index = + output_tile_origin.AddOffsetToDim(x, KernelMappingScheme::DimX, &b_) + .AddOffsetToDim(y, KernelMappingScheme::DimY, &b_); + + // Write to output[index] by emitting code like normal, except that values + // for the tiled parameters are read from the shmem buffers. + emit_tiled_elemental_code_with_bounds_check( + output_index, "output", output_tile_bounds[1], output_tile_bounds[2], + [&](const IrArray::Index& index, llvm::Value* y_loc, + llvm::Value* x_loc) { + kernel_generator.GetTileElementGenerator()(unnested_hlo, index, + kernel_info, y_loc, x_loc); + }); + + // If a tile block contains multiple tiles and shared memory buffers are + // used, we need to wait for all threads to finish using the shared memory + // buffer for the current tile before we move on to process the next tile + // and overwrite the shared memory buffers. + if (block_contains_multi_tiles && !tiled_param_ids.empty()) { + llvm_ir::EmitCallToIntrinsic(llvm::Intrinsic::nvvm_barrier0, {}, {}, &b_); + } + }; + + const BlockPrologueGenerator& block_prologue_generator = + kernel_generator.GetBlockPrologueGenerator(); + if (block_prologue_generator) { + block_prologue_generator(unnested_hlo, kernel_info); + } + + EmitBlock(std::move(emit_one_tile), kernel_info, ksl, index_ty); + + const BlockEpilogueGenerator& block_epilogue_generator = + kernel_generator.GetBlockEpilogueGenerator(); + if (block_epilogue_generator) { + block_epilogue_generator(unnested_hlo, kernel_info); + } + + // For multioutput fusion, emit a tuple with pointers to all the individual + // outputs. + if (unnested_hlo->IsMultiOutputFusion()) { + std::vector output_arrays = + ConstructIrArrayForOutputs(*unnested_hlo); + llvm_ir::EmitTuple(GetIrArray(*unnested_hlo, *unnested_hlo), output_arrays, + &b_, module_); + } + + return launch_dimensions; +} + // Emits a kernel for the given hlo instruction using a tiled 0-2-1 transpose // algorithm to improve the memory access patterns for the input parameters -// which have a shape that is a 0-2-1 transpose of the output tensors. +// with a shape that is a 0-2-1 transpose of the output tensor shape. // // For the purpose of tiling, the output tensors have a logical shape of three -// components 0-2-1 while the relevant input parameters have a logical shape of -// three components 0-1-2 in the order major to minor. The x- and y- dimensions -// of the tensors are tiled in square tiles of edge length `kTileSize`. Each -// thread block of `kTileSize` x `kNumRows` threads transposes one tile: each -// thread copies kTileSize/kNumRows elements from the input to a shared memory -// tile, then the otherwise "regular hlo kernel" reads from the shared memory -// instead of the original input. +// components 0-2-1 while the relevant input parameters have a logical shape +// of three components 0-1-2 in the order major to minor. The x- and y- +// dimensions of the tensors are tiled in square tiles with an edge length +// `kTileSize`. Each thread block of `kTileSize` x `kNumRows` threads +// transposes one tile: each thread copies kTileSize/kNumRows elements from +// the input to a shared memory tile, then the otherwise "regular HLO kernel" +// reads from the shared memory instead of the original input. // // This is similar to the following CUDA algorithm in TensorFlow: // https://goo.gl/MStRV6. @@ -3313,219 +3016,37 @@ void EmitTiledElementalCodeWithBoundsCheck( // `kTileSize` should usually be same as warp size. We currently choose 32 for // `kTileSize` and 4 for `kNumRows`. The CUDA algorithm uses 8 for `kNumRows`. // -// TODO(b/33320379): Here each block transposes 1 tile. It may be more efficient -// to launch fewer blocks so each transposes many tiles. +// TODO(b/33320379): Here each block transposes 1 tile. It may be more +// efficient to launch fewer blocks so each transposes many tiles. LaunchDimensions IrEmitterUnnested::EmitHlo021Tile( HloInstruction* hlo, absl::Span reduced_output_dims, absl::Span tiled_param_ids) { - // Parameters for the tiling algorithm. - constexpr int64 kTileSize = 32; - constexpr int64 kNumRows = 4; - constexpr int64 kThreadsPerTile = kTileSize * kNumRows; - - // Construct IrArrays for the inputs and outputs. - std::vector output_arrays = ConstructIrArrayForOutputs(*hlo); - int64 num_outputs = output_arrays.size(); - std::vector param_arrays = ConstructIrArrayForInputs(*hlo); - int64 num_params = param_arrays.size(); - - // Allocate shared memory buffers to store the tiled inputs. - std::vector param_shmem_buffers(num_params, nullptr); - for (int64 id : tiled_param_ids) { - const HloInstruction* param = hlo->operand(id); - // Add 1 to the minor dimension to reduce shared memory bank conflicts. - llvm::Type* tile_type = llvm::ArrayType::get( - llvm::ArrayType::get(llvm_ir::PrimitiveTypeToIrType( - param->shape().element_type(), module_), - kTileSize + 1), - kTileSize); - auto* tile_base_ptr = llvm_ir::AllocateSharedMemoryTile( - b_.GetInsertBlock()->getParent()->getParent(), tile_type, - IrName(hlo, StrCat("tile", id))); - param_shmem_buffers[id] = tile_base_ptr; - VLOG(3) << "Added shmem buffer for parameter " << id << ": " - << llvm_ir::DumpToString(*tile_base_ptr); - } - - // The 0-2-1 shape of the tiling scheme is the reduced shape of the HLO result - // for the purpose of tiling. Calculate the logical output dimensions in the - // tile from the reduced output dimensions. - std::vector output_dims_in_tiles = std::vector( - reduced_output_dims.begin(), reduced_output_dims.end()); - CHECK_EQ(output_dims_in_tiles.size(), 3); - for (int i = 1; i < 3; ++i) { - output_dims_in_tiles[i] = - CeilOfRatio(output_dims_in_tiles[i], kTileSize); - } - const int64 num_tiles = - absl::c_accumulate(output_dims_in_tiles, 1, std::multiplies()); - LaunchDimensions launch_dimensions(num_tiles, kThreadsPerTile); - - llvm::Type* index_ty = - GetIndexTypeForKernel(hlo, launch_dimensions.launch_bound(), &b_); - auto index_typed_constant = [&](uint64 c) -> llvm::Constant* { - return llvm::ConstantInt::get(index_ty, c); - }; - - // Cast each output IrArray to its corresponding reduced shape and keep the - // reduced shape live during IR emission. - std::vector output_in_reduced_shape_arrays; - std::vector output_reduced_shapes; - CHECK_EQ(ConstructOutputReducedShapeAndCastOutputIrArrayToShape( - *hlo, output_arrays, reduced_output_dims, &output_reduced_shapes, - &output_in_reduced_shape_arrays), - num_outputs); - - // For each tiled parameter, cast its input IrArray to the corresponding - // reduced shape and keep the reduced shape live during IR emission. - std::vector param_in_reduced_shape_arrays; - std::vector param_reduced_shapes; - CHECK_EQ(ConstructInputReducedShapeAndCastInputIrArrayToShape( - *hlo, param_arrays, param_shmem_buffers, reduced_output_dims, - ¶m_reduced_shapes, ¶m_in_reduced_shape_arrays), - num_params); - - // Calculate the starting element coordinate within a tile for the current - // thread, (y, x) from thread_id. - llvm::Value* x; - llvm::Value* y; - std::tie(y, x) = CalculateYXCoordinateWithinTile( - &b_, index_typed_constant(kTileSize), kThreadsPerTile); - - // Calculate the index for the current output tile from block_id. - const IrArray::Index output_tile_index( - GetBlockIdx(&b_, index_ty, num_tiles), - ShapeUtil::MakeShapeWithDescendingLayout(PRED /*arbitrary*/, - output_dims_in_tiles), - &b_); - - // Output tile origin is the index for the first element of the current output - // tile. - const IrArray::Index output_tile_origin = [&] { - IrArray::Index index = output_tile_index; - for (int i = 1; i < 3; ++i) { - index[i] = Mul(output_tile_index[i], index_typed_constant(kTileSize), - "tile_origin." + std::to_string(i)); - } - return index; - }(); - - // Calculate the input tile origin from the output tile origin. - const IrArray::Index input_tile_origin( - Permute({0, 2, 1}, output_tile_origin.multidim())); - - // Calculate the current output tile bounds in each of the logical dimensions. - std::vector output_tile_bounds(3); - for (int i = 1; i < 3; ++i) { - // Only last row or column may not have full size. - output_tile_bounds[i] = - Select(ICmpEQ(output_tile_index[i], - index_typed_constant(output_dims_in_tiles[i] - 1)), - index_typed_constant(reduced_output_dims[i] - - (output_dims_in_tiles[i] - 1) * kTileSize), - index_typed_constant(kTileSize), "kTileSize"); - } - - KernelSupportLibrary ksl(&b_, llvm_ir::UnrollMode::kDefaultUnroll); - - // Curry a few parameters to EmitTiledElementalCodeWithBoundsCheck. - auto emit_tiled_elemental_code_with_bounds_check = - [&](const IrArray::Index& index, const string& loop_name, - llvm::Value* tile_width, llvm::Value* tile_height, - const std::function& - emit_elem_function) { - EmitTiledElementalCodeWithBoundsCheck( - kTileSize, kNumRows, index, loop_name, &ksl, &b_, y, x, tile_width, - tile_height, emit_elem_function); - }; - - // Adds `addend` to the given `dim` of `index`. - auto offset_dim = [&](IrArray::Index index, llvm::Value* addend, int64 dim) { - index[dim] = Add(index[dim], addend); - return index; - }; - const IrArray::Index input_index = - offset_dim(offset_dim(input_tile_origin, x, /*dim=*/2), y, /*dim=*/1); - - // Copy input parameter values to shared memory buffers: - // tile[y, x] = input[index] - emit_tiled_elemental_code_with_bounds_check( - input_index, "input", output_tile_bounds[1], output_tile_bounds[2], - [&](const IrArray::Index& index, llvm::Value* y_loc) { - for (int64 id : tiled_param_ids) { - IrArray& input_in_logical_shape = param_in_reduced_shape_arrays[id]; - llvm::Value* shmem_buffer = param_shmem_buffers[id]; - // TODO(jlebar): Add AA metadata to this store. Tile buffers are - // global variables, so LLVM can't infer much about it. - Store(input_in_logical_shape.EmitReadArrayElement(index, &b_, - "input_element"), - GEP(shmem_buffer, {index_typed_constant(0), y_loc, x})); - } - }); - - // Wait for all threads to reach this point, lest we copy a value from tile to - // output before the other thread copies it from input to tile. - // This is `__syncthreads` in CUDA. - llvm_ir::EmitCallToIntrinsic(llvm::Intrinsic::nvvm_barrier0, {}, {}, &b_); - - llvm_ir::TiledParameterInfo tiled_param_info(param_shmem_buffers, y, x); - - const IrArray::Index output_index = - offset_dim(offset_dim(output_tile_origin, x, /*dim=*/2), y, /*dim=*/1); - - // Write to output[index] by emitting code like normal, except that values for - // the tiled parameters are read from the shmem buffers. + constexpr int kNumRows = 4; + KernelMappingScheme mapping_scheme( + reduced_output_dims, /*tile_size_y=*/kWarpSize, + /*tile_size_x=*/kWarpSize, /*req_block_sizes=*/{1, 1, 1}, + /*num_threads_y=*/kNumRows, + /*num_threads_x=*/kWarpSize, &b_); + TileElementGenerator element_generator; if (hlo->opcode() == HloOpcode::kCopy) { - emit_tiled_elemental_code_with_bounds_check( - output_index, "output", output_tile_bounds[2], output_tile_bounds[1], - [&](const IrArray::Index& index, llvm::Value* y_loc) { - // TODO(jlebar): Add AA metadata to this load. - llvm::Instruction* load_from_shmem_buffer = - Load(GEP(param_shmem_buffers[0], {b_.getInt64(0), x, y_loc}), - "output_element"); - output_in_reduced_shape_arrays[0].EmitWriteArrayElement( - index, load_from_shmem_buffer, &b_); - }); + element_generator = [&](HloInstruction* hlo, + const llvm_ir::IrArray::Index& index, + const KernelCodegenInfo* kernel_info, + llvm::Value* y_loc, llvm::Value* x_loc) { + EmitTileElementForCopy(hlo, index, kernel_info, y_loc, x_loc); + }; } else { - CHECK_EQ(hlo->opcode(), HloOpcode::kFusion); - emit_tiled_elemental_code_with_bounds_check( - output_index, "output", output_tile_bounds[2], output_tile_bounds[1], - [&](const IrArray::Index& index, llvm::Value* y_loc) { - GpuElementalIrEmitter elem_emitter(hlo_module_config_, module_, &b_, - GetNestedComputer()); - FusedIrEmitter fused_emitter(GetGeneratorForOperandIrArrays(hlo), - &elem_emitter); - tiled_param_info.set_y(y_loc); - fused_emitter.SetTiledParameterInfo(&tiled_param_info); - TF_CHECK_OK(hlo->fused_expression_root()->Accept(&fused_emitter)); - IrArray::Index untiled_index = llvm_ir::GetUnreducedOutputIndex( - index, output_reduced_shapes[0], output_arrays[0].GetShape(), - &b_); - const llvm_ir::ElementGenerator& output_generator = - fused_emitter.GetRootGenerator(); - llvm::Value* output_value = - output_generator(untiled_index).ValueOrDie(); - if (hlo->IsMultiOutputFusion()) { - CHECK(output_value->getType()->isStructTy()); - CHECK_EQ(output_value->getType()->getStructNumElements(), - output_in_reduced_shape_arrays.size()); - for (int64 i = 0; i < output_in_reduced_shape_arrays.size(); ++i) { - output_in_reduced_shape_arrays[i].EmitWriteArrayElement( - index, ExtractValue(output_value, i), &b_); - } - } else { - output_in_reduced_shape_arrays[0].EmitWriteArrayElement( - index, output_value, &b_); - } - }); + DCHECK_EQ(hlo->opcode(), HloOpcode::kFusion); + element_generator = [&](HloInstruction* hlo, + const llvm_ir::IrArray::Index& index, + const KernelCodegenInfo* kernel_info, + llvm::Value* y_loc, llvm::Value* x_loc) { + EmitTileElementForFusion(hlo, index, kernel_info, y_loc, x_loc); + }; } - - // For multioutput fusion, emit a tuple with all the individual outputs. - if (hlo->IsMultiOutputFusion()) { - llvm_ir::EmitTuple(GetIrArray(*hlo, *hlo), output_arrays, &b_, module_); - } - - return launch_dimensions; + KernelCodegenInfo kernel_info(&mapping_scheme); + KernelCodeGenerator kernel_generator(std::move(element_generator)); + return EmitKernel(hlo, tiled_param_ids, kernel_generator, &kernel_info); } namespace { @@ -3562,8 +3083,8 @@ bool IrEmitterUnnested::CheckAndEmitHloWithTile021(HloInstruction* hlo) { ? ShapeUtil::GetSubshape(hlo->shape(), {0}) : hlo->shape(); - // If the output_shape is reduced to 021 shape, find all the parameters of the - // hlo that are in the corresponding 012 shape. + // If the output_shape is reduced to 021 shape, find all the parameters of + // the HLO that are in the corresponding 012 shape. std::vector params_012; optional> reduced_dims_021; for (int64 operand_idx = 0; operand_idx < hlo->operand_count(); @@ -3600,9 +3121,9 @@ bool IrEmitterUnnested::CheckAndEmitHloWithTile021(HloInstruction* hlo) { } // Each of our shared memory tiles has 32*33 elements (so ~4kb, if the - // elements are of size 4 bytes), and CUDA has an architectural limit of 48kb - // shared memory per SM. (This is increased to 96kb in Volta, but we don't - // use this, in part because it eats into our L1 cache space.) + // elements are of size 4 bytes), and CUDA has an architectural limit of + // 48kb shared memory per SM. (This is increased to 96kb in Volta, but we + // don't use this, in part because it eats into our L1 cache space.) // // For correctness we need to ensure that we don't make more than 48kb worth // of shmem tiles per block. And for performance, we'd probably like to use @@ -3610,9 +3131,9 @@ bool IrEmitterUnnested::CheckAndEmitHloWithTile021(HloInstruction* hlo) { // gpu core. // // We say without benchmarks that we want at least 3 threads/block, - // corresponding to 3 shmem tiles if the elements are 32 bits wide. We choose - // which params get the shmem transpose treatment arbitrarily; it's not clear - // if there's a Right Choice. + // corresponding to 3 shmem tiles if the elements are 32 bits wide. We + // choose which params get the shmem transpose treatment arbitrarily; it's + // not clear if there's a Right Choice. // // This is only sound if tiled transposes are the only place where we use // shared memory in fusions. If in the future other fusible ops use shared @@ -3645,6 +3166,246 @@ bool IrEmitterUnnested::CheckAndEmitHloWithTile021(HloInstruction* hlo) { return true; } +namespace { +// Checks that the outputs of a fusion with reduction are consistent. +Status AreFusedReductionOutputsConsistent( + absl::Span output_instructions, + const HloInstruction* first_reduce) { + for (const HloInstruction* inst : output_instructions) { + if (inst->opcode() == HloOpcode::kReduce) { + // Shapes, layouts and dimensions must be the same for all reduces + // inside of this fusion. + TF_RET_CHECK(ShapeUtil::Equal(first_reduce->shape(), inst->shape())); + TF_RET_CHECK(ShapeUtil::Equal(first_reduce->operand(0)->shape(), + inst->operand(0)->shape())); + TF_RET_CHECK(ShapeUtil::Equal(first_reduce->operand(1)->shape(), + inst->operand(1)->shape())); + TF_RET_CHECK(first_reduce->dimensions() == inst->dimensions()); + } else { + // For extra outputs we can relax shape equality to allow different + // types (with the same number of elements). Layouts still have to + // match. + TF_RET_CHECK(ShapeUtil::CompatibleIgnoringElementType( + first_reduce->operand(0)->shape(), inst->shape())); + TF_RET_CHECK(LayoutUtil::Equal(first_reduce->operand(0)->shape().layout(), + inst->shape().layout())); + } + } + return Status::OK(); +} + +// Finds the dimensions to keep for the reduction, sorts and returns the +// dimensions from minor to major. +DimensionVector GetDimensionsToKeepMinorToMajor( + const Shape& input_shape, absl::Span dims_to_reduce) { + DimensionVector input_dims(ShapeUtil::Rank(input_shape), 0); + absl::c_iota(input_dims, 0); + DimensionVector input_dims_to_keep; + for (int input_dim : input_dims) { + auto it = absl::c_find_if(dims_to_reduce, [&](int64 dim_to_reduce) { + return dim_to_reduce == input_dim; + }); + if (it == dims_to_reduce.end()) { + input_dims_to_keep.push_back(input_dim); + } + } + + // Sort the dimensions to keep from minor to major. + absl::c_sort(input_dims_to_keep, [&input_shape](int64 dim_a, int64 dim_b) { + return PositionInContainer(LayoutUtil::MinorToMajor(input_shape), dim_a) < + PositionInContainer(LayoutUtil::MinorToMajor(input_shape), dim_b); + }); + + VLOG(10) << "dims to keep minor to major" + << absl::StrJoin(input_dims_to_keep, ","); + return input_dims_to_keep; +} + +// Given the input shape and dimensions to reduce for the reduction to vector, +// returns : +// num_kept: the number of elements in the contiguous dimensions to keep. +// num_reduced_major: the number of elements in the dimensions to reduce that +// are more major than the dimensions to keep. +// num_reduced_minor: the number of elements in the dimensions to reduce that +// are more minor than the dimensions to kept. +std::tuple GetReductionToVectorDimensions( + const Shape& input_shape, absl::Span dims_to_reduce) { + DimensionVector input_dims_to_keep_minor_to_major = + GetDimensionsToKeepMinorToMajor(input_shape, dims_to_reduce); + CHECK(LayoutUtil::AreDimensionsConsecutive( + input_shape.layout(), input_dims_to_keep_minor_to_major)); + int num_reduced_major = 1, num_kept = 1, num_reduced_minor = 1; + if (input_dims_to_keep_minor_to_major.empty()) { + return std::make_tuple(num_reduced_major, num_kept, num_reduced_minor); + } + DimensionVector input_dims(ShapeUtil::Rank(input_shape), 0); + absl::c_iota(input_dims, 0); + absl::Span minor_to_major = + LayoutUtil::MinorToMajor(input_shape); + for (int input_dim : input_dims) { + int64 curr_dim_size = input_shape.dimensions(input_dim); + if (PositionInContainer(minor_to_major, input_dim) > + PositionInContainer(minor_to_major, + input_dims_to_keep_minor_to_major.back())) { + num_reduced_major *= curr_dim_size; + } else if (PositionInContainer(minor_to_major, input_dim) < + PositionInContainer(minor_to_major, + input_dims_to_keep_minor_to_major.front())) { + num_reduced_minor *= curr_dim_size; + } else { + num_kept *= curr_dim_size; + } + } + + return std::make_tuple(num_reduced_major, num_kept, num_reduced_minor); +} + +std::tuple ComputeMappingSchemeAndReductionKind( + const HloInstruction* first_reduce, llvm::IRBuilder<>* b) { + int64 depth = 1; + int64 height = 1; + int64 width = 1; + bool is_row_reduction = true; + int64 tile_size_x = 1; + int64 tile_size_y = 1; + int64 block_size_y = 1; + int64 block_size_z = 1; + int64 num_threads_x = 1; + int64 num_threads_y = 1; + const Shape& input_shape = first_reduce->operand(0)->shape(); + int64 num_input_elems = ShapeUtil::ElementsIn(input_shape); + int64 num_output_elems = ShapeUtil::ElementsIn(first_reduce->shape()); + int64 num_reduced_major, num_kept, num_reduced_minor; + std::tie(num_reduced_major, num_kept, num_reduced_minor) = + GetReductionToVectorDimensions(input_shape, first_reduce->dimensions()); + CHECK_EQ(num_output_elems, num_kept); + + if (num_kept == 1) { + // Scalar reduction is a special row reduction with depth = height = 1. + width = num_input_elems; + tile_size_x = kWarpSize * 16; + num_threads_x = kWarpSize; + } else if (num_reduced_minor == 1) { + // Column reduction reduces inputs with dimension [height, width], where + // width is the minor dimension, to dimension [width]. + height = num_reduced_major; + width = num_kept; + is_row_reduction = false; + tile_size_x = std::min(kWarpSize, num_kept); + // The old Column reduction algorithm uses kTileHeight = 128. We choose + // tile_size_y * block_size_y = 128 to match the value of kTileHeight. Using + // a non-trivial block_size_y here is a way to avoid unrolling all the 128 + // iterations. + tile_size_y = 32; + block_size_y = 4; + num_threads_x = tile_size_x; + } else { + // Row reduction reduces inputs with dimension [depth, height, width], + // where width is the most minor dimension, to dimension [height] . + depth = num_reduced_major; + height = num_kept; + width = num_reduced_minor; + num_threads_x = kWarpSize; + if (width % (kWarpSize * 64) == 0) { + tile_size_x = kWarpSize * 64; + } else { + tile_size_x = kWarpSize * 8; + block_size_z = 8; + while (depth % block_size_z != 0) { + block_size_z -= 1; + } + } + } + DCHECK_EQ(depth * height * width, num_input_elems); + VLOG(10) << "is_row_reduction " << is_row_reduction << depth << " " << height + << " " << width; + + DimensionVector dims_in_elem{depth, height, width}; + DimensionVector req_block_sizes{block_size_z, block_size_y, 1}; + llvm_ir::KernelMappingScheme mapping_scheme(dims_in_elem, tile_size_y, + tile_size_x, req_block_sizes, + num_threads_y, num_threads_x, b); + return std::make_tuple(mapping_scheme, is_row_reduction); +} + +} // namespace + +Status IrEmitterUnnested::EmitReductionToVector(HloInstruction* unnested_hlo) { + VLOG(10) << "Emitting reduction to vector " << unnested_hlo->ToString(); + + HloInstruction* reduce_or_tuple = unnested_hlo->opcode() == HloOpcode::kFusion + ? unnested_hlo->fused_expression_root() + : unnested_hlo; + absl::Span output_instructions = + GetOutputInstructions(&reduce_or_tuple); + const HloInstruction* first_reduce = + GetFirstReduceInstruction(output_instructions); + + if (output_instructions.size() > 1) { + TF_RETURN_IF_ERROR( + AreFusedReductionOutputsConsistent(output_instructions, first_reduce)); + } + + // Build an initializer thunk to initialize each reduction output. + std::vector> thunks; + for (int i = 0, e = output_instructions.size(); i != e; ++i) { + if (output_instructions[i]->opcode() != HloOpcode::kReduce) { + continue; + } + TF_ASSIGN_OR_RETURN( + std::unique_ptr initializer_thunk, + BuildInitializerThunk(unnested_hlo, + (output_instructions[i] == reduce_or_tuple) + ? ShapeIndex() + : ShapeIndex({i}))); + thunks.push_back(std::move(initializer_thunk)); + } + + // Build a kernel thunk to compute all the outputs. + std::unique_ptr kernel_thunk = + BuildKernelThunk(unnested_hlo, /*implements_whole_instruction=*/false); + + const Shape& input_shape = first_reduce->operand(0)->shape(); + // The layout of a reduction input is either set by LayoutAssignment for + // unnested kReduce or by InstructionFusion for fused kReduce. + CHECK(input_shape.has_layout()) << "LayoutAssignment or InstructionFusion " + "doesn't set the input layout of " + << first_reduce->ToString(); + + bool is_row_reduction; + llvm_ir::KernelMappingScheme mapping_scheme; + std::tie(mapping_scheme, is_row_reduction) = + ComputeMappingSchemeAndReductionKind(first_reduce, &b_); + ReductionCodegenInfo reduction_info(&mapping_scheme, is_row_reduction); + KernelCodeGenerator kernel_generator( + /*tile_element_generator=*/ + [&](HloInstruction* hlo, const llvm_ir::IrArray::Index& index, + const KernelCodegenInfo* kernel_info, llvm::Value* y_loc, + llvm::Value* x_loc) { + EmitTileElementForReduction(hlo, index, kernel_info, y_loc, x_loc); + }, + /*block_prologue_generator=*/ + [&](HloInstruction* hlo, KernelCodegenInfo* kernel_info) { + EmitPrologueForReduction(hlo, kernel_info); + }, + /*block_epilogue_generator*/ + [&](HloInstruction* hlo, KernelCodegenInfo* kernel_info) { + EmitEpilogueForReduction(hlo, kernel_info); + }); + + LaunchDimensions launch_dimensions = + EmitKernel(unnested_hlo, {}, kernel_generator, &reduction_info); + UpdateLaunchDimensions(launch_dimensions, kernel_thunk.get(), + ir_emitter_context_->llvm_module()); + + thunks.push_back(std::move(kernel_thunk)); + std::unique_ptr sequential_thunk = + absl::make_unique(std::move(thunks), unnested_hlo); + AddThunkToThunkSequence(std::move(sequential_thunk)); + + return Status::OK(); +} + Status IrEmitterUnnested::EmitConstantGlobals() { for (const BufferAllocation& allocation : ir_emitter_context_->buffer_assignment().Allocations()) { @@ -3666,10 +3427,10 @@ Status IrEmitterUnnested::EmitConstantGlobals() { } // These globals will be looked up by name by GpuExecutable so we need to - // give them an external linkage. Not all of their uses are visible in the - // LLVM IR (e.g. TupleThunk) so we can't give then a linkage that merely - // preserves their names (like available_externally), we also need to ensure - // that they stick around even if they're "unused". + // give them an external linkage. Not all of their uses are visible in + // the LLVM IR (e.g. TupleThunk) so we can't give then a linkage that + // merely preserves their names (like available_externally), we also need + // to ensure that they stick around even if they're "unused". // // We may have to be more more clever here in the future if we notice that // we're keeping around too many globals because of their linkage. diff --git a/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.h b/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.h index 334c0b3c20b..85a0e5328c4 100644 --- a/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.h +++ b/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.h @@ -16,9 +16,11 @@ limitations under the License. #ifndef TENSORFLOW_COMPILER_XLA_SERVICE_GPU_IR_EMITTER_UNNESTED_H_ #define TENSORFLOW_COMPILER_XLA_SERVICE_GPU_IR_EMITTER_UNNESTED_H_ +#include "absl/container/inlined_vector.h" #include "tensorflow/compiler/xla/service/gpu/ir_emitter.h" #include "tensorflow/compiler/xla/service/gpu/sequential_thunk.h" #include "tensorflow/compiler/xla/service/gpu/thunk.h" +#include "tensorflow/compiler/xla/service/llvm_ir/kernel_support_library.h" #include "tensorflow/compiler/xla/service/llvm_ir/kernel_tiling.h" namespace xla { @@ -47,6 +49,99 @@ namespace gpu { // class IrEmitterUnnested : public IrEmitter { public: + // Parameter block_contains_multi_tiles indicates whether a tile block + // consists of multiple tiles or not. If the tile block contains only one + // tile, there is no need to use atomic operation to accumulate a local result + // to a global result to implement reduction. + using TileGenerator = + std::function output_tile_bounds, + bool block_contains_multi_tiles)>; + // KernelCodegenInfo records the common information to support the code + // generation for a kernel to process tensor elements by blocks. A block of + // tensor elements may contain one or multiple tiles. The code generators that + // generate code for tile elements or block prologue/epilogue refer to this + // class in their prototypes. If the implementations of such code generators + // require other information that are specific to the HLO instructions, the + // implementations need to define and use derived classes of this class. + class KernelCodegenInfo { + public: + explicit KernelCodegenInfo(llvm_ir::KernelMappingScheme* mapping_scheme) + : mapping_scheme_(mapping_scheme), + tiled_param_info_(nullptr), + lane_id_(nullptr), + index_ty_(nullptr) {} + virtual ~KernelCodegenInfo() {} + + void SetLaneId(llvm::Value* v) { lane_id_ = v; } + void SetIndexType(llvm::Type* t) { index_ty_ = t; } + void SetTiledParamInfo(llvm_ir::TiledParameterInfo* tiled_param_info) { + CHECK_EQ(tiled_param_info_, nullptr); + tiled_param_info_ = tiled_param_info; + } + + llvm::Value* GetLaneId() const { return lane_id_; } + llvm_ir::KernelMappingScheme* GetKernelMappingScheme() const { + return mapping_scheme_; + } + llvm_ir::TiledParameterInfo* GetTiledParameterInfo() const { + return tiled_param_info_; + } + llvm::Type* GetIndexType() const { return index_ty_; } + + private: + llvm_ir::KernelMappingScheme* mapping_scheme_; + llvm_ir::TiledParameterInfo* tiled_param_info_; + llvm::Value* lane_id_; + llvm::Type* index_ty_; + }; + + // A function object to prepare for the code generation for a tile block. + using BlockPrologueGenerator = + std::function; + // A function object to finalize the code generation for a tile block. + using BlockEpilogueGenerator = + std::function; + // A function object to generate code to process one element in a tile. + // + // hlo: the instruction for which the code is generated for. + // index: the index for the first output element of the current thread. + // y_loc: The y coordinate within a tile. + // x_loc: The x coordinate within a tile. + // kernel_info: Other information to support the kernel code generation. + using TileElementGenerator = std::function; + + // KernelCodeGenerator records the code generator objects that generate code + // for tile elements or tile block prologue/epilogue. + class KernelCodeGenerator { + public: + explicit KernelCodeGenerator( + TileElementGenerator tile_element_generator, + BlockPrologueGenerator block_prologue_generator = {}, + BlockEpilogueGenerator block_epilogue_generator = {}) + : tile_element_generator_(std::move(tile_element_generator)), + block_prologue_generator_(std::move(block_prologue_generator)), + block_epilogue_generator_(std::move(block_epilogue_generator)) {} + + const TileElementGenerator& GetTileElementGenerator() const { + return tile_element_generator_; + } + const BlockPrologueGenerator& GetBlockPrologueGenerator() const { + return block_prologue_generator_; + } + const BlockEpilogueGenerator& GetBlockEpilogueGenerator() const { + return block_epilogue_generator_; + } + + private: + TileElementGenerator tile_element_generator_; + BlockPrologueGenerator block_prologue_generator_; + BlockEpilogueGenerator block_epilogue_generator_; + }; + IrEmitterUnnested(const HloModuleConfig& hlo_module_config, const HloComputation* hlo_computation, IrEmitterContext* ir_emitter_context); @@ -82,7 +177,7 @@ class IrEmitterUnnested : public IrEmitter { Status HandleSort(HloInstruction* sort) override; Status HandleTupleSelect(HloInstruction* tuple_select) override; Status HandleCrossReplicaSum(HloInstruction* crs) override; - Status HandleAfterAll(HloInstruction* gen_token) override; + Status HandleAfterAll(HloInstruction* after_all) override; Status EmitTargetElementLoop( const HloInstruction& hlo, @@ -111,82 +206,14 @@ class IrEmitterUnnested : public IrEmitter { // Helper for writing extra outputs from inside a reduce kernel. Status EmitExtraOutputsForReduce( - const HloInstruction* reduce, const llvm_ir::IrArray::Index& index, + const HloInstruction* unnested_hlo, const llvm_ir::IrArray::Index& index, absl::Span> extra_output_gens); - // EmitColumnReduction and EmitRowReduction emit code for column and row - // reduction of a matrix and/or 3D tensor. Row and column reduction have - // different memory access pattern, so for performance their implementations - // are significantly different. + // Generates code for reduction to contiguous dimensions. // - // Emits code that reduces a matrix of shape [height x width] to a vector of - // [width]. Other parameters have the same meaning as those of - // `EmitReductionToVector`. Note that input shape might not be - // [height x width], but can be bitcast to [height x width] with "height" - // being the major dimension. - Status EmitColumnReduction( - KernelThunk* kernel_thunk, int64 height, int64 width, - HloInstruction* reduce, const Shape& input_shape, - absl::Span input_gens, - absl::Span init_value_gens, - absl::Span reducers, - absl::Span reduce_output_shapes, - absl::Span> - extra_output_gens); - - // Emits code that reduces a 3D tensor of shape [depth x height x width] to a - // vector of shape [height]. Other parameters have the same meaning as those - // of `EmitReductionToVector`. Note that input shape might not be - // [depth x height x width], but can be bitcast to [depth x height x width] - // with "depth" being the most major dimension. - Status EmitRowReduction( - KernelThunk* kernel_thunk, int64 depth, int64 height, int64 width, - HloInstruction* reduce, const Shape& input_shape, - absl::Span input_gens, - absl::Span init_value_gens, - absl::Span reducers, - absl::Span reduce_output_shapes, - absl::Span> - extra_output_gens); - - // Emits code that reduces a tensor of arbitrary rank to a scalar. - Status EmitReductionToScalar( - KernelThunk* kernel_thunk, HloInstruction* reduce, - const Shape& input_shape, - absl::Span input_gens, - absl::Span init_value_gens, - absl::Span reducers, - absl::Span reduce_output_shapes, - absl::Span> - extra_output_gens); - - // Figures out whether `reduce` is a row or column reduction, and which - // dimensions to reduce, and calls either `EmitRowReduction` or - // `EmitColumnReduction` as appropriate. `input_shape` is the shape of the - // input array, which is the operand of the Reduce instruction if unfused or - // of the Fusion instruction if fused. `input_gen` and `init_value_gen` - // generate elements of the input and the initial value. Other parameters mean - // the same as for `HandleReduce`. - // - // Multiple reduces can be emitted in the same loop, assuming they have the - // same input and output shapes, and the same reduce dimensions. - // - // extra_output_gens can contain extra generators for intermediate outputs. - // These must have the same shape as the reduce input as they are computed - // when the reduce inputs are being read. - // - // Prerequisite: `IsReductionToVector(*reduce)` - Status EmitReductionToVector( - KernelThunk* kernel_thunk, HloInstruction* reduce, - const Shape& input_shape, - absl::Span input_gens, - absl::Span init_value_gens, - absl::Span dimensions_to_reduce, - absl::Span reducers, - absl::Span reduce_output_shapes, - absl::Span> - extra_output_gens); + // Prerequisite: `IsReductionToVector(*unnested_hlo)` + Status EmitReductionToVector(HloInstruction* unnested_hlo); // Emits code for an in-place scatter, modifying `thunk`s launch dimensions in // the process. `scatter` may be fused, scatter indices are taken from @@ -205,22 +232,55 @@ class IrEmitterUnnested : public IrEmitter { LaunchDimensions EmitHlo021Tile(HloInstruction* hlo, absl::Span reduced_output_dims, absl::Span tiled_param_ids); + // Emits a kernel for an unnested HLO instruction. + LaunchDimensions EmitKernel(HloInstruction* unnested_hlo, + absl::Span param_ids, + const KernelCodeGenerator& kernel_generator, + KernelCodegenInfo* kernel_info); + void EmitBlock(const TileGenerator& emit_one_tile, + const KernelCodegenInfo* kernel_info, + KernelSupportLibrary& ksl, llvm::Type* index_ty); + // Emits code to process a tensor element in a tile for the given kCopy HLO + // that performs a 0-2-1 transpose. + void EmitTileElementForCopy(HloInstruction* hlo, + const llvm_ir::IrArray::Index& index, + const KernelCodegenInfo* kernel_info, + llvm::Value* y_loc, llvm::Value* x_loc); + // Emits code to process a tensor element in a tile for the given kLoop fusion + // HLO containing parameters that are 0-2-1 transpose of its outputs. + void EmitTileElementForFusion(HloInstruction* hlo, + const llvm_ir::IrArray::Index& index, + const KernelCodegenInfo* kernel_info, + llvm::Value* y_loc, llvm::Value* x_loc); + // Emits code to process a tensor element in a tile for the given input hlo + // that is either a unnested kReduce or a kInput fusion. + void EmitTileElementForReduction(HloInstruction* unnested_hlo, + const llvm_ir::IrArray::Index& index, + const KernelCodegenInfo* kernel_info, + llvm::Value* y_loc, llvm::Value* x_loc); + // Prepares for the code generation for a tile block of a reduction kernel. + void EmitPrologueForReduction(HloInstruction* unnested_hlo, + KernelCodegenInfo* kernel_info); + void EmitPrologueForOneReduction(HloInstruction* unnested_hlo, + HloInstruction* reduce_inst, int reduce_idx, + KernelCodegenInfo* kernel_info, + GpuElementalIrEmitter* elemental_emitter, + ShapeIndex output_shape_index); + // Wraps up the code generation for a tile block of a reduction kernel. + void EmitEpilogueForReduction(HloInstruction* unnested_hlo, + KernelCodegenInfo* kernel_info); + // For each reducer, emits the shuffle-down loop to accumulate the partial + // result to the global result. + void EmitFullWarpShuffleDownLoopForAllReduces( + const absl::InlinedVector& reducers, + const absl::InlinedVector& + partial_result_addresses); // Generates the IrArray for each input of an hlo and returns a vector that // constains such IrArrays. std::vector ConstructIrArrayForInputs( const HloInstruction& hlo); - // For each output of the `hlo` instruction, constructs the reduced shape for - // the output with the given `reduced_output_dims` and cast the original - // output IrArray element in `output_arrays` to the reduced shape. Returns - // the number of outputs. - int ConstructOutputReducedShapeAndCastOutputIrArrayToShape( - const HloInstruction& hlo, - const std::vector& output_arrays, - absl::Span reduced_output_dims, - std::vector* output_reduced_shapes, - std::vector* output_in_reduced_shape_arrays); // For each input of the `hlo` instruction, checks its value in // `param_buffers` to find out whether the input has a reduced shape. If the // input has a reduced shape, constructs the reduced shape for the input and diff --git a/tensorflow/compiler/xla/service/gpu/llvm_gpu_backend/nvptx_backend_lib.cc b/tensorflow/compiler/xla/service/gpu/llvm_gpu_backend/nvptx_backend_lib.cc index 8751e3a9c2a..24f07e68973 100644 --- a/tensorflow/compiler/xla/service/gpu/llvm_gpu_backend/nvptx_backend_lib.cc +++ b/tensorflow/compiler/xla/service/gpu/llvm_gpu_backend/nvptx_backend_lib.cc @@ -177,13 +177,6 @@ std::unique_ptr GetTargetMachine( } TargetOptions target_options = InitTargetOptionsFromCodeGenFlags(); - llvm_ir::SetTargetOptions( - /*fast_math_enabled=*/hlo_module_config.debug_options() - .xla_gpu_enable_fast_math(), - &target_options); - - // Enable FMA synthesis. - target_options.AllowFPOpFusion = FPOpFusion::Fast; // Set the verbose assembly options. target_options.MCOptions.AsmVerbose = false; @@ -453,18 +446,21 @@ void GPUBackendInit(const HloModuleConfig& hlo_module_config) { // * 3-6 gives similar results as 2; // * >6 start hurting the performance of at least dot product kernels. // - // TODO(jingyue): The current threshold only considers the numbr of IR + // TODO(jingyue): The current threshold only considers the number of IR // instructions which do not accurately reflect the true cost. We need a // better cost model. FeedLLVMWithFlags({"-bonus-inst-threshold=2"}); - // TODO(b/22073864): Increase limit when scan memory dependency. - // This helps to reduce more redundant load instructions. + // Increase limit when scanning memory dependencies. This helps to reduce + // more redundant load instructions. // // The specific value is currently large enough for s3d in shoc benchmark, // which contains a lot of load instructions and many arithmetic instructions // between those loads. FeedLLVMWithFlags({"-memdep-block-scan-limit=500"}); + // Use div.approx -- it matters for some float-division heavy benchmarks. + FeedLLVMWithFlags({"-nvptx-prec-divf32=0"}); + llvm_ir::InitializeLLVMCommandLineOptions(hlo_module_config); // Initialize the NVPTX target; it's the only target we link with, so call its diff --git a/tensorflow/compiler/xla/service/gpu/multi_output_fusion.cc b/tensorflow/compiler/xla/service/gpu/multi_output_fusion.cc index d9b06828e2b..01fddcede64 100644 --- a/tensorflow/compiler/xla/service/gpu/multi_output_fusion.cc +++ b/tensorflow/compiler/xla/service/gpu/multi_output_fusion.cc @@ -41,50 +41,7 @@ GpuMultiOutputFusion::GpuMultiOutputFusion() : MultiOutputFusion(INT64_MAX) {} bool GpuMultiOutputFusion::ShapesCompatibleForFusion(HloInstruction* instr1, HloInstruction* instr2) { - auto get_element_instr = - [&](const HloInstruction* instr) -> const HloInstruction* { - const HloInstruction* element_instr = instr; - if (instr->opcode() == HloOpcode::kFusion) { - auto fused_expression_root = instr->fused_expression_root(); - if (instr->IsMultiOutputFusion()) { - // If possible, we want to pick a reduce operand of the fusion root, - // because it has the most constraints. - for (const auto* inst : fused_expression_root->operands()) { - if (IsReductionToVector(*inst)) { - return inst; - } - } - return fused_expression_root->operands()[0]; - } else { - element_instr = fused_expression_root; - } - } - return element_instr; - }; - - auto get_element_shape = [&](const HloInstruction* element_instr) { - // Special handling of kReduce instructions -- the fusion - // applies to the first operand. - if (IsReductionToVector(*element_instr)) { - return element_instr->operand(0)->shape(); - } - return element_instr->shape(); - }; - - // The shapes in all tuple operands should agree, unless it is a reduce. - // In that case, the operand of the reduce needs to have the same shape - // as the other tuple operands, but also we need to compare the output - // shapes of the reduces. - auto* element_instr_1 = get_element_instr(instr1); - auto* element_instr_2 = get_element_instr(instr2); - if (element_instr_1->opcode() == HloOpcode::kReduce && - element_instr_2->opcode() == HloOpcode::kReduce && - !ShapeUtil::Equal(element_instr_1->shape(), element_instr_2->shape())) { - return false; - } - // The elementwise output shapes must be the same (including layout). - return ShapeUtil::EqualIgnoringFpPrecision( - get_element_shape(element_instr_1), get_element_shape(element_instr_2)); + return ShapesCompatibleForMultiOutputFusion(*instr1, *instr2); } bool GpuMultiOutputFusion::IsFusible(HloInstruction* instr) { @@ -205,7 +162,7 @@ bool GpuMultiOutputFusion::DoProducerConsumerMultiOutputFusion() { VLOG(3) << producer->name() << " is not a loop fusion."; continue; } - if (!ShapesCompatibleForFusion(producer, consumer)) { + if (!ShapesCompatibleForMultiOutputFusion(*producer, *consumer)) { VLOG(3) << producer->name() << " has an incompatible shape."; continue; } diff --git a/tensorflow/compiler/xla/service/gpu/multi_output_fusion_test.cc b/tensorflow/compiler/xla/service/gpu/multi_output_fusion_test.cc index dc221f22a74..d16c87ba5c6 100644 --- a/tensorflow/compiler/xla/service/gpu/multi_output_fusion_test.cc +++ b/tensorflow/compiler/xla/service/gpu/multi_output_fusion_test.cc @@ -580,7 +580,7 @@ TEST_F(MultiOutputFusionTest, AvoidsLargeFusion) { // ... // where each of the (pi * pj)'s is represented as a fusion node so that // multi-output fusion will pay attention to it. - auto module = CreateNewUnverifiedModule(); + auto module = CreateNewVerifiedModule(); HloComputation::Builder b(TestName()); Shape shape = ShapeUtil::MakeShape(F32, {10, 100}); diff --git a/tensorflow/compiler/xla/service/gpu/nvptx_compiler.cc b/tensorflow/compiler/xla/service/gpu/nvptx_compiler.cc index de04ed85c30..e934cbda176 100644 --- a/tensorflow/compiler/xla/service/gpu/nvptx_compiler.cc +++ b/tensorflow/compiler/xla/service/gpu/nvptx_compiler.cc @@ -67,6 +67,7 @@ limitations under the License. #include "tensorflow/compiler/xla/service/hlo_cse.h" #include "tensorflow/compiler/xla/service/hlo_dce.h" #include "tensorflow/compiler/xla/service/hlo_element_type_converter.h" +#include "tensorflow/compiler/xla/service/hlo_get_dimension_size_rewriter.h" #include "tensorflow/compiler/xla/service/hlo_instruction.h" #include "tensorflow/compiler/xla/service/hlo_pass_fix.h" #include "tensorflow/compiler/xla/service/hlo_pass_pipeline.h" @@ -173,13 +174,16 @@ Status OptimizeHloModule(HloModule* hlo_module, se::StreamExecutor* stream_exec, /*rewrite_inference_op=*/true, /*rewrite_grad_op=*/true); + pipeline.AddPass(); + // BatchNormExpander can create zero-sized ops, so zero-sized HLO // elimination has to come after that pass. pipeline.AddPass(); - pass.AddPass( - /*is_layout_sensitive=*/false, + AlgebraicSimplifierOptions options( [](const Shape&, const Shape&) { return false; }); + options.set_enable_permutation_sort_replacement(true); + pass.AddPass(options); pass.AddPass(); pass.AddPass(); pass.AddPass(); @@ -248,11 +252,13 @@ Status OptimizeHloModule(HloModule* hlo_module, se::StreamExecutor* stream_exec, // The LayoutAssignment pass may leave behind kCopy instructions which are // duplicate or NOPs, so remove them with algebraic simplification and CSE. - pipeline.AddPass>( - /*is_layout_sensitive=*/true, + AlgebraicSimplifierOptions options( /*valid_bitcast_callback=*/[](const Shape&, const Shape&) { return true; }); + options.set_is_layout_sensitive(true); + options.set_enable_permutation_sort_replacement(true); + pipeline.AddPass>(options); // Choose the fastest algorithm for each conv. // @@ -810,7 +816,7 @@ std::vector NVPTXCompiler::CompilePtxOrGetCachedResult(const string& ptx, // binaries are not available. We don't want to spam logs with // identical warnings in this case. - // TODO(zhengxq): we should implement a LOG_FIRST_N and LOG_EVERY_N + // TODO(jlebar): we should implement a LOG_FIRST_N and LOG_EVERY_N // for more general usage. static std::atomic warning_done(false); log_warning = !warning_done.exchange(true); diff --git a/tensorflow/compiler/xla/service/gpu/stream_assignment_test.cc b/tensorflow/compiler/xla/service/gpu/stream_assignment_test.cc index f2ef11e1e6a..31a5d7a8c04 100644 --- a/tensorflow/compiler/xla/service/gpu/stream_assignment_test.cc +++ b/tensorflow/compiler/xla/service/gpu/stream_assignment_test.cc @@ -30,7 +30,7 @@ namespace gpu { class StreamAssignmentTest : public HloTestBase { protected: - std::unique_ptr CreateNewUnverifiedModule() { + std::unique_ptr CreateNewVerifiedModule() { HloModuleConfig config; auto debug_options = GetDebugOptionsForTest(); debug_options.set_xla_gpu_disable_multi_streaming(false); @@ -55,7 +55,7 @@ TEST_F(StreamAssignmentTest, SequentialMatMul) { HloInstruction* dot2 = builder.AddInstruction(CreateCanonicalDot(f32_2x2_, dot1, z)); - auto module = CreateNewUnverifiedModule(); + auto module = CreateNewVerifiedModule(); module->AddEntryComputation(builder.Build(dot2)); std::unique_ptr assignment = AssignStreams(*module); @@ -76,7 +76,7 @@ TEST_F(StreamAssignmentTest, ConcurrentMatMul) { HloInstruction* add = builder.AddInstruction( HloInstruction::CreateBinary(f32_2x2_, HloOpcode::kAdd, dot1, dot2)); - auto module = CreateNewUnverifiedModule(); + auto module = CreateNewVerifiedModule(); module->AddEntryComputation(builder.Build(add)); std::unique_ptr assignment = AssignStreams(*module); @@ -120,7 +120,7 @@ TEST_F(StreamAssignmentTest, LatticeMatMul) { HloInstruction* d40 = builder.AddInstruction(CreateCanonicalDot(f32_2x2_, d30, d31)); - auto module = CreateNewUnverifiedModule(); + auto module = CreateNewVerifiedModule(); module->AddEntryComputation(builder.Build(d40)); std::unique_ptr assignment = AssignStreams(*module); diff --git a/tensorflow/compiler/xla/service/gpu/tests/gpu_codegen_test.h b/tensorflow/compiler/xla/service/gpu/tests/gpu_codegen_test.h index d2f30ae7bc4..d917320e363 100644 --- a/tensorflow/compiler/xla/service/gpu/tests/gpu_codegen_test.h +++ b/tensorflow/compiler/xla/service/gpu/tests/gpu_codegen_test.h @@ -26,7 +26,7 @@ namespace gpu { // Tests that verify IR or PTX emitted by the GPU backend is as expected. class GpuCodegenTest : public LlvmIrGenTestBase { protected: - // Like HloTestBase::CreateNewUnverifiedModule(), with a flag for configuring + // Like HloTestBase::CreateNewVerifiedModule(), with a flag for configuring // the ftz option. std::unique_ptr CreateNewUnverifiedModuleWithFTZ(bool ftz); diff --git a/tensorflow/compiler/xla/service/gpu/tests/gpu_copy_test.cc b/tensorflow/compiler/xla/service/gpu/tests/gpu_copy_test.cc index 268b48a1cad..a1ed8499040 100644 --- a/tensorflow/compiler/xla/service/gpu/tests/gpu_copy_test.cc +++ b/tensorflow/compiler/xla/service/gpu/tests/gpu_copy_test.cc @@ -46,7 +46,7 @@ TEST_F(GpuCopyTest, UseMemcpy) { std::unique_ptr computation = builder.Build(); - auto hlo_module = CreateNewUnverifiedModule(); + auto hlo_module = CreateNewVerifiedModule(); hlo_module->AddEntryComputation(std::move(computation)); // There should not be any kernel prefixed "copy". diff --git a/tensorflow/compiler/xla/service/gpu/tests/gpu_ftz_test.cc b/tensorflow/compiler/xla/service/gpu/tests/gpu_ftz_test.cc index d0ccd8619bd..5e524faab18 100644 --- a/tensorflow/compiler/xla/service/gpu/tests/gpu_ftz_test.cc +++ b/tensorflow/compiler/xla/service/gpu/tests/gpu_ftz_test.cc @@ -75,16 +75,16 @@ class GpuFtzDisabledTest : public GpuFtzTest { // Check that we emit mul.ftz.f32 when in ftz mode, and plain mul.f32 otherwise. TEST_F(GpuFtzEnabledTest, MultiplyFtz) { CompileAndVerifyPtx(CreateBinaryOpModule(HloOpcode::kMultiply), R"( - CHECK-NOT: mul.f32 - CHECK: mul.ftz.f32 - CHECK-NOT: mul.f32 + CHECK-NOT: mul.rn.f32 + CHECK: mul.rn.ftz.f32 + CHECK-NOT: mul.rn.f32 )"); } TEST_F(GpuFtzDisabledTest, MultiplyFtz) { CompileAndVerifyPtx(CreateBinaryOpModule(HloOpcode::kMultiply), R"( - CHECK-NOT: mul.ftz.f32 - CHECK: mul.f32 - CHECK-NOT: mul.ftz.f32 + CHECK-NOT: mul.rn.ftz.f32 + CHECK: mul.rn.f32 + CHECK-NOT: mul.rn.ftz.f32 )"); } diff --git a/tensorflow/compiler/xla/service/gpu/tests/gpu_index_test.cc b/tensorflow/compiler/xla/service/gpu/tests/gpu_index_test.cc index da8e513a2c3..6814be779e0 100644 --- a/tensorflow/compiler/xla/service/gpu/tests/gpu_index_test.cc +++ b/tensorflow/compiler/xla/service/gpu/tests/gpu_index_test.cc @@ -51,7 +51,7 @@ TEST_F(GpuIndexTest, CompatibleUseLinearIndex) { builder.AddInstruction(HloInstruction::CreateBinary( ShapeUtil::MakeShape(PRED, {5, 7, 2}), HloOpcode::kGe, param_x, param_y)); - auto hlo_module = CreateNewUnverifiedModule(); + auto hlo_module = CreateNewVerifiedModule(); hlo_module->AddEntryComputation(builder.Build()); // Check the optimized IR as the unoptimized IR contains dead udiv and urem. diff --git a/tensorflow/compiler/xla/service/gpu/tests/gpu_ldg_test.cc b/tensorflow/compiler/xla/service/gpu/tests/gpu_ldg_test.cc index ea1fee040dd..3019215c015 100644 --- a/tensorflow/compiler/xla/service/gpu/tests/gpu_ldg_test.cc +++ b/tensorflow/compiler/xla/service/gpu/tests/gpu_ldg_test.cc @@ -48,7 +48,7 @@ TEST_F(GpuLdgTest, LdgForParamRead) { HloInstruction::CreateBinary(shape, HloOpcode::kAdd, param, param)); std::unique_ptr computation = builder.Build(); - auto hlo_module = CreateNewUnverifiedModule(); + auto hlo_module = CreateNewVerifiedModule(); hlo_module->AddEntryComputation(std::move(computation)); CompileAndVerifyPtx(std::move(hlo_module), R"( @@ -73,7 +73,7 @@ TEST_F(GpuLdgTest, LdgForNonParamRead) { builder.AddInstruction(HloInstruction::CreateTuple({add, square})); std::unique_ptr computation = builder.Build(); - auto hlo_module = CreateNewUnverifiedModule(); + auto hlo_module = CreateNewVerifiedModule(); hlo_module->AddEntryComputation(std::move(computation)); CompileAndVerifyPtx(std::move(hlo_module), R"( @@ -95,7 +95,7 @@ TEST_F(GpuLdgTest, LdgForNonParamRead) { // reduce in the foreseeable future. But if that turns out to be wrong, I give // you, future reader, permission to delete this test. TEST_F(GpuLdgTest, NoLdgWhenSharingBuffer) { - auto hlo_module = CreateNewUnverifiedModule(); + auto hlo_module = CreateNewVerifiedModule(); HloComputation::Builder builder(TestName()); HloComputation* reduce_computation; diff --git a/tensorflow/compiler/xla/service/gpu/tests/gpu_noalias_test.cc b/tensorflow/compiler/xla/service/gpu/tests/gpu_noalias_test.cc index 14285459b5a..ca0a78034d7 100644 --- a/tensorflow/compiler/xla/service/gpu/tests/gpu_noalias_test.cc +++ b/tensorflow/compiler/xla/service/gpu/tests/gpu_noalias_test.cc @@ -47,7 +47,7 @@ TEST_F(GpuNoAliasTest, Concat) { std::unique_ptr computation = builder.Build(); - auto hlo_module = CreateNewUnverifiedModule(); + auto hlo_module = CreateNewVerifiedModule(); hlo_module->AddEntryComputation(std::move(computation)); CompileAndVerifyIr(std::move(hlo_module), diff --git a/tensorflow/compiler/xla/service/gpu/thunk_schedule.cc b/tensorflow/compiler/xla/service/gpu/thunk_schedule.cc index 141f3219387..6b2d76764a0 100644 --- a/tensorflow/compiler/xla/service/gpu/thunk_schedule.cc +++ b/tensorflow/compiler/xla/service/gpu/thunk_schedule.cc @@ -45,7 +45,7 @@ void ThunkSchedule::AddDependenciesOnTransitiveOperands( ThunkSchedule::ThunkSchedule( std::unique_ptr thunks, std::unique_ptr stream_assignment, - const std::vector& hlo_total_order) + const std::vector& hlo_total_order) : thunks_(std::move(thunks)), stream_assignment_(std::move(stream_assignment)) { std::unordered_map hlo_to_thunk; @@ -53,7 +53,7 @@ ThunkSchedule::ThunkSchedule( InsertOrDie(&hlo_to_thunk, thunk->hlo_instruction(), thunk.get()); } - for (const HloInstruction* hlo : hlo_total_order) { + for (HloInstruction* hlo : hlo_total_order) { if (hlo_to_thunk.count(hlo)) { thunk_total_order_.push_back(FindOrDie(hlo_to_thunk, hlo)); } diff --git a/tensorflow/compiler/xla/service/gpu/thunk_schedule.h b/tensorflow/compiler/xla/service/gpu/thunk_schedule.h index d3352994f84..43b628a1baf 100644 --- a/tensorflow/compiler/xla/service/gpu/thunk_schedule.h +++ b/tensorflow/compiler/xla/service/gpu/thunk_schedule.h @@ -46,7 +46,7 @@ class ThunkSchedule { public: ThunkSchedule(std::unique_ptr thunks, std::unique_ptr stream_assignment, - const std::vector& hlo_total_order); + const std::vector& hlo_total_order); // Returns the total order of executing all the thunks. const std::vector& TotalOrder() const { return thunk_total_order_; } diff --git a/tensorflow/compiler/xla/service/gpu/while_transformer_test.cc b/tensorflow/compiler/xla/service/gpu/while_transformer_test.cc index c7f51127649..2dce7749bbd 100644 --- a/tensorflow/compiler/xla/service/gpu/while_transformer_test.cc +++ b/tensorflow/compiler/xla/service/gpu/while_transformer_test.cc @@ -29,7 +29,7 @@ namespace { class WhileTransformerTest : public HloTestBase { protected: WhileTransformerTest() - : module_(CreateNewUnverifiedModule()), + : module_(CreateNewVerifiedModule()), induction_variable_shape_(ShapeUtil::MakeShape(S32, {})), data_shape_(ShapeUtil::MakeShape(F32, {8})), condition_result_shape_(ShapeUtil::MakeShape(PRED, {})) {} diff --git a/tensorflow/compiler/xla/service/heap_simulator_test.cc b/tensorflow/compiler/xla/service/heap_simulator_test.cc index fad3215fc81..dc40b9446ad 100644 --- a/tensorflow/compiler/xla/service/heap_simulator_test.cc +++ b/tensorflow/compiler/xla/service/heap_simulator_test.cc @@ -258,7 +258,7 @@ class HeapSimulatorTracker { // Constructor for testing a single entry computation. HeapSimulatorTracker( const string& name, std::unique_ptr computation, - const std::vector& instruction_sequence) { + const std::vector& instruction_sequence) { HloModuleConfig config; module_ = absl::make_unique(name, config); module_->AddEntryComputation(std::move(computation)); @@ -286,7 +286,7 @@ class HeapSimulatorTracker { // Similar to the single entry computation constructor above, but runs the // simulation over the entire module. void RunWholeModule( - const std::vector& full_module_sequence) { + const std::vector& full_module_sequence) { points_to_analysis_ = TuplePointsToAnalysis::Run(module_.get()).ConsumeValueOrDie(); @@ -294,7 +294,7 @@ class HeapSimulatorTracker { HloSchedule schedule(module_.get()); absl::flat_hash_map reverse_position; for (int i = 0; i < full_module_sequence.size(); ++i) { - const HloInstruction* instruction = full_module_sequence[i]; + HloInstruction* instruction = full_module_sequence[i]; schedule.GetOrCreateSequence(instruction->parent()) .push_back(instruction); reverse_position[instruction] = full_module_sequence.size() - i; diff --git a/tensorflow/compiler/xla/service/hlo.proto b/tensorflow/compiler/xla/service/hlo.proto index dbab62f847e..414c6327124 100644 --- a/tensorflow/compiler/xla/service/hlo.proto +++ b/tensorflow/compiler/xla/service/hlo.proto @@ -51,7 +51,7 @@ message HloInstructionProto { string name = 1; string opcode = 2; - xla.Shape shape = 3; + xla.ShapeProto shape = 3; xla.OpMetadata metadata = 7; @@ -132,7 +132,7 @@ message HloInstructionProto { string custom_call_opaque = 53; // Shape of outfeed request. - xla.Shape outfeed_shape = 29; + xla.ShapeProto outfeed_shape = 29; // Describes the dimension numbers used for a dot operation xla.DotDimensionNumbers dot_dimension_numbers = 30; @@ -190,7 +190,7 @@ message HloInstructionProto { // 'operand_shapes_with_layout' must contain a shape with layout for each // operand. bool constrain_layout = 56; - repeated Shape operand_shapes_with_layout = 57; + repeated xla.ShapeProto operand_shapes_with_layout = 57; } // Serialization of HloComputation. @@ -205,7 +205,8 @@ message HloComputationProto { repeated HloInstructionProto instructions = 2; // The program shape (with layout) of this computation. - xla.ProgramShape program_shape = 4; + + xla.ProgramShapeProto program_shape = 4; // The id of this computation. int64 id = 5; @@ -251,6 +252,41 @@ message HloInputOutputAliasProto { repeated AliasEntryProto entries = 1; } +message DynamicParameterBindingProto { + // A list of bindings which indicates that the `target_dim_num` in + // the subshape `target_param_index` of parameter `target_param_num` + // is a dynamic dimension and its real dynamic size is represented + // by `dynamic_param_index` in parameter `dynamic_param_num`. + // + // As an example, imagine we have a program: + // + // ENTRY main { + // a = f32[] parameter(0) + // b = f32[10] parameter(1) + // ROOT root = (f32[], f32[10]) tuple(%a, %b) + // } + // + // Let's say 'b' (param index 1) is a dynamic shape whose input has + // an upperbound of 10 and real size is determined at runtime.'a' + // represents the real size of b's first dimension. + // + // In this case, the fields are set in the following way: + // dynamic_param_num = 1 + // dynamic_param_index = {} + // target_param_num = 0 + // target_param_index = {} + // target_param_dim = 0 + message Binding { + int64 dynamic_param_num = 1; + repeated int64 dynamic_param_index = 2; + int64 target_param_num = 3; + repeated int64 target_param_index = 4; + int64 target_param_dim_num = 5; + } + + repeated Binding entries = 1; +} + // Serialization of HloModule. message HloModuleProto { string name = 1; @@ -262,7 +298,7 @@ message HloModuleProto { repeated HloComputationProto computations = 3; // The host program shape (with layout) of the entry computation. - xla.ProgramShape host_program_shape = 4; + xla.ProgramShapeProto host_program_shape = 4; // The id of this module. int64 id = 5; @@ -272,6 +308,8 @@ message HloModuleProto { // Describes alias information between inputs and outputs. HloInputOutputAliasProto input_output_alias = 8; + + DynamicParameterBindingProto dynamic_parameter_binding = 9; } // Serialization of LogicalBuffer. diff --git a/tensorflow/compiler/xla/service/hlo_computation.cc b/tensorflow/compiler/xla/service/hlo_computation.cc index 0c20d207ddb..ff122b529bd 100644 --- a/tensorflow/compiler/xla/service/hlo_computation.cc +++ b/tensorflow/compiler/xla/service/hlo_computation.cc @@ -499,7 +499,7 @@ HloComputationProto HloComputation::ToProto() const { proto.add_instructions()->Swap(&instruction_proto); } proto.set_root_id(root_instruction()->unique_id()); - *proto.mutable_program_shape() = ComputeProgramShape(); + *proto.mutable_program_shape() = ComputeProgramShape().ToProto(); return proto; } @@ -711,6 +711,8 @@ bool HloComputation::operator==(const HloComputation& other) const { return eq(root_instruction(), other.root_instruction()); } +uint64 HloComputation::Hash() const { return root_instruction()->Hash(); } + Status HloComputation::ReplaceWithNewInstruction( HloInstruction* old_instruction, std::unique_ptr new_instruction) { @@ -795,7 +797,7 @@ Status HloComputation::AcceptWithOperandOrder( template Status HloComputation::AcceptOrdered( DfsHloVisitorBase* visitor, - const std::vector& order) const { + const std::vector& order) const { VLOG(3) << "Accepting visitor with order."; for (HloInstruction* root : CollectUnreachableRoots()) { TF_RET_CHECK(std::find(order.begin(), order.end(), root) != order.end()) @@ -825,9 +827,9 @@ Status HloComputation::AcceptOrdered( // Explicit instantiations. template Status HloComputation::AcceptOrdered( - DfsHloVisitor*, const std::vector&) const; + DfsHloVisitor*, const std::vector&) const; template Status HloComputation::AcceptOrdered( - ConstDfsHloVisitor*, const std::vector&) const; + ConstDfsHloVisitor*, const std::vector&) const; Status HloComputation::Accept( const std::function& visitor_func) { diff --git a/tensorflow/compiler/xla/service/hlo_computation.h b/tensorflow/compiler/xla/service/hlo_computation.h index fc7d2035e5b..c584e4c7ca5 100644 --- a/tensorflow/compiler/xla/service/hlo_computation.h +++ b/tensorflow/compiler/xla/service/hlo_computation.h @@ -264,6 +264,12 @@ class HloComputation { // Return whether `*this` and `other` are functionally equivalent. bool operator==(const HloComputation& other) const; + // Generates a hash value of an HLO computation. Hash considers + // information on opcode, shape, operands, and typically a root instruction. + // This function returns the same hash value for equivalent HLO computations, + // with respect to HloInstruction::Identical() method. + uint64 Hash() const; + // Replaces old instruction with newly created instruction. Removes old // instruction from computation. Updates uses and root instruction. Status ReplaceWithNewInstruction( @@ -301,7 +307,7 @@ class HloComputation { // be a topological sort of all instructions in the computation. template Status AcceptOrdered(DfsHloVisitorBase* visitor, - const std::vector& order) const; + const std::vector& order) const; // Same as Accept() above, but the visitor is given as a function. Status Accept(const std::function& visitor_func); diff --git a/tensorflow/compiler/xla/service/hlo_computation_test.cc b/tensorflow/compiler/xla/service/hlo_computation_test.cc index 1e7a6e197f5..8b50cfa9aed 100644 --- a/tensorflow/compiler/xla/service/hlo_computation_test.cc +++ b/tensorflow/compiler/xla/service/hlo_computation_test.cc @@ -65,7 +65,7 @@ class HloComputationTest : public HloTestBase { }; TEST_F(HloComputationTest, GetEmbeddedComputationsEmpty) { - auto module = CreateNewUnverifiedModule(); + auto module = CreateNewVerifiedModule(); auto negate_computation = module->AddEntryComputation(CreateNegateComputation()); EXPECT_TRUE(negate_computation->MakeEmbeddedComputationsList().empty()); @@ -73,7 +73,7 @@ TEST_F(HloComputationTest, GetEmbeddedComputationsEmpty) { TEST_F(HloComputationTest, GetEmbeddedComputationsOneComputation) { // Create computation which calls one other computation. - auto module = CreateNewUnverifiedModule(); + auto module = CreateNewVerifiedModule(); auto negate_computation = module->AddEmbeddedComputation(CreateNegateComputation()); auto map_computation = @@ -85,7 +85,7 @@ TEST_F(HloComputationTest, GetEmbeddedComputationsOneComputation) { TEST_F(HloComputationTest, GetEmbeddedComputationsDiamond) { // Create computations with a diamond-shaped callgraph. - auto module = CreateNewUnverifiedModule(); + auto module = CreateNewVerifiedModule(); auto negate_computation = module->AddEmbeddedComputation(CreateNegateComputation()); auto map1_computation = @@ -119,7 +119,7 @@ TEST_F(HloComputationTest, PostOrderSingleton) { auto builder = HloComputation::Builder(TestName()); auto constant = builder.AddInstruction( HloInstruction::CreateConstant(LiteralUtil::CreateR0(42.0f))); - auto module = CreateNewUnverifiedModule(); + auto module = CreateNewVerifiedModule(); auto computation = module->AddEntryComputation(builder.Build()); EXPECT_THAT(computation->MakeInstructionPostOrder(), ElementsAre(constant)); } @@ -134,7 +134,7 @@ TEST_F(HloComputationTest, PostOrderSimple) { HloInstruction::CreateUnary(r0f32_, HloOpcode::kNegate, constant)); auto negate2 = builder.AddInstruction( HloInstruction::CreateUnary(r0f32_, HloOpcode::kNegate, negate1)); - auto module = CreateNewUnverifiedModule(); + auto module = CreateNewVerifiedModule(); auto computation = module->AddEntryComputation(builder.Build()); EXPECT_THAT(computation->MakeInstructionPostOrder(), ElementsAre(constant, negate1, negate2)); @@ -170,7 +170,7 @@ TEST_F(HloComputationTest, PostOrderDisconnectedInstructions) { HloInstruction::CreateConstant(LiteralUtil::CreateR0(42.0f))); auto constant4 = builder.AddInstruction( HloInstruction::CreateConstant(LiteralUtil::CreateR0(42.0f))); - auto module = CreateNewUnverifiedModule(); + auto module = CreateNewVerifiedModule(); auto computation = module->AddEntryComputation(builder.Build()); EXPECT_THAT(computation->MakeInstructionPostOrder(), UnorderedElementsAre(constant1, constant2, constant3, constant4)); @@ -192,7 +192,7 @@ TEST_F(HloComputationTest, PostOrderWithMultipleRoots) { r0f32_, HloOpcode::kAdd, constant2, constant3)); auto add3 = builder.AddInstruction(HloInstruction::CreateBinary( r0f32_, HloOpcode::kAdd, constant1, constant3)); - auto module = CreateNewUnverifiedModule(); + auto module = CreateNewVerifiedModule(); auto computation = module->AddEntryComputation(builder.Build()); auto post_order = computation->MakeInstructionPostOrder(); EXPECT_EQ(6, post_order.size()); @@ -217,7 +217,7 @@ TEST_F(HloComputationTest, VisitWithMultipleRoots) { constant2, constant3)); builder.AddInstruction(HloInstruction::CreateBinary(r0f32_, HloOpcode::kAdd, constant1, constant3)); - auto module = CreateNewUnverifiedModule(); + auto module = CreateNewVerifiedModule(); auto computation = module->AddEntryComputation(builder.Build()); // Visitor which keeps track of which instructions have been visited. class TestVisitor : public DfsHloVisitorWithDefault { @@ -257,7 +257,7 @@ TEST_F(HloComputationTest, DeepCopyArray) { auto builder = HloComputation::Builder(TestName()); auto constant = builder.AddInstruction(HloInstruction::CreateConstant( LiteralUtil::CreateR1({1.0, 2.0, 3.0}))); - auto module = CreateNewUnverifiedModule(); + auto module = CreateNewVerifiedModule(); auto computation = module->AddEntryComputation(builder.Build()); auto copy = computation->DeepCopyInstruction(constant).ValueOrDie(); @@ -274,7 +274,7 @@ TEST_F(HloComputationTest, DeepCopyTuple) { auto tuple = builder.AddInstruction( HloInstruction::CreateTuple({constant1, constant2})); - auto module = CreateNewUnverifiedModule(); + auto module = CreateNewVerifiedModule(); auto computation = module->AddEntryComputation(builder.Build()); auto tuple_copy = computation->DeepCopyInstruction(tuple).ValueOrDie(); @@ -376,7 +376,7 @@ TEST_F(HloComputationTest, DeepCopyToken) { // copied. auto builder = HloComputation::Builder(TestName()); auto token = builder.AddInstruction(HloInstruction::CreateToken()); - auto module = CreateNewUnverifiedModule(); + auto module = CreateNewVerifiedModule(); auto computation = module->AddEntryComputation(builder.Build()); auto copy = computation->DeepCopyInstruction(token).ValueOrDie(); @@ -393,7 +393,7 @@ TEST_F(HloComputationTest, DeepCopyTokenTuple) { HloInstruction::CreateConstant(LiteralUtil::CreateR0(42.0))); auto tuple = builder.AddInstruction(HloInstruction::CreateTuple({token, constant})); - auto module = CreateNewUnverifiedModule(); + auto module = CreateNewVerifiedModule(); auto computation = module->AddEntryComputation(builder.Build()); auto copy = computation->DeepCopyInstruction(tuple).ValueOrDie(); @@ -440,7 +440,7 @@ TEST_F(HloComputationTest, RemoveInstructionWithDuplicateOperand) { r0f32_, HloOpcode::kAdd, dead_negate, dead_negate)); auto negate = builder.AddInstruction( HloInstruction::CreateUnary(r0f32_, HloOpcode::kNegate, constant)); - auto module = CreateNewUnverifiedModule(); + auto module = CreateNewVerifiedModule(); auto computation = module->AddEntryComputation(builder.Build()); EXPECT_EQ(4, computation->instruction_count()); EXPECT_THAT(computation->root_instruction(), op::Negate(constant)); @@ -466,7 +466,7 @@ TEST_F(HloComputationTest, CloneWithControlDependency) { HloInstruction::CreateParameter(0, r0f32_, "param0")); auto negate = builder.AddInstruction( HloInstruction::CreateUnary(r0f32_, HloOpcode::kNegate, param)); - auto module = CreateNewUnverifiedModule(); + auto module = CreateNewVerifiedModule(); auto computation = module->AddEntryComputation(builder.Build(/*root_instruction=*/add)); @@ -505,7 +505,7 @@ TEST_F(HloComputationTest, Stringification) { 2, PrecisionConfig::DEFAULT); builder.AddInstruction( HloInstruction::CreateDot(sout, x, reshape, dot_dnums, precision_config)); - auto module = CreateNewUnverifiedModule(); + auto module = CreateNewVerifiedModule(); auto* computation = module->AddEntryComputation(builder.Build()); auto options = HloPrintOptions().set_print_metadata(false); @@ -540,7 +540,7 @@ TEST_F(HloComputationTest, StringificationIndent) { 2, PrecisionConfig::DEFAULT); builder.AddInstruction( HloInstruction::CreateDot(sout, x, reshape, dot_dnums, precision_config)); - auto module = CreateNewUnverifiedModule(); + auto module = CreateNewVerifiedModule(); auto* computation = module->AddEntryComputation(builder.Build()); auto options = @@ -576,7 +576,7 @@ TEST_F(HloComputationTest, StringificationCanonical) { 2, PrecisionConfig::DEFAULT); builder.AddInstruction( HloInstruction::CreateDot(sout, x, reshape, dot_dnums, precision_config)); - auto module = CreateNewUnverifiedModule(); + auto module = CreateNewVerifiedModule(); auto* computation = module->AddEntryComputation(builder.Build()); auto options = HloPrintOptions().set_print_metadata(false); diff --git a/tensorflow/compiler/xla/service/hlo_constant_folding_test.cc b/tensorflow/compiler/xla/service/hlo_constant_folding_test.cc index d12f920722e..4f81dc94e57 100644 --- a/tensorflow/compiler/xla/service/hlo_constant_folding_test.cc +++ b/tensorflow/compiler/xla/service/hlo_constant_folding_test.cc @@ -22,21 +22,22 @@ limitations under the License. #include "tensorflow/compiler/xla/literal.h" #include "tensorflow/compiler/xla/service/hlo_computation.h" #include "tensorflow/compiler/xla/service/hlo_instruction.h" -#include "tensorflow/compiler/xla/service/hlo_matchers.h" #include "tensorflow/compiler/xla/service/hlo_opcode.h" #include "tensorflow/compiler/xla/service/hlo_parser.h" #include "tensorflow/compiler/xla/service/hlo_pass_fix.h" +#include "tensorflow/compiler/xla/service/pattern_matcher.h" +#include "tensorflow/compiler/xla/service/pattern_matcher_gmock.h" #include "tensorflow/compiler/xla/shape_util.h" #include "tensorflow/compiler/xla/test.h" #include "tensorflow/compiler/xla/tests/hlo_test_base.h" #include "tensorflow/compiler/xla/tests/literal_test_util.h" #include "tensorflow/compiler/xla/types.h" -namespace op = xla::testing::opcode_matchers; - namespace xla { namespace { +namespace m = xla::match; + using HloConstantFoldingTest = HloTestBase; TEST_F(HloConstantFoldingTest, ConvertF32ToS64) { @@ -49,13 +50,14 @@ TEST_F(HloConstantFoldingTest, ConvertF32ToS64) { auto module = CreateNewVerifiedModule(); auto computation = module->AddEntryComputation(builder.Build()); - EXPECT_THAT(computation->root_instruction(), op::Convert(input)); + EXPECT_THAT(computation->root_instruction(), + GmockMatch(m::Convert().WithOperand(0, m::Op().Is(input)))); HloConstantFolding const_folder; TF_ASSERT_OK_AND_ASSIGN(bool result, const_folder.Run(module.get())); EXPECT_TRUE(result); - EXPECT_THAT(computation->root_instruction(), op::Constant()); + EXPECT_THAT(computation->root_instruction(), GmockMatch(m::Constant())); EXPECT_EQ(computation->root_instruction()->literal().GetFirstElement(), 42); } @@ -70,13 +72,14 @@ TEST_F(HloConstantFoldingTest, ConvertS64ToF32) { auto module = CreateNewVerifiedModule(); auto computation = module->AddEntryComputation(builder.Build()); - EXPECT_THAT(computation->root_instruction(), op::Convert(input)); + EXPECT_THAT(computation->root_instruction(), + GmockMatch(m::Convert().WithOperand(0, m::Op().Is(input)))); HloConstantFolding const_folder; TF_ASSERT_OK_AND_ASSIGN(bool result, const_folder.Run(module.get())); EXPECT_TRUE(result); - EXPECT_THAT(computation->root_instruction(), op::Constant()); + EXPECT_THAT(computation->root_instruction(), GmockMatch(m::Constant())); EXPECT_EQ(computation->root_instruction()->literal().GetFirstElement(), 42.0f); } @@ -91,13 +94,14 @@ TEST_F(HloConstantFoldingTest, ConvertF32ArrayToS64Array) { auto module = CreateNewVerifiedModule(); auto computation = module->AddEntryComputation(builder.Build()); - EXPECT_THAT(computation->root_instruction(), op::Convert(input)); + EXPECT_THAT(computation->root_instruction(), + GmockMatch(m::Convert().WithOperand(0, m::Op().Is(input)))); HloConstantFolding const_folder; TF_ASSERT_OK_AND_ASSIGN(bool result, const_folder.Run(module.get())); EXPECT_TRUE(result); - EXPECT_THAT(computation->root_instruction(), op::Constant()); + EXPECT_THAT(computation->root_instruction(), GmockMatch(m::Constant())); EXPECT_EQ(computation->root_instruction()->literal().Get({0}), 42); EXPECT_EQ(computation->root_instruction()->literal().Get({1}), 19); } @@ -138,7 +142,7 @@ TEST_F(HloConstantFoldingTest, Concatenate) { EXPECT_TRUE(result); HloInstruction* root = computation->root_instruction(); - EXPECT_THAT(root, op::Constant()); + EXPECT_THAT(root, GmockMatch(m::Constant())); EXPECT_TRUE(ShapeUtil::Equal(root->shape(), shape)); } } @@ -165,7 +169,7 @@ TEST_F(HloConstantFoldingTest, Slice) { EXPECT_TRUE(result); HloInstruction* root = computation->root_instruction(); - EXPECT_THAT(root, op::Constant()); + EXPECT_THAT(root, GmockMatch(m::Constant())); EXPECT_TRUE(ShapeUtil::Equal(root->shape(), shape)); } @@ -190,7 +194,7 @@ TEST_F(HloConstantFoldingTest, TransposeConstantFold) { EXPECT_TRUE(result); HloInstruction* root = computation->root_instruction(); - EXPECT_THAT(root, op::Constant()); + EXPECT_THAT(root, GmockMatch(m::Constant())); EXPECT_TRUE(ShapeUtil::Compatible(root->shape(), shape)); using NativeT = typename primitive_util::PrimitiveTypeToNative::type; @@ -240,7 +244,8 @@ TEST_F(HloConstantFoldingTest, ConstantFoldReduceNoLayout) { TF_ASSERT_OK_AND_ASSIGN(bool result, const_folder.Run(m.get())); EXPECT_FALSE(result); - EXPECT_THAT(m->entry_computation()->root_instruction(), op::Reduce()); + EXPECT_THAT(m->entry_computation()->root_instruction(), + GmockMatch(m::Reduce())); } const char* const kConstantFoldLargePad = R"( @@ -260,7 +265,7 @@ TEST_F(HloConstantFoldingTest, DoesNotFoldLargePad) { EXPECT_FALSE(result); EXPECT_THAT(module->entry_computation()->root_instruction(), - op::Pad(op::Constant(), op::Constant())); + GmockMatch(m::Pad(m::Constant(), m::Constant()))); } } // namespace diff --git a/tensorflow/compiler/xla/service/hlo_cost_analysis.cc b/tensorflow/compiler/xla/service/hlo_cost_analysis.cc index fdfb38b858c..df7d3826dba 100644 --- a/tensorflow/compiler/xla/service/hlo_cost_analysis.cc +++ b/tensorflow/compiler/xla/service/hlo_cost_analysis.cc @@ -419,6 +419,21 @@ Status HloCostAnalysis::HandleTranspose(const HloInstruction*) { } Status HloCostAnalysis::HandleAfterAll(const HloInstruction*) { + // This instruction is used to enforce ordering at compile time. No code is + // emitted. + current_should_compute_bottleneck_time_ = false; + current_properties_[kBytesAccessedKey] = 0; + current_properties_[kOptimalSecondsKey] = 0; + return Status::OK(); +} + +Status HloCostAnalysis::HandleAddDependency( + const HloInstruction* add_dependency) { + // This instruction is used to enforce ordering at compile time. No code is + // emitted. + current_should_compute_bottleneck_time_ = false; + current_properties_[kBytesAccessedKey] = 0; + current_properties_[kOptimalSecondsKey] = 0; return Status::OK(); } diff --git a/tensorflow/compiler/xla/service/hlo_cost_analysis.h b/tensorflow/compiler/xla/service/hlo_cost_analysis.h index 8ced9d776e1..33983119c9b 100644 --- a/tensorflow/compiler/xla/service/hlo_cost_analysis.h +++ b/tensorflow/compiler/xla/service/hlo_cost_analysis.h @@ -101,6 +101,7 @@ class HloCostAnalysis : public ConstDfsHloVisitor { Status HandleBroadcast(const HloInstruction* broadcast) override; Status HandlePad(const HloInstruction* pad) override; Status HandleReshape(const HloInstruction* reshape) override; + Status HandleAddDependency(const HloInstruction* add_dependency) override; Status HandleAfterAll(const HloInstruction* token) override; Status HandleTranspose(const HloInstruction* transpose) override; Status HandleWhile(const HloInstruction* xla_while) override; diff --git a/tensorflow/compiler/xla/service/hlo_cost_analysis_test.cc b/tensorflow/compiler/xla/service/hlo_cost_analysis_test.cc index 6a15b3440c6..ff32faf298d 100644 --- a/tensorflow/compiler/xla/service/hlo_cost_analysis_test.cc +++ b/tensorflow/compiler/xla/service/hlo_cost_analysis_test.cc @@ -387,7 +387,7 @@ TEST_F(FusionCostAnalysis, LoopFusion) { HloInstruction::CreateBinary(r2f32, HloOpcode::kSubtract, mul, clamp)); auto tuple = HloInstruction::CreateTuple({sub, sub, mul, c1}); - auto module = CreateNewUnverifiedModule(); + auto module = CreateNewVerifiedModule(); auto* computation = module->AddEntryComputation(builder.Build()); auto* fusion = computation->CreateFusionInstruction( {sub, mul, exp, clamp, add}, HloInstruction::FusionKind::kLoop); @@ -429,7 +429,7 @@ TEST_F(FusionCostAnalysis, NoLayout) { auto add = builder.AddInstruction(HloInstruction::CreateBinary( shape_with_layout, HloOpcode::kAdd, c1, broadcast)); - auto module = CreateNewUnverifiedModule(); + auto module = CreateNewVerifiedModule(); auto* computation = module->AddEntryComputation(builder.Build()); auto* fusion = computation->CreateFusionInstruction( {add, broadcast}, HloInstruction::FusionKind::kLoop); @@ -472,7 +472,7 @@ TEST_F(DomainCostAnalysis, DomainCost) { auto domain = builder.AddInstruction( HloInstruction::CreateDomain(tuple->shape(), tuple, nullptr, nullptr)); - auto hlo_module = CreateNewUnverifiedModule(); + auto hlo_module = CreateNewVerifiedModule(); hlo_module->AddEntryComputation(builder.Build()); EXPECT_EQ(hlo_module->entry_computation()->root_instruction(), domain); diff --git a/tensorflow/compiler/xla/service/hlo_dataflow_analysis.cc b/tensorflow/compiler/xla/service/hlo_dataflow_analysis.cc index 5dcf6bc985f..3ed3d3c11c7 100644 --- a/tensorflow/compiler/xla/service/hlo_dataflow_analysis.cc +++ b/tensorflow/compiler/xla/service/hlo_dataflow_analysis.cc @@ -466,6 +466,21 @@ bool HloDataflowAnalysis::UpdateDomainValueSet(HloInstruction* domain) { return changed; } +bool HloDataflowAnalysis::UpdateAddDependencyValueSet( + HloInstruction* add_dependency) { + // AddDependency just forwards the value of its zero-th operand. + CHECK_EQ(add_dependency->opcode(), HloOpcode::kAddDependency); + const InstructionValueSet& operand_set = + GetInstructionValueSet(add_dependency->operand(0)); + InstructionValueSet& add_dependency_set = + GetInstructionValueSet(add_dependency); + if (operand_set != add_dependency_set) { + add_dependency_set = operand_set; + return true; + } + return false; +} + bool HloDataflowAnalysis::UpdateGetTupleElementValueSet(HloInstruction* gte) { CHECK_EQ(gte->opcode(), HloOpcode::kGetTupleElement); bool changed = false; @@ -622,6 +637,8 @@ bool HloDataflowAnalysis::UpdateInstructionValueSet( HloInstruction* instruction) { // Recompute from operands. switch (instruction->opcode()) { + case HloOpcode::kAddDependency: + return UpdateAddDependencyValueSet(instruction); case HloOpcode::kBitcast: return UpdateBitcastValueSet(instruction); case HloOpcode::kDomain: @@ -795,6 +812,7 @@ Status HloDataflowAnalysis::InitializeInstructionValueSets() { define_all_values(); } break; + case HloOpcode::kAddDependency: case HloOpcode::kWhile: case HloOpcode::kCall: case HloOpcode::kConditional: diff --git a/tensorflow/compiler/xla/service/hlo_dataflow_analysis.h b/tensorflow/compiler/xla/service/hlo_dataflow_analysis.h index abac398c04f..ece17fc4c3e 100644 --- a/tensorflow/compiler/xla/service/hlo_dataflow_analysis.h +++ b/tensorflow/compiler/xla/service/hlo_dataflow_analysis.h @@ -193,6 +193,7 @@ class HloDataflowAnalysis { bool UpdateSendValueSet(HloInstruction* send); bool UpdateTupleValueSet(HloInstruction* tuple); bool UpdateWhileValueSet(HloInstruction* xla_while); + bool UpdateAddDependencyValueSet(HloInstruction* add_dependency); // Propagate the dataflow through the module. void Propagate(); diff --git a/tensorflow/compiler/xla/service/hlo_dataflow_analysis_test.cc b/tensorflow/compiler/xla/service/hlo_dataflow_analysis_test.cc index 6422346c101..f7a1f19a6f5 100644 --- a/tensorflow/compiler/xla/service/hlo_dataflow_analysis_test.cc +++ b/tensorflow/compiler/xla/service/hlo_dataflow_analysis_test.cc @@ -43,7 +43,7 @@ using ::testing::UnorderedElementsAre; class HloDataflowAnalysisTest : public HloTestBase, public ::testing::WithParamInterface { protected: - HloDataflowAnalysisTest() : module_(CreateNewUnverifiedModule()) {} + HloDataflowAnalysisTest() : module_(CreateNewVerifiedModule()) {} // Run dataflow analysis on the member module. For convenience returns a // reference to the generated analysis stored in analysis_. @@ -1877,6 +1877,30 @@ TEST_P(HloDataflowAnalysisTest, NestedConditionals) { } } +TEST_P(HloDataflowAnalysisTest, AddDependency) { + string module_string = R"( +HloModule AddDependency +ENTRY %AddDependency (p: f32[3]) -> f32[3] { + %p = f32[3] parameter(0) + %token = token[] after-all() + ROOT %add_dep = f32[3] add-dependency(f32[3] %p, token[] %token) +} +)"; + TF_ASSERT_OK_AND_ASSIGN( + std::unique_ptr module, + ParseHloString(module_string, GetModuleConfigForTest())); + + TF_ASSERT_OK_AND_ASSIGN(std::unique_ptr analysis, + HloDataflowAnalysis::Run(*module)); + const HloInstruction* root = module->entry_computation()->root_instruction(); + EXPECT_EQ(root->opcode(), HloOpcode::kAddDependency); + + // The after-all and parameter should define a value. Add-dependency should + // not. + EXPECT_EQ(analysis->values().size(), 2); + EXPECT_FALSE(analysis->ValueIsDefinedAt(root)); +} + INSTANTIATE_TEST_CASE_P(HloDataflowAnalysisInstantiation, HloDataflowAnalysisTest, ::testing::Values(false, true)); diff --git a/tensorflow/compiler/xla/service/hlo_dce_test.cc b/tensorflow/compiler/xla/service/hlo_dce_test.cc index 6c8095d3977..1fa4259a3e4 100644 --- a/tensorflow/compiler/xla/service/hlo_dce_test.cc +++ b/tensorflow/compiler/xla/service/hlo_dce_test.cc @@ -59,7 +59,7 @@ TEST_F(HloDceTest, NoDeadCode) { builder.AddInstruction(HloInstruction::CreateBinary( constant1->shape(), HloOpcode::kAdd, constant1, constant2)); - auto module = CreateNewUnverifiedModule(); + auto module = CreateNewVerifiedModule(); auto computation = module->AddEntryComputation(builder.Build()); EXPECT_EQ(3, computation->instruction_count()); @@ -110,7 +110,7 @@ TEST_F(HloDceTest, DeadParameters) { builder.AddInstruction(HloInstruction::CreateUnary( live_param->shape(), HloOpcode::kNegate, live_param)); - auto module = CreateNewUnverifiedModule(); + auto module = CreateNewVerifiedModule(); auto computation = module->AddEntryComputation(builder.Build()); EXPECT_EQ(5, computation->instruction_count()); @@ -150,7 +150,7 @@ TEST_F(HloDceTest, ControlDependencies) { builder.AddInstruction(HloInstruction::CreateBinary( constant1->shape(), HloOpcode::kAdd, constant1, constant2)); - auto module = CreateNewUnverifiedModule(); + auto module = CreateNewVerifiedModule(); auto computation = module->AddEntryComputation(builder.Build()); // Add a control dependency between two instructions. @@ -175,7 +175,7 @@ TEST_F(HloDceTest, ControlDependencies) { // Tests that a dead call instruction is removed. TEST_F(HloDceTest, DeadInstructionWithCalledComputation) { - auto module = CreateNewUnverifiedModule(); + auto module = CreateNewVerifiedModule(); Shape shape = ShapeUtil::MakeShape(F32, {}); // Called computation for the call instruction. @@ -323,7 +323,7 @@ TEST_F(HloDceTest, CalledComputationWithNestedSideEffect) { } TEST_F(HloDceTest, RemoveDeadSubcomputation) { - auto module = CreateNewUnverifiedModule(); + auto module = CreateNewVerifiedModule(); HloComputation::Builder builder(TestName()); HloComputation::Builder subcomp_builder("reduction_subcomp"); diff --git a/tensorflow/compiler/xla/service/hlo_evaluator.cc b/tensorflow/compiler/xla/service/hlo_evaluator.cc index 7fcafafc097..3a7652a8dc8 100644 --- a/tensorflow/compiler/xla/service/hlo_evaluator.cc +++ b/tensorflow/compiler/xla/service/hlo_evaluator.cc @@ -39,6 +39,7 @@ limitations under the License. #include "tensorflow/compiler/xla/service/hlo_query.h" #include "tensorflow/compiler/xla/service/shape_inference.h" #include "tensorflow/compiler/xla/shape_util.h" +#include "tensorflow/compiler/xla/statusor.h" #include "tensorflow/compiler/xla/types.h" #include "tensorflow/compiler/xla/util.h" #include "tensorflow/compiler/xla/window_util.h" @@ -396,6 +397,16 @@ StatusOr HloEvaluator::EvaluateDotOp( return Evaluate(cloned_instruction.get()); } +Status HloEvaluator::HandleBitcast(HloInstruction* bitcast) { + const Literal& operand_literal = GetEvaluatedLiteralFor(bitcast->operand(0)); + Literal result(bitcast->shape()); + TF_RET_CHECK(operand_literal.size_bytes() == result.size_bytes()); + memcpy(result.untyped_data(), operand_literal.untyped_data(), + operand_literal.size_bytes()); + evaluated_[bitcast] = std::move(result); + return Status::OK(); +} + Status HloEvaluator::HandleParameter(HloInstruction* parameter) { CHECK_LT(parameter->parameter_number(), arg_literals_.size()); const Literal* input_literal = arg_literals_[parameter->parameter_number()]; @@ -1046,8 +1057,15 @@ Status HloEvaluator::HandleBroadcast(HloInstruction* broadcast) { return Status::OK(); } -Status HloEvaluator::HandleAfterAll(HloInstruction* token) { - evaluated_[token] = LiteralUtil::CreateToken(); +Status HloEvaluator::HandleAfterAll(HloInstruction* after_all) { + evaluated_[after_all] = LiteralUtil::CreateToken(); + return Status::OK(); +} + +Status HloEvaluator::HandleAddDependency(HloInstruction* add_dependency) { + // AddDedendency just forwards its zero-th operand. + evaluated_[add_dependency] = + GetEvaluatedLiteralFor(add_dependency->operand(0)).Clone(); return Status::OK(); } @@ -1279,10 +1297,10 @@ StatusOr EvaluateSortInternal(HloInstruction* sort, key_value_vector.push_back( std::make_pair(keys_data[i], values_data[i])); } - std::sort(key_value_vector.begin(), key_value_vector.end(), - [](const kv_pair& a, const kv_pair& b) { - return SafeLess(a.first, b.first); - }); + std::stable_sort(key_value_vector.begin(), key_value_vector.end(), + [](const kv_pair& a, const kv_pair& b) { + return SafeLess(a.first, b.first); + }); std::vector result_keys; // We use a InlinedVector here because we need to convert it to an // absl::Span later, and this would not work with std::vector. diff --git a/tensorflow/compiler/xla/service/hlo_evaluator.h b/tensorflow/compiler/xla/service/hlo_evaluator.h index 07f8d0aad4a..45ed8131dc6 100644 --- a/tensorflow/compiler/xla/service/hlo_evaluator.h +++ b/tensorflow/compiler/xla/service/hlo_evaluator.h @@ -144,6 +144,8 @@ class HloEvaluator : public DfsHloVisitorWithDefault { // Operations that are type-agnostic or always return a specific type, such as // HandleIsFinite where boolean is always returned. // + Status HandleBitcast(HloInstruction* bitcast) override; + Status HandleParameter(HloInstruction* parameter) override; Status HandleConstant(HloInstruction* constant) override; @@ -180,7 +182,9 @@ class HloEvaluator : public DfsHloVisitorWithDefault { Status HandleBroadcast(HloInstruction* broadcast) override; - Status HandleAfterAll(HloInstruction* token) override; + Status HandleAfterAll(HloInstruction* after_all) override; + + Status HandleAddDependency(HloInstruction* add_dependency) override; Status HandleSort(HloInstruction* sort) override; @@ -221,16 +225,7 @@ class HloEvaluator : public DfsHloVisitorWithDefault { const Literal& operand_literal) { const auto shape = instruction->shape(); const auto* operand = instruction->operand(0); - - // TODO(b/35950897, b/27796129): add DCHECK back once implicit broadcast is - // removed. - if (!ShapeUtil::SameDimensions(shape, operand->shape())) { - return Unimplemented( - "Implicit broadcasting is currently unsupported in HLO evaluator " - "Shape Mismatch: %s vs %s", - ShapeUtil::HumanString(shape), - ShapeUtil::HumanString(operand->shape())); - } + TF_RET_CHECK(ShapeUtil::SameDimensions(shape, operand->shape())); Literal result(shape); TF_RETURN_IF_ERROR( diff --git a/tensorflow/compiler/xla/service/hlo_evaluator_test.cc b/tensorflow/compiler/xla/service/hlo_evaluator_test.cc index d95b6ad04f2..4eaaab20ea0 100644 --- a/tensorflow/compiler/xla/service/hlo_evaluator_test.cc +++ b/tensorflow/compiler/xla/service/hlo_evaluator_test.cc @@ -22,6 +22,7 @@ limitations under the License. #include #include "absl/memory/memory.h" +#include "absl/strings/str_format.h" #include "tensorflow/compiler/xla/client/xla_builder.h" #include "tensorflow/compiler/xla/literal.h" #include "tensorflow/compiler/xla/reference_util.h" @@ -35,6 +36,7 @@ limitations under the License. #include "tensorflow/compiler/xla/test.h" #include "tensorflow/compiler/xla/tests/hlo_test_base.h" #include "tensorflow/compiler/xla/tests/literal_test_util.h" +#include "tensorflow/compiler/xla/tests/test_utils.h" #include "tensorflow/compiler/xla/types.h" #include "tensorflow/compiler/xla/util.h" #include "tensorflow/compiler/xla/xla_data.pb.h" @@ -2765,6 +2767,33 @@ ENTRY main { EXPECT_TRUE(LiteralTestUtil::Equal(arg, actual)); } +TEST_P(HloEvaluatorTest, Bitcast) { + // Regression test for b/114735354. + constexpr absl::string_view hlo_text_base = R"( +HloModule Bitcast + +ENTRY main { + param = %s[32,121]{1,0} parameter(0) + ROOT bitcast = %s[121,32,1]{0,1,2} bitcast(%s[32,121]{1,0} param) +} +)"; + string hlo_text; + if (use_bfloat16_) { + hlo_text = absl::StrFormat(hlo_text_base, "bf16", "bf16", "bf16"); + } else { + hlo_text = absl::StrFormat(hlo_text_base, "f32", "f32", "f32"); + } + TF_ASSERT_OK_AND_ASSIGN(m_, ParseAndReturnVerifiedModule(hlo_text)); + auto args = MakeFakeArguments(m_.get()).ConsumeValueOrDie(); + Literal actual = Evaluate({&args[0]}); + if (use_bfloat16_) { + EXPECT_TRUE( + absl::c_equal(args[0].data(), actual.data())); + } else { + EXPECT_TRUE(absl::c_equal(args[0].data(), actual.data())); + } +} + INSTANTIATE_TEST_CASE_P(HloEvaluatorTest_Instantiation, HloEvaluatorTest, ::testing::ValuesIn(use_bf16_params)); diff --git a/tensorflow/compiler/xla/service/hlo_evaluator_typed_visitor.h b/tensorflow/compiler/xla/service/hlo_evaluator_typed_visitor.h index ebed875eb49..b87fc3e3401 100644 --- a/tensorflow/compiler/xla/service/hlo_evaluator_typed_visitor.h +++ b/tensorflow/compiler/xla/service/hlo_evaluator_typed_visitor.h @@ -161,9 +161,6 @@ class HloEvaluatorTypedVisitor : public DfsHloVisitorWithDefault { HloOpcodeString(hlo_instruction->opcode())); } - // TODO(b/35950897): many of the stl functions used in the handlers are not - // overloaded for every XLA primitive type. - template ::value>::type* = nullptr> @@ -596,7 +593,7 @@ class HloEvaluatorTypedVisitor : public DfsHloVisitorWithDefault { return Status::OK(); } - Status HandleDivide(HloInstruction* divide) { + Status HandleDivide(HloInstruction* divide) override { return HandleDivide(divide); } @@ -1556,10 +1553,10 @@ class HloEvaluatorTypedVisitor : public DfsHloVisitorWithDefault { const auto& row_data = row_to_sort.data(); std::vector result_data(row_data.begin(), row_data.end()); - std::sort(result_data.begin(), result_data.end(), - [](const NativeT& a, const NativeT& b) { - return SafeLess(a, b); - }); + std::stable_sort(result_data.begin(), result_data.end(), + [](const NativeT& a, const NativeT& b) { + return SafeLess(a, b); + }); Literal sorted_row(ShapeUtil::MakeShape(keys->shape().element_type(), {sort_dim_elements})); sorted_row.PopulateR1(absl::Span(result_data)); @@ -2546,12 +2543,14 @@ class HloEvaluatorTypedVisitor : public DfsHloVisitorWithDefault { template ::value || - std::is_same::value || - std::is_same::value>::type* = nullptr> + std::is_integral::value || + std::is_floating_point::value>::type* = nullptr> Status HandleIota(HloInstruction* instruction) { auto* iota = Cast(instruction); - std::vector data(iota->shape().dimensions(iota->iota_dimension())); + // Avoid using std::vector since std::vector does not convert to + // absl::Span. + absl::InlinedVector data( + iota->shape().dimensions(iota->iota_dimension())); std::iota(data.begin(), data.end(), 0); auto result = LiteralUtil::CreateR1(data); @@ -2568,9 +2567,8 @@ class HloEvaluatorTypedVisitor : public DfsHloVisitorWithDefault { } template ::value || - std::is_same::value || - std::is_same::value)>::type* = nullptr> + !(std::is_integral::value || + std::is_floating_point::value)>::type* = nullptr> Status HandleIota(HloInstruction* iota) { return InvalidArgument("Unsupported type for iota"); } @@ -2722,17 +2720,8 @@ class HloEvaluatorTypedVisitor : public DfsHloVisitorWithDefault { const auto shape = instruction->shape(); const auto* lhs = instruction->operand(0); const auto* rhs = instruction->operand(1); - - // TODO(b/35950897, b/27796129): add DCHECK back once implicit broadcast - // is removed. - if (!(ShapeUtil::SameDimensions(shape, rhs->shape()) && - ShapeUtil::SameDimensions(lhs->shape(), rhs->shape()))) { - return Unimplemented( - "Implicit broadcasting is currently unsupported in HLO evaluator " - "Shape Mismatch: %s vs %s vs %s: ", - ShapeUtil::HumanString(shape), ShapeUtil::HumanString(lhs->shape()), - ShapeUtil::HumanString(rhs->shape())); - } + TF_RET_CHECK(ShapeUtil::SameDimensions(shape, rhs->shape())); + TF_RET_CHECK(ShapeUtil::SameDimensions(lhs->shape(), rhs->shape())); const Literal& lhs_literal = parent_->GetEvaluatedLiteralFor(lhs); const Literal& rhs_literal = parent_->GetEvaluatedLiteralFor(rhs); @@ -2756,19 +2745,9 @@ class HloEvaluatorTypedVisitor : public DfsHloVisitorWithDefault { const auto* lhs = instruction->operand(0); const auto* rhs = instruction->operand(1); const auto* ehs = instruction->operand(2); - - // TODO(b/35950897, b/27796129): add DCHECK back once implicit - // broadcast is removed. - if (!(ShapeUtil::SameDimensions(shape, lhs->shape()) && - ShapeUtil::SameDimensions(lhs->shape(), rhs->shape()) && - ShapeUtil::SameDimensions(rhs->shape(), ehs->shape()))) { - return Unimplemented( - "Implicit broadcasting is currently unsupported in HLO evaluator " - "Shape Mismatch: %s vs %s vs %s vs %s: ", - ShapeUtil::HumanString(shape), ShapeUtil::HumanString(lhs->shape()), - ShapeUtil::HumanString(rhs->shape()), - ShapeUtil::HumanString(ehs->shape())); - } + TF_RET_CHECK(ShapeUtil::SameDimensions(shape, lhs->shape())); + TF_RET_CHECK(ShapeUtil::SameDimensions(lhs->shape(), rhs->shape())); + TF_RET_CHECK(ShapeUtil::SameDimensions(rhs->shape(), ehs->shape())); const Literal& lhs_literal = parent_->GetEvaluatedLiteralFor(lhs); const Literal& rhs_literal = parent_->GetEvaluatedLiteralFor(rhs); diff --git a/tensorflow/compiler/xla/service/hlo_get_dimension_size_rewriter.cc b/tensorflow/compiler/xla/service/hlo_get_dimension_size_rewriter.cc new file mode 100644 index 00000000000..c919dbd82d3 --- /dev/null +++ b/tensorflow/compiler/xla/service/hlo_get_dimension_size_rewriter.cc @@ -0,0 +1,61 @@ +/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/compiler/xla/service/hlo_get_dimension_size_rewriter.h" + +#include "absl/algorithm/container.h" +#include "tensorflow/compiler/xla/literal_util.h" +#include "tensorflow/compiler/xla/service/hlo_instruction.h" +#include "tensorflow/compiler/xla/service/hlo_opcode.h" +#include "tensorflow/compiler/xla/service/shape_inference.h" + +namespace xla { + +namespace { + +StatusOr ReplaceGetSize(HloInstruction* instr) { + if (instr->opcode() != HloOpcode::kGetDimensionSize) { + return false; + } + HloComputation* computation = instr->parent(); + + TF_ASSIGN_OR_RETURN(auto legal_shape, + ShapeInference::InferGetDimensionSizeShape( + instr->operand(0)->shape(), instr->dimension())); + TF_RET_CHECK(ShapeUtil::Equal(instr->shape(), legal_shape)); + TF_RET_CHECK(ShapeUtil::HasPrimitiveType(instr->shape(), U32)); + uint32 size = instr->operand(0)->shape().dimensions(instr->dimension()); + HloInstruction* new_instr = computation->AddInstruction( + HloInstruction::CreateConstant(LiteralUtil::CreateR0(size))); + TF_RETURN_IF_ERROR(instr->ReplaceAllUsesWith(new_instr)); + return true; +} + +} // namespace + +StatusOr HloGetDimensionSizeRewriter::Run(HloModule* module) { + bool changed = false; + HloProto proto; + *proto.mutable_hlo_module() = module->ToProto(); + for (auto* computation : module->computations()) { + for (auto instruction : computation->instructions()) { + TF_ASSIGN_OR_RETURN(bool replaced, ReplaceGetSize(instruction)); + changed = changed || replaced; + } + } + return changed; +} + +} // namespace xla diff --git a/tensorflow/compiler/xla/service/hlo_get_dimension_size_rewriter.h b/tensorflow/compiler/xla/service/hlo_get_dimension_size_rewriter.h new file mode 100644 index 00000000000..30f44c23a83 --- /dev/null +++ b/tensorflow/compiler/xla/service/hlo_get_dimension_size_rewriter.h @@ -0,0 +1,36 @@ +/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_COMPILER_XLA_SERVICE_HLO_GET_DIMENSION_SIZE_REWRITER_H_ +#define TENSORFLOW_COMPILER_XLA_SERVICE_HLO_GET_DIMENSION_SIZE_REWRITER_H_ + +#include "tensorflow/compiler/xla/service/hlo_module.h" +#include "tensorflow/compiler/xla/service/hlo_pass_interface.h" + +namespace xla { + +// Pass to replace a kGetDimensionSize instruction with a constant instruction. +class HloGetDimensionSizeRewriter : public HloModulePass { + public: + absl::string_view name() const override { + return "hlo-get-dimension-size-rewriter"; + } + + StatusOr Run(HloModule* module) override; +}; + +} // namespace xla + +#endif // TENSORFLOW_COMPILER_XLA_SERVICE_HLO_GET_DIMENSION_SIZE_REWRITER_H_ diff --git a/tensorflow/compiler/xla/service/hlo_get_dimension_size_rewriter_test.cc b/tensorflow/compiler/xla/service/hlo_get_dimension_size_rewriter_test.cc new file mode 100644 index 00000000000..a86aebdd5b6 --- /dev/null +++ b/tensorflow/compiler/xla/service/hlo_get_dimension_size_rewriter_test.cc @@ -0,0 +1,83 @@ +/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/compiler/xla/service/hlo_get_dimension_size_rewriter.h" + +#include "tensorflow/compiler/xla/service/hlo_computation.h" +#include "tensorflow/compiler/xla/service/hlo_instruction.h" +#include "tensorflow/compiler/xla/service/hlo_matchers.h" +#include "tensorflow/compiler/xla/service/hlo_module.h" +#include "tensorflow/compiler/xla/service/hlo_opcode.h" +#include "tensorflow/compiler/xla/service/hlo_parser.h" +#include "tensorflow/compiler/xla/shape_util.h" +#include "tensorflow/compiler/xla/tests/hlo_test_base.h" +#include "tensorflow/compiler/xla/tests/literal_test_util.h" +#include "tensorflow/compiler/xla/tests/test_utils.h" +#include "tensorflow/compiler/xla/types.h" +#include "tensorflow/core/lib/core/status_test_util.h" +#include "tensorflow/core/platform/types.h" + +namespace xla { +namespace { + +namespace op = xla::testing::opcode_matchers; + +class HloGetDimensionSizeRewriterTest : public HloTestBase { + protected: + HloGetDimensionSizeRewriterTest() {} +}; + +TEST_F(HloGetDimensionSizeRewriterTest, Ok) { + auto module = ParseHloString(R"( +HloModule _ +ENTRY gds { + p = s32[3,4] parameter(0) + size0 = u32[] get-dimension-size(p), dimensions={0} + size1 = u32[] get-dimension-size(p), dimensions={1} + ROOT mul = u32[] multiply(size0, size1) +})") + .ValueOrDie(); + HloGetDimensionSizeRewriter pass; + EXPECT_TRUE(pass.Run(module.get()).ValueOrDie()); + EXPECT_THAT(module->entry_computation()->root_instruction(), + op::Multiply(op::Constant(), op::Constant())); +} + +TEST_F(HloGetDimensionSizeRewriterTest, IllegalType) { + auto module = ParseHloString(R"( +HloModule _ +ENTRY gds { + p = s32[3]{0} parameter(0) + ROOT gds = s64[] get-dimension-size(p), dimensions={0} +})") + .ValueOrDie(); + HloGetDimensionSizeRewriter pass; + EXPECT_FALSE(pass.Run(module.get()).ok()); +} + +TEST_F(HloGetDimensionSizeRewriterTest, IllegalDimension) { + auto module = ParseHloString(R"( +HloModule _ +ENTRY gds { + p = f32[2,5] parameter(0) + ROOT gds = u32[] get-dimension-size(p), dimensions={2} +})") + .ValueOrDie(); + HloGetDimensionSizeRewriter pass; + EXPECT_FALSE(pass.Run(module.get()).ok()); +} + +} // namespace +} // namespace xla diff --git a/tensorflow/compiler/xla/service/hlo_graph_dumper.cc b/tensorflow/compiler/xla/service/hlo_graph_dumper.cc index 05cc1593e4e..302eca656be 100644 --- a/tensorflow/compiler/xla/service/hlo_graph_dumper.cc +++ b/tensorflow/compiler/xla/service/hlo_graph_dumper.cc @@ -21,6 +21,7 @@ limitations under the License. #include #include #include +#include #include #include #include @@ -38,6 +39,7 @@ limitations under the License. #include "tensorflow/compiler/xla/service/hlo_instructions.h" #include "tensorflow/compiler/xla/service/hlo_module.h" #include "tensorflow/compiler/xla/service/hlo_tfgraph_builder.h" +#include "tensorflow/compiler/xla/service/pattern_matcher.h" #include "tensorflow/compiler/xla/shape_util.h" #include "tensorflow/compiler/xla/types.h" #include "tensorflow/compiler/xla/window_util.h" @@ -111,11 +113,6 @@ class NodeFilter { result == kSomeUsersOmitted; } - bool ShowFusionSubcomputation(const HloInstruction* instr) const { - CHECK_EQ(instr->opcode(), HloOpcode::kFusion); - return Show(instr) && !SomeOrAllOperandsOmitted(instr); - } - private: std::function filter_; }; @@ -240,34 +237,28 @@ string HtmlLikeStringSanitize(absl::string_view s) { // it to a short string lets us tell the user what the subcomputation is without // drawing it as a graph. optional MatchTrivialComputation(const HloComputation* computation) { + namespace m = match; + if (computation->instruction_count() != 3) { return nullopt; } - HloInstruction* root = computation->root_instruction(); - if (root->operand_count() != 2) { + const HloInstruction *param0, *param1; + if (!Match(root, m::Op() + .WithNumOperands(2) + .WithShape(m::Shape().IsEffectiveScalar()) + .WithBinaryOperandsAnyOrder( + m::Parameter(¶m0, 0) + .WithShape(m::Shape().IsEffectiveScalar()), + m::Parameter(¶m1, 1) + .WithShape(m::Shape().IsEffectiveScalar())))) { return nullopt; } - // Check that both of the operands to the root are parameters. - const HloInstruction* operand0 = root->operand(0); - const HloInstruction* operand1 = root->operand(1); - if (operand0->opcode() != HloOpcode::kParameter || - operand1->opcode() != HloOpcode::kParameter) { - return nullopt; - } - - // Check that the two operands of root are param0 and param1. All of the - // opcodes we recognize are commutative, so we're OK with either order. - auto n0 = operand0->parameter_number(); - auto n1 = operand1->parameter_number(); - if (!(n0 == 0 && n1 == 1) && !(n1 == 0 && n0 == 1)) { - return nullopt; - } - - // If the params are reversed, check that the operation being performed is - // commutative. - if (n0 == 1) { + // If the params are reversed (i.e. operand0 is param1 and operand1 is + // param0), check that the operation being performed is commutative. + if (root->operand(0) == param1) { + CHECK_EQ(root->operand(1), param0); switch (root->opcode()) { case HloOpcode::kLe: case HloOpcode::kGe: @@ -279,13 +270,6 @@ optional MatchTrivialComputation(const HloComputation* computation) { } } - // Check that the root and params are all effective scalars. - if (!ShapeUtil::IsEffectiveScalar(root->shape()) || - !ShapeUtil::IsEffectiveScalar(operand0->shape()) || - !ShapeUtil::IsEffectiveScalar(operand1->shape())) { - return nullopt; - } - // If we recognize the root's opcode, we've successfully pattern-matched! switch (root->opcode()) { case HloOpcode::kAdd: @@ -578,7 +562,7 @@ bool HloDotDumper::ShouldShowSubcomputation(const HloComputation* subcomp) { // Show the subcomputation if we're showing any of its members. return std::any_of( - computation_->instructions().begin(), computation_->instructions().end(), + subcomp->instructions().begin(), subcomp->instructions().end(), [&](const HloInstruction* instr) { return filter_.Show(instr); }); } @@ -987,6 +971,7 @@ ColorScheme HloDotDumper::GetInstructionColor(const HloInstruction* instr) { case HloOpcode::kGetTupleElement: case HloOpcode::kTrace: case HloOpcode::kAfterAll: + case HloOpcode::kAddDependency: case HloOpcode::kTuple: return kWhite; case HloOpcode::kBroadcast: @@ -1267,12 +1252,12 @@ const HloInstruction* HloDotDumper::GetNodeForEdge( class GraphRendererRegistry { public: - void AddRenderer(GraphRendererInterface* graph_renderer) { + void SetRenderer(std::shared_ptr graph_renderer) { tensorflow::mutex_lock lock(mu_); graph_renderer_ = graph_renderer; } - GraphRendererInterface* GetDefaultRenderer() { + std::shared_ptr GetDefaultRenderer() { tensorflow::mutex_lock lock(mu_); return graph_renderer_; } @@ -1284,20 +1269,21 @@ class GraphRendererRegistry { private: tensorflow::mutex mu_; - GraphRendererInterface* graph_renderer_ = nullptr; + std::shared_ptr graph_renderer_ GUARDED_BY(mu_); }; } // namespace -Registrar::Registrar(GraphRendererInterface* dumper) { - GraphRendererRegistry::Default()->AddRenderer(dumper); +Registrar::Registrar(std::shared_ptr dumper) { + GraphRendererRegistry::Default()->SetRenderer(dumper); } namespace { // Gets a NodeFilter that includes roughly all instructions whose distance from // root is <= radius. -NodeFilter MakeNodeFilter(const HloInstruction* root, int64 radius) { +NodeFilter MakeNodeRadiusAroundFilter(const HloInstruction* root, + int64 radius) { // First, find the neighborhood of nodes with distance from root <= radius. // These nodes are our initial set of "normal" nodes. std::unordered_map nodes; @@ -1404,6 +1390,56 @@ NodeFilter MakeNodeFilter(const HloInstruction* root, int64 radius) { }); } +// Gets a node filter that includes nodes on all paths from `from` to `to`. If +// the all-paths set contains more than max_nodes elements, includes the nodes +// on the shortest paths and sets hit_limit to true. +NodeFilter MakeNodeFromToFilter(const HloInstruction* from, + const HloInstruction* to, int64 max_nodes, + bool* hit_limit) { + *hit_limit = false; + + // Elements in the queue are paths through the graph. + std::deque> queue; + queue.push_front({from}); + + // Compute the set of nodes we want to show using a slightly-modified + // Djikstra's algorithm. The only real difference is, rather than stopping + // when we find a (shortest) path, we continue until we've found max_nodes + // nodes on some path. + std::unordered_set visited; + std::unordered_set to_display = {from, to}; + while (!queue.empty() && to_display.size() < max_nodes) { + std::vector path = std::move(queue.front()); + queue.pop_front(); + if (!visited.insert(path.back()).second) { + continue; + } + + for (const auto* user : path.back()->users()) { + if (user == to) { + auto it = path.begin(); + for (; it != path.end() && to_display.size() < max_nodes; ++it) { + to_display.insert(*it); + } + if (it != path.end()) { + *hit_limit = true; + } + } else if (!visited.count(user)) { + auto new_path = path; + new_path.push_back(user); + queue.push_back(std::move(new_path)); + } + } + } + + return NodeFilter([=](const HloInstruction* instr) { + if (instr == from || instr == to) { + return kHighlightNode; + } + return to_display.count(instr) ? kNormalNode : kHideNode; + }); +} + string SaveGraph(const string& graph, GraphRendererInterface::GraphKind graph_kind, const string& dest_path) { @@ -1483,7 +1519,7 @@ string DumpNeighborhoodAround(const HloInstruction& node, int radius, auto debug_options = node.GetModule()->config().debug_options(); string label = StrCat("Neighborhood of ", radius, " nodes around ", node.name()); - NodeFilter filter = MakeNodeFilter(&node, radius); + NodeFilter filter = MakeNodeRadiusAroundFilter(&node, radius); string graph = HloDotDumper(node.parent(), label, debug_options, show_backend_config, /*profile=*/nullptr, filter) @@ -1491,6 +1527,29 @@ string DumpNeighborhoodAround(const HloInstruction& node, int radius, return ExportGraph(graph, GraphRendererInterface::DOT_GRAPH, debug_options); } +string DumpAllPathsFromTo(const HloInstruction& from, const HloInstruction& to, + int64 max_nodes, bool show_backend_config) { + CHECK_EQ(from.parent(), to.parent()) << "Nodes must be in same computation!"; + auto debug_options = from.GetModule()->config().debug_options(); + + bool hit_limit = false; + NodeFilter filter = MakeNodeFromToFilter(&from, &to, max_nodes, &hit_limit); + string label; + if (!hit_limit) { + label = StrCat("All paths from ", from.name(), " to ", to.name()); + } else { + label = StrCat(max_nodes, " nodes on the shortest paths from ", from.name(), + " to ", to.name(), + "

***SHOWING ONLY A SUBSET OF ALL PATHS BETWEEN " + "NODES***

"); + } + string graph = + HloDotDumper(from.parent(), label, debug_options, show_backend_config, + /*profile=*/nullptr, filter) + .Dump(); + return ExportGraph(graph, GraphRendererInterface::DOT_GRAPH, debug_options); +} + void DumpText(const HloModule& module, const string& label, const string& directory_path, bool do_prefix) { Env* env = Env::Default(); diff --git a/tensorflow/compiler/xla/service/hlo_graph_dumper.h b/tensorflow/compiler/xla/service/hlo_graph_dumper.h index 0b11f34abb7..de1eefab776 100644 --- a/tensorflow/compiler/xla/service/hlo_graph_dumper.h +++ b/tensorflow/compiler/xla/service/hlo_graph_dumper.h @@ -66,6 +66,12 @@ string DumpGraph(const HloComputation& computation, const string& label, string DumpNeighborhoodAround(const HloInstruction& node, int radius, bool show_backend_config = false); +// Dumps nodes on any of the paths from `from` to `to`. If there are more than +// max_nodes on all paths, restricts to the max_nodes nodes on the shortest +// paths. +string DumpAllPathsFromTo(const HloInstruction& from, const HloInstruction& to, + int64 max_nodes, bool show_backend_config = false); + // Dumps the HloModule::ToString() as a file into the provided directory path // suffixed with the provided label. // @@ -87,13 +93,13 @@ void DumpText(const HloModule& module, const string& label, // Class that registers a graph renderer. class Registrar { public: - Registrar(GraphRendererInterface* dumper); + Registrar(std::shared_ptr dumper); }; -#define XLA_INTERNAL_REGISTER_GRAPH_RENDERER(factory, ctr, ...) \ - static ::xla::hlo_graph_dumper::Registrar \ - XLA_INTERNAL_REGISTER_GRAPH_RENDERER_NAME(ctr)(new factory, \ - ##__VA_ARGS__) +#define XLA_INTERNAL_REGISTER_GRAPH_RENDERER(factory, ctr, ...) \ + static ::xla::hlo_graph_dumper::Registrar \ + XLA_INTERNAL_REGISTER_GRAPH_RENDERER_NAME(ctr)( \ + std::make_shared(), ##__VA_ARGS__) // __COUNTER__ must go through another macro to be properly expanded #define XLA_INTERNAL_REGISTER_GRAPH_RENDERER_NAME(ctr) ___##ctr##__object_ diff --git a/tensorflow/compiler/xla/service/hlo_instruction.cc b/tensorflow/compiler/xla/service/hlo_instruction.cc index 26786ee950b..21b1dbc1676 100644 --- a/tensorflow/compiler/xla/service/hlo_instruction.cc +++ b/tensorflow/compiler/xla/service/hlo_instruction.cc @@ -93,7 +93,8 @@ StatusOr> HloInstruction::CreateFromProto( [&computation_map](int64 id) { return computation_map.contains(id); })) << proto.name() << " instruction references invalid computation id(s)"; - TF_RETURN_IF_ERROR(ShapeUtil::ValidateShapeWithOptionalLayout(proto.shape())); + Shape shape(proto.shape()); + TF_RETURN_IF_ERROR(ShapeUtil::ValidateShapeWithOptionalLayout(shape)); switch (opcode) { // Ops migrated to subclasses. @@ -101,23 +102,23 @@ StatusOr> HloInstruction::CreateFromProto( TF_RET_CHECK(proto.operand_ids_size() == 3) << "BatchNormTraining instruction should have 3 operands but sees " << proto.operand_ids_size(); - instruction = CreateBatchNormTraining( - proto.shape(), operands(0), operands(1), operands(2), proto.epsilon(), - proto.feature_index()); + instruction = + CreateBatchNormTraining(shape, operands(0), operands(1), operands(2), + proto.epsilon(), proto.feature_index()); break; case HloOpcode::kBatchNormInference: TF_RET_CHECK(proto.operand_ids_size() == 5) << "BatchNormInference instruction should have 5 operands but sees " << proto.operand_ids_size(); instruction = CreateBatchNormInference( - proto.shape(), operands(0), operands(1), operands(2), operands(3), + shape, operands(0), operands(1), operands(2), operands(3), operands(4), proto.epsilon(), proto.feature_index()); break; case HloOpcode::kBatchNormGrad: TF_RET_CHECK(proto.operand_ids_size() == 5) << "BatchNormGrad instruction should have 5 operands but sees " << proto.operand_ids_size(); - instruction = CreateBatchNormGrad(proto.shape(), operands(0), operands(1), + instruction = CreateBatchNormGrad(shape, operands(0), operands(1), operands(2), operands(3), operands(4), proto.epsilon(), proto.feature_index()); break; @@ -127,7 +128,7 @@ StatusOr> HloInstruction::CreateFromProto( << proto.operand_ids_size(); std::vector fft_length(proto.fft_length().begin(), proto.fft_length().end()); - instruction = CreateFft(proto.shape(), operands(0), proto.fft_type(), + instruction = CreateFft(shape, operands(0), proto.fft_type(), absl::Span(fft_length)); break; } @@ -148,7 +149,7 @@ StatusOr> HloInstruction::CreateFromProto( TF_RET_CHECK(proto.operand_ids_size() == 1) << "Recv instruction should have 1 operand but sees " << proto.operand_ids_size(); - instruction = CreateRecv(proto.shape().tuple_shapes(0), operands(0), + instruction = CreateRecv(shape.tuple_shapes(0), operands(0), proto.channel_id(), proto.is_host_transfer()); break; case HloOpcode::kRecvDone: @@ -161,7 +162,7 @@ StatusOr> HloInstruction::CreateFromProto( TF_RET_CHECK(proto.operand_ids_size() == 1) << "Reverse instruction should have 1 operand but sees " << proto.operand_ids_size(); - instruction = CreateReverse(proto.shape(), operands(0), + instruction = CreateReverse(shape, operands(0), std::vector(proto.dimensions().begin(), proto.dimensions().end())); break; @@ -170,7 +171,7 @@ StatusOr> HloInstruction::CreateFromProto( << "Concatenate instruction should have 1 dimension but sees " << proto.dimensions_size(); instruction = - CreateConcatenate(proto.shape(), all_operands(), proto.dimensions(0)); + CreateConcatenate(shape, all_operands(), proto.dimensions(0)); break; case HloOpcode::kReduce: TF_RET_CHECK(proto.operand_ids_size() % 2 == 0) @@ -188,7 +189,7 @@ StatusOr> HloInstruction::CreateFromProto( absl::MakeSpan(reduce_operands) .subspan(reduce_operands.size() / 2, reduce_operands.size()); instruction = - CreateReduce(proto.shape(), inputs, init_values, + CreateReduce(shape, inputs, init_values, std::vector(proto.dimensions().begin(), proto.dimensions().end()), computations(0)); @@ -203,7 +204,7 @@ StatusOr> HloInstruction::CreateFromProto( auto sort_operands = all_operands(); HloInstruction* keys = sort_operands[0]; instruction = CreateSort( - proto.shape(), proto.dimensions(0), keys, + shape, proto.dimensions(0), keys, absl::Span(sort_operands).subspan(1)); break; } @@ -212,7 +213,7 @@ StatusOr> HloInstruction::CreateFromProto( << "Transpose instruction should have 1 operand but sees " << proto.operand_ids_size(); instruction = - CreateTranspose(proto.shape(), operands(0), + CreateTranspose(shape, operands(0), std::vector(proto.dimensions().begin(), proto.dimensions().end())); break; @@ -221,7 +222,7 @@ StatusOr> HloInstruction::CreateFromProto( << "Broadcast instruction should have 1 operand but sees " << proto.operand_ids_size(); instruction = - CreateBroadcast(proto.shape(), operands(0), + CreateBroadcast(shape, operands(0), std::vector(proto.dimensions().begin(), proto.dimensions().end())); break; @@ -229,7 +230,7 @@ StatusOr> HloInstruction::CreateFromProto( TF_RET_CHECK(proto.called_computation_ids_size() == 1) << "Map instruction should have 1 called computation but sees " << proto.called_computation_ids_size(); - instruction = CreateMap(proto.shape(), all_operands(), computations(0)); + instruction = CreateMap(shape, all_operands(), computations(0)); break; case HloOpcode::kSlice: { TF_RET_CHECK(proto.operand_ids_size() == 1) @@ -242,8 +243,8 @@ StatusOr> HloInstruction::CreateFromProto( slice_limits.push_back(slice_dimensions.limit()); slice_strides.push_back(slice_dimensions.stride()); } - instruction = CreateSlice(proto.shape(), operands(0), slice_starts, - slice_limits, slice_strides); + instruction = CreateSlice(shape, operands(0), slice_starts, slice_limits, + slice_strides); break; } case HloOpcode::kConstant: { @@ -253,7 +254,7 @@ StatusOr> HloInstruction::CreateFromProto( Literal::CreateFromProto(proto.literal())); instruction = CreateConstant(std::move(literal)); } else { - instruction = absl::make_unique(proto.shape()); + instruction = absl::make_unique(shape); } break; } @@ -284,55 +285,54 @@ StatusOr> HloInstruction::CreateFromProto( tensorflow::gtl::FindPtrOrNull(computation_map, fusion_id); TF_RET_CHECK(fused_computation != nullptr) << "No fusion computation with id " << fusion_id; - instruction = CreateFusion(proto.shape(), fusion_kind, all_operands(), - fused_computation); + instruction = + CreateFusion(shape, fusion_kind, all_operands(), fused_computation); break; } case HloOpcode::kRng: - instruction = - CreateRng(proto.shape(), proto.distribution(), all_operands()); + instruction = CreateRng(shape, proto.distribution(), all_operands()); break; case HloOpcode::kParameter: - instruction = CreateParameter(proto.parameter_number(), proto.shape(), - proto.name()); + instruction = + CreateParameter(proto.parameter_number(), shape, proto.name()); break; case HloOpcode::kGetTupleElement: TF_RET_CHECK(proto.operand_ids_size() == 1) << "GetTupleElement instruction should have 1 operand but sees " << proto.operand_ids_size(); - instruction = CreateGetTupleElement(proto.shape(), operands(0), - proto.tuple_index()); + instruction = + CreateGetTupleElement(shape, operands(0), proto.tuple_index()); break; case HloOpcode::kReducePrecision: TF_RET_CHECK(proto.operand_ids_size() == 1) << "ReducePrecision instruction should have 1 operand but sees " << proto.operand_ids_size(); - instruction = - CreateReducePrecision(proto.shape(), operands(0), - proto.exponent_bits(), proto.mantissa_bits()); + instruction = CreateReducePrecision( + shape, operands(0), proto.exponent_bits(), proto.mantissa_bits()); break; case HloOpcode::kInfeed: { - TF_RET_CHECK(ShapeUtil::IsTuple(proto.shape()) && - (ShapeUtil::TupleElementCount(proto.shape()) == 2)) + TF_RET_CHECK(ShapeUtil::IsTuple(shape) && + (ShapeUtil::TupleElementCount(shape) == 2)) << "Infeed should have a tuple shape with 2 operands, but has: " - << proto.shape(); - const Shape& data_shape = - ShapeUtil::GetTupleElementShape(proto.shape(), 0); + << shape; + const Shape& data_shape = ShapeUtil::GetTupleElementShape(shape, 0); TF_RET_CHECK(proto.operand_ids_size() == 1) << "Infeed instruction should have 1 operand but sees " << proto.operand_ids_size(); instruction = CreateInfeed(data_shape, operands(0), proto.infeed_config()); } break; - case HloOpcode::kOutfeed: + case HloOpcode::kOutfeed: { TF_RET_CHECK(proto.operand_ids_size() == 2) << "Outfeed instruction should have 2 operands but sees " << proto.operand_ids_size(); + Shape outfeed_shape(proto.outfeed_shape()); TF_RETURN_IF_ERROR( - ShapeUtil::ValidateShapeWithOptionalLayout(proto.outfeed_shape())); - instruction = CreateOutfeed(proto.outfeed_shape(), operands(0), - operands(1), proto.outfeed_config()); + ShapeUtil::ValidateShapeWithOptionalLayout(outfeed_shape)); + instruction = CreateOutfeed(outfeed_shape, operands(0), operands(1), + proto.outfeed_config()); break; + } case HloOpcode::kCrossReplicaSum: { TF_RET_CHECK(proto.called_computation_ids_size() == 1) << "CrossReplicaSum should have 1 called computation but sees " @@ -342,7 +342,7 @@ StatusOr> HloInstruction::CreateFromProto( all_reduce_id = proto.all_reduce_id(); } instruction = CreateCrossReplicaSum( - proto.shape(), all_operands(), computations(0), + shape, all_operands(), computations(0), /*replica_groups=*/ std::vector(proto.replica_groups().begin(), proto.replica_groups().end()), @@ -352,7 +352,7 @@ StatusOr> HloInstruction::CreateFromProto( } case HloOpcode::kAllToAll: { instruction = CreateAllToAll( - proto.shape(), all_operands(), + shape, all_operands(), /*replica_groups=*/ std::vector(proto.replica_groups().begin(), proto.replica_groups().end())); @@ -368,8 +368,8 @@ StatusOr> HloInstruction::CreateFromProto( source_target_pairs[i].first = proto.source_target_pairs(i).source(); source_target_pairs[i].second = proto.source_target_pairs(i).target(); } - instruction = CreateCollectivePermute(proto.shape(), operands(0), - source_target_pairs); + instruction = + CreateCollectivePermute(shape, operands(0), source_target_pairs); break; } case HloOpcode::kConvolution: { @@ -382,7 +382,7 @@ StatusOr> HloInstruction::CreateFromProto( precision_config.mutable_operand_precision()->Resize( proto.operand_ids_size(), PrecisionConfig::DEFAULT); instruction = CreateConvolve( - proto.shape(), operands(0), operands(1), + shape, operands(0), operands(1), std::max(proto.feature_group_count(), 1), proto.window(), proto.convolution_dimension_numbers(), precision_config); break; @@ -394,7 +394,7 @@ StatusOr> HloInstruction::CreateFromProto( TF_RET_CHECK(proto.called_computation_ids_size() == 1) << "ReduceWindow should have 1 called computation but sees " << proto.called_computation_ids_size(); - instruction = CreateReduceWindow(proto.shape(), operands(0), operands(1), + instruction = CreateReduceWindow(shape, operands(0), operands(1), proto.window(), computations(0)); break; case HloOpcode::kSelectAndScatter: @@ -404,9 +404,9 @@ StatusOr> HloInstruction::CreateFromProto( TF_RET_CHECK(proto.called_computation_ids_size() == 2) << "SelectAndScatter should have 2 called computations but sees " << proto.called_computation_ids_size(); - instruction = CreateSelectAndScatter( - proto.shape(), operands(0), computations(0), proto.window(), - operands(1), operands(2), computations(1)); + instruction = CreateSelectAndScatter(shape, operands(0), computations(0), + proto.window(), operands(1), + operands(2), computations(1)); break; case HloOpcode::kCustomCall: if (proto.constrain_layout()) { @@ -414,16 +414,17 @@ StatusOr> HloInstruction::CreateFromProto( // vector of pointers essentially) so create a vector of shapes to pass // in. std::vector operand_shapes; - for (const Shape& shape : proto.operand_shapes_with_layout()) { - operand_shapes.push_back(shape); + for (const ShapeProto& shape_proto : + proto.operand_shapes_with_layout()) { + operand_shapes.emplace_back(shape_proto); } - instruction = CreateCustomCall( - proto.shape(), all_operands(), proto.custom_call_target(), - operand_shapes, proto.custom_call_opaque()); + instruction = + CreateCustomCall(shape, all_operands(), proto.custom_call_target(), + operand_shapes, proto.custom_call_opaque()); } else { - instruction = CreateCustomCall(proto.shape(), all_operands(), - proto.custom_call_target(), - proto.custom_call_opaque()); + instruction = + CreateCustomCall(shape, all_operands(), proto.custom_call_target(), + proto.custom_call_opaque()); } if (proto.has_window()) { static_cast(instruction.get()) @@ -443,8 +444,8 @@ StatusOr> HloInstruction::CreateFromProto( << "Pad instruction should have 2 operands but sees " << proto.operand_ids_size(); TF_RET_CHECK(proto.has_padding_config()); - instruction = CreatePad(proto.shape(), operands(0), operands(1), - proto.padding_config()); + instruction = + CreatePad(shape, operands(0), operands(1), proto.padding_config()); break; case HloOpcode::kDynamicSlice: { TF_RET_CHECK(proto.operand_ids_size() == 2) @@ -452,8 +453,8 @@ StatusOr> HloInstruction::CreateFromProto( << proto.operand_ids_size(); std::vector slice_sizes(proto.dynamic_slice_sizes_size()); absl::c_copy(proto.dynamic_slice_sizes(), slice_sizes.begin()); - instruction = CreateDynamicSlice(proto.shape(), operands(0), operands(1), - slice_sizes); + instruction = + CreateDynamicSlice(shape, operands(0), operands(1), slice_sizes); break; } case HloOpcode::kGather: { @@ -469,7 +470,7 @@ StatusOr> HloInstruction::CreateFromProto( for (int64 bound : proto.gather_slice_sizes()) { gather_slice_sizes.push_back(bound); } - instruction = CreateGather(proto.shape(), operands(0), operands(1), + instruction = CreateGather(shape, operands(0), operands(1), *gather_dimension_numbers, gather_slice_sizes); break; } @@ -485,16 +486,15 @@ StatusOr> HloInstruction::CreateFromProto( auto scatter_dimension_numbers = absl::make_unique( proto.scatter_dimension_numbers()); - instruction = - CreateScatter(proto.shape(), operands(0), operands(1), operands(2), - computations(0), *scatter_dimension_numbers); + instruction = CreateScatter(shape, operands(0), operands(1), operands(2), + computations(0), *scatter_dimension_numbers); break; } case HloOpcode::kIota: TF_RET_CHECK(proto.dimensions_size() == 1) << "Iota instruction should have 1 dimension but sees " << proto.dimensions_size(); - instruction = CreateIota(proto.shape(), proto.dimensions(0)); + instruction = CreateIota(shape, proto.dimensions(0)); break; case HloOpcode::kDot: { TF_RET_CHECK(proto.has_dot_dimension_numbers()) @@ -506,8 +506,8 @@ StatusOr> HloInstruction::CreateFromProto( precision_config.mutable_operand_precision()->Resize( proto.operand_ids_size(), PrecisionConfig::DEFAULT); instruction = absl::make_unique( - proto.shape(), operands(0), operands(1), - proto.dot_dimension_numbers(), precision_config); + shape, operands(0), operands(1), proto.dot_dimension_numbers(), + precision_config); break; } case HloOpcode::kDomain: { @@ -529,7 +529,7 @@ StatusOr> HloInstruction::CreateFromProto( exit_hlo_sharding = std::make_shared(sharding); } instruction = absl::make_unique( - proto.shape(), operands(0), + shape, operands(0), absl::make_unique(entry_hlo_sharding), absl::make_unique(exit_hlo_sharding)); break; @@ -537,11 +537,11 @@ StatusOr> HloInstruction::CreateFromProto( case HloOpcode::kGetDimensionSize: TF_RET_CHECK(proto.operand_ids_size() == 1); TF_RET_CHECK(proto.dimensions_size() == 1); - instruction = CreateGetDimensionSize(proto.shape(), operands(0), - proto.dimensions(0)); + instruction = + CreateGetDimensionSize(shape, operands(0), proto.dimensions(0)); break; default: { - instruction = absl::WrapUnique(new HloInstruction(opcode, proto.shape())); + instruction = absl::WrapUnique(new HloInstruction(opcode, shape)); for (const int64 operand_id : proto.operand_ids()) { instruction->AppendOperand(instruction_map.at(operand_id)); } @@ -855,6 +855,16 @@ HloInstruction::CreateCollectivePermute( new HloInstruction(HloOpcode::kAfterAll, ShapeUtil::MakeTokenShape())); } +/* static */ std::unique_ptr +HloInstruction::CreateAddDependency(HloInstruction* data_operand, + HloInstruction* token_operand) { + auto instruction = absl::WrapUnique( + new HloInstruction(HloOpcode::kAddDependency, data_operand->shape())); + instruction->AppendOperand(data_operand); + instruction->AppendOperand(token_operand); + return instruction; +} + /* static */ std::unique_ptr HloInstruction::CreateWhile( const Shape& shape, HloComputation* condition, HloComputation* body, HloInstruction* init) { @@ -1394,6 +1404,10 @@ std::unique_ptr HloInstruction::CloneWithNewOperands( clone = CreateAfterAll(new_operands); } break; + case HloOpcode::kAddDependency: + CHECK_EQ(new_operands.size(), 2); + clone = CreateAddDependency(new_operands[0], new_operands[1]); + break; } // SetupDerivedInstruction will setup the precision_config_ field. SetupDerivedInstruction(clone.get()); @@ -1680,6 +1694,7 @@ bool HloInstruction::IdenticalSlowPath( // This opcode has complex or special behavior so just return false. case HloOpcode::kAfterAll: + case HloOpcode::kAddDependency: return false; // Remaining instructions with special values. @@ -1745,6 +1760,26 @@ bool HloInstruction::IdenticalSlowPath( return false; } +uint64 HloInstruction::Hash() const { + using tensorflow::Hash64Combine; + + uint64 hash_value = Hash64Combine(0, static_cast(opcode())); + hash_value = Hash64Combine(hash_value, ShapeUtil::Hash(shape())); + + if (!IsCrossModuleAllReduce()) { + if (!operands().empty()) { + for (size_t i = 0; i < operands().size(); ++i) { + hash_value = Hash64Combine(hash_value, operand(i)->Hash()); + } + } + } + + hash_value = Hash64Combine(hash_value, InnerHash()); + return hash_value; +} + +uint64 HloInstruction::InnerHash() const { return 13; } + void HloInstruction::RemoveUser(HloInstruction* user) { auto set_it = user_set_.find(user); CHECK(set_it != user_set_.end()); @@ -1900,6 +1935,11 @@ void HloInstruction::set_while_body(HloComputation* computation) { called_computations_[kBodyComputationIndex] = computation; } +HloInstruction* HloInstruction::while_init() const { + CHECK_EQ(HloOpcode::kWhile, opcode_); + return operands_[0]; +} + HloComputation* HloInstruction::true_computation() const { CHECK_EQ(HloOpcode::kConditional, opcode_); return called_computations_[kTrueComputationIndex]; @@ -2214,7 +2254,7 @@ HloInstructionProto HloInstruction::ToProto() const { proto.set_id(unique_id_); proto.set_name(name_); proto.set_opcode(HloOpcodeString(opcode_)); - *proto.mutable_shape() = shape_; + *proto.mutable_shape() = shape_.ToProto(); for (const HloInstruction* operand : operands_) { proto.add_operand_ids(operand->unique_id()); } @@ -2462,6 +2502,8 @@ Status HloInstruction::Visit(DfsHloVisitorBase* visitor) { return visitor->HandleDomain(this); case HloOpcode::kAfterAll: return visitor->HandleAfterAll(this); + case HloOpcode::kAddDependency: + return visitor->HandleAddDependency(this); case HloOpcode::kIota: return visitor->HandleIota(this); case HloOpcode::kGetDimensionSize: @@ -2623,36 +2665,6 @@ Status HloInstruction::AcceptWithOperandOrder( return Status::OK(); } -namespace { - -// Returns true if the given order is a topological sort of the instructions -// it contains. -bool OrderIsTopologicalSort(const std::vector& order) { - // Create a map from instruction to its position in 'order'. - std::unordered_map order_position; - for (int i = 0; i < order.size(); i++) { - if (!order_position.insert({order[i], i}).second) { - // Instruction order[i] is duplicated in the order. - return false; - } - } - // Verify that the operand of each instruction in the order is also in the - // order *and* the operand's position is earlier (defs are before uses for - // all ops). - for (auto* instruction : order) { - for (auto* operand : instruction->operands()) { - if (!ContainsKey(order_position, operand) || - order_position.at(operand) >= order_position.at(instruction)) { - return false; - } - } - } - - return true; -} - -} // namespace - Status HloInstruction::Accept( const std::function& visitor_func) { FunctionVisitor visitor(visitor_func); @@ -3022,6 +3034,16 @@ const PrecisionConfig& HloInstruction::precision_config() const { LOG(FATAL) << "Unimplemented method."; } +PrecisionConfig* HloInstruction::mutable_precision_config() { + if (auto* convolution = DynCast(this)) { + return convolution->mutable_precision_config(); + } + if (auto* dot = DynCast(this)) { + return dot->mutable_precision_config(); + } + LOG(FATAL) << "Unimplemented method."; +} + HloModule* HloInstruction::GetModule() const { if (parent_) { return parent_->parent(); @@ -3064,6 +3086,10 @@ int64 HloInstruction::concatenate_dimension() const { return Cast(this)->concatenate_dimension(); } +int64 HloInstruction::dimension() const { + return Cast(this)->dimension(); +} + bool HloInstruction::IsRank2Transpose() const { auto transpose = DynCast(this); return transpose != nullptr && transpose->IsRank2Transpose(); @@ -3243,6 +3269,11 @@ absl::optional HloInstruction::all_reduce_id() const { return Cast(this)->all_reduce_id(); } +void HloInstruction::set_all_reduce_id( + const absl::optional& all_reduce_id) { + return Cast(this)->set_all_reduce_id(all_reduce_id); +} + const ConvolutionDimensionNumbers& HloInstruction::convolution_dimension_numbers() const { if (auto convolution = DynCast(this)) { diff --git a/tensorflow/compiler/xla/service/hlo_instruction.h b/tensorflow/compiler/xla/service/hlo_instruction.h index 818d4ede0f3..a54716217d6 100644 --- a/tensorflow/compiler/xla/service/hlo_instruction.h +++ b/tensorflow/compiler/xla/service/hlo_instruction.h @@ -770,6 +770,9 @@ class HloInstruction { static std::unique_ptr CreateGetDimensionSize( const Shape& shape, HloInstruction* operand, int64 dimension); + static std::unique_ptr CreateAddDependency( + HloInstruction* data_operand, HloInstruction* token_operand); + // Returns the opcode for this instruction. HloOpcode opcode() const { return opcode_; } @@ -883,11 +886,15 @@ class HloInstruction { return false; } - // Use an explicit loop rather than ContainerEquals, because copying around - // std::functions may be too expensive in some cases. - for (size_t i = 0; i < operands().size(); ++i) { - if (!eq_operands(operand(i), other.operand(i))) { - return false; + // Two AllReduces are Identical if they have the same all_reduce_id. + // Their operands don't have to be Identical. + if (!IsCrossModuleAllReduce()) { + // Use an explicit loop rather than ContainerEquals, because copying + // around std::functions may be too expensive in some cases. + for (size_t i = 0; i < operands().size(); ++i) { + if (!eq_operands(operand(i), other.operand(i))) { + return false; + } } } @@ -898,6 +905,12 @@ class HloInstruction { return IdenticalSlowPath(other, eq_computations); } + // Generates a hash value of an HLO instruction. Hash considers + // information on opcode, shape, operands, and typically a root instruction. + // This function returns the same hash value for equivalent HLO instructions, + // with respect to HloInstruction::Identical() method. + uint64 Hash() const; + // Returns whether the instruction has a constant operand. bool HasConstantOperand() const; @@ -997,6 +1010,8 @@ class HloInstruction { void set_while_condition(HloComputation* while_condition); void set_while_body(HloComputation* while_body); + HloInstruction* while_init() const; + // Gets/sets the true and false HloComputation for Conditional. The setters // should only be called by HloModule or HloComputation methods. // @@ -1257,6 +1272,7 @@ class HloInstruction { // superior. // Precondition: opcode must be kConvolution or kDot. const PrecisionConfig& precision_config() const; + PrecisionConfig* mutable_precision_config(); // Sets the debug metadata for this instruction. void set_metadata(const OpMetadata& metadata) { metadata_ = metadata; } @@ -1317,6 +1333,9 @@ class HloInstruction { // Delegates to HloConcatenateInstruction::concatenate_dimension. int64 concatenate_dimension() const; + // Delegates to HloGetDimensionSizeInstruction::dimension. + int64 dimension() const; + // Returns whether this instruction does a rank-2 transposition. bool IsRank2Transpose() const; @@ -1435,6 +1454,7 @@ class HloInstruction { // Delegates to HloAllReduceInstruction::all_reduce_id. absl::optional all_reduce_id() const; + void set_all_reduce_id(const absl::optional& all_reduce_id); // Returns data on the window in a windowed operation such as // convolution. @@ -1599,6 +1619,10 @@ class HloInstruction { const std::function& eq_computations) const; + // Generates a hash value specific to a particular type of an instruction. + // This function typically considers the inner root instruction. + virtual uint64 InnerHash() const; + // Creates an n-ary elementwise operation. static std::unique_ptr CreateNary( const Shape& shape, HloOpcode opcode, diff --git a/tensorflow/compiler/xla/service/hlo_instructions.cc b/tensorflow/compiler/xla/service/hlo_instructions.cc index 4c765aa375c..1ea02cf9c03 100644 --- a/tensorflow/compiler/xla/service/hlo_instructions.cc +++ b/tensorflow/compiler/xla/service/hlo_instructions.cc @@ -370,6 +370,11 @@ HloAllReduceInstruction::HloAllReduceInstruction( AppendComputation(reduce_computation); } +void HloAllReduceInstruction::set_all_reduce_id( + const absl::optional& all_reduce_id) { + all_reduce_id_ = all_reduce_id; +} + HloInstructionProto HloAllReduceInstruction::ToProto() const { HloInstructionProto proto = HloCollectiveInstruction::ToProto(); // Proto3 is so sad. @@ -1367,6 +1372,10 @@ bool HloFusionInstruction::IdenticalSlowPath( other.fused_instructions_computation()); } +uint64 HloFusionInstruction::InnerHash() const { + return fused_instructions_computation()->Hash(); +} + std::unique_ptr HloFusionInstruction::CloneWithNewOperandsImpl( const Shape& shape, absl::Span new_operands, HloCloneContext* context) const { @@ -1610,7 +1619,7 @@ HloOutfeedInstruction::HloOutfeedInstruction(const Shape& outfeed_shape, HloInstructionProto HloOutfeedInstruction::ToProto() const { HloInstructionProto proto = HloInstruction::ToProto(); proto.set_outfeed_config(outfeed_config()); - *proto.mutable_outfeed_shape() = outfeed_shape(); + *proto.mutable_outfeed_shape() = outfeed_shape().ToProto(); return proto; } @@ -1862,7 +1871,7 @@ HloInstructionProto HloCustomCallInstruction::ToProto() const { if (layout_constrained()) { proto.set_constrain_layout(true); for (const Shape& shape : operand_shapes_with_layout_) { - *proto.add_operand_shapes_with_layout() = shape; + *proto.add_operand_shapes_with_layout() = shape.ToProto(); } } return proto; diff --git a/tensorflow/compiler/xla/service/hlo_instructions.h b/tensorflow/compiler/xla/service/hlo_instructions.h index d43a8973ccf..b5c28137a14 100644 --- a/tensorflow/compiler/xla/service/hlo_instructions.h +++ b/tensorflow/compiler/xla/service/hlo_instructions.h @@ -252,6 +252,7 @@ class HloAllReduceInstruction : public HloCollectiveInstruction { } absl::optional all_reduce_id() const { return all_reduce_id_; } + void set_all_reduce_id(const absl::optional& all_reduce_id); // Returns a serialized representation of this instruction. HloInstructionProto ToProto() const override; @@ -742,6 +743,8 @@ class HloFusionInstruction : public HloInstruction { const HloInstruction& other, const std::function& eq_computations) const override; + uint64 InnerHash() const override; + // Implementation for non-common logic of CloneWithNewOperands. std::unique_ptr CloneWithNewOperandsImpl( const Shape& shape, absl::Span new_operands, @@ -954,6 +957,7 @@ class HloConvolutionInstruction : public HloInstruction { // information but it is presumed that the alternate lowering is strictly // superior. const PrecisionConfig& precision_config() const { return precision_config_; } + PrecisionConfig* mutable_precision_config() { return &precision_config_; } string ToCategory() const override; // Returns a serialized representation of this instruction. @@ -1325,6 +1329,7 @@ class HloDotInstruction : public HloInstruction { // information but it is presumed that the alternate lowering is strictly // superior. const PrecisionConfig& precision_config() const { return precision_config_; } + PrecisionConfig* mutable_precision_config() { return &precision_config_; } // Returns a serialized representation of this instruction. HloInstructionProto ToProto() const override; diff --git a/tensorflow/compiler/xla/service/hlo_lexer.h b/tensorflow/compiler/xla/service/hlo_lexer.h index 3e2f8bcd52f..d6a2b292a39 100644 --- a/tensorflow/compiler/xla/service/hlo_lexer.h +++ b/tensorflow/compiler/xla/service/hlo_lexer.h @@ -20,6 +20,7 @@ limitations under the License. #include "absl/strings/string_view.h" #include "tensorflow/compiler/xla/service/hlo_token.h" +#include "tensorflow/compiler/xla/shape.h" #include "tensorflow/compiler/xla/types.h" #include "tensorflow/compiler/xla/xla_data.pb.h" #include "tensorflow/core/platform/logging.h" diff --git a/tensorflow/compiler/xla/service/hlo_matchers.cc b/tensorflow/compiler/xla/service/hlo_matchers.cc index 5269cad94d3..d28e79d41ad 100644 --- a/tensorflow/compiler/xla/service/hlo_matchers.cc +++ b/tensorflow/compiler/xla/service/hlo_matchers.cc @@ -237,8 +237,4 @@ void PrintTo(const HloInstruction* inst, ::std::ostream* os) { *os << (inst ? inst->ToString() : "nullptr"); } -void PrintTo(HloInstruction* inst, ::std::ostream* os) { - PrintTo(const_cast(inst), os); -} - } // namespace xla diff --git a/tensorflow/compiler/xla/service/hlo_matchers.h b/tensorflow/compiler/xla/service/hlo_matchers.h index 170ec93a334..235efb19ce4 100644 --- a/tensorflow/compiler/xla/service/hlo_matchers.h +++ b/tensorflow/compiler/xla/service/hlo_matchers.h @@ -385,7 +385,6 @@ std::vector Pointers(const Container& container) { // Tell GMock to print HloInstruction* by value, so error messages are nice. // Has to be in the same namespace as 'HloInstruction'. void PrintTo(const HloInstruction* inst, ::std::ostream* os); -void PrintTo(HloInstruction* inst, ::std::ostream* os); } // namespace xla diff --git a/tensorflow/compiler/xla/service/hlo_memory_scheduler.cc b/tensorflow/compiler/xla/service/hlo_memory_scheduler.cc index 234fcd266aa..d2740bcce26 100644 --- a/tensorflow/compiler/xla/service/hlo_memory_scheduler.cc +++ b/tensorflow/compiler/xla/service/hlo_memory_scheduler.cc @@ -73,7 +73,7 @@ class ListScheduler { // Construct and return a memory-minimizing sequence of HLO instructions // containing the given HLO computation. static StatusOr Run( - const HloComputation& computation, + HloComputation* computation, const TuplePointsToAnalysis& points_to_analysis, const LogicalBuffer::SizeFunction& size_function, const absl::flat_hash_map& @@ -98,7 +98,7 @@ class ListScheduler { // comparison operators. using Priority = std::pair; - ListScheduler(const HloComputation& computation, + ListScheduler(HloComputation* computation, const TuplePointsToAnalysis& points_to_analysis, const LogicalBuffer::SizeFunction& size_function, const absl::flat_hash_map& @@ -111,7 +111,7 @@ class ListScheduler { // instruction. An HLO instruction "uses" a LogicalBuffer if the // LogicalBuffer is in an operand of the instruction as indicated by // points-to analysis. - for (auto* instruction : computation.instructions()) { + for (auto* instruction : computation->instructions()) { absl::flat_hash_set instr_uses; for (auto* operand : instruction->operands()) { points_to_analysis.GetPointsToSet(operand).ForEachElement( @@ -126,13 +126,13 @@ class ListScheduler { // Create map containing the number of unscheduled uses (hlo instructions) // of each logical buffer. - for (auto* instruction : computation.instructions()) { + for (auto* instruction : computation->instructions()) { for (auto* buffer : points_to_analysis.GetBuffersDefinedByInstruction(instruction)) { unscheduled_use_count_[buffer] = 0; } } - for (auto* instruction : computation.instructions()) { + for (auto* instruction : computation->instructions()) { for (const LogicalBuffer* buffer : buffer_uses_.at(instruction)) { ++unscheduled_use_count_[buffer]; } @@ -141,7 +141,7 @@ class ListScheduler { // Buffers live out of the computation have an implicit use at the end of // the computation. for (const LogicalBuffer* live_out_buffer : - points_to_analysis.GetPointsToSet(computation.root_instruction()) + points_to_analysis.GetPointsToSet(computation->root_instruction()) .CreateFlattenedSet()) { ++unscheduled_use_count_[live_out_buffer]; } @@ -157,7 +157,7 @@ class ListScheduler { // HloInstruction, plus some cached metadata, saved for the purposes of making // BytesFreedIfScheduled fast. struct ReadyListEntry { - const HloInstruction* instruction; + HloInstruction* instruction; // The total size of all buffers defined by this instruction. int64 bytes_defined; @@ -171,7 +171,7 @@ class ListScheduler { }; // Creates a ReadyListEntry for the given instruction. - ReadyListEntry MakeReadyListEntry(const HloInstruction* instruction) { + ReadyListEntry MakeReadyListEntry(HloInstruction* instruction) { ReadyListEntry entry; entry.instruction = instruction; @@ -250,13 +250,13 @@ class ListScheduler { // Populate the ready list with instructions which have no operands or // control predecessors. absl::flat_hash_map unscheduled_pred_count; - for (auto* instruction : computation_.instructions()) { + for (auto* instruction : computation_->instructions()) { // TODO(b/34466113): Replace this and above with successors() or // predecessors() when these methods are added to HloInstruction. - for (const HloInstruction* user : instruction->users()) { + for (HloInstruction* user : instruction->users()) { unscheduled_pred_count[user]++; } - for (const HloInstruction* succ : instruction->control_successors()) { + for (HloInstruction* succ : instruction->control_successors()) { unscheduled_pred_count[succ]++; } } @@ -275,7 +275,7 @@ class ListScheduler { ready_instructions[inst] = it; }; - for (auto* instruction : computation_.instructions()) { + for (auto* instruction : computation_->instructions()) { if (instruction->operands().empty() && instruction->control_predecessors().empty()) { add_to_ready_queue(instruction); @@ -287,7 +287,7 @@ class ListScheduler { // schedule. auto best_it = ready_queue.end(); --best_it; - const HloInstruction* best = best_it->second.instruction; + HloInstruction* best = best_it->second.instruction; VLOG(2) << "Schedule instruction: " << best->ToShortString() << " Bytes freed: " << best_it->first.first; ready_queue.erase(best_it); @@ -348,13 +348,13 @@ class ListScheduler { } } } - CHECK_EQ(schedule.size(), computation_.instruction_count()); - CHECK_EQ(scheduled_instructions_.size(), computation_.instruction_count()); + CHECK_EQ(schedule.size(), computation_->instruction_count()); + CHECK_EQ(scheduled_instructions_.size(), computation_->instruction_count()); return schedule; } - const HloComputation& computation_; + HloComputation* computation_; const TuplePointsToAnalysis& points_to_analysis_; const LogicalBuffer::SizeFunction& size_function_; // Computations are analyzed in post-order. When scheduling an instruction @@ -386,13 +386,13 @@ int64 SumLogicalBufferSizes( } StatusOr ScheduleComputationHelper( - const HloComputation& computation, + HloComputation* computation, const TuplePointsToAnalysis& points_to_analysis, const LogicalBuffer::SizeFunction& size_function, const MemorySchedulerAlgorithm& algorithm, const absl::flat_hash_map& memory_by_computation) { - VLOG(2) << "Computation: " << computation.name(); + VLOG(2) << "Computation: " << computation->name(); if (algorithm) { return algorithm(computation, points_to_analysis, size_function, memory_by_computation); @@ -404,17 +404,17 @@ StatusOr ScheduleComputationHelper( } // namespace StatusOr DFSMemoryScheduler( - const HloComputation& computation, + HloComputation* computation, const TuplePointsToAnalysis& points_to_analysis, const LogicalBuffer::SizeFunction& size_function, const absl::flat_hash_map& memory_by_computation) { // These variables are a hack to prevent overflows. int64 cumulative_total_size = 0; - int64 total_hlos = computation.parent()->instruction_count(); + int64 total_hlos = computation->parent()->instruction_count(); absl::flat_hash_map extra_users; absl::flat_hash_map total_sizes; - for (const HloInstruction* hlo : computation.MakeInstructionPostOrder()) { + for (const HloInstruction* hlo : computation->MakeInstructionPostOrder()) { if (ListScheduler::IgnoreInstruction(*hlo)) { extra_users[hlo] = 0; total_sizes[hlo] = 0; @@ -448,8 +448,8 @@ StatusOr DFSMemoryScheduler( total_sizes[hlo] = std::min(total_sizes[hlo], cumulative_total_size); extra_users[hlo] = std::min(extra_users[hlo], total_hlos); } - CHECK_EQ(extra_users.size(), computation.instruction_count()); - CHECK_EQ(total_sizes.size(), computation.instruction_count()); + CHECK_EQ(extra_users.size(), computation->instruction_count()); + CHECK_EQ(total_sizes.size(), computation->instruction_count()); // Construct a total order based on DFS post-order, visiting operands in // decreasing cumulative extra user order, and next by cumulative size, with a @@ -459,7 +459,7 @@ StatusOr DFSMemoryScheduler( sequence.push_back(hlo); return Status::OK(); }); - TF_RETURN_IF_ERROR(computation.AcceptWithOperandOrder( + TF_RETURN_IF_ERROR(computation->AcceptWithOperandOrder( &visitor, [&extra_users, &total_sizes](const HloInstruction* a, const HloInstruction* b) { if (extra_users[a] != extra_users[b]) { @@ -470,12 +470,12 @@ StatusOr DFSMemoryScheduler( } return a->name() < b->name(); })); - CHECK_EQ(sequence.size(), computation.instruction_count()); + CHECK_EQ(sequence.size(), computation->instruction_count()); return sequence; } // namespace xla StatusOr ListMemoryScheduler( - const HloComputation& computation, + HloComputation* computation, const TuplePointsToAnalysis& points_to_analysis, const LogicalBuffer::SizeFunction& size_function, const absl::flat_hash_map& @@ -485,16 +485,16 @@ StatusOr ListMemoryScheduler( } StatusOr PostOrderMemoryScheduler( - const HloComputation& computation, + HloComputation* computation, const TuplePointsToAnalysis& points_to_analysis, const LogicalBuffer::SizeFunction& size_function, const absl::flat_hash_map& memory_by_computation) { - return HloInstructionSequence(computation.MakeInstructionPostOrder()); + return HloInstructionSequence(computation->MakeInstructionPostOrder()); } StatusOr DefaultMemoryScheduler( - const HloComputation& computation, + HloComputation* computation, const TuplePointsToAnalysis& points_to_analysis, const LogicalBuffer::SizeFunction& size_function, const absl::flat_hash_map& @@ -513,7 +513,7 @@ StatusOr DefaultMemoryScheduler( memory_by_computation)); TF_ASSIGN_OR_RETURN(const int64 list_memory, HeapSimulator::MinimumMemoryForComputation( - computation, list_sequence, points_to_analysis, + *computation, list_sequence, points_to_analysis, size_function, &memory_by_computation)); VLOG(2) << "Min-memory list sequence: " << HumanReadableNumBytes(list_memory); @@ -522,7 +522,7 @@ StatusOr DefaultMemoryScheduler( size_function, memory_by_computation)); TF_ASSIGN_OR_RETURN(const int64 dfs_memory, HeapSimulator::MinimumMemoryForComputation( - computation, dfs_sequence, points_to_analysis, + *computation, dfs_sequence, points_to_analysis, size_function, &memory_by_computation)); VLOG(2) << "Min-memory dfs sequence: " << HumanReadableNumBytes(dfs_memory); @@ -532,7 +532,7 @@ StatusOr DefaultMemoryScheduler( memory_by_computation)); TF_ASSIGN_OR_RETURN(const int64 post_order_memory, HeapSimulator::MinimumMemoryForComputation( - computation, post_order_sequence, points_to_analysis, + *computation, post_order_sequence, points_to_analysis, size_function, &memory_by_computation)); VLOG(2) << "Min-memory post order sequence: " << HumanReadableNumBytes(post_order_memory); @@ -555,17 +555,17 @@ StatusOr DefaultMemoryScheduler( } StatusOr ScheduleModule( - const HloModule& module, const LogicalBuffer::SizeFunction& size_function, + HloModule* module, const LogicalBuffer::SizeFunction& size_function, const MemorySchedulerAlgorithm& algorithm) { - HloSchedule schedule(&module); + HloSchedule schedule(module); TF_ASSIGN_OR_RETURN(std::unique_ptr points_to_analysis, - TuplePointsToAnalysis::Run(&module)); + TuplePointsToAnalysis::Run(module)); absl::flat_hash_map memory_by_computation; - for (const auto* computation : module.MakeComputationPostOrder()) { + for (auto* computation : module->MakeComputationPostOrder()) { if (!computation->IsFusionComputation()) { TF_ASSIGN_OR_RETURN(HloInstructionSequence computation_sequence, ScheduleComputationHelper( - *computation, *points_to_analysis, size_function, + computation, *points_to_analysis, size_function, algorithm, memory_by_computation)); memory_by_computation[computation] = HeapSimulator::MinimumMemoryForComputation( @@ -583,11 +583,11 @@ StatusOr ScheduleModule( } StatusOr ScheduleComputation( - const HloComputation& computation, + HloComputation* computation, const LogicalBuffer::SizeFunction& size_function) { - CHECK(!computation.IsFusionComputation()); + CHECK(!computation->IsFusionComputation()); TF_ASSIGN_OR_RETURN(std::unique_ptr points_to_analysis, - TuplePointsToAnalysis::Run(computation.parent())); + TuplePointsToAnalysis::Run(computation->parent())); absl::flat_hash_map empty_map; return ScheduleComputationHelper(computation, *points_to_analysis, size_function, nullptr, empty_map); @@ -600,7 +600,7 @@ HloMemoryScheduler::HloMemoryScheduler( StatusOr HloMemoryScheduler::Run(HloModule* module) { TF_ASSIGN_OR_RETURN(HloSchedule schedule, - ScheduleModule(*module, size_function_, algorithm_)); + ScheduleModule(module, size_function_, algorithm_)); TF_RETURN_IF_ERROR(module->set_schedule(std::move(schedule))); return true; } diff --git a/tensorflow/compiler/xla/service/hlo_memory_scheduler.h b/tensorflow/compiler/xla/service/hlo_memory_scheduler.h index cca5dc49398..7227bfb27c7 100644 --- a/tensorflow/compiler/xla/service/hlo_memory_scheduler.h +++ b/tensorflow/compiler/xla/service/hlo_memory_scheduler.h @@ -36,14 +36,14 @@ namespace xla { // that describes buffer aliasing, together with a target-specific size function // that maps a tensor's logical size to its padded size. typedef std::function( - const HloComputation&, const TuplePointsToAnalysis&, + HloComputation*, const TuplePointsToAnalysis&, const LogicalBuffer::SizeFunction&, const absl::flat_hash_map&)> MemorySchedulerAlgorithm; // List scheduler StatusOr ListMemoryScheduler( - const HloComputation& computation, + HloComputation* computation, const TuplePointsToAnalysis& points_to_analysis, const LogicalBuffer::SizeFunction& size_function, const absl::flat_hash_map& @@ -51,7 +51,7 @@ StatusOr ListMemoryScheduler( // DFS-order scheduler StatusOr DFSMemoryScheduler( - const HloComputation& computation, + HloComputation* computation, const TuplePointsToAnalysis& points_to_analysis, const LogicalBuffer::SizeFunction& size_function, const absl::flat_hash_map& @@ -59,7 +59,7 @@ StatusOr DFSMemoryScheduler( // Naive Post Order scheduler StatusOr PostOrderMemoryScheduler( - const HloComputation& computation, + HloComputation* computation, const TuplePointsToAnalysis& points_to_analysis, const LogicalBuffer::SizeFunction& size_function, const absl::flat_hash_map& @@ -69,7 +69,7 @@ StatusOr PostOrderMemoryScheduler( // and the DFS scheduler, and chooses whichever returns a lower min-memory, // not accounting for fragmentation. StatusOr DefaultMemoryScheduler( - const HloComputation& computation, + HloComputation* computation, const TuplePointsToAnalysis& points_to_analysis, const LogicalBuffer::SizeFunction& size_function, const absl::flat_hash_map& @@ -79,13 +79,13 @@ StatusOr DefaultMemoryScheduler( // the computation. size_function is the function returning the number of bytes // required for a LogicalBuffer. StatusOr ScheduleModule( - const HloModule& module, const LogicalBuffer::SizeFunction& size_function, + HloModule* module, const LogicalBuffer::SizeFunction& size_function, const MemorySchedulerAlgorithm& algorithm = {}); // Computes the schedule for a single computation. // Currently only used by the GPU backend. StatusOr ScheduleComputation( - const HloComputation& computation, + HloComputation* computation, const LogicalBuffer::SizeFunction& size_function); // A pass which schedules the HLO instructions in a module. The HloModule's diff --git a/tensorflow/compiler/xla/service/hlo_memory_scheduler_test.cc b/tensorflow/compiler/xla/service/hlo_memory_scheduler_test.cc index 984a6266abb..bc0d7e2bc00 100644 --- a/tensorflow/compiler/xla/service/hlo_memory_scheduler_test.cc +++ b/tensorflow/compiler/xla/service/hlo_memory_scheduler_test.cc @@ -65,7 +65,7 @@ TEST_F(HloSchedulingTest, LastUseScheduledFirst) { auto sub = builder.AddInstruction( HloInstruction::CreateBinary(vec, HloOpcode::kSubtract, add, negate)); - auto module = CreateNewUnverifiedModule(); + auto module = CreateNewVerifiedModule(); module->AddEntryComputation(builder.Build()); HloMemoryScheduler scheduler([](const BufferValue& buffer) { @@ -78,7 +78,7 @@ TEST_F(HloSchedulingTest, LastUseScheduledFirst) { TF_ASSERT_OK(module->schedule().Verify()); // Verify that all instructions are in the sequence. - const std::vector& sequence = + const std::vector& sequence = module->schedule().sequence(module->entry_computation()).instructions(); EXPECT_EQ(module->entry_computation()->instruction_count(), sequence.size()); @@ -124,9 +124,9 @@ ENTRY root { }; TF_ASSERT_OK_AND_ASSIGN( HloSchedule schedule, - ScheduleModule(*module, size_fn, ListMemoryScheduler)); + ScheduleModule(module.get(), size_fn, ListMemoryScheduler)); // Verify that all instructions are in the sequence. - const std::vector& sequence = + const std::vector& sequence = schedule.sequence(module->entry_computation()).instructions(); EXPECT_EQ(module->entry_computation()->instruction_count(), sequence.size()); @@ -172,15 +172,16 @@ TEST_F(HloSchedulingTest, TuplesAreAccountedCorrectly) { builder.AddInstruction(HloInstruction::CreateBinary(r1f32, HloOpcode::kAdd, tuple_elm, abs_abs2)); - auto module = CreateNewUnverifiedModule(); + auto module = CreateNewVerifiedModule(); module->AddEntryComputation(builder.Build()); TF_ASSERT_OK_AND_ASSIGN(HloSchedule schedule, - ScheduleModule(*module, - [](const BufferValue& buffer) { - return ShapeUtil::ByteSizeOf( - buffer.shape(), TUPLE_SIZE); - }, - ListMemoryScheduler)); + ScheduleModule( + module.get(), + [](const BufferValue& buffer) { + return ShapeUtil::ByteSizeOf(buffer.shape(), + TUPLE_SIZE); + }, + ListMemoryScheduler)); // Verify that all instructions are in the sequence. EXPECT_EQ(module->entry_computation()->instruction_count(), @@ -218,19 +219,19 @@ TEST_F(HloSchedulingTest, MultiOutputFusionAccountedCorrectly) { builder.AddInstruction( HloInstruction::CreateBinary(r1f32, HloOpcode::kAdd, tuple_elm, exp)); - auto module = CreateNewUnverifiedModule(); + auto module = CreateNewVerifiedModule(); auto* computation = module->AddEntryComputation(builder.Build()); auto fusion = computation->CreateFusionInstruction( {tuple, mul, add}, HloInstruction::FusionKind::kLoop); TF_ASSERT_OK_AND_ASSIGN(HloSchedule schedule, - ScheduleModule(*module, - [](const BufferValue& buffer) { - return ShapeUtil::ByteSizeOf( - buffer.shape(), 2); - }, - ListMemoryScheduler)); + ScheduleModule( + module.get(), + [](const BufferValue& buffer) { + return ShapeUtil::ByteSizeOf(buffer.shape(), 2); + }, + ListMemoryScheduler)); // Verify that all instructions are in the sequence. EXPECT_EQ(module->entry_computation()->instruction_count(), @@ -252,7 +253,7 @@ TEST_F(HloSchedulingTest, HeapSimulatorAccountsForSubcomputations) { HloInstruction::CreateParameter(0, r1f32, "cond_param")); HloInstruction* zero_vector = cond_builder.AddInstruction(HloInstruction::CreateConstant( - LiteralUtil::CreateR2({{0, 0, 0, 0}}))); + LiteralUtil::CreateR1({0, 0, 0, 0}))); cond_builder.AddInstruction(HloInstruction::CreateBinary( ShapeUtil::MakeShape(PRED, {}), HloOpcode::kNe, cond_param, zero_vector)); auto cond_computation = module->AddEmbeddedComputation(cond_builder.Build()); @@ -284,7 +285,7 @@ TEST_F(HloSchedulingTest, HeapSimulatorAccountsForSubcomputations) { }; TF_ASSERT_OK_AND_ASSIGN( HloSchedule schedule, - ScheduleModule(*module, size_fn, ListMemoryScheduler)); + ScheduleModule(module.get(), size_fn, ListMemoryScheduler)); // Verify that all instructions are in the sequence. auto entry_computation = module->entry_computation(); EXPECT_EQ(module->entry_computation()->instruction_count(), diff --git a/tensorflow/compiler/xla/service/hlo_module.cc b/tensorflow/compiler/xla/service/hlo_module.cc index 14bf17f4be1..fe8371384c0 100644 --- a/tensorflow/compiler/xla/service/hlo_module.cc +++ b/tensorflow/compiler/xla/service/hlo_module.cc @@ -240,8 +240,10 @@ HloModuleProto HloModule::ToProto() const { *proto.mutable_schedule() = schedule().ToProto().ValueOrDie(); } *proto.mutable_host_program_shape() = - entry_computation_layout().ComputeProgramShape(); + entry_computation_layout().ComputeProgramShape().ToProto(); *proto.mutable_input_output_alias() = input_output_alias_config().ToProto(); + *proto.mutable_dynamic_parameter_binding() = + dynamic_parameter_binding().ToProto(); return proto; } @@ -255,7 +257,7 @@ StatusOr> HloModule::CreateFromProto( // the entry parameters and root. TF_RET_CHECK(proto.has_host_program_shape()) << "No program shape found in the proto"; - const auto& expected_program_shape = proto.host_program_shape(); + ProgramShape expected_program_shape(proto.host_program_shape()); TF_RET_CHECK(expected_program_shape.parameters_size() == module_config.entry_computation_layout().parameter_count()); for (int i = 0; i < expected_program_shape.parameters_size(); ++i) { @@ -325,6 +327,10 @@ StatusOr> HloModule::CreateFromProto( // Because we didn't uniquify the names or the ids, double-check that the // instruction and computation names and ids are unique from the proto. + TF_ASSIGN_OR_RETURN(module->dynamic_parameter_binding_, + DynamicParameterBinding::CreateFromProto( + proto.dynamic_parameter_binding())); + absl::flat_hash_set computation_names; absl::flat_hash_set instruction_names; absl::flat_hash_set computation_ids; @@ -363,9 +369,9 @@ StatusOr HloModule::CreateModuleConfigFromProto( const HloModuleProto& module, const DebugOptions& debug_options) { TF_RET_CHECK(module.has_host_program_shape()) << "No program shape found in the proto"; - const auto& program_shape = module.host_program_shape(); + ProgramShape program_shape(module.host_program_shape()); - HloModuleConfig module_config(program_shape); + HloModuleConfig module_config(ProgramShape{program_shape}); module_config.set_debug_options(debug_options); // The module config is constructed with default layouts regardless of what is diff --git a/tensorflow/compiler/xla/service/hlo_module.h b/tensorflow/compiler/xla/service/hlo_module.h index 8a1f999e3ab..7b9cbf9a53a 100644 --- a/tensorflow/compiler/xla/service/hlo_module.h +++ b/tensorflow/compiler/xla/service/hlo_module.h @@ -28,6 +28,7 @@ limitations under the License. #include "absl/types/optional.h" #include "absl/types/span.h" #include "tensorflow/compiler/xla/iterator_util.h" +#include "tensorflow/compiler/xla/service/dynamic_parameter_binding.h" #include "tensorflow/compiler/xla/service/hlo.pb.h" #include "tensorflow/compiler/xla/service/hlo_clone_context.h" #include "tensorflow/compiler/xla/service/hlo_computation.h" @@ -103,11 +104,7 @@ class HloModule { HloCloneContext* context = nullptr); // Return a pointer to the entry computation of the module. - const HloComputation* entry_computation() const { - CHECK_NE(nullptr, entry_computation_); - return entry_computation_; - } - HloComputation* entry_computation() { + HloComputation* entry_computation() const { CHECK_NE(nullptr, entry_computation_); return entry_computation_; } @@ -135,6 +132,12 @@ class HloModule { return config_.entry_computation_layout(); } + // Generates a hash value of an HLO module. Hash considers + // information on opcode, shape, operands, and typically a root instruction. + // This function returns the same hash value for equivalent HLO modules, + // with respect to HloInstruction::Identical() method. + uint64 Hash() const { return entry_computation()->Hash(); } + // Gets the computations in this module. // // Returns a view of HloComputation*s, so you can iterate over this in the @@ -232,6 +235,16 @@ class HloModule { return input_output_alias_config_; } + // DynamicParameterBinding holds the list of bindings that indicates which + // parameter dimensions are dynamic and which parameters represent their + // runtime value. + DynamicParameterBinding& dynamic_parameter_binding() { + return dynamic_parameter_binding_; + } + const DynamicParameterBinding& dynamic_parameter_binding() const { + return dynamic_parameter_binding_; + } + // Returns an id that is unique to this module across all modules created over // the lifetime of this process. int unique_id() const { return unique_id_; } @@ -285,6 +298,9 @@ class HloModule { // alias_config indicates the alias information of input/output buffers that // are expected from the module. HloInputOutputAliasConfig input_output_alias_config_; + + // Bindings for dynamic parameter mapping. + DynamicParameterBinding dynamic_parameter_binding_; }; } // namespace xla diff --git a/tensorflow/compiler/xla/service/hlo_module_test.cc b/tensorflow/compiler/xla/service/hlo_module_test.cc index 3ae67e4e5ee..620cb7e01ad 100644 --- a/tensorflow/compiler/xla/service/hlo_module_test.cc +++ b/tensorflow/compiler/xla/service/hlo_module_test.cc @@ -63,7 +63,7 @@ class HloModuleTest : public HloTestBase { TEST_F(HloModuleTest, OneComputationPostOrder) { // Create a module with a single computation. - auto module = CreateNewUnverifiedModule(); + auto module = CreateNewVerifiedModule(); auto computation = module->AddEntryComputation(CreateConstantComputation()); EXPECT_THAT(module->MakeComputationPostOrder(), @@ -72,7 +72,7 @@ TEST_F(HloModuleTest, OneComputationPostOrder) { TEST_F(HloModuleTest, TwoComputationsPostOrder) { // Create a module with two unconnected computations. - auto module = CreateNewUnverifiedModule(); + auto module = CreateNewVerifiedModule(); auto computation1 = module->AddEntryComputation(CreateConstantComputation()); auto computation2 = module->AddEmbeddedComputation(CreateConstantComputation()); @@ -88,7 +88,7 @@ TEST_F(HloModuleTest, TwoComputationsPostOrder) { TEST_F(HloModuleTest, CloneTest) { // Create and copy a module with a diamond call graph of computations. - auto module = CreateNewUnverifiedModule(); + auto module = CreateNewVerifiedModule(); auto computation1 = module->AddEmbeddedComputation(CreateConstantComputation()); auto computation2 = @@ -111,7 +111,7 @@ TEST_F(HloModuleTest, CloneTest) { } TEST_F(HloModuleTest, CloneHasFusion) { - auto module = CreateNewUnverifiedModule(); + auto module = CreateNewVerifiedModule(); // Create the fused computation. HloComputation* fused_computation; @@ -154,7 +154,7 @@ TEST_F(HloModuleTest, CloneHasFusion) { TEST_F(HloModuleTest, DiamondComputationsPostOrder) { // Create a module with a diamond call graph of computations. - auto module = CreateNewUnverifiedModule(); + auto module = CreateNewVerifiedModule(); auto computation1 = module->AddEmbeddedComputation(CreateConstantComputation()); auto computation2 = @@ -174,7 +174,7 @@ TEST_F(HloModuleTest, DiamondComputationsPostOrder) { TEST_F(HloModuleTest, LargeConstantToString) { // Create a module with a single computation. - auto module = CreateNewUnverifiedModule(); + auto module = CreateNewVerifiedModule(); auto builder = HloComputation::Builder("Constant"); std::vector values(16, 42.0); builder.AddInstruction( @@ -194,8 +194,8 @@ TEST_F(HloModuleTest, LargeConstantToString) { } TEST_F(HloModuleTest, UniqueModuleId) { - auto module_a = CreateNewUnverifiedModule(); - auto module_b = CreateNewUnverifiedModule(); + auto module_a = CreateNewVerifiedModule(); + auto module_b = CreateNewVerifiedModule(); EXPECT_NE(module_a->unique_id(), module_b->unique_id()); } diff --git a/tensorflow/compiler/xla/service/hlo_opcode.h b/tensorflow/compiler/xla/service/hlo_opcode.h index 70c7d70b41c..127cfd165a5 100644 --- a/tensorflow/compiler/xla/service/hlo_opcode.h +++ b/tensorflow/compiler/xla/service/hlo_opcode.h @@ -47,6 +47,8 @@ namespace xla { #define HLO_OPCODE_LIST(V) \ V(kAbs, "abs") \ V(kAdd, "add") \ + V(kAddDependency, "add-dependency") \ + V(kAfterAll, "after-all", kHloOpcodeIsVariadic) \ V(kAllToAll, "all-to-all") \ V(kAtan2, "atan2") \ V(kBatchNormGrad, "batch-norm-grad") \ @@ -84,7 +86,6 @@ namespace xla { V(kGather, "gather") \ V(kGe, "greater-than-or-equal-to", kHloOpcodeIsComparison) \ V(kGetDimensionSize, "get-dimension-size") \ - V(kAfterAll, "after-all", kHloOpcodeIsVariadic) \ V(kGetTupleElement, "get-tuple-element") \ V(kGt, "greater-than", kHloOpcodeIsComparison) \ V(kImag, "imag") \ diff --git a/tensorflow/compiler/xla/service/hlo_ordering.cc b/tensorflow/compiler/xla/service/hlo_ordering.cc index f5f99bece18..ca6a154809b 100644 --- a/tensorflow/compiler/xla/service/hlo_ordering.cc +++ b/tensorflow/compiler/xla/service/hlo_ordering.cc @@ -356,8 +356,7 @@ void SequentialHloOrdering::Initialize() { // Create a map from instruction to its order position. TF_DCHECK_OK(schedule_.Verify()); for (const auto& computation_sequence : schedule_.sequences()) { - const std::vector& order = - computation_sequence.second.instructions(); + const auto& order = computation_sequence.second.instructions(); for (int i = 0; i < order.size(); ++i) { InsertOrDie(&order_position_, order[i], i); } diff --git a/tensorflow/compiler/xla/service/hlo_ordering_test.cc b/tensorflow/compiler/xla/service/hlo_ordering_test.cc index 2ab8aa57f6e..3ca77e60cd5 100644 --- a/tensorflow/compiler/xla/service/hlo_ordering_test.cc +++ b/tensorflow/compiler/xla/service/hlo_ordering_test.cc @@ -53,7 +53,7 @@ TEST_F(HloOrderingTest, InstructionsInDifferentComputations) { // %c = Constant(42.0f) // // This results in a diamond-shaped callgraph. - auto module = CreateNewUnverifiedModule(); + auto module = CreateNewVerifiedModule(); const Shape scalar_shape = ShapeUtil::MakeShape(xla::F32, {}); auto builder_c = HloComputation::Builder("C"); @@ -126,7 +126,7 @@ TEST_F(HloOrderingTest, InstructionsInWhileComputations) { // %constant = Constant(1.0) // return While(%constant, body, condition) // - auto module = CreateNewUnverifiedModule(); + auto module = CreateNewVerifiedModule(); const Shape scalar_shape = ShapeUtil::MakeShape(xla::F32, {}); auto body_builder = HloComputation::Builder("body"); @@ -176,7 +176,7 @@ TEST_F(HloOrderingTest, InstructionsInWhileComputations) { TEST_F(HloOrderingTest, ParametersDefinedBeforeOthers) { // Entry parameter should always be defined before other instruction. - auto module = CreateNewUnverifiedModule(); + auto module = CreateNewVerifiedModule(); const Shape scalar_shape = ShapeUtil::MakeShape(xla::F32, {}); auto builder = HloComputation::Builder(TestName()); auto constant = builder.AddInstruction( @@ -209,7 +209,7 @@ TEST_F(HloOrderingTest, ValuesInWhileComputations) { // %while = While(%constant, body, condition) // %add = Add(%constant, %while) // - auto module = CreateNewUnverifiedModule(); + auto module = CreateNewVerifiedModule(); const Shape scalar_shape = ShapeUtil::MakeShape(xla::F32, {}); auto body_builder = HloComputation::Builder("body"); @@ -407,7 +407,7 @@ TEST_F(HloOrderingTest, // %dead = Constant(123.0) // // %root should interfere with %dead. - auto module = CreateNewUnverifiedModule(); + auto module = CreateNewVerifiedModule(); const Shape scalar_shape = ShapeUtil::MakeShape(xla::F32, {}); auto builder = HloComputation::Builder(TestName()); @@ -455,7 +455,7 @@ TEST_F(HloOrderingTest, // ROOT %call = call({%c}), subcomputation // // %root should interfere with %dead. - auto module = CreateNewUnverifiedModule(); + auto module = CreateNewVerifiedModule(); const Shape scalar_shape = ShapeUtil::MakeShape(xla::F32, {}); auto subbuilder = HloComputation::Builder(TestName() + ".sub"); diff --git a/tensorflow/compiler/xla/service/hlo_parser.cc b/tensorflow/compiler/xla/service/hlo_parser.cc index 4390145c6bd..9b5bb5d0bd6 100644 --- a/tensorflow/compiler/xla/service/hlo_parser.cc +++ b/tensorflow/compiler/xla/service/hlo_parser.cc @@ -47,11 +47,11 @@ const double kF16max = 65504; // Creates and returns a schedule created using the order of the instructions in // the HloComputation::instructions() vectors in the module. -HloSchedule ScheduleFromInstructionOrder(const HloModule* module) { +HloSchedule ScheduleFromInstructionOrder(HloModule* module) { HloSchedule schedule(module); - for (const HloComputation* computation : module->computations()) { + for (HloComputation* computation : module->computations()) { if (!computation->IsFusionComputation()) { - for (const HloInstruction* instruction : computation->instructions()) { + for (HloInstruction* instruction : computation->instructions()) { schedule.GetOrCreateSequence(computation).push_back(instruction); } } @@ -850,6 +850,15 @@ bool HloParser::ParseInstructionRhs(HloComputation::Builder* builder, } break; } + case HloOpcode::kAddDependency: { + if (!ParseOperands(&operands, /*expected_size=*/2) || + !ParseAttributes(attrs)) { + return false; + } + instruction = builder->AddInstruction( + HloInstruction::CreateAddDependency(operands[0], operands[1])); + break; + } case HloOpcode::kSort: { optional> dimensions; attrs["dimensions"] = {/*required=*/true, AttrTy::kBracedInt64List, diff --git a/tensorflow/compiler/xla/service/hlo_parser_test.cc b/tensorflow/compiler/xla/service/hlo_parser_test.cc index c59bdc0a0b3..ab71f011ac9 100644 --- a/tensorflow/compiler/xla/service/hlo_parser_test.cc +++ b/tensorflow/compiler/xla/service/hlo_parser_test.cc @@ -21,7 +21,8 @@ limitations under the License. #include "absl/strings/string_view.h" #include "tensorflow/compiler/xla/service/hlo_casting_utils.h" #include "tensorflow/compiler/xla/service/hlo_instructions.h" -#include "tensorflow/compiler/xla/service/hlo_matchers.h" +#include "tensorflow/compiler/xla/service/pattern_matcher.h" +#include "tensorflow/compiler/xla/service/pattern_matcher_gmock.h" #include "tensorflow/compiler/xla/window_util.h" #include "tensorflow/core/lib/core/status_test_util.h" #include "tensorflow/core/platform/test.h" @@ -29,7 +30,7 @@ limitations under the License. namespace xla { namespace { -namespace op = ::xla::testing::opcode_matchers; +namespace m = ::xla::match; using absl::string_view; struct TestData { @@ -195,7 +196,7 @@ ENTRY %add_constants () -> f32[] { R"(HloModule TupleConstant_module ENTRY %TupleConstant.v1 () -> (f32[2,1], f32[2]) { - ROOT %constant = (f32[2,1]{1,0}, f32[2]{0}) constant((f32[2,1], f32[2]) ( f32[2,1] { { 1 }, { 2 } }, {2, 42} )) + ROOT %constant = (f32[2,1]{1,0}, f32[2]{0}) constant((f32[2,1], f32[2]) ( f32[2,1] { {1}, {2} }, {2, 42} )) } )" @@ -587,7 +588,7 @@ ENTRY %DynamicUpdateSlice.v4 (input: s32[1,1,25,1], update: s32[1,1,2,1], start_ R"(HloModule BasicTraining_module ENTRY %BasicTraining.v4 () -> (f32[2,2,1,2], f32[2], f32[2]) { - %constant = f32[2,2,1,2]{3,2,1,0} constant(f32[2,2,1,2] { { /*i0=0*/ { /*i1=0*/ {1, 2} }, { /*i1=1*/ {3, 4} } }, { /*i0=1*/ { /*i1=0*/ {5, 6} }, { /*i1=1*/ {7, 8} } } }) + %constant = f32[2,2,1,2]{3,2,1,0} constant(f32[2,2,1,2] { { /*i0=0*/ { /*i1=0*/ { 1, 2 } }, { /*i1=1*/ { 3, 4 } } }, { /*i0=1*/ { /*i1=0*/ { 5, 6 } }, { /*i1=1*/ { 7, 8 } } } }) %constant.1 = f32[2]{0} constant({2, 3}) %constant.2 = f32[2]{0} constant({1, 2}) ROOT %batch-norm-training = (f32[2,2,1,2]{3,2,1,0}, f32[2]{0}, f32[2]{0}) batch-norm-training(f32[2,2,1,2]{3,2,1,0} %constant, f32[2]{0} %constant.1, f32[2]{0} %constant.2), epsilon=0.001, feature_index=3 @@ -1241,7 +1242,38 @@ ENTRY Sort { } )" + }, +// AfterAll with multiple operands +{ +"AfterAllWithMultipleOperands", +R"(HloModule AfterAllWithMultipleOperands + +ENTRY AfterAllWithMultipleOperands { + p0 = f32[] parameter(0) + token0 = token[] after-all() + token1 = token[] after-all() + ROOT after-all = token[] after-all(p0, token0, token1) } + +)" +}, +// AddDependency +// A dependency chain is created from 'neg' to 'exp' using tokens. +{ +"AddDependency", +R"(HloModule AddDependency + +ENTRY AddDependency { + p = f32[] parameter(0) + neg = f32[] negate(p) + token = token[] after-all(neg) + p_after_token = f32[] add-dependency(p, token) + exp = f32[] exponential(p_after_token) + ROOT sum = f32[] add(neg, exp) +} + +)" +}, }); // clang-format on } @@ -1862,7 +1894,8 @@ ENTRY ReduceR3ToR2 { )"; TF_ASSERT_OK_AND_ASSIGN(auto module, ParseHloString(original)); ASSERT_NE(module->entry_computation(), nullptr); - EXPECT_THAT(module->entry_computation()->root_instruction(), op::Reduce()); + EXPECT_THAT(module->entry_computation()->root_instruction(), + GmockMatch(m::Reduce())); } TEST_F(HloParserTest, ParseSharding) { @@ -1922,7 +1955,7 @@ TEST(HloParserSingleOpTest, SingleOp) { const HloComputation* computation = module->entry_computation(); ASSERT_NE(computation, nullptr); EXPECT_THAT(computation->root_instruction(), - op::Multiply(op::Parameter(0), op::Parameter(1))); + GmockMatch(m::Multiply(m::Parameter(0), m::Parameter(1)))); } TEST(HloParserSingleOpTest, SingleOpNoShapeProducesError) { @@ -1950,7 +1983,7 @@ TEST(HloParserSingleOpTest, SingleOpNoNames) { const HloComputation* computation = module->entry_computation(); ASSERT_NE(computation, nullptr); EXPECT_THAT(computation->root_instruction(), - op::Multiply(op::Parameter(0), op::Parameter(1))); + GmockMatch(m::Multiply(m::Parameter(0), m::Parameter(1)))); } TEST(HloParserSingleOpTest, CanonicalOp) { @@ -1959,7 +1992,7 @@ TEST(HloParserSingleOpTest, CanonicalOp) { const HloComputation* computation = module->entry_computation(); ASSERT_NE(computation, nullptr); EXPECT_THAT(computation->root_instruction(), - op::Multiply(op::Parameter(0), op::Parameter(1))); + GmockMatch(m::Multiply(m::Parameter(0), m::Parameter(1)))); EXPECT_EQ( computation->root_instruction()->ToString(HloPrintOptions::Canonical()), text); @@ -2013,7 +2046,11 @@ TEST(HloParserSingleOpTest, SingleOpWithNested) { const HloComputation* computation = module->entry_computation(); ASSERT_NE(computation, nullptr); EXPECT_THAT(computation->root_instruction(), - op::Fusion(op::Parameter(0), op::Parameter(1))); + GmockMatch(m::Op() + .WithOpcode(HloOpcode::kFusion) + .WithNumOperands(2) + .WithOperand(0, m::Parameter(0)) + .WithOperand(1, m::Parameter(1)))); } TEST(HloParserSingleOpTest, SingleOpWithNested_DoesNotExist) { @@ -2057,7 +2094,7 @@ TEST(HloParserSingleOpTest, ConvolutionTrivialFeatureGroupCount) { const HloComputation* computation = module->entry_computation(); ASSERT_NE(computation, nullptr); EXPECT_THAT(computation->root_instruction(), - op::Convolution(op::Parameter(0), op::Parameter(1))); + GmockMatch(m::Convolution(m::Parameter(0), m::Parameter(1)))); auto* convolution = Cast(computation->root_instruction()); EXPECT_EQ(convolution->feature_group_count(), 1); @@ -2121,8 +2158,10 @@ ENTRY %axpy.v5 (alpha: f32[], x: f32[2,4], y: f32[2,4]) -> f32[2,4] { module->schedule().is_computation_scheduled(module->entry_computation())); EXPECT_THAT( module->schedule().sequence(module->entry_computation()).instructions(), - ::testing::ElementsAre(op::Parameter(), op::Broadcast(), op::Parameter(), - op::Multiply(), op::Parameter(), op::Add())); + ::testing::ElementsAre( + GmockMatch(m::Parameter()), GmockMatch(m::Broadcast()), + GmockMatch(m::Parameter()), GmockMatch(m::Multiply()), + GmockMatch(m::Parameter()), GmockMatch(m::Add()))); } TEST_F(HloParserTest, IsScheduledIsTrueDifferentOrder) { @@ -2148,8 +2187,10 @@ ENTRY %axpy.v5 (alpha: f32[], x: f32[2,4], y: f32[2,4]) -> f32[2,4] { module->schedule().is_computation_scheduled(module->entry_computation())); EXPECT_THAT( module->schedule().sequence(module->entry_computation()).instructions(), - ::testing::ElementsAre(op::Parameter(), op::Parameter(), op::Parameter(), - op::Broadcast(), op::Multiply(), op::Add())); + ::testing::ElementsAre( + GmockMatch(m::Parameter()), GmockMatch(m::Parameter()), + GmockMatch(m::Parameter()), GmockMatch(m::Broadcast()), + GmockMatch(m::Multiply()), GmockMatch(m::Add()))); } TEST_F(HloParserTest, CustomCallWrongNumberofOperandConstraints) { diff --git a/tensorflow/compiler/xla/service/hlo_proto_util.cc b/tensorflow/compiler/xla/service/hlo_proto_util.cc index cf33668f5bf..981d06ce101 100644 --- a/tensorflow/compiler/xla/service/hlo_proto_util.cc +++ b/tensorflow/compiler/xla/service/hlo_proto_util.cc @@ -48,7 +48,7 @@ StatusOr> CreateModuleFromProto( return std::move(module); } -StatusOr> EntryComputationParameterShapes( +StatusOr> EntryComputationParameterShapes( const HloProto& hlo_proto) { if (!hlo_proto.has_hlo_module()) { return NotFound("HloProto missing HloModuleProto."); @@ -57,15 +57,16 @@ StatusOr> EntryComputationParameterShapes( return NotFound("HloProto missing program shape."); } - std::vector parameter_shapes; + std::vector parameter_shapes; const auto& program_shape = hlo_proto.hlo_module().host_program_shape(); - for (const Shape& shape : program_shape.parameters()) { + for (const ShapeProto& shape : program_shape.parameters()) { parameter_shapes.push_back(&shape); } return parameter_shapes; } -StatusOr EntryComputationOutputShape(const HloProto& hlo_proto) { +StatusOr EntryComputationOutputShape( + const HloProto& hlo_proto) { if (!hlo_proto.has_hlo_module()) { return NotFound("HloProto missing HloModuleProto."); } diff --git a/tensorflow/compiler/xla/service/hlo_proto_util.h b/tensorflow/compiler/xla/service/hlo_proto_util.h index 1db82dd6fca..31ea2aaffd9 100644 --- a/tensorflow/compiler/xla/service/hlo_proto_util.h +++ b/tensorflow/compiler/xla/service/hlo_proto_util.h @@ -43,12 +43,13 @@ StatusOr> CreateModuleFromProto( // Returns the shapes of the parameters of the entry computation. Shape pointers // refer to shapes inside of the given HloProto. -StatusOr> EntryComputationParameterShapes( +StatusOr> EntryComputationParameterShapes( const HloProto& hlo_proto); // Returns the shape of the output of the entry computation. The shape pointer // refers to the output shape inside of the given HloProto. -StatusOr EntryComputationOutputShape(const HloProto& hlo_proto); +StatusOr EntryComputationOutputShape( + const HloProto& hlo_proto); } // namespace xla diff --git a/tensorflow/compiler/xla/service/hlo_rematerialization.cc b/tensorflow/compiler/xla/service/hlo_rematerialization.cc index 49e46ecd00e..48add75523f 100644 --- a/tensorflow/compiler/xla/service/hlo_rematerialization.cc +++ b/tensorflow/compiler/xla/service/hlo_rematerialization.cc @@ -130,10 +130,10 @@ using ItemList = absl::InlinedVector; // before arbitrary elements. class InstructionList { public: - explicit InstructionList(const std::vector& order) { + explicit InstructionList(const HloInstructionSequence& order) { int64 position = 0; Item* last = nullptr; - for (const HloInstruction* inst : order) { + for (HloInstruction* inst : order.instructions()) { // Add a new item to the linked list. Item* item = new Item; item->next = nullptr; @@ -151,7 +151,7 @@ class InstructionList { // to be monotonically increasing through the list, and so is still useful // for quickly(-ish) determining the order of arbitrary instructions in // the list. - item->instruction = const_cast(inst); + item->instruction = inst; item->position = position; position++; @@ -927,7 +927,7 @@ Item* PickRematerializationCandidate( StatusOr HloRematerialization::ComputePeakMemory( const HloComputation* computation, - const std::vector& order) const { + const HloInstructionSequence& order) const { InstructionList instruction_list(order); MemoryUsageTracker tracker(computation, size_function_, *points_to_analysis_, instruction_list); @@ -971,8 +971,7 @@ StatusOr HloRematerialization::RematerializeComputation( << HumanReadableNumBytes(computation_peak_memory_.at(computation)); CHECK(!ContainsKey(rematerialized_computations_, computation)); - InstructionList instruction_list( - schedule->sequence(computation).instructions()); + InstructionList instruction_list(schedule->sequence(computation)); MemoryUsageTracker memory_tracker(computation, size_function_, *points_to_analysis_, instruction_list); bool changed = false; @@ -1184,7 +1183,7 @@ StatusOr HloRematerialization::RematerializeComputation( sequence.clear(); for (auto* item = instruction_list.first(); item != nullptr; item = instruction_list.next(item)) { - const HloInstruction* instruction = item->instruction; + HloInstruction* instruction = item->instruction; sequence.push_back(instruction); } rematerialized_computations_.insert(computation); @@ -1235,10 +1234,8 @@ StatusOr HloRematerialization::Run(HloModule* module) { if (node.context() == CallContext::kSequential) { TF_ASSIGN_OR_RETURN( computation_peak_memory_[node.computation()], - ComputePeakMemory(node.computation(), - module->schedule() - .sequence(node.computation()) - .instructions())); + ComputePeakMemory(node.computation(), module->schedule().sequence( + node.computation()))); } return Status::OK(); }, diff --git a/tensorflow/compiler/xla/service/hlo_rematerialization.h b/tensorflow/compiler/xla/service/hlo_rematerialization.h index 70d83c04f07..a07d348041b 100644 --- a/tensorflow/compiler/xla/service/hlo_rematerialization.h +++ b/tensorflow/compiler/xla/service/hlo_rematerialization.h @@ -87,9 +87,8 @@ class HloRematerialization : public HloModulePass { // peak memory is the maximum total size of all live HLO instruction values at // any program point. 'order' is the order in which the HLO instructions will // be emitted which is used to determine lifespans of HLO values. - StatusOr ComputePeakMemory( - const HloComputation* computation, - const std::vector& order) const; + StatusOr ComputePeakMemory(const HloComputation* computation, + const HloInstructionSequence& order) const; // Returns the peak memory usage of the called computations for the given // instruction. Zero is returned if the instruction calls no computations. diff --git a/tensorflow/compiler/xla/service/hlo_runner.cc b/tensorflow/compiler/xla/service/hlo_runner.cc index 3f0ca342b4c..5a9b820a9d7 100644 --- a/tensorflow/compiler/xla/service/hlo_runner.cc +++ b/tensorflow/compiler/xla/service/hlo_runner.cc @@ -205,6 +205,40 @@ StatusOr HloRunner::ExecuteWithDeviceBuffers( /*profile=*/profile); } +StatusOr HloRunner::ExecuteWithDeviceBuffers( + std::unique_ptr executable, + const absl::Span arguments, + ExecutionProfile* profile) { + // Get service run options. + se::Stream stream(backend().default_stream_executor()); + stream.Init(); + ServiceExecutableRunOptions service_run_options = + GetServiceRunOptionsForDevice(backend().default_device_ordinal(), &stream, + nullptr); + + TF_ASSIGN_OR_RETURN( + ScopedShapedBuffer retval, + executable->ExecuteOnStreamWrapper(&service_run_options, + /*profile=*/profile, arguments)); + TF_RETURN_IF_ERROR(stream.BlockHostUntilDone()); + return std::move(retval); +} + +StatusOr HloRunner::ExecuteWithDeviceBuffers( + std::unique_ptr executable, + const absl::Span arguments, + ExecutionProfile* profile) { + std::vector argument_pointers; + argument_pointers.reserve(arguments.size()); + for (const auto& argument : arguments) { + argument_pointers.push_back(&argument); + } + return ExecuteWithDeviceBuffers( + /*executable=*/std::move(executable), + /*arguments=*/argument_pointers, + /*profile=*/profile); +} + StatusOr> HloRunner::ExecuteReplicated( std::unique_ptr module, const ReplicatedExecuteOptions& options) { diff --git a/tensorflow/compiler/xla/service/hlo_runner.h b/tensorflow/compiler/xla/service/hlo_runner.h index 2e934bf66ae..bb792cf8c98 100644 --- a/tensorflow/compiler/xla/service/hlo_runner.h +++ b/tensorflow/compiler/xla/service/hlo_runner.h @@ -136,6 +136,21 @@ class HloRunner { const absl::Span arguments, bool run_hlo_passes = true, ExecutionProfile* profile = nullptr); + StatusOr ExecuteWithDeviceBuffers( + std::unique_ptr executable, + const absl::Span arguments, + ExecutionProfile* profile = nullptr); + + StatusOr ExecuteWithDeviceBuffers( + std::unique_ptr executable, + const absl::Span arguments, + ExecutionProfile* profile = nullptr); + + // Creates an executable object given an HLO module. If run_hlo_passes is + // true, the HLO passes will be run as part of compilation. + StatusOr> CreateExecutable( + std::unique_ptr module, bool run_hlo_passes); + // Executes a given HLO module into a set of replicas, and returns a map // with the replica number as key, and the corresponding returned literal as // value. @@ -152,11 +167,6 @@ class HloRunner { const Backend& backend() const; private: - // Creates an executable object given an HLO module. If run_hlo_passes is - // true, the HLO passes will be run before. - StatusOr> CreateExecutable( - std::unique_ptr module, bool run_hlo_passes); - // Creates a ServiceExecutableRunOptions object to configure a run on device, // using the provided stream object. If device_assignment is not nullptr, it // will be used to configure the replication parameters. Replicated executions diff --git a/tensorflow/compiler/xla/service/hlo_schedule.cc b/tensorflow/compiler/xla/service/hlo_schedule.cc index a5780b7551a..8f6eb974c51 100644 --- a/tensorflow/compiler/xla/service/hlo_schedule.cc +++ b/tensorflow/compiler/xla/service/hlo_schedule.cc @@ -46,8 +46,8 @@ namespace xla { << "No computation exists in HLO module with id " << computation_id; const HloComputation* computation = comp_it->second; - absl::flat_hash_map id_to_instruction; - for (const HloInstruction* instruction : computation->instructions()) { + absl::flat_hash_map id_to_instruction; + for (HloInstruction* instruction : computation->instructions()) { id_to_instruction[instruction->unique_id()] = instruction; } @@ -81,9 +81,8 @@ StatusOr HloSchedule::ToProto() const { return std::move(proto); } -void HloSchedule::set_sequence( - const HloComputation* computation, - absl::Span sequence) { +void HloSchedule::set_sequence(const HloComputation* computation, + absl::Span sequence) { set_sequence(computation, HloInstructionSequence(sequence)); } @@ -114,8 +113,8 @@ Status HloSchedule::UpdateComputationSchedule( const HloComputation* computation) { // Map from unique ID to HloInstruction pointer for instructions in the // computation. - absl::flat_hash_map id_to_instruction; - for (const HloInstruction* instruction : computation->instructions()) { + absl::flat_hash_map id_to_instruction; + for (HloInstruction* instruction : computation->instructions()) { InsertOrDie(&id_to_instruction, instruction->unique_id(), instruction); } @@ -128,7 +127,7 @@ Status HloSchedule::UpdateComputationSchedule( // Map from HloInstruction X to newly added instructions (instruction is in // computation, but not in schedule) which use X. If an instruction is not in // the map, then it has no users which are newly added instructions. - absl::flat_hash_map> + absl::flat_hash_map> new_instruction_uses; // For each newly added instruction, this is the count of the instruction's @@ -138,9 +137,9 @@ Status HloSchedule::UpdateComputationSchedule( // Create a worklist of newly added instructions which are ready to be added // to the schedule. Initialize worklist with those that have zero operands. - std::queue worklist; + std::queue worklist; - for (const HloInstruction* instruction : computation->instructions()) { + for (HloInstruction* instruction : computation->instructions()) { if (ids_in_schedule.count(instruction->unique_id()) == 0) { // This is a newly added instruction which is not in the schedule. if (instruction->operands().empty()) { @@ -161,17 +160,17 @@ Status HloSchedule::UpdateComputationSchedule( // Lambda which schedules all instructions on the worklist. auto schedule_worklist = [&]() { while (!worklist.empty()) { - const HloInstruction* instruction = worklist.front(); + HloInstruction* instruction = worklist.front(); worklist.pop(); new_sequence.push_back(instruction); - std::vector* new_users = + std::vector* new_users = tensorflow::gtl::FindOrNull(new_instruction_uses, instruction); if (new_users != nullptr) { // This just-scheduled instruction has users which are newly added to // the module. Update the number of unscheduled operands and push the // newly added instruction to the worklist if it is ready to // schedule. - for (const HloInstruction* new_user : *new_users) { + for (HloInstruction* new_user : *new_users) { unscheduled_operand_count.at(new_user)--; CHECK_GE(unscheduled_operand_count.at(new_user), 0); if (unscheduled_operand_count.at(new_user) == 0) { diff --git a/tensorflow/compiler/xla/service/hlo_schedule.h b/tensorflow/compiler/xla/service/hlo_schedule.h index 0a714101ee5..486ddbf499d 100644 --- a/tensorflow/compiler/xla/service/hlo_schedule.h +++ b/tensorflow/compiler/xla/service/hlo_schedule.h @@ -35,14 +35,14 @@ class HloInstructionSequence { public: HloInstructionSequence() = default; explicit HloInstructionSequence( - absl::Span instructions) { - for (const HloInstruction* instruction : instructions) { + absl::Span instructions) { + for (HloInstruction* instruction : instructions) { push_back(instruction); } } // Adds the instruction to the end of the sequence. - void push_back(const HloInstruction* instruction) { + void push_back(HloInstruction* instruction) { instruction_sequence_.push_back(instruction); id_sequence_.push_back(instruction->unique_id()); } @@ -56,7 +56,7 @@ class HloInstructionSequence { int64 size() const { return instruction_sequence_.size(); } // Returns the sequence of HLO instructions. - const std::vector& instructions() const { + const std::vector& instructions() const { return instruction_sequence_; } @@ -65,7 +65,7 @@ class HloInstructionSequence { private: // The sequence as HloInstructions. - std::vector instruction_sequence_; + std::vector instruction_sequence_; // The sequence of HLO instructions, represented by their unique IDs. The // sequence is stored as both HloInstructions and unique IDs because the @@ -98,7 +98,7 @@ class HloSchedule { // Sets the sequence for the given computation to the given sequence. void set_sequence(const HloComputation* computation, - absl::Span sequence); + absl::Span sequence); void set_sequence(const HloComputation* computation, HloInstructionSequence sequence); diff --git a/tensorflow/compiler/xla/service/hlo_schedule_test.cc b/tensorflow/compiler/xla/service/hlo_schedule_test.cc index 1424569ac1f..0e56e6f760e 100644 --- a/tensorflow/compiler/xla/service/hlo_schedule_test.cc +++ b/tensorflow/compiler/xla/service/hlo_schedule_test.cc @@ -56,10 +56,10 @@ ENTRY main { ParseHloString(module_str)); TF_ASSERT_OK_AND_ASSIGN( HloSchedule schedule, - ScheduleModule(*module, [](const BufferValue& buffer) { + ScheduleModule(module.get(), [](const BufferValue& buffer) { return ShapeUtil::ByteSizeOf(buffer.shape()); })); - const std::vector& entry_schedule = + const auto& entry_schedule = schedule.sequence(module->entry_computation()).instructions(); EXPECT_EQ(entry_schedule.size(), 6); @@ -90,7 +90,7 @@ ENTRY main { ParseHloString(module_str)); TF_ASSERT_OK_AND_ASSIGN( HloSchedule schedule, - ScheduleModule(*module, [](const BufferValue& buffer) { + ScheduleModule(module.get(), [](const BufferValue& buffer) { return ShapeUtil::ByteSizeOf(buffer.shape()); })); @@ -139,7 +139,7 @@ ENTRY main { ParseHloString(module_str)); TF_ASSERT_OK_AND_ASSIGN( HloSchedule schedule, - ScheduleModule(*module, [](const BufferValue& buffer) { + ScheduleModule(module.get(), [](const BufferValue& buffer) { return ShapeUtil::ByteSizeOf(buffer.shape()); })); @@ -183,7 +183,7 @@ ENTRY main { ParseHloString(module_str)); TF_ASSERT_OK_AND_ASSIGN( HloSchedule schedule, - ScheduleModule(*module, [](const BufferValue& buffer) { + ScheduleModule(module.get(), [](const BufferValue& buffer) { return ShapeUtil::ByteSizeOf(buffer.shape()); })); @@ -244,7 +244,7 @@ ENTRY %WhileLoop () -> s32[] { ParseHloString(module_str)); TF_ASSERT_OK_AND_ASSIGN( HloSchedule schedule, - ScheduleModule(*module, [](const BufferValue& buffer) { + ScheduleModule(module.get(), [](const BufferValue& buffer) { return ShapeUtil::ByteSizeOf(buffer.shape(), /*pointer_size=*/sizeof(void*)); })); @@ -313,7 +313,7 @@ ENTRY %WhileLoop () -> s32[] { ParseHloString(module_str)); TF_ASSERT_OK_AND_ASSIGN( HloSchedule schedule, - ScheduleModule(*module, [](const BufferValue& buffer) { + ScheduleModule(module.get(), [](const BufferValue& buffer) { return ShapeUtil::ByteSizeOf(buffer.shape(), /*pointer_size=*/sizeof(void*)); })); diff --git a/tensorflow/compiler/xla/service/hlo_sharding_metadata.cc b/tensorflow/compiler/xla/service/hlo_sharding_metadata.cc index 88329c89979..f5061304456 100644 --- a/tensorflow/compiler/xla/service/hlo_sharding_metadata.cc +++ b/tensorflow/compiler/xla/service/hlo_sharding_metadata.cc @@ -253,7 +253,7 @@ StatusOr ApplyShardingFromUsers(HloInstruction* instruction, instruction->shape(), HloSharding::AssignDevice(kUnassignedDevice)); for (HloInstruction* user : instruction->users()) { if (user->opcode() == HloOpcode::kDomain && - domain.exit_domains.count(const_cast(user)) > 0) { + domain.exit_domains.count(user) > 0) { // If a user is a domain and it is registered in the domain exits, then // the instruction sharding is taken directly from the domain, and no // further users need to be visited. diff --git a/tensorflow/compiler/xla/service/hlo_subcomputation_unification_test.cc b/tensorflow/compiler/xla/service/hlo_subcomputation_unification_test.cc index 11994d99c93..c1073911ea9 100644 --- a/tensorflow/compiler/xla/service/hlo_subcomputation_unification_test.cc +++ b/tensorflow/compiler/xla/service/hlo_subcomputation_unification_test.cc @@ -66,7 +66,7 @@ class HloSubcomputationUnificationTest : public HloTestBase { }; TEST_F(HloSubcomputationUnificationTest, UnifyIdentities) { - auto module = CreateNewUnverifiedModule(); + auto module = CreateNewVerifiedModule(); auto builder = HloComputation::Builder(TestName()); auto callee1 = @@ -103,7 +103,7 @@ TEST_F(HloSubcomputationUnificationTest, UnifyIdentities) { } TEST_F(HloSubcomputationUnificationTest, UnifyAdditions) { - auto module = CreateNewUnverifiedModule(); + auto module = CreateNewVerifiedModule(); auto builder = HloComputation::Builder(TestName()); auto callee1 = @@ -184,7 +184,7 @@ TEST_F(HloSubcomputationUnificationTest, DifferentParameterShapes) { // Regression test for b/31466798. Checks that entry_computation is still valid // after unification. TEST_F(HloSubcomputationUnificationTest, TwoIdenticalComputations) { - auto module = CreateNewUnverifiedModule(); + auto module = CreateNewVerifiedModule(); for (int i = 0; i < 2; ++i) { HloComputation::Builder builder("pow"); auto x = diff --git a/tensorflow/compiler/xla/service/hlo_value.h b/tensorflow/compiler/xla/service/hlo_value.h index b6670d409b9..1f01b0bb365 100644 --- a/tensorflow/compiler/xla/service/hlo_value.h +++ b/tensorflow/compiler/xla/service/hlo_value.h @@ -166,9 +166,6 @@ class HloValue : public BufferValue { // Whether this value is live out of the HLO module. bool live_out_of_module_ = false; - - // Whether this value is live out of its computation. - bool live_out_of_computation_ = false; }; std::ostream& operator<<(std::ostream& out, const HloValue& hlo_value); diff --git a/tensorflow/compiler/xla/service/hlo_verifier.cc b/tensorflow/compiler/xla/service/hlo_verifier.cc index 27fd685a69a..77db7b098a3 100644 --- a/tensorflow/compiler/xla/service/hlo_verifier.cc +++ b/tensorflow/compiler/xla/service/hlo_verifier.cc @@ -753,13 +753,19 @@ Status ShapeVerifier::HandleAfterAll(HloInstruction* token) { for (const HloInstruction* operand : token->operands()) { operand_shapes.push_back(&operand->shape()); } - return CheckShape(token, ShapeInference::InferAfterAllShape(operand_shapes)); + return CheckShape(token, ShapeUtil::MakeTokenShape()); +} + +Status ShapeVerifier::HandleAddDependency(HloInstruction* add_dependency) { + TF_RETURN_IF_ERROR(CheckOperandCount(add_dependency, 2)); + TF_RETURN_IF_ERROR(CheckIsTokenOperand(add_dependency, 1)); + return CheckShape(add_dependency, add_dependency->operand(0)->shape()); } Status ShapeVerifier::HandleGetDimensionSize(HloInstruction* get_size) { - return CheckShape( - get_size, ShapeInference::InferGetDimensionSizeShape( - get_size->operand(0)->shape(), get_size->dimensions(0))); + return CheckShape(get_size, + ShapeInference::InferGetDimensionSizeShape( + get_size->operand(0)->shape(), get_size->dimension())); } Status ShapeVerifier::CheckShape(const HloInstruction* instruction, @@ -1373,9 +1379,8 @@ class InstructionVerifier : public DfsHloVisitorWithDefault { const Layout& operand_layout = operand_shape.layout(); TF_RET_CHECK(LayoutUtil::Equal(result_layout, operand_layout)) << "Instruction shouldn't change layouts " - << instruction->ToString() << " From " - << ShapeUtil::HumanString(result_shape) << " To " - << ShapeUtil::HumanString(operand_shape); + << instruction->ToString() << " From " << result_shape << " To " + << operand_shape; } } } @@ -1426,6 +1431,8 @@ StatusOr HloVerifier::Run(HloModule* module) { return target_metadata_->ShapeSize(shape); })); + TF_RETURN_IF_ERROR(module->dynamic_parameter_binding().Verify(*module)); + return false; } diff --git a/tensorflow/compiler/xla/service/hlo_verifier.h b/tensorflow/compiler/xla/service/hlo_verifier.h index 9fbfd6a21c1..e4d0c3d6957 100644 --- a/tensorflow/compiler/xla/service/hlo_verifier.h +++ b/tensorflow/compiler/xla/service/hlo_verifier.h @@ -95,6 +95,7 @@ class ShapeVerifier : public DfsHloVisitor { Status HandleScatter(HloInstruction* scatter) override; Status HandleAfterAll(HloInstruction* token) override; Status HandleGetDimensionSize(HloInstruction* get_size) override; + Status HandleAddDependency(HloInstruction* add_dependency) override; Status FinishVisit(HloInstruction*) override { return Status::OK(); } diff --git a/tensorflow/compiler/xla/service/hlo_verifier_test.cc b/tensorflow/compiler/xla/service/hlo_verifier_test.cc index 5ddfe0a944f..4bc557e4e62 100644 --- a/tensorflow/compiler/xla/service/hlo_verifier_test.cc +++ b/tensorflow/compiler/xla/service/hlo_verifier_test.cc @@ -35,6 +35,10 @@ namespace { using ::testing::HasSubstr; +std::unique_ptr CreateUnverifiedModule() { + return absl::make_unique("module", HloModuleConfig()); +} + // This class cannot be converted to use HloTestBase. It explicitly // uses HloTestBase to create and test malformed HLOs. class HloVerifierTest : public HloTestBase { @@ -66,7 +70,7 @@ TEST_F(HloVerifierTest, NullInstructionParent) { HloInstruction::CreateParameter(0, scalar_shape, "param")); HloInstruction* negate = builder.AddInstruction( HloInstruction::CreateUnary(scalar_shape, HloOpcode::kNegate, param)); - auto module = CreateNewUnverifiedModule(); + auto module = CreateUnverifiedModule(); module->AddEntryComputation(builder.Build()); TF_ASSERT_OK(verifier().Run(module.get()).status()); @@ -85,7 +89,7 @@ TEST_F(HloVerifierTest, NullComputationParent) { HloInstruction::CreateParameter(0, scalar_shape, "param")); builder.AddInstruction( HloInstruction::CreateUnary(scalar_shape, HloOpcode::kNegate, param)); - auto module = CreateNewUnverifiedModule(); + auto module = CreateUnverifiedModule(); HloComputation* computation = module->AddEntryComputation(builder.Build()); TF_ASSERT_OK(verifier().Run(module.get()).status()); @@ -104,7 +108,7 @@ TEST_F(HloVerifierTest, DifferentOperandParents) { HloInstruction::CreateParameter(0, scalar_shape, "param")); HloInstruction* negate = builder.AddInstruction( HloInstruction::CreateUnary(scalar_shape, HloOpcode::kNegate, param)); - auto module = CreateNewUnverifiedModule(); + auto module = CreateUnverifiedModule(); module->AddEntryComputation(builder.Build()); HloComputation::Builder emb_builder(TestName()); @@ -138,7 +142,7 @@ TEST_F(HloVerifierTest, ResetsShapeVerifierState) { builder.AddInstruction( HloInstruction::CreateBinary(s2, HloOpcode::kMultiply, add, add)); - auto module = CreateNewUnverifiedModule(); + auto module = CreateUnverifiedModule(); module->AddEntryComputation(builder.Build()); // Run the verifier twice. It should fail both times, because it shouldn't @@ -303,7 +307,7 @@ TEST_F(HloVerifierTest, NegativeInteriorPaddingNotAllowed) { HloInstruction::CreateConstant(LiteralUtil::Zero(F32))), padding_config)); - auto module = CreateNewUnverifiedModule(); + auto module = CreateUnverifiedModule(); module->AddEntryComputation(builder.Build()); auto status = verifier().Run(module.get()).status(); @@ -327,7 +331,7 @@ TEST_F(HloVerifierTest, PadNegativeInteriorDilationNotAllowed) { HloInstruction::CreateConstant(LiteralUtil::Zero(F32).Clone())), padding_config)); - auto module = CreateNewUnverifiedModule(); + auto module = CreateUnverifiedModule(); module->AddEntryComputation(builder.Build()); EXPECT_THAT(verifier().Run(module.get()).status().error_message(), diff --git a/tensorflow/compiler/xla/service/indexed_array_analysis_test.cc b/tensorflow/compiler/xla/service/indexed_array_analysis_test.cc index 20cc18f9815..98246d5403e 100644 --- a/tensorflow/compiler/xla/service/indexed_array_analysis_test.cc +++ b/tensorflow/compiler/xla/service/indexed_array_analysis_test.cc @@ -481,8 +481,8 @@ ENTRY main { const char* expected_root_expression = R"( (scalar-indexed-const (constant s32[2,1,1,1,6] s32[2,1,1,1,6] { - { /*i0=0*/ { /*i1=0*/ { /*i2=0*/ {1, 2, 3, 4, 5, 6} } } }, - { /*i0=1*/ { /*i1=0*/ { /*i2=0*/ {1, 2, 3, 4, 5, 6} } } } }) + { /*i0=0*/ { /*i1=0*/ { /*i2=0*/ { 1, 2, 3, 4, 5, 6 } } } }, + { /*i0=1*/ { /*i1=0*/ { /*i2=0*/ { 1, 2, 3, 4, 5, 6 } } } } }) (reshape %indices to s32[]) 0->[]) )"; @@ -512,8 +512,8 @@ ENTRY main { const char* expected_root_expression = R"( (scalar-indexed-const (constant s32[2,1,1,6] s32[2,1,1,6] { - { /*i0=0*/ { /*i1=0*/ {1, 2, 3, 4, 5, 6} } }, - { /*i0=1*/ { /*i1=0*/ {1, 2, 3, 4, 5, 6} } } }) + { /*i0=0*/ { /*i1=0*/ { 1, 2, 3, 4, 5, 6 } } }, + { /*i0=1*/ { /*i1=0*/ { 1, 2, 3, 4, 5, 6 } } } }) (reshape %indices to s32[5]) 0->[2]) )"; diff --git a/tensorflow/compiler/xla/service/instruction_fusion.cc b/tensorflow/compiler/xla/service/instruction_fusion.cc index 7f2d7e7cffc..7559ed1bab8 100644 --- a/tensorflow/compiler/xla/service/instruction_fusion.cc +++ b/tensorflow/compiler/xla/service/instruction_fusion.cc @@ -103,7 +103,6 @@ bool IsAlwaysDuplicable(const HloInstruction& instruction) { case HloOpcode::kShiftRightLogical: case HloOpcode::kSlice: case HloOpcode::kSubtract: - case HloOpcode::kAfterAll: case HloOpcode::kTranspose: case HloOpcode::kTuple: case HloOpcode::kTupleSelect: @@ -116,7 +115,10 @@ bool IsAlwaysDuplicable(const HloInstruction& instruction) { case HloOpcode::kSin: return ShapeUtil::ElementIsComplex(instruction.shape()); - // Expensive instructions. + // Expensive instructions or unusual instructions for which fusion is + // nonsensical. + case HloOpcode::kAddDependency: + case HloOpcode::kAfterAll: case HloOpcode::kAtan2: case HloOpcode::kBatchNormGrad: case HloOpcode::kBatchNormInference: @@ -455,8 +457,13 @@ StatusOr InstructionFusion::Run(HloModule* module) { computation_ = computation; reachability_ = HloReachabilityMap::Build(computation_); - HloInstructionSet do_not_duplicate = - ComputeGloballyUnfusible(computation_->MakeInstructionPostOrder()); + HloInstructionSet do_not_duplicate; + // If we allow duplications, we need to compute which instructions we do not + // want to duplicate based on a global analysis of the graph. + if (may_duplicate_) { + do_not_duplicate = + ComputeGloballyUnfusible(computation_->MakeInstructionPostOrder()); + } auto fusion_queue = GetFusionQueue(computation_); // Instruction fusion effectively fuses edges in the computation graph @@ -564,8 +571,8 @@ HloInstruction* InstructionFusion::FuseIntoMultiOutput( bool InstructionFusion::MultiOutputFusionCreatesCycle( HloInstruction* producer, HloInstruction* consumer) { auto is_reachable = [&](const HloInstruction* a, const HloInstruction* b) { - // A consumer operand may have been multii-output fused into a parallel - // consumer and thus be missing from the oridinal reachability map. + // A consumer operand may have been multi-output fused into a parallel + // consumer and thus be missing from the original reachability map. if (!reachability_->IsPresent(a) || !reachability_->IsPresent(b)) { reachability_ = HloReachabilityMap::Build(consumer->parent()); } diff --git a/tensorflow/compiler/xla/service/instruction_fusion_test.cc b/tensorflow/compiler/xla/service/instruction_fusion_test.cc index 39904bd54b0..58b7135cea7 100644 --- a/tensorflow/compiler/xla/service/instruction_fusion_test.cc +++ b/tensorflow/compiler/xla/service/instruction_fusion_test.cc @@ -117,7 +117,7 @@ TEST_F(InstructionFusionTest, PotentialBitcastReshapeOfParameterUnfused) { auto reshape1 = builder.AddInstruction( HloInstruction::CreateReshape(ShapeUtil::MakeShape(S32, {1, 1}), param0)); - auto module = CreateNewUnverifiedModule(); + auto module = CreateNewVerifiedModule(); auto computation = module->AddEntryComputation(builder.Build()); EXPECT_EQ(reshape1, computation->root_instruction()); EXPECT_FALSE( @@ -133,7 +133,7 @@ TEST_F(InstructionFusionTest, PotentialBitcastSimpleReshapeOfParameterUnfused) { auto reshape1 = builder.AddInstruction( HloInstruction::CreateReshape(ShapeUtil::MakeShape(S32, {1, 1}), param0)); - auto module = CreateNewUnverifiedModule(); + auto module = CreateNewVerifiedModule(); auto computation = module->AddEntryComputation(builder.Build()); EXPECT_EQ(reshape1, computation->root_instruction()); EXPECT_FALSE( @@ -149,7 +149,7 @@ TEST_F(InstructionFusionTest, PotentialBitcastTransposeOfParameterUnfused) { auto transpose1 = builder.AddInstruction(HloInstruction::CreateTranspose( ShapeUtil::MakeShape(S32, {}), param0, {})); - auto module = CreateNewUnverifiedModule(); + auto module = CreateNewVerifiedModule(); auto computation = module->AddEntryComputation(builder.Build()); EXPECT_EQ(transpose1, computation->root_instruction()); EXPECT_FALSE( @@ -394,6 +394,56 @@ TEST_F(InstructionFusionTest, AllowEffectiveUnaryDuplication) { .ValueOrDie()); } +TEST_F(InstructionFusionTest, FuseDiamondGraphsNoDuplication) { + auto module = ParseHloString(R"( + HloModule test_module + ENTRY Test { + p0 = f32[100] parameter(0) + p1 = f32[100] parameter(1) + add = f32[100] add(p0, p1) + slice1 = f32[99] slice(add), slice={[0:99:1]} + slice2 = f32[99] slice(add), slice={[1:100:1]} + ROOT add2 = f32[99] add(slice1, slice2) + })") + .ValueOrDie(); + EXPECT_TRUE( + InstructionFusion(InstructionFusion::IsExpensive, /*may_duplicate=*/false) + .Run(module.get()) + .ValueOrDie()) + << module->ToString(); + + HloInstruction* root = module->entry_computation()->root_instruction(); + // 'add' would originally need to be duplicated if fused. However after its + // two users 'slice1' and 'slice2' are fused into 'add2', 'add' has only one + // user and can now be also fused. + EXPECT_THAT(root, op::Fusion(op::Parameter(), op::Parameter())); +} + +TEST_F(InstructionFusionTest, FuseDiamondGraphsAllowDuplication) { + auto module = ParseHloString(R"( + HloModule test_module + ENTRY Test { + p0 = f32[100] parameter(0) + p1 = f32[100] parameter(1) + add = f32[100] add(p0, p1) + slice1 = f32[99] slice(add), slice={[0:99:1]} + slice2 = f32[99] slice(add), slice={[1:100:1]} + ROOT add2 = f32[99] add(slice1, slice2) + })") + .ValueOrDie(); + EXPECT_TRUE( + InstructionFusion(InstructionFusion::IsExpensive, /*may_duplicate=*/true) + .Run(module.get()) + .ValueOrDie()) + << module->ToString(); + + HloInstruction* root = module->entry_computation()->root_instruction(); + // 'add' would originally need to be duplicated if fused. However after its + // two users 'slice1' and 'slice2' are fused into 'add2', 'add' has only one + // user and can now be also fused. + EXPECT_THAT(root, op::Fusion(op::Parameter(), op::Parameter())); +} + TEST_F(InstructionFusionTest, WideningConvertsAreAlwaysDuplicableIntoConsumers) { auto module = ParseHloString(R"( diff --git a/tensorflow/compiler/xla/service/interpreter/executable.cc b/tensorflow/compiler/xla/service/interpreter/executable.cc index a06d6113e84..7635fbfed6f 100644 --- a/tensorflow/compiler/xla/service/interpreter/executable.cc +++ b/tensorflow/compiler/xla/service/interpreter/executable.cc @@ -37,7 +37,7 @@ namespace xla { namespace interpreter { InterpreterExecutable::InterpreterExecutable( - std::unique_ptr hlo_module, + std::unique_ptr hlo_module, std::unique_ptr evaluator) : Executable(std::move(hlo_module), /*hlo_profile_printer=*/nullptr, /*hlo_profile_index_map=*/nullptr), diff --git a/tensorflow/compiler/xla/service/interpreter/executable.h b/tensorflow/compiler/xla/service/interpreter/executable.h index 3b1ebce0c75..bda13d37636 100644 --- a/tensorflow/compiler/xla/service/interpreter/executable.h +++ b/tensorflow/compiler/xla/service/interpreter/executable.h @@ -42,7 +42,7 @@ namespace interpreter { // buffer allocation. Refer to interpreter/README.md for more. class InterpreterExecutable : public Executable { public: - InterpreterExecutable(std::unique_ptr hlo_module, + InterpreterExecutable(std::unique_ptr hlo_module, std::unique_ptr evaluator); ~InterpreterExecutable() override; diff --git a/tensorflow/compiler/xla/service/interpreter/executor.cc b/tensorflow/compiler/xla/service/interpreter/executor.cc index 4fb67bd0b72..e3e5fa71543 100644 --- a/tensorflow/compiler/xla/service/interpreter/executor.cc +++ b/tensorflow/compiler/xla/service/interpreter/executor.cc @@ -78,9 +78,14 @@ port::Status XlaInterpreterExecutor::SynchronousMemcpy( return port::Status::OK(); } -bool XlaInterpreterExecutor::HostCallback(Stream *stream, - std::function callback) { - AsExecutorStream(stream)->EnqueueTask(callback); +bool XlaInterpreterExecutor::HostCallback( + Stream *stream, std::function callback) { + AsExecutorStream(stream)->EnqueueTask([callback]() { + port::Status s = callback(); + if (!s.ok()) { + LOG(WARNING) << "Host callback failed: " << s; + } + }); return true; } diff --git a/tensorflow/compiler/xla/service/interpreter/executor.h b/tensorflow/compiler/xla/service/interpreter/executor.h index fbb99457847..400c3051546 100644 --- a/tensorflow/compiler/xla/service/interpreter/executor.h +++ b/tensorflow/compiler/xla/service/interpreter/executor.h @@ -125,7 +125,8 @@ class XlaInterpreterExecutor : public internal::StreamExecutorInterface { return port::Status{port::error::UNIMPLEMENTED, ""}; } - bool HostCallback(Stream *stream, std::function callback) override; + bool HostCallback(Stream *stream, + std::function callback) override; port::Status AllocateEvent(Event *event) override { return port::Status{port::error::UNIMPLEMENTED, ""}; diff --git a/tensorflow/compiler/xla/service/layout_assignment.cc b/tensorflow/compiler/xla/service/layout_assignment.cc index a9041192220..eddef850cf5 100644 --- a/tensorflow/compiler/xla/service/layout_assignment.cc +++ b/tensorflow/compiler/xla/service/layout_assignment.cc @@ -2000,6 +2000,7 @@ bool LayoutAssignment::InstructionCanChangeLayout( switch (instruction->opcode()) { case HloOpcode::kAbs: case HloOpcode::kAdd: + case HloOpcode::kAddDependency: case HloOpcode::kAnd: case HloOpcode::kAtan2: case HloOpcode::kBitcastConvert: diff --git a/tensorflow/compiler/xla/service/layout_assignment_test.cc b/tensorflow/compiler/xla/service/layout_assignment_test.cc index 2400b7bb7c4..311bd789054 100644 --- a/tensorflow/compiler/xla/service/layout_assignment_test.cc +++ b/tensorflow/compiler/xla/service/layout_assignment_test.cc @@ -31,6 +31,8 @@ limitations under the License. #include "tensorflow/compiler/xla/service/hlo_module.h" #include "tensorflow/compiler/xla/service/hlo_opcode.h" #include "tensorflow/compiler/xla/service/hlo_parser.h" +#include "tensorflow/compiler/xla/service/pattern_matcher.h" +#include "tensorflow/compiler/xla/service/pattern_matcher_gmock.h" #include "tensorflow/compiler/xla/shape_layout.h" #include "tensorflow/compiler/xla/shape_util.h" #include "tensorflow/compiler/xla/test.h" @@ -42,11 +44,10 @@ limitations under the License. #include "tensorflow/core/lib/core/status.h" #include "tensorflow/core/lib/core/status_test_util.h" -namespace op = xla::testing::opcode_matchers; - namespace xla { namespace { +namespace m = xla::match; using ::testing::ElementsAre; class LayoutAssignmentTest : public HloTestBase { @@ -328,11 +329,10 @@ TEST_F(LayoutAssignmentTest, ConflictingLayoutTuple) { // %tuple.1 = Tuple(%copy) layout=({0,1}) // %tuple.2 = Tuple(%tuple.0, %tuple.1) layout=(({1,0}), ({0,1})) // - EXPECT_TRUE( - AlgebraicSimplifier(/*is_layout_sensitive=*/true, - [](const Shape&, const Shape&) { return false; }) - .Run(m.get()) - .ValueOrDie()); + AlgebraicSimplifierOptions options( + [](const Shape&, const Shape&) { return false; }); + options.set_is_layout_sensitive(true); + EXPECT_TRUE(AlgebraicSimplifier(options).Run(m.get()).ValueOrDie()); HloInstruction* root = m->entry_computation()->root_instruction(); // Verify layout of the root and the root's operands. EXPECT_TRUE(ShapeUtil::Equal(result_shape, root->shape())); @@ -343,7 +343,8 @@ TEST_F(LayoutAssignmentTest, ConflictingLayoutTuple) { // Verify the structure of the HLO graph. EXPECT_THAT(root, - op::Tuple(op::Tuple(constant), op::Tuple(op::Copy(constant)))); + GmockMatch(m::Tuple(m::Tuple(m::Op().Is(constant)), + m::Tuple(m::Copy(m::Op().Is(constant)))))); } TEST_F(LayoutAssignmentTest, ElementwiseAndReshape) { @@ -947,9 +948,11 @@ TEST_F(LayoutAssignmentTest, CopySliceOperandToAvoidImplicitLayoutChange) { HloInstruction* root = compiled_module->entry_computation()->root_instruction(); Shape shape_copy = ShapeUtil::MakeShapeWithLayout(F32, {4, 5}, {1, 0}); - EXPECT_THAT(root, op::Add(op::Parameter(), - op::Slice(AllOf(op::Copy(op::Parameter(1)), - op::ShapeWithLayout(shape_copy))))); + EXPECT_THAT( + root, + GmockMatch(m::Add( + m::Parameter(), + m::Slice(m::Copy(m::Parameter(1)).WithShapeEqualTo(&shape_copy))))); } TEST_F(LayoutAssignmentTest, CopyDSliceOperandToAvoidImplicitLayoutChange) { @@ -977,10 +980,11 @@ TEST_F(LayoutAssignmentTest, CopyDSliceOperandToAvoidImplicitLayoutChange) { compiled_module->entry_computation()->root_instruction(); Shape shape_copy = ShapeUtil::MakeShapeWithLayout(F32, {4, 5}, {1, 0}); EXPECT_THAT(root, - op::Add(op::Parameter(), - op::DynamicSlice(AllOf(op::Copy(op::Parameter(1)), - op::ShapeWithLayout(shape_copy)), - op::Parameter(2)))); + GmockMatch(m::Add( + m::Parameter(), + m::DynamicSlice( + m::Copy(m::Parameter(1)).WithShapeEqualTo(&shape_copy), + m::Parameter(2))))); } TEST_F(LayoutAssignmentTest, CopyConcatOperandToAvoidImplicitLayoutChange) { @@ -1008,11 +1012,12 @@ TEST_F(LayoutAssignmentTest, CopyConcatOperandToAvoidImplicitLayoutChange) { HloInstruction* root = compiled_module->entry_computation()->root_instruction(); Shape shape_copy = ShapeUtil::MakeShapeWithLayout(F32, {3, 5}, {1, 0}); - EXPECT_THAT(root, - op::Add(op::Parameter(), - op::Concatenate(AllOf(op::Copy(op::Parameter(1)), - op::ShapeWithLayout(shape_copy)), - op::Parameter(2)))); + EXPECT_THAT( + root, + GmockMatch(m::Add( + m::Parameter(), + m::Concatenate(m::Copy(m::Parameter(1)).WithShapeEqualTo(&shape_copy), + m::Parameter(2))))); } TEST_F(LayoutAssignmentTest, @@ -1039,7 +1044,8 @@ TEST_F(LayoutAssignmentTest, .ConsumeValueOrDie(); HloInstruction* root = compiled_module->entry_computation()->root_instruction(); - EXPECT_THAT(root, op::Convolution(op::Parameter(0), op::Parameter(1))); + EXPECT_THAT(root, + GmockMatch(m::Convolution(m::Parameter(0), m::Parameter(1)))); } TEST_F(LayoutAssignmentTest, PropagatingLayoutFromResultToOperand) { @@ -1063,8 +1069,9 @@ TEST_F(LayoutAssignmentTest, PropagatingLayoutFromResultToOperand) { HloInstruction* root = compiled_module->entry_computation()->root_instruction(); Shape shape_copy = ShapeUtil::MakeShapeWithLayout(F32, {4, 5}, {0, 1}); - EXPECT_THAT(root, op::Slice(AllOf(op::Copy(op::Parameter(0)), - op::ShapeWithLayout(shape_copy)))); + EXPECT_THAT(root, + GmockMatch(m::Slice( + m::Copy(m::Parameter(0)).WithShapeEqualTo(&shape_copy)))); } TEST_F(LayoutAssignmentTest, TupleCopyOnLayoutMismatch) { @@ -1150,7 +1157,7 @@ ENTRY %CustomCallWithNotLayoutConstrained (p: f32[42,2,3]) -> f32[1,2,3,4] { AssignLayouts(m.get(), &computation_layout); HloInstruction* root = m->entry_computation()->root_instruction(); - ASSERT_THAT(root, op::CustomCall(op::Parameter())); + ASSERT_THAT(root, GmockMatch(m::CustomCall(m::Parameter()))); ExpectLayoutIs(root->shape(), {3, 2, 0, 1}); ExpectLayoutIs(root->operand(0)->shape(), {0, 2, 1}); } @@ -1166,7 +1173,7 @@ ENTRY %CustomCallWithNotLayoutConstrained (p: f32[42,2,3]) -> f32[1,2,3,4] { AssignLayouts(m.get(), &computation_layout); HloInstruction* root = m->entry_computation()->root_instruction(); - ASSERT_THAT(root, op::CustomCall(op::Parameter())); + ASSERT_THAT(root, GmockMatch(m::CustomCall(m::Parameter()))); ExpectLayoutIs(root->shape(), {0, 2, 3, 1}); ExpectLayoutIs(root->operand(0)->shape(), {0, 1, 2}); } @@ -1197,7 +1204,7 @@ ENTRY %CustomCallWithLayoutConstraints (p0: f32[4,4], p1: f32[2,3]) -> f32[1,2,3 // The custom call should be partially encapsulated in kCopy instructions // because of the layout mismatches. ASSERT_THAT(m->entry_computation()->root_instruction(), - op::Copy(op::CustomCall(op::Copy(), op::Parameter()))); + GmockMatch(m::Copy(m::CustomCall(m::Copy(), m::Parameter())))); const HloInstruction* custom_call = m->entry_computation()->root_instruction()->operand(0); @@ -1223,7 +1230,7 @@ ENTRY %CustomCallLayoutConstrainedZeroOperands () -> f32[1,2,3,4] { AssignLayouts(m.get(), &computation_layout); ASSERT_THAT(m->entry_computation()->root_instruction(), - op::Copy(op::CustomCall())); + GmockMatch(m::Copy(m::CustomCall()))); const HloInstruction* custom_call = m->entry_computation()->root_instruction()->operand(0); @@ -1257,7 +1264,7 @@ ENTRY %CustomCallLayoutConstrainedTupleOperand (p0: f32[4,4], p1: f32[2,3]) -> f ExpectLayoutIs(root->shape(), {2, 1, 0, 3}); ASSERT_THAT(m->entry_computation()->root_instruction(), - op::Copy(op::CustomCall(op::Tuple()))); + GmockMatch(m::Copy(m::CustomCall(m::Tuple())))); const HloInstruction* custom_call = m->entry_computation()->root_instruction()->operand(0); diff --git a/tensorflow/compiler/xla/service/llvm_ir/ir_array.h b/tensorflow/compiler/xla/service/llvm_ir/ir_array.h index f4b05f29c38..d6d84994ee1 100644 --- a/tensorflow/compiler/xla/service/llvm_ir/ir_array.h +++ b/tensorflow/compiler/xla/service/llvm_ir/ir_array.h @@ -25,6 +25,7 @@ limitations under the License. #include "llvm/IR/IRBuilder.h" #include "llvm/IR/Value.h" #include "tensorflow/compiler/xla/map_util.h" +#include "tensorflow/compiler/xla/shape.h" #include "tensorflow/compiler/xla/types.h" #include "tensorflow/compiler/xla/xla_data.pb.h" #include "tensorflow/core/platform/logging.h" @@ -108,6 +109,14 @@ class IrArray { Index(absl::Span multidim, llvm::Value* linear, const Shape& shape); + // Returns an index that adds `addend` to the given `dim` of the object. + Index AddOffsetToDim(llvm::Value* addend, int64 dim, + llvm::IRBuilder<>* b) const { + IrArray::Index index = *this; + index[dim] = b->CreateAdd(index[dim], addend); + return index; + } + const std::vector& multidim() const { return multidim_; } llvm::Value* linear() const { return linear_; } diff --git a/tensorflow/compiler/xla/service/llvm_ir/kernel_tiling.cc b/tensorflow/compiler/xla/service/llvm_ir/kernel_tiling.cc index e5fbdbd51b8..1aa85eb8d2d 100644 --- a/tensorflow/compiler/xla/service/llvm_ir/kernel_tiling.cc +++ b/tensorflow/compiler/xla/service/llvm_ir/kernel_tiling.cc @@ -52,6 +52,29 @@ Shape MergeDimensions(absl::Span segs, const Shape& shape) { return ShapeUtil::MakeShapeWithDescendingLayout(shape.element_type(), dimensions); } + +// Given an index for a shape, return the equivalent new index if the shape is +// reshaped to another shape. +IrArray::Index GetReshapedIndex(const IrArray::Index& index, const Shape& shape, + const Shape& reshaped_shape, + llvm::IRBuilder<>* b) { + auto bounds = shape.dimensions(); + auto minor_to_major = shape.layout().minor_to_major(); + llvm::Value* linear_index = index.GetConstantWithIndexType(0); + int64 multiplier = 1; + for (int i = 0; i < index.size(); ++i) { + int64 dim = minor_to_major[i]; + llvm::Value* addend = b->CreateMul( + index[dim], index.GetConstantWithIndexType(multiplier), "linearizing", + /*HasNUW=*/true, /*HasNSW=*/true); + linear_index = b->CreateAdd(linear_index, addend, "", + /*HasNUW=*/true, /*HasNSW=*/true); + multiplier *= bounds[dim]; + } + + return IrArray::Index(linear_index, reshaped_shape, b); +} + } // namespace absl::optional > FindTranspose021(const Shape& a, @@ -60,28 +83,30 @@ absl::optional > FindTranspose021(const Shape& a, return absl::nullopt; } - std::vector perm(a.dimensions().size()); - { - auto layout_a_orig = LayoutUtil::MinorToMajor(a); - std::vector layout_a(layout_a_orig.rbegin(), layout_a_orig.rend()); - auto layout_b_orig = LayoutUtil::MinorToMajor(b); - std::vector layout_b(layout_b_orig.rbegin(), layout_b_orig.rend()); - for (size_t i = 0; i < perm.size(); ++i) { - perm[i] = PositionInContainer(layout_b, layout_a[i]); - } + std::vector permutation(a.dimensions().size()); + absl::Span minor_to_major_a = LayoutUtil::MinorToMajor(a); + std::vector major_to_minor_a(minor_to_major_a.rbegin(), + minor_to_major_a.rend()); + absl::Span minor_to_major_b = LayoutUtil::MinorToMajor(b); + std::vector major_to_minor_b(minor_to_major_b.rbegin(), + minor_to_major_b.rend()); + for (size_t i = 0; i < permutation.size(); ++i) { + permutation[i] = PositionInContainer(major_to_minor_b, major_to_minor_a[i]); } - auto segs = ConsecutiveSegments(perm); - if ((3 == segs.size() && 0 == perm[0]) || 2 == segs.size()) { - Shape norm_a = + + std::vector segments = ConsecutiveSegments(permutation); + if ((3 == segments.size() && 0 == permutation[0]) || 2 == segments.size()) { + Shape descending_layout_shape = ShapeUtil::MakeShapeWithDescendingLayoutAndSamePhysicalLayout(a); - Shape reduced_a = MergeDimensions(segs, norm_a); - auto reduced_a_dims = reduced_a.dimensions(); + Shape normalized_shape = MergeDimensions(segments, descending_layout_shape); + absl::Span normalized_dims = + AsInt64Slice(normalized_shape.dimensions()); std::vector dims_021; - if (2 == segs.size()) { + if (2 == segments.size()) { // The logical component-0 is of size one. - dims_021 = {1, reduced_a_dims[1], reduced_a_dims[0]}; + dims_021 = {1, normalized_dims[1], normalized_dims[0]}; } else { - dims_021 = {reduced_a_dims[0], reduced_a_dims[2], reduced_a_dims[1]}; + dims_021 = {normalized_dims[0], normalized_dims[2], normalized_dims[1]}; } return dims_021; @@ -90,27 +115,117 @@ absl::optional > FindTranspose021(const Shape& a, return absl::nullopt; } -IrArray::Index GetUnreducedOutputIndex( - const IrArray::Index& reduced_output_index, - const Shape& reduced_output_shape, const Shape& unreduced_output_shape, - llvm::IRBuilder<>* b) { - auto bounds = reduced_output_shape.dimensions(); - auto minor_to_major = reduced_output_shape.layout().minor_to_major(); - llvm::Value* linear_index = reduced_output_index.GetConstantWithIndexType(0); - int64 multiplier = 1; - for (int i = 0; i < reduced_output_index.size(); ++i) { - int64 dim = minor_to_major[i]; - llvm::Value* addend = - b->CreateMul(reduced_output_index[dim], - reduced_output_index.GetConstantWithIndexType(multiplier), - "linearizing", - /*HasNUW=*/true, /*HasNSW=*/true); - linear_index = b->CreateAdd(linear_index, addend, "", - /*HasNUW=*/true, /*HasNSW=*/true); - multiplier *= bounds[dim]; - } +KernelMappingScheme::KernelMappingScheme( + absl::Span dims_in_elems, int64 tile_size_y, int64 tile_size_x, + absl::Span req_block_sizes, int64 num_threads_y, + int64 num_threads_x, llvm::IRBuilder<>* b) + : b_(b), + dims_in_elems_(dims_in_elems.begin(), dims_in_elems.end()), + tile_sizes_{1, tile_size_y, tile_size_x}, + num_threads_x_(num_threads_x), + num_threads_y_(num_threads_y) { + DCHECK_EQ(dims_in_elems_.size(), 3); + DCHECK_EQ(req_block_sizes.size(), 3); - return IrArray::Index(linear_index, unreduced_output_shape, b); + DCHECK_EQ(tile_size_y % num_threads_y_, 0); + DCHECK_EQ(tile_size_x % num_threads_x_, 0); + + dims_in_tiles_ = ElementWiseCeilOfRatio(dims_in_elems_, tile_sizes_); + block_sizes_.reserve(req_block_sizes.size()); + absl::c_transform(req_block_sizes, dims_in_tiles_, + std::back_inserter(block_sizes_), + [](const int64 requested_size, const int64 max_size) { + return std::min(requested_size, max_size); + }); + dims_in_blocks_ = ElementWiseCeilOfRatio(dims_in_tiles_, block_sizes_); + + VLOG(10) << "dims_in_elems_ = [" << absl::StrJoin(dims_in_elems_, ",") << "]"; + VLOG(10) << "dims_in_tiles_ = [" << absl::StrJoin(dims_in_tiles_, ",") << "]"; + VLOG(10) << "dims_in_blocks_ = [" << absl::StrJoin(dims_in_blocks_, ",") + << "]"; +} + +IrArray::Index KernelMappingScheme::GetUnnormalizedIndex( + const IrArray::Index& normalized_shape_index, + const Shape& unnormalized_shape) { + DCHECK_EQ(normalized_shape_index.size(), dims_in_elems_.size()); + Shape output_shape = ShapeUtil::MakeShapeWithDescendingLayout( + unnormalized_shape.element_type(), GetDimensionsInElements()); + return GetReshapedIndex(normalized_shape_index, output_shape, + unnormalized_shape, b_); +} + +IrArray::Index KernelMappingScheme::EmitBlockIndex(llvm::Type* index_ty) { + llvm::Value* block_id = llvm_ir::EmitCallToIntrinsic( + llvm::Intrinsic::nvvm_read_ptx_sreg_ctaid_x, {}, {}, b_); + llvm_ir::AddRangeMetadata(0, GetNumberOfBlocks(), + llvm::cast(block_id)); + llvm::Value* linear_block_id = + b_->CreateIntCast(block_id, index_ty, /*isSigned=*/true, "block.id.x"); + return IrArray::Index(linear_block_id, + ShapeUtil::MakeShapeWithDescendingLayout( + PRED /*arbitrary*/, dims_in_blocks_), + b_); +} + +IrArray::Index KernelMappingScheme::GetTileIndexForBlockOrigin( + const IrArray::Index& block_index) { + IrArray::Index tile_index = block_index; + for (int i = 0; i < block_sizes_.size(); ++i) { + tile_index[i] = b_->CreateMul( + block_index[i], + llvm::ConstantInt::get(block_index[i]->getType(), block_sizes_[i]), + "block_origin." + std::to_string(i)); + } + return tile_index; +} + +IrArray::Index KernelMappingScheme::GetElementIndexForTileOrigin( + const IrArray::Index& tile_index) { + IrArray::Index elem_index = tile_index; + for (int i = DimY; i < DimTot; ++i) { + elem_index[i] = + b_->CreateMul(tile_index[i], + llvm::ConstantInt::get(tile_index[i]->getType(), + GetTileSizeForDimension(i)), + "tile_origin." + std::to_string(i)); + } + return elem_index; +} + +llvm::GlobalVariable* KernelMappingScheme::GetSharedMemoryBufferForElementType( + llvm::Type* elem_ty, absl::string_view buffer_name) { + // If shared memory tranpose is needed, we use square tiles. + CHECK_EQ(GetTileSizeForDimensionX(), GetTileSizeForDimensionY()); + + // For Nvidia GPUs, the warp size is 32 threads and the shared memory bank is + // organized into 32-way. We usually use the warp size or a multiplier or a + // the warp size as the size for tiling. This may cause all elements in the + // same column of a tile use the same memory bank and therefore shared memory + // bank conflicts. Adding 1 to the minor dimension of the shared memory buffer + // can reduce such shared memory bank conflicts. + llvm::Type* buffer_type = llvm::ArrayType::get( + llvm::ArrayType::get(elem_ty, GetTileSizeForDimension(DimX) + 1), + GetTileSizeForDimension(DimY)); + return llvm_ir::AllocateSharedMemoryTile(b_->GetInsertBlock()->getModule(), + buffer_type, buffer_name); +} + +std::tuple +KernelMappingScheme::EmitThreadYXCoordinate(llvm::Type* index_ty) { + // Calculate (y, x) coordinate of the thread in the 2D view of thread block + // defined by (num_thread_y, num_thread_x) from thread_id. + llvm::CallInst* thread_id_raw = llvm_ir::EmitCallToIntrinsic( + llvm::Intrinsic::nvvm_read_ptx_sreg_tid_x, {}, {}, b_); + llvm_ir::AddRangeMetadata(0, GetThreadsPerTile(), thread_id_raw); + llvm::Value* thread_id_int = + b_->CreateIntCast(thread_id_raw, index_ty, + /*isSigned=*/true, "thread.id.x"); + llvm::Value* num_thread_x = + llvm::ConstantInt::get(index_ty, GetNumberOfThreadsForDimensionX()); + llvm::Value* x = b_->CreateURem(thread_id_int, num_thread_x); + llvm::Value* y = b_->CreateUDiv(thread_id_int, num_thread_x); + return std::make_tuple(y, x); } } // namespace llvm_ir diff --git a/tensorflow/compiler/xla/service/llvm_ir/kernel_tiling.h b/tensorflow/compiler/xla/service/llvm_ir/kernel_tiling.h index 5ea05b3188a..7277aeac8ad 100644 --- a/tensorflow/compiler/xla/service/llvm_ir/kernel_tiling.h +++ b/tensorflow/compiler/xla/service/llvm_ir/kernel_tiling.h @@ -28,23 +28,165 @@ namespace llvm_ir { // If a shape can be viewed as three logical components 0-1-2 in the order of // major to minor, a 0-2-1-transpose changes the order of such logical // components to 0-2-1. We call the shape being transposed the input shape and -// the transposed shape the output shape. The logical view of the input and -// output shapes for the transpose are called the 0-1-2 shape or reduced input -// shape and the 0-2-1 shape or the reduced output shape respectively. The -// original input and output shapes are called the unreduced input and output -// shapes. - +// the transposed shape the output shape. The logical view of the input/output +// shapes for the transpose are called the 0-1-2/0-2-1 shapes or the normalized +// shapes. The original input/output shapes are called unnormalized shapes. +// // If `b` is a 0-2-1 transpose of `a` in 0-1-2, return the dimensions for the -// reduced shape of `b` or the 0-2-1 shape. +// normalized shape of `b` or the 0-2-1 shape. absl::optional > FindTranspose021(const Shape& a, const Shape& b); -// Return the unreduced output index corresponding to the given reduced output -// index. -IrArray::Index GetUnreducedOutputIndex( - const IrArray::Index& reduced_output_index, - const Shape& reduced_output_shape, const Shape& unreduced_output_shape, - llvm::IRBuilder<>* b); +// A tile is a spatial subdivision of a tensor. We group tensor elements into +// tiles so that we can launch kernels to process the tensor elements in blocks +// of tiles. +// +// A kernel mapping scheme describes a method to partition the tensors accessed +// by an unnested HLO instruction into tiles and blocks of tiles, and the +// associated information to use hardware threads to process the tensor elements +// in blocks of tiles. +// +// Currently, there are two main use cases for a tiling scheme. First, we +// implement kernels with 0-2-1 memory transpose using shared memory to improve +// memory access pattern. Second, we implement reduction to contiguous +// dimensions in layout, with or without memory tranpsose, to achieve better +// memory access pattern as well as to reduce the need numbers of executed +// expensive instructions, such as thread synchronization related instructions +// and atomic operations. For both use cases, we can apply a normalization to +// the original tensors, to collapse contiguous dimensions for the same purpose +// and produce normlized three dimensional tensors. For this reason, the tiling +// scheme class only needs to handle normalized three dimensional tensors and +// two dimensional tiles. +// +// The current implementation of the class is somewhat NVIDIA GPU oriented. This +// situation can be improved when there is a need though. The idea of 0-2-1 +// transpose using shared memory can be found in the following CUDA algorithm in +// TensorFlow: https://goo.gl/MStRV6. +// +// We use a thread block to process a tile because we want to use the HW thread +// block synchronization primitives to synchronize the processing of all the +// elements in the same tile. A thread block can be viewed as a two dimensional +// array of threads, described by the number of threads for the Y and X +// dimensions. A thread block (num_threads_y, num_threads_x) processes a tile of +// (tile_size_y, tile_size_x) as follows: each thread in the thread block +// processes one element in the tile so that all the threads in the thread block +// together process a subdivision of the tile that has the same dimension as the +// thread block array. Then the thread block moves on to process the next +// subdivision of the tile until the whole tile is processed. Therefore, each +// thread in the thread block processes +// tile_size_x/num_threads_x * tile_size_y/num_threads_y elements in a tile. +// +// There are situations where we want a thread block to process multiple +// tiles. We can't group those tiles into a bigger tiles because we limit a tile +// to a two dimensional spatial subdivision of a tensor. For example, when we +// use tiling to implement reduction with tranpose, we want the partial sum +// produced by each thread to accumulate values for more elements before using +// shlf_down and atomic_add instructions for further reduction, to amortize the +// cost of such expensive instructions. The concept of tile block is introduced +// for this purpose. A tile block is a three dimensional array of tiles, of +// which some dimensions may be degenerated to only one tile. +class KernelMappingScheme { + public: + enum { DimZ = 0, DimY, DimX, DimTot }; + + public: + KernelMappingScheme() {} + // dims_in_elems: the normalized tensor dimensions. + // req_block_sizes: the requested block size in number of tiles for each + // dimension. The actual block size is set to min(req_block_size, + // dims_in_number_of_blocks). + KernelMappingScheme(absl::Span dims_in_elems, int64 tile_size_y, + int64 tile_size_x, + absl::Span req_block_sizes, + int64 num_threads_y, int64 num_threads_x, + llvm::IRBuilder<>* b); + + absl::Span GetDimensionsInElements() const { + return dims_in_elems_; + } + absl::Span GetDimensionsInTiles() const { + return dims_in_tiles_; + } + absl::Span GetDimensionsInBlocks() const { + return dims_in_blocks_; + } + + int64 GetNumberOfTilesInTotal() const { + return absl::c_accumulate(dims_in_tiles_, 1LL, std::multiplies()); + } + int64 GetNumberOfTilesInOneBlock() const { + return absl::c_accumulate(block_sizes_, 1, std::multiplies()); + } + + int64 GetNumberOfBlocks() const { + return absl::c_accumulate(dims_in_blocks_, 1, std::multiplies()); + } + + int64 GetTileSizeForDimension(int d) const { + DCHECK(d >= DimZ && d <= DimX); + return tile_sizes_[d]; + } + int64 GetTileSizeForDimensionX() const { + return GetTileSizeForDimension(DimX); + } + int64 GetTileSizeForDimensionY() const { + return GetTileSizeForDimension(DimY); + } + + absl::Span GetBlockSizes() const { return block_sizes_; } + int64 GetTileBlockSizeForDimension(int d) const { + DCHECK(d >= DimZ && d <= DimX); + return dims_in_blocks_[d]; + } + + int64 GetNumberOfThreadsForDimensionX() const { return num_threads_x_; } + int64 GetNumberOfThreadsForDimensionY() const { return num_threads_y_; } + + int64 GetThreadsPerTile() const { + return GetNumberOfThreadsForDimensionX() * + GetNumberOfThreadsForDimensionY(); + } + + IrArray::Index EmitBlockIndex(llvm::Type* index_ty); + // Returns the index for the first tile in the block with the given block + // index. + IrArray::Index GetTileIndexForBlockOrigin(const IrArray::Index& block_index); + // Returns the index for the first element in the tile with the given tile + // index. + IrArray::Index GetElementIndexForTileOrigin(const IrArray::Index& tile_index); + + std::tuple EmitThreadYXCoordinate( + llvm::Type* index_ty); + + IrArray::Index GetUnnormalizedIndex( + const IrArray::Index& normalized_shape_index, + const Shape& unnormalized_shape); + + llvm::GlobalVariable* GetSharedMemoryBufferForElementType( + llvm::Type* elem_ty, absl::string_view buffer_name); + + private: + llvm::IRBuilder<>* b_; + // The number of elements in each dimension. + std::vector dims_in_elems_; + + // The number of elements for each dimension of a tile. + std::vector tile_sizes_; + // The number of tiles in each dimension. It is computed from dims_in_elem_ + // and tile_sizes_. + std::vector dims_in_tiles_; + + // The number of tiles for each dimension of a tile block. + std::vector block_sizes_; + // The number of blocks in each dimension of a tile block. It is computed from + // dims_in_tile_ and block_sizes_. + std::vector dims_in_blocks_; + + // Number of threads used to process elements in the X direction of a tile. + int64 num_threads_x_; + // Number of threads used to process elements in the Y direction of a tile. + int64 num_threads_y_; +}; // A class to represent information for tiled parameters to support IR emission // for 021 transpose. diff --git a/tensorflow/compiler/xla/service/llvm_ir/llvm_util.cc b/tensorflow/compiler/xla/service/llvm_ir/llvm_util.cc index df78726166e..ceea24685af 100644 --- a/tensorflow/compiler/xla/service/llvm_ir/llvm_util.cc +++ b/tensorflow/compiler/xla/service/llvm_ir/llvm_util.cc @@ -244,10 +244,11 @@ StatusOr EncodeSelfDescribingShapeConstant(const Shape& shape, StatusOr DecodeSelfDescribingShapeConstant(const void* shape_ptr, int32 size_bytes) { - Shape shape; - TF_RET_CHECK(shape.ParseFromArray(shape_ptr, size_bytes)); + ShapeProto shape_proto; + TF_RET_CHECK(shape_proto.ParseFromArray(shape_ptr, size_bytes)); + Shape shape(shape_proto); TF_RETURN_IF_ERROR(ShapeUtil::ValidateShape(shape)); - return shape; + return std::move(shape); } llvm::Constant* ConvertLiteralToIrConstant(const Literal& literal, diff --git a/tensorflow/compiler/xla/service/llvm_ir/sort_util.cc b/tensorflow/compiler/xla/service/llvm_ir/sort_util.cc index fd16af67fe9..e22c2173c27 100644 --- a/tensorflow/compiler/xla/service/llvm_ir/sort_util.cc +++ b/tensorflow/compiler/xla/service/llvm_ir/sort_util.cc @@ -47,7 +47,8 @@ namespace { // Adds the inner comparison loop body where we compare elements. void EmitCompareLoopBody( int64 iteration_bound, PrimitiveType key_type, int64 num_values, - llvm::Value* element_pair_index, int64 xor_mask, llvm::Type* index_type, + int64 iota_values_parameter_index, llvm::Value* element_pair_index, + int64 xor_mask, llvm::Type* index_type, std::function read_element, std::function write_element, @@ -139,34 +140,42 @@ void EmitCompareLoopBody( is_signed_comparison = false; } // If key2 < key1 - ksl.IfReturnVoid( - "is_smaller_than", + auto is_smaller_than = b->CreateICmp(is_signed_comparison ? llvm::ICmpInst::ICMP_SLT : llvm::ICmpInst::ICMP_ULT, - compare_key2, compare_key1), - [&]() { - // Swap key1 with key2. - write_element(0, current_keys_index, key2); - write_element(0, compare_keys_index, key1); - for (int64 i = 1; i <= num_values; ++i) { - // Also swap the values. - auto value1 = read_element(i, current_keys_index); - auto value2 = read_element(i, compare_keys_index); - write_element(i, current_keys_index, value2); - write_element(i, compare_keys_index, value1); - } - }); + compare_key2, compare_key1); + if (iota_values_parameter_index >= 0) { + auto keys_equal = b->CreateICmpEQ(compare_key1, compare_key2); + auto key_index1 = + read_element(iota_values_parameter_index, current_keys_index); + auto key_index2 = + read_element(iota_values_parameter_index, compare_keys_index); + auto index_is_smaller_than = + b->CreateICmp(llvm::ICmpInst::ICMP_ULT, key_index2, key_index1); + is_smaller_than = b->CreateOr( + is_smaller_than, b->CreateAnd(keys_equal, index_is_smaller_than)); + } + ksl.IfReturnVoid("is_smaller_than", is_smaller_than, [&]() { + // Swap key1 with key2. + write_element(0, current_keys_index, key2); + write_element(0, compare_keys_index, key1); + for (int64 i = 1; i <= num_values; ++i) { + // Also swap the values. + auto value1 = read_element(i, current_keys_index); + auto value2 = read_element(i, compare_keys_index); + write_element(i, current_keys_index, value2); + write_element(i, compare_keys_index, value1); + } + }); }); } -void EmitTiledCompareLoop(const IrArray::Index& tiled_keys_index, - int64 dimension_to_sort, - int64 dimension_to_sort_bound, - PrimitiveType keys_type, - absl::Span xor_masks, - const std::vector& params, - const std::vector& param_shmem_buffers, - int64 tile_size, llvm::IRBuilder<>* b) { +void EmitTiledCompareLoop( + const IrArray::Index& tiled_keys_index, int64 dimension_to_sort, + int64 dimension_to_sort_bound, PrimitiveType keys_type, + absl::Span xor_masks, const std::vector& params, + const std::vector& param_shmem_buffers, + int64 iota_values_parameter_index, int64 tile_size, llvm::IRBuilder<>* b) { KernelSupportLibrary ksl(b); llvm::Value* thread_id = llvm_ir::EmitCallToIntrinsic( llvm::Intrinsic::nvvm_read_ptx_sreg_tid_x, {}, {}, b); @@ -253,20 +262,22 @@ void EmitTiledCompareLoop(const IrArray::Index& tiled_keys_index, RoundDownToNearest(dimension_to_sort_bound, tile_size))), [&]() { EmitCompareLoopBody(dimension_to_sort_bound % tile_size, keys_type, - params.size() - 1, element_pair_index, xor_mask, + params.size() - 1, iota_values_parameter_index, + element_pair_index, xor_mask, tiled_keys_index.GetType(), read_element, write_element, b); }, [&]() { - EmitCompareLoopBody( - tile_size, keys_type, params.size() - 1, element_pair_index, - xor_mask, tiled_keys_index.GetType(), read_element, - write_element, b, /*needs_bounds_checks=*/false); + EmitCompareLoopBody(tile_size, keys_type, params.size() - 1, + iota_values_parameter_index, element_pair_index, + xor_mask, tiled_keys_index.GetType(), + read_element, write_element, b, + /*needs_bounds_checks=*/false); }); } else { EmitCompareLoopBody(tile_size, keys_type, params.size() - 1, - element_pair_index, xor_mask, - tiled_keys_index.GetType(), read_element, + iota_values_parameter_index, element_pair_index, + xor_mask, tiled_keys_index.GetType(), read_element, write_element, b, /*needs_bounds_checks=*/false); } // Wait until all comparisons have happened. @@ -296,6 +307,7 @@ void EmitTiledCompareLoop(const IrArray::Index& tiled_keys_index, Status EmitSortInPlace(int64 dimension_to_sort, const IrArray& keys_array, const std::vector& values_arrays, + int64 iota_values_parameter_index, absl::string_view name, absl::Span xor_masks, llvm::IRBuilder<>* b, const gpu::LaunchDimensions& launch_dimensions, @@ -367,8 +379,8 @@ Status EmitSortInPlace(int64 dimension_to_sort, const IrArray& keys_array, if (xor_masks.size() > 1) { EmitTiledCompareLoop(keys_index, dimension_to_sort, dimension_to_sort_bound, keys_shape.element_type(), - xor_masks, params, param_shmem_buffers, tile_size, - b); + xor_masks, params, param_shmem_buffers, + iota_values_parameter_index, tile_size, b); } else { auto read_element = [&](int64 operand, llvm::Value* index) { keys_index[dimension_to_sort] = index; @@ -380,9 +392,10 @@ Status EmitSortInPlace(int64 dimension_to_sort, const IrArray& keys_array, params[operand].EmitWriteArrayElement(keys_index, value, b); }; EmitCompareLoopBody(dimension_to_sort_bound, keys_shape.element_type(), - values_arrays.size(), tiles_index[rank - 1], - xor_masks[0], tiles_index.GetType(), read_element, - write_element, b); + values_arrays.size(), iota_values_parameter_index, + tiles_index[rank - 1], xor_masks[0], + tiles_index.GetType(), read_element, write_element, + b); } return Status::OK(); }; diff --git a/tensorflow/compiler/xla/service/llvm_ir/sort_util.h b/tensorflow/compiler/xla/service/llvm_ir/sort_util.h index 556a217322d..685f9383acb 100644 --- a/tensorflow/compiler/xla/service/llvm_ir/sort_util.h +++ b/tensorflow/compiler/xla/service/llvm_ir/sort_util.h @@ -31,9 +31,12 @@ namespace llvm_ir { // Emits llvm IR to do pairwise comparisons/swaps in the 'dimension_to_sort' // dimension of 'keys_array'. All other dimensions are kept as-is. This // implements the inner loop of BitonicSort. It is assumed that 'xor_masks' -// contains only powers of 2, or values 2^k - 1 (k > 0). +// contains only powers of 2, or values 2^k - 1 (k > 0). If +// 'iota_values_parameter_index' is >= 0, it points at a 'values_arrays' operand +// that is a iota and can be used to make the sorting stable. Status EmitSortInPlace(int64 dimension_to_sort, const IrArray& keys_array, const std::vector& values_arrays, + int64 iota_values_parameter_index, absl::string_view name, absl::Span xor_masks, llvm::IRBuilder<>* b, const gpu::LaunchDimensions& launch_dimensions, diff --git a/tensorflow/compiler/xla/service/local_service.cc b/tensorflow/compiler/xla/service/local_service.cc index cca37556173..6c897009833 100644 --- a/tensorflow/compiler/xla/service/local_service.cc +++ b/tensorflow/compiler/xla/service/local_service.cc @@ -96,44 +96,18 @@ ExecutionOptions CreateExecutionOptions( const ExecutableBuildOptions& build_options, const ProgramShape* program_shape) { ExecutionOptions execution_options = CreateDefaultExecutionOptions(); - if (build_options.hlo_profile().has_value()) { - execution_options.mutable_debug_options()->set_xla_hlo_profile( - *build_options.hlo_profile()); - } - if (build_options.generate_hlo_graph().has_value()) { - execution_options.mutable_debug_options()->set_xla_generate_hlo_graph( - build_options.generate_hlo_graph().value()); - } - if (build_options.dump_optimized_hlo_proto_to().has_value()) { - execution_options.mutable_debug_options() - ->set_xla_dump_optimized_hlo_proto_to( - build_options.dump_optimized_hlo_proto_to().value()); - } - if (build_options.dump_unoptimized_hlo_proto_to().has_value()) { - execution_options.mutable_debug_options() - ->set_xla_dump_unoptimized_hlo_proto_to( - build_options.dump_unoptimized_hlo_proto_to().value()); - } - if (build_options.dump_per_pass_hlo_proto_to().has_value()) { - execution_options.mutable_debug_options() - ->set_xla_dump_per_pass_hlo_proto_to( - build_options.dump_per_pass_hlo_proto_to().value()); + if (build_options.has_debug_options()) { + *execution_options.mutable_debug_options() = build_options.debug_options(); } if (build_options.result_layout() != nullptr) { *execution_options.mutable_shape_with_output_layout() = - *build_options.result_layout(); + build_options.result_layout()->ToProto(); } else { + Shape result_shape(program_shape->result()); + LayoutUtil::SetToDefaultLayout(&result_shape); *execution_options.mutable_shape_with_output_layout() = - program_shape->result(); - LayoutUtil::SetToDefaultLayout( - execution_options.mutable_shape_with_output_layout()); + result_shape.ToProto(); } - - for (const std::string& disabled_pass : build_options.disabled_hlo_passes()) { - execution_options.mutable_debug_options()->add_xla_disable_hlo_passes( - disabled_pass); - } - return execution_options; } @@ -145,7 +119,7 @@ StatusOr> LocalService::CompileExecutable( const ExecutableBuildOptions& build_options) { const HloModuleProto& proto = computation.proto(); TF_RET_CHECK(proto.has_host_program_shape()); - const ProgramShape& program_shape = proto.host_program_shape(); + ProgramShape program_shape(proto.host_program_shape()); // Validate incoming layouts. if (argument_layouts.size() != program_shape.parameters_size()) { @@ -220,4 +194,10 @@ StatusOr LocalService::GlobalDataToShapedBuffer( return buffers[replica_number]; } +StatusOr LocalService::RegisterReplicatedBuffers( + std::vector replicated_buffers, const string& tag) { + return allocation_tracker_.RegisterReplicatedBuffers( + std::move(replicated_buffers), tag); +} + } // namespace xla diff --git a/tensorflow/compiler/xla/service/local_service.h b/tensorflow/compiler/xla/service/local_service.h index 3b4f0b50832..f56ba32b04b 100644 --- a/tensorflow/compiler/xla/service/local_service.h +++ b/tensorflow/compiler/xla/service/local_service.h @@ -63,6 +63,11 @@ class LocalService : public Service { StatusOr GlobalDataToShapedBuffer( const GlobalDataHandle& data, int replica_number); + // Registers a vector of shaped buffers of device memory, one per replica, and + // returns a corresponding handle that can be used for talking to XLA clients. + StatusOr RegisterReplicatedBuffers( + std::vector replicated_buffers, const string& tag); + private: explicit LocalService(const ServiceOptions& options, std::unique_ptr backend); diff --git a/tensorflow/compiler/xla/service/logical_buffer_analysis.cc b/tensorflow/compiler/xla/service/logical_buffer_analysis.cc index ec52a24d782..972a5b9ced0 100644 --- a/tensorflow/compiler/xla/service/logical_buffer_analysis.cc +++ b/tensorflow/compiler/xla/service/logical_buffer_analysis.cc @@ -113,6 +113,13 @@ Status LogicalBufferAnalysis::HandleGetTupleElement(HloInstruction*) { return Status::OK(); } +Status LogicalBufferAnalysis::HandleAddDependency( + HloInstruction* add_dependency) { + // AddDependency just forwards the value of its zero-th operand and does not + // create buffers. + return Status::OK(); +} + Status LogicalBufferAnalysis::HandleCopy(HloInstruction* copy) { // The top-level buffer (index={}) for kCopy is newly created, but all other // buffers (in the case of a tuple shape) come from the operand diff --git a/tensorflow/compiler/xla/service/logical_buffer_analysis.h b/tensorflow/compiler/xla/service/logical_buffer_analysis.h index 81f524d84a8..7ffca943d0f 100644 --- a/tensorflow/compiler/xla/service/logical_buffer_analysis.h +++ b/tensorflow/compiler/xla/service/logical_buffer_analysis.h @@ -64,6 +64,7 @@ class LogicalBufferAnalysis : public DfsHloVisitorWithDefault { Status HandleRecvDone(HloInstruction* recv_done) override; Status HandleSend(HloInstruction* send) override; Status HandleTupleSelect(HloInstruction* tuple_select) override; + Status HandleAddDependency(HloInstruction* add_dependency) override; // A map from the buffer ID to the logical buffer std::vector> logical_buffers_; diff --git a/tensorflow/compiler/xla/service/pattern_matcher.h b/tensorflow/compiler/xla/service/pattern_matcher.h index 6152cdc6099..432aa1ea0b6 100644 --- a/tensorflow/compiler/xla/service/pattern_matcher.h +++ b/tensorflow/compiler/xla/service/pattern_matcher.h @@ -16,6 +16,7 @@ limitations under the License. #ifndef TENSORFLOW_COMPILER_XLA_SERVICE_PATTERN_MATCHER_H_ #define TENSORFLOW_COMPILER_XLA_SERVICE_PATTERN_MATCHER_H_ +#include "absl/strings/str_replace.h" #include "absl/strings/string_view.h" #include "absl/utility/utility.h" #include "tensorflow/compiler/xla/layout_util.h" @@ -44,32 +45,48 @@ namespace xla { // // This pattern will match Add instructions whose first operand is a constant. // -// Each pattern type has the following modifiers: +// Each pattern type has the following modifiers, which are described where +// nontrivial. // // Op(): -// - WithName: match operations with the given name -// - WithOpcode: match operations with the given opcode -// - WithShape: match operations whose shape matches the given pattern -// - WithOperand: match operations whose operand matches the given pattern +// - Is: is the given HloInstruction* (i.e. pointer equality) +// - WithName +// - WithOpcode +// - WithoutOpcode: anything other than the given opcode +// - WithShape: instr's shape matches the given pattern +// - WithShapeEqualTo: instr's shape is equal to the given Shape +// - WithShapeCompatibleTo: instr's shape is compatible with the given Shape +// - WithNumOperands +// - WithOperand: operand at the given index matches the given pattern +// - IsConstant +// - IsNonConstant +// - IsConstantScalar/IsEffectiveConstantScalar: Optionally accepts a value, +// e.g. IsConstantScalar() or IsConstantScalar(42). +// - WithFusionKind +// - WithTupleIndex: get-tuple-element operations with the given tuple index +// - WithOneUse: Instruction is used as an operand exactly once. +// - WithOneUser: Instruction is used by exactly one other instruction, but +// is possibly used more than once as an operand (e.g. multiply(x,x)). // // Shape(): -// - EqualTo: matches shapes that are equal to the argument -// - CompatibleTo: matches shapes that are compatible to the argument -// - IsScalar/IsArray/IsTuple: matches scalar/array/tuple shapes -// - IsDenseArray/IsSparseArray: matches arrays with dense/sparse format -// - WithLayout: match shapes whose layout matches the given pattern -// - WithLayoutEqualTo: matches shapes whose layouts equal the argument -// - WithSubshape: matches tuple shapes whose subshape matches the given -// pattern -// - WithSubshapeEqualTo: matches shapes with a subshape equal the argument -// - WithElementType: matches array/scalar shapes with the given element -// type -// - WithRank: matches array/scalar types with the given rank +// - EqualTo +// - CompatibleTo +// - IsScalar/IsEffectiveScalar/IsArray/IsTuple +// - IsDenseArray/IsSparseArray +// - WithLayout: layout shape's layout matches the given pattern (e.g. +// Layout().WithDenseFormat()) +// - WithLayoutEqualTo: shape's layout equals the argument (i.e. another +// Layout, but not the result of Layout().foo()) +// - WithSubshape: shape is a tuple whose subshape matches the given pattern +// (e.g. Shape().IsScalar()). +// - WithSubshapeEqualTo: shape is a tuple with a subshape equal to the arg +// (i.e. another Shape, but not the result of Shape().foo()) +// - WithElementType: shape is an array/scalar with the given elem type +// - WithRank: shape is an array/scalar with the given rank // // Layout(): -// - EqualTo: matches layouts that are equal to the argument -// - WithDenseFormat/WithSparseFormat: matches layouts with dense/sparse -// format +// - EqualTo +// - WithDenseFormat/WithSparseFormat // // Op(), Shape(), and Layout() may be passed an argument of type // HloInstruction**, Shape**, or Layout**, respectively, or const versions of @@ -82,53 +99,55 @@ namespace xla { // CHECK(Match(foo, // match::Op().WithOperand(0, match::Op(&matched_operand)))); // -// Helpers are provided for common nullary, unary, binary, and ternary -// instructions. These helpers can be called with no arguments, in which case -// they will match any instruction matching the opcode. They may also be called -// with matches for the operands and with an optional capture. (The capture must -// be the first argument.) Some examples of these helpers and their equivalents -// are provided below. -// +// Helpers are provided for most HLO instructions. These helpers can be called +// with no arguments, in which case they will match any instruction matching the +// opcode. They may also be called with matches for the operands and with an +// optional capture. (The capture must be the first argument.) Some examples of +// these helpers and their equivalents are provided below. + // Example nullary instruction: -// Param() == Op().WithOpcode(HloOpcode::kParam) -// Param(&a) == Op(&a).WithOpcode(HloOpcode::kParam) +// Parameter() == Op().WithOpcode(HloOpcode::kParameter) +// Parameter(&a) == Op(&a).WithOpcode(HloOpcode::kParameter) // // Example unary instruction: -// Abs() == Op().WithOpcode(HloOpcode::kAbs) -// Abs(Op(&a)) == Op().WithOpcode(HloOpcode::kAbs) -// .WithOperand(0, Op(&a))) -// Abs(&a, Op(&b)) == Op(&a).WithOpcode(HloOpcode::kAbs) -// .WithOperand(0, Op(&b)) +// Abs() == Op().WithOpcode(HloOpcode::kAbs) +// Abs(Op(&a)) == Op().WithOpcode(HloOpcode::kAbs) +// .WithOperand(0, Op(&a))) +// Abs(&a, Op(&b)) == Op(&a).WithOpcode(HloOpcode::kAbs) +// .WithOperand(0, Op(&b)) // -// Example binary instruction: -// Add() == Op().WithOpcode(HloOpcode::kAdd) -// Add(Op(&a), Op(&b)) == Op().WithOpcode(HloOpcode::kAdd) -// .WithOperand(0, Op(&a)) -// .WithOperand(1, Op(&b)) -// Add(&a, Op(&b), Op(&c)) == Op(&a).WithOpcode(HloOpcode::kAdd) -// .WithOperand(0, Op(&b)) -// .WithOperand(1, Op(&c)) +// Commutative binary instructions have a special form that accepts either order +// of args, e.g.: // -// Example ternary instruction: -// Clamp() == Op().WithOpcode(HloOpcode::kClamp) -// Clamp(Op(&a), Op(&b), Op(&c)) == Op().WithOpcode(HloOpcode::kClamp) -// .WithOperand(0, Op(&a)) -// .WithOperand(1, Op(&b)) -// .WithOperand(2, Op(&c)) -// Clamp(&a, Op(&b), Op(&c), Op(&d)) == Op(&a).WithOpcode(HloOpcode::kClamp) -// .WithOperand(0, Op(&b)) -// .WithOperand(1, Op(&c)) -// .WithOperand(2, Op(&d)) +// AddAnyOrder(Parameter(1), Abs()) == +// Op().WithOpcode(HloOpcode::kAdd) +// .WithBinaryOperandsAnyOrder(Op().WithParameterNum(1), Abs()); // +// MultiplyAnyOrder(&a, Parameter(), Abs()) // Captures the mul in `a`. +// +// The following additional helpers are provided. In all cases, `&a` is +// optional. +// +// ConstantScalar(&a) == Op(&a).IsConstantScalar(); +// ConstantScalar(&a, v) == Op(&a).IsConstantScalar(v); +// ConstantEffectiveScalar(&a) == Op(&a).IsConstantEffectiveScalar(); +// ConstantEffectiveScalar(&a, v) == Op(&a).IsConstantEffectiveScalar(&a, v) +// NonConstant(&a) == Op(&a).IsNonConstant() +// GetTupleElement(&a, b, index) == Op(&a).WithTupleIndex(index) +// .WithOperand(0, b); +// Parameter(&a, n) == Op(&a).WithParameterNum(n); struct MatchOption { // If true, actually capture matched item into the user pointer. bool capture; + + // An explanation for why we failed to match is streamed here, if not-null. + std::ostream* explain_os; }; template bool Match(Value* value, const Pattern& pattern, - MatchOption option = {/*.capture=*/true}) { + MatchOption option = {/*.capture=*/true, /*.explain_os=*/nullptr}) { if (option.capture) { auto new_option = option; new_option.capture = false; @@ -143,6 +162,77 @@ namespace match { namespace detail { +// Macro for streaming to option.explain_os if it's not null. +// +// EXPLAIN << "value of foo(): " << foo() +// +#pragma push_macro("EXPLAIN") +#define EXPLAIN \ + if (option.explain_os) *option.explain_os + +// kIndentInc is the additional number of spaces that we indent by when we +// increase the indent "by one". +enum { + kIndentInc = 2, +}; + +// Writes a newline and then `indent` spaces. +// +// We follow an unintuitive convention in this file's pretty-printers: Indents +// are performed by the caller, not the callee. For example, if you want to +// print +// +// foo: +// - bar +// +// you'd do: +// +// Foo::DescribeTo(std::ostream* os, int64 indent) { +// *os << "foo:"; +// Indent(os, indent) // Create a newline at the *current* indent level. +// *os << " - "; +// bar.DescribeTo(os, indent + 3); // + 3 because strlen(" * ") == 3. +// } +// +// Bar::DescribeTo(std::ostream* os, int64 indent) { *os << "bar"; } +// +// Notice that Bar::DescribeTo() does not call Indent; the indenting is +// performed by Foo. This convention allows the caller to decide whether a +// matcher is preceded by a newline, which is important e.g. for the AllOf +// matcher. +// +// (Incidentally, indenting in Match's explanations is handled differently. +// Indents are a common case in DescribeTo [we're printing a whole tree], but +// they're a special case in Match [we're printing only a path through the tree +// that encounters a failing node]. Indents in Match only appear when we +// encounter a failing disjunction, so we just handle them as a special case +// there.) +inline void Indent(std::ostream* os, int64 indent) { + *os << "\n"; + for (int64 i = 0; i < indent; ++i) { + *os << " "; + } +} + +// SFINAE template that determines whether T declares a static member +// kIsTrivialMatcher. +// +// Trivial matchers get special treatment. For example, when printing +// a conjunction of matchers, we don't print "and" after a trivial matcher. This +// yields e.g. +// "a shape compatible with f32[1,2]" +// rather than +// "a shape AND compatible with f32[1,2]" +template +struct IsTrivialMatcher { + static constexpr bool value = false; +}; +template +struct IsTrivialMatcher::type> { + static constexpr bool value = true; +}; + template class AllOfPattern { public: @@ -162,10 +252,19 @@ class AllOfPattern { return matched; } + void DescribeTo(std::ostream* os, int64 indent = 0) const { + DescribeToImpl(os, std::integral_constant(), indent); + } + + // Accessor for patterns_. Please don't use this outside of this file. + const std::tuple& patterns() const { return patterns_; } + private: template bool MatchImpl(ItemType* item, MatchOption option, std::integral_constant) const { + // We don't need to do any EXPLAINing here; it's all correctly handled by + // our sub-matchers (if any fail). return std::get(patterns_).Match(item, option) && MatchImpl(item, option, std::integral_constant()); } @@ -176,6 +275,73 @@ class AllOfPattern { return true; } + // Pretty-printing a conjunction has some special cases to make it easy to + // read in the simple (common) case. + // + // If sizeof...(Patterns) == 1, prints as e.g. + // + // a shape + // + // If sizeof...(Patterns) == 2 and patterns_[0] is a trivial matcher (e.g. "a + // shape") prints as + // + // a shape compatible with f32[1,2] + // + // If sizeof...(Patterns) > 2 and patterns_[0] is a trivial matcher, prints as + // + // a shape: + // * compatible with f32[1,2] AND + // * that represents a scalar + // + // Otherwise prints as: + // + // all of: + // * foo AND + // * bar + // + template + void DescribeToImpl(std::ostream* os, std::integral_constant, + int64 indent) const { + constexpr bool first_is_trivial = + IsTrivialMatcher(patterns_))>::type>::value; + constexpr bool is_last = index == sizeof...(Patterns) - 1; + const auto& submatcher = std::get(patterns_); + + auto print_bulleted_item = [&] { + *os << " * "; + submatcher.DescribeTo(os, indent + 3); + if (!is_last) { + *os << " AND"; + Indent(os, indent); + } + }; + + if (index == 0) { + if (first_is_trivial || is_last) { + submatcher.DescribeTo(os, indent + kIndentInc); + if (sizeof...(Patterns) > 2) { + *os << ":"; + Indent(os, indent); + } + } else { + *os << "all of:"; + Indent(os, indent); + print_bulleted_item(); + } + } else if (first_is_trivial && index == 1 && sizeof...(Patterns) == 2) { + *os << " "; + submatcher.DescribeTo(os, indent); + } else { + print_bulleted_item(); + } + DescribeToImpl(os, std::integral_constant(), indent); + } + + void DescribeToImpl(std::ostream* os, + std::integral_constant, + int64 indent) const {} + std::tuple patterns_; }; @@ -183,10 +349,6 @@ class AllOfPattern { // Returns a pattern that represents the conjunction of all input patterns. All // patterns need to match in order to have the AllOf pattern match. -// -// TODO(timshen): Currently AllOf is still nested, e.g. AllOf, B> is -// not AllOf. We might want to flatten the AllOf type structure if the -// C++ compile error message gets annoying. template detail::AllOfPattern::type, Patterns...> AllOf( const Patterns&... patterns) { @@ -194,6 +356,25 @@ detail::AllOfPattern::type, Patterns...> AllOf( Patterns...>(patterns...); } +// AllOf, X, Y, ...> => AllOf. +// +// This transformation is necessary for good pretty-printing. +template +detail::AllOfPattern::type, InnerPs..., + OuterPs...> +AllOf(const detail::AllOfPattern& inner_p, + const OuterPs&... outer_ps) { + // Invoke constructor of AllOfPattern. + auto make_all_of = [](const InnerPs&... inner_ps, + const OuterPs&... outer_ps) { + return detail::AllOfPattern::type, + InnerPs..., OuterPs...>(inner_ps..., + outer_ps...); + }; + return absl::apply(make_all_of, std::tuple_cat(inner_p.patterns(), + std::make_tuple(outer_ps...))); +} + namespace detail { template @@ -204,8 +385,18 @@ class LayoutPattern; class LayoutPatternBaseImpl { public: bool Match(const ::xla::Layout* layout, MatchOption option) const { - return layout != nullptr; + if (layout == nullptr) { + EXPLAIN << "Layout is null"; + return false; + } + return true; } + + void DescribeTo(std::ostream* os, int64 indent = 0) const { + *os << "a layout"; + } + + static constexpr bool kIsTrivialMatcher = true; }; // A LayoutPattern implementation that matches only if the layout equals a @@ -216,7 +407,17 @@ class LayoutPatternEqualImpl { : layout_(layout) {} bool Match(const ::xla::Layout* layout, MatchOption option) const { - return LayoutUtil::Equal(*layout_, *layout); + if (!LayoutUtil::Equal(*layout_, *layout)) { + EXPLAIN << "Layout " << LayoutUtil::HumanString(*layout) + << " is not equal to expected " + << LayoutUtil::HumanString(*layout_); + return false; + } + return true; + } + + void DescribeTo(std::ostream* os, int64 indent = 0) const { + *os << "equal to " << LayoutUtil::HumanString(*layout_); } private: @@ -230,7 +431,16 @@ class LayoutPatternFormatImpl { explicit constexpr LayoutPatternFormatImpl(Format format) : format_(format) {} bool Match(const ::xla::Layout* layout, MatchOption option) const { - return layout->format() == format_; + if (layout->format() != format_) { + EXPLAIN << "Layout has format " << Format_Name(layout->format()) + << " but expected " << Format_Name(format_); + return false; + } + return true; + } + + void DescribeTo(std::ostream* os, int64 indent = 0) const { + *os << "with format " << Format_Name(format_); } private: @@ -242,11 +452,13 @@ template class LayoutPattern { private: template - LayoutPattern> - AppendImpl(NewImpl new_impl) const { - return LayoutPattern>( - AllOf(impl_, std::move(new_impl)), matched_layout_); + auto AppendImpl(NewImpl new_impl) const + -> LayoutPattern(std::declval(), + std::move(new_impl)))> { + auto new_allof = AllOf(impl_, std::move(new_impl)); + return LayoutPattern(std::move(new_allof), + matched_layout_); } public: @@ -276,6 +488,10 @@ class LayoutPattern { return false; } + void DescribeTo(std::ostream* os, int64 indent = 0) const { + impl_.DescribeTo(os, indent); + } + // Modifies the pattern to match only if the layout equals the given proto. // The layout must outlive the returned pattern. constexpr auto EqualTo(const ::xla::Layout* layout) const @@ -306,19 +522,48 @@ class AnyOfPattern { explicit AnyOfPattern(const Patterns&... patterns) : patterns_(patterns...) {} bool Match(const Item* item, MatchOption option) const { - return MatchImpl(item, option, std::integral_constant()); + return MatchImpl(item, option); } bool Match(Item* item, MatchOption option) const { - return MatchImpl(item, option, std::integral_constant()); + return MatchImpl(item, option); + } + + void DescribeTo(std::ostream* os, int64 indent = 0) const { + *os << "any of:"; + Indent(os, indent); + DescribeToImpl(os, std::integral_constant(), indent); } private: + template + bool MatchImpl(ItemType* item, MatchOption option) const { + // If we're generating an explanation, buffer it until we know we failed. + absl::optional explanation; + MatchOption new_option = option; + if (option.explain_os) { + new_option.explain_os = &explanation.emplace(); + } + bool rv = MatchRecursiveImpl(item, new_option, + std::integral_constant()); + if (!rv && option.explain_os) { + EXPLAIN << "None of the following matchers succeeded:"; + EXPLAIN << explanation->str(); + } + return rv; + } + template - bool MatchImpl(ItemType* item, MatchOption option, - std::integral_constant) const { + bool MatchRecursiveImpl(ItemType* item, MatchOption option, + std::integral_constant) const { auto new_option = option; new_option.capture = false; + + absl::optional explanation; + if (option.explain_os) { + new_option.explain_os = &explanation.emplace(); + } + // Try to match the sub-pattern without capturing behavior. if (std::get(patterns_).Match(item, new_option)) { // Capture the branch. @@ -337,20 +582,46 @@ class AnyOfPattern { // AnyOf will be a runtime number indicate which sub-pattern is matched. // Then we run another pass to do captures only with the help of the // trace. - bool ret = std::get(patterns_).Match(item, option); - DCHECK(ret); + bool matched = std::get(patterns_).Match(item, option); + DCHECK(matched); } return true; } - return MatchImpl(item, option, std::integral_constant()); + if (option.explain_os) { + EXPLAIN << "\nMatcher #" << index + 1; + EXPLAIN << "\n - "; + std::get(patterns_).DescribeTo(option.explain_os, /*indent=*/3); + EXPLAIN << "\nfailed with"; + EXPLAIN << "\n - "; + EXPLAIN << absl::StrReplaceAll(explanation->str(), {{"\n", "\n "}}); + } + return MatchRecursiveImpl(item, option, + std::integral_constant()); } template - bool MatchImpl(ItemType* item, MatchOption option, - std::integral_constant) const { + bool MatchRecursiveImpl( + ItemType* item, MatchOption option, + std::integral_constant) const { return false; } + template + void DescribeToImpl(std::ostream* os, std::integral_constant, + int64 indent) const { + *os << " - "; + std::get(patterns_).DescribeTo(os, indent + 3); + if (index != sizeof...(Patterns) - 1) { + *os << " OR"; + Indent(os, indent); + } + DescribeToImpl(os, std::integral_constant(), indent); + } + + void DescribeToImpl(std::ostream* os, + std::integral_constant, + int64 indent) const {} + std::tuple patterns_; }; @@ -395,8 +666,17 @@ class ShapePattern; class ShapePatternBaseImpl { public: bool Match(const ::xla::Shape* shape, MatchOption option) const { + if (shape == nullptr) { + EXPLAIN << "Shape is null"; + } return shape != nullptr; } + + void DescribeTo(std::ostream* os, int64 indent = 0) const { + *os << "a shape"; + } + + static constexpr bool kIsTrivialMatcher = true; }; // A ShapePattern implementation that matches only if the shape equals a Shape @@ -407,7 +687,16 @@ class ShapePatternEqualImpl { : shape_(shape) {} bool Match(const ::xla::Shape* shape, MatchOption option) const { - return ShapeUtil::Equal(*shape_, *shape); + if (!ShapeUtil::Equal(*shape_, *shape)) { + EXPLAIN << "Shape not equal to " + << ShapeUtil::HumanStringWithLayout(*shape_); + return false; + } + return true; + } + + void DescribeTo(std::ostream* os, int64 indent = 0) const { + *os << "equal to " << ShapeUtil::HumanStringWithLayout(*shape_); } private: @@ -422,7 +711,16 @@ class ShapePatternCompatibleImpl { : shape_(shape) {} bool Match(const ::xla::Shape* shape, MatchOption option) const { - return ShapeUtil::Compatible(*shape_, *shape); + if (!ShapeUtil::Compatible(*shape_, *shape)) { + EXPLAIN << "Shape not compatible with " + << ShapeUtil::HumanString(*shape_); + return false; + } + return true; + } + + void DescribeTo(std::ostream* os, int64 indent = 0) const { + *os << "compatible with " << ShapeUtil::HumanString(*shape_); } private: @@ -437,7 +735,16 @@ class ShapePatternElementTypeImpl { : element_type_(element_type) {} bool Match(const ::xla::Shape* shape, MatchOption option) const { - return shape->element_type() == element_type_; + if (shape->element_type() != element_type_) { + EXPLAIN << "Shape does not have element type " + << PrimitiveType_Name(element_type_); + return false; + } + return true; + } + + void DescribeTo(std::ostream* os, int64 indent = 0) const { + *os << "with element type " << PrimitiveType_Name(element_type_); } private: @@ -450,7 +757,15 @@ class ShapePatternIsScalarImpl { explicit constexpr ShapePatternIsScalarImpl() {} bool Match(const ::xla::Shape* shape, MatchOption option) const { - return ShapeUtil::IsScalar(*shape); + if (!ShapeUtil::IsScalar(*shape)) { + EXPLAIN << "Shape is not a scalar"; + return false; + } + return true; + } + + void DescribeTo(std::ostream* os, int64 indent = 0) const { + *os << "that represents a scalar"; } }; @@ -460,7 +775,15 @@ class ShapePatternIsArrayImpl { explicit constexpr ShapePatternIsArrayImpl() {} bool Match(const ::xla::Shape* shape, MatchOption option) const { - return ShapeUtil::IsArray(*shape); + if (!ShapeUtil::IsArray(*shape)) { + EXPLAIN << "Shape is not an array"; + return false; + } + return true; + } + + void DescribeTo(std::ostream* os, int64 indent = 0) const { + *os << "that represents an array"; } }; @@ -470,7 +793,34 @@ class ShapePatternIsTupleImpl { explicit constexpr ShapePatternIsTupleImpl() {} bool Match(const ::xla::Shape* shape, MatchOption option) const { - return ShapeUtil::IsTuple(*shape); + if (!ShapeUtil::IsTuple(*shape)) { + EXPLAIN << "Shape is not a tuple"; + return false; + } + return true; + } + + void DescribeTo(std::ostream* os, int64 indent = 0) const { + *os << "that represents a tuple"; + } +}; + +// A ShapePattern implementation that matches only if the shape is an effective +// scalar. +class ShapePatternEffectiveScalarImpl { + public: + explicit constexpr ShapePatternEffectiveScalarImpl() {} + + bool Match(const ::xla::Shape* shape, MatchOption option) const { + if (!ShapeUtil::IsEffectiveScalar(*shape)) { + EXPLAIN << "Shape is not an effective scalar"; + return false; + } + return true; + } + + void DescribeTo(std::ostream* os, int64 indent = 0) const { + *os << "that is an effective scalar"; } }; @@ -481,7 +831,23 @@ class ShapePatternRankImpl { explicit constexpr ShapePatternRankImpl(int64 rank) : rank_(rank) {} bool Match(const ::xla::Shape* shape, MatchOption option) const { - return ShapeUtil::Rank(*shape) == rank_; + if (ShapeUtil::Rank(*shape) != rank_) { + if (rank_ == 0) { + EXPLAIN << "Shape is not a scalar"; + } else { + EXPLAIN << "Shape does not have rank " << rank_; + } + return false; + } + return true; + } + + void DescribeTo(std::ostream* os, int64 indent = 0) const { + if (rank_ == 0) { + *os << "that is a scalar"; + } else { + *os << "that has " << rank_ << " dimension" << (rank_ != 1 ? "s" : ""); + } } private: @@ -503,8 +869,21 @@ class ShapePatternLayoutImpl { } bool Match(Shape* shape, MatchOption option) const { - return LayoutUtil::HasLayout(*shape) && - layout_.Match(shape->mutable_layout(), option); + if (!LayoutUtil::HasLayout(*shape)) { + EXPLAIN << "Shape does not have a layout"; + return false; + } + if (!layout_.Match(shape->mutable_layout(), option)) { + EXPLAIN << "\nin layout"; + return false; + } + return true; + } + + void DescribeTo(std::ostream* os, int64 indent = 0) const { + *os << "with"; + Indent(os, indent + kIndentInc); + layout_.DescribeTo(os, indent + kIndentInc); } private: @@ -522,17 +901,40 @@ class ShapePatternSubshapeImpl { : index_(index), subshape_(subshape) {} bool Match(const ::xla::Shape* shape, MatchOption option) const { - return ShapeUtil::IndexIsValid(*shape, index_) && - subshape_.Match(&ShapeUtil::GetSubshape(*shape, index_), option); + return MatchImpl(shape, option); } bool Match(::xla::Shape* shape, MatchOption option) const { - return ShapeUtil::IndexIsValid(*shape, index_) && - subshape_.Match(ShapeUtil::GetMutableSubshape(shape, index_), - option); + return MatchImpl(shape, option); + } + + void DescribeTo(std::ostream* os, int64 indent = 0) const { + *os << "with subshape at index " << index_.ToString() << " which is"; + Indent(os, indent + kIndentInc); + subshape_.DescribeTo(os, indent + kIndentInc); } private: + Shape* GetSubshape(Shape* shape) const { + return ShapeUtil::GetMutableSubshape(shape, index_); + } + const Shape* GetSubshape(const Shape* shape) const { + return &ShapeUtil::GetSubshape(*shape, index_); + } + + template + bool MatchImpl(ShapeType* shape, MatchOption option) const { + if (!ShapeUtil::IndexIsValid(*shape, index_)) { + EXPLAIN << "No subshape at " << index_.ToString(); + return false; + } + if (!subshape_.Match(GetSubshape(shape), option)) { + EXPLAIN << "\nin subshape at " << index_.ToString(); + return false; + } + return true; + } + ShapeIndexView index_; ShapePattern subshape_; }; @@ -542,10 +944,12 @@ template class ShapePattern { private: template - ShapePattern> AppendImpl( - NewImpl new_impl) const { - return ShapePattern>( - AllOf(impl_, std::move(new_impl)), matched_shape_); + auto AppendImpl(NewImpl new_impl) const + -> ShapePattern(std::declval(), + std::move(new_impl)))> { + auto new_all_of = AllOf(impl_, std::move(new_impl)); + return ShapePattern(std::move(new_all_of), + matched_shape_); } public: @@ -560,6 +964,11 @@ class ShapePattern { } return true; } + if (shape) { + EXPLAIN << "\nin " + << (shape->has_layout() ? ShapeUtil::HumanStringWithLayout(*shape) + : ShapeUtil::HumanString(*shape)); + } return false; } @@ -571,9 +980,16 @@ class ShapePattern { } return true; } + EXPLAIN << "\nin " + << (shape->has_layout() ? ShapeUtil::HumanStringWithLayout(*shape) + : ShapeUtil::HumanString(*shape)); return false; } + void DescribeTo(std::ostream* os, int64 indent = 0) const { + return impl_.DescribeTo(os, indent); + } + // Modifies the pattern to match only if the shape equals the given proto. // The layout must outlive the returned pattern. constexpr auto EqualTo(const ::xla::Shape* shape) const @@ -612,6 +1028,11 @@ class ShapePattern { return AppendImpl(ShapePatternIsTupleImpl()); } + constexpr auto IsEffectiveScalar() const + -> decltype(this->AppendImpl(ShapePatternEffectiveScalarImpl())) { + return AppendImpl(ShapePatternEffectiveScalarImpl()); + } + // Modifies the pattern to match only if the shape has the given rank. constexpr auto WithRank(int64 rank) const -> decltype(this->AppendImpl(ShapePatternRankImpl(rank))) { @@ -706,6 +1127,22 @@ Shape(::xla::Shape** matched_shape) { namespace detail { +// Overloads to get a const or non-const operand out of an instruction. +inline HloInstruction* HloOperand(HloInstruction* instr, int64 idx) { + return instr->mutable_operand(idx); +} +inline const HloInstruction* HloOperand(const HloInstruction* instr, + int64 idx) { + return instr->operand(idx); +} + +// Pretty-printer for HloInstruction. Sort of like ToShortString, but with +// fewer %s and more shapes. +inline string InstToString(const HloInstruction* inst) { + return inst->ToString( + HloPrintOptions().set_print_metadata(false).set_print_percent(false)); +} + template class HloInstructionPattern; @@ -714,8 +1151,18 @@ class HloInstructionPattern; class HloInstructionPatternBaseImpl { public: bool Match(const ::xla::HloInstruction* inst, MatchOption option) const { - return inst != nullptr; + if (inst == nullptr) { + EXPLAIN << "HloInstruction* is null"; + return false; + } + return true; } + + void DescribeTo(std::ostream* os, int64 indent = 0) const { + *os << "an HloInstruction"; + } + + static constexpr bool kIsTrivialMatcher = true; }; // An HloInstructionPattern implementation that matches only if the instruction @@ -726,13 +1173,44 @@ class HloInstructionPatternNameImpl { : name_(name) {} bool Match(const ::xla::HloInstruction* inst, MatchOption option) const { - return inst->name() == name_; + if (inst->name() != name_) { + EXPLAIN << "HloInstruction not named \"" << name_ << "\""; + return false; + } + return true; + } + + void DescribeTo(std::ostream* os, int64 indent = 0) const { + *os << "named \"" << name_ << "\""; } private: absl::string_view name_; }; +// An HloInstructionPattern implementation that matches only if the instruction +// equals a particular pointer. +class HloInstructionIsImpl { + public: + explicit HloInstructionIsImpl(const HloInstruction* inst) : inst_(inst) {} + + bool Match(const ::xla::HloInstruction* inst, MatchOption option) const { + if (inst != inst_) { + EXPLAIN << "HloInstruction " << inst << " is not " << inst_ << " (" + << InstToString(inst_) << ")"; + return false; + } + return true; + } + + void DescribeTo(std::ostream* os, int64 indent = 0) const { + *os << "which is " << inst_ << " (" << InstToString(inst_) << ")"; + } + + private: + const HloInstruction* inst_; +}; + // An HloInstructionPattern implementation that matches only if the instruction // has a given opcode. class HloInstructionPatternOpcodeImpl { @@ -742,7 +1220,25 @@ class HloInstructionPatternOpcodeImpl { : opcode_(opcode), invert_(invert) {} bool Match(const ::xla::HloInstruction* inst, MatchOption option) const { - return (invert_ ^ (inst->opcode() == opcode_)); + if (invert_ && inst->opcode() == opcode_) { + EXPLAIN << "HloInstruction has opcode " << HloOpcodeString(opcode_) + << ", expected anything else"; + return false; + } + if (!invert_ && inst->opcode() != opcode_) { + EXPLAIN << "HloInstruction doesn't have opcode " + << HloOpcodeString(opcode_); + return false; + } + return true; + } + + void DescribeTo(std::ostream* os, int64 indent = 0) const { + if (!invert_) { + *os << "with opcode " << HloOpcodeString(opcode_); + } else { + *os << "with any opcode other than " << HloOpcodeString(opcode_); + } } private: @@ -757,8 +1253,17 @@ class HloInstructionPatternNumOperandsImpl { explicit constexpr HloInstructionPatternNumOperandsImpl(int64 num_operands) : num_operands_(num_operands) {} - bool Match(const ::xla::HloInstruction* inst, MatchOption /*option*/) const { - return inst->operand_count() == num_operands_; + bool Match(const ::xla::HloInstruction* inst, MatchOption option) const { + if (inst->operand_count() != num_operands_) { + EXPLAIN << "HloInstruction doesn't have " << num_operands_ << " operands"; + return false; + } + return true; + } + + void DescribeTo(std::ostream* os, int64 indent = 0) const { + *os << "with " << num_operands_ << " operand" + << (num_operands_ != 1 ? "s" : ""); } private: @@ -775,11 +1280,25 @@ class HloInstructionPatternShapeImpl { : shape_(shape) {} bool Match(const ::xla::HloInstruction* inst, MatchOption option) const { - return shape_.Match(&inst->shape(), option); + if (!shape_.Match(&inst->shape(), option)) { + EXPLAIN << "\nin output shape"; + return false; + } + return true; } bool Match(::xla::HloInstruction* inst, MatchOption option) const { - return shape_.Match(inst->mutable_shape(), option); + if (!shape_.Match(inst->mutable_shape(), option)) { + EXPLAIN << "\nin output shape"; + return false; + } + return true; + } + + void DescribeTo(std::ostream* os, int64 indent = 0) const { + *os << "outputting"; + Indent(os, indent + kIndentInc); + shape_.DescribeTo(os, indent + kIndentInc); } private: @@ -797,20 +1316,197 @@ class HloInstructionPatternOperandImpl { : operand_index_(operand_index), operand_(operand) {} bool Match(const ::xla::HloInstruction* inst, MatchOption option) const { - return operand_index_ < inst->operand_count() && - operand_.Match(inst->operand(operand_index_), option); + return MatchImpl(inst, option); } bool Match(::xla::HloInstruction* inst, MatchOption option) const { - return operand_index_ < inst->operand_count() && - operand_.Match(inst->mutable_operand(operand_index_), option); + return MatchImpl(inst, option); + } + + void DescribeTo(std::ostream* os, int64 indent = 0) const { + *os << "with operand " << operand_index_ << " which is:"; + Indent(os, indent + kIndentInc); + operand_.DescribeTo(os, indent + kIndentInc); } private: + template + bool MatchImpl(HloInstructionType* inst, MatchOption option) const { + if (operand_index_ >= inst->operand_count()) { + EXPLAIN << "desired operand index " << operand_index_ + << " is out of bounds"; + return false; + } + if (!operand_.Match(HloOperand(inst, operand_index_), option)) { + EXPLAIN << "\nin operand " << operand_index_; + return false; + } + return true; + } + int64 operand_index_; HloInstructionPattern operand_; }; +// Matches a binary instruction whose operands come in any order. +template +class HloInstructionPatternBinaryOperandsAnyOrderImpl { + public: + explicit constexpr HloInstructionPatternBinaryOperandsAnyOrderImpl( + const HloInstructionPattern& op1, + const HloInstructionPattern& op2) + : op1_(op1), op2_(op2) {} + + bool Match(HloInstruction* inst, MatchOption option) const { + return MatchImpl(inst, option); + } + + bool Match(const HloInstruction* inst, MatchOption option) const { + return MatchImpl(inst, option); + } + + void DescribeTo(std::ostream* os, int64 indent = 0) const { + *os << "with two operands in either order:"; + Indent(os, indent); + *os << " - "; + op1_.DescribeTo(os, indent + 3); + Indent(os, indent); + *os << " - "; + op2_.DescribeTo(os, indent + 3); + } + + private: + HloInstruction* operand(HloInstruction* inst, int64 idx) const { + return inst->mutable_operand(idx); + } + const HloInstruction* operand(const HloInstruction* inst, int64 idx) const { + return inst->operand(idx); + } + + template + bool MatchImpl(HloInstructionType* inst, MatchOption option) const { + // We could implement this using AnyOf and AllOf matchers, but the templates + // get pretty difficult to debug, since any compile error herein becomes + // not-an-error via SFINAE. Also this way lets us give better messages on + // failure. + if (inst->operand_count() != 2) { + EXPLAIN << "HloInstruction did not have two operands"; + return false; + } + + // If we're not generating explanations, this is pretty simple. + if (!option.explain_os) { + auto try_match = [&](int64 idx1, int64 idx2) { + MatchOption new_option = option; + new_option.capture = false; + if (op1_.Match(operand(inst, idx1), new_option) && + op2_.Match(operand(inst, idx2), new_option)) { + if (option.capture) { + bool matched = op1_.Match(operand(inst, idx1), option) && + op2_.Match(operand(inst, idx2), option); + DCHECK(matched); + } + return true; + } + return false; + }; + return try_match(0, 1) || try_match(1, 0); + } + + // If we are generating explanations, we have some work to do in order to + // generate a helpful error. + // + // First, try all four operand/matcher combinations, recording the + // failure explanations separately from option.explain_os. matches[i][j] + // tells us if matcher_i matches operand j. + bool matches[/*matcher*/ 2][/*operand*/ 2]; + std::stringstream explanations[/*matcher*/ 2][/*operand*/ 2]; + for (int i = 0; i < 2; ++i) { + for (int j = 0; j < 2; ++j) { + MatchOption new_option = option; + new_option.capture = false; + new_option.explain_os = &explanations[i][j]; + matches[i][j] = i == 0 ? op1_.Match(operand(inst, j), new_option) + : op2_.Match(operand(inst, j), new_option); + } + } + + // Check if the match succeeded. + for (int i = 0; i < 2; ++i) { + if (matches[0][i] && matches[1][(i + 1) % 2]) { + // Rerun the matches with capture enabled if necessary. + if (option.capture) { + auto* operand1 = operand(inst, i); + auto* operand2 = operand(inst, (i + 1) % 2); + bool matched = + op1_.Match(operand1, option) && op2_.Match(operand2, option); + DCHECK(matched); + } + return true; + } + } + + auto describe_matcher = [&](int matcher_idx) { + EXPLAIN << "\n - "; + if (matcher_idx == 0) { + op1_.DescribeTo(option.explain_os, /*indent=*/3); + } else { + CHECK_EQ(matcher_idx, 1); + op2_.DescribeTo(option.explain_os, /*indent=*/3); + } + for (int i = 0; i < 2; ++i) { + if (matches[matcher_idx][/*operand*/ i]) { + continue; + } + EXPLAIN << "\ndoes not match " << (i == 0 ? "LHS" : "RHS") << ":\n"; + EXPLAIN << " - "; + EXPLAIN << absl::StrReplaceAll( + explanations[matcher_idx][/*operand*/ i].str(), {{"\n", "\n "}}); + } + }; + + // If we failed to match, one of the following is true: + // 1. op1 (op2) matches neither LHS nor RHS, or + // 2. op1 and op2 both match LHS (RHS), but neither matches RHS (LHS). + // We print different explanations depending on which case we're in. + + // Case 1. + bool wrote_explanation = false; + for (int i = 0; !wrote_explanation && i < 2; ++i) { + if (!matches[i][0] && !matches[i][1]) { + EXPLAIN << "HloInstruction's operands (ignoring order) did not match " + << (i == 0 ? "first" : "second") << " matcher. Specifically,"; + describe_matcher(i); + wrote_explanation = true; + } + } + + // Case 2. + for (int i = 0; !wrote_explanation && i < 2; ++i) { + if (matches[/*matcher*/ 0][/*operand*/ i] && + matches[/*matcher*/ 1][/*operand*/ i]) { + CHECK(!matches[0][(i + 1) % 2]); + CHECK(!matches[1][(i + 1) % 2]); + CHECK(!wrote_explanation); + EXPLAIN << "HloInstruction's " << (i == 1 ? "LHS" : "RHS") + << " operand did not match either of the two matchers. " + "Specifically,"; + describe_matcher(0); + EXPLAIN << "\nand"; + describe_matcher(1); + wrote_explanation = true; + } + } + + CHECK(wrote_explanation); + return false; + } + + HloInstructionPattern op1_; + HloInstructionPattern op2_; +}; + // An HloInstructionPattern implementation that matches only if the instruction // is a fusion node with a particular kind. class HloInstructionPatternFusionKindImpl { @@ -820,14 +1516,32 @@ class HloInstructionPatternFusionKindImpl { : kind_(kind) {} bool Match(const ::xla::HloInstruction* inst, MatchOption option) const { - return inst->opcode() == HloOpcode::kFusion && inst->fusion_kind() == kind_; + return MatchImpl(inst, option); } bool Match(::xla::HloInstruction* inst, MatchOption option) const { - return inst->opcode() == HloOpcode::kFusion && inst->fusion_kind() == kind_; + return MatchImpl(inst, option); + } + + void DescribeTo(std::ostream* os, int64 indent = 0) const { + *os << "with fusion kind " << ToString(kind_); } private: + template + bool MatchImpl(HloInstructionType* inst, MatchOption option) const { + if (inst->opcode() != HloOpcode::kFusion) { + EXPLAIN << "HloInstruction does not have fusion kind " << ToString(kind_) + << "; it's not a fusion"; + return false; + } + if (inst->fusion_kind() != kind_) { + EXPLAIN << "HloInstruction does not have fusion kind " << ToString(kind_); + return false; + } + return true; + } + ::xla::HloInstruction::FusionKind kind_; }; @@ -839,47 +1553,211 @@ class HloInstructionPatternTupleIndexImpl { : tuple_index_(tuple_index) {} bool Match(const ::xla::HloInstruction* inst, MatchOption option) const { - return inst->opcode() == HloOpcode::kGetTupleElement && - inst->tuple_index() == tuple_index_; + return MatchImpl(inst, option); } bool Match(::xla::HloInstruction* inst, MatchOption option) const { - return inst->opcode() == HloOpcode::kGetTupleElement && - inst->tuple_index() == tuple_index_; + return MatchImpl(inst, option); + } + + void DescribeTo(std::ostream* os, int64 indent = 0) const { + *os << "which is a GTE with index " << tuple_index_; } private: + template + bool MatchImpl(HloInstructionType* inst, MatchOption option) const { + if (inst->opcode() != HloOpcode::kGetTupleElement) { + EXPLAIN << "HloInstruction is not a GTE with index " << tuple_index_ + << "; it's not a GTE at all"; + return false; + } + if (inst->tuple_index() != tuple_index_) { + EXPLAIN << "HloInstruction is not a GTE with index " << tuple_index_; + return false; + } + return true; + } + int64 tuple_index_; }; -template -class HloPredicatePatternImpl { +class HloInstructionPatternParameterNumImpl { public: - explicit HloPredicatePatternImpl(Predicate pred) : pred_(std::move(pred)) {} + explicit constexpr HloInstructionPatternParameterNumImpl(int64 parameter_num) + : parameter_num_(parameter_num) {} - bool Match(const ItemType* item, MatchOption option) const { - return pred_(item); + bool Match(const ::xla::HloInstruction* inst, MatchOption option) const { + return MatchImpl(inst, option); } - bool Match(ItemType* item, MatchOption option) const { return pred_(item); } + bool Match(::xla::HloInstruction* inst, MatchOption option) const { + return MatchImpl(inst, option); + } + + void DescribeTo(std::ostream* os, int64 indent = 0) const { + *os << "which is parameter " << parameter_num_; + } private: - Predicate pred_; + template + bool MatchImpl(HloInstructionType* inst, MatchOption option) const { + if (inst->opcode() != HloOpcode::kParameter || + inst->parameter_number() != parameter_num_) { + EXPLAIN << "HloInstruction is not parameter " << parameter_num_; + return false; + } + return true; + } + + int64 parameter_num_; }; -struct PatternFriend; +// Superclass that contains common code used by Op::WithOneUse() and +// Op::WithOneUser(). +class HloInstructionPatternOneUseOrUserImpl { + protected: + bool MatchOneUser(const HloInstruction* inst, MatchOption option) const { + if (inst->user_count() != 1) { + EXPLAIN << "HloInstruction has " << inst->user_count() + << " users, but expected exactly one."; + if (inst->user_count() > 1) { + EXPLAIN << "\nAll users:"; + for (const HloInstruction* user : inst->users()) { + EXPLAIN << "\n - " << InstToString(user); + } + } + return false; + } + return true; + } +}; + +class HloInstructionPatternOneUseImpl + : public HloInstructionPatternOneUseOrUserImpl { + public: + bool Match(const HloInstruction* inst, MatchOption option) const { + if (!MatchOneUser(inst, option)) { + return false; + } + + int64 use_count = absl::c_count_if( + inst->users()[0]->operands(), + [&](const HloInstruction* operand) { return operand == inst; }); + if (use_count != 1) { + EXPLAIN << "HloInstruction is used " << use_count + << " times by its user, but is expected to be used just once: " + << InstToString(inst->users()[0]); + return false; + } + return true; + } + + void DescribeTo(std::ostream* os, int64 indent = 0) const { + *os << "which has exactly one use"; + } +}; + +class HloInstructionPatternOneUserImpl + : public HloInstructionPatternOneUseOrUserImpl { + public: + bool Match(const HloInstruction* inst, MatchOption option) const { + return MatchOneUser(inst, option); + } + + void DescribeTo(std::ostream* os, int64 indent = 0) const { + *os << "which has exactly one user (but possibly is used multiple times by " + "that instruction)"; + } +}; + +// Matches a constant scalar or effective scalar, optionally with a given value. +template +class HloConstantScalarImpl { + public: + explicit constexpr HloConstantScalarImpl(bool match_effective_scalar) + : val_(absl::nullopt), match_effective_scalar_(match_effective_scalar) {} + + constexpr HloConstantScalarImpl(ScalarTy val, bool match_effective_scalar) + : val_(val), match_effective_scalar_(match_effective_scalar) {} + + bool Match(const ::xla::HloInstruction* inst, MatchOption option) const { + return MatchImpl(inst, option); + } + + bool Match(::xla::HloInstruction* inst, MatchOption option) const { + return MatchImpl(inst, option); + } + + void DescribeTo(std::ostream* os, int64 indent = 0) const { + *os << "which is a constant " + << (match_effective_scalar_ ? "effective " : "") << "scalar"; + if (val_.has_value()) { + *os << " with value " << *val_; + } + } + + private: + template + bool MatchImpl(InstTy* inst, MatchOption option) const { + const auto* const_inst = DynCast(inst); + if (!const_inst) { + EXPLAIN << "HloInstruction is not a constant"; + return false; + } + if (match_effective_scalar_ && + !ShapeUtil::IsEffectiveScalar(inst->shape())) { + EXPLAIN << "HloInstruction is not an effective scalar"; + return false; + } + if (!match_effective_scalar_ && !ShapeUtil::IsScalar(inst->shape())) { + EXPLAIN << "HloInstruction is not a scalar"; + return false; + } + if (!val_.has_value()) { + return true; + } + + // Check that literal == static_cast(val) and + // val == static_cast(literal). This is sufficient to ensure that + // the two constant scalars are actually "equal". + auto val_literal = LiteralUtil::CreateR0(*val_); + auto literal_r0_or = const_inst->literal().Reshape({}); + auto val_as_literal_ty_or = + val_literal.Convert(const_inst->shape().element_type()); + if (!literal_r0_or.ok() || !val_as_literal_ty_or.ok()) { + EXPLAIN << "could not construct relevant Literals (how did this happen?)"; + return false; + } + auto literal_r0 = std::move(literal_r0_or).ValueOrDie(); + auto val_as_literal_ty = std::move(val_as_literal_ty_or).ValueOrDie(); + auto literal_r0_as_val_ty_or = + literal_r0.Convert(val_literal.shape().element_type()); + bool rv = literal_r0_as_val_ty_or.ok() && // + literal_r0_as_val_ty_or.ValueOrDie() == val_literal && + literal_r0 == val_as_literal_ty; + if (!rv) { + EXPLAIN << "HloInstruction's constant value " << literal_r0.ToString() + << " did not match expected value " << *val_; + } + return rv; + } + + absl::optional val_; + bool match_effective_scalar_; +}; // A pattern that matches HloInstructions. template class HloInstructionPattern { private: template - HloInstructionPattern> - AppendImpl(NewImpl new_impl) const { - return HloInstructionPattern< - HloInstructionType, AllOfPattern<::xla::HloInstruction, Impl, NewImpl>>( - AllOf(impl_, std::move(new_impl)), matched_inst_); + auto AppendImpl(NewImpl new_impl) const -> HloInstructionPattern< + HloInstructionType, decltype(AllOf( + std::declval(), std::move(new_impl)))> { + auto new_allof = AllOf(impl_, std::move(new_impl)); + return HloInstructionPattern( + std::move(new_allof), matched_inst_); } public: @@ -895,6 +1773,9 @@ class HloInstructionPattern { } return true; } + if (inst != nullptr) { + EXPLAIN << "\nin " << InstToString(inst); + } return false; } @@ -906,6 +1787,7 @@ class HloInstructionPattern { } return true; } + EXPLAIN << "\nin " << InstToString(inst); return false; } @@ -935,12 +1817,47 @@ class HloInstructionPattern { return AppendImpl(HloInstructionPatternOpcodeImpl(opcode, true)); } + constexpr auto Is(const HloInstruction* instr) const + -> decltype(this->AppendImpl(HloInstructionIsImpl(instr))) { + return AppendImpl(HloInstructionIsImpl(instr)); + } + // Modifies the pattern to match only if the instruction is a constant. constexpr auto IsConstant() const -> decltype(this->WithOpcode(HloOpcode::kConstant)) { return WithOpcode(HloOpcode::kConstant); } + constexpr auto IsConstantScalar() const -> decltype(this->AppendImpl( + HloConstantScalarImpl(/*match_effective_scalar=*/false))) { + return AppendImpl( + HloConstantScalarImpl(/*match_effective_scalar=*/false)); + } + + // This does not check that T has the same type as the instruction, so e.g. + // IsConstantScalar(1.0) may match a constant of shape int32[]. + template + constexpr auto IsConstantScalar(const ScalarTy& val) const + -> decltype(this->AppendImpl(HloConstantScalarImpl( + val, /*match_effective_scalar=*/false))) { + return AppendImpl( + HloConstantScalarImpl(val, /*match_effective_scalar=*/false)); + } + + constexpr auto IsConstantEffectiveScalar() const -> decltype(this->AppendImpl( + HloConstantScalarImpl(/*match_effective_scalar=*/true))) { + return AppendImpl( + HloConstantScalarImpl(/*match_effective_scalar=*/true)); + } + + template + constexpr auto IsConstantEffectiveScalar(const ScalarTy& val) const + -> decltype(this->AppendImpl(HloConstantScalarImpl( + val, /*match_effective_scalar=*/true))) { + return AppendImpl( + HloConstantScalarImpl(val, /*match_effective_scalar=*/true)); + } + // Modifies the pattern to match only if the instruction is not a constant. constexpr auto IsNonConstant() const -> decltype(this->WithoutOpcode(HloOpcode::kConstant)) { @@ -957,6 +1874,22 @@ class HloInstructionPattern { HloInstructionPatternShapeImpl(shape)); } + // Make this a templated function to work around gcc 4.9.4 template infinite + // recursion bug. + template + constexpr auto WithShapeEqualTo(const ::xla::Shape* shape) + -> decltype(this->WithShape(Shape().EqualTo(shape))) { + return WithShape(Shape().EqualTo(shape)); + } + + // Make this a templated function to work around gcc 4.9.4 template infinite + // recursion bug. + template + constexpr auto WithShapeCompatibleTo(const ::xla::Shape* shape) + -> decltype(this->WithShape(Shape().CompatibleTo(shape))) { + return WithShape(Shape().CompatibleTo(shape)); + } + // Modifies the pattern to match only if the instruction has an operand that // matches the given pattern. template @@ -971,6 +1904,20 @@ class HloInstructionPattern { operand_index, operand)); } + template + constexpr auto WithBinaryOperandsAnyOrder( + const HloInstructionPattern& op1, + const HloInstructionPattern& op2) const + -> decltype(this->AppendImpl( + HloInstructionPatternBinaryOperandsAnyOrderImpl< + OperandType1, OperandImpl1, OperandType2, OperandImpl2>(op1, + op2))) { + return AppendImpl( + HloInstructionPatternBinaryOperandsAnyOrderImpl< + OperandType1, OperandImpl1, OperandType2, OperandImpl2>(op1, op2)); + } + // Modifies the pattern to match only if the instruction is a fusion node with // the given kind. constexpr auto WithFusionKind(HloInstruction::FusionKind kind) const @@ -985,17 +1932,34 @@ class HloInstructionPattern { return AppendImpl(HloInstructionPatternTupleIndexImpl(tuple_index)); } - private: - template - constexpr auto WithPredicate(Predicate pred) const -> decltype( - this->AppendImpl(HloPredicatePatternImpl( - std::move(pred)))) { - return AppendImpl( - HloPredicatePatternImpl(std::move(pred))); + // Modifies the pattern to match only if the instruction is a parameter + // with the given parameter number. + constexpr auto WithParameterNum(int64 parameter_num) const -> decltype( + this->AppendImpl(HloInstructionPatternParameterNumImpl(parameter_num))) { + return AppendImpl(HloInstructionPatternParameterNumImpl(parameter_num)); } - friend struct PatternFriend; + // Modifies the pattern to match if the instruction is used exactly once. + // Does not match if the instruction is used twice by the same user (e.g. + // multiply(x,x)). + constexpr auto WithOneUse() const + -> decltype(this->AppendImpl(HloInstructionPatternOneUseImpl())) { + return AppendImpl(HloInstructionPatternOneUseImpl()); + } + // Modifies the pattern to match if the instruction is used by exactly one + // other instruction. Will match if the instruction is used twice, so long as + // it's by the same user (e.g. multiply(x,x)). + constexpr auto WithOneUser() const + -> decltype(this->AppendImpl(HloInstructionPatternOneUserImpl())) { + return AppendImpl(HloInstructionPatternOneUserImpl()); + } + + void DescribeTo(std::ostream* os, int64 indent = 0) const { + impl_.DescribeTo(os, indent); + } + + private: Impl impl_; HloInstructionType** matched_inst_; }; @@ -1036,6 +2000,7 @@ Op(::xla::HloInstruction** matched_inst) { XLA_NULLOP_PATTERN(Constant) XLA_NULLOP_PATTERN(Parameter) XLA_NULLOP_PATTERN(Iota) +XLA_NULLOP_PATTERN(Rng) #undef XLA_NULLOP_PATTERN // Helpers for unary instructions. @@ -1067,8 +2032,10 @@ XLA_UNOP_PATTERN(RoundNearestAfz) XLA_UNOP_PATTERN(Bitcast) XLA_UNOP_PATTERN(Broadcast) XLA_UNOP_PATTERN(Ceil) +XLA_UNOP_PATTERN(Convert) XLA_UNOP_PATTERN(Copy) XLA_UNOP_PATTERN(Cos) +XLA_UNOP_PATTERN(CrossReplicaSum) XLA_UNOP_PATTERN(Exp) XLA_UNOP_PATTERN(Fft) XLA_UNOP_PATTERN(Floor) @@ -1088,6 +2055,7 @@ XLA_UNOP_PATTERN(Reverse) XLA_UNOP_PATTERN(SendDone) XLA_UNOP_PATTERN(Sign) XLA_UNOP_PATTERN(Sin) +XLA_UNOP_PATTERN(Slice) XLA_UNOP_PATTERN(Sort) XLA_UNOP_PATTERN(Tanh) XLA_UNOP_PATTERN(Transpose) @@ -1125,25 +2093,32 @@ XLA_UNOP_PATTERN(Transpose) #define XLA_COMMUTATIVE_BINOP_PATTERN(NAME) \ XLA_BINOP_PATTERN(NAME) \ \ - template \ - inline auto NAME##AnyOrder(Lhs&& lhs, Rhs&& rhs) \ - ->decltype(AnyOf(NAME(lhs, rhs), NAME(rhs, lhs))) { \ - return AnyOf(NAME(lhs, rhs), NAME(rhs, lhs)); \ - } \ - \ template \ inline auto NAME##AnyOrder(HloInstructionType** matched_inst, Lhs&& lhs, \ Rhs&& rhs) \ - ->decltype(AnyOf(NAME(matched_inst, lhs, rhs), \ - NAME(matched_inst, rhs, lhs))) { \ - return AnyOf(NAME(matched_inst, lhs, rhs), \ - NAME(matched_inst, rhs, lhs)); \ + ->decltype(Op(matched_inst) \ + .WithOpcode(HloOpcode::k##NAME) \ + .WithBinaryOperandsAnyOrder(std::forward(lhs), \ + std::forward(rhs))) { \ + return Op(matched_inst) \ + .WithOpcode(HloOpcode::k##NAME) \ + .WithBinaryOperandsAnyOrder(std::forward(lhs), \ + std::forward(rhs)); \ + } \ + template \ + inline auto NAME##AnyOrder(Lhs&& lhs, Rhs&& rhs) \ + ->decltype(NAME##AnyOrder( \ + nullptr, std::forward(lhs), std::forward(rhs))) { \ + return NAME##AnyOrder( \ + nullptr, std::forward(lhs), std::forward(rhs)); \ } XLA_COMMUTATIVE_BINOP_PATTERN(Add) XLA_BINOP_PATTERN(Atan2) XLA_BINOP_PATTERN(Divide) XLA_BINOP_PATTERN(Complex) +XLA_BINOP_PATTERN(Convolution) XLA_BINOP_PATTERN(Dot) +XLA_BINOP_PATTERN(DynamicSlice) XLA_COMMUTATIVE_BINOP_PATTERN(Eq) XLA_BINOP_PATTERN(Gather) XLA_BINOP_PATTERN(Ge) @@ -1155,7 +2130,9 @@ XLA_COMMUTATIVE_BINOP_PATTERN(Minimum) XLA_COMMUTATIVE_BINOP_PATTERN(Multiply) XLA_COMMUTATIVE_BINOP_PATTERN(Ne) XLA_BINOP_PATTERN(Outfeed) +XLA_BINOP_PATTERN(Pad) XLA_BINOP_PATTERN(Power) +XLA_BINOP_PATTERN(ReduceWindow) XLA_BINOP_PATTERN(Remainder) XLA_BINOP_PATTERN(Send) XLA_BINOP_PATTERN(Subtract) @@ -1202,6 +2179,7 @@ XLA_BINOP_PATTERN(ShiftRightLogical) .WithOperand(2, std::forward(arg2)); \ } XLA_TERNOP_PATTERN(Clamp); +XLA_TERNOP_PATTERN(Scatter); XLA_TERNOP_PATTERN(Select); #undef XLA_TERNOP_PATTERN @@ -1255,31 +2233,10 @@ inline auto WithOperands(Matcher&& m, int64 operand_num, FirstArg&& first_arg, // We could implement all ops as "variadic" ops, but it would make the // already-bad compile errors even worse. XLA_VARIADIC_OP_PATTERN(Concatenate); +XLA_VARIADIC_OP_PATTERN(CustomCall); +XLA_VARIADIC_OP_PATTERN(Map) XLA_VARIADIC_OP_PATTERN(Reduce); - -namespace detail { -struct PatternFriend { - template - static auto ConstantScalar(T constant) -> decltype( - Constant() - .WithShape(match::Shape().IsScalar()) - .WithPredicate( - std::declval>())) { - std::function pred = - [constant](const HloInstruction* instr) { - const auto& literal = Cast(instr)->literal(); - auto status_or_const = LiteralUtil::CreateR0(constant).Convert( - literal.shape().element_type()); - return status_or_const.ok() && - literal == status_or_const.ConsumeValueOrDie(); - }; - - return Constant() - .WithShape(match::Shape().IsScalar()) - .WithPredicate(std::move(pred)); - } -}; -} // namespace detail +XLA_VARIADIC_OP_PATTERN(Tuple); // Helpers for matching non-constant instructions. inline auto NonConstant() -> decltype(Op().IsNonConstant()) { @@ -1318,14 +2275,71 @@ inline auto GetTupleElement(HloInstructionType** matched_inst, Arg&& arg, .WithTupleIndex(tuple_index); } -template -inline auto ConstantScalar(T constant) - -> decltype(detail::PatternFriend::ConstantScalar(constant)) { - return detail::PatternFriend::ConstantScalar(constant); +// Add overloads for Parameter which take an int64 specifying the parameter +// number. +inline auto Parameter(int64 parameter_num) -> decltype( + Op().WithOpcode(HloOpcode::kParameter).WithParameterNum(parameter_num)) { + return Op().WithOpcode(HloOpcode::kParameter).WithParameterNum(parameter_num); +} +template +inline auto Parameter(HloInstructionType** matched_inst, int64 parameter_num) + -> decltype(Op(matched_inst) + .WithOpcode(HloOpcode::kParameter) + .WithParameterNum(parameter_num)) { + return Op(matched_inst) + .WithOpcode(HloOpcode::kParameter) + .WithParameterNum(parameter_num); +} + +inline auto ConstantScalar() -> decltype(Op().IsConstantScalar()) { + return Op().IsConstantScalar(); +} + +template +inline auto ConstantScalar(HloInstructionType** matched_inst) + -> decltype(Op(matched_inst).IsConstantScalar()) { + return Op(matched_inst).IsConstantScalar(); +} + +template +inline auto ConstantScalar(ScalarTy val) + -> decltype(Op().IsConstantScalar(val)) { + return Op().IsConstantScalar(val); +} + +template +inline auto ConstantScalar(HloInstructionType** matched_inst, ScalarTy val) + -> decltype(Op(matched_inst).IsConstantScalar(val)) { + return Op(matched_inst).IsConstantScalar(val); +} + +inline auto ConstantEffectiveScalar() -> decltype(Op().IsConstantScalar()) { + return Op().IsConstantEffectiveScalar(); +} + +template +inline auto ConstantEffectiveScalar(HloInstructionType** matched_inst) + -> decltype(Op(matched_inst).IsConstantScalar()) { + return Op(matched_inst).IsConstantEffectiveScalar(); +} + +template +inline auto ConstantEffectiveScalar(ScalarTy val) + -> decltype(Op().IsConstantEffectiveScalar(val)) { + return Op().IsConstantEffectiveScalar(val); +} + +template +inline auto ConstantEffectiveScalar(HloInstructionType** matched_inst, + ScalarTy val) + -> decltype(Op(matched_inst).IsConstantEffectiveScalar(val)) { + return Op(matched_inst).IsConstantEffectiveScalar(val); } } // namespace match } // namespace xla +#undef EXPLAIN +#pragma pop_macro("EXPLAIN") #endif // TENSORFLOW_COMPILER_XLA_SERVICE_PATTERN_MATCHER_H_ diff --git a/tensorflow/compiler/xla/service/pattern_matcher_gmock.h b/tensorflow/compiler/xla/service/pattern_matcher_gmock.h new file mode 100644 index 00000000000..8fe2d10a11b --- /dev/null +++ b/tensorflow/compiler/xla/service/pattern_matcher_gmock.h @@ -0,0 +1,92 @@ +/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_COMPILER_XLA_SERVICE_PATTERN_MATCHER_GMOCK_H_ +#define TENSORFLOW_COMPILER_XLA_SERVICE_PATTERN_MATCHER_GMOCK_H_ + +#include +#include "tensorflow/compiler/xla/service/pattern_matcher.h" +#include "tensorflow/compiler/xla/test.h" +#include "tensorflow/core/platform/test.h" + +namespace xla { + +namespace pattern_matcher_gmock_detail { +template +class GmockMatcher { + public: + explicit GmockMatcher(Pattern p) : pattern_(std::move(p)) {} + + // In service of better error messages, list out the overloads explicitly + // rather than just using a template. gMock's polymorphism plus + // pattern_matcher yields some pretty gnarly stuff. + bool MatchAndExplain(const Layout& l, + ::testing::MatchResultListener* listener) const { + return MatchAndExplainImpl(&l, listener); + } + bool MatchAndExplain(const Layout* l, + ::testing::MatchResultListener* listener) const { + return MatchAndExplainImpl(l, listener); + } + + bool MatchAndExplain(const Shape& s, + ::testing::MatchResultListener* listener) const { + return MatchAndExplainImpl(&s, listener); + } + bool MatchAndExplain(const Shape* s, + ::testing::MatchResultListener* listener) const { + return MatchAndExplainImpl(s, listener); + } + + bool MatchAndExplain(const HloInstruction& instr, + ::testing::MatchResultListener* listener) const { + return MatchAndExplainImpl(&instr, listener); + } + bool MatchAndExplain(const HloInstruction* instr, + ::testing::MatchResultListener* listener) const { + return MatchAndExplainImpl(instr, listener); + } + + void DescribeTo(std::ostream* os) const { pattern_.DescribeTo(os); } + + void DescribeNegationTo(std::ostream* os) const { + *os << "is NOT: "; + DescribeTo(os); + } + + private: + template + bool MatchAndExplainImpl(const T* t, + ::testing::MatchResultListener* listener) const { + MatchOption options{/*.capture=*/true, /*.explain_os=*/listener->stream()}; + return Match(t, pattern_, options); + } + + Pattern pattern_; +}; +} // namespace pattern_matcher_gmock_detail + +template +::testing::PolymorphicMatcher< + pattern_matcher_gmock_detail::GmockMatcher> +GmockMatch(Pattern&& p) { + return ::testing::MakePolymorphicMatcher( + pattern_matcher_gmock_detail::GmockMatcher( + std::forward(p))); +} + +} // namespace xla + +#endif // TENSORFLOW_COMPILER_XLA_SERVICE_PATTERN_MATCHER_GMOCK_H_ diff --git a/tensorflow/compiler/xla/service/pattern_matcher_gmock_test.cc b/tensorflow/compiler/xla/service/pattern_matcher_gmock_test.cc new file mode 100644 index 00000000000..9ca2fb05c1f --- /dev/null +++ b/tensorflow/compiler/xla/service/pattern_matcher_gmock_test.cc @@ -0,0 +1,76 @@ +/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/compiler/xla/service/pattern_matcher_gmock.h" +#include "tensorflow/compiler/xla/service/pattern_matcher.h" +#include "tensorflow/compiler/xla/shape_util.h" +#include "tensorflow/compiler/xla/test.h" +#include "tensorflow/core/platform/test.h" + +namespace xla { +namespace { + +namespace m = ::xla::match; +using ::testing::Eq; +using ::testing::Not; + +template +string Describe(const ::testing::Matcher& m) { + std::stringstream ss; + m.DescribeTo(&ss); + return ss.str(); +} + +template +string Explain( + const MatchedTy& val, + const ::testing::Matcher::type>& m) { + ::testing::StringMatchResultListener listener; + EXPECT_THAT(val, ::testing::Not(m)); // For the error message. + EXPECT_FALSE(m.MatchAndExplain(val, &listener)); + return listener.str(); +} + +// This file tests the GmockMatch function. The actual explanation and +// description returned by matchers is tested in pattern_matchers_test. +TEST(PatternMatcherGmock, MatchShape) { + Shape s = ShapeUtil::MakeShape(F32, {10, 100}); + // You can pass const Shape& or a const Shape*. + EXPECT_THAT(s, GmockMatch(m::Shape())); + EXPECT_THAT(&s, Not(GmockMatch(m::Shape().WithElementType(F16)))); + EXPECT_THAT(Describe(GmockMatch(m::Shape().IsArray())), + "a shape that represents an array"); +} + +TEST(PatternMatcherGmock, MatchLayout) { + Layout l = LayoutUtil::MakeLayout({0, 1}); + EXPECT_THAT(l, GmockMatch(m::Layout())); + EXPECT_THAT(&l, Not(GmockMatch(m::Layout().WithSparseFormat()))); + EXPECT_THAT(Describe(GmockMatch(m::Layout().WithSparseFormat())), + "a layout with format SPARSE"); +} + +TEST(PatternMatchGmock, MatchInstruction) { + auto instr = + HloInstruction::CreateParameter(0, ShapeUtil::MakeShape(F32, {42}), "p"); + EXPECT_THAT(instr.get(), GmockMatch(m::Parameter())); + EXPECT_THAT(*instr, GmockMatch(m::Parameter(0))); + EXPECT_THAT(*instr, Not(GmockMatch(m::Parameter(1)))); + EXPECT_THAT(Describe(GmockMatch(m::Parameter())), + "an HloInstruction with opcode parameter"); +} + +} // anonymous namespace +} // namespace xla diff --git a/tensorflow/compiler/xla/service/pattern_matcher_test.cc b/tensorflow/compiler/xla/service/pattern_matcher_test.cc index 3f74273517a..186ef0c7911 100644 --- a/tensorflow/compiler/xla/service/pattern_matcher_test.cc +++ b/tensorflow/compiler/xla/service/pattern_matcher_test.cc @@ -14,14 +14,18 @@ limitations under the License. ==============================================================================*/ #include "tensorflow/compiler/xla/service/pattern_matcher.h" +#include "absl/strings/str_cat.h" #include "tensorflow/compiler/xla/service/hlo_instruction.h" #include "tensorflow/compiler/xla/service/hlo_opcode.h" #include "tensorflow/compiler/xla/service/hlo_parser.h" +#include "tensorflow/compiler/xla/test.h" #include "tensorflow/core/platform/test.h" namespace xla { namespace { +namespace m = match; + TEST(PatternMatcherTest, AddOp) { constexpr char kModuleStr[] = R"(HloModule two_plus_two_module ENTRY %two_plus_two_computation () -> f32[] { @@ -229,23 +233,74 @@ TEST(PatternMatcherTest, AnyOf) { } TEST(PatternMatcherTest, ConstantScalar) { + using match::ConstantEffectiveScalar; + using match::ConstantScalar; + using match::Op; + using match::Tuple; + constexpr char kModuleStr[] = R"( - HloModule test_module ENTRY test { ROOT constant = f16[] constant(42) })"; + HloModule test_module + ENTRY test { + a = s32[] constant(1) + b = s32[1,1] constant(s32[1,1]{{2}}) + c = s32[1,2] constant(s32[1,2]{{2,2}}) + d = f32[] constant(1) + e = f32[] constant(1.25) + ROOT tuple = (s32[], s32[1,1], s32[1,2], f32[], f32[]) tuple(a,b,c,d,e) + })"; TF_ASSERT_OK_AND_ASSIGN(auto hlo_module, ParseHloString(kModuleStr)); auto* root = hlo_module->entry_computation()->root_instruction(); - EXPECT_TRUE(Match(root, match::ConstantScalar(42))); - EXPECT_FALSE(Match(root, match::ConstantScalar(41))); - EXPECT_FALSE(Match(root, match::ConstantScalar(0))); -} + const HloInstruction* a = root->operand(0); + const HloInstruction* b = root->operand(1); + const HloInstruction* c = root->operand(2); + const HloInstruction* d = root->operand(3); + const HloInstruction* e = root->operand(4); + EXPECT_TRUE(Match(a, ConstantScalar())); + EXPECT_TRUE(Match(a, ConstantScalar(1))); + EXPECT_TRUE(Match(a, ConstantEffectiveScalar())); + EXPECT_TRUE(Match(a, ConstantEffectiveScalar(1))); + EXPECT_FALSE(Match(a, ConstantScalar(2))); + EXPECT_FALSE(Match(a, ConstantScalar(2.01))); + EXPECT_FALSE(Match(a, ConstantEffectiveScalar(2))); + EXPECT_FALSE(Match(a, ConstantEffectiveScalar(1.01))); -TEST(PatternMatcherTest, NoMatchConstantScalar) { - constexpr char kModuleStr[] = R"( - HloModule test_module ENTRY test { ROOT v = f16[] parameter(0) })"; - TF_ASSERT_OK_AND_ASSIGN(auto hlo_module, ParseHloString(kModuleStr)); - auto* root = hlo_module->entry_computation()->root_instruction(); + EXPECT_FALSE(Match(b, ConstantScalar())); + EXPECT_FALSE(Match(b, ConstantScalar(2))); + EXPECT_TRUE(Match(b, ConstantEffectiveScalar())); + EXPECT_TRUE(Match(b, ConstantEffectiveScalar(2))); - EXPECT_FALSE(Match(root, match::ConstantScalar(42))); + EXPECT_FALSE(Match(c, ConstantScalar())); + EXPECT_FALSE(Match(c, ConstantScalar(2))); + EXPECT_FALSE(Match(c, ConstantEffectiveScalar())); + EXPECT_FALSE(Match(c, ConstantEffectiveScalar(2))); + + EXPECT_TRUE(Match(d, ConstantScalar(1))); + EXPECT_TRUE(Match(d, ConstantEffectiveScalar(1))); + EXPECT_TRUE(Match(d, ConstantScalar(1.0))); + EXPECT_TRUE(Match(d, ConstantEffectiveScalar(1.0))); + + EXPECT_TRUE(Match(e, ConstantScalar(1.25f))); + EXPECT_TRUE(Match(e, ConstantScalar(1.25))); + EXPECT_TRUE(Match(e, ConstantEffectiveScalar(1.25))); + EXPECT_FALSE(Match(e, ConstantScalar(1))); + EXPECT_FALSE(Match(e, ConstantEffectiveScalar(1))); + + const HloInstruction* instr = nullptr; + EXPECT_TRUE(Match(a, ConstantScalar(&instr))); + EXPECT_EQ(instr, a); + + instr = nullptr; + EXPECT_TRUE(Match(a, ConstantScalar(&instr, 1))); + EXPECT_EQ(instr, a); + + instr = nullptr; + EXPECT_TRUE(Match(a, ConstantEffectiveScalar(&instr))); + EXPECT_EQ(instr, a); + + instr = nullptr; + EXPECT_TRUE(Match(a, ConstantEffectiveScalar(&instr, 1))); + EXPECT_EQ(instr, a); } TEST(PatternMatcherTest, MultiplyAnyOrder) { @@ -267,6 +322,15 @@ TEST(PatternMatcherTest, MultiplyAnyOrder) { root, MultiplyAnyOrder(&instr, ConstantScalar(42), ConstantScalar(52)))); EXPECT_TRUE(Match( root, MultiplyAnyOrder(&instr, ConstantScalar(52), ConstantScalar(42)))); + + // Check that MultiplyAnyOrder exposes the same API as Op(), so we can call + // e.g. IsNonConstant() on it. + EXPECT_TRUE(Match( + root, MultiplyAnyOrder(&instr, ConstantScalar(42), ConstantScalar(52)) + .IsNonConstant())); + EXPECT_TRUE( + Match(root, MultiplyAnyOrder(ConstantScalar(42), ConstantScalar(52)) + .IsNonConstant())); } TEST(PatternMatcherTest, AnyOfShortCircuit) { @@ -315,14 +379,22 @@ TEST(PatternMatcherTest, AllOf) { TF_ASSERT_OK_AND_ASSIGN(auto hlo_module, ParseHloString(kModuleStr)); auto* root = hlo_module->entry_computation()->root_instruction(); + auto f16_scalar = ShapeUtil::MakeShape(F16, {}); + auto f16_pattern = Constant().WithShapeEqualTo(&f16_scalar); + auto f16_compatible_pattern = Constant().WithShapeCompatibleTo(&f16_scalar); auto scalar_pattern = Constant().WithShape(match::Shape().IsScalar()); - auto f16_pattern = Constant().WithShape(match::Shape().WithElementType(F16)); ASSERT_TRUE(Match(root, scalar_pattern)); ASSERT_TRUE(Match(root, f16_pattern)); - EXPECT_TRUE(Match(root, AllOf(scalar_pattern, f16_pattern))); - EXPECT_TRUE(Match(root, AllOf(f16_pattern, scalar_pattern))); + ASSERT_TRUE(Match(root, f16_compatible_pattern)); + EXPECT_TRUE(Match(root, AllOf(scalar_pattern, f16_pattern, + f16_compatible_pattern))); + EXPECT_TRUE( + Match(root, AllOf(f16_pattern, f16_compatible_pattern, + scalar_pattern))); EXPECT_FALSE( Match(root, AllOf(Broadcast(Op()), f16_pattern))); + EXPECT_FALSE(Match( + root, AllOf(Broadcast(Op()), f16_compatible_pattern))); EXPECT_FALSE( Match(root, AllOf(Broadcast(Op()), scalar_pattern))); } @@ -431,5 +503,433 @@ TEST(PatternMatcherTest, TestConcat) { Reshape(ConstantScalar(4))))); } +template +string Description(const Pattern& pattern) { + std::stringstream ss; + pattern.DescribeTo(&ss); + return ss.str(); +} + +template +string Explanation(Elem* elem, const Pattern& pattern) { + std::stringstream ss; + MatchOption options{/*.capture=*/true, /*.explain_os=*/&ss}; + Match(elem, pattern, options); + return ss.str(); +} +template +string Explanation(const std::unique_ptr& elem, const Pattern& pattern) { + return Explanation(elem.get(), pattern); +} +template +string Explanation(const Elem& elem, const Pattern& pattern) { + return Explanation(&elem, pattern); +} + +// Helper macro for checking a pattern's description and the explanation printed +// when attempting to match (and presumably failing) on a given object. +// +// We use a macro rather than a function because we want good line numbers in +// errors. We use this rather than writing a helper that returns a pair of +// (description, explanation) and doing something like +// +// EXPECT_THAT(DescAndExplanation(...), ::testing::Pair(..., ...)); +// +// because EXPECT_EQ prints a unified diff if multiline string comparison fails, +// while EXPECT_THAT does not. This unified diff makes the errors much easier +// to read. +#define EXPECT_DESC_AND_EXPLANATION(elem, pattern, expected_desc, \ + expected_explanation) \ + do { \ + EXPECT_EQ(Description(pattern), (expected_desc)); \ + EXPECT_EQ(Explanation((elem), (pattern)), expected_explanation); \ + } while (0) + +TEST(PatternMatcherTest, LayoutDescribeToAndExplain) { + auto layout = LayoutUtil::MakeLayout({1, 2}); + auto layout2 = LayoutUtil::MakeLayout({2, 2}); + + EXPECT_DESC_AND_EXPLANATION(static_cast(nullptr), m::Layout(), + "a layout", "Layout is null"); + EXPECT_DESC_AND_EXPLANATION(layout2, m::Layout().EqualTo(&layout), + "a layout equal to {1,2}", + "Layout {2,2} is not equal to expected {1,2}"); + EXPECT_DESC_AND_EXPLANATION(layout2, m::Layout().WithSparseFormat(), + "a layout with format SPARSE", + "Layout has format DENSE but expected SPARSE"); + EXPECT_DESC_AND_EXPLANATION(layout, + m::Layout().EqualTo(&layout).WithSparseFormat(), + "a layout:\n" + " * equal to {1,2} AND\n" + " * with format SPARSE", + "Layout has format DENSE but expected SPARSE"); +} + +TEST(PatternMatcherTest, ShapeDescribeToAndExplain) { + auto shape = ShapeUtil::MakeShapeWithLayout(F32, {1, 2}, {0, 1}); + auto layout = shape.layout(); + + EXPECT_DESC_AND_EXPLANATION(static_cast(nullptr), m::Shape(), + "a shape", "Shape is null"); + EXPECT_DESC_AND_EXPLANATION( + ShapeUtil::MakeShapeWithLayout(F32, {1, 2}, {1, 0}), + m::Shape().EqualTo(&shape), "a shape equal to f32[1,2]{0,1}", + "Shape not equal to f32[1,2]{0,1}\n" + "in f32[1,2]{1,0}"); + EXPECT_DESC_AND_EXPLANATION(ShapeUtil::MakeShape(F32, {2, 2}), + m::Shape().CompatibleTo(&shape), + "a shape compatible with f32[1,2]", + "Shape not compatible with f32[1,2]\n" + "in f32[2,2]{1,0}"); + EXPECT_DESC_AND_EXPLANATION(shape, m::Shape().WithElementType(F16), + "a shape with element type F16", + "Shape does not have element type F16\n" + "in f32[1,2]{0,1}"); + EXPECT_DESC_AND_EXPLANATION(shape, m::Shape().IsScalar(), + "a shape that represents a scalar", + "Shape is not a scalar\n" + "in f32[1,2]{0,1}"); + EXPECT_DESC_AND_EXPLANATION(ShapeUtil::MakeNil(), m::Shape().IsArray(), + "a shape that represents an array", + "Shape is not an array\n" + "in ()"); + EXPECT_DESC_AND_EXPLANATION(shape, m::Shape().IsTuple(), + "a shape that represents a tuple", + "Shape is not a tuple\n" + "in f32[1,2]{0,1}"); + EXPECT_DESC_AND_EXPLANATION(shape, m::Shape().IsEffectiveScalar(), + "a shape that is an effective scalar", + "Shape is not an effective scalar\n" + "in f32[1,2]{0,1}"); + EXPECT_DESC_AND_EXPLANATION(shape, m::Shape().WithRank(42), + "a shape that has 42 dimensions", + "Shape does not have rank 42\n" + "in f32[1,2]{0,1}"); + EXPECT_DESC_AND_EXPLANATION(shape, m::Shape().WithRank(0), + "a shape that is a scalar", + "Shape is not a scalar\n" + "in f32[1,2]{0,1}"); + EXPECT_DESC_AND_EXPLANATION(shape, m::Shape().WithRank(1).IsArray(), + "a shape:\n" + " * that has 1 dimension AND\n" + " * that represents an array", + "Shape does not have rank 1\n" + "in f32[1,2]{0,1}"); + EXPECT_DESC_AND_EXPLANATION(ShapeUtil::MakeNil(), + m::Shape().IsArray().WithRank(1), + "a shape:\n" + " * that represents an array AND\n" + " * that has 1 dimension", + "Shape is not an array\n" + "in ()"); + EXPECT_DESC_AND_EXPLANATION( + ShapeUtil::MakeShapeWithLayout(F32, {1, 2}, {1, 0}), + m::Shape().WithLayoutEqualTo(&layout), + "a shape with\n a layout equal to {0,1}", + "Layout {1,0} is not equal to expected {0,1}\n" + "in f32[1,2]{1,0}"); + EXPECT_DESC_AND_EXPLANATION( + shape, m::Shape().WithLayout(m::Layout().WithSparseFormat()), + "a shape with\n a layout with format SPARSE", + "Layout has format DENSE but expected SPARSE\n" + "in f32[1,2]{0,1}"); + EXPECT_DESC_AND_EXPLANATION(shape, + m::Shape().WithSubshapeEqualTo({10}, &shape), + "a shape with subshape at index {10} which is\n" + " a shape equal to f32[1,2]{0,1}", + "No subshape at {10}\n" + "in f32[1,2]{0,1}"); + EXPECT_DESC_AND_EXPLANATION( + ShapeUtil::MakeTupleShape({ShapeUtil::MakeShape(F32, {2, 2})}), + m::Shape().WithSubshapeEqualTo({0}, &shape), + "a shape with subshape at index {0} which is\n" + " a shape equal to f32[1,2]{0,1}", + "Shape not equal to f32[1,2]{0,1}\n" + "in f32[2,2]{1,0}\n" + "in subshape at {0}\n" + "in (f32[2,2])"); + EXPECT_DESC_AND_EXPLANATION(shape, + m::Shape().WithSubshapeCompatibleTo({10}, &shape), + "a shape with subshape at index {10} which is\n" + " a shape compatible with f32[1,2]", + "No subshape at {10}\n" + "in f32[1,2]{0,1}"); + EXPECT_DESC_AND_EXPLANATION( + ShapeUtil::MakeTupleShape({ShapeUtil::MakeShape(F32, {2, 2})}), + m::Shape().WithSubshapeCompatibleTo({0}, &shape), + "a shape with subshape at index {0} which is\n" + " a shape compatible with f32[1,2]", + "Shape not compatible with f32[1,2]\n" + "in f32[2,2]{1,0}\n" + "in subshape at {0}\n" + "in (f32[2,2])"); + EXPECT_DESC_AND_EXPLANATION( + ShapeUtil::MakeTupleShape({ShapeUtil::MakeTupleShape({shape})}), + m::Shape().WithSubshape({0, 0}, m::Shape().IsScalar()), + "a shape with subshape at index {0,0} which is\n" + " a shape that represents a scalar", + "Shape is not a scalar\n" + "in f32[1,2]{0,1}\n" + "in subshape at {0,0}\n" + "in ((f32[1,2]))"); +} + +std::unique_ptr SetName(absl::string_view name, + std::unique_ptr instr) { + instr->SetAndSanitizeName(string(name)); + return instr; +} + +TEST(PatternMatcherTest, HloInstructionDescribeToAndExplain) { + std::unique_ptr iota = + SetName("i", HloInstruction::CreateIota(ShapeUtil::MakeShape(S32, {42}), + /*iota_dimension=*/0)); + std::unique_ptr constant = + SetName("c", HloInstruction::CreateConstant(LiteralUtil::CreateR0(0))); + + EXPECT_DESC_AND_EXPLANATION(static_cast(nullptr), + m::Op(), "an HloInstruction", + "HloInstruction* is null"); + EXPECT_DESC_AND_EXPLANATION(iota, m::Op().WithName("foo"), + "an HloInstruction named \"foo\"", + "HloInstruction not named \"foo\"\n" + "in i = s32[42]{0} iota(), iota_dimension=0"); + EXPECT_DESC_AND_EXPLANATION(iota, m::Op().WithOpcode(HloOpcode::kAdd), + "an HloInstruction with opcode add", + "HloInstruction doesn't have opcode add\n" + "in i = s32[42]{0} iota(), iota_dimension=0"); + EXPECT_DESC_AND_EXPLANATION( + constant, m::Op().IsNonConstant(), + "an HloInstruction with any opcode other than constant", + "HloInstruction has opcode constant, expected anything else\n" + "in c = s32[] constant(0)"); + EXPECT_DESC_AND_EXPLANATION(iota, m::Op().WithNumOperands(42), + "an HloInstruction with 42 operands", + "HloInstruction doesn't have 42 operands\n" + "in i = s32[42]{0} iota(), iota_dimension=0"); + EXPECT_DESC_AND_EXPLANATION(iota, m::Op().WithShape(m::Shape().IsTuple()), + "an HloInstruction outputting\n" + " a shape that represents a tuple", + "Shape is not a tuple\n" + "in s32[42]{0}\n" + "in output shape\n" + "in i = s32[42]{0} iota(), iota_dimension=0"); + EXPECT_DESC_AND_EXPLANATION( + iota, m::Op().WithOperand(2, m::Op().WithOpcode(HloOpcode::kAdd)), + "an HloInstruction with operand 2 which is:\n" + " an HloInstruction with opcode add", + "desired operand index 2 is out of bounds\n" + "in i = s32[42]{0} iota(), iota_dimension=0"); + + EXPECT_DESC_AND_EXPLANATION( + SetName("a", HloInstruction::CreateBinary(ShapeUtil::MakeShape(S32, {}), + HloOpcode::kAdd, constant.get(), + constant.get())), + m::Op().WithOperand(1, m::Op().IsNonConstant()), + "an HloInstruction with operand 1 which is:\n" + " an HloInstruction with any opcode other than constant", + "HloInstruction has opcode constant, expected anything else\n" + "in c = s32[] constant(0)\n" + "in operand 1\n" + "in a = s32[] add(s32[] c, s32[] c)"); + EXPECT_DESC_AND_EXPLANATION( + iota, m::Op().WithFusionKind(HloInstruction::FusionKind::kLoop), + "an HloInstruction with fusion kind kLoop", + "HloInstruction does not have fusion kind kLoop; it's not a fusion\n" + "in i = s32[42]{0} iota(), iota_dimension=0"); + EXPECT_DESC_AND_EXPLANATION( + iota, m::Op().WithTupleIndex(42), + "an HloInstruction which is a GTE with index 42", + "HloInstruction is not a GTE with index 42; it's not a GTE at all\n" + "in i = s32[42]{0} iota(), iota_dimension=0"); + EXPECT_DESC_AND_EXPLANATION(iota, m::Op().IsConstantScalar(), + "an HloInstruction which is a constant scalar", + "HloInstruction is not a constant\n" + "in i = s32[42]{0} iota(), iota_dimension=0"); + EXPECT_DESC_AND_EXPLANATION( + SetName("c", HloInstruction::CreateConstant( + LiteralUtil::CreateR1({1, 2}))), + m::Op().IsConstantEffectiveScalar(), + "an HloInstruction which is a constant effective scalar", + "HloInstruction is not an effective scalar\n" + "in c = s32[2]{0} constant({1, 2})"); + EXPECT_DESC_AND_EXPLANATION( + SetName("c", HloInstruction::CreateConstant(LiteralUtil::CreateR0(10))), + m::Op().IsConstantScalar(42), + "an HloInstruction which is a constant scalar with value 42", + "HloInstruction's constant value 10 did not match expected value 42\n" + "in c = s32[] constant(10)"); + EXPECT_DESC_AND_EXPLANATION( + SetName("c", HloInstruction::CreateConstant(LiteralUtil::CreateR0(2.25))), + m::Op().IsConstantEffectiveScalar(1.25), + "an HloInstruction which is a constant effective scalar with value 1.25", + "HloInstruction's constant value 2.25 did not match expected value 1.25\n" + "in c = f64[] constant(2.25)"); + EXPECT_DESC_AND_EXPLANATION( + constant, m::Op().Is(iota.get()), + absl::StrCat("an HloInstruction which is 0x", absl::Hex(iota.get()), + " (i = s32[42]{0} iota(), iota_dimension=0)"), + absl::StrCat("HloInstruction 0x", absl::Hex(constant.get()), " is not 0x", + absl::Hex(iota.get()), + " (i = s32[42]{0} iota(), iota_dimension=0)\n" + "in c = s32[] constant(0)")); +} + +TEST(PatternMatcherTest, HloInstructionMatcherAnyOrderDescribeTo) { + auto scalar_s32 = ShapeUtil::MakeShape(S32, {}); + EXPECT_DESC_AND_EXPLANATION( + SetName("a", HloInstruction::CreateBinary( + scalar_s32, HloOpcode::kAdd, + SetName("b", HloInstruction::CreateConstant( + LiteralUtil::CreateR0(0))) + .get(), + SetName("c", HloInstruction::CreateConstant( + LiteralUtil::CreateR0(0))) + .get())), + m::AddAnyOrder(m::Op().WithName("b"), m::Op().WithName("bar")), + "an HloInstruction:\n" + " * with opcode add AND\n" + " * with two operands in either order:\n" + " - an HloInstruction named \"b\"\n" + " - an HloInstruction named \"bar\"", + "HloInstruction's operands (ignoring order) did not match second " + "matcher. Specifically,\n" + " - an HloInstruction named \"bar\"\n" + "does not match LHS:\n" + " - HloInstruction not named \"bar\"\n" + " in b = s32[] constant(0)\n" + "does not match RHS:\n" + " - HloInstruction not named \"bar\"\n" + " in c = s32[] constant(0)\n" + "in a = s32[] add(s32[] b, s32[] c)"); + + EXPECT_DESC_AND_EXPLANATION( + SetName("a", + HloInstruction::CreateBinary( + scalar_s32, HloOpcode::kAdd, + HloInstruction::CreateParameter(0, scalar_s32, "p").get(), + SetName("c", HloInstruction::CreateConstant( + LiteralUtil::CreateR0(0))) + .get())), + m::AddAnyOrder(m::Op().IsConstantScalar(), m::Op().IsConstant()), + "an HloInstruction:\n" + " * with opcode add AND\n" + " * with two operands in either order:\n" + " - an HloInstruction which is a constant scalar\n" + " - an HloInstruction with opcode constant", + "HloInstruction's LHS operand did not match either of the two matchers. " + "Specifically,\n" + " - an HloInstruction which is a constant scalar\n" + "does not match LHS:\n" + " - HloInstruction is not a constant\n" + " in p = s32[] parameter(0)\n" + "and\n" + " - an HloInstruction with opcode constant\n" + "does not match LHS:\n" + " - HloInstruction doesn't have opcode constant\n" + " in p = s32[] parameter(0)\n" + "in a = s32[] add(s32[] p, s32[] c)"); +} + +TEST(PatternMatcherTest, AnyOfMatcherDescribeToAndExplain) { + EXPECT_DESC_AND_EXPLANATION( + SetName("c", HloInstruction::CreateConstant(LiteralUtil::CreateR0(0))), + m::AnyOf(m::Op().WithName("foo"), + m::Op().WithName("bar")), + "any of:\n" + " - an HloInstruction named \"foo\" OR\n" + " - an HloInstruction named \"bar\"", + "None of the following matchers succeeded:\n" + "Matcher #1\n" + " - an HloInstruction named \"foo\"\n" + "failed with\n" + " - HloInstruction not named \"foo\"\n" + " in c = s32[] constant(0)\n" + "Matcher #2\n" + " - an HloInstruction named \"bar\"\n" + "failed with\n" + " - HloInstruction not named \"bar\"\n" + " in c = s32[] constant(0)"); +} + +TEST(PatternMatcherTest, Parameter) { + auto param = + HloInstruction::CreateParameter(1, ShapeUtil::MakeShape(F32, {}), "p1"); + auto non_param = + SetName("c", HloInstruction::CreateConstant(LiteralUtil::CreateR0(0))); + EXPECT_FALSE(Match(param.get(), m::Parameter(0))); + EXPECT_TRUE(Match(param.get(), m::Parameter())); + EXPECT_TRUE(Match(param.get(), m::Parameter(1))); + EXPECT_FALSE(Match(non_param.get(), m::Parameter())); + EXPECT_FALSE(Match(non_param.get(), m::Parameter(1))); + + EXPECT_DESC_AND_EXPLANATION(non_param, m::Parameter(1), + "an HloInstruction:\n" + " * with opcode parameter AND\n" + " * which is parameter 1", + "HloInstruction doesn't have opcode parameter\n" + "in c = s32[] constant(0)"); + EXPECT_EQ(Explanation(HloInstruction::CreateParameter( + 0, ShapeUtil::MakeShape(F32, {}), "p0"), + m::Parameter(1)), + "HloInstruction is not parameter 1\n" + "in p0 = f32[] parameter(0)"); +} + +TEST(PatternMatcherTest, OneUseAndOneUser) { + auto param = + HloInstruction::CreateParameter(0, ShapeUtil::MakeShape(F32, {}), "p0"); + + EXPECT_FALSE(Match(param.get(), m::Op().WithOneUse())); + EXPECT_DESC_AND_EXPLANATION( + param, m::Op().WithOneUse(), + "an HloInstruction which has exactly one use", + "HloInstruction has 0 users, but expected exactly one.\n" + "in p0 = f32[] parameter(0)"); + + EXPECT_FALSE(Match(param.get(), m::Op().WithOneUser())); + EXPECT_DESC_AND_EXPLANATION( + param, m::Op().WithOneUser(), + "an HloInstruction which has exactly one user (but possibly is used " + "multiple times by that instruction)", + "HloInstruction has 0 users, but expected exactly one.\n" + "in p0 = f32[] parameter(0)"); + + { + auto reshape = + SetName("r", HloInstruction::CreateReshape( + ShapeUtil::MakeShape(F32, {1}), param.get())); + EXPECT_TRUE(Match(param.get(), m::Op().WithOneUse())); + EXPECT_TRUE(Match(param.get(), m::Op().WithOneUser())); + + auto reshape1 = + SetName("r1", HloInstruction::CreateReshape( + ShapeUtil::MakeShape(F32, {1}), param.get())); + EXPECT_FALSE(Match(param.get(), m::Op().WithOneUse())); + EXPECT_FALSE(Match(param.get(), m::Op().WithOneUser())); + + const char* kMultipleUserExplanation = + "HloInstruction has 2 users, but expected exactly one.\n" + "All users:\n" + " - r = f32[1]{0} reshape(f32[] p0)\n" + " - r1 = f32[1]{0} reshape(f32[] p0)\n" + "in p0 = f32[] parameter(0)"; + EXPECT_EQ(Explanation(param.get(), m::Op().WithOneUse()), + kMultipleUserExplanation); + EXPECT_EQ(Explanation(param.get(), m::Op().WithOneUser()), + kMultipleUserExplanation); + } + + auto add = SetName("add", HloInstruction::CreateBinary( + ShapeUtil::MakeShape(F32, {}), HloOpcode::kAdd, + param.get(), param.get())); + EXPECT_TRUE(Match(param.get(), m::Op().WithOneUser())); + EXPECT_FALSE(Match(param.get(), m::Op().WithOneUse())); + EXPECT_EQ(Explanation(param.get(), m::Op().WithOneUse()), + "HloInstruction is used 2 times by its user, but is expected to be " + "used just once: add = f32[] add(f32[] p0, f32[] p0)\n" + "in p0 = f32[] parameter(0)"); +} + } // namespace } // namespace xla diff --git a/tensorflow/compiler/xla/service/reduce_precision_insertion_test.cc b/tensorflow/compiler/xla/service/reduce_precision_insertion_test.cc index 16fa80d53e7..efeec965714 100644 --- a/tensorflow/compiler/xla/service/reduce_precision_insertion_test.cc +++ b/tensorflow/compiler/xla/service/reduce_precision_insertion_test.cc @@ -54,7 +54,7 @@ TEST_F(ReducePrecisionInsertionTest, BeforeUnaryInstruction) { HloInstruction* b = builder.AddInstruction( HloInstruction::CreateUnary(shape, HloOpcode::kCos, a)); - auto module = CreateNewUnverifiedModule(); + auto module = CreateNewVerifiedModule(); auto computation = module->AddEntryComputation(builder.Build()); // Confirm expected state before adding ops. @@ -81,7 +81,7 @@ TEST_F(ReducePrecisionInsertionTest, BeforeUnaryScalarInstruction) { HloInstruction* b = builder.AddInstruction( HloInstruction::CreateUnary(shape, HloOpcode::kCos, a)); - auto module = CreateNewUnverifiedModule(); + auto module = CreateNewVerifiedModule(); auto computation = module->AddEntryComputation(builder.Build()); // Confirm expected state before adding ops. @@ -111,7 +111,7 @@ TEST_F(ReducePrecisionInsertionTest, BeforeBinaryInstruction) { HloInstruction* c = builder.AddInstruction( HloInstruction::CreateBinary(shape, HloOpcode::kAdd, a, b)); - auto module = CreateNewUnverifiedModule(); + auto module = CreateNewVerifiedModule(); auto computation = module->AddEntryComputation(builder.Build()); // Confirm expected state before adding ops. @@ -140,7 +140,7 @@ TEST_F(ReducePrecisionInsertionTest, BeforeZeroInputInstruction) { HloInstruction* b = builder.AddInstruction( HloInstruction::CreateUnary(shape, HloOpcode::kCos, a)); - auto module = CreateNewUnverifiedModule(); + auto module = CreateNewVerifiedModule(); auto computation = module->AddEntryComputation(builder.Build()); // Confirm expected state before adding ops. @@ -173,7 +173,7 @@ TEST_F(ReducePrecisionInsertionTest, AvoidAddingDuplicateInstructions) { HloInstruction* d = builder.AddInstruction( HloInstruction::CreateBinary(shape, HloOpcode::kAdd, b, c)); - auto module = CreateNewUnverifiedModule(); + auto module = CreateNewVerifiedModule(); auto computation = module->AddEntryComputation(builder.Build()); // Confirm expected state before adding ops. @@ -205,7 +205,7 @@ TEST_F(ReducePrecisionInsertionTest, AfterRootInstruction) { HloInstruction* b = builder.AddInstruction( HloInstruction::CreateUnary(shape, HloOpcode::kCos, a)); - auto module = CreateNewUnverifiedModule(); + auto module = CreateNewVerifiedModule(); auto computation = module->AddEntryComputation(builder.Build()); // Confirm expected state before adding ops. @@ -242,7 +242,7 @@ TEST_F(ReducePrecisionInsertionTest, AfterNonRootInstruction) { HloInstruction* c = builder.AddInstruction( HloInstruction::CreateBinary(shape, HloOpcode::kAdd, a_cos, b_cos)); - auto module = CreateNewUnverifiedModule(); + auto module = CreateNewVerifiedModule(); module->AddEntryComputation(builder.Build()); // Confirm expected graph before adding ops. @@ -295,7 +295,7 @@ TEST_F(ReducePrecisionInsertionTest, ShouldReduceOutputPrecisionIsFalse) { HloInstruction* y = builder.AddInstruction( HloInstruction::CreateUnary(shape, HloOpcode::kCos, x)); - auto module = CreateNewUnverifiedModule(); + auto module = CreateNewVerifiedModule(); auto computation = module->AddEntryComputation(builder.Build()); // Confirm expected graph before adding ops. @@ -321,7 +321,7 @@ TEST_F(ReducePrecisionInsertionTest, InsertionIsNotRecursive) { HloInstruction* b = builder.AddInstruction( HloInstruction::CreateReducePrecision(shape, a, 8, 23)); - auto module = CreateNewUnverifiedModule(); + auto module = CreateNewVerifiedModule(); auto computation = module->AddEntryComputation(builder.Build()); // Confirm expected state before adding ops. @@ -348,7 +348,7 @@ TEST_F(ReducePrecisionInsertionTest, SkipRedundantReducePrecisionAfter) { HloInstruction* y = builder.AddInstruction( HloInstruction::CreateReducePrecision(shape, x, 5, 10)); - auto module = CreateNewUnverifiedModule(); + auto module = CreateNewVerifiedModule(); auto computation = module->AddEntryComputation(builder.Build()); // Confirm expected graph before adding ops. @@ -376,7 +376,7 @@ TEST_F(ReducePrecisionInsertionTest, AddNonRedundantReducePrecision) { HloInstruction* y = builder.AddInstruction( HloInstruction::CreateReducePrecision(shape, x, 8, 23)); - auto module = CreateNewUnverifiedModule(); + auto module = CreateNewVerifiedModule(); auto computation = module->AddEntryComputation(builder.Build()); // Confirm expected graph before adding ops. @@ -402,7 +402,7 @@ TEST_F(ReducePrecisionInsertionTest, IgnoreOpsInsideFusionNode) { builder.AddInstruction(HloInstruction::CreateParameter(0, shape, "x")); HloInstruction* y = builder.AddInstruction( HloInstruction::CreateUnary(shape, HloOpcode::kCos, x)); - auto module = CreateNewUnverifiedModule(); + auto module = CreateNewVerifiedModule(); auto computation = module->AddEntryComputation(builder.Build()); // Manually fuse the kCos operation into a fusion operation. @@ -438,7 +438,7 @@ TEST_F(ReducePrecisionInsertionTest, OpGetsInsertedInHeadOfFusionNode) { builder.AddInstruction(HloInstruction::CreateParameter(0, shape, "x")); HloInstruction* y = builder.AddInstruction( HloInstruction::CreateUnary(shape, HloOpcode::kCos, x)); - auto module = CreateNewUnverifiedModule(); + auto module = CreateNewVerifiedModule(); auto computation = module->AddEntryComputation(builder.Build()); // Manually fuse the kCos operation into a fusion operation. @@ -485,7 +485,7 @@ TEST_F(ReducePrecisionInsertionTest, OpGetsInsertedInTailOfFusionNode) { builder.AddInstruction(HloInstruction::CreateParameter(0, shape, "x")); HloInstruction* y = builder.AddInstruction( HloInstruction::CreateUnary(shape, HloOpcode::kCos, x)); - auto module = CreateNewUnverifiedModule(); + auto module = CreateNewVerifiedModule(); auto computation = module->AddEntryComputation(builder.Build()); // Manually fuse the kCos operation into a fusion operation. diff --git a/tensorflow/compiler/xla/service/service.cc b/tensorflow/compiler/xla/service/service.cc index 75f7413b3c3..5ec7fe2aded 100644 --- a/tensorflow/compiler/xla/service/service.cc +++ b/tensorflow/compiler/xla/service/service.cc @@ -41,6 +41,7 @@ limitations under the License. #include "tensorflow/compiler/xla/service/source_map_util.h" #include "tensorflow/compiler/xla/service/stream_pool.h" #include "tensorflow/compiler/xla/service/transfer_manager.h" +#include "tensorflow/compiler/xla/shape.h" #include "tensorflow/compiler/xla/shape_layout.h" #include "tensorflow/compiler/xla/shape_util.h" #include "tensorflow/compiler/xla/status_macros.h" @@ -275,8 +276,8 @@ StatusOr> Service::CreateModuleConfig( } if (execution_options != nullptr && execution_options->has_shape_with_output_layout()) { - const auto& shape_with_output_layout = - execution_options->shape_with_output_layout(); + const Shape shape_with_output_layout( + execution_options->shape_with_output_layout()); TF_RETURN_IF_ERROR( ValidateResultShape(shape_with_output_layout, program_shape.result())); TF_RETURN_IF_ERROR( @@ -658,9 +659,9 @@ Status Service::ExecuteGraphParallel(const ExecuteGraphParallelRequest* arg, // replica 0. TF_ASSIGN_OR_RETURN( std::unique_ptr module_config, - CreateModuleConfig(request.computation().host_program_shape(), - replicated_arguments.front(), - request.execution_options())); + CreateModuleConfig( + ProgramShape{request.computation().host_program_shape()}, + replicated_arguments.front(), request.execution_options())); VLOG(3) << "ExecuteGraphParallel created HloModuleConfig computation layout: " << module_config->entry_computation_layout().ToString(); @@ -745,9 +746,9 @@ Status Service::GetDeviceHandles(const GetDeviceHandlesRequest* arg, } if (available_device_count < arg->device_count() * replica_count) { return ResourceExhausted( - "Requested device count (%d) exceeds the number of available devices " - "on the target (%d)", - arg->device_count(), available_device_count); + "Requested logical device count (%d) with replica count (%d) exceeds " + "the number of available physical devices on the target (%d)", + arg->device_count(), replica_count, available_device_count); } for (int64 i = 0; i < arg->device_count(); ++i) { @@ -818,14 +819,17 @@ Status Service::Compile(const CompileRequest* arg, CompileResponse* result) { "The compile request does not support multiple device handles."); } - std::vector argument_shapes; - absl::c_transform(arg->input_shape_with_layout(), - std::back_inserter(argument_shapes), - [](const Shape& shape) { return &shape; }); + std::vector argument_shapes; + argument_shapes.reserve(arg->input_shape_with_layout_size()); + std::vector argument_shape_ptrs; + for (const ShapeProto& shape_proto : arg->input_shape_with_layout()) { + argument_shapes.push_back(Shape(shape_proto)); + argument_shape_ptrs.push_back(&argument_shapes.back()); + } TF_ASSIGN_OR_RETURN( std::unique_ptr module_config, - CreateModuleConfig(arg->computation().host_program_shape(), - argument_shapes, &arg->execution_options())); + CreateModuleConfig(ProgramShape{arg->computation().host_program_shape()}, + argument_shape_ptrs, &arg->execution_options())); VLOG(3) << "Compile created HloModuleConfig computation layout: " << module_config->entry_computation_layout().ToString(); @@ -930,14 +934,14 @@ Status Service::TransferToClient(const TransferToClientRequest* arg, TF_ASSIGN_OR_RETURN(const ShapedBuffer* shaped_buffer, allocation_tracker_.ResolveForReplica(arg->data(), 0)); - const Shape* return_shape; + Shape return_shape; if (arg->has_shape_with_layout()) { - if (!LayoutUtil::HasLayout(arg->shape_with_layout())) { + return_shape = Shape(arg->shape_with_layout()); + if (!LayoutUtil::HasLayout(return_shape)) { return InvalidArgument("shape_with_layout must have layout if present."); } - return_shape = &arg->shape_with_layout(); } else { - return_shape = &shaped_buffer->on_host_shape(); + return_shape = Shape(shaped_buffer->on_host_shape()); } TF_ASSIGN_OR_RETURN(auto stream, execute_backend_->BorrowStream( @@ -948,30 +952,15 @@ Status Service::TransferToClient(const TransferToClientRequest* arg, execute_backend_->transfer_manager()->TransferLiteralFromDevice( stream.get(), *shaped_buffer)); - if (LayoutUtil::LayoutsInShapesEqual(*return_shape, result_literal.shape())) { + if (LayoutUtil::LayoutsInShapesEqual(return_shape, result_literal.shape())) { *result->mutable_literal() = result_literal.ToProto(); } else { *result->mutable_literal() = - result_literal.Relayout(*return_shape).ToProto(); + result_literal.Relayout(return_shape).ToProto(); } return Status::OK(); } -namespace { - -// Creates a clone of the given shaped buffer with the given device ordinal. The -// shape and DeviceMemoryBase values of the clone are identical to the original. -std::unique_ptr CloneShapedBufferOnDevice( - const ShapedBuffer& shaped_buffer, int device_ordinal) { - auto clone = absl::make_unique( - shaped_buffer.on_host_shape(), shaped_buffer.on_device_shape(), - shaped_buffer.platform(), device_ordinal); - clone->buffers() = shaped_buffer.buffers(); - return clone; -} - -} // namespace - Status Service::TransferToServer(const TransferToServerRequest* arg, TransferToServerResponse* result) { TF_ASSIGN_OR_RETURN(Literal literal, @@ -1060,11 +1049,11 @@ Status Service::TransferFromOutfeed(const TransferFromOutfeedRequest* arg, executor = replicas[arg->replica_id()]; } - auto literal = Literal::CreateFromShape(arg->shape_with_layout()); + auto literal = Literal::CreateFromShape(Shape(arg->shape_with_layout())); TF_RETURN_IF_ERROR( execute_backend_->transfer_manager()->TransferLiteralFromOutfeed( - executor, arg->shape_with_layout(), literal)); + executor, Shape(arg->shape_with_layout()), literal)); *result->mutable_literal() = literal.ToProto(); return Status::OK(); } @@ -1087,7 +1076,7 @@ Status Service::ComputeConstantGraph(const ComputeConstantGraphRequest* arg, "constant computation may not depend on any parameters."); } - ProgramShape program_shape = arg->computation().host_program_shape(); + ProgramShape program_shape(arg->computation().host_program_shape()); TF_DCHECK_OK(ShapeUtil::ValidateShape(program_shape.result())); if (arg->has_output_layout()) { TF_RETURN_IF_ERROR(LayoutUtil::ValidateLayoutForShape( @@ -1118,7 +1107,7 @@ Status Service::ComputeConstantGraph(const ComputeConstantGraphRequest* arg, Status Service::GetShape(const GetShapeRequest* arg, GetShapeResponse* result) { TF_ASSIGN_OR_RETURN(const ShapedBuffer* buffer, allocation_tracker_.ResolveForReplica(arg->data(), 0)); - *result->mutable_shape() = buffer->on_host_shape(); + *result->mutable_shape() = buffer->on_host_shape().ToProto(); return Status::OK(); } @@ -1131,7 +1120,7 @@ Status Service::GetComputationGraphStats( return InvalidArgument("Program shape may not be empty."); } - HloModuleConfig config(arg->computation().host_program_shape()); + HloModuleConfig config(ProgramShape{arg->computation().host_program_shape()}); config.set_debug_options(arg->debug_options()); TF_ASSIGN_OR_RETURN(std::unique_ptr module, CreateModuleFromProto(arg->computation(), config)); diff --git a/tensorflow/compiler/xla/service/shape_inference.cc b/tensorflow/compiler/xla/service/shape_inference.cc index 61a60ef9efa..7e7282a7370 100644 --- a/tensorflow/compiler/xla/service/shape_inference.cc +++ b/tensorflow/compiler/xla/service/shape_inference.cc @@ -391,17 +391,6 @@ StatusOr InferWindowOutputShape(const Shape& base_shape, return ShapeUtil::MakeShape(element_type, new_dimensions); } -/* static */ StatusOr ShapeInference::InferAfterAllShape( - absl::Span arg_shapes) { - for (const Shape* arg_shape : arg_shapes) { - if (arg_shape->element_type() != TOKEN) { - return InvalidArgument( - "Operands of token instructions must be TOKEN types."); - } - } - return ShapeUtil::MakeTokenShape(); -} - /* static */ StatusOr ShapeInference::InferConvertShape( const Shape& operand_shape, PrimitiveType new_element_type) { auto old_element_type = operand_shape.element_type(); @@ -1029,7 +1018,7 @@ ShapeInference::InferDegenerateDimensionBroadcastShape(HloOpcode operation, switch (opcode) { case HloOpcode::kTuple: { Shape result = ShapeUtil::MakeTupleShape({}); - result.mutable_tuple_shapes()->Reserve(operand_shapes.size()); + result.mutable_tuple_shapes()->reserve(operand_shapes.size()); for (const Shape* shape : operand_shapes) { ShapeUtil::AppendShapeToTuple(*shape, &result); } @@ -2038,7 +2027,16 @@ ShapeInference::InferDegenerateDimensionBroadcastShape(HloOpcode operation, dimension); } - return ShapeUtil::MakeShape(S64, {}); + // TODO(b/119580730): Remove this restriction when very large dimension size + // is needed. + if (shape.dimensions(dimension) > std::numeric_limits::max()) { + return InvalidArgument( + "GetDimensionSize's input shape is %s, the %dth dimension exceeds the " + "UINT_MAX limit.", + ShapeUtil::HumanString(shape), dimension); + } + + return ShapeUtil::MakeShape(U32, {}); } /* static */ StatusOr ShapeInference::InferSliceShape( diff --git a/tensorflow/compiler/xla/service/shape_inference.h b/tensorflow/compiler/xla/service/shape_inference.h index 31ef4b2e410..d94385a04d5 100644 --- a/tensorflow/compiler/xla/service/shape_inference.h +++ b/tensorflow/compiler/xla/service/shape_inference.h @@ -232,13 +232,6 @@ class ShapeInference { static StatusOr InferConcatOpShape( absl::Span arg_shapes, int64 dimension); - // Infers the shape produced by a kAfterAll. Trivially this shape is always a - // TOKEN shape. However, ShapeInference serves two purposes: inferring shapes - // and checking operand shapes. This method verifies that the operand shapes - // are all TOKENs. - static StatusOr InferAfterAllShape( - absl::Span arg_shapes); - // Helper that validates the given operand shape can be converted to the // target output_shape via a convert instruction -- the requirement is that // the shape is identical except for the element type. diff --git a/tensorflow/compiler/xla/service/transpose_folding_test.cc b/tensorflow/compiler/xla/service/transpose_folding_test.cc index 7a565bf0768..17cdaa74fc3 100644 --- a/tensorflow/compiler/xla/service/transpose_folding_test.cc +++ b/tensorflow/compiler/xla/service/transpose_folding_test.cc @@ -172,7 +172,7 @@ TEST_F(TransposeFoldingTest, FuseDotWithConstantOperands) { HloInstruction* mul = builder.AddInstruction(HloInstruction::CreateBinary( add->shape(), HloOpcode::kMultiply, add, sub)); - auto module = CreateNewUnverifiedModule("fuse_with_constant_operands"); + auto module = CreateNewVerifiedModule("fuse_with_constant_operands"); HloComputation* entry_computation = module->AddEntryComputation(builder.Build(mul)); HloInstruction* call = module->OutlineExpressionFromComputation( @@ -247,7 +247,7 @@ TEST_F(TransposeFoldingTest, FoldConvDimSwapTransposeRhs) { conv_shape.ValueOrDie(), x, transpose_y, /*feature_group_count=*/1, window, dnums, DefaultPrecisionConfig(2))); - auto module = CreateNewUnverifiedModule("test_module"); + auto module = CreateNewVerifiedModule("test_module"); HloComputation* entry_computation = module->AddEntryComputation(builder.Build(conv)); FoldTranspose(module.get()); @@ -302,7 +302,7 @@ TEST_F(TransposeFoldingTest, FoldConvComplexTransposeRhs) { conv_shape.ValueOrDie(), x, transpose_y, /*feature_group_count=*/1, window, dnums, DefaultPrecisionConfig(2))); - auto module = CreateNewUnverifiedModule("test_module"); + auto module = CreateNewVerifiedModule("test_module"); HloComputation* entry_computation = module->AddEntryComputation(builder.Build(conv)); FoldTranspose(module.get()); @@ -362,7 +362,7 @@ TEST_F(TransposeFoldingTest, FoldConvTransposeLhs) { conv_shape.ValueOrDie(), transpose_x, y, /*feature_group_count=*/1, window, dnums, DefaultPrecisionConfig(2))); - auto module = CreateNewUnverifiedModule("test_module"); + auto module = CreateNewVerifiedModule("test_module"); HloComputation* entry_computation = module->AddEntryComputation(builder.Build(conv)); FoldTranspose(module.get()); @@ -428,7 +428,7 @@ TEST_F(TransposeFoldingTest, FoldConvComplexTransposeLhs) { conv_shape.ValueOrDie(), transpose_x, y, /*feature_group_count=*/1, window, dnums, DefaultPrecisionConfig(2))); - auto module = CreateNewUnverifiedModule("test_module"); + auto module = CreateNewVerifiedModule("test_module"); HloComputation* entry_computation = module->AddEntryComputation(builder.Build(conv)); FoldTranspose(module.get()); diff --git a/tensorflow/compiler/xla/service/tuple_points_to_analysis.cc b/tensorflow/compiler/xla/service/tuple_points_to_analysis.cc index 96f3055c98e..50d51eaeb76 100644 --- a/tensorflow/compiler/xla/service/tuple_points_to_analysis.cc +++ b/tensorflow/compiler/xla/service/tuple_points_to_analysis.cc @@ -280,6 +280,13 @@ Status TuplePointsToAnalysis::HandleDomain(HloInstruction* domain) { return Status::OK(); } +Status TuplePointsToAnalysis::HandleAddDependency( + HloInstruction* add_dependency) { + // AddDependency just forwards the value of its zero-th operand. + CreateCopiedPointsToSet(add_dependency, add_dependency->operand(0)); + return Status::OK(); +} + Status TuplePointsToAnalysis::HandleRecvDone(HloInstruction* recv_done) { // RecvDone aliases its input (Recv) tuple element {0} to element {0} of its // output. The other indices ({} and {1}) define their own buffers. diff --git a/tensorflow/compiler/xla/service/tuple_points_to_analysis.h b/tensorflow/compiler/xla/service/tuple_points_to_analysis.h index bcfcb388f95..0a1d5649d6d 100644 --- a/tensorflow/compiler/xla/service/tuple_points_to_analysis.h +++ b/tensorflow/compiler/xla/service/tuple_points_to_analysis.h @@ -252,6 +252,7 @@ class TuplePointsToAnalysis : public DfsHloVisitorWithDefault { Status HandleRecvDone(HloInstruction* recv_done) override; Status HandleSend(HloInstruction* send) override; Status HandleTupleSelect(HloInstruction* tuple_select) override; + Status HandleAddDependency(HloInstruction* add_dependency) override; string ToString() const; diff --git a/tensorflow/compiler/xla/service/tuple_points_to_analysis_test.cc b/tensorflow/compiler/xla/service/tuple_points_to_analysis_test.cc index 10ef2d38fa2..561762b5d42 100644 --- a/tensorflow/compiler/xla/service/tuple_points_to_analysis_test.cc +++ b/tensorflow/compiler/xla/service/tuple_points_to_analysis_test.cc @@ -264,6 +264,22 @@ TEST_F(TuplePointsToAnalysisTest, GetTupleElement) { UnorderedElementsAre(inner_tuple)); } +TEST_F(TuplePointsToAnalysisTest, AddDependency) { + auto builder = HloComputation::Builder(TestName()); + auto constant = builder.AddInstruction( + HloInstruction::CreateConstant(LiteralUtil::CreateR0(1.0))); + auto token = builder.AddInstruction(HloInstruction::CreateToken()); + auto add_dependency = builder.AddInstruction( + HloInstruction::CreateAddDependency(constant, token)); + BuildModuleAndRunAnalysis(builder.Build()); + + auto& points_to_set = points_to_analysis_->GetPointsToSet(add_dependency); + EXPECT_EQ(1, points_to_set.size()); + EXPECT_FALSE(points_to_set.IsAmbiguous()); + EXPECT_TRUE(points_to_set.IsDistinct()); + ExpectHasTopLevelBuffers(points_to_set.CreateFlattenedSet(), {constant}); +} + TEST_F(TuplePointsToAnalysisTest, DuplicatedElement) { // Create a tuple which contains duplicate elements. auto builder = HloComputation::Builder(TestName()); diff --git a/tensorflow/compiler/xla/service/while_loop_invariant_code_motion.cc b/tensorflow/compiler/xla/service/while_loop_invariant_code_motion.cc index b7c28bfac78..41011176ffa 100644 --- a/tensorflow/compiler/xla/service/while_loop_invariant_code_motion.cc +++ b/tensorflow/compiler/xla/service/while_loop_invariant_code_motion.cc @@ -21,6 +21,7 @@ limitations under the License. #include "tensorflow/compiler/xla/service/tuple_util.h" #include "tensorflow/compiler/xla/service/while_loop_analysis.h" #include "tensorflow/compiler/xla/service/while_util.h" +#include "tensorflow/compiler/xla/shape_util.h" #include "tensorflow/compiler/xla/util.h" namespace xla { @@ -207,6 +208,37 @@ WhileLoopInvariantCodeMotion::TryHoistingInvariantInstructionsFromWhileBody( continue; } + if (!hoist_size_inflating_ops_) { + // Check that hoisting the instruction doesn't cause a significant memory + // blow-up. LICM extends the live-range of the output of the hoisted + // instruction to be the entire while loop, which may be problematic on + // platforms where memory is limited. This can be especially harmful if + // the instruction has a significantly larger output than its input, e.g. + // kIota, kBroadcast or kConstant. + int64 input_size = 0, output_size = 0; + + for (auto* operand : instruction->operands()) { + ShapeUtil::ForEachSubshape( + operand->shape(), + [&input_size](const Shape& subshape, const ShapeIndex& /*index*/) { + if (ShapeUtil::IsArray(subshape)) { + input_size += ShapeUtil::ByteSizeOfElements(subshape); + } + }); + } + ShapeUtil::ForEachSubshape( + instruction->shape(), + [&output_size](const Shape& subshape, const ShapeIndex& /*index*/) { + if (ShapeUtil::IsArray(subshape)) { + output_size += ShapeUtil::ByteSizeOfElements(subshape); + } + }); + + if (output_size > input_size) { + continue; + } + } + auto is_invariant = [&](HloInstruction* op) { return hoisted_instructions.find(op) != hoisted_instructions.end() || unhoisted_invariant_instructions.count(op) || diff --git a/tensorflow/compiler/xla/service/while_loop_invariant_code_motion.h b/tensorflow/compiler/xla/service/while_loop_invariant_code_motion.h index 3031899f71e..bd6232dc0a9 100644 --- a/tensorflow/compiler/xla/service/while_loop_invariant_code_motion.h +++ b/tensorflow/compiler/xla/service/while_loop_invariant_code_motion.h @@ -34,8 +34,14 @@ class WhileLoopInvariantCodeMotion : public HloModulePass { // Setting `hoist_constants` to false can be help if LICM is run in the mid // level HLO pipeline because hoisting constants out of while loop bodies can // break optimizations like constant folding. - explicit WhileLoopInvariantCodeMotion(bool hoist_constants = false) - : hoist_constants_(hoist_constants) {} + // Setting `hoist_size_inflating_ops` to false will forbid hoisting + // instructions where the size of the output(s) is larger than the size of the + // input(s). This is useful on platforms on which it's important to prevent + // blow-ups in memory size. + explicit WhileLoopInvariantCodeMotion(bool hoist_constants = false, + bool hoist_size_inflating_ops = true) + : hoist_constants_(hoist_constants), + hoist_size_inflating_ops_(hoist_size_inflating_ops) {} ~WhileLoopInvariantCodeMotion() override = default; absl::string_view name() const override { @@ -49,6 +55,7 @@ class WhileLoopInvariantCodeMotion : public HloModulePass { HloInstruction* while_instr); bool hoist_constants_; + bool hoist_size_inflating_ops_; }; } // namespace xla diff --git a/tensorflow/compiler/xla/service/while_loop_invariant_code_motion_test.cc b/tensorflow/compiler/xla/service/while_loop_invariant_code_motion_test.cc index 046ccb2d3f2..8e7c4bc8828 100644 --- a/tensorflow/compiler/xla/service/while_loop_invariant_code_motion_test.cc +++ b/tensorflow/compiler/xla/service/while_loop_invariant_code_motion_test.cc @@ -570,5 +570,59 @@ TEST_F(WhileLoopInvariantCodeMotionTest, DoNotHoistOutOfSingleIteration) { EXPECT_FALSE(simplified_loop); } +const char* const kInflatingTestCase = R"( +HloModule ModuleWithWhile + +mul { + lhs = f32[] parameter(0) + rhs = f32[] parameter(1) + ROOT mul = f32[] multiply(lhs, rhs) +} + +body { + p_body = (f32[]) parameter(0) + iota = f32[1024, 1024] iota(), iota_dimension=0 + add = f32[1024, 1024] add(iota, iota) + constant = f32[] constant(1.0) + reduce = f32[] reduce(f32[1024, 1024] add, f32[] constant), dimensions={0,1}, to_apply=mul + ROOT root = (f32[]) tuple(reduce) +} + +condition { + p_cond = (f32[]) parameter(0) + ROOT result = pred[] constant(true) +} + +ENTRY entry { + param = f32[] parameter(0) + while_init = (f32[]) tuple(param) + ROOT while = (f32[]) while(while_init), condition=condition, body=body +} +)"; + +TEST_F(WhileLoopInvariantCodeMotionTest, HoistsInflatingByDefault) { + auto m = ParseAndReturnVerifiedModule(kInflatingTestCase).ValueOrDie(); + + TF_ASSERT_OK_AND_ASSIGN( + bool simplified_loop, + WhileLoopInvariantCodeMotion(/*hoist_constants=*/true).Run(m.get())); + EXPECT_TRUE(simplified_loop); + + HloComputation* while_body = m->GetComputationWithName("wide.body"); + ASSERT_NE(while_body, nullptr); + EXPECT_THAT(while_body->instructions(), Not(Contains(op::Iota()))); +} + +TEST_F(WhileLoopInvariantCodeMotionTest, NoHoistInflating) { + auto m = ParseAndReturnVerifiedModule(kInflatingTestCase).ValueOrDie(); + + TF_ASSERT_OK_AND_ASSIGN( + bool simplified_loop, + WhileLoopInvariantCodeMotion(/*hoist_constants=*/true, + /*hoist_size_inflating_ops=*/false) + .Run(m.get())); + EXPECT_FALSE(simplified_loop); +} + } // namespace } // namespace xla diff --git a/tensorflow/compiler/xla/service/while_loop_simplifier.cc b/tensorflow/compiler/xla/service/while_loop_simplifier.cc index 6f924a29d8a..d30f67dd811 100644 --- a/tensorflow/compiler/xla/service/while_loop_simplifier.cc +++ b/tensorflow/compiler/xla/service/while_loop_simplifier.cc @@ -19,13 +19,17 @@ limitations under the License. #include "absl/strings/str_cat.h" #include "absl/strings/str_join.h" #include "absl/types/optional.h" +#include "tensorflow/compiler/xla/primitive_util.h" #include "tensorflow/compiler/xla/service/call_inliner.h" #include "tensorflow/compiler/xla/service/hlo_instruction.h" +#include "tensorflow/compiler/xla/service/hlo_instructions.h" #include "tensorflow/compiler/xla/service/hlo_query.h" +#include "tensorflow/compiler/xla/service/pattern_matcher.h" #include "tensorflow/compiler/xla/service/while_loop_analysis.h" namespace xla { +namespace m = match; using absl::optional; using hlo_query::ContainsInstrWithOpcode; @@ -302,6 +306,147 @@ static StatusOr TryRemoveDeadWhileParams(HloInstruction* while_op) { return true; } +// Removes each loop parameter (i.e. member of the while loop tuple) that is a +// constant and is the same in the while loop body and the while loop init. +static StatusOr TryRemoveConstantParams(HloInstruction* while_op) { + HloModule* module = while_op->GetModule(); + HloComputation* computation = while_op->parent(); + auto* while_init = while_op->mutable_operand(0); + auto* while_body = while_op->while_body(); + auto* while_cond = while_op->while_condition(); + auto* while_body_root = while_body->root_instruction(); + if (while_init->opcode() != HloOpcode::kTuple || + while_body_root->opcode() != HloOpcode::kTuple) { + return false; + } + + TF_RET_CHECK(while_cond->num_parameters() == 1); + TF_RET_CHECK(while_body->num_parameters() == 1); + TF_RET_CHECK( + ShapeUtil::Compatible(while_init->shape(), while_body_root->shape())); + + absl::flat_hash_set constant_tuple_indices; + const auto& while_shape = while_init->shape(); + for (int64 i = 0; i < while_shape.tuple_shapes_size(); ++i) { + auto* init_elem = while_init->operand(i); + auto* body_elem = while_body_root->operand(i); + if (init_elem->opcode() == HloOpcode::kConstant && + body_elem->opcode() == HloOpcode::kConstant && + init_elem->literal() == body_elem->literal()) { + constant_tuple_indices.insert(i); + } + } + + if (constant_tuple_indices.empty()) { + return false; + } + + // OK, we found some constant elements of the while parameter! Eliminate + // them. + std::vector new_while_shape_elems; + for (int64 i = 0; i < while_shape.tuple_shapes_size(); ++i) { + if (!constant_tuple_indices.count(i)) { + new_while_shape_elems.push_back(while_shape.tuple_shapes(i)); + } + } + Shape new_while_shape = ShapeUtil::MakeTupleShape(new_while_shape_elems); + + // `new_instrs` holds instructions created outside of a computation for + // cloning. Elements added here just need to live until the end of the + // relevant CloneWithReplacement call. + std::vector> new_instrs; + auto add_new_instr = [&](std::unique_ptr instr) { + new_instrs.push_back(std::move(instr)); + return new_instrs.back().get(); + }; + + // Returns a new tuple without the elements of constant_tuple_indices. + auto remove_constant_elems = [&](HloInstruction* instr) { + CHECK(ShapeUtil::Compatible(instr->shape(), while_shape)); + + std::vector tuple_elems; + for (int64 i = 0; i < while_shape.tuple_shapes_size(); ++i) { + if (!constant_tuple_indices.count(i)) { + tuple_elems.push_back( + add_new_instr(HloInstruction::CreateGetTupleElement( + while_shape.tuple_shapes(i), instr, i))); + } + } + return HloInstruction::CreateTuple(tuple_elems); + }; + + auto add_constant_elems = [&](HloInstruction* instr) { + CHECK(ShapeUtil::Compatible(instr->shape(), new_while_shape)); + + std::vector tuple_elems; + int64 j = 0; + for (int64 i = 0; i < while_shape.tuple_shapes_size(); ++i) { + if (constant_tuple_indices.count(i)) { + tuple_elems.push_back(while_init->mutable_operand(i)); + } else { + tuple_elems.push_back( + add_new_instr(HloInstruction::CreateGetTupleElement( + while_shape.tuple_shapes(i), instr, j))); + ++j; + } + } + return HloInstruction::CreateTuple(tuple_elems); + }; + + // Special case: constant_tuple_indices covers the whole while parameter, so + // the new while shape is the empty tuple. In this case, the value of the + // while loop is simply equal to the value of `init`. + // + // It's unfortunate to special-case this, but it's simpler than the + // alternative. The problem is that if our while parameter has no + // non-constant elems, the tuple returned by `add_constant_elems` won't depend + // on instr (the loop body/cond parameter), and therefore + // CloneWithReplacementPairs will *leave the parameter out entirely*, creating + // invalid HLO. + if (ShapeUtil::IsEmptyTuple(new_while_shape)) { + TF_RETURN_IF_ERROR(computation->ReplaceInstruction(while_op, while_init)); + return true; + } + + std::unique_ptr new_while_cond = + while_cond->CloneWithReplacementPairs({ + while_cond->parameter_instruction(0), + add_constant_elems(add_new_instr(HloInstruction::CreateParameter( + 0, new_while_shape, + while_cond->parameter_instruction(0)->name()))), + }); + + std::unique_ptr new_while_body = + while_body->CloneWithReplacementPairs( + { + while_body->parameter_instruction(0), + add_constant_elems(add_new_instr(HloInstruction::CreateParameter( + 0, new_while_shape, + while_cond->parameter_instruction(0)->name()))), + }, + { + while_body->root_instruction(), + remove_constant_elems( + add_new_instr(while_body->root_instruction()->Clone())), + }); + + // Create the final while loop, and add any new instructions created to + // `computation`. + new_instrs.clear(); + TF_RETURN_IF_ERROR(computation->ReplaceWithNewInstruction( + while_op, + add_constant_elems( + computation->AddInstruction(HloInstruction::CreateWhile( + new_while_shape, + module->AddEmbeddedComputation(std::move(new_while_cond)), + module->AddEmbeddedComputation(std::move(new_while_body)), + add_new_instr(remove_constant_elems(while_init))))))); + for (auto& instr : new_instrs) { + computation->AddInstruction(std::move(instr)); + } + return true; +} + // Tries to remove a while loop from the graph. // // - Loops with trip count of 0 can be replaced by the loop's "init" value. @@ -381,16 +526,14 @@ static StatusOr TryPropagateConstant(HloInstruction* while_op) { // performance by forcing us to copy constants. absl::flat_hash_map index_to_constant; for (int i = 0; i < root_operands.size(); i++) { - HloInstruction* instr = root_operands[i]; - if (instr->opcode() == HloOpcode::kGetTupleElement && - instr->tuple_index() == i && instr->operand(0) == while_body_param && - ShapeUtil::IsScalar(instr->shape())) { - auto tuple_element = while_init->operand(i); - if (tuple_element->IsConstant()) { - VLOG(3) << "Found loop invariant tuple element " << i << " " - << tuple_element->ToString(); - index_to_constant[i] = tuple_element; - } + const HloInstruction* init_tuple_elem = nullptr; + if (Match(root_operands[i], + m::GetTupleElement(m::Op().Is(while_body_param), i) + .WithShape(m::Shape().IsScalar())) && + Match(while_init->operand(i), m::Constant(&init_tuple_elem))) { + VLOG(3) << "Found loop invariant tuple element " << i << " " + << init_tuple_elem->ToString(); + index_to_constant[i] = init_tuple_elem; } } @@ -519,14 +662,6 @@ static StatusOr TryFlattenNestedTuples(HloInstruction* while_op) { return false; } - // Cowardly refuse to perform this optimization in the presence of kDomain - // instructions, which may reference other instructions in the loop and - // therefore make this complicated. - if (ContainsInstrWithOpcode(while_body, {HloOpcode::kDomain}) || - ContainsInstrWithOpcode(while_cond, {HloOpcode::kDomain})) { - return false; - } - std::vector flattened_shape_elems; ShapeUtil::ForEachSubshape(while_shape, [&](const Shape& s, const ShapeIndex& /*index*/) { @@ -605,6 +740,243 @@ static StatusOr TryFlattenNestedTuples(HloInstruction* while_op) { return true; } +// Tries to merge loop induction variables of a given type. +// +// In this pass we're only concerned with elements of the loop's tuple that +// are effective-scalars of type `elem_ty`. Some terminology: +// +// - The trip counter is the first element of the loop's tuple that starts at +// 0 and does x++ on each iteration. +// +// - An induction variable is an element of the loop's tuple that is not the +// trip counter and does `x += ` on each iteration of the loop. +// Negative constants are OK. +// +// This pass adds a trip counter if one isn't already present, then replaces +// each induction variable with +// +// + * . +// +// This reduces the number of scalar operations in the loop, which is important +// e.g. on GPUs, where each scalar operation is nontrivially expensive because +// it's a separate kernel launch. +// +// Returns the new loop if a change was made, or null if no change was made. +// Note that the new loop is not a valid replacement for the old loop; it may +// need to be wrapped in a tuple that changes its shape. We return the loop +// itself so that you can call TryMergeInductionVariables in a loop, once for +// each integral type elem_ty. +static StatusOr TryMergeInductionVariables( + HloInstruction* while_op, PrimitiveType elem_ty) { + CHECK(primitive_util::IsIntegralType(elem_ty)) << PrimitiveType_Name(elem_ty); + HloModule* module = while_op->GetModule(); + HloComputation* computation = while_op->parent(); + auto* while_init = while_op->mutable_operand(0); + auto* while_body = while_op->while_body(); + auto* while_cond = while_op->while_condition(); + auto* while_body_root = while_body->root_instruction(); + if (while_init->opcode() != HloOpcode::kTuple || + while_body_root->opcode() != HloOpcode::kTuple) { + return nullptr; + } + + TF_RET_CHECK(while_cond->num_parameters() == 1); + TF_RET_CHECK(while_body->num_parameters() == 1); + TF_RET_CHECK( + ShapeUtil::Compatible(while_init->shape(), while_body_root->shape())); + Shape while_shape = while_init->shape(); + + // The tuple index of the trip counter, if one is present. + absl::optional trip_counter; + // Maps the tuple index of each induction variable to its constant increment. + absl::flat_hash_map induction_vars; + for (int64 i = 0; i < while_body_root->operand_count(); ++i) { + HloInstruction* constant; + if (!Match(while_body_root->mutable_operand(i), + m::AddAnyOrder(m::GetTupleElement(m::Parameter(), i), + m::ConstantScalar(&constant)) + .WithShape(m::Shape().WithElementType(elem_ty)))) { + continue; + } + if (!trip_counter && constant->literal().IsAll(1) && + while_init->operand(i)->IsConstant() && + while_init->operand(i)->literal().IsAll(0)) { + VLOG(10) << "Found existing trip counter at index " << i; + trip_counter = i; + } else { + VLOG(10) << "Found induction variable at index " << i; + induction_vars.emplace(i, Cast(constant)); + } + } + + // There's only something to simplify if we can either: + // + // - combine one or more induction vars with an existing trip counter, or + // - replace two or more induction variables with a new trip counter. + // + // Put another way, there's only something to simplify if the number of + // induction vars plus the number of existing trip counters (0 or 1) is >= 2. + if (induction_vars.size() + (trip_counter.has_value() ? 1 : 0) < 2) { + return nullptr; + } + + // OK, we're going to do the transformation! Set up some helpers. + + // `new_instrs` holds instructions created outside of a computation for + // cloning. Elements added here just need to live until the end of the + // relevant CloneWithReplacement call. + std::vector> new_instrs; + auto add_new_instr = [&](std::unique_ptr instr) { + new_instrs.push_back(std::move(instr)); + return new_instrs.back().get(); + }; + + auto add_binary_op = [&](const Shape& shape, HloOpcode opcode, + HloInstruction* lhs, HloInstruction* rhs) { + // Reshape lhs/rhs to the output shape if necessary. This deals with the + // fact that induction variables need only be effective scalars, not true + // scalars. + if (!ShapeUtil::Compatible(shape, lhs->shape())) { + lhs = add_new_instr(HloInstruction::CreateReshape(shape, lhs)); + } + if (!ShapeUtil::Compatible(shape, rhs->shape())) { + rhs = add_new_instr(HloInstruction::CreateReshape(shape, rhs)); + } + return add_new_instr(HloInstruction::CreateBinary(shape, opcode, lhs, rhs)); + }; + + auto add_gte = [&](HloInstruction* src, int64 idx) { + return add_new_instr(HloInstruction::CreateGetTupleElement( + src->shape().tuple_shapes(idx), src, idx)); + }; + + // Our new while loop will have the same shape as the old while loop, except + // we'll add a trip counter to the end if it wasn't originally present. + Shape new_while_shape = while_shape; + bool added_trip_counter = false; + if (!trip_counter) { + VLOG(10) << "Adding new trip counter to end of loop's tuple."; + trip_counter = new_while_shape.tuple_shapes_size(); + *new_while_shape.add_tuple_shapes() = + ShapeUtil::MakeShape(elem_ty, /*dimensions=*/{}); + added_trip_counter = true; + } + + // Converts `instr` into a tuple of the "old" form -- that is, to a tuple with + // shape `while_body->shape()` and where the induction variables are "reified" + // (i.e. they have value + * ). + auto convert_to_old_form = [&](HloInstruction* instr) { + CHECK(ShapeUtil::Compatible(instr->shape(), new_while_shape)); + std::vector tuple_elems; + for (int64 i = 0; i < while_shape.tuple_shapes_size(); ++i) { + const auto& elem_shape = while_shape.tuple_shapes(i); + if (!induction_vars.count(i)) { + tuple_elems.push_back(add_gte(instr, i)); + continue; + } + tuple_elems.push_back(add_binary_op( + elem_shape, HloOpcode::kAdd, add_gte(instr, i), + add_binary_op(elem_shape, HloOpcode::kMultiply, + add_gte(instr, *trip_counter), + add_new_instr(induction_vars.at(i)->Clone())))); + } + return HloInstruction::CreateTuple(tuple_elems); + }; + + // Converts `root` into a tuple of the "new" form -- that is, to a tuple with + // shape `new_while_shape` and where the induction variables (but not trip + // counters) are replaced with their unchanging values. + auto convert_to_new_form = [&](HloInstruction* old_root, + HloParameterInstruction* loop_body_param) { + CHECK(ShapeUtil::Compatible(old_root->shape(), while_shape)); + std::vector tuple_elems; + + // In the new form, induction variables come from `init`, everything else + // (including the trip counter if it's not one we created ourselves) comes + // from the `root` tuple unmodified. + for (int64 i = 0; i < while_shape.tuple_shapes_size(); ++i) { + tuple_elems.push_back( + add_gte((induction_vars.count(i) ? loop_body_param : old_root), i)); + } + // If we created a trip counter ourselves, add 1 to it in the next + // iteration. + if (added_trip_counter) { + tuple_elems.push_back(add_binary_op( + new_while_shape.tuple_shapes(*trip_counter), HloOpcode::kAdd, + add_gte(loop_body_param, *trip_counter), + add_new_instr( + HloInstruction::CreateConstant(LiteralUtil::One(elem_ty))))); + } + + return HloInstruction::CreateTuple(tuple_elems); + }; + + // Creates a new init tuple, which is the same as the old init tuple except if + // we added a trip counter, it's set to 0. + auto get_new_while_init = [&](HloInstruction* init) { + CHECK(ShapeUtil::Compatible(init->shape(), while_shape)); + if (!added_trip_counter) { + return init; + } + std::vector tuple_elems; + for (int64 i = 0; i < while_shape.tuple_shapes_size(); ++i) { + tuple_elems.push_back(add_gte(init, i)); + } + tuple_elems.push_back(add_new_instr( + HloInstruction::CreateConstant(LiteralUtil::Zero(elem_ty)))); + return add_new_instr(HloInstruction::CreateTuple(tuple_elems)); + }; + + std::unique_ptr new_while_cond = + while_cond->CloneWithReplacementPairs({ + while_cond->parameter_instruction(0), + convert_to_old_form(add_new_instr(HloInstruction::CreateParameter( + 0, new_while_shape, + while_cond->parameter_instruction(0)->name()))), + }); + + // Creating the new while body proceeds in two steps. First we convert the + // users of the parameter to the old form. Then as a second + // CloneWithReplacement operation we convert the root to the new form. We + // have to do this in two steps because the new root needs to use the new + // param0, and during the first clone operation, only the *old-form* param0 is + // accessible. + // + // We have to add temp_new_while_body to the module because cloning a + // computation touches the module (to get its NameUniquer). + HloComputation* temp_new_while_body = + module->AddEmbeddedComputation(while_body->CloneWithReplacementPairs({ + while_body->parameter_instruction(0), + convert_to_old_form(add_new_instr(HloInstruction::CreateParameter( + 0, new_while_shape, + while_body->parameter_instruction(0)->name()))), + })); + std::unique_ptr new_while_body = + temp_new_while_body->CloneWithReplacementPairs({ + temp_new_while_body->root_instruction(), + convert_to_new_form( + add_new_instr(temp_new_while_body->root_instruction()->Clone()), + Cast( + temp_new_while_body->parameter_instruction(0))), + }); + TF_RETURN_IF_ERROR(module->RemoveEmbeddedComputation(temp_new_while_body)); + + // Create the final while loop, and add any new instructions created to + // `computation`. + new_instrs.clear(); + auto* new_while = computation->AddInstruction(HloInstruction::CreateWhile( + new_while_shape, + module->AddEmbeddedComputation(std::move(new_while_cond)), + module->AddEmbeddedComputation(std::move(new_while_body)), + get_new_while_init(while_init))); + TF_RETURN_IF_ERROR(computation->ReplaceWithNewInstruction( + while_op, convert_to_old_form(new_while))); + for (auto& instr : new_instrs) { + computation->AddInstruction(std::move(instr)); + } + return new_while; +} + StatusOr WhileLoopSimplifier::Run(HloModule* module) { XLA_VLOG_LINES(3, "WhileLoopSimplifier::Run(), before:\n" + module->ToString()); @@ -650,19 +1022,50 @@ StatusOr WhileLoopSimplifier::Run(HloModule* module) { continue; } + // TODO(b/119281462): Cowardly refuse to perform any of the following + // optimizations in the presence of kDomain instructions. It seems that + // modifying a while loop's tuple doesn't work when kDomain is present. + if (ContainsInstrWithOpcode(while_op->while_body(), {HloOpcode::kDomain}) || + ContainsInstrWithOpcode(while_op->while_condition(), + {HloOpcode::kDomain})) { + continue; + } + + // Each of the optimizations below modifies the while loop itself if it's + // successful, meaning that `while_op` is no longer valid after one of these + // transformations returns true. + TF_ASSIGN_OR_RETURN(result, TryFlattenNestedTuples(while_op)); changed |= result; if (result) { - // Successfully flattening nested tuples results in us cloning and - // replacing the while loop, meaning that `while_op` is no longer valid. continue; } TF_ASSIGN_OR_RETURN(result, TryRemoveDeadWhileParams(while_op)); changed |= result; if (result) { - // Successfully removing dead while params results in us cloning and - // replacing the while loop, meaning that `while_op` is no longer valid. + continue; + } + + TF_ASSIGN_OR_RETURN(result, TryRemoveConstantParams(while_op)); + changed |= result; + if (result) { + continue; + } + + bool merged_induction_vars = false; + // Notably missing from this list are S16 and U16. These don't currently + // work because S/U16 literals are not implemented. + for (auto elem_ty : {S8, U8, S32, U32, S64, U64}) { + TF_ASSIGN_OR_RETURN(auto* new_while_op, + TryMergeInductionVariables(while_op, elem_ty)); + if (new_while_op) { + while_op = new_while_op; + changed = true; + merged_induction_vars = true; + } + } + if (merged_induction_vars) { continue; } } diff --git a/tensorflow/compiler/xla/service/while_loop_simplifier_test.cc b/tensorflow/compiler/xla/service/while_loop_simplifier_test.cc index 05005e0b262..4950e8269e9 100644 --- a/tensorflow/compiler/xla/service/while_loop_simplifier_test.cc +++ b/tensorflow/compiler/xla/service/while_loop_simplifier_test.cc @@ -17,9 +17,12 @@ limitations under the License. #include "absl/strings/str_cat.h" #include "absl/strings/str_replace.h" +#include "tensorflow/compiler/xla/service/algebraic_simplifier.h" +#include "tensorflow/compiler/xla/service/hlo_cse.h" #include "tensorflow/compiler/xla/service/hlo_dce.h" #include "tensorflow/compiler/xla/service/hlo_instruction.h" #include "tensorflow/compiler/xla/service/hlo_matchers.h" +#include "tensorflow/compiler/xla/service/tuple_simplifier.h" #include "tensorflow/compiler/xla/test.h" #include "tensorflow/compiler/xla/tests/hlo_test_base.h" #include "tensorflow/core/lib/core/status_test_util.h" @@ -27,8 +30,17 @@ limitations under the License. namespace xla { namespace { +using ::testing::_; namespace op = xla::testing::opcode_matchers; +// Returns the first kWhile instruction within m's entry computation. +HloInstruction* FindFirstWhile(HloModule* m) { + const auto& instrs = m->entry_computation()->instructions(); + return *absl::c_find_if(instrs, [](const HloInstruction* instr) { + return instr->opcode() == HloOpcode::kWhile; + }); +} + class WhileLoopSimplifierTest : public HloTestBase { protected: // Makes an HloModule that contains a loop with `num_iters` iteration. @@ -540,11 +552,7 @@ TEST_F(WhileLoopSimplifierTest, FlattenNestedTuple) { // it easy to find. EXPECT_TRUE(HloDCE().Run(m.get()).ok()); - const auto& instrs = m->entry_computation()->instructions(); - HloInstruction* new_while = - *absl::c_find_if(instrs, [](const HloInstruction* instr) { - return instr->opcode() == HloOpcode::kWhile; - }); + HloInstruction* new_while = FindFirstWhile(m.get()); Shape flat_tuple = ShapeUtil::ParseShapeString("(s32[1], s32[2], s32[3], s32[4])") .ValueOrDie(); @@ -563,5 +571,177 @@ TEST_F(WhileLoopSimplifierTest, FlattenNestedTuple) { .ValueOrDie())); } +// Edge-case: All elements of the loop carry are constants which can be removed, +// leaving us with a nullary loop. This is a special case, we just replace the +// loop with its init. +TEST_F(WhileLoopSimplifierTest, OnlyConstantsInLoopCarry) { + const string hlo_string = R"( + HloModule Test + Body { + param = (s32[1]) parameter(0) + a = s32[1] constant({0}) + ROOT tuple = (s32[1]) tuple(a) + } + Cond { + param = (s32[1]) parameter(0) + ROOT cond = pred[] constant(true) + } + ENTRY Loop { + a = s32[1] constant({0}) + init = (s32[1]) tuple(a) + ROOT while = (s32[1]) while(init), condition=Cond, body=Body + })"; + + auto m = ParseAndReturnVerifiedModule(hlo_string).ValueOrDie(); + EXPECT_TRUE(WhileLoopSimplifier().Run(m.get()).ValueOrDie()); + EXPECT_TRUE(HloDCE().Run(m.get()).ok()); + EXPECT_TRUE(TupleSimplifier().Run(m.get()).ok()); + EXPECT_THAT(m->entry_computation()->root_instruction(), + op::Tuple(op::Constant())); +} + +TEST_F(WhileLoopSimplifierTest, RemoveConstantFromLoopCarry) { + const string hlo_string = R"( + HloModule Test + Body { + param = (s32[1], s32[2], s32[3]) parameter(0) + a = s32[1] get-tuple-element(param), index=0 + a.1 = s32[1] add(a, a) + b = s32[2] constant({1,1}) + c = s32[3] constant({10,10,10}) + ROOT tuple = (s32[1], s32[2], s32[3]) tuple(a.1, b, c) + } + Cond { + param = (s32[1], s32[2], s32[3]) parameter(0) + /* Use each tuple element. The verifier will then ensure that if any of + * these get modified, they're replaced with values of the correct shape. */ + a = s32[1] get-tuple-element(param), index=0 + b = s32[2] get-tuple-element(param), index=1 + c = s32[3] get-tuple-element(param), index=2 + ROOT cond = pred[] constant(true) + } + ENTRY Loop { + /* Only `b` should be simplified away. `a` is not a constant within the + * loop, and `c`'s value changes depending on whether we run 0 or 1 + * iterations of the loop. */ + a = s32[1] constant({0}) + b = s32[2] constant({1,1}) + c = s32[3] constant({2,2,2}) + init = (s32[1], s32[2], s32[3]) tuple(a,b,c) + ROOT while = (s32[1], s32[2], s32[3]) while(init), + condition=Cond, body=Body + })"; + + auto m = ParseAndReturnVerifiedModule(hlo_string).ValueOrDie(); + EXPECT_TRUE(WhileLoopSimplifier().Run(m.get()).ValueOrDie()); + // DCE away the old loop so there's just one while loop in the module, making + // it easy to find. + EXPECT_TRUE(HloDCE().Run(m.get()).ok()); + // Run the tuple simplifier to make the resulting HLO a bit easier to check. + EXPECT_TRUE(TupleSimplifier().Run(m.get()).ok()); + + HloInstruction* new_while = FindFirstWhile(m.get()); + Shape new_while_shape = + ShapeUtil::ParseShapeString("(s32[1], s32[3])").ValueOrDie(); + EXPECT_TRUE(ShapeUtil::Equal(new_while->shape(), new_while_shape)); + EXPECT_TRUE(ShapeUtil::Equal( + new_while->while_body()->root_instruction()->shape(), new_while_shape)); + EXPECT_TRUE(ShapeUtil::Equal( + new_while->while_body()->parameter_instruction(0)->shape(), + new_while_shape)); + EXPECT_TRUE(ShapeUtil::Equal( + new_while->while_condition()->parameter_instruction(0)->shape(), + new_while_shape)); + EXPECT_TRUE(ShapeUtil::Equal( + m->entry_computation()->root_instruction()->shape(), + ShapeUtil::ParseShapeString("(s32[1], s32[2], s32[3])").ValueOrDie())); + EXPECT_THAT(m->entry_computation()->root_instruction(), + op::Tuple(_, op::Constant(), _)); +} + +const char* const kSimpleMergeInductionVariablesModule = R"( + HloModule Test + Body { + param = (TYPE[], TYPE[], TYPE[]) parameter(0) + + a = TYPE[] get-tuple-element(param), index=0 + one = TYPE[] constant(1) + a1 = TYPE[] add(a, one) + + b = TYPE[] get-tuple-element(param), index=1 + negone = TYPE[] constant(-1) + b1 = TYPE[] add(b, negone) + + c = TYPE[] add(a, b) + + ROOT tuple = (TYPE[], TYPE[], TYPE[]) tuple(a1,b1,c) + } + Cond { + param = (TYPE[], TYPE[], TYPE[]) parameter(0) + a = TYPE[] get-tuple-element(param), index=0 + b = TYPE[] get-tuple-element(param), index=1 + sum = TYPE[] power(a, b) + ten = TYPE[] constant(10) + ROOT cond = pred[] less-than(sum, ten) + } + ENTRY Loop { + a = TYPE[] constant(10) + b = TYPE[] constant(100) + c = TYPE[] constant(0) + init = (TYPE[], TYPE[], TYPE[]) tuple(a,b,c) + while = (TYPE[], TYPE[], TYPE[]) while(init), condition=Cond, body=Body + + a1 = TYPE[] get-tuple-element(while), index=0 + b1 = TYPE[] get-tuple-element(while), index=1 + ROOT sum = TYPE[] add(a1, b1) + })"; + +TEST_F(WhileLoopSimplifierTest, MergeInductionVariables_Simple) { + string hlo_string = absl::StrReplaceAll(kSimpleMergeInductionVariablesModule, + {{"TYPE", "s32"}}); + + auto m = ParseAndReturnVerifiedModule(hlo_string).ValueOrDie(); + EXPECT_TRUE(WhileLoopSimplifier().Run(m.get()).ValueOrDie()); + // DCE away the old loop so there's just one while loop in the module, making + // it easy to find, and run the tuple simplifier to make the resulting HLO + // easier to check. + EXPECT_TRUE(HloDCE().Run(m.get()).ok()); + EXPECT_TRUE(TupleSimplifier().Run(m.get()).ok()); + + HloInstruction* new_while = FindFirstWhile(m.get()); + // We should have added a new loop counter for s32[] to the end of the tuple. + SCOPED_TRACE(m->ToString()); + Shape new_while_shape = + ShapeUtil::ParseShapeString("(s32[], s32[], s32[], s32[])").ValueOrDie(); + EXPECT_TRUE(ShapeUtil::Equal(new_while->shape(), new_while_shape)); + EXPECT_TRUE(ShapeUtil::Equal( + new_while->while_body()->root_instruction()->shape(), new_while_shape)); + EXPECT_TRUE(ShapeUtil::Equal( + new_while->while_body()->parameter_instruction(0)->shape(), + new_while_shape)); + EXPECT_TRUE(ShapeUtil::Equal( + new_while->while_condition()->parameter_instruction(0)->shape(), + new_while_shape)); + + EXPECT_THAT(new_while->while_body()->root_instruction(), + op::Tuple(op::GetTupleElement(op::Parameter(), 0), + op::GetTupleElement(op::Parameter(), 1), op::Add(), + op::Add(op::GetTupleElement(op::Parameter(), 3), + op::Constant()))); + EXPECT_THAT(new_while->while_condition()->root_instruction(), + op::Lt(op::Power(op::Add(), op::Add()), op::Constant())); +} + +// We shouldn't merge S16 induction variables; we can't create constants of this +// type because S16 literals are not implemented. +TEST_F(WhileLoopSimplifierTest, MergeInductionVariables_SkipS16) { + string hlo_string = absl::StrReplaceAll(kSimpleMergeInductionVariablesModule, + {{"TYPE", "s16"}}); + EXPECT_FALSE( + WhileLoopSimplifier() + .Run(ParseAndReturnVerifiedModule(hlo_string).ValueOrDie().get()) + .ValueOrDie()); +} + } // namespace } // namespace xla diff --git a/tensorflow/compiler/xla/shape.cc b/tensorflow/compiler/xla/shape.cc new file mode 100644 index 00000000000..746ab9e9977 --- /dev/null +++ b/tensorflow/compiler/xla/shape.cc @@ -0,0 +1,107 @@ +/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/compiler/xla/shape.h" + +#include "absl/strings/str_cat.h" +#include "absl/strings/str_join.h" +#include "tensorflow/compiler/xla/shape_util.h" + +namespace xla { + +Shape::Shape(const ShapeProto& shape_proto) { + set_element_type(shape_proto.element_type()); + dimensions_.reserve(shape_proto.dimensions_size()); + for (const int64 dimension : shape_proto.dimensions()) { + add_dimensions(dimension); + } + tuple_shapes_.reserve(shape_proto.tuple_shapes_size()); + for (const ShapeProto& element_shape : shape_proto.tuple_shapes()) { + *add_tuple_shapes() = Shape(element_shape); + } + if (shape_proto.has_layout()) { + *mutable_layout() = shape_proto.layout(); + } +} + +ShapeProto Shape::ToProto() const { + ShapeProto proto; + proto.set_element_type(element_type_); + proto.mutable_dimensions()->Reserve(dimensions_size()); + for (const int64 dimension : dimensions()) { + proto.add_dimensions(dimension); + } + proto.mutable_tuple_shapes()->Reserve(tuple_shapes_size()); + for (const Shape& shape : tuple_shapes()) { + *proto.add_tuple_shapes() = shape.ToProto(); + } + if (has_layout()) { + *proto.mutable_layout() = layout(); + } + return proto; +} + +string Shape::ToString(bool print_layout) const { + if (print_layout) { + return ShapeUtil::HumanStringWithLayout(*this); + } else { + return ShapeUtil::HumanString(*this); + } +} + +std::ostream& operator<<(std::ostream& out, const Shape& shape) { + out << shape.ToString(/*print_layout=*/true); + return out; +} + +ProgramShape::ProgramShape(const ProgramShapeProto& program_shape_proto) { + for (const ShapeProto& shape_proto : program_shape_proto.parameters()) { + *add_parameters() = Shape(shape_proto); + } + *mutable_result() = Shape(program_shape_proto.result()); + for (const string& name : program_shape_proto.parameter_names()) { + add_parameter_names(name); + } +} + +ProgramShapeProto ProgramShape::ToProto() const { + ProgramShapeProto proto; + for (const Shape& shape : parameters()) { + *proto.add_parameters() = shape.ToProto(); + } + *proto.mutable_result() = result().ToProto(); + for (const string& name : parameter_names()) { + proto.add_parameter_names(name); + } + return proto; +} + +string ProgramShape::ToString() const { + std::vector parameter_strings(parameters_size()); + for (int i = 0; i < parameters_size(); ++i) { + parameter_strings[i] = absl::StrCat( + i < parameter_names_size() ? parameter_names(i) : "(unknown)", ": ", + ShapeUtil::HumanString(parameters(i))); + } + return absl::StrCat("(", absl::StrJoin(parameter_strings, ", "), ") -> ", + ShapeUtil::HumanString(result())); +} + +std::ostream& operator<<(std::ostream& out, const ProgramShape& program_shape) { + out << program_shape.ToString() << "\n"; + return out; +} + +} // namespace xla diff --git a/tensorflow/compiler/xla/shape.h b/tensorflow/compiler/xla/shape.h new file mode 100644 index 00000000000..7f6b14ab428 --- /dev/null +++ b/tensorflow/compiler/xla/shape.h @@ -0,0 +1,204 @@ +/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_COMPILER_XLA_SHAPE_H_ +#define TENSORFLOW_COMPILER_XLA_SHAPE_H_ + +#include +#include + +#include "absl/types/optional.h" +#include "tensorflow/compiler/xla/types.h" +#include "tensorflow/compiler/xla/xla_data.pb.h" +#include "tensorflow/core/platform/types.h" + +namespace xla { + +// A shape describes the number of dimensions in a array, the bounds of each +// dimension, and the primitive component type. For tuples, shape describes the +// structure (number of elements and nesting). +class Shape { + public: + Shape() = default; + + // Construct a shape from a ShapeProto. + explicit Shape(const ShapeProto& shape_proto); + + // Returns a ShapeProto representation of the Shape. + ShapeProto ToProto() const; + + // Returns a human-readable string that represents the given shape, with or + // without layout. e.g. "F32[42,12] {0, 1}" or "F32[64]". + string ToString(bool print_layout = false) const; + + // The following methods mirror the protobuf generated code interface for the + // message ShapeProto. This enabled easy migration of this data structure + // from a proto to a proper C++ class. + // TODO(b/29771030): Replace or augment these methods with a more ergonomic + // interface. + + // Methods for accessing the primitive type. + PrimitiveType element_type() const { return element_type_; } + void set_element_type(PrimitiveType value) { element_type_ = value; } + + // Methods for accessing the dimensions array. + int dimensions_size() const { return dimensions_.size(); } + int64 dimensions(int index) const { return dimensions_.at(index); } + void set_dimensions(int index, int64 value) { dimensions_.at(index) = value; } + void add_dimensions(int64 value) { dimensions_.push_back(value); } + void clear_dimensions() { dimensions_.clear(); } + const std::vector& dimensions() const { return dimensions_; } + std::vector* mutable_dimensions() { return &dimensions_; } + + // Methods for accessing the tuple subshapes. This field only non-empty for + // tuple shapes. + int tuple_shapes_size() const { return tuple_shapes_.size(); } + const Shape& tuple_shapes(int index) const { return tuple_shapes_.at(index); } + Shape* mutable_tuple_shapes(int index) { return &tuple_shapes_.at(index); } + Shape* add_tuple_shapes() { + tuple_shapes_.push_back(Shape()); + return &tuple_shapes_.back(); + } + void clear_tuple_shapes() { tuple_shapes_.clear(); } + const std::vector& tuple_shapes() const { return tuple_shapes_; } + std::vector* mutable_tuple_shapes() { return &tuple_shapes_; } + + // Methods for accessing the layout field. + bool has_layout() const { return layout_.has_value(); } + const Layout& layout() const { + if (layout_.has_value()) { + return *layout_; + } else { + return Layout::default_instance(); + } + } + Layout* mutable_layout() { + if (!layout_.has_value()) { + layout_ = Layout(); + } + return &layout_.value(); + } + void clear_layout() { layout_.reset(); } + + void Swap(Shape* other) { + using std::swap; + swap(*this, *other); + } + + void Clear() { + element_type_ = PRIMITIVE_TYPE_INVALID; + dimensions_.clear(); + tuple_shapes_.clear(); + layout_.reset(); + } + + string SerializeAsString() const { return ToProto().SerializeAsString(); } + string ShortDebugString() const { return ToProto().ShortDebugString(); } + string DebugString() const { return ToProto().DebugString(); } + + public: + // The element type of this shape (tuple, array, etc). + PrimitiveType element_type_ = PRIMITIVE_TYPE_INVALID; + + // The array bounds of the dimensions. This is nonempty only for array shapes. + std::vector dimensions_; + + // The tuple element subshapes. This is nonempty only for tuple shapes. + std::vector tuple_shapes_; + + // The array layout of the shape. This is present only for array shapes. + absl::optional layout_; +}; + +// Shape of the parameters and output of an XLA computation. This is analogous +// to a traditional function signature. +class ProgramShape { + public: + ProgramShape() = default; + + // Creates a ProgramShape from a ProgramShapeProto protobuf. + explicit ProgramShape(const ProgramShapeProto& program_shape_proto); + + // Returns a proto representation of the object. + ProgramShapeProto ToProto() const; + + string ToString() const; + + // The following methods mirror the protobuf generated code interface for the + // message ProgramShapeProto. This enabled easy migration of this data + // structure from a proto to a proper C++ class. + // TODO(b/29771030): Replace or augment these methods with a more ergonomic + // interface. + + // Methods for accessing and manipulating the Shape of the parameters. + int parameters_size() const { return parameters_.size(); } + const Shape& parameters(int index) const { return parameters_.at(index); } + Shape* mutable_parameters(int index) { return ¶meters_.at(index); } + Shape* add_parameters() { + parameters_.emplace_back(); + return ¶meters_.back(); + } + void clear_parameters() { parameters_.clear(); } + const std::vector& parameters() const { return parameters_; } + std::vector* mutable_parameters() { return ¶meters_; } + + // Methods for accessing and manipulating the Shape of the result. + const Shape& result() const { return result_; } + Shape* mutable_result() { return &result_; } + + // Methods for accessing and manipulating the names of the parameters. + int parameter_names_size() const { return parameter_names_.size(); } + const string& parameter_names(int index) const { + return parameter_names_.at(index); + } + void set_parameter_names(int index, const string& value) { + parameter_names_.at(index) = value; + } + string* mutable_parameter_names(int index) { + return ¶meter_names_.at(index); + } + void add_parameter_names(const string& value) { + parameter_names_.push_back(value); + } + string* add_parameter_names() { + parameter_names_.push_back(""); + return ¶meter_names_.back(); + } + void clear_parameter_names() { parameter_names_.clear(); } + const std::vector& parameter_names() const { + return parameter_names_; + } + std::vector* mutable_parameter_names() { return ¶meter_names_; } + + string ShortDebugString() const { return ToProto().ShortDebugString(); } + string DebugString() const { return ToProto().DebugString(); } + + private: + // The shapes of the parameters of the computation represented by this object. + std::vector parameters_; + + // The names of the parameters of the computation represented by this object. + std::vector parameter_names_; + + // The shape of the result of the computation represented by this object. + Shape result_; +}; + +std::ostream& operator<<(std::ostream& out, const Shape& shape); +std::ostream& operator<<(std::ostream& out, const ProgramShape& program_shape); + +} // namespace xla + +#endif // TENSORFLOW_COMPILER_XLA_SHAPE_H_ diff --git a/tensorflow/compiler/xla/shape_test.cc b/tensorflow/compiler/xla/shape_test.cc new file mode 100644 index 00000000000..e396897eeeb --- /dev/null +++ b/tensorflow/compiler/xla/shape_test.cc @@ -0,0 +1,149 @@ +/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/compiler/xla/shape.h" + +#include +#include "absl/strings/str_cat.h" +#include "absl/strings/str_join.h" +#include "tensorflow/compiler/xla/layout_util.h" +#include "tensorflow/compiler/xla/shape_util.h" +#include "tensorflow/compiler/xla/status_macros.h" +#include "tensorflow/compiler/xla/test.h" +#include "tensorflow/compiler/xla/test_helpers.h" +#include "tensorflow/compiler/xla/types.h" +#include "tensorflow/compiler/xla/util.h" +#include "tensorflow/compiler/xla/xla_data.pb.h" + +namespace xla { +namespace { + +class ShapeTest : public ::testing::Test { + protected: + const Shape opaque_ = ShapeUtil::MakeOpaqueShape(); + const Shape token_ = ShapeUtil::MakeTokenShape(); + const Shape scalar_ = ShapeUtil::MakeShape(F32, {}); + const Shape matrix_ = ShapeUtil::MakeShape(U32, {1, 2}); + const Shape matrix2_ = ShapeUtil::MakeShapeWithLayout(S32, {3, 4}, {0, 1}); + const Shape tuple_ = + ShapeUtil::MakeTupleShape({opaque_, scalar_, matrix_, matrix2_}); + const Shape nested_tuple_ = + ShapeUtil::MakeTupleShape({tuple_, matrix_, token_}); +}; + +TEST_F(ShapeTest, ShapeToFromProto) { + for (const Shape& shape : + {opaque_, token_, scalar_, matrix_, matrix2_, tuple_, nested_tuple_}) { + Shape shape_copy(shape.ToProto()); + EXPECT_TRUE(ShapeUtil::Equal(shape, shape_copy)) + << shape << " != " << shape_copy; + } +} + +TEST_F(ShapeTest, ShapeToString) { + EXPECT_EQ("opaque[]", opaque_.ToString()); + EXPECT_EQ("token[]", token_.ToString()); + EXPECT_EQ("f32[]", scalar_.ToString()); + EXPECT_EQ("u32[1,2]", matrix_.ToString()); + EXPECT_EQ("s32[3,4]", matrix2_.ToString()); + EXPECT_EQ("(opaque[], f32[], u32[1,2], s32[3,4])", tuple_.ToString()); + EXPECT_EQ("((opaque[], f32[], u32[1,2], s32[3,4]), u32[1,2], token[])", + nested_tuple_.ToString()); + + EXPECT_EQ("opaque[]", opaque_.ToString(/*print_layout=*/true)); + EXPECT_EQ("f32[]", scalar_.ToString(/*print_layout=*/true)); + EXPECT_EQ("u32[1,2]{1,0}", matrix_.ToString(/*print_layout=*/true)); + EXPECT_EQ("s32[3,4]{0,1}", matrix2_.ToString(/*print_layout=*/true)); + EXPECT_EQ("(opaque[], f32[], u32[1,2]{1,0}, s32[3,4]{0,1})", + tuple_.ToString(/*print_layout=*/true)); + EXPECT_EQ( + "((opaque[], f32[], u32[1,2]{1,0}, s32[3,4]{0,1}), u32[1,2]{1,0}, " + "token[])", + nested_tuple_.ToString(/*print_layout=*/true)); +} + +TEST_F(ShapeTest, ProgramShapeToFromProto) { + ProgramShape program_shape; + *program_shape.add_parameters() = ShapeUtil::MakeShape(F32, {1, 2, 3}); + *program_shape.add_parameters() = ShapeUtil::MakeTokenShape(); + *program_shape.add_parameters() = ShapeUtil::MakeShape(S64, {}); + *program_shape.add_parameters() = ShapeUtil::MakeTupleShape( + {ShapeUtil::MakeShape(S32, {}), + ShapeUtil::MakeTupleShape({ShapeUtil::MakeTokenShape()}), + ShapeUtil::MakeShape(F32, {42, 42})}); + + *program_shape.mutable_result() = ShapeUtil::MakeShape(F32, {7}); + + program_shape.add_parameter_names("foo"); + program_shape.add_parameter_names("bar"); + program_shape.add_parameter_names("baz"); + program_shape.add_parameter_names("qux qux"); + + // Create a copy of the program shape by round-tripping through a proto. + ProgramShape program_shape_copy(program_shape.ToProto()); + ASSERT_EQ(program_shape.parameters_size(), + program_shape_copy.parameters_size()); + for (int i = 0; i < program_shape.parameters_size(); ++i) { + EXPECT_TRUE(ShapeUtil::Equal(program_shape.parameters(i), + program_shape_copy.parameters(i))); + } + + EXPECT_TRUE( + ShapeUtil::Equal(program_shape.result(), program_shape_copy.result())); + + ASSERT_EQ(program_shape.parameter_names_size(), + program_shape_copy.parameter_names_size()); + for (int i = 0; i < program_shape.parameter_names_size(); ++i) { + EXPECT_EQ(program_shape.parameter_names(i), + program_shape_copy.parameter_names(i)); + } +} + +TEST_F(ShapeTest, ProgramShapeToString) { + ProgramShape prog = ShapeUtil::MakeProgramShape( + {opaque_, scalar_, matrix_, matrix2_, tuple_, nested_tuple_}, + nested_tuple_); + EXPECT_EQ( + "((unknown): opaque[], " + "(unknown): f32[], " + "(unknown): u32[1,2], " + "(unknown): s32[3,4], " + "(unknown): (opaque[], f32[], u32[1,2], s32[3,4]), " + "(unknown): ((opaque[], f32[], u32[1,2], s32[3,4]), u32[1,2], token[])) " + "-> " + "((opaque[], f32[], u32[1,2], s32[3,4]), u32[1,2], token[])", + prog.ToString()); + + prog.add_parameter_names("arg0"); + prog.add_parameter_names("scalar"); + prog.add_parameter_names("matrix"); + prog.add_parameter_names("matrix2"); + prog.add_parameter_names("tuple"); + prog.add_parameter_names("nested_tuple"); + EXPECT_EQ( + "(arg0: opaque[], " + "scalar: f32[], " + "matrix: u32[1,2], " + "matrix2: s32[3,4], " + "tuple: (opaque[], f32[], u32[1,2], s32[3,4]), " + "nested_tuple: ((opaque[], f32[], u32[1,2], s32[3,4]), u32[1,2], " + "token[])) " + "-> " + "((opaque[], f32[], u32[1,2], s32[3,4]), u32[1,2], token[])", + prog.ToString()); +} + +} // namespace +} // namespace xla diff --git a/tensorflow/compiler/xla/shape_tree.h b/tensorflow/compiler/xla/shape_tree.h index df610102b4c..7bf97729165 100644 --- a/tensorflow/compiler/xla/shape_tree.h +++ b/tensorflow/compiler/xla/shape_tree.h @@ -667,12 +667,11 @@ void ShapeTree::CopySubtreeFrom(const ShapeTree& other, template bool ShapeTree::operator==(const ShapeTree& other) const { bool equal = true; - ForEachElement( - [this, &other, &equal](const ShapeIndex& index, const T& data) { - if (data != other.element(index)) { - equal = false; - } - }); + ForEachElement([&other, &equal](const ShapeIndex& index, const T& data) { + if (data != other.element(index)) { + equal = false; + } + }); return equal; } diff --git a/tensorflow/compiler/xla/shape_tree_test.cc b/tensorflow/compiler/xla/shape_tree_test.cc index c8ff55e7845..2b6c484bc4f 100644 --- a/tensorflow/compiler/xla/shape_tree_test.cc +++ b/tensorflow/compiler/xla/shape_tree_test.cc @@ -52,10 +52,10 @@ class ShapeTreeTest : public ::testing::Test { TEST_F(ShapeTreeTest, DefaultConstructor) { ShapeTree int_tree; - EXPECT_TRUE(ShapeUtil::IsNil(int_tree.shape())); + EXPECT_TRUE(ShapeUtil::IsEmptyTuple(int_tree.shape())); ShapeTree bool_tree; - EXPECT_TRUE(ShapeUtil::IsNil(bool_tree.shape())); + EXPECT_TRUE(ShapeUtil::IsEmptyTuple(bool_tree.shape())); } void ShapeTreeTest::TestShapeConstructor(const Shape& shape, diff --git a/tensorflow/compiler/xla/shape_util.cc b/tensorflow/compiler/xla/shape_util.cc index d0c35d8dee4..f3cc51ca915 100644 --- a/tensorflow/compiler/xla/shape_util.cc +++ b/tensorflow/compiler/xla/shape_util.cc @@ -79,14 +79,14 @@ bool ShapeIndexView::StartsWith(ShapeIndexView prefix) const { indices_.subspan(0, prefix.size()) == prefix.indices_; } -namespace { - -// Returns whether the given primitive type corresponds to an array shape. -bool IsArrayPrimitiveType(PrimitiveType primitive_type) { +/* static */ bool ShapeUtil::IsArrayPrimitiveType( + PrimitiveType primitive_type) { return primitive_type != PRIMITIVE_TYPE_INVALID && primitive_type != TUPLE && primitive_type != OPAQUE && primitive_type != TOKEN; } +namespace { + // Recursive helper for comparing the equality of two shapes. Returns true if // the shapes are the same. If compare_layouts is true, then layouts must also // match. @@ -121,6 +121,23 @@ bool CompareShapes(const Shape& lhs, const Shape& rhs, bool compare_layouts, VLOG(3) << "CompareShapes: lhs layout != rhs layout"; return false; } + + const auto& lhs_tiles = lhs.layout().tiles(); + const auto& rhs_tiles = rhs.layout().tiles(); + if (lhs_tiles.size() != rhs_tiles.size()) { + return false; + } + for (int64 i = 0; i < lhs_tiles.size(); i++) { + if (!absl::c_equal(lhs_tiles[i].dimensions(), + rhs_tiles[i].dimensions())) { + return false; + } + } + + if (lhs.layout().element_size_in_bits() != + rhs.layout().element_size_in_bits()) { + return false; + } } } @@ -203,7 +220,7 @@ StatusOr MakeShapeWithLayoutInternal( /* static */ ProgramShape ShapeUtil::MakeProgramShape( std::initializer_list parameters, Shape result) { ProgramShape program_shape; - for (const auto& shape : parameters) { + for (const Shape& shape : parameters) { *program_shape.add_parameters() = shape; } *program_shape.mutable_result() = std::move(result); @@ -272,7 +289,7 @@ ShapeUtil::MakeShapeWithDescendingLayoutAndSamePhysicalLayout( /* static */ Shape ShapeUtil::MakeTupleShape(absl::Span shapes) { Shape result; result.set_element_type(TUPLE); - result.mutable_tuple_shapes()->Reserve(shapes.size()); + result.mutable_tuple_shapes()->reserve(shapes.size()); for (const auto& shape : shapes) { AppendShapeToTuple(shape, &result); } @@ -372,10 +389,6 @@ ShapeUtil::MakeShapeWithDescendingLayoutAndSamePhysicalLayout( return IsTuple(shape) && TupleElementCount(shape) == 0; } -/* static */ bool ShapeUtil::IsNil(const Shape& shape) { - return IsEmptyTuple(shape); -} - /* static */ int64 ShapeUtil::TupleElementCount(const Shape& shape) { CHECK(IsTuple(shape)) << HumanString(shape); return shape.tuple_shapes_size(); @@ -1155,7 +1168,7 @@ Status ForEachMutableSubshapeHelper( // Let the argument `permutation` be P. This is a permutation over `shape`'s // dimensions, so our return value will be a shape with dims P.I = P. Our // goal is to construct a layout permutation L* that we can apply to P such - // that that the physical dimension ordering of the returned shape is the same + // that the physical dimension ordering of the returned shape is the same // as that of the original shape, namely L'. // // Our returned shape has dims P and layout L*, so its in-memory layout is @@ -1600,7 +1613,8 @@ ShapeUtil::DimensionsUnmodifiedByReshape(const Shape& input_shape, /* static */ Shape ShapeUtil::DeleteDimension(int64 dim_to_delete, Shape shape) { CHECK(IsArray(shape)); - shape.mutable_dimensions()->erase(shape.dimensions().begin() + dim_to_delete); + shape.mutable_dimensions()->erase(shape.mutable_dimensions()->begin() + + dim_to_delete); if (LayoutUtil::HasLayout(shape)) { Layout* layout = shape.mutable_layout(); layout->set_format(DENSE); @@ -1634,11 +1648,6 @@ ShapeUtil::DimensionsUnmodifiedByReshape(const Shape& input_shape, return shape; } -std::ostream& operator<<(std::ostream& out, const Shape& shape) { - out << ShapeUtil::HumanStringWithLayout(shape); - return out; -} - /*static*/ size_t ShapeUtil::Hash(const Shape& shape) { using tensorflow::hash; using tensorflow::Hash64Combine; diff --git a/tensorflow/compiler/xla/shape_util.h b/tensorflow/compiler/xla/shape_util.h index a7a3026cf3f..84a27f662a5 100644 --- a/tensorflow/compiler/xla/shape_util.h +++ b/tensorflow/compiler/xla/shape_util.h @@ -28,6 +28,7 @@ limitations under the License. #include "absl/types/span.h" #include "tensorflow/compiler/xla/layout_util.h" #include "tensorflow/compiler/xla/primitive_util.h" +#include "tensorflow/compiler/xla/shape.h" #include "tensorflow/compiler/xla/status_macros.h" #include "tensorflow/compiler/xla/statusor.h" #include "tensorflow/compiler/xla/types.h" @@ -37,6 +38,7 @@ limitations under the License. #include "tensorflow/core/platform/cpu_info.h" #include "tensorflow/core/platform/env.h" #include "tensorflow/core/platform/macros.h" +#include "tensorflow/core/platform/mutex.h" #include "tensorflow/core/platform/types.h" namespace xla { @@ -100,6 +102,11 @@ class ShapeIndex { string ToString() const; + template + friend H AbslHashValue(H h, const ShapeIndex& index) { + return H::combine(std::move(h), index.indices_); + } + private: container_type indices_; }; @@ -461,6 +468,9 @@ class ShapeUtil { // arrays. static bool IsArray(const Shape& shape); + // Returns whether the given primitive type corresponds to an array shape. + static bool IsArrayPrimitiveType(PrimitiveType primitive_type); + // Returns whether the shape is a tuple with at least one element which is // also a tuple. static bool IsNestedTuple(const Shape& shape); @@ -468,9 +478,6 @@ class ShapeUtil { // Returns true if shape is an empty tuple. static bool IsEmptyTuple(const Shape& shape); - // Returns true if shape is the nil shape (an empty tuple). - static bool IsNil(const Shape& shape); - // Returns the number of elements in the given tuple shape. // Precondition: IsTuple(shape) static int64 TupleElementCount(const Shape& shape); @@ -754,10 +761,18 @@ class ShapeUtil { pool.emplace(tensorflow::Env::Default(), "foreach", kNumThreads); } + tensorflow::mutex mu; + Status status; // Guarded by mu + while (n < rank) { if (pool != absl::nullopt) { - pool->Schedule( - [indexes, &visitor_function] { visitor_function(indexes); }); + pool->Schedule([indexes, &visitor_function, &mu, &status] { + StatusOr result = visitor_function(indexes); + if (!result.ok()) { + tensorflow::mutex_lock lock(mu); + status = status.ok() ? result.status() : status; + } + }); } else { TF_ASSIGN_OR_RETURN(bool should_continue, visitor_function(indexes)); if (!should_continue) { @@ -775,14 +790,14 @@ class ShapeUtil { } } - return Status::OK(); + // Waits for the scheduled work to complete. + pool.reset(); + return status; } TF_DISALLOW_COPY_AND_ASSIGN(ShapeUtil); }; -std::ostream& operator<<(std::ostream& out, const Shape& shape); - } // namespace xla #endif // TENSORFLOW_COMPILER_XLA_SHAPE_UTIL_H_ diff --git a/tensorflow/compiler/xla/shape_util_test.cc b/tensorflow/compiler/xla/shape_util_test.cc index 0c647369a37..60bdbe30204 100644 --- a/tensorflow/compiler/xla/shape_util_test.cc +++ b/tensorflow/compiler/xla/shape_util_test.cc @@ -376,12 +376,12 @@ TEST(ShapeUtilTest, ByteSizeOfWithoutPadding) { } TEST(ShapeUtilTest, NilShape) { - EXPECT_TRUE(ShapeUtil::IsNil(ShapeUtil::MakeNil())); - EXPECT_FALSE(ShapeUtil::IsNil(ShapeUtil::MakeShape(F32, {1, 2, 3}))); - EXPECT_FALSE(ShapeUtil::IsNil(ShapeUtil::MakeShape(F32, {0, 1}))); - EXPECT_FALSE(ShapeUtil::IsNil( + EXPECT_TRUE(ShapeUtil::IsEmptyTuple(ShapeUtil::MakeNil())); + EXPECT_FALSE(ShapeUtil::IsEmptyTuple(ShapeUtil::MakeShape(F32, {1, 2, 3}))); + EXPECT_FALSE(ShapeUtil::IsEmptyTuple(ShapeUtil::MakeShape(F32, {0, 1}))); + EXPECT_FALSE(ShapeUtil::IsEmptyTuple( ShapeUtil::MakeTupleShape({ShapeUtil::MakeShape(S32, {})}))); - EXPECT_FALSE(ShapeUtil::IsNil( + EXPECT_FALSE(ShapeUtil::IsEmptyTuple( ShapeUtil::MakeTupleShape({ShapeUtil::MakeShape(F32, {0})}))); } @@ -546,68 +546,6 @@ TEST(ShapeUtilTest, IsLeafIndex) { EXPECT_TRUE(ShapeUtil::IsLeafIndex(nested_tuple_shape, {1, 1})); } -TEST(ShapeUtilTest, HumanString) { - Shape opaque = ShapeUtil::MakeOpaqueShape(); - Shape token = ShapeUtil::MakeTokenShape(); - Shape scalar = ShapeUtil::MakeShape(F32, {}); - Shape matrix = ShapeUtil::MakeShape(U32, {1, 2}); - Shape matrix2 = ShapeUtil::MakeShapeWithLayout(S32, {3, 4}, {0, 1}); - Shape tuple = ShapeUtil::MakeTupleShape({opaque, scalar, matrix, matrix2}); - Shape nested_tuple = ShapeUtil::MakeTupleShape({tuple, matrix, token}); - - EXPECT_EQ("opaque[]", ShapeUtil::HumanString(opaque)); - EXPECT_EQ("token[]", ShapeUtil::HumanString(token)); - EXPECT_EQ("f32[]", ShapeUtil::HumanString(scalar)); - EXPECT_EQ("u32[1,2]", ShapeUtil::HumanString(matrix)); - EXPECT_EQ("s32[3,4]", ShapeUtil::HumanString(matrix2)); - EXPECT_EQ("(opaque[], f32[], u32[1,2], s32[3,4])", - ShapeUtil::HumanString(tuple)); - EXPECT_EQ("((opaque[], f32[], u32[1,2], s32[3,4]), u32[1,2], token[])", - ShapeUtil::HumanString(nested_tuple)); - - EXPECT_EQ("opaque[]", ShapeUtil::HumanStringWithLayout(opaque)); - EXPECT_EQ("f32[]", ShapeUtil::HumanStringWithLayout(scalar)); - EXPECT_EQ("u32[1,2]{1,0}", ShapeUtil::HumanStringWithLayout(matrix)); - EXPECT_EQ("s32[3,4]{0,1}", ShapeUtil::HumanStringWithLayout(matrix2)); - EXPECT_EQ("(opaque[], f32[], u32[1,2]{1,0}, s32[3,4]{0,1})", - ShapeUtil::HumanStringWithLayout(tuple)); - EXPECT_EQ( - "((opaque[], f32[], u32[1,2]{1,0}, s32[3,4]{0,1}), u32[1,2]{1,0}, " - "token[])", - ShapeUtil::HumanStringWithLayout(nested_tuple)); - - ProgramShape prog = ShapeUtil::MakeProgramShape( - {opaque, scalar, matrix, matrix2, tuple, nested_tuple}, nested_tuple); - EXPECT_EQ( - "((unknown): opaque[], " - "(unknown): f32[], " - "(unknown): u32[1,2], " - "(unknown): s32[3,4], " - "(unknown): (opaque[], f32[], u32[1,2], s32[3,4]), " - "(unknown): ((opaque[], f32[], u32[1,2], s32[3,4]), u32[1,2], token[])) " - "-> " - "((opaque[], f32[], u32[1,2], s32[3,4]), u32[1,2], token[])", - ShapeUtil::HumanString(prog)); - - prog.add_parameter_names("arg0"); - prog.add_parameter_names("scalar"); - prog.add_parameter_names("matrix"); - prog.add_parameter_names("matrix2"); - prog.add_parameter_names("tuple"); - prog.add_parameter_names("nested_tuple"); - EXPECT_EQ( - "(arg0: opaque[], " - "scalar: f32[], " - "matrix: u32[1,2], " - "matrix2: s32[3,4], " - "tuple: (opaque[], f32[], u32[1,2], s32[3,4]), " - "nested_tuple: ((opaque[], f32[], u32[1,2], s32[3,4]), u32[1,2], " - "token[])) " - "-> " - "((opaque[], f32[], u32[1,2], s32[3,4]), u32[1,2], token[])", - ShapeUtil::HumanString(prog)); -} - TEST(ShapeUtilTest, ForEachSubshapeArray) { const Shape shape = ShapeUtil::MakeShape(F32, {2, 3}); int calls = 0; diff --git a/tensorflow/compiler/xla/tests/BUILD b/tensorflow/compiler/xla/tests/BUILD index db34d34f969..f7f090fe4ab 100644 --- a/tensorflow/compiler/xla/tests/BUILD +++ b/tensorflow/compiler/xla/tests/BUILD @@ -79,6 +79,7 @@ cc_library( "//tensorflow/compiler/xla/service:hlo_verifier", "//tensorflow/compiler/xla/service:transfer_manager", "//tensorflow/core:lib", + "@com_google_absl//absl/base", "@com_google_absl//absl/memory", "@com_google_absl//absl/types:span", ], @@ -135,6 +136,7 @@ cc_library( "//tensorflow/core:stream_executor_no_cuda", "//tensorflow/core:test", "@com_google_absl//absl/algorithm:container", + "@com_google_absl//absl/base:core_headers", "@com_google_absl//absl/memory", "@com_google_absl//absl/types:optional", "@com_google_absl//absl/types:span", @@ -297,6 +299,56 @@ xla_test( ], ) +xla_test( + name = "conv_depthwise_test", + timeout = "long", + srcs = ["conv_depthwise_test.cc"], + blacklisted_backends = [ + # disabled because of a break b/119590850. + "gpu", + ], + shard_count = 50, + deps = [ + "//tensorflow/compiler/xla:execution_options_util", + "//tensorflow/compiler/xla:status_macros", + "//tensorflow/compiler/xla:test", + "//tensorflow/compiler/xla/client:xla_computation", + "//tensorflow/compiler/xla/service:bfloat16_normalization", + "//tensorflow/compiler/xla/service:despecializer", + "//tensorflow/compiler/xla/service:hlo_parser", + "//tensorflow/compiler/xla/tests:client_library_test_base", + "//tensorflow/compiler/xla/tests:hlo_test_base", + "//tensorflow/compiler/xla/tests:xla_internal_test_main", + "@com_google_absl//absl/types:optional", + ], +) + +xla_test( + name = "grouped_convolution_test", + timeout = "long", + srcs = ["grouped_convolution_test.cc"], + blacklisted_backends = [ + # disabled because of a break b/119590850. + "gpu", + # disabled because it times out. + "cpu", + ], + shard_count = 50, + deps = [ + "//tensorflow/compiler/xla:execution_options_util", + "//tensorflow/compiler/xla:status_macros", + "//tensorflow/compiler/xla:test", + "//tensorflow/compiler/xla/client:xla_computation", + "//tensorflow/compiler/xla/service:bfloat16_normalization", + "//tensorflow/compiler/xla/service:despecializer", + "//tensorflow/compiler/xla/service:hlo_parser", + "//tensorflow/compiler/xla/tests:client_library_test_base", + "//tensorflow/compiler/xla/tests:hlo_test_base", + "//tensorflow/compiler/xla/tests:xla_internal_test_main", + "@com_google_absl//absl/types:optional", + ], +) + xla_test( name = "check_execution_arity_test", srcs = ["check_execution_arity_test.cc"], @@ -1265,6 +1317,7 @@ xla_test( "enable_for_xla_interpreter", ], deps = [ + "//tensorflow/compiler/xla/service:hlo_parser", "//tensorflow/compiler/xla/service:hlo_verifier", "//tensorflow/compiler/xla/tests:hlo_test_base", "//tensorflow/compiler/xla/tests:xla_internal_test_main", @@ -1865,6 +1918,7 @@ xla_test( xla_test( name = "multioutput_fusion_test", srcs = ["multioutput_fusion_test.cc"], + backends = ["gpu"], deps = [ "//tensorflow/compiler/xla:literal", "//tensorflow/compiler/xla:shape_util", diff --git a/tensorflow/compiler/xla/tests/array_elementwise_ops_test.cc b/tensorflow/compiler/xla/tests/array_elementwise_ops_test.cc index 2180b22cb3b..f6be27bee27 100644 --- a/tensorflow/compiler/xla/tests/array_elementwise_ops_test.cc +++ b/tensorflow/compiler/xla/tests/array_elementwise_ops_test.cc @@ -350,6 +350,44 @@ TEST_P(ArrayElementwiseOpTestParamCount, AddManyValues) { error_spec_); } +// TODO(b/119692968): This test runs OOM on the GPU and CPU backend. +XLA_TEST_F(ArrayElementwiseOpTest, + DISABLED_ON_GPU(DISABLED_ON_CPU(DeeplyNestedAddWithSlices))) { + XlaBuilder builder(TestName()); + std::vector values(30, 0.0); + auto a_literal = LiteralUtil::CreateR1(values); + auto a = Parameter(&builder, 0, a_literal.shape(), "x"); + auto b_literal = LiteralUtil::CreateR1(values); + auto b = Parameter(&builder, 1, b_literal.shape(), "x"); + + // Construct a sequence of diamond-shaped gadgets like this: + // + // add + // / \ + // slice slice + // \ / + // add + // + // Each 'left' slice removes the last element, each 'right' slice removes the + // first element. In this way, we index into the add with different + // multi-dimensional index arrays, which defeats the caching we use to avoid + // exponential compile time. + std::function generate_recursive = + [&](int64 slice_size) -> XlaOp { + if (slice_size == values.size()) { + return Add(a, b); + } + XlaOp param = generate_recursive(slice_size + 1); + auto slice1 = Slice(param, {0}, {slice_size}, {1}); + auto slice2 = Slice(param, {1}, {slice_size + 1}, {1}); + return Add(slice1, slice2); + }; + generate_recursive(1); + auto a_data = client_->TransferToServer(a_literal).ConsumeValueOrDie(); + auto b_data = client_->TransferToServer(b_literal).ConsumeValueOrDie(); + ComputeAndCompareR1(&builder, {0.0}, {a_data.get(), b_data.get()}); +} + XLA_TEST_F(ArrayElementwiseOpTest, SubTwoConstantF32s) { XlaBuilder builder(TestName()); auto a = ConstantR1(&builder, {-2.5f, 3.14f, 2.25f, -10.0f, 6.0f}); @@ -2744,12 +2782,16 @@ XLA_TEST_F(ArrayElementwiseOpTest, CompareGtR3F32sWithDegenerateDim2) { Array3D expected_3d( {{{0, 1}, {0, 0}, {0, 0}}, {{0, 1}, {1, 0}, {0, 1}}}); const string expected = R"(pred[2,3,2] { -{ { 0, 1 }, +{ + { 0, 1 }, { 0, 0 }, - { 0, 0 } }, -{ { 0, 1 }, + { 0, 0 } +}, +{ + { 0, 1 }, { 1, 0 }, - { 0, 1 } } + { 0, 1 } +} })"; EXPECT_EQ(expected, ExecuteToString(&builder, {})); } diff --git a/tensorflow/compiler/xla/tests/broadcast_simple_test.cc b/tensorflow/compiler/xla/tests/broadcast_simple_test.cc index dde19fb65d6..702fb32adfc 100644 --- a/tensorflow/compiler/xla/tests/broadcast_simple_test.cc +++ b/tensorflow/compiler/xla/tests/broadcast_simple_test.cc @@ -161,8 +161,7 @@ XLA_TEST_F(BroadcastSimpleTest, 1DTo2D) { XLA_TEST_F(BroadcastSimpleTest, 1DTo2D_WithDimsUsual) { XlaBuilder b(TestName()); - BroadcastInDim(ConstantR1(&b, {1, 2}), - ShapeUtil::MakeShape(F32, {2, 2}), {1}); + BroadcastInDim(ConstantR1(&b, {1, 2}), {2, 2}, {1}); Array2D expected(2, 2); expected(0, 0) = 1; @@ -175,8 +174,7 @@ XLA_TEST_F(BroadcastSimpleTest, 1DTo2D_WithDimsUsual) { XLA_TEST_F(BroadcastSimpleTest, 1DTo2D_WithDimsTranspose) { XlaBuilder b(TestName()); - BroadcastInDim(ConstantR1(&b, {1, 2}), - ShapeUtil::MakeShape(F32, {2, 2}), {0}); + BroadcastInDim(ConstantR1(&b, {1, 2}), {2, 2}, {0}); Array2D expected(2, 2); expected(0, 0) = 1; @@ -189,8 +187,8 @@ XLA_TEST_F(BroadcastSimpleTest, 1DTo2D_WithDimsTranspose) { XLA_TEST_F(BroadcastSimpleTest, 2DTo3D_WithDims) { XlaBuilder b(TestName()); - BroadcastInDim(ConstantR2(&b, {{1.0, 5.0}, {2.0, 6.0}}), - ShapeUtil::MakeShape(F32, {2, 2, 2}), {0, 1}); + BroadcastInDim(ConstantR2(&b, {{1.0, 5.0}, {2.0, 6.0}}), {2, 2, 2}, + {0, 1}); Array3D expected(2, 2, 2); expected(0, 0, 0) = 1.0; @@ -207,8 +205,8 @@ XLA_TEST_F(BroadcastSimpleTest, 2DTo3D_WithDims) { XLA_TEST_F(BroadcastSimpleTest, 2DTo3D_WithDimsNotPossibleWithBroadCast) { XlaBuilder b(TestName()); - BroadcastInDim(ConstantR2(&b, {{1.0, 5.0}, {2.0, 6.0}}), - ShapeUtil::MakeShape(F32, {2, 2, 2}), {0, 2}); + BroadcastInDim(ConstantR2(&b, {{1.0, 5.0}, {2.0, 6.0}}), {2, 2, 2}, + {0, 2}); Array3D expected(2, 2, 2); expected(0, 0, 0) = 1.0; @@ -225,8 +223,7 @@ XLA_TEST_F(BroadcastSimpleTest, 2DTo3D_WithDimsNotPossibleWithBroadCast) { XLA_TEST_F(BroadcastSimpleTest, 1DTo2D_WithDimsNotPossibleWithBroadCast) { XlaBuilder b(TestName()); - BroadcastInDim(ConstantR1(&b, {1, 2}), - ShapeUtil::MakeShape(F32, {3, 2}), {1}); + BroadcastInDim(ConstantR1(&b, {1, 2}), {3, 2}, {1}); Array2D expected(3, 2); expected(0, 0) = 1; diff --git a/tensorflow/compiler/xla/tests/client_library_test_base.cc b/tensorflow/compiler/xla/tests/client_library_test_base.cc index b98572e24c8..12c02998333 100644 --- a/tensorflow/compiler/xla/tests/client_library_test_base.cc +++ b/tensorflow/compiler/xla/tests/client_library_test_base.cc @@ -107,7 +107,7 @@ StatusOr ClientLibraryTestBase::ExecuteAndTransfer( ExecutionOptions execution_options = execution_options_; if (shape_with_output_layout != nullptr) { *execution_options.mutable_shape_with_output_layout() = - *shape_with_output_layout; + shape_with_output_layout->ToProto(); } return client_->ExecuteAndTransfer(computation, arguments, &execution_options); @@ -127,7 +127,7 @@ StatusOr ClientLibraryTestBase::ExecuteAndTransferReference( ExecutionOptions execution_options = execution_options_; if (shape_with_output_layout != nullptr) { *execution_options.mutable_shape_with_output_layout() = - *shape_with_output_layout; + shape_with_output_layout->ToProto(); } execution_options.clear_device_handles(); return ref_client_->ExecuteAndTransfer(computation, arguments, diff --git a/tensorflow/compiler/xla/tests/client_library_test_base.h b/tensorflow/compiler/xla/tests/client_library_test_base.h index 34148e5886d..65a23dd8835 100644 --- a/tensorflow/compiler/xla/tests/client_library_test_base.h +++ b/tensorflow/compiler/xla/tests/client_library_test_base.h @@ -76,7 +76,7 @@ class ClientLibraryTestBase : public ::testing::Test { void SetFastMathDisabled(bool disabled) { auto* opts = execution_options_.mutable_debug_options(); opts->set_xla_cpu_enable_fast_math(!disabled); - opts->set_xla_gpu_enable_fast_math(!disabled); + opts->set_xla_gpu_enable_fast_min_max(!disabled); } void SetSeed(uint64 seed) { execution_options_.set_seed(seed); } diff --git a/tensorflow/compiler/xla/tests/client_test.cc b/tensorflow/compiler/xla/tests/client_test.cc index 6f2ca84bb64..363dee74b27 100644 --- a/tensorflow/compiler/xla/tests/client_test.cc +++ b/tensorflow/compiler/xla/tests/client_test.cc @@ -50,7 +50,8 @@ XLA_TEST_F(ClientTest, ExecuteWithLayout) { ExecutionOptions execution_options = execution_options_; *execution_options.mutable_shape_with_output_layout() = ShapeUtil::MakeShapeWithLayout(S32, /*dimensions=*/{2, 2}, - execute_layout); + execute_layout) + .ToProto(); TF_ASSERT_OK_AND_ASSIGN( std::unique_ptr data, client_->Execute(computation, {}, &execution_options)); @@ -84,7 +85,8 @@ XLA_TEST_F(ClientTest, ExecuteWithTupleLayout) { {ShapeUtil::MakeShapeWithLayout(S32, /*dimensions=*/{2, 2}, /*minor_to_major=*/{0, 1}), ShapeUtil::MakeShapeWithLayout(S32, /*dimensions=*/{2, 2}, - /*minor_to_major=*/{1, 0})}); + /*minor_to_major=*/{1, 0})}) + .ToProto(); TF_ASSERT_OK_AND_ASSIGN( auto result, diff --git a/tensorflow/compiler/xla/tests/concat_test.cc b/tensorflow/compiler/xla/tests/concat_test.cc index 9811a015e91..4f5b525a342 100644 --- a/tensorflow/compiler/xla/tests/concat_test.cc +++ b/tensorflow/compiler/xla/tests/concat_test.cc @@ -492,6 +492,32 @@ XLA_TEST_F(ConcatTest, ConcatR3WeirdDims) { ComputeAndCompareR3(&builder, expected, {p0.get(), p1.get()}); } +XLA_TEST_F(ConcatTest, ConcatDeeplyNested) { + XlaBuilder builder(TestName()); + auto a_literal = LiteralUtil::CreateR1({256.0}); + auto a = Parameter(&builder, 0, a_literal.shape(), "x"); + auto b = ConcatInDim(&builder, {a, a}, 0); + auto c = ConcatInDim(&builder, {b, b}, 0); + auto d = ConcatInDim(&builder, {c, c}, 0); + auto e = ConcatInDim(&builder, {d, d}, 0); + auto f = ConcatInDim(&builder, {e, e}, 0); + auto g = ConcatInDim(&builder, {f, f}, 0); + auto h = ConcatInDim(&builder, {g, g}, 0); + auto i = ConcatInDim(&builder, {h, h}, 0); + auto j = ConcatInDim(&builder, {i, i}, 0); + auto k = ConcatInDim(&builder, {j, j}, 0); + auto l = ConcatInDim(&builder, {k, k}, 0); + auto m = ConcatInDim(&builder, {l, l}, 0); + auto n = ConcatInDim(&builder, {m, m}, 0); + auto o = ConcatInDim(&builder, {n, n}, 0); + auto p = ConcatInDim(&builder, {o, o}, 0); + auto q = ConcatInDim(&builder, {p, p}, 0); + ConcatInDim(&builder, {q, q}, 0); + std::vector expected(131072, 256.0); + auto a_data = client_->TransferToServer(a_literal).ConsumeValueOrDie(); + ComputeAndCompareR1(&builder, expected, {a_data.get()}); +} + // Describes a binary rank-2 concatenation test. struct R2BinarySpec { int64 lhs_dim0; diff --git a/tensorflow/compiler/xla/tests/conv_depthwise_test.cc b/tensorflow/compiler/xla/tests/conv_depthwise_test.cc new file mode 100644 index 00000000000..bc9bd8a2691 --- /dev/null +++ b/tensorflow/compiler/xla/tests/conv_depthwise_test.cc @@ -0,0 +1,234 @@ +/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "absl/types/optional.h" +#include "tensorflow/compiler/xla/client/xla_computation.h" +#include "tensorflow/compiler/xla/execution_options_util.h" +#include "tensorflow/compiler/xla/service/bfloat16_normalization.h" +#include "tensorflow/compiler/xla/service/despecializer.h" +#include "tensorflow/compiler/xla/service/hlo_parser.h" +#include "tensorflow/compiler/xla/status_macros.h" +#include "tensorflow/compiler/xla/test.h" +#include "tensorflow/compiler/xla/tests/client_library_test_base.h" +#include "tensorflow/compiler/xla/tests/hlo_test_base.h" +#include "tensorflow/compiler/xla/tests/test_macros.h" + +namespace xla { +namespace { + +string GetFloatDataType(bool use_bfloat16) { + return use_bfloat16 ? "bf16" : "f32"; +} + +struct DepthwiseConvolution2DSpec { + int64 output_feature, window, stride, pad, lhs_dilate; + std::vector activation_dims; + std::vector activation_layout; + std::vector kernel_dims; + std::vector kernel_layout; + std::vector output_dims; + std::vector output_layout; +}; + +class DepthwiseConvolution2DTest + : public HloTestBase, + public ::testing::WithParamInterface< + ::testing::tuple> {}; + +static std::vector GetConv2DTestCases() { + std::vector config_set; + std::vector> config_options = { + {128, 6, 3, 64}, {256, 5, 3, 256}, {256, 5, 2, 144}, {144, 5, 3, 64}, + {144, 5, 2, 256}, {8, 48, 17, 8}, {128, 20, 6, 64}, {128, 1, 2, 144}, + {256, 1, 2, 64}, {64, 14, 12, 172}, {16, 9, 4, 16}}; + + for (auto option : config_options) { + int64 feature = option[0]; + int64 activation_size = option[1]; + int64 kernel_size = option[2]; + int64 batch = option[3]; + + std::vector kernel_layout = {3, 2, 1, 0}; + DepthwiseConvolution2DSpec config; + config.output_feature = feature; + config.window = kernel_size; + + config.activation_dims = {batch, activation_size, activation_size, feature}; + config.activation_layout = {3, 0, 2, 1}; + + config.kernel_dims = {kernel_size, kernel_size, 1, feature}; + config.kernel_layout = {3, 2, 1, 0}; + + if (activation_size == 1 && kernel_size == 2) { + // Test for outer dim. + config.output_dims = {batch, activation_size + kernel_size - 1, + activation_size + kernel_size, feature}; + } else if (feature == 256) { + // Restrict dilation-based tests only to one feature configuration. + config.stride = activation_size - 1; + config.pad = 0; + config.lhs_dilate = feature / 32; + config.output_dims = {batch, feature / 32, + activation_size - kernel_size + 1, feature}; + } else { + config.stride = config.pad = config.lhs_dilate = -1; + config.output_dims = {batch, activation_size - kernel_size + 1, + activation_size - kernel_size + 1, feature}; + } + + // Try this layout for all kernel shapes. + config.output_layout = {3, 0, 2, 1}; + config_set.push_back(config); + + // Try other layouts only for certain kernel shapes. + if (kernel_size % 2 == 0) { + config.activation_layout = {0, 3, 2, 1}; + config_set.push_back(config); + + config.output_layout = {0, 3, 2, 1}; + config_set.push_back(config); + + config.activation_layout = {3, 0, 2, 1}; + config_set.push_back(config); + } + } + + return config_set; +} + +string DepthwiseConvolution2DTestDataToString( + const ::testing::TestParamInfo< + ::testing::tuple>& data) { + const auto& spec = ::testing::get<0>(data.param); + const string data_type = GetFloatDataType(::testing::get<1>(data.param)); + string str = absl::StrCat( + "activation_dims_", absl::StrJoin(spec.activation_dims, "x"), + "_activation_layout_", absl::StrJoin(spec.activation_layout, "_"), + "_kernel_dims_", absl::StrJoin(spec.kernel_dims, "x"), "_kernel_layout_", + absl::StrJoin(spec.kernel_layout, "_"), "_output_dims_", + absl::StrJoin(spec.output_dims, "x"), "_output_layout_", + absl::StrJoin(spec.output_layout, "_"), data_type); + // -1 indicates non-existence. + if (spec.stride != -1) { + absl::StrAppend(&str, "_lhs_dilation_", spec.lhs_dilate, "x1"); + } + + // Test names are not allowed to contain the '-' character. + absl::c_replace(str, '-', 'n'); + return str; +} + +string BuildHloTextDepthwiseConvolution2D( + const DepthwiseConvolution2DSpec& spec, bool use_bfloat16) { + const string data_type = GetFloatDataType(use_bfloat16); + if (spec.activation_dims[1] == 1 && spec.kernel_dims[1] == 2) { + return absl::StrFormat( + R"( + HloModule TensorFlowDepthwiseConv + + ENTRY main { + activation = %s[%s]{%s} parameter(0) + kernel = %s[%s]{%s} parameter(1) + ROOT conv = %s[%s]{%s} convolution(%s[%s]{%s} activation, %s[%s]{%s} kernel), + window={size=%dx%d pad=1_1x%d_%d rhs_dilate=1x%d}, dim_labels=b01f_01io->b01f, + feature_group_count=%d + } + )", + data_type, absl::StrJoin(spec.activation_dims, ","), + absl::StrJoin(spec.activation_layout, ","), data_type, + absl::StrJoin(spec.kernel_dims, ","), + absl::StrJoin(spec.kernel_layout, ","), data_type, + absl::StrJoin(spec.output_dims, ","), + absl::StrJoin(spec.output_layout, ","), data_type, + absl::StrJoin(spec.activation_dims, ","), + absl::StrJoin(spec.activation_layout, ","), data_type, + absl::StrJoin(spec.kernel_dims, ","), + absl::StrJoin(spec.kernel_layout, ","), spec.window, spec.window, + spec.window, spec.window, spec.window, spec.output_feature); + + } else if (spec.stride == -1) { + return absl::StrFormat( + R"( + HloModule TensorFlowDepthwiseConv + + ENTRY main { + activation = %s[%s]{%s} parameter(0) + kernel = %s[%s]{%s} parameter(1) + ROOT conv = %s[%s]{%s} convolution(%s[%s]{%s} activation, %s[%s]{%s} kernel), + window={size=%dx%d}, dim_labels=b01f_01io->b01f, + feature_group_count=%d + } + )", + data_type, absl::StrJoin(spec.activation_dims, ","), + absl::StrJoin(spec.activation_layout, ","), data_type, + absl::StrJoin(spec.kernel_dims, ","), + absl::StrJoin(spec.kernel_layout, ","), data_type, + absl::StrJoin(spec.output_dims, ","), + absl::StrJoin(spec.output_layout, ","), data_type, + absl::StrJoin(spec.activation_dims, ","), + absl::StrJoin(spec.activation_layout, ","), data_type, + absl::StrJoin(spec.kernel_dims, ","), + absl::StrJoin(spec.kernel_layout, ","), spec.window, spec.window, + spec.output_feature); + } else { + return absl::StrFormat( + R"( + HloModule TensorFlowDepthwiseConv + + ENTRY main { + activation = %s[%s]{%s} parameter(0) + kernel = %s[%s]{%s} parameter(1) + ROOT conv = %s[%s]{%s} convolution(%s[%s]{%s} activation, %s[%s]{%s} kernel), + window={size=%dx%d stride=%dx1 pad=%d_%dx0_0 lhs_dilate=%dx1}, + dim_labels=b01f_01io->b01f, feature_group_count=%d + } + )", + data_type, absl::StrJoin(spec.activation_dims, ","), + absl::StrJoin(spec.activation_layout, ","), data_type, + absl::StrJoin(spec.kernel_dims, ","), + absl::StrJoin(spec.kernel_layout, ","), data_type, + absl::StrJoin(spec.output_dims, ","), + absl::StrJoin(spec.output_layout, ","), data_type, + absl::StrJoin(spec.activation_dims, ","), + absl::StrJoin(spec.activation_layout, ","), data_type, + absl::StrJoin(spec.kernel_dims, ","), + absl::StrJoin(spec.kernel_layout, ","), spec.window, spec.window, + spec.stride, 0, 0, spec.lhs_dilate, spec.output_feature); + } +} + +XLA_TEST_P(DepthwiseConvolution2DTest, DoIt) { + const DepthwiseConvolution2DSpec& spec = ::testing::get<0>(GetParam()); + bool use_bfloat16 = ::testing::get<1>(GetParam()); + const string hlo_text = + BuildHloTextDepthwiseConvolution2D(spec, use_bfloat16); + + EXPECT_TRUE(RunAndCompare(hlo_text, ErrorSpec{0.01, 0.01}, + [](HloModule* module) -> Status { + BFloat16MixedPrecisionRemoval remover; + TF_RETURN_IF_ERROR(remover.Run(module).status()); + Despecializer despecializer; + return despecializer.Run(module).status(); + })); +} + +INSTANTIATE_TEST_CASE_P( + DepthwiseConvolution2DTestWithRandomIndices, DepthwiseConvolution2DTest, + ::testing::Combine(::testing::ValuesIn(GetConv2DTestCases()), + ::testing::Bool()), + DepthwiseConvolution2DTestDataToString); + +} // namespace +} // namespace xla diff --git a/tensorflow/compiler/xla/tests/convolution_test.cc b/tensorflow/compiler/xla/tests/convolution_test.cc index 211d004ec8c..459add96813 100644 --- a/tensorflow/compiler/xla/tests/convolution_test.cc +++ b/tensorflow/compiler/xla/tests/convolution_test.cc @@ -721,8 +721,6 @@ class Convolve2D_1x4x4x512_3x3x1x512_Depthwise_Valid : public ConvolutionTest { ComputeAndCompareLiteral(&builder, expected_r4, {input_literal.get(), filter_literal.get()}, error_spec_); - - auto filter_r = filter_r1.Reshape(filter_dims); } }; @@ -731,6 +729,291 @@ TYPED_TEST(Convolve2D_1x4x4x512_3x3x1x512_Depthwise_Valid, Types) { this->RunTest(); } +template +class Convolve2D_1x4x4x512_3x3x1x512_Depthwise_Valid_Output_Batch_In_Lanes + : public ConvolutionTest { + public: + void RunTest() { + XlaBuilder builder(TestName()); + std::vector input_dims = {1, 4, 4, 512}; + std::vector filter_dims = {3, 3, 1, 512}; + Shape input_shape = ShapeUtil::MakeShapeWithType(input_dims); + Shape filter_shape = ShapeUtil::MakeShapeWithType(filter_dims); + { + auto input = Parameter(&builder, 0, input_shape, "input"); + auto filter = Parameter(&builder, 1, filter_shape, "filter"); + + // Tensorflow dimension numbers for 2D convolution. + ConvolutionDimensionNumbers dnums; + dnums.set_input_batch_dimension(0); + dnums.set_output_batch_dimension(0); + dnums.add_input_spatial_dimensions(1); + dnums.add_output_spatial_dimensions(1); + dnums.add_input_spatial_dimensions(2); + dnums.add_output_spatial_dimensions(2); + dnums.set_input_feature_dimension(3); + dnums.set_output_feature_dimension(3); + dnums.add_kernel_spatial_dimensions(0); + dnums.add_kernel_spatial_dimensions(1); + dnums.set_kernel_input_feature_dimension(2); + dnums.set_kernel_output_feature_dimension(3); + + ConvWithGeneralDimensions(input, filter, {1, 1}, Padding::kValid, dnums, + /*feature_group_count=*/512); + } + + std::vector input_elems(ShapeUtil::ElementsIn(input_shape), + static_cast(1)); + auto input_r1 = LiteralUtil::CreateR1(input_elems); + auto input_r4 = input_r1.Reshape(input_dims).ConsumeValueOrDie(); + + std::vector filter_elems(ShapeUtil::ElementsIn(filter_shape), + static_cast(2)); + auto filter_r1 = LiteralUtil::CreateR1(filter_elems); + auto filter_r4 = filter_r1.Reshape(filter_dims).ConsumeValueOrDie(); + + std::vector output_elems(2048, static_cast(18)); + + auto expected_r1 = LiteralUtil::CreateR1(output_elems); + auto expected_r4 = expected_r1.Reshape({1, 2, 2, 512}).ConsumeValueOrDie(); + auto expected_r4_relaid = + expected_r4.Relayout(LayoutUtil::MakeLayout({0, 3, 2, 1})); + + auto input_literal = + client_->TransferToServer(input_r4).ConsumeValueOrDie(); + auto filter_literal = + client_->TransferToServer(filter_r4).ConsumeValueOrDie(); + + ComputeAndCompareLiteral(&builder, expected_r4_relaid, + {input_literal.get(), filter_literal.get()}, + error_spec_, &expected_r4_relaid.shape()); + } +}; + +TYPED_TEST_CASE( + Convolve2D_1x4x4x512_3x3x1x512_Depthwise_Valid_Output_Batch_In_Lanes, + TestTypes); +TYPED_TEST(Convolve2D_1x4x4x512_3x3x1x512_Depthwise_Valid_Output_Batch_In_Lanes, + Types) { + this->RunTest(); +} + +template +class Convolve2D_256x4x4x512_3x3x1x512_Depthwise_Input_Batch_in_Lanes + : public ConvolutionTest { + public: + void RunTest() { + XlaBuilder builder(TestName()); + std::vector input_dims = {256, 4, 4, 512}; + std::vector filter_dims = {3, 3, 1, 512}; + Shape input_shape = ShapeUtil::MakeShapeWithType(input_dims); + Shape filter_shape = ShapeUtil::MakeShapeWithType(filter_dims); + { + auto input = Parameter(&builder, 0, input_shape, "input"); + auto filter = Parameter(&builder, 1, filter_shape, "filter"); + + // Tensorflow dimension numbers for 2D convolution. + ConvolutionDimensionNumbers dnums; + dnums.set_input_batch_dimension(0); + dnums.set_output_batch_dimension(0); + dnums.add_input_spatial_dimensions(1); + dnums.add_output_spatial_dimensions(1); + dnums.add_input_spatial_dimensions(2); + dnums.add_output_spatial_dimensions(2); + dnums.set_input_feature_dimension(3); + dnums.set_output_feature_dimension(3); + dnums.add_kernel_spatial_dimensions(0); + dnums.add_kernel_spatial_dimensions(1); + dnums.set_kernel_input_feature_dimension(2); + dnums.set_kernel_output_feature_dimension(3); + + ConvWithGeneralDimensions(input, filter, {1, 1}, Padding::kValid, dnums, + /*feature_group_count=*/512); + } + + std::vector input_elems(ShapeUtil::ElementsIn(input_shape), + static_cast(1)); + auto input_r1 = LiteralUtil::CreateR1(input_elems); + auto input_r4 = input_r1.Reshape(input_dims).ConsumeValueOrDie(); + auto input_r4_relaid = + input_r4.Relayout(LayoutUtil::MakeLayout({0, 3, 2, 1})); + + std::vector filter_elems(ShapeUtil::ElementsIn(filter_shape), + static_cast(2)); + auto filter_r1 = LiteralUtil::CreateR1(filter_elems); + auto filter_r4 = filter_r1.Reshape(filter_dims).ConsumeValueOrDie(); + + std::vector output_elems(2048 * 256, static_cast(18)); + + auto expected_r1 = LiteralUtil::CreateR1(output_elems); + auto expected_r4 = + expected_r1.Reshape({256, 2, 2, 512}).ConsumeValueOrDie(); + + auto input_literal = + client_->TransferToServer(input_r4_relaid).ConsumeValueOrDie(); + auto filter_literal = + client_->TransferToServer(filter_r4).ConsumeValueOrDie(); + + ComputeAndCompareLiteral(&builder, expected_r4, + {input_literal.get(), filter_literal.get()}, + error_spec_); + } +}; + +TYPED_TEST_CASE(Convolve2D_256x4x4x512_3x3x1x512_Depthwise_Input_Batch_in_Lanes, + TestTypes); +TYPED_TEST(Convolve2D_256x4x4x512_3x3x1x512_Depthwise_Input_Batch_in_Lanes, + Types) { + this->RunTest(); +} + +template +class Convolve2D_256x4x4x512_3x3x1x512_Depthwise_Both_Batch_in_Lanes + : public ConvolutionTest { + public: + void RunTest() { + XlaBuilder builder(TestName()); + std::vector input_dims = {256, 4, 4, 512}; + std::vector filter_dims = {3, 3, 1, 512}; + Shape input_shape = ShapeUtil::MakeShapeWithType(input_dims); + Shape filter_shape = ShapeUtil::MakeShapeWithType(filter_dims); + { + auto input = Parameter(&builder, 0, input_shape, "input"); + auto filter = Parameter(&builder, 1, filter_shape, "filter"); + + // Tensorflow dimension numbers for 2D convolution. + ConvolutionDimensionNumbers dnums; + dnums.set_input_batch_dimension(0); + dnums.set_output_batch_dimension(0); + dnums.add_input_spatial_dimensions(1); + dnums.add_output_spatial_dimensions(1); + dnums.add_input_spatial_dimensions(2); + dnums.add_output_spatial_dimensions(2); + dnums.set_input_feature_dimension(3); + dnums.set_output_feature_dimension(3); + dnums.add_kernel_spatial_dimensions(0); + dnums.add_kernel_spatial_dimensions(1); + dnums.set_kernel_input_feature_dimension(2); + dnums.set_kernel_output_feature_dimension(3); + + ConvWithGeneralDimensions(input, filter, {1, 1}, Padding::kValid, dnums, + /*feature_group_count=*/512); + } + + std::vector input_elems(ShapeUtil::ElementsIn(input_shape), + static_cast(1)); + auto input_r1 = LiteralUtil::CreateR1(input_elems); + auto input_r4 = input_r1.Reshape(input_dims).ConsumeValueOrDie(); + auto input_r4_relaid = + input_r4.Relayout(LayoutUtil::MakeLayout({0, 3, 2, 1})); + + std::vector filter_elems(ShapeUtil::ElementsIn(filter_shape), + static_cast(2)); + auto filter_r1 = LiteralUtil::CreateR1(filter_elems); + auto filter_r4 = filter_r1.Reshape(filter_dims).ConsumeValueOrDie(); + + std::vector output_elems(2048 * 256, static_cast(18)); + + auto expected_r1 = LiteralUtil::CreateR1(output_elems); + auto expected_r4 = + expected_r1.Reshape({256, 2, 2, 512}).ConsumeValueOrDie(); + auto expected_r4_relaid = + expected_r4.Relayout(LayoutUtil::MakeLayout({0, 3, 2, 1})); + + auto input_literal = + client_->TransferToServer(input_r4_relaid).ConsumeValueOrDie(); + auto filter_literal = + client_->TransferToServer(filter_r4).ConsumeValueOrDie(); + + ComputeAndCompareLiteral(&builder, expected_r4_relaid, + {input_literal.get(), filter_literal.get()}, + error_spec_, &expected_r4_relaid.shape()); + } +}; + +TYPED_TEST_CASE(Convolve2D_256x4x4x512_3x3x1x512_Depthwise_Both_Batch_in_Lanes, + TestTypes); +TYPED_TEST(Convolve2D_256x4x4x512_3x3x1x512_Depthwise_Both_Batch_in_Lanes, + Types) { + this->RunTest(); +} + +template +class Convolve2D_1x4x4x5_3x3x1x5_Depthwise_Valid_Output_Batch_In_Lanes + : public ConvolutionTest { + public: + void RunTest() { + XlaBuilder builder(TestName()); + std::vector input_dims = {1, 4, 4, 5}; + std::vector filter_dims = {3, 3, 1, 5}; + Shape input_shape = ShapeUtil::MakeShapeWithType(input_dims); + Shape filter_shape = ShapeUtil::MakeShapeWithType(filter_dims); + { + auto input = Parameter(&builder, 0, input_shape, "input"); + auto filter = Parameter(&builder, 1, filter_shape, "filter"); + + // Tensorflow dimension numbers for 2D convolution. + ConvolutionDimensionNumbers dnums; + dnums.set_input_batch_dimension(0); + dnums.set_output_batch_dimension(0); + dnums.add_input_spatial_dimensions(1); + dnums.add_output_spatial_dimensions(1); + dnums.add_input_spatial_dimensions(2); + dnums.add_output_spatial_dimensions(2); + dnums.set_input_feature_dimension(3); + dnums.set_output_feature_dimension(3); + dnums.add_kernel_spatial_dimensions(0); + dnums.add_kernel_spatial_dimensions(1); + dnums.set_kernel_input_feature_dimension(2); + dnums.set_kernel_output_feature_dimension(3); + + ConvWithGeneralDimensions(input, filter, {1, 1}, Padding::kValid, dnums, + /*feature_group_count=*/5); + } + + std::vector input_elems(ShapeUtil::ElementsIn(input_shape)); + iota_int_init_value(input_elems, 1); + auto input_r1 = LiteralUtil::CreateR1(input_elems); + auto input_r4 = input_r1.Reshape(input_dims).ConsumeValueOrDie(); + auto input_r4_relaid = + input_r4.Relayout(LayoutUtil::MakeLayout({0, 3, 2, 1})); + + std::vector filter_elems(ShapeUtil::ElementsIn(filter_shape)); + iota_int_init_value(filter_elems, 1); + auto filter_r1 = LiteralUtil::CreateR1(filter_elems); + auto filter_r4 = filter_r1.Reshape(filter_dims).ConsumeValueOrDie(); + + auto expected_r1 = LiteralUtil::CreateR1( + {static_cast(6864), static_cast(7296), static_cast(7746), + static_cast(8214), static_cast(8700), static_cast(7809), + static_cast(8286), static_cast(8781), static_cast(9294), + static_cast(9825), static_cast(10644), static_cast(11256), + static_cast(11886), static_cast(12534), static_cast(13200), + static_cast(11589), static_cast(12246), static_cast(12921), + static_cast(13614), static_cast(14325)}); + auto expected_r4 = expected_r1.Reshape({1, 2, 2, 5}).ConsumeValueOrDie(); + auto expected_r4_relaid = + expected_r4.Relayout(LayoutUtil::MakeLayout({0, 3, 2, 1})); + + auto input_literal = + client_->TransferToServer(input_r4_relaid).ConsumeValueOrDie(); + auto filter_literal = + client_->TransferToServer(filter_r4).ConsumeValueOrDie(); + + ComputeAndCompareLiteral(&builder, expected_r4_relaid, + {input_literal.get(), filter_literal.get()}, + error_spec_, &expected_r4_relaid.shape()); + } +}; + +TYPED_TEST_CASE( + Convolve2D_1x4x4x5_3x3x1x5_Depthwise_Valid_Output_Batch_In_Lanes, + TestTypes); +TYPED_TEST(Convolve2D_1x4x4x5_3x3x1x5_Depthwise_Valid_Output_Batch_In_Lanes, + Types) { + this->RunTest(); +} + template class Convolve2D_1x4x4x160_3x3x1x160_Depthwise_Valid : public ConvolutionTest { public: @@ -786,8 +1069,6 @@ class Convolve2D_1x4x4x160_3x3x1x160_Depthwise_Valid : public ConvolutionTest { ComputeAndCompareLiteral(&builder, expected_r4, {input_literal.get(), filter_literal.get()}, error_spec_); - - auto filter_r = filter_r1.Reshape(filter_dims); } }; @@ -796,6 +1077,146 @@ TYPED_TEST(Convolve2D_1x4x4x160_3x3x1x160_Depthwise_Valid, Types) { this->RunTest(); } +template +class Convolve2D_1x4x4x160_3x3x1x160_Depthwise_Input_Batch_In_Lanes + : public ConvolutionTest { + public: + void RunTest() { + XlaBuilder builder(TestName()); + std::vector input_dims = {1, 4, 4, 160}; + std::vector filter_dims = {3, 3, 1, 160}; + Shape input_shape = ShapeUtil::MakeShapeWithType(input_dims); + Shape filter_shape = ShapeUtil::MakeShapeWithType(filter_dims); + { + auto input = Parameter(&builder, 0, input_shape, "input"); + auto filter = Parameter(&builder, 1, filter_shape, "filter"); + + // Tensorflow dimension numbers for 2D convolution. + ConvolutionDimensionNumbers dnums; + dnums.set_input_batch_dimension(0); + dnums.set_output_batch_dimension(0); + dnums.add_input_spatial_dimensions(1); + dnums.add_output_spatial_dimensions(1); + dnums.add_input_spatial_dimensions(2); + dnums.add_output_spatial_dimensions(2); + dnums.set_input_feature_dimension(3); + dnums.set_output_feature_dimension(3); + dnums.add_kernel_spatial_dimensions(0); + dnums.add_kernel_spatial_dimensions(1); + dnums.set_kernel_input_feature_dimension(2); + dnums.set_kernel_output_feature_dimension(3); + + ConvWithGeneralDimensions(input, filter, {1, 1}, Padding::kValid, dnums, + /*feature_group_count=*/160); + } + + std::vector input_elems(ShapeUtil::ElementsIn(input_shape), + static_cast(1)); + auto input_r1 = LiteralUtil::CreateR1(input_elems); + auto input_r4 = input_r1.Reshape(input_dims).ConsumeValueOrDie(); + auto input_r4_relaid = + input_r4.Relayout(LayoutUtil::MakeLayout({0, 3, 2, 1})); + + std::vector filter_elems(ShapeUtil::ElementsIn(filter_shape), + static_cast(2)); + auto filter_r1 = LiteralUtil::CreateR1(filter_elems); + auto filter_r4 = filter_r1.Reshape(filter_dims).ConsumeValueOrDie(); + + std::vector output_elems(640, static_cast(18)); + + auto expected_r1 = LiteralUtil::CreateR1(output_elems); + auto expected_r4 = expected_r1.Reshape({1, 2, 2, 160}).ConsumeValueOrDie(); + auto expected_r4_relaid = + expected_r4.Relayout(LayoutUtil::MakeLayout({3, 0, 2, 1})); + + auto input_literal = + client_->TransferToServer(input_r4_relaid).ConsumeValueOrDie(); + auto filter_literal = + client_->TransferToServer(filter_r4).ConsumeValueOrDie(); + + ComputeAndCompareLiteral(&builder, expected_r4_relaid, + {input_literal.get(), filter_literal.get()}, + error_spec_, &expected_r4_relaid.shape()); + } +}; + +TYPED_TEST_CASE(Convolve2D_1x4x4x160_3x3x1x160_Depthwise_Input_Batch_In_Lanes, + TestTypes); +TYPED_TEST(Convolve2D_1x4x4x160_3x3x1x160_Depthwise_Input_Batch_In_Lanes, + Types) { + this->RunTest(); +} + +template +class Convolve2D_1x4x4x160_3x3x1x160_Dephtwise_Both_Batch_In_Lanes + : public ConvolutionTest { + public: + void RunTest() { + XlaBuilder builder(TestName()); + std::vector input_dims = {1, 4, 4, 160}; + std::vector filter_dims = {3, 3, 1, 160}; + Shape input_shape = ShapeUtil::MakeShapeWithType(input_dims); + Shape filter_shape = ShapeUtil::MakeShapeWithType(filter_dims); + { + auto input = Parameter(&builder, 0, input_shape, "input"); + auto filter = Parameter(&builder, 1, filter_shape, "filter"); + + // Tensorflow dimension numbers for 2D convolution. + ConvolutionDimensionNumbers dnums; + dnums.set_input_batch_dimension(0); + dnums.set_output_batch_dimension(0); + dnums.add_input_spatial_dimensions(1); + dnums.add_output_spatial_dimensions(1); + dnums.add_input_spatial_dimensions(2); + dnums.add_output_spatial_dimensions(2); + dnums.set_input_feature_dimension(3); + dnums.set_output_feature_dimension(3); + dnums.add_kernel_spatial_dimensions(0); + dnums.add_kernel_spatial_dimensions(1); + dnums.set_kernel_input_feature_dimension(2); + dnums.set_kernel_output_feature_dimension(3); + + ConvWithGeneralDimensions(input, filter, {1, 1}, Padding::kValid, dnums, + /*feature_group_count=*/160); + } + + std::vector input_elems(ShapeUtil::ElementsIn(input_shape), + static_cast(1)); + auto input_r1 = LiteralUtil::CreateR1(input_elems); + auto input_r4 = input_r1.Reshape(input_dims).ConsumeValueOrDie(); + auto input_r4_relaid = + input_r4.Relayout(LayoutUtil::MakeLayout({0, 3, 2, 1})); + + std::vector filter_elems(ShapeUtil::ElementsIn(filter_shape), + static_cast(2)); + auto filter_r1 = LiteralUtil::CreateR1(filter_elems); + auto filter_r4 = filter_r1.Reshape(filter_dims).ConsumeValueOrDie(); + + std::vector output_elems(640, static_cast(18)); + + auto expected_r1 = LiteralUtil::CreateR1(output_elems); + auto expected_r4 = expected_r1.Reshape({1, 2, 2, 160}).ConsumeValueOrDie(); + auto expected_r4_relaid = + expected_r4.Relayout(LayoutUtil::MakeLayout({0, 3, 2, 1})); + + auto input_literal = + client_->TransferToServer(input_r4_relaid).ConsumeValueOrDie(); + auto filter_literal = + client_->TransferToServer(filter_r4).ConsumeValueOrDie(); + + ComputeAndCompareLiteral(&builder, expected_r4_relaid, + {input_literal.get(), filter_literal.get()}, + error_spec_, &expected_r4_relaid.shape()); + } +}; + +TYPED_TEST_CASE(Convolve2D_1x4x4x160_3x3x1x160_Dephtwise_Both_Batch_In_Lanes, + TestTypes); +TYPED_TEST(Convolve2D_1x4x4x160_3x3x1x160_Dephtwise_Both_Batch_In_Lanes, + Types) { + this->RunTest(); +} + template class Convolve2D_1x4x4x1024_3x3x1x1024_Depthwise_Valid : public ConvolutionTest { @@ -852,8 +1273,6 @@ class Convolve2D_1x4x4x1024_3x3x1x1024_Depthwise_Valid ComputeAndCompareLiteral(&builder, expected_r4, {input_literal.get(), filter_literal.get()}, error_spec_); - - auto filter_r = filter_r1.Reshape(filter_dims); } }; @@ -863,7 +1282,7 @@ TYPED_TEST(Convolve2D_1x4x4x1024_3x3x1x1024_Depthwise_Valid, Types) { } template -class Convolve2D_1x2x2x6_2x2x1x12_Grouped_Valid : public ConvolutionTest { +class Convolve2D_1x2x2x6_2x2x2x12_Grouped_Valid : public ConvolutionTest { public: void RunTest() { XlaBuilder builder(TestName()); @@ -922,8 +1341,329 @@ class Convolve2D_1x2x2x6_2x2x1x12_Grouped_Valid : public ConvolutionTest { } }; -TYPED_TEST_CASE(Convolve2D_1x2x2x6_2x2x1x12_Grouped_Valid, TestTypes); -TYPED_TEST(Convolve2D_1x2x2x6_2x2x1x12_Grouped_Valid, Types) { +TYPED_TEST_CASE(Convolve2D_1x2x2x6_2x2x2x12_Grouped_Valid, TestTypes); +TYPED_TEST(Convolve2D_1x2x2x6_2x2x2x12_Grouped_Valid, Types) { + this->RunTest(); +} + +template +class Convolve2D_1x2x2x1024_2x2x128x512_Grouped_Valid : public ConvolutionTest { + public: + void RunTest() { + XlaBuilder builder(TestName()); + std::vector input_dims = {1, 2, 2, 1024}; + std::vector filter_dims = {2, 2, 128, 512}; + Shape input_shape = ShapeUtil::MakeShapeWithType(input_dims); + Shape filter_shape = ShapeUtil::MakeShapeWithType(filter_dims); + { + auto input = Parameter(&builder, 0, input_shape, "input"); + auto filter = Parameter(&builder, 1, filter_shape, "filter"); + + // Tensorflow dimension numbers for 2D convolution. + ConvolutionDimensionNumbers dnums; + dnums.set_input_batch_dimension(0); + dnums.set_output_batch_dimension(0); + dnums.add_input_spatial_dimensions(1); + dnums.add_output_spatial_dimensions(1); + dnums.add_input_spatial_dimensions(2); + dnums.add_output_spatial_dimensions(2); + dnums.set_input_feature_dimension(3); + dnums.set_output_feature_dimension(3); + dnums.add_kernel_spatial_dimensions(0); + dnums.add_kernel_spatial_dimensions(1); + dnums.set_kernel_input_feature_dimension(2); + dnums.set_kernel_output_feature_dimension(3); + + ConvWithGeneralDimensions(input, filter, {1, 1}, Padding::kValid, dnums, + /*feature_group_count=*/8); + } + + std::vector input_elems(ShapeUtil::ElementsIn(input_shape), + static_cast(1)); + + auto input_r1 = LiteralUtil::CreateR1(input_elems); + auto input_r4 = input_r1.Reshape(input_dims).ConsumeValueOrDie(); + + std::vector filter_elems(ShapeUtil::ElementsIn(filter_shape), + static_cast(2)); + + auto filter_r1 = LiteralUtil::CreateR1(filter_elems); + auto filter_r4 = filter_r1.Reshape(filter_dims).ConsumeValueOrDie(); + + std::vector output_elems(512, static_cast(1024)); + auto expected_r1 = LiteralUtil::CreateR1(output_elems); + auto expected_r4 = expected_r1.Reshape({1, 1, 1, 512}).ConsumeValueOrDie(); + + auto input_literal = + client_->TransferToServer(input_r4).ConsumeValueOrDie(); + auto filter_literal = + client_->TransferToServer(filter_r4).ConsumeValueOrDie(); + + ComputeAndCompareLiteral(&builder, expected_r4, + {input_literal.get(), filter_literal.get()}, + error_spec_); + } +}; + +TYPED_TEST_CASE(Convolve2D_1x2x2x1024_2x2x128x512_Grouped_Valid, TestTypes); +TYPED_TEST(Convolve2D_1x2x2x1024_2x2x128x512_Grouped_Valid, Types) { + this->RunTest(); +} + +template +class Convolve2D_1x2x2x1024_2x2x128x8_Grouped_Valid : public ConvolutionTest { + public: + void RunTest() { + XlaBuilder builder(TestName()); + std::vector input_dims = {1, 2, 2, 1024}; + std::vector filter_dims = {2, 2, 128, 8}; + Shape input_shape = ShapeUtil::MakeShapeWithType(input_dims); + Shape filter_shape = ShapeUtil::MakeShapeWithType(filter_dims); + { + auto input = Parameter(&builder, 0, input_shape, "input"); + auto filter = Parameter(&builder, 1, filter_shape, "filter"); + + // Tensorflow dimension numbers for 2D convolution. + ConvolutionDimensionNumbers dnums; + dnums.set_input_batch_dimension(0); + dnums.set_output_batch_dimension(0); + dnums.add_input_spatial_dimensions(1); + dnums.add_output_spatial_dimensions(1); + dnums.add_input_spatial_dimensions(2); + dnums.add_output_spatial_dimensions(2); + dnums.set_input_feature_dimension(3); + dnums.set_output_feature_dimension(3); + dnums.add_kernel_spatial_dimensions(0); + dnums.add_kernel_spatial_dimensions(1); + dnums.set_kernel_input_feature_dimension(2); + dnums.set_kernel_output_feature_dimension(3); + + ConvWithGeneralDimensions(input, filter, {1, 1}, Padding::kValid, dnums, + /*feature_group_count=*/8); + } + + std::vector input_elems(ShapeUtil::ElementsIn(input_shape), + static_cast(1)); + + auto input_r1 = LiteralUtil::CreateR1(input_elems); + auto input_r4 = input_r1.Reshape(input_dims).ConsumeValueOrDie(); + + std::vector filter_elems(ShapeUtil::ElementsIn(filter_shape), + static_cast(2)); + + auto filter_r1 = LiteralUtil::CreateR1(filter_elems); + auto filter_r4 = filter_r1.Reshape(filter_dims).ConsumeValueOrDie(); + + std::vector output_elems(8, static_cast(1024)); + auto expected_r1 = LiteralUtil::CreateR1(output_elems); + auto expected_r4 = expected_r1.Reshape({1, 1, 1, 8}).ConsumeValueOrDie(); + + auto input_literal = + client_->TransferToServer(input_r4).ConsumeValueOrDie(); + auto filter_literal = + client_->TransferToServer(filter_r4).ConsumeValueOrDie(); + + ComputeAndCompareLiteral(&builder, expected_r4, + {input_literal.get(), filter_literal.get()}, + error_spec_); + } +}; + +TYPED_TEST_CASE(Convolve2D_1x2x2x1024_2x2x128x8_Grouped_Valid, TestTypes); +TYPED_TEST(Convolve2D_1x2x2x1024_2x2x128x8_Grouped_Valid, Types) { + this->RunTest(); +} + +template +class Convolve2D_1x2x2x12_2x2x3x4_Grouped_Valid : public ConvolutionTest { + public: + void RunTest() { + XlaBuilder builder(TestName()); + std::vector input_dims = {1, 2, 2, 12}; + std::vector filter_dims = {2, 2, 3, 4}; + Shape input_shape = ShapeUtil::MakeShapeWithType(input_dims); + Shape filter_shape = ShapeUtil::MakeShapeWithType(filter_dims); + { + auto input = Parameter(&builder, 0, input_shape, "input"); + auto filter = Parameter(&builder, 1, filter_shape, "filter"); + + // Tensorflow dimension numbers for 2D convolution. + ConvolutionDimensionNumbers dnums; + dnums.set_input_batch_dimension(0); + dnums.set_output_batch_dimension(0); + dnums.add_input_spatial_dimensions(1); + dnums.add_output_spatial_dimensions(1); + dnums.add_input_spatial_dimensions(2); + dnums.add_output_spatial_dimensions(2); + dnums.set_input_feature_dimension(3); + dnums.set_output_feature_dimension(3); + dnums.add_kernel_spatial_dimensions(0); + dnums.add_kernel_spatial_dimensions(1); + dnums.set_kernel_input_feature_dimension(2); + dnums.set_kernel_output_feature_dimension(3); + + ConvWithGeneralDimensions(input, filter, {1, 1}, Padding::kValid, dnums, + /*feature_group_count=*/4); + } + + std::vector input_elems(ShapeUtil::ElementsIn(input_shape)); + iota_int_init_value(input_elems, 1); + auto input_r1 = LiteralUtil::CreateR1(input_elems); + auto input_r4 = input_r1.Reshape(input_dims).ConsumeValueOrDie(); + + std::vector filter_elems(ShapeUtil::ElementsIn(filter_shape)); + iota_int_init_value(filter_elems, 1); + auto filter_r1 = LiteralUtil::CreateR1(filter_elems); + auto filter_r4 = filter_r1.Reshape(filter_dims).ConsumeValueOrDie(); + + auto expected_r1 = + LiteralUtil::CreateR1({static_cast(7712), static_cast(8816), + static_cast(9992), static_cast(11240)}); + auto expected_r4 = expected_r1.Reshape({1, 1, 1, 4}).ConsumeValueOrDie(); + + auto input_literal = + client_->TransferToServer(input_r4).ConsumeValueOrDie(); + auto filter_literal = + client_->TransferToServer(filter_r4).ConsumeValueOrDie(); + + ComputeAndCompareLiteral(&builder, expected_r4, + {input_literal.get(), filter_literal.get()}, + error_spec_); + } +}; + +TYPED_TEST_CASE(Convolve2D_1x2x2x12_2x2x3x4_Grouped_Valid, TestTypes); +TYPED_TEST(Convolve2D_1x2x2x12_2x2x3x4_Grouped_Valid, Types) { + this->RunTest(); +} + +template +class Convolve2D_1x2x2x12_2x2x3x4_Grouped_Valid_Filter_OF_In_Sublanes + : public ConvolutionTest { + public: + void RunTest() { + XlaBuilder builder(TestName()); + std::vector input_dims = {1, 2, 2, 12}; + std::vector filter_dims = {2, 2, 4, 3}; + Shape input_shape = ShapeUtil::MakeShapeWithType(input_dims); + Shape filter_shape = ShapeUtil::MakeShapeWithType(filter_dims); + { + auto input = Parameter(&builder, 0, input_shape, "input"); + auto filter = Parameter(&builder, 1, filter_shape, "filter"); + + // Tensorflow dimension numbers for 2D convolution. + ConvolutionDimensionNumbers dnums; + dnums.set_input_batch_dimension(0); + dnums.set_output_batch_dimension(0); + dnums.add_input_spatial_dimensions(1); + dnums.add_output_spatial_dimensions(1); + dnums.add_input_spatial_dimensions(2); + dnums.add_output_spatial_dimensions(2); + dnums.set_input_feature_dimension(3); + dnums.set_output_feature_dimension(3); + dnums.add_kernel_spatial_dimensions(0); + dnums.add_kernel_spatial_dimensions(1); + dnums.set_kernel_input_feature_dimension(3); + dnums.set_kernel_output_feature_dimension(2); + + ConvWithGeneralDimensions(input, filter, {1, 1}, Padding::kValid, dnums, + /*feature_group_count=*/4); + } + + std::vector input_elems(ShapeUtil::ElementsIn(input_shape)); + iota_int_init_value(input_elems, 1); + auto input_r1 = LiteralUtil::CreateR1(input_elems); + auto input_r4 = input_r1.Reshape(input_dims).ConsumeValueOrDie(); + + std::vector filter_elems(ShapeUtil::ElementsIn(filter_shape)); + iota_int_init_value(filter_elems, 1); + auto filter_r1 = LiteralUtil::CreateR1(filter_elems); + auto filter_r4 = filter_r1.Reshape(filter_dims).ConsumeValueOrDie(); + auto filter_r4_relaid = + filter_r4.Relayout(LayoutUtil::MakeLayout({3, 2, 1, 0})); + auto expected_r1 = LiteralUtil::CreateR1( + {static_cast(6968), static_cast(8516), static_cast(10280), + static_cast(12260)}); + auto expected_r4 = expected_r1.Reshape({1, 1, 1, 4}).ConsumeValueOrDie(); + + auto input_literal = + client_->TransferToServer(input_r4).ConsumeValueOrDie(); + auto filter_literal = + client_->TransferToServer(filter_r4_relaid).ConsumeValueOrDie(); + + ComputeAndCompareLiteral(&builder, expected_r4, + {input_literal.get(), filter_literal.get()}, + error_spec_); + } +}; + +TYPED_TEST_CASE(Convolve2D_1x2x2x12_2x2x3x4_Grouped_Valid_Filter_OF_In_Sublanes, + TestTypes); +TYPED_TEST(Convolve2D_1x2x2x12_2x2x3x4_Grouped_Valid_Filter_OF_In_Sublanes, + Types) { + this->RunTest(); +} + +template +class Convolve2D_1x1x1x12_1x1x3x4_Grouped_Valid : public ConvolutionTest { + public: + void RunTest() { + XlaBuilder builder(TestName()); + std::vector input_dims = {1, 1, 1, 12}; + std::vector filter_dims = {1, 1, 3, 4}; + Shape input_shape = ShapeUtil::MakeShapeWithType(input_dims); + Shape filter_shape = ShapeUtil::MakeShapeWithType(filter_dims); + { + auto input = Parameter(&builder, 0, input_shape, "input"); + auto filter = Parameter(&builder, 1, filter_shape, "filter"); + + // Tensorflow dimension numbers for 2D convolution. + ConvolutionDimensionNumbers dnums; + dnums.set_input_batch_dimension(0); + dnums.set_output_batch_dimension(0); + dnums.add_input_spatial_dimensions(1); + dnums.add_output_spatial_dimensions(1); + dnums.add_input_spatial_dimensions(2); + dnums.add_output_spatial_dimensions(2); + dnums.set_input_feature_dimension(3); + dnums.set_output_feature_dimension(3); + dnums.add_kernel_spatial_dimensions(0); + dnums.add_kernel_spatial_dimensions(1); + dnums.set_kernel_input_feature_dimension(2); + dnums.set_kernel_output_feature_dimension(3); + + ConvWithGeneralDimensions(input, filter, {1, 1}, Padding::kValid, dnums, + /*feature_group_count=*/4); + } + + std::vector input_elems(ShapeUtil::ElementsIn(input_shape)); + iota_int_init_value(input_elems, 1); + auto input_r1 = LiteralUtil::CreateR1(input_elems); + auto input_r4 = input_r1.Reshape(input_dims).ConsumeValueOrDie(); + + std::vector filter_elems(ShapeUtil::ElementsIn(filter_shape)); + iota_int_init_value(filter_elems, 1); + auto filter_r1 = LiteralUtil::CreateR1(filter_elems); + auto filter_r4 = filter_r1.Reshape(filter_dims).ConsumeValueOrDie(); + + auto expected_r1 = + LiteralUtil::CreateR1({static_cast(38), static_cast(98), + static_cast(176), static_cast(272)}); + auto expected_r4 = expected_r1.Reshape({1, 1, 1, 4}).ConsumeValueOrDie(); + + auto input_literal = + client_->TransferToServer(input_r4).ConsumeValueOrDie(); + auto filter_literal = + client_->TransferToServer(filter_r4).ConsumeValueOrDie(); + + ComputeAndCompareLiteral(&builder, expected_r4, + {input_literal.get(), filter_literal.get()}, + error_spec_); + } +}; + +TYPED_TEST_CASE(Convolve2D_1x1x1x12_1x1x3x4_Grouped_Valid, TestTypes); +TYPED_TEST(Convolve2D_1x1x1x12_1x1x3x4_Grouped_Valid, Types) { this->RunTest(); } @@ -1217,6 +1957,18 @@ ENTRY Test { EXPECT_TRUE(RunAndCompare(kHlo, ErrorSpec{0.001})); } +XLA_TEST_F(ConvolutionHloTest, DISABLED_ON_CPU(ConvolveF64ForwardReversed)) { + constexpr char kHlo[] = R"( +HloModule TestModule + +ENTRY Test { + %arg0 = f64[3,56,56,16] parameter(0) + %arg1 = f64[3,3,3,64] parameter(1) + ROOT %conv = f64[54,54,16,64] convolution(%arg0, %arg1), window={size=3x3 rhs_reversal=1x1}, dim_labels=f01b_i01o->01bf +})"; + EXPECT_TRUE(RunAndCompare(kHlo, ErrorSpec{0.001})); +} + XLA_TEST_F(ConvolutionHloTest, DISABLED_ON_CPU(ConvolveF64BackwardFilter)) { constexpr char kHlo[] = R"( HloModule TestModule diff --git a/tensorflow/compiler/xla/tests/dot_operation_test.cc b/tensorflow/compiler/xla/tests/dot_operation_test.cc index 6c0847a8757..25091b8d5d5 100644 --- a/tensorflow/compiler/xla/tests/dot_operation_test.cc +++ b/tensorflow/compiler/xla/tests/dot_operation_test.cc @@ -637,6 +637,76 @@ XLA_TYPED_TEST(DotOperationTest_F16F32F64CF64, GeneralMatMul) { {x_data.get(), y_data.get()}, this->error_spec_); } +#ifndef XLA_TEST_BACKEND_CPU +// TODO(b/74459949): failed on CPU on 2018-10-29. +XLA_TYPED_TEST(DotOperationTest_F16F32F64CF64, GeneralMatMulR3LhsR2Rhs) { + using T = TypeParam; + + XlaBuilder builder(this->TestName()); + auto x = + Parameter(&builder, 0, ShapeUtil::MakeShapeWithType({2, 2, 2}), "x"); + auto y = Parameter(&builder, 1, ShapeUtil::MakeShapeWithType({2, 2}), "y"); + + DotDimensionNumbers dnums; + dnums.add_lhs_contracting_dimensions(1); + dnums.add_rhs_contracting_dimensions(1); + dnums.add_lhs_batch_dimensions(0); + dnums.add_rhs_batch_dimensions(0); + + DotGeneral(x, y, dnums); + + auto x_data = + this->client_ + ->TransferToServer(LiteralUtil::CreateR3FromArray3D( + {{{1.0f, 2.0f}, {3.0f, 4.0f}}, {{5.0f, 6.0f}, {7.0f, 8.0f}}})) + .ConsumeValueOrDie(); + + auto y_data = this->client_ + ->TransferToServer(LiteralUtil::CreateR2FromArray2D( + {{1.0f, 0.0f}, {0.0f, 1.0f}})) + .ConsumeValueOrDie(); + + this->template ComputeAndCompareR2( + &builder, + /*expected=*/{{1.0f, 2.0f}, {7.0f, 8.0f}}, {x_data.get(), y_data.get()}, + this->error_spec_); +} + +// TODO(b/74459949): failed on CPU on 2018-10-29. +XLA_TYPED_TEST(DotOperationTest_F16F32F64CF64, GeneralMatMulR2LhsR3Rhs) { + using T = TypeParam; + + XlaBuilder builder(this->TestName()); + auto x = Parameter(&builder, 0, ShapeUtil::MakeShapeWithType({2, 2}), "x"); + auto y = + Parameter(&builder, 1, ShapeUtil::MakeShapeWithType({2, 2, 2}), "y"); + + DotDimensionNumbers dnums; + dnums.add_lhs_contracting_dimensions(1); + dnums.add_rhs_contracting_dimensions(1); + dnums.add_lhs_batch_dimensions(0); + dnums.add_rhs_batch_dimensions(0); + + DotGeneral(x, y, dnums); + + auto x_data = this->client_ + ->TransferToServer(LiteralUtil::CreateR2FromArray2D( + {{1.0f, 0.0f}, {0.0f, 1.0f}})) + .ConsumeValueOrDie(); + + auto y_data = + this->client_ + ->TransferToServer(LiteralUtil::CreateR3FromArray3D( + {{{1.0f, 2.0f}, {3.0f, 4.0f}}, {{5.0f, 6.0f}, {7.0f, 8.0f}}})) + .ConsumeValueOrDie(); + + this->template ComputeAndCompareR2( + &builder, + /*expected=*/{{1.0f, 2.0f}, {7.0f, 8.0f}}, {x_data.get(), y_data.get()}, + this->error_spec_); +} +#endif // XLA_TEST_BACKEND_CPU + XLA_TYPED_TEST(DotOperationTest_F16F32F64CF64, GeneralMatMulMultipleBatch) { using T = TypeParam; diff --git a/tensorflow/compiler/xla/tests/grouped_convolution_test.cc b/tensorflow/compiler/xla/tests/grouped_convolution_test.cc new file mode 100644 index 00000000000..8f7049910e7 --- /dev/null +++ b/tensorflow/compiler/xla/tests/grouped_convolution_test.cc @@ -0,0 +1,245 @@ +/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "absl/types/optional.h" +#include "tensorflow/compiler/xla/client/xla_computation.h" +#include "tensorflow/compiler/xla/execution_options_util.h" +#include "tensorflow/compiler/xla/service/bfloat16_normalization.h" +#include "tensorflow/compiler/xla/service/despecializer.h" +#include "tensorflow/compiler/xla/service/hlo_parser.h" +#include "tensorflow/compiler/xla/status_macros.h" +#include "tensorflow/compiler/xla/test.h" +#include "tensorflow/compiler/xla/tests/client_library_test_base.h" +#include "tensorflow/compiler/xla/tests/hlo_test_base.h" +#include "tensorflow/compiler/xla/tests/test_macros.h" + +namespace xla { +namespace { + +string GetFloatDataType(bool use_bfloat16) { + return use_bfloat16 ? "bf16" : "f32"; +} + +struct GroupedConvolution2DSpec { + int64 input_feature, output_feature, window, stride, pad, lhs_dilate; + int64 group_size, group_count; + std::vector activation_dims; + std::vector activation_layout; + std::vector kernel_dims; + std::vector kernel_layout; + std::vector output_dims; + std::vector output_layout; +}; + +class GroupedConvolution2DTest + : public HloTestBase, + public ::testing::WithParamInterface< + ::testing::tuple> {}; + +static std::vector GetConv2DTestCases() { + std::vector config_set; + // Add to this set if you want a new test configuration. + // Rule : the penultimate number must be divisible by the last number. + std::vector> config_options = {{8, 2, 2, 1, 1024, 128}, + {512, 3, 3, 144, 1024, 16}, + {256, 3, 3, 129, 512, 64}, + {64, 1, 2, 127, 32, 8}, + {256, 3, 3, 256, 1024, 4}}; + + for (auto option : config_options) { + int64 output_feature = option[0]; + int64 activation_size = option[1]; + int64 kernel_size = option[2]; + int64 batch = option[3]; + int64 input_feature = option[4]; + int64 group_size = option[5]; + + std::vector kernel_layout = {3, 2, 1, 0}; + GroupedConvolution2DSpec config; + config.group_size = group_size; + config.group_count = input_feature / group_size; + config.output_feature = output_feature; + config.window = kernel_size; + + config.activation_dims = {batch, activation_size, activation_size, + input_feature}; + config.activation_layout = {3, 0, 2, 1}; + + config.kernel_dims = {kernel_size, kernel_size, group_size, output_feature}; + config.kernel_layout = {3, 2, 1, 0}; + + if (activation_size == 1 && kernel_size == 2) { + // Test for outer dim. + config.output_dims = {batch, activation_size + kernel_size - 1, + activation_size + kernel_size, output_feature}; + } else if (output_feature == 256) { + // Restrict dilation-based tests only to one feature configuration. + config.stride = activation_size - 1; + config.pad = 0; + config.lhs_dilate = output_feature / 32; + config.output_dims = {batch, output_feature / 32, + activation_size - kernel_size + 1, output_feature}; + } else { + config.stride = config.pad = config.lhs_dilate = -1; + config.output_dims = {batch, activation_size - kernel_size + 1, + activation_size - kernel_size + 1, output_feature}; + } + + // Try this layout for all kernel shapes. + config.output_layout = {3, 0, 2, 1}; + config_set.push_back(config); + + // Try other layouts only for certain kernel shapes. + if (kernel_size % 2 == 0) { + config.activation_layout = {0, 3, 2, 1}; + config_set.push_back(config); + + config.output_layout = {0, 3, 2, 1}; + config_set.push_back(config); + + config.activation_layout = {3, 0, 2, 1}; + config_set.push_back(config); + } + } + + return config_set; +} + +string GroupedConvolution2DTestDataToString( + const ::testing::TestParamInfo< + ::testing::tuple>& data) { + const auto& spec = ::testing::get<0>(data.param); + const string data_type = GetFloatDataType(::testing::get<1>(data.param)); + string str = absl::StrCat( + "activation_dims_", absl::StrJoin(spec.activation_dims, "x"), + "_activation_layout_", absl::StrJoin(spec.activation_layout, "_"), + "_kernel_dims_", absl::StrJoin(spec.kernel_dims, "x"), "_kernel_layout_", + absl::StrJoin(spec.kernel_layout, "_"), "_output_dims_", + absl::StrJoin(spec.output_dims, "x"), "_output_layout_", + absl::StrJoin(spec.output_layout, "_"), data_type); + // -1 indicates non-existence. + if (spec.stride != -1) { + absl::StrAppend(&str, "_lhs_dilation_", spec.lhs_dilate, "x1"); + } + + // Test names are not allowed to contain the '-' character. + absl::c_replace(str, '-', 'n'); + return str; +} + +string BuildHloTextGroupedConvolution2D(const GroupedConvolution2DSpec& spec, + bool use_bfloat16) { + const string data_type = GetFloatDataType(use_bfloat16); + if (spec.activation_dims[1] == 1 && spec.kernel_dims[1] == 2) { + // Check for outer dim. + return absl::StrFormat( + R"( + HloModule TensorFlowDepthwiseConv + + ENTRY main { + activation = %s[%s]{%s} parameter(0) + kernel = %s[%s]{%s} parameter(1) + ROOT conv = %s[%s]{%s} convolution(%s[%s]{%s} activation, %s[%s]{%s} kernel), + window={size=%dx%d pad=1_1x%d_%d rhs_dilate=1x%d}, dim_labels=b01f_01io->b01f, + feature_group_count=%d + } + )", + data_type, absl::StrJoin(spec.activation_dims, ","), + absl::StrJoin(spec.activation_layout, ","), data_type, + absl::StrJoin(spec.kernel_dims, ","), + absl::StrJoin(spec.kernel_layout, ","), data_type, + absl::StrJoin(spec.output_dims, ","), + absl::StrJoin(spec.output_layout, ","), data_type, + absl::StrJoin(spec.activation_dims, ","), + absl::StrJoin(spec.activation_layout, ","), data_type, + absl::StrJoin(spec.kernel_dims, ","), + absl::StrJoin(spec.kernel_layout, ","), spec.window, spec.window, + spec.window, spec.window, spec.window, spec.group_count); + + } else if (spec.stride == -1) { + // Check for basic, non-dilated cases. + return absl::StrFormat( + R"( + HloModule TensorFlowDepthwiseConv + + ENTRY main { + activation = %s[%s]{%s} parameter(0) + kernel = %s[%s]{%s} parameter(1) + ROOT conv = %s[%s]{%s} convolution(%s[%s]{%s} activation, %s[%s]{%s} kernel), + window={size=%dx%d}, dim_labels=b01f_01io->b01f, + feature_group_count=%d + } + )", + data_type, absl::StrJoin(spec.activation_dims, ","), + absl::StrJoin(spec.activation_layout, ","), data_type, + absl::StrJoin(spec.kernel_dims, ","), + absl::StrJoin(spec.kernel_layout, ","), data_type, + absl::StrJoin(spec.output_dims, ","), + absl::StrJoin(spec.output_layout, ","), data_type, + absl::StrJoin(spec.activation_dims, ","), + absl::StrJoin(spec.activation_layout, ","), data_type, + absl::StrJoin(spec.kernel_dims, ","), + absl::StrJoin(spec.kernel_layout, ","), spec.window, spec.window, + spec.group_count); + } else { + // Check for base dilations. + return absl::StrFormat( + R"( + HloModule TensorFlowDepthwiseConv + + ENTRY main { + activation = %s[%s]{%s} parameter(0) + kernel = %s[%s]{%s} parameter(1) + ROOT conv = %s[%s]{%s} convolution(%s[%s]{%s} activation, %s[%s]{%s} kernel), + window={size=%dx%d stride=%dx1 pad=%d_%dx0_0 lhs_dilate=%dx1}, + dim_labels=b01f_01io->b01f, feature_group_count=%d + } + )", + data_type, absl::StrJoin(spec.activation_dims, ","), + absl::StrJoin(spec.activation_layout, ","), data_type, + absl::StrJoin(spec.kernel_dims, ","), + absl::StrJoin(spec.kernel_layout, ","), data_type, + absl::StrJoin(spec.output_dims, ","), + absl::StrJoin(spec.output_layout, ","), data_type, + absl::StrJoin(spec.activation_dims, ","), + absl::StrJoin(spec.activation_layout, ","), data_type, + absl::StrJoin(spec.kernel_dims, ","), + absl::StrJoin(spec.kernel_layout, ","), spec.window, spec.window, + spec.stride, 0, 0, spec.lhs_dilate, spec.group_count); + } +} + +XLA_TEST_P(GroupedConvolution2DTest, DoIt) { + const GroupedConvolution2DSpec& spec = ::testing::get<0>(GetParam()); + bool use_bfloat16 = ::testing::get<1>(GetParam()); + const string hlo_text = BuildHloTextGroupedConvolution2D(spec, use_bfloat16); + + EXPECT_TRUE(RunAndCompare(hlo_text, ErrorSpec{0.01, 0.01}, + [](HloModule* module) -> Status { + BFloat16MixedPrecisionRemoval remover; + TF_RETURN_IF_ERROR(remover.Run(module).status()); + Despecializer despecializer; + return despecializer.Run(module).status(); + })); +} + +INSTANTIATE_TEST_CASE_P( + GroupedConvolution2DTestWithRandomIndices, GroupedConvolution2DTest, + ::testing::Combine(::testing::ValuesIn(GetConv2DTestCases()), + ::testing::Bool()), + GroupedConvolution2DTestDataToString); + +} // namespace +} // namespace xla diff --git a/tensorflow/compiler/xla/tests/hlo_test_base.cc b/tensorflow/compiler/xla/tests/hlo_test_base.cc index d8fa00272f8..989a7c705a8 100644 --- a/tensorflow/compiler/xla/tests/hlo_test_base.cc +++ b/tensorflow/compiler/xla/tests/hlo_test_base.cc @@ -99,6 +99,8 @@ void VerifiedHloModule::VerifyOrAddFailure(const string& message) { ADD_FAILURE() << "HloVerifier failed on module " << name() << (message.empty() ? "" : absl::StrCat(" (", message, ")")) << ": " << status; + LOG(ERROR) << "Contents of bad module:"; + XLA_LOG_LINES(tensorflow::ERROR, ToString()); } } @@ -140,14 +142,6 @@ std::unique_ptr HloTestBase::CreateNewVerifiedModule( allow_mixed_precision_in_hlo_verifier_); } -StatusOr> -HloTestBase::ParseAndReturnUnverifiedModule(absl::string_view hlo_text, - const HloModuleConfig& config) { - auto module = absl::make_unique(TestName(), config); - TF_RETURN_IF_ERROR(ParseHloString(hlo_text, module.get())); - return std::move(module); -} - StatusOr> HloTestBase::ParseAndReturnVerifiedModule(absl::string_view hlo_text, const HloModuleConfig& config) { diff --git a/tensorflow/compiler/xla/tests/hlo_test_base.h b/tensorflow/compiler/xla/tests/hlo_test_base.h index 366726d90b4..1d1e7f43729 100644 --- a/tensorflow/compiler/xla/tests/hlo_test_base.h +++ b/tensorflow/compiler/xla/tests/hlo_test_base.h @@ -20,6 +20,7 @@ limitations under the License. #include #include +#include "absl/base/macros.h" #include "absl/types/optional.h" #include "absl/types/span.h" #include "tensorflow/compiler/xla/service/backend.h" @@ -100,6 +101,7 @@ class HloTestBase : public ::testing::Test { // // This returns a vanilla HloModule that doesn't run the HLO verifier on // destruction. + ABSL_DEPRECATED("Use CreateNewVerifiedModule instead.") std::unique_ptr CreateNewUnverifiedModule( const string& name = TestName()); @@ -108,12 +110,6 @@ class HloTestBase : public ::testing::Test { std::unique_ptr CreateNewVerifiedModule( const string& name = TestName()); - // Parses the given string and returns module as a vanilla, unverified - // HloModule. - StatusOr> ParseAndReturnUnverifiedModule( - absl::string_view hlo_text, - const HloModuleConfig& config = HloModuleConfig()); - // Parses the given string and returns module as a VerifiedHloModule. StatusOr> ParseAndReturnVerifiedModule( absl::string_view hlo_text, diff --git a/tensorflow/compiler/xla/tests/iota_test.cc b/tensorflow/compiler/xla/tests/iota_test.cc index 310f3495922..65205f53ddc 100644 --- a/tensorflow/compiler/xla/tests/iota_test.cc +++ b/tensorflow/compiler/xla/tests/iota_test.cc @@ -113,5 +113,26 @@ INSTANTIATE_TEST_CASE_P(IotaR3TestInstantiation, IotaR3Test, /*step=*/10), ::testing::Values(0, 1, 2))); +class IotaR3PredTest : public ClientLibraryTestBase, + public ::testing::WithParamInterface {}; + +TEST_P(IotaR3PredTest, DoIt) { + const auto element_type = PRED; + const int64 num_elements = 2; + const int64 iota_dim = GetParam(); + XlaBuilder builder(TestName() + "_" + PrimitiveType_Name(element_type)); + std::vector dimensions = {42, 19}; + dimensions.insert(dimensions.begin() + iota_dim, num_elements); + Iota(&builder, ShapeUtil::MakeShape(element_type, dimensions), iota_dim); + if (primitive_util::IsFloatingPointType(element_type)) { + ComputeAndCompare(&builder, {}, ErrorSpec{0.0001}); + } else { + ComputeAndCompare(&builder, {}); + } +} + +INSTANTIATE_TEST_CASE_P(IotaR3PredTestInstantiation, IotaR3PredTest, + ::testing::Values(0, 1, 2)); + } // namespace } // namespace xla diff --git a/tensorflow/compiler/xla/tests/replay_test.cc b/tensorflow/compiler/xla/tests/replay_test.cc index 5cf87e565bf..34c7dc7c464 100644 --- a/tensorflow/compiler/xla/tests/replay_test.cc +++ b/tensorflow/compiler/xla/tests/replay_test.cc @@ -55,7 +55,8 @@ TEST_F(ReplayTest, TwoPlusTwoReplay) { client_->GetComputationShape(computation).ConsumeValueOrDie(); std::unique_ptr replayed_shape = client_->GetComputationShape(replayed).ConsumeValueOrDie(); - ASSERT_TRUE(protobuf_util::ProtobufEquals(*original_shape, *replayed_shape)); + ASSERT_TRUE(protobuf_util::ProtobufEquals(original_shape->ToProto(), + replayed_shape->ToProto())); // Run it. Literal literal = @@ -87,7 +88,8 @@ XLA_TEST_F(ReplayTest, XPlusYReplayWithParameters) { client_->GetComputationShape(computation).ConsumeValueOrDie(); std::unique_ptr replayed_shape = client_->GetComputationShape(replayed).ConsumeValueOrDie(); - ASSERT_TRUE(protobuf_util::ProtobufEquals(*original_shape, *replayed_shape)); + ASSERT_TRUE(protobuf_util::ProtobufEquals(original_shape->ToProto(), + replayed_shape->ToProto())); // Run it. std::unique_ptr x_data = @@ -133,7 +135,8 @@ TEST_F(ReplayTest, MapPlusTwoOverR1) { client_->GetComputationShape(computation).ConsumeValueOrDie(); std::unique_ptr replayed_shape = client_->GetComputationShape(replayed).ConsumeValueOrDie(); - ASSERT_TRUE(protobuf_util::ProtobufEquals(*original_shape, *replayed_shape)); + ASSERT_TRUE(protobuf_util::ProtobufEquals(original_shape->ToProto(), + replayed_shape->ToProto())); // Run it. Literal literal = diff --git a/tensorflow/compiler/xla/tests/reshape_test.cc b/tensorflow/compiler/xla/tests/reshape_test.cc index dedc95b5ae8..298136002e9 100644 --- a/tensorflow/compiler/xla/tests/reshape_test.cc +++ b/tensorflow/compiler/xla/tests/reshape_test.cc @@ -618,7 +618,8 @@ XLA_TEST_P(ReshapeTest, R4Dim0MinorLayoutToR2Dim0MajorLayout) { ExecutionOptions execution_options = execution_options_; *execution_options.mutable_shape_with_output_layout() = ShapeUtil::MakeShapeWithLayout(use_bfloat16() ? BF16 : F32, {2, 8}, - {1, 0}); + {1, 0}) + .ToProto(); Literal actual = client_ ->ExecuteAndTransfer(computation, {input.get()}, &execution_options) @@ -767,7 +768,8 @@ XLA_TEST_P(ReshapeTest, NoopReshape) { ExecutionOptions execution_options = execution_options_; *execution_options.mutable_shape_with_output_layout() = ShapeUtil::MakeShapeWithLayout(use_bfloat16() ? BF16 : F32, {7, 2, 3, 5}, - {2, 3, 0, 1}); + {2, 3, 0, 1}) + .ToProto(); Literal output_literal = client_ ->ExecuteAndTransfer(computation, {input_data.get()}, diff --git a/tensorflow/compiler/xla/tests/scatter_test.cc b/tensorflow/compiler/xla/tests/scatter_test.cc index 7e1f4aa0eb4..32de0fdf78f 100644 --- a/tensorflow/compiler/xla/tests/scatter_test.cc +++ b/tensorflow/compiler/xla/tests/scatter_test.cc @@ -129,6 +129,42 @@ ENTRY main { RunTest(hlo_text, &operand, &scatter_indices, &updates); } +XLA_TEST_F(ScatterTest, TensorFlowScatterV2_InversePermutation) { + const char* hlo_text = R"( +HloModule TensorFlowScatterV2 + +update_s32 (lhs: s32[], rhs: s32[]) -> s32[] { + lhs = s32[] parameter(0) + ROOT rhs = s32[] parameter(1) +} + +ENTRY main { + permutation = s32[3,4] parameter(0) + reshape = s32[3,4,1] reshape(permutation) + operand = s32[3,4] iota(), iota_dimension=1 + updates = s32[3,4,1,1] iota(), iota_dimension=1 + iota = s32[3,4,1] iota(), iota_dimension=0 + indices = s32[3,4,2] concatenate(iota, reshape), dimensions={2} + ROOT scatter = s32[3,4] scatter(operand, indices, updates), + to_apply=update_s32, + update_window_dims={2,3}, + inserted_window_dims={}, + scatter_dims_to_operand_dims={0,1}, + index_vector_dim=2 +} +)"; + Literal permutation = + LiteralUtil::CreateR2({{1, 3, 2, 0}, {3, 0, 2, 1}, {2, 3, 1, 0}}); + HloModuleConfig config; + config.set_debug_options(GetDebugOptionsForTest()); + TF_ASSERT_OK_AND_ASSIGN(std::unique_ptr module, + ParseHloString(hlo_text, config)); + auto actual = ExecuteAndTransfer(std::move(module), {&permutation}); + Literal expected = + LiteralUtil::CreateR2({{3, 0, 2, 1}, {1, 3, 2, 0}, {3, 2, 0, 1}}); + EXPECT_TRUE(LiteralTestUtil::Equal(expected, actual)); +} + XLA_TEST_F(ScatterTest, SimpleR4) { const char* hlo_text = R"( HloModule SimpleR4 diff --git a/tensorflow/compiler/xla/tests/test_utils.cc b/tensorflow/compiler/xla/tests/test_utils.cc index 2f18036ff4c..eafa48ed7b8 100644 --- a/tensorflow/compiler/xla/tests/test_utils.cc +++ b/tensorflow/compiler/xla/tests/test_utils.cc @@ -15,6 +15,7 @@ limitations under the License. #include +#include "absl/base/casts.h" #include "absl/memory/memory.h" #include "tensorflow/compiler/xla/literal_util.h" #include "tensorflow/compiler/xla/primitive_util.h" @@ -28,65 +29,113 @@ namespace xla { namespace { template -void PopulateWithRandomFloatingPointDataImpl(Literal* literal, - std::minstd_rand0* engine, - bool no_duplicates) { - CHECK(engine != nullptr); - CHECK_EQ(literal->shape().element_type(), - primitive_util::NativeToPrimitiveType()); - if (no_duplicates) { - // Duplicates may be generated if the number of elements in the literal - // exceeds the number of positive values supported by the type. - FloatT next_value = std::numeric_limits::min(); - for (FloatT& value : literal->data()) { - value = next_value; - next_value = - std::nextafter(next_value, std::numeric_limits::max()); - } - std::shuffle(literal->data().begin(), literal->data().end(), - *engine); - } else { - std::uniform_real_distribution generator(-0.1f, 0.2f); - for (FloatT& value : literal->data()) { - value = static_cast(generator(*engine)); - } +void PopulateWithRandomFloatingPointData(Literal* literal, + std::minstd_rand0* engine) { + std::uniform_real_distribution generator(-0.1f, 0.2f); + for (FloatT& value : literal->data()) { + value = static_cast(generator(*engine)); } } template -void PopulateWithRandomFloatingPointData(Literal* literal, - std::minstd_rand0* engine, - bool no_duplicates) { - CHECK(engine != nullptr); - PopulateWithRandomFloatingPointDataImpl(literal, engine, - no_duplicates); -} +void PopulateWithIntNext(Literal* literal); template <> -void PopulateWithRandomFloatingPointData(Literal* literal, - std::minstd_rand0* engine, - bool no_duplicates) { - // no_duplicates is ignored for half types. Unique values can only be - // generated for arrays with fewer than ~2**16 elements and no_duplicates is - // best-effort anyway. - CHECK(engine != nullptr); - std::uniform_real_distribution generator(-0.1f, 0.2f); +void PopulateWithIntNext(Literal* literal) { + // Duplicates may be generated if we don't have enough bits. + uint16 next_value = 0; for (half& value : literal->data()) { - value = static_cast(generator(*engine)); + // Zero-out the MSB of the exponent to avoid Infs and NaNs, and put it into + // the sign bit. We could be less wasteful, but this is best-effort anyway. + uint16 exponent_msb = next_value & 0x4000; + value.x = (next_value & 0xBFFF) | (exponent_msb << 1); + next_value++; } } template <> -void PopulateWithRandomFloatingPointData(Literal* literal, - std::minstd_rand0* engine, - bool no_duplicates) { - // no_duplicates is ignored for bfloat types. Unique values can only be - // generated for arrays with fewer than ~2**16 elements and no_duplicates is - // best-effort anyway. - CHECK(engine != nullptr); - std::uniform_real_distribution generator(-0.1f, 0.2f); +void PopulateWithIntNext(Literal* literal) { + // Duplicates may be generated if we don't have enough bits. + // Start at 0x80 rather than 0 to avoid denormals. + uint16 next_value = 0x80; for (bfloat16& value : literal->data()) { - value = static_cast(generator(*engine)); + // Zero-out the MSB of the exponent to avoid Infs and NaNs, and put it into + // the sign bit. We could be less wasteful, but this is best-effort anyway. + uint16 exponent_msb = next_value & 0x4000; + value.value = (next_value & 0xBFFF) | (exponent_msb << 1); + next_value++; + } +} + +template +void PopulateWithNextAfter(Literal* literal) { + // Duplicates may be generated if the number of elements in the literal + // exceeds the number of positive values supported by the type. + float next_value = std::numeric_limits::min(); + for (float& value : literal->data()) { + value = next_value; + next_value = std::nextafter(next_value, std::numeric_limits::max()); + } +} + +template ::value || + std::is_same::value, + int>::type = 0> +void PopulateWithNoDuplicateData(Literal* literal, std::minstd_rand0* engine) { + PopulateWithIntNext(literal); + std::shuffle(literal->data().begin(), literal->data().end(), + *engine); +} + +template ::value && + !std::is_same::value, + int>::type = 0> +void PopulateWithNoDuplicateData(Literal* literal, std::minstd_rand0* engine) { + PopulateWithNextAfter(literal); + std::shuffle(literal->data().begin(), literal->data().end(), + *engine); +} + +template +void PopulateWithFloatingPointData(Literal* literal, std::minstd_rand0* engine, + bool no_duplicates) { + CHECK(engine != nullptr); + CHECK_EQ(literal->shape().element_type(), + primitive_util::NativeToPrimitiveType()); + if (no_duplicates) { + PopulateWithNoDuplicateData(literal, engine); + } else { + PopulateWithRandomFloatingPointData(literal, engine); + } +} + +template <> +void PopulateWithFloatingPointData(Literal* literal, + std::minstd_rand0* engine, + bool no_duplicates) { + CHECK(engine != nullptr); + CHECK_EQ(literal->shape().element_type(), + primitive_util::NativeToPrimitiveType()); + if (no_duplicates) { + PopulateWithNoDuplicateData(literal, engine); + } else { + PopulateWithRandomFloatingPointData(literal, engine); + } +} + +template <> +void PopulateWithFloatingPointData(Literal* literal, + std::minstd_rand0* engine, + bool no_duplicates) { + CHECK(engine != nullptr); + CHECK_EQ(literal->shape().element_type(), + primitive_util::NativeToPrimitiveType()); + if (no_duplicates) { + PopulateWithNoDuplicateData(literal, engine); + } else { + PopulateWithRandomFloatingPointData(literal, engine); } } @@ -135,20 +184,16 @@ StatusOr MakeFakeLiteralInternal(const Shape& shape, Literal literal(shape); switch (shape.element_type()) { case BF16: - PopulateWithRandomFloatingPointData(&literal, engine, - no_duplicates); + PopulateWithFloatingPointData(&literal, engine, no_duplicates); break; case F16: - PopulateWithRandomFloatingPointData(&literal, engine, - no_duplicates); + PopulateWithFloatingPointData(&literal, engine, no_duplicates); break; case F32: - PopulateWithRandomFloatingPointData(&literal, engine, - no_duplicates); + PopulateWithFloatingPointData(&literal, engine, no_duplicates); break; case F64: - PopulateWithRandomFloatingPointData(&literal, engine, - no_duplicates); + PopulateWithFloatingPointData(&literal, engine, no_duplicates); break; case S8: PopulateWithRandomIntegralData(&literal, engine, no_duplicates); diff --git a/tensorflow/compiler/xla/tests/test_utils_test.cc b/tensorflow/compiler/xla/tests/test_utils_test.cc index e066b3f4f22..e8f5d7a9a79 100644 --- a/tensorflow/compiler/xla/tests/test_utils_test.cc +++ b/tensorflow/compiler/xla/tests/test_utils_test.cc @@ -175,5 +175,28 @@ ENTRY %sort.148.1589 (parameter.0: s32[1048576], parameter.1: s32[1048576]) -> ( } } +XLA_TEST_F(TestUtilsTest, NoDuplicatesBfloat16) { + // Inputs which are sort keys in key/value sorts should have no duplicates. + auto module = ParseHloString(R"( +HloModule sort, is_scheduled=true + +ENTRY %sort. (parameter.0: bf16[2,1452], parameter.1: s32[2,1452]) -> (bf16[2,1452], s32[2,1452]) { + %parameter.0 = bf16[2,1452]{1,0} parameter(0) + %parameter.1 = s32[2,1452]{1,0} parameter(1) + ROOT %sort = (bf16[2,1452]{1,0}, s32[2,1452]{1,0}) sort(bf16[2,1452]{1,0} %parameter.0, s32[2,1452]{1,0} %parameter.1), dimensions={1} +} +)") + .ValueOrDie(); + TF_ASSERT_OK_AND_ASSIGN(std::vector args, + MakeFakeArguments(module.get())); + ASSERT_EQ(args.size(), 2); + const Literal& key_arg = args[0]; + + absl::flat_hash_set key_set; + for (const bfloat16& value : key_arg.data()) { + EXPECT_TRUE(key_set.insert(absl::bit_cast(value)).second); + } +} + } // namespace } // namespace xla diff --git a/tensorflow/compiler/xla/tests/token_hlo_test.cc b/tensorflow/compiler/xla/tests/token_hlo_test.cc index a2b7c26331b..601c6b06938 100644 --- a/tensorflow/compiler/xla/tests/token_hlo_test.cc +++ b/tensorflow/compiler/xla/tests/token_hlo_test.cc @@ -16,6 +16,7 @@ limitations under the License. #include #include "absl/strings/str_cat.h" +#include "tensorflow/compiler/xla/service/hlo_parser.h" #include "tensorflow/compiler/xla/service/hlo_verifier.h" #include "tensorflow/compiler/xla/tests/hlo_test_base.h" #include "tensorflow/compiler/xla/tests/test_macros.h" @@ -108,26 +109,6 @@ XLA_TEST_F(TokenHloTest, InvalidTupleTokenShapedEntryParameter) { ::testing::HasSubstr("Entry parameter 0 is or contains a token shape")); } -XLA_TEST_F(TokenHloTest, InvalidOperandToTokenInstruction) { - std::unique_ptr module = CreateNewUnverifiedModule(); - auto builder = HloComputation::Builder(TestName()); - auto param = builder.AddInstruction( - HloInstruction::CreateParameter(0, ShapeUtil::MakeShape(F32, {}), "p0")); - builder.AddInstruction(HloInstruction::CreateAfterAll({param})); - builder.AddInstruction( - HloInstruction::CreateConstant(LiteralUtil::CreateR0(123))); - module->AddEntryComputation(builder.Build()); - - Status status = - HloVerifier(/*layout_sensitive=*/false, /*allow_mixed_precision=*/false) - .Run(module.get()) - .status(); - ASSERT_IS_NOT_OK(status); - EXPECT_THAT(status.error_message(), - ::testing::HasSubstr( - "Operands of token instructions must be TOKEN types")); -} - XLA_TEST_F(TokenHloTest, TokenInWhileLoop) { // Thread a token around a while loop. Token is created and consumed by a // AfterAll instruction in the while body. @@ -220,5 +201,95 @@ ENTRY %TokenInConditional (param.3: pred[]) -> s32[] { } } +XLA_TEST_F(TokenHloTest, AddDependency) { + string module_string = R"( +HloModule AddDependency, is_scheduled=true + +// Computes (p0 + 42) * (-p1) +// where there is a dependency from the add to the negation using a token +// with after-all and add-dependency instructions. +ENTRY %AddDependency (p0: f32[], p1: f32[]) -> f32[] { + %p0 = f32[] parameter(0) + %p1 = f32[] parameter(1) + + %forty_two = f32[] constant(42.0) + %add = f32[] add(f32[] %p0, f32[] %forty_two) + %token = token[] after-all(f32[] %add) + %p1_after_token = f32[] add-dependency(f32[] %p1, token[] %token) + %neg = f32[] negate(f32[] %p1_after_token) + ROOT %product = f32[] multiply(f32[] %add, f32[] %neg) +} +)"; + TF_ASSERT_OK_AND_ASSIGN( + std::unique_ptr module, + ParseHloString(module_string, GetModuleConfigForTest())); + auto p0 = LiteralUtil::CreateR0(10.0); + auto p1 = LiteralUtil::CreateR0(3.0); + auto expected = LiteralUtil::CreateR0(-156.0); + EXPECT_EQ(expected, ExecuteNoHloPasses(std::move(module), {&p0, &p1})); +} + +XLA_TEST_F(TokenHloTest, AddDependencyOfConstant) { + string module_string = R"( +HloModule AddDependencyOfConstant, is_scheduled=true + +ENTRY %AddDependency (p0: f32[]) -> f32[] { + %p0 = f32[] parameter(0) + %forty_two = f32[] constant(42.0) + %token = token[] after-all(f32[] %p0) + %forty_two_after_token = f32[] add-dependency(f32[] %forty_two, token[] %token) + ROOT %product = f32[] multiply(f32[] %p0, f32[] %forty_two_after_token) +} +)"; + TF_ASSERT_OK_AND_ASSIGN( + std::unique_ptr module, + ParseHloString(module_string, GetModuleConfigForTest())); + auto p0 = LiteralUtil::CreateR0(10.0); + auto expected = LiteralUtil::CreateR0(420.0); + EXPECT_EQ(expected, ExecuteNoHloPasses(std::move(module), {&p0})); +} + +XLA_TEST_F(TokenHloTest, AddDependencyAsRoot) { + string module_string = R"( +HloModule AddDependencyAsRoot, is_scheduled=true +ENTRY %AddDependency (p: f32[3]) -> f32[3] { + %p = f32[3] parameter(0) + %neg = f32[3] negate(f32[3] %p) + %token = token[] after-all() + ROOT %add_dep = f32[3] add-dependency(f32[3] %neg, token[] %token) +} +)"; + TF_ASSERT_OK_AND_ASSIGN( + std::unique_ptr module, + ParseHloString(module_string, GetModuleConfigForTest())); + auto input = LiteralUtil::CreateR1({1.0, 3.0, 7.0}); + auto expected = LiteralUtil::CreateR1({-1.0, -3.0, -7.0}); + EXPECT_EQ(expected, ExecuteNoHloPasses(std::move(module), {&input})); +} + +XLA_TEST_F(TokenHloTest, TupleShapedAddDependency) { + string module_string = R"( +HloModule TupleShapedAddDependency, is_scheduled=true +ENTRY %TupleShapedAddDependency (p0: f32[3], p1: f32[3]) -> f32[3] { + %p0 = f32[3] parameter(0) + %p1 = f32[3] parameter(1) + %forty_two = f32[] constant(42.0) + %token = token[] after-all() + %tuple = (f32[3], token[], f32[3], f32[]) tuple(f32[3] %p0, token[] %token, f32[3] %p1, f32[] %forty_two) + %add_dep = (f32[3], token[], f32[3], f32[]) add-dependency((f32[3], token[], f32[3], f32[]) %tuple, token[] %token) + %elem0 = f32[3] get-tuple-element((f32[3], token[], f32[3], f32[]) %add_dep), index=0 + %elem2 = f32[3] get-tuple-element((f32[3], token[], f32[3], f32[]) %add_dep), index=2 + ROOT %diff = f32[3] subtract(f32[3] %elem0, f32[3] %elem2) +} +)"; + TF_ASSERT_OK_AND_ASSIGN( + std::unique_ptr module, + ParseHloString(module_string, GetModuleConfigForTest())); + auto p0 = LiteralUtil::CreateR1({3.0, 3.0, 47.0}); + auto p1 = LiteralUtil::CreateR1({1.0, -2.0, 2.0}); + auto expected = LiteralUtil::CreateR1({2.0, 5.0, 45.0}); + EXPECT_EQ(expected, ExecuteNoHloPasses(std::move(module), {&p0, &p1})); +} + } // namespace } // namespace xla diff --git a/tensorflow/compiler/xla/tests/xla_hlo_profile_test.cc b/tensorflow/compiler/xla/tests/xla_hlo_profile_test.cc index ca036f1ae0d..e57d072a063 100644 --- a/tensorflow/compiler/xla/tests/xla_hlo_profile_test.cc +++ b/tensorflow/compiler/xla/tests/xla_hlo_profile_test.cc @@ -157,10 +157,12 @@ void ExecuteAndFetchProfile(string* profile_output, LocalClient* client, TF_ASSERT_OK(transfer_manager->TransferLiteralToDevice( stream_ptr.get(), Literal::CreateFromShape(rhs_arg_shape), rhs_arg)); + ExecutableBuildOptions build_options; + build_options.mutable_debug_options()->set_xla_hlo_profile(true); TF_ASSERT_OK_AND_ASSIGN( std::unique_ptr local_executable, client->Compile(computation, {&lhs_arg_shape, &rhs_arg_shape}, - ExecutableBuildOptions().set_hlo_profile(true))); + build_options)); Executable* executable = local_executable->executable(); HloExecutionProfile hlo_execution_profile( @@ -208,7 +210,7 @@ XLA_TEST_F(HloProfileTest, ProfileSingleComputation) { string profile_output; ExecuteAndFetchProfile(&profile_output, client, computation, lhs_shape, rhs_shape); - + VLOG(4) << "Profile Output:\n" << profile_output; std::vector profile_output_lines = absl::StrSplit(profile_output, '\n'); diff --git a/tensorflow/compiler/xla/tools/replay_computation.cc b/tensorflow/compiler/xla/tools/replay_computation.cc index 47be9f5adf1..ff2c3399928 100644 --- a/tensorflow/compiler/xla/tools/replay_computation.cc +++ b/tensorflow/compiler/xla/tools/replay_computation.cc @@ -82,13 +82,17 @@ struct Options { std::unique_ptr CompileExecutable(const HloSnapshot& module, LocalClient* client) { XlaComputation computation(module.hlo().hlo_module()); - std::vector argument_layouts; - for (const auto& param : + std::vector argument_layouts; + argument_layouts.reserve( + computation.proto().host_program_shape().parameters_size()); + std::vector argument_layout_ptrs; + for (const ShapeProto& param : computation.proto().host_program_shape().parameters()) { - argument_layouts.push_back(¶m); + argument_layouts.push_back(Shape(param)); + argument_layout_ptrs.push_back(&argument_layouts.back()); } return client - ->Compile(computation, argument_layouts, ExecutableBuildOptions()) + ->Compile(computation, argument_layout_ptrs, ExecutableBuildOptions()) .ValueOrDie(); } @@ -149,7 +153,7 @@ StatusOr ReplayComputation(const HloSnapshot& module, << "--generate_fake_infeed only works if the model has 0 or 1 " "infeed ops, but this one has >= 2."; provide_infeed = true; - infeed_shape = instruction.shape(); + infeed_shape = Shape(instruction.shape()); LOG(INFO) << "Generating fake infeed shape for inferred shape: " << ShapeUtil::HumanString(infeed_shape); } @@ -315,9 +319,10 @@ int RealMain(absl::Span args, const Options& opts) { if (snapshot.has_result()) { Literal literal = Literal::CreateFromProto(snapshot.result()).ConsumeValueOrDie(); - fprintf(stdout, "was %s:%s\n", - ShapeUtil::HumanString(snapshot.result().shape()).c_str(), - literal.ToString().c_str()); + fprintf( + stdout, "was %s:%s\n", + ShapeUtil::HumanString(Shape(snapshot.result().shape())).c_str(), + literal.ToString().c_str()); } } } diff --git a/tensorflow/compiler/xla/util.h b/tensorflow/compiler/xla/util.h index 8ce74164741..6722641e9d2 100644 --- a/tensorflow/compiler/xla/util.h +++ b/tensorflow/compiler/xla/util.h @@ -152,6 +152,13 @@ static inline absl::Span AsInt64Slice( slice.size()); } +// TODO(b/29771030): This nop overload was added to simplify the migration of +// Shape from a proto to a C++ class. Remove after class has been migrated. +static inline absl::Span AsInt64Slice( + absl::Span slice) { + return slice; +} + // As above, but for uint64 types. static inline absl::Span AsUInt64Slice( const tensorflow::protobuf::RepeatedField& v) { @@ -387,6 +394,19 @@ T CeilOfRatio(T dividend, T divisor) { return tensorflow::MathUtil::CeilOfRatio(dividend, divisor); } +template +std::vector ElementWiseCeilOfRatio(absl::Span dividends, + absl::Span divisors) { + std::vector ceil_of_ratios; + CHECK_EQ(dividends.size(), divisors.size()); + ceil_of_ratios.reserve(dividends.size()); + absl::c_transform(dividends, divisors, std::back_inserter(ceil_of_ratios), + [](const T dividend, const T divisor) { + return CeilOfRatio(dividend, divisor); + }); + return ceil_of_ratios; +} + // Rounds the value up to a multiple of the divisor by first calling CeilOfRatio // then multiplying by the divisor. For example: RoundUpToNearest(13, 8) => 16 template diff --git a/tensorflow/compiler/xla/window_util.cc b/tensorflow/compiler/xla/window_util.cc index 8ea8dbab257..f113a705b41 100644 --- a/tensorflow/compiler/xla/window_util.cc +++ b/tensorflow/compiler/xla/window_util.cc @@ -185,6 +185,17 @@ bool HasWindowReversal(const Window& window) { return false; } +bool AllOrNoneReversed(const Window& window) { + if (window.dimensions().size() == 0) { + return true; + } + bool reversed = window.dimensions()[0].window_reversal(); + return std::all_of(window.dimensions().begin(), window.dimensions().end(), + [&](const WindowDimension& dim) { + return dim.window_reversal() == reversed; + }); +} + bool HasDilation(const Window& window) { return HasBaseDilation(window) || HasWindowDilation(window); } diff --git a/tensorflow/compiler/xla/window_util.h b/tensorflow/compiler/xla/window_util.h index 1fb9e855fc1..099d7ecdd5c 100644 --- a/tensorflow/compiler/xla/window_util.h +++ b/tensorflow/compiler/xla/window_util.h @@ -56,6 +56,7 @@ bool HasWindowDilation(const Window& window); bool HasDilation(const Window& window); bool HasWindowReversal(const Window& window); +bool AllOrNoneReversed(const Window& window); // Returns true if the given logical dimension is inactive in the sense that it // has window bound 1, no striding and no padding. diff --git a/tensorflow/compiler/xla/xla.proto b/tensorflow/compiler/xla/xla.proto index 28df3b03f39..bdeb1728fa2 100644 --- a/tensorflow/compiler/xla/xla.proto +++ b/tensorflow/compiler/xla/xla.proto @@ -193,7 +193,11 @@ message DebugOptions { // - Assuming that operations never produce or consume NaN or +/- Inf. // - Assuming that +0 and -0 are indistinguishable. bool xla_cpu_enable_fast_math = 99; - bool xla_gpu_enable_fast_math = 100; + + // When true we lower the Minimum and Maximum hlos in the GPU backend such + // that Min(NotNaN, NaN) = Min(NaN, NotNaN) = NotNaN. In other words, if flag + // this is true we don't propagate NaNs through Min and Max. + bool xla_gpu_enable_fast_min_max = 100; // Crashes the program when any kind of verification fails, instead of just // logging the failures. One example is cross checking of convolution results @@ -224,7 +228,7 @@ message ExecutionOptions { // may be faster when using this layout. // // We use a Shape here to accommodate computations that return a tuple. - Shape shape_with_output_layout = 2; + ShapeProto shape_with_output_layout = 2; // Used to seed random-number generators used in this computation. If this is // 0, we generate a seed ourselves. @@ -253,7 +257,7 @@ message TransferToClientRequest { // This optional field directs the service to return the literal in this // layout. A shape is used to hold the layout to accommodate tuples. - Shape shape_with_layout = 2; + ShapeProto shape_with_layout = 2; } message TransferToClientResponse { @@ -281,7 +285,7 @@ message TransferToInfeedResponse { message TransferFromOutfeedRequest { // This optional field directs the service to return the literal in this // layout. A shape is used to hold the layout to accommodate tuples. - Shape shape_with_layout = 1; + ShapeProto shape_with_layout = 1; int64 replica_id = 2; DeviceHandle device_handle = 3; @@ -332,7 +336,7 @@ message CompileRequest { // The layouts of the input arguments. If not set, the default layout will be // used. Although the real arguments are not needed in compilation, the // layouts of the arguments can affect the compilation. - repeated Shape input_shape_with_layout = 3; + repeated ShapeProto input_shape_with_layout = 3; } message CompileResponse { @@ -406,7 +410,7 @@ message LoadDataRequest { string columnio_field = 2; // Individual element shape, excluding rows. - Shape element_shape = 3; + ShapeProto element_shape = 3; // Warning: ColumnIO does not support random-access, so use offset with // caution in performance-critical scenarios. @@ -422,7 +426,7 @@ message LoadDataRequest { message LoadDataResponse { GlobalDataHandle data = 1; - Shape data_shape = 2; + ShapeProto data_shape = 2; int64 available_rows = 3; int64 rows_loaded = 4; int64 nanoseconds = 5; @@ -433,7 +437,7 @@ message GetShapeRequest { } message GetShapeResponse { - Shape shape = 1; + ShapeProto shape = 1; } message UnpackRequest { diff --git a/tensorflow/compiler/xla/xla_data.proto b/tensorflow/compiler/xla/xla_data.proto index 683ccc40f16..85ec83437a1 100644 --- a/tensorflow/compiler/xla/xla_data.proto +++ b/tensorflow/compiler/xla/xla_data.proto @@ -108,6 +108,16 @@ enum Format { SPARSE = 2; } +// Describes a tile used in tiling-based layout. Refer to +// g3doc/layout_with_tiling.md for details about tiling-based layout. +message Tile { + // Number of elements in each dimension of the tile. It's ordered from the + // most major dimension of the tile to the most minor dimension of the tile. + // The dimensions correspond to a suffix of the dimensions of the shape being + // tiled. + repeated int64 dimensions = 1; +} + // A layout describes how the array is placed in (1D) memory space. This // includes the minor-to-major ordering of dimensions within a shape. // @@ -138,6 +148,20 @@ message Layout { // memory. This field must be unset unless the format is SPARSE. int64 max_sparse_elements = 5; + // A sequence of tiles, starting from the tile that's applied first to the + // Shape. + // + // TODO(b/119839262): implement tiling in each backend or add Unimplemented + // error. + repeated Tile tiles = 6; + + // Bit size of each element. If the size is bigger than what the element + // type requires, the value is stored in the least significant + // bits and the additional most significant bits are filled with 0's. + // + // TODO(b/119839262): implement in each backend or add Unimplemented error. + int64 element_size_in_bits = 7; + // Important: if any field is added, be sure to modify ShapeUtil::Equal() and // LayoutUtil::Hash appropriately to account for the new field. } @@ -154,7 +178,7 @@ message Layout { // See the XLA documentation for more information on shapes and layouts. // // LINT.IfChange -message Shape { +message ShapeProto { reserved 1; reserved "rank"; @@ -169,7 +193,7 @@ message Shape { repeated int64 dimensions = 3; // For tuples only, the shapes of constitutent shapes in the tuple sequence. - repeated Shape tuple_shapes = 4; + repeated ShapeProto tuple_shapes = 4; // The layout used to back this shape. Layout layout = 5; @@ -183,9 +207,9 @@ message Shape { // Shape of the parameters and output of a computation (like a traditional // function signature). -message ProgramShape { - repeated Shape parameters = 1; - Shape result = 2; +message ProgramShapeProto { + repeated ShapeProto parameters = 1; + ShapeProto result = 2; repeated string parameter_names = 3; } @@ -320,7 +344,7 @@ message DeviceAssignmentProto { // Transfers to/from the client are encoded in literal form, and the structure // of the repeated fields is implied by the shape. message LiteralProto { - Shape shape = 1; + ShapeProto shape = 1; repeated bool preds = 2; bytes s8s = 15; bytes u8s = 3; @@ -521,7 +545,7 @@ message OpSharding { } Type type = 1; // The shape of the sharded tile. - Shape tile_shape = 2; + ShapeProto tile_shape = 2; // The shape of the tile assignment tensor - this must be the same rank as // tile_shape and the product of its dimensions must equal // tile_assignment_devices.size(). diff --git a/tensorflow/compiler/xrt/BUILD b/tensorflow/compiler/xrt/BUILD index 2ff97914f86..2dae746d034 100644 --- a/tensorflow/compiler/xrt/BUILD +++ b/tensorflow/compiler/xrt/BUILD @@ -22,6 +22,7 @@ xla_proto_library( deps = [ "//tensorflow/compiler/tf2xla:host_compute_metadata_proto", "//tensorflow/compiler/xla:xla_data_proto", + "//tensorflow/compiler/xla:xla_proto", "//tensorflow/compiler/xla/service:hlo_proto", ], ) @@ -32,20 +33,25 @@ cc_library( "xrt_compilation_cache.cc", "xrt_device.cc", "xrt_state.cc", + "xrt_util.cc", ], hdrs = [ "xrt_compilation_cache.h", "xrt_device.h", "xrt_state.h", + "xrt_util.h", ], deps = [ "//tensorflow/compiler/jit:xla_device", "//tensorflow/compiler/tf2xla:xla_compiler", + "//tensorflow/compiler/xla:debug_options_flags", "//tensorflow/compiler/xla:literal", "//tensorflow/compiler/xla:shape_util", "//tensorflow/compiler/xla:status_macros", "//tensorflow/compiler/xla:statusor", + "//tensorflow/compiler/xla:types", "//tensorflow/compiler/xla:xla_data_proto", + "//tensorflow/compiler/xla:xla_proto", "//tensorflow/compiler/xla/client:local_client", "//tensorflow/compiler/xla/service:backend", "//tensorflow/compiler/xla/service:device_memory_allocator", diff --git a/tensorflow/compiler/xrt/kernels/xrt_compile_ops.cc b/tensorflow/compiler/xrt/kernels/xrt_compile_ops.cc index dc62cf7a6b2..2ccdf0f02d8 100644 --- a/tensorflow/compiler/xrt/kernels/xrt_compile_ops.cc +++ b/tensorflow/compiler/xrt/kernels/xrt_compile_ops.cc @@ -33,6 +33,7 @@ limitations under the License. #include "tensorflow/compiler/xrt/xrt.pb.h" #include "tensorflow/compiler/xrt/xrt_compilation_cache.h" #include "tensorflow/compiler/xrt/xrt_device.h" +#include "tensorflow/compiler/xrt/xrt_util.h" #include "tensorflow/core/framework/op_kernel.h" #include "tensorflow/core/framework/resource_mgr.h" #include "tensorflow/core/framework/tensor.h" @@ -108,19 +109,26 @@ Status XRTCompileOp::Compile(OpKernelContext* ctx, TF_ASSIGN_OR_RETURN(xla::XlaComputation computation, client->LoadSnapshot(computation_proto.hlo_snapshot())); - std::vector argument_layouts( + std::vector argument_layouts( + config.program_shape().parameters_size()); + std::vector argument_layout_ptrs( config.program_shape().parameters_size()); for (int i = 0; i < config.program_shape().parameters_size(); ++i) { - argument_layouts[i] = &config.program_shape().parameters(i); + argument_layouts[i] = xla::Shape(config.program_shape().parameters(i)); + argument_layout_ptrs[i] = &argument_layouts[i]; } xla::ExecutableBuildOptions build_options; build_options.set_device_ordinal(client->default_device_ordinal()); - build_options.set_result_layout(config.program_shape().result()); + build_options.set_result_layout(xla::Shape(config.program_shape().result())); build_options.set_device_allocator(device_ref.backend()->memory_allocator()); + if (config.has_debug_options()) { + *build_options.mutable_debug_options() = + BuildXlaDebugOptions(config.debug_options()); + } VLOG(1) << "Building executable"; auto compile_result = - client->Compile(computation, argument_layouts, build_options); + client->Compile(computation, argument_layout_ptrs, build_options); if (!compile_result.ok()) { return compile_result.status(); } @@ -174,11 +182,12 @@ void XRTCompileOp::Compute(OpKernelContext* ctx) { ctx->set_output(0, handle_output); xla::LocalExecutable* executable = entry->get().get_executable(); - xla::ProgramShape program_shape = executable->executable() - ->module() - .config() - .entry_computation_layout() - .ComputeProgramShape(); + xla::ProgramShapeProto program_shape = executable->executable() + ->module() + .config() + .entry_computation_layout() + .ComputeProgramShape() + .ToProto(); Tensor program_shape_output(DT_STRING, TensorShape({1})); program_shape_output.vec()(0) = program_shape.SerializeAsString(); ctx->set_output(1, program_shape_output); diff --git a/tensorflow/compiler/xrt/kernels/xrt_execute_op.cc b/tensorflow/compiler/xrt/kernels/xrt_execute_op.cc index 8c6191ddc06..751329eefc3 100644 --- a/tensorflow/compiler/xrt/kernels/xrt_execute_op.cc +++ b/tensorflow/compiler/xrt/kernels/xrt_execute_op.cc @@ -228,14 +228,35 @@ Status XRTExecuteOp::DoWork(OpKernelContext* context) { TF_RETURN_IF_ERROR(XRTTupleAllocation::CreateFromBuffer( shaped_buffer, device_ref.backend(), device_ref.device_ordinal(), &output_tuple)); + if (config_proto.return_exploded_tuple() && + xla::ShapeUtil::IsTuple(output_tuple->on_device_shape())) { + int64 tuple_element_count = + xla::ShapeUtil::TupleElementCount(output_tuple->on_device_shape()); + Tensor* output_tensor; + TF_RETURN_IF_ERROR(context->allocate_output( + 0, TensorShape({tuple_element_count}), &output_tensor)); - Tensor* output_tensor; - TF_RETURN_IF_ERROR( - context->allocate_output(0, TensorShape({}), &output_tensor)); - int64 key; - TF_RETURN_IF_ERROR(output_tuple->Intern(rm, &key)); - output_tensor->scalar()() = key; + for (int64 i = 0; i < tuple_element_count; ++i) { + xla::ShapeIndex shape_index; + shape_index.push_back(i); + XRTTupleAllocation* suballocation; + TF_RETURN_IF_ERROR(XRTTupleAllocation::MakeSubBuffer( + output_tuple, shape_index, &suballocation, + /*alias_parent_allocation=*/false)); + int64 key; + TF_RETURN_IF_ERROR(suballocation->Intern(rm, &key)); + output_tensor->vec()(i) = key; + } + output_tuple->Unref(); + } else { + Tensor* output_tensor; + TF_RETURN_IF_ERROR( + context->allocate_output(0, TensorShape({}), &output_tensor)); + int64 key; + TF_RETURN_IF_ERROR(output_tuple->Intern(rm, &key)); + output_tensor->scalar()() = key; + } return Status::OK(); } diff --git a/tensorflow/compiler/xrt/kernels/xrt_state_ops.cc b/tensorflow/compiler/xrt/kernels/xrt_state_ops.cc index ffea592491d..3258286c106 100644 --- a/tensorflow/compiler/xrt/kernels/xrt_state_ops.cc +++ b/tensorflow/compiler/xrt/kernels/xrt_state_ops.cc @@ -87,6 +87,19 @@ REGISTER_KERNEL_BUILDER(Name("XRTReadLiteral") .HostMemory("literal"), XRTReadLiteralOp); +REGISTER_KERNEL_BUILDER(Name("XRTWriteLiteral") + .Device(DEVICE_XLA_GPU) + .HostMemory("handle") + .HostMemory("literal") + .HostMemory("output_handle"), + XRTWriteLiteralOp); +REGISTER_KERNEL_BUILDER(Name("XRTWriteLiteral") + .Device(DEVICE_XLA_CPU) + .HostMemory("handle") + .HostMemory("literal") + .HostMemory("output_handle"), + XRTWriteLiteralOp); + REGISTER_KERNEL_BUILDER(Name("XRTReadLiteralAndRelease") .Device(DEVICE_XLA_GPU) .HostMemory("handle") diff --git a/tensorflow/compiler/xrt/kernels/xrt_state_ops.h b/tensorflow/compiler/xrt/kernels/xrt_state_ops.h index 54b06558adc..26a58fa42d8 100644 --- a/tensorflow/compiler/xrt/kernels/xrt_state_ops.h +++ b/tensorflow/compiler/xrt/kernels/xrt_state_ops.h @@ -393,6 +393,56 @@ class XRTReadLiteralOp : public OpKernel { } }; +// Op that writes a new literal value into device-resident memory. +template +class XRTWriteLiteralOp : public OpKernel { + public: + explicit XRTWriteLiteralOp(OpKernelConstruction* ctx) : OpKernel(ctx) {} + ~XRTWriteLiteralOp() override = default; + XRTWriteLiteralOp(const XRTWriteLiteralOp&) = delete; + XRTWriteLiteralOp& operator=(const XRTWriteLiteralOp&) = delete; + + void Compute(OpKernelContext* ctx) override { + VLOG(1) << "XRTWriteLiteralOp::Compute"; + + const Tensor& handle_tensor = ctx->input(0); + OP_REQUIRES( + ctx, TensorShapeUtils::IsScalar(handle_tensor.shape()), + errors::Internal("computation input should be an int64 scalar")); + int64 allocation_handle = handle_tensor.scalar()(); + + const Tensor& literal_info = ctx->input(1); + OP_REQUIRES(ctx, TensorShapeUtils::IsScalar(literal_info.shape()), + errors::Internal("literal input should be a string scalar")); + xla::LiteralProto literal_proto; + OP_REQUIRES(ctx, + literal_proto.ParseFromString(literal_info.scalar()()), + errors::InvalidArgument( + "Unable to parse allocation input to LiteralProto")); + xla::Literal literal; + OP_REQUIRES_OK(ctx, XRTStateHelpers::MakeLiteral(literal_proto, &literal)); + + ResourceMgr* rm; + OP_REQUIRES_OK(ctx, DeviceAccessor::GetResourceManager(ctx, &rm)); + + XRTTupleAllocation* allocation; + OP_REQUIRES_OK( + ctx, XRTTupleAllocation::Lookup(rm, allocation_handle, &allocation)); + core::ScopedUnref allocation_unref(allocation); + // We are guaranteed that the underlying device object won't be deleted out + // from under us, while the ScopedRef is live. + typename DeviceAccessor::ScopedRef device_ref; + OP_REQUIRES_OK(ctx, DeviceAccessor::InitScopedRef( + ctx, allocation->device_ordinal(), &device_ref)); + OP_REQUIRES_OK(ctx, + allocation->WriteLiteral(device_ref.backend(), literal)); + + Tensor output(DT_INT64, TensorShape({})); + output.scalar()() = allocation_handle; + ctx->set_output(0, output); + } +}; + // Op that discards a handle to device memory. template class XRTReleaseAllocationOp : public OpKernel { diff --git a/tensorflow/compiler/xrt/ops/xrt_state_ops.cc b/tensorflow/compiler/xrt/ops/xrt_state_ops.cc index 07d025ce343..a3d63106fa1 100644 --- a/tensorflow/compiler/xrt/ops/xrt_state_ops.cc +++ b/tensorflow/compiler/xrt/ops/xrt_state_ops.cc @@ -95,6 +95,20 @@ Copies an allocated tuple from device memory and returns it as a literal. 'literal' is a serialized xla::LiteralProto proto. )"); +REGISTER_OP("XRTWriteLiteral") + .Input("handle: int64") + .Input("literal: string") + .Output("output_handle: int64") + .SetShapeFn(tensorflow::shape_inference::ScalarShape) + .Doc( + R"( +Copies the input literal into the device memory pointed to by handle. +Returns the handle itself. + +'handle' is the id returned from the Op that produced the on-device allocation. +'literal' is a serialized xla::LiteralProto proto to be written to device memory. +)"); + REGISTER_OP("XRTReadLiteralAndRelease") .Input("handle: int64") .Output("literal: string") diff --git a/tensorflow/compiler/xrt/tests/raw_api_test.cc b/tensorflow/compiler/xrt/tests/raw_api_test.cc index 25464b5554d..abaa17e50e3 100644 --- a/tensorflow/compiler/xrt/tests/raw_api_test.cc +++ b/tensorflow/compiler/xrt/tests/raw_api_test.cc @@ -102,7 +102,7 @@ bool CompareLiteralProtos(const xla::LiteralProto& a, auto l_b = xla::Literal::CreateFromProto(b).ValueOrDie(); bool equal = l_a == l_b; if (!equal) { - LOG(INFO) << "LiteralProtos don't match " << a.DebugString() + LOG(INFO) << "LiteralProtos don't match: " << a.DebugString() << " != " << b.DebugString(); } return equal; @@ -175,6 +175,18 @@ xla::XlaComputation AddAndTuple() { return builder.Build().ValueOrDie(); } +xla::XlaComputation AddAndSubTuple() { + xla::XlaBuilder builder("AddAndSubTuple"); + auto p0 = xla::Parameter(&builder, 0, xla::ShapeUtil::MakeShape(xla::F32, {}), + "P0"); + auto p1 = xla::Parameter(&builder, 1, xla::ShapeUtil::MakeShape(xla::F32, {}), + "P1"); + auto sum = xla::Add(p0, p1); + auto sub = xla::Sub(p0, p1); + xla::Tuple(&builder, {sum, sub}); + return builder.Build().ValueOrDie(); +} + void StoreComputationSnapshot(const xla::XlaComputation& computation, xla::HloSnapshot* dst) { auto snapshot = computation.Snapshot().ValueOrDie(); @@ -203,6 +215,56 @@ xla::ProgramShape XlaCompiledProgramShape( ->ComputeProgramShape(); } +TEST(RawApiTest, AllocAndRewrite) { + xrt::XLAAllocation alloc; + alloc.set_device_ordinal(0); + *alloc.mutable_value() = + xla::LiteralUtil::CreateR2({{4, 5}, {6, 7}}).ToProto(); + + Scope root = Scope::NewRootScope().WithDevice(DeviceFromFlag()); + auto value = + ops::Const(root.WithDevice("/device:CPU:0"), alloc.SerializeAsString()); + auto handle = ops::XRTAllocate(root, value); + auto read_back = ops::XRTReadLiteral(root, handle); + TF_ASSERT_OK(root.status()); + + tensorflow::ClientSession session(root); + std::vector outputs; + TF_EXPECT_OK(session.Run({read_back, handle}, &outputs)); + EXPECT_EQ(outputs.size(), 2); + + int64 allocation_handle = outputs[1].scalar()(); + xla::LiteralProto response; + EXPECT_TRUE(response.ParseFromString(outputs[0].scalar()())); + EXPECT_TRUE(CompareLiteralProtos(alloc.value(), response)); + outputs.clear(); + + xla::LiteralProto new_literal = + xla::LiteralUtil::CreateR2({{9, 2}, {4, 1}}).ToProto(); + auto new_value = ops::Const(root.WithDevice("/device:CPU:0"), + new_literal.SerializeAsString()); + auto write_op = + ops::XRTWriteLiteral(root, Input(allocation_handle), new_value); + TF_ASSERT_OK(root.status()); + TF_EXPECT_OK(session.Run({write_op}, &outputs)); + EXPECT_EQ(outputs.size(), 1); + EXPECT_EQ(allocation_handle, outputs[0].scalar()()); + outputs.clear(); + + auto read_after_write = ops::XRTReadLiteral(root, Input(allocation_handle)); + TF_EXPECT_OK(session.Run({read_after_write}, &outputs)); + EXPECT_EQ(outputs.size(), 1); + + xla::LiteralProto new_response; + EXPECT_TRUE(new_response.ParseFromString(outputs[0].scalar()())); + EXPECT_TRUE(CompareLiteralProtos(new_literal, new_response)); + + auto release = + ops::XRTReleaseAllocationHandle(root, Input(allocation_handle)); + TF_EXPECT_OK(session.Run(tensorflow::ClientSession::FeedType(), {}, {release}, + &outputs)); +} + TEST(RawApiTest, ReadAndWriteState) { xrt::XLAAllocation alloc; alloc.set_device_ordinal(0); @@ -375,9 +437,12 @@ TEST(RawApiTest, CompileAndExecute) { xrt::XLAComputation c; auto config = c.mutable_config(); auto shapes = config->mutable_program_shape(); - *shapes->add_parameters() = xla::ShapeUtil::MakeShape(xla::F32, {2}); - *shapes->add_parameters() = xla::ShapeUtil::MakeShape(xla::F32, {2}); - *shapes->mutable_result() = xla::ShapeUtil::MakeShape(xla::F32, {2}); + *shapes->add_parameters() = + xla::ShapeUtil::MakeShape(xla::F32, {2}).ToProto(); + *shapes->add_parameters() = + xla::ShapeUtil::MakeShape(xla::F32, {2}).ToProto(); + *shapes->mutable_result() = + xla::ShapeUtil::MakeShape(xla::F32, {2}).ToProto(); StoreComputationSnapshot(AddAndScale(), c.mutable_hlo_snapshot()); xrt::XRTExecutionConfig e; @@ -411,7 +476,7 @@ TEST(RawApiTest, CompileAndExecute) { auto expected = xla::LiteralUtil::CreateR1({27.0f, 21.0f}); EXPECT_TRUE(CompareLiteralToLiteralProto(expected, response)); - xla::ProgramShape program_shape; + xla::ProgramShapeProto program_shape; EXPECT_TRUE(program_shape.ParseFromString(outputs[1].vec()(0))); EXPECT_EQ(program_shape.parameters_size(), 2); } @@ -427,9 +492,12 @@ TEST(RawApiTest, CompileAndExecuteWithArgumentVector) { xrt::XLAComputation c; auto config = c.mutable_config(); auto shapes = config->mutable_program_shape(); - *shapes->add_parameters() = xla::ShapeUtil::MakeShape(xla::F32, {2}); - *shapes->add_parameters() = xla::ShapeUtil::MakeShape(xla::F32, {2}); - *shapes->mutable_result() = xla::ShapeUtil::MakeShape(xla::F32, {2}); + *shapes->add_parameters() = + xla::ShapeUtil::MakeShape(xla::F32, {2}).ToProto(); + *shapes->add_parameters() = + xla::ShapeUtil::MakeShape(xla::F32, {2}).ToProto(); + *shapes->mutable_result() = + xla::ShapeUtil::MakeShape(xla::F32, {2}).ToProto(); StoreComputationSnapshot(AddAndScale(), c.mutable_hlo_snapshot()); xrt::XRTExecutionConfig e; @@ -465,7 +533,7 @@ TEST(RawApiTest, CompileAndExecuteWithArgumentVector) { auto expected = xla::LiteralUtil::CreateR1({27.0f, 21.0f}); EXPECT_TRUE(CompareLiteralToLiteralProto(expected, response)); - xla::ProgramShape program_shape; + xla::ProgramShapeProto program_shape; EXPECT_TRUE(program_shape.ParseFromString(outputs[1].vec()(0))); EXPECT_EQ(program_shape.parameters_size(), 2); } @@ -494,8 +562,8 @@ TEST(RawApiTest, CompileWithXlaReturnShapes) { xrt::XLAComputation c; auto config = c.mutable_config(); auto shapes = config->mutable_program_shape(); - *shapes->add_parameters() = param_shape; - *shapes->mutable_result() = result_shape; + *shapes->add_parameters() = param_shape.ToProto(); + *shapes->mutable_result() = result_shape.ToProto(); StoreComputationSnapshot(xla_computation, c.mutable_hlo_snapshot()); Scope root = Scope::NewRootScope().WithDevice(DeviceFromFlag()); @@ -510,8 +578,9 @@ TEST(RawApiTest, CompileWithXlaReturnShapes) { TF_EXPECT_OK(session.Run(tensorflow::ClientSession::FeedType(), {c_handle.program_shape}, {release}, &outputs)); - xla::ProgramShape program_shape; - EXPECT_TRUE(program_shape.ParseFromString(outputs[0].vec()(0))); + xla::ProgramShapeProto program_shape_proto; + EXPECT_TRUE(program_shape_proto.ParseFromString(outputs[0].vec()(0))); + xla::ProgramShape program_shape(program_shape_proto); EXPECT_EQ(program_shape.parameters_size(), 1); VLOG(2) << "Param: " @@ -520,7 +589,7 @@ TEST(RawApiTest, CompileWithXlaReturnShapes) { << xla::ShapeUtil::HumanStringWithLayout(program_shape.result()); xla::ProgramShape xla_program_shape = - XlaCompiledProgramShape(xla_computation, *shapes); + XlaCompiledProgramShape(xla_computation, xla::ProgramShape(*shapes)); EXPECT_TRUE(xla::LayoutUtil::Equal( xla::ShapeUtil::GetSubshape(program_shape.parameters(0), {0}).layout(), xla::ShapeUtil::GetSubshape(xla_program_shape.parameters(0), {0}) @@ -547,11 +616,11 @@ TEST(RawApiTest, DotGeneralWithLayoutTest) { auto config = c.mutable_config(); auto shapes = config->mutable_program_shape(); *shapes->add_parameters() = - xla::ShapeUtil::MakeShapeWithLayout(xla::F32, {2, 2}, {0, 1}); + xla::ShapeUtil::MakeShapeWithLayout(xla::F32, {2, 2}, {0, 1}).ToProto(); *shapes->add_parameters() = - xla::ShapeUtil::MakeShapeWithLayout(xla::F32, {2, 1}, {0, 1}); + xla::ShapeUtil::MakeShapeWithLayout(xla::F32, {2, 1}, {0, 1}).ToProto(); *shapes->mutable_result() = - xla::ShapeUtil::MakeShapeWithLayout(xla::F32, {2, 1}, {0, 1}); + xla::ShapeUtil::MakeShapeWithLayout(xla::F32, {2, 1}, {0, 1}).ToProto(); StoreComputationSnapshot(Dot(), c.mutable_hlo_snapshot()); xrt::XRTExecutionConfig e; @@ -592,7 +661,7 @@ TEST(RawApiTest, CompileAndExecuteZeroArg) { xrt::XLAComputation c; auto config = c.mutable_config(); auto shapes = config->mutable_program_shape(); - *shapes->mutable_result() = xla::ShapeUtil::MakeShape(xla::F32, {}); + *shapes->mutable_result() = xla::ShapeUtil::MakeShape(xla::F32, {}).ToProto(); xrt::XRTExecutionConfig e; e.set_release_input_handles(true); @@ -632,10 +701,13 @@ TEST(RawApiTest, CompileAndExecuteReturnTuple) { xrt::XLAComputation c; auto config = c.mutable_config(); auto shapes = config->mutable_program_shape(); - *shapes->add_parameters() = xla::ShapeUtil::MakeShape(xla::F32, {2}); - *shapes->add_parameters() = xla::ShapeUtil::MakeShape(xla::F32, {2}); - *shapes->mutable_result() = xla::ShapeUtil::MakeTupleShape( - {xla::ShapeUtil::MakeShape(xla::F32, {2})}); + *shapes->add_parameters() = + xla::ShapeUtil::MakeShape(xla::F32, {2}).ToProto(); + *shapes->add_parameters() = + xla::ShapeUtil::MakeShape(xla::F32, {2}).ToProto(); + *shapes->mutable_result() = + xla::ShapeUtil::MakeTupleShape({xla::ShapeUtil::MakeShape(xla::F32, {2})}) + .ToProto(); StoreComputationSnapshot(AddAndTuple(), c.mutable_hlo_snapshot()); xrt::XRTExecutionConfig e; @@ -671,14 +743,81 @@ TEST(RawApiTest, CompileAndExecuteReturnTuple) { EXPECT_TRUE(CompareLiteralToLiteralProto(expected, response)); } +TEST(RawApiTest, CompileAndExecuteReturnExplodedTuple) { + xrt::XLAAllocation p0; + p0.set_device_ordinal(0); + *p0.mutable_value() = xla::LiteralUtil::CreateR0(12.0f).ToProto(); + + xrt::XLAAllocation p1; + p1.set_device_ordinal(0); + *p1.mutable_value() = xla::LiteralUtil::CreateR0(3.0f).ToProto(); + + xrt::XLAComputation c; + auto config = c.mutable_config(); + auto shapes = config->mutable_program_shape(); + *shapes->add_parameters() = xla::ShapeUtil::MakeShape(xla::F32, {}).ToProto(); + *shapes->add_parameters() = xla::ShapeUtil::MakeShape(xla::F32, {}).ToProto(); + *shapes->mutable_result() = + xla::ShapeUtil::MakeTupleShape({xla::ShapeUtil::MakeShape(xla::F32, {}), + xla::ShapeUtil::MakeShape(xla::F32, {})}) + .ToProto(); + StoreComputationSnapshot(AddAndSubTuple(), c.mutable_hlo_snapshot()); + + xrt::XRTExecutionConfig e; + e.set_release_input_handles(true); + e.set_release_compilation_handle(true); + e.set_return_exploded_tuple(true); + + Scope root = Scope::NewRootScope().WithDevice(DeviceFromFlag()); + auto e_config = + ops::Const(root.WithDevice("/device:CPU:0"), e.SerializeAsString()); + auto computation = + ops::Const(root.WithDevice("/device:CPU:0"), c.SerializeAsString()); + auto c_handle = ops::XRTCompile(root, computation); + auto p0_value = + ops::Const(root.WithDevice("/device:CPU:0"), p0.SerializeAsString()); + auto p0_handle = ops::XRTAllocate(root, p0_value); + auto p1_value = + ops::Const(root.WithDevice("/device:CPU:0"), p1.SerializeAsString()); + auto p1_handle = ops::XRTAllocate(root, p1_value); + auto result = ops::XRTExecute(root, c_handle.handle, e_config, + {Output(p0_handle), Output(p1_handle)}); + TF_ASSERT_OK(root.status()); + + ClientSession session(root); + std::vector outputs; + TF_EXPECT_OK(session.Run({result}, &outputs)); + EXPECT_EQ(outputs.size(), 1); + + auto handles_vec = outputs.front().vec(); + EXPECT_EQ(handles_vec.size(), 2); + + const float kResults[2] = {15.0f, 9.0f}; + for (int64 i = 0; i < handles_vec.size(); ++i) { + auto read_back = ops::XRTReadLiteralAndRelease(root, Input(handles_vec(i))); + std::vector voutputs; + TF_EXPECT_OK(session.Run({read_back}, &voutputs)); + EXPECT_EQ(voutputs.size(), 1); + + xla::LiteralProto response; + EXPECT_TRUE(response.ParseFromString(voutputs[0].scalar()())); + + auto expected = xla::LiteralUtil::CreateR0(kResults[i]); + EXPECT_TRUE(CompareLiteralToLiteralProto(expected, response)); + } +} + TEST(RawApiTest, LeakCompilationReference) { xrt::XLAComputation c; auto config = c.mutable_config(); auto shapes = config->mutable_program_shape(); - *shapes->add_parameters() = xla::ShapeUtil::MakeShape(xla::F32, {2}); - *shapes->add_parameters() = xla::ShapeUtil::MakeShape(xla::F32, {2}); - *shapes->mutable_result() = xla::ShapeUtil::MakeTupleShape( - {xla::ShapeUtil::MakeShape(xla::F32, {2})}); + *shapes->add_parameters() = + xla::ShapeUtil::MakeShape(xla::F32, {2}).ToProto(); + *shapes->add_parameters() = + xla::ShapeUtil::MakeShape(xla::F32, {2}).ToProto(); + *shapes->mutable_result() = + xla::ShapeUtil::MakeTupleShape({xla::ShapeUtil::MakeShape(xla::F32, {2})}) + .ToProto(); StoreComputationSnapshot(AddAndTuple(), c.mutable_hlo_snapshot()); Scope root = Scope::NewRootScope().WithDevice(DeviceFromFlag()); @@ -703,9 +842,9 @@ TEST(RawApiTest, CompileAndExecuteWithS64Argument) { xrt::XLAComputation c; auto config = c.mutable_config(); auto shapes = config->mutable_program_shape(); - *shapes->add_parameters() = xla::ShapeUtil::MakeShape(xla::S64, {}); - *shapes->add_parameters() = xla::ShapeUtil::MakeShape(xla::S64, {}); - *shapes->mutable_result() = xla::ShapeUtil::MakeShape(xla::S64, {}); + *shapes->add_parameters() = xla::ShapeUtil::MakeShape(xla::S64, {}).ToProto(); + *shapes->add_parameters() = xla::ShapeUtil::MakeShape(xla::S64, {}).ToProto(); + *shapes->mutable_result() = xla::ShapeUtil::MakeShape(xla::S64, {}).ToProto(); StoreComputationSnapshot(AddS64(), c.mutable_hlo_snapshot()); xrt::XRTExecutionConfig e; @@ -739,11 +878,11 @@ TEST(RawApiTest, CompileAndExecuteWithS64Argument) { auto expected = xla::LiteralUtil::CreateR0(15123899); EXPECT_TRUE(CompareLiteralToLiteralProto(expected, response)); - xla::ProgramShape program_shape; + xla::ProgramShapeProto program_shape; EXPECT_TRUE(program_shape.ParseFromString(outputs[1].vec()(0))); EXPECT_EQ(program_shape.parameters_size(), 2); - EXPECT_TRUE( - xla::ShapeUtil::HasPrimitiveType(program_shape.result(), xla::S64)); + EXPECT_TRUE(xla::ShapeUtil::HasPrimitiveType( + xla::Shape(program_shape.result()), xla::S64)); } } // namespace diff --git a/tensorflow/compiler/xrt/xrt.proto b/tensorflow/compiler/xrt/xrt.proto index 6ab77fbaaf0..378bb9246f2 100644 --- a/tensorflow/compiler/xrt/xrt.proto +++ b/tensorflow/compiler/xrt/xrt.proto @@ -3,6 +3,7 @@ syntax = "proto3"; package xrt; import "tensorflow/compiler/tf2xla/host_compute_metadata.proto"; +import "tensorflow/compiler/xla/xla.proto"; import "tensorflow/compiler/xla/xla_data.proto"; import "tensorflow/compiler/xla/service/hlo.proto"; @@ -36,16 +37,18 @@ message XLAComputationConfig { tensorflow.tf2xla.HostComputeMetadata host_compute_metadata = 3; // The arg/result shapes for the whole computation. - xla.ProgramShape program_shape = 4; + xla.ProgramShapeProto program_shape = 4; // The arg/result shapes for each core of a model-parallel // computation. per_core_args_and_result_shapes is optional for a // single-core computation. - repeated xla.ProgramShape per_core_program_shape = 5; + repeated xla.ProgramShapeProto per_core_program_shape = 5; // Describes how replicated computation instances should be assigned to // devices. There are num_cores_per_replica computations, and each one will be // sent and executed to the set of replica device numbers described in the // DeviceAssignment proto. DeviceAssignment device_assignment = 6; + // The debugging options to be passed to the XLA compilation process. + xla.DebugOptions debug_options = 7; } // Options and XLA computation for a compilation. @@ -98,4 +101,8 @@ message XRTExecutionConfig { bool release_input_handles = 5; // If true, release the handle to the computation after running. bool release_compilation_handle = 6; + // If set to true, and the result shape is a tuple, then instead of returning + // a single tuple allocation the execution will return a vector of + // allocations, one for each of the first-level elements of the result tuple. + bool return_exploded_tuple = 7; } diff --git a/tensorflow/compiler/xrt/xrt_state.cc b/tensorflow/compiler/xrt/xrt_state.cc index 3a99820d7aa..5c7c537c340 100644 --- a/tensorflow/compiler/xrt/xrt_state.cc +++ b/tensorflow/compiler/xrt/xrt_state.cc @@ -183,6 +183,20 @@ Status XRTTupleAllocation::ToLiteral(xla::Backend* backend, int device_ordinal, return Status::OK(); } +Status XRTTupleAllocation::WriteLiteral(xla::Backend* backend, + const xla::Literal& literal) { + if (!xla::ShapeUtil::Equal(literal.shape(), on_host_shape())) { + return errors::InvalidArgument( + "New literal shape not matching the existing one: literal=", + xla::ShapeUtil::HumanStringWithLayout(literal.shape()), + " device=", xla::ShapeUtil::HumanStringWithLayout(on_host_shape())); + } + auto transfer_manager = backend->transfer_manager(); + TF_ASSIGN_OR_RETURN(auto stream, backend->BorrowStream(device_ordinal())); + return transfer_manager->TransferLiteralToDevice(stream.get(), literal, + ToShapedBuffer()); +} + void XRTTupleAllocation::DiscardAllocation( const xla::ShapeIndex& buffer_index) { buffers_.element(buffer_index)->DiscardAllocation(); diff --git a/tensorflow/compiler/xrt/xrt_state.h b/tensorflow/compiler/xrt/xrt_state.h index 73b5584e38f..3664c0cd4e6 100644 --- a/tensorflow/compiler/xrt/xrt_state.h +++ b/tensorflow/compiler/xrt/xrt_state.h @@ -137,6 +137,9 @@ class XRTTupleAllocation : public ResourceBase { Status ToLiteral(xla::Backend* backend, int device_ordinal, xla::Literal* literal); + // Write a new literal value to the allocation. + Status WriteLiteral(xla::Backend* backend, const xla::Literal& literal); + // True if none of the buffers in the allocation are aliased by any other live // handle. bool IsExclusiveOwner(); diff --git a/tensorflow/compiler/xrt/xrt_util.cc b/tensorflow/compiler/xrt/xrt_util.cc new file mode 100644 index 00000000000..3ef8bedc732 --- /dev/null +++ b/tensorflow/compiler/xrt/xrt_util.cc @@ -0,0 +1,76 @@ +/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/compiler/xrt/xrt_util.h" + +#include +#include + +#include "tensorflow/compiler/xla/debug_options_flags.h" +#include "tensorflow/compiler/xla/types.h" +#include "tensorflow/core/platform/logging.h" + +namespace tensorflow { +namespace { + +bool DebugOptionsPassThroughEnabled() { + const char* env = getenv("TF_XLA_DEBUG_OPTIONS_PASSTHROUGH"); + bool enabled = + env != nullptr && (strcmp(env, "1") == 0 || strcmp(env, "true") == 0); + if (enabled) { + LOG(WARNING) << "Passing through XLA debug options!"; + } else { + LOG(WARNING) << "TF_XLA_DEBUG_OPTIONS_PASSTHROUGH not set, not all options " + "will be retained"; + } + return enabled; +} + +string SafeDebugPath(const string& path) { + if (path.empty() || path.compare(0, 5, "gs://") == 0 || + path.compare(0, 11, "bigstore://") == 0) { + return path; + } + LOG(WARNING) << "Invalid config path (will be dropped): " << path; + return string(); +} + +} // namespace + +xla::DebugOptions BuildXlaDebugOptions(const xla::DebugOptions& ref_options) { + static const bool options_passthrough = DebugOptionsPassThroughEnabled(); + if (options_passthrough) { + return ref_options; + } + xla::DebugOptions options = xla::GetDebugOptionsFromFlags(); + options.set_xla_generate_hlo_text_to( + SafeDebugPath(ref_options.xla_generate_hlo_text_to())); + options.set_xla_dump_optimized_hlo_proto_to( + SafeDebugPath(ref_options.xla_dump_optimized_hlo_proto_to())); + options.set_xla_dump_computations_to( + SafeDebugPath(ref_options.xla_dump_computations_to())); + options.set_xla_dump_executions_to( + SafeDebugPath(ref_options.xla_dump_executions_to())); + for (auto& pass : ref_options.xla_disable_hlo_passes()) { + options.add_xla_disable_hlo_passes(pass); + } + options.set_xla_dump_unoptimized_hlo_proto_to( + SafeDebugPath(ref_options.xla_dump_unoptimized_hlo_proto_to())); + options.set_xla_dump_per_pass_hlo_proto_to( + SafeDebugPath(ref_options.xla_dump_per_pass_hlo_proto_to())); + return options; +} + +} // namespace tensorflow diff --git a/tensorflow/compiler/xrt/xrt_util.h b/tensorflow/compiler/xrt/xrt_util.h new file mode 100644 index 00000000000..d9c05a7f340 --- /dev/null +++ b/tensorflow/compiler/xrt/xrt_util.h @@ -0,0 +1,34 @@ +/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +// Utility functions in support of the XRT API. + +#ifndef TENSORFLOW_COMPILER_XRT_XRT_UTIL_H_ +#define TENSORFLOW_COMPILER_XRT_XRT_UTIL_H_ + +#include "tensorflow/compiler/xla/xla.pb.h" + +namespace tensorflow { + +// Filters the debug options provided as argument according to the value of the +// TF_XLA_DEBUG_OPTIONS_PASSTHROUGH environment variable. If such variable is +// set to "1" or "true", the debug options will be returned as is. Otherwise +// only a subset of them will be set in the returned ones, and all the paths +// contained in it, will be limited to gs:// and bigstore:// ones. +xla::DebugOptions BuildXlaDebugOptions(const xla::DebugOptions& ref_options); + +} // namespace tensorflow + +#endif // TENSORFLOW_COMPILER_XRT_XRT_UTIL_H_ diff --git a/tensorflow/contrib/all_reduce/BUILD b/tensorflow/contrib/all_reduce/BUILD index a513aa1e7c4..f6c6560c1c3 100644 --- a/tensorflow/contrib/all_reduce/BUILD +++ b/tensorflow/contrib/all_reduce/BUILD @@ -9,8 +9,6 @@ licenses(["notice"]) # Apache 2.0 exports_files(["LICENSE"]) -load("//tensorflow:tensorflow.bzl", "tf_py_test") - py_library( name = "all_reduce_py", srcs = ["__init__.py"], @@ -29,29 +27,6 @@ py_library( srcs_version = "PY2AND3", visibility = ["//visibility:public"], deps = [ - "//tensorflow/python:array_ops", - "//tensorflow/python:framework_ops", - "//tensorflow/python:math_ops", - "//tensorflow/python:nccl_ops", - ], -) - -tf_py_test( - name = "all_reduce_test", - srcs = ["python/all_reduce_test.py"], - additional_deps = [ - ":all_reduce", - "//third_party/py/numpy", - "//tensorflow/core:protos_all_py", - "//tensorflow/python:array_ops", - "//tensorflow/python:client", - "//tensorflow/python:framework_ops", - "//tensorflow/python:framework_test_lib", - "//tensorflow/python:math_ops", - "//tensorflow/python:constant_op", - "//tensorflow/python:client_testlib", - "//tensorflow/python:platform", - "//tensorflow/python:platform_test", - "//tensorflow/python:state_ops", + "//tensorflow/python/distribute:all_reduce", ], ) diff --git a/tensorflow/contrib/all_reduce/python/all_reduce.py b/tensorflow/contrib/all_reduce/python/all_reduce.py index 25f4b4b8d34..238cdaf8a79 100644 --- a/tensorflow/contrib/all_reduce/python/all_reduce.py +++ b/tensorflow/contrib/all_reduce/python/all_reduce.py @@ -18,842 +18,5 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function -import collections -import math - -from tensorflow.python.framework import device as device_lib -from tensorflow.python.framework import ops -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import nccl_ops - - -def _flatten_tensors(tensors): - """Check tensors for isomorphism and flatten. - - Args: - tensors: list of T `tf.Tensor` which must all have the same shape. - - Returns: - tensors: a list of T `tf.Tensor` which are flattened (1D) views of tensors - shape: the original shape of each element of input tensors - - Raises: - ValueError: tensors are empty or non-isomorphic or have unknown shape. - """ - if not tensors: - raise ValueError("tensors cannot be empty") - shape = tensors[0].shape - for tensor in tensors: - shape = shape.merge_with(tensor.shape) - if not shape.is_fully_defined(): - raise ValueError("Tensors must have statically known shape.") - if len(shape) != 1: - reshaped = [] - for t in tensors: - with ops.colocate_with(t): - reshaped.append(array_ops.reshape(t, [-1])) - tensors = reshaped - return tensors, shape - - -def _reshape_tensors(tensors, shape): - """Reshape tensors flattened by _flatten_tensors. - - Args: - tensors: list of T `tf.Tensor` of identical length 1D tensors. - shape: list of integers describing the desired shape. Product of - the elements must equal the length of each tensor. - - Returns: - list of T `tf.Tensor` which are the reshaped inputs. - """ - reshaped = [] - for t in tensors: - with ops.colocate_with(t): - reshaped.append(array_ops.reshape(t, shape)) - return reshaped - - -def _padded_split(tensor, pieces): - """Like split for 1D tensors but pads-out case where len % pieces != 0. - - Args: - tensor: T `tf.Tensor` that must be 1D. - pieces: a positive integer specifying the number of pieces into which - tensor should be split. - - Returns: - list of T `tf.Tensor` of length pieces, which hold the values of - thin input tensor, in order. The final tensor may - be zero-padded on the end to make its size equal to those of all - of the other tensors. - - Raises: - ValueError: The input tensor is not 1D. - """ - shape = tensor.shape - if 1 != len(shape): - raise ValueError("input tensor must be 1D") - tensor_len = shape.dims[0].value - with ops.colocate_with(tensor): - if tensor_len % pieces != 0: - # pad to an even length - chunk_size = 1 + tensor_len // pieces - if pieces > tensor_len: - # This is an edge case that should not come up in practice, - # i.e. a different reduction algorithm would be better, - # but we'll make it work just for completeness. - pad_len = pieces - tensor_len - extended_whole = array_ops.concat( - [tensor, array_ops.zeros([pad_len], dtype=tensor.dtype)], 0) - parts = array_ops.split(extended_whole, pieces) - return parts, pad_len - elif (pieces - 1) * chunk_size >= tensor_len: - # Another edge case of limited real interest. - pad_len = (pieces * chunk_size) % tensor_len - extended_whole = array_ops.concat( - [tensor, array_ops.zeros([pad_len], dtype=tensor.dtype)], 0) - parts = array_ops.split(extended_whole, pieces) - return parts, pad_len - else: - last_chunk_size = tensor_len - (pieces - 1) * chunk_size - pad_len = chunk_size - last_chunk_size - piece_lens = [chunk_size for _ in range(pieces - 1)] + [last_chunk_size] - parts = array_ops.split(tensor, piece_lens) - parts[-1] = array_ops.concat( - [parts[-1], array_ops.zeros([pad_len], dtype=tensor.dtype)], 0) - return parts, pad_len - else: - return array_ops.split(tensor, pieces), 0 - - -def _strip_padding(tensors, pad_len): - """Strip the suffix padding added by _padded_split. - - Args: - tensors: list of T `tf.Tensor` of identical length 1D tensors. - pad_len: number of elements to be stripped from the end of each tensor. - - Returns: - list of T `tf.Tensor` which are the stripped inputs. - - Raises: - ValueError: tensors must be a non-empty list of 1D tensors, and - each must be longer than pad_len. - """ - if not tensors: - raise ValueError("tensors cannot be empty") - shape = tensors[0].shape - if len(shape) > 1: - raise ValueError("tensors must be 1D") - prefix_len = int(shape[0] - pad_len) - if prefix_len < 0: - raise ValueError("pad_len longer than tensor") - stripped = [] - for t in tensors: - with ops.colocate_with(t): - stripped.append(array_ops.slice(t, [0], [prefix_len])) - return stripped - - -def _ragged_split(tensor, pieces): - """Like split for 1D tensors but allows case where len % pieces != 0. - - Args: - tensor: T `tf.Tensor` that must be 1D. - pieces: a positive integer specifying the number of pieces into which - tensor should be split. - - Returns: - list of T `tf.Tensor` of length pieces, which hold the values of - the input tensor, in order. The final tensor may be shorter - than the others, which will all be of equal length. - - Raises: - ValueError: input tensor must be 1D. - """ - shape = tensor.shape - if 1 != len(shape): - raise ValueError("input tensor must be 1D") - tensor_len = shape.dims[0].value - chunk_size = tensor_len // pieces - with ops.colocate_with(tensor): - if tensor_len != (pieces * chunk_size): - # last piece will be short - assert pieces > 1 - last_chunk_size = tensor_len - ((pieces - 1) * chunk_size) - assert last_chunk_size > 0 - piece_lens = [chunk_size for _ in range(pieces - 1)] + [last_chunk_size] - return array_ops.split(tensor, piece_lens) - else: - return array_ops.split(tensor, pieces) - - -def _ring_permutations(num_workers, num_subchunks, gpu_perm): - """"Generate an array of device index arrays, one for each subchunk. - - In the basic ring reduction algorithm there are size(T)/num_devices - data chunks and each device process one chunk per tick, i.e. sending - one chunk and receiving one chunk. The idea of subchunking is that - each device processes num_subchunks smaller data regions per tick, - and the ring rank permutation is different for each subchunk index - so that a device is potentially sending to and receiving from - num_subchunks different other devices at each tick. Where multiple - independent data channels exist between devices, this strategy - supplies a method of using them in parallel. - - Args: - num_workers: number of worker tasks - num_subchunks: number of subchunks into which to divide each per-GPU chunk. - gpu_perm: an array of integers in [0, num_gpus-1] giving the default - ring order of GPUs at each worker. Other permutations will be generated - by rotating this array and splicing together per-worker instances. - - Raises: - ValueError: the number of subchunks may not exceed the number of GPUs. - - Returns: - pred_by_s_d: list of lists that maps (by index) from (subchunk, dev) to - preceding device in the permutation for that subchunk. The - device index of GPU i at worker j is i + (j * num_gpus). - rank_by_s_d: list of lists that maps (by index) from (subchunk, dev) to - local rank of device d in the permutation for that subchunk. - """ - num_gpus = len(gpu_perm) - devices = num_workers * num_gpus - if devices == 0: - return [], [] - if num_subchunks > num_gpus: - raise ValueError( - "num_subchunks %d must be <= num_gpus %d" % (num_subchunks, num_gpus)) - rotation_interval = max(1, int(num_gpus / num_subchunks)) - perms_by_s = [] - for s in range(0, num_subchunks): - full_order = [] - offset = s * rotation_interval - for w in range(0, num_workers): - default_order = [(w * num_gpus) + i for i in gpu_perm] - dev_order = default_order[offset:] + default_order[:offset] - full_order += dev_order - perms_by_s.append(full_order) - pred_by_s_d = [[-1 for d in range(0, devices)] - for s in range(0, num_subchunks)] - rank_by_s_d = [[-1 for d in range(0, devices)] - for s in range(0, num_subchunks)] - for s in range(0, num_subchunks): - for d in range(0, devices): - for t in range(0, devices): - if d == perms_by_s[s][t]: - rank_by_s_d[s][d] = t - pred_by_s_d[s][d] = perms_by_s[s][(t + devices - 1) % devices] - break - return (pred_by_s_d, rank_by_s_d) - - -def build_ring_all_reduce(input_tensors, num_workers, num_subchunks, - gpu_perm, red_op, un_op=None): - """Construct a subgraph performing a ring-style all-reduce of input_tensors. - - Args: - input_tensors: a list of T `tf.Tensor` objects, which must all - have the same shape and type. - num_workers: number of worker tasks spanned by input_tensors. - num_subchunks: number of subchunks each device should process in one tick. - gpu_perm: a list of ints giving a ring-wise rank ordering of GPUs at - each worker. All workers must have the same number of - GPUs with the same rank ordering. If NVLINK is available, this should - be a ring order supported by NVLINK edges. - red_op: a binary operator for elementwise reduction. - un_op: an optional unary operator to apply to fully reduced values. - - Raises: - ValueError: empty input_tensors or they don't all have same - size. - - Returns: - a list of T `tf.Tensor` identical sum-reductions of input_tensors. - """ - if len(input_tensors) < 2: - raise ValueError("input_tensors must be length 2 or longer") - input_tensors, shape = _flatten_tensors(input_tensors) - devices = [t.device for t in input_tensors] - (pred_by_s_d, rank_by_s_d) = _ring_permutations( - num_workers, num_subchunks, gpu_perm) - chunks_by_dev, pad_len = _build_ring_gather( - input_tensors, devices, - num_subchunks, pred_by_s_d, rank_by_s_d, red_op) - if un_op: - chunks_by_dev = _apply_unary_to_chunks(un_op, chunks_by_dev) - output_tensors = _build_ring_scatter(pred_by_s_d, rank_by_s_d, - chunks_by_dev) - if pad_len > 0: - output_tensors = _strip_padding(output_tensors, pad_len) - if len(shape) != 1: - output_tensors = _reshape_tensors(output_tensors, shape) - return output_tensors - - -def _build_ring_gather(input_tensors, devices, num_subchunks, - pred_by_s_d, rank_by_s_d, red_op): - """Construct a subgraph for the first (reduction) pass of ring all-reduce. - - Args: - input_tensors: a list of T `tf.Tensor` 1D input tensors of same - shape and type. - devices: array of device name strings - num_subchunks: number of subchunks each device should process in one tick. - pred_by_s_d: as produced by _ring_permutations - rank_by_s_d: as produced by _ring_permutations - red_op: a binary operator for elementwise reduction - - Raises: - ValueError: tensors must all be one dimensional. - - Returns: - list of list of T `tf.Tensor` of (partially) reduced values where - exactly num_subchunks chunks at each device are fully reduced. - """ - num_devices = len(input_tensors) - if num_devices == 0: - return [] - if num_devices == 1: - return input_tensors - shape = input_tensors[0].shape - if 1 != len(shape): - raise ValueError("input tensors must be 1D") - num_chunks = num_devices * num_subchunks - num_ticks = num_devices - 1 - # Initialize chunks_by_dev with splits of the input tensors. - chunks_by_dev = [] - split_pad_len = 0 - for d in range(0, num_devices): - with ops.device(devices[d]): - splits, split_pad_len = _padded_split(input_tensors[d], num_chunks) - chunks_by_dev.append(splits) - # Reduction phase - for tick in range(0, num_ticks): - # One new partial reduction for every chunk - new_partial_reductions = [None for _ in range(0, num_chunks)] - # Compute reductions with respect to last tick's values - for d in range(0, num_devices): - with ops.device(devices[d]): - for s in range(0, num_subchunks): - rank = rank_by_s_d[s][d] - seg_index = (rank + num_devices - (2 + tick)) % num_devices - pred_dev = pred_by_s_d[s][d] - chunk_index = (seg_index * num_subchunks) + s - new_partial_reductions[chunk_index] = red_op( - chunks_by_dev[pred_dev][chunk_index], - chunks_by_dev[d][chunk_index]) - # Update chunks_by_dev with the new values at the end of the tick. - for d in range(0, num_devices): - for s in range(0, num_subchunks): - rank = rank_by_s_d[s][d] - seg_index = (rank + num_devices - (2 + tick)) % num_devices - chunk_index = (seg_index * num_subchunks) + s - chunks_by_dev[d][chunk_index] = new_partial_reductions[chunk_index] - return chunks_by_dev, split_pad_len - - -def _apply_unary_to_chunks(f, chunks_by_dev): - """Apply a unary op to each tensor in chunks_by_dev, on same device. - - Args: - f: a unary function over T `tf.Tensor`. - chunks_by_dev: list of lists of T `tf.Tensor`. - - Returns: - new list of lists of T `tf.Tensor` with the same structure as - chunks_by_dev containing the derived tensors. - """ - output = [] - for x in chunks_by_dev: - with ops.colocate_with(x[0]): - output.append([f(t) for t in x]) - return output - - -def _build_ring_scatter(pred_by_s_d, rank_by_s_d, - chunks_by_dev): - """Construct subgraph for second (scatter) pass of ring all-reduce. - - Args: - pred_by_s_d: as produced by _ring_permutations - rank_by_s_d: as produced by _ring_permutations - chunks_by_dev: list of list of T `tf.Tensor` indexed by ints - (device, chunk) - - Raises: - ValueError: chunks_by_dev is not well-formed - - Returns: - list of T `tf.Tensor` which are the fully reduced tensors, one - at each device corresponding to the outer dimension of chunks_by_dev. - """ - num_devices = len(chunks_by_dev) - num_chunks = len(chunks_by_dev[0]) - if 0 != num_chunks % num_devices: - raise ValueError( - "Expect number of chunks per device to be divisible by num_devices") - num_subchunks = int(num_chunks / num_devices) - num_ticks = num_devices - 1 - for tick in range(0, num_ticks): - passed_values = [None for _ in range(0, num_chunks)] - for d in range(0, num_devices): - with ops.colocate_with(chunks_by_dev[d][0]): - for s in range(0, num_subchunks): - rank = rank_by_s_d[s][d] - seg_index = (rank + num_devices - (1 + tick)) % num_devices - pred_dev = pred_by_s_d[s][d] - chunk_index = (seg_index * num_subchunks) + s - passed_values[chunk_index] = array_ops.identity( - chunks_by_dev[pred_dev][chunk_index]) - for d in range(0, num_devices): - for s in range(0, num_subchunks): - rank = rank_by_s_d[s][d] - seg_index = (rank + num_devices - (1 + tick)) % num_devices - chunk_index = (seg_index * num_subchunks) + s - chunks_by_dev[d][chunk_index] = passed_values[chunk_index] - # Join chunks at each device. - output = [] - for x in chunks_by_dev: - with ops.colocate_with(x[0]): - output.append(array_ops.concat(x, 0)) - return output - - -def build_recursive_hd_all_reduce(input_tensors, red_op, un_op=None): - """Construct a subgraph for recursive halving-doubling all-reduce. - - The recursive halving-doubling algorithm is described in - http://www.mcs.anl.gov/~thakur/papers/ijhpca-coll.pdf - - The concept is to arrange the participating n devices in - a linear sequence where devices exchange data pairwise - with one other device in each round. During the gather - phase there are lg(n) rounds where devices exchange - increasingly smaller sub-tensors with another device - at increasingly greater distances, until at the top - each device has 1/n of the fully reduced values. During the - scatter phase each device exchanges its fully reduced - sub-tensor (which doubles in length at each round) - with one other device at increasingly smaller distances - until each device has all of the fully reduced values. - - Note: this preliminary version requires that len(input_tensors) be a - power of 2. TODO(tucker): relax this restriction. Also, the - number of elements in each tensor must be divisible by 2^h where h - is the number of hops in each phase. This will also be relaxed in - the future with edge-case specific logic. - - Args: - input_tensors: list of T `tf.Tensor` to be elementwise reduced. - red_op: a binary elementwise reduction Op. - un_op: an optional unary elementwise Op to apply to reduced values. - - Returns: - list of T `tf.Tensor` which are the fully reduced tensors, one - at each device of input_tensors. - - Raises: - ValueError: num_devices not a power of 2, or tensor len not divisible - by 2 the proper number of times. - """ - devices = [t.device for t in input_tensors] - input_tensors, shape = _flatten_tensors(input_tensors) - reduced_shards = _build_recursive_hd_gather(input_tensors, devices, red_op) - if un_op: - reduced_shards = [un_op(t) for t in reduced_shards] - output_tensors = _build_recursive_hd_scatter(reduced_shards, devices) - if len(shape) != 1: - output_tensors = _reshape_tensors(output_tensors, shape) - return output_tensors - - -def _build_recursive_hd_gather(input_tensors, devices, red_op): - """Construct the gather phase of recursive halving-doubling all-reduce. - - Args: - input_tensors: list of T `tf.Tensor` to be elementwise reduced. - devices: a list of strings naming the devices hosting input_tensors, - which will also be used to host the (partial) reduction values. - red_op: a binary elementwise reduction Op. - - Returns: - list of T `tf.Tensor` which are the fully reduced tensor shards. - - Raises: - ValueError: num_devices not a power of 2, or tensor len not divisible - by 2 the proper number of times. - """ - num_devices = len(devices) - num_hops = int(math.log(num_devices, 2)) - if num_devices != (2 ** num_hops): - raise ValueError("num_devices must be a power of 2") - chunks = input_tensors - for h in range(0, num_hops): - span = 2 ** h - group_size = span * 2 - new_chunks = [[] for _ in devices] - for d in range(0, num_devices): - if (d % group_size) >= (group_size / 2): - # skip right half of a pair - continue - left_dev = devices[d] - right_dev = devices[d + span] - left_split = array_ops.split(chunks[d], 2) - right_split = array_ops.split(chunks[d+span], 2) - with ops.device(left_dev): - new_chunks[d] = red_op(left_split[0], right_split[0]) - with ops.device(right_dev): - new_chunks[d + span] = red_op(left_split[1], right_split[1]) - chunks = new_chunks - return chunks - - -def _build_recursive_hd_scatter(input_tensors, devices): - """Construct the scatter phase of recursive halving-doublng all-reduce. - - Args: - input_tensors: list of T `tf.Tensor` that are fully-reduced shards. - devices: a list of strings naming the devices on which the reconstituted - full tensors should be placed. - - Returns: - list of T `tf.Tensor` which are the fully reduced tensors. - """ - num_devices = len(devices) - num_hops = int(math.log(num_devices, 2)) - assert num_devices == (2 ** num_hops), "num_devices must be a power of 2" - chunks = input_tensors - for h in reversed(range(0, num_hops)): - span = 2 ** h - group_size = span * 2 - new_chunks = [[] for _ in devices] - for d in range(0, num_devices): - if (d % group_size) >= (group_size / 2): - # skip right half of a pair - continue - left_idx = d - right_idx = d + span - left_dev = devices[left_idx] - right_dev = devices[right_idx] - with ops.device(left_dev): - new_chunks[left_idx] = array_ops.concat([chunks[left_idx], - chunks[right_idx]], 0) - with ops.device(right_dev): - new_chunks[right_idx] = array_ops.concat([chunks[left_idx], - chunks[right_idx]], 0) - chunks = new_chunks - return chunks - - -def build_shuffle_all_reduce(input_tensors, gather_devices, red_op, un_op=None): - """Construct a subgraph for shuffle all-reduce. - - Shuffle reduce is essentially the algorithm implemented when using - parameter servers. Suppose tensor length is n, there are d devices - and g gather shards. Each device sends a n/g length sub-tensor to - each gather shard. The gather shards perform a reduction across d - fragments, then broadcast the result back to each device. The - devices then join the g fully reduced fragments they receive from - the shards. The gather shards could perform d-1 pairwise - reductions, or one d-way reduction. The first is better where - reduction Op time is low compared to transmission time, the second - better in the other case. - - Args: - input_tensors: list of T @(tf.Tensor} values to be reduced. - gather_devices: list of names of devices on which reduction shards - should be placed. - red_op: an n-array elementwise reduction Op - un_op: optional elementwise unary Op to be applied to fully-reduced values. - - Returns: - list of T `tf.Tensor` which are the fully reduced tensors. - """ - input_tensors, shape = _flatten_tensors(input_tensors) - dst_devices = [t.device for t in input_tensors] - reduced_shards = _build_shuffle_gather(input_tensors, gather_devices, - red_op, un_op) - output_tensors = _build_shuffle_scatter(reduced_shards, dst_devices) - if len(shape) != 1: - output_tensors = _reshape_tensors(output_tensors, shape) - return output_tensors - - -def _build_shuffle_gather(input_tensors, gather_devices, red_op, un_op=None): - """Construct the gather (concentrate and reduce) phase of shuffle all-reduce. - - Args: - input_tensors: list of T @(tf.Tensor} values to be reduced. - gather_devices: list of names of devices on which reduction shards - should be placed. - red_op: the binary reduction Op - un_op: optional elementwise unary Op to be applied to fully-reduced values. - - Returns: - list of T `tf.Tensor` which are the fully reduced shards. - - Raises: - ValueError: inputs not well-formed. - """ - num_source_devices = len(input_tensors) - num_gather_devices = len(gather_devices) - shape = input_tensors[0].shape - if len(shape) != 1: - raise ValueError("input_tensors must be 1D") - shards_by_source = [] - for d in range(0, num_source_devices): - with ops.colocate_with(input_tensors[d]): - shards_by_source.append( - _ragged_split(input_tensors[d], num_gather_devices)) - reduced_shards = [] - for d in range(0, num_gather_devices): - with ops.device(gather_devices[d]): - values = [s[d] for s in shards_by_source] - red_shard = red_op(values) - if un_op: - red_shard = un_op(red_shard) - reduced_shards.append(red_shard) - return reduced_shards - - -def _build_shuffle_scatter(reduced_shards, dst_devices): - """Build the scatter phase of shuffle all-reduce. - - Args: - reduced_shards: list of T @(tf.Tensor} fully reduced shards - dst_devices: list of names of devices at which the fully-reduced value - should be reconstituted. - - Returns: - list of T `tf.Tensor` scattered tensors. - """ - num_devices = len(dst_devices) - out_tensors = [] - for d in range(0, num_devices): - with ops.device(dst_devices[d]): - out_tensors.append(array_ops.concat(reduced_shards, 0)) - return out_tensors - - -def _split_by_task(devices, values): - """Partition devices and values by common task. - - Args: - devices: list of device name strings - values: list of T `tf.tensor` of same length as devices. - - Returns: - (per_task_devices, per_task_values) where both values are - lists of lists with isomorphic structure: the outer list is - indexed by task, and the inner list has length of the number - of values belonging to that task. per_task_devices contains - the specific devices to which the values are local, and - per_task_values contains the corresponding values. - - Raises: - ValueError: devices must be same length as values. - """ - num_devices = len(devices) - if num_devices != len(values): - raise ValueError("len(devices) must equal len(values)") - per_task_devices = collections.OrderedDict() - per_task_values = collections.OrderedDict() - for d in range(num_devices): - d_spec = device_lib.DeviceSpec.from_string(devices[d]) - if not hasattr(d_spec, "task") or d_spec.task is None: - assert False, "failed to parse device %s" % devices[d] - index = (d_spec.job or "localhost", d_spec.replica or 0, d_spec.task) - if index not in per_task_devices: - per_task_devices[index] = [] - per_task_values[index] = [] - per_task_devices[index].append(devices[d]) - per_task_values[index].append(values[d]) - - return (list(per_task_devices.values()), list(per_task_values.values())) - - -def build_nccl_all_reduce(input_tensors, red_op, un_op=None): - """Build a subgraph that does one full all-reduce, using NCCL. - - Args: - input_tensors: list of T `tf.Tensor` of same-shape and type values to - be reduced. - red_op: binary elementwise reduction operator. Must be one of - {tf.add} - un_op: optional unary elementwise Op to apply to fully-reduce values. - - Returns: - list of T `tf.Tensor` of reduced values. - - Raises: - ValueError: red_op not supported. - """ - if red_op == math_ops.add: - output_tensors = nccl_ops.all_sum(input_tensors) - else: - raise ValueError("red_op not supported by NCCL all-reduce: ", red_op) - if un_op: - un_op_wrapped = [] - for t in output_tensors: - with ops.colocate_with(t): - un_op_wrapped.append(un_op(t)) - output_tensors = un_op_wrapped - return output_tensors - - -def _build_nccl_hybrid(input_tensors, red_op, upper_level_f): - """Construct a subgraph for NCCL hybrid all-reduce. - - Args: - input_tensors: list of T `tf.Tensor` of same-shape and type values to - be reduced. - red_op: binary elementwise reduction operator. - upper_level_f: function for reducing one value per worker, across - workers. - - Returns: - list of T `tf.Tensor` of reduced values. - - Raises: - ValueError: inputs not well-formed. - """ - input_tensors, shape = _flatten_tensors(input_tensors) - devices = [t.device for t in input_tensors] - per_worker_devices, per_worker_values = _split_by_task(devices, input_tensors) - num_workers = len(per_worker_devices) - up_values = [None for w in range(0, num_workers)] - up_devices = up_values[:] - down_values = up_values[:] - # First stage: reduce within each worker using NCCL - for w in range(0, num_workers): - worker_values = build_nccl_all_reduce(per_worker_values[w], red_op) - # NOTE: these reductions will not run to completion unless - # every output value is used. Since we only need one, we - # need to put control dependencies on the rest. - with ops.control_dependencies(worker_values): - with ops.device(worker_values[0].device): - up_values[w] = array_ops.identity(worker_values[0]) - up_devices[w] = per_worker_devices[w][0] - # Second stage: Apply upper_level_f to reduce across first device at - # each worker - level_2_output = upper_level_f(up_values) - # Third stage: propagate within each worker using NCCL Broadcast - for w in range(0, num_workers): - dst_tensors = [] - with ops.device(per_worker_devices[w][0]): - broadcast_src = nccl_ops.broadcast(array_ops.identity(level_2_output[w])) - for d in per_worker_devices[w]: - with ops.device(d): - dst_tensors.append(array_ops.identity(broadcast_src)) - down_values[w] = dst_tensors - output_tensors = [v for sublist in down_values for v in sublist] - if len(shape) != 1: - output_tensors = _reshape_tensors(output_tensors, shape) - return output_tensors - - -def _reduce_non_singleton(input_tensors, red_f, un_op): - """If input_tensors has more than one element apply red_f, else apply un_op.""" - if len(input_tensors) > 1: - return red_f(input_tensors) - else: - if not un_op: - return input_tensors - output_tensors = [] - for t in input_tensors: - with ops.colocate_with(t): - output_tensors.append(un_op(t)) - return output_tensors - - -def build_nccl_then_ring(input_tensors, subdiv, red_op, un_op=None): - """Construct hybrid of NCCL within workers, Ring across workers.""" - def upper_builder(y): - return build_ring_all_reduce(y, len(y), subdiv, [0], red_op, un_op) - def upper_level_f(x): - return _reduce_non_singleton(x, upper_builder, un_op) - return _build_nccl_hybrid(input_tensors, red_op, upper_level_f) - - -def build_nccl_then_recursive_hd(input_tensors, red_op, un_op=None): - """Construct hybrid of NCCL within workers, Recursive-HD across workers.""" - upper_level_f = lambda x: build_recursive_hd_all_reduce(x, red_op, un_op) - return _build_nccl_hybrid(input_tensors, red_op, upper_level_f) - - -def build_nccl_then_shuffle(input_tensors, gather_devices, nccl_red_op, - shuffle_red_op, un_op=None): - """Construct hybrid of NCCL within workers, Shuffle across workers.""" - upper_level_f = lambda x: build_shuffle_all_reduce(x, gather_devices, - shuffle_red_op, un_op) - return _build_nccl_hybrid(input_tensors, nccl_red_op, upper_level_f) - - -def _build_shuffle_hybrid(input_tensors, gather_devices, red_op, upper_level_f): - """Construct a subgraph for Shuffle hybrid all-reduce. - - Args: - input_tensors: list of T `tf.Tensor` of same-shape and type values to - be reduced. - gather_devices: list of device names on which to host gather shards. - red_op: binary elementwise reduction operator. - upper_level_f: function for reducing one value per worker, across - workers. - - Returns: - list of T `tf.Tensor` of reduced values. - - Raises: - ValueError: inputs not well-formed. - """ - input_tensors, shape = _flatten_tensors(input_tensors) - # First stage, reduce across each worker using gather_devices. - devices = [t.device for t in input_tensors] - per_worker_devices, per_worker_values = _split_by_task(devices, input_tensors) - num_workers = len(per_worker_devices) - up_values = [] - if len(gather_devices) != num_workers: - raise ValueError("For shuffle hybrid, gather_devices must contain one " - "device per worker. ") - for w in range(0, num_workers): - reduced_shards = _build_shuffle_gather( - per_worker_values[w], [gather_devices[w]], red_op) - up_values.append(reduced_shards[0]) - # Second stage, apply upper_level_f. - level_2_output = upper_level_f(up_values) - # Third stage, apply shuffle scatter at each worker. - output_tensors = [] - for w in range(0, num_workers): - output_tensors += _build_shuffle_scatter( - [level_2_output[w]], per_worker_devices[w]) - if len(shape) != 1: - output_tensors = _reshape_tensors(output_tensors, shape) - return output_tensors - - -def build_shuffle_then_ring(input_tensors, gather_devices, subdiv, - red_n_op, red_op, un_op=None): - """Construct hybrid of Shuffle within workers, Ring across workers.""" - def upper_builder(tensors): - return build_ring_all_reduce(tensors, len(tensors), subdiv, [0], - red_op, un_op) - def upper_level_f(tensors): - return _reduce_non_singleton(tensors, upper_builder, un_op) - return _build_shuffle_hybrid( - input_tensors, gather_devices, red_n_op, upper_level_f) - - -def build_shuffle_then_shuffle(input_tensors, first_gather_devices, - second_gather_devices, red_op, un_op=None): - """Construct hybrid of Shuffle within workers, Shuffle across workers.""" - def upper_builder(tensors): - return build_shuffle_all_reduce(tensors, second_gather_devices, - red_op, un_op) - def upper_level_f(tensors): - return _reduce_non_singleton(tensors, upper_builder, un_op) - return _build_shuffle_hybrid( - input_tensors, first_gather_devices, red_op, upper_level_f) +# pylint: disable=unused-import,wildcard-import +from tensorflow.python.distribute.all_reduce import * diff --git a/tensorflow/contrib/autograph/examples/benchmarks/BUILD b/tensorflow/contrib/autograph/examples/benchmarks/BUILD new file mode 100644 index 00000000000..6d2d70c99b4 --- /dev/null +++ b/tensorflow/contrib/autograph/examples/benchmarks/BUILD @@ -0,0 +1,36 @@ +licenses(["notice"]) # Apache 2.0 + +load("//tensorflow:tensorflow.bzl", "py_test") +load("//tensorflow/tools/test:performance.bzl", "tf_py_logged_benchmark") + +py_library( + name = "benchmark_base", + srcs = [ + "benchmark_base.py", + ], + deps = [ + "//tensorflow:tensorflow_py", + ], +) + +py_test( + name = "cartpole_benchmark", + size = "enormous", + srcs = ["cartpole_benchmark.py"], + tags = [ + "local", + "manual", + "no_oss", + "notap", + "nozapfhahn", + ], + deps = [ + ":benchmark_base", + # Note: required gym dependency may need to be added here. + ], +) + +tf_py_logged_benchmark( + name = "cartpole_logged_benchmark", + target = "//tensorflow/contrib/autograph/examples/benchmarks:cartpole_benchmark", +) diff --git a/tensorflow/contrib/autograph/examples/benchmarks/benchmark_base.py b/tensorflow/contrib/autograph/examples/benchmarks/benchmark_base.py new file mode 100644 index 00000000000..93c694849c4 --- /dev/null +++ b/tensorflow/contrib/autograph/examples/benchmarks/benchmark_base.py @@ -0,0 +1,62 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Common benchmarking code. + +See https://www.tensorflow.org/community/benchmarks for usage. +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import time + +import numpy as np + +import tensorflow as tf + + +class ReportingBenchmark(tf.test.Benchmark): + """Base class for a benchmark that reports general performance metrics. + + Subclasses only need to call one of the _profile methods, and optionally + report_results. + """ + + def time_execution(self, name, target, iters, warm_up_iters=5): + for _ in range(warm_up_iters): + target() + + all_times = [] + for _ in range(iters): + iter_time = time.time() + target() + all_times.append(time.time() - iter_time) + + avg_time = np.average(all_times) + + extras = dict() + extras['all_times'] = all_times + + if isinstance(name, tuple): + extras['name'] = name + name = '_'.join(str(piece) for piece in name) + + self.report_benchmark( + iters=iters, wall_time=avg_time, name=name, extras=extras) + + +if __name__ == '__main__': + tf.test.main() diff --git a/tensorflow/contrib/autograph/examples/benchmarks/cartpole_benchmark.py b/tensorflow/contrib/autograph/examples/benchmarks/cartpole_benchmark.py new file mode 100644 index 00000000000..4f553be58e9 --- /dev/null +++ b/tensorflow/contrib/autograph/examples/benchmarks/cartpole_benchmark.py @@ -0,0 +1,492 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""A basic RL cartpole benchmark. + +The RL model uses the OpenAI Gym environment to train a simple network using +the policy gradients method. The training scales the gradients for each step +by the episode's cumulative discounted reward and averages these gradients over +a fixed number of games before applying the optimization step. + +For benchmarking purposes, we replace the OpenAI Gym environment to a fake +that returns random actions and rewards and never ends the episode. This way +the benchmarks compare the same amount of computation at each step. +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import gym +import numpy as np +import tensorflow as tf + +from tensorflow.contrib import eager +from tensorflow.contrib.autograph.examples.benchmarks import benchmark_base +from tensorflow.python import autograph as ag +from tensorflow.python.eager import context + +# +# AutoGraph implementation +# + + +@ag.convert() +def graph_append_discounted_rewards(destination, rewards, discount_rate): + """Discounts episode rewards and appends them to destination.""" + ag.set_element_type(rewards, tf.float32) + + cdr = 0.0 + reverse_discounted = [] + ag.set_element_type(reverse_discounted, tf.float32) + + for i in range(len(rewards) - 1, -1, -1): + cdr = cdr * discount_rate + rewards[i] + cdr.set_shape(()) + reverse_discounted.append(cdr) + + retval = destination + # Note: AutoGraph doesn't yet support reversed() so we use a loop instead. + for i in range(len(reverse_discounted) - 1, -1, -1): + retval.append(reverse_discounted[i]) + + return retval + + +class GraphPolicyNetwork(tf.keras.Model): + """Policy network for the cart-pole reinforcement learning problem. + + The forward path of the network takes an observation from the cart-pole + environment (length-4 vector) and outputs an action. + """ + + def __init__(self, hidden_size): + super(GraphPolicyNetwork, self).__init__() + self._hidden_layer = tf.keras.layers.Dense( + hidden_size, activation=tf.nn.elu) + self._output_layer = tf.keras.layers.Dense(1) + + def call(self, inputs): + """Calculates logits and action. + + Args: + inputs: Observations from a step in the cart-pole environment, of shape + `(batch_size, input_size)` + + Returns: + logits: the logits output by the output layer. This can be viewed as the + likelihood vales of choosing the left (0) action. Shape: + `(batch_size, 1)`. + actions: randomly selected actions ({0, 1}) based on the logits. Shape: + `(batch_size, 1)`. + """ + hidden = self._hidden_layer(inputs) + logits = self._output_layer(hidden) + + left_prob = tf.nn.sigmoid(logits) + action_probs = tf.concat([left_prob, 1.0 - left_prob], 1) + + actions = tf.multinomial(tf.log(action_probs), 1) + return logits, actions + + # TODO(mdan): Move this method out of the class. + @ag.convert() + def train(self, cart_pole_env, optimizer, discount_rate, num_games, + max_steps_per_game): + var_list = tf.trainable_variables() + grad_list = [ + tf.TensorArray(tf.float32, 0, dynamic_size=True) for _ in var_list + ] + + step_counts = [] + discounted_rewards = [] + ag.set_element_type(discounted_rewards, tf.float32) + ag.set_element_type(step_counts, tf.int32) + + # Note: we use a shared object, cart_pole_env here. Because calls to the + # object's method are made through py_func, TensorFlow cannot detect its + # data dependencies. Hence we must manually synchronize access to it + # and ensure the control dependencies are set in such a way that + # calls to reset(), take_one_step, etc. are made in the correct order. + sync_counter = tf.constant(0) + + for _ in tf.range(num_games): + with tf.control_dependencies([sync_counter]): + obs = cart_pole_env.reset() + with tf.control_dependencies([obs]): + sync_counter += 1 + + game_rewards = [] + ag.set_element_type(game_rewards, tf.float32) + + for step in tf.range(max_steps_per_game): + logits, actions = self(obs) # pylint:disable=not-callable + logits = tf.reshape(logits, ()) + actions = tf.reshape(actions, ()) + + labels = 1.0 - tf.cast(actions, tf.float32) + loss = tf.nn.sigmoid_cross_entropy_with_logits( + labels=labels, logits=logits) + grads = tf.gradients(loss, var_list) + + for i in range(len(grads)): + grad_list[i].append(grads[i]) + + with tf.control_dependencies([sync_counter]): + obs, reward, done = cart_pole_env.step(actions) + with tf.control_dependencies([obs]): + sync_counter += 1 + obs = tf.reshape(obs, (1, 4)) + + game_rewards.append(reward) + if reward < 0.1 or done: + step_counts.append(step + 1) + break + + discounted_rewards = graph_append_discounted_rewards( + discounted_rewards, game_rewards, discount_rate) + + discounted_rewards = ag.stack(discounted_rewards) + discounted_rewards.set_shape((None,)) + mean, variance = tf.nn.moments(discounted_rewards, [0]) + normalized_rewards = (discounted_rewards - mean) / tf.sqrt(variance) + + for i in range(len(grad_list)): + g = ag.stack(grad_list[i]) + + # This block just adjusts the shapes to match for multiplication. + r = normalized_rewards + if r.shape.ndims < g.shape.ndims: + r = tf.expand_dims(r, -1) + if r.shape.ndims < g.shape.ndims: + r = tf.expand_dims(r, -1) + + grad_list[i] = tf.reduce_mean(g * r, axis=0) + + optimizer.apply_gradients( + zip(grad_list, var_list), global_step=tf.train.get_global_step()) + + return ag.stack(step_counts) + + +@ag.convert() +def graph_train_model(policy_network, cart_pole_env, optimizer, iterations): + """Trains the policy network for a given number of iterations.""" + i = tf.constant(0) + mean_steps_per_iteration = [] + ag.set_element_type(mean_steps_per_iteration, tf.int32) + + while i < iterations: + steps_per_game = policy_network.train( + cart_pole_env, + optimizer, + discount_rate=0.95, + num_games=20, + max_steps_per_game=200) + mean_steps_per_iteration.append(tf.reduce_mean(steps_per_game)) + i += 1 + + return ag.stack(mean_steps_per_iteration) + + +class GraphGymCartpoleEnv(object): + """An env backed by OpenAI Gym's CartPole environment. + + Used to confirm a functional model only. + """ + + def __init__(self): + cart_pole_env = gym.make('CartPole-v1') + cart_pole_env.seed(0) + cart_pole_env.reset() + self.env = cart_pole_env + + def reset(self): + obs = ag.utils.wrap_py_func(self.env.reset, tf.float64, ()) + obs = tf.reshape(obs, (1, 4)) + obs = tf.cast(obs, tf.float32) + return obs + + def step(self, actions): + + def take_one_step(actions): + obs, reward, done, _ = self.env.step(actions) + obs = obs.astype(np.float32) + reward = np.float32(reward) + return obs, reward, done + + return ag.utils.wrap_py_func(take_one_step, + (tf.float32, tf.float32, tf.bool), (actions,)) + + +class GraphRandomCartpoleEnv(object): + """An environment that returns random actions and never finishes. + + Used during benchmarking, it will cause training to run a constant number of + steps. + """ + + def reset(self): + return tf.random.normal((1, 4)) + + def step(self, actions): + with tf.control_dependencies([actions]): + random_obs = tf.random.normal((1, 4)) + fixed_reward = tf.constant(0.001) + done = tf.constant(False) + return random_obs, fixed_reward, done + + +# +# Eager implementation +# + + +def eager_append_discounted_rewards(discounted_rewards, rewards, discount_rate): + cdr = 0.0 + reverse_discounted = [] + + for i in range(len(rewards) - 1, -1, -1): + cdr = cdr * discount_rate + rewards[i] + reverse_discounted.append(cdr) + + discounted_rewards.extend(reversed(reverse_discounted)) + return discounted_rewards + + +class EagerPolicyNetwork(tf.keras.Model): + """Policy network for the cart-pole reinforcement learning problem. + + The forward path of the network takes an observation from the cart-pole + environment (length-4 vector) and outputs an action. + """ + + def __init__(self, hidden_size): + super(EagerPolicyNetwork, self).__init__() + self._hidden_layer = tf.keras.layers.Dense( + hidden_size, activation=tf.nn.elu) + self._output_layer = tf.keras.layers.Dense(1) + + def call(self, inputs): + """Calculates logits and action. + + Args: + inputs: Observations from a step in the cart-pole environment, of shape + `(batch_size, input_size)` + + Returns: + logits: the logits output by the output layer. This can be viewed as the + likelihood vales of choosing the left (0) action. Shape: + `(batch_size, 1)`. + actions: randomly selected actions ({0, 1}) based on the logits. Shape: + `(batch_size, 1)`. + """ + hidden = self._hidden_layer(inputs) + logits = self._output_layer(hidden) + + left_prob = tf.nn.sigmoid(logits) + action_probs = tf.concat([left_prob, 1.0 - left_prob], 1) + + self._grad_fn = eager.implicit_gradients( + self._get_cross_entropy_and_save_actions) + + actions = tf.multinomial(tf.log(action_probs), 1) + return logits, actions + + def _get_cross_entropy_and_save_actions(self, inputs): + logits, actions = self(inputs) # pylint:disable=not-callable + self._current_actions = actions + labels = 1.0 - tf.cast(actions, tf.float32) + return tf.nn.sigmoid_cross_entropy_with_logits(labels=labels, logits=logits) + + def train(self, cart_pole_env, optimizer, discount_rate, num_games, + max_steps_per_game): + grad_list = None + + step_counts = [] + discounted_rewards = [] + + for _ in range(num_games): + obs = cart_pole_env.reset() + + game_rewards = [] + + for step in range(max_steps_per_game): + grads_and_vars = self._grad_fn(tf.constant([obs], dtype=tf.float32)) + grads, var_list = zip(*grads_and_vars) + actions = self._current_actions.numpy()[0][0] + + if grad_list is None: + grad_list = [[g] for g in grads] + else: + for i in range(len(grads)): + grad_list[i].append(grads[i]) + + obs, reward, done = cart_pole_env.step(actions) + + game_rewards.append(reward) + if reward < 0.1 or done: + step_counts.append(step + 1) + break + + discounted_rewards = eager_append_discounted_rewards( + discounted_rewards, game_rewards, discount_rate) + + discounted_rewards = tf.stack(discounted_rewards) + mean, variance = tf.nn.moments(discounted_rewards, [0]) + normalized_rewards = (discounted_rewards - mean) / tf.sqrt(variance) + + for i in range(len(grad_list)): + g = tf.stack(grad_list[i]) + + r = normalized_rewards + while r.shape.ndims < g.shape.ndims: + r = tf.expand_dims(r, -1) + + grad_list[i] = tf.reduce_mean(g * r, axis=0) + + optimizer.apply_gradients( + zip(grad_list, var_list), global_step=tf.train.get_global_step()) + + return tf.stack(step_counts) + + +def eager_train_model(policy_network, cart_pole_env, optimizer, iterations): + """Trains the policy network for a given number of iterations.""" + mean_steps_per_iteration = [] + + for _ in range(iterations): + steps_per_game = policy_network.train( + cart_pole_env, + optimizer, + discount_rate=0.95, + num_games=20, + max_steps_per_game=200) + mean_steps_per_iteration.append(tf.reduce_mean(steps_per_game)) + + return mean_steps_per_iteration + + +class EagerGymCartpoleEnv(object): + """An env backed by OpenAI Gym's CartPole environment. + + Used to confirm a functional model only. + """ + + def __init__(self): + cart_pole_env = gym.make('CartPole-v1') + cart_pole_env.seed(0) + cart_pole_env.reset() + self.env = cart_pole_env + + def reset(self): + return self.env.reset() + + def step(self, actions): + obs, reward, done, _ = self.env.step(actions) + return obs, reward, done + + +class EagerRandomCartpoleEnv(object): + """An environment that returns random actions and never finishes. + + Used during benchmarking, it will cause training to run a constant number of + steps. + """ + + def reset(self): + return np.random.normal(size=(4,)) + + def step(self, actions): + with tf.control_dependencies([actions]): + random_obs = np.random.normal(size=(4,)) + fixed_reward = 0.001 + done = False + return random_obs, fixed_reward, done + + +def graph_demo_training(): + """Not used in the benchmark. Used to confirm a functional model.""" + with tf.Graph().as_default(): + tf.set_random_seed(0) + + network = GraphPolicyNetwork(hidden_size=5) + network.build((1, 4)) + env = GraphGymCartpoleEnv() + opt = tf.train.AdamOptimizer(0.05) + + train_ops = graph_train_model(network, env, opt, iterations=5) + + with tf.Session() as sess: + sess.run(tf.global_variables_initializer()) + sess.run(tf.local_variables_initializer()) + steps_per_iteration = sess.run(train_ops) + for i, steps in enumerate(steps_per_iteration): + print('Step {} iterations: {}'.format(i, steps)) + + +def eager_demo_training(): + with context.eager_mode(): + network = EagerPolicyNetwork(hidden_size=5) + network.build((1, 4)) + env = EagerGymCartpoleEnv() + opt = tf.train.AdamOptimizer(0.05) + + steps_per_iteration = eager_train_model(network, env, opt, iterations=5) + for i, steps in enumerate(steps_per_iteration): + print('Step {} iterations: {}'.format(i, steps)) + + +class RLCartPoleBenchmark(benchmark_base.ReportingBenchmark): + """Actual benchmark. + + Trains the RL agent a fixed number of times, on random environments that + result in constant number of steps. + """ + + def benchmark_cartpole(self): + + def train_session(sess, ops): + return lambda: sess.run(ops) + + def train_eager(network, env, opt): + return lambda: eager_train_model(network, env, opt, iterations=10) + + for model_size in (10, 100, 1000): + with tf.Graph().as_default(): + network = GraphPolicyNetwork(hidden_size=model_size) + network.build((1, 4)) + env = GraphRandomCartpoleEnv() + opt = tf.train.AdamOptimizer(0.05) + train_ops = graph_train_model(network, env, opt, iterations=10) + + with tf.Session() as sess: + sess.run(tf.global_variables_initializer()) + sess.run(tf.local_variables_initializer()) + + self.time_execution(('cartpole', 'autograph', model_size), + train_session(sess, train_ops), 20) + + with context.eager_mode(): + network = EagerPolicyNetwork(hidden_size=model_size) + network.build((1, 4)) + env = EagerRandomCartpoleEnv() + opt = tf.train.AdamOptimizer(0.05) + + self.time_execution(('cartpole', 'eager', model_size), + train_eager(network, env, opt), 20) + + +if __name__ == '__main__': + tf.test.main() diff --git a/tensorflow/contrib/batching/python/ops/batch_ops.py b/tensorflow/contrib/batching/python/ops/batch_ops.py index 55faad983f2..3e4d0dc1cec 100644 --- a/tensorflow/contrib/batching/python/ops/batch_ops.py +++ b/tensorflow/contrib/batching/python/ops/batch_ops.py @@ -18,8 +18,9 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function -from tensorflow.python.framework import function +from tensorflow.python.eager import function from tensorflow.python.framework import ops +from tensorflow.python.framework import tensor_spec from tensorflow.python.ops import gen_batch_ops # go/tf-wildcard-import # pylint: disable=wildcard-import @@ -101,12 +102,15 @@ def batch_function(num_batch_threads, def decorator(fn): # pylint: disable=missing-docstring def decorated(*args): # pylint: disable=missing-docstring - types = [arg.dtype for arg in args] - @function.Defun(*types) + @function.defun() def computation(*computation_args): return fn(*computation_args) + computation = computation.get_concrete_function( + *[tensor_spec.TensorSpec(dtype=x.dtype, shape=x.shape, name=str(i)) + for i, x in enumerate(args)]) + with ops.name_scope("batch") as name: for a in args: if not isinstance(a, ops.Tensor): @@ -123,7 +127,7 @@ def batch_function(num_batch_threads, f=computation, in_tensors=list(args), captured_tensors=computation.captured_inputs, - Tout=[o.type for o in computation.definition.signature.output_arg]) + Tout=[o.dtype for o in computation.outputs]) return decorated diff --git a/tensorflow/contrib/batching/python/ops/batch_ops_test.py b/tensorflow/contrib/batching/python/ops/batch_ops_test.py index 01ee8703a93..9109b9c1c91 100644 --- a/tensorflow/contrib/batching/python/ops/batch_ops_test.py +++ b/tensorflow/contrib/batching/python/ops/batch_ops_test.py @@ -219,6 +219,7 @@ class BatchOpsTest(test.TestCase): @batch_ops.batch_function(1, 10, 100000) def computation(in_t): + self.assertTrue(in_t.shape is not None) return in_t + 1 inp = array_ops.placeholder(dtype=dtypes.int32, shape=[1]) diff --git a/tensorflow/contrib/bayesflow/python/kernel_tests/monte_carlo_test.py b/tensorflow/contrib/bayesflow/python/kernel_tests/monte_carlo_test.py index 13215ffabf3..8b6ed9f041b 100644 --- a/tensorflow/contrib/bayesflow/python/kernel_tests/monte_carlo_test.py +++ b/tensorflow/contrib/bayesflow/python/kernel_tests/monte_carlo_test.py @@ -81,7 +81,7 @@ class ExpectationImportanceSampleTest(test.TestCase): # Compute E_p[X_1 * X_2 > 0], with X_i the ith component of X ~ p(x). # Should equal 1/2 because p is a spherical Gaussian centered at (0, 0). def indicator(x): - x1_times_x2 = math_ops.reduce_prod(x, reduction_indices=[-1]) + x1_times_x2 = math_ops.reduce_prod(x, axis=[-1]) return 0.5 * (math_ops.sign(x1_times_x2) + 1.0) prob = mc.expectation_importance_sampler( diff --git a/tensorflow/contrib/bayesflow/python/ops/monte_carlo_impl.py b/tensorflow/contrib/bayesflow/python/ops/monte_carlo_impl.py index 18d40fc1dff..e83a5485119 100644 --- a/tensorflow/contrib/bayesflow/python/ops/monte_carlo_impl.py +++ b/tensorflow/contrib/bayesflow/python/ops/monte_carlo_impl.py @@ -353,12 +353,12 @@ def expectation(f, samples, log_prob=None, use_reparametrization=True, def _sample_mean(values): """Mean over sample indices. In this module this is always [0].""" - return math_ops.reduce_mean(values, reduction_indices=[0]) + return math_ops.reduce_mean(values, axis=[0]) def _sample_max(values): """Max over sample indices. In this module this is always [0].""" - return math_ops.reduce_max(values, reduction_indices=[0]) + return math_ops.reduce_max(values, axis=[0]) def _get_samples(dist, z, n, seed): diff --git a/tensorflow/contrib/bigtable/README.md b/tensorflow/contrib/bigtable/README.md index 2c44abed5e1..79052bee35c 100644 --- a/tensorflow/contrib/bigtable/README.md +++ b/tensorflow/contrib/bigtable/README.md @@ -51,25 +51,18 @@ BIGTABLE_TABLE_NAME = '' PREFIX = 'train-' def main(): + tf.enable_eager_execution() + client = tf.contrib.cloud.BigtableClient(GCP_PROJECT_ID, BIGTABLE_INSTANCE_ID) table = client.table(BIGTABLE_TABLE_NAME) dataset = table.keys_by_prefix_dataset(PREFIX) - iterator = dataset.make_initializable_iterator() - get_next_op = iterator.get_next() - with tf.Session() as sess: - print('Initializing the iterator.') - sess.run(iterator.initializer) - print('Retrieving rows:') - row_index = 0 - while True: - try: - row_key = sess.run(get_next_op) - print('Row key %d: %s' % (row_index, row_key)) - row_index += 1 - except tf.errors.OutOfRangeError: - print('Finished reading data!') - break + print('Retrieving rows:') + row_index = 0 + for row_key in dataset: + print('Row key %d: %s' % (row_index, row_key)) + row_index += 1 + print('Finished reading data!') if __name__ == '__main__': main() diff --git a/tensorflow/contrib/bigtable/kernels/test_kernels/bigtable_test_client.cc b/tensorflow/contrib/bigtable/kernels/test_kernels/bigtable_test_client.cc index f083ce6f44b..e95dc577184 100644 --- a/tensorflow/contrib/bigtable/kernels/test_kernels/bigtable_test_client.cc +++ b/tensorflow/contrib/bigtable/kernels/test_kernels/bigtable_test_client.cc @@ -366,6 +366,39 @@ BigtableTestClient::MutateRows( return MakeUnique(request.entries_size()); } +std::unique_ptr> +BigtableTestClient::AsyncMutateRow( + grpc::ClientContext* context, + google::bigtable::v2::MutateRowRequest const& request, + grpc::CompletionQueue* cq) { + LOG(WARNING) << "Call to InMemoryDataClient::" << __func__ + << "(); this will likely cause a crash!"; + return nullptr; +} + +std::unique_ptr<::grpc::ClientAsyncReaderInterface< + ::google::bigtable::v2::SampleRowKeysResponse>> +BigtableTestClient::AsyncSampleRowKeys( + ::grpc::ClientContext* context, + const ::google::bigtable::v2::SampleRowKeysRequest& request, + ::grpc::CompletionQueue* cq, void* tag) { + LOG(WARNING) << "Call to InMemoryDataClient::" << __func__ + << "(); this will likely cause a crash!"; + return nullptr; +} + +std::unique_ptr<::grpc::ClientAsyncReaderInterface< + ::google::bigtable::v2::MutateRowsResponse>> +BigtableTestClient::AsyncMutateRows( + ::grpc::ClientContext* context, + const ::google::bigtable::v2::MutateRowsRequest& request, + ::grpc::CompletionQueue* cq, void* tag) { + LOG(WARNING) << "Call to InMemoryDataClient::" << __func__ + << "(); this will likely cause a crash!"; + return nullptr; +} + std::shared_ptr BigtableTestClient::Channel() { LOG(WARNING) << "Call to InMemoryDataClient::Channel(); this will likely " "cause a crash!"; diff --git a/tensorflow/contrib/bigtable/kernels/test_kernels/bigtable_test_client.h b/tensorflow/contrib/bigtable/kernels/test_kernels/bigtable_test_client.h index dac2b16a216..c4a1f06bc50 100644 --- a/tensorflow/contrib/bigtable/kernels/test_kernels/bigtable_test_client.h +++ b/tensorflow/contrib/bigtable/kernels/test_kernels/bigtable_test_client.h @@ -61,6 +61,25 @@ class BigtableTestClient : public ::google::cloud::bigtable::DataClient { MutateRows(grpc::ClientContext* context, google::bigtable::v2::MutateRowsRequest const& request) override; + std::unique_ptr> + AsyncMutateRow(grpc::ClientContext* context, + google::bigtable::v2::MutateRowRequest const& request, + grpc::CompletionQueue* cq) override; + + std::unique_ptr<::grpc::ClientAsyncReaderInterface< + ::google::bigtable::v2::SampleRowKeysResponse>> + AsyncSampleRowKeys( + ::grpc::ClientContext* context, + const ::google::bigtable::v2::SampleRowKeysRequest& request, + ::grpc::CompletionQueue* cq, void* tag) override; + + std::unique_ptr<::grpc::ClientAsyncReaderInterface< + ::google::bigtable::v2::MutateRowsResponse>> + AsyncMutateRows(::grpc::ClientContext* context, + const ::google::bigtable::v2::MutateRowsRequest& request, + ::grpc::CompletionQueue* cq, void* tag) override; + std::shared_ptr Channel() override; private: diff --git a/tensorflow/contrib/bigtable/python/kernel_tests/bigtable_ops_test.py b/tensorflow/contrib/bigtable/python/kernel_tests/bigtable_ops_test.py index 316da9ebe15..197f5578eb0 100644 --- a/tensorflow/contrib/bigtable/python/kernel_tests/bigtable_ops_test.py +++ b/tensorflow/contrib/bigtable/python/kernel_tests/bigtable_ops_test.py @@ -57,7 +57,7 @@ class BigtableOpsTest(test.TestCase): sess.run(write_op) def runReadKeyTest(self, read_ds): - itr = read_ds.make_initializable_iterator() + itr = dataset_ops.make_initializable_iterator(read_ds) n = itr.get_next() expected = list(self.COMMON_ROW_KEYS) expected.reverse() @@ -78,7 +78,7 @@ class BigtableOpsTest(test.TestCase): self.runReadKeyTest(self._table.keys_by_range_dataset("r1", "r4")) def runScanTest(self, read_ds): - itr = read_ds.make_initializable_iterator() + itr = dataset_ops.make_initializable_iterator(read_ds) n = itr.get_next() expected_keys = list(self.COMMON_ROW_KEYS) expected_keys.reverse() @@ -120,7 +120,7 @@ class BigtableOpsTest(test.TestCase): def testLookup(self): ds = self._table.keys_by_prefix_dataset("r") ds = ds.apply(self._table.lookup_columns(cf1="c1")) - itr = ds.make_initializable_iterator() + itr = dataset_ops.make_initializable_iterator(ds) n = itr.get_next() expected_keys = list(self.COMMON_ROW_KEYS) expected_values = list(self.COMMON_VALUES) @@ -141,7 +141,7 @@ class BigtableOpsTest(test.TestCase): def testSampleKeys(self): ds = self._table.sample_keys() - itr = ds.make_initializable_iterator() + itr = dataset_ops.make_initializable_iterator(ds) n = itr.get_next() expected_key = self.COMMON_ROW_KEYS[0] with self.cached_session() as sess: @@ -161,7 +161,7 @@ class BigtableOpsTest(test.TestCase): sess.run(n) def runSampleKeyPairsTest(self, ds, expected_key_pairs): - itr = ds.make_initializable_iterator() + itr = dataset_ops.make_initializable_iterator(ds) n = itr.get_next() with self.cached_session() as sess: self._writeCommonValues(sess) @@ -218,7 +218,7 @@ class BigtableOpsTest(test.TestCase): def testSampleKeyPairsPrefixAndStartKey(self): ds = bigtable_api._BigtableSampleKeyPairsDataset( self._table, prefix="r", start="r1", end="") - itr = ds.make_initializable_iterator() + itr = dataset_ops.make_initializable_iterator(ds) with self.cached_session() as sess: with self.assertRaises(errors.InvalidArgumentError): sess.run(itr.initializer) @@ -226,14 +226,14 @@ class BigtableOpsTest(test.TestCase): def testSampleKeyPairsPrefixAndEndKey(self): ds = bigtable_api._BigtableSampleKeyPairsDataset( self._table, prefix="r", start="", end="r3") - itr = ds.make_initializable_iterator() + itr = dataset_ops.make_initializable_iterator(ds) with self.cached_session() as sess: with self.assertRaises(errors.InvalidArgumentError): sess.run(itr.initializer) def testParallelScanPrefix(self): ds = self._table.parallel_scan_prefix(prefix="r", cf1="c1") - itr = ds.make_initializable_iterator() + itr = dataset_ops.make_initializable_iterator(ds) n = itr.get_next() with self.cached_session() as sess: self._writeCommonValues(sess) @@ -251,7 +251,7 @@ class BigtableOpsTest(test.TestCase): def testParallelScanRange(self): ds = self._table.parallel_scan_range(start="r1", end="r4", cf1="c1") - itr = ds.make_initializable_iterator() + itr = dataset_ops.make_initializable_iterator(ds) n = itr.get_next() with self.cached_session() as sess: self._writeCommonValues(sess) diff --git a/tensorflow/contrib/bigtable/python/ops/bigtable_api.py b/tensorflow/contrib/bigtable/python/ops/bigtable_api.py index 7c87b0daeb0..9f97934193d 100644 --- a/tensorflow/contrib/bigtable/python/ops/bigtable_api.py +++ b/tensorflow/contrib/bigtable/python/ops/bigtable_api.py @@ -222,7 +222,7 @@ class BigtableTable(object): A `tf.data.Dataset`. containing `tf.string` Tensors corresponding to all of the row keys matching that prefix. """ - return _BigtablePrefixKeyDataset(self, prefix) + return dataset_ops.DatasetV1Adapter(_BigtablePrefixKeyDataset(self, prefix)) def sample_keys(self): """Retrieves a sampling of row keys from the Bigtable table. @@ -234,7 +234,7 @@ class BigtableTable(object): Returns: A `tf.data.Dataset` returning string row keys. """ - return _BigtableSampleKeysDataset(self) + return dataset_ops.DatasetV1Adapter(_BigtableSampleKeysDataset(self)) def scan_prefix(self, prefix, probability=None, columns=None, **kwargs): """Retrieves row (including values) from the Bigtable service. @@ -279,7 +279,8 @@ class BigtableTable(object): """ probability = _normalize_probability(probability) normalized = _normalize_columns(columns, kwargs) - return _BigtableScanDataset(self, prefix, "", "", normalized, probability) + return dataset_ops.DatasetV1Adapter( + _BigtableScanDataset(self, prefix, "", "", normalized, probability)) def scan_range(self, start, end, probability=None, columns=None, **kwargs): """Retrieves rows (including values) from the Bigtable service. @@ -324,7 +325,8 @@ class BigtableTable(object): """ probability = _normalize_probability(probability) normalized = _normalize_columns(columns, kwargs) - return _BigtableScanDataset(self, "", start, end, normalized, probability) + return dataset_ops.DatasetV1Adapter( + _BigtableScanDataset(self, "", start, end, normalized, probability)) def parallel_scan_prefix(self, prefix, @@ -380,7 +382,8 @@ class BigtableTable(object): """ probability = _normalize_probability(probability) normalized = _normalize_columns(columns, kwargs) - ds = _BigtableSampleKeyPairsDataset(self, prefix, "", "") + ds = dataset_ops.DatasetV1Adapter( + _BigtableSampleKeyPairsDataset(self, prefix, "", "")) return self._make_parallel_scan_dataset(ds, num_parallel_scans, probability, normalized) @@ -442,7 +445,8 @@ class BigtableTable(object): """ probability = _normalize_probability(probability) normalized = _normalize_columns(columns, kwargs) - ds = _BigtableSampleKeyPairsDataset(self, "", start, end) + ds = dataset_ops.DatasetV1Adapter( + _BigtableSampleKeyPairsDataset(self, "", start, end)) return self._make_parallel_scan_dataset(ds, num_parallel_scans, probability, normalized) diff --git a/tensorflow/contrib/boosted_trees/estimator_batch/BUILD b/tensorflow/contrib/boosted_trees/estimator_batch/BUILD index 14b6fc4ac26..d3b23d949ee 100644 --- a/tensorflow/contrib/boosted_trees/estimator_batch/BUILD +++ b/tensorflow/contrib/boosted_trees/estimator_batch/BUILD @@ -132,6 +132,7 @@ py_library( srcs = ["estimator.py"], srcs_version = "PY2AND3", deps = [ + ":custom_loss_head", ":estimator_utils", ":model", "//tensorflow/contrib/boosted_trees:losses", diff --git a/tensorflow/contrib/boosted_trees/estimator_batch/custom_export_strategy.py b/tensorflow/contrib/boosted_trees/estimator_batch/custom_export_strategy.py index a3df272e692..b314b4d74df 100644 --- a/tensorflow/contrib/boosted_trees/estimator_batch/custom_export_strategy.py +++ b/tensorflow/contrib/boosted_trees/estimator_batch/custom_export_strategy.py @@ -41,7 +41,8 @@ def make_custom_export_strategy(name, convert_fn, feature_columns, export_input_fn, - use_core_columns=False): + use_core_columns=False, + feature_engineering_fn=None): """Makes custom exporter of GTFlow tree format. Args: @@ -52,6 +53,7 @@ def make_custom_export_strategy(name, export_input_fn: A function that takes no arguments and returns an `InputFnOps`. use_core_columns: A boolean, whether core feature columns were used. + feature_engineering_fn: Feature eng function to be called on the input. Returns: An `ExportStrategy`. @@ -59,9 +61,12 @@ def make_custom_export_strategy(name, base_strategy = saved_model_export_utils.make_export_strategy( serving_input_fn=export_input_fn, strip_default_attrs=True) input_fn = export_input_fn() + features = input_fn.features + if feature_engineering_fn is not None: + features, _ = feature_engineering_fn(features, labels=None) (sorted_feature_names, dense_floats, sparse_float_indices, _, _, sparse_int_indices, _, _) = gbdt_batch.extract_features( - input_fn.features, feature_columns, use_core_columns) + features, feature_columns, use_core_columns) def export_fn(estimator, export_dir, checkpoint_path=None, eval_result=None): """A wrapper to export to SavedModel, and convert it to other formats.""" diff --git a/tensorflow/contrib/boosted_trees/estimator_batch/dnn_tree_combined_estimator.py b/tensorflow/contrib/boosted_trees/estimator_batch/dnn_tree_combined_estimator.py index ca73e4af2fb..358404cd946 100644 --- a/tensorflow/contrib/boosted_trees/estimator_batch/dnn_tree_combined_estimator.py +++ b/tensorflow/contrib/boosted_trees/estimator_batch/dnn_tree_combined_estimator.py @@ -36,7 +36,7 @@ from tensorflow.contrib.learn.python.learn.estimators import estimator from tensorflow.contrib.learn.python.learn.estimators import head as head_lib from tensorflow.python.estimator import estimator as core_estimator from tensorflow.contrib.learn.python.learn.estimators import model_fn -from tensorflow.python.feature_column import feature_column as feature_column_lib +from tensorflow.python.feature_column import feature_column_lib from tensorflow.python.framework import ops from tensorflow.python.ops import array_ops from tensorflow.python.ops import control_flow_ops diff --git a/tensorflow/contrib/boosted_trees/estimator_batch/estimator.py b/tensorflow/contrib/boosted_trees/estimator_batch/estimator.py index 38d19976ef3..a178820841c 100644 --- a/tensorflow/contrib/boosted_trees/estimator_batch/estimator.py +++ b/tensorflow/contrib/boosted_trees/estimator_batch/estimator.py @@ -18,6 +18,8 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function +import functools + from tensorflow.contrib.boosted_trees.estimator_batch import model from tensorflow.contrib.boosted_trees.python.utils import losses from tensorflow.contrib.learn.python.learn.estimators import estimator @@ -26,7 +28,8 @@ from tensorflow.python.estimator.canned import head as core_head_lib from tensorflow.python.estimator import estimator as core_estimator from tensorflow.python.ops import math_ops from tensorflow.python.ops.losses import losses as core_losses - +from tensorflow.contrib.boosted_trees.estimator_batch import custom_loss_head +from tensorflow.python.ops import array_ops # ================== Old estimator interface=================================== # The estimators below were designed for old feature columns and old estimator @@ -414,6 +417,108 @@ class GradientBoostedDecisionTreeRanker(estimator.Estimator): config=config, feature_engineering_fn=feature_engineering_fn) +# When using this estimator, make sure to regularize the hessian (at least l2, +# min_node_weight)! +# TODO(nponomareva): extend to take multiple quantiles in one go. +class GradientBoostedDecisionTreeQuantileRegressor(estimator.Estimator): + """An estimator that does quantile regression and returns quantile estimates. + """ + + def __init__(self, + learner_config, + examples_per_layer, + quantiles, + label_dimension=1, + num_trees=None, + feature_columns=None, + weight_column_name=None, + model_dir=None, + config=None, + feature_engineering_fn=None, + logits_modifier_function=None, + center_bias=True, + use_core_libs=False, + output_leaf_index=False, + override_global_step_value=None, + num_quantiles=100): + """Initializes a GradientBoostedDecisionTreeQuantileRegressor instance. + + Args: + learner_config: A config for the learner. + examples_per_layer: Number of examples to accumulate before growing a + layer. It can also be a function that computes the number of examples + based on the depth of the layer that's being built. + quantiles: a list of quantiles for the loss, each between 0 and 1. + label_dimension: Dimension of regression label. This is the size + of the last dimension of the labels `Tensor` (typically, this has shape + `[batch_size, label_dimension]`). When label_dimension>1, it is + recommended to use multiclass strategy diagonal hessian or full hessian. + num_trees: An int, number of trees to build. + feature_columns: A list of feature columns. + weight_column_name: Name of the column for weights, or None if not + weighted. + model_dir: Directory for model exports, etc. + config: `RunConfig` object to configure the runtime settings. + feature_engineering_fn: Feature engineering function. Takes features and + labels which are the output of `input_fn` and returns features and + labels which will be fed into the model. + logits_modifier_function: A modifier function for the logits. + center_bias: Whether a separate tree should be created for first fitting + the bias. + use_core_libs: Whether feature columns and loss are from the core (as + opposed to contrib) version of tensorflow. + output_leaf_index: whether to output leaf indices along with predictions + during inference. The leaf node indexes are available in predictions + dict by the key 'leaf_index'. For example, + result_dict = classifier.predict(...) + for example_prediction_result in result_dict: + # access leaf index list by example_prediction_result["leaf_index"] + # which contains one leaf index per tree + override_global_step_value: If after the training is done, global step + value must be reset to this value. This should be used to reset global + step to a number > number of steps used to train the current ensemble. + For example, the usual way is to train a number of trees and set a very + large number of training steps. When the training is done (number of + trees were trained), this parameter can be used to set the global step + to a large value, making it look like that number of training steps ran. + If None, no override of global step will happen. + num_quantiles: Number of quantiles to build for numeric feature values. + """ + + if len(quantiles) > 1: + raise ValueError('For now, just one quantile per estimator is supported') + + def _quantile_regression_head(quantile): + # Use quantile regression. + head = custom_loss_head.CustomLossHead( + loss_fn=functools.partial( + losses.per_example_quantile_regression_loss, quantile=quantile), + link_fn=array_ops.identity, + logit_dimension=label_dimension) + return head + + learner_config.num_classes = max(2, label_dimension) + + super(GradientBoostedDecisionTreeQuantileRegressor, self).__init__( + model_fn=model.model_builder, + params={ + 'head': _quantile_regression_head(quantiles[0]), + 'feature_columns': feature_columns, + 'learner_config': learner_config, + 'num_trees': num_trees, + 'weight_column_name': weight_column_name, + 'examples_per_layer': examples_per_layer, + 'logits_modifier_function': logits_modifier_function, + 'center_bias': center_bias, + 'use_core_libs': use_core_libs, + 'output_leaf_index': False, + 'override_global_step_value': override_global_step_value, + 'num_quantiles': num_quantiles, + }, + model_dir=model_dir, + config=config, + feature_engineering_fn=feature_engineering_fn) + # ================== New Estimator interface=================================== # The estimators below use new core Estimator interface and must be used with # new feature columns and heads. @@ -437,12 +542,42 @@ def core_multiclass_head( # pylint:disable=protected-access head_fn = core_head_lib._multi_class_head_with_softmax_cross_entropy_loss( - n_classes=n_classes, loss_fn=loss_fn, loss_reduction=loss_reduction) + n_classes=n_classes, + loss_fn=loss_fn, + loss_reduction=loss_reduction, + weight_column=weight_column) # pylint:enable=protected-access return head_fn +# For quantile regression, use this head with Core..Estimator, or use +# Core..QuantileRegressor directly, +def core_quantile_regression_head( + quantiles, + label_dimension=1, + weight_column=None, + loss_reduction=core_losses.Reduction.SUM_OVER_NONZERO_WEIGHTS): + """Core head for quantile regression problems.""" + + def loss_fn(labels, logits): + result = losses.per_example_quantile_regression_loss( + labels=labels, + predictions=logits, + weights=weight_column, + quantile=quantiles) + return result[0] + + # pylint:disable=protected-access + head_fn = core_head_lib._regression_head( + label_dimension=label_dimension, + loss_fn=loss_fn, + loss_reduction=loss_reduction, + weight_column=weight_column) + # pylint:enable=protected-access + return head_fn + + class CoreGradientBoostedDecisionTreeEstimator(core_estimator.Estimator): """An estimator using gradient boosted decision trees. @@ -606,3 +741,104 @@ class CoreGradientBoostedDecisionTreeRanker(core_estimator.Estimator): super(CoreGradientBoostedDecisionTreeRanker, self).__init__( model_fn=_model_fn, model_dir=model_dir, config=config) + + +# When using this estimator, make sure to regularize the hessian (at least l2, +# min_node_weight)! +# TODO(nponomareva): extend to take multiple quantiles in one go. +class CoreGradientBoostedDecisionTreeQuantileRegressor( + core_estimator.Estimator): + """An estimator that does quantile regression and returns quantile estimates. + """ + + def __init__(self, + learner_config, + examples_per_layer, + quantiles, + label_dimension=1, + num_trees=None, + feature_columns=None, + weight_column_name=None, + model_dir=None, + config=None, + label_keys=None, + feature_engineering_fn=None, + logits_modifier_function=None, + center_bias=True, + output_leaf_index=False, + num_quantiles=100): + """Initializes a core version of GradientBoostedDecisionTreeEstimator. + + Args: + learner_config: A config for the learner. + examples_per_layer: Number of examples to accumulate before growing a + layer. It can also be a function that computes the number of examples + based on the depth of the layer that's being built. + quantiles: a list of quantiles for the loss, each between 0 and 1. + label_dimension: Dimension of regression label. This is the size + of the last dimension of the labels `Tensor` (typically, this has shape + `[batch_size, label_dimension]`). When label_dimension>1, it is + recommended to use multiclass strategy diagonal hessian or full hessian. + num_trees: An int, number of trees to build. + feature_columns: A list of feature columns. + weight_column_name: Name of the column for weights, or None if not + weighted. + model_dir: Directory for model exports, etc. + config: `RunConfig` object to configure the runtime settings. + label_keys: Optional list of strings with size `[n_classes]` defining the + label vocabulary. Only supported for `n_classes` > 2. + feature_engineering_fn: Feature engineering function. Takes features and + labels which are the output of `input_fn` and returns features and + labels which will be fed into the model. + logits_modifier_function: A modifier function for the logits. + center_bias: Whether a separate tree should be created for first fitting + the bias. + output_leaf_index: whether to output leaf indices along with predictions + during inference. The leaf node indexes are available in predictions + dict by the key 'leaf_index'. For example, + result_dict = classifier.predict(...) + for example_prediction_result in result_dict: + # access leaf index list by example_prediction_result["leaf_index"] + # which contains one leaf index per tree + num_quantiles: Number of quantiles to build for numeric feature values. + """ + if len(quantiles) > 1: + raise ValueError('For now, just one quantile per estimator is supported') + + def _model_fn(features, labels, mode, config): + return model.model_builder( + features=features, + labels=labels, + mode=mode, + config=config, + params={ + 'head': + core_quantile_regression_head( + quantiles[0], label_dimension=label_dimension), + 'feature_columns': + feature_columns, + 'learner_config': + learner_config, + 'num_trees': + num_trees, + 'weight_column_name': + weight_column_name, + 'examples_per_layer': + examples_per_layer, + 'center_bias': + center_bias, + 'logits_modifier_function': + logits_modifier_function, + 'use_core_libs': + True, + 'output_leaf_index': + output_leaf_index, + 'override_global_step_value': + None, + 'num_quantiles': + num_quantiles, + }, + output_type=model.ModelBuilderOutputType.ESTIMATOR_SPEC) + + super(CoreGradientBoostedDecisionTreeQuantileRegressor, self).__init__( + model_fn=_model_fn, model_dir=model_dir, config=config) diff --git a/tensorflow/contrib/boosted_trees/estimator_batch/estimator_test.py b/tensorflow/contrib/boosted_trees/estimator_batch/estimator_test.py index c155128c0e4..ee052ac6038 100644 --- a/tensorflow/contrib/boosted_trees/estimator_batch/estimator_test.py +++ b/tensorflow/contrib/boosted_trees/estimator_batch/estimator_test.py @@ -25,6 +25,7 @@ from tensorflow.contrib.boosted_trees.proto import learner_pb2 from tensorflow.contrib.layers.python.layers import feature_column as contrib_feature_column from tensorflow.contrib.learn.python.learn.estimators import run_config from tensorflow.python.estimator.canned import head as head_lib +from tensorflow.python.estimator.inputs import numpy_io from tensorflow.python.feature_column import feature_column_lib as core_feature_column from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes @@ -47,8 +48,8 @@ def _multiclass_train_input_fn(): features = { "x": constant_op.constant([[2.], [1.], [1.], [5.], [3.5], [4.6], [3.5]]) } - label = constant_op.constant( - [[1], [0], [0], [2], [2], [0], [1]], dtype=dtypes.int32) + label = constant_op.constant([[1], [0], [0], [2], [2], [0], [1]], + dtype=dtypes.int32) return features, label @@ -77,6 +78,59 @@ def _infer_ranking_train_input_fn(): return features, None +_QUANTILE_REGRESSION_SIZE = 1000 + + +def _quantile_regression_input_fns(two_dimension=False): + # The data generation is taken from + # http://scikit-learn.org/stable/auto_examples/ensemble/plot_gradient_boosting_quantile.html + np.random.seed(1) + + def f(x): + """The function to predict.""" + return x * np.sin(x) + + def g(x): + """The function to predict.""" + return x * np.cos(x) + + # Training data. + x = np.atleast_2d(np.random.uniform(0, 10.0, + size=_QUANTILE_REGRESSION_SIZE)).T + x = x.astype(np.float32) + + # Labels. + if not two_dimension: + y = f(x).ravel() + else: + y = np.column_stack((f(x).ravel(), g(x).ravel())) + + # Add random noise. + dy = 1.5 + 1.0 * np.random.random(y.shape) + noise = np.random.normal(0, dy) + y += noise + y_original = y.astype(np.float32) + if not two_dimension: + y = y.reshape(_QUANTILE_REGRESSION_SIZE, 1) + + train_input_fn = numpy_io.numpy_input_fn( + x=x, + y=y, + batch_size=_QUANTILE_REGRESSION_SIZE, + num_epochs=None, + shuffle=True) + + # Test on the training data to make sure the predictions are calibrated. + test_input_fn = numpy_io.numpy_input_fn( + x=x, + y=y, + batch_size=_QUANTILE_REGRESSION_SIZE, + num_epochs=1, + shuffle=False) + + return train_input_fn, test_input_fn, y_original + + class BoostedTreeEstimatorTest(test_util.TensorFlowTestCase): def setUp(self): @@ -341,6 +395,130 @@ class BoostedTreeEstimatorTest(test_util.TensorFlowTestCase): for prediction_dict in result_iter: self.assertTrue("classes" in prediction_dict) + # One dimensional quantile regression. + def testQuantileRegression(self): + learner_config = learner_pb2.LearnerConfig() + learner_config.num_classes = 2 + learner_config.constraints.max_tree_depth = 3 + learner_config.growing_mode = learner_pb2.LearnerConfig.WHOLE_TREE + learner_config.constraints.min_node_weight = 1 / _QUANTILE_REGRESSION_SIZE + learner_config.regularization.l2 = 1.0 / _QUANTILE_REGRESSION_SIZE + learner_config.regularization.l1 = 1.0 / _QUANTILE_REGRESSION_SIZE + learner_config.regularization.tree_complexity = ( + 1.0 / _QUANTILE_REGRESSION_SIZE) + + train_input_fn, test_input_fn, y = _quantile_regression_input_fns() + + # 95% percentile. + model_upper = estimator.GradientBoostedDecisionTreeQuantileRegressor( + quantiles=[0.95], + learner_config=learner_config, + num_trees=100, + examples_per_layer=_QUANTILE_REGRESSION_SIZE, + center_bias=False) + + model_upper.fit(input_fn=train_input_fn, steps=1000) + result_iter = model_upper.predict(input_fn=test_input_fn) + upper = [] + for prediction_dict in result_iter: + upper.append(prediction_dict["scores"]) + + frac_below_upper = round(1. * np.count_nonzero(upper > y) / len(y), 3) + # +/- 3% + self.assertTrue(frac_below_upper >= 0.92) + self.assertTrue(frac_below_upper <= 0.98) + + train_input_fn, test_input_fn, _ = _quantile_regression_input_fns() + model_lower = estimator.GradientBoostedDecisionTreeQuantileRegressor( + quantiles=[0.05], + learner_config=learner_config, + num_trees=100, + examples_per_layer=_QUANTILE_REGRESSION_SIZE, + center_bias=False) + + model_lower.fit(input_fn=train_input_fn, steps=1000) + result_iter = model_lower.predict(input_fn=test_input_fn) + lower = [] + for prediction_dict in result_iter: + lower.append(prediction_dict["scores"]) + + frac_above_lower = round(1. * np.count_nonzero(lower < y) / len(y), 3) + # +/- 3% + self.assertTrue(frac_above_lower >= 0.92) + self.assertTrue(frac_above_lower <= 0.98) + + # Multi-dimensional quantile regression. + def testQuantileRegressionMultiDimLabel(self): + learner_config = learner_pb2.LearnerConfig() + learner_config.num_classes = 2 + learner_config.constraints.max_tree_depth = 3 + learner_config.growing_mode = learner_pb2.LearnerConfig.WHOLE_TREE + learner_config.constraints.min_node_weight = 1 / _QUANTILE_REGRESSION_SIZE + learner_config.regularization.l2 = 1.0 / _QUANTILE_REGRESSION_SIZE + learner_config.regularization.l1 = 1.0 / _QUANTILE_REGRESSION_SIZE + learner_config.regularization.tree_complexity = ( + 1.0 / _QUANTILE_REGRESSION_SIZE) + + train_input_fn, test_input_fn, y = _quantile_regression_input_fns( + two_dimension=True) + + # 95% percentile. + model_upper = estimator.GradientBoostedDecisionTreeQuantileRegressor( + quantiles=[0.95], + learner_config=learner_config, + label_dimension=2, + num_trees=100, + examples_per_layer=_QUANTILE_REGRESSION_SIZE, + center_bias=False) + + model_upper.fit(input_fn=train_input_fn, steps=1000) + result_iter = model_upper.predict(input_fn=test_input_fn) + upper = [] + for prediction_dict in result_iter: + upper.append(prediction_dict["scores"]) + + count_below_upper = np.count_nonzero(upper > y, axis=0) + count_both_below_upper = np.count_nonzero(np.prod(upper > y, axis=1)) + frac_below_upper_0 = round(1. * count_below_upper[0] / len(y), 3) + frac_below_upper_1 = round(1. * count_below_upper[1] / len(y), 3) + frac_both_below_upper = round(1. * count_both_below_upper / len(y), 3) + # +/- 3% + self.assertTrue(frac_below_upper_0 >= 0.92) + self.assertTrue(frac_below_upper_0 <= 0.98) + self.assertTrue(frac_below_upper_1 >= 0.92) + self.assertTrue(frac_below_upper_1 <= 0.98) + self.assertTrue(frac_both_below_upper >= 0.92) + self.assertTrue(frac_both_below_upper <= 0.98) + + train_input_fn, test_input_fn, _ = _quantile_regression_input_fns( + two_dimension=True) + model_lower = estimator.GradientBoostedDecisionTreeQuantileRegressor( + quantiles=[0.05], + learner_config=learner_config, + label_dimension=2, + num_trees=100, + examples_per_layer=_QUANTILE_REGRESSION_SIZE, + center_bias=False) + + model_lower.fit(input_fn=train_input_fn, steps=1000) + result_iter = model_lower.predict(input_fn=test_input_fn) + lower = [] + for prediction_dict in result_iter: + lower.append(prediction_dict["scores"]) + + count_above_lower = np.count_nonzero(lower < y, axis=0) + count_both_aboce_lower = np.count_nonzero(np.prod(lower < y, axis=1)) + frac_above_lower_0 = round(1. * count_above_lower[0] / len(y), 3) + frac_above_lower_1 = round(1. * count_above_lower[1] / len(y), 3) + frac_both_above_lower = round(1. * count_both_aboce_lower / len(y), 3) + # +/- 3% + self.assertTrue(frac_above_lower_0 >= 0.92) + self.assertTrue(frac_above_lower_0 <= 0.98) + self.assertTrue(frac_above_lower_1 >= 0.92) + self.assertTrue(frac_above_lower_1 <= 0.98) + self.assertTrue(frac_both_above_lower >= 0.92) + self.assertTrue(frac_both_above_lower <= 0.98) + class CoreGradientBoostedDecisionTreeEstimators(test_util.TensorFlowTestCase): @@ -489,8 +667,8 @@ class CoreGradientBoostedDecisionTreeEstimators(test_util.TensorFlowTestCase): feature_columns = [ core_feature_column.weighted_categorical_column( - categorical_column=core_feature_column. - categorical_column_with_vocabulary_list( + categorical_column=core_feature_column + .categorical_column_with_vocabulary_list( key="word", vocabulary_list=["the", "cat", "dog"]), weight_feature_key="weight") ] @@ -509,8 +687,8 @@ class CoreGradientBoostedDecisionTreeEstimators(test_util.TensorFlowTestCase): # Weights for the words are 5 - cat, 6- dog and 1 -the. features_dict["word"] = sparse_tensor.SparseTensor( indices=[[0, 0], [0, 1], [1, 0], [3, 0]], - values=constant_op.constant( - ["the", "cat", "dog", "the"], dtype=dtypes.string), + values=constant_op.constant(["the", "cat", "dog", "the"], + dtype=dtypes.string), dense_shape=[4, 3]) features_dict["weight"] = sparse_tensor.SparseTensor( indices=[[0, 0], [0, 1], [1, 0], [3, 0]], @@ -534,6 +712,132 @@ class CoreGradientBoostedDecisionTreeEstimators(test_util.TensorFlowTestCase): est.evaluate(input_fn=input_fn, steps=1) est.predict(input_fn=input_fn) + # One dimensional quantile regression. + def testQuantileRegression(self): + learner_config = learner_pb2.LearnerConfig() + learner_config.num_classes = 2 + learner_config.constraints.max_tree_depth = 3 + learner_config.growing_mode = learner_pb2.LearnerConfig.WHOLE_TREE + learner_config.constraints.min_node_weight = 1 / _QUANTILE_REGRESSION_SIZE + learner_config.regularization.l2 = 1.0 / _QUANTILE_REGRESSION_SIZE + learner_config.regularization.l1 = 1.0 / _QUANTILE_REGRESSION_SIZE + learner_config.regularization.tree_complexity = ( + 1.0 / _QUANTILE_REGRESSION_SIZE) + + train_input_fn, test_input_fn, y = _quantile_regression_input_fns() + y = y.reshape(_QUANTILE_REGRESSION_SIZE, 1) + + # 95% percentile. + model_upper = estimator.CoreGradientBoostedDecisionTreeQuantileRegressor( + quantiles=[0.95], + learner_config=learner_config, + num_trees=100, + examples_per_layer=_QUANTILE_REGRESSION_SIZE, + center_bias=False) + + model_upper.train(input_fn=train_input_fn, steps=1000) + result_iter = model_upper.predict(input_fn=test_input_fn) + upper = [] + for prediction_dict in result_iter: + upper.append(prediction_dict["predictions"]) + + frac_below_upper = round(1. * np.count_nonzero(upper > y) / len(y), 3) + # +/- 3% + self.assertTrue(frac_below_upper >= 0.92) + self.assertTrue(frac_below_upper <= 0.98) + + train_input_fn, test_input_fn, _ = _quantile_regression_input_fns() + model_lower = estimator.CoreGradientBoostedDecisionTreeQuantileRegressor( + quantiles=[0.05], + learner_config=learner_config, + num_trees=100, + examples_per_layer=_QUANTILE_REGRESSION_SIZE, + center_bias=False) + + model_lower.train(input_fn=train_input_fn, steps=1000) + result_iter = model_lower.predict(input_fn=test_input_fn) + lower = [] + for prediction_dict in result_iter: + lower.append(prediction_dict["predictions"]) + + frac_above_lower = round(1. * np.count_nonzero(lower < y) / len(y), 3) + # +/- 3% + self.assertTrue(frac_above_lower >= 0.92) + self.assertTrue(frac_above_lower <= 0.98) + + # Multi-dimensional quantile regression. + def testQuantileRegressionMultiDimLabel(self): + learner_config = learner_pb2.LearnerConfig() + learner_config.num_classes = 2 + learner_config.constraints.max_tree_depth = 3 + learner_config.growing_mode = learner_pb2.LearnerConfig.WHOLE_TREE + learner_config.constraints.min_node_weight = 1 / _QUANTILE_REGRESSION_SIZE + learner_config.regularization.l2 = 1.0 / _QUANTILE_REGRESSION_SIZE + learner_config.regularization.l1 = 1.0 / _QUANTILE_REGRESSION_SIZE + learner_config.regularization.tree_complexity = ( + 1.0 / _QUANTILE_REGRESSION_SIZE) + + train_input_fn, test_input_fn, y = _quantile_regression_input_fns( + two_dimension=True) + y = y.reshape(_QUANTILE_REGRESSION_SIZE, 2) + + # 95% percentile. + model_upper = estimator.CoreGradientBoostedDecisionTreeQuantileRegressor( + quantiles=[0.95], + learner_config=learner_config, + num_trees=100, + label_dimension=2, + examples_per_layer=_QUANTILE_REGRESSION_SIZE, + center_bias=False) + + model_upper.train(input_fn=train_input_fn, steps=1000) + result_iter = model_upper.predict(input_fn=test_input_fn) + upper = [] + for prediction_dict in result_iter: + upper.append(prediction_dict["predictions"]) + + count_below_upper = np.count_nonzero(upper > y, axis=0) + count_both_below_upper = np.count_nonzero(np.prod(upper > y, axis=1)) + frac_below_upper_0 = round(1. * count_below_upper[0] / len(y), 3) + frac_below_upper_1 = round(1. * count_below_upper[1] / len(y), 3) + frac_both_below_upper = round(1. * count_both_below_upper / len(y), 3) + # +/- 3% + self.assertTrue(frac_below_upper_0 >= 0.92) + self.assertTrue(frac_below_upper_0 <= 0.98) + self.assertTrue(frac_below_upper_1 >= 0.92) + self.assertTrue(frac_below_upper_1 <= 0.98) + self.assertTrue(frac_both_below_upper >= 0.92) + self.assertTrue(frac_both_below_upper <= 0.98) + + train_input_fn, test_input_fn, _ = _quantile_regression_input_fns( + two_dimension=True) + model_lower = estimator.CoreGradientBoostedDecisionTreeQuantileRegressor( + quantiles=[0.05], + learner_config=learner_config, + num_trees=100, + label_dimension=2, + examples_per_layer=_QUANTILE_REGRESSION_SIZE, + center_bias=False) + + model_lower.train(input_fn=train_input_fn, steps=1000) + result_iter = model_lower.predict(input_fn=test_input_fn) + lower = [] + for prediction_dict in result_iter: + lower.append(prediction_dict["predictions"]) + + count_above_lower = np.count_nonzero(lower < y, axis=0) + count_both_aboce_lower = np.count_nonzero(np.prod(lower < y, axis=1)) + frac_above_lower_0 = round(1. * count_above_lower[0] / len(y), 3) + frac_above_lower_1 = round(1. * count_above_lower[1] / len(y), 3) + frac_both_above_lower = round(1. * count_both_aboce_lower / len(y), 3) + # +/- 3% + self.assertTrue(frac_above_lower_0 >= 0.92) + self.assertTrue(frac_above_lower_0 <= 0.98) + self.assertTrue(frac_above_lower_1 >= 0.92) + self.assertTrue(frac_above_lower_1 <= 0.98) + self.assertTrue(frac_both_above_lower >= 0.92) + self.assertTrue(frac_both_above_lower <= 0.98) + if __name__ == "__main__": googletest.main() diff --git a/tensorflow/contrib/boosted_trees/examples/boston.py b/tensorflow/contrib/boosted_trees/examples/boston.py index 54c4ff059e3..09b240a7006 100644 --- a/tensorflow/contrib/boosted_trees/examples/boston.py +++ b/tensorflow/contrib/boosted_trees/examples/boston.py @@ -90,13 +90,13 @@ def _make_experiment_fn(output_dir): (x_train, y_train), (x_test, y_test) = tf.keras.datasets.boston_housing.load_data() - train_input_fn = tf.estimator.inputs.numpy_input_fn( + train_input_fn = tf.compat.v1.estimator.inputs.numpy_input_fn( x={"x": x_train}, y=y_train, batch_size=FLAGS.batch_size, num_epochs=None, shuffle=True) - eval_input_fn = tf.estimator.inputs.numpy_input_fn( + eval_input_fn = tf.compat.v1.estimator.inputs.numpy_input_fn( x={"x": x_test}, y=y_test, num_epochs=1, shuffle=False) feature_columns = [ diff --git a/tensorflow/contrib/boosted_trees/examples/boston_combined.py b/tensorflow/contrib/boosted_trees/examples/boston_combined.py index e04b56afbfd..d640af354f5 100644 --- a/tensorflow/contrib/boosted_trees/examples/boston_combined.py +++ b/tensorflow/contrib/boosted_trees/examples/boston_combined.py @@ -80,13 +80,13 @@ def _make_experiment_fn(output_dir): (x_train, y_train), (x_test, y_test) = tf.keras.datasets.boston_housing.load_data() - train_input_fn = tf.estimator.inputs.numpy_input_fn( + train_input_fn = tf.compat.v1.estimator.inputs.numpy_input_fn( x={"x": x_train}, y=y_train, batch_size=FLAGS.batch_size, num_epochs=None, shuffle=True) - eval_input_fn = tf.estimator.inputs.numpy_input_fn( + eval_input_fn = tf.compat.v1.estimator.inputs.numpy_input_fn( x={"x": x_test}, y=y_test, num_epochs=1, shuffle=False) feature_columns = [ diff --git a/tensorflow/contrib/boosted_trees/kernels/split_handler_ops.cc b/tensorflow/contrib/boosted_trees/kernels/split_handler_ops.cc index 8edb5d6c640..6d78e27e8f6 100644 --- a/tensorflow/contrib/boosted_trees/kernels/split_handler_ops.cc +++ b/tensorflow/contrib/boosted_trees/kernels/split_handler_ops.cc @@ -834,8 +834,13 @@ class BuildCategoricalEqualitySplitsOp : public OpKernel { root_gradient_stats *= normalizer_ratio; NodeStats root_stats = state->ComputeNodeStats(root_gradient_stats); int32 best_feature_idx = 0; + bool best_feature_updated = false; NodeStats best_right_node_stats(0); NodeStats best_left_node_stats(0); + CHECK(end_index - start_index >= 2) + << "Partition should have a non bias feature. Start index " + << start_index << " and end index " << end_index; + for (int64 feature_idx = start_index + 1; feature_idx < end_index; ++feature_idx) { GradientStats left_gradient_stats(*gradients_t, *hessians_t, @@ -845,11 +850,13 @@ class BuildCategoricalEqualitySplitsOp : public OpKernel { root_gradient_stats - left_gradient_stats; NodeStats left_stats = state->ComputeNodeStats(left_gradient_stats); NodeStats right_stats = state->ComputeNodeStats(right_gradient_stats); - if (left_stats.gain + right_stats.gain > best_gain) { + if (!best_feature_updated || + left_stats.gain + right_stats.gain > best_gain) { best_gain = left_stats.gain + right_stats.gain; best_left_node_stats = left_stats; best_right_node_stats = right_stats; best_feature_idx = feature_idx; + best_feature_updated = true; } } SplitInfo split_info; @@ -864,7 +871,7 @@ class BuildCategoricalEqualitySplitsOp : public OpKernel { << feature_ids(best_feature_idx, 0) << ", " << feature_ids(best_feature_idx, 1) << "\nPartition IDS: " << partition_ids(start_index) << " " - << partition_ids(best_feature_idx); + << partition_ids(best_feature_idx) << " and best gain " << best_gain; equality_split->set_feature_id(feature_ids(best_feature_idx, 0)); auto* left_child = split_info.mutable_left_child(); auto* right_child = split_info.mutable_right_child(); diff --git a/tensorflow/contrib/boosted_trees/lib/learner/batch/categorical_split_handler.py b/tensorflow/contrib/boosted_trees/lib/learner/batch/categorical_split_handler.py index 4da25298cb8..d26af584197 100644 --- a/tensorflow/contrib/boosted_trees/lib/learner/batch/categorical_split_handler.py +++ b/tensorflow/contrib/boosted_trees/lib/learner/batch/categorical_split_handler.py @@ -119,7 +119,7 @@ class EqualitySplitHandler(base_split_handler.BaseSplitHandler): def not_active_inputs(): return (constant_op.constant([], dtype=dtypes.int32), - constant_op.constant([], dtype=dtypes.int64, shape=[1, 2]), + constant_op.constant_v1([], dtype=dtypes.int64, shape=[1, 2]), empty_gradients, empty_hessians) def active_inputs(): diff --git a/tensorflow/contrib/boosted_trees/lib/learner/batch/categorical_split_handler_test.py b/tensorflow/contrib/boosted_trees/lib/learner/batch/categorical_split_handler_test.py index a2f708081a4..386dc19fc7b 100644 --- a/tensorflow/contrib/boosted_trees/lib/learner/batch/categorical_split_handler_test.py +++ b/tensorflow/contrib/boosted_trees/lib/learner/batch/categorical_split_handler_test.py @@ -36,9 +36,9 @@ def get_empty_tensors(gradient_shape, hessian_shape): empty_hess_shape = [1] + hessian_shape.as_list() empty_grad_shape = [1] + gradient_shape.as_list() - empty_gradients = constant_op.constant( + empty_gradients = constant_op.constant_v1( [], dtype=dtypes.float32, shape=empty_grad_shape) - empty_hessians = constant_op.constant( + empty_hessians = constant_op.constant_v1( [], dtype=dtypes.float32, shape=empty_hess_shape) return empty_gradients, empty_hessians @@ -486,8 +486,8 @@ class EqualitySplitHandlerTest(test_util.TensorFlowTestCase): gradients = array_ops.constant([0.2, -0.5, 1.2, 4.0]) hessians = array_ops.constant([0.12, 0.07, 0.2, 0.13]) partition_ids = [0, 0, 0, 1] - indices = array_ops.constant([], dtype=dtypes.int64, shape=[0, 2]) - values = array_ops.constant([], dtype=dtypes.int64) + indices = constant_op.constant_v1([], dtype=dtypes.int64, shape=[0, 2]) + values = constant_op.constant_v1([], dtype=dtypes.int64) gradient_shape = tensor_shape.scalar() hessian_shape = tensor_shape.scalar() diff --git a/tensorflow/contrib/boosted_trees/lib/learner/batch/ordinal_split_handler.py b/tensorflow/contrib/boosted_trees/lib/learner/batch/ordinal_split_handler.py index 1fffbb5f660..0476bed2cd3 100644 --- a/tensorflow/contrib/boosted_trees/lib/learner/batch/ordinal_split_handler.py +++ b/tensorflow/contrib/boosted_trees/lib/learner/batch/ordinal_split_handler.py @@ -605,7 +605,7 @@ def dense_make_stats_update(is_active, are_buckets_ready, float_column, quantile_buckets, example_partition_ids, gradients, hessians, weights, empty_gradients, empty_hessians): """Updates the state for dense split handler.""" - empty_float = constant_op.constant([], dtype=dtypes.float32) + empty_float = constant_op.constant_v1([], dtype=dtypes.float32) quantile_values, quantile_weights = control_flow_ops.cond( is_active[1], # For the next layer, this handler is inactive. @@ -621,8 +621,8 @@ def dense_make_stats_update(is_active, are_buckets_ready, float_column, return (example_partition_ids, quantized_feature, gradients, hessians) def not_ready_inputs_fn(): - return (constant_op.constant([], dtype=dtypes.int32), - constant_op.constant([[]], dtype=dtypes.int64, shape=[1, 2]), + return (constant_op.constant_v1([], dtype=dtypes.int32), + constant_op.constant_v1([[]], dtype=dtypes.int64, shape=[1, 2]), empty_gradients, empty_hessians) example_partition_ids, feature_ids, gradients, hessians = ( @@ -708,11 +708,11 @@ def sparse_make_stats_update( def quantiles_not_ready(): """The subgraph for when the quantiles are not ready.""" - return (constant_op.constant([], dtype=dtypes.int32), - constant_op.constant([], dtype=dtypes.int64, shape=[1, 2]), + return (constant_op.constant_v1([], dtype=dtypes.int32), + constant_op.constant_v1([], dtype=dtypes.int64, shape=[1, 2]), empty_gradients, empty_hessians) - empty_float = constant_op.constant([], dtype=dtypes.float32) + empty_float = constant_op.constant_v1([], dtype=dtypes.float32) handler_not_active = (constant_op.constant( [], dtype=dtypes.int64, shape=[0, 2]), empty_float, constant_op.constant([0, 1], dtype=dtypes.int64), diff --git a/tensorflow/contrib/boosted_trees/lib/learner/batch/ordinal_split_handler_test.py b/tensorflow/contrib/boosted_trees/lib/learner/batch/ordinal_split_handler_test.py index 74b0ea6989c..4a1b528646e 100644 --- a/tensorflow/contrib/boosted_trees/lib/learner/batch/ordinal_split_handler_test.py +++ b/tensorflow/contrib/boosted_trees/lib/learner/batch/ordinal_split_handler_test.py @@ -39,9 +39,9 @@ def get_empty_tensors(gradient_shape, hessian_shape): empty_hess_shape = [1] + hessian_shape.as_list() empty_grad_shape = [1] + gradient_shape.as_list() - empty_gradients = constant_op.constant( + empty_gradients = constant_op.constant_v1( [], dtype=dtypes.float32, shape=empty_grad_shape) - empty_hessians = constant_op.constant( + empty_hessians = constant_op.constant_v1( [], dtype=dtypes.float32, shape=empty_hess_shape) return empty_gradients, empty_hessians @@ -1476,9 +1476,9 @@ class SparseSplitHandlerTest(test_util.TensorFlowTestCase): def testEmpty(self): with self.cached_session() as sess: - indices = array_ops.constant([], dtype=dtypes.int64, shape=[0, 2]) + indices = constant_op.constant_v1([], dtype=dtypes.int64, shape=[0, 2]) # No values in this feature column in this mini-batch. - values = array_ops.constant([], dtype=dtypes.float32) + values = constant_op.constant_v1([], dtype=dtypes.float32) sparse_column = sparse_tensor.SparseTensor(indices, values, [4, 1]) gradient_shape = tensor_shape.scalar() @@ -1549,8 +1549,9 @@ class SparseSplitHandlerTest(test_util.TensorFlowTestCase): sparse_column = array_ops.sparse_placeholder(dtypes.float32) # We have two batches - at first, a sparse feature is empty. - empty_indices = array_ops.constant([], dtype=dtypes.int64, shape=[0, 2]) - empty_values = array_ops.constant([], dtype=dtypes.float32) + empty_indices = constant_op.constant_v1([], dtype=dtypes.int64, + shape=[0, 2]) + empty_values = constant_op.constant_v1([], dtype=dtypes.float32) empty_sparse_column = sparse_tensor.SparseTensor(empty_indices, empty_values, [4, 2]) empty_sparse_column = empty_sparse_column.eval(session=sess) diff --git a/tensorflow/contrib/boosted_trees/python/training/functions/gbdt_batch.py b/tensorflow/contrib/boosted_trees/python/training/functions/gbdt_batch.py index ab5713fbe26..9fdc2fc0c2c 100644 --- a/tensorflow/contrib/boosted_trees/python/training/functions/gbdt_batch.py +++ b/tensorflow/contrib/boosted_trees/python/training/functions/gbdt_batch.py @@ -897,9 +897,9 @@ class GradientBoostedDecisionTreeModel(object): empty_hess_shape = [1] + self._hessian_shape.as_list() empty_grad_shape = [1] + self._gradient_shape.as_list() - empty_gradients = constant_op.constant( + empty_gradients = constant_op.constant_v1( [], dtype=dtypes.float32, shape=empty_grad_shape) - empty_hessians = constant_op.constant( + empty_hessians = constant_op.constant_v1( [], dtype=dtypes.float32, shape=empty_hess_shape) active_handlers = array_ops.unstack(active_handlers, axis=0) @@ -1257,13 +1257,12 @@ class GradientBoostedDecisionTreeModel(object): def _get_replica_device_setter(self, worker_device): """Creates a replica device setter.""" ps_tasks = self._num_ps_replicas - ps_ops = [ - "Variable", - "VariableV2", + ps_ops = list(device_setter.STANDARD_PS_OPS) + ps_ops.extend([ "DecisionTreeEnsembleResourceHandleOp", "StatsAccumulatorScalarResourceHandleOp", "StatsAccumulatorTensorResourceHandleOp", - ] + ]) ps_strategy = _OpRoundRobinStrategy(ps_ops, ps_tasks) return device_setter.replica_device_setter( worker_device=worker_device, diff --git a/tensorflow/contrib/boosted_trees/python/utils/losses.py b/tensorflow/contrib/boosted_trees/python/utils/losses.py index b5ebaf19995..220e981618b 100644 --- a/tensorflow/contrib/boosted_trees/python/utils/losses.py +++ b/tensorflow/contrib/boosted_trees/python/utils/losses.py @@ -48,6 +48,47 @@ def per_example_logistic_loss(labels, weights, predictions): labels=labels, logits=predictions) return unweighted_loss * weights, control_flow_ops.no_op() +# MUST USE WITH HESSIAN REGULARIZATION, +# This loss can have zero hessian, so it must be used with l2 or min_node_weight +# regularization. +# An example config is +# learner_config.constraints.min_node_weight = 1 / num_examples_per_layer +# learner_config.regularization.l2 = 1.0 / num_examples_per_layer +# TODO(nponomareva): make it multidimensional so we can estimate several +# quantiles at once. +def per_example_quantile_regression_loss(labels, weights, predictions, + quantile): + """Smoothed loss for quantile regression. + + The standard quantile regression loss is quantile*(y-y') when y>y' and + (quantile-1)*(y-y') otherwise, y' is a prediction, y is a label. The impl + below is this loss but squared in the region where the loss value < 1. + + Args: + labels: Rank 2 (N, D) tensor of per-example labels. + weights: Rank 2 (N, 1) tensor of per-example weights. + predictions: Rank 2 (N, D) tensor of per-example predictions. + quantile: The quantile to use. + + Returns: + loss: A Rank 2 (N, 1) tensor of per-example quantile loss. + update_op: An update operation to update the loss's internal state. + """ + labels = math_ops.to_float(labels) + error = labels - predictions + square_loss_right = array_ops.where(error * quantile < 1.0, + math_ops.square(quantile * error), + quantile * error) + square_loss_left = array_ops.where(error * (quantile - 1) < 1, + math_ops.square((quantile - 1) * error), + (quantile - 1) * error) + + unweighted_loss = array_ops.where(error > 0, square_loss_right, + square_loss_left) + if weights is None: + return unweighted_loss, control_flow_ops.no_op() + else: + return unweighted_loss * weights, control_flow_ops.no_op() # This is classical form of Maximum entropy loss, that is twice differentiable # (sparse_softmax_cross_entropy which is what we go for is not twice @@ -78,8 +119,7 @@ def per_example_maxent_loss(labels, weights, logits, num_classes, eps=1e-15): labels = array_ops.expand_dims(labels, 1) # Labels are indices of classes, convert them to one hot encodings. target_one_hot = array_ops.one_hot(indices=labels, depth=num_classes) - labels = math_ops.reduce_sum( - input_tensor=target_one_hot, reduction_indices=[1]) + labels = math_ops.reduce_sum(input_tensor=target_one_hot, axis=[1]) labels = math_ops.to_float(labels) # Calculate softmax probabilities for each class. diff --git a/tensorflow/contrib/checkpoint/python/containers.py b/tensorflow/contrib/checkpoint/python/containers.py index 242c1e8ba45..5418e2605b7 100644 --- a/tensorflow/contrib/checkpoint/python/containers.py +++ b/tensorflow/contrib/checkpoint/python/containers.py @@ -46,6 +46,10 @@ class UniqueNameTracker(data_structures.CheckpointableDataStructure): self._maybe_initialize_checkpointable() self._name_counts = {} + @property + def _values(self): + return [dep.ref for dep in self._checkpoint_dependencies] + def track(self, checkpointable, base_name): """Add a dependency on `checkpointable`. diff --git a/tensorflow/contrib/cluster_resolver/BUILD b/tensorflow/contrib/cluster_resolver/BUILD index 9e1867ea9d0..f944b7f8843 100644 --- a/tensorflow/contrib/cluster_resolver/BUILD +++ b/tensorflow/contrib/cluster_resolver/BUILD @@ -21,85 +21,18 @@ py_library( py_library( name = "cluster_resolver_py", - srcs = [ + srcs = glob([ "__init__.py", - "python/training/__init__.py", - ], + "python/training/*.py", + ]), srcs_version = "PY2AND3", visibility = ["//visibility:public"], - deps = [ - ":base_cluster_resolver_py", - ":gce_cluster_resolver_py", - ":kubernetes_cluster_resolver_py", - ":slurm_cluster_resolver_py", - ":tfconfig_cluster_resolver_py", - ":tpu_cluster_resolver_py", - "//tensorflow/python:util", - ], -) - -py_library( - name = "base_cluster_resolver_py", - srcs = ["python/training/cluster_resolver.py"], - srcs_version = "PY2AND3", - deps = [ - "//tensorflow/python:training", - ], -) - -py_library( - name = "gce_cluster_resolver_py", - srcs = ["python/training/gce_cluster_resolver.py"], - srcs_version = "PY2AND3", - deps = [ - ":base_cluster_resolver_py", - "//tensorflow/python:training", - ], -) - -py_library( - name = "tfconfig_cluster_resolver_py", - srcs = ["python/training/tfconfig_cluster_resolver.py"], - srcs_version = "PY2AND3", - deps = [ - ":base_cluster_resolver_py", - "//tensorflow/python:training", - ], -) - -py_library( - name = "tpu_cluster_resolver_py", - srcs = ["python/training/tpu_cluster_resolver.py"], - srcs_version = "PY2AND3", - deps = [ - ":base_cluster_resolver_py", - "//tensorflow/python:training", - ], -) - -py_library( - name = "slurm_cluster_resolver_py", - srcs = ["python/training/slurm_cluster_resolver.py"], - srcs_version = "PY2AND3", - deps = [ - ":base_cluster_resolver_py", - "//tensorflow/python:training", - ], -) - -py_library( - name = "kubernetes_cluster_resolver_py", - srcs = ["python/training/kubernetes_cluster_resolver.py"], - srcs_version = "PY2AND3", - deps = [ - ":base_cluster_resolver_py", - "//tensorflow/python:training", - ], + deps = ["//tensorflow/python/distribute/cluster_resolver:cluster_resolver_lib"], ) tf_py_test( - name = "base_cluster_resolver_py_test", - srcs = ["python/training/cluster_resolver_test.py"], + name = "cluster_resolver_initialization_test", + srcs = ["cluster_resolver_initialization_test.py"], additional_deps = [ ":cluster_resolver_py", "//tensorflow/python:client_testlib", @@ -108,86 +41,5 @@ tf_py_test( "//tensorflow/python:platform_test", "//tensorflow/python:training", ], - main = "python/training/cluster_resolver_test.py", -) - -tf_py_test( - name = "gce_cluster_resolver_py_test", - size = "small", - srcs = ["python/training/gce_cluster_resolver_test.py"], - additional_deps = [ - ":cluster_resolver_py", - ":gce_cluster_resolver_py", - "//tensorflow/python:client_testlib", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:framework_test_lib", - "//tensorflow/python:platform_test", - "//tensorflow/python:training", - ], - main = "python/training/gce_cluster_resolver_test.py", -) - -tf_py_test( - name = "tfconfig_cluster_resolver_py_test", - size = "small", - srcs = ["python/training/tfconfig_cluster_resolver_test.py"], - additional_deps = [ - ":tfconfig_cluster_resolver_py", - "//tensorflow/python:client_testlib", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:framework_test_lib", - "//tensorflow/python:platform_test", - "//tensorflow/python:training", - ], - grpc_enabled = True, - main = "python/training/tfconfig_cluster_resolver_test.py", -) - -tf_py_test( - name = "tpu_cluster_resolver_py_test", - size = "small", - srcs = ["python/training/tpu_cluster_resolver_test.py"], - additional_deps = [ - ":tpu_cluster_resolver_py", - "//tensorflow/python:client_testlib", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:framework_test_lib", - "//tensorflow/python:platform_test", - "//tensorflow/python:training", - ], - grpc_enabled = True, - main = "python/training/tpu_cluster_resolver_test.py", -) - -tf_py_test( - name = "slurm_cluster_resolver_py_test", - size = "small", - srcs = ["python/training/slurm_cluster_resolver_test.py"], - additional_deps = [ - ":cluster_resolver_py", - ":slurm_cluster_resolver_py", - "//tensorflow/python:client_testlib", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:framework_test_lib", - "//tensorflow/python:platform_test", - "//tensorflow/python:training", - ], - main = "python/training/slurm_cluster_resolver_test.py", - tags = [], -) - -tf_py_test( - name = "kubernetes_cluster_resolver_py_test", - size = "small", - srcs = ["python/training/kubernetes_cluster_resolver_test.py"], - additional_deps = [ - ":cluster_resolver_py", - ":kubernetes_cluster_resolver_py", - "//tensorflow/python:client_testlib", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:framework_test_lib", - "//tensorflow/python:platform_test", - "//tensorflow/python:training", - ], - main = "python/training/kubernetes_cluster_resolver_test.py", + main = "cluster_resolver_initialization_test.py", ) diff --git a/tensorflow/contrib/cluster_resolver/__init__.py b/tensorflow/contrib/cluster_resolver/__init__.py index fd1263fe81a..390b3e7550b 100644 --- a/tensorflow/contrib/cluster_resolver/__init__.py +++ b/tensorflow/contrib/cluster_resolver/__init__.py @@ -20,12 +20,14 @@ from __future__ import division from __future__ import print_function # pylint: disable=wildcard-import,unused-import -from tensorflow.contrib.cluster_resolver.python.training.cluster_resolver import ClusterResolver -from tensorflow.contrib.cluster_resolver.python.training.cluster_resolver import SimpleClusterResolver -from tensorflow.contrib.cluster_resolver.python.training.cluster_resolver import UnionClusterResolver -from tensorflow.contrib.cluster_resolver.python.training.gce_cluster_resolver import GceClusterResolver -from tensorflow.contrib.cluster_resolver.python.training.slurm_cluster_resolver import SlurmClusterResolver -from tensorflow.contrib.cluster_resolver.python.training.tpu_cluster_resolver import TPUClusterResolver +from tensorflow.python.distribute.cluster_resolver.cluster_resolver import ClusterResolver +from tensorflow.python.distribute.cluster_resolver.cluster_resolver import SimpleClusterResolver +from tensorflow.python.distribute.cluster_resolver.cluster_resolver import UnionClusterResolver +from tensorflow.python.distribute.cluster_resolver.gce_cluster_resolver import GceClusterResolver +from tensorflow.python.distribute.cluster_resolver.kubernetes_cluster_resolver import KubernetesClusterResolver +from tensorflow.python.distribute.cluster_resolver.slurm_cluster_resolver import SlurmClusterResolver +from tensorflow.python.distribute.cluster_resolver.tfconfig_cluster_resolver import TFConfigClusterResolver +from tensorflow.python.distribute.cluster_resolver.tpu_cluster_resolver import TPUClusterResolver # pylint: enable=wildcard-import,unused-import from tensorflow.python.util.all_util import remove_undocumented @@ -35,6 +37,8 @@ _allowed_symbols = [ 'SimpleClusterResolver', 'UnionClusterResolver', 'GceClusterResolver', + 'KubernetesClusterResolver', + 'TFConfigClusterResolver', 'TPUClusterResolver', 'SlurmClusterResolver', ] diff --git a/tensorflow/contrib/cluster_resolver/cluster_resolver_initialization_test.py b/tensorflow/contrib/cluster_resolver/cluster_resolver_initialization_test.py new file mode 100644 index 00000000000..01ff1478c69 --- /dev/null +++ b/tensorflow/contrib/cluster_resolver/cluster_resolver_initialization_test.py @@ -0,0 +1,53 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Tests to ensure ClusterResolvers are usable via the old contrib path.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from tensorflow.contrib.cluster_resolver import SimpleClusterResolver +from tensorflow.contrib.cluster_resolver.python.training import cluster_resolver +from tensorflow.contrib.cluster_resolver.python.training import UnionClusterResolver +from tensorflow.python.platform import test +from tensorflow.python.training import server_lib + + +class ClusterResolverInitializationTest(test.TestCase): + + def testCreateSimpleClusterResolverFromLib(self): + base_cluster_spec = server_lib.ClusterSpec({ + "ps": ["ps0:2222", "ps1:2222"], + "worker": ["worker0:2222", "worker1:2222", "worker2:2222"] + }) + cluster_resolver.SimpleClusterResolver(base_cluster_spec) + + def testCreateSimpleClusterResolver(self): + base_cluster_spec = server_lib.ClusterSpec({ + "ps": ["ps0:2222", "ps1:2222"], + "worker": ["worker0:2222", "worker1:2222", "worker2:2222"] + }) + SimpleClusterResolver(base_cluster_spec) + + def testCreateUnionClusterResolver(self): + base_cluster_spec = server_lib.ClusterSpec({ + "ps": ["ps0:2222", "ps1:2222"], + "worker": ["worker0:2222", "worker1:2222", "worker2:2222"] + }) + simple_cr = SimpleClusterResolver(base_cluster_spec) + UnionClusterResolver(simple_cr) + +if __name__ == "__main__": + test.main() diff --git a/tensorflow/contrib/cluster_resolver/python/training/__init__.py b/tensorflow/contrib/cluster_resolver/python/training/__init__.py index 6d9120a3b96..10d93549ebb 100644 --- a/tensorflow/contrib/cluster_resolver/python/training/__init__.py +++ b/tensorflow/contrib/cluster_resolver/python/training/__init__.py @@ -18,11 +18,36 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function -from tensorflow.contrib.cluster_resolver.python.training.cluster_resolver import ClusterResolver -from tensorflow.contrib.cluster_resolver.python.training.cluster_resolver import SimpleClusterResolver -from tensorflow.contrib.cluster_resolver.python.training.cluster_resolver import UnionClusterResolver -from tensorflow.contrib.cluster_resolver.python.training.gce_cluster_resolver import GceClusterResolver -from tensorflow.contrib.cluster_resolver.python.training.kubernetes_cluster_resolver import KubernetesClusterResolver -from tensorflow.contrib.cluster_resolver.python.training.slurm_cluster_resolver import SlurmClusterResolver -from tensorflow.contrib.cluster_resolver.python.training.tfconfig_cluster_resolver import TFConfigClusterResolver -from tensorflow.contrib.cluster_resolver.python.training.tpu_cluster_resolver import TPUClusterResolver +# This file (and all files in this directory in general) is a backwards +# compatibility shim that exists to re-export ClusterResolvers such that +# existing OSS code will not be broken. + +from tensorflow.python.distribute.cluster_resolver.cluster_resolver import ClusterResolver +from tensorflow.python.distribute.cluster_resolver.cluster_resolver import SimpleClusterResolver +from tensorflow.python.distribute.cluster_resolver.cluster_resolver import UnionClusterResolver +from tensorflow.python.distribute.cluster_resolver.gce_cluster_resolver import GceClusterResolver +from tensorflow.python.distribute.cluster_resolver.kubernetes_cluster_resolver import KubernetesClusterResolver +from tensorflow.python.distribute.cluster_resolver.slurm_cluster_resolver import SlurmClusterResolver +from tensorflow.python.distribute.cluster_resolver.tfconfig_cluster_resolver import TFConfigClusterResolver +from tensorflow.python.distribute.cluster_resolver.tpu_cluster_resolver import TPUClusterResolver + +from tensorflow.python.util.all_util import remove_undocumented + +_allowed_symbols = [ + 'cluster_resolver', + 'gce_cluster_resolver', + 'kubernetes_cluster_resolver', + 'slurm_cluster_resolver', + 'tfconfig_cluster_resolver', + 'tpu_cluster_resolver', + 'ClusterResolver', + 'SimpleClusterResolver', + 'UnionClusterResolver', + 'GceClusterResolver', + 'KubernetesClusterResolver', + 'TFConfigClusterResolver', + 'TPUClusterResolver', + 'SlurmClusterResolver', +] + +remove_undocumented(__name__, _allowed_symbols) diff --git a/tensorflow/contrib/cluster_resolver/python/training/cluster_resolver.py b/tensorflow/contrib/cluster_resolver/python/training/cluster_resolver.py index 40b1e667ee6..99840fb5166 100644 --- a/tensorflow/contrib/cluster_resolver/python/training/cluster_resolver.py +++ b/tensorflow/contrib/cluster_resolver/python/training/cluster_resolver.py @@ -1,4 +1,4 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -12,333 +12,29 @@ # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================== -"""Cluster Resolvers are used for dynamic cluster IP/hostname resolution.""" +"""Stub file for ClusterResolver to maintain backwards compatibility.""" from __future__ import absolute_import from __future__ import division from __future__ import print_function -import abc +# This file (and all files in this directory in general) is a backwards +# compatibility shim that exists to re-export ClusterResolvers such that +# existing OSS code will not be broken. -import six +# pylint: disable=unused-import +from tensorflow.python.distribute.cluster_resolver.cluster_resolver import ClusterResolver +from tensorflow.python.distribute.cluster_resolver.cluster_resolver import SimpleClusterResolver +from tensorflow.python.distribute.cluster_resolver.cluster_resolver import UnionClusterResolver +# pylint: enable=unused-import -from tensorflow.python.training.server_lib import ClusterSpec +from tensorflow.python.util.all_util import remove_undocumented +_allowed_symbols = [ + 'ClusterResolver', + 'SimpleClusterResolver', + 'UnionClusterResolver', +] -def _format_master_url(master, rpc_layer=None): - if rpc_layer: - return '%s://%s' % (rpc_layer, master) - else: - return master +remove_undocumented(__name__, _allowed_symbols) - -@six.add_metaclass(abc.ABCMeta) -class ClusterResolver(object): - """Abstract class for all implementations of ClusterResolvers. - - This defines the skeleton for all implementations of ClusterResolvers. - ClusterResolvers are a way for TensorFlow to communicate with various cluster - management systems (e.g. GCE, AWS, etc...). - - By letting TensorFlow communicate with these systems, we will be able to - automatically discover and resolve IP addresses for various TensorFlow - workers. This will eventually allow us to automatically recover from - underlying machine failures and scale TensorFlow worker clusters up and down. - """ - - @abc.abstractmethod - def cluster_spec(self): - """Retrieve the current state of the cluster and returns a ClusterSpec. - - Returns: - A ClusterSpec representing the state of the cluster at the moment this - function is called. - - Implementors of this function must take care in ensuring that the - ClusterSpec returned is up-to-date at the time of calling this function. - This usually means retrieving the information from the underlying cluster - management system every time this function is invoked and reconstructing - a cluster_spec, rather than attempting to cache anything. - """ - raise NotImplementedError( - 'cluster_spec is not implemented for {}.'.format(self)) - - @abc.abstractmethod - def master(self, task_type=None, task_index=None, rpc_layer=None): - """Retrieves the name or URL of the session master. - - Args: - task_type: (Optional) The type of the TensorFlow task of the master. - task_index: (Optional) The index of the TensorFlow task of the master. - rpc_layer: (Optional) The RPC protocol for the given cluster. - - Returns: - The name or URL of the session master. - - Implementors of this function must take care in ensuring that the master - returned is up-to-date at the time to calling this function. This usually - means retrieving the master every time this function is invoked. - """ - raise NotImplementedError('master is not implemented for {}.'.format(self)) - - -class SimpleClusterResolver(ClusterResolver): - """Simple implementation of ClusterResolver that accepts a ClusterSpec.""" - - def __init__(self, cluster_spec, master='', task_type=None, task_index=None, - environment='', num_accelerators_per_worker=0, - rpc_layer=None): - """Creates a SimpleClusterResolver from a ClusterSpec.""" - super(SimpleClusterResolver, self).__init__() - - self._task_type = task_type - self._task_index = task_index - self._environment = environment - self._num_accelerators_per_worker = num_accelerators_per_worker - self._rpc_layer = rpc_layer - - if not isinstance(cluster_spec, ClusterSpec): - raise TypeError('cluster_spec must be a ClusterSpec.') - self._cluster_spec = cluster_spec - - if not isinstance(master, str): - raise TypeError('master must be a string.') - self._master = master - - def cluster_spec(self): - """Returns the ClusterSpec passed into the constructor.""" - return self._cluster_spec - - def master(self, task_type=None, task_index=None, rpc_layer=None): - """Returns the master address to use when creating a session. - - Args: - task_type: (Optional) The type of the TensorFlow task of the master. - task_index: (Optional) The index of the TensorFlow task of the master. - rpc_layer: (Optional) The RPC used by distributed TensorFlow. - - Returns: - The name or URL of the session master. - - If a task_type and task_index is given, this will override the `master` - string passed into the initialization function. - """ - if task_type is not None and task_index is not None: - master = self.cluster_spec().task_address(task_type, task_index) - else: - master = self._master - - return _format_master_url(master, rpc_layer or self._rpc_layer) - - @property - def task_type(self): - return self._task_type - - @property - def task_index(self): - return self._task_index - - @task_type.setter - def task_type(self, task_type): - self._task_type = task_type - - @task_index.setter - def task_index(self, task_index): - self._task_index = task_index - - @property - def environment(self): - return self._environment - - def num_accelerators_per_worker(self, session_config=None): - """Returns the number of accelerator cores per worker. - - Args: - session_config: Unused. The SimpleClusterResolver does not do automatic - detection of accelerators, so a TensorFlow session will never be - created, and thus a `session_config` is never necessary here, and will - be ignored. - """ - del session_config - return self._num_accelerators_per_worker - - @property - def rpc_layer(self): - return self._rpc_layer - - @rpc_layer.setter - def rpc_layer(self, rpc_layer): - self._rpc_layer = rpc_layer - - -class UnionClusterResolver(ClusterResolver): - """Performs a union on underlying ClusterResolvers. - - This class performs a union given two or more existing ClusterResolvers. It - merges the underlying ClusterResolvers, and returns one unified ClusterSpec - when cluster_spec is called. The details of the merge function is - documented in the cluster_spec function. - - For additional Cluster Resolver properties such as task type, task index, - rpc layer, environment, etc..., we will return the value from the first - ClusterResolver in the union. - """ - - def __init__(self, *args, **kwargs): - """Initializes a UnionClusterResolver with other ClusterResolvers. - - Args: - *args: `ClusterResolver` objects to be unionized. - **kwargs: - rpc_layer - (Optional) Override value for the RPC layer used by - TensorFlow. - task_type - (Optional) Override value for the current task type. - task_index - (Optional) Override value for the current task index. - - Raises: - TypeError: If any argument is not a subclass of `ClusterResolvers`. - ValueError: If there are no arguments passed. - """ - super(UnionClusterResolver, self).__init__() - - self._rpc_layer = kwargs.pop('rpc_layer', None) - self._task_type = kwargs.pop('task_type', None) - self._task_index = kwargs.pop('task_index', None) - - if kwargs: - raise ValueError('Unexpected kwargs provided {!r}'.format(kwargs)) - - if not args: - raise ValueError('At least one ClusterResolver is required.') - - for cluster_resolver in args: - if not isinstance(cluster_resolver, ClusterResolver): - raise TypeError('All arguments must be a sub-class of ' - '`ClusterResolver.`') - self._cluster_resolvers = args - - def cluster_spec(self): - """Returns a union of all the ClusterSpecs from the ClusterResolvers. - - Returns: - A ClusterSpec containing host information merged from all the underlying - ClusterResolvers. - - Raises: - KeyError: If there are conflicting keys detected when merging two or - more dictionaries, this exception is raised. - - Note: If there are multiple ClusterResolvers exposing ClusterSpecs with the - same job name, we will merge the list/dict of workers. - - If *all* underlying ClusterSpecs expose the set of workers as lists, we will - concatenate the lists of workers, starting with the list of workers from - the first ClusterResolver passed into the constructor. - - If *any* of the ClusterSpecs expose the set of workers as a dict, we will - treat all the sets of workers as dicts (even if they are returned as lists) - and will only merge them into a dict if there is no conflicting keys. If - there is a conflicting key, we will raise a `KeyError`. - """ - - merged_cluster = {} - - # We figure out whether it is all lists for a particular job, or whether - # there are dicts inside. - for cluster_resolver in self._cluster_resolvers: - cluster_spec = cluster_resolver.cluster_spec() - cluster_dict = cluster_spec.as_dict() - - for job_name, tasks in cluster_dict.items(): - if job_name in merged_cluster: - # If we see a dict, then we write a dict out regardless. - if isinstance(tasks, dict): - merged_cluster[job_name] = {} - else: - # We take whichever type is present. - if isinstance(tasks, list): - merged_cluster[job_name] = [] - else: - merged_cluster[job_name] = {} - - # We then do the merge as appropriate in merged_cluster[job]. - for cluster_resolver in self._cluster_resolvers: - cluster_spec = cluster_resolver.cluster_spec() - cluster_dict = cluster_spec.as_dict() - - for job_name, tasks in cluster_dict.items(): - if isinstance(merged_cluster[job_name], list): - # We all have lists, we can just concatenate and be done. - merged_cluster[job_name].extend(tasks) - else: - if isinstance(tasks, list): - # We convert to a dictionary if the type is a list. - task_dict = dict(zip(range(0, len(tasks)), tasks)) - else: - # We can simply make a copy (for update) and be done. - task_dict = tasks.copy() - - # We detect if there are duplicates, and raise an error if so. - task_keys = set(task_dict) - merged_keys = set(merged_cluster[job_name].keys()) - intersected_keys = task_keys.intersection(merged_keys) - if intersected_keys: - raise KeyError('Duplicate keys detected when merging two ' - 'ClusterSpecs: %s' % repr(intersected_keys)) - - # We do the merge after all the processing. - merged_cluster[job_name].update(task_dict) - - return ClusterSpec(merged_cluster) - - def master(self, task_type=None, task_index=None, rpc_layer=None): - """Returns the master address to use when creating a session. - - This usually returns the master from the first ClusterResolver passed in, - but you can override this by specifying the task_type and task_index. - - Args: - task_type: (Optional) The type of the TensorFlow task of the master. - task_index: (Optional) The index of the TensorFlow task of the master. - rpc_layer: (Optional) The RPC protocol for the given cluster. - - Returns: - The name or URL of the session master. - """ - if task_type is not None and task_index is not None: - master = self.cluster_spec().task_address(task_type, task_index) - return _format_master_url(master, rpc_layer or self._rpc_layer) - - return self._cluster_resolvers[0].master(rpc_layer=rpc_layer) - - @property - def task_type(self): - return self._task_type or self._cluster_resolvers[0].task_type - - @property - def task_index(self): - return self._task_index or self._cluster_resolvers[0].task_index - - @task_type.setter - def task_type(self, task_type): - self._task_type = task_type - - @task_index.setter - def task_index(self, task_index): - self._task_index = task_index - - @property - def environment(self): - return self._cluster_resolvers[0].environment - - def num_accelerators_per_worker(self, session_config=None): - return self._cluster_resolvers[0].num_accelerators_per_worker( - session_config) - - @property - def rpc_layer(self): - return self._rpc_layer or self._cluster_resolvers[0].rpc_layer - - @rpc_layer.setter - def rpc_layer(self, rpc_layer): - self._rpc_layer = rpc_layer diff --git a/tensorflow/contrib/cluster_resolver/python/training/gce_cluster_resolver.py b/tensorflow/contrib/cluster_resolver/python/training/gce_cluster_resolver.py index 195b68959b6..55e61155c68 100644 --- a/tensorflow/contrib/cluster_resolver/python/training/gce_cluster_resolver.py +++ b/tensorflow/contrib/cluster_resolver/python/training/gce_cluster_resolver.py @@ -1,4 +1,4 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -12,197 +12,24 @@ # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================== -"""Implementation of Cluster Resolvers for GCE Instance Groups.""" +"""Stub file for GceClusterResolver to maintain backwards compatibility.""" from __future__ import absolute_import from __future__ import division from __future__ import print_function +# This file (and all files in this directory in general) is a backwards +# compatibility shim that exists to re-export ClusterResolvers such that +# existing OSS code will not be broken. -from tensorflow.contrib.cluster_resolver.python.training.cluster_resolver import ClusterResolver -from tensorflow.python.training.server_lib import ClusterSpec +# pylint: disable=unused-import +from tensorflow.python.distribute.cluster_resolver.gce_cluster_resolver import GceClusterResolver +# pylint: enable=unused-import -_GOOGLE_API_CLIENT_INSTALLED = True -try: - from googleapiclient import discovery # pylint: disable=g-import-not-at-top - from oauth2client.client import GoogleCredentials # pylint: disable=g-import-not-at-top -except ImportError: - _GOOGLE_API_CLIENT_INSTALLED = False +from tensorflow.python.util.all_util import remove_undocumented +_allowed_symbols = [ + 'GceClusterResolver', +] -def _format_master_url(master, rpc_layer=None): - return '%s://%s' % (rpc_layer, master) if rpc_layer else master - - -class GceClusterResolver(ClusterResolver): - """Cluster Resolver for Google Compute Engine. - - This is an implementation of cluster resolvers for the Google Compute Engine - instance group platform. By specifying a project, zone, and instance group, - this will retrieve the IP address of all the instances within the instance - group and return a Cluster Resolver object suitable for use for distributed - TensorFlow. - """ - - def __init__(self, - project, - zone, - instance_group, - port, - task_type='worker', - task_index=0, - rpc_layer='grpc', - num_accelerators_per_worker=0, - credentials='default', - service=None): - """Creates a new GceClusterResolver object. - - This takes in a few parameters and creates a GceClusterResolver project. It - will then use these parameters to query the GCE API for the IP addresses of - each instance in the instance group. - - Args: - project: Name of the GCE project. - zone: Zone of the GCE instance group. - instance_group: Name of the GCE instance group. - port: Port of the listening TensorFlow server (default: 8470) - task_type: Name of the TensorFlow job this GCE instance group of VM - instances belong to. - task_index: The task index for this particular VM, within the GCE - instance group. In particular, every single instance should be assigned - a unique ordinal index within an instance group manually so that they - can be distinguished from each other. - rpc_layer: The RPC layer TensorFlow should use to communicate across - instances. - num_accelerators_per_worker: Number of accelerators (GPUs) present per - instance. - credentials: GCE Credentials. If nothing is specified, this defaults to - GoogleCredentials.get_application_default(). - service: The GCE API object returned by the googleapiclient.discovery - function. (Default: discovery.build('compute', 'v1')). If you specify a - custom service object, then the credentials parameter will be ignored. - - Raises: - ImportError: If the googleapiclient is not installed. - """ - self._project = project - self._zone = zone - self._instance_group = instance_group - self._task_type = task_type - self._task_index = task_index - self._rpc_layer = rpc_layer - self._port = port - self._credentials = credentials - - if credentials == 'default': - if _GOOGLE_API_CLIENT_INSTALLED: - self._credentials = GoogleCredentials.get_application_default() - - if service is None: - if not _GOOGLE_API_CLIENT_INSTALLED: - raise ImportError('googleapiclient must be installed before using the ' - 'GCE cluster resolver') - self._service = discovery.build( - 'compute', 'v1', - credentials=self._credentials) - else: - self._service = service - - def cluster_spec(self): - """Returns a ClusterSpec object based on the latest instance group info. - - This returns a ClusterSpec object for use based on information from the - specified instance group. We will retrieve the information from the GCE APIs - every time this method is called. - - Returns: - A ClusterSpec containing host information retrieved from GCE. - """ - request_body = {'instanceState': 'RUNNING'} - request = self._service.instanceGroups().listInstances( - project=self._project, - zone=self._zone, - instanceGroups=self._instance_group, - body=request_body, - orderBy='name') - - worker_list = [] - - while request is not None: - response = request.execute() - - items = response['items'] - for instance in items: - instance_name = instance['instance'].split('/')[-1] - - instance_request = self._service.instances().get( - project=self._project, - zone=self._zone, - instance=instance_name) - - if instance_request is not None: - instance_details = instance_request.execute() - ip_address = instance_details['networkInterfaces'][0]['networkIP'] - instance_url = '%s:%s' % (ip_address, self._port) - worker_list.append(instance_url) - - request = self._service.instanceGroups().listInstances_next( - previous_request=request, - previous_response=response) - - worker_list.sort() - return ClusterSpec({self._task_type: worker_list}) - - def master(self, task_type=None, task_index=None, rpc_layer=None): - task_type = task_type if task_type is not None else self._task_type - task_index = task_index if task_index is not None else self._task_index - - if task_type is not None and task_index is not None: - master = self.cluster_spec().task_address(task_type, task_index) - if rpc_layer or self._rpc_layer: - return '%s://%s' % (rpc_layer or self._rpc_layer, master) - else: - return master - - return '' - - @property - def task_type(self): - return self._task_type - - @property - def task_index(self): - return self._task_index - - @task_type.setter - def task_type(self, task_type): - raise RuntimeError( - 'You cannot reset the task_type of the GceClusterResolver after it has ' - 'been created.') - - @task_index.setter - def task_index(self, task_index): - self._task_index = task_index - - @property - def environment(self): - """Returns the current environment which TensorFlow is running in. - - For users in the GCE environment, the environment property is always an - empty string, and Google users will not use this ClusterResolver for running - on internal systems. - """ - return '' - - @property - def rpc_layer(self): - return self._rpc_layer - - @rpc_layer.setter - def rpc_layer(self, rpc_layer): - self._rpc_layer = rpc_layer - - def num_accelerators_per_worker(self, session_config=None): - del session_config # Unused, since this is set manually in __init__. - return self._num_accelerators_per_worker - +remove_undocumented(__name__, _allowed_symbols) diff --git a/tensorflow/contrib/cluster_resolver/python/training/kubernetes_cluster_resolver.py b/tensorflow/contrib/cluster_resolver/python/training/kubernetes_cluster_resolver.py index ddae64839f0..a8eaf33629a 100644 --- a/tensorflow/contrib/cluster_resolver/python/training/kubernetes_cluster_resolver.py +++ b/tensorflow/contrib/cluster_resolver/python/training/kubernetes_cluster_resolver.py @@ -12,121 +12,25 @@ # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================== -"""Implementation of Cluster Resolvers for Kubernetes.""" +"""Stub file for KubernetesClusterResolver for backwards compatibility.""" from __future__ import absolute_import from __future__ import division from __future__ import print_function -from tensorflow.contrib.cluster_resolver.python.training.cluster_resolver import ClusterResolver -from tensorflow.python.training import server_lib +# This file (and all files in this directory in general) is a backwards +# compatibility shim that exists to re-export ClusterResolvers such that +# existing OSS code will not be broken. -_KUBERNETES_API_CLIENT_INSTALLED = True -try: - from kubernetes import client as k8sclient # pylint: disable=g-import-not-at-top - from kubernetes import config as k8sconfig # pylint: disable=g-import-not-at-top -except ImportError: - _KUBERNETES_API_CLIENT_INSTALLED = False +# pylint: disable=unused-import +from tensorflow.python.distribute.cluster_resolver.kubernetes_cluster_resolver import KubernetesClusterResolver +# pylint: enable=unused-import +from tensorflow.python.util.all_util import remove_undocumented -class KubernetesClusterResolver(ClusterResolver): - """Cluster Resolver for Kubernetes. +_allowed_symbols = [ + 'KubernetesClusterResolver', +] - This is an implementation of cluster resolvers for Kubernetes. When given the - the Kubernetes namespace and label selector for pods, we will retrieve the - pod IP addresses of all running pods matching the selector, and return a - ClusterSpec based on that information. - """ +remove_undocumented(__name__, _allowed_symbols) - def __init__(self, - job_to_label_mapping=None, - tf_server_port=8470, - override_client=None): - """Initializes a new KubernetesClusterResolver. - - This initializes a new Kubernetes Cluster Resolver. The Cluster Resolver - will attempt to talk to the Kubernetes master to retrieve all the instances - of pods matching a label selector. - - Args: - job_to_label_mapping: A mapping of TensorFlow jobs to label selectors. - This allows users to specify many TensorFlow jobs in one Cluster - Resolver, and each job can have pods belong with different label - selectors. For example, a sample mapping might be - ``` - {'worker': ['job-name=worker-cluster-a', 'job-name=worker-cluster-b'], - 'ps': ['job-name=ps-1', 'job-name=ps-2']} - ``` - tf_server_port: The port the TensorFlow server is listening on. - override_client: The Kubernetes client (usually automatically retrieved - using `from kubernetes import client as k8sclient`). If you pass this - in, you are responsible for setting Kubernetes credentials manually. - - Raises: - ImportError: If the Kubernetes Python client is not installed and no - `override_client` is passed in. - """ - if _KUBERNETES_API_CLIENT_INSTALLED: - k8sconfig.load_kube_config() - - if not job_to_label_mapping: - job_to_label_mapping = {'worker': ['job-name=tensorflow']} - - if not override_client and not _KUBERNETES_API_CLIENT_INSTALLED: - raise ImportError('The Kubernetes Python client must be installed before' - 'using the Kubernetes Cluster Resolver. To install the' - 'Kubernetes Python client, run `pip install ' - 'kubernetes` on your command line.') - - self._job_to_label_mapping = job_to_label_mapping - self._tf_server_port = tf_server_port - self._override_client = override_client - - def master(self): - # TODO(frankchn): Figure out a standard way to pass in the current task type - # and task id via Kubernetes. - pass - - def get_master(self): - return self.master() - - def get_job_name(self): - return self._job_name - - def cluster_spec(self): - """Returns a ClusterSpec object based on the latest info from Kubernetes. - - We retrieve the information from the Kubernetes master every time this - method is called. - - Returns: - A ClusterSpec containing host information returned from Kubernetes. - - Raises: - RuntimeError: If any of the pods returned by the master is not in the - `Running` phase. - """ - if not self._override_client: - k8sconfig.load_kube_config() - - client = self._override_client or k8sclient.CoreV1Api() - cluster_map = {} - - for tf_job in self._job_to_label_mapping: - all_pods = [] - for selector in self._job_to_label_mapping[tf_job]: - ret = client.list_pod_for_all_namespaces(label_selector=selector) - selected_pods = [] - - # Sort the list by the name to make sure it doesn't change call to call. - for pod in sorted(ret.items, key=lambda x: x.metadata.name): - if pod.status.phase == 'Running': - selected_pods.append( - '%s:%s' % (pod.status.host_ip, self._tf_server_port)) - else: - raise RuntimeError('Pod "%s" is not running; phase: "%s"' % - (pod.metadata.name, pod.status.phase)) - all_pods.extend(selected_pods) - cluster_map[tf_job] = all_pods - - return server_lib.ClusterSpec(cluster_map) diff --git a/tensorflow/contrib/cluster_resolver/python/training/slurm_cluster_resolver.py b/tensorflow/contrib/cluster_resolver/python/training/slurm_cluster_resolver.py index dabe2fe1d39..fcd2a846eeb 100644 --- a/tensorflow/contrib/cluster_resolver/python/training/slurm_cluster_resolver.py +++ b/tensorflow/contrib/cluster_resolver/python/training/slurm_cluster_resolver.py @@ -12,185 +12,24 @@ # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================== -"""Implementation of Cluster Resolvers for Slurm workload manager.""" +"""Stub file for SlurmClusterResolver to maintain backwards compatibility.""" from __future__ import absolute_import from __future__ import division from __future__ import print_function -import collections -import os -import subprocess +# This file (and all files in this directory in general) is a backwards +# compatibility shim that exists to re-export ClusterResolvers such that +# existing OSS code will not be broken. -from tensorflow.contrib.cluster_resolver.python.training.cluster_resolver import ClusterResolver -from tensorflow.python.training.server_lib import ClusterSpec +# pylint: disable=unused-import +from tensorflow.python.distribute.cluster_resolver.slurm_cluster_resolver import SlurmClusterResolver +# pylint: enable=unused-import +from tensorflow.python.util.all_util import remove_undocumented -class SlurmClusterResolver(ClusterResolver): - """Cluster Resolver for system with Slurm workload manager. +_allowed_symbols = [ + 'SlurmClusterResolver', +] - This is an implementation of cluster resolvers for Slurm clusters. This allows - the specification of jobs and task counts, number of tasks per node, number of - GPUs on each node and number of GPUs for each task, It retrieves system - attributes by Slurm environment variables, resolves allocated computing node - names, construct a cluster and return a Cluster Resolver object which an be - use for distributed TensorFlow. - """ - - def _resolve_hostnames(self): - """Resolve host names of nodes allocated in current jobs. - - Returns: - A list of node names as strings. - """ - hostlist = (subprocess.check_output(['scontrol', 'show', 'hostname']). - decode('utf-8').strip().split('\n')) - return hostlist - - def __init__(self, - jobs, - port_base=8888, - gpus_per_node=1, - gpus_per_task=1, - tasks_per_node=None, - auto_set_gpu=True): - """Creates a new SlurmClusterResolver object. - - This takes in parameters and creates a SlurmClusterResolver object. It uses - those parameters to check which nodes will processes reside and resolves - their hostnames. With the number of the GPUs on each node and number of GPUs - for each task it offsets the port number for each processes and allocate - GPUs to tasks by setting environment variables. The resolver currently - supports homogeneous tasks and default Slurm process allocation. - - Args: - jobs: Dictionary with job names as key and number of tasks in the job as - value - port_base: The first port number to start with for processes on a node. - gpus_per_node: Number of GPUs available on each node. - gpus_per_task: Number of GPUs to be used for each task. - tasks_per_node: Number of tasks to run on each node, if not set defaults - to Slurm's output environment variable SLURM_NTASKS_PER_NODE. - auto_set_gpu: Set the visible CUDA devices automatically while resolving - the cluster by setting CUDA_VISIBLE_DEVICES environment variable. - Defaults to True. - - Returns: - A ClusterResolver object which can be used with distributed TensorFlow. - - Raises: - RuntimeError: If requested more GPUs per node then available or requested - more tasks then assigned tasks. - """ - - # check if launched by mpirun - if 'OMPI_COMM_WORLD_RANK' in os.environ: - self._rank = int(os.environ['OMPI_COMM_WORLD_RANK']) - num_tasks = int(os.environ['OMPI_COMM_WORLD_SIZE']) - else: - self._rank = int(os.environ['SLURM_PROCID']) - num_tasks = int(os.environ['SLURM_NTASKS']) - - self._jobs = collections.OrderedDict(sorted(jobs.items())) - self._port_base = port_base - - # user specification overrides SLURM specification - if tasks_per_node is not None: - self._tasks_per_node = tasks_per_node - elif tasks_per_node is None and 'SLURM_NTASKS_PER_NODE' in os.environ: - self._tasks_per_node = int(os.environ['SLURM_NTASKS_PER_NODE']) - else: - raise RuntimeError('Neither `tasks_per_node` or ' - 'SLURM_NTASKS_PER_NODE is set.') - - self._gpus_per_node = gpus_per_node - self._gpus_per_task = gpus_per_task - - self._auto_set_gpu = auto_set_gpu - self._job_name = None - self._task_index = None - - self._gpu_allocation = [] - self._cluster_allocation = {} - - if self._tasks_per_node * self._gpus_per_task > self._gpus_per_node: - raise RuntimeError('Requested more GPUs per node then available.') - - if sum(self._jobs.values()) != num_tasks: - raise RuntimeError('Requested more tasks then assigned tasks.') - - def cluster_spec(self): - """Returns a ClusterSpec object based on the latest instance group info. - - This returns a ClusterSpec object for use based on information from the - specified initialization parameters and Slurm environment variables. The - cluster specification is resolved each time this function is called. The - resolver extract hostnames of nodes by scontrol and pack tasks in that - order until a node a has number of tasks that is equal to specification. - GPUs on nodes are allocated to tasks by specification through setting - CUDA_VISIBLE_DEVICES environment variable. - - Returns: - A ClusterSpec containing host information retrieved from Slurm's - environment variables. - """ - hostlist = self._resolve_hostnames() - - task_list = [] - self._gpu_allocation = [] - self._cluster_allocation = {} - - for host in hostlist: - for port_offset, gpu_offset in zip( - range(self._tasks_per_node), - range(0, self._gpus_per_node, self._gpus_per_task)): - - host_addr = '%s:%d' % (host, self._port_base + port_offset) - task_list.append(host_addr) - gpu_id_list = [] - - for gpu_id in range(gpu_offset, gpu_offset + self._gpus_per_task): - gpu_id_list.append(str(gpu_id)) - - self._gpu_allocation.append(','.join(gpu_id_list)) - - cluster_rank_offset_start = 0 - cluster_rank_offset_end = 0 - - for job_name, num_tasks in self._jobs.items(): - cluster_rank_offset_end = cluster_rank_offset_start + num_tasks - - self._cluster_allocation[job_name] = \ - task_list[cluster_rank_offset_start:cluster_rank_offset_end] - - if self._rank >= cluster_rank_offset_start and \ - self._rank < cluster_rank_offset_end: - - self._job_name = job_name - self._task_index = self._rank - cluster_rank_offset_start - - cluster_rank_offset_start = cluster_rank_offset_end - - if self._auto_set_gpu is True: - os.environ['CUDA_VISIBLE_DEVICES'] = self._gpu_allocation[self._rank] - - return ClusterSpec(self._cluster_allocation) - - def get_task_info(self): - """Returns job name and task_index for the process which calls this. - - This returns the job name and task index for the process which calls this - function according to its rank and cluster specification. The job name and - task index are set after a cluster is constructed by cluster_spec otherwise - defaults to None. - - Returns: - A string specifying job name the process belongs to and an integner - specifying the task index the process belongs to in that job. - """ - return self._job_name, self._task_index - - def master(self, task_type=None, task_index=None): - if task_type and task_index: - return self.cluster_spec().task_address(task_type, task_index) - return self._cluster_allocation[str(self._job_name)][self._task_index] +remove_undocumented(__name__, _allowed_symbols) diff --git a/tensorflow/contrib/cluster_resolver/python/training/tfconfig_cluster_resolver.py b/tensorflow/contrib/cluster_resolver/python/training/tfconfig_cluster_resolver.py index 7bbd189d03d..9db7f47dcb4 100644 --- a/tensorflow/contrib/cluster_resolver/python/training/tfconfig_cluster_resolver.py +++ b/tensorflow/contrib/cluster_resolver/python/training/tfconfig_cluster_resolver.py @@ -12,81 +12,25 @@ # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================== -"""Implementation of Cluster Resolvers for TF_CONFIG Environment Variables.""" - +"""Stub file for TFConfigClusterResolver to maintain backwards compatibility.""" from __future__ import absolute_import from __future__ import division from __future__ import print_function -import json -import os +# This file (and all files in this directory in general) is a backwards +# compatibility shim that exists to re-export ClusterResolvers such that +# existing OSS code will not be broken. -from tensorflow.contrib.cluster_resolver.python.training.cluster_resolver import ClusterResolver -from tensorflow.python.training.server_lib import ClusterSpec +# pylint: disable=unused-import +from tensorflow.python.distribute.cluster_resolver.tfconfig_cluster_resolver import TFConfigClusterResolver +# pylint: enable=unused-import -_TF_CONFIG_ENV = 'TF_CONFIG' -_SESSION_MASTER_KEY = 'session_master' +from tensorflow.python.util.all_util import remove_undocumented +_allowed_symbols = [ + 'TFConfigClusterResolver', +] -class TFConfigClusterResolver(ClusterResolver): - """Implementation of a ClusterResolver which reads the TF_CONFIG EnvVar.""" +remove_undocumented(__name__, _allowed_symbols) - def _load_tf_config(self): - return json.loads(os.environ.get(_TF_CONFIG_ENV, '{}')) - - def cluster_spec(self): - """Returns a ClusterSpec based on the TF_CONFIG environment variable. - - Returns: - A ClusterSpec with information from the TF_CONFIG environment variable. - """ - tf_config = self._load_tf_config() - if 'cluster' not in tf_config: - return ClusterSpec({}) - return ClusterSpec(tf_config['cluster']) - - def master(self, task_type=None, task_index=0): - """Returns the master address to use when creating a TensorFlow session. - - Args: - task_type: (String, optional) Overrides and sets the task_type of the - master. - task_index: (Integer, optional) Overrides and sets the task id of the - master. - - Returns: - The address of the master. - - Raises: - RuntimeError: If the task_type or task_id is not specified and the - `TF_CONFIG` environment variable does not contain a task section. - """ - - # If `session_master` is set, just use that. - tf_config = self._load_tf_config() - if _SESSION_MASTER_KEY in tf_config: - return tf_config[_SESSION_MASTER_KEY] - - if 'rpc_layer' in tf_config: - rpclayer = '%s://' % tf_config['rpc_layer'] - else: - rpclayer = '' - - # Return an empty string if we are the only job in the ClusterSpec. - cluster_spec = self.cluster_spec() - if (not cluster_spec.jobs or - (len(cluster_spec.jobs) == 1 and - len(cluster_spec.job_tasks(cluster_spec.jobs[0])) == 1)): - return '' - - # We try to auto-detect the task type and id, but uses the user-supplied one - # where available - if not task_type: - if 'task' not in tf_config: - raise RuntimeError('You must either specify a `task_type`, or your ' - 'TF_CONFIG must contain a `task` section.') - task_type = tf_config['task']['type'] - task_index = tf_config['task']['index'] - - return rpclayer + cluster_spec.task_address(task_type, task_index) diff --git a/tensorflow/contrib/cluster_resolver/python/training/tpu_cluster_resolver.py b/tensorflow/contrib/cluster_resolver/python/training/tpu_cluster_resolver.py index 1f6803a9ff9..3a1eaccd06e 100644 --- a/tensorflow/contrib/cluster_resolver/python/training/tpu_cluster_resolver.py +++ b/tensorflow/contrib/cluster_resolver/python/training/tpu_cluster_resolver.py @@ -1,4 +1,4 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -12,341 +12,24 @@ # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================== -"""Implementation of Cluster Resolvers for Cloud TPUs.""" +"""Stub file for TPUClusterResolver to maintain backwards compatibility.""" from __future__ import absolute_import from __future__ import division from __future__ import print_function -import os +# This file (and all files in this directory in general) is a backwards +# compatibility shim that exists to re-export ClusterResolvers such that +# existing OSS code will not be broken. -from six.moves.urllib.request import Request -from six.moves.urllib.request import urlopen +# pylint: disable=unused-import +from tensorflow.python.distribute.cluster_resolver.tpu_cluster_resolver import TPUClusterResolver +# pylint: enable=unused-import -from tensorflow.contrib.cluster_resolver.python.training.cluster_resolver import ClusterResolver -from tensorflow.python.training import server_lib -from tensorflow.python.util import compat +from tensorflow.python.util.all_util import remove_undocumented -_GOOGLE_API_CLIENT_INSTALLED = True -try: - from googleapiclient import discovery # pylint: disable=g-import-not-at-top - from oauth2client.client import GoogleCredentials # pylint: disable=g-import-not-at-top -except ImportError: - _GOOGLE_API_CLIENT_INSTALLED = False +_allowed_symbols = [ + 'TPUClusterResolver', +] - -_GKE_ENV_VARIABLE = 'KUBE_GOOGLE_CLOUD_TPU_ENDPOINTS' -_ENDPOINTS_SEPARATOR = ',' -_DEFAULT_ENV_VARIABLE = 'TPU_NAME' -_DISCOVERY_SERVICE_URL_ENV_VARIABLE = 'TPU_API_DISCOVERY_URL' - - -class TPUClusterResolver(ClusterResolver): - """Cluster Resolver for Google Cloud TPUs. - - This is an implementation of cluster resolvers for the Google Cloud TPU - service. As Cloud TPUs are in alpha, you will need to specify a API definition - file for this to consume, in addition to a list of Cloud TPUs in your Google - Cloud Platform project. - """ - - def _tpuService(self): - """Creates a new Cloud TPU API object. - - This works around an issue where the underlying HTTP connection sometimes - times out when the script has been running for too long. Other methods in - this object calls this method to get a new API object whenever they need - to communicate with the Cloud API. - - Returns: - A Google Cloud TPU API object. - """ - if self._service: - return self._service - - credentials = self._credentials - if credentials is None or credentials == 'default': - credentials = GoogleCredentials.get_application_default() - - if self._discovery_url: - return discovery.build( - 'tpu', 'v1alpha1', - credentials=credentials, - discoveryServiceUrl=self._discovery_url) - else: - return discovery.build( - 'tpu', 'v1alpha1', - credentials=credentials) - - def _requestComputeMetadata(self, path): - req = Request('http://metadata/computeMetadata/v1/%s' % path, - headers={'Metadata-Flavor': 'Google'}) - resp = urlopen(req) - return compat.as_bytes(resp.read()) - - def _shouldResolve(self): - if (self._tpu == compat.as_bytes('') or - self._tpu == compat.as_bytes('local') or - self._tpu.startswith(compat.as_bytes('/bns')) or - self._tpu.startswith(compat.as_bytes('localhost:')) or - self._tpu.startswith(compat.as_bytes('grpc://'))): - return False - return True - - @staticmethod - def _inGke(): - """When running in GKE, the environment variable will be set.""" - return _GKE_ENV_VARIABLE in os.environ - - @staticmethod - def _gkeEndpoints(): - return os.environ[_GKE_ENV_VARIABLE] - - @staticmethod - def _envVarFallback(): - if _DEFAULT_ENV_VARIABLE in os.environ: - return os.environ[_DEFAULT_ENV_VARIABLE] - return None - - @staticmethod - def _environmentDiscoveryUrl(): - return os.environ.get(_DISCOVERY_SERVICE_URL_ENV_VARIABLE) - - def __init__(self, - tpu=None, - zone=None, - project=None, - job_name='worker', - coordinator_name=None, - coordinator_address=None, - credentials='default', - service=None, - discovery_url=None): - """Creates a new TPUClusterResolver object. - - The ClusterResolver will then use the parameters to query the Cloud TPU APIs - for the IP addresses and ports of each Cloud TPU listed. - - Args: - tpu: Either a string, or a list of strings corresponding to the TPUs to - use. If the single string is the empty string, the string 'local', or a - string that begins with 'grpc://' or '/bns', then it is assumed to not - correspond with a Cloud TPU and will instead be passed as the session - master and no ClusterSpec propagation will be done. - zone: Zone where the TPUs are located. If omitted or empty, we will assume - that the zone of the TPU is the same as the zone of the GCE VM, which we - will try to discover from the GCE metadata service. - project: Name of the GCP project containing Cloud TPUs. If omitted or - empty, we will try to discover the project name of the GCE VM from the - GCE metadata service. - job_name: Name of the TensorFlow job the TPUs belong to. - coordinator_name: The name to use for the coordinator. Set to None if the - coordinator should not be included in the computed ClusterSpec. - coordinator_address: The address of the coordinator (typically an ip:port - pair). If set to None, a TF server will be started. If coordinator_name - is None, a TF server will not be started even if coordinator_address is - None. - credentials: GCE Credentials. If None, then we use default credentials - from the oauth2client - service: The GCE API object returned by the googleapiclient.discovery - function. If you specify a custom service object, then the credentials - parameter will be ignored. - discovery_url: A URL template that points to the location of - the discovery service. It should have two parameters {api} and - {apiVersion} that when filled in produce an absolute URL to the - discovery document for that service. The environment variable - 'TPU_API_DISCOVERY_URL' will override this. - - Raises: - ImportError: If the googleapiclient is not installed. - ValueError: If no TPUs are specified. - """ - if isinstance(tpu, list): - if not tpu: - raise ValueError('At least one TPU must be specified.') - if len(tpu) != 1: - raise NotImplementedError( - 'Using multiple TPUs in a single session is not yet implemented') - tpu = tpu[0] - - in_gke = self._inGke() - # When using GKE with Cloud TPUs, the env variable will be set. - if tpu is None: - if in_gke: - tpu = self._gkeEndpoints() - else: - tpu = self._envVarFallback() - - if tpu is None: - raise ValueError('Please provide a TPU Name to connect to.') - - self._tpu = compat.as_bytes(tpu) # self._tpu is always bytes - self._job_name = job_name - - # Whether we should actually attempt to contact Cloud APIs - should_resolve = self._shouldResolve() - - # We error out if we are in a non-Cloud environment which cannot talk to the - # Cloud APIs using the standard class and a special object is not passed in. - self._service = service - if (self._service is None and should_resolve and - not _GOOGLE_API_CLIENT_INSTALLED): - raise ImportError('googleapiclient and oauth2client must be installed ' - 'before using the TPU cluster resolver. Execute: ' - '`pip install --upgrade google-api-python-client` ' - 'and `pip install --upgrade oauth2client` to ' - 'install with pip.') - - # We save user-passed credentials, unless the user didn't pass in anything. - self._credentials = credentials - if (credentials == 'default' and should_resolve and - _GOOGLE_API_CLIENT_INSTALLED): - self._credentials = None - - # Automatically detect project and zone if unspecified. - if not project and should_resolve: - project = compat.as_str( - self._requestComputeMetadata('project/project-id')) - if not zone and should_resolve: - zone_path = compat.as_str(self._requestComputeMetadata('instance/zone')) - zone = zone_path.split('/')[-1] - self._project = project - self._zone = zone - - self._discovery_url = self._environmentDiscoveryUrl() or discovery_url - - self._coordinator_name = coordinator_name - if (coordinator_name and not coordinator_address and - (should_resolve or in_gke)): - self._start_local_server() - else: - self._coordinator_address = coordinator_address - - def master(self, task_type=None, task_index=None): - """Get the Master string to be used for the session. - - In the normal case, this returns the grpc path (grpc://1.2.3.4:8470) of - first instance in the ClusterSpec returned by the cluster_spec function. - - If a non-TPU name is used when constructing a TPUClusterResolver, that will - be returned instead (e.g. If the tpus argument's value when constructing - this TPUClusterResolver was 'grpc://10.240.1.2:8470', - 'grpc://10.240.1.2:8470' will be returned). - - Args: - task_type: (Optional) The type of the TensorFlow task of the master. - task_index: (Optional) The index of the TensorFlow task of the master. - - Returns: - string, the connection string to use when creating a session. - - Raises: - ValueError: If none of the TPUs specified exists. - """ - if not self._shouldResolve(): - return self._tpu.split(compat.as_bytes(_ENDPOINTS_SEPARATOR))[0] - - cluster_spec = self.cluster_spec() - if task_type and task_index: - return cluster_spec.task_address(task_type, task_index) - - job_tasks = cluster_spec.job_tasks(self._job_name) - if not job_tasks: - raise ValueError('No TPUs exists with the specified names exist.') - - return 'grpc://' + job_tasks[0] - - def get_master(self): - return self.master() - - def get_job_name(self): - if self._shouldResolve(): - return self._job_name - - def cluster_spec(self): - """Returns a ClusterSpec object based on the latest TPU information. - - We retrieve the information from the GCE APIs every time this method is - called. - - Returns: - A ClusterSpec containing host information returned from Cloud TPUs. - - Raises: - RuntimeError: If the provided TPU is not healthy. - """ - ############################################################################ - # There are 5 potential cases this code must handle: - # 1. [Normal case.] We should resolve the TPU name to a set of tasks, and - # a. Create a ClusterSpec that includes the coordinator job - # b. Create a ClusterSpec without the coordinator job. - # 2. [GKE / No API Access.] We should not resolve the TPU name to a set of - # tasks and - # a. Create a ClusterSpec with the coordinator - # b. Create a ClusterSpec without the coordinator - # 3. [Other (legacy non-gRPC).] We should return an empty ClusterSpec. - ############################################################################ - - if self._shouldResolve(): - # Case 1. - full_name = 'projects/%s/locations/%s/nodes/%s' % ( - self._project, self._zone, compat.as_text(self._tpu)) - service = self._tpuService() - request = service.projects().locations().nodes().get(name=full_name) - response = request.execute() - - if 'state' in response and response['state'] != 'READY': - raise RuntimeError('TPU "%s" is not yet ready; state: "%s"' % - (compat.as_text(self._tpu), response['state'])) - - if 'health' in response and response['health'] != 'HEALTHY': - raise RuntimeError('TPU "%s" is unhealthy: "%s"' % - (compat.as_text(self._tpu), response['health'])) - - if 'networkEndpoints' in response: - worker_list = [ - '%s:%s' % (endpoint['ipAddress'], endpoint['port']) - for endpoint in response['networkEndpoints'] - ] - else: - # Fall back to the deprecated response format - instance_url = '%s:%s' % (response['ipAddress'], response['port']) - worker_list = [instance_url] - - cluster_spec = {self._job_name: worker_list} - else: - if not self._tpu.startswith(compat.as_bytes('grpc://')): - # Case 3. - return None - # Case 2. - cluster_spec = { - self._job_name: [ - x[len(compat.as_bytes('grpc://')):] - for x in self._tpu.split(compat.as_bytes(_ENDPOINTS_SEPARATOR)) - ] - } - - if self._coordinator_address: - # {1, 2}.a - cluster_spec[self._coordinator_name] = [self._coordinator_address] - - return server_lib.ClusterSpec(cluster_spec) - - def _start_local_server(self): - address = self._requestComputeMetadata('instance/network-interfaces/0/ip') - self._server = server_lib.Server( - { - 'local': ['0.0.0.0:0'] - }, protocol='grpc', config=None, start=True) - # self._server.target is of the form: grpc://ipaddress:port - target = compat.as_bytes(self._server.target) - splits = target.split(compat.as_bytes(':')) - assert len(splits) == 3, self._server.target - assert splits[0] == compat.as_bytes('grpc'), self._server.target - self._coordinator_port = compat.as_text(splits[2]) - self._coordinator_address = '%s:%s' % ( - address, compat.as_text(self._coordinator_port)) - - def __deepcopy__(self, memo): - # TODO(b/73668574): Remove this once RunConfig avoids performing deepcopy. - return self +remove_undocumented(__name__, _allowed_symbols) diff --git a/tensorflow/contrib/cmake/CMakeLists.txt b/tensorflow/contrib/cmake/CMakeLists.txt index a63366e1361..124d6cfd478 100644 --- a/tensorflow/contrib/cmake/CMakeLists.txt +++ b/tensorflow/contrib/cmake/CMakeLists.txt @@ -12,7 +12,7 @@ if(WIN32) endif() # Project -project(tensorflow C CXX) +project(tensorflow VERSION 1.12.0 LANGUAGES C CXX) # Set C++14 as standard for the whole project set(CMAKE_CXX_STANDARD 14) @@ -193,6 +193,7 @@ if(WIN32) set(CMAKE_SUPPRESS_REGENERATION ON) endif() + if ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "GNU") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fno-exceptions -std=c++11") endif() @@ -281,6 +282,14 @@ else (systemlib_ZLIB) ${zlib_STATIC_LIBRARIES}) endif (systemlib_ZLIB) +if (systemlib_ABSEIL_CPP) + set(tensorflow_EXTERNAL_LIBRARIES ${tensorflow_EXTERNAL_LIBRARIES} + ${abseil_cpp_LIBRARIES}) +else (systemlib_ABSEIL_CPP) + set(tensorflow_EXTERNAL_LIBRARIES ${tensorflow_EXTERNAL_LIBRARIES} + ${abseil_cpp_STATIC_LIBRARIES}) +endif (systemlib_ABSEIL_CPP) + set(tensorflow_EXTERNAL_DEPENDENCIES zlib_copy_headers_to_destination gif_copy_headers_to_destination @@ -394,6 +403,7 @@ if (tensorflow_ENABLE_GPU) set(CUDA_NVCC_FLAGS ${CUDA_NVCC_FLAGS};--include-path ${PROJECT_BINARY_DIR}/$\{build_configuration\};--expt-relaxed-constexpr) set(CUDA_NVCC_FLAGS ${CUDA_NVCC_FLAGS};-ftz=true) # Flush denormals to zero set(CUDA_INCLUDE ${CUDA_TOOLKIT_TARGET_DIR} ${CUDA_TOOLKIT_TARGET_DIR}/extras/CUPTI/include) + include_directories(${CUDA_INCLUDE}) if (WIN32) add_definitions(-DGOOGLE_CUDA=1 -DTF_EXTRA_CUDA_CAPABILITIES=3.7,5.2,6.0,6.1,7.0) @@ -546,14 +556,20 @@ if (tensorflow_ENABLE_GPU) cudnn_version_number=${tensorflow_CUDNN_VERSION}) endif(WIN32) else(tensorflow_ENABLE_GPU) - set(tensorflow_BUILD_INFO_FLAGS --build_config cpu --key_value - msvcp_dll_name=msvcp140.dll) + if(WIN32) + set(tensorflow_BUILD_INFO_FLAGS --build_config cpu --key_value + msvcp_dll_name=msvcp140.dll) + else() + set(tensorflow_BUILD_INFO_FLAGS --build_config cpu) + endif() endif(tensorflow_ENABLE_GPU) -# Find python executable -include(FindPythonInterp) -if(NOT ${PYTHONINTERP_FOUND}) - message(FATAL_ERROR "CMake was unable to find a python interpreter.") +if(tensorflow_BUILD_PYTHON_BINDINGS) + # Find python executable + include(FindPythonInterp) + if(NOT ${PYTHONINTERP_FOUND}) + message(FATAL_ERROR "CMake was unable to find a python interpreter.") + endif() endif() # Let's get to work! @@ -574,6 +590,7 @@ include(tf_cc_ops.cmake) include(tf_c.cmake) include(tf_grappler.cmake) include(tf_core_profiler.cmake) +include(tf_core_eager_runtime.cmake) if(tensorflow_BUILD_CC_EXAMPLE) include(tf_tutorials.cmake) include(tf_label_image_example.cmake) @@ -587,4 +604,4 @@ if(tensorflow_BUILD_SHARED_LIB) endif() if(tensorflow_BUILD_CC_TESTS OR tensorflow_BUILD_PYTHON_TESTS) include(tf_tests.cmake) -endif() +endif() \ No newline at end of file diff --git a/tensorflow/contrib/cmake/README.md b/tensorflow/contrib/cmake/README.md index 84c679162c3..df5ff6cd532 100644 --- a/tensorflow/contrib/cmake/README.md +++ b/tensorflow/contrib/cmake/README.md @@ -6,9 +6,9 @@ platforms. For details, see the [TensorFlow install guide](https://www.tensorflow.org/install/). This directory contains CMake files for building TensorFlow on Microsoft -Windows. [CMake](https://cmake.org) is a cross-platform tool that can +Windows and Linux. [CMake](https://cmake.org) is a cross-platform tool that can generate build scripts for multiple build systems, including Microsoft -Visual Studio. +Visual Studio and GCC. "The method has not been tested on Mac OS X. **N.B.** We provide Linux build instructions primarily for the purpose of testing the build. We recommend using the standard Bazel-based build on @@ -23,6 +23,7 @@ for instructions on how to install a pre-built TensorFlow package on Windows. ### Current known limitations * It is not possible to load a custom Op library. * GCS file system is not supported. +* Debug build is not available since Python for Windows is no longer distributed with a debug library. ## Building with CMake @@ -53,12 +54,12 @@ bindings. ### Known-good configurations * Microsoft Windows 10 - - Microsoft Visual Studio Enterprise 2015 with Visual C++ 2015 + - Microsoft Visual Studio Enterprise/ Community 2015 with Visual C++ 2015 - [Anaconda 4.1.1 (Python 3.5 64-bit)](https://www.anaconda.com/download/) - [Git for Windows version 2.9.2.windows.1](https://git-scm.com/download/win) - [swigwin-3.0.10](http://www.swig.org/download.html) - - [NVidia CUDA Toolkit 8.0](https://developer.nvidia.com/cuda-downloads) - - [NVidia CUDNN 5.1](https://developer.nvidia.com/cudnn) + - [NVidia CUDA Toolkit 9.0](https://developer.nvidia.com/cuda-downloads) + - [NVidia CUDNN 7](https://developer.nvidia.com/cudnn) - [CMake 3.6](https://cmake.org/files/v3.6/cmake-3.6.3-win64-x64.msi) * Ubuntu 14.04 @@ -66,8 +67,8 @@ bindings. - Docker 1.9.1 (for automated testing) ### Current known limitations - - The Python package supports **Python 3.5 only**, because that is the only - version for which standard Python binaries exist and those binaries are + - The Python package supports **Python 3.5/3.6 only**, because these are the only + versions for which standard Python binaries exist and those binaries are compatible with the TensorFlow runtime. (On Windows, the standard Python binaries for versions earlier than 3.5 were compiled with older compilers that do not have all of the features (e.g. C++11 support) needed to compile @@ -104,8 +105,151 @@ We are actively working on improving CMake and Windows support, and addressing these limitations. We would appreciate pull requests that implement missing ops or APIs. +CMake GUI build (all platforms) +================================== +Install from CMake GUI would be a convenient way to generate C++ build projects. The software supports Windows, MacOS and Linux, while the posix platform provides an extra ccmake binary to run command line GUI. Both working principal of cmake, ccmake and cmake-gui are the same, the only difference is by providing suitable interface for project configuration and dependency setting. -Step-by-step Windows build +0. Pre-buid checklist: + The following binary/libraries should be setted in system path, otherwise you need to set manualy via cmake. + * Compiler (GCC for Linux, MSVC for Windows) + * Make sure compiler directory has been set to system path + * CUDA 9.0 (GPU build) + * CUDNN (GPU build) + * NCCL (GPU build on Linux) + * SWIG (python binding) + * Perl (required if you need ssl support, optional) + * Go (required if you need ssl support, optional) + * NASM/YASM (required by grpc for ssl support, optional) +1. Start CMake GUI +2. Click on `Browse Source` and direct to the the folder `/tensorflow/contrib/cmake` +3. Click on `Browse Build` and spectify a location that you want tensorflow to be build +4. Click on `Configure`, a new window will be prompted out, specify the generator mode for the project generation. For Windows, choose `Visual Studio Win64`, for Linux, choose `Unix Makefiles`, then press `Finish`. Wait for a moment, the default project dependecy would automatically generate. +5. There are a few options that you can customize your own build. **The setting here is crucial for a sucessful build, please check all items carefully.** + * `tensorflow_BUILD_ALL_KERNELS` should alway be `on` + * `tensorflow_BUILD_CC_EXAMPLE` is default to be `on`. This can help you to test build (optional) + * `tensorflow_BUILD_CONTRIB_KERNELS` is default to be `on`, but it won't affect tensorflow function, turn it to `off` if you want a slim build. (optional) + * `tensorflow_BUILD_PYTHON_BINDING` is default to be `on`. Set to `off` if you don't need python interaface. If SWIG is not in system path, you need set it manually. (optional) + * `tensorflow_BUILD_SHARED_LIB` is default to be `off`. Set to `on` if you want the c++ interface. (optional) + * `tensorflow_ENABLE_GPU` is default to be `off`. Set to `on` if you want GPU support. It will search CUDA and CUDNN dependecies if you have set them to system path, otherwise CMake would prompt error and request you to set it manually. (optional) + * `tensorflow_ENABLE_GRPC_SUPPORT` is default to be `on`. For Linux build, this option must always be `on`. This need to be `on` for a gpu build. Reminded that Perl, Go and NASM/YASM are required for this option if you want to build grpc with offical SSL support. + * `tensorflow_ENABLE_POSITION_INDEPENDENT_CODE` should always be `on` + * `tensorflow_ENABLE_SNAPPY_SUPPORT` should always be `on` + * `tensorflow_OPTIMIZE_FOR_NATIVE_ARCH` should always be `on` + * `CMAKE_INSTALL_PREFIX` is the location where the final package will be installed. You may change it to your own preferred path (optional) + +6. After changing the configuration in step 5, press `Configure` again +7. If not error is found, press `Generate` + +#### Windows + +1. Open `tensorflow.sln` in the build folder (Windows). Change build type from `Debug` to `Release`. Choose `Build`->`Build Solution`. This may take more than hours of compilation. If everything is alright, the output window would show no error. + + ##### Python + + In solution explorer, right click on `tf_python_build_pip_package` -> `build`. It will generate the wheel file in `/tf_python/dist`. Install with following command: + + ```pip install --upgrade tensorflow-.whl``` + + ***The wheel name varies depends on you config. Change to your own wheel filename.*** + + Reminded that some pip installation requires administrator right command prompt. + + ##### C++ + + You can directly use the build folder tree for C++ interface with cmake. If you want to do installation for api releasing, right click on `Install` -> `build`. The headers and library will be installed in the directory specify by `CMAKE_INSTALL_PREFIX` during configuration. + +2. For smaller RAM computer, it is noticed that out of heap space error appears. Change to command prompt build is an alternative to do step 1. + + Open `VS2015 x64 Native Tools Command Prompt`. You can open it by press `Start`, then type the binary name. Use `VS2017 x64 Native Tools Command Prompt` if you are using MSVC 2017. + + ##### Python + + Directly build python wheel package by following command: + + ```MSBuild /p:Configuration=Release ``` + + Remember to change `` to the actual path of the file, it can be found at the root of build directory + + Install the wheel file generated as instructed by step 1. + + ##### C++ interface + Build from VS native toolchain with following command: + ```MSBuild /p:Configuration=Release ``` + + Headers are discretely located in the build folders. Tensorflow library can be found at `/Release`, namely `tensorflow.dll` and `tensorflow.lib`. + + * Build to install for api release (optional): + ```MSBuild /p:Configuration=Release ``` + + Remember to change `` and `` to the actual path of the file, it can be found at the root of build directory. + +#### Linux/MacOS (command line GNU build) + +1. Open the terminal, change working directory to the one specified in step 3. + +2. Type the following command: + + ```make -sj all``` + + ##### Python + + **Important Note** CMake generated python wheel for Linux/MacOs is currently under development. Please use bazel build. + + Follow code is an expected Linux/MacOS python package build after development work is completed. + + ``` + make -sj tf_python_build_pip_package + cd tf_python + pip install --upgrade tensorflow-.whl + ``` + + ##### C++ interface + + ```make -sj install``` + + Where `` is the threads used for the compilation, change to any integer less or equal to your computer's maxiumum thread number. + + Headers are discretely located in the build folders. Tensorflow library can be found at ``, namely `tensorflow.so` (Linux) or `tensorflow.dylib` (MacOS). + +#### Start a Tensorflow C++ project with CMake +Here we assume that you have basic knowledge on gathering dependency with `CMakeLists.txt`. Here we introduce how the C++ api works with [official hello world tutorial](https://www.tensorflow.org/api_guides/cc/guide). + +1. Create a new working directory and create a new text file named `CMakeLists.txt` and the c++ file `main.cxx` +2. Fill in the `main.cxx` with the code provided in [official c++ api basic](https://www.tensorflow.org/api_guides/cc/guide). +3. Fill in the `CMakeLists.txt` with following code: + ``` cmake + cmake_minimum_required (VERSION 2.6) + project (tf_hello) + + # Tensorflow + find_package(Tensorflow REQUIRED) + include_directories(${TENSORFLOW_INCLUDE_DIRS}) + + # compiler setting required by tensorflow, to be tested on all compilers + # currently only tested on MSVC and GCC + if (${CMAKE_CXX_COMPILER_ID} STREQUAL MSVC) + add_definitions(-DCOMPILER_MSVC) + elseif (${CMAKE_CXX_COMPILER_ID} STREQUAL GNU) + if (${CMAKE_CXX_COMPILER_VERSION} VERSION_LESS "3") + add_definitions(-DCOMPILER_GCC3) + else() + add_definitions(-D__GNUC__) + endif() + else() + message(ERROR " compiler ${CMAKE_CXX_COMPILER_ID} not supported by this CMakeList.txt, under development") + endif() + + add_executable(tf_hello main.cxx) + target_link_libraries(tf_hello ${TENSORFLOW_LIBRARIES}) + ``` +4. Configure the folder with cmake-gui, an error should be prompted out, requesting you to locate the folder containing `TensorflowConfig.cmake`. This file can be found at `` or `` (for those have build install in previous steps). + +5. Configure again, generate the project. +6. Compile the project with `Release` config (Windows). For Linux users, just compile the project. +7. Copy the `tensorflow.dll`(Windows)/`tensorflow.so`(Linux) from build directory to the build folder containing `tf_hello` binary. +8. Run `tf_hello` binary + +Step-by-step Windows build (command prompt) ========================== 1. Install the prerequisites detailed above, and set up your environment. @@ -292,4 +436,4 @@ $ cd tensorflow $ tensorflow/tools/ci_build/ci_build.sh CMAKE tensorflow/tools/ci_build/builds/cmake.sh ``` -That's it. Dependencies included. +That's it. Dependencies included. \ No newline at end of file diff --git a/tensorflow/contrib/cmake/TensorflowConfig.cmake.in b/tensorflow/contrib/cmake/TensorflowConfig.cmake.in new file mode 100644 index 00000000000..cc04db6e952 --- /dev/null +++ b/tensorflow/contrib/cmake/TensorflowConfig.cmake.in @@ -0,0 +1,16 @@ +# - Config file for the Tensorflow package +# It defines the following variables +# TENSORFLOW_INCLUDE_DIRS - include directories for FooBar +# TENSORFLOW_LIBRARIES - libraries to link against + +# Compute paths +get_filename_component(TENSORFLOW_CMAKE_DIR "${CMAKE_CURRENT_LIST_FILE}" PATH) +set(TENSORFLOW_INCLUDE_DIRS "@CONF_INCLUDE_DIRS@") + +# Our library dependencies (contains definitions for IMPORTED targets) +if(NOT TENSORFLOW_BINARY_DIR) + include("${TENSORFLOW_CMAKE_DIR}/TensorflowTargets.cmake") +endif() + +# These are IMPORTED targets created by TensorflowTargets.cmake +set(TENSORFLOW_LIBRARIES tensorflow) \ No newline at end of file diff --git a/tensorflow/contrib/cmake/TensorflowConfigVersion.cmake.in b/tensorflow/contrib/cmake/TensorflowConfigVersion.cmake.in new file mode 100644 index 00000000000..2a9609ddb9c --- /dev/null +++ b/tensorflow/contrib/cmake/TensorflowConfigVersion.cmake.in @@ -0,0 +1,11 @@ +set(PACKAGE_VERSION "@TENSORFLOW_VERSION@") + +# Check whether the requested PACKAGE_FIND_VERSION is compatible +if("${PACKAGE_VERSION}" VERSION_LESS "${PACKAGE_FIND_VERSION}") + set(PACKAGE_VERSION_COMPATIBLE FALSE) +else() + set(PACKAGE_VERSION_COMPATIBLE TRUE) + if ("${PACKAGE_VERSION}" VERSION_EQUAL "${PACKAGE_FIND_VERSION}") + set(PACKAGE_VERSION_EXACT TRUE) + endif() +endif() \ No newline at end of file diff --git a/tensorflow/contrib/cmake/external/abseil_cpp.cmake b/tensorflow/contrib/cmake/external/abseil_cpp.cmake index 4546dbdecc0..46a193971c5 100644 --- a/tensorflow/contrib/cmake/external/abseil_cpp.cmake +++ b/tensorflow/contrib/cmake/external/abseil_cpp.cmake @@ -31,27 +31,24 @@ if (systemlib_ABSEIL_CPP) message(STATUS " abseil_cpp includes: ${ABSEIL_CPP_INCLUDE_DIR}") message(STATUS " abseil_cpp libraries: ${ABSEIL_CPP_LIBRARIES}") - add_custom_target(abseil_cpp_build) - list(APPEND tensorflow_EXTERNAL_DEPENDENCIES abseil_cpp_build) + add_custom_target(abseil_cpp) + list(APPEND tensorflow_EXTERNAL_DEPENDENCIES abseil_cpp) else (systemlib_ABSEIL_CPP) include (ExternalProject) - set(abseil_cpp_INCLUDE_DIR ${CMAKE_BINARY_DIR}/abseil_cpp/src/abseil_cpp_build) + set(abseil_cpp_INCLUDE_DIR ${CMAKE_BINARY_DIR}/abseil_cpp/src/abseil_cpp) set(abseil_cpp_URL https://github.com/abseil/abseil-cpp/archive/e01d95528ea2137a4a27a88d1f57c6cb260aafed.tar.gz) set(abseil_cpp_HASH SHA256=84043ed402d2a2a6ba4cdddb7e85118b1158fd81fe4ac3a14adc343d054c1e2e) - set(abseil_cpp_BUILD ${CMAKE_BINARY_DIR}/abseil_cpp/src/abseil_cpp_build) + set(abseil_cpp_BUILD ${CMAKE_BINARY_DIR}/abseil_cpp/src/abseil_cpp-build) if(WIN32) if(${CMAKE_GENERATOR} MATCHES "Visual Studio.*") set(abseil_cpp_STATIC_LIBRARIES ${abseil_cpp_BUILD}/absl/base/Release/absl_base.lib - ${abseil_cpp_BUILD}/absl/base/Release/absl_spinlock_wait.lib ${abseil_cpp_BUILD}/absl/base/Release/absl_dynamic_annotations.lib - ${abseil_cpp_BUILD}/absl/base/Release/absl_malloc_internal.lib - ${abseil_cpp_BUILD}/absl/base/Release/absl_throw_delegate.lib - ${abseil_cpp_BUILD}/absl/numeric/Release/absl_int128.lib + ${abseil_cpp_BUILD}/absl/base/Release/absl_internal_malloc_internal.lib ${abseil_cpp_BUILD}/absl/strings/Release/absl_strings.lib ${abseil_cpp_BUILD}/absl/strings/Release/str_format_internal.lib ${abseil_cpp_BUILD}/absl/types/Release/absl_bad_optional_access.lib) @@ -80,15 +77,12 @@ else (systemlib_ABSEIL_CPP) ${abseil_cpp_BUILD}/absl/types/libabsl_bad_optional_access.a) endif() - ExternalProject_Add(abseil_cpp_build + ExternalProject_Add(abseil_cpp PREFIX abseil_cpp URL ${abseil_cpp_URL} URL_HASH ${abseil_cpp_HASH} DOWNLOAD_DIR "${DOWNLOAD_LOCATION}" - BUILD_IN_SOURCE 1 BUILD_BYPRODUCTS ${abseil_cpp_STATIC_LIBRARIES} - BUILD_COMMAND ${CMAKE_COMMAND} --build . --config Release - COMMAND ${CMAKE_COMMAND} --build . --config Release INSTALL_COMMAND "" CMAKE_CACHE_ARGS -DCMAKE_POSITION_INDEPENDENT_CODE:BOOL=${tensorflow_ENABLE_POSITION_INDEPENDENT_CODE} @@ -99,6 +93,6 @@ else (systemlib_ABSEIL_CPP) include_directories(${abseil_cpp_INCLUDE_DIR}) list(APPEND tensorflow_EXTERNAL_LIBRARIES ${abseil_cpp_STATIC_LIBRARIES}) - list(APPEND tensorflow_EXTERNAL_DEPENDENCIES abseil_cpp_build) + list(APPEND tensorflow_EXTERNAL_DEPENDENCIES abseil_cpp) -endif (systemlib_ABSEIL_CPP) +endif (systemlib_ABSEIL_CPP) \ No newline at end of file diff --git a/tensorflow/contrib/cmake/external/png.cmake b/tensorflow/contrib/cmake/external/png.cmake index 1a147e9c8e5..32e6d78e508 100644 --- a/tensorflow/contrib/cmake/external/png.cmake +++ b/tensorflow/contrib/cmake/external/png.cmake @@ -59,6 +59,7 @@ ExternalProject_Add(png -DCMAKE_VERBOSE_MAKEFILE:BOOL=OFF -DCMAKE_INSTALL_PREFIX:STRING=${png_INSTALL} -DZLIB_ROOT:STRING=${ZLIB_INSTALL} + -DPNG_TESTS:BOOL=OFF ) ## put png includes in the directory where they are expected diff --git a/tensorflow/contrib/cmake/modules/FindAbseilCpp.cmake b/tensorflow/contrib/cmake/modules/FindAbseilCpp.cmake index d4f8bb1bec9..944ae3997a9 100644 --- a/tensorflow/contrib/cmake/modules/FindAbseilCpp.cmake +++ b/tensorflow/contrib/cmake/modules/FindAbseilCpp.cmake @@ -24,10 +24,10 @@ if(EXISTS "${ABSEIL_CPP_INCLUDE_DIR}" AND NOT "${ABSEIL_CPP_INCLUDE_DIR}" STREQU # search all libraries if no COMPONENTS was requested set(AbseilCpp_FIND_COMPONENTS "absl_algorithm;absl_any;absl_bad_any_cast" - "absl_bad_optional_access;absl_base absl_container;absl_debugging" + "absl_bad_optional_access;absl_base;absl_container;absl_debugging" "absl_dynamic_annotations;absl_examine_stack;absl_failure_signal_handler" - "absl_int128;absl_leak_check;absl_malloc_internal;absl_memory;absl_meta" - "absl_numeric;absl_optional;absl_span;absl_spinlock_wait;absl_stack_consumption" + "absl_int128;absl_leak_check;absl_internal_malloc_internal;absl_memory;absl_meta" + "absl_numeric;absl_optional;absl_span;absl_internal_spinlock_wait;absl_stack_consumption" "absl_stacktrace;absl_str_format;absl_strings;absl_symbolize;absl_synchronization" "absl_throw_delegate;absl_time;absl_utility;str_format_extension_internal" "str_format_internal;test_instance_tracker_lib") diff --git a/tensorflow/contrib/cmake/tf_c.cmake b/tensorflow/contrib/cmake/tf_c.cmake index 7a30eb94f54..a04142bd249 100644 --- a/tensorflow/contrib/cmake/tf_c.cmake +++ b/tensorflow/contrib/cmake/tf_c.cmake @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================== + ######################################################## # tf_c_framework library ######################################################## diff --git a/tensorflow/contrib/cmake/tf_core_cpu.cmake b/tensorflow/contrib/cmake/tf_core_cpu.cmake index a54cbff33b6..d8884d464fb 100644 --- a/tensorflow/contrib/cmake/tf_core_cpu.cmake +++ b/tensorflow/contrib/cmake/tf_core_cpu.cmake @@ -39,6 +39,8 @@ file(GLOB_RECURSE tf_core_cpu_exclude_srcs "${tensorflow_source_dir}/tensorflow/core/*test*.h" "${tensorflow_source_dir}/tensorflow/core/*test*.cc" "${tensorflow_source_dir}/tensorflow/core/*main.cc" + "${tensorflow_source_dir}/tensorflow/core/common_runtime/eager/*.cc" + "${tensorflow_source_dir}/tensorflow/core/common_runtime/eager/*.h" "${tensorflow_source_dir}/tensorflow/core/common_runtime/gpu/*.cc" "${tensorflow_source_dir}/tensorflow/core/common_runtime/gpu_device_factory.cc" "${tensorflow_source_dir}/tensorflow/core/common_runtime/direct_session.cc" diff --git a/tensorflow/contrib/cmake/tf_core_eager_runtime.cmake b/tensorflow/contrib/cmake/tf_core_eager_runtime.cmake new file mode 100644 index 00000000000..78e4c0d3035 --- /dev/null +++ b/tensorflow/contrib/cmake/tf_core_eager_runtime.cmake @@ -0,0 +1,57 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +######################################################## +# tf_core_eager_runtime library +######################################################## +file(GLOB_RECURSE tf_core_eager_runtime_srcs + "${tensorflow_source_dir}/tensorflow/core/common_runtime/eager/*.cc" + "${tensorflow_source_dir}/tensorflow/core/common_runtime/eager/*.h" +) + +file(GLOB_RECURSE tf_core_eager_runtime_exclude_srcs + "${tensorflow_source_dir}/tensorflow/core/common_runtime/eager/*test*.h" + "${tensorflow_source_dir}/tensorflow/core/common_runtime/eager/*test*.cc" +) + +list(REMOVE_ITEM tf_core_eager_runtime_srcs ${tf_core_eager_runtime_exclude_srcs}) + +add_library(tf_core_eager_runtime OBJECT ${tf_core_eager_runtime_srcs}) +add_dependencies( + tf_core_eager_runtime + tf_c + tf_core_lib) + + +file(GLOB_RECURSE tf_c_eager_srcs + "${tensorflow_source_dir}/tensorflow/c/eager/*.cc" + "${tensorflow_source_dir}/tensorflow/c/eager/*.h" +) + +file(GLOB_RECURSE tf_c_eager_exlclude_srcs + "${tensorflow_source_dir}/tensorflow/c/eager/*test*.h" + "${tensorflow_source_dir}/tensorflow/c/eager/*test*.cc" +) + +list(REMOVE_ITEM tf_c_eager_srcs ${tf_c_eager_exlclude_srcs}) + +add_library(tf_c_eager OBJECT ${tf_c_eager_srcs}) +add_dependencies( + tf_c_eager + tf_core_eager_runtime + tf_c + tf_cc_framework + tf_cc_while_loop + tf_core_lib + tf_protos_cc) \ No newline at end of file diff --git a/tensorflow/contrib/cmake/tf_core_framework.cmake b/tensorflow/contrib/cmake/tf_core_framework.cmake index 7e806685b84..d7b2a1339e0 100644 --- a/tensorflow/contrib/cmake/tf_core_framework.cmake +++ b/tensorflow/contrib/cmake/tf_core_framework.cmake @@ -140,16 +140,19 @@ set(tf_proto_text_srcs "tensorflow/core/example/example.proto" "tensorflow/core/example/feature.proto" "tensorflow/core/framework/allocation_description.proto" + "tensorflow/core/framework/api_def.proto" "tensorflow/core/framework/attr_value.proto" "tensorflow/core/framework/cost_graph.proto" "tensorflow/core/framework/device_attributes.proto" "tensorflow/core/framework/function.proto" "tensorflow/core/framework/graph.proto" "tensorflow/core/framework/graph_transfer_info.proto" + "tensorflow/core/framework/iterator.proto" "tensorflow/core/framework/kernel_def.proto" "tensorflow/core/framework/log_memory.proto" "tensorflow/core/framework/node_def.proto" "tensorflow/core/framework/op_def.proto" + "tensorflow/core/framework/reader_base.proto" "tensorflow/core/framework/remote_fused_graph_execute_info.proto" "tensorflow/core/framework/resource_handle.proto" "tensorflow/core/framework/step_stats.proto" @@ -159,6 +162,7 @@ set(tf_proto_text_srcs "tensorflow/core/framework/tensor_shape.proto" "tensorflow/core/framework/tensor_slice.proto" "tensorflow/core/framework/types.proto" + "tensorflow/core/framework/variable.proto" "tensorflow/core/framework/versions.proto" "tensorflow/core/lib/core/error_codes.proto" "tensorflow/core/protobuf/cluster.proto" @@ -204,10 +208,10 @@ file(GLOB tf_core_platform_srcs "${tensorflow_source_dir}/tensorflow/core/framework/resource_handle.h" "${tensorflow_source_dir}/tensorflow/core/framework/resource_handle.cc") if (NOT tensorflow_ENABLE_GPU) - file(GLOB tf_core_platform_gpu_srcs + file(GLOB tf_core_platform_gpu_srcs_exclude "${tensorflow_source_dir}/tensorflow/core/platform/cuda_libdevice_path.*" "${tensorflow_source_dir}/tensorflow/core/platform/default/cuda_libdevice_path.*") - list(REMOVE_ITEM tf_core_platform_srcs ${tf_core_platform_gpu_srcs}) + list(REMOVE_ITEM tf_core_platform_srcs ${tf_core_platform_gpu_srcs_exclude}) else() file(GLOB tf_core_platform_srcs_exclude "${tensorflow_source_dir}/tensorflow/core/platform/default/device_tracer.cc") diff --git a/tensorflow/contrib/cmake/tf_core_ops.cmake b/tensorflow/contrib/cmake/tf_core_ops.cmake index 9cfa8b90749..6e75963313a 100644 --- a/tensorflow/contrib/cmake/tf_core_ops.cmake +++ b/tensorflow/contrib/cmake/tf_core_ops.cmake @@ -13,13 +13,14 @@ # limitations under the License. # ============================================================================== set(tf_op_lib_names - "audio_ops" "array_ops" + "audio_ops" "batch_ops" "bitwise_ops" "boosted_trees_ops" "candidate_sampling_ops" "checkpoint_ops" + "collective_ops" "control_flow_ops" "ctc_ops" "cudnn_rnn_ops" @@ -32,8 +33,8 @@ set(tf_op_lib_names "io_ops" "linalg_ops" "list_ops" - "lookup_ops" "logging_ops" + "lookup_ops" "manip_ops" "math_ops" "nn_ops" @@ -43,10 +44,11 @@ set(tf_op_lib_names "remote_fused_graph_ops" "resource_variable_ops" "rpc_ops" + "scoped_allocator_ops" "script_ops" "sdca_ops" - "set_ops" "sendrecv_ops" + "set_ops" "sparse_ops" "spectral_ops" "state_ops" @@ -54,6 +56,7 @@ set(tf_op_lib_names "string_ops" "summary_ops" "training_ops" + "word2vec_ops" ) foreach(tf_op_lib_name ${tf_op_lib_names}) diff --git a/tensorflow/contrib/cmake/tf_python.cmake b/tensorflow/contrib/cmake/tf_python.cmake index df7b854afcc..50284985982 100755 --- a/tensorflow/contrib/cmake/tf_python.cmake +++ b/tensorflow/contrib/cmake/tf_python.cmake @@ -313,15 +313,14 @@ function(GENERATE_PYTHON_OP_LIB tf_python_op_lib_name) ${GENERATE_PYTHON_OP_LIB_DESTINATION} PARENT_SCOPE) endfunction() -GENERATE_PYTHON_OP_LIB("audio_ops") GENERATE_PYTHON_OP_LIB("array_ops") +GENERATE_PYTHON_OP_LIB("audio_ops") GENERATE_PYTHON_OP_LIB("batch_ops") GENERATE_PYTHON_OP_LIB("bitwise_ops") GENERATE_PYTHON_OP_LIB("boosted_trees_ops") -GENERATE_PYTHON_OP_LIB("math_ops") -GENERATE_PYTHON_OP_LIB("functional_ops") GENERATE_PYTHON_OP_LIB("candidate_sampling_ops") GENERATE_PYTHON_OP_LIB("checkpoint_ops") +GENERATE_PYTHON_OP_LIB("collective_ops") GENERATE_PYTHON_OP_LIB("control_flow_ops" ADDITIONAL_LIBRARIES $) GENERATE_PYTHON_OP_LIB("ctc_ops") @@ -332,14 +331,18 @@ GENERATE_PYTHON_OP_LIB("decode_proto_ops" DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/tf_python/tensorflow/contrib/proto/python/ops/gen_decode_proto_op.py) GENERATE_PYTHON_OP_LIB("encode_proto_ops" DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/tf_python/tensorflow/contrib/proto/python/ops/gen_encode_proto_op.py) +GENERATE_PYTHON_OP_LIB("function_ops") +GENERATE_PYTHON_OP_LIB("functional_ops") GENERATE_PYTHON_OP_LIB("image_ops") GENERATE_PYTHON_OP_LIB("io_ops") GENERATE_PYTHON_OP_LIB("linalg_ops") GENERATE_PYTHON_OP_LIB("list_ops") GENERATE_PYTHON_OP_LIB("logging_ops") GENERATE_PYTHON_OP_LIB("lookup_ops") -GENERATE_PYTHON_OP_LIB("nn_ops") GENERATE_PYTHON_OP_LIB("manip_ops") +GENERATE_PYTHON_OP_LIB("math_ops") +GENERATE_PYTHON_OP_LIB("nn_ops") +GENERATE_PYTHON_OP_LIB("no_op") GENERATE_PYTHON_OP_LIB("parsing_ops") GENERATE_PYTHON_OP_LIB("random_ops") GENERATE_PYTHON_OP_LIB("remote_fused_graph_ops" @@ -347,17 +350,21 @@ GENERATE_PYTHON_OP_LIB("remote_fused_graph_ops" GENERATE_PYTHON_OP_LIB("resource_variable_ops") GENERATE_PYTHON_OP_LIB("rpc_ops" DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/tf_python/tensorflow/contrib/rpc/python/ops/gen_rpc_op.py) +GENERATE_PYTHON_OP_LIB("scoped_allocator_ops") GENERATE_PYTHON_OP_LIB("script_ops") GENERATE_PYTHON_OP_LIB("sdca_ops") +GENERATE_PYTHON_OP_LIB("sendrecv_ops") GENERATE_PYTHON_OP_LIB("set_ops") -GENERATE_PYTHON_OP_LIB("state_ops") GENERATE_PYTHON_OP_LIB("sparse_ops") GENERATE_PYTHON_OP_LIB("spectral_ops") +GENERATE_PYTHON_OP_LIB("state_ops") +GENERATE_PYTHON_OP_LIB("stateless_random_ops") GENERATE_PYTHON_OP_LIB("string_ops") GENERATE_PYTHON_OP_LIB("summary_ops") GENERATE_PYTHON_OP_LIB("user_ops") GENERATE_PYTHON_OP_LIB("training_ops" DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/tf_python/tensorflow/python/training/gen_training_ops.py) +GENERATE_PYTHON_OP_LIB("word2vec_ops") GENERATE_PYTHON_OP_LIB("contrib_boosted_trees_model_ops" DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/tf_python/tensorflow/contrib/boosted_trees/python/ops/gen_model_ops.py) @@ -391,11 +398,8 @@ GENERATE_PYTHON_OP_LIB("contrib_layers_sparse_feature_cross_ops" DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/tf_python/tensorflow/contrib/layers/ops/gen_sparse_feature_cross_op.py) GENERATE_PYTHON_OP_LIB("contrib_memory_stats_ops" DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/tf_python/tensorflow/contrib/memory_stats/ops/gen_memory_stats_ops.py) -GENERATE_PYTHON_OP_LIB("contrib_nccl_ops" - DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/tf_python/tensorflow/contrib/nccl/ops/gen_nccl_ops.py) GENERATE_PYTHON_OP_LIB("contrib_periodic_resample_ops" DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/tf_python/tensorflow/contrib/periodic_resample/python/ops/gen_periodic_resample_op.py) - GENERATE_PYTHON_OP_LIB("contrib_nearest_neighbor_ops" DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/tf_python/tensorflow/contrib/nearest_neighbor/ops/gen_nearest_neighbor_ops.py) GENERATE_PYTHON_OP_LIB("contrib_resampler_ops" @@ -524,11 +528,13 @@ if(WIN32) add_library(pywrap_tensorflow_internal_static STATIC ${pywrap_tensorflow_internal_src} $ + $ $ $ $ $ $ + $ $ $ $ @@ -581,11 +587,13 @@ endif(WIN32) add_library(pywrap_tensorflow_internal SHARED ${pywrap_tensorflow_internal_src} $ + $ $ $ $ $ $ + $ $ $ $ @@ -615,13 +623,28 @@ target_include_directories(pywrap_tensorflow_internal PUBLIC ${NUMPY_INCLUDE_DIR} ) -target_link_libraries(pywrap_tensorflow_internal PRIVATE +if(CMAKE_COMPILER_IS_GNUCC AND CMAKE_CXX_COMPILER_VERSION VERSION_GREATER 5.0) + # There is a bug in GCC 5 resulting in undefined reference to a __cpu_model function when + # linking to the tensorflow library. Adding the following libraries fixes it. + # See issue on github: https://github.com/tensorflow/tensorflow/issues/9593 + target_link_libraries(pywrap_tensorflow_internal PRIVATE + ${tf_core_gpu_kernels_lib} + ${tensorflow_EXTERNAL_LIBRARIES} + tf_protos_cc + tf_python_protos_cc + ${PYTHON_LIBRARIES} + gcc_s + gcc +) +else() + target_link_libraries(pywrap_tensorflow_internal PRIVATE ${tf_core_gpu_kernels_lib} ${tensorflow_EXTERNAL_LIBRARIES} tf_protos_cc tf_python_protos_cc ${PYTHON_LIBRARIES} ) +endif() if(WIN32) diff --git a/tensorflow/contrib/cmake/tf_shared_lib.cmake b/tensorflow/contrib/cmake/tf_shared_lib.cmake index fdf522f1fd9..62005dd113b 100644 --- a/tensorflow/contrib/cmake/tf_shared_lib.cmake +++ b/tensorflow/contrib/cmake/tf_shared_lib.cmake @@ -23,6 +23,8 @@ if(WIN32) # we need. # add_library(tensorflow_static STATIC + $ + $ $ $ $ @@ -65,6 +67,8 @@ endif(WIN32) # tensorflow is a shared library containing all of the # TensorFlow runtime and the standard ops and kernels. add_library(tensorflow SHARED + $ + $ $ $ $ @@ -96,6 +100,27 @@ if(CMAKE_COMPILER_IS_GNUCC AND CMAKE_CXX_COMPILER_VERSION VERSION_GREATER 5.0) target_link_libraries(tensorflow PRIVATE gcc_s gcc) endif() +# Offer the user the choice of overriding the installation directories +set(INSTALL_LIB_DIR lib CACHE PATH "Installation directory for libraries") +set(INSTALL_BIN_DIR bin CACHE PATH "Installation directory for executables") +set(INSTALL_INCLUDE_DIR include CACHE PATH + "Installation directory for header files") +if(WIN32 AND NOT CYGWIN) + set(DEF_INSTALL_CMAKE_DIR cmake) +else() + set(DEF_INSTALL_CMAKE_DIR lib/cmake) +endif() +set(INSTALL_CMAKE_DIR ${DEF_INSTALL_CMAKE_DIR} CACHE PATH + "Installation directory for CMake files") + +# Make relative paths absolute (needed later on) +foreach(p LIB BIN INCLUDE CMAKE) + set(var INSTALL_${p}_DIR) + if(NOT IS_ABSOLUTE "${${var}}") + set(${var} "${CMAKE_INSTALL_PREFIX}/${${var}}") + endif() +endforeach() + if(WIN32) add_dependencies(tensorflow tensorflow_static) endif(WIN32) @@ -103,14 +128,57 @@ endif(WIN32) target_include_directories(tensorflow PUBLIC $) -install(TARGETS tensorflow EXPORT tensorflow_export - RUNTIME DESTINATION bin - LIBRARY DESTINATION lib - ARCHIVE DESTINATION lib) +# Add all targets to build-tree export set +export(TARGETS tensorflow + FILE ${PROJECT_BINARY_DIR}/TensorflowTargets.cmake) + +# Export the package for use from the build-tree +export(PACKAGE Tensorflow) + +# Create the TensorflowConfig.cmake and TensorflowConfigVersion files +file(RELATIVE_PATH REL_INCLUDE_DIR "${INSTALL_CMAKE_DIR}" + "${INSTALL_INCLUDE_DIR}") +# for the build tree +set(CONF_INCLUDE_DIRS "${tensorflow_source_dir}" + "${PROJECT_BINARY_DIR}" + "${CMAKE_CURRENT_BINARY_DIR}/protobuf/src/protobuf/src" + "${CMAKE_CURRENT_BINARY_DIR}/nsync/install/include" # Please if there is a better directory + "${CMAKE_CURRENT_BINARY_DIR}/eigen/src/eigen/Eigen/" + "${CMAKE_CURRENT_BINARY_DIR}/external/eigen_archive/" + "${tensorflow_source_dir}/third_party/eigen3/" + "${CMAKE_CURRENT_BINARY_DIR}/eigen/src/eigen/unsupported/Eigen/") +configure_file(TensorflowConfig.cmake.in + "${PROJECT_BINARY_DIR}/TensorflowConfig.cmake" @ONLY) +# for the install tree, yet to be complete +set(CONF_INCLUDE_DIRS "\${TENSORFLOW_CMAKE_DIR}/${REL_INCLUDE_DIR}") +configure_file(TensorflowConfig.cmake.in + "${PROJECT_BINARY_DIR}/${CMAKE_FILES_DIRECTORY}/TensorflowConfig.cmake" @ONLY) +# for both +configure_file(TensorflowConfigVersion.cmake.in + "${PROJECT_BINARY_DIR}/TensorflowConfigVersion.cmake" @ONLY) + +# install(TARGETS tensorflow EXPORT tensorflow_export +# RUNTIME DESTINATION ${INSTALL_BIN_DIR} +# LIBRARY DESTINATION ${INSTALL_LIB_DIR} +# ARCHIVE DESTINATION ${INSTALL_LIB_DIR}) + +# install(EXPORT tensorflow_export +# FILE TensorflowConfig.cmake +# DESTINATION ${INSTALL_CMAKE_DIR}) -install(EXPORT tensorflow_export - FILE TensorflowConfig.cmake - DESTINATION lib/cmake) +install(FILES + "${PROJECT_BINARY_DIR}/${CMAKE_FILES_DIRECTORY}/TensorflowConfig.cmake" + "${PROJECT_BINARY_DIR}/TensorflowConfigVersion.cmake" + DESTINATION "${INSTALL_CMAKE_DIR}" COMPONENT dev) + +# install the export set for use with the install-tree +install(EXPORT TensorflowTargets + DESTINATION ${INSTALL_CMAKE_DIR}) + +install(TARGETS tensorflow EXPORT TensorflowTargets + RUNTIME DESTINATION ${INSTALL_BIN_DIR} + LIBRARY DESTINATION ${INSTALL_LIB_DIR} + ARCHIVE DESTINATION ${INSTALL_LIB_DIR}) # install necessary headers # tensorflow headers @@ -145,6 +213,10 @@ install(DIRECTORY ${tensorflow_source_dir}/third_party/eigen3/ # unsupported Eigen directory install(DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/eigen/src/eigen/unsupported/Eigen/ DESTINATION include/unsupported/Eigen) +# absl directory +install(DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/abseil_cpp/src/abseil_cpp/absl/ + DESTINATION include/absl + FILES_MATCHING PATTERN "*.h") # mkl if (tensorflow_ENABLE_MKL_SUPPORT) install(DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/mkl/src/mkl/include/ diff --git a/tensorflow/contrib/compiler/BUILD b/tensorflow/contrib/compiler/BUILD index 1630f010ab6..e4566437c60 100644 --- a/tensorflow/contrib/compiler/BUILD +++ b/tensorflow/contrib/compiler/BUILD @@ -58,6 +58,7 @@ py_library( srcs_version = "PY2AND3", deps = [ "//tensorflow/compiler/jit:xla_ops_py", + "//tensorflow/compiler/jit/ops:xla_ops_grad", "//tensorflow/python:array_ops", "//tensorflow/python:control_flow_ops", "//tensorflow/python:framework_ops", diff --git a/tensorflow/contrib/compiler/xla.py b/tensorflow/contrib/compiler/xla.py index 335ac794648..f867cd15b67 100644 --- a/tensorflow/contrib/compiler/xla.py +++ b/tensorflow/contrib/compiler/xla.py @@ -23,6 +23,7 @@ import contextlib from six.moves import xrange # pylint: disable=redefined-builtin from tensorflow.compiler.jit.ops import xla_ops +from tensorflow.compiler.jit.ops import xla_ops_grad # pylint: disable=unused-import from tensorflow.core.framework import attr_value_pb2 from tensorflow.python.estimator import model_fn as model_fn_lib from tensorflow.python.framework import ops diff --git a/tensorflow/contrib/constrained_optimization/python/constrained_minimization_problem.py b/tensorflow/contrib/constrained_optimization/python/constrained_minimization_problem.py index 41258edd908..6926c0d03fe 100644 --- a/tensorflow/contrib/constrained_optimization/python/constrained_minimization_problem.py +++ b/tensorflow/contrib/constrained_optimization/python/constrained_minimization_problem.py @@ -74,8 +74,8 @@ class ConstrainedMinimizationProblem(object): if (constraints_shape.ndims is None or proxy_constraints_shape.ndims is None or - any([ii is None for ii in constraints_shape.as_list()]) or - any([ii is None for ii in proxy_constraints_shape.as_list()])): + any(ii is None for ii in constraints_shape.as_list()) or + any(ii is None for ii in proxy_constraints_shape.as_list())): raise ValueError( "constraints and proxy_constraints must have fully-known shapes") if constraints_shape != proxy_constraints_shape: diff --git a/tensorflow/contrib/crf/python/ops/crf.py b/tensorflow/contrib/crf/python/ops/crf.py index 656633f0bf2..40e159b8fcb 100644 --- a/tensorflow/contrib/crf/python/ops/crf.py +++ b/tensorflow/contrib/crf/python/ops/crf.py @@ -38,12 +38,12 @@ tf_unary_scores, tf_sequence_lengths, tf_transition_params, _ = session.run( [unary_scores, sequence_lengths, transition_params, train_op]) for tf_unary_scores_, tf_sequence_length_ in zip(tf_unary_scores, tf_sequence_lengths): -# Remove padding. -tf_unary_scores_ = tf_unary_scores_[:tf_sequence_length_] + # Remove padding. + tf_unary_scores_ = tf_unary_scores_[:tf_sequence_length_] -# Compute the highest score and its tag sequence. -tf_viterbi_sequence, tf_viterbi_score = tf.contrib.crf.viterbi_decode( - tf_unary_scores_, tf_transition_params) + # Compute the highest score and its tag sequence. + tf_viterbi_sequence, tf_viterbi_score = tf.contrib.crf.viterbi_decode( + tf_unary_scores_, tf_transition_params) """ from __future__ import absolute_import diff --git a/tensorflow/contrib/cudnn_rnn/BUILD b/tensorflow/contrib/cudnn_rnn/BUILD index 670b5494327..8d35622e393 100644 --- a/tensorflow/contrib/cudnn_rnn/BUILD +++ b/tensorflow/contrib/cudnn_rnn/BUILD @@ -42,10 +42,11 @@ tf_custom_op_py_library( cuda_py_test( name = "cudnn_rnn_ops_test", - size = "large", + size = "medium", srcs = ["python/kernel_tests/cudnn_rnn_ops_test.py"], additional_deps = [ ":cudnn_rnn_py", + "@absl_py//absl/testing:parameterized", "//tensorflow/core:protos_all_py", "//tensorflow/contrib/rnn:rnn_py", "//tensorflow/python/ops/losses:losses", @@ -61,7 +62,7 @@ cuda_py_test( "//tensorflow/python:training", "//tensorflow/python:variables", ], - shard_count = 6, + shard_count = 2, tags = [ "noasan", # http://b/62067814 "requires-gpu-sm35", diff --git a/tensorflow/contrib/cudnn_rnn/python/kernel_tests/cudnn_rnn_ops_test.py b/tensorflow/contrib/cudnn_rnn/python/kernel_tests/cudnn_rnn_ops_test.py index ae839108ebe..a268415f0e6 100644 --- a/tensorflow/contrib/cudnn_rnn/python/kernel_tests/cudnn_rnn_ops_test.py +++ b/tensorflow/contrib/cudnn_rnn/python/kernel_tests/cudnn_rnn_ops_test.py @@ -18,24 +18,30 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function +import collections import itertools import os import unittest +from absl.testing import parameterized import numpy as np from tensorflow.contrib.cudnn_rnn.python.ops import cudnn_rnn_ops from tensorflow.core.protobuf import saver_pb2 +from tensorflow.python.eager import context from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops from tensorflow.python.framework import random_seed from tensorflow.python.framework.test_util import TensorFlowTestCase from tensorflow.python.ops import array_ops -from tensorflow.python.ops import gradient_checker -from tensorflow.python.ops import math_ops +from tensorflow.python.ops import gradients_impl +from tensorflow.python.ops import init_ops from tensorflow.python.ops import random_ops +from tensorflow.python.ops import rnn +from tensorflow.python.ops import rnn_cell_impl from tensorflow.python.ops import state_ops +from tensorflow.python.ops import variable_scope from tensorflow.python.ops import variables from tensorflow.python.platform import googletest from tensorflow.python.platform import test @@ -56,714 +62,989 @@ CUDNN_RNN_TANH_PARAMS_PER_LAYER = cudnn_rnn_ops.CUDNN_RNN_TANH_PARAMS_PER_LAYER CUDNN_RNN_RELU_PARAMS_PER_LAYER = cudnn_rnn_ops.CUDNN_RNN_RELU_PARAMS_PER_LAYER -def _CreateModel(rnn_mode, - num_layers, - num_units, - input_size, - input_mode="linear_input", - direction=cudnn_rnn_ops.CUDNN_RNN_UNIDIRECTION, - dtype=dtypes.float32, - dropout=0.): - del input_mode - if rnn_mode == cudnn_rnn_ops.CUDNN_LSTM: - model_fn = cudnn_rnn_ops.CudnnLSTM - elif rnn_mode == cudnn_rnn_ops.CUDNN_GRU: - model_fn = cudnn_rnn_ops.CudnnGRU - elif rnn_mode == cudnn_rnn_ops.CUDNN_RNN_TANH: - model_fn = cudnn_rnn_ops.CudnnRNNTanh - elif rnn_mode == cudnn_rnn_ops.CUDNN_RNN_RELU: - model_fn = cudnn_rnn_ops.CudnnRNNRelu +def RunLSTM(sess, + num_units, + input_size, + batch_size, + time, + num_layers=1, + is_training=True, + dropout=0., + num_dirs=True, + dtype=dtypes.float32): + # TODO(jamesqin): add multi-layer tests. + # TODO(jamesqin): add multi-dir tests + assert num_layers == 1 + assert num_dirs == 1 + if is_training and not np.isclose(dropout, 0): + raise ValueError("dropout can not be 0. when test training.") + + # set graph level random seed and numpy random seed. + random_seed.set_random_seed(0) + np.random.seed(0) + + inputs = variable_scope.get_variable( + "inputs", + initializer=np.random.rand(time, batch_size, + input_size).astype(dtype.as_numpy_dtype), + dtype=dtype) + initial_h_op = variable_scope.get_variable( + "initial_h_op", + initializer=np.random.rand(batch_size, + num_units).astype(dtype.as_numpy_dtype), + dtype=dtype) + initial_c_op = variable_scope.get_variable( + "initial_c_op", + initializer=np.random.rand(batch_size, + num_units).astype(dtype.as_numpy_dtype), + dtype=dtype) + + initializer = init_ops.random_uniform_initializer( + -0.01, 0.01, dtype=dtype, seed=19980904) + + with variable_scope.variable_scope("test", initializer=initializer): + w = variable_scope.get_variable( + "rnn/lstm_cell/kernel", + shape=[input_size + num_units, num_units * 4], + dtype=dtype) + b = variable_scope.get_variable( + "rnn/lstm_cell/bias", shape=[num_units * 4], dtype=dtype) + + # canonical lstm. must set forget_bias to 0. to align with cudnn lstm. + cell = rnn_cell_impl.LSTMCell(num_units, forget_bias=0., reuse=True) + outputs_op, state_tuple_op = rnn.dynamic_rnn( + cell, + inputs, + initial_state=rnn_cell_impl.LSTMStateTuple( + h=initial_h_op, c=initial_c_op), + dtype=dtype, + time_major=True, + scope=None) + + # Convert to cudnn opaque param. + format_converter = cudnn_rnn_ops.CudnnParamsFormatConverterLSTM( + num_layers, num_units, input_size) + opaque_params = format_converter.tf_canonical_to_opaque([w, b]) + + cu_initial_h_op = array_ops.expand_dims(initial_h_op, axis=0) + cu_initial_c_op = array_ops.expand_dims(initial_c_op, axis=0) + cu_outputs_op, cu_h_op, cu_c_op = cudnn_rnn_ops._cudnn_rnn( + inputs, + cu_initial_h_op, + cu_initial_c_op, + opaque_params, + dropout=dropout, + is_training=is_training, + rnn_mode=cudnn_rnn_ops.CUDNN_LSTM) + # Remove the trivial 1st dimension. + cu_state_tuple_op = rnn_cell_impl.LSTMStateTuple( + c=array_ops.squeeze(cu_c_op, axis=0), + h=array_ops.squeeze(cu_h_op, axis=0)) + + if is_training: + (inp_grad_op, hgrad_op, + cgrad_op, wgrad_op, bgrad_op) = gradients_impl.gradients( + outputs_op, [inputs, initial_h_op, initial_c_op, w, b]) + + (cu_inp_grad_op, cu_hgrad_op, + cu_cgrad_op, opaque_grad_op) = gradients_impl.gradients( + cu_outputs_op, + [inputs, cu_initial_h_op, cu_initial_c_op, opaque_params]) + # Remove the trivial 1st dimension + cu_hgrad_op = array_ops.squeeze(cu_hgrad_op, axis=0) + # Remove the trivial 1st dimension + cu_cgrad_op = array_ops.squeeze(cu_cgrad_op, axis=0) + + cu_wgrad_op, cu_bgrad_op = format_converter.opaque_to_tf_canonical( + opaque_grad_op) + cu_wgrad_op = cu_wgrad_op[0] + cu_bgrad_op = cu_bgrad_op[0] + # cudnn lstm has 2 biases each gate. When converting to tf canonical format, + # the two biases are summed into one. Thus here bias gradient should be + # halved when comparing with tf lstm. + cu_bgrad_op *= 0.5 + + init_op = variables.global_variables_initializer() + sess.run(init_op) + + if is_training: + outputs, state_tuple, inp_grad, state_grad, wgrad, bgrad = sess.run([ + outputs_op, state_tuple_op, inp_grad_op, + (hgrad_op, cgrad_op), wgrad_op, bgrad_op + ]) + (cu_outputs, cu_state_tuple, cu_inp_grad, cu_state_grad, cu_wgrad, + cu_bgrad) = sess.run([ + cu_outputs_op, cu_state_tuple_op, cu_inp_grad_op, + (cu_hgrad_op, cu_cgrad_op), cu_wgrad_op, cu_bgrad_op + ]) + + logging.vlog(1, "outputs: %s" % outputs) + logging.vlog(1, "cu_outputs: %s" % cu_outputs) + logging.vlog(1, "state_tuple: %s" % str(state_tuple)) + logging.vlog(1, "cu_state_tuple: %s" % str(cu_state_tuple)) + logging.vlog(1, "inp_grad: %s" % inp_grad) + logging.vlog(1, "cu_inp_grad: %s" % cu_inp_grad) + logging.vlog(1, "state_grad: %s" % str(state_grad)) + logging.vlog(1, "cu_state_grad: %s" % str(cu_state_grad)) + logging.vlog(1, "wgrad: %s" % str(wgrad)) + logging.vlog(1, "bgrad: %s" % str(bgrad)) + logging.vlog(1, "cu_wgrad: %s" % str(cu_wgrad)) + logging.vlog(1, "cu_bgrad: %s" % str(cu_bgrad)) + return (outputs, cu_outputs, state_tuple, cu_state_tuple, inp_grad, + cu_inp_grad, state_grad, cu_state_grad, wgrad, bgrad, cu_wgrad, + cu_bgrad) else: - raise ValueError("Invalid rnn_mode: %s" % rnn_mode) - return model_fn( - num_layers, - num_units, - input_size, - direction=direction, - dtype=dtype, - dropout=dropout) + outputs, state_tuple = sess.run([outputs_op, state_tuple_op]) + cu_outputs, cu_state_tuple = sess.run([cu_outputs_op, cu_state_tuple_op]) + + logging.vlog(1, "outputs: %s" % outputs) + logging.vlog(1, "cu_outputs: %s" % cu_outputs) + logging.vlog(1, "state_tuple: %s" % str(state_tuple)) + logging.vlog(1, "cu_state_tuple: %s" % str(cu_state_tuple)) + return outputs, cu_outputs, state_tuple, cu_state_tuple -def _CreateParamsSavable(params, - model, - base_variable_scope=None, - name="params_canonical"): - """Create a RNNParamsSaveable for the weight and bias parameters. +# Basic set of RNN configs to test. They can be further extended in relevant +# test (e.g. adding num_dirs). +NAMED_RNN_TESTCASES = ({ + "testcase_name": "xsmall", + "num_units": 1, + "input_size": 1, + "batch_size": 1, + "time": 1, + "num_layers": 1, +}, { + "testcase_name": "small", + "num_units": 4, + "input_size": 4, + "batch_size": 4, + "time": 4, + "num_layers": 1, +}, { + "testcase_name": "medium", + "num_units": 128, + "input_size": 64, + "batch_size": 8, + "time": 16, + "num_layers": 1, +}, { + "testcase_name": "large", + "num_units": 128, + "input_size": 128, + "batch_size": 16, + "time": 32, + "num_layers": 1, +}) + + +def ExpandNamedTestCases(inputs, *remove_keys, **extra_configs): + """Expands testcase with new config dimensions. + + Example: + inputs = ( + {'testcase_name': 'test1', 'gender': 'male'} + {'testcase_name': 'test2', 'gender': 'female'} + ) + remove_keys: empty + extra_configs = { + 'age': [40, 80] + 'height': [5, 6] + } + + Returns: + ( + {'testcase_name': 'test1_age_40_height_5','gender': 'male', 'age': + 40,'height': 5} + {'testcase_name': 'test1_age_40_height_6', 'gender': 'male', 'age': 40, + 'height': 6} + {'testcase_name': 'test1_age_80_height_5', 'gender': 'male', 'age': 80, + 'height': 5} + {'testcase_name': 'test1_age_80_height_6', 'gender': 'male', 'age': 80, + 'height': 6} + + {'testcase_name': 'test2_age_40_height_5', 'gender': 'female', 'age': + 40, + 'height': 5} + {'testcase_name': 'test2_age_40_height_6', 'gender': 'female', 'age': + 40, + 'height': 6} + {'testcase_name': 'test2_age_80_height_5', 'gender': 'female', 'age': + 80, + 'height': 5} + {'testcase_name': 'test2_age_80_height_6', 'gender': 'female', 'age': + 80, + 'height': 6} + ) Args: - params: a Variable for weight and bias parameters. - model: a CudnnRNN model. - base_variable_scope: a string, prefix of names of saved variables. - name: a string, name of the RNNParamsSaveable object. + inputs: A list of dictionary, each being a testcase. + *remove_keys: A list of keys into testcase which are not needed in new + testcases. + **extra_configs: A dict of new test dimension and applicable values in that + dimension. + Returns: - a RNNParamsSaveable object. + A list of dictionary with expanded test cases. """ - if model._rnn_mode == CUDNN_LSTM: - fn = cudnn_rnn_ops.CudnnLSTMSaveable - elif model._rnn_mode == CUDNN_GRU: - fn = cudnn_rnn_ops.CudnnGRUSaveable - elif model._rnn_mode == CUDNN_RNN_TANH: - fn = cudnn_rnn_ops.CudnnRNNTanhSaveable - elif model._rnn_mode == CUDNN_RNN_RELU: - fn = cudnn_rnn_ops.CudnnRNNReluSaveable - params_saveable = fn( - params, - model.num_layers, - model.num_units, - model.input_size, - model.input_mode, - model.direction, - scope=base_variable_scope, - name=name) - ops.add_to_collection(ops.GraphKeys.SAVEABLE_OBJECTS, params_saveable) - return params_saveable + res = [] + ordered_extra_configs = collections.OrderedDict(extra_configs) + keys = ordered_extra_configs.keys() + # A list of list of configs. + # The outer loop is iterating keys, the innner is values of one key. + combined_kv = [[(k, v) for v in ordered_extra_configs[k]] for k in keys] + logging.info("combined_kv: %s", combined_kv) + + for inp in inputs: + # Each inp is a dict + for config in itertools.product(*combined_kv): + new_inp = dict(inp) + # config is a list in the form of [(k_i, v_j), (k_p, v_q), ...] + suffix = ["%s_%s" % (p[0], str(p[1])) for p in config] + suffix = "_".join(suffix) + new_inp["testcase_name"] += "_" + suffix + for k, v in config: + new_inp[k] = v + # Remove not used keys from the new test case. + if remove_keys: + if not isinstance(remove_keys, (list, tuple)): + remove_keys = [remove_keys] + for k in remove_keys: + new_inp.pop(k, None) + logging.info("new_inp: %s", new_inp) + res.append(new_inp) + # Dedup, necessary if `remove_keys` is set. + return [dict(t) for t in {tuple(d.items()) for d in res}] -def _MinLSTMParamSize(num_layers, - num_units, - input_size, - direction=cudnn_rnn_ops.CUDNN_RNN_UNIDIRECTION): - if direction == cudnn_rnn_ops.CUDNN_RNN_UNIDIRECTION: - first_layer_weights = 4 * num_units * (num_units + input_size) - higher_layer_weights = 8 * (num_layers - 1) * num_units * num_units - all_biases = 8 * num_layers * num_units - return first_layer_weights + higher_layer_weights + all_biases - elif direction == cudnn_rnn_ops.CUDNN_RNN_BIDIRECTION: - first_layer_weights = 4 * num_units * (num_units + input_size) - higher_layer_weights = (num_layers - 1) * ( - 4 * 2 * num_units * num_units + 4 * num_units**2) - all_biases = 8 * num_layers * num_units - return 2 * (first_layer_weights + higher_layer_weights + all_biases) +class CudnnLSTMTest(TensorFlowTestCase, parameterized.TestCase): + + def _test_training_helper(self, + num_units, + input_size, + batch_size, + time, + num_layers, + dtype, + rtol=2e-6, + atol=2e-6): + with self.session(use_gpu=True) as sess: + (outputs, cu_outputs, state_tuple, cu_state_tuple, inp_grad, cu_inp_grad, + state_grad, cu_state_grad, wgrad, bgrad, cu_wgrad, cu_bgrad) = RunLSTM( + sess, num_units, input_size, batch_size, time, num_layers) + + self.assertAllClose(outputs, cu_outputs, rtol=rtol, atol=atol) + for s, cu_s in zip(state_tuple, cu_state_tuple): + self.assertAllClose(s, cu_s, rtol=rtol, atol=atol) + for sg, cu_sg in zip(state_grad, cu_state_grad): + self.assertAllClose(sg, cu_sg, rtol=rtol, atol=atol) + self.assertAllClose(inp_grad, cu_inp_grad, rtol=rtol, atol=atol) + self.assertAllClose(bgrad, cu_bgrad, rtol=rtol, atol=atol) + self.assertAllClose(wgrad, cu_wgrad, rtol=rtol, atol=atol) + + @parameterized.named_parameters(*NAMED_RNN_TESTCASES) + @unittest.skipUnless(test.is_built_with_cuda(), + "Test only applicable when running on GPUs") + def test_training(self, num_units, input_size, batch_size, time, num_layers): + if not context.context().num_gpus(): + self.skipTest("No GPUs found") + self._test_training_helper(num_units, input_size, batch_size, time, + num_layers, dtypes.float32) + + @parameterized.named_parameters(*NAMED_RNN_TESTCASES) + @unittest.skipUnless(test.is_built_with_cuda(), + "Test only applicable when running on GPUs") + def test_training_fp16(self, num_units, input_size, batch_size, time, + num_layers): + if not context.context().num_gpus(): + self.skipTest("No GPUs found") + self._test_training_helper( + num_units, + input_size, + batch_size, + time, + num_layers, + dtypes.float16, + rtol=5e-3, + atol=5e-4) + + @parameterized.named_parameters(*NAMED_RNN_TESTCASES) + @unittest.skipUnless(test.is_built_with_cuda(), + "Test only applicable when running on GPUs") + def test_inference(self, num_units, input_size, batch_size, time, num_layers): + if not context.context().num_gpus(): + self.skipTest("No GPUs found") + with self.session(use_gpu=True) as sess: + (outputs, cu_outputs, state_tuple, cu_state_tuple) = RunLSTM( + sess, + num_units, + input_size, + batch_size, + time, + num_layers, + is_training=False) + + self.assertAllClose(outputs, cu_outputs) + # h + self.assertAllClose(state_tuple.h, cu_state_tuple.h) + # c + self.assertAllClose(state_tuple.c, cu_state_tuple.c) + + @parameterized.named_parameters(*NAMED_RNN_TESTCASES) + @unittest.skipUnless(test.is_built_with_cuda(), + "Test only applicable when running on GPUs") + def test_inference_fp16(self, num_units, input_size, batch_size, time, + num_layers): + if not context.context().num_gpus(): + self.skipTest("No GPUs found") + with self.session(use_gpu=True) as sess: + (outputs, cu_outputs, state_tuple, cu_state_tuple) = RunLSTM( + sess, + num_units, + input_size, + batch_size, + time, + num_layers, + is_training=False, + dtype=dtypes.float16) + + rtol, atol = 5e-3, 5e-4 + self.assertAllClose(outputs, cu_outputs, rtol=rtol, atol=atol) + # h + self.assertAllClose( + state_tuple.h, cu_state_tuple.h, rtol=rtol, atol=atol) + # c + self.assertAllClose( + state_tuple.c, cu_state_tuple.c, rtol=rtol, atol=atol) + + @parameterized.named_parameters(*NAMED_RNN_TESTCASES) + @unittest.skipUnless(test.is_built_with_cuda(), + "Test only applicable when running on GPUs") + def test_inference_with_dropout(self, num_units, input_size, batch_size, time, + num_layers): + """Validates that dropout does not affect Cudnn Rnn inference.""" + if not context.context().num_gpus(): + self.skipTest("No GPUs found") + # Hand-picked dropouts are used below (0. and 1.) + with ops.Graph().as_default() as g: + with self.session(use_gpu=True, graph=g) as sess: + # 1st time w/o dropout. + (_, cu_outputs, _, cu_state_tuple) = RunLSTM( + sess, + num_units, + input_size, + batch_size, + time, + num_layers, + is_training=False, + dropout=0.) + + with ops.Graph().as_default() as g: + with self.session(use_gpu=True, graph=g) as sess: + (_, cu_outputs2, _, cu_state_tuple2) = RunLSTM( + sess, + num_units, + input_size, + batch_size, + time, + num_layers, + is_training=False, + dropout=1.) + + self.assertAllClose(cu_outputs, cu_outputs2) + # h + self.assertAllClose(cu_state_tuple.h, cu_state_tuple2.h) + # c + self.assertAllClose(cu_state_tuple.c, cu_state_tuple2.c) + + +def RunGRU(sess, + num_units, + input_size, + batch_size, + time, + num_layers=1, + is_training=True, + dropout=0., + num_dirs=True, + dtype=dtypes.float32): + # TODO(jamesqin): add multi-layer tests. + # TODO(jamesqin): add multi-dir tests + assert num_layers == 1 + assert num_dirs == 1 + if is_training and not np.isclose(dropout, 0): + raise ValueError("dropout can not be 0. when test training.") + + # set graph level random seed and numpy random seed. + random_seed.set_random_seed(0) + np.random.seed(0) + + inputs = variable_scope.get_variable( + "inputs", + initializer=np.random.rand(time, batch_size, + input_size).astype(dtype.as_numpy_dtype), + dtype=dtype) + initial_h_op = variable_scope.get_variable( + "initial_h_op", + initializer=np.random.rand(batch_size, + num_units).astype(dtype.as_numpy_dtype), + dtype=dtype) + + initializer = init_ops.random_uniform_initializer( + -0.01, 0.01, dtype=dtype, seed=19980904) + with variable_scope.variable_scope("test", initializer=initializer): + gate_kernel = variable_scope.get_variable( + "rnn/cudnn_compatible_gru_cell/gates/kernel", + shape=[input_size + num_units, num_units * 2], + dtype=dtype) + gate_bias = variable_scope.get_variable( + "rnn/cudnn_compatible_gru_cell/gates/bias", + shape=[num_units * 2], + dtype=dtype) + candidate_inp_kernel = variable_scope.get_variable( + "rnn/cudnn_compatible_gru_cell/candidate/input_projection/kernel", + shape=[input_size, num_units], + dtype=dtype) + candidate_inp_bias = variable_scope.get_variable( + "rnn/cudnn_compatible_gru_cell/candidate/input_projection/bias", + shape=[num_units], + dtype=dtype) + candidate_hid_kernel = variable_scope.get_variable( + "rnn/cudnn_compatible_gru_cell/candidate/hidden_projection/kernel", + shape=[num_units, num_units], + dtype=dtype) + candidate_hid_bias = variable_scope.get_variable( + "rnn/cudnn_compatible_gru_cell/candidate/hidden_projection/bias", + shape=[num_units], + dtype=dtype) + + cell = cudnn_rnn_ops.CudnnCompatibleGRUCell(num_units, reuse=True) + outputs_op, h_op = rnn.dynamic_rnn( + cell, + inputs, + initial_state=initial_h_op, + dtype=dtype, + time_major=True, + scope=None) + + ws = [gate_kernel, candidate_inp_kernel, candidate_hid_kernel] + bs = [gate_bias, candidate_inp_bias, candidate_hid_bias] + # Convert to cudnn opaque param. + format_converter = cudnn_rnn_ops.CudnnParamsFormatConverterGRU( + num_layers, num_units, input_size) + opaque_params = format_converter.tf_canonical_to_opaque(ws + bs) + + cu_initial_h_op = array_ops.expand_dims(initial_h_op, axis=0) + cu_outputs_op, cu_h_op, _ = cudnn_rnn_ops._cudnn_rnn( + inputs, + cu_initial_h_op, + array_ops.zeros_like(cu_initial_h_op), # not used + opaque_params, + dropout=dropout, + is_training=is_training, + rnn_mode=cudnn_rnn_ops.CUDNN_GRU) + + if is_training: + (inp_grad_op, hgrad_op, gk_grad_op, cik_grad_op, chk_grad_op, gb_grad_op, + cib_grad_op, chb_grad_op) = gradients_impl.gradients( + outputs_op, [inputs, initial_h_op] + ws + bs) + + (cu_inp_grad_op, cu_hgrad_op, opaque_grad_op) = gradients_impl.gradients( + cu_outputs_op, [inputs, cu_initial_h_op, opaque_params]) + # Remove the trivial 1st dimension + cu_hgrad_op = array_ops.squeeze(cu_hgrad_op, axis=0) + + cu_wgrad_op, cu_bgrad_op = format_converter.opaque_to_tf_canonical( + opaque_grad_op) + (cu_gk_grad_op, cu_cik_grad_op, cu_chk_grad_op) = cu_wgrad_op + (cu_gb_grad_op, cu_cib_grad_op, cu_chb_grad_op) = cu_bgrad_op + # cudnn gru has 2 biases for reset and update gates. When converting to tf + # canonical format, the two biases are summed into one. Thus here relevant + # bias gradient should be halved before comparing with tf gru. + cu_gb_grad_op *= 0.5 + + init_op = variables.global_variables_initializer() + sess.run(init_op) + + if is_training: + outputs, h, inp_grad, hgrad, wgrad, bgrad = sess.run([ + outputs_op, h_op, inp_grad_op, hgrad_op, + (gk_grad_op, cik_grad_op, chk_grad_op), + (gb_grad_op, cib_grad_op, chb_grad_op) + ]) + (cu_outputs, cu_h, cu_inp_grad, cu_hgrad, cu_wgrad, cu_bgrad) = sess.run([ + cu_outputs_op, cu_h_op, cu_inp_grad_op, cu_hgrad_op, + (cu_gk_grad_op, cu_cik_grad_op, cu_chk_grad_op), + (cu_gb_grad_op, cu_cib_grad_op, cu_chb_grad_op) + ]) + # Remove the trivial 1st dimension + cu_h = np.squeeze(cu_h, axis=0) + + logging.vlog(1, "outputs: %s" % outputs) + logging.vlog(1, "cu_outputs: %s" % cu_outputs) + logging.vlog(1, "h: %s" % h) + logging.vlog(1, "cu_h: %s" % h) + logging.vlog(1, "inp_grad: %s" % inp_grad) + logging.vlog(1, "cu_inp_grad: %s" % cu_inp_grad) + logging.vlog(1, "hgrad: %s" % hgrad) + logging.vlog(1, "cu_hgrad: %s" % cu_hgrad) + logging.vlog(1, "wgrad: %s" % str(wgrad)) + logging.vlog(1, "bgrad: %s" % str(bgrad)) + logging.vlog(1, "cu_wgrad: %s" % str(cu_wgrad)) + logging.vlog(1, "cu_bgrad: %s" % str(cu_bgrad)) + return (outputs, cu_outputs, h, cu_h, inp_grad, cu_inp_grad, hgrad, + cu_hgrad, wgrad, bgrad, cu_wgrad, cu_bgrad) else: - raise ValueError("%s direction is not supported.") + outputs, h = sess.run([outputs_op, h_op]) + cu_outputs, cu_h = sess.run([cu_outputs_op, cu_h_op]) + # Remove the trivial 1st dimension. + cu_h = np.squeeze(cu_h, axis=0) + + logging.vlog(1, "outputs: %s" % outputs) + logging.vlog(1, "cu_outputs: %s" % cu_outputs) + logging.vlog(1, "h: %s" % h) + logging.vlog(1, "cu_h: %s" % h) + return outputs, cu_outputs, h, cu_h -class CudnnRNNTestSaveRestore(TensorFlowTestCase): +class CudnnGRUTest(TensorFlowTestCase, parameterized.TestCase): - def _CompareWeights(self, lhs, rhs): - self.assertEqual(len(lhs), len(rhs)) - for lw, rw in zip(lhs, rhs): - self.assertAllEqual(lw, rw) + def _test_training_helper(self, + num_units, + input_size, + batch_size, + time, + num_layers, + dtype, + rtol=2e-6, + atol=2e-6): + with self.session(use_gpu=True) as sess: + (outputs, cu_outputs, h, cu_h, inp_grad, cu_inp_grad, hgrad, + cu_hgrad, wgrad, bgrad, cu_wgrad, cu_bgrad) = RunGRU( + sess, num_units, input_size, batch_size, time, num_layers) - def _CompareBiases(self, lhs, rhs, rnn_mode, num_layers, direction): - self.assertEqual(len(lhs), len(rhs)) - if rnn_mode == CUDNN_LSTM: - num_params_per_layer = CUDNN_LSTM_PARAMS_PER_LAYER - elif rnn_mode == CUDNN_GRU: - num_params_per_layer = CUDNN_GRU_PARAMS_PER_LAYER - elif rnn_mode == CUDNN_RNN_TANH: - num_params_per_layer = CUDNN_RNN_TANH_PARAMS_PER_LAYER - else: - num_params_per_layer = CUDNN_RNN_RELU_PARAMS_PER_LAYER - num_dirs = 1 if direction == CUDNN_RNN_UNIDIRECTION else 2 - num_params_per_layer *= num_dirs - self.assertEqual(num_params_per_layer * num_layers, len(lhs)) + self.assertAllClose(outputs, cu_outputs, rtol=rtol, atol=atol) + self.assertAllClose(h, cu_h, rtol=rtol, atol=atol) + self.assertAllClose(hgrad, cu_hgrad, rtol=rtol, atol=atol) + self.assertAllClose(inp_grad, cu_inp_grad, rtol=rtol, atol=atol) + for bg, cu_bg in zip(bgrad, cu_bgrad): + self.assertAllClose(bg, cu_bg, rtol=rtol, atol=atol) + for wg, cu_wg in zip(wgrad, cu_wgrad): + self.assertAllClose(wg, cu_wg, rtol=rtol, atol=atol) - for i in range(num_layers): - layer_lhs = lhs[i * num_params_per_layer: (i+1) * num_params_per_layer] - layer_rhs = rhs[i * num_params_per_layer: (i+1) * num_params_per_layer] - if direction == CUDNN_RNN_UNIDIRECTION: - self._CompareSingleLayerBiases(layer_lhs, layer_rhs) - else: - size = len(layer_lhs) - fw_lhs, bw_lhs = layer_lhs[:size//2], layer_lhs[size//2:] - fw_rhs, bw_rhs = layer_rhs[:size//2], layer_rhs[size//2:] - self._CompareSingleLayerBiases(fw_lhs, fw_rhs) - self._CompareSingleLayerBiases(bw_lhs, bw_rhs) + @parameterized.named_parameters(*NAMED_RNN_TESTCASES) + @unittest.skipUnless(test.is_built_with_cuda(), + "Test only applicable when running on GPUs") + def test_training(self, num_units, input_size, batch_size, time, num_layers): + if not context.context().num_gpus(): + self.skipTest("No GPUs found") + self._test_training_helper(num_units, input_size, batch_size, time, + num_layers, dtypes.float32) - def _CompareSingleLayerBiases(self, lhs, rhs): - self.assertEqual(len(lhs), len(rhs)) + @parameterized.named_parameters(*NAMED_RNN_TESTCASES) + @unittest.skipUnless(test.is_built_with_cuda(), + "Test only applicable when running on GPUs") + def test_training_fp16(self, num_units, input_size, batch_size, time, + num_layers): + if not context.context().num_gpus(): + self.skipTest("No GPUs found") + self._test_training_helper( + num_units, + input_size, + batch_size, + time, + num_layers, + dtypes.float16, + rtol=5e-3, + atol=5e-4) - lf_lhs, rt_lhs = lhs[:len(lhs)//2], lhs[len(lhs)//2:] - lf_rhs, rt_rhs = rhs[:len(rhs)//2], rhs[len(rhs)//2:] - self.assertEqual(len(lf_lhs), len(rt_lhs)) - self.assertEqual(len(lf_rhs), len(rt_rhs)) + @parameterized.named_parameters(*NAMED_RNN_TESTCASES) + @unittest.skipUnless(test.is_built_with_cuda(), + "Test only applicable when running on GPUs") + def test_inference(self, num_units, input_size, batch_size, time, num_layers): + if not context.context().num_gpus(): + self.skipTest("No GPUs found") + with self.session(use_gpu=True) as sess: + (outputs, cu_outputs, h, cu_h) = RunGRU( + sess, + num_units, + input_size, + batch_size, + time, + num_layers, + is_training=False) + self.assertAllClose(outputs, cu_outputs) + self.assertAllClose(h, cu_h) - sum_lhs, sum_rhs = [], [] - for lf, rt in zip(lf_lhs, rt_lhs): - sum_lhs.append(lf + rt) - for lf, rt in zip(lf_rhs, rt_rhs): - sum_rhs.append(lf + rt) - self.assertEqual(len(sum_lhs), len(sum_rhs)) - for lf, rt in zip(sum_lhs, sum_rhs): - self.assertAllEqual(lf, rt) + @parameterized.named_parameters(*NAMED_RNN_TESTCASES) + @unittest.skipUnless(test.is_built_with_cuda(), + "Test only applicable when running on GPUs") + def test_inference_fp16(self, num_units, input_size, batch_size, time, + num_layers): + if not context.context().num_gpus(): + self.skipTest("No GPUs found") + with self.session(use_gpu=True) as sess: + (outputs, cu_outputs, h, cu_h) = RunGRU( + sess, + num_units, + input_size, + batch_size, + time, + num_layers, + is_training=False, + dtype=dtypes.float16) - def _testSaveRestoreVariable(self, rnn_mode, direction, dtype): - num_layers = 2 - num_units = 7 - input_size = 3 - with ops.Graph().as_default(): - model = _CreateModel( - rnn_mode, - num_layers=num_layers, - num_units=num_units, - input_size=input_size, - direction=direction, - dtype=dtype) - random_seed.set_random_seed(1234) - params_size_t = model.params_size() - params = variables.VariableV1( - random_ops.random_uniform([params_size_t], dtype=dtype), - dtype=dtype, - validate_shape=False) - saveable = _CreateParamsSavable(params, model) - weights, biases = saveable.format_converter._opaque_to_cu_canonical( - saveable._variables) - reset_params = state_ops.assign( - params, - array_ops.zeros([params_size_t], dtype=dtype), - validate_shape=False) - save_path = os.path.join(self.get_temp_dir(), - "save-restore-variable-test") - saver = saver_lib.Saver(write_version=saver_pb2.SaverDef.V2) - # Passing graph explicitly, otherwise an old sess would be reused. - with self.test_session( - use_gpu=True, graph=ops.get_default_graph()) as sess: - sess.run(variables.global_variables_initializer()) - val = saver.save(sess, save_path) - self.assertEqual(save_path, val) + rtol, atol = 5e-3, 5e-4 + self.assertAllClose(outputs, cu_outputs, rtol=rtol, atol=atol) + self.assertAllClose(h, cu_h, rtol=rtol, atol=atol) - weights_v, biases_v = sess.run([weights, biases]) + @parameterized.named_parameters(*NAMED_RNN_TESTCASES) + @unittest.skipUnless(test.is_built_with_cuda(), + "Test only applicable when running on GPUs") + def test_inference_with_dropout(self, num_units, input_size, batch_size, time, + num_layers): + """Validates that dropout does not affect Cudnn Rnn inference.""" + # Hand-picked dropouts are used below (0. and 1.) + if not context.context().num_gpus(): + self.skipTest("No GPUs found") + with ops.Graph().as_default() as g: + with self.session(use_gpu=True, graph=g) as sess: + # 1st time w/o dropout. + (_, cu_outputs, _, cu_h) = RunGRU( + sess, + num_units, + input_size, + batch_size, + time, + num_layers, + is_training=False, + dropout=0.) - sess.run(reset_params) - saver.restore(sess, save_path) - weights_v_restored, biases_v_restored = sess.run([weights, biases]) + with ops.Graph().as_default() as g: + with self.session(use_gpu=True, graph=g) as sess: + (_, cu_outputs2, _, cu_h2) = RunGRU( + sess, + num_units, + input_size, + batch_size, + time, + num_layers, + is_training=False, + dropout=1.) - self._CompareWeights(weights_v, weights_v_restored) - self._CompareBiases(biases_v, biases_v_restored, rnn_mode, num_layers, - direction) + self.assertAllClose(cu_outputs, cu_outputs2) + self.assertAllClose(cu_h[0], cu_h2[0]) - def _testSaveRestoreTwoVariables(self, rnn_mode, direction, dtype): - num_layers = 2 - num_units = 7 - input_size = 3 - with ops.Graph().as_default(): - model = _CreateModel( - rnn_mode, - num_layers=num_layers, - num_units=num_units, - input_size=input_size, - direction=direction, - dtype=dtype) - random_seed.set_random_seed(1234) - params_size_t = model.params_size() - names = ["rnn_1", "rnn_2"] - param_vars = [ - variables.VariableV1( - random_ops.random_uniform([params_size_t], dtype=dtype), - dtype=dtype, - validate_shape=False) for name in names - ] - saveables = [] - for name, params in zip(names, param_vars): - saveables.append(_CreateParamsSavable(params, model, name, name)) - weights1, biases1 = saveables[0].format_converter._opaque_to_cu_canonical( - saveables[0]._variables) - weights2, biases2 = saveables[1].format_converter._opaque_to_cu_canonical( - saveables[1]._variables) - reset_params = [ - state_ops.assign( - params, - array_ops.zeros([params_size_t], dtype=dtype), - validate_shape=False) for params in param_vars - ] - save_path = os.path.join(self.get_temp_dir(), - "save-restore-variable-test") - saver = saver_lib.Saver(write_version=saver_pb2.SaverDef.V2) - # Passing graph explicitly, otherwise an old sess would be reused. - with self.test_session(use_gpu=True, - graph=ops.get_default_graph()) as sess: - sess.run(variables.global_variables_initializer()) - val = saver.save(sess, save_path) - self.assertEqual(save_path, val) - weights1_v, biases1_v = sess.run([weights1, biases1]) - weights2_v, biases2_v = sess.run([weights2, biases2]) - sess.run(reset_params) - saver.restore(sess, save_path) - weights1_v_restored, biases1_v_restored = sess.run([weights1, biases1]) - weights2_v_restored, biases2_v_restored = sess.run([weights2, biases2]) +class CudnnParamsFormatConverterTest(TensorFlowTestCase, + parameterized.TestCase): + """Class for testing various format converters.""" - self._CompareWeights(weights1_v, weights1_v_restored) - self._CompareWeights(weights2_v, weights2_v_restored) - self._CompareBiases(biases1_v, biases1_v_restored, rnn_mode, num_layers, - direction) - self._CompareBiases(biases2_v, biases2_v_restored, rnn_mode, num_layers, - direction) + def _test_lstm_helper(self, num_units, input_size, num_layers, direction): + with self.session(use_gpu=True) as sess: + random_seed.set_random_seed(0) + np.random.seed(0) - def _testSaveRestoreOutput(self, rnn_mode, direction, dtype): - with ops.Graph().as_default(): - num_layers = 2 - num_units = 7 - input_size = 7 - seq_length = 10 - batch_size = 5 - dir_count = 1 if direction == cudnn_rnn_ops.CUDNN_RNN_UNIDIRECTION else 2 - model = _CreateModel( - rnn_mode, + num_dirs = 1 if direction == cudnn_rnn_ops.CUDNN_RNN_UNIDIRECTION else 2 + format_converter = cudnn_rnn_ops.CudnnParamsFormatConverterLSTM( + num_layers, num_units, input_size, direction=direction) + + ws, bs = [], [] + for _ in range(num_layers * num_dirs): + w = constant_op.constant( + np.random.rand(input_size + num_units, 4 * num_units), + dtype=dtypes.float32) + b = constant_op.constant( + np.random.rand(4 * num_units), dtype=dtypes.float32) + ws.append(w) + bs.append(b) + + opaque_params = format_converter.tf_canonical_to_opaque(ws + bs) + opaque_params_size = cudnn_rnn_ops.cudnn_rnn_opaque_params_size( + cudnn_rnn_ops.CUDNN_LSTM, num_layers, num_units, input_size, - direction=direction, - dtype=dtype) - params_size_t = model.params_size() - params = variables.VariableV1( - array_ops.ones([params_size_t], dtype=dtype), - validate_shape=False, - dtype=dtype) - _CreateParamsSavable(params, model) - save_path = os.path.join(self.get_temp_dir(), "save-restore-output-test") + direction=direction) + + ws_r, bs_r = format_converter.opaque_to_tf_canonical(opaque_params) + + # Test tf_canonical_to_opaque() followed by opaque_to_tf_canonical() + # returns the original input. + ws, ws_r, bs, bs_r = sess.run([ws, ws_r, bs, bs_r]) + for w, w_r in zip(ws, ws_r): + self.assertAllClose(w, w_r) + for b, b_r in zip(bs, bs_r): + self.assertAllClose(b, b_r) + + # Test opaque_params size lower bound + opaque_params_size_v = sess.run(opaque_params_size) + min_params_size = sum(x.size for x in ws) + np.sum(x.size for x in bs) + logging.info("min_parm_size: %d vs actual_opaque_param_size: %d", + min_params_size, opaque_params_size_v) + self.assertLessEqual(min_params_size, opaque_params_size_v) + + @parameterized.named_parameters((c["testcase_name"], c["num_units"], + c["input_size"], c["num_layers"]) + for c in NAMED_RNN_TESTCASES) + @unittest.skipUnless(test.is_built_with_cuda(), + "Test only applicable when running on GPUs") + def test_lstm(self, num_units, input_size, num_layers): + if not context.context().num_gpus(): + self.skipTest("No GPUs found") + self._test_lstm_helper(num_units, input_size, num_layers, + cudnn_rnn_ops.CUDNN_RNN_UNIDIRECTION) + + @parameterized.named_parameters((c["testcase_name"], c["num_units"], + c["input_size"], c["num_layers"]) + for c in NAMED_RNN_TESTCASES) + @unittest.skipUnless(test.is_built_with_cuda(), + "Test only applicable when running on GPUs") + def test_lstm_bidi(self, num_units, input_size, num_layers): + if not context.context().num_gpus(): + self.skipTest("No GPUs found") + self._test_lstm_helper(num_units, input_size, num_layers, + cudnn_rnn_ops.CUDNN_RNN_BIDIRECTION) + + def _test_gru_helper(self, num_units, input_size, num_layers, direction): + with self.session(use_gpu=True) as sess: + random_seed.set_random_seed(0) + np.random.seed(0) + + num_dirs = 1 if direction == cudnn_rnn_ops.CUDNN_RNN_UNIDIRECTION else 2 + format_converter = cudnn_rnn_ops.CudnnParamsFormatConverterGRU( + num_layers, num_units, input_size, direction=direction) + + ws, bs = [], [] + for _ in range(num_layers * num_dirs): + gate_kernel = constant_op.constant( + np.random.rand(input_size + num_units, num_units * 2), + dtype=dtypes.float32) + gate_bias = constant_op.constant( + np.random.rand(num_units * 2), dtype=dtypes.float32) + candidate_inp_kernel = constant_op.constant( + np.random.rand(input_size, num_units), dtype=dtypes.float32) + candidate_inp_bias = constant_op.constant( + np.random.rand(num_units), dtype=dtypes.float32) + candidate_hid_kernel = constant_op.constant( + np.random.rand(num_units, num_units), dtype=dtypes.float32) + candidate_hid_bias = constant_op.constant( + np.random.rand(num_units), dtype=dtypes.float32) + ws.extend([gate_kernel, candidate_inp_kernel, candidate_hid_kernel]) + bs.extend([gate_bias, candidate_inp_bias, candidate_hid_bias]) + + opaque_params = format_converter.tf_canonical_to_opaque(ws + bs) + opaque_params_size = cudnn_rnn_ops.cudnn_rnn_opaque_params_size( + cudnn_rnn_ops.CUDNN_GRU, + num_layers, + num_units, + input_size, + direction=direction) + + ws_r, bs_r = format_converter.opaque_to_tf_canonical(opaque_params) + + # Test tf_canonical_to_opaque() followed by opaque_to_tf_canonical() + # returns the original input. + ws, ws_r, bs, bs_r = sess.run([ws, ws_r, bs, bs_r]) + for w, w_r in zip(ws, ws_r): + self.assertAllClose(w, w_r) + for b, b_r in zip(bs, bs_r): + self.assertAllClose(b, b_r) + + # Test opaque_params size lower bound + opaque_params_size_v = sess.run(opaque_params_size) + min_params_size = sum(x.size for x in ws) + sum(x.size for x in bs) + logging.info("min_parm_size: %d vs actual_opaque_param_size: %d", + min_params_size, opaque_params_size_v) + self.assertLessEqual(min_params_size, opaque_params_size_v) + + @parameterized.named_parameters((c["testcase_name"], c["num_units"], + c["input_size"], c["num_layers"]) + for c in NAMED_RNN_TESTCASES) + @unittest.skipUnless(test.is_built_with_cuda(), + "Test only applicable when running on GPUs") + def test_gru(self, num_units, input_size, num_layers): + if not context.context().num_gpus(): + self.skipTest("No GPUs found") + self._test_gru_helper(num_units, input_size, num_layers, + cudnn_rnn_ops.CUDNN_RNN_UNIDIRECTION) + + @parameterized.named_parameters((c["testcase_name"], c["num_units"], + c["input_size"], c["num_layers"]) + for c in NAMED_RNN_TESTCASES) + @unittest.skipUnless(test.is_built_with_cuda(), + "Test only applicable when running on GPUs") + def test_gru_bidi(self, num_units, input_size, num_layers): + if not context.context().num_gpus(): + self.skipTest("No GPUs found") + self._test_gru_helper(num_units, input_size, num_layers, + cudnn_rnn_ops.CUDNN_RNN_BIDIRECTION) + + +class CudnnRnnSaveRestoreTest(TensorFlowTestCase, parameterized.TestCase): + """Class for testing various Cudnn Rnn SaveableObjects.""" + + def _create_opaque_param(self, + rnn_mode, + num_units, + input_size, + num_layers, + direction, + name=None): + param_size_t = cudnn_rnn_ops.cudnn_rnn_opaque_params_size( + rnn_mode, num_layers, num_units, input_size, direction=direction) + init_val = random_ops.random_uniform([param_size_t]) + return variable_scope.get_variable( + name or "opaque_param", initializer=init_val, validate_shape=False) + + def _create_saveable(self, opaque_param, rnn_mode, num_units, input_size, + num_layers, direction): + if rnn_mode == CUDNN_LSTM: + fn = cudnn_rnn_ops.CudnnLSTMSaveable + elif rnn_mode == CUDNN_GRU: + fn = cudnn_rnn_ops.CudnnGRUSaveable + elif rnn_mode == CUDNN_RNN_TANH: + fn = cudnn_rnn_ops.CudnnRNNTanhSaveable + elif rnn_mode == CUDNN_RNN_RELU: + fn = cudnn_rnn_ops.CudnnRNNReluSaveable + saveable = fn( + opaque_param, num_layers, num_units, input_size, direction=direction) + return saveable + + def _compare_weights(self, lhs, rhs): + self.assertLen(rhs, len(lhs)) + for lw, rw in zip(lhs, rhs): + self.assertAllEqual(lw, rw) + + def _compare_biases(self, lhs, rhs): + self.assertLen(rhs, len(lhs)) + for lf, rt in zip(lhs, rhs): + self.assertAllEqual(lf, rt) + + @parameterized.named_parameters( + ExpandNamedTestCases( + NAMED_RNN_TESTCASES, "time", "batch_size", **{ + "rnn_mode": [ + CUDNN_LSTM, CUDNN_GRU, CUDNN_RNN_RELU, CUDNN_RNN_TANH + ], + "direction": [CUDNN_RNN_UNIDIRECTION, CUDNN_RNN_BIDIRECTION] + })) + @unittest.skipUnless(test.is_built_with_cuda(), + "Test only applicable when running on GPUs") + def test_save_restore_variable(self, rnn_mode, num_units, input_size, + num_layers, direction): + # Verify the restored opaque param, once converted to tf_canonical format, + # is the same as the tf canonicals of the pre-restored param. + if not context.context().num_gpus(): + self.skipTest("No GPUs found") + with self.session(use_gpu=True) as sess: + opaque_param = self._create_opaque_param(rnn_mode, num_units, input_size, + num_layers, direction) + saveable = self._create_saveable(opaque_param, rnn_mode, num_units, + input_size, num_layers, direction) + ops.add_to_collection(ops.GraphKeys.SAVEABLE_OBJECTS, saveable) + weights_op, biases_op = saveable.format_converter.opaque_to_tf_canonical( + saveable._variables) + + save_path = os.path.join(self.get_temp_dir(), "save_restore_var_test") saver = saver_lib.Saver(write_version=saver_pb2.SaverDef.V2) - np.random.seed(1234) - has_input_c = (rnn_mode == cudnn_rnn_ops.CUDNN_LSTM) - input_data = constant_op.constant( - np.random.randn(seq_length, batch_size, input_size), dtype=dtype) - input_h = constant_op.constant( - np.random.randn(num_layers * dir_count, batch_size, num_units), - dtype=dtype) - if has_input_c: - input_c = constant_op.constant( - np.random.randn(num_layers * dir_count, batch_size, num_units), - dtype=dtype) - outputs = model( - input_data=input_data, - input_h=input_h, - input_c=input_c, - params=params, - is_training=False) - else: - outputs = model( - input_data=input_data, - input_h=input_h, - params=params, - is_training=False) - total_sum = sum(map(math_ops.reduce_sum, outputs)) - # Passing graph explicitly, otherwise an old sess would be reused. - with self.test_session( - use_gpu=True, graph=ops.get_default_graph()) as sess: - sess.run(variables.global_variables_initializer()) - total_sum_v = sess.run(total_sum) - val = saver.save(sess, save_path) - self.assertEqual(save_path, val) - # Passing graph explicitly, otherwise an old sess would be reused. - with self.test_session( - use_gpu=True, graph=ops.get_default_graph()) as sess: - reset_params = state_ops.assign( - params, - array_ops.zeros([params_size_t], dtype=dtype), - validate_shape=False) - sess.run(reset_params) + init_op = variables.global_variables_initializer() + reset_op = state_ops.assign(opaque_param, + array_ops.zeros_like(opaque_param)) + sess.run(init_op) + self.assertEqual(save_path, saver.save(sess, save_path)) + + # Get the tf canonical vals before reset-restore + weights, biases = sess.run([weights_op, biases_op]) + + # Reset the opaque param value + sess.run(reset_op) + # Assert reset happened. + weights_z, biases_z = sess.run([weights_op, biases_op]) + for w in weights_z: + self.assertAllClose(w, np.zeros_like(w)) + for b in biases_z: + self.assertAllClose(b, np.zeros_like(b)) + + # Restore opaque param value from checkpoint. + saver.restore(sess, save_path) + weights_r, biases_r = sess.run([weights_op, biases_op]) + self._compare_weights(weights, weights_r) + self._compare_biases(biases, biases_r) + + @parameterized.named_parameters( + ExpandNamedTestCases( + NAMED_RNN_TESTCASES, "time", "batch_size", **{ + "rnn_mode": [ + CUDNN_LSTM, CUDNN_GRU, CUDNN_RNN_RELU, CUDNN_RNN_TANH + ], + "direction": [CUDNN_RNN_UNIDIRECTION, CUDNN_RNN_BIDIRECTION] + })) + @unittest.skipUnless(test.is_built_with_cuda(), + "Test only applicable when running on GPUs") + def test_save_restore_multi_variables(self, rnn_mode, num_units, input_size, + num_layers, direction): + # Verify the restored opaque param, once converted to tf_canonical format, + # is the same as the tf canonicals of the pre-restored param. + if not context.context().num_gpus(): + self.skipTest("No GPUs found") + with self.session(use_gpu=True) as sess: + opaque_params = [] + saveables = [] + num_opaque_params = 2 + for i in range(num_opaque_params): + opaque_params.append( + self._create_opaque_param( + rnn_mode, + num_units, + input_size, + num_layers, + direction, + name="opaque_param_%d" % i)) + saveable = self._create_saveable(opaque_params[i], rnn_mode, num_units, + input_size, num_layers, direction) + ops.add_to_collection(ops.GraphKeys.SAVEABLE_OBJECTS, saveable) + saveables.append(saveable) + + weights_ops, biases_ops = [], [] + for i in range(num_opaque_params): + weights_op, biases_op = ( + saveables[i].format_converter.opaque_to_tf_canonical( + saveables[i]._variables)) + weights_ops.append(weights_op) + biases_ops.append(biases_op) + + save_path = os.path.join(self.get_temp_dir(), "save_restore_var_test") + saver = saver_lib.Saver(write_version=saver_pb2.SaverDef.V2) + + init_op = variables.global_variables_initializer() + reset_ops = [] + for i in range(num_opaque_params): + reset_ops.append( + state_ops.assign(opaque_params[i], + array_ops.zeros_like(opaque_params[i]))) + sess.run(init_op) + self.assertEqual(save_path, saver.save(sess, save_path)) + + # Get the tf canonical vals before reset-restore + for i in range(num_opaque_params): + weights, biases = sess.run([weights_ops[i], biases_ops[i]]) + + # Reset the opaque param value + sess.run(reset_ops[i]) + + # Assert reset happened. + weights_z, biases_z = sess.run([weights_ops[i], biases_ops[i]]) + for w in weights_z: + self.assertAllClose(w, np.zeros_like(w)) + for b in biases_z: + self.assertAllClose(b, np.zeros_like(b)) + + # Restore opaque param value from checkpoint. saver.restore(sess, save_path) - total_sum_v_restored = sess.run(total_sum) - self.assertAllClose(total_sum_v, total_sum_v_restored, atol=1e-5) - - @unittest.skipUnless(test.is_built_with_cuda(), - "Test only applicable when running on GPUs") - def testSaveRestore(self): - rnn_modes = [ - cudnn_rnn_ops.CUDNN_LSTM, cudnn_rnn_ops.CUDNN_GRU, - cudnn_rnn_ops.CUDNN_RNN_TANH, cudnn_rnn_ops.CUDNN_RNN_RELU - ] - directions = [ - cudnn_rnn_ops.CUDNN_RNN_UNIDIRECTION, - cudnn_rnn_ops.CUDNN_RNN_BIDIRECTION - ] - dtype_list = [dtypes.float32, dtypes.float64] - for rnn_mode, direction, dtype in itertools.product(rnn_modes, directions, - dtype_list): - self._testSaveRestoreVariable(rnn_mode, direction, dtype) - self._testSaveRestoreTwoVariables(rnn_mode, direction, dtype) - self._testSaveRestoreOutput(rnn_mode, direction, dtype) - - -class CudnnRNNTestParamsSize(TensorFlowTestCase): - - def _testOneLSTMParamsSize(self, num_layers, num_units, input_size, - direction): - logging.info("Testing one lstm param size with config: %s", locals()) - min_params_size = _MinLSTMParamSize(num_layers, num_units, input_size, - direction) - model = _CreateModel( - cudnn_rnn_ops.CUDNN_LSTM, - num_layers, - num_units, - input_size, - direction=direction) - params_size = model.params_size() - with self.test_session(use_gpu=True, graph=ops.get_default_graph()) as sess: - params_size_v = sess.run(params_size) - self.assertLessEqual(min_params_size, params_size_v) - - @unittest.skipUnless(test.is_built_with_cuda(), - "Test only applicable when running on GPUs") - def testLSTMParamsSize(self): - test_configs = [ - [4, 200, 200], - [4, 200, 300], - [4, 200, 100], - [1, 100, 200], - [2, 200, 100], - [3, 200, 400], - ] - directions = [ - cudnn_rnn_ops.CUDNN_RNN_UNIDIRECTION, - cudnn_rnn_ops.CUDNN_RNN_BIDIRECTION - ] - for (config, direction) in itertools.product(test_configs, directions): - num_layers, num_units, input_size = config - with ops.Graph().as_default(): - self._testOneLSTMParamsSize(num_layers, num_units, input_size, - direction) - - @unittest.skipUnless(test.is_built_with_cuda(), - "Test only applicable when running on GPUs") - def testLSTMParamsSizeShape(self): - with self.assertRaisesRegexp( - ValueError, "Shape must be rank 0 but is rank 1"): - model = _CreateModel( - cudnn_rnn_ops.CUDNN_LSTM, - constant_op.constant([4]), 200, 200, - direction=cudnn_rnn_ops.CUDNN_RNN_UNIDIRECTION) - _ = model.params_size() - with self.assertRaisesRegexp( - ValueError, "Shape must be rank 0 but is rank 1"): - model = _CreateModel( - cudnn_rnn_ops.CUDNN_LSTM, - 4, constant_op.constant([200]), 200, - direction=cudnn_rnn_ops.CUDNN_RNN_UNIDIRECTION) - _ = model.params_size() - with self.assertRaisesRegexp( - ValueError, "Shape must be rank 0 but is rank 1"): - model = _CreateModel( - cudnn_rnn_ops.CUDNN_LSTM, - 4, 200, constant_op.constant([200]), - direction=cudnn_rnn_ops.CUDNN_RNN_UNIDIRECTION) - _ = model.params_size() - - -class CudnnRNNTestInference(TensorFlowTestCase): - - def _testOneSimpleInference(self, rnn_mode, num_layers, num_units, input_size, - batch_size, seq_length, dir_count, dropout, - expected, tolerance): - random_seed.set_random_seed(5678) - model = _CreateModel( - rnn_mode, - num_layers, - num_units, - input_size, - input_mode="auto_select", - direction=(cudnn_rnn_ops.CUDNN_RNN_UNIDIRECTION if dir_count == 1 - else cudnn_rnn_ops.CUDNN_RNN_BIDIRECTION), - dropout=dropout) - has_input_c = (rnn_mode == cudnn_rnn_ops.CUDNN_LSTM) - params_size_t = model.params_size() - input_data = array_ops.ones([seq_length, batch_size, input_size]) - input_h = array_ops.ones([num_layers * dir_count, batch_size, num_units]) - params = variables.VariableV1( - array_ops.ones([params_size_t]), validate_shape=False) - if has_input_c: - input_c = array_ops.ones([num_layers * dir_count, batch_size, num_units]) - output, output_h, output_c = model( - input_data=input_data, - input_h=input_h, - input_c=input_c, - params=params, - is_training=False) - else: - output, output_h = model( - input_data=input_data, - input_h=input_h, - params=params, - is_training=False) - output_sum = math_ops.reduce_sum(output) - output_h_sum = math_ops.reduce_sum(output_h) - total_sum = output_sum + output_h_sum - if has_input_c: - output_c_sum = math_ops.reduce_sum(output_c) - total_sum += output_c_sum - with self.test_session(use_gpu=True, graph=ops.get_default_graph()) as sess: - sess.run(variables.global_variables_initializer()) - total_sum_v = sess.run([total_sum]) - - self.assertAllClose( - total_sum_v[0], expected, atol=tolerance, rtol=tolerance) - - @unittest.skipUnless(test.is_built_with_cuda(), - "Test only applicable when running on GPUs") - def testSimpleInference(self): - test_configs = [ - { - "rnn_mode": cudnn_rnn_ops.CUDNN_LSTM, - "expected": 231833.22, - "tolerance": 1e-2, - "shape": { - "num_layers": 4, - "num_units": 200, - "input_size": 200, - "batch_size": 20, - "seq_length": 10, - "dir_count": 1, - }, - }, - { - "rnn_mode": cudnn_rnn_ops.CUDNN_GRU, - "expected": 56000, - "tolerance": 1e-2, - "shape": { - "num_layers": 4, - "num_units": 200, - "input_size": 200, - "batch_size": 20, - "seq_length": 10, - "dir_count": 1, - }, - }, - { - "rnn_mode": cudnn_rnn_ops.CUDNN_RNN_TANH, - "expected": 56000, - "tolerance": 1e-2, - "shape": { - "num_layers": 4, - "num_units": 200, - "input_size": 200, - "batch_size": 20, - "seq_length": 10, - "dir_count": 1, - }, - }, - { - "rnn_mode": cudnn_rnn_ops.CUDNN_RNN_RELU, - "expected": 130688, - "tolerance": 1e-2, - "shape": { - "num_layers": 2, - "num_units": 8, - "input_size": 4, - "batch_size": 4, - "seq_length": 2, - "dir_count": 1, - }, - }, - ] - # Cudnn scales result for dropout during training, therefore dropout has no - # impact for inference results. - # (lstm, gru, rnn_tanh are saturated in the test. rnn_relu case is most - # demonstrative of the dropout-invariant nature of CudnnRnn.) - dropouts = [0., 0.5, 1.] - for (config, dropout) in itertools.product(test_configs, dropouts): - rnn_mode = config["rnn_mode"] - expected = config["expected"] - tolerance = config["tolerance"] - shape = config["shape"] - with ops.Graph().as_default(): - self._testOneSimpleInference( - rnn_mode, shape["num_layers"], shape["num_units"], - shape["input_size"], shape["batch_size"], shape["seq_length"], - shape["dir_count"], dropout, expected, tolerance) - - -class CudnnRNNTestTraining(TensorFlowTestCase): - - def _testOneSimpleTraining(self, rnn_mode, num_layers, num_units, input_size, - batch_size, seq_length, dir_count, dropout, dtype, - delta, tolerance): - # Gradient checking runs two forward ops with almost the same input. Need to - # make sure the drop patterns across the two runs are the same. - logging.info("Training test with config: %s", locals()) - old_env_state = os.environ.get("TF_CUDNN_RESET_RND_GEN_STATE", str(False)) - os.environ["TF_CUDNN_RESET_RND_GEN_STATE"] = str(True) - has_input_c = (rnn_mode == cudnn_rnn_ops.CUDNN_LSTM) - random_seed.set_random_seed(5678) - direction = (cudnn_rnn_ops.CUDNN_RNN_UNIDIRECTION if dir_count == 1 - else cudnn_rnn_ops.CUDNN_RNN_BIDIRECTION) - model = _CreateModel( - rnn_mode, - num_layers, - num_units, - input_size, - direction=direction, - dtype=dtype, - dropout=dropout) - params_size_t = model.params_size() - input_data = variables.VariableV1( - random_ops.random_uniform( - [seq_length, batch_size, input_size], dtype=dtype), - dtype=dtype) - input_h = variables.VariableV1( - random_ops.random_uniform( - [num_layers * dir_count, batch_size, num_units], dtype=dtype), - dtype=dtype) - params = variables.VariableV1( - random_ops.random_uniform([params_size_t], dtype=dtype), - validate_shape=False, - dtype=dtype) - if has_input_c: - input_c = variables.VariableV1( - random_ops.random_uniform( - [num_layers * dir_count, batch_size, num_units], dtype=dtype), - dtype=dtype) - - output, output_h, output_c = model( - input_data=input_data, - input_h=input_h, - input_c=input_c, - params=params) - else: - output, output_h = model( - input_data=input_data, input_h=input_h, params=params) - output_sum = math_ops.reduce_sum(output) - output_h_sum = math_ops.reduce_sum(output_h) - total_sum = output_sum + output_h_sum - if has_input_c: - output_c_sum = math_ops.reduce_sum(output_c) - total_sum += output_c_sum - - with self.test_session(use_gpu=True, graph=ops.get_default_graph()) as sess: - params_size_v = sess.run(params_size_t) - inputs_and_shapes = [ - (input_data, [seq_length, batch_size, input_size]), - (input_h, [num_layers * dir_count, batch_size, num_units]), - (params, [params_size_v]), - ] - if has_input_c: - inputs_and_shapes.append( - (input_c, [num_layers * dir_count, batch_size, num_units]),) - sess.run(variables.global_variables_initializer()) - all_inputs = [entry[0] for entry in inputs_and_shapes] - all_shapes = [entry[1] for entry in inputs_and_shapes] - - err = gradient_checker.compute_gradient_error( - all_inputs, all_shapes, total_sum, [1], delta=delta) - - self.assertLess(err, tolerance) - os.environ["TF_CUDNN_RESET_RND_GEN_STATE"] = old_env_state - - @unittest.skipUnless(test.is_built_with_cuda(), - "Test only applicable when running on GPUs") - def DISABLED_testSimpleTraining(self): - # TODO(jamesqin): fix b/117989214 - test_configs = [ - { - "rnn_mode": cudnn_rnn_ops.CUDNN_LSTM, - "dtype": dtypes.float64, - "delta": 1e-4, - "tolerance": 5e-6, - "shape": { - "num_layers": 2, - "num_units": 3, - "input_size": 4, - "batch_size": 3, - "seq_length": 4, - "dir_count": 1, - }, - }, - { - "rnn_mode": cudnn_rnn_ops.CUDNN_GRU, - "dtype": dtypes.float64, - "delta": 1e-4, - "tolerance": 5e-6, - "shape": { - "num_layers": 2, - "num_units": 3, - "input_size": 4, - "batch_size": 3, - "seq_length": 4, - "dir_count": 1, - }, - }, - { - "rnn_mode": cudnn_rnn_ops.CUDNN_RNN_TANH, - "dtype": dtypes.float64, - "delta": 1e-4, - "tolerance": 5e-6, - "shape": { - "num_layers": 2, - "num_units": 3, - "input_size": 4, - "batch_size": 3, - "seq_length": 4, - "dir_count": 1, - }, - }, - { - "rnn_mode": cudnn_rnn_ops.CUDNN_RNN_RELU, - "dtype": dtypes.float64, - "delta": 1e-4, - "tolerance": 5e-6, - "shape": { - "num_layers": 2, - "num_units": 3, - "input_size": 4, - "batch_size": 3, - "seq_length": 4, - "dir_count": 1, - }, - }, - { - "rnn_mode": cudnn_rnn_ops.CUDNN_LSTM, - "dtype": dtypes.float32, - "tolerance": 1.5e-2, - "shape": { - "num_layers": 2, - "num_units": 3, - "input_size": 4, - "batch_size": 3, - "seq_length": 4, - }, - }, - { - "rnn_mode": cudnn_rnn_ops.CUDNN_GRU, - "dtype": dtypes.float32, - "tolerance": 4e-3, - "shape": { - "num_layers": 2, - "num_units": 3, - "input_size": 4, - "batch_size": 3, - "seq_length": 4, - }, - }, - { - "rnn_mode": cudnn_rnn_ops.CUDNN_RNN_TANH, - "dtype": dtypes.float32, - "tolerance": 5e-3, - "shape": { - "num_layers": 2, - "num_units": 3, - "input_size": 4, - "batch_size": 3, - "seq_length": 4, - }, - }, - { - "rnn_mode": cudnn_rnn_ops.CUDNN_RNN_RELU, - "dtype": dtypes.float32, - "tolerance": 5e-1, - "shape": { - "num_layers": 2, - "num_units": 3, - "input_size": 4, - "batch_size": 3, - "seq_length": 4, - }, - }, - ] - dropouts = [0., 0.5, 1.] - dir_counts = [1] - for config, dropout, dir_count in itertools.product(test_configs, dropouts, - dir_counts): - rnn_mode = config["rnn_mode"] - dtype = config.get("dtype", dtypes.float32) - delta = config.get("delta", 1e-3) - tolerance = config["tolerance"] - shape = config["shape"] - with ops.Graph().as_default(): - self._testOneSimpleTraining(rnn_mode, shape["num_layers"], - shape["num_units"], shape["input_size"], - shape["batch_size"], shape["seq_length"], - dir_count, dropout, dtype, delta, tolerance) + weights_r, biases_r = sess.run([weights_ops[i], biases_ops[i]]) + self._compare_weights(weights, weights_r) + self._compare_biases(biases, biases_r) if __name__ == "__main__": diff --git a/tensorflow/contrib/cudnn_rnn/python/kernel_tests/cudnn_rnn_test.py b/tensorflow/contrib/cudnn_rnn/python/kernel_tests/cudnn_rnn_test.py index 1954f6717bb..7e1b4062ce4 100644 --- a/tensorflow/contrib/cudnn_rnn/python/kernel_tests/cudnn_rnn_test.py +++ b/tensorflow/contrib/cudnn_rnn/python/kernel_tests/cudnn_rnn_test.py @@ -536,7 +536,9 @@ class CudnnRNNTestSaveRestore(test_util.TensorFlowTestCase): save_path = os.path.join(self.get_temp_dir(), "save-restore-variable-test") saver = saver_lib.Saver() - weights, biases = model.rnn.saveable._OpaqueParamsToCanonical() + weights, biases = ( + model.rnn.saveable.format_converter._opaque_to_cu_canonical( + model.rnn.saveable._variables)) opaque_params = rnn.trainable_variables[0] # CudnnTestModel() creates CudnnOpaqueParamsSaveable that helps saver save # Cudnn vars in canonical format. @@ -583,8 +585,12 @@ class CudnnRNNTestSaveRestore(test_util.TensorFlowTestCase): dtype=dtype) opaque_params = (model1.rnn.trainable_variables[0], model2.rnn.trainable_variables[0]) - weights1, biases1 = model1.rnn.saveable._OpaqueParamsToCanonical() - weights2, biases2 = model2.rnn.saveable._OpaqueParamsToCanonical() + saveable1 = model1.rnn.saveable + weights1, biases1 = saveable1.format_converter._opaque_to_cu_canonical( + saveable1._variables) + saveable2 = model1.rnn.saveable + weights2, biases2 = saveable2.format_converter._opaque_to_cu_canonical( + saveable2._variables) reset_params = [ state_ops.assign(params, array_ops.zeros_like(params, dtype=dtype)) @@ -1039,8 +1045,8 @@ class CudnnRNNTestParamsSize(test_util.TensorFlowTestCase): # Min param size estimate = sum(weights.size) + sum(biases.size) min_params_size = ( - np.sum(list(map(np.prod, rnn.canonical_weight_shapes))) + - np.sum([sp[0] for sp in rnn.canonical_bias_shapes])) + sum(map(np.prod, rnn.canonical_weight_shapes)) + + sum(sp[0] for sp in rnn.canonical_bias_shapes)) opaque_params = rnn.trainable_variables[0] with self.test_session(use_gpu=True, graph=ops.get_default_graph()): diff --git a/tensorflow/contrib/cudnn_rnn/python/layers/cudnn_rnn.py b/tensorflow/contrib/cudnn_rnn/python/layers/cudnn_rnn.py index 8bbcc7cd039..8e25637ed91 100644 --- a/tensorflow/contrib/cudnn_rnn/python/layers/cudnn_rnn.py +++ b/tensorflow/contrib/cudnn_rnn/python/layers/cudnn_rnn.py @@ -21,6 +21,7 @@ from tensorflow.contrib.cudnn_rnn.python.ops import cudnn_rnn_ops from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops from tensorflow.python.framework import tensor_shape +from tensorflow.python.keras.engine import input_spec from tensorflow.python.layers import base as base_layer from tensorflow.python.ops import array_ops from tensorflow.python.ops import init_ops @@ -322,7 +323,7 @@ class _CudnnRNN(base_layer.Layer): raise ValueError("The last dimension of the inputs to `CudnnRNN` " "should be defined. Found `None`.") self._input_size = input_shape[-1].value - self.input_spec = base_layer.InputSpec(ndim=3, axes={-1: self._input_size}) + self.input_spec = input_spec.InputSpec(ndim=3, axes={-1: self._input_size}) self._set_scope(None) diff --git a/tensorflow/contrib/cudnn_rnn/python/ops/cudnn_rnn_ops.py b/tensorflow/contrib/cudnn_rnn/python/ops/cudnn_rnn_ops.py index d06d0c6bdaa..1ce29b42d52 100644 --- a/tensorflow/contrib/cudnn_rnn/python/ops/cudnn_rnn_ops.py +++ b/tensorflow/contrib/cudnn_rnn/python/ops/cudnn_rnn_ops.py @@ -738,7 +738,7 @@ class CudnnOpaqueParamsSaveable(saver.BaseSaverBuilder.SaveableObject): self._variables, opaque_params, validate_shape=False) def _checkpointable_save(self, save_buffer): - weights, biases = self.format_converter.opaque_params_to_tf_canonical( + weights, biases = self.format_converter.opaque_to_tf_canonical( self._variables) for name, tensor in zip(self._param_names, weights + biases): save_buffer[name] = array_ops.identity(tensor) diff --git a/tensorflow/contrib/data/python/kernel_tests/assert_element_shape_test.py b/tensorflow/contrib/data/python/kernel_tests/assert_element_shape_test.py index 0456463a192..6c5f8c6b009 100644 --- a/tensorflow/contrib/data/python/kernel_tests/assert_element_shape_test.py +++ b/tensorflow/contrib/data/python/kernel_tests/assert_element_shape_test.py @@ -46,7 +46,7 @@ class AssertElementShapeTest(test_base.DatasetTestBase): result = dataset.apply(batching.assert_element_shape(expected_shapes)) self.assertEqual(expected_shapes, result.output_shapes) - iterator = result.make_initializable_iterator() + iterator = dataset_ops.make_initializable_iterator(result) init_op = iterator.initializer get_next = iterator.get_next() with self.cached_session() as sess: @@ -88,7 +88,7 @@ class AssertElementShapeTest(test_base.DatasetTestBase): result = dataset.apply(batching.assert_element_shape(expected_shapes)) self.assertEqual(expected_shapes, result.output_shapes) - iterator = result.make_initializable_iterator() + iterator = dataset_ops.make_initializable_iterator(result) init_op = iterator.initializer get_next = iterator.get_next() with self.cached_session() as sess: @@ -115,9 +115,8 @@ class AssertElementShapeTest(test_base.DatasetTestBase): wrong_shapes = (tensor_shape.TensorShape(2), tensor_shape.TensorShape((3, 10))) - iterator = ( - dataset.apply(batching.assert_element_shape(wrong_shapes)) - .make_initializable_iterator()) + iterator = dataset_ops.make_initializable_iterator( + dataset.apply(batching.assert_element_shape(wrong_shapes))) init_op = iterator.initializer get_next = iterator.get_next() with self.cached_session() as sess: @@ -142,7 +141,7 @@ class AssertElementShapeTest(test_base.DatasetTestBase): tensor_shape.TensorShape((3, 4))) self.assertEqual(actual_shapes, result.output_shapes) - iterator = result.make_initializable_iterator() + iterator = dataset_ops.make_initializable_iterator(result) init_op = iterator.initializer get_next = iterator.get_next() with self.cached_session() as sess: @@ -184,7 +183,7 @@ class AssertElementShapeTest(test_base.DatasetTestBase): result = dataset.apply(batching.assert_element_shape(expected_shapes)) self.assertEqual(expected_shapes, result.output_shapes) - iterator = result.make_initializable_iterator() + iterator = dataset_ops.make_initializable_iterator(result) init_op = iterator.initializer get_next = iterator.get_next() with self.cached_session() as sess: @@ -211,9 +210,8 @@ class AssertElementShapeTest(test_base.DatasetTestBase): wrong_shapes = (tensor_shape.TensorShape(2), tensor_shape.TensorShape((None, 10))) - iterator = ( - dataset.apply(batching.assert_element_shape(wrong_shapes)) - .make_initializable_iterator()) + iterator = dataset_ops.make_initializable_iterator( + dataset.apply(batching.assert_element_shape(wrong_shapes))) init_op = iterator.initializer get_next = iterator.get_next() with self.cached_session() as sess: diff --git a/tensorflow/contrib/data/python/kernel_tests/lmdb_dataset_op_test.py b/tensorflow/contrib/data/python/kernel_tests/lmdb_dataset_op_test.py index d2a72272db1..b9840b1ff1a 100644 --- a/tensorflow/contrib/data/python/kernel_tests/lmdb_dataset_op_test.py +++ b/tensorflow/contrib/data/python/kernel_tests/lmdb_dataset_op_test.py @@ -23,6 +23,7 @@ import shutil from tensorflow.contrib.data.python.ops import readers from tensorflow.python.data.kernel_tests import test_base +from tensorflow.python.data.ops import dataset_ops from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes from tensorflow.python.framework import errors @@ -48,7 +49,7 @@ class LMDBDatasetTest(test_base.DatasetTestBase): num_repeats = 2 dataset = readers.LMDBDataset(filenames).repeat(num_repeats) - iterator = dataset.make_initializable_iterator() + iterator = dataset_ops.make_initializable_iterator(dataset) init_op = iterator.initializer get_next = iterator.get_next() diff --git a/tensorflow/contrib/data/python/kernel_tests/slide_dataset_op_test.py b/tensorflow/contrib/data/python/kernel_tests/slide_dataset_op_test.py index c5a78623225..2527706709f 100644 --- a/tensorflow/contrib/data/python/kernel_tests/slide_dataset_op_test.py +++ b/tensorflow/contrib/data/python/kernel_tests/slide_dataset_op_test.py @@ -63,13 +63,13 @@ class SlideDatasetTest(test_base.DatasetTestBase, parameterized.TestCase): # The pipeline is TensorSliceDataset -> MapDataset(square_3) -> # RepeatDataset(count) -> # _SlideDataset(window_size, window_shift, window_stride). - iterator = ( + iterator = dataset_ops.make_initializable_iterator( dataset_ops.Dataset.from_tensor_slices(components).map(_map_fn) .repeat(count).apply( sliding.sliding_window_batch( window_size=window_size_t, window_shift=window_shift_t, - window_stride=window_stride_t)).make_initializable_iterator()) + window_stride=window_stride_t))) init_op = iterator.initializer get_next = iterator.get_next() @@ -127,13 +127,13 @@ class SlideDatasetTest(test_base.DatasetTestBase, parameterized.TestCase): # The pipeline is TensorSliceDataset -> MapDataset(square_3) -> # RepeatDataset(count) -> _SlideDataset(window_size, stride, window_stride). - iterator = ( + iterator = dataset_ops.make_initializable_iterator( dataset_ops.Dataset.from_tensor_slices(components).map(_map_fn) .repeat(count).apply( sliding.sliding_window_batch( window_size=window_size_t, stride=stride_t, - window_stride=window_stride_t)).make_initializable_iterator()) + window_stride=window_stride_t))) init_op = iterator.initializer get_next = iterator.get_next() @@ -173,12 +173,12 @@ class SlideDatasetTest(test_base.DatasetTestBase, parameterized.TestCase): window_shift_t = array_ops.placeholder(dtypes.int64, shape=[]) window_stride_t = array_ops.placeholder(dtypes.int64, shape=[]) - iterator = ( + iterator = dataset_ops.make_initializable_iterator( dataset_ops.Dataset.range(10).map(lambda x: x).repeat(count_t).apply( sliding.sliding_window_batch( window_size=window_size_t, window_shift=window_shift_t, - window_stride=window_stride_t)).make_initializable_iterator()) + window_stride=window_stride_t))) init_op = iterator.initializer with self.cached_session() as sess: @@ -204,9 +204,9 @@ class SlideDatasetTest(test_base.DatasetTestBase, parameterized.TestCase): return sparse_tensor.SparseTensorValue( indices=[[0]], values=(i * [1]), dense_shape=[1]) - iterator = dataset_ops.Dataset.range(10).map(_sparse).apply( - sliding.sliding_window_batch( - window_size=5, window_shift=3)).make_initializable_iterator() + iterator = dataset_ops.make_initializable_iterator( + dataset_ops.Dataset.range(10).map(_sparse).apply( + sliding.sliding_window_batch(window_size=5, window_shift=3))) init_op = iterator.initializer get_next = iterator.get_next() @@ -233,9 +233,9 @@ class SlideDatasetTest(test_base.DatasetTestBase, parameterized.TestCase): values=array_ops.fill([math_ops.to_int32(i)], i), dense_shape=[i]) - iterator = dataset_ops.Dataset.range(10).map(_sparse).apply( - sliding.sliding_window_batch( - window_size=5, window_shift=3)).make_initializable_iterator() + iterator = dataset_ops.make_initializable_iterator( + dataset_ops.Dataset.range(10).map(_sparse).apply( + sliding.sliding_window_batch(window_size=5, window_shift=3))) init_op = iterator.initializer get_next = iterator.get_next() @@ -265,11 +265,10 @@ class SlideDatasetTest(test_base.DatasetTestBase, parameterized.TestCase): return sparse_tensor.SparseTensorValue( indices=[[0]], values=(i * [1]), dense_shape=[1]) - iterator = ( + iterator = dataset_ops.make_initializable_iterator( dataset_ops.Dataset.range(10).map(_sparse).apply( sliding.sliding_window_batch(window_size=4, window_shift=2)).apply( - sliding.sliding_window_batch(window_size=3, window_shift=1)) - .make_initializable_iterator()) + sliding.sliding_window_batch(window_size=3, window_shift=1))) init_op = iterator.initializer get_next = iterator.get_next() @@ -305,11 +304,10 @@ class SlideDatasetTest(test_base.DatasetTestBase, parameterized.TestCase): yield [4.0, 5.0, 6.0] yield [7.0, 8.0, 9.0, 10.0] - iterator = ( + iterator = dataset_ops.make_initializable_iterator( dataset_ops.Dataset.from_generator( generator, dtypes.float32, output_shapes=[None]).apply( - sliding.sliding_window_batch(window_size=3, window_shift=1)) - .make_initializable_iterator()) + sliding.sliding_window_batch(window_size=3, window_shift=1))) next_element = iterator.get_next() with self.cached_session() as sess: diff --git a/tensorflow/contrib/data/python/ops/BUILD b/tensorflow/contrib/data/python/ops/BUILD index 34dc2379d0c..0fb406f1167 100644 --- a/tensorflow/contrib/data/python/ops/BUILD +++ b/tensorflow/contrib/data/python/ops/BUILD @@ -188,8 +188,7 @@ py_library( "//tensorflow/python:framework_ops", "//tensorflow/python:function", "//tensorflow/python/data/ops:dataset_ops", - "//tensorflow/python/data/util:nest", - "//tensorflow/python/data/util:sparse", + "//tensorflow/python/data/util:structure", ], ) diff --git a/tensorflow/contrib/data/python/ops/readers.py b/tensorflow/contrib/data/python/ops/readers.py index 4601376dff4..aa42782807a 100644 --- a/tensorflow/contrib/data/python/ops/readers.py +++ b/tensorflow/contrib/data/python/ops/readers.py @@ -355,7 +355,7 @@ def read_batch_features(file_pattern, shuffle=randomize_input, num_epochs=num_epochs, shuffle_buffer_size=capacity) - iterator = dataset.make_one_shot_iterator() + iterator = dataset_ops.make_one_shot_iterator(dataset) outputs = iterator.get_next() return outputs @@ -379,15 +379,13 @@ class LMDBDataset(dataset_ops.DatasetSource): (key value) pairs sequentially. For example: ```python + tf.enable_eager_execution() + dataset = tf.contrib.lmdb.LMDBDataset("/foo/bar.mdb") - iterator = dataset.make_one_shot_iterator() - next_element = iterator.get_next() + # Prints the (key, value) pairs inside a lmdb file. - while True: - try: - print(sess.run(next_element)) - except tf.errors.OutOfRangeError: - break + for key, value in dataset: + print(key, value) ``` Args: filenames: A `tf.string` tensor containing one or more filenames. diff --git a/tensorflow/contrib/data/python/ops/sliding.py b/tensorflow/contrib/data/python/ops/sliding.py index bcc383587c5..9ebdca317f2 100644 --- a/tensorflow/contrib/data/python/ops/sliding.py +++ b/tensorflow/contrib/data/python/ops/sliding.py @@ -18,11 +18,10 @@ from __future__ import division from __future__ import print_function from tensorflow.python.data.ops import dataset_ops -from tensorflow.python.data.util import nest +from tensorflow.python.data.util import structure from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops -from tensorflow.python.framework import tensor_shape -from tensorflow.python.ops import gen_dataset_ops +from tensorflow.python.ops import gen_experimental_dataset_ops as ged_ops from tensorflow.python.util import deprecation @@ -40,29 +39,31 @@ class _SlideDataset(dataset_ops.UnaryDataset): self._window_shift = ops.convert_to_tensor( window_shift, dtype=dtypes.int64, name="window_shift") + # pylint: disable=protected-access + input_structure = structure.Structure._from_legacy_structure( + input_dataset.output_types, input_dataset.output_shapes, + input_dataset.output_classes) + self._output_structure = input_structure._batch(None) + def _as_variant_tensor(self): - return gen_dataset_ops.slide_dataset( + return ged_ops.experimental_sliding_window_dataset( self._input_dataset._as_variant_tensor(), # pylint: disable=protected-access window_size=self._window_size, window_shift=self._window_shift, window_stride=self._window_stride, - **dataset_ops.flat_structure(self)) + **dataset_ops.flat_structure(structure=self._output_structure)) @property def output_classes(self): - return self._input_dataset.output_classes + return self._output_structure._to_legacy_output_classes() # pylint: disable=protected-access @property def output_shapes(self): - input_shapes = self._input_dataset.output_shapes - return nest.pack_sequence_as(input_shapes, [ - tensor_shape.vector(None).concatenate(s) - for s in nest.flatten(self._input_dataset.output_shapes) - ]) + return self._output_structure._to_legacy_output_shapes() # pylint: disable=protected-access @property def output_types(self): - return self._input_dataset.output_types + return self._output_structure._to_legacy_output_types() # pylint: disable=protected-access @deprecation.deprecated_args( diff --git a/tensorflow/contrib/distribute/BUILD b/tensorflow/contrib/distribute/BUILD index a87a5624c88..3ecd755d86f 100644 --- a/tensorflow/contrib/distribute/BUILD +++ b/tensorflow/contrib/distribute/BUILD @@ -26,7 +26,6 @@ py_library( visibility = ["//tensorflow:internal"], deps = [ "//tensorflow/contrib/distribute/python:collective_all_reduce_strategy", - "//tensorflow/contrib/distribute/python:cross_tower_ops", "//tensorflow/contrib/distribute/python:mirrored_strategy", "//tensorflow/contrib/distribute/python:monitor", "//tensorflow/contrib/distribute/python:one_device_strategy", @@ -35,6 +34,7 @@ py_library( "//tensorflow/contrib/distribute/python:tpu_strategy", "//tensorflow/python:training", "//tensorflow/python:util", + "//tensorflow/python/distribute:cross_device_ops", "//tensorflow/python/distribute:distribute_config", "//tensorflow/python/distribute:distribute_coordinator", ], diff --git a/tensorflow/contrib/distribute/README.md b/tensorflow/contrib/distribute/README.md index a938f8629d8..81574a2047e 100644 --- a/tensorflow/contrib/distribute/README.md +++ b/tensorflow/contrib/distribute/README.md @@ -134,7 +134,7 @@ def model_fn(features, labels, mode): return tf.estimator.EstimatorSpec(mode, loss=loss) if mode == tf.estimator.ModeKeys.TRAIN: - train_op = tf.train.GradientDescentOptimizer(0.2).minimize(loss_fn()) + train_op = tf.train.GradientDescentOptimizer(0.2).minimize(loss) return tf.estimator.EstimatorSpec(mode, loss=loss, train_op=train_op) ``` @@ -251,10 +251,10 @@ start multi-worker training using `tf.estimator.train_and_evaluate`: ```python def model_main(): - estimator = ... distribution = tf.contrib.distribute.CollectiveAllReduceStrategy( num_gpus_per_worker=2) config = tf.estimator.RunConfig(train_distribute=distribution) + estimator = tf.estimator.Estimator(model_fn=model_fn, config=config) train_spec = tf.estimator.TrainSpec(input_fn=input_fn) eval_spec = tf.estimator.EvalSpec(input_fn=eval_input_fn) tf.estimator.train_and_evaluate(estimator, train_spec, eval_spec) @@ -327,13 +327,13 @@ start training. On your laptop, you can run ```python -estimator = ... distribution = tf.contrib.distribute.CollectiveAllReduceStrategy( num_gpus_per_worker=2) config = tf.estimator.RunConfig( experimental_distribute=tf.contrib.distribute.DistributeConfig( train_distribute=distribution, remote_cluster={"worker": ["host1:port", "host2:port", "host3:port"]})) +estimator = tf.estimator.Estimator(model_fn=model_fn, config=config) train_spec = tf.estimator.TrainSpec(input_fn=input_fn) eval_spec = tf.estimator.EvalSpec(input_fn=eval_input_fn) tf.estimator.train_and_evaluate(estimator, train_spec, eval_spec) diff --git a/tensorflow/contrib/distribute/__init__.py b/tensorflow/contrib/distribute/__init__.py index ab2f221dc64..8ec73654e30 100644 --- a/tensorflow/contrib/distribute/__init__.py +++ b/tensorflow/contrib/distribute/__init__.py @@ -25,13 +25,13 @@ from __future__ import print_function # pylint: disable=unused-import,wildcard-import from tensorflow.contrib.distribute.python.collective_all_reduce_strategy import CollectiveAllReduceStrategy -from tensorflow.contrib.distribute.python.cross_tower_ops import * from tensorflow.contrib.distribute.python.mirrored_strategy import MirroredStrategy from tensorflow.contrib.distribute.python.monitor import Monitor from tensorflow.contrib.distribute.python.one_device_strategy import OneDeviceStrategy from tensorflow.contrib.distribute.python.parameter_server_strategy import ParameterServerStrategy from tensorflow.contrib.distribute.python.step_fn import * from tensorflow.contrib.distribute.python.tpu_strategy import TPUStrategy +from tensorflow.python.distribute.cross_device_ops import * from tensorflow.python.distribute.distribute_config import DistributeConfig from tensorflow.python.distribute.distribute_coordinator import run_standard_tensorflow_server from tensorflow.python.training.distribute import * @@ -46,6 +46,7 @@ _allowed_symbols = [ 'CrossDeviceOps', 'DistributeConfig', 'DistributionStrategy', + 'DistributionStrategyExtended', 'MirroredStrategy', 'Monitor', 'MultiWorkerAllReduce', @@ -62,6 +63,7 @@ _allowed_symbols = [ 'get_loss_reduction', 'get_replica_context', 'has_distribution_strategy', + 'in_cross_replica_context', 'require_replica_context', 'run_standard_tensorflow_server', 'UpdateContext', diff --git a/tensorflow/contrib/distribute/python/BUILD b/tensorflow/contrib/distribute/python/BUILD index 4094e52169a..4c9c35da5a3 100644 --- a/tensorflow/contrib/distribute/python/BUILD +++ b/tensorflow/contrib/distribute/python/BUILD @@ -16,45 +16,26 @@ load("//tensorflow:tensorflow.bzl", "cuda_py_test") # TODO(priyag): Figure out testonly issues that are preventing us from # including our tests in pip for now. -py_library( - name = "values", - srcs = ["values.py"], - visibility = ["//tensorflow:internal"], - deps = [ - ":input_ops", - "//tensorflow/python:array_ops", - "//tensorflow/python:control_flow_ops", - "//tensorflow/python:device_util", - "//tensorflow/python:distribute", - "//tensorflow/python:framework_ops", - "//tensorflow/python:resource_variable_ops", - "//tensorflow/python:training", - "//tensorflow/python:util", - "//tensorflow/python/data/ops:multi_device_iterator_ops", - "//tensorflow/python/eager:context", - "//tensorflow/python/training/checkpointable:base", - "@six_archive//:six", - ], -) - cuda_py_test( name = "values_test", srcs = ["values_test.py"], additional_deps = [ + ":combinations", ":mirrored_strategy", ":multi_worker_test_base", - ":values", + "@absl_py//absl/testing:parameterized", "//tensorflow/core:protos_all_py", - "//tensorflow/python/data/ops:dataset_ops", - "//tensorflow/python:errors", "//tensorflow/python:array_ops", "//tensorflow/python:constant_op", + "//tensorflow/python:errors", "//tensorflow/python:framework_ops", "//tensorflow/python:framework_test_lib", "//tensorflow/python:training", "//tensorflow/python:variable_scope", + "//tensorflow/python/data/ops:dataset_ops", + "//tensorflow/python/distribute:device_util", + "//tensorflow/python/distribute:values", "//tensorflow/python/eager:context", - "//tensorflow/python:device_util", "//tensorflow/python/eager:test", "//tensorflow/python/estimator:estimator_py", ], @@ -68,25 +49,9 @@ py_library( srcs = ["mirrored_strategy.py"], visibility = ["//tensorflow:internal"], deps = [ - ":cross_tower_ops", - ":shared_variable_creator", - ":values", - "//tensorflow/core:protos_all_py", - "//tensorflow/python:array_ops", - "//tensorflow/python:constant_op", - "//tensorflow/python:control_flow_ops", - "//tensorflow/python:device", - "//tensorflow/python:device_util", - "//tensorflow/python:distribute", - "//tensorflow/python:framework_ops", - "//tensorflow/python:pywrap_tensorflow", - "//tensorflow/python:training", - "//tensorflow/python:util", - "//tensorflow/python:variable_scope", - "//tensorflow/python:variables", - "//tensorflow/python/distribute:multi_worker_util", - "//tensorflow/python/eager:context", - "//tensorflow/python/eager:tape", + "//tensorflow/python/distribute:distribute_lib", + "//tensorflow/python/distribute:mirrored_strategy", + "//tensorflow/python/distribute:values", ], ) @@ -95,16 +60,17 @@ py_library( srcs = ["parameter_server_strategy.py"], visibility = ["//tensorflow:internal"], deps = [ - ":cross_tower_ops", ":mirrored_strategy", - ":values", "//tensorflow/core:protos_all_py", "//tensorflow/python:array_ops", "//tensorflow/python:framework_ops", "//tensorflow/python:resource_variable_ops", "//tensorflow/python:training", "//tensorflow/python:util", + "//tensorflow/python/distribute:cross_device_ops", "//tensorflow/python/distribute:multi_worker_util", + "//tensorflow/python/distribute:reduce_util", + "//tensorflow/python/distribute:values", "//tensorflow/python/eager:context", ], ) @@ -116,7 +82,7 @@ cuda_py_test( ":combinations", ":multi_worker_test_base", ":parameter_server_strategy", - ":values", + ":strategy_test_lib", "@absl_py//absl/testing:parameterized", "//tensorflow/core:protos_all_py", "//tensorflow/python:array_ops", @@ -127,10 +93,12 @@ cuda_py_test( "//tensorflow/python:gradients", "//tensorflow/python:layers", "//tensorflow/python:session", + "//tensorflow/python:tensor_util", "//tensorflow/python:training", "//tensorflow/python:variable_scope", "//tensorflow/python:variables", "//tensorflow/python/distribute:multi_worker_util", + "//tensorflow/python/distribute:values", "//tensorflow/python/eager:context", "//tensorflow/python/estimator:estimator_py", ], @@ -145,12 +113,13 @@ py_library( srcs = ["one_device_strategy.py"], visibility = ["//tensorflow:internal"], deps = [ - ":values", - "//tensorflow/contrib/eager/python:datasets", "//tensorflow/python:array_ops", - "//tensorflow/python:distribute", + "//tensorflow/python:dtypes", "//tensorflow/python:framework_ops", "//tensorflow/python:math_ops", + "//tensorflow/python/distribute:distribute_lib", + "//tensorflow/python/distribute:reduce_util", + "//tensorflow/python/distribute:values", "//tensorflow/python/eager:context", "@six_archive//:six", ], @@ -161,16 +130,16 @@ py_library( srcs = ["collective_all_reduce_strategy.py"], visibility = ["//tensorflow:internal"], deps = [ - ":cross_tower_ops", - ":cross_tower_utils", ":mirrored_strategy", - ":values", "//tensorflow/core:protos_all_py", "//tensorflow/python:array_ops", "//tensorflow/python:collective_ops", "//tensorflow/python:framework_ops", "//tensorflow/python:training", + "//tensorflow/python/distribute:cross_device_ops", + "//tensorflow/python/distribute:cross_device_utils", "//tensorflow/python/distribute:multi_worker_util", + "//tensorflow/python/distribute:values", "//tensorflow/python/eager:context", ], ) @@ -187,11 +156,11 @@ py_library( "//tensorflow/core:protos_all_py", "//tensorflow/python:array_ops", "//tensorflow/python:constant_op", - "//tensorflow/python:distribute", "//tensorflow/python:framework_ops", "//tensorflow/python:layers", "//tensorflow/python:training", "//tensorflow/python:variables", + "//tensorflow/python/distribute:distribute_lib", "//tensorflow/python/eager:backprop", "//tensorflow/python/eager:context", "//tensorflow/python/eager:test", @@ -212,10 +181,10 @@ py_library( ":tpu_strategy", "//tensorflow/contrib/cluster_resolver:cluster_resolver_pip", "//tensorflow/contrib/optimizer_v2:training", - "//tensorflow/python:distribute", "//tensorflow/python:framework_ops", "//tensorflow/python:training", "//tensorflow/python:util", + "//tensorflow/python/distribute:distribute_lib", "//tensorflow/python/eager:context", "@absl_py//absl/testing:parameterized", ], @@ -233,28 +202,6 @@ py_test( ], ) -py_test( - name = "mirrored_strategy_test", - srcs = ["mirrored_strategy_test.py"], - srcs_version = "PY2AND3", - tags = [ - "no_pip", - ], - deps = [ - ":mirrored_strategy", - ":multi_worker_test_base", - ":strategy_test_lib", - "//tensorflow/python:constant_op", - "//tensorflow/python:distribute", - "//tensorflow/python:framework_ops", - "//tensorflow/python:framework_test_lib", - "//tensorflow/python:training", - "//tensorflow/python:variable_scope", - "//tensorflow/python/eager:context", - "//tensorflow/python/eager:test", - ], -) - py_test( name = "one_device_strategy_test", srcs = ["one_device_strategy_test.py"], @@ -270,35 +217,32 @@ py_test( ], ) +# TODO(priyag): Rename this test to mirrored_strategy_test cuda_py_test( name = "mirrored_strategy_multigpu_test", srcs = ["mirrored_strategy_multigpu_test.py"], additional_deps = [ + ":combinations", ":mirrored_strategy", ":multi_worker_test_base", - ":values", ":strategy_test_lib", - "//tensorflow/python:distribute", "//tensorflow/core:protos_all_py", "//tensorflow/python:array_ops", "//tensorflow/python:constant_op", + "//tensorflow/python:framework_test_lib", "//tensorflow/python:layers", "//tensorflow/python:state_ops", "//tensorflow/python:variable_scope", - "//tensorflow/python:framework_test_lib", + "//tensorflow/python/distribute:distribute_lib", + "//tensorflow/python/distribute:values", "//tensorflow/python/eager:context", "//tensorflow/python/eager:test", ], + shard_count = 5, tags = [ "guitar", - "no_pip", "multi_and_single_gpu", - # Do not perform the extra analysis on this test, because it is already - # performed for the `:mirrored_strategy_test` target. - "no_oss", - "noasan", - "notap", - "notsan", + "no_pip", ], ) @@ -337,12 +281,15 @@ py_library( visibility = ["//tensorflow:internal"], deps = [ ":one_device_strategy", - ":values", "//tensorflow/contrib/tpu:tpu_lib", "//tensorflow/python:constant_op", "//tensorflow/python:control_flow_ops", + "//tensorflow/python:dtypes", "//tensorflow/python:framework_ops", + "//tensorflow/python:tensor_util", "//tensorflow/python:util", + "//tensorflow/python/distribute:reduce_util", + "//tensorflow/python/distribute:values", ], ) @@ -352,7 +299,6 @@ cuda_py_test( additional_deps = [ ":collective_all_reduce_strategy", ":combinations", - ":cross_tower_utils", ":multi_worker_test_base", ":strategy_test_lib", "@absl_py//absl/testing:parameterized", @@ -368,6 +314,7 @@ cuda_py_test( "//tensorflow/python:layers", "//tensorflow/python:variable_scope", "//tensorflow/python:variables", + "//tensorflow/python/distribute:cross_device_utils", "//tensorflow/python/eager:context", "//tensorflow/python/estimator:estimator_py", ], @@ -469,6 +416,7 @@ cuda_py_test( "multi_and_single_gpu", "no_oss", # http://b/119349471 "no_pip", + "tf_integration_test", ], ) @@ -476,28 +424,18 @@ cuda_py_test( name = "keras_optimizer_v2_test", srcs = ["keras_optimizer_v2_test.py"], additional_deps = [ - ":combinations", - "@absl_py//absl/testing:parameterized", - "//third_party/py/numpy", - "//tensorflow/contrib/optimizer_v2:training", - "//tensorflow/python/data/ops:dataset_ops", - "//tensorflow/python/eager:test", - "//tensorflow/python/estimator:estimator_py", - "//tensorflow/python/feature_column", - "//tensorflow/python:framework_ops", - "//tensorflow/python:platform", - "//tensorflow/python:summary", + ":keras_test_lib", ], tags = [ "multi_and_single_gpu", "no_oss", # http://b/119349471 "no_pip", + "tf_integration_test", ], ) cuda_py_test( name = "estimator_training_test", - size = "large", srcs = ["estimator_training_test.py"], additional_deps = [ ":collective_all_reduce_strategy", @@ -508,7 +446,9 @@ cuda_py_test( "//third_party/py/numpy", "//tensorflow/contrib/optimizer_v2:training", "//tensorflow/python/data/ops:dataset_ops", - "//tensorflow/python/distribute", + "//tensorflow/python/distribute:distribute_config", + "//tensorflow/python/distribute:distribute_coordinator", + "//tensorflow/python/distribute:distribute_coordinator_context", "//tensorflow/python/eager:test", "//tensorflow/python/estimator:estimator_py", "//tensorflow/python/feature_column", @@ -516,7 +456,7 @@ cuda_py_test( "//tensorflow/python:platform", "//tensorflow/python:summary", ], - shard_count = 5, + shard_count = 48, tags = [ "multi_and_single_gpu", "no_pip", @@ -524,6 +464,7 @@ cuda_py_test( "noasan", "nomsan", "notsan", + "no_oss", # http://b/119349471 ], ) @@ -599,52 +540,16 @@ cuda_py_test( ], ) -py_library( - name = "shared_variable_creator", - srcs = ["shared_variable_creator.py"], - visibility = ["//tensorflow:internal"], -) - -py_test( - name = "shared_variable_creator_test", - srcs = ["shared_variable_creator_test.py"], - srcs_version = "PY2AND3", - deps = [ - ":shared_variable_creator", - "//tensorflow/python:framework_test_lib", - "//tensorflow/python:variable_scope", - "//tensorflow/python/eager:test", - ], -) - -py_library( - name = "cross_tower_utils", - srcs = ["cross_tower_utils.py"], - srcs_version = "PY2AND3", - deps = [ - ":values", - "//tensorflow/contrib/all_reduce:all_reduce_py", - "//tensorflow/python:array_ops", - "//tensorflow/python:collective_ops", - "//tensorflow/python:device", - "//tensorflow/python:dtypes", - "//tensorflow/python:framework_ops", - "//tensorflow/python:gradients", - "//tensorflow/python:math_ops", - "//tensorflow/python:nccl_ops", - ], -) - cuda_py_test( - name = "cross_tower_utils_test", - srcs = ["cross_tower_utils_test.py"], + name = "cross_device_utils_test", + srcs = ["cross_device_utils_test.py"], additional_deps = [ ":combinations", - ":cross_tower_utils", "@absl_py//absl/testing:parameterized", "//tensorflow/python:constant_op", "//tensorflow/python:framework_ops", "//tensorflow/python:math_ops", + "//tensorflow/python/distribute:cross_device_utils", "//tensorflow/python/eager:context", "//tensorflow/python/eager:test", ], @@ -653,40 +558,20 @@ cuda_py_test( ], ) -py_library( - name = "cross_tower_ops", - srcs = ["cross_tower_ops.py"], - srcs_version = "PY2AND3", - deps = [ - ":cross_tower_utils", - ":values", - "//tensorflow/python:array_ops", - "//tensorflow/python:device_lib", - "//tensorflow/python:framework_ops", - "//tensorflow/python:math_ops", - "//tensorflow/python:platform", - "//tensorflow/python:resource_variable_ops", - "//tensorflow/python:training", - "//tensorflow/python:variable_scope", - "//tensorflow/python/eager:context", - "@six_archive//:six", - ], -) - cuda_py_test( - name = "cross_tower_ops_test", - srcs = ["cross_tower_ops_test.py"], + name = "cross_device_ops_test", + srcs = ["cross_device_ops_test.py"], additional_deps = [ ":combinations", - ":cross_tower_ops", ":multi_worker_test_base", ":mirrored_strategy", - ":values", "@absl_py//absl/testing:parameterized", "//tensorflow/python:array_ops", "//tensorflow/python:constant_op", "//tensorflow/python:framework_ops", "//tensorflow/python:math_ops", + "//tensorflow/python/distribute:cross_device_ops", + "//tensorflow/python/distribute:values", "//tensorflow/python/eager:context", "//tensorflow/python/eager:test", ], @@ -696,37 +581,6 @@ cuda_py_test( ], ) -py_library( - name = "input_ops", - srcs = ["input_ops.py"], - visibility = ["//tensorflow:internal"], - deps = [ - "//tensorflow/python:framework_ops", - "//tensorflow/python/data/util:nest", - ], -) - -cuda_py_test( - name = "input_ops_test", - srcs = ["input_ops_test.py"], - additional_deps = [ - ":input_ops", - "//tensorflow/python/data/ops:dataset_ops", - "//tensorflow/contrib/data/python/ops:batching", - "//tensorflow/contrib/data/python/ops:interleave_ops", - "//tensorflow/python:errors", - "//tensorflow/python:client_testlib", - "//tensorflow/python:framework_ops", - "//tensorflow/python:framework_test_lib", - "//tensorflow/python:io_ops", - "//tensorflow/python/data/ops:readers", - "//tensorflow/python:util", - ], - tags = [ - "no_pip", - ], -) - py_library( name = "keras_test_lib", testonly = 1, @@ -737,6 +591,7 @@ py_library( "//tensorflow/contrib/distribute/python:tpu_strategy", "//tensorflow/python:client_testlib", "//tensorflow/python:training", + "//tensorflow/python/eager:test", "//tensorflow/python/estimator:estimator_py", "//tensorflow/python/keras", "//third_party/py/numpy", @@ -766,7 +621,6 @@ py_library( srcs = ["metrics_v1_test.py"], deps = [ ":combinations", - "//tensorflow/contrib/data/python/ops:batching", "//tensorflow/python:math_ops", "//tensorflow/python:metrics", "//tensorflow/python:variables", diff --git a/tensorflow/contrib/distribute/python/checkpoint_utils_test.py b/tensorflow/contrib/distribute/python/checkpoint_utils_test.py index d38bdb592a3..31bd0e996a2 100644 --- a/tensorflow/contrib/distribute/python/checkpoint_utils_test.py +++ b/tensorflow/contrib/distribute/python/checkpoint_utils_test.py @@ -43,7 +43,9 @@ class CheckpointUtilsWithDistributionStrategyTest( distribution=[combinations.default_strategy, combinations.one_device_strategy, combinations.mirrored_strategy_with_gpu_and_cpu, - combinations.mirrored_strategy_with_two_gpus], + combinations.mirrored_strategy_with_two_gpus, + combinations.core_mirrored_strategy_with_gpu_and_cpu, + combinations.core_mirrored_strategy_with_two_gpus], in_replica_mode=[True, False], mode=["graph"])) def testInitFromCheckpoint(self, distribution, in_replica_mode): diff --git a/tensorflow/contrib/distribute/python/collective_all_reduce_strategy.py b/tensorflow/contrib/distribute/python/collective_all_reduce_strategy.py index efa99d1fc52..e988b63a287 100644 --- a/tensorflow/contrib/distribute/python/collective_all_reduce_strategy.py +++ b/tensorflow/contrib/distribute/python/collective_all_reduce_strategy.py @@ -18,12 +18,16 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function -from tensorflow.contrib.distribute.python import cross_tower_ops as cross_tower_ops_lib -from tensorflow.contrib.distribute.python import cross_tower_utils +import copy + from tensorflow.contrib.distribute.python import mirrored_strategy -from tensorflow.contrib.distribute.python import values from tensorflow.core.protobuf import rewriter_config_pb2 +from tensorflow.python.distribute import cross_device_ops as cross_device_ops_lib +from tensorflow.python.distribute import cross_device_utils +from tensorflow.python.distribute import device_util +from tensorflow.python.distribute import distribute_lib from tensorflow.python.distribute import multi_worker_util +from tensorflow.python.distribute import values from tensorflow.python.eager import context from tensorflow.python.framework import ops from tensorflow.python.ops import array_ops @@ -32,7 +36,7 @@ from tensorflow.python.platform import tf_logging as logging # TODO(yuefengz): support in-graph replication. -class CollectiveAllReduceStrategy(mirrored_strategy.MirroredStrategy): +class CollectiveAllReduceStrategy(distribute_lib.DistributionStrategy): """Distribution strategy that uses collective ops for all-reduce. It is similar to the MirroredStrategy but it uses collective ops for @@ -53,6 +57,17 @@ class CollectiveAllReduceStrategy(mirrored_strategy.MirroredStrategy): num_gpus_per_worker: number of local GPUs or GPUs per worker, the default is 0 meaning CPU only. """ + super(CollectiveAllReduceStrategy, self).__init__( + CollectiveAllReduceExtended(self, num_gpus_per_worker)) + + +class CollectiveAllReduceExtended(mirrored_strategy.MirroredExtended): + """Implementation of CollectiveAllReduceStrategy.""" + + def __init__(self, container_strategy, num_gpus_per_worker): + distribute_lib.DistributionStrategyExtended.__init__( + self, container_strategy) + self._cross_device_ops = None self._num_gpus_per_worker = num_gpus_per_worker self._initialize_local_worker(num_gpus_per_worker) @@ -67,14 +82,14 @@ class CollectiveAllReduceStrategy(mirrored_strategy.MirroredStrategy): ] else: local_devices = ["/device:CPU:0"] + self._worker_device = device_util.canonicalize("/device:CPU:0") - self._collective_keys = cross_tower_utils.CollectiveKeys() - super(CollectiveAllReduceStrategy, self).__init__( - devices=local_devices, - cross_tower_ops=cross_tower_ops_lib.CollectiveAllReduce( - num_workers=1, - num_gpus_per_worker=num_gpus_per_worker, - collective_keys=self._collective_keys)) + self._collective_keys = cross_device_utils.CollectiveKeys() + self._initialize_local(local_devices) + self._cross_tower_ops = cross_device_ops_lib.CollectiveAllReduce( + num_workers=self._num_workers, + num_gpus_per_worker=num_gpus_per_worker, + collective_keys=self._collective_keys) self._cluster_spec = None self._task_type = None @@ -94,8 +109,7 @@ class CollectiveAllReduceStrategy(mirrored_strategy.MirroredStrategy): "Unrecognized task_type: %r, valid task types are: \"chief\", " "\"worker\"." % task_type) cluster_spec = multi_worker_util.normalize_cluster_spec(cluster_spec) - self._num_workers = len(cluster_spec.as_dict().get("worker", [])) + len( - cluster_spec.as_dict().get("chief", [])) + self._num_workers = multi_worker_util.worker_count(cluster_spec, task_type) if not self._num_workers: raise ValueError("No `worker` or `chief` tasks can be found in " "`cluster_spec`.") @@ -103,22 +117,21 @@ class CollectiveAllReduceStrategy(mirrored_strategy.MirroredStrategy): self._is_chief = multi_worker_util.is_chief(cluster_spec, task_type, task_id) - worker_device = "/job:%s/task:%d" % (task_type, task_id) + self._worker_device = "/job:%s/task:%d" % (task_type, task_id) if num_gpus_per_worker: local_devices = [ - "%s/device:GPU:%d" % (worker_device, i) + "%s/device:GPU:%d" % (self._worker_device, i) for i in range(num_gpus_per_worker) ] else: - local_devices = [worker_device] + local_devices = [self._worker_device] - self._collective_keys = cross_tower_utils.CollectiveKeys() - super(CollectiveAllReduceStrategy, self).__init__( - devices=local_devices, - cross_tower_ops=cross_tower_ops_lib.CollectiveAllReduce( - num_workers=self._num_workers, - num_gpus_per_worker=num_gpus_per_worker, - collective_keys=self._collective_keys)) + self._collective_keys = cross_device_utils.CollectiveKeys() + self._initialize_local(local_devices) + self._cross_tower_ops = cross_device_ops_lib.CollectiveAllReduce( + num_workers=self._num_workers, + num_gpus_per_worker=num_gpus_per_worker, + collective_keys=self._collective_keys) # Add a default device so that ops without specified devices will not end up # on other workers. @@ -202,17 +215,40 @@ class CollectiveAllReduceStrategy(mirrored_strategy.MirroredStrategy): return mirrored_strategy._create_mirrored_variable( devices, _real_mirrored_creator, *args, **kwargs) - def distribute_dataset(self, dataset_fn): + def _distribute_dataset(self, dataset_fn): """Distributes the dataset to each local GPU.""" # TODO(yuefengz): shard the dataset. return values.PerReplicaDataset( self._call_dataset_fn(dataset_fn), self._devices, True) - def configure(self, - session_config=None, - cluster_spec=None, - task_type=None, - task_id=None): + def _make_dataset_iterator(self, dataset): + worker_device_pairs = [(self._worker_device, self._devices)] + return values.DatasetIterator(dataset, worker_device_pairs, + self._num_replicas_in_sync) + + def _make_input_fn_iterator( + self, + input_fn, + replication_mode=distribute_lib.InputReplicationMode.PER_WORKER): + """Distributes the dataset to each local GPU.""" + if self._cluster_spec is None: + input_pipeline_id = 0 + else: + input_pipeline_id = multi_worker_util.id_in_cluster( + self._cluster_spec, self._task_type, self._task_id) + input_context = distribute_lib.InputContext( + num_input_pipelines=self._num_workers, + input_pipeline_id=input_pipeline_id, + num_replicas_in_sync=self._num_replicas_in_sync) + + return values.InputFunctionIterator( + input_fn, [(self._worker_device, self._devices)], [input_context]) + + def _configure(self, + session_config=None, + cluster_spec=None, + task_type=None, + task_id=None): """Configures the object. Args: @@ -232,13 +268,15 @@ class CollectiveAllReduceStrategy(mirrored_strategy.MirroredStrategy): self._initialize_multi_worker(self._num_gpus_per_worker, cluster_spec, task_type, task_id) - if not session_config: - return + if session_config: + session_config.CopyFrom(self._update_config_proto(session_config)) + def _update_config_proto(self, config_proto): + updated_config = copy.deepcopy(config_proto) # Enable the scoped allocator optimization for CollectiveOps. This # optimization converts many small all-reduces into fewer larger # all-reduces. - rewrite_options = session_config.graph_options.rewrite_options + rewrite_options = updated_config.graph_options.rewrite_options rewrite_options.scoped_allocator_optimization = ( rewriter_config_pb2.RewriterConfig.ON) # We turn on ScopedAllocator only for CollectiveReduce op, i.e. enable_op = @@ -248,7 +286,7 @@ class CollectiveAllReduceStrategy(mirrored_strategy.MirroredStrategy): rewrite_options.scoped_allocator_opts.enable_op.append("CollectiveReduce") if not self._cluster_spec: - return + return updated_config assert self._task_type assert self._task_id is not None @@ -256,26 +294,28 @@ class CollectiveAllReduceStrategy(mirrored_strategy.MirroredStrategy): # Collective group leader is needed for collective ops to coordinate # workers. if "chief" in self._cluster_spec.jobs: - session_config.experimental.collective_group_leader = ( + updated_config.experimental.collective_group_leader = ( "/job:chief/replica:0/task:0") else: if "worker" not in self._cluster_spec.jobs: raise ValueError( "You must have `chief` or `worker` jobs in the `cluster_spec`.") - session_config.experimental.collective_group_leader = ( + updated_config.experimental.collective_group_leader = ( "/job:worker/replica:0/task:0") # The device filters prevent communication between workers. - del session_config.device_filters[:] - session_config.device_filters.append( + del updated_config.device_filters[:] + updated_config.device_filters.append( "/job:%s/task:%d" % (self._task_type, self._task_id)) + return updated_config + @property - def between_graph(self): + def experimental_between_graph(self): return True @property - def should_init(self): + def experimental_should_init(self): return True @property @@ -287,6 +327,10 @@ class CollectiveAllReduceStrategy(mirrored_strategy.MirroredStrategy): return self._is_chief @property - def num_replicas_in_sync(self): + def _num_replicas_in_sync(self): return len(self._devices) * self._num_workers + # TODO(priyag): Delete this once all strategies use global batch size. + @property + def _global_batch_size(self): + return False diff --git a/tensorflow/contrib/distribute/python/collective_all_reduce_strategy_test.py b/tensorflow/contrib/distribute/python/collective_all_reduce_strategy_test.py index e3d919dd0d4..8a9e583f0af 100644 --- a/tensorflow/contrib/distribute/python/collective_all_reduce_strategy_test.py +++ b/tensorflow/contrib/distribute/python/collective_all_reduce_strategy_test.py @@ -23,13 +23,19 @@ import numpy as np from tensorflow.contrib.distribute.python import collective_all_reduce_strategy from tensorflow.contrib.distribute.python import combinations -from tensorflow.contrib.distribute.python import cross_tower_utils from tensorflow.contrib.distribute.python import multi_worker_test_base +from tensorflow.contrib.distribute.python import strategy_test_lib from tensorflow.core.protobuf import config_pb2 +from tensorflow.core.protobuf import rewriter_config_pb2 from tensorflow.python import keras +from tensorflow.python.data.ops import dataset_ops +from tensorflow.python.distribute import cross_device_utils +from tensorflow.python.distribute import reduce_util +from tensorflow.python.distribute import values from tensorflow.python.eager import context from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes +from tensorflow.python.framework import errors from tensorflow.python.framework import ops from tensorflow.python.layers import core from tensorflow.python.ops import array_ops @@ -51,9 +57,6 @@ class CollectiveAllReduceStrategyTestBase( collective_key_base = 0 def setUp(self): - self._run_options = config_pb2.RunOptions() - self._run_options.experimental.collective_graph_key = 6 - # We use a different key_base for each test so that collective keys won't be # reused. # TODO(yuefengz, tucker): enable it to reuse collective keys in different @@ -71,15 +74,16 @@ class CollectiveAllReduceStrategyTestBase( cluster_spec=self._cluster_spec, task_type=task_type, task_id=task_id) - collective_keys = cross_tower_utils.CollectiveKeys( + collective_keys = cross_device_utils.CollectiveKeys( group_key_start=10 * num_gpus + CollectiveAllReduceStrategyTestBase.collective_key_base, instance_key_start=num_gpus * 100 + CollectiveAllReduceStrategyTestBase.collective_key_base, instance_key_with_id_start=num_gpus * 10000 + CollectiveAllReduceStrategyTestBase.collective_key_base) - distribution._collective_keys = collective_keys - distribution._cross_tower_ops._collective_keys = collective_keys + distribution.extended._collective_keys = collective_keys + distribution.extended._inferred_cross_device_ops._collective_keys = ( + collective_keys) if task_type and task_id is not None: return distribution, 'grpc://' + self._cluster_spec[task_type][ task_id], session_config @@ -93,7 +97,8 @@ class CollectiveAllReduceStrategyTestBase( self.cached_session(config=config, target=master_target) as sess, \ d.scope(): - l = core.Dense(1, use_bias=False, name='gpu_%d' % d._num_gpus_per_worker) + l = core.Dense(1, use_bias=False, + name='gpu_%d' % d.extended._num_gpus_per_worker) def loss_fn(x): y = array_ops.reshape(l(x), []) - constant_op.constant(1.) @@ -127,8 +132,8 @@ class CollectiveAllReduceStrategyTestBase( before_list.append(fetched) with ops.control_dependencies([fetched]): # TODO(yuefengz): support non-Mirrored variable as destinations. - g = d.reduce( - variable_scope.VariableAggregation.SUM, g, destinations=v) + g = d.extended.reduce_to( + reduce_util.ReduceOp.SUM, g, destinations=v) with ops.control_dependencies( d.update(v, update, g, grouped=False)): after_list.append(d.read_var(v)) @@ -136,14 +141,13 @@ class CollectiveAllReduceStrategyTestBase( before_out, after_out = step() - if context.num_gpus() < d._num_gpus_per_worker: + if context.num_gpus() < d.extended._num_gpus_per_worker: return True - sess.run( - variables.global_variables_initializer(), options=self._run_options) + sess.run(variables.global_variables_initializer()) for i in range(10): - b, a = sess.run((before_out, after_out), options=self._run_options) + b, a = sess.run((before_out, after_out)) if i == 0: before, = b after, = a @@ -222,26 +226,54 @@ class CollectiveAllReduceStrategyTestBase( return array_ops.identity(x) x = distribution.call_for_each_replica(model_fn) - reduced_x = distribution.unwrap( - distribution.reduce( - variable_scope.VariableAggregation.MEAN, x, - destinations='/cpu:0'))[0] + reduced_x = distribution.reduce(reduce_util.ReduceOp.MEAN, x) x = distribution.unwrap(x)[0] - sess.run( - variables.global_variables_initializer(), options=self._run_options) + sess.run(variables.global_variables_initializer()) - x_value, reduced_x_value = sess.run([x, reduced_x], - options=self._run_options) + x_value, reduced_x_value = sess.run([x, reduced_x]) self.assertTrue( np.allclose(x_value, reduced_x_value, atol=1e-5), msg=('x_value = %r, reduced_x_value = %r' % (x_value, reduced_x_value))) return np.allclose(x_value, reduced_x_value, atol=1e-5) + def _test_input_fn_iterator(self, task_type, task_id, num_gpus, input_fn, + expected_values): + distribution, master_target, config = self._get_test_object( + task_type, task_id, num_gpus) + devices = distribution.extended.worker_devices + + with ops.Graph().as_default(), \ + self.cached_session(config=config, + target=master_target) as sess: + iterator = distribution.make_input_fn_iterator(input_fn) + sess.run(iterator.initialize()) + + for expected_value in expected_values: + next_element = iterator.get_next() + computed_value = sess.run( + [values.select_device(d, next_element) for d in devices]) + self.assertEqual(expected_value, computed_value) + + with self.assertRaises(errors.OutOfRangeError): + next_element = iterator.get_next() + sess.run([values.select_device(d, next_element) for d in devices]) + + # After re-initializing the iterator, should be able to iterate again. + sess.run(iterator.initialize()) + + for expected_value in expected_values: + next_element = iterator.get_next() + computed_value = sess.run( + [values.select_device(d, next_element) for d in devices]) + self.assertEqual(expected_value, computed_value) + class DistributedCollectiveAllReduceStrategyTest( - CollectiveAllReduceStrategyTestBase, parameterized.TestCase): + CollectiveAllReduceStrategyTestBase, + strategy_test_lib.DistributionTestBase, + parameterized.TestCase): @classmethod def setUpClass(cls): @@ -269,7 +301,7 @@ class DistributedCollectiveAllReduceStrategyTest( combinations.combine(mode=['graph'], num_gpus=[0, 1, 2], required_gpus=1)) def testVariableInitialization(self, num_gpus): if context.num_gpus() < num_gpus: - return + self.skipTest('Not enough GPUs') self._run_between_graph_clients( self._test_variable_initialization, self._cluster_spec, @@ -279,10 +311,56 @@ class DistributedCollectiveAllReduceStrategyTest( combinations.combine(mode=['graph'], num_gpus=[0, 1, 2], required_gpus=1)) def testComplexModel(self, num_gpus): if context.num_gpus() < num_gpus: - return + self.skipTest('Not enough GPUs') self._run_between_graph_clients( self._test_complex_model, self._cluster_spec, num_gpus=num_gpus) + # TODO(yuefengz): Update how we use num_gpus and required_gpus + @combinations.generate( + combinations.combine(mode=['graph'], num_gpus=[0, 1, 2], required_gpus=1)) + def testMakeInputFnIterator(self, num_gpus): + if context.num_gpus() < num_gpus: + self.skipTest('Not enough GPUs') + dataset_fn = lambda: dataset_ops.Dataset.range(100) + # We use CPU as the device when num_gpus = 0 + devices_per_worker = max(1, num_gpus) + expected_values = [[i+j for j in range(devices_per_worker)] + for i in range(0, 100, devices_per_worker)] + + input_fn = self._input_fn_to_test_input_context( + dataset_fn, + expected_num_replicas_in_sync=3*devices_per_worker, + expected_num_input_pipelines=3, + expected_input_pipeline_id=1) # because task_id = 1 + self._test_input_fn_iterator('worker', 1, num_gpus, + input_fn, expected_values) + + def testUpdateConfigProto(self): + distribution = collective_all_reduce_strategy.CollectiveAllReduceStrategy( + num_gpus_per_worker=2) + distribution.configure( + cluster_spec=self._cluster_spec, task_type='worker', task_id=1) + + config_proto = config_pb2.ConfigProto(device_filters=['to_be_overridden']) + rewrite_options = config_proto.graph_options.rewrite_options + rewrite_options.scoped_allocator_opts.enable_op.append('to_be_removed') + + new_config = distribution.update_config_proto(config_proto) + + # Verify group leader + self.assertEqual('/job:worker/replica:0/task:0', + new_config.experimental.collective_group_leader) + + # Verify device filters. + self.assertEqual(['/job:worker/task:1'], new_config.device_filters) + + # Verify rewrite options. + new_rewrite_options = new_config.graph_options.rewrite_options + self.assertEqual(rewriter_config_pb2.RewriterConfig.ON, + new_rewrite_options.scoped_allocator_optimization) + self.assertEqual(['CollectiveReduce'], + new_rewrite_options.scoped_allocator_opts.enable_op) + class DistributedCollectiveAllReduceStrategyTestWithChief( CollectiveAllReduceStrategyTestBase, parameterized.TestCase): @@ -293,10 +371,6 @@ class DistributedCollectiveAllReduceStrategyTestWithChief( cls._cluster_spec = multi_worker_test_base.create_in_process_cluster( num_workers=3, num_ps=0, has_chief=True) - def setUp(self): - super(DistributedCollectiveAllReduceStrategyTestWithChief, self).setUp() - self._run_options.experimental.collective_graph_key = 7 - @combinations.generate( combinations.combine(mode=['graph'], num_gpus=[0, 1, 2], required_gpus=1)) def testMinimizeLossGraph(self, num_gpus): @@ -323,20 +397,36 @@ class DistributedCollectiveAllReduceStrategyTestWithChief( class LocalCollectiveAllReduceStrategy(CollectiveAllReduceStrategyTestBase, + strategy_test_lib.DistributionTestBase, parameterized.TestCase): def testMinimizeLossGraph(self, num_gpus=2): # Collective ops doesn't support strategy with one device. if context.num_gpus() < num_gpus: - return + self.skipTest('Not enough GPUs') self._test_minimize_loss_graph(None, None, num_gpus) def testComplexModel(self, num_gpus=2): # Collective ops doesn't support strategy with one device. if context.num_gpus() < num_gpus: - return + self.skipTest('Not enough GPUs') self._test_complex_model(None, None, num_gpus) + def testMakeInputFnIterator(self, num_gpus=2): + # Collective ops doesn't support strategy with one device. + if context.num_gpus() < num_gpus: + self.skipTest('Not enough GPUs') + dataset_fn = lambda: dataset_ops.Dataset.range(10) + expected_values = [[i, i+1] for i in range(0, 10, 2)] + + input_fn = self._input_fn_to_test_input_context( + dataset_fn, + expected_num_replicas_in_sync=num_gpus, + expected_num_input_pipelines=1, + expected_input_pipeline_id=0) + self._test_input_fn_iterator(None, None, num_gpus, + input_fn, expected_values) + if __name__ == '__main__': test.main() diff --git a/tensorflow/contrib/distribute/python/combinations.py b/tensorflow/contrib/distribute/python/combinations.py index a5137165403..365ce5cdec7 100644 --- a/tensorflow/contrib/distribute/python/combinations.py +++ b/tensorflow/contrib/distribute/python/combinations.py @@ -53,11 +53,11 @@ from tensorflow.contrib.distribute.python import tpu_strategy as tpu_lib from tensorflow.contrib.optimizer_v2 import adagrad as adagrad_v2 from tensorflow.contrib.optimizer_v2 import adam as adam_v2 from tensorflow.contrib.optimizer_v2 import gradient_descent as gradient_descent_v2 +from tensorflow.python.distribute import distribution_strategy_context from tensorflow.python.eager import context from tensorflow.python.framework import ops from tensorflow.python.training import adagrad from tensorflow.python.training import adam -from tensorflow.python.training import distribution_strategy_context from tensorflow.python.training import gradient_descent from tensorflow.python.training import rmsprop from tensorflow.python.util import tf_inspect @@ -168,6 +168,8 @@ def _augment_with_special_arguments(test_method): if GPU_TEST: self.skipTest("Test that doesn't require GPUs.") elif context.num_gpus() < required_gpus: + # TODO(priyag): Consider allowing tests in graph mode using soft + # placement. self.skipTest( "{} GPUs are not available for this test. {} GPUs are available". format(required_gpus, context.num_gpus())) @@ -190,7 +192,7 @@ def _augment_with_special_arguments(test_method): kwargs_to_pass[arg] = kwargs[arg] if mode == "eager": - with ops.Graph().as_default(), context.eager_mode(): + with context.eager_mode(): if distribution: kwargs_to_pass["distribution"] = distribution.strategy test_method(**kwargs_to_pass) @@ -335,6 +337,13 @@ tpu_strategy_one_step = NamedDistribution( "TPUOneStep", lambda: tpu_lib.TPUStrategy( TPUClusterResolver(""), steps_per_run=1), required_tpu=True) +mirrored_strategy_with_one_cpu = NamedDistribution( + "Mirrored1CPU", + lambda: mirrored_lib.MirroredStrategy(["/cpu:0"])) +mirrored_strategy_with_one_gpu = NamedDistribution( + "Mirrored1GPU", + lambda: mirrored_lib.MirroredStrategy(["/gpu:0"]), + required_gpus=1) mirrored_strategy_with_gpu_and_cpu = NamedDistribution( "MirroredCPUAndGPU", lambda: mirrored_lib.MirroredStrategy(["/gpu:0", "/cpu:0"]), @@ -343,6 +352,21 @@ mirrored_strategy_with_two_gpus = NamedDistribution( "Mirrored2GPUs", lambda: mirrored_lib.MirroredStrategy(["/gpu:0", "/gpu:1"]), required_gpus=2) +core_mirrored_strategy_with_one_cpu = NamedDistribution( + "CoreMirrored1CPU", + lambda: mirrored_lib.CoreMirroredStrategy(["/cpu:0"])) +core_mirrored_strategy_with_one_gpu = NamedDistribution( + "CoreMirrored1GPU", + lambda: mirrored_lib.CoreMirroredStrategy(["/gpu:0"]), + required_gpus=1) +core_mirrored_strategy_with_gpu_and_cpu = NamedDistribution( + "CoreMirroredCPUAndGPU", + lambda: mirrored_lib.CoreMirroredStrategy(["/gpu:0", "/cpu:0"]), + required_gpus=1) +core_mirrored_strategy_with_two_gpus = NamedDistribution( + "CoreMirrored2GPUs", + lambda: mirrored_lib.CoreMirroredStrategy(["/gpu:0", "/gpu:1"]), + required_gpus=2) gradient_descent_optimizer_v1_fn = NamedObject( @@ -373,8 +397,11 @@ def distributions_and_v1_optimizers(): """A common set of combination with DistributionStrategies and Optimizers.""" return combine( distribution=[ - one_device_strategy, mirrored_strategy_with_gpu_and_cpu, - mirrored_strategy_with_two_gpus + one_device_strategy, + mirrored_strategy_with_gpu_and_cpu, + mirrored_strategy_with_two_gpus, + core_mirrored_strategy_with_gpu_and_cpu, + core_mirrored_strategy_with_two_gpus, ], optimizer_fn=optimizers_v1) @@ -383,7 +410,10 @@ def distributions_and_v2_optimizers(): """DistributionStrategies and V2 Optimizers.""" return combine( distribution=[ - one_device_strategy, mirrored_strategy_with_gpu_and_cpu, - mirrored_strategy_with_two_gpus + one_device_strategy, + mirrored_strategy_with_gpu_and_cpu, + mirrored_strategy_with_two_gpus, + core_mirrored_strategy_with_gpu_and_cpu, + core_mirrored_strategy_with_two_gpus, ], optimizer_fn=optimizers_v2) diff --git a/tensorflow/contrib/distribute/python/cross_tower_ops_test.py b/tensorflow/contrib/distribute/python/cross_device_ops_test.py similarity index 79% rename from tensorflow/contrib/distribute/python/cross_tower_ops_test.py rename to tensorflow/contrib/distribute/python/cross_device_ops_test.py index 3e274ba67ca..d6e9521c1c1 100644 --- a/tensorflow/contrib/distribute/python/cross_tower_ops_test.py +++ b/tensorflow/contrib/distribute/python/cross_device_ops_test.py @@ -24,24 +24,24 @@ from absl.testing import parameterized import numpy as np from tensorflow.contrib.distribute.python import combinations -from tensorflow.contrib.distribute.python import cross_tower_ops as cross_tower_ops_lib -from tensorflow.contrib.distribute.python import cross_tower_utils from tensorflow.contrib.distribute.python import mirrored_strategy from tensorflow.contrib.distribute.python import multi_worker_test_base -from tensorflow.contrib.distribute.python import values as value_lib from tensorflow.core.protobuf import config_pb2 +from tensorflow.python.distribute import cross_device_ops as cross_device_ops_lib +from tensorflow.python.distribute import cross_device_utils +from tensorflow.python.distribute import device_util +from tensorflow.python.distribute import reduce_util +from tensorflow.python.distribute import values as value_lib from tensorflow.python.eager import context from tensorflow.python.eager import test from tensorflow.python.framework import constant_op from tensorflow.python.framework import ops from tensorflow.python.ops import array_ops from tensorflow.python.ops import math_ops -from tensorflow.python.ops import variable_scope as vs -from tensorflow.python.training import device_util def _make_per_replica(values, devices, regroup=False): - devices = cross_tower_ops_lib.get_devices_from(devices) + devices = cross_device_ops_lib.get_devices_from(devices) assert len(values) == len(devices) # We simulate the result of regroup called on PerReplica which strips the @@ -66,7 +66,7 @@ def _fake_mirrored(value, devices): All components of the returned Mirrored have the same objects, which is not true in reality. """ - devices = cross_tower_ops_lib.get_devices_from(devices) + devices = cross_device_ops_lib.get_devices_from(devices) return value_lib.Mirrored( {d: v for d, v in zip(devices, [value] * len(devices))}) @@ -118,8 +118,8 @@ class CrossDeviceOpsTestBase(test.TestCase, parameterized.TestCase): self.assertEqual( sess.run(list(left._index.values())), list(right._index.values())) - def _testReductionAndBroadcast(self, cross_tower_ops, distribution): - devices = distribution.worker_devices + def _testReductionAndBroadcast(self, cross_device_ops, distribution): + devices = distribution.extended.worker_devices values = [constant_op.constant(float(d)) for d in range(len(devices))] per_replica = _make_per_replica(values, devices) @@ -132,35 +132,33 @@ class CrossDeviceOpsTestBase(test.TestCase, parameterized.TestCase): destination_mirrored = _fake_mirrored(1., devices) destination_different = _fake_mirrored(1., _cpu_device) destination_str = _cpu_device - destination_list = devices all_destinations = [ destination_mirrored, destination_different, destination_str, - destination_list ] # test reduce() for destinations in all_destinations: self._assert_values_equal( - cross_tower_ops.reduce( - vs.VariableAggregation.MEAN, + cross_device_ops.reduce( + reduce_util.ReduceOp.MEAN, per_replica, destinations=destinations), _fake_mirrored(mean, destinations)) self._assert_values_equal( - cross_tower_ops.reduce( - vs.VariableAggregation.MEAN, + cross_device_ops.reduce( + reduce_util.ReduceOp.MEAN, per_replica_2, destinations=destinations), _fake_mirrored(mean_2, destinations)) self._assert_values_equal( - cross_tower_ops.reduce( - vs.VariableAggregation.SUM, per_replica, + cross_device_ops.reduce( + reduce_util.ReduceOp.SUM, per_replica, destinations=destinations), _fake_mirrored(mean * len(devices), destinations)) self._assert_values_equal( - cross_tower_ops.reduce( - vs.VariableAggregation.SUM, + cross_device_ops.reduce( + reduce_util.ReduceOp.SUM, per_replica_2, destinations=destinations), _fake_mirrored(mean_2 * len(devices), destinations)) @@ -168,16 +166,16 @@ class CrossDeviceOpsTestBase(test.TestCase, parameterized.TestCase): # test batch_reduce() for d1, d2 in itertools.product(all_destinations, all_destinations): self._assert_values_equal( - cross_tower_ops.batch_reduce( - vs.VariableAggregation.MEAN, + cross_device_ops.batch_reduce( + reduce_util.ReduceOp.MEAN, [(per_replica, d1), (per_replica_2, d2)]), [ _fake_mirrored(mean, d1), _fake_mirrored(mean_2, d2) ]) self._assert_values_equal( - cross_tower_ops.batch_reduce( - vs.VariableAggregation.SUM, + cross_device_ops.batch_reduce( + reduce_util.ReduceOp.SUM, [(per_replica, d1), (per_replica_2, d2)]), [ _fake_mirrored(mean * len(devices), d1), @@ -187,7 +185,7 @@ class CrossDeviceOpsTestBase(test.TestCase, parameterized.TestCase): # test broadcast() for destinations in all_destinations: self._assert_values_equal( - cross_tower_ops.broadcast(constant_op.constant(1.), destinations), + cross_device_ops.broadcast(constant_op.constant(1.), destinations), _fake_mirrored(1., destinations)) @@ -196,62 +194,65 @@ class SingleWorkerCrossDeviceOpsTest(CrossDeviceOpsTestBase): # combinations module so that we can pass in devices instead of a distribution # strategy. reduction_to_one_combinations = combinations.combine( - cross_tower_ops=[ + cross_device_ops=[ combinations.NamedObject( "DefaultReductionToOneDeviceCrossDeviceOps", - cross_tower_ops_lib.ReductionToOneDeviceCrossDeviceOps()), + cross_device_ops_lib.ReductionToOneDeviceCrossDeviceOps()), combinations.NamedObject( "ReductionToCPUDeviceCrossDeviceOps", - cross_tower_ops_lib.ReductionToOneDeviceCrossDeviceOps( + cross_device_ops_lib.ReductionToOneDeviceCrossDeviceOps( reduce_to_device=_cpu_device)), combinations.NamedObject( "AccumulateNCrossDeviceOp", - cross_tower_ops_lib.ReductionToOneDeviceCrossDeviceOps( + cross_device_ops_lib.ReductionToOneDeviceCrossDeviceOps( accumulation_fn=math_ops.accumulate_n)), ], distribution=[ combinations.one_device_strategy, combinations.mirrored_strategy_with_gpu_and_cpu, - combinations.mirrored_strategy_with_two_gpus + combinations.mirrored_strategy_with_two_gpus, + combinations.core_mirrored_strategy_with_gpu_and_cpu, + combinations.core_mirrored_strategy_with_two_gpus ], mode=["graph", "eager"]) allreduce_combinations = combinations.combine( - cross_tower_ops=[ + cross_device_ops=[ combinations.NamedObject( "AllReduce", - cross_tower_ops_lib.AllReduceCrossDeviceOps("nccl", 1, 0, 0)), + cross_device_ops_lib.AllReduceCrossDeviceOps("nccl", 1, 0, 0)), combinations.NamedObject( "HierarchicalCopy", - cross_tower_ops_lib.AllReduceCrossDeviceOps( + cross_device_ops_lib.AllReduceCrossDeviceOps( "hierarchical_copy", 8, 0, 0)), combinations.NamedObject( "AllReduceNoGradientRepacking", - cross_tower_ops_lib.AllReduceCrossDeviceOps("nccl", 0, 0, 0)), + cross_device_ops_lib.AllReduceCrossDeviceOps("nccl", 0, 0, 0)), combinations.NamedObject( "HierarchicalCopyAggregateSmallTensors", - cross_tower_ops_lib.AllReduceCrossDeviceOps( + cross_device_ops_lib.AllReduceCrossDeviceOps( "hierarchical_copy", 0, 100, 10)) ], - distribution=[combinations.mirrored_strategy_with_two_gpus], + distribution=[combinations.mirrored_strategy_with_two_gpus, + combinations.core_mirrored_strategy_with_two_gpus], mode=["graph", "eager"]) @combinations.generate(reduction_to_one_combinations + allreduce_combinations) - def testReductionAndBroadcast(self, cross_tower_ops, distribution): + def testReductionAndBroadcast(self, cross_device_ops, distribution): with distribution.scope(): - self._testReductionAndBroadcast(cross_tower_ops, distribution) + self._testReductionAndBroadcast(cross_device_ops, distribution) def testChooseAlgorithm(self): device_links = [[1, 2, 3, 4], [0, 2, 3, 5], [0, 1, 3, 6], [0, 1, 2, 7], [0, 5, 6, 7], [1, 4, 6, 7], [2, 4, 5, 7], [3, 4, 5, 6]] - result = cross_tower_ops_lib._choose_all_reduce_algorithm(device_links) - self.assertIsInstance(result, cross_tower_ops_lib.AllReduceCrossDeviceOps) + result = cross_device_ops_lib._choose_all_reduce_algorithm(device_links) + self.assertIsInstance(result, cross_device_ops_lib.AllReduceCrossDeviceOps) self.assertEqual(result._all_reduce_alg, "hierarchical_copy") self.assertEqual(result._num_packs, 8) # if there are only 4 devices device_links = [[1, 2, 3, 4], [0, 2, 3, 5], [0, 1, 3, 6], [0, 1, 2, 7]] - result = cross_tower_ops_lib._choose_all_reduce_algorithm(device_links) - self.assertIsInstance(result, cross_tower_ops_lib.AllReduceCrossDeviceOps) + result = cross_device_ops_lib._choose_all_reduce_algorithm(device_links) + self.assertIsInstance(result, cross_device_ops_lib.AllReduceCrossDeviceOps) self.assertEqual(result._all_reduce_alg, "nccl") self.assertEqual(result._num_packs, 1) @@ -259,16 +260,16 @@ class SingleWorkerCrossDeviceOpsTest(CrossDeviceOpsTestBase): device_links = [[0, 1, 2, 3, 4], [0, 1, 2, 3, 5], [0, 1, 2, 3, 6], [0, 1, 2, 3, 7], [0, 4, 5, 6, 7], [1, 4, 5, 6, 7], [2, 4, 5, 6, 7], [3, 4, 5, 6, 7]] - result = cross_tower_ops_lib._choose_all_reduce_algorithm(device_links) - self.assertIsInstance(result, cross_tower_ops_lib.AllReduceCrossDeviceOps) + result = cross_device_ops_lib._choose_all_reduce_algorithm(device_links) + self.assertIsInstance(result, cross_device_ops_lib.AllReduceCrossDeviceOps) self.assertEqual(result._all_reduce_alg, "hierarchical_copy") self.assertEqual(result._num_packs, 8) # if not dgx1-like links device_links = [[0, 2, 3, 5], [0, 1, 3, 6], [0, 1, 2, 7], [0, 5, 6, 7], [1, 4, 6, 7], [2, 4, 5, 7], [3, 4, 5, 6], [1, 2, 3, 4]] - result = cross_tower_ops_lib._choose_all_reduce_algorithm(device_links) - self.assertIsInstance(result, cross_tower_ops_lib.AllReduceCrossDeviceOps) + result = cross_device_ops_lib._choose_all_reduce_algorithm(device_links) + self.assertIsInstance(result, cross_device_ops_lib.AllReduceCrossDeviceOps) self.assertEqual(result._all_reduce_alg, "nccl") self.assertEqual(result._num_packs, 1) @@ -280,8 +281,8 @@ class SingleWorkerCrossDeviceOpsTest(CrossDeviceOpsTestBase): t0 = _make_indexed_slices([[1., 2.]], [1], [5, 2], devices[0]) t1 = _make_indexed_slices([[3., 4.], [5., 6.]], [1, 3], [5, 2], devices[1]) per_replica = value_lib.PerReplica({devices[0]: t0, devices[1]: t1}) - result = cross_tower_ops_lib._simple_reduce( - per_replica, devices[0], math_ops.add_n, vs.VariableAggregation.SUM) + result = cross_device_ops_lib._simple_reduce( + per_replica, devices[0], math_ops.add_n, reduce_util.ReduceOp.SUM) # Test that the result is semantically equal to both the concatenated # IndexedSlices with and without duplicate indices. @@ -294,19 +295,19 @@ class SingleWorkerCrossDeviceOpsTest(CrossDeviceOpsTestBase): @combinations.generate( combinations.combine( - cross_tower_ops_instance=[ + cross_device_ops_instance=[ combinations.NamedObject( "ReductionToOneDeviceCrossDeviceOps", - cross_tower_ops_lib.ReductionToOneDeviceCrossDeviceOps()), + cross_device_ops_lib.ReductionToOneDeviceCrossDeviceOps()), combinations.NamedObject( "AllReduceCrossDeviceOps", - cross_tower_ops_lib.AllReduceCrossDeviceOps()) + cross_device_ops_lib.AllReduceCrossDeviceOps()) ], - aggregation=[vs.VariableAggregation.SUM, vs.VariableAggregation.MEAN], + reduce_op=[reduce_util.ReduceOp.SUM, reduce_util.ReduceOp.MEAN], batch_reduce=[True, False], mode=["graph", "eager"], required_gpus=1)) - def testIndexedSlicesAllReduce(self, cross_tower_ops_instance, aggregation, + def testIndexedSlicesAllReduce(self, cross_device_ops_instance, reduce_op, batch_reduce): devices = ["/cpu:0", "/gpu:0"] dense_shape = [5, 2] @@ -316,20 +317,20 @@ class SingleWorkerCrossDeviceOpsTest(CrossDeviceOpsTestBase): per_replica = value_lib.PerReplica({devices[0]: t0, devices[1]: t1}) if batch_reduce: - result = cross_tower_ops_instance.batch_reduce( - aggregation, [(per_replica, devices)]) + result = cross_device_ops_instance.batch_reduce( + reduce_op, [(per_replica, per_replica)]) else: - result = cross_tower_ops_instance.reduce( - aggregation, per_replica, devices) + result = cross_device_ops_instance.reduce( + reduce_op, per_replica, per_replica) total_indices_with_dups = [1, 1, 3] total_indices_without_dups = [1, 3] - if aggregation == vs.VariableAggregation.SUM: + if reduce_op == reduce_util.ReduceOp.SUM: total_values_with_dups = [[1., 2.], [3., 4.], [5., 6.]] total_values_without_dups = [[4., 6.], [5., 6.]] else: - assert aggregation == vs.VariableAggregation.MEAN + assert reduce_op == reduce_util.ReduceOp.MEAN total_values_with_dups = [[0.5, 1.], [1.5, 2.], [2.5, 3.]] total_values_without_dups = [[2., 3.], [2.5, 3.]] @@ -356,49 +357,63 @@ class MultiWorkerCrossDeviceOpsTest(multi_worker_test_base.MultiWorkerTestBase, "/job:worker/replica:0/task:0", "/job:worker/replica:0/task:1" ] multi_worker_allreduce_combinations = combinations.combine( - cross_tower_ops=[ + cross_device_ops=[ combinations.NamedObject( "MultiWorkerAllReduce", - cross_tower_ops_lib.MultiWorkerAllReduce( + cross_device_ops_lib.MultiWorkerAllReduce( worker_devices, 2, ("pscpu/pscpu", 2, -1), 0, 0, 0)), combinations.NamedObject( "MultiWorkerAllReducePack", - cross_tower_ops_lib.MultiWorkerAllReduce( + cross_device_ops_lib.MultiWorkerAllReduce( worker_devices, 2, ("pscpu/pscpu", 2, -1), 1, 0, 0)), combinations.NamedObject( "MultiWorkerAllReduceAggregation", - cross_tower_ops_lib.MultiWorkerAllReduce( + cross_device_ops_lib.MultiWorkerAllReduce( worker_devices, 2, ("pscpu/pscpu", 2, -1), 0, 100, 10)), combinations.NamedObject( "MultiWorkerAllReduceMultipleSpecs", - cross_tower_ops_lib.MultiWorkerAllReduce( + cross_device_ops_lib.MultiWorkerAllReduce( worker_devices, 2, [("pscpu/pscpu", 2, 100), ("xring", 2, -1)], 0, 0, 0)), ], distribution=[ combinations.NamedDistribution( "MirroredCPU", - lambda: mirrored_strategy.MirroredStrategy(num_gpus=0), + lambda: mirrored_strategy.MirroredStrategy(num_gpus_per_worker=0), required_gpus=0), combinations.NamedDistribution( "Mirrored1GPU", - lambda: mirrored_strategy.MirroredStrategy(num_gpus=1), + lambda: mirrored_strategy.MirroredStrategy(num_gpus_per_worker=1), required_gpus=1), combinations.NamedDistribution( "Mirrored2GPUs", - lambda: mirrored_strategy.MirroredStrategy(num_gpus=2), + lambda: mirrored_strategy.MirroredStrategy(num_gpus_per_worker=2), + required_gpus=2), + # pylint: disable=g-long-lambda + combinations.NamedDistribution( + "CoreMirroredCPU", + lambda: mirrored_strategy.CoreMirroredStrategy(["/device:CPU:0"]), + required_gpus=0), + combinations.NamedDistribution( + "CoreMirrored1GPU", + lambda: mirrored_strategy.CoreMirroredStrategy(["/device:GPU:0"]), + required_gpus=1), + combinations.NamedDistribution( + "CoreMirrored2GPUs", + lambda: mirrored_strategy.CoreMirroredStrategy( + ["/device:GPU:0", "/device:GPU:1"]), required_gpus=2), ], mode=["graph"]) @combinations.generate(multi_worker_allreduce_combinations) - def testReductionAndBroadcast(self, cross_tower_ops, distribution): + def testReductionAndBroadcast(self, cross_device_ops, distribution): distribution.configure(cluster_spec={ "worker": ["/job:worker/replica:0/task:0", "/job:worker/replica:0/task:1"] }) with distribution.scope(): - self._testReductionAndBroadcast(cross_tower_ops, distribution) + self._testReductionAndBroadcast(cross_device_ops, distribution) class MultiWorkerCollectiveAllReduceTest( @@ -419,7 +434,7 @@ class MultiWorkerCollectiveAllReduceTest( MultiWorkerCollectiveAllReduceTest.collective_key_base += 100000 def _get_test_objects(self, task_type, task_id, num_gpus=0, local_mode=False): - collective_keys = cross_tower_utils.CollectiveKeys( + collective_keys = cross_device_utils.CollectiveKeys( group_key_start=10 * num_gpus + MultiWorkerCollectiveAllReduceTest.collective_key_base, instance_key_start=num_gpus * 100 + @@ -427,7 +442,7 @@ class MultiWorkerCollectiveAllReduceTest( instance_key_with_id_start=num_gpus * 10000 + MultiWorkerCollectiveAllReduceTest.collective_key_base) if local_mode: - collective_all_reduce_ops = cross_tower_ops_lib.CollectiveAllReduce( + collective_all_reduce_ops = cross_device_ops_lib.CollectiveAllReduce( 1, num_gpus, collective_keys=collective_keys) if num_gpus: devices = ["/device:GPU:%d" % i for i in range(num_gpus)] @@ -435,7 +450,7 @@ class MultiWorkerCollectiveAllReduceTest( devices = ["/device:CPU:0"] return collective_all_reduce_ops, devices, "" else: - collective_all_reduce_ops = cross_tower_ops_lib.CollectiveAllReduce( + collective_all_reduce_ops = cross_device_ops_lib.CollectiveAllReduce( 3, num_gpus, collective_keys=collective_keys) if num_gpus: devices = [ @@ -491,37 +506,35 @@ class MultiWorkerCollectiveAllReduceTest( destination_mirrored = _fake_mirrored(1., devices) destination_different = _fake_mirrored(1., _cpu_device) destination_str = _cpu_device - destination_list = devices all_destinations = [ - destination_different, destination_mirrored, destination_str, - destination_list + destination_different, destination_mirrored, destination_str ] # test reduce() for destinations in all_destinations: self._assert_values_equal( collective_all_reduce.reduce( - vs.VariableAggregation.MEAN, + reduce_util.ReduceOp.MEAN, per_replica, destinations=destinations), _fake_mirrored(mean, destinations), sess) self._assert_values_equal( collective_all_reduce.reduce( - vs.VariableAggregation.MEAN, + reduce_util.ReduceOp.MEAN, per_replica_2, destinations=destinations), _fake_mirrored(mean_2, destinations), sess) self._assert_values_equal( collective_all_reduce.reduce( - vs.VariableAggregation.SUM, + reduce_util.ReduceOp.SUM, per_replica, destinations=destinations), _fake_mirrored(mean * len(devices) * num_workers, destinations), sess) self._assert_values_equal( collective_all_reduce.reduce( - vs.VariableAggregation.SUM, + reduce_util.ReduceOp.SUM, per_replica_2, destinations=destinations), _fake_mirrored(mean_2 * len(devices) * num_workers, destinations), @@ -530,7 +543,7 @@ class MultiWorkerCollectiveAllReduceTest( # test batch_reduce() for d1, d2 in itertools.product(all_destinations, all_destinations): self._assert_values_equal( - collective_all_reduce.batch_reduce(vs.VariableAggregation.MEAN, + collective_all_reduce.batch_reduce(reduce_util.ReduceOp.MEAN, [(per_replica, d1), (per_replica_2, d2)]), [ @@ -538,7 +551,7 @@ class MultiWorkerCollectiveAllReduceTest( _fake_mirrored(mean_2, d2) ], sess) self._assert_values_equal( - collective_all_reduce.batch_reduce(vs.VariableAggregation.SUM, + collective_all_reduce.batch_reduce(reduce_util.ReduceOp.SUM, [(per_replica, d1), (per_replica_2, d2)]), [ diff --git a/tensorflow/contrib/distribute/python/cross_tower_utils_test.py b/tensorflow/contrib/distribute/python/cross_device_utils_test.py similarity index 83% rename from tensorflow/contrib/distribute/python/cross_tower_utils_test.py rename to tensorflow/contrib/distribute/python/cross_device_utils_test.py index e46240abbfa..2303a31677a 100644 --- a/tensorflow/contrib/distribute/python/cross_tower_utils_test.py +++ b/tensorflow/contrib/distribute/python/cross_device_utils_test.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================== -"""Tests for cross_tower_utils.""" +"""Tests for cross_device_utils.""" from __future__ import absolute_import from __future__ import division @@ -21,14 +21,14 @@ from __future__ import print_function from absl.testing import parameterized from tensorflow.contrib.distribute.python import combinations -from tensorflow.contrib.distribute.python import cross_tower_utils -from tensorflow.contrib.distribute.python import values as value_lib +from tensorflow.python.distribute import cross_device_utils +from tensorflow.python.distribute import device_util +from tensorflow.python.distribute import values as value_lib from tensorflow.python.eager import test from tensorflow.python.framework import constant_op from tensorflow.python.framework import ops from tensorflow.python.framework import test_util from tensorflow.python.ops import math_ops -from tensorflow.python.training import device_util class IndexedSlicesUtilsTest(test.TestCase, parameterized.TestCase): @@ -43,7 +43,7 @@ class IndexedSlicesUtilsTest(test.TestCase, parameterized.TestCase): t0 = constant_op.constant([[1., 2.], [0, 0], [3., 4.]]) t1 = constant_op.constant([[0., 0.], [5, 6], [7., 8.]]) total = constant_op.constant([[1., 2.], [5, 6], [10., 12.]]) - result = cross_tower_utils.aggregate_tensors_or_indexed_slices([t0, t1]) + result = cross_device_utils.aggregate_tensors_or_indexed_slices([t0, t1]) self._assert_values_equal(total, result) @test_util.run_in_graph_and_eager_modes @@ -53,7 +53,7 @@ class IndexedSlicesUtilsTest(test.TestCase, parameterized.TestCase): t1 = math_ops._as_indexed_slices( constant_op.constant([[0., 0.], [5, 6], [7., 8.]])) total = constant_op.constant([[1., 2.], [5, 6], [10., 12.]]) - result = cross_tower_utils.aggregate_tensors_or_indexed_slices([t0, t1]) + result = cross_device_utils.aggregate_tensors_or_indexed_slices([t0, t1]) self.assertIsInstance(result, ops.IndexedSlices) self._assert_values_equal(total, result) @@ -62,7 +62,7 @@ class IndexedSlicesUtilsTest(test.TestCase, parameterized.TestCase): t = constant_op.constant([[1., 2.], [0, 0], [3., 4.]]) n = 2 expected = constant_op.constant([[0.5, 1.], [0, 0], [1.5, 2.]]) - result = cross_tower_utils.divide_by_n_tensors_or_indexed_slices(t, n) + result = cross_device_utils.divide_by_n_tensors_or_indexed_slices(t, n) self._assert_values_equal(expected, result) @test_util.run_in_graph_and_eager_modes @@ -71,7 +71,7 @@ class IndexedSlicesUtilsTest(test.TestCase, parameterized.TestCase): constant_op.constant([[1., 2.], [0, 0], [3., 4.]])) n = 2 expected = constant_op.constant([[0.5, 1.], [0, 0], [1.5, 2.]]) - result = cross_tower_utils.divide_by_n_tensors_or_indexed_slices(t, n) + result = cross_device_utils.divide_by_n_tensors_or_indexed_slices(t, n) self.assertIsInstance(result, ops.IndexedSlices) self._assert_values_equal(expected, result) @@ -79,7 +79,7 @@ class IndexedSlicesUtilsTest(test.TestCase, parameterized.TestCase): def testIsIndexedSlices(self): t = math_ops._as_indexed_slices( constant_op.constant([[1., 2.], [0, 0], [3., 4.]])) - self.assertTrue(cross_tower_utils.contains_indexed_slices(t)) + self.assertTrue(cross_device_utils.contains_indexed_slices(t)) @test_util.run_in_graph_and_eager_modes def testContainsIndexedSlices_List(self): @@ -87,7 +87,7 @@ class IndexedSlicesUtilsTest(test.TestCase, parameterized.TestCase): constant_op.constant([[1., 2.], [0, 0], [3., 4.]])) t1 = math_ops._as_indexed_slices( constant_op.constant([[0., 0.], [5, 6], [7., 8.]])) - self.assertTrue(cross_tower_utils.contains_indexed_slices([t0, t1])) + self.assertTrue(cross_device_utils.contains_indexed_slices([t0, t1])) @test_util.run_in_graph_and_eager_modes def testContainsIndexedSlices_Tuple(self): @@ -95,7 +95,7 @@ class IndexedSlicesUtilsTest(test.TestCase, parameterized.TestCase): constant_op.constant([[1., 2.], [0, 0], [3., 4.]])) t1 = math_ops._as_indexed_slices( constant_op.constant([[0., 0.], [5, 6], [7., 8.]])) - self.assertTrue(cross_tower_utils.contains_indexed_slices((t0, t1))) + self.assertTrue(cross_device_utils.contains_indexed_slices((t0, t1))) @test_util.run_in_graph_and_eager_modes def testContainsIndexedSlices_PerReplica(self): @@ -104,7 +104,7 @@ class IndexedSlicesUtilsTest(test.TestCase, parameterized.TestCase): t1 = math_ops._as_indexed_slices( constant_op.constant([[0., 0.], [5, 6], [7., 8.]])) per_replica = value_lib.PerReplica({"/gpu:0": t0, "/cpu:0": t1}) - self.assertTrue(cross_tower_utils.contains_indexed_slices(per_replica)) + self.assertTrue(cross_device_utils.contains_indexed_slices(per_replica)) @combinations.generate(combinations.combine( mode=["graph", "eager"], @@ -113,7 +113,7 @@ class IndexedSlicesUtilsTest(test.TestCase, parameterized.TestCase): with ops.device("/cpu:0"): t = constant_op.constant([[1., 2.], [0, 0], [3., 4.]]) destination = "/gpu:0" - result = cross_tower_utils.copy_tensor_or_indexed_slices_to_device( + result = cross_device_utils.copy_tensor_or_indexed_slices_to_device( t, destination) self._assert_values_equal(t, result) @@ -128,7 +128,7 @@ class IndexedSlicesUtilsTest(test.TestCase, parameterized.TestCase): t = math_ops._as_indexed_slices( constant_op.constant([[1., 2.], [0, 0], [3., 4.]])) destination = "/gpu:0" - result = cross_tower_utils.copy_tensor_or_indexed_slices_to_device( + result = cross_device_utils.copy_tensor_or_indexed_slices_to_device( t, destination) self.assertIsInstance(result, ops.IndexedSlices) diff --git a/tensorflow/contrib/distribute/python/estimator_integration_test.py b/tensorflow/contrib/distribute/python/estimator_integration_test.py index a1355c0b09e..e17085628ba 100644 --- a/tensorflow/contrib/distribute/python/estimator_integration_test.py +++ b/tensorflow/contrib/distribute/python/estimator_integration_test.py @@ -34,7 +34,7 @@ from tensorflow.python.estimator.canned import dnn_linear_combined from tensorflow.python.estimator.canned import prediction_keys from tensorflow.python.estimator.export import export from tensorflow.python.estimator.inputs import numpy_io -from tensorflow.python.feature_column import feature_column +from tensorflow.python.feature_column import feature_column_lib as feature_column from tensorflow.python.framework import ops from tensorflow.python.platform import gfile from tensorflow.python.summary.writer import writer_cache @@ -63,7 +63,9 @@ class DNNLinearCombinedClassifierIntegrationTest(test.TestCase, distribution=[ combinations.one_device_strategy, combinations.mirrored_strategy_with_gpu_and_cpu, - combinations.mirrored_strategy_with_two_gpus + combinations.mirrored_strategy_with_two_gpus, + combinations.core_mirrored_strategy_with_gpu_and_cpu, + combinations.core_mirrored_strategy_with_two_gpus ], use_train_and_evaluate=[True, False])) def test_complete_flow_with_mode(self, distribution, use_train_and_evaluate): @@ -75,12 +77,12 @@ class DNNLinearCombinedClassifierIntegrationTest(test.TestCase, train_input_fn = self.dataset_input_fn( x={'x': data}, y=data, - batch_size=batch_size // len(distribution.worker_devices), + batch_size=batch_size // distribution.num_replicas_in_sync, shuffle=True) eval_input_fn = self.dataset_input_fn( x={'x': data}, y=data, - batch_size=batch_size // len(distribution.worker_devices), + batch_size=batch_size // distribution.num_replicas_in_sync, shuffle=False) predict_input_fn = numpy_io.numpy_input_fn( x={'x': data}, batch_size=batch_size, shuffle=False) diff --git a/tensorflow/contrib/distribute/python/estimator_training_test.py b/tensorflow/contrib/distribute/python/estimator_training_test.py index 8f82b4c92aa..b369a7fefe6 100644 --- a/tensorflow/contrib/distribute/python/estimator_training_test.py +++ b/tensorflow/contrib/distribute/python/estimator_training_test.py @@ -24,7 +24,6 @@ import json import os import sys import tempfile -import threading from absl.testing import parameterized import numpy as np @@ -45,11 +44,13 @@ from tensorflow.python.estimator import training as estimator_training from tensorflow.python.estimator.canned import dnn_linear_combined from tensorflow.python.estimator.canned import prediction_keys from tensorflow.python.estimator.export import export as export_lib -from tensorflow.python.feature_column import feature_column +from tensorflow.python.feature_column import feature_column_lib as feature_column from tensorflow.python.platform import gfile from tensorflow.python.platform import test from tensorflow.python.summary import summary_iterator from tensorflow.python.summary.writer import writer_cache +from tensorflow.python.training import session_manager + BATCH_SIZE = 10 LABEL_DIMENSION = 2 @@ -68,57 +69,19 @@ PS = dc._TaskType.PS original_run_std_server = dc._run_std_server -class MockOsEnv(dict): - - def __init__(self, *args): - self._thread_local = threading.local() - super(MockOsEnv, self).__init__(*args) - - def get(self, key, default): - if not hasattr(self._thread_local, "dict"): - self._thread_local.dict = dict() - if key == "TF_CONFIG": - return dict.get(self._thread_local.dict, key, default) - else: - return dict.get(self, key, default) - - def __getitem__(self, key): - if not hasattr(self._thread_local, "dict"): - self._thread_local.dict = dict() - if key == "TF_CONFIG": - return dict.__getitem__(self._thread_local.dict, key) - else: - return dict.__getitem__(self, key) - - def __setitem__(self, key, val): - if not hasattr(self._thread_local, "dict"): - self._thread_local.dict = dict() - if key == "TF_CONFIG": - return dict.__setitem__(self._thread_local.dict, key, val) - else: - return dict.__setitem__(self, key, val) - - -class DistributeCoordinatorIntegrationTest(test.TestCase, - parameterized.TestCase): +class DistributeCoordinatorIntegrationTest( + multi_worker_test_base.IndependentWorkerTestBase, parameterized.TestCase): @classmethod def setUpClass(cls): """Create a local cluster with 2 workers.""" + super(DistributeCoordinatorIntegrationTest, cls).setUpClass() cls._cluster_spec = multi_worker_test_base.create_in_process_cluster( num_workers=3, num_ps=2, has_eval=True) def setUp(self): self._model_dir = tempfile.mkdtemp() - self._mock_os_env = MockOsEnv() - self._mock_context = test.mock.patch.object(os, "environ", - self._mock_os_env) super(DistributeCoordinatorIntegrationTest, self).setUp() - self._mock_context.__enter__() - - def tearDown(self): - self._mock_context.__exit__(None, None, None) - super(DistributeCoordinatorIntegrationTest, self).tearDown() def dataset_input_fn(self, x, y, batch_size, shuffle): @@ -141,8 +104,8 @@ class DistributeCoordinatorIntegrationTest(test.TestCase, def _extract_loss_and_global_step(self, event_folder): """Returns the loss and global step in last event.""" event_paths = glob.glob(os.path.join(event_folder, "events*")) - self.assertGreater(len(event_paths), 0, - msg="Event file not found in dir %s" % event_folder) + self.assertNotEmpty( + event_paths, msg="Event file not found in dir %s" % event_folder) loss = None global_step_count = None @@ -202,10 +165,10 @@ class DistributeCoordinatorIntegrationTest(test.TestCase, train_input_fn = self.dataset_input_fn( x={"x": DATA}, y=DATA, - batch_size=BATCH_SIZE // len(train_distribute.worker_devices), + batch_size=BATCH_SIZE // train_distribute.num_replicas_in_sync, shuffle=True) if eval_distribute: - eval_batch_size = BATCH_SIZE // len(eval_distribute.worker_devices) + eval_batch_size = BATCH_SIZE // eval_distribute.num_replicas_in_sync else: eval_batch_size = BATCH_SIZE eval_input_fn = self.dataset_input_fn( @@ -285,27 +248,34 @@ class DistributeCoordinatorIntegrationTest(test.TestCase, ]) self.assertAllEqual((BATCH_SIZE, LABEL_DIMENSION), predicted_proba.shape) + def _get_strategy_object(self, strategy_cls): + if strategy_cls == mirrored_strategy.CoreMirroredStrategy: + return strategy_cls(mirrored_strategy.all_local_devices()) + else: + return strategy_cls(num_gpus_per_worker=context.num_gpus()) + @combinations.generate( combinations.combine( mode=["graph"], train_distribute_cls=[ collective_all_reduce_strategy.CollectiveAllReduceStrategy, mirrored_strategy.MirroredStrategy, + mirrored_strategy.CoreMirroredStrategy, parameter_server_strategy.ParameterServerStrategy ], eval_distribute_cls=[ - None, mirrored_strategy.MirroredStrategy, + None, + mirrored_strategy.MirroredStrategy, + mirrored_strategy.CoreMirroredStrategy, parameter_server_strategy.ParameterServerStrategy, ], required_gpus=[0, 1])) def test_complete_flow_standalone_client(self, train_distribute_cls, eval_distribute_cls): - train_distribute = train_distribute_cls( - num_gpus_per_worker=context.num_gpus()) + train_distribute = self._get_strategy_object(train_distribute_cls) if eval_distribute_cls: - eval_distribute = eval_distribute_cls( - num_gpus_per_worker=context.num_gpus()) + eval_distribute = self._get_strategy_object(eval_distribute_cls) else: eval_distribute = None @@ -322,20 +292,20 @@ class DistributeCoordinatorIntegrationTest(test.TestCase, mode=["graph"], train_distribute_cls=[ mirrored_strategy.MirroredStrategy, + mirrored_strategy.CoreMirroredStrategy, ], eval_distribute_cls=[ None, mirrored_strategy.MirroredStrategy, + mirrored_strategy.CoreMirroredStrategy, ], required_gpus=[0, 1])) def test_estimator_standalone_client(self, train_distribute_cls, eval_distribute_cls): - train_distribute = train_distribute_cls( - num_gpus_per_worker=context.num_gpus()) + train_distribute = self._get_strategy_object(train_distribute_cls) if eval_distribute_cls: - eval_distribute = eval_distribute_cls( - num_gpus_per_worker=context.num_gpus()) + eval_distribute = self._get_strategy_object(eval_distribute_cls) else: eval_distribute = None @@ -355,47 +325,15 @@ class DistributeCoordinatorIntegrationTest(test.TestCase, self._barrier.wait() return ret - def _task_thread(self, train_distribute, eval_distribute, tf_config): - os.environ["TF_CONFIG"] = json.dumps(tf_config) + def _independent_worker_fn( + self, + train_distribute, + eval_distribute, + ): with test.mock.patch.object(dc, "_run_std_server", self._mock_run_std_server): self._complete_flow(train_distribute, eval_distribute) - def _run_task_in_thread(self, cluster_spec, task_type, task_id, - train_distribute, eval_distribute): - if task_type: - tf_config = { - "cluster": cluster_spec, - "task": { - "type": task_type, - "index": task_id - } - } - else: - tf_config = { - "cluster": cluster_spec, - "task": { - "type": task_type, - "index": task_id - } - } - t = threading.Thread( - target=self._task_thread, - args=(train_distribute, eval_distribute, tf_config)) - t.start() - return t - - def _run_multiple_tasks_in_threads(self, cluster_spec, train_distribute, - eval_distribute): - threads = {} - for task_type in cluster_spec.keys(): - threads[task_type] = [] - for task_id in range(len(cluster_spec[task_type])): - t = self._run_task_in_thread(cluster_spec, task_type, task_id, - train_distribute, eval_distribute) - threads[task_type].append(t) - return threads - @combinations.generate( combinations.combine( mode=["graph"], @@ -405,21 +343,20 @@ class DistributeCoordinatorIntegrationTest(test.TestCase, ], eval_distribute_cls=[ None, mirrored_strategy.MirroredStrategy, + mirrored_strategy.CoreMirroredStrategy, parameter_server_strategy.ParameterServerStrategy, ], required_gpus=[0, 1])) def test_complete_flow_indepedent_worker_between_graph( self, train_distribute_cls, eval_distribute_cls): - train_distribute = train_distribute_cls( - num_gpus_per_worker=context.num_gpus()) - if (context.num_gpus() < 2 and eval_distribute_cls == collective_all_reduce_strategy.CollectiveAllReduceStrategy): self.skipTest("`CollectiveAllReduceStrategy` needs at least two towers.") + train_distribute = self._get_strategy_object(train_distribute_cls) + if eval_distribute_cls: - eval_distribute = eval_distribute_cls( - num_gpus_per_worker=context.num_gpus()) + eval_distribute = self._get_strategy_object(eval_distribute_cls) else: eval_distribute = None @@ -435,8 +372,9 @@ class DistributeCoordinatorIntegrationTest(test.TestCase, # 3 workers and 1 evaluator. self._barrier = dc._Barrier(4) - threads = self._run_multiple_tasks_in_threads( - cluster_spec, train_distribute, eval_distribute) + threads = self.run_multiple_tasks_in_threads(self._independent_worker_fn, + cluster_spec, train_distribute, + eval_distribute) for task_type, ts in threads.items(): if task_type == PS: continue @@ -449,17 +387,22 @@ class DistributeCoordinatorIntegrationTest(test.TestCase, @combinations.generate( combinations.combine( mode=["graph"], - train_distribute_cls=[mirrored_strategy.MirroredStrategy], - eval_distribute_cls=[None, mirrored_strategy.MirroredStrategy], + train_distribute_cls=[ + mirrored_strategy.MirroredStrategy, + mirrored_strategy.CoreMirroredStrategy + ], + eval_distribute_cls=[ + None, + mirrored_strategy.MirroredStrategy, + mirrored_strategy.CoreMirroredStrategy + ], required_gpus=[0, 1])) def test_complete_flow_indepedent_worker_in_graph(self, train_distribute_cls, eval_distribute_cls): - train_distribute = train_distribute_cls( - num_gpus_per_worker=context.num_gpus()) + train_distribute = self._get_strategy_object(train_distribute_cls) if eval_distribute_cls: - eval_distribute = eval_distribute_cls( - num_gpus_per_worker=context.num_gpus()) + eval_distribute = self._get_strategy_object(eval_distribute_cls) else: eval_distribute = None @@ -467,8 +410,9 @@ class DistributeCoordinatorIntegrationTest(test.TestCase, num_workers=3, num_ps=0, has_eval=True) # 3 workers and 1 evaluator. self._barrier = dc._Barrier(4) - threads = self._run_multiple_tasks_in_threads( - cluster_spec, train_distribute, eval_distribute) + threads = self.run_multiple_tasks_in_threads(self._independent_worker_fn, + cluster_spec, train_distribute, + eval_distribute) threads[WORKER][0].join() threads[EVALUATOR][0].join() @@ -506,7 +450,8 @@ class RunConfigTest(test.TestCase): "os.environ", {"TF_CONFIG": json.dumps(TF_CONFIG_WITHOUT_TASK)}): run_config_lib.RunConfig( experimental_distribute=DistributeConfig( - train_distribute=mirrored_strategy.MirroredStrategy(num_gpus=2))) + train_distribute=mirrored_strategy.CoreMirroredStrategy( + ["/device:GPU:0", "/device:GPU:1"]))) def test_should_run_distribute_coordinator(self): """Tests that should_run_distribute_coordinator return a correct value.""" @@ -529,10 +474,12 @@ class RunConfigTest(test.TestCase): {"TF_CONFIG": json.dumps(TF_CONFIG_WITH_CHIEF)}): config_with_train_distribute = run_config_lib.RunConfig( experimental_distribute=DistributeConfig( - train_distribute=mirrored_strategy.MirroredStrategy(num_gpus=2))) + train_distribute=mirrored_strategy.CoreMirroredStrategy( + ["/device:GPU:0", "/device:GPU:1"]))) config_with_eval_distribute = run_config_lib.RunConfig( experimental_distribute=DistributeConfig( - eval_distribute=mirrored_strategy.MirroredStrategy(num_gpus=2))) + eval_distribute=mirrored_strategy.CoreMirroredStrategy( + ["/device:GPU:0", "/device:GPU:1"]))) self.assertTrue( dc_training.should_run_distribute_coordinator( config_with_train_distribute)) @@ -545,26 +492,27 @@ class RunConfigTest(test.TestCase): {"TF_CONFIG": json.dumps(TF_CONFIG_WITH_MASTER)}): config = run_config_lib.RunConfig( experimental_distribute=DistributeConfig( - train_distribute=mirrored_strategy.MirroredStrategy(num_gpus=2))) + train_distribute=mirrored_strategy.CoreMirroredStrategy( + ["/device:GPU:0", "/device:GPU:1"]))) self.assertFalse(dc_training.should_run_distribute_coordinator(config)) def test_init_run_config_duplicate_distribute(self): with self.assertRaises(ValueError): run_config_lib.RunConfig( - train_distribute=mirrored_strategy.MirroredStrategy(), + train_distribute=mirrored_strategy.CoreMirroredStrategy(), experimental_distribute=DistributeConfig( - train_distribute=mirrored_strategy.MirroredStrategy())) + train_distribute=mirrored_strategy.CoreMirroredStrategy())) with self.assertRaises(ValueError): run_config_lib.RunConfig( - eval_distribute=mirrored_strategy.MirroredStrategy(), + eval_distribute=mirrored_strategy.CoreMirroredStrategy(), experimental_distribute=DistributeConfig( - eval_distribute=mirrored_strategy.MirroredStrategy())) + eval_distribute=mirrored_strategy.CoreMirroredStrategy())) def test_init_run_config_none_distribute_coordinator_mode(self): # We don't use distribute coordinator for local training. config = run_config_lib.RunConfig( - train_distribute=mirrored_strategy.MirroredStrategy()) + train_distribute=mirrored_strategy.CoreMirroredStrategy()) dc_training.init_run_config(config, {}) self.assertIsNone(config._distribute_coordinator_mode) @@ -572,7 +520,7 @@ class RunConfigTest(test.TestCase): with test.mock.patch.dict("os.environ", {"TF_CONFIG": json.dumps(TF_CONFIG_WITH_MASTER)}): config = run_config_lib.RunConfig( - train_distribute=mirrored_strategy.MirroredStrategy()) + train_distribute=mirrored_strategy.CoreMirroredStrategy()) self.assertIsNone(config._distribute_coordinator_mode) # When `train_distribute` is not specified, don't use distribute @@ -588,7 +536,7 @@ class RunConfigTest(test.TestCase): with test.mock.patch.dict("os.environ", {"TF_CONFIG": json.dumps(TF_CONFIG_WITH_CHIEF)}): config = run_config_lib.RunConfig( - train_distribute=mirrored_strategy.MirroredStrategy()) + train_distribute=mirrored_strategy.CoreMirroredStrategy()) self.assertEqual(config._distribute_coordinator_mode, dc.CoordinatorMode.INDEPENDENT_WORKER) @@ -597,7 +545,7 @@ class RunConfigTest(test.TestCase): # `experimental.remote_cluster` is set use distribute coordinator with # STANDALONE_CLIENT mode. config = run_config_lib.RunConfig( - train_distribute=mirrored_strategy.MirroredStrategy(), + train_distribute=mirrored_strategy.CoreMirroredStrategy(), experimental_distribute=DistributeConfig( remote_cluster={"chief": ["fake_worker"]})) self.assertEqual(config._distribute_coordinator_mode, @@ -605,5 +553,15 @@ class RunConfigTest(test.TestCase): if __name__ == "__main__": + # Reduce `recovery_wait_secs` from 30 seconds so the test completes quickly. + orig_init = session_manager.SessionManager.__init__ + + def new_init(*args, **kwargs): + kwargs.pop("recovery_wait_secs", None) + kwargs["recovery_wait_secs"] = 0.5 + orig_init(*args, **kwargs) + + session_manager.SessionManager.__init__ = new_init + with test.mock.patch.object(sys, "exit", os._exit): test.main() diff --git a/tensorflow/contrib/distribute/python/examples/keras_mnist.py b/tensorflow/contrib/distribute/python/examples/keras_mnist.py index 0fd3acd0451..60fda996642 100644 --- a/tensorflow/contrib/distribute/python/examples/keras_mnist.py +++ b/tensorflow/contrib/distribute/python/examples/keras_mnist.py @@ -20,6 +20,10 @@ from __future__ import print_function import tensorflow as tf +from tensorflow.python.distribute import mirrored_strategy +from tensorflow.python.keras.optimizer_v2 import rmsprop + + NUM_CLASSES = 10 @@ -102,18 +106,23 @@ def main(_): # Build the train and eval datasets from the MNIST data. Also return the # input shape which is constructed based on the `image_data_format` # i.e channels_first or channels_last. + tf.enable_eager_execution() + train_ds, eval_ds, input_shape = get_input_datasets() model = get_model(input_shape) # Instantiate the MirroredStrategy object. If we don't specify `num_gpus` or # the `devices` argument then all the GPUs available on the machine are used. - strategy = tf.contrib.distribute.MirroredStrategy() + # TODO(priyag): Use `tf.distribute.MirroredStrategy` once available. + strategy = mirrored_strategy.MirroredStrategy(['/gpu:0', '/cpu:0']) + + optimizer = rmsprop.RMSProp(learning_rate=0.001) # Compile the model by passing the distribution strategy object to the # `distribute` argument. `fit`, `evaluate` and `predict` will be distributed # based on the strategy instantiated. model.compile(loss=tf.keras.losses.categorical_crossentropy, - optimizer=tf.train.RMSPropOptimizer(learning_rate=0.001), + optimizer=optimizer, metrics=['accuracy'], distribute=strategy) diff --git a/tensorflow/contrib/distribute/python/keras_optimizer_v2_test.py b/tensorflow/contrib/distribute/python/keras_optimizer_v2_test.py index 46a1cf41c55..6dfd85bcc4f 100644 --- a/tensorflow/contrib/distribute/python/keras_optimizer_v2_test.py +++ b/tensorflow/contrib/distribute/python/keras_optimizer_v2_test.py @@ -25,18 +25,23 @@ import numpy as np import six from tensorflow.contrib.distribute.python import combinations -from tensorflow.contrib.distribute.python import mirrored_strategy from tensorflow.core.protobuf import config_pb2 +from tensorflow.python import keras from tensorflow.python.data.ops import dataset_ops -from tensorflow.python.eager import context +from tensorflow.python.distribute import distribution_strategy_context as ds_context from tensorflow.python.estimator import run_config from tensorflow.python.estimator import training from tensorflow.python.estimator.canned import dnn_linear_combined from tensorflow.python.estimator.canned import prediction_keys from tensorflow.python.estimator.export import export from tensorflow.python.estimator.inputs import numpy_io -from tensorflow.python.feature_column import feature_column +from tensorflow.python.feature_column import feature_column_lib as feature_column +from tensorflow.python.framework import constant_op +from tensorflow.python.framework import dtypes +from tensorflow.python.framework import ops from tensorflow.python.keras.optimizer_v2 import adam +from tensorflow.python.keras.optimizer_v2 import gradient_descent +from tensorflow.python.ops import math_ops from tensorflow.python.ops import variable_scope from tensorflow.python.ops import variables from tensorflow.python.platform import gfile @@ -64,7 +69,9 @@ class KerasOptimizerV2IntegrationTest(test.TestCase, parameterized.TestCase): distribution=[ combinations.one_device_strategy, combinations.mirrored_strategy_with_gpu_and_cpu, - combinations.mirrored_strategy_with_two_gpus + combinations.mirrored_strategy_with_two_gpus, + combinations.core_mirrored_strategy_with_gpu_and_cpu, + combinations.core_mirrored_strategy_with_two_gpus ], use_train_and_evaluate=[True, False])) def test_complete_flow_with_mode(self, distribution, use_train_and_evaluate): @@ -76,11 +83,11 @@ class KerasOptimizerV2IntegrationTest(test.TestCase, parameterized.TestCase): train_input_fn = self.dataset_input_fn( x={'x': data}, y=data, - batch_size=batch_size // len(distribution.worker_devices)) + batch_size=batch_size // distribution.num_replicas_in_sync) eval_input_fn = self.dataset_input_fn( x={'x': data}, y=data, - batch_size=batch_size // len(distribution.worker_devices)) + batch_size=batch_size // distribution.num_replicas_in_sync) predict_input_fn = numpy_io.numpy_input_fn( x={'x': data}, batch_size=batch_size, shuffle=False) @@ -136,44 +143,51 @@ class KerasOptimizerV2IntegrationTest(test.TestCase, parameterized.TestCase): shutil.rmtree(self._model_dir) -class MirroredStrategyOptimizerV2Test(test.TestCase): +def get_model(): + x = keras.layers.Input(shape=(3,), name='input') + y = keras.layers.Dense(4, name='dense')(x) + model = keras.Model(x, y) + return model - def testKerasOptimizerWithUnequalInput(self): - if context.num_gpus() < 1: - self.skipTest('Not enough GPUs.') - def create_fn(device_id): +class MirroredStrategyOptimizerV2Test(test.TestCase, parameterized.TestCase): + + @combinations.generate(combinations.combine( + distribution=[ + combinations.mirrored_strategy_with_gpu_and_cpu, + combinations.core_mirrored_strategy_with_gpu_and_cpu], + mode=['graph'])) + def testKerasOptimizerWithUnequalInput(self, distribution): + def create_fn(): var = variables.Variable( 2.0, name='var', aggregation=variable_scope.VariableAggregation.SUM) # grad for cpu is 1, grad for gpu is 2, avg grad is 1.5. - loss = (device_id + 1) * var + loss = math_ops.cast(_replica_id() + 1, dtype=dtypes.float32) * var optimizer = adam.Adam(learning_rate=0.01, beta_1=0.2, beta_2=0.2) train_op = optimizer.minimize(loss, var_list=[var]) m = optimizer.get_slot(var, 'm') v = optimizer.get_slot(var, 'v') - return (var, m, v, train_op, optimizer.iteration) + return (var, m, v, train_op, optimizer.iterations) devices = ['/device:GPU:0', '/device:CPU:0'] - dist = mirrored_strategy.MirroredStrategy(devices) - with dist.scope(): - (var, m, v, op, counter) = dist.call_for_each_replica( - create_fn, args=[dist.worker_device_index]) + with distribution.scope(): + (var, m, v, op, counter) = distribution.call_for_each_replica(create_fn) self.evaluate(variables.global_variables_initializer()) var_val = [2.0, 2.0, 2.0] self.assertAllClose( var_val, self.evaluate( - [dist.read_var(var), + [distribution.read_var(var), var.get(devices[0]), var.get(devices[1])])) self.assertAllClose([0, 0, 0], self.evaluate([ - dist.read_var(counter), + distribution.read_var(counter), counter.get(devices[0]), counter.get(devices[1]) ])) - train_op = dist.unwrap(op) + train_op = distribution.unwrap(op) self.evaluate(train_op) # m(1) = beta1 * m(0) + (1-beta1) * grad = 0.2 * 0 + 0.8 * (1 + 2) / 2 m_val = [1.2, 1.2, 1.2] @@ -181,7 +195,7 @@ class MirroredStrategyOptimizerV2Test(test.TestCase): self.assertAllClose( m_val, self.evaluate( - [dist.read_var(m), + [distribution.read_var(m), m.get(devices[0]), m.get(devices[1])])) # v(1) = beta2 * v(0) + (1-beta2) * grad^2 = 0.2 * 0 + 0.8 * 2.25 @@ -189,7 +203,7 @@ class MirroredStrategyOptimizerV2Test(test.TestCase): self.assertAllClose( v_val, self.evaluate( - [dist.read_var(v), + [distribution.read_var(v), v.get(devices[0]), v.get(devices[1])])) # var(1) = var(0) - lr * m(1) * sqrt(1 - beta2) / sqrt(v(1)) / (1 - beta1) @@ -198,12 +212,12 @@ class MirroredStrategyOptimizerV2Test(test.TestCase): self.assertAllClose( var_val, self.evaluate( - [dist.read_var(var), + [distribution.read_var(var), var.get(devices[0]), var.get(devices[1])])) self.assertAllClose([1, 1, 1], self.evaluate([ - dist.read_var(counter), + distribution.read_var(counter), counter.get(devices[0]), counter.get(devices[1]) ])) @@ -214,7 +228,7 @@ class MirroredStrategyOptimizerV2Test(test.TestCase): self.assertAllClose( m_val, self.evaluate( - [dist.read_var(m), + [distribution.read_var(m), m.get(devices[0]), m.get(devices[1])])) # v(2) = beta2 * v(1) + (1-beta2) * grad^2 = 0.2 * 1.8 + 0.8 * 2.25 @@ -222,16 +236,50 @@ class MirroredStrategyOptimizerV2Test(test.TestCase): self.assertAllClose( v_val, self.evaluate( - [dist.read_var(v), + [distribution.read_var(v), v.get(devices[0]), v.get(devices[1])])) self.assertAllClose([2, 2, 2], self.evaluate([ - dist.read_var(counter), + distribution.read_var(counter), counter.get(devices[0]), counter.get(devices[1]) ])) + @combinations.generate(combinations.combine( + distribution=[ + combinations.mirrored_strategy_with_gpu_and_cpu, + combinations.core_mirrored_strategy_with_gpu_and_cpu], + mode=['graph'])) + def testOptimizerWithKerasModelAndNumpyArrays(self, distribution): + + with self.cached_session(): + model = get_model() + optimizer = gradient_descent.SGD(0.001) + loss = 'mse' + metrics = ['mae'] + model.compile(optimizer, loss, metrics=metrics, distribute=distribution) + + inputs = np.zeros((64, 3), dtype=np.float32) + targets = np.zeros((64, 4), dtype=np.float32) + + model.fit( + inputs, + targets, + epochs=1, + batch_size=2, + verbose=0, + validation_data=(inputs, targets)) + model.evaluate(inputs, targets) + model.predict(inputs) + + +def _replica_id(): + replica_id = ds_context.get_replica_context().replica_id_in_sync_group + if not isinstance(replica_id, ops.Tensor): + replica_id = constant_op.constant(replica_id) + return replica_id + if __name__ == '__main__': test.main() diff --git a/tensorflow/contrib/distribute/python/keras_test.py b/tensorflow/contrib/distribute/python/keras_test.py index 0db5844e4c4..e530ab6f173 100644 --- a/tensorflow/contrib/distribute/python/keras_test.py +++ b/tensorflow/contrib/distribute/python/keras_test.py @@ -24,9 +24,10 @@ import numpy as np from tensorflow.contrib.distribute.python import combinations from tensorflow.contrib.distribute.python import mirrored_strategy from tensorflow.contrib.distribute.python import tpu_strategy -from tensorflow.contrib.distribute.python import values from tensorflow.python import keras from tensorflow.python.data.ops import dataset_ops +from tensorflow.python.distribute import values +from tensorflow.python.eager import test from tensorflow.python.estimator import keras as keras_lib from tensorflow.python.estimator import run_config as run_config_lib from tensorflow.python.framework import constant_op @@ -35,14 +36,13 @@ from tensorflow.python.framework import random_seed from tensorflow.python.framework import test_util from tensorflow.python.keras import testing_utils from tensorflow.python.keras.engine import distributed_training_utils +from tensorflow.python.keras.optimizer_v2 import gradient_descent as gradient_descent_keras from tensorflow.python.ops.parsing_ops import gen_parsing_ops from tensorflow.python.platform import gfile -from tensorflow.python.platform import test from tensorflow.python.summary.writer import writer_cache from tensorflow.python.training import gradient_descent from tensorflow.python.training import rmsprop - _RANDOM_SEED = 1337 _TRAIN_SIZE = 200 _INPUT_SIZE = (10,) @@ -212,13 +212,18 @@ def multi_input_output_model(): return model -def get_correctness_test_inputs(use_numpy, with_distribution, +def get_correctness_test_inputs(use_numpy, use_validation_data, + with_distribution, x_train, y_train, x_predict): """Generates the inputs for correctness check when enable Keras with DS.""" global_batch_size = 64 batch_size = global_batch_size # TODO(b/118776054): Use global batch size for Keras/DS support. - if with_distribution: + use_per_core_batch_size = ( + with_distribution and + not distributed_training_utils.global_batch_size_supported( + with_distribution)) + if use_per_core_batch_size: batch_size //= with_distribution.num_replicas_in_sync if use_numpy: @@ -229,16 +234,17 @@ def get_correctness_test_inputs(use_numpy, with_distribution, 'epochs': 1, 'shuffle': False, } - eval_inputs = { - 'batch_size': batch_size, - 'x': x_train, - 'y': y_train, - } + + if use_validation_data: + eval_inputs = None + training_inputs['validation_data'] = (x_train, y_train) + else: + eval_inputs = { + 'batch_size': batch_size, + 'x': x_train, + 'y': y_train, + } predict_inputs = { - # TODO(b/119318587): We should not require batch_size when distribution - # is enabled. - 'batch_size': (len(x_predict) // with_distribution.num_replicas_in_sync - if with_distribution else None), 'x': np.array(x_predict, dtype=np.float32), } else: @@ -256,20 +262,28 @@ def get_correctness_test_inputs(use_numpy, with_distribution, 'shuffle': False, 'steps_per_epoch': len(x_train) // global_batch_size, } - eval_inputs = { - 'batch_size': None, - 'x': x, - 'y': None, - 'steps': 20, - } + if use_validation_data: + eval_inputs = None # Remove the eval_inputs + eval_dataset = dataset_ops.Dataset.from_tensor_slices( + (x_train, y_train)) + x = batch_wrapper(eval_dataset, batch_size, with_distribution) + training_inputs['validation_data'] = x + training_inputs['validation_steps'] = 5 + else: + eval_inputs = { + 'batch_size': None, + 'x': x, + 'y': None, + 'steps': 20, + } + predict_batch_size = len(x_predict) - if with_distribution: + if use_per_core_batch_size: predict_batch_size //= with_distribution.num_replicas_in_sync predict_dataset = dataset_ops.Dataset.from_tensor_slices(x_predict) predict_dataset = batch_wrapper(predict_dataset, predict_batch_size, with_distribution) predict_inputs = { - 'batch_size': None, 'steps': 1, 'x': predict_dataset, } @@ -277,47 +291,71 @@ def get_correctness_test_inputs(use_numpy, with_distribution, return training_inputs, eval_inputs, predict_inputs -strategies = [combinations.default_strategy, - combinations.one_device_strategy, - combinations.mirrored_strategy_with_gpu_and_cpu, - combinations.mirrored_strategy_with_two_gpus, - combinations.tpu_strategy, # steps_per_run=2 - combinations.tpu_strategy_one_step] +strategies_minus_tpu = [ + combinations.default_strategy, + combinations.one_device_strategy, + combinations.mirrored_strategy_with_gpu_and_cpu, + combinations.mirrored_strategy_with_two_gpus, + combinations.core_mirrored_strategy_with_gpu_and_cpu, + combinations.core_mirrored_strategy_with_two_gpus] + +tpu_strategies = [ + combinations.tpu_strategy, # steps_per_run=2 + combinations.tpu_strategy_one_step] def strategy_minus_tpu_combinations(): return combinations.combine( - distribution=[combinations.default_strategy, - combinations.one_device_strategy, - combinations.mirrored_strategy_with_gpu_and_cpu, - combinations.mirrored_strategy_with_two_gpus], - mode=['graph']) + distribution=strategies_minus_tpu, + mode=['graph', 'eager']) -def strategy_combinations(): +def tpu_strategy_combinations(): return combinations.combine( - distribution=strategies, + distribution=tpu_strategies, mode=['graph']) +def all_strategy_combinations(): + return strategy_minus_tpu_combinations() + tpu_strategy_combinations() + + +# TODO(priyag): Add v2 optimizers here. def strategy_and_optimizer_combinations(): + return combinations.times( + all_strategy_combinations(), + combinations.combine( + optimizer=[combinations.adagrad_optimizer_v1_fn, + combinations.adam_optimizer_v1_fn, + combinations.gradient_descent_optimizer_v1_fn, + combinations.rmsprop_optimizer_v1_fn])) + + +def strategy_and_input_combinations(): + return ( + combinations.times( + combinations.combine(distribution=strategies_minus_tpu), + combinations.combine(mode=['graph'], + use_numpy=[True, False], + use_validation_data=[True, False]) + + combinations.combine(mode=['eager'], + use_numpy=[False], + use_validation_data=[False])) + + combinations.times( + combinations.combine(distribution=tpu_strategies), + combinations.combine(mode=['graph'], + use_numpy=[True, False], + use_validation_data=[True, False]))) + + +def strategy_for_numpy_input_combinations(): return combinations.combine( - distribution=strategies, - optimizer=[combinations.adagrad_optimizer_v1_fn, - combinations.adam_optimizer_v1_fn, - combinations.gradient_descent_optimizer_v1_fn, - combinations.rmsprop_optimizer_v1_fn], + distribution=strategies_minus_tpu + tpu_strategies, mode=['graph']) -def strategy_and_inputs(): - return combinations.combine( - distribution=strategies, - use_numpy=[True, False], - mode=['graph']) - - -class TestEstimatorDistributionStrategy(test_util.TensorFlowTestCase): +class TestEstimatorDistributionStrategy(test_util.TensorFlowTestCase, + parameterized.TestCase): def setUp(self): self._base_dir = os.path.join(self.get_temp_dir(), @@ -325,17 +363,18 @@ class TestEstimatorDistributionStrategy(test_util.TensorFlowTestCase): gfile.MakeDirs(self._base_dir) self._config = run_config_lib.RunConfig( tf_random_seed=_RANDOM_SEED, model_dir=self._base_dir) - self._dist = mirrored_strategy.MirroredStrategy( - devices=['/device:GPU:0', '/device:GPU:1']) def tearDown(self): writer_cache.FileWriterCache.clear() if os.path.isdir(self._base_dir): gfile.DeleteRecursively(self._base_dir) - def test_train_functional_with_distribution_strategy(self): - dist = mirrored_strategy.MirroredStrategy( - devices=['/device:GPU:0', '/device:GPU:1']) + @combinations.generate(combinations.combine( + distribution=[ + combinations.mirrored_strategy_with_two_gpus, + combinations.core_mirrored_strategy_with_two_gpus], + mode=['graph'])) + def test_train_functional_with_distribution_strategy(self, distribution): keras_model = simple_functional_model() keras_model.compile( loss='categorical_crossentropy', @@ -343,8 +382,8 @@ class TestEstimatorDistributionStrategy(test_util.TensorFlowTestCase): optimizer=rmsprop.RMSPropOptimizer(learning_rate=0.01)) config = run_config_lib.RunConfig(tf_random_seed=_RANDOM_SEED, model_dir=self._base_dir, - train_distribute=dist, - eval_distribute=dist) + train_distribute=distribution, + eval_distribute=distribution) with self.cached_session(): est_keras = keras_lib.model_to_estimator( keras_model=keras_model, config=config) @@ -358,9 +397,12 @@ class TestEstimatorDistributionStrategy(test_util.TensorFlowTestCase): writer_cache.FileWriterCache.clear() gfile.DeleteRecursively(self._config.model_dir) - def test_train_sequential_with_distribution_strategy(self): - dist = mirrored_strategy.MirroredStrategy( - devices=['/device:GPU:0', '/device:GPU:1']) + @combinations.generate(combinations.combine( + distribution=[ + combinations.mirrored_strategy_with_two_gpus, + combinations.core_mirrored_strategy_with_two_gpus], + mode=['graph'])) + def test_train_sequential_with_distribution_strategy(self, distribution): keras_model = simple_sequential_model() keras_model.compile( loss='categorical_crossentropy', @@ -368,7 +410,7 @@ class TestEstimatorDistributionStrategy(test_util.TensorFlowTestCase): optimizer=rmsprop.RMSPropOptimizer(learning_rate=0.01)) config = run_config_lib.RunConfig(tf_random_seed=_RANDOM_SEED, model_dir=self._base_dir, - train_distribute=dist) + train_distribute=distribution) with self.cached_session(): est_keras = keras_lib.model_to_estimator( keras_model=keras_model, config=config) @@ -382,7 +424,12 @@ class TestEstimatorDistributionStrategy(test_util.TensorFlowTestCase): writer_cache.FileWriterCache.clear() gfile.DeleteRecursively(self._config.model_dir) - def test_multi_inputs_multi_outputs_with_input_fn_as_dict(self): + @combinations.generate(combinations.combine( + distribution=[ + combinations.mirrored_strategy_with_two_gpus, + combinations.core_mirrored_strategy_with_two_gpus], + mode=['graph'])) + def test_multi_inputs_multi_outputs_with_input_fn_as_dict(self, distribution): train_data, test_data = get_multi_inputs_multi_outputs_data() def train_input_fn(): @@ -412,14 +459,14 @@ class TestEstimatorDistributionStrategy(test_util.TensorFlowTestCase): output_dict)).batch(16) self.do_test_multi_inputs_multi_outputs_with_input_fn( - train_input_fn, eval_input_fn) + distribution, train_input_fn, eval_input_fn) - def do_test_multi_inputs_multi_outputs_with_input_fn(self, train_input_fn, - eval_input_fn): + def do_test_multi_inputs_multi_outputs_with_input_fn( + self, distribution, train_input_fn, eval_input_fn): config = run_config_lib.RunConfig( tf_random_seed=_RANDOM_SEED, model_dir=self._base_dir, - train_distribute=self._dist) + train_distribute=distribution) with self.cached_session(): model = multi_inputs_multi_outputs_model() est_keras = keras_lib.model_to_estimator(keras_model=model, config=config) @@ -429,9 +476,12 @@ class TestEstimatorDistributionStrategy(test_util.TensorFlowTestCase): eval_results = est_keras.evaluate(input_fn=eval_input_fn, steps=1) self.assertLess(eval_results['loss'], baseline_eval_results['loss']) - def test_keras_optimizer_with_distribution_strategy(self): - dist = mirrored_strategy.MirroredStrategy( - devices=['/device:GPU:0', '/device:GPU:1']) + @combinations.generate(combinations.combine( + distribution=[ + combinations.mirrored_strategy_with_two_gpus, + combinations.core_mirrored_strategy_with_two_gpus], + mode=['graph'])) + def test_keras_optimizer_with_distribution_strategy(self, distribution): keras_model = simple_sequential_model() keras_model.compile( loss='categorical_crossentropy', @@ -439,7 +489,7 @@ class TestEstimatorDistributionStrategy(test_util.TensorFlowTestCase): config = run_config_lib.RunConfig(tf_random_seed=_RANDOM_SEED, model_dir=self._base_dir, - train_distribute=dist) + train_distribute=distribution) with self.cached_session(): est_keras = keras_lib.model_to_estimator(keras_model=keras_model, config=config) @@ -455,7 +505,7 @@ class TestEstimatorDistributionStrategy(test_util.TensorFlowTestCase): class TestDistributionStrategyWithNumpyArrays(test.TestCase, parameterized.TestCase): - @combinations.generate(strategy_combinations()) + @combinations.generate(strategy_for_numpy_input_combinations()) def test_creating_var_with_numpy_arrays(self, distribution): with self.cached_session(): x = np.asarray(np.random.random((64, 3)), dtype=np.float32) @@ -464,84 +514,135 @@ class TestDistributionStrategyWithNumpyArrays(test.TestCase, # Verify that the numpy value is copied to the variable. self.assertAllEqual(x, val) - def test_calculating_batch_params(self): - # This verifies that we calculate the number of steps when the batch size - # is specified. + @combinations.generate(strategy_for_numpy_input_combinations()) + def test_calculating_input_params_no_steps_no_batch_size(self, distribution): + # Calculate the per_replica_batch_size scaling factor for strategies + # that use per_core_batch_size + replica_scale_factor = 1.0 + if not distributed_training_utils.global_batch_size_supported(distribution): + replica_scale_factor = distribution.num_replicas_in_sync + with self.cached_session(): - # 64 is the number of input samples. - inputs = np.zeros((64, 3), dtype=np.float32) - # The number of replicas is equal to 3. - strategy = mirrored_strategy.MirroredStrategy(['/device:GPU:0', - '/device:CPU:0', - '/device:GPU:1']) + # Input samples of different sizes + input_20_samples = np.zeros((20, 3), dtype=np.float32) + input_63_samples = np.zeros((63, 3), dtype=np.float32) + input_64_samples = np.zeros((64, 3), dtype=np.float32) - with self.assertRaisesRegexp(ValueError, 'Please specify a batch_size ' - 'that is smaller than'): - # The batch size(128) is larger than the number of input - # samples(64). - distributed_training_utils.get_input_batch_params(inputs, - 128, - strategy) - - with self.assertRaisesRegexp(ValueError, 'is smaller than the number ' - 'of replicas'): - # The batch size(32) * num_replicas_in_sync(3) is 96 which is greater - # than the number of input samples(64). - distributed_training_utils.get_input_batch_params(inputs, - 32, - strategy) - - # The number of replicas now is equal to 2. - strategy = mirrored_strategy.MirroredStrategy(['/device:GPU:0', - '/device:CPU:0']) - # 32 is the batch size per replica. - steps = distributed_training_utils.get_input_batch_params(inputs, - 32, - strategy) - # The number of batches is the ratio of input samples(64) to - # batch size(32) which is 2. The number of steps(1) is the ratio of - # number of batches(2) to the number of replicas(2). - self.assertEqual(steps, 1) - - # 16 is the batch size per replica. - steps = distributed_training_utils.get_input_batch_params(inputs, - 16, - strategy) - # The number of batches is the ratio of input samples(64) to - # batch size(16) which is 4. The number of steps(2) is the ratio of - # number of batches(4) to the number of replicas(2). + # Default global batch size 32 for input with 64 samples run in 2 steps + steps, batch_size = distributed_training_utils.get_input_params( + distribution, input_64_samples, steps=None, batch_size=None) + self.assertEqual(batch_size, 32 // replica_scale_factor) self.assertEqual(steps, 2) - def test_calculating_batch_size(self): + # Computed global batch size 20 is lower than 32 if we pass less samples. + steps, batch_size = distributed_training_utils.get_input_params( + distribution, input_20_samples, steps=None, batch_size=None) + self.assertEqual(batch_size, 20 // replica_scale_factor) + self.assertEqual(steps, 1) + + # Default global batch size 32 cannot be used with 63 samples. + with self.assertRaisesRegexp(ValueError, 'not divisible by batch size'): + distributed_training_utils.get_input_params( + distribution, input_63_samples, steps=None, batch_size=None) + + @combinations.generate(strategy_for_numpy_input_combinations()) + def test_calculating_input_params_with_steps_no_batch_size(self, + distribution): + # Calculate the per_replica_batch_size scaling factor for strategies + # that use per_core_batch_size + replica_scale_factor = 1.0 + if not distributed_training_utils.global_batch_size_supported(distribution): + replica_scale_factor = distribution.num_replicas_in_sync + with self.cached_session(): - # 64 is the number of input samples. - inputs = np.zeros((64, 3), dtype=np.float32) - targets = np.zeros((64, 4), dtype=np.float32) + # Input samples of different sizes + input_63_samples = np.zeros((63, 3), dtype=np.float32) + input_64_samples = np.zeros((64, 3), dtype=np.float32) - model = get_model() - optimizer = gradient_descent.GradientDescentOptimizer(0.001) - loss = 'mse' - strategy = mirrored_strategy.MirroredStrategy(['/device:GPU:0', - '/device:CPU:0']) - strategy._require_static_shapes = True + # Computed global batch size is correct for number of specified 1 step + steps, batch_size = distributed_training_utils.get_input_params( + distribution, input_64_samples, steps=1, batch_size=None) + self.assertEqual(batch_size, 64 // replica_scale_factor) + self.assertEqual(steps, 1) - model.compile(optimizer, loss, distribute=strategy) - iterator = model._distribution_standardize_user_data(inputs, - targets, - batch_size=None, - check_steps=True, - steps_name='steps', - steps=3) + # Computed global batch size is correct for number of specified 2 steps + steps, batch_size = distributed_training_utils.get_input_params( + distribution, input_64_samples, steps=2, batch_size=None) + self.assertEqual(batch_size, 32 // replica_scale_factor) + self.assertEqual(steps, 2) - # The global batch size(21) across all replicas is the ratio of the input - # samples(64) to the steps(3). - # The batch size(10) per device is the ratio of the global batch size(21) - # to the number of replicas(2). - # The global batch size and batch size are rounded integer values. - self.assertEqual(10, distributed_training_utils.get_batch_dimension( - iterator._iterator)) + # All samples can not be consumed in specified number of steps + with self.assertRaisesRegexp(ValueError, 'not divisible by steps'): + distributed_training_utils.get_input_params( + distribution, input_63_samples, steps=2, batch_size=None) - @combinations.generate(strategy_combinations()) + # This cases is different for different strategies due to the + # difference in supported batch size being global or per-replica. + if replica_scale_factor == 1: + # Computed global batch size is correct even if not sharadable + steps, batch_size = distributed_training_utils.get_input_params( + distribution, input_63_samples, steps=3, batch_size=None) + self.assertEqual(batch_size, 21) + self.assertEqual(steps, 3) + else: + # Computed global batch size can not be sharded across replicas + with self.assertRaisesRegexp(ValueError, 'could not be sharded evenly ' + 'across the sync replicas'): + distributed_training_utils.get_input_params( + distribution, input_63_samples, steps=1, batch_size=None) + + @combinations.generate(strategy_for_numpy_input_combinations()) + def test_calculating_input_params_no_steps_with_batch_size(self, + distribution): + # Calculate the per_replica_batch_size scaling factor for strategies + # that use per_core_batch_size + replica_scale_factor = 1.0 + if not distributed_training_utils.global_batch_size_supported(distribution): + replica_scale_factor = distribution.num_replicas_in_sync + + with self.cached_session(): + input_64_samples = np.zeros((64, 3), dtype=np.float32) + + # Computed steps is correct for specified batch size + steps, batch_size = distributed_training_utils.get_input_params( + distribution, input_64_samples, steps=None, batch_size=16) + self.assertEqual(batch_size, 16) + self.assertEqual(steps, 4 // replica_scale_factor) + + # Computed steps is correct for specified batch size + steps, batch_size = distributed_training_utils.get_input_params( + distribution, input_64_samples, steps=None, batch_size=32) + self.assertEqual(batch_size, 32) + self.assertEqual(steps, 2 // replica_scale_factor) + + # Number of samples is not divisible by the global batch size + with self.assertRaisesRegexp(ValueError, 'not divisible by batch size'): + distributed_training_utils.get_input_params( + distribution, input_64_samples, steps=None, batch_size=20) + + # Number of samples is not divisible by the global batch size + with self.assertRaisesRegexp(ValueError, 'not divisible by batch size'): + distributed_training_utils.get_input_params( + distribution, input_64_samples, steps=None, batch_size=3) + + @combinations.generate(strategy_for_numpy_input_combinations()) + def test_calculating_input_params_with_steps_with_batch_size(self, + distribution): + with self.cached_session(): + input_64_samples = np.zeros((64, 3), dtype=np.float32) + + # No change to steps and batch size if both specified and feasible + steps, batch_size = distributed_training_utils.get_input_params( + distribution, input_64_samples, steps=5, batch_size=3) + self.assertEqual(batch_size, 3) + self.assertEqual(steps, 5) + + # Number of samples is less than global batch size * steps + with self.assertRaisesRegexp(ValueError, 'less than samples required'): + distributed_training_utils.get_input_params( + distribution, input_64_samples, steps=10, batch_size=13) + + @combinations.generate(strategy_for_numpy_input_combinations()) def test_calling_model_with_numpy_arrays(self, distribution): with self.cached_session(): model = get_model() @@ -572,7 +673,7 @@ class TestDistributionStrategyWithNumpyArrays(test.TestCase, # with batch_size model.predict(inputs, batch_size=8) - @combinations.generate(strategy_combinations()) + @combinations.generate(strategy_for_numpy_input_combinations()) def test_calling_model_with_nested_numpy_arrays(self, distribution): with self.cached_session(): model = multi_input_output_model() @@ -606,21 +707,22 @@ class TestDistributionStrategyWithNumpyArrays(test.TestCase, # with batch_size model.predict(inputs, batch_size=8) - @combinations.generate(strategy_minus_tpu_combinations()) + @combinations.generate(combinations.combine( + distribution=strategies_minus_tpu, mode=['graph'])) def test_numpy_with_sample_weights(self, distribution): model = get_model() optimizer = rmsprop.RMSPropOptimizer(learning_rate=0.001) loss = 'mse' model.compile(optimizer, loss, distribute=distribution) - inputs = np.zeros((10, 3), np.float32) - targets = np.zeros((10, 4), np.float32) - sample_weights = np.ones((10), np.float32) + inputs = np.zeros((20, 3), np.float32) + targets = np.zeros((20, 4), np.float32) + sample_weights = np.ones((20), np.float32) model.fit(inputs, targets, sample_weight=sample_weights, epochs=1, steps_per_epoch=2, verbose=1) - @combinations.generate(strategy_combinations()) + @combinations.generate(strategy_for_numpy_input_combinations()) def test_flatten_predict_outputs(self, distribution): with self.cached_session(): model = multi_input_output_model() @@ -638,7 +740,7 @@ class TestDistributionStrategyWithNumpyArrays(test.TestCase, # `predict` a list that is equal in length to the number of model outputs. # In this test our model has two outputs and each element of `outs` # corresponds to all the samples of one of the model outputs. - self.assertEqual(2, len(outs)) + self.assertLen(outs, 2) # Each of the output samples have a dimension of 7. We should process all # the available input samples(6). self.assertAllEqual([6, 7], outs[0].shape) @@ -648,7 +750,7 @@ class TestDistributionStrategyWithNumpyArrays(test.TestCase, class TestDistributionStrategyWithDatasets(test.TestCase, parameterized.TestCase): - @combinations.generate(strategy_combinations()) + @combinations.generate(all_strategy_combinations()) def test_calling_model_on_same_dataset(self, distribution): with self.cached_session(): model = get_model() @@ -667,7 +769,7 @@ class TestDistributionStrategyWithDatasets(test.TestCase, validation_data=dataset, validation_steps=2) model.predict(get_predict_dataset(distribution), steps=2) - @combinations.generate(strategy_combinations()) + @combinations.generate(all_strategy_combinations()) def test_model_interleaved_eval_same_as_direct_eval(self, distribution): with self.cached_session(): user_controlled_model = get_model() @@ -710,16 +812,20 @@ class TestDistributionStrategyWithDatasets(test.TestCase, # TODO(priyag): Enable this test for TPU. Currently tuples/dict don't work # as clone_model's input_tensors argument only seems to accept list and not # tuples or dict. - def test_fit_with_tuple_and_dict_dataset_inputs(self): + + @combinations.generate(combinations.combine( + distribution=[ + combinations.mirrored_strategy_with_gpu_and_cpu, + combinations.core_mirrored_strategy_with_gpu_and_cpu], + mode=['graph', 'eager'])) + def test_fit_with_tuple_and_dict_dataset_inputs(self, distribution): with self.cached_session(): model = multi_input_output_model() optimizer = gradient_descent.GradientDescentOptimizer(learning_rate=0.001) loss = 'mse' metrics = ['mae', keras.metrics.CategoricalAccuracy()] - strategy = mirrored_strategy.MirroredStrategy(['/device:GPU:0', - '/device:CPU:0']) - model.compile(optimizer, loss, metrics=metrics, distribute=strategy) + model.compile(optimizer, loss, metrics=metrics, distribute=distribution) input_a_np = np.random.random((10, 3)) input_b_np = np.random.random((10, 5)) @@ -743,7 +849,7 @@ class TestDistributionStrategyWithDatasets(test.TestCase, model.fit(dataset_dict, epochs=1, steps_per_epoch=2, verbose=1) - @combinations.generate(strategy_combinations()) + @combinations.generate(all_strategy_combinations()) def test_fit_eval_and_predict_methods_on_dataset(self, distribution): with self.cached_session(): model = get_model() @@ -792,25 +898,18 @@ class TestDistributionStrategyWithDatasets(test.TestCase, model.evaluate(dataset, steps=2, verbose=1) model.predict(dataset, steps=2) - def test_dataset_input_shape_validation(self): + @combinations.generate(combinations.combine( + distribution=[ + combinations.mirrored_strategy_with_two_gpus, + combinations.core_mirrored_strategy_with_two_gpus], + mode=['graph', 'eager'])) + def test_dataset_wrong_input_shape(self, distribution): with self.cached_session(): model = get_model() optimizer = rmsprop.RMSPropOptimizer(learning_rate=0.001) loss = 'mse' - strategy = mirrored_strategy.MirroredStrategy(['/device:GPU:1', - '/device:GPU:0']) - - model.compile(optimizer, loss, distribute=strategy) - - # User forgets to batch the dataset - inputs = np.zeros((10, 3), dtype=np.float32) - targets = np.zeros((10, 4), dtype=np.float32) - dataset = dataset_ops.Dataset.from_tensor_slices((inputs, targets)) - dataset = dataset.repeat(100) - - with self.assertRaisesRegexp(ValueError, 'expected input to have shape'): - model.fit(dataset, epochs=1, steps_per_epoch=2, verbose=0) + model.compile(optimizer, loss, distribute=distribution) # Wrong input shape inputs = np.zeros((10, 5), dtype=np.float32) @@ -823,6 +922,26 @@ class TestDistributionStrategyWithDatasets(test.TestCase, 'expected input to have shape'): model.fit(dataset, epochs=1, steps_per_epoch=2, verbose=0) + @combinations.generate(combinations.combine( + distribution=[combinations.mirrored_strategy_with_two_gpus], + mode=['graph', 'eager'])) + def test_dataset_no_batch_input_validation(self, distribution): + with self.cached_session(): + model = get_model() + + optimizer = rmsprop.RMSPropOptimizer(learning_rate=0.001) + loss = 'mse' + model.compile(optimizer, loss, distribute=distribution) + + # User forgets to batch the dataset + inputs = np.zeros((10, 3), dtype=np.float32) + targets = np.zeros((10, 4), dtype=np.float32) + dataset = dataset_ops.Dataset.from_tensor_slices((inputs, targets)) + dataset = dataset.repeat(100) + + with self.assertRaisesRegexp(ValueError, 'expected input to have shape'): + model.fit(dataset, epochs=1, steps_per_epoch=2, verbose=0) + @combinations.generate(combinations.combine( distribution=[combinations.tpu_strategy_one_step], mode=['graph'])) @@ -842,7 +961,12 @@ class TestDistributionStrategyWithDatasets(test.TestCase, with self.assertRaisesRegexp(ValueError, 'requires fully defined shapes'): model.fit(dataset, epochs=1, steps_per_epoch=2, verbose=0) - def test_learning_phase_value(self): + @combinations.generate(combinations.combine( + distribution=[ + combinations.mirrored_strategy_with_two_gpus, + combinations.core_mirrored_strategy_with_two_gpus], + mode=['graph', 'eager'])) + def test_learning_phase_value(self, distribution): # TODO(anjalisridhar): Modify this test to use Lambdas since we can compare # meaningful values. Currently we don't pass the learning phase if the # Lambda layer uses the learning phase. @@ -856,15 +980,17 @@ class TestDistributionStrategyWithDatasets(test.TestCase, optimizer = gradient_descent.GradientDescentOptimizer(0.005) loss = 'mse' metrics = ['acc'] - strategy = mirrored_strategy.MirroredStrategy( - ['/device:GPU:0', '/device:GPU:1']) + model.compile(optimizer, loss, metrics=metrics, distribute=distribution) - model.compile(optimizer, loss, metrics=metrics, distribute=strategy) + batch_size = 8 + if isinstance(distribution, mirrored_strategy.CoreMirroredStrategy): + # CoreMirroredStrategy uses global batch size. + batch_size = 8 * distribution.num_replicas_in_sync inputs = np.ones((10, 1), dtype=np.float32) targets = np.ones((10, 1), dtype=np.float32) dataset = dataset_ops.Dataset.from_tensor_slices((inputs, targets)) - dataset = dataset.repeat().batch(8) + dataset = dataset.repeat().batch(batch_size) hist = model.fit(dataset, epochs=1, steps_per_epoch=20, verbose=1) self.assertAlmostEqual(hist.history['acc'][0], 0, 0) @@ -875,24 +1001,51 @@ class TestDistributionStrategyWithDatasets(test.TestCase, inputs = np.ones((10, 1), dtype=np.float32) predict_dataset = dataset_ops.Dataset.from_tensor_slices(inputs) - predict_dataset = predict_dataset.repeat().batch(5) + + predict_dataset = predict_dataset.repeat().batch(batch_size) output = model.predict(predict_dataset, steps=10) - # `predict` runs for 10 steps and in each step you process 100 samples. - ref_output = np.ones((100, 1), dtype=np.float32) + # `predict` runs for 10 steps + ref_output = np.ones((160, 1), dtype=np.float32) self.assertArrayNear(output, ref_output, 1e-1) + @combinations.generate(strategy_minus_tpu_combinations()) + def testOptimizerWithCallbacks(self, distribution): + with self.cached_session(): + model = get_model() + + optimizer = gradient_descent_keras.SGD(0.01) + loss = 'mse' + model.compile(optimizer, loss, distribute=distribution) + + dataset = get_dataset(distribution) + + def schedule(_): + return 0.001 + + model.fit(dataset, epochs=1, steps_per_epoch=2, verbose=0, + callbacks=[keras.callbacks.LearningRateScheduler(schedule)]) + grouped_models = distribution.unwrap(model._grouped_model) + with distribution.scope(): + for m in grouped_models: + self.assertAllClose(0.001, keras.backend.get_value( + m.optimizer.lr), atol=1e-05, rtol=1e-05) + class TestDistributionStrategyErrorCases(test.TestCase, parameterized.TestCase): - def test_validating_dataset_input_tensors_with_shape_mismatch(self): + @combinations.generate(combinations.combine( + distribution=[ + combinations.mirrored_strategy_with_gpu_and_cpu, + combinations.core_mirrored_strategy_with_gpu_and_cpu], + mode=['graph', 'eager'])) + def test_validating_dataset_input_tensors_with_shape_mismatch(self, + distribution): with self.cached_session(): - strategy = mirrored_strategy.MirroredStrategy(['/device:GPU:0', - '/device:CPU:0']) a = constant_op.constant([1, 2], shape=(1, 2)) b = constant_op.constant([[1, 2], [1, 2]], shape=(2, 2)) x = values.DistributedValues({'/device:CPU:0': a, '/device:GPU:0': b}) y = values.DistributedValues({'/device:CPU:0': a, '/device:GPU:0': a}) - with strategy.scope(): + with distribution.scope(): # Removed device and input tensor shape details from the error message # since the order of the device and the corresponding input tensor shape # is not deterministic over different runs. @@ -901,17 +1054,21 @@ class TestDistributionStrategyErrorCases(test.TestCase, parameterized.TestCase): 'distributed tensor inputs ' 'DistributedValues:.+'): distributed_training_utils.validate_distributed_dataset_inputs( - strategy, x, y) + distribution, x, y) - def test_validating_dataset_input_tensors_with_dtype_mismatch(self): + @combinations.generate(combinations.combine( + distribution=[ + combinations.mirrored_strategy_with_gpu_and_cpu, + combinations.core_mirrored_strategy_with_gpu_and_cpu], + mode=['graph', 'eager'])) + def test_validating_dataset_input_tensors_with_dtype_mismatch(self, + distribution): with self.cached_session(): - strategy = mirrored_strategy.MirroredStrategy(['/device:GPU:0', - '/device:CPU:0']) a = constant_op.constant([1, 2], shape=(1, 2), dtype=dtypes.int32) b = constant_op.constant([1, 2], shape=(1, 2), dtype=dtypes.float64) x = values.DistributedValues({'/device:CPU:0': a, '/device:GPU:0': b}) y = values.DistributedValues({'/device:CPU:0': a, '/device:GPU:0': a}) - with strategy.scope(): + with distribution.scope(): # Removed device and input tensor dtype details from the error message # since the order of the device and the corresponding input tensor dtype # is not deterministic over different runs. @@ -920,21 +1077,23 @@ class TestDistributionStrategyErrorCases(test.TestCase, parameterized.TestCase): 'distributed tensor inputs ' 'DistributedValues:.+'): distributed_training_utils.validate_distributed_dataset_inputs( - strategy, x, y) + distribution, x, y) - def test_unsupported_features(self): + @combinations.generate(combinations.combine( + distribution=[ + combinations.mirrored_strategy_with_two_gpus, + combinations.core_mirrored_strategy_with_two_gpus], + mode=['graph', 'eager'])) + def test_unsupported_features(self, distribution): with self.cached_session(): model = get_model() optimizer = gradient_descent.GradientDescentOptimizer(0.001) loss = 'mse' metrics = ['mae'] - strategy = mirrored_strategy.MirroredStrategy(['/device:GPU:1', - '/device:GPU:0']) + model.compile(optimizer, loss, metrics=metrics, distribute=distribution) - model.compile(optimizer, loss, metrics=metrics, distribute=strategy) - - dataset = get_dataset(strategy) + dataset = get_dataset(distribution) # Test with validation split with self.assertRaisesRegexp( @@ -969,30 +1128,33 @@ class TestDistributionStrategyErrorCases(test.TestCase, parameterized.TestCase): 'you should specify the `steps` argument'): model.predict(dataset, verbose=0) - def test_calling_with_unsupported_predefined_callbacks(self): + @combinations.generate(combinations.combine( + distribution=[ + combinations.mirrored_strategy_with_two_gpus, + combinations.core_mirrored_strategy_with_two_gpus], + mode=['graph', 'eager'])) + def test_calling_with_unsupported_predefined_callbacks(self, distribution): with self.cached_session(): model = get_model() optimizer = gradient_descent.GradientDescentOptimizer(0.001) loss = 'mse' metrics = ['mae'] - strategy = mirrored_strategy.MirroredStrategy(['/device:GPU:1', - '/device:GPU:0']) - model.compile(optimizer, loss, metrics=metrics, distribute=strategy) + model.compile(optimizer, loss, metrics=metrics, distribute=distribution) - dataset = get_dataset(strategy) + dataset = get_dataset(distribution) def schedule(_): return 0.001 with self.assertRaisesRegexp(ValueError, - 'LearningRateScheduler callback is not ' - 'supported with DistributionStrategy.'): + 'You must specify a Keras Optimizer V2 when ' + 'using'): model.fit(dataset, epochs=1, steps_per_epoch=2, verbose=0, callbacks=[keras.callbacks.LearningRateScheduler(schedule)]) with self.assertRaisesRegexp(ValueError, - 'ReduceLROnPlateau callback is not ' - 'supported with DistributionStrategy.'): + 'You must specify a Keras Optimizer V2 when ' + 'using'): model.fit(dataset, epochs=1, steps_per_epoch=2, verbose=0, callbacks=[keras.callbacks.ReduceLROnPlateau()]) with self.assertRaisesRegexp(ValueError, @@ -1003,11 +1165,17 @@ class TestDistributionStrategyErrorCases(test.TestCase, parameterized.TestCase): callbacks=[keras.callbacks.TensorBoard(histogram_freq=10)]) -class TestDistributionStrategyWithLossMasking(test.TestCase): +class TestDistributionStrategyWithLossMasking(test.TestCase, + parameterized.TestCase): # TODO(priyag): Enable all strategies for this test. Currently it does not # work for TPU due to some invalid datatype. - def test_masking(self): + @combinations.generate(combinations.combine( + distribution=[ + combinations.mirrored_strategy_with_two_gpus, + combinations.core_mirrored_strategy_with_two_gpus], + mode=['graph', 'eager'])) + def test_masking(self, distribution): with self.cached_session(): np.random.seed(1337) x = np.array([[[1], [1]], [[0], [0]]]) @@ -1016,12 +1184,9 @@ class TestDistributionStrategyWithLossMasking(test.TestCase): model.add( keras.layers.TimeDistributed( keras.layers.Dense(1, kernel_initializer='one'))) - strategy = mirrored_strategy.MirroredStrategy(['/device:GPU:1', - '/device:GPU:0']) - model.compile(loss='mse', optimizer=gradient_descent.GradientDescentOptimizer(0.01), - distribute=strategy) + distribute=distribution) y = np.array([[[1], [1]], [[1], [1]]]) dataset = dataset_ops.Dataset.from_tensor_slices((x, y)) dataset = dataset.repeat(100) @@ -1033,7 +1198,7 @@ class TestDistributionStrategyWithLossMasking(test.TestCase): class TestDistributionStrategyWithNormalizationLayer( test.TestCase, parameterized.TestCase): - @combinations.generate(strategy_combinations()) + @combinations.generate(all_strategy_combinations()) def test_batchnorm_correctness(self, distribution): with self.cached_session(): model = keras.models.Sequential() @@ -1065,7 +1230,7 @@ class TestDistributionStrategyWithNormalizationLayer( class TestDistributionStrategyCorrectness(test.TestCase, parameterized.TestCase): - @combinations.generate(strategy_combinations()) + @combinations.generate(all_strategy_combinations()) def test_metric_correctness(self, distribution): with self.cached_session(): keras.backend.set_image_data_format('channels_last') @@ -1088,22 +1253,32 @@ class TestDistributionStrategyCorrectness(test.TestCase, distribute=distribution) batch_size = 64 - batch_size //= distribution.num_replicas_in_sync + if not distributed_training_utils.global_batch_size_supported( + distribution): + batch_size //= distribution.num_replicas_in_sync train_dataset = dataset_ops.Dataset.from_tensor_slices((x_train, y_train)) train_dataset = batch_wrapper(train_dataset, batch_size, distribution) history = model.fit(x=train_dataset, epochs=1, steps_per_epoch=10) self.assertEqual(history.history['binary_accuracy'], [1.0]) - @combinations.generate(strategy_and_inputs()) - def test_correctness(self, distribution, use_numpy): + @combinations.generate(strategy_and_input_combinations()) + def test_correctness(self, distribution, use_numpy, use_validation_data): + with self.cached_session(): tolerance = 1e-5 - if isinstance(distribution, mirrored_strategy.MirroredStrategy): + if isinstance(distribution, (mirrored_strategy.MirroredStrategy, + mirrored_strategy.CoreMirroredStrategy)): # TODO(b/119257215): use the default one once the flakyness is fixed. tolerance = 1e-4 + if (use_validation_data and + not isinstance(distribution, tpu_strategy.TPUStrategy)): + # TODO(b/120435565): Enable tests with use_validation_data once the + # the underlying bug is fixed. + return + keras.backend.set_image_data_format('channels_last') np.random.seed(_RANDOM_SEED) random_seed.set_random_seed(_RANDOM_SEED) @@ -1123,49 +1298,72 @@ class TestDistributionStrategyCorrectness(test.TestCase, # This is used to initialize the model for both the distribution and # non-distribution run. In addition, we add few non-linear layers to make # it non-trivial. - model = keras.Sequential() - model.add(keras.layers.Dense(10, activation='relu', input_shape=(1,))) - model.add(keras.layers.Dense(10, activation='relu')) - model.add(keras.layers.Dense(10, activation='relu')) - model.add(keras.layers.Dense(1)) - initial_weights = model.get_weights() + def _create_model(): + model = keras.Sequential() + model.add(keras.layers.Dense(10, activation='relu', input_shape=(1,))) + model.add(keras.layers.Dense(10, activation='relu')) + model.add(keras.layers.Dense(10, activation='relu')) + model.add(keras.layers.Dense(1)) + return model - def fit_and_predict(with_distribution=None): + model = _create_model() + initial_weights = model.get_weights() + del model # avoid accident usage. + + def fit_eval_and_predict(with_distribution=None): + model = _create_model() # We have initialized the model to the same weight for the distribution # and non-distribution run. model.set_weights(initial_weights) model.compile( loss=keras.losses.mean_squared_error, - optimizer=gradient_descent.GradientDescentOptimizer(0.5), + optimizer=gradient_descent_keras.SGD(0.5), distribute=with_distribution) training_inputs, eval_inputs, predict_inputs = ( - get_correctness_test_inputs(use_numpy, with_distribution, + get_correctness_test_inputs(use_numpy, use_validation_data, + with_distribution, x_train, y_train, x_predict)) - model.fit(**training_inputs) - eval_result = model.evaluate(**eval_inputs) + traning_history = model.fit(**training_inputs).history + + if eval_inputs is not None: + eval_result = model.evaluate(**eval_inputs) + else: + # Creates a dummy identical eval_result to be compared later. + eval_result = 1.0 + weights = model.get_weights() predict_result = model.predict(**predict_inputs) - return weights, eval_result, predict_result + return weights, traning_history, eval_result, predict_result - wts_with_ds, eval_with_ds, predict_with_ds = fit_and_predict( - with_distribution=distribution) - wts_without_ds, eval_without_ds, predict_without_ds = fit_and_predict( - with_distribution=None) + wts_with_ds, history_with_ds, eval_with_ds, predict_with_ds = ( + fit_eval_and_predict(with_distribution=distribution)) - # Verify that the weights, eval results, predict outputs are the same - # within some limits of tolerance. + (wts_without_ds, history_without_ds, eval_without_ds, + predict_without_ds) = fit_eval_and_predict(with_distribution=None) + + # Verify that the weights, training history, eval results, predict outputs + # are the same within some limits of tolerance. self.assertAllClose( - wts_with_ds, wts_without_ds, atol=tolerance, rtol=tolerance) - self.assertAllClose( - eval_with_ds, eval_without_ds, atol=tolerance, rtol=tolerance) - self.assertAllClose( - predict_with_ds, predict_without_ds, atol=tolerance, rtol=tolerance) + wts_with_ds, wts_without_ds, atol=tolerance, rtol=tolerance, + msg='Fail to assert weights after training.') + self.assertAllClose( + eval_with_ds, eval_without_ds, atol=tolerance, rtol=tolerance, + msg='Fail to assert eval results.') + self.assertAllClose( + predict_with_ds, predict_without_ds, atol=tolerance, rtol=tolerance, + msg='Fail to assert predict results.') -# TODO(priyag): Add a test for TPUStrategy with steps_per_run > 1. + if not (isinstance(distribution, tpu_strategy.TPUStrategy) + and distribution.extended.steps_per_run > 1): + # TODO(b/119894254): Enable this test for all cases once the underlying + # bug is fixed. + self.assertAllClose( + history_with_ds, history_without_ds, atol=tolerance, rtol=tolerance, + msg='Fail to assert training history.') if __name__ == '__main__': diff --git a/tensorflow/contrib/distribute/python/metrics_v1_test.py b/tensorflow/contrib/distribute/python/metrics_v1_test.py index c28ab416518..8ac659abe96 100644 --- a/tensorflow/contrib/distribute/python/metrics_v1_test.py +++ b/tensorflow/contrib/distribute/python/metrics_v1_test.py @@ -72,14 +72,14 @@ def _regression_dataset_fn(): "predictions": [1., .75, .25, 0.]}).repeat() -# TODO(priyag): Add TPU Strategy to this once metrics aggregate correctly using -# ReplicaLocalVariables on TPUs. Submit http://cl/208914352. def all_combinations(): return combinations.combine( distribution=[combinations.default_strategy, combinations.one_device_strategy, combinations.mirrored_strategy_with_gpu_and_cpu, - combinations.mirrored_strategy_with_two_gpus], + combinations.mirrored_strategy_with_two_gpus, + combinations.core_mirrored_strategy_with_gpu_and_cpu, + combinations.core_mirrored_strategy_with_two_gpus], mode=["graph"]) @@ -100,18 +100,19 @@ class MetricsV1Test(test.TestCase, parameterized.TestCase): if isinstance(distribution, tpu_strategy.TPUStrategy): def step_fn(ctx, inputs): value, update = distribution.call_for_each_replica( - metric_fn, args=[inputs]) + metric_fn, args=inputs) ctx.set_non_tensor_output(name="value", output=value) return distribution.group(update) ctx = distribution.run_steps_on_dataset( - step_fn, iterator, iterations=distribution.steps_per_run) + step_fn, iterator, iterations=distribution.extended.steps_per_run) update = ctx.run_op value = ctx.non_tensor_outputs["value"] # In each run, we run multiple steps, and each steps consumes as many # batches as number of replicas. batches_per_update = ( - distribution.num_replicas_in_sync * distribution.steps_per_run) + distribution.num_replicas_in_sync * + distribution.extended.steps_per_run) else: value, update = distribution.call_for_each_replica( metric_fn, iterator.get_next()) diff --git a/tensorflow/contrib/distribute/python/minimize_loss_test.py b/tensorflow/contrib/distribute/python/minimize_loss_test.py index c6562463edb..dcc9df4cda5 100644 --- a/tensorflow/contrib/distribute/python/minimize_loss_test.py +++ b/tensorflow/contrib/distribute/python/minimize_loss_test.py @@ -25,6 +25,7 @@ from tensorflow.contrib.distribute.python import combinations from tensorflow.contrib.distribute.python.single_loss_example import batchnorm_example from tensorflow.contrib.distribute.python.single_loss_example import minimize_loss_example from tensorflow.python.data.ops import dataset_ops +from tensorflow.python.distribute import reduce_util from tensorflow.python.eager import context from tensorflow.python.eager import test from tensorflow.python.framework import constant_op @@ -63,7 +64,7 @@ class MinimizeLossStepTest(test.TestCase, parameterized.TestCase): model_fn, dataset_fn, layer = minimize_loss_example( optimizer_fn, use_bias=True, use_callable_loss=use_callable_loss) - def step_fn(ctx, *inputs): + def step_fn(ctx, inputs): del ctx # Unused return distribution.group( distribution.call_for_each_replica(model_fn, args=inputs)) @@ -157,7 +158,7 @@ class MinimizeLossStepTest(test.TestCase, parameterized.TestCase): use_callable_loss=True, create_optimizer_inside_model_fn=True) - def step_fn(ctx, *inputs): + def step_fn(ctx, inputs): del ctx # Unused return distribution.group( distribution.call_for_each_replica(model_fn, args=inputs)) @@ -226,7 +227,7 @@ class MinimizeLossStepTest(test.TestCase, parameterized.TestCase): renorm=renorm, update_ops_in_replica_mode=not update_ops_in_cross_replica_mode) - def step_fn(ctx, *inputs): + def step_fn(ctx, inputs): del ctx # Unused fetches = distribution.unwrap( distribution.call_for_each_replica(model_fn, args=inputs)) @@ -285,7 +286,9 @@ class MinimizeLossStepTest(test.TestCase, parameterized.TestCase): distribution=[ combinations.one_device_strategy, combinations.mirrored_strategy_with_gpu_and_cpu, - combinations.mirrored_strategy_with_two_gpus + combinations.mirrored_strategy_with_two_gpus, + combinations.core_mirrored_strategy_with_gpu_and_cpu, + combinations.core_mirrored_strategy_with_two_gpus ]), combinations.combine( mode=["graph"], use_callable_loss=[True, False]) + @@ -321,10 +324,10 @@ class MinimizeLossStepTest(test.TestCase, parameterized.TestCase): labels = dataset_ops.Dataset.from_tensors([[6.], [21.]]) return dataset_ops.Dataset.zip((features, labels)).repeat() - def step_fn(ctx, x, y): + def step_fn(ctx, inputs): del ctx # Unused return distribution.group( - distribution.call_for_each_replica(model_fn, args=(x, y))) + distribution.call_for_each_replica(model_fn, args=inputs)) iterator = self._get_iterator(distribution.distribute_dataset(dataset_fn)) @@ -341,7 +344,7 @@ class MinimizeLossStepTest(test.TestCase, parameterized.TestCase): run_step() v = all_vars[0] - self.assertTrue(all([v is vi for vi in all_vars[1:]])) + self.assertTrue(all(v is vi for vi in all_vars[1:])) weight = numpy.squeeze(self.evaluate(v)) # Our model is: # predict = x * w @@ -402,21 +405,21 @@ class MinimizeLossStepTest(test.TestCase, parameterized.TestCase): train_op = optimizer.minimize(loss_fn) loss = loss_fn() output_context.set_last_step_output( - name="replica_loss_agg", + name="replica_loss_reduced", output=loss, - aggregation=variables_lib.VariableAggregation.MEAN) + reduce_op=reduce_util.ReduceOp.MEAN) output_context.set_non_tensor_output(key1, value1) return (train_op, loss) - def step_fn(output_context, *inputs): + def step_fn(output_context, inputs): (train_op, loss) = distribution.call_for_each_replica( model_fn, args=(output_context,) + inputs) output_context.set_last_step_output( - name="cross_replica_loss_agg", + name="cross_replica_loss_reduced", output=loss, - aggregation=variables_lib.VariableAggregation.MEAN) + reduce_op=reduce_util.ReduceOp.MEAN) output_context.set_last_step_output( - name="cross_replica_loss_noagg", + name="cross_replica_loss_not_reduced", output=loss) return distribution.group(train_op) @@ -424,16 +427,16 @@ class MinimizeLossStepTest(test.TestCase, parameterized.TestCase): def run_step(): initial_loss = lambda: constant_op.constant(1e7) - # Initial values corresponding to aggregated losses are just single - # tensors. But for non aggregated losses, we need to have initial + # Initial values corresponding to reduced losses are just single + # tensors. But for non reduced losses, we need to have initial # values that are of the same structure as non reduced losses. In # MirroredStrategy, this will be a list of losses, in TPUStrategy # it will be single tensor. Using `broadcast` followed by `unwrap` # gives us the desired initial value structure. initial_loop_values = { - "replica_loss_agg": initial_loss(), - "cross_replica_loss_agg": initial_loss(), - "cross_replica_loss_noagg": + "replica_loss_reduced": initial_loss(), + "cross_replica_loss_reduced": initial_loss(), + "cross_replica_loss_not_reduced": distribution.unwrap(distribution.broadcast(initial_loss())) } ctx = distribution.run_steps_on_dataset( @@ -443,17 +446,17 @@ class MinimizeLossStepTest(test.TestCase, parameterized.TestCase): self.assertEqual({key1: [value1]}, ctx.non_tensor_outputs) self._verify_loss_output( initial_loss(), - loss_output=ctx.last_step_outputs["replica_loss_agg"], - aggregated=True, distribution=distribution) + loss_output=ctx.last_step_outputs["replica_loss_reduced"], + reduced=True, distribution=distribution) self._verify_loss_output( initial_loss(), - loss_output=ctx.last_step_outputs["cross_replica_loss_agg"], - aggregated=True, distribution=distribution) + loss_output=ctx.last_step_outputs["cross_replica_loss_reduced"], + reduced=True, distribution=distribution) self._verify_loss_output( initial_loss(), - loss_output=ctx.last_step_outputs["cross_replica_loss_noagg"], - aggregated=False, distribution=distribution) - return (ctx.run_op, ctx.last_step_outputs["replica_loss_agg"]) + loss_output=ctx.last_step_outputs["cross_replica_loss_not_reduced"], + reduced=False, distribution=distribution) + return (ctx.run_op, ctx.last_step_outputs["replica_loss_reduced"]) self.evaluate(distribution.initialize()) if not context.executing_eagerly(): @@ -478,18 +481,16 @@ class MinimizeLossStepTest(test.TestCase, parameterized.TestCase): error_is_not_increasing = all(y <= x for x, y in zip(error, error[1:])) self.assertTrue(error_is_not_increasing) - def _verify_loss_output(self, initial_loss, loss_output, aggregated, + def _verify_loss_output(self, initial_loss, loss_output, reduced, distribution): - if not aggregated: - self.assertEqual(distribution.num_replicas_in_sync, - len(distribution.unwrap(loss_output))) - loss_output = distribution.reduce( - aggregation=variables_lib.VariableAggregation.MEAN, - value=loss_output, destinations="/device:CPU:0") - - unwrapped_output = distribution.unwrap(loss_output) - self.assertEqual(1, len(unwrapped_output)) - loss_tensor = unwrapped_output[0] + if not reduced: + self.assertLen(distribution.unwrap(loss_output), + distribution.num_replicas_in_sync) + loss_tensor = distribution.reduce(reduce_util.ReduceOp.MEAN, loss_output) + else: + unwrapped_output = distribution.unwrap(loss_output) + self.assertLen(unwrapped_output, 1) + loss_tensor = unwrapped_output[0] self.assertEqual(initial_loss.dtype, loss_tensor.dtype) self.assertEqual(initial_loss.shape, loss_tensor.shape) diff --git a/tensorflow/contrib/distribute/python/mirrored_strategy.py b/tensorflow/contrib/distribute/python/mirrored_strategy.py index 2d75024e7a0..20f1a08d426 100644 --- a/tensorflow/contrib/distribute/python/mirrored_strategy.py +++ b/tensorflow/contrib/distribute/python/mirrored_strategy.py @@ -12,293 +12,35 @@ # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================== -"""Class MirroredStrategy implementing DistributionStrategy.""" +"""Contrib version of MirroredStrategy.""" from __future__ import absolute_import from __future__ import division from __future__ import print_function -import contextlib -from functools import partial -import threading +import functools -from tensorflow.contrib.distribute.python import cross_tower_ops as cross_tower_ops_lib -from tensorflow.contrib.distribute.python import shared_variable_creator -from tensorflow.contrib.distribute.python import values -from tensorflow.python import pywrap_tensorflow -from tensorflow.python.distribute import multi_worker_util -from tensorflow.python.eager import context -from tensorflow.python.eager import tape -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import device as tf_device -from tensorflow.python.framework import ops -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import control_flow_ops -from tensorflow.python.ops import variable_scope -from tensorflow.python.ops import variables as variables_lib -from tensorflow.python.training import coordinator -from tensorflow.python.training import device_util -from tensorflow.python.training import distribute as distribute_lib -from tensorflow.python.util import nest +from tensorflow.python.distribute import device_util +from tensorflow.python.distribute import distribute_lib +from tensorflow.python.distribute import mirrored_strategy +from tensorflow.python.distribute import values -# TODO(josh11b): Replace asserts in this file with if ...: raise ... - - -@contextlib.contextmanager -def _enter_graph(g): - if context.executing_eagerly(): - with g.as_default(), context.eager_mode(): - yield - else: - with g.as_default(): - yield - - -def _cpu_device(device): - cpu_device = tf_device.DeviceSpec.from_string(device) - cpu_device.merge_from(tf_device.DeviceSpec(device_type="CPU", device_index=0)) - return cpu_device.to_string() - - -class _RequestedStop(Exception): - pass - - -# _call_for_each_replica and _reduce_non_distributed_value are not members of -# MirroredStrategy so that they are generally not allowed to use anything -# specific to MirroredStrategy and thus can be shared with other distribution -# strategies. - - -# TODO(yuefengz): maybe create a common class for those who need to call this -# _call_for_each_replica. -def _call_for_each_replica(distribution, fn, args, kwargs): - """Run `fn` in separate threads, once per replica/worker device. - - Args: - distribution: the DistributionStrategy object. - fn: function to run (will be run once per device, each in its own thread). - args: positional arguments for `fn` - kwargs: keyword arguments for `fn`. - - Returns: - Merged return value of `fn` across all replicas. - - Raises: - RuntimeError: If fn() calls get_replica_context().merge_call() a different - number of times from the available devices. - """ - # TODO(josh11b): Add this option once we add synchronization to variable - # creation. Until then, this is pretty unsafe to use. - run_concurrently = False - if not context.executing_eagerly(): - # Needed for per-thread device, etc. contexts in graph mode. - ops.get_default_graph().switch_to_thread_local() - - coord = coordinator.Coordinator(clean_stop_exception_types=(_RequestedStop,)) - - shared_variable_store = {} - - # TODO(isaprykin): Create these threads once instead of during every run() - # call. - threads = [] - for index, d in enumerate(distribution.worker_devices): - variable_creator_fn = shared_variable_creator.make_fn( - shared_variable_store, index) - t = MirroredStrategy._MirroredReplicaThread( # pylint: disable=protected-access - distribution, coord, d, variable_creator_fn, fn, - *values.select_device(d, args), **values.select_device(d, kwargs)) - threads.append(t) - - for t in threads: - t.start() - - # When `fn` starts `should_run` event is set on _MirroredReplicaThread - # (`MRT`) threads. The execution waits until - # `MRT.has_paused` is set, which indicates that either `fn` is - # complete or a `get_replica_context().merge_call()` is called. If `fn` is - # complete, then `MRT.done` is set to True. Otherwise, arguments - # of `get_replica_context().merge_call` from all paused threads are grouped - # and the `merge_fn` is performed. Results of the - # `get_replica_context().merge_call` are then set to `MRT.merge_result`. - # Each such `get_replica_context().merge_call` call returns the - # `MRT.merge_result` for that thread when `MRT.should_run` event - # is reset again. Execution of `fn` resumes. - - try: - with coord.stop_on_exception(): - all_done = False - while not all_done and not coord.should_stop(): - done = [] - if run_concurrently: - for t in threads: - t.should_run.set() - for t in threads: - t.has_paused.wait() - t.has_paused.clear() - if coord.should_stop(): - return None - done.append(t.done) - else: - for t in threads: - t.should_run.set() - t.has_paused.wait() - t.has_paused.clear() - if coord.should_stop(): - return None - done.append(t.done) - if coord.should_stop(): - return None - all_done = all(done) - if not all_done: - if any(done): - raise RuntimeError("Some replicas made a different number of " - "replica_context().merge_call() calls.") - # get_replica_context().merge_call() case - merge_args = values.regroup({t.device: t.merge_args for t in threads}) - merge_kwargs = values.regroup( - {t.device: t.merge_kwargs for t in threads}) - # We capture the name_scope of the MRT when we call merge_fn - # to ensure that if we have opened a name scope in the MRT, - # it will be respected when executing the merge function. We only - # capture the name_scope from the first MRT and assume it is - # the same for all other MRTs. - mtt_captured_name_scope = threads[0].captured_name_scope - with ops.name_scope(mtt_captured_name_scope): - merge_result = threads[0].merge_fn(distribution, *merge_args, - **merge_kwargs) - for t in threads: - t.merge_result = values.select_device(t.device, merge_result) - finally: - for t in threads: - t.should_run.set() - coord.join(threads) - - return values.regroup({t.device: t.main_result for t in threads}) - - -def _reduce_non_distributed_value(distribution, aggregation, value, - destinations): - """Reduce a non-DistributedValue `value` to `destinations`.""" - if isinstance(value, values.DistributedValues): - raise ValueError("You are passing a `DistributedValue` to " - "`_reduce_non_distributed_value`, which is not allowed.") - - # If the same value is present on all replicas then the PerReplica value will - # be a single value. We also handle the case when `value` is a single value - # and equal to 0. - if value == 0: - return 0 - # If the aggregation type is MEAN or ONLY_FIRST_REPLICA, then this - # essentially means that the same value should be on all destinations. - if aggregation in ( - variable_scope.VariableAggregation.MEAN, - variable_scope.VariableAggregation.ONLY_FIRST_REPLICA): - return value - - cross_tower_ops_lib.validate_destinations(destinations) - # We do not support an aggregation type of SUM if the value is the same across - # all replicas. We call this as part of assign functions for MirroredVariables - # and summing up identical values across replicas is not clearly defined. - if (len(distribution.worker_devices) != 1 or - not cross_tower_ops_lib.check_destinations(destinations)): - raise ValueError("A non-DistributedValues value %s cannot be reduced with " - "the given aggregation %s." % (value, aggregation)) - # TODO(anjalisridhar): Moves these methods to a device utility file? - devices = cross_tower_ops_lib.get_devices_from(destinations) - if len(devices) == 1: - with ops.device(devices[0]): - return array_ops.identity(value) - else: - value_updates = {} - for d in devices: - with ops.device(d): - value_updates[d] = array_ops.identity(value) - return values.Mirrored(value_updates) - - -def _create_mirrored_variable(devices, real_mirrored_creator, *args, **kwargs): # pylint: disable=g-missing-docstring - # Figure out what collections this variable should be added to. - # We'll add the MirroredVariable to those collections instead. - collections = kwargs.pop("collections", None) - if collections is None: - collections = [ops.GraphKeys.GLOBAL_VARIABLES] - kwargs["collections"] = [] - - # Get synchronization value - synchronization = kwargs.get("synchronization", - variable_scope.VariableSynchronization.ON_WRITE) - if synchronization == variable_scope.VariableSynchronization.NONE: - raise ValueError("`NONE` variable synchronization mode is not " - "supported with `Mirrored` distribution strategy. Please" - " change the `synchronization` for variable: " + - kwargs["name"]) - elif synchronization == variable_scope.VariableSynchronization.ON_READ: - # Variables that are to be synced on read are replica local. - is_replica_local = True - kwargs["trainable"] = False - elif (synchronization == variable_scope.VariableSynchronization.ON_WRITE or - synchronization == variable_scope.VariableSynchronization.AUTO): - # `AUTO` synchronization for `MirroredStrategy` is `ON_WRITE`. - is_replica_local = False - else: - raise ValueError("Invalid variable synchronization mode: " + - synchronization + " for variable: " + kwargs["name"]) - - # Get aggregation value - aggregation = kwargs.pop("aggregation", - variable_scope.VariableAggregation.NONE) - if aggregation not in ( - variable_scope.VariableAggregation.NONE, - variable_scope.VariableAggregation.SUM, - variable_scope.VariableAggregation.MEAN, - variable_scope.VariableAggregation.ONLY_FIRST_REPLICA - ): - raise ValueError("Invalid variable aggregation mode: " + aggregation + - " for variable: " + kwargs["name"]) - - # Ignore user-specified caching device, not needed for mirrored variables. - kwargs.pop("caching_device", None) - - # TODO(josh11b,apassos): It would be better if variable initialization - # was never recorded on the tape instead of having to do this manually - # here. - with tape.stop_recording(): - index = real_mirrored_creator(devices, *args, **kwargs) - - if is_replica_local: - result = values.ReplicaLocalVariable( - index, index[devices[0]], aggregation) - else: - result = values.MirroredVariable(index, index[devices[0]], aggregation) - - # Add the wrapped variable to the requested collections. - # The handling of eager mode and the global step matches - # ResourceVariable._init_from_args(). - if not context.executing_eagerly(): - g = ops.get_default_graph() - # If "trainable" is True, next_creator() will add the member variables - # to the TRAINABLE_VARIABLES collection, so we manually remove - # them and replace with the MirroredVariable. We can't set - # "trainable" to False for next_creator() since that causes functions - # like implicit_gradients to skip those variables. - if kwargs.get("trainable", True): - collections.append(ops.GraphKeys.TRAINABLE_VARIABLES) - l = g.get_collection_ref(ops.GraphKeys.TRAINABLE_VARIABLES) - for v in index.values(): - if v in l: - l.remove(v) - g.add_to_collections(collections, result) - elif ops.GraphKeys.GLOBAL_STEP in collections: - ops.add_to_collections(ops.GraphKeys.GLOBAL_STEP, result) - - return result +# pylint: disable=protected-access,invalid-name +_call_for_each_replica = mirrored_strategy._call_for_each_replica +_reduce_non_distributed_value = mirrored_strategy._reduce_non_distributed_value +_create_mirrored_variable = mirrored_strategy._create_mirrored_variable +all_local_devices = mirrored_strategy.all_local_devices +CoreMirroredStrategy = mirrored_strategy.MirroredStrategy +CoreMirroredExtended = mirrored_strategy.MirroredExtended +# pylint: enable=protected-access,invalid-name class MirroredStrategy(distribute_lib.DistributionStrategy): """Mirrors vars to distribute across multiple devices and machines. + *** contrib version *** + This strategy uses one replica per device and sync replication for its multi-GPU version. @@ -353,468 +95,66 @@ class MirroredStrategy(distribute_lib.DistributionStrategy): cross_device_ops=None, auto_shard_dataset=False, cross_tower_ops=None): - super(MirroredStrategy, self).__init__() - assert not (cross_device_ops and cross_tower_ops) - self._cross_tower_ops = cross_device_ops or cross_tower_ops - self._auto_shard_dataset = auto_shard_dataset - # Remember num GPUs which might be needed by `configure` method. if num_gpus is not None and num_gpus_per_worker is not None: raise ValueError( "You cannot specify both `num_gpus` and `num_gpus_per_worker`.") - if num_gpus is not None: - self._num_gpus = num_gpus - else: - self._num_gpus = num_gpus_per_worker - - self._initialize_local(self._num_gpus, devices) - - def _initialize_local(self, num_gpus, devices): - """Initializes the object for local training.""" - self._cluster_spec = None - # Convert `num_gpus` into `devices`, shouldn't specify both. - if devices is None: - if num_gpus is None: - num_gpus = context.num_gpus() - if num_gpus == 0: - devices = ["/device:CPU:0"] - else: - devices = ["/device:GPU:%d" % d for d in range(num_gpus)] - elif num_gpus is not None: - raise ValueError("Must only specify one of `devices` and `num_gpus`.") - self._num_gpus = num_gpus - # TODO(yuefengz): consider setting the default device. - - assert devices, "Must specify at least one device." - assert len(set(devices)) == len(devices), ( - "No duplicates allowed in `devices` argument.") - # TODO(josh11b): Require at least 2 devices? - self._devices = [device_util.resolve(d) for d in devices] - self._canonical_device_set = set(self._devices) - self._device_index = values.PerReplica( - {d: i for i, d in enumerate(devices)}) - - def _initialize_multi_worker(self, num_gpus, cluster_spec): - """Initializes the object for multi-worker training.""" - cluster_spec = multi_worker_util.normalize_cluster_spec(cluster_spec) - self._cluster_spec = cluster_spec - - self._workers = [] - for job in ["chief", "worker"]: - for task in range(len(cluster_spec.as_dict().get(job, []))): - self._workers.append("/job:%s/task:%d" % (job, task)) - if num_gpus is None: - raise ValueError("`num_gpus` is required if `cluster_spec` is given.") - if num_gpus > 0: - self._worker_devices = [ - (worker, [ - device_util.canonicalize(worker + "/device:GPU:%d" % gpu) - for gpu in range(num_gpus) - ]) for worker in self._workers - ] + num_gpus = num_gpus_per_worker + extended = MirroredExtended(self, devices, num_gpus, + cross_device_ops or cross_tower_ops, + auto_shard_dataset) + super(MirroredStrategy, self).__init__(extended) + + +class MirroredExtended(CoreMirroredExtended): + """Implementation of (contrib) MirroredStrategy.""" + + def __init__(self, + container_strategy, + devices=None, + num_gpus_per_worker=None, + cross_device_ops=None, + auto_shard_dataset=False): + if devices is None: + devices = mirrored_strategy.all_local_devices(num_gpus_per_worker) + elif num_gpus_per_worker is not None: + raise ValueError( + "Must only specify one of `devices` and `num_gpus_per_worker`.") + super(MirroredExtended, self).__init__(container_strategy, devices, + cross_device_ops) + self._auto_shard_dataset = auto_shard_dataset + + def _make_dataset_iterator(self, dataset): + """Make iterator from dataset without splitting the batch. + + This implementation is different than the one in + `tf.distribute.MirroredStrategy` for purposes of backward compatibility. + We treat the incoming dataset's batch size as per replica batch size. + + Args: + dataset: `tf.data.Dataset` for input. + Returns: + An `InputIterator` which returns inputs for each step of the computation. + """ + if self._local_mode: + worker = device_util.canonicalize("/device:CPU:0") + worker_device_pairs = [(worker, self._devices)] else: - self._worker_devices = [ - (worker, [device_util.canonicalize(worker, "/device:CPU:0")]) - for worker in self._workers - ] + worker_device_pairs = self._worker_devices + return values.DatasetIterator(dataset, worker_device_pairs) - devices = nest.flatten([l for _, l in self._worker_devices]) - - # Setting `_default_device` will add a device scope in the - # distribution.scope. We set the default device to the first worker. When - # users specify device under distribution.scope by - # with tf.device("/cpu:0"): - # ... - # their ops will end up on the cpu device of its first worker, e.g. - # "/job:worker/task:0/device:CPU:0". Note this is not used in replica mode. - self._default_device = self._workers[0] - - assert devices, "Must specify at least one device." - assert len(set(devices)) == len(devices), ( - "No duplicates allowed in `devices` argument.") - # TODO(josh11b): Require at least 2 devices? - self._devices = [device_util.resolve(d) for d in devices] - self._canonical_device_set = set(self._devices) - self._device_index = values.PerReplica( - {d: i for i, d in enumerate(devices)}) - - def _create_variable(self, next_creator, *args, **kwargs): - """Create a mirrored variable. See `DistributionStrategy.scope`.""" - colocate_with = kwargs.pop("colocate_with", None) - devices = self._get_devices_from(colocate_with) - - def _real_mirrored_creator(devices, *args, **kwargs): # pylint: disable=g-missing-docstring - index = {} - for i, d in enumerate(devices): - with ops.device(d): - if i > 0: - # Give replicas meaningful distinct names: - var0name = index[devices[0]].name.split(":")[0] - # We append a / to variable names created on replicas with id > 0 to - # ensure that we ignore the name scope and instead use the given - # name as the absolute name of the variable. - kwargs["name"] = "%s/replica_%d/" % (var0name, i) - # Initialize replicas with the same value: - def initial_value_fn(device=d): - if context.executing_eagerly(): - init_value = index[devices[0]].value() - return array_ops.identity(init_value) - else: - with ops.device(device): - init_value = index[devices[0]].initial_value - return array_ops.identity(init_value) - kwargs["initial_value"] = initial_value_fn - with context.context().device_policy(context.DEVICE_PLACEMENT_SILENT): - # Don't record operations (e.g. other variable reads) during - # variable creation. - with tape.stop_recording(): - v = next_creator(*args, **kwargs) - assert not isinstance(v, values.DistributedVariable) - index[d] = v - return index - - return _create_mirrored_variable(devices, _real_mirrored_creator, *args, - **kwargs) - - def distribute_dataset(self, dataset_fn): - if self._cluster_spec: - return values.MultiWorkerDataset( - partial(self._call_dataset_fn, dataset_fn), self._worker_devices, - auto_shard=self._auto_shard_dataset) - else: + def _distribute_dataset(self, dataset_fn): + if self._local_mode: return values.PerReplicaDataset( self._call_dataset_fn(dataset_fn), self._devices) - - # TODO(priyag): Deal with OutOfRange errors once b/111349762 is fixed. - def _run_steps_on_dataset(self, fn, iterator, iterations, - initial_loop_values=None): - if initial_loop_values is None: - initial_loop_values = {} - initial_loop_values = nest.flatten(initial_loop_values) - - ctx = values.MultiStepContext() - def body(i, *args): - """A wrapper around `fn` to create the while loop body.""" - del args - fn_inputs = iterator.get_next() - if not isinstance(fn_inputs, tuple): - fn_inputs = (fn_inputs,) - fn_result = fn(ctx, *fn_inputs) - for (name, output) in ctx.last_step_outputs.items(): - # Convert all outputs to tensors, potentially from `DistributedValues`. - ctx.last_step_outputs[name] = self.unwrap(output) - flat_last_step_outputs = nest.flatten(ctx.last_step_outputs) - with ops.control_dependencies([fn_result]): - return [i + 1] + flat_last_step_outputs - - # We capture the control_flow_context at this point, before we run `fn` - # inside a while_loop. This is useful in cases where we might need to exit - # these contexts and get back to the outer context to do some things, for - # e.g. create an op which should be evaluated only once at the end of the - # loop on the host. One such usage is in creating metrics' value op. - self._outer_control_flow_context = ( - ops.get_default_graph()._get_control_flow_context()) # pylint: disable=protected-access - - cond = lambda i, *args: i < iterations - i = constant_op.constant(0) - loop_result = control_flow_ops.while_loop( - cond, body, [i] + initial_loop_values, name="", - parallel_iterations=1, back_prop=False, swap_memory=False, - return_same_structure=True) - del self._outer_control_flow_context - - ctx.run_op = control_flow_ops.group(loop_result) - - # Convert the last_step_outputs from a list to the original dict structure - # of last_step_outputs. - last_step_tensor_outputs = loop_result[1:] - last_step_tensor_outputs_dict = nest.pack_sequence_as( - ctx.last_step_outputs, last_step_tensor_outputs) - - for (name, aggregation) in ctx._last_step_outputs_aggregations.items(): # pylint: disable=protected-access - output = last_step_tensor_outputs_dict[name] - # For outputs that have already been aggregated, wrap them in a Mirrored - # container, else in a PerReplica container. - if aggregation is variables_lib.VariableAggregation.NONE: - last_step_tensor_outputs_dict[name] = values.regroup( - {d: t for d, t in zip(self._devices, output)}, values.PerReplica) - else: - assert len(output) == 1 - last_step_tensor_outputs_dict[name] = output[0] - - ctx._set_last_step_outputs(last_step_tensor_outputs_dict) # pylint: disable=protected-access - return ctx - - def _broadcast(self, tensor, destinations): - # TODO(josh11b): In eager mode, use one thread per device, or async mode. - return self._get_cross_tower_ops().broadcast(tensor, destinations or - self._devices) - - def _call_for_each_replica(self, fn, args, kwargs): - return _call_for_each_replica(self, fn, args, kwargs) - - def configure(self, - session_config=None, - cluster_spec=None, - task_type=None, - task_id=None): - del task_type, task_id - - if session_config: - session_config.isolate_session_state = True - - if cluster_spec: - self._initialize_multi_worker(self._num_gpus, cluster_spec) - - if self._cross_tower_ops is None: - if self._cluster_spec: - # It currently cannot detect the toplogy of remote workers. So we - # hard-code the multi-worker all-reduce algorithm for now. - if len(self._workers) == 1: - # The default is "nccl". - self._cross_tower_ops = cross_tower_ops_lib.AllReduceCrossDeviceOps() - else: - # The default is hierarchical reduce and broadcast. - self._cross_tower_ops = cross_tower_ops_lib.MultiWorkerAllReduce( - self._workers, self._num_gpus) - else: - self._cross_tower_ops = cross_tower_ops_lib.choose_the_best( - self._devices, session_config=session_config) - - def _get_cross_tower_ops(self): - if self._cross_tower_ops is None: - self._cross_tower_ops = ( - cross_tower_ops_lib.ReductionToOneDeviceCrossDeviceOps()) - return self._cross_tower_ops - - def _reduce(self, aggregation, value, destinations): - assert not isinstance(value, values.Mirrored) - if not isinstance(value, values.DistributedValues): - # This function handles reducing values that are not PerReplica or - # Mirrored values. For example, the same value could be present on all - # replicas in which case `value` would be a single value or value could - # be 0. - return _reduce_non_distributed_value(self, aggregation, value, - destinations) - if aggregation == variable_scope.VariableAggregation.ONLY_FIRST_REPLICA: - value = value.get(self._devices[0]) - if isinstance(value, (int, float)): - return value - return self.broadcast(value, destinations) - return self._get_cross_tower_ops().reduce( - aggregation, value, destinations=destinations) - - def _batch_reduce(self, aggregation, value_destination_pairs): - if aggregation == variable_scope.VariableAggregation.ONLY_FIRST_REPLICA: - return [self.broadcast(v.get(self._devices[0]), d) - for v, d in value_destination_pairs] - return self._get_cross_tower_ops().batch_reduce(aggregation, - value_destination_pairs) - - def _update(self, var, options, fn, *args, **kwargs): - # TODO(josh11b): In eager mode, use one thread per device. - assert isinstance(var, values.DistributedVariable) - should_group = options.pop("grouped") - assert not options # Validate that we are processing all of the options. - updates = {} - for d, v in var._index.items(): # pylint: disable=protected-access - name = "update_%d" % self._device_index.get(d) - with ops.device(d), distribute_lib.UpdateContext(d), ops.name_scope(name): - # If args and kwargs are not mirrored, the value is returned as is. - updates[d] = fn(v, - *values.select_device_mirrored(d, args), - **values.select_device_mirrored(d, kwargs)) - return values.update_regroup(self, updates, should_group) - - def _update_non_slot(self, colocate_with, options, fn, *args, **kwargs): - assert isinstance(colocate_with, list) - should_group = options.pop("grouped") - assert not options # Validate that we are processing all of the options. - # TODO(josh11b): In eager mode, use one thread per device. - updates = {} - for d in colocate_with: - name = "update_%d" % self._device_index.get(d) - with ops.device(d), distribute_lib.UpdateContext(d), ops.name_scope(name): - updates[d] = fn(*values.select_device_mirrored(d, args), - **values.select_device_mirrored(d, kwargs)) - return values.update_regroup(self, updates, should_group) - - def read_var(self, replica_local_var): - """Read the aggregate value of a replica-local variable.""" - if isinstance(replica_local_var, values.ReplicaLocalVariable): - return replica_local_var._get_cross_replica() # pylint: disable=protected-access - assert isinstance(replica_local_var, values.Mirrored) - return array_ops.identity(replica_local_var.get()) - - def _unwrap(self, val): - if isinstance(val, values.DistributedValues): - # Return in a deterministic order. - if set(val.devices) == self._canonical_device_set: - return [val.get(device=d) for d in self._devices] - return [val.get(device=d) for d in sorted(val.devices)] - return [val] - - def value_container(self, val): - return values.value_container(val) - - @property - def num_replicas(self): - return len(self._devices) - - @property - def num_replicas_in_sync(self): - return len(self._devices) - - def _worker_device_index(self): - return self._device_index - - @property - def worker_devices(self): - # Make a copy to prevent users from accidentally mutating our copy. - return list(self._devices) - - @property - def parameter_devices(self): - return list(self._devices) - - @property - def between_graph(self): - return False - - @property - def should_init(self): - return True - - @property - def should_checkpoint(self): - return True - - @property - def should_save_summary(self): - return True - - def non_slot_devices(self, var_list): - del var_list - return list(self._devices) - - def _get_devices_from(self, colocate_with=None): - if colocate_with is None: - return self._devices else: - return cross_tower_ops_lib.get_devices_from(colocate_with) - - class _MirroredReplicaThread(threading.Thread): - """A thread that runs() a function on a device.""" - - def __init__(self, dist, coord, device, variable_creator_fn, fn, *args, - **kwargs): - super(MirroredStrategy._MirroredReplicaThread, self).__init__() # pylint: disable=protected-access - self.coord = coord - self.distribution = dist - self.device = device - self.replica_id = dist.worker_devices.index(device) - self.variable_creator_fn = variable_creator_fn - # State needed to run and return the results of `fn`. - self.main_fn = fn - self.main_args = args - self.main_kwargs = kwargs - self.main_result = None - self.done = False - # State needed to run the next merge_call() (if any) requested via - # ReplicaContext. - self.merge_fn = None - self.merge_args = None - self.merge_kwargs = None - self.merge_result = None - self.captured_name_scope = None - # We use a thread.Event for the main thread to signal when this - # thread should start running (`should_run`), and another for - # this thread to transfer control back to the main thread - # (`has_paused`, either when it gets to a - # `get_replica_context().merge_call` or when `fn` returns). In - # either case the event starts cleared, is signaled by calling - # set(). The receiving thread waits for the signal by calling - # wait() and then immediately clearing the event using clear(). - self.should_run = threading.Event() - self.has_paused = threading.Event() - # These fields have to do with inheriting various contexts from the - # parent thread: - # pylint: disable=protected-access - self.context_mode = context.context()._eager_context.mode - if not context.context()._context_handle: - context.context()._initialize_handle_and_devices() - self.context_device_policy = ( - pywrap_tensorflow.TFE_ContextGetDevicePlacementPolicy( - context.context()._context_handle)) - self.graph = ops.get_default_graph() - self._variable_creator_stack = self.graph._variable_creator_stack[:] - self._captured_var_scope = variable_scope.get_variable_scope() - # Adding a "/" at end lets us re-enter this scope later. - self._name_scope = self.graph.get_name_scope() - if self._name_scope: - self._name_scope += "/" - if self.replica_id > 0: - if not self._name_scope: - self._name_scope = "" - self._name_scope += "replica_%d/" % self.replica_id - - def run(self): - # pylint: disable=protected-access - self.graph._variable_creator_stack = self._variable_creator_stack - self.should_run.wait() - self.should_run.clear() - try: - if self.coord.should_stop(): - return - with self.coord.stop_on_exception(), \ - context.context()._mode(self.context_mode), \ - context.context().device_policy(self.context_device_policy), \ - _enter_graph(self.graph), \ - MirroredReplicaContext(self.distribution, self.replica_id), \ - ops.device(self.device), \ - ops.name_scope(self._name_scope), \ - variable_scope.variable_scope( - self._captured_var_scope, reuse=self.replica_id > 0), \ - variable_scope.variable_creator_scope(self.variable_creator_fn): - self.main_result = self.main_fn(*self.main_args, **self.main_kwargs) - self.done = True - finally: - self.has_paused.set() - - -class MirroredReplicaContext(distribute_lib.ReplicaContext): - """ReplicaContext used in MirroredStrategy.call_for_each_replica(). - - Opened in `_MirroredReplicaThread`, to allow the user to invoke - `MirroredStrategy`'s specific implementation of `merge_call()`, - which works by delegating the function and its arguments to - the main thread (the one that invoked - `MirroredStrategy.call_for_each_replica()`). - """ - - def _merge_call(self, fn, args, kwargs): - """Delegate to the main thread to actually perform merge_call().""" - t = threading.current_thread() # a _MirroredReplicaThread - t.merge_fn = fn - t.merge_args = args - t.merge_kwargs = kwargs - t.captured_name_scope = t.graph.get_name_scope() - # Adding a "/" at end lets us re-enter this scope later. - if t.captured_name_scope: - t.captured_name_scope += "/" - t.has_paused.set() - t.should_run.wait() - t.should_run.clear() - if t.coord.should_stop(): - raise _RequestedStop() - return t.merge_result + return values.MultiWorkerDataset( + functools.partial(self._call_dataset_fn, dataset_fn), + self._worker_devices, + auto_shard=self._auto_shard_dataset) + # TODO(priyag): Delete this once all strategies use global batch size. @property - def device(self): - raise RuntimeError("Use .devices instead") - - @property - def devices(self): - distribute_lib.require_replica_context(self) - return [self._distribution_strategy.worker_devices[self._replica_id]] + def _global_batch_size(self): + return False diff --git a/tensorflow/contrib/distribute/python/mirrored_strategy_multigpu_test.py b/tensorflow/contrib/distribute/python/mirrored_strategy_multigpu_test.py index 1fd18e09c01..66512f983e1 100644 --- a/tensorflow/contrib/distribute/python/mirrored_strategy_multigpu_test.py +++ b/tensorflow/contrib/distribute/python/mirrored_strategy_multigpu_test.py @@ -20,22 +20,27 @@ from __future__ import print_function import sys +from absl.testing import parameterized import numpy as np +from tensorflow.contrib.distribute.python import combinations from tensorflow.contrib.distribute.python import mirrored_strategy from tensorflow.contrib.distribute.python import multi_worker_test_base from tensorflow.contrib.distribute.python import strategy_test_lib -from tensorflow.contrib.distribute.python import values from tensorflow.core.protobuf import config_pb2 from tensorflow.python.data.ops import dataset_ops +from tensorflow.python.distribute import device_util +from tensorflow.python.distribute import distribution_strategy_context as ds_context +from tensorflow.python.distribute import reduce_util +from tensorflow.python.distribute import values from tensorflow.python.eager import backprop from tensorflow.python.eager import context from tensorflow.python.eager import function from tensorflow.python.eager import test from tensorflow.python.framework import constant_op +from tensorflow.python.framework import func_graph from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops -from tensorflow.python.framework import test_util from tensorflow.python.keras.engine import training as keras_training from tensorflow.python.keras.layers import core as keras_core from tensorflow.python.layers import core @@ -46,8 +51,6 @@ from tensorflow.python.ops import rnn_cell_impl from tensorflow.python.ops import state_ops from tensorflow.python.ops import variable_scope from tensorflow.python.ops import variables -from tensorflow.python.training import device_util -from tensorflow.python.training import distribution_strategy_context from tensorflow.python.training import gradient_descent from tensorflow.python.training import optimizer as optimizer_lib from tensorflow.python.training import server_lib @@ -56,248 +59,229 @@ from tensorflow.python.training import server_lib GPU_TEST = "test_gpu" in sys.argv[0] -class MirroredTwoDeviceDistributionTest(strategy_test_lib.DistributionTestBase): +@combinations.generate(combinations.combine( + distribution=[ + combinations.mirrored_strategy_with_gpu_and_cpu, + combinations.mirrored_strategy_with_two_gpus, + combinations.core_mirrored_strategy_with_gpu_and_cpu, + combinations.core_mirrored_strategy_with_two_gpus], + mode=["graph", "eager"])) +class MirroredTwoDeviceDistributionTest(strategy_test_lib.DistributionTestBase, + parameterized.TestCase): - def _get_distribution_strategy(self): - devices = ["/device:CPU:0", "/device:GPU:0"] - if GPU_TEST: - self.assertGreater(context.num_gpus(), 0) - if context.num_gpus() > 1: - devices = ["/device:GPU:0", "/device:GPU:1"] - print(self.id().split(".")[-1], "devices:", ", ".join(devices)) - return mirrored_strategy.MirroredStrategy(devices) + def testMinimizeLoss(self, distribution): + if context.executing_eagerly(): + self._test_minimize_loss_eager(distribution) + else: + self._test_minimize_loss_graph(distribution) - def testMinimizeLossEager(self): - if not GPU_TEST: - self.skipTest("Not GPU test") - self._test_minimize_loss_eager(self._get_distribution_strategy()) + def testReplicaId(self, distribution): + self._test_replica_id(distribution) - def testMinimizeLossGraph(self): - soft_placement = not GPU_TEST - print("testMinimizeLossGraph soft_placement:", soft_placement) - self._test_minimize_loss_graph( - self._get_distribution_strategy(), soft_placement=soft_placement) + def testNumReplicasInSync(self, distribution): + self.assertEqual(2, distribution.num_replicas_in_sync) - def testDeviceIndex(self): - if not GPU_TEST: - self.skipTest("Not GPU test") - self._test_device_index(self._get_distribution_strategy()) + def testCallAndMergeExceptions(self, distribution): + self._test_call_and_merge_exceptions(distribution) - def testReplicaId(self): - if not GPU_TEST: - self.skipTest("Not GPU test") - self._test_replica_id(self._get_distribution_strategy()) - - def testNumReplicas(self): - if not GPU_TEST: - self.skipTest("Not GPU test") - self.assertEqual(2, self._get_distribution_strategy().num_replicas) - - def testNumReplicasInSync(self): - if not GPU_TEST: - self.skipTest("Not GPU test") - self.assertEqual(2, self._get_distribution_strategy(). - num_replicas_in_sync) - - @test_util.run_in_graph_and_eager_modes - def testCallAndMergeExceptions(self): - if not GPU_TEST: - self.skipTest("Not GPU test") - self._test_call_and_merge_exceptions(self._get_distribution_strategy()) - - @test_util.run_in_graph_and_eager_modes - def testRunRegroupError(self): - - def run_fn(device_id): + def testRunRegroupError(self, distribution): + def run_fn(): + replica_id = int(self.evaluate(_replica_id())) # Generates a list with different lengths on different devices. # Will fail in _regroup() (if more than one device). - return list(range(device_id)) + return list(range(replica_id)) - dist = self._get_distribution_strategy() - with dist.scope(), self.assertRaises(AssertionError): - dist.call_for_each_replica(run_fn, args=(dist.worker_device_index,)) + with distribution.scope(), self.assertRaises(AssertionError): + distribution.extended.call_for_each_replica(run_fn) - @test_util.run_in_graph_and_eager_modes - def testReduceToCpu(self): - if not GPU_TEST: - self.skipTest("Not GPU test") + def testReduceToCpu(self, distribution): + with distribution.scope(): + result = distribution.extended.call_for_each_replica(_replica_id) + reduced = distribution.reduce(reduce_util.ReduceOp.SUM, result) + expected = sum(range(distribution.num_replicas_in_sync)) + self.assertEqual(expected, self.evaluate(reduced)) - def run_fn(device_id): - return device_id + def testMakeInputFnIterator(self, distribution): + dataset_fn = lambda: dataset_ops.Dataset.range(10) + expected_values = [[i, i+1] for i in range(0, 10, 2)] - dist = self._get_distribution_strategy() - with dist.scope(): - result = dist.call_for_each_replica( - run_fn, args=(dist.worker_device_index,)) - reduced = dist.reduce( - variable_scope.VariableAggregation.SUM, - result, - destinations="/device:CPU:0") - unwrapped = dist.unwrap(reduced) - self.assertEqual(1, len(unwrapped)) - expected = sum(range(len(dist.worker_devices))) - self.assertEqual(expected, self.evaluate(unwrapped[0])) + input_fn = self._input_fn_to_test_input_context( + dataset_fn, + expected_num_replicas_in_sync=2, + expected_num_input_pipelines=1, + expected_input_pipeline_id=0) + iterator = distribution.make_input_fn_iterator(input_fn) + self._test_input_fn_iterator(iterator, distribution.extended.worker_devices, + expected_values) - @test_util.run_in_graph_and_eager_modes - def testReduceOnlyFirstReplicaUpdates(self): - if not GPU_TEST: - self.skipTest("Not GPU test") - - def run_fn(device_id): - return constant_op.constant(3 + 5 * device_id) - - dist = self._get_distribution_strategy() - with dist.scope(): - result = dist.call_for_each_replica( - run_fn, args=(dist.worker_device_index,)) - reduced = dist.reduce( - variable_scope.VariableAggregation.ONLY_FIRST_REPLICA, - result, - destinations="/device:CPU:0") - unwrapped = dist.unwrap(reduced) - self.assertEqual(1, len(unwrapped)) - self.assertEqual(3, self.evaluate(unwrapped[0])) - - @test_util.run_in_graph_and_eager_modes() - def testReduceToMultipleDestinations(self): - if not GPU_TEST: - self.skipTest("Not GPU test") - - devices = ["/device:GPU:0"] - if GPU_TEST: - self.assertGreater(context.num_gpus(), 0) - print(self.id().split(".")[-1], "devices:", ", ".join(devices)) - - dist = mirrored_strategy.MirroredStrategy(devices) - with dist.scope(): - reduced = dist.reduce( - variable_scope.VariableAggregation.SUM, - 1.0, - destinations=["/device:CPU:0", "/device:GPU:0"]) - unwrapped = dist.unwrap(reduced) - self.assertEqual(2, len(unwrapped)) - self.assertEqual(1.0, self.evaluate(unwrapped[0])) + def testGlobalStepUpdate(self, distribution): + self._test_global_step_update(distribution) +def one_device_combinations(): + return combinations.combine( + distribution=[ + combinations.mirrored_strategy_with_one_cpu, + combinations.mirrored_strategy_with_one_gpu, + combinations.core_mirrored_strategy_with_one_cpu, + combinations.core_mirrored_strategy_with_one_gpu], + mode=["graph", "eager"]) + + +class MirroredOneDeviceDistributionTest( + strategy_test_lib.DistributionTestBase, + parameterized.TestCase): + + @combinations.generate(one_device_combinations()) + def testMinimizeLoss(self, distribution): + if context.executing_eagerly(): + self._test_minimize_loss_eager(distribution) + else: + self._test_minimize_loss_graph(distribution) + + @combinations.generate(one_device_combinations()) + def testReplicaId(self, distribution): + self._test_replica_id(distribution) + + @combinations.generate(one_device_combinations()) + def testCallAndMergeExceptions(self, distribution): + self._test_call_and_merge_exceptions(distribution) + + +class MirroredStrategyVariableCreatorStackTest( + test.TestCase, parameterized.TestCase): + + @combinations.generate(combinations.combine( + distribution=[combinations.mirrored_strategy_with_gpu_and_cpu, + combinations.core_mirrored_strategy_with_gpu_and_cpu], + mode=["graph"])) + def testCreatorStacksAreThreadLocal(self, distribution): + def model_fn(): + replica_id_str = str(self.evaluate(_replica_id())) + + def thread_creator_fn(next_creator, *args, **kwargs): + return next_creator(*args, **kwargs) + ":thread_" + replica_id_str + + with variable_scope.variable_creator_scope(thread_creator_fn): + # Create a variable in this scope. + v = variable_scope.variable(1.0) + + # This will pause the current thread, and execute the other thread. + ds_context.get_replica_context().merge_call(lambda _: _) + return v + + def main_thread_creator(next_creator, *args, **kwargs): + # We are not using the underlying next_creator for test purposes. + del next_creator, args, kwargs + return "main_thread" + + with context.graph_mode(), \ + distribution.scope(), \ + variable_scope.variable_creator_scope(main_thread_creator): + result = distribution.extended.call_for_each_replica(model_fn) + result = distribution.unwrap(result) + expected = ["main_thread:thread_0", "main_thread:thread_1"] + self.assertEqual(expected, result) + + +@combinations.generate(combinations.combine( + distribution=[ + combinations.mirrored_strategy_with_gpu_and_cpu, + combinations.core_mirrored_strategy_with_gpu_and_cpu], + mode=["graph", "eager"])) class MirroredStrategyVariableCreationTest(test.TestCase): - config = config_pb2.ConfigProto() - config.allow_soft_placement = True + # TODO(priyag): Modify more tests to use this helper and check more + # properties. + def _test_mv_properties(self, var, name): + self.assertIsInstance(var, values.MirroredVariable) + self.assertEqual(name, var.name) + for d in var.devices: + self.assertEqual(d, var.get(d).device) - def _skip_eager_if_gpus_less_than(self, num_gpus): - if context.num_gpus() < num_gpus and context.executing_eagerly(): - self.skipTest("Enough GPUs not available for this test in eager mode.") + def testVariableInFuncGraph(self, distribution): + def model_fn(): + v = variable_scope.variable(2.0, name="bar") + ds_context.get_replica_context().merge_call(lambda _: _) + return v - @test_util.run_in_graph_and_eager_modes(config=config) - def testSingleVariable(self): - self._skip_eager_if_gpus_less_than(1) + with func_graph.FuncGraph("fg").as_default(), distribution.scope(): + v1 = variable_scope.variable(1.0, name="foo") + v2 = distribution.extended.call_for_each_replica(model_fn) + self._test_mv_properties(v1, "foo:0") + self._test_mv_properties(v2, "bar:0") + + def testSingleVariable(self, distribution): def model_fn(): # This variable should be created only once across the threads because of - # special variable_creator functions used by `dist.call_for_each_replica`. + # special variable_creator functions used by + # `distribution.extended.call_for_each_replica`. v = variable_scope.variable(1.0, name="foo") - distribution_strategy_context.get_replica_context().merge_call( - lambda _: _) + ds_context.get_replica_context().merge_call(lambda _: _) return v - dist = mirrored_strategy.MirroredStrategy( - ["/device:GPU:0", "/device:CPU:0"]) - - with dist.scope(): - result = dist.call_for_each_replica(model_fn) - self.assertIsInstance(result, values.MirroredVariable) - self.assertEquals("foo:0", result.name) - - @test_util.run_in_graph_and_eager_modes(config=config) - def testUnnamedVariable(self): - self._skip_eager_if_gpus_less_than(1) + with distribution.scope(): + result = distribution.extended.call_for_each_replica(model_fn) + self._test_mv_properties(result, "foo:0") + def testUnnamedVariable(self, distribution): def model_fn(): v = variable_scope.variable(1.0) - distribution_strategy_context.get_replica_context().merge_call( - lambda _: _) + ds_context.get_replica_context().merge_call(lambda _: _) return v - dist = mirrored_strategy.MirroredStrategy( - ["/device:GPU:0", "/device:CPU:0"]) - - with dist.scope(): - result = dist.call_for_each_replica(model_fn) - self.assertIsInstance(result, values.MirroredVariable) - # Default name of "Variable" will be used. - self.assertEquals("Variable:0", result.name) - - @test_util.run_in_graph_and_eager_modes(config=config) - def testMultipleVariables(self): - self._skip_eager_if_gpus_less_than(1) + with distribution.scope(): + result = distribution.extended.call_for_each_replica(model_fn) + self._test_mv_properties(result, "Variable:0") + def testMultipleVariables(self, distribution): def model_fn(): vs = [] for i in range(5): vs.append(variable_scope.variable(1.0, name="foo" + str(i))) - distribution_strategy_context.get_replica_context().merge_call( - lambda _: _) + ds_context.get_replica_context().merge_call(lambda _: _) return vs - dist = mirrored_strategy.MirroredStrategy( - ["/device:GPU:0", "/device:CPU:0"]) - - with dist.scope(): - result = dist.call_for_each_replica(model_fn) + with distribution.scope(): + result = distribution.extended.call_for_each_replica(model_fn) for i, v in enumerate(result): - self.assertIsInstance(v, values.MirroredVariable) - self.assertEquals("foo" + str(i) + ":0", v.name) - - @test_util.run_in_graph_and_eager_modes(config=config) - def testMultipleVariablesWithSameCanonicalName(self): - self._skip_eager_if_gpus_less_than(1) + self._test_mv_properties(v, "foo" + str(i) + ":0") + def testMultipleVariablesWithSameCanonicalName(self, distribution): def model_fn(): vs = [] vs.append(variable_scope.variable(1.0, name="foo/bar")) vs.append(variable_scope.variable(1.0, name="foo_1/bar")) vs.append(variable_scope.variable(1.0, name="foo_1/bar_1")) vs.append(variable_scope.variable(1.0, name="foo/bar_1")) - distribution_strategy_context.get_replica_context().merge_call( - lambda _: _) + ds_context.get_replica_context().merge_call(lambda _: _) return vs - dist = mirrored_strategy.MirroredStrategy( - ["/device:GPU:0", "/device:CPU:0"]) - - with dist.scope(): - result = dist.call_for_each_replica(model_fn) + with distribution.scope(): + result = distribution.extended.call_for_each_replica(model_fn) for v in result: self.assertIsInstance(v, values.MirroredVariable) - self.assertEquals(4, len(result)) - self.assertEquals("foo/bar:0", result[0].name) - self.assertEquals("foo_1/bar:0", result[1].name) - self.assertEquals("foo_1/bar_1:0", result[2].name) - self.assertEquals("foo/bar_1:0", result[3].name) + self.assertEqual(4, len(result)) + self.assertEqual("foo/bar:0", result[0].name) + self.assertEqual("foo_1/bar:0", result[1].name) + self.assertEqual("foo_1/bar_1:0", result[2].name) + self.assertEqual("foo/bar_1:0", result[3].name) - @test_util.run_in_graph_and_eager_modes(config=config) - def testVariableWithSameCanonicalNameAcrossThreads(self): - self._skip_eager_if_gpus_less_than(1) - - def model_fn(device_id): - v = variable_scope.variable(1.0, name="foo_" + str(device_id)) - distribution_strategy_context.get_replica_context().merge_call( - lambda _: _) + def testVariableWithSameCanonicalNameAcrossThreads(self, distribution): + def model_fn(): + replica_id = self.evaluate(_replica_id()) + v = variable_scope.variable(1.0, name="foo_" + str(replica_id)) + ds_context.get_replica_context().merge_call(lambda _: _) return v - dist = mirrored_strategy.MirroredStrategy( - ["/device:GPU:0", "/device:CPU:0"]) - - with dist.scope(): - result = dist.call_for_each_replica( - model_fn, args=(dist.worker_device_index,)) + with distribution.scope(): + result = distribution.extended.call_for_each_replica(model_fn) self.assertIsInstance(result, values.MirroredVariable) # The resulting mirrored variable will use the name from the first device. - self.assertEquals("foo_0:0", result.name) + self.assertEqual("foo_0:0", result.name) - @test_util.run_in_graph_and_eager_modes(config=config) - def testWithLayers(self): - self._skip_eager_if_gpus_less_than(1) + def testWithLayers(self, distribution): def model_fn(features): with variable_scope.variable_scope("common"): layer1 = core.Dense(1) @@ -305,17 +289,14 @@ class MirroredStrategyVariableCreationTest(test.TestCase): layer2 = core.Dense(1) layer2(features) # This will pause the current thread, and execute the other thread. - distribution_strategy_context.get_replica_context().merge_call( - lambda _: _) + ds_context.get_replica_context().merge_call(lambda _: _) layer3 = core.Dense(1) layer3(features) return [(layer1.kernel, layer1.bias), (layer2.kernel, layer2.bias), (layer3.kernel, layer3.bias)] - dist = mirrored_strategy.MirroredStrategy( - ["/device:GPU:0", "/device:CPU:0"]) - ds = dist.distribute_dataset( + ds = distribution.distribute_dataset( lambda: dataset_ops.Dataset.from_tensors([[1.]]).repeat(10)) if context.executing_eagerly(): iterator = ds.make_one_shot_iterator() @@ -325,26 +306,23 @@ class MirroredStrategyVariableCreationTest(test.TestCase): features = iterator.get_next() - with dist.scope(): - result = dist.call_for_each_replica(model_fn, args=(features,)) + with distribution.scope(): + result = distribution.extended.call_for_each_replica( + model_fn, args=(features,)) suffixes = ["", "_1", "_2"] for (kernel, bias), suffix in zip(result, suffixes): self.assertIsInstance(kernel, values.MirroredVariable) - self.assertEquals("common/dense" + suffix + "/kernel:0", kernel.name) + self.assertEqual("common/dense" + suffix + "/kernel:0", kernel.name) self.assertIsInstance(bias, values.MirroredVariable) - self.assertEquals("common/dense" + suffix + "/bias:0", bias.name) - - @test_util.run_in_graph_and_eager_modes(config=config) - def testWithVariableAndVariableScope(self): - self._skip_eager_if_gpus_less_than(1) + self.assertEqual("common/dense" + suffix + "/bias:0", bias.name) + def testWithVariableAndVariableScope(self, distribution): def model_fn(): v0 = variable_scope.variable(1.0, name="var0", aggregation=None) with variable_scope.variable_scope("common"): v1 = variable_scope.variable(1.0, name="var1") # This will pause the current thread, and execute the other thread. - distribution_strategy_context.get_replica_context().merge_call( - lambda _: _) + ds_context.get_replica_context().merge_call(lambda _: _) v2 = variable_scope.variable( 1.0, name="var2", @@ -358,37 +336,31 @@ class MirroredStrategyVariableCreationTest(test.TestCase): return v0, v1, v2, v3 - devices = ["/device:CPU:0", "/device:GPU:0"] - dist = mirrored_strategy.MirroredStrategy(devices) - with dist.scope(): + with distribution.scope(): v = variable_scope.variable(1.0, name="var-main0") - self.assertEquals("var-main0:0", v.name) + self.assertEqual("var-main0:0", v.name) - result = dist.call_for_each_replica(model_fn) - self.assertEquals(4, len(result)) + result = distribution.extended.call_for_each_replica(model_fn) + self.assertEqual(4, len(result)) v0, v1, v2, v3 = result self.assertIsInstance(v0, values.MirroredVariable) - self.assertEquals("var0:0", v0.name) + self.assertEqual("var0:0", v0.name) self.assertIsInstance(v1, values.MirroredVariable) - self.assertEquals("common/var1:0", v1.name) + self.assertEqual("common/var1:0", v1.name) self.assertIsInstance(v2, values.ReplicaLocalVariable) - self.assertEquals("common/var2:0", v2.name) - self.assertEquals(variable_scope.VariableAggregation.SUM, v2.aggregation) + self.assertEqual("common/var2:0", v2.name) + self.assertEqual(variable_scope.VariableAggregation.SUM, v2.aggregation) self.assertIsInstance(v3, values.MirroredVariable) - self.assertEquals("common/var3:0", v3.name) - self.assertEquals(variable_scope.VariableAggregation.MEAN, v3.aggregation) - - @test_util.run_in_graph_and_eager_modes(config=config) - def testWithGetVariableAndVariableScope(self): - self._skip_eager_if_gpus_less_than(1) + self.assertEqual("common/var3:0", v3.name) + self.assertEqual(variable_scope.VariableAggregation.MEAN, v3.aggregation) + def testWithGetVariableAndVariableScope(self, distribution): def model_fn(): v0 = variable_scope.get_variable("var0", [1]) with variable_scope.variable_scope("common"): v1 = variable_scope.get_variable("var1", [1]) # This will pause the current thread, and execute the other thread. - distribution_strategy_context.get_replica_context().merge_call( - lambda _: _) + ds_context.get_replica_context().merge_call(lambda _: _) v2 = variable_scope.get_variable( "var2", [1], synchronization=variable_scope.VariableSynchronization.ON_READ, @@ -400,33 +372,28 @@ class MirroredStrategyVariableCreationTest(test.TestCase): return v0, v1, v2, v3 - devices = ["/device:CPU:0", "/device:GPU:0"] - dist = mirrored_strategy.MirroredStrategy(devices) - with dist.scope(): + with distribution.scope(): with variable_scope.variable_scope("main"): v = variable_scope.get_variable("var-main0", [1]) - self.assertEquals("main/var-main0:0", v.name) + self.assertEqual("main/var-main0:0", v.name) - result = dist.call_for_each_replica(model_fn) - self.assertEquals(4, len(result)) + result = distribution.extended.call_for_each_replica(model_fn) + self.assertEqual(4, len(result)) v0, v1, v2, v3 = result self.assertIsInstance(v0, values.MirroredVariable) - self.assertEquals("main/var0:0", v0.name) + self.assertEqual("main/var0:0", v0.name) self.assertIsInstance(v1, values.MirroredVariable) - self.assertEquals("main/common/var1:0", v1.name) + self.assertEqual("main/common/var1:0", v1.name) self.assertIsInstance(v2, values.ReplicaLocalVariable) - self.assertEquals("main/common/var2:0", v2.name) - self.assertEquals(variable_scope.VariableAggregation.SUM, - v2.aggregation) + self.assertEqual("main/common/var2:0", v2.name) + self.assertEqual(variable_scope.VariableAggregation.SUM, + v2.aggregation) self.assertIsInstance(v3, values.MirroredVariable) - self.assertEquals("main/common/var3:0", v3.name) - self.assertEquals(variable_scope.VariableAggregation.MEAN, - v3.aggregation) - - @test_util.run_in_graph_and_eager_modes(config=config) - def testOnlyFirstReplicaUpdatesVariables(self): - self._skip_eager_if_gpus_less_than(1) + self.assertEqual("main/common/var3:0", v3.name) + self.assertEqual(variable_scope.VariableAggregation.MEAN, + v3.aggregation) + def testOnlyFirstReplicaUpdatesVariables(self, distribution): def create_fn(): aggregation = variable_scope.VariableAggregation.ONLY_FIRST_REPLICA v0 = variable_scope.variable( @@ -442,71 +409,73 @@ class MirroredStrategyVariableCreationTest(test.TestCase): return v0, v1 devices = ["/device:GPU:0", "/device:CPU:0"] - dist = mirrored_strategy.MirroredStrategy(devices) - with dist.scope(): - v0, v1 = dist.call_for_each_replica(create_fn) + with distribution.scope(): + v0, v1 = distribution.extended.call_for_each_replica(create_fn) self.evaluate(v0.initializer) self.assertEqual(2.0, self.evaluate(v0.get(devices[0]))) self.assertEqual(2.0, self.evaluate(v0.get(devices[1]))) - self.assertEqual(2.0, self.evaluate(dist.read_var(v0))) + self.assertEqual(2.0, self.evaluate(distribution.extended.read_var(v0))) self.evaluate(v1.initializer) self.assertEqual(3.0, self.evaluate(v1.get(devices[0]))) self.assertEqual(3.0, self.evaluate(v1.get(devices[1]))) - self.assertEqual(3.0, self.evaluate(dist.read_var(v1))) + self.assertEqual(3.0, self.evaluate(distribution.extended.read_var(v1))) + + def replica_id_plus_one(): + return math_ops.cast(_replica_id() + 1, dtype=dtypes.float32) # Update using the assign_add member function. - def update_member_fn(device_id): - update0 = v0.assign_add(5.0 * (device_id + 1)) - update1 = v1.assign_add(7.0 * (device_id + 1)) + def update_member_fn(): + update0 = v0.assign_add(5.0 * replica_id_plus_one()) + update1 = v1.assign_add(7.0 * replica_id_plus_one()) return update0, update1 - update0a, update1a = dist.call_for_each_replica( - update_member_fn, args=(dist.worker_device_index,)) + update0a, update1a = distribution.extended.call_for_each_replica( + update_member_fn) # Update "sync on read" variable. - self.evaluate(dist.group(update0a)) + self.evaluate(distribution.group(update0a)) self.assertEqual(2.0 + 5.0, self.evaluate(v0.get(devices[0]))) # Writes are not synchronized for "sync on read" variables, # so device[1] can end up with a different value. self.assertEqual(2.0 + 2*5.0, self.evaluate(v0.get(devices[1]))) # Always reads from device 0. - self.assertEqual(2.0 + 5.0, self.evaluate(dist.read_var(v0))) + self.assertEqual(2.0 + 5.0, self.evaluate( + distribution.extended.read_var(v0))) # Update "sync on write" variable. - self.evaluate(dist.group(update1a)) + self.evaluate(distribution.group(update1a)) self.assertEqual(3.0 + 7.0, self.evaluate(v1.get(devices[0]))) # Writes are synchronized for v1, only the argument to assign_add on # device[0] is used. self.assertEqual(3.0 + 7.0, self.evaluate(v1.get(devices[1]))) - self.assertEqual(3.0 + 7.0, self.evaluate(dist.read_var(v1))) + self.assertEqual(3.0 + 7.0, self.evaluate( + distribution.extended.read_var(v1))) # Update using state_ops.assign_add global function. - def update_state_ops_fn(device_id): - update0 = state_ops.assign_add(v0, 11.0 * (device_id + 1)) - update1 = state_ops.assign_add(v1, 13.0 * (device_id + 1)) + def update_state_ops_fn(): + update0 = state_ops.assign_add(v0, 11.0 * replica_id_plus_one()) + update1 = state_ops.assign_add(v1, 13.0 * replica_id_plus_one()) return update0, update1 - update0b, update1b = dist.call_for_each_replica( - update_state_ops_fn, args=(dist.worker_device_index,)) - self.evaluate(dist.group(update0b)) + update0b, update1b = distribution.extended.call_for_each_replica( + update_state_ops_fn) + self.evaluate(distribution.group(update0b)) # Update "sync on read" variable. self.assertEqual(2.0 + 5.0 + 11.0, self.evaluate(v0.get(devices[0]))) self.assertEqual(2.0 + 2*5.0 + 2*11.0, self.evaluate(v0.get(devices[1]))) - self.assertEqual(2.0 + 5.0 + 11.0, self.evaluate(dist.read_var(v0))) + self.assertEqual(2.0 + 5.0 + 11.0, self.evaluate( + distribution.extended.read_var(v0))) # Update "sync on write" variable. - self.evaluate(dist.group(update1b)) + self.evaluate(distribution.group(update1b)) self.assertEqual(3.0 + 7.0 + 13.0, self.evaluate(v1.get(devices[0]))) self.assertEqual(3.0 + 7.0 + 13.0, self.evaluate(v1.get(devices[1]))) - self.assertEqual(3.0 + 7.0 + 13.0, self.evaluate(dist.read_var(v1))) + self.assertEqual(3.0 + 7.0 + 13.0, self.evaluate( + distribution.extended.read_var(v1))) - @test_util.run_in_graph_and_eager_modes(config=config) - def testNoneSynchronizationWithGetVariable(self): - self._skip_eager_if_gpus_less_than(1) - devices = ["/device:CPU:0", "/device:GPU:0"] - dist = mirrored_strategy.MirroredStrategy(devices) - with dist.scope(): + def testNoneSynchronizationWithGetVariable(self, distribution): + with distribution.scope(): with self.assertRaisesRegexp( ValueError, "`NONE` variable synchronization mode is not " "supported with `Mirrored` distribution strategy. Please change " @@ -515,12 +484,8 @@ class MirroredStrategyVariableCreationTest(test.TestCase): "v", [1], synchronization=variable_scope.VariableSynchronization.NONE) - @test_util.run_in_graph_and_eager_modes(config=config) - def testNoneSynchronizationWithVariable(self): - self._skip_eager_if_gpus_less_than(1) - devices = ["/device:CPU:0", "/device:GPU:0"] - dist = mirrored_strategy.MirroredStrategy(devices) - with dist.scope(): + def testNoneSynchronizationWithVariable(self, distribution): + with distribution.scope(): with self.assertRaisesRegexp( ValueError, "`NONE` variable synchronization mode is not " "supported with `Mirrored` distribution strategy. Please change " @@ -530,23 +495,15 @@ class MirroredStrategyVariableCreationTest(test.TestCase): name="v", synchronization=variable_scope.VariableSynchronization.NONE) - @test_util.run_in_graph_and_eager_modes(config=config) - def testInvalidSynchronizationWithVariable(self): - self._skip_eager_if_gpus_less_than(1) - devices = ["/device:CPU:0", "/device:GPU:0"] - dist = mirrored_strategy.MirroredStrategy(devices) - with dist.scope(): + def testInvalidSynchronizationWithVariable(self, distribution): + with distribution.scope(): with self.assertRaisesRegexp( ValueError, "Invalid variable synchronization mode: Invalid for " "variable: v"): variable_scope.variable(1.0, name="v", synchronization="Invalid") - @test_util.run_in_graph_and_eager_modes(config=config) - def testInvalidAggregationWithGetVariable(self): - self._skip_eager_if_gpus_less_than(1) - devices = ["/device:CPU:0", "/device:GPU:0"] - dist = mirrored_strategy.MirroredStrategy(devices) - with dist.scope(): + def testInvalidAggregationWithGetVariable(self, distribution): + with distribution.scope(): with self.assertRaisesRegexp( ValueError, "Invalid variable aggregation mode: invalid for " "variable: v"): @@ -555,12 +512,8 @@ class MirroredStrategyVariableCreationTest(test.TestCase): synchronization=variable_scope.VariableSynchronization.ON_WRITE, aggregation="invalid") - @test_util.run_in_graph_and_eager_modes(config=config) - def testInvalidAggregationWithVariable(self): - self._skip_eager_if_gpus_less_than(1) - devices = ["/device:CPU:0", "/device:GPU:0"] - dist = mirrored_strategy.MirroredStrategy(devices) - with dist.scope(): + def testInvalidAggregationWithVariable(self, distribution): + with distribution.scope(): with self.assertRaisesRegexp( ValueError, "Invalid variable aggregation mode: invalid for " "variable: v"): @@ -570,55 +523,28 @@ class MirroredStrategyVariableCreationTest(test.TestCase): synchronization=variable_scope.VariableSynchronization.ON_WRITE, aggregation="invalid") - @test_util.run_in_graph_and_eager_modes(config=config) - def testThreeDevices(self): - self._skip_eager_if_gpus_less_than(2) - - def model_fn(): - v = variable_scope.variable(1.0, name="foo") - distribution_strategy_context.get_replica_context().merge_call( - lambda _: _) - return v - - dist = mirrored_strategy.MirroredStrategy( - ["/device:GPU:0", "/device:GPU:1", "/device:CPU:0"]) - - with dist.scope(): - result = dist.call_for_each_replica(model_fn) - self.assertIsInstance(result, values.MirroredVariable) - self.assertEquals("foo:0", result.name) - - @test_util.run_in_graph_and_eager_modes(config=config) - def testNonMatchingVariableCreation(self): - self._skip_eager_if_gpus_less_than(1) - + def testNonMatchingVariableCreation(self, distribution): def model_fn(name): v = variable_scope.variable(1.0, name=name) - distribution_strategy_context.get_replica_context().merge_call( - lambda _: _) + ds_context.get_replica_context().merge_call(lambda _: _) return v - dist = mirrored_strategy.MirroredStrategy( - ["/device:GPU:0", "/device:CPU:0"]) - - with dist.scope(): + with distribution.scope(): names = values.DistributedValues({ "/device:CPU:0": "foo", "/device:GPU:0": "bar" }) with self.assertRaises(RuntimeError): - _ = dist.call_for_each_replica(model_fn, args=(names,)) - - @test_util.run_in_graph_and_eager_modes(config=config) - def testReplicaLocalVariable(self): - self._skip_eager_if_gpus_less_than(1) + _ = distribution.extended.call_for_each_replica(model_fn, args=(names,)) + def testReplicaLocalVariable(self, distribution): all_v_sum = {} all_v_mean = {} components_sum = {} components_mean = {} - def model_fn(device_id): + def model_fn(): + replica_id = self.evaluate(_replica_id()) v_sum = variable_scope.variable( 1.0, synchronization=variable_scope.VariableSynchronization.ON_READ, @@ -629,26 +555,22 @@ class MirroredStrategyVariableCreationTest(test.TestCase): aggregation=variable_scope.VariableAggregation.MEAN) self.assertTrue(isinstance(v_sum, values.ReplicaLocalVariable)) self.assertTrue(isinstance(v_mean, values.ReplicaLocalVariable)) - updates = [v_sum.assign_add(2.0 + device_id), - v_mean.assign(6.0 * device_id)] - all_v_sum[device_id] = v_sum - all_v_mean[device_id] = v_mean + updates = [v_sum.assign_add(2.0 + replica_id), + v_mean.assign(6.0 * replica_id)] + all_v_sum[replica_id] = v_sum + all_v_mean[replica_id] = v_mean c_sum = v_sum.get() c_mean = v_mean.get() - components_sum[device_id] = c_sum - components_mean[device_id] = c_mean + components_sum[replica_id] = c_sum + components_mean[replica_id] = c_mean self.assertIsNot(v_sum, c_sum) self.assertIsNot(v_mean, c_mean) return updates, v_sum, v_mean, c_sum, c_mean - dist = mirrored_strategy.MirroredStrategy( - ["/device:GPU:0", "/device:CPU:0"]) - - with dist.scope(): + with distribution.scope(): # Create "sum" and "mean" versions of ReplicaLocalVariables. ret_ops, ret_v_sum, ret_v_mean, regrouped_sum, regrouped_mean = ( - dist.call_for_each_replica( - model_fn, args=(dist.worker_device_index,))) + distribution.extended.call_for_each_replica(model_fn)) # Should see the same wrapping instance in all replicas. self.assertIs(all_v_sum[0], ret_v_sum) self.assertIs(all_v_mean[0], ret_v_mean) @@ -663,10 +585,10 @@ class MirroredStrategyVariableCreationTest(test.TestCase): # Apply updates self.evaluate(variables.global_variables_initializer()) - self.evaluate([y for x in ret_ops for y in dist.unwrap(x)]) + self.evaluate([y for x in ret_ops for y in distribution.unwrap(x)]) expected_sum = 0.0 expected_mean = 0.0 - for i, d in enumerate(dist.worker_devices): + for i, d in enumerate(distribution.extended.worker_devices): # Should see different values on different devices. v_sum_value = self.evaluate(ret_v_sum.get(d).read_value()) v_mean_value = self.evaluate(ret_v_mean.get(d).read_value()) @@ -676,135 +598,22 @@ class MirroredStrategyVariableCreationTest(test.TestCase): expected = i * 6.0 self.assertEqual(expected, v_mean_value) expected_mean += expected - expected_mean /= len(dist.worker_devices) + expected_mean /= len(distribution.extended.worker_devices) # Without get(device), should return the value you get by # applying the reduction across all replicas (whether you use # read_var(), get(), or nothing). - self.assertEqual(expected_sum, self.evaluate(dist.read_var(ret_v_sum))) - self.assertEqual(expected_mean, self.evaluate(dist.read_var(ret_v_mean))) + self.assertEqual(expected_sum, self.evaluate( + distribution.extended.read_var(ret_v_sum))) + self.assertEqual(expected_mean, self.evaluate( + distribution.extended.read_var(ret_v_mean))) self.assertEqual(expected_sum, self.evaluate(ret_v_sum.get())) self.assertEqual(expected_mean, self.evaluate(ret_v_mean.get())) self.assertEqual(expected_sum, self.evaluate(ret_v_sum)) self.assertEqual(expected_mean, self.evaluate(ret_v_mean)) - # NOTE(priyag): Names and name scopes are ignored in eager, hence we are not - # testing this in eager mode. - - def testNameScope(self): - def model_fn(): - with ops.name_scope("foo"): - a = constant_op.constant(1.0, name="a") - distribution_strategy_context.get_replica_context().merge_call( - lambda _: _) - b = constant_op.constant(1.0, name="b") - return a, b - - dist = mirrored_strategy.MirroredStrategy( - ["/device:GPU:0", "/device:CPU:0"]) - - with context.graph_mode(), dist.scope(): - with ops.name_scope("main"): - result = dist.call_for_each_replica(model_fn) - self.assertEquals(2, len(result)) - for v, name in zip(result, ["a", "b"]): - self.assertIsInstance(v, values.DistributedValues) - v0, v1 = dist.unwrap(v) - self.assertEquals("main/foo/" + name + ":0", v0.name) - self.assertEquals("main/replica_1/foo/" + name + ":0", v1.name) - - def testWithDefaultName(self): - def model_fn(): - with ops.name_scope(None, "foo"): - a = constant_op.constant(1.0, name="a") - distribution_strategy_context.get_replica_context().merge_call( - lambda _: _) - b = constant_op.constant(2.0, name="b") - return a, b - - dist = mirrored_strategy.MirroredStrategy( - ["/device:GPU:0", "/device:CPU:0"]) - - with context.graph_mode(), dist.scope(): - result = dist.call_for_each_replica(model_fn) - self.assertEquals(2, len(result)) - for v, name in zip(result, ["a", "b"]): - self.assertIsInstance(v, values.DistributedValues) - v0, v1 = dist.unwrap(v) - self.assertEquals("foo/" + name + ":0", v0.name) - self.assertEquals("replica_1/foo/" + name + ":0", v1.name) - - # variable_scope.variable() respects name scopes when creating - # variables. On the other hand variable_scope.get_variable() ignores name - # scopes when creating variables. We test both methods of creating variables - # to make sure that we have the same variable names in both cases. - def testNameScopeWithVariable(self): - def in_cross_replica(_): - c = variable_scope.variable(1.0, name="c") - return c - - def model_fn(): - b = variable_scope.variable(1.0, name="b") - with ops.name_scope("foo"): - c = distribution_strategy_context.get_replica_context().merge_call( - in_cross_replica) - return b, c - - dist = mirrored_strategy.MirroredStrategy( - ["/device:GPU:0", "/device:CPU:0"]) - - with context.graph_mode(), dist.scope(): - with ops.name_scope("main"): - a = variable_scope.variable(1.0, name="a") - result = dist.call_for_each_replica(model_fn) - result_b = result[0] - result_c = result[1] - self.assertIsInstance(result_b, values.DistributedValues) - self.assertIsInstance(result_c, values.DistributedValues) - a0, a1 = dist.unwrap(a) - b0, b1 = dist.unwrap(result_b) - c0, c1 = dist.unwrap(result_c) - self.assertEquals("main/a:0", a0.name) - self.assertEquals("main/a/replica_1:0", a1.name) - self.assertEquals("main/b:0", b0.name) - self.assertEquals("main/b/replica_1:0", b1.name) - self.assertEquals("main/foo/c:0", c0.name) - self.assertEquals("main/foo/c/replica_1:0", c1.name) - - def testNameScopeWithGetVariable(self): - def in_cross_replica(_): - c = variable_scope.get_variable("c", [1]) - return c - - def model_fn(): - b = variable_scope.get_variable("b", [1]) - with ops.name_scope("foo"): - c = distribution_strategy_context.get_replica_context().merge_call( - in_cross_replica) - return b, c - - dist = mirrored_strategy.MirroredStrategy( - ["/device:GPU:0", "/device:CPU:0"]) - - with context.graph_mode(), dist.scope(): - with ops.name_scope("main"): - a = variable_scope.get_variable("a", [1]) - result = dist.call_for_each_replica(model_fn) - result_b = result[0] - result_c = result[1] - self.assertIsInstance(result_b, values.DistributedValues) - self.assertIsInstance(result_c, values.DistributedValues) - a0, a1 = dist.unwrap(a) - b0, b1 = dist.unwrap(result_b) - c0, c1 = dist.unwrap(result_c) - self.assertEquals("a:0", a0.name) - self.assertEquals("a/replica_1:0", a1.name) - self.assertEquals("b:0", b0.name) - self.assertEquals("b/replica_1:0", b1.name) - self.assertEquals("c:0", c0.name) - self.assertEquals("c/replica_1:0", c1.name) - - def testDynamicRnnVariables(self): + # TODO(priyag): Update this test to work in eager mode as well. + def testDynamicRnnVariables(self, distribution): def model_fn(): inputs = constant_op.constant(2 * [2 * [[0.0, 1.0, 2.0, 3.0, 4.0]]]) cell_fw = rnn_cell_impl.LSTMCell(300) @@ -816,81 +625,208 @@ class MirroredStrategyVariableCreationTest(test.TestCase): dtype=dtypes.float32) return outputs - dist = mirrored_strategy.MirroredStrategy( - ["/device:GPU:0", "/device:CPU:0"]) - - with context.graph_mode(), dist.scope(): - result = dist.call_for_each_replica(model_fn) + with context.graph_mode(), distribution.scope(): + result = distribution.extended.call_for_each_replica(model_fn) # Two variables are created by the RNN layer. - self.assertEquals(2, len(result)) + self.assertEqual(2, len(result)) for v in result: self.assertIsInstance(v, values.DistributedValues) - _, v1 = dist.unwrap(v) - self.assertStartsWith(v1.name, "replica_1/") + _, v1 = distribution.unwrap(v) + self.assertStartsWith(v1._op.name, "replica_1/") - @test_util.run_in_graph_and_eager_modes(config=config) - def testReplicaLocalVariableUpdate(self): - with context.graph_mode(): + def testReplicaLocalVariableUpdate(self, distribution): + def model_fn(): + v_sum = variable_scope.variable( + 1.0, + synchronization=variable_scope.VariableSynchronization.ON_READ, + aggregation=variable_scope.VariableAggregation.SUM) + self.assertTrue(isinstance(v_sum, values.ReplicaLocalVariable)) + return v_sum - def model_fn(): - v_sum = variable_scope.variable( - 1.0, - synchronization=variable_scope.VariableSynchronization.ON_READ, - aggregation=variable_scope.VariableAggregation.SUM) - self.assertTrue(isinstance(v_sum, values.ReplicaLocalVariable)) - return v_sum + def update(var, value): + return var.assign(value) - dist = mirrored_strategy.MirroredStrategy( - ["/device:GPU:0", "/device:GPU:1"]) + with distribution.scope(): + ret_v_sum = distribution.extended.call_for_each_replica(model_fn) - def update(var, value): - return var.assign(value) + # Initialize variables. + self.evaluate(variables.global_variables_initializer()) + # Assert that the aggregated value of the replica local vars is the sum + # of the individual values before running the update ops. + self.assertEqual(1.0, self.evaluate(ret_v_sum.get( + distribution.extended.worker_devices[0]).read_value())) + self.assertEqual(2.0, self.evaluate(ret_v_sum)) - with dist.scope(): - ret_v_sum = dist.call_for_each_replica(model_fn) - update_ops = dist.update(ret_v_sum, update, 5.0, grouped=False) - - # Initialize variables. - self.evaluate(variables.global_variables_initializer()) - # Assert that the aggregated value of the replica local vars is the sum - # of the individual values before running the update ops. - self.assertEquals(1.0, self.evaluate( - ret_v_sum.get(dist._devices[0]).read_value())) - self.assertEquals(2.0, self.evaluate(ret_v_sum)) - - # Apply updates. - self.evaluate(update_ops) - # Assert that the aggregated value of the replica local vars is the sum - # of the individual values after running the update ops. - self.assertEquals(5.0, self.evaluate( - ret_v_sum.get(dist._devices[0]).read_value())) - self.assertEquals(10.0, self.evaluate(ret_v_sum)) + # Apply updates. + update_ops = distribution.extended.update( + ret_v_sum, update, args=(5.0,), group=False) + self.evaluate(update_ops) + # Assert that the aggregated value of the replica local vars is the sum + # of the individual values after running the update ops. + self.assertEqual(5.0, self.evaluate(ret_v_sum.get( + distribution.extended.worker_devices[0]).read_value())) + self.assertEqual(10.0, self.evaluate(ret_v_sum)) +@combinations.generate(combinations.combine( + distribution=[ + combinations.mirrored_strategy_with_gpu_and_cpu, + combinations.core_mirrored_strategy_with_gpu_and_cpu], + mode=["graph"])) +class MirroredStrategyNameScopeTest(test.TestCase): + # NOTE(priyag): Names and name scopes are ignored in eager, hence we are not + # testing this in eager mode. + + def testNameScope(self, distribution): + def model_fn(): + with ops.name_scope("foo"): + a = constant_op.constant(1.0, name="a") + ds_context.get_replica_context().merge_call(lambda _: _) + b = constant_op.constant(1.0, name="b") + return a, b + + with context.graph_mode(), distribution.scope(): + with ops.name_scope("main"): + result = distribution.extended.call_for_each_replica(model_fn) + self.assertEqual(2, len(result)) + for v, name in zip(result, ["a", "b"]): + self.assertIsInstance(v, values.DistributedValues) + v0, v1 = distribution.unwrap(v) + self.assertEqual("main/foo/" + name + ":0", v0.name) + self.assertEqual("main/replica_1/foo/" + name + ":0", v1.name) + + def testWithDefaultName(self, distribution): + def model_fn(): + with ops.name_scope(None, "foo"): + a = constant_op.constant(1.0, name="a") + ds_context.get_replica_context().merge_call(lambda _: _) + b = constant_op.constant(2.0, name="b") + return a, b + + with context.graph_mode(), distribution.scope(): + result = distribution.extended.call_for_each_replica(model_fn) + self.assertEqual(2, len(result)) + for v, name in zip(result, ["a", "b"]): + self.assertIsInstance(v, values.DistributedValues) + v0, v1 = distribution.unwrap(v) + self.assertEqual("foo/" + name + ":0", v0.name) + self.assertEqual("replica_1/foo/" + name + ":0", v1.name) + + # variable_scope.variable() respects name scopes when creating + # variables. On the other hand variable_scope.get_variable() ignores name + # scopes when creating variables. We test both methods of creating variables + # to make sure that we have the same variable names in both cases. + def testNameScopeWithVariable(self, distribution): + def in_cross_replica(_): + c = variable_scope.variable(1.0, name="c") + return c + + def model_fn(): + b = variable_scope.variable(1.0, name="b") + with ops.name_scope("foo"): + c = ds_context.get_replica_context().merge_call(in_cross_replica) + return b, c + + with context.graph_mode(), distribution.scope(): + with ops.name_scope("main"): + a = variable_scope.variable(1.0, name="a") + result = distribution.extended.call_for_each_replica(model_fn) + result_b = result[0] + result_c = result[1] + self.assertIsInstance(result_b, values.DistributedValues) + self.assertIsInstance(result_c, values.DistributedValues) + a0, a1 = distribution.unwrap(a) + b0, b1 = distribution.unwrap(result_b) + c0, c1 = distribution.unwrap(result_c) + self.assertEqual("main/a:0", a0.name) + self.assertEqual("main/a/replica_1:0", a1.name) + self.assertEqual("main/b:0", b0.name) + self.assertEqual("main/b/replica_1:0", b1.name) + self.assertEqual("main/foo/c:0", c0.name) + self.assertEqual("main/foo/c/replica_1:0", c1.name) + + def testNameScopeWithGetVariable(self, distribution): + def in_cross_replica(_): + c = variable_scope.get_variable("c", [1]) + return c + + def model_fn(): + b = variable_scope.get_variable("b", [1]) + with ops.name_scope("foo"): + c = ds_context.get_replica_context().merge_call(in_cross_replica) + return b, c + + with context.graph_mode(), distribution.scope(): + with ops.name_scope("main"): + a = variable_scope.get_variable("a", [1]) + result = distribution.extended.call_for_each_replica(model_fn) + result_b = result[0] + result_c = result[1] + self.assertIsInstance(result_b, values.DistributedValues) + self.assertIsInstance(result_c, values.DistributedValues) + a0, a1 = distribution.unwrap(a) + b0, b1 = distribution.unwrap(result_b) + c0, c1 = distribution.unwrap(result_c) + self.assertEqual("a:0", a0.name) + self.assertEqual("a/replica_1:0", a1.name) + self.assertEqual("b:0", b0.name) + self.assertEqual("b/replica_1:0", b1.name) + self.assertEqual("c:0", c0.name) + self.assertEqual("c/replica_1:0", c1.name) + + +@combinations.generate( + combinations.combine( + distribution=[ + combinations.NamedDistribution( + "Mirrored3Devices", + # pylint: disable=g-long-lambda + lambda: mirrored_strategy.MirroredStrategy( + ["/device:GPU:0", "/device:GPU:1", "/device:CPU:0"]), + required_gpus=2), + combinations.NamedDistribution( + "CoreMirrored3Devices", + # pylint: disable=g-long-lambda + lambda: mirrored_strategy.CoreMirroredStrategy( + ["/device:GPU:0", "/device:GPU:1", "/device:CPU:0"]), + required_gpus=2) + ], + mode=["graph", "eager"])) +class MirroredThreeDeviceDistributionTest( + strategy_test_lib.DistributionTestBase, + parameterized.TestCase): + + def testThreeDevices(self, distribution): + def model_fn(): + v = variable_scope.variable(1.0, name="foo") + ds_context.get_replica_context().merge_call(lambda _: _) + return v + + with distribution.scope(): + result = distribution.extended.call_for_each_replica(model_fn) + self.assertIsInstance(result, values.MirroredVariable) + self.assertEqual("foo:0", result.name) + + +@combinations.generate(combinations.combine( + distribution=[ + combinations.mirrored_strategy_with_gpu_and_cpu, + combinations.core_mirrored_strategy_with_gpu_and_cpu], + mode=["graph", "eager"])) class MirroredVariableUpdateTest(test.TestCase): # The following tests check assign, assign_add and assign_sub on Mirrored # variables in replica and cross replica context. - config = config_pb2.ConfigProto() - config.allow_soft_placement = True - def _skip_eager_if_gpus_less_than(self, num_gpus): - if context.num_gpus() < num_gpus and context.executing_eagerly(): - self.skipTest("Enough GPUs not available for this test in eager mode.") - - @test_util.run_in_graph_and_eager_modes(config=config) - def testAssignMirroredVarReplicaContextWithoutAggregationType(self): + def testAssignMirroredVarReplicaContextWithoutAggregationType(self, + distribution): # Test that we always have an aggregation type set on the mirrored variable # if we assign to it in replica mode. - self._skip_eager_if_gpus_less_than(1) def var_fn(): v = variable_scope.variable(1.0, name="foo") return v - dist = mirrored_strategy.MirroredStrategy( - ["/device:GPU:0", "/device:CPU:0"]) - - with dist.scope(): - mirrored_var = dist.call_for_each_replica(var_fn) + with distribution.scope(): + mirrored_var = distribution.extended.call_for_each_replica(var_fn) self.assertIsInstance(mirrored_var, values.MirroredVariable) self.evaluate(variables.global_variables_initializer()) @@ -900,23 +836,19 @@ class MirroredVariableUpdateTest(test.TestCase): with self.assertRaisesRegexp( ValueError, "You must specify an aggregation method to update a " "MirroredVariable in Replica Context."): - self.evaluate(dist.unwrap(dist.call_for_each_replica(model_fn))) + self.evaluate(distribution.unwrap( + distribution.extended.call_for_each_replica(model_fn))) - @test_util.run_in_graph_and_eager_modes(config=config) - def testAssignMirroredVarReplicaContextWithSum(self): + def testAssignMirroredVarReplicaContextWithSum(self, distribution): # Test that we don't reduce a non-per-replica value with the "sum" # aggregation type. - self._skip_eager_if_gpus_less_than(1) def var_fn(): v = variable_scope.variable( 1.0, name="foo", aggregation=variable_scope.VariableAggregation.SUM) return v - dist = mirrored_strategy.MirroredStrategy( - ["/device:GPU:0", "/device:CPU:0"]) - - with dist.scope(): - mirrored_var = dist.call_for_each_replica(var_fn) + with distribution.scope(): + mirrored_var = distribution.extended.call_for_each_replica(var_fn) self.assertIsInstance(mirrored_var, values.MirroredVariable) self.evaluate(variables.global_variables_initializer()) @@ -925,219 +857,184 @@ class MirroredVariableUpdateTest(test.TestCase): with self.assertRaisesRegexp( ValueError, "A non-DistributedValues value 5.0 cannot be reduced " - "with the given aggregation VariableAggregation.SUM."): - self.evaluate(dist.unwrap(dist.call_for_each_replica(model_fn))) + "with the given reduce op ReduceOp.SUM."): + self.evaluate(distribution.unwrap( + distribution.extended.call_for_each_replica(model_fn))) - @test_util.run_in_graph_and_eager_modes(config=config) - def testAssignMirroredVarCrossDeviceContext(self): - self._skip_eager_if_gpus_less_than(1) + def testAssignMirroredVarCrossDeviceContext(self, distribution): def var_fn(): return variable_scope.variable(1.0, name="foo") - dist = mirrored_strategy.MirroredStrategy( - ["/device:GPU:0", "/device:CPU:0"]) - - with dist.scope(): - mirrored_var = dist.call_for_each_replica(var_fn) + with distribution.scope(): + mirrored_var = distribution.extended.call_for_each_replica(var_fn) self.assertIsInstance(mirrored_var, values.MirroredVariable) self.evaluate(variables.global_variables_initializer()) - self.assertEquals(1.0, self.evaluate(mirrored_var)) + self.assertEqual(1.0, self.evaluate(mirrored_var)) mirrored_var_result = self.evaluate(mirrored_var.assign(6.0)) - self.assertEquals(6.0, mirrored_var_result) + self.assertEqual(6.0, mirrored_var_result) - @test_util.run_in_graph_and_eager_modes(config=config) - def testAssignMirroredVarReplicaContext(self): - self._skip_eager_if_gpus_less_than(1) + def testAssignMirroredVarReplicaContext(self, distribution): def var_fn(): return variable_scope.variable( 1.0, name="foo", aggregation=variable_scope.VariableAggregation.MEAN) - dist = mirrored_strategy.MirroredStrategy( - ["/device:GPU:0", "/device:CPU:0"]) - - with dist.scope(): - mirrored_var = dist.call_for_each_replica(var_fn) + with distribution.scope(): + mirrored_var = distribution.extended.call_for_each_replica(var_fn) self.assertIsInstance(mirrored_var, values.MirroredVariable) self.evaluate(variables.global_variables_initializer()) - self.assertEquals(1.0, self.evaluate(mirrored_var)) + self.assertEqual(1.0, self.evaluate(mirrored_var)) def model_fn(): value = math_ops.cast( - distribution_strategy_context.get_replica_context().replica_id, + ds_context.get_replica_context().replica_id_in_sync_group, mirrored_var.dtype) return mirrored_var.assign(value) - self.evaluate(dist.unwrap(dist.call_for_each_replica(model_fn))) - self.assertEquals(0.5, self.evaluate(mirrored_var)) + self.evaluate(distribution.unwrap( + distribution.extended.call_for_each_replica(model_fn))) + self.assertEqual(0.5, self.evaluate(mirrored_var)) - @test_util.run_in_graph_and_eager_modes(config=config) - def testAssignMirroredVarReplicaContextWithSingleValue(self): - self._skip_eager_if_gpus_less_than(1) + def testAssignMirroredVarReplicaContextWithSingleValue(self, distribution): def var_fn(): return variable_scope.variable( 1.0, name="foo", aggregation=variable_scope.VariableAggregation.MEAN) - dist = mirrored_strategy.MirroredStrategy( - ["/device:GPU:0", "/device:CPU:0"]) - - with dist.scope(): - mirrored_var = dist.call_for_each_replica(var_fn) + with distribution.scope(): + mirrored_var = distribution.extended.call_for_each_replica(var_fn) self.assertIsInstance(mirrored_var, values.MirroredVariable) self.evaluate(variables.global_variables_initializer()) - self.assertEquals(1.0, self.evaluate(mirrored_var)) + self.assertEqual(1.0, self.evaluate(mirrored_var)) def model_fn(): return mirrored_var.assign(5.0) - self.evaluate(dist.unwrap(dist.call_for_each_replica(model_fn))) - self.assertEquals(5.0, self.evaluate(mirrored_var)) + self.evaluate(distribution.unwrap( + distribution.extended.call_for_each_replica(model_fn))) + self.assertEqual(5.0, self.evaluate(mirrored_var)) - @test_util.run_in_graph_and_eager_modes(config=config) - def testAssignAddMirroredVarCrossDeviceContext(self): - self._skip_eager_if_gpus_less_than(1) + def testAssignAddMirroredVarCrossDeviceContext(self, distribution): def var_fn(): return variable_scope.variable(1.0, name="foo") - dist = mirrored_strategy.MirroredStrategy( - ["/device:GPU:0", "/device:CPU:0"]) - - with dist.scope(): - mirrored_var = dist.call_for_each_replica(var_fn) + with distribution.scope(): + mirrored_var = distribution.extended.call_for_each_replica(var_fn) self.assertIsInstance(mirrored_var, values.MirroredVariable) self.evaluate(variables.global_variables_initializer()) - self.assertEquals(1.0, self.evaluate(mirrored_var)) + self.assertEqual(1.0, self.evaluate(mirrored_var)) # read_value == True mirrored_var_result = self.evaluate( mirrored_var.assign_add(6.0, read_value=True)) - self.assertEquals(7.0, mirrored_var_result) - self.assertEquals(7.0, self.evaluate(mirrored_var.get("/device:CPU:0"))) - self.assertEquals(7.0, self.evaluate(mirrored_var.get("/device:GPU:0"))) + self.assertEqual(7.0, mirrored_var_result) + self.assertEqual(7.0, self.evaluate(mirrored_var.get("/device:CPU:0"))) + self.assertEqual(7.0, self.evaluate(mirrored_var.get("/device:GPU:0"))) # read_value == False self.evaluate(mirrored_var.assign_add(2.0, read_value=False)) - self.assertEquals(9.0, self.evaluate(mirrored_var.get("/device:CPU:0"))) - self.assertEquals(9.0, self.evaluate(mirrored_var.get("/device:GPU:0"))) + self.assertEqual(9.0, self.evaluate(mirrored_var.get("/device:CPU:0"))) + self.assertEqual(9.0, self.evaluate(mirrored_var.get("/device:GPU:0"))) - @test_util.run_in_graph_and_eager_modes(config=config) - def testAssignAddMirroredVarReplicaContext(self): - self._skip_eager_if_gpus_less_than(1) + def testAssignAddMirroredVarReplicaContext(self, distribution): def var_fn(): return variable_scope.variable( 1.0, name="foo", aggregation=variable_scope.VariableAggregation.MEAN) - dist = mirrored_strategy.MirroredStrategy( - ["/device:GPU:0", "/device:CPU:0"]) - - with dist.scope(): - mirrored_var = dist.call_for_each_replica(var_fn) + with distribution.scope(): + mirrored_var = distribution.extended.call_for_each_replica(var_fn) self.assertIsInstance(mirrored_var, values.MirroredVariable) self.evaluate(variables.global_variables_initializer()) - self.assertEquals(1.0, self.evaluate(mirrored_var)) + self.assertEqual(1.0, self.evaluate(mirrored_var)) def model_fn(): value = math_ops.cast( - distribution_strategy_context.get_replica_context().replica_id, + ds_context.get_replica_context().replica_id_in_sync_group, mirrored_var.dtype) return mirrored_var.assign_add(value) - self.evaluate(dist.unwrap(dist.call_for_each_replica(model_fn))) - self.assertEquals(1.5, self.evaluate(mirrored_var)) + self.evaluate(distribution.unwrap( + distribution.extended.call_for_each_replica(model_fn))) + self.assertEqual(1.5, self.evaluate(mirrored_var)) - @test_util.run_in_graph_and_eager_modes(config=config) - def testAssignAddMirroredVarReplicaContextWithSingleValue(self): - self._skip_eager_if_gpus_less_than(1) + def testAssignAddMirroredVarReplicaContextWithSingleValue(self, distribution): def var_fn(): return variable_scope.variable( 1.0, name="foo", aggregation=variable_scope.VariableAggregation.MEAN) - dist = mirrored_strategy.MirroredStrategy( - ["/device:GPU:0", "/device:CPU:0"]) - - with dist.scope(): - mirrored_var = dist.call_for_each_replica(var_fn) + with distribution.scope(): + mirrored_var = distribution.extended.call_for_each_replica(var_fn) self.assertIsInstance(mirrored_var, values.MirroredVariable) self.evaluate(variables.global_variables_initializer()) - self.assertEquals(1.0, self.evaluate(mirrored_var)) + self.assertEqual(1.0, self.evaluate(mirrored_var)) def model_fn(): return mirrored_var.assign_add(5.0) - self.evaluate(dist.unwrap(dist.call_for_each_replica(model_fn))) - self.assertEquals(6.0, self.evaluate(mirrored_var)) + self.evaluate(distribution.unwrap( + distribution.extended.call_for_each_replica(model_fn))) + self.assertEqual(6.0, self.evaluate(mirrored_var)) - @test_util.run_in_graph_and_eager_modes(config=config) - def testAssignSubMirroredVarCrossDeviceContext(self): - self._skip_eager_if_gpus_less_than(1) + def testAssignSubMirroredVarCrossDeviceContext(self, distribution): def var_fn(): return variable_scope.variable(5.0, name="foo") - dist = mirrored_strategy.MirroredStrategy( - ["/device:GPU:0", "/device:CPU:0"]) - - with dist.scope(): - mirrored_var = dist.call_for_each_replica(var_fn) + with distribution.scope(): + mirrored_var = distribution.extended.call_for_each_replica(var_fn) self.assertIsInstance(mirrored_var, values.MirroredVariable) self.evaluate(variables.global_variables_initializer()) - self.assertEquals(5.0, self.evaluate(mirrored_var)) + self.assertEqual(5.0, self.evaluate(mirrored_var)) mirrored_var_result = self.evaluate(mirrored_var.assign_sub(2.0)) - self.assertEquals(3.0, mirrored_var_result) - self.assertEquals(3.0, self.evaluate(mirrored_var.get("/device:GPU:0"))) - self.assertEquals(3.0, self.evaluate(mirrored_var.get("/device:CPU:0"))) + self.assertEqual(3.0, mirrored_var_result) + self.assertEqual(3.0, self.evaluate(mirrored_var.get("/device:GPU:0"))) + self.assertEqual(3.0, self.evaluate(mirrored_var.get("/device:CPU:0"))) - @test_util.run_in_graph_and_eager_modes(config=config) - def testAssignSubMirroredVarReplicaContext(self): - self._skip_eager_if_gpus_less_than(1) + def testAssignSubMirroredVarReplicaContext(self, distribution): def var_fn(): return variable_scope.variable( 5.0, name="foo", aggregation=variable_scope.VariableAggregation.MEAN) - dist = mirrored_strategy.MirroredStrategy( - ["/device:GPU:0", "/device:CPU:0"]) - - with dist.scope(): - mirrored_var = dist.call_for_each_replica(var_fn) + with distribution.scope(): + mirrored_var = distribution.extended.call_for_each_replica(var_fn) self.assertIsInstance(mirrored_var, values.MirroredVariable) self.evaluate(variables.global_variables_initializer()) - self.assertEquals(5.0, self.evaluate(mirrored_var)) + self.assertEqual(5.0, self.evaluate(mirrored_var)) def model_fn(): value = math_ops.cast( - distribution_strategy_context.get_replica_context().replica_id, + ds_context.get_replica_context().replica_id_in_sync_group, mirrored_var.dtype) return mirrored_var.assign_sub(value) - self.evaluate(dist.unwrap(dist.call_for_each_replica(model_fn))) - self.assertEquals(4.5, self.evaluate(mirrored_var)) + self.evaluate(distribution.unwrap( + distribution.extended.call_for_each_replica(model_fn))) + self.assertEqual(4.5, self.evaluate(mirrored_var)) - @test_util.run_in_graph_and_eager_modes(config=config) - def testAssignSubMirroredVarReplicaContextWithSingleValue(self): - self._skip_eager_if_gpus_less_than(1) + def testAssignSubMirroredVarReplicaContextWithSingleValue(self, distribution): def var_fn(): return variable_scope.variable( 5.0, name="foo", aggregation=variable_scope.VariableAggregation.MEAN) - dist = mirrored_strategy.MirroredStrategy( - ["/device:GPU:0", "/device:CPU:0"]) - - with dist.scope(): - mirrored_var = dist.call_for_each_replica(var_fn) + with distribution.scope(): + mirrored_var = distribution.extended.call_for_each_replica(var_fn) self.assertIsInstance(mirrored_var, values.MirroredVariable) self.evaluate(variables.global_variables_initializer()) - self.assertEquals(5.0, self.evaluate(mirrored_var)) + self.assertEqual(5.0, self.evaluate(mirrored_var)) def model_fn(): return mirrored_var.assign_sub(1.0) - self.evaluate(dist.unwrap(dist.call_for_each_replica(model_fn))) - self.assertEquals(4.0, self.evaluate(mirrored_var)) + self.evaluate(distribution.unwrap( + distribution.extended.call_for_each_replica(model_fn))) + self.assertEqual(4.0, self.evaluate(mirrored_var)) +@combinations.generate(combinations.combine( + distribution=[ + combinations.mirrored_strategy_with_gpu_and_cpu, + combinations.core_mirrored_strategy_with_gpu_and_cpu], + mode=["graph", "eager"])) class MirroredAndReplicaLocalVariableInitializerTest(test.TestCase): - config = config_pb2.ConfigProto() - config.allow_soft_placement = True - def testAssignMirroredVarInitializer(self): + def testAssignMirroredVarInitializer(self, distribution): # This test is not eager compatible since in eager variables are initialized # upon construction instead of once the initialization op is run. with context.graph_mode(): @@ -1145,17 +1042,14 @@ class MirroredAndReplicaLocalVariableInitializerTest(test.TestCase): v = variable_scope.variable(1.0, name="foo") return v - dist = mirrored_strategy.MirroredStrategy( - ["/device:GPU:0", "/device:CPU:0"]) - - with dist.scope(): - mirrored_var = dist.call_for_each_replica(var_fn) + with distribution.scope(): + mirrored_var = distribution.extended.call_for_each_replica(var_fn) self.assertIsInstance(mirrored_var, values.MirroredVariable) self.assertFalse(self.evaluate(mirrored_var.is_initialized())) self.evaluate(mirrored_var.initializer) self.assertTrue(self.evaluate(mirrored_var.is_initialized())) - def testAssignReplicaLocalVarInitializer(self): + def testAssignReplicaLocalVarInitializer(self, distribution): # This test is not eager compatible since in eager variables are initialized # upon construction instead of once the initialization op is run. with context.graph_mode(): @@ -1167,11 +1061,9 @@ class MirroredAndReplicaLocalVariableInitializerTest(test.TestCase): self.assertTrue(isinstance(v_sum, values.ReplicaLocalVariable)) return v_sum - dist = mirrored_strategy.MirroredStrategy( - ["/device:GPU:0", "/device:CPU:0"]) - - with dist.scope(): - replica_local_var = dist.call_for_each_replica(model_fn) + with distribution.scope(): + replica_local_var = distribution.extended.call_for_each_replica( + model_fn) self.assertTrue(isinstance(replica_local_var, values.ReplicaLocalVariable)) self.assertFalse(self.evaluate(replica_local_var.is_initialized())) @@ -1179,17 +1071,14 @@ class MirroredAndReplicaLocalVariableInitializerTest(test.TestCase): self.assertTrue(self.evaluate(replica_local_var.is_initialized())) +@combinations.generate(combinations.combine( + distribution=[ + combinations.mirrored_strategy_with_gpu_and_cpu, + combinations.core_mirrored_strategy_with_gpu_and_cpu], + mode=["graph", "eager"])) class ReplicaLocalVariableAssignTest(test.TestCase): - config = config_pb2.ConfigProto() - config.allow_soft_placement = True - def _skip_eager_if_gpus_less_than(self, num_gpus): - if context.num_gpus() < num_gpus and context.executing_eagerly(): - self.skipTest("Not enough GPUs available for this test in eager mode.") - - @test_util.run_in_graph_and_eager_modes(config=config) - def testAssignReplicaLocalVarSumAggregation(self): - self._skip_eager_if_gpus_less_than(1) + def testAssignReplicaLocalVarSumAggregation(self, distribution): def model_fn(): v_sum = variable_scope.variable( 1.0, @@ -1197,18 +1086,16 @@ class ReplicaLocalVariableAssignTest(test.TestCase): aggregation=variable_scope.VariableAggregation.SUM) return v_sum - dist = mirrored_strategy.MirroredStrategy( - ["/device:GPU:0", "/device:CPU:0"]) - - with dist.scope(): - replica_local_var = dist.call_for_each_replica(model_fn) + with distribution.scope(): + replica_local_var = distribution.extended.call_for_each_replica(model_fn) self.assertTrue(isinstance(replica_local_var, values.ReplicaLocalVariable)) self.evaluate(variables.global_variables_initializer()) # Each replica has a value of 1.0 assigned to it in replica context. # When we read the value using `read_var` we should see the SUM of each of # values on each of the replicas. - self.assertEqual(2.0, self.evaluate(dist.read_var(replica_local_var))) + self.assertEqual(2.0, self.evaluate( + distribution.read_var(replica_local_var))) # Assigning 6.0 in cross replica context will assign a value of # 6.0/num_replicas to each replica. tlv_ops = replica_local_var.assign(6.0) @@ -1216,11 +1103,10 @@ class ReplicaLocalVariableAssignTest(test.TestCase): # On reading the replica local var we should get the assigned value back. # The value on all the replicas are added before being returned by # `read_var`. - self.assertEqual(6.0, self.evaluate(dist.read_var(replica_local_var))) + self.assertEqual(6.0, self.evaluate( + distribution.read_var(replica_local_var))) - @test_util.run_in_graph_and_eager_modes(config=config) - def testAssignReplicaLocalVarMeanAggregation(self): - self._skip_eager_if_gpus_less_than(1) + def testAssignReplicaLocalVarMeanAggregation(self, distribution): def model_fn(): v_sum = variable_scope.variable( 1.0, @@ -1228,23 +1114,22 @@ class ReplicaLocalVariableAssignTest(test.TestCase): aggregation=variable_scope.VariableAggregation.MEAN) return v_sum - dist = mirrored_strategy.MirroredStrategy( - ["/device:GPU:0", "/device:CPU:0"]) - - with dist.scope(): - replica_local_var = dist.call_for_each_replica(model_fn) + with distribution.scope(): + replica_local_var = distribution.extended.call_for_each_replica(model_fn) self.assertTrue(isinstance(replica_local_var, values.ReplicaLocalVariable)) self.evaluate(variables.global_variables_initializer()) # Each replica has a value of 1.0 assigned to it in replica context. # When we read the value using `read_var` we should see the MEAN of values # on all replicas which is the value assigned in replica context. - self.assertEqual(1.0, self.evaluate(dist.read_var(replica_local_var))) + self.assertEqual(1.0, self.evaluate( + distribution.read_var(replica_local_var))) tlv_ops = replica_local_var.assign(6.0) self.evaluate(tlv_ops) # On reading the replica local var we should get the MEAN of all values # which is equal to the value assigned. - self.assertEqual(6.0, self.evaluate(dist.read_var(replica_local_var))) + self.assertEqual(6.0, self.evaluate( + distribution.read_var(replica_local_var))) class MockModel(object): @@ -1278,24 +1163,25 @@ class MiniModel(keras_training.Model): return self.fc(inputs) +@combinations.generate(combinations.combine( + distribution=[ + combinations.mirrored_strategy_with_gpu_and_cpu, + combinations.core_mirrored_strategy_with_gpu_and_cpu], + mode=["graph", "eager"])) class MirroredStrategyDefunTest(test.TestCase): - def _skip_eager_if_gpus_less_than(self, num_gpus): - if context.num_gpus() < num_gpus and context.executing_eagerly(): - self.skipTest("Not enough GPUs available for this test in eager mode.") - - def _call_and_check(self, model_fn, inputs, expected_result, defuns, - two_variables=False): + def _call_and_check(self, distribution, model_fn, inputs, expected_result, + defuns, two_variables=False): cpu_dev = device_util.canonicalize("CPU:0") gpu_dev = device_util.canonicalize("GPU:0") devices = [cpu_dev, gpu_dev] - dist = mirrored_strategy.MirroredStrategy(devices) - with dist.scope(): + with distribution.scope(): mock_model = MockModel(two_variables) self.evaluate(variables.global_variables_initializer()) - result = dist.call_for_each_replica(model_fn, args=[mock_model] + inputs) + result = distribution.extended.call_for_each_replica( + model_fn, args=[mock_model] + inputs) for device in devices: device_result = values.select_device(device, result) device_expected_result = values.select_device(device, expected_result) @@ -1307,17 +1193,15 @@ class MirroredStrategyDefunTest(test.TestCase): # call_for_each has one trace per device. To check that the expected set # of variables was accessed on each trace, we first retrieve each # device-specific graph function. - per_replica_graph_functions = dist.call_for_each_replica( - defun.get_concrete_function, args=[mock_model] + inputs) + per_replica_graph_functions = ( + distribution.extended.call_for_each_replica( + defun.get_concrete_function, args=[mock_model] + inputs)) for device in devices: graph_function = per_replica_graph_functions.get(device=device) self.assertEqual(set(mock_model.variables), set(graph_function.graph.variables)) - @test_util.run_in_graph_and_eager_modes() - def testVariableInDefun(self): - self._skip_eager_if_gpus_less_than(1) - + def testVariableInDefun(self, distribution): @function.defun def times_two(mock_model): return mock_model() @@ -1325,12 +1209,9 @@ class MirroredStrategyDefunTest(test.TestCase): def model_fn(mock_model): return times_two(mock_model) - self._call_and_check(model_fn, [], 2.5, [times_two]) - - @test_util.run_in_graph_and_eager_modes() - def testVariableInNestedDefun(self): - self._skip_eager_if_gpus_less_than(1) + self._call_and_check(distribution, model_fn, [], 2.5, [times_two]) + def testVariableInNestedDefun(self, distribution): @function.defun def times_two(mock_model): return mock_model() @@ -1342,12 +1223,10 @@ class MirroredStrategyDefunTest(test.TestCase): def model_fn(mock_model): return two_x_plus_one(mock_model) - self._call_and_check(model_fn, [], 3.5, [times_two, two_x_plus_one]) - - @test_util.run_in_graph_and_eager_modes() - def testTwoVariablesInNestedDefun(self): - self._skip_eager_if_gpus_less_than(1) + self._call_and_check(distribution, model_fn, [], 3.5, + [times_two, two_x_plus_one]) + def testTwoVariablesInNestedDefun(self, distribution): @function.defun def fn1(mock_model): return mock_model() @@ -1359,12 +1238,10 @@ class MirroredStrategyDefunTest(test.TestCase): def model_fn(mock_model): return fn2(mock_model) - self._call_and_check(model_fn, [], 5.5, [fn1, fn2], two_variables=True) - - @test_util.run_in_graph_and_eager_modes() - def testGradientTapeOverNestedDefuns(self): - self._skip_eager_if_gpus_less_than(1) + self._call_and_check(distribution, model_fn, [], 5.5, [fn1, fn2], + two_variables=True) + def testGradientTapeOverNestedDefuns(self, distribution): @function.defun def fn1(mock_model): return mock_model() @@ -1380,13 +1257,10 @@ class MirroredStrategyDefunTest(test.TestCase): [v.get() for v in mock_model.variables]) return grads - self._call_and_check(model_fn, [], [2.0, 1.0], [fn1, fn2], + self._call_and_check(distribution, model_fn, [], [2.0, 1.0], [fn1, fn2], two_variables=True) - @test_util.run_in_graph_and_eager_modes() - def testPassPerReplica(self): - self._skip_eager_if_gpus_less_than(1) - + def testPassPerReplica(self, distribution): @function.defun def fn1(mock_model, factor): return mock_model(factor) @@ -1394,18 +1268,10 @@ class MirroredStrategyDefunTest(test.TestCase): factors = values.PerReplica({"CPU:0": 5.0, "GPU:0": 3.0}) expected_result = values.PerReplica({"CPU:0": 5.0 * 1.25, "GPU:0": 3.0 * 1.25}) - self._call_and_check(fn1, [factors], expected_result, [fn1]) + self._call_and_check(distribution, fn1, [factors], expected_result, [fn1]) - @test_util.run_in_graph_and_eager_modes() - def testTrain(self): - self._skip_eager_if_gpus_less_than(1) - - cpu_dev = device_util.canonicalize("CPU:0") - gpu_dev = device_util.canonicalize("GPU:0") - devices = [cpu_dev, gpu_dev] - dist = mirrored_strategy.MirroredStrategy(devices) - - with dist.scope(): + def testTrain(self, distribution): + with distribution.scope(): mock_model = MiniModel() mock_model.call = function.defun(mock_model.call) @@ -1415,10 +1281,11 @@ class MirroredStrategyDefunTest(test.TestCase): gradients_fn = backprop.implicit_grad(loss_fn) gradients_fn = optimizer_lib.get_filtered_grad_fn(gradients_fn) - grads_and_vars = dist.call_for_each_replica(gradients_fn, args=(None,)) + grads_and_vars = distribution.extended.call_for_each_replica( + gradients_fn, args=(None,)) optimizer = gradient_descent.GradientDescentOptimizer(0.25) - update_ops = optimizer._distributed_apply(dist, grads_and_vars) # pylint: disable=protected-access + update_ops = optimizer._distributed_apply(distribution, grads_and_vars) # pylint: disable=protected-access if not context.executing_eagerly(): self.evaluate(variables.global_variables_initializer()) @@ -1430,30 +1297,82 @@ class MirroredStrategyDefunTest(test.TestCase): self.assertAllEqual([0.5], updated_var_values[1]) +@combinations.generate( + combinations.combine( + distribution=[ + combinations.NamedDistribution( + "Mirrored", + # pylint: disable=g-long-lambda + lambda: mirrored_strategy.MirroredStrategy(num_gpus_per_worker= + context.num_gpus()), + required_gpus=1), + combinations.NamedDistribution( + "CoreMirrored", + # pylint: disable=g-long-lambda + lambda: mirrored_strategy.CoreMirroredStrategy( + mirrored_strategy.all_local_devices()), + required_gpus=1) + ], + mode=["graph"])) class MultiWorkerMirroredStrategyTest( multi_worker_test_base.MultiWorkerTestBase, strategy_test_lib.DistributionTestBase): - def _get_distribution_strategy(self): + def _configure_distribution_strategy(self, distribution): cluster_spec = server_lib.ClusterSpec({ "worker": ["/job:worker/task:0", "/job:worker/task:1"] }) - strategy = mirrored_strategy.MirroredStrategy(num_gpus=context.num_gpus()) - strategy.configure(cluster_spec=cluster_spec) - return strategy + distribution.configure(cluster_spec=cluster_spec) - def test_num_replicas_in_sync(self): - if not GPU_TEST: - self.skipTest("Not GPU test") - - strategy = self._get_distribution_strategy() + def test_num_replicas_in_sync(self, distribution): + self._configure_distribution_strategy(distribution) # We calculate the total number of gpus across the workers(2) specified in # the cluster spec. - self.assertEqual(context.num_gpus() * 2, strategy.num_replicas_in_sync) + self.assertEqual(context.num_gpus() * 2, distribution.num_replicas_in_sync) - def testMinimizeLossGraph(self): - self._test_minimize_loss_graph(self._get_distribution_strategy(), - learning_rate=0.05) + def testMinimizeLossGraph(self, distribution): + self._configure_distribution_strategy(distribution) + self._test_minimize_loss_graph(distribution, learning_rate=0.05) + + def testDeviceScope(self, distribution): + """Test the device scope of multi-worker MirroredStrategy.""" + self._configure_distribution_strategy(distribution) + with distribution.scope(): + a = constant_op.constant(1.) + with ops.device("/cpu:0"): + b = constant_op.constant(1.) + self.assertEqual(a.device, "/job:worker/task:0") + self.assertEqual(b.device, "/job:worker/task:0/device:CPU:0") + + def testMakeInputFnIterator(self, distribution): + self._configure_distribution_strategy(distribution) + dataset_fn = lambda: dataset_ops.Dataset.range(100) + num_gpus = context.num_gpus() + num_workers = 2 + + expected_values = [[i+j for j in range(num_gpus)] * num_workers + for i in range(0, 100, num_gpus)] + + with context.graph_mode(), self.cached_session() as sess: + # `expected_input_pipeline_id` is None because the input_fn will be called + # multiple times, each with a different input_pipeline_id. + input_fn = self._input_fn_to_test_input_context( + dataset_fn, + expected_num_replicas_in_sync=num_workers*num_gpus, + expected_num_input_pipelines=num_workers, + expected_input_pipeline_id=None) + iterator = distribution.make_input_fn_iterator(input_fn) + self._test_input_fn_iterator( + iterator, distribution.extended.worker_devices, expected_values, sess) + + def testUpdateConfigProto(self, distribution): + distribution.configure(cluster_spec={"worker": ["fake1", "fake2"]}) + + config_proto = config_pb2.ConfigProto() + new_config = distribution.update_config_proto(config_proto) + + # Verify isolate_session_state + self.assertTrue(new_config.isolate_session_state) class MultiWorkerMirroredStrategyTestWithChief( @@ -1473,6 +1392,19 @@ class MultiWorkerMirroredStrategyTestWithChief( strategy.configure(cluster_spec=self._cluster_spec) self._test_minimize_loss_graph(strategy, learning_rate=0.05) + def testMinimizeLossGraphCoreMirroredStrategy(self): + strategy = mirrored_strategy.CoreMirroredStrategy( + mirrored_strategy.all_local_devices()) + strategy.configure(cluster_spec=self._cluster_spec) + self._test_minimize_loss_graph(strategy, learning_rate=0.05) + + +def _replica_id(): + replica_id = ds_context.get_replica_context().replica_id_in_sync_group + if not isinstance(replica_id, ops.Tensor): + replica_id = constant_op.constant(replica_id) + return replica_id + if __name__ == "__main__": test.main() diff --git a/tensorflow/contrib/distribute/python/mirrored_strategy_test.py b/tensorflow/contrib/distribute/python/mirrored_strategy_test.py deleted file mode 100644 index bea684e77ca..00000000000 --- a/tensorflow/contrib/distribute/python/mirrored_strategy_test.py +++ /dev/null @@ -1,107 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for class MirroredStrategy.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib.distribute.python import mirrored_strategy -from tensorflow.contrib.distribute.python import strategy_test_lib -from tensorflow.python.eager import context -from tensorflow.python.eager import test -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import ops -from tensorflow.python.framework import test_util -from tensorflow.python.ops import variable_scope -from tensorflow.python.training import distribution_strategy_context - - -class MirroredOneCPUDistributionTest(strategy_test_lib.DistributionTestBase): - - def _get_distribution_strategy(self): - return mirrored_strategy.MirroredStrategy(["/device:CPU:0"]) - - def testMinimizeLossEager(self): - self._test_minimize_loss_eager(self._get_distribution_strategy()) - - def testMinimizeLossGraph(self): - self._test_minimize_loss_graph(self._get_distribution_strategy()) - - def testDeviceIndex(self): - self._test_device_index(self._get_distribution_strategy()) - - def testReplicaId(self): - self._test_replica_id(self._get_distribution_strategy()) - - @test_util.run_in_graph_and_eager_modes - def testCallAndMergeExceptions(self): - self._test_call_and_merge_exceptions(self._get_distribution_strategy()) - - -class VariableCreatorStackTest(test.TestCase): - - def testCreatorStacksAreThreadLocal(self): - devices = ["/device:CPU:0", "/device:GPU:0"] - dist = mirrored_strategy.MirroredStrategy(devices) - - def model_fn(device_id): - assert isinstance(device_id, int) - - def thread_creator_fn(next_creator, *args, **kwargs): - return next_creator(*args, **kwargs) + ":thread_" + str(device_id) - - with variable_scope.variable_creator_scope(thread_creator_fn): - # Create a variable in this scope. - v = variable_scope.variable(1.0) - - # This will pause the current thread, and execute the other thread. - distribution_strategy_context.get_replica_context().merge_call( - lambda _: _) - return v - - def main_thread_creator(next_creator, *args, **kwargs): - # We are not using the underlying next_creator for test purposes. - del next_creator, args, kwargs - return "main_thread" - - with context.graph_mode(), \ - dist.scope(), \ - variable_scope.variable_creator_scope(main_thread_creator): - result = dist.call_for_each_replica( - model_fn, args=(dist.worker_device_index,)) - result = dist.unwrap(result) - expected = ["main_thread:thread_0", "main_thread:thread_1"] - self.assertEquals(expected, result) - - -class MultiWorkerMirroredStrategyTest(test.TestCase): - - def testDeviceScope(self): - """Test the device scope of multi-worker MirroredStrategy.""" - with context.graph_mode(): - strategy = mirrored_strategy.MirroredStrategy(num_gpus=context.num_gpus()) - strategy.configure( - cluster_spec={"worker": ["/job:worker/task:0", "/job:worker/task:1"]}) - with strategy.scope(): - a = constant_op.constant(1.) - with ops.device("/cpu:0"): - b = constant_op.constant(1.) - self.assertEqual(a.device, "/job:worker/task:0") - self.assertEqual(b.device, "/job:worker/task:0/device:CPU:0") - - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/distribute/python/moving_averages_test.py b/tensorflow/contrib/distribute/python/moving_averages_test.py index 7ecc852d205..c492d8bafc9 100644 --- a/tensorflow/contrib/distribute/python/moving_averages_test.py +++ b/tensorflow/contrib/distribute/python/moving_averages_test.py @@ -32,7 +32,8 @@ from tensorflow.python.training import moving_averages all_combinations = combinations.combine( distribution=[combinations.default_strategy, combinations.one_device_strategy, - combinations.mirrored_strategy_with_gpu_and_cpu], + combinations.mirrored_strategy_with_gpu_and_cpu, + combinations.core_mirrored_strategy_with_gpu_and_cpu], mode=["graph"]) diff --git a/tensorflow/contrib/distribute/python/multi_worker_test_base.py b/tensorflow/contrib/distribute/python/multi_worker_test_base.py index 8eec3dc0f6e..147c9b83f86 100644 --- a/tensorflow/contrib/distribute/python/multi_worker_test_base.py +++ b/tensorflow/contrib/distribute/python/multi_worker_test_base.py @@ -18,8 +18,11 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function +import collections import contextlib import copy +import json +import os import threading import numpy as np @@ -271,7 +274,6 @@ class MultiWorkerTestBase(test.TestCase): return config - def _run_client(self, client_fn, task_type, task_id, num_gpus, *args, **kwargs): result = client_fn(task_type, task_id, num_gpus, *args, **kwargs) @@ -303,3 +305,101 @@ class MultiWorkerTestBase(test.TestCase): for t in threads: t.join() self.assertEqual(self._result, len(threads)) + + +class MockOsEnv(collections.Mapping): + """A class that allows per-thread TF_CONFIG.""" + + def __init__(self, *args): + self._dict = dict() + self._thread_local = threading.local() + super(MockOsEnv, self).__init__(*args) + + def get(self, key, default=None): + if not hasattr(self._thread_local, 'dict'): + self._thread_local.dict = dict() + if key == 'TF_CONFIG': + return dict.get(self._thread_local.dict, key, default) + else: + return dict.get(self._dict, key, default) + + def __getitem__(self, key): + if not hasattr(self._thread_local, 'dict'): + self._thread_local.dict = dict() + if key == 'TF_CONFIG': + return dict.__getitem__(self._thread_local.dict, key) + else: + return dict.__getitem__(self._dict, key) + + def __setitem__(self, key, val): + if not hasattr(self._thread_local, 'dict'): + self._thread_local.dict = dict() + if key == 'TF_CONFIG': + return dict.__setitem__(self._thread_local.dict, key, val) + else: + return dict.__setitem__(self._dict, key, val) + + def __iter__(self): + if not hasattr(self._thread_local, 'dict'): + self._thread_local.dict = dict() + for x in self._thread_local.dict.items(): + yield x + for x in self._dict.items(): + yield x + + def __len__(self): + if not hasattr(self._thread_local, 'dict'): + self._thread_local.dict = dict() + return self._thread_local.dict.__len__() + self._dict.__len__() + + +class IndependentWorkerTestBase(test.TestCase): + """Testing infra for independent workers.""" + + def setUp(self): + self._mock_os_env = MockOsEnv() + self._mock_context = test.mock.patch.object(os, 'environ', + self._mock_os_env) + super(IndependentWorkerTestBase, self).setUp() + self._mock_context.__enter__() + + def tearDown(self): + self._mock_context.__exit__(None, None, None) + super(IndependentWorkerTestBase, self).tearDown() + + def _task_thread(self, task_fn, tf_config, *args, **kwargs): + os.environ['TF_CONFIG'] = json.dumps(tf_config) + task_fn(*args, **kwargs) + + def _run_task_in_thread(self, task_fn, cluster_spec, task_type, task_id, + *args, **kwargs): + if task_type: + tf_config = { + 'cluster': cluster_spec, + 'task': { + 'type': task_type, + 'index': task_id + } + } + else: + tf_config = { + 'cluster': cluster_spec, + } + t = threading.Thread( + target=self._task_thread, + args=(task_fn, tf_config) + args, + kwargs=kwargs) + t.start() + return t + + def run_multiple_tasks_in_threads(self, task_fn, cluster_spec, *args, + **kwargs): + # The task_fn should create std_server by itself. + threads = {} + for task_type in cluster_spec.keys(): + threads[task_type] = [] + for task_id in range(len(cluster_spec[task_type])): + t = self._run_task_in_thread(task_fn, cluster_spec, task_type, task_id, + *args, **kwargs) + threads[task_type].append(t) + return threads diff --git a/tensorflow/contrib/distribute/python/one_device_strategy.py b/tensorflow/contrib/distribute/python/one_device_strategy.py index a0d8f938874..e322b6acb84 100644 --- a/tensorflow/contrib/distribute/python/one_device_strategy.py +++ b/tensorflow/contrib/distribute/python/one_device_strategy.py @@ -20,12 +20,14 @@ from __future__ import print_function import six -from tensorflow.contrib.distribute.python import values +from tensorflow.python.distribute import device_util +from tensorflow.python.distribute import distribute_lib +from tensorflow.python.distribute import values from tensorflow.python.framework import constant_op +from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops from tensorflow.python.ops import array_ops from tensorflow.python.ops import control_flow_ops -from tensorflow.python.training import distribute as distribute_lib from tensorflow.python.util import nest @@ -39,7 +41,14 @@ class OneDeviceStrategy(distribute_lib.DistributionStrategy): # implementations? def __init__(self, device): - super(OneDeviceStrategy, self).__init__() + super(OneDeviceStrategy, self).__init__(OneDeviceExtended(self, device)) + + +class OneDeviceExtended(distribute_lib.DistributionStrategyExtended): + """Implementation of OneDeviceStrategy.""" + + def __init__(self, container_strategy, device): + super(OneDeviceExtended, self).__init__(container_strategy) self._device = device self._default_device = device @@ -58,17 +67,33 @@ class OneDeviceStrategy(distribute_lib.DistributionStrategy): with ops.colocate_with(colocate_with): return next_creator(*args, **kwargs) - def distribute_dataset(self, dataset_fn): + def _make_dataset_iterator(self, dataset): + """Make iterator from dataset without splitting the batch.""" + worker = device_util.canonicalize("/device:CPU:0") + worker_device_pairs = [(worker, [self._device])] + return values.DatasetIterator(dataset, worker_device_pairs) + + def _distribute_dataset(self, dataset_fn): return values.PerReplicaDataset( self._call_dataset_fn(dataset_fn), [self._device]) - def _broadcast(self, tensor, destinations): + def _make_input_fn_iterator( + self, + input_fn, + replication_mode=distribute_lib.InputReplicationMode.PER_WORKER): + worker = device_util.canonicalize("/device:CPU:0") + worker_device_pairs = [(worker, [self._device])] + return values.InputFunctionIterator( + input_fn, worker_device_pairs, + [distribute_lib.InputContext()]) + + def _broadcast_to(self, tensor, destinations): del destinations return tensor # TODO(priyag): Deal with OutOfRange errors once b/111349762 is fixed. - def _run_steps_on_dataset(self, fn, iterator, iterations, - initial_loop_values=None): + def _experimental_run_steps_on_iterator(self, fn, iterator, iterations, + initial_loop_values=None): if initial_loop_values is None: initial_loop_values = {} initial_loop_values = nest.flatten(initial_loop_values) @@ -80,7 +105,7 @@ class OneDeviceStrategy(distribute_lib.DistributionStrategy): fn_inputs = iterator.get_next() if not isinstance(fn_inputs, tuple): fn_inputs = (fn_inputs,) - fn_result = fn(ctx, *fn_inputs) + fn_result = fn(ctx, fn_inputs) flat_last_step_outputs = nest.flatten(ctx.last_step_outputs) with ops.control_dependencies([fn_result]): return [i + 1] + flat_last_step_outputs @@ -114,25 +139,24 @@ class OneDeviceStrategy(distribute_lib.DistributionStrategy): return ctx def _call_for_each_replica(self, fn, args, kwargs): - with ops.device(self._device), _OneDeviceReplicaContext(self): + strategy = self._container_strategy() + with ops.device(self._device), _OneDeviceReplicaContext(strategy): return fn(*args, **kwargs) - def _reduce(self, aggregation, value, destinations): - del aggregation, destinations + def _reduce_to(self, reduce_op, value, destinations): + del reduce_op, destinations return value - def _update(self, var, options, fn, *args, **kwargs): + def _update(self, var, fn, args, kwargs, group): # The implementations of _update() and _update_non_slot() are identical # except _update() passes `var` as the first argument to `fn()`. - return self._update_non_slot(var, options, fn, var, *args, **kwargs) + return self._update_non_slot(var, fn, (var,) + tuple(args), kwargs, group) - def _update_non_slot(self, colocate_with, options, fn, *args, **kwargs): + def _update_non_slot(self, colocate_with, fn, args, kwargs, group): del colocate_with - should_group = options.pop("grouped") - assert not options # Validate that we are processing all of the options. with ops.device(self._device), distribute_lib.UpdateContext(self._device): result = fn(*args, **kwargs) - if should_group: + if group: return result else: return nest.map_structure(self._unwrap, result) @@ -148,11 +172,7 @@ class OneDeviceStrategy(distribute_lib.DistributionStrategy): return value @property - def num_replicas(self): - return 1 - - @property - def num_replicas_in_sync(self): + def _num_replicas_in_sync(self): return 1 @property @@ -167,8 +187,22 @@ class OneDeviceStrategy(distribute_lib.DistributionStrategy): del var_list return [self._device] - def _worker_device_index(self): - return 0 + @property + def experimental_should_init(self): + return True + + @property + def should_checkpoint(self): + return True + + @property + def should_save_summary(self): + return True + + # TODO(priyag): Delete this once all strategies use global batch size. + @property + def _global_batch_size(self): + return True class _OneDeviceReplicaContext(distribute_lib.ReplicaContext): @@ -176,12 +210,10 @@ class _OneDeviceReplicaContext(distribute_lib.ReplicaContext): def __init__(self, distribution_strategy): distribute_lib.ReplicaContext.__init__( - self, distribution_strategy, replica_id=0) - - @property - def device(self): - raise RuntimeError("Use .devices instead") + self, + distribution_strategy, + replica_id_in_sync_group=constant_op.constant(0, dtypes.int32)) @property def devices(self): - return [self._distribution_strategy.worker_devices[0]] + return [self._distribution_strategy.extended.worker_devices[0]] diff --git a/tensorflow/contrib/distribute/python/one_device_strategy_test.py b/tensorflow/contrib/distribute/python/one_device_strategy_test.py index 95f4cdb7868..d46cd6f529e 100644 --- a/tensorflow/contrib/distribute/python/one_device_strategy_test.py +++ b/tensorflow/contrib/distribute/python/one_device_strategy_test.py @@ -20,6 +20,7 @@ from __future__ import print_function from tensorflow.contrib.distribute.python import one_device_strategy from tensorflow.contrib.distribute.python import strategy_test_lib +from tensorflow.python.data.ops import dataset_ops from tensorflow.python.eager import test from tensorflow.python.framework import test_util @@ -35,9 +36,6 @@ class OneDeviceStrategyTest(strategy_test_lib.DistributionTestBase): def testMinimizeLossGraph(self): self._test_minimize_loss_graph(self._get_distribution_strategy()) - def testDeviceIndex(self): - self._test_device_index(self._get_distribution_strategy()) - def testReplicaId(self): self._test_replica_id(self._get_distribution_strategy()) @@ -45,6 +43,20 @@ class OneDeviceStrategyTest(strategy_test_lib.DistributionTestBase): def testCallAndMergeExceptions(self): self._test_call_and_merge_exceptions(self._get_distribution_strategy()) + @test_util.run_in_graph_and_eager_modes + def testMakeInputFnIterator(self): + d = one_device_strategy.OneDeviceStrategy("/device:CPU:0") + dataset_fn = lambda: dataset_ops.Dataset.range(10) + expected_values = [[i] for i in range(10)] + input_fn = self._input_fn_to_test_input_context( + dataset_fn, + expected_num_replicas_in_sync=1, + expected_num_input_pipelines=1, + expected_input_pipeline_id=0) + iterator = d.make_input_fn_iterator(input_fn) + self._test_input_fn_iterator( + iterator, d.extended.worker_devices, expected_values) + if __name__ == "__main__": test.main() diff --git a/tensorflow/contrib/distribute/python/parameter_server_strategy.py b/tensorflow/contrib/distribute/python/parameter_server_strategy.py index 790b37f8601..eaeb4d70301 100644 --- a/tensorflow/contrib/distribute/python/parameter_server_strategy.py +++ b/tensorflow/contrib/distribute/python/parameter_server_strategy.py @@ -18,10 +18,14 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function -from tensorflow.contrib.distribute.python import cross_tower_ops as cross_tower_ops_lib +import copy + from tensorflow.contrib.distribute.python import mirrored_strategy -from tensorflow.contrib.distribute.python import values +from tensorflow.python.distribute import cross_device_ops as cross_device_ops_lib +from tensorflow.python.distribute import device_util +from tensorflow.python.distribute import distribute_lib from tensorflow.python.distribute import multi_worker_util +from tensorflow.python.distribute import values from tensorflow.python.eager import context from tensorflow.python.framework import device as tf_device from tensorflow.python.framework import ops @@ -30,8 +34,6 @@ from tensorflow.python.ops import resource_variable_ops from tensorflow.python.ops import variable_scope as vs from tensorflow.python.platform import tf_logging as logging from tensorflow.python.training import device_setter -from tensorflow.python.training import device_util -from tensorflow.python.training import distribute as distribute_lib from tensorflow.python.util import nest _LOCAL_CPU = "/device:CPU:0" @@ -94,13 +96,21 @@ class ParameterServerStrategy(distribute_lib.DistributionStrategy): ValueError: if `cluster_spec` is given but `task_type` or `task_id` is not. """ - super(ParameterServerStrategy, self).__init__() + super(ParameterServerStrategy, self).__init__( + ParameterServerExtended(self, num_gpus_per_worker)) + + +class ParameterServerExtended(distribute_lib.DistributionStrategyExtended): + """Implementation of ParameterServerStrategy.""" + + def __init__(self, container_strategy, num_gpus_per_worker): + super(ParameterServerExtended, self).__init__(container_strategy) self._num_gpus_per_worker = num_gpus_per_worker self._initialize_local(num_gpus_per_worker) # We typically don't need to do all-reduce in this strategy. - self._cross_tower_ops = ( - cross_tower_ops_lib.ReductionToOneDeviceCrossDeviceOps( + self._cross_device_ops = ( + cross_device_ops_lib.ReductionToOneDeviceCrossDeviceOps( reduce_to_device=_LOCAL_CPU)) def _initialize_multi_worker(self, num_gpus_per_worker, cluster_spec, @@ -189,6 +199,7 @@ class ParameterServerStrategy(distribute_lib.DistributionStrategy): def _initialize_local(self, num_gpus_per_worker): """Initialize internal devices for local training.""" + self._worker_device = device_util.canonicalize("/device:CPU:0") # Define compute devices which is a list of device strings and one for each # replica. When there are GPUs, replicate operations on these GPUs. # Otherwise, place operations on CPU. @@ -221,15 +232,48 @@ class ParameterServerStrategy(distribute_lib.DistributionStrategy): "ParameterServerStrategy with compute_devices = %r, " "variable_device = %r", self._compute_devices, self._variable_device) - def distribute_dataset(self, dataset_fn): + def _distribute_dataset(self, dataset_fn): """Distributes the dataset to each local GPU.""" return values.PerReplicaDataset( self._call_dataset_fn(dataset_fn), self._compute_devices, True) - def _broadcast(self, tensor, destinations): - if not cross_tower_ops_lib.check_destinations(destinations): + def _make_dataset_iterator(self, dataset): + worker_device_pairs = [(self._worker_device, self._compute_devices)] + return values.DatasetIterator(dataset, worker_device_pairs, + self._num_replicas_in_sync) + + def _make_input_fn_iterator( + self, + input_fn, + replication_mode=distribute_lib.InputReplicationMode.PER_WORKER): + """Distributes the dataset to each local GPU.""" + if self._cluster_spec: + input_pipeline_id = multi_worker_util.id_in_cluster( + self._cluster_spec, self._task_type, self._task_id) + num_input_pipelines = multi_worker_util.worker_count( + self._cluster_spec, self._task_type) + else: + input_pipeline_id = 0 + num_input_pipelines = 1 + input_context = distribute_lib.InputContext( + num_input_pipelines=num_input_pipelines, + input_pipeline_id=input_pipeline_id, + num_replicas_in_sync=self._num_replicas_in_sync) + worker_device_pairs = [(self._worker_device, self._compute_devices)] + return values.InputFunctionIterator( + input_fn, worker_device_pairs, [input_context]) + + def _broadcast_to(self, tensor, destinations): + # This is both a fast path for Python constants, and a way to delay + # converting Python values to a tensor until we know what type it + # should be converted to. Otherwise we have trouble with: + # global_step.assign_add(1) + # since the `1` gets broadcast as an int32 but global_step is int64. + if isinstance(tensor, (float, int)): + return tensor + if not cross_device_ops_lib.check_destinations(destinations): destinations = self._compute_devices - return self._cross_tower_ops.broadcast(tensor, destinations) + return self._cross_device_ops.broadcast(tensor, destinations) def _allow_variable_partition(self): return not context.executing_eagerly() @@ -237,7 +281,7 @@ class ParameterServerStrategy(distribute_lib.DistributionStrategy): # TODO(yuefengz): not all ops in device_setter.STANDARD_PS_OPS will go through # this creator, such as "MutableHashTable". def _create_variable(self, next_creator, *args, **kwargs): - if self.num_replicas_in_sync > 1: + if self._num_replicas_in_sync > 1: aggregation = kwargs.pop("aggregation", vs.VariableAggregation.NONE) if aggregation not in ( vs.VariableAggregation.NONE, @@ -293,39 +337,35 @@ class ParameterServerStrategy(distribute_lib.DistributionStrategy): def _call_for_each_replica(self, fn, args, kwargs): # pylint: disable=protected-access - return mirrored_strategy._call_for_each_replica(self, fn, args, kwargs) + return mirrored_strategy._call_for_each_replica( + self._container_strategy(), fn, args, kwargs) def _verify_destinations_not_different_worker(self, destinations): if not self._cluster_spec: return if destinations is None: return - for d in cross_tower_ops_lib.get_devices_from(destinations): + for d in cross_device_ops_lib.get_devices_from(destinations): d_spec = tf_device.DeviceSpec.from_string(d) if d_spec.job == self._task_type and d_spec.task != self._task_id: raise ValueError( "Cannot reduce to another worker: %r, current worker is %r" % (d, self._worker_device)) - def _reduce(self, aggregation, value, destinations): + def _reduce_to(self, reduce_op, value, destinations): self._verify_destinations_not_different_worker(destinations) if not isinstance(value, values.DistributedValues): # pylint: disable=protected-access return mirrored_strategy._reduce_non_distributed_value( - self, aggregation, value, destinations) - if aggregation == vs.VariableAggregation.ONLY_FIRST_REPLICA: - return self.broadcast(value.get(self._compute_devices[0]), destinations) - return self._cross_tower_ops.reduce( - aggregation, value, destinations=destinations) + self, reduce_op, value, destinations) + return self._cross_device_ops.reduce( + reduce_op, value, destinations=destinations) - def _batch_reduce(self, aggregation, value_destination_pairs): - if aggregation == vs.VariableAggregation.ONLY_FIRST_REPLICA: - return [self.broadcast(v.get(self._compute_devices[0]), d) - for v, d in value_destination_pairs] + def _batch_reduce_to(self, reduce_op, value_destination_pairs): for _, destinations in value_destination_pairs: self._verify_destinations_not_different_worker(destinations) - return self._cross_tower_ops.batch_reduce(aggregation, - value_destination_pairs) + return self._cross_device_ops.batch_reduce(reduce_op, + value_destination_pairs) def _select_single_value(self, structured): """Select any single values in `structured`.""" @@ -349,30 +389,26 @@ class ParameterServerStrategy(distribute_lib.DistributionStrategy): return nest.map_structure(_select_fn, structured) - def _update(self, var, options, fn, *args, **kwargs): + def _update(self, var, fn, args, kwargs, group): if isinstance(var, values.AggregatingVariable): var = var.get() if not isinstance(var, resource_variable_ops.ResourceVariable): raise ValueError( "You can not update `var` %r. It must be a Variable." % var) - should_group = options.pop("grouped") - assert not options # Validate that we are processing all of the options. with ops.colocate_with(var), distribute_lib.UpdateContext(var.device): result = fn(var, *self._select_single_value(args), **self._select_single_value(kwargs)) - if should_group: + if group: return result else: return nest.map_structure(self._unwrap, result) # TODO(yuefengz): does it need to call _select_single_value? - def _update_non_slot(self, colocate_with, options, fn, *args, **kwargs): - should_group = options.pop("grouped") - assert not options # Validate that we are processing all of the options. + def _update_non_slot(self, colocate_with, fn, args, kwargs, group): with ops.device( colocate_with.device), distribute_lib.UpdateContext(colocate_with): result = fn(*args, **kwargs) - if should_group: + if group: return result else: return nest.map_structure(self._unwrap, result) @@ -398,11 +434,11 @@ class ParameterServerStrategy(distribute_lib.DistributionStrategy): # variables. return array_ops.identity(var) - def configure(self, - session_config=None, - cluster_spec=None, - task_type=None, - task_id=None): + def _configure(self, + session_config=None, + cluster_spec=None, + task_type=None, + task_id=None): """Configures the strategy class. The strategy object will be re-initialized if `cluster_spec` is given but @@ -433,28 +469,30 @@ class ParameterServerStrategy(distribute_lib.DistributionStrategy): self._initialize_multi_worker(self._num_gpus_per_worker, self._cluster_spec, task_type, task_id) - if not session_config or not self._cluster_spec: - return + if session_config: + session_config.CopyFrom(self._update_config_proto(session_config)) - session_config.isolate_session_state = False + def _update_config_proto(self, config_proto): + updated_config = copy.deepcopy(config_proto) + if not self._cluster_spec: + updated_config.isolate_session_state = True + return updated_config + + updated_config.isolate_session_state = False - assert self._cluster_spec assert self._task_type assert self._task_id is not None # The device filters prevent communication between workers. if self._task_type not in ["chief", "worker"]: - return - del session_config.device_filters[:] - session_config.device_filters.extend( + return updated_config + del updated_config.device_filters[:] + updated_config.device_filters.extend( ["/job:%s/task:%d" % (self._task_type, self._task_id), "/job:ps"]) + return updated_config @property - def num_replicas(self): - return len(self._compute_devices) - - @property - def num_replicas_in_sync(self): + def _num_replicas_in_sync(self): return len(self._compute_devices) @property @@ -470,11 +508,12 @@ class ParameterServerStrategy(distribute_lib.DistributionStrategy): return min(var_list, key=lambda x: x.name) @property - def between_graph(self): + def experimental_between_graph(self): + # TODO(yuefengz): Should this return False in the local case? return True @property - def should_init(self): + def experimental_should_init(self): return self._is_chief @property @@ -484,3 +523,8 @@ class ParameterServerStrategy(distribute_lib.DistributionStrategy): @property def should_save_summary(self): return self._is_chief + + # TODO(priyag): Delete this once all strategies use global batch size. + @property + def _global_batch_size(self): + return False diff --git a/tensorflow/contrib/distribute/python/parameter_server_strategy_test.py b/tensorflow/contrib/distribute/python/parameter_server_strategy_test.py index 81a23c89030..83d7473666a 100644 --- a/tensorflow/contrib/distribute/python/parameter_server_strategy_test.py +++ b/tensorflow/contrib/distribute/python/parameter_server_strategy_test.py @@ -25,14 +25,21 @@ from absl.testing import parameterized from tensorflow.contrib.distribute.python import combinations from tensorflow.contrib.distribute.python import multi_worker_test_base from tensorflow.contrib.distribute.python import parameter_server_strategy -from tensorflow.contrib.distribute.python import values +from tensorflow.contrib.distribute.python import strategy_test_lib from tensorflow.core.protobuf import config_pb2 +from tensorflow.python.data.ops import dataset_ops +from tensorflow.python.distribute import device_util +from tensorflow.python.distribute import distribution_strategy_context as ds_context from tensorflow.python.distribute import multi_worker_util +from tensorflow.python.distribute import reduce_util +from tensorflow.python.distribute import values from tensorflow.python.eager import backprop from tensorflow.python.eager import context from tensorflow.python.estimator import run_config from tensorflow.python.framework import constant_op +from tensorflow.python.framework import errors from tensorflow.python.framework import ops +from tensorflow.python.framework import tensor_util from tensorflow.python.layers import core from tensorflow.python.ops import array_ops from tensorflow.python.ops import control_flow_ops @@ -41,8 +48,6 @@ from tensorflow.python.ops import partitioned_variables from tensorflow.python.ops import variable_scope from tensorflow.python.ops import variables from tensorflow.python.platform import test -from tensorflow.python.training import device_util -from tensorflow.python.training import distribution_strategy_context from tensorflow.python.training import training_util CHIEF = run_config.TaskType.CHIEF @@ -50,6 +55,13 @@ WORKER = run_config.TaskType.WORKER PS = run_config.TaskType.PS +def _get_replica_id_integer(): + replica_id = ds_context.get_replica_context().replica_id_in_sync_group + if isinstance(replica_id, ops.Tensor): + replica_id = tensor_util.constant_value(replica_id) + return replica_id + + class ParameterServerStrategyTestBase( multi_worker_test_base.MultiWorkerTestBase): @@ -94,9 +106,8 @@ class ParameterServerStrategyTestBase( if num_gpus == 0: last_part_device = 'device:CPU:0' else: - last_part_device = ( - 'device:GPU:%d' % - distribution_strategy_context.get_replica_context().replica_id) + replica_id = _get_replica_id_integer() + last_part_device = ('device:GPU:%d' % replica_id) a = constant_op.constant(1.0) b = constant_op.constant(2.0) @@ -261,18 +272,16 @@ class ParameterServerStrategyTestBase( if 'CPU' in compute_device: replica_compute_device = '/device:CPU:0' else: - replica_compute_device = ( - '/device:GPU:%d' % - distribution_strategy_context.get_replica_context().replica_id) + replica_id = _get_replica_id_integer() + replica_compute_device = ('/device:GPU:%d' % replica_id) replica_compute_device = device_util.canonicalize( replica_compute_device) if 'CPU' in variable_device: replica_variable_device = '/device:CPU:0' else: - replica_variable_device = ( - '/device:GPU:%d' % - distribution_strategy_context.get_replica_context().replica_id) + replica_id = _get_replica_id_integer() + replica_variable_device = ('/device:GPU:%d' % replica_id) replica_variable_device = device_util.canonicalize( replica_variable_device) @@ -354,9 +363,9 @@ class ParameterServerStrategyTestBase( def _test_simple_increment(self, task_type, task_id, num_gpus): d, master_target, sess_config = self._get_test_objects( task_type, task_id, num_gpus) - if hasattr(d, '_cluster_spec') and d._cluster_spec: - num_workers = len(d._cluster_spec.as_dict().get(WORKER)) - if 'chief' in d._cluster_spec.as_dict(): + if d.extended._cluster_spec: + num_workers = len(d.extended._cluster_spec.as_dict().get(WORKER)) + if 'chief' in d.extended._cluster_spec.as_dict(): num_workers += 1 else: num_workers = 1 @@ -389,7 +398,7 @@ class ParameterServerStrategyTestBase( x, y, z, train_op = d.call_for_each_replica(model_fn) train_op = d.group(train_op) - if context.num_gpus() < d._num_gpus_per_worker: + if context.num_gpus() < d.extended._num_gpus_per_worker: return True if task_id == 0: @@ -426,9 +435,9 @@ class ParameterServerStrategyTestBase( task_type, task_id, num_gpus) if task_type: # Multi-worker - assert hasattr(d, '_cluster_spec') and d._cluster_spec - num_workers = len(d._cluster_spec.as_dict().get(WORKER)) - if CHIEF in d._cluster_spec.as_dict(): + assert hasattr(d.extended, '_cluster_spec') and d.extended._cluster_spec + num_workers = len(d.extended._cluster_spec.as_dict().get(WORKER)) + if CHIEF in d.extended._cluster_spec.as_dict(): num_workers += 1 else: # local @@ -472,8 +481,8 @@ class ParameterServerStrategyTestBase( before_list.append(fetched) with ops.control_dependencies([fetched]): # TODO(yuefengz): support non-Mirrored variable as destinations. - g = d.reduce( - variable_scope.VariableAggregation.SUM, g, destinations=v) + g = d.extended.reduce_to( + reduce_util.ReduceOp.SUM, g, destinations=v) with ops.control_dependencies( d.update(v, update, g, grouped=False)): after_list.append(d.read_var(v)) @@ -481,11 +490,12 @@ class ParameterServerStrategyTestBase( before_out, after_out = step() - if context.num_gpus() < d._num_gpus_per_worker: + if context.num_gpus() < d.extended._num_gpus_per_worker: return True if (not task_type or - multi_worker_util.is_chief(d._cluster_spec, task_type, task_id)): + multi_worker_util.is_chief( + d.extended._cluster_spec, task_type, task_id)): variables.global_variables_initializer().run() # Workers waiting for chief worker's initializing variables. @@ -508,8 +518,40 @@ class ParameterServerStrategyTestBase( self.assertLess(error_after, error_before) return error_after < error_before + def _test_input_fn_iterator(self, task_type, task_id, num_gpus, input_fn, + expected_values): + distribution, master_target, config = self._get_test_objects( + task_type, task_id, num_gpus) + devices = distribution.extended.worker_devices + + with ops.Graph().as_default(), \ + self.cached_session(config=config, + target=master_target) as sess: + iterator = distribution.make_input_fn_iterator(input_fn) + sess.run(iterator.initialize()) + + for expected_value in expected_values: + next_element = iterator.get_next() + computed_value = sess.run( + [values.select_device(d, next_element) for d in devices]) + self.assertEqual(expected_value, computed_value) + + with self.assertRaises(errors.OutOfRangeError): + next_element = iterator.get_next() + sess.run([values.select_device(d, next_element) for d in devices]) + + # After re-initializing the iterator, should be able to iterate again. + sess.run(iterator.initialize()) + + for expected_value in expected_values: + next_element = iterator.get_next() + computed_value = sess.run( + [values.select_device(d, next_element) for d in devices]) + self.assertEqual(expected_value, computed_value) + class ParameterServerStrategyTest(ParameterServerStrategyTestBase, + strategy_test_lib.DistributionTestBase, parameterized.TestCase): @classmethod @@ -574,6 +616,73 @@ class ParameterServerStrategyTest(ParameterServerStrategyTestBase, def testMinimizeLossGraphLocal(self, num_gpus): self._test_minimize_loss_graph(None, None, num_gpus) + # TODO(priyag): Refactor this and other multi worker tests. + @combinations.generate( + combinations.combine(mode=['graph'], num_gpus=[1, 2], required_gpus=1)) + def testMakeInputFnIteratorDistributed(self, num_gpus): + if context.num_gpus() < num_gpus: + self.skipTest('Not enough GPUs') + dataset_fn = lambda: dataset_ops.Dataset.range(100) + expected_values = [[i+j for j in range(num_gpus)] + for i in range(0, 100, num_gpus)] + + input_fn = self._input_fn_to_test_input_context( + dataset_fn, + expected_num_replicas_in_sync=num_gpus, + expected_num_input_pipelines=3, + expected_input_pipeline_id=1) # because task_id = 1 + self._test_input_fn_iterator('worker', 1, num_gpus, + input_fn, expected_values) + + @combinations.generate( + combinations.combine(mode=['graph'], num_gpus=[1, 2], required_gpus=1)) + def testMakeInputFnIteratorLocal(self, num_gpus): + if context.num_gpus() < num_gpus: + self.skipTest('Not enough GPUs') + dataset_fn = lambda: dataset_ops.Dataset.range(100) + expected_values = [[i+j for j in range(num_gpus)] + for i in range(0, 100, num_gpus)] + + input_fn = self._input_fn_to_test_input_context( + dataset_fn, + expected_num_replicas_in_sync=num_gpus, + expected_num_input_pipelines=1, + expected_input_pipeline_id=0) # only one worker and pipeline for local. + self._test_input_fn_iterator(None, None, num_gpus, + input_fn, expected_values) + + def testGlobalStepUpdate(self): + strategy = parameter_server_strategy.ParameterServerStrategy( + num_gpus_per_worker=context.num_gpus()) + self._test_global_step_update(strategy) + + def testUpdateConfigProtoMultiWorker(self): + distribution = parameter_server_strategy.ParameterServerStrategy( + num_gpus_per_worker=2) + distribution.configure( + cluster_spec=self._cluster_spec, task_type='worker', task_id=1) + + config_proto = config_pb2.ConfigProto(device_filters=['to_be_overridden']) + + new_config = distribution.update_config_proto(config_proto) + + # Verify device filters. + self.assertEqual(['/job:worker/task:1', '/job:ps'], + new_config.device_filters) + + # Verify isolate_session_state + self.assertFalse(new_config.isolate_session_state) + + def testUpdateConfigProtoLocal(self): + distribution = parameter_server_strategy.ParameterServerStrategy( + num_gpus_per_worker=2) + + config_proto = config_pb2.ConfigProto() + new_config = distribution.update_config_proto(config_proto) + + # Verify isolate_session_state + self.assertTrue(new_config.isolate_session_state) + class ParameterServerStrategyWithChiefTest(ParameterServerStrategyTestBase, parameterized.TestCase): @@ -616,9 +725,9 @@ class ParameterServerStrategyWithChiefTest(ParameterServerStrategyTestBase, v = variable_scope.get_variable('v', initializer=10.0) _ = v * v v, = tape.watched_variables() - w = distribution.value_container(v) + w = distribution.extended.value_container(v) self.assertIs(values.AggregatingVariable, type(w)) - distribution.call_for_each_replica(f) + distribution.extended.call_for_each_replica(f) if __name__ == '__main__': diff --git a/tensorflow/contrib/distribute/python/step_fn.py b/tensorflow/contrib/distribute/python/step_fn.py index 3dc815f0371..c928b6d9f1f 100644 --- a/tensorflow/contrib/distribute/python/step_fn.py +++ b/tensorflow/contrib/distribute/python/step_fn.py @@ -94,7 +94,7 @@ class StandardSingleLossStep(StandardInputStep): def __call__(self): with self._distribution.scope(): - def step_fn(ctx, *inputs): + def step_fn(ctx, inputs): """Function to run one iteration with one input.""" gradients_fn = backprop.implicit_grad(self._loss_fn) gradients_fn = optimizer_lib.get_filtered_grad_fn(gradients_fn) diff --git a/tensorflow/contrib/distribute/python/strategy_test_lib.py b/tensorflow/contrib/distribute/python/strategy_test_lib.py index 3c0c10430eb..d50b142c5e9 100644 --- a/tensorflow/contrib/distribute/python/strategy_test_lib.py +++ b/tensorflow/contrib/distribute/python/strategy_test_lib.py @@ -19,16 +19,21 @@ from __future__ import division from __future__ import print_function from tensorflow.core.protobuf import config_pb2 +from tensorflow.python.distribute import distribution_strategy_context as ds_context +from tensorflow.python.distribute import reduce_util +from tensorflow.python.distribute import values from tensorflow.python.eager import backprop from tensorflow.python.eager import context from tensorflow.python.eager import test from tensorflow.python.framework import constant_op +from tensorflow.python.framework import dtypes +from tensorflow.python.framework import errors from tensorflow.python.framework import ops from tensorflow.python.layers import core from tensorflow.python.ops import array_ops +from tensorflow.python.ops import init_ops from tensorflow.python.ops import variable_scope from tensorflow.python.ops import variables -from tensorflow.python.training import distribution_strategy_context from tensorflow.python.training import optimizer @@ -45,8 +50,7 @@ def _raise_exception_fn(_=None): # Must be the argument to a distribution.call_for_each_replica() call, calls a # get_replica_context().merge_call() that raises an exception. def _merge_raises_fn(): - distribution_strategy_context.get_replica_context().merge_call( - _raise_exception_fn) + ds_context.get_replica_context().merge_call(_raise_exception_fn) # Must be the argument to a get_replica_context().merge_call() call, calls @@ -59,8 +63,7 @@ def _call_raises_fn(dist): # calls a get_replica_context().merge_call() that calls a # call_for_each_replica() that raises an exception. def _merge_call_raises_fn(): - distribution_strategy_context.get_replica_context().merge_call( - _call_raises_fn) + ds_context.get_replica_context().merge_call(_call_raises_fn) # Must be the argument to a get_replica_context().merge_call() call, calls @@ -74,8 +77,7 @@ def _call_merge_raises_fn(dist): # get_replica_context().merge_call() that calls a call_for_each_replica() that # calls a get_replica_context().merge_call() that raises an exception. def _merge_call_merge_raises_fn(): - distribution_strategy_context.get_replica_context().merge_call( - _call_merge_raises_fn) + ds_context.get_replica_context().merge_call(_call_merge_raises_fn) class DistributionTestBase(test.TestCase): @@ -114,8 +116,8 @@ class DistributionTestBase(test.TestCase): before_list.append(fetched) # control_dependencies irrelevant but harmless in eager execution with ops.control_dependencies([fetched]): - g = d.reduce( - variable_scope.VariableAggregation.SUM, g, destinations=v) + g = d.extended.reduce_to( + reduce_util.ReduceOp.SUM, g, destinations=v) with ops.control_dependencies(d.update( v, update, g, grouped=False)): after_list.append(d.read_var(v)) @@ -169,8 +171,8 @@ class DistributionTestBase(test.TestCase): fetched = d.read_var(v) before_list.append(fetched) with ops.control_dependencies([fetched]): - g = d.reduce( - variable_scope.VariableAggregation.SUM, g, destinations=v) + g = d.extended.reduce_to( + reduce_util.ReduceOp.SUM, g, destinations=v) with ops.control_dependencies(d.update( v, update, g, grouped=False)): after_list.append(d.read_var(v)) @@ -189,31 +191,20 @@ class DistributionTestBase(test.TestCase): # Error should go down self.assertLess(error_after, error_before) - def _test_device_index(self, d): - with d.scope(): - expected_devices = [False] * len(d.worker_devices) - - def mark_devices_fn(device_id): - self.assertLess(device_id, len(d.worker_devices)) - self.assertFalse(expected_devices[device_id]) - expected_devices[device_id] = True - - d.call_for_each_replica(mark_devices_fn, args=(d.worker_device_index,)) - self.assertAllEqual(expected_devices, [True] * len(d.worker_devices)) - def _test_replica_id(self, d): with d.scope(): - expected_devices = [False] * len(d.worker_devices) + expected_devices = [False] * len(d.extended.worker_devices) def mark_devices_fn(): - replica_id = ( - distribution_strategy_context.get_replica_context().replica_id) - self.assertLess(replica_id, len(d.worker_devices)) + replica_id = self.evaluate( + ds_context.get_replica_context().replica_id_in_sync_group) + self.assertLess(replica_id, len(d.extended.worker_devices)) self.assertFalse(expected_devices[replica_id]) expected_devices[replica_id] = True d.call_for_each_replica(mark_devices_fn) - self.assertAllEqual(expected_devices, [True] * len(d.worker_devices)) + self.assertAllEqual(expected_devices, + [True] * len(d.extended.worker_devices)) def _test_call_and_merge_exceptions(self, dist): with dist.scope(): @@ -225,3 +216,78 @@ class DistributionTestBase(test.TestCase): dist.call_for_each_replica(_merge_call_raises_fn) with self.assertRaises(_TestException): dist.call_for_each_replica(_merge_call_merge_raises_fn) + + def _input_fn_to_test_input_context(self, + dataset_fn, + expected_num_replicas_in_sync, + expected_num_input_pipelines, + expected_input_pipeline_id): + # Use a list of one element as counter so that it can be captured by the + # `_input_fn`. This counter is incremented by 1 each time an input_fn is + # called. We use this counter to check whether the `input_pipeline_id` + # matches the counter in the in-graph replication. + worker_id_counter = [0] + + def _input_fn(input_context): + """Input fn for testing.""" + self.assertIsNotNone(input_context) + self.assertEqual(expected_num_replicas_in_sync, + input_context.num_replicas_in_sync) + self.assertEqual(expected_num_input_pipelines, + input_context.num_input_pipelines) + if expected_input_pipeline_id is not None: + self.assertEqual(expected_input_pipeline_id, + input_context.input_pipeline_id) + else: + self.assertEqual(worker_id_counter[0], input_context.input_pipeline_id) + worker_id_counter[0] += 1 + + return dataset_fn() + + return _input_fn + + def _test_input_fn_iterator(self, iterator, devices, expected_values, + sess=None): + evaluate = lambda x: sess.run(x) if sess else self.evaluate(x) + evaluate(iterator.initialize()) + + for expected_value in expected_values: + next_element = iterator.get_next() + computed_value = evaluate( + [values.select_device(d, next_element) for d in devices]) + self.assertEqual(expected_value, computed_value) + + with self.assertRaises(errors.OutOfRangeError): + next_element = iterator.get_next() + evaluate([values.select_device(d, next_element) for d in devices]) + + # After re-initializing the iterator, should be able to iterate again. + evaluate(iterator.initialize()) + + for expected_value in expected_values: + next_element = iterator.get_next() + computed_value = evaluate( + [values.select_device(d, next_element) for d in devices]) + self.assertEqual(expected_value, computed_value) + + def _test_global_step_update(self, strategy): + with strategy.scope(): + global_step = variable_scope.get_variable( + "global_step", + shape=[], + dtype=dtypes.int64, + initializer=init_ops.zeros_initializer(), + trainable=False, + aggregation=variables.VariableAggregation.ONLY_FIRST_REPLICA) + self.evaluate(variables.global_variables_initializer()) + + def model_fn(): + train_op = global_step.assign_add(1) + value = global_step.read_value() + return train_op, value + + train_ops, value = strategy.call_for_each_replica(model_fn) + self.evaluate(strategy.group(train_ops)) + global_step_tensors = strategy.unwrap(value) + global_step_values = self.evaluate(global_step_tensors) + self.assertEqual([1] * len(global_step_tensors), global_step_values) diff --git a/tensorflow/contrib/distribute/python/tpu_strategy.py b/tensorflow/contrib/distribute/python/tpu_strategy.py index f5b4531ba8c..39ed8f7cf10 100644 --- a/tensorflow/contrib/distribute/python/tpu_strategy.py +++ b/tensorflow/contrib/distribute/python/tpu_strategy.py @@ -21,25 +21,28 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function +import copy import functools -from tensorflow.contrib.distribute.python import cross_tower_ops as cross_tower_ops_lib -from tensorflow.contrib.distribute.python import values from tensorflow.contrib.tpu.python.ops import tpu_ops from tensorflow.contrib.tpu.python.tpu import tpu from tensorflow.contrib.tpu.python.tpu import tpu_system_metadata as tpu_system_metadata_lib from tensorflow.contrib.tpu.python.tpu import training_loop +from tensorflow.python.distribute import cross_device_ops as cross_device_ops_lib +from tensorflow.python.distribute import device_util +from tensorflow.python.distribute import distribute_lib +from tensorflow.python.distribute import reduce_util +from tensorflow.python.distribute import values from tensorflow.python.eager import context from tensorflow.python.eager import tape from tensorflow.python.framework import constant_op +from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops +from tensorflow.python.framework import tensor_util from tensorflow.python.ops import array_ops from tensorflow.python.ops import control_flow_ops from tensorflow.python.ops import math_ops from tensorflow.python.ops import variable_scope as vs -from tensorflow.python.ops import variables as variables_lib -from tensorflow.python.training import device_util -from tensorflow.python.training import distribute as distribute_lib from tensorflow.python.util import nest @@ -130,8 +133,21 @@ class TPUStrategy(distribute_lib.DistributionStrategy): num_cores: Number of cores to use on the TPU. If None specified, then auto-detect the cores and topology of the TPU system. """ - super(TPUStrategy, self).__init__() + super(TPUStrategy, self).__init__(TPUExtended( + self, tpu_cluster_resolver, steps_per_run, num_cores)) + @property + def steps_per_run(self): + """DEPRECATED: use .extended.steps_per_run instead.""" + return self._extended.steps_per_run + + +class TPUExtended(distribute_lib.DistributionStrategyExtended): + """Implementation of TPUStrategy.""" + + def __init__(self, container_strategy, tpu_cluster_resolver, steps_per_run, + num_cores=None): + super(TPUExtended, self).__init__(container_strategy) self._tpu_cluster_resolver = tpu_cluster_resolver self._tpu_metadata = get_tpu_system_metadata(self._tpu_cluster_resolver) # TODO(sourabhbajaj): Change this from num_cores to metadata_override @@ -145,7 +161,7 @@ class TPUStrategy(distribute_lib.DistributionStrategy): self._host_device = self.get_host_cpu_device(0) self._tpu_devices = sorted(device_map.keys()) # Only create variables for the number of replicas we're running. - self._tpu_devices = self._tpu_devices[:self.num_replicas] + self._tpu_devices = self._tpu_devices[:self._num_replicas_in_sync] # TODO(sourabhbajaj): Remove this once performance of running one step # at a time is comparable to multiple steps. @@ -214,7 +230,17 @@ class TPUStrategy(distribute_lib.DistributionStrategy): return enqueue_op_per_host - def distribute_dataset(self, dataset_fn): + def _make_dataset_iterator(self, dataset): + """Make iterators for each of the TPU hosts.""" + + worker_devices = [ + (self.get_host(hid), [self.get_host_cpu_device(hid)]) + for hid in range(self.num_hosts) + ] + return values.DatasetIterator(dataset, worker_devices, + self._num_replicas_in_sync) + + def _distribute_dataset(self, dataset_fn): worker_devices = [ (self.get_host(hid), [self.get_host_cpu_device(hid)]) for hid in range(self.num_hosts) @@ -225,12 +251,11 @@ class TPUStrategy(distribute_lib.DistributionStrategy): # TODO(priyag): Deal with OutOfRange errors once b/111349762 is fixed. # TODO(sourabhbajaj): Remove the initial_loop_values parameter when we have # a mechanism to infer the outputs of `fn`. Pending b/110550782. - def _run_steps_on_dataset(self, fn, multi_worker_iterator, iterations, - initial_loop_values=None): - + def _experimental_run_steps_on_iterator( + self, fn, multi_worker_iterator, iterations, initial_loop_values=None): output_shapes = multi_worker_iterator.output_shapes shapes = nest.flatten(output_shapes) - if any([not s.is_fully_defined() for s in shapes]): + if any(not s.is_fully_defined() for s in shapes): raise ValueError( "TPU currently requires fully defined shapes. Either use " "set_shape() on the input tensors or use " @@ -251,13 +276,13 @@ class TPUStrategy(distribute_lib.DistributionStrategy): initial_loop_values = {} initial_loop_values = nest.flatten(initial_loop_values) ctx = values.MultiStepContext() - def run_fn(*args, **kwargs): + + def run_fn(): """Single step on the TPU device.""" - del args, kwargs fn_inputs = dequeue_fn() if not isinstance(fn_inputs, tuple): fn_inputs = (fn_inputs,) - fn_result = fn(ctx, *fn_inputs) + fn_result = fn(ctx, fn_inputs) flat_last_step_outputs = nest.flatten(ctx.last_step_outputs) if flat_last_step_outputs: with ops.control_dependencies([fn_result]): @@ -265,11 +290,6 @@ class TPUStrategy(distribute_lib.DistributionStrategy): else: return fn_result - # TODO(sourabhbajaj): The input to while loop should be based on the output - # type of the step_fn - def iterate_on_tpu(): - return training_loop.repeat(iterations, run_fn, initial_loop_values) - # We capture the control_flow_context at this point, before we run `fn` # inside a while_loop and TPU replicate context. This is useful in cases # where we might need to exit these contexts and get back to the outer @@ -279,38 +299,70 @@ class TPUStrategy(distribute_lib.DistributionStrategy): self._outer_control_flow_context = ( ops.get_default_graph()._get_control_flow_context()) # pylint: disable=protected-access - replicate_inputs = [[]] * self.num_replicas - replicate_outputs = tpu.replicate(iterate_on_tpu, replicate_inputs) + def rewrite_fn(*args): + """The rewritten step fn running on TPU.""" + del args + replicate_inputs = [[]] * self._num_replicas_in_sync + replicate_outputs = tpu.replicate(run_fn, replicate_inputs) + + # If run_fn has tensor outputs, tpu.replicate returns a list of list. We + # will flatten it in this case. If run_fn has no tensor outputs, + # tpu.replicate returns a list of no_ops, we will keep the output as it + # is. + if isinstance(replicate_outputs[0], list): + replicate_outputs = nest.flatten(replicate_outputs) + + return replicate_outputs + + # TODO(sourabhbajaj): The input to while loop should be based on the output + # type of the step_fn + assert isinstance(initial_loop_values, list) + initial_loop_values = initial_loop_values * self._num_replicas_in_sync + + # Put the while loop op on host 0. + with ops.device(self.get_host_cpu_device(0)): + replicate_outputs = training_loop.repeat(iterations, rewrite_fn, + initial_loop_values) + del self._outer_control_flow_context ctx.run_op = control_flow_ops.group(replicate_outputs, enqueue_ops) - # Filter out any ops from the outputs, typically this would be the case - # when there were no tensor outputs. - last_step_tensor_outputs = [x for x in replicate_outputs - if not isinstance(x, ops.Operation)] + if isinstance(replicate_outputs, list): + # Filter out any ops from the outputs, typically this would be the case + # when there were no tensor outputs. + last_step_tensor_outputs = [ + x for x in replicate_outputs if not isinstance(x, ops.Operation) + ] - # Outputs are currently of the structure (grouped by device) - # [[output0_device0, output1_device0, output2_device0], - # [output0_device1, output1_device1, output2_device1]] - # Convert this to the following structure instead: (grouped by output) - # [[output0_device0, output0_device1], - # [output1_device0, output1_device1], - # [output2_device0, output2_device1]] - last_step_tensor_outputs = [list(x) for x in zip(*last_step_tensor_outputs)] + # Outputs are currently of the structure (flattened) + # [output0_device0, output1_device0, output2_device0, + # output0_device1, output1_device1, output2_device1, + # ...] + # Convert this to the following structure instead: (grouped by output) + # [[output0_device0, output0_device1], + # [output1_device0, output1_device1], + # [output2_device0, output2_device1]] + output_num = len(last_step_tensor_outputs) // self._num_replicas_in_sync + last_step_tensor_outputs = [ + last_step_tensor_outputs[i::output_num] for i in range(output_num) + ] + else: + # no tensors returned. + last_step_tensor_outputs = [] # Convert replicate_outputs to the original dict structure of # last_step_outputs. last_step_tensor_outputs_dict = nest.pack_sequence_as( ctx.last_step_outputs, last_step_tensor_outputs) - for (name, aggregation) in ctx._last_step_outputs_aggregations.items(): # pylint: disable=protected-access + for name, reduce_op in ctx._last_step_outputs_reduce_ops.items(): # pylint: disable=protected-access output = last_step_tensor_outputs_dict[name] - # For outputs that have already been aggregated, take the first value + # For outputs that have already been reduced, take the first value # from the list as each value should be the same. Else return the full # list of values. - # TODO(josh11b): If aggregation is NONE, we should return a PerReplica + # TODO(josh11b): If reduce_op is NONE, we should return a PerReplica # value. - if aggregation is not variables_lib.VariableAggregation.NONE: + if reduce_op is not None: # TODO(priyag): Should this return the element or a list with 1 element last_step_tensor_outputs_dict[name] = output[0] ctx._set_last_step_outputs(last_step_tensor_outputs_dict) # pylint: disable=protected-access @@ -320,10 +372,10 @@ class TPUStrategy(distribute_lib.DistributionStrategy): def _call_for_each_replica(self, fn, args, kwargs): # TODO(jhseu): Consider making it so call_for_each_replica implies that # we're in a tpu.rewrite(), and update TPUMirroredVariable accordingly. - with _TPUReplicaContext(self): + with _TPUReplicaContext(self._container_strategy()): return fn(*args, **kwargs) - def initialize(self): + def _initialize(self): if context.executing_eagerly(): # TODO(priyag): Add appopriate call here when eager is supported for TPUs. raise NotImplementedError("Eager mode not supported in TPUStrategy.") @@ -338,7 +390,7 @@ class TPUStrategy(distribute_lib.DistributionStrategy): tpu.initialize_system()) return graph.get_collection(_TPU_INITIALIZE_SYSTEM_COLLECTION) - def finalize(self): + def _finalize(self): if context.executing_eagerly(): # TODO(priyag): Add appopriate call here when eager is supported for TPUs. raise NotImplementedError("Eager mode not supported in TPUStrategy.") @@ -346,7 +398,7 @@ class TPUStrategy(distribute_lib.DistributionStrategy): return [tpu.shutdown_system()] def _get_devices_from(self, colocate_with=None): - # TODO(jhseu): Change this when we support model parallelism. + # TODO(jhseu): Change this when we support model parallelism. return self._tpu_devices def _create_variable(self, next_creator, *args, **kwargs): @@ -383,12 +435,12 @@ class TPUStrategy(distribute_lib.DistributionStrategy): return _create_tpu_mirrored_variable(devices, _real_mirrored_creator, *args, **kwargs) - def _reduce(self, aggregation, value, destinations): + def _reduce_to(self, reduce_op, value, destinations): if values._enclosing_tpu_context() is not None: # pylint: disable=protected-access - if aggregation == vs.VariableAggregation.MEAN: + if reduce_op == reduce_util.ReduceOp.MEAN: # TODO(jhseu): Revisit once we support model-parallelism. - value *= (1. / self.num_replicas) - elif aggregation != vs.VariableAggregation.SUM: + value *= (1. / self._num_replicas_in_sync) + elif reduce_op != reduce_util.ReduceOp.SUM: raise NotImplementedError( "Currently only support sum & mean in TPUStrategy.") return tpu_ops.cross_replica_sum(value) @@ -396,27 +448,22 @@ class TPUStrategy(distribute_lib.DistributionStrategy): # Validate that the destination is same as the host device # Note we don't do this when in replicate context as the reduction is # performed on the TPU device itself. - devices = cross_tower_ops_lib.get_devices_from(destinations) + devices = cross_device_ops_lib.get_devices_from(destinations) if len(devices) == 1: assert device_util.canonicalize(devices[0]) == device_util.canonicalize( self._host_device) else: raise ValueError("Multiple devices are not supported for TPUStrategy") - if aggregation == vs.VariableAggregation.ONLY_FIRST_REPLICA: - return value[0] output = math_ops.add_n(value) - if aggregation == vs.VariableAggregation.MEAN: + if reduce_op == reduce_util.ReduceOp.MEAN: return output * (1. / len(value)) return output - def _update(self, var, options, fn, *args, **kwargs): + def _update(self, var, fn, args, kwargs, group): assert isinstance(var, values.TPUMirroredVariable) - should_group = options.pop("grouped") - assert not options # Validate that we are processing all of the options. - if values._enclosing_tpu_context() is not None: # pylint: disable=protected-access - if should_group: + if group: return fn(var, *args, **kwargs) else: return [fn(var, *args, **kwargs)] @@ -431,9 +478,7 @@ class TPUStrategy(distribute_lib.DistributionStrategy): updates[d] = fn(v, *values.select_device_mirrored(d, args), **values.select_device_mirrored(d, kwargs)) - return values.update_regroup(self, updates, should_group) - - # TODO(josh11b): Need to implement _update_non_slot()! + return values.update_regroup(self, updates, group) def read_var(self, var): assert isinstance(var, values.TPUMirroredVariable) @@ -453,14 +498,10 @@ class TPUStrategy(distribute_lib.DistributionStrategy): def value_container(self, value): return value - def _broadcast(self, tensor, destinations): + def _broadcast_to(self, tensor, destinations): del destinations return tensor - @property - def num_replicas(self): - return self._num_cores_override or self._tpu_metadata.num_cores - @property def num_hosts(self): return self._tpu_metadata.num_hosts @@ -470,15 +511,15 @@ class TPUStrategy(distribute_lib.DistributionStrategy): return self._tpu_metadata.num_of_cores_per_host @property - def num_replicas_in_sync(self): - return self.num_replicas + def _num_replicas_in_sync(self): + return self._num_cores_override or self._tpu_metadata.num_cores @property - def between_graph(self): + def experimental_between_graph(self): return False @property - def should_init(self): + def experimental_should_init(self): return True @property @@ -500,14 +541,12 @@ class TPUStrategy(distribute_lib.DistributionStrategy): def non_slot_devices(self, var_list): return self._host_device - def _update_non_slot(self, colocate_with, options, fn, *args, **kwargs): + def _update_non_slot(self, colocate_with, fn, args, kwargs, group): del colocate_with - should_group = options.pop("grouped") - assert not options # Validate that we are processing all of the options. with ops.device(self._host_device), distribute_lib.UpdateContext( self._host_device): result = fn(*args, **kwargs) - if should_group: + if group: return result else: return nest.map_structure(self._unwrap, result) @@ -521,17 +560,27 @@ class TPUStrategy(distribute_lib.DistributionStrategy): def get_host_cpu_device(self, host_id): return self.get_host(host_id) + "/device:CPU:0" - def configure(self, - session_config=None, - cluster_spec=None, - task_type=None, - task_id=None): + def _configure(self, + session_config=None, + cluster_spec=None, + task_type=None, + task_id=None): del cluster_spec, task_type, task_id if session_config: - session_config.isolate_session_state = True - cluster_spec = self._tpu_cluster_resolver.cluster_spec() - if cluster_spec: - session_config.cluster_def.CopyFrom(cluster_spec.as_cluster_def()) + session_config.CopyFrom(self._update_config_proto(session_config)) + + def _update_config_proto(self, config_proto): + updated_config = copy.deepcopy(config_proto) + updated_config.isolate_session_state = True + cluster_spec = self._tpu_cluster_resolver.cluster_spec() + if cluster_spec: + updated_config.cluster_def.CopyFrom(cluster_spec.as_cluster_def()) + return updated_config + + # TODO(priyag): Delete this once all strategies use global batch size. + @property + def _global_batch_size(self): + return True class _TPUReplicaContext(distribute_lib.ReplicaContext): @@ -540,13 +589,14 @@ class _TPUReplicaContext(distribute_lib.ReplicaContext): # TODO(sourabhbajaj): Call for each tower should be updating this. def __init__(self, distribution_strategy): distribute_lib.ReplicaContext.__init__( - self, distribution_strategy, replica_id=0) - - @property - def device(self): - raise RuntimeError("Use .devices instead") + self, + distribution_strategy, + # TODO(b/118385803): properly initialize replica_id, instead of always 0 + replica_id_in_sync_group=constant_op.constant(0, dtypes.int32)) @property def devices(self): distribute_lib.require_replica_context(self) - return [self._distribution_strategy.worker_devices[self._replica_id]] + ds = self._distribution_strategy + replica_id = tensor_util.constant_value(self._replica_id_in_sync_group) + return [ds.extended.worker_devices[replica_id]] diff --git a/tensorflow/contrib/distribute/python/values_test.py b/tensorflow/contrib/distribute/python/values_test.py index 268393ee801..538b859f3d1 100644 --- a/tensorflow/contrib/distribute/python/values_test.py +++ b/tensorflow/contrib/distribute/python/values_test.py @@ -19,12 +19,15 @@ from __future__ import division from __future__ import print_function import os +from absl.testing import parameterized -from tensorflow.contrib.distribute.python import mirrored_strategy +from tensorflow.contrib.distribute.python import combinations from tensorflow.contrib.distribute.python import multi_worker_test_base -from tensorflow.contrib.distribute.python import values from tensorflow.core.protobuf import config_pb2 from tensorflow.python.data.ops import dataset_ops +from tensorflow.python.distribute import device_util +from tensorflow.python.distribute import distribute_lib +from tensorflow.python.distribute import values from tensorflow.python.eager import context from tensorflow.python.eager import test from tensorflow.python.estimator import model_fn as model_fn_lib @@ -34,10 +37,10 @@ from tensorflow.python.framework import ops from tensorflow.python.framework import tensor_util from tensorflow.python.framework import test_util from tensorflow.python.ops import array_ops +from tensorflow.python.ops import control_flow_ops from tensorflow.python.ops import random_ops from tensorflow.python.ops import variable_scope from tensorflow.python.ops import variables as variables_lib -from tensorflow.python.training import device_util from tensorflow.python.training import saver as saver_lib from tensorflow.python.util import nest @@ -324,20 +327,20 @@ class RegroupAndSelectDeviceTest(test.TestCase): self.assertTrue( isinstance(merged_estimator_spec, model_fn_lib.EstimatorSpec)) - self.assertEquals(model_fn_lib.ModeKeys.TRAIN, merged_estimator_spec.mode) + self.assertEqual(model_fn_lib.ModeKeys.TRAIN, merged_estimator_spec.mode) for device_id in range(3): d = _device_str(device_id) - self.assertEquals(created_estimator_specs[device_id].loss, - merged_estimator_spec.loss.get(d)) - self.assertEquals(created_estimator_specs[device_id].train_op, - merged_estimator_spec.train_op.get(d)) + self.assertEqual(created_estimator_specs[device_id].loss, + merged_estimator_spec.loss.get(d)) + self.assertEqual(created_estimator_specs[device_id].train_op, + merged_estimator_spec.train_op.get(d)) # Scaffold is populated by `EstimatorSpec.__new__`. - self.assertEquals(created_estimator_specs[device_id].scaffold, - merged_estimator_spec.scaffold.get(d)) + self.assertEqual(created_estimator_specs[device_id].scaffold, + merged_estimator_spec.scaffold.get(d)) # Also test that we can undo the merge using select_device() - self.assertEquals(created_estimator_specs[device_id], - values.select_device(_device_str(device_id), - merged_estimator_spec)) + self.assertEqual(created_estimator_specs[device_id], + values.select_device(_device_str(device_id), + merged_estimator_spec)) class PerReplicaDatasetTest(test.TestCase): @@ -568,7 +571,184 @@ class MultiWorkerDatasetTest(multi_worker_test_base.MultiWorkerTestBase): multi_worker_iterator.get_next() -class MirroredVariableTest(test.TestCase): +class InputIteratorTestBase(test.TestCase): + + def _test_iterator(self, input_type, dataset_fn, worker_device_pairs, + expected_values, sess=None, split_batch_by=None): + devices = nest.flatten([ds for _, ds in worker_device_pairs]) + + if input_type == "input_fn": + input_contexts = [ + distribute_lib.InputContext() for _ in worker_device_pairs] + input_fn = lambda _: dataset_fn() + iterator = values.InputFunctionIterator(input_fn, worker_device_pairs, + input_contexts) + else: + iterator = values.DatasetIterator(dataset_fn(), worker_device_pairs, + split_batch_by) + + evaluate = lambda x: sess.run(x) if sess else self.evaluate(x) + + evaluate(control_flow_ops.group(iterator.initialize())) + + for expected_value in expected_values: + next_element = iterator.get_next() + computed_value = evaluate( + [values.select_device(d, next_element) for d in devices]) + self.assertAllEqual(expected_value, computed_value) + + with self.assertRaises(errors.OutOfRangeError): + next_element = iterator.get_next() + evaluate([values.select_device(d, next_element) for d in devices]) + + # After re-initializing the iterator, should be able to iterate again. + evaluate(control_flow_ops.group(iterator.initialize())) + + for expected_value in expected_values: + next_element = iterator.get_next() + computed_value = evaluate( + [values.select_device(d, next_element) for d in devices]) + self.assertAllEqual(expected_value, computed_value) + + +class InputIteratorSingleWorkerTest(InputIteratorTestBase, + parameterized.TestCase): + + @combinations.generate(combinations.combine( + mode=["graph", "eager"], + input_type=["input_fn", "dataset"])) + def testOneDeviceCPU(self, input_type): + worker_device_pairs = [("", ["/device:CPU:0"])] + dataset_fn = lambda: dataset_ops.Dataset.range(10) + + expected_values = [[i] for i in range(10)] + + self._test_iterator(input_type, dataset_fn, worker_device_pairs, + expected_values) + + @combinations.generate(combinations.combine( + mode=["graph", "eager"], + input_type=["input_fn", "dataset"], + required_gpus=1)) + def testTwoDevicesOneGPUOneCPU(self, input_type): + worker_device_pairs = [("", ["/device:GPU:0", "/device:CPU:0"])] + dataset_fn = lambda: dataset_ops.Dataset.range(10) + + expected_values = [[i, i+1] for i in range(0, 10, 2)] + + self._test_iterator(input_type, dataset_fn, worker_device_pairs, + expected_values) + + @combinations.generate(combinations.combine( + mode=["graph", "eager"], + input_type=["input_fn", "dataset"], + required_gpus=1)) + def testTupleDataset(self, input_type): + worker_device_pairs = [("", ["/device:GPU:0", "/device:CPU:0"])] + def dataset_fn(): + dataset1 = dataset_ops.Dataset.range(10) + dataset2 = dataset_ops.Dataset.range(10).map(lambda x: x**2) + return dataset_ops.Dataset.zip((dataset1, dataset2)) + + expected_values = [[(i, i**2), (i+1, (i+1)**2)] for i in range(0, 10, 2)] + + self._test_iterator(input_type, dataset_fn, worker_device_pairs, + expected_values) + + @combinations.generate(combinations.combine( + mode=["graph", "eager"], + input_type=["input_fn", "dataset"], + required_gpus=1)) + def testUnevenDatasetBatches(self, input_type): + worker_device_pairs = [("", ["/device:GPU:0", "/device:CPU:0"])] + dataset_fn = lambda: dataset_ops.Dataset.range(11) + + expected_values = [[i, i+1] for i in range(0, 10, 2)] + self._test_iterator(input_type, dataset_fn, worker_device_pairs, + expected_values) + + @combinations.generate(combinations.combine( + mode=["graph", "eager"], + input_type=["dataset"], + split_batch_by=[None, 2], + required_gpus=1)) + def testBatchSplitting(self, input_type, split_batch_by): + worker_device_pairs = [("", ["/device:GPU:0", "/device:CPU:0"])] + batch_size = 10 + dataset_fn = lambda: dataset_ops.Dataset.range(100).batch(batch_size) + + updated_batch_size = ( + batch_size // split_batch_by if split_batch_by else batch_size) + expected_values = [[range(i, i+updated_batch_size), + range(i+updated_batch_size, i+2*updated_batch_size)] + for i in range(0, 100, updated_batch_size*2)] + + self._test_iterator(input_type, dataset_fn, worker_device_pairs, + expected_values, sess=None, + split_batch_by=split_batch_by) + + +class InputIteratorMultiWorkerTest( + multi_worker_test_base.MultiWorkerTestBase, InputIteratorTestBase, + parameterized.TestCase): + + def _cpu_devices(self): + return [ + ("/job:worker/replica:0/task:0", + ["/job:worker/replica:0/task:0/device:CPU:0"]), + ("/job:worker/replica:0/task:1", + ["/job:worker/replica:0/task:1/device:CPU:0"])] + + def _cpu_and_one_gpu_devices(self): + return [ + ("/job:worker/replica:0/task:0", [ + "/job:worker/replica:0/task:0/device:GPU:0", + "/job:worker/replica:0/task:0/device:CPU:0" + ]), + ("/job:worker/replica:0/task:1", [ + "/job:worker/replica:0/task:1/device:GPU:0", + "/job:worker/replica:0/task:1/device:CPU:0" + ]) + ] + + @combinations.generate(combinations.combine( + mode=["graph"], + input_type=["input_fn", "dataset"])) + def testOneDevicePerWorker(self, input_type): + worker_devices = self._cpu_devices() + with context.graph_mode(), self.cached_session() as sess: + dataset_fn = lambda: dataset_ops.Dataset.range(4) + self._test_iterator(input_type, dataset_fn, worker_devices, + [[0, 0], [1, 1], [2, 2], [3, 3]], sess) + + @combinations.generate(combinations.combine( + mode=["graph"], + input_type=["input_fn", "dataset"], + required_gpus=1)) + def testTwoDevicesPerWorker(self, input_type): + worker_devices = self._cpu_and_one_gpu_devices() + with context.graph_mode(), self.cached_session() as sess: + dataset_fn = lambda: dataset_ops.Dataset.range(4) + self._test_iterator(input_type, dataset_fn, worker_devices, + [[0, 1, 0, 1], [2, 3, 2, 3]], sess) + + @combinations.generate(combinations.combine( + mode=["graph"], + input_type=["input_fn", "dataset"])) + def testTupleDataset(self, input_type): + worker_devices = self._cpu_devices() + with context.graph_mode(), self.cached_session() as sess: + def dataset_fn(): + dataset1 = dataset_ops.Dataset.range(4) + dataset2 = dataset_ops.Dataset.range(4).map(lambda x: x**2) + return dataset_ops.Dataset.zip((dataset1, dataset2)) + + expected_values = [[(i, i**2), (i, i**2)] for i in range(0, 4)] + self._test_iterator(input_type, dataset_fn, worker_devices, + expected_values, sess) + + +class MirroredVariableTest(test.TestCase, parameterized.TestCase): config = config_pb2.ConfigProto() config.allow_soft_placement = True @@ -580,9 +760,9 @@ class MirroredVariableTest(test.TestCase): v, _, mirrored = _make_mirrored() - self.assertEquals(v[0].name, mirrored.name) - self.assertEquals(v[0].dtype, mirrored.dtype) - self.assertEquals(v[0].shape, mirrored.shape) + self.assertEqual(v[0].name, mirrored.name) + self.assertEqual(v[0].dtype, mirrored.dtype) + self.assertEqual(v[0].shape, mirrored.shape) @test_util.run_in_graph_and_eager_modes(config=config) def testVariableOnAnotherDevice(self): @@ -592,9 +772,9 @@ class MirroredVariableTest(test.TestCase): mirrored = values.MirroredVariable(index, v, variable_scope.VariableAggregation.MEAN) - self.assertEquals(v.name, mirrored.name) - self.assertEquals(v.dtype, mirrored.dtype) - self.assertEquals(v.shape, mirrored.shape) + self.assertEqual(v.name, mirrored.name) + self.assertEqual(v.dtype, mirrored.dtype) + self.assertEqual(v.shape, mirrored.shape) def _assign_mirrored(self, devices, v, new): for d, var, n in zip(devices, v, new): @@ -714,14 +894,13 @@ class MirroredVariableTest(test.TestCase): save_path = self._save_normal() self._restore_mirrored(save_path) - @test_util.run_in_graph_and_eager_modes(config=config) - def testFetchAMirroredVariable(self): - if context.num_gpus() < 1 or context.executing_eagerly(): - self.skipTest("A GPU is not available for this test or it's eager mode.") - - with self.session( - graph=ops.Graph()) as sess, mirrored_strategy.MirroredStrategy( - ["/device:GPU:0"]).scope(): + @combinations.generate(combinations.combine( + distribution=[ + combinations.mirrored_strategy_with_one_gpu, + combinations.core_mirrored_strategy_with_one_gpu], + mode=["graph"])) + def testFetchAMirroredVariable(self, distribution): + with self.session(graph=ops.Graph()) as sess, distribution.scope(): with ops.device("/device:GPU:0"): v = variable_scope.get_variable( name="v", initializer=1., use_resource=True) @@ -747,7 +926,7 @@ def _make_replica_local(method): return v, replica_local -class ReplicaLocalVariableTest(test.TestCase): +class ReplicaLocalVariablePropertiesTest(test.TestCase): config = config_pb2.ConfigProto() config.allow_soft_placement = True @@ -756,15 +935,14 @@ class ReplicaLocalVariableTest(test.TestCase): def testProperties(self): if context.num_gpus() < 1 and context.executing_eagerly(): self.skipTest("A GPU is not available for this test in eager mode.") - v, replica_local = _make_replica_local( variable_scope.VariableAggregation.SUM) - self.assertEquals(v[0].name, replica_local.name) - self.assertEquals(v[0].dtype, replica_local.dtype) - self.assertEquals(v[0].shape, replica_local.shape) - self.assertEquals(variable_scope.VariableAggregation.SUM, - replica_local.aggregation) + self.assertEqual(v[0].name, replica_local.name) + self.assertEqual(v[0].dtype, replica_local.dtype) + self.assertEqual(v[0].shape, replica_local.shape) + self.assertEqual(variable_scope.VariableAggregation.SUM, + replica_local.aggregation) @test_util.run_in_graph_and_eager_modes(config=config) def testVariableOnAnotherDevice(self): @@ -774,11 +952,32 @@ class ReplicaLocalVariableTest(test.TestCase): replica_local = values.ReplicaLocalVariable( index, v, variable_scope.VariableAggregation.MEAN) - self.assertEquals(v.name, replica_local.name) - self.assertEquals(v.dtype, replica_local.dtype) - self.assertEquals(v.shape, replica_local.shape) - self.assertEquals(variable_scope.VariableAggregation.MEAN, - replica_local.aggregation) + self.assertEqual(v.name, replica_local.name) + self.assertEqual(v.dtype, replica_local.dtype) + self.assertEqual(v.shape, replica_local.shape) + self.assertEqual(variable_scope.VariableAggregation.MEAN, + replica_local.aggregation) + + def testTensorConversion(self): + with context.graph_mode(): + _, replica_local = _make_replica_local( + variable_scope.VariableAggregation.SUM) + converted = ops.internal_convert_to_tensor(replica_local, as_ref=False) + self.assertIsInstance(converted, ops.Tensor) + self.assertEqual(converted.dtype, replica_local.dtype) + + converted = ops.internal_convert_to_tensor(replica_local, as_ref=True) + # Resources variable are converted to tensors as well when as_ref is True. + self.assertIsInstance(converted, ops.Tensor) + self.assertEqual(converted.dtype, replica_local.dtype) + + +@combinations.generate(combinations.combine( + distribution=[ + combinations.mirrored_strategy_with_gpu_and_cpu, + combinations.core_mirrored_strategy_with_gpu_and_cpu], + mode=["graph", "eager"])) +class ReplicaLocalVariableTest(test.TestCase, parameterized.TestCase): def _assign_replica_local(self, devices, v, new): for d, var, n in zip(devices, v, new): @@ -795,22 +994,15 @@ class ReplicaLocalVariableTest(test.TestCase): save_path, _ = self._save_return_saver(sess, var) return save_path - def _dist_scope(self): - return mirrored_strategy.MirroredStrategy(_devices).scope() - - @test_util.run_in_graph_and_eager_modes(config=config) - def testSaveAndRestoreReplicaLocalSumOneGraph(self): - if context.num_gpus() < 1 and context.executing_eagerly(): - self.skipTest("A GPU is not available for this test in eager mode.") - - with self.cached_session(config=self.config) as sess: + def testSaveAndRestoreReplicaLocalSumOneGraph(self, distribution): + with self.cached_session() as sess: v, replica_local = _make_replica_local( variable_scope.VariableAggregation.SUM) # Overwrite the initial values. self._assign_replica_local(_devices, v, [3., 4.]) - with self._dist_scope(): + with distribution.scope(): # Saves the current value of v[0] + v[1], 7. save_path, saver = self._save_return_saver(sess, replica_local) @@ -822,19 +1014,18 @@ class ReplicaLocalVariableTest(test.TestCase): saver.restore(sess, save_path) self.assertEqual([3.5, 3.5], self.evaluate([v[0], v[1]])) - @test_util.run_in_graph_and_eager_modes(config=config) - def testSaveAndRestoreReplicaLocalMeanOneGraph(self): + def testSaveAndRestoreReplicaLocalMeanOneGraph(self, distribution): if context.num_gpus() < 1 and context.executing_eagerly(): self.skipTest("A GPU is not available for this test in eager mode.") - with self.cached_session(config=self.config) as sess: + with self.cached_session() as sess: v, replica_local = _make_replica_local( variable_scope.VariableAggregation.MEAN) # Overwrite the initial values. self._assign_replica_local(_devices, v, [3., 4.]) - with self._dist_scope(): + with distribution.scope(): # Saves the current value of (v[0] + v[1])/2, 3.5. save_path, saver = self._save_return_saver(sess, replica_local) @@ -845,7 +1036,7 @@ class ReplicaLocalVariableTest(test.TestCase): saver.restore(sess, save_path) self.assertEqual([3.5, 3.5], self.evaluate([v[0], v[1]])) - def _save_replica_local_mean(self): + def _save_replica_local_mean(self, distribution): """Save variables with mirroring, returns save_path.""" with self.session(graph=ops.Graph()) as sess: v, replica_local = _make_replica_local( @@ -854,7 +1045,7 @@ class ReplicaLocalVariableTest(test.TestCase): # Overwrite the initial values. self._assign_replica_local(_devices, v, [3., 4.]) - with self._dist_scope(): + with distribution.scope(): # Saves the current value of (v[0] + v[1])/2, 3.5 save_path = self._save(sess, replica_local) @@ -862,7 +1053,7 @@ class ReplicaLocalVariableTest(test.TestCase): self._assign_replica_local(_devices, v, [5., 6.]) return save_path - def _save_replica_local_sum(self): + def _save_replica_local_sum(self, distribution): """Save variables with mirroring, returns save_path.""" with self.session(graph=ops.Graph()) as sess: v, replica_local = _make_replica_local("sum") @@ -870,7 +1061,7 @@ class ReplicaLocalVariableTest(test.TestCase): # Overwrite the initial values. self._assign_replica_local(_devices, v, [1.5, 2.]) - with self._dist_scope(): + with distribution.scope(): # Saves the current value of v[0] + v[1], 3.5 save_path = self._save(sess, replica_local) @@ -908,7 +1099,7 @@ class ReplicaLocalVariableTest(test.TestCase): saver.restore(sess, save_path) self.assertEqual(3.5, self.evaluate(var)) - def _restore_replica_local_mean(self, save_path): + def _restore_replica_local_mean(self, save_path, distribution): """Restore to variables with mirroring in a fresh graph.""" with self.session(graph=ops.Graph()) as sess: v, replica_local = _make_replica_local( @@ -917,13 +1108,13 @@ class ReplicaLocalVariableTest(test.TestCase): # Overwrite the initial values. self._assign_replica_local(_devices, v, [7., 8.]) - with self._dist_scope(): + with distribution.scope(): # Restores the saved value of 3.5 to both variables. saver = saver_lib.Saver(var_list=[replica_local]) saver.restore(sess, save_path) self.assertEqual([3.5, 3.5], self.evaluate([v[0], v[1]])) - def _restore_replica_local_sum(self, save_path): + def _restore_replica_local_sum(self, save_path, distribution): """Restore to variables with mirroring in a fresh graph.""" with self.session(graph=ops.Graph()) as sess: v, replica_local = _make_replica_local( @@ -932,72 +1123,35 @@ class ReplicaLocalVariableTest(test.TestCase): # Overwrite the initial values. self._assign_replica_local(_devices, v, [7., 8.]) - with self._dist_scope(): + with distribution.scope(): # Restores the saved value of 3.5 to both variables. saver = saver_lib.Saver(var_list=[replica_local]) saver.restore(sess, save_path) self.assertEqual([1.75, 1.75], self.evaluate([v[0], v[1]])) - @test_util.run_in_graph_and_eager_modes(config=config) - def testSaveReplicaLocalRestoreReplicaLocalMean(self): - if context.num_gpus() < 1 and context.executing_eagerly(): - self.skipTest("A GPU is not available for this test in eager mode.") + def testSaveReplicaLocalRestoreReplicaLocalMean(self, distribution): + save_path = self._save_replica_local_mean(distribution) + self._restore_replica_local_mean(save_path, distribution) - save_path = self._save_replica_local_mean() - self._restore_replica_local_mean(save_path) + def testSaveReplicaLocalRestoreReplicaLocalSum(self, distribution): + save_path = self._save_replica_local_sum(distribution) + self._restore_replica_local_sum(save_path, distribution) - @test_util.run_in_graph_and_eager_modes(config=config) - def testSaveReplicaLocalRestoreReplicaLocalSum(self): - if context.num_gpus() < 1 and context.executing_eagerly(): - self.skipTest("A GPU is not available for this test in eager mode.") - - save_path = self._save_replica_local_sum() - self._restore_replica_local_sum(save_path) - - @test_util.run_in_graph_and_eager_modes(config=config) - def testSaveReplicaLocalMeanRestoreNormal(self): - if context.num_gpus() < 1 and context.executing_eagerly(): - self.skipTest("A GPU is not available for this test in eager mode.") - - save_path = self._save_replica_local_mean() + def testSaveReplicaLocalMeanRestoreNormal(self, distribution): + save_path = self._save_replica_local_mean(distribution) self._restore_normal(save_path) - @test_util.run_in_graph_and_eager_modes(config=config) - def testSaveReplicaLocalSumRestoreNormal(self): - if context.num_gpus() < 1 and context.executing_eagerly(): - self.skipTest("A GPU is not available for this test in eager mode.") - - save_path = self._save_replica_local_sum() + def testSaveReplicaLocalSumRestoreNormal(self, distribution): + save_path = self._save_replica_local_sum(distribution) self._restore_normal(save_path) - @test_util.run_in_graph_and_eager_modes(config=config) - def testSaveNormalRestoreReplicaLocalMean(self): - if context.num_gpus() < 1 and context.executing_eagerly(): - self.skipTest("A GPU is not available for this test in eager mode.") - + def testSaveNormalRestoreReplicaLocalMean(self, distribution): save_path = self._save_normal() - self._restore_replica_local_mean(save_path) - - @test_util.run_in_graph_and_eager_modes(config=config) - def testSaveNormalRestoreReplicaLocalSum(self): - if context.num_gpus() < 1 and context.executing_eagerly(): - self.skipTest("A GPU is not available for this test in eager mode.") + self._restore_replica_local_mean(save_path, distribution) + def testSaveNormalRestoreReplicaLocalSum(self, distribution): save_path = self._save_normal() - self._restore_replica_local_sum(save_path) - - def testTensorConversion(self): - with context.graph_mode(): - _, replica_local = _make_replica_local( - variable_scope.VariableAggregation.SUM) - converted = ops.internal_convert_to_tensor(replica_local, as_ref=False) - self.assertIsInstance(converted, ops.Tensor) - self.assertEqual(converted.dtype, replica_local.dtype) - - converted = ops.internal_convert_to_tensor(replica_local, as_ref=True) - # Resources variable are converted to tensors as well when as_ref is True. - self.assertIsInstance(converted, ops.Tensor) - self.assertEqual(converted.dtype, replica_local.dtype) + self._restore_replica_local_sum(save_path, distribution) if __name__ == "__main__": diff --git a/tensorflow/contrib/distribute/python/warm_starting_util_test.py b/tensorflow/contrib/distribute/python/warm_starting_util_test.py index 5d57d144c1c..b0bcf9b1745 100644 --- a/tensorflow/contrib/distribute/python/warm_starting_util_test.py +++ b/tensorflow/contrib/distribute/python/warm_starting_util_test.py @@ -44,7 +44,9 @@ class WarmStartingUtilWithDistributionStrategyTest( distribution=[combinations.default_strategy, combinations.one_device_strategy, combinations.mirrored_strategy_with_gpu_and_cpu, - combinations.mirrored_strategy_with_two_gpus], + combinations.mirrored_strategy_with_two_gpus, + combinations.core_mirrored_strategy_with_gpu_and_cpu, + combinations.core_mirrored_strategy_with_two_gpus], save_with_distribution=[True, False], restore_with_distribution=[True, False], mode=["graph"])) diff --git a/tensorflow/contrib/distributions/BUILD b/tensorflow/contrib/distributions/BUILD index 60f6b90edcb..3079175015a 100644 --- a/tensorflow/contrib/distributions/BUILD +++ b/tensorflow/contrib/distributions/BUILD @@ -72,7 +72,6 @@ py_library( "//tensorflow/python:nn", "//tensorflow/python:nn_ops", "//tensorflow/python:random_ops", - "//tensorflow/python:spectral_ops", "//tensorflow/python:state_ops", "//tensorflow/python:tensor_util", "//tensorflow/python:util", @@ -80,6 +79,7 @@ py_library( "//tensorflow/python:variables", "//tensorflow/python/ops/distributions", "//tensorflow/python/ops/linalg", + "//tensorflow/python/ops/signal", "//third_party/py/numpy", "@six_archive//:six", ], diff --git a/tensorflow/contrib/distributions/python/kernel_tests/normal_conjugate_posteriors_test.py b/tensorflow/contrib/distributions/python/kernel_tests/normal_conjugate_posteriors_test.py index 29eeaf43c51..ab3c07172a6 100644 --- a/tensorflow/contrib/distributions/python/kernel_tests/normal_conjugate_posteriors_test.py +++ b/tensorflow/contrib/distributions/python/kernel_tests/normal_conjugate_posteriors_test.py @@ -82,7 +82,7 @@ class NormalTest(test.TestCase): x = constant_op.constant( [[-2.5, 2.5, 4.0, 0.0, -1.0, 2.0], [2.5, -2.5, -4.0, 0.0, 1.0, -2.0]], dtype=dtypes.float32) - s = math_ops.reduce_sum(x, reduction_indices=[1]) + s = math_ops.reduce_sum(x, axis=[1]) x = array_ops.transpose(x) # Reshape to shape (6, 2) n = constant_op.constant([6] * 2) prior = distributions.Normal(loc=mu0, scale=sigma0) diff --git a/tensorflow/contrib/distributions/python/kernel_tests/wishart_test.py b/tensorflow/contrib/distributions/python/kernel_tests/wishart_test.py index a60056c444a..cdee30bbc42 100644 --- a/tensorflow/contrib/distributions/python/kernel_tests/wishart_test.py +++ b/tensorflow/contrib/distributions/python/kernel_tests/wishart_test.py @@ -147,14 +147,13 @@ class WishartCholeskyTest(test.TestCase): x = chol_w.sample(10000, seed=42) self.assertAllEqual((10000, 3, 3), x.get_shape()) - moment1_estimate = math_ops.reduce_mean(x, reduction_indices=[0]).eval() + moment1_estimate = math_ops.reduce_mean(x, axis=[0]).eval() self.assertAllClose(chol_w.mean().eval(), moment1_estimate, rtol=0.05) # The Variance estimate uses the squares rather than outer-products # because Wishart.Variance is the diagonal of the Wishart covariance # matrix. - variance_estimate = (math_ops.reduce_mean( - math_ops.square(x), reduction_indices=[0]) - + variance_estimate = (math_ops.reduce_mean(math_ops.square(x), axis=[0]) - math_ops.square(moment1_estimate)).eval() self.assertAllClose( chol_w.variance().eval(), variance_estimate, rtol=0.05) diff --git a/tensorflow/contrib/distributions/python/ops/bijectors/softmax_centered.py b/tensorflow/contrib/distributions/python/ops/bijectors/softmax_centered.py index 15c241d5d7a..74765f19e58 100644 --- a/tensorflow/contrib/distributions/python/ops/bijectors/softmax_centered.py +++ b/tensorflow/contrib/distributions/python/ops/bijectors/softmax_centered.py @@ -168,7 +168,7 @@ class SoftmaxCentered(bijector.Bijector): # log_normalization = 1 + reduce_sum(exp(logits)) # -log_normalization + reduce_sum(logits - log_normalization) log_normalization = nn_ops.softplus( - math_ops.reduce_logsumexp(x, axis=-1, keep_dims=True)) + math_ops.reduce_logsumexp(x, axis=-1, keepdims=True)) return array_ops.squeeze( (-log_normalization + math_ops.reduce_sum( x - log_normalization, axis=-1, keepdims=True)), axis=-1) diff --git a/tensorflow/contrib/distributions/python/ops/sample_stats.py b/tensorflow/contrib/distributions/python/ops/sample_stats.py index aa680a92be6..978e627d663 100644 --- a/tensorflow/contrib/distributions/python/ops/sample_stats.py +++ b/tensorflow/contrib/distributions/python/ops/sample_stats.py @@ -29,8 +29,8 @@ from tensorflow.python.ops import clip_ops from tensorflow.python.ops import control_flow_ops from tensorflow.python.ops import math_ops from tensorflow.python.ops import nn_ops -from tensorflow.python.ops import spectral_ops from tensorflow.python.ops.distributions import util +from tensorflow.python.ops.signal import fft_ops __all__ = [ "auto_correlation", @@ -157,11 +157,11 @@ def auto_correlation( dtype.real_dtype.as_numpy_dtype(0.)) # Autocorrelation is IFFT of power-spectral density (up to some scaling). - fft_x_rotated_pad = spectral_ops.fft(x_rotated_pad) + fft_x_rotated_pad = fft_ops.fft(x_rotated_pad) spectral_density = fft_x_rotated_pad * math_ops.conj(fft_x_rotated_pad) # shifted_product is R[m] from above detailed explanation. # It is the inner product sum_n X[n] * Conj(X[n - m]). - shifted_product = spectral_ops.ifft(spectral_density) + shifted_product = fft_ops.ifft(spectral_density) # Cast back to real-valued if x was real to begin with. shifted_product = math_ops.cast(shifted_product, dtype) diff --git a/tensorflow/contrib/eager/python/datasets.py b/tensorflow/contrib/eager/python/datasets.py index 3aed121233b..34614b86a75 100644 --- a/tensorflow/contrib/eager/python/datasets.py +++ b/tensorflow/contrib/eager/python/datasets.py @@ -52,12 +52,6 @@ class Iterator(iterator_ops.EagerIterator): TypeError: If `dataset` is an unsupported type. RuntimeError: When invoked without eager execution enabled. """ - if isinstance(dataset, prefetching_ops._PrefetchToDeviceDataset): # pylint: disable=protected-access - raise TypeError( - "`tf.data.experimental.prefetch_to_device()` is not compatible with " - "`tf.contrib.eager.Iterator`. Use `for ... in dataset:` to iterate " - "over the dataset instead.") - if not context.context().device_spec.device_type: is_remote_device = False else: diff --git a/tensorflow/contrib/eager/python/datasets_test.py b/tensorflow/contrib/eager/python/datasets_test.py index 6a508fc6ba9..257d02057ae 100644 --- a/tensorflow/contrib/eager/python/datasets_test.py +++ b/tensorflow/contrib/eager/python/datasets_test.py @@ -26,7 +26,6 @@ import numpy as np from tensorflow.contrib import lookup from tensorflow.contrib.eager.python import datasets from tensorflow.python.data import Dataset -from tensorflow.python.data.experimental.ops import prefetching_ops from tensorflow.python.data.experimental.ops import threadpool from tensorflow.python.data.experimental.ops import unique from tensorflow.python.eager import test @@ -208,18 +207,6 @@ class IteratorTest(test.TestCase): y = math_ops.add(x, x) self.assertAllEqual([0., 2.], y.numpy()) - def testTensorsExplicitPrefetchToDevice(self): - ds = Dataset.from_tensor_slices([0., 1.]) - ds = ds.apply(prefetching_ops.prefetch_to_device(test.gpu_device_name())) - - with self.assertRaisesRegexp(TypeError, 'prefetch_to_device'): - datasets.Iterator(ds) - - for i, x in enumerate(ds): - with ops.device(test.gpu_device_name()): - x = math_ops.add(x, x) - self.assertEqual(float(i) + float(i), x.numpy()) - def testOverrideThreadPool(self): def get_thread_id(_): diff --git a/tensorflow/contrib/eager/python/evaluator.py b/tensorflow/contrib/eager/python/evaluator.py index 7949a3f6da2..51443d24829 100644 --- a/tensorflow/contrib/eager/python/evaluator.py +++ b/tensorflow/contrib/eager/python/evaluator.py @@ -22,6 +22,7 @@ import six from tensorflow.contrib.eager.python import datasets from tensorflow.contrib.eager.python import metrics +from tensorflow.python.data.ops import dataset_ops from tensorflow.python.eager import context from tensorflow.python.eager import function from tensorflow.python.framework import errors_impl @@ -164,8 +165,8 @@ class Evaluator(object): self.__call__(example, *args, **kwargs) return self.all_metric_results(summary_logdir) # Graph construction - call_op = self.__call__(dataset.make_one_shot_iterator().get_next(), *args, - **kwargs) + call_op = self.__call__( + dataset_ops.make_one_shot_iterator(dataset).get_next(), *args, **kwargs) init_op = self.init_variables() results_op = self.all_metric_results(summary_logdir) return (init_op, call_op, results_op) diff --git a/tensorflow/contrib/eager/python/examples/densenet/BUILD b/tensorflow/contrib/eager/python/examples/densenet/BUILD index 2dc196f550a..e2154fcc5fc 100644 --- a/tensorflow/contrib/eager/python/examples/densenet/BUILD +++ b/tensorflow/contrib/eager/python/examples/densenet/BUILD @@ -3,6 +3,7 @@ licenses(["notice"]) # Apache 2.0 package(default_visibility = ["//tensorflow:internal"]) load("//tensorflow:tensorflow.bzl", "cuda_py_test") +load("//tensorflow:tensorflow.bzl", "py_binary") py_binary( name = "densenet", diff --git a/tensorflow/contrib/eager/python/examples/densenet/densenet_graph_test.py b/tensorflow/contrib/eager/python/examples/densenet/densenet_graph_test.py index 4b3cb624bc9..24f6b007b52 100644 --- a/tensorflow/contrib/eager/python/examples/densenet/densenet_graph_test.py +++ b/tensorflow/contrib/eager/python/examples/densenet/densenet_graph_test.py @@ -119,7 +119,8 @@ class DensenetBenchmark(tf.test.Benchmark): with tf.Graph().as_default(): np_images, np_labels = random_batch(batch_size) dataset = tf.data.Dataset.from_tensors((np_images, np_labels)).repeat() - (images, labels) = dataset.make_one_shot_iterator().get_next() + (images, labels) = tf.compat.v1.data.make_one_shot_iterator( + dataset).get_next() model = densenet.DenseNet(self.depth, self.growth_rate, self.num_blocks, self.output_classes, diff --git a/tensorflow/contrib/eager/python/examples/gan/mnist_graph_test.py b/tensorflow/contrib/eager/python/examples/gan/mnist_graph_test.py index 12b39b0cde4..e73841fbf72 100644 --- a/tensorflow/contrib/eager/python/examples/gan/mnist_graph_test.py +++ b/tensorflow/contrib/eager/python/examples/gan/mnist_graph_test.py @@ -42,7 +42,8 @@ class MnistGraphGanBenchmark(tf.test.Benchmark): # Generate some random data. images_data = np.random.randn(batch_size, 784).astype(np.float32) dataset = tf.data.Dataset.from_tensors(images_data) - images = dataset.repeat().make_one_shot_iterator().get_next() + images = tf.compat.v1.data.make_one_shot_iterator( + dataset.repeat()).get_next() # Create the models and optimizers generator = mnist.Generator(data_format()) diff --git a/tensorflow/contrib/eager/python/examples/generative_examples/cvae.ipynb b/tensorflow/contrib/eager/python/examples/generative_examples/cvae.ipynb index ca27a85a229..1a08cc0fd06 100644 --- a/tensorflow/contrib/eager/python/examples/generative_examples/cvae.ipynb +++ b/tensorflow/contrib/eager/python/examples/generative_examples/cvae.ipynb @@ -470,7 +470,7 @@ "\n", " if epoch % 1 == 0:\n", " loss = tfe.metrics.Mean()\n", - " for test_x in test_dataset.make_one_shot_iterator():\n", + " for test_x in test_dataset:\n", " loss(compute_loss(model, test_x))\n", " elbo = -loss.result()\n", " display.clear_output(wait=False)\n", diff --git a/tensorflow/contrib/eager/python/examples/generative_examples/image_captioning_with_attention.ipynb b/tensorflow/contrib/eager/python/examples/generative_examples/image_captioning_with_attention.ipynb index 3acecd283cd..12c5eff2b4a 100644 --- a/tensorflow/contrib/eager/python/examples/generative_examples/image_captioning_with_attention.ipynb +++ b/tensorflow/contrib/eager/python/examples/generative_examples/image_captioning_with_attention.ipynb @@ -1,1184 +1,1174 @@ { - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "colab": { - "name": "image_captioning_with_attention.ipynb", - "version": "0.3.2", - "views": {}, - "default_view": {}, - "provenance": [ - { - "file_id": "1HI8OK2sMjcx9CTWVn0122QAHOuXaOaMg", - "timestamp": 1530222436922 - } - ], - "private_outputs": true, - "collapsed_sections": [], - "toc_visible": true - }, - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "accelerator": "GPU" + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "K2s1A9eLRPEj" + }, + "source": [ + "##### Copyright 2018 The TensorFlow Authors.\n", + "\n", + "Licensed under the Apache License, Version 2.0 (the \"License\").\n" + ] }, - "cells": [ - { - "metadata": { - "id": "K2s1A9eLRPEj", - "colab_type": "text" - }, - "cell_type": "markdown", - "source": [ - "##### Copyright 2018 The TensorFlow Authors.\n", - "\n", - "Licensed under the Apache License, Version 2.0 (the \"License\").\n" - ] + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "Cffg2i257iMS" + }, + "source": [ + "# Image Captioning with Attention\n", + "\n", + "
\n", + "\n", + " Run in Google Colab \n", + "\n", + "View source on GitHub
" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "QASbY_HGo4Lq" + }, + "source": [ + "Image captioning is the task of generating a caption for an image. Given an image like this:\n", + "\n", + "![Man Surfing](https://tensorflow.org/images/surf.jpg) \n", + "\n", + "[Image Source](https://commons.wikimedia.org/wiki/Surfing#/media/File:Surfing_in_Hawaii.jpg), License: Public Domain\n", + "\n", + "Our goal is to generate a caption, such as \"a surfer riding on a wave\". Here, we'll use an attention-based model. This enables us to see which parts of the image the model focuses on as it generates a caption.\n", + "\n", + "![Prediction](https://tensorflow.org/images/imcap_prediction.png)\n", + "\n", + "This model architecture below is similar to [Show, Attend and Tell: Neural Image Caption Generation with Visual Attention](https://arxiv.org/abs/1502.03044). \n", + "\n", + "The code uses [tf.keras](https://www.tensorflow.org/programmers_guide/keras) and [eager execution](https://www.tensorflow.org/programmers_guide/eager), which you can learn more about in the linked guides.\n", + "\n", + "This notebook is an end-to-end example. If you run it, it will download the [MS-COCO](http://cocodataset.org/#home) dataset, preprocess and cache a subset of the images using Inception V3, train an encoder-decoder model, and use it to generate captions on new images.\n", + "\n", + "The code requires TensorFlow version >=1.9. If you're running this in [Colab]()\n", + "\n", + "In this example, we're training on a relatively small amount of data as an example. On a single P100 GPU, this example will take about ~2 hours to train. We train on the first 30,000 captions (corresponding to about ~20,000 images depending on shuffling, as there are multiple captions per image in the dataset)\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + } }, - { - "metadata": { - "id": "Cffg2i257iMS", - "colab_type": "text" - }, - "cell_type": "markdown", - "source": [ - "# Image Captioning with Attention\n", - "\n", - "
\n", - "\n", - " Run in Google Colab \n", - "\n", - "View source on GitHub
" - ] + "colab_type": "code", + "id": "U8l4RJ0XRPEm" + }, + "outputs": [], + "source": [ + "# Import TensorFlow and enable eager execution\n", + "# This code requires TensorFlow version >=1.9\n", + "import tensorflow as tf\n", + "tf.enable_eager_execution()\n", + "\n", + "# We'll generate plots of attention in order to see which parts of an image\n", + "# our model focuses on during captioning\n", + "import matplotlib.pyplot as plt\n", + "\n", + "# Scikit-learn includes many helpful utilities\n", + "from sklearn.model_selection import train_test_split\n", + "from sklearn.utils import shuffle\n", + "\n", + "import re\n", + "import numpy as np\n", + "import os\n", + "import time\n", + "import json\n", + "from glob import glob\n", + "from PIL import Image\n", + "import pickle" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "b6qbGw8MRPE5" + }, + "source": [ + "## Download and prepare the MS-COCO dataset\n", + "\n", + "We will use the [MS-COCO dataset](http://cocodataset.org/#home) to train our model. This dataset contains >82,000 images, each of which has been annotated with at least 5 different captions. The code below will download and extract the dataset automatically. \n", + "\n", + "**Caution: large download ahead**. We'll use the training set, it's a 13GB file." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + } }, - { - "metadata": { - "id": "QASbY_HGo4Lq", - "colab_type": "text" - }, - "cell_type": "markdown", - "source": [ - "Image captioning is the task of generating a caption for an image. Given an image like this:\n", - "\n", - "![Man Surfing](https://tensorflow.org/images/surf.jpg) \n", - "\n", - "[Image Source](https://commons.wikimedia.org/wiki/Surfing#/media/File:Surfing_in_Hawaii.jpg), License: Public Domain\n", - "\n", - "Our goal is to generate a caption, such as \"a surfer riding on a wave\". Here, we'll use an attention-based model. This enables us to see which parts of the image the model focuses on as it generates a caption.\n", - "\n", - "![Prediction](https://tensorflow.org/images/imcap_prediction.png)\n", - "\n", - "This model architecture below is similar to [Show, Attend and Tell: Neural Image Caption Generation with Visual Attention](https://arxiv.org/abs/1502.03044). \n", - "\n", - "The code uses [tf.keras](https://www.tensorflow.org/programmers_guide/keras) and [eager execution](https://www.tensorflow.org/programmers_guide/eager), which you can learn more about in the linked guides.\n", - "\n", - "This notebook is an end-to-end example. If you run it, it will download the [MS-COCO](http://cocodataset.org/#home) dataset, preprocess and cache a subset of the images using Inception V3, train an encoder-decoder model, and use it to generate captions on new images.\n", - "\n", - "The code requires TensorFlow version >=1.9. If you're running this in [Colab]()\n", - "\n", - "In this example, we're training on a relatively small amount of data as an example. On a single P100 GPU, this example will take about ~2 hours to train. We train on the first 30,000 captions (corresponding to about ~20,000 images depending on shuffling, as there are multiple captions per image in the dataset)\n" - ] + "colab_type": "code", + "id": "krQuPYTtRPE7" + }, + "outputs": [], + "source": [ + "annotation_zip = tf.keras.utils.get_file('captions.zip', \n", + " cache_subdir=os.path.abspath('.'),\n", + " origin = 'http://images.cocodataset.org/annotations/annotations_trainval2014.zip',\n", + " extract = True)\n", + "annotation_file = os.path.dirname(annotation_zip)+'/annotations/captions_train2014.json'\n", + "\n", + "name_of_zip = 'train2014.zip'\n", + "if not os.path.exists(os.path.abspath('.') + '/' + name_of_zip):\n", + " image_zip = tf.keras.utils.get_file(name_of_zip, \n", + " cache_subdir=os.path.abspath('.'),\n", + " origin = 'http://images.cocodataset.org/zips/train2014.zip',\n", + " extract = True)\n", + " PATH = os.path.dirname(image_zip)+'/train2014/'\n", + "else:\n", + " PATH = os.path.abspath('.')+'/train2014/'" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "aANEzb5WwSzg" + }, + "source": [ + "## Optionally, limit the size of the training set for faster training\n", + "For this example, we'll select a subset of 30,000 captions and use these and the corresponding images to train our model. As always, captioning quality will improve if you choose to use more data." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + } }, - { - "metadata": { - "id": "U8l4RJ0XRPEm", - "colab_type": "code", - "colab": { - "autoexec": { - "startup": false, - "wait_interval": 0 - } - } - }, - "cell_type": "code", - "source": [ - "# Import TensorFlow and enable eager execution\n", - "# This code requires TensorFlow version >=1.9\n", - "import tensorflow as tf\n", - "tf.enable_eager_execution()\n", - "\n", - "# We'll generate plots of attention in order to see which parts of an image\n", - "# our model focuses on during captioning\n", - "import matplotlib.pyplot as plt\n", - "\n", - "# Scikit-learn includes many helpful utilities\n", - "from sklearn.model_selection import train_test_split\n", - "from sklearn.utils import shuffle\n", - "\n", - "import re\n", - "import numpy as np\n", - "import os\n", - "import time\n", - "import json\n", - "from glob import glob\n", - "from PIL import Image\n", - "import pickle" - ], - "execution_count": 0, - "outputs": [] + "colab_type": "code", + "id": "4G3b8x8_RPFD" + }, + "outputs": [], + "source": [ + "# read the json file\n", + "with open(annotation_file, 'r') as f:\n", + " annotations = json.load(f)\n", + "\n", + "# storing the captions and the image name in vectors\n", + "all_captions = []\n", + "all_img_name_vector = []\n", + "\n", + "for annot in annotations['annotations']:\n", + " caption = ' ' + annot['caption'] + ' '\n", + " image_id = annot['image_id']\n", + " full_coco_image_path = PATH + 'COCO_train2014_' + '%012d.jpg' % (image_id)\n", + " \n", + " all_img_name_vector.append(full_coco_image_path)\n", + " all_captions.append(caption)\n", + "\n", + "# shuffling the captions and image_names together\n", + "# setting a random state\n", + "train_captions, img_name_vector = shuffle(all_captions,\n", + " all_img_name_vector,\n", + " random_state=1)\n", + "\n", + "# selecting the first 30000 captions from the shuffled set\n", + "num_examples = 30000\n", + "train_captions = train_captions[:num_examples]\n", + "img_name_vector = img_name_vector[:num_examples]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + } }, - { - "metadata": { - "id": "b6qbGw8MRPE5", - "colab_type": "text" - }, - "cell_type": "markdown", - "source": [ - "## Download and prepare the MS-COCO dataset\n", - "\n", - "We will use the [MS-COCO dataset](http://cocodataset.org/#home) to train our model. This dataset contains >82,000 images, each of which has been annotated with at least 5 different captions. The code below will download and extract the dataset automatically. \n", - "\n", - "**Caution: large download ahead**. We'll use the training set, it's a 13GB file." - ] + "colab_type": "code", + "id": "mPBMgK34RPFL" + }, + "outputs": [], + "source": [ + "len(train_captions), len(all_captions)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "8cSW4u-ORPFQ" + }, + "source": [ + "## Preprocess the images using InceptionV3\n", + "Next, we will use InceptionV3 (pretrained on Imagenet) to classify each image. We will extract features from the last convolutional layer. \n", + "\n", + "First, we will need to convert the images into the format inceptionV3 expects by:\n", + "* Resizing the image to (299, 299)\n", + "* Using the [preprocess_input](https://www.tensorflow.org/api_docs/python/tf/keras/applications/inception_v3/preprocess_input) method to place the pixels in the range of -1 to 1 (to match the format of the images used to train InceptionV3)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + } }, - { - "metadata": { - "id": "krQuPYTtRPE7", - "colab_type": "code", - "colab": { - "autoexec": { - "startup": false, - "wait_interval": 0 - } - } - }, - "cell_type": "code", - "source": [ - "annotation_zip = tf.keras.utils.get_file('captions.zip', \n", - " cache_subdir=os.path.abspath('.'),\n", - " origin = 'http://images.cocodataset.org/annotations/annotations_trainval2014.zip',\n", - " extract = True)\n", - "annotation_file = os.path.dirname(annotation_zip)+'/annotations/captions_train2014.json'\n", - "\n", - "name_of_zip = 'train2014.zip'\n", - "if not os.path.exists(os.path.abspath('.') + '/' + name_of_zip):\n", - " image_zip = tf.keras.utils.get_file(name_of_zip, \n", - " cache_subdir=os.path.abspath('.'),\n", - " origin = 'http://images.cocodataset.org/zips/train2014.zip',\n", - " extract = True)\n", - " PATH = os.path.dirname(image_zip)+'/train2014/'\n", - "else:\n", - " PATH = os.path.abspath('.')+'/train2014/'" - ], - "execution_count": 0, - "outputs": [] + "colab_type": "code", + "id": "zXR0217aRPFR" + }, + "outputs": [], + "source": [ + "def load_image(image_path):\n", + " img = tf.read_file(image_path)\n", + " img = tf.image.decode_jpeg(img, channels=3)\n", + " img = tf.image.resize_images(img, (299, 299))\n", + " img = tf.keras.applications.inception_v3.preprocess_input(img)\n", + " return img, image_path" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "MDvIu4sXRPFV" + }, + "source": [ + "## Initialize InceptionV3 and load the pretrained Imagenet weights\n", + "\n", + "To do so, we'll create a tf.keras model where the output layer is the last convolutional layer in the InceptionV3 architecture. \n", + "* Each image is forwarded through the network and the vector that we get at the end is stored in a dictionary (image_name --> feature_vector). \n", + "* We use the last convolutional layer because we are using attention in this example. The shape of the output of this layer is ```8x8x2048```. \n", + "* We avoid doing this during training so it does not become a bottleneck. \n", + "* After all the images are passed through the network, we pickle the dictionary and save it to disk." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + } }, - { - "metadata": { - "id": "aANEzb5WwSzg", - "colab_type": "text" - }, - "cell_type": "markdown", - "source": [ - "## Optionally, limit the size of the training set for faster training\n", - "For this example, we'll select a subset of 30,000 captions and use these and the corresponding images to train our model. As always, captioning quality will improve if you choose to use more data." - ] + "colab_type": "code", + "id": "RD3vW4SsRPFW" + }, + "outputs": [], + "source": [ + "image_model = tf.keras.applications.InceptionV3(include_top=False, \n", + " weights='imagenet')\n", + "new_input = image_model.input\n", + "hidden_layer = image_model.layers[-1].output\n", + "\n", + "image_features_extract_model = tf.keras.Model(new_input, hidden_layer)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "rERqlR3WRPGO" + }, + "source": [ + "## Caching the features extracted from InceptionV3\n", + "\n", + "We will pre-process each image with InceptionV3 and cache the output to disk. Caching the output in RAM would be faster but memory intensive, requiring 8 \\* 8 \\* 2048 floats per image. At the time of writing, this would exceed the memory limitations of Colab (although these may change, an instance appears to have about 12GB of memory currently). \n", + "\n", + "Performance could be improved with a more sophisticated caching strategy (e.g., by sharding the images to reduce random access disk I/O) at the cost of more code.\n", + "\n", + "This will take about 10 minutes to run in Colab with a GPU. If you'd like to see a progress bar, you could: install [tqdm](https://github.com/tqdm/tqdm) (```!pip install tqdm```), then change this line: \n", + "\n", + "```for img, path in image_dataset:``` \n", + "\n", + "to:\n", + "\n", + "```for img, path in tqdm(image_dataset):```." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + } }, - { - "metadata": { - "id": "4G3b8x8_RPFD", - "colab_type": "code", - "colab": { - "autoexec": { - "startup": false, - "wait_interval": 0 - } - } - }, - "cell_type": "code", - "source": [ - "# read the json file\n", - "with open(annotation_file, 'r') as f:\n", - " annotations = json.load(f)\n", - "\n", - "# storing the captions and the image name in vectors\n", - "all_captions = []\n", - "all_img_name_vector = []\n", - "\n", - "for annot in annotations['annotations']:\n", - " caption = ' ' + annot['caption'] + ' '\n", - " image_id = annot['image_id']\n", - " full_coco_image_path = PATH + 'COCO_train2014_' + '%012d.jpg' % (image_id)\n", - " \n", - " all_img_name_vector.append(full_coco_image_path)\n", - " all_captions.append(caption)\n", - "\n", - "# shuffling the captions and image_names together\n", - "# setting a random state\n", - "train_captions, img_name_vector = shuffle(all_captions,\n", - " all_img_name_vector,\n", - " random_state=1)\n", - "\n", - "# selecting the first 30000 captions from the shuffled set\n", - "num_examples = 30000\n", - "train_captions = train_captions[:num_examples]\n", - "img_name_vector = img_name_vector[:num_examples]" - ], - "execution_count": 0, - "outputs": [] + "colab_type": "code", + "id": "Dx_fvbVgRPGQ" + }, + "outputs": [], + "source": [ + "# getting the unique images\n", + "encode_train = sorted(set(img_name_vector))\n", + "\n", + "# feel free to change the batch_size according to your system configuration\n", + "image_dataset = tf.data.Dataset.from_tensor_slices(\n", + " encode_train).map(load_image).batch(16)\n", + "\n", + "for img, path in image_dataset:\n", + " batch_features = image_features_extract_model(img)\n", + " batch_features = tf.reshape(batch_features, \n", + " (batch_features.shape[0], -1, batch_features.shape[3]))\n", + "\n", + " for bf, p in zip(batch_features, path):\n", + " path_of_feature = p.numpy().decode(\"utf-8\")\n", + " np.save(path_of_feature, bf.numpy())" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "nyqH3zFwRPFi" + }, + "source": [ + "## Preprocess and tokenize the captions\n", + "\n", + "* First, we'll tokenize the captions (e.g., by splitting on spaces). This will give us a vocabulary of all the unique words in the data (e.g., \"surfing\", \"football\", etc).\n", + "* Next, we'll limit the vocabulary size to the top 5,000 words to save memory. We'll replace all other words with the token \"UNK\" (for unknown).\n", + "* Finally, we create a word --> index mapping and vice-versa.\n", + "* We will then pad all sequences to the be same length as the longest one. " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + } }, - { - "metadata": { - "id": "mPBMgK34RPFL", - "colab_type": "code", - "colab": { - "autoexec": { - "startup": false, - "wait_interval": 0 - } - } - }, - "cell_type": "code", - "source": [ - "len(train_captions), len(all_captions)" - ], - "execution_count": 0, - "outputs": [] + "colab_type": "code", + "id": "HZfK8RhQRPFj" + }, + "outputs": [], + "source": [ + "# This will find the maximum length of any caption in our dataset\n", + "def calc_max_length(tensor):\n", + " return max(len(t) for t in tensor)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + } }, - { - "metadata": { - "id": "8cSW4u-ORPFQ", - "colab_type": "text" - }, - "cell_type": "markdown", - "source": [ - "## Preprocess the images using InceptionV3\n", - "Next, we will use InceptionV3 (pretrained on Imagenet) to classify each image. We will extract features from the last convolutional layer. \n", - "\n", - "First, we will need to convert the images into the format inceptionV3 expects by:\n", - "* Resizing the image to (299, 299)\n", - "* Using the [preprocess_input](https://www.tensorflow.org/api_docs/python/tf/keras/applications/inception_v3/preprocess_input) method to place the pixels in the range of -1 to 1 (to match the format of the images used to train InceptionV3)." - ] + "colab_type": "code", + "id": "oJGE34aiRPFo" + }, + "outputs": [], + "source": [ + "# The steps above is a general process of dealing with text processing\n", + "\n", + "# choosing the top 5000 words from the vocabulary\n", + "top_k = 5000\n", + "tokenizer = tf.keras.preprocessing.text.Tokenizer(num_words=top_k, \n", + " oov_token=\"\", \n", + " filters='!\"#$%&()*+.,-/:;=?@[\\]^_`{|}~ ')\n", + "tokenizer.fit_on_texts(train_captions)\n", + "train_seqs = tokenizer.texts_to_sequences(train_captions)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + } }, - { - "metadata": { - "id": "zXR0217aRPFR", - "colab_type": "code", - "colab": { - "autoexec": { - "startup": false, - "wait_interval": 0 - } - } - }, - "cell_type": "code", - "source": [ - "def load_image(image_path):\n", - " img = tf.read_file(image_path)\n", - " img = tf.image.decode_jpeg(img, channels=3)\n", - " img = tf.image.resize_images(img, (299, 299))\n", - " img = tf.keras.applications.inception_v3.preprocess_input(img)\n", - " return img, image_path" - ], - "execution_count": 0, - "outputs": [] + "colab_type": "code", + "id": "8Q44tNQVRPFt" + }, + "outputs": [], + "source": [ + "tokenizer.word_index[''] = 0" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + } }, - { - "metadata": { - "id": "MDvIu4sXRPFV", - "colab_type": "text" - }, - "cell_type": "markdown", - "source": [ - "## Initialize InceptionV3 and load the pretrained Imagenet weights\n", - "\n", - "To do so, we'll create a tf.keras model where the output layer is the last convolutional layer in the InceptionV3 architecture. \n", - "* Each image is forwarded through the network and the vector that we get at the end is stored in a dictionary (image_name --> feature_vector). \n", - "* We use the last convolutional layer because we are using attention in this example. The shape of the output of this layer is ```8x8x2048```. \n", - "* We avoid doing this during training so it does not become a bottleneck. \n", - "* After all the images are passed through the network, we pickle the dictionary and save it to disk." - ] + "colab_type": "code", + "id": "0fpJb5ojRPFv" + }, + "outputs": [], + "source": [ + "# creating the tokenized vectors\n", + "train_seqs = tokenizer.texts_to_sequences(train_captions)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + } }, - { - "metadata": { - "id": "RD3vW4SsRPFW", - "colab_type": "code", - "colab": { - "autoexec": { - "startup": false, - "wait_interval": 0 - } - } - }, - "cell_type": "code", - "source": [ - "image_model = tf.keras.applications.InceptionV3(include_top=False, \n", - " weights='imagenet')\n", - "new_input = image_model.input\n", - "hidden_layer = image_model.layers[-1].output\n", - "\n", - "image_features_extract_model = tf.keras.Model(new_input, hidden_layer)" - ], - "execution_count": 0, - "outputs": [] + "colab_type": "code", + "id": "AidglIZVRPF4" + }, + "outputs": [], + "source": [ + "# padding each vector to the max_length of the captions\n", + "# if the max_length parameter is not provided, pad_sequences calculates that automatically\n", + "cap_vector = tf.keras.preprocessing.sequence.pad_sequences(train_seqs, padding='post')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + } }, - { - "metadata": { - "id": "rERqlR3WRPGO", - "colab_type": "text" - }, - "cell_type": "markdown", - "source": [ - "## Caching the features extracted from InceptionV3\n", - "\n", - "We will pre-process each image with InceptionV3 and cache the output to disk. Caching the output in RAM would be faster but memory intensive, requiring 8 \\* 8 \\* 2048 floats per image. At the time of writing, this would exceed the memory limitations of Colab (although these may change, an instance appears to have about 12GB of memory currently). \n", - "\n", - "Performance could be improved with a more sophisticated caching strategy (e.g., by sharding the images to reduce random access disk I/O) at the cost of more code.\n", - "\n", - "This will take about 10 minutes to run in Colab with a GPU. If you'd like to see a progress bar, you could: install [tqdm](https://github.com/tqdm/tqdm) (```!pip install tqdm```), then change this line: \n", - "\n", - "```for img, path in image_dataset:``` \n", - "\n", - "to:\n", - "\n", - "```for img, path in tqdm(image_dataset):```." - ] + "colab_type": "code", + "id": "gL0wkttkRPGA" + }, + "outputs": [], + "source": [ + "# calculating the max_length \n", + "# used to store the attention weights\n", + "max_length = calc_max_length(train_seqs)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "M3CD75nDpvTI" + }, + "source": [ + "## Split the data into training and testing" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + } }, - { - "metadata": { - "id": "Dx_fvbVgRPGQ", - "colab_type": "code", - "colab": { - "autoexec": { - "startup": false, - "wait_interval": 0 - } - } - }, - "cell_type": "code", - "source": [ - "# getting the unique images\n", - "encode_train = sorted(set(img_name_vector))\n", - "\n", - "# feel free to change the batch_size according to your system configuration\n", - "image_dataset = tf.data.Dataset.from_tensor_slices(\n", - " encode_train).map(load_image).batch(16)\n", - "\n", - "for img, path in image_dataset:\n", - " batch_features = image_features_extract_model(img)\n", - " batch_features = tf.reshape(batch_features, \n", - " (batch_features.shape[0], -1, batch_features.shape[3]))\n", - "\n", - " for bf, p in zip(batch_features, path):\n", - " path_of_feature = p.numpy().decode(\"utf-8\")\n", - " np.save(path_of_feature, bf.numpy())" - ], - "execution_count": 0, - "outputs": [] + "colab_type": "code", + "id": "iS7DDMszRPGF" + }, + "outputs": [], + "source": [ + "# Create training and validation sets using 80-20 split\n", + "img_name_train, img_name_val, cap_train, cap_val = train_test_split(img_name_vector, \n", + " cap_vector, \n", + " test_size=0.2, \n", + " random_state=0)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + } }, - { - "metadata": { - "id": "nyqH3zFwRPFi", - "colab_type": "text" - }, - "cell_type": "markdown", - "source": [ - "## Preprocess and tokenize the captions\n", - "\n", - "* First, we'll tokenize the captions (e.g., by splitting on spaces). This will give us a vocabulary of all the unique words in the data (e.g., \"surfing\", \"football\", etc).\n", - "* Next, we'll limit the vocabulary size to the top 5,000 words to save memory. We'll replace all other words with the token \"UNK\" (for unknown).\n", - "* Finally, we create a word --> index mapping and vice-versa.\n", - "* We will then pad all sequences to the be same length as the longest one. " - ] + "colab_type": "code", + "id": "XmViPkRFRPGH" + }, + "outputs": [], + "source": [ + "len(img_name_train), len(cap_train), len(img_name_val), len(cap_val)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "uEWM9xrYcg45" + }, + "source": [ + "## Our images and captions are ready! Next, let's create a tf.data dataset to use for training our model.\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + } }, - { - "metadata": { - "id": "HZfK8RhQRPFj", - "colab_type": "code", - "colab": { - "autoexec": { - "startup": false, - "wait_interval": 0 - } - } - }, - "cell_type": "code", - "source": [ - "# This will find the maximum length of any caption in our dataset\n", - "def calc_max_length(tensor):\n", - " return max(len(t) for t in tensor)" - ], - "execution_count": 0, - "outputs": [] + "colab_type": "code", + "id": "Q3TnZ1ToRPGV" + }, + "outputs": [], + "source": [ + "# feel free to change these parameters according to your system's configuration\n", + "\n", + "BATCH_SIZE = 64\n", + "BUFFER_SIZE = 1000\n", + "embedding_dim = 256\n", + "units = 512\n", + "vocab_size = len(tokenizer.word_index)\n", + "# shape of the vector extracted from InceptionV3 is (64, 2048)\n", + "# these two variables represent that\n", + "features_shape = 2048\n", + "attention_features_shape = 64" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + } }, - { - "metadata": { - "id": "oJGE34aiRPFo", - "colab_type": "code", - "colab": { - "autoexec": { - "startup": false, - "wait_interval": 0 - } - } - }, - "cell_type": "code", - "source": [ - "# The steps above is a general process of dealing with text processing\n", - "\n", - "# choosing the top 5000 words from the vocabulary\n", - "top_k = 5000\n", - "tokenizer = tf.keras.preprocessing.text.Tokenizer(num_words=top_k, \n", - " oov_token=\"\", \n", - " filters='!\"#$%&()*+.,-/:;=?@[\\]^_`{|}~ ')\n", - "tokenizer.fit_on_texts(train_captions)\n", - "train_seqs = tokenizer.texts_to_sequences(train_captions)" - ], - "execution_count": 0, - "outputs": [] + "colab_type": "code", + "id": "SmZS2N0bXG3T" + }, + "outputs": [], + "source": [ + "# loading the numpy files \n", + "def map_func(img_name, cap):\n", + " img_tensor = np.load(img_name.decode('utf-8')+'.npy')\n", + " return img_tensor, cap" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + } }, - { - "metadata": { - "id": "8Q44tNQVRPFt", - "colab_type": "code", - "colab": { - "autoexec": { - "startup": false, - "wait_interval": 0 - } - } - }, - "cell_type": "code", - "source": [ - "tokenizer.word_index = {key:value for key, value in tokenizer.word_index.items() if value <= top_k}\n", - "# putting token in the word2idx dictionary\n", - "tokenizer.word_index[tokenizer.oov_token] = top_k + 1\n", - "tokenizer.word_index[''] = 0" - ], - "execution_count": 0, - "outputs": [] + "colab_type": "code", + "id": "FDF_Nm3tRPGZ" + }, + "outputs": [], + "source": [ + "dataset = tf.data.Dataset.from_tensor_slices((img_name_train, cap_train))\n", + "\n", + "# using map to load the numpy files in parallel\n", + "# NOTE: Be sure to set num_parallel_calls to the number of CPU cores you have\n", + "# https://www.tensorflow.org/api_docs/python/tf/py_func\n", + "dataset = dataset.map(lambda item1, item2: tf.py_func(\n", + " map_func, [item1, item2], [tf.float32, tf.int32]), num_parallel_calls=8)\n", + "\n", + "# shuffling and batching\n", + "dataset = dataset.shuffle(BUFFER_SIZE)\n", + "# https://www.tensorflow.org/api_docs/python/tf/contrib/data/batch_and_drop_remainder\n", + "dataset = dataset.batch(BATCH_SIZE)\n", + "dataset = dataset.prefetch(1)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "nrvoDphgRPGd" + }, + "source": [ + "## Model\n", + "\n", + "Fun fact, the decoder below is identical to the one in the example for [Neural Machine Translation with Attention]( https://github.com/tensorflow/tensorflow/blob/master/tensorflow/contrib/eager/python/examples/nmt_with_attention/nmt_with_attention.ipynb).\n", + "\n", + "The model architecture is inspired by the [Show, Attend and Tell](https://arxiv.org/pdf/1502.03044.pdf) paper.\n", + "\n", + "* In this example, we extract the features from the lower convolutional layer of InceptionV3 giving us a vector of shape (8, 8, 2048). \n", + "* We squash that to a shape of (64, 2048).\n", + "* This vector is then passed through the CNN Encoder(which consists of a single Fully connected layer).\n", + "* The RNN(here GRU) attends over the image to predict the next word." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + } }, - { - "metadata": { - "id": "0fpJb5ojRPFv", - "colab_type": "code", - "colab": { - "autoexec": { - "startup": false, - "wait_interval": 0 - } - } - }, - "cell_type": "code", - "source": [ - "# creating the tokenized vectors\n", - "train_seqs = tokenizer.texts_to_sequences(train_captions)" - ], - "execution_count": 0, - "outputs": [] + "colab_type": "code", + "id": "AAppCGLKRPGd" + }, + "outputs": [], + "source": [ + "def gru(units):\n", + " # If you have a GPU, we recommend using the CuDNNGRU layer (it provides a \n", + " # significant speedup).\n", + " if tf.test.is_gpu_available():\n", + " return tf.keras.layers.CuDNNGRU(units, \n", + " return_sequences=True, \n", + " return_state=True, \n", + " recurrent_initializer='glorot_uniform')\n", + " else:\n", + " return tf.keras.layers.GRU(units, \n", + " return_sequences=True, \n", + " return_state=True, \n", + " recurrent_activation='sigmoid', \n", + " recurrent_initializer='glorot_uniform')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + } }, - { - "metadata": { - "id": "olQArbgbRPF1", - "colab_type": "code", - "colab": { - "autoexec": { - "startup": false, - "wait_interval": 0 - } - } - }, - "cell_type": "code", - "source": [ - "# creating a reverse mapping (index -> word)\n", - "index_word = {value:key for key, value in tokenizer.word_index.items()}" - ], - "execution_count": 0, - "outputs": [] + "colab_type": "code", + "id": "ja2LFTMSdeV3" + }, + "outputs": [], + "source": [ + "class BahdanauAttention(tf.keras.Model):\n", + " def __init__(self, units):\n", + " super(BahdanauAttention, self).__init__()\n", + " self.W1 = tf.keras.layers.Dense(units)\n", + " self.W2 = tf.keras.layers.Dense(units)\n", + " self.V = tf.keras.layers.Dense(1)\n", + " \n", + " def call(self, features, hidden):\n", + " # features(CNN_encoder output) shape == (batch_size, 64, embedding_dim)\n", + " \n", + " # hidden shape == (batch_size, hidden_size)\n", + " # hidden_with_time_axis shape == (batch_size, 1, hidden_size)\n", + " hidden_with_time_axis = tf.expand_dims(hidden, 1)\n", + " \n", + " # score shape == (batch_size, 64, hidden_size)\n", + " score = tf.nn.tanh(self.W1(features) + self.W2(hidden_with_time_axis))\n", + " \n", + " # attention_weights shape == (batch_size, 64, 1)\n", + " # we get 1 at the last axis because we are applying score to self.V\n", + " attention_weights = tf.nn.softmax(self.V(score), axis=1)\n", + " \n", + " # context_vector shape after sum == (batch_size, hidden_size)\n", + " context_vector = attention_weights * features\n", + " context_vector = tf.reduce_sum(context_vector, axis=1)\n", + " \n", + " return context_vector, attention_weights" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + } }, - { - "metadata": { - "id": "AidglIZVRPF4", - "colab_type": "code", - "colab": { - "autoexec": { - "startup": false, - "wait_interval": 0 - } - } - }, - "cell_type": "code", - "source": [ - "# padding each vector to the max_length of the captions\n", - "# if the max_length parameter is not provided, pad_sequences calculates that automatically\n", - "cap_vector = tf.keras.preprocessing.sequence.pad_sequences(train_seqs, padding='post')" - ], - "execution_count": 0, - "outputs": [] + "colab_type": "code", + "id": "AZ7R1RxHRPGf" + }, + "outputs": [], + "source": [ + "class CNN_Encoder(tf.keras.Model):\n", + " # Since we have already extracted the features and dumped it using pickle\n", + " # This encoder passes those features through a Fully connected layer\n", + " def __init__(self, embedding_dim):\n", + " super(CNN_Encoder, self).__init__()\n", + " # shape after fc == (batch_size, 64, embedding_dim)\n", + " self.fc = tf.keras.layers.Dense(embedding_dim)\n", + " \n", + " def call(self, x):\n", + " x = self.fc(x)\n", + " x = tf.nn.relu(x)\n", + " return x" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + } }, - { - "metadata": { - "id": "gL0wkttkRPGA", - "colab_type": "code", - "colab": { - "autoexec": { - "startup": false, - "wait_interval": 0 - } - } - }, - "cell_type": "code", - "source": [ - "# calculating the max_length \n", - "# used to store the attention weights\n", - "max_length = calc_max_length(train_seqs)" - ], - "execution_count": 0, - "outputs": [] + "colab_type": "code", + "id": "V9UbGQmERPGi" + }, + "outputs": [], + "source": [ + "class RNN_Decoder(tf.keras.Model):\n", + " def __init__(self, embedding_dim, units, vocab_size):\n", + " super(RNN_Decoder, self).__init__()\n", + " self.units = units\n", + "\n", + " self.embedding = tf.keras.layers.Embedding(vocab_size, embedding_dim)\n", + " self.gru = gru(self.units)\n", + " self.fc1 = tf.keras.layers.Dense(self.units)\n", + " self.fc2 = tf.keras.layers.Dense(vocab_size)\n", + " \n", + " self.attention = BahdanauAttention(self.units)\n", + " \n", + " def call(self, x, features, hidden):\n", + " # defining attention as a separate model\n", + " context_vector, attention_weights = self.attention(features, hidden)\n", + " \n", + " # x shape after passing through embedding == (batch_size, 1, embedding_dim)\n", + " x = self.embedding(x)\n", + " \n", + " # x shape after concatenation == (batch_size, 1, embedding_dim + hidden_size)\n", + " x = tf.concat([tf.expand_dims(context_vector, 1), x], axis=-1)\n", + " \n", + " # passing the concatenated vector to the GRU\n", + " output, state = self.gru(x)\n", + " \n", + " # shape == (batch_size, max_length, hidden_size)\n", + " x = self.fc1(output)\n", + " \n", + " # x shape == (batch_size * max_length, hidden_size)\n", + " x = tf.reshape(x, (-1, x.shape[2]))\n", + " \n", + " # output shape == (batch_size * max_length, vocab)\n", + " x = self.fc2(x)\n", + "\n", + " return x, state, attention_weights\n", + "\n", + " def reset_state(self, batch_size):\n", + " return tf.zeros((batch_size, self.units))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + } }, - { - "metadata": { - "id": "M3CD75nDpvTI", - "colab_type": "text" - }, - "cell_type": "markdown", - "source": [ - "## Split the data into training and testing" - ] + "colab_type": "code", + "id": "Qs_Sr03wRPGk" + }, + "outputs": [], + "source": [ + "encoder = CNN_Encoder(embedding_dim)\n", + "decoder = RNN_Decoder(embedding_dim, units, vocab_size)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + } }, - { - "metadata": { - "id": "iS7DDMszRPGF", - "colab_type": "code", - "colab": { - "autoexec": { - "startup": false, - "wait_interval": 0 - } - } - }, - "cell_type": "code", - "source": [ - "# Create training and validation sets using 80-20 split\n", - "img_name_train, img_name_val, cap_train, cap_val = train_test_split(img_name_vector, \n", - " cap_vector, \n", - " test_size=0.2, \n", - " random_state=0)" - ], - "execution_count": 0, - "outputs": [] + "colab_type": "code", + "id": "-bYN7xA0RPGl" + }, + "outputs": [], + "source": [ + "optimizer = tf.train.AdamOptimizer()\n", + "\n", + "# We are masking the loss calculated for padding\n", + "def loss_function(real, pred):\n", + " mask = 1 - np.equal(real, 0)\n", + " loss_ = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=real, logits=pred) * mask\n", + " return tf.reduce_mean(loss_)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "PHod7t72RPGn" + }, + "source": [ + "## Training\n", + "\n", + "* We extract the features stored in the respective `.npy` files and then pass those features through the encoder.\n", + "* The encoder output, hidden state(initialized to 0) and the decoder input (which is the start token) is passed to the decoder.\n", + "* The decoder returns the predictions and the decoder hidden state.\n", + "* The decoder hidden state is then passed back into the model and the predictions are used to calculate the loss.\n", + "* Use teacher forcing to decide the next input to the decoder.\n", + "* Teacher forcing is the technique where the target word is passed as the next input to the decoder.\n", + "* The final step is to calculate the gradients and apply it to the optimizer and backpropagate.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + } }, - { - "metadata": { - "id": "XmViPkRFRPGH", - "colab_type": "code", - "colab": { - "autoexec": { - "startup": false, - "wait_interval": 0 - } - } - }, - "cell_type": "code", - "source": [ - "len(img_name_train), len(cap_train), len(img_name_val), len(cap_val)" - ], - "execution_count": 0, - "outputs": [] + "colab_type": "code", + "id": "Vt4WZ5mhJE-E" + }, + "outputs": [], + "source": [ + "# adding this in a separate cell because if you run the training cell \n", + "# many times, the loss_plot array will be reset\n", + "loss_plot = []" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + } }, - { - "metadata": { - "id": "uEWM9xrYcg45", - "colab_type": "text" - }, - "cell_type": "markdown", - "source": [ - "## Our images and captions are ready! Next, let's create a tf.data dataset to use for training our model.\n", - "\n" - ] + "colab_type": "code", + "id": "UlA4VIQpRPGo" + }, + "outputs": [], + "source": [ + "EPOCHS = 20\n", + "\n", + "for epoch in range(EPOCHS):\n", + " start = time.time()\n", + " total_loss = 0\n", + " \n", + " for (batch, (img_tensor, target)) in enumerate(dataset):\n", + " loss = 0\n", + " \n", + " # initializing the hidden state for each batch\n", + " # because the captions are not related from image to image\n", + " hidden = decoder.reset_state(batch_size=target.shape[0])\n", + "\n", + " dec_input = tf.expand_dims([tokenizer.word_index['']] * BATCH_SIZE, 1)\n", + " \n", + " with tf.GradientTape() as tape:\n", + " features = encoder(img_tensor)\n", + " \n", + " for i in range(1, target.shape[1]):\n", + " # passing the features through the decoder\n", + " predictions, hidden, _ = decoder(dec_input, features, hidden)\n", + "\n", + " loss += loss_function(target[:, i], predictions)\n", + " \n", + " # using teacher forcing\n", + " dec_input = tf.expand_dims(target[:, i], 1)\n", + " \n", + " total_loss += (loss / int(target.shape[1]))\n", + " \n", + " variables = encoder.variables + decoder.variables\n", + " \n", + " gradients = tape.gradient(loss, variables) \n", + " \n", + " optimizer.apply_gradients(zip(gradients, variables), tf.train.get_or_create_global_step())\n", + " \n", + " if batch % 100 == 0:\n", + " print ('Epoch {} Batch {} Loss {:.4f}'.format(epoch + 1, \n", + " batch, \n", + " loss.numpy() / int(target.shape[1])))\n", + " # storing the epoch end loss value to plot later\n", + " loss_plot.append(total_loss / len(cap_vector))\n", + " \n", + " print ('Epoch {} Loss {:.6f}'.format(epoch + 1, \n", + " total_loss/len(cap_vector)))\n", + " print ('Time taken for 1 epoch {} sec\\n'.format(time.time() - start))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + } }, - { - "metadata": { - "id": "Q3TnZ1ToRPGV", - "colab_type": "code", - "colab": { - "autoexec": { - "startup": false, - "wait_interval": 0 - } - } - }, - "cell_type": "code", - "source": [ - "# feel free to change these parameters according to your system's configuration\n", - "\n", - "BATCH_SIZE = 64\n", - "BUFFER_SIZE = 1000\n", - "embedding_dim = 256\n", - "units = 512\n", - "vocab_size = len(tokenizer.word_index)\n", - "# shape of the vector extracted from InceptionV3 is (64, 2048)\n", - "# these two variables represent that\n", - "features_shape = 2048\n", - "attention_features_shape = 64" - ], - "execution_count": 0, - "outputs": [] + "colab_type": "code", + "id": "1Wm83G-ZBPcC" + }, + "outputs": [], + "source": [ + "plt.plot(loss_plot)\n", + "plt.xlabel('Epochs')\n", + "plt.ylabel('Loss')\n", + "plt.title('Loss Plot')\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "xGvOcLQKghXN" + }, + "source": [ + "## Caption!\n", + "\n", + "* The evaluate function is similar to the training loop, except we don't use teacher forcing here. The input to the decoder at each time step is its previous predictions along with the hidden state and the encoder output.\n", + "* Stop predicting when the model predicts the end token.\n", + "* And store the attention weights for every time step." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + } }, - { - "metadata": { - "id": "SmZS2N0bXG3T", - "colab_type": "code", - "colab": { - "autoexec": { - "startup": false, - "wait_interval": 0 - } - } - }, - "cell_type": "code", - "source": [ - "# loading the numpy files \n", - "def map_func(img_name, cap):\n", - " img_tensor = np.load(img_name.decode('utf-8')+'.npy')\n", - " return img_tensor, cap" - ], - "execution_count": 0, - "outputs": [] + "colab_type": "code", + "id": "RCWpDtyNRPGs" + }, + "outputs": [], + "source": [ + "def evaluate(image):\n", + " attention_plot = np.zeros((max_length, attention_features_shape))\n", + "\n", + " hidden = decoder.reset_state(batch_size=1)\n", + "\n", + " temp_input = tf.expand_dims(load_image(image)[0], 0)\n", + " img_tensor_val = image_features_extract_model(temp_input)\n", + " img_tensor_val = tf.reshape(img_tensor_val, (img_tensor_val.shape[0], -1, img_tensor_val.shape[3]))\n", + "\n", + " features = encoder(img_tensor_val)\n", + "\n", + " dec_input = tf.expand_dims([tokenizer.word_index['']], 0)\n", + " result = []\n", + "\n", + " for i in range(max_length):\n", + " predictions, hidden, attention_weights = decoder(dec_input, features, hidden)\n", + "\n", + " attention_plot[i] = tf.reshape(attention_weights, (-1, )).numpy()\n", + "\n", + " predicted_id = tf.argmax(predictions[0]).numpy()\n", + " result.append(tokenizer.index_word[predicted_id])\n", + "\n", + " if tokenizer.index_word[predicted_id] == '':\n", + " return result, attention_plot\n", + "\n", + " dec_input = tf.expand_dims([predicted_id], 0)\n", + "\n", + " attention_plot = attention_plot[:len(result), :]\n", + " return result, attention_plot" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + } }, - { - "metadata": { - "id": "FDF_Nm3tRPGZ", - "colab_type": "code", - "colab": { - "autoexec": { - "startup": false, - "wait_interval": 0 - } - } - }, - "cell_type": "code", - "source": [ - "dataset = tf.data.Dataset.from_tensor_slices((img_name_train, cap_train))\n", - "\n", - "# using map to load the numpy files in parallel\n", - "# NOTE: Be sure to set num_parallel_calls to the number of CPU cores you have\n", - "# https://www.tensorflow.org/api_docs/python/tf/py_func\n", - "dataset = dataset.map(lambda item1, item2: tf.py_func(\n", - " map_func, [item1, item2], [tf.float32, tf.int32]), num_parallel_calls=8)\n", - "\n", - "# shuffling and batching\n", - "dataset = dataset.shuffle(BUFFER_SIZE)\n", - "# https://www.tensorflow.org/api_docs/python/tf/contrib/data/batch_and_drop_remainder\n", - "dataset = dataset.batch(BATCH_SIZE)\n", - "dataset = dataset.prefetch(1)" - ], - "execution_count": 0, - "outputs": [] + "colab_type": "code", + "id": "fD_y7PD6RPGt" + }, + "outputs": [], + "source": [ + "def plot_attention(image, result, attention_plot):\n", + " temp_image = np.array(Image.open(image))\n", + "\n", + " fig = plt.figure(figsize=(10, 10))\n", + " \n", + " len_result = len(result)\n", + " for l in range(len_result):\n", + " temp_att = np.resize(attention_plot[l], (8, 8))\n", + " ax = fig.add_subplot(len_result//2, len_result//2, l+1)\n", + " ax.set_title(result[l])\n", + " img = ax.imshow(temp_image)\n", + " ax.imshow(temp_att, cmap='gray', alpha=0.6, extent=img.get_extent())\n", + "\n", + " plt.tight_layout()\n", + " plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + } }, - { - "metadata": { - "id": "nrvoDphgRPGd", - "colab_type": "text" - }, - "cell_type": "markdown", - "source": [ - "## Model\n", - "\n", - "Fun fact, the decoder below is identical to the one in the example for [Neural Machine Translation with Attention]( https://github.com/tensorflow/tensorflow/blob/master/tensorflow/contrib/eager/python/examples/nmt_with_attention/nmt_with_attention.ipynb).\n", - "\n", - "The model architecture is inspired by the [Show, Attend and Tell](https://arxiv.org/pdf/1502.03044.pdf) paper.\n", - "\n", - "* In this example, we extract the features from the lower convolutional layer of InceptionV3 giving us a vector of shape (8, 8, 2048). \n", - "* We squash that to a shape of (64, 2048).\n", - "* This vector is then passed through the CNN Encoder(which consists of a single Fully connected layer).\n", - "* The RNN(here GRU) attends over the image to predict the next word." - ] + "colab_type": "code", + "id": "io7ws3ReRPGv" + }, + "outputs": [], + "source": [ + "# captions on the validation set\n", + "rid = np.random.randint(0, len(img_name_val))\n", + "image = img_name_val[rid]\n", + "real_caption = ' '.join([tokenizer.index_word[i] for i in cap_val[rid] if i not in [0]])\n", + "result, attention_plot = evaluate(image)\n", + "\n", + "print ('Real Caption:', real_caption)\n", + "print ('Prediction Caption:', ' '.join(result))\n", + "plot_attention(image, result, attention_plot)\n", + "# opening the image\n", + "Image.open(img_name_val[rid])" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "Rprk3HEvZuxb" + }, + "source": [ + "## Try it on your own images\n", + "For fun, below we've provided a method you can use to caption your own images with the model we've just trained. Keep in mind, it was trained on a relatively small amount of data, and your images may be different from the training data (so be prepared for weird results!)\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + } }, + "colab_type": "code", + "id": "9Psd1quzaAWg" + }, + "outputs": [], + "source": [ + "image_url = 'https://tensorflow.org/images/surf.jpg'\n", + "image_extension = image_url[-4:]\n", + "image_path = tf.keras.utils.get_file('image'+image_extension, \n", + " origin=image_url)\n", + "\n", + "result, attention_plot = evaluate(image_path)\n", + "print ('Prediction Caption:', ' '.join(result))\n", + "plot_attention(image_path, result, attention_plot)\n", + "# opening the image\n", + "Image.open(image_path)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "VJZXyJco6uLO" + }, + "source": [ + "# Next steps\n", + "\n", + "Congrats! You've just trained an image captioning model with attention. Next, we recommend taking a look at this example [Neural Machine Translation with Attention]( https://github.com/tensorflow/tensorflow/blob/master/tensorflow/contrib/eager/python/examples/nmt_with_attention/nmt_with_attention.ipynb). It uses a similar architecture to translate between Spanish and English sentences. You can also experiment with training the code in this notebook on a different dataset." + ] + } + ], + "metadata": { + "accelerator": "GPU", + "colab": { + "collapsed_sections": [], + "default_view": {}, + "name": "image_captioning_with_attention.ipynb", + "private_outputs": true, + "provenance": [ { - "metadata": { - "id": "AAppCGLKRPGd", - "colab_type": "code", - "colab": { - "autoexec": { - "startup": false, - "wait_interval": 0 - } - } - }, - "cell_type": "code", - "source": [ - "def gru(units):\n", - " # If you have a GPU, we recommend using the CuDNNGRU layer (it provides a \n", - " # significant speedup).\n", - " if tf.test.is_gpu_available():\n", - " return tf.keras.layers.CuDNNGRU(units, \n", - " return_sequences=True, \n", - " return_state=True, \n", - " recurrent_initializer='glorot_uniform')\n", - " else:\n", - " return tf.keras.layers.GRU(units, \n", - " return_sequences=True, \n", - " return_state=True, \n", - " recurrent_activation='sigmoid', \n", - " recurrent_initializer='glorot_uniform')" - ], - "execution_count": 0, - "outputs": [] - }, - { - "metadata": { - "id": "ja2LFTMSdeV3", - "colab_type": "code", - "colab": { - "autoexec": { - "startup": false, - "wait_interval": 0 - } - } - }, - "cell_type": "code", - "source": [ - "class BahdanauAttention(tf.keras.Model):\n", - " def __init__(self, units):\n", - " super(BahdanauAttention, self).__init__()\n", - " self.W1 = tf.keras.layers.Dense(units)\n", - " self.W2 = tf.keras.layers.Dense(units)\n", - " self.V = tf.keras.layers.Dense(1)\n", - " \n", - " def call(self, features, hidden):\n", - " # features(CNN_encoder output) shape == (batch_size, 64, embedding_dim)\n", - " \n", - " # hidden shape == (batch_size, hidden_size)\n", - " # hidden_with_time_axis shape == (batch_size, 1, hidden_size)\n", - " hidden_with_time_axis = tf.expand_dims(hidden, 1)\n", - " \n", - " # score shape == (batch_size, 64, hidden_size)\n", - " score = tf.nn.tanh(self.W1(features) + self.W2(hidden_with_time_axis))\n", - " \n", - " # attention_weights shape == (batch_size, 64, 1)\n", - " # we get 1 at the last axis because we are applying score to self.V\n", - " attention_weights = tf.nn.softmax(self.V(score), axis=1)\n", - " \n", - " # context_vector shape after sum == (batch_size, hidden_size)\n", - " context_vector = attention_weights * features\n", - " context_vector = tf.reduce_sum(context_vector, axis=1)\n", - " \n", - " return context_vector, attention_weights" - ], - "execution_count": 0, - "outputs": [] - }, - { - "metadata": { - "id": "AZ7R1RxHRPGf", - "colab_type": "code", - "colab": { - "autoexec": { - "startup": false, - "wait_interval": 0 - } - } - }, - "cell_type": "code", - "source": [ - "class CNN_Encoder(tf.keras.Model):\n", - " # Since we have already extracted the features and dumped it using pickle\n", - " # This encoder passes those features through a Fully connected layer\n", - " def __init__(self, embedding_dim):\n", - " super(CNN_Encoder, self).__init__()\n", - " # shape after fc == (batch_size, 64, embedding_dim)\n", - " self.fc = tf.keras.layers.Dense(embedding_dim)\n", - " \n", - " def call(self, x):\n", - " x = self.fc(x)\n", - " x = tf.nn.relu(x)\n", - " return x" - ], - "execution_count": 0, - "outputs": [] - }, - { - "metadata": { - "id": "V9UbGQmERPGi", - "colab_type": "code", - "colab": { - "autoexec": { - "startup": false, - "wait_interval": 0 - } - } - }, - "cell_type": "code", - "source": [ - "class RNN_Decoder(tf.keras.Model):\n", - " def __init__(self, embedding_dim, units, vocab_size):\n", - " super(RNN_Decoder, self).__init__()\n", - " self.units = units\n", - "\n", - " self.embedding = tf.keras.layers.Embedding(vocab_size, embedding_dim)\n", - " self.gru = gru(self.units)\n", - " self.fc1 = tf.keras.layers.Dense(self.units)\n", - " self.fc2 = tf.keras.layers.Dense(vocab_size)\n", - " \n", - " self.attention = BahdanauAttention(self.units)\n", - " \n", - " def call(self, x, features, hidden):\n", - " # defining attention as a separate model\n", - " context_vector, attention_weights = self.attention(features, hidden)\n", - " \n", - " # x shape after passing through embedding == (batch_size, 1, embedding_dim)\n", - " x = self.embedding(x)\n", - " \n", - " # x shape after concatenation == (batch_size, 1, embedding_dim + hidden_size)\n", - " x = tf.concat([tf.expand_dims(context_vector, 1), x], axis=-1)\n", - " \n", - " # passing the concatenated vector to the GRU\n", - " output, state = self.gru(x)\n", - " \n", - " # shape == (batch_size, max_length, hidden_size)\n", - " x = self.fc1(output)\n", - " \n", - " # x shape == (batch_size * max_length, hidden_size)\n", - " x = tf.reshape(x, (-1, x.shape[2]))\n", - " \n", - " # output shape == (batch_size * max_length, vocab)\n", - " x = self.fc2(x)\n", - "\n", - " return x, state, attention_weights\n", - "\n", - " def reset_state(self, batch_size):\n", - " return tf.zeros((batch_size, self.units))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "metadata": { - "id": "Qs_Sr03wRPGk", - "colab_type": "code", - "colab": { - "autoexec": { - "startup": false, - "wait_interval": 0 - } - } - }, - "cell_type": "code", - "source": [ - "encoder = CNN_Encoder(embedding_dim)\n", - "decoder = RNN_Decoder(embedding_dim, units, vocab_size)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "metadata": { - "id": "-bYN7xA0RPGl", - "colab_type": "code", - "colab": { - "autoexec": { - "startup": false, - "wait_interval": 0 - } - } - }, - "cell_type": "code", - "source": [ - "optimizer = tf.train.AdamOptimizer()\n", - "\n", - "# We are masking the loss calculated for padding\n", - "def loss_function(real, pred):\n", - " mask = 1 - np.equal(real, 0)\n", - " loss_ = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=real, logits=pred) * mask\n", - " return tf.reduce_mean(loss_)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "metadata": { - "id": "PHod7t72RPGn", - "colab_type": "text" - }, - "cell_type": "markdown", - "source": [ - "## Training\n", - "\n", - "* We extract the features stored in the respective `.npy` files and then pass those features through the encoder.\n", - "* The encoder output, hidden state(initialized to 0) and the decoder input (which is the start token) is passed to the decoder.\n", - "* The decoder returns the predictions and the decoder hidden state.\n", - "* The decoder hidden state is then passed back into the model and the predictions are used to calculate the loss.\n", - "* Use teacher forcing to decide the next input to the decoder.\n", - "* Teacher forcing is the technique where the target word is passed as the next input to the decoder.\n", - "* The final step is to calculate the gradients and apply it to the optimizer and backpropagate.\n" - ] - }, - { - "metadata": { - "id": "Vt4WZ5mhJE-E", - "colab_type": "code", - "colab": { - "autoexec": { - "startup": false, - "wait_interval": 0 - } - } - }, - "cell_type": "code", - "source": [ - "# adding this in a separate cell because if you run the training cell \n", - "# many times, the loss_plot array will be reset\n", - "loss_plot = []" - ], - "execution_count": 0, - "outputs": [] - }, - { - "metadata": { - "id": "UlA4VIQpRPGo", - "colab_type": "code", - "colab": { - "autoexec": { - "startup": false, - "wait_interval": 0 - } - } - }, - "cell_type": "code", - "source": [ - "EPOCHS = 20\n", - "\n", - "for epoch in range(EPOCHS):\n", - " start = time.time()\n", - " total_loss = 0\n", - " \n", - " for (batch, (img_tensor, target)) in enumerate(dataset):\n", - " loss = 0\n", - " \n", - " # initializing the hidden state for each batch\n", - " # because the captions are not related from image to image\n", - " hidden = decoder.reset_state(batch_size=target.shape[0])\n", - "\n", - " dec_input = tf.expand_dims([tokenizer.word_index['']] * BATCH_SIZE, 1)\n", - " \n", - " with tf.GradientTape() as tape:\n", - " features = encoder(img_tensor)\n", - " \n", - " for i in range(1, target.shape[1]):\n", - " # passing the features through the decoder\n", - " predictions, hidden, _ = decoder(dec_input, features, hidden)\n", - "\n", - " loss += loss_function(target[:, i], predictions)\n", - " \n", - " # using teacher forcing\n", - " dec_input = tf.expand_dims(target[:, i], 1)\n", - " \n", - " total_loss += (loss / int(target.shape[1]))\n", - " \n", - " variables = encoder.variables + decoder.variables\n", - " \n", - " gradients = tape.gradient(loss, variables) \n", - " \n", - " optimizer.apply_gradients(zip(gradients, variables), tf.train.get_or_create_global_step())\n", - " \n", - " if batch % 100 == 0:\n", - " print ('Epoch {} Batch {} Loss {:.4f}'.format(epoch + 1, \n", - " batch, \n", - " loss.numpy() / int(target.shape[1])))\n", - " # storing the epoch end loss value to plot later\n", - " loss_plot.append(total_loss / len(cap_vector))\n", - " \n", - " print ('Epoch {} Loss {:.6f}'.format(epoch + 1, \n", - " total_loss/len(cap_vector)))\n", - " print ('Time taken for 1 epoch {} sec\\n'.format(time.time() - start))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "metadata": { - "id": "1Wm83G-ZBPcC", - "colab_type": "code", - "colab": { - "autoexec": { - "startup": false, - "wait_interval": 0 - } - } - }, - "cell_type": "code", - "source": [ - "plt.plot(loss_plot)\n", - "plt.xlabel('Epochs')\n", - "plt.ylabel('Loss')\n", - "plt.title('Loss Plot')\n", - "plt.show()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "metadata": { - "id": "xGvOcLQKghXN", - "colab_type": "text" - }, - "cell_type": "markdown", - "source": [ - "## Caption!\n", - "\n", - "* The evaluate function is similar to the training loop, except we don't use teacher forcing here. The input to the decoder at each time step is its previous predictions along with the hidden state and the encoder output.\n", - "* Stop predicting when the model predicts the end token.\n", - "* And store the attention weights for every time step." - ] - }, - { - "metadata": { - "id": "RCWpDtyNRPGs", - "colab_type": "code", - "colab": { - "autoexec": { - "startup": false, - "wait_interval": 0 - } - } - }, - "cell_type": "code", - "source": [ - "def evaluate(image):\n", - " attention_plot = np.zeros((max_length, attention_features_shape))\n", - "\n", - " hidden = decoder.reset_state(batch_size=1)\n", - "\n", - " temp_input = tf.expand_dims(load_image(image)[0], 0)\n", - " img_tensor_val = image_features_extract_model(temp_input)\n", - " img_tensor_val = tf.reshape(img_tensor_val, (img_tensor_val.shape[0], -1, img_tensor_val.shape[3]))\n", - "\n", - " features = encoder(img_tensor_val)\n", - "\n", - " dec_input = tf.expand_dims([tokenizer.word_index['']], 0)\n", - " result = []\n", - "\n", - " for i in range(max_length):\n", - " predictions, hidden, attention_weights = decoder(dec_input, features, hidden)\n", - "\n", - " attention_plot[i] = tf.reshape(attention_weights, (-1, )).numpy()\n", - "\n", - " predicted_id = tf.argmax(predictions[0]).numpy()\n", - " result.append(index_word[predicted_id])\n", - "\n", - " if index_word[predicted_id] == '':\n", - " return result, attention_plot\n", - "\n", - " dec_input = tf.expand_dims([predicted_id], 0)\n", - "\n", - " attention_plot = attention_plot[:len(result), :]\n", - " return result, attention_plot" - ], - "execution_count": 0, - "outputs": [] - }, - { - "metadata": { - "id": "fD_y7PD6RPGt", - "colab_type": "code", - "colab": { - "autoexec": { - "startup": false, - "wait_interval": 0 - } - } - }, - "cell_type": "code", - "source": [ - "def plot_attention(image, result, attention_plot):\n", - " temp_image = np.array(Image.open(image))\n", - "\n", - " fig = plt.figure(figsize=(10, 10))\n", - " \n", - " len_result = len(result)\n", - " for l in range(len_result):\n", - " temp_att = np.resize(attention_plot[l], (8, 8))\n", - " ax = fig.add_subplot(len_result//2, len_result//2, l+1)\n", - " ax.set_title(result[l])\n", - " img = ax.imshow(temp_image)\n", - " ax.imshow(temp_att, cmap='gray', alpha=0.6, extent=img.get_extent())\n", - "\n", - " plt.tight_layout()\n", - " plt.show()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "metadata": { - "id": "io7ws3ReRPGv", - "colab_type": "code", - "colab": { - "autoexec": { - "startup": false, - "wait_interval": 0 - } - } - }, - "cell_type": "code", - "source": [ - "# captions on the validation set\n", - "rid = np.random.randint(0, len(img_name_val))\n", - "image = img_name_val[rid]\n", - "real_caption = ' '.join([index_word[i] for i in cap_val[rid] if i not in [0]])\n", - "result, attention_plot = evaluate(image)\n", - "\n", - "print ('Real Caption:', real_caption)\n", - "print ('Prediction Caption:', ' '.join(result))\n", - "plot_attention(image, result, attention_plot)\n", - "# opening the image\n", - "Image.open(img_name_val[rid])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "metadata": { - "id": "Rprk3HEvZuxb", - "colab_type": "text" - }, - "cell_type": "markdown", - "source": [ - "## Try it on your own images\n", - "For fun, below we've provided a method you can use to caption your own images with the model we've just trained. Keep in mind, it was trained on a relatively small amount of data, and your images may be different from the training data (so be prepared for weird results!)\n" - ] - }, - { - "metadata": { - "id": "9Psd1quzaAWg", - "colab_type": "code", - "colab": { - "autoexec": { - "startup": false, - "wait_interval": 0 - } - } - }, - "cell_type": "code", - "source": [ - "image_url = 'https://tensorflow.org/images/surf.jpg'\n", - "image_extension = image_url[-4:]\n", - "image_path = tf.keras.utils.get_file('image'+image_extension, \n", - " origin=image_url)\n", - "\n", - "result, attention_plot = evaluate(image_path)\n", - "print ('Prediction Caption:', ' '.join(result))\n", - "plot_attention(image_path, result, attention_plot)\n", - "# opening the image\n", - "Image.open(image_path)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "metadata": { - "id": "VJZXyJco6uLO", - "colab_type": "text" - }, - "cell_type": "markdown", - "source": [ - "# Next steps\n", - "\n", - "Congrats! You've just trained an image captioning model with attention. Next, we recommend taking a look at this example [Neural Machine Translation with Attention]( https://github.com/tensorflow/tensorflow/blob/master/tensorflow/contrib/eager/python/examples/nmt_with_attention/nmt_with_attention.ipynb). It uses a similar architecture to translate between Spanish and English sentences. You can also experiment with training the code in this notebook on a different dataset." - ] + "file_id": "1HI8OK2sMjcx9CTWVn0122QAHOuXaOaMg", + "timestamp": 1530222436922 } - ] + ], + "toc_visible": true, + "version": "0.3.2", + "views": {} + }, + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.6.3" + } + }, + "nbformat": 4, + "nbformat_minor": 2 } diff --git a/tensorflow/contrib/eager/python/examples/linear_regression/linear_regression_graph_test.py b/tensorflow/contrib/eager/python/examples/linear_regression/linear_regression_graph_test.py index 557ad427521..d412b25b368 100644 --- a/tensorflow/contrib/eager/python/examples/linear_regression/linear_regression_graph_test.py +++ b/tensorflow/contrib/eager/python/examples/linear_regression/linear_regression_graph_test.py @@ -36,7 +36,7 @@ class GraphLinearRegressionBenchmark(tf.test.Benchmark): noise_level=0.01, batch_size=batch_size, num_batches=num_batches) - iterator = dataset.make_initializable_iterator() + iterator = tf.compat.v1.data.make_initializable_iterator(dataset) x, y = iterator.get_next() model = linear_regression.LinearModel() diff --git a/tensorflow/contrib/eager/python/examples/nmt_with_attention/nmt_with_attention.ipynb b/tensorflow/contrib/eager/python/examples/nmt_with_attention/nmt_with_attention.ipynb index 480777d9487..66d52a74943 100644 --- a/tensorflow/contrib/eager/python/examples/nmt_with_attention/nmt_with_attention.ipynb +++ b/tensorflow/contrib/eager/python/examples/nmt_with_attention/nmt_with_attention.ipynb @@ -768,7 +768,7 @@ }, "outputs": [], "source": [ - "translate('hace mucho frio aqui.', encoder, decoder, inp_lang, targ_lang, max_length_inp, max_length_targ)" + "translate(u'hace mucho frio aqui.', encoder, decoder, inp_lang, targ_lang, max_length_inp, max_length_targ)" ] }, { @@ -781,7 +781,7 @@ }, "outputs": [], "source": [ - "translate('esta es mi vida.', encoder, decoder, inp_lang, targ_lang, max_length_inp, max_length_targ)" + "translate(u'esta es mi vida.', encoder, decoder, inp_lang, targ_lang, max_length_inp, max_length_targ)" ] }, { @@ -794,7 +794,7 @@ }, "outputs": [], "source": [ - "translate('¿todavia estan en casa?', encoder, decoder, inp_lang, targ_lang, max_length_inp, max_length_targ)" + "translate(u'todavia estan en casa?', encoder, decoder, inp_lang, targ_lang, max_length_inp, max_length_targ)" ] }, { @@ -808,7 +808,7 @@ "outputs": [], "source": [ "# wrong translation\n", - "translate('trata de averiguarlo.', encoder, decoder, inp_lang, targ_lang, max_length_inp, max_length_targ)" + "translate(u'trata de averiguarlo.', encoder, decoder, inp_lang, targ_lang, max_length_inp, max_length_targ)" ] }, { diff --git a/tensorflow/contrib/eager/python/examples/resnet50/resnet50_graph_test.py b/tensorflow/contrib/eager/python/examples/resnet50/resnet50_graph_test.py index f3bb978875e..fb7975d8fe8 100644 --- a/tensorflow/contrib/eager/python/examples/resnet50/resnet50_graph_test.py +++ b/tensorflow/contrib/eager/python/examples/resnet50/resnet50_graph_test.py @@ -142,7 +142,8 @@ class ResNet50Benchmarks(tf.test.Benchmark): with tf.Graph().as_default(): np_images, np_labels = random_batch(batch_size) dataset = tf.data.Dataset.from_tensors((np_images, np_labels)).repeat() - (images, labels) = dataset.make_one_shot_iterator().get_next() + images, labels = tf.compat.v1.data.make_one_shot_iterator( + dataset).get_next() model = resnet50.ResNet50(data_format()) logits = model(images, training=True) diff --git a/tensorflow/contrib/eager/python/examples/revnet/main.py b/tensorflow/contrib/eager/python/examples/revnet/main.py index b702e91f922..9585f3565f8 100644 --- a/tensorflow/contrib/eager/python/examples/revnet/main.py +++ b/tensorflow/contrib/eager/python/examples/revnet/main.py @@ -72,14 +72,11 @@ def main(_): train_one_iter(model, x, y, optimizer, global_step=global_step) if global_step.numpy() % config.log_every == 0: - it_test = ds_test.make_one_shot_iterator() - acc_test, loss_test = evaluate(model, it_test) + acc_test, loss_test = evaluate(model, ds_test) if FLAGS.validate: - it_train = ds_train_one_shot.make_one_shot_iterator() - it_validation = ds_validation.make_one_shot_iterator() - acc_train, loss_train = evaluate(model, it_train) - acc_validation, loss_validation = evaluate(model, it_validation) + acc_train, loss_train = evaluate(model, ds_train_one_shot) + acc_validation, loss_validation = evaluate(model, ds_validation) print("Iter {}, " "training set accuracy {:.4f}, loss {:.4f}; " "validation set accuracy {:.4f}, loss {:.4f}; " @@ -218,11 +215,11 @@ def train_one_iter(model, inputs, labels, optimizer, global_step=None): return logits, loss -def evaluate(model, iterator): +def evaluate(model, dataset): """Compute accuracy with the given dataset iterator.""" mean_loss = tfe.metrics.Mean() accuracy = tfe.metrics.Accuracy() - for x, y in iterator: + for x, y in dataset: logits, _ = model(x, training=False) loss = model.compute_loss(logits=logits, labels=y) accuracy( diff --git a/tensorflow/contrib/eager/python/examples/rnn_ptb/rnn_ptb_graph_test.py b/tensorflow/contrib/eager/python/examples/rnn_ptb/rnn_ptb_graph_test.py index 63b5c4c54d1..770484abed9 100644 --- a/tensorflow/contrib/eager/python/examples/rnn_ptb/rnn_ptb_graph_test.py +++ b/tensorflow/contrib/eager/python/examples/rnn_ptb/rnn_ptb_graph_test.py @@ -82,7 +82,7 @@ class PTBBenchmark(tf.test.Benchmark): tf.ones( [PTBBenchmark.SEQ_LEN, PTBBenchmark.BATCH_SIZE], dtype=tf.int64)).repeat(num_iters + num_warmup) - inputs = dataset.make_one_shot_iterator().get_next() + inputs = tf.compat.v1.data.make_one_shot_iterator(dataset).get_next() with tf.device(tf.test.gpu_device_name()): outputs = model(inputs, training=True) @@ -124,7 +124,8 @@ class PTBBenchmark(tf.test.Benchmark): dtype=tf.int64)).repeat(num_iters + num_warmup) # inputs and labels have the same shape dataset = tf.data.Dataset.zip((dataset, dataset)) - (inputs, labels) = dataset.make_one_shot_iterator().get_next() + (inputs, labels) = tf.compat.v1.data.make_one_shot_iterator( + dataset).get_next() with tf.device(tf.test.gpu_device_name()): optimizer = tf.train.GradientDescentOptimizer(learning_rate=1.0) diff --git a/tensorflow/contrib/eager/python/metrics_impl.py b/tensorflow/contrib/eager/python/metrics_impl.py index c88c0f52eea..566246de495 100644 --- a/tensorflow/contrib/eager/python/metrics_impl.py +++ b/tensorflow/contrib/eager/python/metrics_impl.py @@ -24,6 +24,7 @@ from tensorflow.python.eager import context from tensorflow.python.eager import function from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops +from tensorflow.python.framework import smart_cond from tensorflow.python.ops import array_ops from tensorflow.python.ops import check_ops from tensorflow.python.ops import control_flow_ops @@ -354,9 +355,10 @@ class Mean(Metric): def write_summary_f(): summary_ops.scalar(name=self.name, tensor=t) return t - control_flow_ops.cond(write_summary, + smart_cond.smart_cond(write_summary, write_summary_f, - lambda: t) + lambda: t, + name="") return t diff --git a/tensorflow/contrib/eager/python/metrics_test.py b/tensorflow/contrib/eager/python/metrics_test.py index 9d2d172752c..39e5957f5d1 100644 --- a/tensorflow/contrib/eager/python/metrics_test.py +++ b/tensorflow/contrib/eager/python/metrics_test.py @@ -49,18 +49,6 @@ class MetricsTest(test.TestCase): self.assertEqual(dtypes.float64, m.dtype) self.assertEqual(dtypes.float64, m.result().dtype) - def testSummaryArg(self): - m = metrics.Mean() - m([1, 10, 100]) - m(1000) - m([10000.0, 100000.0]) - self.assertEqual(111111.0/6, m.result(write_summary=True).numpy()) - self.assertEqual(111111.0/6, m.result(write_summary=False).numpy()) - with self.assertRaises(ValueError): - m.result(write_summary=5) - with self.assertRaises(ValueError): - m.result(write_summary=[True]) - def testVariableCollections(self): with context.graph_mode(), ops.Graph().as_default(): m = metrics.Mean() diff --git a/tensorflow/contrib/eager/python/network.py b/tensorflow/contrib/eager/python/network.py index f801d9a47b2..5cc0c4f23d9 100644 --- a/tensorflow/contrib/eager/python/network.py +++ b/tensorflow/contrib/eager/python/network.py @@ -24,7 +24,7 @@ import weakref from tensorflow.python.eager import context from tensorflow.python.framework import ops -from tensorflow.python.keras.engine import base_layer as keras_base_layer +from tensorflow.python.keras.engine import base_layer_utils from tensorflow.python.layers import base from tensorflow.python.ops import variable_scope from tensorflow.python.platform import tf_logging as logging @@ -220,7 +220,7 @@ class Network(base.Layer): avoid_names = parent_network._owned_layers name_uid_map = parent_network._sub_layer_name_uids else: - name_uid_map = keras_base_layer.get_default_graph_uid_map() + name_uid_map = base_layer_utils.get_default_graph_uid_map() # Figure out which names we have to avoid based on which variable scope # we're nested in. strip_name = self._default_parent_variable_scope.name diff --git a/tensorflow/contrib/eager/python/saver.py b/tensorflow/contrib/eager/python/saver.py index f9c716360c5..1d0d6c6c14c 100644 --- a/tensorflow/contrib/eager/python/saver.py +++ b/tensorflow/contrib/eager/python/saver.py @@ -115,6 +115,11 @@ def restore_variables_on_create(save_path, map_func=None): class Saver(object): """A tf.train.Saver adapter for use when eager execution is enabled. + + `Saver`'s name-based checkpointing strategy is fragile. Please switch to + `tf.train.Checkpoint` or `tf.keras.Model.save_weights`, which perform a more + robust object-based saving. These APIs will load checkpoints written by + `Saver`. """ def __init__(self, var_list): diff --git a/tensorflow/contrib/eager/python/tfe_test.py b/tensorflow/contrib/eager/python/tfe_test.py index 4454abfb966..8c35dddb5a5 100644 --- a/tensorflow/contrib/eager/python/tfe_test.py +++ b/tensorflow/contrib/eager/python/tfe_test.py @@ -87,8 +87,8 @@ class TFETest(test_util.TensorFlowTestCase): x += 1. # Without a device context, heuristics are used to place ops. # In this case, ops.reduce_mean runs on the GPU. - reduction_indices = range(x.shape.ndims) - m = math_ops.reduce_mean(x, reduction_indices) + axis = range(x.shape.ndims) + m = math_ops.reduce_mean(x, axis) # m is on GPU, bring it back to CPU and compare. self.assertEqual(3.5, m.cpu().numpy()) diff --git a/tensorflow/contrib/estimator/BUILD b/tensorflow/contrib/estimator/BUILD index 37f253d9c11..a888379f13e 100644 --- a/tensorflow/contrib/estimator/BUILD +++ b/tensorflow/contrib/estimator/BUILD @@ -16,7 +16,6 @@ py_library( srcs_version = "PY2AND3", deps = [ ":boosted_trees", - ":dnn", ":dnn_with_layer_annotations", ":early_stopping", ":expect_tensorflow_estimator_installed", @@ -25,7 +24,6 @@ py_library( ":extenders", ":head", ":hooks", - ":linear", ":logit_fns", ":multi_head", ":replicate_model_fn", @@ -47,18 +45,6 @@ py_library( ], ) -py_library( - name = "dnn", - srcs = ["python/estimator/dnn.py"], - srcs_version = "PY2AND3", - deps = [ - ":expect_tensorflow_estimator_installed", - "//tensorflow:tensorflow_py_no_contrib", - "//tensorflow/python/estimator", - "//tensorflow/python/estimator:dnn", - ], -) - py_library( name = "dnn_with_layer_annotations", srcs = ["python/estimator/dnn_with_layer_annotations.py"], @@ -144,17 +130,6 @@ py_library( ], ) -py_library( - name = "linear", - srcs = ["python/estimator/linear.py"], - srcs_version = "PY2AND3", - deps = [ - ":expect_tensorflow_estimator_installed", - "//tensorflow/python/estimator", - "//tensorflow/python/estimator:linear", - ], -) - py_library( name = "logit_fns", srcs = [ diff --git a/tensorflow/contrib/estimator/__init__.py b/tensorflow/contrib/estimator/__init__.py index 80d59627620..7d61247e7ef 100644 --- a/tensorflow/contrib/estimator/__init__.py +++ b/tensorflow/contrib/estimator/__init__.py @@ -58,8 +58,6 @@ _allowed_symbols = [ 'multi_label_head', 'poisson_regression_head', 'regression_head', - 'DNNEstimator', - 'LinearEstimator', 'boosted_trees_classifier_train_in_memory', 'boosted_trees_regressor_train_in_memory', 'call_logit_fn', diff --git a/tensorflow/contrib/estimator/python/estimator/dnn_linear_combined.py b/tensorflow/contrib/estimator/python/estimator/dnn_linear_combined.py deleted file mode 100644 index 7894418c4a1..00000000000 --- a/tensorflow/contrib/estimator/python/estimator/dnn_linear_combined.py +++ /dev/null @@ -1,34 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""dnn_linear_combined python module. - -Importing from tensorflow.python.estimator is unsupported -and will soon break! -""" -# pylint: disable=unused-import,g-bad-import-order,g-import-not-at-top,wildcard-import - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow_estimator.contrib.estimator.python.estimator import dnn_linear_combined - -# Include attrs that start with single underscore. -_HAS_DYNAMIC_ATTRIBUTES = True -dnn_linear_combined.__all__ = [ - s for s in dir(dnn_linear_combined) if not s.startswith('__') -] - -from tensorflow_estimator.contrib.estimator.python.estimator.dnn_linear_combined import * diff --git a/tensorflow/contrib/factorization/python/ops/kmeans.py b/tensorflow/contrib/factorization/python/ops/kmeans.py index f384d761a84..3eb396a29cc 100644 --- a/tensorflow/contrib/factorization/python/ops/kmeans.py +++ b/tensorflow/contrib/factorization/python/ops/kmeans.py @@ -26,7 +26,7 @@ from tensorflow.contrib.factorization.python.ops import clustering_ops from tensorflow.python.estimator import estimator from tensorflow.python.estimator import model_fn as model_fn_lib from tensorflow.python.estimator.export import export_output -from tensorflow.python.feature_column import feature_column as fc +from tensorflow.python.feature_column import feature_column_lib as fc from tensorflow.python.framework import ops from tensorflow.python.ops import array_ops from tensorflow.python.ops import control_flow_ops diff --git a/tensorflow/contrib/factorization/python/ops/kmeans_test.py b/tensorflow/contrib/factorization/python/ops/kmeans_test.py index 1ab5418fe46..2f7cd131d3e 100644 --- a/tensorflow/contrib/factorization/python/ops/kmeans_test.py +++ b/tensorflow/contrib/factorization/python/ops/kmeans_test.py @@ -27,7 +27,7 @@ from sklearn.cluster import KMeans as SklearnKMeans # pylint: disable=g-import-not-at-top from tensorflow.contrib.factorization.python.ops import kmeans as kmeans_lib from tensorflow.python.estimator import run_config -from tensorflow.python.feature_column import feature_column as fc +from tensorflow.python.feature_column import feature_column_lib as fc from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops diff --git a/tensorflow/contrib/feature_column/BUILD b/tensorflow/contrib/feature_column/BUILD index bbe335be3e1..1cd83bdb5de 100644 --- a/tensorflow/contrib/feature_column/BUILD +++ b/tensorflow/contrib/feature_column/BUILD @@ -14,6 +14,7 @@ py_library( srcs_version = "PY2AND3", deps = [ ":sequence_feature_column", + ":sequence_feature_column_v2", "//tensorflow/python:util", ], ) @@ -32,7 +33,7 @@ py_library( "//tensorflow/python:sparse_ops", "//tensorflow/python:tensor_shape", "//tensorflow/python:variable_scope", - "//tensorflow/python/feature_column", + "//tensorflow/python/feature_column:feature_column_py", ], ) @@ -51,7 +52,7 @@ py_test( "//tensorflow/python:parsing_ops", "//tensorflow/python:sparse_tensor", "//tensorflow/python:training", - "//tensorflow/python/feature_column", + "//tensorflow/python/feature_column:feature_column_py", "//third_party/py/numpy", "@absl_py//absl/testing:parameterized", ], @@ -69,7 +70,7 @@ py_test( "//tensorflow/python:parsing_ops", "//tensorflow/python:training", "//tensorflow/python:util", - "//tensorflow/python/feature_column", + "//tensorflow/python/feature_column:feature_column_py", "//tensorflow/python/keras:layers", ], ) @@ -89,7 +90,7 @@ py_library( "//tensorflow/python:tensor_shape", "//tensorflow/python:variable_scope", "//tensorflow/python/feature_column", - "//tensorflow/python/feature_column:feature_column_v2", + "//tensorflow/python/feature_column:feature_column_py", ], ) @@ -110,7 +111,7 @@ py_test( "//tensorflow/python:sparse_tensor", "//tensorflow/python:training", "//tensorflow/python/feature_column", - "//tensorflow/python/feature_column:feature_column_v2", + "//tensorflow/python/feature_column:feature_column_py", "//third_party/py/numpy", "@absl_py//absl/testing:parameterized", ], diff --git a/tensorflow/contrib/feature_column/python/feature_column/sequence_feature_column.py b/tensorflow/contrib/feature_column/python/feature_column/sequence_feature_column.py index dd6da35ed00..9b3a5c58aaa 100644 --- a/tensorflow/contrib/feature_column/python/feature_column/sequence_feature_column.py +++ b/tensorflow/contrib/feature_column/python/feature_column/sequence_feature_column.py @@ -222,10 +222,8 @@ def sequence_categorical_column_with_identity( ValueError: if `default_value` is not in range `[0, num_buckets)`. """ return fc._SequenceCategoricalColumn( - fc.categorical_column_with_identity( - key=key, - num_buckets=num_buckets, - default_value=default_value)) + fc._categorical_column_with_identity( + key=key, num_buckets=num_buckets, default_value=default_value)) def sequence_categorical_column_with_hash_bucket( @@ -265,10 +263,8 @@ def sequence_categorical_column_with_hash_bucket( ValueError: `dtype` is neither string nor integer. """ return fc._SequenceCategoricalColumn( - fc.categorical_column_with_hash_bucket( - key=key, - hash_bucket_size=hash_bucket_size, - dtype=dtype)) + fc._categorical_column_with_hash_bucket( + key=key, hash_bucket_size=hash_bucket_size, dtype=dtype)) def sequence_categorical_column_with_vocabulary_file( @@ -324,7 +320,7 @@ def sequence_categorical_column_with_vocabulary_file( ValueError: `dtype` is neither string nor integer. """ return fc._SequenceCategoricalColumn( - fc.categorical_column_with_vocabulary_file( + fc._categorical_column_with_vocabulary_file( key=key, vocabulary_file=vocabulary_file, vocabulary_size=vocabulary_size, @@ -384,7 +380,7 @@ def sequence_categorical_column_with_vocabulary_list( ValueError: if `dtype` is not integer or string. """ return fc._SequenceCategoricalColumn( - fc.categorical_column_with_vocabulary_list( + fc._categorical_column_with_vocabulary_list( key=key, vocabulary_list=vocabulary_list, dtype=dtype, diff --git a/tensorflow/contrib/feature_column/python/feature_column/sequence_feature_column_integration_test.py b/tensorflow/contrib/feature_column/python/feature_column/sequence_feature_column_integration_test.py index d8ca363627e..bcc25b8de89 100644 --- a/tensorflow/contrib/feature_column/python/feature_column/sequence_feature_column_integration_test.py +++ b/tensorflow/contrib/feature_column/python/feature_column/sequence_feature_column_integration_test.py @@ -53,19 +53,20 @@ class SequenceFeatureColumnIntegrationTest(test.TestCase): return example def _build_feature_columns(self): - col = fc.categorical_column_with_identity( - 'int_ctx', num_buckets=100) + col = fc._categorical_column_with_identity('int_ctx', num_buckets=100) ctx_cols = [ - fc.embedding_column(col, dimension=10), - fc.numeric_column('float_ctx')] + fc._embedding_column(col, dimension=10), + fc._numeric_column('float_ctx') + ] identity_col = sfc.sequence_categorical_column_with_identity( 'int_list', num_buckets=10) bucket_col = sfc.sequence_categorical_column_with_hash_bucket( 'bytes_list', hash_bucket_size=100) seq_cols = [ - fc.embedding_column(identity_col, dimension=10), - fc.embedding_column(bucket_col, dimension=20)] + fc._embedding_column(identity_col, dimension=10), + fc._embedding_column(bucket_col, dimension=20) + ] return ctx_cols, seq_cols @@ -148,8 +149,8 @@ class SequenceExampleParsingTest(test.TestCase): """ example = _make_sequence_example() columns = [ - fc.categorical_column_with_identity('int_ctx', num_buckets=100), - fc.numeric_column('float_ctx'), + fc._categorical_column_with_identity('int_ctx', num_buckets=100), + fc._numeric_column('float_ctx'), col_fn(col_name, col_arg) ] context, seq_features = parsing_ops.parse_single_sequence_example( diff --git a/tensorflow/contrib/feature_column/python/feature_column/sequence_feature_column_test.py b/tensorflow/contrib/feature_column/python/feature_column/sequence_feature_column_test.py index 2163af0b438..d5f74028298 100644 --- a/tensorflow/contrib/feature_column/python/feature_column/sequence_feature_column_test.py +++ b/tensorflow/contrib/feature_column/python/feature_column/sequence_feature_column_test.py @@ -24,6 +24,7 @@ import numpy as np from tensorflow.contrib.feature_column.python.feature_column import sequence_feature_column as sfc from tensorflow.python.feature_column import feature_column as fc +from tensorflow.python.feature_column import feature_column_lib as fc_lib from tensorflow.python.feature_column.feature_column import _LazyBuilder from tensorflow.python.framework import dtypes from tensorflow.python.framework import errors @@ -109,13 +110,15 @@ class SequenceInputLayerTest(test.TestCase, parameterized.TestCase): categorical_column_a = sfc.sequence_categorical_column_with_identity( key='aaa', num_buckets=vocabulary_size) - embedding_column_a = fc.embedding_column( - categorical_column_a, dimension=embedding_dimension_a, + embedding_column_a = fc._embedding_column( + categorical_column_a, + dimension=embedding_dimension_a, initializer=_get_initializer(embedding_dimension_a, embedding_values_a)) categorical_column_b = sfc.sequence_categorical_column_with_identity( key='bbb', num_buckets=vocabulary_size) - embedding_column_b = fc.embedding_column( - categorical_column_b, dimension=embedding_dimension_b, + embedding_column_b = fc._embedding_column( + categorical_column_b, + dimension=embedding_dimension_b, initializer=_get_initializer(embedding_dimension_b, embedding_values_b)) input_layer, sequence_length = sfc.sequence_input_layer( @@ -148,10 +151,9 @@ class SequenceInputLayerTest(test.TestCase, parameterized.TestCase): values=(2, 0, 1), dense_shape=(2, 2)) - categorical_column_a = fc.categorical_column_with_identity( + categorical_column_a = fc._categorical_column_with_identity( key='aaa', num_buckets=vocabulary_size) - embedding_column_a = fc.embedding_column( - categorical_column_a, dimension=2) + embedding_column_a = fc._embedding_column(categorical_column_a, dimension=2) with self.assertRaisesRegexp( ValueError, @@ -206,7 +208,7 @@ class SequenceInputLayerTest(test.TestCase, parameterized.TestCase): categorical_column_b = sfc.sequence_categorical_column_with_identity( key='bbb', num_buckets=vocabulary_size) # Test that columns are reordered alphabetically. - shared_embedding_columns = fc.shared_embedding_columns( + shared_embedding_columns = fc_lib.shared_embedding_columns( [categorical_column_b, categorical_column_a], dimension=embedding_dimension, initializer=_get_initializer(embedding_dimension, embedding_values)) @@ -244,11 +246,11 @@ class SequenceInputLayerTest(test.TestCase, parameterized.TestCase): values=(2, 0, 1), dense_shape=(2, 2)) - categorical_column_a = fc.categorical_column_with_identity( + categorical_column_a = fc._categorical_column_with_identity( key='aaa', num_buckets=vocabulary_size) - categorical_column_b = fc.categorical_column_with_identity( + categorical_column_b = fc._categorical_column_with_identity( key='bbb', num_buckets=vocabulary_size) - shared_embedding_columns = fc.shared_embedding_columns( + shared_embedding_columns = fc_lib.shared_embedding_columns( [categorical_column_a, categorical_column_b], dimension=2) with self.assertRaisesRegexp( @@ -315,10 +317,10 @@ class SequenceInputLayerTest(test.TestCase, parameterized.TestCase): categorical_column_a = sfc.sequence_categorical_column_with_identity( key='aaa', num_buckets=vocabulary_size_a) - indicator_column_a = fc.indicator_column(categorical_column_a) + indicator_column_a = fc._indicator_column(categorical_column_a) categorical_column_b = sfc.sequence_categorical_column_with_identity( key='bbb', num_buckets=vocabulary_size_b) - indicator_column_b = fc.indicator_column(categorical_column_b) + indicator_column_b = fc._indicator_column(categorical_column_b) input_layer, sequence_length = sfc.sequence_input_layer( features={ 'aaa': sparse_input_a, @@ -342,9 +344,9 @@ class SequenceInputLayerTest(test.TestCase, parameterized.TestCase): values=(2, 0, 1), dense_shape=(2, 2)) - categorical_column_a = fc.categorical_column_with_identity( + categorical_column_a = fc._categorical_column_with_identity( key='aaa', num_buckets=vocabulary_size) - indicator_column_a = fc.indicator_column(categorical_column_a) + indicator_column_a = fc._indicator_column(categorical_column_a) with self.assertRaisesRegexp( ValueError, @@ -530,7 +532,7 @@ class SequenceInputLayerTest(test.TestCase, parameterized.TestCase): sparse_input = sparse_tensor.SparseTensorValue(**sparse_input_args) categorical_column = sfc.sequence_categorical_column_with_identity( key='aaa', num_buckets=3) - indicator_column = fc.indicator_column(categorical_column) + indicator_column = fc._indicator_column(categorical_column) input_layer, _ = sfc.sequence_input_layer( features={'aaa': sparse_input}, feature_columns=[indicator_column]) @@ -616,8 +618,7 @@ class InputLayerTest(test.TestCase): categorical_column_a = sfc.sequence_categorical_column_with_identity( key='aaa', num_buckets=vocabulary_size) - embedding_column_a = fc.embedding_column( - categorical_column_a, dimension=2) + embedding_column_a = fc._embedding_column(categorical_column_a, dimension=2) with self.assertRaisesRegexp( ValueError, @@ -639,7 +640,7 @@ class InputLayerTest(test.TestCase): categorical_column_a = sfc.sequence_categorical_column_with_identity( key='aaa', num_buckets=vocabulary_size) - indicator_column_a = fc.indicator_column(categorical_column_a) + indicator_column_a = fc._indicator_column(categorical_column_a) with self.assertRaisesRegexp( ValueError, @@ -918,8 +919,9 @@ class SequenceEmbeddingColumnTest( categorical_column = sfc.sequence_categorical_column_with_identity( key='aaa', num_buckets=vocabulary_size) - embedding_column = fc.embedding_column( - categorical_column, dimension=embedding_dimension, + embedding_column = fc._embedding_column( + categorical_column, + dimension=embedding_dimension, initializer=_initializer) embedding_lookup, _ = embedding_column._get_sequence_dense_tensor( @@ -956,8 +958,7 @@ class SequenceEmbeddingColumnTest( categorical_column = sfc.sequence_categorical_column_with_identity( key='aaa', num_buckets=vocabulary_size) - embedding_column = fc.embedding_column( - categorical_column, dimension=2) + embedding_column = fc._embedding_column(categorical_column, dimension=2) _, sequence_length = embedding_column._get_sequence_dense_tensor( _LazyBuilder({'aaa': inputs})) @@ -984,8 +985,7 @@ class SequenceEmbeddingColumnTest( categorical_column = sfc.sequence_categorical_column_with_identity( key='aaa', num_buckets=vocabulary_size) - embedding_column = fc.embedding_column( - categorical_column, dimension=2) + embedding_column = fc._embedding_column(categorical_column, dimension=2) _, sequence_length = embedding_column._get_sequence_dense_tensor( _LazyBuilder({'aaa': sparse_input})) @@ -1055,7 +1055,7 @@ class SequenceSharedEmbeddingColumnTest(test.TestCase): key='aaa', num_buckets=vocabulary_size) categorical_column_b = sfc.sequence_categorical_column_with_identity( key='bbb', num_buckets=vocabulary_size) - shared_embedding_columns = fc.shared_embedding_columns( + shared_embedding_columns = fc_lib.shared_embedding_columns( [categorical_column_a, categorical_column_b], dimension=embedding_dimension, initializer=_initializer) @@ -1101,7 +1101,7 @@ class SequenceSharedEmbeddingColumnTest(test.TestCase): expected_sequence_length_b = [2, 1] categorical_column_b = sfc.sequence_categorical_column_with_identity( key='bbb', num_buckets=vocabulary_size) - shared_embedding_columns = fc.shared_embedding_columns( + shared_embedding_columns = fc_lib.shared_embedding_columns( [categorical_column_a, categorical_column_b], dimension=2) sequence_length_a = shared_embedding_columns[0]._get_sequence_dense_tensor( @@ -1152,7 +1152,7 @@ class SequenceSharedEmbeddingColumnTest(test.TestCase): categorical_column_b = sfc.sequence_categorical_column_with_identity( key='bbb', num_buckets=vocabulary_size) - shared_embedding_columns = fc.shared_embedding_columns( + shared_embedding_columns = fc_lib.shared_embedding_columns( [categorical_column_a, categorical_column_b], dimension=2) sequence_length_a = shared_embedding_columns[0]._get_sequence_dense_tensor( @@ -1218,7 +1218,7 @@ class SequenceIndicatorColumnTest(test.TestCase, parameterized.TestCase): categorical_column = sfc.sequence_categorical_column_with_identity( key='aaa', num_buckets=vocabulary_size) - indicator_column = fc.indicator_column(categorical_column) + indicator_column = fc._indicator_column(categorical_column) indicator_tensor, _ = indicator_column._get_sequence_dense_tensor( _LazyBuilder({'aaa': inputs})) @@ -1250,7 +1250,7 @@ class SequenceIndicatorColumnTest(test.TestCase, parameterized.TestCase): categorical_column = sfc.sequence_categorical_column_with_identity( key='aaa', num_buckets=vocabulary_size) - indicator_column = fc.indicator_column(categorical_column) + indicator_column = fc._indicator_column(categorical_column) _, sequence_length = indicator_column._get_sequence_dense_tensor( _LazyBuilder({'aaa': inputs})) @@ -1277,7 +1277,7 @@ class SequenceIndicatorColumnTest(test.TestCase, parameterized.TestCase): categorical_column = sfc.sequence_categorical_column_with_identity( key='aaa', num_buckets=vocabulary_size) - indicator_column = fc.indicator_column(categorical_column) + indicator_column = fc._indicator_column(categorical_column) _, sequence_length = indicator_column._get_sequence_dense_tensor( _LazyBuilder({'aaa': sparse_input})) diff --git a/tensorflow/contrib/feature_column/python/feature_column/sequence_feature_column_v2.py b/tensorflow/contrib/feature_column/python/feature_column/sequence_feature_column_v2.py index 67ffb939663..0d34ad16185 100644 --- a/tensorflow/contrib/feature_column/python/feature_column/sequence_feature_column_v2.py +++ b/tensorflow/contrib/feature_column/python/feature_column/sequence_feature_column_v2.py @@ -26,7 +26,7 @@ import collections from tensorflow.python.feature_column import feature_column as fc_old -from tensorflow.python.feature_column import feature_column_v2 as fc +from tensorflow.python.feature_column import feature_column_lib as fc from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops from tensorflow.python.framework import tensor_shape @@ -226,10 +226,8 @@ def sequence_categorical_column_with_identity( ValueError: if `default_value` is not in range `[0, num_buckets)`. """ return fc_old._SequenceCategoricalColumn( - fc_old.categorical_column_with_identity( - key=key, - num_buckets=num_buckets, - default_value=default_value)) + fc_old._categorical_column_with_identity( + key=key, num_buckets=num_buckets, default_value=default_value)) def sequence_categorical_column_with_hash_bucket( @@ -269,10 +267,8 @@ def sequence_categorical_column_with_hash_bucket( ValueError: `dtype` is neither string nor integer. """ return fc_old._SequenceCategoricalColumn( - fc_old.categorical_column_with_hash_bucket( - key=key, - hash_bucket_size=hash_bucket_size, - dtype=dtype)) + fc_old._categorical_column_with_hash_bucket( + key=key, hash_bucket_size=hash_bucket_size, dtype=dtype)) def sequence_categorical_column_with_vocabulary_file( @@ -328,7 +324,7 @@ def sequence_categorical_column_with_vocabulary_file( ValueError: `dtype` is neither string nor integer. """ return fc_old._SequenceCategoricalColumn( - fc_old.categorical_column_with_vocabulary_file( + fc_old._categorical_column_with_vocabulary_file( key=key, vocabulary_file=vocabulary_file, vocabulary_size=vocabulary_size, @@ -388,7 +384,7 @@ def sequence_categorical_column_with_vocabulary_list( ValueError: if `dtype` is not integer or string. """ return fc_old._SequenceCategoricalColumn( - fc_old.categorical_column_with_vocabulary_list( + fc_old._categorical_column_with_vocabulary_list( key=key, vocabulary_list=vocabulary_list, dtype=dtype, @@ -441,7 +437,7 @@ def sequence_numeric_column( ValueError: if any dimension in shape is not a positive integer. ValueError: if `dtype` is not convertible to `tf.float32`. """ - shape = fc._check_shape(shape=shape, key=key) + shape = fc_old._check_shape(shape=shape, key=key) if not (dtype.is_integer or dtype.is_floating): raise ValueError('dtype must be convertible to float. ' 'dtype: {}, key: {}'.format(dtype, key)) diff --git a/tensorflow/contrib/feature_column/python/feature_column/sequence_feature_column_v2_test.py b/tensorflow/contrib/feature_column/python/feature_column/sequence_feature_column_v2_test.py index 5ecd85807c5..ca4398a1420 100644 --- a/tensorflow/contrib/feature_column/python/feature_column/sequence_feature_column_v2_test.py +++ b/tensorflow/contrib/feature_column/python/feature_column/sequence_feature_column_v2_test.py @@ -25,7 +25,7 @@ import numpy as np from tensorflow.contrib.feature_column.python.feature_column import sequence_feature_column as sfc_old from tensorflow.contrib.feature_column.python.feature_column import sequence_feature_column_v2 as sfc from tensorflow.python.feature_column import feature_column as fc_old -from tensorflow.python.feature_column import feature_column_v2 as fc +from tensorflow.python.feature_column import feature_column_lib as fc from tensorflow.python.feature_column.feature_column import _LazyBuilder from tensorflow.python.framework import dtypes from tensorflow.python.framework import errors @@ -111,13 +111,15 @@ class SequenceInputLayerTest(test.TestCase, parameterized.TestCase): categorical_column_a = sfc.sequence_categorical_column_with_identity( key='aaa', num_buckets=vocabulary_size) - embedding_column_a = fc_old.embedding_column( - categorical_column_a, dimension=embedding_dimension_a, + embedding_column_a = fc_old._embedding_column( + categorical_column_a, + dimension=embedding_dimension_a, initializer=_get_initializer(embedding_dimension_a, embedding_values_a)) categorical_column_b = sfc.sequence_categorical_column_with_identity( key='bbb', num_buckets=vocabulary_size) - embedding_column_b = fc_old.embedding_column( - categorical_column_b, dimension=embedding_dimension_b, + embedding_column_b = fc_old._embedding_column( + categorical_column_b, + dimension=embedding_dimension_b, initializer=_get_initializer(embedding_dimension_b, embedding_values_b)) input_layer, sequence_length = sfc.sequence_input_layer( @@ -150,9 +152,9 @@ class SequenceInputLayerTest(test.TestCase, parameterized.TestCase): values=(2, 0, 1), dense_shape=(2, 2)) - categorical_column_a = fc_old.categorical_column_with_identity( + categorical_column_a = fc_old._categorical_column_with_identity( key='aaa', num_buckets=vocabulary_size) - embedding_column_a = fc_old.embedding_column( + embedding_column_a = fc_old._embedding_column( categorical_column_a, dimension=2) with self.assertRaisesRegexp( @@ -208,7 +210,7 @@ class SequenceInputLayerTest(test.TestCase, parameterized.TestCase): categorical_column_b = sfc.sequence_categorical_column_with_identity( key='bbb', num_buckets=vocabulary_size) # Test that columns are reordered alphabetically. - shared_embedding_columns = fc_old.shared_embedding_columns( + shared_embedding_columns = fc.shared_embedding_columns( [categorical_column_b, categorical_column_a], dimension=embedding_dimension, initializer=_get_initializer(embedding_dimension, embedding_values)) @@ -246,11 +248,11 @@ class SequenceInputLayerTest(test.TestCase, parameterized.TestCase): values=(2, 0, 1), dense_shape=(2, 2)) - categorical_column_a = fc_old.categorical_column_with_identity( + categorical_column_a = fc_old._categorical_column_with_identity( key='aaa', num_buckets=vocabulary_size) - categorical_column_b = fc_old.categorical_column_with_identity( + categorical_column_b = fc_old._categorical_column_with_identity( key='bbb', num_buckets=vocabulary_size) - shared_embedding_columns = fc_old.shared_embedding_columns( + shared_embedding_columns = fc.shared_embedding_columns( [categorical_column_a, categorical_column_b], dimension=2) with self.assertRaisesRegexp( @@ -317,10 +319,10 @@ class SequenceInputLayerTest(test.TestCase, parameterized.TestCase): categorical_column_a = sfc.sequence_categorical_column_with_identity( key='aaa', num_buckets=vocabulary_size_a) - indicator_column_a = fc_old.indicator_column(categorical_column_a) + indicator_column_a = fc_old._indicator_column(categorical_column_a) categorical_column_b = sfc.sequence_categorical_column_with_identity( key='bbb', num_buckets=vocabulary_size_b) - indicator_column_b = fc_old.indicator_column(categorical_column_b) + indicator_column_b = fc_old._indicator_column(categorical_column_b) input_layer, sequence_length = sfc.sequence_input_layer( features={ 'aaa': sparse_input_a, @@ -344,9 +346,9 @@ class SequenceInputLayerTest(test.TestCase, parameterized.TestCase): values=(2, 0, 1), dense_shape=(2, 2)) - categorical_column_a = fc_old.categorical_column_with_identity( + categorical_column_a = fc_old._categorical_column_with_identity( key='aaa', num_buckets=vocabulary_size) - indicator_column_a = fc_old.indicator_column(categorical_column_a) + indicator_column_a = fc_old._indicator_column(categorical_column_a) with self.assertRaisesRegexp( ValueError, @@ -532,7 +534,7 @@ class SequenceInputLayerTest(test.TestCase, parameterized.TestCase): sparse_input = sparse_tensor.SparseTensorValue(**sparse_input_args) categorical_column = sfc.sequence_categorical_column_with_identity( key='aaa', num_buckets=3) - indicator_column = fc_old.indicator_column(categorical_column) + indicator_column = fc_old._indicator_column(categorical_column) input_layer, _ = sfc.sequence_input_layer( features={'aaa': sparse_input}, feature_columns=[indicator_column]) @@ -618,7 +620,7 @@ class InputLayerTest(test.TestCase): categorical_column_a = sfc.sequence_categorical_column_with_identity( key='aaa', num_buckets=vocabulary_size) - embedding_column_a = fc_old.embedding_column( + embedding_column_a = fc_old._embedding_column( categorical_column_a, dimension=2) with self.assertRaisesRegexp( @@ -641,7 +643,7 @@ class InputLayerTest(test.TestCase): categorical_column_a = sfc.sequence_categorical_column_with_identity( key='aaa', num_buckets=vocabulary_size) - indicator_column_a = fc_old.indicator_column(categorical_column_a) + indicator_column_a = fc_old._indicator_column(categorical_column_a) with self.assertRaisesRegexp( ValueError, @@ -920,8 +922,9 @@ class SequenceEmbeddingColumnTest( categorical_column = sfc.sequence_categorical_column_with_identity( key='aaa', num_buckets=vocabulary_size) - embedding_column = fc_old.embedding_column( - categorical_column, dimension=embedding_dimension, + embedding_column = fc_old._embedding_column( + categorical_column, + dimension=embedding_dimension, initializer=_initializer) embedding_lookup, _ = embedding_column._get_sequence_dense_tensor( @@ -958,8 +961,7 @@ class SequenceEmbeddingColumnTest( categorical_column = sfc.sequence_categorical_column_with_identity( key='aaa', num_buckets=vocabulary_size) - embedding_column = fc_old.embedding_column( - categorical_column, dimension=2) + embedding_column = fc_old._embedding_column(categorical_column, dimension=2) _, sequence_length = embedding_column._get_sequence_dense_tensor( _LazyBuilder({'aaa': inputs})) @@ -986,8 +988,7 @@ class SequenceEmbeddingColumnTest( categorical_column = sfc.sequence_categorical_column_with_identity( key='aaa', num_buckets=vocabulary_size) - embedding_column = fc_old.embedding_column( - categorical_column, dimension=2) + embedding_column = fc_old._embedding_column(categorical_column, dimension=2) _, sequence_length = embedding_column._get_sequence_dense_tensor( _LazyBuilder({'aaa': sparse_input})) @@ -1057,7 +1058,7 @@ class SequenceSharedEmbeddingColumnTest(test.TestCase): key='aaa', num_buckets=vocabulary_size) categorical_column_b = sfc.sequence_categorical_column_with_identity( key='bbb', num_buckets=vocabulary_size) - shared_embedding_columns = fc_old.shared_embedding_columns( + shared_embedding_columns = fc.shared_embedding_columns( [categorical_column_a, categorical_column_b], dimension=embedding_dimension, initializer=_initializer) @@ -1103,7 +1104,7 @@ class SequenceSharedEmbeddingColumnTest(test.TestCase): expected_sequence_length_b = [2, 1] categorical_column_b = sfc.sequence_categorical_column_with_identity( key='bbb', num_buckets=vocabulary_size) - shared_embedding_columns = fc_old.shared_embedding_columns( + shared_embedding_columns = fc.shared_embedding_columns( [categorical_column_a, categorical_column_b], dimension=2) sequence_length_a = shared_embedding_columns[0]._get_sequence_dense_tensor( @@ -1154,7 +1155,7 @@ class SequenceSharedEmbeddingColumnTest(test.TestCase): categorical_column_b = sfc.sequence_categorical_column_with_identity( key='bbb', num_buckets=vocabulary_size) - shared_embedding_columns = fc_old.shared_embedding_columns( + shared_embedding_columns = fc.shared_embedding_columns( [categorical_column_a, categorical_column_b], dimension=2) sequence_length_a = shared_embedding_columns[0]._get_sequence_dense_tensor( @@ -1220,7 +1221,7 @@ class SequenceIndicatorColumnTest(test.TestCase, parameterized.TestCase): categorical_column = sfc.sequence_categorical_column_with_identity( key='aaa', num_buckets=vocabulary_size) - indicator_column = fc_old.indicator_column(categorical_column) + indicator_column = fc_old._indicator_column(categorical_column) indicator_tensor, _ = indicator_column._get_sequence_dense_tensor( _LazyBuilder({'aaa': inputs})) @@ -1252,7 +1253,7 @@ class SequenceIndicatorColumnTest(test.TestCase, parameterized.TestCase): categorical_column = sfc.sequence_categorical_column_with_identity( key='aaa', num_buckets=vocabulary_size) - indicator_column = fc_old.indicator_column(categorical_column) + indicator_column = fc_old._indicator_column(categorical_column) _, sequence_length = indicator_column._get_sequence_dense_tensor( _LazyBuilder({'aaa': inputs})) diff --git a/tensorflow/contrib/framework/BUILD b/tensorflow/contrib/framework/BUILD index cd747df4d69..dad50a3a730 100644 --- a/tensorflow/contrib/framework/BUILD +++ b/tensorflow/contrib/framework/BUILD @@ -47,6 +47,11 @@ tf_custom_op_py_library( ":variable_ops_op_lib", ], srcs_version = "PY2AND3", + visibility = [ + "//learning/brain:__subpackages__", + "//tensorflow:__subpackages__", + "//video/youtube/personalization:__subpackages__", + ], deps = [ ":gen_variable_ops", "//tensorflow/contrib/util:util_py", @@ -66,6 +71,7 @@ tf_custom_op_py_library( "//tensorflow/python:resource_variable_ops", "//tensorflow/python:script_ops", "//tensorflow/python:smart_cond", + "//tensorflow/python:sort_ops", "//tensorflow/python:sparse_tensor", "//tensorflow/python:state_ops", "//tensorflow/python:state_ops_gen", @@ -311,17 +317,3 @@ py_test( "//third_party/py/numpy", ], ) - -py_test( - name = "sort_ops_test", - size = "medium", - srcs = ["python/ops/sort_ops_test.py"], - srcs_version = "PY2AND3", - deps = [ - ":framework_py", - "//tensorflow/python:array_ops", - "//tensorflow/python:client_testlib", - "//tensorflow/python:random_ops", - "//third_party/py/numpy", - ], -) diff --git a/tensorflow/contrib/framework/python/ops/sort_ops.py b/tensorflow/contrib/framework/python/ops/sort_ops.py index 1921a77c1e9..42184a4e55e 100644 --- a/tensorflow/contrib/framework/python/ops/sort_ops.py +++ b/tensorflow/contrib/framework/python/ops/sort_ops.py @@ -22,173 +22,7 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function -import numpy as np +from tensorflow.python.ops import sort_ops -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import ops as framework_ops -from tensorflow.python.framework import tensor_util -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import nn_ops - - -def sort(values, axis=-1, direction='ASCENDING', name=None): - """Sorts a tensor. - - Args: - values: 1-D or higher numeric `Tensor`. - axis: The axis along which to sort. The default is -1, which sorts the last - axis. - direction: The direction in which to sort the values (`'ASCENDING'` or - `'DESCENDING'`). - name: Optional name for the operation. - - Returns: - A `Tensor` with the same dtype and shape as `values`, with the elements - sorted along the given `axis`. - - Raises: - ValueError: If axis is not a constant scalar, or the direction is invalid. - """ - with framework_ops.name_scope(name, 'sort'): - return _sort_or_argsort(values, axis, direction, return_argsort=False) - - -def argsort(values, axis=-1, direction='ASCENDING', stable=False, name=None): - """Returns the indices of a tensor that give its sorted order along an axis. - - For a 1D tensor, `tf.gather(values, tf.argsort(values))` is equivalent to - `tf.sort(values)`. For higher dimensions, the output has the same shape as - `values`, but along the given axis, values represent the index of the sorted - element in that slice of the tensor at the given position. - - Args: - values: 1-D or higher numeric `Tensor`. - axis: The axis along which to sort. The default is -1, which sorts the last - axis. - direction: The direction in which to sort the values (`'ASCENDING'` or - `'DESCENDING'`). - stable: If True, equal elements in the original tensor will not be - re-ordered in the returned order. Unstable sort is not yet implemented, - but will eventually be the default for performance reasons. If you - require a stable order, pass `stable=True` for forwards compatibility. - name: Optional name for the operation. - - Returns: - An int32 `Tensor` with the same shape as `values`. The indices that would - sort each slice of the given `values` along the given `axis`. - - Raises: - ValueError: If axis is not a constant scalar, or the direction is invalid. - """ - del stable # Unused. - with framework_ops.name_scope(name, 'argsort'): - return _sort_or_argsort(values, axis, direction, return_argsort=True) - - -def _sort_or_argsort(values, axis, direction, return_argsort): - """Internal sort/argsort implementation. - - Args: - values: The input values. - axis: The axis along which to sort. - direction: 'ASCENDING' or 'DESCENDING'. - return_argsort: Whether to return the argsort result. - - Returns: - Either the sorted values, or the indices of the sorted values in the - original tensor. See the `sort` and `argsort` docstrings. - - Raises: - ValueError: If axis is not a constant scalar, or the direction is invalid. - """ - if direction not in _SORT_IMPL: - raise ValueError('%s should be one of %s' % - (direction, ', '.join(sorted(_SORT_IMPL.keys())))) - # Axis must be an integer, not a Tensor. - axis = framework_ops.convert_to_tensor(axis, name='axis') - axis_static = tensor_util.constant_value(axis) - if axis.shape.ndims != 0 or axis_static is None: - raise ValueError('axis must be a constant scalar') - axis_static = int(axis_static) # Avoids NumPy casting error - - values = framework_ops.convert_to_tensor(values, name='values') - - return _SORT_IMPL[direction](values, axis_static, return_argsort) - - -def _descending_sort(values, axis, return_argsort=False): - """Sorts values in reverse using `top_k`. - - Args: - values: Tensor of numeric values. - axis: Index of the axis which values should be sorted along. - return_argsort: If False, return the sorted values. If True, return the - indices that would sort the values. - - Returns: - The sorted values. - """ - k = array_ops.shape(values)[axis] - rank = array_ops.rank(values) - static_rank = values.shape.ndims - # Fast path: sorting the last axis. - if axis == -1 or axis + 1 == values.get_shape().ndims: - top_k_input = values - transposition = None - else: - # Otherwise, transpose the array. Swap axes `axis` and `rank - 1`. - if axis < 0: - # Calculate the actual axis index if counting from the end. Use the static - # rank if available, or else make the axis back into a tensor. - axis += static_rank or rank - if static_rank is not None: - # Prefer to calculate the transposition array in NumPy and make it a - # constant. - transposition = constant_op.constant( - np.r_[ - # Axes up to axis are unchanged. - np.arange(axis), - # Swap axis and rank - 1. - [static_rank - 1], - # Axes in [axis + 1, rank - 1) are unchanged. - np.arange(axis + 1, static_rank - 1), - # Swap axis and rank - 1. - [axis]], - name='transposition') - else: - # Generate the transposition array from the tensors. - transposition = array_ops.concat( - [ - # Axes up to axis are unchanged. - math_ops.range(axis), - # Swap axis and rank - 1. - [rank - 1], - # Axes in [axis + 1, rank - 1) are unchanged. - math_ops.range(axis + 1, rank - 1), - # Swap axis and rank - 1. - [axis] - ], - axis=0) - top_k_input = array_ops.transpose(values, transposition) - - values, indices = nn_ops.top_k(top_k_input, k) - return_value = indices if return_argsort else values - if transposition is not None: - # transposition contains a single cycle of length 2 (swapping 2 elements), - # so it is an involution (it is its own inverse). - return_value = array_ops.transpose(return_value, transposition) - return return_value - - -def _ascending_sort(values, axis, return_argsort=False): - # Negate the values to get the ascending order from descending sort. - values_or_indices = _descending_sort(-values, axis, return_argsort) - # If not argsort, negate the values again. - return values_or_indices if return_argsort else -values_or_indices - - -_SORT_IMPL = { - 'ASCENDING': _ascending_sort, - 'DESCENDING': _descending_sort, -} +sort = sort_ops.sort +argsort = sort_ops.argsort diff --git a/tensorflow/contrib/gan/python/estimator/python/gan_estimator_impl.py b/tensorflow/contrib/gan/python/estimator/python/gan_estimator_impl.py index 219cc199d79..3593b501bb7 100644 --- a/tensorflow/contrib/gan/python/estimator/python/gan_estimator_impl.py +++ b/tensorflow/contrib/gan/python/estimator/python/gan_estimator_impl.py @@ -113,7 +113,8 @@ class GANEstimator(estimator.Estimator): add_summaries=None, use_loss_summaries=True, config=None, - warm_start_from=None): + warm_start_from=None, + is_chief=True): """Initializes a GANEstimator instance. Args: @@ -154,6 +155,8 @@ class GANEstimator(estimator.Estimator): config: `RunConfig` object to configure the runtime settings. warm_start_from: A filepath to a checkpoint or saved model, or a WarmStartSettings object to configure initialization. + is_chief: Whether or not this Estimator is running on a chief or worker. + Needs to be set appropriately if using SyncReplicasOptimizers. Raises: ValueError: If loss functions aren't callable. @@ -187,7 +190,7 @@ class GANEstimator(estimator.Estimator): return _get_estimator_spec( mode, gan_model, generator_loss_fn, discriminator_loss_fn, get_eval_metric_ops_fn, generator_optimizer, discriminator_optimizer, - get_hooks_fn, use_loss_summaries) + get_hooks_fn, use_loss_summaries, is_chief) super(GANEstimator, self).__init__( model_fn=_model_fn, model_dir=model_dir, config=config, @@ -215,7 +218,7 @@ def _get_gan_model( def _get_estimator_spec( mode, gan_model, generator_loss_fn, discriminator_loss_fn, get_eval_metric_ops_fn, generator_optimizer, discriminator_optimizer, - get_hooks_fn=None, use_loss_summaries=True): + get_hooks_fn=None, use_loss_summaries=True, is_chief=True): """Get the EstimatorSpec for the current mode.""" if mode == model_fn_lib.ModeKeys.PREDICT: estimator_spec = model_fn_lib.EstimatorSpec( @@ -236,7 +239,7 @@ def _get_estimator_spec( else discriminator_optimizer) get_hooks_fn = get_hooks_fn or tfgan_train.get_sequential_train_hooks() estimator_spec = _get_train_estimator_spec( - gan_model, gan_loss, gopt, dopt, get_hooks_fn) + gan_model, gan_loss, gopt, dopt, get_hooks_fn, is_chief=is_chief) return estimator_spec @@ -321,11 +324,11 @@ def _get_eval_estimator_spec(gan_model, gan_loss, get_eval_metric_ops_fn=None, def _get_train_estimator_spec( gan_model, gan_loss, generator_optimizer, discriminator_optimizer, - get_hooks_fn, train_op_fn=tfgan_train.gan_train_ops): + get_hooks_fn, train_op_fn=tfgan_train.gan_train_ops, is_chief=True): """Return an EstimatorSpec for the train case.""" scalar_loss = gan_loss.generator_loss + gan_loss.discriminator_loss train_ops = train_op_fn(gan_model, gan_loss, generator_optimizer, - discriminator_optimizer) + discriminator_optimizer, is_chief=is_chief) training_hooks = get_hooks_fn(train_ops) return model_fn_lib.EstimatorSpec( loss=scalar_loss, diff --git a/tensorflow/contrib/gan/python/estimator/python/gan_estimator_test.py b/tensorflow/contrib/gan/python/estimator/python/gan_estimator_test.py index 3d6bdab0ad7..bc9021050bc 100644 --- a/tensorflow/contrib/gan/python/estimator/python/gan_estimator_test.py +++ b/tensorflow/contrib/gan/python/estimator/python/gan_estimator_test.py @@ -48,6 +48,7 @@ from tensorflow.python.platform import test from tensorflow.python.summary.writer import writer_cache from tensorflow.python.training import input as input_lib from tensorflow.python.training import learning_rate_decay +from tensorflow.python.training import sync_replicas_optimizer from tensorflow.python.training import training from tensorflow.python.training import training_util @@ -82,7 +83,7 @@ class GetGANModelTest(test.TestCase, parameterized.TestCase): self.assertEqual(generator_inputs, gan_model.generator_inputs) self.assertIsNotNone(gan_model.generated_data) - self.assertEqual(2, len(gan_model.generator_variables)) # 1 FC layer + self.assertLen(gan_model.generator_variables, 2) # 1 FC layer self.assertIsNotNone(gan_model.generator_fn) if mode == model_fn_lib.ModeKeys.PREDICT: self.assertIsNone(gan_model.real_data) @@ -95,7 +96,7 @@ class GetGANModelTest(test.TestCase, parameterized.TestCase): self.assertIsNotNone(gan_model.real_data) self.assertIsNotNone(gan_model.discriminator_real_outputs) self.assertIsNotNone(gan_model.discriminator_gen_outputs) - self.assertEqual(2, len(gan_model.discriminator_variables)) # 1 FC layer + self.assertLen(gan_model.discriminator_variables, 2) # 1 FC layer self.assertIsNotNone(gan_model.discriminator_scope) self.assertIsNotNone(gan_model.discriminator_fn) @@ -121,6 +122,7 @@ def get_dummy_gan_model(): def dummy_loss_fn(gan_model, add_summaries=True): + del add_summaries return math_ops.reduce_sum(gan_model.discriminator_real_outputs - gan_model.discriminator_gen_outputs) @@ -168,6 +170,35 @@ class GetEstimatorSpecTest(test.TestCase, parameterized.TestCase): self.assertShapeEqual(np.array(0), spec.loss) # must be a scalar self.assertIsNotNone(spec.eval_metric_ops) + def test_get_sync_estimator_spec(self): + """Make sure spec is loaded with sync hooks for sync opts.""" + + def get_sync_optimizer(): + return sync_replicas_optimizer.SyncReplicasOptimizer( + training.GradientDescentOptimizer(learning_rate=1.0), + replicas_to_aggregate=1) + + with ops.Graph().as_default(): + self._gan_model = get_dummy_gan_model() + g_opt = get_sync_optimizer() + d_opt = get_sync_optimizer() + + spec = estimator._get_estimator_spec( + model_fn_lib.ModeKeys.TRAIN, + self._gan_model, + generator_loss_fn=dummy_loss_fn, + discriminator_loss_fn=dummy_loss_fn, + get_eval_metric_ops_fn=get_metrics, + generator_optimizer=g_opt, + discriminator_optimizer=d_opt) + + self.assertLen(spec.training_hooks, 4) + sync_opts = [ + hook._sync_optimizer for hook in spec.training_hooks if + isinstance(hook, sync_replicas_optimizer._SyncReplicasOptimizerHook)] + self.assertLen(sync_opts, 2) + self.assertSetEqual(frozenset(sync_opts), frozenset((g_opt, d_opt))) + # TODO(joelshor): Add pandas test. class GANEstimatorIntegrationTest(test.TestCase): diff --git a/tensorflow/contrib/gan/python/losses/python/losses_impl.py b/tensorflow/contrib/gan/python/losses/python/losses_impl.py index df0342c80c5..a0a86c6337e 100644 --- a/tensorflow/contrib/gan/python/losses/python/losses_impl.py +++ b/tensorflow/contrib/gan/python/losses/python/losses_impl.py @@ -36,7 +36,6 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function -import numpy as np from tensorflow.contrib.framework.python.ops import variables as contrib_variables_lib from tensorflow.python.framework import ops @@ -47,7 +46,6 @@ from tensorflow.python.ops import gradients_impl from tensorflow.python.ops import math_ops from tensorflow.python.ops import random_ops from tensorflow.python.ops import variable_scope -from tensorflow.python.ops.distributions import distribution as ds from tensorflow.python.ops.losses import losses from tensorflow.python.ops.losses import util from tensorflow.python.summary import summary @@ -740,11 +738,16 @@ def least_squares_discriminator_loss( def _validate_distributions(distributions): if not isinstance(distributions, (list, tuple)): raise ValueError('`distributions` must be a list or tuple. Instead, ' - 'found %s.', type(distributions)) + 'found %s.' % type(distributions)) for x in distributions: - if not isinstance(x, ds.Distribution): + # We used to check with `isinstance(x, tf.distributions.Distribution)`. + # However, distributions have migrated to `tfp.distributions.Distribution`, + # which is a new code repo, so we can't check this way anymore until + # TF-GAN is migrated to a new repo as well. + # This new check is not sufficient, but is a useful heuristic for now. + if not callable(getattr(x, 'log_prob', None)): raise ValueError('`distributions` must be a list of `Distributions`. ' - 'Instead, found %s.', type(x)) + 'Instead, found %s.' % type(x)) def _validate_information_penalty_inputs( @@ -817,7 +820,7 @@ def _numerically_stable_global_norm(tensor_list): Returns: A scalar tensor with the global norm. """ - if np.all([x is None for x in tensor_list]): + if all(x is None for x in tensor_list): return 0.0 list_max = math_ops.reduce_max([math_ops.reduce_max(math_ops.abs(x)) for x in diff --git a/tensorflow/contrib/gan/python/namedtuples.py b/tensorflow/contrib/gan/python/namedtuples.py index b9ac1bf1513..969b68449d9 100644 --- a/tensorflow/contrib/gan/python/namedtuples.py +++ b/tensorflow/contrib/gan/python/namedtuples.py @@ -213,7 +213,8 @@ class GANTrainOps( collections.namedtuple('GANTrainOps', ( 'generator_train_op', 'discriminator_train_op', - 'global_step_inc_op' + 'global_step_inc_op', + 'train_hooks' ))): """GANTrainOps contains the training ops. @@ -221,8 +222,17 @@ class GANTrainOps( generator_train_op: Op that performs a generator update step. discriminator_train_op: Op that performs a discriminator update step. global_step_inc_op: Op that increments the shared global step. + train_hooks: a list or tuple containing hooks related to training that need + to be populated when training ops are instantiated. Used primarily for + sync hooks. """ + def __new__(cls, generator_train_op, discriminator_train_op, + global_step_inc_op, train_hooks=()): + return super(GANTrainOps, cls).__new__(cls, generator_train_op, + discriminator_train_op, + global_step_inc_op, train_hooks) + class GANTrainSteps( collections.namedtuple('GANTrainSteps', ( diff --git a/tensorflow/contrib/gan/python/train.py b/tensorflow/contrib/gan/python/train.py index 7ee39f304ab..4c7bee41b33 100644 --- a/tensorflow/contrib/gan/python/train.py +++ b/tensorflow/contrib/gan/python/train.py @@ -114,7 +114,7 @@ def gan_model( discriminator_gen_outputs = discriminator_fn(generated_data, generator_inputs) with variable_scope.variable_scope(dis_scope, reuse=True): - real_data = ops.convert_to_tensor(real_data) + real_data = _convert_tensor_or_l_or_d(real_data) discriminator_real_outputs = discriminator_fn(real_data, generator_inputs) if check_shapes: @@ -924,6 +924,7 @@ def gan_train_ops( generator_optimizer, discriminator_optimizer, check_for_unused_update_ops=True, + is_chief=True, # Optional args to pass directly to the `create_train_op`. **kwargs): """Returns GAN train ops. @@ -939,6 +940,8 @@ def gan_train_ops( discriminator_optimizer: The optimizer for the discriminator updates. check_for_unused_update_ops: If `True`, throws an exception if there are update ops outside of the generator or discriminator scopes. + is_chief: Specifies whether or not the training is being run by the primary + replica during replica training. **kwargs: Keyword args to pass directly to `training.create_train_op` for both the generator and discriminator train op. @@ -980,6 +983,9 @@ def gan_train_ops( kwargs, model.generator_scope.name, model.discriminator_scope.name, check_for_unused_update_ops) + # Get the sync hooks if these are needed. + sync_hooks = [] + generator_global_step = None if isinstance(generator_optimizer, sync_replicas_optimizer.SyncReplicasOptimizer): @@ -995,6 +1001,7 @@ def gan_train_ops( trainable=False, collections=[ops.GraphKeys.GLOBAL_VARIABLES]) gen_update_ops += [generator_global_step.assign(global_step)] + sync_hooks.append(generator_optimizer.make_session_run_hook(is_chief)) with ops.name_scope('generator_train'): gen_train_op = training.create_train_op( total_loss=loss.generator_loss, @@ -1016,6 +1023,7 @@ def gan_train_ops( trainable=False, collections=[ops.GraphKeys.GLOBAL_VARIABLES]) dis_update_ops += [discriminator_global_step.assign(global_step)] + sync_hooks.append(discriminator_optimizer.make_session_run_hook(is_chief)) with ops.name_scope('discriminator_train'): disc_train_op = training.create_train_op( total_loss=loss.discriminator_loss, @@ -1025,7 +1033,8 @@ def gan_train_ops( update_ops=dis_update_ops, **kwargs) - return namedtuples.GANTrainOps(gen_train_op, disc_train_op, global_step_inc) + return namedtuples.GANTrainOps(gen_train_op, disc_train_op, global_step_inc, + sync_hooks) # TODO(joelshor): Implement a dynamic GAN train loop, as in `Real-Time Adaptive @@ -1066,13 +1075,24 @@ def get_sequential_train_hooks(train_steps=namedtuples.GANTrainSteps(1, 1)): train_steps.generator_train_steps) discriminator_hook = RunTrainOpsHook(train_ops.discriminator_train_op, train_steps.discriminator_train_steps) - return [generator_hook, discriminator_hook] + return [generator_hook, discriminator_hook] + list(train_ops.train_hooks) return get_hooks +def _num_joint_steps(train_steps): + g_steps = train_steps.generator_train_steps + d_steps = train_steps.discriminator_train_steps + # Get the number of each type of step that should be run. + num_d_and_g_steps = min(g_steps, d_steps) + num_g_steps = g_steps - num_d_and_g_steps + num_d_steps = d_steps - num_d_and_g_steps + + return num_d_and_g_steps, num_g_steps, num_d_steps + + def get_joint_train_hooks(train_steps=namedtuples.GANTrainSteps(1, 1)): - """Returns a hooks function for sequential GAN training. + """Returns a hooks function for joint GAN training. When using these train hooks, IT IS RECOMMENDED TO USE `use_locking=True` ON ALL OPTIMIZERS TO AVOID RACE CONDITIONS. @@ -1105,12 +1125,7 @@ def get_joint_train_hooks(train_steps=namedtuples.GANTrainSteps(1, 1)): Returns: A function that takes a GANTrainOps tuple and returns a list of hooks. """ - g_steps = train_steps.generator_train_steps - d_steps = train_steps.discriminator_train_steps - # Get the number of each type of step that should be run. - num_d_and_g_steps = min(g_steps, d_steps) - num_g_steps = g_steps - num_d_and_g_steps - num_d_steps = d_steps - num_d_and_g_steps + num_d_and_g_steps, num_g_steps, num_d_steps = _num_joint_steps(train_steps) def get_hooks(train_ops): g_op = train_ops.generator_train_op @@ -1120,7 +1135,7 @@ def get_joint_train_hooks(train_steps=namedtuples.GANTrainSteps(1, 1)): g_hook = RunTrainOpsHook(g_op, num_g_steps) d_hook = RunTrainOpsHook(d_op, num_d_steps) - return [joint_hook, g_hook, d_hook] + return [joint_hook, g_hook, d_hook] + list(train_ops.train_hooks) return get_hooks diff --git a/tensorflow/contrib/gan/python/train_test.py b/tensorflow/contrib/gan/python/train_test.py index 64d67061990..841f25cd7f1 100644 --- a/tensorflow/contrib/gan/python/train_test.py +++ b/tensorflow/contrib/gan/python/train_test.py @@ -519,7 +519,7 @@ class GANLossTest(test.TestCase, parameterized.TestCase): """Test output type.""" loss = train.gan_loss(get_gan_model_fn(), add_summaries=True) self.assertIsInstance(loss, namedtuples.GANLoss) - self.assertGreater(len(ops.get_collection(ops.GraphKeys.SUMMARIES)), 0) + self.assertNotEmpty(ops.get_collection(ops.GraphKeys.SUMMARIES)) @parameterized.named_parameters( ('cyclegan', create_cyclegan_model), @@ -528,7 +528,7 @@ class GANLossTest(test.TestCase, parameterized.TestCase): def test_cyclegan_output_type(self, get_gan_model_fn): loss = train.cyclegan_loss(get_gan_model_fn(), add_summaries=True) self.assertIsInstance(loss, namedtuples.CycleGANLoss) - self.assertGreater(len(ops.get_collection(ops.GraphKeys.SUMMARIES)), 0) + self.assertNotEmpty(ops.get_collection(ops.GraphKeys.SUMMARIES)) @parameterized.named_parameters( ('gan', create_gan_model, False), @@ -759,7 +759,7 @@ class TensorPoolAdjusteModelTest(test.TestCase): # For [pool_size, ?), the pool is full, tensor2 must be equal to some # historical values of tensor1 (which is previously stored in the # pool). - self.assertTrue(any([(v == t2).all() for v in history_values])) + self.assertTrue(any((v == t2).all() for v in history_values)) def _make_new_model_and_check(self, model, pool_size): pool_fn = lambda x: random_tensor_pool.tensor_pool(x, pool_size=pool_size) @@ -836,6 +836,9 @@ class GANTrainOpsTest(test.TestCase, parameterized.TestCase): self.assertIsInstance(train_ops, namedtuples.GANTrainOps) + # Make sure there are no training hooks populated accidentally. + self.assertEmpty(train_ops.train_hooks) + # TODO(joelshor): Add a test to check that custom update op is run. @parameterized.named_parameters( ('gan', create_gan_model, False), @@ -923,8 +926,15 @@ class GANTrainOpsTest(test.TestCase, parameterized.TestCase): model, loss, generator_optimizer=g_opt, discriminator_optimizer=d_opt) self.assertIsInstance(train_ops, namedtuples.GANTrainOps) # No new trainable variables should have been added. - self.assertEqual(num_trainable_vars, - len(variables_lib.get_trainable_variables())) + self.assertLen(variables_lib.get_trainable_variables(), num_trainable_vars) + + # Sync hooks should be populated in the GANTrainOps. + self.assertLen(train_ops.train_hooks, 2) + for hook in train_ops.train_hooks: + self.assertIsInstance( + hook, sync_replicas_optimizer._SyncReplicasOptimizerHook) + sync_opts = [hook._sync_optimizer for hook in train_ops.train_hooks] + self.assertSetEqual(frozenset(sync_opts), frozenset((g_opt, d_opt))) g_sync_init_op = g_opt.get_init_tokens_op(num_tokens=1) d_sync_init_op = d_opt.get_init_tokens_op(num_tokens=1) @@ -959,6 +969,32 @@ class GANTrainOpsTest(test.TestCase, parameterized.TestCase): coord.request_stop() coord.join(g_threads + d_threads) + @parameterized.named_parameters( + ('is_chief', True), + ('is_not_chief', False), + ) + def test_is_chief_in_train_hooks(self, is_chief): + """Make sure is_chief is propagated correctly to sync hooks.""" + model = create_gan_model() + loss = train.gan_loss(model) + g_opt = get_sync_optimizer() + d_opt = get_sync_optimizer() + train_ops = train.gan_train_ops( + model, + loss, + g_opt, + d_opt, + is_chief=is_chief, + summarize_gradients=True, + colocate_gradients_with_ops=True) + + self.assertLen(train_ops.train_hooks, 2) + for hook in train_ops.train_hooks: + self.assertIsInstance( + hook, sync_replicas_optimizer._SyncReplicasOptimizerHook) + is_chief_list = [hook._is_chief for hook in train_ops.train_hooks] + self.assertListEqual(is_chief_list, [is_chief, is_chief]) + class GANTrainTest(test.TestCase, parameterized.TestCase): """Tests for `gan_train`.""" @@ -1036,6 +1072,44 @@ class GANTrainTest(test.TestCase, parameterized.TestCase): self.assertTrue(np.isscalar(final_loss)) self.assertEqual(17.0, final_loss) + @parameterized.named_parameters( + ('gan', create_gan_model), + ('callable_gan', create_callable_gan_model), + ('infogan', create_infogan_model), + ('callable_infogan', create_callable_infogan_model), + ('acgan', create_acgan_model), + ('callable_acgan', create_callable_acgan_model), + ) + def test_train_hooks_exist_in_get_hooks_fn(self, create_gan_model_fn): + model = create_gan_model_fn() + loss = train.gan_loss(model) + + g_opt = get_sync_optimizer() + d_opt = get_sync_optimizer() + train_ops = train.gan_train_ops( + model, + loss, + g_opt, + d_opt, + summarize_gradients=True, + colocate_gradients_with_ops=True) + + sequential_train_hooks = train.get_sequential_train_hooks()(train_ops) + self.assertLen(sequential_train_hooks, 4) + sync_opts = [ + hook._sync_optimizer for hook in sequential_train_hooks if + isinstance(hook, sync_replicas_optimizer._SyncReplicasOptimizerHook)] + self.assertLen(sync_opts, 2) + self.assertSetEqual(frozenset(sync_opts), frozenset((g_opt, d_opt))) + + joint_train_hooks = train.get_joint_train_hooks()(train_ops) + self.assertLen(joint_train_hooks, 5) + sync_opts = [ + hook._sync_optimizer for hook in joint_train_hooks if + isinstance(hook, sync_replicas_optimizer._SyncReplicasOptimizerHook)] + self.assertLen(sync_opts, 2) + self.assertSetEqual(frozenset(sync_opts), frozenset((g_opt, d_opt))) + class PatchGANTest(test.TestCase, parameterized.TestCase): """Tests that functions work on PatchGAN style output.""" diff --git a/tensorflow/contrib/gdr/gdr_rendezvous_mgr.cc b/tensorflow/contrib/gdr/gdr_rendezvous_mgr.cc index 94f522c04e5..fbccbead03f 100644 --- a/tensorflow/contrib/gdr/gdr_rendezvous_mgr.cc +++ b/tensorflow/contrib/gdr/gdr_rendezvous_mgr.cc @@ -170,6 +170,14 @@ class GdrRemoteRendezvous : public BaseRemoteRendezvous { // Record "call" in active_ so that it can be aborted cleanly. RegisterCall(call); + // RendezvousMgr already aborted, shouldn't send RPC call any more + if (!call->status().ok()) { + done(call->status(), Args(), Args(), Tensor(), false); + session()->worker_cache->ReleaseWorker(src_worker, rwi); + delete call; + return; + } + // Start "call". Ref(); call->Start([this, call, src_worker, rwi, done]() { diff --git a/tensorflow/contrib/hadoop/python/kernel_tests/hadoop_test.py b/tensorflow/contrib/hadoop/python/kernel_tests/hadoop_test.py index f7f1189bb93..bc941ae9f23 100644 --- a/tensorflow/contrib/hadoop/python/kernel_tests/hadoop_test.py +++ b/tensorflow/contrib/hadoop/python/kernel_tests/hadoop_test.py @@ -21,6 +21,7 @@ from __future__ import print_function import os from tensorflow.contrib.hadoop.python.ops import hadoop_dataset_ops +from tensorflow.python.data.ops import dataset_ops from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes from tensorflow.python.framework import errors @@ -47,7 +48,7 @@ class SequenceFileDatasetTest(test.TestCase): dataset = hadoop_dataset_ops.SequenceFileDataset(filenames).repeat( num_repeats) - iterator = dataset.make_initializable_iterator() + iterator = dataset_ops.make_initializable_iterator(dataset) init_op = iterator.initializer get_next = iterator.get_next() diff --git a/tensorflow/contrib/hadoop/python/ops/hadoop_dataset_ops.py b/tensorflow/contrib/hadoop/python/ops/hadoop_dataset_ops.py index bf398b838df..d3fcc8cb2a9 100644 --- a/tensorflow/contrib/hadoop/python/ops/hadoop_dataset_ops.py +++ b/tensorflow/contrib/hadoop/python/ops/hadoop_dataset_ops.py @@ -40,15 +40,12 @@ class SequenceFileDataset(dataset_ops.DatasetSource): For example: ```python + tf.enable_eager_execution() + dataset = tf.contrib.hadoop.SequenceFileDataset("/foo/bar.seq") - iterator = dataset.make_one_shot_iterator() - next_element = iterator.get_next() # Prints the (key, value) pairs inside a hadoop sequence file. - while True: - try: - print(sess.run(next_element)) - except tf.errors.OutOfRangeError: - break + for key, value in dataset: + print(key, value) ``` Args: diff --git a/tensorflow/contrib/ignite/README.md b/tensorflow/contrib/ignite/README.md index c7db0b77e25..5a8c650fb92 100644 --- a/tensorflow/contrib/ignite/README.md +++ b/tensorflow/contrib/ignite/README.md @@ -54,14 +54,12 @@ jdbc:ignite:thin://localhost/> INSERT INTO KITTEN_CACHE VALUES (3, 'LITTLE BALL ```python >>> import tensorflow as tf >>> from tensorflow.contrib.ignite import IgniteDataset ->>> ->>> dataset = IgniteDataset(cache_name="SQL_PUBLIC_KITTEN_CACHE") ->>> iterator = dataset.make_one_shot_iterator() ->>> next_obj = iterator.get_next() +>>> tf.enable_eager_execution() >>> ->>> with tf.Session() as sess: ->>> for _ in range(3): ->>> print(sess.run(next_obj)) +>>> dataset = IgniteDataset(cache_name="SQL_PUBLIC_KITTEN_CACHE") +>>> +>>> for element in dataset: +>>> print(element) {'key': 1, 'val': {'NAME': b'WARM KITTY'}} {'key': 2, 'val': {'NAME': b'SOFT KITTY'}} @@ -74,23 +72,22 @@ jdbc:ignite:thin://localhost/> INSERT INTO KITTEN_CACHE VALUES (3, 'LITTLE BALL ```python >>> import tensorflow as tf >>> from tensorflow.contrib.ignite import IgniteDataset ->>> ->>> dataset = IgniteDataset(cache_name="IMAGES") ->>> iterator = dataset.make_one_shot_iterator() ->>> next_obj = iterator.get_next() +>>> tf.enable_eager_execution() >>> ->>> with tf.Session() as sess: ->>> print(sess.run(next_obj)) +>>> dataset = IgniteDataset(cache_name="IMAGES") +>>> +>>> for element in dataset.take(1): +>>> print(element) { - 'key': 'kitten.png', + 'key': 'kitten.png', 'val': { 'metadata': { 'file_name': b'kitten.png', 'label': b'little ball of fur', - width: 800, + width: 800, height: 600 - }, + }, 'pixels': [0, 0, 0, 0, ..., 0] } } @@ -100,13 +97,11 @@ jdbc:ignite:thin://localhost/> INSERT INTO KITTEN_CACHE VALUES (3, 'LITTLE BALL ```python >>> import tensorflow as tf >>> from tensorflow.contrib.ignite import IgniteDataset ->>> ->>> dataset = IgniteDataset(cache_name="IMAGES").map(lambda obj: obj['val']['pixels']) ->>> iterator = dataset.make_one_shot_iterator() ->>> next_obj = iterator.get_next() >>> ->>> with tf.Session() as sess: ->>> print(sess.run(next_obj)) +>>> dataset = IgniteDataset(cache_name="IMAGES").map(lambda obj: obj['val']['pixels']) +>>> +>>> for element in dataset: +>>> print(element) [0, 0, 0, 0, ..., 0] ``` @@ -126,18 +121,18 @@ Ignite Dataset allows using these two aspects of distributed neural network trai ```python >>> import tensorflow as tf >>> from tensorflow.contrib.ignite import IgniteDataset ->>> +>>> >>> dataset = IgniteDataset("IMAGES") >>> >>> # Compute gradients locally on every worker node. ->>> gradients = [] +>>> gradients = [] >>> for i in range(5): >>> with tf.device("/job:WORKER/task:%d" % i): ->>> device_iterator = dataset.make_one_shot_iterator() +>>> device_iterator = tf.compat.v1.data.make_one_shot_iterator(dataset) >>> device_next_obj = device_iterator.get_next() >>> gradient = compute_gradient(device_next_obj) ->>> gradients.append(gradient) ->>> +>>> gradients.append(gradient) +>>> >>> # Aggregate them on master node. >>> result_gradient = tf.reduce_sum(gradients) >>> @@ -145,7 +140,7 @@ Ignite Dataset allows using these two aspects of distributed neural network trai >>> print(sess.run(result_gradient)) ``` -High-level TensorFlow API for [distributed training](https://www.tensorflow.org/api_docs/python/tf/contrib/distribute/DistributionStrategy) is supported as well. +High-level TensorFlow API for [distributed training](https://www.tensorflow.org/api_docs/python/tf/contrib/distribute/DistributionStrategy) is supported as well. ### Distributed File System diff --git a/tensorflow/contrib/ignite/python/tests/ignite_dataset_test.py b/tensorflow/contrib/ignite/python/tests/ignite_dataset_test.py index ef29b5f14a4..ff5d4c458c8 100644 --- a/tensorflow/contrib/ignite/python/tests/ignite_dataset_test.py +++ b/tensorflow/contrib/ignite/python/tests/ignite_dataset_test.py @@ -21,6 +21,7 @@ import os from tensorflow.contrib.ignite import IgniteDataset from tensorflow.python.client import session +from tensorflow.python.data.ops import dataset_ops from tensorflow.python.framework import dtypes from tensorflow.python.framework import errors from tensorflow.python.platform import test @@ -65,7 +66,7 @@ class IgniteDatasetTest(test.TestCase): self.assertEqual(dtypes.string, dataset.output_types["val"]["NAME"]) self.assertEqual(dtypes.int64, dataset.output_types["val"]["VAL"]) - it = dataset.make_one_shot_iterator() + it = dataset_ops.make_one_shot_iterator(dataset) ne = it.get_next() with session.Session() as sess: diff --git a/tensorflow/contrib/image/kernels/adjust_hsv_in_yiq_op.cc b/tensorflow/contrib/image/kernels/adjust_hsv_in_yiq_op.cc index 478b716d883..108da044946 100644 --- a/tensorflow/contrib/image/kernels/adjust_hsv_in_yiq_op.cc +++ b/tensorflow/contrib/image/kernels/adjust_hsv_in_yiq_op.cc @@ -115,7 +115,7 @@ class AdjustHsvInYiqOp : public AdjustHsvInYiqOpBase { *context->device()->tensorflow_cpu_worker_threads(); Shard(worker_threads.num_threads, worker_threads.workers, channel_count, kCostPerChannel, - [channel_count, &input_data, &output_data, &tranformation_matrix]( + [&input_data, &output_data, &tranformation_matrix]( int64 start_channel, int64 end_channel) { // Applying projection matrix to input RGB vectors. const float* p = input_data.data() + start_channel * kChannelSize; diff --git a/tensorflow/contrib/image/python/ops/dense_image_warp.py b/tensorflow/contrib/image/python/ops/dense_image_warp.py index 9c7ada7afb7..7930b8317b6 100644 --- a/tensorflow/contrib/image/python/ops/dense_image_warp.py +++ b/tensorflow/contrib/image/python/ops/dense_image_warp.py @@ -25,7 +25,7 @@ from tensorflow.python.framework import ops from tensorflow.python.ops import array_ops from tensorflow.python.ops import math_ops - +from tensorflow.python.ops import check_ops def _interpolate_bilinear(grid, query_points, @@ -60,28 +60,40 @@ def _interpolate_bilinear(grid, msg = 'Grid must be 4 dimensional. Received size: ' raise ValueError(msg + str(grid.get_shape())) - batch_size, height, width, channels = shape + batch_size, height, width, channels = (array_ops.shape(grid)[0], + array_ops.shape(grid)[1], + array_ops.shape(grid)[2], + array_ops.shape(grid)[3]) + + shape = [batch_size, height, width, channels] query_type = query_points.dtype grid_type = grid.dtype - if (query_points.shape.rank != 3 or - query_points.shape.dims[2].value != 2): - msg = ('Query points must be 3 dimensional and size 2 in dim 2. Received ' - 'size: ') - raise ValueError(msg + str(query_points.get_shape())) + with ops.control_dependencies([ + check_ops.assert_equal( + len(query_points.get_shape()), + 3, + message='Query points must be 3 dimensional.'), + check_ops.assert_equal( + array_ops.shape(query_points)[2], + 2, + message='Query points must be size 2 in dim 2.')]): + num_queries = array_ops.shape(query_points)[1] - _, num_queries, _ = query_points.get_shape().as_list() - - if height < 2 or width < 2: - msg = 'Grid must be at least batch_size x 2 x 2 in size. Received size: ' - raise ValueError(msg + str(grid.get_shape())) - - alphas = [] - floors = [] - ceils = [] - - index_order = [0, 1] if indexing == 'ij' else [1, 0] - unstacked_query_points = array_ops.unstack(query_points, axis=2) + with ops.control_dependencies([ + check_ops.assert_greater_equal( + height, + 2, + message='Grid height must be at least 2.'), + check_ops.assert_greater_equal( + width, + 2, + message='Grid width must be at least 2.')]): + alphas = [] + floors = [] + ceils = [] + index_order = [0, 1] if indexing == 'ij' else [1, 0] + unstacked_query_points = array_ops.unstack(query_points, axis=2) for dim in index_order: with ops.name_scope('dim-' + str(dim)): @@ -112,16 +124,17 @@ def _interpolate_bilinear(grid, alpha = array_ops.expand_dims(alpha, 2) alphas.append(alpha) - if batch_size * height * width > np.iinfo(np.int32).max / 8: - error_msg = """The image size or batch size is sufficiently large - that the linearized addresses used by array_ops.gather - may exceed the int32 limit.""" - raise ValueError(error_msg) - - flattened_grid = array_ops.reshape(grid, - [batch_size * height * width, channels]) - batch_offsets = array_ops.reshape( - math_ops.range(batch_size) * height * width, [batch_size, 1]) + with ops.control_dependencies([ + check_ops.assert_less_equal( + math_ops.cast(batch_size * height * width, dtype=dtypes.float32), + np.iinfo(np.int32).max / 8, + message="""The image size or batch size is sufficiently large + that the linearized addresses used by array_ops.gather + may exceed the int32 limit.""")]): + flattened_grid = array_ops.reshape( + grid, [batch_size * height * width, channels]) + batch_offsets = array_ops.reshape( + math_ops.range(batch_size) * height * width, [batch_size, 1]) # This wraps array_ops.gather. We reshape the image data such that the # batch, y, and x coordinates are pulled into the first dimension. @@ -182,7 +195,11 @@ def dense_image_warp(image, flow, name='dense_image_warp'): of dimensions. """ with ops.name_scope(name): - batch_size, height, width, channels = image.get_shape().as_list() + batch_size, height, width, channels = (array_ops.shape(image)[0], + array_ops.shape(image)[1], + array_ops.shape(image)[2], + array_ops.shape(image)[3]) + # The flow is defined on the image grid. Turn the flow into a list of query # points in the grid space. grid_x, grid_y = array_ops.meshgrid( diff --git a/tensorflow/contrib/keras/api/keras/layers/__init__.py b/tensorflow/contrib/keras/api/keras/layers/__init__.py index 3327a9f9a61..9e19884df85 100644 --- a/tensorflow/contrib/keras/api/keras/layers/__init__.py +++ b/tensorflow/contrib/keras/api/keras/layers/__init__.py @@ -20,7 +20,7 @@ from __future__ import print_function # Generic layers. # pylint: disable=g-bad-import-order -from tensorflow.python.keras.engine.base_layer import InputSpec +from tensorflow.python.keras.engine.input_spec import InputSpec from tensorflow.python.keras.engine.base_layer import Layer from tensorflow.python.keras.engine.input_layer import Input from tensorflow.python.keras.engine.input_layer import InputLayer diff --git a/tensorflow/contrib/keras/api/keras/utils/__init__.py b/tensorflow/contrib/keras/api/keras/utils/__init__.py index 47cd01b924f..3b9fa1b230b 100644 --- a/tensorflow/contrib/keras/api/keras/utils/__init__.py +++ b/tensorflow/contrib/keras/api/keras/utils/__init__.py @@ -30,6 +30,7 @@ from tensorflow.python.keras.utils.generic_utils import Progbar from tensorflow.python.keras.utils.generic_utils import serialize_keras_object from tensorflow.python.keras.utils.io_utils import HDF5Matrix from tensorflow.python.keras.utils.layer_utils import convert_all_kernels_in_model +from tensorflow.python.keras.utils.losses_utils import squeeze_or_expand_dimensions from tensorflow.python.keras.utils.np_utils import normalize from tensorflow.python.keras.utils.np_utils import to_categorical from tensorflow.python.keras.utils.vis_utils import plot_model diff --git a/tensorflow/contrib/kernel_methods/python/kernel_estimators.py b/tensorflow/contrib/kernel_methods/python/kernel_estimators.py index de7530231db..1626e55b9b3 100644 --- a/tensorflow/contrib/kernel_methods/python/kernel_estimators.py +++ b/tensorflow/contrib/kernel_methods/python/kernel_estimators.py @@ -90,7 +90,7 @@ def _update_features_and_columns(features, feature_columns, mapped_column_name = column_name + "_MAPPED" # Construct new feature columns based on provided kernel_mappers. column_kernel_mappers = kernel_mappers_dict[feature_column] - new_dim = sum([mapper.output_dim for mapper in column_kernel_mappers]) + new_dim = sum(mapper.output_dim for mapper in column_kernel_mappers) mapped_columns.add( layers.feature_column.real_valued_column(mapped_column_name, new_dim)) diff --git a/tensorflow/contrib/kinesis/python/ops/kinesis_dataset_ops.py b/tensorflow/contrib/kinesis/python/ops/kinesis_dataset_ops.py index 75806dbbeb1..c392adbb1d9 100644 --- a/tensorflow/contrib/kinesis/python/ops/kinesis_dataset_ops.py +++ b/tensorflow/contrib/kinesis/python/ops/kinesis_dataset_ops.py @@ -34,15 +34,12 @@ class KinesisDataset(dataset_ops.DatasetSource): For example, we can construct and use the KinesisDataset as follows: ```python + tf.enable_eager_execution() + dataset = tf.contrib.kinesis.KinesisDataset( "kinesis_stream_name", read_indefinitely=False) - next = dataset.make_one_shot_iterator().get_next() - with tf.Session() as sess: - while True: - try: - print(sess.run(nxt)) - except tf.errors.OutOfRangeError: - break + for element in dataset: + print(element) ``` Since Kinesis is a data streaming service, data may not be available diff --git a/tensorflow/contrib/layers/BUILD b/tensorflow/contrib/layers/BUILD index e6596bfdfb9..9ca6f8df5db 100644 --- a/tensorflow/contrib/layers/BUILD +++ b/tensorflow/contrib/layers/BUILD @@ -78,6 +78,11 @@ tf_custom_op_py_library( ":sparse_feature_cross_op_op_lib", ], srcs_version = "PY2AND3", + visibility = [ + "//learning/brain:__subpackages__", + "//tensorflow:__subpackages__", + "//video/youtube/personalization:__subpackages__", + ], deps = [ ":sparse_feature_cross_op", "//tensorflow/contrib/framework:framework_py", @@ -253,7 +258,7 @@ py_test( "//tensorflow/python:training", "//tensorflow/python:variable_scope", "//tensorflow/python:variables", - "//tensorflow/python/feature_column", + "//tensorflow/python/feature_column:feature_column_py", "//third_party/py/numpy", ], ) @@ -277,7 +282,7 @@ py_test( "//tensorflow/python:sparse_tensor", "//tensorflow/python:variable_scope", "//tensorflow/python:variables", - "//tensorflow/python/feature_column", + "//tensorflow/python/feature_column:feature_column_py", "//third_party/py/numpy", ], ) diff --git a/tensorflow/contrib/layers/python/layers/embedding_ops_test.py b/tensorflow/contrib/layers/python/layers/embedding_ops_test.py index 124515e5a64..295c721fced 100644 --- a/tensorflow/contrib/layers/python/layers/embedding_ops_test.py +++ b/tensorflow/contrib/layers/python/layers/embedding_ops_test.py @@ -21,6 +21,7 @@ from __future__ import print_function import itertools import math +import sys import numpy as np @@ -36,6 +37,7 @@ from tensorflow.python.ops import gradient_checker from tensorflow.python.ops import init_ops from tensorflow.python.ops import math_ops from tensorflow.python.ops import partitioned_variables +from tensorflow.python.ops import variable_scope from tensorflow.python.platform import test from tensorflow.python.util import compat @@ -48,11 +50,13 @@ class SafeEmbeddingLookupSparseTest(test.TestCase): assert num_shards > 0 assert num_shards <= vocab_size - embedding_weights = partitioned_variables.create_partitioned_variables( + initializer = init_ops.truncated_normal_initializer( + mean=0.0, stddev=1.0 / math.sqrt(vocab_size), dtype=dtypes.float32) + embedding_weights = list(variable_scope.get_variable( + "embedding_weights", shape=[vocab_size, embed_dim], - slicing=[num_shards, 1], - initializer=init_ops.truncated_normal_initializer( - mean=0.0, stddev=1.0 / math.sqrt(vocab_size), dtype=dtypes.float32)) + partitioner=partitioned_variables.fixed_size_partitioner(num_shards), + initializer=initializer)) for w in embedding_weights: w.initializer.run() embedding_weights = [w.eval() for w in embedding_weights] @@ -256,6 +260,13 @@ class SafeEmbeddingLookupSparseTest(test.TestCase): embedding_weights, sparse_ids, sparse_weights) +# pylint: disable=invalid-name +def local_variable_scope(): + """Create a variable scope named like the caller function.""" + return variable_scope.variable_scope(sys._getframe(1).f_code.co_name) +# pylint: enable=invalid-name + + class ScatteredEmbeddingLookupTest(test.TestCase): def setUp(self): @@ -266,17 +277,18 @@ class ScatteredEmbeddingLookupTest(test.TestCase): assert num_shards > 0 assert num_shards <= size - embedding_weights = partitioned_variables.create_partitioned_variables( + embedding_weights = list(variable_scope.get_variable( + "embedding_weights", shape=[size], - slicing=[num_shards], + partitioner=partitioned_variables.fixed_size_partitioner(num_shards), initializer=init_ops.truncated_normal_initializer( - mean=0.0, stddev=1.0, dtype=dtypes.float32)) + mean=0.0, stddev=1.0, dtype=dtypes.float32))) for w in embedding_weights: w.initializer.run() return embedding_weights def test_scattered_embedding_consistency(self): - with self.cached_session(): + with self.cached_session(), local_variable_scope(): embedding_weights = self._random_weights() values = constant_op.constant(["foo", "foo"]) @@ -288,7 +300,7 @@ class ScatteredEmbeddingLookupTest(test.TestCase): embedding_lookup_result[1]) def test_scattered_embedding_multiple_partition(self): - with self.cached_session(): + with self.cached_session(), local_variable_scope(): embedding_weights = self._random_weights(num_shards=7) values = constant_op.constant([4, 4, 5]) @@ -304,7 +316,7 @@ class ScatteredEmbeddingLookupTest(test.TestCase): self.assertGreater(embedding_diff, 0) def test_scattered_embedding_coverage(self): - with self.cached_session(): + with self.cached_session(), local_variable_scope(): size = 8 embedding_weights = self._random_weights(size=size, num_shards=3) values = constant_op.constant(["foo"]) @@ -316,7 +328,7 @@ class ScatteredEmbeddingLookupTest(test.TestCase): self.assertEqual(len(np.unique(embedding_lookup_result[0])), size) def test_scattered_embedding_multi_dimension(self): - with self.cached_session(): + with self.cached_session(), local_variable_scope(): embedding_weights = self._random_weights() values = constant_op.constant([["foo", "bar", "bar"], ["bar", "bar", "foo"]]) @@ -329,7 +341,7 @@ class ScatteredEmbeddingLookupTest(test.TestCase): embedding_lookup_result[1][2]) def test_scattered_embedding_lookup_sparse(self): - with self.cached_session(): + with self.cached_session(), local_variable_scope(): embedding_weights = self._random_weights(num_shards=3) sparse_tensor = sparse_tensor_lib.SparseTensor( values=["foo", "bar", "foo", "bar"], @@ -358,7 +370,7 @@ class ScatteredEmbeddingLookupTest(test.TestCase): embeds = np.random.randn(n_embed, d_embed) idx = np.random.randint(0, n_embed, idx_shape) - with self.cached_session(): + with self.cached_session(), local_variable_scope(): embedded_np = embeds[idx] embedded_tf = embedding_ops.embedding_lookup_unique(embeds, idx).eval() @@ -370,7 +382,7 @@ class ScatteredEmbeddingLookupTest(test.TestCase): idx = np.random.randint(0, 5, 10) idx2d = np.random.randint(0, 5, (10, 2)) - with self.cached_session(): + with self.cached_session(), local_variable_scope(): embedded_np = embeds[idx] embedded_np2d = embeds[idx2d] embedded_tf = embedding_ops.embedding_lookup_unique(embeds, idx).eval() @@ -398,17 +410,18 @@ class SampledScatteredEmbeddingLookupTest(test.TestCase): assert num_shards > 0 assert num_shards <= size - embedding_weights = partitioned_variables.create_partitioned_variables( + embedding_weights = list(variable_scope.get_variable( + "embedding_weights", shape=[size], - slicing=[num_shards], + partitioner=partitioned_variables.fixed_size_partitioner(num_shards), initializer=init_ops.truncated_normal_initializer( - mean=0.0, stddev=1.0, dtype=dtypes.float32)) + mean=0.0, stddev=1.0, dtype=dtypes.float32))) for w in embedding_weights: w.initializer.run() return embedding_weights def test_hashed_embedding_consistency(self): - with self.cached_session(): + with self.cached_session(), local_variable_scope(): embedding_weights = self._random_weights() values = constant_op.constant(["foo", "foo"]) # The first three sampled_candidates are equal, so the first three @@ -429,7 +442,7 @@ class SampledScatteredEmbeddingLookupTest(test.TestCase): embedding_lookup_result[1][3]) def test_hashed_embedding_multi_dimension(self): - with self.cached_session(): + with self.cached_session(), local_variable_scope(): embedding_weights = self._random_weights() values = constant_op.constant([["foo", "bar", "bar"], ["bar", "bar", "foo"]]) @@ -691,7 +704,6 @@ class EmbeddingLookupSparseWithDistributedAggregationTest(test.TestCase): index += num_val return grouped_vals - @test_util.enable_c_shapes def testEmbeddingLookupSparse(self): vocab_size = 13 batch_size = 10 diff --git a/tensorflow/contrib/layers/python/layers/encoders.py b/tensorflow/contrib/layers/python/layers/encoders.py index f42112206d0..3671633c8d7 100644 --- a/tensorflow/contrib/layers/python/layers/encoders.py +++ b/tensorflow/contrib/layers/python/layers/encoders.py @@ -84,8 +84,7 @@ def bow_encoder(ids, if isinstance(ids, sparse_tensor.SparseTensor): raise TypeError('ids are expected to be dense Tensor, got: %s', ids) return math_ops.reduce_mean( - embedding_ops.embedding_lookup(embeddings, ids), - reduction_indices=1) + embedding_ops.embedding_lookup(embeddings, ids), axis=1) def embed_sequence(ids, diff --git a/tensorflow/contrib/layers/python/layers/feature_column.py b/tensorflow/contrib/layers/python/layers/feature_column.py index 222404b19db..00d819ed0e9 100644 --- a/tensorflow/contrib/layers/python/layers/feature_column.py +++ b/tensorflow/contrib/layers/python/layers/feature_column.py @@ -1015,8 +1015,7 @@ class _OneHotColumn( dense_id_tensor, depth=self.length, on_value=1.0, off_value=0.0) # Reduce to get a multi-hot per example. - return math_ops.reduce_sum( - one_hot_id_tensor, reduction_indices=[output_rank - 1]) + return math_ops.reduce_sum(one_hot_id_tensor, axis=[output_rank - 1]) @property def _variable_shape(self): diff --git a/tensorflow/contrib/layers/python/layers/feature_column_ops_test.py b/tensorflow/contrib/layers/python/layers/feature_column_ops_test.py index 6fb4b9ff353..7e6eafaa0d6 100644 --- a/tensorflow/contrib/layers/python/layers/feature_column_ops_test.py +++ b/tensorflow/contrib/layers/python/layers/feature_column_ops_test.py @@ -27,7 +27,7 @@ from tensorflow.contrib.layers.python.layers import feature_column from tensorflow.contrib.layers.python.layers import feature_column_ops from tensorflow.core.example import example_pb2 from tensorflow.core.example import feature_pb2 -from tensorflow.python.feature_column import feature_column as fc_core +from tensorflow.python.feature_column import feature_column_lib as fc_core from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops diff --git a/tensorflow/contrib/layers/python/layers/feature_column_test.py b/tensorflow/contrib/layers/python/layers/feature_column_test.py index d90d6ecf7f6..cab8da808b6 100644 --- a/tensorflow/contrib/layers/python/layers/feature_column_test.py +++ b/tensorflow/contrib/layers/python/layers/feature_column_test.py @@ -27,7 +27,7 @@ import numpy as np from tensorflow.contrib.layers.python.layers import feature_column as fc from tensorflow.contrib.layers.python.layers import feature_column_ops -from tensorflow.python.feature_column import feature_column as fc_core +from tensorflow.python.feature_column import feature_column_lib as fc_core from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes from tensorflow.python.framework import sparse_tensor as sparse_tensor_lib diff --git a/tensorflow/contrib/layers/python/layers/layers.py b/tensorflow/contrib/layers/python/layers/layers.py index ac9561c7693..403b522ce45 100644 --- a/tensorflow/contrib/layers/python/layers/layers.py +++ b/tensorflow/contrib/layers/python/layers/layers.py @@ -35,6 +35,7 @@ from tensorflow.python.framework import function from tensorflow.python.framework import ops from tensorflow.python.framework import sparse_tensor from tensorflow.python.framework import tensor_shape +from tensorflow.python.keras.engine import input_spec from tensorflow.python.layers import base from tensorflow.python.layers import convolutional as convolutional_layers from tensorflow.python.layers import core as core_layers @@ -1958,7 +1959,7 @@ class GDN(base.Layer): self._reparam_offset = reparam_offset self.data_format = data_format self._channel_axis() # trigger ValueError early - self.input_spec = base.InputSpec(min_ndim=3, max_ndim=5) + self.input_spec = input_spec.InputSpec(min_ndim=3, max_ndim=5) def _channel_axis(self): try: @@ -2015,7 +2016,7 @@ class GDN(base.Layer): raise ValueError('The channel dimension of the inputs to `GDN` ' 'must be defined.') self._input_rank = input_shape.ndims - self.input_spec = base.InputSpec( + self.input_spec = input_spec.InputSpec( ndim=input_shape.ndims, axes={ channel_axis: num_channels }) diff --git a/tensorflow/contrib/layers/python/layers/layers_test.py b/tensorflow/contrib/layers/python/layers/layers_test.py index 8ead6336a08..0a4d2c6d4cb 100644 --- a/tensorflow/contrib/layers/python/layers/layers_test.py +++ b/tensorflow/contrib/layers/python/layers/layers_test.py @@ -3811,7 +3811,7 @@ class UnitNormTests(test.TestCase): image = random_ops.random_uniform((height, width, 3)) output = _layers.unit_norm(image, dim=dim, epsilon=1e-6) norms = math_ops.sqrt( - math_ops.reduce_sum(math_ops.square(output), reduction_indices=dim)) + math_ops.reduce_sum(math_ops.square(output), axis=dim)) shape = [height, width, 3] del shape[dim] @@ -3847,7 +3847,7 @@ class UnitNormTests(test.TestCase): image = array_ops.placeholder(dtypes.float32, (None, None, 3)) output = _layers.unit_norm(image, dim=dim, epsilon=1e-6) norms = math_ops.sqrt( - math_ops.reduce_sum(math_ops.square(output), reduction_indices=dim)) + math_ops.reduce_sum(math_ops.square(output), axis=dim)) with self.cached_session(): actual = norms.eval({image: placeholder_value}) diff --git a/tensorflow/contrib/layers/python/layers/regularizers_test.py b/tensorflow/contrib/layers/python/layers/regularizers_test.py index 51faba30c74..5cb00b76847 100644 --- a/tensorflow/contrib/layers/python/layers/regularizers_test.py +++ b/tensorflow/contrib/layers/python/layers/regularizers_test.py @@ -141,7 +141,7 @@ class RegularizerTest(test.TestCase): dummy_regularizer = lambda x: math_ops.reduce_sum(2 * x) array_weights_list = [[1.5], [2, 3, 4.2], [10, 42, 666.6]] tensor_weights_list = [constant_op.constant(x) for x in array_weights_list] - expected = sum([2 * x for l in array_weights_list for x in l]) + expected = sum(2 * x for l in array_weights_list for x in l) with self.cached_session(): result = regularizers.apply_regularization(dummy_regularizer, tensor_weights_list) diff --git a/tensorflow/contrib/learn/BUILD b/tensorflow/contrib/learn/BUILD index 61185f65a9b..14065fcee51 100644 --- a/tensorflow/contrib/learn/BUILD +++ b/tensorflow/contrib/learn/BUILD @@ -24,6 +24,11 @@ py_library( exclude = ["python/learn/**/*_test.py"], ), srcs_version = "PY2AND3", + visibility = [ + "//learning/brain:__subpackages__", + "//tensorflow:__subpackages__", + "//video/youtube/personalization:__subpackages__", + ], # This library should not depend on sklearn, even though some of the code # refers to it. (The code handles the presence of sklearn conditionally.) deps = [ @@ -269,6 +274,7 @@ py_test( name = "estimator_test", size = "medium", srcs = ["python/learn/estimators/estimator_test.py"], + shard_count = 2, srcs_version = "PY2AND3", tags = [ "manual", diff --git a/tensorflow/contrib/learn/python/learn/estimators/dnn.py b/tensorflow/contrib/learn/python/learn/estimators/dnn.py index eabebb7e881..10fbd60ba2d 100644 --- a/tensorflow/contrib/learn/python/learn/estimators/dnn.py +++ b/tensorflow/contrib/learn/python/learn/estimators/dnn.py @@ -28,7 +28,6 @@ import six from tensorflow.contrib import layers from tensorflow.contrib.framework import deprecated from tensorflow.contrib.framework import deprecated_arg_values -from tensorflow.python.training import training_util from tensorflow.contrib.layers.python.layers import feature_column from tensorflow.contrib.layers.python.layers import optimizers from tensorflow.contrib.learn.python.learn import metric_spec @@ -38,11 +37,12 @@ from tensorflow.contrib.learn.python.learn.estimators import head as head_lib from tensorflow.contrib.learn.python.learn.estimators import model_fn from tensorflow.contrib.learn.python.learn.estimators import prediction_key from tensorflow.contrib.learn.python.learn.utils import export -from tensorflow.python.feature_column import feature_column as fc_core +from tensorflow.python.feature_column import feature_column_lib as fc_core from tensorflow.python.ops import nn from tensorflow.python.ops import partitioned_variables from tensorflow.python.ops import variable_scope from tensorflow.python.summary import summary +from tensorflow.python.training import training_util # The default learning rate of 0.05 is a historical artifact of the initial # implementation, but seems a reasonable choice. @@ -150,10 +150,10 @@ def _dnn_model_fn(features, labels, mode, params, config=None): "input_from_feature_columns", values=tuple(six.itervalues(features)), partitioner=input_layer_partitioner) as input_layer_scope: - if all([ + if all( isinstance(fc, feature_column._FeatureColumn) # pylint: disable=protected-access for fc in feature_columns - ]): + ): net = layers.input_from_feature_columns( columns_to_tensors=features, feature_columns=feature_columns, diff --git a/tensorflow/contrib/learn/python/learn/estimators/dnn_linear_combined.py b/tensorflow/contrib/learn/python/learn/estimators/dnn_linear_combined.py index 3d85533d92d..2ade6b7b6ce 100644 --- a/tensorflow/contrib/learn/python/learn/estimators/dnn_linear_combined.py +++ b/tensorflow/contrib/learn/python/learn/estimators/dnn_linear_combined.py @@ -38,7 +38,7 @@ from tensorflow.contrib.learn.python.learn.estimators import head as head_lib from tensorflow.contrib.learn.python.learn.estimators import model_fn from tensorflow.contrib.learn.python.learn.estimators import prediction_key from tensorflow.contrib.learn.python.learn.utils import export -from tensorflow.python.feature_column import feature_column as fc_core +from tensorflow.python.feature_column import feature_column_lib as fc_core from tensorflow.python.framework import ops from tensorflow.python.ops import control_flow_ops from tensorflow.python.ops import nn @@ -236,10 +236,10 @@ def _dnn_linear_combined_model_fn(features, labels, mode, params, config=None): "input_from_feature_columns", values=tuple(six.itervalues(features)), partitioner=input_layer_partitioner) as dnn_input_scope: - if all([ + if all( isinstance(fc, feature_column_lib._FeatureColumn) # pylint: disable=protected-access for fc in dnn_feature_columns - ]): + ): net = layers.input_from_feature_columns( columns_to_tensors=features, feature_columns=dnn_feature_columns, @@ -292,8 +292,8 @@ def _dnn_linear_combined_model_fn(features, labels, mode, params, config=None): linear_parent_scope, values=tuple(six.itervalues(features)), partitioner=linear_partitioner) as scope: - if all([isinstance(fc, feature_column_lib._FeatureColumn) # pylint: disable=protected-access - for fc in linear_feature_columns]): + if all(isinstance(fc, feature_column_lib._FeatureColumn) # pylint: disable=protected-access + for fc in linear_feature_columns): if joint_linear_weights: linear_logits, _, _ = layers.joint_weighted_sum_from_feature_columns( columns_to_tensors=features, diff --git a/tensorflow/contrib/learn/python/learn/estimators/dnn_linear_combined_test.py b/tensorflow/contrib/learn/python/learn/estimators/dnn_linear_combined_test.py index 4e65c180d8b..d46a873bfaa 100644 --- a/tensorflow/contrib/learn/python/learn/estimators/dnn_linear_combined_test.py +++ b/tensorflow/contrib/learn/python/learn/estimators/dnn_linear_combined_test.py @@ -36,7 +36,7 @@ from tensorflow.contrib.learn.python.learn.estimators import run_config from tensorflow.contrib.learn.python.learn.estimators import test_data from tensorflow.contrib.learn.python.learn.metric_spec import MetricSpec from tensorflow.contrib.metrics.python.ops import metric_ops -from tensorflow.python.feature_column import feature_column as fc_core +from tensorflow.python.feature_column import feature_column_lib as fc_core from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops diff --git a/tensorflow/contrib/learn/python/learn/estimators/dnn_test.py b/tensorflow/contrib/learn/python/learn/estimators/dnn_test.py index 2bd57597c2e..ee25cebd484 100644 --- a/tensorflow/contrib/learn/python/learn/estimators/dnn_test.py +++ b/tensorflow/contrib/learn/python/learn/estimators/dnn_test.py @@ -38,7 +38,7 @@ from tensorflow.contrib.learn.python.learn.estimators import run_config from tensorflow.contrib.learn.python.learn.estimators import test_data from tensorflow.contrib.learn.python.learn.metric_spec import MetricSpec from tensorflow.contrib.metrics.python.ops import metric_ops -from tensorflow.python.feature_column import feature_column as fc_core +from tensorflow.python.feature_column import feature_column_lib as fc_core from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes from tensorflow.python.framework import sparse_tensor diff --git a/tensorflow/contrib/learn/python/learn/estimators/dynamic_rnn_estimator_test.py b/tensorflow/contrib/learn/python/learn/estimators/dynamic_rnn_estimator_test.py index 1d8a59281a4..28c4964527b 100644 --- a/tensorflow/contrib/learn/python/learn/estimators/dynamic_rnn_estimator_test.py +++ b/tensorflow/contrib/learn/python/learn/estimators/dynamic_rnn_estimator_test.py @@ -668,7 +668,7 @@ class DynamicRNNEstimatorLearningTest(test.TestCase): sequences = centers + noise inputs = array_ops.expand_dims(sequences, 2) - labels = math_ops.reduce_mean(sequences, reduction_indices=[1]) + labels = math_ops.reduce_mean(sequences, axis=[1]) return {'inputs': inputs}, labels return input_fn @@ -722,8 +722,8 @@ class DynamicRNNEstimatorLearningTest(test.TestCase): inputs = array_ops.expand_dims(math_ops.to_float(random_sequence), 2) labels = math_ops.to_int32( array_ops.squeeze( - math_ops.reduce_sum( - inputs, reduction_indices=[1]) > (sequence_length / 2.0))) + math_ops.reduce_sum(inputs, axis=[1]) > ( + sequence_length / 2.0))) return {'inputs': inputs}, labels return input_fn diff --git a/tensorflow/contrib/learn/python/learn/estimators/estimator.py b/tensorflow/contrib/learn/python/learn/estimators/estimator.py index 8bc869db895..9132b2209bc 100644 --- a/tensorflow/contrib/learn/python/learn/estimators/estimator.py +++ b/tensorflow/contrib/learn/python/learn/estimators/estimator.py @@ -1066,11 +1066,11 @@ class BaseEstimator(sklearn.BaseEstimator, evaluable.Evaluable, chief_hooks = [] if (self._config.save_checkpoints_secs or self._config.save_checkpoints_steps): - saver_hook_exists = any([ + saver_hook_exists = any( isinstance(h, basic_session_run_hooks.CheckpointSaverHook) for h in (all_hooks + model_fn_ops.training_hooks + chief_hooks + model_fn_ops.training_chief_hooks) - ]) + ) if not saver_hook_exists: chief_hooks = [ basic_session_run_hooks.CheckpointSaverHook( @@ -1493,7 +1493,7 @@ class Estimator(BaseEstimator): # pylint: disable=protected-access class SKCompat(sklearn.BaseEstimator): """Scikit learn wrapper for TensorFlow Learn Estimator. - + THIS CLASS IS DEPRECATED. See [contrib/learn/README.md](https://www.tensorflow.org/code/tensorflow/contrib/learn/README.md) for general migration instructions. diff --git a/tensorflow/contrib/learn/python/learn/estimators/linear.py b/tensorflow/contrib/learn/python/learn/estimators/linear.py index e100bc7a1e7..9ee8d8004bf 100644 --- a/tensorflow/contrib/learn/python/learn/estimators/linear.py +++ b/tensorflow/contrib/learn/python/learn/estimators/linear.py @@ -37,7 +37,7 @@ from tensorflow.contrib.learn.python.learn.estimators import head as head_lib from tensorflow.contrib.learn.python.learn.estimators import prediction_key from tensorflow.contrib.learn.python.learn.utils import export from tensorflow.contrib.linear_optimizer.python import sdca_optimizer -from tensorflow.python.feature_column import feature_column as fc_core +from tensorflow.python.feature_column import feature_column_lib as fc_core from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops from tensorflow.python.framework import sparse_tensor @@ -155,8 +155,8 @@ def _linear_model_fn(features, labels, mode, params, config=None): parent_scope, values=tuple(six.itervalues(features)), partitioner=partitioner) as scope: - if all([isinstance(fc, feature_column._FeatureColumn) # pylint: disable=protected-access - for fc in feature_columns]): + if all(isinstance(fc, feature_column._FeatureColumn) # pylint: disable=protected-access + for fc in feature_columns): if joint_weights: layer_fn = layers.joint_weighted_sum_from_feature_columns else: diff --git a/tensorflow/contrib/learn/python/learn/estimators/linear_test.py b/tensorflow/contrib/learn/python/learn/estimators/linear_test.py index 597ca4e86db..dfc76bfde6c 100644 --- a/tensorflow/contrib/learn/python/learn/estimators/linear_test.py +++ b/tensorflow/contrib/learn/python/learn/estimators/linear_test.py @@ -37,7 +37,7 @@ from tensorflow.contrib.learn.python.learn.estimators import test_data from tensorflow.contrib.learn.python.learn.metric_spec import MetricSpec from tensorflow.contrib.linear_optimizer.python import sdca_optimizer as sdca_optimizer_lib from tensorflow.contrib.metrics.python.ops import metric_ops -from tensorflow.python.feature_column import feature_column as fc_core +from tensorflow.python.feature_column import feature_column_lib as fc_core from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes from tensorflow.python.framework import sparse_tensor @@ -1745,7 +1745,7 @@ class LinearRegressorTest(test.TestCase): 'place_holder': constant_op.constant([[0.0]] * num_examples), }, constant_op.constant( - [[1 if i % 4 is 0 else 0] for i in range(num_examples)]) + [[1 if i % 4 == 0 else 0] for i in range(num_examples)]) place_holder = feature_column_lib.real_valued_column('place_holder') sdca_optimizer = sdca_optimizer_lib.SDCAOptimizer( diff --git a/tensorflow/contrib/learn/python/learn/learn_io/numpy_io.py b/tensorflow/contrib/learn/python/learn/learn_io/numpy_io.py index 29552d24f1e..59a67636ae2 100644 --- a/tensorflow/contrib/learn/python/learn/learn_io/numpy_io.py +++ b/tensorflow/contrib/learn/python/learn/learn_io/numpy_io.py @@ -27,7 +27,7 @@ from tensorflow.python.estimator.inputs.numpy_io import numpy_input_fn as core_n from tensorflow.python.util.deprecation import deprecated -@deprecated(None, 'Use tf.estimator.inputs.numpy_input_fn.') +@deprecated(None, 'Use tf.compat.v1.estimator.inputs.numpy_input_fn.') def numpy_input_fn(x, y=None, batch_size=128, diff --git a/tensorflow/contrib/learn/python/learn/learn_io/pandas_io.py b/tensorflow/contrib/learn/python/learn/learn_io/pandas_io.py index b4ef055f5ae..e9df7258a35 100644 --- a/tensorflow/contrib/learn/python/learn/learn_io/pandas_io.py +++ b/tensorflow/contrib/learn/python/learn/learn_io/pandas_io.py @@ -53,7 +53,7 @@ PANDAS_DTYPES = { } -@deprecated(None, 'Please use tf.estimator.inputs.pandas_input_fn') +@deprecated(None, 'Please use tf.compat.v1.estimator.inputs.pandas_input_fn') def pandas_input_fn(x, y=None, batch_size=128, diff --git a/tensorflow/contrib/linear_optimizer/python/sdca_estimator_test.py b/tensorflow/contrib/linear_optimizer/python/sdca_estimator_test.py index 64766718823..7a5354222f1 100644 --- a/tensorflow/contrib/linear_optimizer/python/sdca_estimator_test.py +++ b/tensorflow/contrib/linear_optimizer/python/sdca_estimator_test.py @@ -524,7 +524,7 @@ class SDCALinearRegressorTest(test.TestCase): # LinearClassifier requires at least one column. 'place_holder': constant_op.constant([[0.0]] * num_examples), - }, constant_op.constant([[1 if i % 4 is 0 else 0] + }, constant_op.constant([[1 if i % 4 == 0 else 0] for i in range(num_examples)]) with self._single_threaded_test_session(): diff --git a/tensorflow/contrib/lookup/lookup_ops_test.py b/tensorflow/contrib/lookup/lookup_ops_test.py index 5e99ef46051..9b2c2dd87cc 100644 --- a/tensorflow/contrib/lookup/lookup_ops_test.py +++ b/tensorflow/contrib/lookup/lookup_ops_test.py @@ -25,6 +25,7 @@ import six from tensorflow.contrib import lookup from tensorflow.python.client import session from tensorflow.python.data.experimental.ops import counter +from tensorflow.python.data.ops import dataset_ops from tensorflow.python.eager import context from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes @@ -2737,7 +2738,7 @@ class MutableHashTableBenchmark(test.Benchmark): def benchmark_many_repeated_scalar_insert_scalar(self): table = self._create_table() - c = counter.Counter().make_one_shot_iterator().get_next() + c = dataset_ops.make_one_shot_iterator(counter.Counter()).get_next() value = variables.Variable(1.0) insert = table.insert(c, value) size = table.size() @@ -2758,7 +2759,7 @@ class MutableHashTableBenchmark(test.Benchmark): def benchmark_many_repeated_batch_32_insert_scalar(self): table = self._create_table() - c = counter.Counter().make_one_shot_iterator().get_next() + c = dataset_ops.make_one_shot_iterator(counter.Counter()).get_next() value = variables.Variable([1.0] * 32) insert = table.insert(32 * c + list(range(32)), value) size = table.size() diff --git a/tensorflow/contrib/losses/python/losses/loss_ops.py b/tensorflow/contrib/losses/python/losses/loss_ops.py index 619294b5182..709a042bbce 100644 --- a/tensorflow/contrib/losses/python/losses/loss_ops.py +++ b/tensorflow/contrib/losses/python/losses/loss_ops.py @@ -22,7 +22,6 @@ from __future__ import division from __future__ import print_function from tensorflow.contrib.framework.python.ops import add_arg_scope -from tensorflow.python.compat import compat from tensorflow.python.framework import ops from tensorflow.python.ops import array_ops from tensorflow.python.ops import math_ops @@ -60,41 +59,12 @@ def _scale_losses(losses, weights): """ # First, compute the sum of the losses over all elements: start_index = max(0, weights.get_shape().ndims) - reduction_indices = list(range(start_index, losses.get_shape().ndims)) - reduced_losses = math_ops.reduce_sum( - losses, reduction_indices=reduction_indices) + axis = list(range(start_index, losses.get_shape().ndims)) + reduced_losses = math_ops.reduce_sum(losses, axis=axis) reduced_losses = math_ops.multiply(reduced_losses, weights) return math_ops.reduce_sum(reduced_losses) -def _safe_div(numerator, denominator, name="value"): - """Computes a safe divide which returns 0 if the denominator is zero. - - Note that the function contains an additional conditional check that is - necessary for avoiding situations where the loss is zero causing NaNs to - creep into the gradient computation. - - Args: - numerator: An arbitrary `Tensor`. - denominator: A `Tensor` whose shape matches `numerator` and whose values are - assumed to be non-negative. - name: An optional name for the returned op. - - Returns: - The element-wise value of the numerator divided by the denominator. - """ - if compat.forward_compatible(2018, 11, 1): - return math_ops.div_no_nan(numerator, denominator, name=name) - return array_ops.where( - math_ops.greater(denominator, 0), - math_ops.div(numerator, - array_ops.where( - math_ops.equal(denominator, 0), - array_ops.ones_like(denominator), denominator)), - array_ops.zeros_like(numerator), - name=name) - - def _safe_mean(losses, num_present): """Computes a safe mean of the losses. @@ -107,7 +77,7 @@ def _safe_mean(losses, num_present): then zero is returned. """ total_loss = math_ops.reduce_sum(losses) - return _safe_div(total_loss, num_present, name="value") + return math_ops.div_no_nan(total_loss, num_present, name="value") @deprecated("2016-12-30", "Use tf.losses.compute_weighted_loss instead.") @@ -187,10 +157,9 @@ def _num_present(losses, weights, per_batch=False): # First, count the number of nonzero weights: if weights.get_shape().ndims >= 1: - reduction_indices = list(range(1, weights.get_shape().ndims)) + axis = list(range(1, weights.get_shape().ndims)) num_nonzero_per_batch = math_ops.reduce_sum( - math_ops.to_float(math_ops.not_equal(weights, 0)), - reduction_indices=reduction_indices) + math_ops.to_float(math_ops.not_equal(weights, 0)), axis=axis) # Next, determine the number of elements that weights would broadcast to: broadcast_dims = array_ops.slice( @@ -606,20 +575,20 @@ def mean_pairwise_squared_error(predictions, if weights.get_shape().ndims is None: raise ValueError("weights.get_shape().ndims cannot be None") - reduction_indices = list(range(1, diffs.get_shape().ndims)) + axis = list(range(1, diffs.get_shape().ndims)) sum_squares_diff_per_batch = math_ops.reduce_sum( - math_ops.square(diffs), reduction_indices=reduction_indices) + math_ops.square(diffs), axis=axis) num_present_per_batch = _num_present(diffs, weights, per_batch=True) - term1 = 2.0 * _safe_div(sum_squares_diff_per_batch, - num_present_per_batch, - name="value") + term1 = 2.0 * math_ops.div_no_nan( + sum_squares_diff_per_batch, num_present_per_batch, name="value") - sum_diff = math_ops.reduce_sum(diffs, reduction_indices=reduction_indices) - term2 = 2.0 * _safe_div(math_ops.square(sum_diff), - math_ops.square(num_present_per_batch), - name="value") + sum_diff = math_ops.reduce_sum(diffs, axis=axis) + term2 = 2.0 * math_ops.div_no_nan( + math_ops.square(sum_diff), + math_ops.square(num_present_per_batch), + name="value") loss = _scale_losses(term1 - term2, weights) @@ -674,7 +643,7 @@ def cosine_distance(predictions, radial_diffs = math_ops.multiply(predictions, labels) losses = 1 - math_ops.reduce_sum( - radial_diffs, reduction_indices=[ + radial_diffs, axis=[ axis, ]) return compute_weighted_loss(losses, weights, scope=scope) diff --git a/tensorflow/contrib/makefile/download_dependencies.sh b/tensorflow/contrib/makefile/download_dependencies.sh index 0a07588f07f..b396c527673 100755 --- a/tensorflow/contrib/makefile/download_dependencies.sh +++ b/tensorflow/contrib/makefile/download_dependencies.sh @@ -34,7 +34,7 @@ NSYNC_URL="$(grep -o 'https://mirror.bazel.build/github.com/google/nsync/.*tar\. # 1.10 branch does not work. `make distclean` fails and blocks the build # process. For now we're hardcoding to the version which is used by # TensorFlow 1.9. -PROTOBUF_URL="https://mirror.bazel.build/github.com/google/protobuf/archive/396336eb961b75f03b25824fe86cf6490fb75e3a.tar.gz" +PROTOBUF_URL="https://mirror.bazel.build/github.com/google/protobuf/archive/v3.6.0.tar.gz" # TODO (yongtang): Replace the following with 'https://mirror.bazel.build/github.com/google/re2/.*tar\.gz' once # the archive has been propagated in mirror.bazel.build. RE2_URL="$(grep -o 'https://github.com/google/re2/.*tar\.gz' "${BZL_FILE_PATH}" | head -n1)" diff --git a/tensorflow/contrib/makefile/tf_op_files.txt b/tensorflow/contrib/makefile/tf_op_files.txt index e779eff6890..655c7eefcb9 100644 --- a/tensorflow/contrib/makefile/tf_op_files.txt +++ b/tensorflow/contrib/makefile/tf_op_files.txt @@ -157,6 +157,7 @@ tensorflow/core/kernels/mirror_pad_op_cpu_impl_2.cc tensorflow/core/kernels/mirror_pad_op_cpu_impl_3.cc tensorflow/core/kernels/mirror_pad_op_cpu_impl_4.cc tensorflow/core/kernels/mirror_pad_op_cpu_impl_5.cc +tensorflow/core/kernels/multinomial_op.cc tensorflow/core/kernels/no_op.cc tensorflow/core/kernels/non_max_suppression_op.cc tensorflow/core/kernels/one_hot_op.cc @@ -252,6 +253,7 @@ tensorflow/core/kernels/split_op.cc tensorflow/core/kernels/split_v_op.cc tensorflow/core/kernels/stack.cc tensorflow/core/kernels/stack_ops.cc +tensorflow/core/kernels/stateless_random_ops.cc tensorflow/core/kernels/strided_slice_op.cc tensorflow/core/kernels/strided_slice_op_inst_0.cc tensorflow/core/kernels/strided_slice_op_inst_1.cc diff --git a/tensorflow/contrib/metrics/python/metrics/classification.py b/tensorflow/contrib/metrics/python/metrics/classification.py index ac123608650..062deb74b16 100644 --- a/tensorflow/contrib/metrics/python/metrics/classification.py +++ b/tensorflow/contrib/metrics/python/metrics/classification.py @@ -175,7 +175,7 @@ def f1_score(labels, predictions, weights=None, num_thresholds=200, return best_f1 best_f1 = distribution_strategy_context.get_replica_context().merge_call( - f1_across_replicas, values) + f1_across_replicas, args=(values,)) update_op = compute_best_f1_score(tp=update_ops['tp'], fp=update_ops['fp'], fn=update_ops['fn'], name='update') diff --git a/tensorflow/contrib/metrics/python/metrics/classification_test.py b/tensorflow/contrib/metrics/python/metrics/classification_test.py index d6a670f97b3..e789d2cb9df 100644 --- a/tensorflow/contrib/metrics/python/metrics/classification_test.py +++ b/tensorflow/contrib/metrics/python/metrics/classification_test.py @@ -291,12 +291,11 @@ class F1ScoreTest(test.TestCase): labels = labels.astype(np.float32) predictions = predictions.astype(np.float32) - tf_predictions, tf_labels = (dataset_ops.Dataset - .from_tensor_slices((predictions, labels)) - .repeat() - .batch(batch_size) - .make_one_shot_iterator() - .get_next()) + tf_predictions, tf_labels = dataset_ops.make_one_shot_iterator( + dataset_ops.Dataset + .from_tensor_slices((predictions, labels)) + .repeat() + .batch(batch_size)).get_next() f1, f1_op = classification.f1_score(tf_labels, tf_predictions, num_thresholds=3) diff --git a/tensorflow/contrib/metrics/python/ops/metric_ops.py b/tensorflow/contrib/metrics/python/ops/metric_ops.py index d6932f6e4b6..7b432f8bd20 100644 --- a/tensorflow/contrib/metrics/python/ops/metric_ops.py +++ b/tensorflow/contrib/metrics/python/ops/metric_ops.py @@ -24,7 +24,6 @@ from __future__ import print_function import collections as collections_lib -from tensorflow.python.compat import compat from tensorflow.python.eager import context from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops @@ -46,32 +45,6 @@ from tensorflow.python.util.deprecation import deprecated _EPSILON = 1e-7 -def _safe_div(numerator, denominator): - """Computes a safe divide which returns 0 if the denominator is zero. - - Note that the function contains an additional conditional check that is - necessary for avoiding situations where the loss is zero causing NaNs to - creep into the gradient computation. - - Args: - numerator: An arbitrary `Tensor`. - denominator: A `Tensor` whose shape matches `numerator` and whose values are - assumed to be non-negative. - - Returns: - The element-wise value of the numerator divided by the denominator. - """ - if compat.forward_compatible(2018, 11, 1): - return math_ops.div_no_nan(numerator, denominator) - return array_ops.where( - math_ops.greater(denominator, 0), - math_ops.div(numerator, - array_ops.where( - math_ops.equal(denominator, 0), - array_ops.ones_like(denominator), denominator)), - array_ops.zeros_like(numerator)) - - @deprecated(None, 'Please switch to tf.metrics.true_positives. Note that the ' 'order of the labels and predictions arguments has been switched.') def streaming_true_positives(predictions, @@ -3247,24 +3220,20 @@ def streaming_covariance(predictions, # We update the means by Delta=Error*BatchCount/(BatchCount+PrevCount) # batch_mean_prediction is E[x_B] in the update equation - batch_mean_prediction = _safe_div( - math_ops.reduce_sum(weighted_predictions), - batch_count) - delta_mean_prediction = _safe_div( - (batch_mean_prediction - mean_prediction) * batch_count, - update_count) + batch_mean_prediction = math_ops.div_no_nan( + math_ops.reduce_sum(weighted_predictions), batch_count) + delta_mean_prediction = math_ops.div_no_nan( + (batch_mean_prediction - mean_prediction) * batch_count, update_count) update_mean_prediction = state_ops.assign_add(mean_prediction, delta_mean_prediction) # prev_mean_prediction is E[x_A] in the update equation prev_mean_prediction = update_mean_prediction - delta_mean_prediction # batch_mean_label is E[y_B] in the update equation - batch_mean_label = _safe_div( - math_ops.reduce_sum(weighted_labels), - batch_count) - delta_mean_label = _safe_div( - (batch_mean_label - mean_label) * batch_count, - update_count) + batch_mean_label = math_ops.div_no_nan( + math_ops.reduce_sum(weighted_labels), batch_count) + delta_mean_label = math_ops.div_no_nan( + (batch_mean_label - mean_label) * batch_count, update_count) update_mean_label = state_ops.assign_add(mean_label, delta_mean_label) # prev_mean_label is E[y_A] in the update equation prev_mean_label = update_mean_label - delta_mean_label @@ -3447,7 +3416,7 @@ def streaming_mean_cosine_distance(predictions, predictions.get_shape().assert_is_compatible_with(labels.get_shape()) radial_diffs = math_ops.multiply(predictions, labels) radial_diffs = math_ops.reduce_sum( - radial_diffs, reduction_indices=[ + radial_diffs, axis=[ dim, ], keepdims=True) mean_distance, update_op = streaming_mean(radial_diffs, weights, None, None, @@ -3926,9 +3895,8 @@ def cohen_kappa(labels, po_sum = math_ops.reduce_sum(po) total = math_ops.reduce_sum(pe_row) pe_sum = math_ops.reduce_sum( - _safe_div( - math_ops.to_double(pe_row * pe_col), - math_ops.to_double(total))) + math_ops.div_no_nan( + math_ops.to_double(pe_row * pe_col), math_ops.to_double(total))) po_sum, pe_sum, total = (math_ops.to_double(po_sum), math_ops.to_double(pe_sum), math_ops.to_double(total)) diff --git a/tensorflow/contrib/mixed_precision/python/loss_scale_manager_test.py b/tensorflow/contrib/mixed_precision/python/loss_scale_manager_test.py index 1b0383d24c0..c922d0cd11f 100644 --- a/tensorflow/contrib/mixed_precision/python/loss_scale_manager_test.py +++ b/tensorflow/contrib/mixed_precision/python/loss_scale_manager_test.py @@ -29,7 +29,7 @@ from tensorflow.python.platform import test def _GetExampleIter(inputs): dataset = dataset_ops.Dataset.from_tensor_slices(inputs) - return dataset.make_one_shot_iterator() + return dataset_ops.make_one_shot_iterator(dataset) class FixedLossScaleManagerTest(test.TestCase): diff --git a/tensorflow/contrib/mixed_precision/python/loss_scale_optimizer_test.py b/tensorflow/contrib/mixed_precision/python/loss_scale_optimizer_test.py index 9009df0eefe..33f9a43e803 100644 --- a/tensorflow/contrib/mixed_precision/python/loss_scale_optimizer_test.py +++ b/tensorflow/contrib/mixed_precision/python/loss_scale_optimizer_test.py @@ -132,7 +132,7 @@ class LossScaleOptimizerTest(test.TestCase): x = variable_scope.get_variable("x", initializer=1., dtype=dtypes.float32) dataset = dataset_ops.Dataset.from_tensor_slices([np.nan, np.inf, 0.1]) - itr = dataset.make_one_shot_iterator() + itr = dataset_ops.make_one_shot_iterator(dataset) lr = 1 opt = gd.GradientDescentOptimizer(lr) @@ -182,7 +182,7 @@ class LossScaleOptimizerTest(test.TestCase): x = variable_scope.get_variable("x", initializer=1., dtype=dtypes.float32) dataset = dataset_ops.Dataset.from_tensor_slices([np.nan, np.inf, 0.1]) - itr = dataset.make_one_shot_iterator() + itr = dataset_ops.make_one_shot_iterator(dataset) lr = 1 init_loss_scale = 8 diff --git a/tensorflow/contrib/model_pruning/python/layers/core_layers.py b/tensorflow/contrib/model_pruning/python/layers/core_layers.py index f0ce6fe0396..1fa5c8cb485 100644 --- a/tensorflow/contrib/model_pruning/python/layers/core_layers.py +++ b/tensorflow/contrib/model_pruning/python/layers/core_layers.py @@ -21,6 +21,7 @@ from __future__ import print_function from tensorflow.python.framework import ops from tensorflow.python.framework import tensor_shape +from tensorflow.python.keras.engine import input_spec from tensorflow.python.layers import base from tensorflow.python.layers import utils from tensorflow.python.ops import array_ops @@ -119,7 +120,7 @@ class _MaskedConv(base.Layer): self.bias_initializer = bias_initializer self.kernel_regularizer = kernel_regularizer self.bias_regularizer = bias_regularizer - self.input_spec = base.InputSpec(ndim=self.rank + 2) + self.input_spec = input_spec.InputSpec(ndim=self.rank + 2) def build(self, input_shape): input_shape = tensor_shape.TensorShape(input_shape) @@ -171,7 +172,7 @@ class _MaskedConv(base.Layer): dtype=self.dtype) else: self.bias = None - self.input_spec = base.InputSpec( + self.input_spec = input_spec.InputSpec( ndim=self.rank + 2, axes={channel_axis: input_dim}) self.built = True @@ -393,14 +394,14 @@ class MaskedFullyConnected(base.Layer): self.bias_initializer = bias_initializer self.kernel_regularizer = kernel_regularizer self.bias_regularizer = bias_regularizer - self.input_spec = base.InputSpec(min_ndim=2) + self.input_spec = input_spec.InputSpec(min_ndim=2) def build(self, input_shape): input_shape = tensor_shape.TensorShape(input_shape) if tensor_shape.dimension_value(input_shape[-1]) is None: raise ValueError('The last dimension of the inputs to `Dense` ' 'should be defined. Found `None`.') - self.input_spec = base.InputSpec( + self.input_spec = input_spec.InputSpec( min_ndim=2, axes={-1: tensor_shape.dimension_value(input_shape[-1])}) self.kernel = self.add_variable( diff --git a/tensorflow/contrib/opt/python/training/lars_optimizer.py b/tensorflow/contrib/opt/python/training/lars_optimizer.py index a8dafd9a4cb..205d6c39491 100644 --- a/tensorflow/contrib/opt/python/training/lars_optimizer.py +++ b/tensorflow/contrib/opt/python/training/lars_optimizer.py @@ -18,6 +18,7 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function +from tensorflow.python.framework import ops from tensorflow.python.ops import array_ops from tensorflow.python.ops import linalg_ops from tensorflow.python.ops import math_ops @@ -162,3 +163,14 @@ class LARSOptimizer(optimizer.Optimizer): math_ops.cast(self._momentum_tensor, grad.dtype), use_locking=self._use_locking, use_nesterov=self._use_nesterov) + + def _prepare(self): + learning_rate = self._learning_rate + if callable(learning_rate): + learning_rate = learning_rate() + self._learning_rate_tensor = ops.convert_to_tensor(learning_rate, + name="learning_rate") + momentum = self._momentum + if callable(momentum): + momentum = momentum() + self._momentum_tensor = ops.convert_to_tensor(momentum, name="momentum") \ No newline at end of file diff --git a/tensorflow/contrib/opt/python/training/nadam_optimizer.py b/tensorflow/contrib/opt/python/training/nadam_optimizer.py index 155ff5b3f4f..960826407b6 100644 --- a/tensorflow/contrib/opt/python/training/nadam_optimizer.py +++ b/tensorflow/contrib/opt/python/training/nadam_optimizer.py @@ -18,6 +18,7 @@ from __future__ import division from __future__ import print_function from tensorflow.python.framework import ops +from tensorflow.python.ops import array_ops from tensorflow.python.ops import control_flow_ops from tensorflow.python.ops import math_ops from tensorflow.python.ops import state_ops @@ -83,14 +84,14 @@ class NadamOptimizer(adam.AdamOptimizer): with ops.control_dependencies([m_t]): m_t = scatter_add(m, indices, m_scaled_g_values) # m_bar = (1 - beta1) * g_t + beta1 * m_t - m_bar = m_scaled_g_values + beta1_t * m_t + m_bar = m_scaled_g_values + beta1_t * array_ops.gather(m_t, indices) # v_t = beta2 * v + (1 - beta2) * (g_t * g_t) v = self.get_slot(var, "v") v_scaled_g_values = (grad * grad) * (1 - beta2_t) v_t = state_ops.assign(v, v * beta2_t, use_locking=self._use_locking) with ops.control_dependencies([v_t]): v_t = scatter_add(v, indices, v_scaled_g_values) - v_sqrt = math_ops.sqrt(v_t) - var_update = state_ops.assign_sub( - var, lr * m_bar / (v_sqrt + epsilon_t), use_locking=self._use_locking) + v_t_slice = array_ops.gather(v_t, indices) + v_sqrt = math_ops.sqrt(v_t_slice) + var_update = scatter_add(var, indices, -lr * m_bar / (v_sqrt + epsilon_t)) return control_flow_ops.group(*[var_update, m_bar, v_t]) diff --git a/tensorflow/contrib/opt/python/training/nadam_optimizer_test.py b/tensorflow/contrib/opt/python/training/nadam_optimizer_test.py index 85e05ce71ce..a4372f64874 100644 --- a/tensorflow/contrib/opt/python/training/nadam_optimizer_test.py +++ b/tensorflow/contrib/opt/python/training/nadam_optimizer_test.py @@ -52,14 +52,19 @@ def nadam_update_numpy(param, class NadamOptimizerTest(test.TestCase): def doTestSparse(self, use_resource=False): + # need to use a larger value of epsilon here so that + # np.sqrt(v_t) + epsilon doesn't get rounded to 0 when + # the dtype is half and np.sqrt(v_t) = 0, as is the case + # when the gradient is 0 + sparse_epsilon = 1e-7 for dtype in [dtypes.half, dtypes.float32, dtypes.float64]: with self.cached_session(): # Initialize variables for numpy implementation. m0, v0, m1, v1 = 0.0, 0.0, 0.0, 0.0 - var0_np = np.array([1.0, 2.0], dtype=dtype.as_numpy_dtype) - grads0_np = np.array([0.1, 0.1], dtype=dtype.as_numpy_dtype) - var1_np = np.array([3.0, 4.0], dtype=dtype.as_numpy_dtype) - grads1_np = np.array([0.01, 0.01], dtype=dtype.as_numpy_dtype) + var0_np = np.array([1.0, 1.0, 2.0], dtype=dtype.as_numpy_dtype) + grads0_np = np.array([0.1, 0, 0.1], dtype=dtype.as_numpy_dtype) + var1_np = np.array([3.0, 3.0, 4.0], dtype=dtype.as_numpy_dtype) + grads1_np = np.array([0.01, 0, 0.01], dtype=dtype.as_numpy_dtype) if use_resource: var0 = resource_variable_ops.ResourceVariable(var0_np) @@ -67,21 +72,21 @@ class NadamOptimizerTest(test.TestCase): else: var0 = variables.Variable(var0_np) var1 = variables.Variable(var1_np) - grads0_np_indices = np.array([0, 1], dtype=np.int32) + grads0_np_indices = np.array([0, 2], dtype=np.int32) grads0 = ops.IndexedSlices( - constant_op.constant(grads0_np), - constant_op.constant(grads0_np_indices), constant_op.constant([2])) - grads1_np_indices = np.array([0, 1], dtype=np.int32) + constant_op.constant(grads0_np[grads0_np_indices]), + constant_op.constant(grads0_np_indices), constant_op.constant([3])) + grads1_np_indices = np.array([0, 2], dtype=np.int32) grads1 = ops.IndexedSlices( - constant_op.constant(grads1_np), - constant_op.constant(grads1_np_indices), constant_op.constant([2])) - opt = nadam_optimizer.NadamOptimizer() + constant_op.constant(grads1_np[grads1_np_indices]), + constant_op.constant(grads1_np_indices), constant_op.constant([3])) + opt = nadam_optimizer.NadamOptimizer(epsilon=sparse_epsilon) update = opt.apply_gradients(zip([grads0, grads1], [var0, var1])) variables.global_variables_initializer().run() # Fetch params to validate initial values - self.assertAllClose([1.0, 2.0], var0.eval()) - self.assertAllClose([3.0, 4.0], var1.eval()) + self.assertAllClose([1.0, 1.0, 2.0], var0.eval()) + self.assertAllClose([3.0, 3.0, 4.0], var1.eval()) beta1_power, beta2_power = opt._get_beta_accumulators() @@ -91,8 +96,10 @@ class NadamOptimizerTest(test.TestCase): self.assertAllCloseAccordingToType(0.999**t, beta2_power.eval()) update.run() - var0_np, m0, v0 = nadam_update_numpy(var0_np, grads0_np, t, m0, v0) - var1_np, m1, v1 = nadam_update_numpy(var1_np, grads1_np, t, m1, v1) + var0_np, m0, v0 = nadam_update_numpy(var0_np, grads0_np, t, m0, v0, + epsilon=sparse_epsilon) + var1_np, m1, v1 = nadam_update_numpy(var1_np, grads1_np, t, m1, v1, + epsilon=sparse_epsilon) # Validate updated params self.assertAllCloseAccordingToType(var0_np, var0.eval()) diff --git a/tensorflow/contrib/optimizer_v2/BUILD b/tensorflow/contrib/optimizer_v2/BUILD index 3ba3ee29ec7..6e401406308 100644 --- a/tensorflow/contrib/optimizer_v2/BUILD +++ b/tensorflow/contrib/optimizer_v2/BUILD @@ -48,7 +48,6 @@ py_library( srcs_version = "PY2AND3", deps = [ "//tensorflow/python:control_flow_ops", - "//tensorflow/python:distribute", "//tensorflow/python:framework", "//tensorflow/python:math_ops", "//tensorflow/python:resource_variable_ops", @@ -56,6 +55,8 @@ py_library( "//tensorflow/python:training", "//tensorflow/python:variable_scope", "//tensorflow/python:variables", + "//tensorflow/python/distribute:distribute_lib", + "//tensorflow/python/distribute:reduce_util", ], ) diff --git a/tensorflow/contrib/optimizer_v2/optimizer_v2.py b/tensorflow/contrib/optimizer_v2/optimizer_v2.py index 467dd86d8fd..73a556f0b29 100644 --- a/tensorflow/contrib/optimizer_v2/optimizer_v2.py +++ b/tensorflow/contrib/optimizer_v2/optimizer_v2.py @@ -24,6 +24,8 @@ import abc import six +from tensorflow.python.distribute import distribute_lib +from tensorflow.python.distribute import reduce_util as ds_reduce_util from tensorflow.python.eager import backprop from tensorflow.python.eager import context from tensorflow.python.framework import dtypes @@ -34,7 +36,6 @@ from tensorflow.python.ops import math_ops from tensorflow.python.ops import resource_variable_ops from tensorflow.python.ops import variable_scope from tensorflow.python.ops import variables -from tensorflow.python.training import distribute as distribute_lib from tensorflow.python.training import distribution_strategy_context as distribute_ctx from tensorflow.python.training import optimizer as optimizer_v1 from tensorflow.python.training import slot_creator @@ -446,7 +447,7 @@ class _OptimizerV2State(object): if v is None: if colocate_with is None: colocate_with = self._non_slot_devices - with self._distribution.colocate_vars_with(colocate_with): + with self._distribution.extended.colocate_vars_with(colocate_with): # TODO(josh11b): Use get_variable() except for the legacy Adam use case. v = variable_scope.variable(initial_value, name=name, trainable=False) self._non_slot_dict[name] = v @@ -657,7 +658,6 @@ class OptimizerV2(optimizer_v1.Optimizer): var_list=None, gate_gradients=GATE_OP, aggregation_method=None, - colocate_gradients_with_ops=False, name=None, grad_loss=None, stop_gradients=None, @@ -680,8 +680,6 @@ class OptimizerV2(optimizer_v1.Optimizer): `GATE_NONE`, `GATE_OP`, or `GATE_GRAPH`. aggregation_method: Specifies the method used to combine gradient terms. Valid values are defined in the class `AggregationMethod`. - colocate_gradients_with_ops: If True, try colocating gradients with the - corresponding op. name: Optional name for the returned operation. grad_loss: Optional. A `Tensor` holding the gradient computed for `loss`. stop_gradients: Optional. A Tensor or list of tensors not to differentiate @@ -704,8 +702,8 @@ class OptimizerV2(optimizer_v1.Optimizer): Minimization (and gradient computation) is done with respect to the elements of `var_list` if not None, else with respect to any trainable variables created during the execution of the `loss` function. - `gate_gradients`, `aggregation_method`, `colocate_gradients_with_ops` and - `grad_loss` are ignored when eager execution is enabled. + `gate_gradients`, `aggregation_method`, and `grad_loss` are ignored when + eager execution is enabled. @end_compatibility """ grads_and_vars = self.compute_gradients( @@ -713,7 +711,6 @@ class OptimizerV2(optimizer_v1.Optimizer): var_list=var_list, gate_gradients=gate_gradients, aggregation_method=aggregation_method, - colocate_gradients_with_ops=colocate_gradients_with_ops, grad_loss=grad_loss, stop_gradients=stop_gradients, scale_loss_by_num_replicas=scale_loss_by_num_replicas) @@ -733,7 +730,6 @@ class OptimizerV2(optimizer_v1.Optimizer): var_list=None, gate_gradients=GATE_OP, aggregation_method=None, - colocate_gradients_with_ops=False, grad_loss=None, stop_gradients=None, scale_loss_by_num_replicas=None): @@ -756,8 +752,6 @@ class OptimizerV2(optimizer_v1.Optimizer): `GATE_NONE`, `GATE_OP`, or `GATE_GRAPH`. aggregation_method: Specifies the method used to combine gradient terms. Valid values are defined in the class `AggregationMethod`. - colocate_gradients_with_ops: If True, try colocating gradients with the - corresponding op. grad_loss: Optional. A `Tensor` holding the gradient computed for `loss`. stop_gradients: Optional. A Tensor or list of tensors not to differentiate through. @@ -776,8 +770,8 @@ class OptimizerV2(optimizer_v1.Optimizer): not callable. @compatibility(eager) - When eager execution is enabled, `gate_gradients`, `aggregation_method`, - and `colocate_gradients_with_ops` are ignored. + When eager execution is enabled, `gate_gradients`, and `aggregation_method` + are ignored. @end_compatibility """ # TODO(josh11b): Test that we handle weight decay in a reasonable way. @@ -832,7 +826,6 @@ class OptimizerV2(optimizer_v1.Optimizer): grad_ys=grad_loss, gate_gradients=(gate_gradients == optimizer_v1.Optimizer.GATE_OP), aggregation_method=aggregation_method, - colocate_gradients_with_ops=colocate_gradients_with_ops, stop_gradients=stop_gradients) if gate_gradients == optimizer_v1.Optimizer.GATE_GRAPH: grads = control_flow_ops.tuple(grads) @@ -848,8 +841,7 @@ class OptimizerV2(optimizer_v1.Optimizer): """Scale loss for the number of replicas.""" if scale_loss_by_num_replicas is None: scale_loss_by_num_replicas = ( - distribute_lib.get_loss_reduction() == variable_scope - .VariableAggregation.MEAN) + distribute_lib.get_loss_reduction() == ds_reduce_util.ReduceOp.MEAN) if scale_loss_by_num_replicas: num_replicas = \ distribute_ctx.get_distribution_strategy().num_replicas_in_sync @@ -892,7 +884,8 @@ class OptimizerV2(optimizer_v1.Optimizer): raise ValueError("No gradients provided for any variable: %s." % ([str(v) for _, v in grads_and_vars],)) return distribute_ctx.get_replica_context().merge_call( - self._distributed_apply, filtered, global_step=global_step, name=name) + self._distributed_apply, args=(filtered,), + kwargs={"global_step": global_step, "name": name}) def _get_or_create_state(self, var_list=None): """Either looks up or creates `_OptimizerV2State`. @@ -927,8 +920,8 @@ class OptimizerV2(optimizer_v1.Optimizer): def _distributed_apply(self, distribution, grads_and_vars, global_step, name): """`apply_gradients` for use with a `DistributionStrategy`.""" - reduced_grads = distribution.batch_reduce( - variable_scope.VariableAggregation.SUM, grads_and_vars) + reduced_grads = distribution.extended.batch_reduce_to( + ds_reduce_util.ReduceOp.SUM, grads_and_vars) var_list = [v for _, v in grads_and_vars] grads_and_vars = zip(reduced_grads, var_list) @@ -944,7 +937,7 @@ class OptimizerV2(optimizer_v1.Optimizer): with ops.name_scope(name, self._name) as name: per_graph_state = self._get_or_create_state(var_list=unwrapped_var_list) # Include the current value of any dynamic hyper parameters in `state`. - non_slot_devices = distribution.non_slot_devices(var_list) + non_slot_devices = distribution.extended.non_slot_devices(var_list) state = per_graph_state._copy_with_dynamic_hyper( # pylint: disable=protected-access self._hyper, distribution, non_slot_devices) @@ -989,7 +982,8 @@ class OptimizerV2(optimizer_v1.Optimizer): # Use the processors to update the variables. update_ops = [] for grad, var in grads_and_vars: - update_ops.extend(distribution.update(var, update, grad, grouped=False)) + update_ops.extend(distribution.extended.update( + var, update, args=(grad,), group=False)) # Give the child class a chance to do something after applying # gradients @@ -1001,8 +995,8 @@ class OptimizerV2(optimizer_v1.Optimizer): update_ops = control_flow_ops.group(update_ops) with ops.control_dependencies([update_ops]): - finish_updates = distribution.update_non_slot( - non_slot_devices, finish, grouped=False) + finish_updates = distribution.extended.update_non_slot( + non_slot_devices, finish, group=False) # We said grouped=False, which means finish_updates is always a list. # It will be [None] when finish() returns None. if finish_updates == [None]: @@ -1017,8 +1011,8 @@ class OptimizerV2(optimizer_v1.Optimizer): def update_global_step(global_step, name): return global_step.assign_add(1, read_value=False, name=name) - apply_updates = distribution.update(global_step, update_global_step, - name) + apply_updates = distribution.extended.update( + global_step, update_global_step, args=(name,)) # Add the training op to the TRAIN_OP graph collection in graph mode. if not eager_execution: diff --git a/tensorflow/contrib/predictor/BUILD b/tensorflow/contrib/predictor/BUILD index d50b52b8ff1..53a3bc63e1d 100644 --- a/tensorflow/contrib/predictor/BUILD +++ b/tensorflow/contrib/predictor/BUILD @@ -42,6 +42,7 @@ py_library( name = "saved_model_predictor", srcs = ["saved_model_predictor.py"], srcs_version = "PY2AND3", + visibility = ["//learning/brain/contrib/learn/tpu:__subpackages__"], deps = [ ":base_predictor", "//tensorflow/contrib/saved_model:saved_model_py", diff --git a/tensorflow/contrib/quantize/README.md b/tensorflow/contrib/quantize/README.md index a1f2b590266..9085d9fa719 100644 --- a/tensorflow/contrib/quantize/README.md +++ b/tensorflow/contrib/quantize/README.md @@ -28,7 +28,7 @@ Since it's difficult to add these fake quantization operations to all the required locations in the model, there's a function available that rewrites the training graph. To create a fake quantized training graph: -``` +```python # Build forward pass of model. loss = tf.losses.get_total_loss() @@ -51,7 +51,7 @@ The rewritten *eval graph* is non-trivially different from the *training graph* since the quantization ops affect the batch normalization step. Because of this, we've added a separate rewrite for the *eval graph*: -``` +```python # Build eval model logits = tf.nn.softmax_cross_entropy_with_logits_v2(...) diff --git a/tensorflow/contrib/quantize/python/quant_ops.py b/tensorflow/contrib/quantize/python/quant_ops.py index 6f659347fba..8619708cdae 100644 --- a/tensorflow/contrib/quantize/python/quant_ops.py +++ b/tensorflow/contrib/quantize/python/quant_ops.py @@ -138,7 +138,7 @@ def LastValueQuantize(inputs, if per_channel: if input_dim >= 2: batch_min = math_ops.reduce_min( - inputs, reduction_indices=reduce_dims, name='BatchMin') + inputs, axis=reduce_dims, name='BatchMin') else: batch_min = inputs else: @@ -147,7 +147,7 @@ def LastValueQuantize(inputs, if per_channel: if input_dim >= 2: batch_max = math_ops.reduce_max( - inputs, reduction_indices=reduce_dims, name='BatchMax') + inputs, axis=reduce_dims, name='BatchMax') else: batch_max = inputs else: @@ -263,7 +263,7 @@ def MovingAvgQuantize(inputs, if per_channel: if input_dim >= 2: batch_min = math_ops.reduce_min( - inputs, reduction_indices=reduce_dims, name='BatchMin') + inputs, axis=reduce_dims, name='BatchMin') else: batch_min = inputs else: @@ -272,7 +272,7 @@ def MovingAvgQuantize(inputs, if per_channel: if input_dim >= 2: batch_max = math_ops.reduce_max( - inputs, reduction_indices=reduce_dims, name='BatchMax') + inputs, axis=reduce_dims, name='BatchMax') else: batch_max = inputs else: diff --git a/tensorflow/contrib/quantize/python/quantize.py b/tensorflow/contrib/quantize/python/quantize.py index 338923f7512..21d1b121309 100644 --- a/tensorflow/contrib/quantize/python/quantize.py +++ b/tensorflow/contrib/quantize/python/quantize.py @@ -160,7 +160,7 @@ def Quantize(graph, # shouldn't quantize it, since the activation will be Fused into the # Add at inference time. consumers = input_to_ops_map.ConsumerOperations(layer_match.bypass_op) - if any([consumer.type in _ACTIVATION_TYPES for consumer in consumers]): + if any(consumer.type in _ACTIVATION_TYPES for consumer in consumers): logging.info('Skipping %s, because its followed by an activation.', layer_match.bypass_op.name) else: @@ -195,7 +195,7 @@ def Quantize(graph, # Add at inference time. consumers = input_to_ops_map.ConsumerOperations( layer_match.post_activation_bypass_op) - if any([consumer.type in _RELU_TYPES for consumer in consumers]): + if any(consumer.type in _RELU_TYPES for consumer in consumers): logging.info('Skipping %s, because its followed by an activation.', layer_match.post_activation_bypass_op.name) else: diff --git a/tensorflow/contrib/resampler/BUILD b/tensorflow/contrib/resampler/BUILD index 38fcca03116..bbf10996759 100644 --- a/tensorflow/contrib/resampler/BUILD +++ b/tensorflow/contrib/resampler/BUILD @@ -13,6 +13,7 @@ load( ) load("//tensorflow:tensorflow.bzl", "cuda_py_test") load("//tensorflow:tensorflow.bzl", "tf_custom_op_py_library") +load("//tensorflow/compiler/tests:build_defs.bzl", "tf_xla_py_test") tf_custom_op_py_library( name = "resampler_py", @@ -50,10 +51,14 @@ tf_kernel_library( prefix = "resampler_ops", deps = [ ":resampler_ops_op_lib", - "//tensorflow/compiler/tf2xla/kernels:resampler_ops", "//tensorflow/core:framework", "//tensorflow/core:lib", - ], + ] + select({ + "//tensorflow:with_xla_support": [ + "//tensorflow/compiler/tf2xla/kernels:resampler_ops", + ], + "//conditions:default": [], + }), alwayslink = 1, ) @@ -94,3 +99,26 @@ cuda_py_test( "//tensorflow/python:array_ops", ], ) + +tf_xla_py_test( + name = "resampler_ops_xla_test", + size = "small", + srcs = ["xla/resampler_ops_xla_test.py"], + disabled_backends = [ + # TODO(b/74459949) Support BatchDot in CPU backend. + "cpu", + "cpu_ondemand", + ], + # TODO(b/112295522): the OSS build will not likely work in the short to medium term, currently it is blocked by the fact that bazel does not allow py_library to depend on cc_library: https://github.com/bazelbuild/bazel/issues/701 which may not be resolvable. + tags = ["no_oss"], + deps = [ + "//tensorflow/compiler/tests:xla_test", + "//tensorflow/compiler/tf2xla/kernels:resampler_ops", + "//tensorflow/contrib/resampler:resampler_ops", + "//tensorflow/contrib/resampler:resampler_py", + "//tensorflow/python:array_ops", + "//tensorflow/python:client_testlib", + "//tensorflow/python:platform_test", + "//third_party/py/numpy", + ], +) diff --git a/tensorflow/compiler/tests/resampler_ops_test.py b/tensorflow/contrib/resampler/xla/resampler_ops_xla_test.py similarity index 76% rename from tensorflow/compiler/tests/resampler_ops_test.py rename to tensorflow/contrib/resampler/xla/resampler_ops_xla_test.py index f87ac3360c9..d8ca0eab276 100644 --- a/tensorflow/compiler/tests/resampler_ops_test.py +++ b/tensorflow/contrib/resampler/xla/resampler_ops_xla_test.py @@ -63,8 +63,8 @@ class ResamplerOpsTest(xla_test.XLATestCase): def testSimple(self): for dtype in self.float_types: input_shape = [1, 2, 2, 1] - input_rgb_data = [0, 5, 13, 54] - input_np = np.array(input_rgb_data, dtype=dtype).reshape(input_shape) + input_data = [0, 5, 13, 54] + input_np = np.array(input_data, dtype=dtype).reshape(input_shape) warp_shape = [1, 2] warp_data = [0.7, 0.6] @@ -151,6 +151,55 @@ class ResamplerOpsTest(xla_test.XLATestCase): expected_grad_data, expected_grad_warp) + def testOutOfBoundWarps(self): + # (x, y) are both less than 0. + for dtype in self.float_types: + input_shape = [1, 2, 2, 1] + input_data = [10, 5, 13, 54] + input_np = np.array(input_data, dtype=dtype).reshape(input_shape) + + warp_shape = [1, 2, 2] + warp_data = [-1, -1, 0.7, 0.6] + warp_np = np.array(warp_data, dtype=dtype).reshape(warp_shape) + expected = [[[0.0], [27.62]]] + self._assertForwardOpMatchesExpected(input_np, warp_np, expected) + + # One of (x, y) is less than 0. + for dtype in self.float_types: + input_shape = [1, 2, 2, 1] + input_data = [10, 5, 13, 54] + input_np = np.array(input_data, dtype=dtype).reshape(input_shape) + + warp_shape = [1, 2, 2] + warp_data = [-1, 0.1, 0.7, 0.6] + warp_np = np.array(warp_data, dtype=dtype).reshape(warp_shape) + expected = [[[0.0], [27.62]]] + self._assertForwardOpMatchesExpected(input_np, warp_np, expected) + + # Both of (x, y) are greater than image size. + for dtype in self.float_types: + input_shape = [1, 2, 2, 1] + input_data = [10, 5, 13, 54] + input_np = np.array(input_data, dtype=dtype).reshape(input_shape) + + warp_shape = [1, 2, 2] + warp_data = [-0.1, 0.1, 1.2, 2.1] + warp_np = np.array(warp_data, dtype=dtype).reshape(warp_shape) + expected = [[[0.0], [0.0]]] + self._assertForwardOpMatchesExpected(input_np, warp_np, expected) + + # One of (x, y) is greater than image size. + for dtype in self.float_types: + input_shape = [1, 2, 2, 1] + input_data = [10, 5, 13, 54] + input_np = np.array(input_data, dtype=dtype).reshape(input_shape) + + warp_shape = [1, 2, 2] + warp_data = [0.1, -0.1, 1.2, 0.1] + warp_np = np.array(warp_data, dtype=dtype).reshape(warp_shape) + expected = [[[0.0], [0.0]]] + self._assertForwardOpMatchesExpected(input_np, warp_np, expected) + if __name__ == '__main__': test.main() diff --git a/tensorflow/contrib/rnn/python/kernel_tests/core_rnn_cell_test.py b/tensorflow/contrib/rnn/python/kernel_tests/core_rnn_cell_test.py index 245fa68eaef..7d57b0413a3 100644 --- a/tensorflow/contrib/rnn/python/kernel_tests/core_rnn_cell_test.py +++ b/tensorflow/contrib/rnn/python/kernel_tests/core_rnn_cell_test.py @@ -906,7 +906,7 @@ class DropoutWrapperTest(test.TestCase): def testDropoutWrapperKeepNoOutput(self): keep_all = variable_scope.get_variable("all", initializer=1.0) - keep_none = variable_scope.get_variable("none", initializer=1e-10) + keep_none = variable_scope.get_variable("none", initializer=1e-6) res = self._testDropoutWrapper( input_keep_prob=keep_all, output_keep_prob=keep_none, @@ -922,7 +922,7 @@ class DropoutWrapperTest(test.TestCase): def testDropoutWrapperKeepNoStateExceptLSTMCellMemory(self): keep_all = variable_scope.get_variable("all", initializer=1.0) - keep_none = variable_scope.get_variable("none", initializer=1e-10) + keep_none = variable_scope.get_variable("none", initializer=1e-6) # Even though we dropout state, by default DropoutWrapper never # drops out the memory ("c") term of an LSTMStateTuple. res = self._testDropoutWrapper( @@ -943,7 +943,7 @@ class DropoutWrapperTest(test.TestCase): def testDropoutWrapperKeepNoInput(self): keep_all = variable_scope.get_variable("all", initializer=1.0) - keep_none = variable_scope.get_variable("none", initializer=1e-10) + keep_none = variable_scope.get_variable("none", initializer=1e-6) true_full_output = np.array( [[[0.751109, 0.751109, 0.751109]], [[0.895509, 0.895509, 0.895509]]], dtype=np.float32) diff --git a/tensorflow/contrib/rnn/python/kernel_tests/core_rnn_test.py b/tensorflow/contrib/rnn/python/kernel_tests/core_rnn_test.py index 5cba54dd3df..ef372b947ce 100644 --- a/tensorflow/contrib/rnn/python/kernel_tests/core_rnn_test.py +++ b/tensorflow/contrib/rnn/python/kernel_tests/core_rnn_test.py @@ -227,7 +227,7 @@ class RNNTest(test.TestCase): def testDropout(self): cell = Plus1RNNCell() full_dropout_cell = rnn_cell.DropoutWrapper( - cell, input_keep_prob=1e-12, seed=0) + cell, input_keep_prob=1e-6, seed=0) (name, dep), = full_dropout_cell._checkpoint_dependencies self.assertIs(dep, cell) self.assertEqual("cell", name) diff --git a/tensorflow/contrib/rnn/python/ops/gru_ops.py b/tensorflow/contrib/rnn/python/ops/gru_ops.py index b30ca7882fc..251a933eaec 100644 --- a/tensorflow/contrib/rnn/python/ops/gru_ops.py +++ b/tensorflow/contrib/rnn/python/ops/gru_ops.py @@ -21,7 +21,7 @@ from tensorflow.contrib.rnn.ops import gen_gru_ops from tensorflow.contrib.util import loader from tensorflow.python.framework import ops from tensorflow.python.framework import tensor_shape -from tensorflow.python.layers import base as base_layer +from tensorflow.python.keras.engine import input_spec from tensorflow.python.ops import array_ops from tensorflow.python.ops import init_ops from tensorflow.python.ops import math_ops @@ -165,7 +165,7 @@ class GRUBlockCell(LayerRNNCell): num_units = cell_size self._cell_size = num_units # Inputs must be 2-dimensional. - self.input_spec = base_layer.InputSpec(ndim=2) + self.input_spec = input_spec.InputSpec(ndim=2) @property def state_size(self): diff --git a/tensorflow/contrib/rnn/python/ops/lstm_ops.py b/tensorflow/contrib/rnn/python/ops/lstm_ops.py index 4db431f85a4..b043026bc55 100644 --- a/tensorflow/contrib/rnn/python/ops/lstm_ops.py +++ b/tensorflow/contrib/rnn/python/ops/lstm_ops.py @@ -25,6 +25,7 @@ from tensorflow.contrib.rnn.ops import gen_lstm_ops from tensorflow.contrib.util import loader from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops +from tensorflow.python.keras.engine import input_spec from tensorflow.python.layers import base as base_layer from tensorflow.python.ops import array_ops from tensorflow.python.ops import init_ops @@ -385,7 +386,7 @@ class LSTMBlockCell(LayerRNNCell): "scope": "lstm_cell" } # Inputs must be 2-dimensional. - self.input_spec = base_layer.InputSpec(ndim=2) + self.input_spec = input_spec.InputSpec(ndim=2) @property def state_size(self): @@ -628,7 +629,7 @@ class LSTMBlockFusedCell(LSTMBlockWrapper): self._use_peephole = use_peephole # Inputs must be 3-dimensional. - self.input_spec = base_layer.InputSpec(ndim=3) + self.input_spec = input_spec.InputSpec(ndim=3) @property def num_units(self): diff --git a/tensorflow/contrib/rnn/python/ops/rnn_cell.py b/tensorflow/contrib/rnn/python/ops/rnn_cell.py index e159dc95796..8a1c09f171e 100644 --- a/tensorflow/contrib/rnn/python/ops/rnn_cell.py +++ b/tensorflow/contrib/rnn/python/ops/rnn_cell.py @@ -30,7 +30,7 @@ from tensorflow.python.framework import ops from tensorflow.python.framework import tensor_shape from tensorflow.python.keras import activations from tensorflow.python.keras import initializers -from tensorflow.python.layers import base as base_layer +from tensorflow.python.keras.engine import input_spec from tensorflow.python.ops import array_ops from tensorflow.python.ops import clip_ops from tensorflow.python.ops import gen_array_ops @@ -2752,7 +2752,7 @@ class SRUCell(rnn_cell_impl.LayerRNNCell): self._activation = activation or math_ops.tanh # Restrict inputs to be 2-dimensional matrices - self.input_spec = base_layer.InputSpec(ndim=2) + self.input_spec = input_spec.InputSpec(ndim=2) @property def state_size(self): @@ -3089,7 +3089,7 @@ class IndRNNCell(rnn_cell_impl.LayerRNNCell): super(IndRNNCell, self).__init__(_reuse=reuse, name=name, dtype=dtype) # Inputs must be 2-dimensional. - self.input_spec = base_layer.InputSpec(ndim=2) + self.input_spec = input_spec.InputSpec(ndim=2) self._num_units = num_units self._activation = activation or math_ops.tanh @@ -3183,7 +3183,7 @@ class IndyGRUCell(rnn_cell_impl.LayerRNNCell): super(IndyGRUCell, self).__init__(_reuse=reuse, name=name, dtype=dtype) # Inputs must be 2-dimensional. - self.input_spec = base_layer.InputSpec(ndim=2) + self.input_spec = input_spec.InputSpec(ndim=2) self._num_units = num_units self._activation = activation or math_ops.tanh @@ -3323,7 +3323,7 @@ class IndyLSTMCell(rnn_cell_impl.LayerRNNCell): super(IndyLSTMCell, self).__init__(_reuse=reuse, name=name, dtype=dtype) # Inputs must be 2-dimensional. - self.input_spec = base_layer.InputSpec(ndim=2) + self.input_spec = input_spec.InputSpec(ndim=2) self._num_units = num_units self._forget_bias = forget_bias @@ -3444,7 +3444,7 @@ class MinimalRNNCell(rnn_cell_impl.LayerRNNCell): super(MinimalRNNCell, self).__init__(name=name, dtype=dtype, **kwargs) # Inputs must be 2-dimensional. - self.input_spec = base_layer.InputSpec(ndim=2) + self.input_spec = input_spec.InputSpec(ndim=2) self.units = units self.activation = activations.get(activation) @@ -3558,7 +3558,7 @@ class CFNCell(rnn_cell_impl.LayerRNNCell): super(CFNCell, self).__init__(name=name, dtype=dtype, **kwargs) # Inputs must be 2-dimensional. - self.input_spec = base_layer.InputSpec(ndim=2) + self.input_spec = input_spec.InputSpec(ndim=2) self.units = units self.activation = activations.get(activation) diff --git a/tensorflow/contrib/saved_model/BUILD b/tensorflow/contrib/saved_model/BUILD index f0947fe423f..269443b2c65 100644 --- a/tensorflow/contrib/saved_model/BUILD +++ b/tensorflow/contrib/saved_model/BUILD @@ -102,7 +102,10 @@ py_test( size = "medium", srcs = ["python/saved_model/keras_saved_model_test.py"], srcs_version = "PY2AND3", - tags = ["no_windows"], + tags = [ + "no_oss", # TODO(b/119349471): Re-enable + "no_windows", + ], deps = [ ":keras_saved_model", "//tensorflow/python:client_testlib", diff --git a/tensorflow/contrib/saved_model/python/saved_model/keras_saved_model.py b/tensorflow/contrib/saved_model/python/saved_model/keras_saved_model.py index 27b5b6d22e0..ffba514bb96 100644 --- a/tensorflow/contrib/saved_model/python/saved_model/keras_saved_model.py +++ b/tensorflow/contrib/saved_model/python/saved_model/keras_saved_model.py @@ -25,7 +25,6 @@ from tensorflow.python.client import session from tensorflow.python.estimator import keras as estimator_keras_util from tensorflow.python.estimator import model_fn as model_fn_lib from tensorflow.python.estimator.export import export as export_helpers -from tensorflow.python.framework import errors from tensorflow.python.framework import ops from tensorflow.python.keras import backend as K from tensorflow.python.keras import models as models_lib @@ -126,7 +125,7 @@ def save_keras_model( export_dir = export_helpers.get_timestamped_export_dir(saved_model_path) temp_export_dir = export_helpers.get_temp_export_dir(export_dir) - builder = saved_model_builder.SavedModelBuilder(temp_export_dir) + builder = saved_model_builder._SavedModelBuilder(temp_export_dir) # Manually save variables to export them in an object-based checkpoint. This # skips the `builder.add_meta_graph_and_variables()` step, which saves a @@ -228,9 +227,10 @@ def _export_mode( g.add_to_collection(ops.GraphKeys.GLOBAL_STEP, clone.optimizer.iterations) # Extract update and train ops from train/test/predict functions. + train_op = None if mode == model_fn_lib.ModeKeys.TRAIN: clone._make_train_function() - builder._add_train_op(clone.train_function.updates_op) + train_op = clone.train_function.updates_op elif mode == model_fn_lib.ModeKeys.EVAL: clone._make_test_function() else: @@ -265,7 +265,8 @@ def _export_mode( model_fn_lib.EXPORT_TAG_MAP[mode], signature_def_map=_create_signature_def_map(clone, mode), saver=saver_lib.Saver(clone_var_list), - main_op=variables.local_variables_initializer()) + init_op=variables.local_variables_initializer(), + train_op=train_op) return None @@ -307,31 +308,11 @@ def _create_signature_def_map(model, mode): serving_only=(mode == model_fn_lib.ModeKeys.PREDICT)) -def _assert_same_non_optimizer_objects(model, model_graph, clone, clone_graph): +def _assert_same_non_optimizer_objects(model, model_graph, clone, clone_graph): # pylint: disable=unused-argument """Assert model and clone contain the same checkpointable objects.""" - def get_non_optimizer_objects(m, g): - """Gather set of model and optimizer checkpointable objects.""" - # Set default graph because optimizer.variables() returns optimizer - # variables defined in the default graph. - with g.as_default(): - all_objects = set(checkpointable_utils.list_objects(m)) - optimizer_and_variables = set() - for obj in all_objects: - if isinstance(obj, optimizers.TFOptimizer): - optimizer_and_variables.update(checkpointable_utils.list_objects(obj)) - optimizer_and_variables.update(set(obj.optimizer.variables())) - return all_objects - optimizer_and_variables - - model_objects = get_non_optimizer_objects(model, model_graph) - clone_objects = get_non_optimizer_objects(clone, clone_graph) - - if len(model_objects) != len(clone_objects): - raise errors.InternalError( - None, None, - 'Model and clone must use the same variables.' - '\n\tModel variables: %s\n\t Clone variables: %s' - % (model_objects, clone_objects)) + # TODO(fchollet, kathywu): make sure this works in eager mode. + return True def load_keras_model(saved_model_path): diff --git a/tensorflow/contrib/saved_model/python/saved_model/keras_saved_model_test.py b/tensorflow/contrib/saved_model/python/saved_model/keras_saved_model_test.py index a65b2ce4661..93d73e1b484 100644 --- a/tensorflow/contrib/saved_model/python/saved_model/keras_saved_model_test.py +++ b/tensorflow/contrib/saved_model/python/saved_model/keras_saved_model_test.py @@ -29,14 +29,12 @@ from tensorflow.python import keras from tensorflow.python.client import session from tensorflow.python.eager import context from tensorflow.python.estimator import model_fn as model_fn_lib -from tensorflow.python.framework import errors from tensorflow.python.framework import ops from tensorflow.python.framework import test_util from tensorflow.python.keras.engine import training from tensorflow.python.keras.utils import tf_utils from tensorflow.python.ops import array_ops from tensorflow.python.platform import test -from tensorflow.python.saved_model import constants from tensorflow.python.saved_model import loader_impl from tensorflow.python.saved_model import signature_constants from tensorflow.python.training import training as training_module @@ -255,7 +253,7 @@ def load_model(sess, path, mode): outputs = { k: sess.graph.get_tensor_by_name(v.name) for k, v in meta_graph_def.signature_def[sig_def_key].outputs.items()} - return inputs, outputs + return inputs, outputs, meta_graph_def @test_util.run_all_in_graph_and_eager_modes @@ -332,8 +330,8 @@ class TestModelSavedModelExport(test.TestCase, parameterized.TestCase): # Load predict graph, and test predictions with session.Session(graph=ops.Graph()) as sess: - inputs, outputs = load_model(sess, output_path, - model_fn_lib.ModeKeys.PREDICT) + inputs, outputs, _ = load_model(sess, output_path, + model_fn_lib.ModeKeys.PREDICT) predictions = sess.run(outputs[output_name], {inputs[input_name]: input_arr}) @@ -342,19 +340,21 @@ class TestModelSavedModelExport(test.TestCase, parameterized.TestCase): if optimizer: # Load eval graph, and test predictions, loss and metric values with session.Session(graph=ops.Graph()) as sess: - inputs, outputs = load_model(sess, output_path, - model_fn_lib.ModeKeys.EVAL) + inputs, outputs, _ = load_model(sess, output_path, + model_fn_lib.ModeKeys.EVAL) # First obtain the loss and predictions, and run the metric update op by # feeding in the inputs and targets. loss, predictions, _ = sess.run( (outputs['loss'], outputs['predictions/' + output_name], - outputs['metrics/mae/update_op']), - {inputs[input_name]: input_arr, inputs[target_name]: target_arr}) + outputs['metrics/mean_absolute_error/update_op']), { + inputs[input_name]: input_arr, + inputs[target_name]: target_arr + }) # The metric value should be run after the update op, to ensure that it # reflects the correct value. - metric_value = sess.run(outputs['metrics/mae/value']) + metric_value = sess.run(outputs['metrics/mean_absolute_error/value']) self.assertEqual(int(train_before_export), sess.run(training_module.get_global_step())) @@ -364,17 +364,17 @@ class TestModelSavedModelExport(test.TestCase, parameterized.TestCase): # Load train graph, and check for the train op, and prediction values with session.Session(graph=ops.Graph()) as sess: - inputs, outputs = load_model(sess, output_path, - model_fn_lib.ModeKeys.TRAIN) + inputs, outputs, meta_graph_def = load_model( + sess, output_path, model_fn_lib.ModeKeys.TRAIN) self.assertEqual(int(train_before_export), sess.run(training_module.get_global_step())) self.assertIn('loss', outputs) - self.assertIn('metrics/mae/update_op', outputs) - self.assertIn('metrics/mae/value', outputs) + self.assertIn('metrics/mean_absolute_error/update_op', outputs) + self.assertIn('metrics/mean_absolute_error/value', outputs) self.assertIn('predictions/' + output_name, outputs) # Train for a step - train_op = ops.get_collection(constants.TRAIN_OP_KEY) + train_op = loader_impl.get_train_op(meta_graph_def) train_outputs, _ = sess.run( [outputs, train_op], {inputs[input_name]: input_arr, inputs[target_name]: target_arr}) @@ -401,8 +401,8 @@ class TestModelSavedModelExport(test.TestCase, parameterized.TestCase): output_path = keras_saved_model.save_keras_model( model, saved_model_path, custom_objects={'relu6': relu6}) with session.Session(graph=ops.Graph()) as sess: - inputs, outputs = load_model(sess, output_path, - model_fn_lib.ModeKeys.PREDICT) + inputs, outputs, _ = load_model(sess, output_path, + model_fn_lib.ModeKeys.PREDICT) input_name = model.input_names[0] output_name = model.output_names[0] predictions = sess.run( @@ -463,11 +463,6 @@ class TestModelSavedModelExport(test.TestCase, parameterized.TestCase): clone.compile(loss='mse', optimizer=keras.optimizers.RMSprop(lr=0.0001)) clone.train_on_batch(input_arr, target_arr) - with self.assertRaisesRegexp( - errors.InternalError, 'Model and clone must use the same variables.'): - keras_saved_model._assert_same_non_optimizer_objects( - model, model_graph, clone, clone_graph) - def testSaveSeqModelWithoutInputShapesRaisesError(self): """A Sequential model that hasn't been built should raise an error.""" model = sequential_model_without_input_shape(True) diff --git a/tensorflow/contrib/seq2seq/python/kernel_tests/attention_wrapper_test.py b/tensorflow/contrib/seq2seq/python/kernel_tests/attention_wrapper_test.py index 8668c67cf95..922f21b98b3 100644 --- a/tensorflow/contrib/seq2seq/python/kernel_tests/attention_wrapper_test.py +++ b/tensorflow/contrib/seq2seq/python/kernel_tests/attention_wrapper_test.py @@ -154,8 +154,8 @@ class AttentionWrapperTest(test.TestCase): if attention_layer_sizes is not None: # Compute sum of attention_layer_sizes. Use encoder_output_depth if None. - attention_depth = sum([attention_layer_size or encoder_output_depth - for attention_layer_size in attention_layer_sizes]) + attention_depth = sum(attention_layer_size or encoder_output_depth + for attention_layer_size in attention_layer_sizes) elif attention_layers is not None: # Compute sum of attention_layers output depth. attention_depth = sum( diff --git a/tensorflow/contrib/summary/summary_ops_test.py b/tensorflow/contrib/summary/summary_ops_test.py index 4d1807130c5..10e4556dacb 100644 --- a/tensorflow/contrib/summary/summary_ops_test.py +++ b/tensorflow/contrib/summary/summary_ops_test.py @@ -152,6 +152,27 @@ class EagerFileTest(test_util.TensorFlowTestCase): self.assertEqual(len(events), 2) self.assertEqual(events[1].summary.value[0].tag, 'scalar') + def testRecordEveryNGlobalSteps(self): + step = training_util.get_or_create_global_step() + logdir = tempfile.mkdtemp() + + def run_step(): + summary_ops.scalar('scalar', i, step=step) + step.assign_add(1) + + with summary_ops.create_file_writer( + logdir).as_default(), summary_ops.record_summaries_every_n_global_steps( + 2, step): + for i in range(10): + run_step() + # And another 10 steps as a graph function. + run_step_fn = function.defun(run_step) + for i in range(10): + run_step_fn() + + events = summary_test_util.events_from_logdir(logdir) + self.assertEqual(len(events), 11) + def testMaxQueue(self): logs = tempfile.mkdtemp() with summary_ops.create_file_writer( @@ -279,12 +300,9 @@ class EagerDbTest(summary_test_util.SummaryDbTest): def testDbURIOpen(self): tmpdb_path = os.path.join(self.get_temp_dir(), 'tmpDbURITest.sqlite') - tmpdb_uri = six.moves.urllib_parse.urljoin("file:", tmpdb_path) - tmpdb_writer = summary_ops.create_db_writer( - tmpdb_uri, - "experimentA", - "run1", - "user1") + tmpdb_uri = six.moves.urllib_parse.urljoin('file:', tmpdb_path) + tmpdb_writer = summary_ops.create_db_writer(tmpdb_uri, 'experimentA', + 'run1', 'user1') with summary_ops.always_record_summaries(): with tmpdb_writer.as_default(): summary_ops.scalar('t1', 2.0) diff --git a/tensorflow/contrib/tensorboard/db/summary_file_writer.cc b/tensorflow/contrib/tensorboard/db/summary_file_writer.cc index 3f24f58f03a..22b6f09d0cd 100644 --- a/tensorflow/contrib/tensorboard/db/summary_file_writer.cc +++ b/tensorflow/contrib/tensorboard/db/summary_file_writer.cc @@ -73,7 +73,16 @@ class SummaryFileWriter : public SummaryWriterInterface { e->set_step(global_step); e->set_wall_time(GetWallTime()); Summary::Value* v = e->mutable_summary()->add_value(); - t.AsProtoTensorContent(v->mutable_tensor()); + + if (t.dtype() == DT_STRING) { + // Treat DT_STRING specially, so that tensor_util.MakeNdarray in Python + // can convert the TensorProto to string-type numpy array. MakeNdarray + // does not work with strings encoded by AsProtoTensorContent() in + // tensor_content. + t.AsProtoField(v->mutable_tensor()); + } else { + t.AsProtoTensorContent(v->mutable_tensor()); + } v->set_tag(tag); if (!serialized_metadata.empty()) { v->mutable_metadata()->ParseFromString(serialized_metadata); diff --git a/tensorflow/contrib/tensorboard/db/summary_file_writer_test.cc b/tensorflow/contrib/tensorboard/db/summary_file_writer_test.cc index cd3f712256f..ffbfb9533e8 100644 --- a/tensorflow/contrib/tensorboard/db/summary_file_writer_test.cc +++ b/tensorflow/contrib/tensorboard/db/summary_file_writer_test.cc @@ -15,6 +15,7 @@ limitations under the License. #include "tensorflow/contrib/tensorboard/db/summary_file_writer.h" #include "tensorflow/core/framework/summary.pb.h" +#include "tensorflow/core/framework/tensor.pb.h" #include "tensorflow/core/lib/core/errors.h" #include "tensorflow/core/lib/core/refcount.h" #include "tensorflow/core/lib/io/path.h" @@ -104,6 +105,23 @@ TEST_F(SummaryFileWriterTest, WriteTensor) { CHECK_EQ(e.summary().value_size(), 1); EXPECT_EQ(e.summary().value(0).tag(), "name"); })); + TF_CHECK_OK(SummaryTestHelper( + "string_tensor_test", + [](SummaryWriterInterface* writer) { + Tensor hello(DT_STRING, TensorShape({})); + hello.scalar()() = "hello"; + TF_RETURN_IF_ERROR(writer->WriteTensor( + 2, hello, "name", SummaryMetadata().SerializeAsString())); + TF_RETURN_IF_ERROR(writer->Flush()); + return Status::OK(); + }, + [](const Event& e) { + EXPECT_EQ(e.step(), 2); + CHECK_EQ(e.summary().value_size(), 1); + EXPECT_EQ(e.summary().value(0).tag(), "name"); + EXPECT_EQ(e.summary().value(0).tensor().dtype(), DT_STRING); + EXPECT_EQ(e.summary().value(0).tensor().string_val()[0], "hello"); + })); } TEST_F(SummaryFileWriterTest, WriteScalar) { diff --git a/tensorflow/contrib/tensorrt/BUILD b/tensorflow/contrib/tensorrt/BUILD index 20bcd2447e6..784acce444a 100644 --- a/tensorflow/contrib/tensorrt/BUILD +++ b/tensorflow/contrib/tensorrt/BUILD @@ -29,6 +29,10 @@ load( "if_tensorrt", ) +exports_files(glob([ + "test/testdata/*", +])) + tf_cuda_cc_test( name = "tensorrt_test_cc", size = "small", @@ -491,6 +495,7 @@ cuda_py_tests( "test/memory_alignment_test.py", "test/multi_connection_neighbor_engine_test.py", "test/neighboring_engine_test.py", + "test/quantization_test.py", "test/rank_two_test.py", "test/reshape_transpose_test.py", "test/vgg_block_nchw_test.py", @@ -527,6 +532,30 @@ cuda_py_tests( ], ) +cuda_py_test( + name = "quantization_mnist_test", + srcs = ["test/quantization_mnist_test.py"], + additional_deps = [ + ":tf_trt_integration_test_base", + "//tensorflow/python:client_testlib", + "//tensorflow/python:framework_test_lib", + "//tensorflow/python/keras:keras", + "//tensorflow/python/estimator:estimator", + ], + data = [ + "test/testdata/checkpoint", + "test/testdata/model.ckpt-46900.data-00000-of-00001", + "test/testdata/model.ckpt-46900.index", + ], + tags = [ + "no_cuda_on_cpu_tap", + "no_pip", + "no_tap", # It is not able to download the mnist data. + "no_windows", + "nomac", + ], +) + cc_library( name = "utils", srcs = ["convert/utils.cc"], diff --git a/tensorflow/contrib/tensorrt/convert/convert_graph.cc b/tensorflow/contrib/tensorrt/convert/convert_graph.cc index 26d54eb156c..812948bb303 100644 --- a/tensorflow/contrib/tensorrt/convert/convert_graph.cc +++ b/tensorflow/contrib/tensorrt/convert/convert_graph.cc @@ -82,60 +82,76 @@ std::vector GetLoadedTensorRTVersion() { } TrtCandidateSelector::TrtCandidateSelector( - const grappler::GraphProperties& graph_properties) - : graph_properties_(graph_properties) {} + const grappler::GraphProperties& graph_properties, int precision_mode) + : graph_properties_(graph_properties), precision_mode_(precision_mode) {} Status TrtCandidateSelector::IsTensorRTCandidate(const tensorflow::Node* node) { // TODO(laigd): move this set to TrtNodeValidator where it should belong. // LINT.IfChange static const std::set candidate_ops = { - "Identity", - "Snapshot", - "Const", - "Conv2D", - "MaxPool", - "BiasAdd", - "Relu", - "Add", - "Mul", - "Sub", - "Rsqrt", - "Pad", - "Mean", - "AvgPool", - "ConcatV2", - "DepthwiseConv2dNative", - "FusedBatchNorm", - "FusedBatchNormV2", - "Div", - "RealDiv", - "Rsqrt", - "Reciprocal", - "Exp", - "Log", - "Sqrt", - "Abs", - "Neg", - "Transpose", - "Reshape", - "MatMul", - "BatchMatMul", - "Softmax", - "Minimum", - "Maximum", - "TopKV2", - "Sum", - "Prod", - "Max", - "Min", + "Identity", + "Snapshot", + "Const", + "Conv2D", + "MaxPool", + "BiasAdd", + "Relu", + "Sigmoid", + "Tanh", + "Add", + "Mul", + "Sub", + "Rsqrt", + "Pad", + "Mean", + "AvgPool", + "ConcatV2", + "DepthwiseConv2dNative", + "FusedBatchNorm", + "FusedBatchNormV2", + "Div", + "RealDiv", + "Rsqrt", + "Reciprocal", + "Exp", + "Log", + "Sqrt", + "Abs", + "Neg", + "Transpose", + "Reshape", + "MatMul", + "BatchMatMul", + "Softmax", + "Minimum", + "Maximum", + "TopKV2", + "Sum", + "Prod", + "Max", + "Min", + "Relu6", + "Square", }; - // LINT.ThenChange(//tensorflow/contrib/tensorrt/convert/convert_nodes.cc) - const bool is_supported_op_type = + bool is_supported_op_type = (candidate_ops.count(node->type_string()) || PluginFactoryTensorRT::GetInstance()->IsPlugin(node->type_string())); + static const std::set quantize_ops = { + "QuantizeAndDequantizeV2", + "QuantizeAndDequantizeV3", + "FakeQuantWithMinMaxVars", + "FakeQuantWithMinMaxArgs", + }; + // In INT8 mode, we will always apply the quantization ranges provided by + // these ops to the relevant tensors. This happens regardless of the value of + // use_calibration. + if (precision_mode_ == INT8MODE && quantize_ops.count(node->type_string())) { + is_supported_op_type = true; + } + // LINT.ThenChange(//tensorflow/contrib/tensorrt/convert/convert_nodes.cc) if (!is_supported_op_type) { return errors::Unimplemented("Op type ", node->type_string(), - " is not supported."); + " is not supported"); } std::vector input_edges; @@ -170,7 +186,7 @@ tensorflow::Status BuildNodeMap( tensorflow::Status ConvertCalibGraphToInferGraph( const tensorflow::GraphDef& graph_def, tensorflow::GraphDef* infer_graph, bool is_dyn_op) { - VLOG(0) << "Starting Calib Conversion"; + LOG(INFO) << "Starting Calib Conversion"; infer_graph->CopyFrom(graph_def); auto trt_rm = TRTResourceManager::instance(); auto calib_rm = trt_rm->getManager("TRTCalibration"); @@ -220,18 +236,19 @@ tensorflow::Status ConvertGraphDefToTensorRT( const std::vector& output_names, size_t max_batch_size, size_t max_workspace_size_bytes, tensorflow::GraphDef* new_graph_def, int precision_mode, int minimum_segment_size, bool is_dyn_op, - int max_cached_engines, std::vector cached_engine_batches) { + int max_cached_engines, std::vector cached_engine_batches, + bool use_calibration) { // Create GrapplerItem. tensorflow::grappler::GrapplerItem item; item.fetch = output_names; item.graph = graph_def; - // TODO(aaroey): we should have used single machine cluster like the - // following, but the problem is then wrap_conversion will depend on - // direct_session and cause double linking problems. To fix this we need to - // fix or get rid of the swig dependency. Here we use VirtualCluster - // as a work around, and we need to create a session to initialize the - // underlying device before calling this method. +// TODO(aaroey): we should have used single machine cluster like the +// following, but the problem is then wrap_conversion will depend on +// direct_session and cause double linking problems. To fix this we need to +// fix or get rid of the swig dependency. Here we use VirtualCluster +// as a work around, and we need to create a session to initialize the +// underlying device before calling this method. #if 0 // Create single machine cluster. Note that this will create a session and // initialize the gpu devices. @@ -264,7 +281,9 @@ tensorflow::Status ConvertGraphDefToTensorRT( #endif // Create RewriterConfig. - tensorflow::RewriterConfig rw_cfg; + tensorflow::ConfigProto config_proto; + auto& rw_cfg = + *config_proto.mutable_graph_options()->mutable_rewrite_options(); // TODO(aaroey): use only const folding and layout for the time being since // new optimizers break the graph for trt. rw_cfg.add_optimizers("constfold"); @@ -285,9 +304,10 @@ tensorflow::Status ConvertGraphDefToTensorRT( list->add_i(batch); } } + parameters["use_calibration"].set_b(use_calibration); // Run optimizer. - tensorflow::grappler::MetaOptimizer meta_opt(nullptr, rw_cfg); + tensorflow::grappler::MetaOptimizer meta_opt(nullptr, config_proto); TF_RETURN_IF_ERROR(meta_opt.Optimize(cluster.get(), item, new_graph_def)); if (VLOG_IS_ON(5)) { @@ -433,7 +453,8 @@ tensorflow::Status GetEngineInfo( << "but this shouldn't have happened"; info->device = *segment_devices.begin(); } else { - LOG(ERROR) << "Can't find a device placement for the op!"; + VLOG(1) << "No device is assigned to the segment. " + << "A device will be assigned during graph execution (inference)."; } return Status::OK(); } @@ -564,27 +585,30 @@ tensorflow::Status CreateTRTNode(const std::vector& infos, int pos, } } } + + const bool calibrate_int8 = + (info.precision_mode == INT8MODE && info.use_calibration); + // Build the engine and get its serialized representation. string segment_string; - if (info.engine_type == EngineInfo::EngineType::TRTStatic || - info.precision_mode == INT8MODE) { + if (info.engine_type == EngineInfo::EngineType::TRTStatic || calibrate_int8) { // Create static engine for fp32/fp16 mode, and test validity of the engine - // for int8 mode. We don't want engine to fail at the calibration time. - // So we are constructing a FP32 engine here to check its validity, and if - // it is a valid engine then we put the serialized graphdef to the op. - // Otherwise we skip node creation for this engine. + // for int8 calibration mode. We don't want engine to fail at the + // calibration time. So we are constructing a FP32 engine here to check its + // validity, and if it is a valid engine then we put the serialized graphdef + // to the op. Otherwise we skip node creation for this engine. Logger trt_logger; TrtUniquePtrType engine; // TODO(sami): What happens if 1st dim is not batch? TF_RETURN_IF_ERROR(ConvertGraphDefToEngine( - info.segment_graph_def, - info.precision_mode == INT8MODE ? FP32MODE : info.precision_mode, + info.segment_graph_def, calibrate_int8 ? FP32MODE : info.precision_mode, max_batch_size, info.max_workspace_size_bytes, input_shapes, &trt_logger, alloc, /*calibrator=*/nullptr, &engine, + info.use_calibration, /*convert_successfully=*/nullptr)); TrtUniquePtrType engine_data(engine->serialize()); segment_string = string((const char*)engine_data->data(), engine_data->size()); - if (info.precision_mode == INT8MODE) { + if (calibrate_int8) { // See above comment about why not putting this inside the 'else' branch. segment_string = info.segment_graph_def.SerializeAsString(); } @@ -596,7 +620,7 @@ tensorflow::Status CreateTRTNode(const std::vector& infos, int pos, // conversion. string prec_string; TF_RETURN_IF_ERROR(GetPrecisionModeName(info.precision_mode, &prec_string)); - if (info.precision_mode == INT8MODE && + if (info.precision_mode == INT8MODE && calibrate_int8 && !TRTResourceManager::instance()->getManager("TRTCalibration")) { LOG(ERROR) << "Failed to construct calibration storage"; } @@ -632,6 +656,7 @@ tensorflow::Status CreateTRTNode(const std::vector& infos, int pos, .Attr("cached_engine_batches", {max_batch_size}) .Attr("workspace_size_bytes", info.max_workspace_size_bytes) .Attr("precision_mode", prec_string) + .Attr("use_calibration", info.use_calibration) .Attr("OutT", out_types) .Finalize(&trt_node); if (!status.ok()) { @@ -864,19 +889,17 @@ tensorflow::Status ConvertAfterShapes(ConversionParams& params) { } segment_options.minimum_segment_size = params.minimum_segment_size; tensorflow::tensorrt::segment::SegmentNodesVector initial_segments; - TrtCandidateSelector candidate_selector(*params.graph_properties); + TrtCandidateSelector candidate_selector(*params.graph_properties, + params.precision_mode); TF_RETURN_IF_ERROR(tensorrt::segment::SegmentGraph( - &graph, - std::bind(&TrtCandidateSelector::IsTensorRTCandidate, &candidate_selector, - std::placeholders::_1), + &graph, std::bind(&TrtCandidateSelector::IsTensorRTCandidate, + &candidate_selector, std::placeholders::_1), // Input validation is already done by TrtCandidateSelector, so we don't // need to check the input edges. [](const Edge* edge) { return true; }, OutputEdgeValidator(), segment_options, &initial_segments)); - if (initial_segments.size() > 1) { - VLOG(0) << "MULTIPLE tensorrt candidate conversion: " + LOG(INFO) << "Number of TensorRT candidate segments: " << initial_segments.size(); - } // Get the EngineInfo for each segment. std::unordered_map node_map; @@ -902,13 +925,17 @@ tensorflow::Status ConvertAfterShapes(ConversionParams& params) { continue; } curr_engine.precision_mode = params.precision_mode; - curr_engine.engine_type = - (params.is_dyn_op || params.precision_mode == INT8MODE - ? EngineInfo::EngineType::TRTDynamic - : EngineInfo::EngineType::TRTStatic); + if (params.use_calibration && params.precision_mode != INT8MODE) { + return errors::InvalidArgument( + "Calibration with FP32 or FP16 is not supported."); + } + curr_engine.engine_type = ((params.is_dyn_op || params.use_calibration) + ? EngineInfo::EngineType::TRTDynamic + : EngineInfo::EngineType::TRTStatic); + curr_engine.use_calibration = params.use_calibration; curr_engine.cached_engine_batches = params.cached_engine_batches; curr_engine.maximum_cached_engines = params.max_cached_engines; - StrAppend(&curr_engine.engine_name, "my_trt_op_", t); + StrAppend(&curr_engine.engine_name, "TRTEngineOp_", t); status = RegisterSegmentFunctionToFunctionLibrary( &graph, curr_engine.segment_graph_def, curr_engine.engine_name); if (!status.ok()) { @@ -969,16 +996,9 @@ tensorflow::Status ConvertAfterShapes(ConversionParams& params) { &graph, alloc.get(), &engine_nodes); // If status is ok, we successfully added the node to the graph and can // remove segment ops. Otherwise graph is not modified. - string msg = StrCat("Engine ", engine.engine_name, " creation for segment ", - i, ", composed of ", + string msg = StrCat("TensorRT node ", engine.engine_name, + " added for segment ", i, " consisting of ", converted_segments.at(i).first.size(), " nodes"); - if (VLOG_IS_ON(1)) { - StrAppend(&msg, " ("); - for (const string& node_name : converted_segments.at(i).first) { - StrAppend(&msg, node_name, ", "); - } - StrAppend(&msg, ")"); - } if (status.ok()) { LOG(INFO) << msg << " succeeded."; for (auto node_name : converted_segments.at(i).first) { @@ -986,7 +1006,14 @@ tensorflow::Status ConvertAfterShapes(ConversionParams& params) { } } else { // Graph is not modified. - LOG(WARNING) << msg << " failed: " << status << ". Skipping..."; + LOG(WARNING) << msg << " failed: " << status << ". Fallback to TF..."; + } + if (VLOG_IS_ON(1)) { + msg = "Segment consists of nodes: "; + for (const string& node_name : converted_segments.at(i).first) { + StrAppend(&msg, node_name, ", "); + } + VLOG(1) << msg; } } cudaSetDevice(old_cuda_device); diff --git a/tensorflow/contrib/tensorrt/convert/convert_graph.h b/tensorflow/contrib/tensorrt/convert/convert_graph.h index 1c9d82105a7..1f39f56f639 100644 --- a/tensorflow/contrib/tensorrt/convert/convert_graph.h +++ b/tensorflow/contrib/tensorrt/convert/convert_graph.h @@ -35,7 +35,8 @@ namespace convert { // supported by TRT. class TrtCandidateSelector { public: - TrtCandidateSelector(const grappler::GraphProperties& graph_properties); + TrtCandidateSelector(const grappler::GraphProperties& graph_properties, + int precision_mode); // Returns OK iff 'node' is a TF-TRT conversion candidate, which will be added // to TRT subgraph and later converted into TRT engine. @@ -49,6 +50,9 @@ class TrtCandidateSelector { // GraphProperties of the graph whose nodes are to be validated by // IsTensorRTCandidate(). const grappler::GraphProperties& graph_properties_; + + // Quantization ops are only converted when using quantized precisions. + const int precision_mode_; }; struct ConversionParams { @@ -63,6 +67,7 @@ struct ConversionParams { cluster(nullptr), is_dyn_op(false), fixed_input_size(true), + use_calibration(true), max_cached_engines(1) {} const tensorflow::GraphDef* input_graph_def; const std::vector* output_names; @@ -76,6 +81,7 @@ struct ConversionParams { bool is_dyn_op; // Whether to create engine on conversion or execution time bool fixed_input_size; // Assume non-batch ranks of input tensors are fixed int max_cached_engines; // maximum number of cached engines + bool use_calibration; std::vector cached_engine_batches; // list of cached engines }; @@ -95,7 +101,7 @@ tensorflow::Status ConvertGraphDefToTensorRT( size_t max_workspace_size_bytes, tensorflow::GraphDef* new_graph_def, int precision_mode = 1, int minimum_segment_size = 3, bool is_dyn_op = false, int max_cached_engines = 1, - std::vector cached_engine_batches = {}); + std::vector cached_engine_batches = {}, bool use_calibration = true); // Method to call from optimization pass tensorflow::Status ConvertAfterShapes(ConversionParams& params); diff --git a/tensorflow/contrib/tensorrt/convert/convert_graph_test.cc b/tensorflow/contrib/tensorrt/convert/convert_graph_test.cc index f10729987fd..2d2bfeb192c 100644 --- a/tensorflow/contrib/tensorrt/convert/convert_graph_test.cc +++ b/tensorflow/contrib/tensorrt/convert/convert_graph_test.cc @@ -85,27 +85,42 @@ TEST(TrtCandidateSelector, Basics) { ops::MatMul(s.WithOpName("matmul_with_incompatible_input"), incompatible_feed, const_2); + // Quantize ops. + auto quantize_attrs = ops::FakeQuantWithMinMaxArgs::Min(-6.0f).Max(6.0f); + auto quantize = ops::FakeQuantWithMinMaxArgs(s.WithOpName("quantize"), feed, + quantize_attrs); + + // Get GrapplerItem and GraphProperties. grappler::GrapplerItem item; TF_EXPECT_OK(s.ToGraphDef(&item.graph)); Tensor feed_tensor(DT_FLOAT, input_shape); item.feed.push_back(std::make_pair("feed", feed_tensor)); - grappler::GraphProperties graph_properties(item); TF_EXPECT_OK(graph_properties.InferStatically(true)); - TrtCandidateSelector selector(graph_properties); - TF_EXPECT_OK(selector.IsTensorRTCandidate(matmul.operation.node())); - ExpectStatus( - selector.IsTensorRTCandidate(incompatible_matmul.operation.node()), - error::INVALID_ARGUMENT, - "transpose_a is not supported for TensorRT FullyConnected " - "(op: MatMul), at: incompatible_matmul"); - ExpectStatus(selector.IsTensorRTCandidate(unsupported_op.operation.node()), - error::UNIMPLEMENTED, "Op type Sin is not supported"); - ExpectStatus(selector.IsTensorRTCandidate( - matmul_with_incompatible_input.operation.node()), - error::INTERNAL, - "Failed to convert input with index 0 to a TRT_TensorOrWeights"); + for (const int precision_mode : {FP32MODE, INT8MODE}) { + TrtCandidateSelector selector(graph_properties, precision_mode); + TF_EXPECT_OK(selector.IsTensorRTCandidate(matmul.operation.node())); + ExpectStatus( + selector.IsTensorRTCandidate(incompatible_matmul.operation.node()), + error::INVALID_ARGUMENT, + "transpose_a is not supported for TensorRT FullyConnected " + "(op: MatMul), at: incompatible_matmul"); + ExpectStatus(selector.IsTensorRTCandidate(unsupported_op.operation.node()), + error::UNIMPLEMENTED, "Op type Sin is not supported"); + ExpectStatus( + selector.IsTensorRTCandidate( + matmul_with_incompatible_input.operation.node()), + error::INTERNAL, + "Failed to convert input with index 0 to a TRT_TensorOrWeights"); + if (precision_mode == INT8MODE) { + TF_EXPECT_OK(selector.IsTensorRTCandidate(quantize.operation.node())); + } else { + ExpectStatus(selector.IsTensorRTCandidate(quantize.operation.node()), + error::UNIMPLEMENTED, + "Op type FakeQuantWithMinMaxArgs is not supported"); + } + } } class FakeCluster : public grappler::Cluster { diff --git a/tensorflow/contrib/tensorrt/convert/convert_nodes.cc b/tensorflow/contrib/tensorrt/convert/convert_nodes.cc index e2988f5f2a8..25a34dd3503 100644 --- a/tensorflow/contrib/tensorrt/convert/convert_nodes.cc +++ b/tensorflow/contrib/tensorrt/convert/convert_nodes.cc @@ -54,10 +54,10 @@ limitations under the License. // would work! #define TFTRT_CHECK_EQ_TYPE(val1, val2) CHECK_EQ((int)val1, (int)val2) -#define TFTRT_INTERNAL_ERROR_AT_NODE(node) \ - do { \ - return tensorflow::errors::Internal( \ - "TFTRT::", __FUNCTION__, "failed to add TRT layer, at: ", node); \ +#define TFTRT_INTERNAL_ERROR_AT_NODE(node) \ + do { \ + return tensorflow::errors::Internal( \ + "TFTRT::", __FUNCTION__, " failed to add TRT layer, at: ", node); \ } while (0) #define TFTRT_RETURN_ERROR_IF_FALSE(status, node) \ @@ -130,7 +130,7 @@ void GetOutputProperties(const grappler::GraphProperties& graph_properties, *dtype = out_shape.dtype(); *shape = out_shape.shape(); } else { - VLOG(0) << "Unknown output shape" << node->name(); + LOG(INFO) << "Unknown output shape" << node->name(); *dtype = node->output_type(out_port); } } @@ -181,16 +181,55 @@ Status ValidateTensorProperties(const string& producer_node_type, if (shape.dim_size(d) < 0) { return errors::InvalidArgument( "Input tensor with shape ", shape.DebugString(), - " has an unknown non-batch dimemension at dim ", d); + " has an unknown non-batch dimension at dim ", d); } } return Status::OK(); } +string DebugString(const nvinfer1::DimensionType type) { + switch (type) { + case nvinfer1::DimensionType::kSPATIAL: + return "kSPATIAL"; + case nvinfer1::DimensionType::kCHANNEL: + return "kCHANNEL"; + case nvinfer1::DimensionType::kINDEX: + return "kINDEX"; + case nvinfer1::DimensionType::kSEQUENCE: + return "kSEQUENCE"; + default: + return StrCat(static_cast(type), "=unknown"); + } +} + +string DebugString(const nvinfer1::DataType trt_dtype) { + switch (trt_dtype) { + case nvinfer1::DataType::kFLOAT: + return "kFLOAT"; + case nvinfer1::DataType::kHALF: + return "kHALF"; + case nvinfer1::DataType::kINT8: + return "kINT8"; + case nvinfer1::DataType::kINT32: + return "kINT32"; + default: + return "Invalid TRT data type"; + } +} + string DebugString(const nvinfer1::Dims& dims) { string out = StrCat("nvinfer1::Dims(nbDims=", dims.nbDims, ", d="); for (int i = 0; i < dims.nbDims; ++i) { - StrAppend(&out, dims.d[i], ","); + StrAppend(&out, dims.d[i], "[", DebugString(dims.type[i]), "],"); + } + StrAppend(&out, ")"); + return out; +} + +string DebugString(const nvinfer1::Permutation& permutation, int len) { + string out = "nvinfer1::Permutation("; + for (int i = 0; i < len; ++i) { + StrAppend(&out, permutation.order[i], ","); } StrAppend(&out, ")"); return out; @@ -198,16 +237,15 @@ string DebugString(const nvinfer1::Dims& dims) { string DebugString(const nvinfer1::ITensor& tensor) { return StrCat("nvinfer1::ITensor(@", reinterpret_cast(&tensor), - ", shape=", DebugString(tensor.getDimensions()), ")"); + ", name=", tensor.getName(), + ", dtype=", DebugString(tensor.getType()), + ", dims=", DebugString(tensor.getDimensions()), ")"); } -// Return whether or not the broadcast is feasible; -bool TensorRTGetBroadcastShape(const nvinfer1::Dims& operand_l, - const bool operand_l_is_tensor, - const nvinfer1::Dims& operand_r, - const bool operand_r_is_tensor, - nvinfer1::Dims* operand_l_new_shape, - nvinfer1::Dims* operand_r_new_shape) { +Status Converter::GetTrtBroadcastShape( + const TRT_TensorOrWeights& operand_l, const TRT_TensorOrWeights& operand_r, + nvinfer1::Dims* operand_l_new_dims, + nvinfer1::Dims* operand_r_new_dims) const { // *************************************************************************** // TensorRT Elementwise op supports broadcast but requires both tensor to be // of Identical rank @@ -232,52 +270,59 @@ bool TensorRTGetBroadcastShape(const nvinfer1::Dims& operand_l, // -> T: 1 1 1 -1 3 5 1 // -> W: 1 1 1 1 3 5 1 // *************************************************************************** + if (!operand_l.is_tensor() && !operand_r.is_tensor()) { + return errors::InvalidArgument( + "Broadcasting requires at least one of the operands be tensors"); + } + const int max_nb_dims = nvinfer1::Dims::MAX_DIMS + 1; - const size_t element_size = sizeof(operand_l.d[0]); + auto compute_output_dims = + [max_nb_dims](const TRT_TensorOrWeights& input, int broadcast_num_dims, + int* output_dims_array, nvinfer1::Dims* output_dims) { + const nvinfer1::Dims input_dims = input.GetTrtDims(); + std::fill(output_dims_array, output_dims_array + max_nb_dims, 1); + std::copy(input_dims.d, input_dims.d + input_dims.nbDims, + output_dims_array + broadcast_num_dims - input_dims.nbDims); + if (input.is_tensor()) { + const int true_input_dims = input_dims.nbDims + 1; + if (true_input_dims < broadcast_num_dims) { + return errors::InvalidArgument( + "Broadcasting beyond batch dimension is not supported ", + "(tensor #dims ", true_input_dims, " vs broadcast #dims ", + broadcast_num_dims, ")"); + } + // Set the batch dimension to -1, since batch size is not supposed to + // be broadcasted. + output_dims_array[0] = -1; + } + // Copy to output dimensions (stripping the batch dimension). + output_dims->nbDims = broadcast_num_dims - 1; + std::copy(output_dims_array + 1, output_dims_array + broadcast_num_dims, + output_dims->d); + return Status::OK(); + }; - // fill in dimensions - int l_s[max_nb_dims]; - std::fill(l_s, l_s + max_nb_dims, 1); - int l_d = operand_l_is_tensor ? operand_l.nbDims + 1 : operand_l.nbDims; - int r_s[max_nb_dims]; - std::fill(r_s, r_s + max_nb_dims, 1); - int r_d = operand_r_is_tensor ? operand_r.nbDims + 1 : operand_r.nbDims; + // Compute the output dimensions. + const int broadcast_num_dims = + std::max(operand_l.GetTrtDims().nbDims + (operand_l.is_tensor() ? 1 : 0), + operand_r.GetTrtDims().nbDims + (operand_r.is_tensor() ? 1 : 0)); + int output_l[max_nb_dims], output_r[max_nb_dims]; + TF_RETURN_IF_ERROR(compute_output_dims(operand_l, broadcast_num_dims, + output_l, operand_l_new_dims)); + TF_RETURN_IF_ERROR(compute_output_dims(operand_r, broadcast_num_dims, + output_r, operand_r_new_dims)); - int max_d = std::max(l_d, r_d); - std::memcpy(l_s + max_d - operand_l.nbDims, operand_l.d, - operand_l.nbDims * element_size); - std::memcpy(r_s + max_d - operand_r.nbDims, operand_r.d, - operand_r.nbDims * element_size); - - // set -1 for batch dimension, since batch size is not supposed to be - // broadcasted - if (operand_l_is_tensor) { - if (max_d != l_d) { // if broadcast beyond batch dimension, fail - return false; - } - l_s[0] = -1; - } - if (operand_r_is_tensor) { - if (max_d != r_d) { // if broadcast beyond batch dimension, fail - return false; - } - r_s[0] = -1; - } - - // compare broadcast feasibility - for (int i = max_d - 1; i >= 0; i--) { - if ((l_s[i] != r_s[i]) && (l_s[i] != 1) && (r_s[i] != 1)) { - return false; + // Compare broadcast feasibility + for (int i = 0; i < broadcast_num_dims; ++i) { + if ((output_l[i] != output_r[i]) && (output_l[i] != 1) && + (output_r[i] != 1)) { + return errors::InvalidArgument( + "Infeasible broadcast scheme (", "batch_dim: ", output_l[0], ", ", + DebugString(*operand_l_new_dims), " vs ", "batch_dim: ", output_r[0], + ", ", DebugString(*operand_r_new_dims), ")"); } } - - // output new TensorRT Dimension (stripping the batch dimension) - operand_l_new_shape->nbDims = max_d - 1; - std::memcpy(operand_l_new_shape->d, l_s + 1, (max_d - 1) * element_size); - operand_r_new_shape->nbDims = max_d - 1; - std::memcpy(operand_r_new_shape->d, r_s + 1, (max_d - 1) * element_size); - - return true; + return Status::OK(); } inline bool DimsEqual(const nvinfer1::Dims& dim_l, @@ -381,8 +426,8 @@ size_t TRT_ShapedWeights::size_bytes() const { string TRT_ShapedWeights::DebugString() const { return StrCat("TRT_ShapedWeights(shape=", convert::DebugString(shape_), - ", type=", type_, - ", values=", reinterpret_cast(GetValues()), ")"); + ", type=", DataTypeString(type_), ", values=", + reinterpret_cast(GetValues()), ")"); } // A fake ITensor implementation used to check whether the TF-TRT converter can @@ -425,7 +470,9 @@ class TRT_TensorOrWeights::SimpleITensor : public nvinfer1::ITensor { void setLocation(nvinfer1::TensorLocation location) override {} #if NV_TENSORRT_MAJOR >= 5 - bool setDynamicRange(float min, float max) override {} + bool setDynamicRange(float min, float max) override { return true; } + + float getDynamicRange() const override { return 0; } #endif private: @@ -489,8 +536,7 @@ nvinfer1::Dims TRT_TensorOrWeights::GetTrtDims() const { string TRT_TensorOrWeights::DebugString() const { string output = "TRT_TensorOrWeights(type="; if (is_tensor()) { - StrAppend(&output, "tensor @", reinterpret_cast(tensor()), - ", shape=", convert::DebugString(tensor()->getDimensions()), + StrAppend(&output, "tensor=", convert::DebugString(*tensor()), ", batch_size=", batch_size_); } else { StrAppend(&output, "weights=", weights_.DebugString()); @@ -627,11 +673,10 @@ void ReorderCKtoKC(const TRT_ShapedWeights& iweights, break; } case tensorflow::DataType::DT_HALF: { - Reorder2( - {k, c}, static_cast(iweights.GetValues()), - istrides, - static_cast(const_cast(oweights->GetValues())), - ostrides); + Reorder2({k, c}, static_cast(iweights.GetValues()), + istrides, static_cast( + const_cast(oweights->GetValues())), + ostrides); break; } default: @@ -753,8 +798,9 @@ Status TrtNodeValidator::ValidateNode( Status status = ConvertToTensorOrWeights( *pair.first, pair.second, graph_properties, &tensor_or_weights); if (!status.ok()) { - return errors::Internal("Failed to convert input with index ", i, - " to a TRT_TensorOrWeights"); + return errors::Internal( + "Failed to convert input with index ", i, + " to a TRT_TensorOrWeights: ", status.error_message()); } inputs.push_back(tensor_or_weights); } @@ -786,8 +832,11 @@ Status TrtNodeValidator::ConvertConstToWeights( return status; } -Converter::Converter(nvinfer1::INetworkDefinition* trt_network, bool is_fp16) - : trt_network_(trt_network), is_fp16_(is_fp16) { +Converter::Converter(nvinfer1::INetworkDefinition* trt_network, + int precision_mode, bool use_calibration) + : trt_network_(trt_network), + precision_mode_(precision_mode), + use_calibration_(use_calibration) { this->RegisterOpConverters(); } @@ -812,13 +861,18 @@ Status Converter::ConvertNode(const NodeDef& node_def) { TRT_TensorOrWeights& output = outputs[i]; string output_name = node_def.name(); if (i != 0) output_name = StrCat(output_name, ":", i); - // We need to check the name before setting it. For Identity op where the - // output is the input, if its input is one of the engine input, setting - // the name here will overwrite engine input bindings which will cause - // runtime error. + // We need to check the name before setting it. If the input is one of the + // engine input, setting the name here will overwrite engine input + // bindings which will cause runtime error. if (output.is_tensor()) { const char* tensor_name = output.tensor()->getName(); - if (tensor_name == nullptr || std::strlen(tensor_name) == 0) { + if (!tensorflow::str_util::StartsWith(tensor_name, kInputPHName)) { + // TRT initializes tensor names as "(Unnamed ITensor* N)". We rename + // them to match their corresponding TensorFlow name. + // Note: ITensors that we create internally within TF-TRT which are + // not inputs or outputs of a node will not be renamed. This is a + // potential cause of confusion if an error message or warning + // mentions the unnamed tensor. output.tensor()->setName(output_name.c_str()); } } @@ -930,11 +984,14 @@ Status Converter::TransposeTensor(nvinfer1::ITensor* input_tensor, nvinfer1::IShuffleLayer* layer = this->network()->addShuffle(*input_tensor); TFTRT_RETURN_ERROR_IF_NULLPTR(layer, "TF-TRT Internal Transpose"); + MarkQuantizationRangesAsInferrable(input_tensor, layer->getOutput(0)); nvinfer1::Permutation permutation; for (int32_t i = 0; i < dims.nbDims; ++i) { permutation.order[i] = order_with_batch_dim[i + 1] - 1; } + VLOG(1) << "TransposeTensor permutation: " + << DebugString(permutation, dims.nbDims); layer->setFirstTranspose(permutation); nvinfer1::Dims reshape_dims; @@ -950,6 +1007,38 @@ Status Converter::TransposeTensor(nvinfer1::ITensor* input_tensor, return tensorflow::Status::OK(); } +Status Converter::GetWeightRange(const TRT_ShapedWeights& weights, + float* out_min, float* out_max) const { + switch (weights.type_) { + case DataType::DT_FLOAT: { + auto inp = static_cast(weights.GetValues()); + auto result = std::minmax_element(inp, inp + weights.count()); + *out_min = *result.first; + *out_max = *result.second; + break; + } + case DataType::DT_HALF: { + auto inp = static_cast(weights.GetValues()); + auto result = std::minmax_element(inp, inp + weights.count()); + *out_min = Eigen::half_impl::half_to_float(*result.first); + *out_max = Eigen::half_impl::half_to_float(*result.second); + break; + } + case DataType::DT_INT32: { + auto inp = static_cast(weights.GetValues()); + auto result = std::minmax_element(inp, inp + weights.count()); + *out_min = static_cast(*result.first); + *out_max = static_cast(*result.second); + break; + } + default: + return errors::Unimplemented( + "Data type not supported for GetWeightRange: ", + DataTypeString(weights.type_)); + } + return Status::OK(); +} + Status Converter::PrepareTensorForShape(const TRT_TensorOrWeights& input, const nvinfer1::Dims& dims, const nvinfer1::ITensor** tensor) { @@ -964,8 +1053,9 @@ Status Converter::PrepareTensorForShape(const TRT_TensorOrWeights& input, } if (can_check_shapes && TrtDimsNumElements(input.GetTrtDims()) != TrtDimsNumElements(dims)) { - return tensorflow::errors::InvalidArgument( - "Reshape shapes are not compatible."); + return errors::InvalidArgument("Reshape shapes are not compatible (", + DebugString(input.GetTrtDims()), " vs ", + DebugString(dims), ")"); } if (input.is_tensor()) { @@ -976,6 +1066,8 @@ Status Converter::PrepareTensorForShape(const TRT_TensorOrWeights& input, *const_cast(input.tensor())); TFTRT_RETURN_ERROR_IF_NULLPTR(layer, "TF-TRT Internal Reshape"); layer->setReshapeDimensions(dims); + MarkQuantizationRangesAsInferrable( + const_cast(input.tensor()), layer->getOutput(0)); *tensor = layer->getOutput(0); } } else { @@ -983,10 +1075,123 @@ Status Converter::PrepareTensorForShape(const TRT_TensorOrWeights& input, this->network()->addConstant(dims, input.weights().GetTrtWeights()); TFTRT_RETURN_ERROR_IF_NULLPTR(layer, "TF-TRT Internal Reshape"); *tensor = layer->getOutput(0); + if (precision_mode() == INT8MODE && !use_calibration()) { + // If we are in int8 mode and not calibrating, we need to explicitly set a + // quantization range for the output tensor of the IConstantLayer. Here we + // set the range to [min(weights), max(weights)]. + float min_range = 0.0f; + float max_range = 0.0f; + TF_RETURN_IF_ERROR( + GetWeightRange(input.weights(), &min_range, &max_range)); + // Avoid setting range to 0 because TRT will throw an error. If the + // weights are zero then the range doesn't matter: using 127.0f should + // ensure the quantized weight will be exactly zero. + if (min_range == 0.0f && max_range == 0.0f) { + min_range = -127.0f; + max_range = 127.0f; + } + ProvideQuantizationRange(const_cast(*tensor), + min_range, max_range); + } } return tensorflow::Status::OK(); } +void Converter::MarkQuantizationRangesAsInferrable(nvinfer1::ITensor* input, + nvinfer1::ITensor* output) { + quantization_infer_.push_back({input, output}); + quantization_infer_.push_back({output, input}); +} + +void Converter::ProvideQuantizationRange(nvinfer1::ITensor* tensor, + float min_range, float max_range) { + float symmetric_range = std::max(std::abs(min_range), std::abs(max_range)); + quantization_ranges_[tensor] = symmetric_range; +} + +void Converter::MaybeApplyQuantizationRanges() { + if (precision_mode() != INT8MODE) return; + + // Infer ranges across marked ops. + PropagateQuantizationRanges(); + // Apply ranges. +#if NV_TENSORRT_MAJOR >= 5 + for (auto pair : quantization_ranges_) { + nvinfer1::ITensor* tensor = pair.first; + const float range = pair.second; + VLOG(1) << "Setting range for: " << tensor->getName() << ": " << range; + // TODO(laigd): if 'tensor' already has a range set which doesn't match + // 'range', it should report error. + tensor->setDynamicRange(-range, range); + } +#endif + + // Warn user about tensors that are missing ranges. If TRT fuses some layers + // then these tensors may not actually be required, which is why this is + // just a warning. If we are still missing ranges even after fusion, + // Builder::buildCudaEngine() will return nullptr and we will catch the + // error at that point. + if (!use_calibration()) { + // Get all tensors from network + std::set all_tensors; + for (int i = 0; i < this->network()->getNbLayers(); i++) { + nvinfer1::ILayer* layer = this->network()->getLayer(i); + for (int j = 0; j < layer->getNbInputs(); j++) { + all_tensors.insert(layer->getInput(j)); + } + for (int j = 0; j < layer->getNbOutputs(); j++) { + all_tensors.insert(layer->getOutput(j)); + } + } + // Find tensors with no ranges + for (auto tensor : all_tensors) { + if (!quantization_ranges_.count(tensor)) { + // Note: there may be some warnings for "(Unnamed ITensor* N)". These + // are tensors which are created internally by TF-TRT. The ranges for + // these unnamed ITensors are always inferred from user provided ranges, + // thus there will also be a warning for the range(s) the user missed. + LOG(WARNING) << "Quantization range was not found for " + << tensor->getName() << ". " + << "This is okay if TensorRT does not need the range " + << "(e.g. due to node fusion)."; + } + } + } +} + +void Converter::PropagateQuantizationRanges() { + // Propagate ranges across edges in quantization_infer_ until no new + // information is added. + // Note: this function modifies quantization_infer_, it might be better to + // modify a copy instead if we for some reason need quantization_infer_ + // later. + bool information_added = true; + while (information_added) { + information_added = false; + for (auto it = quantization_infer_.begin(); + it != quantization_infer_.end();) { + auto input_tensor_range = quantization_ranges_.find(it->first); + auto output_tensor_range = quantization_ranges_.find(it->second); + if (input_tensor_range != quantization_ranges_.end() && + output_tensor_range == quantization_ranges_.end()) { + // Input has range but output doesn't: copy range + // TODO(laigd): consider reporting error if it a different range is + // already set. + quantization_ranges_[it->second] = input_tensor_range->second; + information_added = true; + VLOG(1) << "Copy quantization range: " << it->first->getName() << " -> " + << it->second->getName(); + } + // We can remove edges when the output range is known + if (quantization_ranges_.find(it->second) != quantization_ranges_.end()) { + it = quantization_infer_.erase(it); + } else { + ++it; + } + } + } +} + Status Converter::GetInputs(const tensorflow::NodeDef& node_def, std::vector* inputs) const { for (auto const& input_name : node_def.input()) { @@ -1043,12 +1248,11 @@ TRT_ShapedWeights ConvertFP32ToFP16(TrtWeightStore* store, } // **************************************************************************** -// Constant folding functions -// TODO(jie): once optimizer kicks in, we should have done constant folding -// there. +// Constant folding functions for weights. +// TODO(laigd): we should probably use eigen directly. // ***************************************************************************** struct LambdaFactory { - enum class OP_CATEGORY : int { RSQRT = 0, NEG, ADD, MUL, SUB, RECIP }; + enum class OP_CATEGORY : int { RSQRT = 0, NEG, RECIP }; OP_CATEGORY op; template @@ -1063,84 +1267,10 @@ struct LambdaFactory { case OP_CATEGORY::RECIP: return [](T t) -> T { return 1.0 / t; }; default: - VLOG(2) << "Not supported op for unary: " << static_cast(op); + LOG(ERROR) << "Not supported op for unary: " << static_cast(op); return nullptr; } } - - template - std::function binary() { - switch (op) { - case OP_CATEGORY::ADD: - return [](T l, T r) -> T { return l + r; }; - case OP_CATEGORY::SUB: - return [](T l, T r) -> T { return l - r; }; - case OP_CATEGORY::MUL: - return [](T l, T r) -> T { return l * r; }; - default: - LOG(WARNING) << "Not supported op for binary: " << static_cast(op); - } - return [](T l, T r) -> T { - LOG(FATAL) << "Unsupported op type "; - return l; - }; - } - - template - std::function broadcast_r(T val) { - VLOG(2) << "LAMBDA VAL : " << val; - switch (op) { - case OP_CATEGORY::ADD: - return [val](T l) -> T { - VLOG(2) << "LAMBDA VAL : " << val; - return l + val; - }; - case OP_CATEGORY::SUB: - return [val](T l) -> T { - VLOG(2) << "LAMBDA VAL : " << val; - return l - val; - }; - case OP_CATEGORY::MUL: - return [val](T l) -> T { - VLOG(2) << "LAMBDA VAL : " << val; - return l * val; - }; - default: - LOG(WARNING) << "Not supported op for binary: " << static_cast(op); - } - return [val](T l) -> T { - LOG(FATAL) << "Unsupported op type "; - return l; - }; - } - - template - std::function broadcast_l(T val) { - VLOG(2) << "LAMBDA VAL : " << val; - switch (op) { - case OP_CATEGORY::ADD: - return [val](T l) -> T { - VLOG(2) << "LAMBDA VAL : " << val; - return val + l; - }; - case OP_CATEGORY::SUB: - return [val](T l) -> T { - VLOG(2) << "LAMBDA VAL : " << val; - return val - l; - }; - case OP_CATEGORY::MUL: - return [val](T l) -> T { - VLOG(2) << "LAMBDA VAL : " << val; - return val * l; - }; - default: - LOG(ERROR) << "Not supported op for binary: " << static_cast(op); - } - return [val](T l) -> T { - LOG(FATAL) << "Unsupported op type "; - return l; - }; - } }; template <> @@ -1148,15 +1278,18 @@ std::function LambdaFactory::unary() { switch (op) { case OP_CATEGORY::RSQRT: { VLOG(2) << "RSQRT GETS DONE"; - return [](Eigen::half t) -> Eigen::half { + return [](Eigen::half t) { return Eigen::half(1.0 / sqrt(static_cast(t))); }; } case OP_CATEGORY::NEG: - return [](Eigen::half t) -> Eigen::half { return -t; }; - // TODO(aaroey): can we support RECIP? + return [](Eigen::half t) { return -t; }; + case OP_CATEGORY::RECIP: + return [](Eigen::half t) { + return Eigen::half(1.0 / static_cast(t)); + }; default: - VLOG(2) << "Not supported op for unary: " << static_cast(op); + LOG(ERROR) << "Not supported op for unary: " << static_cast(op); return nullptr; } } @@ -1188,50 +1321,48 @@ tensorflow::Status UnaryCompute(const TRT_ShapedWeights& iweights, return tensorflow::Status::OK(); } +// If swapped_inputs is false, 'tensor' is the left operand and 'weights' is the +// right operand. If swapped_inputs is true, those two are swapped. +// // TODO(jie): broadcast is needed yet not implemented. -// Only implemented channel wise for the time being -tensorflow::Status BinaryTensorOpWeight(OpConverterParams* params, - const nvinfer1::ITensor* tensor, - TRT_ShapedWeights weights, - bool swapped_inputs) { +// Only implemented channel wise for the time being. +Status BinaryTensorOpWeight(OpConverterParams* params, + const nvinfer1::ITensor* tensor, + TRT_ShapedWeights weights, bool swapped_inputs) { + static const std::unordered_set supported_ops = {"Sub", "Add", "Mul", + "Div", "RealDiv"}; const auto& node_def = params->node_def; - // tensor is the left operand while weights is the right operand; - // when swapped_inputs set to true, those two are swapped. - // TODO(aaroey): use a set. - if (node_def.op() != "Sub" && node_def.op() != "Add" && - node_def.op() != "Mul" && node_def.op() != "Div" && - node_def.op() != "RealDiv") { - return tensorflow::errors::Unimplemented( - "op not supported: " + node_def.op() + ", at: " + node_def.name()); + if (!supported_ops.count(node_def.op())) { + return errors::Unimplemented(node_def.op(), " is not supported, at ", + node_def.name()); } - // Check type consistency - nvinfer1::DataType ttype; - TF_RETURN_IF_ERROR(ConvertDType(weights.type_, &ttype)); + // Check type consistency. + nvinfer1::DataType trt_dtype; + TF_RETURN_IF_ERROR(ConvertDType(weights.type_, &trt_dtype)); - // Check scale mode + // Check scale mode. auto dims_w = weights.shape_; - auto dims_t = tensor->getDimensions(); + const auto dims_t = tensor->getDimensions(); // TODO(jie): addScale checks for input tensor dimension if (dims_t.nbDims != 3) { - return tensorflow::errors::InvalidArgument( - "addScale requires tensor with rank 3, " + node_def.name()); + return errors::InvalidArgument("addScale requires tensor with rank 3, at ", + node_def.name()); } - // default to element-wise + // Default to element-wise auto scale_mode = nvinfer1::ScaleMode::kELEMENTWISE; // TODO(jie): maybe use a permutation instead to support more cases; - bool permutation_flag = false; + bool need_to_permute = false; if (weights.count() == 1) { - VLOG(2) << "UNIFORM"; scale_mode = nvinfer1::ScaleMode::kUNIFORM; } else { - // no broadcasting on Batch dimension; - VLOG(2) << "WEIGHTS DIM: " << dims_w.nbDims - << " tensor DIM: " << dims_t.nbDims; + VLOG(2) << "weights dims: " << DebugString(dims_w) + << "; tensor dims: " << DebugString(dims_t); + // Make sure no broadcasting on batch dimension. if (dims_w.nbDims == dims_t.nbDims + 1) { if (dims_w.d[0] == 1) { for (int i = 1; i < dims_w.nbDims; i++) { @@ -1239,72 +1370,70 @@ tensorflow::Status BinaryTensorOpWeight(OpConverterParams* params, } dims_w.nbDims--; } else { - return tensorflow::errors::InvalidArgument( - "Binary op cannot operate on batch, " + node_def.name()); + return errors::InvalidArgument("Binary op cannot operate on batch, at ", + node_def.name()); } } if (dims_w.nbDims == dims_t.nbDims && dims_w.d[0] == dims_t.d[0]) { scale_mode = nvinfer1::ScaleMode::kELEMENTWISE; - // default is element; + // Default is element-wise for (int i = 1; i < dims_w.nbDims; i++) { if (dims_w.d[i] != dims_t.d[i]) { - // if dimension does not match, switch back to channel; - VLOG(2) << "channel"; + // If dimension does not match, switch back to per-channel scale_mode = nvinfer1::ScaleMode::kCHANNEL; break; } } - // if channel as candidate, validate it + // If the mode is per-channel, since channel dimension is assumed to be + // the third to last dimension, we need to make sure all other dimensions + // have size 1. if (scale_mode == nvinfer1::ScaleMode::kCHANNEL) { for (int i = 1; i < dims_w.nbDims; i++) { if (dims_w.d[i] != 1) - return tensorflow::errors::InvalidArgument( - "Weight shape not compatible at, " + node_def.name()); + return errors::InvalidArgument( + "Weight dims not compatible for channel-wise broadcast at ", + node_def.name()); } - } else { - VLOG(2) << "elementwise"; } } else if (dims_w.nbDims == 1 && dims_w.d[0] == dims_t.d[dims_t.nbDims - 1]) { - // channel wise and broadcast required; - permutation_flag = true; + // Channel wise and broadcast required. We compare the last dimension of + // the tensor shape because of tensorflow default broadcasting rules. + need_to_permute = true; scale_mode = nvinfer1::ScaleMode::kCHANNEL; } else { - return tensorflow::errors::InvalidArgument( - "Weight shape not compatible at, " + node_def.name()); + return errors::InvalidArgument("Weight dims not compatible at ", + node_def.name()); } } + // TODO(laigd): we should add validation_only support in TransposeTensor() and + // PrepareTensorForShape(). + if (params->validation_only) return Status::OK(); - // transpose last dimension + // Transpose last dimension. std::vector permutation(dims_t.nbDims + 1); - if (permutation_flag) { - if (scale_mode == nvinfer1::ScaleMode::kCHANNEL && dims_t.nbDims > 1) { - // we swap the last dimension into channel for trt. - // because of tensorflow default broadcasting rules. - for (int i = 0; i < static_cast(permutation.size()); i++) { - permutation[i] = i; - } - permutation[1] = dims_t.nbDims; - permutation[dims_t.nbDims] = 1; - TF_RETURN_IF_ERROR(params->converter->TransposeTensor( - const_cast(tensor), permutation, &tensor)); - } else { - return tensorflow::errors::InvalidArgument( - "Transpose cannot be applied, " + node_def.name()); + if (need_to_permute) { + // We swap the last dimension into channel for trt, because of tensorflow + // default broadcasting rules. + for (int i = 0; i < static_cast(permutation.size()); i++) { + permutation[i] = i; } + permutation[1] = dims_t.nbDims; + permutation[dims_t.nbDims] = 1; + TF_RETURN_IF_ERROR(params->converter->TransposeTensor( + const_cast(tensor), permutation, &tensor)); } - if (params->converter->is_fp16()) { + if (params->converter->precision_mode() == FP16MODE) { weights = ConvertFP32ToFP16(params->weight_store, weights); } - // prepare weights + // Prepare weights TRT_ShapedWeights shift_weights(weights.type_); TRT_ShapedWeights scale_weights(weights.type_); TRT_ShapedWeights power_weights(weights.type_); - // Maybe I should do a switch if (node_def.op() == "Sub") { if (swapped_inputs) { shift_weights = weights; @@ -1312,6 +1441,10 @@ tensorflow::Status BinaryTensorOpWeight(OpConverterParams* params, *const_cast(tensor), nvinfer1::UnaryOperation::kNEG); TFTRT_RETURN_ERROR_IF_NULLPTR(layer, node_def.name()); + // Since quantization ranges are symmetric, the same range as the input + // will work for the negation of the input. + params->converter->MarkQuantizationRangesAsInferrable( + const_cast(tensor), layer->getOutput(0)); tensor = layer->getOutput(0); } else { TRT_ShapedWeights neg_weights = @@ -1323,6 +1456,25 @@ tensorflow::Status BinaryTensorOpWeight(OpConverterParams* params, } } else if (node_def.op() == "Div" || node_def.op() == "RealDiv") { if (swapped_inputs) { + // We need to infer the quantization range for this intermediate tensor. + // + // x -> [Recip] -> 1/x -> [Scale] -> s/x + // ^ + // need range for this + // + // We have the quantization scales for x and s/x - can we divide the scale + // for s/x by s? Only if it is a scalar. + // + // Because of this issue, fall back to BinaryTensorOpTensor if we are + // doing INT8 with no calibration. There is most likely no performance + // penalty by falling back here. + if (params->converter->precision_mode() == INT8MODE && + !params->converter->use_calibration()) { + return errors::Unimplemented( + "Intermediate quantization range cannot be determined without" + " calibration. Falling back to BinaryTensorOpTensor for ", + node_def.op(), ", at ", node_def.name()); + } scale_weights = weights; nvinfer1::IUnaryLayer* layer = params->converter->network()->addUnary( *const_cast(tensor), @@ -1342,8 +1494,8 @@ tensorflow::Status BinaryTensorOpWeight(OpConverterParams* params, } else if (node_def.op() == "Add") { shift_weights = weights; } else { - return tensorflow::errors::Unimplemented("Binary op not supported: " + - node_def.op()); + // This should not happen. + return errors::Unimplemented("Binary op not supported at ", node_def.op()); } nvinfer1::IScaleLayer* layer = params->converter->network()->addScale( @@ -1353,8 +1505,8 @@ tensorflow::Status BinaryTensorOpWeight(OpConverterParams* params, TFTRT_RETURN_ERROR_IF_NULLPTR(layer, node_def.name()); const nvinfer1::ITensor* output_tensor = layer->getOutput(0); - // transpose back dimension - if (permutation_flag) { + // Transpose back dimension + if (need_to_permute) { TF_RETURN_IF_ERROR(params->converter->TransposeTensor( const_cast(output_tensor), permutation, &output_tensor)); @@ -1398,7 +1550,7 @@ tensorflow::Status ConvertConv2DHelper(OpConverterParams* params, int group) { return tensorflow::errors::Internal( "Conv2D expects kernel of dimension 4, at: " + node_def.name()); } - if (params->converter->is_fp16()) { + if (params->converter->precision_mode() == FP16MODE) { weights_rsck = ConvertFP32ToFP16(params->weight_store, inputs.at(1).weights()); } @@ -1445,6 +1597,8 @@ tensorflow::Status ConvertConv2DHelper(OpConverterParams* params, int group) { nvinfer1::DimsHW(padding[0].first, padding[1].first), nvinfer1::DimsHW(padding[0].second, padding[1].second)); TFTRT_RETURN_ERROR_IF_NULLPTR(pad_layer, node_def.name()); + params->converter->MarkQuantizationRangesAsInferrable( + const_cast(tensor), pad_layer->getOutput(0)); padding = {{0, 0}, {0, 0}}; tensor = pad_layer->getOutput(0); VLOG(2) << "TENSOR after: " << DebugString(tensor->getDimensions()); @@ -1486,9 +1640,9 @@ tensorflow::Status ConvertConv2DHelper(OpConverterParams* params, params->node_def.name()); } -tensorflow::Status BinaryTensorOpTensor(OpConverterParams* params, - const TRT_TensorOrWeights& operand_l, - const TRT_TensorOrWeights& operand_r) { +Status BinaryTensorOpTensor(OpConverterParams* params, + const TRT_TensorOrWeights& operand_l, + const TRT_TensorOrWeights& operand_r) { const auto& node_def = params->node_def; static const std::unordered_map ops{ {"Add", nvinfer1::ElementWiseOperation::kSUM}, @@ -1499,50 +1653,52 @@ tensorflow::Status BinaryTensorOpTensor(OpConverterParams* params, {"Minimum", nvinfer1::ElementWiseOperation::kMIN}, {"Maximum", nvinfer1::ElementWiseOperation::kMAX}, }; - - const nvinfer1::ITensor* tensor_l; - const nvinfer1::ITensor* tensor_r; - - nvinfer1::Dims dim_l; - nvinfer1::Dims dim_r; - - if (!TensorRTGetBroadcastShape(operand_l.GetTrtDims(), operand_l.is_tensor(), - operand_r.GetTrtDims(), operand_r.is_tensor(), - &dim_l, &dim_r)) { - return tensorflow::errors::InvalidArgument( - "Binary op broadcast scheme not supported by TensorRT op: " + - node_def.op() + ", at: " + node_def.name()); - } - - TF_RETURN_IF_ERROR( - params->converter->PrepareTensorForShape(operand_l, dim_l, &tensor_l)); - TF_RETURN_IF_ERROR( - params->converter->PrepareTensorForShape(operand_r, dim_r, &tensor_r)); - - // get trt type & shape - TFAttrs attrs(node_def); - // maybe this part has to be moved into the block of rsqrt later - nvinfer1::DataType dtype = attrs.get("T"); - - // check type consistency - TFTRT_CHECK_EQ_TYPE(tensor_l->getType(), dtype); - TFTRT_CHECK_EQ_TYPE(tensor_r->getType(), dtype); auto op_pair = ops.find(node_def.op()); if (op_pair == ops.end()) { - return tensorflow::errors::Unimplemented( - "binary op: ", node_def.op(), " not supported at: ", node_def.name()); + return errors::Unimplemented("Binary op ", node_def.op(), + " not supported at: ", node_def.name()); } + nvinfer1::Dims broadcasted_dims_l, broadcasted_dims_r; + Status status = params->converter->GetTrtBroadcastShape( + operand_l, operand_r, &broadcasted_dims_l, &broadcasted_dims_r); + if (!status.ok()) { + return errors::InvalidArgument( + "Unsupported binary op broadcast scheme for op ", node_def.name(), ": ", + status.error_message()); + } + if (params->validation_only) return Status::OK(); + + const nvinfer1::ITensor* tensor_l = nullptr; + const nvinfer1::ITensor* tensor_r = nullptr; + status = params->converter->PrepareTensorForShape( + operand_l, broadcasted_dims_l, &tensor_l); + if (status.ok()) { + status = params->converter->PrepareTensorForShape( + operand_r, broadcasted_dims_r, &tensor_r); + } + if (!status.ok()) { + return errors::Internal("Failed to convert binary op ", node_def.name(), + ": ", status.error_message()); + } + + // Check type consistency. + TFAttrs attrs(node_def); + nvinfer1::DataType dtype = attrs.get("T"); + TFTRT_CHECK_EQ_TYPE(tensor_l->getType(), dtype) + << DebugString(tensor_l->getType()) << " vs " << DebugString(dtype); + TFTRT_CHECK_EQ_TYPE(tensor_r->getType(), dtype) + << DebugString(tensor_r->getType()) << " vs " << DebugString(dtype); + + // Add ElementWise layer. nvinfer1::IElementWiseLayer* layer = params->converter->network()->addElementWise( - // TODO(aaroey): will tensor_l/tensor_r get modified? *const_cast(tensor_l), *const_cast(tensor_r), op_pair->second); TFTRT_RETURN_ERROR_IF_NULLPTR(layer, node_def.name()); - nvinfer1::ITensor* output_tensor = layer->getOutput(0); - // pass the output + // Pass the output params->outputs->push_back(TRT_TensorOrWeights(output_tensor)); return tensorflow::Status::OK(); } @@ -1789,6 +1945,8 @@ tensorflow::Status ConvertPool(OpConverterParams* params) { nvinfer1::DimsHW(padding[0].first, padding[1].first), nvinfer1::DimsHW(padding[0].second, padding[1].second)); TFTRT_RETURN_ERROR_IF_NULLPTR(pad_layer, node_def.name()); + params->converter->MarkQuantizationRangesAsInferrable( + const_cast(tensor), pad_layer->getOutput(0)); padding = {{0, 0}, {0, 0}}; tensor = pad_layer->getOutput(0); } @@ -1796,6 +1954,11 @@ tensorflow::Status ConvertPool(OpConverterParams* params) { nvinfer1::IPoolingLayer* layer = params->converter->network()->addPooling( *const_cast(tensor), type, ksize); TFTRT_RETURN_ERROR_IF_NULLPTR(layer, node_def.name()); + // TODO(tmorris): Average pooling may not be entirely safe to infer + // quantization range through (at least forwards - backwards should be fine). + // Max pooling is okay. + params->converter->MarkQuantizationRangesAsInferrable( + const_cast(tensor), layer->getOutput(0)); layer->setStride(stride); layer->setPadding({padding[0].first, padding[1].first}); @@ -1813,110 +1976,290 @@ tensorflow::Status ConvertPool(OpConverterParams* params) { } tensorflow::Status ConvertActivation(OpConverterParams* params) { - const nvinfer1::ITensor* tensor = params->inputs.at(0).tensor(); + const auto& inputs = params->inputs; + const auto& node_def = params->node_def; + if (inputs.size() != 1) { + return tensorflow::errors::InvalidArgument( + node_def.op(), " expects one input, at ", node_def.name()); + } + if (!inputs.at(0).is_tensor()) { + return tensorflow::errors::Unimplemented( + node_def.op(), " is only implemented for tensors, at ", + node_def.name()); + } + static const std::unordered_map ops{ + {"Relu", nvinfer1::ActivationType::kRELU}, + {"Sigmoid", nvinfer1::ActivationType::kSIGMOID}, + {"Tanh", nvinfer1::ActivationType::kTANH}, + }; + auto op_pair = ops.find(node_def.op()); + if (op_pair == ops.end()) { + return tensorflow::errors::Unimplemented("Activation op: ", node_def.op(), + " not supported at: ", + node_def.name()); + } + if (params->validation_only) return tensorflow::Status::OK(); + + // Start conversion. + const nvinfer1::ITensor* tensor = inputs.at(0).tensor(); nvinfer1::IActivationLayer* layer = params->converter->network()->addActivation( - *const_cast(tensor), - nvinfer1::ActivationType::kRELU); - TFTRT_RETURN_ERROR_IF_NULLPTR(layer, params->node_def.name()); + *const_cast(tensor), op_pair->second); + TFTRT_RETURN_ERROR_IF_NULLPTR(layer, node_def.name()); nvinfer1::ITensor* output_tensor = layer->getOutput(0); + // Set quantization range for output of Sigmoid, Tanh. + if (node_def.op() == "Sigmoid") { + params->converter->ProvideQuantizationRange(output_tensor, 0.0f, 1.0f); + } else if (node_def.op() == "Tanh") { + params->converter->ProvideQuantizationRange(output_tensor, -1.0f, 1.0f); + } params->outputs->push_back(TRT_TensorOrWeights(output_tensor)); return tensorflow::Status::OK(); } -tensorflow::Status ConvertScale(OpConverterParams* params) { +Status ConvertQuantize(OpConverterParams* params) { + const auto& inputs = params->inputs; + const auto& node_def = params->node_def; + if ((inputs.size() == 0) || + (node_def.op() == "FakeQuantWithMinMaxArgs" && inputs.size() != 1) || + (node_def.op() == "FakeQuantWithMinMaxVars" && inputs.size() != 3) || + (node_def.op() == "QuantizeAndDequantizeV2" && inputs.size() != 3) || + (node_def.op() == "QuantizeAndDequantizeV3" && inputs.size() != 4)) { + return errors::InvalidArgument("Invalid number of inputs for ", + node_def.op(), ", at ", node_def.name()); + } + if (inputs.at(0).is_weights()) { + // TensorRT will automatically quantize weights, so we will ignore ranges + // for weights. + params->outputs->push_back(inputs.at(0)); + return Status::OK(); + } + float min_range = 0.0f; + float max_range = 0.0f; + if (node_def.op() == "FakeQuantWithMinMaxArgs") { + // Get ranges via node attributes. + TFAttrs attrs(node_def); + if (attrs.count("min") == 0 || attrs.count("max") == 0) { + return errors::InvalidArgument("Min or max attribute not found for ", + node_def.op(), " at ", node_def.name()); + } + min_range = attrs.get("min"); + max_range = attrs.get("max"); + } else if (node_def.op() == "FakeQuantWithMinMaxVars" || + node_def.op() == "QuantizeAndDequantizeV2" || + node_def.op() == "QuantizeAndDequantizeV3") { + // Get ranges via inputs. + if (!inputs.at(1).is_weights() || !inputs.at(2).is_weights()) { + return errors::InvalidArgument("Min and max inputs for ", node_def.op(), + " must be weights not tensors, at ", + node_def.name()); + } + auto get_weights_value = [&inputs](int index) { + auto raw_weights = static_cast( + const_cast(inputs.at(index).weights().GetValues())); + return raw_weights[0]; + }; + min_range = get_weights_value(1); + max_range = get_weights_value(2); + } else { + return errors::InvalidArgument("Unknown quantization op ", node_def.op(), + ", at ", node_def.name()); + } + if (params->validation_only) return Status::OK(); + + // Store ranges for tensor + params->converter->ProvideQuantizationRange( + const_cast(inputs.at(0).tensor()), min_range, + max_range); + // Sometimes, TRT may not quantize a tensor, either because it chooses to + // execute a higher precision kernel or because of op fusion. In these cases, + // accuracy will suffer if the model was trained to expect quantization at + // that tensor. We should consider adding a clip(tensor, min_range, max_range) + // operation here to ensure that any arbitrarily placed quantize node will + // execute as expected. However, this will negatively affect performance. If + // users train their models in a way which models inference as close as + // possible (i.e. not quantizing in place where fusion will occur), then there + // is no problem with the current implementation. + params->outputs->push_back(inputs.at(0)); + return Status::OK(); +} + +// TODO(pdavoodi): we should update relu6 implementation once TensorRT supports +// Relu6 natively. +tensorflow::Status ConvertRelu6(OpConverterParams* params) { + const auto& inputs = params->inputs; + const auto& node_def = params->node_def; + if (inputs.size() != 1) { + return tensorflow::errors::InvalidArgument( + "Invalid number of inputs for Relu6, at ", node_def.name()); + } + if (inputs.at(0).is_weights()) { + return tensorflow::errors::Unimplemented( + "Relu6 is only implemented for tensors, not weights, at ", + node_def.name()); + } + if (params->validation_only) return Status::OK(); + // *************************************************************************** + // TensorRT does not implement Relu6 natively. This function converts Relu6 op + // to available TensorRT ops: Relu6(x) = min(Relu(x), 6) + // *************************************************************************** + + // Input Tensor + const nvinfer1::ITensor* tensor = inputs.at(0).tensor(); + + // Relu operation i.e. Relu(x) = max(0, x) + nvinfer1::IActivationLayer* relu_layer = + params->converter->network()->addActivation( + *const_cast(tensor), + nvinfer1::ActivationType::kRELU); + TFTRT_RETURN_ERROR_IF_NULLPTR(relu_layer, node_def.name()); + + // Large range of relu is problematic during quantization in INT8 precision + // mode. Setting dynamic range of relu = [0.f, 6.0f] helps with quantization. + // TRT only uses dynamic ranges in INT8 precision mode, + // and this does not affect the FP32 path. + params->converter->ProvideQuantizationRange(relu_layer->getOutput(0), 0.0f, + 6.0f); + + // Create a constant layer to store the floating point weight i.e. 6.0f This + // tensor will be broadcasted uniformly during elementwise `min` operation. + // The constant has to have the same rank as the input in order for TRT to + // broadcast + nvinfer1::Dims dims; + dims.nbDims = relu_layer->getOutput(0)->getDimensions().nbDims; + for (int i = 0; i < dims.nbDims; i++) { + dims.d[i] = 1; + } + TRT_ShapedWeights weights = params->weight_store->GetTempWeights( + tensorflow::DataType::DT_FLOAT, dims); + auto weights_ptr = + static_cast(const_cast(weights.GetValues())); + weights_ptr[0] = 6.0f; + nvinfer1::IConstantLayer* const6_layer = + params->converter->network()->addConstant(dims, weights.GetTrtWeights()); + TFTRT_RETURN_ERROR_IF_NULLPTR(const6_layer, node_def.name()); + params->converter->ProvideQuantizationRange(const6_layer->getOutput(0), 0.0f, + 6.0f); + + // ElementWise Min Operation + // Min op is a nop for INT8 execution path, as the input tensor + // to this layer will only have values in range [0.f, 6.0f]. + const nvinfer1::ITensor* tensor_l = relu_layer->getOutput(0); + const nvinfer1::ITensor* tensor_r = const6_layer->getOutput(0); + nvinfer1::IElementWiseLayer* relu6_layer = + params->converter->network()->addElementWise( + *const_cast(tensor_l), + *const_cast(tensor_r), + nvinfer1::ElementWiseOperation::kMIN); + TFTRT_RETURN_ERROR_IF_NULLPTR(relu6_layer, node_def.name()); + nvinfer1::ITensor* output_tensor = relu6_layer->getOutput(0); + params->converter->ProvideQuantizationRange(output_tensor, 0.0f, 6.0f); + + params->outputs->push_back(TRT_TensorOrWeights(output_tensor)); + return Status::OK(); +} + +tensorflow::Status ConvertBiasAdd(OpConverterParams* params) { const auto& inputs = params->inputs; const auto& node_def = params->node_def; if (inputs.size() != 2 || !inputs.at(0).is_tensor() || !inputs.at(1).is_weights()) { - return tensorflow::errors::Unimplemented( - "ConvertScale only supports tensorweight: ", node_def.name()); + return errors::InvalidArgument("Input expects tensor and weights, at ", + node_def.name()); } + if (params->validation_only) return Status::OK(); - const nvinfer1::ITensor* tensor = inputs.at(0).tensor(); - TRT_ShapedWeights weights = inputs.at(1).weights(); - if (params->converter->is_fp16()) { - weights = ConvertFP32ToFP16(params->weight_store, inputs.at(1).weights()); - } - - TRT_ShapedWeights empty_weights(weights.type_); + nvinfer1::ITensor* tensor = + const_cast(inputs.at(0).tensor()); + const nvinfer1::Dims original_dims = tensor->getDimensions(); TFAttrs attrs(node_def); - - const auto data_format = attrs.get("data_format"); - int channel_index; - const auto dims = tensor->getDimensions(); - if (data_format == "NHWC") { - // 1). NHWC is really N+C - channel_index = dims.nbDims - 1; // batch dimension is implicit here! - } else { - // 2). NCHW is really N+CHW - channel_index = 0; // batch dimension is implicit here! - } + const string data_format = attrs.get("data_format"); + const int channel_index = + (data_format == "NHWC" ? original_dims.nbDims - 1 : 0); nvinfer1::Permutation permutation; - for (int32_t i = 0; i < dims.nbDims; ++i) { - permutation.order[i] = i; - } - - if (channel_index >= 0) { + if (channel_index != 0) { + // Permute the dimensions so that the channel dimension is the first + // dimension. + for (int i = 0; i < original_dims.nbDims; ++i) { + permutation.order[i] = i; + } permutation.order[0] = channel_index; permutation.order[channel_index] = 0; - } else { - return tensorflow::errors::Unimplemented( - "TFTRT::BiasAdd cannot apply on batch dimension, at ", node_def.name()); + VLOG(1) << "ConvertBiasAdd permutation: " + << DebugString(permutation, original_dims.nbDims); } // TensorRT addScale requires input to be of rank 3, we need to apply - // transpose as well as reshape - if (channel_index != 0 || dims.nbDims != 3) { + // transpose as well as reshape. + // TODO(laigd): this doesn't match what the TRT doc says, fix the doc? + if (channel_index != 0 || original_dims.nbDims != 3) { nvinfer1::IShuffleLayer* shuffle_layer = - params->converter->network()->addShuffle( - *const_cast(tensor)); + params->converter->network()->addShuffle(*tensor); TFTRT_RETURN_ERROR_IF_NULLPTR(shuffle_layer, node_def.name()); + params->converter->MarkQuantizationRangesAsInferrable( + tensor, shuffle_layer->getOutput(0)); + + // NOTE(laigd): for some reason we need to apply the reshape + // unconditionally. The default shape has nbDims==-1 and it seems the + // behavior is undefined in some cases. nvinfer1::Dims reshape_dims; reshape_dims.nbDims = 3; - reshape_dims.d[0] = 0; // 0 copy from the input - reshape_dims.d[1] = dims.nbDims >= 2 ? 0 : 1; // 0 copy from the input - reshape_dims.d[2] = dims.nbDims >= 3 ? -1 : 1; // -1 infer from the rest + // 0 means copying from input; -1 means inferring from the rest. + reshape_dims.d[0] = 0; + reshape_dims.d[1] = original_dims.nbDims >= 2 ? 0 : 1; + reshape_dims.d[2] = original_dims.nbDims >= 3 ? -1 : 1; + shuffle_layer->setReshapeDimensions(reshape_dims); + if (channel_index != 0) { - // maybe we do not need this check. concerned about TRT optimization shuffle_layer->setFirstTranspose(permutation); } - shuffle_layer->setReshapeDimensions(reshape_dims); tensor = shuffle_layer->getOutput(0); } + TRT_ShapedWeights weights = inputs.at(1).weights(); + if (params->converter->precision_mode() == FP16MODE) { + weights = ConvertFP32ToFP16(params->weight_store, weights); + } nvinfer1::ScaleMode mode = nvinfer1::ScaleMode::kCHANNEL; if (weights.shape_.d[0] == 1) { mode = nvinfer1::ScaleMode::kUNIFORM; } + TRT_ShapedWeights empty_weights(weights.type_); nvinfer1::IScaleLayer* layer = params->converter->network()->addScale( - *const_cast(tensor), mode, weights.GetTrtWeights(), - empty_weights.GetTrtWeights(), empty_weights.GetTrtWeights()); + *tensor, mode, weights.GetTrtWeights(), empty_weights.GetTrtWeights(), + empty_weights.GetTrtWeights()); TFTRT_RETURN_ERROR_IF_NULLPTR(layer, node_def.name()); nvinfer1::ITensor* output_tensor = layer->getOutput(0); - // restore transpose & reshape - if (channel_index != 0 || dims.nbDims != 3) { + // Restore transpose & reshape. + if (channel_index != 0 || original_dims.nbDims != 3) { nvinfer1::IShuffleLayer* shuffle_layer = - params->converter->network()->addShuffle( - *const_cast(output_tensor)); + params->converter->network()->addShuffle(*output_tensor); TFTRT_RETURN_ERROR_IF_NULLPTR(shuffle_layer, node_def.name()); - nvinfer1::Dims reshape_dims = dims; - int tmp = reshape_dims.d[channel_index]; - reshape_dims.d[channel_index] = reshape_dims.d[0]; - reshape_dims.d[0] = tmp; + // NOTE: for same reason as mentioned above we need to apply the reshape + // unconditionally. + nvinfer1::Dims reshape_dims = original_dims; + if (channel_index != 0) { + // NOTE: according to NVIDIA dimension types are deprecated, so we don't + // need to copy them back. + reshape_dims.d[channel_index] = original_dims.d[0]; + reshape_dims.d[0] = original_dims.d[channel_index]; + } shuffle_layer->setReshapeDimensions(reshape_dims); + if (channel_index != 0) { shuffle_layer->setSecondTranspose(permutation); } + params->converter->MarkQuantizationRangesAsInferrable( + output_tensor, shuffle_layer->getOutput(0)); output_tensor = shuffle_layer->getOutput(0); } params->outputs->push_back(TRT_TensorOrWeights(output_tensor)); - return tensorflow::Status::OK(); + return Status::OK(); } Status GetTensorDimsWithProtoShape(const Tensor& tensor, @@ -2053,9 +2396,9 @@ tensorflow::Status ConvertConst(OpConverterParams* params) { uint8* data = reinterpret_cast(temp_weights.data()); std::copy(data, data + tensor.NumElements(), dst); } else { - return errors::FailedPrecondition( - "Unexpected data type: ", DataTypeString(dtype), - " at: ", node_def.name()); + return errors::FailedPrecondition("Unexpected data type: ", + DataTypeString(dtype), " at: ", + node_def.name()); } } } @@ -2070,32 +2413,41 @@ tensorflow::Status ConvertConst(OpConverterParams* params) { } tensorflow::Status ConvertIdentity(OpConverterParams* params) { + // TODO(tmorris): TRT's Identity layer does not get optimized away as of TRT + // 5.0, however once we know that it does it would be nice to use that + // instead. params->outputs->push_back(params->inputs.at(0)); return tensorflow::Status::OK(); } -tensorflow::Status ConvertBinary(OpConverterParams* params) { +Status ConvertBinary(OpConverterParams* params) { const auto& inputs = params->inputs; const auto& node_def = params->node_def; if (inputs.size() != 2) { - return tensorflow::errors::FailedPrecondition( - "Binary ops require two tensor input, at ", node_def.name()); + return errors::InvalidArgument("Binary ops require two inputs, at ", + node_def.name()); } // Constant folding should have been done by TensorFlow - if (inputs.at(0).is_weights() && inputs.at(1).is_weights()) { - return tensorflow::errors::Unimplemented( + return errors::Unimplemented( "Constant folding is falled back to TensorFlow, binary op received " "both input as constant at: ", node_def.name()); } - // Try to convert into Scale layer first (for better performance) + // TODO(tmorris): TRT plans to deprecate IScaleLayer and will replace it with + // IElementwiseLayer. At that point, we can remove BinaryTensorOpWeight. For + // now, the performance will be slightly better with IScaleLayer because it + // can be fused in more situations. However, most of the benefits of + // IScaleLayer are when the layer performs both a shift and a scale, which we + // don't do except for convolutions. + // + // Try to convert into Scale layer first (for better performance). // Since scale layer supports restricted broadcast policy and op types, we // allow failure and try to handle it through Elementwise op - // (BinaryTensorOpTensor) - Status status = tensorflow::Status::OK(); + // (BinaryTensorOpTensor). + Status status = Status::OK(); if (inputs.at(0).is_tensor() && inputs.at(1).is_weights()) { status = BinaryTensorOpWeight(params, inputs.at(0).tensor(), inputs.at(1).weights(), false); @@ -2103,7 +2455,10 @@ tensorflow::Status ConvertBinary(OpConverterParams* params) { status = BinaryTensorOpWeight(params, inputs.at(1).tensor(), inputs.at(0).weights(), true); } + // If both input are tensors, or one of them is weights but the conversion + // above failed, try the conversion using BinaryTensorOpTensor. if ((inputs.at(0).is_tensor() && inputs.at(1).is_tensor()) || !status.ok()) { + if (!status.ok()) VLOG(1) << status; status = BinaryTensorOpTensor(params, inputs.at(0), inputs.at(1)); } return status; @@ -2133,6 +2488,20 @@ tensorflow::Status ConvertUnary(OpConverterParams* params) { nvinfer1::IUnaryLayer* layer; if (node_def.op() == "Rsqrt") { + // We will need a quantization range for intermediate tensor if not using + // calibration. + // + // x -> [Sqrt] -> sqrt(x) -> [Recip] -> 1/sqrt(x) + // ^ + // need range here + if (params->converter->precision_mode() == INT8MODE && + !params->converter->use_calibration()) { + return errors::Unimplemented( + "Intermediate quantization range cannot be determined without" + " calibration for Rsqrt, consider replacing with " + "Sqrt -> FakeQuant -> Reciprocal ops, at ", + node_def.name()); + } layer = params->converter->network()->addUnary( *const_cast(tensor), nvinfer1::UnaryOperation::kSQRT); @@ -2156,6 +2525,48 @@ tensorflow::Status ConvertUnary(OpConverterParams* params) { return tensorflow::Status::OK(); } +tensorflow::Status ConvertSquare(OpConverterParams* params) { + const auto& inputs = params->inputs; + const auto& node_def = params->node_def; + if (inputs.size() != 1) { + return tensorflow::errors::InvalidArgument("Square expects one input, at ", + node_def.name()); + } + if (inputs.at(0).is_weights()) { + return tensorflow::errors::Unimplemented( + "Square is only implemented for tensors, at ", node_def.name()); + } + if (params->validation_only) return Status::OK(); + + // Constant 2 with same rank as input + nvinfer1::Dims dims = inputs.at(0).GetTrtDims(); + for (int i = 0; i < dims.nbDims; i++) { + dims.d[i] = 1; + } + TRT_ShapedWeights weights = params->weight_store->GetTempWeights( + tensorflow::DataType::DT_FLOAT, dims); + auto weights_ptr = + static_cast(const_cast(weights.GetValues())); + weights_ptr[0] = 2.f; + nvinfer1::IConstantLayer* const2_layer = + params->converter->network()->addConstant(dims, weights.GetTrtWeights()); + TFTRT_RETURN_ERROR_IF_NULLPTR(const2_layer, node_def.name()); + + // ElementWise Pow Operation + const nvinfer1::ITensor* tensor_l = inputs.at(0).tensor(); + const nvinfer1::ITensor* tensor_r = const2_layer->getOutput(0); + nvinfer1::IElementWiseLayer* layer = + params->converter->network()->addElementWise( + *const_cast(tensor_l), + *const_cast(tensor_r), + nvinfer1::ElementWiseOperation::kPOW); + TFTRT_RETURN_ERROR_IF_NULLPTR(layer, node_def.name()); + nvinfer1::ITensor* output_tensor = layer->getOutput(0); + + params->outputs->push_back(TRT_TensorOrWeights(output_tensor)); + return tensorflow::Status::OK(); +} + tensorflow::Status ConvertReduce(OpConverterParams* params) { const auto& inputs = params->inputs; const auto& node_def = params->node_def; @@ -2692,6 +3103,8 @@ tensorflow::Status ConvertSoftmax(OpConverterParams* params) { layer->setAxes(1 << (nbDims - 1)); nvinfer1::ITensor* output_tensor = layer->getOutput(0); + // Quantization range for SoftMax is always (0, 1) + params->converter->ProvideQuantizationRange(output_tensor, 0.0f, 1.0f); params->outputs->push_back(TRT_TensorOrWeights(output_tensor)); return tensorflow::Status::OK(); } @@ -2716,9 +3129,9 @@ tensorflow::Status ConvertTopK(OpConverterParams* params) { op = nvinfer1::TopKOperation::kMAX; reducedAxes |= 1 << (nbDims - 1); } else { - return tensorflow::errors::Unimplemented( - "Operation: " + node_def.op() + - " not implemented, at: " + node_def.name()); + return tensorflow::errors::Unimplemented("Operation: " + node_def.op() + + " not implemented, at: " + + node_def.name()); } nvinfer1::ITopKLayer* layer = params->converter->network()->addTopK( @@ -2732,40 +3145,52 @@ tensorflow::Status ConvertTopK(OpConverterParams* params) { return tensorflow::Status::OK(); } -void TrtNodeValidator::RegisterOpValidators() { +static void RegisterValidatableOpConverters( + std::unordered_map* registration) { // TODO(laigd): support all op types. - op_validators_["Const"] = ConvertConst; - op_validators_["Transpose"] = ConvertTranspose; - op_validators_["Reshape"] = ConvertReshape; - op_validators_["MatMul"] = ConvertMatMul; + (*registration)["BiasAdd"] = ConvertBiasAdd; + (*registration)["Const"] = ConvertConst; + (*registration)["Transpose"] = ConvertTranspose; + (*registration)["Reshape"] = ConvertReshape; + (*registration)["MatMul"] = ConvertMatMul; + (*registration)["Relu6"] = ConvertRelu6; + (*registration)["Square"] = ConvertSquare; + + for (auto quantization_op_type : + {"QuantizeAndDequantizeV2", "QuantizeAndDequantizeV3", + "FakeQuantWithMinMaxVars", "FakeQuantWithMinMaxArgs"}) { + (*registration)[quantization_op_type] = ConvertQuantize; + } + for (auto binary_op_type : + {"Add", "Mul", "Sub", "Div", "RealDiv", "Maximum", "Minimum"}) { + (*registration)[binary_op_type] = ConvertBinary; + } + for (auto activation_op_type : {"Relu", "Sigmoid", "Tanh"}) { + (*registration)[activation_op_type] = ConvertActivation; + } +} + +void TrtNodeValidator::RegisterOpValidators() { + RegisterValidatableOpConverters(&op_validators_); } void Converter::RegisterOpConverters() { - // vgg_16 slim implementation + RegisterValidatableOpConverters(&op_registry_); + op_registry_["Conv2D"] = ConvertConv2D; op_registry_["DepthwiseConv2dNative"] = ConvertConv2DDepthwise; - op_registry_["Relu"] = ConvertActivation; op_registry_["MaxPool"] = ConvertPool; op_registry_["AvgPool"] = ConvertPool; - op_registry_["BiasAdd"] = ConvertScale; - op_registry_["Const"] = ConvertConst; // TODO(ben,jie): this is a temp hack. op_registry_["Identity"] = ConvertIdentity; // Identity should be removed op_registry_["Snapshot"] = ConvertIdentity; // Snapshot should be removed - // resnet_50_v1 slim implementation - op_registry_["Add"] = ConvertBinary; - op_registry_["Mul"] = ConvertBinary; - op_registry_["Sub"] = ConvertBinary; op_registry_["Pad"] = ConvertPad; op_registry_["ConcatV2"] = ConvertConcat; op_registry_["FusedBatchNorm"] = ConvertFusedBatchNorm; op_registry_["FusedBatchNormV2"] = ConvertFusedBatchNorm; - op_registry_["Div"] = ConvertBinary; - op_registry_["RealDiv"] = ConvertBinary; - op_registry_["Rsqrt"] = ConvertUnary; op_registry_["Reciprocal"] = ConvertUnary; op_registry_["Exp"] = ConvertUnary; @@ -2774,20 +3199,19 @@ void Converter::RegisterOpConverters() { op_registry_["Abs"] = ConvertUnary; op_registry_["Neg"] = ConvertUnary; - op_registry_["Transpose"] = ConvertTranspose; - op_registry_["Reshape"] = ConvertReshape; - op_registry_["Sum"] = ConvertReduce; op_registry_["Prod"] = ConvertReduce; op_registry_["Max"] = ConvertReduce; op_registry_["Min"] = ConvertReduce; op_registry_["Mean"] = ConvertReduce; - op_registry_["Maximum"] = ConvertBinary; - op_registry_["Minimum"] = ConvertBinary; op_registry_["Softmax"] = ConvertSoftmax; - op_registry_["MatMul"] = ConvertMatMul; op_registry_["BatchMatMul"] = ConvertBatchMatMul; op_registry_["TopKV2"] = ConvertTopK; + op_registry_["Relu6"] = ConvertRelu6; + op_registry_["QuantizeAndDequantizeV2"] = ConvertQuantize; + op_registry_["QuantizeAndDequantizeV3"] = ConvertQuantize; + op_registry_["FakeQuantWithMinMaxVars"] = ConvertQuantize; + op_registry_["FakeQuantWithMinMaxArgs"] = ConvertQuantize; plugin_converter_ = ConvertPlugin; } @@ -2798,7 +3222,7 @@ tensorflow::Status ConvertGraphDefToEngine( const std::vector& input_shapes, Logger* logger, nvinfer1::IGpuAllocator* allocator, TRTInt8Calibrator* calibrator, - TrtUniquePtrType* engine, + TrtUniquePtrType* engine, bool use_calibration, bool* convert_successfully) { engine->reset(); if (convert_successfully) *convert_successfully = false; @@ -2813,7 +3237,11 @@ tensorflow::Status ConvertGraphDefToEngine( builder->setHalf2Mode(true); } else if (precision_mode == INT8MODE) { builder->setInt8Mode(true); - builder->setInt8Calibrator(calibrator); + if (use_calibration) { + builder->setInt8Calibrator(calibrator); + } else { + builder->setInt8Calibrator(nullptr); + } } // Create the network. @@ -2826,7 +3254,7 @@ tensorflow::Status ConvertGraphDefToEngine( // Build the network VLOG(1) << "Starting engine conversion "; - Converter converter(trt_network.get(), precision_mode == FP16MODE); + Converter converter(trt_network.get(), precision_mode, use_calibration); std::vector> output_tensors; // Graph nodes are already topologically sorted during construction for (const auto& node_def : gdef.node()) { @@ -2882,6 +3310,9 @@ tensorflow::Status ConvertGraphDefToEngine( TF_RETURN_IF_ERROR(converter.RenameAndMarkOutputTensors(output_tensors)); if (convert_successfully) *convert_successfully = true; + // Apply user provided quantization ranges to tensors + converter.MaybeApplyQuantizationRanges(); + // Build the engine. VLOG(1) << "Starting engine creation"; engine->reset(builder->buildCudaEngine(*converter.network())); @@ -3026,7 +3457,8 @@ tensorflow::Status ConvertSegmentToGraphDef( } } *common_scope = local_scope; - VLOG(0) << "Segment @scope '" << local_scope << "', converted to graph"; + VLOG(1) << "Converted TensorRT candidate segment @scope '" << local_scope + << "' to a GraphDef"; return tensorflow::Status::OK(); } diff --git a/tensorflow/contrib/tensorrt/convert/convert_nodes.h b/tensorflow/contrib/tensorrt/convert/convert_nodes.h index 5cc28b33e7f..f1c4c121ae6 100644 --- a/tensorflow/contrib/tensorrt/convert/convert_nodes.h +++ b/tensorflow/contrib/tensorrt/convert/convert_nodes.h @@ -92,7 +92,8 @@ struct EngineInfo { EngineInfo() : engine_type(EngineType::TRTStatic), max_workspace_size_bytes(0), - precision_mode(FP32MODE) {} + precision_mode(FP32MODE), + use_calibration(true) {} string engine_name; string device; @@ -109,6 +110,7 @@ struct EngineInfo { int maximum_cached_engines; std::vector cached_engine_batches; int precision_mode; + bool use_calibration; }; // Constructs a graphdef from the segment in the given graph. Adds placeholder @@ -145,7 +147,7 @@ tensorflow::Status ConvertGraphDefToEngine( const std::vector& input_shapes, Logger* logger, nvinfer1::IGpuAllocator* allocator, TRTInt8Calibrator* calibrator, - TrtUniquePtrType* engine, + TrtUniquePtrType* engine, bool use_calibration, bool* convert_successfully); // Helper class for the segmenter to determine whether an output edge from the @@ -392,7 +394,8 @@ class TrtNodeValidator { // Class to convert TF nodes to TRT network. class Converter { public: - Converter(nvinfer1::INetworkDefinition* trt_network, bool is_fp16); + Converter(nvinfer1::INetworkDefinition* trt_network, int precision_mode, + bool use_calibration); ////////////////////////////////////////////////////////////////////////////// // Methods used by the TRT engine builder to build a TRT network from a TF @@ -422,8 +425,43 @@ class Converter { // to add TRT layers. nvinfer1::INetworkDefinition* network() { return trt_network_; } - // Is the converter operating in fp16 mode? - bool is_fp16() const { return is_fp16_; } + // What precision are we targeting? + int precision_mode() const { return precision_mode_; } + + // Calibration will be or was previously performed on this network? + bool use_calibration() const { return use_calibration_; } + + // This should be called on the inputs and outputs of any layer we create + // where we know that the quantization range does not change during that + // operation. (e.g. Reshape, Transpose, Identity, MaxPool). + void MarkQuantizationRangesAsInferrable(nvinfer1::ITensor* input, + nvinfer1::ITensor* output); + + // This function should be called when we know the quantization range of a + // tensor, either from a quantize/dequantize node or when the output is a + // fixed range (e.g. SoftMax, Relu6, Sigmoid). + void ProvideQuantizationRange(nvinfer1::ITensor* tensor, float min_range, + float max_range); + + // Should be called when full TRT network has been constructed and before + // building the engine. + void MaybeApplyQuantizationRanges(); + + // This should be called on the inputs and outputs of any layer we create + // where we know that the quantization range does not change during that + // operation. (e.g. Reshape, Transpose, Identity, MaxPool). + void MarkQuantizationRangesAsInferrable(nvinfer1::ITensor* input, + nvinfer1::ITensor* output); + + // This function should be called when we know the quantization range of a + // tensor, either from a quantize/dequantize node or when the output is a + // fixed range (e.g. SoftMax, Relu6, Sigmoid). + void ProvideQuantizationRange(nvinfer1::ITensor* tensor, + float min_range, float max_range); + + // Should be called when full TRT network has been constructed and before + // building the engine. + void ApplyQuantizationRanges(bool warn_missing_ranges); // Below are helper methods for op converters to add different layers to the // TRT network. @@ -440,6 +478,13 @@ class Converter { const nvinfer1::Dims& dims, const nvinfer1::ITensor** tensor); + // Return OK if the broadcast scheme is supported and compute the shapes after + // broadcasting. + Status GetTrtBroadcastShape(const TRT_TensorOrWeights& operand_l, + const TRT_TensorOrWeights& operand_r, + nvinfer1::Dims* operand_l_new_dims, + nvinfer1::Dims* operand_r_new_dims) const; + private: // Verify the provided batch_size is consistent with batch_size_ and update it // if necessary. @@ -457,6 +502,12 @@ class Converter { void RegisterOpConverters(); + void PropagateQuantizationRanges(); + + // Gets the min and max value in a TRT_ShapedWeights + Status GetWeightRange(const TRT_ShapedWeights& weights, float* out_min, + float* out_max) const; + // Registered op converters by op type. std::unordered_map op_registry_; @@ -472,7 +523,25 @@ class Converter { // Store the weights added during construction of trt_network_. TrtWeightStore weight_store_; - const bool is_fp16_; + // During conversion, this table is populated with quantization ranges per + // tensor. MaybeApplyQuantizationRanges() will use this table to set the TRT + // quantization ranges. Since TRT only supports symmetric ranges, we will + // store the range as a single float = max(abs(min_range), abs(max_range)). + // Range refers to the floating point values, e.g. min_range = 0.0f, max_range + // = 6.0f for Relu6. + std::unordered_map quantization_ranges_; + + // Edges where quantization ranges can be inferred (copied) across ops - from + // first tensor to second tensor. PropagateQuantizationRanges() will propagate + // known ranges from quantization_ranges_ across these edges, adding the new + // ranges to quantization_ranges_ so that they can be applied in + // MaybeApplyQuantizationRanges(). + std::vector> + quantization_infer_; + + const int precision_mode_; + + const bool use_calibration_; // Batch size of inputs to trt_network_ added by AddInputTensor(). During // network construction it will update this, use it to verify the batch diff --git a/tensorflow/contrib/tensorrt/convert/convert_nodes_test.cc b/tensorflow/contrib/tensorrt/convert/convert_nodes_test.cc index c3a39395f3a..a95ab8dfbbb 100644 --- a/tensorflow/contrib/tensorrt/convert/convert_nodes_test.cc +++ b/tensorflow/contrib/tensorrt/convert/convert_nodes_test.cc @@ -35,7 +35,10 @@ limitations under the License. #include "tensorflow/core/grappler/costs/graph_properties.h" #include "tensorflow/core/lib/core/status.h" #include "tensorflow/core/lib/core/status_test_util.h" +#include "tensorflow/core/lib/strings/strcat.h" #include "tensorflow/core/platform/test.h" +#include "tensorflow/core/protobuf/config.pb.h" // NOLINT +#include "tensorflow/core/public/session.h" #if GOOGLE_CUDA #if GOOGLE_TENSORRT @@ -47,7 +50,9 @@ namespace tensorflow { namespace tensorrt { namespace convert { +using ::tensorflow::strings::StrCat; using ::testing::ElementsAre; +using ::testing::ElementsAreArray; // TODO(laigd): put this into some test utils file. void ExpectStatus(Status status, error::Code code = error::OK, @@ -69,6 +74,32 @@ nvinfer1::Dims GetTestDims(const std::vector& d) { return dims; } +nvinfer1::DataType TfDataTypeToTrt(DataType tf_dtype) { + switch (tf_dtype) { + case DT_FLOAT: + return nvinfer1::DataType::kFLOAT; + case DT_HALF: + return nvinfer1::DataType::kHALF; + case DT_INT32: + return nvinfer1::DataType::kINT32; + default: + QCHECK(false) << "Unexpected data type " << DataTypeString(tf_dtype); + } +} + +DataType TrtDataTypeToTf(nvinfer1::DataType trt_dtype) { + switch (trt_dtype) { + case nvinfer1::DataType::kFLOAT: + return DT_FLOAT; + case nvinfer1::DataType::kHALF: + return DT_HALF; + case nvinfer1::DataType::kINT32: + return DT_INT32; + default: + QCHECK(false) << "Unexpected data type " << static_cast(trt_dtype); + } +} + NodeDef MakeNodeDef(const string& name, const string& op, const std::vector& inputs) { NodeDef node_def; @@ -111,6 +142,35 @@ bool TrtDimsEqualsArray(const std::vector& lhs, return TrtDimsEquals(GetTestDims(lhs), rhs); } +// TODO(laigd): define a parameterized matcher that can compare against the +// vector. +void ExpectTrtDimsEqualsArray(const std::vector& lhs, + const nvinfer1::Dims& rhs) { + EXPECT_TRUE(TrtDimsEqualsArray(lhs, rhs)) + << "expected: " << DebugString(GetTestDims(lhs)) << "\n" + << " actual: " << DebugString(rhs); +} + +template +void ExpectArrayNear(const std::vector& lhs, const std::vector& rhs) { + ASSERT_EQ(lhs.size(), rhs.size()); + for (int i = 0; i < lhs.size(); i++) { + EXPECT_FLOAT_EQ(lhs[i], rhs[i]); + } +} + +// Eigen::half cannot implicitly convert to float which is required for +// EXPECT_FLOAT_EQ. +template <> +void ExpectArrayNear(const std::vector& lhs, + const std::vector& rhs) { + ASSERT_EQ(lhs.size(), rhs.size()); + for (int i = 0; i < lhs.size(); i++) { + EXPECT_FLOAT_EQ(Eigen::half_impl::half_to_float(lhs[i]), + Eigen::half_impl::half_to_float(rhs[i])); + } +} + bool TrtShapedWeightsEquals(const TRT_ShapedWeights& lhs, const TRT_ShapedWeights& rhs) { return TrtDimsEquals(lhs.shape_, rhs.shape_) && lhs.type_ == rhs.type_ && @@ -121,8 +181,7 @@ template void ValidateWeights(const TRT_ShapedWeights& weights, const std::vector& expected_dims, const std::vector& expected_value) { - EXPECT_TRUE(TrtDimsEqualsArray(expected_dims, weights.shape_)) - << weights.DebugString(); + ExpectTrtDimsEqualsArray(expected_dims, weights.shape_); ASSERT_EQ(expected_value.size(), weights.count()) << weights.DebugString(); const T* actual_values = static_cast(weights.GetValues()); for (int i = 0; i < expected_value.size(); ++i) { @@ -133,11 +192,12 @@ void ValidateWeights(const TRT_ShapedWeights& weights, // Fake ITensor implementation for testing purposes. class FakeITensor : public nvinfer1::ITensor { public: - FakeITensor() {} + FakeITensor() : dynamic_range_(0.0f) {} - FakeITensor(const nvinfer1::Dims& dims) : dims_(dims) {} + FakeITensor(const nvinfer1::Dims& dims) : dims_(dims), dynamic_range_(0.0f) {} - FakeITensor(const std::vector& dims) : dims_(GetTestDims(dims)) {} + FakeITensor(const std::vector& dims) + : dims_(GetTestDims(dims)), dynamic_range_(0.0f) {} void setName(const char* name) override { name_ = name; } @@ -166,7 +226,12 @@ class FakeITensor : public nvinfer1::ITensor { } #if NV_TENSORRT_MAJOR >= 5 - bool setDynamicRange(float min, float max) override {} + bool setDynamicRange(float min, float max) override { + dynamic_range_ = std::max(std::abs(min), std::abs(max)); + return true; + } + + float getDynamicRange() const override { return dynamic_range_; } #endif private: @@ -174,6 +239,7 @@ class FakeITensor : public nvinfer1::ITensor { nvinfer1::Dims dims_; nvinfer1::DataType type_; nvinfer1::TensorLocation location_; + float dynamic_range_; }; TEST(TRT_ShapedWeights_Test, Basic) { @@ -265,9 +331,7 @@ TEST(TRT_TensorOrWeights_Test, Basic) { EXPECT_EQ(1, ptr->batch_size()); } EXPECT_EQ(&itensor, ptr->tensor()); - EXPECT_TRUE(TrtDimsEqualsArray({1}, ptr->GetTrtDims())) - << "- expected: " << DebugString(dims) - << "\n vs\n- actual: " << DebugString(ptr->GetTrtDims()); + ExpectTrtDimsEqualsArray({1}, ptr->GetTrtDims()); } } } @@ -286,9 +350,7 @@ TEST(TRT_TensorOrWeights_Test, Basic) { EXPECT_EQ(false, ptr->is_weights()); EXPECT_EQ(1, ptr->batch_size()); EXPECT_NE(nullptr, ptr->tensor()); - EXPECT_TRUE(TrtDimsEqualsArray({1}, ptr->GetTrtDims())) - << "- expected: " << DebugString(dims) - << "\n vs\n- actual: " << DebugString(ptr->GetTrtDims()); + ExpectTrtDimsEqualsArray({1}, ptr->GetTrtDims()); } } // Test constructor with TRT_ShapedWeights argument. @@ -305,9 +367,7 @@ TEST(TRT_TensorOrWeights_Test, Basic) { nvinfer1::Dims dims; dims.nbDims = 0; - EXPECT_TRUE(TrtDimsEqualsArray({}, ptr->GetTrtDims())) - << "- expected: " << DebugString(dims) - << "\n vs\n- actual: " << DebugString(ptr->GetTrtDims()); + ExpectTrtDimsEqualsArray({}, ptr->GetTrtDims()); } } } @@ -341,34 +401,50 @@ TEST_F(ValidatorTest, ConvertToTensorOrWeights) { graph_properties, &output)); ValidateWeights(output.weights(), {2}, {1.0, 2.0}); } - // Convert non-Const. We test the case where the non-batch dimemsion is - // unknown as well, to make sure the validator allows that. - for (const int32 non_batch_dim : {-1, 2}) { - const int32 batch_size = 12; + // Helper method to run ConvertToTensorOrWeights() with predefined parameters. + auto convert_to_tensor_or_weights = [this](const std::vector& dims, + TRT_TensorOrWeights* output) { Scope s = Scope::NewRootScope(); - ops::Placeholder::Attrs attrs; - TF_EXPECT_OK(TensorShapeUtils::MakeShape( - std::vector{batch_size, non_batch_dim}, &attrs.shape_)); + const auto attrs = ops::Placeholder::Shape(PartialTensorShape{dims}); auto feed = ops::Placeholder(s.WithOpName("feed"), DT_FLOAT, attrs); auto add = ops::Add(s.WithOpName("add"), feed, feed); grappler::GrapplerItem item; TF_EXPECT_OK(s.ToGraphDef(&item.graph)); - grappler::GraphProperties graph_properties(item); TF_EXPECT_OK(graph_properties.InferStatically(true)); - - auto& node_def = add.operation.node()->def(); + const NodeDef& node_def = add.operation.node()->def(); + return this->ConvertToTensorOrWeights(node_def, /*output_port=*/0, + graph_properties, output); + }; + // Convert non-Const with #dims > nvinfer1::Dims::MAX_DIMS+1. + { TRT_TensorOrWeights output; - ExpectStatus(ConvertToTensorOrWeights(node_def, /*output_port=*/0, - graph_properties, &output)); + ExpectStatus( + convert_to_tensor_or_weights( + std::vector(nvinfer1::Dims::MAX_DIMS + 2, 1), &output), + error::OUT_OF_RANGE, "Input tensor rank is greater than 9"); + } + // Convert non-Const with #dims < 2. + { + TRT_TensorOrWeights output; + ExpectStatus( + convert_to_tensor_or_weights({1}, &output), error::INVALID_ARGUMENT, + "Input tensor with rank<2 is not supported since the first dimension " + "is treated as batch dimension by TRT"); + } + // Convert non-Const. We test the case where the non-batch dimemsion is + // unknown as well, to make sure the validator allows that. + for (const int32 non_batch_dim : {-1, 2}) { + const int32 batch_size = 12; + TRT_TensorOrWeights output; + ExpectStatus( + convert_to_tensor_or_weights({batch_size, non_batch_dim}, &output)); EXPECT_EQ(true, output.is_tensor()); EXPECT_EQ(batch_size, output.batch_size()); EXPECT_NE(nullptr, output.tensor()); - EXPECT_TRUE(TrtDimsEqualsArray({non_batch_dim}, output.GetTrtDims())) - << "- expected: {" << non_batch_dim << "} \n vs\n" - << "- actual: " << DebugString(output.GetTrtDims()); + ExpectTrtDimsEqualsArray({non_batch_dim}, output.GetTrtDims()); } } @@ -405,7 +481,9 @@ class ConverterTest : public ::testing::Test { ConverterTest() { builder_.reset(nvinfer1::createInferBuilder(logger_)); network_.reset(builder_->createNetwork()); - converter_.reset(new Converter(network_.get(), /*fp16=*/false)); + converter_.reset(new Converter(network_.get(), + /*precision_mode=*/FP32MODE, + /*use_calibration=*/false)); weight_store_ = &converter_->weight_store_; } @@ -432,8 +510,21 @@ class ConverterTest : public ::testing::Test { return converter_->GetInputs(node_def, inputs); } + Status GetWeightRange(const TRT_ShapedWeights& weights, float* out_min, + float* out_max) const { + return converter_->GetWeightRange(weights, out_min, out_max); + } + + void PropagateQuantizationRanges() { + converter_->PropagateQuantizationRanges(); + } + int batch_size() const { return converter_->batch_size_; } + std::unordered_map& quantization_ranges() { + return converter_->quantization_ranges_; + } + private: Logger logger_; // These members are ordered in a way such that the destruction order is: @@ -504,9 +595,9 @@ TEST_F(ConverterTest, AddAndGetInputs) { EXPECT_EQ(nvinfer1::DataType::kFLOAT, inputs[0].tensor()->getType()); EXPECT_EQ(nvinfer1::DataType::kINT32, inputs[2].tensor()->getType()); EXPECT_EQ(nvinfer1::DataType::kHALF, inputs[3].tensor()->getType()); - EXPECT_TRUE(TrtDimsEqualsArray({1}, inputs[0].tensor()->getDimensions())); - EXPECT_TRUE(TrtDimsEqualsArray({2, 3}, inputs[2].tensor()->getDimensions())); - EXPECT_TRUE(TrtDimsEqualsArray({5, 3}, inputs[3].tensor()->getDimensions())); + ExpectTrtDimsEqualsArray({1}, inputs[0].tensor()->getDimensions()); + ExpectTrtDimsEqualsArray({2, 3}, inputs[2].tensor()->getDimensions()); + ExpectTrtDimsEqualsArray({5, 3}, inputs[3].tensor()->getDimensions()); } TEST_F(ConverterTest, RenameAndMarkOutputTensors) { @@ -552,7 +643,7 @@ TEST_F(ConverterTest, RenameAndMarkOutputTensors) { {{"my_op", "my_output"}, {"my_op:1", "my_output_1"}})); EXPECT_EQ(2, output_tensors.size()); for (auto output_tensor : output_tensors) { - EXPECT_TRUE(TrtDimsEqualsArray({2, 1}, output_tensor->getDimensions())); + ExpectTrtDimsEqualsArray({2, 1}, output_tensor->getDimensions()); } EXPECT_EQ("my_output", string(output_tensors[0]->getName())); EXPECT_EQ("my_output_1", string(output_tensors[1]->getName())); @@ -577,8 +668,7 @@ TEST_F(ConverterTest, TransposeTensor) { // OK. TF_EXPECT_OK( converter_->TransposeTensor(input_tensor, {0, 3, 1, 2}, &output_tensor)); - EXPECT_TRUE(TrtDimsEqualsArray({5, 2, 3}, output_tensor->getDimensions())) - << DebugString(*output_tensor); + ExpectTrtDimsEqualsArray({5, 2, 3}, output_tensor->getDimensions()); } TEST_F(ConverterTest, PrepareTensorForShape_Tensor) { @@ -590,7 +680,7 @@ TEST_F(ConverterTest, PrepareTensorForShape_Tensor) { // Shape size doesn't match. ExpectStatus(converter_->PrepareTensorForShape(tw, GetTestDims({2, 3, 6}), &output_tensor), - error::INVALID_ARGUMENT, "Reshape shapes are not compatible."); + error::INVALID_ARGUMENT, "Reshape shapes are not compatible"); // TODO(aaroey): we should check the case where uninferred dimensions are not // an exact divisor of input dim ensions, e.g. for dims {-1, 7}. @@ -598,14 +688,12 @@ TEST_F(ConverterTest, PrepareTensorForShape_Tensor) { // Infer shape, ok. TF_EXPECT_OK(converter_->PrepareTensorForShape(tw, GetTestDims({-1, 2}), &output_tensor)); - EXPECT_TRUE(TrtDimsEqualsArray({15, 2}, output_tensor->getDimensions())) - << DebugString(*output_tensor); + ExpectTrtDimsEqualsArray({15, 2}, output_tensor->getDimensions()); // Regular shape. TF_EXPECT_OK(converter_->PrepareTensorForShape(tw, GetTestDims({10, 3}), &output_tensor)); - EXPECT_TRUE(TrtDimsEqualsArray({10, 3}, output_tensor->getDimensions())) - << DebugString(*output_tensor); + ExpectTrtDimsEqualsArray({10, 3}, output_tensor->getDimensions()); } TEST_F(ConverterTest, PrepareTensorForShape_Weights) { @@ -615,8 +703,7 @@ TEST_F(ConverterTest, PrepareTensorForShape_Weights) { const nvinfer1::ITensor* output_tensor = nullptr; TF_EXPECT_OK(converter_->PrepareTensorForShape(tw, GetTestDims({10, 3}), &output_tensor)); - EXPECT_TRUE(TrtDimsEqualsArray({10, 3}, output_tensor->getDimensions())) - << DebugString(*output_tensor); + ExpectTrtDimsEqualsArray({10, 3}, output_tensor->getDimensions()); } TEST_F(ConverterTest, MaybeUpdateBatchSize) { @@ -656,6 +743,178 @@ TEST_F(ConverterTest, AddAndGetTensorOrWeights) { "tensor/weights my_tensor already exist"); } +template +void TestGetWeightRange(ConverterTest* test, TrtWeightStore* weight_store) { + TRT_ShapedWeights weights = + weight_store->GetTempWeights(DataTypeToEnum::v(), GetTestDims({2, 3})); + const std::vector values = {T(3), T(1), T(2), T(6), T(5), T(4)}; + memcpy(const_cast(weights.GetValues()), values.data(), + weights.size_bytes()); + + float out_min = 0.0f; + float out_max = 0.0f; + TF_EXPECT_OK(test->GetWeightRange(weights, &out_min, &out_max)); + EXPECT_EQ(1.0f, out_min); + EXPECT_EQ(6.0f, out_max); +} + +TEST_F(ConverterTest, GetWeightRange) { + TestGetWeightRange(this, weight_store_); + TestGetWeightRange(this, weight_store_); + TestGetWeightRange(this, weight_store_); +} + +TEST_F(ConverterTest, ProvideQuantizationRange) { + FakeITensor fake_tensor; + // Assymetric range + converter_->ProvideQuantizationRange(&fake_tensor, 0.0f, 6.0f); + EXPECT_EQ(6.0f, quantization_ranges()[&fake_tensor]); + converter_->ProvideQuantizationRange(&fake_tensor, 1.0f, 6.0f); + EXPECT_EQ(6.0f, quantization_ranges()[&fake_tensor]); + converter_->ProvideQuantizationRange(&fake_tensor, -8.0f, 6.0f); + EXPECT_EQ(8.0f, quantization_ranges()[&fake_tensor]); + converter_->ProvideQuantizationRange(&fake_tensor, -8.123f, -6.123f); + EXPECT_EQ(8.123f, quantization_ranges()[&fake_tensor]); + // Symmetric range + converter_->ProvideQuantizationRange(&fake_tensor, -6.123f, 6.123f); + EXPECT_EQ(6.123f, quantization_ranges()[&fake_tensor]); +} + +TEST_F(ConverterTest, MaybeApplyQuantizationRanges) { + // input -> infer1 -> infer2 -> infer3 + FakeITensor input, infer_1, infer_2, infer_3; + FakeITensor not_infer; + Converter int8_converter(/*trt_network=*/nullptr, INT8MODE, + /*use_calibration=*/true); + int8_converter.ProvideQuantizationRange(&input, -5.0f, 5.0f); + int8_converter.ProvideQuantizationRange(¬_infer, -100.0f, 100.0f); + int8_converter.MarkQuantizationRangesAsInferrable(&input, &infer_1); + int8_converter.MarkQuantizationRangesAsInferrable(&infer_1, &infer_2); + int8_converter.MarkQuantizationRangesAsInferrable(&infer_2, &infer_3); + + // Input range should be inferred along the chain and applied to tensors. + int8_converter.MaybeApplyQuantizationRanges(); +#if NV_TENSORRT_MAJOR >= 5 + EXPECT_EQ(input.getDynamicRange(), 5.0f); + EXPECT_EQ(infer_1.getDynamicRange(), 5.0f); + EXPECT_EQ(infer_2.getDynamicRange(), 5.0f); + EXPECT_EQ(infer_3.getDynamicRange(), 5.0f); + EXPECT_EQ(not_infer.getDynamicRange(), 100.0f); +#endif +} + +TEST_F(ConverterTest, PropagateQuantizationRanges) { + // infer0 <-> infer1 <-> infer2 <-> infer3 + // | + // infer4 <-> infer5 + FakeITensor infer[6]; + FakeITensor not_infer; + converter_->ProvideQuantizationRange(&infer[4], -5.0f, 5.0f); + converter_->MarkQuantizationRangesAsInferrable(&infer[0], &infer[1]); + converter_->MarkQuantizationRangesAsInferrable(&infer[1], &infer[2]); + converter_->MarkQuantizationRangesAsInferrable(&infer[3], &infer[2]); + converter_->MarkQuantizationRangesAsInferrable(&infer[4], &infer[1]); + converter_->MarkQuantizationRangesAsInferrable(&infer[4], &infer[5]); + + // Input range should be inferred along the chain. + PropagateQuantizationRanges(); + auto ranges = quantization_ranges(); + for (int i = 0; i < 6; ++i) { + EXPECT_EQ(5.0f, ranges[&infer[i]]); + } + EXPECT_EQ(ranges.count(¬_infer), 0); +} + +TEST_F(ConverterTest, GetTrtBroadcastShape) { + const bool kIsTensor = true; + const bool kIsNotTensor = false; + auto symmetric_test = [this](const std::vector& operand_1_shape, + const std::vector& operand_2_shape, + const bool operand_1_is_tensor, + const bool operand_2_is_tensor, + const std::vector& expected_operand_1_shape, + const std::vector& expected_operand_2_shape, + error::Code expected_code = error::OK, + const char* expected_error_msg_substr = nullptr, + const int operand_1_batch_size = -1, + const int operand_2_batch_size = -1) { + auto create_tensor_or_weights = [](const std::vector& shape, + bool is_tensor, int batch_size = -1) { + if (is_tensor) { + return TRT_TensorOrWeights{nvinfer1::DataType::kFLOAT, + GetTestDims(shape), batch_size}; + } + TRT_ShapedWeights weights; + weights.shape_ = GetTestDims(shape); + return TRT_TensorOrWeights(weights); + }; + + nvinfer1::Dims operand_1_new_dims, operand_2_new_dims; + TRT_TensorOrWeights operand_1 = create_tensor_or_weights( + operand_1_shape, operand_1_is_tensor, operand_1_batch_size); + TRT_TensorOrWeights operand_2 = create_tensor_or_weights( + operand_2_shape, operand_2_is_tensor, operand_2_batch_size); + + // operand_1 broadcast operand_2 + ExpectStatus( + this->converter_->GetTrtBroadcastShape( + operand_1, operand_2, &operand_1_new_dims, &operand_2_new_dims), + expected_code, expected_error_msg_substr); + if (expected_code == error::OK) { + ExpectTrtDimsEqualsArray(expected_operand_1_shape, operand_1_new_dims); + ExpectTrtDimsEqualsArray(expected_operand_2_shape, operand_2_new_dims); + } + // operand_2 broadcast operand_1 + ExpectStatus( + this->converter_->GetTrtBroadcastShape( + operand_2, operand_1, &operand_2_new_dims, &operand_1_new_dims), + expected_code, expected_error_msg_substr); + if (expected_code == error::OK) { + ExpectTrtDimsEqualsArray(expected_operand_1_shape, operand_1_new_dims); + ExpectTrtDimsEqualsArray(expected_operand_2_shape, operand_2_new_dims); + } + }; + + // Both inputs are weights. + symmetric_test( + {1}, {1}, kIsNotTensor, kIsNotTensor, {}, {}, error::INVALID_ARGUMENT, + "Broadcasting requires at least one of the operands be tensors"); + + // One tensor and one weights. + symmetric_test({1, 1, 1}, {2}, kIsTensor, kIsNotTensor, {1, 1, 1}, {1, 1, 2}); + symmetric_test({1, 1, 2}, {2}, kIsTensor, kIsNotTensor, {1, 1, 2}, {1, 1, 2}); + symmetric_test({1, 3, 2}, {1}, kIsTensor, kIsNotTensor, {1, 3, 2}, {1, 1, 1}); + symmetric_test({1, 1, 1}, {2, 3}, kIsTensor, kIsNotTensor, {1, 1, 1}, + {1, 2, 3}); + symmetric_test({1, 1, 1}, {2, 3, 4}, kIsTensor, kIsNotTensor, {1, 1, 1}, + {2, 3, 4}); + symmetric_test({1, 1, 1}, {1, 2, 3, 4}, kIsTensor, kIsNotTensor, {1, 1, 1}, + {2, 3, 4}); + symmetric_test({1, 3, 4}, {1, 2, 1, 4}, kIsTensor, kIsNotTensor, {1, 3, 4}, + {2, 1, 4}); + symmetric_test({1, 1, 1}, {2, 1, 1, 1}, kIsTensor, kIsNotTensor, {}, {}, + error::INVALID_ARGUMENT, "Infeasible broadcast scheme"); + symmetric_test({1, 1, 1}, {2, 1, 1, 1}, kIsTensor, kIsNotTensor, {}, {}, + error::INVALID_ARGUMENT, "Infeasible broadcast scheme", + /*operand_1_batch_size=*/2); + symmetric_test({1, 1, 1}, {1, 1, 1, 1, 1}, kIsTensor, kIsNotTensor, {}, {}, + error::INVALID_ARGUMENT, + "Broadcasting beyond batch dimension is not supported " + "(tensor #dims 4 vs broadcast #dims 5)"); + + // Both inputs are tensors. + symmetric_test({1, 1, 1}, {1, 1}, kIsTensor, kIsTensor, {}, {}, + error::INVALID_ARGUMENT, + "Broadcasting beyond batch dimension is not supported " + "(tensor #dims 3 vs broadcast #dims 4)"); + symmetric_test({1, 3, 4}, {2, 1, 4}, kIsTensor, kIsTensor, {1, 3, 4}, + {2, 1, 4}); + symmetric_test({1, 1, 1}, {1, 1, 1, 1}, kIsTensor, kIsTensor, {}, {}, + error::INVALID_ARGUMENT, + "Broadcasting beyond batch dimension is not supported " + "(tensor #dims 4 vs broadcast #dims 5)"); +} + // Class to test various op converters, using both a TrtNodeValidator and // Converter. class OpConverterTest : public ::testing::Test { @@ -684,15 +943,21 @@ class OpConverterTest : public ::testing::Test { // Reset the validator and converter. validator_.reset(new TrtNodeValidator); - converter_.reset(new Converter(network_.get(), /*fp16=*/false)); + converter_.reset(new Converter(network_.get(), + /*precision_mode=*/FP32MODE, + /*use_calibration=*/false)); // Reset other related artifacts. scope_ = Scope::NewRootScope(); validator_inputs_.clear(); } - void BuildAndRun(const char* input_name, const std::vector& input_data, - const char* output_name, std::vector* output_data) { + // TODO(laigd): test fp16 and int8 support. + template + void BuildAndRun( + const std::vector>>& + input_data, + const char* output_name, std::vector* output_data) { // Mark the output tensor as TRT engine output. TF_EXPECT_OK(converter_->RenameAndMarkOutputTensors( {{string(output_name), string(output_name)}})); @@ -703,25 +968,33 @@ class OpConverterTest : public ::testing::Test { CHECK_NOTNULL(engine_.get()); // Execute the TRT engine. - const int input_size = input_data.size() * sizeof(float); - const int output_size = output_data->size() * sizeof(float); - const int input_index = engine_->getBindingIndex(input_name); - const int output_index = engine_->getBindingIndex(output_name); + ASSERT_LE(input_data.size() + 1, 3); + void* buffers[3]; + for (const auto name_and_data : input_data) { + const int input_size = name_and_data.second.size() * sizeof(T); + const int input_index = engine_->getBindingIndex(name_and_data.first); + ASSERT_EQ(0, cudaMalloc(&buffers[input_index], input_size)); + ASSERT_EQ( + 0, cudaMemcpyAsync(buffers[input_index], name_and_data.second.data(), + input_size, cudaMemcpyHostToDevice, stream_)); + } - ASSERT_EQ(engine_->getNbBindings(), 2); - void* buffers[2]; - ASSERT_EQ(0, cudaMalloc(&buffers[input_index], input_size)); + const int output_size = output_data->size() * sizeof(T); + const int output_index = engine_->getBindingIndex(output_name); ASSERT_EQ(0, cudaMalloc(&buffers[output_index], output_size)); - ASSERT_EQ(0, cudaMemcpyAsync(buffers[input_index], input_data.data(), - input_size, cudaMemcpyHostToDevice, stream_)); + + ASSERT_EQ(engine_->getNbBindings(), input_data.size() + 1); + TrtUniquePtrType execution_context( engine_->createExecutionContext()); execution_context->enqueue(/*batchSize=*/1, buffers, stream_, nullptr); ASSERT_EQ(0, cudaMemcpyAsync(output_data->data(), buffers[output_index], output_size, cudaMemcpyDeviceToHost, stream_)); cudaStreamSynchronize(stream_); - ASSERT_EQ(0, cudaFree(buffers[input_index])); - ASSERT_EQ(0, cudaFree(buffers[output_index])); + + for (int i = 0; i < input_data.size() + 1; ++i) { + ASSERT_EQ(0, cudaFree(buffers[i])); + } } bool HasStaticShape(const nvinfer1::Dims& dims) const { @@ -736,18 +1009,7 @@ class OpConverterTest : public ::testing::Test { void AddTestTensor( const char* name, const std::vector& dims, int batch_size = 1, nvinfer1::DataType trt_dtype = nvinfer1::DataType::kFLOAT) { - DataType tf_dtype = DT_FLOAT; - switch (trt_dtype) { - case nvinfer1::DataType::kFLOAT: - tf_dtype = DT_FLOAT; - break; - case nvinfer1::DataType::kINT32: - tf_dtype = DT_INT32; - break; - default: - ASSERT_TRUE(false) << "Unexpected data type " - << static_cast(trt_dtype); - } + DataType tf_dtype = TrtDataTypeToTf(trt_dtype); ops::Placeholder::Attrs attrs; TF_EXPECT_OK(TensorShapeUtils::MakeShape(dims, &attrs.shape_)); attrs.shape_.InsertDim(0, batch_size); @@ -826,6 +1088,11 @@ class OpConverterTest : public ::testing::Test { } } + // Expose quantization_ranges_ for tests + std::unordered_map& quantization_ranges() { + return converter_->quantization_ranges_; + } + std::unique_ptr converter_; std::unique_ptr validator_; @@ -835,6 +1102,11 @@ class OpConverterTest : public ::testing::Test { TrtUniquePtrType network_; TrtUniquePtrType engine_; cudaStream_t stream_; + // Used to create placeholders with shape and data type information. The + // created placeholders will be used as inputs to the node to be verified, + // thus we need the shape and data type information to get a non-empty + // GraphProperties. + // TODO(laigd): consider use this Scope to create the NodeDef to verify. Scope scope_; std::unordered_map validator_inputs_; }; @@ -958,15 +1230,15 @@ TEST_F(OpConverterTest, ConvertTranspose) { Reset(); AddTestTensor("input", {1, 2, 3}); AddTestWeights("weights", {4}, {0, 3, 1, 2}); - RunConversion(node_def); + RunValidationAndConversion(node_def); TRT_TensorOrWeights output; TF_EXPECT_OK(GetTensorOrWeights("my_transpose", &output)); EXPECT_TRUE(output.is_tensor()); - EXPECT_TRUE(TrtDimsEqualsArray({3, 1, 2}, output.tensor()->getDimensions())) - << output.DebugString(); + ExpectTrtDimsEqualsArray({3, 1, 2}, output.tensor()->getDimensions()); std::vector output_data(6); - BuildAndRun("input", {1, 2, 3, 4, 5, 6}, "my_transpose", &output_data); + BuildAndRun({{"input", {1, 2, 3, 4, 5, 6}}}, "my_transpose", + &output_data); EXPECT_THAT(output_data, ElementsAre(1, 4, 2, 5, 3, 6)); } } @@ -1048,15 +1320,15 @@ TEST_F(OpConverterTest, ConvertReshape) { Reset(); AddTestTensor("input", ok_params[i].tensor_dims, ok_params[i].batch_size); AddTestWeights("weights", {4}, ok_params[i].shape); - RunConversion(node_def); + RunValidationAndConversion(node_def); TRT_TensorOrWeights output; TF_EXPECT_OK(GetTensorOrWeights("my_reshape", &output)); EXPECT_TRUE(output.is_tensor()); - EXPECT_TRUE(TrtDimsEqualsArray({1, 3, 2}, output.tensor()->getDimensions())) - << output.DebugString(); + ExpectTrtDimsEqualsArray({1, 3, 2}, output.tensor()->getDimensions()); std::vector output_data(6); - BuildAndRun("input", {1, 2, 3, 4, 5, 6}, "my_reshape", &output_data); + BuildAndRun({{"input", {1, 2, 3, 4, 5, 6}}}, "my_reshape", + &output_data); EXPECT_THAT(output_data, ElementsAre(1, 2, 3, 4, 5, 6)); } } @@ -1070,15 +1342,14 @@ TEST_F(OpConverterTest, ConvertMatMul) { "Input expects tensor and weights, at my_matmul"); } - // Get the NodeDef for Reshape. + // Get the NodeDef for MatMul. auto get_matmul_nodedef = [](DataType dtype, bool transpose_a, bool transpose_b) -> NodeDef { Scope s = Scope::NewRootScope(); auto input = ops::Placeholder(s.WithOpName("input"), dtype); auto weights = ops::Placeholder(s.WithOpName("weights"), dtype); - ops::MatMul::Attrs matmul_attrs; - matmul_attrs.transpose_a_ = transpose_a; - matmul_attrs.transpose_b_ = transpose_b; + const auto matmul_attrs = + ops::MatMul::TransposeA(transpose_a).TransposeB(transpose_b); auto matmul = ops::MatMul(s.WithOpName("my_matmul"), input, weights, matmul_attrs); return matmul.operation.node()->def(); @@ -1094,45 +1365,845 @@ TEST_F(OpConverterTest, ConvertMatMul) { node_def, error::UNIMPLEMENTED, "Data type is not supported, for node my_matmul got int32"); } - { - // transpose_a is set. - for (bool transpose_b : {false, true}) { - Reset(); - NodeDef node_def = - get_matmul_nodedef(DT_FLOAT, /*transpose_a=*/true, transpose_b); - AddTestTensor("input", {2}, /*batch_size=*/1); - AddTestWeights("weights", {2, 2}, {0, 1, 2, 3}); - RunValidationAndConversion( - node_def, error::INVALID_ARGUMENT, - "transpose_a is not supported for TensorRT FullyConnected"); + // transpose_a is set. + for (bool transpose_b : {false, true}) { + Reset(); + NodeDef node_def = + get_matmul_nodedef(DT_FLOAT, /*transpose_a=*/true, transpose_b); + AddTestTensor("input", {2}, /*batch_size=*/1); + AddTestWeights("weights", {2, 2}, {0, 1, 2, 3}); + RunValidationAndConversion( + node_def, error::INVALID_ARGUMENT, + "transpose_a is not supported for TensorRT FullyConnected"); + } + // OK. + for (bool transpose_b : {false, true}) { + Reset(); + NodeDef node_def = + get_matmul_nodedef(DT_FLOAT, /*transpose_a=*/false, transpose_b); + AddTestTensor("input", {2}, /*batch_size=*/1); + AddTestWeights("weights", {2, 2}, {0, 1, 2, 3}); + RunValidationAndConversion(node_def); + TRT_TensorOrWeights output; + TF_EXPECT_OK(GetTensorOrWeights("my_matmul", &output)); + EXPECT_TRUE(output.is_tensor()); + ExpectTrtDimsEqualsArray({2}, output.tensor()->getDimensions()); + + std::vector output_data(2); + BuildAndRun({{"input", {0, 1}}}, "my_matmul", &output_data); + if (transpose_b) { + EXPECT_THAT(output_data, ElementsAre(1, 3)); + } else { + EXPECT_THAT(output_data, ElementsAre(2, 3)); } } - { - // OK. - for (bool transpose_b : {false, true}) { - Reset(); - NodeDef node_def = - get_matmul_nodedef(DT_FLOAT, /*transpose_a=*/false, transpose_b); - AddTestTensor("input", {2}, /*batch_size=*/1); - AddTestWeights("weights", {2, 2}, {0, 1, 2, 3}); - RunConversion(node_def); - TRT_TensorOrWeights output; - TF_EXPECT_OK(GetTensorOrWeights("my_matmul", &output)); - EXPECT_TRUE(output.is_tensor()); - EXPECT_TRUE(TrtDimsEqualsArray({2}, output.tensor()->getDimensions())) - << output.DebugString(); +} - std::vector output_data(2); - BuildAndRun("input", {0, 1}, "my_matmul", &output_data); - if (transpose_b) { - EXPECT_THAT(output_data, ElementsAre(1, 3)); +template +void TestConvertBiasAdd(OpConverterTest* test) { + // Get the NodeDef for BiasAdd. + auto get_biasadd_nodedef = [](const string& data_format) -> NodeDef { + Scope s = Scope::NewRootScope(); + auto input = ops::Placeholder(s.WithOpName("input"), dtype); + auto weights = ops::Placeholder(s.WithOpName("weights"), dtype); + const auto biasadd_attrs = ops::BiasAdd::DataFormat(data_format); + auto biasadd = + ops::BiasAdd(s.WithOpName("my_biasadd"), input, weights, biasadd_attrs); + return biasadd.operation.node()->def(); + }; + + typedef typename EnumToDataType::Type CType; + for (const string& data_format : {"NHWC", "NCHW"}) { + for (const int trt_input_rank : {1, 2, 3, 4}) { + test->Reset(); + NodeDef node_def = get_biasadd_nodedef(data_format); + + // Add input, dims_array will be like {2, 1, ..., 1, 3} + std::vector dims_array(trt_input_rank, 1); + if (trt_input_rank == 1) { + dims_array[0] = (data_format == "NHWC" ? 3 : 2); } else { - EXPECT_THAT(output_data, ElementsAre(2, 3)); + dims_array[0] = 2; + dims_array[trt_input_rank - 1] = 3; + } + test->AddTestTensor("input", dims_array, /*batch_size=*/1, + TfDataTypeToTrt(dtype)); + + // Add bias weights. + const int channel_size = (data_format == "NHWC" ? 3 : 2); + std::vector bias(channel_size); + for (int i = 0; i < channel_size; ++i) { + bias[i] = CType(i + 1); // bias will be {1, 2, 3, ...} + } + test->AddTestWeights("weights", {channel_size}, bias); + + // Run the conversion. + test->RunValidationAndConversion(node_def); + TRT_TensorOrWeights output; + TF_EXPECT_OK(test->GetTensorOrWeights("my_biasadd", &output)); + EXPECT_TRUE(output.is_tensor()); + ExpectTrtDimsEqualsArray(dims_array, output.tensor()->getDimensions()); + + // Build and run the engine. + const int num_input = TrtDimsNumElements(GetTestDims(dims_array)); + ASSERT_EQ(trt_input_rank > 1 ? 6 : (data_format == "NHWC" ? 3 : 2), + num_input); + std::vector output_data(num_input); + test->BuildAndRun( + {{"input", std::vector(num_input, CType(0))}}, "my_biasadd", + &output_data); + if (trt_input_rank == 1) { + if (data_format == "NHWC") { + EXPECT_THAT(output_data, ElementsAre(CType(1), CType(2), CType(3))); + } else { + EXPECT_THAT(output_data, ElementsAre(CType(1), CType(2))); + } + } else { + if (data_format == "NHWC") { + EXPECT_THAT(output_data, ElementsAre(CType(1), CType(2), CType(3), + CType(1), CType(2), CType(3))); + } else { + EXPECT_THAT(output_data, ElementsAre(CType(1), CType(1), CType(1), + CType(2), CType(2), CType(2))); + } } } } } +TEST_F(OpConverterTest, ConvertQuantize) { + { + // Input list is empty, should fail. + NodeDef node_def = + MakeNodeDef("my_quantize", "QuantizeAndDequantizeV2", {}); + RunConversion( + node_def, error::INVALID_ARGUMENT, + "Invalid number of inputs for QuantizeAndDequantizeV2, at my_quantize"); + } + { + // FakeQuantWithMinMaxArgs attributes are empty, should fail. + NodeDef node_def = + MakeNodeDef("my_quantize", "FakeQuantWithMinMaxArgs", {"input"}); + AddTestTensor("input", {1, 2, 3}); + RunConversion(node_def, error::INVALID_ARGUMENT, + "Min or max attribute not found for FakeQuantWithMinMaxArgs " + "at my_quantize"); + } + { + // FakeQuantWithMinMaxArgs ranges set via attributes, ok. + Reset(); + Scope s = Scope::NewRootScope(); + auto input = ops::Placeholder(s.WithOpName("input"), DT_FLOAT); + ops::FakeQuantWithMinMaxArgs::Attrs quantize_attrs; + quantize_attrs.min_ = -6.0f; + quantize_attrs.max_ = 6.0f; + auto quantize = ops::FakeQuantWithMinMaxArgs(s.WithOpName("my_quantize"), + input, quantize_attrs); + const NodeDef& node_def = quantize.operation.node()->def(); + AddTestTensor("input", {1, 2, 3}); + RunConversion(node_def); + TRT_TensorOrWeights output; + TF_EXPECT_OK(GetTensorOrWeights("my_quantize", &output)); + EXPECT_TRUE(output.is_tensor()); + auto ranges = quantization_ranges(); + EXPECT_EQ(ranges.count(output.tensor()), 1); + EXPECT_EQ(ranges[output.tensor()], 6.0f); + } + { + // FakeQuantWithMinMaxVars ranges set via inputs, ok. + Reset(); + Scope s = Scope::NewRootScope(); + auto input = ops::Placeholder(s.WithOpName("input"), DT_FLOAT); + auto weights_min = ops::Placeholder(s.WithOpName("weights_min"), DT_FLOAT); + auto weights_max = ops::Placeholder(s.WithOpName("weights_max"), DT_FLOAT); + auto quantize = ops::FakeQuantWithMinMaxVars( + s.WithOpName("my_quantize"), input, weights_min, weights_max); + const NodeDef& node_def = quantize.operation.node()->def(); + AddTestTensor("input", {1, 2, 3}); + AddTestWeights("weights_min", {1}, {-6.0f}); + AddTestWeights("weights_max", {1}, {6.0f}); + RunConversion(node_def); + TRT_TensorOrWeights output; + TF_EXPECT_OK(GetTensorOrWeights("my_quantize", &output)); + EXPECT_TRUE(output.is_tensor()); + auto ranges = quantization_ranges(); + EXPECT_EQ(ranges.count(output.tensor()), 1); + EXPECT_EQ(ranges[output.tensor()], 6.0f); + } + { + // QuantizeAndDequantizeV2 ranges set via inputs, ok. + Reset(); + Scope s = Scope::NewRootScope(); + auto input = ops::Placeholder(s.WithOpName("input"), DT_FLOAT); + auto weights_min = ops::Placeholder(s.WithOpName("weights_min"), DT_FLOAT); + auto weights_max = ops::Placeholder(s.WithOpName("weights_max"), DT_FLOAT); + auto quantize = ops::QuantizeAndDequantizeV2( + s.WithOpName("my_quantize"), input, weights_min, weights_max); + const NodeDef& node_def = quantize.operation.node()->def(); + AddTestTensor("input", {1, 2, 3}); + AddTestWeights("weights_min", {1}, {-6.0f}); + AddTestWeights("weights_max", {1}, {6.0f}); + RunConversion(node_def); + TRT_TensorOrWeights output; + TF_EXPECT_OK(GetTensorOrWeights("my_quantize", &output)); + EXPECT_TRUE(output.is_tensor()); + auto ranges = quantization_ranges(); + EXPECT_EQ(ranges.count(output.tensor()), 1); + EXPECT_EQ(ranges[output.tensor()], 6.0f); + } + { + // QuantizeAndDequantizeV3 ranges set via inputs, ok. + Reset(); + Scope s = Scope::NewRootScope(); + auto input = ops::Placeholder(s.WithOpName("input"), DT_FLOAT); + auto weights_min = ops::Placeholder(s.WithOpName("weights_min"), DT_FLOAT); + auto weights_max = ops::Placeholder(s.WithOpName("weights_max"), DT_FLOAT); + auto num_bits = ops::Placeholder(s.WithOpName("num_bits"), DT_INT32); + auto quantize = ops::QuantizeAndDequantizeV3( + s.WithOpName("my_quantize"), input, weights_min, weights_max, num_bits); + const NodeDef& node_def = quantize.operation.node()->def(); + AddTestTensor("input", {1, 2, 3}); + AddTestWeights("weights_min", {1}, {-6.0f}); + AddTestWeights("weights_max", {1}, {6.0f}); + AddTestWeights("num_bits", {1}, {8}); + RunConversion(node_def); + TRT_TensorOrWeights output; + TF_EXPECT_OK(GetTensorOrWeights("my_quantize", &output)); + EXPECT_TRUE(output.is_tensor()); + auto ranges = quantization_ranges(); + EXPECT_EQ(ranges.count(output.tensor()), 1); + EXPECT_EQ(ranges[output.tensor()], 6.0f); + } + { + // QuantizeAndDequantizeV2 Range inputs are tensors, should fail. + Reset(); + Scope s = Scope::NewRootScope(); + auto input = ops::Placeholder(s.WithOpName("input"), DT_FLOAT); + auto weights_min = ops::Placeholder(s.WithOpName("weights_min"), DT_FLOAT); + auto weights_max = ops::Placeholder(s.WithOpName("weights_max"), DT_FLOAT); + auto quantize = ops::QuantizeAndDequantizeV2( + s.WithOpName("my_quantize"), input, weights_min, weights_max); + const NodeDef& node_def = quantize.operation.node()->def(); + AddTestTensor("input", {1, 2, 3}); + AddTestTensor("weights_min", {1}); + AddTestTensor("weights_max", {1}); + RunConversion( + node_def, error::INVALID_ARGUMENT, + "Min and max inputs for QuantizeAndDequantizeV2 must be weights not " + "tensors, at my_quantize"); + } +} + +TEST_F(OpConverterTest, ConvertRelu6) { + { + // Input list is empty, should fail. + NodeDef node_def = MakeNodeDef("my_relu6", "Relu6", {}); + RunConversion(node_def, error::INVALID_ARGUMENT, + "Invalid number of inputs for Relu6, at my_relu6"); + } + + // Get the NodeDef for Relu6. + Scope s = Scope::NewRootScope(); + auto input = ops::Placeholder(s.WithOpName("input"), DT_FLOAT); + auto relu6 = ops::Relu6(s.WithOpName("my_relu6"), input); + const NodeDef& node_def = relu6.operation.node()->def(); + + { + // Clip tensor values and set quantization ranges, ok. + Reset(); + AddTestTensor("input", {1, 2, 3}); + RunConversion(node_def); + TRT_TensorOrWeights output; + TF_EXPECT_OK(GetTensorOrWeights("my_relu6", &output)); + EXPECT_TRUE(output.is_tensor()); + auto ranges = quantization_ranges(); + EXPECT_EQ(ranges[output.tensor()], 6.0f); + + std::vector output_data(6); + BuildAndRun("input", {-100, -1, 0, 3, 5, 9}, "my_relu6", &output_data); + EXPECT_THAT(output_data, ElementsAre(0, 0, 0, 3, 5, 6)); + } + { + // Input is weights, should fail. + Reset(); + AddTestWeights("input", {1, 2, 3}, {-100, -1, 0, 3, 5, 9}); + RunConversion( + node_def, error::UNIMPLEMENTED, + "Relu6 is only implemented for tensors, not weights, at my_relu6"); + } +} + +TEST_F(OpConverterTest, ConvertBiasAdd) { + { + // Input list is empty, should fail. + NodeDef node_def = MakeNodeDef("my_biasadd", "BiasAdd", {}); + RunValidationAndConversion( + node_def, error::INVALID_ARGUMENT, + "Input expects tensor and weights, at my_biasadd"); + } + + // OK. Note that kINT32 is not supported by IScaleLayer, so we don't test + // DT_INT32 type here. + TestConvertBiasAdd(this); + TestConvertBiasAdd(this); +} + +template +NodeDef GetBinaryOpNodeDef(const string& input_name_l, + const string& input_name_r, DataType dtype) { + Scope s = Scope::NewRootScope(); + auto input_l = ops::Placeholder(s.WithOpName(input_name_l), dtype); + auto input_r = ops::Placeholder(s.WithOpName(input_name_r), dtype); + auto op = OpType(s.WithOpName("my_binary"), input_l, input_r); + return op.operation.node()->def(); +} + +void CheckAddedLayers(OpConverterTest* test, bool expect_scale_layer) { + bool element_wise_layer_found = false; + bool scale_layer_found = false; + for (int i = 0; i < test->converter_->network()->getNbLayers(); i++) { + nvinfer1::ILayer* layer = test->converter_->network()->getLayer(i); + if (dynamic_cast(layer)) { + scale_layer_found = true; + } else if (dynamic_cast(layer)) { + element_wise_layer_found = true; + } + } + EXPECT_EQ(expect_scale_layer, scale_layer_found); + EXPECT_NE(expect_scale_layer, element_wise_layer_found); +} + +template +void TestBinaryTensorOpWeightNoBroadcast(OpConverterTest* test) { + typedef typename EnumToDataType::Type CType; + for (auto swap_inputs : {false, true}) { + test->Reset(); + NodeDef node_def; + if (swap_inputs) { + node_def = GetBinaryOpNodeDef("weights", "input", dtype); + } else { + node_def = GetBinaryOpNodeDef("input", "weights", dtype); + } + + const std::vector operand1{CType(3), CType(7.5)}; + const std::vector operand2{CType(2), CType(3)}; + + // It requires the dims to be at least of rank 3 to apply an IScaleLayer. + test->AddTestTensor("input", /*dims=*/{1, 1, 2}, /*batch_size=*/1, + TfDataTypeToTrt(dtype)); + test->AddTestWeights("weights", /*dims=*/{1, 1, 2}, + /*values=*/swap_inputs ? operand1 : operand2); + test->RunValidationAndConversion(node_def); + + // Make sure it does use BinaryTensorOpWeight, not BinaryTensorOpTensor. + CheckAddedLayers(test, /*expect_scale_layer=*/true); + + // Check the dims of the output ITensor. + TRT_TensorOrWeights output; + TF_EXPECT_OK(test->GetTensorOrWeights("my_binary", &output)); + EXPECT_TRUE(output.is_tensor()); + ExpectTrtDimsEqualsArray({1, 1, 2}, output.tensor()->getDimensions()); + + std::vector output_data(2); + test->BuildAndRun( + {{"input", + /*input_data=*/swap_inputs ? operand2 : operand1}}, + "my_binary", &output_data); + if (node_def.op() == "Add") { + EXPECT_THAT(output_data, ElementsAre(CType(5), CType(10.5))); + } else if (node_def.op() == "Sub") { + EXPECT_THAT(output_data, ElementsAre(CType(1), CType(4.5))); + } else if (node_def.op() == "Mul") { + EXPECT_THAT(output_data, ElementsAre(CType(6), CType(22.5))); + } else if (node_def.op() == "Div") { + EXPECT_THAT(output_data, ElementsAre(CType(1.5), CType(2.5))); + } else if (node_def.op() == "RealDiv") { + EXPECT_THAT(output_data, ElementsAre(CType(1.5), CType(2.5))); + } else { + ASSERT_TRUE(false); + } + } +} + +template +void TestBinaryTensorOpWeightWithChannelWiseBroadcast(OpConverterTest* test) { + typedef typename EnumToDataType::Type CType; + const NodeDef node_def = + GetBinaryOpNodeDef("input", "weights", dtype); + const std::vector input{CType(1), CType(2), CType(3), CType(4)}; + const std::vector weights{CType(10), CType(20)}; + // There are two types of valid dim pairs which requires channel-wise + // broadcasting: + // - input dims (X Y Z) vs weights dims (X 1 1) + // - input dims (X Y Z) vs weights dims (Z) + // Here X=Z=2 and Y=1. + for (auto weights_dims : std::vector>{{2, 1, 1}, {2}}) { + test->Reset(); + test->AddTestTensor("input", /*dims=*/{2, 1, 2}, /*batch_size=*/1, + TfDataTypeToTrt(dtype)); + test->AddTestWeights("weights", weights_dims, weights); + test->RunValidationAndConversion(node_def); + + // Make sure it does use BinaryTensorOpWeight, not BinaryTensorOpTensor. + CheckAddedLayers(test, /*expect_scale_layer=*/true); + + // Check the dims of the output ITensor. + TRT_TensorOrWeights output; + TF_EXPECT_OK(test->GetTensorOrWeights("my_binary", &output)); + EXPECT_TRUE(output.is_tensor()); + ExpectTrtDimsEqualsArray({2, 1, 2}, output.tensor()->getDimensions()); + + std::vector output_data(4); + test->BuildAndRun({{"input", input}}, "my_binary", &output_data); + if (weights_dims.size() == 1) { + EXPECT_THAT(output_data, + ElementsAre(CType(11), CType(22), CType(13), CType(24))); + } else { + EXPECT_THAT(output_data, + ElementsAre(CType(11), CType(12), CType(23), CType(24))); + } + } +} + +template +void TestBinaryTensorOpWeightWithUniformlyBroadcast(OpConverterTest* test) { + typedef typename EnumToDataType::Type CType; + const NodeDef node_def = + GetBinaryOpNodeDef("input", "weights", dtype); + const std::vector input{CType(1), CType(2), CType(3), CType(4)}; + const std::vector weights{CType(10)}; + test->Reset(); + test->AddTestTensor("input", /*dims=*/{2, 1, 2}, /*batch_size=*/1, + TfDataTypeToTrt(dtype)); + test->AddTestWeights("weights", {1, 1, 1, 1}, weights); + test->RunValidationAndConversion(node_def); + + // Make sure it does use BinaryTensorOpWeight, not BinaryTensorOpTensor. + CheckAddedLayers(test, /*expect_scale_layer=*/true); + + // Check the dims of the output ITensor. + TRT_TensorOrWeights output; + TF_EXPECT_OK(test->GetTensorOrWeights("my_binary", &output)); + EXPECT_TRUE(output.is_tensor()); + ExpectTrtDimsEqualsArray({2, 1, 2}, output.tensor()->getDimensions()); + + std::vector output_data(4); + test->BuildAndRun({{"input", input}}, "my_binary", &output_data); + EXPECT_THAT(output_data, + ElementsAre(CType(11), CType(12), CType(13), CType(14))); +} + +template +void TestBinaryTensorOpWeightFallback(OpConverterTest* test, + const std::vector& input_dims, + const std::vector& weights_dims, + error::Code code = error::OK, + const char* error_msg_substr = nullptr, + const int input_batch_size = 1) { + const DataType dtype = DT_FLOAT; + typedef typename EnumToDataType::Type CType; + const size_t num_inputs = TrtDimsNumElements(GetTestDims(input_dims)); + const size_t num_weights = TrtDimsNumElements(GetTestDims(weights_dims)); + + test->Reset(); + const NodeDef node_def = + GetBinaryOpNodeDef("input", "weights", dtype); + test->AddTestTensor("input", /*dims=*/input_dims, input_batch_size, + TfDataTypeToTrt(dtype)); + test->AddTestWeights( + "weights", /*dims=*/weights_dims, + /*values=*/std::vector(num_weights, CType(1))); + test->RunValidationAndConversion(node_def, code, error_msg_substr); + if (code != error::OK) return; + + // Make sure it does use BinaryTensorOpTensor, not BinaryTensorOpWeight. + CheckAddedLayers(test, /*expect_scale_layer=*/false); + + TRT_TensorOrWeights output; + TF_EXPECT_OK(test->GetTensorOrWeights("my_binary", &output)); + EXPECT_TRUE(output.is_tensor()); + + // Check the dims of the output ITensor. + std::vector expected_output_dims = input_dims; + for (int i = expected_output_dims.size() - 1, j = weights_dims.size() - 1; + i >= 0 && j >= 0; --i, --j) { + if (expected_output_dims[i] == 1) { + expected_output_dims[i] = weights_dims[j]; + } + } + ExpectTrtDimsEqualsArray(expected_output_dims, + output.tensor()->getDimensions()); + + // Check the result of running the engine. + const int expected_num_outputs = + TrtDimsNumElements(GetTestDims(expected_output_dims)); + std::vector output_data(expected_num_outputs); + test->BuildAndRun( + {{"input", + /*input_data=*/std::vector(num_inputs, CType(2))}}, + "my_binary", &output_data); + if (node_def.op() == "Add") { + EXPECT_THAT(output_data, ElementsAreArray(std::vector( + expected_num_outputs, CType(3)))); + } else if (node_def.op() == "Minimum") { + EXPECT_THAT(output_data, ElementsAreArray(std::vector( + expected_num_outputs, CType(1)))); + } else { + ASSERT_TRUE(false); + } +} + +template +void TestBinaryTensorOpTensor(OpConverterTest* test) { + typedef typename EnumToDataType::Type CType; + test->Reset(); + const NodeDef node_def = + GetBinaryOpNodeDef("input1", "input2", dtype); + test->AddTestTensor("input1", /*dims=*/{1, 2}, /*batch_size=*/1, + TfDataTypeToTrt(dtype)); + test->AddTestTensor("input2", /*dims=*/{2, 1}, /*batch_size=*/1, + TfDataTypeToTrt(dtype)); + test->RunValidationAndConversion(node_def); + + // Make sure it does use BinaryTensorOpTensor, not BinaryTensorOpWeight. + CheckAddedLayers(test, /*expect_scale_layer=*/false); + + // Check output dims. + TRT_TensorOrWeights output; + TF_EXPECT_OK(test->GetTensorOrWeights("my_binary", &output)); + EXPECT_TRUE(output.is_tensor()); + ExpectTrtDimsEqualsArray({2, 2}, output.tensor()->getDimensions()); + + std::vector output_data(4); + // After broadcasting first input becomes {3, 6, 3, 6} and second input + // becomes {2, 3, 2, 3}. + test->BuildAndRun( + {{"input1", {CType(3), CType(6)}}, {"input2", {CType(2), CType(3)}}}, + "my_binary", &output_data); + if (node_def.op() == "Add") { + EXPECT_THAT(output_data, + ElementsAre(CType(5), CType(8), CType(6), CType(9))); + } else if (node_def.op() == "Sub") { + EXPECT_THAT(output_data, + ElementsAre(CType(1), CType(4), CType(0), CType(3))); + } else if (node_def.op() == "Mul") { + EXPECT_THAT(output_data, + ElementsAre(CType(6), CType(12), CType(9), CType(18))); + } else if (node_def.op() == "Div") { + EXPECT_THAT(output_data, + ElementsAre(CType(1.5), CType(3), CType(1), CType(2))); + } else if (node_def.op() == "RealDiv") { + EXPECT_THAT(output_data, + ElementsAre(CType(1.5), CType(3), CType(1), CType(2))); + } else if (node_def.op() == "Minimum") { + EXPECT_THAT(output_data, + ElementsAre(CType(2), CType(2), CType(3), CType(3))); + } else if (node_def.op() == "Maximum") { + EXPECT_THAT(output_data, + ElementsAre(CType(3), CType(6), CType(3), CType(6))); + } else { + ASSERT_TRUE(false); + } +} + +TEST_F(OpConverterTest, ConvertBinary) { + // Input size doesn't match, should fail. + for (size_t num_inputs = 0; num_inputs < 2; ++num_inputs) { + Reset(); + NodeDef node_def = MakeNodeDef("my_add", "Add", {num_inputs, "input"}); + AddTestTensor("input", {1}, /*batch_size=*/1, nvinfer1::DataType::kFLOAT); + RunValidationAndConversion(node_def, error::INVALID_ARGUMENT, + "Binary ops require two inputs, at my_add"); + } + { + // Both inputs are weights. + Reset(); + NodeDef node_def = MakeNodeDef("my_add", "Add", {"weights1", "weights2"}); + AddTestWeights("weights1", {1}, {1}); + AddTestWeights("weights2", {1}, {1}); + RunValidationAndConversion( + node_def, error::UNIMPLEMENTED, + "Constant folding is falled back to TensorFlow, binary op received " + "both input as constant at: my_add"); + } + + // Test BinaryTensorOpWeight() without broadcasting. + TestBinaryTensorOpWeightNoBroadcast(this); + TestBinaryTensorOpWeightNoBroadcast(this); + TestBinaryTensorOpWeightNoBroadcast(this); + TestBinaryTensorOpWeightNoBroadcast(this); + TestBinaryTensorOpWeightNoBroadcast(this); +#if 0 + // TODO(b/119560144): it doesn't support FP16 constants and the following test + // will fail. + TestBinaryTensorOpWeightNoBroadcast(this); + TestBinaryTensorOpWeightNoBroadcast(this); + TestBinaryTensorOpWeightNoBroadcast(this); + TestBinaryTensorOpWeightNoBroadcast(this); + TestBinaryTensorOpWeightNoBroadcast(this); +#endif + + // Test BinaryTensorOpWeight() with channel-wise broadcasting. + TestBinaryTensorOpWeightWithChannelWiseBroadcast(this); + + // Test BinaryTensorOpWeight() with uniformly broadcasting. + TestBinaryTensorOpWeightWithUniformlyBroadcast(this); + + // Test BinaryTensorOpWeight() falling back to BinaryTensorOpTensor(). + // Unsupported op. + TestBinaryTensorOpWeightFallback(this, {1, 1, 1}, {1}); + // Rank of input tensor dimension <3. + TestBinaryTensorOpWeightFallback(this, {1, 1}, {1}); + // Broadcast on batch dimension, should fail. + TestBinaryTensorOpWeightFallback( + this, {1, 1, 1}, {2, 1, 1, 1}, error::INVALID_ARGUMENT, + "Unsupported binary op broadcast scheme for op my_binary", + /*input_batch_size=*/2); + // Incompatible dims with per-channel mode. + TestBinaryTensorOpWeightFallback(this, {1, 1, 1}, {1, 2, 1}); + // Incompatible dims. + TestBinaryTensorOpWeightFallback(this, {1, 2, 1}, {2}); + + // Test BinaryTensorOpTensor() with broadcasting. + TestBinaryTensorOpTensor(this); + TestBinaryTensorOpTensor(this); + TestBinaryTensorOpTensor(this); + TestBinaryTensorOpTensor(this); + TestBinaryTensorOpTensor(this); + TestBinaryTensorOpTensor(this); + TestBinaryTensorOpTensor(this); + + TestBinaryTensorOpTensor(this); + TestBinaryTensorOpTensor(this); + TestBinaryTensorOpTensor(this); + TestBinaryTensorOpTensor(this); + TestBinaryTensorOpTensor(this); + TestBinaryTensorOpTensor(this); + TestBinaryTensorOpTensor(this); +} + +TEST_F(OpConverterTest, ConvertQuantize) { + for (const string& op : + {"FakeQuantWithMinMaxArgs", "FakeQuantWithMinMaxVars", + "QuantizeAndDequantizeV2", "QuantizeAndDequantizeV3"}) { + // Input list is empty, should fail. + NodeDef node_def = MakeNodeDef("my_quantize", op, {}); + RunValidationAndConversion( + node_def, error::INVALID_ARGUMENT, + StrCat("Invalid number of inputs for ", op, ", at my_quantize") + .c_str()); + } + { + // FakeQuantWithMinMaxArgs attributes are empty, should fail. + NodeDef node_def = + MakeNodeDef("my_quantize", "FakeQuantWithMinMaxArgs", {"input"}); + AddTestTensor("input", {1, 2, 3}); + RunValidationAndConversion( + node_def, error::INVALID_ARGUMENT, + "Min or max attribute not found for FakeQuantWithMinMaxArgs " + "at my_quantize"); + } + { + // FakeQuantWithMinMaxArgs ranges set via attributes, ok. + Reset(); + Scope s = Scope::NewRootScope(); + auto input = ops::Placeholder(s.WithOpName("input"), DT_FLOAT); + auto quantize_attrs = ops::FakeQuantWithMinMaxArgs::Min(-6.0f).Max(6.0f); + auto quantize = ops::FakeQuantWithMinMaxArgs(s.WithOpName("my_quantize"), + input, quantize_attrs); + const NodeDef& node_def = quantize.operation.node()->def(); + AddTestTensor("input", {1, 2, 3}); + RunValidationAndConversion(node_def); + TRT_TensorOrWeights output; + TF_EXPECT_OK(GetTensorOrWeights("my_quantize", &output)); + EXPECT_TRUE(output.is_tensor()); + auto ranges = quantization_ranges(); + EXPECT_EQ(1, ranges.count(output.tensor())); + EXPECT_EQ(6.0f, ranges[output.tensor()]); + } + { + // FakeQuantWithMinMaxVars ranges set via inputs, ok. + Reset(); + Scope s = Scope::NewRootScope(); + auto input = ops::Placeholder(s.WithOpName("input"), DT_FLOAT); + auto weights_min = ops::Placeholder(s.WithOpName("weights_min"), DT_FLOAT); + auto weights_max = ops::Placeholder(s.WithOpName("weights_max"), DT_FLOAT); + auto quantize = ops::FakeQuantWithMinMaxVars( + s.WithOpName("my_quantize"), input, weights_min, weights_max); + const NodeDef& node_def = quantize.operation.node()->def(); + AddTestTensor("input", {1, 2, 3}); + AddTestWeights("weights_min", {1}, {-6.0f}); + AddTestWeights("weights_max", {1}, {6.0f}); + RunValidationAndConversion(node_def); + TRT_TensorOrWeights output; + TF_EXPECT_OK(GetTensorOrWeights("my_quantize", &output)); + EXPECT_TRUE(output.is_tensor()); + auto ranges = quantization_ranges(); + EXPECT_EQ(1, ranges.count(output.tensor())); + EXPECT_EQ(6.0f, ranges[output.tensor()]); + } + { + // QuantizeAndDequantizeV2 ranges set via inputs, ok. + Reset(); + Scope s = Scope::NewRootScope(); + auto input = ops::Placeholder(s.WithOpName("input"), DT_FLOAT); + auto weights_min = ops::Placeholder(s.WithOpName("weights_min"), DT_FLOAT); + auto weights_max = ops::Placeholder(s.WithOpName("weights_max"), DT_FLOAT); + auto quantize = ops::QuantizeAndDequantizeV2( + s.WithOpName("my_quantize"), input, weights_min, weights_max); + const NodeDef& node_def = quantize.operation.node()->def(); + AddTestTensor("input", {1, 2, 3}); + AddTestWeights("weights_min", {1}, {-6.0f}); + AddTestWeights("weights_max", {1}, {6.0f}); + RunValidationAndConversion(node_def); + TRT_TensorOrWeights output; + TF_EXPECT_OK(GetTensorOrWeights("my_quantize", &output)); + EXPECT_TRUE(output.is_tensor()); + auto ranges = quantization_ranges(); + EXPECT_EQ(1, ranges.count(output.tensor())); + EXPECT_EQ(6.0f, ranges[output.tensor()]); + } + { + // QuantizeAndDequantizeV2 Range inputs are tensors, should fail. + Reset(); + Scope s = Scope::NewRootScope(); + auto input = ops::Placeholder(s.WithOpName("input"), DT_FLOAT); + auto weights_min = ops::Placeholder(s.WithOpName("weights_min"), DT_FLOAT); + auto weights_max = ops::Placeholder(s.WithOpName("weights_max"), DT_FLOAT); + auto quantize = ops::QuantizeAndDequantizeV2( + s.WithOpName("my_quantize"), input, weights_min, weights_max); + const NodeDef& node_def = quantize.operation.node()->def(); + AddTestTensor("input", {1, 2, 3}); + AddTestTensor("weights_min", {1}); + AddTestTensor("weights_max", {1}); + RunValidationAndConversion( + node_def, error::INVALID_ARGUMENT, + "Min and max inputs for QuantizeAndDequantizeV2 must be weights not " + "tensors, at my_quantize"); + } + { + // QuantizeAndDequantizeV3 ranges set via inputs, ok. + Reset(); + Scope s = Scope::NewRootScope(); + auto input = ops::Placeholder(s.WithOpName("input"), DT_FLOAT); + auto weights_min = ops::Placeholder(s.WithOpName("weights_min"), DT_FLOAT); + auto weights_max = ops::Placeholder(s.WithOpName("weights_max"), DT_FLOAT); + auto num_bits = ops::Placeholder(s.WithOpName("num_bits"), DT_INT32); + auto quantize = ops::QuantizeAndDequantizeV3( + s.WithOpName("my_quantize"), input, weights_min, weights_max, num_bits); + const NodeDef& node_def = quantize.operation.node()->def(); + AddTestTensor("input", {1, 2, 3}); + AddTestWeights("weights_min", {1}, {-6.0f}); + AddTestWeights("weights_max", {1}, {6.0f}); + AddTestWeights("num_bits", {1}, {8}); + RunValidationAndConversion(node_def); + TRT_TensorOrWeights output; + TF_EXPECT_OK(GetTensorOrWeights("my_quantize", &output)); + EXPECT_TRUE(output.is_tensor()); + auto ranges = quantization_ranges(); + EXPECT_EQ(1, ranges.count(output.tensor())); + EXPECT_EQ(6.0f, ranges[output.tensor()]); + } +} + +TEST_F(OpConverterTest, ConvertRelu6) { + { + // Input list is empty, should fail. + NodeDef node_def = MakeNodeDef("my_relu6", "Relu6", {}); + RunValidationAndConversion( + node_def, error::INVALID_ARGUMENT, + "Invalid number of inputs for Relu6, at my_relu6"); + } + + // Get the NodeDef for Relu6. + Scope s = Scope::NewRootScope(); + auto input = ops::Placeholder(s.WithOpName("input"), DT_FLOAT); + auto relu6 = ops::Relu6(s.WithOpName("my_relu6"), input); + const NodeDef node_def = relu6.operation.node()->def(); + { + // Input is weights, should fail. + Reset(); + AddTestWeights("input", {1}, {1.0f}); + RunValidationAndConversion( + node_def, error::UNIMPLEMENTED, + "Relu6 is only implemented for tensors, not weights, at my_relu6"); + } + { + // Clip tensor values and set quantization ranges, ok. + Reset(); + AddTestTensor("input", {1, 2, 3}); + RunValidationAndConversion(node_def); + TRT_TensorOrWeights output; + TF_EXPECT_OK(GetTensorOrWeights("my_relu6", &output)); + EXPECT_TRUE(output.is_tensor()); + auto ranges = quantization_ranges(); + EXPECT_EQ(ranges[output.tensor()], 6.0f); + + std::vector output_data(6); + BuildAndRun({{"input", {-100, -1, 0, 3, 5, 9}}}, "my_relu6", + &output_data); + EXPECT_THAT(output_data, ElementsAre(0, 0, 0, 3, 5, 6)); + } +} + +template +void TestConvertSquare(OpConverterTest* test) { + test->Reset(); + typedef typename EnumToDataType::Type CType; + + Scope s = Scope::NewRootScope(); + auto input = ops::Placeholder(s.WithOpName("input"), dtype); + auto square = ops::Square(s.WithOpName("my_square"), input); + NodeDef node_def = square.operation.node()->def(); + + test->AddTestTensor("input", {1, 20}); + test->RunValidationAndConversion(node_def); + TRT_TensorOrWeights output; + TF_EXPECT_OK(test->GetTensorOrWeights("my_square", &output)); + EXPECT_TRUE(output.is_tensor()); + ExpectTrtDimsEqualsArray({1, 20}, output.tensor()->getDimensions()); + + const int num_inputs = 20; + std::vector input_data(num_inputs); + std::vector expected_output_data(num_inputs); + for (int i = 0; i < 20; i++) { + const CType value = CType(i - 9); + input_data[i] = value; + expected_output_data[i] = value * value; + } + std::vector output_data(num_inputs); + test->BuildAndRun({{"input", input_data}}, "my_square", &output_data); + ExpectArrayNear(expected_output_data, output_data); +} + +TEST_F(OpConverterTest, ConvertSquare) { + { + // Input list is empty, should fail. + NodeDef node_def = MakeNodeDef("my_square", "Square", {}); + RunValidationAndConversion(node_def, error::INVALID_ARGUMENT, + "Square expects one input, at my_square"); + } + { + // Input is weights, should fail. + Reset(); + Scope s = Scope::NewRootScope(); + auto input = ops::Placeholder(s.WithOpName("input"), DT_FLOAT); + auto square = ops::Square(s.WithOpName("my_square"), input); + NodeDef node_def = square.operation.node()->def(); + AddTestWeights("input", {1, 2, 3}, {1, 2, 3, 4, -5, 6}); + RunValidationAndConversion( + node_def, error::UNIMPLEMENTED, + "Square is only implemented for tensors, at my_square"); + } + + // OK. Note that kINT32 is not supported by IElementWiseLayer, so we don't + // test DT_INT32 type here. + TestConvertSquare(this); + // TODO(tmorris): Looks like there may be a bug with this layer for FP16 + // inputs. Disabling for now. + // TestConvertSquare(this); +} + } // namespace convert } // namespace tensorrt } // namespace tensorflow diff --git a/tensorflow/contrib/tensorrt/convert/trt_optimization_pass.cc b/tensorflow/contrib/tensorrt/convert/trt_optimization_pass.cc index b30d94b0282..4ac7e21d348 100644 --- a/tensorflow/contrib/tensorrt/convert/trt_optimization_pass.cc +++ b/tensorflow/contrib/tensorrt/convert/trt_optimization_pass.cc @@ -67,6 +67,9 @@ tensorflow::Status TRTOptimizationPass::Init( TF_RETURN_IF_ERROR(GetPrecisionMode( Uppercase(params.at("precision_mode").s()), &precision_mode_)); } + if (params.count("use_calibration")) { + use_calibration_ = params.at("use_calibration").b(); + } return tensorflow::Status::OK(); } @@ -222,6 +225,12 @@ tensorflow::Status TRTOptimizationPass::Optimize( TF_RETURN_IF_ERROR(static_graph_properties.InferStatically(true)); tensorflow::tensorrt::convert::ConversionParams cp; + if (use_calibration_ && precision_mode_ != INT8MODE) { + LOG(ERROR) << "Calibration with FP32 or FP16 is not implemented. " + << "Falling back to use_calibration = False."; + use_calibration_ = false; + } + std::vector nodes_to_preserve; for (const auto& n : item.NodesToPreserve()) { auto tokens = str_util::Split(n, ":"); @@ -250,6 +259,7 @@ tensorflow::Status TRTOptimizationPass::Optimize( cp.is_dyn_op = is_dynamic_op_; cp.cached_engine_batches = batches_; cp.max_cached_engines = max_cached_batches_; + cp.use_calibration = use_calibration_; auto status = tensorflow::tensorrt::convert::ConvertAfterShapes(cp); VLOG(1) << "Returning from " << name_; return status; diff --git a/tensorflow/contrib/tensorrt/convert/trt_optimization_pass.h b/tensorflow/contrib/tensorrt/convert/trt_optimization_pass.h index 71b51d13681..3e8dc0978e4 100644 --- a/tensorflow/contrib/tensorrt/convert/trt_optimization_pass.h +++ b/tensorflow/contrib/tensorrt/convert/trt_optimization_pass.h @@ -38,7 +38,8 @@ class TRTOptimizationPass : public tensorflow::grappler::CustomGraphOptimizer { maximum_batch_size_(-1), is_dynamic_op_(false), max_cached_batches_(1), - max_workspace_size_bytes_(256LL << 20) { + max_workspace_size_bytes_(256LL << 20), + use_calibration_(true) { VLOG(1) << "Constructing " << name_; } @@ -67,6 +68,7 @@ class TRTOptimizationPass : public tensorflow::grappler::CustomGraphOptimizer { std::vector batches_; int max_cached_batches_; int64_t max_workspace_size_bytes_; + bool use_calibration_; }; } // namespace convert diff --git a/tensorflow/contrib/tensorrt/kernels/trt_engine_op.cc b/tensorflow/contrib/tensorrt/kernels/trt_engine_op.cc index 019446813a5..117039683c0 100644 --- a/tensorflow/contrib/tensorrt/kernels/trt_engine_op.cc +++ b/tensorflow/contrib/tensorrt/kernels/trt_engine_op.cc @@ -124,8 +124,10 @@ TRTEngineOp::TRTEngineOp(OpKernelConstruction* context) OP_REQUIRES_OK(context, context->GetAttr("segment_funcdef_name", &funcdef_name_)); OP_REQUIRES_OK(context, GetPrecisionMode(precision_string, &precision_mode_)); - calibration_mode_ = - (precision_mode_ == INT8MODE && calibration_data.size() == 0); + OP_REQUIRES_OK(context, + context->GetAttr("use_calibration", &use_calibration_)); + calibration_mode_ = (use_calibration_ && precision_mode_ == INT8MODE && + calibration_data.size() == 0); if (calibration_data.size()) { calibrator_.reset(new TRTInt8Calibrator(calibration_data)); calibration_data.resize(0); @@ -252,9 +254,8 @@ int TRTEngineOp::GetEngineBatch(OpKernelContext* ctx) { cached_engine_batches_.push_back(num_batch); VLOG(1) << "Running with batch size " << num_batch; } else { - string msg = - StrCat("Engine buffer is full. buffer limit=", max_cached_engines_, - ", current entries="); + string msg = StrCat("Engine buffer is full. buffer limit=", + max_cached_engines_, ", current entries="); for (auto i : cached_engine_batches_) StrAppend(&msg, i, ","); StrAppend(&msg, " requested batch=", num_batch); LOG(WARNING) << msg; @@ -308,7 +309,7 @@ bool TRTEngineOp::ExecuteTrtEngine( std::vector buffers(num_binding); for (int i = 0; i < ctx->num_inputs(); i++) { const string input_name = StrCat(kInputPHName, i); - const size_t binding_index = + const int binding_index = trt_engine_ptr->getBindingIndex(input_name.c_str()); if (binding_index == -1) { LOG(ERROR) << "Input node not found, at " << input_name; @@ -345,7 +346,7 @@ bool TRTEngineOp::ExecuteTrtEngine( for (int i = 0; i < ctx->num_outputs(); i++) { // Create an output tensor const string output_name = StrCat(kOutputPHName, i); - const size_t binding_index = + const int binding_index = trt_engine_ptr->getBindingIndex(output_name.c_str()); Tensor* output_tensor = nullptr; @@ -491,13 +492,14 @@ TRTEngineOp::EngineCtxPair& TRTEngineOp::GetEngine(int batch_size, } TrtUniquePtrType engine; bool convert_successfully = false; - VLOG(0) << name() << " Constructing a new engine with batch size " - << batch_size; + LOG(INFO) << "Building a new TensorRT engine for " << name() + << " with batch size " << batch_size; // Up to this point, calibrator_ can never be empty, since otherwise it // means calibration_mode_ is true and this path won't get executed. auto status = convert::ConvertGraphDefToEngine( segment_graph_, precision_mode_, batch_size, workspace_size_, shapes, - &logger, allocator, calibrator_.get(), &engine, &convert_successfully); + &logger, allocator, calibrator_.get(), &engine, use_calibration_, + &convert_successfully); if (!status.ok()) { if (convert_successfully) { // This means it fail to build the engine even when the network is built @@ -567,8 +569,8 @@ tensorflow::Status TRTEngineOp::AllocateCalibrationResources( const int64 workspace_size_bytes = workspace_size_; cres->thr_.reset(new std::thread([cres, label, segment_graph, shapes, platform_gpu_id, workspace_size_bytes]() { - VLOG(0) << "Starting calibration thread on device " << platform_gpu_id - << ", Calibration Resource @ " << cres; + LOG(INFO) << "Starting calibration thread on device " << platform_gpu_id + << ", Calibration Resource @ " << cres; auto err = cudaSetDevice(platform_gpu_id); if (err != cudaSuccess) { // TODO(aaroey): should return error here. @@ -586,6 +588,7 @@ tensorflow::Status TRTEngineOp::AllocateCalibrationResources( *segment_graph, INT8MODE, cres->calibrator_->getBatchSize(), workspace_size_bytes, shapes, &cres->logger_, cres->allocator_.get(), cres->calibrator_.get(), &cres->engine_, + /*use_calibration=*/true, /*convert_successfully=*/nullptr); if (!s.ok()) { LOG(ERROR) << "Calibration failed: " << s; diff --git a/tensorflow/contrib/tensorrt/kernels/trt_engine_op.h b/tensorflow/contrib/tensorrt/kernels/trt_engine_op.h index 8fe06758914..b545f497f32 100644 --- a/tensorflow/contrib/tensorrt/kernels/trt_engine_op.h +++ b/tensorflow/contrib/tensorrt/kernels/trt_engine_op.h @@ -130,6 +130,10 @@ class TRTEngineOp : public AsyncOpKernel { // The finalized calibrator for inference. std::unique_ptr calibrator_; + + // If true, create calibration graph for INT8 mode. Otherwise, we are using + // user-provided quantization ranges. + bool use_calibration_; }; } // namespace tensorrt diff --git a/tensorflow/contrib/tensorrt/ops/trt_engine_op.cc b/tensorflow/contrib/tensorrt/ops/trt_engine_op.cc index e0c7b627237..92405906eb7 100644 --- a/tensorflow/contrib/tensorrt/ops/trt_engine_op.cc +++ b/tensorflow/contrib/tensorrt/ops/trt_engine_op.cc @@ -16,6 +16,7 @@ limitations under the License. #if GOOGLE_CUDA #if GOOGLE_TENSORRT +#include "tensorflow/core/framework/common_shape_fns.h" #include "tensorflow/core/framework/op.h" #include "tensorflow/core/framework/op_kernel.h" #include "tensorflow/core/framework/shape_inference.h" @@ -39,18 +40,19 @@ REGISTER_OP("TRTEngineOp") .Attr("cached_engine_batches: list(int) = []") .Attr("max_cached_engines_count: int = 1") .Attr("workspace_size_bytes: int") - .Attr("precision_mode: {'FP32', 'FP16', 'INT8', 'INT8CALIB'}") + .Attr("precision_mode: {'FP32', 'FP16', 'INT8'}") .Attr("calibration_data: string = ''") + .Attr("use_calibration: bool = true") .Input("in_tensor: InT") - .Output("out_tensor: OutT"); -// TODO(jie): TF requires concrete output shape for concrete input shapes. -// This is tricky for batch dimension, since we cannot ensure which input -// would carry the correct batch dimension (for the current stage of the -// implementation, we do require all input tensor to carry the same batch -// size, but this could change in the future). Hence we disable shape -// inference function as a workaround. -// .SetShapeFn(shape_inference::TRTEngineOpShapeInference); - + .Output("out_tensor: OutT") + // TODO(jie): TF requires concrete output shape for concrete input shapes. + // This is tricky for batch dimension, since we cannot ensure which input + // would carry the correct batch dimension (for the current stage of the + // implementation, we do require all input tensor to carry the same batch + // size, but this could change in the future). Hence we disable shape + // inference function as a workaround. + // .SetShapeFn(shape_inference::TRTEngineOpShapeInference); + .SetShapeFn(shape_inference::UnknownShape); } // namespace tensorflow #endif // GOOGLE_TENSORRT diff --git a/tensorflow/contrib/tensorrt/python/trt_convert.py b/tensorflow/contrib/tensorrt/python/trt_convert.py index bb81fbf93f3..74a2c2392ad 100644 --- a/tensorflow/contrib/tensorrt/python/trt_convert.py +++ b/tensorflow/contrib/tensorrt/python/trt_convert.py @@ -63,19 +63,20 @@ class TrtPrecisionMode(object): return [TrtPrecisionMode.FP32, TrtPrecisionMode.FP16, TrtPrecisionMode.INT8] -def tensorrt_rewriter_config(rewriter_config=None, - max_batch_size=1, - max_workspace_size_bytes=2 << 20, - precision_mode=TrtPrecisionMode.FP32, - minimum_segment_size=3, - is_dynamic_op=False, - maximum_cached_engines=1, - cached_engine_batch_sizes=None): +def get_tensorrt_rewriter_config(rewriter_config=None, + max_batch_size=1, + max_workspace_size_bytes=2 << 20, + precision_mode=TrtPrecisionMode.FP32, + minimum_segment_size=3, + is_dynamic_op=False, + maximum_cached_engines=1, + cached_engine_batch_sizes=None, + use_calibration=True): """Returns a RewriterConfig proto for TRT transformation. Args: - rewriter_config: a RewriterConfig proto to append the TensorRTOptimizer to. - If None, it will create one with default settings. + rewriter_config: a template RewriterConfig proto used to create a + TRT-enabled RewriterConfig. If None, it will use a default one. max_batch_size: max size for the input batch max_workspace_size_bytes: the maximum GPU temporary memory which the TRT engine can use at execution time. This corresponds to the 'workspaceSize' @@ -95,6 +96,15 @@ def tensorrt_rewriter_config(rewriter_config=None, use this list to determine the batch sizes of the cached engines, instead of making the decision on the fly. This is useful when we know the most common batch size(s) the application is going to generate. + use_calibration: this argument is ignored if precision_mode is not INT8. if + set to True, a calibration graph will be created to calibrate the missing + ranges. The calibration graph must be converted to an inference graph + using calib_graph_to_infer_graph() after running calibration. if set to + False, quantization nodes will be expected for every tensor in the graph + (exlcuding those which will be fused). If a range is missing, an error + will occur. Please note that accuracy may be negatively affected if there + is a mismatch between which tensors TRT quantizes and which tensors were + trained with fake quantization. Returns: A RewriterConfig proto which sets a TensorRTOptimizer to run Grappler. @@ -107,13 +117,16 @@ def tensorrt_rewriter_config(rewriter_config=None, rewriter_config, rewriter_config_pb2.RewriterConfig): raise TypeError("rewriter_config should be a RewriterConfig proto.") + rewriter_config_with_trt = rewriter_config_pb2.RewriterConfig() if rewriter_config is None: - rewriter_config = rewriter_config_pb2.RewriterConfig() # Layout optimizer may add Const nodes followed by Reshape nodes, thus we # need to run constant folding again. - rewriter_config.optimizers.extend(["constfold", "layout", "constfold"]) - rewriter_config.meta_optimizer_iterations = ( + rewriter_config_with_trt.optimizers.extend( + ["constfold", "layout", "constfold"]) + rewriter_config_with_trt.meta_optimizer_iterations = ( rewriter_config_pb2.RewriterConfig.ONE) + else: + rewriter_config_with_trt.CopyFrom(rewriter_config) if precision_mode.upper() not in TrtPrecisionMode.supported_precision_modes(): raise ValueError(("precision mode '{}' is not supported." @@ -121,7 +134,7 @@ def tensorrt_rewriter_config(rewriter_config=None, precision_mode, TrtPrecisionMode.supported_precision_modes)) - optimizer = rewriter_config.custom_optimizers.add() + optimizer = rewriter_config_with_trt.custom_optimizers.add() optimizer.name = "TensorRTOptimizer" optimizer.parameter_map["minimum_segment_size"].i = minimum_segment_size optimizer.parameter_map["max_batch_size"].i = max_batch_size @@ -138,7 +151,8 @@ def tensorrt_rewriter_config(rewriter_config=None, "maximum_cached_engines items.") optimizer.parameter_map["cached_engine_batches"].list.i.extend( cached_engine_batch_sizes) - return rewriter_config + optimizer.parameter_map["use_calibration"].b = use_calibration + return rewriter_config_with_trt def create_inference_graph(input_graph_def, @@ -150,7 +164,7 @@ def create_inference_graph(input_graph_def, is_dynamic_op=False, maximum_cached_engines=1, cached_engine_batch_sizes=None, - rewriter_config=None, + use_calibration=True, input_saved_model_dir=None, input_saved_model_tags=None, output_saved_model_dir=None, @@ -182,8 +196,15 @@ def create_inference_graph(input_graph_def, use this list to determine the batch sizes of the cached engines, instead of making the decision on the fly. This is useful when we know the most common batch size(s) the application is going to generate. - rewriter_config: a RewriterConfig proto to append the TensorRTOptimizer to. - If None, it will create one with default settings. + use_calibration: this argument is ignored if precision_mode is not INT8. if + set to True, a calibration graph will be created to calibrate the missing + ranges. The calibration graph must be converted to an inference graph + using calib_graph_to_infer_graph() after running calibration. if set to + False, quantization nodes will be expected for every tensor in the graph + (exlcuding those which will be fused). If a range is missing, an error + will occur. Please note that accuracy may be negatively affected if there + is a mismatch between which tensors TRT quantizes and which tensors were + trained with fake quantization. input_saved_model_dir: the directory to load the SavedModel which contains the input graph to transforms. Used only when input_graph_def is None. input_saved_model_tags: list of tags to load the SavedModel. @@ -191,8 +212,9 @@ def create_inference_graph(input_graph_def, returned GraphDef and save it to the specified directory. This option only works when the input graph is loaded from a SavedModel, i.e. when input_saved_model_dir is specified and input_graph_def is None. - session_config: the ConfigProto used to create a Session. If not specified, - a default ConfigProto will be used. + session_config: the ConfigProto used to create a Session. It's also used as + a template to create a TRT-enabled ConfigProto for conversion. If not + specified, a default ConfigProto will be used. Returns: A GraphDef transformed from input_graph_def (or the SavedModel graph def @@ -322,21 +344,30 @@ def create_inference_graph(input_graph_def, grappler_meta_graph_def.collection_def["train_op"].CopyFrom( output_collection) - # Create RewriterConfig. - rewriter_config = tensorrt_rewriter_config( + # Create TRT-enabled ConfigProto. + session_config_with_trt = config_pb2.ConfigProto() + session_config_with_trt.CopyFrom(session_config) + rewriter_config = None + if (session_config_with_trt.HasField("graph_options") and + session_config_with_trt.graph_options.HasField("rewrite_options")): + rewriter_config = session_config_with_trt.graph_options.rewrite_options + rewriter_config_with_trt = get_tensorrt_rewriter_config( rewriter_config, max_batch_size, max_workspace_size_bytes, precision_mode, minimum_segment_size, is_dynamic_op, maximum_cached_engines, - cached_engine_batch_sizes) + cached_engine_batch_sizes, use_calibration) + session_config_with_trt.graph_options.rewrite_options.CopyFrom( + rewriter_config_with_trt) # Run Grappler. transformed_graph_def = tf_optimizer.OptimizeGraph( - rewriter_config, grappler_meta_graph_def, graph_id=b"tf_graph") + session_config_with_trt, grappler_meta_graph_def, graph_id=b"tf_graph") # Optionally write the transformed graphdef as SavedModel. if output_saved_model_dir is not None: saved_model_builder = builder.SavedModelBuilder(output_saved_model_dir) with ops.Graph().as_default(): importer.import_graph_def(transformed_graph_def, name="") + # We don't use TRT here. with session.Session(config=session_config) as sess: saved_model_builder.add_meta_graph_and_variables( sess, diff --git a/tensorflow/contrib/tensorrt/python/trt_convert_test.py b/tensorflow/contrib/tensorrt/python/trt_convert_test.py index 9f2eeac990d..dbf8dd26144 100644 --- a/tensorflow/contrib/tensorrt/python/trt_convert_test.py +++ b/tensorflow/contrib/tensorrt/python/trt_convert_test.py @@ -47,9 +47,9 @@ from tensorflow.python.tools import saved_model_utils class TrtConvertTest(test_util.TensorFlowTestCase): """Class to test Tensorflow-TensorRT integration python API.""" - def testTensorrtRewriterConfig(self): - """Test case for trt_convert.tensorrt_rewriter_config().""" - rewriter_cfg = trt_convert.tensorrt_rewriter_config( + def testGetTensorrtRewriterConfig(self): + """Test case for trt_convert.get_tensorrt_rewriter_config().""" + rewriter_cfg = trt_convert.get_tensorrt_rewriter_config( rewriter_config=None, max_batch_size=128, max_workspace_size_bytes=1234, @@ -162,7 +162,7 @@ class TrtConvertTest(test_util.TensorFlowTestCase): node_name_to_op = {node.name: node.op for node in graph_def.node} self.assertEqual({ "input": "Placeholder", - "my_trt_op_0": "TRTEngineOp", + "TRTEngineOp_0": "TRTEngineOp", "output": "Identity" }, node_name_to_op) @@ -189,10 +189,11 @@ class TrtConvertTest(test_util.TensorFlowTestCase): execute_engine_test_value = ("done" if expect_engine_is_run else "") execute_native_segment_test_value = ("" if expect_engine_is_run else "done") self.assertEqual(execute_engine_test_value, - trt_convert.get_test_value("my_trt_op_0:ExecuteTrtEngine")) + trt_convert.get_test_value( + "TRTEngineOp_0:ExecuteTrtEngine")) self.assertEqual( execute_native_segment_test_value, - trt_convert.get_test_value("my_trt_op_0:ExecuteNativeSegment")) + trt_convert.get_test_value("TRTEngineOp_0:ExecuteNativeSegment")) def testCreateInferenceGraph_MinimumSegmentSize(self): if not trt_convert.is_tensorrt_enabled(): diff --git a/tensorflow/contrib/tensorrt/resources/trt_resources.h b/tensorflow/contrib/tensorrt/resources/trt_resources.h index 840da6e78d8..aac9e5c7bd7 100644 --- a/tensorflow/contrib/tensorrt/resources/trt_resources.h +++ b/tensorflow/contrib/tensorrt/resources/trt_resources.h @@ -39,7 +39,8 @@ namespace tensorrt { class TRTCalibrationResource : public tensorflow::ResourceBase { public: ~TRTCalibrationResource() { - VLOG(0) << "Destroying Calibration Resource " << std::endl << DebugString(); + LOG(INFO) << "Destroying Calibration Resource " << std::endl + << DebugString(); builder_.reset(); engine_.reset(); // We need to manually destroy the builder and engine before the allocator diff --git a/tensorflow/contrib/tensorrt/segment/segment.cc b/tensorflow/contrib/tensorrt/segment/segment.cc index 4f64b7a9522..d8f63779e64 100644 --- a/tensorflow/contrib/tensorrt/segment/segment.cc +++ b/tensorflow/contrib/tensorrt/segment/segment.cc @@ -33,6 +33,7 @@ namespace tensorflow { namespace tensorrt { namespace segment { using ::tensorflow::strings::StrAppend; +using ::tensorflow::strings::StrCat; // A simple graph representation to mirror tensorflow::Graph. This structure // helps saving memory since segmenter modifies the graph in place, preventing @@ -406,22 +407,42 @@ tensorflow::Status SegmentGraph( // Use a union-find to collect the nodes that belong to the same // segment. A node value of nullptr indicates that the node is not a candidate // for TRT. + std::unordered_set unsupported_ops; + int num_unsupported_ops = 0; std::vector> node_segments; for (int i = 0; i < graph->num_node_ids(); ++i) { SimpleNode* node = graph->FindNodeId(i); if (options.exclude_node_list.count(node->name()) != 0) { - VLOG(1) << "Not a TF-TRT candidate: " << node->name() - << " (excluded by segmenter option)."; + VLOG(1) << "Not a TF-TRT candidate, " + << "(Op type: " << node->tf_node()->type_string() << "), " + << "(Op name: " << node->name() << "), " + << "(Reason: excluded by segmenter option)"; + unsupported_ops.emplace(node->tf_node()->type_string()); + num_unsupported_ops++; node = nullptr; } else { const Status status = candidate_fn(node->tf_node()); if (!status.ok()) { - VLOG(1) << "Not a TF-TRT candidate: " << node->name() << ": " << status; + VLOG(1) << "Not a TF-TRT candidate, " + << "(Op type: " << node->tf_node()->type_string() << "), " + << "(Op name: " << node->name() << "), " + << "(Reason: " << status << ")"; + unsupported_ops.emplace(node->tf_node()->type_string()); + num_unsupported_ops++; node = nullptr; } } node_segments.emplace_back(node); } + string msg = StrCat( + "There are ", num_unsupported_ops, " ops of ", unsupported_ops.size(), + " different types in the graph that", " are not converted to TensorRT: "); + for (const auto& elem : unsupported_ops) { + StrAppend(&msg, elem, ", "); + } + LOG(INFO) << msg << "(For more information see " + << "https://docs.nvidia.com/deeplearning" + << "/dgx/integrate-tf-trt/index.html#support-ops)."; // The segmentation algorithm below visits nodes in reverse topological order // and attempts to merge nodes along output edges. That means that subgraphs @@ -439,7 +460,8 @@ tensorflow::Status SegmentGraph( std::vector order; order.reserve(graph->num_node_ids()); StableDFS(*graph, /*reverse=*/false, {graph->source_node()}, - /*enter=*/nullptr, [&order](const SimpleNode* n) { + /*enter=*/nullptr, + [&order](const SimpleNode* n) { order.push_back(n); return true; }); @@ -548,7 +570,7 @@ tensorflow::Status SegmentGraph( std::set& segment_nodes = itr.second; VLOG(1) << "Segment original size: " << segment_nodes.size(); while (true) { - std::deque in_nodes_que, out_nodes_que; + std::deque in_nodes_que, out_nodes_que; // Find an input node that is not eligible and add it to the queue. // Nodes that has no incoming edges should not be treated as "input", // as there are really no inputs to them. Similar for output nodes. @@ -594,8 +616,7 @@ tensorflow::Status SegmentGraph( // their outputs. In this way, for common cases the number of removed // nodes should be minimum. auto remove_nodes = [&segment_nodes]( - bool is_input_nodes, - std::deque* que) { + bool is_input_nodes, std::deque* que) { // Run a BFS on the queue to find all the input/output nodes. std::set visited; std::set logged(que->begin(), que->end()); diff --git a/tensorflow/contrib/tensorrt/test/base_test.py b/tensorflow/contrib/tensorrt/test/base_test.py index 18096e0ff1e..03faf1df243 100644 --- a/tensorflow/contrib/tensorrt/test/base_test.py +++ b/tensorflow/contrib/tensorrt/test/base_test.py @@ -56,8 +56,9 @@ class SimpleSingleEngineTest(trt_test.TfTrtIntegrationTestBase): strides=[1, 2, 2, 1], padding="SAME", name="conv") - bias = constant_op.constant( - [4., 1.5, 2., 3., 5., 7.], name="bias", dtype=dtype) + bias = constant_op.constant([4., 1.5, 2., 3., 5., 7.], + name="bias", + dtype=dtype) added = nn.bias_add(conv, bias, name="bias_add") relu = nn.relu(added, "relu") identity = array_ops.identity(relu, "identity") @@ -73,11 +74,12 @@ class SimpleSingleEngineTest(trt_test.TfTrtIntegrationTestBase): def ExpectedEnginesToBuild(self, run_params): """Return the expected engines to build.""" - # TODO(aaroey): LayoutOptimizer adds additional nodes to the graph which - # breaks the connection check, fix it. - # - my_trt_op_0 should have ["weights", "conv", "bias", "bias_add", - # "relu", "identity", "max_pool"] - return ["my_trt_op_0"] + return { + "my_trt_op_0": [ + "weights", "conv", "bias", "bias_add", "relu", "identity", + "max_pool" + ] + } class SimpleMultiEnginesTest(trt_test.TfTrtIntegrationTestBase): @@ -92,7 +94,7 @@ class SimpleMultiEnginesTest(trt_test.TfTrtIntegrationTestBase): g = ops.Graph() with g.as_default(): inp = array_ops.placeholder( - dtype=dtype, shape=[None] + input_dims[1:], name=input_name) + dtype=dtype, shape=input_dims, name=input_name) with g.device("/GPU:0"): conv_filter = constant_op.constant( [[[[1., 0.5, 4., 6., 0.5, 1.], [1., 0.5, 1., 1., 0.5, 1.]]]], @@ -105,10 +107,10 @@ class SimpleMultiEnginesTest(trt_test.TfTrtIntegrationTestBase): padding="SAME", name="conv") c1 = constant_op.constant( - np.random.randn(input_dims[0], 12, 12, 6), dtype=dtype, name="c1") + np.random.randn(12, 12, 6), dtype=dtype, name="c1") p = math_ops.mul(conv, c1, name="mul") c2 = constant_op.constant( - np.random.randn(input_dims[0], 12, 12, 6), dtype=dtype, name="c2") + np.random.randn(12, 12, 6), dtype=dtype, name="c2") q = math_ops.div(conv, c2, name="div") edge = self.trt_incompatible_op(q, name="incompatible") @@ -129,22 +131,21 @@ class SimpleMultiEnginesTest(trt_test.TfTrtIntegrationTestBase): def ExpectedEnginesToBuild(self, run_params): """Return the expected engines to build.""" - # TODO(aaroey): LayoutOptimizer adds additional nodes to the graph which - # breaks the connection check, fix it. - # - my_trt_op_0 should have ["mul", "sub", "div1", "mul1", "add1", - # "add", "sub1"]; - # - my_trt_op_1 should have ["weights","conv", "div"] - return ["my_trt_op_0", "my_trt_op_1"] + return { + "my_trt_op_0": [ + "add", "add1", "c1", "div1", "mul", "mul1", "sub", "sub1" + ], + "my_trt_op_1": ["c2", "conv", "div", "weights"] + } - def ShouldRunTest(self, run_params): - # TODO(aaroey): LayoutOptimizer adds Transpose(Const, Const) to the graph - # which breaks the conversion. We should fix it as: - # - Detect the invalid NodeDef earlier before adding them to segment - # - Let it able to change the RewriterConfig when calling - # create_inference_graph(). - # It will be good to add debugging feature for Grappler to print the graph - # after running each optimizer. - return False + def GetConversionParams(self, run_params): + """Return a ConversionParams for test.""" + return super( + SimpleMultiEnginesTest, self + ).GetConversionParams(run_params)._replace( + # Disable layout optimizer, since it'll add Transpose(Const, Const) to + # the graph and breaks the conversion check. + rewriter_config=trt_test.OptimizerDisabledRewriterConfig()) class PartiallyConvertedTestA(trt_test.TfTrtIntegrationTestBase): @@ -153,7 +154,7 @@ class PartiallyConvertedTestA(trt_test.TfTrtIntegrationTestBase): """Setup method.""" super(PartiallyConvertedTestA, self).setUp() # Let it fail to build the second engine. - trt_convert.add_test_value("my_trt_op_1:CreateTRTNode", "fail") + trt_convert.add_test_value("TRTEngineOp_1:CreateTRTNode", "fail") def GetParams(self): """Create a graph containing two segment.""" @@ -190,14 +191,16 @@ class PartiallyConvertedTestA(trt_test.TfTrtIntegrationTestBase): """Return the expected engines to build.""" return { # Only the first engine is built. - "my_trt_op_0": ["c0", "c1", "add0", "add1", "mul0", "mul1"] + "TRTEngineOp_0": ["c0", "c1", "add0", "add1", "mul0", "mul1"] } def ShouldRunTest(self, run_params): """Whether to run the test.""" # Disable the test in fp16 mode since multiple matmul and add ops together # can cause overflow. - return run_params.precision_mode != "FP16" + return ((run_params.precision_mode != "FP16") and + not (trt_test.IsQuantizationMode(run_params.precision_mode) and + not run_params.use_calibration)) class PartiallyConvertedTestB(PartiallyConvertedTestA): @@ -207,13 +210,13 @@ class PartiallyConvertedTestB(PartiallyConvertedTestA): super(PartiallyConvertedTestB, self).setUp() # Let it fail to build the first engine. trt_convert.clear_test_values("") - trt_convert.add_test_value("my_trt_op_0:CreateTRTNode", "fail") + trt_convert.add_test_value("TRTEngineOp_0:CreateTRTNode", "fail") def ExpectedEnginesToBuild(self, run_params): """Return the expected engines to build.""" return { # Only the second engine is built. - "my_trt_op_1": ["c2", "c3", "add2", "add3", "mul2", "mul3"] + "TRTEngineOp_1": ["c2", "c3", "add2", "add3", "mul2", "mul3"] } @@ -257,8 +260,8 @@ class ConstInputTest(trt_test.TfTrtIntegrationTestBase): def ExpectedEnginesToBuild(self, run_params): """Return the expected engines to build.""" return { - "my_trt_op_0": ["add", "add1", "mul"], - "my_trt_op_1": ["add2", "add3", "mul1"] + "TRTEngineOp_0": ["add", "add1", "mul"], + "TRTEngineOp_1": ["add2", "add3", "mul1"] } @@ -289,7 +292,7 @@ class ConstDataInputSingleEngineTest(trt_test.TfTrtIntegrationTestBase): def ExpectedEnginesToBuild(self, run_params): """Return the expected engines to build.""" - return {"my_trt_op_0": ["c", "add", "add1", "mul"]} + return {"TRTEngineOp_0": ["c", "add", "add1", "mul"]} class ConstDataInputMultipleEnginesTest(trt_test.TfTrtIntegrationTestBase): @@ -324,12 +327,12 @@ class ConstDataInputMultipleEnginesTest(trt_test.TfTrtIntegrationTestBase): def ExpectedEnginesToBuild(self, run_params): """Return the expected engines to build.""" return { - "my_trt_op_0": ["add2", "add3", "mul1"], + "TRTEngineOp_0": ["add2", "add3", "mul1"], # Why segment ["add", "add1", "mul"] was assigned segment id 1 # instead of 0: the parent node of this segment is actually const # node 'c', but it's removed later since it's const output of the # segment which is not allowed. - "my_trt_op_1": ["add", "add1", "mul"] + "TRTEngineOp_1": ["add", "add1", "mul"] } @@ -373,8 +376,8 @@ class ControlDependencyTest(trt_test.TfTrtIntegrationTestBase): def ExpectedEnginesToBuild(self, run_params): """Return the expected engines to build.""" return { - "my_trt_op_0": ["c1", "add", "add1", "mul"], - "my_trt_op_1": ["c2", "add2", "add3", "mul1"] + "TRTEngineOp_0": ["c1", "add", "add1", "mul"], + "TRTEngineOp_1": ["c2", "add2", "add3", "mul1"] } diff --git a/tensorflow/contrib/tensorrt/test/batch_matmul_test.py b/tensorflow/contrib/tensorrt/test/batch_matmul_test.py index 4b888081787..f42308ecb7c 100644 --- a/tensorflow/contrib/tensorrt/test/batch_matmul_test.py +++ b/tensorflow/contrib/tensorrt/test/batch_matmul_test.py @@ -79,12 +79,12 @@ class BatchMatMulTest(trt_test.TfTrtIntegrationTestBase): """Return the expected engines to build.""" if (run_params.dynamic_engine and not trt_test.IsQuantizationMode(run_params.precision_mode)): - return ["my_trt_op_0", "my_trt_op_1"] - return ["my_trt_op_1"] + return ["TRTEngineOp_0", "TRTEngineOp_1"] + return ["TRTEngineOp_1"] def ExpectedEnginesToRun(self, run_params): """Return the expected engines to run.""" - return ["my_trt_op_1"] + return ["TRTEngineOp_1"] def ShouldRunTest(self, run_params): """Whether to run the test.""" diff --git a/tensorflow/contrib/tensorrt/test/biasadd_matmul_test.py b/tensorflow/contrib/tensorrt/test/biasadd_matmul_test.py index 7545bb9df20..053b38ff1c0 100644 --- a/tensorflow/contrib/tensorrt/test/biasadd_matmul_test.py +++ b/tensorflow/contrib/tensorrt/test/biasadd_matmul_test.py @@ -41,6 +41,7 @@ class BiasaddMatMulTest(trt_test.TfTrtIntegrationTestBase): input_name = "input" input_matrix_rows = 4 input_matrix_columns = 144 + # Note that tf.nn.bias_add supports up to 5 dimensions. input_dims = [input_matrix_rows, input_matrix_columns] output_name = "output" g = ops.Graph() @@ -74,18 +75,18 @@ class BiasaddMatMulTest(trt_test.TfTrtIntegrationTestBase): x5 = nn.bias_add(x5, b) x5 = gen_array_ops.reshape(x5, [4, -1]) - x6 = gen_array_ops.reshape(x, [4, 12, 12]) - b = self._ConstOp((12,)) + x6 = gen_array_ops.reshape(x, [4, 24, 6]) + b = self._ConstOp((6,)) x6 = nn.bias_add(x6, b, data_format="NHWC") x6 = gen_array_ops.reshape(x6, [4, -1]) - x7 = gen_array_ops.reshape(x, [4, 12, 3, 4]) - b = self._ConstOp((4,)) + x7 = gen_array_ops.reshape(x, [4, 12, 4, 3]) + b = self._ConstOp((3,)) x7 = nn.bias_add(x7, b, data_format="NHWC") x7 = gen_array_ops.reshape(x7, [4, -1]) - x8 = gen_array_ops.reshape(x, [4, 12, 3, 2, 2]) - b = self._ConstOp((2,)) + x8 = gen_array_ops.reshape(x, [4, 4, 3, 2, 6]) + b = self._ConstOp((6,)) x8 = nn.bias_add(x8, b, data_format="NHWC") x8 = gen_array_ops.reshape(x8, [4, -1]) @@ -94,13 +95,13 @@ class BiasaddMatMulTest(trt_test.TfTrtIntegrationTestBase): x9 = nn.bias_add(x9, b, data_format="NCHW") x9 = gen_array_ops.reshape(x9, [4, -1]) - x10 = gen_array_ops.reshape(x, [4, 12, 3, 4]) - b = self._ConstOp((12,)) + x10 = gen_array_ops.reshape(x, [4, 3, 4, 12]) + b = self._ConstOp((3,)) x10 = nn.bias_add(x10, b, data_format="NCHW") x10 = gen_array_ops.reshape(x10, [4, -1]) - x11 = gen_array_ops.reshape(x, [4, 12, 12]) - b = self._ConstOp((12,)) + x11 = gen_array_ops.reshape(x, [4, 6, 24]) + b = self._ConstOp((6,)) x11 = nn.bias_add(x11, b, data_format="NCHW") x11 = gen_array_ops.reshape(x11, [4, -1]) @@ -116,13 +117,18 @@ class BiasaddMatMulTest(trt_test.TfTrtIntegrationTestBase): def GetConversionParams(self, run_params): """Return a ConversionParams for test.""" - return super(BiasaddMatMulTest, - self).GetConversionParams(run_params)._replace( - max_batch_size=4, maximum_cached_engines=1) + conversion_params = super(BiasaddMatMulTest, + self).GetConversionParams(run_params) + return conversion_params._replace( + max_batch_size=4, + maximum_cached_engines=1, + # Disable layout optimizer, since it will convert BiasAdd with NHWC + # format to NCHW format under four dimentional input. + rewriter_config=trt_test.OptimizerDisabledRewriterConfig()) def ExpectedEnginesToBuild(self, run_params): """Return the expected engines to build.""" - return ["my_trt_op_0"] + return ["TRTEngineOp_0"] def ShouldRunTest(self, run_params): """Whether to run the test.""" diff --git a/tensorflow/contrib/tensorrt/test/binary_tensor_weight_broadcast_test.py b/tensorflow/contrib/tensorrt/test/binary_tensor_weight_broadcast_test.py index b53cb3c091e..169835956c0 100644 --- a/tensorflow/contrib/tensorrt/test/binary_tensor_weight_broadcast_test.py +++ b/tensorflow/contrib/tensorrt/test/binary_tensor_weight_broadcast_test.py @@ -26,7 +26,6 @@ from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops from tensorflow.python.ops import array_ops from tensorflow.python.ops import gen_array_ops -from tensorflow.python.ops import math_ops from tensorflow.python.platform import test @@ -56,10 +55,10 @@ class BinaryTensorWeightBroadcastTest(trt_test.TfTrtIntegrationTestBase): ]: a = self._ConstOp(weights_shape) f = x + a - x = math_ops.sigmoid(f) + x = self.trt_incompatible_op(f) a = self._ConstOp(weights_shape) f = a + x - x = math_ops.sigmoid(f) + x = self.trt_incompatible_op(f) gen_array_ops.reshape(x, [5, -1], name=output_name) return trt_test.TfTrtIntegrationTestParams( gdef=g.as_graph_def(), @@ -70,7 +69,7 @@ class BinaryTensorWeightBroadcastTest(trt_test.TfTrtIntegrationTestBase): def ExpectedEnginesToBuild(self, run_params): """Return the expected engines to build.""" - return ["my_trt_op_%d" % i for i in range(16)] + return ["TRTEngineOp_%d" % i for i in range(16)] if __name__ == "__main__": diff --git a/tensorflow/contrib/tensorrt/test/concatenation_test.py b/tensorflow/contrib/tensorrt/test/concatenation_test.py index 465cb022964..c3576f81d97 100644 --- a/tensorflow/contrib/tensorrt/test/concatenation_test.py +++ b/tensorflow/contrib/tensorrt/test/concatenation_test.py @@ -79,7 +79,7 @@ class ConcatenationTest(trt_test.TfTrtIntegrationTestBase): def ExpectedEnginesToBuild(self, run_params): """Return the expected engines to build.""" - return ["my_trt_op_0"] + return ["TRTEngineOp_0"] if __name__ == "__main__": diff --git a/tensorflow/contrib/tensorrt/test/const_broadcast_test.py b/tensorflow/contrib/tensorrt/test/const_broadcast_test.py index e32f0478661..c1c883312d8 100644 --- a/tensorflow/contrib/tensorrt/test/const_broadcast_test.py +++ b/tensorflow/contrib/tensorrt/test/const_broadcast_test.py @@ -64,7 +64,7 @@ class ConstBroadcastTest(trt_test.TfTrtIntegrationTestBase): def ExpectedEnginesToBuild(self, run_params): """Return the expected engines to build.""" - return ['my_trt_op_0'] + return ['TRTEngineOp_0'] def ExpectedAbsoluteTolerance(self, run_params): """The absolute tolerance to compare floating point results.""" diff --git a/tensorflow/contrib/tensorrt/test/memory_alignment_test.py b/tensorflow/contrib/tensorrt/test/memory_alignment_test.py index bc7c90081ff..104bac43a0b 100644 --- a/tensorflow/contrib/tensorrt/test/memory_alignment_test.py +++ b/tensorflow/contrib/tensorrt/test/memory_alignment_test.py @@ -68,7 +68,7 @@ class MemoryAlignmentTest(trt_test.TfTrtIntegrationTestBase): def ExpectedEnginesToBuild(self, run_params): """Return the expected engines to build.""" - return ["my_trt_op_0"] + return ["TRTEngineOp_0"] def ExpectedAbsoluteTolerance(self, run_params): """The absolute tolerance to compare floating point results.""" diff --git a/tensorflow/contrib/tensorrt/test/multi_connection_neighbor_engine_test.py b/tensorflow/contrib/tensorrt/test/multi_connection_neighbor_engine_test.py index 11be4feaf7b..293f93d8a78 100644 --- a/tensorflow/contrib/tensorrt/test/multi_connection_neighbor_engine_test.py +++ b/tensorflow/contrib/tensorrt/test/multi_connection_neighbor_engine_test.py @@ -25,8 +25,6 @@ from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops from tensorflow.python.ops import array_ops -from tensorflow.python.ops import gen_math_ops -from tensorflow.python.ops import math_ops from tensorflow.python.ops import nn from tensorflow.python.platform import test @@ -60,14 +58,14 @@ class MultiConnectionNeighborEngineTest(trt_test.TfTrtIntegrationTestBase): b = constant_op.constant( np.random.normal(5.0, 1.0, [1, 4, 1, 1]), name="bias", dtype=dtype) q = conv - b - edge = math_ops.sigmoid(q) + edge = self.trt_incompatible_op(q) b = constant_op.constant( np.random.normal(5.0, 1.0, [1, 4, 1, 1]), name="bias", dtype=dtype) d = b + conv - edge3 = math_ops.sigmoid(d) + edge3 = self.trt_incompatible_op(d) - edge1 = gen_math_ops.tan(conv) + edge1 = self.trt_incompatible_op(conv) t = t - edge1 q = q + edge t = t + q @@ -83,7 +81,7 @@ class MultiConnectionNeighborEngineTest(trt_test.TfTrtIntegrationTestBase): def ExpectedEnginesToBuild(self, run_params): """Return the expected engines to build.""" - return ["my_trt_op_0", "my_trt_op_1"] + return ["TRTEngineOp_0", "TRTEngineOp_1"] if __name__ == "__main__": diff --git a/tensorflow/contrib/tensorrt/test/neighboring_engine_test.py b/tensorflow/contrib/tensorrt/test/neighboring_engine_test.py index eddeafa38bc..3e1e4b088ba 100644 --- a/tensorflow/contrib/tensorrt/test/neighboring_engine_test.py +++ b/tensorflow/contrib/tensorrt/test/neighboring_engine_test.py @@ -66,8 +66,8 @@ class NeighboringEngineTest(trt_test.TfTrtIntegrationTestBase): def ExpectedEnginesToBuild(self, run_params): """Return the expected engines to build.""" return { - "my_trt_op_0": ["bias", "mul", "sub"], - "my_trt_op_1": ["weights", "conv"] + "TRTEngineOp_0": ["bias", "mul", "sub"], + "TRTEngineOp_1": ["weights", "conv"] } diff --git a/tensorflow/contrib/tensorrt/test/quantization_mnist_test.py b/tensorflow/contrib/tensorrt/test/quantization_mnist_test.py new file mode 100644 index 00000000000..31cbef89e23 --- /dev/null +++ b/tensorflow/contrib/tensorrt/test/quantization_mnist_test.py @@ -0,0 +1,290 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Script to test TF-TRT INT8 conversion without calibration on Mnist model.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from tensorflow.contrib.tensorrt.python import trt_convert +# pylint: disable=unused-import +from tensorflow.contrib.tensorrt.python.ops import trt_engine_op +# pylint: enable=unused-import +from tensorflow.core.protobuf import config_pb2 +from tensorflow.python import data +from tensorflow.python import keras +from tensorflow.python.estimator.estimator import Estimator +from tensorflow.python.estimator.model_fn import EstimatorSpec +from tensorflow.python.estimator.model_fn import ModeKeys +from tensorflow.python.estimator.run_config import RunConfig +from tensorflow.python.framework import dtypes +from tensorflow.python.framework import graph_util +from tensorflow.python.framework import importer +from tensorflow.python.framework import ops +from tensorflow.python.framework import test_util +from tensorflow.python.keras.datasets import mnist +from tensorflow.python.layers import layers +from tensorflow.python.ops import array_ops +from tensorflow.python.ops import gen_array_ops +from tensorflow.python.ops import math_ops +from tensorflow.python.ops import metrics +from tensorflow.python.ops import nn +from tensorflow.python.ops import variable_scope +from tensorflow.python.ops.losses import losses +from tensorflow.python.platform import test +from tensorflow.python.platform import tf_logging as logging +from tensorflow.python.summary import summary +from tensorflow.python.training import saver +from tensorflow.python.training.adam import AdamOptimizer +from tensorflow.python.training.checkpoint_management import latest_checkpoint +from tensorflow.python.training.training_util import get_global_step + +INPUT_NODE_NAME = 'input' +OUTPUT_NODE_NAME = 'output' + + +class QuantizationAwareTrainingMNISTTest(test_util.TensorFlowTestCase): + + def _BuildGraph(self, x): + + def _Quantize(x, r): + x = gen_array_ops.quantize_and_dequantize_v2(x, -r, r) + return x + + def _DenseLayer(x, num_inputs, num_outputs, quantization_range, name): + """Dense layer with quantized outputs. + + Args: + x: input to the dense layer + num_inputs: number of input columns of x + num_outputs: number of output columns + quantization_range: the min/max range for quantization + name: name of the variable scope + + Returns: + The output of the layer. + """ + with variable_scope.variable_scope(name): + kernel = variable_scope.get_variable( + 'kernel', + shape=[num_inputs, num_outputs], + dtype=dtypes.float32, + initializer=keras.initializers.glorot_uniform()) + bias = variable_scope.get_variable( + 'bias', + shape=[num_outputs], + dtype=dtypes.float32, + initializer=keras.initializers.zeros()) + x = math_ops.matmul(x, kernel) + x = _Quantize(x, quantization_range) + x = nn.bias_add(x, bias) + x = _Quantize(x, quantization_range) + return x + + x = _Quantize(x, 1) + # Conv + Bias + Relu6 + x = layers.conv2d(x, filters=32, kernel_size=3, use_bias=True) + x = nn.relu6(x) + # Conv + Bias + Relu6 + x = layers.conv2d(x, filters=64, kernel_size=3, use_bias=True) + x = nn.relu6(x) + # Reduce + x = math_ops.reduce_mean(x, [1, 2]) + x = _Quantize(x, 6) + # FC1 + x = _DenseLayer(x, 64, 512, 6, name='dense') + x = nn.relu6(x) + # FC2 + x = _DenseLayer(x, 512, 10, 25, name='dense_1') + x = array_ops.identity(x, name=OUTPUT_NODE_NAME) + return x + + def _GetGraphDef(self, use_trt, max_batch_size, model_dir): + """Get the frozen mnist GraphDef. + + Args: + use_trt: whether use TF-TRT to convert the graph. + max_batch_size: the max batch size to apply during TF-TRT conversion. + model_dir: the model directory to load the checkpoints. + + Returns: + The frozen mnist GraphDef. + """ + graph = ops.Graph() + with self.session(graph=graph) as sess: + with graph.device('/GPU:0'): + x = array_ops.placeholder( + shape=(None, 28, 28, 1), dtype=dtypes.float32, name=INPUT_NODE_NAME) + self._BuildGraph(x) + # Load weights + mnist_saver = saver.Saver() + checkpoint_file = latest_checkpoint(model_dir) + mnist_saver.restore(sess, checkpoint_file) + # Freeze + graph_def = graph_util.convert_variables_to_constants( + sess, sess.graph_def, output_node_names=[OUTPUT_NODE_NAME]) + # Convert with TF-TRT + if use_trt: + logging.info('Number of nodes before TF-TRT conversion: %d', + len(graph_def.node)) + graph_def = trt_convert.create_inference_graph( + graph_def, + outputs=[OUTPUT_NODE_NAME], + max_batch_size=max_batch_size, + precision_mode='INT8', + max_workspace_size_bytes=4096 << 19, + minimum_segment_size=2, + use_calibration=False, + ) + logging.info('Number of nodes after TF-TRT conversion: %d', + len(graph_def.node)) + num_engines = len( + [1 for n in graph_def.node if str(n.op) == 'TRTEngineOp']) + self.assertEqual(1, num_engines) + return graph_def + + def _Run(self, is_training, use_trt, batch_size, num_epochs, model_dir): + """Train or evaluate the model. + + Args: + is_training: whether to train or evaluate the model. In training mode, + quantization will be simulated where the quantize_and_dequantize_v2 are + placed. + use_trt: if true, use TRT INT8 mode for evaluation, which will perform + real quantization. Otherwise use native TensorFlow which will perform + simulated quantization. Ignored if is_training is True. + batch_size: batch size. + num_epochs: how many epochs to train. Ignored if is_training is False. + model_dir: where to save or load checkpoint. + + Returns: + The Estimator evaluation result. + """ + # Get dataset + train_data, test_data = mnist.load_data() + + def _PreprocessFn(x, y): + x = math_ops.cast(x, dtypes.float32) + x = array_ops.expand_dims(x, axis=2) + x = 2.0 * (x / 255.0) - 1.0 + y = math_ops.cast(y, dtypes.int32) + return x, y + + def _EvalInputFn(): + mnist_x, mnist_y = test_data + dataset = data.Dataset.from_tensor_slices((mnist_x, mnist_y)) + dataset = dataset.apply( + data.experimental.map_and_batch( + map_func=_PreprocessFn, + batch_size=batch_size, + num_parallel_calls=8)) + dataset = dataset.repeat(count=1) + iterator = data.make_one_shot_iterator(dataset) + features, labels = iterator.get_next() + return features, labels + + def _TrainInputFn(): + mnist_x, mnist_y = train_data + dataset = data.Dataset.from_tensor_slices((mnist_x, mnist_y)) + dataset = dataset.shuffle(2 * len(mnist_x)) + dataset = dataset.apply( + data.experimental.map_and_batch( + map_func=_PreprocessFn, + batch_size=batch_size, + num_parallel_calls=8)) + dataset = dataset.repeat(count=num_epochs) + iterator = data.make_one_shot_iterator(dataset) + features, labels = iterator.get_next() + return features, labels + + def _ModelFn(features, labels, mode): + if is_training: + logits_out = self._BuildGraph(features) + else: + graph_def = self._GetGraphDef(use_trt, batch_size, model_dir) + logits_out = importer.import_graph_def( + graph_def, + input_map={INPUT_NODE_NAME: features}, + return_elements=[OUTPUT_NODE_NAME + ':0'], + name='')[0] + + loss = losses.sparse_softmax_cross_entropy( + labels=labels, logits=logits_out) + summary.scalar('loss', loss) + + classes_out = math_ops.argmax(logits_out, axis=1, name='classes_out') + accuracy = metrics.accuracy( + labels=labels, predictions=classes_out, name='acc_op') + summary.scalar('accuracy', accuracy[1]) + + if mode == ModeKeys.EVAL: + return EstimatorSpec( + mode, loss=loss, eval_metric_ops={'accuracy': accuracy}) + elif mode == ModeKeys.TRAIN: + optimizer = AdamOptimizer(learning_rate=1e-2) + train_op = optimizer.minimize(loss, global_step=get_global_step()) + return EstimatorSpec(mode, loss=loss, train_op=train_op) + + config_proto = config_pb2.ConfigProto() + config_proto.gpu_options.allow_growth = True + estimator = Estimator( + model_fn=_ModelFn, + model_dir=model_dir if is_training else None, + config=RunConfig(session_config=config_proto)) + + if is_training: + estimator.train(_TrainInputFn) + results = estimator.evaluate(_EvalInputFn) + logging.info('accuracy: %s', str(results['accuracy'])) + return results + + # To generate the checkpoint, set a different model_dir and call self._Run() + # by setting is_training=True and num_epochs=1000, e.g.: + # model_dir = '/tmp/quantization_mnist' + # self._Run( + # is_training=True, + # use_trt=False, + # batch_size=128, + # num_epochs=100, + # model_dir=model_dir) + def testEval(self): + if not trt_convert.is_tensorrt_enabled(): + return + model_dir = test.test_src_dir_path('contrib/tensorrt/test/testdata') + + accuracy_tf_native = self._Run( + is_training=False, + use_trt=False, + batch_size=128, + num_epochs=None, + model_dir=model_dir)['accuracy'] + logging.info('accuracy_tf_native: %f', accuracy_tf_native) + self.assertAllClose(accuracy_tf_native, 0.9662) + + if trt_convert.get_linked_tensorrt_version()[0] < 5: + return + + accuracy_tf_trt = self._Run( + is_training=False, + use_trt=True, + batch_size=128, + num_epochs=None, + model_dir=model_dir)['accuracy'] + logging.info('accuracy_tf_trt: %f', accuracy_tf_trt) + self.assertAllClose(accuracy_tf_trt, 0.9677) + + +if __name__ == '__main__': + test.main() diff --git a/tensorflow/contrib/tensorrt/test/quantization_test.py b/tensorflow/contrib/tensorrt/test/quantization_test.py new file mode 100644 index 00000000000..28353273ede --- /dev/null +++ b/tensorflow/contrib/tensorrt/test/quantization_test.py @@ -0,0 +1,144 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Model script to test TF-TensorRT integration.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import numpy as np + +from tensorflow.contrib.tensorrt.python import trt_convert +from tensorflow.contrib.tensorrt.test import tf_trt_integration_test_base as trt_test +from tensorflow.python.framework import constant_op +from tensorflow.python.framework import dtypes +from tensorflow.python.framework import ops +from tensorflow.python.ops import array_ops +from tensorflow.python.ops import gen_array_ops +from tensorflow.python.ops import math_ops +from tensorflow.python.platform import test + + +def _GetParams(add_quantization_nodes, dtype=dtypes.float32): + input_name = "input" + input_dims = [8, 8] + output_name = "output" + + def _Quantize(x, r): + if add_quantization_nodes: + x = gen_array_ops.fake_quant_with_min_max_vars(x, -r, r) + return x + + g = ops.Graph() + with g.as_default(): + x = array_ops.placeholder( + dtype=dtype, shape=[None] + input_dims[1:], name=input_name) + x = _Quantize(x, 10.0) + x = x + 5 + x = _Quantize(x, 15.0) + x = x - 5 + x = _Quantize(x, 10.0) + x = x * 0.1 + x = _Quantize(x, 1.0) + w = constant_op.constant(np.ones((8, 1)), dtype=dtypes.float32) + x = math_ops.matmul(x, w) + x = _Quantize(x, 10.0) + x = array_ops.identity(x, name=output_name) + + return trt_test.TfTrtIntegrationTestParams( + gdef=g.as_graph_def(), + input_names=[input_name], + input_dims=[input_dims], + output_names=[output_name], + expected_output_dims=[(8, 1)]) + + +class QuantizationMissingAllRangesTest(trt_test.TfTrtIntegrationTestBase): + + def GetParams(self): + """Create a graph containing single segment with no quantization ranges.""" + return _GetParams(add_quantization_nodes=False) + + def ShouldRunTest(self, run_params): + if trt_convert.get_linked_tensorrt_version()[0] < 5: + return False + # Only test static engine mode, with or without calibration. + return (trt_test.IsQuantizationMode(run_params.precision_mode) and + not run_params.use_optimizer and not run_params.dynamic_engine) + + def ExpectedEnginesToBuild(self, run_params): + """Return the expected engines to build.""" + if run_params.use_calibration: + # In static engine mode with calibration, it should build a calibration + # engine. + return ["my_trt_op_0"] + # In static engine mode without calibration, the engine building will fail + # since no quantization ranges are set, which results in no TRT nodes. + return [] + + +class QuantizationWithRangesTest(trt_test.TfTrtIntegrationTestBase): + + def GetParams(self): + """Create a graph containing single segment with no quantization ranges.""" + return _GetParams(add_quantization_nodes=True) + + def ShouldRunTest(self, run_params): + if trt_convert.get_linked_tensorrt_version()[0] < 5: + return False + # Test static/dynamic engine with/without calibration. + return (trt_test.IsQuantizationMode(run_params.precision_mode) and + not run_params.use_optimizer) + + def ExpectedEnginesToBuild(self, run_params): + """Return the expected engines to build.""" + return ["my_trt_op_0"] + + def ExpectedAbsoluteTolerance(self, run_params): + """The absolute tolerance to compare floating point results.""" + return 1.e-05 if run_params.precision_mode == "FP32" else 1.e-01 + + def ExpectedRelativeTolerance(self, run_params): + """The relative tolerance to compare floating point results.""" + return 1.e-05 if run_params.precision_mode == "FP32" else 1.e-01 + + +class NonQuantizedPrecisionsWithRangesTest(trt_test.TfTrtIntegrationTestBase): + + def GetParams(self): + """Create a graph containing single segment with no quantization ranges.""" + return _GetParams(add_quantization_nodes=True) + + def ShouldRunTest(self, run_params): + # Only test FP32/FP16 mode. + return not trt_test.IsQuantizationMode(run_params.precision_mode) + + def ExpectedEnginesToBuild(self, run_params): + """Return the expected engines to build.""" + # The fake quant ops are not supported in FP32/FP16 mode, and will split the + # graph into three TRT segments. + return ["my_trt_op_0", "my_trt_op_1", "my_trt_op_2", "my_trt_op_3"] + + def ExpectedAbsoluteTolerance(self, run_params): + """The absolute tolerance to compare floating point results.""" + return 1.e-05 if run_params.precision_mode == "FP32" else 1.e-01 + + def ExpectedRelativeTolerance(self, run_params): + """The relative tolerance to compare floating point results.""" + return 1.e-05 if run_params.precision_mode == "FP32" else 1.e-01 + + +if __name__ == "__main__": + test.main() diff --git a/tensorflow/contrib/tensorrt/test/rank_two_test.py b/tensorflow/contrib/tensorrt/test/rank_two_test.py index 74a4a059257..0cd733dca13 100644 --- a/tensorflow/contrib/tensorrt/test/rank_two_test.py +++ b/tensorflow/contrib/tensorrt/test/rank_two_test.py @@ -68,11 +68,11 @@ class RankTwoTest(trt_test.TfTrtIntegrationTestBase): def ExpectedEnginesToBuild(self, run_params): """Return the expected engines to build.""" return { - "my_trt_op_0": [ + "TRTEngineOp_0": [ "add0_1", "add0_2", "add0_3", "c0_1", "c0_2", "c0_3", "abs0_1", "abs0_2" ], - "my_trt_op_1": [ + "TRTEngineOp_1": [ "add", "add1_1", "add1_2", "add1_3", "c1_1", "c1_2", "c1_3", "abs1_1", "abs1_2", "reciprocal0", "reciprocal1" ], diff --git a/tensorflow/contrib/tensorrt/test/reshape_transpose_test.py b/tensorflow/contrib/tensorrt/test/reshape_transpose_test.py index bbc724ab18e..207944468ab 100644 --- a/tensorflow/contrib/tensorrt/test/reshape_transpose_test.py +++ b/tensorflow/contrib/tensorrt/test/reshape_transpose_test.py @@ -79,8 +79,8 @@ class ReshapeTest(trt_test.TfTrtIntegrationTestBase): def ExpectedEnginesToBuild(self, run_params): """Return the expected engines to build.""" return { - "my_trt_op_0": ["reshape-%d" % i for i in range(7)] + - ["reshape-%d/shape" % i for i in range(7)] + "TRTEngineOp_0": ["reshape-%d" % i for i in range(7)] + + ["reshape-%d/shape" % i for i in range(7)] } def ShouldRunTest(self, run_params): @@ -117,7 +117,7 @@ class TransposeTest(trt_test.TfTrtIntegrationTestBase): # Note: by default Grappler will run the TRT optimizer twice. At the # first time it will group the two transpose ops below to same segment # then fail the conversion due to the expected batch dimension problem. - # At the second time, since the input of bridge op is my_trt_op_0, it + # At the second time, since the input of bridge op is TRTEngineOp_0, it # will fail to do shape inference which then cause conversion to fail. # TODO(laigd): support shape inference, make TRT optimizer run only # once, and fix this. @@ -136,7 +136,7 @@ class TransposeTest(trt_test.TfTrtIntegrationTestBase): def ExpectedEnginesToBuild(self, run_params): """Return the expected engines to build.""" return { - "my_trt_op_0": [ + "TRTEngineOp_0": [ "transpose-1", "transpose-1/perm", "transposeback", "transposeback/perm" ] diff --git a/tensorflow/contrib/tensorrt/test/testdata/checkpoint b/tensorflow/contrib/tensorrt/test/testdata/checkpoint new file mode 100644 index 00000000000..a603e1aec91 --- /dev/null +++ b/tensorflow/contrib/tensorrt/test/testdata/checkpoint @@ -0,0 +1,3 @@ +model_checkpoint_path: "model.ckpt-46900" +all_model_checkpoint_paths: "model.ckpt-0" +all_model_checkpoint_paths: "model.ckpt-46900" diff --git a/tensorflow/contrib/tensorrt/test/testdata/model.ckpt-46900.data-00000-of-00001 b/tensorflow/contrib/tensorrt/test/testdata/model.ckpt-46900.data-00000-of-00001 new file mode 100644 index 0000000000000000000000000000000000000000..88a998f184b275121e1e76eb51d2310da149f10a GIT binary patch literal 686728 zcmZ^Kc{~=;-?n|Xj9RmRSG3St9?y&5urjzmJsfHzGns{ zTb3xLMJq*?h?MB{dp_^`{P8@W_xv++&bj8C&zU)&bFOn;-w_fL652F)Q%h&qpE+aH zLhE<9GomF79Wa_nPw~Ids}2P-C)WEh=l}N7@0S=bZo5w~=B3iiUK3Tu+vfuve0?!9 z7VpH&N*8B(Urx}T(k--~lLmcd(wGr?Y|q#~GG#tE%wqQ8qDEB+VBdTs2*&fnh03b^x~UwTeKpxRh1 zsFKcNc~%v(9~y93d;PhB?Cs`kvPqq<_3Syn+f18vY~4)u9-#>AwVn!YaGA#02RGrR zR}Ju=L*ls5gFr0)_GJo!&Cn5z*P@^$KGZc;CF_-@PqeU@v*pA+-=GfO#6K5roERLGU>aokw>i0 z(|7SxB;{y93#(q|#TES}m2XUFp3sFnN8L#yhwV#sppbNnm5_Q&_`8s+{#%y$)SS)9>h%QD|4{b_cCE={6Ja*xtA6jr9r8;xs6;}VKIsgw8U z8;SZnuZ*8uyFIdu!e=L*+S#WZ_hV&dzH`vmJS&)u#u(;wV0MA=Jaj46>)nu@$&B*(v7|LbW(2w_gLpgwDeyleLnL$ zckQB?bp5GYv~aE_@8GpUdfRwEX?}hy9k;awo!x8qXh%P9j&P&o9-)8R{mFdqLNey(QSM{Y1adO(EA4#bEB9e+HSMzK2wjt^ zNITxiM90MD(@Cpx$Wp_n^bWNzsDGm_k7#eEhu=x?2EV=`w}{?D*$?M1`cqc4{=WNk z_ zU2S2^E6wbpom1}6u$`vI%HPvJhBxw#q|5R0M3&ROf0pn{ec{E^NoFU zk^gymf#YA4yheh|TJ{|#N5(jl--2jV<3LB#2+F29TS?dVgEme3X2IZK#%GLOyt6>Z^~?pK4G7CJnMW45Hln?UZ0soA7Q+_z2%?7Z#+ z0_P(?SssT&|#Cp4I8(Z*$ z9BTq!Ea+UOQsG+Vh(G1BShi1=3Hp3aumo2=^M^BO>=K)g_x8l`r`o&tE~bC68_^W| zz}8OO{C-u%tzv8b4kby!A!`lxMT4k{7tYCSw^NlY`lBZMszD&@{8nB5M@|ARB2>W7 z2+Cqr_k^=uYxLNAcTMv9d!^aG3|jaLcHU<1_`+q&&3epI4;$rw=4@jtSd3S^On=JS ze_cl4w#A=ix7vy2er^YgTX~yRb*i0BwI%XtCvE)Ma3AYy@LXS35! z$MD6DTCu->k!GtcN#zgyloI?@{=k>LB2v!lcV|B|iD!K=$>hr}4lkz`{o)IkMDRat z3u4Lns`4+AQ&W;tkTvx|1f2m&m&vgIoGu{Vw-m>-oT1!on71Q!+yDvW3E zhgT#2Zd3XWmT zPO0U`q$yOaxMqS6XLhr9&o^K_E^*|a`Y@y7fyf{~_)-lkCYxO2rMUpr3FIgO6x!-AKHEzq~GeR}|{`qqS_cMO6ek?i6z84aQ zkE(4guXo+X()?s1kXH$3J^K8bb%pOB@Uz;_UVl28-ELjWda(R8u5&wqfB9V~+c46D ze=AF!-TSnZuRNZ^jyzby-~RnK>)`timb6zLu0QC@I>}yx8+jYDJKc2z^GteJhAqq4 z3aRO=XFCTfm^d{-+yZ~T{Kmcbxk4RQ?bLSmI70kqrK*A)*Q0_}i!@kkx^D_33|6z8 zf;8ESxEZVfHEn*XU^PobQRz`N!y#_w-Gft}hdild~5SX|;G?2B#*mabijeR`~lFMBo{cPsYC-$_K`^R_U! zYKt^}TI4e(-sO++SaF!Ms0JP`n81qkmf`iuUD%VnP1tZ>0VeGrgwNS%j2W#c!`}Aa z#N`KP;fudK#WIsavDiCqn9Yd}%zuLlF1B?J?&m#e?oah$<~EU7mGVst_j`t|{@sC< zd(Oo5z8c_UgcJV#%Q}3->Kg8FM+$G->4aS~UyISIIoN@%*_cs8J67^;5{pxl#Q&x) z!hSQ=n0WXgw&#rvcDTL`+oYI+>nv8q_s5Fi3#H4kAF+m*{ZV;b&AADy$x*~F)?dV& zGG1WEwI5(Vt*>EUuXJLke@fu3YgO?5E%Wdbx`WttTX9_P*D+jls|a9kW|-cB9*nbO82e8u#ulh_|SD>yr5$cn>~3I z>$La4_WTjYuW%G`5u0sz;2|@tuxJLpIrawTFIS8ei6IQBgkeHU)bXfp6+Fhj0+SHG zjW^G2$M>c%*wt=L{O%_ncG2__R@b18`{X{wY;4Cd!|~tP5?fmgOHaa9=}?%>6+MjN zJiyYw72u)~Jy=NaGwj3SaO{Y|BCIF28Ee**$Ln?&;v@F6FpDcgm~LD!#(9&1&Gg}8 zIhNb7E?IfJQr{5I3$VqFPb|jsj%~wTyyG!_GkvUOHjEXDjcZN_iA zh~t~}XJDPZN!XR7`&d}|5e#3@fvtV^0N-{)7#Eq_fE6#u#%#ZRH2*D|*F^EZQIvt6M9M2dvG)3d&~T>-a<1<|+xipqj+XI}P!NGYhej;BVNm z_!8{Kjx1~+D( zvoegvwqRQ^l@)UM_0eb8(#QJvuV*}LgOCuuVpSMs%ZkT@1S+_t^duI{nvHwhkjB;f z=irkILh#e?WAM0*FR|T}I_~9h5o34U#%jAv@wfl+-7_i)^q&=cK2fz zRw}g-TXZQG%P+fw%L@O(3K-2+YnVACo*CkM-N0 zz+P_zEcbN*X0mu0&KVfRN>9q+w-(Cd%5(o=tEGkU+>Q4zpRaxx!tY@x`gQQ@JqvNI zSTlU}d_`QKvk0T>BC%Dbwb-g}j@a5;+y$1@UJh`uuFbL7_BdZFRk%_?eV$rsQv~tt;|RC zYX>;xtH|Uf$S3^B`3!h&V?&ReJ_lU^@pNrBRmwPP@U!$rtdNZ`DklMc33C9q&~0*de0LZ8@n(3*#EcB|Jl}m@(#wA{~MoG`1Zf< zf0jD_cYFWAq`gl3Z_I!4|1VVz;_~~s8tS2KKJxGFg4I&?iO%}H^R{`%!&~NGgv4e% z)3v~CD{5vu$8PC7)&8J7Uq~<`L*wEMN@Tubzb9dCurN`vzb8>s>z(T&r-j~1t77bD zq-cAX2VSaCPBi|$QpUeo?VoaSJQZiP533v z$j=ioU$buY%%*Bnh3%U6%-7tmmY-vNKzVpUgIB#-G|FgP zcLbNG4^dFv1(+DCC3@T=1)E;xQ-7E30;iKjbZG7%DVMg6(HPLCXQ^L>YN2aP`<-%P zm1HhlYw;rGk0AD~lZ#M|ZXwzqyn|!af{3#ommw)GUr?&lNo*TG1J$Q{!Nbafnk^!r zyDfeT4z?)^5-g(VJ=c6FHC6M<){i2nv*bI-H0cU1Bw5g_=hYF5)&vUTkq~7PaT_+w zQiru3!vL`#2w9_jl;WXE1j@3UnHLUlANCp1`$n!%%?m;q!{bpz4m$x^eVBtHUdDk@ zYd8#^lSBLGxgp|_FHCjr;v}y3MR~_W(CHyN&@iQ#Lk2F;OH`87nRa5;LupRU923qh z(wzDyBh5ncY)10>X;Pk7$go`AAn~jW;70$UCHKaGM8q&$I5tH)sXv9;s;X#vHK3lW z8(@1#H8rN$459w}C_k|vig)8Q{Iy<$xI1@HOP+SpLEQ*Sm3qX#15)a3kNC_DNDJ>&+$ zp2QHk{%#q~jzes{scUq`jmNO-M>^U)`@LY%!U6E;xdj8g6hX%6F+aLDBg{Do`DqH% zZ4smJrSCaxUcZrS4GV_AH<7fe)fc!`)(T5UV-flh363jn!`iLJTobV3dbucb4KZ)} zTVO9f`_aN=fXKWIEJgP`PVKA%96BI#m?SIJ^}_W5UH$-L52bCfS}!jkDye`?Va(^J^fYbX;H}?**HA-h^P# zk$9TshD!M1kWlA`lnVC1oj?lQ!wVqyUO8IfoDY45hv^MNf6%r;J)jO`(LV=M0YB)D ztk7bxO!*C!oG>a>Ac=PCYSSVoRrpKe3qbbr5p*o-5=5;Tg11sqwD7)Os_Ebenwb-i za`*p$s4XO6c0d4*A9uq8t1{Z(@Cfr$Pn*fGz{!-k(r82Q6z9k#EzZc=WMYQ9DV<+m z2pXH_LPyGTWan5#`1SXL{&QiJ6rqK3CnZ4B{3RutxC@G?Fk;y`lFU}w1Wwf_ASvqt z-B3{oMFV%}xz5r^I1{Kh79vPF`YNf64^d|wQ{m#}el%&?10f5P(ei>$*!d;|*1qC^ z%MK4z*6yaR!Kl_QTsVS`fIW6@I_Ap{HtM zh*U;}vfQEyD+2#fd)uVYzo7X@ZTtg#ad08`-3&n&_)XB@ErM2S1VcdAJvd@8pNSC3 zqgRgKr7L3}kareNYqqUl(4zLcg70&BA$hSFx#vnKq=lAL=(;aMzV{rc-u;s3kdJ`> zLn#1853ymj%s2kNuoh?tsNr-ic~6H**TZXVQG|_!(5@dPkZWotH*dW~#Yam%WpcHj z?^E-WR1|pfHQrYOdrFy(7d9f;A-$C0&QKzr{>wf;nFnD?0`&TrGdP?LC0}$dW)|x{ zfEcM>CZNilh*)?8#W!7|+;;harBNEO(Ze3@86G6{h%R{T7(lTq#87kQC-_W6}Ya0s&*nqXO{T&3UZYP#KG9krx1hn6mx zq%_x^fkp=vMt4~-6!CK5-nJa{*k;-E`~3;?g@3@0^G-m!MS_1!DjcX6MHz2@3WOJ~ zW8UAAKm~`2AvAjhvKV;?CtEF;8e<#k-MKiyuX~+@@xv*yA>cFo#{5D2cRz)PuVzbW zrV=w}Jf+S^zK7=*3I(6lTafG(QAp_8%Iu(6yrUyN)BAz~FDB(UHL+?r655$bB^znM z_^fiGm%IwweK`yKBoD_BiT~bG%deymHO)qZ?bIzuIkF%4OgNJi;|;O34DfML6ub^y-!w}vj0>G+UNMNT~(LRFY1FFKA-4jg5c*16(nm`fXW}1 zLEF|M&bj4Am0CqpY@<|+IUxKSzREbGTR|n%#*q6g|Mn)&C{6>bvH=ma0|BM`NrIVk;UtKd zgp%gLx6q1wVWbz|0ox2CLGtr36gPR(TDlt(LD-A*YsKQOI*$I4Q_ z66DdbiuKIMni1IaY%P7~@LjgQmnuvc)e_f#cB1G@`yn#SR!~HV(x28pgxS*7L?Gu4 zRhX@eZq5}(1x4@B9H%0p{*Mh$>Y^$fpYxQmu1aO3>~2i+`UG^de~5~mkRl?FKZpD_ zX_TlxMz-hILG-SbAkmx55u5QC7JuFcvyRRs8>lM;|H3TZ5Q{>m)=NR<6$+HScLEdN z3qCO^^vLl@^wNJXsUCNYev2s}+0F8_r&|?Ub(11GA^MG*$595Kg1w-#`6Bc`btk0p zbRhLbQS&K%kQZ%(-hgY+i7TNi=?AFK--Zxnm@Amb9wW}Kl&M@f)&UPXO&H~Y8n&5( zF=g`pD6DsOWnKkF5{pKYK+y(Ac)1(V`ED7-`a4}IBE`I zXcR{qA4(zwmqpO^88a&vepp3)^_4;*O2(WmnWv$zg$IAf0tv0#lVB?^fjDB{D7A0t zkas}=oqV3fEc~-xuw|(^{FT??$>m-mG_5A-abHWQ>aS(TpPdPB*Z7hj?foI4?-9J8 zbV1hN(}1N|2`Td~^L~hI_cDSpPfl_Qr z|9uHte*7O&nDGc?EEN&*vmt)A#ZgO^$3V2xEb3B41f1NSKxps0MLxfDo8r~yQFDK} z(fc)I(DqM@X^Vzrh*`t|?Yu4U$l)cqDIgWL<~mRpU)Law#}^=DemxAWS4InEwLmzw zkJ@s^ABC%S!@Q>|T-p8F@pWiU%Z3T|r$GdFAVSUpDun66uAKykeWyLVLw zEIz!BIZ#&xCp_iBYmEjS{dN<)w%!9K)@?BD*{~9F4sod1G_?D87R1l7p;ZTR$h8jb zMDzk*I;trc&IN>^ggaAEY|#z2B%4)}{sID?D8kjm3CgZ30G^liq1u%%spzzF%CCW@ zXAW%zqb-}bFJ`_(-^*)>S%0SIotr4!nC3r{0V(tq9dX9MZa(9|WoSdi3ib-eEi_vE z4jwyifd)wyqjIf*s*?BwCN|HZQN{*rwa>wDnGiKp;sf=D9Y8m9L(HDp@N4BuXcqJ1 z9akuWwYvx5TW$?)oYMt2Rr~3GcN}00*9J$kD!BDXk+*!(1>DQuP)dheQQo(|U@sgE zW-$@qekB10d&;Om^C;xhdkxI{UXq^#CZw@r7NwiKkT>*g3CuIOjV{lYLWtf6rg;f) z{kb)@SG<$#I9Wpc@GXTk(JQ%DEqZA4vSE-*TS@ne3czny9r;)#1O13@fyDG#g6BYk z;hbQ&7$r(NOJCzP8YR+VTk05XCKOKEI@7Z`Ecm+PBPh9UBOCs{hSs00MC!cxVAvfC zc{>ZBx;7nTK0X8sQ3P5dv(ebgd4T)%3JAxAdOIhj_9oU4n$s@8u2T-db&x zwTUahu3^L`=Lp(WG@EK(E(2X&b?^g!L$uC)LkS0M0l$gou!@|?*&Fc^9jtgoY|5}F zrR(gN6=j3gD*RE$r$Zp=!=-L)p93pqiJ*X7C)%<#7NoX|G7CCehAGaMQu^FvFX-8cB3I)_V66KSPV66G&szAKJSZlGyd)^NT=53I zVda8hPzjWNl|)}-i@_u-367ldM!S}VL-aBw?kRZ_Zud$n_{)!A#GSPuE%qmR8s7wt z19xD;MhG2hmq1-Ig&g;;64>^837nP5Mr+07knr-?zz?qBtVDP!BT~6My$PP?XMo6s9dz5G_W-5c zgrXdU3KFltuDPvL@90j-ZQBx1b?=0;bTFvDljE=qX2I6$Y2<;d0Z=U!L(3{=GdPw3 zpB)Pr?y z`?Z4QxeR)T^(pdCiZFiqsxUfSX-Qa~IEjRJ*1(4aWrXGADZ*fnFpAi$$U*@a{9G_Q)Z-ZeqV2X=tJTsO@ennT=sRZE3m&_EGEWiaXfj&kCxBFK3T;9W5f z>^=rTzT8I;Aa5|JpvY=EnV2d8SoUTs@0D^Pd{5HjJror|Ux%}Zs>^!_N#D)9=nvSNyaeO1_I@3*yJ ztY^h2d#eyqpCyA*a`h@QBwt^UqVPf^^Z;!7qGoH1*CM06kSgQ1~v zTE(=Ax^G@jeQI7tTdutik7{f{pshh&5SfAI_MBwT^p>Qu^M%0DMG6^3XYuS;cfyrJ ze(dOvvP@uqHRvrbV}8EPrvm;?7K5vZ+F0W8}!A?vXyG8e0W z-3r1?$GvKpA90l$Q!YXJ29wY^+65QOXVVw7zEUQEu1xK=$7s6GGq(l=ikGF0u6FYM}~^JVHcT4Xg-PXWX=% zxn~}+1o89JxP3FGsOui3h@;#OA)k{egGmc`YAc7L*KD92%tT=4(INOX6X!rf1w>sx z0zKT z5k{p_*pYAN!p1;Rv?rn*)|*9BtJBMZ*n5#kW9o zX;Utc8vx9OKR~peBXo7()QXc$D7@kUwKLX~KC?)j(S9=w?Wt;v$M;JF-?A2sPfbx< z=Qn_GP;`x>QEph2y?X9rSIVPpk&4di?` z<>W0YL%S-r!TR~t@U1imJ_k#pPj&{V#OxrYA({p9W`2Ze+5mS~Zx)e1D1kE9u^FQK z40Jkap(4vbaC)PFnzkMT#|KAfz43TzI#LGI@n&>MYzhi2ZKzqAH{kA!F>u_dfeKpw zAnVgbP~{m+svT&f#neB5NUSBDe}9D9?^@51XxxVy0lC-I&2CsN<3<|YDQC=L`e<2wqrFK%` zhd!?(zJ*NoSx?mkK4lJMnh9)J*NM-DCa`u5SYS zOL_=5ITiFsr5n;I8FaA8mr+TNVV{(rB=RNhk+Vm7;OHoiVZu}Sm8X8f;dm_k9JgV9ob{mTV@o(rkVtP# zn+q}5KBAX@t--0e5tfT@re1{&!D&BVlmyb~+G1Z~+m=^o;%zu}S<{m8*0bj2^d5zq zsb}c}#qsFhTWd(NkA|4DXCUhEAWZbFsLb+UNOR|O33NAFTJWGgzvh-l-jI!6~6pPraD89 zz|2Q$=%eH^vRF3*{oebY^4>j6#oaJs`ZnlLdi_SQt(M0Pn=GXAa|_{9O#up>KMJ?M zt|VM^RZ#KM0XKY5i-H}tUMza zyr+yZ=WPQc>E#SRc^2>;WRVx+2cggB!6Y@GL&o@mW=1Lz`Fjd7^HR_TgU9?WIvMmu zY8N+SPz0G#2kA|d*^rt39qGR!2_{4U10UzXEjM-cgG^22Bk~nG&YM!VbGWphdMT9L z>3}EWTj{7qDZ%6dMaHE*5k>9o2E4YJC|^^~wtpvtvPuI;1-FN!bDatI&Y3{^TvtA% zI(0FpZf~UCOnV{Ql?~{X9kCGTn+n1I9FSD)S=gWF3$F|diBCg{2>U6HQWEchPf{4{ zIyVYy)x615a?9aU+w?@ZFUw?$9;TEV?j!R%Vesr=F?0u+QWoNpB-;Q$30nhi?1quW zvn#}zqK(ucp#qT3b`Tt#UrBk*aYx^hI^YFsHt|f1qTLoB2KhiY=ElWD$S^8Fa`$?u zbmJ4Shc_SgTdP9RY!<0^*b7uPJp+rvBs3s)2b>J&aYPnfWrTkXLb|dM9TH~8VN43( zN2(7s??f%^4Y_1`_-?w`R+=%nznk1=_YStdTfqxmbQ3}vtO*pEK-Ry01;4HL za#UAWqB(mLK=*hW?3wu+cel%dupi4HT;~rAEI$e>Xj5m{W@mD(7@g|8wwv))sL z_&ody9oFrGZBCW2ck>(KN2MEM>+z22=#PdxT!J38mPC0QB%lr328G79aPxc>;qBjq zw5?_l+tN}PN~Mt7C6NVJ=d9%-z3Kg@$Q${MOn=`?pr*!;qopP>A}eZ{&yu# zpZy(FcljDMJgr}yzUs(X^ZXWM=ZliNMX#U-ZgoW1s$m$b_M>9l-@wBNN9ID0GcVl0 zknVJw&8RyJ3O*eQr&HHSB8ltIDS0)DJRsjfamS-68{<4GgjPjy=bd4ElOFiX3M2Iv z6?9**7%u0}6!czx0Pn)&xNrVk6R?xZz(aDB&XymC9ooB?bE{R*Ia(3Tm(ki@aFohjgxehv?iBh9BbJ60hPVk;>Pp6DpvBMne5W6u8y--?9q2%M#!I=4#%VvsF zq9s{m<`);_@^>2;xV(o{Z4p$Jf0L-+E6%uvIMKzIk5_Ug{bAm#0kA1NRhg*OL-DWH zfVQYIJtFfF>dbe;`q4M2dCv}beAow0Z=M3XnN^_o)D@0>{{~lOjtV>r@AAcBq17`7`#hj*$F#AKW@J2%_Q*FndIc*`ma! zZt6;+x6$I%Q14C>Gx`gQAEd+C6Uo$>%GX4gR||25xJeXzbc2_B5Zpd-3z52R?Bn~S zcxU7M=~suPs9Rd@jH)n0s7-4Mrxo4`Oig!CwU{!}=#zpAE)(?4t@BVyY#E%{^$Oiz ztpjGOgwS?3Z|YA}5Rw0TE7^J@0J%@ez?p(d!E=Q^m?(Y$N6JnzNgHgLJtpe(pWHy6 zMdV}ZFj~&ZOm2bdK`|ty>_eNWN+WhiI+(f@p~=|A&{UjE_?gM0sjo4h^Q@Dat@D-Y zi;CbR+0Uk)?6D%x|7oMJf=ak=D?rzshbyv^$~X$RH8^j)HO-L|;bLPGndPW}O1S5V zS(SNcY84H%poIATYZx}LqbOII3And|L5f{rRK-S3`tooq-G0*(zIQII^vaOph#z@^ zGQa+SY?sRdSGT92I=$|j;Uv!>;RSeoF9GX2*+^v9O}?y`E{A9$nFBi+N=N+?CoXj> zB9zLgwzw`>CGrqba(UFkb8EPJ?IO_XYD0$WYs@)UI|{FlRv}6lSmetHLcr=K-S}pkUEP=fLhEYR~#UQnyf^jfZj}_XvZKT?_mU4xu1a#GBJ~}D9AI@;fA#)MH=8kVjEj|*K zx10haLKf`O)QLFvI#_empT6ww1BcePaSbl62D?Y&gjt6@=aINHS|E`{wVs+*{&#)h zdY3k#uiYBV*#t|J5$PW2X}3I$%H+79^4%TatndhOvxOdc>aygWIs-MN%5kiRE69>zSUc@G@3=*yoN9$(f z(~P4B2BrE@w!%C)Z9MGb)3$a0%cg-9rT{ zRpD*z^qHctohwxxjr3GI zMuO4!2kP#kCaP*g6Jo&~gy*b>;Bw_D1;v_h0~fE@^JltNu$F#u zw*wx&eS_9)sf54bhhXUsMf%OO|H+Z z?-#W&x>sb{7AU0o0gqtkk#Y7Gt7GJ52UaawgDye87^K7>w zT^AwP@}vhm&hG%Nbsms=`8;iSMw)(TRzufiO3>nNCxH8S1o_EVfxF)Ox||N8zx2`#1QUy@sQ0V?kLCj{$X|1YRxHq2?DYr|;G_!_N1{L&ZZ;nWEurUc>q!gfEzJ!!f?tyv4IOqs9p*sco#QiD+Hj1@Sl_$foSoi{5 ze&(U$H^q>}eh!&@lOS6yYGDs&9+P;~i+!#iP|WQ>P!L!NHqTu{oYVg$Ncnyl_5K0) z;dGsfYwtwlm?~9G+~Bv|mSDzr2qCWQOrBoPT6Dfv}SX3W9Oj{)_^SMU57U5Vl@5F5It`OA~M8% z>7|V+#CYjGYR`NjbTT=W_*WK4*~$;W`KV@Q=hrd#xnvG&RryjvMrln2-Yy64W*$Ox z(l5XVILye^6@*9OO*n-w7aTNFM03{{z$?yi?isE)8MRK6i<{?>`SQ&`JgnlM)6E9S zBPz)2yDw>|5g<6YRvitudlPd1HUr;s0eKABJ`>=G%O{s^*6g9)(*YVew0O#ds&5~!;?LzdDKq}N3eyI(dy zm~Mc;u8=P*mPZ))_UyPvLs@;cXopg+*JYJ;@DV?~@Cf zx7ERCa~At@|1Xfz%ZC2uL})lXCQuf-OH`cnr4IRxA@!JY@NSnzF^P$^|8j;sqjfR) zc26N?E@-6Ey5%{=F^|aNgQYOxbry8aJV%KfDKvbmgK~H=Nr^n@1D;b3$1}YaiH95{ z;+J;P<83Q=ee3l&8YM!^eic>7#}iS&;XGm}F@--h8IG%LEdzsVYiV4kgxyf?!w#Cz zp!Mt&1){Avu)_5dy(BIJY#t|&nTwl{*v=+!KWGn6p9jI>HwrM2Il^6jW0W8(G3y|u>}4qF|B-;8`$LqA zMJ%)7K@{wLFhL!Uv!b4Sx(W*$3#mnwOJIJl7@F~Cgl+N72D}*+6yQ=$1$iwbgAP@~ zJKrAa&v+rzv0ivOzAKK|OfQDZBp(*l{sDCPDjlKV1}*jv;M$RBbS!cSz>RV+bsd1; z92TXyz=pcGC>ed@-h)#$`)J>-Drk{;KLj*r(*nPDc0=p?HdweKnwy>*2mDvNnW*%H>9D`&ptSfRsAz~2S`!k; z;No4zXZwD7=r2vK87O2_WTFT$K>?Y)*%`K|db9t2n#XIrkO)&34#9x5B?#B`!!RCC z$Z{FbsF@F~>l6s>CNI5~BKo!6WpO+dos0`dIv0kC;;ve} z(Bu}k-$V*o(c#>>i$hSap$ptSOTgUnFEsNpn3R?R{QB`yvTc7&o7gP1y;VhqBP)W;3F>yNNWJ{Laa~WXBF@*~-*2g9?-sdd`GyA?+&U5iGahYP41 zGMTVGEQRt)z77k0Hk}GfgLaocQiQO-z2kSfu-USFu4_h)?GGEwQyM`ci zGKM-HeVDqEx=mmmM*5~~A)E<}0N$aQ%&MF` z*zfR!abBi?j_W6YRJS_E&EXhRTQEIVmb$=7c?w-k6Ht>=$q?T+pKveq5sW325oaAT zk(gB!@uH}emO%dGFV1tp_I5wALR=X=%1J{5pF)Xj&F>&CG8=Mc#DQUXI=#nZ1H08V z8k)DfM2!nS5}fAoX+QWEmv9Xr63{Tc@6NQw>2?#c{Ntg97X}%-KZBxmB7E@6fECx$ zxLfAca~PrQM(Z;+^}m0;d? z4#1lCotmPro;9PlZ*hUcjfT9^ zUuG0rHwj5xl81K3Y%qP^Mos9y0xCbA_x{^3RpQVM&qYExQxos0&g{J~?4M5xoyr2c zp0&)_-5V%gL>?U5aKY|Xjqokb3?%$%W z1G&@Z;k9WS(wQZW&OZ4|x;B2M*YsbdE(ScH{|`my;ZN27$8np;9+9HHrP8F3+;i`_ zCG9~fQb|ckdub{&viFE&tCVDg`+0xPF+;YDkX2@g6b<8det*EZkMp?qe$IJ+Ua#lt z^-0WHi23Bt;4NxZD3&h>q}hwG#o1Zt)N3bAM}Gjl@H5PLuYyf3Zh~9a&k|hCW8`yp zcetvV##UuAxeEJ-VUOx+=rUabtEY1YBb(n)Fdmojr zQx6-|=fnQRuQ^Az2Y{ft75F}BgEPDMaA|2gg!ykFl}csduaPir2>*zQh{eQR`5`1K zsRTa`4jJ4DTM8N4ez-jIBzSW_ar(s-@pxVyc**BeBa9|PmFHot*RP28|3c97m1|M) zF=gQYcL`U>cCyZ|m?V_WwQjOd(*lWZQxN>3N8j=M2Y>49$-m|1fOce~yofS*=X)Jk z?v)Vqq?z;a=RNAiLM3|I`YE~CE{*l=!XGM1&iEbpOU)oAZ%kOb}=_%*B)+O$3LepjZAixH@$TL@UVbK;s?87cjzT_f@eUTt!o9(?P|PBqXC=)TO?uGvIz3p zoe%i$x=OH|D}*Y%LKz=d7yQ@oQ_`>ZP-BWa$w5DTGCpkqyaU9Xl=-mW(GWQ7sf7SnDIDYS3!d%GB9>WwfiONZPTFRM=X#Y6 zJ4S5D5hYI)<8DPyKbJs3hJQd;uoay(iupHngZe>K!jaw3@irA!it$n3R&Rk|n<3Wq z&KtxwA90LV_`rA3gYYtoWOtZu#dlKVxhx4$T&Oq;pYvxZrHP}^qSQn`5oMkkpVEmC z^E{AMDJM1hh2f#^D8!1TW6#@TF!kyVk_~0n({MBNPuByhlfnYK%b{zZC-qL)6bt+s zLb;12IOlO9p*S3m7s<%ri9CDwx~>q2^|mNhhoUa8egk$t(s2CIOe{Cg6{H8cVX9Ms za&=z~Mls*;M%!I5lJNtKO`gKN((|aqt(3Xmfx4zGK;Nw$M&-?hRQH^p@W@b`jwJt~ z2$fn8NeSYd*|Gw@-{NP0gjBd(>kA%o0{C9vL2zLO5;y!?z-{~*k?d>%M;^SxJ(0|_ z>h*CbWO!iys5Fj*Sp>vQW?&x)VeHGfNbFhk4Q4hDaU3>#z+>A0V4K$9|Jshi_-0vn ze^v<38G2$cwb2Wui5B4{?L*++_LYkBap#nrI?IZZe+oD2^U!!x6~11T4g(4etQe7L z)<3!&WY$V^+lMuYuS5W0wLTXqvE6`wWf6>-2GjXIt?<2G6<0S_;UBgvh$z_4GS0~& zo_vtRbIuiTdSA%Wiz0nE-VxX7ga_Tc-Rr{0EOsCSmGr=tZba$lC_?;#D`@lf0PM~F zf_~q?#Nt2!EPAm3PJ0|<&5S!i#L`J*9nX(tKSkmQS{56=3_-4aE67Fq|IjzhRg_cY zJtEZeCOF$)N1v;6@n2tmczNOlyqw~vmLATA13{6X5&DwIuHOtl^Z&zVm22_l>lOwh z<&|Wi)n)wY*^X0K#NXSrg7!R!{By0gjf+Mj%^$wiEnp2XdT_1hS2u( z3fjTk!{xKl*lb%EWw{8!SnDp>S>{HCol^w&J}cTCl#E`Hj~z8ufd~D z6CfX*abzf;=mdaps!wcQdz z*HYN@L<7;R^cyu_@kIXw8d&$XNl_t>l)#`h1J)di!q1mDvi{EZgA65gJXgB_-rn>g zf2<0{vP37G^tFb=r-N8$%>J+@CWUFecWdd1TTYbv{l3#cBf-1p7ZcaHNtCf z0TpW$0Zo}EK)MLxB~`je^TzEbYuRe;X%`@bj@N#(jESYe;q=yan+MzLlGf-?Y7k783 zz+DFuLZad=j-1FriA=XiRQ!9+M{XsQmk5xTJJ-Sj?I3)uRt&%6rGj=$9*R_tgXJ$d zWS0svkA`rGCPzH-VJoii@SvIs?|_`co`O%xd_VZlF*CwJj@-O0lSiH*q~7Z zmyDxe_a+g1Ph>4reGWvd*?%y#?+vvp!-(f2z#|5C8PI;$`SEafB)ugn3}lAIVBSCt zd~do(-4QB6YQF9eva1_Qj0oe8E40{4!fSx<5Dn&9AuwSwj!QV6px3{YdVY0`EZJ5K zTB-6>@9WiQ^lm5iZ=Zq!+b1Bl=LN`rl0u!mQRF6>I>>$W36_5k#@jO8p?t}+yt%I? z$h|suiJqZ8_SrM;;OOOwE?0jCm|O++2?OZcxF z?eY{HdSK1=Q&i>&F+5qe9GY!naoNc(kf8W@7A1^^%Dm4PY+nVWl-#ShI+DzbFD<_!LVVz3PBIuB+xrF|6h){Zr)e;ut`!66}O< z3B2t`Fi7teWNIfzugjh^Fk;8Ttl}IHu=hi3K)iKnzQFw(AUFL2WD4ckI<*4Se!>FA z9@wCrt7oAi^9C#~`VP*18z9dA3&<)m>y0kMpS-ise~;s1ujr?>~Ne9{U%p z&rQHq>IBG`^}*M-Y79qcjGB#|$?sg+K-(f5FCARQsW`=CavZy%f36dE)nW;};l2u` z*&f1fj4(wKqGM1jCkYixn!)Q{CpIcICqM08&E2gKNXjrwzS^xeYE4fqYkKiGY-_Qm zy9`$mKN&!8RUL&l>U@WhYgMqVD3W)G$!o+nG{OBkK76&8hK)_)xbgEX(oQdu=l*n( zWp#_G+z@(#ToOd-<0XmEaBL1<&Mkyf-_&UP?W(YF!Do0bE`<*l6(I3l4jQk25+q5T;6HIW@!l1^HxNIVD1x~boDb&dB_VT^Nl+)Ywgc``10mJ8eX=|$nqWsq($PezLBvS`qPpl+V*@S|1NHuty#1TIf zOVP&73Opb34chS32SRRC0`V#d=1bq8G;H{B^L!C>m>6Jxp7)qIt7A)woyyAHj>ix}egnMt%8q6@nJYLT1K2WMr7bd#^ylD$i$l)h!alOli`FH4jI* zNkZO!Gfvp&b4Xl&EslP~WUVe*!icLiED983HJ;R=0)BXcF4;ze9P)uT=Y4tHd0z?h zXR=(24GBbt^HXxtUy9u&^d27b?!boiXQ9nE3uSGrzze!EK+Q26C#YVa&7%Yf0fzw2 zl-GAcCb*o3&UoSH+6m~lL>=UyQ^dwMQ?PD+Ds29A2rLqm@vpBcP`UdV!6Vm$JMRpO zZ8wij$SH%BO>$Uk{s`5mbpb>#CX+T#e8Bgd9PD6N^qRXTkkKYjh+);hJ1a5jmeqfR zd#4B76iX$#8WW(GBM#AvKH{E1OVBvNp?!1zaBlpnfot0_>C5OZIc8OO_~}mQ`7;8m z%05C}z*%_A>|Z|1*D=|dSFq)S8?IrcK#u1K#2ns5t5xXnI-MOzL)LRJ&`L*mlsTI*fD|u5D^(L=G*XbQ*`!U>$D|p&zCB6bdcU7COOfU# z&dSr=2p#z9RX_{eP-Jgq7{l2b8}19!kLXX55;o1v=ef3@Cdz(G;D1A6IQi~hC|mjp zBnD++k7`h@;cUS5^nJTDXnS@@QH>V4dT|i+?KUTmbd`!%Ou%3Oy^8$tuvzx~r zeEJ0>w~gV>U`=-K&}Q~Mot;E{sU95o>kiug72xY5d7#sDnmj4=m#j4(1;e>>>Cf(G ziL<)C_IJiPin7aPqf$L|;>)oXDQm@dhNgGs6o(x3#TALc^v&@i03uMEqqo$x(ojTGG0 zVJCYZp<16zL(T$zI(MBcezVdB8J>Rw`z5DI3F%Bw3;u&jPERtkVkS^N=fQlADN(+G zhVURZa(}?ZTP^g#KHZKoIPA+#wmSsoHHDP9^b%I0Km-;vvV;U{j80Cv5RGR-fO~EQ zZOo0s?A6IQEBzf#cH_g#1V6*IsYm31p(@XBT^;2moeWIR*T-K?h{jUD3AQ0vIds zMy~4viN;mU=zPNgFn<{d_l&$Dq9~qzp}z-?IViJ@J_S&#svvK!ND{N<+JS>9VZY=i#Zcr$7Zm3t1L-u z3=UY3rW6M{G_N0e!O7b_SXL%)9C2Wz7_XsNI#XyR&^+Eoq<+mca*fB=Bw zMN(VmJ&dl{$5PlX$bDw>1DQ8H=Y+1os9Tx?$Clbdn29&lMhoBfa#hR@G2mvAc;sl+5lQn=L!F-xj4pI zll3EN43tfU@yb2_2zg_BXt3}DIVTTRlyxJ#s9Hik>(7JG58~L|ViR{{l?^1vsN#d^ zKD?ckThL62yn%D(5LGds4=*w4Caf?1#XHmsS3kJUD`->z2P>8un;IMs^V zu8Ls6WMOt^j3VV+(Z=m|u^_tRdm_dyj{ z^A>^y@_=W%%Sf;JVR`Ra88EzeFWtoc%B!u3rsR5`vHHH0f!NMKLTAMa(nLFl@H%;) z6&F8`62Dvq1Dw-rLRFtv9w^HWeKi;7-b=wZ=9v=b_LXuLL`!lT!smmffe+~;{TN2E zDXzNlAErC};Vi>q%!$k;$__1|*Y&hR+ox_=&O8IIj4p;tyAMDHAB%gZR~%pZd7L^- z#iOgYg+N5nnw_&y2rv3ZBEu$4`e)A_`twRLyfikB9_syzwq6nE=xeBh(W(9Dh~qI@ zy>Bs|STF_qMwpz^j?Zwh)R;IAzd>l|6r6~<$jl(AXr1jfNLeloy8V|(d&bYK6X0hT zS59#9=W79$R_4y5RM1nMMexuq4s@kP$ZwO)c@C^OSSu<8mOo4(^mJ|DP{azteeW7L zB-aQgcRca9UnsoEI7f!Ew$dk;pM|I+la!{-Gh+Y8B0Li?0Iki>VPE!YP^U&<@zGYe z!*PeZcQ?QVZ$_5eS;YHOaS8a+MYwueE7PG`}Xg1(Ax5Z+&}soW_5pq5tFI?Gqx0Yo+<>h-Uj^GI1Txps)w>7 zX&m2~2Jw;{4N`Vk8LK zB$MM)4ZJ(&Qh@K44fj4O1##^zvZwboM7s9?TmL^u5?#!_w2<-ScSOP47(S{adlt?; zIDu{jx+B}GS-gVxWw34RE0&sgjEa_!bl71*I`gLi+ib56*_E&lc5h{7<2q@aR+vo0 zDOG|=bRk5ac*?nC^qST4-!ZoLoqkYMcPEU*K7r(i1pJ~u2+aD^=&c8A@d*Jz9A+L2 zW0h$Tuel#JY6R0uXh(91?iaH8>nL@Ji=gG~G_9|o#QyV)A4FT!@iy%om{iih(NI{vf!wNHfTzB z@;>QKqKftP%)3&7TF|rwD2HfV!dApmo&ab|I7oc21arm1&`}sqTgwX38?8>09u|+d zru&Lfl%Oju7_^0Wuh0&u+#Mt_vjlYWDp5uBF{t{djxW8Ni z-MX_F_bcp%6JC~_K>c%sL$C^yVGf{dGyRFFqlbX+l{h}iEr!6pcd&+>%T3QTg&v+i zXip`=A~zWWF^x8uEs??(q?qpQgjm=cQHHZW&H){I5uMi=PMaRe%C)gHC!HDYX}41h z4oKnw|K1wnib*Rn&1nSpqf+=4=P3Jd#SX&fZVo>B=|9-pS_Ud(i>R9_X`tQiPVI0= z!Phq~LwEnX1Qzq^(7$7j#FDS?D2u*q@^_FCZF~0$RbFoh1$T1kqif3G3Y#D2)GLys zseY_nmJ!>3MJ9eQQqHF&Im(;pw}+W&-e;5UQ06HI1Dx|AZtrM>!3udTSut z>o~Wg#0(DW2$G^Tm(b3``7rO{arkH7M-8su4sDCL1huKSR?d zAEf4gg3?fq2O!QP4>@US@^m2*Y_=l;4i};8$`TM@T?fWD4RG@^f8?IKl%ujyj9N)9 zVt@S_jlEiW;O>!3JkmtL&y$((?@9(7m^Fb57ax*Y*V9qw3pc3Qu!agcKSXS{e+Nel z-6_^|07z}{C*Pe2#swQR014WGm}oQbou3ai{{_-b2mQz$l6h3y%?>Kr^&D*9n}|QA zra;~|efYg0iaI?M%hP-L75Hl^QFNgQHhe@9YD+TFo9#`!ogQWA+sb@!nW}+JkJo|T z4HHs(;RI5Z&4;E9O5B*Sb|_tEO_?2XMUPd*V27~?WQ_r(pZXFi3Lg>{dLvkClNfe$ z5yFlyvzdM{4e-6d&zmux(z1kQRj#Nq zbq6bm=M5guNb0Vq64X>2gyX0B@MfJsP`=!Od@coXk6&y7$NVkym4k8ksBRnd3Irh4 zse7M5EpF9Kb6!VKg!>d;-R0+uP&Oa>`~XlYWRRa?uU@|XyAdu&OqS9b-|UR^q~ zZ63ZY^98zA?IPru(BawMEl^o4iU;2|V2hRYz*{HFUSH=!j0<+aZuexcwt0)IWF^@X zzN^WJBS~bDq7OJN2qfEXCmLkfhTxg`?a&}I0V;`n(AM{q_)j*Tyd!{ktJxbt>UcU{ zA+a4cx?YB_$JWxn9#%kd#9XeDqdWFhv>@&+O@jkJxUf{&9OQbBaAWO9sS#bK6FjB? zN;!Yvg7!2ycsv}{x+TGjlLzRpm056Ldjx^AWAW!NAw=g)DwxM@Mz1>f*}GT!f^hz4 z*p{^!MaX%;txA8m-6cU-+sWd+(QCM!%zl|RwkCPo%^VbRKiigz z9E{beWZgedq}2s6_M4%Qe2ZS(t|P47x^S^@Tds;w7#(m|mfkOkc-EtjNdbEg)FmcF zbrw}Xse?P%R@I?<7bf6dZ4OxGso(-tE%v^4cdU0e3Yt%R2L}={y$^Ho@kWxKt~&Dj?(2%H*pDA&M z4Y%qEqp(oyTXc?>wPFC8Uw%O!I=Fx@h~dMM+K8T`OH^ik06YF@%6hmFJ`)0X^w&J< z>vb-?j;N;1vxH!zr3i9j4iU-wMe<+PMj_AVkub041HQ4npRju!0nOQkuy}_6afiXHj}~ z=S5QT=057lM?Qrbgb0gZ0-QWY4d`eiu5ASf$j&)2d4W zbL}$T;ek{lLXDXPUdUjsNGb5|uw-i8%eDDOVBkczP*N@VU^{#NoF%l~YtYka39w4sF zih~iuUPId-vVGqXxOAIhyVSqH!lBnTeVGv<7N&cWw=QZzNWcclw7iy>t1$zD%x~My z>}~YrK7%JULk>RxGP=f_+Y+~hdv`}L;=kg^O?YAqQjvUEz;74bct#jcaks-e;Yj=$ zb79Nb>%dE-2t#`*e5KiwHODjz?oM_S$}YoDzci4O`)~)^7}5lFJ^D1C)NiON6U;BL zQNY>KJ&;~d&FV`@qlWf6!A^rf7+|vT;sdJiGgK7IdJjS6zlC6Rcbu8m3h{;vABbHs zhc1pJ$Ri!0aBJsda?X}M_;>RmUb>1#-pMUQM29>(lywrFO!ucOkSKP0^B>{=+Zoe( zX0U0)H?TQqN*>uSiC5(5ai|;b!0Jja9B*^RmP>k3q7s4j3;v*&PKMG7^&HC8`xKWu zRR!W=OY^5fQsL#NZ20SCM3v0%W&hlviMKMYx>a)>+9+2Hmczq{x|0i`d>_E%{5oP& z`y&{qS zV2a>fvY?n@iVlY)l^ zd{AUYHn{mqLhG(4Q2Gm#euOIbY>_YZXxNyp?KldV-Ko_527bbq;i@OAROzd8#jxd< za$n24;hpZ9P^Xk$WHDiUskpG#a8?(qPxP4cfdv z7uO$J#6I^egerbWo7NZ7;;wl54I9862G{t-{u?z)KmO~)El-4I2Jb1sk%Z)70a z zb|LxV@iA^3lPSFO@F)B8WDvrc;m9_9GdEfA8;>m(4wJekac4p!D)$P5ocY_Jx9J<& zVHJk@vv-5-Ee*WB>LpmECQ=TTr7&t13m@(yDmrco79^4Md&c9{3yy=uSB22~C50Sb zNS{Id`j3d)Tn(8w8<7}V&+fGN0V8=$5USY<8O>>UUbqB%r+)^uYb=fo)K7*|?L>U&+UX1Ls-%r}keT*u*iphnJj_`X)E-pFW2Om|xK=GOq2;3=y(@j;W z;j;`OQfN*2_iG$>uKTPof3U@#_s4_j&{-v`s-_Fq>`p(j7eHbJ0xs zb5wu%DyR)CfH&oDKtnwM=6`ob=l=9lOYTbGYpd0XRN@yo6(|q!kIE^nsBe&eR3Drk zjG;k(1@3CyV7UJ%6)K!*+$?Acmwq>ac49IpmjuBH5k4rnSBZ<68GP*KX|A-kGS|L& zi$TxEomAVJBuJe}!vz;_f>Q_!ZOJQ#nFpfaF}j?qTsRH$i%Xz&Lpy$E>k2COm67d( zl~n2xEf7C-kt>xb20wNu@Y3ihQjjk z6wb#YWq5nX9HLHI((aZsP+GKresSaxvdwh|$%WM{i?dItmP1D1von%)e|0U+`zMUQ z0ErZ634C#kCcbbL4XQ&D@S{5}&};CLS{_zQ-Lb8QR%1o(7lzATUh)dR395juZ|cyv z@EEbR{~EEjDGt8=!f?ZK8l0>j;{yW0P=Di9cWS<`;N`8em8_0ZZxR0j(>GvYNkv)6+YwvXNmrZql4E0~aKl9ZJb|yMn=BHgt-u1C#3Kl);WiC_&^q zN_~9?XZ%fsNuzXlgj1k?)lTB4l_)-RJO*1ZIYVP9O`7%f3?(~P4S$v#B*otChG!+O zvEZ#qxD|+?)Yk?&tz+TdYakb%n~!C*y|IpX0Px*Zz&?SQMDN=f>gtyw*2<`ZbkWCc zEEVQ^-tOrJbgvSuVKLV)_eLRR6QrDt2IUgDnf#E<@C9dYQhsZ>sD=~+_hrxMVC7j9 zJL8C+9s7tk309--SKe^aUJ$Q4v>x^u)`Iz3BjmM^gAQu8gJIYv+BClrW(Mct8}9`v z(E~Lols}2Kv$tYfN9w?_9zNV~t1+HV2wX%^A{WqyHpvv)WqPZ`H8Ct>4|7VC1l4v35y)BI-Z zS&3V=u{T^`=KOzA#O6Q+BKu1e6}Odi@_X)k)zLrq61lq78X;zYp>2rvNsaRi#@v=90!Io%Q_@*U+`>t`=YG(p) zx>W(Vo%>6Q%nB{3w{{~+oz3xSDJvtFP0^^0{>l5j$eB`?E0F<0?N8+&qaMg4bv>iH(T4o=R z(M2M7eA8<<_DhpHA&nmiXj~N;(K2 zzKP@U!V2_dix4exPmA$}m~K14TjVrf1z4*5qzfBcVT08|_|%R`p97+}`};na|Cxlh zPcjVlKQaeW{|8#A5Su5Fw*=Do#j)||2l)K+GT|1WLCF|19U9G{5Wm5ILM2PlG>b*Q z2oS*!9tr`+FEFQmK`WIRHG{T=M8THcWcMr7Zb}%%z>+G1`+d0DKRlItRJ@s1pdhr z{oB>h!S;phI)yGyBwr!#`;SseKJFdxyLQ04qq*Re!OY4t{BYu+65X-8f!(qunqIn0 zlAd=`5~MfXq3)ev*w%mg#P+RBh;m~V>d`t0YpTj%pRFbm4i|*if5h?HIA2)2G7NkZ z=i-xBW$`L=1L%0Oo84Hk5TYMBz~dSXu7m6{5PQCz8`ZXta68$H`3Xn_-}AmJgbRb2*o9&XJ!JMm0-GkFJ{3P#PD--;KXAbM(@_p9 z;=GvrkuW|XI0HYm6&T;Q407g%P_Vu;`mQ z?$Ca>p1zzD%e#7c18%#~i5~cEBI17QLC`iaT#`3VolV}9dvC7`yY{0LC2Bp&3ug3F zCDRTp(7Ycq<-U^@8pgPmgJFTM8Qitr!E-3m1S`=ml#1_ds^aM(%6`2snUV6Cc;u={ zUVJVN>nt-tzPFY%l+A*+Z45)U#~%+LtAQIstHD>f%^*_#H0)^K4Puhza53;T+;+K1 zENjxGeznFz#y9ni1DZdmhcm6429uX_M0qUUdzn{g&9y!#GC51`Xy^vVF$qFCumU&X zBD9Z8g1%kSShevJl)g#a)OR2XXHv`2@SVBz*+OAbZNCewII@^t8Dk7B>3(>w>1)E( zJ_Df+N6-xUj4E^wkU84jgsZ+Dp(tsCkK2S18ALAVD~?hpy#KQDeNK}0CH2VRNgq7P zatCL}K}P3y0;lw?R9BM(nVV}uy>l2K!v&2&l23rV_PGp(Pfha@v+Br)M_o~AOBqbe z%g5GZ*N6>U*TCVc>p*pEEbrv2L?#1x2F;t?LbN;Z*a7P_KL|o+i_rQbvT$pYG6fD-Aw`GD zGzh2Tz7|)Q^Y0~`*jC2TT6+qPCJwWw_I-fY9!79z#uN#wr=!&i*sS%nTJUk4Gqn{WVGgTOPT(R(>3jAAio!8jcduF z1Vt3P`vZu&c@qn*8hO1}oS{NL1+Fju3M(D+k<1+r{EI^n$Bs;p*Sn+eec}mNcxSN9 z6PCf=3uQ3cGJ{fix^aH>EK$3(veeuEM+qe2H3 z7C)vfAMJpRLBX&>@E~-KAA&>HPvKSneloX2n;s4HCN)Yf;I`lOXh2hvF4ZnYucv0w zV8>T#ldT%etmX&7jn%N$q6;GZcA~7^P0%jIkHgPhhs6(A@JC3UXK*n9=8UJ2n+A-K z@jCCk71!fA>&FWr{{SOr=~AqPOfQ7~r9PGtlhy4mmjz3v_i4`ZABb$)YEZC^Lu18E4oEhMoa1tpwczDNbh0dgO8n^yrRw5Tv4txf zf;!k85C)5Nytog2Tj8KoGGS{XO{Env%#FYcFhs{8H(Zu?m(h;~Vn?v@s3xk5w<2?b z%gFVC9z@HPi*$faJ~IkE!^W=^;P9>z$Q!IhUvGXwx>H-ZE1XsEMjt-Nwa&urbF4ts zvy;e5@TQ9mrLj`{1m~SlJ6@)sm>03Fh4>y_3lrkEA?Va@dTGRb`a{}(G@u~vz(hDa z9qS@(Dt*z>tR_}JTa_!kM;^|-@FHH^t-$FG3z=E(4Y(LRAnX;3K*O#QS;u{a(vC5t z(Amkb980)RtpwTrJkZ^nMW22o1n#63>5&@3ORSVaZdwoELSQ$373>09G;`4C1_5fp zs=qL_H5kroz5)RsGh{bH!$|JrrEy?|2QVV?QOmWQoHQfGdBsWuIvT#UybH1Fl(cCtLE_Tp3o$(Rup0Dn1Ar* zm@}yDX$7OdOebsl9jdQZg5K&fMx}oKP5Sk|gSnH@I8xpbKA-*o3lG&2g@+12qvr{9 zGYt3TNPA-V=p)$YCkzh%rLzY6pA*5s^4QBq72BQ?!CL3iIF9lukY3J*Z>eNa4Z5S~ z;AjUq>JiEFXfp=4MGxWhpCM}eh!`(z`7>m5s~PvFMnb9G6k)6WhOn4TK@Ucw;GlH~ z4jq1pey0Ut{rsaOFGU@_&$A_6FWv&5cSoSkfa%H9!Em-c4bARJfzVG7UXy_s$>_-J1!1)coldK_bv=GZTJVDPFCnA~R0pRrvp_dna zfoaezl(x2jT5=|Ax{yc}jQ)fheun7!n)RH?fKFufs+^<;C{X!t5H>OWinc;!lDBRZn4K8kZWr2c|#BOOvNCi zg($s66o|ArfmhFcgL~`$fYnU_tkLk&U;*jQ(Xt*T=S3Qk_vi5u7D=BtW;cH`nfYWm z6MhRlRF8oU)>DuT^~J)|!uY_4#clzwpI@ zuiXV${;HczZVMyMr8=S$Ulu~Dtv`@@ig^C-a!Q`700%W+qT5wJ@mHZr__^I1mXp^& zK3D*BmsKKT@d7-3r^Q(Pwf3+23YgrlkbfZm%S z|2Rk5M&l+qu&^0rZt&o^)r;cU8|mbEp*_Uh20bLY!;Un)J;VEnd?7eb7=oOSg0uK3 zBxI9IJl7Dyt(E&>apEAdc$NYq+Y}(%PM*BK;WiMLo8h>ODJ^PI3fVVS(@x)#h)P!z zqAyvDt(8+vro5Sl_hwBYuEzw%>75j)yS9cVeYrtg!|y342FmpV5MzQN*#2*hg|I2dSi^k2oCGRoLzN z0qf2_VthRzygh9><6f)9lpCjr@&FSUOAN)277ijE)pz6`yMD6YXclC96R4_7-!`e8dWNT0loJ`p z*&ui$l2^Uj1D3`9!{>85i0GFtMB4Q&oZGj4q3T%Xe5ZBNa7ML^b=CGN>HVY{YptGU zbbo(1pKZfBDNGUNXE@xohn92mR)tX;-ZzkKn-N^_`;MoCh1f~XEQ7_qOSpv|lhEuD zk$2{~2Nr~XNRvGc$-I8(%(eq>7bcf>%nkH;jd?*u_XwL$J$Uc4hbZI8H#n1SK>PU5 z!6!!kQvEw0!1OU+Fmis3%FrtMydMQ^;i9Ob;1Ql&BS9F;ydWws?1Qf=uB=*_+q}v} zO#XSf2jrMV(#38}1}1GB>V(*|S49DkLkusfCvEU=q}||XOEEMmj3W8{zVz?6LEs_t zl=ykQ3cIOvgMP^zR1D4YDMTHj`qoCm9`hZPesv)+JkbSCOMM_BWg)rm zgC4Zp)dK`S@$*m*&`V8&{i3~KET0L7PA=lL>s8~2`*P64_BkBe=|<{ggCbGMu&RM2 z%$aay30NX=3A~bc!{YyO7Wt2g;h)h`O(-&?NQMe!$WSEj{=NTz>s+qA_gd>&pU-{!n+JfYtTf*8 z`!YOO@*9olkg#mSe&TX~03(0pK~mL02UYlGqP1V&P~}!S#IdCp@wI;1Nm3mS|3b<+ zGQPR!*Q-xxU@?R7$GibAra73tUQuZGl}Tvd^#pp}KcRDpKk&ckc(~UG+}%19)}b>F zCtewHuhzG4JUzUy8lC^U%khO6)nCMNz9Qr}`I>aQKLwU|&$0S89LB@82jHx=61*IK zMb7rGpzmA>a#H6BmQkJrC2u?CLMH*Nw)YhZG#n;B^vqCmH!hM9;?ME1D<9CYMIW%A zb|>M#Q;}x-FJx2RkIHp^DPghGIoN8h2&wlbazvy$VQu~e;Qy8f6$=#^4zw4bUNQrg zb{wN9=RXktJcB53+)w^m;|oQdn7l~4Z`O1@Av$BFaYgw~O!BZHA*B!;7Mo%2>Qr!h z*aW5qTm<27h>q&#u#*g?8Lz&?5y=JYJfi{f<0So^A6`xkZd;5So3*g2rx#ppkAU1k zLF|`I)82&S*wiX%U{+2sk9~WOndZU3?AuOUF)twvY7Rh8>Nc`pEf2RU$PkZ2tI-V! ze!L^a2+637VG=DR4C7#V6B%)o@VcJoYZ6qB;7$XGhLJGOh+CAYEoh$2h zPN)5-Ot2<9(c`@%EMJ}!G$J|#r-S>j%tk)E*VPRz*r*4OHibZf;yBuRp&73#^d=Is z=sS^z=g1^`ADTZBNw&AeF(k`}a0j^z*@uSd62cCx$9ad2N4-Af%Rg=Hs%aJ6tr*`CF3DWg5_SYL+CD4r1? zayEGhv9wBy_4BR}0g=jthWa>}8dd~rKkzXx9iQP`yK{}h@DBsYEiYkQV*=V`3ZT+t zl2iF*7~vyAP;mPYl|gsw;K~uATt1lE6Whv|AZ1z4X>Ovz=rf31c7e!!{EDi(n2#hk zhoQ>%%c%LdY&h=H28W2}47P+EGdFt}?Ooo4w;2|L?vtz3&sa9Tl?F{`lf>h zgM+2UvVh2o;4Jf*L_2sF;C59j3L4U&v~(Vpt6pJ78lPvRN-W}dDg}^+g6^Qvdy2c5 zScSWn1wzRdJ?5iYONfhcC;XN>u=M{MW${Wmk!AvZWSs{IeqWSn_k>RK8b(q)+4IIGoU4u&i4)xM4Kj;$hXlzuS>7ORl7#`GQXcX8+8lh`I@0S zC4g19KoBY%)mio)PSF1$Nc~~+6Qs6rj&-al2yA+rIK@6GnC%>gUR(@AZKNv1=RGA% zzVxELwbR((vmUWF-kfVd&jKGc(EGOTSgO-W3r6W2l5(IV^LAl5>g*Q<+xvw`IetI) zw_!RQkYs^*v=$DYw%}+B{3|^zB26w;_lNAm;Y_LP5qQK@nwk1~8nVQnqTPdfupoXh zcYJF&_hkGmYpP6-eQL2KA>2O-B~oK_pFNLZ$Sfx7{I`RDZvd)LIKbLo3li^~I0XV8sTu*=ZubAe!Q zCdsUP8-P`h`M@5da*p|PQL3-Ng3g^u;c3Zma9Vf=BAx`n2E$e?7WRR@^DrWRY#t}S z%61VhDW0UBg*fw&#Sx~!ns*%iZatzXt`M|NHp6}XS*j`hH%HSh5`FD|j_-FZ!CrmE zgz6r1Qr{w&*tDyIBK4E;$T1-%Po4yfbxXkU+9EJndy4G7zk?KzUBiB}M1y>Z0NwhX z&ed602qN@4&$s0=cYUf7GEt{zl12%5puH08r@wJNW&VZr5?|q#X*E%IcmP(G%Hi1+ z%3$Ck%)Meqfki_yu};p540Pv~R$e*FzMP#0_x_oK%2Ove>IP&%!c)-T&4#@yk8q3q zCkWWkLEKql3n{xIpriIDlDuJs^JW0tz92k+I))Rteb?D(IapS z%SRRF*P!%i16+_d;Kn;D<7~PoR7O#zIv;44VCOrz@2`+njMpSfE&5Bn?oOd;H>%xdN)FIRSzgof%5inp2Npb z&H0REB1T{p=Oqj#>k+HZ@xpu!hUGufIr4A+KmacZrQGAB=!OU+!CHVFmp{b5$_wyl z@f7Q1Wg=V>rgNB|T%pkH6h#IV-aq<^{ChK?xkv_O_D8^9 z!^@bpza2t5qhMs#hn;Qf4bu*~D!CRIt8EBHX3Kgf%d?!=L;ZwDbYP6Wq&B9PhoEY=U}NWwI~ z0TO0p=$t|hTH!-?{cY*Lk)i>WRtt!TDiLf|V6`vo{!LQhUJ04IY&G2CMo>;EJUAx> z<5#i?(7AOUc^^LuO1l;k<_B7+CskiVS+*v11M6Oxs1>Hf-NINYKd+D<=Pjrw4{w8L^DdH?)CP_7^z&1t%JnpS z4i*;;Aa=wJj}}%#a(Xzyxo=6dY~zC2=_g>KQ3+Ou-_w@P2*?~AWoc`?2hFXWoJOgq z_*T~#3aX06hPyU^VUH+W33?0;W_QVei%s;|A&d~wsm86HTZrFmK0I}`bswu_B}7MO z!>d!RSo@4Aym}H(kexDIm7pk;vay-EWRQ%G1)Rkma&^R2VLtpF{f5^~Quuyj3Kj2h z6aL#B3rBBjQyhi}-f^!5h73YTi{YrcegTn?bEP9i#<(@cnWR__qHvx@1aJ zoc(PF3j^vYr-pj0uq=V7-z|+s)onp};sMAJvTTQB*^0#hHl*9U0QctB1Q_-+;NITa z4Hp_70nU=9(49E=F#m+83l+w9ynn*+TeC<#R1oL5h(W&HIE0=JgNJ`*@d}!~?x80~ z3XPv8j%C<^$G~!KV}dnwW<^tVrT5`HzYo|(NHRO!2HG7=9DI`TT&DOJ;DF9!mq^?h@{KZ7E0>mc~*gY2bFt15%!h zLeaz`d{bJH{gIgrr}mlQuu389f0TBC&-398^zT>Ui5bkSnZYFNZ-S{v(l6jScWjo5nCUzol6mEilf1m?HzC|_jy0B`&K87|q!fnazi zj6Zt~uDA*Kj&5Y{|4w^cpY&5zPB-D&-C&g5BuLMI=OL|1n{n3lE0ugriXE_~gi5Rz z;p(~kg_}(a*#$4$%Bv4eLeUk1a#oT+?N>2cIQSMGKhS|$fg@l-cdrv37(#+oA#_f^ zM6IOnke2`0R(xI6MdfrrkYzyq5XrI|3AEIUH3HEBwpWt1s zhA#hD!8#vZK<}lOGPQj~amgL}JG;;Ya^>RjajOroN^LoJ*XcN%f|_8oFqaB1C?aev z&G9b!{C4(`C>D$KLt3F;X!zbYN=A7hCa3qo9Zf!pqp=nmpPvGi6E;*ck&Z^>hsdx^ zV$7%q6P&dFgyG3>5IVc&30il6Vom65#XOC_scF$9n4%qRlc6%qWr1ykIz0o8b~T_fs{=J)R|LxpjP^r z82Y(^`F-#e2pF8E#v(Z&PxH=Q8s8%G)7livz#5&`;UKg2F}$x@1AZwFLuS?nMqSEp zV#y^Y8Z)ZJTvcP8Gm@HE(3&bj%@^U;qSxhL&I;i_ zf9$z$+f=ymF1@S+XH;0df9d!3sU(-2Fekp>5yDp#`Pi3nGw0f_COAxUfEL`@Oz#V6 z)~OXg-e)b1`79RTQ#&=`zWN3T-DZF+#W*BW+#h}a8pL?J!jkODTLqm~7F;pfW$|-E z1l+!|6>c49Wep5k5askeJV7%vw5fGO`;-@&TRR0Nhs)rF@MR>Z#>K*qyBOwlFYqs4 zEYnaW2I4Y=*t_r@<|T^)EH$(Uo4+a|92?tUWHB#ZWbuKD5AoondMEF-Z7`tMc$&*L zXAWt5_OnzpQ^0xIKJNY-1vD?ui`cu_oxy7(*8H$sc&VC$JbHehu%W#^$FOcSV9incZY)qQgP4z zZKyM?1qpl$D2sUO0cJ1isDn9jG|NJmX-n@<-gnC({nj>Em4!+3le$nReUkE?eh3BY zZ6UowfN8x&lkN%gfmU@aX{POodu%t8D%z@K>nI;yL1*HcXJ6BIJ2WG^Tae5@mj=Bb z>o~Vf3t5Tbw-}EXisSIO)!?9YgV>RmN=Pg#fs421;po--^sHNwN^kFmUAaGz>3daf zaZW$fraQ9Bx^ob(z(KgYpU%M7SF;XQ#}kz_qiv-=7pIji1b5oWEBoU#N9Er`_&DGK zAGWk$xm*4qXL^&H`ht)8r4|7yLjzc=^a~Q1DWLfW?~wmt0)FSWLkjAHbNe#!qX$Q@ zpHvc_BReo>Nk94>nNQ@)$FszXSb)11lE+uQ1!7$%oT@v5rzLt((BC?cG&J|f2*Ma(ybmXCq$Y?*QebqIC$gpZDX|E)XoR@xvX%lUZ-FatZohybbO9Lq0 zUq_%(GI^h^R2FDoY6F{#6W})34-#{qKx9KU{J9r`JD3e##Sv6K zGY!6fj==}6W`I^(9Z`Pt0fWc34+a9$DdDU0#A6Lf%<&S$kO5`G_Ojmhsz?rTJ_%tw`d5jdM^(_F{|2%uRH6Q615TxV(_QqHSz(S7RBBX1q(vE> z*Pp<*#-d1Nsi&m=3lD1c?=iZA(T3f&$UyxWfB4C%#pT-4c+p~h@Q!uCjm#zZ%ltPE zt^I`CKhvS%(jw^DYrzob-UhL|59yir3G%GLGdQTV7=wKPt7tY-0qc;4`cAVay)I;t&Uob28A&A9p=!Z}vR=0B`q?ZRn^EV~f zeK(oN=sZixYuFHK{d6~6SQY}VeL~XT=84GqYUKB;n0mdM2YX(b5u4wgFo=81wh|>n!QMO zT+|x3P|N@RKyotigwm&N-07z^5c(jDOg>%)MfKA7RAwozh+`N(71#-iu>occQKu zf8^LH#4KIv3XQWjSSR(*B5LU@4Bki3+o^zpL|zecqia|SJ;JykYZt|zqy}-tf=p`_ zIs+TG8LEsPz){ysrt5SIwmc?6YFxfVUQ`)}Hf92~xcd?0b|s*P&p)#G1iayOz8sj& znt*9WIC1L92$JEq1(_suB6de8+}A$^IybMt4VS~j4J=Ieu=gMt*BscZs6$Gi1$cpG zEwxr*7|lfcG8{+ep=2_J()B$9TemvGiFQ*Yqp+ep}$IFK++?H#FhqdkA7G7XqH2mdIb2cnnPU!%G7Sd2=d(JIjUSX znQ9xUM$2RTsSYI`)aJYfLM8&prnGZNXVWF-5BE-7mDmGT(#wchn(=qB)(RXmO2CWG z{&b(tf$X^zD8I#&OfmK+v=t+W=9@Q2iS-o__n?{;ey|$U*HjY+?mh$F*wy5^d$j+z z(4BC#IfzpebYKLf!aJP;h`ri`gi3F~XJ!Fb3*m=*Hk(M5k@t9_Qxa6!X5kbZRfcjR(KrZ(f*$!Q0uAFv|1J2-;>6dJbDuF~y~LJ#@@Fa0DwspAT|Wu_pZKxgWh+vE zXFJH!Y?JBHJiJB45egFl4t*mz?%Ip-*-kb@8y^C@-C>;hXD?yXA1N~CtUOV=Whov# zDMkHg8G}te^6ZJ9CU82k$kaRL}i3oObF(D`NQXp}=6n-%#&drOU9W@HSQ5YqPg$_M{PLm`!!R_QU z$!aokeIlSga|>KHaED*KnxK2N3tjT%!7~a{j9)2jlv21J#HZxbckGH#Fr5b{Uf6)O z+z@qtbpWHT#}alQNWeSBqT#LJB4$Sa2joG9gXBOmIA^}YrzTsWB}4+J;;B;e%{!j~=yaqGUSfbB;=VCh%G3)hvPNBkuYx$?4W z{g5IMBopX7QiVB3n)Yk=uNxM6>g9XY*$ zIia%wCNyT4@(+K2U1$ZAy}JtvgbWBWqF`8WCt63lVK*BG!$bQ3n&ZC$J4Xk@!X!J& zBh-e_^*;^oQm?ZDSE#`BPzu%dc^bT)yhp$H3*cl27WZSZ7ont*g4hOa`1Hyb5LN38 z{|(1-WW#CBWQZ|TMc&17z53|q(hp1{vY1>a@f&u^aB-K$Rw&lX#5FrlfB?G^P`4p0 zdK`*oWFz2=@_ypkZwLBb1>tLXS3yxhl=%Kjp2dihz?QpqQRgEkpku;=u_ev{tF98l z$Z!E{+nK})^?OIkDpgXw{;AA!i^TA`B7q{Yx@_twHnY~^N3(9UWlV6f8lMcUr6Hp z2z3M_QtnkQs4U=PZ_A(NaQZ{=fL#o1SooeWH+n`i5AfoMkv}-@{vXg!O#&FT!3|%h z&=9qF<0LafaBwJQ0x>#y*F|IzK4R`N5ADkX-^d*pIX7?{t%W+(9oE}@NI!l;Oi zVUTu{7jLl~WA0m<56YwGknnXkGXFvxHtPf#-UlU8d1K#Q;f`#dvsZJs+ zC&^a+D2Tb1;<#oc3JZT0#kWpXf##|zh{-H0Z(4no>9WWh3rO}t-87rp_wOzFbb}>y zKaZk1!(7#`3%cVNnw_IMM}g(hG-#Qdf$OJzshlkx^cqf*EynC>`+&Iq4| zx-@%Y*QYqL(KQdk>ECi@T|NtMujjM~dcvAx@sO!yqB5ytX&bqWEm^i*D8B2|?1Ec@$sZUII z!nV&!*n(W&> z20kz8zEocW#2A*dejJNN1zF;ZJqcNO%K8;rTf7tT(Y>kcycoDnTGG$m8#?rd=3p{d#?}7eBX~J-%g{V{j-GeenZ5Lb7mZ76j9A)gUH!p4`eTqVHsZR zVB84cLcC!DDe2}1DM|nEX?p&Da&RqxVhD7+8G?|tUtr%>Z=%k(l^9WSBO|}+QR*s+ zXs5XV*=MnxnGtdsWa~62!L^q-7oYZ{&?XLi)vclL4#bH8{ZTZ3A&h=kelu1)Zh_N7 zztP2s7ocb82zv3s@<+sj!J1&{f#T+}Nt^OeP2 zi<5Ee;%H>}=oZs&_bf~?&oa}eo}i}E|8VZ)1YG5z$(ozCQUi4Ux_s{%I-e~FdL21< zUvmw}J2t|`B|jO@ie58#cP5gGRdgSNc@SQYG~mINA~Hn!p+EveaKbXO3ds0a9N$pqzUn6ZVEJW%Fe! z;AX2BcqQx4Ty&@uUar$ZYiM`TY$F#Lj%`MZ#8!jIy;aDk(GQd_zGb@Sogwg@dtfsO zWP&pR8CIT@Ql}zRIPF5gL=>$0h>^nCt>7}^387Q8Ur&<*&mYc0%(Kr(bVV!Z;U<(n z7==H7-G&Nff++sSHRPAqeIUGjoSd!8W5}wd;`{K@@T+41(N&u4uh2@fV2#Fg>;LS_U!1R}RD(Q+P zdArr0Tq`(<6GtZ)mZ_h)Vun%>&lbY0u|U?lpRds5b0f597lk{HE8zE{Sv5_Mg?LsG1*gr?Q~;?;AL|nRfWlJ)Glb4DejMD^?F^gt|6;8@s zv@juwqJj#cXy*e)WK|%3PUn{P%vHi$1}|CsOPvh5TF0z0SLNzunt1zOyUbR3Ak7&MWBddkzsDhoOPy>w8KI;Dlvws3*J+ zh}$7=frraWt;!N6R=$5rSXFvJmFs?*nOp?uDVtmNZ9Ox`VmY+ms>Z?6!tm(gPq07f z1tm&f;J@dSkb12Vf7{GQBzhRJ4zwR(-(8VLvvT~%HUAC5xsXu&c#9QOy%%K~ImVRd z3%}wVcy*uURGN)cKo^wd)<6rrW+ksQ2OTRd>Qm=elwBl9EdP-Pi2);=Fu4rEv}!lw z<8cAJF-M#$8k~!}3OupA356Giln}=Tl*l=nk*KBq7EKG>Bwh9ubAH|V4&6sLLgq;= zIFDbUo_m`JbvlD8*qO$0dFcczPUXTT?d=f!6DaS(GstDi8)|B~)bEje;BC%TimB{4bpqfv7n6?u_mGl+je?`$B`bwC0z&I{d9W-Q;jVD9wCA)`I4!%9lWrN?rt}~ zEWwvzQ_)I?qr~{Z!({`znTW4o5=Qw_afP-3HjACY@;j0#Bde!`g+X;cN4EC{FLkpZhoBhbsX>>3z@MCp%z+(t56ZSR0<# zNdor!+eB!=AmneDLo<1c*`BWZsE1p;C}n{Itl)bt<#N?sl&Z)Ys1Z%tcd*5l+;mO= z`_ePY%@ckM`GMWA{g^6LDiuJp0SBDc2B72GfX2=zLMpwd))V4FwFw3GnGe99o`Lah z=f`{>4v^i)-$ByvdDvT#2-hR^Nz?3N(7x*i#{(?!0{RSnp-@zLP2SP?P-NjPl3=tsP{{(zF@ zvj&yRy&!Wh3@Zz?!=oiCbSHB&O#H>LBdC{N+nn&ls0^r%N+e^L(IlVY0=(BlhN7~2 zLE~~hZkZi{oldSuNox+Q`t(6rFp3o*ZVWPMk71K$I@Z~?8aKv?qn?m3RJ2u(IFq@N z(k!=xtil+$xslEiD^sY#KpbZ@n{s&tL#dwrgQUQdTxuZhCHm`0dmLzQMah+B5HDWG zeV#6g-xfL%C0nZSZASq-$CSj%CtIM$>Jco>-Uas0`k*uDB1iUj41W95gXH^G4};^M zK(_oW%VqT&5V)2}?H{Ch*y1v9c035~Osybq=(EYHAjxP9 z679O7|6VE9oR>qZrujhQs}GsI{UYShE<>%fHL&lMJHD$+q1Kt%2fqVT^)OGw5tW(Z{ zA(KCx^#&?9jm!oy77uebHYGYub@9

fqr`pXZE#0}>}m#; zqfeTpC_?BMAC4ABc$fA7Oee_T3bGi6%lkkuEQ)DA`v#wt_9V8n(EN!*Y$~S0naGt- zA~R-%VSHr{4&c%(h2#CO?#eT$*FA)`&YUGRr+M+(%T?eqaTwRUiULb=6;t^7N9s&n zCj>5ZrSu$+;-xQ=K&JW!VbC;(J}do&f*WpNz{|sm7vxfsu1ncB=4k%qPetzDSZxqp zwFaneD{|@973$T=-(gXD4ee#`AR-h*v0WAIvi9OI@b$ycam}A*Fg1bg-f~745sVcY z9MC=Q8Bl$_h|Ksf4A}xvB=cG$lKcCLk@BSg_O^wPMG=c|hN1&Jm)wWnKbFLq4YR07 zC7fgOvI~X$REHnigYb>bQcP}|Add3-$qGF!f$Td|$@&?KI6M%5|@z(SIYb zzG()YJBzS8MbEI-r)A>o(5Lw8@Iw%YO`@J%l*5~AeIY-jmz?LZ0w)(WX6E7qmd-7@ z*L{`lg+{DHR_}j;&I(W3X(faeFBZYuh!o&2zlOW%`Fn+BBzpDVCu(Bu1GIa-CsQh= zvBe18yOEa1r*^MqzHEqx^8#OiqQ7%PXKKO3#)h8Fis453`}A(94`LiB#O!*yLl=F7 z<#C3>$GioYJC-rYl7?M0zq<)5th^0BbacUIANBv-OcmpKwH3bY$$?0XM-bN#wC`7!J?!#KL1)Iq zu^2sP*L>Jdub3kE_52xFR@X)ORn^1tTn8%Zd^hK^-Y}TEq%xYXP}J!RN1!j&;STM8 zZIgRXm?VWTQf7{U*GvMq#VCW>Y%YH6Q3m#-zo|jvS~62UAM~_UxcmO&$C?FRxW)?z z(s~aOb#^oH?)XFaJycB{^@@XaiQkE?AE)t^)Dz zoe1sf^t#KhO#J0gP!cezIdFaBFmXtCF;lZCA8G^Xd*uLLsz}Qdw99*;qox=hSBk^cgee$S zrx}mya$u{$PHtmXH!Gm!F}SQMqhzBK!LT|D>t1()<<=drtlAwqW($$^DpB@@nSAg` zptBwtZCGk$Ju=9;fxPza=iU=?W_;OZ#Nv9JU^99?q9^tQylyAK{uwQ#^6fO2t)EQB z?{*}QhE}o`MIAy%uH4}A#X5sM-9KK-C(ZqpG!DzUmtoD6Htg)V{N0(C@z*#LNPrKW?10 zwP!K(NgqXtPA>RXUpQ)wIstKaE8$K|8khzx#IJWX;=}9LpxI?dND=A?>Ds&-IUim? z`L1C=rkgX)Iu`=F)Gk5%?VHR4Ui8SXNttp{`Ur0&a$q_;9NT9+EYrKzgKWoQ$k%T| zp`xOn+uT)vu|+P?lM@cYYt(4sk^%CgGb_@Wk0_U;n(UsF@#I9=i83`=XVyeu2F>m9 zho;6U%D1lqZqlxoehE6Wrl8JTIU;~2YF|NvQ#h=2pgEiar4X`^7cY2d3Lf{jlj$Nn z_}iXd=Sht10d;Ikxq?H{`d?ZE*OI5=m7oBYK*p@WK@~te)NcSi#{xh&e#BR-K=M zsZJrm-`NJ`F~4BoWidQ?(S!1IJi$dq6q~q6lYJU=H-WnyUi*BY_Eh(QSj`;k^?zE$9IWdp^YP zY6c~5(`lwhD&=@~9(`fQV7+&Xu%Hah!ku9;#WE6z&wi`8Lj~(`w}l&C$Jd4#`5zd* zUr97NEJZ48nuBkr+ql2n-_WciUS_am0g~8P303qN@4n&_GS)PcTH~;glgEl<6>k&5 zXJ2(=<85??lcxFg%s$3rLtM~xp;=gkmY5oQN%**0f>I$r7I)``XF07<7SIi=RsMrm zuNbgN{g3N;;x9a{NP|ZbX{*khLhIE@nPwJlpUJyy63~ z{n%#aeUCv{Yo8A#e|0%G^P3_6XFL|>zm7g?kgV%6U6lW5A6k69mSjt&gQf8^tWjeI zmyBzmJXR4GS&HDhCp(}w^gL{%xu{-eMmZIGPNO5Iz5%z&7yNo#QKi92YNv=FzPacJ z%@OEh1eO*c?*(Pxon>ZiLJ(<3e ztDyJfPw*L#WOypmtTP84*5~9Q=pG7(%~*iF#%VRtw($k&Pqfk52w^O+ya8GRow&yq zm4nupHZ1tvi8ZBHfm7uKgk6qO$4RSHp7;pLCLG5iPqS@J<5~7*Uc6uwBS;gM?k~iH?QYs}j#p#o5x|?Gx z70A`RpMd>d4|AxR1cqhjZa8}GA5;%U;99;A>cJ@7;*CAni!kuDH7Xz;^#NU7ou_{N=~Xbbw}H=xPS9S8Pk8plHkvJSol&Ja z3P-es;fQWD64>F7-x1QxW0${iv->hx`?U}(*s_(n6-GZlLHYRon+ITg3ZwLnhhR4$= zV*;;pC=@dB4NBGfLCx72m=31MM32J|-NnoPED``m`MhAahbwuD*#OQGIr!xx32f(@ z4{=++Le=&xD&vm^Xl4ju7mgs@JmQLUXpR6}DMhXe1&FbWO=#kKHRko~CD8CFx#^)P zX}A6=9KU;kY#GmoLr+5S$2)y!r+75nSS@|UFd?z7gNiF=! zSVbJ^xJ=nDR)m#}6TsEH$Ikgs0pe#$$ZvYmIP9A>5xYAD?$=~;Pw@xA>J{;T<5y5} z2O1!AmIo(nDFoRgDezO^C%W{z4r`7ELF$ALULSpw8=Bw^U({-;oEZ9!OgtHP8PCAW z$Bhs=k_y4|G!sOaO+MIJi;VIGiPSU7l*QFZj^W)9baB}kQZq^hj;KqpcTA+=)w>5# z)}N(NwyhmDuus97S4-Jr6>Rp_f70x0=c_0^%L&FD;YN8|DKkGc+o2rI1?+X_4$!&c zW@1-Z8t(t_0~oHVP⪇FY)zwJuP%E)n$02H-&P5Nh@wfp6!( z;DVh599K}IdV<4oHY-MbslGS)=h7VGgI+aWD=39qRc{aLSnoE< z=WQHo|DZZGc#xhUYb;@}w-|%uHAS%e`*Ga$NCEUMR^!>#10ZPrTm8SJXGreU7I1%* zj>2!B0n!G_t9Bd%o_%YG4Hf+4yEi$o&Nl@P*U7V19cY4+V>h{>uR<7E+qua2q$Kmh z_fl#Xho1i}S4N|rr?AKVE6f)w%$UlFK5(_W4h*b$@vW&?{DgNF%Ip^6*~NNnv!qm_ zDQXM#zL#Ei>-Mpl;#*Nl<4W|#Uk0X}qlkxgF5D|u<)C}JIMYY*FbG(Vz}0{cuwnZP z7Nop@xo!PqQcNgZ5?_hLMLC4s`+UNCfeVQBMo~)^ut?!&##EU7AJ!In&gAfwq9Uau zz{zPDc!kBXZh78=(C!X&ZIuaZQK|#hnH6x|s+6GJrC1AJ2j`y%4sG9r=c@I~(@IrI zvs*7f(c}P%pB2E$uR~ylZz&n+Bn8d8C@5Gb$ZWD%g@f|@;dhQWo>_FCX2R3H&~vWP z=T!^_(NBRbR}W_FKFssL1ccNC*sGt;adlN6fXZ`c((lr zQN~C{AHG*p8!Pv*kHw2|O?1V$`YLx|kM~-(*R>vo9?dphynGUz)Va_#c^C+}Y;1DF z2`PW4d2&y;!r6gq%-WnkVBhnUBXV*#Q)|!_o6Omh3&#}D@2+KVa&0YTys}I^N^5}G zcjO)YKQRVQCwT;G6;6B^+A z-BRjzLN9gh&KG1kbDmX2bLD=k_F(nHOYzYix^U=yIpY1o0qYbVHWhLW)~HXFKc;yi zU$xsnhBp+ZzAmB!dk(?7aVsi0I1Q)u*Tc+s9dR_20W-XwsB!Qp-c~A!yZ^XRl{SqO z@3IF(NQERf>wF^d`;iXfTfKsw32Zqs*d5az8zIgl%P$ubzo^v zueJX^z^x=+9PlxLdNoBMotsj4SX6+$R8Rmv2y7s8ii(P#35&sW>SA@_59YXh$5M{B z?I&m#bP*b zRDf@;w}Ix16cXvM2bZ2Cv@YL+Jd;)q&D%tvtZyZy%if8$y?X_pO~R?wp%QrP#SA%r z&JVABBMIrF;<)`qF=4d11d0{+P>Y5eSi4Q4s5ks~Dfh8?h;VgfEgqf(ihG(gGzc())9vV0$M-p|!i&tBbN^q0N` zj)^gMm1R4494#cRPI}C8q)Ex#7Yi+qVTbF+&%}IVtCS;W&4V!87Jwk%gSk{1)ey$_xH-z9Qe+*X#5{Rc{ z0hper51xGXz}{~Mnx1c=O+xSrIWX|WI)r9BE-A+ zqGefr>|E_u*yGy+?;>bdt&cAfk2EI^xrM;pbK@X*<}>FSipLwIufbLqH)8=;g zAJ#-g2xuxKf!)f3?4hy;>Z`r~GHqmN_o7P>s<}`5&Z1nnuEt3)_I3eY8KgjIWi-O7 zcso$Z`w1eWJlIjtmXVY?fHyaJgKlCFSLnaJ(E12$ysX;mZ-i>KCCx+Q8 z7^K_kAYFeyxwiTsw5YBl)TexLClAe|r@z%b*HkKA6ieW57Yw+YiTikYcMEj?nj&8YYIzaN@Q~2s9<+|;a9w%-~_3W^Ulh@4pQ9=6jeIu2c9!MK+33vw`chi`bt-cQ$=_sSCN@q~EzDd}%<6 z->>v^OuGY?{%r?}~^6x~tf!7HwgfkvY|$fZTY=zs0N zu&D-a#1|;ewM3esdy*4xdB|=c=({Zaajpd;~_ne5kP<|H{vwFehWb zjKV#UP`Jz)Uhc3}zgn9Ge(~9i^K@s$exDh|jU;HsQXnK= zQReOr6a-n8IM+^m3Pe{-GcM=ufMe;jt79#F_hP;T@5l+k3NikuD|k6_=*?qPt(!vP z=Du_;I22zM7i30%dWn1NWw_VfN9gnSBPNeh16+Q63zHk8P-`|XUa7hU)=GXX7g@TW zEw4WUyhZjzZEqOfKP*@tZEgZT#x=PSj(qS%tDTHlxe4n&$snTa-=YHHiT|VMJRGTd z|2S?FG82^$qM=F1J@=j~qNUOi8A=? zn^@9MCK)}|AEM66ivf*)V@;}v}KvkJhOxN&(4A>uHwV4ZQ+pkvX=MmkT#jr z(@12>GF|;IQ$(q472(>-xCN$`LVv9xgjK|$OwVYnDC|s}Uz0^0DmSeDQx^;#Wy`n| z(IFru7>S?ZPT+Go0a{cV7<~=_yk4E#eMto;?7qjd{h&zMdwqo$x}9*QK$P;AFGs%p9UfmQuh3U=@>MHQD?sj{d$@I;r7Uej@g_C6wlzdezo*}Q+mNZBS> zaBzUMUnPt4zuSOtX##G`Zia1H+h9CoAJHbA&g4&X&@}C)4AbaK*3@ywtck&JP!?krq2|b zcy|C;%Xfie$wxH1`Z*Y^lgD7#w2&&wg7B&iLbO0ZVi3NUkjgYy2w6EcS?nBLW6y7}EKWuoZ75nH8)KMEH@&%Cv? zNa_$W-}M9%aXvfrQWsR;41|Ve#{anO9ke`q2)|a(#kx|3FkblyETiY*bIjc;-}Vpa z%KkyTUh8+&z(6h#KX#J;O*&LvGA5|iM{*!H+#Oj?$l@KN&D8Nj%6NL|JX&wA%I?@Z3k0&b=y#HRG1B z-&ME5hqs}?C&FZ8{>4y69rE-;^B1JfsYEiWvj?@sg_7Qm%53|PP_XRpqivSGhAypV zyg?@Oyi>1@aLX=)m1)f&yh$0Y9=t}VmyAR5P&RD)tw0ppW@8E8Oql=nFD$we4;O4a zS*x{QlV|Jtc#Tcc_yXz8dTpmf z-Js4i9HtZt3Ss?QdE?LXzJo-_X5!c5`*eTSerR}=j9!bs#M$IesCmoGh8aG_szq+F ztg{-K>Mr3eeQ63M_j+Nm#ZmUPr%MsJ+8EUPJ;>t@6Ig$keWBYv zIC)u*+~0T-?iEkKCyDjgc5V*O)yD=j4EXSKl{m0JLV>YS9^SsS8huh~<$R6eCyQ6| z0q1rh=g-YKxN1)uF83P&g|i}{E@BViMNi;D{5|sirCgR=-fb*DqXn$NvruEB$^JOf zNc|O&L}v15Dd`=dkat;(y|=aeN)AZxP_5~pHe>GP-XakoBXX8whi?r{j%*$8u%EMlwJ zdXWbX*aOI?fVz!5ss1pPm=PD`u3lINBg?-7(K-*GHtZxd$EM-m`fuQNqY;Z}#uEo_ zCV{qRBj}&hXRlk)4_1T4_&E0iELZenk9&xc#3dcr9c)XQ&+dYNy>GA}!%?iuRz=c} zNvL2vFmF{C&%h9sDN;tbPA691Zfun~8nYUSn$)~P^?le={t#=Ki>|f5+ zXF4;-N^2pc*qhL}(nS3ydk>n3S|TSV9(%tT;wg9Dt6Y8a2zq>KEi|pz#AIbCy!7N0 z@73vX_T>}hoV9`{;J>t0e6*F+@2eQ`ZaH|#Jc~Y1IylntnxbW*|P>>{{4q#GXt=S^CfcT zQW349s|1?ifmmkw3L^b|EQ`uHPIcVzg6D&$*x!P3u+ozEu;11SL|saG4y)7Qg@qP6 zVJv|63M_&RJ#+A?h5C3wX+8?SwE#p6^2quXVT9R^OuV9V7KzaSYkHPKQECO{|DJ|3 zJAxtWxB!)tHx1X9OXKU=UZ_x608+)g&@EL*Tvb}a8?;KKN-l4KvqQ{z!&i;;c>fML z{(Oaho``}d=4_$0I|r6M7snauAIY@rUo3Ru7IeJsgd=|w0I`ao!+1MwD((QHxRNd^ z*@B&?TA=A&Iu9#<04c_cmDgEFEqu70{>%S?!=dkBsx<()9=$;B+SUT++RuU3pNG^7 zhB4mMyAo87+cVF&KbUtoF`i``;M*gBuRPjd^tmS-9gh5suJ}&EGiFw&{O>Ly>a`UP zZI_ zY5b2luLp!8qYWPTYx5vaZP*P0e}AW}KOaIJ2X@iyx?;3S!UW=lUh}RUmcUV=&p_zB zCbi!FAl*Gl z$mYQY{upRiZ-Ms%lGya-TR6JS4;>22rCumK1aD@3obf%Jd$6S!knsw#%ci?Z?o~U? zdo=@dhi+3+pASZOUx#` zm{|*dz0U>#Wi4*lmvoeN{~yl1(hhvwOlY%7hO*>)RO_!fpt!6Iox5iSThz-Dhv^H< zw&lZ{%mjr*CfTULY)tz(TG%fY3?K{yI#lZ&o`vkrVJ8Ctlz@eXb#T& z6ic}8k--(a8ep5WDDF%Ah91Ok0EyY{AZJy9AD4Jj`X1S=m-{HR^r{MZRJV|2rDRBL z+$f3B8xc&Me2C`s?`N0`vy59b6oM8nV~ag(u3BGv1mCv%06VsBf%O)@&<3$g?7spd z9|adu;gm3?B0?btH=SxdK{0ImP#*_XFM!bvy`USZgKmxcK}dTC{b z#%>x`FTVs4jzZKKvF#93u$1`VkV{)=grbBSgJj=BU-%Yz7h;yI2jZq$b->M)_`imy zgv~Vp>W=a%SZ)^#qdE!jBknJ(*OFr=26*861`AQQ*$s00OJpx(E|@J zDO5im&L^B72a}^)E#Q6HeU^!u2-EeyONZOe!>fXh!uNqe>>_H7y1GoEIcGoYa1CIs zy*C%HTk#SPoO1_{TNK&Vs=(!Ed`))>glQvH%#w=M#&54&f%-GIV79K6h{iIYBr-#V zdJltN`R_nl)*er8_Nz)?_>nc5c?a~aG3UCx3@SY40c<_l4<{{eqXj<~u%?;)W`1K;eN=7V%bNz@HD>sV%=j! z6EFWpK!3*zQt)d7=aJMpTb*(wyK8)$x%@Hxj|O=mFFBEyc&z zS|H&E(NK8c1?Ub6;%Dgu3OJ(!Nq^(Aj#M)c(G>+bLSs{gFADHR`YZ)i-|?fKk8uJ@-DnXCWG+v z)TBHe`smY-a^YybF2$zy5c7`&;EGRyEbE&^Fl0~y;*Y(MWLXm@W2+e6sbLKGaXmAm z@`79SGzgUD&|()PaUfbm+7E=``dMAjznTl>@$HbL`wYw)w~@Lpc@%!grQa)ufQe5B zS{5EeS_^l<)ck00*l~-awW^$FHGNcAJbWEFj}8!LZs!k z6aCD7{^sxh$bU!X;!oz1c*b>xzSKJlHO+F|(_3O$?#Bi30*AwdONIowSHBq6*;GNR zMJIMX5r|GZCBm5X0@(OP8eZO;2Yb&J;eCGt2&tE$G_gy9+x2ic`+69c-akH%O$8mm zGP)690;obhXZ6%; zoc(t{M8q3I!pJ#@X~;#_CTsD2d*Nz@>Pm__97R5yCy)IveJ4flKY>T0>IC1uLDJ>o zD9?4rbC8}_fNzHQvfGDm!^-V{kluglaMetP?HnZti}kLfB~w)!cQw!9SQfrO-_B(a z&mOu^Zv%gVyjL1s5S&5`mM}B_=`8r0-Ai3J`%Rot7=+;ZP)taL!>_F}ApO+?*N{g^cnFnlt1FLG7JVD2d{rK zRC~)J5OW;?9}OG0e(en+elq=n5fOT^##PALH=pF}yo*IH>4RNy6qq|YAj#Brmioy< z9HX!Vj*6ffo%{4Q`R>SV`1~h^ls(~#Zf{XSma4Vn6{er0$S)1nfo|-dE485V{yeZd zr~+F)ied-9F<32%;dZbhzHOC`Y>Gvx&D%UtXP^i=HA0f6`^;d+s2BIm%WWwB+7;q$ z&@E z+8f|=*Q4*}|ATg;Al!egnJAt27&cl{@Z?Y#5nF0SDgP;8fYN+8?V}F$v&#>nojsuS zt1T%qt%Uw&2UE|V-+}+6R}hai*78ah(?sTx{k(^7zf;%s_mNL7y&%7ltLUm{*O@Mo zJJUa(1G4`rE6=dDb5~#c3zywqBcqHrIR3y=BfHq9swKH|G5l1-Hr!>P*!Ui{HKo9^ zTgALFULtw_!vhGP*9wK3TIt2IV(_e1owh%A2VLN3AZI-#oS}V?JM(KEeCa$l@JB5C?skIWwk0@lUn{w(2lo;t8QxR}Z%0*$+AK9%e;cdp znZxdlVe)KRH$jYSK(Pf(H+Ah|D!X5nRv!1Gj`VK<&F`{Ylrjo>29|IpgGIi0C<@P@ z%*bMnI{fxdfha*;kc|?@i}m_ri8TY*eg+tmZauIgeWlI2A%K6MEi&(}j);wVm- zDXQu|OHo0D2l2Vc4caV&ct0;+A<<%{r#x1JI2;M;*!BpPN7g)az;GBc2A)@)9Q0r} z=j_2ZE7O53&t%!qFs(al1FGdC)Yw%)uv*%UcUgraFDGr3H%AV4N^fC}kF4U}NOD7} ztbeffzpeP1T|O)G5D$$1tmD3{XPE6J>!{(7EL^{}n-H+tPNZ(=hPu@&LFa`Ly?GLo zDGQfTJ@W-9du3x*`4unfPm(RssZN1pf*$jUuwIO+*ptjtp6?C;OJ5QYejKfN&15(S@8C4wJV?2aKyp;;$pgV6U~}&- zZF(pWe$ZU}?6Uyj{e)rIuE0>gkI5$tEu{zIA_)U!NtjZ{_^416B$p)fR$PrB?foV3 z$EdU1=8f<1$K+2$y7^D|&(E4D`YMD+mha-C^~Kc4Z5?h#v{ZFz-va0ye@32W;&^^I z0ckAa(qBF);)|14(G?wxdvAIZ+mHNYm6d-4NzDkBV{`!P6IIG|s^X#KRVI9qnIRwj z9fd4~Iaq(IELl0tkDV*!DHri!7#w;9H@C$?&%Qr&uQS8VaZ^Ea8&dHm%79tt{}T3R z(g^v2NR~`PF%lLt!`ceF;6VEaYS?KXefrg3kWkT}QnddAC5cUV)d?B!HVK1>X9Zx# zUd*^$5IM;DO+4*82aCN5u)8;lap*M@c^AhyWM~$lAYXm1NSNmbT_Jid_1TMhn*GK>dNhyZFUZh zzRW;6d-EB`IrD&>QNUgg%veuEK7-Hgb_jLMpq5U4K{GBHP_?6+>b>6w+CzP$JSBi7 zUzb6ZDNU?POr}R}@IhJA3Fz1;O>xdlLZQ-YrP({XJ@h zZUGhfro8IH$^+E%2{|lZ6N3YdBf&k{kJl_v2(3+tOrJZLoQiNlzO8K__ArQ3Gb4=s z#Gim^`g!sp)eZX-52N87`S|8}3O#We16?-bvt@W-bjU9}6qHTvQN6%9bH4&>TLvSO z==*egj{@=ZM?U21mvUqoSEx+6rEr+SvCVY1NZ*brGmP^+ma)nP9aYaY6TG3mY-Ffys!{&&9 z{drB`W}gRxtQ2r;YNl?OXM<&E2Yc76g;-?C=1PT=YuNF!Ddalt9W-2YlzedI8V+z| zyzEPt5qlmML)GwH%;)fsRLZEKcI@Ee-V9HpZC;i$9G`V$$FdVp?%Dwn153DkslJf- zXEA=VdMza|dIPT3FC}_Pqu|N35G-;kL@}bPi2+AXSZ>URKX7BXtH=Jr5$9pvy+BRU zz@iVP78KA274ukS!Xr?-Dg*UYk#wx=2I%y;g(mvb@g=!LxPMfWtuZ+Pc1;v)Y1D*Y zf2OhT&0vV*YLlN%ambh3s#wOKH&RVeH_@*EFT8(P5faZC5{ap9=zc>MY-^n*o6TyO zzulF0LDG`Lb0`GjO6%00#R!SwN$5Zg(<}W~}tOK>qay07_ z0GHXK*s4kn^Tj`a6-zL%B(&)u1I9P5qbi_lL{$iiAN*+Z#@*KO&r`eq)(LaS&p=Qe*wLJa z{WRX9;3MVuq_Tz4*0E}MWn2#dj(dq8x0Ybp*dACgXB={i$MCLcT{PN1&U6;PQ@g5W z;rwc8GV?s+RJDyoW?^q3;b#&0vo;@P?JmXt{ zJbJG;9fuaKM|MgCl_P2E114{_27l)I!U>=4MqH?K2y0QE_%TQ@z> z67O?F2DP~Q=GoO~k>6j`XQoS4hzz2OiOh`79LSlmD-b!IO{Xr6z;h+mz^dIeFRrN# z_QcG=lh=Z9NIcE?_`8}sSfovlZOQ^~A0>KKwjca5qzNfPk!vyCPAEAuy=sF9{POS* z7-Y`kPV-pkl^Am--%Wwr2S0F#+X+T_9A2Mm5J}$x^geI`O|@;KN>@sNb%GatB%%VT zI`QEZ2NAekyh58cJS5ifW?7RiGSs=MHw*(#gz4=x!Nu(bDDsj87!{S_sMo^y(~6&D z$s=j{R&pZ|e$awC)SnFpZJO}OaD?)$ic!BR1>3AT!O%>F{`GznWSkVh@>}lVRqLWa zO6(nx;QAI{8*~EeRi4D3nz4!&S!^IGsSlRWBau z%v=Wxr4Ms2^-zTO>S~w}YNkR9d0;jAh@3h96rP?fL2sN@87`zJd38Yw>aOmE0uJM< z@EC&k4=zI*-4B62!Nkr~PbkgF$2!`KW3*(FnlF2b(;vZ9A^wGs0zCMONLo`P^ED zDHjIYJ_f+X?SpXXTO;|)Qj)xrJw#1MUx%$n9>BIQe<}I2I}qit?U56eKmw53r+bq)kSTL>EAMpPttgYG(gkaKn?GhV3? z-F5fKHQR2%21j>Z_)ckDb?G9xP@2iCyhlkb1hET)jStv{?`eDkep=xsxqAf8Q6cd~_N%I%S~hvQtKvq!%Iku#;p@QYUQg z`a*nBHK6XFdCYuI);ytezo`%33t-vBf1F2Nc~~b;j_|ZHVKRqCB(4Uc^*>5sq*;$pf7QzB$kyf2uD{8uPy4BrTY`A9`|eZfmxtg#b!l2b@GcT& zC9{X)){-y=C;V)n{jh9ejv%$$q=or!|e$a}FObP_(?1ShbRGrI*TF>n)8|n9 z`6ud^UQWgwtw0V{kBKHZA1p500Kb?$aD`p~;y8W>sb9&|DTgBbt>-Feh%cx*?tT}D zUUBS{EkdVUXeGO&BR2QG_U;>M?*7fo&-n?D=Px41S{|Z`Ixpg| zPZzFq{sLuH3((!kZ|KRt%dj?W0kT&sz;~?c&{ZJ~F15Xod^3I;dCZll25L3oHr<46 zEppMab%n_1nJ_!IdNq2^QYCE(Wi-OD7sX|5ai3N$l1)ZPkUK)UQq$m>*-@ghi-~u?JfLhOU(sX=kgn#3H}G; zwQKO>y&rLA!zEZ6UjOR zct)a@`9771v?vSq66XEpM`&^HEt-Z+4U6y^H3=+yatPIyyUdS$5)e+R0JGdyaJl^u*(|wP<-Q$AuU;kiGL!*(SF6JzvwOq~mk?0*sG^k1 zWQ-DjiBmb(zrm%3Mpz+G356pmXbHm^51KINB(Mc3`2!XdWu#2C`qvVUpP!IsqhENH zXGL+~0dej$<2pZ=sS5*1;`pYy1>3*Pi*@S$Vfbf#70M~*|0y$u{-i`gh@d?rsXri> zF6#y-#RDjy;w*i6*LCPWw~KPt-AqWi#6d`)30rLWb$Uudo+#}cfd{reh-Fb9NI}K=ziHQ6J9z)P6<(0lkg{wpjtII4 zbxx8vp!z%I)TzZgsoe$s*)QbTtUD+GKT{Uzv}Zbr zNodO}e|*pWIqb|S0lIb^u4NXZGC4)o;Hd`udB7g&W#)4Nrgf;2(|3v9#T)1ovdZiw zPcw0ykUNCvwFAxc%vNcMUzpO1&)=Xuu$;E??}w0pSi0Nx z8#={s24Y&A*kAWX!0C8EHD?mYK;z|fY{MtgMkIlZ_ppP5!e>bxo)#>+`VNj`OGhSA{eefQcUdfND`z$RLiIXKU$BQG zr?NS2fp>Yr5sT@oeEpDg^b_pP?tx>G4=M8FH;6de2Bj8ZSZt9x%$)p2sBCP7>zo9} z)xLz-CRB;1oYunMiuL43f+?lBC>idnZDjUo{;-HBpcXCG7Ymfl;|i=z=ZreY=PZRCOZ))!>oKQ=0GeS?$}@^ILJR^n&>cA`^r zJ!r^XN5UJOQ01u<;KSv=(zL2i4-yf{f(DO3EbY-xI84Y9yJhWQ(fV`<4$dZKyrQvW@({e-lH3G`h}zNnQURm2c-VQ zi{700kkC4ANM98~pq?JVtM9qU7W}FQOJ@y;j^NtZz0rMGZd`6IRI zu^Ia{|5hZuT#Yg~Q$twHUkYbiNbFurfI(nC(zbriTB$l0`)2g9mJQaT^KU%S&2>Wf zy?7dyw>C$jk0+RSZx3hlF^tBGpMZqcCgQ2NP7`8N@+ z9STs^a+j6i^Ne?b?;Sjs%fQ1n&tP1jhTXhKnd^9Q5{45C>6o2kP$6iE3mT2kfls+C zg>hkQbLkP3=e>|vJIuX!R^JmiBfZDDtapLrrhirRAwFkT*#<#!!_+0#8^+1@UEZ0z(y{|K{!B#L z*NX7Sv-4nS(a8%-To0>GNaMV?!>~6-mzWhQgzDlqJgfXvc2b8s@i&#ZlU>tJO9}r5 z>5IYCIOEeky?#DO)QaJ3G12OHIX-+I3-huJrtxRXZD4)C4K|kuVgA@ybn0p}TPDOG zN1b8v&tLP{0y%Q@eun#>SnEhzxCcX(dnK;-RuOg8#qAa%9n8c%u3Fi*I!^_X$1N1Z^=fv=kq{EG5{Vi{IEAkxxg9ABHi}$k*@m- z*pVybs?}a#w6}f($E9HttL=F!Y|#5b9-p@i(E2avjM+z=HcNxqc^SMWpcNcXECNX{ zPoia~1{`c!!1#zg;Q6V=>gzip<|&)Amd=2X%nL|-w>)%hGp?R& z7=QztQ;0<6ZXDV38CEXwM2$0dL3n8=7>lUmHpc&Hm)8K&HpO^Etq~u%w~#Ht?4OS< z|4uA#Vdj57Tv73gYQlEO{v4NHnI!z3r7DuTnoA3nlH~IXPw1i_Kig zGiw=FV6Y{bbWRhJ61(umdF6z-xd0ZK9z@X+cX;b6#kpHc2{4M3Wxo;I!_g_5i*Mwz z+3kA5vF8@v|&ed-cIHYzOj5xDZMVrq-U!)?#v-pH+GSWd6 z<-_%h`^cs*oMp;JS4lyEJ{j`P zYkA&;p!j`oEBc3;{D+KkWmloGom04JMFaY}$BF#2D~9sbEn|h`6q3hHH3$jiQ&`TD zgt#jetOLd(_)TLyoS!u$J!ir|iRglL#?cHrRtUe!VEO_oS1A$ZzP{(~QI2=W3v6ZW zgKkO`!On$w@YjgpIzBTXugc|;p6TI~rgrnT!52C&K}kp;@Ev7{hs=AXz)NuBdfOCG4wfJU_Ljl! zb&1%`W`NGQcQ^~L=YP#!rGDclbxbBgzQTEsI9~CpjCF3cc!zPsQU5@ z@9-J|E$L@O7O@U*SbqV0b@ah~$s|O#i^2k_5-51ph4=m;QP)g1wO}(vrEhu-e5M!4 zYhP*PIqrmaDf3~g|6Cw4K8yGu6v|Anis=7;+pR;TMAAP!#!$c0n>&!y&JRhm{Mdd)F`Qk^gY~=$2yD9sAe zS^X)qKB%!X-rmLEbz~stb~tHgr9(HitD-*bSJcs6S!jTnW2Ft>XBiF+gTwSNDAomm z@!|`#wtykND4dR4yXI4ijs&v@$}5O<;z1b1?1{Yc`=rn8WqA25h8g0@A@%+nQ1f&F zvKh{$%)2Y7FN~vdPselmKW0zd@@g4rRuTs$Z%dh8;~O&gcP^n|8v;HS6z+*|Vaaqd zd^h>MaM`j3R_iqp9~A7tzkd>;{)gC^Z9x8JCla3Nn;=a8GQIrJFq|6g#lI#Pm-*H_ zxID%1ZF2sQW*f&~?$uTBK5`+Y`Az^ARcypJgfUbH3gK0GK0sYh!BR*Z!Z#+7@{YOW z#DYv%>K9Bpni#93q1c~-fvF|i;6K_Ltkz9 z<2#Q&F1Lql5=(2@q4oeY!9dv>TiNk17mk73!3x$4;1ZdgK z!3%Hrk&a^zssdz~43zIY!hvz54H#aaM_(*_6(C*k@lSt`TB zANqIfMK&KDK*H%Z(RjWRmHo)aWh%c2S6?x%POA#tqM1t6O?WU}zk{%$ycAb`6~xxD zm1t)s1?Mf4MLgB9%k`-16Cvq_;nOo838=k_o zplHs|GBM1`E`SBj;$WnfPwBHqkjqYHFJ<}#Uyu<%nep;yo!wpH_{44a!Q=-{B}`(M z;y`fODZ*W2B1y_37Q}5>PWRbFL&HEBu4#^7ZH&-G7g9b!sA&ys!MnM;k!of8+geb$ z`xc8ZvxNhJl@LBFz-_jX!@p!LDfPp}Sc_{5Rs#>=?12#CEpsm%?q$hk%N(M29{j`! zU8zdHX$U8JY&@t|=9yNoOhR2TC+YAf^U*NF645Z1VLKJl;8vQ34h?;P!(*K=xKAIp znO;Jb8jBHkLm%tL(m*P;UX;KY8+Z*Vb%fg@rsI=xhjaU;2@20mLRm8w@JZnl6UTC}?c#pW4-Nx?j~Q?#`v<^()^MgZ1UvqyAvXO9!;MQ;lZ(#Og68@{>dsFE zoMLhX4_S$`?>O~?T7?|0sK|$;^GI`R|I}xXu>kz* zF&^$^I1pv6r8riy6cNNi`gmv}c`x8Gx}0N7ocYagb&KtpdlAO(>J$Qh9P8lsNya;+ zrp#HzuLD}@%i;6>2)zEsN7#N(j_nMQWRaAh(Xx#aBrzQXR~U}B)@y(leLe8BH3xKk zf+>+vJ?dRc26bz$2yJ&K6MUAxr{~_?1%5(_z)CRYeAqVt+o_L)CHS!29D`unssMEJ zmIQXLoq>u?*WoAO4HJQvH7lLbRWZm3UjvsyEk%pPK`I5y1xu% zq?{$c^`?RGBN>KqB!pcGy5K?)&3h)CNW5iurNhz1on1LyH^l#&u_~gUhXBwzBDF!zww-ZAL#UNWT2;8rl08zYPdU^PYZu5u^wt~LnwF}1 zuKNgaOtLxxSLbv%M`RRrzZfyL` zQElHv{9(zXzoShhINqYE02HI{lF|gYnZZcgpZs@M7hXP-YN%OFy7sTh5ueg z@&<9#h>{kmd+i|cg4e~qX_(GS9xuTvzt`~wc&3ET&k3Hr_ff+5rwO&?>m+;`xQ(Bu zBw+u`Uy-u&K;_;wn96-2imP^pbNCWQK++?dw`aX4Y{I@ePA3R>{W&gAWLt9w9)Ptr&-9@eQE1$C-2e?;PyPUQbH?T>>iVY0yt16dCQo9(Xbj zy9eEers4(E4w)I2-EPJWCrCj{LL1Z*YIr>k#?AlrqvJ6<$Ww|N=&H3z5a@QF(kMR* zdSAcb;Z3W6y+RfG&MLD-d5J)T3YiO7xx?TWBTKa3|9t96LjbRQV;YvFCy*aA>+#+-3{Uk=pm)~M zxTfzev!@>=E-#8<9Ik$F(BM7spRhUE7uyYPuNKg&_VBYcLS)&?a~W@|B|rB4bDZ!_ zSWL3L-;rWFOhM}WW=dkoBgCD1mt!*T25kIn0;U#L+$_DbOebA~y=h$s4s0pny*WAo zjXN=*d(zl$?n+cvngiBhYX9Gtz@HVi_`7}t;rKWKCV2bk)P2G@YjQExv}0V_3l0-W zpPp<~vA)VZb@dHArxoEvVLpCp_zEn;vbA-aCEvyYvu$NGHz=8t_}uh;W+yx;Sl+4sDUfjItR>0?ZnSdD(N z@}R^q8Qcc@gI=rvR#1nXTy?!ntX9X-vKQYttA=J}4SVqh^3P?G$Crm=ci4x?rQteA zvu+}xo>q_j3eiPse}W*cUl*QJ%1ctc*Cg`#1%2F1>lzeO>magM^g{Y!S}0a^wCGWQ z3VaOq1DovM$l`<}C`|YWF|L{@IfVmgd=TW%q~56lpTo`X%h9W31ySPMZp5GapJ7RD zIIc0_9^~`B;!aDaa~!ET@M9XEfnw2m(T!i#P~}8Is`Dr0H7-rdlzZ`Yv6WFd^*dIb%RRX|UQZ*vw}Y$qdYOTb{d4Y^HN1A95i zpf+9+^$)qg<)8M4yFz7DJYza)imrvAUER>e!)l;Ho+8e5Zib}Bn`r3DCfLBS!Uq~^ zQfu3rv4vg+x$Xg8UA65tk9q{k z+X~^pcSa}8wx%->ibL%4Ky$^;QZhx?ymr0FNmv6_ORv#3`du)QUsqY|Z zRwPlZ6#>QvzLAfpy|C8~@_=B2Xj1hVwGa0E3!;|k9vF9iDa>g(4IbyCsk7avZ#EaG zHkdvI{kr!Dh<Hy`u~i}s+zX&mKa`C29*K8Jo&cWZ5x83SBSh%t zd#Iv^Ib51>5au|AfU+@0*} zcn%gstDs%yu7KLLA7Ewp3^sLhCDZ18w!64jOEmPi5^v(&W@71qwYYm0wI1t~i0JZ1 z4WFW41)aNv@F(xw#Z9~zLL@Gr&Tc%g4az2KLXutrIE_?73Y#3cPLFvfN&qXcDL?xSQ^(TSXT+*S25w&i*t3NpDtOz}U4{I`2!dE38Zb*Jm0 zi!cE5<7?xY_M>3P;k$4s-_9;`aW!#dy9P{248@ZonxN#>26FGYhY$km;Ao{iW_wN* zHC*~lXzuYPlULQ+JyrXGme=nmleIp=ytq!H;K*w5jh_tOV-MrzCC}kj`V(yH=VNH% zlMg_h0ZybIEQKxItikk^DkKkOgC;dDU~`Xprye#-1lS?$&CqVhZh#rN@l-U&+RBC8 z9Pt27TY7{4v8Td2{m$U^r>OZi=dTlI$KOYHsBt5;9CP^E^9l?#E9OM@vc(*(KSUO{ zT;Rm~g}9clyFK3O$dJmD*PyH<(r{RxsIXPw`315bjc2APt!@|DDVU>dbR(%f< zIgQ!_xoV{_x!Zm;CyXWZtA7uRQxs5FZGR}$2nKi)vSG4Ndx2Y zfC}oYz)cI#ms5q%V|pa?-u;7kbzcQFtP5UuKFKwN7f;c&7)XzDw`{h9!~yG8F%MvE>Qu%MC*@Hr3u=xrPGl!og){4^oes|D)!&8shaAp~1BWfIT7??OJ63mTRd! z1)i+|4VVr-l|c}$Pzzo=%DC^+oxw3_B>Ft}Dh!Syi0J6v zn0-v6?Ski79CR#_wBtIXm<#&6-s)Fz*DfM>#GZ*S99#g@_6Nd!kJIocuX`XLlTNUy zvl;V~hDv%j+3j!X@g{OL5(b~pb*zU8F?6m3+1S*~3giPDUGp@^k z+qZ|{Et7r`E9P_qLQ8?09ULT@I3XNvM-PX^afi_9!6{H$cfXzvRh|6=gLN%*D$v>@v z1Sk5#X?{BL?#~0;yy1{#tVtSO?*>Cs{0a0i8H|4(q2}hK!L?DRu%nUtiBX1?c>KI> zytRX#A>liB(NN=cuy)=UyU@iMkbSxrJfl`E@4R<`Q+w_Qaeei{YtT7}ziffM_ghc=DD)L&FF64UZ@QtGkF&|{gVoV&MMd(v&21?Dp@jlh zQfs8QZWHyf*dr`52?Ec4@6m;ETZp{t1Htji46s<22cHBw$jI^<`WbnIaJy&9`@M1? z+2{Rj!s2h9onzy1B5p<5^l?HXrMCAH_<0_tG1jl63BJx{>ArYP$F%x+k5 zHICq~cm^)-=JMVYtpexg_b|&K0l7WrA>q25C7L_Ek03pKA;GrSLP~}|p=wek%)asp z-hJJRvd)x)@~QsZvh)hD+?x;MQw-p|(JfT2=noe9R(7%8R`|{8P{7xziXwE`+#K~w zXzXM!C@jf^sp&^yg83%cb4)|zsvH1{wrjXQR?UVl)H=7zC*Ki26vv2GepN@dzD=aD z+YL05*bXb-g;ITF0MwpThiP6zc)VFJ?1b40q5G!nW~XL+%DSVOPe6OdPx2o+D` z;P5VLZjbpJltS(QbK2$&>iT3T{$7KcchFKzUb2sb!YjdOf1)aCS#t)ICp^c-j(<-? zS>%zcZbUu4823AyGcnrncP(x5&qhKuw=3c*{U9}8+=4n$JbZ2rN3-9L!Oo@) zfy>PvFe8RKf8&P(5&rBRGCx2L1C7pj{<#~%uv#@(xAp=aT^S8#C7rXAsJTiB zS{JZ;&Z9B6uo=kp(@<{xn~j{YB@syHQ7~*^Z-^Z!&-<5swwU9Gv$G+^B&&ptv^Ll7hJ_NkXcf`5}UkNohDk^_Ll5VAgMBE%U z{{8k!;bg}PBx}hL;ex3t!jV@`aXIDX;2ENga9>|^ytg%>J|hL(%tv4|%@k3_@l;@U zq0T+&ZGcp@zeBn9)9k9TYm_I>G>}S7X~bNfIq&Yp zbD-KU7`c`f5cjVx!BX9+_0_}eh!c8#c5zQ_s6Arqh{Y#Y!yoGgVp_?27;r!duglfV zS~C&1dr_l?=7wztM|>*b+n@^k%2aqs{nwZ@D;Y2NRc!ZlaF6U!o);l5P7jZJcbJHC zIRnqN6?uiG^I%z%In1igLiGwCgi2+5G3|-RvCC5niTkS$!dI{7;Pi7FkuW0|erMdr zZ<+>zpRztOYDlv4udabK!$doi!L{%wuL)9W#=^)R4WgtE*I}&112|W1LJp|BMl>IY zffH3es3>{`ni6BZ^SlyTyZ53L5XPKtJg55b9chn zauYoBd`jr7Jci=z=AbCnesCCXNcO*?0v;jyL|)-Te2mXC_-%4d)Y$1Hnz>C4#sxGB z-|t1RT{8e{&=BEEowD!`FE5eaBcnxs9Uee7Cjg(ZN(aij_XUTKQMgCSDd^PX1Y19l zsC&^@lj9}SqzU7-onI+k0>sRPy8Kp6fP#1U@CJWFn@&;EY2QJj1p}|6rQL7C@^?dE@N;V4u&{HOqQi3B$S;xP z6CwDFnk@3_(eBuL6)#ea+6U;@u_$UB-j)2N|CShdutBJLC=0D)?}9J!AHeCHy>R&` zKk&6VNS*Ug1T&1^fwyTO_JrCyH_Y`uc$!|p!qYtP;UiW;Z$o_%F^(EDv40ENB1H&s zTZ|8@d`irs)(@x0MWBh?TO8YE$KmDP0Bl2UHc@w`1Ue6mM7MgC5@U1Ti$*7#QX|Rd z2)6M+{Og`?!Y3h($mI7U*hH=6Pgl<1Tpz253^JqeNkt_bvq!I>cwP>=6?F|vd4a^t zX+ms6up*jMeTn?^H4joNsJ*+t2f-MRfP?gk@jfA)nQ%BUbg2lYB7AT%t#g2@nVbT4)+ z+NYkStwL|5^nzF+p&sUcS_p}C39Il1N_>Cf03zfLuJ&r?|Wq;V? znGDmEsMdYF08&$ofvdL;GESdEzW53FY-^ISFTp!m~h zVbT0NbSP>Tr_YjGSoq0{SXs^;&dt|nfx9>sj238eO?Q3-i!cL`V)!6Ti3ozrZ=FR7 z7yH9&t%ba%^B;xi#YQk3bP2n(ZeVsO1QOS5p27Gz+N6DdTPXbflc>ux1h+IE_j2Yo zcz60H+B9wsHLvU=>P5^EO;Sn+e!?2OXPPD$hu%jMYl1oL4dI)pt~H-{7Uyf5#W zmKwV7-~}4F>aL|qh!$+%e zGqN);z}%G_Vj4j0)7=8oMd$GRo@H?V;8863l@4!&sR3lK?#cBiA59*J+(7Vm8)4{6 zS7h6}7S6e7qG>5z&@O6yR(|SA;?}?wB2KsKklE!g1lM23Z(d)Hb>4Og?`KNwxtq-= z!UYwutJ)a1-dhHj^!Iay=j0*p<7EW9PXl3T6N&9l^(Fpn4B}QBCZMj1L*V4POZb(S z6OjEzL(KT;7Fb`R3Gd`uJYO?w6C)Tn?VavbVfYp zJh~q9`Fjp8ZkP+$RBLi~%NcZTS2!$D(S~%Lukgj0+H1vIgSQVKEP7b=8qH5BAji#0 zrOrv*D2i*~gXZxC*rx0h7Dn(3 z&qN<}Q;EtlO$c9LV_UaX1FQOSQnbiK05%!Ju)_v>A-ZHKc*YJRj%6?BY8Om{Ux8Qc zrloPfu=X?b8vhHcd)o)Rg;5Z;!Vd3e!6AqC@*u~ZC?ZdN=283aD2saiItocQw?lF0 zMBX{itB|2l1J>gXqxiAoi4U>Y;S_bgiGt%RVGp}v!qD+8IBs|hrx#HBnGOrbH(gr^ zYWHVyTz2^4BUd}z3z$m89utVN^=ui;o7D8xQSMglV|V&ipKkzNfo z*FJ**^^8$(S%!u;w!oW{H&O1_W!U+pYS>82TX@2oO2Yn>6L0xy4zV@rGi1HL4UgS@ zVQZZZ1P}X)pN11i-SwtONoQg9hu@>wXGfifns?Or|Fe^j-{+^DGjAL0zWAM3?p1+Q zloVia<}{))(*-Lt-w!u-gkZMUI-}h4nb^wH4KQh)JGwbJ9kvv-zy%+7{Gm+_ocq-e zuYTPXwv<%C@K15@5$HH>&n^0bQ6KqPxQSohT>YIfK8E32G6Gi7|zk@CVG|(99CA{}vsWD80C_<;^7K*v!3b)j2L0I|`%6g^% z=h6vmnNcS=7^{xft>g-$bS=no^BkdB{V<{LzA1Lm4+Kzm{U~);l!D6UHLzQ2G4|b5 zAJ5@eL3B@V=sR7Be5k;}KK9Fp;8Ovd^`oi1(5dfaI@G)RC+s3jbsx$*H-0U+tk_C$ z3Nld9uR@p?^cpqICmTK-7@Cu;wZd`d`j-r=tt_3fJ?z=XWFV?{OTFTc2UXk>?!V z;rm}{9fzi@4bd zt#InXUA}9Y-J?7GA>NAz&nEwdhl8r1ap?tQTE<+!^h+1fz-xcz?&vBD1TP!TgL5uXNW2_|x?UwKu>B@VUJPf~PMbIM1Wt zb&;AV)vyRgyBL5^-(G07X1MTRJ_|TmQDjzqFRXwl7fmnC0l(;%u;x}VOi(!tyH#}2 zDBq>jdZkIc1=9-1kx#XKyWcYf8_vy!2M;^*`q_*on)ml4_B({2wR?WT zp&SJ$c3lI76}2#X!CN9L=sJuztcuL$o(0vst9F+Slu-q>{_b0|Ij(c92I|(Ppe~tS zurA2iw*STN!eJ|HvfoGE0`@^?Vrq^l`NJ#+%$zk*;tCaTN>oB0YDzfwO`oAcfe$CO zkD_ScV0-*>MTKzkqj223QJuio)S})Mx+wWx8l3Ix0xRF&0`oUrMD-MAQqpQpq;Mb^ zID`vUE9)8;Gq0Vb9#+Pjk$NiQXiQ*jH zMWw?2FtWN|cAu>y!Ct2YDyjX`*Cl(w&oL|ENG3*VT}0TPtbAhDLq5EFRR*lBIHBY| z2G$=-f{Ok6#F%3RBCYr)NFNmkR!h`)mbgw&fHlK6w#wjkI=Wwwb*BkZoJI*&A7iAf+cfSQ~l{k z%-ceUGOVMBC-Jw44PHXbLi7=KnrMlt@2a8ABel@Xq~4^7AHb97E0C^|Xji9V4>xiw zcnUGE&~m?h#O+F3uIlM%U{ULluM`-NwnfGGu<8T&jbUd*Zw)xm<8U|86N6*W!!i+j zx~719c}@|1Z}B6PHd-NP5D_&u%7|?bVqwq{Mf8#-0MkpIkc+Y@T6?q@c0ayp*T3g` zP+4yRQ)B1aT|^PKf_|we4NefJ`0qGhFH--%Oz)B*!dGygTEBBMemn|VSxlWX76Te5 z%Zb09(QqvKituIa6;$5IAIgT3`0|(s_|)b~FgzTHFWq+m>K;U(DLE$CoRK=9*-6FoMDLz*4;&Vz0I(geTi_5 zqQkT961=o}{CTH%>io+Ha@fF=_-D>?3@e<(duAVPdta*r&h)xWR!&kBPRDj)8_h6e zGFJh5m?*&OrROpIGcSpyzPC7Qjy)ur?~MZm&TP24$O^Z&xDUFC)x^*qSMjWb7qC9? zFFBQS4JMlx5L56N2=0Fo-)VD(K&KWEX>rt+9qf7K8L7Q2XkCkGL%6{fgv9-sJjBoh@;lX{h??aWEdobtEWFeYwD*ay^M0R&C#f|Qdo)a zjp;)Un{YxDo_h&8GWo?{#wDm({zeJ#m_U-qFluVcZuQxxKjrq(&^d4|n9 zg^>;MUP3t2VkdZ`M||Ab57Y;(Mi9itd~cS+%)-<7s$B^p_u_Co*%z7jsDR`oGf~*dK+#RrV&VA(OOeUgNa2|GB2?Xc z63h-ff{WCC&`l2x6K57Bl0C-wK+)7=pwl}KZ5o*fONL(u4NeIjVTHlPPy5L2$%m0! z@7`Sd>}nX}8AF|gt^uNPHE7CYZQj%~4x+`;6(auwYlVB(c;K6DSAf-7$GM^;nObbBH%(A@Swc8{&ShEtq5+fECTEKn~MI)thF1%& z3ahLV$e-J53nNE)m^CJB$xv6SB%Nl|ddHp~6;9 z?oS-*4vgTf?9y8ll{^>|&jr)7O@Wn0X`I|5>@H8kAvRuBxk@dF``^`7( zCA@p|*I{HYmeXX70p8@!Pvwhf+KZ3Md+58$Uro5Qu&Ab0FN4ePJ`fkfTM z2s>@V3HJ4XqUCHnQK$>SE4ZS&I~2KNCJf;QZrcr(MY?D~e`^p<;-dp7Vk5lSE7Y} zxb{YR6TFCiR*{^OWij{`uf5uCc4C^MbCiilU9%^I`qfOR&S-6h3WjCQn>E zLA(KHj^}JZ{?>nBMs>VUQ}+Ym{Gb5(G#=ra-p@wigR8-HsV=EFC)h=HH zq;ZJs6UIj;dyb;cH?JVBHXDHL^TV+G)l6#r{Z~=$h-0L=N~B%Pnlo6qZKx>W>S;*m zsRG`_7ZEQ=1KEEW41+J2q1hk%6SJr>{gEr@gKG_SwpVs6HWw^lUFcNU`mL1u{u_cv zk?zw)HqVy7N9bW@6AQ zryCmX`iK3XDvlf7qK{U&bVl>f3_}VQL@-6S0e4dG0fK=V_~tJ+P(QOn*oDC) zXe3v_j?MxwQcSV)m~k6lzyGLc(iTIJpkxs2@U!Q=c*BRO37*92S&KwHv)&Oun_olm zFr^yR9P0&JjkJO+EGpX~rdxjEW{yk9GY7G>4|2G-6vVh1E zP&W0`hS!1Xh=RarpuMD!8@u2Q*p53w-qfMiyT15KbV@;R_6whAPSrx|zh&dGhlb&? ze)mPEZ)l0O{B6i`vq%+Pd!qtphc^&Tp;yTpTbl?CO%kks??ofZFX%S`quS;{PN9E zdz{*5F**&VI@!SM_krlQ>q7z$(L$%Eslay66quBm4Wr-^D!#IU2v@y{D?cW1ok51& zSA8anUb_5*C6%Gb08*&F>oIYfGzFiArMypqo>xOr-M(zo{&+nnVL6ywCe%bRg_3ND1_iuw>0rj-Sb3Qy$(Q^zV^Pl}9IG(PsA$$T6 zv+NP3o<5rB_If0|e$2=GcPXG^>iutXfIc!#RVMD=iNPAbrQ?m%c~(6feh~TUPvO>n z735-&1xl=`#IN}Wp<(M+*s~}XZRn~f?3<)uXPJD~_1`lTPoQVErs}Jb-$aGo_x%7e@i!Fs=)V zo%aTGcicf^>ne$LPp(6BRv$Q7|5o@7>7vkaov>fjzUgPK2r=QF&gfl>A7^Vw4REiI zz|Y*trj}nkMFTe%a7y2X!m_Yiw%4h>!H%f^h25Ri&~)l7ALGxXp?u#r;OIqx%{*PO zs{YDJ(Mv=JS9IsziRF-m%`eFJD@m|@yc0Lv86)iFW@oo>qz2kj%;Ii8ZBDqH>q4Ea zOP%dPeV@Gi%))kUl`h%)2p^v6KZm;HC8EhayFj5GBIoF)qanL%ApB2P(9ZM1mTS$% zdfzp)o4q)k{L_TvwTmo8Gmr0t-Gge#h~uZ=aMyS^b#fGVbiW1Nb*XuZ>V;_VI4{_? z@(58oi6F-GP{6`1f56N^+u#RB1s&Sz3gpLh+)w2#RIwYVYmY^5>;y1A-yfdzKSLBv<#D;x8K0(2p-8urGB@DE zK(e9MPQ-mCfbz;Dy!-vn*esJ6YM!Se5v_I>9{>0X7Ao3E`$h(-KIdcKC)U%!a#{Vu_ByC68a=>t47HUrP-Wnf=!$lKa^H2(4FT-<1b zmuS{vPps_T7<4n>J_PGKa^}>W!lPymM#HCH!4@7#LcM3^6VEyYf|XVjHegsMqN`3Q zcxkH$1CGZN;a6fI_|Ge{SFsE6WL*}#`)0#Cw#|pgkC=%kyL3Xsu3W;(oi4$kA)Q5w zcML?Q@&XC&z+Y(4oETx&pnH(4pbli5Rn~Tv31W3EKw95^5uPg1b}I=U{&BGyHzY~{ zKe5mOhL?)a`3(ue9}|Nymem+yd7h(i;{Z0ksNkbeX>SS4+xZ$1i*?bL9_pCm_%yqi z=R>h(9|W`1U!mEdsnC*cg!x5MK4}V|-a)iPDVsDzhp$~GkNFQ4O?wESkTXE^v62P$ zhxTD3LgUf*dIvDyY6|gt(oh|>uL!SJ8@+q{fH?mq18%hh6LYJ4pnujC0)EZ{lW zTm?DTYQZXYl#8E@xTs23XOJ8cIxFPlc3XL6r;4Mk#Q8JW#VmQ{L+=K$ygZk(?e~V* z%-*EM7t?s}oRcY~J$&MoKpO8zn0KU>#t_rqygQ%f)p2-6>o4VhTe7Rem@CIlV5e-= zVT*Z_Jb=+liX{yP&nM*C@3<|1mi>X7N@GarrD4j(EdI>rb?oooKG@XVJ4fqP(Bj0% zah*9oOG?|3@E(6IWB;MM%W2-2E_^0WjG+~{sl%9tThs(b`;BPb)uN1O{p9KKIj*x@ z`|rHart^bX?`3q5V$AEMCM?~ez@EC~J^Qlzs0?xbQ}=C*uh#x)Y*(ffqjgx*@`5Ez zpM9QA+i9@bk--y}B^a;lFy_($jx2ZIJ~Y46rE0l0h%qCUJ&}`Z`Qu>b4|HmySDi%aYMs zRB)-o_FFx6VSATq()6EAmD}Z0>D7&{=eD{fwn*-K*MWJoZEee+BpdU;5BtgLe8P}@ z@9%5c*3cVA7@H&S9Bfx+TP$(9-_PF+{fX6E>G;g)-K1rhO9`0x&<43U!>1AQ| zmTu?Lv^2cFZZ@sYbjRyVpC&C6haV0Z2ySojw(*;`fju#cL(2*CpTJ;kFR*3oY0cMO zJT2rnBS&bFO7q4fNcwLnPyC9*9mX^rs?K(@>rc~1O})nGCB>44*VmiL_*j}hOs7Y> zIBw^!B)5H3!8Cr}tTpVu#eS@yz57^V+N@_`bSRELcJOQ0=i>6V*U55chTe$Kp!L=g z?WRL1ZOt6h4%fe{>j>M({&uwPEIV}uTZ$zOZ>^jr*M3!yJTvbamoYX;dBowt+~@pp zIsMu7!ymEman7`!t31uDr8FFty<9$NdD}2;&sfH`<3E~dykVnKWOSQK%wTBR^0!TM z_`TR$GlH*+^P?gzwrjs*=Ml7z+VXRG?`d1^udb47M`ov7mb%+)cKmn~cBrKV-M;ew z=+JtLu{bmPwkxx(EOGk!Mia)i5f>V0oc)m+G8l?a)EOFi7_IXXcI&aH=R450`8dsy z!D&smc4m}Z`&=-|`^czoh^f=od=87A^PA5Fup3uvvWMf6et*&GrV1`69ob){FNKp#9~AwO{{BW@ykpYK!$C`~+K^ zH)T@~I&541?(rqTV71NsuB8^Xyj{_>-NUrJ7#r+oRCiccG4?8AJwxC0LW$P9-yV|5 z6Vv2h{KE8+;`D!AE-8K4Ch=-fJAagwAhBRyrg{#npxaAuS5 zU|V}f3zq2BQ`&xajZj9vdJ6}+xUFThjt6?~VRY2rbC|~ClGZX9Iy@hSrZum;IJx)~ zJF-5AEiQB2&XvJwO}}+RCz_^po;c3ZJ5B4bq2z|#^1&1J1$VM%P+jYLmLmHYeUHSN z^rCsh{Ntw#?y#<6dZ&1TpFjhN@Zu5U#t=^--)(02FOr7wfi zn$P`4qTI5tbMDf8+VF0B<(6~Ce6cnWu3#4(n`d*OX*5l1-E%EtzvWm@T6Sw${}n$N z7{b_gsir&4yY+QHCQr;;9iS$Y{x3Y6@HF<0~@S)x{$vfTn4S<(+3t^YqW^ zE4Mt6{m3R{dndN{SBK2<+nZQo{>zasXg`SKHrm}gtgD#5_u+bm-ll&kExYBcX<<5O&{2>oVnitnEoV~Qj zho+y?B4kPP+lI;0R?F?*h-*bO@2|#OxxC9S+OjHb)A`q8kFnodb*62tof*d1{|UM? z_Oz8LFP$-@iSgHTI|*;;^+ODvm}h-Jj}Bu#d3lcYcEudJ?vnmw)Ab`yZqRA@^=pxF zk(O;{aFWJo%e&*|AihebDSK?vBi5Y1DzskNMwYbC#I!35Lm7M8^2$q(J*Lgr7CKCq z=FM=H{r_{i)UCsqPahbvc@w>D#j<9;p1@$Yl|8g@fn4A8V<*wFZvMW*Zp$N1Pr7qd z@F76Yc1+4A*3~~cW5jtM6fBvxGHKs;#uwtU|8+&xPGoo|S834t&dknX^2Ge^nSmX) z|07&xhaH$m+s{&KZkO*D_ED~VBG**fFX4l($hAZ7GB2y*IGf$?p@(h5tFbi3d_Bbs zsh!@cl6M$s`hSHx*^>KBOjkBk@}7UU+Db<63!l!&r2h-E>cJRW>wYh3|9!N~ZI^HD zp{um+ZLvc7Mbh@qTDd`PIcw_u%w?9dSQB>Cv8>Na#(iv`Oa5<)vCKLZ()UkW-1gET z(;hQzwB%?ot#2+*@*W`NnZ9)>LnF;=FEklGntkQZdAj}u<3F`)e_m#SOdFHp6=nCK zd2~RL5cQJ--;Z@X0@$}wEv`d|0_IkYd+J?bkyBx*i8M7NRFdE&U1es5X_N4-fhbs(j~42;-!)3&iIaB30qDu}Vmkh5+B+HF->d8R{qOC$n!T!N25p;hl1;n!O`_3V zx%R1D*ht%-JMO&9JBZYNE2Tv?F6MjK;ghOadcT0iuw6EaY5#U_`Y>@R&wqwZF^v6d z`y8j^*6;4ITl1*ucYI9g{WMRuae^z&AO5y|za^+Fm60Kp=QC^#-FC(`$$$QJIoqAx zXxq#V3w~~M&&QFr|L;8x%_GK$(HltHAjSJ%;ZkEerj9}n{-AO0eBUnP!)ql5be;S! zEOB}1w@WOIcR<^A;z2>Xx-4>iE7$&&UG~sAkV|Z6+uF+9ePAZ5=Gr>G=+7f|hKZzJ zruI)^+KNEag0ZJ9ue`K-^AU#r^Q1&t-yabXGWLIp=*7^;!)Tp1@#I>8Pt+nBCs-R} z^lD8bj?dpc)wXS(IDJ>)34QN@em){Q&l)?v*ruH{I9m_J8N6%m4p3vb;0R zYg+wRZv6-MJjPd^t;s$*HHS5!ci|1Oja@v4GxiU(w(qd6V%{ZZ>Y27Z`19OMah`?2 zTDCZpUP~h{cNo*$CYj}RDTC%O)gl>eDV8)`*IXqdLz@k1WvUz3(X_&SCJgO*Y?wfb)sgVElaQgmcCjnnp6ZM8nLIH@;_zJ{&!QXcPkDNHeXx|<{f9jPgZG-zC9$i0DoYvtT z|2nz$bsbx6Ev9>8T`8CMM)6Y{Kb2=00X_YJXLFPHuQ#=oByMl<)$)&f7;&f1np8T$|Jis*3t2FfIGUe%UcXMZ^ zZlAetpzS>VG)~6eNbV__v^*SXp2_dEtX*e!(K>W}f2dtu0(wg7S;{93Timp1yJt=L zONY{YX{fhhlwj|hXdBPPN7&=plK+SnM+A)hkbbYjx{5KMKYPN^*A__ZUgIzOAL99< ze?F4x(O&4-_$cH1#~51o{Nq-PUTx{0_8Tm>{ZG24OSc{0P8D+PcbQ=)up1K1_sKD1 z`#5x^+kfOzNqZB^DB4lkuFu=*AWp9=7{}Cqrjq2p@Mdn5jNZOv`~TzHVm{t-pY>t6 zWNbHpRVUMrwx(Uba1s5TyLFy8p3uDqZMT+*sa!iUZ;udcJYScQtYF9P7JZ!7^L=DJ z^PXjv--)Tuw%GF0vD>cDJ`|Vf?ed^`N4@ov@qL%j=yvUBi|N+yFk5wkvzcJb8xa{1cgNYm&4Ixy|u!YYN< zH?_EnjPKvvn8wgZ`PvIz+}E@AJRU*YX67&J$JRWd>GH?_9NB~D_Bbzfid;LQ_Xn_Y z@CX}z@+h{Ztr~52sd^IKX2kgFLeqAA-desizGJ&&&X+iSv0ECA_c~KD1|_92&X~pj=pRSLQT>B;zDAM|8U;jzBgVsFa_@6#Ge7El&?0CiQ z>}o$d8l%S0f$2+lbKN_9uZr=;%ur_J?Eki$mfhRCnaLB=WY~ASPf^!e&wg8&N!MX$ zUHjwa$3$gx8xiBTzH06z(D`eZsmU$>aMw$qvb(pyV1SwJ*fh!e^~xbDXnA6spt(jJ zZmVMY?6BFi&Ej}itYn;b#=ccDvd=v~%Fu}E{_E0S62V^gGK}V5-w;T*CwcTn^~#LS zt;==3NxpwR70UlU6gX-*-`BUR?G6nS_P7D@bp1cE>c-U9pjUU>mDyU3H2$|*GUh8z zFBx`$*2zNiAag&7^Q*`0?y&v0GIQ9slX7YMcWBO}^%G;Xy)GI{o^x&UZ`df&JS#nl zXpFYyRyHPl&4Tf4*7}3&9qTM;TMHxmF>U_po&g=!Rg7)6W;H{<^r+<7k-c}PjO>1K z3mKZW`m{~k4qjw)dgL^kKe0?j2B$S`!?%5M%U;Zt%=_z6_*<^;>f(Odq;vXZ7IbQ2 zE${b`wm)TUFw>U2ul@h}zZ>PxX$-M$g1lvP+*j0F#$FM>hM{R~gS`09(+=#N4}IwV zXws6uj9zVNFNV0wwcmH4Io+10@TSNuA0BHiID)7i`j$R>!TupEF)r6tmC;W*{Sng^ z+RBud-t}oI(_g46XV5qo2d6N2VxHLB4`uuz4@a7}eOx8$z;u0@e&I(g23uM#4Yk(m z(a#8JS{nBFI+>PzZ`WcO9i-(WX6Lie>Pv$9A=6oT+sbHPc|5abYz~i*{dOhgYcDJ) z{5N0zT2eQbn9gwPWSRT_qAB*2rz50OtRG6x*kkk3X+H-G#xQcat&Z(*U5Rm{2drf3h?OQ8Pfnmt zylE}dXK2UgKPUGVu&u^O-k(nI{m`!8K0Bq#wO{$uAljBSrmy6dub*_0wfqA$|L4b9 zR>NwEkJxVi{%3xu8UFvWzid=3L*Law^6l|-!E+h=2RIOne_Q)SUVP}9WP#d*3v|2m zK5WMLwk_?@OV8!nclM`b&X|8M$rx%|zT|^R{LThFtwVyIuzT(CrS^k($vK}eqezJJAd;_Kb`uL4QzfB*Wk?QKs5^`-UMG;DLbe95N@<=VGzhzEVogq59_ zTmCq5mLUGia+a0-3ihXSYv^{0m;4)Nw3{%pU77NAg^@BCLuYq< z{7)+(CbP*$(*J$Csm)-wmCbpSMC;x*f7_y0bbX`=v*ngwS(cM|Xlfdpc)OcT&P=9h z55HAn-v6&!ZeZF%TWophwYlkxp9im#%zt{1=4KLA=55} zWlCkV4)2}fF!UuBlJAuxv%1RI8)RO_(6r{27yqi>W%RZ?`;xℑcvgW3R;jc2hETSNkKk-S*$8C;4{O7OSyhJuRzr;3-;{ zw&mOV=h_^c=E$ykV$T{?(S^1(?(tcs&wrURNk;d!{B6@+4-_%2&u0tNlNSn8mbFl@p!l`1HBmWPDg$Z_d!P)wgY0EA)>|h=b()zZ3r_(=OZ6jeI_e z_EFn>e0MBui&}u9-13F#h?P^F%(ghx%u4rjr1fmxT*LJLu2BE~vVUg89>z9odm)We zcKn5m-UGXRWBl7z-?r(RGev^2DHrLw4P=|hw4>IvUj7<#>p$0Ahvwl8wUS$|H0}xi z(w705x5J;aB77zIP`#3ok=1*oB)mVp?CfP#KKs<1P%%e_@Hs zrk*~|o}0dl)+fqsJdG!hCvwbVx%R0akvuarmwl64u4k0Ux8OUo_HtjcJ~x`ta#RJ9 zc|c-0!rU1h)>TZ;nJpRf6~{lRY@>OXrFzL=yfSP5f2^3V^<^U5$F6o?NApLF*l69l ztabdMxvAXtJ*46<-3G6j4VI|`Des$z$?Wx=dF)WdTkKKQl6CgC)1EW!v()IAjP6pt z_QKLkjA{QLGr!U}w-f7_{#49o(OfTA7kQYQeh2d(-s?l#Ht0)OySfY+B$*@G7DH{u zZd%rauUxrpXI6lswMBg|_L_|D?DVTHG_6y^X2#D;Z8h4J*;bY~?ND}rq1Qj0K;t+V z?2xf{-SYOwaoS?;wf!kbHx$zL2OXQpV7H}H%U6+G|K4_z`AC;)Z?oI-h|_Zo;sm1) zb!LCI60-A8OV&N@2lPAu9@P1Q=F?~C03)FnVX?~>%Z~0<= zc~@-kXGZ7N<;g4DY2LlF9>^_UylbI=Gczn>pi(-Um|R6arxf2r(xDh!U0R@z9`4uthS zF5^Q#wyZzo;Yjm1H~6fgzL)5Gn>s*)!IqXw!@4QSa_t+lM=}Pr&0c>0STtmzVE=|I ztbQ&jnG2mB(D#?C7s1%Cu*|w$nNr!U!@W;~8Gps}G^2U(AOD_-Vp;(XGX7}I-!`rx zyun)B)R)$$--h=8i*>Q?Cl@nRAbFM@8SeGZrdC4q501|~INXz8p}&E>V5&F$oI0j_ z*iuT8HN#RauC%;uIDPUw%i>kD3f1T2%>o!3{TL!1K z>~7Dy(KM~|#Bsgc+qA6DDze|8rE~@s+XOogB(tne4HJYfPNnga>Gd(2V|8KAcF*da> z-!fjrx6I98H_ZxT^9>|zJ+O-c^ZzIILLy^NYrgj49?I(w;5vhT;L-{;${ zi%8Agw2xZrB98wkh@tbJ%&C)We?p%h{5yu1Z1$}3W}9U%rR{hAe2!@=W^OY(TvuXw zU3>o^2E?Brhcto0|R({eHw8PGi9@}Y-M|G(;g=UHFI&+Lzqaqlfr zhBU4iPke>z8@6kQgyP@tS;1%4lEVe`{vR1V`!Ra`qmabZg1XZ6+%~^9RF&4pSa^}q zt2NEjW)5qOVG#Rm)oqsFrt>tuPvL1sUz36p9oAKhtHM`g=tuMJ((wkT(6rVvpY4*o z7qrf6FCMTl%jQV&0J{IvUfIN8x24y==fiHBCr)c0Q=oZ1jxv>7KKQ$b0REg2s1BOV z4mK^w5cAw|myDT-WA%BA zzD-WAX`jfGDK{_cIMZLaA00va&M9s>9ZKn6tdZ1-H2q(p>QF!S{oC=h?&Z-BWpG;a z9jG}%w~^L);&^SLWd2LT5;kK~YnpmnpUej@ZL_XyNnt-%WzqP_Pyfx867;{*;kpvz zZrSv2{@;w^L-hUh_;{pD-}AXYKBlz}{}tDIaZS)<`h>Qv!1iss_OCj)f%Z{btj9e! z(E9&$lK*_T_WQ3OLH7tBHyUoc)K!zFHCZOdtaV;{@pW2OeDj`eH2;9(YueQX-*Qeyh7@n?e#!jx1?%L`56$>`oVD%k z9=3K5#Hx?!PW!N+ayZlftsQ2~_(F>LUtxJkFr#DV0Ao51uXr-L#jLdJFs9k-1+2fn z6=;6%Ao+DD)@Nf&uH5!NXiyo=TQuFDwymx1xdvMVaVNF2&L8t(8`?>pA){=EGPIh> zUplO-7`ra!0b~Cqr5|)#xX@X4?b)g2dQ2N>t54hXo92-=)(y948`gShGuUnEruAxO zbZ%Yt>6fHmJhXMV-13~qQJIUwKk#$Y*4VmqA?bQQnV8PJ|E-@tyTiJQ<#zA$@BJre z;Xt7{Z>Fv6w+Dw$|N01V`F~xR+m&paZak&yK+j6EUHive9VoZ{^Q(^0zPrDgkZXUS z(GP0+17W>0*ud6a!;q%^JNVzX@&_mWjc5Fqp3<`AdM6lv9k6wx<9DiM_xsXZ{BMj{ z%GX|)&=exbd^(@jebCMJpT+rPlwA96+*GE0cR*ME`=Qo?a+{NN0jw<>Cb0ihAEoPG zU%ig$uOFoSr2Fvp^lO{nH1!ChNA0IX+Wtj-B=f+ev>Q||2&D1<6>k0!#LC(rnS*zv zp|oATRn5~Gxveob6^x?oztw0mQ0q)iV2UC;owKuEWn`T~>)_^xtb< z2d#fgcaI12*H8XB&@RG0EIq<6LG0tj;y2|Q=;xf#z^;9H%V7VXoo!1iCygPUpDr(1 zqlu514RFrz!bxp7_oI)@m_DbkppOFZwY!L74x|6yvU6ztKk`Qf(0dMOg+58!yGmay zdNLHc%Mg0_E~bgbQyI@;}H42W(ttsuk0{r&+#VbNd7nM?hAPcIi}Ro zsRxpih~L8ot$RxABPN@*5&uStiqp5|HE8Rm*9S>95PXj1ny86-ZdEit(N`pd4iQ2Z@bBid0Fl{+OJ`R5R=PmKmZ<= zr(R1l#I*h`@W=PspBNj`w%wcu%Hs1^)R8>T4bpWF>>l4I!a)bW4LDf%`O=}WC z%(1eZbvW>Ik^#OCPVh`wS4CkQYJgn4KboT|is?QHX-} z9qg!*IaV0{qT+fe+o72PWOv+@;9)%BODko@oRPEyMLPl!OiuCK8DJllr(Q4FJ8Atl zRQ3e2vHblgRR$}+>WC5Fop%BC*Nd>%iR=Jz>XD@id5_I@uceb?q6H-tLEl!s3I(#~ zqI*CZCO0AzpBFG;NYfwnG_g+y;x(L;w=!2gC5#~A;Cj=3Zq1jZ*`If(!B9*VT}jw^MDltXC z_91-?Ag_N{+1KhZoojggfUQ_Qq~;IEcPQK^wf!eY{9FXa^S7%adQ}xqWCG-Gt|H?| z%Z@xa2-+-7_ZsjX_;1M{M_X^!bu;qDXbxKJ{}64j#pg(ECFRgwg|GNMTpF{py#9g^ z=>OzLw?Q6VO+K4i_%ImqbcTi1o%GyJGtlO@}*luHQ<7HXY;&xh zdL4Z+9oqOf3jaScshZzJ(tIWBe#nfORNqxtWQ4E#W`x{fl(XKSPV0Y{2l#IX)?KDw z9~(UHe9e`E_XWlaz9aW{X#tF&rIK$cRGUANY?t_UASbfD4P6`WIKV&lrA5+4*Hfk{yVUDj*{e=fXSgZWMBWa{Fx(~VuAm*)t|R< zjJEvzRf*8rGzNd}-x~3YGzTSk9lDMcKO3Ke6jyN2%>`co##>Fv{kq^IzK28b|1B*n z-wEv)J>3S_d;HL1lHG`xKB?Rp<}FbQGW2ULit!xokBU9`!nkGTKce!_K0R$yQ1f~_00Z*bwi#sB8+EGWC<`*n~HUeN*D1k?Lw z%@SH$2%Dr+?GI*u>c&6p1w9wXT?{TcrIr9mL z^GiexEI248_wwXr$oqYOkaX`RaG6s6%y?*HO+PLR?4}dr00{Pnm-YnVg|Mgey)--3x0nD3)Um0ytbZme& zOWPRzJ{`y$G^9WOzRY-jjn~jZ1nHiG%+oCfyuMa!U^j+SvR5AZBaO?HzgxN$^51=m zn2=Oqv6(ahxigcK-eB(qBTM%0R8(g_Sw}J`%^U=$xhseD- z35`?9Llml$0WTVm2(eO+T=}Q|KWO!EXd}w61#Mxmbqz=fyZbe*B9#d`T{_7p4?WJp zzwO!DkIy)6louJ%KL64Z1_7Tg{pkgLlGdYPRfe=Ay+*sbendYn!Jn@KUG_pB`d==S z*;WkqLAm6-jM?^cKsJ-3PqGa)4`TE~+BPg->?}ePz6Jn$hMsQ(zEF9v@|d^ABpn3L z!V9KA#(rTnt^Z#iy%E`$_ePIC9*=(8eE`@iXZH~DUj6w1&IVG@x7OM^TQz?CfDx;^Ey5i_vYGBLf6F# z4jA1ld=P9~D&OTROG!G2dhcZ`fIYT*==TQ>=^Yk34VWr)eQ^d|u-FV>oLi0mZX1SS zyBcpl0`G6>q4-NcAIA4V7ymyGz2=hp|7BOaU;GUVt1qg}L83-vfWAF%h2+^nm0eQe zL+k$u^DIF3y%q>!OO@O8%cb^B|3T3{$AX@C5^a8%_};2Txe z-rqyin(Tq->^Owl?yv$pbH4t8_xuOClIIy!mafdZk3t(q{J_6|DE7SqX_$<$!)a2P z4u_zh1sHtE;Er8clO&yDr^C9Pc$%q_72>X!>E&r zg_Sq2yAAo3r)PqEy&n18j*8g_h);(hP0RfA6a9T@GvN36QUUv=G!K@yjadcy2$ddD z%?7f^jwK@0I;{NBDFS&P!xo$!xe}e<;Ro#X*|!7Q*Z2Wn!=jU=qs=YF=MY%k)RKCD zH!DhVo(7{^`CBvI$FX|pOQT;CqOXjfZRE|CJX@rBu>6DfER;&a%42&U0{s{H8B1IL z!A&UEjcr4FtZPE=@&*B3*9!a`5GG#|(FWTBEC22KlQs_cjPY?Emx8=%)-sZA3!iy1 z`|o_&1MQ-y24c${pbD|6cC>56lXMXE)+4%uxTlR6LtB4b|5r`xe|sZCezAmY|KB5f z6M(S)nFPRo_9I^a?2Y$0O41$DkRr4Hh2zc8by;>G21{3*0(gXP(&>rD z43Z9_&OE;dh-tu;R@ylF7hMz?FKo4+=eYtc-PsO!l*WvKywlF%&u{|oZ|Uwv$@ib9 zQojOwxBUPAU=p%wwdCgkL8nWPy|NNo))oQZo|%q<>#|hdZ+9+YpNwV>vrj64e)=&m9L1j+%Xf` z@BamY2K10S{{GnB&60#8jTyR{e*L@Gb4?%^nliMv&LjeMho8yzvs``5Wd5e-lxU3l2cOcP;bx z4@XNSYqrvuYt3DNZ0$aL9w=Rhr^+p<(pr-~Hq8^KX}z zC_$g3<)V&nMg$?%qHZ^}gdWxwAl7bbm!Y1IBXJ+3Wjf1ioBV?PrQTsZ$eUKZC#}7P zRTay8|0~gSMkn3D*A!kP<7YYO^bb*gP3vF5Y5aF@_VmAj(8>Ms)U|)RN)w58T#cGl z;Oi^ZA?INKkDTTR{h^bm(*7jka~_O-GwKW&PfoM(bwz@A*>L7{Jc5TvlSX-*xDr3h8SY8{d^~Ew6Mq@B9nCwp8$GPl1P|n| z&njR#Pc1ivu`Ste1KN&r`XR)_cw+8H%G{>aJLe+#hdzPv!;8BG`z3}WeeI4OLVN#D zUvLk!*-^z1TG?Zntiw(5Ni{}&hoeuweFu8Jzj1=?th__^wMxvMM^|}JPm=|HUQzLp zKgs_I!a|w-R}m~j!m6Bs{Kt8EjB=I*^zpT~+5mhp>_%(|7zF$V1SW(lsC7iO-qZ0{O`5 zu8eZ0o9O4i!jbvl-qVOmVk)g`%)tgxm+_hQt!E?9Uia&P{{~TRP*2`LQ<#^QW6CeS z`VsP{_nQLpk?c^?yyea5V#vdUL(p#@Kf^D$i_c4H+7nTNPE9?ZsM7X-|4a+eA5U*N zL@WEMDM6%jcq+2wQaI`}5<65JHdDOIs5hlAEc*4SkwKTc2r~Bbpr`Mtq7ik)~Uvp#QJlsR4O&Ef2tNVF=95 z(yBH#uodH3Fl`g?Gqd10$=?rp#_$=@85UM2SosUR-0>A3mlvKPO#Hp?q(Ymed9tcE z02#$>`u%bC9aHi0Bu}J%{R3p>!(o88;+Eu`#YoY0=#MlmQ(oo&zj=n$yPFIKvKv?L zCH2jR!qGDOf9X{Py2A-S+g8_B#Ar*Y@;X}ouQv4u_doeKd~J_vw@&`B>d{NC)TFFf zWj8dn4~Wy6s$H=Ehj~cu=Tup`@~)AY(6_r@9>CsP%Z~7+dG{zkXY@mw56dr?S3o;1 zbpYS0>?Q*Hs61HteuO@V9kuSKrz4O#al(={a*X?P0(f- z)UZcNYwMuRA77UJBBk3 z{|WwjzMR(oNUhg^zP`H}ZJTQx))8+RBZqF&iA6bMT!6j3il2Zs!|WM-qk>kB&Q^@p z@gW}a8!9A$euQrJ2KEy&wdKPgK0#wjKUwgE1uOC2KRqf~2(hXBCGv|&ZA{c}Df|L_ zQWCBw*+A4EeJrrQ-1H31|4@gjO)ds))#!Q%VjU^{x95K^xhkL!!{1}{68IT@m^?O% zc^+_#5Oy%7vxY1{o?gM9#|8N%z&<)&Q*BLJ|J97oLqwT z?&AXNEjp$R^%yG4K3Bo)>N_U|#>k{z9^mXR>Q2(#|Dx?e5_lK+d%6@Z@3aEQ9wZ?iybsmi|ot!s#go%Pd!e7n&u z#&&aRzEA7lzM(uo*IF1tTc62s78iT+P&M{vv}|(;u)R(71RP5X`Q)>Ugbz$<^NEQt zMlbtT1A85;$Y)5*P8c)#LB$;tIvcfLh@b!HW?}~Hqw-+oW9^N!@ee7$*K|azeA@b5 z&pSkeD$>z09+{|a;zfYZanOKzb_t`*iZ0JiNg2AiE?M@gYd!P&GU7D{NT1<5SuEK zaodR0#zcKZH9mJ*Ry2!N_RA$LaZrF6x+2g|IQWYxz|dMh1jfHA0^h?R@cx#{X+wEHJMCnCAfj?#8~pV#ad`TOU|Uys8!qO)0=rWpAK8JUZ(L3`&;Wt5xtv6$As zJz*_17@qvdE1(ZaV`KTPZ#qN;(WV&$NX!ejeD z{+Rp-NW*jv@eC!EF&fh~*?N7=iSS7v{#Q>Vz_?KPFL#rt?f<$v{{a8%TfPVD&j_?lRko@n>cAgMB-8JRYwD3S&vSTm9e4jCn)8 zLbTwhIq>b0iaNx`bV*;|pZk+^5OvB8__sWwO8WJYB^kQn{yq=THz7qR>mVPr?=wF= z82<&V0kr*wz^A6!M{hv>UA1$8y)GK$y_uG{35qRkK{?#whrXt^Iz{<_&yN96MN?@qaOqh zkshu3g#P~jy?ygxA4a*0o9MrPo4W4;@OxYP23r43IiHcoW!C87B@XDM`}i8M^W-$h zdpo{?Q72)ebh=Jo^4nI(%y@tk+_40-E5Y~JJ62}Qo_d|AxE< zPsq6>X&`nzH=>PgT-4CB9pE?qvVr|y`%$3GYjqf=|GO?Qwvm7E_0KW|^8b6Qp6&v9 z2s@b4uPSAzq4G5lckR=)FfLU74L`_p45GfIB@*1z-Cyd{_MOGc43HYXdL+``Oc=2} z57_m>IUVXLc`Ex@#dJ2zvxmM}o1X#sOZm+t-I;5c-(v{7iM0BBwb(dgFSvis3_1<5 ziFzV!?D~{8{;x)E1F>xO>PlPRvUa=pnX$+|wPPd7@xu3dbi7_b|Mg8z!#ts@({zyJ z+nz`FAABUt@i>18q+#?j=i_<0h41@f`5Q0~hv^;$;d|nuQ!OvbDtH9@hg#55R_;Z#h zca{dB2fn;m>t!C^MUOvikM?@bpq}u6$@eG-*0A~ zpt-`9oi1Szxn$cEVT?r{~c*(B6{C{;(~m*mPyb&le&8s_B`azgpRqlHwn% zxd!skVMx;&SILQ&3_1eFfBDNCM!B{@DX6p@k#By`4aiu2qJKYB+V)y}SGcSCT9+B< zoQ?SJ3pFh=fc`&-?k^&+WJ#-LXTb6DaYGRB+i_k3$?hGsMx=6QxK272$cY>M!85q$ zoHA$^RYt>ik3E%!l^cGk1DP|C7isJ5j_)|Mv};)P@1#=ueHR9Tc!gE_0)3c#Q^SdW z^8fcQ$-Vu#Bfb`XWN`>7HXW=j@E(Eb=cz(4UXA(K;)hF={>%RNE@rn2N7mJ=(e&Vx_vMy0#bXp%s6(<5 zVm|T^xc|qT#?Q=Qyfb?%GqyPu3oF;n(}DbMarioKkHeDt3C1)2isU!KuySWt)v7V# zS5@1D7|+nb>mfE(=9RAvt^dX&@i|Sg_7U3t-{@8(<`)k^%!fH3j`H0BhPUf)c;;iM zYacOHS7&8!wL{Rx%%ul_?D%Fk62`(ly&z9#SXiCK)@sob^}!%67kkJ9`{-msmM7Ep zjeR$?LEkHRUQSzo)iB=v$MxWvmM)PfJI@D<%ltDR;Q#*!9Er=*>7=7Av5}k|PzcB0 zg?ru61@R;Jr$xv0gJC-#$h-$=(wLp)*XcFGSX*z#`~C07S`ZJDwQ-7-8S`nw zR7C4k9*}?R$7Sp{5vBM(7uB{YbMZas(s`0^`lWev=k!6+RgHytGd7$9!Bc7X;rz9=Wlp^)knl0JMzHz3H;NOvE8~kpeP;Ih_IEQui0e|rtNbYCP`uw zq#QXA9*KVW;Q}yPw3k5Mjg=uX+luL5ttnZf+E^V0WFN7~bF&BSvd@2y9V-zh8I1#W z1;ox|v}F!A1MpDWVAW?lmJ46e?|=2E=_8)Y87S)UXbLjEtPR}%j`+-m{#Pu)*H5Up zOl58U5@_R+#K|C!-Wdq-FgZ`_^^k`NhoJA?RcwFjY9)xjdv`AQJ#C^+IyGSU|L@zg zUBR=X!{R0)%}?Z4_wPWwt9qlq<3FRP*Tn!k-}Tpo_V%gA?}-He-_l8AZ$sa-a`AD# zwB8liPsr{ZrRBt?)H}jAs2ApL-@m7=w8=!Kc}f2=NIT zQ)(VC3+-!y?`^sXXOQkA(lT97HqhF6LOT@15?vfiTR&mfVbNHX?MVLJ1k`j3K40zo z_&SWgjk=o5wqo`K9Ulhy*Ea74d7bZ;B;8-*CHF{aAL#N%ecMF_`_h2?8`B2HeuETT zC+Q%tQpOg7F+IBG6++OY)60UR{Fv{VsQ)x8^xh3`fHBwMPz}NJVed3Ck(VXinUV$D zZuJsVFkXk0D3f%%jqqjkgQ{bu)oFyi%@2&>FYWj_U@8w*zVJ|kwtus8@P6Tao!*mkfn)<_2X+;Yt`|3b^AmfjI-=&jF(6LgTu#Dx zac(VU)QKaI|He(ifAb5YZhkc!$QFLVzd&c8jEf6Gp-<9s$io`@@u_T7ab_TzTp0<*ipr#?@P0LD$1^wyVc`E&wb38g}eF@;V+R-4Q))VEgDoIc;LBmq{)&qd< zT`+{UzI$I?aZvO&BvDm9|HmsfSv<}`O}vP)5jzUi?P-RKd80Ifk~F#fAh{QRsewqiWD9*+ev z$MW)S(V#7O56Xf47!A7yH{tu@c&YQhMUyN<8qWCnx4_+3VcSxrVpcklbP)9c!J~jL z4#fRDkkJ+w&^x9tP_v`2K&;z`dcc_XW8-J93B13h8||E-|D!d`fW1lg$#XAL zlQ2d$lJ@DqqrC`uWeQp7^`N0**PY|MQbn#ShL;syPyLSlBgl zG~fv`m7ITC&M0w88BZ>ZHd9w#UxK=gb(->9P16mf-(e zI&FgF``1xxGyz_*_XW_dgzTk*k~soFV@h|}-9mRKIRpPwdf?|t>3I2`xA3t`sob80 zuR)8=Gid9dnHPvI>?{&o(hsT;4Z`>TgYPFnUbnfYLHki)8H?RgCHo(DHhl)`=*^zd zuuouuzVAOV_G2pWCkuyK17k(t|EABgNjTEH^{2a&+L);0o}K}G@jtVcw*JnpK=FbX zJR}-1L3d=#1NTbjUbCQ{3)T^|as)m#HIHtCvM~$d!Q5eMK|RS|i?Sh*hbm7kTc7t3 zXYYIm{8x+etfkfyW&W4pw08D6KNG}nXPX0U{TyFMamYeG8ht}o_}*I^*j1C~2JM|u zUMRDzm@V&gMnD^L%KCzQ+VpO~e!^B2pKQoO(3sM`KeJGw=2T$M%4@bDK6JdVTD?g+ z2p*3qcYw^^TzqYhpc84t$T*Q+nL*7)l^o>hlIK8=a(ozUD~Hxq|KvZSl?h^w;oGmr z*8-02UxpC+u=L7-V3{#@Uu{QIi|zvXDZR5Gwlp5VyqQVs|7pibz?V7e5}{AhvRh_{ z)Q%AMM^CHlMt|+XwsKYgWnlxNx0o-&db57%0TW@Q`)?>%V|_VMxn#mTx#{ z1bwSWCQQujp{S#Ys+!J}>dkb$6e;=;|Xm9+OtI!{+EL}OiS;AlT^D^)g9oCbCq4!En z=4a|I(iY_^>WPf^EE*Sp@ zk{3hXEmN4^x#|2@G?kneiLA%(9j>t+B)eU%)sfnR4u_y|KWq@?x8V1#Ykpa9Y!UTD znmRm)c5H-B%>q6R_}~gsf}cng)|?j=_unkqK5ZfDx6}n-JW|{Nc`wwwfqj9H=`3CS z_8^q4Y3~i}op2Rjza!*cdOsqy2SKMxx49;ZXQ|f!8|=2WG0JVPQ-Lv)#+l)Q&p|i+ z|2ZIChvnnD?m)KYn4|XsQS|HH2f+4%>a8$7>n;?*JXMyiJny*V_hN3MVW2&CDeWU+ zEPbCVbDJjrP^cN+^8{$eX})6_pBswoJX-(VKd=FPe~A|D_=x`aLsU695VdJBN85JV z0$#^3ry9by2wgdtCup6eM!u5ypAsK$#}mqTfc=Er#G&IE{UCUVbYJ>7bj|EyAb&T* zfU(_pr|@s22st7>6L=29p!r@htsj3}97H1?wAg#6d!WMEjX=-bJjtGqb4)5Jb_6a{ z8vT_IeH(u343Ld%3?pG2=9ZGmOjsCwPKu}4Nz($vZqT;F0Mk&$^-*&fT2* zwEj2591sus^A>Sx&_}a=T?hWR9cZ%0WU#A%%e>}>bCCb8=@%*7hXdGu0@L@$x4x?O5volEtft|0|2xQz_n`!HpE%y{DJ=}|0&sro5 z)t?Uh{ud_3BjAsI~v-J4?zxly<`d!Wjcm|_}liL1mp)O+_sn`XJ#-eKkSAqXy z2QaT2kMb&@^)KsjA<#Yh##ewr#ebPvD&h#%qdng$A*~~B01TeJJLFw$w~u5yRi0Y5 z&5^wSeAOKU+VSetIi&k`<*p5Ij8NsNWpkg=HEG_xfL+H<;QOgm9<01F5x>t->+XHZ z2f8x^L!nR7JWIRI5^egi7d2^=NBxeC1@>MWcNN?-Fuv9=a$;%zzvUI%Bx`&1fq?+e zsl$?l5tk|Z{C}(RWmIg1pC55o{!VH~XlRAlRQ zRa9lia`e@=o?w1C!2!1$lRve+I?%l&o!(uGIswHDby4l$h*CV||%1pXR5D zjK1OXU(-vg`ZMYNJxB7J&C)#Ow)nbuj=L4D|EVkTk+ok+P`;BpnxoDI=e}lJ4THRc zUa@7i71L|J%@4#J%l~{m4wrpe25A_5P})YB#}w}lS6Eo#3-14mZ1D3`7zXv~`H>5F zsPv_ZNM)UUAuN$Rkas%Qyd7jmiaigZ67XP*2;(F}Y8x7m;)jSdqT?`@eo^?i}!e;Qw2iBzF(;D~03xe{;r7A=%y0G6M4a z4R4P6P4ryuaS#tdLOG+{@34<39hU1V{F@>3Du(@mw$kd}QB6p5w`f$*JQTFiK%^Sl z``%xXQ74_gQhUjKd6f2NfLGhilw{k+KPzO8|CwALq5thapzr>ETLJLs^r|$Ibr3w~ zKaU0W59E#m7zCY2-v;n%6%?lmgIz6!iadOtU15N~Gh#d);j(|Lgz=Y4tLKZx-fmnE-f{)=B2s*$dt<>Lf6v(;G#Sy-kInML>3Nh$=~U>85bTI7(w-`Lt?n zk^JS|Kwf_37I-d~$|sshzy8s6@J=8b)gBD{w6ttqkqctBCPOr1Axl`bBm;~yy8y|0 zg?LK^@IhLpvwX$EV8|cT?E%P-7{rD&Om@X0*~e66zOne_jg27wh7HW|Z+V2T^H6Pj z^(X=OE&8c|QtPmCk4cBbj~^DGMIX1LdoQg3JV~F9!L|}OR?_w>IxLKqKPwi>hE{Tc z-{y`}Nd9Z39wn8TurNA0Y$|%C#Tf8kj8kQ_C2#mhln#fx6JIm4TYr~!?qq=U6CWG8 zQ?$^t33V}j4&vYQ>TwZ)6*VnTW?M0tr3#Y!g<|^xAiMm%8k|pJJP&@}qqPOAqh1E@ zzN1>drvl&Zw&KrcDi2ovv=-kZqt?l}DS~)x|Az0OQtNxfl-X}N7a%&i?Z4kypOFIW zb$c%1E$Y+$PyQR~?S=e5c3FY@+~!0%U_YUI;Ab0|?|+wcol)%!H_-ndrerY6O?-&2 zkr6tGv}!MYW+GzCagq&0{n9D6V&3*L@r{LvXodbn&{j3WB>n%{CR^waA=6oUu5KRW z=d9fX^515%APtkfl{}8o4=VQe{B(4#WiR0W^j&F;a$8Dnp;TF{JgNoX_w{{l3NWZV z(VNU_G*!2Y%}Zj@T}_R^&f?Iga15VJJxsElDo-tsud{bue z@=)ce<%aL7!dl)5fb;ZjAmjL#Q+tV0WwG)yg-yVhbtUv?2j0bIh))W8iz~gS3KOi> z0p7d`2cW&rgC*ay(&1BS>l$Z6-|~0kd*R=*Vn}Vc549wfsTk7bvNxAd`6sEso<4S3 zjB+uiDRZh|8=Zb&>v~LnDSyL z1;}sb`vusW)S*nW+w1Ty$isv~&?93K(ANIx!1j|KF~h7o%%Ry!+W=eC+~-DklCt-z967(mX?kYzMl%w&L$W(sfuq&Cjv+hI15p z=JHkGo_C;ph;tPmVc!x4}G0{{{PRt z!Q)A`Eh)i&XS*{TqAp==4*Gb&D&YULODVKIV7v_qzezd>`HOeW0A2l1ByD}i)4AeH zvp1vd*Kg)ZPRKfDpP z^T{lKnQg^z?f1U0C-9F?$Itmy)z?5gtZqnLzRZ|sA00=y8GV2cNeiYxY(fW-vfCtU zRs;=8yB`Svx=nkG1{l(HSUz`trD)K$P^2XL5IQz!78onvbB{uMxm!bIwiUy>yK+9{ zKew$~gwbBl#@7|FJa!d4{Y-ivz^U|qo%T->J^F&bms?Ef08+ZURvJU zARb1~*4JT-BUUecg`YoKbFyVF=-bY{#?tzL@%#AV>%7vkSbj!qEYNjy)+&@*hm|!t zuGd_h8jCLPvjV+-rX93ZmYWi^ccBMemTx6 z0JH;Ui|fzpFi*9Ou6#_c7}$!{ZN20IbQ%sJe?u_jLzB#yF=u1N7sd<)eplWd3gbf6 zZC<8NYGb0lF1s1%s;;14AF}uQSYy9j5AAZSC#tk)J?Q`af>uF2&KI`Pwi|&@P3J7Y z_k1yWrEeKnud+%kg?Jd9mC>D4#%N5}nfhL6Y;`E`O_b)sc>kXszYXwEah7M|dpXu7 zwY2rOG$QO5J{>DwH$nxySH%T9`Tf^|_QtS&U7rQ>RBXEPT<0<|L6`g83i1w*#iaY4 z_ZZ1L62Z%q_75#VC!DPWb|us^KQ}T3I1d%aOBFx&*0S;)Z5;OxeFsP*Na-NY%v18DVzO7u_|euj8|iR7GR;lYnGk5vpi z4b_Bpyf9@0-CssqlI(7og3rmRw$PQ2$lnv0&&PkeWJx9c8754wwd^I(Ma4Lxa}M~e zv~D1+A8S-MitUqMpm#4^MQePT!M)2b@Cd}Je8&9E8C4fn?sfSa^i^?EEQs?J-Hlb@i?(wK=G@U@pXo38*vx(>@f2-qgdwMsy* zeb7Xgp2YWvY*h>(R!plFyq{BL>B`GTZh$uSvDpFqUaQ|o!ss?&ARHrf7}B&`Y2#~_ zt?dS4XXzy-wWBoOp`dfnW@-Ayt1-auUQ?&g`hWBJG0_|67Ex1Rkw~vs8o+4lcL8E8 z`w<8GvotPKUQzWT^k3;_KCpNABRh~1GQYC${|ii%J4+L8D~mh$)4=?v{;}lUiYjA} zHjmc7(rJo7_dCzYwDn?-NO9tSd*CHGw^0R?`QZ6yXR23D!T961mX6_KgdH4+P%d((mDtvEEd|1+vG7Oon)v{7eJ-v&eMV zpR11}(^f49@gFgEF>G5ZZ9U!RkY!(#f$hQ| zM@CzG+fSl|jC5MQTN}i^e2NNfz4mcG@%Ll_qM$udxa)R4;I$kV4|&~&ZmE&R`&<6c zR%I9?RfG8;pFU+D#KYvx*T1Fhi*(onT6!SEUf}nmUEPhvbUYYse_|#{2f;UeIezca zc#N;15_BRhST2u@*_$Y;=~0ai|Dgf!8#HVo*1OZPk5vrQE~5$BczAVhke}P^P8vg3 ze$0YxMA*TU2AoQ;Z`2M3@o(~~gLYB*<-+Y&f1o7LniGF?L^vDj;S#@J!c&ee$ zHmm}~#l5!?wDbwev0@h`sOO9I7no<#Elr=3A({W|tA7mI!p-VE zY!i$(tA0JHEGj896>iQUrZRt8JfBCDQ*h$aZ{zx}Dp|^<(&{I&pSVZsz zCjQ&+Uj#&#!}f|ke*@V2QT_s?VYrpHlHV65^f0BF3q6sywVOfw$Ny%27qPf@hSbIc z-o699fbQ4tQ)ug-*a>ToQw zT^@$Q_Qd3G58Wa&X4MldG;QYLezpg`W{BnMG>0?l#OkTn*EqdLLlB{n}(w zKO%UrwB*7bTK_-3K>YCo-Px8PKdf1jq z<7IzVWA^wtaG?LrF`gh^{pxZc4dXfM6$aMDvHahz5j%SdZElPJ@i#15#n^9pUAu<2 z5e4whrk|kA`WVvBfBJg0Bdsp#N7~kZ08{S$+!mf3jN*V_($y>O$-PrRn&$wyg2PY3o9zlQ?&m5d9MKaa@xWY+WO&(hSh$n(nss7qlE28 zV}V_YiP^wz3^%>>zrCSayA|F8ACkUH_SUJoO@5Qlm=g6`Pi6vp3^+Cb zgP;@Xq31DT)w^5J1NV9e8~@<{E70v1-VZRISxsx8KZHzYDf`zG=$m%c4RAj_-AsOl zGz5f^`a@@UF`q2N*AFX#*s9NA-Z$=*OuzoWJiaSvd%u@vwEi#R_Y`GYq@cFl@1Vo{ z;{Z?4Wj*M_Qr&wn-fGn^|~PsjKh6=y+wsywy4e!c>-{L?P5E?vC!6{9U4=agJ>KCr2^r`pt6*?xPah^U{fX#?^j&G6r# zCFpWbkICHsbH;tNH&e!a7;1WvvE8n&d;)DI^k8ZG0~0{stoVYqK2Lj_C@Wwwn%)|Z z9L<^xd<{*w1%19-mB^@5S`U#=|1}))PgBMBUQCQGKpMukVa71fE`&^HY2q+PRDR}a z6yq5_b_?uVRGIeK^xuCy{`ns0>ZZ)0^*`+BcClW-Zt;Jg)*xHc@O6m^GbMY)GjFDo zY^Um_mcw=@Lw@}q`2N2jfCp(9zrosbq%svly1exA4ACarAz=IuI4L<>AkBm2J=+g{>T0)7G{V%%5V0NcJV9d9Zx=CVZcZO3!ml0b?yS_7iCi zM)0K_x3n)_B#%xB9)m7a90HzKI+p9fd1H*{X+fgQeHG(h7`hwwv$pN{*>R`+OGs_+ z-9v-a9+(`{b#X|1&9U~LK)#}&j?tF93KNnJ0xzu>{=d^x&K1+jKD!@RtMZ~dlE(Uu zY^=}*JWb~%^Iylo?l7JNW@o8?oB;aP;DxuH$sIAgpJ1}veJ;`3LWi3?kdJysUIy;6Gi>|Zb832egO15OCJJcFdZII_?{qD zA5&TGxa8Zp?%nb6n(`qFk>;Nqh<_I(O=HT>u3w4j9X9~Jjr3q%|DV_}fY$$P4?dvx zkUEzut;;@RoVZbGxY*!=5PhVJ+a7bXC-gs}Bu*^N|F`@TOZ+$AFuuiC@b8KHFB(Z| z``b5-WX25Wc^1jbtP^5*--CTg?TFEcE8tBUt^di1DA3ibS0-)!hIzK)Ss7ZW{1{!K z{I)KjpE^v+hW5oR{RZ2D(A8Obxp)*DFE56k0rno{)sp##gJ?PSfY)b9xDoR5}YDgyuaAv9R(0qyWm^%f;va1z%%HZD?4Z0q(PO@>JTG!`bM9L^}}wJke#?wp6|;dP;zY zO3$)s12KKqZxL<%uRM2=qm?aU@FxdV9eD!SwMPv<2ZCW4ipD{IsMvJnM{SjmKV%X9 zEL?D-0+r_Fhv2^pCrx9@x77R)*DrYmY_Df0lW?SEQfwE~)|vge1MX>yQ=ZYrv9DKy zxTa81_|*IX3c*X22dJuhCa^azUIk(u z`!1Q!N$X?E+gVE97j-`C0z6TR1P_zbyx$GB5fctU50J|iw=BWm!P=a+Gs+Dv9}3!t zkRwvZy}yAk>`?ms;jj@W?af+_q827eNc8Sdz&k5Y8H^bW!{#gVnhT*rI?bL}1=<^< zjsEEle*gbD+m#f5)jipaen?|s`N04F7uU)jzkxk_j!4!esC@1k-D&;5!A%47*gKLv zZE4vR3Kwgi9nwK>2WKFQFXP_|3|$~;tD#$D->brK$N1v?4a-+-69YfnUXtxLeJ&@n z{|hI-5WTTvf&1r-efW6-44ZlhyPA^Pn5geP!4}Y;BueJ=L>-oDWkiWrJcvf*ZM0B3 z^E|LFE!Zk~mu$8CxA&ihZ<6)$X zehmYYd1RSQ>;EH@S)fm|uC{}eil03rShQq+0{UAn6}_V23wsm`bYIxN z0FG%w21~z1w2EE^dx%QEd_pt(F9sOCCHU`lVf9J<{%!yBRaOyfulTFGfxUNX%Sg6; z*=)e*2c}c{3Uit(9-Vd{jA7M14;kgABwYi3O3Pw-_u&VDJuyw@0E1dz;7}tjs>~3( zTD?Wlu0_Dk!C&#;499psuT6!0fhtQ^_M5&Aj_tb9JwQH~?Lo48wSyw$p~H}-36T`h zhc&Z+Z=S31^#tj9BCoGoPV4{JyH9}q^Uq|_);IZI67?MT6!D%r2zCFR2l|NqP<+oE z!;U{RjbuBak11VQH4^Hn(tim2JU;Xtq+xOaUsPoFKW$uD_0jW>fqcW0t&DO%TfUHV z5cXaAUR{ml&)u6YBxurUgzr?5Uc--?C39TSbxZKy?(9AfUkAtdx^2hzgr#LV%ZpD) ze%mVO0sfpjT+*Fn+m@CjMn9<7XE$C)!^aK=^7|Gy!Ldc<`({bMH!dHB&kOH4+$W7e zLVj0`p?GY{cXacPXkpgO&wyv;ekI83+LyV_2@L7`+ zuj4>HKQ&^+1ePpm$X;jYoAXKhEUaoNd2Oa>nq-aqZ;h#?@u>Up$btSWG|1@`i2FNb9q4oQ2|w5u{$?vyANk@6lzq5m z18Dna`dW~N@q|C|C6zH6({*@$nm9x6C5qLbsp!LKi}AoF+B)TaK7f9`i-7j|r)V%v zY#z{vZo6SC^l^>`_saL9@b?x>=di-9;QoQ-Wx2|fp`c%+IsPsuMqJi$~;ZH#uQtQ7x^cOGs{YR{~TUn?u zBMGz>Z?Gb?&oTQL$#$wdwVXasaz^X%LQi0?@{%zmjOEsnZzQDU)wX-6oV3`%nRK=f zt9r3Tw`-gBa`|W5?P4{p`0hh;xX&%iEq0Dn;oTbC*YKRV4W~VEFwbMh1McS0?(B!= z1_Fy`Norj-r*pS(^>j}<8gU*)*>Vq6Y#21^vpn}_=pTN#T_*R1h9>uw^9@c`bAQ&| zp`q*$k3>7g6KCvB<$kpqb9=PloySk^(({{b4d)%?);?Rsdj0CU0O|TzCEaBNM`wCB zgNX8jT*r-DSgl)=+2%eEx%~~_aJ*_BtJkT!u+2&#Mekv=jugv=X^;`<2p}E=d!Hxx#b1+${LdsxNFl= zS#G&Kw9-qGcsKVxF%PmMcg&ypES+aF6TOJ(ioT)~0e` z)i2euq~U^`qfLCJqX9N~lRgN%cgShH)Q%RIzKZ6lX})F6=bp8{pxUVFG3F}gOs<1T z(apKMqF&J|Ms+Rh<0)f!?djEACyOy0L*-1hs2*u*eQwwCbK7>a2X>9&vD|9~3uAS- z%IYXTQ0^|b$D6*~w^#3R9e;-kK53uhb}6~bne056cP-C_=h-lomk_U_cqMSI&XtSD zSdO8M-P;Qv@TTqd5?uCO%=g-Tnm4duDDum_w}!^zZoLX!ow@aU7HGtqs%ShrWT=*s znjz?KcAT>fy`r~%_BPX;+8vzoN9CHT1*P_5{)@YT!7P5V=S^OHAfGoY>`TvYsd_Af zwC$|=gbT=WmAitI2kQl65~iSnw_X}%MK<=Fe%@NqNE+MxkcxfBYJ1iK&2OCRb;J5* zn~QnpCPuUN2YIQP4_?TdXrRq&Ghf2#)i+HoXw<~+CUTegPQxqNKet}z^;T%I=bRIA zyXWZIv$WrHW1o0%pTt@6N+MP5ZwPwv9?Om4j@fvL$I&m~4R^89c;sfro}xN|qpsGI zKRvpT8{SvJK(N@Gv)T3s$M1R|chveAwOPSV z{HMA*SsRNlajXOE3|ec2>^&*b8Ble<$ zS5>ul^b@pRzGBz6+d#X60iJvv{r=pXtdW8U-2%?hd=GZf&b@+>Pioj_Sn(WZYj?x0 zKis&(=LfOkDl6F+rX;XF`bKl|-bJwYn2%Kt8f&P!zh4T!s;-KCP$88o9_uPt-ZPb* zZF-e|BXJb_kG>Xrk&+81?DYV_gTBUUU3!9ya5$O2y?^_2qKK zsu}94-3BRrRF90DE(nRL<2zn8P@kI;%1`M3ffHgh&fuGX!&zQEiK8{5hI2T6CA+Y! zg+FuJHP++#y*Oi&+&TTWjA3_o9L#Y)(1U-oY>V1sBv-I*h@-}Y_lfFTtX`_$EIhBe z&^th7WYb*!kKjK1`{ncOwz-eyx2dOa6e5%beH31(8)wL~ANq?{9Jgz8RFl29>|&jrtaB4;)boBVXT_|wQnTBYqx{y*i{F3MSk{=P^BlKbS^}S$ zp=^hMi-M0Vd$kq0sqEJ6q3XR$CJXLgf6RV&t&v@D)v;9^d|q&s zhMTfXCzW+u8})+sYwl*j+nHJXAwCy*brESca?Ls#p+jyNPLI^!zTUo6BX_uxM)smM zR_UNMg7$d6TE2B6&q2JfS7z0D^|ED4d9!XL*!p}sz{znA5@^aj;ax}>#w&Mu!i_Po zWAzwp$9^MH3!JKgVVV#7Dl|ho*UICUP0LF7Is%#v~4+m;Lae{m);L~T75MH zhp)`yMD}vzw>aolH%)`}XS6WwlQ=dCCd9-7b!H3j&YYS-K6heb;NJhH3d&TkE}jHAD@X>@&_vc8k9aVcE*v z;`A~eXWG>Kfj6SRrRv&|-`T#Qm$+Xl=kb<=UR0m-{ixc~?vL1`-)ypN$=b}`chf{; zPPQJuWqTQ?*=xOB%XdHSx5Q$O+`>3sn)NeVdDTU{FCn>H%dtH*UgqBAvHg2#99@U1 zUW+qTYfIm$`hJM2#){N1HLkmo{phC&c5z~3wUIA$?b7wF)noPz)jxd1SZ$V)oO;Q- zw`u`vyjdBMm+a!!^i=KVuw5-BSZp%GWsB;xZL8GA%O~2pw1%;VJQ^l=ZDgsj_N0PZ zmkl;*btXS+%e0F!u8P8ftR052((Wn8EV;?5x~hS6^_#Pglh^ z*+~7IL6n_S=p2^msi}fRkH4}`iw>$@J@$zm`P@jbsUU=%wB`vbYo7(rA?C2!1lu^Z zq8AFP^QLZ9Pvpd^JY~!AR1T(V`ySmXaGTXzFmL5>^<~DT{Hld}I0r-I^(~|Q)ax$= za$I(YbM)+fvRVRs_|g8eSe14voKK%mnKnH+$$F=*!g(_<(^mfOJ+;{Uoq{DpK5Ndt zGfiDtK3=`z?HJa`tPL!O(>DB59clcjnOy$(UsDAi-bQou)?4srczsd-JgiyS>Iq*^ z>7~c&ej-Xe#O$-ig6655Q!j3)98aFWZoF8e_TB~O*sJ2h)r@kxu%os9dyY)@ zVk?Gi<=DKo=6f#aq9)Yb#rNDkfTgO?%Ccx&t6t#XE~xYV&I*oIRWmv@M03tV9nKSn z{_4f%-Br%N9>|%s%8<26VTGo%NoJ2(wxNP+Iuit^opx}#8Oigxb60W$8?G97{47&1 zT|SqqlVQX4U%i3V@bEG}p^~MR`c{tH@27&r{m^`MUB4*K*Iog(m!|IFOub_un7r*> z_a3!2oWAYzIL#_2Sccbzv-@li+wHro##cMMi9aD=CV%6OlbpSk0=wj&_c>?sl~~u8 zEf5&~*rHNXSEk-8Fj4V#U@z{)J4)>9Aum|kMuFTZ@B4AYt^w**XNR+2S|4QR8T3a& z5+<_tP15F?yeqR)77XLuPffIanEQ-VUNefb+h796U{N+-f6X*bKPMyg>kht}c1J&P zo;H5rEE%P$v}4LnZeFPiOT9dv8&y@ti<+IzZ|r)P-^DPMH<2Y2_|I+SWjF_Dqz>xN zy|LX?V{XJL-t*U)?AJ$FcBO0D*rCZocr9bkn@`wdq0ZjJ=XD>%=j-1c!I?Q?kd1YR zo>s5P%DkV;vbb7L+*AdRZ>jofZMAJ%=EpY_dfW9~^vbR+tGC8(hHC^xG02k``%UcV~JgO0;4^4Hh1rFC;4b_KU@#$wn@vA zXSgwn=h^M6Dr>2|`lY07)eWB_G|JD`sjHp)$uECZA{ZYar~dGJ4?(v65ze+7Z*{NC zu265SQ0E*^&)^L6zs`Q+n9U#kDV=3p@>%_G{y2S=L62GYY)m)_=gRr6`DfIwEwd2t zjacf{D}B|wuRNn(X!la}>#HZqHjeA~xlZkNqqR%;PI=*W`WX{AYnPp|^G+{NPu#nf z)%3K7Af1!SzO$!A-LR;>_lw-w9K-q%Dt#&y*{hF_VU=}kR_Es&XXPA;RI{1TrW}8> zj?eBm$CBSx&Kdf4tH9^rP1b6`M#0zMM7Hu8efGdZBh?4$j^OiurLy>}Bc&-sqM_0>E2EN?8L|=@Qb^h= zeMlOTQA(7ihW7gT{{B1voPVzC@pxbF*X#9sKi($D+C>FgDR=OFvk8=aEQ5egelcgA z-Ox_+A@0nV>D+?>vx#pClk6<|GOF^e0tbJ=rp(yo!fjG{8*I%d>@Rnr2{iK712G` z{IMLjwL=E}oE;!s@0fsscitd7l4h}+FHa)Hx1NMzIrh@-B~tV$+fiUHc9!g0d=`DF zl}7ybcFO!KK|M1<;4ZJ2o5C$5kCZXAe(XVZ+m&7@hW#vV_)DXRk$wo|B$35Yfxgwd_D5rIQd1U1&zI`W;HB( zsZ`N7{DitDV-D?+^CWxJ_F0l_i}NKfe;H>D7u#Y3BYk|n+ZA7~T}R*NB3QNKJ?Q9z zBw`@g0^eKvnM`PvLxJJ<8T*-E(Sl`)#Ad%#kgk*mN&-C4b=5?W)Ov>U{JMm^Wi^NE zey9fqqvz7!=S;<`@L{mCTL!DAodPX)pOY_ls3N`DnmG6f5B{1Z1KF}d*_lSS(1`?N zWR`(>dkvo;pZ_SLDxgv3=-PfdxOp=kIi$!**`1~%6>`|Zqf$)w2S4>wgR$uC9#`f_ zs3cQYltPNB@599I1k&o&2723LcP%Lkad5mYnlAjL&p!4xrxeojuqb<6c9j`N7nnDp z9y*H`pQ=jetyaREeJ0+Q)FV7)IGZlWbILg-I(4 z!%w&^&7o(6GxHwnTj(c@-`NLt_q4Xe5TJr-Gs48);-IJ)uXb>Vg_J!wXsp7oC2xniQWzMn@W2D~$8FS!As$stgc??`W_rPyCOv*~l$3P`r_ zI$ixMmp+(vf!_Qkp1*JD2%Y%R3Y`9IB)jiuKZAXy;=zgCZCJmN(fN3lePypBcqDkC zaq96rdQ5SVpl>5BV4TvZZP=Y_4th#anRyHir1j&U)gVh|D^uZagGX|Y(aQx=9K2j3 z^YACj=tO*`JL~Tg+eR}A*}+_z|8{_1x!YrE_hX$QUCt7 zJoVu*LnSmMx0t}RhMDNF)-04Uo&@|V zthv$QuY_H#1d0?b)_E2gP9*Jl3YD!63N@)jswYvG3wU)%_RZG?P-A^Myz_lG(RDG3 z2wo76ZGi&EpV!UqJ%3A78Rm$dv~cV%{Z42meT?8$24GLcOJuq38hE$(jLy%fIY{~U zG=BHm{iJkgG}sUu3!mvYl6UwnRD)kQVXWH0ecV${F1vFC8O|(ZXJ+0d|D>6*11asq ze02#za6AA~8Jn<+L=K4ls!7E?=o8ets-Y{BN6Cs1-s)1`9~ujF-Nbb!cC|1 zMUR{mMA7bAuye7YXjf?*s<*6AKJI4Lo2`5VHp@Q_%@F(Z_a%;!cV7pHpG_)=y+s9z2Lo;nZjl+GcSbOvDY)@fvUni5(3$C64B=`3B|3UP!b$eW8WBF&TYeobiqwXR?-Qh>j@T5qb`+i$6i1GH6N+0za+?rdX97v^!bgAHiR z3vmA_c*(5WV9z{x&`#^0pGPJVuZYRp)^RS)tGN-!8=T_46!u$gh(JmAI=j~AJrno3 zjc@2&#JMGQ5Zbeo(4JZi{o@{$%+>H3zN>}>h{;c-Gv}!@+kI2u&VR=!(R>A}DLk0_ zRkIH)KRLvRisMCDq#Ht4uXElPKBFgFS(KD*$GG3Pji-BBG9Tvn)6HcCawo=fn8~Pv z(VtUFC@|B@*TK zBJZ>Vz-g-|&g)zT8a~FLmNAi5g(U*fdeP{0-YKE;fF?AvddXdOqh#lRyD;6HL2tT> z$oT6#5Sl5%=N9V-itdca^PH1i@XZ>f38rMp>n#+Uqzkq%8R&D|L{*yg9njv66cYASl zdN){ce;@4bO9_mGh8D}35}EvmMWFSWqmV7zLEb&{2(=X&3j$U}GW$Gk5oMk$DU)S( z%(Ruwbkl~J=)PMg^{Xy{s+(bhORhG9@Og=hhuUAf^0N~C;;%j%VsV2W9o3@^+aAyd z$$9u`bQ!JQ)`^BJ0_BRs=Q8(x%wft_g_21QDkxQJfRMJRW-=bWgppy%?9plk&cGH! zw^xZcFlqp`v!N>E<457oQ@QBN-vl)8{KSUH^V)c$jU$=m;fQ43Y%%{4n z7k*^ADQzt$yyTUP{1H(C9GG$e{v9|&>QK7m;&pizZ@bLclGxAe{tG^=chC;hExaX+ z4pm2t;VEL}{0q3vy^=hAFbH;u*73^Xs?fZUF5<*m7gC&>L#cdefnrxrk~v{flv{x( z87;P(Gv3=lKDBy`B790&zCr*9j0h7RjlD*k`LluWw>%9k-mJj=<_Z(ts5_B zHzS=R5(266gG|<|W2BLKI_Itt1O$dk%qzx4L;NZOub1-}my5S7%5U>fzqt{2Atr<= zYwM2jJ_e=dfx z=`+KTG8bX1LYGC&+F+D%wE~`3?xQ1;_hK8dKKkK_kLVsP&iuTp#az4+#2h^F5d6A3 zgpNI52r89h(YnxhBmm-^X|5Z$t*HafJ=Moi`$}jK)T<&~Jpd>EqmSM&rSJaQN!@@4 z@y6V2p!o3xn(myYQ#<7;Sz&nzB_uv)&uV^v-;p6EGTLPB^r$23k&7;T;{n&bn*gaF z&yJqF!~M>;#xBz$I1`gT^iy=xeAdl}i2GUuOfEfR9W|zc9Xm}?p_9An?>9lT(ex}( zL_Glh9}xUJEgDs;VIVLd;izRauRK?dD@j!ao|BX4E6T>)4x1HJ(c1=Ap*f#galVS2 z+PN25I2Pjaty=I2dQL?o#R^&mf1^*IS`e(B1bk|>(Qwf)DS7XR%$q9@;9K3b+`~|R zE+Ik$6MbgjISo(Ihd~Jq(;^+{5f_JIQxgyjss_i-*l|*3xrmftKZJwLjIV8 zP?Yo`i|SAFV2lgP9TF^nl7`_>cEd&Z<5(z3PCrRfcoH`?;{{jn-wrmjUBdFQrxUt; zFwi{6Lj(2rNr5{bg7MQ#D{_1I0XXm$Y5hnqL4(WRlHuDEK{3CY3SQL!N877OsrU0K zxxIIJ>%_lty$bQ9D!78Ac5CDCK16XQ|11o9MDSZrLYQpIYE2fy)=~Og`ipc-uC{>clWz(yeEvgSx-_3Za`XTv z=OD{(ItHnwGp|T17tMt}AAb;HV?Lyon<@y8OQUQuoQcixI>17qlCKRPi;P5&+^2jP zrs^hRwZ&6_wUx6_`uk~Ow#Re6<(X~NseLk>>Xq-L+JUX0re8zWT#A5mbBd@k=VoBl zSb}~{`9@Z_UX(d~+8;gJvWMI7<^X5mjp0o+iH%s4O)np-)-V_ugs-MTy7-Get(%fU z8IFx}2eSOY-xGccOPL@oMV$`9rsE=-I2nrszbsh$JD9s6@@CZbFv!thFZ$)zLmsh+ zA(NiybAJ2Qaooxd;TPE`%PA&L(Yq{Z;oAJ;NI4)8GzThkFmeG{_dp4?=t)ktXV%bW zPd|{SoK?Z0&-PUCwIFn*JDrr`Okq@SG~YE^Mx>%NK!Tsi^m$?d-WHbu4j;=AUK_pu zrY08<*1zIV;rr$6QoAflE;yHZ^++Hv9{z>8dQy?D*IWMkrnSh`Wh2==>51&(Rzu`| zIhre5*@goLHPQ9qe2b%vbLqmlxoZD)6u^-D_h{z2N67xB9qGkVcx>8j5E9;q!oO0g zZd53g7nerAl5>PS?L_o-=?;8Vr41H$RzX?OEOfJ70AfzI0DrAG_P@=_Sb3~b7;DKB zy56#a@6@|laUveg+xURkTX%&!LHZEx)63wLv&W7eFvInW#BvkG^f5RLi+DCKUguq8?n-JmMj3bk@%+^dABjCOdpVJ|+_c^8JP z>!+@o)+5=?c}R4z3=QrRfR+0dVbi*KGZMSlKDz)k zupn7o@!m4nS*wI*+DW5@%bJ02PX>=5){Y=CPPG*VC2 zh`8#N4+THMRbb)Q^>A-f9&xZl3akuv*WJOw5O>~)m1@PEG5XLYoC)#uE$Ugc@EIy=VvmuHycdd|qE;+E#l#j((h+ohgB`gAjEnCg;bjj~bDV{!s-`C@YESwC)x>~H>RK{3+v zdnLCq;T3b|?ZgvQ@{R1bJWIPgoWm4fYlPu33#k4^V@iM?aygOm;KW`z!4iw(toxm8 zG_WelT>hycEtOr4Ozv)BE$)6~u^5C{Y&DKPcL`+&h|d$G3EHUy3$uIREw;cME_B`DyEVYxMkq}<3&`D zd=9O-(u(fD2I$`#3tH}m7K`H~=+ksBn?I6Ax5m)41>1upqtZ!*y5bIr;QYCtkaun{YSuMO*hyQzCjR>H1Ii$*-f=z@LFJ zdb{2W#2oB`SM*kc!i~qlh1D9EJs(1*7@enoOWK<6_O3&gGil*Br&wAo^eCG9N|}~Y zw#1q`l62w-M3n17=JU=p+T`_N`mya@Uas$6y03l-Xx(EZ+qFA|DcqbQ3b$G*I>lSZ zY|qQa_v%_02jZf}xxPN6I|B&#KaMk}Lewa^C)>EN<`>l5ZT5_4ZoKxOZXVLPx zh2qU~HqhOhNlvmpfk`cTLR(1gp_eT%BHHSz$f&Ln{QBZ+JRY8lJ%;vkb2R@l)6}c+ z*Sm4du97e0ZhN6f@F|OwsD6&_>iTKw7p-IzR(0`1{woF{+Wri~YoKifvr(j392N9) zB{kD&7@zbo15J%_%!yS`g|*j}>BfsDc+v0}J>;ZE`)juHt_Aokw)+ZPiYfLcRB*DrEJjN@nMhNYNiQk6x$mk9S-3 zF$WG={(t^|t09_%f$!2i zOmx#yk?|*cI=AXSZdR`(lcKbQZvV$IHK{#Z(;pMYg_%$PsjFp@l}nhwUs!PapEhs| zazgd@+sTFA4`jU4ve7?Mn)BZA4)5$Mgpuj7SU#ha<{yhw3z(XLR9*j|R~JI)oTE7u ze{n31va0~8t?kI7T9RI}YdbKm_<xY!N+NixC#FgXZ{jsP^Si}vdDtd?|*=<)t8ALE{{)yt-=AjTkx(KN_bCo92VD*fJYRw@xUY2edsv) z>-GhEYl#cYCG8nKljn40H74IiECOyn!tnZv2RJ(?7w1hD;e3nPg18($widl(n$*O| zo0)0cm)vf`*`x$b)6-DMN-brU{7NKGg{Ode?*h6&Rh^l%XczqUFbk|$q6fTZ@8hOW zHsrU$XN=lB$Tmf-M8MP$m(#kmr-H+T-?@NUQLN6DKXPJh>zC0lZe|M_hBKMDe;+el z?f0aOC(MZsUIUren(^%j;{VvNb4#r!NoS?~2k8=)-5Nw`2iPqZ{N*mm*5u9sx z5s39a2MUp5!t33+)cB{1jO}-S=ADfeO50t}{1v*hCW9nxnR-x2_38<<*d!)srxl}H z@6Vq*)sA#qvXI@K&?np!1n_Uq*}_jtiv(GQVHUm53!}7pYAo!E-?B-80&5dkE8Cejk?0XHx9SmDH_2w}c5Nmx00N$%65QYTORO$-K&nzJ_>_n94&n5n%rkRXg zQ4}@b%R^-L;R7Ds2`MQJYn-QK20y%=r@Adtm6ER!gRbMlRQ&>7@>Qe>{-@SV?7vq) z&7Ns)a8gx;sEkOa(l;1jS^@3OurBH6<$piTEWauy5vLLh-93m7EqQJtgcqb=7YMziv%6(WCW` zj8B%KvhqLilis(H;?K>Az3*g2`dWp+w{7X_Jk{dg8xp~XXf(laNg^zul}M}MJ5 znU`?R1qds`8@Tw8YfOt~FbsWg1g8GnMDkKoh+i#6SW(}ctGMWjO;^}+zLOIX|57$< z?D7M?ouNf|@BV~`DyNX=?ghX!rRU7s=4EJIkqKeyVG05t6oB=A_rWXvPr&hOHkA3& z3A=Ujc}~X33M@O7gA%$R*EeiLU0eJRR~76d)}Mw{!$mci8n~G=;B-NfsTtK1J}B@M zsi8i}5XeNR@=NMxOlWffI9~dlsk5gPgPAZcuz(Q7DVHJ-JDNKyxrKIXlu~z_sIq#4 zL9}h3E&anxiL_RHjzgmtQqS*5(a#zrjbzg+;iBqTlwerH^3%&v^1_ptSdyo(J6;cU zWUoN03}28lD;9tSgQ@IUnSH%fCHd z1M3w}@yI4wT2Yw#RG#JQ^wFrq(}Vno?&lnIi)FQof$xL&G)ZQou?O_4mC+ z_53H`yO}j(BvYvMh?)!+1Yl-q@q8S<&kmXIk`+$*by()nCO$K_Kc61@=S!p;l@fnj zC$aKTr|?*L1^_i=%>*wWA7Y2;f6J4=t2Fc6jVZPo>;C+0N?3v^H6_mHA zf!vvT$WU(!7kK3c6<#djG+W+LR*D;-_~mxk)oe-jpm2WW$ywM_LYa#+xs3~SzYFhK zEk-M3)!55kCD5t+99dROa+NO4pg`*Z%xRdeb~rHvwHR;U5%?GqPmQ3#?Bn*rjaF$`O%g6)>6gO3td zLGi*r%&5OHdetWd6(UaXS1eaVmn?h8qVs`*L@{}Vyrcon=wJcv7@UbzE{5Tx8#C$5 zSas#%yixe3@-^b`VQ9t3F}{Woz=TI9FgkMyIb`PPxNmBs8cxkYa_oDy>EXtS{_GY$ z{JTPa>49rd7p*~peQDrxOcpqsxDwY`<#Y9}4_Nmt>u}lacd&GBwYk0T7BtbrlPg{~ zvlq+!h|O_TFve~rKeX#BS|)r0M)Hn>n`-A#(NAfllu<;r=Psi@|ES>Ar61;OcFrZ0 z+$j2A*ko*0)(`cXUz;l{%Ar>a6JS6LLL0M&*m=pXVCc$YRH#jgAk_32%Kms1we6ck zOuONLQq|(gw7Mi2QE-}KX?iP1UsvXeuO*=^`yXN>)eu@UWJIN|X#w(B<3P9Pv$U_b zKG@QTaq0T|;L&gu@@*T{J$pnRSUx+C>Y@VhKmXU%l&?yhkz6qyROp5}Pb@%YvuBV- z1{s8xl@wm9Cd-{j-GpxhY2XVhL+FnR5^QpR5ArE$4{Mj|*(Y-wyh9{sMAmbRKTE3Lyas!CJ;fnu!T}0jWDFjN>%;~^FE8J$I zNV^9rBBC-@u&yM9-lu+)?rnTZe!h`{f=7;%recp}Vik^|Pdi?47A>jx(ccDGdBYp? zYA?|T9$is0{dxuUXDZWwoa5=m7JI0~-hb@PU$ww%@FiMhDMxoaYXIGrfR0!e%Xa>) zg&8-@xTOl^@LPio^36GkuCHGJw1@AJrZ+F)?lb$?5Bn?FY|F1~R-+UBc6vX4l5rKy zU7HFvl!S6qHk<%{DqfT{m0|wuSJ1_Ur(gQwfn+18K@zm`Tn)|HKBUB%9*AuYvvTL;D(`j@^ z;yv_yYXX@zc`4!Vo6i*w>5H7QSK^5G*F_g%%o$>bq;T-iJ^D4e#v2^7!QU63;>+6Y zfkE@vYQ5?@MfZedlh$?HNg0V_RF9qHchgL4Z6XXn37QZ9W4y zS5r0ka(g~={aCq9)y*c9pudfg2-$#v!^B-&Q^oJ*j>u$R6*70j-_q&dbcl!#7s&qG zhMY!X5q_4Y%jxdk%U!m07hqFG?)m+8M&Vc;2z>R0Ydd}#Yv&TBSulI~W9HwdqnNxfht|vsz-n_;m}yt% z(yebS8E|C|7hk|*hIt0`uPY{UYd{(EtF4x4*`h=CD#^hDjS}LOEnmjkq7mve)^lqW zN4S_Ty0F}T7rXUN3|b{@QzuX7!q}q@$g9Q<8NR3i6P5=TFntQS^qv^{wQ{F!!+8hd zzQ+&@dU%^1n4d+ZOUiL;zCU0TmMn(1Di6XlSJsn8&-96{ zPH49EbD>LM7rehxjoh{CCp*w|i=2}>8^X_?n%k~;pmy_5{QL*c3A@{?K!Yd}79H;* zH$69|evK3pabxGWgw>to=6^>}`I0>R{hlm{+q#976B2~|qD^E$sseO14Z--@CU7sq zjheRYfZ!>oiS{nrHQ{f6AhzGKfIpXY5=*z13KkTlz}NF?L|w5QSJUeS+Y+Z(W?94| zALV3~fpd%DgBV$4|JM@P0yXgPMk8mdYe0PCdZ2H}0o91;5Prez<8Wx^8~oqhN7RAb zpZNPHMZwzh6kO_F0Y6cD$AH zo^rf#i!)Kf$ioGDmTS(|_MPB3%SgX{8_56aV}WIWFI0<^Blkr$0MR*0vCOJRl+&;xj1qXZS;4GfO<{} z8Wom;oA&m2#(XDAJLo?-|J{QM|INPvD|5Fp)8^`nQVq_dnM@+b&U?cYe6VHY1YPv) zysO0bAZMbYwGJl=D!4n#O+<Es8`In2$(&-gpGtU;~r z(-q79Rx-hstN3zr#Q;9Nhd%aMlkxqU4XKzg>R!Pwxw}7ZaK!Y9ydY&lD;$3!QgHA= zK2KdmQ&-UcdVFQ4m*z5+gAWf*an-a{4kfeQ9Nm>c8s?kz&GYlrW_e-8&HvgyDPfocu?? zgxrocT<%v>Iyz21Du_hxUw#PpRarrsZL*?&CchaN^a6$*Ge%Liy(F4GmDHEa#0rx_ zS-lc@T=O3f$NrS0Z&cgk+YY->L}~(QqY%!0+9nQSk_q&2zqJ1UxG=4~V&rIB16cIO z8Hia6kadeRmHWY#iZD?Dy7m{iau0i8G+mAs^xoj?&F4`j=mhsPvl*b05K66H4<_?U zxw}UffUXlAl;o=A0*e`$= zWvQsS=oXXmy;#2Pqb%(;4g{JjPB9P4Whq;mldM?62-UXOj(PJ?j8X4SM(1sn7}fB3 zShhQ!ma7nB?+6ad^oGBn^N*y_o6mkA?mn*|P4Bl0vDI=;<-laTyd#RW6q_VCWs_^Z z(Da|o*MrsI&d)gZ)wC0&hvzetTH4CMo(;?tk9_jzdReO1$%&a{*i7H9pM_TIWKqK^ zaq?ync3kzX-QY|>BE!3=$t7*jpp%AMh3Q)#(oyd*z2B^lzS)Q z@ZV@I>Q9MBM}j9&|JEX0n!}LGNC$Lm_m(LgC%uVPF0u&yG7*~hi9wfh0XQqIM()u0 zjd0escIZROgIzN>6TKyWg&TPxEFG?d5ABX*v&DC!(BK-2AJ?={=X!0@tL8i|4a+39 zSGmHYscF(Py^o>5Kgq{6Kl78uTl+Map%?Hnu0J2Zvb+!ZZnT^EYZtiA`$O)B~FmdUhH zp$W*AKP_`CZwF&FrIxdLs4luFzl3=xmy0Ww6fi~?%`{51TTp$bgy5A#1~a;FgdmX~ z7l%}+UN>c?TJe#hQVazz9N;tJ;`(gi(05q2TM~aSU(M`%(?vg%SU~%Rjt~>G0HEQp z8GiP;3SXHYiyy1fqUb$+%+Xu<7PlQ^nNNZSg6PCt&%AL$OKvkd5Xw_GyJXC?#io+y zrr!Zxw|dZIrCR!MO%>(1>?`}QdKc!wAE!yQ{`zLry~PzQUc>{QF1uOwb|W{o$sJ1$#dA}8 zX!>mZdGi%#fb2w`4=z_`v;Y2kOs?^sj3NsKYCmQK(i=Tjf_>iCf$`5SD)d%7s$JAY zj(@j-XDMy@x^to2PVGuC`XianJGq~`yFs4%U{=OTUF`?a&6<3-y$L9)JP+U5m`#P9 zd_nFOnJ_cP-9wL>43{m*+ZvN4A9|UT@?!mUMnfO6SJ6^{?9Y@bSA?L~V&j?2s7MS^p8wYnKJX$>ylL;JosYjso)4bQLswrpcVkc7kG0 zQ*q_xpH%GQ0C;bG4a|I)M;_A6Atx4mV&7R3B8oZ8&K`*o1sbeJ1A8;4o5pSgIC@0n>sz+~H$~2(!}toh_mnSBUF|#!t!IVqn$JMh3TsL^sSG}$Y|;9jQuwVK z@`|c&Qs;+W^1}|#L@$L$sIuqztl`EIQBEkIs<(N>ecHJLe)Kz~GAH~1^=Y9S^#5fE z`@_^ojp=~+C`dyFZE zmqEx{>lx{^#1Tp?JWNO%-{5i`5fGQzNAGwE*sdSi;Kc8176x0+5yk!_u`4?e^~D#m zqeauHzM#{TXQU-EzwiT1Zg4^$-uLjmZzrJB7cIct!BH85b;fAXHZM`6`W!CxumEjV z-iv3nInonsx-$G43+pz2M2jrnqvnJ{-s0>>Y?ikNRkiRE>I|0EGYwn~6Lm-=>s!Dk zwrxk9t<$jksc%SiOEz>nYK%6VPy%~joCID0$~Y>(4nsK#%l)|~ba?=g$f=C2p##vD zqh_SVY$difGKaX7UjZld8ZY49Mbu#C2dHZ$AYBd9kET3Kk`n3TpE;>P)|81A~x;-BH>Ye0Z(RQqL_a$|sbq|)`z8LLDR##6J&4nY~im2nJ zG#Z$62$&A+$8t6g$-sbCSe~#K^5e`%4gX5G$)y;F6xC8U{Y^N}pcm9%MgV_*v4SUG z@<_h9ExFIV1~(?n!9IG+*@8Ex*`NxRkzNmoOo!F} zr`g@S9~0^<2e@zM)E{IEZ=EzDt8Wbem+2)0Hkg934CFse_eJE>Qarr<07#_HanUCc zCBp@h2D(~s&N{>`{w)L#AJ>Crd-gFCY$4=AXTTQ+a(GIIl$jqz7fF}rt&I1}1jX|u z7jRU{5l%biH?wgt8eeJl5}2wQYfO)4=}p5q0=IYOf}Q91pg-d$ZoVBvO}TN3Y1(#P z+xD0Q`WsNd+#b#p)?`yO^8AQ@ESx~JJt7zh*C~wnWl2!gAOpM#OY!}E_Z);gg@_C5y~=Zyg;dQUt!XkX6V_qa^1yC5f!UQsPPX?h49{WN4=CSy&1p)%1q<3wh53V}01 zQkb0YDfFU;Pnn1rx0qgEIl=GjUcUL+2K3=G54hKjTHl(fO?*!WjBg}69_ZrPEn=|YZ7yA?9-`wgUl&O>i)hzlSJ{j^ z-%*yYNGNtMO?IS_qJP}Jie9=!lkbfSftXK0bptU8KtV9OSX>k51Ai zb4$teh)j0$XE6zh0F?UUjI_FbDSblj1^K)*8`M1;r$f_5(Yb*#h|+Z_^hcTkOIXf1 zMh)nwwa`8>+k|pIPN2y*GKA7*JLrFHiD>754sE;08Sgv%3dM#G{{>@P^|>j#K2U+n?3px!JUtp0pvkqe%=MdxarOF8TJv@-j!-L-RiE%wWXBTd z-qdwu{bV0dcJ2;-XoI--N#oeTbqN=LT3oPaMhEV=6UV?K#-QYI4c5G-1ahi=BVtak z`VpmN%$k&C#Eim)U~x{;d58$=!n|doCApKppJQ=Mp{R@feMpH` zKCm0dEgq+FmlAFM`wKm%NuJB$F48&^a~R!m7i0;m*-Vj-FXPmDpC9u_Lw2rBHn^yA zRsQPDXN;gA28X_onn-GIG7`JaWA)jS14+Vn0=Za1uz39qVB@dGCG5F? z=G^a6eajW-1?_J_Hu9^PXLYusiTY2EJ<-F9cK0#nK_aHpe}vOm~`|kGi#C!u>IZ( zc=?Hpb-p|k7!!bwbo)~>H=L-RjYI6af97Cm;X`J8ID`vcVoz^59l=@EFpQq(9GZ2T z$?S@}i4TrjFq59mr3WTGW4gwRn2QOPf=e1&K&o05g~jeB|Jqr|I2Grh?~`)4_f~q` zL$NYgt)a<1P&rIrQCO#XtMoW(pD#v_vny$5?{w;rQxbOGJe7Kqc?&)1y{4Ix_zu)G z{zebl_u)@bC*iF#`#9qduVgP?o`HtbPND@$2f>dWUx}t=Ex3K!Z+0j2#bM8l*jH4#hlo;;ENkIp{`2Gi_0Rl#rY?9Cpet|p}y6XQWgXBCoH`_2P?^cdN?WPjY4IXzWD}o28={U{M0$SJ9_Y0T``yZb_4-m zEj8jPGr{BdpzM}9w0MR;7;5jrioxZeVZS=ERgu>ZT^bAW*4d)M^SPXj<8$h|Y$-SE zt(E-zk;Bm7*%%aG@Q@ITyU1@nI1^WoY~V_-#Nh|WlJQcV{mALfVJuiV3fD@9fEVOp z?l`v$R3#6>q|@n>H^V~oXtz1(xU!eb-P=Gu5U+u)8{J5kx8J~>-dL%c{r9;=E4oOR z_qP!>!I8B~#DVIKej3|_Akg}hjG?(dx~sDY{#qx; z4-Nf9uitA=dd|PaY@Gj^!CPL6&MGewEh_G0UTosHAL+^h6|*Wels$^3rEV0Q-oXxmo)>1V5)@AmypPZoX;=u_IZ>>OklZI@1-y)jdUn8h=cM<$P+zOlvcMDfeTS|q` zyU0v^(5qM~u7qxHsb>_Gp0X7cBz^r!pKxiVo&dI_FvXp=%o)2D!Y%(YnfXW{eAH#n z_W#qzI~FBbyp%pHkSjT39t@`lG^6K%F~vE;wX@1e*H&rz(}h(E;aTHMM;%4#Fo(b! zg(tL=bPz+fJb(tKv#3h^3fu|G6@Il=2B(TA3#1l&z)p=P(Vtp-?BlqGR*7suZR9~l z-{=r~z+oG6Eb}nEu=EvEcJnbaYlESnVb^TJDOHViu<#W%tPtbm zE0n3be|oXUT5TBG@=v9C+FzjhT@0pGJ)>UT&m_MrDZ{mPB7*0eL(OYdo3g&6lOM4t zlX5Fs%vFCo4B+%*+`Bwqd2a(ArLX-S*vw>zwKIS5HMTrttDHl*!zH(|jg%#4*V9A2 zKUx02oQd*zVncwe)A-X^GvSpnmC|s%rEA#SO%><$^TacCiHi!a$*t>oR7d|x;!^Nq z(mB|KD7igPbmjd`!ttOE?CAC6nk^*B=H+_a>k7aZIXd%cw_#wuQlC?L-b7aQz9s|j z@&tSKT%htCR47lMMt=0xX^=>KOZZH2K>LPUk2rtrIA-t3A}=+fkf57zYkd9Vup5Q)Wt=COZAG3Qqg=oL7fvnZS)~`a}=sNF`LF)KuxfvVA`>@mbbURBWKks&`X5X`1LW2{FFCZoDCd7 zF8_UpTW_yGJ}r;&0#`-kw3>vsi<0H^JKm%3JMN+-wg&v@zDk<$8dU| zzSIPEd}Nv3^PaK3!_LTWt1G)vtb)1Z4VdV+h4goa^@K#OBzZt>xkc-oUiNyHKK{y_ zvoNI<1Q##;XZAd)j4^mH!FXw|Vo!xEBX{a2paWu0#rI4PV|e^^r2m2hF!SFVw5Vl} zmLqsjGN%eyJXsFZAGTtvlx8x=x|ESB5$AT?e+#|W46%oX_o2RZJY?iTFxGnWvHa>W zI_`ihUA2KH__x7}IT>((A?r=Z$*YWD#7jfsRPF~sm1--bkJ{qxztvc?Rss?)HAJRC zXHd+-H!6!V+ThPh2HAZ%h!nXefROsk9v(^t>s{Z$4|kAS=p_+Y`B@*i&hX@dqsA$# zu@|`WuCt)*=5ClBcp2)cXOKj<9O0xL$SOa!;Via#CoPHj!xf^f4l5BI`Z4lP1%S?}1AuW3p00k-GcijLhuGTX6R} z18{Yy0>u@a;;QeyBz5^$Ipy+H@R->MoYKVMpC_}ilm89!o%|bcd9FAeZDxu}W}CwP z5rQ8)&jcAg*~`1qvsq?xy^O*hC6){LPn-2Lwm^%N%s6KyJNi*FQf}LG2@d~wgVs(} zq?2ca5mSdFu?f(j0=2H7%Ik$XUOi{1;e~$4KaJ0R_6|g8u95iIBP;n?&uXB3jtgoz ztO3rQ5P@~RK^XccasS8BnfO!j{eRr<&YYQZ=GylqyCU2)f`?&Wpcg}ggU$58m75Hh0{nl!Z%{Xs3 zZNHbLL*r)Xn;$`R|GCA!icui`r;7x>pQLif{VTjEJ_Jr~9RRaMI_k%j@-XC*E2XFL zfHL)#0L8a%aaH=;z}`$6jcj~|{i|AOhgZieB8R@yLF;wt|F$Z_@HL*8R;r|zdE`)T zkJ_2x$rW&XVeei^3anqrgnkk*Kl;$*x( z$cG+shn^qYq4#D$-oN21EB23KYh!+4omT=J+UvslEVo0!Ltm`o(|X~+yd$KKUL<}h zs3H$2yrnO03y`qC_y=ZqRgpCVq8+#ATjHPN0$8QF4&>b&2Nh8O1pTq$q^_+1Hz-F$ z{?OqzDe6+g%LmwO$z~uC8%bR*=%6zUb;R(3yVv+(+Ov}3}=ypTloNYGFF<|mu{;nHpLxE?{Q*m?4_8{HhJ>* zG*RyE_croe>vrUEveMwF+DGz;it`^|APr5rr#l(@^c#8rU)sDLZ8o z%{wiB3k&A|#F9aF_=Qa^JFjF8(>XxnKD~An_huvcA>IRji!dRJBu41}@jt#^$wnUo zXyWWkG4j6}WopGg64@?uArn4i0GTb*h+?bt+?~oQLVlG!F!fH$>@zKLXc~~mgD{rwm2^D*vM5}*4BJU8caLb%6quYv{pZr~cryC7XR3m)NB8mtNmMjfkflMbIOr~%C>U`b#S^evL5 zR=8GEDBvAYcHukcbnO>8y7UB63OC}mL@l5yxC7i5^CC*qa)6Sb@|%9}VH_{Z(V%j` zHR|WBvvOs@awz1jA`DSEB^?(s21i}a@kFx(zsuq!ioQ1qj~kukbU)OfH>&5jSccD_ zp=FxRDJ$SiaaSg+0x{2DIg-9foUm&HAYSWh@jZhyv@P~F5efl&(V>AzHg~59eM{?^ zVA0rhv<+FHX%{i<2-86v;W_IpL`-?Nfk&aAlcX%;8n0Rh1C_vm)m-|(R=yi zEdP(_cAf{j(_;>`az&nHM@|4acS{fGjd4dNb!u$3=Sr0PPz~a`O$wH6Sx9+VGb;J= zg!f;-d$g)%j8q{0Ff!~@rZz8GIPLRU;eyIqCOk<`=&7y8|4*|=y)1P#`gCR`e{(zH zkM*Wf1ympQPx?pAdXU9PJTTTj{Olrp_HZ9Fu5^J_-Ekc0-aCW8zK@gl*~Bq(54}S1 zvD?YSHD$ouw1tyDkd1#m-^IDt>vCC>m+|k=gLv|$HfHODG*x%)8GgKpraYgxq8mC6 zj9YOXqtw5S*y->Vj0YWL<~6D?{~cKi4@}ml+eJ)(A61laXi+l=o_NYA^e1pRqyw72 zClz0JK}?#HEmEkm7R|r5oW1V^@{83+J|}Dxi<^rW^|TWVxA6%f{h^MMkaFs@f3<)ola^bp@w>(+4NXYtBobG=sUi0CAbOc9(9|K^VUd?Jv8#+Bu}T?H z{9hQIP0NAmWGO13*Fix?Z;bm_o=V2G7eIMg9@i3Y4+aWy1To7Mh@qF>V17&*WtSO* zFB^)o|IV=#w|OCdgP|c*UD`yy3A{<{|6xx%c;Ar~-v25$D=3LR;w{=g4$a0l+GFV@ z;fliO&M0^(DMI6P!3uir2YD#&Ivr{hsRBpSM4{DK1JN67BKNB#Pdi(0GcQNag@5`p z$)&7srCziTU_9#@C1X5F_0ehc3T8F=<;-2)y3uZK>x}}=!?uKT@L4Cc6sxDz>|AlI zcon@=J%@Ppb{T%tfqAc%F!ZB!8I#6-yTT!ZC{ik@nC$Xd1XgOVr)>^56Ys-G>c2;O zcqYlD@D!)bawOl`;KDcd{b?2PdxY7sw~(hBzmyyQ zq{{0p41u}X!z!Yz5#wGJ#;cN_2L_KNqC*Wjj2}p+x8LMbJ%=SJ$F5KK?2@s{y3dA2dGhTiT=*ezd z7%I1l+L>cPH4S@k;pMA=%5u?3C3%?}8`ee;!uDUv(0ii_P?kB+B3 zh8AT;a$bcC82dI~#zUmv*Z0k!z301;?xJ^M>Df>8-&Tm{$5e5{#y0eGEoW}alY20* z$X8C;PLXmV^+#5XvLuYN8=p8QToZWdFp4AWsUkv(%IzaE79K88}eJ`uY2C7G~i zk#MyCFn&LlgA+TWX{)2T!UpYQX#I~*YVlKS;N`zssJLS?nmQsu*M2&S<-28p;?`_N zHziB!@bA0i^rm|-zSv(N;uO$XgRyMM#rd?a76ETsgu$crr9eBz0#po>IAnb%9;ls# zOw-;khxd=eA>p#J`+%0lqmM*lO)1p%>>5H=aual=zo`9r|%hZ%b zk+5#bHSj#bkq-WlL9V>`1#j~!1s$;|XyDXAZs>eHkjnVRI$A6sEW_Q&Q`?rpv$^MS zj9V0S;>I}j=<=Z4kv~7+_gey38Prb7J&=Z>&C*1mQyjlf=^>qe*^ZrEHG=WpjdZm1 zEADu1C)6`_)f@}Uqg%Y9;A`0kkq+cYeO_zJy{TV8x?VPgTbc>2;=6Wa&p|avk3F<< z&RI-1b<4A#*0oS2zkKNhi*L}LPGe+I#40l3r~$Y1*fZ{^n-(_aISRYwBA`p6uVCgK zRroq*A=!H@kM-RGNUhId^!Wp7I@bP$Fyx&m54wLoi1X8?v;!JxD4z>rJ^ZOYDxQRA zk~s5kHsFW!ZD@T}0;`J{3QyPmw2TitOHBURMC$cK)2E9M2{IM`f|uT2RC4P@{y^L; z7_V@cPEz^C<-xo)3!#nuZd z)DX;}{#(C+mps!$_M`t8`yj829d10sZGCKty_N{*1xg`;veIwVu*@``VEa{0xFL=A zbxNWd-3@;i#*|#_bMFR`i*;hd@`Np9EQ2-gQ!kwidM;-jZpd|Kn8D&(T!i?wBO4# zE;#rk+5U97lFHK|HRmLTEI)7)ZW<3_^BwZ(%CcW<-3ui;`T1Y^a@+wpZ$}rHFo`DL z+E{Zc@6L04V@a%dZ>_NT>Tjt0GurA>_i1Q8Y6f1uSb}|<{mAK?q~Y>>f%ezU8`1bt zRT9n+2U%?s;NYtDQ0IOvC`&S;cRHTrUGvH3g47O>`vy|b?~RKEcWm{+rSxK}s)Qt> zdy5*#-LDPD2Kw0-kAtX9=3P|ab7#Jo=P!6pY8rf>=*pAlAAt*oHxS>_Iez|0Q|Kb1 zSxv-Ma$3)a>2;(7p{&%>#f=Z|L_F+V8a1wqg^9(H_J8r z)!|NWP0)b#O~>)?J6qwTlvw;k1}pxvPofo*WnpASBPkP}3NGDrX9Ew`z+Y@|B7oHf_t^2lF z1+91^UsQDz3McH~nqA+>%erRt+b69=wTwVcK{J}RH)e&&FC@7s-`lD2b2_h)T&tWY|%;5?l;Ay>(=23o(v9jcV(ZCYQlhQSuUTfq}N&+gV2bh zoLcm8a?E}ny?n`ioy;BT@QeR(V%Jv*VCIjhA0^hbuX8jR@q7quLAS}96X~31^EffD z(-YpkF2@a;>H^Jn8DZ4;6LMnn81dVIqUV%-#0tw=$UB=_0evAzF6GPv+V}ZQ>cTf) zSrdH?W}SErnKRP{&A;|u&M4Oj7mjHNSIYfn?CRzyooVJ!=E$xhpPm9 z0NO~c)h+nU+(K&Q_&MfQbDLtpGHDpLJ&!@oChQ~^88r3WW!6Y~F~4KM7Use)L&oQT zNWc1~0Mpk)dnJf)#EnLClUANKONlr|}ST;SHx|7mZ)deTE1aQ>f3nc5Kz}Hga@a~mI;ipwq zRz%i36d$@1uD3hHu&S%ETvGz`W=kx}Guvs8|+@pUinplKOgAo~C4jdaTy zD_0`rq}}YO@^!p)ARnDu?J;#z(mdt_b7sBKMtInX?c9I}j!UfE|{vN4j z4Z({I&uQnYj$rX~2TI#uggtffAeri3kG|Z<#k=O#z(YCm?9r;X^kLDrg&uM6_=1J7UXJjG=V$mfzJ}`JMrC@qLfdMiucuBY|cj!z2^QR}( z!am{>^ZUVE?H$|R!yhpOU;TDDQ&jN>bR5d!PCJVzY+r1cjq4S(KJPsYAC6H>;df>H zN=C#U$V=rSiD-s$e}qJQ8uassF)m)cA7;p zW7A)fdUsoeeGB=7zjiC+u1Dzh&hukVjvJ8qd-s6;#B!TT_LBzk`HEV_*y!hs+pT9fZf7~(_UIa8w^Vc5p9|XjUGnNWy5ZJn z5c%`}giG^i(>QTRzmuoOKqYQF+r{orWo-VxgCLe9=I?u+GK4gBz zTl|0izw@RW*q^_b%zRWn&AqgZ3sdpuwyJKP?(Q|4e{Jg#PHF>I@LRthm>X1b;_LE= zvdhEpYwj$g6^Ab|+K+rd|2HX0 zIah8Cr)*t__iY|yGmDJax??8jp2Ag8-nIa4=hUgOTLE1AL_hd%Q4$;~U#8=u=Y}%# zvPi=lQdGs31>p9Q{c!2cB(koPr2?Lo%StQ_!fQ8dA)hZvN1n2KxTgW1z`>F=T+?kA zT72g`szUZJJxO8;PCV2|P1CtTt#hm9Pm<=N3u7bn)iZ~9fw#BNR^HLFQIQG?;^)57 zr={C*VD2MapP^1Sf9hqw6@7x)#V<9|-4o$#5g*7VxCU;`YXJGWd+^(^C#1qvP1x_E zq*xpt13dj^!jAQp9Dj8W^{KmtTfAGHwqBS;w7kE=WNW>PD}>ZhX>!rd5fZgD6Mt3C z)A2hE$rBec$^KvVJo_C_aXS!_#}!*)y7q6p?LYxIp>>#>v1c;%vFJQ6WO0Tn+v_+NG&!#z;N4w)a!J+>6KxD3^hBC zO+8x+e}0|C2?9o$xmMPUN7ex16xNbof}=pz=PW#xw+oLt%L#ImUkawm%;e9lTFRMc zePH~&NnYyBeBnfUjO=Lh9=Pahu_o#%W{#aJCZBnl0M+DU$U8!lX-`R}Z>GNi)kp4v zdog!evB2}BjLa)WMcj#dHq8&tG!DalN$RMwYBj{i7cx(3&A7pF52iBIAIZ9>Go>C! znWet3nbA3EV(*@qG2Ur3`1nI)nr$)Scdb^#J6uAs9iQO`xSqs^bT{#>mE-8t!U%Nk zK{M{)a@qICyZ?4auO~f$w zVp)b;gUHtgHN^PaTJ|Dr7ficxhn+b;+-h@nF5j=%Sum-|fFHTR7<^e8$_lhz5Z0+p zaLDqT;+4iR2Iyk4*ro|Q73)Q%Ja0y8(p|dcNetz%what#jAl;}lK^(n;opi1!XqQ6 zVaIle9krLE^OG*Xe@Su7Zr@7+#mCW1TSOk}y7!&gcj6f{^`9<(vF21l*G7*b;wBS3 z=`^`^*+{Cb*OI%$$Y9GGCe$hOQa17YG@4&GQ)4i9DRp+S6#ZC9f?na1N2QoOFq?AGb)!FZcJw?aOUIc`oj=80jO&Z>Coa92Lrq_|Sml$=4GK2DlD&69 zmwf4=2PQ32qo%ChL$p*|kV-B$Wcg=v@#f)TUfDfenyp*Ju9HnB(`H2CqiHpek;T0+=F^S4C@0M*EPNE|uJ9&b&5&T^VnD4SVoy*w% zi%qyYn+|_gg@X=UfcBx1nhDKP(ErF5*x|YxrnhF1HouT|5PLdE5pOSvs`6)LCwF^lVZpqZ7RP zQ^fbL)`W>cV)X6!e%|@-GawsslB@{!le>Mb8%BAFxJ`Q=aiVh;&I+E72etPhC(~5T zENn@ix7vsT9A=^6K4VhPkK)?RNb2rINyPslW72n`g!WZD2tOCjuyWXU7P-1;mVv`_4b?utbA>Xfh0tl9t;d#;5cn-JGlX#jm#EjmqRh=1bND0KKMVwp`zlgTnE zMKtl0SZr*~SE(vyeoE%zHMKNXE_0A6vR%LhEbn1DueQ;XzZ#;(-+KH5kyn^I?mNJ8 zV+rux7NehOz!qLpsQ3Dl^>uimj}5hWi2otkmvmC?sb}G5eS-)TEYA7=)A( z*}PIVOKt^T9dLyG)%}%yx>t#RFmj3jMHMj#owrGO-!E3%WC+scur1_X#p_SrxP|#8 z@{R~K{eWEcclbQ754{_erPsZv0i_o=0-`_++rAqn&G`k42R|CO>-E#q`WSX1+y|=3 z{HDF+#h90(UW|*W<0wa)kIHwt%Mb5%Vb*3RGJUuHBd(t_p{0G!@a9U7%l)Vo{pLU8 zIWWnHlX)jb|I9Vz<}AJhVb>j%mKXD&p@TeZ`FjMaDic%@_G10Z-jO-sUucD*292Ef z-QeaveOR(@1NS)OB$bga$=Qhgr5YXf(GN^M(|yhIL|&UZ&qhm{TdX&VlYhR(e;|(? zd$HT+NsyDq>lB#)|y@k<>ZGcCc)RB<|)9pAKEidHe;kN$m?cspJ^D>+`{J? z{xep3TR2X?PhSJ?y!l2R0MEgKC8dH@VG}rHsR4ExfSgK|KHMK!!#=oh2ue|x2$lPx zqFkJmh!u68jto1_dp7MSto>X7{G1nybd3g>zflV|(~qbRzsjkaY6gUgdZa{*Rd8&M zI!b!Cg4I&%qM1$yHu!`-1o=_)pD1N$0E)1ASq1Ii(M!1px5+(mJqxcyi45ZBl4O;y zJ3(pBdZM}ho1CnFxk}iwK5kJ#7asT%2lso4bAe}eq8=M}_0ZHhI1n=j9V-+d?cGYC z&mx-*(ftp+Xeoto9c3ILA)4)p|Dby6HhjzEEM=bEj$Ocfg`~ZU;QG_6;p@Jy#9C?| zncQ)Z4ZnN>Pp0%Rv7(ffa@~N^cTK`s#Xay;tQ2@XIEpr!a^)qtVP=lfjZp7m~uH{t4 zN^#1k;1T`sl^gzUtwcW`qiDIh`EnEYyG6ZSn_)!L8rk2$eQ?R;#WD%ab#i(peQ2ty zDGs~h!|ge84yi?+V~@_3VhkNFYIJT0hQFpSVnUBfGIvhbkRAO2c!}RNV&9sb=>5?= zB~eEinc*CQWUTuInwkmJ99?%@d-|EY^N0WqBwm7k;wDlTy0Un=t$8>tH43X%B9YFg zjMFyGV77V3;lQtL$oAS7!ki+x-nBmor^I}^fBe1b)BQQ9H~cWqb>1eTZuTVV>|Fva z<~5VSv5!g0dYMeQY&4cyBt<-_?n83#WLVyR7s0LN>1>eIFi`z_2~1-x(cg-A_K}$- zlGo6L`>u#H;V>C#*Hoc#B8l7?5&};gen>2j)K^&GSq5YJt_zfJSF;xc)pY*AD?vog zP1O4%P1E-DRP=PV1`^MSLe{)PR5}vFDeMtoeP$TCyqKZx_rQ|+e1$}dbl$VyZylqr z=!gRno5}l54-0hj&WYu{VcHS*<}kWG4HZ+tqpn&f2L+ zeSa~R)0DjpTK=WN%1ATK8wxwn@~5vzDt{97J3SmUd`pF1GcS{5n2y8Bl>TsN&6J}U3YZQDd#_Q zcyJf5m5@Xm#AOKnA$hsgtL&8M?#bMSPnWop@2b(sQztpLUy(_loUCQP_AX6@docOV z@=VNL1z;PnoO78~OFUFcMpf2!nuXctNvCr$$ZNAW%b#RSYaBYs6>7di`90H7dE#C8 z?x-Wtxz3-Mm0^t+m-KVphVSst_`^b#|K>As8#(r?!$Tz6G4g_CQ@DPu4@B4FGxUYk z_S%+RXVDb3Z-mB6XENb#8f9*)iWqD)_YGm0yQvgF8fv@$gntlT?I z{_6LKOm2+1h{=tG>gmVw?8MZ(MhhiL1WE9KE!bR6Jo%J8zy!ScTiknb<2RuX^6k-Mw0klNPAb zKiRqXdDlUxRxnEoG-T1$S@tmO$84DRFdmo;hu~FS38bs_Nri3q9vS_Sd_#mz{z#uX zk8!6cn>snxkUMcbSK&-x0G)dJAf55{5V5vqC2^iIz|ChXu%y@ywq;umH`#avRFj&C zvzFYacN9-2N5iLb{n}Y1=bJ{4cJMXhA1{Qi`Yp0nvoDj6zDI!AJ-g`cDb{5Dz9m%c zE>n`cpTybDTSea4TnM|=oCGB<&&i(0bMW3fFNvpPOUW0T7tuq4&-l~crKGg$Z?Ldb zz&{zI0k!i4bZO&a()7zhI$ZyYtkKXixYKf_(&dE}!Vjg9!Z1r7otoJo^r}gL8ng7& zM7dwep-KnFeVGk!8_x#eMl(KGsiX0F~27e%5q?^i27jB$Zaupv3|^&1g)&>r~S-Aga44J6;n%2Kl((_~|tXyFy(9#YFH z5_-M=#yLqFgC8bGv2Sl18OQ!4sKi*>>i&MN?r$d5m-q*W3+Ky=xGAvsbsxR%B_ifr z{Dumb$CAPBgL0kR7pB>)P8i^NRCqK09wQy~M%XW>#Lp6LSETO_La&&e{M}9je`HS< z&`uS{dseQY6z(T62VQq;T5NBpFQ)Ed^tx?@=Ob(2`d9K?mh5DH?3!7O+4v2lyEKQe zI3q#Y-^u1W1D|qdT1X)ooy!J!`SRENh`}*^ADF@_1=+UNa*jx@BtMKDfR&ScCh6A| zGegq0L_>ibkob~-R9#;pWz+wf-v;!B%(!kxx9Ut zAM`)73Og21TtW6TTH(}6d@V%o=<&P`S7v;GDZ9F$(G)w7*G+TpUOfSl4@2OQ zV~^!OJx>L`zjwobK}pzQy$oG+=PjojpvE6mdru!vuz=StG>}`SkCV!Kp5gc#i||G_ zeY_yq1g|m??TVY_x&BLgVEAJ`I2ZJaYkl8HW>o6Hu=QJ27wcYytcd+y+&d1OXM|Cf zr6dFmJAkO14x9_y2Il-diiOb>2w5YCv~_E_o|i6^O0T7Gndg40ym1QEAhNzKJS5pS5)hW8;h?msu$09br!8;Q5E@ZYS9$^0H`g5~4g>%ACKZfn= zZAC)Cd{Wf$IqjfV=@O`>dHNRJJ(=k*4Ia_u-6xS!ATKkda= z*f^xb6S8ykHY?=reu5fApAqXsg8p-tiEVCeEX7%Jag&v?<=z+^IwZ!wtQ#*#XpCky z`IiHY6%qnCYZ~F`aRZJ^0K+fD;C8zbfEq)FD#l7(eYL7@Km9f!n=+I%<^@s z8PBCf>7Y9&^3l)^pSR(H#8GHh{7z-r!y@|Zj$AnC zegJmPzXTo~w!o$LHj>Y#&K7lvXc=F&-$8zVY6v}iXu&t@>(t_U2h860RR}T8rrT%N z(b^gd$;d>8R1&(gOYEHmzs-&DixydS$+{dE7xP6R;=@4i$KQ$fn;zh4%Nj|PT})pe z8=1A-)UrxL=yGTDu*2fD^NrRoT0&4o}7=`|m-tb(*M!F%;nr9B;kvFk{JaA-| zy!6$*%rYr&{O{uqu6M^8M$7%7Rf=m4vnQcj3%VVIO{3!c#+o#yRXvHE&)9I&ZXE*o zHdv$-W$x?|8zc>ga)n3oISvj`2%ph`2QvNe`04fY1A z;!okglS;Ic|C+*IMBP4XDoAbJ=Qx89C!JdMGjvrKcB7i;3aPVfTvs4mUzI@5?0t=j zU>x^T|1%0p`~ZK4i!(*)+KgPs28Mn03%FZ8g?iQJ$V(PK03SrberYj|+giYVdss%F zPuCM}lP^LU8X4+ebCu!iMor{aEJT8o(-hI{g)1f{frmr=us3wEPM}dF*#}!;HdTU; ze0)UP)YRf-@6_c>p7P*I>v;HK$!eexK!8pKO>FKT$z8*7IL2f>)_tOYRD8|~lpc3L zL;Gsb`R@;VSuYP5rh7o$2T_K4sXnOn!D3+MCrNz?$fNQ%#=(_qrvimy6}o)O9bRwJ zWxQI+8<1(qs52pqyEdjtrOXe)Po489U?ovrJd(=PXj>O@!hOF6_%$v*^QzlR1xsh3JfiE4^l< zj$ZUOoph?+N&Z@Vp1tX1%4K<{Vc19tr^ZFY9gc47{-0{lb;T1h@J0ald|Da#2a5Ky zdRov759>$HedO@3Ruh7#>th{c-6xD(pH$umoDz@wK>;Z*;- zAhACW+a^j83w8_t$`GdXUd-sGbR*H>c8{v-Ze_mRzOCjfZ#+ zGBc6kg)++8xE>3C91@z34dkgr2@IFxjl zI5=ONcJ}Ny(l?YPPVfdOC24c3MYSJ6adbC!(=%84?50TV+Yw9clj!C3Wv(O+x#r-i z_$v~?eW?TrO6&TTT;U-gcEf8$D+-LZ|kJBegv|_)l{mRDQvNi(8DzUfDu?>`NFvSU|JMGa7{V{^OyJ zH_}*YeI@LTJPC4@az*d6SYRbF2Rc8^(}|4sM~{a>NG5I@Q25pX{#*!#djeue<0F_} zdfAxxn$XI9aJ^41|CxfeG#tZs)=vhX3MKLQr);n%JezFaxD75Ie2teB-va`V6sp2d zivfjyU~QHvd`?>t)dA6Poxu|_4d0V@d?ti}v0_|V$yNNZaWELZx|Zi$em`|7(R;W&ES} z{CAH2hXrtc{yEaFP>iaW#>e_cX5htB?hBHxG+Di?9-$u^)Z;mRQLxt~hSyMfiVGRv z%&YKeqEEk>kgZAUfHFrMz-aJh5aNvy{60?KTwq3T@*V~Cx@$r4l>`C1;5_*;@hP-7 zBw10XDZKAr&0fhm2S@04bob*&^!a(;*~{rEG}-llipvy7PKVN=iGL)8I@-K?Y}r0uqZ2|VCx_{4mkCN?Rdry^xI`5zC881mr6p9a>i z(2lHnZOiu$lF(Rh`G(N6&!N{fcjHOKZ7SxF8z-M4O*5Wq^q4~o{rS}hA@jYAob_B@ zIQ+BGF1ga#r{ozc?Q`!Z1 zDO%Ezk^qZ%5frfwQa#JBpx27e=(?|>XY|?~-p^Jw=FLE`sLx@7Y5vbwMfFA-HWue| zD;@tZjaJtM^S&eX2`$Yv6U&Fsm&ACo(bRM`h@;&|!c5qu!-6t~2G z7k^FcJ$7Q5BLA480kQR~gs^5LkC4>Oh9A6Dc+FSeFnFmX(7N%Fq|eu(MY)TZ5xWK& z&iq6jm)54FzKM8xQl((_YDK>G{Y)&Ia~LM=?PW!!N@%`WGJJ77n(^A*#0{+vXQuBs zg#NbmGR{V=jO$l9zQ{+!`;k}ycW0dibs=qhvFjD6_RksaN?#}EC<4LemPz9?QnE}^ z#$wH`*$iCHo52V~etF7*B=2kw#_tTCQO_1+p>Kvl#J_I}?Her+ai>wRLrw~9E&L;J z&B{Ts-V|zH*9}us)5xRECE%>z4|ccdO*Z_kDF2NFxHp4jjC9`$JYH3y$qwe^4(-=? z-|#Ql#*qbZ%S46jhZBWp<6Z(xw7dXQDksqLg z>(J3%3056NHE_m~3c*NrB#M?k3IWtcQ!F;&GHV)X)&tm*oG$09l!7Kv1<2ogt89UM z9J+AN2b{i$<#9m*6I{JPs2JidOdj3L+;_~wqPrfGxaF{x#)}_tsV~8wYf-`2jNSrY zKkITHG@wO13dZJ}r^45|T9V?qB#?TNG7{W5{$V`5}v$8lkqJNp>zHPjUSXZQ?=Wj zd@2;VNLSY&2zQcihY^HI0s(DLO^3nwl*Ze0%#GJjQzF%gSy zFGdYMXBk{^oJm?f&e&z%ChOm8z<`)zJcDn1g;jx>aQ|=(9(4rVfma2zL@^&1Z#jXs zN)qtrESL&Gf<@&PT|sLX=DF!P%hS0SHBUz{O= z2%0C+dZ%sSpP|ceSCtT4({myJrkQd-8kD%wf(>};F>Ar{HKxdWcn{Y(r2tOZ>r9zy zOyO(-tAXaFRG3+KMQduq3iM=+G`7rcMC8Q$$y1c$2SsYza?vWcOQxMaeI zjC`7g=9su~v*UCriOS2|ZzD&#BmJ9Ubs?>(zxBYtRF)D}Q?_N)6NoW_tQ?mx&q+Xr}h2M6)~wkzbtd3Ru;Y7Q28p@7rAJg$or z@#1>E1D2Ji*UYVBkF9nG`5#|`a|3+25#<25qa&Ms{6(F2*Wm+7)13jF3{~ar4-PU_ zy@!SEhDt(9*}IG{{XY+rGGF7`N6mVV9;kMH8~^4}8U9xfPqHz*8y}gSO$}~1z%2D& zrK0ru8C^0I#E}0Oyr`rS>IdZEw>>YIFUL8?=HqiDwfz8D@y-#1QTMQtZ3X_^cofSM zHSByNJN~S9C$Ylldq(~HQJ!XOE9doKFRxQ41xlnvtE6o&Vyb@>k)n4AkaR0V`S}{m z^%*GkRd$lb1OSWNaqb4%a=LPfq^esWc9549XBp#n#r;qMZ=Fp;g71I&;73YV# zF|+@!LY`$Ua(AMSFyHn+XGHlonQPO`nQund30G^Zu;7#te^H1We%js3M*A^*;)oX4 zyK@u2O=AYbuGxcp`pfy$bqBt>OFB3{$-zqERz5ZDb0cF@u$0N0sR=WhD;XkW8ur;w zBdxim0*7EdzVVAVM#OPs#Cd+CvB5s_?pc=Q#l2vAp8jNW9xS%n8=b~4^n_OL2MqZ0 z8--x=4MjE~=@H@P-3kqjBovpgoM2X^6p`D}0q~_`5H(!%VzL)^(miI%l*2V46&lwq zh+6Cn3a{w!KZ$(aCs&<=7v}JB&@L|oTU((1$rQ%oQlp^BA&HUOeH`rw{lGZIJZIMT z>GD_dF%f0QQr>GDWdn}SldH0hq>_y$xa>)>xM;o#Rr^B76^jXJ$q^UzL;Kt)-5@C% zJorYbU?;Ld+ymFR0FN}5uW_oqg4KzTSjO58Nw+A@*;apOCFFu#>udwHgS9ezpO@fpBBs{^U2M#{t`?7_Kx@Fkv{bfvs5 zHE7;uZcqxXX|gqYe-OhX5@2tk7Uh~=!}C$-Ad_=53CENauIayAVzH|(-E;gu_6a+m zd^>9cw=>F`Y#L;Ug8#;WgZV-H2#OTA2?kqwm5KY}-;v6OZ$;uE`e zH#26J`{-xymvJvUZ1H|k-f7=WeeRP@Ieh8nuQ6-tI9-3rA9ghc!A}d$knK4LORt#; z`sYZ)t-GZSYWK^48OtSLcl#1t_)?NyN?hR_Ld4O-+u^iL-5FZfa6ZAipGVl8l*gsg zaX5&KVZB%EWBI&Tc;8+P|55lsKh`=#THlK0DiT(atM2B|?>=udd^@rcHe9wK)0*c3 zmzz_;knkW4U24JW$9I6u#RZ_U(~XOuEy>R>GNE+!1eTGSPie>=|3hbd*)(9!7Ez7qH>4tG-H@Js|PQs~k zPQt;;WnAdbJ!t-$Qgwx9EjnzAKdP*qhhp=+NskDIn;49sOzujfFV{^?_gt%>-#(k_f(jxHNOA_bpo`;_WEaC1RfRC@s3V{OFO zG0*h)#FNX+c%2E@|8b2~$;LBaxM>FS-Q^}qdaX`Bs5fTq5=Q>01Bm5H-=^UtG21$oo*nvH#;sMHh!+x0ilg#cu&V2V+ zj{hF`LQ6f^##&1R!;TIFlaMsi+ccAvKcu(Rfu9$IBiT67NeC#!Zx3j{yzq6{T3(>yVNb)1~rzx5qWF%DTMotrB&Uqic`3MXVi{Dvc^78}?c-0pj{&KMV|e$hoxtu_C;ff2NoRLwC^T!B zAZqAIAkAkp`A8y>PCFb=?BHhuv$RFhqw9}ie(VM^^?MPVd+!IH(&-J}EU@IpGbU59 z(;fh;xNB6|i(0Gp^h02DeH(BHiGnWaGhjl)F_^u?oYmFh5vFz&xxfL{Kr1Sdang^!kDtk!I}kTvHyr>nrqA%?fptV8Z_K&O(_25auG{bGFeOd?D_v6V~T>4l_< zbkybTQ=yUJ9K=4Df{y;zNX~FCXPK6v_P3NwF1?4NDC<9OY;{NcAB6n_<4@h10 z0&mG!PwFQ9A4g~YNX7Suab>$RXU>_qu6>V)D9gPw*BY%#B19pHM2PkhDoavGR3zC| z_N7!pq6iT!q!5+TLPe#0`_AY4)BOkL&YbgpKhNvM@&|K8wDxCABYSo|!!?wS1Cy`* zxIoehHJvm9_fm6#>$#8k6Q#LKbfZz73;r5VtC z^aRba$bN_JFFfGDMJ-Gx`JofJH|R>Bd6?IJbRjWs{khT=LpJ4u1; z3PE+qW%LQ_r|o%#u~`TBeTk_8<(@&_5Gm%Y3Veq>x0prv*yALx@i=ai+JS4Y z6XpD!Urr|r%&=2y2gsgtPX(vuT?di)IYGB4+mXB+4-Wa8lgovC0e4tHE{ah=W3H1@ zP9lk90PCB}xN{s2mUKtIRJu7wMxnImXbH;OMl#crPtdb2U4@H2>d+^sMp9j(T>1B3 zXOiuGkfp!(k`G}pF>ug>t|qOB#)1(Pm^mQDeA!Rj$tB2?mz(Lv2SxDxN*N~0Gy~Rm zJEH~BL#V}i4Y1x@iVwJ%Gu7wQ=@HIX0Wy{p)VMAnw*fom+2bap+HObiJ+?9p((JqB zLK+ftU81~VRTx?Cp$~$unG+iQA>iufSR^MG4f03ZK~Sw1_I8xd_AN8;vYq+lue>t0 zeYuqoEZNNXYI(rQkHm zfQ_dur1A$|ky;iX1mSm#ga^#4$?Kx|LiKDJ>h0xv71geGsCL$s`bhIA+jv`GC+@|( z64l0@*d&nK?}^JxZRx~izr~Q-)_WN1lS;66ie%>Kw2^d{fOWsLBY&4{OfzXCw)Egk zT4FmP43bl1f^8$&-vv~}!8QDpz&0@sF;a~X6aUtdb$uL1S?{0Gw4{Ukni_&fecW0(mpIY~dTQfIV;OX#3; zR$TYFWehvZr}WpQft&S~%*IC>fX8+%aO``BBTDSDEFQjp zabrZKR4xfFUNIldIX(}q;3R>rX`S@a${Kv0+b7tIMjao_>3gytadR^+{h<7tWP#opeEU^t*!fSBDe$_AS?|;)WP2pIa-xdJ`aa8s zezY1N8yCV1N)ubyolL36ZIfH7TOqtSQpi}Od?e-v=9+wYa}*8QE>^rY>jzq8I2VPwAgtn}^M=YNg#jIaC9k#d0!18lP zq2dxHd_n(Gtm$qP-O#Z^xXCS^xpwmmv+SG{(hZ9=-eDmrlbZAqt4euD-%iuU=iU&) z;h6b~zm->@3r+)=dxjsbpMU|c8@pkIDk99KY)b$VuiCy zBtTgP!JOXVk8_*uaqdioLb|z{`S5Wmv471|5LthaTbe447TVN9#l!O0dEO*>N$m*B zNsZ>Q-8SlaPKogMaRJkG-~k2S?l5VOlHvY*vqmoWOcM%aU8M)Thx z4E}h|sN_>Z&(KfCq4kd3k%!5&11C~u=e%^z)wEFN$opydWA{i@)ugZfl0HY31sP%g z?SMQTqsZ+lYSiAWQ!E28kBHkXN5r1K!l-=M4mjVRQIA6|3(wZ9KrJ8NFh}-HC)JLx zKxY=Lpn{)mpnn@Kpr+xzr15nRZgF=iwM|`%E7RM{IpTc^p1G7Rv3}5wH|ewqaU!@3 zTUw$`LO#NnD9D)Xmn5~qy_8MRY4qaWIJ$lR7CPvx2rS}o=3~|%FzMNiEJf{*ZOcip zz;P>bE?vX)X)lIlos&$;uPiuy@lWXPMWY)(cjA&_udo@n6X{l5kvZHtho01Q6YP8X z7yX#iL$^(jKriG&!R)Pn8PK)|_ZF8%=cf5gONi`5!($@A;_PE^j8~6;a=;O5_ys{ZdCf_LwM%L7G)>_A zHJn-jo|EU}j*>3RMsU;E>v*7Fvf#=d3uby+Jbe=53U&m(XBqJ^#u1J;q|SP$fuA3> z1OtT;SXM!55{uo`!qs6D3uX4wL!kyYZt+hEgjxg;C}2 zzB4e)Qy8+@jrwi$iuRPsplXQ@mEquW6!&c!cTHsqbrP1cxrZP0O6jG<0zs8gvXacN!7yXN&Xejrae%WtxLrNie4ZByxileHy;6^w_-u|RXEuS`Kl{<-yE4*7 zfwh{HsuTZwUg3XiPSM;0wfHuv2;O>_c#Q#vnqND#VyBhvY#5Bc}Mk|y`}F}@Xj z>?y-_G9`5E0kp2EcL4~5bjJA|=| z^2zMa3l-lkWIcM?DrD`dxul=#Bf{R{0{u=j8r&Ou#yU}rWjjO*@ss&|XjtN!z*cw@ z_JRaD;_OB!sz#uH2?tPe^GVQ-Q=mvkhK{=(#VAfj(leu12+ZTu$&m%yjmPG+BZH(; zFrK|rU`VC`vk7-JPb5S`CBTc^y6hCby7wPgv2Y)esD20yeS)C1N)Kj_>B>BCdO<%~ z?F33XlgY*NfY3m(08D>5N=F-<0%bW*a6H)$nT0-LHbs2_D^|4h!WK6++jGkMPR=>CG;wMh7_(hs5~EQgH1~NQOc4S zR3fiW$o&0DV{>oeqUpM5`~gCXj_=0vqGzHVMG^dfi?gALHOm}*+)Ah@2f~dnYhW33 z8Nc?V8n4;cBG~mbmGN7Dfj+(U3$yL(L4=)K&p)88fYQ6-@E}W3X4jtye06RqT+sSJ zJmyC>qSfzVQO{HV>(3D%*WQFrIHF*f>q+e5V4y$2o)HUk0b64`kgjYQ^LckLc-8cP zKBdx+KRoDzd!5dLM=fw6`%M7@CMrbl(brs+1Li1p9EY3hcSvpzxkfn7|H5fp@{?>C zDk4_3)G})>-4-?rc*MSUUW{$d5g2=*Tj3_&P4qW8!*i2-7;xYa?v}63YI zL_ZzZN$8$~>CAmiD4w;Y-&Qt)D}$lTK;{bB@yo`9M_C%N)`yE3wP4uvRj28voN~di zu#7pbHzKg#YY(N}gZP;3cjBQr&Drk#f~j`hkJWk1BgC{Crf)O1g{BZ+cWT6d&*&Qx`gnq4go55kjyN*5lqZ=FP?KwosgIA zp*I0v+?P54o<*2SKk31t%!8-E?MVwb_u@3l(p`ZUWTa7%*WJ+v)*vNeYx(HccT&d(PwTV0`(|}zo+5o;Ry9=HrZ$ybIQXtV&3l7~a z<&5r5WbB&{*gH6(F%uqw3CPpX`YoO?^H5Q z9OQgKhIL}(E6ydXPxZNAR(2%uZ&MRmcsdW+CT)Wc#AYHArV56wA`sBw(B2L|;fb}$ zaKA0P!-kpo_s}#Du6dPScXkGSo%2bsJ;H&n-gF$z^Om4T_IaR{W{I4tv4!-v;Cr0z zfO^);3Y4@rUPfxOOhE$q2zb2XllcZBC`m#b)-Wl=CgVEX_2nSF_=Yv`d@+RjXUOnZ z`*4x0=sCeL=Nhyw@HiCIe}J7vnS$|<4}`T_0b&2;zLZu@5)xSuf;PlFU^y$=@O`-) zzCUr0cjXLwPG1y5XL*Ft({?^4CNwqaUHxw8YfqiBVfGOGmLP}ZGC3$&tqPEjC1~v* zukn`n7Pw?;hFjiFz;0viX{Grf!tW*hymvprAum!vl?j?}#xT@$ z=L~hI{xbhpDM79nCmF35d$_9{{3$!WCiQ1S51z1O76>T0C5Sk3oc73@$5>@FG7pEh zbHDB>=SLlt;U>wxf(x5Shxc~W4A0B( z!{__NO0kV#ue2dEu^|}*?U3OfQ3ebioQ%?^{S?G^Eh3jT??v^cN2rR1XG~7PAxecx zB$ti4~(WPg7x>Zwo0!-pP{f9%T0t>rvn>sv8l^7=GV*??psv?M7@ zJ%7cJhmz>}5;JOn`UKe|_8wDwbzRV)2H|{`rRQ^Tk>;-_Gm-jEnw)vMl0PB&4N3Wp z8rk;zrrMWCkaynJBBS*&_?My+U~h>7-OVWwc&wRAt1VqmKkiYYUaqqgJiA>%{(d_f z+oPDqA8EJ5-k7+ePoao2CYentgnh*WxOqT4`zKks=Pj~x%7<@1dlS(mvxv425sd0B z1+cC9DmhLL@H4DS(Ea!%enQVSa_O$^Xsk$w+}Gts|5kgCc&nzPeZA|X7c4wV+7%xo z=S|*}4AhJx4?FQd@CQk$jB9a}zAN!$k;kar7U6l1$Hr{ zy9WdXvf<$VxpXFa@B?jHX24X}_t8pQ-%}Dk1XInAq;44KfEj~#=&+|k{Ojg-X#Crt zN@kP?HE+gt{D-46a1Y-^jW$-YnJRrWc$*}mrJF#lfwSs+0ZNSBn3?Hu4bw``G~-MM4hX&%}o9B)oG-Wia*zDHHUju#ve5GRWiHQ7ow2% zZl!X6w*T*a4ZR3nPgU6OV|Ls*Lylr!_}hLxqV5kFQy;7UlE=@o9Y(VD&9$RKc%V)7d{=P=53Y-Q#+RnUVM{PFZJ>MT4tuNHF?1M4!v&`q_m36NR0wp z%E$EqJ$zyrwRfcr`AkNY`|e6Mb(%Aq+n`yAOOA^mDV_;tH{8K{Zkvu~PFXSBB=&6L zRRJ9ooEaybQ)I=i@qq7{~*RHhNthOY#2w;Qk`aUi z^@&cB&p|huw-PVLotdkw&q-13D4Hm|pr+L5L(cJ!2KtR0Vk*WJY`f);_BMEef8RcW z>pkl@4oA1q{qY63d0Ha5ne}|@Q)Ty&RUk$`pW0zC?}cHRABw zw_M?ME;+r{7L7M7!0ggDz=3<8ITn50yatD_FlVBNF>p|0);DRxuU1HC_9ERTKa zc01DgzMcM_cmaeuon<)kP2jTBPY`?UB9W+HCCIKj1m6F<1M*j2mQpkIL#fU=@cZ&= z$q$!5l8K7nu{DjqxU)N7Q~Q#C2zAeF5_-?>qz0E~GlZ)=ck)lD(l+~k^mTF@m-Qxa zm!fs}hQGnaFQl z4qe&HC5?(HRJS64y0`HjZK7^X$?7DaZj&0rFOw9PQ@e*@XN%nFB^#u>H9W`*_lvp47Z!0>NxOoVy#>bMSJ|Y{ zz6Ofpcw27%3m%#{T}FMVm!N~#o^6hmj-dCtCU@X;9CbyZTQ14p3jZzB$L6>x{yFoH z{DKu;e9aStu~fz}F0#)uTB54WUA`g;yivc!&n>OMhN|D9&ORWX*8P<_lsk^kHQf)C zEuNAXe;viM=z@>Z$$!yEzKgIx{On0p9ZJ{lcrpjkBW`!;)KV@cFsk!m)u5f3~@s+OZZNAHB+3U zNJzw`G98|?q3QcbMNQVLwoXk9nr<8?l)i`IlJ6twkwwQi8^5O!+xR=RY5!tQvT7DF z4=iHj91_7nV>71x=@x16DU7grHjS_zTh3X3?JVc{;48ZL>mtTQI)UE4aDpb&Ae<`X zGvY%*govU(_V*i$WyMZoW>!LCy~1qOT{@D~f#fT&{43A>7-{8RJ zf)oY`3E{5|dDxQA8nETceR}bE8GKz%ER((@h*O?1!g;ykF{r-0k6s)ffsbjn;A?tr zQhH8}#H<_g1WQTdtO!tmXPw+}@l*rK?o2-%R#s)25(DUcgB~z=;1csB_Z)KDdR*y5 zTo&9u#}}Qg4n%LpvT^T!C+M(H0BVOhNb&ryxXDH7sH{`)0*c@7D0yeMQ^=u)X| z@4{i=-D>!C(>u<;HP0n4u#f?wDF3V4q!h<=F*rc=a}L>|6%07@xre*6QPhB56pg zJ)M!;uz`4YqJ*(FV2Hz~@`*FN^~8|n2qU%mDp)GDoY;H8k9*r$1H~V|3C&Wj|_aLW&WlPs!ROYp~8R3pA}E1%DfD z0oMKsr1xyCrwbzdnRQxv^nIyiWXRA`L7U=Zw7K{sCo)}&A)c(loP?bar1&D=Fi6_{ z+=0Jn4FS@9J4o}pW=K>*8=kn8K|sw6@OIe|lNz^7VESH(JTzx1ovHD8lEm)As9_lkgHUQzr=b`N#%k~x)>e~Wxq zrw4~q=29jv=F!!XYf!VZxggLbgOa{0N$tnakX_SEuu}6?{1N`yBbVwW_c^;`WFG~#-FGzYmnqL zPJ`#L05D^LHqaRo#Fm6YL%RBGX=XBeA(hnwac6F;%@iIn*7oA3-;OA}XX`3(~QUTy7SM$c5C zPqmn0rm8f$en$$?Cyyep%2u$y$4}52YX%&DPQZqACzbzZlneIGMe|dVnTgjEL}{Ws z9hm-_xKa`atBFq7)z*rs-o3#&`(T=oP7-INwCV(34J-LMw#(7Bg(_@sDI2B*CWCuA zO|;;C5>P9-1|M&ZRquKkg*?_;;-W0$Y4OoTcynSL%-nw+6O~8hO-#DIjvdm!roq`ZuTsIg@S)O*Uu z25t$_fv`Z_)c6e7{aP?;{BW5LKB!6$?S2biC$uqsRm;h-FdM}~A_(3x{fFER&m{Ni zM}rHmbm>XSX~a-R3#!NMlyc-A679WlX#VZ*{8<9i`5vkXgz}EZ%pu91Okw557q({vlzkAJLZu%9$@401y zk~B#a-1~?qjFv^|o(fD`;2`00q>OM6Yv!uoEl1S_UWl@dIr46_ z@4`4i&wDL$OKH0RCG{fNPz!KfH=o`Rvl}ltF#^}$PM@h)AxU=Y5};MX1BAz2$FfUo z&|yt&U?AKB?4xIc_`X?8yZ01caodut`{6?G{%1w(=qeKs!UtgXj6`CI$qAS`yPbA@ znG4+mjo>SukJL5tAsVf?fkJgZNU}Q&l=E^x^16g9_nz1Q`FiRIb8X}XlbFZu|9|x| z_ocX)z0Y|8IAt-~ytIlvXFYCe2ld_c1OIsE5{E0C_ehJ~%m zXI^)(gSFEGq;p~>-Jnv%-iZbTf3ljvuCKvhV*5fex#622%|Mq7^W&jvQCylgkw6aF zmXSSkQt5~wW?G6)b4HD>z!>fqG#FxmQeMi6JoFZ*)n&}djjp~G_>;3 zsOrbKXp;4bfk$OlL@dU7zE;Md7_UpfI_d%lf1!@sXnQcCN{;@noq>~ z-emU0;4s!}39Qdw2H`y&nmwBmsfoSBqC0%4f?7>dyfh5qQnRp?)K?@grHwfYW8p-> zD>CWzaY3?4lJM!?YO=Oc(j>J}nwn*rqI}BZ9ZX4erF`y6QB_Xsfs3vc^Y^qQCu@2F z>3`T&#r%mPZXFs!rW{K%)CW3n!J1k@Ydxa84)aN#XE&;J8Rs+&nR9x2t7vfVAv3Z_ zkS)@+`0Zb=@iYuUY!{wqhu}dR$niv#P~-PyYLZK$4QzLWM8Ca zco+y*7^`u=PDv|%*^x?47Ov)6z9+b~70$#rgEYGOAqGWHouWPl%-4vDx{Qpaw^JX2 z>*&Cw9CGpdYNj@HffPTEraHz)NfBQ;pecJ6#8EGpQ*K4{^vlhRj7zlO=CQ?GZU*aI zecev|O8*YF#40lN<4b`3w0UG$e4p}7>7F&U4ZKMLHIAEkaOOH)hNxTBM^_7Vrz zs}sCG6O7tWF_`W6h??qMA*{Q%jNQHcrR91!)S@tJaz>98wfd$tGqQ!CGXH3>eSnoR zf%bWnVrnI2x|8_Yar-L8h0=fKELYKZeV!Rg$)bY!_WF27g=nKrl0l$(2rWp$yesW|=rsM3SkkjoJ2a8VefX^vO;@s|Z>6|sm46!^Mw`For zw*DVFql*Wyiob%~zI?3BRS!JgSxvM(2xn?N)WFSkJBj$zF)61Y6*TfW3m(un!KV)| zgIo6)b1wf5mr|9=hN+uRFwR-lwBgYn!a5_sGX<(Pp>^3cs6K#eme;Mr6@?&_z7 zMb_ujF0B8cM0tQIk}?n;ygeI%$KUy?we@fQ<;f! zmN()^KcHecx+br3WV!kpK1#PPmm`lqJGpuR7&okKEs&PhX3j<~AT-_MDd&+>s>r$< zs_xICS~gk=zrMeUhH+CS@#7d3eBYe97F(*op7Zc^e>UQSOQs9Cvz(ZPub(nXj&F?? zI@xfy#J{2KGbgC}USn)BLYr}^{=hNoj6xC5D%5V86;Kg;RqV&m7kt{@!{h=|jasrY zh3&pBBV@Z`r9=B8n9$(ez_aW*W$?2}sMhI#=3PF=NQldkEnDo6b+0=$prtQt9CxBV ztND_H2FIxA-KkVt`yeG+ql{gy@r14Zi*bRhK5vhY54sS~XN(m3`Kvxp!MbNf%%dlg zWc`3Zc|?nkBF>JYIqglTI8q+yjK8J7?vn-*I}e}_hBlgQ8uLM#!!9JSEnsxIe4t%m zHlw`3h3Clq4Sk)B(N8=V`}Jy6GU7%y?fpH7xtKUENb67MSBPQ_Jnwv{+UffIQg)gAWhGr;NJUE7mKT_0p+f)IZahOcVtQN#f-$z~8HbUwbCy^n? zns7Y^ItZl&?C|#bf#7YAMi%0XCpV}Zu{pUwSE5?a3K3SKlijNUWFc@vwsNL zeQ!f3J4s()G;{zcOS@5HA(zQdgefvK)+4-I=W%5$?U{EKC3v_;6xBZ>AvBmXgIv~q zj8@ZqO13-ckwp)O$kNSc7z_7DWY^k0lUv?M1L|De4X z1-;c#oy8t3?%&cxmNBLD#W)CGHuTe4_1$EpWg@g)JO(%BP6r2Pig1J{BKbCNdId|0 z8|Xb=YXpu5wxZOI=SI@mI;c&Gi*Fm!W4gKzW2~DHTIc+c@ZV5^<{7j9^WZ$(*;oYd z>u`~bgyt=`~xL{br5?7^p=338>Z8b z+DYQbxm@Cm-5;qfih*cO`Y3Eyq&YL#YzFde<_yhqq)KhVsZ$Eh!eHs^LPwi5)Sp#d z3^tlUx$nNDG<{bex@R<m-K?@`CJS z<{o_eAzVXXCV3wPt< zX7zkZqqr9=tZGH{|c*Jz?m z+Q-m}_fNoy4~y7y%2mO)>Uqqp)b;eek$v=R%^9SL#42O9??Y+?*Mer>^Zd~t*&wfP z9SToqQ!qc{LlUPoL?vW zxuy=N9`I%C*4+XwmA}EYi~8tm8&5bgUJ3eR2Z_J72c_DM{6Kz}3*oG62<)DYLF2or z*skOHs7~eyw5xFE`+P7Fe)}g0r=fb`u7-z*>vT-nHxr@x^#@RKdIU;eGoM(cqspXV zO2G8a40QEgq>@7xn?W_yKm&Taj1EC-ctdXi%`d1YT=ufxtEB_5%4!tf)3*lK)}KW` zEvjeeb=mZwZ$ER?D;D|98aKANq=1g#Q~03`h&H_U8Q<++4O!-q=!3z0GbH|Ugj|i)}gfkN{+TjHs>EEti+NeFTB#^zFg}eD>iXJ_}8yb z*#1kI*k++8bTf~E`KogjGc$Pbx`{K~=cfVnj(@;px?1QmgU))PbZvC35%cF&j@RV$V8` zlQUb7&<)>|L4eUi!MAhfK)Cobu(AWF%}A8-c$5Qb11=C6iN?IX(eLP>eh2*4@rH9L z;S}QAC0Y4`>{X=c zGl)}PAAu`BbjZ{{V#u=mJ`n+a5C@BXf@S^-gkqOPiGLZd(2FN41n<6mha)XBY0udS z=)-DXsJQeEY}5E47&5XT9G_h8&r=6Wf;Tn>Mhu=6)C)l!WejX z#(AbS;x#SU-bG}n?P1nUTZ6ivKt+FY1l|Pls54LGzbpf|c*1|U*Wf0CWt?>1%flx2&l(5 z028?!sjprG(BNz)(fd*w>%XW>DL0+P$t6M}NP%VO-;rd#dlU;dp8Q2UYSa>j^f`0& zrspcL93!N$D4iR4lh3u>u?eK-exzAOEaA_)NX1Uqm9>%vNb6lDr7jKlk{d=x=z5P#4@&sG*JCx6sDzG!uV!HnTT3oV(p4gP*uThHK*df%DyB z6Fu2<1oPuuLNnE6IH%xKD#PV0uG$m?O0?U^*QG0{twAE1g*D-C zMg{<4vIVlUv-p)E+XVrNJpzl4Z{(hiboyq)1#-&`3VU)?N?6;b%L$rkiZTq|DV*>u zBy(SjvE7#>U^o9gIlkx(Qe8a^)!WK|x+6yTNTmtNiyq?3;Z^e8<9>Qi?*;T|`xpL4 zza3<$Fb{1Cmm~F5+URqe)LBM>H2QIe%av(AMy`}8Agdhbh|irLLoVOM1Ew-hc`lK0 zRKT1h`tFu`X18H5RrW>9m@B+Wx#sDswQ6@G^A8x;IkT9msPzSVE-hug;2hYHF^|&B z&{lh5o`J^g0EHjL*!g7)*?!E2Ie#{alInOwR%=9&C&s_yvCG!r;^~`(JF|Jr^&cz; zTYR^GWg1bdY#oIk7DZA$^)2|V!ZD%I-{l-J#TKNscBRJZMjvW8bvFLG-yG!scA;)i zm8ACSMabLFhIksXgh;b6pjm1^c&`^p`DPG8r#JFsL#8$@{`@iN5HgcgJoS?Fv|*SD zw=q&h<2f6s$fOK)mrq zswLI9yeSp+((7VmLf|IuEDv$6$vh|UZBSi!vi2qlH;q#o^vuwb>#0P`N+qN=l?5?J zrstC4OD_r9KlM>|$l(9{f7@SYK#)lX{!DDOiQye*#((j2rUUFU<(@LU?@eQeO}!UBd2=@b};70+SQ;tK2NOV0nu1uGrJ$lC{OkDcDl7vuYZN zZ*nC`|7hZkhpzzY9s&IB`zTiUOJ(w`7~y;BCu*Erz-aHe2*U@Zm=C*-Lb*j2C_i}^ zIlLXg549y?*RI$Jjdj*C6RZboGBwfA@~;YMRbRn;zI>fm!iW(as}mW&(Q*(HxgWu0 z>KY?f0pxu~3sjHX0$}1cZf_od^6LihGtCcihp^oe^%I5kn}gCIaWsLChob! zP5P!IlvB#crp*$>>ES+YXxc6`Ug$fBrpGv_IC#V(B;$fA(o2w{Qx?eA!|8$r5qR-{ z3R26~R$}>gxcqMmv@|D$;f2b>Mf6MBY?lHQ3>||h8+6c*vMI^JZI?WCT(@Z>xm_}G_GJ`jt zHW@cLz9g`ozaS4;atq!Eq1ve~==rl=Qg_W5IU}(c_pZLn`)~iRP~k(1WyUh&d+$)C zpN|?B-<0F}uB}tGnK*Luas$}Anv9np_nYPd-i1rc z$*pHfxWb8rTqWljO$SS;S7B=O@+-3B#SnY`T3t#J3yU`X1+Q$&nH))zId0E`P)ul+OsN z6Zx1>x>rME@(5lu>KbDH+~*-xMkaaED_GKcx|Q>gkj#IONeeeOQoax z8i}&rVacx4PVBBW0NLZp#H<2eEECSgFP9&eY&2&xs#S3u%Xcc!b5BG^7U&h%GbLAdJ{HWr=KQDvv%ct+ z!2a`5HUklZ+7=;3Va61c8@hpi`*nl4)3TlY0G)t_5yOfb9{VDUsp(9Z! zdkCJq6bfW`BEUe_2OLS+%{SS)Dn!TFJWxHf z$Yj@NLolaHfpomaVX}$^$Z*$OdUe>>|1$1qm7^!hq`T;v>yyyefr2sX`=#7wO|W3U z7PQHtQ-XcJkWzQ`u+AExlp^s}|$0@G;7edn9TPl?E*JfkD=Ao1gZzPd1Mb&Nhb zu{2N4eAQNJT>lWB@)iJ}!_!H5?qhODg@6qbvk13OCE#TdO?!E}gJD4~<@;_Qojmy- zemLJDDE_q`Nw|ohtGh+1qD4Tk*jSueHOwVV8OS@Y;zT8`I!5W+mtg5h%JAcRJ&DIM zZoE%h?#Rx4Zo#%W1%i1!640uwR^T~UhKjyzQflwth`4XKXqsFC>KO#Y=%s6X*ZP(C zu(>jCPUxg&o7QD~Z&(qvYdD-igc!T??-i5}P;hxgBs|WQKo$ihoYMQL;@4U{nV`*v z3{PKzSzt8Bcuk%;(tLZEIZ~MkrS*I8_g^Mx|DwgfFTDn8-QK9Sz#|z)k(N_<5i>@w*`6lrZOf(6#b+~ z4ZYb$Gw1vrNfCVoCHvofgigjsbYzD%sg;%oo;V&Cyqcu|Jmcz+^3pF;KUxZXJRI^U~j&xaOxSn)H4^=T=<7=^c@Br$0P(lYl>(kI}tklU_RY!QHku_ z6=?V0htN(e7F)HflOA2&hu>b$LGeNNIQ5DXXuH@UQ2G8CND?nVoBedr(7PT&X!@EE zE^5PL{zwVa?urptHY<~1C&L8o*>&(_nH6op-XDH8^5MN7@8!7Qx9YCK4yQ zq?B^bqby!Hsxpq`gx;Kk?r2qVBEwvGjn5w;wfQ!Tg5*nnoktuhkX%NWPkE3hgMyW8 zR<)wa2MbBD1YNRk#0WUsT^C&1I|n?qJCC@F>t}7P>cRu|UPTslNp#e!A4Jf)BHC`d zvCN}FCp1Go1nG`^#|j@F#nv}(p*Oy3qp^4ydRrz#2X9zQ`nVL)LE87w&L2;3^POtU zoUhaITc)g6dtSOqj3S5p)KrUmc`O70GYWuWggHt)qzaw{$O9Hp3+BCYWfmLhgR7C2 z0q|~)BkaF zCVn-1{~y<)?wm7o&fNBW-)X%w=SC?+)+CA~BuSBd%Tn44l~O4!6iFK`5+O@=QQ2D1 zN+{W~ezN`U_xB&%$KyU`&YAc7^?E*$td&UTQ~P%)KL+Bi!LcN6ZYQIsF~omedX$-W zU?v*q89_-?gfQHC2B<$WkF2S7C&vmeaKQ&dtfx0yV*TF(WZi;G=(xu{*jZ`JPl(wB zIhiPQ3K!^gE{VZapFQC6Kq*?ms0VKU8I3OJTSA9=b-2OCjpDA*{I}!YaNh46{7mN- zr=7f%-blaUT|Td-w+%g`qh`Ed`uy{4EvGX~Socf%RJ$45e9A;J!&``cJk6yB3`b+l zPC1bIWT&w4&l`NuJcvwsE)+{VVOaLlTJe=kHFh>*t)sN0nK8}vVn?|vv1R@F@M)YY zNx7*8`ZM<7BRZ#a7474|e_^qhHEHBymH@`$q?Wih!B{ac&=KzsynvYH)$)G9IneYEDdU7 zM+DfSvYO_%Y2eJVaqzA0GkWaZLf~?#lFS$w51ViOhb`;wkPQpu=rP-k$*hXg5R7>Z zLRuE#ozK0r?WA2?liu#rBF^z@En)-Nb=UB)+j2(wX!n=N{rVC7O17h=O+dyED9NuH%YVi8$dG^9*HE_tj7H*VEVCzGbSYz{D zC{nhRKCsuEws4f>5DsR;Q^_yb;t5dPtUD9uSMTJX^uWES zMBt&L*JOm0kGY`bmM&;uZW^4Qc!)Pkiv^A~znQk-&t?~H2LdalVF|P73LpJMgYMN< zA>>T6vX99&Cf}lf`RboUg_k8!ZmRv-T}a z-{;A2iy;VklEPf+anQ81b4MkPc7iX{hXB(*5vnBbVp=46-lenuN$%BQApH3(GCb)x z)exPB9#wAQoLmiHLHTul+6DwWH%fR}siw>eDSr}w{0^|O5YYX{LWS=oJw+Ec5mPZL z4bW1v7zO)!f%&8sRHT*9loeHopZr)a4sn-ZZpupA)9;hflb|-un2OtU{d^siw0s5% z@W_A_Yp(O>7mTMo9}gP5$7 z^#5-;)KinqXM!1Q3YhjJk`a1EgVi}&bYS~dgJf-R3mZP_hS(=cO*mfuoB9s74rCy|P1qrEV1oWMc*v?>WZ3CT-;&Z- zK5Xh~Fx~tPBjC2N!+L4r84L2!kieb~jk_QWcs`L$sBXXqLCn5FI&&xQ3~Ex*3Hmwe_>$XyI&FQW}k` zKWMVA%VHQJtHLZ&5z)4@j*?v8X|Q5!A3M8!ytr=REaZPHj(qHWjjZliqDPr-?1z5| zqIsXp*yw=8c)+ewVamxvY`#$=8}WA(r5ZiR$Sp6I+g$xwvF-Fqv`9{t=x1z{WI$gr z52j7vsH5-ymN%wf_cb(mIC29O{cnP_G zXCr$1^Ao7q|CtZrho~$i8Y%l~=^2#YL$6*A0Lcs(E}y)Ep6Vrwu8cVXy=ru!aY8)Y z>EJ}lyaC6q#l!mqU|yJskecz&)tU&Cuei=2W~3`nRnpt z|0QBcwjL(VuBAc`aD34hIpS~O$KT$P#UGK}N8%fnaK@XW*op5jd{uu@G(*->VEVca zsfni;PpsO&9{VvK>VG=}lXlKx9Z5CLJ3SdKHSwSy-z%fPAMW6-N7uqD-I46oPv7`E zbM>%HoF;$U`UVblu)>b+&#;by8h<0R9d~y(qkqy!>F(j#Y{-sac5ZSC*ms{s3xC#7 z5~rJ@@}vKl%*qmeT~#)zjTSRILjH)>2!>Hxx~W=XtQqn$m1M%#??r0M@}b8T4_>$N zHBi{ ziJ721V#m$#0(q4*eJ^Og()|sq(C}1i=G*R>0?*zU{N~F}puLGw9G|k2b(LXB%U&qK zyf0xZVl7C_o5SqLj%7O5GB&8%LR$FQD3#qi+XL)5V@;ZKk3!yGjlFoZNo#?(kSTB& z&1xI2;hyf&Lk5#f2{UmO8;@b<^&9FPTq$Q3h)_?t-IdbfD7zPGQcolO}%i z=+4`kz~RJv{)NL{@Wmj4T`L_$M%$icY<-)!-8;7+|J#9#^O6DlaD6xVulO}Cee@d5 z7{@8h)zM{7&RN0sow@{{TU$&59fCoW1Vl`vng zYxul4ht{vv!8g;zSbytz`cITEZ$ESpURd!GEwHFGytjHUv>a(fyIa+W#AVLJ8q6j( z->)dD#z8c;F$+E9&%^rTl|V*y2mj;4DiW||J0EHv%O8BIhWB|iacb)t(S%3~ddu=W z{+qfSjg?oRve%!CyHi)--jPP} zG%TpdAXBAuXzgcW;??$s&M^B4mAM&c{hky8gFe!`9KX@p8JiUgR>@&aH${~CvmXqp zs3DP{km@So*Z|VXBxf%mT~P{Tg^VmSGPFbN`}jRtbC=SYK^da^K6{XrFdkh9)}z7W zvwXuwH5j?m5-qwaW9&Bg805NWp!UCoyq~Nx6Lm#ioTv9qLFKU*Qxfx>8SD1~T#uP8 z@#U#<+YYz!r`t_B^0D;XC{RTHBfN6?3L z81ySv!DeDkJ8ZkfH zTL{yV8I!aq@c~msy3@mzu6(vYVZd%GI$br1dA816@XxZCPSK5&pK@}5we>5Z|2FH0 zy-(f}SK71m>z{~|u)>&8_fphHp0?7eP43Ke=kd(xqrPBqEJrLKwo$h02I%Sk%`mm^ zE1-Uy`$SKlB2T(~pTQ+zCx~n6A?50M*7W=Jne^VNnSy284Jf1am-w(QpiiyT>gmb!gyBmHD#s`~ow*^Ic^O<>fY0TeGyhqI;} zqz9d|slWHWf|0W?sk((X#mcXj08Xxq367@3UUvq;1j#%h=u8ZlQ*f0EtFfk6l)7`l z%0H-4@=Kuo6J4cCFXfq>Nf#vF7)Rm4oMcJA<_e|#aFeimI7j8jt0w+vxIF);R*88f z*5OW^KZ4DU4eK~Qe1krIf>`vx58p zp(Qh!;=p8LJ(#Dcxm%)7~Re@?l7R+JZq7Rp{-E;@fPM~kvXbrQsSmeiAD2YBL2=17Ok03z>Hg_ zgI?^80JAi{!=a%OUOz98Ule4(6$Z%ITFeSW_x&wMyoA#U(k2R8@6961L(?gx-**|A z;>!vbavM;_GYh!r(Q!E5d@SzqmPVuOjhIDcru3@J(eRhs0nv=}OQGlKE@U~kQREzt z(fssUE=IZz%|bPdoO&-4tA0jwlvQ93YShrluNa}e`vG*Xeid5%Cl!3##V|z{As~B_ zzEaVGXvXIINiGYm=U!iyW8~EK*hq-;Xi4Kf?P8B?W>b|i`d@bRO3^M@_w4{j9{YiD z>o7jy6sObnXbPnsmd$W)myoi^ue9{nE^~)Hx zdAQI@|IK71Y#`8Ja1G1^KjmkJZQv71lI54}T!>`u2Jsypb@1!h4DxDV3~kq%3@scC z8RUJ;j-GdxUNrL?thuO(y@Q`KFTx|3sp}%BD>t=RyXgnOTG<=yfkGu=y`}@Ho<@sX z+kUZNP76sfpD#RV@=yCse+_c9&l2vtVJn=fS_g+tyO7)V{`8U5YBnr&p0U5(Xw>yQ zhvlFOH?Bht4<@yfy6R4)m@#YFWqRuD_~Ii#C*K7wTfKl=kz&E^Eq=zEs;7{_-(kX? zI~VYBBg(?C(*iJSjEQ*WuUw##avtrn6>GiM?_+h&z5v=?@=)2a8p~gBV`mnfVeXE0 zV4l4D4*O%`iR;(PaBq#Wu=wg)BAmG&Ex5OyKk;0Ine&^`)0}v=@N0lLb$BHkmvsP- zE%H`~e|m@gE@p*r?hbiRw=%TqRvO#@hlO3iWw3lwB2iPmA>PyD#D_#9tWZy(t=t1epN2#ndMpp06JC>>nG%gh zPArDY<}Xtmy*Qrr^O-NUS+P@G{56C%a}VNgZac^B3}2yhMy?f^U!a6r@6U*21Aet(mA8`q|8xukqD&#T zH=U#hjU_)$W1_#IHxVDDx>C58?W?8BQeKzsE^a{pf(6}SEYdUWohPQ%NkY;K1& zxbwyn&SZkvYf2UP+(k*s@Et|ZS>FvEgyEdd&6#kT-yYWTstX@#pn=n;D)Vmd2e9dU zHN5o5CoC$GFf!A>;h8lrP~!6kiV|NTE4MX>{gU`e?*IA!t7G8n;Umf=ah>eZ1bw^P z&mqZaz0R&ljTcLa^n_AUc6w*(o}eXHR|yqu<%O~X??J;B74h`ZPE6RF&6+YLImU&m zm(U7_CG2UxLQeL4K7KP_oqONY#twH`vQX+j$!~UrhvT>l8ZGgA}!yz}yG@2LrR?aL|brSqf2_y2mZ zpSR4xqh>#5C9Xhr#rqESw%HBvXhf1qro06^Q{$CdRkDy3^N!Cp(Bx;VEMcDS*vfTE za%GdgjMEBL@x-s*O5;X$9^O?PLO;!%%0=kJ!~ZPT;=GNQRg>mZbP)d=t@g7f^RHZB zX2v~OL}II0hAD^h~1ek4$pmv54sZ7RfhK{9`%eUxpjhXFp{R7Ixw<{-_x z3iRP~?R@^b#Zc1MmGC4&^j7ZNfMbmh!r1U9@JX)}c;&qkH73u7Kgafgt-HJB!;Ad+ zuq0LZsxTg(E;i*VD(BNP)*K{ft@UZE2x;cRo)qRWbKdss^Z?qu>I{A0ttp!;tB%dt z2sEKiS)gd`$CTZtsF$4!l_os;#vHc#&Tm*WnfDCWW2lBbylH1Aiaj6Ec$g5z}g7 zgk1kc%Naf{kh8k-o>)dNCeFu45#T0cH~&r)(%AQw%uefO7W*oJvR)mc-n#) z78>YompYD?*UgmpAdP`}|6Qtk-f6~fvnSXSX%67r|0ol`k9@;Y6<{#wD$;+OV>d1Q zD%57*kf`^eKBg2=Fy`PNFk=Z26M1ZWgZ*Nw|_P@X&{L@Z*s^6uw(c zoH;F4fYK zh((vPK9juo2OjQhQH(Dx`1DhedUud5Ui*yQg!JSmpBu|g8`@8CErv=}zNW(1^dRo0 zwxeCoLROd)?@E#yRtiCvnZl$vG0r+rDm+`{B3!&nGGp^wD57h!XosZF>|aZF_PD3S zU-z+;Rf@{xFT7B}e$iJ%1@eZ%p)b*FVyL!?&&;;~oQMUgITZi(_BAg5n>}Y(v%xOM zai0)poDmgWy`ymG=6$eMc{f+~dMxD{eg+9Da&-ngezQI!cR^pgHFVnd9?R7(Wt*H^ z7;k4k`p8UETJG{cQBT4FX!25DC{B*$xA;|~iaVa%pR#4xNpKN8jZ9^u`|sK5r)IDd zqw}#_=m&O@RTs-nvlV)UlnTZ;>(TEu2IW81yjCc>7eSwio5}0>|K(H_RA`^~X2k8I z9pfI*uHmY&m_GDcib>J=POFrs0GYNF{-lRLb?j#*ovo~FaBkcVf%VNC+OPH|zhC+vV)`-LnCD&j4BZv4M*1D5I>GD$yz`S)%l{>dQ(k*p@MGmeg|e9)jBj@wQQm6AAG_eg=#7|)S2UF&-Are#EhUaf;yXgY zFN0C&#>c<})bQ`7LkR~{4z(+tG>+m_;A+X;G~nKM{?aUECTBoS%={grc>GEPqjdEw z6LmXYuzsxy6=11D{>vXrZY1C0t|zGTNw;^Qli4G@--B-qkjzj#?az^qO^2yfB!xj* zdb%$i)}j*{yTSDVG2GXq4XZaqF=p9$FxkcgYSx87&+$A-3AsTTJxWLK_t}yk&Zw7g={3RGd6fNFfB|P}q0_i5^ZpgFEXWP)vqxXh#g4Gmr5HGW0lq+MQ`!jM(W#sh))u>1_1zE&UdfqEonUKnlC zAcYTw=vm}C7c&=rq#*frP3~@e9vTkvBxA?Ls2p|CLwZ*T>V0<^pv{_)eKMKXY#!t; zgv}z0O!7s^z7z4teJNhT-bTA_woq!fW66424{)bDnd9p;1+Pvcyvku07#76+fB!!) z(H@=nWy`3V`Ow;hUxAl6-2PO?ZYbgR<4yF{xUx4!4lcx|Z>Hin`_>%N1zWl4y{jA|p zM*PyPnQfl>TCx3FBdVEfBut{Ou(tXXd=cVf=leVqE*P1>Y9^W)Xuj2F`hSgO;}0I; z#a*kA@TM&nI+D$fJdn#KT;{9eLrHEC-Qs zK>_RWYBCrn@keP)>Jcbb8=$}#q^Z=mfel_%3cmGz1{H=s(T)kvCC@crTB0%ST@V4y zRE7N8^uOTJ%zSpq=_9;C<`-sR^Fi*!j4(9vQ5L;j3)lwz9J`Iqifn?7GPXqy!i96@ zu@h7eu@541sLfTTjAK)r{OqeU6v3DV=6ZS@zsoUC5;=|1eT>r76MHyQ9gXT=D?vl275M_{8~=?~>co(e_LF#QJ^?6BzUybj-9MDkG}! z@aYOBUn@g*6%~oD{WJs<=^-$+*;`sTd`(OD6FX@Bfe8%$6jMlFzGC0i>KaH8E@o=UAvseEc>YZ5d?!&6+#7Mr;vJbuJ;XgB| ze((jncIXIFR<}mo@Gw1mUxjgpOF*BXg4oxY(K|g2aoyVh-mJWc$m#YFTuxW|ENjpFm8CJ=&CAE0FvzXm&)jytb{j;_*vuDAIF0NlfbF zM*AhArtMN>L&;p6QKh6YY0^Q|^I|gg=`qE8wH?%KddiK}&}SP@Oj3;eT5BAyUJK4{ zyNF*6wD3nu!CQy>;lJnTGmM`jE$>&~D;cbJR`H0Cg zacYe*KfN-Y%`?9NrfxJKD??6Gp0a&Rx7l<}TN{X{mrB979jWA zM8Mtd%J5$4Qd02x5%?Z84$GRAa$|g4X+_6y-h3z)OgI!xuU~eS8TMVpH*_Coye6?s ziJp>zqSUdfu>0M|H}?ZvllCK!R+cwoO5}K{GW<#IMR5uK~9A-I1siOueNvM^B(M{SLU^H zAqu669apXJYlU;@klv7-wtTtVtmh0_F(#42t1I~%&t2>S21P9VGKF9M<{pmmau;md zn9V(JQlr*AJ;1ECPEqkK&B7jQO$Aif}Wc-xE~e)F;maBHDIpIZ6>zC08GW!}xe+ctG^A12Sjb!zI!_scQGt<7E5%}_wn0D$MYJ4l{Pm|5WK6bmbBBC(C(dI;i#<(cr{rJA9P2euT!V$ z^7r@R*Z;1;{130-#1t!dFKG`NjyeT5gxvw-2ZpKtd|SD{+IH}zdnWd@trRslt(;mZKn+Fcwi^aybGWa>JLgv^MDAvBygeSUB#A7t;QN@H5xp*6I>LonFFKx;r zBclun>zU4vTytZ;uL%%6U)_%1rn-Tb^QV%u-SwcQWSB9Et1uC5Ifa8-MpKhtZv-N& zN#DPwfy4WwVNr4geKDs5xDUFL&TlJWQ2&2e{q}ybI2Y1!9sy+Y_|b66HCMRw$`btk zdA*q1Er%s6RJ6UfUFr1B!%Bpypo#cLN@*@k7=VxwP;;??=p zEIbsz*9x?R!P5oWDwo6XsOJlX!5?Vh=BcF{rGawRRH07K$Lki_ue^>; zJ4HyLSrz_!rkTVKj8o(a_w{`5jVw&)`W?KFM=PE zJ0o}4Xwxe|-9>{`{gm(?Hz?ub9y7ImcAaPKRttdPsiW|EN;*3zt<3%#UWt~nE9w31 zv21CBNL}| zMSnNb{3xDrdA5n_qwT0ntsXA);YogHWEy|&xr8;eS&~mwkLC}zK4e51C7^$`7soM!utkzW=~aS#MSo>Fid3H}?0jpN0?H)l@|D2JEzdL*I>P;?4 zejH2955mQ=x$JGUs{YmhVE~kQ0hAehoaYZ9p0^^`t#z z1H%aB5Z16n!U6h^N%+X2jr&Wfkxhn_xO}2mSNS4e{b>{b_>`yJujnCEy}XFLQrUtQ zDE$LN0UvnQsGQmc-Z8-uk$Ur&KS4xcEDUhWg!tKHAp8c9meK>*DU$#LTQXt6xi&t% zWeG5vE01?Pe9Wn8WYdf0b`oPF0Fy1B)6(vx^srxusP||zU9`Q9{+LjWUo_^Tgs02V z^aY%}%)-4`eMc_Xe}1__%)xjzp}3lyS(wX*`vtO(4YI`&e-yj5!bx?Mi8OBY1413; zQ>;^X0DSkULzLYIn1Hz|O0f@zarnOsi*+OXtfo(;$j$V);;X#o1=J3zEYM6Ra31*<<^W27dRXsxe1g%V#@3d%422ZRgq z!MMY<%+KyM@X=9gI>0*=RNk~9?emTaw%NZz;j;mGc)%UTyG$gb&q_fX>sYvfHD^K` zbVLaehd^ulG0?;bwJq(e)E`RS+JRV=S931-Pq);T<$yRnu5Sz6g5#Jc_ zW{1WUk;K-+Y#jepYqxCz(z2EoPWY0-s%%z-<;E9y#Raak)K@ijNwku(aU9LubC+X- zdgC}>X?Zj@^B*s@XeE2?*)?qIIvZc>`bVk0d;n@zU*d-xZgR1=N0A9X5AYH%Iy*+B z$-mAFX9wl)%ALxUxJM`zpk3#Me4d{$9P}B(CchsP%sN;PssGy0l5MB)<%=1#gxy8Y zJ?#SPBmd(T9W;P*qr+Hr%>~3|!Et8CjJuqa!eV53+nDiP_Y*TI86>Ig89sQo9gTDF zR8r5;We0grcFX)qXn*t;YU<3VaEKNDD_IhxP_xL*|F)77wL6(rE2QkN3>D)JRTmA} z8aZ6vr-NDTG1y6~lnxcR@C#L%AwO45xb2rn<;tNZ=yR?e3CB8+&F06MrUlDM*&;_K zc$GHlG(3r#rR$*5W+hnEzn32<$t1?H5BVK^*SPY3I{5N#L+*p;9n{E9q!lhC@Lx_R z!D-2UNM-w90=Je{rB))dAcC-OBz1o zON*Cm97V^R%_OVlJfL&xo#~YdnrJAuo-BKvNLS1B(V_QlDx9;E$1W!-nI5N`;0dpW zB==T~)z zY&zT_bTL4+lGyaGAU!A@Y=V|)ckq%O0rSXMAZ7&>bV6D*GjIP##^{j@jAZM;i`h!N z!Jr#4lQ>@*?Y8njKSr|Wx+U6uP7Q6+je`1pysPdl_!ko<{O+snY72_h4wU z7o)g1P5kB90r=9&hW5AlsIcYpR8$x!&*+&%2u>f$rOySq3%o@c!pb~HwQV&O;sZZg z#9uaR(`DZpc<-xGj7MLEy7zm?>^e7>se5P0n7T{{cRqFUH7^%}7M~vaMC)#|kgAK+ z3;*}DPw!RI?)?uAUVBfvyPhgXtNPOe!?pA&^-4kaK|jINYBzrE+%oc)Ya!XE;zayd zH)gj!;^HTc(0kh_Q#qW1*i+w^VoO%j6H83BuRif%l>0|hX9eRxbpAaMCmTzzCImRU zqySOL8W6d)OKj6q1MU`{WH$VE6kA$Y!Z4l|D^K7+spc0j{YMpi_2)j9{56QWn7SQ4 zNhnaPmvF1H&W)ny8YDtDDh^3`UkANX+U#ldrD(%Y4)5AXi*HEA9JMBc{N2YA=IokP zI{*2~VG}XLj;`i-&hjw&#(xa)eRB!k^srXa{aayJls%fRc-w?r?acYB34Tn=i!sFO z+h6?YvlhDa@EAHb_c)ws`x%VBo6EHySLJ`q7)8>`V#$PVO7cA4;AWpFLl4%-(RrVt z*maT@+^IJQ{b#nv_`;+0IQh<4aC_5Z_~%|1%>B9)g-lh24>i*1ij3Q%Po|raY%eXj z(4FZx@kKwsICv@)KQiVA4a(^!af-A`nlyUosv&kvHKK3)Nuvi8aunik%j0bU2btKI zJ=Di}@n~X30Z85Js<>QDjMf=7kSBXJ$y=wNOwiOyKD}xl9+pzl@ZIIcD3pFei~1$G zT$aGF@3%Sk!DzbRMjc83i&TdGu4dk?utt}5rgII8gV9#^KrZ8xDH_&}WTel@q5Jxk z;Op*SXy-JF#0m)Sto)5z*A-zm+qwZoe=Fqw-0(v4+!1xY@EosQGYQo1>}5i0P1Tku zSE2UoHt_OY4?L(g7AFnMBh7{(dfxY3TDM#PtIgE;4cjWA|Lxc4n&o*>R-g=WPj%p` z(ks!CRnbgT(rd=7K1=l5)|(Dj%%Jz3?@(+yk%%7q8lg|WJi)9?1Lnn(MIhyahEnOu zC}we?93Q_}^o@eM>&?Nba&6vkc7b4~lqd4m)fRECiEwOW1$o|-1@+%fhj)h5n6`Rp zJGy@cZRGwCroux)-8DT-dIQI7vMQviY_-_f&qE-OoWsYwm4tlpDPr&>S8O}*i`6(j zM%?Y`Dcsz?Nb3ZjhjL7_gas2s!q;ye!QG<_$mYfvI?Sz_?LV>DNYE*dI?Qs}thHM? zPd^#_ZEin_7XhW(ZNcmXb!|3I?+f_qI~i(=X7Qa-4F75ZAaa)-`3X8Zg&B{MxbM~~ z!u(0A!01gY3BGp()IY01GuKoYjP~ndQ$8eu&D#QC_WL@#Up$9ZxL(X~ZDX14UFtA4 zRfTA@)WRPQD#F**63+jkeJH#+i=Xnw3GWD9f~F5AvWoR4Vmtj1cJELk-o*7QPWXCD z(tpCj)U!~&v#$<0PiUbUf_hks&N7_%$W46jDHi+B%fJ@jd-p!Y*2XJN0zbe}tfzwQxZ1se;L1POlgRK-r&X#n-e+HnmDE7(QnrSYwe zwdlpT0kHdhG?cm9$9LoDqCOH+Q~8o75!)SJ(o)Z9gvo6VwA4X+U}{U}r+=W$=>WjeO*3^{~YV4Z)9dHEX2%-c#9xdhcJ83!feACfz2nEpDsB@bh9)U59y zWXlPKNkQ@Kt0VIE4{kgVhnB2ik6K(H{srgQZ$m`up!7>*-%1I4%F9?Kr7-B0T)=Do zF=OVhpTs&{@Gz=UJ%)l^Xm)it$7i{2#tDCAdDrn**fU*i`04a0EOA#t>$U^XV5gM0 znLWvGrE*B%jDGH&tHe33noKmEBUrsCTadY;o`1}&p?1W)KvP*(r*51NyYR0A*nDFt z>@r%%UY4!IpFUZj<^UBs`pb3L>*FK7QDqB7u6x)_YYU>fT@9CB?&gjidxw)^>~z;vZ27p|^4+PO*lW#NII8K8a@o{QHhukhuB0}X z6t_39TN3Qa0$N|_Vr8S}c%%`19$Y2t8J76pim!t(x5q?k=_P_Tf zwv5$2V9ps7=U|n_0Xyn!8>{1C!Or?Kf}<}khJOE@gMOd<#lft2Wkpk=9~n&XTC6*R+9NafuE9uorQ82E08S)72KchGt7hu#r%5hgLuh+ zf~Nn|F*x{$G?qExfwwz^(CRCKxZ6Hf@ZfBsu&7dmGmn3OpIt`b<%x#~E&r66vs#8H ze<~U6gsEtz!ejJIs}_tI{w)}Emlt=$8<6YKO_Fmq$X0f{D*od2lLXGsLB_|W>HO$X z#CTvD%zd~NE!kqEr>C$T|3p=ALH%X;c->v9hipJu2clr2;{s@EY)*CjJIhbGg}}h4 zI6UKUl4!#FrS$iNEo9fLo6vD3!{}}BW=@aOwj1w%oz5#OrsKzm6$VbK;V_?c6qpht zP*L<`u58Q_jKtX}g^c>nIGLG~PQ~Z^&1Z&8(yBS+@QZE~SOhiIokRcUFhzN<%8`F} zJlv&h&->cMvBS_v;mMsM{jk${AiR8)g1wuRc=I{PJS*4asrj_R5ywoXdy$ZT~kylJPzZSLYOh*IZR)YFn!&sp&nCkbtuV7aC zhAsWjCNAjND9%}akImS3mpJvS3Evzg8nO-T|6>>nE!5?NC;z?y332=QV=D^jf9(mZ zL2bOzmF2C>smmML`E#CftFqIObn$L}nT@PsShF4L?9hy7*xv_nCv@Q|qcn1KXEuLg ztvm@b&*G-8au+HfRT4F0h;1~RCwLWJz;k~&s(4+z^61IlH@1ZX)uC! z@pfRtuSz=ce^iuDJV<4~v^TPwtgp)lEF8w3ADd(gN~^s0QZo#b6CtX=(C zbScC{_-}3!C=lw2t_?&A8uD(Sb||ZP!(@cry5t$S_QwS#@o%vItYvIzNF(!MR1kf~ zpb#z!Fz23h41qWu1L4QRb)v4T)kt~uThaXAYjBHs8M^FtnB7>IDXP`UW~X@{!NXP` z*{#N%Y?n4KG_30tG?`1$Pdk1IT({>cEX<6h9VfYw1>tf8rfShe@^{FiTb9h_<5sHW zlI}%jnKYx=JV;*}C&``^?%>C1b;)1NKS)1Ne{ZPO5ie+WIZ6*r>*eoHTL3>cU*TWQ z%TdUwk)}7Cq39(p6@tKZUj+R%^GU%ro_Gqk@>7mq<~qNrGlR4$IWaDn9_Q32STII~ z^c-`dW_)v^*MF?mS?pU+OBf(>5t(y9it%b-qQlbb9PUz&GCIH$8EtBgS`oRq#f{P| z)M7$kjpBY8Oa*AkQer)MG8L*W2mY%DG=41>J-s6b4n02w79G7T4C<(-r>A0iw{f5R z&+~GOu3QHddOJd)C$5dTr?HQ`T9e2(|6RxohB=GBewHwFmUn9|(s|GLiGol{UNCCk z(+y6qY~>q@_JJSUD75m9tKq*j{oqabXe4dmN6yVNVWbn^68ER33ht46n1_j#%;;1( zL64%X+^C=ca^bfdDcSOiyD!_sO>DJ7i+BI#|Ec|88he@nnVwEejqg!Y8@-q-Z_XNj zaoU3HW!HkZ%dH^jH=xJ4>|@LXS#ZTm9eAP47v9zGCTFd$_ zF>SS=T|83hw;0g#mU1v^e)wIFI?v+J`A?~H+tx$sQZR0z z#OT8mZDv`IKfQnHc{ur2j=kxeJMfK*7M?%mHd*mN2OV88m;7AOjN~S@Fkrne5*WSY zj~z&4e3vUQ_Y7|;%`bU_+T9MK9sTL@>0^)K@}=s)q7^IN>^#nzTfE@+-`Z_kPxiA7 z`8wh(>JfYMVz$!4H>c5FH6!74tE+5LI0d&CRfr<0Dxsd66YE%f4Hw%xGWVxhvOgnQ zxkG8ok=5*Sj;Lj`N4`t59cyy%l^5ES?SawYrt@3TfqY}`Vs8#VuLIg(Fx&|? z&|$oegBV!U<+Ftv60hiyVdl8^Y3{Z{DvCe(j#(F?%u2n2ydlzL`9TGI@;@hb)OSyI zQ*kCM*&k75kycFEP_JCe6s(9muQ77_B#8fg-$W@x6k`=LgFhi(g?XXPJThOW zh#Dk5;Ktm3BK8XI!{BF{aBz!BtB7UNhRG>&avPhb7ww32?zVy*LV9F8P2Xtf&9@z#N={myb1wXX6y_94JLC>LprycIcC z`@=7-LL%i`0Mahsfw!`{>B~vV+<*U!X|lT(-n4QSmQI|2M(sb$*h-wruilB#vI7*v z3i(2f8yk_L?seYRZyNu0c{?)_ASd4TW)VLB?1TD5jcd#U)e$t-Qj({wccUHl+3`M( zxe&=!BHekG2CK5K(xw{^A@8F@Txpa88i`v&^2eN0X1!aO_kSgBuqy=EpOADxG>+!+n>NQ92jlW>!*8P2<^57!PolQIWot!$UKovSP3V=|76>s`mI93Rd5RG;ANRrIiS z-#yNGq#0c;TnAEP8+b|I8nmb8Fe`%!buW!kz_(<+flm!x;G&llyjeRNQ4K~=F*gae z95sLgFKx;6nQtiX{_*&UlmdTJVG6yHb>=164wwXY)A(L3W4%9u&*}i^mdJwXy}na% zdSVaqR%t|mW!7?vHtlG2!)(DRQLFG)tggxhwI0&DZ=Tq>wE~BJ$RWD(Wh7Zyq59H9 zOY}Q-HoJdFnzhtO0Fkfc_|0^%{L_JLSfR(s^lgf-eR9!q`+L zWjiZemQ;h}-UP{Re4rr6+4qgt?~mk9Do!E~67)p+$eBH}Y&B28Mtnhiqd-gHE7#lE zAuwE6%mi1i(R{H#3k$7&%5SdTNCoCjr1K^~yk6}HFiSfIU;bslY_^bC%s)s8B=g&JOH;IE)amf zE#$Xc1b^12mOnGQ!Y1RIIR-vuBxCL+_Oyri^^sv_)g`SCH@So=wQGeSPD$xBqnjCSG`j@pF?`?U5Z(4un zJ;v+XKQBCkKVJ2-pHr{GUQpSl8QOK7>D=zcy4owUsRb9I^PAJ;ZQ>FzcgtS9{-Uh@ zt0rx*-8~k+{npBxU9e!xJaol}SvQ4b_wjh!^6SXm{=DptQ&j?~1I@f0Gf1d~TBPi- zCcivo7CVWruw4??j+M4f2Vi0bpQ*7NOnEfK^xTd!`7?M5PjYaej)g~q{L^0W*kets zogD*T+6K{u4*$p6o5xf6y#M1A;q2$wcabg0dXDAXbE8GHqC%9Ely)stQrWlc6|$wM zghWX=_jRKbZI)6YWvL{oXphS8=<|5Je(&%5_5OUnpYP-Goj=SqGuJ%lnrr4B_qk{0 zy7DMLSR1+BT3e{^I({(Ax5-<++O9(TocMuv)-`}C5Urx#kN04%I^JyYZ5o|?}k9o(INwJDljhRoym#~=C8QjiX1D3VQ4Ca}ZXEKtx zXIQ)VEv&(EkTO&1p~X2WQu?)yFal(%=%*d3d3v_Lc`mYA^v$=YSPaeyW%#}pQ`tx_ zq;n)~8C`*M7`&)h>i$)UTrZuIlsj=W)~C2>nu;+z%DJeI^b@xlxeJod(6)EF^S*JS zIW8?hbm86(UG>t0l;PC~N!gqo+?84~yn~xsxWS$o+|f*HhGnA;*ZbN-`X|#1lpp7S zo7MJ+@^E;(-QN;U%_GCZKr>yHd%3Dxh zDe-&OX~sNtZQjUxW$O8jG29zb^^|1fO7%zxrY%s+5$?r@Q?(tJ()GUmsH z(7j@kD9#HVh0f7uNhSXd61_CM zSkh$R9rJSd2<1hLmP`g^l+`D^znQ9+YC?O_#yYe_)?(uiIrbCX1eN#O(-^{MioIs5 z6nongM0F|>Z`b~(QrSW~ zONYx|Oy7CEfVI(cs?_Xf3RIc0SSGt%jU}{YO0RpIKyx`7C##&-%riLVNyW$eS(&{; zuQ+>`Go(B+xTXctOtjaUK~-{N{gjuq$lqkoin#B`Bs7n)&O4^DHqC#+656knTzGRi zy;u@a6x`dS<$1z$dJXfqbCv7OoR5sqH{~DU>FlR7+fwrtr`4G;)}aALgw#!jk)#84 z^;Q~Ju}6f`wI_seib=>ja$+f62X-@3PQBs%{2oSEc*){PpG;vH)+F88EfK08KVX5Xlh z_Nbq6bjEb*;d^$}g#!fjLZCe3YIGCTSv8I7SyxI8tZU}p+^i>+dW^zccg@PYW^E2l z>q;BXX+MRQ{@?{|x^6ygyl8{jJ)afy&USJ7R=bt#tTiQ!;uBGf>?tQC4!?|H#{A$= z)0C04zfCmDh<=BgtB_@;+q04N;qxtv8&%g?rTd>K#yxw)m_Licrnz5YO__mF!I2RPmA{ z)O}{U<~QlC+^bWBCLEaCjMe79U|qU6)8uJdJWJpeK+(#VH@}N4B{j}mW9S`Op)gl} zoA3^W9d&ifD(Z=+-mGu=SDC9{Suie#@1T9&tSEETLWh_2TAQ*UD2%m;e8cPguEu=2 zu!g&6UoX?dQh_5Y>ZI6v zy^>zDd!4N7%P8Kk7~(eR+S9v|WO#3qS{Uz@Llu(52I=Y2CmF5bkqqZgMCG!>L zEb2;ATSmLlMXl|};wi+`nGC!b=bg%(O7D1+!+nuzMBh1lj^41khhF}%i(-`$B00YY zae1F4cu~zo+^5SPn!eg4eCO82%y$Uw|3`Y=r3|hd;Ih=yDFrjq=#>^ej^%C}Ec>#lBY(C;KqIqus~uCmj!H2WUQ95kn9i%O zp2=P3;m4EdsWg9Z_6@5r<(2vMuZLLH)`J|OnZX(NeD2rmq zL`QHXMR#!(=ppoiCH|JATQ*Z_#x13UYthWxE51GH-XmmSd5p|XF#iGw=*uz zzM{TfO^qgXwv%DtHH~-HqMm+e^Ag_fmNiVjTeBDj-^v)nAynF2Z4+w17Ax+a!?C=C zxMFU??RjQB^=iyRh;A;MGhasSx-(7Wa0s_22BX$}_G29S#n)+{zMVO-%$)k=gAuJ+ zZw^f^(U;K}(MF9r;z2uzR!Mzn+Q}WhBSGmhNM%lc9$_K$Nk_ZnD`D~EX%X$Yb2qIx zvWmVxM8rZl3!`}|cGKpVXtCvurZG8pTj*(bD=8JD74$5#*HWb?H!`(Dd+EwaMqIhR zD2uBCHvM3Pl;vsfhm7*5IOTI~n;8CyVT|fNf5xYhchpnm3wYC{BdIgrsWVIt>T3}@ zcTz>W)EUajO1uLuh~DV0U=bvLi>56ZN0(pxl`b<)fqJ{qiDJ=uljrPC#AT$ zpH?AaZZR!bm3I3^IWJN}MXprcpB98}(|)A2OT_C@(&d@7b)>xc$_YwL@e=NLLY(NQ{5kgHGol2EIBgvNX3(CD#PDs7gcH&w5ZdVWX2hJi%8R~y!eUFU4Ba5* z3D4$6A~Sb;uu6fK1k>pjh3PP%&2&8)MpFwD=S5|z(kdkjWPVG$;si9Q2>TwS2c1p?{Am6k@uXbZ|GC&wSlQ(@;yNtu0HZr)$bv)U4H{rJl&4 zU6Y+AV_hfCylVWKJ~;h`RNw1(hLWfS#qxMQXJeBkqbY)7(flEgXZ@>_esJ18-qL{; z%%kTC=`oAz^qkKh8B!|W7;4v!QX5~3b0Z%wqP^`dWaO@TJwv?AgZ`y-0YmRarg>bz zTt<=o8*`yAO3v(G$LI^|J`2~O5K1_1Otq7_#nqt9<}SRtlIxI}WG7Bw9*w6TyOMj@eWE*mjO|4lwK?|Kx-h}j!N|1^G=&b)or{Q1L0)Ky)Xw3wVK znXO?$-_jed(aoRoD7(96(}T2kQFaTyO4qE8q|b1n^U|BcxwBcfX?8zyxF;K(7|$(L zRSwt){rf(YXPlQFr-#_jq{{5cqb3Jn5chMVx&sG;~gryKyNMU zr8Ld!;yyOBkc!Z^qx=3^tM*~8BBS+Fo7Cs& z!F7FgRLW~`Ib*xMruomb45~@VS>CE+xfJiUr>I|U37^|}=S?Hc)={%=UZMuBnBdg- z_R`;f7NMu(FQvMI)L07wEGQ95*BMHSWY|5P3=8Xawq@tMan^&JLl(`qSF%@@o=`d= zXk_HsWU%#y%-AO`yrQ1doXT5!{0vR|Svl)Q;|@*h)sl=nl^oVjv0Cny@G%B#*R&YB zt|5IY#fLSsT7|{U`c1h(ok^XsL$h1G?hC8tPnZ8c$@I zC~ZZOEW08&*@9Oa!DumX;qeS@n5k_G7<-&gu-w!&EY`&Mv8HW{W!igY$^;cOumXmq z*qoGkuyfTKo0BydA%3JP7J&5;oFprR`Ol@KZ5M5KhO6HK zAZ(mN^kqIKLZ_dDir_h9PrVyZh4~-9k2iAp9NNf9Q!dU4! zF|uR@L=3y)J$A)pqgx$Zh!`SD&hCY0+&wlTS=->Kj7Y6>$_U`~6~q=7L9Jx46yyqZ zXs|dLwhixvm3S1HGP4LQj50xW^#!2sw*$Y?0C@P|BV-Q*fFffTczhD6O_pYZ?(!@= z*V7puSBlj|CO)$fH}N5LfBzu7gmcGT*jI&UXE-Dyu=&t@i7*%TZ%IfokuB8!D!ErM z^7{c%pSK7+UL1iB5s<5!x0M})v9Le?C5ed0?ydk>5MM+7SlmcF_`H);URVNW zAD<^h)#Krza7?FnS=;=@o9~cXyZ>+$Mw$aWIB^5O@iaS}aYr!3uQ#*#(s=^1gqYzkN^C+uJHy4zJvQ6! zZy=o=*Z;v-B%GfIhD~hf1L5S9TkfP}e;OF8d62ee?h+?G=hSw#E`%v`F_P8ShObSe zk}0l@{NLBFS%WDH>+Pn=B8*Odf(YU&&+1w+TbZR9d^WJ0W-)D94{K89-V8% zgYrjkH4 zjW(UfjB2BP2imk}odT8ij}Rcl8jVn~-Ky^j!8OSsS+UmU?7WlYx8VQ~6Sh?$FGKD; z$g=UcdmKK$@_<;sj1cI1cx=-Y35c*Q}oXblB&>s7*iVM*gf7A(alVLQCVGp~=2a2H7gF|WMh`T5p`|`1nxYl+1kA5flF5}OwC*#S$3wTrhU)2Bq*Z+`);IGt!;r;wc zI%bdApZKr*$yl0@tNGXTztZko`TyW|B5Jb#;*sxr`L_&z<^Atsnz8=hGX2FzUmW@$ zF;=hna z$T6tw{ExnWe$My@-~Wva^?s*xC;5G~&-2Z-!~P@I-Y+x$WBz|?2W{5*AM^gF`jdII z=F|UU{njTB{6i;`7*(Ao|IY8#UB#cw`zzR8+5JDp|EtblF$am4{6&|gT={QtKB~CZ zlnsbfpZuisM<2CLjsFYp^S=DQ<@*coKgAc9Q*{1Q{{Ja2Jt_ZxX>XGM-j=O@^w7jh zs+vq&_e))vjQ=$lwxwF%9o_Ip=7|TdHO`uxt)47zwXRbCi|<}f6Ynqjzl%-ZSN`3< zQ~WJ|>wj9{>_7CyK{!wT}Kd zv45-+|62aPqy1PDfBM;_YWlaVQm#dR>t?cU()+J}aK%4Xd?|F7x)jA%XCI>cH1MNh_Fb6o$Pr^OXAf72IZL;j}! z;y>ON_m8~4%Kl(&D=7Ha^#4nG8RulqzvKCK?BMVNBrJ?HPSd`60vp zb3Zv9GBMLdlfyZ-E^Mng?is&`L%8)vuGfu%Q?_v&ldk$nP`AN(byf6>b?4ae$s+8D^J|$ci*ngd3(Po2*C#Pv zb+{=_o?i~Hk+A&QDasPs*=DC!EU;o2inBdR%GkH+nkj>-@~jNsx0YKRQrYvnAFxKJ z9$~+KMqwvXvssm+>C*n@ROVIxJl4#EQ>^Ig8(5;xbuR8`gPt zV-ENlBp8csx z__{%sK|mB|;X<)LV>2mBWb1qS1W!(hmE6kNRBgf%AK}Qf>hKg()n>D=8+OvGid(sp ze3Ri<@NvrpNBw0soG+BN{B5kHd)}YDAmkuR^MDCwZ`6FtnW)P$t3BV+@2w7dc;jV@ z$vS!$SID_-Jgxg};H+iOmsEC)&UKc{6LaY;_RCnW>`C@TsWWur(^h}vSrq@7J<96V z9yv75Zk?UYRyu#4lPZ?W7@w;pTQj5}yRj-vW~z*lbjrYGX2U{pnJjZHR<*R9>?M;r zmUA-Ka$(9jwzl&}dCto7w8BeImB-4bT18svuzi!XE%#VXuwIXxo+aGxXLm0;p=Khy zv%5lgXZZ8oXv^0{Hde!YG0yE9$7gEp*=`l9|AH-PmZepfP$naJa5|??dWJ>PHgond zLHQp!>2WEXw8(L#;BT`yNz-RCk8P@D#|ba#lkS~t>DNzKJ8iYC%)9l^ zrD8d(Bg7TU7bm>s76oekc@{`+67%9L7k%%6WJ<=9L(?ieS!_0K1~ou8f}{P+&MMYE zhP|coF*7!3iq^iLSEX&2G-*oTkGBYPQ&1IdLTV@}-e&4He3Ls>(aqkc{G8cgiz)TG zc(TL&?bz1&jU0)kpZ?6Z$o@=@N_!SNJTO98{pmsWqxWo;$#IzsHbW)stlxXAZd8+W zh2EQN{6H)7yBtj#H@Ih2`RzHoZQ_-xoyb~dsv(o(C^#Y68b6QYV?RLK^D2)mC6-9D z4s6kqo?^wRn0|#B9p}!my2fII$XqKMEd|!a_yCL52Uf7E>CT+0#+jBI1yM6M1=ewv z6n$YIksLLiY+qvASyqN^GUxe=T{670%Q)+nXsSwOQ`ijg(<({a<@79w;N*JwaRzc; zurqfpl#%V#S5)XyXDck%xB5~zOJ;wuGG|bI3nz#lM!)=i%gkGwq-D?cg>gzmdN~d6 z-PH>}B{Bl;uVQ5#f$jK>E)!+%&X} zEgJQJ!EZRH@;2=TT~d|KR%+rng-@AUlh-Z)YaA>3%Da5^pseMs8&GLF4FA?<#Ako~4HfqA8zqpHd2 zX1yy*XFsmWJ!l?5XS=Mzhe#TWIMKN|Jt|X5w=ZU9qHk zj*{+a=FPErmPNb>4Z*sDRwsYWm6lo3u1axEWUn)PWV!CfP4@hmQS9|0$(FCWj?XYV z&0%g?oXavl!jM@!kzsWX(~%jQ@sWOi6_vBxM#(DC|A|3Sn4EO!mWx)~qNno99#R>L*);i7RvFN-E4cyC#d(Z=K7LE>D?WHqycQ?P$c_ zyZ?(mb!fiaa9iD%cAVw7IFv6$Fc?|hih{%A2#>m2 z6r;14=)Q7{knz;P^cEZuG_>3U?~P_CInoNNdzuI$w$|3~s}-<^iVunGuc;{L=6Paw zgbuX&7$Q~Qw-6_C6*S8G1=c#Jz&dsW6vm}MMC0JKh9)kGk#fLX!hR7&>8WV&OA5$5 zEQE=tdr^T%KGxf{lHcHjfnS&!*}afS&bN(&pr{KF+^>P=n~0Oy8*QLL;w1XA+YrU8 zYoRo09~9F27!2?kcz(neR5N%CIi^wYk)960hMg;r%AGi@@SF!sO!4Ih!Y!R9Fqe@ z`7F#)Um6csAOSHwqS$BEXp|)Lizqqw1=23K!>yqyc=CNVs>qhXDNglps(B}PcvQlc zd+8{*O$2-_Z^GA^RCu{;Kl12QLEcFPg8nuu)Ln2Ai5^rWLvl2UfUaF|YI`-(Bjk|i zl2Yhwb%L}Ip{&<^Q5h~u6g;dzr?VbmJL)zOkrg$>G?`Xp{h$yV+g6OlwRgkn)%7Ui z@qK>u4P``Fu0shQ6>(27ODu8q6_n|48*v>7cpcb+7M+&BLrSW_C76ac9{&zoo=cMc z4f~Uw}-Ou#aQN@48k`x646}4v90=NVAmQWZ+=EN8IYJqWF;cS6v-HYB1Rk3LtYEo zVP<*^GWK+UOE=7c-ByM!?f!;(pV7hi%uaAZ7a&AT3iGNvhkiv^V3wDrVNpd1(6VkS z5?QB?v3}#IHU1TPAW$K~++^X(eGxM6W(IZ;O+~$KG=cJ?+km%ELs6TmQAENxI_h_Y z@a#1OuSFDSp1TDWPrZSXw>~FKbK;SIMl&&Vn-FtF&WQBiV2 z=ijIz_t(XI>(p#mv{?`GVs62p8;<;qyZ6NLHx}YB z+^mX*t8OEQz#Bx?*ax_BJ`n~h$FQmmOQFX}3oL!QFsDEV6dRX;&T2EPZ_gxeyvxZ^Q#c0+j2hV)I0Lp0>LGSAflxlwzXiIjWg6si; zAASPHja1R2GcLs3;h$)p_cfx|hf4Gpk75zaf{@LdTfi5i;qsU70NIs;${UQKQ8pL& zC8|6PfF2A;}7NB5y1LYWmg?$7+@UYncf!=EMuVY?`2X9V%e^eOQGSY)=*WiD|UXt4U<6|pQlg<%4n%?!LC>?3)WfSklopjEAPyZi6ai zJzxc$;@2%|69n-2*jot&!gjHr;NnIK>2qr(_-X}X^=e8;Q7?fo9a12hHa&sFoA*)K z$@hHR>@pe_J&J->i%umMdAGJ zSa}fjKVidF))V6B;!9vH!Gut30ao(0pSU$!8Q*(w8fu72M*dVi>_=u*mHU)btoF+W ze&(-}Fr`HV51KO{90P<$*w$AAS)B=8!(w3dr4cAihltr-=TPaHWc0-C2Vt3qVS)O# zpvOFjHRmh_*#}Jgb^#4plg$`SJrMakTn5>fDu|8K3*gg%CBz=XQXu!7L6@G(;K^oD z=$WT0W_BwZxjxPZP40DcDzqOh#&4oRKRGyYZ9VLr>kn~TvSG~}e>56?3?Z4<@WC(^ zlefBxx+dJ=&7q|=2ivXCJ;(%)+1HV~HI^vYMY6k4JRPjpa!+%orZqTAZT!|M4HzeV2H8_ z<#!oizgF+Uq(wrp!k~UQ;Giu$TMz{2Mq1GK@+Vj`>$+g<;4erW$%fz0>3E)_G&##6 zpBT6xNhSopMeh4-3Ehg<#F6Wdpu}$i%Uf|3I>e2!Z?4}6Ow<=Uw@#s^uVGxkuWv^; zf6hjkpXR}i7qLh;@+NGST8b>z9)a9bbU2V0N{}7X1aV{Uu$y)buy_4sw9@Dy5lqM+ z>le=iH;T%k{&@+u^|KuKJEbBIZxymKA)7z?-VSctRlwY>Vo&>PH`D>v^9zaqSC=f2xW04PHZQFYiGi?sf1X zMg;d5s)dn_QGyq0f$(hRDCVAQjHLyZgG&4!ID4rFbE-^%q4*JEvkfNj&UYbxec8e9 zdZYtW6|ym(&NEPzGYpFc4B_~7V^Y3~jfdoBqbIQkp^e`S_p+_vllTm>*xm>nMa!T> zxf|0TWuZ|&P4efjgV@0odyp5=Cf}~UD+r!`hoJRkg7s;2{PLF{@U`g*dKRn?vut9K z^o6gGC-ICA4r1iq1fdg^xO@m!X&^$AO5yat1sFe~O3s562?6LGtPZEb3<*(w84V4tk#G(4t1cfT|f8sjW$l&t;PDn@+;< zLnWXT_d>Xz_Y2clBtgnbpGCdz&!QutsW4-O3u2u-g(U37$f7(k(qWk-IWAQNVUi3| z@^Bc@S7=Udb-4#A+*VlWeM<1v(H}w@mOw5=fjlgI9`-MN&L8?tCtqksqfsp}6cW9s zX1QK95f`US5>o4s*pV*Kd+iP06lIdTYbr{7wjE5z*TK)oRK(pE1kc-gV6N|Z>&QV( zut||2&t6DJRdYWQc{dBOcs&VNR-%K_snrmDtsC|2)FBm?Z3fy!J>>oT8>V=!yhd7c z4Km?ZW4_zZgUWRRxpwLa-|6naaG0)3z}9bUMZdDr;Z%n>@b)m!rwkcjU!Q^UL=@2Rdz(=5rdYzm=Lex# zYzTI(^@4L$HFSY^iMd{;qa80}VPRyxfWMqutr{y!c0eMbWvq=PnN!FrnYW<%D;jAk zJVC8*??Evo2o=z$Sv_TCk8ep9#TnSyJVVF;uNtj>Hz7giZxrkc}=x&DZjX zH>Ynx$@XF5T?7M_+n)#a;|Ji{Zwkv})yWg2BtBXg0{W$kVRchH##;Fs7-?fD<=|(m zxB3@mmn{o|`Q=d7`r`AxkrUcy3&rF}5p%n=(KDTg#TEl>tk5m!Gv8|7m-8UBVsLAj0|nMh55Z*k8W+DlE?OSROd&1 zLR;1=kUn8s(bj#>2``O9@IF=^y|_J&`ols9%VbTs@U8{+Z5gnt68=UAdUAqaU>$*m zFLTi1lRH7QTmxZOW8mPh3gZ6m2a!u#_;Oo%vBef)M3$-xtdLSb&sOa~!v#ZNc0m<; zb7cTbw-M;Uq4Vg|F9oC*cndqx;|Jnqdyv;*QB153gG(D#!6^F+9Gap{?mI)pIgiVb z)SF5ebs0oX^})nF8&R@&Q6iEI-9Yv{lfK=zOf4e+>4Q%tM>rDuBkt`$Tg_4AP`21Gb?Oh0PyD zce!2Ypa_*jOFV!&MGs5J5FwlRo=B`d4RvqafyH!xC%Tt~AooLE$Zxd(2Bv)kg)n!- zuNKF2H+6snaSJ2FZo@6zOf;yp4*RWm4}R;MBt)!az@SP3=X^|ob4L@j6=dox< z#uqFwWh=^lz6nuBrjp?jNwB~7AlhH%f(detp_0#rP<(MW|D~4*?#+BA_$Vt&cE@R> zs1pYG@}-T?EA?2Q60Cyx?We$K)JkkB_Xfm2JPp_M`Y^>%8fm?8H}sBa3w~|UAc=M1 zM3M0fbmjmXS9ye?!QW+wq1XUo38v_=a6H$gUm;#UT@1Ca;=%Dkx$ymijR&vn=960_ z@vEAjvAy3nKzFM=s(hb^#MXvjTLMR_URt&YbYF)H9xHAGxq=Kd@2LO@st!PziWs`i zppg(fAL<&lk?Jc=vZ&Jx6;RW$;X^TyS0#$-dOv`sgcyF$(hzO7y#q^wtBG%kE&w?% z5ZKN@l7U&+?3|NmN2HoS!Fw^>J{JZzv+6)`Hy!EK#9~DUX2El}K4RsFI5H!3(Hct$ zlxOu@@KYfPV+XjSKJF}N9=rjCk7AHSZ7cF{S&nr~J&aNDMAXz00r_7up<~>R7XCxbdREAu5_BD> z;Il>R(RbIonAEOgNZFVQ@h@A@>TBCzlye@nTzHQ(wn^Y(FSt&eYQ6-qi?*QB=i%^E@Rr!$UWfJi&lR-qPe;u8 z`k2a3Kd9dQlOI}92|hh?_&O5=)xM_CD&Qbg{Rnw@3<&=R%_sZ~z7q#Vq6lN-EyPC8 zDdbtB?@%=}3*1}Vv6!%_U>SRmn3vv1U_15!mKcaA$31QR4L)`;r^ji{g2LOCt5id-Vd8|I-DTriFrJ5)JY` zeItGZ#lqfhX;?9ui_)0c$Zf`3v`=9p8VR3`=3f3u+;*5krgM7vBGUzejW-FHDBKHn zXLz9Si-B5WeqgyJX>d1+PIlB65qs5`s7}U=sQCGsFe?%Vw*doqb}EA~sP=#k!_Qc4 zgbvo^V}&FpE+NNd)@V+n9dz$JL)7+LVlHu=Kz|egd=o3Qec>5mzlT8Z>(P5;y^@bg zMo$43dyc;EZG?T#rjXxuD&xsUm*HpkX^?d4BlO++kty#on9YE6awpd43K#g zh9nGS!0+@W!g_xw%(d3Vy_SbzSHzsa0+T@CJQKZ>zl$;j0vK()2Hq0NRztRcisnsV ziH2pUV*OhfYa9l>Yhp-AR~;05XF%2A>lmw=iey}vX!Xn(l-OBxMfaA>#T$Uy%cMogRfA=S^>N)*Mx$n5g;b?w0PrG71B780gU(rWVNACkYCp- zFgDQ^xWi`JoQb+=*e{WUOn{5HB@x<&BqZYfMDd14WtsxpquY`Y{ULrN+VQtJ4JR?(d*> z7GZX~qrmoxAwKe>4rz|$AYOqsO#B+h5-ZctrEpy$*Yg7$JDiCWEVRfYHOZLb`Dcj9 zrPrJ+RR$xJ2y&EHSQ)*Q-%eBzIXB!1x9P(F7ZfJEtTsb0Kb!=Z_5sDDe+J5p41rf* zHcEN60wwd3_(R{q&;{jU;vLyZVD28U<_jCI+L;YK_w>m2h8n zFBvxF3UbWO$JGY-WW|S6l=dZ0_>FuJUHb8m44rix%?>BQiO$3Hobo_YSrzAH6(U{t zM2sCOPrg%2Bi9NR;4=q*k}fCXP}ngoSn#N)rbcT3pZ}~FvsBSRE6)${m48jgOQ-bm z^;XtnFO?I}-P!d}V8Ot1Bg;_MIt1-cGRO@NMX=e|q#-6V8t-yl2o)DElGR4as;n%*s~;Z&+b#Ez)7($^eNzqe-P8!2oQ=^vKP}wVa|-M)-HdHJ5{cG%{X*Z{ zS%UDYndjDHoAmt>95uk<;1{6aT$ zW=a}d{>VYWDO1SoU^eM>=LD*h)r48CF?ic2KXB@C#%fe`AUWQiFegjV?mO*7(yLIo z_2Lfb#jYYMEK-qwdI}MZPNH)a1S(l&hT3oF;~FjF=*bS@cjmjMm|mlB{^)m*tTR$r zZTb^(XKoTORayb_EkdqF>LBVvBMV=R5;gD5QMX?^UZX6AT0-YQWzHgjXn#B5?NlI$ zSXW78SBvpi$Wl@C*8R9%=q;ovy$8?U)ri2Oi?lfS1t$m|3bATNN*#A$hi{X0UEn?e z!OJIYdH1lj9%mpXWEJX&d;)v77h#@R%`lv}4yi4lD%iZakIX2OhFz3y@V40xFK*Yt z@19fm)J;rc?U6cI?yrce>}~*T$ttwTn?pX$)WIb3Z-R#j6Rv;O0E>mg9+GbX3p44~m09KZ2jTBr1OI#2XPy<>^u8G`EUU!V z?z;jZF5{S&o;tp-AeP*2cONscO$8=5f^60NgvBlG1q1bDT(N5tXs_zTD|QFNwumB_ zy`};!VXcAPO6Or)KaP^OKjc?Agb@$EoJa2l`@z>J0{^9317iD0s9n~HAMiRUa4jmp zJKKza`)MiJ*?$OkO$Hd+d<;G%hZ8{?uR~SuL*nz6fs3@ z0)?h9&`XJ2(rb?kMr^!*w=Q0QUOES$LkHZ^RNDw#u>BdV=ZwOyQ9iNt0SCAAEyR~8 zmqM@F3Ub4~d)TQ?TPXOfjhZWL35Orn!~~a)g&4MCQv6W3trd!TI_zLq@NDeVdR>UC z)gg&;U7?r4HnOj5EoAj4qNQVrxDt}KIeP3SVWXW25>1jeC&gz%UEw-`r}cYe6L}Xc zj#Gs18w~MGKRT>)d4XHTP)Mqj91*j4B~djp0~@M(T+?DLi7yhHYAv%WiyVAT!)9dL zTFWPD<1bYkVDP{%GCQ>twHe9cCHDDP_f_%QUWwg;wLSL2{~yr^7rTdUifiCEh2QtH zHVMC#=YF!j64{5}bK8eM4(%hI?%fsaeawL>H-gwNmw?~nG=jfCiyX9k0)sMvxLnLE zNKpJNxFOsJ3VPZHt%>i+w5uH`2h*#`eR2%li#~}5o=HG8U-y#Pdxdw7j)Z}y^aWz^ zM-gD39*29QQ*rm2mr&X)PEI%YMEXph1MO>P5iB1C7NDR~g=O)SmpNy8tGhs1iYj*U^|*7$jv7XvNPo#7K!oSW6>$c(XHE6h;tB z!;jnnD^OuS92D!Oc%A0!8yojsS;+bzK-K87a~ue%TQGQgQ$NR1X#x|c+^u5Cdcj(^N){UyQ`V#s_s@Wen>+iZ*2LBN|i)w z;*junixPaZlLL)4SHMuVoN(V6h{fnnA)Pio;agfWpu)r(Tm<>Vnc%%xWZ--_?C~4w z*0(^BCB0_eL?Wi$R1ShdDw+3iD+2C}*s=#?yvW*2xtkM0fVji7>&+7IIQhGM>* zc&cEY^;G0Dng((%HX!jm3Y$|hTyy`>Y1lkv9ro?oAk_R8M?(}U+NID6A$8gO-o;!r zYx_mGcPbi&=k7=5{~tx?;aBtb$MH&&($G{YSxF&ebkF;9uN|_Rjh8HutAQSr^_cYpuD=kfS_?mh2w&g=bpJ>Pq4g{W28~BE(Qw5q#`$K7xsz`aQdv9Xq*y6%M8kJl%%KCYboLx zOSuwr=>$A{T8#IFI6HuhZ%9YAzy5U1+oKo|vl#7d<#;Fa^dY6O zlPvux4Gu7wS0WXPaeYxF;n5MiZs0>_DX4>cT?G2yybFK&c7wD>rOz~8=4@EF)a})A)t0X;pIokAj6bybJ!itQZ;bi|i&@;jT{|xKE zLfb3w{(U&E`_V~PE6ujvo7hR5LxR!1>n7~d9Y#(pWWrXpG<;j!iVib6N&AZk_}%^x zi0=nsLG(@d(|ecdtP*g}>Jx-Z7be597JF#6HYFbw_o0Kl7XJ6&4bsvoi_U^7+!no4 zIMw^3kjv=6@U^9IZN(k1cwvioj6P$_Zv&<%y@l@oP!DUx$>j9U0JQ5l56&%#_{vj} zd$d4{r+Pt+EZg!OGpD>Dampj(uO)^6;j=ZF+bME8M@_Y>YegnfrlThk;I$9bi z!AWw1ejC^glZV{IqgPttp`@2`#<+$&6$o+t7+n_bwj9N4FG7~48vFhvh|;bi-to>o z82oD$_zz}K-55_|bE^urs_NsfS}Ms*PUi+U1!GJ0RvaPs91D9=QDPRZ=qzcEM{JjCsjEY5pPsWd>jH67c9-f|9R3A z50pS|$O@kPZUuJm?@g?jF67-G{{(s7!yvQ90PJ`A^Um$KO;&H0=0^4XAP2)5&_7+3 z$>lYXqDTYGi!xxH#+~&1KqhgUe+LJK#!wkyIoOBfVT%4g8lph4$YCz(DDTF}9_sMC z_%%+iG~%uOZ2*@i&%m4^6f~X;AW6MMPTQ=8nO)nUC{l-gIpP7Sf`>3hQJR@LE3wWk z!>IS7tJvY`L4RLJ$J73MkwY(PCI1+=CPm^dpQ$in?-n@L8->&9KANdwNo-yI(2>UN zbnv7M*Ks+6*yz0=3AcLa7&~1S7dr~Hr>T)jml&+iR%b33BzuEd41I7b6Bh^!%X_tD z8TSEcsH!^H&C#V*Q?^6qM>T9MD8KOdiC-O#&W>3~zL-Jf%EPb@SR2#>W=&ZDf!=dY4beMKR=cpIKlAz zyE0zmoG@HD$q8jA*P-v37V_CngYA#rfD0;ZpyqH5YM-5ey}AzArScEEmb@gJ(gu)S z$%n!E61-Ud1-PevNSsH4>|A-g>UNw;$-c#wi(>k#;uIV^Zh{Y&xZuh&=C~s>fjY9w z!mEkigxLRs5iAwwdnFLHHMhYLweZJd6?VR9HV*z&#oFPzWR{r>r#mA>m?!%L79H0@ z^Y|nj?9PKg!#qsRD?%5^eOwrshh>slFl*ML@_{-H$h01${Ppd4`ZNc+KU-mGe>8P) z&ZQw+v{CAF1xc}+2?+{v5WM{kOy}K(<tz`^8@1W?*R0eY>=F> zOiBNflf*UV8~K#CoosgTgV#qwXh?MurgwVbt@>o1P~$N6r<9^bdjx%vBaJDe)gkQY z21HS$STo}dM!xXH)uL;ny8+v9>0c*k$Q;7YP<@9NG_Uf5P8}*2rlgAsdgcoDOt2z* z$Gv82xtpRjIvLoo=R2ltUxCe=c;dsxr}4^uS*B^Y4Vu*l@Xh=6A}>KUwCk)CAD6p{ zGH0pi(wys}_=GainO*&;`ay|Rd{5yOKR4qC-YO&IReG%S{vNdYFp*`%4`Iuj-(ygs z9&9{W1bQ7q;e6gtY#CnwgNqgja{he57i;H`xOhDgNlPb%k9wF>!#{Mk5n*e}Y*G3T zZ_%RQ7ILQTyhv_R4u9rB6*enfmc^Y~$b3geFli@O@ZDL9txc~mG5Qr+ZP#QWUd6ob z8fg~x?I2c;*@5Y@A?$TIN!Eo3G1b(X>$;voZ}BbA$^ATwt9^(XN2J8t zI?}N0;suGI&q!) zv>CIxW*1QT zB(gj-l6|W(N4bZxus2}>v5)3MPhv5uJbHtifi$f6@`1dp+>A$4%&=(gHBr@^W>UUt z1iLY{kLHx_Lcb02TxXjz3t8uFea0l1s@&CpS>F_S+uuwTR(wl?vcPDZ&zhL~7-gn% z`xBdg)s^I_d$1KtFLGMPWY|nJ5J&hrOvmtYOV^QpdQqja2$Y1KoKc!m1``v#ETVOhVU9uckaO*Jo zi^LNny9a0AQ{`8Bj^eVqv$&a+awyi`NoO)Wc6nZnXgZ0-aW4 zx_WG`Ob#EtB(?3k6>gbn%z{iMo(%0h*tdm|#+H3BG}wzj=2jK{HGhv_G7+lED#@Bx z+G4J6Bn$tuQt(`*1zTTcaj(9VVAnJo+~52WC*WDW@xWKoxcv;4F&W-&#~pJ% zjB?>k=uinT+^JroXc0*HUa|<*YbWlhjDu^Ugf9wA7_UOcsLjoE!wolL*kQd5Zyf; zO%IGzV{eB%6F1D+$=AH&O|l~I!HduP*c-uder9Yjw)B=`ebQD`+0rYz_&c5R9T>`- zg%|k~#|1)2T^|jsHRCQV6XDCF?=Vg07M28WX5K!Wz~GD+cj;UZcPQwy+&r)m94}cWrvA|%2czsIXbW{&j2+hg=5N5C*hX(b4=xmI}M3TVkbsleq9>w0OX^LH3TTC?DzK2fAZr5r|duBl|ww7KQoj6A>bHs5h-V@p0c^_; z9y?iepI_?w3bju6iDGxYS37JDFN5`9)X z0+ZebOZ4t;^2=l&?(;i_GY-aK=GtUTTr^A^yRZhIReMvjV=8>>xCGqraJ0x>bPq#k zcH#2(mqp*zl%eS1ND}pU181xmD{fk(!SwBKV|rpbto50|jTn$Y)54Wh7^jG56=Lb? zz4f#P|6tl<0jQ3x#DoG}+Jbew6Th_Z%Cd0LbM_r~IKRQmBNq$YHn|HS?~Jf5eHj}Z zPO#l%rr59O9gaKlp1eEz7R#RB$6=S1Se|Yw+SE@ei*x5;l-^eH-#@)n>A$J0pzSrz z=&-{)xlt_FEQ43&gShI>ZF;+>jsCWY=YB*NumICe9A?{2xnd7I&?Uw>LM7g}u|J8i z>2TKVXijI|AHgC{Y+|l4|7h#_>-cQ!pR!M5bVZd`N}~B4E-dP$9(%ZC6+IR^UThxw zr`%tBP1Ip-!)!9+u`)viyCeS+y(c#%n(A?MdNhG8ovtbh&+kV04^r$wvKZG@@ksjU zp)B&|ET-_-i0$b=j`l^i{I!{Cu(TnI65lvHIC+RjcUK=@-zb&^g^p(3EzhttPn9W+ z7Ga{VwYYHfMRJa($}9?nm>GQy&rY1h;^#kP3l_RiUlS{0)=`JG#pB4|y{kAaPa)o! zpu@CYDdKhqX;yb}2@&^Nu(@76)Yn+zX){W}OM7M5Mxlg}l5+&7>=E$JZJQ;EC|Jej zH7h{H$uNFWzXIErSdJ^QZisq}71=6NFP8pPit9Qc;7>4?CuQ*7=K9ejV+YerMSs-e_;kUx~PbYxY~+m zOj=QtufuHQ_2{8nmN+Hy6%ys!?4P#=PI~0P9#DOzr}h$lX+I~qyI;XaZDqk@#jDt> z9wpJyFA26Eh~zD4x=1?CUSv+)D(r!(I*U$Jpq3|2i=3Bb@NKU&i*_&cq1KCD@&dO3$_)q(# z+2Si@5T5l4lV>`KVy*A-GgQ23Zt6_2cl0kD8LrJAA-FMTowg}|>C2}$eEL;3@vJ*v zu24$Ud)JtK-F%qu9huC}d60mjZ}xnjj8KSE*JOGUmAp^WMEI%Ikg@Y;5jAS5;_`jC zdCnZx@T~+Ze@nB%;kx1+$3dKzeV0C3JVt!t62Y8;T(Y3ILNsizG&}!bnxOY_Ez9s# zV1~Y4_(e|Qm1?`qhCT^l%aRVWPtxjaSJ7hjHzx|OM^y7ae6&Nw%X`@2Tt|_=MEiPh z*@53jMLE)FF=hVtypsy1}}I6yC) z_7{!Ws>dQtQ`pO;_vn})Yh zX15cbG@1bCq-^v&Ek&uHtz&lstlz&eIL!R3m{;;WI@ zXzY)jAKEP4vM~bJ4!-PjJZ#M`mKF&!(&qvY-!t$S=<%u5raGu3 z!f83EHmyLnJ(ale=2EnVe!9hMiYVJ!gC%F^v-TfxWKpT4cHHg5uX_RVT{A=q??o8- zeKT5KJjr_I*s%rLd7^Qjo{0?JtFX}TCa_LJlW#e%lj&RtU=!-T)9Pgwq6O7WbZcf3 zH{<#YG=K1wtgMda51V7hb!WUHcMQ}~Vc{~S_i_wh_w-L3enycsMUFtt)0=Vrt3(ji zkLAtrTp>8|ZVEmgF@#xYSO{iS>}6etjo6{~F08H_%5;8O@O_<%ar+EOU0P+0=T1Gq z6B73w@6%YepNb`pcSOUTJks;;U&j1t{Z}K*f|XRq{!x6-%hOe{lGh#2^g!}&WT#a zas{aoJo59rsG`9ZpS7n`Y~^6VqVaGpU?-L|8eoHt4$k`(O+4}sLYJh6B3*xjj@|#3 z>bRHlCR~3De?CgQA?q_RralM^Rl~sYwxq7DkOrL@jj&Kn1f{#K!>5p8sDENQs>W+$ zRpDY%H~5RFKKzB!WmjO$$YwAqO~d;!n&_c?z`Cn`8L>(j04}Q;7lH!Li|*z5t0?o* z=f$Ak4=IQctK;>Wcytf>f%_-b;Qi03;L{pOy@xHqM&%}+O6o`!_xCo;IUNjZm;jsp z3&!qkNsy?PiBZ`~u>NQzq(-Gco{uyex9B}N=A{lv742k%gblVXFBQy!uamV}^HIXH zBHcdenD_MwdE_8X!@NR?=vOv&n%qJksm(1gX46#|s;WVJujnzU@u5PiXEN+YTqVp)x{p}~9pszRS~_Rg zXpoD`#O`Qc^qv15>ph2qxOaT%g0KTnZ@H0bc6xzQs0)63Hyu2B{AuQok3#dsf2nRx zJBk0$gpzm}&shBwnf2U>oUl{nq%LgX$-j~2jxUm8`hFR>RgZ!rQz7U zDzH-bkFa;oCA{j=45}NALH?a4wvP>?qxq6(v}q^r@n{jeI8cRay*3MHM~q<+ePViV z@kY>@bd!E7c}EwV982C9|HaUsb%-zcmj2c zR>OzarTBF6J>GVsS7eRWJE|Ko8y*i&haWw|c;+8d!C|5-8>s4o#I1oKjqzma?u*!x zVZvsHY2u6W=TxKMvT)HSL-@P)7cKoHAdfUJOEkSQ=wJ1M1U@)lzGBKk;&hQf*TWz@ z;++O_-Z@~$ZdLBTkiEF&g$sUP93woMCC_q)ER_7Hg&=QXCF#MYqgS;uXW8S0M;0vS z@#>5)X!mAl38X077>`Or`iWumEl?osczInTu9VscCr+s2s$YL_+&&MiSKNt4b_E!? z`L`tJ7>F(bh4hoNJjDDm6;~)!jA@)ed$L-2e~#rz6d-b;h7qZx1vT zTce!u?D7S6b3oTam!0!_iff1Q*=j2bUc=v~ctbu2U)}pgbtlau*Up85mg#V=fwvd~ zeRVnInLYUKoe68Okz((gwu4x=7EiC-03_RhDV=|WvMG#qPMrj&cTa|>O}8XoL@^9l zenR`zpNK=vf5K%SGRWr74Ma2Ch;ge5aQ6!-XkOw-4}JA5PhV(<4=jrDNTeR^e0vu& z)@pDjR(UXd%wcqWeGp1}FHyVX3Tm1A1BC`^a7iy6rOtmWuNWOg43`IBn3R9f?9o1W*%>9sE@A+B>2C^S?Vpg<8uvGM9FiF zwTXED-XS!s-hmcvk!1O4F*&@{3csmENa}pMQ2(YGlv~KcQ!^PhiBb>^=&&d}N4U@O z?B^UQ@ZVhxjaqfk&~_UWpBS+S9a}Ki?*dmRcJT_9nZv>{fy7Bkf$@JP;O81kEI%5G z9Pc?6mr=a+E`-ec-h{K3Pr*epRrFI|I9YeJ2j(2S0jBd8!`jQU(S2$JzAh5dg)e)B z!QT^UdPWKv_I5QLIbtG~3^j#45mTVy;7$m5TZG@1Y$NZ->T#bpiSS>H6h{xlqm5c9 zWa{{n_cee!l#H0R>?)8`mdCG#3}ZI52;a>X!E5(_5VAyt?Nc4azNU`Flt3PC9HvN1 zwC~W3qesEi5#7Am=WdhFk0T*wP8bY5t;H?7W<_#En?XC~BmH5h#;&hcfx9EZ@bQE? zGQxEzQ#O5%#b-Z&U}*)-b@~HqPJO}Ows&yCfeJ}>r;gUkKLDAh_2_ULgj+2Xz$ES% zR&N>3I<}WWX8uDQVe}m~DgOaaaS?=!9Kk}BUc-VVfCu&+#%l}egwlPtVToS>j-N1y zwY%2f#Li_fYIX{4mhr&QxISv7z8rPe{)DfIau5@C8Fpqcbof0U3r9(D%Gae(_MjBY z+&&)PmG@9RPeqns?}5)`uk&(BZ=un@V020~YMECr1XrcH;k(Efh-$l?;{uH&{v<08^YWdH%Z3d6? z$+Ts|Mf{6@kBD=r~zCL9l{Jodw_v#JdCor zhdvix(m1~W?!m1u)URV3eqXCCs>|`k{X(HA`s9A_i+e+=>*NG8ZEM9QY2I{w$4Qa) z1_#lB8Rwwo>m(L$JCzllw_>J25j@>nyNPB)F`bs@M3%(Mg1+N#)Y-p;wThpYuUlpY zD{aoRBW~Y_^O*@`wta=!9FBFXpoK826V7|2 z5ON)+ZNCezbRMGC$H_P^RK^@-=Z8d+ zk1~xY(uyNZa`EDJ+huH?sS?LmLk#I0BhiMH1c4SxI1rP?H~zg7#Z-r_YStD^7M_F4 z&p+bC_4nq~X3WNMrq;~vavpAw4k6C(8ZpDhfaP4;Ps4Y#V3(UBF21GDl+TYA{}-8# zIYCV#E9Do~qh{5KZkk^c&mB60t$fAs>2znR%w)Oa77tLI5H9}lW)?FzCQX^SD_+{B z#ZnWeu&+xm^Op~O1B;xepkE{GDk+NW}H4&Le?GRQrIuVuG1?Vi!C9@@&FvC$h zc&@)y*ffcrps_oc-JBJVexnS8PnS2~k{=HAu=QJ!bA%SOMVs*~@-jt+&n;QVArn@A z?1(7K^8>y*sL9Q^IgEYJeIYtB>49iwwFQ6HxotG5_aC<|?YJb*YA#ZnmMGejW6AHn zD9284hDy`r^Q@ZAx^1GqXoElv8n#>^rM0*C2PN4>xnl|9YPGv;mi{vED9`7)ZdGN4ZS%xm zzwcu;{(8*4wUVA(9m#G8ETQ;jqd;_ChkaZ;lo{zI@K^ngk@U81h;*mF6}9gC%Ackx zxpz%6VG8eGVC;PfxA&?n3kq@M_e`}#!Bb^+NT&jOwi>d%6J0SQy$6qxOyNs~nfwpt z6sC<{%(peakK}?MhPy7o5ySKZ`KDfE+6s!zXB$X>XE?cgM@zJLj2hjXlnb@!hDs-9 zkT+jf(vz(lFjzx?Tkfl&7Y4yL0}(UP(PCeotFrTBcfwq0X|Olk!nY*Zq;f(pAaX@W@T6(+fFX|Mrjb^J89op<%IC8oSy2$>yP=sk0g{=NUcd}oF)TAuqN z$}YSHeqDxuJ%c>ns}j*dN&ckGK!muy9QB;cMCG|#LB-L79-T29rz$9;VE8ot+ABju zMiItro6v}57ETc?tCZrNUUI;j+e&c!U>lSyIf=n`0hG3zlc_`f*(BwaIA_%`wDvOP zE6s3+scSvaY4kpPTk{T+CUYR)7er2;$!8Z@f0LifG?%CQrbtR{T(KddKiEY&icZL*H`$_%j%eeB*WOg+fP)+l382K z7Ls+^5sTHFSj3BDoLbl|TK)Ap?okz^)b$XYc{`MnaeKf%Mo4zqRI==zpZIJ;KK43S z;;X1(Z0e^hk|%k7+4@WH@LmR4d)ke7emNqVJ7Xg|@AL%M4s>%4*M>-ZmUnT*;8>#k zCJR4hoP~x9p71O_lN2wT1mdH5kp4Ciw-)z%jA)re&I_1i$mNL}V4jFVvzsMZv+fC&Tn$3`lPhrRoRMH}d8Fz+IvI>8t|=!x66jVIA@nM7&y2K*+QkJGsYw19UK zS{-Jh>|1N*mX(g#uWRYlOtDbtD8g=w2b5AC|1gZ39idAdj-JJzHvb`$><^pO63}(bDA0K}7XOyyqm3ww?u)%g-XB(i zl;ul#DZZ(5gV6E93h$l1h6X425Z#+p zy@lfjI5IpqlY*Zd%PA6rcuffsi-}OYCm+9P+u|=#1L=A&n(2l_VyDELzoIRVdbAIN z!C#@)4d<%R_1bRoJ8%hF{b#}Alxy*8Q!>xqs-NsAG@yyDc^I+92;U8c!rhWL_{6yw z7pwJ>Sqro2oN);-y?+3771X(3W7fj)GIf0Fa}!6DRG`Kc2V602Br9_pf*+0skgKw# zbjpMz-oY9HI3KMq^Ub*f`^)c>6|Psn-SHZJ|2__OR+Q7$VM!8}hk*p)e-ZVjYp{uu zn)P*lEoyHngPHqG;IF?HlaC(@xXlc0PMA~8XS%R`zQhOMT}6LC+X-D;F9??wI8$>; zE~?Teicat7ggK60$e&(IW+%=={!9t8q4EfLoSXvsNd_S9P{j+D8=%irg;~GaShC$i zGCN?H7*jX?!soN4*uP94SnI98q;1E6=f=@&XIhbPgWeH*!hMG1FM6aselJ=W2f+2i zQ^@i)D(u-+NlkwukEOi{I9K^5ChH8+^bh%Hbv_&1+&+@->(8M!@D5e79m#&!41-Of z-ry`fk+-7N1|1J%k`H}5@b@_}F1`^6jZ?nklK&?2&Q7Sa-ZSM7FP#4kgPI=^8(SrA z+~Z$3`tuV>rpgyWrzkFFKf!_KCU9^Q}f8>(>D0S^)uOVd?jh#Wi83#p9aCXTx#%D z9YJ3Qd#=BwoBX}d@5L_cK3YUe3?*EpXM=P?%26DhD$HFFUd{zJdePjA?al1@k&@YUWs|^kRn^&K800!RhYOk14Aag z$6k%OPGmY&Z&^}osr?G9NGdAbv?A}&lfuN!F=RzT83Jqk}zy$ zATgNTO^qIh(Mt)|VESJ-ZqaK)FqIN!%=-rIp%;ZmE_tH2niR-xnS0y$f|5}cs65|%XE1ki4N<+QWtgh znsk%rInHF2T^?!L>jztpIKb2KW;mcA4ITzfxNW(DL?g0;4}M!PeS;!v^tb>4w)=7B z=|6aInl~0%6qCeFD(GY%j<@s;Se}drjqmv_oNfJ$ZfvfBt;^C-IYx{94N+&F!S6v~ z0R2>RD*zYk_X=1LR+0 zqGX+)1eNphEHXisS&s_9WljHK(207wNic#5BNw6Vr>WF%=~!m)s}slHOhUEr>$GQY z9Nv@oG79&4)3)wR-l!^N)V!BQ$n&F+urdesUB5%M&nvO7a?{z^`5yQ;YACBL7zf5t zMY zqH*0b&fR?>HY7nQ~szLm3X7-f$g~|6`K2F6zdN z2IE#+VepFUc)MFNd*+8H8$W3#o#%EArS85f`&D5K`&C_$8W+Q@RxjNCUpU4XWY9MO z8;Hz5Wj5;LBQo#C0N#Ds!D}YAICEq@ctixlt)aE}>$MNbjA`bk$a|8tv+{9kY#K2= z{Q&gpYFOdAMZ}oNFp--t$?p9J{=@g8!Ce)qmRQJHE*zkxKkuMN(5~{@-0N_^DhCWq z;*i#P!Pzyn7^fBl<8+mnaQSKBt-fd2aPu5q>6U!|l_Q{JO+DxBuE@#c-GtFEV*|5|@ z3d_26N$#mySbF9?@tDakH}3Nkj(_=vec7pwp~JJ0Y&`_e-!H*t`*^BXqr?7wFCghw zi*WLgTuicjj*Uj?q&r=rgHAP}g7w)X?6xN=k3R!Ppq2{%RKT6PzqpnyWw5_po!8c= z#Z6DRi(T@Ypq#VedXuEN;eQLT(O(BoCCF37uZcA5uL9?GUy+M={2pfM^+8#fk>vd2 z#2Jbf;o*S;kl!-~I=X(~VAWzcd2l#6DA!M}%*e+#vC3Sou_twMNhRl_lX2;D3JHw| zK}Ba8=-=}MgRDsIfmnv^pVdh!D+gh(k2UXITLd-q$|X$N4(6+wakCeTspH%l&TpJF zU$NsL%@#xTcay=fJmW{`RV{nVcWsoZI!+=LmVD7w)T+DGD zJo)kmo^rQB!^avl#Xpqi@l`}t^bO#5DPwYDM=<_*@`Li*B^;0~m*B+82Qa2S317dn zBdb1m@e0jzQTD+HCb(>jS;I$h#RHq5@x=q2{^2sYQ8t6h2)=QC#mQvT(I8H|EtxEG z|IJNxF`{Yv2o$Py(5~tRPV47uvO_tW6mQIguLpuLYJDV6v#yiQww?xY!9w_QvI7&2 zPsV_edd$-02Pz!9PsXpi32V1phO$mm8ZxdKcIw0;dAb#zrsTpQlSrNvS5bB~@iTA2 zox7Y@%Q)y>`-A7*SAhEyGGU@;Axo*dB($6)&7V4TD7pP80E#Q#<-<=*xEMuGkO3Vb)9Le-E^F;GEr3D6NS6JHqoxqn@pfGPk6b4*_Fyn}G#0RiOCnh5zPM#3AH5c)megR4367!-$GfRx%})a;}S zG;cac+|L-{qI^gADs_^2bjHv{3k`7X8^poIV_2@uABZ(;z+*9ssN8Q){26i(-+2Yl zmb(YZ{J2j1pjC!ujmN-0UJY&L4iM8!5%HPgNqFfuI9o7cLmg$9vQQVN^iJm5jHTJJ z3~zK9y$=L62jP*Z3DUY5e4kwi#*?h6{0$veYPc2rlT%@#^A37g`y1VVlh9wc@~P7Z z0SaE;#_Z?vqC+1TF`uyvimqPctot3g>3!R9$dy36+q)LbZ0aREsjW2C=>jB-G2wQ* z%5zu01fyo+2NJXIExmv6GH>S@NA%|(z`whH;Wt}k`aw@P@L(N!*P~u_(7<^C~z%& zOa|{>#}^9G_-FH9dV1_tFyALbFYNvZYf_X!U3WLO55$7=s2R{RRxe!-swX)-MnR_%(xmCI!(p7`nPbe z?^oheycmMR6Ub$yS>*CbPvQBwMqF=X8}DPpEflDIg24Z5;67Tgu>-rne#2~B zBFT+rcV=L0_G=O+nRRr@qzVi5!^o~X1w@sl;Nrq>_(b(J?aW;T1AB|n<=j^M?XVWl z);~g?bc=+;@{iQ)dnDmaoT7~Zb`Twq1U&;!wvE9 zRK61Dc71|^x${xnY=++B!odH5nEH>n2;<|%Vwb=iwz;Ok!|Zi9Q?D3O{+*)Hit}K6 z^qkX*>@DNC|I7k{Z!_*xtO4Bqb_b&cg5f1f-y7A z2VPkXC!4S9uq`rin6#r1Z#*gki;ieC*dWa*JhmgA)1uHWHUPFxv!iO4Du~GlGP<0TpA1t92*$uOqlrz(cd}T* z1X-FM2+HnTsMIP!nSsV+xK&~Sbj3Zm6m^e(!oVdKJeCQi0{PoV7x2mc7Ug+v+N)GGF!>=0(?4HEuu+dNf_vDnIN&F2+YAk|> z)xr3(O-M~`bs&5544iR71Q8XQXcll4q$ON{smguCYJDu)d~F~D=SH!U{yy0B=o_4u zeFF{eMsldG$Xq_o1|`uAKzq2I}yKT|MHW80?d)1p9HjaLJt(ur0Zb6D3TRX~Npc{>Vjl#mvSyZA8d1dZ?dOhiZ3h+SZ+aAJNm&q?Yx zOkJ6Z`kAS4Hn|3ujTws}-NEEYML51mnt~A}ejq-yh<9Shb}%iLmx)ef7^{e?Xff5YmZv(cg;oK7;o zMt4X@;^`n;Qh)LVCZzoUn@MdDHnI@ac4Fni9EMn26(8Yuq#lwpE3YFO5W)w~QuN9Kp}kZ*jc-ReY2Ef$U!(*<+2w&=eht zUTGO%5@xks<$g-B*5s$s+X5E7MS%y!@PgN>Vx zkzE0~bn2@p5UdF0>4vbm2>K>KOAqO8CK@p%>;SOOs<8k{34P&vuFrS5|{FRK5uTj6cUwJ zaG*$!liCFE>9QJ{O;cvGcdkHR-5`?Cz5(u5Y$4)bnyj&D8LSIr_>07C;WAu$Ist#Gye8%IH$zF|O#1q2x3F)i3aVY~CmxBb ztlO?W=e3u}GAUUtp(=kP8f|z01FyH^j9sxb;eiLO)$|eGaMFi4^S+YKEB@G#VTn83 zN3cf@CD8p(igR;~B~kXZ%LkmlH4@oxv82<& z8)wL8z_dqtIMMG0SpPFGKV2u5oLy(3+oDpCmGIQ`T8^M!vVnxx(?-2EmBUN9YC82{ zB=O@f#viLyS%cnkz<3>ZJLP%}GgU%f4)H$8l@r#2StQ>3`Ipp#(qJc86{#GsLmIX;r)b$eRpL$}Kq?!DNM z^FDALOHEGF&AB%q*|r?#>OSF3{b>oTPi7AbslQ`qhaG3l^3lpk6;Kl?6c=S;P zVor;&;Ul9>qG_lf6^ji6_QKoSq>x{K8IOeO(R*Tv)}K`lGtBLv=yw3|V`VgDRyMH5 zN?cF>aBlmGr!YqK09=)>tc?xT>1y!{K^+jO$nubftO%t zkrF<*T8@u0+;HB`k7-vlA+P8rDcu?_4ExlL=Uu9hE=@rGlV^A!Wh;JzIMR}S z0X926!96hsq^l-Vp9=pkZ`O6zDc-PT@t2(d>NU&%>&yn*PyS8Jo|7) z9*Wh!l7N${+_?>had&w*}%{hhAisq!^ z%1|71>LkxbuBK)d!`b~HIau8lj;5A=0HIU4y}K@uq zCQsD#I7GhA8jULH1(?;_4&#OdQ(Xl`_V;WuHc2>zqtMtV z@i7K0{3{HX+eOZp+hEYJOJod0;r5*q$+6pC@R?mWUP|$USO25v%mbU4uVJ_8x%2ff_0}g`_n#DtTtK)h z>53QjskJ1iE!ZlL4l5pJiLj#W;gbb1IBK5L)lyrIafOoY_2o<>tZ+0X4ZnD z;C-~~o`wM8{_#xIY+~r74;mH;*yrR$y}2s*r>{Zznj4BfU&S+18i{MN3T%CO39tC7 zLZglcjCCzTAHQTteYHRro{FS> ze;r^(;dI)*@dQ@;ejz*fY>1LA|GQ^gB>S5+*sY6dEc56MvR^Zr9+-2UTs&Y%ls{;5 ziM{-e;>vZwKgWHzsv;Tl__M>(^S4mt-CEQxI)>kDjiB!GNHSCvjk{kp2*jG*sqs8j zk}B?m@&~J^=YLwLQu&-L@`@IQgyrJWK56l&YRGK;M!vWtx?hb%j)LAX|{{ZhQS^kCGozO~`)a`%=@5<2DZ8m(IA4e^>0}GRSP`d`UmHYya;-`f1#k-5DrIZ@eXlYoaO3FewCi2 z1sUB~)H9pU0;}WhtvawX=?Pu_=sTSEGa|G0|D@Aa%F~S&7vS4WPu%u>6EY8g&~4*! z~|FXZASgdlSdL9?km9Wg%nsDag)vP7cL?1$~pdFniYvbUt>Hyy{p= zuWVRPFHef4XMajEsn{Pv?Gr{!qP3f5{MHZ{o@}OLT5=)2o5SPZ~?b^w{8Aj`l3SP7daK;u!}yocgbVTHD_eG;Qk#XRAci zc#=sX_q-!keE-U%dnrttp@GH?lOc21M6%ytDX1MMfY zL)+olr$Gn_T7>FPN2B!CRy&hnw@_=1G0gMl_p#p6toD8x@lD!-8B%Ylk%c)f;QvOF z8s69;_=;DyiQv$6-m9VY1&_awpiAzhQpvv4aQuNhF8*@^>rX8a91e*Cqm6mEP^p(r za_ED!m2vd7nmi%?5ulMijN6>-$ali@x$JZ|`hL}Ix^ILvTAksU$a@XY;OsYW>5Ycb zVmEN~-*Dn|@gq*s*oS?4()ed{27J+|0?tMT4ZoylBxrvvh*?7VGLV6@mv)t zil)^Uj?h?>ZLmdhBecZa1MhPVLM`tp;IVu<>L2Ib?h(US{&q3gpwx%WCsdiA)@B@4 zwhT*$_JOaY5f-l60PXF%=)CqL6Lt=ABW=35)!&v z12(_ki2H{2+PHjui-#5|vAP?w%-g~yTD~`vE_1%0nr8|wY0@y1`bRhz zSV&IXHx%rJU$`^i2^y~T!yp4MP!Yw#&>b7BEr|t_t_C!G@dG|bEWkxmZDE^?JG`)( z4OI%YkSsic-pV{1{N@Jww|Nw%*<8op%35gks0R|SKf>29C0NFROsGjUby9LxP0oVVg^xYKg^seAFHicFjfHvr1$Vl+$VHsT_4`qFOD;Y+rm5-P0ZYT zmQGfy;GIJ{Fty8B_`Wrs-!StqdvBu8Mi0ZKS!JM* zc@XEGG6wHM2~cd*NwP~*!F_!s6eQ1pnmH?QAiM(H)f`5abBe%P3Bp`?IZOtR{T!e2@;}{UF@1w@x^1TsRyO z$S_bmh09kz#loOm_?59km{&O$X@x%1Rf)p*Y=X0NZ3TO#j>8zq>A2>Q3|XQe&F%iE z!)Ju6$oa-<)L+`bbFf##x6mTAK9@qX9m1i_PJqD^Cqs^&EH;i6XCor_V5WHNh?B=bAd1HT(?BAq@SrVbjh!@2vR*QWvk zogdOTFd^GFS)$n2QX0se!D&^W1@HesUzKYJI4_+YYR{ zs>5h(vvVm>Sq8L)gADQ>eZp);;` z!;NeqR*y3SI`t*hcyEfLD-%#wY&jMr$Y7v~E*oWi8GSb-;Kb#+=pNJr3)Py)93xrq z{18Hm|Fog(=;4B-tL300Ooz^LH`04p2}K^3xWPBB^3=u#tl1ZV?QaG2;pQB)^xTg{ z@k0DJcQHrJ1}jbJEV^^XJW=@?QBwp=)%<9gj->QLy9KcwZ)~h z=ff(JQ&CO(^z_WS2H`~y6HGRIF3g=6 zkHsN3QT6#Rh&HEq?^_%ij$DTuBqA_b?gT!6%s=1=*g2C1Rb`e7`Epb z_9v(@72^@)u+|7XJ9ZlEeRBf4()6h9;lp^&@C?y?7*1?HxgzOK#(5@uj!Z@dJtjwh zeNYw#NHpVvsPQ=7>M-WcG9-foUvZVn%BuPM?xA?iFYGMr7u<{vqzcQPpwpTjXmQtJ zy>+@+oO=v6%T2*!lZOfKnwjxAM+NqppSk8c+rYYs)1YAPIr!=Fj4Jb4!|3hOEO_ij zaB>vmWLEc~-8FBleRxcV-loxrwNjhmkF$os*_2!L%3;cL5nZ zYp+pg6s*kDS1*K>D_4`y>LHC_m z-8MazDO5`02yRHWp(zg)xPL`=Fkx&i$hq6X;*4Y*+0;yQ1;;_mcqFX$8i|&>4}hW0 z7QwVBC6G1F3Nl0ciQx}>4ES>wAD3K#>eZd(*vrkJoj3$bbY{YsIs>$fctGwvT1;cx z>xtu~RPv+j81&s&07G>*s4pMGUDJ!h<@09=!rPSj-qv^AnUIIB9+^ny@-vy6ifl)y z8umth<9X1J1exxmnVng$?cS_3+_b%g7E1FB$KXx4`OAFz#UKF&?|cQ{^?CH1XCXLM zZG{&CWp12K4v8{Z2a@jpf!Umk(zj^h_GU$asFX;+IRUcVCB)lm<)^AHn@I zD5n?HN8p`MLvBWX3gj-2qoKbS+pRlPhiRcVAp1`&D1Pxr7k)SERin<1j{8L>&Q61~ zmZ#{U8$p6Z^;{Ah@QNfUo+V58LhGFN+W>`Lkmkite8f>nYM#; z`3Z2J8iQNP$ML-ICm^)k1$<$T>b-h~IZ?lbeVbEo;9voU?sWyZdv$m&R1S6Sig7B@ zuc&(_}fK1qqXnet35+@~sZiie%$CpFIf z<0HX9o(3Iecn=mXjR5aN5n3HyUu}8(V~1z0X2R(LdTz zI|@>iWH8+QAO1Cy#J&n?a=+djhC$3*cm{sM#}sI(hzHrv0$jCa zJ!INx!X48vQk!xcp7(vhp!#!k+e$n13|S7pn_GzXiWrpDXn<*rcQEeJZ>-;6Kr(K9 z#nIUr*fJ&pbfgvNrn5za$j8t=gJ&cpH6DteO#;<~Hn8LKRNY4pp|hd`gkH45`k|#n zZ2JqkiFe|>`A}Esw{R3hc-esQ(iTeoRpFfSnYiTYbUa_jI~ca}JIzN);IL=}=iIsi zgD3eo7}F)UV2q&CTm(OkHa=v-IhLd7Pd)>}rJR`6VJS4BRH z^act)1cSq#S5;oNLi~_hgDGW~1h3{Y{2h~oF`gs2MURd`X1_9gYi)qrzDF>qTYHA%<_eXkDKoH3)8t&! zLt#?$5^`eWFi>f|56QkFw3&7pWUA|7>-7TocUpwxPZLP%t-w6V@1#BWB-L-Sg>t!G zOc^K6Y)y5z`xh_MD04d;8*0STJos*ZRFlwg=6X2sOa;8X=RjzhAv)My#MGa0Ld&1^ zFx>PC_KevNXMOIWCEtT{pBe~ln;*i7^3NDtx)A=m{u=w88w5qGX5u{l9IBa{PxCq$ zy&c2v7*{sp=A3J|Wt$n8T|2}(dm8zjMIET8jpRb3>Tu6FYuq`>6o&Nf;ulwQXyi6y z)@*AiIKuOv{1B@KtA(;I@wBWk9CP?*u-az}1jY`M$|3;-$2*fLy%PoZ?21W8l?Lxr z?57*%l%Sbi4{Z`4g$2hz)%FSuT8q$MqX=B>?!$#fzF#PJ7vA4#p@}XzIBA0fXSa1E zH%X_1_Afj}msIDGRc$&f`G+1Hk4;C3-8|FJB1X`F1Id3v7+RvtNryL%g?HN>*}d>O)9#Y?jV_Z$LX{?D(rEG6ec|@ z#e1DnENWpYScsF5YDt#(y7l`Tc%Q3h9|`!u{QtjxXP>rxTsT zsQ%jbbWi9TC>i+$!_x~O|3oEhcv1?PYL}^Z_Cox$=%1i{`U#rwNFH|%63pG(EKIuN zP8ZJlYA3H0gw^}wgiXz6E!xiE9{Idc_{m=7Rh!tok5*JKgz6=$l z!~mkpNxxP-y7T;{;EwN*nf91Ev`7htm&>qkzf>UY;z+)Ckwmp4ZW8CXPngxX5@hde zfROlnsy(;~mcAyK$=5WpyLRKfzz+Lc24 z|E`g7Deag%@t>eC;EesH@?>&c?yFd<9%kP+GM;X*>%*7!sd%p=ogQ%bPQ3gVk;JP?%<*G1D(G}@ zrK`@5HCYgwNVaRbXw@)`I2P8tvGyk{z_Fq;0WQ|RDsA`e=?IP&4q6tV$t^1 zWi%+7KvIr};zz+LRO(N{@vIRIe-8;7gL^Ra^$JnyssAv2>k(pAqAH3RR*rd5!?1JVL7ehk1{FI_GS@X*S^QHwxWdkgE-564 z43}ElJIhSr7Cz1ruH>`pMjHEI-T5JI-H!2~GCiMn7iXaaK^z&k8vM>ppbbO}Co5=+ z_64T_6 zyvANx(u5h!F%&&{zlq-J78mVvUcou(SFt+fAE@{FA2GYbcN;4BGhOc*S~qbxo5f3% z$BsLL#oGC_HtL`#zg3s{?KNOioMeGVoZ-B(B|tlxA-ixds2rV#w+e={VNvIC>472I z=IMn;eBWW}&Y7ZvCX;b}OgKB?+>hxh>zT*reEZo2Mr`l>WA;OBvUpwouxNAx&j@Jn zWVv@wk^-MN_9%$j*DQa|s#yq@eVZUE8rLTZ34DoL7VHtJEjrD1m*0j{--cDIzCDfm zv+8glyOpLLal(Pt(cIvV?QH3-2z-%1Y3x1VeOX6o*ccnwa4X*a#LQ!M=A0?1-5o6| z@?M6&AN^oGYn|Bp_`7&?Zzi4|6G2b=8`-N&-M}RL^qJlvA@9OD$PTXb#ZRfnM6E|$ z@ct|fRFqLCQr78qU!E0;hK=kL?ag#!>g#`UGsp4X%OMvis+3^oW()Dr>~~;)Qk9kz zw2?9PO3eSWh&~pM!cSp;aDu>-eHwD%{Cg6~Xz_6(+5g7V_i=m|S$hXOe48)2(__f4 zw2RQU`3)WkYNZn#Ywe38M`GArj&pq#j+<^rkwH-?Td!P7e}1@&Ic*nl=yxief4`Sx z|I)yZ_vP5Tm<<>VH)xFSbx~x69@*gZgN-i@r-Mtw@x^^B(bYc>z{j*6!}uAyyucf0 zTa{y4vIYy`S!-vd%-D}v7W9R~anYEbyBH|Vv2W(h_|#S%9t!O6WqvMk+c_>L>H_ARcqUkIXBbR;BEsadE%5o3 zJ_~ndOls;;9OV#+4;Bm9&9Tnx+|$E!$DOV0acm-Z6raWWJD#vbCg-`pifNp}G83jd zAS=>c`4-)FPZT*E7>>0WMvOBNFhzEZrOz9|b{NT#nNOB8@@%Yq-;Eg3&^!i~4VvJE zx&MiV4?9dx@ZEw>Tb1n(9jKw@Ss^rK;3#`|KMTzqC74vzZQ<~9$*g480WM&XCfl*A zK-A6i;qA(HV@l6qmiM`YZ8#yvGFs<~QVdTrvEC-)c+Ug7Px|mA$t~37U>C}mhSJX! zNj%@Uk>A0NViMSa;VzNb|G1QyEWJk8zO!VW+brlW+cr$B=X3sTyyM5f7(NAPvsuTB z@SKi6JCGaBg6554D?0Pr1kWJ_Rye{0kAI5>M>$ETY2AQ*T{qx@|0lFx_kd>c^9L2Z zKK#?CfJ?XMz=AE!C_hOHmb@~?A^vXNJ>~(%*au-=$sUkXu@fk}T@>CBbV9m;8ZN8* zK&H9yY9yCTRI15D<(#^zo|Up7TQWpXPptvHrWRrR=m#kIT>-Znp2RpsMWN#9-Js&1 zC3xmkfxg%DxX7`cFdUWf<{UNlscsi~p4kTCp0b?h_7*a5X#q|(OF##iE5a?W@4}R- zaqO*+91GoG3}X|=W8K70L^oF-|CN}qu%1NzoFU6yJi~Jja&%Z=%3M%7q>AZF#^ZHE zL;U+NA2j*wMZ(W3!t#-wxbn#$ANoCr-&&60?&L^}ydeh0LRByv5l$YJ&j8QZF`%*F z8D5b4O!wb9k6&gUhpP|n@Vmfte0pdcUfev6&h39rh4!|1pPxl%nC8>)2n)9SsWyfR zpMZ>AEhPL?Ml}-(hJ!Y^?Y<+=jMaeVf(7vW>lxa&a|S;`)8#m#^I-_bWR}~# z{`w9bCOo3$Mpmd$@d{QxT!xot_~Afz4XRF0#ZA`7(fmJc{8%K*Y<8}+Vk6dc6$|_ zQJshW*_uSodor3E?-u6&JIpg7&w~48c^pi=M~s$L(rVBBAeZ$DW=n}k{~~1+tJG#5 zb=ROcO(d|aZx@Qsjia8%;^aXPf5sQ?6wI2b%&C^o!b#R~!lHtafIRba;{@JgmbDSg zuiE3S(RQeGF##9PD5vl3k3jdo%RK8`mCZl+OE~pY0NN-8kfHI#5M@$;x?Co1>v)Tc zH`ap_A6vE_c#S-b2|{nphqGblgu_imWah9*bd)R)`PL{3*wh)z>FG?!?C<9g!9uN{f{(UTk?}ubl;+t7Y4|^ zYk8Pw?+a(fZKmt2KcTY6Ao1R!%o60ZxaeC?iA!*pp!U@;CRPw2G`=_iLVS4U-}x{I z^z?yiJ1)?`Koe{@mI(SlK_-CTot7_7GUk1b8sjK= zc7zB~J9Uzn7mR-k$q%xcsjVRHfM8g-!aT%}Di9-5aYIvJM`?@-s zxI~IIwsE}wQkq3nR)Nx-7f={?ls}t`aD{aa;++JV=qibGe|ZSEY`1}HAqK4dtPBoC z4PfBdAN1rJp-_8s8oj zn1D~Vw}Z|)MXu)1R}@=1l{}GBVn^@@T6m~(uCK)q_6*Y5_rH?r+;MPbRRgv1D}exh z#5B0+GS$}VB1wA>0iKToz>q1e75571ly$5k(ik73sqQluto^Pbg ziT5f%-{cx_J93W933y=lUz9E<9(o1_JSW2QKl0est;9}CDf69rJ!TPGA#|`7V-+7? zqtXU19QGm{#8dcZ=*cB=BFPJVewdSrQ%+>eVrvw*I%0FvN$?z=0@v0JXP-ySCoA6$ zLE3Ttb4+>$M^=mQpnM$O;@zj}wyPk`(?uYbb4nQTEsdI3KHxohytDT;@9L{v0iRkX zq4b;E^mt7@d6SThK^L}wTFFK@EK0)polQdD;+OP=NQJph?xgO5U8KRq9R2=j!XNa) z3E$H3WKFy9{q&Kz>(*rM)vh8K`rJkhM#RC=-;ZEVxiX%8QV-I(?=i|R6Y^FN>?%Ah zurr8+@L(_6OyxND!rwy0112yxsT^;#xk1h&GpJoA#vVPN4mwfx#3K1Vt&R?X7u`Rp z*p+U(z!P;~dasb+VTOr^wgQWh!XSA`T)k*N=A;ker+;-gR3C!1#%D?JxbfhSSjF$j zc+TeVG0djqI&o>{c|iZ=p;nO#!QHoUDEcz>@Z}{*T$iOG8oq~g*><6wXNCb zFy}}BdKK(~-mQD-cE@&s`n&UZ&2S$afMB~nL%b`q^f=b_RniCVaxpprNkrFZ*llnV zuGyc(sUu9_mO?yS?hJ)AQiG)$Gw`k@-yypTJgb>c_Rp%t{?8*>>fbD!G4m8Hyqtwk zQ~CS!VLRcHJ;8K`S_$^wj3h36Ce*P|9da_@J-=cY?_#XHv99zS$zZ*ODq7J zfVZUQF7HG)qm<4xN5|ME)G96@HJy8DgzZMW#rN>Ut?a-nNq}P`it%6A6WDuN7S_%v z6Fg2&zz>D@K&hq{_Nj$|VM8LB__GD=owj4i97|e%APbX@AEGy*49xJ_xJx0QE?u~jOxX901pccb^4wBkS9=CHk5b0?w@>N&TbHSLSQ4h}3x_GWUoqPu z#%{2p3}bzs(9=7l_{x$is)(1+crPX_SS1a7b3Ox*(0xTp7mX!OZI8KrO{a; z5WTO)&>BTtmMqS=Ers|tUk@j~m~al|f6`1+|2u-V&rOL{N{PVVUY$uzQzq^M^)%_3DmUg#H-to8#v0=R+b_H; zB>YPfF}}ug`FKwM*<4AMvMU*F<@IrlVjFewcv z_(!+Eo(e<0`?3{f4Tmw;TO*iXB)^+Jpou3ce!;hQ>A1=z1Ra_t(!JJWtK!e`P92^- zrtDcG%y3g+bBAAms$vZ`cTJ|iV{je*TJ6JgDDDxhA0LH}ZJn{#vJxdt<#8bBIxL)j z8XNkap=tjybf5o&GL>ccch^oja>O!vFi?@*@iv8H$v(o$=i_0Ilp=R2{sPsvJ&dRQ zZqXHK*^tU}#a-AJD0^}Urq4-&zFmqeR^lMF2u^;fR_fv+1g9F=oh>K z-JSS!j4+Gc2Uc@1GFXnHmv#k5swzWpjQ_tvc=zH zsjqr5tV|qEAAcZZLTm=H3hRdIuSGcL)g{Q`dui_@w3fTI@*jqO~LmB2Hvu%K1ewKlcxQ9MdVB z>`O^p`z2Jg8KMiM{4i?zABb9x*x-B&HdZGJ?7#B+KD{R4v)o{OZ8ioQr!5CFjdk$p zqXJ8Dm;(z+LrIL5BG$>Lz|Gn5G__KO9eUCRALM7*Dty&qWX@R(*y0K2_;*{2ZvoHv zvxlUoUNkN2H%)LfWDaOUK19nipI^QB>@G*AuBM=FJRU;7B%z;t7F}SIfI&ZGc-G8j zL7HzrCVkT97R|c|zGt6drraX9d*7apuo@E9j+e*c_m;HEON#CJz8_(99Y%zQf`_~U ztCw*{?yot%yPX9dzYn2z?;xz$t%CbhBe2gR6ZaZOR-&&Wv(Pgpi70{eR(-E{!p?sJ`8gtN|0K<>Hpk+kb)oRZu#5&8TEhC@ zi}AhPTG+F!pE%vhCHu!*paIi^afG)H`}8#k%O2#>d;U_~)#zffC^s8JHg(ZVg<3)U z&K;Psv>X#P)VTRdCQL3_mpf@9#@wcBum`hiX>?Buc`4(Mx@-Sn>=$(~>T0J6a`9-w zzr(xh`Oo#G0JaH`UIjJh}#m%5LK|E4S82%iWl8FP|43;Bod=4fd5y#U@P3I(73Zbs*Q zU-|iWK2-Kt;vR<>R7g5Yo<1!VKHhT%6FpDRJ9-23V#-hA&UXc8@2?|{U%avFZ6rKO zsUn)2YIsLW6`7PDh+uG>KC_i($vb3OnEF;!-|h$pb5xjDdMVUyd?zrkI1h!k%KYxj zR|xMTF=4m_yIQ-7tTt5T4!zDrRWaTx)D{C55_I4!x{|Qp=8YwT6%kc_CLG| z9s#bqN2nNm|e}$kER8%eD-y;JCsbVZ*aCzby4K# zh^07TUM%U0?razI#+}e z$EzE-uE*+ZMzR|$b1OhMYjsen_(8V4ET+k$syP3V;h>`VnEq$77+pkG`2G4}oZC`I ze9+lG=E@IDw-Q5T;V`aj_Y|mB_(-?eUqVCd#0|f_(}J=a95MU7A&TAJ!EI7!xOcK9*0%m2%4NPZXjvC`?(1iMcKnoX zSkMfwSJZK?X;<;tw3BdTm5jZ?-MjR2+8|2Yi-#$3YET=ah2F=6q^9LLYVH?e=Pr3X z*7gSv{SBRNQ!9zyJLOm+B#^AXVVLvMo{bu@9sLrz(5@na8!|Fs3tS3`d-Y>d_y&nZ z`9k|~>eBrFU5|JtE+o0H0fSV#g$@gjU{+2U&P#ksCa%*GnLnO_bu&+Zjh;4LxLub0 z+?RrH=4mkRZAMJx^BKXe2Kyol*seKB3B;wVMcUw#`PmCxuk)IgfWQJQCXSEJA6CR=k(!gH5JEczS{++k8r& z-R2CTv2BPhL$hu5RQa+Tb%D za+OBG*3)iOGif4DcFwT#z8gcecU-{IwR*T9x(SP?jl3D@d~T=Tf3{o+B{wt(0(a z&@32oIBajOpNUm_d9P}Jq0qD~AO8#*Qu1XSX6}1WS9?}s$#PXtPBP&Apm|8lQt=RI z;V^eAOqr1m_nO|3Tf+_Dl)ng-5AnIQ!(*ZI!b7|h#1Rwka@_E5H&^{870+$@4Dn8( z*u8i&m{p(U_RQv;ecjodyG1FEb?<~{m6z$#?N!v!rHnlI7z0k@Hj>|FHOwi*p62-1 z!}*;jaGrfD254!4;S(#EF<%qhluWoQlQme=i~>}-F&7s9Cy9$|kHLkJ@#xbuf+@=- zRJm;p;$jz^$LpSwAkRhG|8$dNuYDD4R2pA6O=*1ogbjRsyGa&yskQXbvU!$C7^4JCxM#36bpS%34T6+E{XBfe6l91a5iK$ zOpORy5(B8nEY+ zPT>+^490$ZMto+gVDF8|;9jpMI2YNC*~5iGr5k&ME2{GGQCloO(~!pF#kySWj8x3r zzgzf9b0XK-e+cbv`jf^t{O92t&yw@c;@4R_=$3KWxUW^2+k8=$os&I8E_I)=Fy3T+NxmSth3n>`&Qj0A(*av^yoA8SEHC!Jnhr!ABar{RM`!}Qip>^bK zuA!m|C%w=n`NG8l>4B~EzOe*}g{L@Q&5Z8+d>{NuZV7&Q%z~`vk(e*@hxE1oCV4WN z4VYQX zFC2H=6`Kw;U|e#Uux>&mW^Ne9W*pDL%fZ{AXas+cEG)qfbOEaLAEB3QLdd->9%L z*Ot?a$ICFKb*U)mxCw|!>IkwD*Fw?xBDP&egY#M(%6pTvxgVRfnDW07tc<^phr3py z>B+ySJ4=yyZ&-lq%9FScD~J6*3vXY1dnsBtp5Ps~Zg9lV25Q$1ho5cbkY08LyA~CJ zNB?d9{+CB$SKY<6b6cp3MhX~OuY<0dl_)-0jwOH8Vq*`h;PZ#})a>&~vNq`?|FX`LeW7))wXJFa;V6rbupQCfH zfv<%Wckz`Oc8)xVnd`qnboN9vo@9ud#g)10zYhdoWydhvIa#d=sJkpkarFXKbY zOE59%5WW-eEVah-xc{&^`LmZ|o%1h@bzkD_}*-6x7_lo#svW zNJbqhhK(0xxYfGDNSInK6tc~HlVUQ14fsn2$t&I z6t->8#C39C1%bJ`DCt=YD#1&j@7_Ez)Zq@+D`kk-DDTP@h_oyM#Sw$l|$C0JDXQxF+?V}5xjpHEZZ0^S{^ zQzCB?UB?y(Wj^^I7~^io{td`XbkU_~3YN&+mnKnw&u)ZMod*F6j(5r+Qg?P!M& zCP|P7fp<{z#4)ZqsY2Su-hd|;ow&WcxwKm1M z0fav{+i?!BBIw%}1(5Db@w(p!u#Tyq@2%VEzvcFj|MdpXMIN9(B7150{tHA*DjL(Y zhQrFgG0;z!VBjy_wA-EvuiI5whoUZ1(RxHjeY++Y(C;DN6i&kGpz5l)f}hme*a50P z-Xu{?gx+`b<8!a)F=KkDplw1U%KjQod&(Md&L!Rnv30BwE2<_I6T}5O{)E%e5)V7>)YdiULI6! zkAT8r5zYw>#V+|*}(rq&R+S4bAtSa5a{|HzWh z`yq2pBsB^8g*_!BuzqzW7QDSp-!-;i!Rg0#c|jC=s`OY)&obP;Z6A(xi=?BXhjW`o zT!KnzBiI!=f?K_=5iK0T$ciLMyb>{*74ERbUX^5?=k9^y*L;S9$7&#>ErX8kc>tb; zo5->a!??@a6xaiaVhp|PZWsKfp6IPN!?8cZdH!lERBd~Qg(u;CSSH^|vZoz#5!l~#4b47H!ebhtMM9%8({%T0Lqx)|9mH6CcM75vzz#1)<8yGlng@y@&N zWYGnFE|k(t%f9Ti`%}^g`)Z=l<$w|ws42yM3dOj$7Lx34kR=PbGX@HN@b|574M^M3 z3a$&ZDe+DsUk7Y3xpy0+&okiiU3786a6XSc^(N>&zd~x_44F?qhvKCMBrYof^=)sF z5AU0()PO8!{Vjl4PL%?kr@z4SS02PP@w?Fw9o)L03e47wz;5iNa7BxAaO$TDp({w} z;4kXRsS9_lc~}+E8Haz_D|+4KG7h+o#YDTwL{mjmFlYT;7-M%GvCkjRNPYpCoO0}Y z-j3_HKE;B&0;*uY4io$*!q`vOF}?CO{*XU`V!lTzh_|N@2d>w;*2@;44VfF z%EegZ4+%J~I0C&kE5b`{A$7fb1I~7&+qpHBL#B5G990|?RL`A9XOMfx#NR!1Yerl-l zWhd<#XTi@=9+Nv3{Qmeo@2TZ`9-O`y3rd~_8xF3ZPkr614s8#kV@5wDU(Ua+Dsw!J z#gqI5XM&cJ`SDJeIC?Flo2y~>%y8&P;5+J_!vS{iS*Ttkm?Zj58~N|Y!eh=jO7{@B zp4)=)<35nk8Lw>Ezt8x5pqtp4$Dy;DG|Y?ojHm9d!1N(4)b~?o=a#;L-G^m}#z|v- zM)(v@UjIpQX0E`njCL|7HVB7ZABiSMCD<*M3*ZAuu>apz`UaoU9W&F(>eL)u;-|I*SQNw zP^t<1)QQ3Yb8B>erODpAl~A>vJ{Xc01~_~tdEXI1^?yeSY>wSRhdrNg?w*^NS}I4^ zub;rgN-tvI&W+e<@m(-fumgM-pC&uKmcehmt7v)RJ}zHiPd9Wo;T|(XW;nP{kiWp1 zSp~~;FK+Ka*QfeSKb3boee}XS=}_TMts}VZ(-k@>dI8_gOTm)Jvsf;nif7K1!0=P9 zL}Abv^Nc3(xf3N6cbN&?)_5A~eU?_;DWI=<8;O~~m`ocS4MU$4;2yt2@h~xD-eXgR zEfZ9jbP*%g@5K3zlqFohx({|$Zvwxe5$wND@_2K<7}w%4g7x}Mg;V1#SW`kBn)BS5 zExUNWTV*5e&6Q?hR2p8;bHHqq;ES~yvsGviC~+cuJ?$b0qn^UGAT`kLJWDSfkRd<1 z6R7dqv#^185KOR~if^u8L|dDi7+0>!%q%V7j@e5)a~%op$=o!Y725<`7yrUtrRS;i zvmB^Qd=C;v@37Fn5a*vi09lf4RorX-9uk*~dwxlA`}loG-x_%m^kWbB{B=aXE@SBQ zxkA_M*T!qE!LUy4DL?DHFMN!#R9^cv7T*sC`y_eJ#;y$NhP8pq#+~45slv^?X2R(! z9K{2z32@o69W7ceV^9%*+-xP34Vj5uk+X#(UY{bTUxkt2ZZl+Er=j)Ve?)uWEfhUc z;=AH{FfpotW;cAN)>}hi?;>p|k{PUAc6%3G4LuBw(~Dv9!%>{W%=cBr^R2L*&o+9m zV1gFs*L3ks58{w|302=c5vslNfO!}6p>Y-OjV@MXarN#X)^QSkKaI!uws-haSVze{ zak%X721y=f1mC;ii?)e0LS_o4ovA~IrJIBU?0l}5XPp?#)#e{RQNO*>vZ)Y*~FAmg!NlTg3T`89p5sKcqth`f{`AI8^mD0>n(D? zYB&bG_zA%qLa1THLI@4oYNz6V6ff*Qi$M#Y;}}gh@LVcI4r~*`{tjhs{0ZJ4rf?3% zy!b=LFW!aFnvR`)L6vi-ET-;9hQYDFro!48nV9F=BaGUp0o6NCp;~SZN@n*US@8*G z`JTb3um7XyJREZR{y5%LT2j$YsVG8-=sBNT6j@niuS6m%UzvqyNNH%PBrQ}bm9(Do zIZvdal86)`gwl{QLw@)7Cv>~dx#xV&`~7+;GNo2TqicIW+b;p%rfx*pV?T(}8Gi0+ zIE!66B}4WL9nn2n27fKoVRN`g_;Zgwp0Is{nWcK@_ss!qw(g_$jR){s+&Db^G#{i# zjbQTbifrXrCFba~80*)xlJ3d;+T5hbHaEQ>g@a0X{_SWmD^kY5qsQ>`Ol5Y`{1$Y4 zO~$Y9((t0`V-j1X$o=Ne{+pj(1%st6c(=0`jh$BG&W?P6ShyLEX;fqW8)u`}nrmdj z>jZGiSK_YzPAjd+^nl<0boqUeJYE;Z!BGc&@MtQ-Z8C{`CUXzLndvlkB;baf{p9*v zRqpNNejF++z=EBMbe_=Z z=o@-4K!@8CB**z~TL$|Fdoe$hXS%JMh21x{!vwkCC|g9qV|D;J%uHy#;!C_2be5n!8?ZY`c?>%JqQcdp5u_hQ8HJm%V z=MHg`$)sJLa_m4*A#KVsC+5a8IPYC0Xx;u3`h24WX%mkN7EUW9&TsFb#3nTszWg9< z|Jz0Oj#FV#!$0#~Yz@|O{SD4|-Am2*oXY7tnpkk-Daq=orCK}Y(!9mzaNCM zB{t=dLxXd{d7~t~f1wJ#mNu2Rf0AIa_J?5kLu2Y7=T1h;6rc^^_k(KQH1I++c7CqJ z=K1r{N`D*}?NNpabr*?^!z0r9!GV4c8O4-jM}y7QRwBWlcP-P6S-f;F?f-2HvUhHR z&k3GQ6n>Xzjy!}HPaMYNN&~FeZ(?(})P~4*sd1&gCS|8RZ}W5F8gg;@IXILq1=l(X znNrJe(LfJ#?DbV|?Uj$giP^Fn`8FG+x`p9#$MIyYl$AFehdWdaoSD^c}jG>MK3WNeP3s zpHIVH|3(tXXV2a3pX0TQ`b?NS}A zp50D%PCdb%3ys)n`z}$(&8%A@c;Ap2%3Qoy1P*5UDu^;GWUZSaj_GY@GQFf9NWe8JhN>+I)8y zZ_m${PekG#&KjR?uw%{>`-muPFa6hUYY=CW-$AX-P5-C>Qz-C=|$?ST)V9o&Ii`~kc#orrb zr>T{V-{;|Iy`cpiUU-P&pPVHImx}QB(ebosjuO-F7Euqet%3)9KDR+y8;Hab?AkYx z{d;^~#>HSqYp+9?_Y9w&18>ATk)Dz^WOq4ds`qgm?LeSI8((s$jz5~ zrB%m8Tw7!4 zFAt5IThVKv-rBA+9R0T@vknCtYuV{X*a}TE!I>~Ec5PNEdMuD;0c+$0JCc1-W=1=e z^}H!~`F?;NSlTR#b{mE|DqScz)re`^XF)``JXdIMz@#^J(7x-{SePD3$Mrh%vzn8F zvk$tg>Mr&S04-V5~ZjFVa@AnvXj4>z^-A^Z* zFGq9I_P5EIw|u5J(8W4e&qGw_yH*HHrc9(BW?ixO4E=7ff!jOC^ZVm`P`r6PT3hCq z6q671=tmnEsFh|jo0Yhr|3dKIFBSH{NP>M_y2)nCKQ*X`ID&iotF5=q$i-8W-{GRe z&qS*(XyVLAvuz%^?q^fJNwV-y(lkdqL+I4vC)#w`g}o~AVB;ii(akcSP;LPs!BtDk zLb_(-)@OULa$Tp$XhtHwc{SPk(T~-ljJPhYZ}cE6H6XnkwYp6$b&Oh>V|J*}KV)nQms?u5>&ydz~`n8+?U z7N-=CX9K5X;ez&at1T>uP07$>eF=q}K0lWj>h#9OUSl-TzC-yZ1qEx31ZUI(a9pe_ z6y@F&9KKzIi{9v1gUZ8!cBY0vYi+~ntuBu&kFq$}n0Q|4Kx2S|dyZWSFSJVpK#yk>T{s!-io z5t~n4!Byo956mcsPbJU)&nt)VOGhwO-4N?_zx9ckoD>seEM{MvZd)C?7E~gCdZ|g;|^*c~yW;FX#9ArH;GuistxFWJ}>IV$J;$P;o$wU;izL%|< z89>`*qeXwV^W4#)5oC-7A$Q+NGG&)GJga3OVv^(F9)$3$jYrHZWJTE`*Le6g&_u3w zk0Z9;7pZFMn?Ob1NN-7hAx?fJz98 zRu(wR?&LXywdD3|MP?lIoBC}X#6AK<<=KWz< zemn_P)jiQlCK7^F5tfJZod&1d=+w$P6bw({tcv*%o3wyV-#Q?$;q%cu$0>4aeO$rY zHyg#q%_9nRRrJ=vKd`$o17}+%(W|HV`-s59s9<*$KfOtY6ACv7H|haCPd8%A&hZZ2 zm&%N)3UP)-6&|>8%u4Y2G^W2Q=J$d89OcYTbUr$UScuPry&AD-{`R79Nl!bD@97|m zcJDwh^9Sg&DhDMT8nHJh1MBvOkOz*{@G-`(=TfWbw0P)ivKCSE1a*3_dzvNj8!^Nh?~V20Ll@t{0n z9gK0?h07*Q0GkK}T(Kk@>m*uf_5HQP?|3{!F1$vQruGVF244n`BfIGO5vsIWex=~1 z$f4Bu%VXYqDbD43_!H+m7lFr#>sbB8jr_?R$$eZKPYOp~pt;)<@qVTaw5X5e3LA$( zdXN@NRp}@H;%Z>&iZSf)sQd6dJq&#YHxTCtZ(8RdhE+eS(erUD?Nk0lzG&IfMBQNg zt2`dv*781?BYbq~MN}Sr+BlOp%4E$}W%-_fK3k;XpVQiWnrhd`o3Ks4} zu5z>>GkFA*MxMd$m1B{gZ`1FIHJDH`25d#TcsFGvW4IN+%1$7r+8L-H^_GrmK8E^x z)WKlLPk30`5jV{Gg6x+eCs8Mh26HN)!LLx*#dSjx-W6=htHagK=`d3-4qfl@_w(Bx z^1hBw&~mbs_NqjJ!ObU-kP{A4-gRW3!f5@!)@VWn0 z5QcRqyT^df6=lG?hD;iNAqU2WCF0Lf6Cvh+EZsZ90%WcAVTEuQ?)9ss>3Qv>qCx}1 zv?rl=T{a4?Y{8~o_wn?mIOd{x~6ix!?ho!U5hD`yUxW-deV z@~eUcJcskipVw^{o+?@!=eLAHPF3UG>MXae4H=pAR6jF_qYyjTc5)%%qPtD}uL7t*}{f z1c;q`Ndz|z;aKN>yno*tTMtQcZOunX&#g@8$XH2)0s-i&PXd!MpWx0L6=u!5qpky7MOA|d#BM>$%IR)Adx&cG(|3|Rj-9ep0} zBJy!DU~}pRI?oaa_GSxl?X_EkT$KcmPoR;+^3Xc+;v!t zHLQO|`Y!*V_wPN0s?BR~1J7l8vTB;(r$z)W=*q(xKkj2?<0n)fV~1x3Yhc;wWvH+# zNbp-Ggcg-I;!Hn9c7o?0$-mL#nr}TP%NIw(ZSNTL$vjD&r)7}f%=<#M#qWe=QLbd{ zu+^mel|I%l1xQ`B2wU65>C!K)KbTn)#kAK3H=+z z@%5i z*lDhU!ZTrTC`6OpOnXR&zb(Oi^Cj?%nKVqztH9}p;z*C`LHbJaIN#e<{Cd-XgeNF~G8*-((=h1H92c1&R z-*sF1Qh|&y=d(^hkXCk<=17gAdi>tZc|s-Mr@oEKfqp0~AHz+Lr$qkB2mE)_4cGl# zL@XR6(8Dqnr|OiDo9fzN>|=q=#+M-L$4lC>O^j^Zs6jeD9D(lDn^C1h7yDKw(G0r` z%q`D`gx{0O9uG-V2aR#)c3@c9(3Ud%=-h)fZXAoJQX?M6_rLTgkJJC$VSwL86#>hU8IJjCG{_FS;UcFP(aM zl}K+15v4_Gz@a>M(YDpw$~5*h5&8Ez%;eNUHqCpqXzAw+9GJffxoxKS*g6;W#id}` zGetV?uP&KaugsDyZsW|3XY3XKy$#CWW7BkVDo#(=>G|%d5lL-Z}ZW zcWQr`{BtGtaBCg$pP0pWPL=6(ryfD(%q?(k-3%PPSC>WVq=|Cc($U1_plGw_68c9w z1Y2Kkz!~4};D6#KxGDB9esrz`_#Y|`9maPOmFWChg(Y+r5uH>G_TiMdXy*IPj1DP-a$zfSFAlAI?~*Q zO+wwb;np!Heo?V0QD_{`v$o#YU|o{z$=CSq zh%OuBAh6Mv%0YTOiZ*R(X7g6 zSQT7HW}8=V6aHnf^xW?}_u(5G6lvjI#n)7Hu#=Wvm=8nKZwX9Z&tng~&xvgHHOn4< zjHH5GE4*)zfUnH2SQKR;d}P#KlhXw z&uKtf^#FajO0MAG6{eB&4Y_4AIq?xYaDC${$Ub}#BTo60^=?~llhv71CfVFf{j)FF zET8R2?c2gcu|4m(oi*b{axeKl*&~Hh)aguU`koCZ0%qcWPi-!o^EnPwbfPDtAncXwvKYyo{f*`jfnl$qh~tN3I_=W z8{BQaNeZp~e|iX}xO^4WUJ$d9T&ywjIdT5)M73BVj-O8MU5`Hjgqf5xC`54*y}&L$c_9kys$`{ zwY!Fp*@hE^*Jdj-=MT3zlXru}b^HJ$mbsW06NH-%^X{EV#i&|8jIGsB#h16M@P$Z` zr4L-TcAGUGt7mCj*MBp^vdZ}+UCoc)R_?|Gy)`iDMJ?9MD!{24@}j;!C&5TXgw4)U$<5z^nW9vkv8~TXWJ8oNF^9r_Jm6$++DHQGZU?f6^FcOQmD zmxBd+i6j%}-Sgr*r(`74a zk6>1k9G_F_LWM^OEMtm>b*%0$cF=1F-Clg1EIU|T=4#-_?bFH?mC{IjS^ow5>sQQP z74^V+-l-*}2j0fwr=>CwRJ}{Ith|D{j$O`Y(r+=lGTE|n*$L=*tDS0`ke~|{rZM}O z+M*=`hE|;;%d8vRpYlA9tuQccOMviG@Tr8wZ}Z2q6Ya}3x;n5v_cE%ps+1;=KZ)k90hoOA4&$zeK)KFxmLB10 zEq{3wYb!NmBlW|X2yBkVAJ-R^(^F!%c$8)eGC7JGS4#yMP_GFK&A?oDV zanCI-vc?0W;KxfVG(X`;Yff9hdbJXKn=FJ?Q)WZ ziB?ogvA*uTB-D+Afq{qQwN(k#+p!*|r=`#&nJz4tt+skcWi~$Z3c9p+@cb?% zb}n=j`!C@Rriq^;&LhIXN9&~3Oy_KD?(Qa1@9JQhRTca<))bE)3xp~;NBoqc4(})I zW=WqFxb#OY_^r(fR-0_a2G&Q6EfY~&elq)XP!^siH`0dddfdpR8wA!;nfO74cNZKj z<{K-I__L=H?Daqnd;Sos_S8kCrCN;5X+8By&>1NQ5Pi2L#_1{#PO*?=l>8bT&c2kR>hsRz3AE&Wc=5UkQil(7NQy^42m2gegmGCOn z36(EO!cy%-f$}X=vNb6RFN!I0LE7>BU7;*zvvWK5vu%)kIv+~!RN!H2Dm~!U=ddp1$DVthqrkVrhR}{E~Mm#e`bvN0h^aAu| z-GaW&e~DSc=~`03sCc?2V7pDR)8XN6!qe+TT<-j&veLX(}29u|1kQ~ zDvVjK3uAUAqWPL+^7N=RO}%-Kwg*q+&g6*ETW>xHmzs-lp94hbZ5N97ULL3G+nn+A z;Am!k-W%FmG6hm94uYtmQtpC=8ETs8b0ZgNU`U=7IO&FwXT?fH-pUHrzVM~7e3xlu zQW)natp<|oAHny*8w-hhMwj4U z!U(o~cOecceJD4 zn_oi*&(Qeb*NEvVKLxtVH;BYFdwe&TWc4KV67+4V#HoWtG$5@FlSfUV!+O@@lwFmW zZ=}q5zR|{DDRJg$G)}ly@dZh1%7rg$U(rMCJolhkkI}(soIPhUw%01q2^t)?+29{7 z5ESyRnKfV*(}gOkA)v81l)kb5hZCoM#q_Tk^tJ6Wn6OTkI(es||2W=rJLU@d?!8~? zK0g8Hjm=>rmr1kv2V6jCyA#{@u4J{Y9rTd07VP1<+3A71iEap^4iZU%9l3+>eQg2W zeVAQJ`ybJ+6ZdeeeKwz6xx$|*L&+p_3AXH06c&AL$6xW+v9-RD>rvIm(tx|*H(Q1M zp0)zp3Au*y>GQJ;tPpZ#%# zsUK~BYL15%59fmQu3&~Z!-T|%@Of4jWd0S0GalvmMpTRVeXAhitqe0saKQZ3^{ALw zMsLLYgNn&9xVYg0HTf1nea+kHwDn7A?-^4xc_~4sKkLCwVM92rCJFl|xLUmu9zzqA zC&cO6R$3i-6`$QRU|V;$q1RRkc;y!WM|?F2g}GGM_b8`r?}>>=o}*ZTtd-5MQmZYa z+Tnp=J~y=QJ_%o{fOa~~lEZbx(A^pP^uK6HWOL^(Kp?PFTCS(%wf zXydx~pAeE2Si+dqU=S9@6mAS6*y@l;S@CpZZaAv`T*bI&uH?``M<%eB#`Q+Z?9%Ap z@ZTRfJbFxrwdppa*@@ZsfQs0zOApcX?p-p`CmhcVSVBLa=Ti*bOs6+XbEBiTLEhA> zSp8EIJzCCCVP!wuTj2z42YX5tAKC%L)dXzgKckpi=UI*&@EJgZ4YCZa?t>% zH1d1#;pgz)!7RL@`Ui`SRbj-Wm%>9U^M$1f!9=v|Gzzya#f2w&F}`s)E>USilVJuh zt}zGxd$yA*cb=xxcrH=($00H}=@BfcnuSsw&p@nf4?Oa}g(FQ|seH#ebc%As;YV)Z zG>bsc50hfsqvwN;raXNr8wVOocz2o>?=N(V10#1Yls8Jm1=E$$Wu*`=t-TDfRy|~b zTLj6CZzbv0FKM;3DPH-Z%N;Vkj-M3U(1+>ctj=Tj=tmE>?2kh`*Qs#QG8Doqc$P1p z!Jc-Z7>ueX!H2zCG-k0JoZ#c8$Gywp@vV_)ZtFsX`*|n4Nia>SktM}rQejBwW>vaN zMD2Mt$(5b=aBk^C{IFygn_JE^ti!eNo{Sb){*K2@wN{|Qa(OqX6lmJ*7uxycQc+Pa zI`@mArG*__(R+kVCL@`n;F!=_Ne5Luc&^3ig>ZdU1kbm7j0%LGX&q}2%s(W4SPSpwC+Z_C4Ct>ZZ8n{ z4Fur1Cnns9cPhkWQ@T8u`P2JR#aTY&r|Y|4z#0mW*fN;L(;Z;_%i;0@V?wSn7&7kSuOkm z*5`H$!ZJHaiDCxv$XrD>a7x@>^%Ua2^B{4x`Tc>=74?FYlIYdC$i zEPG=ri^Z?zV!dfOnDu?8NgX;YOIZo@R5aOj!9EDya14u2%qHCfKD02#1ReQ3;AidO zj4YdjyARFb*-|pF&HXq0%9UYbpAl%Q83X^?f`osb6l31SZN*Qzo{%YV%53rKl;Fjc3Nnqxje%=w0SO7q1YY=9(O=G3r5|bHjwai(^UY*HkjRrvZyA z%+Vxe7=BnPgPZ>w50cUHT*j^{Y#$}0Yd&a0`7KX)6On=kj}Mmoj+SA@TqoXtx(N&< zv`{``G@*jmaPeOPM&0cu(=%>jf_oCNU3mp&_3wplvzI(SP>jE}Q4(BFm0-a$kS^3J zMXj{6f(0-BVdwsp7${xEXD8DkVr{;aM2<09%OH6kaT+H%)>7p;CFryC0KD3&%yJx; zV6yce9HXVjXeP4vfpU%X+UwUxF_ega9WCxokUl#s- zC(9PyFoA;E`FMLU!|JE*2{4yU#-$frVa0KEke#~~rDSfPx>rBhb6iLstz1slRNa7U zwpH*Wtxw>Z91iu_^HATK60t;Cw(gY*CyLf%KhC~DiF!3Wmzs~q>&Ky)_GDPHAd;x~ z{3DICDtPwMU}>@PRDqFDm8l>4hr@KkU{+KWamZsB9k>S-@?~J`p%8R|RE!-j$=wOm zXP@qwf@1k1P+RpG&bcn39{;^AZM7SquX|SE(Sb{}taB#$x?(%bHrfe0Yl8`?Z9vz( z>P+r-5UPHjMUTI`40CGZa1XmqwuyPuLFvTOt@sXSebxl&laBDJP!*4qtFZL7g*ZyP z1ww9BVD?o7HrcZREJtp}x>LMk^6NFyQPN9%t#}XNl{&a+uoG>2zTl~Z&q8l0UD&!# zNE|g2fm?5Db^5*wEZB0E(1f=oajNAwO==oWTKo&ON{?e5_XAeM0h-MT51w9J{EW#Iv8Q@vD|H8xhlpOZm*} zw6{q(|JO2{_e_bt4GzEsmYSGlc@5W(=if_JO3g;+W2Z$g+MUb7sJhW?^R~}| zU&n>`Qa*;Z1T>;)n>N#wsKvDB6u(Ox#O7g2oXvDw>=1d;_$QLAYH}E&VixUsHiFGc z^d;?HilA3)2NJUaY3r15+^{{6{yt2BuJI?;%^Kv7sxNV_Jc22yTI_j@1|A05Qm%n1>&n9{r?Qf}&b+ z7E)^>&?~5f-3dmhaz72KZyYA>m!@zD@wZTOX#o5HU0$$Z2#as9q)qxfRO9 zs&o;z|5pea*vax|+EV^k!FQ15OK{D<0fCeC1^jlP8n>LShJ*iNFgLQA#Nub_&`xlP zdljnwQ)as`Lzwf-kky4NvxJ_DR70y3D~vT+)YJ&rvwACJeR9J2x|(o&d6eLgb0fe0 zL{Nn@LZBpR=*87yyGm&F$q^~cuY(e zOJmlLIux_LL)H4b@bB^ucuH|N3htc3Yz&5!(4(OHG7`s(lSIcKsTlb%4mD1V-+dv@G?<=0tpX;G){wQ^eQ3i& zJ$M@%NH=ylk(M>vF#MdYFzKZ(+Viv1B!xw^CHOSj44r`M2FC0ZGvz)6{-V!$r{$Gh z+U!AQogj35A8szFrxUFA2*dK83F210!TI5%VCxEPBKzC|3Vfo8nz9;~>7@X7FSJ5O zNtodLY-54Sih9ueavKL$j(~FKQLy6PJ@~GrjWY~W(W1%#);=i*Q z=#CN|`@!=wy+?zu=s!sFJ4RJhpW*_$JSZ$Zim!euGm?gkOajWXTkeaCGPKY-s6#81F;UnxaG?$37ek|a<`1RQyonh z(AXm^IF*kFPn;q5#y`N#KX`VU!*FWz-TH74XOG&89cO=J$V;o1w++wmXbZgB01i?mq0u(jX0E?O|7_D;X8#LwcR= zVqm#B7Q3m^oNJ!ME2kW15%_A&T738CHgPu_!@6CA zQEcR2dZ9y;iRY%2gy!&hU9au1(_%M0dcL04@2Q1j@d?zjN&+ug`O||^R|Ojf$8y>c zqtF#>Ft2zX`s}DDQ3Zkcs^<~i_j3mIRgNV7E(c*vKHpiMj>6`yFgnP0zL$j@Mm7F_ ztm^pz{%V-OloE54O&Z1hDm20G7k6XDpGp*#iPNnwwfViJ7!P)YaRk-xjSky?}2ybr~LG9pI;?$LidmFXbLX%PK<13!yIOt21 z1eG*wQ!M5;2(diE7{iAL3$>3mpt#<1Y^a|AW^^L@S9IZa;s!(WgF#~7d;VTL9__qT zndQM*@V)FF_<2^MYn~Vvd_$ho84!Z~q)C{ZDakr5HlXuvNpLi;x}>^M{W!hd7hs>a}s zHxk^2o&HdJzLhq1EW$634N*CH8PL#!gj0j`18D<+7WIhg;3`$Hv|pUCo$!T7U!{>cLIOhimh)bm^5mD(Vr*i(KRTz z_emQTL^r_26VbFQop+N49*34!RUVR9ptE8XC^EISZ{? zT1GIJefi{-zYj$DTn3#Px5zz_5*Rt0L7#8~NZ%MlWwm>(Bqoj(v>gyb8}}5!wiXVY z(`Qp9`F(-_mw8b3+J*Z2&%)cAp5ls}^Jw)v78*~6QG<#b=+lx{YHh2CaidPa{R7G< z^W_6st&s)#3RS|k_lS0A3oSP$imS?XH z+{1BO9)Nr6HJCX~A95QlsN9bPoIoc*t5_H;-~WFvd!Mz_}Ns*}-Zak^a&de`B+j~+xi&R8DNQ?vq zxYOtKDi+@Hfgsy75L&7V*Qi-y%I`LuwK$x>A!&FYDaAG3=d%(XMmXX{8{VtW1|NGf zf$;Kdy!T-mjVxRP(oM?56vSEV2N$}^P5|pS6bSR%A7J3v4ODW~Xl}LTYtk^DcTH6q za<6z!yT0*c9QdAzRP>rA_ob09G5w^+R-A1sl;h^cNOCs{LrXh~>d73#JPB>tQfVG6JeyMsuaw*&w@b66uo$kiCLnez{!m zN+k=OcG|+G72k07%Qci7Rc4yY%&LwwVpj zZ(0Xm_scN-7+cJW-w3A`KM;&j5s|i(LxPG^ZFp^LDrr~|2Qi<{;AO92TzkAfsWdae z8Is%SjO=8T`lti4|JK0TuXp&~*CJ3kmje0?gYf9mGkAaA3Tu~~g~5iq_||KHJTMZX zkz_h5q=yra)<1&TERDwU4zYbxZSl!pRb0}YkIQquk-nYo)W@@e+|<@)Z^W|6j;jly zFXkF8{ZR4}!BBLQ>jlv%!v0&Y>=jY09Z zsrM-gNB7>xo$LS7=M{qx;$hAfx!K+3;+l z#{0q*`ft!z2M0i6d=dsO)@JIKr^v0aSmFE5-55RB2MhK}fa|2ExTxkjIM4J1vMB*R zvo)lD)?92*DG)S!OTd-$a@+~?VEEUYgKBznaP67^x-3nF$;il(g5ImxeX|ZzxKxxA z#e;8IBt-T$5wU4A1hP$IVD%)P%RBDL?CVk5+`WstcyGfs&{kgnj|*$?$;)b5K*hPK zlC9vht{R#j+yL`k{-{$HjkPPc;?*&)3JH>Frjq^UCMF^K*_Ck1CVb%K+JB4<){2d~)ez~vbZ@N{1@E_hu; zlD3=@Hor3flXWxTddXh=si=Xi`z}LMb{@u(uLAS^BdE0gWGq`ULa1V)jbfjo=<t(s5cgtz$qB2}rc^)!lijaTNFmpvIJpGbER2|J5X=sa`5)zzcY`f3X)`AQv0%b@F!6VevW%e60d~e%pKK& zlm{R2*XLotB^q3E_0U|ch5kF>gtr`DfY)6c7`fywE#&XmN42}++m9P@{hAAqZMh!T zz77Iuz3XI><`^!*b%2Wby&@LUzhH*x2ULyY|8^&Ptn*8N{^=XwsL>@XNO6SC)pEqU zKNE%G0r+W{2E?SEg}B8|_<64vHb1$D=?9Zx*hghp@@yW4HvYiORkt8|gbxiqr;CaF z{Vg~98q`nHU=q8sQD?0^J{}OqolnKNUj7cFt9LAz{dkRO6;WXD{3dPrAjJ)rJV;kU z14f)(Ks%S~vDmW{$t9lOP_eL%8c}_)IW0yH8}Tf_ZY|tWX$XGLo@392t>}<(A5!Ip zb0aRB;=@x~pb>o&x80tC*S6+Sm1(@+^6y=A{@X1)!^M-8w=}rtGQp@hG#7kUdeWLe z2VuL-cc{$|B+B<2gbMSQ@)?-d0#E+`^d)T>>h}jgLCjL{kQq*r7K_srMdKj)_;s8) z(+aPx;rDqj6tMVs59rRU#e#F?aN3B`u~SMQZk;CA7stQ9E1QY)kUpHAb{!%vL_u5h zXfh$H1xMt}gJIb*@Hshy=lw{qo!fXu~dIk ztlEVW#L95u_48PLLleyY`-B}+`>E6RWzc$f7nDzn!(3}eFiTE{ya^fPSAu}=@;{{J z?Kb4#@(^GTDBAvrqE*{X1>d>^+yM8wUjQ4`dO_P7e&%ef%`_Sk$&w@mJX)!Um;V^ToP4P3N<-N7b<_x(HKs zhH=Fs_CQJ$p8+1u^H&EZVt2L=Sz?z%%vHjWGyZ~xZ=)a(dDE5jFIv`r6Rp)w6Lm|T zcRh3z794h?MJMl~=jUb`yFC-0J^o4?8ls`fK%R@ZoCBfK1-RkyRIm^$hPAzn1U$Tp z%X0ZS*aS0N>tsMw>h!_xoHo1ns07*@BFNzH9P+?kh~IhF=q87s)GntU=1+@(hi>^O zR%B*1OJGb@ax-{G`6x_!_nA(p9?4C2Ux9;Ii6vX_3EgvgA=)Xy%Hy&I-0hx*FE*UR zj!Hvfx#=NJxm->Ev{gYf&x`WZIZA7t#xOC7!-8;&iO|rgK{a=b;hs2lK*9(qE=nqd zC{G-Nu9K26^sF4G`&OR%Z`q1sQb(+QCErK=qX9ytr9Xi+D&gHJwNyc|hOA9(r8*%~ zVQRe+T)QQXkDoPRrq*K;Jp2-xQE^@qAPaO-` zyhnv!Q}#n8y$@2ddi+`wM=iga5V6B!geq}Q!TE{C|0p`|uo~Y#j<>Zp6={kVN@nN2 zpOaPD*`p9&GcuD9qNE`-X_uCE+MRQM?&nlU(IO-%5{2walKh_Ezg=BjU3JcL?)&q3 zzh5sHSELAMmrbIkn`YoE$A7%9l>=1$)`ynK$M6kr7m}4=2nN2X zyi+(EAI>YM^%JAWzP(?7Wcu(8-g7yvqh>J7&!O`NOsMKBeY*+22I&ShRrc?UOMIs( z!EpGcuAKreA3u4T2XhCju}Gtl%l#XJ*UU@Og!QmoI)?LC-)IaZdxI~EO8h%e_67sGDD4BPUcW`SaO5F;`C$#Y z;yzqYKLZlx{DFc>O?*329J%qd?It*sq04(*&@q>VGowDN5ha#YirFn+G*1z58l3y18! z!>ff?AxpIca*AI=tF#EF?+T$8oGQr|@hIfV`C}E|-3N(NcRA)q9kx!1hrQml@L*dp zA%@a)#+n#-*sF;Pwta!t(-p)peVCb1ra((%ZRk7ST-#??GRekz1;%lqC+|*RITJo| z0dk`JaS?lm-)a95>tx;{O<@z5n`z|&wb*3j>oEX7V}@XM%M#kUet_^xgTcr78ny0z zjI;|UqgQWRS(6v9@e!|L8v6GzY*ADc?fbEay>)du9W$p7>%|ZTV2B@Yy1Xrg#-lPALxh$v&UoPnn=wVs^H=`%5>HlB+X%FBCgG9F zdPwRY=Bb_ui?L>~MnX zirHr)Db%MDgU;|d_S7aFrvIQUv-PBqyXTsSEO#7Yb0*tUgCkYkGozT*oSX)8UkPAg zZzB7yahzTLrVHHNl7)NvA^00l7X6+u8@+#-fahLB=-fO_UQl;CXgS8xkSINRw91Fv z4*Z3E%ogKcTPMH?y=^q<;1qZ$&BIM;y5!iIFnGB~k$Sl8Ak9KCTgx}kscAt!T&bMQ z_UkI+s~fM-t;P@8%(*oGOsEahp ziSsmFKZ#c|^7PjgHKzE#Kaf0kuZR?h!LyIo$kyfEG~W9@^c1h>cYHocBcepuoAH8S z2Y>MNzrngk9^f2jOQ>D@P20;;RA|nT$)J@oi(e`FLtH~HK!4LKR&T(D^LJfla;9bT z%l4}=JBL+7vyCL-d}2R+DLeu5;4hTin#Zomcm*qz+VS&d?ylMv3K?&1f%Zi;o;ueD zt?!@5T6H?0t4luMf|ai1O2uX3{UwQPeHzW5GcbW3)jNR0TL@3{^$TLKDV2VlHU)RT zBG8K2~*renayv|A;-mC+LwA|g2b-uL=tYdDI!^$!gwQI6H*Y-ekN&q_c zWgX9h<2dbX%%taTC9)eE4501VHkwlNj2~)UDO`G_lpZ|OM7EaZg7}zU`0O)FSnO3K zJn&&6{c&=D^vyX68~dWz0fYOcW*?4ZR@kJlEkPgg&u_+} z*`G>)9sWBHK20v%3k5+PtTu1C^{<|1XI6Pnt}w z*pB+1q~TXuZ{}T|R1LGBVqs6EHLjKPeT%PkI&B=dEdGW9D zm0!~sdv3mbIcq6+omUj~D`kTWRYN9vG9snsEy80Km(j@Ec63GHfPE`g@Gps5(Eon? z$4=c+&VS{uNV^x^;@4eU!lrIG2#3qw659X^xVtZleY`t~_D7r09c4G5%Yox^-g00| z`s5i6-MMyo54>S^Z8*DW><4I_B+as0C0Ucndi3DIKB#lNM3wf)Ql%bUdU~lUU27B% z3L+D=AHdmX?M%wHO7x#m4S9He1zSBu9kwhp<6lef zL)C*UbWaL_|IkFXHkE)-TZgrf`wTl_6BOHZLf55qDBLrijgCr#+hN;4A@(mk;h6Qw zhP&r6abI{gdYd8hy#c6lj7W)JU9{Ej0Cp54(vwq_;VYJ)f1|_c$l^{I9JL(k>Z>IQtuPUe-fV#-k;{Mt2oBn zf%5Tmcaw;8hA#uPmmh@(WVNX8_i{A8F_?X#BMGO=(#Wm0sWfJsJoFvX6s#Tyfi;Wo z!{@q42=|EqdlLqRE*zy9Y!^g)m8M%dmY@WN;g?K4Lcf^~V|V|>@ZpjUB&itK=_uBK zg1ju2`P2o1=@&tgRTZ5os)c!K0^+cAI>@cQL8O_5Y)pwB?YpCha=+%oHQ8RMZ&jv$ zAMd1xRKLLzgLIO+_BNOm^~0;AX7;(F2l4$9&X_$CW2d!UqP6GWfV19xkY2@Q&lYRb z^+IK~C19TLjl37c3h#mQZ+Gx;TumoFn9YtqTzsyqXgbgwvgqUbJ#>DRn+)cjka8^XGfPVV$Vr*v6cIV zz&ff8I<5afIoIi%@mdp3u;$S2)JO$J@8OJ_B>foBi4^LTncCqsJjH@}p!#>19h#X= zgc$|!p>ZZ(CwwF2m)ro$lbmD1@haMGUI}Z~MZ&?kXZaD=WBCb+^{jK54wafaM>M@P z1!kSt%oYY|^Ru-+@@<#GigcrA*wD>EcENK64hirgq73;2iO_nyBFoDt-}Fk zep@Kb6Tbs@5+t#meJua?>yol0$7o1Tm9twA;S68jya$OHilX4xA)G4Dmo2|vN=MII zfk(v&eB=d%DUwO_z|j)A`h+)ia;28(z?|}@N3u)Z(blv$F>{8f1Ya8V4^cTetU=B((2&(S%2kz-&@F~Mu`e;1ma23 zTn^Ed${XgJ89Re=*6zJCK zF*Z##w^w}t%fehz z!!bafi_6j!B|f9#>cEtC%;vmrd7!aJg3U4c3;m&wsN+9pa(9{))tsY%`|Bnn9ljV% z>CUHKp9(l$g&~eHn@?vCOrq1rTm?s+E4*KYVpQ$$Q_y*S3GHaE;E!2qNuw%$!YBP0 zdTdiJy}WWK|6_A1U+Rz+J?qOkQ*KVTyR=K0jT&fR-{dUDj(wc}_e(F=6)R-7o%jMJ z(hq=EiZ1k8S{S*_*rH>C^XP7QtZVM2<7;2_eEt zZayiJoJ3bkpWzGds8B8Qr?7Cb4?l6!UUEZKjlR4opw1WhaM8`eE-z1HyXc!L96uGu zf4!IEU!F0xOKS9ClL}R7PlFbHd?*My3b=bMBTWaLC8?xq9(3&;0pIqocv=d_ddN4A75p9j;k~KkC!0Tvwz^BgdAS0BjV?!HFK=FRY>neK3I1y<=9B7kb7Je z)xN3%%jRYxqm~SM9b@QIfdw=@{lr|loDDUj<6)g$-@FvP=U^L$xtx6p*q^7+I_Dm1 zx0m_+4gY2YOuxoBUr78qEKhwd8)Qm!h|*-bN$?-b5U z`1B(gwAp~q?+JqpE-z2tPzhqYXE+sM?XE$VFv2 z|Jq$es{R(kje>6=CD{Wzt;K1MX)Byr{RHF=s^OKnL3|nC*DyQ~k2maCi2r(DM7b}o z@FvaAfRIBeVBUNN_RZ)4o4K)U{NjCR)o47(bF9%FMb{x>mp>Rrf96C(?+9j%ZcgbUqF>qHrL%hMPdE~n11-YSv6l3}|7 zEajYuTdi>Ls(x%#EKU4P3#P!c`5x*iXkjDzY+ zZ&0;(7e4fN1~!x5G;ee6@7_%?EOmzojxT_FGOu9i zVkP>%A{xHCRD!kSL!`=|O~XRW@Ce7}j5}jXtoo$ri}f*dONKaikMCpGo}30UDt1i0 zw=<-eTw=FwSkHv+O_+CWV+LPL^c0N*o#Ze0p-&@@Ze`=)1o;<~LW^d0FupA|qAQ=q z;nFive9MQc*t<5Jq@`YyMT+9cwCOBzG@8%9#1DicYbMf|v|t?fNEhtWCi1Nx#ZrS~ z^7PGrEIG0E0N3Flx_q$i_UzS9$PaC-9u@b3qFv)2PKHp)a4AV{NH#y>#!@o7MB z?a4HaA+S3dNP9Hb(shA4G`=*RW(>p=s{@jv#hWGReiL^(?OQ8NQkSMRwL+S{Gm~$` zxs~^M9;08qR-^I$bE(3M&ma@~4Yrmf(tRlwIF<9jtg}4=ooYG4A!lcDUH&LkC>qkO zo%ZyCc|8bUUZRtJyMc|~QF#9E8ay5+Au=LbwD6@CRSo(Jzm}zukcw&iDN`F@&&m^A zwxJg3Z|i}MEGPQ=>OWzze>7OhE7PcG8LH8y%fuYJ1`E~?!>tGqn_D)C+=d@?hlRWmKJX z0Y#_i@pmQOhvJbz$lvb-t9g>_*%XfFSEPgX%!-2CBX4XcUsyxl#CF5>V~X_Wob`0K zHe>f{3^1ZL0e>VfmmHL8gcAb@jD7U@o4LQ$vV?yyPdOIOUovI9RvFQ?NEZ9e>>+>N zr=#qxyQpu(UcOoPM(U}%l1|S)#Wz>cpo=w|(e>9;VTzhD$2=W^jZ3+&?fncj-1mpq ztQHj7`oq36x$uws%vV&u2QPCj!?Ma^?tk%$D5k%L^;uSSztkVX*`jc=P~!rn;CF!e#bvq%PLJt;05xv1DjO zj-~|&`8~6z(*a2bsL_887v8L-W1b{4VLwK=JLq=yQ|kp7r(*(I6Wpodf=uepQ=@0r zs`3~8mw}|Xd02CPH8|Yp;GCSJXlqag=29FqDm{#rSt)_wS{-;;SksNpQgp@?e|-G% z7{&=-A?N3#>1-Sv83r`S$|t%=t!2*I;a7%H5B`_tEBV3Ci2w1oMVh!>aX5*n@Qh?_Z&6N48g@ z?aPALm6`Xb-)I>tW3ZW4oy|tc7tTPj>Sg9#br9?wjRfB5Zji{^L(K-8Ai8P@i55)2 zVy;)`)!z??h-K3B)<7}@e$8NQ)5h~H&SC3x zHu6JWhJ#hR0(RKenpPePHAieJLhjCj`bZ`G{ZSzt@Hq?a&6DYz zg88UsKS5C!YT@``GIiTg1g)=q_^H7b=+B3bAa0e!U!k`dd8c%PM`{^$;aJZqhfDYx z{S=P*4#CXE3y|E_4W8@7=`(#dnm1U3=c&8Ghp<9c$>0e{c)VmCE=$t9x(HZ%U;_#` zp+Ij<{08AiqhYZ60+}xLgPu7zmugJy7OItNLj>oXTvF`~&1FR>=VBuHw5b4!&sRg| z+DBy93Rx;Qv>3fJ55NL-JFxT6vhxTEp_%sP%wUfMIuUpjl9?-_$%!6tIw_Q0t@95O zpB%!?4#jku+)os$Z-#R^lHqoIJehd!1WgRwgw}3(2xrXy6MDNp0?XD!wCUG!`uUO@ z%pR1b=KG?U+Uvi-{jv#v-TpfeY`1_vPA`vd7%W9SHtOLi!ZCExqZWwMj08__6OsMe zU2rF*lgxAo#L4RosAZx9B>_sJC=+XR{KY$HdVh_bKP4BnOm!Bya=Uu#E>OiREn^Rnk>)7P{rQd*zJCGH*9VEL zOB34kH5$lx1L`foQn?*Duwq^st!7JkH>#@X2cZfiI=mr;pBA(4v*tsRPK$8HkHd_G z*=>4EF$fZt2f(TezQRt)PY~ZQ0=PR{lwjV#&K|H5LE0Qx8$I35C*>YYref?pXC?OI z{YKVdMlSp~+gE-vrv}DcPy~7P0J#37f%kT575oiKLwl1OV0ft?d&e}DEMr4qbk}0u zzwweNPdf{XZDqN0bqMe7$?#F@^`%O{#s_=48r*kUHIcZOu>IPBvI4kIU@G+5MR}Tf|mlz zYBpaM)jldC19`2ik#jP2o@s!$RwS_M1rLb*Uv;|Q^AwDk^c=RoFcw}eh@_nbKIHfo z0v@3gL?1lMz@lj)uV;P;)3IHRmWKWV-@h?zxZx4lx-En8KD3=44t>oreP%-Od>Lw+ zbcvsC;z%7{U!+@eF2K2!Z=klvm|8qBVAf8vf%#K+QCFe3o!^mS+r4es)K0I2UvfMS zxBYP7x1K&k?$$d|&twH^&59${zXzE^Y6kR7^ItHkSq&#$YDq+VK5-vXqDDP35UuCW zD4C6bnL;2d9BqK!-K&TLe;5VD_^}=97D0;U5!-innz36KA32vc^GicIiC6M@hzta% zc9=x-_H(li8EyJ|@t`efoB&oY0lvCjCbT9Wbc}UrxTF);r@I5+AEofE)#d4M-8E5L zSR^vv>5JA+R})z)RPoWNLC}9Fg82A<{MNYw^s6WxvJcO|iUS{@c->Zr2@Ii8<}aYA z>;;={R3MBqE{CR04XS+G1$O)sk>$nH*m{NWc-?MmzUjdm#3@ISjx$^XsT}uHE8!Jg zGg}02P5 z&C}&u&HW^l)HKBUwLvIr@qQ*H{*zE@b_8sgs{-qLwAjb%9ih~0I(}5FK^u4Ng=Qv# z?U-Q6PF9*v(YpVj>7USUk@p<@we=i(Fi?Q3#00{e*naSKxq|8!=;4?pwp5Mtu+aLu zFkE~QCwg*yUv?tCdr+KeOGJZmt|nc&O_?5oPyE^4(sbmS5N4cCq4Ga{=&!|D^r3;W z=<4`w$Qy1WFWz!C)Fq2~cF)>wlcf)K(&DoR#uUJz&A0f&CHJ6;UkTy|*N80k3uy22 zN1%aL6OTFJwDDM)-OD2zpiXfRR_H$^9+#%G-6{HZbB!9vrY=jn!&Y2pM7@CAcbm(8 zTe^?ZSvs(J@i5ZaaE->!y~p>Mdy$B%DzMjzTZ!S+?=Ez(U^~v* z_8qytcqlAiMDaAy7gByE8(8BoZ0M}Se7>yB+*2!qobF(b`|%Q0O4@)`-wSNt%H=Te zBkcZeFVXFP1-+FD^vgS0CNQE3+@2QT3(GISw_z*LdpwhPH9RPf3?x5Un!+`Q=?`d&fH1_M!~F$Bn|s z*!w)sW1o=G;(pk+A`SaIzm7XLCnIg`F<|2+OXa8D2lLO4_U2Dol6 z6=n=?K@SVLGxhiieDJvv^K6|uJ|*nLgCqq@9Y}@MwZ}kZ&2#uMl!TpDjiN~})rj%D z3BtTvld(&hD%rg@A7|SfhaIE;kk9Qra_q_#NLhMCc%bPYif9{;7tH#N?#qeMKjWLp znU^(q^A#k=>q2<;&W{VeLaB z8epMJ9Pdmfaf5+)ozr3x`C|~@T$;r5a#BXYnR!I5(-P2%P|oFJf)B?^GIPttaOUNo z=(>Lim(7|;eQIQEr)b_KykG6aZd|jV^#a2sQe(i;wTAGL?!co-g=k`)EsVSnAiV{g zr{uIfs7}zuGfjgL)(-)>cd86OD;KXQwS_OOv82(V5-;c;kHq?R3T#@uvG?}P_}oct zoZ@SQZ`GZL*0wF!GDJY!@J)R4LqBPYzKGxYgmU?YQCun>2*-+clUFkyfazTUiCeZB zKUuB`lg|D|9X2AU*y9DpEz9xZO@7#FybdX~`U_1T0);o>Hi*KnlBJO)R0pd8)^8_wOO0*x)PTsriworvDY^pLc+2Ut@4S>`BBnvpB(b z2^^cKh}@2L6OCqPa8#Ux!}Y!h1&1cm*Riir%imC2cg_!^p*zeIL#DW1;7zmxr{J$+ zwXlm9*V9Rz$HY%D!29p`A$7}gthMGoy18#BJ}n->%M6sKI;DzKY;h^523DhNolNjC zTZQY>v!Q*t8Y9?R0)H+jfZ~zqFqE8u42HSSEb9h1_Entms}O;pmwVvW{=?DIb3idP z7Mn}O!4FSu>(7#-gT}`NK8q5_$Cz?>KjSIHC8a=i$3^I|2*XD=aopYW zi=lYtDeQXsFbe!^NUzi?P;H-6I8{p$%US3!cc(X!VfRB&!_Doqn(mMXt0&O>)&j8e zDu&@%TJZXc01KWua{dr$Sl6?cG`QWu%4_Ds)9HWE`QACO+dKl%+Y^}Ij!JO#u{pf+ zAI~JqN}y?KCbVz-Q4+P|9(g^{5DuI8;c`$M97|-u^?DPzzr+TueiDO}?*1n2e~WPG5hhs5MCA? z1O3^L)TW#gkeQz zwAfNo*pQflB%fMfqk2g?LFNv+=g~yMtZi{X@Y0H(-2FH0!dtvse==SZU`VyYVqnFZ zQe0r=fj;%d!|Z*JQS-oBc;lLfp5D91b+`raZ2LXvu6D)38Vj&{tSan0(b0cIu)`aEs+}Cz~xvK zhaN+evX@=1)FiaVDFakii(ui`xwJO@D422C-vWkn+$4v?Zt>|P!FC$?x~vv5m8USF zwiij}u!y%%W&#`0(t_$POS0QuO48T)t?>KT52U%+p6Qt3MFL|C*d5n3`LpwMslvVk zP`FE+&gptZCI7pKMa~++em_8$(m}bB>v&JgDH#_X}zl&~Z*pc*a;HIyjT-D(&YScskGVUc0AI z*IL5%>868y!yTCB^+2cq#jv{P5gg96flf|=XenWST( z5NY-WvV;5S)4$;`VP_&!{9X>*;g?)y_yMFPXn@l^eO}eh|4`BbU3yvVB)E-hBrX^H zIsQl`T5~G~|C3XtO4F1@nI)V{?=rWq%eE11Ym9|)e?{cRf-B@})>g1~$)*3^E~C0z zE%*_u-Z8;KLN(8x<1Y&IVp+GvSa#e;vOK_q9gY?;F+jlxgN)(o|*s-GSjKu zIYRJf4gL%NUMT1-K@SxsicIfTVIRIwct7kU%^JIhYK9GwZ;4F{xj` z{7;0i{8bFza*boXhAB}?tx&$E+(EdICW9qTHV`{G2Rx74ZGGWe)5lA%q4uGP^rnF8 zr^X1;lkd*(-SRE=@0thU5du6s<2E|?Z5pd_&kmMLCNcMZ%Cm!${lPl;30th6h0Hnx z_(8n{gr%KkBVHVagl*2Wet~3VZd(90Wu>TGNGZRu_9#Ch?>9WVZYOG1QlyVQj==OO zk#>fgEn(j&8JgsK7(;S2Eu{2fqs7x)N+F(ibK(N{P3TvG9VxJ!S3pXZ;7}K^u(qiNZvnI?Y7NKf( zhI79NElRF}{?2Yt3Tr^xhOX=~vl`g4x`pjMI)ZK+EJ8ujhpCbMSXxl`2Y%*ngl`ok zkbL(Ktjzt3%A2yFqk%==10<;uj}^^a778R*QuHh`4J{Nc2OAj+_7!fRH3H7{crr7qy;v zPDl#XF#G`_m$ucjt$U2f&o9Gwd^uC;|e4by`=31X7Ctu>mF8H-IDx?JAg)J#eipWC->S> zrqw11nrrbe4<40crR<}CU%hRuAB?MT4LbynK(4cT&0@@!6@U%QE2q@f`;WA z0EP*>!ZV4L`+W!fUX#FoX!;109)E_%>rUGp@Rp+dG6GLNRf12s8+}#zow+)(2xPu( zWlMkS(OJti>AT7P^f*EE$Cy}1ExgEXwI89{hXe4YqdLUuS-fp;vmGfL+0E7!Rl(Wi z4)l3uB7MF4J8ZUY2KO_c;LxTiwCeX%tkBR7Bl?eE2#J9?lYtXwh5|46KCIZFj~}&N zhQ?TV)?m&R>TnGZrt!64b94NtuMuMUHXf+X9Y()om{)C#ca8g|9f_>~%g>@n7 zAU2kBYec8o&fwUl%Pkh+!!qM2FW&>ygEnBJH?d^cwFH#yDOTYVuAk+AUe*(E^D`oc zIIbrB%;hGH_Uu%bB+ z*kB=?x!MEc+BlbkRT_+<^Ys?T44%Y!bVabhG#J0&W+}aYRp{wA9G@yu4o2thC39CVL*IkHaeFu+WNfQ} z)SG$a(fsT9vr9NE_It)V8|?>$o9>|Qq$YgurXP8=?Lwm|VGjKfKHT^bduDXtUpQj@O?PNTuEf;QGlA?a*vNG@N2S7IQmb4netSXDY4rL66rr6;|15#y{<%u>XK2l;RHAkq!07O z&9Kv1Be-?79!+^`iZ${>KzxB3IU+C4T)*APF%;{ukNh#@YZru%Ep~=)2cGbPKXSe- zi86TGph$;gjA8uBWP!yVt`~hji?phkkb{%w5+`&F>0O?OC+eo*z(uxX_TJ;TYkd)% zyHkhz-+5xjXbJo}J_`^2iiG~_z3|!l8F;iC(}iCyLD^|38fK}=_018NtsdaKoCb7G z&_npIRe_9)lAuk$1F*~a8s18uWJtO62`&1mhP}-e;Qcf6ZO<6_bG`Q$Xwd-?34fA^ zAIl6Aew-h9z49Ww;O1OAT)vZ!cUmCC?Ia#+4&YcN#mw3|N)CyBpzgSJxJ*}r8W%ms z4$X@AG4m8%fIdy!6*7s= z+OipUDmH-6@eGua?2TJaT*B8rE(HC%P3V1YE>W4e6_yyv({a)sAa3c2ZyxI+j^$k6 zdSp7*6@Eu+{w{#oUq3+D4;A{jJs-X$s?*&G!MInMLtM{l2HW@YjO+$8a7#(V6%O8z zZas=NW`x0spQiZB7a3;U{$T9$U@cy>OP(y!It6MqSD{<~A+A%;BLlZLf@;KUxSyDe z^QK6XTc3>>_1zaBVX-;${nrFgiJy+`vi`wX*$oxwo;cM>9YFz8RfxlCZeRb&AEyO$ z!4H|!&=hu&sP8*V61_OrOtd;KcicwayiP^#g9nNJi*mB*z*LAXPQ@J@3-V0ERc!EZ z1YK#D!Z(dpV~?99B$-SjR@>j8BQ`8DsO4ePkOMI7KoBqEKsl<8?FDD9YuC{~3Hoz> zWA}xTaOKc#{5$<0uCQd0ws8It|tNddo%`VNg}Gr@FpIllg9ukdwfC(^Dn1M;Q^IyG;T z|N82%q`)1XE^x%5$E2w8Z-5Z9Slq386Ae@rp*@E$3uV0J!79gu1aS_UfuJDxy;qSD z@8>#=-VcazWChuGQ<+-ZE<<^H+sV^>OuB6v1h#x7{9rFZR$X>TrgaByKg5TH9XIe6 z$pKPZFb{7M7T|z!Sy*}ULj0-lBYO5Bo@0;7b9<~37(aIeQd?i)cb_a_VA4v6;#xOi zBgbqT)LqE5KqJnB)+gN0xy}uzx`W!#ewaL`1sA8vP$)Iynv2M?fR!z!2&H}x4o)CTPIjr7?fytPQn6pZBW4|tZY89YDzjpM+WCmE? zQGh3@6}Gtb85pTaL;0!WQCda^}}U2>CZ=D*^OfOpily*7HQM(k_k{f zsR`b+a@{)jIlK*CmtpyamDo3KtuVyR7@vynf(QRe)73>3oF0zFChF5b!aW|ll;Ii2_oJ^25Bxgs;7jYWkY%40R1_WsyKDzoC_M@6 zRz&cQEnW+g8_x!d&Fy=>}WA{KNC7wczCweX+sUD7;4yiFc?-Q_*fk*qN6H zK3@gA<(1p8G{^b!w_k&0g@95Ax}nsWo4@Uf;%O}7W;WaRq01Swuy$_~E~#@DR)qb9 zN^);|q&p`}sFrd0TZ$py5$39vxKz92)}Pj=U4Lm3=0G+?=B8tps+gQesk; zmO@1%pL4ZkCl6{`N91(=d)IA&QXauPlg{wHA!hg@!B`xj~0#4QiGe?7w+b3_b5 zlQtbM)daS`){#G1>a=jU0AG1o1k6=QT01rZ?Bo*(xmAU49~+PK(l^4niU?5L!(~4Y zt%kDjm$)^(7qxdH{QV0L&VNfrv$dp{yEz}o!ioVd5358AFhg8is)>Pig0QLZ3o83? z6{dbGBNqbi2!o?f!4>l>c(zZA@cNBq5dE>1c)L76ozIf6Y|VK{YOuf>WGpqf)FDjz zVS@}8t1|PE2>Mz(h<;=p9JZCjsE9j<&Hs&DI5zroj_o-teGx{gMv38@@3@%bqv$$U zz^658 z;S2ZFnAnAPVQIBC6XKH!VKa;IzmpYM>~a-)Vy}Vsi50P zdUU~@QhepZZSbEF1s_L+*yH4Md_)+E{dbq619SSw1{V{OwW$GG6X#=BQ#E{P+7|9y z(I5<0s>0iL{zj@RBEacKB+33YC}>#|NLF7DL2vB#z&Z0$+r94Nv0cDi)W~#ke(7R( zWMM*k&&|h=rpwTRg@$DRvT10UgC3MWFlKB&1!2oZ6*AXdoQ}0OoULy6w=+Fj@PbNrKTgNNx82F*j=x- zZ5sR|w0hc4#&Fr$2j+j_UvLhXzO%O56D5 zS0F*7ne1M35%-^#<=74z$eknGVbi`a>{BAojATE-&+;Ac`{#}%zHJEK*tr~!@&3eX zQXhwb-_iyLlPs|4+72SO%7WaKehfp;!k{NWoC)!m4F9;K?yznooJ=}Kvwi--0}QR$T&KXi)( zE=UN%5!dYS4E-_^6cCKcDMFACQ2~>(HQal^D$pgQPJVPzjIgKju~A z!PhnLXLlL)IA{+m(H>M1rHo%C+{0FN6k6i-Az^(zInou$Ep>F^0hgac-y7fynT%Sk zl(DN`m{9OX05UJj@rj38Od-eMu;{n}BHMY$+T00=Om4t^vl4i>u?HzU_zo*fU17#q zBW&mRn@pDL5oSz3j(c~W#=VM$Bxpe%c4~6KyPxENHPejFoXLdJY1VjRpf$V?*+oz7 z(5I>4Bhcrb!LHD_j;G%p&%SA&!x&8y=U;Qy6%EKlq0o)j_;NW{Y1o$IFr+sG3(aI< z_V6oyp}{?xn6{Y)EfV0Ojmq?hYa3%JoJ?(^UegruI5J=N5M1AG#jlWG!El>mnxu3K zHB5{pA)482ed1OwOH>PmL%mRGHjIk3?~~wFMX*2nHmvZegL}Uwpk12bkkg(E8Z#5v zkpeM}hn@|K-fyELN8@m53$U3_?!n`^1mgYUsZ~J&|G9YpeCX6<58i#wf8^|g73%MR zlb#eD3M!&UpMQkzjhSfbb^-ZbvH+x=K0(k%Gn^OO2RZfH5cyRf{|&rHm+NE^x2K^@ z@VI`#pe+aTYj0!k1V0B;n_4LQo=iJlUJx4p_l!N7w1H?`)1aO=;!%@MBTTGqgrjFx zP}$Q$vOc&Qb;fzIQC$CEXYXA`TSA{6;&q_<6TZZTc}F%EXwzH|*LjjoHDop`$?wyS zfEAZZK=Af0`SyDkS;OtsLN995FP>NNhN3#|?DqgZWVOTH?bB$;;8Hwdb_Rcwi^2+L zr`rWt_*0o(-{Ecc6Ie0O1EJX$+0+wP|3}ez_~rDzalA!4?Y)H}NhQ^DU#EO!Cp+08 zNo5v=P-$ywC{3x9_K@niuTw}%B?=*AWMoAlgx~r71&!zRoO9ow>vO%|*dA%o@F+!( z{Xd_eGm39(6r{$s2Y#UiMS^JeGY~_=K7q3NB`R}p8oNMAjup79z!tZ@BJLJW?7ubh zC~r+5uFA+@UBg#U_rW3Nse2ho_AhFPaeczh*q6r^CF`=;KgHOymnB(C6^49xp4o78 zvNHxcDzj+`25bZ-;jxRe*lp&b%#6LtY>HzFO(}D2a5^`JceiWV`7cs{7lxulN7$s{ zQ~Y_9WG>QEU**_Uql2g$AOqIsrL>`k+Z7G3z|OD@sCnoip;NOP!W`3ab?0;VeCM_8 ziXpCp6Y-^Z1^BCcvSh3#{TKw+5X$--cF`xvck96)^4D%rW%6 z(Ry7{!?MJohS!@{QrikX9(j_=zv|0?Q`0ymFTBAQ{Tzrlbc5lM%y+JfaGv!|6XbZV z%dneXp(%@MNUTO4o?JlD@+Cm1eiSYbQO z_RwUH`sF*%uvS9UcB$HI_NVwX_SvVoY?!(@TP<*bS^j7mD}8VrC&iZIpUFz>V5Mrq z*>ii?Hjbqe*jJ5xHb2=G=`7L^G!r_OO>6v?<45M+Eu;~{)V5<~A^IE%YS?I4#eUgq z3Jx#IsYPT4ZgY-rIJ0LF+wL-gd;U4%41oo#hK@L!TOP%{pV^OY{Ag@HIK9F3GlSko zR@ho^O14d_I?@m+IHB=g@@ag&)1V>7GmB<@>L=zc>)B)>N%oc2RQB)8WE2|zN5}em z$;zY|a4+xxIJb&5C`jbs7PD$5`9l;~<(-6PaY0t>ouu8>=L9=9PsG(hQfy8BJapn2 z*-}GSoFx+2;N~k2TRWB6Uem*DWSIq3yzrj*?c0E^XIf#Qab!d4<{en})rcNjSwP*! z^w@}Yak~tX3u!lRG_VPRY*?8HtKmK0_W1NtGWmxf`#$)!jpMUWTaV#xT#Bao*LJGtzU>BL`W3wXyMkChv*#t9F+s-x1aOk}lE5&u-e@K>tI+s9~J zlZwOToL7iekk4Dn*qz~TkZR|W#Qwkr!Fl$Wd+Rf<(;mUNwQ-n{F2F{-lE?9F>deUE zKV)L>7yR&c5}T?~$DRu+Xc$?|Htfb)y6A%`h#weZJ2fA%f%S}SR)sxo+H;Y)w0$*B zD*R?UW8X#`xiuZyEOywQprsA`LxS8cWisvbO=lx+hcu*R2C*vTABfrS)#Prvl5NJN z_v}k87u=>|V>|E3EOySUt&rKfgK=0tnT=}b!Fh=y(5q_3j%uZFyAQ!=xZT{=P(&A_ zA~%z>f3h1kt#N02Jx<`ky?Ja*9gi{9f6NQ$P_@w>yhpnATJZUl!`Qpv3MgvqVC~P@ zGmYsiY+MoYg;*T+hXiANwz`$;3r#Y^0|_nsY|nY* zxX(C^;TWX0%0(bhEW}0~DC4{;ome#O0x_mxT9am*N!# zad!M@DN%nJ4>p0C%*uBiL{#DfeUa=0gHkzAwmP2tvoy!;N1swz=ifxZk?Ys}{KZ$7 zROF1?_aM!X<2*i}$YeTjbKBuevcy`A@mp6*?o>)Mr!GD+sBEjpO%r8c;th^lwn~L@7UcSv+Z zA#_v-THq1R?~BK@1Qj-y%g(~PZ}2j^&XJxOPADXN4G#>rqq0gMybF!QRTa97#@c3- zs^?>4UJp&*JRAM;ms5ebW4xB}6zrPb3azuRqD8bI^ZsuhzizD#^U09nS>I?Z+HRGr67F%F0uL&2#*||lIBAy*vnqvU2AS9?+oNYVW}`vb?_qJ z@82qTe_ERrF`CZ#6YIcb0+(6Zd2zfTcgS>cm6$Ffy(PwT?soRskr0r)f6}j03 z35&Qc+;|rWUM|BfU#tQQcGthCM zotw)|WW@8L@Yek&5T&~nEZ=3Lm7gD&{vM)zq=Ce%9j9$)DfPPWfLG-x!&cm$13MDJ zP*+nHdV6HZ(mi*`G_rv52D9;dV<%2?yo^TE&AHv#1@yueiW5$B;3G$03^48FO)L3E z)2AlT5blhjZzIoCZ_{NhJj{TqIpLcRIbfbOh1#8qv@vcgp_$hOnNC$zR{s}^feI(F zAiWICV^+UygzGvxKxTu9;iJ8LaZz<)bR;Z1=)>u*wot<5cTaalVV zPUG`8{tSi*|3YDJMHH@nr@%H|9H*)}xwLeO8Xnz~2g2KXuvg$Tk$9UAzcMb;u^a6; z=4}h6VI}lX<{gaTSf(lD3Md)5!rl4OJc%w!V%>y5#^?me@>POB#VC~|th!DPD0h*uh?%s`T$?dn@R`2YB}IR5 z{fi;7SkPHnj9DLB-%kQ^Yiu&QQctORH ziY^}~(Dy_M~u5Lac(um)^8^zC0sYE^uw zi_RcfeB%T38)QSDRs)$3!1V|Y>>@5tEcwSdZ`2N>IS_bCmF;MyNY1vw@3TI1^jjh| zSlNsw6I;+j;XEn-a2|AhPU6|ggRm@QoEm0z;I+HE&~B?9^-XD`iY2#rX?p81z(s`E zWgLaZJDun`cn|WA^HHH(niZHMLe|dmg(>URSvh6S0ia_*q;HF`0j-6wudNdtR5Te+ zqR%ACWn=t2S;q5gDpr?V1EJJM;E=9{&R&*~|7#+~dEMp5D22hz5pHjxikmliZ^z;} zG7PxeVw{2(^om#09o=$_zDy(jo9d6*(g8HvJBN2OrGzK;bsh{jy7DDX2UD-NbKw~7 zLYsf3kUgRfbzTQ=t=z$jsy0BpQEnA9*gw{ zm$JV&>ERVHuen7T@{#kqOk(%$pG?+WRHm)kW5o7g1a;IZA=}q}!V|TE?0-IS#P{O^ zp26KMkbS-cQ?`51F;QFU)#MFJb(ax2ql=`X)eQpnH}Z`2zmmc=i5L=FP4^TFvI`A` z*t5smh@P1$n||#D);vFj?6djQ?T8rb;C-JSkBg;S)&22ti#z#k)5A~N_#ON<9D<9s zVpREX74YvafK}0pA>nBMfv+g}M3B_Em&q@kXe0Njn_)+>wa1ku1H^;HE-*8XhJl@w4f|PBsbaMV6 z+&lS#?h#M~t%V)9d#eij;gcRr-&Be&k(=Std9I%tp986uC-MC%LF)4}4DaC&k(&boOBM|@7>>9uRII#ZiOF#D`TR!CGr0Ib5G>Jo2_5Gj z(+gR8^jFJ0)M|c!{~UAiGacy9RD?I@ zyx^CR3a!&T3yqlq@LoWaE#IQWtS){?j~05tf9etVPAUh7IptK;fl>Z%Z9z8t+%giQ z6OSWZPjS~)cQp4`Mk|FscyvsL|EeMdBpuTkzXf{;mL|H!eOAxowdPv><+Yu_F$jaMq;;1oW#2;fy?Go{F~2K zV@}}{E{|1YZF0-$rq2a1yo7M?lQXnRHVrpuPQfC7Id*DMJX(FVgS{*6KpgkCmuDpK z7A?`C7HcH%z`kN$$KN!ZY?V^qenA1nYhtjmyoPSsYlJ$($1pT%9LHz(lR~bC*FU%( z1*VtrU0bFyo0a0hYWqbLEV|0udN3a94W81=Lpto-t`ID}9Rg1_58I$x6Jg2*Rp#RCDQtM? zR0z_V0N;z|Kv&TZo}$n-ZujChf4#038D901FEsxgKFa3g2n$1Oem@N(r$TNMEp=ta zw@-;R^=!r;dhhY6hA{jyIfL@CPheZ+Eu#N^o=umU5#Bkc#4bK7&2(<@f=1u}!0qT8 zD%y4$k2i9;gt!)_&F=`D113?E^MKy5=waEu^4+?F&<`XxuGe?UO6!< z9KW(dXu)NJA4my>Z>M=Xm8&42MhO*t%c-A(Kk2%kiD&9Ik`&=tuvdK|G2L&EpDm6f zZ%G?Wmd~Ll=W4O*zh_~&VJzVb-%Ulqnp+ zd+TPQ;He7yBOC&e(p~i_2U9^{*&&Yq*8>H!}Tr;+WY+S{oqhG4PI!T!N z8v)0+EQU|{KD=-XCFuS769o1q(HE4PdpP@oOVwR^|A!hXrAV^_LyO?a_2b|@QH|)9 zXL1e)3$!}!3zHu2!{0tdc;#9ImoX=Rpjsc=ZN7zD%cl@rq{Ob0O((nG8o|rhd%VZ@ zcG1==J7AXeTYM2Tm72_G2HpFAiSOr`wW7w{9com7w>;Yk(`W`x{dyX`Mj!HnW0Fvy z&Ar~3+m+GJTx9h#btxokN0NDV(u|7mXCi3Xz;|8L3WnRcJ3_lO#1{&)p)y{0sr-B0 zDpw6w>WCvg(1^#9+ufj_`3=--*FjoSA`Th8hUZRa_#4-SfJBlZqZ2BO`#4v{k4cwF z{PzL6e2D~`9sQ7hx>E&RW_VzM)H`bReGA^c`Gh>;GO9iEzMwtELwKztu76zz??ZRc zGn@zZ@E0wZz1s;hj9Tfgh7J_0XE4#=EL}K-h5Gl^bRAEMm2O&(7iJ+@7^4RptV-zA zj)`n+@evxj6W znD9;Xroaldc{qAD-Dba;0sDPSgVy$Ovp~yO(0&|*0$g4noPGdPf~v`Y?Eo(>$^tf* zt;1}e^YCgo5Zm`$=2tgzd)2S@(lqZRBGBtp9~j5|kl*uYz^m=BNp>$|@n{yxx+j34 zZ#YxD{wgl(xyqh9d!tB1Q_*JzP1NPX{@hlGrFlgcl zHAgc-qoT-*E@0-Oz;LGzD13~Q~WBq5A= z6y(B?a4t0Uuf%e-EZ+DqpN5}{W6GZ&!2<`Ii9oO)^^@+WuCwD=kxhIM8mz;V*-|*L zdo?$2Y+(!pY+%E+bvSO+3Ho8S=&l(~!)vNxx>y<97Yf6nuTePcm&9LbRzgqx$j4V7 z^uTVz58B_h9+ZEt#I(eCw07>nO^0*XZ{-%)tD*yob47`;y{GE&tz!{cmz%~^mh8cSXjz&fAPUYo_4vs5 z8Z0<^4L|I;&GDQC(7q`b1ZG|2`ON=GU*vS4Vy_gMEv-VYcaGqZbOG1vG%>?LJBjhx z16Y5e1e4S4u*TyeCVQ0Oz`tv-@{u?!4x9#kY6|!So!DPvji4K62){x@(Dj8I$b3(S ze$!9v@Yw;L*JUr@pV`QUiqv6oTQk@xor9yIkJ(==$KFef!O~;N;AK!nPG;A`%>s@Q z|F(v-w(N(3Kp!fjC5UD>qR>mv7D{uJ@Pu9^RwxVebRD--G51M$hszU&Yo_7FsxtKV zxCUa<8j!Ei4XXc!>B^i|cGIH2Z0EOprr_8jddwx8dHp~SogB(xE;rkJZg879;B%C& zmAV0It+<4A=mbO_tpt~^9i;a3BWiMR8XnTgg=rDjVW5f+n&Xwk^=t(l$VkBY?K3f@ zXE}tgQ-|W6ju6-Eik5{zpsN%{gHyx7&aJ8b$niDo)JAI@eHsF?jh-BDJQV6e(^w*q z0cOeDm`puG)L54ZE3I=#|0hZXOaig-O&hI_k7M7gaA0M3r_dWIXPB|nV%9EMQq2n7H6vm2 zSMXPgHRI+V2=z|`(6(iS{oZ#2FZ)^GcuX1zj0AjZREyVQiqLME5wzx;!hiBnP@CRK z-WPY_-xF)lE%F=Ns@hA$Y(wCCsX7W90p1Pcd_5m48P`uO)X;wtZ7ROWW%veQ?HPn~ z_Ura*!}%69bMEzzqtqCyDSEk+Argw%Pge)EQZbRVsZFT2G8ZnB~p{q z1-1`-QL(Ry?#W+@kMGFXPCIUhzuujPv5X_c>d*(mTXYz`pB~27J2bGXH5x?hu7bs~ z%XHwv9MF2&hF`QwQDAwfwOLjL^af->ir+5sKJ_SDA#H0j}YAZ-HFRWYPk%t3{ z9BP6;dndtb@7tVnE)Zs_TxB|(599El4wyd6VV&!Oh`=;6&RJXsT@^LllH?n>lUjyR z-4Ce;zZjh)${|2oj4yDay}o?ST4?#N4jZ#pgXRiP2(U8b`6)l60f!{v;HB-@s~`#v zkd=9H&$AX2>IQdC^QqnmLz14==+c_74cN*Or z)r8}<`BbdI6eny}1g*H~ND1EL(RMM7>LLB$b5?}1%c0~Vs zL)Cxv(+kGQc+;c~AM8_QD~|`EEY~MEZn+$@t#qi$jtty1c_NeBFcEi5*Tln|yZ4LX zAP9Fo1Vy)Ty11i}9<6%@8S<~dDpP?@z4etJEm==Y9`&OCaS_Nn@e{a}clZ&ffKm?| zAh6Y+c27TwP2TC~mCWtA-8odh>1GXW)j7nkdK3fQk*%cVd?>aw1BwKv;yCjSFGh>A zZS%`%<9bJ85Gu*~1_CWzB81^bAJTsNA{0}UW}hpR;`?G5_Fc9)8a#Q1qOT8Q)ZbjZ zMx~g_SwiH=T`|^lOD#^TyFgz3j=_TcXxpoPLo>ar54ZcKSxfGk;c$FG5YH(uCE7*KA5p&bO^ZbT|*jb!gz1CG4gl4^_ z4`&wezU!WYKVQLJ56l1rZH8mG9G;NQ^~Z;D2U2$xMNj_$GNW zI*NZIGalF?9eqI6=RD;3$^>9{k^y7}1{7 zp!ic0k6K0G+OwiiWB8gE7o&~&w+8WV^lrL1y_Xz_aznx2t~6!s2J&sma{M>eN3Q)F zL`&^KyrmaKJ&sRdOcS;7^V{pFITe7+5n@*!NJ6>f78q!$<4;Ri3sN56U|Y#qlDF{y z)~sF*JKC-Bj8OwFEtSH{U;@cA#>fm^Lr5)LMhaSrP_|PF_WP^`^Y6>atrzX^@ty!v z@aQ|bzrVw=^zWmglOzP(JW!u6sswwh|6!BXM7G*)h>EblT(yzHd1Lq(`8s?3^4RAJ$&SJunjiaF<_5WD-0JIv~9rduM! za8-IfW*6|uiX=d2OtXE1oMgv^{z z;pWAcRLmrmsB1Wbh~Yfc@qCFUh7;H&SI+=Xi*q^MRcGI%=+M+2Dc172KL*R?p}CYJ z!&{n9x_0M6`NG%e>mAC!Xj_3d15TsAmJ%+kN{7FP+dyieJBkV`VPkDhZuqJTMRWLD@IeG(-r#C&xR_g)8 zoGa+%^#tR1=Slmm2jmII2Q-_t4^8~9;LRNhY?=RajLA_4;;;&T_a^ZZADt%N`*g^E zk{95x-xN0Yp&_G~unGi}`^f3`y*81{x?s_l39NwqTUZyfkCrRH=Fgun8_xM8g1$vR zdNl-txyL1x+AtB6_T5IsjyjMt(nq6Qjw9_YOz`J5{Cg%6mul~+z_<1TD!1~oeDCG$-4l!5IZf*uO)>;GJ zdg|;J?%lR{{c6%Y^E261+DoI-FT;-c%Iy8vy{O}Jj&_t}!={bj>A;uSfMO%0eo6^& za~>L;zYUueOSsJJ7KSYNj2i>A@JhNkqq}ejZ40H@!m(5gnr~J&<*qn;Aj<+hGAz+@ zasrQ9b30pW|3huD({RIYDfSd(z#>US=sq)uaWPY=+u8)YO$}-J)mOB9+bOg$m;{;8 z>oCz@lue72Vka*Rrf*H6(YsLxYdgK+`-KhsS+jq?zI!tkRvkq#6)xLeUj^#r zL2!NXHgx#Pd00hbh?{mfSS1f)yvbLPs4Ik5HdPeFYv?+Tn|fuHEc!;d;-Nkq!q!L~ zT0V7zntl>v%R;&PLBe6mD?dP+d?n~D;X6<=!u8|cEdye=5LcVu;ob2Mz*Dm?kgazZ zYJ7D+{%x;;o#z`c{LUuW+Ae^`Q_jJ->Sem{pB%fuR*GFca}sWju7`hd7C7p;5q28Z z;zPGLG(%+~dr{^f8tgky><-GZI|_=?C!iN4_0>?_CKvwOI1_T_YO+oddr)JtJo9a) zBQP^XQP(Sjj2zR!ua07D<{K+$TyzxwlRN`+|MbDqRu-RUec_3je4_(dDQLPZA9TYH z@&*#-kktpi!A7?(NM3glYA$y{k3cL6yosx`l8+!#x7(oHD-UDOJb?L|V{!J-Rjh8F zfQ8}Z)YADrb^qf4W}I(Wz~m)*j)&nV=MX5Lkxukmze2IiYxFCcK>Z3~`!UV!d9ZE7=DZOGwipMJ{K!=Lt3w+)&`yPwU3QS=8*u4pd#ah>V*(0FmAuoHNE1 zd=?3zzgPsSPWu3f>jr62=|`U2pLi6w%k}eXcf#%n5yt&|HHmn+(I(e>I;0r2fUHX! zF~9Q|-uO?(pUdQ!L(yEO#ZQN%^=W9cyNZZxn#2zO8Q`1e`J+{lEi?*<@TDpwS)Y67 z_$m*#0%Q0F?M465tp?lCwEHxUZ(_$fhyU23rx`3Du5eOq6{{J~7va>*Hwo2WCD z0oJ zhKfaD{%8?M>nh>tutKb#cp44;mFX*G3y`9*_@MF`eChDl@4IrNA+| z5S{P5hUPy(aIByRo?HI_R{juX-|L5gw9iCfF*j>%@uR`+tHDg>1<2gd!@NB&z-nMN znoe6s&p#=F?CH07#yzPtH%}EUV$bsz$5x{A&?>a4UIJ$tPhtI88<46xi=?mu9^Y4E zjIVM0i?c4I>-b+x2n)eyDY@KpxSo1@se;eX4v^cu5Z`>dLli@}`{rDE(z{Qa36Hr& z9M>Uia%jeu;LG5-i2IxlOhJQ!Y8aH>NQ$=oMHAE8P^UZ*8{P*)k8K55>es=BMLJ|| zAIC{qr^WbZ^q}suGWc%gfgw{aK=F<_;CCVfwjaF>63kn8qFzJolS=qy3%ZGE|8E-V zF2!azR$#I49j*(zno;>7jR@M_*yUiFmOXrQUY7zQn;slKAH*vuMA zJ)&@LdN&mE=TK`e112n?2j`nEN0;Cl9McR0kK;}J{?u-IKsgmQI8TQ3PhGHQha6*2 zy%;|zS#q=UIrO>tV^VC{Otzo41!vzUWYOn|pt&lQ?`cs*?})9!g{ECtQS_hx;MBkXxX%}W$oNte-7${A zsnHOXD}fJqY0!7<97sqClYv@Gs<=!DE{{BgTERK6RHOtxf z1I~UNh8ur`Nab4sI-py1y*kc`xI`>j(>as`tSMA$i(Oe1wKa)v}ODoC*6+&U? z2+z>5jS5HJgPS!=`NIn=z;ZaAmRB6$a-|&ny?l&+)-e)(`F3LH)Q11_D)6%EOFARi z0dEJr1^=}S^sZ3C9j+yKUbZr8ROX`5w%Lin1$ccQ2e^2aeVo?JT!q)^5_ZfY3l1MlNd=9lK zZ;uCM={?*Y;7y+F`SU#4?m~#u_zu(e>Y?|Zv(TWfz$O{Z05!$gP&4}$eS5+XLh=$} zm?@-6zlYFnW-RpYP=@<1VL-IXNWhtU3Jyrg5q#B3Z6qX_#;t+%uEK@%*$!=LdfkWCm={6++^eMU zbqAIE!FhiK2I@u~+QG_S7Nx~C!P4+5zE*35{NA&W{PPw5{3-_2LzHu$y{G=iR>9K? ziS_eL_ST#23`3>Z*D%b@c6IE`!Exa|=v3k3iv4T(PdD)4ltLcdIhBHmdY@>)Yyy&J zrr1n;smAQSUPDVe9+L^&oiy<89vt604TB|(Ah1K9X)3M6|G&Gdk4=V+Qe2{V97M*(V1B%v7=Y{L5u)VRWjP;L3kh-`6<|kEP=;shw;%xNG~#+8>$JFxrByHP8VoMhg`U zML=`)93b1Z060$F$H?ngv_y}aQ$&;Q%AJHPUjnBe8bC>p0{_`0J{XDzp{}DIyV8~5 znjf8L;57$~iWcM3Bf6~kdKu(s%f*4c8hkNREu3W^L)#n@&?xX2sqQ}p^FMxo z+uhRC(Z&J2IKOfP$I-E?;`SFe%!EtDhoR~HQl!W^ARW!%mq8L3?s^GZ5-!2-e+^V3 z@+{B6>jF-i5eH*_5oqKy1WsqTT^f%<)G_XdGs%Upu_zkt8?rIF?ia~Dt;sY>sPVQt zNHSI%dx*?XC!Vf)2s^CXs7t09`Nic38Ih5&wj!EJh3Mj&&z>lo{D)*~xDt=`&nPBF z*1K}dwCOT)Q1x0OCakU_V!JtR*zLWfrS>3A->-ugQ}sE2oFJR{!3tVmjFSxRJt4Mc zFHUdjg!I&ES}OMn;@$}}`FC5uce)VsG#_cq<|bIssDWntOYp<~L}0jC(`M@|v(F7Vn=5z{v-L!x~$TwkwDMa-tKS5A(D z&-)ZGeBVt^8FZ8AZ`#cD?{j#v^81MCqv_z7F2!@{_(p7=D3K3>?{MCmA)-9DkCack zh7*+{iEM@p8qS)EulBGuhmI?-%AF}7p8vG2d#e<(CGTNnZ4PiWj4Ap1BKTR{P@Ql2F^#2{HfX)w)Q)1 znOQ-)PJbjER0!*?-G{|ll`yCJJjayadiVk>NctK}h_kGwE=lWP`1m>a@83gu=&lm3 z75hfBQzmeGh)B)EYe9PFdipSNn3T21GPR2JV9_KEJA39}SV}c2s;>sqBf}t8H%waY z*l=$6FrH%gd*mAyVYVfYZci0sZ%pq(fz>v6h-hMzUKsRf?C0Mxp1>*=f1{CsX`C}y ziIzW#<|{Kdp|s{TTuQ4&s_BPDyi(ey_zIguMc~mA0rGx4n(jUH5e(%v!nY++_^bXJ z^tt7OK?Udg3BH1UnsFGN-2r!Q4nQi`anNs!hxSwn^cs#p`%rsc{&*^yzt&;m#g;;l zZyYM{D(QWGCSBUHg%qq#fl*(K+(zmB_up}- zN1Q!>E)%RztFZT8PoN6{ zOCX^CDZerFLA~Ff2%1&B28R`9K-}hf5Ewhj>pS5GjoM#{h1F+zTJtJ&KPaPOp+m&s z$!yI2!`%Z+J)r!(C~P=y0F%B5QW2?r(Cn?r{C7zmekvuR&(BdD_?k;nwkFew&-EBZ z_69vNI~uQ9%)r|H!tjXPM()T=*518G18k$n{(q4qH-8D7(rBbBTl#q0c0J)&QDJtp zJq}lP5Hxw32H&rF+tdaNBatmfn~#D_nR_q&`Ev@_J1QV!8k{4=VjLYevk!bLpPzutcCW+<)BYO6n&b_IYzag^ITsL=&Txn zOIv=xmCDQbQgS(2u~Ue(tIVMB#@*O>*cHOVg0Od46b-0qq3)JNpriMJbNA{n!*RuY zp~wc%(37j{F%2eCj^Q}=T#gz2uMrw+mx1&2LFg@Ppk9-Oxf`)BD$9pKu$?0&2Ka)3 zB0_Cv7x;y0p#96)9P8yX*7d4_OWqOw?S@haIL`ICFF&)1u+F1H_7ZGUwFg`n{L3GH zFiH=3JVjU0DtNlN6`Z$r81ZF291^d-u_CejyeJ*5VG;86?0U z1eCmd(dD)e??`$U=<2LP<;~u>F*6E(HYzc3nfBC6CJFKmwu5&{AgVuHnjlf%q@pf@hXh?pvIcZ#?zJJU%>ya4+J{L(*lDS5_wA-r#HKx(<&CS z4O4jgG#Ypv7pGy)wGddpT@mFur+Z3_E@}m&5K$jJhF+He;|1Di@azM|t&fH3=iKaJ z)hr0|k%OJr4&yJ5iLmC|AQ@VehvReiV3JE$eSJeGbXh8ac-C3eJ?x0Xm1R7O<~NXX z{3dP=^@D_p+k8`;L%Df8aUTwc_OK#|lF-F}Yo5{8$~bh-n+J-S*TLbW2$L?Y!9EHa zf_Aog%4ca}ecxlO-lcR(y}@BI^?XY>H9K5D|)&oWTe&|w@RIp)`fFSx$? zIk(48LGFq=6E-Obhx<*)ofVw(w{shq&OU>QcQlyDVSS`hGhlF8I_g}hChA;A!z*2m z`9qZ9#=jfnfUh{N+n~v0pX>$uvM)F$MyX1IFD2dSL~4H@l?P=sZ<1%;7RlnOSFPm0 z90PV`Y#y)c=1Tfy)lMpFGlc%lGho(*P}(z^h_gTD;fB{D*qA#`br(dEec#)-ZgC@? zT38FmL`U(#xma31&iw4#Z?l~HTnmA5Ey?IGT*=FWCELZI6)g#?t{r(J?cFgp7dmdkGd z)s0EC??MTdt$su^?>!-tw;jafa z;rz>0e2F|UB!k@D>u(Fk#!AFp=3k)h{9nGO)=!$VFcw@hPou!53%u@+tEfeI5**S> zry(NG!B#Ga&fEAJOO_}xcFw6F*gisy@2Hda@-?(BUV|w(|Bx!Zh(=xSix{V~hZt_t z#d3))koPK+_$jqPqh$+OB`L#NJPyDDo$0`ay(4v#&x8BzZDfmYCv8(52Tw)^YAv|A zcl<0|pVN-!rQe8*P61$;KAzlS3x~D-@N0Cr-0qD%T={K+Gu^W2sm1Q3*2@mMA954m zjx3vj&{AG`Knb{}34)#XIc|P;kh`N8pmLEB9J9LsH`)J)cWNeX-dqL;p09##A1@rm zXVi334u9o|A@XUPH16Tf_KCUvQ0);!mV4#mgRk!hPh~E6IgLQd)#DJgHwL4hlo6*- ziWpq24fdx8`Fm``VZ0|7%lhK5BylIdw=slvht{LV*;n*x&lb2BZ~^ibtwF_&eKb5H znygHbMCp+Vl*snLMw3y#^xAmT6@5dRmg?h>eL7SmKcH3LCgO{;UodsyYpC&&VWQkZ zF>#+Nrgjz7i+x_qPdML+f(GxvZtnnje?FBvZ%E)Z5e3j6Xb0cIXdJm5L^V~Faj}XU ziT|v~Z}#oR7w?qWFE93^owp|bx%ZITyjVdFa(AWFCF`+Y;5>|f?xJ(_WtmTdM`40H zOSV|aF_YgDc(=5YEU2X@wV0(#%_~8by+?1%lw-seuHqPK!7v(N3vPn3SiYWn2WkYt zhR5#Mt591Dx-sKvY>FZn{2#YREi@~wK<9`zwES%&HZ{hR$`1>8`@=i%fWbGy6Q02Eo_->-Gt$2GqUmFPV zrOF6=6g_}18jeEOKOV+yilf*55j?!H11II~;(Xw3q<&-$O<%PY;%eW3|AJ=7i8u#& z#{}82CQVf9&BoP6bLfYYE>JNX&wIf2_#M}A+0NSn(rI)A%DLRiPo@&)$*tsf=z4P5 zOg)Bve}R9PdXl&!8Moc)M5dqt%#P%7p7ZmdzEuFVS>Z%f85I#?!`3G;M#-OYRxg4EI5`-sxB=o?V;Wr!|J~el5EOGXUJM5zyx%p zL4KfR@LCo@wYBMJ9qUae z5B~(&)C6$h-ZS!ZHbZUPF1YTZg$|k{#3gSl`C6KdIkXBd3sqz5m^+nril+EhhdUD< zMfDAD=xL?jXgzxd-Z-#~_^9eL&her!{z;S>YbhtC%K|`tiZ0_K76Nix1liyW9*TRY z;iV-67JYh74svW?)9X*sMoylUSZ%~i;U(awH#^By2YJ{vTmwO)^GQsGh$8iQ3g4^g|1>v^54 z)tSko6(swv1oRqyhW)Dh_!-%rxb{XU-4b*U1KMuUEgMAGuWN$AYH2lZ)G`I-r6+QI zf3EZQE|lmi>tT9827E1)Wc^hev3<8HT#>IJN&aTco>y+jU(`=e?H?is=6**DV8PNr z3*4U+LCESrKC`z9Q<4cpu3128x0!=3*NZ%DFUcD2@TB7v-^u-AKKv@pL04xZU>!_x z%9Q13nc|2(J-4v(fEIYG>7&)%AEfq_Gd?~Vh8Hhz*|%UIS-W5oJbifs%PaQ-DfWcE zAEHd%jXGZWj8K^4*TGAeorB9aF2=KKD^bd!0Mj$GiGJZ#vb^*hUs(MNIu~XWEB_VL zG++U=8(4t_TYxSFj+j<9!Z%fZ#QVOb3(~C3z@^8LH^J=`?s-#<7iP%9nW;^vAXkPi zTkev7@sV&~>@TssS%jA^RFm14EHJC7p3M20gU!ni<9(SRa?o9#4IDOPjE7I4WcVgn z<;-7J(ssSEY8}$@xkQ7YE1sYg}^Qd1nD@=%O%C-x+ba6`*ow-?o^l= zu;A_nX7Bg{F1zG#{6B)>Tz^u|5|r=l#O@>6D0_7cbWR(Aze9hyMI7vm@Irn@ zGrH^(Va`Y=(EMymqR%lp3%5toEhj|SU2(2-GcR;OPyBgM4(58DokAFDBumpb@5Z?N z)3}PBhg{!u(An8e9-Rt82i|3z*DwPoSZ1QxBpcZD<_BL^ejk1pEyD0z?sqPq0`-p~ zakhRJs=rppcM3t6sGm+tHXOl&t6t-qDlsAwBEcS47(^F!u2ZbTWyvE3aAN?U_xr5~ zE96#O`TF2}3_1~kEzDNlob9o&#JL(iTS~Gan-mD^`x3^Rgwfh^2vhZXl(v~-dg#%^>Q^@Pxt}hoS#bk++JWR&O*n>0Mz2xXUw)z=y)kc zt}UAY4f>N{ziByM%gZ6_jn?9mE5$r!ogvF@hl!t~HhWAZ2|7I9L;B3^_+TUs#{DBG zJt)nFylAG4+mne`#|dm0&gN^?4bz7*7hsn7CDgbZkL{tg*!5I~H8C*Q|GR|W2XZmI`!4EUZl*fAf*R(7HRrUjH=K%mO04+ZgjoE5SW4(Y04PQ_K=30o3vt#4Sd zrxp6$3b0Wcd5-c@%n!c>c);#0TFiO|vsXs}*mu&5kgxFfI_I|9)P(U?0{l7iBYaL+ ziGvcWa6Vs|h}{n%!up(><46RluF$|n&P{f`FAJHZ46ID;fhMn1Ons-oz7}>y{qJ{B z)%FK!nL1!zWCb0#6N-nlQ}NhebsGIn4{fHKU_|NAScrv7swinJ0J%U8{8{jCS&!ojk^LQ^C$pif+()Z{L7+k6&8$MqK#~>-jaoI9j{X82h zD^w64|K(j`?pxgCbVESJ(#^2b7o*$~=(2toAxqBP+tZu~V z6P1{gXB&yo>{uADGDEXt8q~JU04~^%^VnY|%&hFY=qVXRb)1{1NTnWpe)^n9I;7B{ zW0f>kU`J(ha87Mw_8x)qw!Ml4}Fsx^Z7N~>&EtAK- zTfd>I^*AwEoCV*L1mL?P3Ar^t_U)KYdPEdx@Y!2v*t#70wjM!)#1JBspp3h}PGV+v zkK#YN7v?|70;Pvr$w`&#;B)F2bjW2xQM3wvy~({J4kYq6DmPi8Vb&jIlF3IQ3&_5`;{Q8vtkF_`FqPd*g&Hoy?8&2$;qro{Z(qV=ODyq^rxU!` zdJO9%IQC*zC7r7pU@dv)8uF`*7*8`v=7vB9ZTO9-^XCjr>UG6EnV#5QC(qo}e*zC~ zbdg_&r_pDxuhT;ukADNbjFy`>z@Ef+L}*jXf@O_*>>jT=lBHPy%7v@2VudYl_fi(c ztdgKs*$WLzB$&}_UEFy^n7T$?0F^09WasUJ^GU?YA_Wx3`tUM#`_KLu{`y7Un&QK^8zV_ZQjVg@dWx=P;KnJ`nQR71q+By4IJ#{+TmsL3K# zHhSM_?%R41_Py6+qj$)#D@-TA^Rfh(ym&5o)YU>ad8@NIwpvW_Z*h!?GiJ6meB^m@ z`{>P0uc<|<0yC3i9d@n=g-O8~u(N{lT65zd@)PG(j(*PNM4wPG#~_k;vb%(d4Hm?djmPC}w=f|2F_h?~!PdgFSfbE^ZXLyV zyTuM|b-Q@`7i@!c32D4+ZvzUN9J@6^AHQpa;;LK9d<*^*vh7eHKIi)QbJYjw@FfR4 zwwb^;6*cB%N-y>AFM#2?tB|Cgeksf&qJt>+ba@rINv&6JTxZI@&!mJre1-}_4P~Wx_i^Y$@(Dn!Tb>b;pB|4FDNbe!vxE}K3 z{+B?R3^>N|S@tw*Gdf{^Q8c3qJGj}R$nGk%->*f5dg?*CJs);EO|OoYYCu{WOCq~! zQF6o)Pfg!XVlPNB4mFXuae*>>lKiEak2P2)hkW|>>oC=jl7||J6l=%(srWKUm-#kO z04Lv^M>ioAE(0r|a{JffM^ghP`|wkUEwn<1En&PL)5Mtc#iC3+Qn31U1Z^^WxWAVY zv90^g^V)q9ZtTsWxrMTze54M;f9fzp&OX)IDgT3yIgXelu?IZQg`mXF$6$Ew1Zr4T z&}vI*=5Kc*^lCN1#uvJ5CY^;27kcT*>*{#9^AHC(C8YTn8HSqx)3VK419b?JPkhkRJ?`zl`l}>a| zouM+v`>}RwBL4LiMZJ0>xU8*-lLc#^ zXfThOveD#~5~!|g=cE63ew0uWaV}dA8@XqG-Eu8ftNbl0MaE%hXbd>j>_k(Ibdodj zmA6ZJDR>%X5@UaPv=1(zEg@AHDxXSg3f5AMSGw@4_7)12I)Mw%gLlGnC#-1nLfg`6 z{`8Z(U|-7xJn>^5NZz}P#qB~kf5|c;dA1)vkJyep!@2(z z!$)qOFj^YNdF`GO1p%L@w(m8a@1X`g!dEf*w*XVDCBZ>G4{bXdd7|-!Xdar%Ke+uf z@-`?let-Mvy-NYmXmuO(WEGgUcj*v*q?6mR{X*;f*OamEr+&Im$tUAuaLFK-hUnA~ zYdNl`_=Avh_Fvx68F099@tl4Dg3GgIGVnH*x|$w6)xu^2|=A7S@1DbV7wrz@(D!TY5?Xfo3S z{3SQShK;?DxP3Og=*0C6{`u2O5z9dEmBk4g0%*;i$7G-@h}_3=$W5J2z(twOz2O8) z#Wk6X_yF{KDM35}V#%jVHn?pl087@|V{}#+|I(ji-1T-blsr>J{~lvlIQc0RQc@!F z+wKvgnBPzrTnSZgPr_%X>3HmJ6mP@Qzi46_f_7oiU^SS6s_%Yb#=RUk9W#VBZ+WXqX~a|u7^>#GW=tSysf-u5+MJQwwaZnxuOnh;@|>_yM~GG$zbc)m`g-6#(^l+ zCEx@ZCzzM?2L)q!eFFA%^P%t1DR}hf z8@fogVtt(kx4UHc!)L!jx`8FrnRpfM{SJW8eXnrU>)S9cA`AXMSj?F*z*{lF10F|4 zq5GdauzjJ+vH4>;7VH)}_0oL4@sJ#y6917NnVg69%0V>PiN&;^*CSP}XP+RHt7?CG%mpVb+I+rkAiMMvm1#yq$mPmJ~By zq(DY4l#%1y|EbR#E~~XW3T_Usfx;68`14@~piCW_XR0z|CsJsU@ocSAw7pFAqQ(r!G1Wv*2Nvf1%)}RFA$ViM zboOxj2;LmMh5g53;CHw;-VU0Khx!!QtT{^XYI+aWbSqJrvIe-;smC^kDzJyMzku#1 zRXiw}POps@;j<2Zbk>z1{l%eJEPoqEro=+w;$7r3H?R8aA;6x%2vRBa8057h@b;7n zdhpglDsK7!JEkt=+!ZImsdqB0+M>$nHU6NbPsXs}?`<4a2;xV}ora@X23W8-6JOTF zz-Vm@e*DLhy0ho_cDW&VOE3xdpR@)2o0$-H#gqTNw@JTA)Y$KbSGQ#Vp=mk%iX5SFoYb0B#)#LaPA}>!FAXctwWeyPw-c|Er6wHmID-Q|4w) zIqeGUh-M^wNtI`A>Ddw{wi|aU1=AO84X|g0Ga6=?Lvc_w=`|_C@4?%_W10b-`{ypT zAad*+Znl>Ht_B73jPTdfKzy){@8(whM%>$uo<#N5Ie7>gd1E1q1I%Fv|{G!LsF6{I5$@7{w7~ zyz#$nbk!zqPBDx)v+*_D!&gAaEUBaA6SPb4~5xb-IqMiTsL(5EzUNz`a=AF#kjiqA$gsl#r{!x%TrCe z!~g9O4%?!mVC`MR8)I$c!k%KV{L1j-N>)LSnI6thR%bSxEkdwm2)tzmS?R~)i|ei1J^rz53xG2XjCW- z3x`_pu-*v@XF2XbGA#j!7Kwqc^Q1|ZBc5jXzwl$nb(!U(NHVCoVHzc8FOrPp={+PY5 zw@|AbAACRM77mZ!BUKOdf#<&k*Ej}1>C+-GnNk6BFRbU)1&d)z>_yUHrA6k?p8>Ac zHfU-4k{4&c5ge4tz$2%cKK*7!cU+WY4s+*$$}ch1^?jU|BwUKAkq;wdoj&NkF&(W2 zRhhxVvP{h6A|jG4!VcuegM!~_x^H(h2^8_bhD zl*qjUe2B{Dr?fQUJ~%3Qk;R#_!PRvJ>a>LtgUbi`={GqhyQ2`3W&VTSSj#cYk6uD! z1vhM=xzx|yo4(?Dr*>TaBsuFhi0~V*Z`xxbCv6T3j~KHhU7^%sTPixeJ}0EbB65w9jIq z&SY4p)lH*>cGHXuFD^584gNRs1l1*$;h`gQsGr<;b;3VGsEHAw^74DgzGZW8gHSc3 z4UCh?^Hm5xr-`VB-N)NHr|6Ga<)|%s8J}(wV%&EOqtF3Ej?K_Qh7#@r?~Oeuv`wP) zf&yB-bOiZ~b>#8WY?$tE1PzY=fW9vW?-c^B?ji8IcAB&!E_tswtQWt9lpH-z3vr*#$C5<~b!rW3} zIP>+ab#(V5zNdaOMi-DBrRW`QI{XG>=K_itkOJ5j&w^wT;p`ovws9? z?u7G9V=b*x{<}?nzZGJ)i9VuZj+`f(>rx-qDj;c*O7NxNB^`er%{@yoaQHq0qIN3K z*ZGg?oGr#*Z-0Tly$Dk_qX6@{dEiNNeGCth1#Pnr9B=0e6ieLW1>Nle1y8OszU2a4 zvMZ8k6lT%I8V`w`$Pfg4PX#972&y_J(290x@J^hEo$E`u4pRlxo({$n#W!#s5n=Ol zlTlmUhvU3Afr0~{=eXzzi7xL)!Q~P^8?%Q&%TIpL{Pf3%Jd??49jPoZUQE5n=<8h!# zjcGe!!1TU*O9#e7(97T_@p~R*eOB}eEK5;B^Rg#AL-Pi#X?V%|s~iM9dBRLT$3v)= zjK{fV4@kVuRif~%k4$ZHz?8SIxvZHwEmh8dCKUaOwGoO!_My`RBm;-USZOEn%bd}L z^jG4zmjl|!4h`=;Fw~=Rm5ui5Ttv(py}cM{6Qrbo`RlS&7IDVIAT2w7kQ-D zSl!7*jf6D(JS9WWCL+atD7ap;=-nP0Ps2pPz0vnfRQqD6mgfs{^`vbMzRN&E)rBa1 z!Rq?_zAl1wrj{Wk%QAz2;VeCj0nTP|dvb3U>5eu1#le zCfonN$;eYzXI+0y7u4^0M(dli@K>G|%Ud48783>bowPT{s@jHyGZS#d;3c}?u@;Q9 zS)$1rXSWk*CS(_7|};p2UFi?ozDX(Tks@JLxXXUhIf7!$HoSK4!-~zt67XKImmT zT2J8a$W69hYH_Glv=F?L9@h-`eL=I!Pib||H8x;Y1DPx>TcfKg4HYkE<1w$ zwv1%*4A<^s`xa`l5+kQbT)T+P?8hl+e=XWZ_tZw*A!2iT-lv6_b#DTwy_^J>|B0}Z zA~&MzyiWA+YR1@b$LbuXWB4Gt0@uydW;d27umflEIT%tG296oQjDL6VbXY%Dr$5F< zKUMbfoH3lgYPBuTT9Wm-E5tmH=s}-nLbUzPY@5(rDRzyX8*^b^7#g`+fYF$VVDaW+ zT9(sJzs^3-yILW`ZWN6bWX;=+H{73~?ys2;As>k{|0UyWV_=8BKH$$vv!qh`|FP>n z=&+qbnKglC8F(ak7c1MHYh$Ong}v=qiONeWan9RjR9lvecWqQ^=8UYz&wnDQlEp;q z3f5ucV{hW*54&s@Y5T(FQSKi6pC{k8tN}y2-E9*=5}Ye6NU>Z8INf^#dF9~*J;Q-- zU4{m!6g-YJVa!xXwnRA&k7_1hMnMf~<;-FqC!WQlY%^GW7-WhD#n_je$K6|Axu*MY zAg^vZ%POt-%ieO&VoFuaSRNvLZ6)wm6Y*QH8$6yI}qs8GL(v z0R(>fkFEOf3O{d6Vneq6!4pCTf}G4i>o3B|*dMK7V=#7!Klz{~zS|{H6YrGEu2)#g znw^;~C=Hs#E?*Hw=9XmOe5;wZ5i3vG?xt=o-#;AQ{hd5n1&Ws>DO(6 ze@-6OHcPO23yiJlCM(`(uq6riDxKD%haQ@p1wPE595Vj16kB*?|wHv3^Ye){6XW(|4^WJf>FpQA9(=GLFkT50=h z0>^uc1*)^xpzAQ_Yv!+^Wt@{{%7ZTEXw);9=JlE8OXTxvVy|OwLM`g_?!;L)7ZZof zse()2Vxg(vrp>sgFLm%&B_dj%vDP%g#zjGked~D>CsnM#P{+~*HO}@^y+b^aKkPCn4MCq?K8;>~R=sib}UG2_s0j8ibTZupfI7PCVxy7;?o67>HgDq?z2BFkE8mMT3+to$Gn2{c#h2J^ zl51I~J0XJKJWUK-zMoj;&!pzE={2`p#ciBt%&qC*oHwNf0X9{e$9TQoVz!e^DW><@ zvFj6^ZN!F*Y?Qi;*sb+yWY5-I_O)*$`!Tne4lYze(U*02e%JJxN7EiLd&4KPeS1^{ z2O^}{m(?xAM|Pe-G-?HF9`e>YbDYa1F%x5UG<_bq zuv-yMetpAwGNSB3uj6#5-c2lc5khn%_|#ubh<&u1hiPZC@bz|XwmHL^oj5s%oiwP0 z;~d$NHB-Pa=M`}5W;ys-{o%dlo#N(ye(1b@6p!v-MelN%#81^JzNWhG#5YU`+oG@33A!rs&0!!vF4a_quQN(|9#LT`a?dZQt}`ma*d}Yj=reIv!v8$qBdrqk8k1;1w-}lzmvb2f6RLK;lB(^hC+E1i z#uKg=ruq39?T|7hJdZwzF394oQCJW2CjP)u+6o>r1u)k_8%86hK~HWm>3ocItKv^o zc;ru1J>JuU&5{hZOs8+huVX_Vw?CP57bZ4W&<&T*dcwXI zk4*|CYUiHAnw_uFHuwke7BQwJ_kIyUloXzE8{zK>>cWe0PSsW`t+Fs@u@+1Jp1DBKDi6QuBQ!4N+ieWc)YQ0y_FbzcZ8x!)lL6B&p_^QKsUT>hGQuepi^)TP^uMKnPXU$w2(Af_(1j{7aDzb z7&8hN&`6I&Vq~O(e&SE~Kg4F??y!sCFZU0dWj6DBeS~S@)GpLHqlRwD_Iz{MLtv*k zUb#<3h#7uuNROV`fb(s&xqGb?gsvW^3+mR>a>c*&#H(f^dnN$l-rIq9*H;uuGUqjK z(S`R$N9jr9d|33X7-xRZz&Tfc^7({g>N7{^S;wnT&^tz^JGS#sCknU!I!@QV)nm34 z$T99(85kWrK-_i|Vj24t6>k?24f!|JFLj)Eb-e`NLtx6?Z-%UMx!ugMofGl0-#;+k z#(BbzYBEZ^Ti`R}J57U8+7{vqVyl%gPeBXMzB1z_rZ$pAKcuL)NDU6pvH_3h3Dm>u zI_#sRcuwX$+{jjCXFStlZATj5_uf!AqWqn@n@t9xO&Zuc-w00blVO~L_j0*i0m+HD zhNZta=gg;YK2N^`54mQ;t-Y5P9DB#TgM%8dF5`{$U&*f|<&p^;alMbNZkxc=)Qgx$ zDAIzR!?fo`BR_FlF_#lkgrx_BnJ@eX`tbZ8e(;Y+{4S>ol62-1zr1TA>fbHFO54-W zwh+15Nf_okm7<LpYo%2V4Jg?!W<0h+d^i7CGgUB~N~H%*zD$`}PUdl$yl&MO#(} z_@{wktt6AvvmLstFJtt(S&*StgpJCQWW1_?4vvU2HGLWw@27?h(&E_mxs9Cd_`x?{ zVa)O1xsY+{V}9qgXyErB;rIz}Xx55qe15teZ=b$stAnC(ltr zsE?afZ-L3i=b_@D3nf*m?A8G}oOVZ*ta2TPQx{cO*|ZkQT>D#9p38Nx4?c$#l})r~ zKFjmGGmYlkKgMuhaUZp z|MD^qcZx<}bjkp=6&>XX89s)yl@xbW+`viGOhBqXhTnDd8rCTmRBws&hF8IAtoYwH z^7r~`+R@oZUIa#i=us9-%u8UYg)n2aP7xoRHvkiU5N!{NM%`CKm>z)0WO7`>>-E*? zNA_|Vs$z&fDMoz5Rv=HO0GX0;;w|=xxTwv;otG|i{I+6RbtepL)9b5Q+1u!x5J2u0 zmyu$_5)!bq9Yb4GaQU5ajQ0&gTc-@9i;^IZtw+b=Oe*p?3UBlYG2IXMkpuqUVRgk6 zV*hb265(y28rI7%)XBj6QG+C|Qj%#ceb0Ak?grbrvzcM_o%GeP9)2%7&Ci-JgGm`q zB{atzPKumCmy*ESMc1s@=j}_uDxebOP2SP=wcWUNVHxUO&BjF!i}Cg5hg7oRJk@Pn zPSS^3xqHKB>{66R_eLw>uiE z0*kO4xWMHpjJ<5dgh44bk=r#y0QL=z$DKq=mt9teBB@5Hg@0dn0&fw7va2K(|R;O+BYznce4 zubzDEN#pfDu*aur4?Pgm&ML38O(fFf^U*% z0Ns;{Lb*kF;nO^3OO^`A*`EN()<1lAnIWp#w*+_km@%hXN?-v!O}yPF;T-e5sJpQO z|96*1<2YvC)%UTuEmw*8B(sE`i(@F;#J$r=AT6KzfU1{CvORNOVBPcT1?^j;nM98f z{ws&*1y}u)kQp*!{q|*7hjMdG_ofzHgMgiB{X$2WiKXvgl~ zM0=kpHg2w`Mh;v)LU@EOix==s-}FF>mHN zliqAM*O{-wJxgT4d}S{k$`O#er~T-Wr4CfuR01yhU&OEmNv7UtGGE!_0G>%S=kE#s z3o)5)7gQ3eqpt zx7A(!v2|hs= z>N(b7a0M33%>#wpSbAu&3>jFg%q|MO0A4-mWW$d$@Rn|b=q+{hW_JT!yxR#xY-{;( zU9ohx!9HH^<|=rw;2qyBQxSzvg<20U3C63|$@H$>2eb=W1C6=k{IXpI)X!L~dbhC* z>tvM-ODBh6%is@^H*Cb{=*Yscv!yscX@nTwkjDi9DY*Ac7$LumS&M8P`eK?i{1+|G zx;B}R@Co^x2Q!%NUF3xqou-nKw04wVb{m7|Yv98+BkQY2)bMj$Big86C)Q3~7`8nM zj;&maYE2(e@7_=H`dlQ>lw+s-t(9kMTx*e@U`hJKPU5}uBB(Tc#F=6JT=(F}WpQ3&R1|6+mu5$eFVG3M)v)4gIvg#1Pq%rxVPN$;=n9%m zJGw-%WmW|L{dq5vW71EqCj`-iu4nMAX#xFm*Amy0dvyK3<79UB5j2$)#cX&k%yz)4|=+*U_HqE!?^=5&k)C$3Evqy2ntM zt@aUOL&byP>+@RDKllYr_pgDaD;{9NKPC2sE}?DO-1$FFkdo*sE4+XWLf2msq6{kGNKrwLWOuQ7w~q4 zquIv!FfjQKc5BYY2P%(^Y1{1<#U>;nSypQdZ?{4ffty37Z(_v;`rCT zWPfoacnBof>Xi#|sNDcHIx1;b#XVBHB%gRVm_kT;U2BuBG=)BW zoGExW8pkMX*CP$lQ3Oj>F+aPB(Nr3PaKZo3r(+-HIwjB)mzOa7^fUI4X29k&F^-&FR-wf!(2LTgS+1!!UKoLsD`VH;7fi!&uwlKe$ObPu5V_r`Ng96NK%x&5c3?D z)NVzmZ4c7b31WZw= z+51S6P2ZXUZ;S_-cTfF@*;XA)?MR1@G2bb3-Ua=(tMEO``>Ia~%W{kmOV+Sq0{fO@ zyE^BK!!6BvzOnq#>WhC~kiDXN1YRY+bnX>lcJ5PELGMwPbbtJX&n%G*Yun0;+uzBy zl$VhBu|??cPlm*ot|3qFOk@>P{$RrIzo^tk>8aIfw!Mc2sNAT&K$+u^He8g%0C82a zrItJY7jC0zN;gqGq8cOra2YM3c=UPwgX?FQ5!`VQ*X|Rcmkx@uIlF|hUr~*9?0!&f z6E4Tfop525qRzqT$II!}ob_;8EQ2*Jie&A~B2hkl0xPmo8|TcaWEY(mkfLZAoAODI zu;L&0EG<38{8@1VrO)Kzq}le^Ik5{XcJtVSlBTSsxiy>8RffANi*Rb*Cpw?=U|qc> z!>-!VhMSfy7SxO#}~H8SfOoilXzdzw#N4|yI;r-rkrh}vfnjO_eTh}ot$Mm zeWo*3|C3y%@iw+qjy{jea+6= zc<(9UHJHs{*J_4T`zxx`{0;52+Uglv>?lIXbER~E>qOf8zJ)^qMsRLD!$w>Tt^TI6 zwML155JuifvTCMB*qBXS)bvFnvpIz8NlhFSxU7C)BfdznrsnlI#zLiwQD1b0-hX$I zca*<{IsR)1+Bb2qA&n_D+MXHco8-g#AGlESSo#4?=FYcX5^2o7Nv^g}Y<<}&wH)KO zeT1}+2@%VgEqJdl6B%-m9Gw@#`W(+eMN12|E#wzH`0X3>Q;>p_UIpWX@6t6X1y!W} z!T{~-o-4=~Imx-D9jTVeMAnmQDOOmRQU7q(o(62be}n3k_=6)vV7x^&`^Wt{ z&Mq0{ORTKIlvEj3e$r$%&tH@Mvu_^@|64^OPCcYaT@(jD3E`Fv?d*nf&J)GGBhs6k z(RoyueQ-SxgwD1x^JjY3_`Pn&QfW&YzK|liYby)B8`+Ydxuwi;{UaD;JeQpu^;hu4 zbfWF)%ij>qOa#JW^8~hKd$^r+4ZEUzoM6*cw44$zIH7x!_s!`n$3VzJ9T!=4|9(4x zj&~uOXE2ei`LPKj+rJ4ezKWviuTZXe;>zPiOyZmwywNK;V9NF8A573@>6vbt z6rjlJfFC_SoX2R6))JQwAw0>QF4!ZHj3j+FYEPv$^;Ns+!5^ty|JRtEVedzC1!;^= z{8V)0vX=W>b=bu+BJ3A;MRs=1Dp+_^lpQFkr!h|@v1UgITd+kA7SGqC)nS&_JGOh{ zY$Yc_FHs?z(?c1@Uw!yt-3NM4ve>3;qNq*s$7Q&7$tIlhVqMKQ-z>cOBawe1<}(%t znjkZFg?tNFXVola@ZQ$t0x@58Y`Ry-AKj|L%G?OeB#w%t}`L05o2bC%jhi*lLE zrv`LhgcB6*Uyn~-Z^GF4dVbZq6yDF5*Qklau`ce7}&mOx6;#3V9108#U?unWGr<;TzeUC@D}&d58zJT*!P~9ae>N*nEvp z;m!e-#NKBe){P3|zs#e!TT??|wR8j9<+g^^ca30b9F5R0*o%!mJ4%=BQUm_Fz4&wC zIh0Vl$jK^vN>T*(W(y4wv)+ULXn(z985gC(Dy zeBpZ5-t3h6Vs>V@kVl)m|3RwH+L3gg!F$>#9sNNBn_!ia2V(JbGu zW*kyTldTjT=`FVD3wT9V_p71dG!s1NR*tb{`NY!-n7&&-kBR4k; zlSh`@>i$k5e`XgD*Rc-*TWeq?j;@+n7z2;6M!r#n_f+QCQw< zVe`{Ml1*UFV2FZ0dv|mmJ{a7^<-C;8Zg?AdHT|Qx!NGzDiR~ z5Kl1XgOcy=g7UE)m=RHq(?TBM-U+=Raq%WxFw3oy`g1e4t=OH z6mRcCamGJPlHGVWnk4paq53~lP;v1HJ(R`rPOcV*+f(!z+vZ>_|Mr4D`(Fk4y*WnB zXM_VQ-;AvZoMRz%AuOF$53Maa5NIq4x%&r*m*G^bj5>{SVVcY*mnU#0XppSBJ_AFK zOOSK@OK>Dh7BUyFfq7Cd;mM|A;@D+IFDx)%#*Z~w&j>q$|GTEnxpn+u*8Iz4?zsl) zm03l6bbo`;c0Ybf*BV&8b1T&JWhYx;PTve z8c##RqGT#u_8M~av+3x>6Er_o7A^ZjA?mgy<2h@HZ=3Uw#2r_LvO80pW%jaVJz}Z<~oG(bo!E;)K_5{I11;{LuZbYa=41IlfeZlDgjBgRaNf)dvuTvrw9vx7w4 znh8=*IM#LZU66Begqh>|Oy6P++9Z<+ zcm#r@N6 z!BECMd#`nF!4@$CMl0bVZaXT+ILh?U*+2Y2Vv`cM3T0qm-9rdeDujvkT(9QC9p0xEisX1WliUs75YaBE;=;P+fQ2aBm7?o5- z(OK;j$#Q;54PweMaVy6G8a8C!xR*heTqBux_&>}!5`cjL&7|Dr0^rLV*m=Z=8T_fr z2x%Vx6}JeYW>AC23_HlyxwWVfBTJHBjL?ZX>)_{%h3GQl9!`+8g6zJ9){ADP!ZWkC z=r_-r_i&dL`n85vUoNHA@6QrkcOskg4JU(cye%)dK9e>GdxNNGHeT=iNdYxUQF3i69~|5NM|zI0MI#-AcC?(#v<=FyBLpnF%66sberQJpq*!E+mz^S3u{m2J6?hkRPqcF?R>bFx90Hmwx>} ziq89=tM`rLW)UJIGpmeBDh=a(Ux&u0XwlXrRMOH`DtlyPW~PvgvMMRw_jSMDrBEa- zq&;bfN}=U@zJGupcsS?W_jSEq&*yenBG7=Fv$kV-{!PJhc7S}#zE6_($Z|o052@pf z6~yOAI_^sN2XV8!$)x=EJn>iw1aDNC#MWNn`Tyoqh|LzXzf?g%#c%j?>lfH)?82oL zj1G4H!Us$D;_i`Z!nflx=pXkS*to=q{Yz2gygUV9JT3@st4YI!l0L{fBEnnlBe?5= zE1-46xpN~>j!=TWt2%97m z1r8M^LjM1i3vCa@w(PB>{58Y#9^;Ut4~OKI4A9g4Nava-VY%@-^s-ZdF(p0pdaExH zTV)9RrNwCTybA9$oe@8JqRsp-<_X-QuiHz28J^yJ5vy%qjaA%Q>{pH&`MXIhm zE6YsHfn3*QCp=HtPkdr)Z)m<6s($pMQGgXlg(=xzo*2k3{`O+onHl7I_!qjQ?mX*U zat)@tZ579QIil=9j{TaBFww^6&g@g(VV2i&3ImRm2;9`hJFGo2jCndIki5WHJKdFu zbpId~rg!yX@}el(`s+5X?J{9@K5uZcb0aPZ`2ovij!|x^5^=b+n~`@F9__qylG%R)-{svk#Q(v^9QXf zT8Zi^FEpKS4)sU>qVJ>9>6=pypuTi3OB-Sioflr%FUguqcS-BxGw%!bMjJ-5va8)F z&!X6?ZM`s7PS+tU_9qFFC`Y9lp2sCuLSLOB&^>6yEJE|?ue*sbRMFLbLP8!r(K}Mq zYujRHlV(9GFSXDTRUwY){%t4tPmwJ%Hy|B#VfdzOHd`oICH%Q+J9#aNfMtHC=$|9I z&}xc5nm8{fMH}woIFpfV$7eIv^ma0n%{zv=b9tW7xHi$R3u~CxZ!O`XvrZWB{2cjL zX)E6TUkZNw`i2unk7wT=#G~}68u8?_wfO#XVAZ#bhvbD>GX`dfL`*9aC!W?~dyU88 zriSxa_E{MY+WWEZOZgpqX+510HcT`&Yz?j)^+}|{40x8nL~(|-j#&TC33kY3Jk#i>~B-#ATg{cG3@ziE4Jfp#vWmb486e7`MlRYF*rFAL+4MbTZKj zVsxsAr1`^`z;R_T3I`Gd_|}@eAMM7LC2(XbdN$&F;kygX3m zj0RKT^PQ?~BQVFY3YA9uz@NWO(dwcI-cxV3`9La)hBsmV2}x-4JB*7G6ljp*H<)N_ zz!vu0=G1rnMJiGeE!FqM!?X8s#&$9Gi`{Zjg)?^e&zsL&hRqa52YA3GlWNhaH5Z7# zX&jS_0@2JivFO%-6q5|<;$m#w?Bf~V1ua`FIujJgj4KML$B+j5sCEUME$qbFPs%tL zKM^eLWksgr)-lH%zJEWa9s?q8lA&8wSy6r;<|K+mH$L5`f7VaNIVVm!c)xMO^w($X z2Yw~9pUoTbqE;$T-IhdROUsC(lQ z(I&HNf?0{1>_7CEh`nXLa}NZ0qLc%+<&G5n>0rMJTl_4XNy;W+;F2gt@|VH=8e81id<*~0&48l@Sx{%CDO%dsiGlCu z34agY2gQ2x;oIoVIDhR>wv(T`RAyFyKKBHzrtuww)MC-twK{B&?{zvK_=#pubFjv3 z0=i9~4wFV3V~#@+OkZ?eR96M|TaH&c_&#YCUaQq*vw7~-jm1xyTc;Ino^lmF+TA5@ zmgKV^b0k>hS~+I1=Xv$G?|GneF42Ai-EB_-vsrWdUN$^zw7AEJ_jxQo&kpR|f)BEJ zzs9TAIApX0Q@nhYDonq~M({J>OUEt?W_uTi?$;F2+B|)9Ki!TSwbn6-BVpv5)pE3$ zbIg9|#7u#CV-A=L|KR#)%Bt^JiU-_$L0LnW?dg%=rcX@~ZSAZRty-RI-~VU51WB-na@ujaoNG9v#8X zjD6!Ujg1fu1~=ov)K{V$|1;psvo}7dj&WGGd{adu3k&LPagqhk&N=7BMvQomsdL1l-|J&oqPZTveK-p;k6s~T=3anNYGXzIwN8+# zbAUCqycdnKRA#?|W`oU4Q!Lh*#j4Z4klq1VR(7|XJU8C}+m>m7W6J~*EGdOGgN}Hh zHjYO1<+Ia!0@<92v#_$&oJnv8K=4x)rH1c^SDn^)av%=mDtfDJBl-KP{6(PporF-)6UR)9gDzlhfh#yTp^3IE6wy(`WO*OzOKQB! z6)!rJ;3I2!$hxf#*VL0>>aljHSGNTp;=91WXj=U>g8#5Dvz+m%Ooc+NF z-1clnrfJAMu-t(?dwbyE{6nx$k>^Xjyb99fX~AiHXl_Za z>T+1V=sC4nnv6rYJfOJ~#Mn7v1U}yVh0NhpalV@b4H#Qa*T)#+jD;pFL+%3f*$%+A z#r8Zydk0-V|2?iU?GUKH*hxkV=yRQ$gZXYQ;t?l1@_zPR9R4sIzUo!c6`D6NM67`L z`7by&I}pdWRXD9`0dDMzqAwk%!>^=9QaE}d3Ij56$f=C#8480Q^Enf)Im-a|2Q!fk=(hL><-g)#flJb+y{TJ6eP ze!=0zlAzI7fpe$JaO$xq;KYor_;_6_%?@}iG<+K^3~iYUw^p|Cd^a5o`ru)g((8kv z9m_Gs>M;#*Pqre z!@S}O{Gw{Wl~;&Jc7rMxcW^9B-){v`Cj+5Us}e((AB1J!V`$Iskz`3|0o_*q5UzUm z+ex}j!+$)3sq^q7vWU;Y>HAgE#tGBmLwG1;YdsX^w`PHgj4?(oIRt7g={R-xo$7@a zBjI$60(aDm@193o6$a{f3v&2f%iy?&c25?Mq_lhiWW>udH8VZI@49}0jei3cb}XXe z-@2@D&2{X2@qubKcMFaL=8%0likQEg-#yP$=K9a5;5!|CZo1TP7HmQx{I?nYcq-5B z{TanS3p`7d&ZqUE9pnekT$*I&NACDI!Ig@1aC1wc-ranU&guYYRX(E`tx1^f9*VM{UK>sb!XFW46WBb+Xpg*S&4OO)A>J(Y+^jHx)%kgRRzMfWu2$wI10YqJxnqgMLieT;j`vC{5DmQtv9#>OPLt=Ig~)0FijZu?l!K^ z(ja3a7QG{xzQ=?bB|E;B#%{ZwAfn>Ay;T8gk7YojA@W zg>*X(g`<;%eI-6{N6Swr2`yy0cj6rn_FEN1f=uVxn)Y?b^*pO`)dLCi;J z(|l|fyFCLAosnaA@@3i82fxXzF^#COM;5%2V{lh@0#0931Gzl6;N$0N6yKTw+UtK{ zK<*ne^%H@H1bSX!cJ+r}|B_+(MSyzabJf z?)`wPEFJONgCZOl8*;U6WHvm%tAuZC9f)-8BFHz&5!B7Cp^8PhnBNu!7hBJf#)wtm zvUe<6h)!YFKi>0UQYzFHiy=LuS-9~I&#E22hrBQy&P}~S`1|uoGUqJMNwQVNtQ$(0 zd@2gQrBUsPFsr89?Zfg%b!u#YdQ2_nI*{qDabSQ0*h~>!AFyyDbMs0`pNzx{&9{}Ir%0D z+!N39e&PiM+hp;v?OobE_bj~hZo{VjJow&HP1ep2!hdv7;M@I2C^mB>H-z)>Mr#D= zAH4(iSQmq{))?;YZXG^vm`5xd;bfI8D*p1KOV7)3(u=yNca;hR@O+$U zd}gOVbO={1e;Ri2=gWY@3Pe6@hOKy?Bs$cpva|i#oO6^ejN$(#WWOJtwlcv_x;<1y zcP7yeq4-%`wv5|*e{U}}9eIPFhEU%$tKlK2Z~A$ec;t5bo^I3vfMES2JN9gMh186#lN zGZk7sr%LeJQJobXn@ana?Ze1fTR?*H{i`ttu<)N2&g@u0cNNKTHhMvzGio6^4fIl< zNnV22X?$-lW;fB=HxG1eN>HNOk;Y#>3yLep!@tj`fdm%NjiP9*H&e&c8lNCECz1L; z`$oMQx1!A<{#)Ks2v@>+KYK?N?iTaC6;20^SIgp@A^A`~wq789|C)A6T|?Jvm&w|a zKlG)%lknxCdF1Z77MOJ-9Z&h41MyBNF8|bJh%CQL7tX4~!-*-V^&uHX@~@4<)R4=t z2}7-5Y4~1hfJgP+;hVuE9KY`st}JV$aUIIsi(XU0d!OuU1-R|kN$~X1K&b_R zIJ_zpckbSazx$IRZqYqDIok!lMDxt2#!3uSr2PIg5p3>!!`SZ)=$t9bj4lsSmG%fh zc*R?Ec_YDel1@PW9Nu%|?1WuqbKr4aCEn7Ep$B%C(a)pDa0_i}p!^IYD`U^|yO0th zcd?iZkM4s=IV*g`_X-M6jDf-j_E6#x0Dmlh)A(E^+!p3P$#0) zb>pb*_+YRJGQ`-$HMnN#SuCG-5h8o0!-G-D#F#bFr=#7WKAGogKDtTe^{--vAOX@- zH(=W3XVss2Z}8cc+2|T6#avUvN%Q_f(7L7vnHGOg>R}Xc8HqUZOan|`C5dZY^{|D1 zZ?xah;r71thG``c{NC#fb$T6C9pkYWD)@}=f-ogc`I|2eiz|Yj=59Nqd5!S$tOK-U zM}nNfIRK8s+|xX}So#TFX#EB@9-WOoJM&2X=_=kmcN~tjNf7-puJnR19IWON8ZQ|P zaeiwtOs$U2^vMH#nRYs!D}dHVoAIUd7%n(z8m>9&kL#|V#K)7z;aImw{m>*( z?K=f7_mTxR9ak_}DijTa#lpa(o!GjHgH6fTDVyhvYsh~%YvT}bI(M*o+srA@^`8>n z+GxVf`P2Xkb@NbNH44hQ{?J3`Ix*+vWgHk9h=*<);!Bev!K6|0P+naqkh{K*JP7_K zPlHT^uag7n;xB`!mELcM4^ znlHHln;Y+uzkClU;)pa$DUXB4fuoq^3;}kCO{v;KOM!#&XS=6y8aO527p@=YdHb35 z^rh`3&}acnXj6n?eGj1_lJ}HL6~VOrlfu$5M%;zh56HBqN=&)Cfd9G6n7}_D*NtL< zxf^p8Ed^-jFN;(DO(037b>Ka1CSCAIC5$>HjhZb%(fCL-NVo5F&6%Dl4x?yNieHgg}RIROipJZ zjZZTHjp2-4J~sOhOZNEvm2b{Zwp z;a_i%8*R-rMt=hg;pa#D+X~^JvLagazr}@HZ{vpDLGbE*0-ov-W7B~TlXAdF?}D3SL_g0pS}VsZF0%P0aXl6It1RI6$BFHr)^cf?gaJ1X6h1? z30s~lBK3EoQ5x*^^lr*x*ce#2@-oW=?z9f=ue^2AQRVFw=lkbcV+^2)3hcRR0 zGu)E<8FyNqM2Db85-`31zxLJH_dxG@3mW%);vP6>zMK&)#k>h7uQE<2ISkkA`^A zU*qLa_U|dw^@+w^i}*Uh&@tqy))=a2qKEnZ3!tobC?1_B%~U3Y;*aKCw14qG!PQZ- z;pFZ2v}uC|9yZFPp|R6(MZ_+cE)&gXB!cnqa(!frZooU9#eR6p58C&D&lR21AwIo{ zxWCWCZf5Z~94D~~JO1jkYy0Qoj(P(q@O%nhdb>dX*F|tW6pQ6g!XWQZ7daa}4yDqY zA+2!^DBSsg67qNH-xbYh;k6F`_->`cgNxx|KsMbwz7*3QcM`8tFYtV>Jp60XZ;VBSJTxOlZk7a8qIgz$=@+s zX|bm~J8e1>yMr5XQllljYKkG-!a~78Mjv+FzDbNS9)aWWsjxdY34$(aVYy3=4Kr>jaFO;t!k0@e(39_kJ}Vmq_uM=%_meSscDo1e_dK+d^V27Ka(Q5t z&7XftH-h_rKZH3;{^H9uZK&zyOnXMh(Dre9+&jKcxY$V&yJdjruCTzhrW$ay@-NBe zXLg1*1|<9Bdfa+u4lX?K9yjH@AqRT=>4(gpBx_R=T@sKC`G4iv@Z2^*e6%@H-Z%l? zjHNKeqgA+c!!p>arNKt;6ygy*Hw?2+!XNMQ>3aVd!NcuQuzGC@M*q8pmN8ZMvT`U9 zNPncAJ1U^#gE!y|L=R~oo4=!UNf!?^pa#CU1dH3;wyMtw^mu6bxnA53qix9`+c zM|}MU22KrROr99?IKHEC{|$U?X%}o;8Aa8kVP>uOszalA*Fi@d9bk5-Ii2q=57nWQG{@6#OTW>R!cFLKKM@{THxkqE zd-QIh2FZJwLRJNP!;iNnBG9PDX1jnT)B zfWtf+AV;N9b*>Z}qBjV&E}e8-+CR+PcA1t%4CQLVcG7Su4s6a!;lr_`Ah;nD#07d( z^)%nZc@c_{O}vZTQH=Ly-Vw%)JBF4f(zr*1_Xb`+hN{vN$oeO@aP^ip^vKD@L#`uf z#rpNcev3O+7&k#f_B#Fs8%+L7eTz=pV+E0X9&5%wK38D$fy(pFoC6*Ez%M=(jGq)@ zUBMlkUldB-&B~-V*S-eF>6YB{l>^kIas*WCOvfnuLSgNr8Mbs!8@|7|5#@)Jm*HIE3m3txyc8O%l?|Kfd`U%`HHbaS z&^SefvuO|^#|_~$jCGK4AqC73f?S~z@6EOfl9rl;3L)1DRq zihY*}G+*T4`66?Oaxf%SU52PQj`yC7vB&uO57nCXTT#y;gmmr8puNf0L{*!|W4HZU z*6`&w*IeZ!JX?DSk53#aa@xW7Q1u+xvNic^<)UkB@Xafh|I+{``@R=76}{y0=Ss7H zQeAS!0a0#fu*2#blh}mv7~Hy5j3(cI;#G@R{5!~!xd#^5i!H55-^3rRe`dbD(orRc zHodXBE(ApwK2l7*t|8zYC#bU ze@m}(RieOfDSnQu46)+ArPg^(3zVwJ5iF}5})5GxO-vaz# zT7e-=vaI0uO;YR~Orp{%2yVGWf>X0Zo8H~$-z&?gYlAGy>A8gMNkidczzuR;o1*H+ zOK?gmo(js!utYBcePMEU@t!&C zZmRb3^mLdW=VTwzDaG{9i*c5$49h5gg5$l6ar{;@POy<@D#af_i?TjiRdkgs>fVX| zrwX|@(PC8C7DBIRtJ(Li96*n$B_ge{6*P86IJ)mLvwu^cfV)&*W3A^=YCS@kmFSnF z=YlQ_ouSF<;TajMx`CICvT5R5dFHp!hw1)VfWG2Xdm~{n_BqXlr71?t8;T}ml-cJ+bTOqR{Oqfv23r>%;Ab@i^uFg24`C@j+EK#x z`d;NT3ofk5Zw8KB(93Rrj>VoeT6TFm9L42(0{IMPAOR^;hw#M?pmRo&GiuUda}s&S z-kd9Vu%}#5_j8uLsMf QTTyTBF6p+*VQjE(bE{StZkbt|B_@yU`&f;Q>z4@udZ$ z&k35{UJ#Aif4JmH9Px-sp^wC+C}UzGynM@!tv%a9h4*sN;p%R*Tq?zurw1~Vu_5fF zeG@+Xdjd-M)Gsz6Nm+p~K`Os|6|gM=0qIVz(Wxu<$p9BCTD@n3#E1WS2bBA;wh=e!R`K z&+j{jQc2Nho#iI}8xV_=ze&UFV+>1`3UQf|v&g6M4(r%F84RYL7e+lhDr#CU$u`># zW8%dw+?`4c}I1rkO(`t{1pFAsO=Z8;bC-ctYd+AgH$EuiU&T+QD+lo0^ zHshm+EcQD3FxUsQ+c!pCqe}%g%(7iwBWCE_EaPs zc|fX)ZNKC3KxNUAe+Thz+&DD*Hwu)GjJvA$z#K~R_R~t9?`1hN9J5P0@j_C#DE@3Z zdZo5fi-}rPX5UJYpUg9qdt?u`hwtLM^gR$!kjPdYR%JYvn6)dN7tSKxRQgIPGkxzw zm+zYZ59b-O_vs&ScH$M0UGWU0h6*s}l00ip?xyx;2OYvR-jTD*5QF^k|Hhr3PJ9X&7#IeL)J^5 z<5ARSZSULgKc2TYuQrI;pUI^8N|U)`V=M8Kmo1EL@Fzdi_JM|VsmNDLi4~}tW2>qr zG(GO)&tgMyO!IU)bBPzOO41YD;_tXaG$RBH{?4TIzj(}Y=9xTi&xvj*`q&%C$?%S+ zdn7V$HJrB)GVAdN1aZv?B+EGsALm!&kX32uKk*;#7#dT}x+-vUk+MkhRE|hJqL?kz znaO-s&gQOrSPEm~6GRuW1V@#Qz}qTuWP$BjJiaWP%A8PU&fX7bo@fW&*maF_O!r1f z&vv@IPZ{;%C$fr1Wp)#NPLQ$Q4I;PEL$Io64EB{ov#!T+%=G#T&Uln5_e&~;ne^?)7V;+usJpgONxs3&HitSn4o0)9&D5 zMP~Z&8b+lJrsk$u#zes*$7X~ zf76iV!@1;f>)(rd9N)@u~4BI|7qao6Hh_3@Flz`UMgH<&HEddPQv8& z+3-pt0sHIU2uDX+!hW0cAf;bQo;7~JN&h9&1$VUBu32(yqYKZrTk{Ct+sHG6gl4qT z`a{wVj%4}kt#GI90sQQI7qtAMNRjRmysAGB@;2q+E~{YLXO>KSvd&WRSR-i5rnuzn z6rs!bU0^O@4YEo>L_DO{?xpEJVmh=B!#C7p^Y9$Orh;^EpTVC6l?zF=vozD1EXVbi z%YfQzAN~NL$WEzD0Wy6m&EwBtMlvQKwP6vTuc(Hj+cM$&@(+Luw7D?bpLnlwHCoMV zfVxegboIGDVgB0$@@Il2KCTO*^PMJ>;W=G2=#Mf^{5G1&_{71Gu{vmZ(g@-e!a(d2 zBgoj=0p)+?IV;^L81ksFdhtdnc62De_uD@bT(+zOy_8u(@j*Sd?TiyjXE^gQdKE7pk}J;=mx`D-9+3uJSI)yJQXlA4BQSkJ+f)d|r?dX$+2K zs^qec8i#p0Z0Pu@nEU43G~&#Ru^`U#1t=#L2L&J=!qSa6I{N_Y6c$S3}QR zL)qdfsnqwmGMzN`0!T>sRH+6$Lh;%_>{&e^{4C!=I$t*MbE>l_F<*-s?Anbp!wu*d zsgtm+s)n@iJ=;yCA%KzcoYay`T&lecyl=Om;r><7R&^7m%1Cnw^CFRB6VPO(F;?9) zWL*lmc=Sa%ZKedqOuU1?PCmeGyZ6DexvF$M*N8`!Ib+-5Tg3G7Sgao<$6}^%@Wsy= z7i^UjXr0-Is}?SyzBVf$WSAFv*7RZfLp`pYXASra)xgTL3+a$|C*jZU+qCy+0`7>B z#y*E|gm@iT5#)zD{lVn2ixM|}At9E7p~!6#k+Ts_@awY>pL)*$uc9W>G*gbPsJ;m* zn{;WOp*qyv)L`FLBZ*hbE8L#5npir0$4mN6=(XY+DPJeXnuIrWjL&_f|K1A^YyZME zuCa7Sk}T)4;vpS%G@7mq8OqK!mg3oNb-1GSj5z!(`m#< zin$nW`Vs97<1x(PIa=W`Hq>Vi(t&DPsAhxNzZbxQX;*Q0f2#264pV{Y&bJ^~7m5*m z9>S)C+qnGfL>hQcRiL`g2&>z@p~udg^y)`LNy8hAsm`H4dCp0iMgYY8DnhGLTWH)L zNG?1W#U`EST?y&S!8~v^xhkuPSNe_cs)P}Jcd-iP{!s43-;J@Sg5t zIP4$KBuQNYU%vX{g@eWrm~KNQc0UrfM(|vQfH?H-*5$Mg#uE3J2dH76KW+ZK7q&0T zp)iMM+?3p+=Jw~&{o-u0c~S-F{&F`%(C1$UwgPv|_uJE)rcdzr4aC*uL z;X>V?!g8%AaCuoZ2J^0{XEgyZ`r~ckFGZ#5j<{r6dL;$)+!Z*7B~yh)TehLvcr!@u zZb$vl%jkS)De9UC(AoMGsw9j7t(d#mzug*kju}NNLUKvx`>%LymK@q#E)y8lEytb| zQ#!2O8I^A)qf5^L^15^+T59law4#f|Ew;*To!=|Lx`sM5*4l}M7gM24CV-C3e@ff7 z$W!MpimWQ}8`ftE(CxN9>lawS;LZ{3>HSB7TRFuTYMV-py0lp0tJ4@h`ye$|;&XLl z%Aj1jL*O1#ONvqw>0KLwBlp))Gl^JoLplb(`|xhR&^j#5-i(h7waLgxzIUJQhTroo zD9_&mm*_h}({H!IYS(edeG)*tqIf5`)Lyu+H=OCp%qE!y<+xFOfU<3JY*eT=XXvC( zq%?=HkfZugr!q*cFB+hqM&G18E5hOFnHey#sfi~37{#5ucnf3>?WRMOR?z4fS}asG z1nw5pz_sXbYQ1PU>&&f(KKXcT^ofP)JOfVBQ-Rw(w}`wfy#y)w9|d;|BB8YPDjIMh z=yhL&9S@Az73)E|GN=Uhy4T#eAAdyR7_dMe4cTYR5mL!C50NDli!K6=;FG zr#YN@6o3cvtBJ2e3zlB%6M9Q+0JXRm=q+}`Tnx;RrBGXMgj;o7t?)?L(sq@ zP+*cM#LzR(aQuO0T&$f5A$-PX?mk->bBEtUw-k{?#mVTWug&hCUMJ)~&+tTkFHQ>2 z6D)3%;ry$@@xPp(GNJjNLa(yrz50`4P;KVHat-S(Qen=sQcU2SjULBUY<}^7c9gK0# z@6l@d1(;M{CiGiy1;V~8B&Ih~V8$kMoP2W~A$EZfxGGc_wT#9bv_A}18RKN z+De$b{4XwP9Ku3x6=3sx{_e`VaK0-?k;d2#p*^1&n|hJJ$)U&L*+f+Waby!!4Kt#* zzn$j0e!)aIpbsM!rJ&E6(Qs4uFOl*%Eyx@;2EUZ6^Eub8cvM1yRR{V?lrB ze8kG#>THBZIXcMvvpf9O78<;&U@`C0v&@xeXHvpZ${EqIdME7W8Q9CaZow|zvt&fW ziN-xw(#7XG#=3Q&!|EgC=K2mW){)}06JC;rKanU?ISDpS*hY#hTCgYU1i7*2Azq5p zXPd$r!L#=O+R9JE;!G!+ZK}_onVQM<^*7*0q$=~2yAKAoZ)uvo4108bJ-l`HfW4D_ zG5g|EQhP25w!MFjM!p*`&bS_4eGI@}wHC~IzXK+f#6n=10d5Hor1^KtNMY|oVy<)$ zLcC-+ty~rI!pk3IY;&o~KQH)MJerMqa~E$PFhy76RJi1tBYgU#8m(&+==#(h80uY5 zd!D<%9VtzO>pfVvT1YofJB>Q8^l|5kTd4G-SMV@23a_sT!J_X`w5{?3E!DYzZZ^_P zX!eu)m6*_20UEgy3rsr%uxaIawzN=%idvrBU^Y`w1lH)4%Ols)o4`b$gbJ?i>f?)F#f1JVZ)m|*kM13uOZhG`E zY!3;+in0;*(mh)C_uYdyLG)Q}*4o`5Zg+y|A~()(?f_@ys>T9-TxN-eXHawYZ=MP3 z#I6<=<7(NbSo6h|b1-|)g)3|oY$&>g3zQMtw(wljBQbD(TMm|<7Sr83B2oU`D0p65 ziFdziVzJB=tnLrNkRJwY%VXX#{Jw!+8Xtm#{vVhx@3M^>DaAdBxCGz-ekMaNEg|E= z%piT;4AvO56)yg_jx}5|Vv+&w+zE?ScxlQa=+|q8f?Mv~;EzFEbGi!S4UY>}C7wfD z3mukH`5k%WE53BOD_pSX8=lQ~;ACpAp;gLjVf}L>SQf1cJ0-i(M3!g79Q_2s6H?4R zL4|#-H{^aasH69edPp-H&S$iY?Jb&Kp=MSJL|Xqs?RSR+C*E`ljc3TSTjuw;;i0uc zg}ol6!M~Pg0H=}77p72&#jK^f3 zj21Ml;#rw>T~y`oMp&mghu#$3Vfm6>IN&}3?Dl-aDM3T*1D-!c6U)i$ZYiIiIM_|^ zE~_RI?;}}3feIP3_Z;UtvOw^%GK_S3m!f~wQ%u`i%(`0|aZ2TRB+?&9(z0E6@1ifn zw)U`~S+cCn$_8FG6qCB($5dJFf*@h{Fi?_jVB@y`!BLYeSj2a2!NTG$yjrsxr~lO? zK{x9V9{S)F_ZA%0^@OShmf?;k0a)l2Cj9YjD7V2(h8~O_``2Om&`+|(YwI{W<1P9m%fL> z%$!oTJaQMV3Cbrn1%|M;@=3LymOFd&Op=YBevcb^S%ujQkwCS}v7C2=8r!ZPO&Yfi zV-d4un4ba98vVHp8^v-g|Dq+N@t%OaE6#I1Q_YCdq;stKJ`v`%8 z^KaZ!I+Rt+c?Z9FKd%m-@%|lJPfn(oz_Eq9*}ius^m6ZEcDPa(p7L4REjU0AZse&r z565Hlieb!b<6WfM+T4A9Cb2@}fuMMO7_QNNh2blUxkYoDn16%}YbI~WiBxUYtMvgl z|9im<{crJp&S&VG7)GWV1d&lG534_Z&!^ANtFkJ}pD*q5IVbm@_#%BID^wFCz!_ z-^Ad|W81O2;ye3P*U4_g-oXDB4Zy)$iRd(1QD`3-3SMhwLGv*^cFX!27GC85Q_iuL z(U);t(;c|USDu1&#$nf!WbW3X3@FT3heJy8oQW*Y`8{C|wRaBTKMMm^=j4LX-^0@E2o*x5aOxN0Wf z8Rz>qq<1u|N#8^yjwE79ff`pndMJ54RD$~wY>3CxBj9tk0)Hp}L~go`;G(B<;N5+a zq#0c&hll&aB&(s!A@@F>_tYYT1}aQ%u03AABSRM&_rbY>iBKG+hbk>c1fE?xg%0;dgoTIG=AEPeyWWN3FT@Aqq^essybwD&d}78jLF` zMwQ1~!C@Mo&Evn-M;)^0?Xn$oxkwE>(n4VNI&cFdx$MIxi4!Y{SLdOVeGOXwx zdOixLQ`dV4K8@LkCKor52U7F##pC(-<=rT{@zqzMq(mrQdHO?OUD`_nE6xk@E!!|I zZ8aJ%kfu>9|Hs~2$3+qR58UL%UeX9s3aEf0DpI$os8|RpiiwF$7$~-MhysG5lwhDB zV9{`UDt3biVh6UUh#la=4$rQ4?4jt_@AG=TuixAs%*?0eGw+#OcK0?W3k7$|gzuAS z=*K5(u(w6BB3b$+LHX-KbcNv@f!fj|s8MzacEHve>oLX-t*@ys9np}C&B69yLZzXo z(Yq9EU78!#*MWxik9aS5I_ss~!a`fDk>C8j`JT%277%H+STjZe}7)t{D12qw(Zeblr6u%xY6b6$H0cENM|qp7YZg@fj1phtF;qpqhf z3FPBWVO~r<5v}==kcXd{*e~)|EDc;NgT3rqgvk!SFF173MHn$p#s2)eIzhs;j-qL$ zpM-Zib`a^>pFrgv=wYl}71Tc>7n|}dQ8?A#7*h;T#CG{y6fQhC3~L(txcNCJF_{04 zBiI!0&dtXPTB2!pzhI%>>LoXHbww5Pwe9N{HVH@c=!r$T7~A`$e`x+c-3xOb-^2cx z#YyzD-#XN3qY-MpKLkY+EYQ$NJ%pbR1Pd*fvPGw#?Lb4-3I*;)P1u1s4>3)an=oQ& zP4hi69|ai~jtl)#=GeU$*&ls0a}(On`h-v=vP$@>?gnOUJw(XJ2^H>L)B)Yfn`_5B ztb%ppxMPtsWw7#z<>+GnIKkZ6^Go#=mtcJs#0Y0%&Ce$pk74@#mJ0UW2`(AYXM$i~ zMWkInj}*K5sSUQ;i*)RtTQEdkrV-d|_50Y7Wej`OZK+s)HET5c;|Brjiz?>kuo+W! z*A_i8Yr^)sDq|%z_UJ>`r&yute5?XFjtv=@jIp&wpcB#yu%*3dB1cUHk^0qP#NW0f~0NN(9xH+V}Y&$ zd!FzddSS5}b~c$KG8$xTU*464>R6>=I+<+rb(OYgs=f_2hBXv5ZY&XuvE7Zzs%;hy zRXB|DN|Q^1vhQKN9+Wm;=bkF$C+1-F-<^etZ-)xI@S5LKmYpHYaFr2V|2`GH&wYpO zco`wgmd(SGW=3Hrr^;d{uQmxMkIEFTEj)y2r#)ys#n<{l#*8-H1(k+=SK#9}o^KWr>7IFEBf=wSo`j>LM3WB-U-jCT!|1 z6Z<{OrV55`T`KTCF&SIm`2=<-dZFN|?*hTN_kGa#H9^>A&IV!n784tVXm|9%Ef#v| z_z6MXYja^!U@ZFPZH?f#(i^PbShnac)0pPn7YiHvYl!T&mtqRKrAKN41$VhqF!IB_s;(~pcbMh=K z&7&(iKYJK@Y?z!V?wE!BOoM!Em_veKQq3!Ld9Dnm!P63qJ)vQ5G+iCrIxZXAa^4=D z#^qzTr*Fj;ERwNLRP@DKu*wejgSd2px*7egc zto~>oI;hed4Pq@qR}Q!;xbtv6+HHZ3eWGo2^F5-?+pO1v&<~<`fpXFfVfTb9m~rz? zSEsBYSo9`k`~HS?=qmdfbj!jTVc(^%h5Up5Xy>C_FvFWsrMV7uSdn#CbZOZZtkZT) z5wD9icCE`))VbnSiGF~{?pX6aRpsM%=$^p&!qIe&eb25j*zh^i(BX}r(DP%{G4$CT z?9>P@y3S&G=|fEeY=VM6I{dA>pdh{$Q##&!zlU1{YFOLH-YoG;>0WLararC{X4cUB zzKWqIg~!k7l$wpRLznZtgdJS_qTF#8?S`%0h+ST?S+K6*23AJ31cV8 zqy86%2)k`o71gGk5o)hZ#ePhx!xYbc!@iWeVUcySuvvYsVZKibg-E@YNG9N%olab` za7S%M$-_z^%5b1zQHp9J_n^bV*dRUOL`P5Th0%xR_uFQo8xjX%G1#Hfh{4V8vZ}$b zGYRh4n_<Maofh-?NT_4Z}Pzt%?#%vwt9#XfREfr=^Ceyi~PcQ<#BiZHvKt z?x>@|RYwIkI%r`9A>r61u59!5`_r(k@4uk)tSs$U9Q4MJ=Ia%5xM=g-^v4W3f3^E) zbz3+xf2BY^s7kO`Q`X*mx3h3|_YmRqi2=fs&HF;C-BM7UyS=fuyUpzf#@S(GF5MEw zE%Hat#IDA?{l;O%aZJ(K0VY^l!eqhqE22_s<3m_z6HnBm@)A~CdQ-U8#uJjax)pPQeTI&TH4-zyO7Qc#3dEL6uFo8Q4|Vo`wgT-*Fy-XL#G z--v@=3(^;@q?KWx7dZ>gSTtV)dsr0>D^wOmL?6X^cin9_Vb5H2=net)y!rj8%3(XP z`CjWW_0tyxwk{jd)8}sp!k(@`{q;N8-?_X5Ju>Q{U_^pBrX-v#oL*Ofmbo=xrMq@x zwQL`u$%`Ra)tLi=)Q+pM{a*DLihakvR4Uo8++vJv>-rrVU1flF-fxP^3^`izd03gi z$ktQvy!qM3{f;fiV#dY_J1HMV(^#6K>TEuiYSMgPK0~|I-#`XyKJAXa=$wuXuJggl zh6LIfCHF+T^*twi;8%y;X}a9oysiC~VieOmsUmtas0)VlSs}!xoW#O-TLtM0KA{tjW(ahr z<=Z_8uEx4obVEhO1*qDrMr_d)1(EK|(`el8k-`FGq2T!C0>O`_M9eJuzOZ8WOZ0uC z8Tu(X6uWWh6M8{00rT|N7fe=D#rifsBXEt{b?lT`m~hT<4%WS68uofbGWI59JLWdf zx%r;`80_AuCqjiyEL4g8T%h`C4!S1n6~?pfgg#jJK)6Zu0yg8zYILinHdcE}-M;!k z5@z9=B0QG9#a7n#2eyfBRoXvszOcZQi8YO}#{`i^=(-{~;c?G*!XJ)`XnkFTaDTR& zaR0KiC^L#D(k+WYyGG{Pee1YhFw0O)l(=8RzGRlRD0pdAX{uHzYUp)ah{?>w`cyWR zJmNNA3;J=6pkS7&=yO~g`mt~uwz2uQpD)+S&`Fxj_hHt>V{<1Mh$7!s9y=D> zvG9{X|FW!|j@l9I!QvcY*~ObewnjMis+8UQj8i`}ByOCYeDk|4IlVUuu58~g7!=qG z3x6;N9a>Z@@Ozzs-7saM?#s*3@yYV`xt%JpnZo94SPMoA)(6)K^_;S>e%BYEZe=p| zN4IYlD%Fm{PQR@XR5U;Iq&`DWSd`?4UYvX$Tf`|BSoe5`S_G-1hm_4w!N?Wp`yc7e z?{Eml9749C4)z+@ly|o=GlLgc;^aw!usgej<4k&Ck5pyQvj)w_#opQi2M;IV`{r{r zj5Tawx2!-^yH}=Qj>-$+i^|j3>Mp^;2=k6;xym+lz~xTp@H3ghD`#1vQ=e~RBb%=a z-C|;Hcd2G6HgXJ4@VTN~c<@m)=Kbh2#&i!7W>f`W3p^hQ#)djJ-(wJ5D(~}1C^D`W zI;)-%E_Yzqotd%Ewqrsywr$8+p|x-c;M4rSwbr#)^LfZ*G-%ukOli|r zY@C&h=*DDu(V>!fG-zNVmf5rqJ7KFX;;9!m|85m5=;`V&eEDp`;eq5D}4J}VXZ(Vzcaa87EdY+kr>>p{uM?OQ)%niBN*D3q$R*%xa++Cgu zpN~uu2yR{#R@^rb6v?P!<74_`Lz|y_`NgVOkWp5KJwNyz+v2fNaCokgAkBSd>6nAB zFsxeyHt)qkfeidW0+0YC00}?>kN_kA2|xmn03-kjKmw2eBmfCO0+0YC00}?>kN_kA z2|xmn03-kjKmw2eBmfCO0+0YC00}?>kih>Wf$a@k8zS+7la~(?b)xkB)~|%9yS&(F zi`UDoj6q4r@p5oLu$6dQydGDGk$n(yJGTi*ME!o_Rf?X_Nh`-sjn|zU`G;S)ZhEIa?S* z8T;gnu>U9*jX}+KbvJ+Xni_0XmwHk9#C7#grb?CLy4Ih9O#gPogqN8~CM5cRf7zgI$Xd^@iOll;qg2&nh13Z={ed)7!bY4L*N zI~7Wk8@&7~RW8l8y|N_%do?t*i(qb+r+r zNI4O&u(uaQM(FU8MK@TRA|8K?P_CGd-%;~gB$YF6EUgyh?DIsLVA41oUt7-1q<4|J&ha zYF+StzNRbL9i4Jc>bROjNmAvl^TXRqx<<6yR+>L98+h5U-7$*i4z!gPr&#_A$trKj zMG2dIT4K4KFM3iZS}>tIc?_Ya#_E`c6v^+>8e;@)Ah5Fgzf7QrV=?x>kF4H zxn~6S&)U+rC13r_O`<>Cu4LU_(B}(e3)mLsxo^y9?O$|ikgfLc$-tL3#Z&*mer-^H zYO1A2*iQOpC-MJ*kc*eXG8NDlE*tv!1z9(YE&=snck?LQn9TWg8sagCdO>jlxg95D zco~)6e4Eri{&-~PQDSa>KaaAVXm7Xsd|SRdXRIHSapTWCCf6n&$MNvVY-#Zx@<+An ze>iVV5`|@L+-1t#O6x;hc6#6f8<&h7n)$}TWUjcrmzF^{al7`^tK<(-a%zuXYnyK@ zGqChQDlS;FQeag*(2Wmm-X0XnkoM>9*hMEKmw2eBmfCO0{^=N4n^)E_h%ey8xFM| ziOr2m#Dj^WQ4AK4~??l^VsfFBB7t8)sxJDko~Ik?IRI zX2xSm9~zgFY+A>w9n?s%Yi;wZcBGn&F|JqN9%wt)M4vJjxQ=(Oy4`)0ttx4^EgoCi zXQxqbsd-4VwY|ut%pq>KaG*2E%6^C*S;uvrc`kpQqqgTh^8EjVc*8d1n5?H2A9%OG zCdj0P-7)QQ30rd$TZx?Tyrh2S@&D@lzu&i268+(^&$_JveO^x(3C=~Id$@>leEDT> zFPq)hmjM2G9Jb!8P~{@rcWn};gO~V6P3d7?ORep{7MUN&@zt{J$vLX?b_VtGMH-a+ z-u81+mln}`l_td&w;Pe{N}f}f=Jz+UdXL4F?aN}}T{8bajj46!EXaMWPzP|t{fl;| zl5Fty%9w#tkN<<;S+?u^e;-;nWqVopJ(4oFcwfKze80cldSIBlMsDR<2?n#*_xq% z>LZe=*W%%%oQOSgttCZ9=#U}#w?-p1?kM|V(i!sT1PP0U&7vO%g2 z*SF^ryW{6G5XX$_KzVQS!JVTi`LvE}-nqkXZCd;MXctjmW=v5h|SZFd@M2f3=G)at$OBm(zHeHoFf zOXRt#^uuqT{}Z}bn`Dm6+ep@}-t7b9_S$b?L&PIWjj)XxL|wd`xmo&imUin>pXny< z_cwLHk)aazgyAuk^HnI@iT1|!T4)Yf1{iE6i#5^L~Y*M}eu-p5nMH&W+uFpFHjME|LCODQr! zhnEAUZ~3o1|1Zj53i-S}@jhW45`Y9C0Z0H6fCL}`NB|Om1Rw!O01|)%AOT1K5`Y9C z0Z0H6fCL}`NB|Om1Rw!O01|)%AOT1K5`Y9C0Z0H6fCT>c2&_3@NM5seXqy`4{Xb7f z-5{@roPrxiloL6Eu52m>#4g`-LLu(*Ns>mPBKXMHKbg_d90XE$w%BDuY1b8AY-)F{j2tx zuiq(o@HU*HjD6X-pVzqlP2WC>{^0uZrx8Nid(%%*`ov@Eer^Bzc=|Q8ygTM^av^+5 z7wwXo2cZ`)O}|TjSM}E+LoX1#l~f0hNk@W{4MUjoqYe#iEP_8^K@USR_vr1 zZllq$<(*p?!$QI~Vc>nrToU<;musu~brx^8x1PS>3uTOoSsE9D)f z6^@&1TsTKSt~cpy-PCjk5U26(aPs{>N^`A#d;LG})z5oXj>o5ge(~HtoqZ3+?b9En zyt}7+(nU3C;~S%YiZT}5PW5!vZ}X5A?^iO76*no{zU{$Il`%U{hOk2(*--B7Aare&*G?U?QgZWAFT0W; z!S&52`7Ukwv?Y(5{_{O+ZP~O({&7_!Wj|Fh)6d9qzONAW=cZ#?FW!<3n2CaV?55|H7AOT1K z5`Y9Cf&X0sTk9LFiR>R#^y)#>iSkycdkInBW!rKe?o>KcVqX;33qNln_nQcNq8t<8 z`pf+uo2d%F)Dv-tvZiMzGABZo`8b7=6Rrz$-o3YMYfo~Fg#DkCB?-plKA-1IE@i)d zSnltSr#~F#QgZ&29NJ?W?R%Bd$L6eL3YVU~&&ct#CvIz-$R=%YeEH(}bz7goqbPmi z_AAfKlPYf=3vX9WacsA(G=E&?v_81qG0gHu+DePlIsX;OYINKr37d$IN7`K9(qf6r zMs0N?kA+SR&uf#t@2+k|vvZh(ak|3En99_5cur`RkTGy~-79&=TF!u(#q)~b6qEh4w4`Eh-SdRyiC`5e#}uHTmF zNY>LGTfW-|6ALKc}Cy2M(s(&73A`uU_`@TCewJrO$NFQk96OopiEOTRv^c$ChuDh|!ka@8nIlqkkJ`fq8EmJWfTU2+8Xgor0K@_y4`^xEoxHif6NOK?S&e@mPur+Z-$H$m1g6t}`|}6qvwfH@Am`ZJk?JiJaQ%b5Hbj z!?tw4k|&+5rHs+Rp#bzbd|u1=_bTL4?h$wR@z6#P-2`&2@3c%WeMJ?xZ{2_WD)Rk* zn>}Xy_PYD=nlsOLB0QH3-a1*VjQ&rYz?9=UfQ3{p^OFBm-XOKVu<_w zO?~b6LQ2dD9lR;qiT1`m7&Qs7lyg``#!U=xm-rq(u&yH4lz80ZtiX1?|Ia*z0p>{D z_wsVaAHV;v<1w!FxKWiF3g_xE<>I_34&BkO7_uab4b_wj3`U#^KGHFGHU))2n` zX?gXOhSc}}sqM@G9Ps#k%)=X zi7}1196-?%I%(zRzBwhWDUwvjGG0ew8z>vsPS5GWR*~ zh^#MLooa*Y-_2PCO1w^dR`@!p6Ls;jDe*DLS6qhIjXyCc+w|5pf7)i*a43CkSQJay zPP8}f_lTC`3Hpa*GFJ!IW{I&kBsF!D7H_6*a=J8mYuWp#&SY#tw{PDx%Dx?;YpYyd z*00E==9T*BYBLE2{x@He_q?{{+gkqh*W{^|5`NR7t^-^CgC3M|+z>_pObFh61JTY4>G@B^{M6d2JD~x%0`;*k<7_n(9%GHwhh4 zR(u#oj_=E&p(OtUAC3U|HRbc<_mJ>0WgBY}b@5WJY7oU1*XO@DOWCH2`~6M*hoKLp zuU((clllK?%sm6Y0+trlKc6{ib$l?%29Nh>szbK4xbx1Wx9fkn!FPYqhjsBRZK55jwmKFJDi2wx+oL zKdDD0E>Dr>-?LkuH6d5O#ijiJFVFb74SG9lI~Y6v%t;%z+;Y6X zUn1ele%eEVDIssG9DJ8Lzh0@0l$>yXWbl-14-IFjy0+rCmb?0lBG*CVg|U=54AJRV z(%PqWTTX4lZ*5xp{AoMUOrf;3&uq2hl(EOSX#7XHT$I%U=l>_68Me&I45_g`Y;}<; zZ=E0BZdzy9Zd+;oxGeNUNxNfgxTub6Mc-{HncB{KW8QbZWvw@A4bq|R#*}tRi%e9Vyw>LcL|5wNVrFULPY{T(b z9V%PS|34dQ3vx2h8B6*9vOH;|&6-i~L9S8!jk;$K=mK`M%Y(?{{}J6&^6_}T`qTvd zJeJy3{_|O?VBYr+px3%tX+i_X;l)S!H zP>+$?8~@m`?XdS|HD6jAHa3zC+d=O-@nVxXV4v&&jsHBasMM5>nJjD{hJ$a{%gno z>K6Nw#QTDENB|Om1Rw!O0227WOJL;q1akjH-RkDS)+8Qw`gwjNWMc~0e>t0>McI!* z9=W!;?-qHa55@j?|4#`-U-79g$s7pTugVcIi%VMTgNBWh8soArKdZ{}f>d4W7;Uu= zW)CSQbmLWQDaZeH{kBr_A?%5=3%loUZ3x@HEqB#1D98URg0D*)H{h`k#2+We^EZ7V zG5f&zI$5Zm zFJbfgX!&oCDWt{3Wl6`C$>aYn+%0Xg-^K51v#)fEjao>{b5Om8|B$d%7;%lvudTjt z`NZ)Ve|7xN|F}S+KRlLe{Wiei6Ri)>6+2y{93$>E{a~%SDi6$I0++4V%cbS_@b#Hd z!OKV0f_O;`Hupo|3{&zq6#zn(|j zU)txf@;XXiYMT(sc0!u3G#Tsh6A;gNTmX42x@nf=v!5$pOpto~|Kwd#yMF&)VzHQ# zvxZeJWp1VACoVJY!YF=cl&D_Exh%1r`CM}TkAMFU>EJ=hNxZN2)^oVIzn$Y_E}Lxd zcpiz}z`7-3f2yk{na9WceC|7;XUKMt$T9xPJ`e}j_chr?GI&ATLatxJH=_0$MMmiG zGBj}Df9?2xUq>T&{4Wk0)*%5%01|)%AOT1K5`Y9C0Z0H6fCM0c|3d_7E{i9_=Rhc=OYGojHc+t{ae?^ zExJ?ET9(JMk{Uxcc4+pwfPqqVtz)#+*4+A(99zPB>UF)7vGV z+v2gMeU3CyuYHtehs$RA)k<9teHLya<%YIwu-1tfdJ_uBH9*MPTaI_xOP>Gd#&2(P ztThAHS}BV9*#3Ar7ufXLEtIgmrr2^_-S7I{@Ydh>S1~V_y#N2U?d{**|M$vGWQ+UZpYo4C`4?Cs zKwkBsGl<(_(el@NXoA%1|CTdbuAvjR#p~AQ>wkN0xV6vk+Gk>;DKUr5E|OS3zl$NR z@Avuyn1^KDpV!pQHc%$n;I^mT3Z!0NKUS`N_y0faYd~4gr!ylc*LI5MC9OVlpkN_kA2|xmn03-kjKmw2eBmfCO0+0YC00}?> zkN_kA2|xmn03-kjKmw2eBmfCO0+0YC00}?>kN_kA2|xmn03-kjKmw2eBmfCO0+0YC z00}?>kN_kA2|xmn03-kjKmw2eBmfCO0+0YC00}?>kN_kA2|xn>mcU!zbL2C%mz1iT zw~l^dumSmeKs(*KC zc|WSSe((sPwRoGT&p*;7!*oo}AH^qpiSm|eKk_-yb@RVc#`0iLmlDFZt#Y)M3K^p< zoBvdvRUSjhqks1#%GfU;P5)6YTTaMy!gCePcx>yN+LzKNu3O@FrrmvfxEdri1|Fxq zpNE4(+8txaite`3;-<>bNO`KyR*CggbU3Wt<7plHj`x;KLdMe5YctPv6OD?iO*R&4 z#y5h#uB;y-VcRuW+lGkqt8&gL7wuo^+tPhrQc4-4lWsC#qOd>Rs=C{sw>)Tepgxu^>56~ChUUN8$tbc5*2IF>GSx0&QZ}=qR zV8UO#oPH~mG8SC_DdyvE-$N`Or#%b{it8}v0f;9rzn|Qu{s?Vh zK=@UaT_WEpLCEmZ^7_qf;_bHT&e%bT^^MyJomcH_%f2o77d7W1RJVu9eStifBf_rH zZB)D7LrmD?rG39K6prHh_SU<2zx!>Rgl8;}?*Xfp?-~(X$vukN_kA2|xn>I|Molf1b-Pq+1+roq3+np@fk2?9%ey-?JSSQqE&IsmZi? z&ShooV2b?;t;i%Iu7N`bik{F(D|cxuAaf%0*$sgb=Vb`_y4M;xvL9b5OQucZ|eg{cWYiSspJ>%1@8kK-O{F4#v6U z8j$AuE7{rx`7A>3z-as3q}PX=6j$uMS|DPzeCKl4sY#4!(4`BMxg_F>m#fFg|JD0{ zJsh4;#^@Z_!eLQ(geBg`0|j=5cwcz^NpgwJW4SDl>ocW|dbQKz!Ff>6tJ}%<|9XE9 z{_T19$5|RAbJLp%Wc}U4Nucf!u!xf1z0LhJTIYkem)QDHY;pPRewPdJ@?U;reFx;X zj<@HX%U_@W4@}VoEc?wmL!M7wew0P>!28PCJ|nwzpLqMCV)MV1skeutGP zbKBalb$gV2rFG8iyBZ^3l~{}GW>ug0RsLp_d}fVIAoY0srD!ZUFCoH#WId;A23aTa*k5;&A|rHonN_y^zxMv$ zQ%^U;_y3B+hIL2)5`Y9C0Z8EgFoD?@eqJ}HU3BVTYZR}hT9pv8S#Rcm{g=bt^KA$j zQ5toylGraI0TQ+s_(C#2 z@%XLl*1<#4TFctgu1|}W=nwbdqRxOr>SuMJ+g;a4xvon#?~*l_SqS*oGTFLeV+=r^ zTVrjtiGHotJfs{W6FQ=tslg$cpC0vdo6(3o2Jsv%MoayU6aM0*b=UorvEcfkHU0nj z@&DSYNtEwc*jx?Dc0xSAU)BATTCPcZUh|5~^@4cu|ks_re%0&yNk{r%Kph&%baN$VK*-%9?L zitESN`y~-}iksAb`MU^VN0k24Jjt90nemZtFu9*2-o7O&m+Fov@TH4xY>H5u-u^(bP{ztjEA9euO|L?B2 zXnQ(5S!%2vmQGUTt@Fd%n;x|P+I?yMxa>JQqTT(KSS_)Y7DqATB+2UFwHOJTXYT^Z zH6YFRSF!-(Y2>q70?in0<{90qs<6Wowyo3b2GG~&5#J^Hy#2PE%wN={5Wmw4#w9)8tPfLRL#FSlseEq+l{og-hOFX|}d0i!P5!VrQg`Tm1 zXWi(LVBHdO^o}*j<$2B)%Jn8bfd$rtEncc;)5&!@{npRtR~az3l5)bH7F0)(5jwo= zI-%Qt?fU;QhgQMs|HWa$IwSxIKmw2eBmfCO0+0YC00}?>kN_kA2|xmn03-kjKmw2e zBmfCO0+0YC00}?>kN_kA2|xmn03-kjKmw2eB=A2<;03)0EwA=E4Nrf>lS2$y-&$JF zQ8|nr#@Td1;Cw|~-Y~QY=pSFsXW;GDpRE0*wA*13jQt}Y&|B*aMOn1do)c&x=38mG ziB_cEr96_XzxE7eEbVbpv3gpl0`Bi0GMlb`ca-AcPI;hTB4%{Kaf!BTmkP$ViXjp? zeVW0eZ&=6&v1ic=&q0MaJC^*XUn9o&q`6u9RmEo!r1!HF5;7-BC4~ zuD9bIsPo@qv;m3&iN1rsyi+h%*-bm@Ur%e;p}^RuNMk&EWd>|2FXoW8l|yzi7H$!e z?a%M>=$oQHf%fESVIclNw{`Tnk>}}kD~6JGKaSp$$hn1{jLbgMQ;aEBQ%Ji$!YMSh ziM#2SlTondjA}YF61W^NCbNy95;68;8svPKo7|)OJY53xby;T>Z#)VGF<<*}fsMoY z`=syl@}598n7>@{xXCj4tkP?sFCr(R?6!Uz{Y2--^zpOb(CBN!6qoN+1%BfUwlI!F zB+vtUoS`2c8$-seTo45Gl^=B(O>dZtc~&Yk-P^Ch{HyGZ0{Ijuo>g>vnyTm&+z-f& zEpIS9-YGJjpL_t{0le?Jid2OG&r*Q?y-y_Lt?v<9*|dWoc1XE5D7j_obe)PyIU?7j zI&U(MJ__6D7iV;$56()ZtM4{t7|k9A`tGy#IsN3vZ6Ge;r)rA;KdTQ3ATA*%%Hx9; zF;1;o&s0@$B=Z}!=RP@K_3pFjPNpsE1ov?rA;&mcV+r&MZ${GXpF5OC+Z5)(NLnrg@k=7iLFqWfke0Ag9c*8^(v+B;=4C5t zls1s#!uz!^No0Cw?_<;re#w}-lt;goRZHuebdDZdpF?{SF9-U2#~lLVjd*2@lGX{mb;B4tgD$H@5^ZWuuK4>?5YzC1gp zD7)k~XlET>#jxEujy_91NMU)A9__^W(?FiG;RZQ3y}5J9daprUX{k;dLF^w~7URB7 zJZ-T_16{{{6Wzo~m$7858l%Veet-?mVMvZUna82R@r<_Ab0vM#=Op0kSTU8fFYpXx^c$A|&U6${9gvIKyqK4aT*11$`d1OR2))cQJTDLC zqz4wL7~pjg#JzZO08M)>1JuWOW^rj*>7b6Ycy8~+`C(@S@}HO%!H->V6O47}z;KS; zPGA1(%8Q&a_6fWxUeD+`wHLYJfij$t>T7}jkW=owvr7%Qu7`H=`j}2dhFwzSc57J2 zcgRrUiO!U93eWg)e2NWt6GIS=U{eaRCuakjm&ays_bPA==aq4)l54mpLrZy#!9KLK zS^YU3Zn=TJ&o3(GJ>I2+>@qGx@Ej**_;40{isc&KO=96~(p8$l+rfSR1+@;U1jN=Z`Fx<71l#a^}T)g7Ny@t!Fn3*5C;?YV*nb9n4#zSAX8Uf+%j-rA^$sAI5SsH4OQt6Sr|*nFg@ypE>Zfenhfo zAJqjp7S7G$)XekXC3Nq|x;EdPhixB(Tub7qIR5 z#Lr-l?Bc?@?3mAzr`2JbquTr*0@ES z1%_{UQB~bp9j>M$j+@naW_dliWukkWg<&Lk5S#rQxT7nRSy=%Gm|u&ZbAs0UusjV)xtgOx_sWP5zn>3aq(Xt@$Y~PRNkEUi_zex(w&Q{k*5U z-!TIMEcsJB-XQy5?&36dU(EMlKH>H2X$;01Ua*7fNT0ukrbem82%pQ? zn4`zv;q#KC{LzkYbmB1IV8tz-NJE2}uAt9)-EX%7=Svh<$I*z}(YKHh{5g@K8LP{@ zyy-Y6tjvUW{&Xr}J?;2MSU;x)VYR;Oa-d(I=LFM3k#4~-spKz(@aqoKNaOQT6=1j~y z4#tbGtp!Lp*jA)Y6L8da3`{mz6f{)3uHqV(0VmEA%;TE}uvF;r_ z0><1SCYIm#@g&wFrhqliZUWzXk{rlK{?a#oUDabS7W;!8Ss%>QKwTqUkL6M_8QEPt z0MVY1f$X_`8}!Y5Bm(_rB>S*r!{SJucz*at{<8w|VW}ERht>fJHuML!EgBh_$)8ma zr~7oqz!ROAx$DM~-x;0B4M=>kB8Zi*z5&T}tYb!D@kno@7)1^Bnaq6sFeL45D)Pf} zD;e+8xID6NN98K|#MRqp4qJMi4 z2HN}TO_7wDOPLr40lwNFb;10veK&~aaApJDC}%8-e=~yKx2FmdZ43oA{?jrPthxJ< z#g1GMBdF{FZRf+Mpw1na&d@tuLbAabM0q(f^OL+7p0OX8?t@Ag`*z+`2(4{le3+R| z=A5)Vh^f7}f%KiQtSjrRQ5I;wn$U87;ag5GWXP}wjH^X!$+6|mswCI!cKI3f`0*(; zL1nyxg3((<|8h7Pf71PZ^sW7(=)$pSAlDdfBdD}ny*po>68%-QwjJHY?U=TmLz zSGgg~lz}D8Ymw*ZZpPc0uLBi;?Z7H^kej{Lak7sceLn!*^4JWftHD_$A=i@3jfjnx z`^wHDZ!1!mPmNg&de`~L1h4PFuVIxeD|}oca@%tzQar8?@C^*|qxY3N0rcvbj!5_V z97J<=KO|$?C?LDiKN$4i@s17R>=%mcIAH?vO3vW1Hl;0MEveE1c6h(07J zO3$vWaoL4TnU(R(>FhhSX}jG(e{r7qv=!%Gka^FN(Ie;Ntz{gu`E_f^wwY&;UNyOl ziV_!whTH)~ztR0!#^p~*4tQUIcCUb+Q{Hx@+t?T6eBnC$^HwyCQEeNG+)x=$>blH6 z2y)XH?MUx&(Vy{DZ8m6^eAvJUGSpxlRFGp@_#UI3b?S@grTEjmg~iO?*GGZ5`lj%K zzU}Hjq_?~d-9M2zUCB=%Q)W$;CM4#)_^tP#yF;j;T~X< zZLk{X*H#Qrys=;dVE=T(C?H!CsLs%73S}00?*+Dljdvn)779qnk(Pb+>QPsjU)~L5 ztUJbGBwh$ao~NHi9A0cftk15ckNO@CV$HbB1hH_QU%$1iF}!~K?t123(-5E=n0|n9 zH2yqu>IODrv_%rLc>W$%wDM7=r^8NiJ#8J-vfajgCuqhZ%(RyO0iZp~i=L@Hn%PUC zdEK7g%=mcQRQ`-8kfC}onm*-%4^!jVY~)T2lkS4!vS<|#1aWFP*Ch0H zVC7XgfTnv8?D@sCbm7}7)iB68!*F0i&U zj)aly(oF6kKIL~o9Gf>Mfc(IODaatFF=U(}uG%E8x#we9UsO=g9U;Wn3rz z$w5ny3#X4M%J&OX4qs`>{vKbgJlWhz7l@jfi^Iy-OPGilC*{U>4?Geba;cj=< zz7fM&TNY%IdEzl&O&UOsVMEzARnIB=wrY;QRRtGyY>h*b{)dEIe2E zT@RTkCzyQ(aV=!HtW#J$)1^SCWj$3_s(qV6V|3A<0op|=exU5H+%oWBY(L;J7YV z$x3me^5O9Xtb0*Ql$($Lf&A(xhU&B_6TopvlHwHAs0c@uQ#+M_uC`Y->w1m}$P3S- zaZv`N;rRfXjqzsCr{VmY%!M6V#xxE)&m674AB+#TePqBQ>q|SWRhlFB2E^1~X^sqk z%2DBl>!|cd9S@EHvYl-J+unsH3?;8lYB+Bn7**_ka_$=i;W?3v7 zik#_aN8e-|&3fz78Q5+1iDs?T?T#RdrO4x%sig1b%t(-P(!`6%n8qO0Llc^h6DQmR z-;?^!DN6J!o+_s=oj?NL8LEvJTv2dv&Su_S_l=df!9rE%Ys z3e;G%qlSp-od7U)*9c4Y%>(jkxZRYdjUe8p>n*=IMZHyE-KX^-*EJE#+1i_)@upt& z=!%x}g>%0gRGzN&01?y&sn#c2Ar9RlRfU>KtKfYOYt3w5WW%j4pq+ZKC3l>!YW_a3UPIMT@Lkk&QAZM=t0-TL)d%zX)wn0? zvu+6T-0Kz4-|zGoS!!d(Tv5hhcui8FpO~M|I&fYVafutqpnuz_(y4h}d{rC)m>hYd zr($WdgSP1K1+Xrmh@r8NX zcWo!?6L8-s-};BFek2zVa?zU z0`qWlfH!k$gb%|)D-rO;S(|yefj%eaBjkB-of{ogCwXl`-W}=)nC0_VgV?zJr?2~1 zCx^G}tCc8(0{Pf=X{-%2f4TZ$y5Rr0vFku~#Uu-mo7~wmJX!Nl5PLx{XMRt1S8nQ~ zo9t)Ltd#cosR8@sWy5$sHu=!&N>undCzZgwSYN#kw*PpadwfsuLguL`vDk&&_ZdcH zp15vVpIQ8g4zswry=3_Oj_bI5-{b6Q*VEaLOCE9WTHXWk$5;>JBweMm7j<&uHe9mh zRSsOmIh&ip7g-+VgznHJw-0IVGQb{>>vYJSfB%b+{lLF7?~0QqJ9-<9fBnF0wxwq* z>;1|-9JE_Fe_;G;kfY_FIIIyj$;bUFA1K!z$_ZjO|qHGUX_b_$L_CpfERPnjJH~EKeu=AB9L>!FxvIGwIu|vymZ?jnmoWQevc*%WII0uzm zwuu)Xykh!y6_5;%Jq!igs3)6@c~8!$k#Y`aC}X`!IhcQg@$yR8;bS0v{^}XB*xf$&;B+X^XD3ayWLfS|Ws6=o@eaRK0lC~hMdP?? zu|VHzH(K*MU%bYWy( zk~v^J_2|RNex_#BJpQo`d@Ro?CcF zOiyvy<1TPLt{&zNPmSOo^r3US`&P0Z%{t1BeIA_*~}eBCn>0pHo5n%IS-dU;cI}$1-qD5JeO=KmbtoGJMNN8!8i6W$w)et2kWs4+{ ztd!q zOB;U)C9Y~;WUgzPL(Z~akeN~k;u{i?o#r6q+`38y=9LuohWX6RW6)+)Pi!`EJ<&)vIR*xMevSsIx$`5BT)JImO1h6 zDYc=>2i#xd!fg8D0k56!Mja9!IDhe*Xe9qG4Y)iz4+Ydq+Okm(&od%iyMLcB&7qD) zV!@L~F!|Y8S zN|^=ejIJycS#gq9TX_jxsD4V_80dt}_ll|QF`VC}!Zkdw=DC53y3lYZwhhnjPc|8I46-}71M!{=g-S4A}av#Uj-Fs2k- zaNLjk>h<%XxQ)oRP#{M=6($!AVm)kI>BB5JSWNXc97B#5<9Ip0-_oa^|00)kMu3+~ zeCZz(RoITMd-R%474qVqRxHR^Dwa>3zsY6i z+B%?BJI_*?FD4>2i2*wF>lbGKsK0o<>c%l959y4dd(N2f?1T!{TOmI@Hwp?l(2$n` ziucoOX7DfSO5+>wsHG47oluF!SLmb1Biup5$`Ho9_AWWotALw*WF(kocb{&rGp6%5 zji#Fx+`w3FFR#Kx2X!_+e>F+=YNKW4s^~`T9W=3lVC{T=(kkwi)ci0B^g{O^p1U*r z?U`CHabLBvp22p9uL6rc?SvW93+ZueKT7m*hHpG>0duzrpy&2X#^ctQ;XZM+Lu?~& z-ZW}hKY!+50_pZ7L-n1;{FwSRVEX;fOqm<2XEH;fa+r3p{SdOA_=Wi7XU(jbcMF9U zJ7FDJIr_140~4{mm{gq&*9=rxJh<}cz3 zM&_n#0Z1KT3N~U}hVv?;fUh&sZrf;J?mU%_r9lo^QS~8_&CS0)4Wn_fv6?WYF zv4XUz`?+Yf48`Fc)QtF;YuxsZppaFKY2o6j{N~pI+^P))(0u9x);=>^?zWp)Pr~`1 zH+PmCG#^+A?7Js0%8xW)fLbLiKCqr@*}YXs)F! z2;9G97t2u(Y9o-A=@`!J9ozcy6r&C?DyjYkEfDaW4JrR$hID!ti=tIA1bNvuq6-xCYFM zF=4zO&L)AR(G~c3cMW_qg4I4Ys}FLg2hdrQd9qodbM?OR%XMDabm+?7i#L9d_?9pK z%K<(GZqg_B%c7Ntt2a5Rq`Q3}BJ4svVR^9D3*6|w!RnL$5PzGGpVi%0Pw0(B@2y06 z_V(+`T@TkcsC5qEU!9>0i>GD^_sb6;_bv~5ok}_guDnb3-HOBWN&U=bQl9nK+o+lZ z9XwdlnSKRNG;JX-+$Q83a`WWWyiUk2ICTz<)p$og8WqjHjHxAD|BaTnu-izw+`Vo! zy7n5eyXv`oi%KMM=;2uvsn8+*#hOy`{JRXR*O^!8{^>`IrkymV+&Jm#xe^pT=fO50 zeJ<1L+6W&(Z4*N_>?kuXx6&1q&&>pCO&Z#-(mt~rZCIGdb+(F{cn5qq{}A`#mTGOF zb;VEFD3(CST?BMWTo0)4)D+l_90lf_WJ&!S5(1gyUx9MyBh z>hfIb9&Yb{9DWkZ`t}>9jrZhss#{9l_3C4Gs~+bjazDV2GiC~Q(yn z5tn0ftqpF{b^}j6VpN>%_HlSwohqVw5rqbgnb1EuOt4s&;c2GWLdT2mcsILAwB}E- zp#8#KGBvPY?Q)?ty)w&&+%Ya!#J^ zBKdbNU!K=TZ1&k{rEe$ADQuEL>q7=CAKXkfR{Jsv%}UFFfhJv}MdeqzPXrS+HBrZ! zxR~6a+2i*-(C4gwu{6SKX8z~!STBwj43_ztLA&{y7vLlXF z1D~NikgA?a`X~1oT{vSY_xA61Xq>PPUAK$ZXmLp*+O*pVlh9sin;ojq(&8So(y2*S zZPIMQ)YMu|_lJX3i*2gfv8@G`B7UlUACQh-eMEY6uF3#8k}A{+BkFy-$T}-qxxbEI zi35$>5U1cXw*UJ+uvBYkH88mnMKTuiRoN>`5r1J(FmHkT2@MfeZ=*eLOYm_#?wth} zd7ZWetVUc442U~IeBw?~Q(x0X?D8z3YHPLtxxxL6H@}b2{Y6nrx?=^$YF`48{eQ~% zCf00BHs?eRtNkGCrUA!c4Vu+)mHg7k<6E2MXe!+f6*w39p&Je9YEEkwlIy<2@(^t>cf~kSiWG~E;BucX5;eq$7_$D{)O+K7h ziI$E(C{VgmLhYSm4}*Hj1w^?jGn-mOoL;LmehNn9uFLq_TqM8VSFz1OyTRM&aJh=2gxw=|jN8)#z;tV5m-nq@*)SpEg1vV3^z^>VLp!niLYEQq6 zoMK@(9h-O+?T9zU`k1>+8-%8%lYHJvwE z`ih-;PpL^^(?HPn4b*nG&)jLv_hly@iy`hOMyRD6%a^jPM7NI)v`(>u_%V%c2Qvmuv z*U8>UG$fbNCd}xtI1cZmKH9LeLvt0^7D*~gnnr0Pk@|+cg6T6%1v>0=;^xmEgDh9h;eHVo+R@?L5DtF?u1 z<`Dq>uAC%gMrNocj1JN;UFAWKWwHg)yHvQD=6B@=!!@l#0wr0lNhb5`qNAC;{aZeV ztyi&aI0X87RnhoIiI^`uP93cw7H$8j9GsVhjBkx)%$X0eYd>iyr=0NunSQavhpaz% zY!uP1Et)5Ipc5)@;dDgKd9#7a>W=kLY0Y+^u~Y`lO;Z+h>ZS@vy=1|frghMwIv%f! zNc}8@o8d(a zb}xKDRd}YO1ab#aHmzAcB>S|ytKEIMC%>fFg4zqrY2_obA(bj*ympLy`G7OID^*Fb zXzVGfSk_dj)-9AMny95(;ccu z6$`rBk_48{e42gr%%sO?igg0F7OGp->dxMpCZEeSMaE0U^D8xsV1oY|thXx{D$(Rl z8P&HR&cH*~2K2Y;y-;EDVL|VO3-HcUJ~Q^tFG0=5WT^cxnofQ2RG_yyok@&dFHm(G zkYA5X@HbDYbT$!LU~FtV$ntzQI>?+{8pxdPnaP>0GopEX08>N+_MUA^Zn*pyIcvRP#sI^&%JrybcoMpQmI`yyB$ zydBwWV&u*84baQ^9_nHGb3x%Rb2ZWYe^SRD?ToWTPHQA-?V%5HCOhPny^uQm;51#} zIj>XBY3vr5r_u=17tW^=ZE{Wec`fqEcNz4alY$=e^fUu^)ze!qyda(|KFxb__%RG; z;(_CJN};0Jh8nCHAwS}KEdR)ZNDXxvIoyXOj2}?^@t29s%TBA=SENv3JS*yiyba;9 zT}^T4Q<6Tg>@Hfjb}Y|_&EaggeNXL$-5HYvUtcEQD~wey;S1iLm`WZx8iW6D#Co++ zl=Y!+@O^k1Wv%|67ww-Un>O>PaJ4>*SRI$GHvfn$dFVzcMF)n%p_4^mv#AHQ_pt%k zW`wM>Yh6KSj5qI5GLY*vK|Gtf**Z~v%M1)!3egS4dgj@)F8(}bsn!_IJHga5Us#u| zp6WM#c2djMLBT_F1*Dw37+Q_B7bv})rKi~Th|qd4f$|{d%E=E}S`VeB$Q_b5qZdcG zz!wiCttV%LUEFr^motlb3jXB~kMhHioH!%-7F|pA*lZqsD&Nj<9#d zJj_EHWUFqQ~g6jo?Hf$zB&nXA3i;iY+7 zsQm5+j8#;eL;9`1X!tIisLIc z{K9o&(OAU1==a06t;O?eNbN_#;POKs>fyj7ah)xi%iD7{dsv5wefc=QccKeu3k@HR z-?t7C@3X~cv=nAKAfBoCtZ8TR4eD;5H##?1$OK8d!S8baxO+$&;5%2Ake#7{^LwOC zfESjjQB-;&)G}8j{X+hcOv4%^y(SanN#!x44jyK<*}Mfi4__tE5DsKQ^LDzf`3G@+ zP@WtcFpG@H@BrC985m2sy^w6btc>+cG$p_cxz93|KW<@JOirfKl4UX=QAr7}*)o!9 z@b8U--(YFLeCmqlUNopI1sDEZ&Xu-v#&RAlF5qsylgX?~`bju9gi>9Dr)92X1Yo@1 zD_-y(eFz@j8!KBRL5J9%0)2xcc|R^{kzofCfn8lI)aE~d$#w4Z?8q&c$3sdgRC#v} zh(1#XvK+1v*4YmU3zKEE+k{;>{UsHR+w?J&JLG$vJ&b7n7qvHsH%*+lfV()2bD^9Y z_WLlbuzvx*fC4bfO9@>%^1rh}u1g0w)?MZ@e6uu6Z?#Vun*NAkgcBwA1{abaEd)w|&cN4_WkOuX>Soc_F4khS$FdGc~K*2|JdlgSiE zDR^K_jbzkB6S(3<3Gu|U0+;o~!kS(eoX1fck&5*cAJ|3KL`BezN3X$mkJpm^Q(Cc{ z@pAgWCGZ@`jgSINMH^aFmd1^2>Vcfu;`5uv?~KVY@)xO-;h#z4{4(4>=AZwK^BUER zCwZF)V3#+QUbgx(#m~$pWBoTl{~1SdePsF(r2I^O-jA{)WZByaknxZMt+rG$R(CGJ zqii;W_cY=Cpex*SQF5^5Og3&Wyxj_YD2pD^9baz71f-|CB;Yq1}g<{w$)ZBu`+CuBa#H9n8^U!6xI zO~`;A*F^X}eiFCBfR)FXE<;u&%HVF2u!MSg;is0sUmqN*qGo; zlq;xU-X}M_6@C9pM-TqP@8c#$(rp*N!nw;9lR4A1k>de(-q_`{P`}4CuHERvaPzyP z(w*PUa9w-)zo2Xk$0MCrxv+AUA+GO}JNAQniL=lIZ4P=lRtN2BA3=L2-D3{9`@tpq zH*?l=r=!18mdNvt@C-t~A2Os$;by(9OvNt+^lfY!!ODJQx*eBb8AvM<2|}x=pF8D{ z($)l2NM*ni!TOxZ<1UaQ`~S*WR-npNybpUueKW56NMmc}+L8v)wmlSMxM=an#eExy z&IT@8lwi&w$CMN0iifz(yBOxNfFvHAh(wl(4y3wA68FWsu~;sT>ibw1qPi<_5}rv- z72g4rp0JQ3dt1DRbY19K_Er^^&W&z3th=cS3;@3*BbG;#U+~vA~<*? z*=4qk^Y{Bo;MgO+M=Qd;{Av?eHEkxaD>{e@4WfxiJx>s369eZxXV97FnXtlK3QRh2 zod{MB=b;-5c=!D4=~vH!sMTT9$QCq;ymhY|@A(AXkAYU)Or&x!mB+j251W#rAQPs7 z@&|oUR?#%-XzK$aJ!m`Z>Q=|~vodNXxit9zmUGZB2X#eWrbfh1hh3>GQD?FqX=OTt zw%dCF8e_?*EH$StI*acuiFkhPDv8I6lzS&ImMBhbhATLg)kzQ@bFlu?O2lpB`{Y$r z=<0IvjPwAeU1%17%1-8RXIu^((h?Z32E@o0^)^?S!4;&R>)|C?!+k#xHa z=!>tTkA`jo#Hx+_-z!pO8}8;~Sz}38+WU%u+!qI){KBRP{!T4B;@{4hS|8K5Y5;Fi z=coiKFKv8OWAOr@q}shG)5~W2;2YljpS+Wq<-8^ zmTZdQ>-riJKktoV0tCz`?V}G^sj5+ecONW>++*{ISp(8q2FeeN{1eC18OAbZ>LrNp z7XOM9yzL5Os!7RCzs!@LZ?SH8Pi4ZWiS*VVDK!6%4efAHn|9J)!TiWs&9g0cP!{Pt zxtZk$zy8Z5_%59KGqwE7kWcTKJ%V<<638v-SxdaXdYu2&={fyvv<^obDKm@Ya%daB zQ>4{cHSr$pna#XSbut2%OLfHiu)kVp_60fDEBiP#%PMK7Wo1MsZA1K!>0(!{oyhwh zxkD}U%@Nu4Q=T(5ZQ?!VEo)Sn`jIY}U*Xz<=I#$d5(R3!hRU(b`UPWw_FDyd?)gZW zn{AeYJyU7<110a|N?zzPvVvcnFSkAUnKLrTvEvNYf6Qoyi;yBR)rjMN9de=~Zx?V{ zFY7~gMkKw8XQNyi{e}0ZrHO2Pcot4Ep3mNVbb{C`!6oOep2j=RMloNk3e;;0hUiUi z9gY9?U*Y<^|G>|yF0~SU|6hNRMGyVn#!a|h%L$y5LibM@D{vjrM#&cYa`ld$0GH&CSSz>tpx8Hb=5ZaIpJW>s4dIZCpzwemf=$z|}YScygeYKtT11}Basbs@k zT6LIRRbQriZ=;m1-9J~|fVDe~+oulkdp`EkBK!Y(qE1<@fveG3A-AdGj?h8#Y2eNSCt2aWaM{vd z$qfI;2*PhlEc+n-J}>a?X1OXOPhQfhH&)vo?dN?~G*?)+WDT#PG(;s~_AI`Hw}7{P zvl;)Tb3L77{Kcd>HYSUmHy9n(GZ65SOlhO{ zCORX+C1rQtrl@c2S(etHQhBqsT)}<#{OSR2Qu;V$(`q$z`(Hk3v#^BTy`Y?bVANJ} z|F;l2>rxqiU9>vgwGR-n(# zUnS!fx%JLu!Pb!nq}Ii5WOVO&b8ig2W|mCp5TrX-6MHR8)HaQ=W}2CDp6a$%$=l1O z3N0cU&a&|9{85HvE9`E&;fUgn;rzsDWtWSvVyLi;~=ePOrOwq$jK!qi^+e!nfX z<=Y0U$-b0kZ2)Yk3ZeuF8v=H5KfC3xNxHK==yy6J$eTHx{`=qw?Q(RoAfC8I zb==P68qL`Ore^eTw|FfT>>qcRm^`3v>KVn#+~;-Bn&~IC9<*GC=YsD8h402j>QaY> z?f-i$CS$!HFIi*s&-yyl|Ih}X^{m(MHdZEn{W(Y;( zl$|^89DEt|7|;Epc6kIz5;s#w_Pd`7l+{i}<61RP!p%&uGfZfE%DN0cH;BKVWrO~L zIsWd#dzSXFv#AuWeJFsnK{o7RlRZ#3yBzZ-!n?Qa4JfO=hu1LXm!#N78UTE;{stQz z=Q0&$MW~>54BWTDkvY1?9WEX%&+_sGcs+XZZ~~liB^rJ%%LUvkgJ_a{K;6??I?43HT1gd36?62hDqBYTXXmnp}yxNmM6X19+dWPV~k2U;NYQJy1n>2 zlx#nS)0WyaG5K=6uzkizZsGnp;GTl;+||B|a1KYf|2{V!^Dz0XJFrccI;7+1-`h;yk`8b4c^s znc}9&xL=e9?qfgYy~lOcxhE2uScI@Q`yYdCDvjuN!#L7nXA7*pnapfl6O6G=tXz(? z>bJw&Ds|}fi*w8~qzL0R6IlP~>1dQUfQ=jUVEr#6G_$mualT>8^cs~gEv4t^kE{oX z)9(kWuNYJ!^%IHi|DV_}8r1E*ipze@bRtO~E*N>=8^~T{Wcs~#0>_P?q4lReoL(~h z7XJO|s>?QSXJAC(A@;C@Gul$+Mt7e(i}hp<@|m`CmjF?Dllj+W0k0RUGnaWkkE)s8+z2c$Z=*lxy`zkB zu8l?a7u0~q4mx1}!6Nv(_b^Vc$jD@8J)BL}3C}QXsM!abHO5nxU=!En#Sm$iJVJJM zf*Sc+LYlCXX+&mUN1$TMXTtBf8LY!HvCT(RALDnNrf%?p8BY2Y++HQ_PebdP;$Twm z7TWUTOKLe&hh-Phem*q?*IUWGNvPESI4qC53S~Ylhe?J8Y}WfSV38%q%nF~4nn)Kw zK2*UxUS0bb?A6?Y8eScv)OPry#LsKNK8t?#owYXs%T~ey%WCrAH+ig^B{d;1{p(yv z3tplfVVdyBV_A02_+;)hxkTuxkOQTr9fWl!Nm<(?Z)xpKn(V^)_i?`ZtM=fhHkgez(E%(pab$C_Dq{d;$I4A&2uOJz{~nLJjHJ z6AhIvB9!qo0Yxx3;J(X7>_o2$WNNV(J&TrR&aBWyu6t+Vx?aA}6TbZU56vsMK}?&! znAbB^19iO40Q>o;;9Au!prwq5MoyW9UQFqRAh-bSRsm$d@7K7zc)|kf#W`v=liw!x zks#7-S&1B8Tb%SC%T73+183h0!F1VyJIML-y;1TfU2t&wYo=I91-c}!Lgy5v>BOC@ zVcV?rP%Sr;iP*3ohA907r<>cLd`~=EeytnxI-+_EYnEsOl|7=-_;;UxYnwW*Ls6M; z9%Z2$%a#z)m#0!UMh7DQcn{`dv?N>ImW?Lr1Vgv1aP)KADz0Nl9L8EPMul}-mPY*O zACKSjF4Z!-W7Jre%t)sHogJIwe-M0m8pzJQvX3!caRbHY-DbGeQ`p+e?L^rMU7)cs zjGeOe6_UJt6zd{TKNJ3O+rq{|W0sv^&vq6I_a#&P;P#b=P^6_YHQFV>AWqmJmXzg6^M@lspX?|cVS(6JA;`yE1mFW)D=y6dtj`oDpcmIG_`xs1KJzYV2^ zPo=h&i~uh+Rk;hUg~Iq%eAqA1&x^jf2e>)!mOi}kF!-gu5KYv~!n!U@=RovUf;pMw z%*I9bk)vOyftR1A&}X*Zg{8kAfNk%~*kgTmm*tvWDxb?&9&2PQA%! z^;R?IO)ikWo-~@6BR<^cOaB9Y!TZ4mf7nT;EH z5B97)1n(qB!J(fALC1!BSY8p&%4_oRT<&M64PNZ@k+z*;E#ofS&$>Uu3d9o>Ft<8H zbX^O^*sU4@IDXP=u>3#@)sNnV-&8uz={z%@^ZI` zX$k8ot4#+9Tkk5ohqPeg6v|b1t(M2}Hgd$|cbLBhp|2JZ=6|7@Z6k}mD^k^dKIAj6 z@l?`o@tw8B5p!{wMkjmKxiTkl9;>}d+TE9CkZvh{`2F~_#nh}7`dA-!4J(M7OLWN> z-u}c=pG?Y2Nqk0AUfGeB95+I1>aUhz8Gub8Y4Ik8F47F8>!S>?9F&zb^~hoAFwH@0 zHL-l>ck&!N|92lr#CMCYG+qlWT^C6PR+(a6+VdW2E3M3=Mvg(WOPC`S93x47>3pL# zX(WPJa^-xXWU#Y~NVq%PRiA-+% z8D8<12#goqq($~BefU51#@vqpHhuiaU$5CXg51`(8iTxP7J?BQb+o0w z!s0G<^Hmac?1A{~Lb{;}sX~NcIhqzaQja?xYXYT}gmPgf&hxo8m^6CxL_2qC1ZA|_ zOViakgEZY125+~Wqfh_6NI#7EighZ>eA7yBuhLxp4)Qu)j{rOKlC^hhpQApQE}%UU zU!?n^7)B;#k~AjN*K z*g{t2h}U5i?0KHqx&dA^% z9V^E9udE?cih8wCTIb1rUtZ33b+T#S03lW7NWAg~xI zI_4w%VGEX_OQw%Hbibaue)TPxo1{!BrrPsH&$HBSPwC};h7uZoj(_EiZA_#;IO(fQ zTJ5ENKsJFqc;Oehwnjo*#DkH(C2^r%gDKLdDUV_cT5{xQ+T-V9>S|~^74R~H*nj61 z!F%zCkU_cB<*Ib5Dl~y!zG6F-*VcyhHZc7m@9!P)IeACR>zWn&M@z+X7SmGWKag6j zVqXVtxwkRT^QM*%gANLq@9b%tv2MP)@Ku~7j*z`|@|Yj0>poSk^o+7U`wFKwo_R$y z->4#|%zwzyK5RlSBkxjzr(da+`@iwbN@FputD+pnESon``RnTy z{A@TFN7a8^h=pP09bN8I{)kNQ~kS?cj7v0nud4-xwD zxXrxkK0VyWdvj#y#(8UKE1^#^5zVg*24lNzadGoSE9l9O|D=9J3)~Gw_ysjQgoeav1)-^_XG| zLQ=4$tDYT1f8Qg? zB-%~H=@XbZN>?|UqRnPA7q&HkSFWIwdKny{kzv2ei->i(w1Uqequ|#-D z%_EpyuM8-a$1kWrUN1E+w;JPi+IC<%&mB>;hn?+kJM%j>o!(>Uh-nsVJ51L<6}Lg{ zMgr_v|kBgzJ9CI2_s_Y-itSx)z6KA(@%nmj=4=9 zlDbX>j&3BEIcnpwL_F!ffdD9pZMdTMGpBxF46Uh|U_nVP6Mw7-zjybikn7Eqm=|*o zV7|EvH1O}G_-cA&^)imjn)S?LqanJeZXJ2{@jjgAt4}i{^rc9e^>LUHi)m@sWHaXM zVsTxxyy&EU4er2nqH;yS{NhX|wC*6|=n&1=m%L`+oMdo0D4SMjG$T?+CE+%|s(1`{ zi26(`HI5n1JH>DUaPe!!`iN=h$NeSXm=m-Zd4i62UpCxVteQ-R+iZTo3A(6!G`WAv zM5^lFEHJr$Xc#YNnmt)IE}H2)B>o2XYtf-4KL5e#T%G^{-8nn4!tG46=UU9MuH1FZUGZ^&9DyIlWnO{F-}nEN0C@p zpaJf|gNzAB5@yOab4-uj!*YG}hpccx)%af%FOCs&vY%;0ZjX1rjTp~izA!q`_9uCGD>`Y zC%Z%34|ccZVVu#75m4-lXWA2UsNBgJjQ$%xQ1~dGns6?ixgirt`aZqDEMb)}uN9p~ zsc#)xaI>Y;u-+qvCX*{~9HX6Df6|-W_tP4DZH9Yg7E^vKkY4WlmhiQ#qm62|(~Ws5 z5M5aXJRu#tQluiP(%MjHnValr;&4nS5 z3;%FJneTV*lPfQY+xf!LV*kb>eWu+xJ1pD0$^nL1y$ItRJHLp!{%I5aP`TH^2 zbHgBgpSmTyA8Z%`#Tu}h>L+FqhMGZ~zA!cUrBe@A>TEn;ac|5s+ll0NNg_`#f!q;rFT$y&oz)4 z-ko;_X%BW8Hk- zd=eaSE2KLhU~SIMWsWUFe6) zaX-by`mcNV4D4`8$8+iPgoWTLZx&=+E-=Lt*Wx(Yv1f=$#fOINKSv&?;`}+Xny^eg zayULRk06rN+VMQH%V0EHwrDh#Rbjp!O!RYLPdRr0Ki75W&xrz_y2&`$6BA3L9~91? zYBdt2|GWwoZB0XF?qBHA4^2$Wr+MtJ*JW_g(JNqya}lsxPzQ~q)-zWf^QfgE$C;LQ zvw1>)&9vjrLgvRoe>id8MU2IDDxkmJ7yF5pI+%p?-zhMMfq1Ma8wGK@fBu)kygi;K zUOSe&ZbQ!(buy@LH482cK=%_<$Sp1ASkB#jCd`H3rvNYF5dGwzG>EaDO?S_a#r=Qx zE=Owl>`MGBY7a3LO@KQg4L&lFVJZ))F@8T%V6)m~WKdFr#w8XrA3rb0JRb6t1lz+> zVW^-EKCLWd%v+=Af=$_=d4n5HpA@nkx3SdQ#URDJ1FW!lg4a?HN>314o??4G@m-tK zCl`mvKeu2*cr1KvL9^#W1BUz5(=0PGYvdDpHHg4i7i_yB{BaQk-jb7e^?fIATCmvn zrikyhnc{O>O$rhy<t zn}6VftrpyrFod%A2=C*cEW~A9I$s8MGlqm+stnf6dbb2%)*S#Q=$=G%5!T?>mRc-l z>|q=DdvgH{lIjE2MR!=Z*o*>+&B!|}3g-*SS90x2m3q~2J2lpHXgTRfA45+Ca0_7ExI4uTZKdoUusY%Q&ZuJ7kiVLQga8{M~-Yvv@3vH#$xPdqlq z=NGUP`bycOZ4I`a85kaqg={xoFQohqRV}h70y6TGE5o zeRv1+xlySR6&c^ z9Lq+eX|SVbhJl}d-PwVVV$^dj8-)7}A*Qp4luOfv1+gn!AO)9x0)3CL#C~qrgH+MM#F}}9xTTaUIT9bM=o%|&hQraBy}0q!-RklAWO-Q z(AzMA72eDt<^|+2rMC;nY466up-2~C?s!v@nmJ_$``qSa*ST`g~sVND3L@W2SLeosHPh2Hq5$mj~^ z(OV&nyr(~dVQK8E;kCTdX5sxgjsQ;Ujb@E=4VkViFSxpLJ+efW<(MfF3i73je#0%XI^g}CqrC75t zd)8aHhBYpo$>eKzBGu30v#JN~Y5|jkC|rLdd{bG$muRqc@lsU!)q&{>TExbU84HsV zHp7gooXa}5EdhqygpZn#{Lt)=N*|RHcV2?@y#uD)< z3bM=3q0UDaK+;?R8oMt5S_Lj=^mkc-dmUVux$8N`kQ3O0usA&?@x*9YV(I}8d`txA z^`D{in`J-u3{%LA&0%UpTopO?&rL>aroE+&#xH`8er=Qx@u+{fo-j33M~lWKK!HRDlhCmT z@J^jXN2+%{XQ^ z!jZdX!E@Qh@XAW@9^d@h@wl!MQcJ*?^I~6DsTuo$*1u^m|E44;sS*2!)ax`Re|P+E zo_|>)AdE%i4_y7-Vk3zSYYBeL9Pf+RPko z(`H-v_S^}Y6DgkmSCA|+axMjWGP{t>7){3BM*#@;^w@DFm9Pb!2KMI;0%hK5SXViN z*`{SkYp+UT{B1`Q{mW|T#Ze!bnEq{$S}0zlEHtR4^`AY$vW=e_jkc!FXEHa<$L|&g zzA;KYykF0ENkz56X*M~Ui9SMfT?d>0A-DNSWI1TJM(7Wu#@-G!(_kggB+ybRd);K+3x(9A!)*s%2*}nB)-A=JD zQs1++#J%$4I4yj_0hv92ccM|d3*dFNF!=H2FO;T#0m~!e1(~-K9Gc za0#zncwWVuC{diktK2B|PcEYM20BHciV-CPnAJCPwiuz&ZgtCOxB+U9?N$V3)RHF?+2r= zg8ZtHmbp{k&3?wkZ&C1LRS@iZMA0bilfWdolJZP6*JqO&M;}m&?z>AAfXd z{tRQd@ziN%(fkOQ#Qz6#mzu$wYPOgzVox*X_49@t++Gs8nyH+$VvHx!?SFxLy8#Jw zUW2+D>cjZ1sXyS^L@7q*qZjkZ)f4C__dtcNTCzp%Ifyhp2{u^nq(@&H2hV7zg25|m zq1mWW>|#=UN96OgcPLfrIaq4rj<$^T2eswTB#wcKY8ZDe}F`wYuWn^Odk5|5634WiWQ^y3iUq&Y>3)=2;KZ<`K z0i;!gcj2EZF`7G8p ze3WhC&V5cbq26t>AZGptWZ$=lZ9cslCG0yu9wh+d_dS#oJ5Cl_X*XfMcjt(|50~2X zG3#=hVBsVOtZSWVmSEawT~hM=M?_y-LM-?0WR5OuCih<-2~|$+0gnq8uv6@Q&^ijP zWZgDdwze!?rtbI~T(2U1)_XDxyVMB36PcdOi%mHgSG1p30_L)*hEb?umN)(C!6h_q zY5}vM!wNibc!uZ6J?T@Kn(Z^0!(0`@raJ-_L-D?X2>1PUC)jEu2fg;LVtr!On7iE$ zaKU^Jv`MxLm8=!|nb~m@*#2+>@wwxeab|VQ=5}=uSQ0@edxuaDh5k$~re@&&KE~7+ z-L%>v^8Z7f^>#z6CJ66a9T)2Tm<9XtoIK!6)MM6N=A)98i|}`N=C|d*fV&gdUygYs z`&IHfNS!hUIfg|s>9V%$G2jU6fDL471A&3o9;V^wtl>8MZB8_n%kB8kuztdO7C1h>_ridjr$5aaCpTaovgZEJmL}5Ye<4O8k!;@M zOmmX24zIQTOOuYtC*SIKp=fYXk%kYc;hbA&Ijo4HD4#$mh`+)he^jby9jypj3 z4yDML^Gt9VlF?!Rhu8e;am3GG_S`&vD#yJ2AK8&L38$-9-pBOo%Izu3XKKTHxHtCe z@y?DqjNfesJgK{xi*cFtFLQ{65(M$}Pc5N7ayAjXd<@2sXQrJa_eR z@!s*Hp54P3(xDsv4<9jE;(flQ^WsS3CD~+zQxc~2NSQ<}+h0SjHir0p70ns0e-RIr z33Bvhr99pr^`+z~{yt38(>;dVoBD*j+?$WfNKZM&$q&iEX+rWck%xn5Iz4sAVDz`#b;?m?`J30P~}fn;`&R; zuf>=XtgOj_=r(*-N zGf3e$hjBl0EB_DAvOkQ5^{2f4BN?$l?9<3gWgZ#5_BFNx|NyX{Vx;BucvyLRpd|Wl2hDAI8A=d!dDwed*vEzT3(>yr_&aiG*s;(zEVDkDnZ(i7mV zSj1hwVFkCXZYq$7W5>{Ts~#Y5iXdyO0myojBgP#(8-brKckd@ZIOdZHMULR5uQqPK zP@dHdil6QXK36CT=jjoQ!>*^v@R;R3xcT)uypD+49s8S-Jzv4zJq}XZVEaOA5PIzk zh7&#W3U1@gk{ZHNj~3&7&VfCvi0ZFTdr2%?AWt5)6vm0zNPv%Lw_`fpNS+8!UN#rg zOjLI6mNWP_y7=#YSM>hBv?}r(aU$R%ct7BY@fE!vry(nbv1v^+G0r(%oTvR-0y?{y zU>qZETmd8A%;#{UE5v*o^{0e*F3aw>9l|xla71v8y)NN$e;p0Dr^AY55==w5*A->K zCfk+3Y?wM+l*$Lk_#c2)KdZkZJ=qhtllVjq(_!inU2^a7=^*nC1(d3CiNZPaFnmKR zFO1(5S8pP(nALM~d0G#yrvR|zmb(~t5e|p;9f!jdZxBrf&Vj54p?hzd)Z_)Su$%l*q`1i#@qb77I$e+ zJw78kw-}NyJRI;?U%t{Agid8;=*=x7z&C4Fr>on2EO{m_1g|OkJMG0|$F4LTbjg0g z{pa+mfH4cX=(SLmk)FvF>IH<0+avXBaGAyf&%|}R?j!Lt+XLY|DSR%xYdj2eEbigF zwZcFj|K(-J*}=Y->^H|`pFe(o<&Xi?nz&vZM{q>-Jn{U??@>jDTaxULt-B76X?5@) z=T_V7P=5q2l@!A4kjFYMA!q0ipFGFbR!1$$>5bB&nFpL6MW}$wGn^eC?-uSD)jv_L zpSs`_2~ApN@)?J>jR*Pmeqq>tfwFc%5eCl{9@jf6_NN?kjENeAJIsgaI@SHwx}v&Y z@9FhE#anWRWe%ZJU{E>JLGjujprm^NT))u*GG)W$8MVdmXtfknUsZ|sH&6H4gW>r$ z826kvl8#bc1RS4{NWB)I;cqI($@yNm>{uT=izxiK7S2*%uanUB+BDDK8};)B&FAVQ zIP~*|5x)}7!*J81a&wREQ)o4{5})6?4&;E#K`AKZh9QV@aYS9yTcD*%IFQ_Af&F~! zdbrm?w{9XitY;W-p1#g8+O7vAEwlu~bIri_ISpWab)`eKfh1KTy@pJW(uAvL6ocCV zLBy%?%JP$6`E%UXBlz-IsAImw3G&vxV4bSsaD}J`W1!kaU+C6R3+-M6g7<9}atdYF zxHpeCYnT?rDI9a@pyZy7!Tk+){stK%0mTIddWxIx8lvAJ?})-Z(vDAc0>GTgE1Jq1O5`3+JjI;&_0i#5cR73} zSxE-ZoWLCUS^zHRNvO&HRtG!QnZt>SHbOg*M&8!>`@#DTK+oH72N>TvLun~&L zqRF1;OlN1xa=RX#fC(j$rbQFzY*|c2d>hfYuyRGBEhB*=R ztbO+M%+ah&yd}wy2njaEb3|0O;a3kB%Z)Z@ucGPH&FSR5KLd{ZigR%H)uYg7SdgxB z<2z=42bZpV8_QSMpp1S_c%--f(SX4;+i9j7rnoqc-B}1k_J84TE!9T^#A<0Al!v-Q zoeo0#ebxAM$#SK2Nh-Iut)Lbrx&T)jKDLb#;r%v~aGYOd!~eZ(h-c(6Ms;HT zL0EdVju^4~GuXA_p1_H3;W)>~!J+NKChp(2@(!Xpo-ZpDcB%Oq^&L!b*j{)H+UVT? z?-UJC!TP_zbnZSB<@}th*DutW5GFnQWzaqs89W;13vNVJfvj2?G|zknxba0#fiIh< zuse7fKV1o;`IL&#C%=r`{y`t0Zv0%K$;J;T8P^Hc?ze`^>c1!o{Re^F4=jJ+0_jiQ1Mq8U#k1T;p^PtwNhGa^$Vh?W#dBWk{xmrh`CRum}7@3Yvh<7}- z)DSHN;d)<&(&em*!=d7{d-l)wt|nDHf8%l7x9c|`dlst=JzpT_vUVffI({swycXf; zI4TGx?Y#qXPWw8>1tIcbc{^PHnCCb$vl%&z=y#Z#*QOXt2)}p2B$t&23<@+%mgSm* zpv-z%yZk%~88-m^@LcFSG1oDlnF#K91~Pq3H*~M>pT^r4RG~Ze$vOKCQ$sWehb`JO zv{Sfe7kpJTf1d~=Z}yRYYr0hj&O5@I0XtZCOAoE<{)TznD9wT6PT!_%1+8jx>rQgN z++U6D(?mF2m+68Q!!J3WT)K~|+$)LB`XVx|`XRM3+uEr`E*aX)RMd<%b*3CwM1Z-m zMlvoTH!u!P|r2pQW1b={2-_Bls@TrdhuDgaZOLC=9f+MjiDHBLAQ?SQ`F2? zI@Jja9h+hIzBH{FDpJxp^kYm1a(E$a$WsGuSHt)l`ieQ}@8skkF5M~^X>doQv#1LD z|4B`gfx4>sz}>M^ZQ1%Mn0v6G$xN7k_IymevvVm?LrFoF#I@MIcQz7j{GJF z)|jZD2*!-&BI}OnOlYV%?Qu1P@ESVC`R-KXIHYHf8t7`E8)Nqr5W9*kZ_RK#d{by?hJfV2aA@Nqf z-gu=D;M`%zDYxD)pS`TXaeKpO?BlL=qYWBb&2{|46q;>nA5J;Om^pczIEZdk=YnmL zGEPGOc$Bi$XV9B!Kqc0W2WDgY9lZv(Bh!a_&7;do$vO9xVL`2m?mWFKa&7xPd5Ud~ z#-10f6h(X$C1D(PyiZYbaWH*2HH=w!c#K?GwW79+{YNk_;ydW3u2WqG19E-~HiB

Ns|pe=$w&@=*#Eepfz zMsnUuu=V3z!tl;GsM4#AzW7YSJU@CR8}BnRHw6oAbgjkfuc*E#o%|bxuIS%E@iipu zy+!iMPIyk?58hHDDA33%BsQ8{PPj&@5yQ~nySJ2sl28)gW4vz=pWDLkHhP5bt zV>8P4XhP*D{_QJ8HkE9P!-m}-nncVAyb6MLxnme*)4F82)2)cLvT`uyM-oaJUPi1o zV|DJH_8Xy$!TYeRA`#sPYr*)x`PnaqC4#ws^*vBr*~fk25X`-|^%Q6ON@Wapl^_Sx zrk&9Sn`@NBb;7DD2(N;AOqbCk?ZJOBP3oz{qga@{UB# z!aAV}hHKH5P$1sVs@@1BovReV881&1MM@Kw*O=mZG7hZ&5%UupIli+qK*?BXF+81{?YD6YX zu^)}@pS2`UOmL8;gsvw=Wpd|RNbfRS04fjs(|f(OKnk65VDGbt(B-&XY`hY6f0{p_ zPU(yLy72q}#$5!*FWZJ#-XlS1MvyY!Ec#(qc`3$Oly`P+m#q2r$?!wD79L-t&S#Jw zL&tEw`?6-tTTwYtf`X+OzE0IqZt~S4_|V=4K3*WpF?lo|lc$E#BP-_De{df< z4#K^01ibg*-#JXhiuL#D>XS`WE~y|@&Ehc30H-&c@1FID{^|#&S*Vk@e`}$!>385J zCjsio8o+Zb_vjDec4!|djG;;{d2mLIzo@;HCmL|yicX#-{C_nHedmmZp@Jr;>3;;g zn2>^wFE|3$Ke2<;c1*&wylr(Fx0@ZY5ySCmdkyO^^I%vHLv~YvFwW5inbiZ?r2A`N z!qM;Czwx0k^!_i*qpgN@WYV^GNY|(x{}#1B)S5~3`5q!icq|2}$`8oD9k)Q`h;N|w zkv&*wEc8+HSC+i~?i|;5_(!zTSRd_#Qq;8ac*5fSNIWk8#!JFD1y;tUv?mDDGRfNp zJ?ImlF;;W%Z@;a#aC_0am&(>2IfdUX+R5HOpRP{lDvaF^^$J;;2vs--eOsSLJe>6u z^Qdbn58W79P86JNmt3267ya}&hxpM`;n;7diJ@7ntWa)Uk9dyw%n20p+FUya%Ab?L zZ6`PWA#NAG4otYm= z{%B94>&S8P;;t3aFk%5Q&S(x;(zP69?|ea6y2XoOeTnuU2Zz^VynF+?$-ZM981~q$ zH%SxE(YV~nNpsPjc_w6h+z)c1R0evxiPdR0t%??p$-b$p(DQg{%zL4{3)$M^G5M^I577+ePl2a%Nx2ktYZqNM^KZD zzd@hplX!KH>u?{xs@u`k=a&*`lX%Rjd{)*FvYPb?C~C7~;~m<(ScRv1;3M@skd?WL z-iy*su!EWH8bm!#k!IZI8_+>{hUlKpL3FaQl?o4K^RPq?;ZQnvTw@qJFSOc;0VgEl(yi+~o6 zbM&@-3dOm-hl%%(<4lib<1W>%evB?8ro%5|M!;Q9g2@wL9zEp8!*ugNB+scSm(H7ifoYePvKYkdEym9x+IVbRPt}f|Ov`?dKubfl5jqTL z|B5}#jlf-u`c;VH<`2U-F8@@3o_Woq!O_FCf6f=UxO**JQ6QlFwsCNNpfZnF-1>fln#W7+*~)zy7mw@=G>Y7^5rit!6LOB$S*8$rux2lq+zq&4Rl&Hb#;MG-E& zX$oY{f=*~$#E#b&Qwr$3Qw3B%C?t%Rkg|DyT*Y|j+L_?HIDM59^qG{CRNSB0n;x-`R##%oi~OAGUFX{@10e{X*%g4ofJJ_JTS%WfNM^GKV&Lb(Xdn_lX+& zWPqNyrB=3bQ>f=0TGFzE>$dju$og=R(-d>!Ef;w4=}UkZMP(tE%KX% ze$ighKUPkLgYIW>zxe?XOpIF=N9!AF&vUTE2ieL*F_*8)zznU^jL`NIxym&$XVseF z38A0K^)D4s!IpE(ZuPg)qgzkFven&;>xwbVS@(;W-&{Q)lxsg>o?l30LLy2TC}oU! z)DX#Kg#P~shv4VTz1z2#tIM`h&Ko6ouA{dx^UF3axl542%&a6F8`b;){QO3bGR0ogJ zyB0t^pMF+IQejIrl6vM+Zw4)lR&V#n0w@k8~vs3viQ4~E(z)6b#+vNiw=*pOok?#<>+5^JjU}^ z2~%-?IODZEgt|VAPkB|I1dsX?;jVA*QGI$6%st0wirFYCm_J5`O{wv|&iFf~`+5TJ@U&nm zChH;X*64%+R>|;M7O2p*7rs-KSre%-shg0ky(;dnXkE#C$jT|cjAQj)MelB;@abJY zKf}a9Q`F#oRYn+_1(&xuQ-!JnkD<5A#^SxlDklyvX-g-#+q(rN6qPfVb~s^LY;s-8 zD=v`3z&XXX}T zxs@wrf$mWw5iX+i-duyJyt9xeLrLp;+V2wMF3M9Y&t`~#I%;R%B!jUMkLW^vW~J1eEyE2Jj+9VV@7K~VYkpJ9_r77C_Z!hi;ylFP|A$YO;rSzi zdv)Um-ItNYC^zj8z2ln%y83An^Ec7MIPddo7&d~S_kJ#C;=_~GL}kB`vY61gPfxPqgOsHT3HuS$gy- zCEUl`y}f9%R;^Jp6HmR4&%$F}?FdN=;|$VUFXvG&-ZS{!V8IHK`!q~UL$ae3*RNCe zpj-KS#P9u=*`eouPGf$C_u265>{wmLig3o{tRJ6OHgh%g`eQiLW_yC$w{JiXeVc>& zcj@8!#reaSelH1KPWL)yR^CB0{M;{UuA?UJmxT%wW?M}4pUkFGJ~YwCB@z(5hd}C` z8_|WRPvEzP6s6m8nL2x*nts3dAcmFrC13A#W0Ah7?{hp7n7h3vk*(QZJWo^|OblJF zXVUka3ozfRBLi6eow>!p8K0=B0hYY8FNQI$oCctFKi5zMgVULYq$W(Fe5b8LaE~e- z_`VvZ`RvE=NMYPiqo>~ZSyXnOoE$w4#?r+3c=V_7A&NFPFxDA)iLo<2$+QdoM&9#g z*P~&={lvnbb@UqalHMd$Ozk?;&&%*?(5ud7Z9CfJEqP4`+qfp5C)3%x;thrR|D?;A zThiCszZnL09p(wVPjSmWU*&q#+%{+y#(a<*2&7lO|H^zHJx)9xy)%Q!O}DJj+IwDj zyz+)V(ouiB>32%QU^xFc(9k|0=G`pK`*^JIPrRa+E3BX$HN9x50ZDpx(RJqWprj!` zEy1Af8(-Ja)r4~1(Sgg(-(Nt@GBIJU>W1i5P2n{D~Tv9>TakcwU^6=!SyCUTsmoMd`@$k$7FBo7$L&(F+W-KPzIoxA%VJmR*uy z@+-Jh#qwO8i`iXN1EI_qUVROk^7Lu#2u*sGtudMxWlMYhNu^>vlj!<(eV+8|1GpW% z*1JsDxK~ujRSwhfIE&ib&-w)swK>tpj5*xOGgb`wOa@jOGtGY#5p2t0no~23zc(Au z*`x*|b=3`IsIYs9$iG*ZxAZPzY}R``ehwHVq3-euX83CbHZp$xAE}}rs2^u=JVhP z`r%eh+Kn(_%Jv>6H*4~F?b1VxubT-iaod7dKB}1+Z0MjO=9KAvk6w=g`|N4Ib_d4c z(@5eqZK-YXCX>pm{>a#SEaBaBh-2h>EWtjjOrmJ?E?PPf`T!FcNrn&JPO%f4y* zzGIl--}CXE{j({#R7{XCZs?02OyIOk@^kcTDq+V}pw=EuH`%686W1=|X~!g>{ErCq z_v!HFcrM}^Mke5~yKSIEdq$Zyy1?6tf5#mWRZRk2N zo{`z4M6FMMMqNu=g2$oD>-lI*(k_%B7pUt!_6i;I^`DQ9&{N^`&5P}t43L{re5#nk+v72j*R5@8`k}`hEqO44=^a^t`O`V?Ceq0iGI3umx@9oA6SHCDV=aBj}G)&QW7;9iYyIMvK=9^F|B&E&p-j z32s}&?|~G5`oY{T#7rtd-o+YH(Uq)^*~LPiR@I?9shrR3{RutImiORy5Q^r{r~Wp} z@}9jsv0~`a^{Ak$y-@p-&IQ!VQj=za@h03Fj74f{W8wS#G~wj5 zlNdKkk5$}f8_UUYKOEum*tck;zz#gLyoqTRV&#imCP<)x*b8V=L@tK6wB+9%iwO2~ zYb7Yr=|b9CO(4n3>SQN12bVJ!dhHbDFUfWUjW7P`O+2@LPG;{+#cjKuJ`KJ(Uc==l zCL1HQ5kmWw1OD(y`4LbsT?4;=8Nq(@Z*2VM+ovG*1A?E;02441b-sL$=@v{#5?VU! zeYdFHV<9ZP{cJ{(=idH{Ss=>W_-Qwmwb)K6M(QD~Zz9oqx2Ba~abXfUCpZf&i95@k zC8Lh%ysMCnov7H#;bzbL4boEQ3-<&pUJ;xnKyhUbd6#nsZsVGw>vvlT>qV7B_{>Nk z6X4Fx93P6uiwM49Kp%8$=@Z}At*z?8@K5?DA*$>GJe+j^H}$cQ(O{gA2%+u9Eoo)~Y_R{0h4_`q(H9 zmZbmlLnOi_`kNbE-+u~@R5%U8)9T=WaV#=b)d!Y(0i3cG#h9kT`+ph3h4mmfjKDBO z_1fRfMi;zUJ<&h8to~mze>!!?ej4zTh!Brq;aOiiW;Z^0M}~&YAqUs$aW)P5ppo^8 zxIZFz6O?BV_xA^DhNPLxLfip224$%Dk3qAr*$GZnsn z9?sn+c^3%%CE+nudEg|5?Xve6{w=C!ZL5Ia$q4Vg+8_49#9k9IZle7E5*cMdNnL&i z#%X%PGZHV5zZrW^c)3jzO$~(*hVF-(t690$1OF@J^A@4|WMPUutbr1ZzCy#JP3RXK z#P6d1&&K;Z*o7=}q~L~=>>a4SWIEb^QF!iqxlK&d=cbiJbDs+}sQXNIwp}eevONWU zmwAG|{{2Nv51NJPvS^DsFdfk*rrT-hfA$=SHwe_rKSnkv55kVCN~o-NAu21r&NX?S zLT!s?@09-QV|}hXrXgoR4k@$cpH9es`ID16rj#-L%G%rYx%$X+%ozup_50%PjL9QeGp8f;*P2T_>3-$LJl|wMUrphbh`rFp$ zMB;B&TDf4G3Ks1B{$}u z+(qbXlsF|ZB-G=M1@HK&WX#8l0RC}?cPn?PRZ&0qOo;pRykcLzjX_QNNu| zlJ8eB81JCzGEnsiySCN|Sf8lYR61(Vj73UbtZo?*#`=z1yL18f>;_gpS!3=FjQ`YJ zO+x!mJ@k^d#ypw-A``vep)NfncM-!s-!~J9^#Ag2cFJbNd`IoaWGFZ;20AHxBE~FX zeHbPew}{u5S)153Nkm(JnQr*}L^K{d<9v3Iwmq6;e(gdKGW7ybAGrj!omEH0vTQ5} zQF--HR;D9*mg;zfJiqT1&!2~ZdW6pHII{2BPSF4SDQQPcm%SgF#yx8wl!-051v<96 za+3Gm74x>WUJ?M_alFn6efE+i`&nNIz3v&{=#(^E?r47u`noNU%vo^_JWG9qs#>?< z_YEHe;&vw{Ttq4|>|OMkz-~0h!I(;T$faz0UqJt{5g=<_AGs2!%QjYS#B(XG%u9^J z-K)l8x`_HDO24IuwcE3?eF*D}fMS{h{ftUUO^H@F-*PmtjoN9WaAp5<&h%R1w376gY zft96+V3dDcOCNB&$~?WglRBft-nWU~i&Dg1WhTTxp1OK8m)7#Ihd-iIfcJe_*rhp% z()#L!@s=kCN$I#nWcJO8Lc7S9RLYWEa&zQlMkY=k9@T9S?}a?q{F8Uy8!yikeKY~C zX%DDXV3Vvgb%Ac!*G0Y|h7on%7lkr7b7o>OD}QZ${}c9Aok2bN6S+f5e5SjXB<0ti z5U*SLg;!)dCBo>hyDw6U=4POqi(aC};VO)BZol+WhV@H(a;ga`xc-)_F#=Tt%tz( z4njRM(A3^S8NQ+5;zU+Qf00}o@%Vc`h9#n3Vum(;w!2wP$?LQ6>_zWgGnHj7^mt-E z|Cy1+v@T-DoiJW9N|+U&!`c2;|6zh70 zu39^Ryk4&dX3kDz7S=*~_bd*TekKr287D_;U0eo?fAGMQw)c=@;sHFp4iRf4_klpa z7HaQ}I~dkMt7=emGadKsNbL(c-Zhzca$gGnZa0aCNu|OV>^kQ#o+|JueqVLeit(CW z1cbh58Rp3h`rXS<#5ZmVrqjl+2Z@x&@u>b&5s}Yb3Zm;ra1<+8UF9uN#@wyJKgHkX zQUxbu?cPi6>$T!A`NhQ2pjayEfj;9?q{2w`Wk92#c#LDo>jTh2i%)l^+R)sOnWUTd zS0YL!74CVQgY%^p6yPxy4}(C-o%=`_{}<1>1erS7sY0z35xqs}p5qtVbm~jWEA%6E zcCioBptcM5N0et<=}esQ*X4K*dw^lBQ=LusDDS|1vPd&09ZykWylXaf+Cf!g#99gLK(yHISCZ+Q3QQwx?E}ETBA>Z=`%<6nKN} zecX$}d(ZIm>^)Nb%)cc6_jTy0^^RPqsfOo%#<0=24V4p9z?As))cm0ZXo>CyGgcEQ+)8pvd+1g@W zir`uW72-A=)ZDn7#C7yO^@T)h!%zBaaUOkF{T9&ni=>;Z{K2$h)()eHG-V7;-pjnP zjl|=ptv-v~@!N>$=N%`f+?mHDS`8AV#k&~C>KBB>`D|L^$a!w^dQZ%UgF>Opj15st z%h8|oJ^pTt$G4HGl+YHPnc-){c$+R}ri@oWMIAPjhDIV?Q9OZLGb@fcwr(pEVs#BX z%};06#mO-XbzY%1^I1$>(>m^n#0^yH+id#h(go62e;P9hhihTr&>ZIDlzJvB^*8;E z5O8jOC?KCo7)r_fSVLXA?u==Y;=c#4V=XTekQ8Z1=Z;~0RRl&J0z!RJvgVyDj{kG+ zOQ8QcIl)^~z%M_e7FXKvcXzIqCGl!3 zf%|?zuafAoxx~zq`%TA&Ym*abl95!qB;4Dua&r+cH0@>(Bi~*J2Zmf_qy9Xl^0E`f zy!yTJBt`6+Ow~{~7|G{jz{c}&)C;PBjz8=Ke(CMw6BKH(X63YTi>GB@%AvQ*PLcLhyajqZv(jD zo=@L>bRPF%Kd=W$&3(8Jk*EJM)>)14$*i$-yUr(q*qy`FfljI;ES8!Zas|!&xfrf= zzb_uMUMuEf{4W}Mh}W8O6hI}P$p(DeV0`}5-^=>Aj`sTr=I_1;B$v&`b0jrCkGb-@ zkC>t!4nAA$VES$tV|?a(l@qUN(;_s{h6D|}bQwEvXXyt1*77Q=$?5Y!;|Uw8`-zcu zwcjcFx!)0ipRAn5E3-m{PY$OY42JC_tNeL_vRlK9pZR7}pAUMW{L8ge|2tdzZDYR4 z$({Xz``S4qfh<8j|K|%+dEA7So63gq56!P~K~7Caf(f+EX0FuqLhSfXV_9j1fTz!bsY=BYv-#b3~u718F8 zADPPCTwO2N<#L5fr%A%J{b5=bUXmuUcXHuN-KUfnMH`yd$Jgn%xZ+GC)bh%Z3?_5TYt{Hc?-c{O}OCl z=f%@kF{-gk2X zImms!-ezu1}FA%cxj>Zsn-+HmHI5@le@#g{HZBm zYW#4fdf#2%grU#m7yb1B&U{8r7y7HJ+e>J+)t*EPUTc~N_5V3K&vOO)$K~3K;{UvV z*?_hgB$2A_KS{Fu75uAkTVSVC2qTdNcyvvV)|q@BWK>kqdC@Ui`_g>nC2SIr=ed#U z6ZF@RKTog0y2C5-BN=%EV|8ZK5wg4@f!Z4)R5;OP&yF|V25?sS4~geU%Z%ZWoV69dpw9bS8xY_A$+f`YjTdDelHXO%p$Lt=3!;Jf+&A{3@Bf@-kUy%x#zFH1q@1K>=}u;+SxKda z{V?BPrS~hN;b@~r=$T@bcusDu$l}zw{8ny$Ay3UYF@pkOiE{Ic<|{pjyoB5`(lkHtOwtEU>9}uKs^cfvgfz}+~@f? zE|STCE&S~j$p*!pMQ$hF?9V#i)A8hM_Na+8cJLt)teR7E-U}@+fy*=aZ zKy?M7Px`uxNMv`CvZ;=*`65=%DBO^rGI=6pt&&0d#%jsEHz7gifm8ff8BK)7um$$^ z9iHHPu8hLdLn%bo4+qtb%oM@Puso7Du+v^+j~&cE1$Y58&R}(IF<60 zBY1SWR`6%y2(ab#AEUm9HG-~FQm8XmS$EYRKe?TEuECQWDZ9;wegciFx_GVcJ-Upv z3^Y`+=!s$)5AFqaLto%6<19hxETJHIL?k@_IY!{2c@)T3X+WRNR)U^vX{7ZaSg^1= zQ{MH?Qw*cr%@v@v=0+Q;qzH!#vC`rDhEVu|)r5N1Z=_n5Er`2u9_}3G13(@3{+_PI+x|Qnd)<+07#XecaY2&Il$g8to1(6+O+P;#28>@S@TCsD z=Uy<|#FsITvs>!pq&_&x3Xj!AOVWXXr3)}THlQ-8ESlRP^^}~JGYS~4e9X6=7ecHl zSV-M$I7f8d`_74Uyr8C&Inu-{;22QMzeSJv^2Tm^uM}}@?qlrtTB}Y?7-*4|n5UN1fuBID-WF2dKbvHu)lGh200UubNxeK?k9pFY|V;o_h5|O>hk@ z6%an7heq# zuJJ8s;_+G9f1iwGGQ8hV7M@29bI+~AzQ0AhJ#nN3uXlXOFykZ5BN=Gs!wmn-XomRF zBERgw>$olDdg=iazT-L>JogS7T`LR25?S5Do84{bNybcK_vuQ|Ja-J$JIP!rd5xmn z=tCV;>q<8MPf@}&sxX-KpQw242qNchC5Km^LvEuiD3uAGc#ggDU}J4*9&E#7K!n%- zpBjgSQQy|~QpzjY{p|mrw{@fn>aq+1o8ksw^n?uYzUInZZK@`FKZXy+Z=f!jT0>tA zR<~H|={+jGp`OYoKH+`lfAGexW&JPw_gl4LGTLPEzxt8YM`*Omya?C3bUzyZH};rC zjazjUm0g`mjc{RY$y`5v7k~Rj@a8(tCvRPLfi)h=xUYl8@fgO3)=1fj zpBgc~qVh|M6=Z5Bt;S_E);Xf)9yXqUi^3yRZ4`&D4l_o=H3p`SVDIgsO)um2e$Ha; z+C?ymH$0<;Q3~*i*92-_6np;nzo7=0b+ovhFBz=wdr|rA$J+k~XVE)R%Bl?hA3CL~ zG{YEmE5iHvKB{EwVwuy5E#mJKTSF+b{Ph_2c_7M+QNl2V`(JR-I2zI^(MYw@nJU@m zLJIEzNa-%t9(HXndw(ISZ{~OBfABE0slvaP-&skWYz;*t2ig3QvyOqvKmG7j?sAOZ zGdu17!Si}dKGMoL1}mnl0c&5fG6JV8XVe|l2wKj%V3;!o|G<*#cKF%jWG)C$|L4P3 zgiB;yDa>B0i~Qe@Lml>IXhV`5Rr1vZ(@*F_QoL8N^nVB~bXi%osNSqZM@mkKl?{3e z?Sw_|nrjv58{HwWw6I7#&i7_Yi^r^}%zw$z(t}#0$B6qAX3hndmMI{une6y_(77Bo zZkdMhep3Ix{uyRKJuyFo-rA`vf>Rh@hpxr&o>e8zdl6-YVTbA+72p5;jE=(aa1p!l*u$!C%Jiz9_3iQ0#@|yMpq42it9ch z#$vdux`k)r&wl9l>`PSKGWK3}!zBaUfB6Yklvm7KxNtxlmn|;Nq%@)$snV(n@mQU- zvYjk1W&H-Wo5>-|@lojW6!x5cRIwYE8+9p?a6ZZU9v0zR)%Xm6P?rnixXEG#Ez~0tmlKtZJ*t4|q-Z%y+`j1jf9YNOZuB7g7ia$Le{!)6zCypF z|Ciie9s|W;tp2*_RY>4luZ~W9TErSdGV7QW17co8qk0Md9 z!7`!kaRw~Dbqd}Y&CZjR{vUDQj%7?oZJVp`*)6}U8d%6TQ3{_W{-@uWHmRs`C=x2a z)dV}vNrGu@>rs700~E>s=Y);J^I&n)N6I=9QCq`xXlFSeas3IQ)*$~??Z3VXJ7)4A zm~uJnD5jTw8hiE-^|_=?9rt&H|3BH7F0)Wv|8c|yYU3d!hA;Ffi{<~Ky0uBM)Yg;D z)OOub=unj~-biHxZX?z94u&Zzv-vhV#zfD{rd~vcd)f8NO@a?692pO4)$B2ovSJ(@v={n>z0jl;F8P?YcCQu;3z(>JcU6b{BW|KYyz+5U z_lAPtl18b~=wIE`eB^?zHR{rqM-%ORuMXo|Poc5>fA`!^Fx)mreob_jf_X-|;NF`9 zsI|pX`=W}R=9GwV9rxeM6?TS7OOD+i2$im6*$Xs!p!)Jh&^RdqzM5wu|K$QeX&1Cn zZBr{g|HGtt@KZ`PrpJ^qP_TS)Fr0bVkQ%mln%Va|E^^yvmI(%Qiiw+vYAA2daV;rV zX_ME9E(m?dHx2e%XrHz?m2eY$K#MF;RF_>beLY-bXJ&8io{svS z065x*GzPT5=3BOzSjonbc zvo@6Qc9pyM^D;SdnTh&2D5ob1HA#c7Ji%nGC@}oi7xgM-6FL8DIm|+v zY~HHFLIqggPGY}3J*^^qI@7H7nEo{hVuw6Z(?5$FZZ1S9L_JGvWp#HQI8`v zu2fR<(2QiXLu0&g(vJ1Aem!plISsSzxznz|vOUXCqEiqUZxRZP@?22sqBDZR1Cr=Z zoEHos8Jbd(Mm>>PJa^+@tqrMW3bz+WQ^zjw)CTpufo=3~eE!#5pUWh!Q_=V{ydHF& zaiqHxrD@5EFwCcoquV(8Pgq%iXdaB&69JwZXXpxJc2a)ck)S{@BzP120p*o7z@Qcd z?ay0YG7;-ZTOaFXDohqB?iwok-GOoO{yMOS5VokCU+!=HYIo$nckD9W_ ziR2BGD?O8Vj!K?N^7^_~$SvJHMfsV59JO`MXplF;*h+-6C@E+dEQE1oEG#$rJ2{2wd5x%`-i3EF>fa+EC@=9Nw z6}I;|^X*>MpwWlT$hWija-+5zKnwGO3XczTgBSd*c5$G1s>Ur# z1lxBmhMl@8_6H-K(CH9l@@GyUO05~E@uV?`Dss@|^%U*2SE+6lc&MF%I_s5N+zLO&H#M?%QW2v#Y*a2Ni4ZGmcc6{4zBJdCTlt}#4Vx|*^+HbHfTGY$Cs z5b^o{QDz4Gaim%Ba{LEw)0Oim{N-z+=wl+KYi;N7TzN54baYl%6t2OVfvoyuzKR1~awN09e}t?Xu-$&hnKr=q*$W4q5^ z^Nlx*_<(MW_>HnxhFFTm)8!lLJ}y4R*TLt+ zeE7*yQl)FdMdI{o8b0xO3c|UH@+)3v6MIQ>YPe%Bh_={D1UU|<4Fq_aEG|r;O2^HX zn_MAbuUWlAX2Jae+-}3MYvixtpMa{~7um6rPGDZ43xD5m2V|-ChlFZLY8}HW!OT&S zXmyV!qruxl?Vbd7Orom<4F z*;?}R&ZP)4f84_M|5vNZP^a-X`#k|e#s{Z_P?`g}4z;~&QQXn*Fv+!BaCdAr_4?&` zIA&)F^-EzcY18msuv=M%$}(SRI_<+`5MyDFye@ibkDO*N7cy^%xvf`g*tbJVLBv;4 zGL6>5dy9jY4jEbg2xoj$+nHmgwwikVp>R^-y4(mF6g`D*EPxvwY6sL!@JI5r;N$&a?rV0C1}yrr1&&%U5y zt7RfvCg!ZIE*|?MN4!(~xPAtQXO%?1({xdAxp_}&-;#f}&sSf%(tEGxGMh(-P@!cF z;=P3k|6~=&C@C;hfXNa1e0>LqQ)J+tuk+v(V;M@mgVkRuKhaHAhB1mwM`u&(>nkX; zxSQmNdsfVANe&$9WqlKiV96Fe#rTMxyZZ^oP(GKm&%Qz3)?&zxpG9Ip8%S#P}>E?oU$~d~k@^x8)~Y{r51rUHL1{6Twa3iq{isXCdT--`EI z*K2MQj}P0^=k&9vhob_hzYtj81BA3m^SA;%}~|c z{fy8DEc&zNC_R2mK4<1Ch90xX3TPzl#PwYdEu^n=XykeKH2ol$N4ga7VfE7prY201 zK3Dx0G|3FJoA(E}_(=S($iG zHduV0{wOi_*>lXX?(WZu~n-@fZFtFR#Nq(>H?()km13wvlw9T?6;7 z!&-dj^}WcKxW4Tu>i=+sxV*swdaHhu@EXqgv(lL~L%KLSOMIs16WWNIA-u~f$S1<4 zC6RCL#854@Bol2zG4~$Dz?E+gU>t3s9V{tDN|#(z>58T#pkT05L1nNC>J261{DR3R z82*#3GE>(2fwLznMDl-*Q2$>_EkKR3d)Uv^gAUHaqIjo0-jUiELko(0Q%qz#TuY@cCtusxJl7Q{pI}MA8wO81fKcaHW^-kflf!EaGbjP42 zm250L6ZD2q4n79Vvc3TbyZ2LO;f}OJK^u7V<}V&sMt^<5vBVklP*x1*gY_a>+B=yX zRo~5kMoZ52Z4=NgB|bOLg|(Ab?_U5+%@ok&A6;}~i!`v65{6nhF-BH1VhR;RpQr4z z#!*i^30}jzcSMiS=0iP|gZpu3@)mM=>n3=5mj(!X8HuuYW=hu{F<|DZk5u5=q>_Gb zDW&c|x8R#e!%^Qt6VMG~XqhdFxG!qmYr%tUvebrOVSd5QpBH(9M0UEjgL@5v6lhk&gm6Cc&wcqT#D)9AmK*dLyyJ&Qu1%I_J5}i88GIb>=o4S z?!`RLQ*7f@WKE!HQ+4o1C`)b1ib7R0Oz4`+4cvFrJSf+S1z?NuPC`;jnkulZB5RLb zrrb?qn3CW&+|C{YVTQBZvD*U24WH;s|;S}614&2Ys5u$?)$>;dg_aTJF4>VPob zj>k^s{0lv%Uwbkmh%JR?@ARlmsS&ivF;!0X%stE!!zE1ZxRt>BcnlNWI7F{*u1Be* z)0J8eUn3WtnMrY4lj#7#by;(u%e-1p2rWIG8DHgd%x&uqI;FahGdSFpJbv0xYRG0R zHGMYg}oCN>>E3B@aqKbl{1%- zTj3;by8IdD^AD1|P#w&6wmXN@hY6bM%IpZ zwfcFk)6PQX-rFv^;?@dcjV*`qae`!aSQKavw!-CePH4;29E}Ex<3=fRKV7Eo^Y)AH z&)%&F5$>0zbGMKy=Q8P;^gXTTE7@kyx0Te|6}XRY>Dh-Un?mgvV%bhyiGy#voyI#WtRe9&80k zaWFs8{GYbutAeH}$Ol^n&^f1$8-tGRPfUf*v>^0;2l+T0y!VTv>PY?Yje0ym4?3_fxB%Z>IQgh&I#~Tr` z4H%*L?h}}^G&p{DHx(7b&l4)pYxoRarWJ)A8}KCU;{-8}Z8dyZ1l{1_fpY22oBT^` z(UC#(u3yGaw48uW-8)T)Na$dWHEOh(yB+9=%R;ho&=JT?IJ*FopWcU-7#@JD)7j+7 z*}>48LpuC#2`;^#R|AQNJ#+EuwF6jO;WNxWM-G(hRrwIq$G^=1;b6lcO}Cn$c_%iC zV_Ur?Cm|F?5}&EaC5&13%c{f#fx;MVk|9q521GX!iLa_)>y2 z;m7m`v?QA%k@goUK+aQF@QaI0O&2C!#)>eLh z7VUCF>z>nytHq0u#P{0h*EQ4FHG3Pfm?ld4oESi@U-0(l!|G1M&73Wo@@8P zYX>?YWG)@=bZG%~$P3`vl8--XMZX`i$2s`F{IT6vXB_B3xUr`2BBL+F$hrOauI+#D zk8f9ylV$(f78`Dl6*K5FhM#+Og7noNqKKS)m?##D68*S-3{JD!!RLEzl^p1Y(+5lm zqu#Ul!x3A&!&D8QerZISiHaf93A)gij&p=uQ9E4Y%Ikab*`n~B-gdyRaZnV2%7r1J z0lz_6r;nwhocUk0-FP;jYaU;Lrs-+oih2FS-+i8_`wiY1#>D49Z25RE)`LC9`>qI6 z^G@Z9y6-Y0ug3)e`DzzyAUb-*xc}Tx?PF*n26td~glU{Fw^!al;V(t%RdzDr9GeVoeH1@TaxW z#Obv;Vog{p{?0!KotHEOLm32m^1=z!u0WRf*px~>D(2nWcF&-bxo=!>pQCT^eK-Hb z1?JLiy~q0&dG25-y5Z?5Lf-zj$hXCJh@P`8grCU*Y;&avfz)h(tHXJEe9T!7$W%~X zh*yW?57^Svvq{bG91KG}bJF4VJmh=NM)IxX8QSGhYeL}dR_JR49pquso}nfI31rwN zW%5I)9njN6vtxL$xIKBkRfQa^okvzFI^(|{u{d{s)Ctc0KiVddXtL~w2RX{|qn)-Y zlN5l8fG?h&;J3 z3))QZK@8qk;RX8YC0`zv`u;=juveG72*HiTfW{r0DnuuEJ#wA0Bi)W0imsflM>zNY zpi;CK-A2=aqdjhtZMVjuuK~@pZ(_VN;X7@-GX<_ZNv-o7tg~EDx1mfrWb6AH&}}Ky zh3(7=Nw;nhGQLz8D*sVS%xF3y(SIw^Ep57Bf4kI>4UgIw!i<1gk)ZQ*B0};xzuzhL z&)|x?BydNyT(UrjiLz$};xvwrnDpUP@|6EW!hmDrC$ZuM; zBAo=ts@xn&gWNzr(c-LUBfnoDFD@6N;V$n8rKw(`{9*=KThT>g(gLVf=qfm^x{~1F zoAcU7z#Z)h;Ni${eRoYvV%Ymne)z?X64=8$n%uZ=HL)qp8BY#*h4sJ9L+6nH_K9Z| z$)H{t_Pq5WS3VVLG(6eP0x~S*t*b74;jK|^89|8IrBn`Xza!w&RiuEF^I8S(MUJAU zw`Y(AVgg{^arz?%z8AQ(bdGZFm(G?3|D8}Xs(n=w;mN@vYtx>gPTgmNb}o%cA)cRD zg?x;TA&2LbfVADeH2MAN>Ipki(5;SIBG^O}c{BL5x%AVMwKAoamkOWNRn=<*gN*~CUv;K$ob*GUE8iwSlWjC-Uk4l`bh*Rbj+K_qULD27n#v_ly5?H9b#bdWe3dKJX|#Tw!M4xS%Nn^qaDYc-vGb1{qPZm%Q0o4o>cJQuwJ z^q-^R*2E3PCeS`~*&Iyp+cCmTqXBJyrV4ar!M;2wb>~+=ms`*Nq&AS>Aaxm`R`Y-# z9_5yX>KVuaI#1QZsX_ZDP}hVV2}J9`aw;onHDRq)LWVzOKxSH7iNswR7-zo;N;9hi%%CO68_KqL3~*2 zQxHy+>r<~^FT{1iZ&DnrG;&060y?E}7}RsPtQLJcvVMLJmjeBmm zPFNonCnt~S;|dm?Xeq2pl-L%6_!9VR|o+OI^Jlxv}{*-gw#y zd9~{$q#FCLy=jL|6Bv_%2A7eAp<4-#uRB6XyPjC+G>-~fEKLl#`(Yt#B}fx_5&V=s zZ~fnJSQUR{%o+2?G7*zdFY?AvJvx*~qjNCqX@M%2aU1P{eio_{Sxo=<{+743br!UZlsk0AphlclL=zyr0?LITn z_%=G(ePJE8N`~X7ZnqkzFP|k0i7;|zn+yId=LRYI>59mHv;&kkz4I>6mp=~gMXNBL zT^+X_{&}iq)T#3EX}q$LcmMxJ>;p0V2zzqVFKg_=rnC6>18sPxl^S`SDTM?-6C^&i zUB!03zK{5meZ+YwNz8Wr2_io-hq|P~ThHvgxr5@|*(NlKXp}s&A2Vp=;j(bcvz+sT zbmqR;5_LX?ToHFvJv#r)+rV4`X9>gTm?^*Pd7Y}bkdS^A3mmN;|id-R! zgLs&LI~03~w{mAFxgu-IEPoH>@+X$~)Tl~m^bC?IpU>008WN~Fr*P^dHiebo<&@2$ zFXYvMkGM_qGD>ZZ0-CrokO&ViB$M$X(c4jLsYo*wyyv(*Rr(}Xbm2jb6QF8NzCeWXTIF42ycr1r7wjIU2|94zR2@~A& z9BR24O$5;XVxN3lI5uGhsJehXG^8C!9o#l1df`kvzb}b7T|gK5SpnHR`S>24Yo`YK z)VpG1xTCR$x4dQUH+MXBu={j4c8NBAUR?Dd&H z|IeZd2q)oO;zIc-6|(Luw)j{X;hpt>T=3F>a;$hl-tO2%sEQV2JD0r1-Z;(1{RZ7} z|HtpK(?~695ED(~_$mcFdOa1Ra)rK7*Jt%0zaHn4F?Vaog4~7F>7+!eH)kh4_hS{d zP`8A%7!C#Lmv|;%vNt+G9R^Q4p_Zm!#h$CbC2xJah8iAjqDIPm3C=zX2d^y$PqGul zHjn;$KK~as0Q_IN%kwqQ)i;aUVM0By6h*ZUqHg3Xk?UW0>wns!1$h0@Q|Lq4Ot7}! zBzS?cVU}RkB9Z9T8B6Nf;#x3Y1kDqs4_O`oW89+Evyd-_m%zF~HmM9*5ZDAy&8H)c z6W?KK-&R1UaBVV3%QiLy`MEs&-_M~RVer(?@=_T9^;=h5q2-tgO*Wue_JhH^i7J{hn zNy;Jo_kZ~2{<=o~i#5sppK-kdR=>Fd@k|Ls5>zKdLc;!EeAKYIa8XOveq_p#XY0PX z#uyep&eRTo`Ka_h*K{Q zPm3FtuhS2$9Xcy|b89}}yKvVCBz$K(p!@b+7Qb9>+T$`j9pt`u+pPgS=vOJiX$kU9~C-qy=&evnNuiiw z(_!Sq0iNFU{oRV}`Th=acB_V=AP0V4#nZeyayDWGaN}QXxKBF|_OLz=$`39O2k9Op zUWAt>+=Tt}T|ghb5w3t%;uIR99svGx%Y7LdEpn!p=jWv+wHtN{aE6ob4*;HrXGZ`! zQ5O!=oVWaYk9gMQbRaKo{^LR>uzs2?{QB@1+*@z~iXZ$6c<{X^hTiu37n6aT_P{7_ zZRL}~hV2EUnWcs_S_3U7dZ#`VB<*6Hy5zI8Okm6o91vOzb1-bCLM zf$Q$^Y`Xt;&P87C7=}F`G5EZ3`3S0>kDm0H4fii5kbL{M@X99<}eML9O8$n`&Vuo>N&M#kI#bTvHx^{4ph z)HRTno4z#tFfGY`0Z4PU=mazqE)E-nD$@)X_dtifaL#`jg2*~iD}EoH^Fx3ym#!zS zi1XAoJ}oXitw?WZC+iXNmL|jx-~DVeE#K-fV zIo*>4_4Tq5L@r`HeOBwPfW3?K0DoCNbKp9=6++y+{y%y4i`;jog4GD7`vau;6d{Q8 zWo4ob$s$M#s|_+dZ_n4?MYjS#nhsVupSSU6h5UCeE$$-|k_`{H${=h-W!|=_~hK<0sK2|ct*2bJy&!lxmY6*bBwT3xaoU`5a#{B5CKX0gdy)Nzjhq1`sE8$Of{C z9fPXW#y9|wq+saHDUjwK%=_N>vLKGnZ)t7lgFG|B(UUCH22@ktC&Hw!Idz!ssUaP-Z19w`dK&?f=Qwed!9Uw>uj-V;2nQ zS58_Z_hjxs`!1DpaQWRJe-{TaE#Ye-!E!u12J(jhg*X52cAw+qzGG~8Lg6rMdf5AYLF&BM6j zraSoi7__|@S3(TEg(9mftWn zFBQ}?(flf`>?H%s9oP(urt7p9TJOPmAl~g*BB0ZC z`6R@8&f5p)(oqP~<>xQ@mH}1uNg$7Gzrt$XUC_=bJy6!x-y0F9Q`3AoaP#G#Q3Cu4 zvf}>}&W+{PnM|lfLbmv@CZw5P$CTD?S}=-YK(g5Zk97un+ou}7VTlL^SPCFpA)(6mps3Nj)1_SX6-tGp!+e0ov ze>!>cTRhomr!_Fws`{TQ%o8t0^RSXNdvA!T%L=I7af? zY>y6-Yqtb(qiG?BwRyJe+`Pkg{l(T?7S2Cuk+{2md-APIrzUoWm3*Zw9#uq(7H2IP)%+pZG+i7c=y562xm`Lv}>0!zaMb?Y4GzZ zw|hhOnCE}m%CTMpUVCskk|Czd55E$mCU$MbRjBESEo>3|@0uZ|q6)gCnFR1H&i;tw zr`K9r1~1&X0hH;v?ljFIZa;{#GH9dOeW?cZrp34L>p!tq1nK{uO`fBZB%+^GfQ--L z+zlUlL390*z~_B__zl4OkLGN?tk=5m?s)wpd+x7%-E-*C>>u!<*g=r)q{zSXWOip7 z924pQ!dK5q1GL?(3^_LOZ}}vOq!jpNHYpWiKci-wghgbDG(6hJ7*kd0D1fb}yx zj-fU)3*3$57C6sxFLU#XgqOE>m>cC9XR1=Nq`^criW!JMwKd zYvAvFB(Nn8NfsEyl)sYV*G=geHX?9;DV-FwT>=j!e7Y^qy7 zVD>3$)|3N#PwY43x9&9d=$fALro2Y&&Ep=l+R+QT0f)6%bkhKAd3G?`kaL$ZxJEJe z9Ixc-Y-0O9q<$}f?71t2C@gbCdIor7>xXw`;Qqf~s2vL%;n_i2X-KebZ6cAzN1Hjg zVJ~6mPXe0ayAIr6AIMU~o|nBqE0R3WoLLn}_L&nv7E-65qF=W&*?VVWG~3cLWhPcB zo^^!4-@SN?%wCm;y@K9B(CZ}XeY!OqDwZifdzLzrJEx#xG%)`xEAmCmW@N#hSoPT< zF|8{uL{N+^t#<)Q+U7pOo*jpPzAceWV>2&CNtkY`V4iq#4U4<-n6{=XfOS;5hBFWQ zD9CSYqWnr%;+p9jSQ!QTFx^cYzkqArA#1$dkPAi`GP`z$qxp=zSpExDLOtL&;R7`y zx9p#xX?eW4?)t!dMcr+E=v)zPZEidGxErw_Yf~(a|3B=dNe;HgyA0KIoCG{?i}Gc^ zEgwTN#FH?CEHm6|EEwyXs-foBJW;xG5y=`mtNWT+Yw8+!K>D83F8KMe63jM0AG8No4mn$! zVbQ5n>At0rs84?naww`BYjSo^eGZ~a9%|u#p`Fn(I zwL)a<#V9f|xz*I{=NHzGh=<_bf1XeR_&tQ#GnK?z)#U`OSx09#pdC#=(DJ)HjGn75 zGW_v^%S+1}MI0&~oMn1rGuj|^3#&EY^_%y(+2YF*m6+bP!}R$PmjUk^JS)h(<__rT z`+m%sqPyripIFmmavb}T`yTX7bz)Q&QK?}uz1L%=TC-SFw#(4Pw*qw}`uaHcr{ffc z?qnkdE4?VoE(z9}^ZVHf$82E+&eUDl#C8&Y#|x^a`I1P!B0n-!?kCzHm}F1>ZBAFk*`js&5~!G$G4_}~#tLm` zqQXY6O-i0ep`sB^=<0<{ri=CqYH#06!$YRc(9PbG23#5L=$oJwn2KXcg16}pF2y1$ zOXTFX3|64RzSVO6G#AME7!CDzPe7b)Njy#F6y47KisXG}3Sy$+$JVV{Lf?DR3^~-# zvrRBy!rJTVJs4p=cRBtXPGHynN&*>m^bLT1H#9s7MVjQnp%riFH)a?ZVSbz;VXbSL zem6}1T<<=ht5A0~wpcL?smksHGFlX>i>=f?23ynjF_V9%V4J-q@CN@wXo}Y;bR%3+ z-Xamz2{$*!{W~=1d&23gG`1?*J*orBH9pgTURrVn^wTt<5AgFHo{ZRq9f-~P4y3Yc zP;Qm2ItOp`6(*4xi}e1n#ENbw0zP}&J=pv24`5x1-LQ4Q1YK?F0=zRuRwcGtK!e;Z zLVVodjQn=WFx~jeTxrGDc(!^|1zvc&TYlzwE#}|w8R*)khKtCVkd-Vw>FugB+su%j z=F6zH7)?A9TgEo@P6jqUbKS2E7la(H2nt3R3{brmud-?}xR#i_06-Nr0oW#>bb{*!2te)ehMkhnPPm>mx@ zfDv#E>%K57G)Hk4R?v-&UVlTSj`-KL(;Zc!z55Q6n*zSO5Abi z^xTqd2zE?OlpYh+tE@c#B*{uIWpAEiqj+>hnq3-PDzPL|^{J6cHq@4&uL`H9;3)3@M7k|zt4wj`N>kn@yk8Suw z{5&5=+#WCz=i>h;_Q_)Qf?jOZq8@zHCmZE=yRR_i?L)8%+bQDL(LC(!lTGLiSd{Ta z@e6Dn5-q=fi7T_^M>a9Lmj%w$<~ocJr|u1d_s>&Ie7~$GpeHvIR2UN5zO)hmYf5v(ubFnpDFa@p^f-4k~0PgXp=jhol;Dh z@gSR52y6coy-eFnRkIxiyjbsNiDDl$G3xNr^N6YHWAx%XKV14!Df_1U1d*||8J(KW zCG4YCW2c9#=^7t&m6z`#Bt-YtQ0up~%74{~Kt@lKY;Ic^RBWI&Cht{HA2de|Q>017 zBMj2ftp@bzo@+_8m>t#pZ&zK%m-yU+ZfgF8IH6ULgUB!FVy8S?KoL4x)SuIFDvf20 z6so!gU3xZ=@lLH&efRKJ)zDE&T}$$v*2g<5*g|)&kX--2hoRYW%W~H#EY9Mr3R?zUzj!Yh%JkOzZ2HP{Vbn*d9VHN{-WNn2Y0?9_$mRR%XFFK^ z#XITI?L$;*bQq=83Nd15LZ#=O2_}SnI}z=0YySM-Ej!Jk$FF5%-5;TM^*T&PPKIJ_Dqp~udU%HmvvQZ7hHh~zOMZPkdBEu#den!c zt0$yk`x9%4`8ynuiR~popMUH+ioG6z)Xba3s5761F~NwrRD`Mm8WWVr3`xmm!iQwU zd&Sh~DXAVWS$qrQoP#U)99U1>$eT|+4R>R82VG`=q_MCwa-{Z%BaCPa z7Qn)%7PGJK@gc9F)7aW_oShg>5P?g+vFdl8l924@;Uc0XF=$Qjpk_;%0cYP%N_5BO zCW5*Uju2ZnQZk*nY}#M}Ik(=1blVxMBA72r_mH@+Qkb4+qIShu-Y7*#>0#0=RNw!N zgj-q)HdwSC&HdUSgV3$9s0;?FJX}G|YE}e&?vtV|CD|%X?zvkod%NQma_A`o_!i-| zeebbqGW2r|`@@hh^sT;xwD#GA;EF8jf|7>WGV5Y2c{fdAg~kkC9`FF=%>N<_R$7Al z|FQ>40sc>c_klu6|TTmnj;Or^XI{VFePN*iy)?a>t6sIMkhc}60)VdgE(K-V9oco$m3)_v(S7?Kp+@j>2Jv4N-em#$; zhfL7}E10Zr6CIE?eK(l@bF0e{w1-Y{@qgUwA0VUF!;DQ3Mpb8MBOmMR<);ewqB}1> z$HX_2lvRrs@!Guzn!Rxn^_-|?8)_(#o~5qHxwdPJlZ)=a6~XB$M2?q6u-XHvw^JYO z_4sY7-;kw*xV~p!H4vaXZ&Xo`lU#!BH?v|73N#|Su6YpG^>SGLfuUY%^Lv)_q?x7N51$a>->o6K^@{d5dXZf?phVT{okk_ z1=gr`@u>FmS2~o-yH8ZM#ScYAtRMCGIF5EnKP2*9e$deamyy!w38E@04hx*855~*) z#r2d*ge@AYau~Z4mOyFG&ye1g62ORb)}?zV1%vrN;`d7G_|sm~ANP7_4s)!TDl=Ke z$FfaP1=G8foLrf*-0JnDy~HwhSag=sA+-$Y)nEM0p0DdgmWC;txn~M$sn5TFw*|?Q zQ}r*XPs6P&j?Edf|CbuD|L^*J2Mu91NPek1EAeITL$>dyLF~n2J(Wx4MM|A@8&n-m z?~&rn|In25Nz{T2Wc#K=3hXSU5@=fLfo;|S4jNG(#|@M zNS>`E9R+ll*IJHBhWQnP@p6ugIu;t%LAJi#O)gH_iJto#NM^utl+nq@XyM&-nTO#m zsGppk>GJ1?5b1fx$nqVd?7`_WV#EC+a_GJR+?J<}M(d=q-}g%38S_L@^}uF=F%?N@ zU^j?hp<3}%3Q5?K;uUCw!D4pFCN>!^IIKb&m?yb^L>Xg-S`k^f&e+&~O=Kne0^{zg zD>N(n74o9XMwu>=?zpTN@1FhqGw<-XX)<6RKG+=(h#<6fY_Y7R)nj)ox+WY0}Nqpar)BqWVU6W1646m zhV*rCeCC*qQ8W5patC!^;j@295Z{i+P~YDqFm^j4Xrb#Ue(}o#gyT1w^6R^=)TO(Y z`ZnCF<2>GkSm?cCZ_9hivU1JFf15rgT`t%npDPT|;D;4x{r)2M`5;&7FqMruypd!G z=}mL7G#Ng3jw*NGcb1$8dPUtCKCcjTE)f7o|ln-Eh zdGvyrqY^L8Bh`Dp+)(us648h?jn-DEjAZqSCXy=@ToKyM6uIK8gNl!B=VDF!LfCDF z2GGG%_2Pv$#@OzM73lhp@8hMD3VNpJwz5S3Y?9T?KTE87Igc7@2?Km_=YXjbuc1et zSu*ZBuTX*)(z(7Z87D$X8T;5!qUNX-QZ;tWgqx20So?ow)r%xDpPo7n8dF`t0c6= zE5&b*DbgRvHR2m&ycr={R}o!zRN|f&fi4X?$J%?6bKlI$?tXX5AI< zNK5LMYS8UiX7Pd*)XyM<^=R6Oyl^lT%>O1!Mkscvw47G<0}>mt##&nv$WVbJ>~{+H z$RMHP_>qRkY-)K8c_gF;`;%JF_NuJFy&mPWR+WE~crmsJ^o470Aynu)UHiFFKVt5y zPOJM>fF~Xwg36ZfqO^N9vFDr%Bwx$8Q3+$SRFlQu(0|^_R&A(uWJONL%HNmxu1tiB z5PEfLQrKO4?2OwYwRNK>%Ky6AW$qt_VfSt=7>Wz&C8GdJ(oKbK#(jRe) zKmW)0z9HKlJ!hMqu!Pk1hLia13mjb6U)1w2Qf5x((O7(6u|oTHcXUX-A6?|M4VrIC zqe5>iN7MZcFiRsTBzjT8EQPHUWgfN^O?FyJMNSH#h05dXS@Mry?I$vOlO2c!RP7RTDlUztqNy2+NL_&usfh z`#9`NEfUy3iFImY#k)Jj!**aY3*b4V`64~`ETUU{@>T+QF6AA$>4O%k>g=R*yseI@ zD5FcIExbU*V@s3-Lw;hu!x`v}>>iGePmp#iDv7XSL5Z?uZy@eIYWs^*EW53SiXFFK z0$qJN(S>sCd5|p4lkz!eDf;=7EZX<935Mrhlf0)q4aXlDqax4Aq0wRL&;jvG`8MGq zouIx3+;)W+BVb&Ooo3(%iB7Kr^OLFZ6r6ZkoC=q<{G% z`8oQQ=skyUEKE3$6rWBfJ(Ui?Q5Lp@MZcI?)B6R4!N)al?{tRv%`a(4RKKq3C6z!8 znT-!9KhGKbtDq!%r}QG_j(0We6S;--7ZENBA|rX|434nZA2z4nNd}P{@ei!OdsnJa zDe1_36KN#9mBvgf)iL{iGE}@;%>^@7+C)~Iy2w6X<*nN>_L+LN6E$1mxsaG8a-6*W zE7lBdki}0N`HR9aSycD?1JFe_WR`3)h`}LIx({vdAs**XQqjz0<)af3jB;shIy8Mr z+wYaSByXhPoXv%hpil2IjMurj!G+8>{Z55=?o~47)O#QM405k}8~&sD8T0MlBvJ8O zhSXPzfjO06V2u@GZeu zHk^*EbGgB=4N;m~sg;B*Q20V6>DVdzo8{rxP#Sxo?NWu9@=A%wmWynIW_{F2a5rnr z_^am3^UZjz-%fIL<~k*{Da*9}yfHjb*>i(fdAJQh^^zH_X$BZ6evgXn;9)1H ziZtTu!ZN{_`!PO(Ra7E|HdyT<8CPp{Lp_hu9sR}F2fof0TWa(gU2*%W()NdX+LaMi z$lc3qU9$>7R@ZV%SZs=)DZSm zLzPDJFb_8$wq&1Hl$E$?9>9ES)=5@GtcK!W39uGrCgA?MLJAbZq6-`qq)uvmFnzo5 z4k@0p4e^@{A?M7?q;?TjGLJOJ@uboW92u@7f0~C=TCu~(vVmSK#_t<^FWH2QpSZ1n zte!_^^(k@jf1%}_wXE)QrKWeknPJ-(eI-wi#lWf@UvxnxKG@w~0_^!$#mK>vj%dJy z24S^L7j2yCqT-@gD;4FMF%`F!Q_SWkawYHpWT|BX#*SV$1u{48i~KCJt%&xVdJaC~ zUmTk11NuC$X^n`}K1KeS&B>a-xHBQ3X6Ms^9p9jjZe6{I{r9^HHEJx+@eRF5W#djk z%0xj1bG<%7f2Olqoe|@vdf;2O+Gay>ZL7&#R&zxaSpOHgI>=^4-%@yZyAT!2OEF!Y zpG%u3wTDr>V;B3%2AJ+Y8cQtEwAFo+{nBJ*JS9~@ad0VpH&B%^&HTNHKN6RrMn^BH z@}dXSx)Xxrj*pM&YP%mu*~N9q&aX1H6bdCdcn!#+KoR<# z;Q}mmAfLF0#v{A;UIO2<_45kQD|ZTJnJZ6|TMd_>A~xz&_ULkSo@KrGr>Zz{JE3sK zr{&W0ydw?R^<9VP*XH^oYD2=zG7g?4X1m|y>UTdheVL_1JI6i`%T1S69SvaZc8W%t z6T(fCUl`(_b`ERL3d%&^T+>w8wF$xO8r zFgOCp!R(8<%I7l%6#1+Qas0fFikpWc$&ZU!w&I)N+$<#%wsAEay~B<22&^_K&F;{$ z`*NK6Rw_hkTpgdo9h+iq)DcB$G1P*$`x%z+T_|g}OEOmL;)uyDYvf$CQfQx6Cs0dt z_EQ(vgrRZwEg6;W&n4WF^@!9ZB>S=BLpmH{LZY?ZWX|DwIMQRKR`2*XMqsxyr4ZLe z*_=76@T%w_9NjgDj7D4**|2#RY4zQlNKKAmyz+V}cq{lbEN^MXA9^ZbPHaGI3+|#6J?H)xA z4qcA`J>NLfR9K)@WWqfW%ujOXsvxM8=f`tGO%HB){EEIiaFRSWRtj5%9G838zj5%72RXq0hrLNQcf^FtS+_UCgtR} zA3lF=iK**}3B{KE^6a$RbEqj{NBQ=fwaC(sj7uM-kx|+$ay-G96KrsSl~O5p@BRt#w>i1GZcdg05!H;A0;8^TiF9`w7G_q;oHT>V2nZz5KDt|j-~vc;BP-V4Sc zZaDW*ylq6NCPrYRhZw}5cr)>N&lPxf`Dvv2aWMhS;^9a7#pU7$9F>tKtQ1qeU4i>- zyopykQ=~Q|Lkz`sg0i`^q^e8#`>n^L7Z8(Y-(i(|rE!-NGEj=%Lwq)8FLd9b4m7?C zBb*;SCA81+{LMcO@5av@<=9fm(Q&lp8u2VK0cw7%$EVAs{kn&PkF9wL@mQmYd(YLz z$u0RHUO1}>{Mwqy;PT@$=+MTuwDi%}BD>mhaj!j{psu$B@-fC$A&~C*>k+K&+$v(l z-%9*-?Hx3D9S;w7(vF8|d2EXsl2v+~Tcp6|>Pb9P^E;le*n>-0^n$UV%jXNSwo8rA z+qX}~;5}GUlJM+N=ZACiw;#U=+F5dY0NHIm8_1?{eIRNuCyKCl(8k;p(+Sb~GWg|* z8NBP=9#kF82Kf_P`x{W$fX&H z!IT!w##iU^@XpZZ86bQ$lthqPF-YWK4xy-a3l~)OLhC$u_^x$dhEeBoMXck<4s0z; z6dlwLLtk&>`J8+5#utf;<=ISe>mTwp0iClx8DFD^b1-|0u;K%anA`XO!H{<#I5;-w zl4HjKUml*}@cSE9#JktBL`F@d zbiemJN3wm#L0fUt4zK6=fIQB$#Gew=_;sT*SjX7})MuRI1Elx~ny4cSU6m*xwiIe2 z`6c&38KQsMA^Vv;KyM&VipYcJ;t}mmpsXuDQ-S@zlDq*JE0Poyh^Gs3@FPw?aFsia z_`vHxkPkbbh_8B}hW4ME!6iy9NTH|wV$8p%2-jUa-{Mi@pRl0g3yH+oUOeOaR#5Kc z?W&**GXZb>&W%pYtzkRP!QMiD|HcS*jybXVB+ve@9nFGGb$g*UYtESIbpiAB5+Zf= z!tk3~R^;lfIoM{VI9?~5A}Ibm71PLN0Di{O7Q)Gg9)WkRT#nQ~#1&sRAm{el;6Vp6 zaqEjtgwBJdxQ=Nu@vJ8a@S~Z=vsG|scYwB?^x1$V)y9H%E{%U4;g-il--=8;E^id% zHVm{H7UO~N8~n<B}#~x|9(eDt;^C*BOg)OZ9#kdHf4dEMyzJA2 zpEK9M9y{{J&J&ms9x%v4yR%QDw(~Uc_rL7OUg88M^Bh7_wi*Ea`rF+E#y_q+pNov6 z`$Kv6?Q$2SVXDY$K-W7V2Tc=fCN6IZA|`IWMb)l&-~swmn6(E_uH3w5GR?87#^>-$ zr%IvtJr2a?4`zJ2?)qDCKPx$O$-`XY?5E|U(!oC!rt3HR6U8O~E&hF5tuxohx zt5UJp$rSjhZVGnHO$gIl6hZ5~QH+v@;y}IgHn(BN{C^W%{QoN=)x_o^4?=2q5jpkA zkMv3(!p3U$!=7_Ri61I$;67i7&u!dEoQLU?+mT9M4NoG@VK5(E@a&YMZKc5CWKzgpcp3q~-mfCouEoYy@tve0HpBpzD zihWZ-v_@`H$9D@TZ)lq!#su}*Q21|oS&j3ODhk)wct|4B^7;fT;C-)V<6aLUr`nu+ zb25$Uix6OSshngiJ9Y=G|FPfmu@fb1@i}Kxn7gIgS#o9m=tSdc<&EaO3QzZSsHCmv z0pB@W)`=hwm4BH^M3)e`qbY>=#u(T%=?a6^)P(N4b&z|5q{FH$9#nJ|HXZZ#Aw%|{~ zH$IXwx#lHpf{75i7xHnjp*pfrESqj>cL}-0(!uMs4W@0rtrcVRCLZ`Qf1CTNcnOBOm|X3Be&!1xxG=uAKh~JltnY)r>cUwC%aKn zhjuExgUy&h#+el3x{I8Q;7QoL&K#`&-vrxHqv$pHVpioG(z2W`L5*FvWbdguSKs{Wzt=` z4B%_q;!1>@;LVnmDyv9Sl+fF zOiwLFzDK76eKwH0fh4O%&I)$9Pi_}ljF<~y)YK0#M7rdXxPeKT_~IR}89s$VbolHz zdhqUbdf%!G@XPI{%(@goL|}V`$xEA7`1S0|%pWsN@R!4fWScd&u}*16BJa~~nwmzu z#daK&)Db+zL=zdX0&?pbW~_IH4D<{zx#D8L4*5G9xj8pPtMgJH>u}3jLSmn>I;qFh$ICuS@ zI0vWpOenJ(zi%Xif7#;e>|hmLmon<*u7xUr73#(@`X}Z3pL8oc-yMsZdX_OeFTv>S zRVuL3*$(OV96!FU@i|25hXd5~SUf1_!&zNw-R;$c?uiPS9ihdv-r}|V_5a81b>!VP zS+;8bb6QNf7CGl-7_`zIr7FiW*usP$8nAvq?tZfQc(%w+s?9nI*x3$^wAO%vJDSW{=l(>Les7lFNBrGCQ41?u3`~PC$L+;dw|1xSr%Lk-% zxRfCfF;Ak)_y<0t!el4NFJb&~Uq!58<&o4fAC8YGJ*}b6+dg#dgf^4`|@FyJmT1niSBsunbg_KW701|cRDD8!5fn39uGVKQw!ni2?E9F~$ z)U?dR4f3{s2-g2o$VJ*VpDEHY>5ItNQ%@*|VaT{u-Av9p9|=F)d|a;6{1n`j9f2;l zHX{BwiDFw}4fyEXG}Q2*CVRp9dSaTefTYgl)1ieP@YDcJ`B`3v`mXFtlOLOUIMC8LULTjMRG;dpmilYb)uZ?h}$&i;B zE2L>`t;kdc2mc50D9FyRKpMO<{Wwsx^&x%{YfhoqGtr84lLQ zQRf@HkVFr03wC^b?q_QS$Lbgg|7m_8C?5W67mZ(TQ_ndheZ_oSteZHUH^LqW} zk9%g$x%PAB+-JS6HUh7&bc}{fb!rA)ON*MchoQz5p(UOonDBP3MBd^N?sugdIniZF zJH#gk8Zw{U2S`}|zt@I&;O~N!@u$mk+T6`na@|*n9{poArp2bSmm;-?F_-q=FK^Uh zG%ZJ={I9Yw=T0PazmSPqe;va#R_)h=T}$QzEqO7m)G?BNnKJ`wW?y2eZ(N2O$|mFg zewJ3tS+7avjNJr)b(p>JLk+%?quk(rNF{R{(vc zg^IlF2HR#hgE{sPglU;WgRBYk#~7}hrLo4rp4|kt^OsGtnV5h9K+^CP{|^~WPJ}tZ zJ!p9|G0s0dVK%XDJXgZ0 z-*u$Ek*;8R9$Kdfm&Nz_{`{&8PxMN(itehoLoPU6LtRier|+4jF=FYwt*X~JSeUAV z_1#k&4fWze=rb94^mTs%9%&i^E+i-;p;v#L?@CxmUeb)HQyMQ(@+uCokgrN|Z6*V(=g zXyK$|DF3}a;eEuJYW4TVIz3ktQK4UDVZ;dNve!B@!?jc#~ z8gYqMKBf+-MVxQ1S01u3eA6r1$5lnVb=7D3+I|7l^$V3;>lG^UsGf_Y{{O`@CM00n zxZXbk&xDu2(My7uJL*HAt5O>C!XZZ}_WO!n?VdvKt!zSrV)jVP-_q3Xpj>9H{vIsb zlays(`p5)0!F@4&8SjnueFlQlbIxKLRd)RX$HwZ=O|#!Z@%${b$Gr}gn;A2Hhq&{% zT{`DMZ0j|FLQ4dk>JMA_YN zm@tqzAloI>ec&jZT(+0~^0{^VU&OAFog%M!~F;`@t;?N$g#Fq z?>=)g+5dl0-y7U|P0W~35LuPYs8EZgF`J$-GQPWj?oE-zW$$-#M425^)VhcY%zMl1 zlw6QyqQ}F$H6qC`Q$NOS=tW%K4vm(yswI-tZ75fv3Cg5II$Hl2u8@;&~CDa2;g* z|9ZbIB^^?eux>ARD}n)!Ip0U+^@WuGtdWwO=C#ZOY23>I@qR{kP%QB3sDUj9mg8?q zJ8~YCB$k1s$4~KwITPrcZH|0VkfDkw}xtRT!hP@ zJDt5Vw~vyd@=Reszn7%@$XXuXBxe-i?&^tU_*tn&C0V-=3Za}|O|SCrWaO~Bq=K10 z56{vXe#nC#`?2ru$G<0^e|Ys5&#OE0MIR={V7y5?IRDF!)5A#fnLov^qSgx3^MbHW z)%P0s@0X>O*+(q;l;kN+czZxNQMZ%$c6Bk9?XDj;#+zPmO?tU-=cHM9X2KrfF@Y|Y zXId(vE*!Xy?W)Edz$B|BQ{mOA7>DfiWP7TgyLrN~^ z2Y|&*o!E=lfqggZ^90j)vEea!IJt<>^XL4WhYY+$HXjKTc1k$E^v|F7`l~$s)z(p8 zje8Tw?sbyXeRnOc` z%+*RJuY_L`gOefR!iQYU45J>~czcC2-elpRp1d8}PsUKEVy@%5Aoc%)(`{zEQ}9qP z5xf77qb@yXTK4-dnY+`!O+G*^Ix?8go7^&0nAIy@pwz;hxowb5z0 zDDv3Ke$*8|&i}v!Q$-ASe5565dMS`>3_6Qt_#JwYx*044ExkA&7%bm>ft|QXsTI>I zZ!9B?U#1aR%Q+hi@0pD096fnR9CPOs&-XH09w zfV}WiMn0Hwp z+nRmPUg$wh&kVsl_GBAVL-sln-@~NyA^-c5`u`zTnQ>*|Rq>}CV6)>T+&KQ+8F}IS zc(Hhw{}{oX71`E7|DS@!ymKM0oQfm&bc zu8te8f2f=*|GYedz;z%bm-Cy&!Vj-4z-jDrL%%jsQI_*<#J=|{+D}UT|6qRmw^WI9 zm#(AEH&2LEfz2GA{(8!imu+H$lDuZKG0xj_?H*xYZ$@TDK#?RxkGRm= zmeNazCQh$A$=9cv2g~_(J#egmy1eU;&nH%v>@NwT0VW5iK8pyRN03>fsIC1y;iFwi zntiL|GFHK5%nz=h_`9jXgowMoIc^mLcA*RL^q`XwQqiChzDQbl_T9tH81|WM1G=s$|Mx zovKy&{iNsr$uW6jjlIsaSyc3^AzEug>6N;@7>ykT%KJ#pr!{;2uZKYt>bYn!c(^nZ zniV&|%zjI0)xgQ3AH$}ou{P@&S78y;kV!O07EotA4_GD?%|_8(IY_$ypR5s=QkwD# zmVt`;xc7QWveN7sZxK=&uP=NLn!(I7Z)H@#*nNV^8`9o z?;uyE$n0j6L`UDNS>Y);$v_Q>r>cVo+w3XB<^GIY? z_uG#?NZns=80nAnGlvjs$5Cjnr4t?X8LEfgeh=PGoeSX6Xxc8wVdRZX)!MCI(Vg`MK6B)oji129n_A>Cymou zEmKW|=s>#ywN>Ow1+;&rtW?67qW-&dr1O#2aySje8`@cAmieP8FF=7i~qOkDnKncN<~6Rb~7(f4_96MnBMQdl#1oR$65 zIJ%^xC!KUETeqO6Cvs`L239cHOzw77gV1E1J|jJr>t}7gV)Xb@x@F+EH+bBrwbES! z4!Nu)x91sU)BX#e|I4*&k<9kAx4Jco(QHfy9N=W-GzwB~`6E|Y2J9NOZ}^+-!$%~Xrf4iD-<+(f)Ck`vygYz$W#T>e;O z5qSFq>b<~?ijO3j^9lrImJ|e~`~Sq#qk3R@431V(^|2v(iTSV5fM*KSB$t8ok9jjl z#}x~d&c`|^TjzvJzPC}#U^z!BzWO2EY<`lORvD!nwNjRdncq0n!aIkm^>FF&!Fspw z>&RpU!H_v{W%N?hNZ~IOZ8WsBu0_IXY@+kNr}g(B?Y0{SltH3i9_1SGW4n$_R=m zYJ_j$e*KggjY-M0?I>`I&~(Q-nd29s^74qsGsRGSPZV{&eE5o<()*m%k|z5 zF~4z2b8fnhww!IMu$SjOm16Z5)VRzNs_kNzn&&7>aOqSElB~W!s}4Gd>qzmoc66?3 z7L+Y*QFjfEBt>h&`ThS@9`fkWFoD(J>4w6EbNbQ-Y0?0V7o>5MM~7L(M3CV9QOa^m%*g}pbSBeCU=^R|8puR7?)QZRPGI>{{D~JSX26s-Do5?m25DHvb5ZYNch@s z=%%0&@%qR2(0pwxi_fihj8uPIrnBd+rpsZe8C#zgzZeIz!gT12nNE@glSkA2Mz7H* zzxoKSE1sp9cQupPo|Qot2L{r_%S`I{8nL8k+Yr?=z30K~pxUax_>w_ZlxevYhqX(yx6Pb0}er`#2# z^Z#U`e3owIf27s$Y2#< zbVwIk&iH7N8(Q3Z;k^&U{-x$p-)yQ%ccv;>5o6^g_lsSGI`c-*-s4>@Biax5YdNdR ze6w9@Roiu#jP|KPtL~dwy$xLheJt{*mmZGH#Jn?P#aaWavHg3)$emaES54O=tItHx z?GM~^EmV+VzUB`4-u6L#S8NOB&;O;uAl#oJ=W7~;w>^`FmMUPbH;&WZIjt8p`2JYR zWne4-$D?XU8gKRk>GT*5fG?y{nRf#9^y8ah)(C*JGS0+Uo#H{ zQLmrB0tGJ*^8JeLJVSa7C0QRutUZ@SCJy}L_hbCcLMpjhI#VOPh&QQJocVX6;nG8l zFFlJ#mA3W;6$b+U;-^;|`Bz`)^F%XnQ^bw+yZK#zUm!M4kTrBcF8y2Hy2e< zide2naxh;Os%tbcPB*=YFS$5Nhef-ofOUt(;wSl79+ko6R8i$FYD5Uv4}Bra?UU-p z>)*Q_Bn%G{jhIbP9ox=hoT!3DWIv6o)Px3Z{BQI-M7*Q05a&7ezAx6ko&FbZ$BuI1 zLO~Savs6It7$Kn!RvHsC{J1lsW4)DmnZpNrQTw(X;q4ks9|j566fv#X$0B?`wX>FL zw>iT4|Ks}@_1n313SMPi%2InisWEvB`xpNLAB$L(`Jd+fwt z<#YIU5U<0@_d(>XbKE?I+=v;}`A>CNM!2yj#tAB-c^=1YXX7-5#oH;#Vr{T&%y~Yq zTQ2slDdz%~QLjX9Imv{1T>zC7d;{b4I*<9R@$;#vm-;P-)z{L$2K1qjlX7 zaPQG)DqhJC(|(XQ9rOES9YUBt8jJtCS)8^(3B0Yd2g_53LgJ{GSCxT{)K~4DW6@ZK zl<98(c%sM4%i?A0t6;j8U%Bs#P5b9Hq1lu8u(2P=J+8phANhDR#$n-3)mW2DzH&OU z?+c$LfKlQZIA8Cx3fO;lmT1yEf}f{pdi#a&Kj(_`bHWOEnk;Uv9vAQ8*U2QYo1XN4 zQX`hMdG>(PmCPcS+>Qak>4_9n9t;Mbn2Pb={oIG|86A_cPB#+vQ_|T+V1DQv)wy4T z2!3z_>-}ZbFnsqT`Vc=GTrq6r#Xdk9n+{Z2bMrDRfA-oE+!NkZ{Z6D@lm!d#g#pIo z5Ov_uF0dfRlt}Ol#%)g0j})-+#t^W!-$(E%i`z%V;^wHxk=psuSSI$Jy^K5efc}=l z__P~k>HKrtSMD1Wd;W>V`FJSJ=f8Fu1QzZ8%D-QCw4l}u^~AaqWyVuI#yW$mO#?t= zm?mM^#rZ>h^ymc-Z#nhSpEmMjE+L-9`HiosFC-PW?4l-?&LMq=7Ld_uFF?r1iInFp zBfdU)6v*;)`WG;uu4x_S%V{MY$Gu1qdlprK%$ARQ8O|0T7Jtn&qt2xrrsiB0@%5~< zAWHoFTp0cylY2g<49-o&a?Kf(4Rq3iFkEM5GG!{{>f-O{U~yRsH%I-=iu3i%@_kwH zifTN(9?P-S=Blv1<}fr_C<1BALr7hRW4ydySN6uwacPHn{e21>`8KOthmhyDsk$qE zKsd7w!wn5E=4Cb-CH;>cdqVL zJZ&~F^CXEnmBYZZ_TSRDAd&Q|t=EZn178qT9aSh?j*FWW>^l>^Sd>d?Ap|8CW)!*6A4w`Er(?jfEbOli)|)698og5;51T;SO;RXmdOntQuQ@ z7-_5roAQMi7pcxir+jZgGiNBw%hxccdwLQp056<6%O-b%3yEoTv$|VmVcdd}+Wm`2* z6F)yRZH69exmaYZyzI42eIl;Iad19;b1b*#vqL&JAjuOUH-#OP?4Wzd_~#2?3*RjwB9C;3OEhB7G8wM6j#H7(GV5t9S6Vddr(=~ zSAnbIX0jzXlaho-3(v$zV<#$^BF|WEze2lsF_nIs+uy&VqyWiT&xLz7rea;`MXNw^ z6A8x&r(?dmmvg#&e?0;X->OdD>^~eG(9eOVbW^D9P7g3nN#l5FJnU33)j<|!4KF3~ z8`gpC2m0Z>LBsthuWt-~X7w+=V?e<#OW@Dx1EF(J5LA1z1HSJ08a?0C4sX1@Mrje; z|89OiS8`?8AY^+^jC4017UmTciAm!~fe4RCH@;{ z=@cC0;v*!E`3QD@wndBLxJG;;-F>5u zikO|?Vl6<0&wA23-WQ8vKF&ayVG!$aB;!1c961%OI=>U$DG*_tO)pkp9vjWH5nV9^ zro#$o+)yXpyD=O&e7JKy((O+~{HyiE=qI00;PYox#<4eO@uJgk;S)#Nx~dOY zt2qKztE?0h4wy%^)D>ddCkDM0iXAWEXVxyWT)JT1zFCCFp*K*^2cji4+t9Rt>0s^x zM|xlscfRICRu*paI!1Fogk)_(0C4;HSDVAq8m*s;CWS@AXYqb~8Lho#Aw72@5}kBT zgzJBGLf;>YF>l{_rBK27Cd#R;0o&^j!zD?>QMLS%4UV)_R|H7S9<&m4uBX|^h*zu5?+gw|Dl(H!^ zLCfFkqv-vS;K9Bf&@-YKT2#dG<#AOVLHo5T;!3z*;hl2{vPkfsDD}lW?TmJY48lD7H3fNY&N6MwX))(DW`;i{Y0?RZ7R5PxQLMX)(6qKug8L!3=j~129pd{j z-=+kBUuVVBfEdHD^;J`^ zfq!P-fo~qa-|*_}yZqb(o5x-cPu$e!o}}>w?tV64pMS+R>UF1e-x~ywhwD!3zHbH1 z7_a;N|Gb-R&G{F-{}bk)@6!DeJiWeE4!BKd(b>+Y{S)@}s18v^(}q6(O#A=(Tx{FP zXA|j>+b~-n-%cll!$PS^S=!`4>0-w|2=qVbgojh-Ce(J`l@iGH}UXI`uSv{U6CN zaC0I*hWJn9>xMBJo6)=bx%-}{eHrt-73GCxstXL~$4=ca&wstt8))l|^_n~7cF!}` z@&A%@$NIVdqTP+(eb4o|#N+i+`OVwu8@&UkOZOV{Y27&g-F+(6|G$Jisy|Khzr*=| zP4Bt=*k9$u(m~H-{+0Hh(trJ17wda)M21c`&iCc}_-8h)Ox5kbr2nY;sX{l-|LuP5 zaV>A_Zn#X>I(*)#`~5$;Kh5OY!}>tmzrLy7+j{*g?#TZ{&(S6GFB<<7{C}4}@5#LX zvh4o}pVF7Yd*BzoE%?ihyDG=>xEbH({*}k#FH5ujD{c0jWFEd_={DY`N%oz8!Lhi+ zdbxk4vFY8{nW_c<`nJ0B{{OhUS1U8u&uznPQ|G(AjI+vX6uKS2TM2U|r;ks={R{b0 zA+2U+Lh3(mrBgqsN)+eM7t~!@&G%8Bytz&1H0w($I#lVF`~G}8$I6?sG?~dxPe7d> zs*IbAJ)PvN2%RRqgtN&j=w>xH2KOC>&Wqg0WtslCPj&Tf5_Dm9(Iux<=!V6J zo(_y~`oKdyuq`PW>1M&sFQkv%75N3eUZ{f<(&=-~Kvh#N)`tBBb8*c&FW!dphjRWF z9@$(+5pQhy^ui6jsiEVC;6CAz+dr7#;1*1Odd?;Eewm`A)aE>TJKT^7A0{Ts0-BN3 zKNjd5cmc~&E7wVfS8PI42F_y4om63f=sc|2(MytdE0k_DO-F4*cVdT}7gxw3NkJQmi;pl;C^c#yXEX#^Eckp{~HsI6%Ba`SO!wKAyv zb}LHo5aM!hY~sdJb0qti`8_h>p*j5|S!HMF16L&EIjdh7#^JXqYGO@j~i zaQl&i?Mk6TswJ)UC6dm-L!?OEo~QB&J6mo80*c_$Zq>atTw0tuKR3o zeGRrsrC+QchtBO7L!wl4w zdy7&Zkxq~9F$wOP?v2arfZ`Ttl)9h3)JlVvu)sou`4A`gU<@jul7UI@jqd# zP>cKhMAvR4jiH4WJFjA{j2?^XniH6!PzT{ix8AgLKPjDbEFNtrPn1lExk#F|B{5mf z+&c8TfHdIj zQH0B*=X^2T>(xPI4k$opUE}CZVn2OtV>wc|zm|c^e3)!2MaFN&Wb)zj+w=sDF}SQ{ z%SC8fZfPogX=^#TYvOw3y)g>?{>J(}H=-4uyn|^s`p^npP_ng=^M5c6-DVUP6rjzn zQyE5eEE+IzAK$+UUgylLwo(@bpIAudWTnxeFJFKjf~CyYy@*kYb3k>5%1mZ`2Pco5YN_4;5!#(!uW&f(D{{qZ62>mO!fc=3Pc7HN+hILE|dM?5p%R51#S7VA9;CNF;f5k zM8(ifrhVNT=3!9+x~ZTn87;qt3GOiqO*cKogxYSR;?KW9zb`5?qgSpJg}*z)_|K__ zLl>Q6O5FxZmbO|lKi4agLz-`a*h8IyOZxB7@^vX#Zq_d-YipCFeNbH?f_@Rv8`qVM zn-$R6RY`({cBk?F|FDloq169B8LLr`GMcBLG8Hq)bWdgY>a#!TE%gD~i1dWZ4QFFr z*!}+*(s<*ASGaljxCl3R)+8R&wah6a?N<+xT>9+B+-uPSnx^@Twsd~xQJ^9!vS`QO zRE+Qi_|{Vesx5vmp7>}K-TTZZe%yR`n26TZk*C*P952~4UKY+-rbrJPZpPSsl%9d{ z9LtO{x`w>I4n`3Rlfmh&I&g$^E5`G87r~EBBgvW(d#SK4AFS`M4?zueQ|U z8|sY7DP89D#na6El_JT#Tt#$E{2q86R%C3?bL*h8C93GL;|Sa?RHQgd#`x|*%k~Uo z%r{Jd*}4XjY|m<`|0WAs^P(PN1iOJwtr+Xe+E#j27Rx{FJh$G!rUk8&27UQ41n&JT zfQ2C=Wx-0$6zw>@LFujuR@i-YYZIuEl zj_i-?8OKFN#v(Kh*OBh`y($O6jOvkp^_NC>Hd2~ifWupU0_#=sdI~g|aSsF!TLW`e zbL&k;3oiaetFZJd6|}Ti>^;g+Ab2^6^q$Y{6+Uvi0Mm#h4iW;Tr94i+z&T|5Yi^$| zY?ufx$8d3971U@r(c}h*+vEYxlx(Eh9JtuH8lVy5p6R2&*T)pH0~&W;rJ}8KVc8uo zYzyxIFY(JI2YEZr8Tn^@b(wGwmZe+&?t7Q{?!VHTj4i=Kol-*gW*&6v5g^_@WiQXO z8|R1XVRCSLG<uF%z4OdDt*Al*xa;;8?_Q=VpkGxC*p_%3lr(39jNLO~ z!`@VEze!2Pyd6F4o>2{3{_J~QaA+s=ddtB{#Mjw2&0FB> zPi}2{-Eak)z z>y}UqXSYFx{B>A~|E>dFU-S4Ty+4BMsiES@Z}gycg$N#AW{+`qzub$a6O z!m*t+8;bZcX4AW`=$pl0zU_CB<-D2v`f1UfKjoz&mquDHT+H)6x3VX)5>CMHO+N9k z?o}m-D{+bY;=AnuX}jWd5nOylfVK2?u=A@YPJh@Cj^$B(xe45N zxq$0|o1r$<>yaXo`p^DLZ!bl8n9{#Lb$@(4DUP{F_`h2V&Q31@()fRb@4$n2oFlts z5V|BE4{HaFNA}5%JbwGQ(bSNAB3@TkuY)}b;gw}u`Lr=%T)&IW+i))f*XNWY+I)Vm z!Do0|&g`EL?Gm^>KPLf-}W zALrISre6$(!m%0{CPphAUiH(*_@gtqvuv#X?*2Dm_?F(d?0t8gf(z|^P)nQ~dfB@Y z$mZMd^4i)w$9mo}BY7E^^H*@$v2?nxZ$WcO?+b0f^LIYR-L!z~PqX=9_gBG<_De9G z+CJR37rMh1O5^|GelZJkT<;A3{cWqiL$(YZfNiljqsTj ztGIn`Q)aaiTU+ypY%z)H&zsznRD7RC8ut9N=E2f(XqocYyq2i1Klzg}{Y$2b#rs9% zwre=8=HMtSznZEnd7=1$=wp0^FhuS*ZZ{G&uL@Y2-Iw*osiJWwe-L9g#u9#^apDQh z+Hu@Vtw7O^X>mKd2YPV z;{Nk8pA^W$u<512mr3h-AJXjgL~>M#nOHsJ**Pf^FCu>e8SdSK z7-(%u^w=_)T(D_?psGAs5TvpL%eCE_TjOML@2DRBt4&&0*7knA8H0;AfMB>F{!BfFO@s46!LgTKB`0hz=kJ{qF+*v#p?tEb( zF*_n2*JJj5ck?&WF|HERO3(g`)2sbn5gwD}MKY1EdARsd5r4_d;H|ciVNi zPF)aiG2pHDeB^P@D|6pz+1+1+_vlmP*JpqFWE^GgH~r>yGR;>aRM{UXmITfq&CR*J zNJkG6RMCPEGI6ad`Or(1*Xw(1A?B|!yN;N0(S($5vLS2waiurCF`sa)q zo8EoZmmU)vdY&K+ye)WJU#t964m12Gid*!!Sj8;P{-EAeb{!YPcIw$I^2FxZVy$hX z`8HD;Q%$t^Cl@!>E^;+5GmC55I1kJAdhiY5RMiRM$F<+~=uy+D=8PWqTDN z{Ir?${5=EX)dtsN*`3lmF`o4U6S8uh3@LsdL2f=}B_dvI!f+i2PU7#(;F62D=m@vJ z&e-w*xrdxXX5IdCHj$NK_lcv_=Qo$ctKz*$<5E4!d7&P8*7qf8l&&rc`lf{G_4|5@ z4DdB0LAebTuJoAK;l9=hv2McbzwDbkTk5~-vprw7kL70Y`DW^)uubQ=j^*SdC+ zL^Pvkf1D@v=`CIxV@{S12@&gMwTb1OBB(j)6#0|f&)4Ph6@SY3OJXcZmzD5(vh<#{ zMv2}`+eoRs{j=|M@}NZF=zH#jL#7_d^#60tIUW5L3|l0^Wzd_RNOqpxfobVIA5AUN z?TvXpV3bJbr`&$fosQ!~OPb5D9p1DdzKsm|XhLqiHy)=e9`+Zkl)CR*7CnFr9DGqc zc~T;;_vPkVvWxCSOYfIj}VK zG`P9zw6rwJ@!B6hqbuFl66g2r#P&`4l1Y_aL0I->f*EjJz9F<*6AI35Dxfwsj>L3& zX=USl!zX{nq+8mRk%8VV@_oZPSm?bSmpL`5ow!E)8UK&gxr^~xS#BR2_?OIV8hg!p zbndTo_{NEn#>ggCN&RvDe6)-BIhz~banRgvBz2m~QrrQnOE_OWeHQysuNwLa-`UlI zqf@P5<0@C;ePIics#cEqPZ*fVmwh_@?yqttvN#%d-{rx;8Q%fe`2y#!_)#m?HMv6# z{8f+TyR&NgUvg%RP6D(0B~u!{O@zz4s~D%*OBcTHzM;L6-s5GL zc=y3}0Wo7?bb}C{bp8%?7z3O?uTmO=L3*8?vJIXpwFj?~6><9G)C*9*Ne+dEoZ@YI zZ!sVPO*dlrXBL-1*op*TlF7v{5xV5zdXygRMSVFa!^^Gjp~=h9e$y0IwX2h(M7nTK zs49FMz8UB3ttvt9w{d%&SXtO>Y0hQ*?KvFO<-ZRuhx^T5)CbOAtp=)!GWdGy6&cU- zYTP@LRJ+#2%N{v#IC8dojA_lh7!RM%HiqR-kMrKbrZJS8H5z#2n4z+p(Uj4I`Jml+AJ+SE;}A@@ zANQS1Us4B3^tpXznf}~&R=b9R-<}0w-eEHpN!Q=Qkx%L0XQR$4GvMN-6iRQyJ;L~1 z2z95=KyXQg0#g6KMA-XS{2hReW6&m8`uP=p} z!@2c3_FZNApEhLR0bi`|s_;Y}r$Wrl`LVD%Ze?)3kuP88GscFn{(mu!?nndFdO-(S zjo$>e7`-CuOA0ZKh&i0kIu`%JQEsl3eYTf;0>;@*Sa;9dMDj{T1~C>UQ_@(d)NPGG zkO)S=-x$`)ianmXh_Vz;M3vlHu`yC`_w|u{^3%OvdR!E8g<@ zpV9pe!)jLIvhT1^MrQMeBf_N*60Bn2FbD2GmX+W^8JX`mPb7NkRey-=^yOz z`a$zfPbTH=YGks-1{56DmZ;4Q!1UMm%g69%X5`XuHc0(bNiSl+BVID z-$asOCj8n33ul%wg}Hcn1{z=fnpPhhh(ewFff=3yz~v-!WITnNE3_Rr9$u5vqZH1{ z(({jfLWPqf;Ma&HOx@XH_^pQK*Z)sWN#o@nyxxiFI3@$M14z16UtZL*PKK$S5(!={ z%AvkBZii>%?U_`2Zoc_OjXgXxCIQ{OWCv|!G?{0ykQ{V2nWrT^|1WrXI*$f9LuvO> z!(gB7HLy>rIo6|SeF$Y? zu5BmbcF>p#-gg*({}T?;WTp{!#+}uFUdvVZ)qtUcuG@e{(RyI7qDrT18o;<|xiN=V zsKd>9zp=j2*}dS`r$5otK{oVJ@ot6VI|2nto+Z#vt_~O+s7^SR^rt}%k ze{+imbL;smzt$$iqzreXJ?e#Y@6-fF>gO2i!{%uw7K-0K`AU_NLQHF;wHrOM?-N`m zI{w8(y_g%*v9R&Z-1(-=TY9;1wxC}ik{lBqDA(ZY^zA)^=?#Xfh5O6`k>e{7^x66y z&AJgwk8m(Yljq*Ux*xPQ!2H(tEyeXkS?vKyI{mvJq$`=me9C~Z%;6};W9@z~B#aK- ztS_&8PXp>s+6nI@Or*0NZA7Q0jzVu26vEZc!>P_Aw@|j77>0Q~M?Q~kFxHk&F(2pE zF^u);V5F7XgZUjdmv8%6x_#RuSci3UN{JT9OnTk6kJP0H`b3YL$vTcmYdq=6?zCDE2k5mjZMfcAbzK$LYriJ<8K8w(5 z9B@$N>TgGbCE1iBBWeBKgAsWKOZ|zgU@V%ifNO{5@a5ql9oM{^90y`uofINmzoWF# z-25wBPKm-j=#(_B%N!G0@>qNbEUdnS47UHI>kZa0ZO=*O)z?VWcy%hK_3K8m=-J&; zn4#Vem|RQ{by9E1Dj9D4*Afhw`l)IX6OCMQnEwk}!Jv*lH!qa=jV>^cIwkN@)oLLA zeUq+|F%?ti@g5`ybJtH;Y(A5p-O1Yq$?jN6|)r9^DQ74n7B*egeYH0CV+FOf}o!6Zg+ zqLi(p!KShh#`JSLo?8sNZ;EAMX*J~dp`au#uE_bXL*eX6d4%ln1+Y)$G~|7QTeF)# zc_pUjGw%?oZ8DIb*I@D4OJ{u{${zO-Sr6<3$IY4e*SvazlM=KDdqGWT@dNI6x&3D> zj+0*u%J|OtG*sHV3r(~v1cFjq=#%(PUOMNA7R_ivRs)3ewL^1we!-^P89Np~b!mpY zhuv@Rtqx)v2zoWhjb6M_sPkF_4%~1H1uh7aGnsT0f4g_5xuEt{pP;wbQM68>7~Ang z2#NDpTI^+UN(Qcu$R_(u4JFP6c!(9NhhVxku@5lZu{qITj^NK2BtLi~vD1y)A8}PH z39Mat8N*+T&_KUueT56FCc=;AFF?y&51bx8eip7*ZjZV72bQN{pC)8m6eV6Btc_k5 z8)6%Odu1cOzp5|BVPVYQ+``Y=Hg3YU#CE=(vw7W@;(~beOg@^o`}#Hrww*gf4XBAj zZd(J%frI;Dyt;`7c>QFS&mx7Lao0dli5a?4%=K6MuhgIvE#{D3(Xr540MMi@y@~6; zGRY9ZZ)$#9f5Cpy87v>mv*dOOw00cKmy2Z_H-B_;axC&lPlgN6?8E$2dp{;V8v23v zJ-PbTF^n7c1suqG!Jnx|m$S(df-b;$_ zk4~8-Vz{_1+`84z&;&J_2>0Lyvn#hgsWWUw(wqosPNbSaDl{5CbyhG3%IQo|~-pk6t!YSkD zfvPK&;H|r|eV7d2OsM%hD;x@Np93x>c)_?BU zQ7Fr^4BY5f!r5PW3Ch+yhB7zK;L9T>b}H4^${NF$_b-H@E_TqYS3Me|HcNQ$EO)*u zw<3$uoOOWbZKK=C<4fa%!>yK&sf~*(;IC%{I;$}q9dN0WdA2c(_DdG?IIOMyf1HQ5FY1JEXD&wN0w0X;xp*_FnUe!*3-kmncWeHVrF{YC&+udKGV;a4 z+aTnF0o2ylMG41Wg2ir)NW6PCfC|&$LA8b0HYcujV7?1`oKydNasv6UGUyEfLd8E{;Ib7)NxjzA5$d$p~Bz4~SC1@X`h> zQ*<16Hd#9ULLE_4(TU9Cc-(k7%L3=CnpG<(TaCwULCk1wu4l`k9SYwacfu&co)|V? zb3M`-Zb(Y~58--$&}9g^T3km_neQ-dvehsnL)xbuZU^2hnPx-k=D zFfUd&d4+2*?w%X|26h^aq-Y&wvFN!Oh&K%f4&i5j!V82{Mf3UcIC|>{kDGexDmwjp zAg(7YE$MC)fz%fmb#r_$wU2y6wVU1&MD;S3m(~TK@g)*Gvl&U0edNX=Ec|T$U(}EA zi&(D9U+k#84;aixp=KXd@R&PuIKpB*f;UD}{uRms+pc2dkm847jxYbS#zLWL_>S%_HEwdoygDkV9=3 zD)RPb%QI%9pDA^VTwbFf&jGq`7l_AYIW->T600H7D<^c%xYYu-P>d zKAFeuFAVhw0hSQ~)C1LR;(01D@P>zl;8vfpd_7_1oOxpch$XbC1;1WVKbUID;Rx4< zU0cKL=KvZzz_DY0=FEnDQ2{G%{J^>f>Yi&Lv2*4#On;Mom=J~C z1J~2<_@1q2}o_&{F5X;S(uy~1IxY(L?9ckd;r*cZCXAPF^z#?w%i6YpH+w*{|aPWS7 zIxr)b1D{?Z@cjx)AXii2GaE#z+Bb$f(`R|HOw&Lv@btpu-k8=WbV2vz@zdaK~ zhus2AeZNtkia&#t>z!D}S7QPGKi=H-1lNPY|Hal#4^taAX6{|OOIlK37coH* z#6lD?Kr9qn5Wz+zMH&SJK`ChvK|)aw6cq!pQEXI9LJfunRDjM znKN@IPGtQg^p8zPc_JSDePoiNM6nKmY|X#1WZ1hJ(GcQ$fNYHj6Y7sMdR9sJKJrRn z1)16O0skR+fi!)xg-_kBbq%h!7 zx&3!?@#VXuFb*&{e49mv#oLn&FFBaTxB0*d!$GvZH#uK53|#n94*WIG5@)tB`i(#R z6p6~na74>3P4S?W4{`12NIbKp0+a>sB{l_{U=yb?XZN1# z(#3rkSjqUj7qu;IWFa;z!`#cr?Fhri-jM+1?tuJb~u!&PYW+EQxwvj@g=0>>I_ve`8BXvSWk2wd8 zc5Xv!2cPI1F)u}U{yx=jGV$w5+lfcS1@Ri=Z{FsLBusjN6YKm!74C)2*TsG8TAIh5 zLl&X4jXNG?c+_)0X@}b<5d4_&r!ai#Rea1@Mt|xru2&UzQ2D>_mybK(gU8MOGZx=0 zx8i<$I}O!SqTz<!FkCSGrf@ZEJ@hSgqez}`MB#3Y>O zVK!?%TC57q7O`xV|5W z@;VJDY)1m4U-uV>+4~Oc*XCTTuU(S(6EBS?J`BR9wDlqSqAdr-G{>0BV>;b6;(g-H zg-v+N{W&0cI?gDE{AH{J}q%^QFJV{`S7B0R#F#Q1~p(~x`)M=|65{ZuDJ zqX;%H>;j@Iw{8*M<=KFB9E%o@r@y#G|6}@638kc5%;usTw4%kxGpJ|Jy|t z#Eq}|#31-*uguxLnR|9SgSRtB0WaTjjhigg>lM)yak&)H;Lvr8ZF#3qTzCKBfBL(I zu1WZNgE9D(b2)h4ZynAkgR2PEAFCji?zI%di0}#HVIq9yeNkfmh|>Q)SeGexV1hkf z=e-hr3qsx?vW;;O`a{h;t zcvQdLKyXWnggUdyb$F4|LF|2?3^s8bjd=foW_3Ewn-I{13-$lmrB+Lj++52a#*H?7$G>K15%%tE zF?_(4F_?zRmH)I!GT4sTb)V4z72*F<&glG$eqU72L-gHOGC==DX{jr}i}794jmO{h zG$US$^1vfuoL0rdhz6l9Ah&QTM%*|H@PxVXoEL8xJ>Ri1-Qx14oHjA8km1?vxl2P4 zZJSoqab%*GVO0UkxrL)GF_lY3M8Wx0(d`#av9Ih?dPeLvZ1D zkw`Eo5U0m))#Hqx2Qi`z;CWw}c}P??Aiq_{YPTDrYxBF++@Z__)CTUhcD$@&1*)?@ z_9@nUlY`RB{5Rl_-#kO*?I&A{=_uKAA1_a=MQu3N;Ep$p93UQfM&j?s+hNjU|LON+ zU1fX?+H89+rp@h}{eSvJM3>diu~?^`2fq5AACm3Nb^rA0{b$7DO7`KX?)hu^h(`66 zI9#azF9Jo+i|;97e$_Q@q`Cv=*eo)~$jqnFa@UWJwi;tn&*jHVrz>j>w8|d*qKcN? z;V;W*Q1|J&pkS4>+z!_ikiXXi3u351!-kkHxPH@D7~AbdKU$9q!e`@h&C(yx{eobe z7c44^k-M4MBp;UXg@1qJcBp3ucQSR zUr%D?oQ#&;F$yF4`s`>A|5k+Wfp0eOz0Ta3=}wg7$DB9hHr={TJzg-)bpA3YS;@5r z1c%Zvmdx8hZo#t@O-JcG<5>^1!SFekk>45<`J|OM`{u@3l;*5p+1W}H7a>3S>E6q8d&H?X3hWLJ_S;9MV z6Y51FhHdeSWk-0m^PlCoautm;!~uH#V~EVar+@M?S- z$bIxqOCr=)eob6Cch?XLI$4%r1y)@AnWD9D(BeY#v-jyp8!7z_#?rmkMBp~kFIiKPThA- zWE*?;@@x7&f{ON;>W3ptd0lV5(YcvyL2|)vD)YNGcj2*9bly^u9+X_5{JJR$s1C>j z3)eTab;~`Ty3I-KB2`PeXCJ`yA6d}TuD#Kqu%pzi#)F23)~DiyDMtilPsi}L6nhdS zp_x?hgQ=3+8Z$WVBXr@HOG<)Y^G8vmo=<1>pI}jLnH5;Z)&}}?Z?l#OeOcaMnL0i4 z`d!sOKKb0!gEq*|{gHq)+HAK#T}v^GJJCCpa`;wHaf+Gs5TSZsDz=~tjWv-Vk15#* z9hb!Exiy)Cr@H6aOQUeRu%EtE{P8S`n>Iz0_3<C^taq(}K=}P<$6cW03=EbDrh0CncMh=V z5(_=k#mUnAA$Eb%mL>r-`rSY^JenpR<8dD0tlNHHl%D0=(zgyDgIY;5WEX}?C?&X# zqv9uS=Uz>>wPr@2gv8_lcmAzu8nV$7*s}U7@|14hKL&-qGvL&QgOtq8k^Ccv6$Kzl zo1f(93##nwkWY0jiAi#0dnfB_&fLNOB8<1Ok982+(6N^OmR1gCzIUL6_W#79fP5q? zds)}G&I^y}UQn7rNuP=1xCWi0o;vpuQdGUZC{5+&g67_WI;{s)8ciB=OzWucE zHm`u5FV~|vE%ml5)OL4KIhFg9wsc>su@m*4(aHTSq;Tz6Xo3=~& zWHpP`uxt@5>Zj@2pK6GwgRPC!gytq{=k*;bhC7yUnogLCjpa44m+g}NIzFK*Jv zG`@tn$X#Ts77AD!f?{cEQy569nJKs7W-?dV@3>rD*ee?~_II@-mS!ma?-G|eSj=@p z?)r}lRGMEZ5^a|ZxyQTWN$&eFxdG{gn1l0FI__61R@S_VZGY5NZTC~Yv8}Nb*tY5c zt>LE5k2ko%j$4|H;{PViiNeNYhEOB4qS)1y{p@K+eR#usC#s@nE4LxRLTwH1$yN$v z z9_{{N^R4}{CiiNLFgoQZ-s-2H)%QfkRRoEZAN%|5sjy0dF z+sE9cXoo}kd+$R6-y+L4SE`}B%5N#2DU+8SMEzCvG8d?xZd2FJSZT}JDipNG_mZx7Pgo|P>0-uU7TzWneY{=6zeyiO6}i}Pe+ zEB(b?E_(&f88=fr2aD2!CAPCPGfyEny&sHy*17U*{Jx?EmjA8~OIgR*b`4gfiN7Nv zIG@BQtiRHllQVrUD`D0Yc40GPD^|-jLAcaHk8|$&|6BJ5E*X#YZ%sh-D%+Cy8cBCV zhw%X=e4_6=e9@PqSkL(ouFx+hN}n|gBHrD2)+1iqb#+D%N^*a=LVp@WS2T0ZQ_@#2 zvsC(@t)2BXTudi+%Jcu^|0VC-|D;pfyz!1ti}9#Bd*a$~2PgmNWO1FMwu+wXS9-aH z?A62-nN0|meIW-6xhde5_+i+XM@4wW*wx(p*hTClXJ+kvvy-un62W?BWQzAxqIVYw z#=g+ofkWg=zQA6(L?WCoa+7$oyR|&NALOfYdK#ZGxs7BEZ<;% z4(j2LtJk5t^yzk7q0db8`(@ca89Ry4|Fq>dtB&i@X^MLvHXvxc0=qnQ8a~>BvB61s zkBRG>>?)6$3=fH6lbRHW=+R89*hM~(h{s)=Lxg>F6(@41ikRoO#%POWR@BzN&-^(N z#EMSFW>b`wyg`?1=zD`3Ts0ll*)qO_D~zj->XR;J#^YbuFPVqLbjZK3#q({rNJfCk zTFi%FY`y~5oI-fcu@%Mg-LU;G`mH5A9^0KV68C(2@IT|@FaBfI>BRV7y_nKNVf-H5 zSwxo0WrPb`@d|GXBC*_~+W5)!USin9A;vz?Q*k4%6F!VNW~?Jp-_)SGtiCgMFUq#_ zai!D*+^lgSlioR(_;@LhD7auIZp%u}JkI#=eF(4Zupw@@dJgu>i-|>|6zPQ0V~0Dr zmCp;rbw7KyMchZmN@lo3>Lfh)9+xnGwh0s5WzHMjwWWl2f_7pUQ9BdMJrEsdL9`hD zZ~I06Q`;gqV}hpOZSH!goG49a+P|}4T(_4KXCWho%}m`d)N`4D$_BnxAj*oC;88=2 zevb&2%k$@i6ft`>txe8iI8olBlxPHZF0qj9$O4#kYb#qTXD;!kfR9nWvgG`y%6OXE z4z80TW0%rrSb^Z9&+bBPikUCRvTvD%Z+DBsU+Z^b3oPYO{`qiygjemy4Qw8d83V8M zw&I&63JBkKjba%U(a`*61Cf8RitBz@20u4>B>sNvF>GFr93lMvvlY!Ypz)d;K8qlA zdT^PU9>nPNOj|{;QSoYAL*;*G9ip@jiSb0yZN`2~^nU*4O>vu-<=jSf2jYxBd{O=? zszc1@T)j8A&{vN5{bfWFE*ZeA+c*A@#Lx5Q5)&Yf9rJeKOqk7_3x%%zXLIv%RfPCE z5^coeel}<5xSqO_JBj<7+xvDpSKpAr!dBJcqYRwz%I-9=Y&?(&+ptxqX5e^unYf)k90UBt#58nPm+ORKpLcLk-r&Wx zI2ps(vq*mO#N%D}A$qTFioln*G4=o8go^t|JKBS&4Bm!l+*HFO79~9nj6y@&6PFd(6dQ8$big8#5nG#FQT2R}1 z`T|foeeyNF{^)Jig-?25x}SW`KOWo;26VtI^tU|3^qswwGsPy2Etwv)w%7B z7lq|_;&LHY^tRCs$p4??mu8eF%Kx^8r1h`PnWwWYM@rT3_O$dy*u z`@oC}k*)X9`${?dNxufoUJye`eme_>?Oox3SspzHGvo~3 zV`4F1r?x=-iDqEC_Dt|+K^FetSsWET?26iY*WfnUJ7zE1k6hfR1A78*P)<7YsJ0$! zYPE_3O1G>_M|V0Md8go$DSE^&HefYK|A0cj@f0gT8y?#wkNUhn%@$v>W51Y(-0TAg zeo?a$Wx8RlIDPdydD2Uch5EOi9>FtiWzI6zY*M7Gj~ej~jJgU9cPyhVHCEy=^)A%6 z+BEXb9mbcTajXKZ_U#A6>2$S^R!3cT465SaYGHPdN>VA zCujph1#%Q36&i*_hLR}v35Q6EOnR;Xpj}l z0ONB8sp&~|pu{E*^*<>AV4}1PU0*IBMq0_xzEStVE|A8v^~&LHk9#R@x&~ldg#{!HFXesTx>_`1V%25E4rh zXp_B=yOCSMu6$HWMXfLfV{=U?DdUOIDt$k-ze<``Y>T5T>Za4S4>h=znv9-p(FtZ; z#Msyq^{-mQ{UCFu4UOq{#AP~OrU(B1sEX*9|6{=2mj8pga#tRV9#;)YW_v;Zgx8cz zT^rs~e-1W{Vwp4S982uG77LhUsZ1-@v77yWq|B?sQ{-F<#QA#*6RA zczRtbRK$Hd-pf1X^q2_;VfxdjbZT1>(Ku0s%CoYg4X+NfGUj*CjyZ#1;j&)Z*k=jP ze|r!ujA4q`8Ue84T{U}Sy$*%jG5U`R6H^iWWuDS-XW}L5V*3=t>viW6;O?Ua9KQ7n zdg(?N%=)_veA6_KFtpI8-VCxR?GuZ60lhAOuxY@nxJo=Pj|OhU&AUi;MgISO#-xK4 zrYF&RqTY9)Tc3~c{vwVO(NdngaUa9zy|Z?b8LUS7A)w)ro-I%=#ud$lOBbC4ZFaN3 za%&2A)jmX7?)f=-As$J-P7vIsI+5WU%RVOr^cpbLR zI|?{om7y@cE;RDg0?|i3$+9s?#CAtT)ZUI6zBG5k4qnz8KfzP^C=Y0nkL-+B%>zZhvtEvTk_wod19i%mnyT>a@37YMM{(m`p+3kY3ltIILZ6&0P zZZ9Q_nZcXu^_DO6O)7Wo)&PomE24kN^ka(j)JTO$%{qY>$C|F5;H&LXGDS1tVv^3s zkEEh>mpyk+g9K$$C?R+sb)8CBHw-tnEu|Ev+?6vs8A9bd@8B&RG(-4)+lHX=Bf?u| zwoq`Wcmt?b-$Pw$?KZ1iEh(!vWk^tIeHNVAXGw9BJ+-VawHOQI|8hrde>HNae=A?g zRF$G}XWFR#h}@ei2Na5eZO}ZtcK15G0O-)}*3YON(pL$fJp^QPL+QH8C~WhLNag;+ zq4?XAq5KovJP`7<9C&0j3o-&1Qrhh^=u6fou=bxg7}Yl!onIT4$HI;pVRX3^n%XFe z%fn8WvppwG>`%qufrz65E8;ga^{v#F+kH@>wAh&{Y`h8m`qE&jTt2@3N0$+lnVfEP{p(WD_N;N&aXyGIx+r81oJ z$H3ZdN#iJm$T52qY$w{#*r{*Wm{l5r#k~&T!PH#!IsKS@P zvdAB1?4Heg>wZQhc<4JEZ);5L^J=4n{&sj*_IQwaycRH7X(6`Km}=AF101CQ}@ zWltR2r}efkLSbc@H*at8ZS|nw@q|lG5aM4GKb)fCCu^KMxd2QuD}h!Wao|Lh3zGfm zdA_9ogEsLvcpKjiv-joeck2)kY`hI56`bwTdiTtnmze?69aw??s{R{;6A9mrtO`ghH z1S#2-S(WoFChFnY-RUDsjveL)wkfF0dt^(^JN1tC{U6_e_~$O%D&(-TSNj6qqA7=H zq3=arZ&;pk?}bL<@t`kW^z60x{NK1wQ_yr}o4nU>ob2eA<5cor`%%-2JgBmrr|Av1 zQ}~YzLIqp5{o&i?3;_SdLFoK{brMf;zA0@WJ7SceaHxx#Ls&t7sbrqr*)2F#5&)+d z+hfV^x1u@oVRsburB+d|T<)Huk8C6D{ ze;UV=U9<*NG{2Vbf3gWWUCHF1EVQE<&;3z!`+QRx3tXuBs$sogtmAFCb3>M7#$|JK1n^bQrw z?&Muh!hNeNwaKv+Y_yG_Z$AmN6VzH#?~l$>H&WXJ-Woii=BW*E7hKGyXDv&{&Nz_N z+m#XAekC>CJ?2^TQK3)Un(804`^wAk*VNMjrxSlbMyUjUoeh8wt1WpM{m1#yg}``T z37a>6hdCWqHrqlp1|K=@p)PAr;@KbkAiMvJ6t8sG7`0X1C*YGHYcl zcoo+oC^=TW?0y9q<=d2DRUz=RHV}*C=xBDG= zN%S|3YT-Tmhua6hDKlBlhI#|)LVFeTe|J~zkxVaK24Bc#@@@D98HMU=Qnn&lo$%@p zT>nRkYrnBm_40=_%q$}t)_JgT_&dRd~WrDn>JGjzVC6hZBm8Clgx_9 z`djw0lYC$CGiDL+4fYwGPmAKycZJo`?Sqk6|NV!=`B{!~L)RA4{WTnbOT?<>4o1Lq z$&WC2V+Kk73;-!pq~X4TJ>WV0R&eQN6xF%-0Wr0CmfXBXT}-P&L5I{jtbcH2C=Zsz zf!)>n`LpxBYZ9yE?cRAy^Da*ER_{J$LCsdS5zKxT27`?sf|(iG0zvWxUFktbvZ87| zyfRfvzHYLz-4RZP?6&L(LKxGGHhq>Nn6^@O!+a;O~d3sR4oq$VNciA`7qU0;D)2wYz#=x1g0-%rw zXdb|Z#_%2rZD|4>7Q+(tOj^3HK}AW!Sw7V%k{3Uah4}m2s~k4{V&YP5N?Q&dNA4h= zs#bx~ClA7T5A2z_HvSCyPKL*uBfKJ+``>3+Tal7Xo zwb^k*p=}`kBQyZv4Loava9qjL0L5P!e~O~=Hm}*ZobDvF&RTM=kF;LN^wYdYQ81We z4k!F*2Xkd3z`jDpeqg#rF=`vsevEi2!eQq-l`69B#)hhA!QGX65Uxw+Ubs`!Kc6aa z(m#7Q{}Si_v|ofn^e8>~{h##LV=(f|nmy!gt9NjtY7`zhHv9kU7t^JOfZ_CIFkGIY zYoqO0{H-rVzVN*ZR{8CO-!JLno!4n%(B2TyHG0QC{oH^s$N$vVS;zRc9F2c~)zcK= z%pob1-y(4q&(ge0PAUG1-p516-vdLngv0|UEs(p%65KgC8(+j~Ml_~MO@PNUa=0#8 zj2`_)VH`81G)BL-MGWPpI|39ct&VD*V35t73q5mIoA1!EE`X3n-!HS-n^#|b8 z)_=Yn=oz(mFV+uiOR`X%je3y0w}G)s8XWOYcUJ`abTCA`cZ>;jhOWEjpf;Te^CqKm z`VqXl2_w6YR{s#o_f390;$zwCX7KLp4Y=c zY`1|wMz?^vPe~x)1!F%iq9@Nqfbu&7N5Ea0YvJ}mGiq0CAgZfl^d(e>{iee(?WsPH zwa`ZCJKY=MLl&F5jh__v$9i9a#0@e~d8|q3>$bicESF_yOk4XIwJXrDm{{fCj&O+T z4xKSa{M&7hE?km)nXCwrgO`uQLY=*5QC_3x9ZKe*mbjbFA&;M`jl zBfQsdG4`8_T;kw8JQ|D-S0|~vRbc!cCO+b~kXl5~rXfYR-(wZ}y)57x!Bg>~5@HsM z`7Ua+%lPL)Tas$>;pWlgspz%fmq`@ZyI&yk|AWV;F!zajYv)tW;gPWC`fjR0@t+Q6 zX>%NDp7LLxMtiSSLsPwfXS`V%q2l`9#xi$;j)8B){K+MI#Q9E#Z$Mj(i{dlvRINlP z^#6nUe_qjA>e&N6g}b?e`$keoe^Wo4;<{U!A2gxZ8VwyV4Rj6|op{QO_p6EFqeU{wcpTkX;;fcpp z)VI~7-lof>u@X)29#i7JT`*}EIa(_nwPUbbk|3w-MYzx2Ifm&GP3X6$-+dT4z}!tO zEl{T}2X&)#t=7}x_LpzQs8MBU2(Oca1r>F|j0&rrMVVFELd!4L#Pp>ZGxqH_j+BVo zp#Qj1oX@mC822gCnP8%KO=)kj6xFk9pb5$iRA|NDT28&5{!-v^O*<{0h<3Q{nTeVA7fIP0%VGulU3ypxTx~eB z)!VcJd`20@TrD79kJxDPmaELqxpRxv`63Gtg%+}R*sXFG;5~$^sHA~A@#-94w;~c; z_p%rI;;e<`6CZ<`{TvcgE<*Z$?w5x6U!O*?^J*WG*W^tErpgn*H&_XT`+v^Cj6FgC~M;!x6YL?6%j^%}O0+g+)1UxEdRyDkwUr~_JIgYV>? z>l?9`^-*MH<#tZZDNT0xATE0?@)IlK^(Hk}-S@K7!zMuI*^FIW&E->sQrr_X#!N?F zBS%c$N~=lrvUTcwVW`wX7`rM`wEt%nR}~1jXHLn#dgCMCzvnavFc`qy=G zg!iC&$!#s#>h)53S8=DU@$0w1YN8=GuD_8Qxjzx+hiED}Ur;3dXLb-8yGGM8YYy_N zZ){*`r=27AHIW?0gy-;Q-~;uC=~LnTjedqc1E(=eb()}go&^8Y{TO^jJq1U-Qs?wq zsIX?KN&pk1i-P>Uqv1u%G`9KuM7Zr^A^Xg0HTq4Q*suR3yM;!Rt%MK2Ts z>(a<=S!Qs$;ZE@#UqFQ!`^nNzik#Nt@QIxsCHspd8((arG+9LlIhJhf z)*m70X*$mnd_BigoOeWV?8p(|Npb_(X#wFVA6r|i%K3s3=TmH*R<)B-Q_muus(Y*o z*#~#7jrcjL)Apk9j_X>wAqS3UC*y*?fOg^ler*%;jVL?r#mOB%Vnk&4P4J+j znk`**2`W$VBBc4}dQ<*Do($&aWK4YQnQa4SO z){RK{+lClE>dO?>U+jl`v&&knuZdaw(cixjo(C@&R_}Wt_3&b?`i^}P@Jx&^b(x=R z>o)LG@Q0hlmT@yN+_*MEFrzkH=#xHOJ5@qcHZIf@j=HhKrlZCc(7KWg-wchR$7>7a z+=!j@#Dgm2%c4u**!Qsl1HDktdT|U9@vx4+Y}5>JKxY<5a+`@DCNmvi*%JiLYpUgc zSB0aou);!xqc=^_AbGSq`;N*nsZQA?Ft`3ZR#&YDua0^rC|}|QUI^p8Yh=?pvM=7! zEh`UbV`+qStYD=Yk#A~8j~C~XO^;)sx#}`B zM`Vm%1HC#jAil~$rFpftWVf=6c>iDS{TR$nKPp&B*swOBF8Q|*pO z2yLAgS*U$Ka~vEA&*2F5|D=Ll320#ao-=L#W+cn^6qVIoRcV-_yH|!z8Vi?Y{$bx* zEhEr0UBe#d9$~Y3+eD5<^j`QPjO6>Z*BP4U1;FgWII!esn1x8Dj$F+ki>}AP2vtJv zY0E_T$zD&zj;bcziwHGQe#o;0Fki|QW^TPLQxMf5%VAl|8?Iac9B;i6@FK%yZ#6{2 z%~QRpv&IqlrurO1lM@+oM=ON>={9A-W;0cdAe$cS>dS6UZ*dGOJ76yLE~thG`OKEH zv|zC_56_b;T$pBauE|>MV%>5itK}US`^YmEXO4NG^8H7lxPE94YuAcBL_koR>}{_P z((z+o!p}QunLGZ%7`1bfR>linE+O~CB+^52Z*5m;m2fQBs|ez~*Ac>1Eu51XUd`ImWu{><4(M&4RG z@t!tzahF!)X-Vl#(?0^b;Kec>d`l2!o8W7!Ib_!^*lKs0CiIll{cm27k>5R6pnWib ztlT#rzVF}1m)z&72Gz#1I7`#me_p!F1};w(`1KUXD)CadbARN3^k6MPW!hFF50N+YFA zQ&j&<)^4cmUxd~hHxk`|wQVkLL+$~S*DZx>3>f?9={FfYx@jZ)@Y}x_yDSmx-^VxL zB8)Nr4sSI;Fn9WZcyA-hFRC9W^b^`hOucuD3{$-*UZ)7}V`20z#=pSPm^65G?FDlC zc2nRSo{kj|GSpPJuW(n7GWF-oC-FMUcJgKM8uLTWXYRoi2;S~bM(ZP)2??y`HhZjO ziy^pm{tJGj{Am8Be^%64CiNu7{MV-AjCaK7u&e zd>mK0K%jB2GR*kz`>=UEx#i<@as5+S&e-4-6;$`UmTRQV^a@lyZI?0BH+>15gm&PO zo^oLQX2vJ-=gr#??B&!3@qUGyj|=@TxJ?{jP+;&6W7En%9I(NCszd!zwY|^7JDR z5xYp-&%$$fgyWUIFm_&N0~nWh4!!%|2Auu#0?5`TW*>EE7(+Z4`Xk`hbXKq>(yc(; zKs$M#`vRY9$k?N2&wT@K1s%YwEE&7-bv``cuuvA^``9*;+<)X>3>Oi9Dt7)MosJy` zn9WHrq3#S=CKn2q$4x;zJG=9rP10eNxny{R5yB&abMq9yw0=)iu67}#Yd80c3w0j2 zkwX7};A59xv8s64l)zS%BWN7tP?duj&I@MO#gV z_C5~b>KdJK8~(J=1D zd6@sxRm{_@7m0+wuM?t4X>u(nZVU#yZoGmq5`Q>n^j4yFsJ&nlueSc%SHE4x67w(+ z>j2^JCCK>CkHF;3Mo?9HlhALCSK@g4QIjiyh>pgwy2Dw4RoKT<4mBCxXXJ@2thgH1KI57B<}sg64cP=r=;P}$(XA!E~(p3 zY1nBoAI^3dhj8mHI*<1(eL%M6|KWsgekP__RNu`9IBHwF(`Ed^qE{qkr3KDe^I^!F zQJ}EtCw%PVLiX&o1HEj18lY*fc0p`Ku`1*C|I_hPwtJtT?gcg2z&yC;w)&1!_5b_RJuZVd1{ znMp1e#&kG5mB1GtUcvRcK~oFBWavHKkcu!lL2P$}h&RnLy5u?^#y-4i^LCVGws;R5 zkPC)aq#6CmHI`1OpJfHl*klxAbGc^1K16@lfi(7&_QPPxWDNwnreiwqBB<+;jV3{8t4a>v0hG4`seRddL3hHmg(xie+j__(!lwN&&U? zTIDW6MriLl396D&0YT)Pl^LXdlnhL6TPyAlQ_FvTY3+`-!t!S>;{Ef1kSx~aC8Nof z{v^Wg-c@3`CP(Jda3d$SQi&Y7_b@5+@Wjn8trz3MmB*8Ngh$MxK7O{?ikR)o*zhmd zv4yDfPDJH|@hIW3!LztcL^sAMI1AI_nKOcUkHW<=>A12U4qlyr#>Ko>58&62Qj|rX z6y^2y8F(G+O6~|a#@)TPncZ-65o))8=_D}?b4y#qbCHM+(X$Q@!iBziL09clRKC@7 zoOm8sUZf&iw>?BKnw}mA&*J$wjF>VY{*J01ED@LY%T|K1r@T=)W!)C?_k?>~yA*Y> z&x?Q(&65z{$DC0@_-ur_M+^o3{Tt^W1TQBS!`tHq#WMM~M9bz(vXAH?lHZ*sDzAl# z>(!BRhY8PCpz`~jC&6=#g=C91^PR#E7sK=GOJN!RUtfsu>6S76jQ`^P)t5p}s{ZE> zR+O%B7X#9AOdr(whoE)^Rso@nT_JEe6bu72eNh{%QkieoUtG$a1?0VMh)@4iiej2F8_lLk{<5PdAKlK;Tj8$BRB$4w$_o^QjvgPFUa4HuGtyWS0~Aep%X z5&GyPH$7zh9`%f=BPMk-ag;^%ik?cHU!;xtJfJHz4J@9Y2$oGZgmC-i`J%Hug_;OH!_j>`77lUk1BSSMT)o|M2#2`b5L$prANb z+@6DTIx*?lc3@JQJnG*8kG;g1ROY@YCwhZ;E)c={erputf4NLZv+Peq$C?g3vhgg< z$lddWs9xp)sCxxmiH(D2D;eMKK02e&m{qfqg&W<+lRDd( zHlNT{fmYUj&@r6x@gky6da^ljoo4P5H)6Sf?IYA<^hrbYsCF>+&!syK6IPvb#dUlC zW@JYM53}-xeqDIv(jEJNVA_4)_g)E=8_42Qp!L79^D>0dhp(lXb$|%|f1jCD1BhDD zNZ!_Ed@zXev~3yvEz$1{@>TfI-GBXeW>qP?xrbRdww5Ts5qZM6vT2lB(c zJ3nliE>0KW+8gl<_0edLcs%;zang-14+4X(fTv^xIW8{(J{ed{I?yV>V1+5e%k6on zAM1%32#-_tU63(B3G5r8MvQJX20PBkq4ah$W(_>;%L<@-;x+1{YTIz)f@2Fj)4}M+ ziRu?UyKl`P%G3lTe|#okzU~qdp-rR*<-*1g%^0~};*ff;`6)YdZfj-8M zvIsT|%Op95DPma`rFoPo!+iq`|3&X|-)^A#MZa@S1|obvXE=)aP_o2R4BxD23IpFT zJ`qIaT^;*C)IBB!<^{c4xI;MwT2Cgy!^)NT%YMd3o%J&&78fe-_CL6)bAic>Qp8{L zM}P4B^9H0<=C%+SPDW%zrz_bU(gxNIF?z)!*sKN2T37VD^jZX5cI3bJLx0$(2%$|3 zZnfm!y}*X8HAK^Ob5c<`3jaMEEN;{1SVeg zWcpi`&k@rgIp?1Edm$<#dQxsl;dwRY;6}DD;`#kD*1zwn(62S&$jl*|-oHor4G{@I zQ>$0(|DWV|h|70R(1&5k=TLp>>86l>b~<@Z?*UoUX+tTVV(!-a8md6Orwd1K^Ij;7 zV~AkQPGySeN^-D==cODdMeGS#;B;Rs2O?UdB;23}Z!h`h$y?xYq!}(@*Wx)*Iz+#7 z2~ddGi^`f+XM=5ruH#>Gx%ktSvS4{#9f%J844cXoq2JYBF)v&c{>2)_;%39c8BfW8 ztHO2U;yi9swk*7)>O?fHT}X6Bs=)gl+3;HxbKk3}wGjv}L`f~08yPOPp!s(;d}1iUQHi`;4>jZ@O;9L}0~ z;P##u2*)Lvz2uE^N6>GzV=};o;=o#EZP0zbk+ke)?sKP0e?sFYMd>hcZt)r<8@r!; z0<43PAf)jDm`3>{8Ty&tja>}*x6fbwiP4GpEgcDx&)JH3@hZm$e|Y*D!qexdN%4jl ze{I6E1EMOk0e+tGfVltJ8#>&YfCX3X168Mj3HdwBc&=88263r9q~@!!@a4l2a$)>o zy!de;4Ae>jDQ$J6+^zYbbjTb;!bP~yb2n+#(uOO}DkR1z&chF_nm`omr-AyZ%)DLl zU=*%+uLia4sM~62T6TzZF8+evsxGBr* zr@f1+5ns3NF@VdWZjvj;JV4)P|M7=7p;fDqyw7Vnz{}M29RpPNsBjO?IDv<>%@L0m6~!oYZe#Uv5!lrn3DgxW zz#Xp$GQcwlej25MFYFlPUcWDZLCKZ4@7SHFj<2;v_!MIW_*^oRvv?u_ZC%F-QG2W<7lKIb9zJ|>L$&W@?qg!5nNfOUV?!uk$#041Kn&Uuw!PTXd|Gi~BF#m)d8&BlnH z$@LQmqh-wcS~OQ2(`f)pHBS&5#xDkepUTKv18=|@_DP_2CmGTW%$R5u#$p$hP30FL z`YxpUkjZAuw?&lx_eoq{2$xGsgK3Wj#eKw|stS1@G;y7U4n(A^B(O}8hFK0I`#qQm zdJ5EGZ>IvOlJN!K`#cGhN=1QBjvryWDRT!ex6uq9nV|<1bBsX0?JBU(Usimk^U73vP}z zLvlLiTN}AKBMUlh3xXSnZSV&9iG1={8N5j?2BPnOLf9GTR!{_SmI@5J{1e{3HUK}3 z*-B0usD?XkW@EKsiKOf_bLf=SFI|}A236w1@X+jg(Eq6(WvaXe-dH8zbXj+j3+8Q- z@?E%}^mrYDcz-T27X43%zKG<(Ys^>Fo*L~!pm89bSoot1ddFCCyfC3V3*qoN&okwEFTWZ!_#>6@Qb?dzM9M|mJ zA1Gz#Z%+g%(6|ufnf= z{h<1qKKOGsU2Z&gK1h5UPJj6@7U3JzXKdU=xSb6X1W~nSuRoASTb5Hp7W3pTFR)RFo}q!}k{2r_ zup@(})VzEt>LPUsS6)_1o-;p0y;v}dyEP?Q`G}PVzV${w|53ztBIo*elJo1SU`6&4 z_#$#Km3{RFx2iK6k1exe^#5P^kXA0?u*2>PI3Jq;I};|#o&8>eznz!=vMjs{!? zCRd8Iw!E6C5Zz`8?3HBTmT(u+zlw|DeNTAT+V^53Y=OE{$_u$aNiuk0kus8nTS_Tl zMyn6%=bM#}xj|A|JYnopEHNRLx_0*~<$kaoi2VP2Nei%>_EJ;fiOC3sSe&GW55#lb zyDbDmk_U++1EV#}_Z{YCg=*8DoEd`8rxU2klp;di*9ge1UqZjwb4z9KU>?YM^$qm! zXlhGF9gp_=h}~H?0+cnBWA@{ok_nm`ntb&)0FPQ@Eb52lE9C5sN51BR%ZErAw?)+Z z6T0N&j$wi#A~=~Xigq^&0o)oX31&t*^704NNNp>9*@DuOS{?y53aau!G+whs{Y3sM z?t=%*5dXIPnMJ8uCux)>E+-b|rUNN!ocQXo4#{)0eF^STvPC>k99B{RHeIxSRnT0J zaZ{J<=0(}{xkgi`p43u-A53&+7@eU;ZK@$;K!N-Zr+hsZyBysK{d@HE=cO7oAI%e( zTyUUdr$k~?HifI^U4Egl;an0JYmg-Q7n`x3fdzPsE26Zk5~^Wo6~X$s6cbEgJM`=kvU*r3%Wr`)Fb-k%qmz zWGB}D51-*DXukAS?#zkHy{JaV9s2m zk3hIK$bWcMtpD|nfuQ2jPq=wgKk?;L23>mE0q3}ECYA5_a;{uv?0y0U7lJt7lo6`a z^Qo<^YQS^XNvc~tjJ?EARsMc-w*2qyr)4MSHPV*$KQWnudwGW?w29iLbou!M%4Fv2 zIegp!fKf&fifKOSl3CwRs+QF|3j|3U$T_p0@FixRBt3SznF#itCj;MlX^8xgb_Xn{ z`}hCi+jjEoTEiniBHm(Dt|evXdHoRR^?Rsu{9LYFPu6J4E7#rDxj~sK9$kv~Rz#!$ zt@A5Yw4*unAO94t+mQm8v7}63GXEs_`YsymH!Bdhy&Fk=xpfw_tU4>`ZSBIsv;QAe zUmj1z`#p}rU1qZ;vWphVzRojOD59b$N{h6pR7z=4*&|DlT~R76Lc4cy=b52WXhVcb zsYodm?fdVV`}+2I{pKI;z0aKIob#M#JFA(Ui`7K6ov@VKYlLgt_@%OGCL3x;@R49& zdE=%6zQq17$QGQz-`+;;V`|6Y2{qR+ub?=6*Wy0HqHMAL(|`;cpPTJkH*WRoh;=Om zs`NPZTf4@gA%jN3h?x^dJMAb$D&o7)(xSznCU;i{?{0-{Ag^!zMFh7R7hn-X0mjC_CArenD-LfP05 zm$&=armM+} +r3qemi|_Ht!?HKe<1lt}rWaAw*QY#j9!98av(gGT%|(B0i`0_j z|K&++-Gn|=Cp7m^=#KFhm1CCmsE?kajmobk?B8yDtel^7ADzF;3eOJg9C?G^FfLK) zl*;!hJ@_i0xx|JCaysQ7&hx7-`Ut~D9Fw1YTt`w*@;tgC;+cx$^2DpYu0XtuMs~rDJ@ECQu`=^sV9f zO>fh>IQ8oIx!O61?Dtzr@}cr}`0A^|y{8X6Jo8z#zsdRK7xO9kLb6RcSk{V&9~npgcM+lV3DX3x9MW!LIeL>;#!8b;5Z3 zeEYz`r^27k&iJz_M)tRzOHgs}NVKcVgYer~&!3)VWnbx_hTgFqHBonhI#QT6mpG@A ztrxtvM_*~rxvZpxW6MD6tMV3=eV|i9$x&n z_G!eCSjoT;e&+*%&FSnraEezpo2Lb}IO^7O7AD=p%>G6&eXx_%>Q^#|3Z10ikdA52l$Lf}*@x|dwI6cajHLYk3pU<^< zG3^KcoQ*X0!E)j)0b9nPb^5!}XZ~xkFpm(fpOq)Sb7zg!qmFSj5BK>@vqnf%2B8*L z!-Ov`P2p~_BYxzjREK=wsxjKI zd?Yb$QXgt@Wj}_$a#IXCu?zX5H$$-WyHp>vMd2qFntlVd1k>@vQbi80LnHNw*1eUe zqW3h^bjL;1eq1tkk8diJ=<@mT+V6$y9vH z_q{nK*j8n29DTYHf7LS|S>rN`o$(AS7Utetv9xD<2UB_b%A{DI4p2;p-{9UjO(}0)r#X` z`dVwDd4MuDH+UuTV#9dUKj5x##kvK!$Gvv0&%srOLc{EM-l;knBqMz%Ca@lhmOauz z_ciavWKSjGGg72})ho~3!J@N{{yW22ETR9==klJy{EK4c@3=;xo`wfd!y0*Xj?5=a zHbN5jIcUt`k!d6K$F_|NmCB13%Wr>9{(~Xquq1c;Gh3YZA`h8!;_koSO4udk!a!#LcG3nUk%Y2N71EquYP0Y3hALkT7vY8Fzy0zCY)fOpPK>T_;e z;>WdL87Pg%ewk#7wSHWSr|wSYd)HpX*Av=8^D83>_Jp-S=XW-qZL&)+B2AGSFU=;g z;MV#BxbgSCe{>1vZO5!sm5F0pvT)gBHMlE7VH+>%qBYCC@w}S1inos5##_f+#k-cj z6UH2uC5kk9F}@*`&8t%PZmUzmRbNx&*pUBBb z>+4_WG`kp%Cllw7;>K}{*9OEZs2Lr7RYeNVJNtxS6#M9<@>vKPvH0ibekCZ-0BJN;t=VzmQUDQ)nAY@KZvgIUFY{rQkb^ z9f&y_)cNoBBx5w*hKZh=;{Edv;Nzz>30eJ}!tM*7{*8}{S)OofU?WEZ74sE$UFnIO z;w9rL3PpnWK%5wpriCPI+lC4LTaJu2?%*?TEdIS4Y`&P=%MWgl=8PKV&jE=)TdMMq z;PP@m4xb}`V>sCF<6?ztKZXcr$y;-}5Rv0Z;_cDZg5!IE(6rK)Ycu@*p;_wx*0iN+ z`uON<$o;0M+kiW-dV|i1kmR$M$t8RT#&Uc7uB+1gE@6S%_>MXs?p;Fle|G80Z9~|G z!wU(mT6-+MOd97f+|TfHZgUNOz(7)`Xsa{6>c~BT|Jzpr^TXqWYs}gas|slhOK?5K zU;b5|uP{CU)AjMkJ|`jk|DJc_llXgtHZ39Y_v>RZezCWcw>N7SHnw-J zqJ~j8UgUgQxZXEjK6sChAbAMl=&os;98`#{ke-eXG(7zoZ`6uHJa>bbS3&--X5_H` z|DDBHJo0A=#ye7kV8^sk>Pk7slh)1BbHGb98(n@whnJp}#1p+L8zHJZjCuSB;o7XL zvE>(Di{!?Pf3=PmvZ@6;8w#-9_0HI3M`_%W|Clk{I@$CvOQPRc$LU_8l34zvQPZ&B z=K}<<%wBS1Rz6#SSCO#)-&p;cmBYbJB*tJRH7ovoKg@&OH@|W3%$PKu@Ep|V=48o^ z7Ub~^5%Kn=81<+Dc&N->`4suF*!z!ze6td1E`bSF2Dkx%AilR^aLd{le$0>g{2gQ5 ziN%vLu{BeVqb~e-v|}2BHBAf@9?5^eU$^WO@<8b`?`Fz*#Z%qs0<-0&n0PRa!zJlv zlHjCiA}5E+D`Ro9#unZbHkqS!joBHb>gO#qHb9!=dzGRC*X~MuD{i&wi9qvDAhGzy z0MD8W|$CFpSe-(mQ}&j`0O|K#sX^=^FJh^rVje-Gv{ek7sv zqL#yJxZSCyRlL=k1d6n86xUv@`vq3?XEiq_0e@=csm7K6@Sd>pC^!DCb3bFw-Z`j( zBxZ>!a>I%acH=S!lu)!qRZ$>`PT7$~33rPQa(s!+ROct%>*W4lZ6L($-460|gQYo` zem$r~B>8{kPpU|3O+0MhC9pVfTDb1wVLW^eC5Vcz=IE4__{5)Ib_g5gT#4Qo%M$bo z14X~HH}QAVj&r!}yx_urUT-XvpE8nP?*AW$d$WhM?k>|B!M|v%gvD5(Nb3(7bivIu z0oA$y|90pdcUHP9|49IU&rzHq*Uq1oR)Sqx+lKx0DL|)OFUDU+X5bMzdN_K2)JRF~ zC;YgEG?)Ckq-MV5cBvc<^Khx&1uU{yhj08S8}&_kC)B#aXJY)qDPTrXGef{B)Q!CP~|K^OL)*oz{RsqlB{BmUD;S}WB z`7-R%u1xt(If#CyV+E3Y;5^}M8B8m!hHsZC$;qP+h&8+zj_0Z2-|$hkAJNb+vG|*5 zGyjeIa9?$pbPrqLMJQU?wO~2>$15(=PsgL0Z1C>PQ-Y0});Od3j;HA>tw%X^`E`6) z|8MT~fP*uq{uE;R;u`K7s=!}6T0m&O_T#u$j>ggyD%7_-194a7ZXdHa1kE3#R$t!}))We@!HuJw_9} zU#MVS(G;TVY#xqTd!bj3f5Yz!e=6*Y8bZZU(>Ok<1VjP~l zKU2QtJjoxf|NAMMhke$r<#=zkE}g5VajWbjGF;10PjvwY!^!pm|Hz1B-qr7KaEbn3 zUfBlcOY;9I>_A;m`X`}SSGbBmD{t{Ex0fm`;;V6V?V2miK|9P(og^9Fp@^RxZR~@3 zg&USM2iN0rt3v(RjYR8&Onm7#5ie`-G2ZQ_!>>N~znp!#6YnTYim?CG5_!v=2!kUH zf{Nv+5LfiVyUIeib!6~~h9EE}j-&JLo-|^ji=1%JWeT_1x{O!y#h-9-*Tl?`#aKkE z25Qj#QgB!6fBt+%SyMR~8TC|}6LOf2lvHQT#UqDrq;!zu(diD!wOaDbPY)56=k)V! z7YVufY`b7KvD5GkpV->LPx~T6)LHN4)}7~1j&kjX-znMNf{Sn3bD!5#M-WZt)l|w9 z`-CQMbd*wC4|B5rm3gNecRyG8^WXD$jtEn&&39JZ_58akUL&Px@*he3&xjYq9M%0Q z9~7i_sbSc|kIi*oV$tWtnDP3F>WrjL-TFHtRMu7gQazloPh}A!t*!fhd=ui9^h))= zCl>g7^J?t)A}#dNSU*B`>q=C=UxkbP8-}%fm()%>{Qq~)_o^ct5`~7ID)`}#C(vv3 zNMcAOTQzG&xbUQs8d`7aL_~!rad8yQxtFnTc&Mu3;u?f@5)h6zGthwX4R~q6FOE+a zmuU+;tr>hMbR$mJ79%&cAEQY}%!x;L#EKicb2-`5H+NJPSF@*9Rz2C zK5@Tk9DFOtT_sc>#%po;EM#l534VFYc}(2>3v;d4r`KwvVr(kzyDy!}u zFKCd-h&9r+~wdB@EUG@zgw+) zu0G+vd%rNzTn)1i8X~eCwN-o*0H*5Vi6yKG!spz}Ma$Nv2>zR5g!)P{g*a75pid5* zRxz1YhxO|%=5YBMG+Q~g_am3DqG8(zVxQd`;nl`Z+_~!!bx+NupckvpX1R0Z)@fU~ zzbEhdt4t}DoJm|{m5T0e!S$Z5;&(XC;$Wc9D~0t2Myg*sm4%_^9>e(@g=l==u?|O$+ zUUlTDbbp(z+#pm`pT0CuB{H{x8yl0R^GbWxC#tSpAB6^JWU6k~yT`+mmdG!Sxvv(W z=BBi?>IJ{YAXv5cV1b(3yBxgm(g@seOd4U6B8?vy9(R|?H1BF)oX|lm&&g}2;(UC~ ziUU~ky92@{&F;K>z8pv2aNDQ-7dcrmw%vo9t&-FrI46rMzI%o(^b92y;<9Q_eALx0 z{oStS{JsP|*M5tG$F9`Fw6-n5tEY%nqLfqd?I{MxAD*dFHmM^R&JUB}F3WGZHiqM` zjhhF$qU$e~qh?7YZ2yATPMkx0c!bGSX360xi$d{3xFa?*-bd+r%|_v`W*;IhI)LME z;`&L*t3Q=0C5Aloaqu*Js#S*kw{=NG)R|>!;cW+y&R0TJ(YHJ3u{Z>uoc9Rn%jqU= zmT%_Xy|ZkR>MnRnWlrfjv}%={+ULe6Dj&BT!c-%&RW1HbC3IFj#;q+}RRTQ^;JeO` z;P#JO7A;t|v9Iz6-7LOlNguCY!yd2wwMi}N>3>+%^9;1w>aWU`6J_|O#A?*AYq{Eu z=Y|5c$k`n4PLI{##%|Lio}hS5#6R91JyXWbqvOO3Vr{FUT3pt7u3eJ+Sk;)2X#8_| zqRReq2kcw>A>|k5lT^*xOofg8C>G*vj>h)73j5_v@LEfW{@=bxvUk)H=0NKIDL@Yp zH6`mYCO=oXZJ`W$hba2DE?xH5Cl+e8sA@*r@{b1@<7+b`ggrqfgmy=rT7>l|^uAAk z%JQ^R0y#;}jp6a{T{?~EpXkP)<r%yv*tdBM0$9<})*F*oWMQ7Z3^K1pjQ`YW^v>b5Qxke#Y}&g>KtTULq1VfFaa zv@m&%&gUvGjzq8*XAx7p~>jqytB!wG}J! z8dXwuk;)c}Eu`_>pC2PTHhJ?aZfzrw${>DsNCa|~tm4lxc!-6R@=>bcEunS!obcK< zZT?>!R*7eQS@jDk^{wA3rP?1>f%WuD&sf8Je<|Lec8J-dCh=!i zk~NUnxb}uA#hmw4LFUw{KF}8u3qJ9cBPD^cPJ?->)r<&BiO)u-HqKX->_owf z7%i@ky=C627B-`m0%mFmpPa77!{jHcHKkOje3^cpm{mGmZQLpiw0%PpKJUdnPA&%n zUL&#g?>QQ7^;W8-9tlNX2Om&9maT-S)eR7{TQJoPxf=MZLKZdQA4M$Bi8+}a#yNWE z47W!$UIG91gzhdU)u1d5BrU8)xO3iq`4#DVIM|W;AJIGW*C2CbrMa=61a}e4nB8c2 zv&2s69ZdX&WzzMbQAa8KQi}AIm$ho)uifPF@(WU( zSgFGfj`w-z_FuC17d znfbr(Z5z@B`^0VBJG{z4!nHu^8?GFA4a2kLmHr!l77Lo8PprCUqJZ7{h-vpq>e|1r zdV}va^2UxkxD$v($0OPhhkkggtaR0P8(!d^fYpUgk@$J0<3m?b)aaRu(D>MD z6x3M?&Mdb@R$#NyM{pGWDLtISCDeNjGTm_!x2|Pxm_mG>XMmhbTE@}(Lv#n#sT;s9 zn@epNPcBRS0*!a&5#>Kp(HMDcr30nq_~S7uf@WhbU4L9eja<>ZP)kbkR zdM&I#&v;G4&E!(>-0{w6W|yQMXTCK?`U$z;m3zijz=i2;ujXy;&Ka)BHO)Ja(zi^_F|{3ukr+Uq_w66&9!K0vj8)G6G9Bh zu15KZNgSO<`Jb_qPZs03v$8Pt_B~1~bq@>L*JcoT>|JgT*OB;_ad?Jy z8Lm5U51XVKPQ1P4PJB|9Bj&I&{83lW;VP4gIC+u8g`&eos&ZzGVJXq}W%4zAJczPPRXQKaJ86oW2o-jdMDSLWg4EmvyrEi6AA^ z{GkzkIz~ZhO@~zf7x#xDHl&qeWxh7VoXw!Ql zkksEL-pNY*FrR%UJ7^YeSI(N@{Ka$ z(5yN^r&%vHDS%H{(ODy-22K!6?iF<7_lrdOxrs_=zS$F98-DUV`+7&n|pg=@ph4-tHsK3+-FWhSxT zIY=mMJwT8#41yk>i^fMtW1Dv+*9mu&pygq*Z3w0$=uwFEY^cq>A0oZ1nANtkE@=m}u95IR)r* zd>e+9bmhjsc|N&1fOz#DLv7H}c zfW4-8__iJgOe0f=F#qNzEZ0!OjVJBkZ27J7@rn=D-@%U+?8Fzgr6B%(Iq2ms5kdAY zPz-frx%FfJ;uF}NUot47?<^dslo5Ok*2C4mD{!=$77QV0MoZ&6h|P`Y`!YYgJLwjt zJDb2CpO)_bXhjIe%k7oxG4qHD{`aeP@*iX_VIyHRX49{S>1`RujWgoa!S7o#;U5p9 zpM2+N=yhI;@s|?+zAw}`B-q&~=Ju(p-DimR%cXGwQ?Bm9Hif<>Uep#~qO5*=!Q{^h zjoFEqkKbP5vS;BOEftmb@Y)VTtUt$+sQasqZW}ueIWS6A>3o+e?r}8~yPkXkD>%9q z_b+A;#k7eCd^j21yWu!L>i7cTr2Je&ZZ5>Gsz_@eh?}d0t*fQ?8gp4qV(!_U=q;~< z9IaUcG%7o%3fp2epQAzJ_gn7!+=x(OC?p?QSs+w;IcX`a!T!?unlQHd@4tEAwOSrih%m&DEuDhuF`Lo(R{nUqr-%r? zv6%4KI2Qe-C)F)Fzrj(J@^9FmjgI*ChdgA)7DwT>LkZYJ-vI zOWF*R=>NI=|3-Hh#jO_>DHYY|^TKkM^1OEtgf2+Lmv*4|S-B!&^Sg%}J;So;iDn7< z71hyM6I3~QU2z~BpN-04J{8vdupmMZesK>sCx_d9+5ow;_uDCXxJPdUwlzVBZ*}{F z8ElOsszc4NByzFf^`Bt8x$ru&VS*utqw*pla<<+NH%(3Db!*f4d4oM*5G?y_P#zN^y}EB-G; zso_}%|K5nj1T62Du4^z4^~f;C%8U zlnupj9aExV+70gAGI2bTzpz+PhWx-Z%a-8(HBZ9x{Zg>&o<>Ahy()jbIuuN7E7**7PP@VJFVaQ8lSw>? z#~auQGiDB2Zot6GU?NgJ5os?xPDE-GSfb)%!rOWz zdOKJ|Of_;rPo2C+bT#Z%@bmzPL0J`%w@W}=CXR4x#>P4w{ENwZVw`Lu;T&?BXjM(b zo_2;|ycMH}t6zTe)jI2l#F&djwN)dwcG3W$dC;C9<_NJl=TeA>*6RXo`y*KD&`VrV z>xF#XqCCRoS0{4HE1S6N!&7ST_=W!sjNzq@GeNgUOdt8QK#a+jO7H)g3zRtgFVuyj zp`{M^%8ZemtjO29V*FY^;qvK;#MH_Z#TgZHn8$lPVVK$)Y;AE8c44QYlB8xh@~|yU zkb$2fOub@8+Fp~|nuqiMpS;$Oj8B)w7+Q}>MGGv}aB$-f>=7O?QC8}3)g>gkM3G&4 z4iY0P&!Tsy`eOb$1jmbkuqEg~aW!)7dV!*0m@a;3@BDxB&iBPVtep9Ry`EyMG&|!E zvN+`__M{6Va&5zitS?%`^ap3KKk_kHkzu943wp1 zf;|yLPER5|Pv~$Nkc!eo=~Nqp+GI2)8ce z|JG2N9{d6;SYSbX8B9mBqvVu+T=d7rWf|cvZckB3+#oW={W2%B!)wLCEpi-P$&cFq z(UCl2EdG3h2pxTDKUx}~HbPSWo#RjB?a`=(J|AnlDy<7nb^j#h#1{#_eEothm>fgg zSt!+oNMuy$%AH(Je#@5>vi9FzursEf7M<-l#JZUzvNzk#Qk4#I)R%%atZ?2mDtqgo z$h%b5$vNhA)u`a-_OGF3b?lm3S5^o%SGz1Ib&T(Yj@w?Wa4gSTCDwO+X18#Qj@@T! zxBd1)1^b}CZFXlegY74W_Sns6erk8IPGqllkF?kKO(DqFB-?!8%z;?)|NZX&y2=(Sezm7^x}BWEGhHF)fFJJ8Q5dMnNxslMQpb+!o(6&8b>GgFwU zW5NE{UHLfaLg(DL&!AbwEpF~FXTfHStf zB6-bs=yuOAIL`G7-M{cS^|4@YP1WI0_HF1xy3yo4lQ`HddedUThWxjt#&wAsbw$IT zHBL=pZX7vmtI1Q(h;>(jAm;#8;JJnyHi)9 z{(_{+RHlC@ggO7|E3`n8ImPzjN>Fx+>QIxS>tNp9`*?=99#jhNy|Z-k{Wawd;qUq*bSVG*%bm#UJ}Lq~$|`&NmcN3DP@ zcfZks;4JWY} zq7OH#SXuQYB9ld(qI>KD_EKmo81Zrdx^FoH++`laRZ(Mr?_wFMf6*T}dT+6KG?h=r z9s5NVESXHL)KOr9JZgW-c(!Ge z9#|v{AvM?QQ4c8@s$^&-6#9*$W^bfL)e#7CIXO2VhJYe@gcD)3CM794CJ z$5`K-PKM>21buTq(nBE*(Ag)R%87CVM$flVrE#{*)|AJrmW77MIeso1fEj?5Fxp<-zAZL1cjLby`jXGQjhGQWHe^lb80HhU7#zJjPE@+?6tkhKo@rbe z!rbuEt(kSql5O)pb%c|N`;G7?g8DOlo)1SB=CDa0&M*t4^`i;0UAxR%>6|!pnKURIBMTM zI&A+aCeAbgwyYrFFNdojX3bHjRqKMmU0Wx(Mdk+CHBFXwQu!tl8rM!__P$aB zs&cbny5el$-}Qo&q;`bLj=D_eeIXpbvI5F346H8xAfg-dya2NG8~MZ;2fM%Qgg1Ta z!P{k#FjnObD0x3bKdo$l`P&|mD+A1(tZq6n?z5`NM~g>;`TpTh%}R!YsuHvvPL@5Q%{ zjsn$l9s;8!yJ5kIkyJ9iiCxxeCNi+gWv%u1vj0W4QV!{7M6v~$H8=mH z`{-gXG2D1(8;qDR4w!1yz~U|)u=LX*mdx*F|L|%-e@s608adAD#^985R1R~jz6^R< zFBIj)OsAfkj}xUY5-}B}*V*OQezEKGvqT%8%Gb!U3xJ$MOA8|8+7W zJN+}HFQ!uu{{Ci0eEq~MjAB7i$165tsS5e4wX{2!pj zt%LOMRsp0L0_JTBpf?Og(@I^s&_B8zJQ?wWta^BfEFSAd(TD~!bILRL={O1lV;;eq zv5v4(&Yt|LvY(#Jbc4N^8hz2Gm|BIn!O_XNjOL9LvTL3YE(*KNu(u-N-1Rz?PjR zFrobjd9OJfD*7LR`(JH=OwD~bF4~N2-Ea|h->Ij|lFh-}10iJnq>;?qz2{-ww^-1c zafenf?t~g~r)c9JRxm>U0vS`uXZD@|V8;Ct`utLfVaaYR_+4E}*O=b|d)}TW_0&q~ zIZi?t*7F0Lhp)hzr$6a4RbudGnHhEZxCUq`<$(#^YG7>LRA8EK0$a&WS}6O4Ts763 zeDbo#sr>s6I-&9uePCNQOv#mH4(k~K(4$QmZ>bY2-KK$seh#_w;6}%zn-a<6M-Wi= zW-*kh5Hi0%4ZxO^#_AZyz4VPZd1`95GjMnm3=foS1jkRdIjxsCv5VKblD5&Ebkn9| zFm_Om@!Ncjwy7GybRawEXk`rko*{%|56UsR3eMo**dk`-dqWX8I~zDUU1Lu04>K3? z{$q9rJ^}3uzA$Nh|1sjhJCtppw|J;QiC(N{1n2RmLIKc%1D zMbYqsg0pC7#MD~1$$6r|u^p6Y#a$6|?<~75U`^`j zB}kRI4(I<>0lyOt!_uAxxX*hoS^v`s{iaxbKjWr@*b(JyO|mLP_!9c192+nc}) z`Oow&2R%^Z9|{M@R0BF|q9{M-GFv30AUb$PnVlKnC3>GE)tX!i2B}|)tr|- z%4!wPgikN4Q}-;zV!mi9e6T+nZhaIEJe2FZj)U1hry!8yPy*VinfR(GrZ&Zd z8K@~?#tIHIuFeb-u%?0DocRYz>i9uoFAsj7Za^(~bBz4ra2Mvwj-vur0$>`ENiP3Q z))egdC}Qh!L=TL9i2kGo)_gXqqJAHWCD$mAVEuMgiS8bittl+a2ZuDv$saQ2V*9lQ z&~}GAke!@Jzm9hXhaW$IxAH3?JJ$rx8?6LRxZH=w4hX2!9pA}{`B#8Z7{RDT-2gG> z@lM?$Ra%~IBvtS1h8s)E33y+rV7`$^D_ z{Q$uSa>%~PH|fItm!Q*nFHodGQ+KqQsf6}OkhyjN)jRq;FuJ1+4YVz&*hMs4{>y_w z+%jP7rFb|VbBBMLE#Yh13RrwR1{P1hO0GBj0fSmZAmhh9di`@X=Hs8kbeO3oP_40t zznySKTdo4!+NQ|pT~30X$*s`xyes*nSC?_Hw_#f>j3~=%vTWUz)uQDJvg{cZk?4nZ z0h@qWGLLudr<8x5WA5%wfF{<`TC;r7Z0h(?S;&e>Tot>&38t4SW^_GF@vdNo`R#zR~hB?cK|Qu z9BBKo3?`pj1mdU_s%1I8)Vt`XJK|&UGw+#&$yJSMr_K zn~zba7uGr%g!j^GPt-W^KfZuwCo`d;C=KjIf08e@o&_G?p2H{Z>eR}?5e!f|087a> z(AE)8m-4ofN++!84ISkW)4N1I@Cl^XnHf4QTT}|8_X(lpK!W2;hug6I@gK6|c^=u& zbQ^kyT><0fz6FRV8m_AqLNl`wAdOc{M!P49%UpQW;+`&2v{8p~uS}xl_$6%MMmZLsK1L`|7$nA>3}%3PuN5A51YeMK_IYP z+zV4*{)X>U?}KBHCo;U>kKmkwC2(OtHY_t-P0!Kf!R5LX;a>i#$l8q7=KLiPYT zcJeuJ@O3eqx91KV8Ty>IUu_8Y!%mQY`7y2EI0#+O43OR@8=%t#oU;DZ<=8%OjrKo3 zmY$v2=hzkQ2!lQ+LZS96r%TG=;A`~(sL?71Kec^;wn!$(v$6;Fj4Pc#I$ZqlTQ^-( z97E2T6ADg!iKJ(H)qyJ;4$waDj3BSLiJrbI3rrij0wJE4Gy6Q~+_w+*q?y6)L!X@Zr#0bJYcuLGI~u&*tw%kfw5Y=g z#q=3A6=)strGq{;gNZfHV1bGXWPV=&Yr-<%-m?AVjEhRle6=L`D_)sQ!J$X z#Pb-(RvG4<^$ccg^ekp))*DbHc4Xc)uBA5Jiv*h;9>W&Olbm$Hj2bif64_m114j32 zf%VjQ7!fHDr*5|u<7btf{Jug zB4?FxH6w%5YLfQKIaeQ>T;qRJqekgXyJ+FxOm=0zrt^YRjnty;kAZQ^eR}>h4XVpA znX!C6hWTOX&P<(N3`{^8z_;K`(u!Vk%Uc;nZ)hPMoH>GO&)oxL8hXH?%&}zNuZ;}x zX*u=4YY*f4Nkqzj{|zp_mII;BP2foPz2MfQqrkZM1=tDv9Svz3I5gg&gM12T*GY+x z1?un!?h4%!$AG8lspOK{5U|WciJ5_Q!fQ6aL1|bjTO+n1(=uGxf-R0Bi#$b9!zCLq zg-T~9`wAA--z8@};=xqiM8^dy zDAM@#E9Q-4tvi>bL4A*OrGyLp*+KVginPgMMNrIaFBl`5=%px{d7=`weBm+PW)4)@ zbV&XhaAAz!OryqI+yFnmeg(6tU(rSh@|4V#6XfD|o9R}SNAwaWZ*o>-2yAPf58TbY z7{Sd{va8USCQPE~MKz)9Jk8T=%aR0El3Jq1`N9j4@Vh~cQt^LGbJ7cDVZ?7%{g#dB za{q1G@tQrD>KP25r1?WjlVExd?>?CS0i*c&DRA+=#bE2#da~_{0kwM35qCwvxi|s-{?ZA|w9|k?h9c7w{0ZEB*Gn($ z>L(A|Yyf|h9snD?`C!uyIq_XR6{lojBslFo4zi;hfnOU%9^J7M+&v>pjs0r?=p`1S z@$_=)m)$;4{$7iH@xlu>O=zN?mj8ga)<0#)M+@2aE?1a{h{a&S_90;T&W_n_T;`NFOKgEVOx5K4gUE+W(5YAFgpeO9)gL=(t;Mw;gcvxgWnjU=yeos<` zxnm}PcjoWNMTK!tVM98E%dUekTF1ecNP`2Z)0`Gqc|r1^C9`6v41B)613IP!gW2pQ zK-|<8KYbr0_HtCBbZsxw`+ilxqXi9MYj!s5IOzcgjtqdfEhoz%(d6=C_MVYBGwMro)$ie?1`o+))8v zZkE6h!48&J6D_Kf-zm}^`;BU|dBkqJKaT0{m1kE~?55Pu{be`Q+p!7nRGg-~ZXxCQ zEZtPLAMzQ2lfKuV;kLujn^KE^sd22d+ zHsK~UbJAw#{38vbV`J57#^m|2hn7Y$qqIeg?`>IDZzj%GMJPCz4UQM3rYA5nrjKH& z*~>-nusYK|?K>Dw4l6|VA1^_jzZb}#W3PgqgYj_fn@&iY-T}c|N5hQ0 zALzGE=Fke5z=<^lVE>$>AZZ~2eQxK_RdV|9!rWTY;+-$3dZz#^&0jz_yqUCnQ9_2Z z1eoE!1RMyp14%E;;41Z6;I?5e+~N}pjZ)V`?|?frJ~h2s_m&fUXEX#=_VqgfSznN_ z$cVAHZ30~#6seW3daLY8ip6se*TaMffU(l?1mCu(QE$5sz|cY;N-5o((%shKq*_)0 zf36)tJxr`%{bpFP>*wBf3b1LS)}$%1sM`auTYWz}nVcf(F$;q}yL+jmkF{Ve^N`Vx z$^$+o>Qv|w9#c6Z5(MQrRhww~LXD(QFayznALGUWlLhNZPp5dY$#4aj9U%+Ts(MN9 z3r=KN_bS>vejV(Ii6g7hzB2D0E@w5F`%vamKUgk2$XsqeOWl^MWtut_p`oP@g(H{y zp3Yw|9g4nL!j~@Bu1f}0m>PO#p+N2{Fnp?-fnU~XH%hptIVwMyA3p)YA za%V!D33s7<_er|U(vv*Ct%`g$?-JO1av>Zl@dK^$zsWx-C2+>^Dq7`j8hDZs4z^{F zAqjD+SnV_oZtHizrtTf&%A*^|9AiVUw>cV=IXr~HxvPQKQ*&_c_qXZ`<-h2Uazili zQ#*OxR)ws{yCJq_AHlqMf;o0F0~{&QWWL^iEp8r{LVo`u$0+v$IH^?#sp}FOLc$&3 z`N|XsF0KH$OBwvTx)|&g9)=wg-QaJ{!_Y3Y7#w)w3w8-<_;PGC@O^R@JoY(4dKi0< z#gpw|*V`K~f4d1B#2&$#lds4c^Uv^!jurfx<_5n%p2p0Ny9Z6v1t6`=owS*kNvjEF z!_k8mD2;*uYR2>N)b>MsD)Q8H_G{fu=K7mLYPEZv$Z$t5;}67Ob$A~1#n(U&J3c&A z^_(Q$R*BOCM!-;C74rJl@l3hoOe=R#ff}VYnle1E3JkIwVB?Ap^aZPtz#4vlZ|uH+ z)|uh3yKfyhT9L`>o|xiX>J=ht3kzXa!e!2%7x=U5T0XK}-5#Qnz!S_Nt>l`_h6wRN zyCP8E;{xUWE}yTiFL-qfK~K z-p5Cz{Hhd?{N^$&ad&|;6IYQRZWOROD<_DM&d1bLZ4dUzm3a0<_d4c7>kg)Dem;BF zQidHrlZAaV_K-de2Vlpka(Zp|TTp%OE?jDu2D(36fyVL_c*MU)e5zR$8jkT`Zd2*Z zxq`dQ&cG^4@%=vP&3lB;{jQ{0L;vZ~DvJuveoD zzhTVZrDaUe(8z4wf& zV(HpN$vFolXAlv|uu@kyU;<2t2Vx%rRBmz>_@4Lx*GmLbPxf!bCto??JEyji@$Lk^{t*||M1`S@xd`sEa7JNYQ^14s z5g=1A1RgtY4`yvgr2h0AX?JqZ;zw! z;sm1Q>mBOuZfz=`I7gh)kHeO^xne)R9D%+3%h(x5SKbe=T7e{&i*b+1BK)<%QR*}O2R@M?z`wE;C&WZoA;;|=)CrkW`~we)QS`B`_~*t$ z#O)g*{PTyO0p5b^R6}nKe(2j{U}2s@{P5<0Q%h+C_t!P*(t#1Ud%;cAQ~DNKu{S}n z8}lgZ+XU6$>y9Q*=Rr%b1#COKAE+@k@KcHn&|2&inQ@ejCAZgLOY-|L+lhFVEGLt9 z;f6A|W;%*(-myedJk}PJJT`$54xQx2SxFG9?&EE5QATmBS&%&}2x3$fQBy!7`4wb< zwA+_3iNP7*@xXHM^5p<5x$8yNhDU%mUDu)K{#`ahm zwiBua385X44)CY37Agg${Ms6i>Y{T0ApZJ^=l z*RCS3^~I=GJ|8*m4#O=!Ffrd%VPMyNPi*!0eQ@L=4)RxIlV?|GVcO{`sP?7+ep6*1 z5Ig3Hi?phtr{7nho`4wWz?VUVIt%b0xHFpZDGe2xG-Fi~o`~^iHHeDIB1;x51HJK2 zu-j`UpdQKq2Eq#1o03<;#&|8)9WV2;pl&bG#RC#o;>BMo zxLvaZaQe;xSkPt9=`rb{e5yd&=PgQ5VS8&_h3A04aLJ z4_YlRf{MkX(A9Di=nj`g9~yee{(X)xsV|8>V|OwTEpkK)%uF3imp-7flnnUmJJqRS z`&s-g6>*fbcL+Flc#7Ila*%pdZUl?z7BJ+~GOEU^V zgw18rZ73Q(GZp}m+C1P|`VbDzjmNSNOG1&mPq2*HEx=tA!QhL&Anoc#%Ed+8ksXnZ z3&dTatkSm=`@i>7kpr`c)8Fer57eT#*W8GR%NKxpVjSKprVAvq_Q88|{lNS3+i)m1 z4EO}ALJ2cFMtj3*^z-y=m?bgIb6pFD?=*5?^!H!zsH6p;PDg-~>l8;s6MT1MI!p}0qi0c)y#pE7*SQUlM)EeWS-G6~> z_WlX|v*N(c7w>^l|7CKqbvPON_yg-ja5Jco&B8<*Bmm9S5+2DFK^Hvz!LCK$f!%`w z_>6D>yEk2f=U&~Jp1(aLUFO!o%G^C*#-upByi|{5i(G(9-1LBr{9-6nI}<~@yYbh9 z)kL)VcS`*HBy~G20zb4Pfxn;YNez^BQ$tcu$qySO3FYuP=yTvLeoCJmK?y}8rN{!x z9!65GS3YCyrAB;X)=9#(G=X@rXBB_v(+9-F($)AQu}4teJ)1haCX;`V|DM=&RGu&5 zUQY?f>N{o|=fc>rdN4L46zh92L0YK!!V)C{`w(7%EmFA)pB;fbU=#tptoEX3We=%{ zt4)OG$Mb0MUPZ^c9U0Vcp%7KIGMXB5K8SQD*HSYz(qXsh4Kit_2guXU1BK_aVSL05 z_#i|dly5SI@3xl1*XfR6Y~_CBzpxh1NZL&~PZq-!&!c#ctvBPN#yvlzxf^ z-(I4Fni;T}5NEBWoGPMls|kPV{Lg2EvseVaH8Ys!@AeRo&+WmSt94-iD_5YsLKPQP zivS_MLFBKQ17vCa7BJYXfqXhdfpxPnyrO=Dcfj`(SiQCd*cJRFe>t4SZV&08s8l-i zlzC0sET0X>0<~d75gke#ORlY3aSJrP3xt8uUa(C|WZKq@z`+YNu<8RORUdi5gnldh zSKLGRXp<9pHbDvrJ?tkl+%1q(%ThT1V;I(f?BK$m(s+TtE4ZS+5FUnlhUmG8fpl7r6Yjr+ZwF5!x{v8h{2-U@<28qmrU}>#{9=bP*IZ&7J(_i z_@p*SqrW77h=sxp*}}MSWi&+NQYgs#DpT`P4lnD#JxnEWJ$xs{gz_R`V5zM#Se%!ToAu*Vwoy}JQ2Z?=PO>=DTvD#9kT7xNyJ{3I8j7J|uUdw}PcJn%|w6*oCV z5I){`oD#z-h^^KYxN@p2|3$ns)sppq+H}*1S}?&z_Poo~ud)e-B*N>;R}GJf)p4`TEG+(&x7Z+ zZetfc1dtAO9FEp}01aZ1kY(OPZl4?@`DL=mTf+;!r%U0p40ZyA78|ms^E~W2^$fIb zkwCeRiosGZ8XQ~XkBn!@41k-Q{;!+oVIW zGA}l$7vhi?J`MBQbER=gxNmQ=`WISns#(y+HHPs2BBC6S5t0u&dhk05af z-&NN|e!V>bRt8t{KIVj?9Q`8j>nMhnSI@vBk2Ar(vv1&y+CyZJ&K7J(+7kFERR_3= z4U;%_5z6h*2G=;p`Ddo?5fS?vkn~s?!5VHyw}^ej*|c2BrbZv^IkB3*{(1(HKQ&4o zS084&Mq7X-Z&$L6%9KI=)ifCK>n8+!JD9aj3jDGcz^$%J5sC{+@U7mHXw^mv9W+qD zuf7jK7RT2>)@=mc`D>6s3?KU6Z{Z!x`cB5^R6vOWFZl3#Cw6g66Q$~{$Myqz=p8DD zPujLYUy~ox{xmvNi+DG_z}X&cXg)~o{Qd~9+Pn;UrH^81v!0<}ZodJxcIB1nq+jf$8)Zw0cG;)auv+0=Kn;;#DKClYRr2xO)I)ZxG<; zNEJ|5TBa`}UaCQ5h-x9(+5Jd=wh?8)x`uzT2_x5dN5jv3(|f$iCFDA5ODJzsN6P8y z<4Z;*@uF|WaD~}1tR}V`T?uJIGO0Ik70Z+OOkG#POBX}WCB%u(@~fzos;_Y4OM{5} zash7gYb8kXsDowaenqVp_Fr#-2D>k! ziH1)=1csMwynmmy-UDDMdl-&^cMW6vkjUZ6UwiAxRt+0c8O!f>(8jpKav9K zvk8IxGt~arPRi;*3$@+SfuN$IiT58$h@mi1zApVTRpnAc*?YTEuiVz+X^$1DkN!Wf z?i>O17B_*XoGnDnZS#M8)<*=^*q{3@uC5lqCSF^Jng zUsGy_TBysODa2)WEoyCX1O;C5@JvQ3HQ@G<8Mdhb=3l#l#}&82$2s$m?N%uM!6d1CMGj4ftON$GnSdWt59MCF!-$+Mm`u?Y%tTiM@Bdf=mlkdT+xyEQ zO(crCwRI=q)wzUtxk=e^&)zEPW@$M8R`@D@17|6*={m%}yQmZWWiMdtvNPBM>v`Cd z=6txQ!Ga8_NP$^dG)hw_s&<4uRE`XS{77P`M272GD$*l?tp^bG8Y>_zxBACyx1AH-C|9UA{xz_02HM&l9U( zeiYQTY5Gg61_pxy_eOLZ@L*i$0rHB6pF`h z5Lt*vRsy{ZyNLDg1&PYuE#!_&4Nz542S4`Df&BevF;w{pY*S1E?~1OHnh#^aWe*5V z3&Tjm`K8#;%bl?Fv>4{|QHZ?h*#o9#-T@(J#o)xd#pF8E2jCJ_kJnkBpsG)vM{#>0 zex^PjuDwzWnhh$5jrp241*-QG>WY$lP#X=U_Ehq|$lbw&pQVFfy+)pAy*mDVbtmT+ zHwR|aUm{x?)xiquyI9ZR9dM2V0j8N~Y{^n(sCdI1oLFc8JU>Vx*<=N{e4iF?VfPr- zzAch?6sSiyOlaVru56&zPle#uBh~pqyC`Da0tTY4&*isY-3Vh=lwgmyc90@BhJkCz zRam6z0No@v^FGyEV^bY;;I>T~9-0CK?Ncu?<5?eK>@1+_&yEsJ&+k#{MiKmyCt;4M zPhzQn5;}$QH1VO`O8EDCchI>FU0Ax+dBlo1iU&OB;!ijH1pHt3(8=lj-?~p5z!$5} zplfQ{FQW4RgyW;|LG>PB`XG_quN(&yxr0Eqcm!rAU*T1EZiVZ8+OPu~`&r7jZ$Z;( zf8Fv5ODym8Bam%f4+pCA$ndaeY=(g$^qzW&fsRyoa*-Bv*EkL9E?mIcKFERi0S@f# z3x~FW86cyMhm1a4M9qFrsWK-&uInHE|xFk>CNxBSpZ* zty`c(>k|??eh>0C$-t|__hGM3BCz^+2@Cy@jpZtoU~4!zkcxn0=iveD`Oh*~zugaJ z%t(M1cV^(`Lf7$R{-;(Bp3{O%R67+LbQMVj0A%ZQ5Sn&6Ye)P4l@Hg$| z1m9>0YP1Qa-0iqjon8c(u~8QsHjah4^fIhW;tly_qzi2N91CYDBtwBtF}!r|H`qMA z7ZA6037!IYxYT1;+-##Il5Kd#v)$WA4L;2$K73E7(&wkbx0c;h@TNnKBegb;qAlM1 zx%b$7jd#bXB^?6%x+xdFuH9Qc`QC>=`on?}zR$sbExiePyJZRC@(HL34&h^^>3FW< zCRAqVic4OLrK;8BiHFm%0{%PS;7v4Y*Ka6kKvFj!&QFZ7Zw%2@q|6`w2!M=E8}O2sLd__`2i zWWM4#%Fl)5ZC^NGW(Mk?Wx`ia%V6KKwIH|oEB5xCJy@0$Og`1`gDjc@cyD(KURmCR zg&J{jA*E(~XnH(&?j?+R$x`^ns{u)UV&RgVB)Yy?0uHV>f_#}a2-||;r|@v{m|{1i zQ-cs+& z-Wr69<{N?!pVg4Zm1vg!?pX3=N+cYv>H#g$8Q9OjZ(y?B2h`6ILFEtE0QC+*D3hEG zCNhShmA*RQzmS2`;m@GK%M*ONA_KnfT?M{q7l9MWm!V4b0{A7h5TN8TxbHa*9RpXv zV%l7+Jvfeas*!7$qGt#r-8e}&q4GH0Q9vcVd1o|d1&fAGH=UgC|XB{ zd6T1??yKI>zvgkr*ELW;xuTmBlh#Donl1E=BpDpX1AKCF5}cotV=W zFFedR9rwTtsQ{1p__G~WRPaqxe266r!lxd>W3gv=N9yL06`7S7*sBYkn>Bzv+YiFq z-!bU2dkcK!*M{#O@Fr1?4Dt5JAPj0PhdTqZsT}kK1R*w9o8pE>UOGX-1%mY#i!dwq zgV^Af=VVvbEil!i4C(`Wz^+%~JR*M##HsljamZ1O|kE|@r^PkP2q@5AS+U~O@a0hG#uhy0XSix*wOq~@vd zmu0_1`5rv{*c}OC)36!tv|$y#Lf3>Jsa8#hSG=P(3mXz)GX5bC{J1&6E?fa4K9%*0e+(4s;ZZe)yq=sXXaTyrqH8AsrGRf;T*Qw3KPOrXWj z`J|a;Ib&ra9b^ly0iR=JQ1Q|>hQ6Q*-Vta8<_E=qgd6JcN#-%qNca*sowWutZ!-gr zbGJjM3lBh?yAoIyc^npCLimS8k9cjTcfpjyt+2gtH6~MsQ)f?x5NDpR=O1q+@tjdn zB5(CRqNZDn>WEtb!{XbhNjU+mdesfkd8YwW4732v{^2BLCIgbK1F-pm%D9YiKD@Qe z5X=$$hVT7+k1~3h3eLI=;2SrV;&$f}z=>)VF3+Le*E7a$3s=appT=JAWm% z;Oh(0AB=;#m?@~CzYsC~H9_0AP-xvH1b2}tctfrod>)udPRx(OB*g1Tq@jz}Dnwzy zCo9PaZ8`}5Y9~?a%#%mOA#pe1?-`D6_f@sV{tyhdNH;pa@wQi|A_0s0`!l1nssGA-C@g^-bA?8ql|(+~pP%6BY}ATGv~!-YXuOJ6(i^J-Mv5 z*>hm@x9f1mgM3humJ4EDDWUiuIOx~U04wVSf!O{8EDlCu3HL8zxp~W>mPsQBcwG;Z zQl5dAx8}h8KNlnGhv$j=fw!q0e(qFiiWEO`(_G@JNeKUp>ku(hvXoz3EyXv^Si*~0 z$45&Gqe0fivxMW0_4sMK6Nqi)LD;V=r<%vN!Sw}ll<9*@Flfa>kaVyDtE;ETXGgRk zM=lf-a=8KQ3%gi?x9*Zlz1p$6qB4k22m2`hzpRKYDCSTkY-l7G14#iAQ2O`+$=Azz-5FiXI;9UQ=Gy~+UX0R{N< zHLUXK_%C-;IdaN)8n4?y82{8o*1HS)K@y5&x4Bw*K@IzdpzdJucd2{Tn7pgsEuxTLfLXv&<0Cyhr**0jItl^H_B z_i{G=rtvT}xiTN6*+t{8?+Fsdp{L-y-SIe~(uaT2^gspEJie>+ImhPM2!#2U+JWvW z2XI2=H2G=!HL$&LIa81$4VE8NhB8KnVfHyqU^E@u!L%EOvpkkFw?8Wc%fp=D>XmJv zU|SA(t1KCNvM30UY+GQaH-pl>=1jzo%pqjuq)}eC9A%zbiFF@biS|9#p*{t+U|@3_ zT)XWjBYDRzEcN7ltnorS=)3F(l;#wIt#{{R`Jb)erPo8`hHWq5u`g?hu@#d1JLU#d zoPH3cxb7|WY*PY}m|H}|&3I1o7`ONw`c|iP|5$T<530<=*$lc8kz#92`@R8v;GVl3aGIv-6$~<@l?n5agL*|f| znrDz_Ei<6UYB7*+z`|ZUbz$8}E5h#1UyF+`dV?-YcEJ3D=P6|uRbm9E6V3*rgvIVuF7kC;>*_y(lD|?`ab^y#vU5r((T0y=! znFi9*pOMj)w$pn$UsoMoqw?WL#b>R4lL`q6I8XrxN$0b8n5sxgIaGkbpc;sLQikz%NK^gau zQ$f|m>12pY8I%$%gR0kef$OTW z(Dk$)7|z+i>B>?;t$EAIn_Lxq?6E0GT5EvI|47F)ua9E)Mjt};#0Yp~c{U8pS3(V< zfNyR(U;V(cr+uM5uh0ORMk-|QL?fKo^c{v&%p_?gk9c{UVlpZs z1)~Nfrg>;*^7xK6_)T;f8f&WLb-d8R&0pSuZ;$7JkJ+2xQl|p=V=4se9*{)s1&{E+ zoi|{~sa`@&MCm5jD#+JfxYhA)-X1Fd;0|iz`C+Q?h$P>f(xO(}IZN0YtVMjgImEHS zczmqGiW>YX!*}+uz#lIWC;UDX!_S6*|M|`uV!2xsrSqVIU-Z_L?-k2&^l|kj?rX~8 zC8xx0Ug@#tC(#aoQ0;>KITH9tWcOMD>%K!F-f{tw6KWI zAeip`5GIN_0a?)jRuowe9ctzACTa)ko3jpj#S{Sz2fE3k=`J}{?HKuUpq%t=90WD= zEFSgI4x}^0aD$jO?49Ne{F3S*zNN{U01MlQikTkh`8tx)-mOQdjV?rA*9lXlVwGsK zRT;HqpAKx9j_KMBY+?M647%6v0$g8q!3ef4I(Htzyx-!LX(JutcOv+{{q);$uc*W?UlS(W=8K@RCO6cvVj+ULg5YNR7@D2d&J!7V$P@Wy zhMOCFfqIOUEQxnqtnJAv9?MeY7Y>5^4;iF& zz*B7PiV1A}lXDwLIqnKSSUT>)m@yaapte}IK_BUA|^u;GnEfO~8vDsgm$;-myD z-&w?=Rm=k!VS@NFO>@?eM>_VD;Q`loZGu;xg!7(y^GLgs%c0?oMED`Bl;pgGxR~cX z6x9@o($gD3k%=OuqjelFuKZ3Wn~PI7VrLSBa|>=Fcbb?uYEOh6yaGHzjCti|I@6pH zM*SdjV5(mp67!aZn?eOC+gMvv`#=cP`5Xr{xd6CQ^&Gi~ejn-?euc-!pMbp2`_X=n z6{JHq18r(vfHDIeP%b|M>X6Fln#5NiKkqiS|DqG(j3$EfK4;KF$!y9qE(G5$GzVDt zj#EP}RYa89U7S3?$NMi?@%8Io!0Lt=w2TA5<*)=$bw?U)ZNSOJzDLL+)!FFIM=2nPLbRe0iR7SU9)3dr9GrqcWOQSC9asqi`q7m!~GY!at^ zDt%wDw4f{WJy|e4e>np+jtRr_F10}U%qh6K`~)_%Cl1UzJ)i6{GDVc*Eo{57JQ9nd z!3GB_bYa?0UuV@EvVh6qp+g6Gt$r%-Zip68iVKBKX*S^Q%*8MX7=q&q6!3y|QLuPu zJJ9zC0L`%n;Hl|;a>ZADP%IroE?L1Mx5*rbYUhgJ#y~A3Iqx3k2ve}~C?j}wqdd^E zl*F&)t${K72+-RQL8jL3h5pX($b<`Va9x%pxYHPpF;E{bM>h=jcQ}d6Y;<8Y_cEq= zu@zV@@vMn-S_o@ZU_qq zyidet*bal}1O$4HD}dS&1u!`G6ZD5hWcRERU{k7s?uiTI#I72oo_7wPIuwArZf6k( zSmu<^F$^x9-H6|MCqc|G83U%?LX`Ybc~G?IBH$H#gu*8!fZxkg7@w|+ejfP@^$Z?> zc7G+HzdQ#RKQjRF_Z9Ic>)N@)o2{@dMygO$&2`)r}?{BHQJ)da1&zZ}MWDuwA5 zx1q{c0kCo63C}-X8Le1RK<1`T^YpK$*TF&*YtwBl(CaHnj_;I)QUk*HrRZo-R-}NA zrz+usKg7_L9b*_Jb_?6@v+E$P|p`ah(@4*H6c|d&ZZ+*^26DAG%E@-Iqq6HfW=+H!5Jm+Z2pv z84I%mz3}!rb9i=_2wD)m3T;rqFw*TliN60fPAQ5Ge^h!#C|T z;ZNPUk9)N@psi)+pb1-sZ@(xHy~rjA@6%V{B&@~PwLgdLIyfa&CP6%_D!@BNxtQ?t zM$)vmlK10d!8C^=jcIQY!kW5P1CyM$uqRUkVu2sAkI$W`TPe<`QS=6mAs;t@8IXi{}h``^qmEVXNJCvlCe$IF5#;~%3< z+d~Hsv`7*=_%n!Cq+Ab`l(Q&nr|l?MPY*X!S%V)mJ&SL0i6oO(J5uvG@x;wc7bLN_ z8%K&yG0&Mzn0C@^WTFOPBDoR{dz6!X;w+ zWH{xL=ti+VCjybHnNa(XCQ>S_K+#Gip#7={&Nt8nN15L+r~9+;OWC>1pp9?fTiw~v zDLRW38=mF`W{jf@@a@E2Zl@OKTVWUE{jGR@tD#$sQB88Y`lK)*G9PR7`As4SWoRlP+htIF}G zFGr*pNyiAYH?h>I1SS5^9BclzfhD~8t7s^>hKKEG{|J&39zYid4RCYw6>@Zc2zssd z6ifTM93FDt3(6lYhO)ah03*^4sPVMXa>@rrg(d@*XCB$QLk;d*cbfhvaEOdK@*U`y zbHHBKP3+)>^<<;4B2behr~Ts}^D2&cz-oy|a?3IeBz)s3k)TF%lr96m4@(G_(OkHxa~txhDkRc0VyLEVgUBZ2 zE*0&ng;w-V``@g}1~Y_YK}2H|s887nO~kr*9~<4t_NKYOxc?qj-pC*>-m2#ZkJ~uz zU0X>U>=ob_PwwWsU3Dg!a?eo)&ZWqzN}o!3)d1h#Pmz0bw#kH-x~~EizfRsACce8ipZT8kL~-|0jV^?V}KM8M{XY^YXFT!V*Y9w3vJ_!2%q}Yq%IXx&#x}LvH-cOR{*>t5T|DLkKdP zY@bYi^28TaUfUS1#x{HaHk$$>TT~9B87bo*~`<>-jH(mJntP@?3aw4 z@c#h49H+UrumHHuVIS#cB?M9yXTho~b3nLu5a4LnfTo46(0$)5@cHd}kSZMkwF4z^ zVd)o8QnLi;GjNv2%!z5b`y%yvvoG4*v6a~JD+R23w+GkS--XO1{3!1QLPW-{J@}E5 zN^s~_0ob)S5~qgupvaMHXw$|cKs9nXzQ-p9?>Oa&Qq&^R`t3hi7q0pOV|gpQ^BWtR z`vn6BD@=y)2!& zFn*TuzqpcGY3hj7^dfP$@r|gjT>CBFYsMrp? z39bPBg*D*Kp%QSm=`lQTRGHkHn*}v9c<|!Y{Xk&;2GYDD4clzAly_=P9&o8zgF%II zkU72-?zxl#HqOI=wPPw+!uo|hc8+5$8F~)a?XtmA)iiK4Rt@j$R6?&_tHY;G!m!#y z3Kz7Tjxgw`2R*V5Ao{r)+;~A4%u>&S-k0lupZgkceWU`)phoCquz*mXlptCLa5SUy z7BA&uF8al&qsBWd(SzXo=)mDj^fI&$y`H6rU*8I0rR!m!Qe^_(N!y|~VhI@k{2a*m zc@mn7zrb8$8ljlG8e+*cL!U$OXf^#aSftK|J=ccua~CpkxdbKnty2yLJim|N)F_6$ zw@~wk2JwV1DDo~nPEa(H0^9xW2b*LlrA#{z$0*En~8 zw}J%nk(35UukXaRp5sDWs~wnPq!ji}uY-K}l7qGTU5ApyE+FjqkhQ*kF`P&(1v`uH zVJ<^1VDx1!SQ@K`lz{>?DqD@c91*~z2eyGEIa_!+wT}EgSk8hk?n3*{XK?=ziSJ>m zp;-gSYKRbgWI82s1TH7y}_o;-Jy7{1~%uLIWipBfsF)8;Wir{v$neoL8pmnKbKcaV4;~5 z`NXM<^;6*-=%1y4w>nz_)vsqk-hDS{<#Z2xoR9}%>dRob%MtRzt$xgYB$em6rjtBf zsY!m-A;^r>r@(PuArQ*1L(a=oscX7%_~9f?Ds{2~9CHn1sWJ}2iV_Oz@QK3! z&$|>b&3+|#ATu6*%v*uARXb53!wRS#Ylgq3&qQcD^?_#x z=!VZsfcdtO0qxa*zi2fk&^?!H{<#LTT6U4eWhX-M*IiJ4*CF`V*@nOK+?z`{z{Mh* zVZLuQy5u(FYn zFnY{qnF@(|nNb@%W~bcVY`xB}%(UQVwekOn&VPsO9R`~*Ie5TfK;{wSNdE$k*v8&k z-|Kvi3w;Ztq;v_WNN1A<}vFX}lTF*)ej{i$J?o{|o#utwa2KLO2(d{Ji z+xmVt-9Ig7t7A_eG>&5%md|7#txBO=c^~7fdcdL`@xs}ggO+oO(oeEK^2C{cI^Dm! zzKyjO%+SjdwU$!KT+M|^HtGW#xYkDbjKSh)=Di3(ws^V-cU{nuKW^uL8ka7A&wt%d z^v(0I=3lzIztR+sco)jLJa{@YUiFiPb5@SbiwVR3_uS||mHvgCXAI+$XW8?t#o3{6 zme!VC(4xf^(j7i{*fEb@y+M0hHpa1dpz^o3k3WUSf0~|AAw7Cx_)gl&*Ebmpe#Wtr z!WYrWEDpy*ae%EaDL~g>ytmf)42%AEFL!(LJVxojYW88)rRKBllA0UBdWCnkKNRD$l#gvJwnknF%JBfGJ{-Sh<=y0%YdzNQTu_0zqb)&F?^qC_lU z1`nU+jC6-GEB#s=hAr=L7Htsb*`!1;gEPXJ9I*`g-Io{t_T!ew(GB#@Iy&e5vgw1QIykehY+mw@{r(rtD9l@fdCq@gI#5i8!wjtAHh!0` zwdiPK#63C75Mp3VE5CD0;{@K{zP&9}~r=n;}yxf>CC41Q^L3?Q@ zznuDKeEYjM$p4}Fi(wgR{gw4jpk?ycZ?m(s`VyzrYMSS68!T)5WLR_fj*P`+d8NPJ z{mxSoox_-4mS8*nD8NA^GmQOy>-`oF`E#T;!d&gm{yvBH}WKCbO zXcjy8#|7Got2Jz!xe1(}q&SB2SRfr)dNA6i#Q*m8`#YHYC8N1FpK*0pFFWMI3Fh8K z^_;Trm)XaYwsh#M#KtI-N#r!evEAx6+1n=)&j(_s^pN&A)=GQcB&FwW4x#=8~Vn82x zBWZ&>Rx@5$^wGcdEu&peee=h8hW~6-|NBeVvNDu5zF?hwx%V=*LgY!dhr@(zXrDDB z__L|~-0YK_^)8Z(xM;cmn0mXbGss|lZ>DFp&ZVhaJf@!uPNO$0aO5~Pucbv!U1y9r zlrhG=v;X)w_;pDGb0X>s=lFa>uHr&dhLIbGCM)rfqh@Kv+&tFLxZ4oJ>6rQNTQ9IT zyV*WpznZQjR8DJa{KD4RQs;2kp^*J9afR)jr;-eFqaJ#<@7R9~fB)ZaS^uQ)7XnU& zc5}X)N)%`BbTHEh-ad2f)5XR^TZ42yKeslO-PL1$Zy;GG)g{&7PotHzN1u_1MRTe> z3>jsM-Pj>&Z#hLQmuXgMzhu%8) z#bNSwltbmBN?Mj~JR@%?y7qC62KRQoCgWARgss=LIUJb)Z}yZ|2kj7p$+#O*N|(&p zRx5qIkttsAf~(Jc>@e`ig&w}yx8~bhXj9-gO4Ik)V{bCr=U}>x%}{uKo&Msb#BcYf zCH=Q)ul6_Dxk?If7asRu>@zq}d8FTrraf#}tI(KcZ=zB}-yKi<+v@EK@sYM{UOZd< z#2LE5msd3f;i}wVZEFU*B%X0-{!-3;nG(j-x|G^K-Jbu^_rFKLTr^YduQ#WJHvh|K zYps3wjNZ_&8E)Lm>M<*l4MdimQ8>1p@!R~t|2~y}{{2ru;KpqMnorIMy`CjVUt9Ca zzW0e1dqO~wwvvV%gkv|+Zxc)DtH%8Q_Wbikol@q~4jtM|PIK)nmKfb=+amG*9r4@!?zlL`c=ih}t6g-Ma5JPZ=+$%1{71SwScJ54qV6!WoyW+ggPfW>1Ukj+%tm7)|PO4vk!*y$RXr&>11j`D&#_8`|-S zd+p?P#?~_&ns1v0^T(4U#<%vzbcG)s%9K>IJu+15Lh&z60W>Cp+9;i>l-o2o|Cwb z^X^eb?a`n&j2E6F>^X)6b8gYkT8*ifjIx{C|2UsmuuXv3T|ARL@o~KNkmq>^@UW1x zuS1?bdC-LUyzY=gUc)3O_T8mF-tT&R9-(C+P44QVO2(D3gY1mS;_Tgf$ zDF?o;2L14@b?hDFV$S)S3u)q4uCUYcJ!!5A1&qQOzi64;s{Z!%_Pe;=UR!Pb_)bMl z&JTy`+{P*Md-wC~F8Fb*#A6@Wjzg}4ufrnCjSAJ)zpL*bxqlG&zY&2MeGT;6ZUt;z zl@YddYg;WnzK3m+wv=;m-F&v+ULR&b(;@cFH#L8J-1^g4?~!=GteB#6^VXeXvV-lp z_KlU??g0fJbM*xC^4C`8FB3uTjXb;Go+tk4{QaZvec$32*xN1Pa~*#*DN*L9R<(d%o|=xOYH8vBcJExo_f-lBOfP5xpc z?PCv{wl(o1W5<2VKi*Eg9*WX#?$l%_)ZKMxYc!_)Ji3#y<%G7wsH2C?cJ+e{d5#2q zG4ojyh#(k!#U>vGh5OB zn(H!m1@*uF9XQZwV%z2+Y;)dG<)WF|%&s@x8Y}w`8wyWfe3hy(SN?zf@}GbHI}nJ^ z5@yV@aH9z(-{Bl}SaJwSJ_Udq`MxcIm2{J$fQzoRpKxc^^! zP3egbj5mMXF7I4#t+_M7Q+;4rg8t~?y|W%f%~4rnru(<&rGBTZ|H%A*BY+8SW;cCx z=PW+0T{G)Uxb;-8FMS8OmBv3ZO27Lnw)#nF20iSHV^wcbr`%s!a#|z)wNCavy>pwC5d^JLcH^??oaNf%ReOhPtyK3W(GUAM)Hd!Buhg@sVyp{z zVtsV$n}6H-GNQ%fUw)Q1=6s)Jt|=#cH}9N=TP9yKIpCzuE+=f(%H^ht6^{8Tf0gH- z=RX$#T8g-Rimf0+M{*I{K19a$OM54KUw{;yI&R0FDgMsEukfiu;d=77dH(-gr2Z+{ z{{#dSdap1^9~*XOi8MRmiyoJ6{F2Tx*RB0=(}*d&tC#lkaVlqk8TZH63;z?a`IC7J ziIZgbmVc>DioRJTR+Mk2E0;kl-Cb#SIy|Lzt#dMsnOt8z_r=%W9^XE6m^1xW$#6Y6 zO`NgPI__Ci!(MZ%lwG^HjKK-M${b0{qX#tp`{F87-9j0wq8w?5+i!6M#w8fBjeYch z2}{Pk;wJl)=r)JgO_iL&ls_MrKd9bYqiW}08|ZR|cFW4op=OOR_mQh|ja1Dqhwp($ z>0iHF(eoK?fBSiMbeHWV@x-i7}0ej{)S z)(ZHFu?w%g;J631GeQ!-)#3tKH38LE>5Ek#(O)h8$dEPpeSfLn#br@a7&D{5l%r3c zW@L{W(_=1XGgc=!&>oMRW_Q_ZG5VZa7)4FLe{Rq3ru(Pm^c{s*%Osc5C*LG-eTMBB zWqf`5k?W;wMfI=D?j#?2`RjeE;%Km*d~1@7NyEf9>lZ5=}9PrU?l@t5#KY z9^0Vw)TK`QnSP^wOaDAMn-#(T@GAeV3i>DA|0@wdHIM$qcdD+C|CfEL@Ag%?9=}pn z?M0>fCxaup{Yy8CCzja$?d|e_6HvOd?v!GaO|R_BOAi#^%-yMYO5R6l|7eNAOMwFU zBOfV+1XaFLreBNQhV~}AA+>57ld{=!M+s;LSLj&pH`B3ib#}13!+|z^ZXBDcBLcFC z_;qQ``dQL6rkIS9rGxYioo?yMnX=MLtrtose-W4VfLEjwq=?_vvjMqjTDo5EA@I+iR3ABSIL-PvJ%#JIg-4mVp1!=Es|Ot zvp|a7tD(3$JV`#teYxD3%Z`et56dYQ*XAi|47)0i*K~G1+>Yj$bp0>K0LxkGw4Pn~*XNai16#J+h6}L`nR5DF-k6M(L zWR$q->(tk(hH}oz>ND~cE3{2jCy6XY^MWPHw&Al>@20F&`Z6A>5=#4EZ+_6;-e+Zj z!~etDcLqh#bm0;<%}$=pyTFojW@i=zQBV;@f(aE7#6(arU;ragBuNGZ6p$z)prR-U zND@RrMNl!J2xg2Z2Gr~OReiVazwiBVr)ukTPfyiUZTC5S&U2ogLu56{HJf6|jkM$P zVUK(gb9SV@mKVx>x+|$2hl_yg>z*(Ma+yr~YIP>Cr;c%an>Q1i!eNH^Com!+)R{le zoM+tNwovn}0;s~a8glp48uAG8hFa^2Q8RLjsEpwx>b^!N**QEw=5!QkR%e9kl*XUZ z5AW^OKR)Zc7TKAuzfx$c*|~n1{?iS&^d!=^IvAq>De6~5W-rNsJ*ZKbQF{nFIa&Ze z$ln8hH$4VDyAcCdFJ2G3>=ollM!T^n-UIN)r!1`HsQ^n9>Ea_ZT+pfjbEN*|F7&E2 z0l(N)MPw?x$EiJO=n|Ltglohj+`;Y|QU61VgYll&f_)?&A8EvKNuQ93`oEGV5k=Cs zUzhZHP*1H{Zz&5c?T~HV??^2*%p?2FTBs(AwVcm`k=&)1Jvi_%$q_SjxyZr<&KLfC z?#sz3+-tf?9QaZ^=XLiVLi4f~x@pZ#;`1n*lgaYHAEh)B29jSG>(wEAky;s1ncj*2 zTVpwq69Q2|bD>RXArM`y0i4*_4@NnyW<{Oj0b}d5K_&5OFsphA(7AX%$MD-pZo(pW z;y7ysp%HAv_1e@;bluC~`0JhH42;wfrMH?macB_t)D$KB_ug;ZocdL~4~!xF)3IpY zwIn0X(V3QH_IOFz?ck9O+GTPJGo<3n=hKg zxtQ8lxfR@zavCb$KMy>%E(;8LoxmQMae*ncZe-W|G5`T5Td-i6COEokd|q*s=7Yvk?QX8P#e-Wlj^#YYV*!KQv1^Bpu(@uQ8RLKQQ4uQE}3KgP)%UY zQE9MLR&}?xRHd&>RLSh9RW>4*Jw22&pX0vj$(hM>>W;KYe0NMez|KC)Q^Ud%xt zCsY^OxfX^i(zE59cX6b7D?}-Kj>=NJ6Unuwipk9Pe)6CcP0B0aI_3PTgc|>N3#FM^ zqP7nk;X>O<^xpC%=vCVRG=26OID8};$&igBd)Ljz#=3JchPyEq1v^06Jr-t{)}1r4#Mf>>gd{6aN$eL`iw)8LB}i;!Z$CB*wY z@E`2{pa1-SUFGW_Y=6cf)Y-xU;wq|PCh>9TlQTEbjNDAvQ^gTUUMxcg4VPjuI|>+w zkfp5PWBQDY?|JlZgd6L|!$F{LjgXONr_U0Cjr6Yd>sbHmeg7Q?mE~0Qls)7(qa>Y>d*SyD6YK>uvq00km(dS-nrQFl9(erA zbdch&N2qosbYmEd-sZSrCNgiV=d=K|!G+lBjBIqe!B_b3U^Tj}MS`wHZXv-cH&I(? zok(RpU^+P7q#B%aTX=HYUM=I@7bHGbbJf=Vy&s$bORmnDtA&0K<6OUZgV?jclB4$_ zkaMkMnD|X>;QW4Zn)CXRDu;hbpW}TdgZMqvAwS|`LhVtxA`jWXrj&zBgmYS_8MfV zVl4aT(~a!A;Xm2oZ#TljuZExnQ!cW4NZCQodbJ9Fw zE_GV{EIIG(3$n-JAZa`Vk_Ma4%YHx6B-OSKlR@NVIMC}7bRnuA+9|n$KKd~YEt#%? zoH;Ydjy&Uy+{hb8f|es_^Y>3!495fOOCJVfgXNg^>q4|`as*rD5{nFTa^bcqQuMB~ zG5*WxIT7A%%N4ia9OmO|MD@m8jzY^PBKIlCt=V7CS#_ruQ>Bmz#39f6naYD*mlPz5U5EZl4<;${ZetpV4lB)z=*$J-ru@LiS-O z^p6ji6hdda=pKO%=xkyCcU}K4xh*u1)@<3ptj^p`%aljca+OZgvhXHa^71RR7*;Ax z@-2z>&ojND0-{O28Q*QX6Yjp2h3n<3;p2z23C%7q?3#@eD*qFO5#Qs9(KD|w?~;5p z$~glT?;pWBdyBC5feZ0_A~Ckhoq;Y11u@H}R=nvt1Gz;NK|!7ip=3!ja-^dUiFxxA z+1|=UI5$mUU>HC&Kf0swx(wp6r#5col#940cjB$mb$Gwfia3VXVZMSM^v=&3tSb5# zp%}dZ=a0l;;c*xYB^u!!O4l$ok9>U1DiM|+2BMEvXk#}m5F&6q72Kwi2X4Fjl{Gv# z2Qu=~f%aqXz^|_t0DrHf%XDMq|QP{;@n1t>cwJ)U(r^E!B7-~UTewtTf~L2 z$nk=eMO3cYGo8)UVUZTKX>P9N(SXO~ulO8u@3AFh^FR#QS6VImxbqJL2Is)ylfJP2 z-dtpF?=RSJlQR^pbPe1zrvo-Zvf#h-fb9C-0)(|5pp=@mVj=bL z!wOt)=}l}-Wf6SpLqAq{avm1Adl^0$=!1@zdLUyNmr)bPdfczz6`JpJ1NquK3?9uq zjq;Xyp<@;l=6mxJve;<=KI1)%{Q6jcReNT!`WUNV!PJ;nv zF5r%{wygMQ9d@FV2H0sF%s%C9&Ypdq4!fD`hMu@Kv4)A_1=sLU!Ji*L_#Yzw1H8Yf9&i7KduHO_H%F2TAyl2p(CAP4M%L>^01BKLOJir9N zF?8g77o1Vok1emv!shn5;Rdt6qN*1=k)FSM;+#e2aX-&%uf09e(pi}&4Y^8NA8+Y zkx4)0yJ}RZ^S@Fl$4l4c8_OP2^KSSOzNU|G++YO~p4g1Py=8{0dASo?X8K~+s`jI8 z8!NEXvsFaK+*PXD50|KlewnMnQ}(EfMv_!tvk=w#L+uj98;0t8dM~Qof1)6@uYJk4 zdFjpFJXJwJG`I+sWohz(9a9C<^xgOetK#|NR~q<^YW{rR1*f191_WjCN}%$XK=ire zUBo(e4xD3po6Q_Yksk5@vTj)uf+%?+yH2y=!#y%EyHyvhIC2`QE9FnjXembb=s44nl&k3xh zXbe%Fe*yDqi$NDu=%V`u;?dAIt?=xe{qPiK6cni038g!xL9$)x;CjJA&^g5%n!&%! z?)W~!$_vsKem?R`cir&s4g=@tUSaLdRbi*zx`GwobkF*R67sgRoO8) zQ$79o9Nu}JCmO>2wK9X}>3NJ7d_cxk?CaxAZf)RwZkj-eVG@1o?1AiZO+dQ>`{A3j zuOjDW_QBE2MA-P88B$bs7|GttMq^L(BRegwK-+xQqxYs?K|;OD5n7Nd#1GX&-i9Y2 z?IYg*8V5PmatUiu^2Op-?uPQx6|lK++t6#WW7tG^ArckJhKhbwAlo((`0?cl_~@HZ z%=KI)ym8nEKR=I)FK0c(&B|6{&XzOLCG$35-N3YeeeUV0!xkX!%~9q^ZaDzu@z@YH z!K~b;2B_Zt%@RIWXDwT6#je_YfMxP#K9+w!6>0pDjx4`B8?7byC^lm;%IKPc43A`? z*UfIBkGd8k%=gEkEq(;sE+-RsZ<@v!c_0P{3w{HLiwzLjlg<9%k^$(IvcR?Senh_{ z8Mw9I5M5TZ8jJh61$yUMh29omMOMVCXFBPrr`^0@n%8X%u$GaU;g>OF&Hd1E@m8di2jiDH_lof_l8DMcR*_ zfDhsTW)>o2_&4nVj2!G3-`CD$m>R8OFrO^{Pka7GGe)4YoYDVzJ%e`Y4Y_lHA{F!J zFzHKWT78O-l(WnCkoXdWI(lmZb>ff~HFV*iT>KmnRBG(z22@-SJigi@ILbEW?~BV4 zuxWGn4@3)jTrW&;ho&gdczc$6xpF??wtpV4qe+4PB+Q#5_7n3N1$Nwdr(bf6l@xd< zv@vcJT8xL%&toO7%i-N~*5I){Fn*6UhF5|sv7YnV=;d9$m~xaWkv5wSI*oWk{E1B9 z1lA5#H}40C|OXe`uGU7npr}pHJWE|CQuQ?rQmvZ-wQj$QOpgYTt{b}WK>bxD zFyeqHcYrOdTO6R)xYAV(+Rha^tRd8VwjUF{ymnR+b>oA|$r^LXxy4N=GfNFr^{)ac zlWFKx2YaOYXc0WUo{9Ef#L%ennOLoo06imnhP5+}U{@zUqyEtr*aD5isBzFMbltW2 zNY5TAs?=_QopWtMaobk%;Is_#Gf_ewT3l*Xd}F3uGQUw4wl|f!S$~$i9de6QXzQdV z+yun^r~rIkv;=(-^9qN~gyL}}HblMmYivV_F^010vBI`8Vx#JOINWPHxcBf`u!Jc@ zwWbCj`Vs}$YTGI1Vs<7>Un4?DX&6!$)QGLhwZnY622sOk2Q19g7yY#n#O|-IMp)C- z&^y~vOvZ0Rjq0!AI+fS3mOeUiGCCiR`ZyPVgop{#mSjwlcntNfD#V_7L=sJbio&IT z*rJNEV1d)?Y|fWRxu`SuHUGReCh{qY5$w-$5F|2g3QP_=z`KQXxN_$SFuvOyp1Y(U zI@fp=I{4@acv+P)D#v98|vm^eMjG*8Xf~+@C7-hV^fd1Ehxf7p)_=g;z{^x zxe>Z;KaSV>&Lp0cX5zN%6|oyXZsJzR0^HTWnUD;g!l3D&(DN!yn2cUdG?VJYHJLY!Zp zhhz6>u#;0Yk^G;Bk*fZOh|70>w4R5e%Qr0p-^{c?zJEx9MN`yJ6Eq7lD-Zw<7DLeH zm&?%hS6LAATO-urS_i+p)Q|3+N}>l76_AB)#_avkamYUXhsZVla&+hTJ@od*N9d9D zDoC^7EdrY(=)4uvU?2g4>^{zd3qE-xCKirJ>$SaVDI*&sbTU zy`8F7|3_EydVaQQjC!7G_v<9W<^2sjB3}vBXrhQW$z8a;@hJjbrh_wAFUAb+3-P;6 zJkGu;u9}o{lRi0qSl{^Nn4slAvfj*OJAKvkAOp?xDf({2W$Ejm9omBNg^V657uXUP z!kD5J#kf0n8-xEanbGigKZD&k=Rfz3{2i-4be4MAamZZ#Vh?pnVIiqm>Ph z-zYWd3|P55@sbamWl;&oqsSdPTge|GIplY5OxE$xmTKMRM-G@hCO7SWM5<{ylXb2q zC`Z?DrVW0W{adaD9xP6vjn#TUbH9FJ8Q!Ne{*jSSdGz%wU(%{IX@rX}7hfN89L7&y z#($po!t1}76EQ}qSWRUyiW$^kOuZ~3_T(n`R;M4Fb$T20y(Sg@GA{@ERpQJln`;j_ zx$l8N4Hg`6sRmw_d;;R zkhEw6J6;*F(^Rjs5X=gg?mnH}|0x$R(jQ|$k+HOo8zTTWv7>zJ;chaVQ7m7dt0Q0a zbrW^-Z|wn(oF#Ib_iriSLZjT%u6cO|yzy&@96d^@g}N#K7vn((Quu2_NA zVf4_9l^DHHj{xJ3vhPRcLK$vpkOt_A)Ha`jtHu3Lp!Epr!D$AZ&^!Y^ai$4Yl+J`N zrRri8tU=gv`%EO7zZA`O;iCh?I}mH;7&1QyLbi!FBcq>!;gY@+Of(P&inc1FV=haO zO_mVceq0|E*u=pTp=!uaFI@z_J_TD@bseTIZhW%0D zV$6 zW#}t%=ewymjLwp@%sYTQUvC0!UULCju|Wq@ymJuE(KJUS7YHyGI*E9{n~REgd(oW5 zDrnD~P3ZnvLC9IoWu$J24PtcaBy=!{LeJVcB0thp;I3G8)P3?DEP8Sa(rb@}pL7%G zXzDtIv$`BUIxqr-_?E$^rf?DOX=%tc_Y07$=OFx~4?yfQmto5CvuHu)ETnUp4YW0f zLLV;Qg9a5n{MUZs+ub-^FX>^8k#|`+hYJziYzA_-D;aXt+Qk0w!+H1mq3*1y$jdq!Y<1)o{DFfa3!KU{f_wCIYPK6`s2EPsSd#ik z+bJ|!eNk4hhG=NH*5ye>%@nj;V`j!VWL}~nykx?L9sQ>kb+uMS^(rUfB~F{!_gT&G z=e|xvZQwGJU;7YZDlLSg1JA>=^7vS;djh%=GDI$@goAMYFtP!4KqE%e&|@Z+vYDmt z$e>MMsZ72r6~2BRrLriMlza5a?25mUuMfMCgH;qc_9Ou3Z90IR%iRe1&Jf^-j(1=` znu~DU56A9(tU~tBfw87afT%eWh;!oiV3!XR!zme4@vY^LF@9SK?zv6}L(*K)bt)v5 zyc{KJmh;4?uZBym4c}ME?7YDlGS*Z3Ip?3TDStzWx@FE6z4pCmrSEW;8} zX?!P&G<9NILW8iz*UNF$%fHd7<4&l0SS5C{@Bp5xpMaG+hoN<{P{hhL8+Q6ZVE*^w zkx+kQq;NF?Kl_yiw@`nJ-f?E(f7Tr4oLa@iR8BQwpLFX84cj_Y(d7D^!Q{Wlha8mgC8Y}nMEgV zjauM&+aqCb@CdHZV1}#mbck7p{IK{;8}!EC@9dawI!J8)fijlT4zee&?`7Nr`WT09 zqAa8Gz0B34D6>0th*=@;V8{$bERV^DXO9gkA9+0JC{A%RRfI7sS5 zvB?{EUdgoQ`^#q2bSRXsDBoFehx(ywPP@^fM_Uc~GXIgsrkK)_vrTB`^DSxnFPhOz zLv3kiOKfNXiBis2yCaydnKI{we=>))tpKm2k~p(jF+{4i88N!ogkzvsgHJuMT7Kso zNsZ_zSy#*2sq`C1sG;Mr@(o`g`RF!FIl)$>eh=KFRPJvezDIcDPPx&@)7ka}Blr{k zfm=^3uaM(;qa^lSW{Wo%sdH{;l8C;*7`{GZ8XOaq3pLS7q4_WL;Rusk;Jx_<(E1n0 zpk5mtNYYF*oGg#lzo#xRE)dC#-&MTQRoq=>pk}70vq5{7;g8q8#$Q(4)^~C$FxNGE zC+kpqLTaqerS5GgkZrkdN!`7PSl~@dso+i{l25^ujph+PNV zZHue90>qcU>*aBk8y-CL43GWHB_`>F@M1(Qk2C&MICtksp|(3iFw=ZYcq;yq;GS2n zAY^!jFsETs5Q$Dj?7MB@!0@?H-0u~T!8R?ZCczP!=MO;1_HV$lXLBHFp%3VEDGF+c z&1MO*Qy}XfKVT;`3bw^$(6*fi!Mpc&f^FNSvWj(-5q$`Hy55JUjk>n?1$D~w}huM`G~l@ z75lU#0PA2E!_?k$XmrVa#7R2>^DXj(cN8YURe}iQ&zmMpJ|0b;*G{GGF7}W~%WPyx zCp@Xld~d6?*C)u-Z4hO->=F6M$%R_9>^Noeov?U#HA=2pv0J`3|Gms_S2*=?@gLGz z(T%*+&a!^8EL~n1+-5obi8%%AR3=UJw_8@QGG&|je<8)K3FMB*c+$;9DwAx6$tAt- z!R1SfkT0}auyg8M=*Z)Kb|<$GGE`R*%=n{;Qw=OS(2xaKfdBxlgix6AwdX5%L`>yqacR z4#pC5XCB?mef(t+=XOH`w_t^{j$Q&^`?A}Z=DpBEs<(uXbOP8TYUA(nbjE^cTKU(q zbo$cIXdAFA#|B!i%vc6qSz30!6&;6c+->yA^ z3Gy|Nv^Bo?S>hw+!D+@xU0v)!U^=?*ojVquph2kX9AMYKDP*^;RAIkxb%)AwnULXV zG=S-^2Se_D1RoSGg#PB`L6^jF0t+@IG6wb`?)mXVcrlNtA4fSW>GN@}>uT(DXE1)I zV=Kp&mIE7(Ec1E)R9fyRlc@VA0P zpnJhhD6Wo$8J&AtUH-hwPK0wNy zzW;0cY&|^<_WEoB?{qx^-Wyg%vJbe!ihT>9N6CvI&$s1s@nJcpi&_C{N5 z?aOvZ6P7R66)Zsa6fZshS^p5^5%@8w==!T7sb623hxjQ3jcH}9d} zeqNNOoV(<>n1iv+xJPt0a;Low=Qw`f#a-t^ag*qmIny^ea`ov33)_UJ_35>#Db&BPGDnw1<-J*hE3Clz&7Fv zuxYCt)ZZe2iuE(-{t63;8)Jj`91tWVO7HPgwK4c~AOq)mONg@?-S{pCWn%6LE8G;U z1J`(Q*$)qUv9+fVK#STSP?FThT+d4eL_K159HqH6U0cOHw;+P| z#?~KSZyCp%xHL%wZ3rdy>w#SLwoKybLQ~{Y*g9xas~0;?mVwsXp`l9E=U}Cu9&DYi zNtpGj4cVA_6{*=g%W7wfHg(43y_J^5cUj>OL3VT1WqBL!TeaGe)YiISnR)O4sl=~? z%Dx%GkG^!nU*$U3jtj9UZ$JlmT4V$&Kl*~WP&ViW_ylS_^EX#JU&6O5R^!&)sO5w< z0(@hGJDiADZM;Ti6c5K9bI@JE1IEFI*w?Y0{RQp4KZ`E8HV+i5wQ!?Kmy`eo`#@b8@aIz59sBd5%oJ z;rvnA>M#e{@SX&#feq`(_{%qCwta|v<(qWb%agiP_pZ~V%B%Zed+P)gqk0{*9J~Q- z>bws9Vlkkjc6R`OqlKWyzJBn;-0hIICY>ViNvq_E8I(@kU8*uDlhi&}OSJ)uNyC>B zRLcJQ)YdmMsAO3w;W;TG-phv(xLt=>n05sBpI<;6l-7pi-;C7Q_DvpD5 zYMUV_j{!Yx_(jjV;tUEmzJiiJ2SGE3awW_5$wk*jR8>d+6sQO`J4%{lcf{sFuOv|& z(PB&MJ1QD;Y*lo$%%EN4_Ru_20|?k%3TF?1uqSIK{Gj~}XaKzgdl)pRM}ouUfBi+t z`|EJQ#Gk)8WV^q~1~oKurZHOjYZ+42t%-?`ICdvkcRQ3l-IxJe(~g`b+Fub zcBFh$cNN9ec9GRtj>-d`n#ez{L#$8at|Z{YSMas>HzJC}1HA6i3S6PtEk-mkU&bZ~Q z!t-R#d@b3W^2Ov@yaq|yb{cVXcZ0I$-AB$^*dP+eFL2~{F_;wN4==+e;OyQ6WOUk3 zfDn$ru5Lz1^tIP$G;KEOHlU2W-B-l6|5A&b`rF12%o>uZVyy#*j&@o>KIBe9Dg#f=cj zj>Ym#-dqy?_yGwQ6|13ZJ~yH_ELNe*6+IF45Jgz3w+lJ6=p#HIZ-uA3WTVOBFmlM_ z3~2qw8`xjI3+O88Wx+01K+1tsz||!d$aMsO%PAQEU}O&XBQ#>FjTZhT`!y0Zvk8w( zn1lQ7)FckjbFh>4Z_vGw$1#QSL?X27tGr~5y(}hfh&mPDORd_hPT|k3*Og%>%qGO7r31_q}GKVDD*oM9uv# zH!2dRSG@*XZq`C4J`}UlM?JyPDH`A^%N{^zpAYPqQwx;6`T(T3UtxzFN|%|e{6?Lx zeoRS|S5XU#Hd2%Chp8DG){p}+>*Pq726;MHnbP<9%I=#%15#~yK>x%0Kt{?PR>&JC z;1Ev(_?vfUX?`#Vc7Ez(A!}u%Iem_mIXjNLGbomoop@qZHozh~J&k3S^$}#X1t44O zJSKB`vq(_DQRI!E;R(L%Q4#vO+~XBGgbSu@Ru;HGQvSm$#sYH-YvE|`d?N0tFaBoZ zcR0IMLR{KihZDb3h|Tn$7+Qm1mn?C->Ermn&O1kC)x*Q9LAaXmfQH5up+iUIpo8H# z_PZ?ufNgRkSY;3k#9e#?Iw|YvXMVrCj-THvAus zt9VY+f1XeD%u*CRnf)IQYz6*HfsEt8@Ow6R=6DsG%GL%XeLnzy zm1MS&h0=fU(}vaeX~iNMyc`~4*KVFrs29gW|9Ka>@Bi{EmFoit(dV8BF`rd4%s}fE2`P1c5(hB)trxW=yc$g|s6-8JUx^+3bA`W@!{M`hxv=qr)$E>+3( z)^n18RZMZ+>^l-3-XSV}J452V`;s{QPoAiMTdkUZUAB0QYOJL4s)Z=a?1ajSkC~!% z?w8a~%@PSFxz{8LD;hhN2z_m>2^f$mtg+I%h5b^)W#Fp?|K2<^St$6 z^ZsS+?kK61ggV^zgf`v-PZ4na)Z<=HKi zTJa6ubXElqu)mBQTf7=~{rLzrjGTt35Fes??v{)|~GTnv(5%gL~gFE~i?^bqUM=hLmfw1--2{7#maeCU*S zryiD1sxOAN<{twXHXtPBoJC?zDQBwb(n}KviOm3zq{^RkxxVk{00!c54%p?8CL){>hm6!{ForM3{aph(j22=xgJ?-&B>jkGxISC>pPz!*n8Ai~7`*%8 zTCJs#qAAfuW`_{t~AnDXu0&}(oh zRw(R5FZ~4Y#$W|BY)UhnbJ-ESuxk|_wAdetO?QXO#vEYNEBg@BCBty}iW$&z`@>-L zg%a3mY6i?4o`qyHo?~0)4WZ{XZ$eM}TQQ^d16YNF6}~O=I{H@QuYdcf6a9MXD1JLA z5w5vo0@wvFg5qg1ba8hkg0FFeC)0C)u82JNMUy#l?N|)5QLlz93LU3XB21|tum<_% z^L_cGr4IEv_XOpWttdb5WKU`L2g#T3b3{TeOtLZOP%zf^0Q#x&1G4f#2RwB9IM6!t z0<7zvk6gqGkXIi&WxGNIq^|j0E57$f8Ch6NmVGgjS!u18sa&g&b%u$lzquW1Y)UXv zY_AQ4`<6p7pf;AY`QTqZuo!{t&#(};62#PW8k(gSfnJ(jj~yFS!ZPT?P&;-2JLG1D zW%?=LC2oh&s1Q%2*d_`M>fVaS0ZLY*I<{ z9e$~p7S2sG{kJx9+xBT_Rmfpfb0-QXKQo0I?%aTEyBSE-huP4ltRj#ju>e1d%Mt0m z7JLc65(~VdkHn^w;IC(_#>aMP6V>xhVtmUClqNoh#iJ=i_9{Nd??VHpGuVWaT_)f- z2lQ~DbweCm>ph&(C$~5Yg=(COJuaN&q?7QJwaL)z@0DN((2nd~GY#42=Z-u&%z{_{ zN`SxYAB9ugX=q=nm&pApEJ1F+79?x;aHoCHmKfxe@OB+;6JGao6&oyEAuQ@s6#vyf zT&K1Qdl8Tc$0OAJZ)n$^@e6d zYVI_$W_c;d&%St0o=y_Vf4tmEJ>445F`%ascLzH;9m?GtGyS8S@7iJ96+ap{Zl(1c zo7Xoumu|Up>_)wCk@q+@b*KRK&XeGgRgqYmJ&4b@GsG4weSogH`VF;M7lU7UxR+4) zRDjLY~I3+mD0-n;vXu zS8p2!)@_Jp-4)rf91bh6%cd)`UFF^ePd)tgUmrEqJNs#!b_YMfV8*O$y%m2-3^uAc z>4gd?z2xKXbcYwvxNmkD5>M^5xfW|uxe7YBI7JVYc&grM9JO0Cu1B68*Q+6)Q#aIw z7daW@j7z}~t=<@axUCKYCU4^S*>Uvc*$iZ`j*l6*m=Kop9^wU$v~aJ`2kfp@KKMO{ zF>Ki48=lwB!APYxq~L=yR&>>kFh5qRYP7UORTEQI>k3I$^@!iB*6LEA`qQ4Jx_5uN z+9{1nRp!=kwdXgkvR-V_fD)-e*4^4s*0;q%psq{{m|kH7j8FT{S~WV(+WWne_3iT| z_7Qs?CV|~99(BAhO z(O07_XjMfT)_q3{`x&T%mY+~YmZlY9FCl$wZ`MI9;z2J$^HxV4m)4=Gu|KevhC8|U zhi?#0!O7g>)x+H8&?(&8t2B5P-h8gV#ujd2!CEePNQ*n~SOYi|Fax^L%Z5kPdtiMH z6+54Z^b7Q;#Qnk z@XChcA6SkrpL19i`ThOh!=hSbXot7y;U9If?AR=se_kf}UiYj_dEqy!ANReiUT%~~ z%<878+{oN68cdlkw!0lAA-5+9N197Tl{;cpoSe>yHxE3LJkVB?tqhq<9d5g6r8VcJ zN$@TyIqURJSy<>-nb(D>lu4PE*^B+izs_6TxhO&QaByhCDinC5VTUf*P=WNi7Q-=h z31CFsQdrG62vJ+&gRHi)gqo3LaLMvSFyg=y_{erHY#Ve9T)jfb+`In_^sTKI>WEE% zD{>o!sjmcLpA!O+IW@*DHjNV1)ZG=F`>|NG_l1?Hd5cgGVOAlC@K}=%m8$86Mo8Y3mBj8sRFKDZ&C!E=?0x$J#h28rP zf(y@c0qK;azh{Uzz;k;M@af|rW!|{;-o~0pUWDd_~>^DczjZF ze>OvGbuvx@{xp~TwTEJcri03A-;I)4KkN*;Z=nV*@jiyH+Hu2{)Tss?#8!j0r&shd zGouY=1Al{2@{nP=$O9bw%VEj9I0V`7Ncj1m7U>1HS2hRN;Da_Vq-0x4%rm!16+ zN~P`9CL8_BNyc+ivSd3Y3rv-e^|iOzs^Ogw%iuipY)3o%fI-5Wau0zE7H$TWIxj%| zT2;_iBnHkCB&e33iWMJ!?y5RA9Ibk|K|`hdV5OvBot?@NiG#{`oRQkL0G`^v^-FX# zaftp?2(dPO1kTjPK+~3_f(Z^306q_5RsV`%`##wY7F@m#=EPw{kF!5+p?VT}WgCZE z%6WLbk~*P8xnYaZ-^ihb+cECt7~+>1{`Gr%!5;YvTAQVWnPe8 zV7)az$1307!qPLo%$jxh8AppSBTBz;Ighuwb5y2jaqLcHabCSy$r+i_hEILnr?})@>!&bAVtw;w|mUpv` zeS^XB<<@{==^kMA5ehuJTFCzME&wn<0Jb5oPApL^6>SRKAz9+SQyl*r5ob{u;e`WbJB+v%Z{khZ@|1NyTt zO_84A4IH?m%m~m-1QZ?epo`B<;q@EuBXWifVuingygp6>?Av89p7sD%-V~1faX!wz z8f^{FeUr)_)VvP#nPjuuvrO5-Ynnh7T+CG6NU^_GjshyZaq_G^v&p=3snn!lIkjwa z0X5_FW9sLpFVva9RU~nmF0YVxQ4XUDMF0ACn5oJlWc|12_^pv7TxK(ih}eG(E8aw4 zmf<7Vz4<2yKd6I!pHyNwj^AZC=qs?gW9h8+C>;iHkj1i@LuZW`A*_r#g!OP2pJTUm z0gf!UlDSxCI7pkOFkK0K#ZC0w<=#{AF9rz=C#&gsNiKD4L_rx`ZjxG4G%%cvF)OZX`U5-E#K8^S>NLBbF{jx76rj zCwm+4o|U1p#Ke4AA^*B%%V(jvOz_p}{ecx$OO+dC=l*QBdgV84mF5|2WfUAEb-1Xh zdinkj=|q0J8phZuegDE#P5d-TYJa{;b-_0)bsf38R90pqpA*(i1w>7;*?YOidi|n3 z6dG$_Gf$<8+N>$G&bYKjUMg^t7p2W7)_AapZvonb63ioBK}^CcPejc1OeCtZR0z}A z`8aT2PP}AViyUV2#Z3q2sXT0cD{1uSNHzeel68tpMD<&COP;*X7rUJblc-)?1U)UX zhxQK%Vf&NQ(21Iph~|g2aC+GPY31CbpDI7n`9*QdI3TFKjv!Bm?{35M)DknSF5C|LLw7Mc#RZJs1U$Cs4=DJ`G=Lx2O= z+-L+&F7_q7$n*Fmy-b8Z#vtr}Psc;v#1O&-41Di=2y>SWU}2TL#I(fOa`_8{D(l-w z*0vp#M-K`V*Q2xKe^|>%x#uw|qyG>&+%`**D<~HCyW1OuDlsF&6#^sAzN1E+2OJDk zj+J6z@+qOA)k2{<>V)vxlbg_>ZXEOORz0?3U;@}ugMtfNY@i8^D`Z$03dKbqf%mLD z4ZldXgNBVs=(|c&cI^WWP^*LBFKRVgIO4>-Uq2OYSDL}uRcWx%%VWGxPuZMJ)k>aQ z+6?Zg- z)n}HHMJeN?G@M4ZU{{Fur5~|2=W62WS98u`JQ-IMU>s7-BC^@%@g?eBqF{~Z&w>T5SH79PqNCe$S1kivLuS`AlyG zm*q*GFlr{_6MhzdVG3j$H#*6jr_@QKJ{ZUtt3ss6=2(kJ^OX|+VjqdNhABCi=RjPu zQ{p2-Q_!Gbh)^Zx;*YmS5r-?Z|I3?-J!8DbW|njiSEoLv=+>5Y+`|He=hpoc&90j4 zsrx7odvKW2!9&SrVFIOUNu+{`s-QD59MI~0j@_`~7nZAM0edKLDKm6hJ8LLf#9p*f z0zpstY-2wNRHW?!q5exD(QjE123N9_2w7FoT40qJw{8vTyY54N0KyBtIph*LAYT^zCeU?lhL0 zGLc4ODeY;ldL&x*SfMm|L*1nK3eK8VYQVtO-fvM6sd^u3-!kX9qGa|Fz4gK~@VW3M z@a58z$dXZGWK`V+*X-WT;>>mdT~7*t_@7L~B>tqCHeO^=g_PyvxpyshM|fH}uhqAt znei-0mE7vQ;D+qbnM*R&!Uje|(ifKFl*^1Q_v;upmR)9qbzNgLr{83x_S|ITR&+Ct zN8V*9(@x-8Z!BiUx(DLdhv7w87`}zohPSo2VkyJnXq)j~jPGGjbOWAnd4!H2r5`u( zS@YQ>C89+@#631auq>^g%2Bw(N@XHVcNP`eeL&Sg@#ukNYW(FwOF|;PgAVv6VaLq7 zFx|2Yyf0dUZ*zEr?a!Qt?Xyk-+tz1cbDvpZk*O636bdl6ADh71egW$5u@);^Jc8(q zTo9jM`;+9_t5nezwZ`zlJ`;a)3byGk%o=5^Glky7Ms9(N72bYsFFcVznPowzq<9l7}GQUquwfKxeYgZ>_SM0={!E3_7b^%`sTZf-SBz!bG~oB*5{TX!d!n}?0q-xY z#&KUJeq*qSNV?X}*!qv(blpB!GF7>btm%XcqmrA&dfDc|#0*;Xyi(IyT2>Wf$;=q^ zowW#4jd~$r&r$S#4F?UE%E6wA)#zph9r@l{4peszqH7j-BN@s;m?v~%7b*oP^h-X{ zN>W;t9s%L4YwrNFq39}>`#1UAN)vqmdcqBbAPkhsnM z;BsLclz-M1_|JAma$OSCC4=lMdiv^{etpzeV1Q9;-M-UC-;K} zkAK0!cgi7-G0D+q8b0Ws*7-dgoL}0hwciFCqld+VkCNk=)LQY zo85}TIE{h$&PxHDqvL7%`&i%eQbu QT&r(7{J)mG=07+70#W?Io&W#< literal 0 HcmV?d00001 diff --git a/tensorflow/contrib/tensorrt/test/testdata/model.ckpt-46900.index b/tensorflow/contrib/tensorrt/test/testdata/model.ckpt-46900.index new file mode 100644 index 0000000000000000000000000000000000000000..537976571337508ab1798d33646c51d62a146ecc GIT binary patch literal 652 zcmaKoO-K}B7{{M?_I=0MogH^(wux~~=iO!59m&saN2svckQAw~kSs%%)g4XRT+=Nh zgNBkC=&*$L1rbTnDPkiFp&&^SDX1<%VW-$j1|1~p;HBA1$Nrax-;d}29G)NSZ2*=- z0fs}s^L=J6)s!5`j+>(#l0nDDqX}sTE)!x&MVY7A5vyNw4}h2Zv%_a&>4v_c)R?o0 zWrm}a(=B8q?tRSg#NUuer%ovWY$H@jb5p@Ck?|*+DpX*Q+G$_D@VboEPMV{`X2z+D zPTeJ7K%^&byL8KKyuI@HjzF{q;dnq9bvZ1RSP8yAyuuq+LCv(^zZ_3RA!Lg7}l+MY+QWR z`P4^(2NyxmaS@Xmb?3IZ^d=^_cL{_De&{BVT5jrDFL;!+IXq@c@Aebf#aWozMM1Yj z<4LibfUhbfO-NDrMWX7FKkaQkMj=@#fG5jQLhbt5dWmHN$_lvA2cbVcU9MXeXNWvi zyd)s?#6kd@Nk}Dk>$XZvfFxrYKsC|CV*6Tw?8>7?54EQwx^Ap3#?O13^N91%0;v># zZ>%Pymb^osi7HM~UgBc25cIb?U-eMyMnt#nBjct$kT0*Fg^Cb<9wj01z_;D+{mN{G zhCk^UC)7l8K-X0m;$OmZw|j&N=nsI^|BbH>op$`^3*9RV=TM05MDA1w)o0p{0c h3V>WE;06BYmXEb8zr@@!_LbEC)4gF`3!iCQ`3q_5l$rnl literal 0 HcmV?d00001 diff --git a/tensorflow/contrib/tensorrt/test/tf_trt_integration_test_base.py b/tensorflow/contrib/tensorrt/test/tf_trt_integration_test_base.py index a725d0651c9..495a9391a1e 100644 --- a/tensorflow/contrib/tensorrt/test/tf_trt_integration_test_base.py +++ b/tensorflow/contrib/tensorrt/test/tf_trt_integration_test_base.py @@ -30,6 +30,7 @@ from tensorflow.contrib.tensorrt.python import trt_convert from tensorflow.contrib.tensorrt.python.ops import trt_engine_op # pylint: enable=unused-import from tensorflow.core.protobuf import config_pb2 +from tensorflow.core.protobuf import rewriter_config_pb2 from tensorflow.python.framework import dtypes from tensorflow.python.framework import graph_io from tensorflow.python.framework import importer @@ -42,14 +43,15 @@ TfTrtIntegrationTestParams = namedtuple("TfTrtIntegrationTestParams", [ "gdef", "input_names", "input_dims", "output_names", "expected_output_dims" ]) -RunParams = namedtuple( - "RunParams", - ["use_optimizer", "precision_mode", "dynamic_engine", "test_name"]) +RunParams = namedtuple("RunParams", [ + "use_optimizer", "precision_mode", "dynamic_engine", "test_name", + "use_calibration" +]) ConversionParams = namedtuple("ConversionParams", [ "max_batch_size", "max_workspace_size_bytes", "precision_mode", "minimum_segment_size", "is_dynamic_op", "maximum_cached_engines", - "cached_engine_batch_sizes", "rewriter_config" + "cached_engine_batch_sizes", "rewriter_config", "use_calibration" ]) PRECISION_MODES = ["FP32", "FP16", "INT8"] @@ -65,6 +67,34 @@ class GraphState(object): INFERENCE = 2 +def OptimizerDisabledRewriterConfig(): + """Returns a RewriterConfig with all default Grappler optimizers disabled.""" + rewriter_config = rewriter_config_pb2.RewriterConfig() + + # Turn off all default Grappler optimizers. + off = rewriter_config_pb2.RewriterConfig.OFF + rewriter_config.layout_optimizer = off + rewriter_config.constant_folding = off + rewriter_config.shape_optimization = off + rewriter_config.remapping = off + rewriter_config.arithmetic_optimization = off + rewriter_config.dependency_optimization = off + rewriter_config.loop_optimization = off + rewriter_config.function_optimization = off + rewriter_config.debug_stripper = off + rewriter_config.disable_model_pruning = True + rewriter_config.scoped_allocator_optimization = off + rewriter_config.memory_optimization = ( + rewriter_config_pb2.RewriterConfig.NO_MEM_OPT) + rewriter_config.pin_to_host_optimization = off + rewriter_config.auto_parallel.enable = False + + # Run only once for each enabled optimizer. + rewriter_config.meta_optimizer_iterations = ( + rewriter_config_pb2.RewriterConfig.ONE) + return rewriter_config + + class TfTrtIntegrationTestBase(test_util.TensorFlowTestCase): """Class to test Tensorflow-TensorRT integration.""" @@ -139,11 +169,15 @@ class TfTrtIntegrationTestBase(test_util.TensorFlowTestCase): is_dynamic_op=run_params.dynamic_engine, maximum_cached_engines=1, cached_engine_batch_sizes=None, - rewriter_config=None) + rewriter_config=None, + use_calibration=run_params.use_calibration) def ShouldRunTest(self, run_params): """Whether to run the test.""" - return True + # This setting combination requires quantization nodes to be present in + # order to build the engine. + return not (IsQuantizationMode(run_params.precision_mode) and + not run_params.use_calibration) def VerifyRunForEngine(self, engine_name, graph_state, expect_run=True): """Verify the state of a particular engine after sess.run().""" @@ -194,34 +228,35 @@ class TfTrtIntegrationTestBase(test_util.TensorFlowTestCase): def _PrepareRun(self, graph_state): """Set up necessary testing environment before calling sess.run().""" # Clear test values added by TRTEngineOp. - trt_convert.clear_test_values("my_trt_op_.*:ExecuteTrtEngine") - trt_convert.clear_test_values("my_trt_op_.*:ExecuteCalibration") - trt_convert.clear_test_values("my_trt_op_.*:ExecuteNativeSegment") + trt_convert.clear_test_values("TRTEngineOp_.*:ExecuteTrtEngine") + trt_convert.clear_test_values("TRTEngineOp_.*:ExecuteCalibration") + trt_convert.clear_test_values("TRTEngineOp_.*:ExecuteNativeSegment") + + def _GetGPUOptions(self): + gpu_options = config_pb2.GPUOptions() + gpu_options.allow_growth = True + return gpu_options def _GetConfigProto(self, run_params, graph_state): """Get config proto based on specific settings.""" if graph_state != GraphState.ORIGINAL and run_params.use_optimizer: conversion_params = self.GetConversionParams(run_params) - rewriter_cfg = trt_convert.tensorrt_rewriter_config( + rewriter_cfg = trt_convert.get_tensorrt_rewriter_config( conversion_params.rewriter_config, conversion_params.max_batch_size, conversion_params.max_workspace_size_bytes, conversion_params.precision_mode, conversion_params.minimum_segment_size, conversion_params.is_dynamic_op, conversion_params.maximum_cached_engines, - conversion_params.cached_engine_batch_sizes) + conversion_params.cached_engine_batch_sizes, + conversion_params.use_calibration) graph_options = config_pb2.GraphOptions(rewrite_options=rewriter_cfg) else: graph_options = config_pb2.GraphOptions() - gpu_options = config_pb2.GPUOptions() - gpu_options.allow_growth = True - if trt_convert.get_linked_tensorrt_version()[0] == 3: - gpu_options.per_process_gpu_memory_fraction = 0.50 - config = config_pb2.ConfigProto( - gpu_options=gpu_options, graph_options=graph_options) + gpu_options=self._GetGPUOptions(), graph_options=graph_options) return config def _ExpectTestValue(self, engine_name, method, expected_value): @@ -291,6 +326,11 @@ class TfTrtIntegrationTestBase(test_util.TensorFlowTestCase): params = self._GetParamsCached() conversion_params = self.GetConversionParams(run_params) logging.info(conversion_params) + + config_for_trt = config_pb2.ConfigProto(gpu_options=self._GetGPUOptions()) + if conversion_params.rewriter_config is not None: + config_for_trt.graph_options.rewrite_options.CopyFrom( + conversion_params.rewriter_config) return trt_convert.create_inference_graph( input_graph_def=gdef, outputs=params.input_names + params.output_names, @@ -301,7 +341,8 @@ class TfTrtIntegrationTestBase(test_util.TensorFlowTestCase): is_dynamic_op=conversion_params.is_dynamic_op, maximum_cached_engines=conversion_params.maximum_cached_engines, cached_engine_batch_sizes=conversion_params.cached_engine_batch_sizes, - rewriter_config=conversion_params.rewriter_config) + use_calibration=conversion_params.use_calibration, + session_config=config_for_trt) def _WriteGraph(self, run_params, gdef, graph_state): if graph_state == GraphState.ORIGINAL: @@ -400,10 +441,12 @@ class TfTrtIntegrationTestBase(test_util.TensorFlowTestCase): is_dynamic_engine = not node.attr["static_engine"].b self.assertEqual(run_params.dynamic_engine, is_dynamic_engine, node.name) + self.assertEqual(node.attr["use_calibration"].b, + run_params.use_calibration, node.name) has_calibration_data = len(node.attr["calibration_data"].s) if (IsQuantizationMode(run_params.precision_mode) and - graph_state == GraphState.INFERENCE): + run_params.use_calibration and graph_state == GraphState.INFERENCE): self.assertTrue(has_calibration_data, node.name) else: self.assertFalse(has_calibration_data, node.name) @@ -438,6 +481,11 @@ class TfTrtIntegrationTestBase(test_util.TensorFlowTestCase): # types. scale = 10.0 if np.issubdtype(dtype, np.integer) else 1.0 dims = params.input_dims[i] + # TODO(laigd): add debug options. E.g. we can set the input data to be + # continuous natural numbers: + # seq = np.arange(np.prod(dims)) + # seq.resize(dims) + # input_data.append(scale * seq.astype(dtype)) input_data.append((scale * np.random.random_sample(dims)).astype(dtype)) self._VerifyGraphDef(run_params, input_gdef, GraphState.ORIGINAL) @@ -449,7 +497,8 @@ class TfTrtIntegrationTestBase(test_util.TensorFlowTestCase): config_no_trt, GraphState.ORIGINAL) # Run calibration if necessary. - if IsQuantizationMode(run_params.precision_mode): + if (IsQuantizationMode(run_params.precision_mode) and + run_params.use_calibration): calib_config = self._GetConfigProto(run_params, GraphState.CALIBRATE) logging.info("Running calibration graph, config:\n%s", str(calib_config)) @@ -519,27 +568,38 @@ def _AddTests(test_class): use_optimizer_options = [False, True] dynamic_engine_options = [False, True] - for (use_optimizer, precision_mode, dynamic_engine) in itertools.product( - use_optimizer_options, PRECISION_MODES, dynamic_engine_options): + use_calibration_options = [False, True] + opts = itertools.product(use_optimizer_options, PRECISION_MODES, + dynamic_engine_options, use_calibration_options) + for (use_optimizer, precision_mode, dynamic_engine, use_calibration) in opts: if IsQuantizationMode(precision_mode): if use_optimizer: # TODO(aaroey): if use_optimizer is True we need to get the inference # graphdef using custom python wrapper class, which is not currently # supported yet. continue - if not dynamic_engine: + if use_calibration and not dynamic_engine: + # Static engine with use_calibration=False will be static, so we want to + # test that. If use_calibration=True, only dynamic op is supported. # TODO(aaroey): construction of static calibration engine is not # supported yet. continue + else: + if use_calibration: + # Don't calibrate in FP32 or FP16 mode + continue conversion = "OptimizerConversion" if use_optimizer else "ToolConversion" - engine_type = ("DynamicEngine" if dynamic_engine else "StaticEngine") - test_name = "%s_%s_%s" % (conversion, precision_mode, engine_type) + engine_type = "DynamicEngine" if dynamic_engine else "StaticEngine" + calibration_type = "UseCalibration" if use_calibration else "NoCalibration" + test_name = "%s_%s_%s_%s" % (conversion, engine_type, precision_mode, + calibration_type) run_params = RunParams( use_optimizer=use_optimizer, precision_mode=precision_mode, dynamic_engine=dynamic_engine, - test_name=test_name) + test_name=test_name, + use_calibration=use_calibration) setattr(test_class, "testTfTrt_" + test_name, _GetTest(run_params)) diff --git a/tensorflow/contrib/tensorrt/test/unary_test.py b/tensorflow/contrib/tensorrt/test/unary_test.py index 8736bfb6449..9fc50e05952 100644 --- a/tensorflow/contrib/tensorrt/test/unary_test.py +++ b/tensorflow/contrib/tensorrt/test/unary_test.py @@ -107,8 +107,8 @@ class UnaryTest(trt_test.TfTrtIntegrationTestBase): def ExpectedEnginesToBuild(self, run_params): """Return the expected engines to build.""" return [ - "my_trt_op_0", "my_trt_op_1", "my_trt_op_2", "my_trt_op_3", - "my_trt_op_4" + "TRTEngineOp_0", "TRTEngineOp_1", "TRTEngineOp_2", "TRTEngineOp_3", + "TRTEngineOp_4" ] diff --git a/tensorflow/contrib/tensorrt/test/vgg_block_nchw_test.py b/tensorflow/contrib/tensorrt/test/vgg_block_nchw_test.py index b0271a04b36..b29626d2c28 100644 --- a/tensorflow/contrib/tensorrt/test/vgg_block_nchw_test.py +++ b/tensorflow/contrib/tensorrt/test/vgg_block_nchw_test.py @@ -76,7 +76,7 @@ class VGGBlockNCHWTest(trt_test.TfTrtIntegrationTestBase): def ExpectedEnginesToBuild(self, run_params): """Return the expected engines to build.""" - return ["my_trt_op_0"] + return ["TRTEngineOp_0"] if __name__ == "__main__": diff --git a/tensorflow/contrib/tensorrt/test/vgg_block_test.py b/tensorflow/contrib/tensorrt/test/vgg_block_test.py index d7c165784bf..9b0b1896260 100644 --- a/tensorflow/contrib/tensorrt/test/vgg_block_test.py +++ b/tensorflow/contrib/tensorrt/test/vgg_block_test.py @@ -67,7 +67,7 @@ class VGGBlockTest(trt_test.TfTrtIntegrationTestBase): def ExpectedEnginesToBuild(self, run_params): """Return the expected engines to build.""" - return ["my_trt_op_0"] + return ["TRTEngineOp_0"] if __name__ == "__main__": diff --git a/tensorflow/contrib/timeseries/python/timeseries/BUILD b/tensorflow/contrib/timeseries/python/timeseries/BUILD index c230919168b..4b90b596b28 100644 --- a/tensorflow/contrib/timeseries/python/timeseries/BUILD +++ b/tensorflow/contrib/timeseries/python/timeseries/BUILD @@ -104,8 +104,10 @@ py_test( srcs = [ "estimators_test.py", ], + shard_count = 3, srcs_version = "PY2AND3", tags = [ + "no_mac", "no_pip_gpu", # b/63391119 "nomsan", # Takes too long to run. "notsan", # b/67865658 diff --git a/tensorflow/contrib/timeseries/python/timeseries/estimators.py b/tensorflow/contrib/timeseries/python/timeseries/estimators.py index af68aa03cf6..146ed9f2713 100644 --- a/tensorflow/contrib/timeseries/python/timeseries/estimators.py +++ b/tensorflow/contrib/timeseries/python/timeseries/estimators.py @@ -32,7 +32,7 @@ from tensorflow.contrib.timeseries.python.timeseries.state_space_models.filterin from tensorflow.python.estimator import estimator_lib from tensorflow.python.estimator.canned import optimizers from tensorflow.python.estimator.export import export_lib -from tensorflow.python.feature_column import feature_column +from tensorflow.python.feature_column import feature_column_lib as feature_column from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops from tensorflow.python.framework import tensor_shape diff --git a/tensorflow/contrib/timeseries/python/timeseries/estimators_test.py b/tensorflow/contrib/timeseries/python/timeseries/estimators_test.py index ffd838be40e..7d780559f97 100644 --- a/tensorflow/contrib/timeseries/python/timeseries/estimators_test.py +++ b/tensorflow/contrib/timeseries/python/timeseries/estimators_test.py @@ -30,7 +30,7 @@ from tensorflow.contrib.timeseries.python.timeseries import saved_model_utils from tensorflow.python.client import session from tensorflow.python.estimator import estimator_lib -from tensorflow.python.feature_column import feature_column +from tensorflow.python.feature_column import feature_column_lib as feature_column from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops from tensorflow.python.platform import test diff --git a/tensorflow/contrib/timeseries/python/timeseries/head_test.py b/tensorflow/contrib/timeseries/python/timeseries/head_test.py index 90c7d8ac1a9..8f692d94da4 100644 --- a/tensorflow/contrib/timeseries/python/timeseries/head_test.py +++ b/tensorflow/contrib/timeseries/python/timeseries/head_test.py @@ -38,7 +38,7 @@ from tensorflow.core.example import example_pb2 from tensorflow.python.client import session as session_lib from tensorflow.python.estimator import estimator_lib -from tensorflow.python.feature_column import feature_column +from tensorflow.python.feature_column import feature_column_lib as feature_column from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops from tensorflow.python.ops import array_ops diff --git a/tensorflow/contrib/timeseries/python/timeseries/math_utils.py b/tensorflow/contrib/timeseries/python/timeseries/math_utils.py index 43c5267e632..aab33064386 100644 --- a/tensorflow/contrib/timeseries/python/timeseries/math_utils.py +++ b/tensorflow/contrib/timeseries/python/timeseries/math_utils.py @@ -802,7 +802,7 @@ class InputStatisticsFromMiniBatch(object): array_ops.shape(times)[1] - 1, self._dtype)) # Co-locate updates with their variables to minimize race conditions when # updating statistics. - with ops.colocate_with(auxiliary_variables.max_time_seen): + with ops.device(auxiliary_variables.max_time_seen.device): # There is a race condition if this value is being updated from multiple # workers. However, it should eventually reach the correct value if the # last chunk is presented enough times. @@ -810,16 +810,16 @@ class InputStatisticsFromMiniBatch(object): auxiliary_variables.max_time_seen, gen_math_ops.maximum(auxiliary_variables.max_time_seen, math_ops.reduce_max(times))) - with ops.colocate_with(auxiliary_variables.chunk_count): + with ops.device(auxiliary_variables.chunk_count.device): chunk_count_assign = state_ops.assign_add(auxiliary_variables.chunk_count, array_ops.shape( times, out_type=dtypes.int64)[0]) - with ops.colocate_with(auxiliary_variables.inter_observation_duration_sum): + with ops.device(auxiliary_variables.inter_observation_duration_sum.device): inter_observation_duration_assign = state_ops.assign_add( auxiliary_variables.inter_observation_duration_sum, math_ops.reduce_sum(batch_inter_observation_duration)) - with ops.colocate_with(auxiliary_variables.example_count): + with ops.device(auxiliary_variables.example_count.device): example_count_assign = state_ops.assign_add( auxiliary_variables.example_count, array_ops.size(times, out_type=dtypes.int64)) @@ -829,11 +829,11 @@ class InputStatisticsFromMiniBatch(object): # the series are then members of fewer chunks. For series which are much # longer than the chunk size (the usual/expected case), this effect becomes # irrelevant. - with ops.colocate_with(auxiliary_variables.overall_feature_sum): + with ops.device(auxiliary_variables.overall_feature_sum.device): overall_feature_sum_assign = state_ops.assign_add( auxiliary_variables.overall_feature_sum, math_ops.reduce_sum(values, axis=[0, 1])) - with ops.colocate_with(auxiliary_variables.overall_feature_sum_of_squares): + with ops.device(auxiliary_variables.overall_feature_sum_of_squares.device): overall_feature_sum_of_squares_assign = state_ops.assign_add( auxiliary_variables.overall_feature_sum_of_squares, math_ops.reduce_sum(values**2, axis=[0, 1])) @@ -869,7 +869,7 @@ class InputStatisticsFromMiniBatch(object): state_ops.assign(statistics.series_start_moments.mean, mean), state_ops.assign(statistics.series_start_moments.variance, variance)) - with ops.colocate_with(statistics.start_time): + with ops.device(statistics.start_time.device): series_start_update = control_flow_ops.cond( # Update moments whenever we even match the lowest time seen so far, # to ensure that series start statistics are eventually updated to diff --git a/tensorflow/contrib/timeseries/python/timeseries/model.py b/tensorflow/contrib/timeseries/python/timeseries/model.py index edd97b2a4c1..a8cd4287e00 100644 --- a/tensorflow/contrib/timeseries/python/timeseries/model.py +++ b/tensorflow/contrib/timeseries/python/timeseries/model.py @@ -27,7 +27,7 @@ from tensorflow.contrib.timeseries.python.timeseries import math_utils from tensorflow.contrib.timeseries.python.timeseries.feature_keys import PredictionFeatures from tensorflow.contrib.timeseries.python.timeseries.feature_keys import TrainEvalFeatures -from tensorflow.python.feature_column import feature_column +from tensorflow.python.feature_column import feature_column_lib as feature_column from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops from tensorflow.python.framework import tensor_shape diff --git a/tensorflow/contrib/timeseries/python/timeseries/state_space_models/BUILD b/tensorflow/contrib/timeseries/python/timeseries/state_space_models/BUILD index 3c07a74ed8a..125750e7639 100644 --- a/tensorflow/contrib/timeseries/python/timeseries/state_space_models/BUILD +++ b/tensorflow/contrib/timeseries/python/timeseries/state_space_models/BUILD @@ -40,7 +40,10 @@ py_test( timeout = "long", # Moderate but for asan srcs = ["state_space_model_test.py"], srcs_version = "PY2AND3", - tags = ["no_windows"], # TODO: needs investigation on Windows + tags = [ + "no_mac", + "no_windows", # TODO: needs investigation on Windows + ], deps = [ ":state_space_model", "//tensorflow/contrib/layers:layers_py", diff --git a/tensorflow/contrib/tpu/BUILD b/tensorflow/contrib/tpu/BUILD index a0a9cb3f31a..05d2ebd2e8a 100644 --- a/tensorflow/contrib/tpu/BUILD +++ b/tensorflow/contrib/tpu/BUILD @@ -14,6 +14,7 @@ load("//tensorflow:tensorflow.bzl", "tf_py_test") package( default_visibility = [ "//cloud/vmm/testing/tests/tpu:__subpackages__", + "//knowledge/cerebra/sense/im2query:__subpackages__", "//learning/brain:__subpackages__", "//learning/deepmind:__subpackages__", "//medical/pathology:__subpackages__", @@ -215,7 +216,7 @@ py_library( ], deps = [ ":tpu_lib", - "//tensorflow/contrib/cluster_resolver:tpu_cluster_resolver_py", + "//tensorflow/contrib/cluster_resolver:cluster_resolver_py", "//tensorflow/contrib/distribute", "//tensorflow/contrib/framework:framework_py", "//tensorflow/contrib/tpu/proto:compilation_result_proto_py", @@ -263,7 +264,7 @@ py_library( ":tpu_py", "//tensorflow/compiler/xla/experimental/xla_sharding", "//tensorflow/compiler/xla/python_api:xla_shape", - "//tensorflow/contrib/cluster_resolver:tpu_cluster_resolver_py", + "//tensorflow/contrib/cluster_resolver:cluster_resolver_py", "//tensorflow/contrib/compiler:xla", "//tensorflow/contrib/tpu/proto:compilation_result_proto_py", "//tensorflow/contrib/tpu/proto:optimization_parameters_proto_py", diff --git a/tensorflow/contrib/tpu/profiler/pip_package/cloud_tpu_profiler/main.py b/tensorflow/contrib/tpu/profiler/pip_package/cloud_tpu_profiler/main.py index 63641e00c5d..a081c4354a7 100644 --- a/tensorflow/contrib/tpu/profiler/pip_package/cloud_tpu_profiler/main.py +++ b/tensorflow/contrib/tpu/profiler/pip_package/cloud_tpu_profiler/main.py @@ -90,12 +90,12 @@ def main(unused_argv=None): tf_version = tf.__version__ print('TensorFlow version %s detected' % tf_version) - if FLAGS.service_addr is None and FLAGS.tpu is None: + if not FLAGS.service_addr and not FLAGS.tpu: sys.exit('You must specify either --service_addr or --tpu.') tpu_cluster_resolver = None - if FLAGS.service_addr is not None: - if FLAGS.tpu is not None: + if FLAGS.service_addr: + if FLAGS.tpu: tf.logging.warn('Both --service_addr and --tpu are set. Ignoring ' '--tpu and using --service_addr.') service_addr = FLAGS.service_addr diff --git a/tensorflow/contrib/tpu/python/tpu/async_checkpoint.py b/tensorflow/contrib/tpu/python/tpu/async_checkpoint.py index 1cf7f9fcf67..1b09ce173a6 100644 --- a/tensorflow/contrib/tpu/python/tpu/async_checkpoint.py +++ b/tensorflow/contrib/tpu/python/tpu/async_checkpoint.py @@ -80,6 +80,8 @@ class AsyncCheckpointSaverHook(basic_session_run_hooks.CheckpointSaverHook): self._summary_writer = None self._global_step_tensor = None + self._last_checkpoint_step = None + def _set_steps_per_run(self, steps_per_run): self._steps_per_run = steps_per_run @@ -137,8 +139,7 @@ class AsyncCheckpointSaverHook(basic_session_run_hooks.CheckpointSaverHook): last_step = session.run(self._global_step_tensor) - # Save the last checkpoint synchronously if needed. - if last_step != self._timer.last_triggered_step(): + if self._last_checkpoint_step != last_step: self._save(session, last_step, asynchronous=False) for l in self._listeners: @@ -174,6 +175,7 @@ class AsyncCheckpointSaverHook(basic_session_run_hooks.CheckpointSaverHook): logging.info("Checkpoint finished for %d into %s.", step, self._save_path) if not asynchronous: + self._last_checkpoint_step = step _save_fn() return @@ -183,6 +185,7 @@ class AsyncCheckpointSaverHook(basic_session_run_hooks.CheckpointSaverHook): logging.info("Saver thread still in progress, skipping checkpoint.") return + self._last_checkpoint_step = step self._save_thread = threading.Thread(target=_save_fn) self._save_thread.start() diff --git a/tensorflow/contrib/tpu/python/tpu/datasets.py b/tensorflow/contrib/tpu/python/tpu/datasets.py index c694e9c1bca..8d6245390fc 100644 --- a/tensorflow/contrib/tpu/python/tpu/datasets.py +++ b/tensorflow/contrib/tpu/python/tpu/datasets.py @@ -133,7 +133,7 @@ def StreamingFilesDataset(files, with ops.device('/job:%s' % file_reader_job): if isinstance(files, str): source_dataset = dataset_ops.Dataset.list_files(files) - elif isinstance(files, dataset_ops.Dataset): + elif isinstance(files, dataset_ops.DatasetV2): source_dataset = files else: raise ValueError('files was not a string or a dataset: %s' % files) @@ -156,7 +156,7 @@ def StreamingFilesDataset(files, source_dataset = source_dataset.prefetch(1) - source_iterator = source_dataset.make_one_shot_iterator() + source_iterator = dataset_ops.make_one_shot_iterator(source_dataset) source_handle = source_iterator.string_handle() @function.Defun(dtypes.string) diff --git a/tensorflow/contrib/tpu/python/tpu/datasets_test.py b/tensorflow/contrib/tpu/python/tpu/datasets_test.py index b58d05eac56..52d87b80040 100644 --- a/tensorflow/contrib/tpu/python/tpu/datasets_test.py +++ b/tensorflow/contrib/tpu/python/tpu/datasets_test.py @@ -70,7 +70,7 @@ class DatasetsTest(test.TestCase): dataset = datasets.StreamingFilesDataset( os.path.join(self.get_temp_dir(), 'text_line.*.txt'), filetype='text') - iterator = dataset.make_initializable_iterator() + iterator = dataset_ops.make_initializable_iterator(dataset) self._sess.run(iterator.initializer) get_next = iterator.get_next() @@ -94,7 +94,7 @@ class DatasetsTest(test.TestCase): dataset = datasets.StreamingFilesDataset( os.path.join(self.get_temp_dir(), 'tf_record*'), filetype='tfrecord') - iterator = dataset.make_initializable_iterator() + iterator = dataset_ops.make_initializable_iterator(dataset) self._sess.run(iterator.initializer) get_next = iterator.get_next() @@ -121,7 +121,7 @@ class DatasetsTest(test.TestCase): dataset = datasets.StreamingFilesDataset(filenames, filetype='tfrecord') - iterator = dataset.make_initializable_iterator() + iterator = dataset_ops.make_initializable_iterator(dataset) self._sess.run(iterator.initializer) get_next = iterator.get_next() @@ -154,7 +154,7 @@ class DatasetsTest(test.TestCase): os.path.join(self.get_temp_dir(), 'fixed_length*'), filetype=FixedLengthFile) - iterator = dataset.make_initializable_iterator() + iterator = dataset_ops.make_initializable_iterator(dataset) self._sess.run(iterator.initializer) get_next = iterator.get_next() @@ -177,7 +177,7 @@ class DatasetsTest(test.TestCase): dataset = datasets.StreamingFilesDataset( dataset_ops.Dataset.range(10), filetype=gen_dataset) - iterator = dataset.make_initializable_iterator() + iterator = dataset_ops.make_initializable_iterator(dataset) self._sess.run(iterator.initializer) get_next = iterator.get_next() diff --git a/tensorflow/contrib/tpu/python/tpu/keras_support.py b/tensorflow/contrib/tpu/python/tpu/keras_support.py index 08f58a5f5b8..ebf40827e45 100644 --- a/tensorflow/contrib/tpu/python/tpu/keras_support.py +++ b/tensorflow/contrib/tpu/python/tpu/keras_support.py @@ -81,6 +81,7 @@ from tensorflow.python.keras import metrics as metrics_module from tensorflow.python.keras import models from tensorflow.python.keras import optimizers as keras_optimizers from tensorflow.python.keras.engine import base_layer +from tensorflow.python.keras.engine import base_layer_utils from tensorflow.python.keras.engine import training_arrays from tensorflow.python.keras.engine import training_utils from tensorflow.python.keras.layers import embeddings @@ -438,7 +439,7 @@ class TPURewriteContext(object): self._default_placeholder = array_ops.placeholder self._default_name_scope = ops.name_scope - self._default_make_variable = base_layer.make_variable + self._default_make_variable = base_layer_utils.make_variable self._default_random_normal = random_ops.random_normal self._default_qr = gen_linalg_ops.qr @@ -486,14 +487,14 @@ class TPURewriteContext(object): gen_linalg_ops.qr = qr ops.name_scope = _name_scope - base_layer.make_variable = variable_scope.get_variable + base_layer_utils.make_variable = variable_scope.get_variable logging.info('Overriding default placeholder.') return def __exit__(self, exc_type, exc_val, exc_tb): array_ops.placeholder = self._default_placeholder ops.name_scope = self._default_name_scope - base_layer.make_variable = self._default_make_variable + base_layer_utils.make_variable = self._default_make_variable random_ops.random_normal = self._default_random_normal gen_linalg_ops.qr = self._default_qr @@ -728,7 +729,7 @@ class TPUDatasetInfeedManager(TPUInfeedManager): dummy_x_shape[0] *= tpu_assignment.num_towers dummy_y_shape = dataset.output_shapes[1].as_list() dummy_y_shape[0] *= tpu_assignment.num_towers - self._iterator = dataset.make_initializable_iterator() + self._iterator = dataset_ops.make_initializable_iterator(dataset) K.get_session().run(self._iterator.initializer) self._get_next_ops = [] @@ -769,7 +770,7 @@ class TPUDatasetInfeedManager(TPUInfeedManager): def _verify_dataset_shape(self, dataset): """Verifies a dataset is of an appropriate shape for TPUs.""" - if not isinstance(dataset, dataset_ops.Dataset): + if not isinstance(dataset, dataset_ops.DatasetV2): raise ValueError('The function passed as the `x` parameter did not ' 'return a `tf.data.Dataset`.') if not isinstance(dataset.output_classes, tuple): @@ -1012,9 +1013,10 @@ class TPUFunction(object): optimizer=_replicated_optimizer(self._cloned_optimizer), loss=self.model.loss, loss_weights=self.model.loss_weights, - metrics=metrics_module.clone_metrics(self.model.metrics), + metrics=metrics_module.clone_metrics( + self.model._compile_metrics), weighted_metrics=metrics_module.clone_metrics( - self.model.weighted_metrics), + self.model._compile_weighted_metrics), target_tensors=tpu_targets, ) @@ -1184,12 +1186,9 @@ class TPUFunction(object): # pipelined loop. return None, None - if not isinstance(K.learning_phase(), int): + if isinstance(inputs[-1], int): # Remove the learning_phase flag at the end. We currently hard code the # learning_phase in TPUFunction. - assert isinstance(inputs[-1], int), ( - 'Expect the final element be learning_phase flag. Got {}'.format( - inputs[-1])) inputs = inputs[:-1] if (self.execution_mode == model_fn_lib.ModeKeys.TRAIN or @@ -1379,6 +1378,7 @@ class KerasTPUModel(models.Model): self.train_function = None self._fit_function = None self._eval_function = None + self._stateful_metric_functions = [] cluster_resolver = strategy._tpu_cluster_resolver self._tpu_name_or_address = cluster_resolver.get_master() @@ -1393,10 +1393,10 @@ class KerasTPUModel(models.Model): self.compile( self._cpu_model.optimizer, self._cpu_model.loss, - self._cpu_model.metrics, + self._cpu_model._compile_metrics, self._cpu_model.loss_weights, self._cpu_model.sample_weight_mode, - self._cpu_model.weighted_metrics, + self._cpu_model._compile_weighted_metrics, self._cpu_model.target_tensors, ) @@ -1466,7 +1466,7 @@ class KerasTPUModel(models.Model): assert not self._numpy_to_infeed_manager_list # Ensure empty. infeed_managers = [] # Managers to clean up at the end of the fit call. - if isinstance(x, dataset_ops.Dataset): + if isinstance(x, dataset_ops.DatasetV2): # TODO(b/111413240): Support taking a tf.data.Dataset directly. raise ValueError( 'Taking a Dataset directly is not yet supported. Please ' @@ -1492,7 +1492,7 @@ class KerasTPUModel(models.Model): y = infeed_manager.dummy_y infeed_managers.append((x, infeed_manager)) - if isinstance(validation_data, dataset_ops.Dataset): + if isinstance(validation_data, dataset_ops.DatasetV2): # TODO(b/111413240): Support taking a tf.data.Dataset directly. raise ValueError( 'Taking a Dataset directly is not yet supported. Please ' @@ -1551,7 +1551,7 @@ class KerasTPUModel(models.Model): with _tpu_session_context(): # Managers to clean up at the end of the evaluate call. infeed_managers = [] - if isinstance(x, dataset_ops.Dataset): + if isinstance(x, dataset_ops.DatasetV2): # TODO(b/111413240): Support taking a tf.data.Dataset directly. raise ValueError( 'Taking a Dataset directly is not yet supported. Please ' @@ -1676,14 +1676,10 @@ class KerasTPUModel(models.Model): callbacks, self, do_validation=do_validation, - val_inputs=val_inputs, - val_targets=val_targets, - val_sample_weights=val_sample_weights, batch_size=batch_size, epochs=epochs, steps_per_epoch=steps_per_epoch, samples=num_training_samples, - validation_steps=validation_steps, verbose=verbose, count_mode=count_mode) @@ -1700,7 +1696,7 @@ class KerasTPUModel(models.Model): callbacks.on_train_begin() for epoch in range(initial_epoch, epochs): # Reset stateful metrics - for m in self.stateful_metric_functions: + for m in self.metrics: m.reset_states() # Update callbacks callbacks.on_epoch_begin(epoch) @@ -1923,7 +1919,7 @@ class KerasTPUModel(models.Model): if validation_data: if (isinstance(validation_data, iterator_ops.Iterator) or isinstance(validation_data, iterator_ops.EagerIterator) or - isinstance(validation_data, dataset_ops.Dataset)): + isinstance(validation_data, dataset_ops.DatasetV2)): raise ValueError('KerasTPUModel cannot handle a Dataset or Iterator ' 'for validation_data. Please instead pass a function ' 'that returns a `tf.data.Dataset`.') @@ -1998,14 +1994,14 @@ class KerasTPUModel(models.Model): self._optimizer = optimizer @property - def stateful_metric_functions(self): + def metrics(self): if self._tpu_model: - return self._tpu_model.stateful_metric_functions + return self._tpu_model.metrics return self._stateful_metric_functions - @stateful_metric_functions.setter - def stateful_metric_functions(self, stateful_metric_functions): - self._stateful_metric_functions = stateful_metric_functions + @metrics.setter + def metrics(self, metrics): + self._stateful_metric_functions = metrics def _make_train_function(self): if not self.train_function: @@ -2230,10 +2226,10 @@ def tpu_model(model, strategy=None): cpu_model.compile( _clone_optimizer(model.optimizer, optimizer_config), model.loss, - metrics_module.clone_metrics(model.metrics), + metrics_module.clone_metrics(model._compile_metrics), model.loss_weights, model.sample_weight_mode, - metrics_module.clone_metrics(model.weighted_metrics), + metrics_module.clone_metrics(model._compile_weighted_metrics), ) if model_weights: diff --git a/tensorflow/contrib/tpu/python/tpu/keras_tpu_variables.py b/tensorflow/contrib/tpu/python/tpu/keras_tpu_variables.py index 28d3a938510..8b0b240dc73 100644 --- a/tensorflow/contrib/tpu/python/tpu/keras_tpu_variables.py +++ b/tensorflow/contrib/tpu/python/tpu/keras_tpu_variables.py @@ -217,6 +217,10 @@ class ReplicatedVariable(object): def get(self): return self._primary_var + @property + def _in_graph_mode(self): + return self._primary_var._in_graph_mode # pylint: disable=protected-access + def _should_act_as_resource_variable(self): """Pass resource_variable_ops.is_resource_variable check.""" pass diff --git a/tensorflow/contrib/tpu/python/tpu/tpu.py b/tensorflow/contrib/tpu/python/tpu/tpu.py index e3e791faacb..def57da20d6 100644 --- a/tensorflow/contrib/tpu/python/tpu/tpu.py +++ b/tensorflow/contrib/tpu/python/tpu/tpu.py @@ -1001,8 +1001,8 @@ def rewrite(computation, `rewrite` is a list of tensors corresponding to the tensors from the output of `computation`. - All `Operation`s returned from `computation` will be executed when - evaluating any of the returned output tensors. + All `Operation`s constructed during `computation` will be executed when + evaluating any of the returned output tensors, not just the ones returned. inputs: A list of input tensors or `None` (equivalent to an empty list). infeed_queue: If not `None`, the `InfeedQueue` from which to append a tuple of arguments as inputs to `computation`. @@ -1111,7 +1111,7 @@ def validate_inference_rewrite_for_variables(graph): Raises: RuntimeError: if validation failed. """ - if not any([x.type == "GuaranteeConst" for x in graph.get_operations()]): + if not any(x.type == "GuaranteeConst" for x in graph.get_operations()): raise RuntimeError( "No GuaranteeConst ops found in the graph after running " "tpu.rewrite_for_inference(...). Please check that you are using " diff --git a/tensorflow/contrib/tpu/python/tpu/tpu_context.py b/tensorflow/contrib/tpu/python/tpu/tpu_context.py index da6bdf67d68..67246244794 100644 --- a/tensorflow/contrib/tpu/python/tpu/tpu_context.py +++ b/tensorflow/contrib/tpu/python/tpu/tpu_context.py @@ -41,7 +41,7 @@ _NUM_CORES_TO_COMPUTATION_SHAPE = { class TPUContext(object): - """The context of current input_fn invocation.""" + """A context that holds the current configuration of the TPU computation.""" def __init__(self, internal_ctx, diff --git a/tensorflow/contrib/tpu/python/tpu/tpu_embedding.py b/tensorflow/contrib/tpu/python/tpu/tpu_embedding.py index 3fe896426a7..ccba8a46c7c 100644 --- a/tensorflow/contrib/tpu/python/tpu/tpu_embedding.py +++ b/tensorflow/contrib/tpu/python/tpu/tpu_embedding.py @@ -1069,17 +1069,14 @@ def _create_partitioned_variables(name, 'As TPU embedding is not optimized for small tables, ' 'please consider other ways for this embedding lookup.') - slicing = [num_hosts, 1] - - # TODO(shizhiw): deprecated, use tf.get_variable()? - return partitioned_variables.create_partitioned_variables( - name=name, - slicing=slicing, + return list(variable_scope.get_variable( + name, shape=(vocabulary_size, embedding_dimension), + partitioner=partitioned_variables.fixed_size_partitioner(num_hosts), dtype=dtypes.float32, initializer=initializer, collections=collections, - trainable=False) + trainable=False)) @ops.RegisterGradient('TPUEmbeddingActivations') diff --git a/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py b/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py index 7cb8c4aa7f1..a9dc542ae5e 100644 --- a/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py +++ b/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py @@ -298,9 +298,9 @@ class TPUEstimatorSpec(model_fn_lib._TPUEstimatorSpec): # pylint: disable=prote host_calls['host_call'] = host_call _OutfeedHostCall.validate(host_calls) - training_hooks = list(training_hooks or []) - evaluation_hooks = list(evaluation_hooks or []) - prediction_hooks = list(prediction_hooks or []) + training_hooks = tuple(training_hooks or []) + evaluation_hooks = tuple(evaluation_hooks or []) + prediction_hooks = tuple(prediction_hooks or []) for hook in training_hooks + evaluation_hooks + prediction_hooks: if not isinstance(hook, session_run_hook.SessionRunHook): @@ -335,7 +335,7 @@ class TPUEstimatorSpec(model_fn_lib._TPUEstimatorSpec): # pylint: disable=prote hooks = None if self.host_call is not None: hooks = [_OutfeedHostCallHook(host_call_ret['host_call'])] - hooks = list(hooks or []) + hooks = tuple(hooks or []) scaffold = self.scaffold_fn() if self.scaffold_fn else None return model_fn_lib.EstimatorSpec( mode=self.mode, @@ -2169,7 +2169,6 @@ class TPUEstimator(estimator_lib.Estimator): builder, input_receiver_fn_map, checkpoint_path, - strip_default_attrs, save_variables=True, mode=model_fn_lib.ModeKeys.PREDICT, export_tags=None, @@ -2184,7 +2183,6 @@ class TPUEstimator(estimator_lib.Estimator): builder, input_receiver_fn_map, checkpoint_path, - strip_default_attrs, save_variables, mode=mode, export_tags=export_tags, @@ -2201,7 +2199,6 @@ class TPUEstimator(estimator_lib.Estimator): builder, input_receiver_fn_map, checkpoint_path, - strip_default_attrs, save_variables=False, mode=mode, export_tags=export_tags, @@ -2783,7 +2780,7 @@ def _export_output_to_tensors(export_output): elif isinstance(export_output, export_output_lib.RegressionOutput): return [export_output.value] elif isinstance(export_output, export_output_lib.PredictOutput): - return export_output.outputs.values() + return list(export_output.outputs.values()) else: raise ValueError( '`export_output` must be have type `ClassificationOutput`, ' @@ -3059,7 +3056,7 @@ class _Inputs(object): @staticmethod def from_input_fn(return_values): """Returns an `_Inputs` instance according to `input_fn` return value.""" - if isinstance(return_values, dataset_ops.Dataset): + if isinstance(return_values, dataset_ops.DatasetV2): dataset = return_values return _Inputs(dataset=dataset) @@ -3084,7 +3081,7 @@ class _Inputs(object): The initializer must be run before calling `features_and_labels`. """ - self._iterator = self._dataset.make_initializable_iterator() + self._iterator = dataset_ops.make_initializable_iterator(self._dataset) return self._iterator.initializer def features_and_labels(self): diff --git a/tensorflow/contrib/tpu/python/tpu/tpu_estimator_signals_test.py b/tensorflow/contrib/tpu/python/tpu/tpu_estimator_signals_test.py index 3786e52b949..55235556de0 100644 --- a/tensorflow/contrib/tpu/python/tpu/tpu_estimator_signals_test.py +++ b/tensorflow/contrib/tpu/python/tpu/tpu_estimator_signals_test.py @@ -71,7 +71,7 @@ class TPUEstimatorStoppingSignalsTest(test.TestCase): with ops.Graph().as_default(): dataset = input_fn(params) - features = dataset.make_one_shot_iterator().get_next() + features = dataset_lib.make_one_shot_iterator(dataset).get_next() # With tf.data.Dataset.batch, the batch is None, i.e., dynamic shape. self.assertIsNone(features['a'].shape.as_list()[0]) diff --git a/tensorflow/contrib/tpu/python/tpu/tpu_feed.py b/tensorflow/contrib/tpu/python/tpu/tpu_feed.py index e75a09492ec..d5957b7e8ec 100644 --- a/tensorflow/contrib/tpu/python/tpu/tpu_feed.py +++ b/tensorflow/contrib/tpu/python/tpu/tpu_feed.py @@ -26,7 +26,6 @@ import numpy as np from six.moves import xrange # pylint: disable=redefined-builtin from tensorflow.compiler.xla.experimental.xla_sharding import xla_sharding -from tensorflow.compiler.xla.python_api import xla_shape from tensorflow.contrib.tpu.python.ops import tpu_ops from tensorflow.contrib.tpu.python.tpu import tpu from tensorflow.contrib.tpu.python.tpu import tpu_sharding @@ -92,8 +91,7 @@ class InfeedQueue(object): else: raise ValueError( "number of tuple elements cannot be inferred from InfeedQueue " - "constructor" - ) + "constructor") if number_of_tuple_elements <= 0: raise ValueError("number_of_tuple_elements %d must be > 0" % number_of_tuple_elements) @@ -293,9 +291,8 @@ class InfeedQueue(object): self.number_of_tuple_elements """ if len(input_tensors) != self.number_of_tuple_elements: - raise ValueError( - "input_tensors is %s, but should be a list of %d Tensors", ( - str(input_tensors), self.number_of_tuple_elements)) + raise ValueError("input_tensors is %s, but should be a list of %d Tensors" + % (str(input_tensors), self.number_of_tuple_elements)) self.set_tuple_shapes([t.shape for t in input_tensors]) self.set_tuple_types([t.dtype for t in input_tensors]) @@ -451,8 +448,8 @@ class InfeedQueue(object): for i in xrange(1, self.number_of_tuple_elements): if devices[0] != devices[i]: raise ValueError( - "input devices for shard %d are %s, but should all be the same", - index, str(devices)) + "input devices for shard %d are %s, but should all be the same" % + (index, str(devices))) with ops.colocate_with(inputs[0]): return tpu_ops.infeed_enqueue_tuple( inputs=inputs, @@ -792,18 +789,14 @@ class _PartitionedInfeedQueue(InfeedQueue): Args: tensor: Input tensor for partitioning. - dims: A list of integer describes how to partition the input tensor. + dims: 1-D np.array of the list of integer describes how to partition the + input tensor. Raises: ValueError: If the tensor can't be partitioned by dims or the num_cores_per_replica doesn't match the number of partitions(dims.prod()). """ - if dims is None: - return - - dims = np.array(dims) - if (dims < 1).any(): raise ValueError("All input partition dims must be >= 1.") @@ -823,11 +816,6 @@ class _PartitionedInfeedQueue(InfeedQueue): "partition dims = {}).".format(tensor.shape.as_list(), dims)) tensor.shape.assert_is_fully_defined() - if (np.array(tensor.shape.as_list()) % dims != 0).any(): - raise ValueError( - "All input partition dims must divide exactly into the `Tensor` " - "shape (tensor shape = {}, input partition dims = {}).".format( - tensor.shape.as_list(), dims)) def _partition_or_replicate_on_host(self, tensor, dims): """Partitions or replicates the input tensor. @@ -840,16 +828,39 @@ class _PartitionedInfeedQueue(InfeedQueue): Returns: An iterator of `Tensor`s or a list of partioned tensors. """ - self._check_input_partition_dims(tensor, dims) if dims is None: return itertools.repeat(tensor) - else: - output = [tensor] - for axis, dim in enumerate(dims): - if dim > 1: - output = [array_ops.split(x, dim, axis=axis) for x in output] - output = nest.flatten(output) - return output + dims = np.array(dims) + self._check_input_partition_dims(tensor, dims) + output = [tensor] + shape_list = np.array(tensor.shape.as_list()) + quotients, remainders = np.divmod(shape_list, dims) + for axis, (quotient, remainder, dim, original_size) in enumerate( + zip(quotients, remainders, dims, shape_list)): + if dim <= 1: + continue + if remainder > 0: + # For each dimension, when it cannot be evenly partitioned, XLA assumes + # tensors are partitioned in a greedy manner by using + # ceil_ratio(size/dim) first. E.g. 2D tensor with shape (5, 14) and dims + # are (2, 4). Since 5 % 2 = 1 and 14 % 4 = 2, [5, 14] => + # [[(3, 4), (3, 4), (2, 4), (2, 2)], + # [(2, 4), (2, 4), (2, 4), (2, 2)]] + ceil_ratio = quotient + 1 + num_full_slots, left_over = np.divmod(original_size, ceil_ratio) + num_or_size_splits = [ceil_ratio] * num_full_slots + [left_over] + if len(num_or_size_splits) < dim: + num_or_size_splits += [0] * (dim - len(num_or_size_splits)) + new_output = [] + for x in output: + new_output.append( + array_ops.split( + x, num_or_size_splits=num_or_size_splits, axis=axis)) + output = new_output + else: + output = [array_ops.split(x, dim, axis=axis) for x in output] + output = nest.flatten(output) + return output def _tag_sharding_attribute_for_dequeued_tensor(self, tensor, dims): """Tags appropriate XLA sharding attribute to the dequeued tensor. @@ -866,13 +877,9 @@ class _PartitionedInfeedQueue(InfeedQueue): elif np.prod(dims) == 1: return xla_sharding.assign_device(tensor, 0) else: - tile_shape = np.array(tensor.shape.as_list()) // dims tile_assignment = np.arange(np.prod(dims)).reshape(dims) return xla_sharding.tile( tensor=tensor, - tile_shape=xla_shape.CreateShapeFromDtypeAndTuple( - dtype=np.dtype(tensor.dtype.as_numpy_dtype), - shape_tuple=tile_shape), tile_assignment=tile_assignment) def _tag_sharding_attribute_for_dequeued_tensors(self, dequeues, dims): diff --git a/tensorflow/contrib/tpu/python/tpu/training_loop.py b/tensorflow/contrib/tpu/python/tpu/training_loop.py index b6c350ecd75..0187b4bec6e 100644 --- a/tensorflow/contrib/tpu/python/tpu/training_loop.py +++ b/tensorflow/contrib/tpu/python/tpu/training_loop.py @@ -166,8 +166,8 @@ def while_loop(condition, body, inputs=None, infeed_queue=None, name=None): # control dependencies from any side-effecting operations. if input_arity == 0: inputs = [array_ops.constant(0)] - return control_flow_ops.while_loop(condition_wrapper, body_wrapper, inputs, - name="") + return control_flow_ops.while_loop( + condition_wrapper, body_wrapper, inputs, name="", parallel_iterations=1) def repeat(n, body, inputs=None, infeed_queue=None, name=None): diff --git a/tensorflow/contrib/tpu/tpu_estimator.md b/tensorflow/contrib/tpu/tpu_estimator.md index b6514e19dc9..552febd80bd 100644 --- a/tensorflow/contrib/tpu/tpu_estimator.md +++ b/tensorflow/contrib/tpu/tpu_estimator.md @@ -89,12 +89,9 @@ handle training: dataset = tf.data.TFRecordDataset( filename, buffer_size=FLAGS.dataset_reader_buffer_size) - dataset = dataset.map(parser).cache().repeat().batch(batch_size) - images, labels = dataset.make_one_shot_iterator().get_next() - # set_shape to give inputs statically known shapes. - images.set_shape([batch_size, 28 * 28]) - labels.set_shape([batch_size]) - return images, labels + dataset = dataset.map(parser).cache().repeat().batch( + batch_size, drop_remainder=True) + return dataset return input_fn diff --git a/tensorflow/contrib/training/BUILD b/tensorflow/contrib/training/BUILD index 00295f57f60..f6427ae05a2 100644 --- a/tensorflow/contrib/training/BUILD +++ b/tensorflow/contrib/training/BUILD @@ -26,7 +26,6 @@ py_library( "python/training/resample.py", "python/training/sampling_ops.py", "python/training/sequence_queueing_state_saver.py", - "python/training/tensor_queue_dataset.py", "python/training/training.py", "python/training/tuner.py", ], @@ -287,28 +286,6 @@ py_test( ], ) -py_test( - name = "tensor_queue_dataset_test", - size = "large", - srcs = ["python/training/tensor_queue_dataset_test.py"], - srcs_version = "PY2AND3", - tags = ["notsan"], - deps = [ - ":training_py", - "//tensorflow/python:client_testlib", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:gradients", - "//tensorflow/python:math_ops", - "//tensorflow/python:platform", - "//tensorflow/python:random_seed", - "//tensorflow/python:training", - "//tensorflow/python:variables", - "//tensorflow/python/data", - "//tensorflow/python/data/experimental/kernel_tests/serialization:dataset_serialization_test_base", - "//third_party/py/numpy", - ], -) - tf_proto_library( name = "protos_all", srcs = glob(["**/*.proto"]), diff --git a/tensorflow/contrib/training/__init__.py b/tensorflow/contrib/training/__init__.py index 3547e71184e..87ce57ef060 100644 --- a/tensorflow/contrib/training/__init__.py +++ b/tensorflow/contrib/training/__init__.py @@ -59,8 +59,6 @@ from tensorflow.contrib.training.python.training.hparam import * from tensorflow.contrib.training.python.training.resample import * from tensorflow.contrib.training.python.training.sampling_ops import * from tensorflow.contrib.training.python.training.sequence_queueing_state_saver import * -from tensorflow.contrib.training.python.training.tensor_queue_dataset import enqueue_in_queue_dataset -from tensorflow.contrib.training.python.training.tensor_queue_dataset import prepend_from_queue_and_padded_batch_dataset from tensorflow.contrib.training.python.training.training import add_gradients_summaries from tensorflow.contrib.training.python.training.training import clip_gradient_norms from tensorflow.contrib.training.python.training.training import clip_gradient_norms_fn @@ -79,7 +77,6 @@ _allowed_symbols = [ 'FeedingQueueRunner', 'get_or_create_eval_step', 'StopAfterNEvalsHook', 'SummaryAtEndHook', 'wait_for_new_checkpoint', 'add_gradients_summaries', 'clip_gradient_norms', 'clip_gradient_norms_fn', 'create_train_op', - 'multiply_gradients', 'enqueue_in_queue_dataset', - 'prepend_from_queue_and_padded_batch_dataset', 'train'] + 'multiply_gradients', 'train'] remove_undocumented(__name__, _allowed_symbols) diff --git a/tensorflow/contrib/training/python/training/tensor_queue_dataset.py b/tensorflow/contrib/training/python/training/tensor_queue_dataset.py deleted file mode 100644 index 8896a95327a..00000000000 --- a/tensorflow/contrib/training/python/training/tensor_queue_dataset.py +++ /dev/null @@ -1,201 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Python wrappers for Datasets and Iterators.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.python.data.ops import dataset_ops -from tensorflow.python.data.util import convert -from tensorflow.python.data.util import nest -from tensorflow.python.data.util import sparse -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.framework import tensor_shape -from tensorflow.python.framework import tensor_util -from tensorflow.python.ops import gen_dataset_ops -from tensorflow.python.util import nest as tf_nest - - -class _PrependFromQueueAndPaddedBatchDataset(dataset_ops.UnaryDataset): - """A `Dataset` that prepends a queue to another `Dataset`. - - A vector of handles to the queue is returned as the first component of - the associated iterator. This vector can be passed to - `enqueue_in_queue_dataset` to add new elements to the queue. - """ - - def __init__(self, input_dataset, batch_size, padded_shapes, padding_values): - """Initialize `PrependFromQueueAndPaddedBatchDataset`.""" - super(_PrependFromQueueAndPaddedBatchDataset, self).__init__(input_dataset) - if sparse.any_sparse(input_dataset.output_classes): - raise TypeError( - "Batching of padded sparse tensors is not currently supported") - self._input_dataset = input_dataset - self._batch_size = ops.convert_to_tensor( - batch_size, dtype=dtypes.int64, name="batch_size") - if padded_shapes is None: - self._padded_shapes = nest.map_structure( - convert.partial_shape_to_tensor, input_dataset.output_shapes) - else: - self._padded_shapes = nest.map_structure_up_to( - input_dataset.output_shapes, convert.partial_shape_to_tensor, - padded_shapes) - # pylint: disable=protected-access - padding_values = ( - padding_values if padding_values is not None else - dataset_ops._default_padding(input_dataset)) - self._padding_values = nest.map_structure_up_to( - input_dataset.output_shapes, dataset_ops._padding_value_to_tensor, - padding_values, input_dataset.output_types) - # pylint: enable=protected-access - - def _as_variant_tensor(self): - # pylint: disable=protected-access - return gen_dataset_ops.prepend_from_queue_and_padded_batch_dataset( - self._input_dataset._as_variant_tensor(), - batch_size=self._batch_size, - padded_shapes=[ - ops.convert_to_tensor(s, dtype=dtypes.int64) - for s in nest.flatten(self._padded_shapes) - ], - padding_values=nest.flatten(self._padding_values), - output_shapes=nest.flatten( - sparse.as_dense_shapes(self.output_shapes, self.output_classes))) - # pylint: enable=protected-access - - @property - def output_classes(self): - return (ops.Tensor, self._input_dataset.output_classes) - - def _as_batch_shape(self, shape_like): - return tensor_shape.vector(None).concatenate( - tensor_util.constant_value_as_shape(shape_like)) - - @property - def output_shapes(self): - # First output is a variant representing the Queue - return (tensor_shape.vector(None), - nest.map_structure(self._as_batch_shape, self._padded_shapes)) - - @property - def output_types(self): - # First output is a variant representing the Queue - return (dtypes.variant, self._input_dataset.output_types) - - -def prepend_from_queue_and_padded_batch_dataset(batch_size, - padding_values=None, - padded_shapes=None): - """A transformation that prepends a queue to a `Dataset` and batches results. - - A vector of handles to the queue is returned as the first component of the - associated iterator. This vector can be passed to `enqueue_in_queue_dataset` - to add new elements to the queue. - - Below is an example of how this dataset might be used to split incoming - variable-length sequences into "head" and "rest" parts, where "rest" parts - are re-enqueued back into the dataset. A more realistic example would - perform some calculation on the "head" and modify some components of "rest" - with the result (before re-enqueueing). - - ```python - dataset = tf.data.Dataset.from_tensor_slices([2*x for x in range(10)]) - # Make a dataset of variable-length vectors and their lengths. - dataset = dataset.map(lambda count: (count, tf.ones((count,)))) - # Emit a queue we can prepend to, and counts/values as padded batch. - dataset = dataset.apply( - tf.contrib.training.prepend_from_queue_and_padded_batch_dataset( - batch_size=10)) - dataset = dataset.prefetch(1) - - iterator = dataset.make_one_shot_iterator() - queue, (count, padded_value) = iterator.get_next() - - # Split the padded_value into two pieces: head and rest - rest_indices = tf.squeeze(tf.where(count > 3), axis=1) - bound = tf.minimum(3, tf.reduce_max(count)) - value_head = padded_value[:, :bound] - count_rest = tf.gather(count - 3, rest_indices) - value_rest = tf.gather(padded_value[:, bound:], rest_indices) - queue_rest = tf.gather(queue, rest_indices) - enqueue_rest_op = tf.contrib.training.enqueue_in_queue_dataset( - queue_rest, (count_rest, value_rest)) - with tf.control_dependencies([enqueue_rest_op]): - calculation = fn(value_head) - - while True: # Will raise OutOfRange when finished with all pieces. - session.run(calculation) - ``` - - Args: - batch_size: `int64` scalar tensor. The batch size to use when performing - padded batching. - padding_values: (optional) Nested tuple of scalar tensors. If provided, - the structure and dtypes of padding_values should match that of - incoming dataset's `output_types`. - padded_shapes: (optional) Nested tuple of `int64` vector tensors. - If provided, the structure must match that of the incoming dataset's - `output_types`. If not provided, the incoming dataset's `output_shapes` - is used. Any unknown (`None` or `-1`) dimensions in the shapes are - treated as being unique per-batch: for each batch time, an unknown - dimension is replaced with the maximum given value of this dimension - across all tensors for the given component in the batch. - - Returns: - A `Dataset` transformation function, which can be passed to - `tf.data.Dataset.apply`. - """ - - def _apply_fn(dataset): - return _PrependFromQueueAndPaddedBatchDataset( - dataset, - batch_size=batch_size, - padding_values=padding_values, - padded_shapes=padded_shapes) - - return _apply_fn - - -def enqueue_in_queue_dataset(queue, components): - """Enqueue components into queue from `PrependFromQueueAndPaddedBatchDataset`. - - The components' dtypes and shapes must be compatible with the `output_shapes` - attribute of the `dataset` created by - `prepend_from_queue_and_padded_batch_dataset`. This operation supports both - non-batched and batched modes. - - For more details, see the example in the docstring for - `prepend_from_queue_and_padded_batch_dataset`. - - Args: - queue: `variant` scalar or vector tensor. - The tensor emitted by the first component of the iterator associated with - `prepend_from_queue_and_padded_batch_dataset`. If this is a scalar, - then the `components` input tensors should not have a prepended batch - dimension. - components: Nested tuple of tensors, each with a leading batch dimension - if `queue` is a vector. The structure, dtypes, and shapes - (excluding batch dimension) must match the nested tuples - `dataset.output_types[1]` and `dataset.output_shapes[1]` (the non-queue - output types and shapes) of the `dataset` emitted by - the original `prepend_from_queue_and_padded_batch_dataset` call. - - Returns: - An `Operation` that enqueues `components` into the dataset(s) associated - with entries of `queue`. - """ - return gen_dataset_ops.enqueue_in_queue_dataset( - queue=queue, components=tf_nest.flatten(components)) diff --git a/tensorflow/contrib/training/python/training/tensor_queue_dataset_test.py b/tensorflow/contrib/training/python/training/tensor_queue_dataset_test.py deleted file mode 100644 index c1657fec7bb..00000000000 --- a/tensorflow/contrib/training/python/training/tensor_queue_dataset_test.py +++ /dev/null @@ -1,355 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for TensorQueueDataset.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np - -from tensorflow.contrib.training.python.training import tensor_queue_dataset as tqd -from tensorflow.python.data.experimental.kernel_tests.serialization import dataset_serialization_test_base -from tensorflow.python.data.ops import dataset_ops -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import errors -from tensorflow.python.framework import ops -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import string_ops -from tensorflow.python.platform import test - - -class PrependFromQueueAndPaddedBatchDatasetTest(test.TestCase): - - def testNoEnqueue(self): - dataset = dataset_ops.Dataset.from_tensor_slices([0, 1, 2]) - dataset = dataset.apply( - tqd.prepend_from_queue_and_padded_batch_dataset(batch_size=1)) - self.assertEqual((dtypes.variant, dtypes.int32), dataset.output_types) - self.assertAllEqual(([None],) * 2, - [x.as_list() for x in dataset.output_shapes]) - iterator = dataset.make_one_shot_iterator() - _, value = iterator.get_next() - self.assertEqual([0], self.evaluate(value)) - self.assertEqual([1], self.evaluate(value)) - self.assertEqual([2], self.evaluate(value)) - with self.assertRaisesOpError("End of sequence"): - self.evaluate(value) - - def testBatchedNoEnqueue(self): - dataset = dataset_ops.Dataset.from_tensor_slices([0, 1, 2]) - dataset = dataset.apply( - tqd.prepend_from_queue_and_padded_batch_dataset(batch_size=2)) - iterator = dataset.make_one_shot_iterator() - _, value = iterator.get_next() - self.assertAllEqual([0, 1], self.evaluate(value)) - self.assertAllEqual([2], self.evaluate(value)) - with self.assertRaisesOpError("End of sequence"): - self.evaluate(value) - - def testBatchedWithBiggerPaddingNoEnqueue(self): - dataset = dataset_ops.Dataset.from_tensor_slices([[0], [1], [2]]) - dataset = dataset.apply( - tqd.prepend_from_queue_and_padded_batch_dataset( - batch_size=2, padded_shapes=[3])) - iterator = dataset.make_one_shot_iterator() - _, value = iterator.get_next() - self.assertAllEqual([[0, 0, 0], [1, 0, 0]], self.evaluate(value)) - self.assertAllEqual([[2, 0, 0]], self.evaluate(value)) - with self.assertRaisesOpError("End of sequence"): - self.evaluate(value) - - def testBatchedWithBiggerPaddingOneEnqueue(self): - dataset = dataset_ops.Dataset.from_tensor_slices([[0], [1], [2]]) - dataset = dataset.apply( - tqd.prepend_from_queue_and_padded_batch_dataset( - batch_size=1, padded_shapes=[3])) - iterator = dataset.make_one_shot_iterator() - queue_handle, value = iterator.get_next() - enqueue_negative = tqd.enqueue_in_queue_dataset(queue_handle, -value) - with self.cached_session() as sess: - self.assertAllEqual([[0, 0, 0]], sess.run(value)) - value_1, _ = sess.run([value, enqueue_negative]) - self.assertAllEqual([[1, 0, 0]], value_1) - value_2, _ = sess.run([value, enqueue_negative]) - self.assertAllEqual([[-1, 0, 0]], value_2) - value_3 = sess.run(value) - self.assertAllEqual([[1, 0, 0]], value_3) - value_4, _ = sess.run([value, enqueue_negative]) - self.assertAllEqual([[2, 0, 0]], value_4) - value_5 = sess.run(value) - self.assertAllEqual([[-2, 0, 0]], value_5) - with self.assertRaisesOpError("End of sequence"): - sess.run(value) - - def testOneEnqueue(self): - dataset = dataset_ops.Dataset.from_tensor_slices([0, 1, 2]) - dataset = dataset.apply( - tqd.prepend_from_queue_and_padded_batch_dataset(batch_size=1)) - iterator = dataset.make_one_shot_iterator() - queue_handle, value = iterator.get_next() - enqueue_negative = tqd.enqueue_in_queue_dataset(queue_handle, -value) - with self.cached_session() as sess: - self.assertEqual([0], sess.run(value)) - value_1, _ = sess.run([value, enqueue_negative]) - self.assertEqual([1], value_1) - value_2, _ = sess.run([value, enqueue_negative]) - self.assertEqual([-1], value_2) - value_3 = sess.run(value) - self.assertEqual([1], value_3) - value_4, _ = sess.run([value, enqueue_negative]) - self.assertEqual([2], value_4) - value_5 = sess.run(value) - self.assertEqual([-2], value_5) - with self.assertRaisesOpError("End of sequence"): - sess.run(value) - - def testBatchedOneEnqueue(self): - dataset = dataset_ops.Dataset.from_tensor_slices([0, 1, 2]) - dataset = dataset.apply( - tqd.prepend_from_queue_and_padded_batch_dataset(batch_size=2)) - iterator = dataset.make_one_shot_iterator() - queue_handle, value = iterator.get_next() - enqueue_negative = tqd.enqueue_in_queue_dataset(queue_handle, -value) - enqueue_zeroth = tqd.enqueue_in_queue_dataset([queue_handle[0]], - array_ops.expand_dims( - value[0], axis=0)) - with self.cached_session() as sess: - value_0, _ = sess.run([value, enqueue_negative]) - self.assertAllEqual([0, 1], value_0) - value_1, _ = sess.run([value, enqueue_zeroth]) - self.assertAllEqual([0, -1], value_1) - value_2, _ = sess.run([value, enqueue_negative]) - self.assertAllEqual([0, 2], value_2) - self.assertAllEqual([0, -2], sess.run(value)) - with self.assertRaisesOpError("End of sequence"): - sess.run(value) - - def testManyEnqueue(self): - dataset = dataset_ops.Dataset.from_tensor_slices([0, 1]) - dataset = dataset.apply( - tqd.prepend_from_queue_and_padded_batch_dataset(batch_size=1)) - iterator = dataset.make_one_shot_iterator() - queue_handle, value = iterator.get_next() - enqueue_many_more = [ - tqd.enqueue_in_queue_dataset(queue_handle, value + 100 + i) - for i in range(1000) - ] - with self.cached_session() as sess: - value_0, _ = sess.run((value, enqueue_many_more)) - self.assertEqual([0], value_0) - rest = [] - for _ in range(1000): - rest.append(sess.run(value)) - self.assertEquals([[100 + i] for i in range(1000)], sorted(rest)) - # Going back to the original input. - value_1, _ = sess.run((value, enqueue_many_more)) - self.assertEqual(1, value_1) - rest = [] - for _ in range(1000): - rest.append(sess.run(value)) - self.assertEquals([[100 + i + 1] for i in range(1000)], sorted(rest)) - with self.assertRaisesOpError("End of sequence"): - sess.run(value) - - def testEnqueueWithPrefetch(self): - dataset = dataset_ops.Dataset.from_tensor_slices([0]) - dataset = dataset.apply( - tqd.prepend_from_queue_and_padded_batch_dataset(batch_size=1)) - # Prefetching will request additional values before they are - # available to the queue. - dataset = dataset.prefetch(buffer_size=3) - iterator = dataset.make_one_shot_iterator() - queue_handle, value = iterator.get_next() - enqueue = tqd.enqueue_in_queue_dataset(queue_handle, value + 1) - with self.cached_session() as sess: - i = 0 - while i < 4: - received, _ = sess.run((value, enqueue)) - if received.size > 0: - self.assertAllEqual([i], received) - i += 1 - received_last = False - while True: - try: - received = sess.run(value) - if received.size > 0: - self.assertAllEqual([4], received) - received_last = True - except errors.OutOfRangeError: - break - self.assertTrue(received_last) - - def testDatasetWithPaddedShapeSmallerThanInputFails(self): - dataset = dataset_ops.Dataset.from_tensor_slices([[0, 0, 0]]).repeat(None) - dataset = dataset.apply( - tqd.prepend_from_queue_and_padded_batch_dataset( - batch_size=1, padded_shapes=[2])) - iterator = dataset.make_one_shot_iterator() - _, value = iterator.get_next() - with self.cached_session() as sess: - with self.assertRaisesOpError( - r"Incompatible input shapes at component 0 between " - r"input dataset this dataset: \[3\] vs. \[2\]"): - sess.run(value) - - def testEnqueueWithIncompatibleInputsFailsWithInformativeError(self): - dataset = dataset_ops.Dataset.from_tensor_slices([0]).repeat(None) - dataset = dataset.apply( - tqd.prepend_from_queue_and_padded_batch_dataset(batch_size=1)) - iterator = dataset.make_one_shot_iterator() - queue_handle, value = iterator.get_next() - - enqueue_bad_structure = tqd.enqueue_in_queue_dataset( - queue_handle, (value, value)) - enqueue_bad_dtype = tqd.enqueue_in_queue_dataset(queue_handle, - np.array( - [1.0], - dtype=np.float32)) - enqueue_bad_shape_no_batch_dim = tqd.enqueue_in_queue_dataset( - queue_handle, ([1],)) - enqueue_bad_shape = tqd.enqueue_in_queue_dataset(queue_handle, - np.array( - [[1]], dtype=np.int32)) - - with self.cached_session() as sess: - with self.assertRaisesOpError( - "mismatched number of tensors. Queue expects 1 tensors but " - "tried to insert 2"): - sess.run(enqueue_bad_structure) - with self.assertRaisesOpError(r"Expected component 0 to have batched " - r"shape \[1,...\], but saw shape: \[\]"): - sess.run(enqueue_bad_shape_no_batch_dim) - with self.assertRaisesOpError( - r"mismatched shapes at component 0. Attempted to insert tensor " - r"with shape \[1\] but queue expected shape: \[\]"): - sess.run(enqueue_bad_shape) - with self.assertRaisesOpError( - r"mismatched dtypes at component 0. Attempted to insert tensor " - r"of type float but queue expected type: int32"): - sess.run(enqueue_bad_dtype) - - def testEnqueueWithPaddedBatchFailsWithInformativeError(self): - dataset = dataset_ops.Dataset.from_tensor_slices([0, 1, 2]) - dataset = dataset.apply( - tqd.prepend_from_queue_and_padded_batch_dataset(batch_size=1)) - with self.assertRaisesRegexp( - TypeError, r"Unable to create padding for field of type 'variant'"): - dataset.padded_batch(batch_size=10, padded_shapes=[1]) - - def testOneEnqueueWithPadding(self): - dataset = dataset_ops.Dataset.from_tensor_slices([0, 2, 4, 6]) - # Make a dataset of variable-length vectors and their lengths. - dataset = dataset.map( - lambda c: (c, c * array_ops.ones((c,), dtype=c.dtype))) - # Emit a queue we can prepend to, and counts/values as padded - # batch. - dataset = dataset.apply( - tqd.prepend_from_queue_and_padded_batch_dataset(batch_size=3)) - - iterator = dataset.make_one_shot_iterator() - queue, (count, padded_value) = iterator.get_next() - - # Split the padded_value into two pieces: head and rest - rest_indices = array_ops.squeeze(array_ops.where(count > 2), axis=1) - bound = math_ops.minimum(2, math_ops.reduce_max(count)) - value_head = padded_value[:, :bound] - count_rest = array_ops.gather(count - 2, rest_indices) - value_rest = array_ops.gather(padded_value, rest_indices)[:, bound:] - queue_rest = array_ops.gather(queue, rest_indices) - enqueue_rest_op = tqd.enqueue_in_queue_dataset(queue_rest, - (count_rest, value_rest)) - with ops.control_dependencies([enqueue_rest_op]): - calc = array_ops.identity(value_head) - - with self.cached_session() as sess: - self.assertAllEqual([[0, 0], [2, 2], [4, 4]], sess.run(calc)) - self.assertAllEqual([[4, 4], [6, 6]], sess.run(calc)) - self.assertAllEqual([[6, 6]], sess.run(calc)) - self.assertAllEqual([[6, 6]], sess.run(calc)) - # Get some final batches due to prefetching. - for _ in range(3): - try: - self.assertAllEqual( - np.empty(shape=(0, 0), dtype=np.int32), sess.run(calc)) - except errors.OutOfRangeError as e: - self.assertTrue(str(e).startswith("End of sequence")) - - def testNonstandardPadding(self): - dataset = dataset_ops.Dataset.from_tensor_slices([0, 2, 4, 6]) - # Make a dataset of variable-length vectors and their lengths. - dataset = dataset.map( - lambda c: (c, c * array_ops.ones((c,), dtype=c.dtype))) - # Emit a queue we can prepend to, and counts/values as padded - # batch. - dataset = dataset.apply( - tqd.prepend_from_queue_and_padded_batch_dataset( - batch_size=3, padding_values=( - 0, - -1, - ))) - - iterator = dataset.make_one_shot_iterator() - _, (unused_count, padded_value) = iterator.get_next() - - with self.cached_session() as sess: - self.assertAllEqual([[-1, -1, -1, -1], [2, 2, -1, -1], [4, 4, 4, 4]], - sess.run(padded_value)) - self.assertAllEqual([[6] * 6], sess.run(padded_value)) - with self.assertRaisesOpError("End of sequence"): - sess.run(padded_value) - - -# TODO(ebrevdo): Figure out how to use run_core_tests to test state -# saving of an iterator that's had some tensors enqueued into its queue. -class PrependFromQueueAndPaddedBatchDatasetSerializationTest( - dataset_serialization_test_base.DatasetSerializationTestBase): - - def testPrependFromQueueAndPaddedBatch(self): - - def build_dataset(seq_lens): - return dataset_ops.Dataset.from_tensor_slices(seq_lens).map( - lambda x: array_ops.fill([x], x)).apply( - tqd.prepend_from_queue_and_padded_batch_dataset(batch_size=4)) - - seq_lens1 = np.random.randint(1, 20, size=(32,)).astype(np.int32) - seq_lens2 = np.random.randint(21, 40, size=(32,)).astype(np.int32) - self.run_core_tests(lambda: build_dataset(seq_lens1), - lambda: build_dataset(seq_lens2), 8) - - def testPrependFromQueueAndPaddedBatchNonDefaultPadding(self): - - def build_dataset(seq_lens): - - def fill_tuple(x): - filled = array_ops.fill([x], x) - return (filled, string_ops.as_string(filled)) - - padded_shape = [-1] - return dataset_ops.Dataset.from_tensor_slices(seq_lens).map( - fill_tuple).apply( - tqd.prepend_from_queue_and_padded_batch_dataset( - batch_size=4, - padded_shapes=(padded_shape, padded_shape), - padding_values=(-1, ""))) - - seq_lens1 = np.random.randint(1, 20, size=(32,)).astype(np.int32) - seq_lens2 = np.random.randint(21, 40, size=(32,)).astype(np.int32) - self.run_core_tests(lambda: build_dataset(seq_lens1), - lambda: build_dataset(seq_lens2), 8) - - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/verbs/rdma.cc b/tensorflow/contrib/verbs/rdma.cc index f7c979e8632..9db80f6b573 100644 --- a/tensorflow/contrib/verbs/rdma.cc +++ b/tensorflow/contrib/verbs/rdma.cc @@ -30,7 +30,6 @@ limitations under the License. #include "tensorflow/core/distributed_runtime/rendezvous_mgr_interface.h" #include "tensorflow/core/distributed_runtime/rpc/grpc_util.h" #include "tensorflow/core/distributed_runtime/session_mgr.h" -#include "tensorflow/core/distributed_runtime/rpc/grpc_util.h" #include "tensorflow/core/framework/rendezvous.h" #include "tensorflow/core/framework/tensor.h" #include "tensorflow/core/lib/core/status.h" @@ -1028,7 +1027,10 @@ Status RdmaTensorResponse::PrepareRecvTensor( return errors::Aborted( "RecvTensor expects a different device incarnation: ", parsed.src_incarnation, " vs. ", (*src_dev)->attributes().incarnation(), - ". Your worker job was probably restarted. Check your " + ". Your worker job (\"", + channel_->adapter_->worker_env_->session_mgr->LegacySession() + ->worker_name, + "\") was probably restarted. Check your " "worker job for the reason why it was restarted."); } diff --git a/tensorflow/core/BUILD b/tensorflow/core/BUILD index a701b38d4b3..575edfe7a93 100644 --- a/tensorflow/core/BUILD +++ b/tensorflow/core/BUILD @@ -95,7 +95,8 @@ load("//tensorflow:tensorflow.bzl", "tf_cc_test_gpu") load("//tensorflow:tensorflow.bzl", "tf_cc_tests_gpu") load("//tensorflow:tensorflow.bzl", "tf_cuda_cc_test") load("//tensorflow:tensorflow.bzl", "tf_version_info_genrule") -load("//tensorflow:tensorflow.bzl", "if_not_tx2_llvm_or_windows_cuda") +load("//tensorflow:tensorflow.bzl", "if_nccl") +load("//tensorflow:tensorflow.bzl", "tensorflow_opensource_extra_deps") load("//tensorflow:tensorflow.bzl", "tf_cuda_only_cc_test") # For platform specific build config @@ -112,6 +113,7 @@ load( "tf_additional_device_tracer_test_flags", "tf_additional_gdr_lib_defines", "tf_additional_human_readable_json_deps", + "tf_additional_logger_deps", "tf_additional_lib_defines", "tf_additional_lib_deps", "tf_additional_lib_hdrs", @@ -300,6 +302,7 @@ filegroup( "platform/env_time.h", "platform/logging.h", "platform/macros.h", + "platform/platform_strings.h", "platform/types.h", ], visibility = ["//visibility:private"], @@ -442,6 +445,18 @@ cc_library( ] + tf_additional_human_readable_json_deps(), ) +cc_library( + name = "logger", + srcs = tf_platform_srcs(["logger.cc"]), + hdrs = ["platform/logger.h"] + tf_platform_hdrs(["logger.h"]), + copts = tf_copts(), + visibility = ["//visibility:public"], + deps = [ + ":lib", + ":lib_internal", + ] + tf_additional_logger_deps(), +) + filegroup( name = "platform_env_hdrs", srcs = [ @@ -519,6 +534,19 @@ cc_library( ], ) +cc_library( + name = "platform_strings", + srcs = tf_platform_srcs([ + "platform/platform_strings.cc", + "platform/platform_strings_computed.h", + ]), + hdrs = [ + "platform/platform_strings.h", + ], + visibility = ["//tensorflow/core:__subpackages__"], + deps = [":lib"], +) + filegroup( name = "platform_other_hdrs", srcs = [ @@ -841,6 +869,7 @@ tf_cuda_library( "framework/dataset_stateful_op_whitelist.h", "framework/device_base.h", "framework/function.h", + "framework/function_handle_cache.h", "framework/graph_def_util.h", "framework/graph_to_functiondef.h", "framework/kernel_def_builder.h", @@ -884,6 +913,7 @@ tf_cuda_library( "util/bcast.h", "util/cuda_kernel_helper.h", "util/device_name_utils.h", + "util/dump_graph.h", "util/events_writer.h", "util/example_proto_fast_parsing.h", "util/example_proto_helper.h", @@ -901,6 +931,7 @@ tf_cuda_library( "util/stream_executor_util.h", "util/strided_slice_op.h", "util/tensor_format.h", + "util/tensor_ops_util.h", "util/tensor_slice_reader.h", "util/tensor_slice_reader_cache.h", "util/tensor_slice_writer.h", @@ -1038,6 +1069,7 @@ tf_gen_op_libs( "batch_ops", "bitwise_ops", "boosted_trees_ops", + "tensor_forest_ops", "candidate_sampling_ops", "checkpoint_ops", "collective_ops", @@ -1085,7 +1117,11 @@ tf_gen_op_libs( op_lib_names = [ "string_ops", ], - deps = ["@com_google_absl//absl/strings"], + deps = [ + ":lib_internal", + ":lib_proto_parsing", + "@com_google_absl//absl/strings", + ], ) tf_gen_op_libs( @@ -1187,6 +1223,7 @@ cc_library( ":batch_ops_op_lib", ":bitwise_ops_op_lib", ":boosted_trees_ops_op_lib", + ":tensor_forest_ops_op_lib", ":candidate_sampling_ops_op_lib", ":checkpoint_ops_op_lib", ":collective_ops_op_lib", @@ -1340,6 +1377,7 @@ cc_library( "//tensorflow/core/kernels:batch_kernels", "//tensorflow/core/kernels:bincount_op", "//tensorflow/core/kernels:boosted_trees_ops", + "//tensorflow/core/kernels:tensor_forest_ops", "//tensorflow/core/kernels:candidate_sampler_ops", "//tensorflow/core/kernels:checkpoint_ops", "//tensorflow/core/kernels:collective_ops", @@ -1386,9 +1424,7 @@ cc_library( "//tensorflow/core/kernels:summary_kernels", "//tensorflow/core/kernels:training_ops", "//tensorflow/core/kernels:word2vec_kernels", - ] + tf_additional_cloud_kernel_deps() + if_not_tx2_llvm_or_windows_cuda([ - "//tensorflow/core/kernels:nccl_kernels", - ]) + if_not_windows([ + ] + tf_additional_cloud_kernel_deps() + if_not_windows([ "//tensorflow/core/kernels:fact_op", "//tensorflow/core/kernels:array_not_windows", "//tensorflow/core/kernels:math_not_windows", @@ -1413,6 +1449,8 @@ cc_library( ]) + if_cuda([ "//tensorflow/core/grappler/optimizers:gpu_swapping_kernels", "//tensorflow/core/grappler/optimizers:gpu_swapping_ops", + ]) + if_nccl([ + "//tensorflow/core/kernels:nccl_kernels", ]), ) @@ -1437,7 +1475,7 @@ tf_cuda_library( ":gpu_runtime", ":lib", ":ops", - ], + ] + tensorflow_opensource_extra_deps(), ) cc_library( @@ -1577,6 +1615,8 @@ filegroup( "util/stats_calculator.*", "util/reporter.*", "platform/**/cuda_libdevice_path.*", + "platform/**/logger.cc", + "platform/**/logger.h", "platform/default/test_benchmark.*", "platform/cuda.h", "platform/google/**/*", @@ -1671,8 +1711,8 @@ cc_library( cc_library( name = "mobile_additional_lib_deps", deps = tf_additional_lib_deps() + [ + "@com_google_absl//absl/container:flat_hash_set", "@com_google_absl//absl/strings", - "@com_google_absl//absl/time", ], ) @@ -1763,7 +1803,7 @@ cc_library( # registration of ops to prune code size. cc_library( name = "android_tensorflow_lib_selective_registration", - srcs = if_android(["//tensorflow/core:android_srcs"]), + srcs = if_android(["//tensorflow/core:android_srcs_only_runtime"]), copts = tf_copts(android_optimization_level_override = None) + [ "-DSUPPORT_SELECTIVE_REGISTRATION", ], @@ -1775,9 +1815,7 @@ cc_library( visibility = ["//visibility:public"], deps = [ ":protos_all_cc_impl", - "//third_party/eigen3", - "@double_conversion//:double-conversion", - "@nsync//:nsync_cpp", + "@com_google_absl//absl/container:flat_hash_set", "@protobuf_archive//:protobuf", ], alwayslink = 1, @@ -1787,7 +1825,7 @@ cc_library( # no proto_rtti. cc_library( name = "android_tensorflow_lib_selective_registration_nortti", - srcs = if_android(["//tensorflow/core:android_srcs"]), + srcs = if_android(["//tensorflow/core:android_srcs_only_runtime"]), copts = tf_copts(android_optimization_level_override = None) + tf_opts_nortti_if_android() + [ "-DSUPPORT_SELECTIVE_REGISTRATION", ], @@ -1799,9 +1837,7 @@ cc_library( visibility = ["//visibility:public"], deps = [ ":protos_all_cc_impl", - "//third_party/eigen3", - "@double_conversion//:double-conversion", - "@nsync//:nsync_cpp", + "@com_google_absl//absl/container:flat_hash_set", "@protobuf_archive//:protobuf", ], alwayslink = 1, @@ -2045,9 +2081,7 @@ tf_proto_library_cc( srcs = ["protobuf/master.proto"], cc_api_version = 2, protodeps = tf_additional_all_protos(), - visibility = [ - "//tensorflow:internal", - ], + visibility = ["//tensorflow:internal"], ) tf_proto_library_cc( @@ -2187,6 +2221,7 @@ cc_library( "platform/**/env_time.cc", "platform/**/cuda_libdevice_path.cc", "platform/**/device_tracer.cc", + "platform/**/logger.cc", "platform/**/logging.cc", "platform/**/human_readable_json.cc", "platform/abi.cc", @@ -2199,6 +2234,7 @@ cc_library( "platform/**/stream_executor.h", "platform/**/env_time.cc", "platform/**/device_tracer.cc", + "platform/**/logger.cc", "platform/**/logging.cc", "platform/**/human_readable_json.cc", "platform/abi.cc", @@ -2641,6 +2677,8 @@ tf_cuda_library( ":stats_calculator_portable", ":version_lib", "@com_google_absl//absl/base", + "@com_google_absl//absl/container:flat_hash_set", + "@com_google_absl//absl/strings", "//tensorflow/core/platform/default/build_config:platformlib", "//tensorflow/core/kernels:bounds_check", "//third_party/eigen3", @@ -2943,6 +2981,7 @@ tf_cuda_library( ":lib_internal", ":proto_text", ":protos_all_cc", + "@com_google_absl//absl/memory", "//third_party/eigen3", "//tensorflow/core/grappler:grappler_item", ] + mkl_deps(), @@ -3008,7 +3047,6 @@ tf_cuda_library( hdrs = ["common_runtime/metrics.h"], deps = [ ":lib", - "@com_google_absl//absl/time", ], ) @@ -3033,7 +3071,6 @@ tf_cuda_library( ":protos_all_cc", "//tensorflow/core/debug:debug_graph_utils", "//tensorflow/core/kernels:function_ops", - "@com_google_absl//absl/time", ], alwayslink = 1, ) @@ -3393,6 +3430,7 @@ tf_cc_tests( "platform/profile_utils/cpu_utils_test.cc", "platform/stacktrace_handler_test.cc", "platform/subprocess_test.cc", + "platform/vmodule_benchmark_test.cc", ], deps = [ ":lib", @@ -3406,6 +3444,20 @@ tf_cc_tests( ], ) +tf_cc_test( + name = "vmodule_test", + srcs = ["platform/vmodule_test.cc"], + tags = ["optonly"], + deps = [ + ":lib", + ":lib_internal", + ":lib_test_internal", + ":protos_all_cc", + ":test", + "//third_party/eigen3", + ], +) + tf_cc_test( name = "lib_random_random_distributions_test", srcs = ["lib/random/random_distributions_test.cc"], @@ -3421,6 +3473,16 @@ tf_cc_test( ], ) +tf_cc_test( + name = "platform_strings_test", + size = "small", + srcs = ["platform/platform_strings_test.cc"], + deps = [ + ":lib", + ":platform_strings", + ], +) + tf_cc_test( name = "platform_env_test", size = "small", @@ -3668,6 +3730,7 @@ tf_cc_tests( "util/bcast_test.cc", "util/command_line_flags_test.cc", "util/device_name_utils_test.cc", + "util/dump_graph_test.cc", "util/equal_graph_def_test.cc", "util/events_writer_test.cc", "util/example_proto_fast_parsing_test.cc", @@ -3798,6 +3861,7 @@ tf_cc_tests_gpu( ":test", ":test_main", ":testlib", + "@com_google_absl//absl/memory", ], ) @@ -3826,6 +3890,7 @@ tf_cc_tests_gpu( ":test", ":test_main", ":testlib", + "@com_google_absl//absl/memory", ], ) @@ -4099,6 +4164,7 @@ tf_cc_test( "//tensorflow/core/kernels:identity_op", "//tensorflow/core/kernels:immutable_constant_op", "//tensorflow/core/kernels:matmul_op", + "//tensorflow/core/kernels:topk_op", "//third_party/eigen3", ], ) @@ -4392,6 +4458,7 @@ tf_cc_test( "//tensorflow/core/kernels:random_ops", "//tensorflow/core/kernels:shape_ops", "//third_party/eigen3", + "@com_google_absl//absl/memory", "@com_google_absl//absl/strings", ], ) @@ -4871,6 +4938,7 @@ transitive_hdrs( "//tensorflow/core:core_cpu", "//tensorflow/core:framework", "//tensorflow/core:lib", + "//tensorflow/core:platform_strings", "//tensorflow/core:protos_all_cc", "//tensorflow/core:stream_executor", ], diff --git a/tensorflow/core/api_def/api_test.cc b/tensorflow/core/api_def/api_test.cc index 6f988569159..d38a8424eb1 100644 --- a/tensorflow/core/api_def/api_test.cc +++ b/tensorflow/core/api_def/api_test.cc @@ -182,11 +182,14 @@ void TestDeprecationVersionSetCorrectly( for (const auto& name_and_api_def : api_defs_map) { const auto& name = name_and_api_def.first; const auto& api_def = name_and_api_def.second; - ASSERT_TRUE(api_def.deprecation_version() == 0 || - api_def.deprecation_message().empty()) - << "ApiDef that includes deprecation_version > 0 must also specify " - << "a deprecation_message. Op " << name - << " has deprecation_version > 0 but deprecation_message is not set."; + if (api_def.deprecation_version() != 0) { + ASSERT_TRUE(api_def.deprecation_version() > 0) + << "Found ApiDef with negative deprecation_version"; + ASSERT_FALSE(api_def.deprecation_message().empty()) + << "ApiDef that includes deprecation_version > 0 must also specify " + << "a deprecation_message. Op " << name + << " has deprecation_version > 0 but deprecation_message is not set."; + } } } } // namespace diff --git a/tensorflow/core/api_def/base_api/api_def_BatchDataset.pbtxt b/tensorflow/core/api_def/base_api/api_def_BatchDataset.pbtxt index 639d962874d..32def912f83 100644 --- a/tensorflow/core/api_def/base_api/api_def_BatchDataset.pbtxt +++ b/tensorflow/core/api_def/base_api/api_def_BatchDataset.pbtxt @@ -1,5 +1,6 @@ op { graph_op_name: "BatchDataset" + visibility: HIDDEN in_arg { name: "batch_size" description: <

nzV5r`>aWFZ@~FJ&qoh8GvH6 z+0@p;wKa2?Ttj^V;BRDx(?h!8%43ETJ!aD)`Fc}gcHCRvb zFESleNLPvbNq?zpSuOe3g{uT*oELNlu>3*56}k`btOMkFnh$Uz$1qdD^HhLC$s@El zc4w}F-Hee>xkmbr96`;Xt^}rx1R9C?iqjepxcBof@r1ls{srO7i2YI3k^96Q1fMv0 zY$eMLd^q#y`Sj_`xt#vIcs{~!$HRC7`3HnW;!TkYBcnxMxWC!MI2*a$xwly7fxS3| zj%QQ@D(f`baLQED7}6B-KuRx49%(bu3!fCa0$8t)Le3C3v^&r@xEgSe4PirgEbb5f zFN_hB$qOla$aj&NusHZ{ut)e8E(7;G^ewa+S4&8R$3o@!vG^^7$MAA&AY}sm4Q(>{ zFiM51;kDRaBA-ac?je!LV^~6n#k~$ffouM&!R7e1$R)rnzX}fwH~6K#*1=5tKEhT4 zigYCnriy59DXmCeA?K@uJqEV3wF$YgSR z(r!`#^$wH8)9_?iFb=TP>OrKD!$Y>I<=g7%g~hRTDJ0gdAvV8*2f zS_Y$V?FlWRdgv9r3Au#OQ5SIxV*o!wbc&zEI!Ad)Od$@&4x`(RVlfFt4zDvL3fh28{C=W`a%Xz5=?+4)0lKo~^B=$h_H9 z3fOsPt^Yc=d9L`@`=|Opd)=uP{DiOz@Qd~W6KX2ji2WiJlVzm4SS|XR6r$f| zKV^p*_o>eTm-#kn7O@T14^2njLT3nG@cDD1=~x^(3Aup2Bk!blq`k%F6LQ1sP*P|^ zaAM$kU^gI1%=bSBeR`$0ziX{+mX+&JdT@b*zT3``mL$VPtxa=QdtJxZQ*`gu1eHly zrb5(Hl<&dWQcp!Et-$!+uuZd87AN^q)3^3X-RXK<<5fU0x(Yxo=NfYwAjzca0aZV1 zFUfXjT-tGJgUl~Utf5qot$9Gs9#+_9Z=IwHcpnVuPd!NQ|D>?q23ML(^GUS zR2vj=%3dnD`hbpVEH!<%WZ0dKt&UAr$aKI^WF%Ob&c*JfE`#lg>4iSq(AB)xw#FIZ z>EUw+Ug4CWZ($%*gD&6tAc~)e3?mLAUBDoC9{wFJgmdF>fj;?L0uKE`GEzE#S}2Hp zM(<(i(X3|@*&+f*yI3gasW0t0Gk|uO%A{&2Vj!} zu*m_~f*yI3gasW0t0Gk|uO%A{&2Vj!}kU5e$nVv$6r%j+8 zq}r)shJg`esyOGk>FjyTXh5)&QWhaCp!);^z7@0%sYh2}3al4t0dP@zNb^YL#Qx;_ zARAjquczlzw@{YTRx&OCo9sS1iAiC+r5&WNU=-2wXnSab7!&DQYDtp=u*m_~EF1)L&bGjpJuXE7dPa8L2`phWmSgh1m6GW-O>HE1bJgWnQlxT?^;;GN)uP!y9|a?#tXVzig!ZjAFTJh32@9Vt8&CXlifmW*=!^Z5w4>Y)Q9nwQHP(?p>ZZ z&tqVKTInKrdikaWa)Ug;;CvGf;0F@=;~c>r!Gh2_TpgfS=usYVKlTN=2S0$P!UfPp zLJi>s?86R{qbaM&$4Ny*Hc3Pw(8VkXCx&-{`;Bv!lg5qZt>x|F{^n@7zxg_WL~xCN zfLq4y#J<9IaT<8h0ym$pKTlxhALr$9_A}cuq8Ma`k$#DuPjdkh(t6S$@?;83eM{*^*+-d5#nT+r zany&D!;~IWKkW>23THAegO3

?IejSe2Wu@mjT7Q9xT&1qtZU4#%$;m0=M6iX8AD${ zJwP4=Sfv|C-AHci4dQ_+fm3V(B*r^J^5Bq=JA52pgrA9996AyB?d$2CD^jidFqwHq{uCPUL+b6ovVRi@IaJ(};D_o@?e zMdKp5UKQ3}HHYb4Lx4WDBhO?bxi$mf{^~QK-xbHY2$4=V?+W^N$XM(HD+0!}AS?;>#S?k^7 z{RSwe(*u!V8eu8{n9ga2dg_)ovDWEx=BoRhikpnt*ie6;vZfztbSF!R+3d8 zl0J~NQVvnARz8#KWC8g^)pAX=_Lrtuldip~-J?_JGmTDTFH@muoVh)Ck1;dJNCn1) zUi!tRur=Dz$8q2O#-8m+aKMhXjz{)P+b!#V_9(a1+tb(R9_I*KEtWh>u~`8)U31I} zO-GCmjXzCi&DYFS%WvyZ$3*vNFTp4Cs=PmZvcQ?}8T>**Dd85ujb{;#K%ZbUG!L%` zJBDZB6$`3FyTt#BH;bo>SBs_y#X`I&OH38#2v75#a&~bdc}aqf!Un-A z{#qWJA1&Yu?EK4u)q;z>JTAgpEhrcHM0~*p&V5!FmWN4aX<73*6L|stA;Cw1L$F?u z5AMW6c)0?RctoT=a#du{$YD_rV}8aZ#BYr|7q>2cSi9p)XS-X)H!LVQ=w!@{NmVAF}X1xqw}KfMU0BL z9kD0kvv{ z|2;rQZ$L(Z`)whl$J6k&xRc-xvnhNr)HXOY@FB1?=nHiAD~h1C%F;)7Tg#t3w8=5nLUj&kxS%490t1s zGnbY}UPt^&tS9%To+P``M#zVU@V{{e9EP8c|AoJSx8effV8{u`Un#yRp1q#0-tNAw zUb$P~vN-ALmW@!CGx)xf~w zH;gpBF{T)@^@V!3K?%%9o%P4G!!^Iuv%oXeY6fV?nr@nA+CDlbxGw+IH|m~gUuh0% z5N&^LNRtT&roRn4jOE7BCe(BX5Ia}te;9U}mYcK9!-03QulcMc%R1H~GL-?t&wl-D z0~dU+*1R3?Jb4z6<%9K>EzWMXeYKfwt?jezE$juh*48|W!#oA}FAP?r)oB@TRvIrG zyP0N~-Wz#_M(t~@Tc2V~*H2crRy>v^%ZRe=vRFAo-c}}QxCW?ztLmRf&eScb?NO6i zeXlyTjsbY_QzYqirW$F@*4l@)18d*aEU5XXW^~Q{8nkv{ji5RfSYa;II_fEnEgFAG z8PbytadM+Vtm3NkG|hCY4Fpr8>8AOoS!MDYBTS3UC#{R^GaPD%+|k@|)m~%Ib6}34 z_Wx`x?5~~6y@LTotwmssZ@uqgphJirvINfgTlzBsPlC44VX)eILWMzk&>z}@Zw1!p z?%_SbR>7mebs-z>5wsW1flFW%drx{pxlKJtg{Wt#vl)4uT>dh_TK+h0n3Kl036=`p z@^D-uyBX&c_bYERua-NQXXg#$4d>qCHsfpg+jt_*a&RZe=I-Hb=kDWtP1g&&-5sSgP&-h4ROkY6jMD0(-s2tL7cpO;O8;D2AeJJIWwbaA3EsV#k zee4K!9D5Mk&cZlseske+p?Tq8@k+qu8kFInB0?`dyozZnkJ9S(~_Vw>4a zj+DKQJB5En@Lljha9^}K@=$a^j3`)Z0xoGwVt)LESVP>1go_DiB9b^P zseg(nd3$2}WM`^5bxq>@xK)V>DaK?=%9qrEX(Q7pDKk=Lr7==^CihC}9W759+G&0M zO13LN;Ab`a8#SNY9m`@pj;)Q$pl%7TWK_lQqnAev6q%8qj%s36vu_z^=qA?z*GdvG zzC3jYj~K3Twlh}C^35N)`_ksHUf4u|`K^?9;sh9BNG{;P(^ zx*Eb2(FjJJdPPZrWQ3y&s9zRYY>l;&Lylg)r;V?w-|AxBI(Lfow{e0S=V`4utZw6q zbI#L0uwD#Q`Xk*7OqRxvYKkYr)zWb?kZrG(jaNOl4-5bC*3_Tf9d(}sKfQq3mvf&y16SnlZGUgf^D=Q$4BILWRCQ5hTMm2nd0_2e ziA}<+*QVb^pYT|5ogRxo7$a=D=rRkbvuq4|>gc<-bZ-wWws<&z@utfO4{mI}+ z!KZ|W*gv_F1LNF!pD{Gt+0nYjo$H$xS_%7uyKQrP(d=8IwP+jn4#G~xT--D5dD}I1 zPTVGTTkmr9P&LI_9;|Y;*PqjFw4{3L5i;#b5HS}9|59S%R>&=81#Le1A9Mj(#eFPZ zPElbr)?9{_wv4O}Yzhq}mmINDU3^w;9Ur=c%cUE?9j8NT@ z?W)sNQ|oFQAk{$?v8MmeCx3fawJI6>r?ft*fmJ*3&$8m*Wg{Bi)vc^CR2=#>tc)eW z)ym7F|9UHC)fLt6mTa!-S>dQ^T`|5qOR>S-5%x;e|wU?L7-z)&kTyAm18_`Y9LB1qX z&h3W380brFCQ29A2>uafidzZ}Fs+ngQUg&5Jr4!3o~$YV!}jDK6i;9j6NK(|?gFAi zFkZNn8XvkJh$Y679*6%$Lre?I^qK?FVHNZbAcS_MXE6qbbAt%BnlJ|ahkcRQHTaA0 z2dFA>gqipy2tMp_UB^{WBj^swc;Y^29UjF_G1KV73C6G&9*?Dv0ELv5MT;U|phZ$P z(I!S$#y;S>*em&$qKpZHqwd5hGPyBwdLOP)geCRL-jg^vx<0`i=SgUt_MzE>HeK@j zqz;Peor2^(Z=IhrtND=@aN66Lo8ofbg9uq-Y0O7nbLId#oq3(PkJ~D0n5a8rIoBX= z!>%A-W#scV0e$B&wt#X8fysCIJz~2F-$0vuZ=ov;Dg6s150T;V#6+SLf6TMaeZrM) ze(s`B3t9O{p?jz!3s0fnCuib?p0Cz6{sUAeVsdUF4XQ-En)_(d0Pl5ymy@ zGhzWn#~R2lVvom`;d(>O(KCcNUyWmuw*~$Sya~VF-OhOjmk+ISUv$typHVXP9qDY) z<`8*?g(eV^(K4hU6k``_&M7+?KUp7G#@jkN`*}|X(>xbUOk;P`PxA!JVeKx>Fqb9R z%$;xR?Kt7N7I68Cooy_U=9|`&u8@Cj(BsSVwe(l{a(%}=6|O>`4qppR!`%)kLLGwt zgxRELv~K9)PzkV`)kCMybyzYI3~_zj026-?cQ$wk?##Z*HX(CE^?@=^XU`gZBPB$C z$(X>HjJ^rn4X301P%%6&{KRJqc7vvbXWP$dP5L~~Izkkn`7cKEajX4I&l$%6FOArX z*@yB9c|dp=V7c+0llUUA@^lFv@)!G3JZpTT12cR(lO`kS@XQ#$e6o3D$$#T3u}c<)_7QLjINpKHHO_(>=t zeJChW{^EOk15&*e2m$HVq3+yL?*bJbP9Y;LILRpXS zc5e&9ed1%%S85JDmU;nD=Wk(Ssak%nSj7HAeL^2Dcx(zhFuHv4n52>FKuI<2iFvcgFRJ z+nr!d+@G>E^I!{k#+j5e8I>&#w}4tr&&zJ%ZZR_NT~2m0a`O2kP1>g{Mb7M2y1dry zALO^stISMGULGrpSrR8snvs4dU6I@&@lNuKjNG&nF{bDqk*~O$g(o8MqT>-c(XAo} z$9|5#7WJR7C8ruP5B$+e5c$zPp8(OHp`7IRf9Cw%wx4J zWiynk^l9!%;r*W3)_bN5J;g+^QjD(+S4}?M05wyy#K5%_nz9V<3?fTcSAkFDpAVbG>6I3 z(K6MVZ)XDM!9$0_(#p9lcs;Pj`Nf{^*$6thAH5;hMDKF%zm7{Li*}D@l5vJT#eULQ z0Yob?b}W3&&ojZQZwii5qBtTSulTI-=}Oe^*Kd&ZRoPX?OznZ0+>wL51!T=q)$JwS^79$pY~IchA=>`PrypuT$=obG=R91(8xclP{nwe>U$?ZVrG zcl@EC8#fk$31z{<{)yfN-lEWR+=1|C+(q9T^C{C6GpygPX`z{-{cW6QxFo+@r>uHX z*{`}!_4Nu-DgJxI=fOqRKh?Fl4XmEOODD_+(;XpF6&Q1R_=TE*0IamlCR zX1^wv(<)w;nM-s2eyz%usvCPXT&}6C*jU{~vQpAn!j;Np2eqfoPmJGHd5Sa2y=t89 zfTp=B#>jAOb+j}-QvFb_G<~uUG49a9z<~PISYn=OG@Ir+S>8(_Bk~pc9T*cDgfc19 z$ri+f%O`xn_F?z2m5gGxg1MW`V4k6k;t*L=sh?V!=1z~W$t5irS$-gtpA9G z=u&tt;UB^RWC2l+MI-;Kp}TQzqDf^fWAc@>eDqYPeYlWaEGq%Q1*4%BU$Bj*|qtFg~x@T z3y*-mc{Kl@LZVPz@a4bBzn(A4qxpfHC+EqUGw;f^nFHBpvL~fp;UcHuJlx0Jd~O*# zM2Dy_eV_YGd?DQ;Oay;ARg9h;!}W3#xoZqTH!!u_IKe2M7Y))xd8K?%9+YYDPHp1$ z(^_f{HI=TRcTp9os|hGMluA>JsX4%uDNP(rJWSzqJM#nmY3e30{%YxuxKg2+AIptp zdzdrKb#@vr^A15FaD1IW3eO6|>|DAv`D@&p*q$n+<|X5?=|i)J24ddCC&{BJlscAb zrp~bu;fja~KQSj$-sJ7%m{famZ+vw$8CHfjhY9k0bW+S6Z-`d|)pvK|bYf}jI;jgi zO&CJWz{d53$CKyCC6Q#*A3GC&D^A41vGoZheVMBjR`5@lAE~ueIaAKv=d|2G<_&r$ z^*3ra{a1Dm7v%m7p2^wtA=#SQCYi zRV!8im*s+fy6!-6R}lpir5nXq(FRpyjcehrL_&2 z2|J;3^DMO4KELFY?I-kY6vZ;=7VAdD2Uo(E;0olJWxv&CEkVvg+f2Psy`>v_9@ANB z;6u#!MZacqYDh@;wB;cRr6 z?US&tu;N|rG_X#- zmsK;b(p~ho%=_#DP{3PK+4#O#3n*&R(80@07J@!uF+?CS~K4w6K52nsof>v&6m_a6r4a*zMjKou_Fm+<~Tt^deC z{r`UV>&E^a{`!z1@_-y47KvrV^k7?Hr@z<73}y!(<>Z?bf?>-MU+K&gsZq$vY*uhxfw@`4Da(hmhm&Oe_T( zjqMV+1aiVmL`bdhxTp?2n@phWWEH=F-H>&IX=XHK{ll*0Eas}Y-FS1+Ow`Yt&HIJ) zMsM)v3J&6xcpJew{vvD%agu0(Z^wBgpLmMx!WR?o@MhRntS>PJHweDq3&>OCFS@Zb zO_`anJT8kaz;9!%vB{W}SVS5LiWmrvYxqVyoft-LpuUqfY@}c^mX96gkK<3`XQEEz zBDW>?0&@K~;$1-9JPB_s zKZ&2o?ZsV;?m=siTF!A+a;$yG7VHrI9fHCGLL0nKU6Z`*z>q+$e}T85XN+s6vx%GS zIpzFdEier*eKY?tWf>KQZTjJ+ht``GwJ}G3OI=6(KwDqePWw*BH4M<5)!xuwGcGZ& zu^cgVFuc-N=zeQftM{ornicv5#%=oT>U51!*GGFvQ)Ad@)|=Xzij1#}&x}IzLEAX{ zb;~Bp3fph{ch>`7qrj@b-4Go;7VZ^%5xN*D3V#ZX3fD&8Fpe@;vre-Qakg-m@It%> z$X)h+wvMBOTgA>scgJ2tC9(F<8t5@&06dZX0xp6(!9N(u%!Tj*CKfvoZ5*u=JssN= zJ0HCmQ!@X7XThtPPH1#=YiMau?SJVT85kM75IE{z=tVq@y>kN>BdelMg0;TSo;Qvh zTXSn0^EOjM^IhW^om1Vp_GN9B?v<&D^^B#+^jW`AGeG0f4K(gBKGH>N`_{hHywR5$ zmzmqy{Enz&f}OM>K15I z)Wx-TG>dg}b!zQP!y}u})y(B}-|^-6NBVkrPrHY?nz{E1HOjA(a{aevz(1c zPwqO7g8P+yfq9qlhxMGjoZT4y8*a=MqjmWA`EgiX0;LQj7k`FEczFVZ_(r4>4e`Ag zoA^bg(bI_?SOOl#8uM*PZ(a(z8vTOqNB{6TAWOh^mB$^74CUVjYxe|Jf=%ZiL_TqZ z>=E!on8&WiX0Q$LarRd3dQ>5JjCzm+@LBG15AnL8*Lmsa3Vx6;5lj;Z1hxD!!4^S( zbPkfroyFmCZ*sdLG}>8k9P?lpzKKv$&B$5AM{0n`E#%NU$PBVS$tPX-3alxA8@C-B zLh9qc>D$6fq@TYWZ77(=Z@}Hi8p7xYO=r&JTt$4m{(@6@IyG5XOy%KfYz|SETtHo; zr8I}?OgZR>qC#<*TclRZ@|h;OM#ii?Z$ zOB+jf%kE0oh#CqP3D=AEibsgDgd;>l#bYJ^lO2p}7S|{4gzScdC0;6W2@9yb_yepy zHdXLQFbAXXb~qn9CD@Izuzmb~0urBt!2r>Yp;|Ovpcf?ab9tMQ5Le8#!0#E!(U1P^ zo+4jz@O&UC(A%*W}slS?uPzlilrIKOO5GkL^P1a`Q;b16z03 zA@?mu)Ox|v(=;ACpz)aPpx zYfskZXx?k1`ZJ~<<_dFDOT2lSQDin*W7a}zI~&K2I3C&s_SR0N=ZtrWU+Y)<76ev@ z$4CANcMBzjCWH@#zXdl2!a-Yf7R$sbKu&YHoHDpC>pBBwoPgRv&RCn6B(?;)z#PEt z&waqT$7XV$bB}Q5b4GLiW}7)LkX$atu3)WX;;e^o54f6XVjN^pjCB!K;GX|saB!$F zsPk{|uMc{H5x?2@-1psk&Qt3yc3pH1cYbu#x0l<-+TGT#<`X8Lv4zQN+G?(6xnPbr znGG%UU9=(fZ1pWoiZ-z}MYUAZLBHFGSZ0|Am^5adwa7Bt+SJzCX0{HpKD1UkN<0sJ z2?1y5O7sep#ca&H%6J0FVoPJQ89f-QAR$A?AXwj6>saqu8{xO?1zbOS46{D-A$09%#C#&#JW5J*pemw66K9dU5rt z>W5Vqsy0?%uc=kNs2!kQp}wL{)}GWA>jvn*89JLjnmDFP<1R~rV~VTF9p|ZbS)Ci4 zOC3$@z3s_Pt!srl!;|lu8*CbR6#XZrhPJVGv2U=ua}eY!_b2-SJRZIei#e0HUwFmX z2oj=_>3HEax<56Kc!7<_D)1BJEwCC3sUjgNz9V`q+$1E0KC%LPiY`KoTnNeM>e+AL zQ}93VDb{MZJ)+@LSeQQu{g;=)OF;@a58&x=EjWk4C)j#U3uHQPD{l*LF0T`k%pK1e z$gY7E>`v?g_8j&Z_I2(h^eg`oYU0_^QDCJ^E~=M^I?WHu)c9>J_+#Iq*B z5!MjaYxo&A#M_S|=u6}Uw=;5-SAh29L)d%#BQcHiQu(4U;)#-Y=?7_^Y@ck0Y_qJ3 z?1HqutRhY=pQ*@Heo%If-yXj$eqel9S*c7>dg2Dk{L&ASM&c~d|7eO%621`j6`huN zS+!3j})fBf{i% zVRvP*m@gS?nGx1W_yjCu4~Hox2Aznk4(12?2g-cgy&HUb{}cau-y?6LFVXMwcMU}R z3j^&#UqiP8KfUkVuU#%D>{x3%Z{P3S?JTx0vL>1HjX(9Rb@$ZUYA#oetCE7FcePNp zv({DHuy&_vnkr9qK{dOUUrVdnsBEf$N~Ky|tI&vbZe4f%RztkeqDS<1bp3Sm^o>jr z%TPP%-0QmM5&HUhr+9XI$w05*=fLiOBDgtd4DJQ1cbl)JPwi6#HU+ie-Ovu!Hn=g| zftAgC0x4o`qM-;GT^B21>|=d`3pneM+q|~C^++NzjQaziMG<>GXFhi>#|~eBkHdf9 zo$PV!p0JB0ft9R*j3cpVcxx~*^fo*pd_Pnc8XU?CrH2oMM@ODT??*Y2q>v?WK9Cz& z;#=s^yS_U)&OVO04!!e_tFLRA^KbhV3t?VtYGqul&((d=4%c1PP1Rl0u?z*q!^SIy z76yfpVfI=?wr+N(W0P~O^SN`p^SbkkdxaPCW%#=UmIglhS^hkK>%gy|I`SE6%G}P9 zu%(>UoF?paRu!`&YaMeY<1Ta%n!^~!#F?+4?$BarRcvtdV{{xOWu0KZho>{`jOGk; zbY5g@xOu27crug`t_atQ{)i&chT&y_yWVeZ*rl~ctT)Vw#@+hcI+?Dm?yN3Fzeayi z*FwwI1XYFA@~ZX~C(0aU`4uB7ODc7h!>fi?r`A-~yjJ1r9PQuwcte8zp{|c%u(6B5 zp{I=pjBQN=ENWY}tFG^HfE{E7M*A*%y1H{+lbodvp0mAcpnI%mm#+Sk3@+4NV%yB>NVA!P814+1+;@+Ecz|( zC+Q$DOO8sPNT!Qlib{oBXdT%W|Cev%jYQ(P`?wrlJ@hYr8^JyPK$PbBxm~!kxjt?I zLh`orJUo=YRL}sci$w(o1cU$={3pOMvtXH^o8XEd9~0m!iIr5MkP_7wO%?8>pHXu1 z5>|(w$9sev=Ius*qEq?pv2y$sPU6R~OM)wWh`)h{B1&XEV&PKULcncq0VbG-Jna|@YlzJ&g07^PKaHQ@|lO`#DwI zp1j8VH<*!NP@AZCl$9Dp-=u%jxNyGktf)$|P3Du0k7LP;;`nh5Wn%!V*e>lO-7Q%y z(MaY=vc!6Tq~mEVl|$bUc9(pV@#R%qc=5m2Z0p+ZnF|>&8S|L$ne&)6jMt0> z3?cL+`aIe^Rug4KTSZ2Ob3$7JOMSmRqdj9ie|ZpZKi>lXmOx^##^2Rj>w4hmZR=;5 zXKrg=Wj<{ww0(6{xmI}UdjIFy?B4D=;+o^;cx>*o?!)dA?k4UM*8}$)?+xE8|Ipx# zQ1kHea7|bqsSQsCSLhCpie8BMp&I53z$r30OF17nZq8NCa5lt}FnU3YW8I@f^ifnD zJs7wis;oyMK~0i9_ky~5UL1SLq(x~L*s*^{O`P(-pihy z?w-yfE8lcbcSU_v^{8fV&8(WnHG-N|)%~gxD?3%RC@(IpF0qyf%6^oes)VYEnsKUa zwFhfu;5@4Sq~4+KsCld<^=%A&jBkxD<04bcwB6j^dfT4vZ0r2#kU4032itnkQ3ASQN;7D8{uTW>{xx%Ny4B>2g9sP$+pxcu3@FQ4r?6#m( zFa>Wy-UX^aPQ1d-^H=ePAzi>){u}v&oJF&-$M`;?l(bSzI+gAvJS<8UcM?w&j}ptp z-$i!Oc5x5MXGwSI3uzkMRiOPhZgaM&cv_yE0zD>^)))!@ow@BEsg1F`KzvQjs zOwzfM^P)uI3_1s>oy+txQ87RgNd8{#mN${VmSst|iBm;2^dhR3IDwzXjtO@1%h5$> z9rQBqIkF0{)<onZ_6u%a4o- zeGgE93jYoN9shv9+dz*%UH>ibLicLNQ|mc%sgZ9usPk&wT1pqvUeG3LH)$TIPt`JN z*Vm3#FHpBoFIAVQ9qL?-MpLCV=^XlE!)l|&*x9tjw8T``lwulUvYGy~NbThg$|d(K z^p*JE1tg(`;qQ@_F#_5F-DG@Yx|qqVO>jHTUtA?}8WHd|@U8(bMu>2@9(Fr+5UzsN z@E~?BXBzh{vX|%MokDZ?-_bVcX!H~M3mt>5O;7pg)E zfQQxvKEv$H_zG#Dtx!d*M{H|!cjRn%VCZy!7nm9-_bYvSJR96yUB4V%?C-37tzuiq z#&IxQZ9SvCv%K+u3u#=}9m8zHEayyfjdP7VO)t#>%PCuh{kq*_J7`;C+iZJfTWI@c z{cas(yJPEUAL{tyJnnAd75KLMR`^#2S_AF#EVMIxE%GUPE|w2XgzK2l*3l2)|Anf9{{~ILmf@<1 zF4{UKjkSnA49i1p0_}Z|JjdL1T+JLzTe(?eI$`LnAEaxco21LuZPVV;>{s`%WvOmd zFRJQSxw)dIqD@ts>TA`O8bj?EO^%kKv+8c@?fMq_G~H9}b#0FJiN>t?Tf179WH@4k z&3V>8_7%=Xt_`l~F5IPZ{_6}n)Q&>uRo51Gm0RSod-%Slz6aiy-rl}dK8{b|P4~3$ z5<0R%`d-*TW|M5-&tdR4zqT2+ifVYe% z{ggtuSlC)vMYj?VUx8OS}tMY#~SntKd+jT}WLBgF`ga`{{NYXn}b89|Y| z$kXI|GLKqB(bP1OMJ^?8QU_@*Jz1D9>LeCP4oig6C$jE}BHf$ zZOk^zN6Z3dUB;bQUepJ8`L2M=d)jl>GtKkB-2f=~Dwoam!8yym#M<4$HJ>ytGd$53 z=~o$U8X6lO>ig@j>7Hn7)D2W=)jg~7D<7AyEo)GArR+}m@rv%1yQ*&1tf=jxzN8+d z*{Id&1cn;JeWTs<$C7JTISkINZl5F7FNZKMsd;n*AU9Dd!9_6@A7(EC>j$V{W{N=tT~tZqc)an}sRD zrT{yaQb(vI)OIjkme9NCa%wr5NqB(cvjjbXByefYN_KO$6K)EhW$K{zvA5w5fe!wb z{_{Su_qAK-zUXS<%65LT7uv@=mN>rH7urwTX~#fEe|rP#19Jm&UsE%q!LZi&)YR5u zw%oMbvedCIv;MNSv?beo*2%W}z_Yk&^;_csDxPx<@$~SG4qOh_1`C6If-Qq1f*ph9 zfscU)!PDXA(fhFnv3~$Y%w?SguEhzCjHg9A@eiXT(9@^@y@DL#*5{t)hLK9%Ti!7A z4*x3u6K@Z9ANvwZ$~*!!i@l1@ig7_ku_E@r=)mZXs5rVmyg#%pgoWA#3cc@K1&%ki ze%2auz;wzq&g3!jjPLak-5cF9-FvN1Gg327Q?B{0>8WX=*{Rv9NzyD-w^wi1JkWjD zU(;*#2Mk^V%jh!THD(D*kksiPM-S;xIf}tEEnZ2aDR1ObG>#i@mhRrf7Dy( zZS3pp``63y20Ti@aSAK zZ$M9h*?0b*B+)Ztz(}iiGWn!JUzPOdBuka%Ml5Q@XB)lk`D1ya& z@gz}4;Q+uUB=k}07ukp0NzSBRQX;U9z5$)|2OB0>$(IY}VVg05U>kavHxDs#+OZ?7 z9`J28%8>w;u$}m2%p|p;4Ae| zo(-N{PfHimvB*|ny=XaZ9&FlUoMyajFzF^}?x@aJZ>yREFk(R2&C=}BWKReZ%YZ75hI1z6H00T7+W} zKeUro1RrMK;TXA(0GoV?JmNj#$75`4gLk$jOT@crotK2F`^kh z0?WpxVe_!nf~#mYuPyS8vmO4x%wyIvHb7*wb*KXHi!4uxv##Sm`xE;kd#-(`t)mUH z<94gH(sI>ex6HD(u;tlrI`+75&wkG(PmO22_qs3We-T(192z>)=}t z+!v{Tyg%UI7tjVC0_Uhh;CJ9*aA#;$SQ*WL)-ld8vKTL*Lg+By6>*I9%!_b+&Pssk z>D+S zMy0F4RbkcL+Ma4$n`ZcAqAfEmQ^B#mC3b5D#496K1$|K#AJaJT5B zm;gEixcULcNairsZ1@R#Gj|AY9C`qF^kwL9e!M_|4aSqnUi32|BqBwZM9szX#Wr!J zq*8i7_DD8VmLwY?lgai;#*0SL&&XTEYC=dRQA6m)!UUm`wviLa8{|aFK+U833fqdV zi8hFmL?PicQ46tEyj9XvS}aYI6-#R+N5nfskAzx!9DR-+Abce(7p?{PaggjqjK%*5 zV165P9Xb$==RM?p<+SA7WOLYa;q&k!_DA5CUWBK>v*9jqOV%KUCH5`0657FN$C}Gt z!u^3Lctemz+(n$ToabCK(ve5r>DXIniSST0I6aNuy5ygo=iKj@WNT14{#mVGfR)$gp$kTKR{-ZCTk&SE>h9;>DyF1 z)t?+sOvi^}h+sN;9cj&#aRlrrE0twrc456>31JHE!ukMKg*`Sj_AELmW`~|K#Ef>3 zIyNgdFsciG2`vka4J{11{A|C`ch7gqyU4xE5wv!+G&f&2nM@bWBFh@H%6LnEUE5!C zRoz`(zqWo&qpD988_HXh{aw1Nbbi^>a&JYXGOzkd&3Vj>S5VoYwJAaW_VkBZ+W!tI&P`<4`)=*-&D7-f^KYB8zf?6GlZ?Pr{2Y-1c|TxC=Nz8(R}j1A|(gMs(CfQhgUvj(ynGDVD$(A3zh=*;NM zSSLshWyOf7Df}wr4?Yd92^Izh_%Yv3kJt6qNjYaY&fCA)V8AaXS+dOUjl&HFoklxS ztJZAMywn`jR_ZbhiwuwSZooD)+D%%PZmqu7c*qjAuCgN5Ki20qwXKQ0jiaS=i<5Nn z90Tn6cAlMMBdpIY4=o=oORR)F(P?nKanE&kaAi4fIet1CJGTRW*X?NQxNo0tp9M79 z2**6GD^^Lr~XcRplU=#bv zFXSsQXMa-T>6XH6q85^^(sR=PNdn?_k`l=s$qsR?a20)-a*>_Lk>m>M9!Oc9(0|cg z=@ax_;SBK;2?HQPrR2P%hh(fIUs@ttD>o>-%0uzR@jv3H#t&8cRB7 z?6k};6UyvjnW%#9OsyeKo)Z3C=REIu=8 zd1oGJ$_CSEv09>fQ@yjgOZA{CV}-oDN2##nSn@8VxYAMsZ5ve7g!cSZuH*^{u~#n?~5OF;$Jk7x&aG|$Noah6TKI86W0|_6fF|Yr6*7q$dANKJPqpzxZefzH}ZmA$&>@XrfVcE zJUBcnJU-MjVDh&0baxK~Xz|Xm)KPBt+U@}zcE_A$>0vo-?rHvP#;huv+wt1vcE@`r zxmUXOI4?SmIW{=!02Tkjxe7Rd-JMUJ3tY3^S3Ev%d;e~bepH9nhHr&ehRN{r(2r2v zaACMpq-(T0NKs!!zel{lW7dTJgz&I3@-R9jb|;z>eGn}HyzdTU5Yxw;&l&_cVm338 z8DPv~d;!kW`Pi*kOXxcE8InTFqyL0A2WR_By&Jp-y?k%2dz9;;W3zpqZKd_KWv6A3 z#b};xK5V*a3>rv7txm7)r44E_w9mAMb(i(O4Udc|rf0^@hK>50`soJ0L1L0xu37cg zKGqToWi?tK*p}JXI(j?foc$cL?GtRzK`K1OI>++f>^J|gOtFDq)ENZX&l|^RdmY>f4*zT3O32{Vn~?hfKRni20p)ou#w&A6vHLl=F)#;_mCcWaUK zaztx{O@$@GUgDRMH&UB)rj#i?C&4AdB{JF9xW)3ja+Umx{FeNre81cw|E4&iOpHGt zUq7K!!ms$8_#VoZih1&0^2PE4@~`r?iZzP9iaqikaTBEn#8*W9L=T01g~Np^dIhB> z9^n-J6sIpqj;IHzR*bu_C=x6VibL8^yYQyajo_@{*&q@Uh5ifH z3C;>=ec9d%?vQh!1Go3FmYNcc2Mh&YvHK z(4uiat9~5$;rmhf(^1s&_u4=I6`M*Xmt87vJNeCl>!IP1zoOS;W(EphWplWPxL3KYklzT0cb0b+)uW>TS}fHGal<+(AC7|b%gf#|iE`T|)zwyHa7kCvF^f1v|aYIQ<$r8y<$qLC*2`Pca zWkM_6m|jB71Wx)*Y#BcdIR(FD6vjqJiz3}455slCKZF1JL!OhabmvV+(7w)g1u%+7 zmZ|33rh}&Wrg+nE;~elUuubhP1vc98%OQ3)b6Ok)j#G|D&LY=b_gi;W4)qlA{Qg$qS5HL=!Zx~ z*csRdvdnz8FfI)B&}SmMm2sm8!EY|#I$=NoPt)P_{!J!8Ol%?KIW7*6UF^iOpS z^%BDXLtn!kLwCbkUAFdzX0>LyW{&olZma&RVTbXy@rUt@NoS6;R#|P<_g1+r&vx7@ zx6ZI$wt6kg&2gq1$xchEeW05Y#Rprz0ZzQUfsZHL&9M3lh~19h?r zP2v9!q#%8WgG7*+LLMeJky#`}t|DuJ65U7eL2ofl*hloQ2otpyazWO5i1d=rC?9PV zb`aS_`H~6JEZIPS9YYkmmEwf{i7S$H$)S`{sm)T)rl2W5lU2!%pk2S$)}0fLkvlD<$ov4B$Cdi64nzQ3v5JS_vG~<@6ZgLy(l_336 zy#rS708EMB$G73X@C2Yf9^iGc(-=;?B{bk2%BY4yQlt~@5EqJDNwQ=;wKeq$-X4uTv)yTdV0GHT@Rf@9A;a<`oWrIb6EFV%1nO@e(k5) zNi|=p+En?gE>(M~1XVq%o>rZ&(pGJ&4pyJ3d8%4ho2lNfnX0>|FEMaU7Sk41xAYJ+`94aEk z>%|wutHswubAxpqtZ0mq%qbSmKi!4FPmCfo>;HjYHeEEXW(Z}wF;~wt#1Hx z?{Dd9o@w4?skVA-_w55621lIpkR#18!+|waPp;UJm;-MNB3K(D0# zr4mStXpG;(7UCa(?yn|VP`8C+#cuIB$!%$axDj%je6C`HvT?%K#OkE`$?lZ)X^YZk zq$Z^#CTAoOiJ#(0r9G~$te>=w^nfHuY!to|HWKDi6N$lumq-WdU>7l!h!ID~Mr0w8 zOkz}P>KpkF`GDL<9i|z=t3dO12Fb>7q8@%48-{NoNYDm#V@Cy}1u8){whYtZ$y6D= zR9FDoXG~EG(RHDb9!RUGHXrNeNsnDJThdv4OW2M!kxxKBJ&LyjpScaO z7)#{8<8|ZxgY4#x;-W}8Ps4l7TgWr;Zh|DSJ^GGU%j*R6(qN#U79xF+JD^i6VGm>V zW%gq3V76zTWPD>ltc|Qu%pAsU#u8>-W-)XqmKi%3dmq~dT(gSM@Q^8x>L-0?y`#Nu z58LyXYm(!oZ#uhs2fXdElMxO9zaiIy5?L3QZ25kF<{lB8q5(*lp-vMq9>B z&`qify^hw4%nm;cAC2^gu8H=Dr7@m?Ugi;|jd2GO$0{SEBNHRVXj)G>B5+9lq+JwfW3VQXz!Z$i!8EGNucKw3OjzhBo{dl>YdU#iDxTk1CHn(ONto*1--Z-zaF zafV%nRz|yVxT(Z+%3NyxXi74&4Ss#8p~5)8bOB(b+tSyXX+3Y%+gdmz&Lg0|G~NBn z!}gW<7Wq|ve!$>A( zN@yh%i(QM3j5xw0BfBH^Fef}Xv@Vzzc;r)h7rR-Yhx^h#-{!T9w!~YXSk;yV=5iCm zJjom~Keg<#Zm{(OF6BpCzKyWeSc|M*t&4$oEwHdHKh4)nXN*?kE7N0Bo~eha$kf%` z$b8QvGS{&#unQeK?PKg+>|22Mfm;@tUFQ3irj+PP z#Pc(qE2M?5=)Lq{x|HfbZ3oO_4cU%5O+BC-)EzMMUx`0T_DLPGiE^=mQoNA=EuSE7 zCm$I1MS4#%PLe9wCH^IH3SZMRsFq|E-T@mRFbUoQC2&;mTF?bJkHZ8F1s?>}*b>}^ zC1W|*acm*>M9@L-KLLy@fX3KQ7|11bGoXot!WdmiU!*tFXK4$)Lbz6xFQ&u=qP?Pa z;vM2k;*dCBazip#+7~z^=i>Iu=P0SP+#gB;prR=I4p}efzubia3qRfwf z7tc$`OW-EHO}dvw@dzShs<#$S>)Y=qQ%GBgh$@7wp zNsAND#M>3^C^%Bc{*;t%L_^^F%%{$h+lh`~ zKTeULm!LmC9ZlplLqyzrY&+`!%lo?;&d(#!gG*e0{~x0u2i@TdDn{##2bPGJ2Jomj5{ksAgVCKFFriN|-9BL7n9laD=3zY(9PeUUCvziwX zgpUW81kU<@_;z}Yz?E}56pp&~gVxTLDPU!fwQ4QHtov;}?ayqZt$wguW~uSCUaGsH zEzoK8e;aq1hMGHBoR-fPnPs)PC78~~%w^_Oi^WoDZEUkx7g`g+>>g|#Z#{1PYTNHf zarJQz@eK7Y_iguc0%#yJFbwn@{{%J!yM?H5ApAVykIadbhucK*qOYU1(ND2)jNMEz zYb~oTY=_6N8?me5SMVyhij~7U$NY!+95gTjj0&g?^uO5s=&;DI(1PHEzzBbi&*%B# zehA#v3C>iSbF%B4YrgBZGwi7E zm}ehuqpZCxy(}9orz|HeHp_2oxvjZ9!+zMl8gw+DIjnYuJ(pPI85%K7Sf2Yn3mHUfHSxOv?6oCYW*X`#5*N~Odh9; z`yxN6@GEkaE0q-RHWwvaNO+N8OL&q%Cv1)%7cY-LrJSH#3?A*KGyv`KPEoD+pW>+E zk0MQ3poHTU3HuY{ljo#tPu-A4rnDiq_nQTmkQh%pzNt2|HOdpco zA$?T(gLH95tBe5|%QLQKs51DOnVHQp`(}>MoR_&Zvo>>97M=Y#+n>EI+n9AYs~~G+ z*21j9tYFra?ACSa)%lwJPj+_pmMnAT?#%X?_cBIgjLI04@hlxnC)2;Cj|YFX&ZwW^ zORtmuKDAE@o_s0sUHmFVpSYh=o5U`DAo?NfE__bwsi)*jkTyRT(EJ?U7O?-t$bQ8h z#hw7qXXUb@%#NTz=7pX@CqVn^Ih4pSGww2fGMh06F%(eqC?}jAT;YG}o#eUaZtc!@ zLC!<=CbsvWi+IFz3+ylQn5SFrTaH^i<`rt18}< zuPnP?^5swTSC=B)j}AYY|7i6i=f{>GeSa4Jbp1q%vWq4Zy({Ya>*TM%uO`3S{r=~- z=+A^d1HoDRXL#|wVsXjGlB}}(AS(%0HmIInGhD^5U0qvQyI#FQb3`lA7a3AbdeaT_ z7K_}P0`!g%v?TkP+nI-(KbRj`*kD)5V*7J|4g2hQ_Qv)&aD1^}a+n;I4%j)sNw`M3 zA9|Yk8U~U>y~9xCVB~vbJ;)ae!>7Wzk&4K?Xi?M>y&7#6-5j|bJ_?eL%)nUxUEc-o zdCz3eB2PWf3wNozm*=}D-+RFepwBzgy9Kxazn!BTKW*)-A@dybQS)Dx8P=t?F?N$( zZ(nVv?UgoCfpxF7*18z@j{n&wJ8n8AfIUDQ`%Li2QtKAWPP5-M$aLC>8g2TeI=5D!o2z}U zVQVY2r*(bxC-j^2>^3f-_b8J zTr%`CX!NJ`&-8;0T;o<_H`77$et;Onj5{SIU*ttTblY$<+AN!j!%#bV^1_o78&g%`^R3W9qc6%gTx5{Fi$& zcXV#k+}XK}>gClR(O_`HXv3b3S~U97uzSN54f@vaU+;NtL2j>HN$#PXvbtq;0@>%X zhh%fJL)oI7*STBjy{Nad9#trz~5My)~_Vlc- znFBKFW%SSJoY5^~PX;TqLuS9sahbjhZ90-ZFYQFCCFS3grYTEOx}{!EeUa);)u%2_ zm8WK;u1md>IwDH-bu$v%OvZ?6~g|)!NNQAd1@{h!jEDJg7s)F?-iHg+=9oljxhQ_W1&>W4bWa5 z$@mUBnL}a~k#*rWp~BF)(CN^9;JM!fnSblhlRy#B;c8c*1GQ;P#|_oG|7rdK`weGR zt*^?jt*cq4dt^D_?jG74%Lxa(yL?XriqM;Iaqy6rcGtB<^enKibhWx$ZMCX?W&M)( zzl}eqe4F)o^asr>A@||?2Ui}9elYAoqX!4?=iG|JwXp+2-1yZ(<}dhNQ_yco2UH*c6M%A@<^D zu>`E1z>RJ}U!#-IUA&RJ2yYWV9e+(G2nR}z#_d*giT5Q;N*n?5rhN%#6!|iySVfH? zE(tCmYSzS9N@$L+pXa!{nHzS$0;)RSTj4nil8hdHtFJh)JUl1TAmR@{iKNF&(SD$n zzBL>SsYA^}PXjZ9+rozMzVNz;EarpeGp8|cGPW|dmSo)yJCS z)uXD$RF189U;eIqUd5cMEY-%^PqkWgt>%t;rRsQ%tEQp`sTouKqiSj8=kmVg-twar zuPU}wJScBjwzq^|;w{ao`c+HoKbf%`@hRFbY7 z9LJUYN#bXWPwA67y-sd!pSqXpUakMJv8*ZFMAL9;{ciOh=6UFJGm@_=*VD8Df{*23MSehpJN>V`zoHinDN=BdT^6cbnI@^;uEIl*z zOVZc)S@G`^E0Va$@u`lqGpW&}mkAC zvqDthAAiVu-PP2w3*-#f96HcFJZb*|=8M#R*Qzil85U`_*R-o_TLzUxivKH}P`0Xc zL`iP3^EdV9O7Xl>ygZ}gYWak+yQRlU@08R4MA-lHL=mf~*N@=$mPN8Z1%G6}-~L+l zYeCW4qDj9N{JK!Y|C#oE*;mV_iVuB1vA+)bdhkpBm!qHQPwIE$U+sOad)E3H_H6R= z<1g>N8S$b2=hCljzn6bq_<7bR<)^ZbJ3kHk+P-LQF;!|Q#megd)sj_NSaqeQjXFUC zX@?k2n#Y3%(Hzhb&ol3~o^?!fOMI7tt)n)`#^kY1GVU|}$4q8wpgs@=#ev@XX6Oau z2J;Q`8FLZyKW3QK0uC~1R%hU@+reEOwHT`%WEs!zSLe} z|Htv(Io_S-8R)s}S?$^5j=B=v7k?pTHXt0?dvcLHRrrN6t-08iKtJi+HDz)@&7{1=Kg% zCUnd%_VxA-^ezC&=x*;Xe?oAlKf!Bt{b!$Q?x`dmlIlHX3`01~?--mq528?gU z_qX5G-(5d?6$O8|zOVTiD8hc{{pnw_x$<6ZV@q!fjpz0cDL?^`LU}baJ{hr9USCa>vp-*e9^#v;meVw(M}8_*jw2cl?l2E zE`he`Riq02$zRXki$Nrp_=T+_-cw!aS>$*;7wq@okq3zJM18C`>fz>bi+J4yXTftF zf?e25Y^C5bf1RL|9|rhf;y2{mc>nS1;n)7(&YeFsd7yCsPx8KR5^{1RAqgM}h>-}y z0|7Y{yDN&qYQ>gX>VZ<}p{ysUMX8Hg>{t;&5N)mZg|uF*)}tV(T=fD3JUBv-1ad&` zkaM4%{R8&Loz7Nr>$Hc~ehi)fu~@O?T0soiOI4a{lsor|18#|rym`)2!gJH*mo+W5*hmbz1%nwub!uwtqIrc*4))ZY96Z2D&tiL z)QOt#(U`HQu`{FhMhoB!TBO74G-Fj`p;{Wu{~zgiqTkgDYU*X zj`%7$sytN}{Nvc7{tW&l*D6yW8RVtZD%YsCsSwp+Wwx>sI%u6EmxkLF@8x{Cb|_Ka zCGVFXkmD7X6!+kscbOtce&+pK+0ubD>FK^5J*(ew-mQPPx96wc$UbXteoyY(lGoyH zbJv<~?DbN&>EbqB+@_1$ba9(5ZqvnWy0}djx9Q?GUEHRN+jMc8E^gDsZMyu&ba7AK z+>^II&E##=J$d_6Ox{>;yL;IE-UC6h)`9H<;RBV@iPFgaN04FKDy`~&(f{9G_m|f7 zZNS)X9inSxBmO&R7bmUvr(lSzm}ZJn1&-~6S%-9To68m z;7Rxc_C;%*Qyd!mV@IWZs@21?%ot*rZcuiwoXPbMR{;pRg|0`g{jO+SCH@V* z6FN;|;x%##^)uajrwf8ckEf1m#||2(e&o?Z2O;D2<&48d-p zNw`N?E({II3ECB08d51b7+Nb%6<-ZK8hT}NwkTe-U-Y>sR79Fo5?myFDTsn(US?of z;DNxRz?#6Pf!+c{_(WJQI5)v3;CsJT-d}k2JdEGOvvG5|G+rt12JbkZ=zl%n_rO9y zabW6%ul)OX3EVQ+&GWfz-bcJaZmVxB_)0vVt6n=jS2FV%59tj023j!n0c2f|Asx6e z=YUgyYr!`lcd+%uGOQdu2%meptHqfIlYJ6+MI=Hvp_=eBVGrR`gn`y0PoQ(2kNyoZ zB>zO`qamn28U(HGV(^NhiKWDENh8E(=o(}_ArH?cY(T=$Ls$o~jW`5a-9}7<8o@YA zMXzExL@jZglt=C&RT0N9D<;MEVRNuMU>PT&#jwZyI}(f7q20d;mkh?4&N<21VK1?r z7*8-48#ApO*LU%!6Pb^q1Gj;TSgHUkK{%RtbHSa*SvxXEf|H*yyGUiD>i7jZMWA48E_%`F6II^Fj^hs{~b zk#UlIv$@T@etw*PaKHx;hphKM%rEx!V`X?I(PflZt z0jhfeDTDY1Sco4{F;+;NNlGH6kk*itq z;jz4J?2q%Q@o+9m<#WInmn0|C*=Zkvy>ph z1vmwcSjPp2*lBgjUGo8@taXIhbF2$2&rOF+L#ET_UdtZqT6+&f^?MvFr`|c}D#az^ xlL!K2BQhW1AYx<{k_+U|bKs4BA}2zTx{9{$e|O2Jo9Sr`3IneE8xGQue*sgL$#(z% literal 0 HcmV?d00001 diff --git a/tensorflow/core/kernels/fuzzing/corpus/decode_wav/7501f79cb067da108020579ed654349c7933d22f b/tensorflow/core/kernels/fuzzing/corpus/decode_wav/7501f79cb067da108020579ed654349c7933d22f new file mode 100644 index 0000000000000000000000000000000000000000..3be6a61cbab76ee7142caae58d99695ac573a46d GIT binary patch literal 54715 zcmWJsWpo-@8y;YQVPF&u5cf8zyQZ$G>(<>Z?bf?>-MU+K&gsZq$vY*uhxfw@`4Da(hmhm&Oe_T( zjqMV+1aiVmL`bdhxTp?2n@phWWEH=F-H>&IX=XHK{ll*0Eas}Y-FS1+Ow`Yt&HIJ) zMsM)v3J&6xcpJew{vvD%agu0(Z^wBgpLmMx!WR?o@MhRntS>PJHweDq3&>OCFS@Zb zO_`anJT8kaz;9!%vB{W}SVS5LiWmrvYxqVyoft-LpuUqfY@}c^mX96gkK<3`XQEEz zBDW>?0&@K~;$1-9JPB_s zKZ&2o?ZsV;?m=siTF!A+a;$yG7VHrI9fHCGLL0nKU6Z`*z>q+$e}T85XN+s6vx%GS zIpzFdEier*eKY?tWf>KQZTjJ+ht``GwJ}G3OI=6(KwDqePWw*BH4M<5)!xuwGcGZ& zu^cgVFuc-N=zeQftM{ornicv5#%=oT>U51!*GGFvQ)Ad@)|=Xzij1#}&x}IzLEAX{ zb;~Bp3fph{ch>`7qrj@b-4Go;7VZ^%5xN*D3V#ZX3fD&8Fpe@;vre-Qakg-m@It%> z$X)h+wvMBOTgA>scgJ2tC9(F<8t5@&06dZX0xp6(!9N(u%!Tj*CKfvoZ5*u=JssN= zJ0HCmQ!@X7XThtPPH1#=YiMau?SJVT85kM75IE{z=tVq@y>kN>BdelMg0;TSo;Qvh zTXSn0^EOjM^IhW^om1Vp_GN9B?v<&D^^B#+^jW`AGeG0f4K(gBKGH>N`_{hHywR5$ zmzmqy{Enz&f}OM>K15I z)Wx-TG>dg}b!zQP!y}u})y(B}-|^-6NBVkrPrHY?nz{E1HOjA(a{aevz(1c zPwqO7g8P+yfq9qlhxMGjoZT4y8*a=MqjmWA`EgiX0;LQj7k`FEczFVZ_(r4>4e`Ag zoA^bg(bI_?SOOl#8uM*PZ(a(z8vTOqNB{6TAWOh^mB$^74CUVjYxe|Jf=%ZiL_TqZ z>=E!on8&WiX0Q$LarRd3dQ>5JjCzm+@LBG15AnL8*Lmsa3Vx6;5lj;Z1hxD!!4^S( zbPkfroyFmCZ*sdLG}>8k9P?lpzKKv$&B$5AM{0n`E#%NU$PBVS$tPX-3alxA8@C-B zLh9qc>D$6fq@TYWZ77(=Z@}Hi8p7xYO=r&JTt$4m{(@6@IyG5XOy%KfYz|SETtHo; zr8I}?OgZR>qC#<*TclRZ@|h;OM#ii?Z$ zOB+jf%kE0oh#CqP3D=AEibsgDgd;>l#bYJ^lO2p}7S|{4gzScdC0;6W2@9yb_yepy zHdXLQFbAXXb~qn9CD@Izuzmb~0urBt!2r>Yp;|Ovpcf?ab9tMQ5Le8#!0#E!(U1P^ zo+4jz@O&UC(A%*W}slS?uPzlilrIKOO5GkL^P1a`Q;b16z03 zA@?mu)Ox|v(=;ACpz)aPpx zYfskZXx?k1`ZJ~<<_dFDOT2lSQDin*W7a}zI~&K2I3C&s_SR0N=ZtrWU+Y)<76ev@ z$4CANcMBzjCWH@#zXdl2!a-Yf7R$sbKu&YHoHDpC>pBBwoPgRv&RCn6B(?;)z#PEt z&waqT$7XV$bB}Q5b4GLiW}7)LkX$atu3)WX;;e^o54f6XVjN^pjCB!K;GX|saB!$F zsPk{|uMc{H5x?2@-1psk&Qt3yc3pH1cYbu#x0l<-+TGT#<`X8Lv4zQN+G?(6xnPbr znGG%UU9=(fZ1pWoiZ-z}MYUAZLBHFGSZ0|Am^5adwa7Bt+SJzCX0{HpKD1UkN<0sJ z2?1y5O7sep#ca&H%6J0FVoPJQ89f-QAR$A?AXwj6>saqu8{xO?1zbOS46{D-A$09%#C#&#JW5J*pemw66K9dU5rt z>W5Vqsy0?%uc=kNs2!kQp}wL{)}GWA>jvn*89JLjnmDFP<1R~rV~VTF9p|ZbS)Ci4 zOC3$@z3s_Pt!srl!;|lu8*CbR6#XZrhPJVGv2U=ua}eY!_b2-SJRZIei#e0HUwFmX z2oj=_>3HEax<56Kc!7<_D)1BJEwCC3sUjgNz9V`q+$1E0KC%LPiY`KoTnNeM>e+AL zQ}93VDb{MZJ)+@LSeQQu{g;=)OF;@a58&x=EjWk4C)j#U3uHQPD{l*LF0T`k%pK1e z$gY7E>`v?g_8j&Z_I2(h^eg`oYU0_^QDCJ^E~=M^I?WHu)c9>J_+#Iq*B z5!MjaYxo&A#M_S|=u6}Uw=;5-SAh29L)d%#BQcHiQu(4U;)#-Y=?7_^Y@ck0Y_qJ3 z?1HqutRhY=pQ*@Heo%If-yXj$eqel9S*c7>dg2Dk{L&ASM&c~d|7eO%621`j6`huN zS+!3j})fBf{i% zVRvP*m@gS?nGx1W_yjCu4~Hox2Aznk4(12?2g-cgy&HUb{}cau-y?6LFVXMwcMU}R z3j^&#UqiP8KfUkVuU#%D>{x3%Z{P3S?JTx0vL>1HjX(9Rb@$ZUYA#oetCE7FcePNp zv({DHuy&_vnkr9qK{dOUUrVdnsBEf$N~Ky|tI&vbZe4f%RztkeqDS<1bp3Sm^o>jr z%TPP%-0QmM5&HUhr+9XI$w05*=fLiOBDgtd4DJQ1cbl)JPwi6#HU+ie-Ovu!Hn=g| zftAgC0x4o`qM-;GT^B21>|=d`3pneM+q|~C^++NzjQaziMG<>GXFhi>#|~eBkHdf9 zo$PV!p0JB0ft9R*j3cpVcxx~*^fo*pd_Pnc8XU?CrH2oMM@ODT??*Y2q>v?WK9Cz& z;#=s^yS_U)&OVO04!!e_tFLRA^KbhV3t?VtYGqul&((d=4%c1PP1Rl0u?z*q!^SIy z76yfpVfI=?wr+N(W0P~O^SN`p^SbkkdxaPCW%#=UmIglhS^hkK>%gy|I`SE6%G}P9 zu%(>UoF?paRu!`&YaMeY<1Ta%n!^~!#F?+4?$BarRcvtdV{{xOWu0KZho>{`jOGk; zbY5g@xOu27crug`t_atQ{)i&chT&y_yWVeZ*rl~ctT)Vw#@+hcI+?Dm?yN3Fzeayi z*FwwI1XYFA@~ZX~C(0aU`4uB7ODc7h!>fi?r`A-~yjJ1r9PQuwcte8zp{|c%u(6B5 zp{I=pjBQN=ENWY}tFG^HfE{E7M*A*%y1H{+lbodvp0mAcpnI%mm#+Sk3@+4NV%yB>NVA!P814+1+;@+Ecz|( zC+Q$DOO8sPNT!Qlib{oBXdT%W|Cev%jYQ(P`?wrlJ@hYr8^JyPK$PbBxm~!kxjt?I z(w?`S=i#CJrGf@nT`Vd%ARq*|;6DM5nFY%P-2_(z`IrD-Nvxz2g_NkiXsU20{fv^6 zm#{kgJl-SZFmE^d6P?O$kCo%Ea1uX;T@qa3L;MXq6j37U5et{%76NW_3oyYvB#$=> zRRXNofHdbQ;BNrg-$uTNn}>75XToujNF)}!&3q1bXNTc&>|vbuoadZpoB|HP+0Uus z_T)9@zrlFG#)b2RXGK+#Z8D#1d>l(&6vvNiC>sM<#dc{Q>2Aqt ziAFL{k|ovyBppv{sT}%-u)E}=j4!W>lf{jexuizvbE!_kmBdTlOU6n~(w?$Kvj54B z%2r9|h}Q{^&?sF+t|k7(-(vl+%Ys?_Li7+?!iz`la&EKt!rNF|nbW}Cw*ktAU}z%L z3-ZKvK*iAXSm)@MXuH^n=)K6Za6-6acxL!?NEtlmkMqy-|MDY&*@2CLW&ZBoR<0Me z`j!o*-G=KxfArJrtR-rX)iTs$)FE{{%^CH&+RLi-svjzcYJh50&4HR!RTq^{^-z6C zo2FlG*l6rwx@)>>YGs;W+-NvqV3<_qJ=S@4voq*E;(6@p;Cq`st4;F#D{!*A5 zxgG8nR)$-IH;2U$arA4fjL`v1-FWyvI3LWa9_%`>g*le#XS!M4VIMpi%&L~?Lf#)_ z2+|pOg)9QBGMTfJ{T6NjlduE4?jXDeZUNVWqpUTo7OXNR%v{KL$(YA{&z#4sVZ3H6 zUqK~5L z=)vfhEPSw8Y&9?8yX)R<$veR^j`Mt zboX=?S^1`ex-06Vsz)_@Yi89nt`XF%s_s{nSlOwfMR{>)b&0h^Q1+wzR3%hR)QnSY zt36mN1LsloC-n|>N6lj`sc&QGV|;6L85fyirtRkT*4y@UXItk_hs;6SJJ{A+Cs|gR z@0s6QdfWbX(C&ubS-uwjxBhDXOMr;og7rdGp5dSGZORyDF0!QKsd4)Pl&lNruW(a4~>*zmp0^OFJhabV3W48sR zf+=_t@-9#Xa^e+sp1+DW4Cw;a^54iGRpQ5zVrf(9D)BkdLD2{iOH?NOBn$|pq9wv}^lf^cu)ZiuyhXy66~rx<|0Qo7 zXOhm9oEIeuXV5u7?OdjpiHZS|K=Su;x4en`wJb}zO`IyKp%+oL#0mU7c1*C7Uyd$9 z>!6o;&yiJtwLW5V*}qvGnKa`nbP}2ZHGy)Wx=;hC8#uEelChQ92{7Jv%rwTRSbk(w z=zD+)RQPZB@AwA<-UfOE>iTba7rIwFo?6eDOO1TPL7i9Y)>68V_JTH1yGipveX5pG zyS{d`dV#uyda1fZ?NH}xG@2@{N$1cP8&(@F#?GcKrX{AjrWDfMQSg1P%gP= zp|8aME+7dl41bTbj1kZd=qBSE)5T0?ZGziz{^BZ;(};k#fp-mfF+zmH^|0HqgK!nB zh6k~8In%grk-a=0?-ZKD|BkjnN28z6U+5TgCGRlO8j&F}?keOpkK%jz9R(-&y-*cW z06erV@EK-j##cxKZG|dgJz`siXsAVt;4tYZco< zHjabgYU>&8o#l-OTu9@(?igkpW;th?Yn*G`X?kf6SWek0?APrc+dv!uQ+Z|g+`%uRp=W%xvufVt6x5B?N&>Co;XQ7?pYmraUbFqAAD)cILEp{3zgBn6t zp#DrL>nU?CK(Y(WuS_K~p3#Jn#r(qT%i_ZG;Qer4SjU_VwT^xW|1VS>{5NO{whUKA zbkWu^X{<%`VOSn&6KL;yW9<7(z$+RDu$(+NXo{UBWn-6UPMZkzU&X1}_BElYKy zdQnxs%FPuu6>X~8R9~yM)EH{VXmYd+omF>JZ`Zfbr|F(*uWNI(Pc&xD-`dr>B*PIS zY|gX(v9EA8a&2%;ci}FT^IvDsp>`BHue!FltK1@w-NW}a^*!*u^!E0x@^O3uZ@Q<2 zC(paY_s9P#D31({i5Lb(2WAcP8|yGEW=~{$+0(eAfhOoAxP@u)8sZ1Zqt21jsiDF} zqW+>((Rk5*QJUyq`XpIKd?x-PY8BLJ#lqIYD!P@>A=C?d3n4I%CkQh{UqwyD1fX*kiA=IjoGYFqIu2OPS$YV~rV+Y; zUPXr}f<8jW(?w)YvLC4?0{9y25MVJ^d3}(4P6j)X{eXRr(}w$kJB`7(O|+;%><5$wo*O(oNDt*+*HOI7VDb+~+t_el$)LcSzPj zW|#hu92c#o)2X)PWnwH*iuc1|T!tz6O5R=WY!1Y(VohOfWwixsYzpTBr!}WOrw@n2 zwlUi2r%clz%TmX>%=*jP(w1!VStr}>1JB~B)o+alsCdpb#M8q!I&e8y8!Qa=3APN52zCsX z2R;TK1W$*bNAJfT#Qp&oF_(1~xE3ckGM*Og#6OIVKu@Cr^a^r_Tc3NH8%8R5Z+XMe zJN&EsPrNOabnmr3%}C8SO}XZ~rl+QfW~XMaCP}kU-Cn(2^Fa4q ze@(B|A24_gEThjb(3o!iX>DuoV~^R7I(hCZ;Qoknv0Rk9!2Q*A&Gp*7#B1@f{ZVhB zx3RCY?_V#+8}KLr$0_t&bWKI{3dv03h|WqLe3#4;Ex3^ zz5zW2X5%U34euR454Z~#@omIhayPY=P8X($mWg%Z`r=lizQT+2OS-vmlJKH%q6ilA z#gjxGg#!SakkCh|Ut}M0CpnXPNr}KZ`UZ5;A8eRlC0{O>hi%3Lf^Fzw-aN#}X~&MR zdce2YC`STV!g5Yi?tV^BHXA<9$^=O6W1WV#aW;V|Rn2S18^(Kxe&PS-JNf6pe{o?o zAko<@C=(pU67kXaLi`lSXa2&~f{pxSep~)D{xLoWIF5603E7<5My;a4R0!~^6~fn| z!O}5tY4V=(N%E)iY{hH2Aud;bQBkSPjpxNbSE`lWkP*N_YJStSM2{7_!is}Y7ve_ z{LoHT5qy|^hhyYE0&Man@`(3{ACIxI6@mqV2ZEh~j{H`<5SNYoh5UzH;H~8E!X^`a z$Q-geIg5HkFBFD_l_HP0MDj&aEf$GAB3g`z8;I|V4@kyKr%4Y>wu*O(nhIx7$B1V5 z2rL_$hRwrP3$CKsytc?U&UW|%Gmlxz*Z`5y)}ac(FS0x(&bp5O>`&~E?78-#wvIN) zj@zx)O3PJ?-7?GC!j@;h>Dc4KJ^MYEJT;#6-s`@g|3zS7aA@dM=v{DCV2XdguY+$r za9^bU@&15+UqBmp2%Mu1f!~3L!JVO1VP!M}TE{rY$YQ*J3ZcV*SHv;aGcUsRIV%CC zr*j8#qntyWRh)+$h`W@dV6)&ytVOK3%oh+Z_C3-&!j1e0w~lm<$^ffq7JVMR3G~RN z(5w(MbUr8un*0NOgFFLWc8Ai@+dj&+(7MO6-i(<<2BTJ>*;Lz5C9646U0hvI69;tU z7?rLDSA|t~YkR73ZJObeiMGtNOa;eY%LL04^AmH(GSQY`A7#I7-|D#F)H#niNc%$D zKwE;XmrZI@ zl1k|T*(2FdS(0poOeWhW87~?|KO=7us|g{QL=B-E3loG&+D1+wZ;%ry12vECD{L#e zCfXoM5`~1*L@mTt@m5JwX|Xg-RxGWR91-sjJrZi^ar8NQfbf;DT(}nC$3e0eF&6(L zfcb6Eb?87ep7)UZmD7@Qlg(kzh0nu_*dKvkdJ&!i&xX6eEm?yYme{x0N@xe89cwOo z3HJx0;0-|xWOnJK{8c1Ixpd zSUkqWWY{{#u2citW=hf*@g9jC4?!s3+n?|753QB*t6)Mm>qh`5Hs3A z>e#HW{7zmFpOkQ0`~!8LgxyK1djy$sh+p>BiQ;~wbV z;mUQfT|b>g&fTtfPtd#CzauajxT^mP*+OqavqN1%TSJNAqVV3x{OHM;3TnX^3QdUB zi+Q6PqgZSww4ZU7v5j$sIB)-Eg8{#oWXUqWHx4%#bQ^#y!{F!IkB_<@o7n?A#9gUALpHygWOvneo);IF{qEYmKfKBWt zzmTuMoc&3Sr&|iQiCRduO3z9ECkcq#NlGMlBs;{l!d3KT%0+e}N0KY3dmv?bLjOf~ zrBBdzg)_uYBn*HEm6G$49+I(=d})bnt=ypSDi6gM$Nz|*8b4I&m$T(l;xw{TvMsWS zvePoZOenL9WugkYGqr{|jo-#c5Mg2{(F=RdKgGYyZ_3x9)#zQc7rF*$pSH*#L=F`4 zB6bSA0{k{V-~{e0UMaXIm+`lu9eAgZ8e}ss2i<_qK@(9Hnuopw8Ov_L1?)OLm-s}0 zrU!Y5FyOQBf%rQ71zrLC3pN=58f+|8N;--41VSDnRb(}_7hL@~(Q45Rahi05tR(J+ zVsE@K@p9rniE9#*k`5-dNlH)r7LUiL#s3rUj;~Fqlk_r)lhh+&mQt)3BX210ELX)H zm8m3Y;)%kW043HEEr}*XeZq+6#NNdE#YRT=MRFn^!;J6_Fq^*lxW0FuM}W)jw+*mPvG~lW z<(+w?DH}|u#cGM_P4&*|F4cpoj1}_o9;L#PW5o}Phn5ISZSd#PuO$Ly)~{_tJ$&kJ^q6vQ&2eb7n91?Cu5Q+P9b zBv%d^7Ol}Y=sv!gzl(nreZ*VI%SO|{+!+a&W>3O`7h^vKF9j7?KcXGz(L5(Vkl)Gc z;C>g--^dGgB~uRkny!(w z@Zj*Q@c2;AfXUm|)7?D~pv61KQb)PnYr6+@*d240rHAFTxu^NF8MCTvZpUkv+a2$j zBHas$>n4TFo8#d~1>Zcq028l^(xn|W{ z`&dgXl+|c`U|VKi>*(!_bM|x0wokA<2dVHB>m18_v)}x~GQ|ddQD+cjKW`kP?R9J? zt(Yxht8Z^=Yh^9B^tbdkA2RJSA?A1Hb(YT7e{9)~Q_e50h`X=%j<0LL8LSR-Vl5eK zS<~1#oMxOYoDN(K_Xc9-74kO#zw(<4{Uv%Msw@5` z$`P#*HWii#dx>94-biiInNp_ooCKE)m&jya;}*;B%2o0&@>}wg^8Ip${F~y4GBN&q zeEoz@3BTfV;(I7tD(1<1$rsBH$iK?lD%L3aD)z{G#7&eQ5ML4X6Fn666%H4w=oOTj zc!X2+BtbP_DliM; z@r}e-l1Yvr@`>R@Q(_zegVnF1j?vfYHuM#WMKQ=*cp+v09C`+L{zz;W7QxoxWATgF zM4-D$@I%A{aw^?J#FQjSH%hz8f`D zx#^t8@Ff<(TFrb9a{a!H<%|plA37W*Bd(ZQ(sedN_ zLW{=ztom`}hwn$_Pe)P9-)sN;S8OVoTz0AaWre)zYW3f$L+T7&xuKOAw_XITtf>9H zeTZYDQ|@ly?c_HHu7`$4{)%3YnHea2mCfNE;$G#pLVhD0-dWyRRF94VXtA7^ir(aR z#TwyraWnQyP{Qxbmw=v664n^7xd7(G{>BdzT;Nqy(8EM;#SJAbB}*hbB`YLLC8Puv zmkF(OV|ocS6FBKNv1R-;qQHJ|^V2Lx6rWymwutEQ~o^QBqP#aQ>_lyDKH6vtfV>qc#&_C5R z)JqHl41EoE4BZWDb=leW;}4AjXg zG=={^kb?9f4iZ6P3VE2^L}rl?xr(d>N^~E=2ff8KVIR@IB23g;$OT#JA<|1eqkObg z*g<3y>Bm((Wl zbbNEAN}&Z>|C>UnyrW1|6w2p<9J9G>x9pj;gXE(~Dcnj;CL0io@z%szGD@02mo$~m z7A=*yr3+;9Wc6h~0dBpJtdz8uGJxk;C4MA=MID5@XeDq^m(yc}4?$9zC*%oVQ0G9O z^bT0P1283iAK!-m!V`e{c!1Z%PGdOnme7EAD5Dw*Ns&&pLtH3sCCQTYj2kL@EPXG_ ziR&$km(7*^0M4aVTu;=B#_9G#xoEg(il{<3kDf@jz+VW)3uX!!0xK%zRdbTqE#cp+ z5bGe!1?h(${s7u3e%2h;6y_U7Cy)}p2ImdteMTp!acoEQMkFs>7F-5e4ccJmAR3$! zToim0`0P*e|MZ>qt@DlcCHs*?(-bUkzqahPoZ>j!I=&0*bdDKq^s__d#E zC)IqZYE$K}x>W6{5>)l5dRle9N?Wz9I#_+C=Ba93ZKis^W~%OWGac+T2 zuupMjq_)he)#w7vn% zy}zZad8T=nrP}JT-M0^P7#wlVLyk1Z3iR+|({{`k4ywQxnr_@F`CLFZ!VVrT&t>iyqPs(AVh47_yCfO&OM!)^*kd z>uL*bxow_h?rUCX{$ULan5(n@6JBX1&)(IopiQ|tx^le^3(FlT48-)xn_=- zp92qgj74U>Vw>Q2>OAb4?`C=RK6YScplR?{Xj5cyY$>BT>i}GbGsc+;z@HZ8B3UTnZ?kdSZ3^C?0sw-aLp=0!$YP(s-N_o^^W$s zJ#5cku1SvbwruMuGspD7KpRB*H0`n4ziJv(_o*(dc2>bv$yEocs48V;qsqfomujj& zUpNP(af^-h%o{Antj}ys91>Ti=Ye;&FC8$7>d@%0DKt5JKGHrKh$x~BVz;4x8EqLi zK{u%`^g3EEGCTY*d^FM{x+dBqmd1DndYMO6u+re|$ z_0FNN_XMeDhOM<_y$LmUvz#z*0cr7A{eE3*?P1V+eyJX(ZK>O&Yp${V(*=N$ZcATlruDp4Z)@R@IFEq-(scJP z58GGbTjW>y`2mCfj6dH`_@DYuf@~-+k{KNab!JWgxhcfj0N-SH;^c7z96!kD`>^sD zE1{K8EOsqAGU5o2jO>os!<_Ko(7Iq=;E_-1UF>Fo9_~x~e4E!Y+7fSlVpUrfn9EHJ z^CWY~{M53`y1~{DxRf7l`8L8ZQbXaBm*~4n!dpq&5pVVy$Q@a5#4X_xh*!FA*v1LEoS>T{{%3u02*UIVIY^#&44Bn3S)F7eUaWwpQSDI3gKE&zL*jhi1v!w ziFb%Ei9_Oi$qmU|XJN-M-o#0BDKVutu{VJ_X1S`E^kq2Nd%`%_Z75esqxGZgNLgr1xHPgpIwm?5bd%mj-9Wwj15Nzj zz4@Mdt`$y$W!+|HOs4qReh{*luar1mUJy$ zRJye^rwlFMS6*4(pkh@;RmG^vH_wqp5@G zU^oo+8hfq1vtzw8(=G69^4#}M^zRB}fSLOym>RkTaHvIOcJxwgEmR7aJq?Wn%xYdl z5I!DU5;*Js;oIpo0$0xMP&n$^4_Z50rht__)~dA(v+lR`v_G?rw)(+tnWe_lda3S) zwm_%V|83l58fxxjaaulGWR}(DmS8#`GnbiDEf!0qwXw}&U1&`NvwN_0y!E*Ct8Kp{ z#ns0>#52^p+_&A&37~<@z%bBr{1ey^>=vTJf$;N)KQbp$9&QuKi@uK5MnA>IF?KV> zthKDVupJ)5Zp5yJU%{*3Dpn5b9P=OMbI`yDFe;!n(Enohqr)PBrG^>5oxn+@>)p#7<%fpe?#6YyVS&dIKGuKBLt&ak7t zW1fAujk5N(^s;QUoU)v>*et)T<+kSb4EtgGYS7Vq=CIls_AFaVYt&q1lAB%{uNpra zD~vCVKaD+2FHJ8@q^a7lM?YAn&`LD7)TE|VqtPPzTti=D1JfGQVN=v}&3we-vR<-J zb&~GYp84KhzKi~GL00%`WMZrcN?~4L^=0qoH0EyQ`nm0qSKLpalk4T|;3`16@DF;7 z&lP;-PvnQt1*o3)k~bM7$!oEE;xM^`T1X?pVp>jb0M6hB(2C3jtM!i%6YrD|GI^XX z?u-1O!mr3xu2fRM+gy}zA>l=WE#XN5ov=B6T)aH~lyZV{F?h6_(g3u_J4LnPe~P1u zKZ-PEff9~aBB~6k(GJQyT zhxAeD57NaMtuh8=EYG-_p~~QAW@a|a?3+10b6)1w%-YOZS#@yYM-!r=F5CLE8LWK=X5WTfqJoBl{J5 z6ng?ZpOwpsGCP6>nHPErodE5t=TIWU%(%<^$!x|P#85!ZqnvPhaE1S=carCxyR|#t z1vwAdo7moiF5(f>EwI1HW1eogZ#i!9m|vJuj7{|YwL8@JRVCFM07o!a?yKZguBv!b zzOw9o$(KLTUtNlHKRWzq{-f28oF7|$^!-`<)AbW6$}XBv^scDuuamz5znc7R_xqpU zqCXS<3i#+u_FWjZ>UY_rseD4J>fIja`?-t+!{C19V{Is>RhRk!!N6mj(W>}Zn#@J1E zy?wQvwpZFbwjp-2eG5Rai_XQaC196CV^6?c0^Ez6?(?9}iFp_Jgn`bX;z&^}gE5D> zlZAqA=~I@C^*4M8o(Q*tN5L@Mk5!kI&oVJ@F^91F!u!}$x$St5P>Fy5O14z6jc?~2 zK>p%}*migud|=G(U0 zhS?al2iCpTTI*uqJN{>%?6~Qe0QLZJ>@&e5ORZZhJI#L6Ak%3hYP9K>>fBm^Zm#yZ zhOMp8p4Ro%pU`j8kJHx$U2&IokXEND)HK%oR>PYAX*y{4Xam}=x(T{Mu!|%?e@DN_ zaLLfmpwXY!KhqC3aE)7y-Ao6~`vLY$w;Z*oEZqU-|7W{vU+x^@)_6wvuKK?Q+(9_Z ziF5$_13yC}nSa5JfJ6Usu^+l>XRiC;z zRi2uWx-Ruj>Y&s^DP_t3OX`+*JYh?MDqf{r3V6nA#TiAa@|JRVyf8tOz=$tbbW|*r zca0k)dnX+yEt9MlR|xwH2Mh1e=c&152tSG?2-c&yyjNU`a|<5JI>P7!jfGMfH$Z!N zB;z~iWDbc{MAn7hgbG9FLZ?Ibf#-e`Wd5x~PXa|ihpSzM4%DVG9XC|#{-^l|>^GcM zwZ1C9wytKG?vdqyyL)JJEGHcB?(#hiC_-<-#lb^f+FjQc(X+t5($(s2wbiQnmGw*B z|2F=d@@>}V(H~52?!B1tB<|6ghunwnA6$7b`oXXVjUF7lpK~9&AGvq*{(%RD4=s-x zKH)rF_f-1q+;hdNgg2w#o_*K(!=;bIK8^ak{cH1YWuKRPn)YSHx6*H)f6V&xsC-(D zUQ=!~+Ge^H{yCvD(J#WGJ~uw z+|H<#FT?i}?}2aikjN$?_!|6*U>2_%Tf?x$&`5WG$o0UUY~dTv=o{+xs5h&UR3B@O zSC6V1Q#rQcefhicc@=Z2vQ!&uKhQ^nT|76Z~?(`iFp99SnFT0TI z%YsRPX@U7cDk6jej8Cjg_B?n5YdkB&yvI;Or@`L0#>^B}99+Wg#61Jt1wUdz9w5t* zFJMownb8o+iT)F6;_vRsa{Jw7-evx_0YQ)q<%G5cmjzn-{|Dg{;YScjZ>r*_Hjj z+`NW|>w9xnTqSlK~A@cluyX@!GTv@L4lcOfQGTc~4HwlG7y zPdrnk5H_dZQ7(F*gcWy3W(FNhNd8#SJ7HO3>qIo^NU}Udo>DJ$WP0mNNftkQeVrk7 zzt?>NJ{O)Vsn@k$Vb1WJgSjW``ZF%4VQHG=D@g?@aN3BpDH(mT%d?ZS>1e*0_H zuLVVGizfYA@asYm|7Y6wWnV3yDn9i6#Qr+y>%lMiUygpJKdIl1f3^3y?pf<+*t5yc zkH5V8X2gg7pG&{C{a*fc;pbVOl%L8z?))_DYx|A2XS$f%-rg6bE|io1qts z8_YM%XUs*+|CnJ`3pmK6S)GBqZU=XDkh2_dfaZ6Gpp9TJnubhg|7K#0n%I)qhgiE< z$LO-?@@SLDqVSe*pRg=k9qJUS6QumFJQLifozv}OY@4ksEPpJctn)1$Ew8O?`%-&_ z{U67B=XiITXQ1b@XSHXKJL*bwk8^)@zXQ9e*un9kcA;0H_u-M@lyH{_3KENH&|=0x zW&l3Mor+Gz8iNGAKGA}Zk}6^negbbq2rxT(1m*Kg962`~X$bZ@EaII;TC-J*7Es@4 zo6s@8*w@=T(7OO6qr1Jo{0YIG{sgbp^`CvJxu?EReXr(I$e|P=pRTTW;`o898pa}b&_osi!=E{4ujWx--=LW*iQNLY(U(;2EsidF}b6#^F z^e((0S<@&zu>F#+Z85p_@wgsrzz3uMq zZu!{V-QAsA-RibG5CafthGDw9`@O#Z#ruPc<#ORdVwicJ`@Zh;JPy_pZVGPY>od7XNOx8M#evkS0o10R%CB!);?!>jyp%y;#Mnj>n6GP z+VpKTyrsOAs?Dvu!)-gXFD*##G`Y*IF2_6H?Ud8e(0*^b$88tqZ_O9B^S3+EUev+S zA+Ce9-Q2uct(N`OCZ}W0>KtqHWzBXpnVfwxdqNX*vt2oT{yNpXs;NJFUdH{Dr%8_! z)1^)cCGMQ4x$tj6e}S2g;dSC%XLhG=rv`wf5h7OO+hHBiU+{c71gYH?VH?b8)ma>*H?ln3IXp7lGwcl0qsL-fFwOCk35moVFqhkh$wAeJpF>B} z0ZlI~oV2wF^$hh2O$_RMM?9_Fznw0J#qkDK<-MJDhs!bAb-^8WQ#=RVD_t4RwT|`9 z&F;>gj=(x7cEsC$m^}LC+AFFU+>!b#mZ+L&8Tu9a|Fos53yR#v9`aoIdpPejDRNXe zjYj7&7Fc>%J6fAr*IA!iTU+0mFPjR?J1w29&20JhT>B;43)^~l1@(2JomTs8dzOO+ z^Z$=7lgIB7d3*Y*y#?;Oj^nm=mc!8czHStl_~t@$73?^cTE@XH`HiU`B!k!{hiQs= zxfyFdV47(<2_3Xb;|)W(UZWH19NNyhDqW*)rw*;Zp??8u-eLM=-6c((YKU^E;-Y+| zY*IbHeqQ|++1ZBH@?gUn*^0V@HQCj^swvg@8lZhNS~q@# zNy;+C?Z(nZiQ;YJB1KbGo`$FwnwDB-T7!^XJZ5p2^UZ_J|Cp(k3zkCIe_XLYbZ+zP z^iA_mgR`SIzMIf)Kjt-f9=fKvvfZB?rPe&_aqCa}Mb|mkS62tG)0^x)Msa= zh%zzlaUsaGUn8_6^2kT28T9qc(VRA%TBe?XWp-vfqpP6}IFq6y52B8vH!`=go3c+c zh4c{lE%7oTgnx%G#LvV{!Q`NaV*eu~5l`c9;|REe*mD>mWVl*m`@!7ThwYEc!d}7- z#ZhtBV0StVQ-MB%%8D(HoQAwzDD*nSh%&JI@CEoX{0G7VXey6K_eV{Ol|_fd7NCNt z8z=;|EOs#3FFHB46qStbhhbs9!(C}=v`NGo{uDVG{wqlK5A&qCTYAm@DHFauTsgpOfAvUP<7_?~3~->Lz+G!i$SU zA>j%kO>|iFP;^R6kUWTgE!~{BU)n!mjzli%AUpy)0I`rK8ZJ@`zX{qwi%Z14!(PRj zz+A(4M`zII(^9CfV4{8k`x|ACDx-<0V)O^>Q~W%_5&SXS9(deG#)_kbaQz@cD~gC| zhPjJ5kJ*SBgJs~}V?RT(bq#I+>{OrNdg0P=5?nH5NB2W3DvxlOuz_eK{KQSc&chU< zd6@awrnr6h--KTTEp%cR;jK6iG%5Sz?%)>?9E1R|kW@vyO|avGcm;kdz7zf_5L0^M z_QPs%A+{Yh3>o6Zs6No7bVk#ozrzPZrvn{)`#mPt7s$Saah@1f;Mzb=)u{(h<`_fygPyYH-TN4@>$?bCNzANqXC{DLW(U9_yI_6zRI z*3Z{Jr+w-4h5O~;m*+)|Ul$dxFP>GLSUjS*;9Hk(nZ-B1-uRmGbz4yw{?C!GmETtV zF#pUi?Nv7H*Nb02V8+2J@BHiSFLn99zgJfNs-9C@S(hy1H$0Mi6e9HkZ6Cc%zsfM! zC^uN3q42lfqC290Y8YrbYRQ0I_g%;TTpQhQJmua#zCr%+fi;20ftP_9{z+b}o8@?J z?Qdys&V-5M74u9J--t7;)F04S>n9iv83|^*<*ntU)nMIkJpf&f$H3P}1ad{1v(Pok zo#9RMkIBO|;71YflTx6i zd;&T__vm98cNr6yH&|>=9M{9WjQoc*Adh+R{B=AAbRKU)=P?6$4%y{@k#D>&{J#Zv z1WaLH;bh?mNDf+rn}KXHQt*<`<4@(Q_&o(PgQ;Piz=IEG)3%fo)a z?8PFu<&ZLefm?&0gwMto;|}2N!klFXHVIpXL9jD$D!ha^mb8>Cp@=C%p@WGfwIiJ* zc}ZKyJt*ra97xBkfUka$lu1GpS3t932qps+A5Dz3i(HFjM}tvatQWMP-$a^*7X*9z zOT7EMTJJ?)gMVXiYFGwmL$U}Gbw$;&!>B&!o|r`JB5W@#ADfMxh+P34lM?9DoF#RJ z5`-ItGld(3C{bTgosa=7^alQNewa6z*O^z%>kbWbBX1?|9HQjLad8|a>m1xi zlIbDpb4og7Kt+`HaPPQ=$6=FEKf<5e(_nL3i_bF4 zI?uYude?f!YPTd>ip^TH!+aAmF@>f@kOt^!>}}|%U$4ESPFC$y)h%t0L9v+O_p{ zvYLhx`2oda)mDvDciq$!P8b(MV#ng%3K>tfr`~o}XloyPknu!?TR{U^B{?@rhweU)$_ zUJ_p-nJmFdL=r)~E}>asdLk*YD)D5}=A?`yd*YkKVTos?&l4Krh4D)u&66EBNz8!z zUrn49o;6-cox~)uNxYKb@gow>OCKf9PO3`EO@5QyH|3ucZAvIbnQ|!QadPvdRZ@C_ zIR2hwfaHE$=eQPeTg2CdKlvp7MMx^11t{hijtg7Swwk8$)QGosIr^h z2FPw*sS)yR(nO+^kc0Q)a`98~WPB4`4t6r8EqZm#73mf!gLUtuQ1eim5I)o`m=Z|$ zZ}RX>c+(%qLoeIZ!`*7Pr*mKEE^NlYI^9?A&Qau^c;9qsewB6NmMVm&t zyr7{|Lq9p9=&kxolc%+7GxW_3;|vUgT<_C&G%Po4Fl>hG+I@r7u+fkUE5I`CTJ1&6 z3w5emrgE#K>X=HW>Z`u29;^PUnxJZ`Dp1{0<*C=J7pk|ZGhipWMGNO`dc5hN<(IAA zkqOg}#K1Z@H<=W@j3Qx-m@C*-xLx?g1O_pIgaYEzCpwNfpS6HvL3#>AqOamLl9%x* z3D4udN*r+q;wC_2;)pm?{1H|G`-Oc3Dg4>I^GF3W${5IMgv+DyJ^cNGsDLZHET9Qi z@Lxfzy?{3aP8_+sK|C!F%^%2XK)Ujd!+SBL9j}4cgMXb@$}MKkVYOjyrT?OSrL3l$ zf#1k!vVeS)l_SgX|bSwUd|6wXJ&NST8 zz0@>^$>mrjU$MBcxbb`=Lq1A&v;IN-fA#tG)H+UW>)I>uXKBrvn)KR_wZyuvb{f|3s@JJ z<6$Onl5w4`rah(A({kuR$b?t35Kbc}h0EvixShC{fJ?TJ_m$5TRtw!iE37w{h(zMc zkPSW&H#z=l!kh$F!b`YTY?sz1kP|k-Jt7uQOPHJRQ+hV3Jh@d$e#)qn$0?eW>+pB) zl=&$~Q*NfrPO&F%Oco^%Ov+B|DlL#Yr0WtFC1ybu_nEYXw08m{-WV4a2SgWzy#$^3 z_u=mJ9(lr@%yF@{GE*5lY1b)3$cgYaiL{YeK;%GD|0gaCJeU#4`%Pz}ZONnKt47rx+oVslThAqc7BL)3(v#w0~>p>UYW; ziZ6``jX&k}a#7=~#!HQB8%yP`hQfy74Y>^#*&f*<*>TwlSxJ3feYCEwZfjjlZADFM z_}w?uT&;NrIyVL%Bp%q28+9shWi{KV}7JEOz+Tmm1ad1cIS||nb!!Lr^P(p|o z$`37tsr<)KdFXsd7a9_w#bU7_N{(^k#sQntPTUWts7!KK;0?4O@`$B`#{>^yFWd)` z$<4_k@&j@;#ZCQ8n?nCc*V3=hQ|YH^U1{xTOCS$*0=n$0fZx!D@`^l$oDXjbl8g8r z_=OK}r?J~H`_YxCL8zM81n5`}iFzaCC_DNp@;EXH)|EA}A*idU9CTY~=d{P3z&^s2 z1JPzQA%Wn-CqcvP6TSn!J$?k<4X0a!aR`16p@?{gB!K?dBUoo0BR(T^gXeM(I)O|= zA-)-WY&?E1j*lbZim@?_741fKL)FKw#r}qq{iBhR@ZaIHp{GG4SQOY1=oT>fd-(78 zZusu_NWOXABVL2I#JkGd1!nM%yx+WZFUPaptpZvA#reg7axQoN>wN9(?z-Zh>Wzb@ z!Jc42$R3&#z7bA~^ovGgpMkV70{aX&5QFgV@yUeEuzTUdquB@yrsD)FK}9Ga48tG9 zX|Wl=nhQbdyE(cQ`T;5m`%poc5xNps;}841zHEQJ@3?o0C*31~Nk^k=sLSZQ;p_*^ zhly}PdCQ4$CAoIFmbgEA(tMr$*ud1lkw8hn9QY7$25tqjLob8oz^1^vz~12g(3mhi zLXC=J6|sfTl^TtCjI9L@$PxTA=%42imJ@CfdJ-p+S^)p7h-@c!1Uh2}$hZBbFJvBo ztnLujdKQPRh1A|K?q_a3l7-CV9^(w)Oy}I@lyg*^KHPE0b6yF5u3(K|pP4=GmbnuwnC*2p_CihwQ1=k7ZYey%?LC7_z9Y3It)xx>W(FD@uR>wRb zN6?+?919$?983q!ao=&rdDumR$!ZgKhG&)cp-=0#1$G7B28RUu1xEXIz9+tIK8bI& zH{W{&R%V-gCm;vb!++5K%FhVAhUV*&P|Gked;q3iokFzG@}MzrC$KFrCUC>g_099l zbB}YW9nbA0wlmO-d1m@*n5&-%v!a>$tGZe(Nt>u?rtYMArz}<8Q@Rwb6}HCiiV?~> zm0dkwb4zniGeYxAytJTOOWPCZCzYBN8iD4m`mEZeUa8p)C$Kv8AwpBiP-+v( z15!4S>}trr$a%!s*gfbT7!q6^mXMZ_nh@U*b(Cs`88VM&VS+OPx?L95Txf$VW*?_l zQt~Nqg{3y9-KIV#eFakMYT_^QO4=hvA;SqNyw2n{4r7Nh}i|rnD^Ey^J-&DgTwH^eAV8^E%&td1YDzB9bi>A%3f|-YREU|JKnqh^Pce8 zebT_yU{~k_SVC(4AzVA40jIKolr@_P%l%bQYMj_6AuBg_5zK; z+RJXr+QSe+x)jg6L%mEKgD)hUBTuF|=>r&Ax)}(8rSxU=M7oN~qwEA~PW^f z`WQ+#LMgPY{?Lm5(2D=iivQ4x|Imv6(2D=iivQ4x|Imv6(2D=iivQ4x|Imv6(2D=i zivQ4x|Imv6(2D=iivQ4x|Imv6(2D=iivQ4x|Imv6(2D=iivQ4x|Imv6KcE#)V{ZVJ z!wq&ea~+*ZpGR9xIY9i03t;LoPjUIgBcy>O7BLS$8;f8cgX)uxe~F$OMMb<33Tirf zJ(TauP$`&8xC4ZC#6n^d;w5}jTn+9o;$P&};OYs#s0zRgj_<3xw8T3T)+_16PTU2 zhXC#RkoScUqjj;B=!i#MS5RG zJjB2zGJ4Zb(C*PP86MVf?iTJw_GZQ+nw++YnFOFeHM;?Da2;4n0W@#{qNXjWanus( zb9w`)3|2A-w0cTD?E_QD?FkT;6WpKN+sJp`U4A*A&F=$9=S6~_!d9Yn!c=||=N)4N zBaIag`rHbDf$@23d98R|kYAh;>=3g()5c_SMj%Id7XWu}K-%+f3I~X#;w?fZKLKPidF57TN z-%VetcR&QOwP_1L9+p}bS}ep-+d$o9s&kGVbr7|iE>?@o3*oM?yBR%F*W zIy<`B?pq#%qN<7Whf{7}1Qmtj7ON#_-fa>A7GxoS4}TbU1AfwK9dEm4n`5taGq`B4Eyy*^xyPe-9MVB@{Izes!~sZm~B}8M7|8_P7f8$ z6xSMS{H#C`dbafjcUaJC?5(HALU0HoU$7b(Yqz<*U(4q zlE0K!H$bqnc6n8L<&x@kvgJyOrc9l!ax11P+BKeN*eH8bZ>tN`d1Zx)GIg=0NJUrd zZ6qrHQax4ORDaN{)YfT#YFlcbYbI)iARcOBa+y!tLXOeS!}gxmt(E~crIYL3;@jnO zdB;F7Z5zbWP`*6hM&D)ss?eDj16zg-04TFE+7KIoo`Knb9e}?_yg@Fc{Gt?5{j`Df zvos5J7ws@(EPzo>5ZZe|f5jSzbP}8pZWEG(_XK$1K9MoznO5YK*f8+bMXU$-oi)1 zLeYLPQ_?xUeSC_fjo2l)#*Y&$7wIISgq!gh;`N}i#0h!|D1v_cnS7r>CfXfWCs`;t zCradh1R=^4za7{%mnx zTuhk5zlangt#}%4U-lH{LdI9tx7J zN!)#$t04K#Ll!}y<{py4y$z3cJog=s1+uWk{GZ5C>@<+Jza&;sT-4*#1jx=7;ZHZ}zUVvH4jPvmM(P*nqlSKF zwsnS;Y^gS|wCUOsQZyEk)UT7U=9c$@ozGCDX z2!_9nLrn`zR%3xNYcXITp1w5|{gWIKL2 zZo0}mOS~30%Q@V3%R0nvb-Z%eY#l9KjjwdGv_G`N4SsW18_%wHymUA7CVO(7L2Irh z$>OoxwGDFeJyqV5-mC6^oe9ph&Mi(S1P&KEGrfm`apA+kJbw?b#k1X)4+?;LP|a-! z{P3>y_&fp6W6v0`5kPZSBX461W98w_fp*?;Zn^6KAe9b)0C0qFw7bmy+&0}Fv5m8; zECJgN_xXSzoDwSa&+@i&H#jdk>zqpGCg)(sL)&cICVQsiKl^XnSSZB`oJ(EzT#e37 zj!YZZvc|N*RATW#ee9=wF5qEW+l%Zw?I>Gwi_~()HpjKhd&WoiC4(+!lYa#uFb$E7 zu}(30Lj03E1mc!2wCA9e z!7w6>MeH6(74HFWC(@hi;!HwBf&rq1qPKz-ya`+(=RfvFb`KEBL;x~;m_30lWIqL6 zUOH_obvbn|-M|WQpCFYSGn36oV0alVnAaJn7~h%O*fCBmw}9J>)t#oI+@sBBUSp31 zOltu*g;l3&N$ z$@_{-f_P0p%L?40R+$MXDwK0L6os z9HlIy;;HrILl7OzBR(LA@LG%#T>zk_IamztU+fqR3q2e(D#tK`@jnR_gac6jc}E-x zVD~}fZNzSb?F1&2eA<%BiD-foCk5594gZnw8MI%Iph!6l2NFJFE@=;PEUx=!t&;z}oTnr4V^O4>bZ3w6wWKlB zRQerIrv5{HOzugdkmRJ^lxNfu+BgGU?By~=LI8V~ZYn~ZAa7S3GW7APSOES7*4pbF zkDOPWqa4XLf#nWh<~JMLn8K!Fv&kZ}{%2!2Zaeze!%$zyvi!1+29;GjVDO&Vf7qE& zRqAR_a;$Mqbxm;p11j~2-uvz>=Oh5@i9sz9@Yub#eNqq!{^dU9y5+j*n&%n>)fcC0 zru&+!mqTdHHdPw9ru!C$t(~)tXP7U?H^r0hy6Ff5FudH}=+L>p`>zIv1s-@$y6-@7 zrjz@od$;Gh*W+b+SGv2pGF=B<$6T}EbJIL+eck-?ywy&=U1m+UKDErTPPJcgf_uTe z!&zvrvwF;%jVXp(hBYR-wYPn}eX{kj>6r1M@uQ)c{+Vg<7HmIKfyfaVrQgu-MQN2|CR8_8QtK6^H*tk+YUfxe(RQJ)f*I{%Q zbX34?M|Ed(3Z2mK(Xh*afMi)_skOee-3Hv?6UThl0*}zU34mTjp6=d6?+ovCkVhv3 zrw46;7Qw_Y3gpQrV_Q+v&>u1Tu{oG9RF11bX?_c3LgTO%SSvOQOT~0VC!z17mtdY? zo`EKEJ=TV~gm%YFQ2Cr3nj4xHPK-{BU5yQo^@=_YTSCo2$V!Ul#>!ADv1?0l_+{Otp?P5uhz2UQw`q*^LR@?`?5`PZ& z677oMLYe?Mcqup|G%4(jRDc#~U+jGJL3B;53Z=nJgsA*6(hX9SSWRd_7)tn;Fo?K- zbcb9?=|Gi}mk=xQ2k~$42K;?O4e0{4KfOD>4{bQ;;8@f>fHW(hhhPOMV5TwojIOj1 zlv+|Lkx2{@8i*XIbC#0M5wXPcAat7!H5?fRo-^2X7!3Lg>NM(g>}u2(c^i2Y$&QqS zGK0bZJ)jE6gH1z6f?ESC{p)=_y#&t$cW2P~GyfteATz3UQe3&t#jv~6gZyF@U}wu5TbyZ7)$3=SVOa$G#@^O7HntOcWAupeq-nP4o?)Hth=!#esFJG` znos(nrcGv%d5STp^JoIHVk4Ae4GZP^*EIjMAaH~jpn#!t7fZKsQadAs~V}erRWBV<8);NcAKvh3dLE) zC52aMRu9yCR4q_WP==H{RCiRj;cMT~4%Qa~2yd#sr$KL8XkBQ_w)Y2k(sKZFKC#bs zOn2Gb3{M*n5I*#)f+NExq5M)4QifKAXGHcz_n|ss8*nXgN1#sG0j?;26StD8D6<*o zSznl==`8ADat3)Zxho}uc7-v3rD1)8-B34Xn32Zp$6NxaY!kZ$@;9$F(w5_7Enu-& ziL8a}W>6iyFMJ}U%%(O?AAd7hK`HO_jp!!D> z_Y$du7X)2Fp>v8`$~z&PBfc&EpQy87EHaFZFjp~VGg<7<91ZsYcMInwh%LqJF|5I? zl>p3sjI;;z`*H4OPA!|j(QzzX32z_l0$Yokf^b*{ve2J`KKwm=jj(TAZwWz?9d}qf z7F2;pg(F07;-oBmA8LlJKywv#0>RlYJtMz|U{N zC-5dAKS30Kf{(y{Egvw%7eIXcfV+UZpKAxTD~8vSPZD;Az27CldVUpul~5~sCVn5s zj~6D~O*orS8($dDPhd-XB#M$hr}RlJPx+O+Gig&|t+b2uKtfUcYl#MAg_bzEc(QOU z?+jPK)j@S~JTgeoM8p(+MW(S2GLJF=O$770lUy%vJ^unP330P0!5py-y#wPsAj=u- zcdQ4@gA6L;2R)bGn|2;_!2)^%eG#2a>rDAVwBoK~7?{;q2H_=8a`py=xcpzgzA zW$3?Pb>OeyhtTWDASmr6#w?NkaOEfuFA84>w~btnaH8q4X{gDl@6puArtr0}FFXWb z)n6i{C@K0Zax-!t%6GBg`ar+HmW2t2l$SuB`<1HV}4W>aRuc@Ww zZ_8=Z1j7eirS_<{sV+mm7XXyc)teO$<^Rb0gV=AcB2_s>c?J~mGn52HbNM~lNZCl3 z1oc!5S9S#%^ettz;;gbveOxQm-PZkQ5Lwdgsg4h}O_oyAEn}K-jOm)SmlN-9?-By= zc!h1Oy@{jV@zH5^(LK*Rdp#H2lU-70TM&|yok3SB;A_3!Jg?Z>+l%&g1$j}-Rqtl_ z#=?HBJSYqjp&WMK-#3VkC}X+k1XNjMSU4Uu>V!~9pe>a5{tmT|e2P}a+M(LVn!}^B zHhds5J}Lo~>S)w|=mpqR+yg8gI~G>_6EN4I-cQH9!6;GH(S4D7k%u5w-WyGhl!Hp< zYVbsGT2K+_5I77-L6<)m_!yx3jUa^X;hgW_J4VBmcQv3}XPFZ$D=jYAX*V^kF%B`l zG)^~t16|uNgG1L}f5_0;SZ*NeA)M%WdQ(2z)SaPT{pg(qIp12618b1#K9PJIY8+9K57SMDzZ7l6y>S1aR zT8Ms@HHABwm%(oURY)OUBqWIB!c-wyP{W(S3-TWE*YHWaPTX}IIjote+!W3T7L(DA zHi>$SI*ZnU){k@@6MTwg+jaFj5J&=j=dJpe;8gP4lX2E}t%B9W}3bf)#7J%aD_CPp{03q#QU^HksKhS7KEbBrdl5I1 znv!%Nmd(N?;W)V0u*w~SW?}L84X_4!itC8(61^VQgaqNck*k1h>WOm3PR0(!zJUDv zMCfzC7#t2Fvn%Kq*ps-fxIUmWxPZ|CtR)Wf6F`|uVudIriifgAe@5l8i-41SiXTIm zihqjTiP?>Lfn9X|QKto*(NU2iLj+?@aG&?{#m6 zNAEaqrCa3Yd?}~CH?1SxcTMEbpDV|ooE56HKuEz{hb-l~$9^jSusJ^k@=N^*>>nrtj z^WF5cb8m6WyqN)MV21CW2kYJmbD;k251th7Z$Ms5^Hg}R`_~5c1d5>wIXx_i(P6*i z!W>1bP|HyISZmY-bOpK{02=hD(U>~iL_!z*CyX1Fj_QocMAHBSwH3`k?TH?Snsg(o z6Y71mUF1x7U}SwX96cYciOh}EMh-=nMR!F;g|`Kp`>%Lq-huw50k!|Im+Y=_Cc1R4 ze(pD}Mn{^B1H0RPfMVZa{$O0C*J_9Bm_?mpy8x|@7DQ>N>zD$Z7Ur#TYzAP{ zrl9jNUXbt1z|b*UF-+VDf}e;X4Te|WA<_YgnYxTRjxv(+8&umL;;-d#_z42Oz{S5JSS`53 z%SG_K)q)C9P{bE(;64CdgCABg2KGYkL|&MGNbo`67OWQ(z?FC~ueCrV9u5$g|AQ3| zjsI8rBPk_$Yts3ob;(0hrl)pIZ<_fz^L9ooZFSnGbZJ&j6IGLFc8{!G8SB&Mryoks zPLHHT(_3X;$qZ-C%Q};ZO_wDvO&lq0E&Y&?8-G7;WZd1jJ)oB!CE^OEAOkqB*?LY7 z-durRu$$itQL=Mb%>WqR9wfR{#xpoe<-^o!H}M3(1@7ZaxJLX4xZa+?nxLj$2eUOE zYE$e|v~6Ta_|| zi_VFSiV0)ApqQTwp#ILV|0kjr#WqL3gNW>Qs4y5GnCzb(m>DvKZ$^qE8za9XU1Gh^ z3$ZKlcETfKh?ofCmqMzYHlLbBx`I~$;AK8x20?@O;t7CDYl^>!Z^XOtp9p`Et^ri^ z3GE_7!Q9RK$XLo8$5wKr+-;!vodzlz5`yJ2IUQK7>A943q%WifN^jaJijSbc2GLk_ zDawo@!mPFgeG@AB;aDW}{CgQ>m1 zv>Px?Z4G<1XVm5D;X0c^YHY9XuHCK4*K&0=a1!^ouDyPto@dY+znWDRk(FdwZ^|&e zHW@)(@;`GKtVX6=s?BRn*~aw%%jpBcreb5rJi_|gl3~sP>6_22fmwNH(=o#^eTi-k ze5N{me?0|GDVrGj7(MX1oNZDVpMsk8pdN4NXNZDopvqWk-eIW#eFMRI&%EBW%Jkj5 z(^_cDu?+*r;or7%_U4W;c9HeB31`}GdSyo7-|B4JZDU|c@3+5)s(+Hp=KSKcIrCg| zT`gQQoq3L2yW2JeKpbWOReJ2>Y#PfIOE);5cxU066^2&^pDDwVZJMmhQ$JRwD@n@j z%0v}Y)mABMye>xrH2I0_Z2hvj9<|MC?$>12GaCdAQ)Joo)>?V(*1CV|2GqT+T~s^1 zc2wbyfAnI(GwA(LzxoXUb1CCaElHu@=$g>YEr>n=w{}^)~3NwAK)a zSeC*G?NZkax6Z8so&7aett;0}bPsXea<*{2@)QOJgW2~#2GydcUyn-M1&CuMYjM%IuG4FwkOgiax}6o>O?)l?u9+&PaJ{xj{KT>mv)eb zrJbYAW#)2Q^H&Jg^2Z`EZWiAuST1-ca5!kvPu8R%%U0cp%+KVW`fEM+XBccS&95oug< zDd3@>;u}dvDSfCF)U~w3fO37z-p7f96TN|OVoBt3_yRn@vx0Ae_ksta)sjOAGo_*gae^YTW%AJE4=Jd$ zEvW^`7Zc4%!&5G$5K{4}L(}?Yh|;&GwomtD+A`OqE=*dLnvwxqjf~Hk0{{?D&6trf zCySZUGrd<@?*vuakWT9hR&u;y48M7k^7w_6?!@No$BA`GO=(+VE1A_&enKJW9c=g? z?ix~clj5dl8CLHB?@BT$xgv81j})u*w6oMG3v3^deOdF^&z++1LiPnhAA6?xzVAF` z3hll3N#G)~34f?sS|tguq#uc_Huf;@^-i_N^y^W567z(UJca7Z+LpFMsNb}0lz#~8 zBejM%4Y|r`=I??0&^7a8V=d;YXgIT8_rISrWy9TFVEwYh?o`yt06i4^Pw}$mjWN+@ z^kq0oEfahwf1duZF5jEvSzvnT07OSf;#*|3D?aF`{-)km?o;6$SDkX4_L*yFtSnI5 zaIXBOqQp6X@hqc*v<>lERtG;qRy-AGa7o2Giu*5B$` zTC{ppC9gbFCy1uSo*OY$jpgXt&y5-8Q(+xxE^dRRN$rRRinW#Xx@?dv$4kOChIdE) z=f9%ut=;O_XA8ph$?VQRXT%1(K6=x`V|}KeB|6O0(XqzYIyf!55*Lc>cFqeXaPElK z67qdJFguy^QBMsQoYy%mlQwbM1`2gUbW~48q}tmaaQPeUnSlm8h5jUhx1ETTQxkD* z@ON01^o4|5*o*j8$YXIKRZFC^=QADj6%<`~Q*;RB7rqcXhg~Cl%HB$TO(?)!kLJa~ zlMW&zwly@y?(z-AQ%Gl{X}+4!Ec}&dYaK=R3LTHG)Rfm}%kMO{G7hk$ zn4P{!!HC9O;i>Abh|}It?y5J|(CX_NvD$-LQf{mm4BfD-u*@~~F zzlS%ztzTJdt~^vS^fz0Es#E<=C=XQ5sV{2SE!$k(v(jCiS2?btxq5@IBkEsYC#A34 zUDmszP>F{*p4AwqxGonN&ssK`h}Mg`uIjt)nE$fro~?6ab<`HTXHQk+stt|#yUJ8!gM0wFiUaMsH?C~-QVrGzTKFQ0DtolZ(_Oy zINmX+r?f2MRc~=H3ovkcR0~`qGCd_Hei^8b=dcD6-SoAIwW8hB76>6_ZptR%82V>& zOG-O(A~lOWoi|Yxg**~BgP9?jEPBrUM?5{{hE#wcAUQmS1QV{r@8zZQ zHw#8{h9Y0%UW;yX5AtOa719lTDf~CBi6~oKD;N(`u~vcuEC=-~xsjy7K8r?(J=s%8 zj|r3b2gMVZUok>oJKs!_TQE+zoR$m-b)e{xAID}BqAWX(6|jX9Vp{BYz}t0YG-nQo zwT|G4t1+Vq<2jc|T_Yu!GJs*CKuNj`kB<4hH&B(dIEI@#jbx_F485`P4tNgRf!J~FJ~qHa=ayFQ2f0lbu&b&V)Q{QB4S#voIR8B?L*qc4;6K179wVGKBtm$jak6k1c1ksmrh^Drlp{IQ>2 zo_&}(Y~;A+q3UrPBfafG-3!w@2jcq&eHy*p*~NY$cqA;0#$k2=KJrksYjm0Cv)Qb> zWy^^U4c^nGs-Bo?t*H)D;9}%M*ye9(59u>(S5X@%AJB2$C9W0r3C>>F*349rG5iC7 z?{E@GF2p}09ODWSE8?fqU!bnxHZV>zXq3(L$-Er#x40wxZ!`+sBifrLPxT6gw7u{w zdn*~8u{&umCxMbgJHot9drF!~HL?fri#X$mD^R_$e-X}Nl7hAFNr4vV&$vzK^}cqV zd#D2JD&HkHJ^G12p}i%ai#XjP|Ip|J$XWcx&x}f4V*PndC(B32L;E;qM^E3tnMjuZ zqLl?Q=pVKT_QQr<`k`KXq=~P<+1q{6e?9CCef8wqCAQm+Q{HH3e#9Tl4YmqZ2U`b^ z`zyUCf<|;5b{gt#R2}UQ86V@2pVGS#mPUU@cAy%trwQwb>G((#2_j(@`abGh+nCZY(igx7`GtyBa@Qh$2 zC=z8+AVX@BDqk&%rlNm&OY9fvOh23d&j+ zUwHrK_2mzje$MzkwCYm*IE7#HTH8`~w&dB@CncY%uKm9LV?*i8%BK}2zgN_h)bcA| z)><`V8aw@_)Ee{#&xRN~)WOl-rS#0V*P8p1;RHXncxFwCYLVRn9xO> z9XBOz0FuUf&JBv2r!=Ouk_vc_86DWScVeY59V|PugnXVLrCx_~!X)lV$*1(isT~B@Xs^gisdAQ^C*`KlZo=8( z8qzM7WVWldm$pg0@&+bFgQ3+fP%XNZQF-{DKhBqByQ-7HNzrS|LXXO} zz^F5wv8gPrv;+0mZDS0rlruD|Oj*83vHkwJj{DZ8CaM+en3h-OYu2E#zmBC}W=247 z*Bs9CME0)UnL!bpR2~dBhKZ=_F+!+5kQ5zB$Y<c)8*3Yx5&Z1xXS)sO z?=5W`L15R#JsH&_c(z4ugx7dJy9)dpVUqhH5cN(B6b5Fy zFI(+~J^D$O8LkZ1DNCh}t(UrpvFjn86{js$b2T#c5!E>LCwEFEkV_VR@hiB`z1@2&5%z7*#txLUi|}RNMc3L#_kOp zlz(eo+GE!Ceuw|DXRcFV+v#w5=DNMED&Kv7V(?pRQPk_MwLf(&_r}Be?!UJs^9XTAD7+4f2iatXfh z^ivF_mIda^s(W;_|G@sTJa%AHOy!nOs4yeDT}%t7Z9@>KwUF(W~)FZB^yQnl7@Hvd%I@u2LQ} zoUuKzeADKt&uaGSP{srLzqC>d)4SE(%JNA2U9-~q(KXnz!+xCw9T^Etn)qW zz~!g~{{>qb9vvM>U{R-2?074x0P~r+kNAMNlKGXRX6@!MS!d}Zxg_>f+DE}TF`0Uw z{F-lxn-u324CW&IDdJJW0Cz6)Ej^2!%bkg=6@|nXkf*H4+!wqV;u-0>=7r7nW_FZ% zAt^AKf0biJY~l@ZBPAOp+xQhM8?!5NH|}D*O42svO!ohpZfSP1+1ji}DO~9UkkA}V zu9Ef@FBaP+nPP+JO>$l4?UeJv?My9Q#W^WV1qBC2@eS2O1Vysb5-LD)ms6x71=MBAmCQX7*`iqx!RS_x@A(Lu~5=HPxr(7x|M zF{KCzATPKg_9bfP|5GGTXEFy0{z5iVp8=ZKK;g4mBA2-V)1e1N=$?cwf1+95kw zZ2?<4m!8aVu-;M&=sP$cxR-ew#r-5l;~0_&abHC9BsnRy|F5I74sJ4g+iyrs+ES<9G)8l- zFT5|}fNT!R9rNuq?gj2+KsYzet#(W_i}iOpKgncDglU5rubv?}(`b<_*POBru+K5+ zJ4Z;iG;We)D6e!T%kgdN8&UP{hWzGrEfZVvn-TS+s}$8U>W|j%Y#=smsXJTq?f0Cz z$u-@ovGu z?VOA32aG%PDt3q9du)ANaoqESSn*rYdQoH2tF-y4Q_?(X@yR2>ojYH&IKCu-mGCJ} zC>#w+j^FWdqN%avF>vl!8jKc0H&ax^Px!O=JferVl`Ldd@B_jH(bt&i)MM0Fw3)PP zlo--_JO?cYvZ!4I9Pt#c1Z9L?2z?611hf59+%j`m?N)YGozdCty5L+ud|VETgJZpq zUB8{jor}P9SZ6zHZ*=zxuZCTLP7b|v7udu0wT@@b8Sc66E8cQ&23zBc3)F|Z!>__` zM|OboRDb7t`$&6%?XtbM{kkmySkzHoxu?pL>-D>GZD-BL%|zQDu6eFzo7~2Ci9Hyn z*ifQ8sfg;lpxU8s(ata}Fz!^JQ1w=SSH4&EHmo-~^jhsiwMaY4qIQmUU2@#E?{Ne? zgTg(a|3LNtCCqbgx=ZOe2E=UpU2wb776P@2LOa*>(4Jvx*B}-9+Mfa5VVX?ScC;g; z&?!E3)V1Ae)3uXj8#^v2wLsMqQQrhy$O5&nbG~|j{+#)SeV!BNE_O?u5+}oT6o?kn zofjc@j8R9 z*wW@aV`rH!8AlotbaQpvjajx~p6>pKj^(=1YL#}3vAcCJC~+R|)H)j+uN>{xLj7jV zdCd;(1bsK-BNNpe2AaYJK$CD-zret;q}%?qO|q9b7kSnK_4QI`f$gmsZ$eo|xHfxh zK^-UAlkL_3{{D>H?j0Lg7;yPsg6Zs;;P3$3^AQM`oi0q^SzwHBhI^Ka>ve=C!?z)B zBle+h<1#?%#flf;kK*F*ba&xROf%C*-4`aLof{FTFe3R1?oHeLxc{( zfaXDgw<{bDeD-EK*W0x~5&O`!+IQI3!;|J86xj;d4&8>BintJ|_Yd{%^~gXzV3ud8 zFV}zD>-OJ+a1mrszbQlKAbs#!Fxyulh1eP3m4ATXBA)`Oq+gUvBn{C^+DdMw_F)eL zB*HFEIe##p&KU>F`P+CUF|4>faT^jsiFP0$eUZ>LmL$@}6p9?;jp;wKUgqd?`{zE& z$&$wReyZ_EkO`|7sQ*Bw3yJmMk*0X=_A3eYJ5QB5of!>q) zr1W3ezogH|o~hl;xzOzIX+x8HC9V;>#iYa^;>Y6tar61tm=`I@xZ0@O@Rx7~A`yNV zs)E3vm5^7VQuiZJ&TO#2>>Uo8>yjJe+2p$Elma!T#>Mc)`t07Vp2tqVHP6@$@H|^p z%hgETKm*`~%nEaoO<}ua9$>07)6Kt)3gae&PInz>t8JPKsxo=4EF>$EXQ-R?ji&dO zTJujssqVVANYht4Qx7#QH+}(Hp4Iw+`eOY=lLQEGD^35I1=j7L0`{$Ky#15yq%Fzu z(y0Ry%CFvZAOc9xg&nM-#D&@hb>&+yaaj^)EUR_mkL6 z`9`<1beu_?w~PB4cmtvq&gQS^0TP0R@}0lddqz{auCnL}8!IQg6%++~9MvD*0I3AM3Fg5lh!9AtF1 zXjI&|1ZyHT=~(=90g2h2G@S4i&qq%OPpzWhJ8<5U0_oCx_h)yf_g<(EY%x3)kpG_G zdjDr1%Kz2hCFFpdhVMtV;;M)qTo!5`Y;x#_|5ZR3o&_m{s3N1mzI9jRUhs;~;Nf`4 zuC31N-lUKf2+QXC(w$Okl(X7j7#b6pyWieQqO@A4)%;T&b)^E0XHlv9P zelGJhN@YdITG>STR^`IZB5;O%s#&390L_|8Ck5>1Z9}D{i~XhZ0uYQ{V;BvZj~aw}aQosWcN0ZIkxXhvH{7Yu zYYI1Bsa;d9_!%hr@%^uFh%fX{p#s)-Nl`;#&w}*#SKfs`e*Apvi|K1pVbWLj*Z40B zK27_u_;3?!Cp;X*L9IjHMv<_ua56$N`6MNp{*kqWI|yhM%6V6~H#q&+6Ip;(r;6!5 z?iZ0eVR6!$q>qV{lO$rDs6lW`h!iEn_Ke#fwbOHy&Js`lZJ^wH=x|;SNNOMMJxj=1`z$9>EBp~ zx#xk~`8T(MJBcgciTG1^6ds&Uj{Yy^K|;U8apGwS>e$J`G=2)FD+|VIWgX#+;w|D| zj4l%F5V&~5S;J@_sr9rK^nV%G*kAaP=#N}Cdk8N-`m-YTT*S}dhwjNK{1`e6S26YtLZ;7OR{Hlz0oy3cWidAj9^N6N>j?X zEHymz4BK99FHngJA_bJ$B6U8se`>o^4V2W}eKM%9yt5sQd_Qohq> z(FkN6;V$tVSwx@ATEUvnD4|-3ChS)9Dbz8%=(-TM z5?2!E6Q<(e*h+9F97?JtOvSE44FLk|o~Xazp3uI)7T;9of7Y(HTQ;X9)qK^EuRW@r zs6VaWs(se^qk}E`N7gJmArr}7%h-yX&O6FVg-C_hi40}>3p%d8-uMVe$<4+E+7a>x z?Mvkfu&a73&zILJ6d(`CQMM|ttB>mL7%3K?g>3x^vLpQsR80<0AuiW_F{N5*_Giwa z9;bH`P`UN<#(FEfQ~dpdOaE87?oal-bPlt(TM{jOt>bO?ZH2ZphrojfwE8D`Y|e|W z1`opj(Lc|p@htKDaCLL6v!AieGVeAfnqOLOTLPB*R-RRCQfT1Hba}dBYv&5}0QG6* zeAyn!mX_r=4mE>7#O6$9}EplDw zbyXkLQ593uO~=y*wG%*1v`y#NYIG&0H})D2CAdG76-@K*@!oQO1X8&s>n4y(dhOWe z-S5BQza2=8)WKfC+QMyqsrQT*<|F#Xf;}iIFbVJrLxMX3_k#N%ClLLSUStHy&!}SHq8uxt8M=Ty~Hl$1X#e^?QyYO0-8*ZlivZDCDpO>x!5 zis`==m60l@{I)e+X&Ke>uBEWGU6Lj3l(tA8w)g4S(Ls^@kbaeNWWD686hGxF<*A)x zbZbn5%yov#+LK`CYS7Ts>(o8V;hF{2C4!HCT4!BxHKN74L76@ZRHF4^MrQ%4! z!h}2V8{*<(#Xx_CiM=Q`ruEBynR7U|z00Srv$_uLlAqhZ%Z)C|TwBiR9DB~8F50e) zF2ggor(6^7j=LzL$Djm@`N{mRfIWC0-B*AWwh7P1T#P9dJm4`o4DyABc=e5pI3+#bHTXiWPT!VHSYl5AH7tl74+hpIny}l-2FT&m%%>C+{L=g z?!l?zyync{^x({8=dxC@wCp(UbZ#SiHUmo8LAZ+75rz`G;g4c&qO`~@;8%aRKm0V9 z7x>}34-^y4fR)Jbb#vdfEinx+_5?GZ+vaTJRc&jhO|eYzT0T_Hmrv+8Av-TCl5LZH zY0qqzx2t9Jj#u(R^*Up=WwUv@(Q3GE9tWPCzik-j-=0q26#oREfBp|r0(%cR6p*-I zxNv~-V!Pe;>y{(tWNQhy&3v|RvA?&TH;*z+FdZ-tw2JLY#|5B~H@kQF_JHJOX=r+& z$eZJ(c;|W1AT5pzqC-RzF1FmTJT-p?T>mm{y5>>mEV)d&x7F8-ZOv;N*FIL3CA-@$ zmfmSaw03E`Ej`uN+Nx`rD=C&ilxq1L+3@z$(%x;0Tb0dgTBMS@ZM#|mb;GK^R1U9R zTkENvRZ~^@v|?|iv2uOI+F#JJd8M<03oUJcx9NB!Y<#x-fW>xdZR=vao=3~MC z30m5{9Y)0?f) zyURZj{gO}MP64TAElb4P&mX{D$0X9B^pgxW;Gu4^uCs4*UUJGgYk6!zY7AR+IVMps z9kAmheqUa9Rvwi{dP>YC_odt>{e#mZm%v4kuR)(*8bE}9L}DPsNFY2F_6)THcLX;R zeK;x(c^4fEQa^FnewbtE0-zi2zz+eu=PmLuayL>nz9*m{Nq7?BFT!{HIRb)GLi^c&i3Yp`IaT4uxa62ors~wH5?%pB3V$U?^7TY&lS7)(vs%@BYy3VJ& zZG2|w>Dc9=1^y0>4_*(#!lv;3kQ0#EN4$2xBJU240KDCQK?;zqhl6{=iD7R*<;A%F za`tcvT;n`d{?u@3cxWKe-Ou^N-OmSp&tq^61e0H*|6FiN=w8?z84EE(_QPQC^+2)y zHgYg>FPs~G8gav3gNZ*4ejYjnHWzUZIT$?veFixXF&18mC`4Z(JfLg?;=6~$0Dc(p zK6x~CKJ`1fmNjg#-nDfOmn5?gCA1BC(XIC=1Ge)usxx=Dq(J?$K zP>QDWNrF;gP`F6=I%ZP5Cb2U4XY%vJj<^*<0*?&1n9;oX{1F^FT}*BUn!o*IK9Kx* zNlQs5NMM>smC#O88;E1D8={h-c@afK0=WR$9^M?t0vT?1@Li}N)EdkU8iKPzy@S01 zqQFBx#Wx4=8PTrHU0)@e8a3Snc&m9f-lFxD7bjk`<=v(Y-y!E-sBM%zi#XhV+isYz^_tUsWB zrHoTv?9`|O+K0N&+C!Rtn)jL~+G+Z=fC;-|T5n+J6xvyaIP+bz$H+E74N7C!Y_t?u zMwssy;QBm6H}eT=g00FtR<~K82hwyntd?d})MreDOUt7p&8K6&)aCuw#U~t8ZD50}&z~qIzLYpg*8Un6CJx#0>-% z_BbjQ&A=@tej%TtXvs%ObwoUwMk4}J6J$)NTZrQcw@F>;pV?&IR_;&s5B6nVi_jH2 zA?|}{Z%lOzBo-O>BW^=HCt*lJUHs_yXYqfC2}yxOOF~}UKG81`FMgc3J>h)paX~bH z3BOQKDmooc5Q`HsVhaUj{Pof2gxr{&(SLBFnKMCmL=tNUBSh^|MOYLR#$E*b$-)F%Hqxc&J#N*q&UP{7Za7G?|~y zlM8OfisLFo>=>$WMND3tFP@$tjQ=GXD7++iE1WIrh>eX~C^{(U4!*0YFt0%X(qZwBq~!ElS&y?C(#fg2Qs$<;&YY1mCr6e! zAWfa56jQ|86D08m;zThV-ZN%4-9S4}kD`f*voJ?dd1xW%>Zn8$kwc=!MOhG0NFin| zj*VN2zJwSEJqF~%#Sjwg67)bME-VR?p%dX$1Qhl%bOR9iCXD`!W{GB}HmG5$^zu8h$?Zg0n&OV?tmdSqyNW7*C4DQU$os33G-Fja z0NlRr%WIx-FOINqelK$wp-`Sxauim2E*s(yiKps-9)7uR9K-Zs zF1D<;^tS}etIRFt6}EO4&o2&Sd;N~LwvBe73+uV={?8?F5A=9Fr+mkPtHVb^zkv$u zW*84&gF1+Lh0a8JKn}qio(VmKxDYiFK>>XkHIQ=HFT^X5lsF7sA4&^Ag5=07m=C@Z zJ_A}FZVMI#JHjF8H~0@29P%slIrLvd3{8R*hfW1o2UCN`gSlaOqz~j^xY0k`v)BF1 zv%|aFlj)pdNitZqVyy<8-P1aUbZluK*AdmZOM}*5&>z;M%jdMtXuRLpwe?TwC~2^D zWecTseA`~2qpWH#mYO6QK%G989Fq2tUT$60G@&t}d5ol6dsYXrW32R8i?}Jf3DYva zb#O~&zW!?w8+{LL53PDwKxCWAgO0Gm9)f5{*`znVrh4oKvCbhMm0nEvtvrf zGkJeyu+yu)szvGJ4D&!^kl1?G+3M0bgtlRp36@P(h|On>vE^CG=378wnBX|&dkD0f zs{mWP-rF}Y4RR{Vj~<8hU?S*AnC-atgs#Mu1UBIUVF~dMav~k%bvc#n(JUU*z%1ch z;SUhtgwF)B=x)M`akrAzroKqQC&?0*q#e#V+-*euH_!#&WlPq_)ZIzbl18L_Pn(nRF+DN$ang;% z_lfpIoA_jWR_wDFsc;|2^PLskitgn3Sur#^Wh`w8YaKV8=itm_SF!f8<(w*RZ!QM*xKp|Sg`P*fLFqxcOwJ?C1+w&e#2l)U z9-$AS#uD4G%Q5dTn{h`7e}J~GdDu^=PsnSiFX-{;CxG=ngWQCAht9+06DkO7{6E+v zY!beeG=Z8=2@sFq_?VfP8@SPg0(>+UsE2`i>MCI?X#}a65Wx~qYv2*+XZR*m4d{>P z2X2E+Fc#2qkB2XZ)P)kknMV+q5&SoJ$(Qe(Y!#Xpn)g}GS{9j48jZRPjak)8{aBr) z-p~nC6n11Q7OM7WoAfD0B+zPK&?+@HZA9N{Ofol`cbZq21f~Jza?56`*m~G<$FkDe zZ2iNEwp1G-`n^DQUk)}y9l8P|*PLo8u>J++LT2Ane_wyt8|!%ib}i+>N08g#G@%I| z@GbB@_k9fxiY$oKhP(l#|3Kha=tzVIy$G9(c!~TQlZv;1ZCx4hFm5V#H_l3!M_Ejl zfMk`LX#shJIqY)IIQD;Z3uy^~M1Dy}F}u3AYHZi3MZ`y$xhm$5UA3cCwLL zOn*QZ(RNY?(ALp?jN`027M2;JdPp?{CEkZWLXd$@piV3Yy&Aa^S&Y<1Ip9lSn?OGQ zImqXipyr@h$XBqO@C)$wp!ufzsexJkG2mR8>uvCM_`3NvxVt;T*40*}b%gc238w$8 zUe}qfoT^lG;x)0_i5f4+3a?d<)vnclFbp#uH5M6Xn_t@_9$w&Yzty$hQmUV(TdrSX zqS;uEF^&SqYFC2Sy zgbzL#_C4(OzxFzOdm_K0GSTx91#v|rM#h~|M4`H>KD(oNlWI`9hJG>dF zv#W8l@rwxg#7o4NxF;wJ;t;$Bb`N$RNLrg=)v(tf3sea`11~`+fY|*w!irjp&n3l@ zwIn$~hWUax9f1aC1j>VkkTH1K_m3;X{)cs&g$3q_XSCzBE}c%lU*}ONDgHMYN(R#Rp4K<%aT#a#$s)8eLsd zNw1PtovONC`5o|F+y-jfZ0U|xTJxf&bIr$FQzb2uL|L68x${)#36)RXs1@k%=pJjI z0U^8E@YC$H^6f9}2#~5e@7n1}@)N^np)29<;E!MyXfu2aYA4WwtFea(cd5@C|rvy3WKlmF2U4?6+PjR`Nc1|CD zfiOO9a{T$&4Z@ZDgPbDfI{J9(4boKnTnq$EhtUKa`3}j5^B_mVWiWWuQIrppi*LZc z#}#1DVmqy=)4D<|2zh zwH3(AS7PrIQfXb;V$M+3eZ~v=cKTt)Pi7Bx3{RvN#PDBoebJG`zj6#+UuDlrdLi5) z=okN2%C+?BbXsb1;4ru=s<|wzt&^)-U!`?voVu!BCI7U5MJfKAL@;4 zAY39B5TS&}q&h|>?-#F*)t@?)P>UZx?;|WrSRVhBzl2r_hVjGMidaIR>?bq`}cV;#@`4CvAyiRY-IhbQi8z~OOUXK4Od3ahu^1HZ? zf* zle&mm%Q!=U5hmka;Pzn)kxtkk=yzxxqA%7)SclsVZ}FdU7kf8Fn8*QWT2y6ZZJ^Tk zEuagrgS$Ks=X%%Fa0Gjk#;5Jakb{qG$IVk6#{!!m`ym=I==~@3EyN8E3XSq(Jr?)y zfD#HrJwg3~mHRc$r_KuB0_YedH7W{vJQ4@9BZp&0W43xRBpepgZ`gEsXK0$V|ZC&fK)^J-x$IDK;dPQeb8@};V z?<-M2WLt%2i_tHGD9kKnKX;AsqhfX&to1)hoNSPMg8a`8hU|WOjqF{=@pgBMsqsq_uI-sZpq;8+ z()pikWZRTBNqfJJ?Xq1`OY7{G=H_cnjK;flaaBW0ua|7DOln#nTiYRS-BEk<*R2w6 zSyRQ?YH9Vls%Mo6)%MyR^?%fRYUNd_m4V93wX6naeS4Lvlv8}D_)KZRulW@ttDaPG zs`^$%RbQ*|*UD>7SM{o@t*vWu%KugV(SE2&`P=%ts_Ck9i~K+3^iHSpEbw0+P~B3e zsMo6os#~=Urd5u?uF>`i5WGw^BCH!6Q1=S&J^$X|wqTq8-@u+Q7P1mD40ZTMGhJMYMC23)IE*)s#KBPpIW+FHyt#7py9h89gcYh+m08qMCS} zTt+!UdWm;|DEVUickBREPlO4v64RfkAV*WcHJ*vq&Zq%uYa_W>`9bDHrFwUSav zNhJ-!^~JPd|D;GjT)|EsLKqg+8Yv3HBC5y%cs0@%wFLePwgY|yl#SIxVj(fm!HD7L zt=J^Y8u0wq127y5dxU8sjwc(5b}Sb;7k(A$4)Od9UnFo6{sP|@l(hM=ZbUl730(Ep z5o>^9R!8bV`a#%=Go$+=x5F*aBG_nT23|pd(cTeaqv}HWerND0Y!7M@rWd9J0O+q_ zMiFWZR zoFG%jNMx7t<)B7(DSsBfk(arI7|?-EnVYs7GXfNa7#kwYvAZ`#MQh zirHyt2L%%qHiw}~887WCS=V0O*%Rc7VvI%F1hreik-ctXwassP+K%f)8~oOYW2hs+ za#PnsMOXe-UeT6W_BpGaPCLX#u-J|F^;x=neX{k4Col9PObjGDYV<~Rl8$BB<+A%0 z1VBT8-{Q$~HQD>xZrCuc1%3zQ7&&`(;>?QbFMBysgo;Ic>O}-4KP@|V93?&P$ssqTI4OSTI!nW8rmD? zNm4rtGneZ&I=A9YY>s#ku>#hWTbE-Edas@UHnj5(mVusC-!LaVI2Z)_c zN1Q!kVOTmWLi<721F%y`^d`B*t~0KPo8VpR-RRpHmg>(iN`DgpSf~JVW!L{N2k=pQ&;1R#m+uJ+JTkM_W4~J~9<)~CF3}h;w zBc8&S0_JQHVD@SeH=^>8e<2ScZ-YZ;97>EPU?P|d{0`EeKwmx;e*`@Sc?mfRZNU7* ziHUc~V`;ZQ8_^)7O9kBtl WNlrnGkLrbbhwV@LMl&+Uvi=YHGrS}K literal 0 HcmV?d00001 diff --git a/tensorflow/core/kernels/fuzzing/corpus/decode_wav/782051f8120182b860c7fe1b265179cfa2fe03fd b/tensorflow/core/kernels/fuzzing/corpus/decode_wav/782051f8120182b860c7fe1b265179cfa2fe03fd new file mode 100644 index 0000000000000000000000000000000000000000..a0d8a6ec48c983fc3870d173c5b4c73eb474eecd GIT binary patch literal 66439 zcmeI*^>Fmcn6okVPJs4T?=)Wq>a1l zPEO8R@AJ#~2i|p_wcE9tG+j-yKTY?2U*GHM>d1lp`fZp^r;Y8E*LU{974cjejYf~W z_qjo%?HEU=0kr@3mj8Jh{C}>5{^y4Ob2%~`4*!2{qCTYrC;>`<5}*Vq0ZM=ppaduZ zN`Mle1SkPYfD)htC;>`<5}*Vq0ZM=ppaduZN`Mle1SkPYfD)htC;>`<5}*Vq0ZM=p zpaduZN`Mle1SkPYfD)htC;>`<5}*Vq0ZM=ppaduZN`Mle1SkPYfD)htC;>`<68OI$ zkote&F{y{71SkPY;D4O}@W1Yer~{$|C;>`<5}*Vq0ZM=ppaduZN`Mle1SkPYfD-uM zEdW%Xtx0O=tr(%0+$d_eQvajj-^P9Q?kZFH^9n}oEoH1WPdlgiY{S6XF|{@I-5b|6 zY^bwTPpPV`JX0Ymzf_V`*!N?>hu^=YSIlTw+L&6szUbWN^BF4QR2NvEalofU>gukEp;V)AB zI8@lVu&Ah{%Bk4WG@*WHMN_f0xUk}M-D<_zrg6>ArXx+F=IyHU>P+B1 zk?R^_|7v0yvW$@RR|nla-+S4=Ik+}h>)#UC6e4H~X#If0sEap0+7nYM{+0a!>w*UH zZtObV4&iapNs&^NDjq0)Cgw=KN~T6-#syL?WM9h}omrm9j)9~1#Bve`rPgJx%{kC& zX3KS1eKQuPi&Ki@v*U=wgsl1Pm;9t{6-@gO*HV5}zCPwc?CE$_!rXXH?5gN5QBvto z!fD(b&S}0=)<5Aya(>d2*gvG0Fj_zpeUL7S?jJif)*uf^{tf0)Kz&a*&)#?{x{w?UKXb} zqZ?jJOy<={TSVD~eOUeA>hPNo5LSh^gWnMwJO}&?tOs|46M#}$A}yYt3-!m=5b5|5 z=uk-S7X}I7Z@hv%oNHv;i6|r=JWY3#a=*wI4jcgQFuU;{ar}fEYC&^`F3<{~B}64# z$LYv<%UZ)QW8IK-poRVh7=q+5l{`T3FDnr)As_mk!JEJ)bQInh|A3~#_klgMn`Cx? z7Pu8c(M!BZvV8eHiIOuI>kI7wGQcHRF=wz)C-R9?C09j+;3_A~K#2mzLata$m#r5s z;7rE4BQR0Oc_R>sgOWyRiS(1SoAjl4phzQFBiJb^i5r!AKdm5TS;B^Bo-8QViPEGW zqZP47W9CLriY}L@MD3Ss6E5fX5L8K?#k7qd9aAZpBna_V3mc>t<=fFmYPV9>aKBbkJH7U1J);>j_^n|_Uq_1NoUWrv z*i_tfO8d#O)%o4&w9{-Xi`{rd-%9sTpJCnY8AQGfu>%>7A9|xYT_>?@aM^t`0(b!P zTRg2?74~knzihZ`hTlQki>+o~;J6u05E<;^E%NpvlfjP|$&Bap<{V)4CB`5ea5?=2 z@Bv-V{*_?J>Dvg&Ph*~IVFRBzw;|F*mL1*#T=*;Agshp(I z^7Z`R7=`F-WC9w6|4y_ePGdr(9Hiwz`xiokFQ%>Sr4NGNqc18QSXBH>K4|suWfCs!A$K%IeFe)nqk4 z)m&=&Sm&x5TY0ByUfmtVYxNMrc*6pXw`rrYLa|WsOhGjJw9m{t?E?Eh#(k;+1*W{+ z+)F#w@X%;5)tFZsdZ-ebQdL>%7b=zNi1wZJ-@sBDC+K#>=ge3CY1C*vMz7;oV11}Z z@E7k<=QUfFRcaYwVL4)aYskLjU*3Q1DVDQlhShG5^ArX4($0b%fGcFKf06f~TjRX$ zSm}BmVlhpEmV)oh?$91`qd(*kxwra`(JDX=@MmbFAND+OR@;79Hrq;FUwvtTN#50t z{??I}c9y%A5-aLh>)91NPp%1!4-^Eq22#CY$2L3U>jAAIl5jh?oSfyIio@B?3VjX!QHf1 zzyRg><^fIIROj(4`!_}L&}l#6QptN>=!IQvzFaQ080Z=7vz2At)joi{u@6h z;X&fPWO>?_j9D2EQ^_P>!iR*cWGrn`MoAWwv#sTWmf99wv!XJrX=Bpbr$29TvxB3{ z!_Gb0cg;!9=#!S6N=ubxtjy`xW>lNaE!Sj8)4C?zjQ=-|h&RT^BxR?jq=JcCqbEx^ zq77n1dQ{pkN-fWokC9#zB(YCo3D9Z!3?K(j65fzciO!QQ;Ef^u#PZP!1F-$>7IPC?9@I3b#u@5{PiVAh2bpZ>JH~38E0QL*!F?1eX5S;J5 z?E!*^A&@zqJ(Y0)Iz!fYi=AHEWJi?G9Ol3vrok4YEuk2iGEm`d>Fwel5o!fY1aHt5 z1fsm}yj*{t-{T(=d>h(9Uqx>n`tBR&8fw31AK-ZwDuO-u3?hi^r&k2m`dWGt-H+XC zd|SxBXe;Pk+9UFQpoedpYoYxw`)=1JPZu}KzQnjwJ5KvV_gde^5NEh$Y;Dz9J;r02 zHBHwQ*Oe9(!yvIObDedS*?O4v>R0Iu+BMqw+7G%H28MB(zCy!Ld6f=@TyeIstuj@e zp+9QyYwMbd6ung=bldd%^d9|n<4tpcF-Cp8p>@r(%9<)eeNZ*Wc;C{)JXc%LxVy?w zx~ha$IU|oY;k(`HL4CC@!n)MB*>u~IX(Jta&!52( z`g0JVef146x2NrA-k zaqXf`2xoIQveb+@%mJL6f~%4r(tBbPe*^mojv>1v_d66>hE~HE@HyNZ{wsVjBne4F zGFl!q44DlwL({!_N4|ZDUFi7Y&L>|1|3VYM!L+&%FFYf1HZ1V5g9E~i^us_EEjn~5 z@YZ+0w=eKpXgpmDo`6b$b@UZLF7yDtitI;ejB@TO!5`cbMlPm9He#!ZJq#Z^Q}|sZ z5sYR(#I8V7KqtLNxO-?IttZq9>xj=m=-_$!zjO`o1Rjp>W{zfGVG6J_@HkXL++c3x z?iKZp{*VNv6(-Gy-7LE#S}5Ep`c1M^J}W_xGBJ5z+;@qOzk;vj={YmF--R_%6>*G& zhcWM@Rl;t3J8zDtr@SKmQnD=NNJ4tdA5pudm&96;MO+=Fi?2=k5l_g|MPE3-FmDrE zi2vB@#E0cs@>=mq?ib=Z+8#a$90Tqmy;zR~?IbrvuQ?;IeSjuBHmnTarvD8-1D1wX z`NMoBE#vYXd7v- zNRPiupi6iF>_FXM8&cry>U!V=++RGc{kwu$fsU>$i^QnV7w9hsn=dIp)S0RsbuU%jEel)^oh>cpT1C@m#U|yM z=4wrb;U7bpwp(O(^`}zU)W2zVQ(yH4BWe51A+vurXX$E{z3L~_-B+G8c5>cw-*c;6 zPnHr=+?VUzZy)4bX)0E#E;W}|uT!pscZcW!lhVo9sxN2$n%JK_UKh=usi|W7DcCVaT{Ozmr z(~!@nzO4A#;~VSy?&7QE`zv3R%_;2pzTp+`&DM|5_vIxYOO3^A3SWMvecu16)rYlj zjc=}g*!BH-`LoKl<=P*Mzo29*O?jIxZ9FZjvwvhB&feUnxZ{wnRb4-J9{JPZc13NBt!vw;+r4SOyPdmrsAZp4 zD_gg1J)xzv#q}&!_Nx|Wat>w>PJ0l)SFVvwjqa7OI%Rp<#57rISxQ>k$g~T|6B9IX zpW_!LHKqcYBeSYAV$$ZObV%u*sz_UtJ}0d*xqWcsi+Ut?CuBh`s$lwbq@mY}b=wQR1uF=~l;6L&DP6Y&~< zN2D;D<1 z1ff9YfGIRE4AUj>68r^Y9Z`WC2hRg2v|qX4I4 zW6+VYKES12Y-r| zASBENZKTDw*7sAenzkRFgZ0Gc;S=y`Y%^AeRbm7>6C4~?1doy9!|ZTN@@6nEd=<*U zSL06+GdO~dhg~5Mc#Oue3wdLC)ohsckokl8E8`2+1`VM5h;r6N?k(O4?o##&W;7#2 zY-jpdHOv+Ga-=WP10T-3#O%ndV`cKJLb`Oc^onRDZxu5gi$ez!W!%wHYxM5uVyQy7 zo8O5i<4Bkz2tP59HH3R#Fio5-e$IcwzQs&r?c`MRF7Re^I#rGJ8u}Ra=wItpT8^e$bG%ZnY}Kq%pVbO=FEyH`$qn2(etlIco8nrw=lio?nu zP10t5bDm1CCCvNm4424dvA3|cHz%8|wp3qc_&Y5(Qn!JjHT0>F4!lAqBWw6TdOCaq ztH$oaT!2Y?6@EuM4wQlyz(IhP7DLYii(oN25Iu;tz?U;NFc&dTGbUgwf$pRu5Dec& z&#{VGb@*gB7kEh<5ndcT=l#`Xa@=ta@*MX4=gac|j=SbG(*ny`dpAcv+aKoV22}q} zhwB-J0@Ecc$I;i7;c4mp-BV^CuHUM>(a=wcXipm-8{>7#=I+WaiaiZaYGNw=<#847 zD_qqp8jm-b8h=!OE}LEYsk}vX$C^#mqpQoRMps_1Xi<5olB}Fsb*bt`wWIc3!x9D3 z;HV&qUVRUjKCR1Ewbky>(sf&Opze)kmAWaiirj2Gs>s*Ow!U=>-Q#U8lhN45bkO|F za>eq}oMf787-;BZ0<1y%JLfxR!1cF(8SN3+9q1qQxNzqccT!+RSOW9~Hq%;$E5m*0 z7J3RjIVAEg@Wut?^aY3#84OONEu#$q^q>R04X%UiU>VID{uye(J2Baev)CG}gdyR8 z9F(~Q{TnzE?m~-$L|7QhMtaeolUGPvXiWG{=v?rT&*xnq>D)RyLn6b6FJ+7eB=le4^KhF=vJf#QbFUfbf%d( zp6CM8!n^&Q11ungVdF04-D7`Xa@Z_x2`8SN$ZW(3{5#_l50qYrl1MM}h3wNz30uNd za9FH%_yXh;qQ!4AKe7h0TXKHpbreBSbuo&VRH=$Hns|)wU^Vb-L^niP!c0N6z%3dr z`AIyDzlb%Gun@)U`2t9EmG>v32>Bg;kHld=p&HN`9!j<#S>gHg)}R19K2fPMcqz$3< z0tUf-(IrSOv=$nL#jzQ}Wa(4s2C-cr6%+|mrE{Z3MA@WrNhk3)$)+eQ8i{7fugS&n zJyIs7ok@*Ju823pu8CWj7?b`s=VNQI<@wC~jL(^Uvb|YZ8OKsjrVY#*(9+d*U&nQw zbnSCnPt0j*F{$O^Rzyor*7x+qY4nV_Ssk*UX3?^&SuI-Cw_DsHzy0YpZ(E4dM!0$ z_$$&LD@5lYI2;BKLthaWX2zSamXTC(d*Gr!kL*t8gDvUp$S(d@KBGT5aKwxC3J7Ko3&EyPpYKG z9rLR~Lit){znlYXHo$(nE5X}xIfVQMh-vn9HZ`xL&1-mTt^zV!hcSrp0( zF7_;NDC~=!RjzTKJKilmm49z=0$EI6B47EJySTPN*1Ha;Z)ms`?MoyxIqHuITn|`$ zH{5-knU3R*&2E=p96Cw*eH&ft?XZ2mU1IZ@Zt5zUixi|n-IS*itF|{I>Z{rT`Wc38 zMuE|)mbhVtLb zTUMT}T3V$pA6Q~2>Roc9R8cy#w0G(Ivb`0TE1y@}>J0V#+O*065vb zm!ciOQS?H9g$%*kVU6$sU^nd;t(3kW`~o?l@z6|g2pE8m5YL(K@WYUv{s$<)?2Jte z0fxY0uo$=wtw2hUY@`*mfX=3krqiL<&_DEh!CYS}pCqs)xF+cJck!Ncy>UYB58gY0 z;ba1t5quu}EBp)42ACA$`$o9@&d1)Ukeb$>HaL__hRGA*CE>%NB_TO^%s=0ch5iEO zKzo5zk<=+fwx@jr2&fXs3vUZ(eZL2;hfw+(`gMAM{sVL%SMec47@Y!Q;gk>?P@qm? z2{9J?1Mwjbae_%Q-OTpP1I!H^o{%lGMfZ=%j5;R9M5Jh{^pfncq(Cs9dyAvzrHfTE zhP+YsKw2fE#W3S1B@B-r7t=1PvrHI0F(EExM9QVaLvgurLlbz3s(4wPJ9>jWAv!j0 zV)BiQzFA+=pQS8JKA!wX%FUF)NgZNWNL2je1keSdR_#)bXZNLlg z5$JP(2oDMefc?l}G!BUaztGmu+eYr=I=mIqfO^P;l8gbI72I6T4W^Lzjvd6uFmUE^ zyg&3C%|w?X%P}XqAKn8dLM@TWXaOoiFT>x#?{qr73sgWT*gvt~6ERpc7Efd|hTtqD z7Lr4K;B_d6@Dg|M0`xpQ6EZ-3k$dnX`s%=C_f~hQ@4w)Spu+deqxaR3m&1S3Is;>Y zz2VM*R^Da4|3WW-Lv)l};rqikH272KdN9qm$UVb#*JXD<^LF(=@E7@NJ%p#VZ&`3F z*(o4$Z?S!gRGbsMSG_$w<6LcAAKXQrkzUN#${Tdlm>PA{3<2wI*HhOWTaMABTWWk} zHW>TsM@G&IwT3mOuC^4{YEQa%f#t^V zui%PccJMdf1ZTZPK0N($exn)pgY`tBzG)s7a|I ztMAnIX;`Rut8^+!MS?=3h-&7j(wY`3Dw}%f63jDg@9bXtS!dW&=kFHE2e(K1UN@04 z@EZ6J)|G8$A7>0gMeug?5F?kF&Tuf!u|}~sv8Qpi@!8UT(HYU*qdG;=q8nm=(yLI+eaFqeE&zLS^jE_!mj@ zlV&H(N!XaMC{7f!A=1~=B&AQgdXrHcRG80^-M_ zEuw7c;;8r1bm3~wK=w}lJn6Hj9GOnER?u7co0u!UEI|2cZWd<=3uKMt3=j`%S+%TF%o&VPjDAcT>p%7*rVX71REB2*2`G(7BHCdy zk(201;yQ5{KZh^CqtT(zVXz6lkHgH~jGdSa*$K}?ZX;hIIh`c`CT(E@@DscMxr?ko zMj)Ng*7&2yz1%~-((U1wU{oZJLep zF1QVP0KJC-csE>x%Aq9SJpduAF#&oR7!w*DoE1C~+!<)&``i7u%jI6|`z??Zbo#$} z$9txD^1XlgLcY(Qg$~lX&{6G)@$dD8-3rGiJHxru1x0d@{{mM3R>OEk;1TXbICW35OdQD0QG(A4U8n_R}5`dPZQdeZ1HueZE5R~Xmm+vse% zyT(h_fzAe37njd5&e6?LXdh=SF)Y`0GO=LX{fIssmV z6yVdCcGfVqnq_1qvwyHwu;N)w3=y#tyMUg>o-oF;i&+zhT;vCM9Q-}9!&m`jfyW~C zXX|izIEy}={!h3l_^W@u@0j|cr*^O)XWqnnfB%^AyGLhJP9vO%=^^REFl!k!1aj>Wez4iPMeH@wD^Qf2s;pk5&22 z2O2Ndey@qGbJP*_@9NgnW!4{U*wXlm;-0di>3;L}=CjIU3TM+s&2#N&O)C|oI;%Zn z-r+dxs&JHFNzlNOWCE& z>1-i?g2MCb3=Y9t;jZw1$Z%Z5yvH8K zRde%sBY2&->)6pu2p^3+g-Spz*cJYU)T1IS0X>b(!#)w0h@n^lau~KEX?P03#cAjZ zcqx*L@$hljV01jP9o`P_M9k<7{2Zg6S;_3joXt3b`;Z1`b|e)WKzl%&1DL={NDj|} zeg*5mXYf{3i2Z?HM2eBqSQW8?u?L@w^oBk_3Zx~bLun`np+)*0?H~~D2)BSs5Hq$N z`#JLLMuHOhj?k;XMqjB94R$5vU=K?SNHD1_5 zx^i7&=f5_a`K77Zw9z!jm~0rSU#Axsiu4(Jw*HN7g*K=;sjoDr+jdwX^KJcKnupEL z8pqXbuRGf)ZtkaEs!mf`nx-`wntG~!Q7M|YsrT#Z^cDIs297b`m~CVm&ghbLg}MrT zp0P2q*J!r=Y}K0P884U!+c8Iaq$BH#^nr7%bq2O}uqIoVWIS*7n1yDmnXu_?arTyu zZ_Wg_%PsV1-K4vn=d8Pri|N$bWswg2HtQPu9%qtU;PyE{o8Fvjd1`y@= zD7GdU-5Q$ai)O5brOsFRn)fzuXg<|k)BHxIRNqh!Q*CaZ-n_OcPdQDgP_9ruY3x$p zsdi<}x0>;F?)q?DY4yU&NfrGn)Rn*2%&vP?-@f6Gy1~`YDyqua6)P%sS9Ga3U9K!! zTK1rfSrJz?s3x&)aXqUcx#3p**1EV_q-Iw2r|N?><7(E|w5X>k_crfPxtixi9!;%a zDHk=b(48=!w(YZ5+16R!82>Q5GMq5hn{q6dEE6olEl!KU`pNdfKGK=&zU$fVyB*jO zTo+tG?hFkG-6FF>hVXHK2lqhcA(qJR-8H&9G&9sli-l{j1H^+!e{Bi5xi#9(^-ZhDahQu?*NnZvm77T3{IPZ=|PnHIz?Z2xHhNv=w-cjP(Ql_QAx+x)SL` z(=XE*krn4S^c*-#%b;DM-+9uHfBemO>Mu8;I7P2A(@W=QR z><(f>cH<`)dIG>sLc767a0hH2euU`4I7EEIGl}`k?Od@iN!Xruo4ud?Z)6^7hBzoX zEzA&p5{kqH;=v-eaJF!eXp1;Ob|Kmr+ctJwRA13retW?%;S@m&ULE@r>oB_q=LS2# z60wwwi;NkpuDrftr|e;5dM%gxgIN@*$nWv?@YGz4H;nh3SIV2oEoO-sL5##sGv;wn zzLPhL^Ma9wtMF$`E2o0joxewLOYD^{lTDFrl!nF0;&x)c4{3G)ovjn#zr(q_X3v~tDk?g@p{uxq{ zOM<`nY|bs#3&tCIzHyR;ZX?Y+eG65fsZ_mE_q%SkF4E^W(QK$=yS=;ZspX4txlW*6 ztcx~YF%=l!>+WgZY5ZD9U!v=&-K7a@qYRx*NtTt7$)G%wS3g(x!*I?T?O0&THznvF z>6w;E_7?U5k$uf`Te-cx)8RZAnLw~R)<&8qC#?e*IixPNr^0*CbJKO%)#M#YJ_)rC>~#(@?=vKu@~uVoL(UbhwXT0$eOxac(GI`8 zz3s7?G?iK=I7ru8C&Qx9ZPOI0yJ$XX#rhV8d}Flnn|`?dk#4*uz3K1z1GQ`G2P>yE zjcACjoK=)k__grBkBXv{Qenk_%8bfWmBuP|&E4v_DtYDZ%Hrz#^&Z6|#hAK*6|0MT z{=B4x*C1$mKs+DQu?G=QXX4-qd{Busd7_^@q6a?Ws&*P8znEw ziYq2ozpLA&C}=*Wn$SE($yAJ1+SOZ)F_w;|{<@{=r>a`@2d!3@s_&-%SAWa+*fPm> z!#d5p*)T%?#^|-(ccb1Tu06I57SOie@z^=s_1x9Lwbl8?Im+F~_c8Ey;CElxGuyYA ztfBkC9>9CD&S&)&`5nO|a((c4FqYgO+D&&s8uSpp11IsbtO0`Il3yjmgi+i(tYWs7 zcSX2dd{dkz+9O!W|H{kee~;9A_1x{8>zs1_R*73$E#z@Bnb(+;IEVQ&gn%enBoG?; zCj^bcWkQfYnbV%Lk-tj9lb1xLh~KiWVOr#8Yz(oJIhGA^P);M8!Aa&m=Z@e=Sj(89 z*cPIHqRsJb;?SrKqQL@};F&m4Ha4noe0Oo#Vc4qWt zFqoSo{m#+!djXNJIFiyF^kad&z8udYcOS3GPY-qrJn^M^*LaS2-Tu*JDVY}N?)l-0 z^4<^JCdZTS16BT00dMdjSxvSjt0F0hHn<^t8n^)+0VjuB`=7Y0+^PP>q>~H>)sY#w ziy>a*>^C3y9efE^f{pM*ykDdXMziZb3?aipXmkkr}WbxhA`02NE@Ud5{nID zFgeEsEhNjt4cx7G9JCGS2YpAnVaISOb3Z$gJBqi4r{)Y|l@aZU<;*;;S};?TAb7!w z!WHOP%!n$GWymn}0-nqq$ZElw!xS(^5><@NTvGf~v?WF%A1ay8|BbVRlh3Q?d-=J7 zHKIw8E^<;#o&1$dB3ARpa5MM{VX-JvxQV-+c?aK)8X%IsKHMktIy9K}I?^SM2Hu8a z$>#xMWa8&Qcs(sIw9vQIdB#TA`r7iX0&|Vwrv8Awo8gHe@~yhR)k0N4vp}_7-Ag-7 zJ6Y{&DvxADt(&?i4>sJcJ6GSTv1j9<#_q}&O;ei1Dv9Q*eumj$nPui1dTK9fcIh@5 zL#Ey)hS6#W8|RppS>mmCt*vbaTcf?!(ZjVRlBo%7PmRM2sPS$j2M_6w=zAL08XWqa z`hae{Znw^^yP#*AZ#g)g2d-aj(@Z0c!z~vb?_HVhE}qlg9bU|n;2P;zY3pa%Vk$LV zuugM2-EZ9o9X4~Gp_iUxIBIa}vkgL%+cecUH8K<1TZ?PPtLmB++6C5auH()}mVEsQ zb#pV`d_mb)F-S2+DQyxrtyFGOrjnU)>cN>mNfjLnAy;?c4$pWjkjh^?fm-g zieS?;RXbICRej{+2xVr&vB*B^wkFqj$h6x~s2Q!g9+_k>Zl2aOv-z5Or1pjGwqdQw zXkr?dX$iGMJxDJ!zp?JKZM2E(>+Nmq+kG);AXkndWwTN zk-GNJaJSGqGCMK>)gyd6G&VFM)Govdvw*+g7<3IX4jv1A0jEF}@JM75G?Bh1JUBd^ z_5jdA@yJW$5K;|qMV#0k;ys>#`O%NWNe&<=lbsXI7hybCUT1Z;$yRS-=!#E@K$l z7g!Xk2}0x>(iD0~s{(T160|2Vonawhya0KGc!)|)9`8V;zkU)cLX>bi^fzz{m=Df{ zmPV$#wqOS%yNvVrD0DC4LvcnryPAEQ`4Cql%b@{a4!A9nXK#)46gq%)@Cs~4qyjO( zPvCfbFsnP~9Xp#@j`2|iJQG|?-%HoiAJbj*L7*C*ia%jYW?aS9*eqO2bY`7p-)HS* z9LGmsI5rt$V>^&lkQYb*Z$K<$BJv70Kzeux!R9>Y?&AhHBO*OfJyDH!!lgI|TMOTg z^s_I78$zu@-AP+O=dbt21z(Ysq#}U$+PQyrE^#VcW8H&X&m5QRY)6jky*tkBcP?^$ z^E3s#;X?W$dUbefkn8K`5qtW0=lZ_|J_e2l0)g?tpM#9x*uZ>$e}CUV25AaU1y%ur zY2wgkG8~zt=nO4`XwVVpENq5n!*YlVB+{nSSP+c=#+=B0#9qmKf_dQY&~I=QR!^iw zCThg&K9Q-*Ys5GVf?v{4&go3F&FJ~f`&-=Z zb-)|o5a@w_!WLl$A^-E0 zc~ia9A{n8B42IMs9Ju1?=X&HUb+z_R2`mX2!&Ab;!}saqAptxB;(-?6Ftiz70x`h< z=wHKbavE7k4hbCzoexb7jSN+Ux`c~E8_3E4FK{hzikwHA2L$QT@GJi*cZGxJm=(#y z{&)Y45p{Hw03|>PPy&PPy&PPy&PPy&`<5}*Vq0ZM=ppaduZN`Mle1SkPY zfD)htC;>`<5}*Vq0ZM=ppaduZN`Mle1SkPYfD)htC;>`<5}*Vq0ZM=ppaduZO5p$J G0{;(uPT$r5 literal 0 HcmV?d00001 diff --git a/tensorflow/core/kernels/fuzzing/corpus/decode_wav/793feab2deb35e284a975f6527d76a8be5540fe6 b/tensorflow/core/kernels/fuzzing/corpus/decode_wav/793feab2deb35e284a975f6527d76a8be5540fe6 new file mode 100644 index 0000000000000000000000000000000000000000..8a9216e10b777da5e8888e5549ab39346de8acf2 GIT binary patch literal 61 zcmWIYbaQJ+V`K<-40BD(Em06)U|?VbLYFlR3>^`S3``6H3@M3;j4}MdI`3@W^d5e! Q@m%Bik=G{gmhuM!0L>l}?f?J) literal 0 HcmV?d00001 diff --git a/tensorflow/core/kernels/fuzzing/corpus/decode_wav/7f41ec3a9805c6b8f3656c4f9f6d0ff7dbf8a329 b/tensorflow/core/kernels/fuzzing/corpus/decode_wav/7f41ec3a9805c6b8f3656c4f9f6d0ff7dbf8a329 new file mode 100644 index 0000000000000000000000000000000000000000..aaa91f2f45bd5730550211f4a080e6119d15b48e GIT binary patch literal 6150 zcmeH~S8x=^x`1aVhn?A(jgoeSBrqT_!LkrwlQANgEKE9ZFd`W^L|K3Y2utK7Fgjqs z1|y*0AWXE$BBx`JiN+!brA@O@o5O58TXn1M+kLwa_f}DV{ZHLp)!p@f{q>(WZ1CXH zIU-=X#Z=iMTbZ!QtWJ<)}pcCO#ynuW{ zgs?i~9dZiOVK2~u*muMc@fBtaeUJheGdqFJXWG-P>1p&Xu}7RDu}Pw=QFch)QM!n2 z#W~qVssPD?&Vx2l4zvboK#Q=MdTa=>glta+iABU;_z3bmwM4v+X<&Mb*HbI#V##4i zPi6;0u$1H$y^AT5{LXZzx6`8~Uo&R$Q>KtT!u}=AR(+&t-)ePornG>$NcW=W&??Ca z)*|5~;}UU}*~GM!Oky{3FIWdXRXm3tN&iUYQ?se|q#HkpeTG%xXNVlqO*6)$VmA}=@{8@MND;8 z9n$2caLV`GMK)bJoZBGrO8&(nEF>wGn7PjKX$p;MOUj5;iF&S5Cto0&&aPrhxgRC- z=mFGD{4D-Y@pbNn+%Fp}eIu=qZId>$|6p+H2vJ3P2_-R|N~hXm!?7arpJX$xLk~e2 z@qS@PXh5Vk3`Fw68~wLDv;BxLLFgPT^k@0L_AGFB^CG?@?#C?!);#NT+iPp4MQtuO zO}19I{A$-*I+-r$JL<0)yYhXFzw?+mk3ViaZ#ru!wym}QWF26>Z>r~O4Q2X0dY_@h zRA?zT?a;S1Sol%KUkuIWO*WIYkM))1zU8(>X4~h;cb>CvwwE|+oi9As0-p+{!sW0u z_H$%-=x+FA^i|}K@ZTf4*hA3)unani>_@j^#YC9+1iy^@fbggW{ycs>RuR7&Q^xxN zYk?c0Ja`sz7k&kQ0lySwfXm@RkdE(-Wydb2~PsC2fHQ*X}AzTK!foZYr;gunM z@Lpi5Ff~*q90)G==jA2n_d&(z!Tp?S0_i0mfp5< zYnJV@Us<1+HW~5^K7O2Kr{y{y(~Z{MGdwiCv#hf9bp%~8 z*Gwnt-0QmF+3OwS?d|(2cq&{LS)Iu8*zVY^_(HHzsfZ3Xrc8*aH_9$}qo%dn=I zC{tJK2FnlA(xS!t0Iq z%-0<sdnwQ=utEm=Z^D5ZFwSS;32b>gk!vE*XB4Ym*^uwSvkxRe|u zK1BOyn%OMTaNXI3k{T{g?vRa&u&Fxc-cuUM%^TE#%HhRZ^52#JQx^@`H*x<$O)0-sL20Y6&Y2Ul?PQ$)$^oV>cZ5|)al8|$w5`Nsv_yK zYOOp=wnDZ+zFRRxo++CmpQxCw{4Hr;a`)s>$(2dxm5^ek+#@UCb~D%LuJl~-b@5`F zWBM`_eMDSAL-ZbMjF@E>({SSd`7>!GQ^h868r7NDf`>5$W``e%GGaBs9llqAjL-=o zUFZ?43hWN<4=xKX5Pk}-^ILrtz9KK?&G7c~{OQ`@y5W?ythP!<)x8&1^i^^j=rX@3+3S?XZ1ke`}|$2Y3;$O(bYY)}PR&=??2U86Fv9 zreoHZwt8ERJ=HePBDYyv;w=YT`Z`c2?y7c*oxR)|-!Xr2&=}MQ3Wc)BjOd!kkZ^i< zW@LZld1#Xm2{~d5AuC#dAH^`V4jv7i6TzZNpfBK#_l_&$#XuF9hm6Ipp;r(PyMz6N zev3{+mmxOvF5VfVk$UJmkb$b>@73~voqV-WoxDu=mjSn9T@xhJ34Iy7B8ngxO z1YY?6<(c*A-p)}COf84`*q`pslUwA2>+gs_L<@7opI2Fzc`*PbB z+gfXH%VG0u^LR63`q}V8Uu(E#$kiX$8k>eSWjFU~{##Q~Q)yFmV^!m(rgP0Y?Ok1- zzC?dopJ6=Azv1&t&&`9ZHCEKxVA*AF<@(0c=uP%Dd0N~X-78()oFkkWZlkBf+s-#K zuq2cdy&hW=*8}CyF62Bi6vgpp*q_KXcm{kGR-m)7r^Fk23JY)<(p1@5=~!;5TF@|-fn)e`0wB-eckvnIE@~T9O&r8;lU-;l{Si|qUQezd zKE<~o9pGiiaNJDppo+=)#2XyK7oq*(DPV&r6`Bo4p$X6f_%;?Mejsu39)1@agdZRZ z$S)~?e#F#B=COWmr2MI3mNHfKSk*siPtwk$ElIgaRjRH@^~w5_1?u*i$C^Q@J5tM2 z$E8Lz4Vu;(U-G!5pz5*mQ$?oyHz_Agm)(_(mLFC6lb>rYrQT7uPTr$RQzpx2OL;EJ zt>QAdE8JA+8ktJbF|lH+Y@u`xyPx@nE)<){8XUxOk--oI-V?0@qtIcv5|$y8VGg8$ z%4k_=q%c;f3+(W3448s9gBt?Z{b_-;U?4bHhz6Gn{ld?}7ll9lzk45eJZ{*v&T+!| zgS*20#<`*;-8Rzlr>PfzMPJ^0sxiM&m532dGVS*|k1k91y>_0qzqU%dNJr_U+TL1+ zR?up-WjeJ%!F&0krtRibi`|5qF7addrKV4)!3T;*$l&_`mU0_*r3i=!sAv zs6$&qmeB45z2$+=0{Vbj*c>uODuA6(Ih+j-fI5IT0d>50EF5KH>*H@ld!Rqy0(1j@ zk?2Egz|-(a*vrIg@e27C{T5q-I^in#5d0ec9?3_(gguZF)!I#=Du`Y8=>8~BFFjT4*N zG&eLq&@%c?#$~3|4}$w3xDSH+Ah`b)oVk^$njd8zZ^<>gOj65UOCM{VUGM1N`8aSx zKthl(EpW;=*xT7N+x^Z(xchm=d8hk!1%?Z&5~D4LsPS0~!b zIYb+35Pg$Lkz`0}Bw@CO^K$jv18%4^O(vEVNL|t*d97lMa)8pN{9xGs$*>1hH-BM7wz5UUR~{yjzNIn?(WXu?mD;(?(Q}W4DN#t?(RCc$CI|SyZf#C7rgJU zcdakA3KlH7(&<#4=j^>d=jq$EQ>T>^0l}cQ13ONfJ~xXK2m}Csbhsc8Y#a;-KmjKh zKW6Tj;r##61upv<2Ad^jrRK!g$jx{z)DE-Z4T)jcG;}tmMyo>IK>+T9@1zcqeaTPs z7Iqcej#G&M-0};gvkpkobrpLaZ?XVTZ3HlD(n>kDgSPM6h z8^cy%Dlj9NKWQJW7TSdgu3U0dT1`BI%MiJ_QfdxT7rF}Cfrij>qy$}v8L-a8EV2q2 zCT0*n@E+tGYpDgyBH?ji6LvdGaFp;d^C!Dd_=au7Y-N5Ej$$qJM|LiE zg8Lz^ugK9<$yln&70+RBGR>KZj6%48vkFDRe*AHn-N0574(8U2zHlyPI6aZ+$sC}@ zP!p&sq!&MrHO0>1mxzX>m%7VbV*9df=-Ffs>bo!?^~)y9#wgmUAEhtIvSz&01T-Tx z#p=t-G8rlBFGVFG2`TLMKvAOB6*SMCYL4dE?Ot_3(s%}2MHGmEuxyz zkur^9le$N`Fzt7_UOG)Onp?uH6de#wX4+Ca@XPpL^aIfsX;9KhoG0ET*&^0*Q(2rk zL7XG~gq#>nWl>eI?$|=|FH(!^(W6jys&&j2Z5#g?6U6(*)(0Q^CIpelz(~#T++f|n zDBm=113wZt;r-7!$JW>O+5XFxYfZCkGY_%dcV2TCtToIx4Al(x%4?fimA^4zmcFLb zQ(-)QES;?H``j;zFS{d zpI9aKeXcR?D~^qhWv-v@FTQ)BdXbfp+c9zSNW6RWS?qk`d;EQDL|mVI1snobK_`*@ z=w@sY5hLp0w~@aQ6RLrmrA{YzrJg0_sn&wkf`>p~cs%k9{tmZ?zXI9de0VO%r1mE3 zC#xk-rZ%R|CeNod;Bt66ybAORMkcq$7Do-?=b_<|;n8!EL*e;BJWxM4D{?-uGWjS9 zRuQBpvl7h`rxVqZ(WEKWSuiE_I8h@$BiiMvAepS5P{i5TKar*3cqkaE7A^|y4UP>| z4rl^@`pbO%d{*ySZx8Qz&nS=0bKVR3vVEIk!{f(v7vD$K_%6xAE}vwCcqHzCR6}VLUto}5eL#3orj)BJ7J^ny2MuE z4dEtQk=3Y{w2uDH%oOhA_H%u?6`WX9E_x#FFL@7w1OFTjRulSo-FYYQC zF1;uFB%i8Ss+g%5r6`m?k-d{nl}?sUlCskFlB?nm;-L6|SS|L6f}*=3Oyn1u*-C6n z<|MV2n2J3`rofdU5bOeY1l56ZpgOn}V1QOYN5Czx2}S_lfLu@o?f}LEwZRV1Yv?_6 z9r_cR3?;x^@HpTD#)4hIIB*}>0L%yK11kgxTD!LdZs#G-h^_=)(hc+dE{m?%~j z{TVe!$ym$S@mNN@RovY7e}(IRh3kKX>;JQG=|dj^uRJwejh!v*+iZ31x2>m4UPA}{ zbA7Jqg{^_}l;gYYgL#9ouQ6ciXWe0aU`pzL(?2)9G8b5v*ju^6o}_1-n{)5=-03+gtJi(f|8Ksc~6-ZZi`GAcGLJ}$O9 zIx$`n-xBG?m*OY>MSivKr>BN@sW;#4a9p#`aV)dr)`ylJw&C_{Tc(*Z*S7Vrtu-|^ zywa`FrRf;0OiSvP={oDH8sC^!TRNJ58oBbtrn%)640-xn#)YO?CPVpi%LA9h*T@(2 z-wM?T4-a(>p7am)HS`_!7X?>`_J-<4M<&;Ur_l{~H*77MhJ8ZLfwzHQ&{Je7QXifN z*T-ah$-PUdnCe1OWZ^LO37H_;(75oku%fUoyPH9TKSVj=DZ(901{-JUQ!cy*QGr}V zekA`Ue-Ul*Mf_*g1{;76qHgj{$1x_KnL_Qu-=h*_7E@K)}+pt!M0I$vd6yKEm%Y~_>WLLF5tNmmZcC$`2{Os>Y?cG@~^G z(&nUD)vZ)TiiF%EKdf*oJ}V!m%}sBbmZegu!ixHeUCP^v)zZ3>1(J2r-LhfQT*)x$ zAlYd7b>%))BULZeG38Y`BwH-?N#=-lv-g#g5JeI%pR&Y&9O5X zm~X|7kmY1LZKgA+n#3kNhRHAo{1(Vgz6);;d=F(u&qlH$jl<_cyTkj#^TN|22g7TE zw!p5yLOY;WjDw@ij?u7402gD9XP2u(7bkRz zcadwf+wc5jKV}PAo7jT3t@c`ubM|zb-O|L|u{>s&VYp$eP@bu;pj&KgYu;tW9nkF8`GNbPjEFaOF?mst3)TlO0gnXA)Z)|(pewLaAOTE( z5c&+Qh2BCN;Md4pER2i-YlHV8FDL-!r0yg&{P+4fF+Uzl9)hd#QsWhxht@?-LJa_G zVo|J7WM!yg@U)Nfi34>4?fq21>p$il@AkW1x@GQNj`{XY_SLo)*5j55mH`&le8l+0 z@Y8tD*wJuWS6VD7&K7>z?WR8kQL@8nVleoAOM3&7UnD zZ0~HSt;G7LBf~SvSL#;<%6v}mdhcRS19uO1wzu53%wIXsGc+sOF!3O{JY^7UgZ@OW zB3)1%|Ac)*?!jZ>yRZzMfPEzLm|>hilr2t|To(5c%@#gm#xlk1G42NcG#(axmyohs z(wCBr5>680ikZjc0^EuT@Sd0%c?F+_!gupJP+xPTgdIy zB613mheP-bv<*BAECJG?32*`$2)%@#U@_uv5+|SI&#(^oAz}{MjS?_#*>}RpTu{_g z`cXDsp04;$(MGvPxkI^0*-?2;QCnH8GN`AeRnh#X>5#rXeOr3J^th%(Q$Z6@^;3ow z|HX<7x<9z~{HC7mEPi4vkEqFm7((Qxr{i9%M5 zf5m3Wbn!%PKRbz;OPk4eIEZybIzbTl99RP;pyTi{Sb_|JMIggBpR1xhBYh%;q3yx- zA#?aqcwOj0Ff)`H4uv~K65;ug*0E2qn~`t9H~yDCpBMJ5ah-Ml?cL?gb1!ga*?U^Q znVXyL7`ACIl#VG?@JEj_iEgLfr?0Eushg~8qdTXYp{Mj>T??H{7t!f-tMq9`naOYJ zV%};=w>r$Y`If1-X|}nZE#VmC=DfRocLI`7@8G1st{@lb9{mv66-kS3idv()`KNbV zsAl{mTHlVCAj3;R6ei=dJoS**Wovbmc%+d6CaFy z<*UVaQ<&%sCGU+_+34AKquL2_6F^#cy3lJTw4%-HMrxcJ>zVQfIGO{`LU zZ+v9pN%C$IO=QI!k+YGSkwu~T0kiLm7xnh?%<`DMzkI*>27BkZFFJ(wg|=qaRpy$e zkL5#5mrTE#&YK|19P0t=MN5;|+F0w@{pi=o>F|QknBdVs%Yfa#-+$b1^B?vf^X>CK z^0fB+=^?xZ&j(L^Z(Z+tPj$~6kJB^FyTy`b1?)jD<6Z8V;rZPo^f;t0*mK2m!Q0th zVM9hQ)p6NWDK2Xv@2D^->MHLkZ>j!NUsc;w8OlNO&Wf37|7CQ@Sf&=p3WaT%Y*$P}wkw|h+LWxHU574iPhIj-AsgZPhsu$J~oRKVy!-?k!NAi{+AFPFZ!`hHz zDHn+lN6MZS`StQ)(QFpji6#=F20fP3v9?eB#Y?Iv=Y8#YtTP&F{VYvLrZ~e zfEBn41b}W(IWmg4M~$G)66N>=!b~<}jO$j9-#FrwM!6$-~snTSfWUGWHW{H+WGviMac?oR{ zjWh`U=AY%)1_lMY1s?iNd0Ahrz^h^!l{qIE}$T_F3v?dh2)_Q;oqYl zVsB#mVzJne_-}D4J}5Rd+Bw=WRyE;BjRF^d-39XGnnWsT0h+;cU;(_FuUt-KB9aDk zU2>~t5q?RnETMGMePlW4*h1`1aY>8AdN)#3j;|2A5-ZFa0brbq2 zfQVrMJVb7!IuMo60k9uT17w2#1X17|SPVV~cR?YfF+Kygp;zGQkOmqJodDMeswP@S zRs2d^EqoyKA$TiLFK|1sDYzkoMeav_ge!-~MDE9YiFmR+fyB;)ItA|dzX$4uN5*C) z?WvKe-l^-r5WZgZfkluD{0m-#%z`QCKJY#jOpHl7$>yoA$(4yb zUP7OX9ZXCXOy=dbGW9fiF!;+S@=fsH4sPPBT7yVgq)U8Ia$T}%a(=RBN+;L~wuO3t z`%_Ef=E$zl-+`6>T>cpEuMyZ5OoX~E4v&;J$-s)y)(Xtw)GfOL$eJ(53 zcGMGwh5EP_)?U(SOr34j92f21Ea~O*^jEc8%eIt}+IZRTWk1SbeP_P&Xk9KUpJSY0 zn593W|I@I-@SpyKexm+{?u@o!*#W*o`=y{x-n{L9SO3QJvciFt>;m%GH>%Jqk% zk3DR&*;d%6Ip4c&o)w<8p54Cj!4=_K5m!VJWnv5Xx^yrWk168Q<2B>0VsPvOuXeu2 zwekM(&#@)(@rlNX(ebCTRf#y*6CX@;#%950V67mLsw2n*4&Z--=pWT>>Aw4dl8lTHlCwwRY8ltde9o_l&l9-1IGeosrAWjiO=z=iOI<>soSahf>}@w+6PnO z-Ej_^fWWXCItX1yMq>&LL5Cr4VISN81+m(g1#3%Grk>K%>2z`xRtdX9sOdO0ja-W# zLxqSOo)4c!dZ1g;Gw4$k#!nCoX(nzFL-23NKTrrf3Q3S{XfF$FMvS%t8Q zu%>Vt7ZvRltr4zZMl<=s;o|1viNe}c3w$%S75xZbgJ+(o$c@>3C!O3B5ri7QYbQqGys{ z2@$aq3u7%QfzZzF7v+h!N*~JF$Un)n(i^<;s3$LwPgWQd8H&qNrMN41NLVV&;aYIL zxYyh?@ij>m$vFP~jhA#*Y)>1QUN8N2+EUeJ#b()0ajvM9=&E><^s;P#!l%mDEXatZ zH%)((-Xz12Zb|=^QJT3sb7lHTbr6TqQ0;IEAgxPGwgK5m6t}9{yim3*QSh z+-RYQozK)@Z*UzXFC@<;Z=|4No|;I<(*I1`t}2mlm%JDLqP?UYC$L>eQ+O}b14;*t zf@^|51wB&}67OOiqhzFEI4`&+Ff-r{tO(u=9uD;OfAiJy?ec=&1@1A<^LEDm&3?mi z-SN%-!`jKx$@0WJ#`L~CT0Y0*F$GO7Qw4L(wAh534whFd_Zb_M2g^5_O_p4X&UBgo z6R;Lp4p~lEx>>hb<<^-NyLqGeiD{$hk|o_?b_Lx_Tr(Xn?DuRJES1bRO#961tUlWV z`zA+Qr^|WB)ygGt=&iG?`>cztx2>zKA1sZnyKHCeog6zH)0}PHL%bdQg90rAH3IVk zX97U*P7n(}410o8{TqB|y|>(FohA0}_Vtd>&Jce*a(#9;@f>qYTdj` zYgL=lHo(eQR+w5EyXqsl)B0RPx{z8d?}$=|7_m)yry}N z@+RiD$w%_%=C3X|TG*|qv}j22`J&;4(+h6p=jEe?sY0k2E3Kv7svEBFV(=SRnBQ4< z+B(`g+h*G|wo%sYmU))#)-SdbcFtk94|Q~NwfB^H#NMjDSAIv}dGK4Pe`F}X?tY4X z3s>h|jZ1+p{$IZH{$;`G;Ze~U@oLGIf`iZpl;*vJ$@nGw5dIZ=fh3@@@N%Rrk^ygk z!tfNFq3Td6@+z$o-r&5VYm(*i!|F;INanbVXX(=Pi<-A-PnG}5cFMH!N(xxmRz6Zt%8xj0WUUG_}YQeIi9NgJQBG%F`dk*UsXli4sMTO&{}QPoX* zp}{n2wNvR;{;e9M8mf3G+bONk z`s!9{_i8KZI_g5Y`-c7HIp(F74c4x<+qO%#X0~zG4VGgTz^1eR<(%!dd!zn?fro*% z!3V+Jp`1vc==bRH*uQZsaWmdIu8B8^Z;HzjvgD^!A<&kWuj%kVa8F+Jbw#Sd4sbLW z2K`VMI0TR6MPE~LKJg15hOJFV_5j;vhO*T$xlTfl*VtBkpY;$CB=x1PLV02(;01x&K%?)pk zWJa~&PC>ozo~MVaw_~=wrG2ISq~oyblc&_TEKogoJ+R5Y-FMJ8(~kyR{?q;g{$u_I z{(Rp(|IFak(2MY(=+#)`_)>nMQz!KCiTroykB>;6ON9km@EI@8tDuX~|4=`A2_1q6 zAUV)ourSpnDNH^{8j|~xmy;b*y#yNp6x;$73EBy!r>uYK4%Cg?2nI-Lun-t|0l;u0~ z>B6r?CrSimLhTscHvK-ml0Od_-Wzrp+8G~~bLJM7Ue?!EpLKyPW!r9V?Y!x(#kY>x;%f`z_$kehgQnz%wtgHNkyo=(QqMmZQ@|<#;@_@3Ta+EyI=P^UXDzQ>L zT6{-5RPs(zC~hcP%XQ~O!he{WOo*1ykH{u`)1bjDyr8dyO~tPgb;xVP82m9>2{{2x z04D>R1d)_Nuv?%25eNzRCG@-1_1 zX|8Ou98u0uEmaRyH&@vdv*c%`nUbmE8oX_ILA*qoCx53DsNbsn>IUkU%3Q@ZSw*Q< zyg;NE9%IikN9dhY5xIb@MqVJE;wyRk=K+EtKcRM@7`P-jE|?@}AgC#*E~q2u%%4>S z9IzE^&s#^Wz#L#is%K(E>`O!xDGpx^-wOAQypD8@R1e<>&iAkKJa(S37g#CFK2xyV zUoJAm%FmT&mTxrPGn~)^`gQt|hPj3&hQ)?_gU3+QXf&3V+e{vFo@JHQVeMesY+Gcj zZmVD$Xmi>AaVXqH9+6KSm>)}E-lEB;0X4KRg}XZ=An5Wv-YD^#Hz>fsw)K!F1kKF#4`|2D=73&e&%0i|bC?b9=;b!d2|P z64nM}fy|&Iv?M$!G&k_C&*eKC?8GONn?h^+=RB0V(Dl(x`~D3y3f}Q;c0Y9faD;5% zO)GUrN>`K}(J6ToZMnWccf_DD{b62f30v-(WhO=WKgOD-BI_1Mz<$v>!Lrnvu=||L zUC*4q>`ScmEFQ~M`zd!#;$t++Z4=etwz5iL zHirl=P$#jW$O7~TF^2Xsz1VWb&Kq2xnYm0Bqo-u#Cf;;ChTo$Gv0s^?WP9{8v=3SY z`{B8~jN1z&Qgu^5Q`Ld7z>U%z{+D5r(PrpxIBU3Kd{-VfWm@`JMCQrm ze;bb&?;1xNc9#89^rDsa>+H1OB`s;=bhGV+DTEBjk>5;j= zsg4mg9y1?yZuI>XCQt@2Aq24Y&Ccz{%#3}Is=}h@i-T=&3RpQN>w<@pvws-IiRk}U?7gC0*$!25)tQq_POn|fDv3z7Q2T3C9kQT^&v^OE6 zud%(ja-KrzBWlVmV>L8P^hY1S-yk#80{MWJU^3n$J%moi-;$@uo_IF;7ve^%@X-oK z2(ZVG|zlODFZQOzhFd-^HlyCzWLYg2ok#DFI z-+=$dzq`JW0@#@3MP}rC1dF#wq$S42r$#q~VuAC)Y2jaycm$6e3*U+S7cGqZ3MRbi z?oyZ7tM@zj=C4}V6FwKI9T^igg!R!X$@4%K^osv%E<*hv3!qPtJOeRVkOf^rc3^k0 zs@P@BiCc*Txr5qAej?UWb=XW{Gqxvvm^?xKVE^JuxFh0evi6F`yiI#Zc~{j>^Dwh_ z#nMU}D+?-fm0naV%r4IOk=8G*TY8hsJ6X3Xz&SfBx+@&1croX7&gzQwvm=@BvX|xD z%K5#bwnDv%ML8R)jL%(MP_3B3!cD>`-I2Fv>Op%`u$28T`0?IzBtLlKNx?+a(rbHkM z%DzcIh`)(mi?)d}xNFQHx+jAR>v01`3GpIXfpS7xcA8JARxFS|kZ+N%kyaAVS;ak*9F)$Nv=FxxPnE2f#JMG`i(SO!370S` zx(7pY$3^?NW2}yu$&L~p6Fy`Y^9^l+x=DSbu2YA3NU=L3U`Ehd`UmUel%g5JGWH19 zSbSgfUP!U~nFXwqea7^mC-RB+XXYNQBRi8;Y8U?+BUyz=;1kIz)GXf5>WcRyM^X*w z7JN28i+2>qNo0~!5=2@@(nU00_>ny$tRgxnMkFp#AvZ#lCGIT}^UG&Y)KYwmhZ(c! ze)MkoAbAv1pk=5ZmyX z64nY+V)N1c$U{DvJcX#RVc2kNJZ>lYQr&5Ywh}0I3o)VuegMsZ2SOA~AqHeDFCeR9 zI<$;muO*6nv+b)R-#;Ty~x^N;d=#uNGs-6);dIMQ5V z{bsN4{OnY@y1V{%wRiV+UvjRsH??uL>UOMvyH|J?rCD%JgXJ;LEO^?R2if@~{*>kNs z%)QNOYc+cd`w^?pY&5Oq^T2u*nN97e>s;y_%zIU4&noW*@1Oj233s{cuQPKItm)xfrbSn4>i4i@1oRg>ODS;&8Rh<3K{vG6I| zkr_uHp(XsYmdW+wo6PsT*xoA{Ej=vVDG`Y4h>u7oDT2z$sxno#v{~uPGnZuz$$X(% zt`;lLO2>%1@lfp%`AFGwaS^vkv|eJBn8mHde~6w5`>_!F2RmHo;KACJ>}jR}^OiBQ z-`Jf@9p)dlkjoQw;Xu~RWC%M72e2s5m{cbl;5!ip8HTT=)7S)klsttW#7lUwCZ*0$ z&nY$Cj0rO-mJ)Nan6#m!vKSE`6MMuBL~q!AtX4RW+rU{kv3RpIM{!;GMKwY_MRiN@ zMwTV%B$^=3kqwa-NvDfuvD4X$!b-x?yxTilv_s_QW^+MqoMg1Dfpns1x$r3afL$k? zD+GlK?uJMzej;iu@^FtuUU67FMw}G26g?L9WZn=Hv2pkd5~ErYNfbx2c?4Dkmq6#? z?|eh-#{MC~RJ!ni2oYoQ?*ab-egJW>D_jhR`9{_WUx3;G z^O76*ETk~rG_gNEA@(g=7PBQjr-lN9`6MqJ+y*=s1O>Gq7ms6~LQX+*K_gfj-Uz>j zMnW5)O;Bt8h(X>cJW z`X@D&RLw>O|^>U?KRG{{$98Yr&`>H}xsrA+k0w z%;)!>_G$f70(pMU-_y6)JIC9|x7HW;W_kB}ws|)PUWQ{4Cek<5HYg6wjLe7)i08-O z#(%^zV*kc&$F1>R@uJw7Sl?J$OcE=IwvD!p2*WEvgTn_RH6oXJocq29_O$hE_w?i$ zK9f`D&^teQ{`NNt?Dp02Ytv1_oWs*m#D^^?I`p~;~mp~8?e zxX%BFXTKA3w0AUc4s>pGWZIuu8Jo!-a13>{x3e~t^FE-z_jJ0UZdh;wR;C$#A?3>_CxSlv> zI5J!ro=+akbHp{oxymudvB^2wy@hwt`|v-LTY=*tad=5k=pXML9s3BuuHp(VkM zfdzqK!PcQsJaaKE)FNCG?IoCi-XQJ~&G;4WB#&f&Ltmm&WHx9_-A$5$Z0H$cLvyfA zn3_099b%@iyI3K6mCm3g?0C^i$wJ9lQ8nRmhGM4C^T?gpM>ql&fp_30*lzqcK96gR zOopV;O`e=&kO#ojq#zMYkb()o3Lq%>BB%(SfOn$JumxCk>;zH+&WGz@b%+MUF?0ab z6x;$m=b!thU;?ND{Xj0^t*Fzy?Kp$UplXo|sNrk@cZn-x-jRA@Bh`v+%elEEw^;N+ z+**1<`k!R7cpuk-Rr3|FwPcsPr{bFIwPd11A`MFxisx`eY*W4xWpWop`^7S`g1f~m zp&v8zM5m<1C zMy@6IhcKHxMxUVSQEAjC>K;=f?9MG>QM!n1LO&NOC7q;R$v(+P$-mO~vWxO>a*@2N z?7eK3;-j*@YPm8;o-KaL4rD8F>HHYQ56K{@T((SBU)EOoRXk7>6E+vRghKH^=^oiJ z86~qwo6FBBd#N+kYn4KIZRt_z3;8i+FV$7$c|}v@S+!4NNf%}8Nq40gR6mrnRa?_8 zriao)n(3Oo8YF#0I+EU1ldd_bS(Ls$V`t{|%pDmIPpR%Ij!1@ZmxNYPuH2}i(#ljf zmH8?`I-NBllQuY_cKWRsK<^+(lRWeCfI&&-8OH?lJFQ%v#94px%?kDU-ea23rA?zMr4?P5&NW6?a zi$6-u2U|h!fP1Mqsg&R^P{-pZsYIvP?da0POYk!)#W?sIXyj5ajVnMt^v`u(= zcx9}oU=!>?a^QY|O2F{0LJ=%N-h%%n|BRncilD9NWvmW59`1(xgO%bddH1Lf_6Q_X z-BK}d2l^M5iDp9Y1WSPW{5AeX*1;yo3Old_)kQd$YcIS&OUY08pTq!)p!X4-;e`SR zphW+`{n!p0D$K35mL)GW?zx%xCYPSUDF-}gJ!^Ka0lMS z5|Uvak3GUa7~^3Z+!4Ko+yNHHP6yWozen!JAIG(khaqdEIB`1lx1bp~2;7!x7ONAU z6S<#!4DJH3#N5b=NY8lVPF_o*Cm?9WWlvO zo^&)YG<-JPCN!8wIbH|9g!+c@NS$!pSLi6SjIqVsTLKRPS3I@s4$Ex&L#NH&+1i&c zl|{BCjuxIufyJTR@U+mY;Lvb!R20*NwtKrd3!F`SEkcVU`=ZTaC*nVnPXrIZTksON zDo`WdIV_CKNInHoP?4Mw)kGFY`^Nu>?}>GYEswcl_v1^FTESQ_4>*$QlxQ2D8?O;x z7#Zp>ac}m{41NyX56<<^<;%auwb(h@dC&FAm1AF7K1^rPNzGjxvz$Y0JF|NNuo_wPS<6%-dd$=_RWvanKNqTotV zhtlcV7dpQ#q0QEsv}%LUSVccwo2PGW$##zSy!3{>$NZ^KakOP}G_(Pmf?q^`KL)u!Q9PN8&y7%otLpG>v6@Y6(?p- z%3hT{BU6^XQqwuZ#5-8+D^|;bGaF@X%9dw;O%Ews^0+{6rBnGp=~Vo#h{^BD*79-Y zO!X^eu5_`mE4xWNS@}?1OJ$KQm9&#Clymaa5=>mqRTs|U!R5ZfUeZ0v=gNJui{eh= zc@nv#t*{+Q;}7wlL>8676fpItMKYEZ+-*?2$n5%Lt>g#F}^)2`S>BnbThy+eK@Co(qrGI0YL2i=4E z!&AV6NlE-p{7Nbv>dn*4tw;swAZAme*+00q!Y=f6tO@cDT9*_v-5HEbLpLM!u$?@P z4)XA!3`>J6g0DarU5rbx)8K$)|M&zRc-|DN7dan19|#0zMi#~@#{JO`;US@sq0!-^ zk!0k*&~#tIJ>6FjN{?=fq=H)CTQB9G9e_j6cvQ+AT^E*w*zohHCUG?3jeQTF2pWU? zL%x90d%}6ZVQ{wc9QCyg+z;#wTn;EgO`^LJdy-_L0guVdO=tyYpkwf0Xsh6D@&u1c z0z3xkM=0a~v>hA{9Y<@B{rFbzDqf98V)KcwL}P3+v=!(G;_z@D&7O-_#~x!QGLN;3 z3dM&-2ZaOKo2-MID#@437VTwzl1<1nWM8Tl{V#P7f6LFREy2DLi|8Bd0@h2tC)yAV ziB|YKxKXN6OdWd@f5MNLos5^pHpPD>B1wHpoifMT{lAjL@YAFJ_m}X0NzDI}nExd) z|4U;2eoTjv_QhJ& z%37aW=9=TC1J*oeu4khgc3!p~HQh2iEE`sx>w)K)ud{!b|C;}z@2k6_J!le`-kAoOnDWuah+&&yrQx8V(D2-- zE5A_Q%eclc*05ARP&Y=W)y>u2D{EfTv}k_ehr%Jn!ID(*_k!tp!+&-9Ri3w~aANV( zl7^)#in|v){FVQc{WbU3mS4?(9r~&JG5g2OAM~%x{BDI6if5KEr8%XSO4b!;7NLa` z3f>m{Sva_Gd11{Gfo_{&qcLEZ%zw3VEu))Zm}}YZJmlH#&G#&GJ-4s0J+b@B7QM>@$2U_JU@@Q84@dp58@dBS=uPwj)|Gfo z?xsxSSo|?>Ki%Oo+Dkm#d6xW(`iE?WuSWFHP*~5)?h8EGl7X9$cVI7pJJp}3uu5SJ zJ&v6rHSAR0LdWSnq=s0AwdQf~Dd=qs#ueyO(3@J76ep|jko4@-Js#9<0>%W(Q>Cfq zU=p5&bs|0zZ}FWd2)_iMLz|JwXd&vyb`j^OM(ictdK)No3avs$sAL)TBQGo`@i=cc z;eED)@Ef;C?3d)q?RM{Na zOIbrjUDbBgP9>pwtG=8DX~LQV86UHt3b|Q*HPuv;_-Sm5VywDz`p~RYh4mFrW%tQ^ zl>R{TTI0)DRUwsAkRz+GCw-6#R`ip1l#}wevTgEq%15dTYH`|KwMfl>ePv&zUjZxg zm44-H`7x1)!Z0`d7CC~QAr8@Pgue+frasvZTMl=CZo_8m8(E2tk#?*(vK!h0)J%O$ z+)cbse&G?}<$N%3m5>lV4CL*8Gf#po$4N>@cEm5kN+3J6E>$4VfZL%QWC(hWXWJG- z&w$y09NLFwl76Z`ZJ@T37jQc|5qSsg2JeGz$bcB}$J9_}5#PzbV4hKBgbCkAyd=-_ z1CB-nJ6JIek@rj87nq&tAuBGW@9`*7I~4V zLA2uc1L#IJa%?KXzuZ35|KcG|i0T}`IjEpkd zXb$@(u*Qx=enb{Ur$smO;}Ys99mWEkJ(#nP{Xg3Q`%kCP)6bLXQrd4?E}OE-|KJDD z=9r|`u#I*saG30mZJR6}<9glYvisVWpv8l7QGj% z71`?V=$`11IMz6ux~1OgerK?6RKbrWaj9xaYvT9RZg2xOgI>tap_SM=U|?#X;3PjX zQwgjLDUbrJ6Zwf`c~<2#Z$^Kkmx=1j&ng!yzDw@0euCsj*)V)NzKO@wOPDBEC>p}$ z^4IxEE+=<0+2S@*gSfh|1$Blv&yVBQpcnDBbVY{X9!MI?OXNG`CuK@mmi)N#e%g-o ze9d-sOT`OGBe78w;w}hhur1gL+;m9~Sp_*FZzjWJz2rNUZ`7+bvo#s2SJKboev(zv z4$=#vWBgEV6&4WYv+daNOn+(|F@zXP_M$pdjfgc!XP{E@RAOlIMlz6;Cbz`9C6=U` z3hXI+YN+4~@D0ZB%H(WnD|s0yNU0N(6CG2Z1T_SAlm8``2nxYmcqsG(7z(TfZ^8?( zUerkT5i^GT3#EAjaT&G^m*XUM1<@ck;bJgL@Gt?z&%{nA768kTllXu5X}DLaELIwA zoTv$G2iJip(vIj&tC+`BL*g6iLyn^5c!XIb`6hLUvY7$+OjM0!Va-sI-{<8wvL#cM zsYSIVNwPVyg_uNN;zTmDJX5lV9*EyWN249lG-L_99=%QMq54o5K_JxxjpM7M(wI6K z0H#1Qfpu|p=(m6;!1!0Vo478yYxtLkC-6IV+>D%vK8%RMSHhcPBT`dA5)9WM%n`P1Tw%3JOn9tIx-)-O!OjS#CEbCwSz3>Ly+2(pWe?rCP?T}X<7$F`lHi%W$X?Q zOR~Vzq%l0)8+AP8E74L@mT5P?w@U}-d)Gqu3(sEf7|-9Xi7wDv8akCAlK#k%KqGHc z_Z!z5=U2Pmal|t@us(b$x*|3%_A(L*HVwkj6#^g96hDi$hCTvQK?J-nxFMjxCU6;g z1qaB7^giJv;Uj)rFeYwBxJuox6rF;9uuo?SY&-?Qa~QhYS1>nisTtPdV?|YTMhn zwt1HZ(!w31q4)!Vk@qd^U_S5itw^B2D5M|$8*v?L{yt4980K5)-?SYZI2PjPHr@e$bzdAfWY|9da>+VoEu`!X`pi`8FL9nwCe zx6f>rVO0&09v9UU_mR|<9^(Oi8ShuI(mL`q#U)v;Bvb4c_m*~%CHU2}J7YoffPugo z_y?*$osbIL4Aca_gJ!T7cw6u;c{Vv3n2z9hKdcUPJdqIvqYdK~c#d*1AOTJT6i-<0 zgCBv%c#`)FZ~T~a2M2$ICq_llcA*2l=brPP9N)mu z*7(ol#>BPQotPoB*!N%Bq3lEbOd>foQBJJN-qJ{huvrw;w~`~zk+(uEyRAx zN`m+Scnfp~X@pNEcJt$;yZD{7s*zLa4V+wBQQA;+ncczO=9JQL^0@2}zt_)OsZ4%E z-d)B@CrZ2VQD(O4q{g06KVz`Eqim_Tp`@2|q@<>(n0?Fq#kLk+U}KDo(NX_UzU!ECpd)Pk-SXLX+^z9V^Ne<6TklwD*Klu5Zx^29 zedPJ+ZRq#;{|p=pczsJf3D*I47vK570RL%ML!SGc;Bt7rdySq89@aO<*VLyBl!tzW z{|sFWoDS&2Jrnnm4P)E<-JIKPIgZipFWz1Lxq+pD8-WgiCq9iY>TT$`>r6PlyN3D_ zfu(-RrM0Z*2dA5x-kRmsnzqq)js1hQxAl%?h$&ZpzGP?7(vt4Fk@`NRn!E{LDt&za zap&h>Un+f<{_2ufHSb`aJ)bSST9BEambWGEYr*xBkoJytKylY!i@&z{eEM_Qmx*6r ze7o|~n3q*hTxcy?TNwC=e!urs@iU|7LaDj%8Ub?CGv1j?R|Z&d9*t@R1U~ky_f{MMZ*gI>(UAv2lWl`7OMRZY$ z^p-$Edhg}Qeed`GmASjQySd9S!;GWJz}w`>_xXI@@7G)BGCu-gtkd?x=AA~!e9Zcf z?HBtidr$j5+k4w2#}IcZu#aqZ2b@dXYy6F950ruV2dIFcozK>Yas{2jpnjBI@URM8)rQSV$G)apzmVk@|of)Q4H?`?KV+` zA5WY@&WESbuyg{jW>G*C@D*bMoljj4BeX>B-x5Q3N*F=3lRKJiXTRh{3a5!O#4X|% zl7*pP!#;!=LJCBS1-C`o@T4vWyBH(ChO`Mr^9p!1f`yWbFm}Y%@QdL$BBn$(g)a{2 zFMJQM53~7KB-DsO5lch93Nm>!cv<}Ue271We}wPiUlpDf$BGy7$FL^QXESzlY6Svu zq`i272e`hE1*YjE#`$%Ef{g`3c@A&@20+Ike1_Jz( z0Qx~mA4UC1?m^xFPX*+_McfGXYpRGOBTOUe2r~S7{5Zl@QZzggBxbTbR|(sCum~)^647^ssKdimbFMM`<{Q_ok!fU1qLjyCnr+ZQ`sYR=O?U zHqkD0E_E;U^mUgwsP<3b2|`02Q5 z*sstWY%Ojgeik+heK;^WumDwpQDMXJZ}2Da4Y++^LUNe=i4;Nf5K77C=@@o7Yc}Hp zwHLe#UQY8dPOx8c$MJUyN`#BW6GF~}`a`QjYeQ+_{UQa?MbSZMMG#sMgjNKh6+vjl z|5wn8|JarXzXcWezfpnrOb)X@;|D#0UQ3-0?*JC}b<}(+3?Ct%C#@rD@B;uaVkEW% zN5=C&VtgyU2jMJ!6b^^I3`w!$u}e)&c)UiJYQN0C~XDEdp>B1sG(g}jw?NJfP0mRuAa7nXp{8bO#SM2W=WSjiH} z6Y)6l1#yG;zBpGrOdKLU2ev&Uc(d76j0dy?sua!xAg#TWx8yeBC-D2cfv#fm(KAr< z0(vAVP#-{{W(R!8g1~P9IWi7Rv8n<}08R-H6!;&5owd$A)HTwXI~zc*cE3H* z-p7hD=Ni`={?cF8P0^mxEYp;zwVewUkL9=8j8w-bX~ygJ8&;S%+8#I@&N|mu_YcoOZ*O0~??N31 z_^R``$9N6l1t1H&C%z!PpoCCy)NSxa_!)d0IKlb?KoyQSlX#7Io3w*+g1Q7?WD*!7 z7^|2sSUGGTyM<%t*7M4FZCno5&S7z>+-}@|xCOig{AK*}yu;k%Ak46WSxE0m%L23e zWz==lUGP0}EU7>72mUCw43mjzNAE{5kaTaeOXBQmueZdTubEz%UYHV1(~N_RC=<q6$r#stq);Z9*4x^3opjZ>!1Kl<5=ODYh(qXl*RcHRZdGKt%2wweHkklOlEJ^OSXu+j$6w)$Pog|cs4Ja|61@yC=h)YZWiK2 zBg97W4e=6iMHnnbL@UG>B~8FBY72QP$rOu4XTdduDH4b(0f;tQ(3{tbBWESEN|+(c z35*2*!ZeLagqM@=6VnL-Jd8u2C`f}D3cZ8aSSEHTR0iG^rogno8)S08gnEtUq6eTl z0;>X(kWOE@H_toIo8z&%a4wDWsq=~>*LK3}F$~i8(%lDWvcGg3{cfFH^SA20a)RPs z$B2&P_GDQ~>$m3pO}$s~dvNR6^I2Xdm))25O%fMf+Jzo9S-hRKiBe8@2QU*N(q?ih z`67{nufa42*7&=5m7YvwGjsp|DCR*zpW4Ote)e+%CV-SL1Sam;=yZRo&j$wKSstcy zq2rjl(VvCr-54uZce?YH60b_s?9&f1iA~)AMS2;4$Lht!xn9ZA_AC@ za9ZmD;cYm0TbX`(UB_`q0YB3QRN4}#HFo_B_~ z5P<8uxB}Mw7OXo1n@f%&S)dL6#jaV7D0`Vr=^Bgz6fVLb{A1`5W)PHv-hmuL=AZ}S zc*Jq|N9aYUN0{|k7#E2@O~@lz$RmlvFsT5JKf+(?j&_W)O>!qg-H1zYS%E$7^^SFp ze?idz%aszifEkZY4P5f`5Cdv3rWfWGb_ekX`2|^lzlnMPfJI^?95oG7fxSii08`M{OR_Y zhR5nk@IUBQ2n>&bxoVSfhOP~;`3@^ePLoA=1V4=zN+(vynm~rc}2@L*$riP z?R$NHeX545#K;ylGHXs%{Zn_Y{jMfiQ>+-)=4d=oZ>V+DK5JT_*rZ!-EHvTFQ%xt0 zV+><-@3oKh`_1?551c~hak~^)x;EJ>9b&iMcL?a|=B<;PD#VNnRs*2i(Mo z70~&^*iwdt_L}N}n`lGWIDuDmRXkMOP4ewOj65Zg52MA>y|Ig9A9wLbxg$PD#77gQ zvtp~_u}S-rOOjQI{o+NjCg~JukFKv0@AkB&Kkl2+qhC_j*dfvcfJ+d@<|GYEnUu08 zIWJxy?H6@7{7D!&To)b^mCz-o3lzCeGMi85?&RSC6L*+M2`)@i1h?5ywDZIW>_zlK zOcE)Ib4NT!GF7mWIf;A%cN=pT&BA7rhSHhrgY0F@y|g*-MRFyv0IY}k^d4+H=L+LC z`6zS>5g~n0=}^vTj45pVtl8t5Ilo6mvS6?$=~Rzw7JZ)ts=J} zK*vGEPU0FuGByMy_tv_SUFn_)NH@;%A=ib z?RSd_Fgly;WYap`9(}Pf&g{2poM(Mi=vNR1_02ubCN*C%FSO71iXbuhAbgLs6ni6Z z-8T;~CD-{|(JaDO@*CI-6DeN8dVu|qVXqM>G!Ancr4Jw&Uft)*u(Pw`HM zaKk-fFZUp$n6eqz^Sd%$@kfVcMtLKzgr$kjah5Q4QI(Wzcm(|}`xYRpKji6IJ892I zMEoJ}ykqg}2@N%7nv^;8I=pNy9?kY|`cN0Hfyf}g# zlNCKO>^tAh+Q?EdHS~py@0>NE(Qmb=d^9SQ{s4W6EBDHm0U#V0n(cVmyZCk5>xMd=J?aWcL{vsG5zpMaoLz6q|Cj-e#x@H z{K@*pJuvVIosGVVeDlVG#c!M?*5J{7M3~6BC(a*NzSv*ctDpX zJMTNQTr!s!#C_O^9;E`S@~hqTi;Mi_M~{ulHG)GpLpztfZMO%IH~SqTm(#m{#2vzOQ~j<28t;h-1C>)Ac zH^~RK%xr!nKd7?JJqY(Kgjkf2Iza*#Evo6xdx9qN3}scMSH}s z*ivkLVgA#YWE2?F3_Lwa|K5;p+2+{lxdT$?$GwXjb>KCB-+2z11PMV4#Z}bzKrw1c z;FWu@?U-ehZMDnp>lTPa&U-WttNkVjm)JeeJ?*Z(j>%R9IK;cmLfZs@+>_e1_5t8k zFwTP45;|tLb~FoQ<2p+;8S1XeYwZ){Qu+5bTr;metX5U~ps^n?f81}K-XN&SskvI; ztBKcA(ekZnK;7KRf4|wjj48YDb>p{;f2rRORo<#OR`>d6c6sKf*0;>}`%1CjH&m5Y z>niig-;|-sj(zF&dFuz=`&*w6elMzdS(jR)`myG#qIA`l{a-uEvnwZ853WwAobml< z`J{@E)#K~PP5au?I{O>>w#^ovZmB{iqqH}v_L{BkI1~%|j2Va#Lzx%?^2%N7%Jsd5 zey4aC<5?AS2OuoR)A|5&>lL0@)KC1YXrVAt6eW2rIWP8$swKZiJeM9zIF%@kD~|pd zel2`UbX8n-w~W-XRB38)N=-_7iZi((;YZx5ggq&hy~p&c?^oJ)Vy{za6)C#zO)1K> z_dO1!Il3dsL%QX3PwhT4S&Pfd86cq!>b!f0tp_z`iZaIR!v#Fm&1(k!X4%g-38 zbfWZXbXG)XSXub$sJ1ScxQX!%u_4l>F+F1jbdgE(x@Jr3qI-lli8cwk3a1Jy`G4~! z^S6Sr0zJNh9TuCXzGA zmq`q=ftthpRYDJaFFeBfLU~Pi01U^+DUTR4KoYkbYbLFVJeX+5E=KM4e?s0sQ^~{V zIL5!!b(BI9nmC0pkvNO|hT5L3+V+g_Dn#N0(Jz#{6h$%l{2R?8)!J{WB3RBE9EPZX=64sKQc$K z!q_l-J%Au=;?!_|;SJ>Jco+Gvc+0u(ISuTCtWHLM+98UDG>aHQI77gblE^bD{otpR zNn{quMHot04QOubfxqr3$xb3uw!qz}@l*sZfax#|z6R5%Iyes=4&MbnUzoOozJnRY zW^$4_OWDT(7D2=)qQ+4&NiT@!Nhirw?iiZi?KgI0mWzF4(bKa2iJW7vFOPL1dWY$Ag4U5L!z`nt0X8 z9R|$DC#e0vQ-wt>LXAe*1IJKxpuTJfQjQD={DD06zww3o2KiR{u6iY&n=YYio$Dva zuda6g=FW4CaP+mlGbZczYfq~0gMPPP6#vWVswgT-2!6VJusweOl7VjkQsNrM2ld8! zFf3>YpsJd%^*9f}wwnR{unXJ|K244$F2~hisAySWbD%6RATS8YL$)GmAgO#1KoJDK zHt%3>oQLW9U|(pvYFTcYYs@!n)7R=}8#aJ#`9}lJw8hkH+HU@0SzwE{PqN>!@3Nn= zA=WV4eA{4zUJ_6QOO#KmCnNYc+;&Vkug|%{YwWrcrvAfD$ zO|8kSolu|O^0Ylw5z+ZpO$QtIqpE)t!{k&szkOxLEx-)?UFqmlf&9x%P`%09|zDvCj% z|D^7ve_;+|tzs;oB?9ItiBUjbKu@NQA=?R=glf_V_y+j5X7~hkC3OdUhq4!L0Islw zBrT;o#YnnFNum`q*0ZCyGkH~l0`Vrvb?^Z)OteSvFaNVJRI)FmI&@LkzR+nQ4?|?( zo|v4ty@~0`zb4y~+q?gjs!NOSd9TOl)F~;gsqH=f>UFA5|MVaIx@62B_}9RK%)1%M z1Jvn9`j+%r*Lz=YL0?zj-_wQt&Hcsw4SiSiTAI4CTaV-c$$80!q>YIO;^)Vmi<=XV zPCS@AvfKHj+5}hJO6mQmrx7LLUBWC9vba!~BpAgX%h&NR+zi%D`Vi`G6gOy;^b+cD zeIYYa30_YpJ%c>meMWyOc%_X-BK)&Kt-*TVCLh^1(EGP%k#~__g>(;)0w^TOH`cq; zi}B3`3|+s^;79o@J=Jca=Peb;)gO)E=o$0`(^c1=J1P{RO2qG7w?nIX;aR(Dm~51_a+4T;9y zrkh;MW$FYY+hmhVAa}PcA;Y!u>1G3-8B~&`|3}D3Za`CzLuxU z)z#__>Nn_T>3`F|){arbdH%YO*S_^NOOeeWHAl?9bNC&GVXgO)H!BHeYHpWjQyv`gz|YbWA$di!I09z^4$n!1FGp?x4?L^EOk^#pU9%u?sNC=t{X{XfY(%@_|zpeX7;@F@5?bsE6!XEUBL>gl<(dDK!+YUZN!hW+p~Y9%ZaE15f$M*!#9P^2Z;XsumfQwVgH7) z!l8&y5%0pUg|b5qioXg63*QNGqEew(kRyPFr-ToM=S6tQgOFEYd%}-}jSXEcX%+Sd zyvq4}k$@|lBJ2=+<@W`oVe*eJkxPl}23!M^Ro9yMp6jHOhp@!6>c_{T}#v zSK?3O&SDP(-KPc0kX&$mApmkP0h5Tii@A*1g_#M_uS>rf*BZQO!%L8^TxycgX4oyY82 z>u2*EQ>?K@m#ZmK%}|c-kjtSqO4Ew^m$ff`MpsX&im1HuJ?-21(w-kn-`#y{cr*P? z@tdb_W8aVb*!44}bXn=f(uU92&-*^z{1o#!<1_p7iO~|M0!ZsLHH-Q`u4V zsCsivW!>_I8W3#dwfx;`mkB#oD@UrDRa?{(G_7jAs#rBjr3X=#r|R+AGkU4fZ@dep z3cGBt?NyGE&R<-!!97oj`-OXnYn}tLG0e{lWA*8}uDV9uHC?urr@^W>tB$McRCClP zHFzCP|3-h#pf(&c95?(llz;}ma1gbNvE*9k*`$sb7Y?*>t@fmOPXYu^e^ejz2Xr%7 z3?D%<1D?QQ)F%`jy&25`0f=6pr)4_fJ~0a1VEj(m4d0_?(eBdb&~GuAEHN+;U*$aE zv~WteA-wHe8i2vy1~7Oj=NVY!J>q=je&&th-{I2*qXqK?Qw0YFdchvhoj8sEg2&}8 zL^L4> z*bFBA*ZP-T-yKICO2?nh7S}Eh!29~8NVA^<$b%h$Q>c;XVVH1eEi@eBL2=MrC}NOa9MY@~KV$tRd6Dm4DS?DrB@Bv-Wbl^wy(_fgk!B`ZA*PBs)sY zOF@6ZHvJrJj0&x|-d^8c*c-a}sDIV?Yipr=aR&rEv^P4Rs^_XD zore?{MemN$?Zxfyl*@FN&7oF}nQQKD+OF@V=~UfN%QR_PyE;@g4HR9rs^sc_HCMD& z-3tS3x@1~mc38IAMD}wwueH{e>>TZOc~^kE- z8Y?<)n@}^DRSn-Expm zlXqsQ?W#(pvV)`$b$$jpjoZ3%aE{DZ@)c_phdUQ&<4ko{H>i*6ZNKG|dairQT~F+< zK{4N4Q=z%PjqcpxKH(*JOTj3UrBiAm!Fjk`)x%I?nPmUR@yIpWcL+)Fy?34iy&3|$ z+*W9Ruvh*a#b!S`WGNeHA{2Z>f8rJ=79n9%lJwo9Xyq zcUd#cR{a>AOt(^hAJpLPQ1w!f+NaBfV3-PNe>FfRzv%?HKPa=Dur9FRjIegA>ZxkH z_L1SV1>)(fbUIsL3%)) z&)mRvvhFZAG!=OOfrP(G41*nD7r&i)l9Er#CE`e>g^vtjhO7_Y6SE_B zd3;jhxNg6vEb9&@Esf8PD~tOcUy#rzDIq1YXHn16wAHC=Q~Go(Nc=nDY?8M7m((jM z=aOe6@Z)aA-iZ?@XcFHie^351u}8c+HY_eAVR!Prl;6`n^yKt8*!O+kl)m=fb9<6g z7A4B!qT>@2>r1hUg>g8 z+D&?~%ag9@35|)0gdK6ur1zp5!l@zuic3OL!*7L)L!*R$vd7awF&60*(Fj=XyYOou zChD3W<4^I&`<$2zH;Agzrvs4zwFuMcn)&c1XBfQQkdu{bD>>3ZDY;Z#w>lU zmZbmJw9Wp){ml0YQK1%N63`q3@@c%jU=A;s!wcr{{$I`E)&39V@C^Tj9G-EO`A#s0 z7tG-Wb9liVUNDFE|8x#-(#9SOH*Q|Lc;v#N!SZNQusk|g9vv)?4wgp;%cGwK%cFzk z(ZTZQV0m<~JUUn&9W0LymPh~hmPbp1<5L6(jKv0380zn0W3Ir7hDiBm4s6bGGpaMY!f(ir`2r3X%AgDl4 jfuI6G1%e6$6$mO2R3NB8P=TNVK?Q;e1Qqz-s=$8&YMv>6 literal 0 HcmV?d00001 diff --git a/tensorflow/core/kernels/fuzzing/corpus/decode_wav/8dffe4c5c26d891b578fd2ea4b9adfc0c96ad5f7 b/tensorflow/core/kernels/fuzzing/corpus/decode_wav/8dffe4c5c26d891b578fd2ea4b9adfc0c96ad5f7 new file mode 100644 index 0000000000000000000000000000000000000000..54a777d22c159918c5e85078560fe363c2a78b10 GIT binary patch literal 71409 zcmeI(hjSBmw>R)@TCG;S%a(h^1>5vu!1T~ULhoP#0TKv3m;^%THFOAs03nnBp@%>c zLhsea1@|V|lGS@(t@ibuxp(Hy`xo4K<|oaltVSB?qt)&?XZ0OBeCW{i^I&MwfQduq zeYG+{1VIqoeja=og0@eFAq4ubCI538`QJAp|Fht~ug5YB^ZzXZcYy#900KY&2mk>f z00e*l5C8%|00;m9AOHk_01yBIKmZ5;0U!VbfB+Bx0zd!=00AHX1b_e#00KY&2mk>f z00e*l5C8%|00;m9AOHk_01yBIKmZ5;0U!VbfB+Bx0zd!=00AHX1b_e#00KY&2mk>f z00e*l5C8%|00;m9AOHk_01yBIKmZ5;0U!VbfB+Bx0zd!={2v!UYERWAHx1N`(adO; zH=S>M-}G1WoRr z8}=;>{1)02UK_3tZKgIxNN5o>6gkNDiWkQCV=EM23tkZU>@cUVph3J{c2s^$u9c@L zhAZwVgvz(dS<#vCRO*@Ri#g*nE0YAVte79-gh?aP8Zy`B?8}>*`(4(Mj3pfusbvY- z@nlkB*1~Q}dl>S<=`Z4Q)fd$3V$Z}KPtYYUND#&q#k`4Dsd~s}i*kg=C7!5{5`Ri5 zOuiHMy^4^7KDAXIIwgwB3n&VsFHt zv4>*|5_TnJB$mfD#?{0($Bj@wS7@c1Wb0ILY-?Qq*x4$r_z~|bGKT9BOpz{=Z<0Te z9u^;w%#$qS~0|J|~_YRjB?$ zsTGbShG5%~40I_`CLAd<%7cnDFSbj<_}FUYbZJEVt*l9PR=rj2ipp2LRXmb6%W9?h5>#|fkR?*d z7{zFXP288e78}8Ip`HbDLp5|yW-DD6`Z(b5ZSwW?2pwXZ$KL3&xZgV6=Hix=hQ4*n z8%tVynM2kD%M(MY-lvUfx>>KR|E&H>Be#_>ha8N1h&$E(tFfm})>78;v*DF}i>Jin zaY0VL-DNp%$}`?JWjJ>EN6=3r0xHA(-el2tFe>fqy{_OKibD}0yFbrc@u(!c;Z1${}-yaeGu;XeKlasnaYBYyA`Q$EI*-za`bWM&x3go2LNqFD3`nULw48A;OQ4f*0y+cEr@Ms1^l^Gx zNbH~E+Z(K-r$wwm#4WcRYP+w;4INEetb-kDN1m;Z0c=eA{BS*4e%_ z7w8gO(sWt+2RfbZkl~r*FKQVi4Ex*(ISchaTG|YLOTc}ES{Eq@e;GLJx#-MtsO)3x ze0OYc4LyYZCGeLk)qcvxb+}yd{!(f;bPDZ`oTmqc76qZ%gn9RD}wWA0rz=EdMP}t@FKole5D6HkeLL4}9zX$T8O5)qcZX?qIvu z`gev;(`%?H)XDG`DlNdcx4N)k0aipNb6n^O`t!haU$XbLM;{mj9bolh55#YyH<5wp zD){@z-pEXN082rP#&5!Np_|BZjK@mC53$#9T;y@3a zn#ODjF83|-z4Pw(NCKUipOH`CF3bs95wSsokRHed?0e!pd6(4T=b;V{W>No{gxSn&JH5AjM)3J3NiNr5&mK^3M5E*E1 z=so=#wKLoqszec1cbFC0;wf}_o&UJDdS{1NPq zCmk&3M8^U9DBCE@6Z38BR_Ar^4S$q>pEuo;=2`76^QnW@@Gj^PG7S9_xd0DhLX<6- zL)qc)dG)dkl~6fUnl5q)9`HlFdcj~ROGQPWQ4dk)#k`ICD`9@(t)xFv)ajcuKF_$F zMkfaoUnFLw5b4u1%CoSXt+}^y4ITSrMQ1qDC!}}laKGb~?(Y2Cy$iba$?1?WI6XTJ zN{h-^l{2*SxXzn$*JP>E`y^jU_$!`Fuq4DLXQ!p6p-Ed}W+;X7^$J{dST!_SuO6hH zpt>kc791lIvE%R@B!`nMyR4oWGf}ljJdWJYx`c{lp=eSFta%MQb2!yV&eQ)hF$`KqOp!|3o^j^YRnZy89zbQ3ILzaby;C?3J3*vnZeOoN>zxB{7UgJPLtgXE0B z%-2!S8=$~ZsL>u^e?tI=b z;T7ox<$Kj13aezj;0}ktceTIn$M|w~EsH>2Gp)=o%-M)CqKZU86S2|wJd_vtEMRgM zx|X_R?l- zD&bs8RO+F`4zb@y?^2yp802EB$}956!ZE}iq>Y)xXqjv9@8~^bS>)SLYoNw|$M>sG8Bm2k zLHgheSo4ufk=Mb6-V?6RoX_14g9Di7@O=18R#k6c8E|B zMOahhTBt6#AfWab9B-WE-fSv|nauPG{}|x-R|ha^NBAfDaU>@)m^v9)6L=OXj4X!Z z(Q7O#rz>w4na%El#xdq_V>l=BHxfgh5&kTS=dZ;-g@1-NK#ypDD4)t_hOyjiAKICg z2Ksn!c@Wy|!f;>$L+Kr#9Tw9<%iFT=o6o(|PZBw!7mT>y3+Bo@zGh z`dPbs)V@*fS+LBf8GC6WE>iW-9*knCCoKw0&_n&$g|fq!m~Kw4d*dQkz*l~&+R?$e;o9N{tdMR zdi%z^b&eU1fHTT7##`u3_nN)^+Sh{7F5Jv+ozc|Vpwf(PyJadccF><~8KX_tmNc^( z6gBZxhN>HNeVV!2>rInuRh6qM&(!p&S2UJ3zOC<9J*({9ThFUeuYZ2C@@>I8e#x$~ z3zd7TAAFc!-2ZvgBk_|hFR_vp3omw-T&lcR-KEm- ze#sl%%dcK-ebZVzzifQPpo;9WDJ36@$CW;-_@tVw-_qQ*t*>3_`NpNNEYMjr+?IO7 zW~V!t2}#fw$N)r*_D9%}`@yO}QTPFRlp7L#A}JO6cn)5cpqEf6IjvAf_fdZxJtrzD zIyvS+%rP|`T@iCM@!#~l*#|qOXI@MBkZ?BP+m!Om`FRChUUy0Fa;}P!fth2)U22RiQlHKNS~G-mG&VuJ$-EYnUraXZSk)Y7AH5SA(>;dYBOTf z7o>Ji?U$xWU(;cJdUZ;-g!<^!st!>TqsrQQ!|}?sQF~&d;;nIcYO8#-BujEc9*VAy z-;nfe(!zwdu|?|k?nFLLx?XZsIz-VaYJs{rdZ}WgXe6%}`Iz&JOy#ZRNw_!Jjp!A4 zEL;v>LbB0ws0uG5KN3LvEOs69OX$a-BD@ev!58vQh_b{hg)SaRYVi4JDugmDSjk$- zdBFXStiq3?rx7+*gqw-?+{J?4f-&5#oK>8$WC8gLM?~8AtK?tD2;-ha?UuaaK49O# zJ?wqlo1!VQ)3Q9tR6#j8i11>cLmTPmk%#C+a;T6cddFYJJ;{NI3GA`Nbn+oLM1CN; z5q-!3y!BicF`PY%n9FMx*~OCtBMBSAhU^h4VuUZU^;{G8I_ECC9H&_V)JfZeYlA() zwa{MHe4;<+3(izdEwPDcAgT$HJr^CxXu?P6DU5*0rLTl1G8eEM&bOSqxD6cxa~N*~ zMgC#O3yQ^)#I*tz|2FSE?`!TGqBEOf?;$JsXGK@VKZ%wJR`O!F5po+Z$gktAMdPw!K`UP6NE!*%~AFmtP`f+QbE?tl7Z?*PnacXvG4r<@Gs9Ghh6Lls7 zY1`xCdgWfbtD~cvEyd<=rUf&Z5@=9+--brkz_TzTdLE|R_u<3g4y?;WEpdY-LU_<4 z<{5MpsX))5BM<`=3r|E#Sqk=W_I`Fp&I;~&-eTTy?o?tG(vNmiVdfh9SAH44fir_O z2zdyNVU~n{4Semjy03di_zwpE4NmkU?i;pr>mvIpS6}x~=l8bzX13{>kz?YTPg>78 zgzh2U41aFm8~+E_r=~61%S}VIxZ$|vA4`H!+uBc?ulb?rPF-wus4~9ld6lzU`7hw}a&S`Ph<_Cr1j`@I~`d0#R$hfyL!kWEl7Q_T#9?Qklb5|M`% z1>z|+ya?CgBhlH=a%dD{Lfz;!^gGOjet-hZCs-4w7f--FMXVvpxk@1_Wb-z&e@6~6 z`A|G2Cm13d9{}H{&(qGx1m=3=*YMxLU|?OiC#oR|ICZQBRE=H6jI2g>C*nS<2>y}| zgdI>3Ya{y#`vluae9CJRYDKsBz1WSAi=N9Y!m7y;LR@&8{Jj0qi@bA!VS>TDII=%^ zm&X?kl&WMuiqu>)TT7(zcZrutrU|!^h4>*Xfi)9Xu(#lKm=2plbl}-|Q^i{zLGBClv5S<-!C(60ey9WxHU~nxY+VeOE)u_Q^z>C&zokH_*@at##Mh zUz&%Q(yf$J?|tU~#Q(K_1cgFh!dsb@)F9tX7w&rJWrvI4>*z(~EHnxlfQ(=bVK2o8 zVQaB*M7)42OHtibtyj3DDru=KO|>9;Ote#_R`yc7Q*MkVV(=Kg`l4EqP>?z;{X|-9 zN>ze6ZcY5Eq}UEmb6$2rb5Ca$X1vZEoE^x@$~cmCEPZ&^uv~AKJw3ncW$ZSn)3ltH zj?;6O3bmW~_5K(~gP`<5PB~?CG$(V@z&Q z7C)^ysW|aPLR4Ip>b)dExK5xK-jdvsZ&8#={}H_9$caL-f|tNs&T-&RS#R-fL@|3l z&S5d=LF_H=C2X7)BDa02xQ#j+nn?G9C1@_(mCg@63R*%b)QN!9o9uq%+U?%%n?xOs z%%PLodo{I3={@KD)7?JXvAi^FZYwl=X54MOV`yocXdsPG4Vw%qLl0eYb3uL2dTmpG zoy<7ESl`yqu+j9DHQ7<<+~GKDE3h_MhdPsdM}wN+?ZB46hTuBNNtZ?@hL`vkxizjO zo*M6D|MkG;pf0pKJe4k^&(V)UE4(7-2*(Y#C-^aw2fb;ZnH&zqQkN)u@Um~PC)0h@ zy~*bdDI&+{P;i5Hor~pK=u$d^)+@%U)=~|v(YH+0DRkRfas36uFw-3KR*TdUGR$nv z*JQNJHW5y)>w!JSHqc62axF^@eOeYa&8`>Lo~wP(z-pe=w4&Z!y}t6B%G~NxHOp%B zmBY);r31?^S7<6et{7PH{KM|5->UD|IvdQ5lKS-OVU^FS_SH|+Jk%65PN>^iJE~sO z5L2ID^HX(tO-5~ILuc*awvnb{qg6j$_hp;cRBL%`DKc+0RhfUX{q9I}!R}eE45!V3 zx-vbRe3-wN=Px^Mh3&_^VJeL-4?PPCDPe?ycEV-Q3uquDXE2D1Eg}BosMy`nad0uh z$43!eiDuS5WEXShw=OAW+zelBni@T94C2*DkEkiD0 zEAetX8_&ZQ!2)PJ3}cV6KjA;ZgMxWMCAB%cCL9Rm2Y&TF@nAmXXVN&yUFm3rKy^}i zQl=GM6L5uuc%1~1tPnw>a`94gxlv#mqi2a;@_KWJaJjrq?FG_!_zy}REDJ0T?hg^v z?qH68v2Sp|8iK=psXM{6z#9LdfG;$juAtMYe*X8~=)g_t8a;)6PSu2drUK#HbS>S5 zuA$aZhVXjkIC2>~gw9|(h3@!jd}*O2w1;NG`u3sP*@zgL0xv|qK_8;kXftaXXJ~sV zlgctNC&FKa7euZ?ui=yMUHCXk;9T}=yfZ!&k0VBKdBUU8j>_eVCeap7Jhl}Xik0Af zi6a~pZ?7OpG)}xmtQU^pe;~V&D|iz{dg)wwqVxejnxkP)A}nkTz8oLTKEp}j4d-{{ z&*w?GW62usCK0XZ5o3?ls6SRNl&lsm6&8vcB>~AG=^FX;_OdfMwn64~#+jbf^S|ZkgR<6ZiW-Rk<%k2q{8;(v+ zv$NS%?=J9e_RaB1op&vx&1}n!_8(5fbjZ}-yw>bC?J!ZsDaKt!m+_2AV7uxT`fqu^ zcFwkrwR~zn>wfOd^yT}H2et z$CPcBS$)=7mRapX^nnIW+Z0_xtH!X%vDJIj^S8aw^pn1|mD74gJ47==GeN6rQM9bm zZq(#7Uuo!Gi&xiI$JCcMeW{t-)W81Yy7Ia}-J1G^jr}y?mf5JAj<*kE^~3)c-^J;LtkaBvs>o2Ueu2@JTP7}ueDmNJj-$esdwu~m}Ismjy=u| zPPuEHtFvp0E7N_;E%fa0jP*wO7X%lCdIxX$_}(u5vTzRk8T=#DH}Z_mZXcNxFxMiJ zB4Z+5BYcLB{KAT5ufZp?CSh;TnOGHTEIu8Z2LHf}WIls#AqFe~e~2HzYgt=x5Ag%} zoRdg|*e}UrLPT0DnIU@0@4?%^`%(}V?U&w@k5+C}-HQ5LJtp>W9358?R~08n=#!*M zxs+l_+7>rD>R0(f86oeZ7@}-bc2eC@-HzHHeIwc&6{nh{_(-`#{W@-N+;`ECyiPcP zpU68xs>xX1z4mqfW3gCzN6M8H3Xkz#aJmse0^!c&4dRYw4?z}3>cSZPgtkU*Lp4Ya ztDN1R{ETZSS)7yj-?*Qw7EToJYcI%;5v916)dBk*`59S=F2I(xkApT7`^kLnY0fzI zZam25a61TU1=o1DIeL5rHVn-{w;~zvmiCQMceH`ElGx4}K$ux~SP7hw{C>h`f^1$T zAz^D+bJ4Z%ZrB9>1AE~SsGc>8bB8;Fdx4`TKIa(7-uzR7oBZ9}qnt4WhnPVKi0yb0 z7C;iw%NQS@hCgDNF%xSPDG=Tl?GaJJG3`Z^iLB-H;;1-6VlC@hdjWigX^P}U`q56x z7-|f~hab_^w1&ciU45T;mU=YaNxl)@d+u{Cfjh_h+!yZ)c@}%$`CF&}Qw$$~Ynd%! zQE;eV;U64W5PCfS)H6{E>m>Zr%Eew4W8bW2zR%RAbgp7m~kxewy-g@=MmSYfh z2s_2HvF5SVm9S%3`VJWMI*h)SpmbEu;v+zOfML;48W{L65>`}y0;xu~-YZS5>Dum;(3Vbbbg&4$^ zu~wr-WH?%Z{lI=ij^Mu&|B>l4Vkvbe1?E zoFT}J+uPaumv@J6Tl)a@-{6NpT3~kYU&>8~BYK)e zo%av*{_Uyob_&d-mPRbhOy*POCOid`vc_Oy)Q%j)HnEmsT=ZY~E#sqS)5Y|t$f3yT z$gIfNNL3`CDT}P9t0^&ck@}hb0{Q|8!z$)c=x1M*TkQVa^Ox`c_`eFk+XDhX00;m9 zAOHk_01yBIKmZ5;0U!VbfB+Bx0zd!=00AHX1b_e#00KY&2mk>f00e*l5C8%|00;nq z|0Mtt>>$Ao66_$s4ifAj!44AaAi)k2>>$Ao66_$s4ifAj!44AaAi)k2>>$Ao66_$s z4ifAj!44AaAi)k2?EmZF3AhOafB+Bx0zd!=00AHX1b_e#00KY&2mk>f00e*l5ctm* zNchkHh~N_f0U!VbfB+Bx0zd!=00AHX1b_e#00KY&2mk>f00e*l5C8%|00;m9AOHk_ z01yBIKmZ5;0U!VbfB+EqF98u?00KY&2mk>f00e*l5C8%|00;m9AOHk_01yBIKmZ5; T0U!VbfB+Bx0zlwDUf}-$a+7}w literal 0 HcmV?d00001 diff --git a/tensorflow/core/kernels/fuzzing/corpus/decode_wav/91d787a9298ddc015efa783a92c4bdba8af0d7de b/tensorflow/core/kernels/fuzzing/corpus/decode_wav/91d787a9298ddc015efa783a92c4bdba8af0d7de new file mode 100644 index 0000000000000000000000000000000000000000..826747d852b00397d859fcd65e7404d42cd329f5 GIT binary patch literal 44 rcmWIYbaQJ+V`K<-40BD(Em06)U|?VbLLdM$nSfNvXH%oWdz)c9C literal 0 HcmV?d00001 diff --git a/tensorflow/core/kernels/fuzzing/corpus/decode_wav/92c065286f956f086e977556358f6b54b12bcacc b/tensorflow/core/kernels/fuzzing/corpus/decode_wav/92c065286f956f086e977556358f6b54b12bcacc new file mode 100644 index 0000000000000000000000000000000000000000..77b8f518b4c46b2cef89bd7a12f23ea49c8505ab GIT binary patch literal 315 zcmWIYbaQJ+V`K<-40BD(Em06)U|?VbB9}D`3>^`S3``6H3@M2vi7`MmkG`(|Z1BzM zZ^8ef-+Vu}{?TA|Vt>tH${Ed`!&b=7$mT7j$;84O$T5L?4p$J@d7f^*M!q1d<})(j zGT#)#{3lTJrPwr?y_pzUP|bJZZRLF|kRX`EC&pLHgUU;13ulq#DB?QH`HTHF>n+wb z?2p+Qfxd&={jTVB(CZ6tuD{lJCH=DZMda(1r6bdk-8y`FOs&knu--nrZid@lvw IK6w=e0IQK?#{d8T literal 0 HcmV?d00001 diff --git a/tensorflow/core/kernels/fuzzing/corpus/decode_wav/a35c9bb71792b60a13dea23a41b41847ad4b93d6 b/tensorflow/core/kernels/fuzzing/corpus/decode_wav/a35c9bb71792b60a13dea23a41b41847ad4b93d6 new file mode 100644 index 0000000000000000000000000000000000000000..45d6b6fa606fe4a25d00476bd4ff785bb80fbba4 GIT binary patch literal 44 wcmWIYbaQJ+V`K<-40BD(Em06)U|?VbLYFlR3?1H#3``6H3@M30KvhAW0M{r97ytkO literal 0 HcmV?d00001 diff --git a/tensorflow/core/kernels/fuzzing/corpus/decode_wav/a6ea960c7b4d42772888280277b26e645ceee904 b/tensorflow/core/kernels/fuzzing/corpus/decode_wav/a6ea960c7b4d42772888280277b26e645ceee904 new file mode 100644 index 0000000000000000000000000000000000000000..14954c595882e6d35c5f77f36dfcb9ad315c606c GIT binary patch literal 22470 zcmeIaWpEo=)GaEhLCh@6oW!wXh+|CR#11(;!3j(3Vad;d7$HL8+x_s(H>@^D4eMg47TVP430A=_d`#S=?iAGWec>-|(yp$+` z+scwr3!qfs zA;E;t05(7^@G7E;*q^+F5=Zfp7m;g7LnsfaOK2w;EsPx6HfkY#HS;1fo3WokW>J}M z=!Y1qm?ex%`d<1l=46JR_LH%kb%|BONfWk{#zm|XcjOc>9@D$iXVHbsLY9HaW{$$f zeZ~$(d*(RSc6J4;jXse!i$0WoiaM1#lNv{{lWq~a5^s|3kuxZE>NEO1#z;mUZ7F34 zwVLVTIr($>Q-!@Hufq$Y3=yBCF6ks`v*f<0l~3W1=D}PKm%e)%WNqnhrmt;sdGi;7P$y>mk!dlDP#6HEGOYcQJ zK)O#lOnb?$;JLZ^oH|Z1cMnI-n$I9nFOhFjoMZuc3N?xvM;uICMLA57la%mzAUc$b zwjsUzwJ6R%65Z~8HU9x|*6c|Su%T2$G9SvdneY)|+ z7nXm_Dnp{~kt#v;T$7^hsVUJC^&_>{G!J$64XaIC%%_dL^dEK2+FJEy)lrp8y;+Abo2-^%irSP zidfF%B_q(`KnYE4C`a z6m&UXPEo8^^jF5KOSD_`{dBczmS&B1xh6(cr~FU7O1ng>(tOarY~wmI9d74;o<#3N z&p`KO=Qu}(dLo{;A8b{<1TU(D>pTF$=D5prO5U$%|&f_IkRC^#dk6i*Lp zlTMNT8CDQxkn|Kc2?GMX;H=OpEEl~ATOQsuEJ`dEdxdGjV$l=f7G5g1kXyt%!k@tF z$eq9&%bz0nS9C(0DgHxzLG(}n@YnDh+yeFy#&dcKeGcs{{g{I-{Mrjw)M`?Uj{lS!bGPe%>~~ankw77PQFF?^;2(A43>{* zoi2OO+N7-2o;22*_M0}E>&u(*hg({n03Ny8ooV9NFRXBW=Nj zfDtYrT_zIYMsOJL01x6X;CkZhp`4H)v>JDlFcKO;d=5W_2*h{9)9^BQ61)O3!S6{) zL^{+AY$Py%7vMm!m0-l5z_an&0)X$S_XYANdKS@ow|k2a7ZUKAJnuXe?&~h4v(9nL zKHmP#mSSycn__iZ%1jrG9z(X#ZQN^0Hs3Ub8%_FbT|W(~TBLfUj?qLaV-#!Dy>!I} zl6j$Nq)}}mTdK{AEE#Rx+f0_xmKT;5TfOUrC&Fh(?*{MUIugi^h>;S(&%ZXlSG9iWV0WpcMP9<$;xcGD%OqbzgI^92{3?S05+UWp`e%&#*e%K3VzJV)IJVF4Gocj^Uzyrv6Vo zLw8PHp{i9sSNBt0Q)pTTwx-Fu%l~R!)w-$mh3uwmN9zN*Qt@6nQng-nR~4P za0q$`4S-3cGGZ0<9GnI|1Nrbw;!koNeF6)|j^>1O?{h}5moneer_q}k7g&$5-FTK= z&86`F<9*`pHljf24l6RArko%CLiPPXwkQ@wy`alKHV(2RLfOwlyM!iijlC6}# zv7Islu7qyFuVDj>5~E2((n>Opa*y<$G>vkIx`+CLe3tZKQ&&^wk?TkRX%UNJ4&z-GxW(nt|H9ve#fXmzBL!mKOpcZvV6SC& zWItt3t|CuyQ?W=%RdN(LiZ+E$ zp-^mAhN<~lr*?pDuRh#h){%7oX@_f<>be*M=CM|m{fOhKi|ZNgp6x1jvwVY*kr?(3 zLv|qsG>e5*YxT{=gF9k%~rTVm7Me>sLZ#@Sa`@0yvW zRmN_H&AKG*PtADkJ?$LrEiIrgFq|^n)o1I&40w~<%xmj!wcB>ux7y#?r`aFae>&H@ z>7EYWe!exnZ(hKg=k4yRK~#YsxD3L6KmZBh&2T!@9*_}w1KS7-@K10jaEtL%2@JwV z+yLAv+@{ci=&tTtL?7w7m68g&jHN+*j2y&n!lngh(>#*ECiyTYsOMlIfFr%5@ zm?-NT+sSTbe_{{dL~?1|0*;NdidV}YF6bpN3C;*#3+C}Z@EW+gIa*c^#vZDHJdqSm zJW7Pg$&|mSIkczLQ4|i@OYBEnMD!2~NG$SxvWrZiuAz0NccKSr$7xI&gLZ|+pqprG zY5i$;X+!BW#zy8wb|jb0OX1Do9^|}bOIWw*3Di9DOVTNFF{O$!hnh=oVtiz<80YA> zX?LkO>UJ`PBqbG*%tSWvEQW0!VMs8Klt*4jkz%o8J1Gke1Iw|<{v}Z1&+;evulmJ- zKp+%)On3(lfc)T8XdGM$zk}bx1uzpn2Fr+p$Z6EibORI5-pMXuTiByH|8QzK4DK@S zHJ(hcPvj9z69bZJF;$!@nv9`}{lY(l#e&TOwP2~BBVUI_(rFwGJCXB{J3#PFM3uZ}y<+cDZ;hAaTjbl}Tk9R*?&f&kmSWy+EY?53>W|^-gG#3IoD#2^tU^^i z)mK&9ly?+Gib{n|F;cNfeq0`_=%?@~UZ_rL+UeHmcNhj5pBV2MyBVh&cIYqY@kWK| zuw|*$WJjE*U9Vid+%Iu-oEZCdyKwLE4+)P2ZI@-e`Iq@`%QACs^Ct^rU1M#wZm^Zwr#aiXM!D{|HoFcxgN{h&RA-s91Y7eE zG7asCend=>) zG=)kw!V+S8QYyB(3dxsA-5~`&8#gSNgP!y9Jwx21+{@imcd@(18-X13Mz}4GE7m!t zLAtZ5r}FPDiyD=6Z)%oQj46+=Ost$=@ufml;iw#3jZ``+wp975>9u*kM%3?Wd8$lP zM{D2dnfl(k{kmuBd<9)0R4!26P(M?j(73faO`Q6#{|UjGV(mlID@Sk7HV@s$K}LJF zIvvh-o}llF|4v{{un*pVWxzCwh?Y;gO}R_DN0L#hs72HxbR3Jwtf6mXma_9X3t7_` zNzA(}GV3^V8Z(7HgyJIR5v$1qXxFgU+GvO93+NkZcc|NF4OBmAI@w4~rM8i;P?H(= zSq0o!{$GMB(ebcV(yL(sNxo#O=!fu&I6_(!UL7$bvM6F~_&?$DD5U+$gl$RPQ&yxn zQi zNVSiI>_t3SWG@NK!cD**$N$7P;xa>b11tP_{;~dizuiv{9t&+FBtui-DB@0d4!EC? zf@|_WM}GKh*nR|%G9(qvNBg6b5v}KlE8SUbci7CfH?|hr0K3)Zu#I zN1T1VZKHjwGtbo%t4P1v!rOirUAkn=6=g_Hk`I*^C_8Bwx|O;onsVg@d1h-rS*Gm0 z>|U!;o}vWRYOTwVWA10^X-TqduspYF&#*9}6Wz*K1ZW!i0CW}ax?VSZyA zqA%35jW**n(+U${I%u42Jc&h$TEh)}l}@ddXl@lbt2o#)rII;fu4Sl7&NO{t6=Ghj+e8harY%^t>MwsrKsOAgi zLd%-AE7pJP+g-anGrTjrRh~DVn^+BU%&m9*P6nresbc`lJjOG+3X9pZDO&Py z>O^`ga|b(-eVQqvqvW^5%PYa3}E%_*(o{AOU_r8bYn7*D@vS<1e9WSl^|6QBpN!iZ|0ZE*!kze| z@wYn8j?0PL9yc>CK91PoQ0&g=im1$}RgqI8k3{Z@+!^^e@<${m3W)w3{XXhS1UGzl z*f((>@p~~ODG{Thl_Hw>u=pSGDG4ln82&nPYt(_rp%L?>GI0;l5#dy!M8p^WDOQQT z3A+hy@Wk9Z>@}=O%yo=+bOwDPEtdL<)Db=bRO75cMKB8Y75@Qv3N3(-K*zwn*u0Mo zeGL|3@q>u_2Tvp<5$+Ps6E+k60vOfg8QLrxDwW2$Ylv+0bO8ca42{OS_Io*AF+`1i+CHhLI@;> zilAKRDaLE%g9iYVuo&nD`0*X_OK}53@}ND~Az1A{gr4^G@EmX%9basdtnqENrb5GQ z-FVF~l|liuQk&-2zxwsErd{=@s_4o~W$E8fm1KS@d4Knv7X)Rsyb>r7LU!%T``I_^s*SC&eZX@|q_p6j++RD0l4YiFi&4QLkGM8Me zTBI4MYu2sNk1)veX5D?=Ae~uzME6ub%y`rs*A{5IYg^^m?0n;@au4(j_x|l$=X>mX z>6_)9;s%^7+jGlMb9YlmQ={pMX|_>d0QIYN2X%G2N%~>~Y=X>h%_l8-%K^(lOO53* z#)3p){6c$sp<{|O&fVS%`HFpukaV=ze=FDn*A@Q}-;9e2?F;6GkkCxrXB-p18qXu> z2w7kYG>-Uy6pLZW6VxrVd-Ms6yNpT98!R>_jO*fF<~`xH@E-HS1snJb{tW(2zJwpg zd(Or2{^fn+e-R84-VriIgGEzCV??_}X3VHj@$E1#~U)R5X^`FA=sg?^H;o;jA;lkt#dpbWLZj}Ys;I|lJL&&0jLd&oc=kYStYLLwG8x}!?WkMH z?MOa24<^CKpp9S_@PV)o;DIZELX5px2Tg$zps(OT@GjUB+zCVj6$BnI3sgc<;snw% zvXmmBj3Q6PD)4TklO#838@V541BF9*NnVM4`bAPl5}vpc3IHPsak%hcRG?enS|A~a z1ht|57*>4~Nc1m426)Td``sG%MNf-&Gcw)Z9Eb}x2Y5k8P!&3i8;H**L;*{H{(t~T z044(~fe%0#cmY03%A*{m-o|j)EaU%X&OmFI!koJJK>)2 zPOQ3qNQlMDaVzmB@zr=DZ~~;mOd<Q`#kbPq;PQ z6n;H?Sa^=~rsRn@S3Fm=To@8e6h??Pi)M>9iE!e<;zkhzs}))VD+GT2RDK@6j^7un z0}TAt{Bt}7H;fB%6s&Vt9*Lo&)aR5Alst-<(jChk*B}sx!Ts=m^0Hi?>=CxOww2~N zrW)fN<4R+cahh?2sjJyzo?}^HSz@_sIb*S!qs(7T8k5a*(}Xh>8kZO|jroQF`kuOt znoFt}swiwtXw9SMYWE`7%6&djckH!Q_H1hZ&Rr8PQ%*zgLV18rq|4@8dZ@{ zHsgo<8|CYf5?;ytFS1YhA5%XJe!ueFv$u2K#=mpEFZ($6^Zb(XuZHitf0mYwFW*z{ zD$lG)tQcGItU^{HuV}1TTk*YoMEUaart+Hde&uh=R+o(~8&&qTtg*aZWp3iw%eF1Gzjv6OMXud0wyVi` z$@SEo(VuaK{)wK>yv;h! zCUIx*zX>hkn_*9+x5J$gdm~>(TnLwjmr18e0jXFj4A({^MRkZGMg5988ND?+F4`LP zCTeulnaJl6E#ac@0;wV_A#93-f#tvYFblS7+|ov=QQ9VTOaBZX6LCKBQPkY%U(uN{ zZ(;_=-jCJ9qOpqD;@HPA$f=@F9fd(uCo55n@oQp2`Mu8DpMNP>&}a^6WUgR_b? zlxd->sJ$o(asp)uZ|rbA??6PN-_C3MBF z4LJgRu&S8ipMoZ%87PGIL1KL!yjwigZl5dP<#hhz?BzV-_-U8h&RhR%Ta49QGUGzS z3;jYpPQOe?);YDM+GCo&Dw#Z^l`hL^$!!@T9b_(KE#jDY`9iU{RI*O`GCVfod3dSR7IrXfQdm%OMAA|65kmn7L<5Dfg1P+j zyc+H;E`zt0$K_K6F2Mm|P{1)_XRJU1*Q8pS7=appGiX21lWz%+bS{;oRkDvM;kgv#qhtYm-?{ zSXN=tWvC_Dl5HVb3NT##48!?9j3tKI`diwU>SR@a-koOY+UorriV>W znzEayjhu$`hAY^$tbSd6hlY<0#Kzu@$xXDDpe#=QRGzOWQP7oH$}LL1%Bu?3^wkEn zDY`4VT783|*)Y@4+c3j$&d_2=F}^YWGNoH)v~{rVv6flIw)VCb>ptsMYog6%TV`)( zueCk37d!jA?|2sbD$udPBwQMP8{rQ4H+%x_4`;w4EMJyFZO|cDN@^hcDe<&;##mM@ z`yb9iE`c|KH;UJVr^LEK+1yO7jPsGbh;@PaH^vE^WL&4KXisTPv=ll*AI_{}@i?uV zSgwG}=jL)RVcny}{89l|R3~zZEEwJ_5Q`<3B|jx6!ls5_jhGj~ig<~|iXD*+5#)%? zSdIvV(<0_a{ER#sT@{lSn;knY_HnE__B!@=K$*=f+xNHphr#hD9et^^VMm zv_)=+S`rl><%)b3nHo7Df)Q>A^Gkf{I z*qB1vOw1v2h=VY!>Bm^h!MM`kY=4<=r>BGK5Z13yS^&!@^8xc?lgvmmHJN6ccbo5; z%gnpXk4z_x(+tD(|LN}P=IIKx+cg;)P%}tPSG`l*kbh~7X#FW`l8IaAv|egm-&!tn zv=p}d*^=2}ZraW(ntEV-dZW_}(x;dg* z)cjZTmgW)7N17EaH(KKq1V>|!hG6%+kDR~vIyHUtv2jAkF%L< zuk3k_0gl-Y*eP)>b$`OJ!$;p^WG8wJjYS_KFAxBYK>27kx(vO9@jX@Oc~pyz4A4TM z5Q38toZv)EL185xAcc{cZ zSLp5Nr)j-u-Dm~0NwgEx_mnl1LP`eZ75Oi6Ha6rW2k|}DWqt^r26hk*;A?TiarL1| zAw_Ux&>bKL*}+$V$AKvruB;D@#9hUu;JXq?KzHB-@CdBJc*pT@1nhyL;jZu}s0Y*? z8Us1O8Q=(z2hD>^h<8XrGK>5O!&%3O&)`1TT1Ma(7)MzMC1I}(hxUO2kOY1OLIexm ziR*)F3SA2g3LOX@4V3u@`Ol(H5gt>I*#9wL z-jnAAeA9hLd}Th9?}N|oyM-j6FAw?7ln=5_vPQBtvN&uF`z>cQ_cJ$} z7tfo`J;wQiGm~?hQ^iqo2689zp7YBD^M&h#i-nVgje@uQZ@l@uxxCpt2ConIA?F9j z%{j%9a2#wm`x%?ac4E9hdqxlXW$HHaeBwK39@qgO5C-6FxK4Ntz7rPd>G+=bet0X+ zi2EB~h3`lZW9w%IK84U1_yT+f{sj&JbAbS%BjF<6fuBYgK=9*V;CJ9xVl%lBcQ+&s z6$RS@T?4EA8U9QDU;d%~B9x7`BDIJPp`bm`i)e(ur(cI|Lvzs=NTu(Zx6m`yecsi> zWpbW$UUV9rXPp-uC+x3nxwb=Chd^cfVN0{8+PB*}*$QkH+XDL@JKes)w#YWe#MRal2>M`m{ z)eekWNjn9m*DcAJb1X!-L z!8W43!H#p9U46Yt$a-`FrW}a~hyqo9AP^q>I~0Xmh8v1gh8~1gg?IQk)<H=!C&6R>Q~p~)i_j}vD&z^Bm?ncRTp+wBtQPV_y+!RrNuqW`@W@Es-5?I~>JWq}5sp)=_OmmLl_B(^2D5!%h8Xol84W zOVx_CAGB|EclE~%kBys5i%lM*);Qc`F+DOb#55iMwRN{fSrLr2y>9F4h;jYv_IW1x zP9u|0i9ahaBRDM-i#v@|;tKF{2&aG_;4MfD{{@ePHIM;P!35F=(pd5d43Xj}wnV~y*KUkuyymvnBeSre-fYsPC1YqXkZtzB~zyNggiR1Q!`?pZc0+#+$+WXs)AJt_?D&VTH>eJPys=rrP)r|Sox9)7+U-e@f8X9^vPH8G@ z{@!w3_Ox}G{I#5{=%CmqKY{5qeza6HlbR1T4Qh(RQ0=3pby%Eu*4(XmMf1v*m9lQF zMe-xcE9!jh58YiusyWR1!=CFZ^Ps-Ys29ay(V#D;v)hMzgnNwd58MJr!#SiqSjE|m z8cXd*;Zq>$FxoBa}{d_Tg%Dd#bbLZLAXI&w^8L}YWMJZf09H##eJcRRSl%DAo_Z5@jf<|qD_^d|Xs%H7UT7cea< zZGGCq^tl;r>3^gzN`IbyIDJ)mpNwUhNm=r&U)ey{zq4;p6+CPpx(caThlJqvUE#pU4 za^}#qV_hP<%uVXsesx51SXe|w?11>6iBRXIUCwmI#V1DO@eA2Im}bfcI1$rdRxj8VVmTEv_v{290>nc%oj}L zj}t^WQ0y6-xHCPk@JqUzr2 zEAkG_7aHoDHp|Z|+o|SgblRzEn*3LDeA9>~UGx1GY4fytY0c|OMWwPTvX;?kXf05^ zQGHP)%SS0h$`36?b@JNe`tOYqtC*OJ8;|g)s^eI?dgwH_}_=L_~C@NxCwzVh{9(> zzN2&fz5L&>99)9s*)rr^Pz@B3?oeKkH6$jr9eo^i5AhS&8AD=wq1)sSjGjE9=%;A5 zz{grhDS+*ulaPd$5N3n7p*s*CazP<7mGOw)h1!|=k~WiukOvVLKnpRxpbX~?{6OoG zu4tZrZNL?p3~T^z;I{ZcmexUuaPruix`)GBr~P5pJhvb5~i&xo?{iu)J~w7Vv;eq!?&nN7B^ zb*Nm7F#?SWZR<4oPQ@MheA$HNAx)Q>hBp^Avt-?sEN!!Zu zQ}>`|R@K?^jb+;^#MNNU>*}3VLo07rrc{-bYrY4+wtZ!MGk^Q?^GubmZducVrX>wa zYwuJ{`KkEc|J$su55LbU|EKamb>ClW8gg40@~y4SvYx61m~v#kq2AEf+=#`onMR*( zlm4R7VcKd*vcGgU`$i$lyiQklC)07*KG)IaB)ez1w_%;7Fo(tY-7^S5u)I|5pBkKn zam`eG1@0v7C9W^wEkvXK!R{g~4LcaYjkJUdr0c~03jXH5;-?7l5>a@ch=q}RV=u?a z;`+sXj?YWlnH-yZHi?pKN?DeAyz`V~N1`_Ids1bxH${-PHrv%bwcDMHq%Ovk*Gca> zH6{GlsdLh%q-TkJ64xhIB)#a|C5@0~O?}(tO?tb`7n#>G!!mi9C7Cm_W@g89pPsw0 z&*^?!`hk6&z4qi1dZc%wWwA3}XHc`2WUk6Mk$F4&cGuRfG2ME0SM=DE_bjht?%tmE zo_RfIb&t=wk~XSKr!E&#=XZINc{t}~EB1+Y6y-AeDfa-*a8jMJTAs1LS*7HI2BgFD~UJZzk!{>lSsLz&3V9)<$i_MGJ3bVr6vAgt}q%JL_LJu8^Hm%v1sD z+v;oDr~1iysdkT=pw3neR^C^>*UUAYw?#M%Hoh&@y1|@f(CV(~<%V>l3sdfl#k89; zokIV^a0y#OFD*3dS?he8+rA#FYfm~+$1i6JrquBw^8*h;^Fn_HT&M)yg()~D`#vKU zMCM!VJ?c5?$@E-sWw;)CD9Bo0o|o*qfjmUwy?Y%QZF|hc=4h;vm)KUi1DIke&Ru7_ zVM#U3)33umIalw}RcbUUvRb13qAxezH2uUnxVtn$^%C`7?KES8wa($gs)KCT4Nn|$ z75V9X;(CK=%_dvV*?Ksco{hdkC=o4jb!y|7G=>vH<$C7+*E`t1Cz$B} z;5q4<_08@5v)Mcs3%5e&@vY!SpuCM^sfb|g8ng41(;w-Nk@}LH&n+e+XgsHH)VYn2eY}T=?h39YY=-MdJBZ6cEpQb& z4QCTyk{{BivX^l^oI5NYLr3XJB*T|Uku*1bIAa67n7WHxNP@^ElzY_wD442=tR^mo zbBUFdh0LAY*@6M0za&jkTg2fgE7mVfk6ayI8PO}|OZ)lp#YyDOwB%4nWxKO6P}IP1 zc6dS5*7h6Y=XFX>`lIuSE^||9$#Xi*PWYNo*6Da+*W|=5F`2hBOVSsmE$Py=^YNrd ziN}(SsikR`x|~cIpD0YY9)BkxEYXnkA*C$kVp2vYUwmXjc;c3nqAn-WKW6f}?(X)X zTbFLG?8%vwE;Ewk3GF&1{g&7Nme>E5*Z-E+|Nl*1|2v)kPUpYV`ER$}Z@1iUx7=^H z+;6wsZ@1iUx7`21E%yV{mi~6j{dUX!Z@1k4ICp*@9P<0%kpFKV9P-=q`P=jP|ElNn zw@Ku;N#wUls(k|6d4bbIkw% literal 0 HcmV?d00001 diff --git a/tensorflow/core/kernels/fuzzing/corpus/decode_wav/aa526aa853333f0bb11804b5243df411452cecd2 b/tensorflow/core/kernels/fuzzing/corpus/decode_wav/aa526aa853333f0bb11804b5243df411452cecd2 new file mode 100644 index 0000000000000000000000000000000000000000..c0cc5c469f9ab41222d76349f960610e01d4cda8 GIT binary patch literal 21331 zcmeHvsnaG8+Sp?K!6J?1B~1ObCK7;OKJ?f_9I?Kv;+g z&78Jk+C=akne(AN$JIHoB(w~rd!PA}aJ>k6Vh(8nVIi`V5RN3{24XSraN=RgIr0c{ zDRm!xE4>%3kS1qNWlg8^=^LqYkg>3sxQzUp(B{-Lj6USU?VvjlBKiIE$^lGw}W&uCp}vuOgxYNn3CVvGdGL;5ayJYyVlC##%k zrcI>IrVXZ@p-iL9q9l@S#5;segxkaiq#Ux1@{;y|K7yV{T}mEAsbV;}4&FT8G(oTM zcjDF2y2!60r)aWB68=!w$|LheaS@K2L*@=<598e61qBZye4@VLEPfN~88ezakX6WV zFm5vOOdMk^L(59zPUeXOd%_2a84+{&a_&OTROWi-7Su>gQR*93IoHL> zXV>%+;Qz_AuM8ZJA8uC%Hlqg3o;$lKQP_wV6zXpZ;Bha0$ z_x4#XymyQ@&9lOl>6~I;Xv=osomXr>Ow05m^ks%xeUdIhdq6W@|JwAAQK3uKJXNG9 zUa2zF-BrbEf_8-ZhU&5Ap>C~VoAIo^r}m3RqOMVHRUB71l^ZoHbO$sC74b@)dWh<- zQmWl$(CEAAt8`ywF5&XZJ+!MBi=iInOE= z(V691;=L2t5`5>AyMH^sSyRm&O{5_fA0<`e&fWe6#({{(asd;1_*xtaXIj zYpkiZ4YqoV(fE&HnQ^0msC%m&q@QSr(MM@0nhgCQ{SI|U#W&e@S%i!xo3^i}M#)sIQ?F3PD(d9_D%Yr&s1>Tu+Bas7y@TE5_}884ndtt@b=5J>o?}1n zXmV|HpLA#XCI@$7Zy>vf0|+~i2tq0THugED7WWaq0iOl0gtG`dath@oC5+aVL1t;0 z1o{VZfRsliGRhcnj7<758lF+jYQvt-I7Ex2`)OGeGjR|pmb{hxhkSxuOX^8n3)U)+ zFq$})@(k?VOSF30e99@}FNA}i0KbMw_;fr5uZ1t-_Y(@q5!AP2CovLym*<4jq<-W_ zq{jj;N#tnAm^^Snm>IblWEj0m%6s%Uh?vIt#x z_pl~GfUo7B7gz*k!uJs?#GN9d!@|Nmf-FIi@VQ_cHcj=u) z&NU8#BgWCgUTH0~zO`^n8w?YTugv}JryWnNLDOyH0R0$EU*$I0K z)B1YDLBnQay^*Frr^cvd;KVA!6u0D2^2_p6Wr-@NxvsA;NDMhfv00y z*YIC(D^`y=g<)ZK1aRILp4Yz7=y{*ov(r=PbNT`vgZrbq+;!6_chuSM*v8v_S~Dz7 z=BXBksnl>u@7Cq&UHbiobmMJ#`T7zVBk zw*yy#+Xa8euON8vQ?MD>*Eky%!Ym8D2#UaOEeovjqrr1@jT9chz0i&eD$7(lr;9<%$~R zD`j8B4VkL-uhuMS7wO-vYg)IozHYhQva9uxR4)4@AEDT&xUYy&T~^nrM`+5lee^%| zh`w2O#29IvV{dVUIa}=}+fLg$Yqn*OCB~+*Z*(L$2fLT}asqFHn?efc0PYC>F}^=S zB$g5?@vq?N@JpD7%p&|D)zKy}Ayy1q%z4Nj&RWX&M4L{N&@VBcg57wYRmCCm{^fq< z?B*~zZl;9xp1hiy)5=4GL{6w5iK0?_?c}+S`{6J2p>1l1~jnp0F)ui^sefVT}C4L}L zOFl?hOP)`vBjSjQkUV$-wizSF&4L5CF}Sbr2LehuK_-$v6F(995YLg8kq1y9S_%Cp zV;doehBh}$AyQ4dxU+3w*?tONthyhVMLAk7oU&AVLxNGV*|L$@Fkdo9}lyzH0V-bt8cJ( zxVO=L(6!U8@x1dCy5G2>+)*C4C*K?Jtn&6iOVMZEN>{PttKDvct=rAFEGKM5wmQpd zQ?y~Qu2R!Q{X%g-daq?#ivS#hS~;@Aa=ScJepoh7mM6O{TO_B**|M%Ov&<`#$+pTP zlsvUV-Cwg`E7ln`M9shIVd|xt_WFQvtc7ViW`E)2xQDssIE!3N??B&gZ;>~`x5uaR z9Rs`ffV-1h;g0a`_Nn|u&>`FbI1BEHOUAx~B0^n*Xn+~q5vs==$NhqrA%(rY$)j8?~6je`)BxHqK)Wi zG!KpUpY%@-dy2Prn)!K*J$Jtg|?i(3~HTurF zt(r9TAJusE1NB_>9W_q7Om{|iUz@9q&|wTNBiG!|VzchHZMS{2O}9O={c&t`(cB51 zzTS1-pB|hi&(p=%!JoltkN|fHUjom^S}+|khTzh`et$=_(RUe5@JsyZ!HOU;nCV~deeNoAz;=}- zV0vtb(iLf*sfFrp>Kp1<%{I*)b*>7p^vTY*hPU*PTxzs74wg)4u5VU1k8c^<8YgX* zewEP`sj8J4u{KikT0KNNTGv-=)v$FZb=~wMj0$tIy{-GL7w^M)C%f-C^BrmSS+)i% z$=1U@(lOO}#68fv9=u9IC!=ruKSCU=6D!3f;nR@@WCO_Evq^E3KD2lAa7GN{Cj({v zWI0$8)>l@4b`*!oS;n@q*Klii!}vY<2L3t0JN|s$XKn*$FI&y*M&C!#ktPzwgyRH+ zlurJe(v|vxGLpPaetN~d0<(rE_jdTKxF zed=HumA;v=nH9xhaWlAcIfvOFSmDgOv=mAn=?(D=sfb)jo=fRLYodRlGwBy-cd7R& z5M?KcOcW6diADm8a30h)PeCP^N6aG?kVPO?>?C$XBH%I**}n(M{T=;j{_Fm*Kp+qb zJ;Q#4`{VuaH2gTE1o?=3K$am4e;;q)+EcvToBEK@iIR22sWLj*_k!)W)Y1(fsT z)8u-RnD`udhCc=$!0pG*15e*hC>esGnb1JU89D^jLGwd>f`5YDLzjXt0w4T^=3JB8 z=rMAPYxQdFZcT#vh7wlnk}Z;MXdTz`Q(|qL)8MMlZ&=;1zah1e+;qIDxv8CGi=;&| zsrg%TR7-POLEg2TrEHn)_8B8kEZ_^=jgQd(Ww7szASg}@&Rcn!2fG&o z>6+;|;!W^v^L_Khp-<7t{@lQ#;N8%6r~$eGv7w2fLtqb4{TF>}y*E4+?!zvfqo>_r zjj*=0oHF$>&M^)(O*N^E<4h;a11ulRlT9Abb*W%}U)7{a8b9qs{o+ zC^T+0bTSMzTre~m;*3UPvnk7LG_5j4nIcW2P18*mO{L}&)>!)x#~9~W*9P}N58@?z z6TRcWeyR2D^z}noey{&yz!O*;X!3Ur|xs$eP`lu8v*lcpW2Yp36)Kcf%o9~#aY?WVhyxi+R_t8=+)p!<$znh)oH z5SSUNf?~0^aYOM%NETr~!9(al{6hEzV!I1DM2H|pkT#JoPzcmg%1jDMUP0E7K9gpX z|D|lF4Q8BS9%8Lx6FGJ4aQ05tRMt+gyVBUr>{a zC~?f<*n@F9+pyys+eqVf#x9A`M*obih!(}@VxYL1xV>%o@e||6#P^J!6#puom(V$3 zM8bxI2MMwSN@8MShs2?Y(-W5_?oX5_79_Eg-zIyKcO>hQ&Lk~MnwYdI>3ote>3(vT zl=PI+@tNSKr`D&a#sEuI-)8b2LebxFubaL1>_m&6T;rN`Wj zDi&{v7!p<~F!L?ESKJCtf6hm?hV`DgkP$=wNM%z}Nqa#brNe*0Pr}cDm*diKL2NIu z=Uvcy=n`}p`Upi~44CKGN^A%0C`<&@F^Kr%eH%URU9+4o99wM{W8AZNT-YhRvY*e0A@ikT2IK4*y*s#|aZi+RXFzO6j48siF z4dV?z3~!8h(4DNYd<3!KxFye$WeEevAIn{<-r8)1Z6j<9`$WfUX9sttHwGQ#hXSVp z<$=NgA#mP*#h(_C1eOM?g2v#3V29wIz&-ytbeu2IJJs{tecN@*Ior9~neP1LXmAX4 zmOBT#Zo5F}b1ii3bq;pa*d|#k&D~9?VX@(y;cw#t(>n7Mi{7HKY_+g0&1R>0jKyHt z3nJJZ+Zy{?P;F;9y^eavcgH`DTMooYbFFZ5ynWERKvgIKvlx3AM~1Dq_c$|dC43j2 z33rDl!7w}w*A_Pzr^i0Uj=>FukK^YOx|80J`BVmN1g(L3fMOw?B>qi6@fP?1{4YEX z{{!C_Z^bRazJ;CvJmLsd=O1w&7Nrou9D1)enXq_2yCdw*fv$=V^MLY+02X_*81@{PdBljbh#~&~Bhc6Nji24@& zckKPRq&C<#m*d27=VOP)vSSlsyT+x*cTDsoO-bq27MB`Gy_R-4ZE{*p+M=}f>3JCw z+KtW(W)5hd+x|~x|ICf;hGq;;|CqKcZD1Nd?R08m+r|`c^3CKi$%JGynVb4GZEyOg z^uy__^ucMO)Vh?$lkyWsB%~({Pw11-FX3nc zF0p6gu*7MJ?gUjlF@9;AOL4~7eX%*QYh(MxJ&OAj=ZMq9t%(bdONiSM_ix;&xYMzX zF;AlVMO}>C8z~dZMC(NPqOTFxBjQ9)MdQVsNNyxX+!WC(VqJKC*eGGKV49$jU&yQH z4CjpI{L8+@TEax>=V+1CLUJ1E3xS0^g{R`qVunCdp*YNA%xBC*OgYpK8WWNPcKE-c z=h2(!RrDoVf&PPDLA#*uyj5r-|=4X;pWGPFOlCS zf4@=u9}xQ=5c?kx`@avwLjM7={{gZ80kQu9vHtoFTs;$0(*K zP(^p;b;S<(Jz1fwLS~hXkZqBkl*Y;W%G|Qoiqon#nhn}ry1(?#^$+x&^)qz4w3oCP zz07dbwA5m-`5b4RZ=F3|Z(PURZM?&MRe*lD>n8-B`TO}r{#^eaKQF)wVwKPkhz5lL z9C;7)3G*2H4f_GR8hZ`v$K}F4EF0Gcz6S8jdb|(WK(vxrlmu#5>M?Q~;(UA!mWGjr z)`otBx`%oN*9SKQvjeOBd;LTFLVqjT8%^=CJYSqM99M1gEmO>UOdE~0#!05-#$Lv+ zCcI^xMPk`xEwN2^v~iAf-g9ns9(4rmQI2VjQb#dhC-A=MXm|7rTH>GRkM;Krkb`Bx zdC(fnDXbU1K$uIOP0L~wGBOys3;|Qdm_@%tZ_l98EabCf3Q3QI6XJ=PfCpGjx=QSV zmtk_Dp~0@`1rN_X$TiZn!bNcvxvD*pzQdkKm&tz3GS@Ibb6)X6`m1?SqrC2Y^^)>2 zWyt{fT2TJITv2YX7+B@2u$OPE^j6Vo@@j|I?`eJ^&r-&yKWZ7;UYdiNm&$w@O(u{p zRNPX&RGw0~)EZTy@^7H|NLLl99~<7KgXWq#&WVh|`AZK9K==LpH z@|-&zk9O+TrL1d0k7+&c^gNk&zeh%QZI=U`pLJTATbL{C?CE@}OISB+w}@`0&I>xu z&05heC!>4D`V3S0inQHrr=^@uncNnWwm0Lic4yLSQ#~mQ6CcIBhjQBQAuc z3kL`W3k-ZLuLt`kqYrHt#S64)D4`DD8D|Yvfaj^m*UOjbHwUu7lQu9I6PV;5>R;wx z<0t!jqECG@(U}2FFf&97LBVwYU~~(L^-uB7_6Pi?02HY3Re6cN{XRF?1jT{b&;-m$ z%pXi6)FE^~u+pFBAM4Ne+x)cPiO>#gI(`}wP1udhg%4sgpeFw-ze}Gycz7Efi{fIWVvfewo@_)0_1hMaUx&~-s=ZxS8G{%tA4s+r2%I+te>Ml z4MItc?v}Pvqg02ht*ShAt-4jcM~%_k(!2)P-zZJ2`iinqK0-E9dbwq-WNH(?X<^e2 z$@%8&7GLuQ$*RU9^(l4k+Ua%pdTwK#gxAWJ#maxm*2vOir=@MA*{xpyqqIVLx3#QQ zB>m93M4Bq^s3d5F`sKzsCZF+`@r2Q8$TbW%JTy>@myD|c4tve=k8P)Ok9($Prl->V z-hCUWMNYW1&VTGP?J170)-qE^(@E1G%Vqlodx^c9%jSx8opDWYFZXmsKL;7uE-(uJ z!QVjI6L_TKltkKQ#&~uPyMdvh;~07Lmox=n*XEGbq+yhav{uG0Rx0Z(Lr6nO9|%_w z6#o&w8b1e~j?KV~#63r12xswkVFW&cyMPs99$>O@gRo|d8#fqE#$CaUgemY1fYQ&z zR%6aV$)Tlzv*>yhMc<+HARTuQ-xXhu|BO5a%>N|JU}$EjJUAk>2=YODX?b)EQ_5V=O5q&krt)(6_XT%_$ze5N&hTClPs87YUktw<_8DB= zi+C$K9+?%rDaIAEHReQ2a_pkmjd4fgPbcnAexEWp4O1*q{iTs;)@g_9TC_&Z zL(KqdlCelVUkX-$VrhfUR{XF#tO4U}NsSGmu*&D>`^F@F<}&YQ`*%?sxxa$j*E?i21$-go{0!94*( zI8ZoEI7YZvXcTS-O2M&$H+&v{24Bw47t9fsgtdlu5ygpn0Uq*`=y3QUK?HXKGoL0W zR}UX)-l^N81$diHk9q8HbgIyhY*ny_|0%f+-K|o z92Z`RTMcw}8}L){Dfl1oVfa4an|9-3aOGGoZZ<5(iwF~l%Sj?~IC&qi90CY~m` zh&xDq$(zV*@*C1B@ZT>JlZY6?DtrJp0-FelgVBM`fg6F8pf9Kn^#iJc_kmRZB42+` zsq3Ii<+|){_H6ad@Jj-TK}mocv5l9MI;h9kIE)lpg*lC> z!Vqw$U>d?8FoFIx9tlTwAQ^;O!b>8S+=FtI8ckown8EDGTE_a#oXoTWjmjW!f3vx@ zTo!LE?>djgf5Pu6cqE7yHUeeW-tcGP=J53q@uEMX_TqhFi`XE(DIO~BD!Lv1Jgi69 zJmCsKh(A#fDcmZYBitf{!Ul#l3h6-A(ac}T_w%Ol@_2Q;K0vXd-uZEYc{K{C}QP%q#3OrT1s2W zTBI$}t)qZ;kk(q=GNENl^A(AwDb#qcVSWAKy8PN1)w3!`mZy}?{4M=S{&B3BTfE?V z%h&ubnV$!KTJ`behj|~8KRP~@ewp`eL2=m+-LJiWN=nC&jxl37@WdTb)!J zs8cj-Y-*I$H-RLNE?sc-9O^z$h7p|`Eo9>sMzCIVa zJa8l^3*Ey^06b|vo`!rvZW7$Y@#GN{4`l{*B=rhV0&gX)C1J@F@=MY#QcuzgqL6rk zaG9`_IEG{dnm!Wc8YPo@kt(Der8Uq}=-cRh0W@ zV$);RMbRR|#ScV7M2{l!A~GX(gx?VU;S>3nd1c(w96Eaqb1=h1Q&4)6Wuz4H67oxO z1|1Ym$ zM|=C?ya}Fd?kbnpneTJ}R=TI-nEj7UYQ1O~ZC(s|t`>cP?zOf+3u%{YNE(N_M14Zl zN6{k9X{EJvZSK)LsD&%-FK?&psIsUMHR;-kTDrDH#84qP?4deq?I!Dn!KUu-9(Hn7G&@%R8^O!wad^XcDjtSEl-JDd#XAYELfp>0W?oarzv4 z9o3Gvz%6mWS?VZrY<85`hFP$tY{N%gzILrTQ`JMIQ$?!BsrP7Z>7<5(ruF8wKne7~ zO124X(YEupRd&=~Y2RdDZ7;QVbL2So*qdz2Z7;3sEc4ARrcx{%?J;ZjSbj`i(LjFt`(BeCg8GAFUT#=`G_Vx0@a}J#WfwqBOD_vKy{}b9w!S z`hl!q8i zyP#{z;eDw;UZC zOoOs8JFxfQNysUrACiLzLB1@(oAE~w5wU^fCnr;r>0_BStbf=A94vP@cO0(`kKinI3S=dj|gZ~I*r%&8}In&s7W+9^ueGl~}c?2mM98-x~30(#2=ro5n~+Ng^de$={M~lHrnL5?S-D)vQy zWsd(c$A6jQ|1Wa@m$Zd&2Fa+*7y`cniiOk%$`~gLFc^ z;=AFy;K$${Kp#3B=Hllg#e{oA0f|X^3Oe)?gqKKfPzCvr*T9Fc8lMK_d?e(m)2G4 zTIcHN8ti)N`st#%*v?%Jxm{)>+rC>N+e+IL+dEqy`!&Z5SA?g?yWiIpwV?C;xBT&e zLBU|?8^#4Rt1of8;bHhs_*i5+z`gijHCurj<|JZ5M3$tO%9W&$T#G08&e4t%oRs4{95@Qog3o@S05u7VcLQofrjah z>5H*IdT0`+5|e}t1JBP)Oa`_O?mO-m?g{P)ZXPaxO~PKr*fG8U_Y1?USYh7kFSr^*&*=V*+)3mp^f3hOIN zsrj6#zwxEMM7uyU1+WZrG}qM)DxxY{nWpF=|0pYyJ&@U@*-~?BAL$rbquin>P~K5K zP>xYnD0V4~3N>Kc_Nww#QK}l{Dy2aAL2+JTSFBa;0~%(vqDbMFdu1o2;#PHYt3=Th z*>tdRc;ofPjHc;LTbq~?w}jKurj;w5F3VQDSDsK!QlHUSwLaY@eUW~*{-qu<^e}ug z;DDkTu@YmN04Tj{QN1-*L+TeoV+hCVa z@6h*PL*S-=DVpJX;+YLPzq2m2s|c(T#+&MUhN}D<1Ny*&pfc16GY4COJAcussz zl8~oT`_jXii&>M|Q#hvpCi$8_NKhzPEpUMhkj?GO@v;YV61ZQvT;4Grj_=}S@>2yz zg}87<_^0p@;TyxlBMcEy;^&bAqG!b@V}8fv$6SkU7yTtFEow7BK-P=KMC64>g{273 z2nGl~^J{rt?pv;pJDyj}Ys1guC-Oh>{MQWokaTKaowa;12>PU=e0J zGz$_y*ib^y5;zy21a|ww{4$j7&+_Ymrg^)+JTN456`F(Dhk1m#jWIzZpzqLO%nQsC z%rEF`2p@6)7w?o%4AdXf9(w_M9`x<4m;%f_s88s4V2u9&D)o)`k$exmOF`t!_i{bO zt_+vbS?S#4`skKw*BkLKG896?>Z%e*<>*PhLukHEjQ z-e>itp&QXD=q}%O?+MRO_b&G|_i5J#=U&Gydl87VY75pf(p+dNH10PX*B{s2)_&7C z)f3eeb(s3I`h(`a_Jr=4ehY9Rxb_OZu0pSZm4 z$=HaKG6GTIb$A7WC4MH3C7l9T21FhY zFiINv4DjvupjJ}{(n@JV0b;g@mPYlFe~|7IsYEQ1N$g4dNTh-*2|)xVjZ`E5B3qEd zND;zC+Tjrx!W{x8uA$KCpfKni z`s_MyP4{YirLru(^ycr+zjplU`78J{=U4bI=MQg*q~teXF)jlhqjPcN_mb}g-=}{k z6kjR6QM{}8K`~tNPf4$z$9`9p9xF#GBdX3;ovHd&Rarfzwol#py1(nkHZ(N!Y@FIu zD*4rXv*ktWa_KuMNtPfxAUy@V*uR_0B}B=QrU6Zf01tWEv;oA4my*tsm6BD>t6Dm@ z7D|uFuPO7@zcu%DnZ^jqZ(9#%sT=jSLp>-2qCp=pOXC3a6nci~hr0uhLb?+70j#n! zC63aU%p>C|L#YL{2!K_x7|R&r8S@!>#v0~KmYSWzO$K`@MX+0NLvUI!S8!1LtQ z6Lv1VIpVT7CTdJnWRxUI8a*_|6VnlRmym>2iJg+HNku6OQvXeRpME#vemi`7I4e49 zW7gyBc{%3nA=!(vUu7T7UX$HBXL*OTj?#{`xwuY~a&PCZ?L4~cplc$sO0_$~xW4P0g9tZgooE_-Apa;@wH%X&*ApIlnum zcNm;?qJ32Rd1-y(*G5VrA|lJ<`X~QM#kX79{#-jKIW;nmx0Qo)Rs3i@b< zlzEJ650i*X00Mh7(iUxr?h@5O93#3WnjjK|74kN6=klHj%;Aqj#iBuCocKu?k3W$& zjvoc$-azhE`Z!_{?iX|>G%Iu$LSPhBQ7^HZLOTB=Z&$||{W|$>DXO@sNiw{$P5_fo z8l7U(M&-MfPEDukgf;zYI@WO;ze&zD&u__<7pM*D1l4lo3e^zJFAZOnA=k)JWiRD5 zX@cZZLw(a$=|y=P#axv}JxxiK)=H9_hBs*>51U1j>Gh)OcNMYqwZ*E|=c(r?*z(mA9QZE&9-fDH;zJ|~{VAIC@HP|6Im)Qe+c-4z16Ap6D4C>tStqt zgQZ~rOKy~@Tc=BR%kD`Rv`ml;YP!-iOj0Oewses*)e=<~`TG`Ob8(Zgp>qSK@l-=> z-J|N+mFLSgm+mMJtAeZFRqd`ETyeJ|qq4Y6^(*+p{Dc0}`1AXpbCuq@UsrBYn#~4rdn!!+z8@&u(^*T(ey}fF>crZgTu`5AgXwUaIm>3(f{C zB?Y)~PD5{?KG+XM*(I~{y_~f6`iysJAKNyi z{M)u&+LpAJsl8J-rk1C@Zr45wn`O!T(EfdPn+~r#+~^R|f!m?D!_1Dea$~#9=uy!7 zY~O8t;XaO@`+8uzWp}1_WaYfep>$l*VNK4d4tI0!c53Yu+qrueS+{+8FY}Um?C);t zp4WYLm*kGuvPQOV+x}AKg7)t_9PRq12dzgz*UC;WIv&Uwo)yZJWoj~WGLL0Y(>|sA zmC}?lDy1N4Q$lw9k~aI}&c!rD?iAe$X9|_vLiS&*{Y-@U3#hn;(hDey2=j1k$P#!S zXboNrH3Y{5e)^9Gtbqowj*FpGJc+Oz$wXwxD#C4K5^i_!w6DxnD}93~lM(GNbux6Vz1IxU#`hYp!Nj z3#zwNN7M|cHPmtIC)Ql4h$?&XqTZ%2gF2zbQUok5%9Ozx}W3VF-B;bOb z0~7tJ=w9zSca3whbE@-iC($*`y~4BC8|9PwTD)sL$KA)>9o(0kInKv!vTwaN&qMOw z@;&w?d-mIN%=?T*#uxxdhg(;<0$@Z#qN~n&%am!DuiXHap@-I~sZgmDBxShzyS7Y! z+wcbnk@lzr$|cJE>goCvOP$>dT9sVqEq9{ty6=zYx$`|3ZZXAj!P?EiaBub=K?!KF zv#ptJQ0ZcUpz4n%$MnoL5}3%Jcn13S1ylW>-KW7f7dmB*3(hAVTks)RH#!(&b=%+I z{R5hoW1dmI2(+_T@7!lUV>@Xp2QsclX1(E;cAx5lVt{gjy1nj&p}|nAJE=|9{n6#= zPHMWSRPtjowQPZ+tLlu#rcXEJ7y_yZvKK9J&HW_(B#AAVAZVnU7FkX>;y`h~$Gy$I z+(Nb#S!S83=DGHL-aYNjX!Xol*an2y=7ZjMjnO>?|58T7FJ zg0@lP(&KI8-2`+`a6NV_Qb*iHSOKeX*YRhOT*4dDW7;&Y@#!Z=w}Z_K;Q+@uXt%1IoW-Fh+r-BrHaH5Gu$8jNP0${Qknf!<$6b$fMB~Ap6OV zS}U%I>>2w#enE0k8mS#MJ(MJGb3PUy{g;>}UKYJQesl8tw&`g@+MQ}YFO!-+x9yyi zA1S46Po{QCPi-ID;ckcG>_u5i+IMPqGVN*ViFAEtN!FG2r!&T<3Q}$+-%E)|)unyT zD9yN>mebao9F-zY-Ih_<{#5pt4%|+AJAdxnzOyrTN(XZLnQ79LHf__=Qrh*+l4Ps1 zuD1(MU6wdH&KC7Gazac?e0VaqZB`mCV|vE>v~He!mmFAU2bqTfOLRA2Jicl?Z!WdA z*f%=c0zz(&JJadfvhms z@9<9w-VS*nDfSZ>9h8WyLw+C*ou~vgp;hQUbh>X6Xg?BN_ne0v18h|$ivE=PzT%whjr5qb zK$<0`O1HEQYl&*^Ey-=FYiO-E0dl4y1xdV;a4oT2;<<$9NnKJK)0C;3Q(h%LXwwi) z6aS8QCeDh!6&(>77j~I5l;Ni=BpoH1iMxnfk;OO`bS;1lv=6ieMTf$_(m&k)$G89zFQLf7c zv*Pk)8Yx~rPW4Lnx6xyGqI;w+P@t_v5@J(c^DbFCWr^~d>WBsiY_+SD(_~LT^af+5 zWMbuE?KChS0;?UZoF&bae5yCsH8oU7hO~B)7PjD;jOvM8HXFV$-0AK5qQPg#wetcX-lRTp#{O&rrx-86NCVq0rmbKhob>r&+y zEl(#^mnh<8Ijzr|Ur3fT)1(XJ)$(NdDcK8IxT2S;R-K~Vr>{1NZ4{@$g?E2*k{xeV zOifqq5Z!m3UAI~v(jPSRFg*jxscyDPtI*1}^aL4kmT`;Wg`vba$h^|Zc4WE=+_|3b zo>tFiPnV!|uX&^~(J1!z^QEIL=u`iS06mx=`hqzIgRy#qpTzB?4Wv1wnE*(wCmKMO z`9rjmMp52VlBoM>A~0<7KJz~7273wTJtu*)h`obd%Z_AsvlKY8-3L7Pz0bW*+yTcn e`vQl^Bk_j;S>6gAWSyUROo&6*!s zkifv4{cStx*R6BsRg)Od;C6#LO`17BfrKE40gjGWAZXJN2E>F&Xu{a}V@H7JRJ*S` z8+%%W=S1^@xc^QlgVmmEL>lvkb7yeoa^;*%Ru3iv?u+c=A4B`0-vnFn)p&bsJyt`G zp~m47e7RsUXD}R#O!SeYKt|tePWASt>2^)p& z7B~fR!bF5gjqrr17CnoMqikdaKaX9Xb(3jgG-j<}mva_zRopJTIcPfS$!UwwO3YG{bk`Jd#g5!*=6~i1&CCY#Y{_7>(-%U-9|mY4SJSP@1Al zk6RX#LFeIju$I^)OiC;y4FpB>2gh}MBc4hOAvaJzNGmo%FbT`Wj_}9wC-T!#2Xcwq zoO=R*K+m!wC6N8b=M7xO>4|Y zjqUWWb!FN=>eZ_KDz|#MZoXl=ZigyWZP518Tvk``PAv5&WrwuAQD&V#N#u2${=zAM4ip=IFQ9@!Up7M;ngV0faPqEjLeV-e>xr;7Cl zZo-+#nZg~*jp62T&vFKFdT?gLL*Y!$4R!+iD!V8AAG3maoOO}ClxGz@!Cv$4@iLI> z?6oX}u`1Nuzs)~7I3qMZxGyj%R36&u?**>txofdY?)+oVax8Nc+05o!rabd<6JmIx z?`a%iN;Jmn_`15rp2l_BrmDBqYpWI2Se3X6tzKT;r6yDTUb{x$N%u!hYL;l{Ym!uj zHFwpEv~#p7%`5$5tI*lR>2cllX8A^VyLryIhBzBL^IfH$HQs~X`hiiA4a{?#jYxOy zI*x+-jeU`MkMWoFg1wC05dH^l$Q7ft_z(CoSZxBO^duL5j)r+T0)+TZBop=VeHfeg zO{LJ&h@Ds*9>N;(tw>K^61p1wiXK4!^4cMb!Dp4j9f%C(-vMj)Bvyn?;~zpkbA;^S z@FSSVuES=q_3#PyHtu> zlFXgS;c;(qJ0mpOQE&otV;H`PP*P3EnZzfmpU5TT&^yUAvM}Sk)g;hC5IZ%%|{#WacO6?0N{Q^FE25jll<)IR(nRu`Kh zcr2KWQFt4ikDV6m!C2USejfpe&&FVYXva|v8Y|EV;`!OU%}9_d=9=LTjKs(%-wtxD+&_r}+jXzCgrpD%2(^P}VWVA#r`Ia_Tjt#LtvI%T09ZL6E&tjj(r}EDCuMUk1 zuLyMsCIrWa4u`%6Hu^&WYh)(N$jL*_aJifkxHszt17@6r+CYwItEeQp7`n*p$L`C0 z$hpsEa$j(da^`VHasFYOI4_ZGF2*intz_b?M{qZ|l4)cdVo;2AVV3{C?@?f2Fh8L6 zZSbuRxC3FI$@{|l!*kwU<0^Dsatw8Rve&hhTF2O2mT#t$Mz5in(PP|Zs$;%riZz<_ z&2*hKLDekPZFQ0+z9y-9iMpL`j{z~yH1#v8O*~71d6uQIwWZZ$8DM#2DYqB7A9>^a zj^NeERVahmka>;q6p}@kL}xL&F;+oBhL%CFzO&Y`KCm{z@7VLXKK5v4UFIW}g9$P6 zqW2?8aIN3N3qrxjF*p-U?6;gkPJQ+nRwITXyg1mzzsj5JIp-vuw7Z_WgNyHWxK27I z*j&~(HnDAwd4XxOX^pX!;go)&exM%L9asNQ{ZT(ucT%0J)>L+@Y*^K@YFOo>%2kz* zDlS%Rth`ZGQ~k20pK7`4swz=)N?WMyr~9t&X#8a47|RX2&2je0&I(tIyV7ZKY;Y{G zH?sA#B|0?D<*qb$u6ItLarkj$MN|cCXYFR+WOwBt$T#jU_Ct6a`~VhnCUU>>3bElN zL?zO(!s~QjYA*2-8;6zQC&}AjHRe+VLR5TL^hUTzND94V8TJfah#0sKlFQYx-@>Qi z74T`+YPc<;=2KXRKLFjzOXDRWd7OvvG`I$wL*SEa9j6&Gjkk@rg*S)S0ZHVJ_X~%!sAnC4zP6LS6%8D?1bZhus6wqdWME(P_LwgoVuFB^7Fj*~u5MLrt@c+}SFf&7sKr{BwySQNKGtB?A-cQTKH9mu2F9>? zu#I%=bKZ9gy?s2B-FrNwzenJUe~(`g*c>nf_JP&A-P_!&@+$nB0-DesXeVns+z@WZ z%49x;6wy|ZV3>@oixx5Vvp&Olob|{ZUTfZZBpw;U{Rz;bfIW{hk2{BBgD=7-;J@%L z_E>gz*vXQ>N>+cy(P$*JEf61k7aAXW5G)A}4CVw=LkB~n!p|cQBAjqS(Cok9&-O3& zE^zCdKO7uKFZ&$3&hgjT+d0JXkL{|NFfB5+Fs#;PYrkrSYOiUhXfJ75`aHuC!&QAV zy~4mSdCVef7n{Sr$+6b)!ZFTq!|~O%+=F@3e4YGD{GWU*UyiS(|93zY{sJ{-?qEsS zQqF2lBX%mQg4v$6jyZ#I4>|G*}Wi6-*12h3Z6pMvzGT&{F?B&vzH>)Y!t7o2Gcf9^Diia#3Z}8>%ZEFBAy#Xgj?~^hexB z(oSNM9FsnkOcTEn6$`h}TCz31m2cpUKw`Q3xg1^{bQr&t;6A@UO7nc&PTW~sFEQ;q`u4v_s__(!N|C_8jE6cY}IqtQFe7jRd02p-EG!uh~?!FkTf;}Dz!oC
u;c|X{FwT((STL#koJ=9k*t=e zC37ViVjV!zakPfYqHhYjN@`@N)%qW>tYLLE=Y9(ArtmK1ajMOOYE?X%3Pj*bU zN;+G-PI#0?=?Zc!u@!%Z^}((PX7cmV!)OsN7P-f{!`=sPXKiCn1$W;DC=-IA2~ZEn z9o-2PLeruhBVQwJq9-Hw!_Px;q4uE}p))~c;Jh!!H`n*uhxljtH~N?Qx_VkTUs~&$ zHyHQmZvg$#N4=|ts5xH4P>ogvRc+K~RqJZ5RIji8S#7WGSG}t0U{!K;r)qEYBh_I| zif);HqoJGep7ENog>k%LqyD6xVXQXowam4d90Au+_Y-$J&tuO%Z;HQfpa9(UmqXm} zoluvMGSn=zIV28?Bj2JWjCNq^#=`%?xnNdxW7mSs%rQ(K)5Yovd*M-FRy9W#@ctr$ zkdDY}WFcUciJV>RcW^zJgzez>4#9ijW^f%i!dk;>#wuaL%ms{BjJeDY%(=`e#v8_b zh7fuhc@b$Et%|TBEy5#0S-~y-CEh>oQSLGBVQ$3J$2;G*#UCH2@^$vqI3L=3TKkyi znp&Gyna-H=t>5ew&gJgfp8wpNT|1mdowHpWx7Bsdb;Nbj)yP%keCV3(x#@lF8yvV9 zY#Le?stU=&HK9r13SFU*k&96uRKV3Fjln#ks~A%7$1HMh|FFv}=TjJdUU$ zha%S_9izRVO$-imE29)@56z4&k6a6vg@VCp!QR0Q!LpzwSPFY5m7eyXt#pdYk#)`6_*{03vn?)CpDuZ-?fG@kr{ zhsY+{A?_ymBIzo9DQzIzA-gEsE;}M?EE_Ee32)Lv=@?o@kD>3=!-SuNC3ItI9od7V zh<~wc%qtKIo}$foKe$S+9`KMvVKdo#fFrYq-41BW9qbRB#Yi*sFJFxP#D?O3@C>3Okwn<>A9xYollVi* zgjYqY#7`uJ(#FzN;`5?IqTwQzs6_Z#=od;wi-qUuJM>&(T~UU3i-ax9i&-WgCT|&I zl+KY{5XB3p(^)|6T%nhW3IUQp@(*&CypjBkEJM0ooGhxM7g9CEN&EtKT(FB@iY`QJ zp;ve>kX3-SK4x>-e^~9AG~*g{3YrWxg0i97P(7#%I5Q!Vv5naQFy1!I6voJCZg^zy zhoAD7`EL5|`uh3b`MddR`)+#{xK`VrSI#ifYu6R(R~yWRj>av<#m3skB;z2X)%dSjYAdxcLZsP$dD*^74n8h@jd+Zf|L9nXf=`t zJhV>mS!PGZH%JX_gUX`aqT3>S!skN$gJ=9a{}g|zPwCz3-r(x&{B7@KdvED&5nF>+ zj-BCb?H=Ws>4^ngNbS5~A7UM1K5v|3m}A&wd}Z>RPg~1uH*9X}A?sr6X6tL~0_%6n z56b}SU2A*WVEbRk30EVJz`Mh{+_%!-5@?_2!Cj&2;m?ut(OhT>^g4PydIlO)td zzDz0W8FLOmvWv`bOeHgx(TI`3{L1Xj;=*&`18{Fx%bW$ZjC>6J7px3y4HyH>Llt3d zq-9hZZ5DYHk_TJ)+j<|nkGpC)o7kDwQj^GdQr}TGK-)|^QJbmVuDPu~pz2%0s=isd zu%b`-=CZ1?Ru!!(uUDF@^fjZ^SsI4cqP?ZF>6+w6uLhX+SR3_YVAvx@nhbp#f(C$K&2soYUO6Lb*V#x!^p@ss3H=gDc*VBtbh zUs1AXoalfkMYNSZMV1g>h+%{kuSHxTrU5nMAm(EG_%G4_c&7kX$a&k)?Sf>$TgH(- zN+Db%Y$>duTL|q!ov^160`qvhFirGL)L2XaI%k&1B>TnL;@P4TfYqF%2hnUAq4Vff zbdVzGqjW4?Kz1klkSfBDufYxj7ITf)3(4iAvE$hf+2=W}xG%X=dA;}q--Wc`#&I6Q z^;uO6oY9Xln3=`;!a4vCVz=Y;=N{#vT!>rAJ&wFVjv}LoK9eYAVShmyoxpL$rpTB+M0c5Q`*7Btq#^Sy#m}C8GSUNK+aVVfmXFlWb>9 zuA-5$y^@S=A2%Rwacn*1e0f65O<5h;aH&GNNg6NvB&!v}h)Ih15<|+5#fV}K%i76o z(w~wOqSbUN)tbCQj3J8gJ~)iaFeP8fyT_fyf!GzS$*gUx)?kfI=3L~oVD{|2NZmT)9U=_m}y&V>1yVhP8pW!pXv&9 ztMs?@4fT(7eRbEhPc>DldeteF-79j-pOmgGsaJBfmA!%UQe9=ePBsweMG@{%Cf+>VH5n74RbcIhq2Aea=53@C$BWF1&!~K@O&F)3bz|g-OE3 z06UjZN2$fs4lrF7)4S%;&?E|I$ zPyUC2Gocre2hoSo6#ygVu+9P3;v`4L)1V#rN6_Ku8B~B?MGkZ8a?fx>NICBvZwPvq ze~tf{x0k!0eVHX?9)+4jUq@y}xgeuh9{n%UKe96-jvNRb2yPEz!8ZPU&wFQ{{jIf+ zrOM)rYs(+}vs~f3zsrRW9)C*K?RXfxVwLf&% zbsF74y+_Y7c=i1asit3+*0x@@sO^}8=ei2+j~FM*NxAY|-<;Q-Z(NH#W-r?p@#K3N zdOLczdN>}xTM0N$zWbVMuB)Yclc$$&cR&~F8O>+JvtF^fz&dyqJIp@I$>sJyN_nOH zADEojK_rk&%0ONtVu=dEL9VBUQX5Dv$s?x_iTDmIiWTCwz$251XT(=>HaQ-DB5?Bc z=xHz;Pa|)6@A)~vUATmAC+3iQsBLtrFh#UftQFT4w-EIfUZP*oO@$MMmxL2Uu$V8N zC~7b42iSy!K1Tf}dy%`y8PqFE1lG}app*V$Lj)`Na=~0|GbRvhN00F4A_h(yc9_)- zzQaa262KCcaT;?EaJsYE@CjBrKyokZ47{DQ2~4R7Z{+%zZ22WcxiUMJ7yCk~Qg)49qdX?x6>}$MM+`S6I|h;~6q(9N z%6p1lild5R#SO(mMQ25<;;=j*>na&3{6*fxv#@l*QvOK(I{pi^5_t=*=Q-;Pb2DQo zG$v|})QJ{FA!uYY6d4!Y7_E-*gKq)p_&0#cs;I z!JX}H?qu2*TJtQI%qL6(je8AK4R`cL?RfRw>I;?ID`o?X=vQ*9II}ph=;q&ke~NxT z`?H|1Q*lOVo$?NqsnyGBvQ^b8wq}yHP`A|3+qA(l-9F!S)8p|L`ab!;2et&8g`!~} zw2M^$A7S6+7`TrCn|y^l<~`=eVr*==V7}m?V3(jhzXdPIWh29of02v4mHge^BC%UUi!pIM@dNQe$vEj$=@H2`@h(wg;dJUa(F7lk zWnxpYx!7vKH8hjg8u`xI0e@uXFl!haATrW2SO)k-hP%j7+y1ZZsqL{X+cwzR-U``p zo5fOYzGk+WXIh$Bb8NTld!4xZfcvt$%Dvum!yE9u^e+eu4xSFa53KS}_8suH^R5T( zi_|yH=lAXRYy6LZbJWiN$Nwm>E4V78jHE&980Q%ojF(V8bOi8<7{+?$CAcnUCBXDl zZhvlsbC|P=^N0g+mv9tp7W|mCkTr+-65>UFgnNd$;h&+F;f@g*U=>XwFG9C~9=RNx z8Ds`81Ox%2ufKPIyT8+BSK52pMp_qG_L|q5Fr!Fs&3BJf;G-I(sswT&3@6LbsV>owguMy z);MbqtJIccXE>4^9qlYzjM z1<+x@)eka8Fo&>a!B5$nxr2CP(SyLFFF}X$V+9IqAf8C}pq~pN5h=PXYAT*5wu;Lo z<GO0y;cH>3a4o=(Lu3zP4E|RD z^IM_o(EeyF?-BPKr#a^qo5P+1Uw{{~KLNk=5d;e~Nb(&ntOW>M0AC=Tu&;x==l~W`^pp`m#o+ov)7?c9@!5 z5S!cn%qenD_mI9Oeyx8*usoazonS0r2AMYi!${=BaUUW^bdI1v@I;^onNMB(A_fW0 zp;6u#;5;|r<)S(KDSRC|6s?UeLf7y|Vetf)+(8N`h#Eu|69V!x;Brmr(ew*?g7BSi zuh2<#IR&yYw!%tG{cf zGuz2_{&EyJ_BdnR0ncjRPX8$2s{R+W2Hyo|1v>?|1>-{np?%?bkyFuXs2O80G(K7< z>WOTOV9{OB0meDTcE%CLHAV&C>tUeG*l-R!5O|;SnF#9$t3RtgQ^XhnO^MEo%!tg0 zc7WtiMwEycL$8Crz_Y-bK)%195A*JFdz|kalw-R6g6+E%2K-{8Im7h9FjTMCsx>1t zD)koiEA=5wxi($DQ2$ux0&GLA*`#4<*Xn8vhs`0&Dhp!yYk6T+SsU3}*_%7II7kQ2 z-p`h6xGiS%Od@uG0dTGdkzmVA|C-;-^lBYM$m%-HgSOb zO1=hj_7^pdZZ6y|Y9`qxJum%F;up7(6iMz%c8Y6+tLQ6~lk7l_AeU43LCW%!9!7Vj zPtx~<)5T9E41fsbk_(b;^uE_)LJN z2YHy#<1_L8_&WS0UIzRNHt7c%Yz$RQI*9cILLMfo$x3P;xcaf8)uNl?6zOnTQOr%n zzF1-WmG~9$YvL0U4kffoNR9sMQ#`vwQkq*juJlG}hq6xP z=PKG%4Xj~mKIm2&$5=Ypw>a&tCGIV5le@fU=6HNhgX9QrH-3c>Zi2V}05|m+mh&G@{^Md?H{vdCV z8_7!Y2UP>6QWw!&(Fai%ac%Jg(L&)IdOUTJ{6x&aQ?T}c`&~r;ATQbFOgZptI)_t2 z14A=Ir3^*+#HcTE}QZss$l?xrs$%u;Q2+21%_u2}a( z*GlJJ$0hr5`vylXpyFRTRsjdFtK*qtzH^rAs@v;n>)Qj;kILZM(CyI55E*(A{28nr z$`5r2caC%gDeCLUkFW=L%AE#o2K{aB}S?FxnBJS#nLAomSTpUX9bcVcS8vZ{6K`;d4poAa05k8_;6f>(v^J32h4A49pCi3m`#J@ZUhK zz)Zi!o9Vge3Of4Raa%7-u`%9oP@kt?qt6BR+H!SC^_PlQrP7k$#qWw+7H1Vr`8(-1 zS}^uk#m}QZy+6x;*$bNgS^M|jLSxaSlFOy9%H$Q-D*vfItV+|C>RXs_%O%jtir7Bb z2H7_{sJi)FlI^cKG} z)&QS_o3P)4B7R4{1oV6ou!exm`7sCf4}OH;0mr{PzO{OG7iLAY!9QK)w4S756z=sx94b=peiFt@{<_B z9RB?v%glAnbJ9+w(_SF4w)XKRjYx9QI5cN*>(ej3giwWb(Lg~e+5V3AvMtS2mT z%XG^Xi^sgo6k|LNdMHd&Hxp!X8RqJH=x}YkTB(wPPS;VjQ}qGlDjU^2Z8PvXBMhI7 zMw8Dhu{O4i1nBb8ndvrpM)^kptoQ|W1sOO1?fc`A_Bx@@(8(!%pf6h6}y1zGE1(nCI{ytGBw zPGlA3O2$hwWc>kl3{vb-isSmmFHg`W29ri6H%UI1geLt;tWLBi{zx2?csn5`p;i2u z*rv*Ag$8K-?+T&vt|DHMFP{T)%%-wEvgguvl20O~a2qv=tVb-uTM}!@2x$ae(quYQ zv_#^P&X>)V)s_7Mxb;%9Qqoq+0G?xo_^}8UwG-~9mB2w=MvoRg0!e9(kSBaeodIzm?2;YEU1)M$w^>0hySpG ztV1vtq#r)`BWS1iShHD^nQs{#KuY)qoHv;d7#*O7(VdZ-;ha!OU@2%dXaXGrXkc<+ zVc@O*i!Z_V%X`MV&O6GR=uH65g~hYg-P4uteB>BpH(C9bkCqIp-EzQOV*IQ3X+GCX ztomBfs=`-sxzb%BsOVPltl~n2reb?#pz>_hv+8v<>8b!WvX#;UP?0emFNbXPMYJ3+%@Ht+~# ztIfFij%lW;w`qatr)jYHthuW>$0RY%GaNJUjUA0n;|p`KHOrpnxafT9zUtlTllt0t zYdo7hE8Trv#ZH-PCeU6@oTD9M9q%1K9K9U#?Wce`>1Y*Oq-Ku!m-)4&%<|BD-4r&x z03PsYv&?eUI^O=wal|>##q#L9Z2t^@rTkRiM% z?-+WKUnvM-Rd{RSCXq@#2VI0a6qgzZ-0K*sJ*gnykZl25_Yn&TJJAH_&gJw$dL_M; ziX$BGD-#|hx{}(aEKHf6 zoRAcsn3h1qe~u-UwwT(oKGIs!gOUWXLHJ(SK$uNUAO;d1A{D5E-NY0kN*pB{koiO+ ziBT=7@8k;dA-SJALNkQdfadK8l8vE69sCS71m8lCpbhH6jtNEyss)+YQcQ~{QYG{f zVIF9oF-6TpH-rYdKV40|r6lxVI-dSbU8i!X401ER5~QmK@ay<({1(8JJ4R!f)29?Ht49?4IT^w=b`Bpt=Kg>7gf`4sfiBX}F|o?8)% zuz3D^UKid9WDj>F7e!KeYTg^(0-m0C3nYnc(f7O>UI(C;1_J%G0O^I?1)X9EdkCvH zvj=l0vn}%!<2wUlZDfsPW-<0K7Bg!z3!%f&^ys1Jhv;_Tnw15I295q?AL%{k8Rc=g z+3sP^iS`TDOv`B#$M{iC>qWX0&GDLHRrM-+RpwVZD&UI5ih~tYg|fUs`H_mtRTZEw zoCVUjMTR=24d&yP=hjAci8I~(&@;=M3K&IYa8$?`oD{kcZX5B36_I+;JJ41}YsM|m zO{xvOiPQ6T%eyXTDP)8_Ef%M}|Nhnd3oj3b8i8x7ZyxIUE7U2QvCztQ^Kl zXeAVlUXP3j+e0J5d&0I5Cp0j)E|BAY>{WUexmciw`^q-Y>M@Tp$6B6RROb1nQX|7O z(G)a2Gw-%+u=W8i3M}6&i-316Ftg3SOxKNP4Hm;|;}c_!v752L*xA&; zbl)g4)w0aD3GKUVqivmRTY&e0n-`j#rU&L{=CS4_=1MbUIb*(W8VU9ZEYvsAJ9T$- zE%hElp=rH&qIs`b0nq84g>5}-726v->$*{INB_Ja7H$}67i|LdX8dMugZZ4vTqRG& z-z1>1v6u|p+XsOIkxvDv%|eb?Bbovn&fUPh{v{qJB8A=Po0OV7K+dI}(c^@*geO7b z`Gw9F(!$sDK6)TsOtqtS0OqlVY(t%)9#VGdE|~eR#h)bmrFPi_xmZCdUdsQGkC(TR z_mBB1y)PLnNtWyu{}wrfZ|Lb%bFu<&hxHQ}1@C|oI3{=_=mea{A%gmXkAg~UF>b{Y zu`KKawg7u7Xeanj0OJavF%A%VaxvWmXd_J3a)Z7=xo}R;@Bav!()FdJ1d7PuP6^FCn~Qhb7SAf z^5Syhxbg22t|XmK9+Ofxt$#*Z=BkYG=?l}oriN0UCx1@*lhh!&CW)0aC2?fp+(bjd zqWH71HbooxL0P=COx#GECw?wwi2o60)7`1nAnh3pjwG`03Q9^hpds*q=2GX#9YlMu zA*VplL(rF>ipKMrAR_L4wvBa=@lSL|q;7;AE(wkf6^GYHMn|TAa?-nq3&?k$zme~U zC)a)7x!j?*-?X#r>#fZ!UbD|EG%q%4^_z5Q+H-1HwXu3u)w0SV6`#uNC6kLiMV*Tm z7H=!gDnU#4mzI~-D_d1oQ8u#tZFzh}c}1X-U(KjdsyeDAnr|APwi5{DmK*Yo8RiC- zyOyEWY}+53(q3fSYYPLzy_0>BEy*?(#Fuq#uvKR1YcW|gmM@l_)?(XtyUcOl-q_Bx zGwga>m956s(Z1f1?h?2+xgU5Y_;&l#z|eggNDkfxK-4TeD{?uy7AgkRo`yyMYBe`3 z2%QKl_Mh|p^zQN)fGy{;E9|vxhb$e@a^Z%gn1y&B1s+ZYnV)o6Y8OOGB&KvcM7#hW9|rILiskH|qg= zlCzgV=(0|bfks;yV!TEvl{^7nXuh;$6 z^$6Ij;~mNN8&;k*3dZ{h%V~?-@{jeG)d~ba!1l~u&#}$%85pop$0X-@=RD^hN622+ zKG!zXN?CfEdzd$xPn%Dft>!0_ zWrkOVUxx0+SH_n{(pahAs~f0QXe8>}DpFmnR%;Mlw!XKao^g%wh%sWkZaQjqS}xnB zI7ruO_dHJz?Z04sDYJRw>DB{46udb9U%8gjRBecZOlYwl-I%Jpz|aupz6Sb-kr za|Pe{6Zk=NKC0ur;!Ofk@>(pHI705E7SM>Wke1UMfHk-Q)FQLNa{VjB#JePfOdg|+ z`6@r8@F}vDE0q*5Hy6fTjC&bpje8nL$8C-s8!L}JtsJjh1Rm|8)C2MHUQwy|PjO7~ zSCOL3Q^K)|xC8OAiL;Y-ByUKeQ%h2+k~bvHNz^BNO8A+eOf)1y$$yf!q)1Xnqz+1L zmpU@_VX8Q-MOweKWog&as?+%C>FG_hyW~b^wby4L8KoU&l`^$OdtIFP% zt}J;_^!0D#cc%jF&vrkX{%r9x>*toAy?+({a{fXJG7H8Ryf5hd`_ym$??!*x{8{lw z^mqK<{@^V9JGAhAp}6Q%QAWuF5S9eW>s3yx8d}Y-SzS|JvtG4aeN-dS73h-!UfJws6VQ20l9JqQ@{L#IR8;j-}DNI}FLxfW>>*&MzSItHSTbpIINJ?}-& z1@|QPLU$eaOINY0hx>;+*K^SWpwBbIvjx}ye;gz2zpQO6LDOv0G1D;fbjuR!Xq(Zd zv#qw#wsNc6I>=_SZ2<^&$+5_}7;KYh==QscfPHbxbpaGQG0%Lj(BCmw7%qsWF=jJ& zu~1Mhea5n~{(&#U6W})RNEn9uuxhh%Sw`k<<{(yYct3jzw+-(xDiIJs$`%W@^KHC? z$S`h@ZG*SNci|!IkL*rtI|x#rKzG1Kyt9E1zQNwMp2sdGD4E`|nXKj3YU^I>JnIhY z5G%v_(6Y}`V_5`@$A4{;?6>UW!6qP%Z3cK`iDipW1n+Dp>uWx}9dP#;@tD9k0y?+eqSccXbQ( zm-T)0YTX&#bKO8a*Rajd#dyec0ASBF^D%R^xhuf@f35dy%N&DTYWHyOHQzVCD*%T$ z;dWqy;1_5Fa~RwJ*o0=JJ__+S@_Pz)Vl9XoGDv-=X;F@Nme?g)CmJc5FWN0yE_xvn zONPoq@>#LnOS+nzk-|(ll^mO#pVT{vPD)E^m0TyaX}T|CbgkC4Sy|z%f3r_z zkIHVGJuACGot(PE>kX_Qso%Xpvj$)5cdfs?Uhlen>%7R$%kGga$v&J_QoE#j-dMr^5~y<&;H zbIbtQd+AtdiDbRFOxRaAP)bJJQI8XO!r$L^luq_>MsC7T;<$!?S1A~1mA`V1BX4dtF|?)V}XsOt5scUDy!?3*Dd<+ z$M9?N_nBWteKfwk|8n}%n8#}#aUXqnc=h3^heIAVczEbR)&uB4`2Mj62Os7?GC!{W zl=E!eGwJj5FBGri-i~^A?tRCPmp={pJo3wqZ%x0Kd|CW?>eu1li@$&QIrH!1(y3KC zb*aH%o#9gWW(UtkzC!yzIlCjgjMIpBK+v1mOU@(zAx03RNt%3&2k=*bPO*qAVjq4E zi^J*&T<8|`4LT9s%^SfB^EUBQ@i%0gu)pM3%pOIjSa00K_~9UM+8=jTkt<`0tEth% zWx+*6#hMUJ3eNWSai4HCalx+FKvw5^%G~Ebl+n#+@fP}*g=UBAg?*u?;nb)x(g)Pi zw}k>hRj_IBnSVxLdq^MJA6gfdMZM5G=2Ye_#x@2USViraG$@n*1{*g(S(DM28DMSU zc0?_F8GeBH0F0|gL?#i&*WgzLGkIOuYKAq6hP(QL&WE-{Gv9DlS6{nVwYfT>`cu`3 z%8?bL%g2;`D1Bc#w`_JrM)k&;&ovrVjry)?W%Y?FXH{7hQZ>5rXT_58FQvUpJ*CIW zUYBhtdsy1MWM2`#$WxqE@w&$aF>A+2a|n@HIQkCIz}Id zWPt?#RR6pH6&6B%#%ESKdoDbjHI5Zz-e;(wGhp*uLuL{y1}r(c{b*77Xu32JnWnx3hsG${P&Iw#mqNUMjZ^uQk3MJBqABRuQ8l>%^s$RFbM3 z7{itQP2i`EOX`(8tyXq+ui95?U#t76p{z07NL_zQ-7a+=Wq+=9F=JBg-Pz@}c4z*V zom2lvT~F5XY;m0_Sx9!b`d=INYba>=qv62D7n=NP)UV$1tic(+M85)ySuZn6uZWup zAA!PnXS|$;quE#)c3!{$cEckOI8&R zFUl@-{Gt9{Eu347m!_3nD;-~QulRWJ-J&Xh2nT+hEMOJ%_!;=oyg>Fh@2~97yWgvR z&o5Y8F!A^N-xmw`zfyiI{bv4L_ObV8_O}7w4t>r2dh84RS@nM0>wPb@&s#pno=9bOSZOVwS~AM>E3Q_xQpKqu z%^>|L(>PEenhi?gIi@|9bM~n&iT84#WyA_unLO4h#skKG%tWRd>IGp?3@EN|hF&sm zGT$-y$;54KaW1LJ~ig0F)gLL)*+p-y2GL>5z_MT|pC zKYW}!1)YR71QB{&q8TA2tBHyDNxT6ez-;JIl+QDAdw_zwGEdmL``uRFGlOl%fu(k4afrnd5} zZQHie6sc|7Zc{5oJQ>^eUcYnxfb(XbYrojN@~X(B*L3EY^{n+-_sve=4(C-04u}^= zorR!t7tuPw7k-5>I%0+ zFfD&_^Z89$def(;+(>v3cQ-aU#vVly z9}{K^y7PPSjXV^$9qT;36Lk~W2QZBQz6x7_wuOJf>!}oCyG?@@Xw)nZ_Xx*@Mg;o= z=LQ!CNx}AkJO1&3@gZ%vX@nR-gtLRa0;>b4;3!xi3I)v}MChmgmyh7z?)Sj@)cepx zL=;}uels-YcA|0+b-_olq-ldyFEotW zng+TAx&}u1HJ&n8uJf1OVKdpDLsj0*ZnZgV!yG4^K_|(z$GOCjVqalfWnbqka%vqL});#Me%VWzbcn9^cBkgAEWow#^0t0|I4ui|< z61uy1s@$!eS8WF@1*W~Q_I+N@H}H&$jDO+AG0*fL+>)OgdO|daX|Nf_8W$MR#@&WV zhH_X!`=h_0tJJDAB8^R5r1`69)NIipwHLIHq2?W^jn|w~)hhbS2gpuJm(-80C zTVH>)Ay?{eSYE%dc29MBmFMr+Dr|NCe^8fyP?vvDmw!-~e^8fyP?vvDmw!-~e^8fy zP?vvDmw!-~e^8fyP?vvDm;V>)@~`vuuk-e=^Y*Xv_P@3B#;L8U=QgtB@rqCK#qun9 zxhzqZ+xP}DO!H-z8^1UHfBkus>_y{jS*9Xih1UuU^GuV>epoBsZ?YL%8v7Zq8_A{< zrbUnpIAgtK-{{)n8SfnrvF_)divXV7@7B3)ImSEEoo{X5&H3g7=C9V1j$@7wjy7() zJKlZBUE-PNZ5?qx|W8f7ttP4l>q9RMA8uZ zlK-PN(l;?PnTP2DYJm6xe;OCSzQiuVPQr{u==;8}ncrlEIYTVsDxreQ4G{)m=}!;XN1hbx1sW26F<>A(3RwD z<~DlI1nlAc2zO*{WECKE*G1k$Hb(Lh<;WW3A6RNi$K4?GBv(`a&_&Dx%u;3xX8n+MUnF)?K^TVgiH+>QAX!-_@6y^DJldnTGK z*(&}d>?nLC#ERYv1A>JDig2&+mhg}W7j;weEM{Hou9#lYQ=_E9Hi9yk&5HzF;UJ+> z@QGglfF&XO3UeuABz-yUC6z{E^e==pEDy&bJ|bVEA7H2B%CP$}+u?a1 z68RWj1f6|60E_UbEYwxhanu^ra5N3`3jGe2&X;3)Lx$uYrYj~16NQO~weDR2EXv32 z#jVEcabGcG(bG|jkX+OZbS7pe_80CaP7S!a*;q5i1(35|m@C+sI2+D~Uqtwezl^hD z{a6`x1GXLZ0lE9>66Mqe;jQe@&TkhvW@AKZie|7bx`Nhx|*Izt%nfAK) zZE6MT{gn6f-`7-NDmJ`3|1PPbT?M;hPsOA6jUQ%zT=jAC$Jmd9KeqnV{!{A5iytn0 zNcgbveGtB$OXD{q$0u35YS>wI)rO_JOXCVV@}lLQq$2iydp6&s~-7 zVozW12;XwwUEdSmMDJ)f+R3mzGWRkS8dD(vdd4`(z|&)NOSHSSRoaodQa#RyHN7yE zn|0=0=H0;7xeL7GSRe@}*%vuRJ5$_AUM#RnXZmvkrNK+#Hi%ZpHxTQLg_M3##2=Y} zc!!`PmmoPPEh-<=fE|jzK}Y~<%I>%Nxg2@Vf9P2|frL zMTJobk`54ud=<4@w2Lq1lrXwb6{O#UBp8Q|Bj-@dX)7QU)0TFVq9^qQR(Cgi39bgX z5JxcS*mbxT_%8UK__25c{xN||3=j&5(Ig6a3t34%K-o;aMKjQE!^TfBd@W@(qtj`h zD2e2C#6*G*((nY_e(WktKKeCk2bzOf0Mw7inB~~f*mUei%x=t8h`nt_$Dx0qIOvHO z1vUy_LYPO4B8f-?fQg4D6cEY@Zo+zEXVOX%3$_&(!cRX*NF^Zg3xRagAC-cTgkwVm zp>v`1us^JcbOkEq^H64RroWr_n|r5Q?LO&g@UHQX3)VwczdpnXJHpDyUPLjn3n~^p z8{HMnL#Lxhp%()4=Ns^ijuMJslln4c;{UlLpX#9wppj^B^3T{#MOy%?knesPs772z zzd)Zs9uFJ+Iqvti$$(8NXSEBY$K(@~ zXw_xaIn4vzC|#6hn+m0BsqCS+u6U)MVmxk(cIa(fTT|;wQ@&mU3ym^;uE7OR*dc%g zmuls@FZxpkhw+J-0{QM~Hn)9+Q{*al1{{B#IRJt6`lp3%M5aatgtGiFco8pC`|A!qvic(?f#XPm!1CyWqd+PxEefw6JV5m73yATTCL`LU#yYyeaM~ z+XZt|<5b;p_{r^bF6~dXT1ixiG!?q3(ydhvpYy1*rR#zx z#edfS)qCId95BYCtjBC^oOI7B-<|+I@ZObSVHwr>c$h?g)wVF-wGV*Q`E75H;I?pP z@U^ELaGnB}+R-+9I%xFmB?}5quNBbVfSUc82 zF)Y_U(DpIhHka8^9=1Qqm*sq9HX1O7W4cD*P;5 zJN!@54EkpFBwjbca8X^9EqZUP6{@n_m?e^*(e2|alBT7VW)YiEvLmUA#G~=p*kTD& zGBk_ekLlks4rI2<&TJN+cRBBU?#$-1o3(0kAnQ)% z{%k|j56w?CE6*90$xlC@b|qb$sn2?y^DXCOR*MW@T1>hmb8XIsW(RZM_$*m^Vn$Y0dXvu0>vJ{D&o&Wd&P^GVV2^nlT@oiv5~Xo6nrESN#^yZF zYLkvjIhA}SrAf+>zSc98|W+KjnP@!g_j1V)V3@!-v3w{l*4XHxyLdX4!-H+{MR=o9xmE!2{{^&(| zS37&zCRs{Me+@*_7wZbw6W^oYv#=I10hYlyVRTR*2nI_)oN*I*0lOY@jQ#O=%xy$% z@Q81z_li#yFa$>iNZthJYOCG!5wg_1jJu2lMwNb`4xtUJshZB(65Sfz72QXDsWIMs z-h9xc(69u1|^4YKEo8LD}(PtsdGMs-YySAI||R?br|&}jkb8LeNY z?WuOk4%fe`8z&_wj;pfOR#k)gp+=+nDt{;YB>$rzDWjDX^)dYlGuwPeKSm=~u5C_YyLY4}D%=8-K0;L~wLuIx-8@4|N53J3WwZix3m zy#J8j7cwFrWArd69aQ)Rr(5vlOu`vK;ve zyhF zI5ngU90`mLZAQ@00!$~&Rk$B~i(C#95udw$n)vbN=ZW8M{oM7d)8C~v?HXvZb&d7X zws4psjnnki`c9@=*t45p@By#!Bz$13GiTYKxa)lb{PVm{SD}**w2&zdi<9V{=w1(( zX0gNU{OsxO_X8i~S8z;tB2-jl2r!o;p1}Uy3oM1)o7s&2LA*Pf9b=a8qLvG9^G0x= za&!1dkwDThdPdClgwrX~l+G#d(u%S+XD4JI%_3zRbLKTY&}4MBBU6+4IqPS(H;31J zSxZ-8(}F84vYHuko@Kqvs7t?>(Ijhi*2Bz>nJY4XWIb-utU0Q=wdspy&vO&=9_O9Q z6X$XA-sg?apU^VCa9q0?9S?V2+Zofz*?wC)RGZuaN0jMkN{9^~(6 z(XV-=sl2JSX^W<1Ih3qd>BZ@F=>yYeq^?ZPO`4s!J>f`PP4vd73nGR<#o53rW^QNT z7@r{k*oQWQJQF_!%|cj1_d|{01bl|YM?mf5(pF*GmkflH=4bgK7jIy$iU;UOzZ=k0T@YF za1z{P+-+PxTrqAgzBlm-`7UiUGoBs8sp2f;ykglINsKj&YNiCN97y3>VUeJeTg*yf zoMePq7rDpyGX#}_V$ou87hw)JgR_UXQ7}n(T3j15Ju#B}I>`}tDY|8pPB4ylo8Kzx zckIH1i3!*^QFMiPpKvBnxf=LP(Nb}a_@=O(a3}O~YIzpu{?rRxQK;CM_$!G=ldmN8 zji<->qAq}UEKMSaogRNTp;zL=gtKueF`FgxB>iK$#OEabO8lDiDcPCSEunAh;^>#r zn_|DmwM%H4qR+r&%}gJeG9WQNwn02oI8RtBZW;4D))mbcH}Vkd9rWXrjik-Qfy5AD zKK?nT1u6}ecz#Ai;eoJdFfMWgu?AU-Y=P_?Srjby?eVZY6THkolW;jgfLs^hhqn3G zdU&p>F1oke9}esd-U!|grUf_ouwb{_<6h~pdKUUGhd&?-P;Vk>f%R^?{g#DfsxpqU z46>IvF55EA%k{;YV&x*)0;#59zcjCLfpl5@xmrbytY%3~=W0ZC``T9xHI18PdGd7m z59ys+@;y-PE zHCC4YqW`(__f&Op{cPD#`DE#9SXt@*=UXMR^69VYs`vG}Mq?vFs;^sGn^kkS3Rl&p zT33R7dWt#f4=BrL@_@YnO_0^=R6Vwkh`}A#1HRiRJ z3iD7STgO%pQOcB=nj)Q7-^Db_^4n5r)){~4d+7f&B*U_KQya-s5V#*q3f}c~bC)>A z*lt?K*{8a#U;=6p9vr;mlXz}A4fZ+q365>9pPrT8L!R!QQtxE{*MJEwfEMHmpxOVy zHzhdmx3S|;zao3V3dBeHF!u;`V83dlte~+&LfVWrDx#igQz_zKPhrrh|wLG#u=RRte*6{ zltGmJ)Nyn&^Cs&zXED$$pRp%1Z_(>%KH4sNTPB}7M(78!$|gRJPp+5ED!b;9%MR9$b_9`}YURLYpH4kSf$|aI`csB2S;B|5C1G!yIJN>fLA>b#a$)tDv#9L!`s2r#8c&Y z>DBsr1^A)y0mTLBDtER4zyBOarkU|9Gr zq;J%0;6JyZdC7}NBvLi$B_*D5ihEWpfzeY%G+7c6zTjy%G;Rz2AW@>k5uFkOnND@ZBiLh5T8pO#KZC2hhNV5R~+tt65WafBoRg!i`V9Snr_ST)v# z_FK*b&p_Wv;Ie)3R=Nu9Z_Qpqx&FQ3o0(wSX-k4t=~C+v<8OU5P>ME~?^p!(qs|$w zqmIMY3Fc8Irs;=?W_{*3;6;Ye;P^TnIO>^hA8+#NPHCCC$@*%;Gk`QO^b0fs6;=MV z!BUTDTn)~#1&RZT+lo@fCB;EybD;Z=(7o0d8VgJhjmz~7nmejo#mmMQjfILHs(|X5 zdbp;Uda|Opu|)%?p-5WXcto~M`9yPC+e1^N8m=(QTFX|*<|`g(w8k@5tbMEPyXBiv ztG%OoqP(f9&~!9ZSR{@DXR>pqW1-Dqeqg#`F0fTPCb@FlHuqxBe9uD9NNZV3RR!SlBh@$jJY1SEN*YiNXculK(t@5L@1W*i(MU`kT^1F zUE;GiTFh0+UrFbfmT|We3R5R++pxks1Fxrdv!_+^fWvsiJgi+K|*0Nz&L9 z@oioWr-1XGJ)Ohjt>qo&yksii=6MM3#>8S;VUjUJ(N~c}BkmwR5cIY89}MV&m7zp% z2mXhwj@%770%(7OXA@YQioIxfr8EFl-0V|%7CKuwZa6Qxrn`>22l-evp2ZwwJ9@VzgcKWqOA0h5CjvD31o$(PT~7fV4if4|I5JYblEt6TC0|!-J;+$v(ZC z=-OeQWu0l=VSnwC`2Gg&g#Lsthc^e;`$C>6UcYywuPpE+G6(e)iAH=6j}Q3*Z-QG8 zxw!q5*{tI{lCTsihTrTv3O3ud~NobuENPG}4jyWW%Z^S)| zlLCftPU5SibtwZ2H zJ!5M6>ojxf)|AGiLy6qPn+aJ7dmte|O5KpQBQ20(ON_+6k2)my%^Ao1LuHT#;3B9` zh%t!SNG<9tT7vc=Nl0TPF5D2rhI)jC2EX|C`$l@BJ&W9T-Gd?jF~-xu{mr?@vBNI2 zjj$I0RO%y$!BzoAelIk=ii6w3@557q&%JY99LF{5ZqqXTQ0*IaKi2}~Y2^>qbWMBh zM9p;7Fh#2DX@k11YaO$WQ#-8s$=^qRI{y*>RaDQf*GT)wKFh_*p33iv4~kJrz0#x{ zuXHM!C~K5vkRE@~zc;FlBx9a|tDmVo2#~LJ`Voe1h7N|8hDXMFQ7?Ph*R8JWo?U(qAUp+uPky8Squ=0*_bu_w_vZx?f*Ij?$OvW*z9j)gn2zs+ zlVYY}x?o3uLhm`Q5I>CYka(AjrZ=*7bKUS#y(7pFQ252XM_e1Hkv)QS7$^Yk=`-lH zuq&!!ETvDNa$!(;k;o+0l2%jC(5}$>(kmFVfH3ik>*seBiNsTdF5VeH)lXr+XFg%{ zVJv6VFAvbD0Mihe4J>YHMdq5?idnaU|Jxf%^xt&lEPr`7$|bM-H@U$h=wD+8sB za+q`+&&M_5oCF?;LM+2iz+c60z+b}uM`%av3oP>o;KwqM-jJ7*dl50X5f}}&CoJ(^ z!?(o?@C$L}*r(umJ&P6MOYr~U#{Lw!!3tQ*9}(vCc|g2WJ?V3#tG&U ztHLtKL^A|+ZFFvpAN+$8WGx$6jUyTjjSFS#WxHi<H(@#ifAxTuv8

%-uw9ad=iv|Ymhv-0EUv+@FfTOB+cPrr(HknT9OD0OK`d3x=WX?2At~UhPEtC% zMu^zbD&E|}oVzSCB1?k8O=^Z^-;PYhw^ltqUT%1{Chq;7@MoM7UIM1KS(=HH0uQ@7 zR~>l(LplMzp8~FSTpsgp71Bz=7)2QoN>=!UE@Lb7n107V0(PV4Q=&>x*YRl}Z?a~}e2JC*gx=Z-iY{Ka>;#$Uyb<>)O~6`wG>uu*P7`Y^ zb?d1yBzV7erLow~AAudXUIIru3jQfwnS@}YxisV*8n82!MUSboH1 zXZ|gWmta-%J7bjLU&RgHDA-?z^=paJrkJv1iil{9R92E097zB9En~6-m!sUU{brDX zUjM|fL@L7~5YwFS3H>FaER~y|hhv~7cdL{?Z8KeNIJj^kqwIU{h;pcA*97(|Rr!vl ziZmxGoX#e53AAyCpH)737@vT|@m<|QF0L_`7T!gLz@rBHz*?kUq$_^BLL`ff3+<-e zp)YKen2O$%8y?YSGp7pu$tk67530ZEYNLymHxJYDcib|x2kEpWDM>k={pLI+$Fa_b zv1jyCdCEmXBBQP#%JMh6SRD zkdX%R?Oiu`y;ym?GNdzjtl+CY+dCcQm9HFn>IeAtx5#oiS=LMNtZAWwh#!TfRVaB; zR$?QKp8a;)%6SKooQNIq^{B~MYo-3t)256WHM31aL3X+3WIxYuxnP#f2@|B#lI4_1 z)}#Z&J+Yr;C}6*>=)qeyOa^Lowna$_;UX4rT68(;k|6(ZDmm6=PltDI43&+yo0b-E zkt)1jsH*IwO_TkTI%#X(B!Uu0r3;ix?)%(o2+BKE2yihlQZ zGL5}Dp4*kdbuK}2G2wwC4pccgK0FTkBl-i~zj?RVmiDxZt5JbLf6%#pLN5w?l7-v7 z;fIAw>4LiNRgk+~fJHVcJ4G<*YN%Ky`JXI{rAc)2AuTq%G4J6e{KD75vx`x!o!1gG z7fZ@-TnC3=;E{j+jR37ARo4GLwq_c(>9w|~$Pn~CaA`$Ot^J(*$0({zW^Su>wFKtG zrGT#y4z!9N{)yCmF>CmiRat&Prri``uLwRbP00*ZG2k7}8JOmQfz9N#NNx0DZ2C1; zofKC6R*e~NK9PYf#x=2?v=lmzoNJr0;!H?$xTeHVjlj-xflgQ!AkRTjw%8f6m{=3^ zSCrxffpD3%nN74fX5L(6w~>G^>n=p{y9TaAeQQDoc_CJv^Ip*rJA#Wb&SaH*djygg z_nHcA^`XMJISJbDppN8niJqlbLK|PS2UAZo`c-nR35^_)S85K6S4-+-%uTuEAR*iY zp!CsK5{)C{##>dYA1k=aS7;)mJrJyw774bgch2F~CGL*}sRtJJ${ZH?p1qWKQNfX- z@qx=DvFIDIkhFwo>PIl<6_xLC#Kv9~lNE*aOgr9+%6Nd=0P-YrPE29v%C6SxmEw`C}W;8H<#)V2>Uxp^xNqz z*+4$4FNuLt_J{F)rBI^6V67acOavM5a`SxPXfaeKLoJIHh8VN{Z#V(DleC4PT~H87 z`V$UI-67*g0A~SK!~!*)#4+X4W`J9(#@CwOOCzi$k8qWAFS2E;!lmGm`VRq|W9Q7P zuUbS(#!ax5#7H=b;cqB^XoFQH>0f@t6Z|TEhDknI)zwlKcw3ZCSFc!IOl`gEHYXqJ z+EnN|V{(BX)00H@H&Oc2I-Lc|)(cDMmZzzzuL{V^$_WVI8dRFCLW!Hi5)JY9B4M-> z=%U#mOm+t0Z>{{}*d_dT)TO7u&gB`3bV0UNYwr~uAWlPs#!@EAU52g$9TTideh24% zzT9zni|+p3t~a~s!VVx)@s;YK+S^ua%WGq1{TiY|8MHQS(x^YWn{vs9hV{-UdKB_~ zo-Ii+;7eMcOk5&m7MB+bED_@eJ4x3nr|CD7uNjtl)dG9T$4u3X;Wlw^VrOycJB}98 zHUbfx!j93HSVBJwBvY?8nG0;v`hCP>g1BwrwfY<>1yH)95Gn}gUu{vq_@1D-`r&!{a3lWN+4<*|Y4%5)O6)%hPB-_Au*|#QmWKs(|7ERuB%TBO}tnLBiv| z>SLo60+ji;fWBJSCfY4KS)QhaSYez^#U$=|Mq~*%Y!TttV)vD~4#rr`JRnSh>} zU+9Ni$WL8Ria}6cRT`sOy#?d^`4D!fE)H#|@IvmC-BzX4Iae9;4JD!UpWmH$5&!gQ zNgTPB;lgBK+0+YQg_ZUnxvrAa_uovQ>JDv*A`w|VvH3b45;)1mi~137m%T-W|NCsA z)$~O1%(mYCw_UqS%914@I-H~vr?%(S-0Iphxv^@yyxkB}oTxBA%SQDFgO}>`VaI`n`n{0{qAV;_s9|B`)pAO;z3 zI3{)Z*pT)y7I~fS;ygvDHH(!Y_GQIm$|nKZfYSRQXmJ-cFrMCuk=zV(Xph;Xu}E&WF*10Me(ffu{~(3T zUyN}SE>%$aC@f=M5*i_;J)_<}$Yy*ER~E^zLz`9to`0Mu!+CCI-|&(ECHg*fml1FM zhEJy$E204_w+YLEGM$%Nf&_!na^Pu?pP^rR?_5|q%`8%-k$pR5d;6=;;_ww}{xB7~ zRiv=a=Q51fU`V*xWO~Ti#K#*PX1p*0^O3^4)Un4UkUe8AV~h!X(Q%?zZ`%AC(SS@l zgF#y+>GJL$GKLSxos@TqNcqA-)muuyh%4!5yjd??X`!~kd@fxRSJqnd?byO4T>e^E z?;73CzjSW{U;AI>zcH$5>Cqmhj{3S$n|p`G_ZoRh%KB{|BVBp_*u$45RZ*rgPF-b7b$Pq#+Er_r8=j8e?p6tGcW>>sqy-@sUTtj~$w?{_T z)o$Eiw<-6+EMG2qnr@D~Q<#Fg4{xZ6^?NA=D2b!fu9g7G&%j}mt)HBetI~~ikeI9J zi8}ZxWx-Lr))?FB>^GJmAH{)-j2A1CooT* z!oGx7vam>qYO6JYeC{JneSgOvB`5zeLnmm7%v5G}>1zV?2rfw=!4eys;^oATnAEoo z<;2{3K;zSxTa`&k=(_Ne=503OK{Va%B`uL6}bkI_U<|szQma#qDD6F4V zQ%^*xz1lx*AZLK1TGHtrc#X`0^|-{22uT(ep#!*l8N+}(rtM8Y&d7J9xh)uzTvv~g zJ*rDB&Whn-Mh_(3&M)YMgQ%ffX3tO|04nfNM=#+UI~f|{mJTkh0)J4UAlPXoGXX?M z(cr%q1fQLnbrM3q$*(l(KpRI9!*O32X=v6WcOn$)s?9MMo^B_0&gjg&ms(^cHd?e0 zl{?tlN2;V1s-1dsql6KD^L0}E+6cWcHbpPNV<}g}wVak^FdXt25U@(yRhe7&i)V$?O{qafZ|}s>x3VE z!ES73=@v(VO9g6kNpaIp+lKJ% z(GQ74dtO-w|1UGA2JZZ7=u?Joj>XB`e&Z^caU6S&QCJdDb-|ZcTIIH?)dA~Tte>f}Y~{sLM4U%;U9AM)8b+;leB;Mm>>I`obKwvA2w zKu!mig-#lgNsVM-t9#=~F9})Yg0b7<6I^GbTZdQNl8P_;Ec1caSU46~V+MJe|Fn9F zFqa}#irbE_OQ?8g6UG`FPiOiZSktRZ8{N(@sp(v`c-(ITLr{dME*q%jcSXbdB#(w@ zZ5q=`4Jm%s%QoJH=crPbzRj^CRfvl{W>;Ge_$?fqd}Pk12as!*2%k%hR0a3HhW-6= zFm0w~R7MQ#4WF}_vi_mU8{8SQX)59{1#NBRTDDNJ_@OY@%F9In^o#4LXn18}X z4%6IbGc#XCf4k`9^up}H4alk8nm2l8IJEv$clP8i-!$?0;&U;kaW)|&VMcC{uhC?W z;russ!S51BE#?ueu*~I zk$+nXJX_Hf-SbJ&+?4H4k zyeVq#1N$ME#z6KR8QjI6w&aclRO^YUTJ~83$KwVq175i0(^(-el?v%M$r@TPS&Z1+ z1;4G9uu-D%9Wwc7B1acNTQk#6K@A$gM*pgHqNVG=M&{e*LE%)jV zrfiRvhomNJxor^j_!ToN=bA(c>Zr~B*Dt92|3u(P+!x7zBv=89jgcG0O5AVhQeKPg zgxS#WwH!@p%3({uQBrmpYI;iEYrITWWi#4UtI4cZGv+Ez+K$u_nyUAwMY>pGFLlAI zqI@g~>qM3B&zvGC8KKSu&+WXEtW?rjU+mZ(IYz}6d^hd&+-lRX9E9K(_nrI_(kFNx z^leYEMh|!#w&Xw1f(y2W3=^!kW@^?#=6^Sq<(Wx;_WscCBOwZjq~n6KBK`XMmBN9| zdw2QEnDM5K-?!1ow3$yVH42_sbU93ttTBB=A;&w3@(g~GJC5r`;0DHy5nIY& z+S3Eau9V)CcESZ;wLT;t30|&NPoP5VT&cg?kF{QH?+Y}PEg*e43P8nbSHm8B)vCms z;O0g`7~>j$;N3%3u*EZ&))%2mQWo#ygK6bs_v%%mDJR3nsA9rHWX#gT#k}n@CUaVK zq=VQ0Dzc0+3J=Xn{O6S3Qv>HWKl4J=*M~9DcKJ_t3)8SQ_#Bb+A%YU6Q}`GjLfS=F z?mJcTAU=T@b-Nm|O>#X#Cja0{FD-UQO}=IJIF}B^vQ7Bjs<4a7P*e1Qe9qs!L*ac0*?5= zrW-%M8H;H-D~dxz&S>mpssXh|xvu6DqzcpKk?;VC>tdYwl7{bGi1>HTS>5E;VW4hP zw?uNB%Ap3#M(WdHP86@%AFZ~_7MdGV?1_T43sf!)ettkqbeJzq{S*$?#O zyb+ZKub9v`u3(3lz@e8!a9UW{ORy4=i!+R{@W1EGZJ+>Ydcba=C!I}}Lc)XB2(~51dlZ+?M&YyevDhnrl zC7>KtvZSTaBxhi(Cr1@SMnMs3rPNVj&(RMYrfW3@`{mM(;ALC!r#*Fyd|K)Hoe{cpkr<`)DV*DE%%PgN=uJS}i1<)|x?0B@|8bY^tfBkVNdjdX7V;;P2Hjz1-ZK{O7dyFp*MPL}vlMiBQ)P?B;0V`jWvy2t z654?=*!f@CC)XIOAo2}YWE_rsRl0u_^qEwD03ww}I+6lG@^P=V^!T@|*)_`e^H$1h zoE?!aCXAvpJJUyIDI%W6Kt3JMtl+b;ScAMt+a4d(6Y52BWj&l?#K~+kCS#m6|1QnWqBa;Kirf_qZtX-=EkCQRk z;S=tcn$Moj?v^m;+qMV+=)mYsxZh+)6e%nc91{~GlJ&HIVuF9*lpzuDFcBzG#QsTn zS~*a3q*bhoGTvM=eqX|^tz7~{*s>l@IP87mu{dQcw49t7bKNJ%u=HS7trSIO?gJ@jKzC#VXwtV*Z!AFv(&AaZ9d8;Y#9ZU z9dOj@>P zG-p<5uUMy8X=jfDA~xMveOxFF?yUKS8NFl%51+0$%wQXvzV`Mersgu8qHSzAU8J^g zFD7cc*|YBJ!8N))4Zx-0c)fwNG~kzp_t0j*r77gO*sabU*kP+GTozt+3Or)9Qbv82 zga0-((j&?+XGF6iS+jO|Yu1-;Y~XtnT|D2Wj@BGd0;a5a)bPIBm?y8Kr4!Sk!_F#9 zJx*pa3>3=5Qe^N)Z#YF>c-mM4IyBs-4Du>ew7TjI5e_6fDkssFx~-e9ZjPZ_3Dr&) zkeeieM(y3pTN~RR`k9InyORzn2cbSq*o7KpHqnrSaJRWuwwse|Cl3hO#xvP5-J5*eUMP~14mdAM_GCNzw^7Tm^~*5DG%PM046wE&Q(C6W2t_k&R#b=OV-EPW zQ)5_Ew7|TZ<&n)H$+rTtrCtFineu9ejw*W)TajZkCtp4~@+zB;YtKWWuCY{_Mefbs z%7o`Yr9E;;rwZuJB0?17e7V2+{5T*-qCHy-<5uZT83H2FU6-5}j2pIf%T1`+h5(NfB^Ojd@*w40wE#p2jrMlR#&RR_6#Z+Ff7>FURNhBT2{4+-N z85KC&Ze~xc>Ps=}rqTD=Xe_N=<^{mih7Uon#7>$WEH7L2ADw{Y(9G$Ca!N*5FxTzY zA{tM-auL{I?}`QTy5KCSbi@~^u6Zq?9^$Domsx%D>_@xF+xE3N?E4C@HbX% za5fhE$vU!EI<^IVx?VfLV zzDEMTi56kp7P@C_tdi`amAP5d8s<_4hI1BkS-?6V|47T%`!~|a60@(^J zkV6Y2BZ!YJE5d?g+os#GFdGTm0gR{PbRgAwVgl+3;lsVMb1WDS^A*yYsOU2IFsTFx zX_x{@z~b+_Owrp-+vCmqVq-hrUsU=V6`2SINe#@gaZ+rJXG5#< z5g6AQ%?OE2UwskqUpSd6C_&9$_noiaZ|q!x=Hp_IrMs3X#`H|@7?9CZY+JgK*hL8Q z0u@^D?^FC)gY3zIQbZ*J}0+~lo<<Gq&K`!N5nwK> zTBG|Jp74{&U9mhSAeuky^1ABYR+k~i#F+{#XppC2b&ncJ#wF?cqVV9_}IctBmtT zTM;toKDpIpB0Iw zsBF?*SA)$9+2w&;x<(Fum9bRIPvQt{4I6>_6Fh8tthi(3;wfia#ffICngUIKJG^b4 z9&ctu!69BQAx*IuXd+BOr&t_{SYl$!ilT+U4ev}yWEMUKBeEnyR#~)(qd9K4_67EA z8CMRO&`Wj{A!Aa}vG(9FMP#q({|J7ydc?DW7vE)Cz(>KB^GGT(1;tvl+jat@){yfF z@_(gLib3`5Lbv?F`RFxci7k@BJd*dC3u5Pzqd_J8c^sZ4C^kjBw-SoI=;N7d`#a9h z+v>&4TPAQ?xZJYTjC4Zv!k(MLD)@pffkD9&1N#U3Z;^LH7GxQZDPIdza9G{&RE=%s zp1Op}WsVNH1fDnszBJ{#1KaXM7faK<>+fSmyy_v*#eSfy+hqr0`u(nxT~s6ez$r^1 z3fyo~y{*Ve`?SCSWJJ%orbl3gl zkkr7PxOI_N8u^pkOH6JnhnC(@n1Dfe>B$P*6SsROW~H^v6>GB{NuQQz=b`uR$`b9w zXzjO9Fg@*th^Wq6%DQ087y1z5E_z`4<~8)pH-WptkyrJ@#d5aJ9A@6E=@rrfZNimj zh?UgrcK^N?H8@$01XE7>J}B5HC$pPw(hHzQdS;4=LJLb=#hF9f=m1O4Gc>$0>Ouaeq$viBdK z<~XP!OSpZzabZhYuJBiRv@j8(AY zaArT`Onq4J4zf0YAdEfQCV&R7XSi?!K^Pzu16;{QXakH#S1x#QY@{o)>hvI4psqSF zn$D0WcLZB;(O$lgvgtVI5K$~qepjm=L0X9*M>=b_@)KF((G!KGTLGxJENQD2-*lk;cl7?a<%g^KpV(StG6g`8y}GF*6|&|RF8 zCGrdowF%w1GUYiG=S9J_g6>O@dx&lqLYO6VZ*yZDd!urub+*W&e42|L`+Xh z+BioaEGLJ#na^7am!8Pv!Vdyh?MWEUR?Pk0_`SuyI>0zFLj^ZZ(8@vlz!in@?C?Z; zgi^)A+TvrjaM+!gX$7K*g3b_9quPA!3LSojz#10IsDd07I76i;g~gAxK&YqK?eJyk zWytq8nNYe@lMQxBiuD1XkZYgX!*l`>e5SP~r|FMrEy&67tZS12%N1Mdh`apDCh3{$H$_n=0Z)O4|U-8EA20_ULz}~V&=H(fBxXIBl zk^^#Zzgg`dqP!KSEiziPaP>FwZbeY8Ywn|1xNqdk40rbnZ3dbT zm@YJ{PBmL(TDQ-VFXV*DVyXhZ(qY&;ic{UpXn4AAz8;oJS0+BOOpZ%z6wOGXjnPw$ z$fh>?OZGczh}n@PFm`^}`=3{tI|h8xakOU8)mfI`N_lrGj}S9&T~ZrK%drz={D z9=^@8aN#1e0GR_=pj0raIxfJ9RWl0Gy=v7~U0c5jmCItareepu#wny=W#ZCQ8oHgR z#%~QxiEl5$%@cyE5qrvhY6WzORNGw`&<4ZN)dm{v0~$ejL`G%#TlAHPDv(D@;Hn$9EJV*C746 zV_aT5bROuz1|?S^@4yAYOaO~izJ*ZOHj@*LWle4YfL@iqTSa4w6+vV0Y= z+LgE%kcV90675-6z6l)({+Pigb9KS$_A1qp0FLBhR7BJvlvoZe9;4nE24-Mnne-8! zauKw0AunB+^ox>PJiPqS$)hqDcWy{KpgtL_$@lH zD{F^vT#%hSAZ8`X1{VY;6lWLZ)`zVl0c{Dhn23M~!6&As5Dc+lu3_4dI2r_D1d_>k z79%v>BIKg}Vv4zyh!@Ao;uKm0k-8F?e?o-4Q!@<>YVl!pEe9BBR^EbowA1%OR-5)# zvf#J@HRcPG>I*yGoyWwZNi~yvMr};+*z3V%9jVLS4Rj5zlTFFxQ$Y}&(=?$yzCp8D z8ThiHq$)CYeYqnBW*z#xn98+9r3wyLY^rqzrWE;7RN$@-7YuTD`MS7)%N334`v@Nfv}=gVzNb`?TQ>cSKM!o8<^8Pi;{-F9CTksQC5Il4wX=i$k{ev_^e9itrnjMcc_rLjlntd;* zfFG>oG%AYcZ=yam6g`j$aks6pxZ+K1DsGOVG>P`?h7O16;|v*$a_a_JMAR1jt@?|m zQ@TtJgT-V!-+yGsiOh~zp6#*O3N@Dh0Oo?HCmzWYfp9`lcKt2f;=;yS0|Hy#MG~6j z8Fi>^1MHnT!V&gSc21rh%`TgEmhrLUxriTe7|J4pWizx9J|tu+_uEewm`;E*xY5dQrk{Tw$HX>@#ladj*7+oABy|4+d^p?e zSnP!@?Tf8Ei!I2DT@4hwK$C8qvF9AvGdwE#=a=1z*kwU+Ao610YbtNCBbw=#{gK#p zL19uS2W3V4Sbc4cQ3f(?) z9j|ZyEecps2~;t{zitTrvE;fWx}J))Hu)r>M!{fRl1@T~PHW&#nC#F>W{FRksW@1B_J1%+n*^LPFa*)tN*$NIdN33M%Zuarb<0U3 zo?k;)S|Q~tM1Or^<(Vy|Re(TehLe&D+eV-agXk1U2L`Wrk5QeD7_+m`GVuh(zUU8V z_Jhdl8EAZS%A4-O!*DY0J>jILuPB!g;_iv11jguc9+;*D3niXpRP#H;)YyOVC`_!E zjH_V>jb~5gquS%0I6z4{wmQU;XCvmiv_(T0~eeOR{+0|K#m2NE_9o1)u=p<+wPk! zyaU=rrV2I+R>H)KV*p*w7UC*|!qi`Mp|_^_lBDAGx9L_VBzx9_ri&qVW4MIaiYj)Z z>p>GAGN(l_7Cwk*+y#cRHvL^HJh?+})IwLWktu*E=&&P(w3l_TmbLsH{M~+tm%(bg zBc2AbjX6U~M3;<+Eq=c>nrQJpIFp6`2`4i}QwC3t^ybAgK#>q_21Hr|zd;t;o z@jQCjZ&`A*hGdd2*%nxKycf(zLwPBOnQ!im=;5KdU!)sq?*JLZX*wp)!wlcBuEek} zhhe{o14u#24lV~rHLO$t=_QWDwv*WaK5)JIxUk4e@XDLtF_S}{98f>bI0DXaON%ERJ3$Os(F(lIV8jE|K#7g`*TuzSi%CJfK?6?sHXXPfk7 zo4x0R!WR$B7ZvkG;t7)7C?&f~N+yBxGdbdo&Au~>(+-Zg_8z2`v)LAek+1JvpZ4Fa zw837HOfJlLk&$d^KHq}jRC7`Fc|sM|P$TTVHh*R}9NJnN$b}-*kkls*x=AuO$x`6j z!r6DSod|g};0`w6Fg6_h87~LfX2QebG*S6H#%I$okLF3MEk?UqCuzZ;H8Wh*44M^o zv{3dY_`DrY$|++O9<$jSYvP!h5mS_&cAnhLYH{U=aRu&7epCu#tDTUixCDt>dH#Su|;q2b}voD(Lre*(t(l`AxY(*1|%x2u~^3 zH1C~CsaUdP7u<=hSK)Y;sNC%M3p%PKqpRk4VNaW{?3=UtZZ;R(-voTsth>0{;S>R1 zwlgdoBvekgYT2~X73_0!2D8UjezWnn-YVMw3Y)m6f?vj20)i45 zf+3+2#xrq-C1n5w!&u^S7yPa6OmV+Nyu=||co#B%Z^B#+*=EVU>|x3_;T5J5PB66$6dZ z89Z--Y3^cocg)a9Lrt3eQWi&1dWs#|_r>;NA-i2;&*0bh>uEFIk5z>f()8i>tw9r1 zD_*;)L|27_SXz2V2@o^l#7*Uge;HiIOk81vsBrr{m?(RwR6z?GZ!}gv;{q-mbv4|3 zl}T-=r)5kSq-H&IHcz|?n9s%=uH=@KwmKXrB5i;H6q@pq#XwaI9*Svjvr`5IvxRo; z(v;i$xE|Yl_7cE+b;!y2I*PS(%%I;jgTv0<(u%nPQcB5iZ01zmtq-+LR6@tEcs-X!&Haq)Za=b zsUH`4RV|KfBov_9(W)NkPEmbT6(PXHWL2gu`!Hu6CnA2z6MRK2tcqQ48tfKExR0}Q z*ki?vsALPcQF?0t5lAoP^;Oo}=)a-JCYdB|N*2!K7U81x%;kp*!jB;&&6aZ_AWeuv z!ar4)nKBZj?I;o()8!U=h+;*4-37z7`y%s%a5%O*g3&_<|U% zOZ$hJSUo%j=Q{z1xV>R4Yu>7Ex{-NU-BJpJW!qR>77obqy&c{!WS+~=1oF^@JT!dc zBg8{Rt)feNhqtKK8Lu}3n}7=t8&(u?B1S>8-;#a57eCLwj`G8)>os7JjAl6AQ#C7Z zpxS0{$x2ofbVrPegs*udAL_Z^#6SydcSV&**RHU;Lc=2Fw@pP|CKVjwn~dCVbjm&u z8fb{T0$i4iT*L2W35inUBo>#3ru{Njz<4gJ*ewf&7vr_Zei$UMOZ6Ne!sMC=($}&! zproA1A;+&}q3g07+Wy^k@fFfgi$jrkFRT15dn{fyl~|Srvt;Pci1yv_8T~MNv7+E` z>WXc!fara(BOtnsF1F{bJh%)PEz9CVQ!PIopuh@i((b)I4d)$AY+BZ0RU8VKM@j&4LJ&Y0EIKtlVf@0xi!l<* zb3q`AWvP(F&3$R^TQl;|o?d345sk(Mn8f9qfx+arcHj&(CU1ZSed+dH;XN%4qdvnp z%CKvvVS%k}R+AlmB@rf*ZNNH;xGxD$IFkV(qd{`~fT8maz>?DN}_VmyY z?hYV_$0QxV5(DUBuXf{xGI zscyKu3ts1h7Ek1~^*|+tZ@`!!0QL;b_7sCGZ#o%W5 zRW_zkbu#O_0HtLxxXjeeD6eF6MD`W9hCsm#2Dk5Wz-N*|R?D4Rzd?@+?WRH)^PM2> zh5xnGbT??gBN-e?1|lBGs0DK0WOdf`JW>0q3uYtG)oBZ*$u#`HH5=4O#$V3AdbyzU z8yI=ukUJTCNao>_F^6RT+i!fn(Vk@323M70t*P6&Zc8-d~!5wyf)hb2|H9=^pCtZ`KX-bbB!G z)^(Pkt~rB8DO}Vv7cknA94DEf?;gpH#SJyPKy?c>dJ;Xmo&>6^PRA1`Q^IA&(-G(j z^mK(rvc+73$dM@=s+SesDtS2r!L+yQNtt&kTZGd5d&=qu=kGb{?^h!STnm3WQJ z#zZXa*sk%i-Cd%jKwuA@ZzP6XtYXb}T#9iP`*q3&xCpRj$%RtmKlb^QT|V2bFRDwr zK?ZwZn+9NjYbN-N-m^KV-iB!^6lt~>FB)z}o%DOPRTgX47WR`LMSNM(wduABCq4YH#VX(+BUCk+P;#aNcclrWV2msm^>{Js|5 zs#karp3;BY4}fU^a|r#`R8<$I(gdnhe6$aHYTtL7U4l(qa7=l2m}X`9Xi|v4%`6HN z2_!^J^JT8}(mO6Fg~c*oFfWTKpesSVs++-K5Gzj`5d-3ZT z4*CIR_F7o*$4Z21)6AJ+RY%p*Xi*l(-~A)m*2H5w$2 zbS>8s^%(ds&>gYC9aVA?un)N&Kw2+l^A_+7Ap!)G$gAIi(OsA@ESS&*j~Qz2))}}k z0a&(+gyCKAziPzgoj~p{K%2GPB28ViaZybrL1R9{E;TSn)Cq*sSf%7LIGKG}o%)~M z6!5AEvCb8K{ssJS13Ntn3&Y0$f)BXk6PZ)Jq$Qf1vHO7wn%fbxlMU%qt^y{hep-NA zPzSG^PZzA}WpVt4Dj3-SPtZiZEau<&7n~J1QZ^ivjkOH!UR3s&@Zn{V5%RNg@qqOfEILiji{65JlG}(dj%t%H&@kQo_ zE{t@6h|_TmwBCmu!RQP&x7h+(B71U7$!usT8*0jipdJ+k`E0$E0W>s#p9_if%1sR> ztj3a!$P%n^!fyDLRPuHwjc2YParwyPZLXfX9UyUXvr#JrD-A0=U|@~mgzIA0K!G5k zd62F!a7m7E4xzH^C9&;Yr6be*yZqYJgo;XV`&I%^C^Pb$>@&MGX``}M-Dk=cR+xhj z3CFN7Fkyg_;Kvh8_C}8!9;*N!naFZ*t;&^rVi0UD;ZkHP$5@7N3r~kr15U*NRI%!f zmFi&tAI6KU2W;;g_RUdz#kVw~WdBCQP4hkO30|oW$8T z3##G5lNwTsY_HId4qP-qebl{mT(AJebdo_%4c4+wWyY{mX*CR0lB^%EI3@|t}`(b#69r>D4-LP0U8rD5i5chK5*Kx4WLUB5vh zEwV|VS7^uP;AOrvg84EP!nH?*#bE293PE*KBOaAdXNo`;FInS8AB88g_Hgm|Wwz z@7L-I9Gap6h1tnr1#Sc4GQefnD%>AtXMIPvtP8vg=q|t+@Rrdo(4BCKeUW4Fa2ao- z*~B3amLx)~c16hO#4zr}F7D6pPnMOC-y{O2W9l{yQ?OEeQ5kp)4lzrweaVbaT?{$zq})5a9&(Lw{@^ z0~0rFdgf{{(@LO$^7#d^G$BtWq=$S>V8R6CCnF@>9?pjLye>1AQ=oKBT= z1t6HsKGa0{L5Xq=l|g3Y?J7Ax|wY2vJ*$gczLFSYY8JRgTUhxK$E z?8FpW`YZ4Q_p&?krUP$wFtWl`c2boXcx&vhg`46G#msP*CaMz{olX|P7g!NoVs)sT z4s@z>;O#;mliM%6j0;vul;VBfSpz!*V#{%Dml0rU8T6eSesH2N87s5n3DB3|w(`?n z!|Y3wk9;-!7Q;SVoHQ3K{!Y6Spf1CIje%@!Dp!tn>1(gT&G`#&ku2 zN-e*FS^pf`UJ=0q3gvV(~0KJJI%H-mdJZ%nvLN*tZx zI0grWsQ>`lv3LnLy$<1CZWQAp_X5Z2J0VtpiZQ4&ipu}ZdQRY0!6%9uvCCa9F1bav+f z$eO?~gYphwKe+iAl(73z(5h~AtZtLI(BU#PX7wSNh$Y`BUiih=tY*}#vavEM^STi> zX1BUI-0T8*0kI7$mkl)h(y(&baA&l>tj9Icy~y^pVIk^r0^T)&zb03}m)_}YEUU*r zXc6xCBE>Lq^I*;BSo7vq4RhJ{48tmu%es#H&fk+ODEv6C>}=VF0}NBcx{mwC%?f33 z?jAN}8Au_cRV#0J?BCPa8A$?P3k$TxT?!uSGAo63BXcdPXa?twRL7mWjHlTy?QF53 zgm|TA#9qYqEEV6NqBY)L(%&;%@wl)dWvjRlRk1y96Sd}%ZO;>;%t~Ik6lwkf9u7fu z)u3GVgHR!|{o1V*;tEUTZ&5*je^%zQ4p+JYtE*0@p}Fp~1!iZZuS5ly3Jwexr$=^$ zMLdzJaX*`v&rrw1u*30)#%2#9CI;=QI-G)1I2qOB{xWK}G_MA@oc$2m+h9i-=V51r z;@CEHRg_FaSO(>op_%0nSVPIy3omFz?9!FT#i^a4R`YJ-9`s#!v96}!u0Zw-sx8V6 zeQ5`kXfD~PP)LsF^KL!rr`|@=sXls~y{iqXsz&T+ySi!54*UOX5b2iy1$z)cL?%gVe7``?7bz2*A{zm0r>33%p4TDbbEDDMq_cM7)EGHB_=7u zfULzO#i2t%<*ILVr{ECT8f3`^dGy&UvJm)r18kbe)4Mw}pVWkS*gKcx;?CzlijOeW zl|OaFj1~3Xke!?N2)Z8}t{R{#aRcqN{I2z1O;n%giLNZdYYJbhR$Dv&hEpSa)fH7E ze6Aqdy4RXv$@rO3<7&x%b?nmPV#lDd?k88^t^mY2q`@ogPb?9E=aQ~CDBF|p%}VAo z%ooekjz}jwQAbrxks*q$)~^h#i0$$Dm)tNzKm5R|)VM%BJ`Ms}N=t!pfEqDvKB49`5Q_e;)tPqHn09QFsmCb;FfH(@%XB zWVainetV}M#`h*^b6zP!E;N?|EpuqSp(W1^`NQ7Yn)B%ahV#QbInUQspN}>>q26ct-RqOC6#0^heg6i@?>@FN1PZWD}R{&zhQ&Yw?G-un1a#g?xGm3*5rNV4~PT6o^52>65 zZQIV105(E2rw%}A49cx+AzG#*!4gYbcd+x^7Issx?tVg;V%;DQxSsAfT4}SlBy|2z z)C)nUJ-8ZtE0%r&?`pcy$d@-lEA z{s)$$;-F!Yv1rVw*vY~Wig`g|E%=DVv9dV57N?@JbP76hP6Z>(={P*lxGcGcTGAO^ z7+xz-7Or8W+f~s~L%gP&&I&-7d=ul!F4o5x$X$$B6$#b7g;mO>q*?Jz`S@>ArpiTY5iv{DoeG070NNJl+cMW- z5E0|~%WQUdgS*nXza0SNcQb2OUEd`9& zrYrRG7L-tp9vDEJ(dFnO1*M%hm_CCqI$W!LVe5wV+wikB>}Sh3#TE8tKHHLcpAAVB zts%$0&IaomD=Qnv^4Y%hcOV#fSf3>GtSnccq;ZwV3g)rZvAa}g+yqT0 zSEY^~Ws9riHLSEm*3MP*z4%3W@kjG3x=#(Lu5l?%r=*iS;Y=jSUCC~;6h8w$1N<^& zZ|9g?7pI2kD%V)TCbR@70N@8DVF<29h_^#u)C0n?OLg|IL1Krn1IW$MxEN8eK#UC2 z@QwOBkvRhOOcK9t3`&2&p^{a+2=A0|LSs*_Q!G~^=;XdKM{;qKTGS^pJUN)JPEmUY$ zpj!QDzCoys2Ps)4ypU`-1x{E2BjK5vK7&^mu2sW*-nhJ6`8X)_3XVm+m8Y_ z413uE67;G$ZiY{X?7ccSTQ17}9(ao=^>(k9&ADd>nb{~;zlkV9EGaUY?HE&7ARbFV z&@R46sJ$#tGpraDEF%z;fR?(ijjtplRR9v%N>`pLLQh8AaKUMCC*Tz0OduQCwsG0o zSE|5jR1rYWa8L@kX~wkyA){TWcjbUWvZ0S``$qFXx~2eFOEw#ot>=_&Oj=A;6kDDZ zyVEOnw|5l1vh57B8R-qQ0TMMf?gF$UbixN@K{Qd&xGmU&ZxpD224$l`&Pq}#(RtQE zphgtrvT<3ts?O?`VKW*@Wlrc&E67g@+Q9|Q;esf%H2h;Ju&fG_<&gs78rm1K4L$)j zJG$$|2I|EUkYcCbLac1-2oPq4U|qXF@%;h>#yl9FkwJbG(<5bA8*aFO{ZsDe!;n++ zj%8ypcBejmN8V(C|hBQzs-bT@ufGzOP3GsqQZrq zReRq4sM3>cmU4jrJ;LC=y2DCr8)(x(PeL0G79%!V?tH(>*W3x)_HhC3=^N-y4zwpA zA(?r0d)`961lZyGq4U%Ylo$cRSlFN$84TPQsnORtM^@)N3+HsIpaaO{P~xg?sYJzw zUApgEBP)f57BL*-QyXJO7>yUI(p4NrCfaJFG82nc8>%{J4_Ch4>Hu;GJr00wc4K9GqGUb98&*ItH& zW~6>Tc@Bf18_NZp?oGN`b#Z-I2;Ctwznaa)x1MAd)aIre_=|XN3zAk zyLQIbRKO1&wSn&hdu&q93SvZQN6w6KN__& zWz{+{F7@D$ES%9VKtx@kr6rTOZH==Sx&4UWD1#|_&;BE0N_m({NV}Lai}|Mn1O_)# zvJ)+$JO#aA|Cq3FOwD;*5LgDINokf;t8G0TBu9XXbhR^zdka&nh0WL!R4v6TyTmTw z9_4uG*vY`{ScuXqeOt}FQqdZJRRJW2*6l29GX{qzT^D1ZuOjg=zVKoI5VJxKi zx;wQJ6NHp9bc-zvb%Z9A6|0U|BcPDBT)|~D#^4Rkh%d7}-cBwP{~;V!FrH~}MCd>= zb8j-`GCid0Nh*lOl-OlDX)9J;0-3u0s&WEx-==u&X)h|-kOwA#CPyx-KO7T7yDYpo zbuP3ZE|UfqRL9jLhL7#rF9Z;3WcW6X2`%Xg+0t#i*Q}g>AwXOry%I$NKn=(A?S57{oJ zvngW2h-RylW^b-4ZX`so+3<^Sq>`2=E|!XMh+=%ARHw!%%H`A*MD(Al7@FwDAKP+j zpw2dKd=k@_hvdbZ{)Q$TpnL)cIcTR{he%b#6dx+kul3LX{jE&1O?K18S zHwO(bi#uVP>`)Urr#(aEFX!0VNG6ki{Ei_p|PTxCFBvXR(`r;uTozw{L%Ab#>aH&u;lj1Y<6?TP=>SrHsSZ%@vqF zUD@ey1#uMFMJ#CGv?ZV;c@^BoSHzChq^|%I1CQB|aCZFTZ0#K!SzmVjbXhmX#I>|^ znt_CFlE^5F^}uYxR=`zkz-Gwc5$>dTXeU$2S1ve74CENAN!+#9spN;gX?Vv?0nHQx zHW4(7ORMv#kA?ONf^k*UitJ&%R=F z2~D3BH=^c9FQ-vqT)V zuy2)(tl8kX869YL%hv3f=9O;Ryn%~12g^!Uv_A|%gNZ$)X!A~Pzp%G&VO4PX1{rlA zHxpCMTKmoHY%?d?#2dcoMImgqUTMa$n<(x!^u-ly?>JHPe%uJgD`KcC$rvpw$W<2j zmEkl&yfTsH2$h}fLIHiIg^hQ*NN}rgWykB9Ql7KMzK-ZsLw(4Vb!pKnT zNJfMtKwt?^Rf^ljStF0|0+;}lQ1>a^vRiZltf39G|5n}4y0g)OrK!CzaE|b6;EW44 zZoz~tNN5321IWo{f&(0?133*Rz*+eDO2jaDgl0F!e}Z{nQx`V%3qW(>6KG*rFTJ6> z!h{eBpAxiz8Yojoiw6t9gnG^+O<(eG;alrM#Ruh`WE~&t{h&yMh~cv3(`9R)%Z$=x zen*CYfhw8sUX*hzU`La@eB@qu^EtjS5hE(g9CE{k$fE$8wE>N>;c9}j#E~UE-N)EW9zjXCTTVqek9wh zF){@TH#y{WvfLv1>`InrBp*$x!1im&R&RIx-&^ynNo4dB*{okc3PeK{upmaPB@dv4$z*>9_!kv4Xv^r zac5gr0dy2AS!YkW?6~SG(4Ahs-<&V*7gvys0jUD}a}gFaom!P?3gan;dPs2d)Sq_5 z99Axiq>`0_nJgI@C-!ShEQus}=<;skqEawJ$7_1n?h4B|f8u}{Noj^;s?iQBt>8#Htn@+!Bb123Z;Abf zaNh;vUk{wbIxWfiE6G|ZP{d|Db>9GswN6k7YF23q9hm`$WyjToFf9WVY@oM7K+_GF z6o3;u=&@Qz1w5%dD~4#CCp(DW12ZT`x>IT_#Vj;VjL(WdG>T)O>w)XZ zE^oUi9AZt@O0mr@$|>!6G!*QnT9}CMqYhzZ+G^J?)r6i*qOJhoFJmUnDH6^G9?}6q z8VE)QM8*QlKw<_I=GsnsU-(Xd)!Kk z#yrAFcjN6FXovmtG{YH2L{L4f?d@N3N`8S0Lz8IxfaW48d336AZ2}7hBZBYFI)IIs zLpu-UACwneHPcwlQEoFM+UBqQxo4}`*eXrR#JhhqP+j73 z=Sh-Ff`8D~!*9|8?C`5hks5#~%f7F?3o8vwV;b~W20dn^hVbQ8$RZli8f+#3Y=CE= zeUk2f)%ZreoN}#f=&5dG!*GRN;yao_f##@DtD%h&X*WD!nS$16a{!s<^S=4GR%&=6 zf+b8gA>z00p_~>hNlV6#Z<{Bmt_@>Up0{LpAsc@v<=$@_M?9%7B(uMWl>Ln zX-1Ws&|)4iK9i|{C&X8RkG3LkV@nPchjW4kOvT-t7(q@Zs(rIf5MYORur{h7j$#Y1 z9w+@oFf}0?WCKPlJBO^h@T)E_Pokuk`m+^d1Vtc{sIp-Kl_&}nzhjy12lg{=J)1;A zu&!B-uc8`UBmSuyZm$O@2J3I(?{sngEOdPrM!G@)O6HQp8YI~?NVc#oTjG*!`;Ir{ z%lczivNSH)1SUbq9;efpEz|aiIO4QIMhf=XsuIb}HJF)Ml*_Zj5*ek{Y%wM>=4tG# zePvTzUC?a;0fGm2*I>7&`%GI=e&ULs|&V~c-01hakQ#5OWMp!qny)Yew9GJ)2c zT^;+u(@J;U%+Ys5Ymlq`XGq@^i&Eilb}f6V^fvV8h|p;5PeW-Hhf?-+i5Hj|NRzy) zJ1(G-Ux@VU`Q34U#5`MI^bHlFRU6Ycyvkm)t={E7XJH~}wc&}+-g= zbrd3IWkUFO=fGN0MYGN=RObPbNP@$#_LAS3u`UuLrX#%Sz1A?cCcyO z{x^q@=AIPx0R0hNI>nqu+5>IV67lkLAK;i6vbYA^%$zMxi{pRdXP^19X#!iiA`Cqe z`PlD#;wGG@EBL3RX~j9cNK*BhSF?tdS)O9B6(po>5dR=zjDV^6cS=%w61}&-&DB1v zLV{3FNA1f^SG4WroQwbFr45p^l00=KX(e8c)l`1s?7EX{5HX2sA12AAvgkbzNCr~Z zE)tuE5N(^aO6fl0Y;#V+-#khJ2(!qJv%rcQj`}tgXQT`A0eKX2M2BX_irG;Y!Z;WW zx)OfyRD*{MHoP|#6qk`>e|}jH_OHT=*w*^}*E4v%Yv4-0<{Y8WHA>LyXom$gB4bxu z=iH@Ngv0)=CTmJH_+lVwGC;6;8F#MD&s8PRa$qRKLEzMETP)ee*d*x4N{ZLb?d+7u z;$0l}ng<8fz^PksM9<~U4E8I;3|FD8{o@GtX_c3LZWbunoE@zr?2b!n?8x|J1s}jd=jZ%!+GvC*mg0~1Q z3ZtH?j8|@0X}Gpa?NE&hKaxnCSaW{W#TTKy7@7?kK&r9jBiV07v}y^^`cx?sWCaG# zCE~0l{PBPT6RsJLAx)~9@A;^Cl|@m5Vhlp&DyGK;$`4nKhGz@yHUqL-=SswH>ok_C zY&7y!Iwsfu8l7%~8?q(MY>U4ZbL^q|_016&d98LycAcsh|Iozv%>lHVEDHOv&ud zedRAD)$6ivDv4(}6WX3o#2Iq3yp~W#ee}?z@Ix(rkN?xta>wrMr~QpLjGXgnJ$;_J z7*v(J*v^pD3C|nW_oF}01pRSeeQ?D%jlvkBB2s)WFwM}J6xxK}iiQY8o!8K56VkDg z{~EL6T37ncYlZb*}3Ng5SPCz3)-B79@;WrxOX_`rDTCw^Q##Pkz=@cK}_ zJ|6XJ7us8*vuo4IJz{Q8&L29t;}`S6PaS^!nB?9B`44vSRV!$;joPCmANGQ|qyi6= zW>+xwNnos0l@^LB&8d%Za2KM7?oCA7{Y%sG>WEV)RK!1|{Wgo1Y{(BH#^u3X2W7Nn z)G={wNJw0L`oK)96fLtC?3ON#{H0(8zGFqQj+aA+`9j3Jb)`G2xk6$`rzSY?Pmf?! zh!alup$gop}zg71#8{7X6Nt@|VG5BqSD%aRyI9BiynHPga1_)t6Kak2SzlhP%NUC@_!oFDn$ z`T~;^O)$Dp`yZ9vjXoMvTxyMS;!TNVw_9Xbfb4U&dK(`fHq285wD+wZ@dj7GJc8-FTvN{9??i^`|>>Ap{#e zc}byDRrd3I1k$P-yr!Cbx#SB77YE_**$YMBY(c|%XjY+Ev`|N_e5PsN+2bxJ%wRMw zm%V_P+icGCKcRK4v}A6moI^TQgk2JTL#xQWWK3dWEE4N7^VV zRFjpa776@MpBzCl&Aw&$thn}HQNL=dK1wO~hp&ca{rDZ&_UG{Dmjc8&(O@AONC;q~ zsPe0+Zo5d7n4y@Q&NrPd=Yep=0bSXF4M-mAL{f-(lE=_%f;Q6xWr@tH4CgURma3!0 zK{i{kg@S%b?-wAJiiZdg&1x17k&j&P2Y}n7_I{gv;rK}onM0WFpDrV z41L)-BpqXrjX{dMK}uDX#N|4-3n3DmAoJvUyOWQJB1CN5E6p8n(4Q~|kT7_eD5&ba zl+40f^Io6cQfg0Cte>`&&QgZ;qonh&^tY|>A5-wJbDx_uS{Ul)in_&&!}yL%tEXpG z6v|lWn(nxjb-Z79KUFj-6^OO1!nyfn@4d7i7##CSu%|GaYWd{ax%De1Li}ZrWvlaGw z#s*B&ab^H&Mtt8pElgYSC$;nnqbMKzOuIGk`i7Z&^G2z!cz6&Mzj}SV%z8sAffHTU z|Kj5CJnmF(`*#a2K91e8Wxn6Rx)K6IWY(Lq!;xsOF>3j#in}_)f;{h)79G#H>9cj~ z30u+#M=;_AS3Cdj+md8AdLDj5B~jIAS~HaosSEeSZhCl$AIt0WGG0)SX-5tg6P`Zo zeL52VNBs;0cMZkz%30suJ?4J@%bowXlFAQlB9 ziO&gkON#eu0)eZ9+g`94wb^S&W*g17+yAC}K=nTeq0;62^=)I)h%5Vq>n?g~sb?w=JL_r0 zqD2X3|A1(SKyMqrSHyzG*7w8X1vb)|voBLn7bV@#0}+Si)j&ppjQKCiC%%5jB! zno3Bf+9%#gyM*%Ht0i%&Damt3&5Mx6wftsqQLJE_Se5(0LN9u)n^H)Wy{ynim=L(2 zMylB_orSNV?8XC|M|-YB+UMsuA}~~6f|*$Hi9y3olYFlDXeUDX*abUTUlJ+5ABHTZ zbm&K>c#1_hbl7L0r~Zd3j;i&J)j$-^3IlQbC*|Q+G_K{S6Na8B`+UMhjnGeL32?DR zss))fw?|be$6w51gX6fD%he)?ZD}RcwEhXwpnt0>Rh>W78xPkU-T%y~>(&A^fBM~wuKT_W1N(76wgFFlh%N%&|+M~&_*Qu9R+ z$J<-tS>Im4(YY6=syf$xe4Gz7rXzEV;+1q{ec%Z*Rplu<7%n6b$Sal8Q%Q0!ax!IoT{9X&NFC0n?X1n9c?-T3?X5 zh6ek4$R`fTT)t*wJuhk#TwzY6aO z;4=<=ChD^pV@%2UdYF^d?@k|CQu}VCX5EjwU~7kUGp^Dw!K(61W>xG$VEz5pNQ>gt z;X*qP+-t@kQZSd}kuf@dZtJ(_$Y0ar$++#>p$#sBtCx4iUvsB~t36hv#hAL&&`-{7 z(JuadH$CHX7`52;OR`V$1}Q!-_{%eC4^Ac63wE}h(%N(z2hP;@_NY^~6Row{0&5_oH%^1o)2UfmRtJSMUBdW%SbV-SpZU|~Cv-%t(F>}h{5(zoj zuO2qJRIOS|p&h*&&*=QoXRtgTaK^+f8^6T~ugi_HmekVxjrfpifA;vHq!??Z(KIPa z!0IQ^8p*13Lwd9mf&#{s(O9I=7%%^DX;gP91e^aNiz>McYQc8U** zpZ`E8c$m=d5SN)(eN5BV7L-p0^b+aqCv&c*9CXUN~N_ z*dT@tL_~b$F6WS;sjM#!Hi>m5R*U-(?(#K~4Z(rsQ& z9&g4lxND$WD@=(ES{-PQ6RQuW_R*VLWJ^%qveWwyVwP~d*qfXo%Y7s$tUKMY8a8V~E)RD~nz?cLSpZcyavlsV-H~e6NXrw7*0r%G20U z2R<;F%@B6Mq9t2<&!R*Z(p_2I#X|(T*L5*9`UPsIb{rMPNNAzzQb$JRvyCLl zDz~y`0Re+@8tivv%6B9x^L3YIUsuUF`1g#y;q;ls2d?TWe#we#&4w!#L-`EHce%S= zVwBk}V`2hK=9YQHIqbO@iur=)hTE^jR4+SSs3iDfK@SZ%BmO5m(UDmtQwc=&drU_c zbN`jGqpc>%n8J@FzhLSs*xP-Fi#-d9XAXLXKi>z6EC|tWTE+Z6qcnS7y9+up>jC0N z$$3{<(%ED`J5P$FKY^A399$j4Ez6)QyY&4w!|F|LemzNwRWH%079)P`NBY%2F(4XP zua%DI_{K&&M{^B9xqL1B@d^S`0it)8ZO`*?c~%m#_Md(i(l=vahR8ZTnLebmL$F!S zVwPwg<$X(>pep!mJ;=Xx6CS$z+lc$ZD!v&X!A$%Em~@s&!rtzWD2of)<);G2I<^&w zQC10_ms#z0TPU$_iBrsw#;+ImD;+GK+1rXf(juE1+*u_je;V4JDyeV>;;Q?eiB!Q* zkg_zLttoIKOt0?BKF?&z%c#*=i=72T>?*&!sSQIg7ADs~k!lwKH&>L}p5-ipBj?y@ znL)*srYc_3i^Y|C$$VfU{BtvyoE~F}Uv6>G8uSaCU3agV@$=V1i)@`j)ft$YI@VlT-$zyWliJ5F89nm;Q%oKGaNNV^E=0y( z(x&}ZYfa^VQ1Ll8Tpvy>`FVMm`e`z-lLK&Jr;9GAXUl+iiq&=oKY_eJ&87wh*KW`ji|v4Wd`Jug06@ReN5BlOT*vIZ=;hp87YWp_8T(2*}6tg2@C|vc^)V` z_Ea0e*f%I&PZmEjn-N5pWu#Xp?u*gDWHw6TxdNOo!+y1GHO$%6;%vAb@H(D&bEyGQ z|E`5QHvDvU`5^rXfQO)L$;qMUY*c80xk;YX{3}U4u*x#2yVUF~5MkN&N3V{#QR#%` z9nSUWK_q~{AA{3du)1|Fb6DdnkW_9uMd{M|uWrE+cOK$F;)>bf(4xWQtT~K(W}hWm zMqij!=h1+#*m8ZN(01q)MRw*4{KpJEC-9^020M*3A2a~wTlkxCn3eN*KXZPdbN(Z< z!IB))WEHqR2%@zfd{OPJmxmfAhm}yC?=JPHI&GMUdlJ{d3okXBP;1^SE-1a$`@?Z=g8_juzLP7_VqLwpn*ejP@QMTSFfUnAtP zPOWhY8034IF{8R>c%%e{Ajp?<`gFX__^2agu*jjiBfHHEe|YiFfm!e9yOlts(Sol6xAT0X?kW^gY;|@TD z1uG^{82ay3Q{}WZUx?;1rvr886kD;?t*^Scjr8r_C)N&T)~bFLCK)pQCp{1#vJ zpbF`!z%H+P3E_Pe({P&AFr2p+nnMwS(jGGCe($;9#N|F0bS2<1Kt8ejaHjQ=|I~d= zXr$#^+oQf|vs&!5MwI{3!qsyl)Xzl&-gym^^YtG&E#9-4^Q08W?)y^gap}%+4yb0*YVJzx`N4m=?-}+(B@7z$AW8`QO@g z$?W2wU?p9~i7cN%n4HsWA3jrBMO9vx@)O;EHsxLUr>QtyGQhdhp{n zDsfCi`aN%EYDyvwepgP!Iw@#kQY-%x^JkNUS<6*e{*rv!&j%JR0U;;S@K7qIHH|2y z^Od0L9YE=z zLM3f(wa*D!j25oij%ij`!2xYHD}Vcwf8xPBGCDhT;{nHS?6g9S|-tG1`<$)cqxDJX?u_K0=ZSkJa9;rZftv6d}TStZkWSZyt#H0 zRIIYM3R>n}%9Nm7!cIbUF={pyIz^?wo3pR<+>vJZ@4{f^TUJ;c2^`quu==<)^KB zv-)FCx-9|1J-AVj+f&(l`lEqc&kC-)K(Fb?ThYloB1Fi%Aa{12{$N8!7cazBdtt*ft@vi_1z5K6SB2rAzP^-U=hqu8>R0EKbA_e zv~t?RT2Dod)Q%SD%BCz6;k!x2W)Lt-x;xke(Yo>=YA8AE*)~bS(8;`xRwrnCPt$z( zx>yP*?|eSVeV%}T`DehUCBvp7Fv_O|-TY(W(Ec@;v^oTRrdmEIRt?nw%6d1wmI3@o zMY?6N2!7UABqm|#nrCVAE2Fg=jBJZVm-jH`={ zAa2`A%C3jQZlNuJPsQw55B{!s;23@7y`urnjI@*>pYvlMbJEg$ZUN{HX%7b%8H+o; zh)!la?EHnb(8h3`sn_IKSGPy5-sG=uF#}L`6A$t3u-IaJj?o;xVDL|ymz2GKmb{FD z2mWaTNF|ELNHD>p|LjA4VAreA*aX^}FR=dNu;FvB??G1tSg1Cp5r@64z9ivpk(Stx z{_{^;&Z`7*KDPET7W1t{Ob;7z7A#F{P5VPjS~Z$ppX*S>LXay;uK=2wlR5m$V+u2|AotMhug!ij$C|^!jjPJ&=ax#ea zSfC+nMiX@s8X87FkCbK!lL3FWMjX92Zv@giN)_OT{Pab zEH4y&9j&`ObFw!&UJtP}rsoibLUy&qHwkObzhe2jNwi;=`Q+A5>B;pM#GaiR=-nc@ zoFJbk8^-QrX{EZ;9LKdL_!tlZ3u8!zzDn0nqmFsGivC z>&d(bkS98-sb5P+)J2y|rb@HrvN=QP&?I`k^EzEVl*vHRDkbM-8QG#_T{1Js82kBH zw9ak^QRAntL7fs9YS+yK%S{hj?635z_zhQD;|bPJH76cMpz!XIe>31w>^H&e+>yiq z$yc6fjd2nle+NXHDvDr1_k1ezpJ356SSTGse_Mz%7i?n^o!$}@F}wGYGEL#kY=?Mo zQ!~t*mx+cO`r-(1TC^;V@qK~pQRK6yhy! zbY|nS*ab2k>8`x9l^070(R}XbwpA)NKKc+%IVs6(|Fuv6sST-uCc6Agb{v$@1`h*+ zZhN+fSBA6kM#vp!&!t`njQyNoii5^6$P*W63-BA_%ez66=V87>#UAw&Q;yFIpXmGp z8X@ZNc$N{w=E9H9nQ1oH3z0`;6dh77BC^2@2os>~X`UVhosfHx!7|F9wg8gukuW+om4N zG6}DeS4nn~rhFsfPO4I6F=%WnwPUii1ItkGHWl9PE=r!4PHZsP82Te}LyAN$b$7yB zBqDzP`{Q*c9>mt3^oM7UtlOc&L>Sa?_&fikY5HV0EVW;kkM4YPr+Cb_ zGi75`hsPTJy>-7eKUU4AQ9;U>n)Bhf9zQ7cgpd|p=(xJQ(+2iC#Vow?N=-rq9mEAk zb8zl_1K1Uje%Hr$UHnn^&Td<`lQd}12+v&sT3nKzi$f^yr?jD}B%@h73&=uLfGLC} zL(Qs#ufnfSN4#hEGFC>HT+B)(3gLSGTKK3(N91))N4? zEz9@KL)_ht@ec&fEq-Q%?ai&{;`tG-{em>USluJNR9KRK_t}Y)Z0gZ3<*O*2eE{ji zf-+!u?=oI{jCueq*Fg713VEo{hfa14^hVZVf{|C| zFC?m&ilCczzurK6I*aOp#VJ)wSiadX^$T^#2PJd_G>1CCt)`X1$IY)q%|ZV*s!R!fWWy=@+$FNRvxUZujE31b-C|LXMPxRjSgUv*y!dqu zmZAr*v4cD)4kFu0vK+@dkXh;xV^X_#p7|^Q_xAN_sxr!Z+#~K%pOdR~v2w)FpSjr? z)4v5MF7A)F(wu=SLu5G^gLO4Xt7M#O(t&oP@C1$;zx>+P`$ynJuu=TAL7v<38{ZPR zW$MlQLzl!(>P-JEkR%;*3zoriv|8t9#e*`RczUiZ~Kq3KVTjT zjR{i1%eHIuQ>~7SLaLVuY@RuhnNol_3twhFvnAf;JCzzUqfvL~wP?xn_~BY0kphO$ ze+Ji;isfyxIx%Na2t?>*iKqJI|4L-J`&|W2K1o6Z8&$`E_i=)??@R8XVNY zt#^}remo39!6py8o$zQ>qAVuLr_)M=A8vrPqW*+Z#t603sjECWD}8-~r-jw&$8f3O8Nz+HBHxF5ZN) zCy`o)9U5j(6*rMO*IYMu<%SDuZs*3U=x4-1+fqcl&>0h%`^7d&0qpJ;62^(?AaK$n zz05sat<5M&uYssnW?d8p>QcST!1v>&&FKk;UfW@8N&WZ=H&x93;wecAnN4xcht$MX z4b*42T%{d3oVMUjB_vPiSER6{T?E|BLtImT3*_usr-{je8dyh;Vc zzkz=hO^T!7#w5B@4hsee55DaBamSHc2x+J5zAeqfYze5<+eDjm-~N<0j*R|P42z3o z)WwMHKBrwr?LFUiihrhTjrFb;!zFW%eI~La!4XpFfq8kg!vC-oxA6V1S3GO}0YT!t zMQm~1O>6m~?O@z||X&+D-O1g*{oNgBv^k9yr!0k4w{Riw?a!UpWsdi);J-QP$@-*1SMg)hZK>+8leKeVBz%< z4vJ^;U;-CkX_3t(sC|{>Psh~d*z_kV?H0^eAI{~4i(6Bf=pbb-@)>2&T-A!B4GxLd zQH(IJOyeEZ_7%|!|6K}h(&RKqIt*SYR|!uU8MjSaDym|6XM#sfnYCw!W}|ZFj0^3w zjis&IuOySdL~w{IK&m)ti+@&TCVMG_IL^vl=y9(k&4A$=;7@Q|bd*eFe5zBbzhX-G zUIw&EMT81CI>>#4l8SE&k)$y3sCQ5Mw7HG^j6)%gqv*m);y94l9jAIl4}JjPUi8KwVhX`m*oRQCLc-BDz*<|GqqX_xlP3^jA8j& z4M*>4fl?24Lbke$(bu_B_($q;6@dbJH!g?cmgQza(_fYu1YN4%1Wgb4V!rU(0VxQ34 zdm~^@aQTjZ&;Tl!AtbYeAC3Jx%zdEd1@DxzoZBlmvKiJxLfaTxIF4<=>WQbC=hV{a zH9K$&^IB`y7E|1&1@b)F8rk`RB)zYR%^3W49tyk`?Lz7|)Kae~U;2x)$PzK0xd|iC z2NW==gHGhpEGXPol{uX{CwRT1ap@5_SGyD~j^~(iKiCcYX5Y?5r~5GoqkN0fn?JZ|FQxL!XbYtDv|zcD;*|-btW+11y?DTTow%bG&<) zMK!C!vNvz}?_&1eWNoU{R<->%+KX3Y!G@Ei$Ufy_NYa|7Vu5_&()IzV`8!v118TPW zF9#erDfiXK{>HrGH+)5lg*;Av*34AwU~+KJC(3{AuwU$FaFDN zdIGhpmX?kb3i8M3#9tTUMJ&Bb8p}NJgeUI`;<5#H)sR6BIGE+TWqR%$)N=Qq3Kot=N?%FEe$ z$JOzIz8yvUSPz9?RXfzg%!XxV?-Q(MUkx`x*Q1M(CL_S_lBEjM5J1u77gq6XdU1#x z2s&lk>qt2}om!VSjHzyu!;?ETI33_H2GI02mprG==#3Amsf>JA23DDsnLvmQOm?h( z)4PCXg>*jy=^J!kcA`i$e=2xovA$=5#Kp6J$u|We(Y4H3B%%|QV-uQ1|20AR7TD}F zy`M52flXPe9@9Z4lWh%}UmQrv*UMAyX3}Fv3Uxb*Wb0f?>7T>Ih&7K3>*~Fy>Hc#$ zMygjtrhqHe4JWgzBiq1`2G)qYz-HqgA|OXD-$-^~DX!~CEEdLgyR8aSo5t-V5gS*}RS}fNa%UR;9Y%(9%vjiW+mWeaD--`s$0sjh zUcKF04^LmZtZeKFLE1R$H&8hh!AQttRNs^`dB=gNpGMNg2+tdY55 zT5pXUK&_o!8JAd<`+#m*{@e|J`8@PcKP1i4YUuFmKqH4l*R4ptv%WBL@6LanXOWzW z+B+IcPOYBH!CEMC!}ZRw_jokSvf25p470H$?@v*k?Kd+2Rr>HDxK1+N4Mzuf`XPK- zJTZWK+2|qhV6XP)7MUiTkk8zfepu=&L>`~ThcNHAuYUhp(B`47LOzW=bc~wrBA+Va zYXdYGx5C&S`+c(0+l4a!TJR9W6f|p`J|WhybA!|R)Fu1NpZ5r!%Y-lidE*L2d);pG zaB9_W(L!G$QMdto+Q&ByG0-=@t>AorRC&l?7EroR-evz8k8>_!tC%f{`bW{#tRU$n z;rq(M;ee$@ldpd&T}wS&fVTSc&v2>iDD9|k2=`mnI^B_()sn206xwmze#Yz~wO=>g zh@$T5Nwn01U(hy|eKQkZIR|auan--PgxarM~i&SY{Y1_6=I6{*x^wxCA$M(+|%NX`=7h7^v8bT;o|)W~;|&S#Wt zQ;=bqvxufO{C6)|QU}66UMnc1TC*hv*#^C12_CMFfb{hj$%Rf!p?fZfPgazJ<$Wu? zyu6_c;Kph)@C~JuLb#!}btk^)xbGRGdzF>T07M8du+-7wGV2ZsF2|n-WQLU+;$nRwB^YE3_v~2-?A{Y zw6|Qi@kzP^Veh_ltM~bdQ-KWn?>RxDQcJ2_!>4=a90=Lb>xi{^j>fR}>j+PE%)+=p z_2k0%{TA zW9CmHyzt7tB|P(4c)IEGiVSCcR(RyqbyIlxqzbL*(YrPM60Fi5u(V)`kGpkg$()j{k1c0CbLCe|n*!H!Y z{+C>l@NDrr`L6}tb;BN!Zm}K5JO76rmv!Q!Wr1Fy1j)UxK-ag-=_z0T+?G@8%9gv8 zGxSd7_xu~eN3#9p6Ak@Rq6;&-Qt=D7vRt%gpN_fdwbr}ai>7%6M*Cdl)6!Ibos-hk zc(vVxvyw;e)zi{y|9^owS+x_eH!^&z$=mqeIPpbyofhyG$&a}>8~#-lEtL*k1K8D- z$Mp23?XSEbdb8E^PBDD?ob!a&b8<58Fw_o8!C2}A<8nQva61<=46!fJwgSD)OS@>hhH z@LSQX8ZW6Rf$9zZ>lct(o441}=`S&hkaLrl^90w2->Pve*T~s1R|_*oM4nT7o$~R( zdr30Y-I^0<+(G~+)f;T#`ofj^geUvph~VBvz^C(wNz0q&cY%-b8DE4s3Lz*t0r={` z4Ck#aM}Hbq%-ns}69kj1{{{EpCi?#a$B1sSh;ATO^Zx?Ens=`MbHn+rNkx(IqG=ba z*|TZsA-zjTZtneB^T49C{xCFNO>uG6^nLS^t13j(0C- z#CZ3=_-tb#*Cjvmsp7jM@6xv`+eN2mEk&}k{La&poTJJt;qJ;(&kNdpg(JC-{=})0 zvqg}zr$*?@Q<&|s@C(MOU%xLDd;v`{7GM;fHv<@c6n3p6axV5iS@R=T@%L91mb`m2 zQau^@{%XeD3kbu%y_ys3HDs^JEK}ZEe{&0ah1sq53)|ksIo&Yh=N@_bt9S!l_U_nA zdd-*GP)}db4Po4cm3v2OPW36OT z^g-&UWFE8rn`JGtWT4*hOwK*I`kU9U;7Uqtf69R;Nk_8%JDo=ycSvsvN}p`8;_?&T z@_p)#ut8WOuWN=M4yQUd%m0W7YU70=zdTYQd=iO|jOWS5P^myK?EIL}9OR5l2l4*%h@Z}$Fb z;rRW>!g0E}($iC->JiEKy1F1u#_MFRXCu`rFVsn2wqIRooX1$`>$Do31tyA7a2&UIwwE~xbPwhRf z6^o6+8oTVh>YiC|>af%AW&~iTdsr4Qw=`Q#?}YJqyAE##II(8D;dBuP#Ac#}r|s-` zb+eA;KQ$BBYDo<5=dbq9x#Wa#I^xwf$MCfq!l)3B3VMm?8J=#C7e%ddz%vp~MpXxc zOSGZ6dTgKe_8u;Ey53$MrRhj!P{4Nm+qdNTF<{fH`FI{^{_3W7e2TH z;7aar-o>jLOd`jZe3su?4BfTQp8tLwUc+07l%@JY5KQVPVG=CRQ4xo3^4 zZCxs_uTprB$xn!)|GT*GOt_un%-yT={1Ga1D)fjKiSW*jz2&0%*8iOtSoze>daxq; z+?^Q%J?aXmfm+&OB2&gHkZymgxiYjY^cjTOYbIT&zRdviea;MidY)}_8vhF|1<2@efD>k&n7@87(OSKJv8cE_vPg^DuPfnsch^Hu3Avnc z7k(+4$80^*-g?McdB{Pw_d8eZ{uxBqyZyCvsM83LmqEVO>p)o;JlzIF*L zrAah!;0QP$Y(-7I%6X0>!pxPQo6716ynei!k|lk3PP|$PSW}Pl4Y={GqJvi^F9><{ zpYlh~7!n_crcrHq-j(;Mo$~D8&>ztwju6NA2*xYcP6zo%@4;IYA~da619x?hSyX!< z7sA&c(J2&9P_!crC5t2hYcwy%Y@28;-j>j`{n?BCH{yYaxl7SuhyK_tLx;yT(!y|O z4p+%eP1Z2NH+NFuX{#ovZp=Wrkz<1e!czt1uYsPuJv!fq2b%tP*}sD} zBBtkzVz^}+a`w8?n);EKb%hC$j`f5dF|KUHlnpL6Tb6iVs1>{f$)(4gGXmw{~DTW@#4EplH{cid?#cLaa|qnGug zkjHe@2i^SJr6K<#rjFAJ;J%O<7`(xRG;U+14gdc;zLvdc{@pFQAbEj5EV{99d0&+@;>S>cvatQB?^@d7B*e|Zf zl-x5a+ah@A2vMW|qto5Ob@^y%Jukr(o(~|}-$tmn<*sPdTUaP!w9D{jU6=O#@0)Hi zaJHPXRHG!8w2e1P=W4SX_zsXeJkk1t<~qc&@p5@2bzc)0w@$U8!}!06YxJnq~` z5;0Bh<7oMM`TgRh1Eb4*tI9Ler=sgJ`{GFw5A0F$It^Bku!}fxyH{4!ytE%#K6~Ng z%62ZyF^Ek*>5}IEb~VHxIC|Ct1=DT?54i+k5Pg|o^z-qp-PNBVGQ}3y9D`^KcNS`c z95rygmvWE%*fQYWEVf7w*d>WQM>Z(FU7B&vmICiCJi4=Q+9luD{CfRucI*Q;E0XM( z8}9EayJN?=zk!2bG_9yK>m2}zZ9?7IWY*{#k2&7MwHHP2W(&WKFX_-0B8 z@IldAcn%)hu3|5BuQ@UUDE~MVpWXTv@3?<=9AyZ^0YakI2~PEQT6qqGoe!3)`AG&m zEeSoD-`Zcdd_>0)$!mWD$x0?|ceRPjfRgeMIbPqIsWq9!N%+|&V>*resPcXt| zeXlt9`q0wTP+cIu_IJ$>6ss}yaJYANENFYUd9;s!p#Iw)A~|=pMi$cy^`*iw@w%cw z%Fy&sBa{8h|GqYJ*G>v_ZzvqUA|x8Zhp-!*y=2`~`g^{F$dqj@J-6iaT1rC(Odg)! zniHhfU1DaGs<$eEnYuOTd^a~k(5JuC=??$;iCQM3(QfONGd1ExrzF!>*B_vIc*-U$BKso@PHN^m~vg;{jFFdv{o%{!>p z%U0hbG#g;~cAsGEo0h-i8n&poru`mAD=zG-Zzv^dJan5XeZ7ezlu-Zg+Ihx1w*eqc zl;3tQqIY<0?>RR$QB2&u$n$UF6*4fc9_>fQm0%AYmo0FoAw1o%Gkw36RmsVaEJ3q;aY+HwaB-Rpp(mw^`sP%IcImv z(If|HU5}Ey@tg5m%_i*U7y20c>jaRQfv*TGJ;+<(osNs3*pO&mKF(Fp>g&Rshh(qz zRj27{{oU(ArA+UcAGD6|OzDDgBfeKQGf(w*K+lbLC33JovDn6$qOf2wK`eUq?F;EJ zX%V#Jr2kWF_gzMp6tinKKagPoy z@rtAu$-hcBCSFDpR=W{nu9ANHKZ?Ha*IohUqYroj9?eZY{sq+gUCZ*JUl&?Rw#~0p zEddga(YtjX3OU?KMu;kr3ZjQ@FS@qaIGM}jx+why@ z()9-1^DV(HRQLMKMAqw@jJD_T&Tv@3M>~tJog?s`WFIJ$E|%frZ28F2uY;l-YWy*- zBQL+c{V?;4^bZ~_%dycvyGQWs`|P(jHjYo)tnHJ^^y33NJ+Ai|_r6B) zmGCX$+&}L^KdiCU+Q*h296ymdO=qy$8HMsm9P_%-KX>M^t1VQI`o~Q3ip5;h zH~uMNHh;T2mU6TD0f~KA3-04b@YnvKBSuH*Q4nk2<-q5yl*^3|GKhNUi<&~0sHRZi z5y3pqd%y4QTAqhso#)7Vw0jtSn9UES+FaABAC=|zlke(Q-dV;miO?4CBYw=!Av50_ z*P-#l9SlzL1C=(e@415c2^IAzs`Mw#5^T%bXPB4vcMPX~;<|r~u=qI6%;x@ntd8S} zXnX+0Kao`P`W`%Y&+_?BRLv87V8cz}{77fjBZjWVAhxqdv`@Q7ja7GrAaZAF=hx9D z4*u~9dPN(;**WXU*zRi1({>8i4;m5U`phE7-S60IjO&czhu!o0yK;IH^KiXxHTn68 zaIg|Vspl*>ngbiI7zdV^2XvSRw3r7aHkQ9X!MbnW9V+M7Y32CB{o(n_Uej;d=7Y}8 zK>gd9^4Vt(1`)<1;}~O)a@+iWoV%{&unl+7eprrv*b9?*n+_fKo;Xh%hc}t$-rd2+ z2YrIuFUro|KE2;(bbQ{w(kXu4ZM=1>f2!#ofP2i^JyM@Ei|2W(e%F2v@qM;A(|yXF zqj219pD7(TzN?cRUFoORA9frbAM8iA)n9~Da$LQccO4z|{)R*mFvGvo+wHd+j*92e zsq4(SPdXPiwT@z+{4|~WEsn`?)kQiVvqi&k$dMH_jWv5Nd?!N5E|muvfUHjGo}I-@D`!EagadFacr?%_Fu^7zb% zkHh?|d*nWCInnid82j_ngt&1ccjITxrJ7=Es)!qXuYYRa84er}3U60@?s?s+d)LA0 zt|O|s;#18l0I8aewp_yY&_#~E?7Um~paZE><{LDo9!R>T*bU~z%6q)O#a-)Zew=Rh z)49hub=;5Jo$<)X`aOc?DCQM=@UHtF)&sHh``0|9)_$15cIL-Tm-g@b82V;AiR{0~ zdCwVv_XD7w5~1)b;DIufQ^t211c%;QUV*4S$5C-Q98fJV03-zOX2 zC~((&xAubu$+f?Uz4~W`+;wNV-{xS~u?s#(hSgpDOx5yX^A!=zP|TO{cy` zIapeMd*8P)zh_x}e)xD%?2Kc-$L6~VoFBj4r*9cPulK@BpVVIa+ef*F>sDy(_wdX! zJfG+0`_Oy?ul&-9dj|3zvrvMb??}fd>R{Da)T_SfiiTEX)mrz>b)x`BmDICu9j{5N zo^e1W>hnmi8i(92?hSX<_nOYHSWYu`zkVyKZ|pLZzvu1hIo|kwl`{%69YNQP&93iC z#yo}8uY<|Y&za63^^XFsH=s0Hxw~`jg>m}+d4tABCwbkU<&46(67>e_*N#gY7mC22D7>(-h6!M`wuwE{&phG@7o;3XFDptZ>yztN8Pu#yQk@H)_XLb zwLc8`-kNVe+H1ue?d_Huog<>Rqbc#cX`?%my9`=-dz3V4Klukw{U$KCW%-@e)M4}0WTcd7KT zA#eQ;J-hFI(jxxg_(#L`gA~ErkbR$(h^O=JlaEUW3Cci7Hp16z^W&sV5k z?1g2zQ$Nd{;e)x49=;WMyL)Tj$3~RCwa>acj~IHq{SH%%oX%S2!JAscx_|zdhT*B^ zO;@O#pN|h>2+>Zqj^g+G{d`yOLECS4cicbwhY7z82V&36$G56}zLk!gDNnSX!#AvRlnBM^ZB9s`&C_!wLJylXQ)#j4?~=?8_oYsC(?*4g%Y|ERDpDy4nC zajT`q!G3Sn_pkSZS6W*3>)L1RS*5(WvEw=ramj~%RQx;6;ufM4(uz0A%(-Hzj(eBb zYK-+Q!x0Fs`Q3KHOGo|TpeCQj+Vk$aKIe*fqmT2mof%)hS7ki>Bvf1PqxZPOHS9-6 zHg>CYj3b>}{qD-VJDcvGcN<53c1Vyvpj!Rg>-0Ct$*mO@@|F+rMn`{Bc`MgjJe$o~ zuajxs@|E;qqK#)bih7e!-9Ad*JjH~+4E=0(|K6lj`R#VTf8M?LKHL-e&c+fWJ>N9Q zKIQ!+*R3zyK(zmc-EsyK94|VeNeJKmhv^lLAI4+){&}& zkRaWI6xT_cWQ}7k6QWsq;T%; zJnEU4^-T&F*9Ur^=U|T;pH3CebDbY=$I2%q!YF)q>-)y@QT&KkKEln{?dl*OIbi?v z+4|caaVos!5#N^ve3y7``J+-voUQA8=@s;Q%Wpt_JH`7A|AxDU{81{}@*5XbKaOyl-x zXB5xH&F!|CswM-FfcBgWnAyvJ>Q8n^EU!SXx)gtA$i zA9?pSZ%Yh2`-B`?2g!Zh?senYovVGWD<6=0<@;{`oGsly%iXiP-s@9=)bF3|?pXkj z^$qf02Nk|WsxjB*%>}LcK1tSH{f({PSO?~v2T`sESbfy{<|mVwc>vUPK;?R)uU0MfO0L}X>&x~pEU$n?!4Q4&S50StiQ2-%vGmJSZ~$5I{@mLYeAg3wcB2IFulV?SnO{ybO+fSZhBPHut|+#HHd{eiBHdUFK_huF)9%k z*#IH4c7Hc-*t)}TB8dK*kRgG(KkM$^Jw`iS zJp~=iJauuJvPRPy(;Z|C@!qYT(N59#gMoN`uRm+g*sN|+)V3wrg6zI+IM(80 zr+%CU{ImG1a{Bz)J5nAzi#qhpa&GsQM^bOU5170{ri+Nbf{F$L@D$;l6WI3YI`?N&)Wm7B_#rQNWk`vPHe-$7TU|DgJJg-gp5NJ^VSy0~z< zSIbCqgYKYCmLr#{a*Xd|giLcVtAqSaWB%&nuENMMIAr)t8%eYekCQlIcr(#+cxue9 z%ecVb$yOvXE2yoj2Xk)Ge>3jO2)SvQ?C}D5b7Aa$VF-RS-{?X@vP`ndTpVAx7i(jDXBG1Uy9D6Ok>vRu%o$y~xYex6E2GPkUJxJnCjuEM(s2=U# zncQEAuP}u1 z!UU%=5)-Eg3VpBAU0?Azo53SEjD0Vr$vu0zhP^n(nq8t8Ekubn&`SI`LS}^mimep| zOVGJMXcc;zEB;WT0eI1%HzA+kR%7TZeSgE}#6(uD`d1>o4F3TRuD$JDcL8nJvApU2 zIIem}J^(pz1m=|H1T2F=wgE`$Z&;yRrUE%Ptay7ulcu(O*hk5VadJJ zI3=dZ)w?DWhY~gBbuCfY$6#nfZVwqp?sw23uDXOq4YjkI9?=2#h$e|$4&swdy-mJ+ zgI9*G_hCEBfo7(|nWJt}Jd8Hw%muO3K6bm_{_NFMC6fQqTl{7Cn_gojd+OoeD?`~@ zNyPAAk(<@3k;BrCE{6rHC&RF%G3pYy$8)Zvzl=XiE@7dCsG zZ{bttmx!`2uIIm>Z1{0x|1`p^jaaH<^aU2ayOQwb)=?Xs99pA;Zc`{@A*!7?>c$fr z!`0^eIJRNO4y@sMjbSZ~US*SO$eeDJX$$KG6PSPUcEoGV+{!uiT2nQnHOJ{Pc}f3g zneEqA2!>TXNIkv+`KqM2cQ4rV^(vMo(Mctj&8lpQ37^>LB%nDBmNnFtG)u)hmhC&c zh8>_|riv@)*nq(X40|dOm_dE-Hv_6(rpPp$mZ%tRxGv}dBj+0r?oPW2A?4D8OL%@f z7poreRYuUwbCX-{(m}5e`m^(*X^-sMOXCKShkf>OniwCuGyOMjX03g+k)!3XQjMm= zb2PKpqhp$G4x@1&8<&72xF~P%xNM4goi)twN)y{GZc){~Jd>(=GwBeW+-gUIr103o z>uCX>+!-hL)8)!J3BXQ=r_SrjgFsDQ>}^-U$?+3np8-6mqhU0!J@6j#@Sb5QerjZbT(L zZn619C+LDlPZ#6+mg1SYt$kjzf{|2TDoj%%BlYw8hR;Kc44*Uk=cu|;lv@uKso*eN zh*-whCR&Y?g_^(};xb7c2$ge{TAyKX>F404=kPooJ`K0kfPj=Qr>5+}sPLKkUMDk` z>mD!TO^?)LdgKQ|(4+4>`EbdK2O43?0s1$??XEEmZaB7*eC4JUr!!aFzVA}DPp>S^ z?L#Kfc0Kt=PevIA&5G(*8BP_sTd~*}A+&(*d0h@l`#3ii-n2?^B>In%2Y?40>=ER{Zgl9#|W!Nzn6G5CBY zSy*D`O{hoZaVi(UG#`>#K8!c3)482)$?J+z+qT;!TK>pnV6*liyMJ+l3%$5Qp}T)5 z3R`afP1;4A6gS*;PS7&ATjA5y$-}zF$T4|`ar&}}@zwTYwF}AW!i`n;@|nD5>6aIh z0%4`Om@dzCs>d&7Y#1I?2(-A*~ow{wv83XqZEgp&X6m4{^~}e>|0*w zJA@Vn`9*&Ue6=#g`-ni-WQ2Q7W%=biaL zExqJH?uBfqmD-UMH8~>_!^zQi_mUqFggx@N`ugxdqnSt9x4JV-b*ncxw>Hd@8GWCJ zYBkwHFqn#bnO-x^K+3JuKV}b7I;%l-VStrkAGD`5UVN+*-RIQY7g35-dZx}q!X5EK z|K?}|naqj#{}R(Bke1^qNxyJOIy7X7X;v)f?H*kw67wngt%XYvpIUUWNv@Oj6Q|w4 z3_?lw%s?EvBs!p^x4gTnUD=-a3kTV66L{3iptuekk7>u+YK}0@!oDv{cEOc4Lu=IQ z3RCA?-}U{Snm6d|$6pe;aX#`s(TbGh>ucDJyVIn~JLa+4U+TID7Oq&V$gSXEjeI6X z9@DP4sZYoJoLtFcFXc(xKo19%6>6oJeEPV&JExu}=V%Uj$oi6X2ug)~KQpt|c1@Xq z@O1X2%*L7dXv+j5`lVky?{JGN+Xyp<{gDjR$$pYyV;b8+A%IXPz4y&W-56sgDdFp+ zllna40_tQQ$LXSG^-gTjpv>A>>U?%P4S4f(;hDwj@!B3@)`Kr=Y%HfOvrg|>uQlxH z_G<6;h5pl5kcRwjpIq*r;}vPcN%Mdn2Oie0n3l#6utv8{F#&|je$i-t7+&mBtCLq< zwVqw^OR@(QyAB$KXQNq^$;Mr~+BpN|nL00XEULSV7N->tnk^9m=k=6eUU}0!JG(X} zJN#{1+MYE(yt#>z#JW=|cDh zVa_n4u`|R+Dl|JB?@NlEWARZqV>Eu=VQw;`@jDK`l|4LS4?IT8Of(jXtbR~3%8}DA zxDs=SRR`{&_sxF7=S$9Y?!)~hB?Um}LSJtr6K)|?SuP1?;F9q*3b zW()3^)8N_Dz!{3OORJFkY|GVEFLL9dHTX=IB)eDVIg4%59H4=#h}`f_-=I5$?Ih7{ zT7%nYWvqz0V8|XY?P%r>KuVX#=oeJQkZs@!+E89|2_Oe#kyKyEg|J(^6pl5arW48z zh-f-W`A!FS)tRP1BYpwP`3#E8*^Jjpw$QGRzLyr_H{9fnyT}_%(ZtoP2 zuG$XKEoP7#)$z_g-)*5*r(RcCMa3!C@DQ!yrTh%;mgk z8Oc_xy-KfmIOf(S1?Q&P)~e0UQ*wSB&znoMHYs>Bgp*98a{h3l9NRLh#cV~l<`fgU zUp#2nsH&&?Lor1O!e~8CcU|#BPSk+6>ilxf=}Cmv;riaw{D)vL6_gmlbY^D=q@^i# zMxkC6OR9Hb#-wTPx!JcwzBNU^7jg<+njERF8~J9XZDTL3XtlNfavT_3@tx^!zfj_% zj`g|nW4Lwln;jt!Mrnv$LM6B}aYtunh)#PFZJ-Nzer1Yg=Mnicb}%kqh09mEPaT=z zW?*JG@?%=#p^j=y=xA!SjWs%97VuP7#D@R7NH9e6>FL zl4C1rRcBzX(!eYat5~k;oQh{^#3ct*R*tw(VIb z91*d?1#{Zg)Y_LFPq)q&&T|=LLq}n82mLDWU|b$}R9EzTbczMBI*-anaVHV$wOEKa zp?qY67Bt#RvEe;&5CrLOMm-FF^_M6Y8%QoR3)YY79N9RSrFam{6u zZ00`46}!8Z9SZU=&2(&u$6e;FS7+FK*-&KPYkTV4MUR@)zO^#zX|&05Xq-^3IpF`H2QN))a8 z)i>!&N%Wj8Bs(7Ut@{djf0z5E=2Gcy`#mBHO5@@o(ka2jJ2mTWGQB^Wy31Lj)cUc@ zm94G2J-l?bOVb2d{$GN;d(TO|o`pHjW?2 zp%>mmqO+yyY0~K$LBE*CsuL_ZWY9uynE9R3>KkO_;HeR#Njp>{?)xzJ{ z4h_JnOv6n=sYy=BWj3CKicFI0Y-^{)*r%1M&@Xj{d zVA{fmw2uSFE>Xrl3#-jckTNAwS1&mL^Ll#m)J)o_O25<@?sS2ARQ(7uBku?~R1#>- zz=mX9v~8McCXPznt?D*Zr%zwEufHq^?e@6)2ZIO?Geu0XLIx1eX|WxXUO@6aiQrWS zw{<(ey+tNmSMc76!UDiHh2`nf^k!|cOsTe19$KxF}B}##R$9{2z zppCz>motO65{F$VCY@Nmc*0G5UH;+4cj36Y-M?XT<&)};9>YfWi>KI3F_3j%FkCgh zcnw!WFM%J`ZR}Q-?1LMS&zO$#@d-|*f2qTAZ>b-9Ia4{~!FM-4-yQbdTEp`Hl8R3o z;b@o4pt-ld(VZjE@NJ2AWUJawAYiiF6^hv+UO@5bqy|~DIMW-$QxbfxP{Li zl>39*=Lja{yJmcE!|2Az`)?3S-ikfZ-FR}}TG68sjLmXv&Xs-Gz7N}_9Go#Zcc`&r z><68(CkPx^b)Fq@X4@G3WDlC@veQcJ%(Tkox|@y(#`jI|aXe@54_*-Lg2E?ktycmr zH4r_L=|YY+-oeiQma^b$koe&|`x*H>J3=IWz0-a>?lJ2kh8}-0_a*Fv&4?Y<`aF{j zhFBE=nZ|rFjX7k6Wo)T)c14HRvFmL#HBA>=+IecSWCW0wV%0S_1vQ9Cu~>00_sMvM zfJW|UQJlbY1ZY%8F5_ldLsE$~Q`*8-f`tnB^EDrk!L_y5|VGxoTw>3D6=3J>=51JH)$?N6wPP$ zA5N*ADMHshCrbq38;_}9&T3lV@ZrOoHS&mfH?HtrF6)0$@E|u8igHu2Pv#oC;QS`d z`RU=`PoX{FpfB%?9qy!sS*5|AD%O4M`^JiD!!Ir+>2Lr2>lJo~Zl@YC10;jQmHxpp z)pu@v+2lFIjS$hMmBhG>)^#VD*aFnMdg}LMdxtp~xL@Nesp&Ebt zXMB6)cDHG(ZQ|tK?0y#^UvD?USW7z-!b_1drn$u8f`4)(2bxT#we93G zo#Ys<*;#!?vbI^*(^NHzwZ}duardU=mDG4=xIB=;$G>y1p5sv)QT}QCtb^a=amz?B z#l^7y3#R{9msb6vxbezbRW3tUJK%Nms)WY>=uxMv(jcYT)aiEF=-ww9S=WQx+};=p z*aG$Js_R45Na}sGa7JAgly3JpkgOJHU9F_A47xfsMkFqhgUY+BGD!<{qc?jCD)K*e z%AREn%vS@Wt>yMbk67*2y~3(2TC+Nnt$wp^TmX88U*PFx>Pgng=HRX-i=ey+6?P0Y z-gSPsY^85Fb+zm-66gIxm@fevsoz$hr*$iQ**GOv&Lu|d4Jxk)a%oh*8nhW zPIo;<%hh1-SN2#fZQTxV^h!$XtL>xi!2O3uZX0-7&=b4GtW2 zM_AjNYKRP_;6W+C$oY|t;P@_j*qNQuX)2uRTOh^5=~s@>IA;beO}rJDOL^5SnHL=v zdCSTmn}RoWC2P0cykE^LG;=8&LEmLho44dS-g~{Kx|5057lqmrg-EG=Uw*oO$OtpD)N$pV zPR&>bD>=6ZLl{RtS#@xQc@*)`W!x=hCG4&p(VFMF%O3i@5zi3CWqZMlSTDLm9{vd zZLB?ik^7o$7j%kBbQ+@-!Q9`cWL+`Dr*O^Jvn$}g!DBnEL>aj>ZFD5)UFXM%LS z@g6a77OVW-Q;$^1c9U*JGo~`gi&4N>Qxa`dFTT%>vgqjF>FyGYLyX5T;{x+Pl{1+I zqzAiyo{qZlA){8j{G`)0@d}oZGsejhDro!(pLtPdr@7wQqyWgja0yDHFH()s&(f&zgJIoM@sZ?lv&Qn zdT^*>U>_-s!$)mYQlzrpSbCg_Rw)v@%nuZF`W3g_CT+vv#cb6$wGZ#2lDM6X=v>y% z*eYol0f*@Z-$fneyw==WU{&NrbSGst2q2-`J!`$k5wkfRdnEHYMFl7y_Z}B5i-K;2 zUO=VO8>@6(569lo_rYJx=JNhbo;sr=#$$+K1a*b>qD|y>_GdG_|78VVO{a@$^+b@* z(UlY^PC-#7ibWMZ!-eA?ekPg8o=5yve)gy`r-&mIFEMH=pRQD`qio zxRMA(tBEag@~ zvOY-m56iC6(O%_SaD9rY7A1XdZ|omCR2La#OW&%E9zEup9J#`U4y7Br3XPy=(%9<6 zVgI?XpPn}zc1e71_uH^o0WB(j&qzQyf1C+R?QfIe>}yplv{@eRk8-IQjM)4~@8FC{ zU2kTNhr80ji_?OmN7}3H6-EA-t!~=BMhLB2>ClO1Jw50qU1TQ>*Ns-*Wn<`+5tRum zINL=VceBRX?k^`UHbrOwk3E`!w8|K$#Vz(rnYgpsMmiIJbq(Z_iZ9HBt*eAF>z&|R zQR+@CS$)w%@UU}pk7&D>`i%@(pUCESlR!)pVGB#y>R;&rez|jVf-K3w%nf|LkwFOl zRHoH-60ph0)g3)Eg9m1h+euEWgB_#2+;%S*5B?EV?&}nz+vPdl2+usHUud_G;O~(X zlh>lD5lq(*ljoCflHHy9Xrp*~j9nA`+jk6$h%&EV%~SxdpiXnXf_fUG9Zyztf~Kn6 zK~)dPMF(n@!p&J@LQ_=@1QNKvDqbtL;?HPaN}rRVR5S+XPCY0BZ>pDx0O!`q=J zb%gDk#c<0`2LS;yxvyQ&?Kc{K_wAycgl^*q`Vmyu!f}|>uQ)4iKk?f@#`2ngfq2pF9Ays-e9C-ZajU+iIXjFkdxVxKCk z^g<3rQ`2&A)@>9+r5r0`p1)Ck{GT~-%9@%}GN2GJ$+PkMU@l74j7ga8iNZCz_ z?6zjzR_rnShz~)q{F?47ngX4T0w(JalxOh?jZZ!1$ZDK=%w}?rhzJ* z>q7Cd6a8~WWfzP6D4om8j+@Z)!>Yar!}UXpkd*C7k$|y!dpFL6=5|$gv&LcG9BUBA zY~z+%EGAy|j&C8QH+AE`fYa#n$ zWLA+GKL?j_5u4?lgHRe#sq6*_Qf-KyI!m8@Zf5#d&s;34nY^$hC@vNUS zPt-L1xCOe$SK{X_98?nekZN}xb_tJaJfyV3>CmiT)^oK{&$>;0k8fJK5MOiTCgDsl zXNFk~#WwR_3#W(7tmb*t!pYMhH+G$n?UlFSF0T~X9#$Ca#Bn*bKME$c!2P$`F4io5 zp>j^PfSm2xq;3`X>}l6`yMM6{9qqQkie|iILWe-a@^6g0{e`F9ds^4AIQgmPA(Kcg zH;{ON(PFs~LwCg%oFBz>-RXvkbxgFL3-$%mJ&IlKrGMM*#fSZ3l+zdbC6lj?5l1_4 zcr7~1<^4+%GbKh%REi=b$mD*aj&*=hjtzonw zj29Q2t~$bMlM<8!lGL?slkt{?o2Aoe{qhWKCZaYonywoU7b9@km1Iw_7q|Ox1u$o{ z5WGQTxCdDYS$bCV>uJu&n~pmH5&RZ$1Ir0_n6SEihsb|W)-mC9lsXH%Yh`QMYo%O%;+ zi>t9jk+==yzWBp*X~3QZWY)$akQvO^+>P0Iil0H zlU~dhTGt#DoqvDhy`KENZD``6{fugt?XCb1&GDRSZF0up{@$j{lHI#bYf`p0n`t;& z^VKOhOh-Vkw>0NWHME*oyXl5^P4&blx3FCFJ=O+IS&zN%IVCRU961o#rmXG%4yclzd})^tZiXFjiq^2g|8?;|*a^R%+76Wt)=n9-e&FLDzz^QVh*&k~WD2c|T= z?s3#l%QNcL`z8!*d7SI-v=Yz1Nd|-9+%VX2K<&Ao99+-oup{Tn=#D;Y|C^;p*&Aba z1HeW1aM_IBl*|@hNgXrS3nROYM z*&=siM*373F zb7sgW-@ce>dj(*^Pp2}hSRjqKjN{#$&dV(b=tL=JKip@OS2r>T)H6*bn@)(2!$pzo zbK#f7m%PZla>k(yPtzu*X^69bd}8@)=UvOiUR}iJdD_uK50xVpm84-eA>N6X#9i+1 z_(96G(XLAFyAQj2u^JXOrSW)DBU|f-z5FZl18;SA!O&N$ml13G!|`G2cN8y>SS0W~ zSuXDKV#RS?l7Hzd)am3{+OB+fzB#v{oUD5Oes6cPb{}w8YEWfL$B?<3G*zDzi9{z* z%BsN1(RZ@Pbv9C>5%;ZWM_p)`bLx8Y?oakrY_r`W16pRgloVf6?V@*LQ z2#j(+X2)rP`$%JctVvtv%Fn3OBhj&FWN}l|dzf#1Ljb@DrVt?SPUSEpcaW7%bgr#E zR#gi6p`L40k5lRp_2R}ym~`d*;^0z=gelN%*&Mb6`{DK+Lr8ZY-9&EeTp&VjV?%{2 z5zH6>mDO_5CTR{W1iUAgOlenKqmO<x0wEh{thEC1!iYrF(1i+iWgKd(Q-# zsLUAlXnu=}47QeA@q2U@RL+ON>C$@7^@_}I{O81YN4vM%V>vy0VMh;DjtPu2^r7)^kSwmDU96zaX1nY1nF z4yz7<8%E3UVEh`y2HnoYFvKI6xMU3=lMCo}W2~mO43V<;4)WCL*RLBN+eR1#I3P3vF-?%ImxW8G(bUqGxE2TSA!fBfBBWtt zAmkTWlTO14Xdxf8Ymca|SVo#Tx~)vZoNr`t99H90`x`}*nNL&lGy@Z+luBe%HIl~$ zoDH+V(k-KC(mZ|E;^eR=BI4SjUkKU43Wvxk?L_sP7Wt}f^|Q8hcE-H zGnqOx4!V_*2zfN-?rQLN;8ImduskD^EqOBGEvXb1-2i%lkwWF-+2-Zcl7yrZnuWVozzTz=GR@A)Gn#PX}QU(AGdj0rz?-5 zbJtcgaJ!akxskn6x-Pa9yI0ovHD!Xe zMG2H?DcTkn4fgHBs&;Fdy_WTxW|dL{6CO6L-=;mlo|WjF|4c3(!@bDeEfz-Kr|UgN zC0Inkxqxo7pEc2*R_q-Z+pIZ;CqdAEuhSgXMr8)El$w)WIvl43o5K2sw$g|3zt1xO zB2UJJOvYFYreo}uXnb;}F=2flWX&JWWB#$l-FFku55CJ}BUCu)Oi<|OU@WJc3T=Ab@w$sWxGgABHQ zvU`j&L8>Q%p6@M4+7o_u&^~6y?oEqCn3upQt{K|rFG)%B$N9kPIwT~9fr78a!Mmt+ zW*i-ih)9ywlJ|;Mq`+d%@*EpqdvF=lNE5TU66;!RoA%;~{dspuWur2CXC!M=xx3+o zOG4iBh*xJejFNSQUZX)>p5sT+mO%!Ik_mT|OxA!z)w2wWjgvdfdUXrENNo2Q_BpLd zo!3}TPo6E-Ox|Z&4^u@PYhBp;hNY=x%-d+EN_1WD zn3SxMAVc-~G8hpv7=|&JI;afiN8jnh?ruXRuNh*}c}cxQ_r~gl^7^A;Iz#QlWt>b! z>=|uHwlvm$G+)tXB6ZJZH!SS6#@?trc!YQt0#C&@-7-{E9*yMd!zm>i zsMXY4Ts|b7{;dIx-E9r6#BZM?VOtsgS$vzqXo%LV_xP-Oy-nqZ_=jn3|IaGyH_{cl zl4`V;i^w@LN_58C!nTig-h`=Z3%%vyX|kw=32e9N$*YLU2X0PMGcr{!14b9*r+&bZ zJ4?M%)l?TZ`miqj(r30C@Lq$D+WYj19hkiT-Y-{v?&L-?cb5_hE1EUe@)S zJ&mZQHH*n*Bgsvtu~tR*#KHRvejfXfk2_5VeNFWh6$q$BCvk;_q5ar7SVpyOoaeAm zNnrJ1cc04!#;}9dQ#Rcgx;I=LiT`fV+Gc~%LLUY@eexYUyI^|NX7!>~w~kOp(C7kX zRD?XFrdn>|zI|Og2mxn&v#k9aC-%?(;WZn<7e=klMy6_Jd6~@-plAGAdvjdrk?6<(C?ftaT zX5KDS_+$Xyg8w_8r7Y#qm8vzFPu0z=F~3`j?x{zbuxY!+1;F{h1WF6Ke{0Mu$mnqQBmw(G({->uZ?{Y+SwaDd+LQad!T~o{Fy^9As zK^;q4ydC`ieUn~C%e1%tu^MS`o003bVdmK@X-p&QZyQF?Kb)OwU&SB?>hn0N(`uYM zb{pnr9S&!C1^}64-3Aw*!GTYhMA}^R;0SD^vJA^jGhfF)_KMT96WLw*&EX?`H|^wK4T9F=S>%LPJ z05)S-$(%SI@~c%EU(FvZ9Cf?h^zbX`O!FqS*l0~B!8Bv5eT@gNbxf-=q=Q7hbpnNS&)8Dp6VJfuAWlQ^5`yY zez_3Wn`Sp-Y}7H%8o&MCxmF9C8;91--5TSXXh%<9s?Jl6H?2jgFP3Twx3WZT1rYl|JEv%h8^b8PA(UT)c>BNH3( zLEX67LG@fbFS8`)kB|JAu4r@7+RnzURhzB*N@bz>?c*5_`4}~fW27;v65}r2Oz>KK zhHICr(XPJW#Q!1$+ZBR6-0nE2Gw_IePm4k6)?SuWHg}XW%d&Os4xUQYz9=8j?F)8x zj^EW1JFj0KfAZnhww_$Km8<^MVV&*{V=^3NxHsZ?$|KWrwEX4J$M`xEYGg~ik<~Uo zNNzRywp(h2tQE4wR8+gymO2I<5XGxv@~(lU#){LP#DkAU;Sfn3<9LqUXxhPM4Xx>V z%fsPp5abjWLVLvFp)%n1$>L!O8-#P^4&91{#q8Ra%r7NfccG z((}Hu`K7z>Y~S*yDChkuPQ0%=zvd^^TAcYK9NZHL7i-ub}JTlrMvmz1|>(Tdz>$O*Dlv2 zpOfL_Kb^ye95BLcJ)^nyGU-cIoCnpvNwpcnQ7xfLG?zw%_uAoM_a$p<)x|-IX8=J! zzQ4wpYMERZr~J6^+8syBqN{v*kH+mdt!JlQldb3~=nEJxgOmpkn=@VioFrH{!tdX? z?a3VuXYz!Q!=pdHmadd}J4$7)E{``Wb}J5d6H7msp)>2(jXuIOLq*jEbuso#qrYWT zMN<7&ih7-Saiq;><0XLhM zU(`Vlc?H{RY`^)#wR4&1RYMpo0{h2E)I7NiWqfk=DbZ;n#bU+#{mi#(e9SHS9w$pL zyWIx$j>9f@N2_az`FWV*&Jh!tCvhf8bEuw(m|br#Z^0+|a}nQV2Po->Ng_DcpZ!5( zRLG@=9hrp7Bhkk*(ZreVBu!D%-QD6GcF%&cR8)(8lhOmTl)5O)NZ#o z=h5hcGA;tVL2ur{1Qo**e(zJPG}(}0vA_FqKMRI`5BFO4nUUk6me)v^T5P)1a*gH+ zpG+-kvW4vqnP9?GpUk=)CNu01X2UdcHEqD-o^7Vw^z~>QS)Jn_o*CJQFCK~Q@ z+(qr*?Myb?;@;K&QYDp^(AcOsCz`V%&ThAS&F7(4+<}qL(VDj`0WZT`>eGd?QD$$0 z^qS4@BT&Ohz0mx+kv7ZE=_=gl`iOJH>D65dzIZ2x)A+K~jBMAL8RQ^dlXl&=DXv{g z3pGAx3VJk*_&h-^qx)#X*%Ag{VcIozR-;$tiPJs6_d;>yXsxflwY=qwUyVI{TiS)S zv~fB-w(&z$-OEUrSEY4$p+#4RRc+LBRh-ihY7iwmgS#ORc{4VsC8G}{>8sUc7>tZ(RN6ZYJqBGkVq!cm>Y0T=GjP~)OX6=8={hDEA*mbQ1 zwO-Y#S+n--3>-56>tre#iqlA&nl+#AtQMb`ztSKodQ)AIkXa`MN53JRZZMUaAY-%6 zr8maR2r~bH*rHh=a-&sm!A*y%I2oKZGt=YdD>;+g;WbcRK^L@WC0RN(_h@@f96Bi` z!B4voJ3{;YFMpw>dw(t^%Q*GU7tqmIqRQf2bjye!|jDd{NsDh?5K2NA%CU+ z08da=zneWFrAtM@Sw%#4;?!Nf$m->ABoC7)^p~Lu#~FsWmO_PzQ#G_J79H^IGUN2| zagiX-Bhjjiui)(Ov3$>G1N_{bj}qFa-1Q^Rgmh z#eqA!meYNbP@6_JHQi@qjT%-nT9Kh2nr*ImG*HJtrM;yXR9yTT{o_06LN|G%WA0au z!)AJvNbYY54yv<(?tisf( z{I*vjcjrd`K3kaDt;8+x=vDpYg)eG(+p>!mZO=%q!r-RjHu+=-mFk7j);kY6*}QGz z*i(+_Eue}z5(J~y7#9%PY`V(kksNX`?CPH;hB(GV+sqE#xf}UmuN)#G^9a zrtPBZ^dR6q7#KFj$(Ozy9R`c)G!@%L;~JX^SyQ4y56Rq?oM|4~aW5pgZJ*RKDOmbX zzhBd-a(nZk*thzVw-TcU=S*x!l$dJ8V#RHWl>NHs`fN#cEVzeQezhd(fx=U^pwKpN zUxB^5KR+1hbeA1g)Vyvm+j{^o2=hO1W5!3)2)8=HDrWF_*>fg1aSrh02C&?x~~>#6Q}Si|bm*>=YE@ zyzANauTj;5gInU)!?eRzIl8`TR)I6+nO7L^ojq1BE=Nxt+lC&4SiVTmP_QB`% z&}{AbnVHv(>&i8y(ix#%3~06DIbLk3m(o`$6==+l1>RRu2-d2@vDVX*Z(^^}HfdCr z+nCeG6vxUNYj)?+Xk)TnsB)F8m4R|e#m|4{aV#5b`sTHK{7FVjoC0b~x=uXT?KGGp zIbJ)pnkD)cwxp9`Qn8usGF8m>u|OS0{m;g*8g8eJUD>!d3r~7#cBjY+yu4VmSy2fT zXXA!88v{k#&Di8-{cI<5>Q#x7&!lWa2Wrxl{#h8a*}hc-IxIr-RC&%%X7jq;YsI?I z0RzcIEM?XlA8+GJCg0I2_~?zAHT8C|)kGB4?T*@AmRkk;Z{HbE*oi7bi zVr9u?6?jNHI=s_9Z9d7xYVzE%Uau-9iBdY%ToB~^c#J5-7&vgh?=gQY3NxP;{o!oC zYGE0aV2?0`nUzaRB;MhN`WRPMo#^r0ecj!!kS(Uvyz#B~Pn$iQH+ z6gC=b2AzM$C4Y}2s(EIkr*Fhk1<3ih3Jbe#lRGsszrjHN7GzrZN2H7_M+1)r!fGrj!WkJ7cRe z=00jIO~9e(;FugCClhrB$CknrOd{4a?!}F(t$EsOSXBxPX@)xE3^CQ7c0pVkL7VHn z^s9*3&woF^SeM+yE0d7lBx|#6`pHnRFTaA8RQ=%QQdm+J^VJ7#LA=8IblZ_)^$Ykn zukB#^did>qwPy?(f?{SUOUxSi;$|DY_$M{o$!TkT?{%{{OpEV~!x`F=d06+4=0%jR zXUo(U_79=P1eB?oziM#dLio$1!NJi%oxqdxn>w8y*4;L6FRP&Po)s=$#))l2*n=h@ z8Ag*I+ul!INCWW~DcWR-^-U7h=QNg-t*m_T0Eb$q9Wtkh_km<~1>%<$u4)9F2l^oemU)_wMx*_4 zV5s?ZjY4UeHlDPu%T!MAV-oS24(xNu+0{8}0SN>KC%Nw?p`YnUh3#YLep%dHtoQ4* ztB>}Z&{Wch88proLnz20&p_nLFy>0vi&{;}ee?+wKF&RvDFCMHwb=~fPox%Wx8<3h z^}=WnsWKa%4($@jv7%@rXU@nMfEc)wG?mpssGdiflCRUOe^=y-h8@4 zrlzzY#!x@XzUexTQmXPZv~zyq5&H}F#+;7KQqbF*Fe>_M*OgiQ;Ak`*j4MJ22v=}( zvNUgl$sGvCvn$h}qI^m?%;&M6=f`L`s%;O6I2)exlRiEe31PvsgImm`@;#_rs*~sd zftZS+cJwVH-8UEi#jHQ*u{?&7ME$tDizZ25z1u5?HK?I(L)px9MxEJQiLR^s;Zx5a zC-Ec1Wz$mUTQkgCWe%lA#L?|XHR(4x$4P=Toe!pdAl<>8YTE3m3tUXc!afI=kfiP` zZymcKAT?QaFRxQbu<$%JcZ%Oi@EC|#sx*GCFa=&J*6m)Qj^)jzb#2p%rWZ#7qRX#a zsk;d-yO_i?n;OAIHBQOi0q4rz{N-~s=$NIvx=H8AQ!9Barz58H6h~}xVj6{7UkR*| zju3gL>AR>U#~XLMWOEtUb#ij^GE2?&6xk)7ecC&a;l|ybJI(;TN9((AErX+4RB%00 z#%!0V{k{uY(zjUSC_lCc01 z6`y`rf11JC2O|!4(e!mB`%TNiMD#A7?%RWUNod{bZZ|y@QLDhtutweoeUQwOT{MKPtwab3nx{W6qirY1c)UmS| zF>zv8AsJ!UvEO2gv07CtQC+B166)b4S?0MH9V<_nv;kA*T*Lf=ddis=WzS>x9tw^F+pixF<5TjYdmkqYBh)w?Ae-5f5b=Oyq&&2C?Wrn(0`#I=&Z* z!~d~O-ex*oqa$;qp{%xe1OyNQhmY^G4k?l}4?G z>_ViYTF+~yXmvA@xdwN(WVU zHOJw5pW!k_3n5+L#~}PT%)@Jpk4xJ8D798}zGIo}Q|3!L%z8A&VRF=~PtqeY&dT;? zt{HkV+DQv^>aKV>K=6373x^dg#;JSxty>LuU9;llnWcNK#Vz^FyyU8k<50kP=sQSg zj%%?H4dz?Y5^ctt6@{oet8puRXNhe_)Y9#!l8|klLKM4$g8U=XhnyCozU)h5y3DSL z4)Bit>QT-vsX=!b*^nthoDy_H(y3!9JxnQbF-;luSi$=Ia_Kg^Mz`ppTi4Tt-e583 zqw%PTJ(A>qW&$y*-(Ak$?%wc~3~)>LeRZymFQZpX|U)C@*B`g_wXX| z&1aWTT}eDpdKUMpiDgJb6ciPm%=T&O#1v$H&}P%v8Pc8v}6^`HZmz8cP6V) z;T<<(#UoSNI%9fadd*+{H+xfWIYl};g~}VX+QL|){5B{N&a2X-POtR#OK8ehrE=Z5 z|8-k`sGw8W;EzQ4?+Uh0Coy76E{Ts;;I%hJ_r75JS++zTiln56({xB1M(cF9_86d% zD4QyyM6!_IzttHkkDV`R+)yE&o;XVua?tspm^Z#x17rlH1Z!38G)2`zC>8N+yhk*S zd>wd{5M(;FS1Y&sK)3b~&bAvkE{I{OI*HFo*4gH}A(uU*y;mvR_2%pVgVLfSBxbBy zr&CY&saIHy~UPrPTf66I4RBHkG;Z@DMLlj5W|Gho0GPFq%ga#!R{P2eOVMzPC9 zcIkm9zo6TpIz(>YMETHRYP6?(TNGehR6%-pvDq076737!8cH9wrxW8-LhBe@s78M= z%bF8_)O{yqIRN`XjQ5=Mf=#MhRt0|q5CCdrlIQb+&8iDR!{1M4a8xO>Mz$5*5^BKj|c_uT}joCvmkU zQBGohbJ*?u>N2T%-cbl+UjH5}b&7VsBtB6QVH%t@E?;`h*)tuTlGZ)4_{aguMwTM^ zw`cd}5)$dIPb})xw?lPXp(s=i?MuAq2m|a)FV_tN>5Z8m8d>^jN$!MUdUKiLbQ9

*4alK!GI9;sLL-s$!BfAng8xRYV) z0*5v%kvm@|nX^kuQQw25<(oBS=KCr`a?Ek+ybpnagP8TtSFNv_>t2WkHRr(vPjdpU zYiVH~-~Mpxe54j#?^9gEju@16^GdaUSjn11t$!42(r@zg*QDoXuU2eUJd@W~4xs6h zslUyyByR5c^0=B-vKTB4Fza^|m>xaa&}hG%-9=t$Iy{fV94jSB8z(6WNl@0XuhTh8 zDHJ`G&HlHYF%MB@E+NHcn`R#k@f>ENVJe&*nEg+R?RwfP>E2%bZtbeiv!WjU&gmtA zz}Z$_F0wSme0f#<6sG=X-6($Hd0=zDY9to3DKmw4=E-iwS7s(I$hZEWw0gsIF!f9I zn^!l@MEBIloV~KJ4;G*Cv(vO|&Mc12S=V}fOw(yOH@AlY_QkQ}>;5?NB`|8dfTetj zgDy2FyJ#S~WBdT~%BC``IGvn-Gd2xF>W9hf!@}gBK95`51bYj|Vlccix|EL#JFf>R za8Uc+dPE_}TzL9LKawqh9`U4GE-KT>!8Z!%?o8p+7E_9$lM-t+*ASJwxd;@p`ILEr zlxDnUYR~#P{k3Mg(j?SM;e9eleRlOIok>v&a{SuhGmy^fR!Nd=F5~O@tI22HC z_>_MoC2sKKk%dErK<0ty&7o5#*cJisb4fT7#4s*syX@>zCNg5ai1fp^IMOjmcma=+ z3nzIElHF7Daj7@~1=qb1L~I)p_(68dBmXf4AxRZeS_5Tja0b zr!cZig;Ka ziK?fGcjf%-)U8#O4UT|8*NOv#k|R`>&AaEu0#x;#6c2Th{ptAT-@Vv!DSiN<+TowZ zWtm6J04-`$dw)G7+G$FwbN)U+>JAxR9W)7DR>q9X^f?lK*NqtC)N_O;P48u(_qH2a z9FSwKBgaxYG5fihF%I!_1C({c)4Q5!{>f+(x-Ga$GHzZI6&N&&CjG|l8ky|yd&k|j zE>`~z1-s#acaH6cXI;g{>16`%w9`s9MGT}Ras_k-2Ee?jH{#_FMfy&Dh|Xw^&ftu0 zaUG8y@Jrn6&JRVOS3ke3PTw8F{m8bTn|45G-no7Zt4E>U7zLwWVYaaT_GR3HWODiC zEZru@tQ1YqkI9O~+7@k9E3U?LZF7gm=bv=c-SpnvV)XE_#8J*bv%qw~ee>LVA$gb* zPzbe5Zs3#0G_8Zwqyy97L&w`L8JdO6_;`-#+D6S}t7mJ8ITW$pNdMg|476Mi&x3~FrY$sQ}BHwNLThZWSGL_!^ayr`Qr)vf47SrO? zmuxb>8$-Z{A?{;(s%xhNYK}+7Cyi;jg(AaK24nnW?a@vkW`51r;LD+*98#r6=iXE_({Sq}bf<$x$m2SwboM zQZNp@pxc1B{MB^d#Y}^0^Iw-F79S-vS%=;IUmP@&xD*~%IOmwd1BQ@mG?M22)x-6^ zYqKK#BF`#setEDFGM`-kd_@su#Xv6KNE6B#RmgI8<=1esY_ds$AL(~b6^k%Ug?=EZ zFdidDV-|^L6vgARl`e@!{WD#H&Qu+qyAM2MXwNsTs8-pw!!902r%B)?ys8wZ86hT& zhvV1%MKe1hBTM->Z$K(m># z$rNgHOdA@#&>RzrhB-7K<-e|GTLgdjw0>^M&8$8x89F7(whw~u2G5-rhxu1caG#~k z&l$0&DIR$;y2HbyXvcQ-Lwu|PPt}RS#fs62>O<>1ks66Sq{h$QpWf*ueQD`DY^Dko zgNmJW5|dwyauu1x-HE|IgF)eAn%t&>l1pG>R8Yp}Gp76EX>&upggMD!^I}(fM065< zY?c?p@dxw3Z^$!uD(=*cRk=%?2`SRNeZwm>SqW%(Q?)@et(c^q4;A|tKR$E6uigN9tUQJRrBG1i(y zUkLT$o_5>rJSiTsdG1|gS8(fAv-7snB}2=VGPOI0W7TQbX@jQ}QCd|AqMLWId39K1 zt}4}Ns@+O`^SipptP-1V;Q4P$VQrLuYdiVqW>^~JwLiC)U<5}0eqakElId6M9YlUu)jQ2ZCwye3M))JEg*e8wPTzgG%^0F(&4@uDJ zY1#w%&;kLN;II>|WEdPBd*bDk(F4;L^g!Sy&@K-N(7ifVrXL`c9*8(G4aD*~$qvtp zw<%rWw;ntf3$s%9a8A1J!&r|Y{BF=8qM3@uU|`pnq!waK){+MrMRE;Z*5u>7mC5-;9b2NTuVDlX{~Ct9)QjEcUioIR zk>AW&D%z}{#RuNf;u9OSpIJaV$P0tJ39Iz#E+mxM_`PjCnhYMCjN+VzvP8E$MqzjJ z&$H_8Rd#pJV%aO9yDFmVGAojUDw<9^NhN`YBzE!bYPl<42_M}_rv08S0V0v?Ihf1| zlAWtToKbm98Ol9}%EYol(DQ444rO<`+_G9h8SW>^t)?R*nJURl@?1?JrutO@Sf7{Q zH3g$Z3XCmZ#ha0F9)xdl6AOW8Uk@m}kXeT1cx4rhSPtZqkO~$-!EIta-PLeVrqh43 z6%#8w^K4jUvN7b{*Toxm&!ndmoT7?QpZAj(}<`cz>~L{I|ltUSaihPbXdaNR)6_ zE)Jyw?G~1JIJzU(oZo&5Z{x)oilrW|hdQia~uaJ0DDc z2_8c0crF*NuP%{OweRk7f7mOmlkw`wb-YrWE0dF~Qtw(;oNV~c;Aw}c99%l09yriq zb!%g(Q^7L5Q5UJ5x7PG7tHR3^nGNby0{4n|JAd^%@_>=9;_3bPuN5d=ZNN>Bo_QZ) z=59Y2s5BU=JEq&z2e(xwrz&rZvTk^sIjLmTw}rg>F_GPvP}5l?#X~3SZ6`rpH#XVA zK{fJsbgR~g{LBYg-WmZ06xmR@z4r-Q&%~%FDz&w#SX!GhbRk zJutF3Xd0s)$aL-!X+D}^^J|hDnlx-faZcqu+esS;UI?cMA;Z#cJvWmKKdo*2j!9qr zm=PEnY?q!|>>o=dnkYT>r(;*3Fo6$lU z|4tx#PbY&$@0abk(_?z%Qw$wh4Z=_{3y-?-(|FKyb`3if4JuMcavViW`y8|1goYFP z)#q>>-dadM&Fa&Lk>;Q+OC)PcL%#R165X9`2kSO|pYFz~jm=_1^UBCfdhV|6?$gC% z8tIBhOfHOecGXvUQYVMOjRv#8EqIL;DodmdJsC2jzBPtxT%kN+VOm2O0{fL=xj8=U zD}c0-qo_6)h`q<@i(R?*`>O;C#Idoko%aXn8^%02&u2MFD-Nv%sbYY z0mj|>d{{qw%+ zL|6!;)3N*8O?;HOvGHhy%xjcqH0*{C?}=2d|S1XjO3} zeq3ZjGud5cac8J+&U}%cR7LJC8qJNH64c8*->7f4p6ild@43!3p7Xe5XG%O|66rje zSc<%f5ut7$oZC?Z>#GLVPM2cD`Oxe|_(!LRJ{9!uQ*rk`PcM)X7)`2RJ}ULADUq*X zycYYEtuNnhbL;4liKf3gJ^nECd6z;FYGlNn?4sGm4~0e!xZ<|_7N%%kr6Z*Z*N;-U zZDeGC+tsJt&UUGbYxB_+sPVGSHFxs#MGgdJhkfjIEQitkc9yy>A^G6eOnY(=W8*oiy;3x?~Ni64k(d=pS`{ zks^17=gjI&1*1}XUC+3)9`D2UHtlNEK?^BQ{3$8e--So^_Gg^3Cm3GIlCWg>exJHz znC<BWP>*K0KJHIjGB0KOdN4vCKmmVHP^EO(|!=si51 z;KrZt{pO~YUG&N`e!1J8ww*@fgDZ7XgBg>mg)u>{(>&$ancmqw?L|1z73;wR8aF6-W8Z%ILL=Q4%dvFO8 z^s*O@rlpk_(nN~iWLDecd;*#~IM_Mk@=a(4LAm?k^E}FwsqOz|8tYebA&KK`JmW;W z#Aton`Md5fy&DR~z}Ph?iYc<&ZMic{t9N2={mQs4^?%7rXl=mOdH&ntHTGD2hB^Gk zU}|hIYg8alKn#S`)_NOxTX)sGo|q}~-l;gYoJ;fObZRqUAo<6A$>b&RJo6kX-eBnV zAYsOs=KGjoeA2_Fl8x&o2e2}XMTtQ;KIcQ1d2{yH^7W z>_J}Ar>XIvVGwA-0+^3Cn#EpVdOsjNLKB=zq~a2$m}m9jrJv5dNe=9{Oo7)!r*PQ1 z5f9As^8PU{FLe;4+Oc(&n3;z(W{W*2ibi$u;yVTCYCDK&8N)u)mfMzHbrP*TTwp~C zs8*yR`c4k7qljS3G}-JW>+WipZ5bNxpKVFAkiEExhW2g!Kw6Hf)g;htt!m3 zZE-)|ZdtkOnWNG(ttLs>6tT5TmDylAFV2N8ab&-eS#$W3!-~`D#k28x*B1s{#$?t8 zYR7%hWUc7FtchKb7|b3&BFh|j8}lKb_F!uO1uFu)7q_Z2N2}n_ z@1Zuor^niPs_i^oOEwm^iDA$VJVaXaJ0ppG2BY`K^qljme(nv(R(4x%=~FT<_XaxF z%;BYFoe%RmproX7^5&2z=jCLj!!r6s$r9DYoeAf}=#+J2KidnU*E4gIOy9`yPd_&u zT8SV2xGHTQ^}H>fKeB1%!pWV@OTMS=H?t2%SNYqcC3o@(WHLcz09@aTt`pWXx71!w z7Q0T0dc=DcyJxbZ!*R%l*4*Pa_FAeb;dk5DQnIZLohdTbL1HQ60$_}b{n1@$szIj& z0`NiOa!a6E+$YI43!Pp1JpCg}AnQ#~&3tjd&wf=RqSSaJV~6LWImtXfRb}bZX$Je> z(VE43D7jkrD6v5&31lZhx%3;?Wz*MH6evNZu0EO7dP=&n^L;KfY`Ula2Ye*9fOZGi+5_=Vcjh#!L-SyRf1&Ap8CB{N>8z4 zt&7`B5MmYN?>JBit+}q?v06-o*7@ohvk`QHEQRa62}z9oYhWrRx;ocDd)&~4do`z9-#CFO^BRNx4yU{t z&V6#-)V(yXubSs!yS8c3o@I_SnqIpT%Lmh=4Pn}ynq71`Sn69RHj8a(R^#jh z6;5+tC0bvDirvz#=mbZ~^TSGsP2_VmW7Oy$7fYr#$x6s%bh=2?;O)R?U`}-~&%e4j z{ts7BVa7ye6+UXCep+ptpV@`yUQUp;kpqj41e2~)wNWmTB{jNxUxwzB1f;@E_}hd^ z6K-ub#J9O$B>C*vn>en`6}eh;MQx7x_~H05KjYd?lGT|g4$oxrPHm=4wvXcpCC`$X z)~3Y=NbUzynf)!|-4!vqcJ`}3%H2eJy<(ETZ(Hr~L=rD_iA=1$#S?Xph~Y!s4`pkquLFj2=E`oNqJto7p)D8#5EEmQ6da1FXBp z4C{%=i&RN#lvrB6GM-SxTBWe3W!F`?W#4gTT{soQrePVoQ3U^|9dmDIA9}JEC!gm& z?h)ilw~qB)oBQt3UfeD%_PAtijC{(tzV6{kG&`uz^$z{4YczuASCj!q3yLwD`9Ww8 zVAnjlm^2avwd@Qm)-~N;HPnAJE7?QeUNe8O^)!+5{|+OKjOrsaEt=&elA8;lr&|wt z__**-GHfO1$E!WGxmh0`POMyAjcV5g>v}i&qE!yjuFH0}YOUy3WA`lRUi;P8uy@n| znbirdJ!X_@?!i|Mpo^8`FS)+` z&Rf+5tk5Ah-N3MMU=J&9WZ8flZd`nC$31XzL=+&2m8y*bh=3QgK>A#*|o8x#=1a}r*|`|Hn1h{o3br~C5_$frqgK4 z>bI`i$sy5cC|?%M&Sz+NZ|6Yir}DYnnkpNrjO5*6`%L8OxB#_HEuFVI^LhpoWml^j zJF&7=E|U74e<>J6N@(2rItc-Kw`oVi#I)rp*#jnf`OZ+*EYnAGE1Kygy{lT?Nmd{| z$M0^IV0!;$M?!*WE^@!8NPIx}cH4)0*Ek zThL}W6%z@QR{L?*?jPB$ux76Ad|Gex&g@r{hCTDN`5ONrMJ5BP2F2$=@o_26nl>y3 z*S0hC*~w4eE~}B~ZYNM_XyLQ5r+tvGN8-WE;Ooo)-;B(1yD4`fHTU`!uReywZQCc) zFbkKGm#gz2XW}z5QC+D}&X?C%cuQlJNkBMgfg0IJih{HG>jCq&)g3BwmmQRY2hCQ) zB`Tw_bxu!5#cHh0q|AR@QZ&I$e91LdhkyJS1g1ztbY(iuH@MN+} zq7{c-UFwar2&ow$enEFb5%!gP0qlFBTfn+c)|L7(f}qO*J!d$M`o&H{07Uu-|TG;rD# z=0!p-U2=F&^HG)ts&7qYc)P_4PK)oGg)>4bWI6<$ta8#p*7NrGN>{4M5a`&=HSA?A zVjYX&*S^=>3hazwlSInyaI;Hx(L(vBnyMvm&(Sqh$V=pu8`YnM- z@rtx21-=yRU53T2yr;Rt*?K-cx~p`8q(m3pl*qQ2lEqx`R($0(^j#3B^;hMVg7Qn? z08VsQY1-3`=-~*AL{1lF2T6$pOUcpjRwo9$$RDi#n!fG3CW9i^b9YgAMoG+=eF_fU zL!`zz9tH`VA*aIA|g^kidF=jH)i&VB-WV^A&;%Q5ZYtlCdg?o+STKesLB#t%R zu1x+eL%-_e+J~~g;EV2JtIu+0=e?7yIIP&Nwmhohr(z7g(L4rxZ3*l8o*0o1Q&P-U zym;GiyjYc@J6kS9(#7TCazf~r-O$2B#)hrOAn7fGRBFG|<48Z%_W2_8BLf3yQ(OnH zrLMV?2guf)WX)DcM>mg*E6(uu|4yGr6~k(vq0M;^dB4X&`(^krNe-ud zrMgxF3?S!YjOyUv>lp)GsI`(kf%f!iPjC%TQ>1me6ytUCYAs-6p82i+RQslten?{= z#(5W`)y(&$G~91IXsN&&ls4XTU^oS5*t-6 zi0@iG@ItLz=_?U=VAjb4GmEq+!lR zayKucPLnnEHCN+qk#4x%?Eccues`IBUa^bqdCBm1g*W=)ti+SeP9DBIeS4j;qB|U~ zezc5=?P!JF+e|(Fi6Olhnh=fgI*G0k5N(%OL(XEbbN~!a?vW{$u41&Kj?tFxAR|+B z!@E0Nv$*PZcq@L0_E$d4d|at0BUhPWtIY7UDYi1=Zm!vdx0u$VYK>NM{bHa_p!PM2 z4@uLZwtN~bX3cR;891*C;IKt@6f#{8EYvTvR2$_ePA4ySIdG?YneH06wfPxb7z3fj z^3C1!ts%pJdzF+$4d8t^b07n`+ zXJyM7_&+O&1tvX7XFa1cXEkn7F)lt@zp;MhJ?w6YrL9TMY5|?y-?I@MF1)3+S~I)M zYWbA*n{e_FRSjMDflfKp>YJ)nSyklu9@8@5 zO|hYX_jWf}V3iV*#b`)LB&TsARkR#+#e={|fUM3-$kG>+diqIGjSH!Gg*&0S+EI{3ZnNM z&n))zNyY8}BcC0?y+w}V<+7WmUZwfhwb_#RrSfqUO>PF?ZC|~KYx-0VkOdUmdmD0f zvK}5&RgY+R1L0vg=&m@kaD|Tx1aa{cq?j9Ls?3GXyu@wGeVSfTdC(lh1NAkz`{dmR zOVeup1yTLXI&2m^b0mJ6bPTC<0-KB_U!ujWUGMQ@RSC)T6Z7k8s+)A2sPsdt%$K&B zq`^PkJ-B^uNlVeJM1@*Bo7C*NXTVXnShw>6-l`snZfhrvX(lUbKbg~ChV!%_pjKQ) zu9s=Om&C@*n5_^%YfPru%IEMxXhkK0#*CPY$G)_ylDU05Fe&BtroT9J70~|p;@Yr^ZWhz_e0<8#x1lD`VDt~&}E9**+lia>v4J5 z)y3UDX@4~?CVi-$-G^>%eVB)^MGQ$}za4X1@fnkyZ?jby9KfSk8r^UqBum4t4mMCs zpfI!Um3nq5*Y$SkQvhaIvtlsBEV`|BaAM#vQG;qgKxb*Nk?Nfxee+Y$_VO!hRB8L^ zkY$;08AdsSqrYS=-m6ieJs3ron%%8FT`r2#$h%!yt!a&?ttHgEo83qK5+uadQ$bcs zM8P$DR?Atjm&c1rLyl7uZBBvXbBt%|d>XY@qPX#rifM_E5uzo$dXc;);#G~|E{Lvd8q-5x;bcMq%Kl>C3YF5U zTSHPh^Sk2gw-PMUJADRdZIz>@E0kzQO}b|Sr{n`*(0YOfSpFwwb&QijSij`Iz)`Zj8cW5(q3uBBb<(;#Eq*T3YA9~w z>Marj3FlIM2mby;LCHFVsN;;i9FEKYZuXbU`8GS`RmQG!} zM}~DoZ$VRzp>>UkXcQ0+Px($A=ILH-+NX$#t3P^C$#DJmPqP_cd;+XII|T|-{;Iqyj=wb?Q1pf!EImb#f8tN;Et6|G0fEb~_) zDje;0(E;MkeDO{kd0-x2EnU^<6VSR<_7&*Y&|VyU=Qvr-<~UshOkvVop@UgdDZ|fk zrALQfrOhg8fz2+xytlOiRY%RDFizt@D;RE%KX6yzR9E7zV#!7e19|C+(ulXU;!+!G zODEf0c_Fu>Mn8dOE(PuG((mJxRehI zn;psB+-~9?^K;PGFxAi{)a*L+AFFo40(5)t!%&BQB{6|9*=|wqTd32+FfWWR*ZuPn zmswq3ZrIbuRXZzQAB?^*bOR_{YS&I!GjyyT&(T!s;`O_6hs%+`-(NcKJCeitN5Bsg z2#VKerh?lk$+>i+^Z+Oh-F>=$v{kr6t7s1|@H^x^ClAxsPfMezE;E?_x7_UxYPy~1 zqb6y2>{a9S>R#XHtG7bYwefYBZ!*#mfeyEW3+BQDPUvXcC$?;B*5*Nl{28>P531aq z3i;sccMB5Tr_L{5%mT`Fdj?Q3e(Bxiln&$)v+_HvV%m-D?$sLQUT&~uof6k@_n%g5 z?z0`KJLK+sd5vj}OUF$xK5kIYL@iQ)7#V`#!yp<*lpd(xzoJ@K?}11@!S=j zq)nM1nL|-qMGMQEzM|8@?Bn7b2f_48^5IOq;&QZCm~CxNzTZ4~FDJ+!&jY-C@S<^A zsG`Sn2CwbxXQ(dO3!7W44q|_DH5FWT+hT{F5VB147)N~@#y6kF+pspOWcs~S3>elMmSoFZSE_?{GzJX5e8~$|SzZL$n!s`6) z14S~Rq|>zCy)rQ<-8vc4Hn(w#aouk*(^jH`mqtctVq*-rFo0uAxidS6YDW*PM(MT` z=_Fb#(_<7mTir*^W2(Jr508$=y~OlNV$$$oG)eTiXFWX8FL=V>yEBNfm5YjNm{O5G z<|D0&n~~W}d;cTn#Zc?5=t!pNI5U!Roy2JFc|;E$>mte*)5H&oRLsqQP^-k2tR4r9k=hfxup)WzpzW1#jOP4T~2 z6GFFKEnQs+O(oI69hj{=kPbW$F^Gk+2eYc@@2qEt?3}!k87P*WYuc`U1RiTX3>~Dz z99`xM95BY!W^kVkWF^6jE)ZuNV>3IHZFjlM#LR>Ar_c;I`84P}Hp-`dyep5&HfNz$ z)mWC{EFYRCiNkvidU9}2UhvZ}Sj#FB^r~#jw%aa^QGv@r*O#IC*c{$jLOsCL!1vfc zsylIWXIWT5+6J@dJbmyOMISonp_TSEI1Qh0*h)RJ+4gaovHCbw=_Rf!r${+J{hAro zMtkGI$9w?DZ4=E^%tCNtv=t~atV_xS6oy-usGSdL^yBQV7|-}Fxy)KtsE?l^Br|a4 zg=^*o0mB~7RqF9~fS;~=>K4D_lJ&5uicXgtZoQ;l^UO<1J2d(imcCDeb zW2mR+lih>pvO_-R-7&AK8vE^77f+{)NjbtSVai*)^Sa!IP>NHlWKs)h!oo7Kx~3qP*UcG+!tQlD7L%*&gm)3HWF zmdBU(gR(RZW~^A1tUA3&Rihmz3E|S?nmy+`4Ne~}J%p|R&5D&06yqriEsXE*$+0RV< zX|Sn>i?7&5GW>p((i$5{^st%k91UKxb0Xj*P>`k*+`+M#f^bJ?Bc5=SZYsQPHws4> zQn6*TI%i44Vx=0>DgpP}RWD^xoB%}O464-LA}0Ay^iLRt7ITMnci1901jZH%-J;lA z(1O$|TbmFIdiztGuocb>BjKM&ri}ndtusgBEnhTjagB@z#!>o8)s9vKB zX!32a%)d%zT+i{^yvK9j!o-JU^xKQ-SIE-Ey>rAjhu<{0;0eOB3!!>6SgKOp4iAb3 z`lJRhrfQbSX#`SJl;oWlL;3ijJb2DaDuofsI{SRj*KsgZ9@skRK~ZX?KTK;=f&`Qz zo{p9jMhp~*C7xxoQcFLJeDijISJT=x?MZbkvHg0JeLiVb>($hhH~V$EY?WAz;*=i4 zsoR`ns%cQ^IQHWLuFC3=e$QlaDEl0}5|uR|vMi@1?%DB$$8c_ticx}?v2J#)mwqE# zdec43ZUJn_E-T--6h!bed8TQ@9=9FhZc2RFt^JQSw5NQvxc%Um>hafnR_xJir!|O* z1KU@2pM7iED*Bc87Vhedou2$^BjsK9#a=Xl!a6nD!A3nb9H7q}Jps(j1T@8)!ntD_ zTOsjV4K7DO|LuNli*Z&^w};bUj7_3?9gJ^5>;xq}xBcL|u#$}zlZk<>Ctr+9bTTlL zD^LCSFLH>#KDi!~QnwUUJ9P2$%kCi*_on1uGw&1H=BA2(?8E?^=G1?eh)m#-V1nK9 zNK%H`RWVPxvd;8OR-GA04zR0hE{=XUQJ$qs@b_x>>Zr~+o-oIHgN}@e=Hq-sy&`{u zytR{K%>J0Ta}V)okt~*G+{tdtIcuz09ix9rIDOH*n{JW6u=RGRXhO{9@N$N{s+~%f=BxqZ^*IdEs0p^ZBY1mkyyd z=$(e6hO!hqT@W{%JGSozNgsLdMPE7h_MEzFP3M1u(&hh+9vjOIT&YG-D0AV^)JO~y z>zz8sC{2rQi5A)y)+HeoTGR=bo^(XKYlL19&pb}#PS7&xG+HoioU}P)=!|5)6Y7Lr zODdqUfQJgZH5dW?rD${!lT2!WS<1dOxNFU;#x-0S>P1GIh3-f^G^zE>j73oJv26F8 ze=g_OnF36++(oxTY4!X&fV1;nnHU;EK|;fE3g+`viPtnl=fy5Yz|MG`K_h-mv@5&0 z)1vCYdb`TBG>B8Rs#VLWa8tFV-jPHtJgg?B^zeO&ThZW<_1JB2*w7aZ$(Uv^KYYwgJ8{O9SbAFyhlsCSK&OLe8KPJ#m$2O zo8b-)5`feVH@PtEm!T3h4keCh@aZdRC>Ik0d{{}$pnoc3t3$6u-MUlALbm37)o2XI zrdK{n!UdnT*Fn`nksWMANi{q;3yDhPGArg83%mS){CNJ=PMF8Nh;YWCaUHkEeJQs@ zSVG!H@fvOIt5u9}0og|1<4&M6uGF8@(d~@UoA}6})j62cXD<Ur~)G zRwJ8NT7c=btE}cD2Io-z*Z>_n5l^G7BTHMBWVo2pzok%&t@9c&*Tc^<^ySu!P0 zXHSeU=uAgI8Gey>$JTsdyd)@?83!nIWF)o3?fBJcAZU(W9SUXKd^w!01Vq*Zd@*CA zmFaxe*r1V6)#uwczRMln&Q-mm%%Lb13_jy162{b-BqtF`G%67^*! zbs!a&@8B0VxKGFdaqkL83>%TAuNR@J{h_6gVkvz7V2Tu^v8lS zm0ieM)Ki@rF+t8F#(A^jX)vyYS$Xa?Cgk-Kk)Ddju`$^a9antW@0g!Ztq=A8I)aNF zRHPcetUh7>T7pG?bmLIsrbD%0G=DCS79DqrXbPS@bx74_#@J=Za)lx=Dp#GD4ZxI% z4`XNyGME+BL!r>dxf{QL8EO?5Kl2h=IaQ_bT`QUh``XyoPj)elsKHY57moxzH zF}s5DxaFKRZAmaHbFC`-Q>-dGZrP%;yFW=3CF+-Tk|Nh8CZ>3c#?#P-;@g|s@Z8|y0ha!FZ+oJ?qNMDf&F7TfsbS@~)e>sblTP7*a9s^IK#UI|< z4_Io1ozoMuQVaE6*I`T6Q;!P}AaW@v869z9z9N@WBwn}HS<>4VVp7-iWL1(1z{OAF z`ofH7*fJMloSx4&N8n15aJdPf;A?lei$Xn8#T9Ky2++ph_r1}#p;T4vDVzPQY+wOEP^=2(Nl4smNKrbRnff{i7}J+ zIpZnz2v_W!ah8%9gS>-GHN2Dwqdczl~Cy%qU}mOO?XZuBX`GMU4U1;KD{) zWR|Se@utXp{IUegV0v6Kj+*?yr*|glwn%Iper4M#*m=>^kn=Q5mZ8MvSimlHT-}e` z?%JQ-LFhr%U=S^PyTl#zu+tq3f^lVxp~2BN#)a6Qwg!afu|6Ws8^g8jORL)_vcxs# zy0)94#i;7J87jV#Re?ey%6gM?@#q_AaCPr=`LTyF63!XhNsTl2CflWI4;aA=oXFoT)2sD)eC zkKth89KXbyRFD{I##a1JMDahhI}iPbg_eE72XL;apZ7V&U>K)a6|PXBsR|8us>1}P zFr7~_+_dRB*@c?io(G+SD{3~!JdfcvhPQ!I zS!ULkH*^=E;BeS$ zENI88vaiP!Da#!o4Nd^PsMGi-WR4Y(UzgK-i<*TSzjNz7)9e9?*SJ{TceuYPrc#tg zG2$-CRi?~7eDfB_FIc3M>1Y`jls_VcCBhI!RZIzSIw?inP8y3!hnyalD_V?bmKl+J z5`>e{R=w(H-*%UxqU=vEFX1VtiPP>IUXk0n>CjAQU7n4+5> zUc~_i_Vgzci+NE$U);ERUlMBvSB1Cw!v&MuI za5uf*UQ?O~xe*7}%=|v>9B%iU5Pk6>n1ECfFjW`w z`wdjefLsaH(%kcw9*?u$g#91^Y*HUY(aD#LHrH#JTST~aY8KtzBJ!Tx| zy3~gO`En!I$isJaN_lS)mtnLzAQ=97oTLXf;WjwI(8)xqxYtkyy8M}!f)nawYv$PJ z4CL8IoqwfrfXSr;qLpZah5jm#J4%$THepZGw7ainRU;Vdde35=dtm$f#ZZakmB*v= zs2a=i8XE4gyr7)ZW~t|ZjEIMoTp|t*^~215R^NRd5WhM0vUGysxFBwcuYw{1JH;k` z{>o}EceH+a-(^wkOfz5dEQj2JzK~|@y;9)almI%`$?okhKC)W+@?h_IqckWPVnt)1 znqcnueAxxDZY>6J&yFr`czZqLmDubfavW^N({irHhJ2%NqhaqdUX%r4Wf3|3RG>a6 zjqTj3YPU|LmEgU&cp*9>8fWRxZ3Pxn)xr+&5L2k#N8^ zd|cqfCC}DnNXl{7LL`uLi4M3+sG>Y0lRdRlNYwW_5n_whiU_{{gbI6Z@0IO~8aa1+ z(JIk8B7HbQV1h3{WP#}T2u|U4(>yMEB|uVqiY!9KHd^!vP{TuzchV4Lu_T!DaRmpr`0`a^xFBNzhP~Xr-eFVk=eFIC3pvgiJz)5GN^alKj`CAy&Lwn zd`;&b0zk|nW>6D(j3Te{A?0gjn(7Yi?T?p0<%%=Y|FZBBX_vDX4RXYv9B|yPXuk#- zBX{V3t*0jbH5z&JJ$LEPkXa@KT2cBDH!0x6AoEgB`|r}{eznR1-z3=?mh7&X^CPS-S96xMLPK zc98~m7GAQaL+xUVJ1Oc}Shq|(cZB?45*+oy8U@AXO2WXsa}5gBn-qD23fU2)6c7u_ zum*|pBKrPVXhEB7nrOt-))yz)`D&i?_lMLEcXaK#DR!~NKmia4$k59b{_5~6q7;(U zMDr*75+PjwBDPm1h=(2ll^ zfRc`|hL`7@_E9=(l}#`iA)T}PH=CBW`Yu1v=}-C_ii8^=U&(fQj#?`=4h<({1Q)mM zXF|1^N-H9~c#Zwg<=Kp}*|t9289AMlm47b2#`ChQ8GTuOk!*dJ)ckR*49hPm_>ah5 z%xF^}1OtY;WY!W3Lm9zCgLrN8S2#L{PV}ZJU4^-sad8`0n@ix#y(z*SJ`se-7xZFU zbQs=Fa-}Q-+38Z>iDr3`$jZSAoWB>;*K^CWS$86S!UTTkAKA1%pWg=cxWLF#;kh%S zLfCOYx>M+-7;6je5 z2TwP{NqlkV#@!qTk`~Nq+K%GA(E2T$4%?rp2Ya9BCzXGnxc(_cYMXZ2;3=x9Ts)mz z>sf}x2ed(=TS~xJ8YHzaj6|1~VAZp$UETRvs?RG%r)+IS3O&KY$Nd7r6_Ys{$k-Cx z9$xHE+OUc@1RwLwGt*$CeIL(Vsitl};guJDlxKe&&9?dOL4veXZw>1TJ=?k_m*K7$ z{Wbmd=x@69V(FLDKS`T(l7-S}lJlYxhOI`%rhrRC+GQ_8GOY=aAmRbsR=FAWe`IFP z5~qiRX+M)yhn@_w#m6M;qSmUCf1YE;W?o?ba$jGzfN)ZK7l&K4#N6SUyeF{pkeR*d zV^0rFGpt^!vYtFn z>eNPJV#Z--rEo(Sgms!j0VRQLt-pfK|E(QY z0uNT3GE_t1Hh*M`=GhOn7V5UgVDXe2ah{;rYg6>`J0N5IFf-LtGMFeTfIHkOE)P8O zl1}p2o$TbfGm#;&Pok@NZi+JmTd>&rhp4$0yfC7acEdExuuBXcx92n+r%(HhHCPg&)4wEzKzvdy3_sBOYPTaY zGhM2yk6H>hbMMC8THc@4MZZ;S>VDLJRc|#SrPq9ci-Ar4-MQFTkA?hg1|vMk5>!A@YK*0yHbhEet;!62DEccmb>j^^-}6o1%t+7PjMz~{);C8T19c2mo{ zew!{s(Ey$Wn&C=)lE(Sz8&|a{tc)cg^mt=s$vh;Gi!#Yoqd0*|UdQ2mZ(#8JXot8k_S){Uo#UGdH=7M4s^N{E4CC{_5QS>K_s;Gs=eK+zV z_g9l9fJVAJcqgmme zu2CC}x}%#VBRG#t$OlP?b8g~8HUX51mp*1au zQ=lX>9qO4;zh+9gq6tRXW%ct;%EpYRJ41!VC5bMj0qF%{p(40~{i@cd@*$V4PSY8(Zb#u)vhhcGEUDxnWB&2M6|*yo7VKfPdBf-&rGx;G2NErWK0H-h)^4FVWRI z+uv2r@hFim==i!%j!Vr!U)2KNZl(va~PmC=}98bbyd0DO%M; z#_=eR9~5l5hrcgbD_#ugiYOK7FSScracijXsxpGbmPFoIO+)J>*z$RT%WNWfII#Pa z`^kkFn48j`AFN|sJ@1qlRN62jgLQ4imwIKfbeiMSt%v)RT=rbUs6alDe|}3t%_$r4 z;gcL5 zIJ&luIb#C`_uxt6TRfeYHFQuAVBsWko{k<>*I38+7QjH(t7{`2!o}~1N&aw-84YK_ zclHExt;0Mue1o%9DgHyP@K9Pls}x)BC3XErx2Jma-J%Xhd*g7Y1VzoE<#4C$XFUqf zo!O|cGkh^wgK9#_S*Uj#N?&p%lZ$u*KdOQ>QcuGOc~^Y-25)NL^g7dnWnM zDPY#N2KZUo)eCaRyeE1J$A~r=ZX5UK>|gg%(HV>>>;|94@|c|zm7gX=<1ONkmm$w~ znkX$&z89|l%uo-u7~yf>o)0yf%&)^W7^2A{Jg`MPn6ZoeaM7uG-jm+Rek|uZR&Zz+ z_5>ho3)_F)jnu-WuEw|p{C0HXCC9WjiLs^JElCX~^7cw|mN{w??_=7}SE>rExwUSEAbt22`+9ut zU27m1+q7>drhC0Yp_#XowQw;D^?bUndgT>cJwAxKcdeLQKzkdx?LV-Mnjp>DdE)Mm zv*x8v0N&G_?pRteE~7WWVj5_u^Q!d}$$>z+=s)`)vERp_N{4}t;OE{bGzDD7NC5r{ zAZ&ORK1Ym6MTbROfx!Lo0AMj0X1ns*eIB#XxA~a`C0Z-gX)M1a)T+5VwN6~!tvSP)i{5|!pk<_7d zYs7Yg$_fZZ;EhCQrxD&d)4#))^RU_dvFAG8(dHL6c2LX}d@k$ZIhUAd*!i1)?cD+} z5?g6RaDsp)A-RFeI;htW7}6~?Iw0M_lww<&Q>4Hxtim|_gH`ywtv_i>?Ccn6_}p$G zO9jp&2F8>=H&0RmrU-q=d6sBOT$6q?LAEYn$t~RL4h7DHfxpik+yW-K*G*}q!LgQb zRPM$d-`{lqE+fBLow2=?C7@*k=-gw-fZ6280YHGJ{WQ}adnwyVvd`i4+*aJE@ZKdhkC z1}jdhb`7~q+itH)1yW5V;zG{R>s2`=P2_CYXchL7mqIy-Q_plPK#Tt9b$PgC*fRXN zrp?2hPN+62W_MhmFD?U~3`@vY8w(dsLUBN+J~n%3(yS70Tm0xv3hUk}(`u*C-l}JF z6xCj(@v&mND$-!xv_^K{I{os%l!obvjB< zeZDL^;hNvMSzj)DPK6GSSIgjF@Gy$p~qQm1&_Z+hrBb z(P&(bYG%D?+Bmw1Im7b`bRHI#A)z+>u}YN0f?G)RrJo|l$b^Y5)>wJ5{VI1yQ85)}gotZuyANtPcKj z%D$CVT78L}e9Dq}JZ^|DQA{*4NZ?!VaeoKU)9NIYdypC%KKp4US_nVJ=a#I(R8C2g zzlCqGJ9!||p~aQ;!#-IvtUI|7kV#nHnOo9fyZKRmE%K+-ssZyv{~Rk!cTi3jy|FJV-3p;KgZ+-Op*j4dP1dEulsF4ny-poTI zlECjzilzg}_0uAsCW(@=9t16p7c8_R+)YZ=&!?1jrl~0WM~w*83lsJsKl>2dUf;ys zn)mY^x9+i;#YoPS!OUa~n!f!!g9$5Y#{_PnCW1h7n70PKiZHPF2j-=}m*`eUjbyA; z?F`mZ?&S($igvbOb2|p$ZvPrTs3oNfvNiOR?zL(K4>&@1;nD+N+F4c&#DvwvuvLoP zV=8dz7eDZAhQ?f@L3nvp91a+G=d5*^8MQl|uHFo-fj0;3;hlI^=23C9AJ3u;}fK2E`o{gkyT% zgi&7ayKGMDvcXQthPP&Dz-Ip7v>7zbHG4{!@dPr9LqvHv1!pe&rHwB}v(9#KZRM8e#%6C{(HmlDq@!_HJFAI=bq|*Jg<54=X1g`{j=hO!0XT9?Q$&w4`tNn zu({HHEMww7PG;!eiIPy+wU+e=bjwB@biM??Z=Pq}OB5Vby`BXHe>#`iAbz_&n{>U- zcs%@+uYB43`fL#~)7VikJ?il6rxpF+;ONyP3JJl+y{N6_lVRsLI@J_4Sgi}QC)_`l z6Pbv14*!E>M>?Js7=b8FlKIyj`a^CQL$a0EXF1J30$N(HUO3GP;3@LbC^eC7ZEp{3j_#Y7C0B=&{5g~FA zii;9E3JLrN@zG-Q04SWeS%5eP;s0`ZMDhO%|8LO$$^9b~{ulngL;hdEf8l@S{x?KK z)X-7rADo3!i-}7W40HwTgL0t z^_8axGbBdL0)FAh6%vgvHik>xH<6rXs3A*;2Zu(No`zE5$Q?+-kekc`lsNM9i75Yr zaz~*lPMrKqeo@7<(D46Q|6n!lVmhr=xtD~FRui$CJ%-ra)Yd_wmtm=<6k3~jT0-!< zm-`g(9jF|84ki0b(>HLf9c4HFUlidMlFT6h=_!W^7Ni2GT*^KN>XGFTz7jHqSk?Sj z+d#-?8dSMCQk-z$HGvOM1g|UDXLSNI!*mumR+^j;G2T4{&<#dxf;5Se@@^Mx3 zT^fOp?-e&c=t4cdXzn?R9FSoCSgh$h5@LG?t*DBa{~hhr}7_qotdwe823?TsT2-O>xB$jw=_FRfJ$jkfgx>* z1Ih`pZaLB6+8eY)1eyrb%O34FCe9=0KiaZv8-z3grZ^*z%U*1v17gLjpL`XsRw(IF zBKkYT;V=MS3vM!lz>8;}v|OY9GKUxHttX#;5jKp-<501mVht+9v}XfKblNy`%Dx$r z{YpM4qV<)SSVRLBb6zHcUq3ekyxq)BJaNyj^k-v#%7^M-4;mjTPnFQGoUflTiyZOX zes01EqaXf7Lr{{{2Gk_-AG#bim8Fo;hkmC*!$WTkKTlP}L+ghuZHk>-^*~s zKctI-aG};$FDMH~hxdrh1u=_aAjm(t%+b|Ie)j%$V^cXT>N3K8tpnr8g_LG zBK*6$^<(o=!4eQsZcwh4mJPLw3&CDGz+}3fOhim~|Fk-WkYq&Q(*A_bj;HuGEZ*Ut z80t?T2#z;`C8OwVt8O*(RGOYxu&cFbo11_0QK3m`3C+PbI@ihlb|z^5#7qH!Oms+nwG;-?2HPjrSmOJOYY(!l!#gp57IVgvEes6bnUA zi+sk4<4db24zE%Ne2%X<5ZJhUeNa9t#(VO2L1sWe0u7=>fU;BrbYWr3?oF&}F6eZ_ zTD>EHP<@JxQ7Kd=J(RwQ`{=WqObjhUFJ=qz_PIQkqDpkG%>SODKD zM=igpnR>UQke?6llgDS$~yEH5Y~YUdc7;=BXPc>W&OYv z;-eD{%e;Nbw znjZ-Wz11BN2~yeVz{L37#K&3pe1}~md`&ckP#A$=O&Bv_qwcD&MmQ9V@nBTa{S|Z$ z;>)CK(#bO7AW~748UU&!ODb{w#E4N83jMiclD6?=<^z)s)X{#;T0&43Ekb|E3gkI( zyOa1|YQ&LE#kz(NXZeNXjbdmNz0O3>xlgPV2Im8VFcxKw9ehi%Jfwy2Yiqj!zgieB zllX8AO%Uyf!eXG5yPCd7tLuz6%ngvs?q+fU{puJ>`tDl}&NA}A^oCd{X+rU3kX^L; z+zpjZ75D1FsI6p!pGtwfQZU4!;jb11)TkW@&3I`_s^=?H9hPGI)PD-mUhvlvAq0&i zHkF^b$h=&ubN(~~$%{50s=id>aob&P!Hdjtj{#mjT>2)CCfhbpI{icY1pwLeM=nY=rZCJY+Oq{6tDLjm+mY^r;A*PB(RsScZnni5YY6XH^1 zwW^O8uInvBiOxEc`IpDxzv)xNLQyP8YwjQ**LEccnJ=ETZKFQObFD8!Low6}bdNkD z1K>jUOla~TZ_vi@BL09c3tZN%gxmJeY9h5(nuR=xMfMV=+Yl=9i(na7<_<%x=Mtxh z-MXtLI7oFsYRsh5TKHC{y`3PjslAqrg8H|@J2LhMCO@$XMEYfvq$WSdhdDWp?z%ab z^nMabBY^s6dj?VK*Ws57>~(Ip32OIFQ%Urn3&o)nS2|d*QOm*z6?z#CMKMwv`VxSf zHwi>S-%r~jX!kk#JX!q!pH=QpUS9&39&V!FcaK81XeQ(M@Oxit;{PE z#2qz9=_)u@W9w;1jcUc?;|}{;w5yJU!=x?|F-8ghMahzPWvZa=Q%dHqV?n_nnH3B7 zgqo}%SM8i^E>+e5Y?r4BTQF}__TJl?4sQH&a(lY|Maee&xvtRs6X}Xv)3$V&Pzg}L zCmdo4+IwrnRJaXSGn&h~rkFq^+_wzo259Z%EfQq1{YC3tXZN>$uwb6?B2_q&=nyp0 zlOYTuVQ?Qq3%OsmD-X*jN8;A+HA=k@qC@X>HSO{e+U$j{CvGJti>Q2JG=yMEgLdUkmUpTCci~|I`3DnoC7fswQ48@_`NJ`>_jf!yLq$4$aGE zIV=0C%$qy4uZOQ_TBYpJE_h#9F?c*xv6c*l9gUbdu>UEdrFaF=phf8%N)Y3=0cKa@ z@)~m6lYMu0Y~LaBsL}(Ya4iLLk^7=|W4RL_s)%6dbyH#P5?k>TZw3DQK(qj*c@qwL zWL$2H2Z!&^oC#YF8fLyR9iawhP+pu1}U*Sjc;48WBmzj77oOpEttSfOS5@W+Jyq#_@4)++JW(gFDuL&I(wa=G^{A^6| z(f9aA3WTr050elT`1RMI5b&SccuR63At3xd+bmccVtiskdBEjbY=~H~e?Ke)a1af} zsHHeAG9nbcK|IhYuq0S#AvCKzGNqUo5xG_^BJdTm1js*mPda{rP9ewV8s9|!?-gv^ zIRj~5o&D_2Ja>HP`G?^GpmTIurGS7EU(BT_RVx1!3bU1Z zqNQ-M(R09m0lf#oN!d$oaN6Zpd})O*gC4q43UA4Xx2Swb_*5l%a-4ay;6?c(#khTU zX7#Jj5x>?U+|&avLP65%?P$zUW0hS_j~&rXtj$f;Y8sG+;UR~?`eO?;y#KODQio|% zI;U{(-eLoE@-2(gsMQ@FUC)sic-X2b+ewm%{VV--ku;}19$+l}RcU=KSs|vQbP==? z$&bmPONP=Z2?2d^BAzOU>C6?t?uhI+zTA835dyt+k4$T7yV((|P@459zssnMJAdts zYJ=IO%Y$#c+m(?3dN|{jm5U@U(u{p^&J8AUd9y zfW}r_WP0U_#QYuJivkx&`5-1hP;HRoD}tvNesu81OUDlv)(wErJ=WxCQhgd6M-x`1 z%I79y_-jmVw^3dPe2b3-rl_h)T?o;-rgNpVO)V zB*tca>jK(A;KMp`6PPbri-wNHuj5B=njzNVl)4QvhTNM&EQUeNIS_-ry@qnph9 zEkW-iLC5ZHgQwgpy-LPE&g?RwXb}T26#9F>%Er7{N+2_qFOl+`InkP_Zaf^1*W8q* zZ$kqZ;orVH$vBSQO_=Ei%q_pvN(FC9>>4aG-BRuyX)Y^*9{`plh1v#eqGh-N))es4 zA6`yMP#jx#mZN^?bgcU35=zNHrx6q&dS$Su@%bg3V zn)(Z>H(NvzPYje%xb#j<7iB&7H|s}eO&!fVVn8dZClCq~J7`m?(1YmIB2Z&<6UX?W z2mQLbbMnZq+%#Z$WS({G^iUS~%Ug>oMkJ#tFhC|Y!f)Q2EXv38Kq8P7m1VOWLzd&&9^hBrerd zwA9*eac9(g0G+&s}4X6r?_gd?0M$+zHgFEj>$QOe*Bqv#XC< zSUH43y~yx=8XD3v9AbzG#l$CSsPt1UspV)+qn!-@htM8Ek(l%bclv^5bmMhIujFio zJ$jd-%Svs#8z&5DI$IaOsa{p^`L#UQpMY7&l@!wceWF{7B%W~u!e8JNKvj%DBzLIh zNEu}?pq4Jc>bS8{kQRJ_;aW-ni4DBxn-9L;U;Ar32m<+()zbI9q-ZF<$&lu!w{zsH z1V(c15Bsi8`>x*4`=NX!zB!}A5l+&zuzWNd<62?0mRvv@=)H7#c-pfOyQ~0JJ>)LR zzWRGVw&k1vrDQj!68B`DLT#mc3d3)uwe_Vw^Rb8h%r2Icw>4$iy6UPYk$->4vNCaCOAW#bWS_Gl~x4eR;!Cy)+P!*u#o2JYnUcll6x+h5wIPM_$Oc<9J zF4qz1x_U%5m66@(wnOwLOi+Dsfj@@ zCiwM`tUaNBh+477u4@V19<;9wHD>h9_;y1>E9VoG+~vjWc~l3Nr&n?n`E8LWwKofy z4wb^VY9Ps8M~zzdZ)wccmW~JlnE@WwXl7aJdo48MNF`|&tlZpKb1lwT{ zJUznj+HdAK^g*H8DwEoOD~-pc5TsQHB(=)l-JDQXwGGt-6s&vp!u@6~RJ6_%zbI5S zDj}!vcpz)0B0Z#<`6IpGLGx%Dy%Bh|JtTMMnnhf-tJG6gQEY2mAUs>5vdL*@&u5uZ zd&=a$>)f;Z;~GsISg+32-Uqvi=a`(ln4e^-!g$JVWxNKVls+}k#sPK0qP`x0I2N3M z;s|t@5uQs#ed=ihJ^T~k0hE+zQauNzJ{<;v(CXr=D_}v`P6FE$>Pu9@559RS#sUAz zFe%gl7F1$gK4(WiF3MF_WwJ4OG9CM%>pEOIO`M*fI z-?6f=Ye%&wGz)Q;<9n+s(W-VANdS7)@rK7=|$C3aKMAp&5&I6Z01jf_famjt3~G>cR4-w&*)J{GKOp8eN}q zif)%wlE!I#fBmnB@tO8#XZ;RVV1Zy#{ebfbtM=i_6N;#N(@<{@ zeSxLqbI9WE&cUOobn$jeb4_4x-a)lpr$yiBTphboVAZV$2n^^Mk5b*Y`zXv$=(9 zg5aXDTC}?5gy<5kp1obAVM)CJER~x}*FSHr{K1Y0DZ4-g*?jr7_b?y8;iCE7*l{G< zsfvIbv<-YloDP9G{$!8ULD0n+Ur$asNP5C_s!kV>x@rJ2v*k^kmH5C!vlL%He!eW8 zgqEKn-doG1R)0I$o>u&wxIlfNjOb|so|}f-TH6w^r*~!K$MgQ#YpxAIh@k5eiXKh; zr#C>^9o7$(4rNJH>4sb@1Q$m&a$`HkAM$oEV0k3;DAh#_*YkqR(#4qZ{%k z*2-di-bag@k1->x3HyGWykG!ca!(q2Q$C&SA$4Li-8sswbguD^gFheIPKMieF2Le| z$)h}SDCc?F<^opYKA;wk<0pd1PP1WRrodDAEu~izR|U3LU+XOut-Tjt#0d)o4E%bE zBUnD!JG$yq?jTo@kC^m@M$ra8%yMd?1gL#m+2Rw%UhoKselteTdBKB5`k&mufc{+N zwD#0;5qo!|v9c4qd{@*M;hprcscx*{QZt-V0s(tY>4M0g}q%tD6Ojr!Q(;~Ro<|MzeYuj%5^3d z41VtA?D339-4V6{-;ivGz%8XBmPUOoYW%fW7MKJ^48(sink(rMJ9grosJ8oh_KtJZ zW@dwW&wZ#e9QUbvkzfRZ6;UY#$jJL7ed&0qoN9=yofbWwv{qR!Ah~fPve9%2*!cF@ z`N8}6nQ=FJOZMZ9;}!S8eHAg37PYKe<+Vav>QZ#Jdo%ky`^0!J$gqa`8`kS{%0R8} zL9wCkvZ=Frwd6grjDahB<fz5z;m6iUm;Tw+nB1^`T-2lf(?-Lp0wr*o7hVl5y1{D9iuS}rWJrvfI$Y^Y(J>Hh1Gc~<; z!H3i}+rtY#BG6aWe~&n}0|B``eAmq5e?3B8kuihW|7S6SWTa@nmj#wtM!+ z@GCh7g4p36>1pIu9>h#h(NWs8^b@)i(ojy>?Q7_m*k`Z_idW5NSCELeK7aJ4zH1jk zXUdWk(FbM35R8g{+1HxJQ*#Uy0q`F=IX<4dk+hXB@glY0g=qFLf9IAQ?4+e>m znIeU<_l-);Jny{*r%&)Va_LvSML_*pxHg?RP;wpL9{iD!LfUNyq3c^uPsxyl3)nCrz4YdFCtOP2tfbE+e9cH&dejdkZn|GhOnY9Fd+uYXAGc zx%6=Py~b#VbSJ(|eJbORPu#N~LvYBSStslA#CG>0x3ogls|`E6zVP~i4}A6=w4V0w zBjIB)moqZRf`cfx42(hY>L(ykIa1dAiqgPe!oc6(mHU)_-f>&~rSNnDkXQJ0{D%2j zgGNX^q)mYT=Y|d4*b07$N4`{Slj?nSOQz6RWRX{wHmxN~c8xd3AOsgAK-LX;Z{W_J z^q?V_f{4t!@O>zXOVcw8UWSP4p0{vRpbrY*t%v^eYp3ijM&bGQHos5$Hy=!jPTaz6 z!wz^_%-zh^gW#dT8vvO76#g6I2_jtpgks|RNM=uFtA;LYGe6`SCYgzO|1K9S4k~~Q zra<)*a+GzSJ|gI#**@_lXGy`fGgl7@qEj5id+{-m^7C&}`h~UE|zYM@@ zkJQ$%rQn)8vK(9nr6Fz_9p>(KIYFd*zds8Ll8F8(@{;jdQE64LfFbL^&#DC6+UgT$ z^IXuA)U$t~iJTSj@v`QhV5o~=u+t0mHq+ha;T7q)5F~c{rXL8D1ZOJFLi!+7)4ma# z^W38+A>>C$^ja0N#(sw0yDFpg2MRA#G}-e$(RB4hCD?#yStCABgJaNB4`lOl{#5<( zIQ;g#+;cGND2)5pR)FVP07e^|sjfn_j%J zE9o5smWi&R@Y8;g15yJNJU%C!hD8Pf5TMmeK&3!?Z7-h|2gjZ&6_UeV&*ui?wkoun zD*N-J|2Y3pS=xI(zrs7ZcYeAA7|eaTU$1!}DePaE|D|3`&3+-iC9LMjv7sjb6|kK| zyT!iIzM#7Mb$9W!d*kAa&2l#&^PzgbRWtic(Gw5nkm5yhShgp* zt38KJEqf0Z{;jWV%MZTY)Va19D6zNNfY;SV&^TFuKfs!{Fv$Wm ze0f`-!?Z}L+MJGRnN>w~5ur-nXb0ymP3mLn^rQN;=+Ecs!WblCWzk`5WBD;!*x=R)XryUuBZ{co zSbi|%E011)us0Px4%0l5O!x~q6!O9eJvL}?ARsBhDIUcrHaMY(gkcXJTd0?8v0(zN z!`NG7CT#_`Srldka@4;knh*u{w;kO#2q7hMt?Be5Rf9p8Jx z*Uo@d{rPcE_-XaV)mXV%iWUe{RIz<*Wy8iOw>m^GnK*SL&sA&w^xwaM_)-{!gz{w( zS_?t75Kaq$G$PCEI`K%%U5kh`msCrV;b*R{o@?yQH5R8VA(E<49L#rcyRhYmkGgQo zj46KV>&z4$opok*FzUeVXUfQ$vh+^K?^D*2_jJuBr3q&+M0`etjFzoL( z<%>=Es_@pH@|}dyNluFk`-1$Z!1>CC6APGen-Q;wW(>|woE>s@Ms%&5hKJ>p(-XZN(A@$39nj$c-$SIDfGCxKNS`1R zPX6Gl?67@&m$*Mn#FvTqG{{=_*(^&DT^P}a5uF&7Jq!I9%Z?C1ow9?DahvUCSN8}( zheY&eM2ALnXhe@jbZJDNMs(`GX>3G%oQSUz`yQ}(624Ew2a5PYaqI@;BSn0rh|d)9 znYJfvoO|G7{37;zAQzd+=6!z@E^uU)&_)-a93f!jflo)e5!;mmGGq!zEr}OO88RG_J~aQR0*FdB32}dCM$Yt-0=x9Zl~4F2n(SO zvK$P?aXVZl#OlGFSjZeeM*@Bf&?L(Ag#8Hyrzm?}g#8D$m$;oC!RrGuJW%=6j%dKC zu@^w`;|>!PUgsTk*X3-Nvup1Ik=Od2=4TJDYT<_8gfU?CS$kjvVyMgt6J0W~5~5E6 zIt2s_z_QjqQD^tt?zr7^yX$sDB|1;`MENJ`Y23*H*xH@7`PlgusxUKWw<*YbE3rEN1H3Uas0zbm0bp*fFCh-8Jm4tHwGT7k& zVrxQ*yB%39N%oLdPrOKXSU^Jd4* zax01c858mkiX{NKs+EosR9tg+3Eui&TI~S|d;kP|p#N`lY7Kkz_U`W8-@C)n~VrB_?i))6QqO*fi4yt zj#zTpXA}0>gnc%BHutB5EJlPRS^#<6B<1*?5g#<-i^ly)fHI)TX-WMoYXkTuk+U>^z*#utEZAiq@bL|FJnt~MjA@Rz(}*Zh`GebRibZr_%=?R?591~)Kqp4@q7osXA0s+4 zq9-4v=tO5mbY?_vDoFzRbEk!i=ux4HlYgQPIyItKBf2%BUpFo9W`h(OW*Hx-v&jy} z9yGWO`d8T*=I)%Go(`S7X+#tHIH8Xd`Z%GFHCim8lM^~w=^fC=Ne3vZ>>;|zB@#M0 zp_8#AXDi}9=;VJkX;R@Gzv?LO1ipbxssYxCO@Mgu<-@g(YyF=pCDE+Vbqi ztCyH*RXQ(~Y9siQ`roKWrQ_7RFF2~qbe~2#K_t!&p4AQ+Vip!Y`)ZF;bCxhlsSi9lL3!1AbRJt_cr1eO>9Fv=ga1o4_O4a!H8R`lKe z-ru_QuUFMBy{p7G_zY<982*lNe!b%VM3$S)k}TqN!Gwg1sqyoO5M3|>3#8`){*8a7 zL;hRgzZL$sRuH|feAh{-{s-n}+6h2*J{+urqBVX`tK)9Ul=ZzRezzQ}WU?i&A!fZR zU*Qf*mvco*;Hc^Ntx|ohUR8uBR7Bd?$qPkTs#xsQJ`V+v>VoqZLVQIVPy z$w86rQ=|jQNU$!@mjxNM{;SJvy2$cNdTVe~Y?{tcDzD9_$===sWwOiv{>YwkkIiiv zL9L+4mI7WfqM>oWtrBJ6WZF+llxnpu}LL_f>oG2@gkrOk&U%M;3e^d3?p=n1U3l82r8g6FwW#Y3By5z zRwOS21qn_TTm^d3ECdz;j%LJ=OE`FuSaZ;nY<6V{LN8J-avF>#HNqL>df_T3o1AP? zmdv6us=7pouII`>rj&PP=m4NLG`;JOU8z~N6xx;MWBuqDE*-jyc~UOt0Ran1`4SX|o2Mv*O1t65$*55=@?=39ku_*^w|iMBPQw z!U+Cm`Ga#+9$AGZD8>!zAa6rtBH~?IR|?I_s=N-Yi;>lNY%2M6v^6bBvwQOYL>`N; zcSA0tRRpDqKRM*iLu!ca-)R? z+`NL$s+rDS4PoE){$I zwbIv%fa?uhfkVn`#E8?2k-y0%Ie|0EJK>*_1LyDe(f_szvK!C65yHhr{y-?M8Gi=8 zU)EQZZDG3Ae;*$8tus()b2ZNM#i6-D$?8hx9EIfU?^fN?tz)&kB$Ey_&~`9 zQo=XkduR6hrGUP6*U!sAIqyTpOY{Q1b;5)b7xDZ_|Kts-R`>r2jnn201ye%?1C;VQ z$*9mA$sj$Gb7*4MQy^2zY*OaaR-AzzoNrB#N+rIXD#aiDdd<@Ai&&*Fv2zV9rN7@{ zRi0e?{eUX30M59Ej%e~I@LqvAuFGXRr3|}dNNZP^SrWYXhpzbhAM_tpn9{N-ZJ3}> z4FTUhz`zZ5xFsbdNC(288Sn>HBC3GhOVH|HUgE16N-~zVLH+!=F@1f_|AB7#Z-xI> z`2TqYkfN=E{7c-o;phL^K^cg@Kv)zRVO-RO#J)&mkp#u0f7GO+;B3*ovZ~L{Q`LZo zCV!ZI@bLNw91IHiRV+oac*6S;FVU-4iK4Ka!PNVlXA+^h)9+*`9=$G|6U?WE)AM z&clr`Nt=>@WN`CKXh#y;2t4^N75OKi&;i=X`7k=cIhZt^aJ>G!HFx&_6IMmEN_xONEQ3+Bmy-uySQYB zk=KGBRF>X_Yf}aXm*zHDMx6}0X(R^_U>1qYmwO+=<6+}NS{2DVTUOAvWFRdo4}xT* zU5@AFBS{R}Iw;A_`-%@29`_-+6C{r*@(l%H5)V?2NM1ZBF7%e#~%KCR*|EyKBL$pO;-@4pz{kyNL$GXp0 zwdXoL!Lcq>_W>zvqWlBo>q*K<+EdUVNpMr;w@u{90KP8S4kT}NlKX+OeZ6xPG!!P{ z#pNz{TCckXzBrcyVV5+V$de!$WELxw&FTCk1F+0q#O=e`1 zFWy9%g|QX3R+w5Fg3)15ZJ=;KP~~TH%gZxXa7vSr-C)RQB3?9ku_ogiw#z1hMH9iI zN!W-GRfnC!pvikT(A$A?4y11~g(Y*rVt+1iFDbgmx@N3P=BhJSohi)X0dZ(T zr@nd<#>M_oo^?rxP~>=(R#$n8h7070acFknEbiIw zz`}zRQe9TUF3Wa@hXJx=Aa6w%3~epRYrL%(bQpz)c=8S~wo9r-$vBw{2;9fgYb1rz zN^n1sJ$N9Id0S`N`e$1e``RY@y6zE{aY-mK*KMcdwh(gmxGDLChJ1PC!=O84S`Ar( zhAg!(!VKB}h8RYNtV26PBP=mPw!tB4z>tM_$Z`ae%#cNCfb(I9gg#_#9)b=JS)hi< z)k6g8A!B;L@eMW?O(Kw_ttp}`c?i&1(py3&Kg2&rvq~mu*iKnc<*f8vlHgcL>LM=Q z6HXt1BPHwmI)V5}Nu4BQ_>voJ^Ad{QC-L>7;BB%3XlS6@)WgabhUjvM*Itz+SX`+B zBO(w!xV~sfcKt`W+x9_7C!Y#_OzWT-b8r!v_E+cn9K$ZC}_~5a6uqe?6ZORJ(>{qhEcks z(u(}~AgmRmLL|Xm7K=~aqcX6Vk``e-tU4fL(;>J^qi9Ly(=Lm+-S*d$tDyfTxY}%g zoe#XRnWRX*TG!$>d2Yxlq7!%auTR)X!8Xa8Snps`!YIgkQWJ%BR%gzzYV~2Q!c!}Y z#fpIM*X6JlAaj{?fB@AAUopv)(W(lC?<8v2h~gJ~~G;!Pz*PazK)F-&S@yQDNZu zT^dLq6$W;`FUTT-a)Fsw@Cq*KDj zJ|Gn@>q9pqh>|%|ojCJFK++@$T6BL&!wll{c zAd-!c?97gmG*E#OIU!F*lCy~MCHLRyWhE2j8eLhsE`N}gQSlRbQF-K2DKn(-{86Yn z@2q$xTB>BZKcavQ>qh_w!8l6LM39noa?Li$RHc~8x>`^TNyaAmC?zxHgKSR->jG?p zbiPPdnGvrkWqE|86e!X$CxCL|C%}PZ?FkM(5lkdBjp#-m@14_I5@Jd|tRiZmrnE$k zOaVI%^yCr2T87(H^u)lExH*}s;Jb>RtP*gOrl~e17336QTY!!^nV}8>yO-BtQ$i|H zWEVwRP01v=GmX~56p>}G(3W{rqi+LEJ4St^-4 z9{?{9?j;1U?Jp|8gdh@2D|iHlL9WZ72`?8x;zv*mNqQ=P%%Z?+J2hW`eASR87&+}s zBYD165})=A-0bqit)=WVj*>4@$gc>{NPXlVmNZ{$=rpWD?W17LuPP3JAfznAw&n5| zi|!+6ifBMx-~*&y6*T5e8v;%0x=4}|_H{zEqlqP2vX=D^GRi-|R3h*mXii@V?8@H9 z`$zK;yL4~L>fNO`x(Li&bb=0F$u3Js-`be6#P<)9!eO@(m`0T}=x6hToWiKFbT!}+ zqQG@b6!D2wd}Lu3k|C`lw`LIRB_P?dq+OnZkxarPpj-+3Yy-+R}ZBGdv13fXtDb@L#SZzUSQslUXCP%`NU20@+g_>h8Lulll1UOtZVt`0drB1$cYv@1bRC<3 zMN~8qLx$XONcAbzJ+;ct1`EQ(08)@md;&6!OGq7^xGY=l7d1pzeKCJF^@DOgknADjPj&PTS@p;a zC@Vv*-SwUJ*)+U|q!L3Y{%tY7vu*B-#^CKtL`Q?1&KJ{ih**H(Lb_BgP*M z7M^ej2%jA({Oq{kjT}Q%LPYW0VZT97>RYY?zTTV>^jJKLfwg z9$-_$Job7uvHZ`^cI;gxM0<%zrAZ_Vl3+`}!dNrAkH*?DWpOw=&d!*PIk>Gp1r-q1 z@~d&@+18z`OE*|~Bba!#`FMm#gL983n0$70!-v_)_uD3c-j}&1e@;sfLORlip&VlQ zk=A>&&<>nH7N;HBfXCqO5wS7PHW{rEqSd$MXs268RLQm@MTC+(J0&~dcb{}!Oees+ z_W5mNdUnG0gy@`Yaw;F1oObf|gk_aczz2zy&pgO4+k-&1(@6dEOFbMCMt&-%cMi&2X+ngfq>BwK6g+)pqH!b$?-QJK zA>q$%2v$h6Lbw%$yRU-2PX2{f;;I6l+=BB34~l45M0=K=Cx7s-UhvjI1m6TzL#`UA zu$wjIq;xu(E6$D3F%l>RQ-awfLDxvz!!e1MRD#N}yGCpPVo(vu9c0{D^*bpiv1>}@ zCZ++Ywl>cs`bH9cV>z=)xo$@wvB4#Aj$GBdob0j}4>>WUJmpESqeAvTTjEAPU{xI( z#z{~!)}M^?8CF-cMlf+9*+JDZwdrDFfz2Sn$5Qs`{j83Y18+ShA35d35=`Vz$MnWk z34!i1u0acz;Eb8nfPfPv;ZMU=74~6t){=jB^`aDr*uycWinyLGw^8|r~DHCDeDVfJ1yv)rc*I5i3kv@RD4)*&|-a_N$3c9kiI7p9Tm*(rblNLypkS`r;B ziuOQzQ6b~1+~I%}u-p{bw+f&Q`6n)Jcv-YhzCwYMQV}7M&l)&1_a8YUe!cJj6+9gn zQi21hbXcnR=fu$6Bbc|9NdPrOGow7B$PIW10S&2Gk7Ue?se1ipsVsZDR4%wg4p;Fd zY^eA_jAIpk=5=cx9`f%gzY<5d9~n?xoSiGy^D2^3f{&=yTOD4}z0gkNstWJ_D%GLX zP0;>K9_hH(kU#K17^pN^NtTrgkAh0kTk0oN%BYa>uTt>`|3XJdz|rcr1vtBtNK)kpnzjER_CDcXIpEL!m-r=$(9kxL zRdJ}{PY3mA$G`9>+3}j74SJP~J@}}~pN^eqZ%6|RB>shW#8ss+xqIE&1CnZRNy@9$ z9ksI*3x20uBjqL)8_I*#yA+4~PVrHn1J5Yh07bxhWdk<>AgI^+T=EMnt>Y|rLJT&5|vG9a& z-mF7-a%^m;%stpp^vKvxnRZ{aLH2O)ZtS`VO-Ohy`ZMpIuu5gWSAuXqwzH5pKJX~~ zg0lRYLsCKIN~fVFUPPT0I{bLespxQFq6|t8CYfye1RR;3?a3r!=Hbw^{h6}L=JaQR z=hshuP2X-!&o6#Ww_DTmvtQGfThsHaU(>f+)AOTW6P8t68qG6GJPY$(F<@k&kY5&FDKD9J-)qQ$DZe}YRfg`c{zOz?o4VLYa5N+H`)YE6ugh*% z!TBnGfcWC!vF|^q&c7a>>_^hlTRCvgnQyITh;O;id6e}?>7gMB*+N{Zs1lJLKq^ zphdE;Ejx14yDqJ1!D%t_NNmWc(5xvLm5Nm=s8pLaC*t7(*=BeVWw;ULvX^VI(@UQf z^qD4D3^)|=Cl$VO!jU|+?%KknRdrvUA>9sv`^Q!>Athif|MUCoi1{pzMe7QnD<5(B z{8MUG|Gh%8vPj+gz>rgu6(mBhidLUh7~aN)B>CMmJ6nclQAUW?|Hs~!tx1kD$=>T( z;zDdfy#I@x#LS#ULJ^Vm_ssOnsjs_IA;ga3_)Z|nU5ijM85vHKI=F-RGV^7I+Wa~F zYuCumFneM4k{Rp8@p14`to3^Op>LWAPK6I`43)7A3;-HTTY!(-sarIH#Q9kicfdIo zs2r$e4qP3jgoID2*!zL4orAsCFlzykJ(W-btr1#|N^C;ej)%aXnL^JLm?^e;DwPnx zF7!+XGv%J?WJRU-2Og0$LQ)bo?LoUfrFeb}y-QWr6sR!J1VL0=Ag`6RBUbB03OV2i zQgST;udo2>+%hL@=>w&YY{?bPWo+CG2vp|6_SoQ#GHv!U5uH*zif)ygQDH~q0i#G# z?Upy8F_0vpzulpToanRNZ4TK!+b*o6WER=+(X$)gwT|J8&D_Shi8fFqqBWcvnUKh+ zfG{o!qIz*-gU4Sk06y`5C`j=s!BRKTDh>>GHJz+k@RF0&cOql)_Fmwx2b%zE(cvQq z6s7pgYa=p^?si%$2YP(r!9c}q8AhelHCP2FyM(DT(wzg_*~)%cUL3e5r_F}!BPSuW zOzaA#PAQ4`p~?sNC`vD7&q}QHBJpTJe19cJl&W)BFo#S%Kn-RT-e^XTieY7SqHsZx zh7%bLE=U33WLMzzIB8{ox5M^wAsC9>hYGW>=lqyG?@^Vok;eT2*12c`wz)u4>1F;C z#nNCx>$IkF(xD;iYnRokBjuZz50<8Og{FSN@YCt=Qa;*ls*S^ZSLCk)7-6<(iQXKr zU?7P6^uxqC8kHuaK{R0bWC;lCaXDr|hN14B=IV8-vfVz)M(?$o`JGVQX?;8Gv-w-W zGR}!yGIt)DyQybZ<9V54=c0SyrZK+R;eZ}6$16SPeKF5o=h32?Q<(9S>cPav(qq*ejV-C*)vLv$6e$3~3>*|hyiB-( zMzj4gd4~31Xva-y$r4Wpe60bB9MplcB~!}4bv8FiN*1_nTOuV0h6C&;Wdk~(dvffV zkDf|~e>0R5`r%|h$;}fsf_51BRBW;Ki(I)jwq-O{9Q|rpjrSIoYMI4CzZ!qc4*2#n z4K;)Bwa&Etu3v|kPC}{@#nPf%Em1tIrmH8Xp84ph=a9LyWO1gPxt-Jv8wGq5pB6%@%CfhG@ zUkxQ%a%_3UOmT`r%PhW$or&#dXjK@8glT9@MY3Pyfv~oY{K(6Aod#$|9k;AvVPF{z zm*4k(#Ty)p$zIuyS?}F?M}W;pz26MOOAkd ztTO#}DN-4nbxkKkM_F**-8~jd?q5ck2mPW0DwoY$KKE+Wx8bZ{zvQOL$rr-$ozpAE z-?`HmL3rC?PaMX;6-$Ca0A2i>iyYI70&h%SV%o%Qvk8*DaaHbUJXzz@=!3Rt zs}&mmmF*Y(do+Hx8)~9U%WI!pPdJ&4k=o2%MBrmA)85!G`ED?CG3G~CrgFuMO_Hy~ zOMdH?EVl+hofR{7XQOWcM(G-7gIHH-YK*e zWU2kyrQru&YMc{Iv5_wid{1YcTxXqRVvRZ*vr;-&BI{YqMN75Y9}H6MXKaj*Gp=N# zuyHVZOO;h0D{kD%(YO+`rqe2VgGAc6*%r4P#)?o6!1Zd8 zHe?xQn%QpW#kuIri_Erc0xSpZfnK|8sJ7Txm$P#&GJ|^UX6ZzkFl~k|)=aOxnJ5_w zG;mC=GeE|N#K!65#?x%2vp25)jGfat8cVaugvT&sV{baA! zrv5Vz+)gh)8(XvqQ!jkyv04}x*I1d*er?j`8zUPqsm)-4|8(Xu5#@D3w>uelMX>83V1ugeU z=XQ?H)||0tE}gFlje(K!6jNht#O50V6Ix3zI$RPBC!hWV9bMDEdvoEgy z45fD=#m)j5dl4Iha|y*;;}{P66&p{p zQD=vlXQZ$}pqw)-M z*w_r}$WWpc*`x_$XFIPGJ9BdQo;e?#tLe@9{xkN>zA-S=LK%290x;Ndx_AD?cmAcb zB&I$>X6lAGNxri(duL@@yPFh0L^H;V9QKR)9WHKDpXA2BP<6w_EvCQOW;Kmy%i4jL zVX3HJ8v^f8G4=BF08vu0wk~1oc3@2&q9H^&0Ggu`U8P>xmF9 z3QfUq78MF*$0rbSVnt-a_QuVB_VVA%&HuH_S6_bnFS64&xA{$e^JX2;UE`^QJqKTS z8mQ3EIu35`P@Q^ncQAK$^6v6CN6P4qZuEwexfZ>?H}edCk=@i+>XYc3E&daZ)zK2< zsBBKmL`luPm|;)Owd*j|{`=t7OEfUK>zY}Fqk+6JM_pyL(eKu2+I70YAJ!8YHviep zj#=Y|=oL%rDmVCl(p*Y6w#vB^TQZGqoUeWIz1-S(ny;i;Ep?YLi!lKQFL{5_-%IXZ zB=Fqm?gccLCkLP>Q)`q5C1^*ce$1Ep1fIuW|V@ zNLV2hDLh!m>;qC?1ODR%$ z7`&^f{|I}24cPTQUe{*U;PEv^=Q>1Y&8NpU9v|?@53l_icnfoZ1BG$7=8#)G+7^Os zZKat*;Lr&oVk{08gI=LZ96P?n5&sanifnq1Lsc})<24>q+fx84Mv`J6DF{Uwn7Won z_@(`o6`!KVfMVDu1bo`Yh)={XJ;+nKoOs)Kj1=Yi1j_n+S%}2m_J` zBp0T*H!m{eon`>Fz~D6`kVrl#S`7o!Y%OS#f3;9FBqWYTi&(UoQ#LJA)EPQzqm^Zt z87U%OJ(kQ6MFwh)&`A^qj@ib@F(5Y?G6j;{F-#2WRumYE2}8pgg{FCtyBvbRu-47U z%`xl?JIQ!2434_9!|c@@;l4gMv6#dhzJCk@$1kH#5&0$ zCv0;t1z@vl$OKHLq8R|WMoU%511bv3izzVb?@Ia|`ZxcNC{*IJYm|zvT>Wl-AAByZL#`ftw!Q^c>y? z*Y4S+n-EW%r);jr^wFlj_T5pNQ6K1?O;2s;7wDpGk!A)TZ8~Rf^3G<^B5E+qIeYZV z#05CP0w<6J2H8b+Og6s$+}UQdDXpk^JH_P1I&?b@K>Yc z+1bydyFg7IPEN#XGkEe%2@slNP8htD=nNE^us;oA|7Lx(caeG%4}>Q1j!i_}4Rf|L zb=#gU?~X0Q*#*h_(WTB!e%u7asmYW#8SXccgn!LJ@LwjEXhyBd19qg&0l9K{$4huUA!_(3wEmrV zdZWm}BEQ5($#j(t;HET4$o)gyZU_-5N%|rPx4_k_)-i7OB5by~3Q5F`i!<=5P!POSSFpLnzDZhBB+VE#GQ@P=T+HQR1h-;7PQ3az`XW9zISZKo7htFqf<~)a^qr01B=|)(syx3IqoTv z=uQcpLTBE?hHfJ}3ZYW&e5fd7ch z5gY+W;It;}G7CeDHaRU?)au6DNh!(fA~WnsYCUCPxfB zu}beaB@;uDHyT5gaR(g_VaT|kWJutUF-97TJSWxx#YlE7qK7GX827wle2HJ00)(09 z=PCG?Vm>M;+9KQ+(YRMkuWbzK6}iLwGJ+TO7Jfd(>+*JzhnI!kB7ZT))mnztBIBLO zN?jPN&bkc&u=qdYU+K=WMSLd*JZuC$Df{gbr6@J7V?o-3M#mr)0f=G`J!CiFfPTLO z8ycewnSAROR%lZ|A@*Y#LFlmWjHNS$3u-j`A~w+3qV=pI0a3k4s2j=r&4T_g+7Cp6 z?;?$0lHNxb!O;>JSL7B^gya{kpjIJ7wmF7Ki%ReiG=w}M^N)X7CIl^D;%c5KH(Rw?cGF3mn0? z5!(=a+A%wQr}gxT2R#Qh8YsPTy?5z!ij- zbdOu7?Jb{)T!L^yHXr6jRg3z5na&Qsnh!I{W-o+f3NCSLW_xM(GVV=(6S|g0_8Kpm zPBl9TM{H?^5ED7*M*>jli3`c+FI3(p9ebnc{Ed$Dop9Skx%nH(=I=DxohtJeTFf=V zOB7L)m&<^`3A8OWo77ujqyYr3iNX~P8vP9|tHpK}F=9WvziTt<-WZA5HBw4I){L{8 z$+n4~oq$s}-{nJ>-lckeRTi$YKe+r3d~L44{Bx#PkN{nm!QGsY`o>yArDG1>vyp+2 z^4Z>5F2CU5vhzNAwv=I-n5dM+9oV=gJk`S@L2LN)ZQI{`&aeB|ErIfnh!b)D+`lrO z$9AvV;RbJKo-taoyrwNSLaRbP1vQ2v7++Ul%}Zz`YAbUdGmkE>nV zO3@(Y#Qs&=2+WsooYw1%ac6opKV%EfKrpTHcKaMY&A<+m23!oCp7S6~LR%MfXd4H{ zasBKT!MPUXG9MZ~hQS^9+$OH&p&D;%xflW*B7!W*!M=cU4u2!uyqUaMo+gi*;CR}e z_IXSB%_cawI+qPPeGm5?x@$7i!QO9gzL!<;RGEx~M($gf-}O1&2cLl5YuA==H-6r@ zh3D@O!PP8bHMnj19zSD%VI{d%R|KF1o|1>`9YzUxX@o2IlI^yc6JhI$pjj|qA$z-k zHnUaj!F$jK*2wCE-5sG)pTK;9thveG?}6kVJ{zWw=etdiS70^*@%C_1G$6kp6x#^J zYcT7d-EtFjD86sec@x~X=mg~c!zKtzFwzok*EJkqY67V{q;E@5?;t%<2@Z?-?YY5Y zV_RS!Hjcp7YW9Qb8-`)R;$HP{@#JZ{5j%9dvuD|iC;`qiz7 zrn;P<>@-rYY(qyD zpi5lIe+~=WLHX^V?h~{9s$WA@+hUT}{kn8(d)KcEB?3Kw>KFOW8)cPA@ z-P(Isd_}BpMy=}tfQ*ZcP7<&>iva*9wrs7?(>)7UXXY1mSCF^|A<8p7qgVcYBuk4O z*c(^$Kukn_A8_*h<1?>V)@Wzw4`KTNpUUnV`*J@VflB*1H6BQ$n91O6y~`I|e24}d zIP6MBSYo5*hKv)A;l$gB$2QA#mRje2w#jy{VAl&lIT}>Qez^YpKOehg9;GJvG_v+E zFx(prIt-(351h?M60B7HLDuYBnk7+^aN?~V0Vht72)07)W)VsI*V&I716q};fQuoi{& z79pZN8F|xs0krcTPZ`6Q_BG$so9jT`({`L6HGeGP>?ntMpv$wNh>!gWCaw`pmvLf! zYKF_nbNA$45)Mv%!B^aAZs==!+W00Qm_cm2XPF?xHqEd}VLhp;7W?_Kol|3x-tK1u z;>I>@7X&gk=*I!`Cq;vwp`}&X`*oW}*QdYDe&YeQ=VfzCw7d1Wb+_ri&b~vmj6VIx zeflw}Zk+w?-SnF341Vmtw;!D>Y63LP(*7^o!|#2mG75 z;NP5j#t!Xd60VcvKLGPK<$Mp!#ZKVu-9r1%w!LkImZQEmGDx4cemZ&RPt`NzgZ(a1-X04`Q-wigmZ;s4(ne`j-fO!zz z{PrY7##waxA^*9?Jg3wtqO2PN9*^v_;`&&4>&lpS;FQ)RNhcw>F`WM8A$1#*jU@oY zRfQ!aKcq(VZCeAv@dUS5_9?q%XWkJ2XjAEl;LkM8w(hN&tPDXTx9;d&*DmA zcs!D5wxQOil$E=%vDB6fUoXW%=P%gewkAtfN?#VJNY91G&~{gAeI=wQr9Y_Jj>n6%>o(zX?zYPZba=fWth1qBn|(Vg2f z)bx8oi{(BLTAh}^lDT0Z;@>lEBTv6Ak2-2v*C4ZvotZ$!ghs-- zQ})vO-e=PWJpBAZB+wxtx07`nF=|J*lW^WK4J+txGb0x(Gpz0c{5m@fxzt~iW-EQQ)xPpZ=|9dov@5cQt zQ-;;--)gvTcnVm5zu}Vvl<%mO#v0$|U}BR$kX-R)O5$ldY-oPyNhhBq-s+$3f|o^? z{1rGq5!a31e2*X6q4*9z2$7t(-LaH89k0bvR*L9_v*j*r@aI_Qx{!A&V!Q3rhjQ;x zaN~z>h*jbW?by5`;2YP-8Q7-cgIUE4`-`1PN!53GQ@$<$M?kp0=(1O!)BMYHeH05f z(Qcod;pazNvJu+XTh7lC(XH>80GJ_o3VZ2ly_le)@Vm&D%C^jTAKBouRrD%9|>cx{cJ= zG=({8&%pe$?wy+06vj4_1AH&)$x<2fMC1_$+h0@v@PE&pu$)|ygEqWrzOv-67m=&nH4szQFs|%2$0m=ZW8%>|= zwCg!-{SRXS4@|((wBIx+z{uCA*H0*WqYmXeL_n|PPk?!y^?ictw^Mah9WM9!Fn=Ay zuKRLj7e)&j8vku0HY75Jt9Q?U;qNpquLEK zPQG<>1k?#mv1r@+FuAs}0^K>!Q$5?1X}wt?ah~;`Gv}}{Tx{hYPP=X*cmKHYt9sKw zGzwKZe{a0X=xBULymoR{swtG;8zSo_Rx%~xHq0l`4cPxGRv#>3IO>uOW!E$qRuy=y zk0xBA-;5yB7hmlyM3J_(AIt7mFIN1%+#Nk>j9#XM|MxY8O*)7>8D?@@96$ zHn~tw@B$koRWa5u0AQQBVgR(!o{{b>C#2cyP7h}|4)l^<p&0|t5dt+D+eH3 zA`$iTl#yPOAi;8XADn|b#`W2~9Hfk2DTxB*L19f`CRY84duD;NOoP*zrX_S(feZT-OL_rH~!$;b|$f^tUqCH zmQ$y9hh9xQ+ZWzG+nDuXL~g=XH--L6JvJ8JdZ#Hk>Pj<{O%ETp%U3_uB{lb4_eiQ# z+INNQ>@=^k0?UI9s0PAg$?ZB3GpoEnN*kp`pDdkL92t?4X%U>jXPmZ3A^E^Q7}t9v zJO zC6-#tH{!%##Bvw>t{IcKbV?d}XN?xQSGcCVk7Devav@cDuYIHiT7TE_gPE6!4-Upz z_Nad_ZqE;+m{&=|&*$d0N1pRRHnKeo&YF4>JTH`gV}YPpPW(iMSRmNn9P`q7eYC^4S}_F)!|e|8S)6)p_l=%_w%ykmjW zq=nac81&wj9K{a=sRJhYXsu|Ng~`RxT3g^0QP2ByqJGO>w!+&t{PZDGdzKU|6yv1U(=&cV z5k2cz>OPvGm3yyNGToUUT-dH!IUCKUX=^gxVKhp+z?XeEsMxrgZGAI#RXqH>@6YfN zx>lZKN0Wq4TlT7o?G5bX_i3ap9U+oA(r(N(s)u&oKBL+AVhVbTt8tI)$M+{h;3>OZ z$6fB@Mw(4Qc^LPCmtlzuF8_K}(tcl+SqRu0*vIcv?|Qyw9rgKG2y5*+zpV5k%i(O( zJPdoN4`EQZnIAeDv=eI z;5d7ho@0}v;Pne(E2a^#bvxZ>#5Q8C7#eNJzyufBXm9PKZeqx~Akywof+ zJ#p%!(uSYgYJaw(tERU2R$lak5yci=+qvQEXeQ(RyNNZrS?~pnw`+Z0(azfg(!bH~ zH?rwz6>d0xv&72~8_tgt&fofxgb#Ov?6t=fRYoJx*sK2LbdIY7Lm%9qoX(PgY*$9Y zEvCY?si1S)8X%WKoX)@$SfH$>Ln~3bmAe7t?7F@Ui4Xl+M$Xi?&u8rdi^j!S8|_5g z=D7w?ff6eQ&>us&hkv{r)j zZx5L8{c-n<3FqT(NNeivR{uVJewe#s0*#o=^59{x`44w=UhU{>s3l9Y5e(X@m|o^J z`#6;R8`;Oh#!er|5W?Y7=!A7a47rR$IyN=a)$JlA0#4F*pi5(q0U_RnW1QmrxHQ|73~?;{V2t4utLojd zuf2kX!r$;_kv0}rQN)=1wv`oM^g-`87XF4;?tPhcu-Ei4B--`cb@^Hso>9VTcDh!; zdPCV?Pgi_0tu@5@4!YhCrK35w?~xt=iu@IqVG8G zO|WZ9lK&iB)~K%?AVzRMMA3OZA-7+G;vf=#8s|u}nUG^3h*l`*ij&vG$_>nKp!J3l z=tRaZ&xgkq`798B16t3BczeHvZo}Me=)xo@iNCI9@)Ax3_mU7q+M5ZP&$`1c-PsI-sfoJNNV}#c!^)K zdwo=C>RGS=vAJ%byO_NrKR)gg46o#4Crd$dhw}UPfnPx7mU@OYF ze`mzTTKRtM0}Y9WU_krWxyzR|P%<=UY62B4AUABc$cQY8|KGJ&i%s#xY1}C4 zl;3w?yRWC5H!ewIPdBr!xaiq%k;Cqmb9-cSx0ps;tHC`F54CjAA1B(~Hq3q5H(U3l zyQK?NF?9G^Ey>m%a{2PIud|Z69IR%Vkps4%lHDSAZ3C38O z7v(9Vw)P{uC)xFW3r-!yn0q7FVF6Cnfchct(DR;aTV#8jdGjFKo;rR3stMn1%@Z}R zput~pqfPqM@yBd$COk;;`+O+R^Ih7(RpyiOwsq4gV+W7x=Z%Nuk{Be(cwC`kd!F6r z2%vW&<{{qam|4D)rLuC2`%VPc;`_pCAf z)=_<*ph< zBw}s2om2{kFt7?u#_C$ALP7+tDE&cqxhO{AAxn`sPP4}I&`LX3m#ydwGH>w3q;D&T zpQf;tPT2%fa~OD2knE|;>(*+f#o^De4so!?yH~ljh>mXIf7Q&zri19+m{%l07hbG- zAU^OL8S^wI_c$g92Rae(@l5Y^#*a%-==Hr{0ATxSHf9-#B$t;ZW`xRb6p6j(Mf3as zG~s3P%ZAl|v4k|#{{WY4;&*UaOl$W5m+JQZ-Db01ZzC=6zSquaQoit-8=)i(y}m79 z*FL@L_4YK`;Zv_SNlU(fkg?PKmiE;|9K?P5`Db7qzX0pw<1W9$^$C!6970MVN865)vh2w8~a++?!M+y~gW;m)V(nGtY`P z`EN2SFPEpO#y(}|BA(Yfk1hiqWjdz?JKe|c`Z`R?<;$~|z4_<=)+NZwO)fz_v3a%h zq-lJ&>`#-`38ZbvzwFEn&b}N(S`A_je9+U5qJ2`S&m4NZM)K^{BacPLjLf%<88ck> zR_opx+dj_=+-`d3uA6WF>>0C+)?wxLTd$7U3AShaVk7ZhmXG_cZ>7`@n1?x<_Stry zBZ%D2K1X+1w=Q>yU&mqChB-Q#46gU8#k90A{XOjqe6aHnT;hYN_Tgo*=*I`i{nMf! zSN}*$+vXj8Sn!`O*4#B*yB?6~0~R=J{600iScx}^09(QJM1{qGrIMKj9KM8FT1dPZ z@2qsVkJpqBPy0R~Ec2yb*Ma2;q`&m}vvQMLR!^L^z0!!~wh_&ZB21B6km=@eQXU#u zb%=QVS0NpC0(rVFZC>E@+)(G|-RT>5@i2_J`peqphSZ15clLa@r8`Y$bRF@+qt41r zOV<;p_)7bPj_zo)k63EkSgz2#U8|q!z4z=-u1x3h%=oCMT}HHhLP%{5V>?vte#v9? z!^;)n1P$T02*=8eBkE}#;q7*MNHLrX{WsI=jQfIWj)X&jP<9=#;KG% z?t#1PbxyhOVnNOiK)D+wgl8;~;f1V@7RR25Y}XZB}mVVo&R+t*3p1 z+Iak=BAbJnYY04 z2JpNgwCz3<{o{QiwndifL=5pHZ`nVoonjjFPK%JBPXm(p>Q4=;+s)|9*+_d6$Unh_ zic1M``*MW~!-$WX6lt5{pfr05kopW;J2GAB*uI_8G~BKI2d4M!+UoU`x&gqp^_02? z0Gk-9YxXx10}6=ybR5qB_D;dIyH9mIBXqx36n(pAO?R`NQvY$!_AHkv?a?>=J^S;X zrS{DBEbnOh<(_@7M*6U4!y5n#pZ9Ed+_PZ?VBrIB?2|`1lxM#Mjumh8HE_gtz!5(K zM|=W~xCD;)0XU$S>#$Le_Fuxrd#>gKY~0OR#%~{CBR#@KTERy81RGRk#X-sTTi93$ z3BQJo;~i`~h5z0bS+1~gEMeolK@JhePStxi)!OGs%&sLVQn;x|G;!MuM^7XF5P|Rg zwB@T72!L)-^xo{L{+h1@swKbKQ?@c6!}FNY+iE(({Xn-5+g*w| zeqJz7Lyz8TDnIn-`HrA1eY}Zb8NsvH{l+Vi_jLU!=ll?$seeuH$*yO{Y`kXRq3+}T zet5(5jX{nUpI4FfXB74JPNvQ(EkC~jlTXa*Qv~r?6k{UYE`Obv)wK!aQ1~V~*mAyP zSQbuw#(uZbvPq2Rz3JYvzOM(3AMfG!dC*rumK!?z8f3T}MjG z0?+uM-WeNz=oqm3+e=g*1NK?zH>+WWJ`YV#$JE^N_SE!P_}!q_N{<$2*Zlz^8%r-% z;}8+hIPzbBw+!G?58QDZbX17wG+lhRUa#lU+hSv~Y)TE3uH)vgF6(t4*fPk(Sa>b{ z%BBqe^{K_uqgwu|E;0^3U+=x$MxEYMKW0g^z-ZBo zTi#w%u`-jnl)kn<8Z`8X{|e)QXgV?3y-IDnZPe1~1Ufof_atL(yVUM+eQ2RseT^Z* zruuJc!&X8Oem`_Qp1tSW7hNAqJ^B@VP;Nx|X*dOG@h|#8_?Ox1-$9ax3?(uHdcB_t zyI|rSP(F8H>pgg^{Mw7V!GiUnx_vxQg{I|uuj+{A75V*Cjg@cr30o>8TUX!lqo0Vq z^qY0PvG1(z?dndl<@bVuKb$wK{L{P*o%lR&pENrE*}SFw7xVUk8T`-YtY%gg#=8go%5O}TV55q888 zXZK|F;3}~r*u3k)Dro>Nu4cVJu339T#=Uz7JP!St?D=BoQs19+Qa^3n@ya|e22GaH z?sg=A?@tqOAW-mRuHHh>LFxqt$#g2qQ2FjHu6Ac zT&yHZ2lvMBFAEs+gtE#f-M|}LQ5MFGApCPPF&k!K8{zqQTJ4y67321|GE2Xk^$!c+$+Fp6EX8{j zg_fc_k5XTKfTt_F-nrb`K&T(rTY=}2KLRJCOuBCYPg{*mU=#%-i`dD zIPwuwv+$FKaLDi=t85GQ2QkOHsp;BL!B4QVU7P+>b8{>FZqWY8jQkvozna<2okM*$ z==tiug;;lpFtsf|pYLY%_#}6E=eN_9w(H&<%L{q?EG#9>pK+e$#z8?gMBx(5WT&$) zWpKG?R>bJ*8J`m83#8YV+_2Z3*#9}yYJ;u^nM1le{>Ak(D$I9XyyDh}C4H*l+HF^5rg7Mzu zX?eeK`~2o8`oiStk*X;QA~)4trYVW z-9CSgfegpbrGH-cD9v)^dyn2(DR<#ZQcT}|9wWfT2`ST05Ayu!HfCno=Hh)5vHHu$ z$16}iD(s(R^DMj`Z?XWz?`<+4oJhygOVtCv-?h>=xPCCfYNgkWi;MU`1I4G2x6D>uj?AM=gb)+TGi#X$ESzq<;g6&`wgGg(uH zW#M@7TD?!!lkIIQ{XmNj-`9f%3b~*+5+!MGLBHO0JiP;wFT&zI+hkr%A8$69OM|xl zZK7hon^M~v$3!>zsnVJ}zCT^-GZa{_+i@j^*9^#d-L98_-Hfm7YO`D6B>&_Q*n4!h z*=F^*^!E09(`lPxCVmc}aVfnDwx_Y*UCzn{U>Al{bLktv>bw6eC0ItLnoHkSoM`@w zI1%?=nQ8s4!4xJfsO?$tOcI|Yyb$aacb`I`}Ly23=BKx$mOr%rV1F344 zm91mn9t5(7IZc&jizY^3%0P@QA~>>$BovGALkh3>NiwyBio6_zyUJCJzZ(h5#YS=+ z0B-@5`nQK-<$e65#lRUJA&&PBUz%G+aLi}&ku5e4|?>m@M@()XPKWY#~$S{W8u|G2dRIS+fuu-bb|R>cPqSF=|jW%wIlfdtaEEtb?Zd=jQx!^$u~k z%jUMn;k9xL7H|0WP%QqMIRiVB>z|g3_t7SJ6?*b$XV3CO ze(y7~1oq^g0`XqBkJW;|Dfh78;Q{j;5-qpW?Ki0WphBcP9B+rX&dX3fhI_Pf%k}%} zI)KIc-L3=ZX+w<5p15WwvV!VHDUV>Vce0&thWgLF~hSgmRKR@7V z($a6icfH+P2>0Pm(bK3yBeur`Qsy>6zS}$c*U1RZoRS*RAwmkx_zp0JyhN&BZTkFl zGTV)6e-goLEW859e5Hh*4$yT3(sVFAQ7_;JODl zxdTXgh${P@&9`z7^YD8Vn$16Dfz9$A;|x$pq0;ZM0?VyBGCM_9V){t)lq`Z8Te;<2 zJ%lbBB)#h?nI*{6hwXbA?>JfhY1LL9v>)%JvT`etR{!=;?07HITF099X702Ak~c@k zH^Pu5keD)U(OeUeEEfDCAGwWgrK)Y^dUHlh#Abr}0J1ADwn-ov7m5liXH?5}C0mMF z4OzG2BwLy1X*!zMh<-g$Yq$#1i2pvW@L2kd%CCv2o)l`WV}nf;snMqMY>QO?a>(Sr8hI>&R z0!_lW=v4coUX3f|OSC`FK-==m$rI{WL-#}c zs0RGUh2Y+1;BS}XEw{$8(njbAG8f_d*R6lPodnW9r)D=Yk9qohJ13<4R8VQc{YyrMQn_+9vejGtF9*1$I{PNebk zO;;f;s^IswD<|l_`+!gId)t*W@P}au?~q`kkYVA>#Criy9|3*4V5#Z&a`W;kxXRnZ z!VZHKozSK53ve-+`6%GISA0*|?meE*fYs;ii@mt&;r0yfU14)UP1$UR{9}*$_TAoP zSTB8-bKRDH)ko{a{W$76nI-hm9&;dZ{j0f;il%%7!WCXaS+9TQkE416aZuB z-u89>F4b*m|5g8{`ff`&Q62B@sNdRo^F3Me&{{|8|3Gt(N5<26WZ!E)o7eodnrwrG z-}UlS+06 z`7%(Ey~hYoCW}xP+c*@?r{o4NJG-g5T759rCL55qGe!-^3m5w-|Dj9Hu4BW{8o8mF zs5?J46~T-q-j&-_Fy(>LdsA-fk&gN_jS#<>m~FLm!_W+l%+7u}zgo1`kvlOr>9!gR zvXtGorfy_!@5BDGdk~75ytqXquOh>&P7J;6Rv0Yv;@7Q6pT&c{%OVHq@iwm7wbTLw zY;9+xhZP?-kMX*z-E&p`*arD(XQykcI)B>EE>l~oeC%2$Li>e{Ht6B!`?GC-U1wvS zML9A|lRt~L?|40#DEIBej*^$>JbmtuG5>u!r-&HWx!+c~pRuZ8`8N6PHu?P+l#9|Z z=;7yEf#N=b;+@enMea2BH{-ykQk`H}P3*{?O>+q-N&ekrUgy$ZTN)>`td6MrM)bFZ zZga1pEW1AgzHf7{AldIH|61L4pT_Wvbyh@DEAKcUy;pAN>^9e+^8;r)=@GlK-opk~n_&bKAGCUvg7Vy?O-RIM4CJR9u1gqh?_OAyZrqimxVzU-oxFaUHJ(owgzs z7Od`c+3b+UpWD?DOR|tRH2Ic#FCUA30RT5X~8Uih1&3-K~aARg5(}Exx-@ zoR8gnpim$4&-mco1dN)Tix{DAi*KN8>m@FeX)}>7#w|wq$IlS8p!|mm%>9@){Jaob zCi(guvBgsFiLG(N+-cV^LecgCjBaKr1H24=L~qwWYh1(OV|`O+F4_0V(qI_$@bi;6 zV6N|qOJgxs#6U_>GMna4W7%VaB4ycDdEScL;wY<`WI}HWt%Z5xZVeZ@;rw(mx2_n< z+J~-ud78+S=nf}un%=rV?qW8yoP}-ba1)}+HeJHsWM+9{;MSupQyarRrRtsg9drt1*PqR?reKU5ao-R?fmBhP_Bfklhk^N5-#ZHd7Aw5rnp1b z5C_NVGs!r}r^5*Q0kzy?=ZL&|BpEJ;*|pRs03WQm?jy*(V|fi8;i+sqf5WfD^YuM; zzmkHBA@Mfmo^3!Eew>?a`2EER9Yu7nX4?;RR@x0;o1yN5?31Ndd%dahhfX^a6ef`e zi34>qTeHy)1`o)3ycSechLZ;&f=y6=j%*ep*|aQnKo&*g5s#7}Ko zRNLBok5TtjisN{g$PsGDwx-wMP}JsnK7_ug?5#f+g{Za2RO>sT@Kl2O#B^zNn00AC zZtyh0*){>%o8FG)x1mzTIDKBP(saw#qp3wt9GMru`>^ZW2iUtX>X$Yz%FA98d9jM+w zvys-%UB1JvGmmm+GVA+QuAb>0IDs_ytOXi)2h-y|m3lh;!a%*=Hyy9HZZ0-AI31g} zwx>*+L9@Pi<5@;qDPg?9lm=G|U5$^qK`LGdWJ?PA08L}Dm%C@eAAImb3tmXA4sm7TP^dk{#%*R zAT%xfsiZAUH-Qfzek3K{7G1*C;}C3t`_w*UwdA?p@0wbet!~Y%(oIGPegSh<<96@; zRbpTUWc9f6EJ%CUa_Qx){G3*Ovw=KUzT7q2H)XDnXQ4vlZOIq%$AyQ+vC^Nm51(ny zJXw6r=AXT1pCts}w=`nG;iw24SbkllAMpyMZf$1wIIQ)o_48qYvmSK%t?1P0jl!g^ zIcww!K3~6Xw{>ED*LOQ_Ykm}2^l3Pghb$Z6zKKZsm&chnNfk~@y23B%xM0_nFnV2d z&|%ysCLJRdz!Vt^d?(=+65XV@icjK4eqT3iR*koBO?lY#SoifF^9<+GK&T4+bcV~X zbD7Tb&cDZz#aN!OE5;-)e3*tNx3NuuNS!U|KjI*f7^Ot9S#sIWb!Ukpwg+Zy)y<{c zBL9Be-agJU5^U??_qweWsmCq64)9R`m%kmugDm!K z)#V9n)2KBrx0kdx3XR@r7vayUwr-2!I@jsCyINbj6ePGeTajdMfg+OeAn)iwXU z2_02&KD5hSA-496v7##=+1ji>0LjPP`HS2avfvH~u{0gyVNr|otIJB<)#q_M|F*$c zwD)GFGTH=eS4bPW^J!0vyWwGuM>Mm&!F-!;JVrue#-p}-WTZJK7#iGy0_d^a+SQcn)7X1cLmQVZ7T$r@+MUcwtu}l6Fl=)ZupeDhe%nSSay4aCAE{}3gouI> zKo%eE{o&9h{`Zc)VL6_ThSfz3vcSHuP(AbTvzeVu<1VXqt~mU=Rat4v#}8BQs3Dg* z+kRV>OUC1v$HILrE){)23oXM*9ir40 zkhOgp?Ohx`eV98L+uEfT+5V}@)tA(4V!mAvg*bYu?$$-p{KNOh1^LEeZOpmo*YUKM zZ>QrpvG-6-W4fI60+%zZ4ak3%$g|1z?Dzx8{n?<-#C z8lQU@`7_P_=&04n^{4c08ezlt@wGtr8hY)=v$quF5Lag@j&mCrgY|i_;2@r$>)8hV z`E0)46x`55`!LUj>$5S$nOw~As#dWkem5!lLQU{g>S4%!(Z}Xfyn?y~mwG5N(eH8l z>MrC~-K}G^BpALw_VIr8ls89w6wblle$X|~kIyY-z4%l6=5j^6mQznP;|)TOO|n7f_(iVrIN z=HvSghg(Bm$URnFqL%t;-^GV@*+wr3iQaeVyJ!7U`!MU87iB&mGPsq2vK%91B;;7R|;QV*_2Z^T_smat1}ZaQ_`one$V{9I{VN5kKhWR1JF zUt#o(X<(13yMzS`+4c!+X)y4FK^3Xye6vGGdHgn)ZAI_%N#Uk%c9u)@cpLgQmF+Cd z=U}DjZW2zxdjGI);|=!S*Qe1?bltt}D(Bw^>A|9V7IR~cf_5!9cfsPIf5H6REa+R& z_p`fA-EWY3?f%+IUu`@>RWqG=oZYX$yZ6Gay0`T&_cre0ulVeWyeRsG30m)76WZUV zZ>4Ww6du9mFVKeR^xgpHvU`Ia>a}}!%CNU_w2&f~?v0jN3mco?1K+~paQv`+QWlK` z25Z`7*Df$SPx>2Ys^7zGT8-*K)?$FS&}w}PvxBbetEp-j@HXoY_d)b6wJo;%gSp-U zfqw_HR}IA6k2}?}eeB18Z@-_WxUxnXvEzf8Z9^bW{rFBFx5&Pl+jm^`davG5>z8}= z!Q^2dp!bi!qnj~)hteLS$WB7(UxP;)Too4Q!V|T;eLhk;AbqtxF9!YF+{;dOT?7;j z`8^JBB{>$waR=x%J-46+76APv(qDG+dN%$BkXGz<>*Upq=PkiH75&bQXGZ_etMEkG zJ;v)=AnxDc`C3WI-Q~_D!&s%Ae`Cu%w>4p|%OGK6&r}8t*mvbVQ2Y&tHrn3WF_F1a zx`n#`BG;V71e8_e)7*&n*Oy$O|hUGV%e<5;*gw$8ifSsTOUje@F6%~SN9jsZXA zT7Fc-hh=9{zVF2CJ}>4~tEe!K|eRTt;Z7|&cwxD z6zmNuwQ0NkzRYD$y~+Ck;c2-ZcaTUrWCqVdPW+qfmV3Krf66-Kd;>wpV#c>cH}0pp zrsP{9V9GwxL+`RKIvps#Jw8vhDNf25g!gUMvWU04%+hkKD7zwOv=um5vzx8iqY8LQ z1-(%icQCkR+L-GiKtDflRnOFZE7~?Qd7{Nynm}XuWwTGC!i#9)xw;OWkF71alk`zL z+D3CF{L>&N%MC{sI|&mV@P-Z7+=cgch#G)g(ERM?xUPL&`Sw-qw}k_h@zq3JxZjD2 zURO+`sd51+6}i4IJS;AR`W58zUET8)Z`#T}!LmM@VslR2U)T4mYWep4rl{Iy;o{8v zKEwfNMZ3IZm|3gT`e*Meqwj0}g*I$o$!$L9J9@Qu5qB3BJx<7z0^=PQ-iki<#yw-S zE)`f;>^eppAz}QoD)dII@qQp~a$-`vf!*rV$$Soaz{$h@mfIRJf19)$Pb%vp#jhM$ zfgaT5@kXzB4ue>v{c97peK%um z!rHO;cE*POtAw^n_i6cv-1mlV)9(t7s6AQWuQUcIva0U$dXGO7`jBE-kNKZ~#EW$0 z3rHMFyaUHYk$)a6+E$-JDp@;&TI!*DznnE|c1zbtOjENn+SuLR z8zV%Fj^Cdb%5s0PM(XtT)a%E+JM70UjW*>#H=y_}l(#rG{|LgxRDA>B%)5zuyUm~&fY`+=m``LcmFV{lceJu051ly;M-7wAep<~uRcau{7u9yZC(arJqQ~ViQ zP?8i4-;;dda604AA_MxJ9LoSX-<9g(DF5qnm*p}g1zz94$T<#kQ8@KmtLF?`tG-=m zFGzk*By5bS(AoA?j~nZ?aM)_rTi2~~8S?gVUpx$X6L0wCfD~>cwwTI`Dj_WBWW{^K zH>>i_%U!3w3Q7;9n&A+g(2$RQ=gA9qQiSXsIkj_ z>dtnl64-8kUW&Ki96xs%;=13#^BmEBS?Aez_HW@it@}K{^Bi^PYvg)7&fz}GxPJ}K zT}l5i&lDK`9-8xWmy4;!Oi;kz&2uab~EolB0gs>ZOXUL^-_a05MS>y@2^oysM)?o8^6X$Xi0-ByVZCvc`G$l zZ%**P^SrTAPwA8AP0u`#`O6Z^xf&^;p{~U!KET$)DDF0umcAH8`mp?xYmYt)AC}(@ zDD!cC(pcBFnx8z*&vjew=2h~o57;)eefNkR({0>C;@ctbz8#uh?4|S?NyU}h+uX-R zUApP$9OvhHjTz8xJZ*ZubV#>SE4XT*8VB%E+Ix(kQ|}!{7^@#%t_yy=jdb(0;K$1H zd`uq~{II}_?*>FQP+Rcz8eXe^?AFxmwTj<*IkxBk>jBSRj?~|!@NvLR{O|^5ueHDF z)e=M6m#`Y-Qvr3L)aEZ$ztd4^Xfe~v015m`Vh6p2HZ2RSIy01{b_gTt_HN-s-NoZJqR|2Um=`e zXUqLVPqzUDlYD$2PG@<_3gL{kzhCY0>Sj*+Sa7WFFnP>J*T893aZcU*?k(-o0M&xO zJHC#KGO<^8OW!E?TbO9xr|)it*?Vsl4?oA2v(40`vK#s}g{J*(Q&_3nrWijAJxtRs zf7lew8l~M5IFYXpr96eE@3zv|se5P3AB}w-!(-r5-_qx)(IXG|Y-(hl$nsSESnW1t zzSO(6RCpf<=o&X{yM35?nJf^4ey#!mrQX~Oue4749ahU`)b>=}*hOB|`jJu4xzmKG zE3`INKc{oY4Oe`<$4@qjzEN-3GwnC)p>1kx9;2lVKX1#BXT-4jTCqpD#}>#J(~a^K zBw4A6itmZT&t+dyyQTI1DXhBkhElOow@FhJk^X(%mFedB`BQrC%Kkin+hul2+lJrk zt}KO6{M3wuo&j(%fQRK?VZET!^Imypx?b(XZkVF~JwXLI$I}V%6-3r`uE<+Q1 z_EBs_m#$^Q*Qx8eFJqW=e_^PG3L1y@X9kXUr|0Q|O-<+M?q-5x{&73T!Tj#S!M3*e z>pJU;nc&vHTyrkX7>E4JMlKAJtTNcs$ECHR7gh5z$9?ws?d^75A*){PRh4xg@Nyd# zvvhcX<@|2IN$S<@wj%arzrA12-|P3)r|moc=sU2bL795%pJF>-6CV5L9%i59!yns= zt|Ileug_cM%?$^w=z61<_fdeKBFglNS9&13bo8cLO3SblS3aNDYb+{mn?_T-!fMYZ zvRf+ldxasd!%;Dd?v13)wr(Wf~YL`#pe z&~KBw4!5jlp?slH>dLQSBDQfR*5jG>#&3+Ky`t*Asd$`6-okSCsT}Dk_ir zaWYrBst%d&D_CsOXN}LtP5KZQW4nBPvq?J%a+^%Qo@4)ZlTs7bm29mD{RdF-j!GE$muIM0Fgf;NJwub?{{|J_WMpE#ehC#HgTy^U#e=Kjeq+Im-LOg@Vi@B$)39DHlAi}eW*M}uKPf5k5Ryui;+Li**0dHCnw*N zx1Ro1WZgyabhdp1KSPgOM4S~}{LH&;t?b*1J2SK`zdjUvjw1EBN9*HHAD_>X_iXCJ z17^4sO;E% zeH7LWIc@;WZ^IdMfxEHYYa{?SE!4w+TaIzIC?eTcoZ1IuH5Of{UJjk>kFusmfF6sk zhZ_|n`FWGNCu}vAy#h3glQ+Ask^0n@Xmt`jMT^tr9pf6!73Z7A{Hv3w744g4-U{)w zaP^m-sN8tAL`@euYmNN5FyYPW}e$8+v? z+iD*WS6LCdZzaD92;elU{v}V$yM_EbAk~>n)43u9QeK&!O(3{veBJi_7*Wk>M%%Z+ zx16eaLCK7|y|o7IV>jQZRsB3u!vXY=b?E2=$MR0~Fm!c$Kc!1fTQGW=(+AP=vvB^8 zC#+Zfal%^G&lC2nQu&W33{`(MVQ;v$|8&BV`+CA2Y4YuxUOb@fq3RD)VRDK$QDgoH zA@xphw{G7o$g?HFms9ahcDI;{wOr#7H_MaoHw$u^xN+X9GPjE72UF>1x*(pEyxpN4A!+ zOt&>O73jlM9K$3`y#-KQO%pbXEG~`#O{P20U)u>1;c35vH`@1BfXXQuqxeaX> z@g3YF9>xC8C7iI$OFtzov@0lN3G!WyoLCiGb!W7ZX(c?@WoN&Db&R5eU-#> zi@$==T$=9H%P#XaLFJp%@zO!Vko8fI>s~ec{dLcrlXmY|_KB%;l6r!zirM3CImQsK zK-!Ws!dgx`0ZFj{_}HiIW7r8KT3Vi#?9ie;{@a&s0ZD&F#&Lc8CYG+0u=e^TmlxIz zWmcgO{==fhJ9L$<#bMkNulug==3crYasM=GVi-$u2_SCvya{2Eh?MDWDww<*)J7Z`(MEl=-N!%w>HX?owe;$JZf76(;3Y{LGEU}{T0blj{ zNH`%+_AXuG`oAZSf<-15sm`E-Q^9BH%_5h_K^=TLZGD;9#NYh3&c2&03AhrA%;#m? z->LTH*RS>G+CG!K`EKOZ^6q-kl7DTsC;xIf#AmH@5Kj}lu5kPiHJXPil4E!haTecy z?LcTM^vxbEapz_`MKqhL+sP@yA++;?AV?}d4A;pD6F*%P|0+0-j>Av-uZ(pSp^NWE z;X|)zhYSHPuV=w>9&20g{optTa|zUT!ShPS0VGd%a5Bu7Q)=$$Yyzd7`)xLgyo|{;zf7t+McaH)m@U; z8oF>Nb7EZ|E$ciU^?m)D5NUFG(dqA5Z0n1vLs;_$6)M$y$i^isN+}G%UnZdEh%F;k zXz>G;=vNM+-UGit`)?L*T|YlBp|SVvEDwi!mOHfIsU-Vih?_eK`+FXCX8V3`D{LFi z<1{5#>v&02=~6kH!G1`_I54@E76@Y`dv1?O|17|YiNL9JK)UkA{XO+P_s2jbV*mVK z@V|}$;o5n)*z2`b<5?6o0?Y93lrtg8as!am2~3xS)tvPmB~{L|f<|uaw_>hyGQbgY;u26EXSzgGd!St;PAGVIG*& z7ZMzMnGpt>FFt*}h{Br;U#lN?%x_b_H{%;#q!I*L-;OeQ(S6CU4}XWhWF-~)`k)Z_ zr0d$)bawJ_Jdd6T_@6c;O4KD67#w>7LurBs^>5ny30#b-V=3#}FC&#DsAJ_=k8Z0x zb>1Z>nZ&hgarL$;C9vnY$_-tDMtopx75}$(d!=t182KY|{P4Ng4JhykQlm5cN@%Y- z19`QvsvR55BH#HW#_t}>wgOjEOPXDyyszS*NJ~KwTS`Gt`_%wn;Y*S$+^$xoKy=$` z{J!^Hi`B_t6cWDr_*RX|h?S3y;V^Vb%70iEEV)__ zd!p!udvk5Nm9^?+fbSr@ptS0|y@Cf=vCE(^;&6#7CHiyYA6sh_I+%owKEg|MH1T8Q z0$w-T^c~ims91#i%`cBrQlSm`TB^HT+~s}*d2JghehbsbaJMX?wd#N)X3>Xt%&mZ( zE5(c!!{6{yRtn#VR)b-J@qSATmkup<&G4aeEH68bHzx9bd2xQg`*yytP=#UQ8W!E& zZ7s%+^45&?gv(M^dw+jsJj|?5ZpK{Gvbb^YJQ(*Wx^98<^OBgd(?Lz0;A#jcnpy(0 zU_bie`9{@B)R0hFkldq<-hb_1saLFL-&0dIzLfv{@60!T!Y|RAx%H|>{*=N{QOcGt28pfg7`aI6d=<#Lwl}xzL~u*&lZIK{+l`RQYxH%OC(-XKn>={28-28oaf2-%qpuEQFX)hh z7|q=r=@VC0cX=`GffXrczvZ)3r4;+s;~M6At~S2@w!9P*?vhvsHnje#e*00s7moTG zcoXueik5`PjMUR9CJ;6E`IAU_@kr*gbK2_O6)!72rd>yVQQhnH*%HC()Uto+{mCJ* z%U`dyPo+y6*((MPBX?;A4x@wAE)ElSiv|wwq2K$P1s534CdU0Ez7Eq5h<*<5Nlb3K zA-_p)ySus2xo#1m4qUhQeO%&i!4MYLl;C{t;=*>~Fi6;@)oJX|=4UcQ;_$nZ*dfso z5IPTLaE{O2zdNC@x4u5>jEHyOmP1^p+elDYBt~nrXc#|DMns*Z*#i`COkNNW0e%_<6}W&e1ws< zX?%2+Y7$T?pz9r6+O87@n}!WRkFcWT8W4?XG|Z0yIz5UQN4^rFp+}gCh5v?*^V*YS zQ8dm)kR<;gD*i##{3C7f2hscwqHSrLKUdi- z9oyF@G&h8A+x!=9j5b&|{g)Dj-3xgPAbzp;E-tChk;f}jJu9*K$K>C(&j6KP zUoS(=%5BifS$qWD2X&%Kn7)W;KZx>{h@i}hmSWDomIk?ftqu_R(RuVM{tt_1Jmv0f zT3NdpM@D)4!7cyY(3u8Az-{>H$nH2c+NHNr_&j5?Y_r3K1LQWWlDu2K`RL{RGEXk1 z6qoP4doX&X;qe|$>3^)0!N{n%fikxjQYm1Pm&4`8sF;jfwi)8(+&7;KQc}+G#yuE~ z)bQBh6LdOO%3)+w-iVyr8>kep&dcF*V^m5`F54{ea-N;fB@tuX2y-qh#%AEekTPXE0cMJNb zU5TL5z zrkzQU{>NQO^p$rD=B97C`TWq?Dd*%rx%Y9q{vqmeMi*I$D`Xv>PBl9mBx$irtC zRyIEEOpNqD?nwysK$k|09O~sfoL=--)z~PiY_m zYvVt{Hq2r$m5j1~r-7*RjCVQ4UNXdfAjDoG)P5k;UNX#nAk1DO+|UReB@-J9T#W-Yw=Z5H?huU7&gbDQ-{`F}%$e)m32VtLic^w4Eyj6S~cXhdHck z%tRjE=R|e|+5@G;LAUwv{%cHCA&k&D)kh*EKlIZ0!b@&E(ILn|p(_RIL7^iBQf~O{ zf$Ad}>VNIzR2bf8!DKK5NiTFIMomXi98yabx6hZBU6D~;8BY9$GB2&`f?E^Y!_d^) zdjP00!=UZixDO+ROviE^_VLwVqi!IF*@r+0%S>Vu4y9FHY1z6!7+*r91%ir6w`XE6 zJ36sxHjtC-L*zUZ*B&$OuXOv7(>YXdxh2X-A#kVzjGq&O@a@nR5$PS<(7F0>uJ0Gw59E?1ytZ3!b7fq{a z?+;KA3o4~7oDD~rmT9=9)qTch@q4fF|52B}*Z80MTnnT4_Mc1i>%+{~yw`G4RC{}- zkRU!1yT~&}WErtAWp(7b;xzn^YQ*KzF zHB-(Igc5pW#^f85EQ0w=B>$g(Q4gG^y_j;t`~Gt>k{@R2|8X*zE3tffq2s%gou(VE zPQ-cP#zZB?{1ro`Wl15wlnVI!NX%p&RG=7dXh*$2@n8Py7H@f||8f~pzoNadzqDWK zgudAy{tSI1`1kSAVf%kVKGTar)?Z{!A(foXoVvJ*5)!F_*`!%hE4gxu8yN*w`RXdW z()W}Bp^K7j+8Wgy?b_9E6|?%)wdR5o<>>sV`|sH$FfvYmrNAy#>us`(N1oY}+{jnY zvbtPUDI<>=NF`6c%vK2p+5CI5#(ZCT_I5daMmhG#=zS#6YX9^`=CkqlA4&a}#;z~- zi>ad0`Rl4mq{fC5R0BqTHC66 zE-1xxd}=1tcScfJ0w4A@xE>vj`gsl~2(cMniFgqiSywg!9lmBJ8`&0*i`WH>7MI@_ zz6O{o#KK>qYu%=fTdI)r8<02A0~H>w30kQ0ZX~TZhhb}7k^1L{wJ|*Oon=_Wu$I-s zTdOat$n#cVrmlH@4)DcXAaS-7LX%iapSu&mB}v*zdr zeB2R?;drq)}OE8o7HuE;zD7oN%ruWk9eODd41 z*;8J-H-LR?h=DgoGEVR>c)4vB>uykIO2fmvF8WFQkJka*;VfY@Py5@#2`umMjbuy3 zf9#5@>5LX%?s^BC%kSCKeO1U2btamZJy;M5K3DcHv#8!d7a7j1lt=t2MQ3WRpPHiVeSQY>B)J#U~8q`yDPUk z@Yq*bKq*j2=?vVR5b>fis1nsQucu^6=h5Uz;(+e+kL{AF%R~tQkH{m_M7L?riX0gg zDFfU_L?{CMM?|LQWmG6e(yhuwgbk5n!Q=0KFy9Q{3yhG1UN@v!&CSlO<10`cWi)d7 z<#f45jQSPGSZ%)j(BRR{ZSI|r&AGi5Ro*D+*}D=4hu{9-{TKr)hQmvZab#V$rWz^{ z*@ONafyD||dW*51L|&IhX7dC>-W zVCx<~pJdBy+c`8^WJamkLbv2k&BuQAwT9^=obUx_zk@cDg}HBk&fu#97L)#tIQ#1j zpo+!RoQibDqMsz7qkbIx;YM@&!R0k=jH_-XVSUNA-$_J|0=7VxwgzAszH_I0yAnRYj1{@D=LT- zk%CJE`exFux6@#YZ>D=4z@VT)ekr>#l$JBhoUD3FCf-EyRnM7JKbJ@9hQ-ec|4Ps1 zQ>!=EM;l80L#ytVM(Uv0KWD*cR|((U=DUSvP|0yY94KyP%E?# zZNO-4;?%UwVeMUpwmp-LG0~C-@`L^aaEwEO=$YS?Nh+HMCC&a1jXVswJ!&s7vU}%k zVv0@ba2{BlasNiW0~Rt0p8!i&Gmnps#RVF_#Oy4*0~#Th~4{G zPi?ZVJg`_{_aoX)@S`#m4{E)|toJ{B5dX1-Zgb?v+}M`}#<4W3_G5l9d4$KteTMAq zIGUorl1#$(RI8oBSVipH{(-VoQC{IIs(&B{;H1lU|}TlB=Eva za2U6}-|9DH*(+X@4|AoNfr)n$HVK<^px0coVw(OF;7newyY*!2!0q&i{E5mZ9qe%!2lf6ml7Dr(7sxbX_91s)aClF2+R*4k zxMZr9)#%@e@W7N`-%Y|MB1!m-SAnl@(k>qJK-i+Z!aXgw2GDo+&Zqsv%c0UKD>)79 zDttFzxODfueGjbcp+N35x{@5;%FOOYXZEt09B$ToH;uN@yTD#<@$ zM%EsYu-cu5BCmnn`kL-%?lRf?w^qLgDcFwMnL^U^kGajytCtuFBr8kvp!+|({Y)Xr zS;*x;`z-$7sP|m6S~MHH4=nlz5*Hy3hHG7;Bg*xKEJFjF^-wu&&hHQE-Kr5#1h)C# z`Ck)fD0i@i3j9~`TmrScQ$>TShS=S;vhavGX=WE1+J;rc5>mdXmg|y#tGa?iE6Asq z86xKP(?}T!MzS$Efpr%Mjpb$YQo2s?@v@zM4g#i5+;9fb^uIkB*3L&fE3RV3bsY9O z74#z$T!2!Q=ZLoVBMC9EzX~ep>2U|wL?qJiOK;wT4E(s8OUAkzd?1?rm|rM*bOByh zH?p$TGHl@1AVD>Tkx3l*79FDH?^TODK0eMgPw5*%^JL}XGtBsU#cctd2$N%yT$zWA zxMvH8X9y;R;&$dZw3!DV_m9%Ou*~ytiPedL{LD$n=;j)iveNMjGTc}+k|?KjC_nw$ zP!G$I6j)UE6H`3tkg(*gWQes#_m0KvTX6QUi~1j%&NzLS9QDr7S6_z1y!~b;6bY>G zeM$+{8!FA`+))8Q(FAGUwqT9GM63nd7O&_lmHmo49ysq^>vlXq6$RV^-|AilEgY7F zmcI=h-u3XCmg?pGBM7Q~)7dOH(&A_#Ini@T=CAj+C!f#gY=Y5&pWCn&CnxUCe+DUWXR;$FCx@ z&;tN#bqLr?9^!kwjACDU*0RAUj(Vli1y<@1Nk1#@kermWt11;Z<`E*72aI5ms4z)r2gX$+-NCeFiv@A7w-%RXe%g6f0ZhLBLjIk#GlY z1JTB&AM%>jO4Sk>T0I*`rfk5~C^ZZXS;i%BCn%@N#vcL~0NS{^O2!}5l5zD^%sq-^ zF|~1%a|?Ht!*8zhhaK-C;?dCpk-j-Mu!bK}DsdIgV=I#P&SNW+!!BT}lS39P-gOHW z*y=T5m&eEraabqFP3Q9fTpRMOw~eaCoQBw zx{8H4Yn=6HRQE8(H7h;saZQMaRl#NL#7Gj+w&Bp$=h%_2d+Q~RYHhxA<~Qon$<)8> z-r_Z0qI8P4p(mrQ69@~$wwI1hZI01Wgu$fO9+j-Qj?nG=lOJ;z0f&u1f=Z`Jk4AgQ zn4nUS=lE40a0OUJDbb*Zr#)myPJAKM!6q0~omF@sf!v5;+pk{VJPk zxNfKyqtYst@(=G?ua2(>-U2wU)w+bxtED;rWQXd?JD%o}s#$bmkB5j7r zO|_74hh_iv9hbZBJeEkAw+dGb?Yl+1X#UczBFD2O0V3m8`wRfL)pn}qMR&4VSDO`) zXfs;+(!$+Y79vW!zP)0>I3`u05Ko?PwNyI5;#}m1O%>1n)ABuY#c-@VC(#0)Y%l1v zwuW-xBGgFRfdb>Ib|SKj)?6m~KK>&Msr!hJ(C6@dwRS#m@k^S?TDV{+wSBkOH+k7= zcBNd(6&~%Kqmmd#4WdXXV8*E;Hai(b)id0Jj+DDJmWBa`UTBsZbv1@BUE!{io`>P{ zD%C(&rhvM{C9pUynI;ugCNJnMzcgUrw^LFTGT9`L)+_Z-auPQ0bvJmAF73P&@DR!8MJ{|M@FSIf9Y(#vky7}BCWInFu(tb_e(w_ zq!H%2vY5px{9rx{jov`okozI=rg78xbRP3mex8_35VGw9)&d3qCaVAL0rnLYBaoTD!7;A#-tgpvKun zWyB~xidIXLhRP?8-Kh%Y^%nnYL{AtZO+mN9ld<`r(4jun4No2$05Q?G3%-)De4`CF zG4^~(C1iWFel1ZOVD z!At0{Z>XsFew@rsrtcy0W~8d;u!9Jq69_*SO7SI#Rt%@(_`(}?;W}`!(hbMyr_I`5 zYeLXm?%iIJmhARbh7)KWsjzqzpLjXUup@XFZYhT#=U#M_9y98&+gc z=t+dN)SwZ(h2ig6Rv;3E;52G5)|yO@+%~p$g>ZEe_M_TTvDO?yy~R0y0MjI% z-92iy?k&0$0ol@4-&WOLB9g{_q#xi{o^p4tcB7j$OWqh21J{*L(2rGabl^)2ZX}0F zw=K+0@0Cveb-ugi*1W>N(Ho)$fuwh#k0k;jYAFu2IN!Q5(mVzdLe11M2-C(;WF%A) z6(+G{m1McTf)WLf z0{%0aV}tbuB>zAWbzwTCpJa;Mx?eWHV4Qx|kOr4K{T+mX`mr0rQ{%>N(`MwdeDt(?{S4{h2PX2pcKb=O$U=o7CY zeRU0~46cEYiLND~xY_4z3~6Gm)=ZQV8dlIV+XcDs^9!>%lh1*81pNB>x+N7r{4ug6 zh16R5uv#B{m;Go5=m ziU@UuVEENB=>w>hz2P=-IBhp)sX-Wp*MHXX(6WkG*`>}#P?kWFqAu?X`Z3^D0fhH; zk#~j;9+Se&u-m*HbCk4UVbZLxu^{GyO=zfR4-)x_oXRZ8C3c0Q7STiKzo-#68gD|` zu`Yi7tmHb@Ydt3e-Y~Qtidw(w8wkK_nz)$5`|~bi%zF*QF!@|)+{vRm+tkEvvyCxs zk@Kue%9gnkNGJ}fM{c#sd9@&=W0b_+#(*qz_m93CK8}KWgK55Paeo| z2GJ*d^e@kaA&jq)r<%D?-ent!P4t8gkAwybZghi$+o)R9<5T26;O;?806QG3ECC~Y zAs1VWDDWh++Y!(|t{5Mb`AuV29aNgy>i+dEOvM3n=pDb|cMS{}WAaWqJfeGdt)JWh zkhFbBIy!wB7G%%qgME5C<#(gsv^W+GZK~Xp`d(jWn%@X8@!Nm9S^r`suWkhL$u8AO;@>B_HYY;V3iE%|O{tQ(kZTCgdn~756)Ay!7CoRF`a?X= zz$PFF^Ayh=;)^{)sPs$O_Y*nA@r3OeIx0@LWbXwxvK7ve0}xClWS*{?l!6twyM>Jz z5s><|xXnY?Dyb*c@nCO7Z>dTjb{gYjxd&MCM;@|?Q<9sQN7Y_c8jhqXWTbb@xU${)dxQ-G~u*{lkCG4Xpj6o9Fvyl3X%5E=!{SFu=zt}+RNoWoz52evK}e2MUp zI~-I9Lq=0nMwrtDcgD~jG~_U<0XZhjBW;3Hby3ZG*x#g2wLXIpO{h!4IGbpPL<7FP z+!S&H*v;l-ze(kU;vb6yv5Ir;5=Ah9mta#^uLvKC-*Fk3Uo-+w0gIuoc@yp70SACw z&0kq)A1>V%z!EQ7=7yg41ky~*19+hTrnWH@n41ECGFLvaB?6Z~tl-JMV2=`g-P2Gp zA@5-&vK5kRvoh5Z88hcHF7rilqm^q)zqr15El5#rGJ{$!L^`U@`44`0Pw8qfeC;g> zts9;wm0MM1NJ|fU7K;7tuiF9nqG1zkCHoUP;v6q!yu9tG_J+jNB(Tt|ThqmtA-Spq z&k&Bbx&}1M0~%@whNKj6V2J{VoVi4PuUW*}J|4e1!BPDgJ78x7!5A(lA$_Y0^AFu? zF7sbTPm8lJwv*6RWEnU>kdhn}r=&-me#bU+Tk_kKiBs0O$sC=$*PP#cNCfsagVxrv#db`O9#*tx`@o-PUUDeRZ(M11>aO)y6 zcKD@0n;|D<^TLj)NHZP?HQZK^tnK z_1UVaH~Gi!>AsF>BKe`a9&3eleOYz200zSxEmTM_p!LX}}KVtL9tG)^>$Rmkzl$i4Z%rVLj`Q0wkq03fcFLXo)o z%&u8d)`rNoHSfb^Yavw*wzmKdUfk?6%M6N<1e@A2j<09bAz3^=N;UtX6~eL6cLDGM zGFM4);4~`+9NU2#9v6WLRyJLQJ>4yHC8;0sr4eWWFOx>xB;|98BK8#@FcBR^zs!Ki z%-u5!dY;FnlSx9?gEwQmMdTEs%}3?6M{Szwbck6)fk{h#BOX6*>P>R!)g^VgS6q^w zIA^+*mp+dMArMDR=)zCmDs{Yfz6)Ga*^mtcg>c~oIy!m&VL`x_)zti&An%m22U2U| zi#JaZz50lylJ&jzTqNfz&;&_Y>ci^bHvqDGG$RtA-Txd#P8H&8cSZlAN+%nkjLONf zSx{e6GNr)Js-B8R8SnGk`l5(E{YiNjEn(`1vLC)~9^X6w;iVbf4N0iUWr5e}U^7HQ zh!;zh%DW;N53TijnJO|08YbltBc5A`6e*RF_}8S^iYiJXppizfal#_pG+0v_Z@+#6 z&OIb&yX1$&d0<%_2RIs7gf$=qJ#|5<|5ke)n^c%z@`TC<-_TLe6Vnb5$Asw}N7?lv z)0Ue1;nGfFJ|bR8UZPZm%WJu&ksq+qlhdnRJ4_(&A z#F3;Fen`{;YUyxKNP7Y5)#Zk>DAT1*>0G08+qfOP2_%e+A4jB(r{o{<=wz7o#IKd9 zyUyU3O05DItm<9Q1u5b?ENG6<*QNBS@L-uuoSfXUn9Tz=5!1ejsd^a0adY4KdYDpR zn(*gf#eB5}cTS;R9Lo)5lVPiDZxySCr?@pa$9c_)`@qQP5J~2qRSXs=8~@lse*4Zq zoe)Y#v5SCk_b9j>?9}JhLE%qTfi1tALR6`FiYX$j{ZL>q!8c39a7)y?cBI~+frc$v z$X|-}fe>{ab&4!QR?VxFp24~>lP+5_lNhTHVoNweXDeqjqOqQ0JJ0MPB=?D8Ac!Gz z&3XWKj7$M7Vs@M6weQ@uM_ot)9J`HU2qnUni$>Y-OjQ15KB`kk6h?)(T+|4E_cz>} za47Hw%rR@}V|@|7T!6Y3D+ANYGrc1}nep9I{SrCqEW-p9nkjs@8|OSkUc!;8n}?bJ z^OA$9L9Vrr@Ym$G(u;>P24>|C6^eOtHt`eVxXB9KWfY5CGB|Teun;iEHA%~b69wF$ zf-j&qGNWb^+^tFCGJV?sSyxD+5vTCsT0$yQZgMP`-3A20@znS2U^;qHY{ys(vZEZIB5IJ4PQf0*fU&>{V~rSj zZ@uuLIX#<<+alTG*?U!Rpg9T%ll8j56k{#$`B(p5BbcZn6l-I+^Gi`>P*yM%Wtl^Y z?U+y##S?di5}G4wR~pkc%GP&&>^!8>A*1Y65@B6zxqo|Z1Ap1<)*&X&dnjyKzqdX$ zd7w%=@ez!X^DLZxyKJ{Q?MDmG(KPJzhr?Ue3u1283e5@8#NEp@+H*)5Fr;f>v^=mD zKd9V^lJEtCi?}`puIFDjuqYwerzaAI#zSU>BdCHg>mdiMt-tI^iXy(uRht$j?8?zR zR}7$rNTTv86Z1)3XC@i-oao2{h+zdy7$)jzrVoe6#*pMRa|S>WS*Rjx#N;xX%qF{X zM_5u(4JN{l2IBWjrEPeCBELWeL+aeUr-DClaWLGxdWTNsl0huk*XeGin9h4Eaqr8@ zf?p*zrUI9-J3)Bs>3rFsF{(B=DpMX}`T2kq+^R|BzpBhEIkQqMrcFERc6or}Wc@lS z1IaUW?N6wST-H(^^6Kruk}1F>Otm~(dZ|V0{bZ0eB>IqM^_M{wf!bHLz?~Wa)>+v+QrtYUDy?i>`1te~ zHocR4ny*!ov9*WsDxA?vrvv{)d)fkzbim#K|1`j=C7Vdrbn9k1ljF1gD%0|J?+~_L z-#|2ZK(M5wnkO@af zmI*IG6ZNZs-oCi}(FC8ZEY6wZLN|vC6ro9{0yi;;0dS+N0gitf*h zk=wYjtNCvGy(1SZ!@xEqy%k@R|7U9Fltj7 zBU>^h3RgNP4Q1tiHq-v#!Ds>FlXj&OI(Ip{VA@6O5KU=0Rss} z``ImjR*9E_#<;X-QHRh3lAc8*AGnK3T;sXs}tC83Bwa`N`dj%BUc4A+>^F9L%Dl9l=~pLw;pae;!46O3j&d}*pj1&HS5 z5`#t}l;YO~XXwPs?$X=REVVtmsoF_dNX6tb0xQBRJre{lTx<(h-6F@igko%Q~X>J1RzA_9S?< zzLjK#>_mbf#?&lE{3Pni`&(vzBQ z`8LDaaS4Y99^@i+rs9qSGazHSwOPB-XbCAtQ>kZbyD?tc2+3tl3o&ZU1$Uf&(_Wc# z1Z2yViK{zX2GwJ40D{h_P>fs3E^SKiFnJ??mkhy#Yp$L}oy#{G5k!^*i>X%ZDUC{e zWXY zH(1=Tyn%@B^Q7iDi*n7!qZ@eIz#fEZH=82G8jlE+F}kh}>Ch^oL-X886d|1Rmi@aH zVBJ6Z>0gjqfH$cv6Wrr$SMX=z6!%`ZU|Yp9z2xPS%n9Kydts@Gw;=Sag;RkpYG~Qa*{+|>cMGz z0@x~DtHMJy$&}1RoQQe#na=A`_>K_`9Gu*=PA6f>D{|}m*i0 zq(o{8e5~JKKW5r27c`96@wuAd+x{*x5J!o?8cX<6{41~RAIBSFWqEZ%xsqyrz9@z@L?nkeCV3dQkDDwi}`jXsMM_`hH8%nT{;QXNx#pj0FqLVmI#>P<@RWZ8ZA7=4F?O{gzEWub!4owUlwMnYI2F|vi8z0>uJASKJ z4{zw&hs5M9jH!dlX2`I1U#Wd+B6>J+my^v&KR%Bi!pv??%9+xw&Xns_dnOC_%D*70 zU}1Q<&1By>Nae>p3DsY5o=~0^=5Spa4SSpDJoPoKFSA`=Xd9f2j(CT5$Fp;b=d+#d zOMhW}_?d0p$Ak%N@}YZ6QNi8K!B{b0gC_3f7IOy(6hE_}vdWCM$$s5gx0QdQb6{8w z8ogu_mgA%&7HuvF?)!;jLEV$+P&A1fn%XcXuInGcyz(kOiYphd=n@&c7$F64DvMCt zClc-i0~H{K=~ARg=;HA0@n$+n=@smWBPNtBIc!@SsEiU|kqp&lo|U5oA9Xl*1%h)e zuh#MXf}$^;?Pp*|92rGgSbH@pnV5JU?XI9%A#C@dt$eRw-NFo@@KzW?YT$IEM^hrtLOT z4G!TnfH>ajs_-fn%=IhoxBd)Z$cnj@VG^+Y!6oxy`b89k&e_u|t{^0xy24aZ>HZdzN*R@S1>#CKyGkW97{6=f|jH z&gOdR2@x;2$fjbA=pZ_?VqZtF%UOB%)n0~t-Y&6ipMEU5jA15IDzFeYB+A6UD4+r! zd8m+wB-IPfP8h}2lMeVU7}*W85V4-qr-iUOztS1d=G+v?^+qDZr>L5%q0cJ%Y-l3E zvFfFuB}Kb&r6P)(li|!`H#oKRi{Ms^3uun4CO!cEiF%&q;!u!SxY@)AJF%0~2b0@m z?xDB({h=@XUZ-9%;l-F0A450tMX{>136Anf+!-DupRrTmNuQjqI9qlhtpzupSbk>& zioWI-|5h2b6HVxwU*J@2dlbWP_S>TUoVhm4TZ+w5!z=POVXMoO8GLI)hd_{Cz53!MdH@b(!9>E7nyE*@$G6Po)zLGM%Q4Xwe)4lo6UoaiiPWMbBYw@i@|T%%le z7Az9b8*u~9BYr4)#4`Z}>j%VnlB41zcU zv*~8_Ibf?IvpQ*Hd{!P?Gi=9?E>l5gQF?h~4ljQ(0@k!z@}mGJDtB_?NCFm5!z9G1 z0(rWtDoJc1E(b3i7DC4oS`c_ymRpYGg@d7;1qi>Y2R*b{B7cva>SN+ABAN_EY)`TW zMC5ICI$uwVL`tg2sX)(Hn^g56LXH+yeB`a-dDy7R&SLtU8L(>hWO!Y0aI{V<=XfS2 zLZP5(CD%6>F0B-zY}ev}7xg6vJ(3P4Iek82Ah7`|cZEsmB)yq8L~>?9X;+x;PG>_8 zFMAnAox-w6{Vrx;6EB#NWRd_C_j`;%Qwz#D)<$rn9bf4h{YCLH!GwbMhjLSc&h`0= zyuUE+?swcVtuU(Z?;WyYFf#LMmE~%<*t{olRld*X9^?6C>Rw34$?VmW zCRl2mcuCXTWVQs6JxVRNDRuRu5#qRXEBQhG%RmMBThZkQ1Ysi{O=NIU#IrY9GM5%v z_~&1r`Hh78ctDw@R2h+MzpaLb#Dhv4klE2qAX4Y(EssqYD6tB!-Mi|u0(yQ1T zB*!AF05(638R#>E3@=c9snBRS0ZMi{y+4rbx?J8;*E>iJGgIa4EH)*a;N~ncU4#&^ zoOYPp<$AKrbME&sAMUXJAng3puxz(AWztHX;w<2jH>9~ro`Xxaa&(sL6CC~$tt=5_ zJ_L>l4v6W=>);13A2lO~3lBztndN(P=;Z&A_rsum82_bSLGRV|k7p0YN{B^ml7S+~ zfo0NV>KVp2@_S?Zh#{ThG0W(FaFgX3UHRAY&_GGWefxs&239H`I7`W-BOC+=WeFw3 zIXFvN<0-HckejD1oV@iCFVE9Fj%JCbuRB}Dn;vT-9H)=s1Da@3xc}-|%{oFuXk5Nl zm1N;{wU_7VEcA5mZ)C$VwO}5+Qig3M!5_#zbK}Ii7`R59ksullDrRmCF!6KzF+n?X zONZ`_4Y{&QVI2=E=rf+NmKPl1xRJ}9X-NMSfU_#@vP(fT8M$*Lz_f}f(36<9yXkSJAwH)h~WXy$cAFj_LjW%qyabrxZ9Hqo|*Mj9G-cXxMp z3-0btaEI{Y(6|QgG!oq1X(YJ2CXEDljgaJWC-=FNb7tRDlW(-E*50-1UG7;FggKFk zcyB6lM4-yqC#6OQpe&;icN`@fb!8lziXY00!x=~~nRI4M>FR_l3(5TE9F^FMG>?1_ z%e3-RTv91KW@{!{ zF)>F~DbW_%^;g?h@^bUmA&*~d6w$*U<>Z2X*!)0uv2kHB??Xmb`9?7-^l#bU!+4VX z{$h=`|L3iEvD{OfwAt{H9@I~(`<8GS=$x0Rbo0(%vx?4`X=42FTbp0yH~j>fR=!+GTUz!zE;LLHL1*l#H0XPB^&(u?LP=_M$?{&hKj>c}) zQbwYeS={$h-B?j3 zquFkyTl{ZD&kdW3pxOk%dp9jm>8H9N4>&W{p5+mT zg?_iByZbDbzU{A5v&&>784E(US*`G-41%fBxyXmC`l4s$dM|e)<|IMJtNeRb4|PHF zxL-Cpq|K$ljuh<|(I;vZ_3V0SBt5naK2*W%h7*!iBYw@a6Ueg3kvPbB;shlsx8R=& zo=buB@AuEpkuMyQ^0mi_Mfb-vIBE#o;t1q)wUMREx?tdB4Yw$KQj+ZgwJYcp8pK9L?$QHx`m>i|0ZpH&kK4eaEmi~My7Q~lULr2Fn)@2`*|AY^RV>S?RK}2=(~7#(14pFKQH_uX z=6J>_%S%afHQTM;W$alrfkJYm)Z|Mv+26E*o+A|e^o5avlXS(GQ?Bo?qArx%W>3E zO7)q{`uC~1T-qm7_fPU{g0tX%(<+@S2VT7A!MVrR{N>ORAx&!SX@aGc#=y;*^XI;h zU9}B=`R~AKeV=f{v^9zKGdA&OcIto$*<&PA0D~_I`&U*>nE;hh>@Jb~VrPo`I1gTg_^SYzGyF@%34gr#octB*OZrLKmhcxJk)&=FG2Q=-`4fZhv+_rkynVdvh@`+` zvs-zI9Qiq_4qd87U*D;B+HI0-<=~=|J_j$7U#VNhoD2HT2pRkI4DiOrt5Qw)A0K@tGV{j+O1; z=iVlUb2$aq8=(kFEeKvxh#K0zm#H_)%`E2FNU$w>X#4@vo&u8N2CH&Ua4r*3;yA|C zh< zA1AI%EXgU()cP^7U`39Q$KEe1JI;U&pvy`M{iQp99OS!b6^%5VU-~M3Dm=X|$snu0 zL24?|=lG&WEUw>7mkKs({Uy?-?qzJ~s0fL-u-SrB%Ik5QR%YX$=AqR7REMfw0=+hdGZ@n^t}0J0 z*0)ZW@EqHAenR#1t+wOXY>|;jo+bm6bL?14q0tyBJ&m3X{iPf_qo!L+~d*q?KVQ7q;0?rz20zrQT9Xq zoEwo28zZx(BNJ}sPokUBOKv+6r-8M{NR)ywbZf-%X=;4r1GjRz82A`@9xkypDL@|u* zlwhMVr55V-X<9{0d?x1K>IzRuaMN`KchL#`}8 z6%-JwKz>T^Zn|qU|7iZ^GnFt$bQHywa{ODyQC~vO7&_ye={(@M z6Bu}9&q1r8#3R738J|bcX8c1Sso@3sg@0z8R>#@ELpg&Hoz#V<@Mq-3Ja#8?4H=Fl zc23B7i+iR7#P=t$b>gxc%LOg!@N!@4N*MdJ9j)HKv}BxSZe1I4N7ir`+Pk%W;H4At zUgQ_=j{Brow&=mI@Bb}n7X6NK0$By1nDKakf~NNLS%NVC&b#p)<{?aIKS=g!S2iP+>b$uohCEWAAR`gu-k{90Z*dLKsOc1P!$J7ue z`|2_I!!!>=yx`+|pnEo41C=ta0940%9x2l2MHJxl(d*c)?<>=i5Burd?F!(=1O-l4 zV#I`=ls(_i_*#6#Fl+;H!cg`?P8D@c)TNSKOIt0my6!MoJ!9s#8jRkzneGRLna9B9 zUHS_w(fU9;(f4}^rHx49p976Cy+U9dGt|GPuW#!`(UcR6?jqIxapUXjsmI;GjHNvu zYF*l^>O--gl&z>R_tS7j45w;d1c@cue!2eXXdp3^&NIe{?ww(bkRcp@fDUjEx*G}N zSW&H_2lt~@K!n{YgalpR?cx1xDg$uJ6VGLU+rN9l{X=Xj8}vJTZH{AVV7vt)o|lgV z-)bz|HYIxE8c8p&ZVl?jtoVDs&dH2l(zZE?U)0Y6WVbjT$5bXaqS+njVSoEm%Y?9q*y_4}U6hJVg@^6ilw+Q?XgBJo^? zF-Y=Drn#)vCTSUG6$>>`N4%|VJgaU&BZsZm+i^Xd{jXY;iQ1pLWVz}yhKtV=^gnb& z#|<#GipGU&Y3{Qz`c1>ay3E4}CxAD#w9Ph9pZAr&fQPKSYV`{-G~c@#VWO?f?Iu`) zIsSqlz9X89Atc6s!-#uQf_|P+N=oGMV{0}~ur+2qRQaHT#(=E774UEb4j=^_)0Lfv0TDpbd>cr3qB zUhQnDXN4p}By<;xeb|9DW;F;a+IZuTO7!HtZ{y)4jT>E#*N%6gl0 z1w}jP3%3p$Dcfm@-8ctwL%j#12-Wn zBmXPaFwXdrWlP!vB@b*WVP5%;tvR|tiSmt?Krd}q=F$|);s3sv{-phA^fNlw{zFxg zk|Bjq>&Sxdu}>1=esDXyrW5<=qCL8n_vSou3hfme7^i69wPmRY#|mc(=Y?<*l~SM?cV8g1=J{c~C2 zg599Qz_2|bO+Jk67UX?H)bwtt)Lo+KI@kI)S4_^M9WOz`NsaNlPozTV!<0?@jV&qK zzo#7lt>HCnqX9zQwrchm`59trvgVa_S31poH^JL0kj zu)e-;7lx2zpq(-PcB3G)=UsG;J_8!42dxn(EonUE!C&bAb6I# z4bn{`P_>ER_Ro=E(RBOaa3dQR3@o39QK2Bw23q0{r3(R_YfZ+&?(K;g?zfz-M&SVA zBpyX$1V*+X}M+|wE%$#2@5XqInmc}h4X;|hxhU%GoGaTlN` zHLq4glTk%Pn>sjL5ke_)?JlotufIu}l=fLHagJ)_j5kRU{G8tU8$H4P7%*J5mNiX1 zdN#daO%?A`!CG|QrJ82vqUrwHu92cdv->a4ya3=uxuH#FcC@1o%6YRsAqQ@L@cD0B zB1$(?bzK(i_ny7)%+R$z&&*=y2(d3V4~!oLV9*6Q5|eiprM4j$@J*HM9h@DZ7zzxX zwpSnY&dzZ3)_1wI&31~_wBCvE@}15-3{&>rTi_}6v@PV(%P@VM#+L8!P$V>HI3I9z%@^eMiPk#`v?+6ES3#l^`{xltS~Dxq-|~w)$#3{5f?%<(+2}Z?-q@6a(<}ik z8p$oPAE@9R75HRspKFZ9hYEt`(M*s5H=a0I_w-#}^W9uZVwqq`XXwqo0?GDr1kP~h zq~Nf7gFG~hlq0H=Z@IfVOYSiYGE4|c>Q9xAL{*my*C;sm@z|jH-59Y#k4dWPqcUti z0)DojvjGtz81DYo z3f-puzOAl(bY+^)5h*fqJ2{4vQ5tx8_~?t_59k2fUd$#T3xk95Omw2kC9ilRC<{xG zNX0PUptQGpsf1^8!H^UEY9c>f+lz|zx$*w1r(D}GabO|=&m4L*O+RT}F;0<)vT()0 z2}_A}0nuhk<-6jSdXTk$2-dIJ2nIKk@H0;kRQPM1&S>79xxrLSO@Zcl5-I83O7Cj6 z_T5tPq&B3{+*S(Oa~T?6-NAoXlt{sYmGtVpWxqC^@$bO=wlO6^$}m&>(7&qA6DL<| zJb7tYpDdZ3Q#fXJ!p7{%uAG&{iBHmM<3DDUk7}%epSVWKCUm*U)^TmN3@GGBnkaL@1*p+y@i~7HC@z5UTn{pn5aQ1 z`?=uzfWUGBE~Wiuxz76LXZ`oY9S1@xefc6QQD@40c4G=VhqxeZ|3y1EjFmGdSCa)n%w%XmhL`sh6z)$PhDXLmwh4jXOdYf)9QFxY492-kF^%5OM%a$9 zExhO>w-$dS^HC-%=V$+lf%mEggj{Z8*Qt?$3m$A`6Rwg5(eC)0)sH|@iKA={Cx7X7(1P$*0aZ@su!jqko0 zjfMoTQb|CI$6dPoc(+nj>UIi$^n~IH5vJPTQRBY3ZV`_j_5caeGj)(^H6> z$CkUG+Cn`G#VxsHx#0$KDq&k9apE=LZ)hr|3B#%SfLXC~tg#4spjBDTSnCuA&bL6x z&dGkDYxCH+R)pjaO{v6l08NqZ`=sa-{}|+(V824Ni9?#3cPhwsEK{Iq7O4d-lyw%1 zezuSPy${JxHHVnaN2-XZD5ENvdnERIEtHBqa)D8WDowPf)gM{wT)fHMov6-}y*@bF zBG=J5gEgW>^-vxqd6_I&RkrNlCBz-9p*QA=_I2_lo}s@zex+K5{9xR06{-1^cWoFT z3|gp)4hu1z%+-%us2FAsTQ(5GGR~FQ>yAoI^RJVg=g<=3hk#l5sDpp{WJp&;7`;el zqx1DQu*T-uOkA|ovG#@Y;?#S_1O^7P!Hsh(*ZX)blOV*i?= zN^t=UR>SnZml{H&)rd?9CA^4Kuw=vb2@!1AV&-~BC{W|PG3|cca)!Er%E+NPt_oq9 zk?IMAT7-Tys)mn@3nF`mNOST~C5y!4Pcy}^q|8`i_6>bKB~Y#VNa=c;gO%Kk?oHK3 zkf90oMTbk|EV7pN(X$L;rb(ci9GzCom(`Tzfqey?@iNSF*#>FbA7@{_N&NLh#JL{& zk@aJ+zdP;KNu;?AS(e+imo&9O+Jv#WAH~={ObO@s$g&{HSPZ4rUJg_wo4!X|Ia`lT z2<;}8-g48$2_jzRq`EBBSpV;107zzR(NajRV?PCl-xK5Uz*f+b&g`k!M-C5V9aOt= z&Q)b9X_R4yCS^MTpzhZo+QOC++DyFpYWm;L^W5o?r~<%65@V)6fukMzUT^LXQC8^0 z`-CKEJnZDaDp~_2`PaRCr1E}~c;jlI{$XO9l8eX7d651^hRP4A&nk7a95I%>A*$mf zxp#{%oSw70a*k~ZC>+w1h1f;JVO?8xa!my_Jy4kLbF~=u^SpC-=F6e|NqKWU{p?x% z7OZQsUUZL4Ldro)2q}0Y8r;Fm|FHlk?vx{(!>!xYxh*plO?W8ul)I{G;G)l`LGIgV zDG2aeem94$Vbl;f9b6r)K;-aYL5;}da#Ww}&84M_5{J@%u#b;-!7lu+r+18PV=o%f zO>>w0?qFa&{1(dqVyXJyAZM$?7?K}nf{m|Z)UPJl_a+(XY1w=DpH61D{hAp#6F(Y0 z3O*kg{8V7fwL}fw)c<2(op}013+3#yPL$`saeZ-FmDW<#vI+{u1CM#PtiS-mA#d(W z?IyR?S~y;*`v|FhAMQ|@_qm_Bm7Z%@>(?moW`10CgVnJ{1C_@1GR9+#Ef5<4NHPe?&)$vx>^F&8Hvgj`PY(cuGeLZ(TDDlZ-> z)*83>7FjI;iRr#vh?WzvQicLkV|Vt?Um#W zuhwq`1KRQ<;kZm3Ao9obq#m%5>7t$fHG7*tQJZ+ZdebA}s;8|8%tPg@K}V)JhkDBC@UdTU=FGQA9dm!_Y$}K2N6~DkRmjx6523iV?PhHcc80iP)r- zx2YDtC@_fnbt6a#DEWciUq_!BB@z}4>8XZR%i2b7cz0u3T^tBqS*(*bV};$ghPaynsLbp zCF2pNQz^dS)qtpN0?Boec1=y)3XyjXDUYgaIu5)gs@Ef)t2In#k@QMgoCIoYtPH(e z@klQ#b>BUR9EfJ8p*CTQ9?EQ~Wu*9$Q_*1;h(Qq1TvjcnZ6;g2bJ`NAgS2;`u#zLJ z+KVOn@;7o?zuokd+iEyj`pB4OmpYZ?$E(hp5~e71x3$;MTaNg!MHz*S4l#_9l2#AI#>sM-T1NZAw7|wG-{9Y15hYIT*mo&gJVyp6dXy67R-I#CcC0VCv6julq7? z5OvN#b%+zpN=9vF%~+c8^YEmaqPfLLwtz}jFEMBRMEII1)n2Oq&Zb;yJlW;`)L$3v z3+@5^mEghN&ieks2(~2Tbh~mNP08m$ex;?*;7}=~PYKI7-xW?3$6WN{^=anoSTYI)suk0^pE z8~q_$*nCj2(V9-N7uxQus+o?BVtuO;m!~fn-{AQwur`}9cAg0_)bxZu&<%mpkkcAK z?YJq*7Q3inb9E%(-Wz_;sl9B?_~UQ`m;OiJ)IFh}+(m6D1F-EBlmqk1(m}zW>qlVm z1_IM^1Rd$)3RLx($&H!fx^cD_hCPv_OzGQVqm6wFg|NE&d270A@g;3Ou=ft+H4xqu zWiv{i@%L0<$oPB~BR0>wu!T*tTESStPJ&!Z@u?|j`u{{agp^t8m*uW3AjPB>Ba-v4 zMayzj5RzTAO7jz7?1(;%d&S-7q#{z5by`+PAwf*Fg~z~|-u=?rb3 zCnT#|Qce6TD%iw?oitF&P;Svf^>NhPwam$2p1hitFRC4rHpR~CXV*HAK$ijKNC>;v z&IDc!?U!6ql3XGYUut21A#r;vfvA0JV?kh)@h&90!18hIyfnDst< zYVP7er+1riSxxwxZ^s1CumVlcBB3HcmC$9Q${44=a(rea3WM>!Bdvp3L$oNXQsN9G zqUiAbg)062fG9FNSW?xam3#s?MvY$f^;e#lzl8H*40zdGDR;($t0nmO&_+VZ64%Fl zpV%z>oPI1KxOQu^53;2B^J#tkF5WNG?pX0t8sokPy^uK#cDlm%pUNI8nSwm7<1tUr z2h7Q`?hr8lsY4nym{}E!s)h+xOL*&lIZc0o{HF>7UY8oe4B^dH?9=2LVvm^XUTfjC zFjq3cLCbGz);z{|P4Q%ZC8u0ix2OtC`^5BxKdNf8fQnYsW^uBq1|N83Y7SKCEj0gR zGFH7|Tox3F6o3C;1xFn!^GJ0l5wNbtnt#-h$@rw+yUcb5+N-G3U#Xe%J@?3S5iP;w zoIo~&LShir?$a6!D zp~--6T7xCwZ8kcJf;Qs#Jjn70cuiHYTz!Q*1*wF-c%sBTgjMuz#-g-fO;uhadFal4 zD=ulEF*gt5_D=4ex7ushUd-M94-zpS{s#JMbo)Oq{o4WI{`WaI@UZC9pqLfGvQ;{p z*Hcrio1*yU*_TuvOxwIa2J8uULKA0PnqCtYyxKGiXALP{Cz2z6=g(pRC&i)mF`{Qa z=;q@-fd~9SnaTU6Mk|NYf-Htu!UK9LVZ3FfFK_etM=ch4oF^sG(3$78!@2=xC(k(j zeCde~p;9KKx}O_Mp3z1YNtS6I+#%LNme1788Mn5mN$?Zn&R7%i&ri0cHScI7MzmUVJpdxftHP2G|KHw^PDihg02 z&3Tly5I&p{aGLc@8HEaEkyLl2;eQgGrA5T6J!h##>;rS3($X(#Tsc=ND(|jVj~itB zB8Ng&W;w#P-l_zi^Ce?DyE;e2DgNVBfK-VhU<4mzr+BGd~t| z#P-v4l+H_Hu6rl?{TK4%w*2KkfZ44*N;kHvDAFEp(+B-=KW6iTSf(m_*GkiNTZ-a$ zG)hCSaL4Ww3m3tXxNfDxK+@aID=N%Oon2?Q34U!BVz%<PUCBtNypc|AK zNQj&8sEIe};tyebafP}G$>9mff4~zN`fF$N-tc-Ci7aqq`A!JKZ&!wkPB-_iSpuR; zFus$gN@1p2BTew=dJ%Xjd{$N38t!T~2OBVEApS=C;GdTj4m^W+wiPd&EiU5?@Kvtm z&K878Dx4#K5zqn&5JhNt*|O(X89wQ+*Ua0q6FZd!hbFHSVb@}VR?2>q)akUCs%X)1 zx7hR5iojW2kk=Kz7~3`dH}-`yh+T;VJn%*Z*QO#?&PbMbXZncp{Ys%Qk-aOO5WK;? zIAMPfG+!X3N+9-;#fnwi->>+r+nznYGWz75ZC{Vt_~-YZoGV*q2_M5(<6T+e$nX|+ zUPidJ5Tvdyb1F;rs@I}WKNkc-)d;Em#r*oK5`QCPRw7+B5q?A3mkN|-{(JUG=HQj> zab}|lwDg`l4DavR@kBW<%A@s3iL>cfhNn=3?TX>%!LP@RK$bnfLmw33%OHVr9OuHh zKa%i0yz@`7b*FW8{vRI*OS%ObQUpq@_+!-mNMd&%p}g+BV+L$|e+P5|CaiplIjU%z zo9gbUg0+a}ES7kED-vSa^W@ONQfx>I3`s%rfg{d*%}y(kNgrsH>oymRWRPcktLQbA z)MR{@z{J(OJn*HO_F#6DjuzONR9-2l)4A}&r^hPsvnPXZ>ZQ4&91;dBqfL&IZb!oe zh~rv6N*)?d2;$?qjUXbqC7XFC!$o5lo~81n=eub`kt)!kx#8|RmLOKI?Z)G+j%#Rg zgA`S;_X&YlzI4E-;7!)?)!G_yE3GW(ntJ&UEQ*pDAEP`IRVvt#XOPdt7V*6yscwH{ zFIP8NjTL9zg}GFd$!F`IKDY7Ow=2Cvc`pr9md;g3uM0UVF=))IPwMi~BL7A{r9+mU zhDE9*5VVzR-~%Dl2{io~Ssub~`b;usv2a7mwzmVx7>hq#OKPU0BRIiwoM2uk^KZI@ zB*F6}1%6$ol&4g(Xm)^Z$@R(o@pCN2aGG)7@g~2nhmNfnj^5O1Eq%a|B zZ~>Ww%sF+&BD&%^FB!Un8$5~|vmDtx zeQ(PezWlOy2KM&je^faQMphNs0A6D$N2(;W{VHaYKF*Be{n*C|$9prZk-1`DgSRC! zvq(-krLu`uMiF;KEVC09MVHhVl8@5e})aTz@A%U*g|3T@gq|9pWq;%SdTk5YZC z_$(ypmz&S>Ectc;;o^mo3?K@}1+>hAv$4AeMFb|x7;v-c&(F7ZUo8-0f+?ovNrQCU z6EY-Oyp1KyO61EO;*__DQW)Vi?(oeLeZ1nNUh`8typS}$dlV}2C(Od%mZj(b61qu@ z;HppEc{t&V#VhV+U$+LuB4<7n^$`3)zRl6^=wiXhlJ*Yg4MrVZSH>+0p2R2wvuOX! zrfy?1mkZ&c@%0LvJw6J@ii~;*SkC_W<{saXU#dRBx|%aD`c8PsavSM{8GrcMgto|z zK5Jas>Nxow{^VUqD{8QZqL|epo+B*Vo5OWE2pH zOXy*a0n&ZYn7R$Q_(JmE-?kJP0)um~<=@{$e`PZ8UcTwH-tt_3|CaKfz~8waB8UI$ z$Bc|2je(1Y4wt^9U9qv$)nBiRvwt-yeSdeJH-7*7>-6Iv{is7x;ceC}j#jE(xU)lp zu*gYv;fP>RU&X2*xE2n2PV3p9FL8f#@-^oZaZqexEbGIq?fr|*$( zzqZaP<#O*(k8UwQX#gpG9?^vTRoOk$$+$W3Pc{-2|GjT*-TA_&P@iFJ-hf!|^XKLXPuRpO2=(Sg()32r*(&ZV`i7%wx?DqlzTgwc)`vAlOgz!ZjpaVUW7Ly$Quf z&iuC-NW|u#nnbW-_q>fE-?rzjFLbk8fvolvc|wY!?XFirk|=Z;J3Ocz-RzHREtYi2KKuiKz1*xjEEM;W2f$K@IFRX`B#}Xh&8>5rw7Gvs%I*C^C(Mg9SAo2^bZhhjw`Jr%If;6ETGCQ}X9so~2!#A{39%?Umdh|m zH?lZuyC^fYhyzzrSLrb%8oy(o6q-f8UkMtE5w0=Hgyg6}bw3A+-&MR2^hUiZXZ!c zSSm~ho+2B( zxmJ@sr-@b<=khlrRdmQ-kEoRWx$5&-A@9EJs~Pm^ia5T%7}Nd4`>7_7I_PHEv~WXs z5Vpe9P8?Zv4zks{@ZBK#9?*n-$?DXj?jQ=2mvFLNGaB&Zz%s{^@O+83#=Uu7s`2%X zeWH{q`?xO~lh~>I0b{s^CV!fWWcvM@ZElzPeIw`5Bor2`?amBCsD&q3wP?T{>kztN zJMna?2SSBJ2U45i%RU8K`rfi}-^eO68^7~F@ppUG(g!{fN%lk(NOyKLV$!%xkg`c^ zhD_K{uMPJKgAe!NddAzYP?B2>62ea`A_0*&o#o&1PfAo_fieV!#mbhtT>X?jqAEXR zB`C95ylA7tQwEob7CAz6u<0|Pw$y2KCyanoHa zX{i;fwvX1-?@)QQu?!TxsTgSLop%Uyk<>CUSex_Zt%2UhBe9txQ(OiA3Abc~Q?JUB zBZY}((j8zadC$GWuf9&fjeWWhGeE;iTe0vDB9soX zIKVE>IXl%XuK0q0@4*Q{ifAaMfYX|VVQVW*q{_>v%>pcB=&J_A2_@`UD7#?VK(D_^ zk~Wf&nWE?LdnK&_7}7-QF6n+u(*=3UxhZE>xpC0y)d$Hp`rPCiuq+AFHUnG1#-Eok z6Y-x?5*gKP$eeXl@2UI`Ct45WEvmk3PjSJejnZuZ!^dFhOMj$vf6qo>A@B-kjO6qV z%Qv2sX~ne2YV(If7~=f>?J-I_CFu7!ZJfm6SMOmW)WqMnqEiB*%7S-_F;A#|KBPiG zdapZ5VCpLuj(*Us|ai zkPMNbf7JimNF0?J!u<@@E8tPwt-;*G5d4QoA|5|jjgd^SU%fh;>;{P6shtu%u}&%W zlP|m-dAt5WRvShVjVqg}G3AUJstdz^jrwovk57jX2EU%U#+W$Imn<;TZ!T+Z4C5G)7-|1@YV7Kq5UzS`Dw-Q9*r>C7abYwx^SHfEvvPpwx`{ zY~1JmcY8pJKl!??C|}NkvTew3VOv4XUo-ctosM5<=#UmmC!tPFuJm*$B*OAg6)f!Q{>zDP1jDLncVfO!rFvi$Bm48P7wNd~* z+%`y}4F7|b;&`p7)-E$3D~v<&bgJ(OsVN;%^S)aX_B(shB{jKuv-3s($uIz_OxRkO zJ-qBwV7eRV$O3H6$QjB{LL7!HLK{YnjY5G)fhPxT_J3l3dQfzW^vuxp!OsxFzCc4`Y8GabalA9l2XnMyEs#9^h~j!3(u_?Aw>4ZQ6V?W#%>#%sqzaAy!|Hwu*Z5@7q7+znlWUk!iyDU!aI^{FL+hn&$7s;C3 zKU$4%=p&M&Qsy1iOoG>n5Glbzdo$`zG5fXrqhldD_?Xg)Op8-HaX!!22SUNH!$5!P zb;S>firA4{@;y9cY_ybYu)0R9MRS{OeML+rC0`u@#H3LTp8X-W1LVvhkxi;dbG%4V(|{3S2Md&E#SL6`Q~)i8|KWJ;p>GEpo!jpFb3K zw-E<0-%|Q>!nstTtCsQx#)LNnnm0g}lw|E#;4{Foo^_RAkLVfolsjsDx8nrcm&B#e z0CtfxhK`Yr6iTuruL->C1?nThR^`S?fue(8Gyzl43Tkv+u&yDD7^w$c%WiW@)4MI0 zmq4^Es2RTuV~S~z@Vob62ZBB*|;Q+!+K*415#3*CarjZJxgf~U4zk~6k6ZffEIrPw_2PYz8et;-xQ5nDGh_g z;8d+z4vh3?z~B;2_o|U3R~DqS2%vP#qKA2*>;ZtK9ui`oXdG&St#%Vcdml8aT1iUA z1vXWvxDhay+;$r_pYrvZyT*J32ax06>_=yEV#-kVPdj+x@wBF!$ha#Tvaor8!9?^=>bxlx)S|jn$ZD|BIwF2wZxcvVZu|vn(Z}fZ8qep`s&d z>L;-ggac)OhK^KJt+R4ZTuL3_5L`3Y{Bx8PN$68m;0zeI6;Ur7XAnVM2<@wAY%ll8 za3ug+X8OCkbM5&B(;#lLsiBcei;=}9)IBOZ*cfp$8bKDBwz>cTm6carhzzNntkei+ zvlb|1XM;0Lg1}heDI4sO0hb7LB{E&YMfihwTsla{)+yCuWTzLb7UDxnsZgj>G%u+O zBSzjbG3-R37gYJIK+7_g0gWnDY(_ymV0oNDiCUjpEP=*EjxB~elGvOf91*$(AWDg& zXyrzr#EW<9;~tT{Q}%bjmqp;ac0~998K%cY3E@Rc@zH3Izmm7qZe3lC{$!wNLq$Jo zhOc4dG>5LQ+jo{&uL`>wQ_;e}Hg&)cqK#BS)|dy7W!{$6#4)tq#~CTXj4)@_eARk^ z%Zl435cNb5Ms6i?HcUvCQ|Xf1*Iw2#QRhT&&j|Is&-1(idgTbs)B!?3M z>7`D)9GsVOq@zmKUS_8B6pZLj@`Vjqr!V6z^2|XA3>>fAFJWM=V@%cGW#bIUjX)~+ z<8dETij5?fc3iXX$aq6{saKUo%7vSR10lwD0W|oyYlzSd6f^W8V8SM|flHv##F58c zH32-@jjzKTQ1|AQeYFrBEFj@a%!KMoIfj;8iXX1YS^!F|p8%n@p`lNl}1>20pe2)&@(8168by`5NjZ9MHk9pJ_w6(B8Vo=%Fzvx z3o}$DH4JYeS>H?7#MClQZ^7h)*Ntk_;j_J}xtPHD$5ofe9C0#yD%Lhn9+fXBh$#I~ zRtjEf7;$2KXUImH6;3!C{#QIx-Fa#g_h(FNN*ne_eM!7jNA(l5rd$9C8_O|5H~&Z^ z`o_Nua{Clf3oAJyB37dOS2V*`Zn(N^5GqX~5$h2N=471##K6+=6c&sN_fAKccYw;^ zyqk)#e~`0p!W7N;K85aRRo#g{ub%(26nPPutSJLCK_!wpl(MG-=?{mnwq$`8p8#pN z)Kkb6J_1Md8k*qy80t}q+J*kr(IbJe?0ij&cqK>Pd=Moek1u(Y)qk=X2#TayMDID8 zi(nZA7SdFMWAc&Yh+W^+j;UxhaUC<5%EEiC1ZCtDWfD~XiB{QE7!^hOAsLa<{e|=5 z%f4qmCt;^SlcpMe<<}%}ThN;GW&$v@Sfz4PTF(QJhY0f*L)d3F82zJOIB@Zp z&Q{n(mLLSXv{IDN z!-$lBGluKInvSYJ2w8xO!=5>zrTQ74y|ga0cFWr50N z4gF*ao8?+PwzKE;m_D2z-}gYts0jtu6Q6FLFC}3ek&w2MVc2tlzNu zud@4}JL<1vzpKvqn_TB(O(ohn0peMs30)nYZ> z6spg{z|R~}>tI}gT~W)h=mC=pMb*j~4IKZ=aVq5C$ilr3rP4rjA9m-*HgQx`d#(tG z&gcjnx&+AO3+Td)TzQmVA6czLzxZp^{=0ilV2y6ugv36fDL$d6sVV-S00uDm&%_n$ zmqLIHJkM!03&UckHK2b|tWc6EfG6qaDXxXM!BGSpL24wqD1rPusB(zFgV2SZL|Kl! zg+^bIpaT}tMO2Px$k7SP2^FzE-YAe9O+j1~>`5X()RY+7{{`+91nP^5G+Z5n(-sym zL`fa9icyjR0?P=^gy*Oa&?xfC0$%!#Kx>7zUJ6nJ{W44qHVL@w5VZwlM%)B+rkiBs zdrrnv*CWUl3QJz7qo5Uw`MgBgs*tZDSzQ+zya-r4sW$*wfg$sO&g*;-w^em7xHsT8 z8jlIV?L=n!<=Tjc#31w*DcO!P%XYwffy5lJ9*v-|Q)GjIwLHO$5b9t6wQKpXp0Z1X zZB`PdCGX4Wr$rBOABEnDOAr-87gos#xz24~$C3D}Z0D711Cpa*aY zTo4MrVzU!3fIJ9HYg4>D(Rg2NM=Rpf;3eg2Jk^-$WGoMQlsr?5Jk-C z!C_!4*OS)+143y;--$>%zykw@z>BZsngo`@8CoNG#wl|2o51>lyCAsns$H%L9F1zr zux#>?;O4+xL>WYExYnW~9a=z9*^J|~5>5$hg=)YZ*dU9l{t<|d+lw@pd(qt^NtN)T zACuQJyyi#||A8-47a;sy_YUDM zo>x+s18b)JsL@i0rR!u+X@zJhsUce0N0IW#;_e_yO%zU56jAe3elyc<0VZ`|>uaW; zDNgXbNZlkLpwdf_`pIfNQ0ebVEuXNhNjfcq zBPR)Yt!JS3ddAeKABm_E8p;GB6Bkw{su?Ss>jILB*#bxGaLsfA)r@Or6{Vy~6DqL& zbR^a$AJT0q0H1@>K`dsyf~aoWLU_5DU3l~&I;5&V|;3cuUmzWZ4ibPcctg33O^^!jD6^YJGm3K)DlTy#4 zB!AA$!3pf3R;4zaWGhkcPJ1EF1x>_gguUrnG}7R}MN*;w8y#fTKA}z*8x8}$V&b87 z^yFp|gI735q(Qa61HuB?et?<59`nTy)5Kqv&hsuIRTJ&ACou+=Hq zyC=l)c&5*q)KT19q?0klfE5>rKWIUe%up6v_l3|jF7yEh4mcCDXzU@kUq{c`6VLX` zHd=K_h&<*zJoAaK)im%}6(yfSRJ+W?BM0Q6E&w9)lu6@0E#BHjjRB1-aT(XC>4B*X z#cy=G*g3Ta_!5lD6C=Gy2X2UALMNINDw0I26$;$aOK!c1gQiP-<_IITAM)q}V*6(f z?my4qfKt~%Utn|k+F#c=HvdRR+`&0@SBLx^fPha$)?%qgWB?+Ew`aK5`9Cm!e>3nu z_*V9a8#q#XjE&=**079kd7s|g%OdDy>g9%7amMzn6JX>tw)nCCAOlxG^e><1#Xxd9 zZ`)#I&i6mwR9(2^ziGto6lRK$2*{|ucjhDo4?^JNfmkKrClNA^%KyjSyQRsF>RO}k zZc98^;@e zqtc>N7)R9daIm~AkYc=%8LPw6-ZmCAX<>|s7@LG(Er*2=@F)t#C}HC_MhRy_qUkIW zv{ei6hF#A5#7zt{Z^d&=7g$EY?ePGJY4fA)T4M*-6U?E)Fgg^BXYuXIw#XOM$!Z)E zNXx9E`-Wx5C;sX2S4xA`Y^;tJ#E~$`sUnVq4N71ji3CSkm2zp&tzHH172Uxz4UNT# z!RT%wYMhtpJvtYnwu9ra|Eu8u99|%?XIuR<|RN@B? zs#uukUc_RDnM<6V=^l~485!E5v6;7+1>5a!A`d$-YBM4>TOcn)@bUnPKSGwBKC|eG zhsksn*}+07npkXVVDoM=J-zA94?6^u8KH=Ai3`sLFch%``a^_a3+#t@e!Ymr@3Wv* z7O&6Z^dT;=Mdm|DJ{yc)#5AQAaj(Hu#`MT^Ut}C#Q?(&_?}*rYAY3m(^)f~;!}B5> zAp#NFKw8AR13`Ehf0qQ{k#|nU+C`9E1lT=9JZ#?Sfpi(E-wG?@@Zd;k04c#33Zc*n z>x{+wZ3j1|6kEZ&5v#Q@3*7= zH`@XC3c5`(-k*hw!~su%Mn1sh4QXPUnHHv%X(NC_h{a>H5MUv|LVyJTtu-ZkVL~m0 zTG-4=pp`(Yj>7Nz*qFJo1ud|a#wIlOsj*LuwKvw@P`epn0wUWmrl#^iU>cbwraX#% zmeEXe5k+A_6+#tZN}xiZ62I@`|B__?xeP`!f9I~A43`_8TFN-Loe`_jqU;6uAqGa! zP`DAPosnYG$5n6-bP$o3g*UQtB<$Lf@1@xMmiV}=UfS9(y} zlAN$4CqyO<32Sn~nv77S0ghS$;YSzjJ#9T*9hYHr%ZOT#;rM6an373NJt@%K6+|y7 z2&3FxBh!cW|9uP?Yu0CN;q1*kja)d3Ej~qrOQ)q4OR~b{(!L|@P5s-`QCP`;u*3eR z=UikF-SEw#_g`qsU(aAnrAtmKJSCa$GtS~mDEzgzJ*1}h)6>p!e4_6OeF|*{9+0d8 zz$36y!twUUt!)BOZba(an@5jrd>))MgD$&IdB2bekFuLj`iEg=)1QT{zdwp*Hr5}k znU5KSjW_3}>*kFN-1Wm{*Qd0t`al=|Y!5BwsZm+mtFp5Vion4CONrWIfc{4Y_c4Qo zWqP2>ze7Xk_cUY&Re!QM|KH8fc6F1*FpB?DF5}w_8EgmxWMr@*nQTbL8l1717O)`& zY)HYXT0mt5U<$yPQmy|4oB}WfU@E{=fT;je0p{;afA9zYT15XBGC+BTs1lZDrqnHb z4-x9NDZ>iY5zxM^&t0;nhGV{0C1GPd!~i&^-BjkidJng1QydQJKdwZo3TMFddwb#-L4IQZ7>ySC^C z(c;p@t5J!WaFPsOeeW{;FA(`N9EC|Drb>Lr99iI!SgAOwFKC`B0)<)}{lYV2MVT@V zQDIcA9ud^i%#;8N0R9E(E$j%lD2fir!Y;{Q|TX*Ru|YX3{*{ZR&sSL>8m74)3=NShfDjW`b)k z{txQ3{(c6m)PHB-EI#&MLi_)T3>NC!7Shi&@+hfl^FZq~;?za2Ln6hl>n6hl}I#AZlKydP-Pvcb?<2)z(9bs&nx5JSZF>xU=tY(6=#>gdm<%3 zzf2(#iJxe?V3HpNlbi;Jy1RVQbBK;Z^cy--yymV(Q}B7L-ZRu z=r<(v8d5vrOlsFT{se*5az@5)h0#pK00ty5a+>dPdi4agApg=%@fC5_< z2r#%GStP_rsF4680Y(Ch1Q-c076(9xkxfhhc-!x2$Dj$SV!Nq7khuS7#s4>C0Fo1t z{Nbb&kpBa#;D0y+EYN?V(E`0C(py^oXQp=};(uDuIzje76hO~(Yv7it<=gvqXK>zD z=GSVTWp5M~pTtFL2ZnX%Ps|2b>h8S|2C@BX7;PNO^FqF@J~a7*-v$=mf6`|6&t$;r zYGElz#gWc*QQi{89}Ehw$i(jOSGtdlCvoFR+;}D=>@)xSb>q_L`;2*n;``J8va$bs z1~7sM=`N86$ST8ltr<BMapHql*no<^Zh zCj!6YE;#yicD^bF(hS{J9*-B8$KgD)Z}LQXaEvyHeh%S`!>`I}OpBcex3BC+=uF_E zA{2L+t&kaS)S6?wQw4hR(n7borNu&(^W}H3-`3UYsGJpE|2oQtvRD!5%EY9ICb0#Ww5$V+ji^S;w6zVh!zFaMpHrS zHhq1nR@uq2VuDOm*RW+k*D7OKJdAF|6t$h%?mw%u`KXz=sO+{)qNUjxoIa#dUu0_H zKt`?{Wm_Qzg&2p`mzfj8Y1Q{(t|WI0D%dzrX9QrVv&wWm>v`K;EbK}~A{xt}#mx+c zNN63N&*4s5Y)*@FRfKxO%~@%zA`|-&Gu&qSWfTkr72%roGW{(?YjPbx9iwKhsE_lp z0kKe=EUhs0MD)F*a__)}Cb(Vs+=lP%B!+gOkdEmkXaP*0&MSTc>@kBO%EEEy8?r!p zG%g@|FeX4@)cX^TxyG$VC)Y$`GBv<+GjT(2o-N53Mx%*KBPLQ$oQS|eo=wfrN4l>7 z(e7~_YvYMHshkBsq1|DTsP$qU4Wwv_GNWD0E?gBId6FsNC`|eT_EFCrnY+30Y^0j9 zuObXZJ?Czuhvx%ElnW$R-eo*H;~pKA zrB`H3B@0eIro70KIWD!pXB+Ez^U)3aPQU=%&~je3+IN4pUHiDFF!m?IWYi5&8^FRS z(YCPPbsYCGl%3hC2pxnte96_1cj9ln)7y%N3D@qBO;m_F$`rUaUFb?orbS%|r=V7q zNp;G&hNe;Vt;rs`ZCvJnlrole5IGRtkEj@zeE;I<~d<6vh#z$D;uOU%18YQh4nP*o>fJ2%%6LEvZKAAT}o zV7?Y6;?k|V{YK`dX*U!nj@`!l=*btPz(~R1jN1gitciI}((5h+6q6j|oxE%~2D7L6 zCYwBBz?iy+^_^XYk8+=0cL zJOepx$!FJ1T{2ZHk%rg^@0J@4f2{ap1|SKl{fC9s9KjNfoTh%YQ{u$1c}i|Ol}&!V zRbK0DAZ^25)>l}rcGi~{9Gl3wF2%N*lGRLkE*)9!Je$qe#Epk{kLhH3!LG~R+}uXq z^{+sj$98m1a3&JGQ8t;RhMOuMAacmDA(wQ^Q{t?pVFkVmCJ>Jb@L6Uz4;V!6TU^Oo zMekYLxG;?s9jWMdnHVz5A<41oK2rW?q8bULhM(1*>wREuEws(}0Z`p>S%Jq2936E> zKn2b!`eeNxGHk7IZufHG-NY|}Pcqgdh|GrcJQu1H7_nQT`#!+y zfm`Odq&dT(=s;Y1l#xNhx`+I?*VbS}x}Oc^2_Di(%G1DiD$iXF1ak(73PA5qk%3#u(>xMk9P@j#&L z0CWA-yV*UP^K2DJr;k^p&^r-Esi;BR56c$A7~gZivEUpq`}*aYCpa{}@y_Hk(qAsG z5t|^bfn7t~x>MVQ9|=TY&?z;>XBmu|=VxAELab2m01=ClXi&+b$-HhR27P4QsRwGD zwQm_x6P7#(II#P2SWF4QSPW4<^GdK8WJGS?7s0e`|Bw`@VKLf_VeFYsVex#VXY+UT z7j-Au*-hPB>$geWGDdD*Q02mn-1=zG>{K=^7@Y4m@~Ir%>MrX281qDfcXCJ9h=aNcO3< z?FW%6WWVA!knCVr3bY>)DJGLrS*B(0XmM@vy$c$5RvYei%w&cuE(ifUu;his6lxai zN1akyZlivsS!_poeqNrtdP&#gOXiyAj1MIW`1e)X3mkQM>U6On2AY-vb=*AN?@iBC z50TA3-~h+fCHpe`%qqQ|_k|<9GX8R5<%K-DkbO6GPP|BG%Bcv25tcDX;P?&3aZ#xl z)8y*t1ZB4kS1C74vsnudL0( zK`z|;!d;Ptf%A~5gZcyvi7Py*B75TnVh=VeSLi#*>ylF>yA!8e z3njnnF;F=*d~j# z*rEVm_g6` z9|RLjPd6xa)z};m4X&MGtKzb5hefgZEqbwNJipXc(#h}|(^&Tbr4)f_glEPmM+B)~ z^EL<;6KxQ*>Uo!7Q2hf3mn)kA157RhBwT_rR~j5z;;Tb1f|F8s)U>D@p~wyxR>hR%IXJg(;ji*?Qi| z0BR0|o$IK`(_VhZvhGtv^Xs~ovVUa$`M)#P$s z#<|`KT9eF!R^%Zpk}YS z8*ZfjjadrT@=epAJ}1FZ{(xfN2Thk|>ViSNBk67X+U(EweW1@KDmj-5?*#E#z)kU) zx3~|5-ZyX@k8~{{H!|M#NqfRK#A*7liuLmR{M&hZh+eu$udzo~b^9hJ|0i0?Ng92FQ$$$MNMQ0iZn5H}k|PO3<3#r!;r zAmqQpb=*4yH9VV^9(GTt);T!Nmb;bPyrLE#BOJ0V{GK;9sI@=b&s=$L-?I*;7Cs}H z2Jtt`2)57&_1CKHf#;Tcu<-au+BXZh2{&AGdmE3$-6qwgdFR9&WUd9Jt2R!lBYlk0uU-0fh-05rPiPBZzIx$O!ASaR|+PynXm^hHjP@`m}6GRYs- z)dvR;oLOHyHz&6^yF|%C$Zea}HsTEScM^{&+`kJ*EM*_>1A+@IPQA3PamzMIX#>XB z|H#c=Quf&_^g4NjB$TCO;qO<^%6ETK&*!20w(U|Uv1gb-8x+Wg)H@l^Vw2nt^B=sr z$klyHO4YyG!g7UlSvQ5^h7BZz>&=2kw~0j&6mC+FP~Z{@Cx_H!d1&uh%A-!Z2!L}( zlKoJKP~-3qkQWrI>%p@{X{$KGg##-?YzVidLuk|jiWip&dq5?mE!7vRu()7CLFZ>3 zg5%*8(mt}xcxU9K?Q`hRAh$Jc(u>ZXZ!daGB?c&l#YDCdLTCy*EfgM?X6T!jwNKA@ zAx)aJWSU+-4k=`+LVL>_Tvpg@^bE_C3;XT?r-(u~gF=$1D0Dl=)7qUJ+FWtouD{M2 zRA{pATCzo+mUdC>z&3H=YR?A>K!4h8iehkOT2SZd3|qYK`(%+#7GvBep%*4QhD7F)lQZ3;22f?zY zlq}&j5otTkoJ9{y2#nxcK2)fF ze~=a$xSle-C#1Ot6}7p(vIJL>hwyr5#04L-Qp_TD!MTHKN9-lA7$ztS%+^EMPW{|o z+Eo-dXRWkgJfH{8A(3 zcY0~YMf?f0O2lQ7Q2w(JsKaX}@2r$A;+{t5+r#Q+Go5OcvY_~pA&P#v&OJcWvP75X zbE`uB0$sUmgv(*;4NZBsTG*pxSu;5jg=l-|;TBurGNEk*GjL?Gxq+meHL?G-YKZs%BK1(Hx<;bK;KRyOKb$!^A zKXx4+Iro;@M7uc?$$H>&9+1ZglsBr4x+C{7Pidd?$RZ5a*=sW>EaeJLS=K2aZV=1; zu&nX``D4ux)!&T=#;|@kfD3ZiX|H0q%YImFnS;A9RMYp!dmPsr2J_VVyvLezCJjL9lM;1lDyuQ5tn@-z@4@G4A$vk@>h^R~T4-$xq0+kR9G0mT|shtVT= z;M!I@6ZI3S)bN@IiuO)aq0-f3PZ}-I8_4HF;EtS1(ABd^v7c^0bOpXoz66qu@Y3mJSB6 z9Fp*0_~YSCN|VN&#HssA6*~Xbc$eVhFwdJ$!2y=Og?3|HW<s(Y&WqJoKHk{?8ep;5%&P|oHj zD3Fd=eLLp`8Rf6=?^1SE(1PcwfuRZuL1)NN6<uM$P3@9)!QG>9PprQtD&ozcx7Xy>z>*HJ^fS;e=12wK7puTDS(OdT*_i%*2J9lrf9yTh`OA7(FLHo}>D6EvE<(;B6j_9Fb|o28QWXj$ zLIFc4D@l{RA^_jD1@caGvMVUYq2CV*f*+H>k22U@L{Cv2Ze(!!6k6mcJ(sN{E-SgVd9(%7y3msUNxJRhs zq9TBxHwhD}^jnxPC!sUF2mk`MR@~)kylD4h>u7j# z8Dy%0_Y9Xken3oZQy0Jh$L-nDwqiZJ-gyWNs2;EXDT6Sq*fEhwGYkUl%r6uR^w!~!_LUDDT!)TETdaObqKl+$dZ@p)=NrQq}%?u*CBs1%xHIl!sm*+jBW z)PC=HTp^dM;5#KBH;>; z?h3{eff^UCDNmo00+Th*>@8JC`l?0CoiFn zccP;ZscENs#PweQ+iJJ>&g2T5$4q%K1yaWgb`yegfaNa3Aedpvd@vB1Fpxo2O9VdS z^7qOMYAUl?g)mDQj}V=saLeGqEO$DA8z3n6YtygOC{WNyge#yac=0}~=Tg!AEO131 zV6R8Or=A;vt5Q_)h|x6BN0_r~n(b3R8k?2G4UJnx$Kiz1US)SZf~_)NSC;8zVfP+h z-9DAa(6Hy73`)c9lzlAh*~DX;1Avf%tMptcQRKOIz+j16#V$i?0X;@o_uV0_o%dFB z75v(jCKG#Q?(T>o#@6nH)E0av%=2lJ0G`|Ieye*MDu7kJ^`xP5ZQ2W`ZJD~Fy2unK zmKKi~#^c+EFxDijEvu6)Y}hz7?W%pb&nPS_9GrM=yCN91b?H5U3CjY11k%p7?eS7Jf0V0U%Gxx)k{E_>#T+Sflcm6*SX?YaSJfZZE_l=-Sxjxk8azYxG3oK*rt_E1N zktN+WuWbTBQ(SG8{mJFOFqpV+t-g4)xm`#FA?6D$7}=!nA-^qa%hIj;>|X!S&IgA_ z;RXP1>oT@w?-dL#3!Y`!h_Jj<~!JW(7@k}8`V9=2OpaXj#tK2N3F_=MSqH!}bE z^7}dZCexd-A-Y(*$&lG4f&@7YgLxnEPUG#uskjJuOfAijM=;t$a_*dpcVSFtJ+cAf` zT~xh1<|qp%%BU=GV*ci|i86>b%&wW|a&2@-eL9`dB$r#)k3kgWd04jL0TKf>)EWAt zPsfus46mW?$ud)Z(2KQih4nN2ULI!*BTAl`Y*_Whr;cuv$E#q6LG!Mcv}k*=Ule53BmbBoWZ=WOhiqS1+yHCO37#>jCEb zJtEA=O@vIVG?!oYHtNEKg1k29lUk;Une&R_iwg?s($_@K zyG*;0f-KjV_+OciE_&X5uvK+O3%WlIX;66xm6;16>cUj1EZlsn^KyX1!;_yFft|+6 zg%w%?naDX$gep0;Ll*{8^?_qZ(@}hx^t%H*2q-*YqUvq$9n_XKl_MEmp5#+!tjNk? z%g9tQW(M`Q2Rrj(YtN-=f6sKJp9N6yG&DH%qB{@_O*b;876MJG_(&9xb#H)G?@ zO*Ys_8>3MeXbbr>w8giMK*O>xW;u_%Q;SyYS%tTR?Zb4fYWB5YmBP&_mvd9_XIXwL zeBC!}HfX7Z+=q!%Z$d+WLy3^)dRS+kbR2{~mG!WM6y1Og8fs;-4iJ9Vh>mURNC5(; zIwTC3C2!zM+;fiS<<^RQ1z6XaD%+4Ri|eJCi%YO&EOTfoa0rLFfJ>uMvS83E8h#*j z(p^VgF{o{dug^>$?=q@Da|LGXIBcTczN-j<78 z)QG(p209x00S6+?#pE(s;~~}reelxVTF0n#yvv;LuwVyV*xMmV*k}nCRyW2myu?QT z^mhnDkLI{&;tI8GzI)DQq8me9g$J4D#4wE@kK@m?2~98HluEsa=q_-MV$_SCxy609 zL1p-`V&q+C^xk{mo%ajOQV%x-ZkbLFh*=g_W)*|DWv+L3y^n*0NwsmFC*(_^)0(Eo zS@+!8T(`^z4FgbLay*1&F87Iu7A&?c2=@={CDc`aU_ey&t@JSQI;R)ZCTnESq&GBz z@4P^U@~+V0uowgxORfT$4{F-FbskE#2X|~Z>L|tiRpnY2KLO$Lp!?Pa=flfd4PEWx z34QLZxCd(IbYcxI@z`Zha ze|h$dMAY8Z`$3mAChP=<1uuDYA_wyru!;W(oF=ynj$=J_a^!i zJm*$b>zn&$p(%Z0=wg07i3lJe-Y?&!V6dF|hFL7u-yE>IRSWg$hq!MzzT<9PcUA~G z$aO}BI`_8+$OjA`?;U0@?OXPTc<=QAGdULLk8Ze7%Q~d{J|PRYyRg~!KnsXE`{DHW zb4SuMH}eQ3-}`c3Z_aznmVG&h>$E`ORbN+5?+oW`LcFF5$;OyXR+xRgkZ5q;$1(V=P)1bK1n6dXVCM%iNGfmE``!% zAzmrZZNh0E(0nHXv_ZBFM;6&lDD282ogK1BbW$D$aS^h(xS@dF~U;$qn|!_(;DPGNBousWaZyc5TDNHjW>Of~H+ zE+W)5o>_Ur=Ym67n%wRvriKK4P~Mvjk%ye8<~!dmE~1f+;pJHAuub9mUda1*pMHkv zuX)PTth1jI-!CX~>7N0Llk?K^T&WjQSDArT(P_(Dy_h!k^fv1T(2%YIaI{7vCNvyU41_ z+rNcJ+Th`MnZ(L_#o?^%&g5XRAn`vqY2xT=FtQqkbW(TQ?e1`b>f=mdO!dx0`vgGr zn~NsR_VPtZYaAOarxhxmL&F>LUgWrn9qn$3U&@u~l5(*&tyiZ%*hrPKpHlSFurJ`c z@cxK@WitRQA+L*6%2XPBg6y8cyQFGfaxe+$MTAmbT@7|*=m)i4|rk@#SV*oT@z` zKJbonp%8i~uRP`L1q~K>_0aTfR|aFsJOwq1&?A*Rd55%|OF>(j?Gxkf=7*c*)UREv zSj-AgZ~Xthf49Y^g$*Rb;=|y5pYE+I7?pw*65Fp+K8Vx*GwM0RV<}8LYsk@^v0w;% z{HQzxk1o{NV|YTXv9gzI>+2lm8lNemmhCxQW1m7fCQ6+R_DneP3b!FYA=WO_H_EoU zw4F+EWq_Ym1|EU_1OrLs_7})R(2PljJPPi2&a`DpSp`#zgN?Srjl6Oa79IpE zbO;zn!q4eOckVNCbfnE$jYBhmp;F%v4va=Vg41Z$=A>(lJ-yF*=vE&XS?J+B)4s$) z)#+1%YzWK+OMna$7nBw~OtYJE2L`a&q?VpZYc_Sq08BfTG1ta2L@!c?dw{vA(*D3a z`sVbg@_uccq~A{NPA(p5_Z^dq-T0z$rNtRU)wBNs82}J=bZ1d7>%%vpB@TT87%QR#(>Z$uqa}&fVHWn zR$jaf?l|*|U7^_f<}#5-Qn?K(Zvb2D>==|1ZX~r2pkCAYZb{n<><|^^gGd0B01T_T+y@R@jt2$JH^GbB0SQ9$ zWv(?i_CuJJ5$+ z8tQE#cHnHDe}f6cZ$_l|!P_RF#4;trodtohIJ>=SA;nJ?8%FsGe!#+IZlxzIF zr(17ya{ddGu(Kz6;kXdhRq5KjN@=^KQHS#Oz60q4oA;n_{1L*gTp zS<7XuQs?d$*+SZ);&TRl8_o#lS*WOfFk4^rKHfEIuqL$gs??6JE{SW~ajwWY?u?xk#p}&jYCB zx0HT_xH$05NGq8vCCflH^txM5MOr8)ivnYSCu=xl8t(^TagFJZZXNb z(~O?0I{D!k31|ko16|l5jjr3HktbeQ`tt05HUR6wsC6Brx`sV@u!(gC0AuzSOuD>8 zN#njIY#H-Be2ZgKyvpQCZW_ZI;;5fPDJl6*BIIE@MLMaTL_w9~O^6yuk z0a3{;<7AE3aG!KH45bGNLFC?UjSEQ)rcvJ$Y5y~BgVMh-65N~Oec86`r)7C`)LGhT zoZ%a?QOYUC&oyX$|7x6PW2)M0!Gk7sWJQx;=3DABj;TSvySSyt=~QI#eeOOYOVo-o zarl?I(f=xtU+NFhAv{P_+9QSNq5ybbr`{W0!y~+rr%s-sf_4zL4uml;qnG=_Vf^KT zI^=_LVM=gYrV5N#jA;c5Ea>ji8ypU6U=ttI*G=!JZgUE>9TDJ^z321rPTL6wXI(1Q zp?5sCIp5j?<+nOtqgUb)Z+R~?1qT<|_v2m4lkW|a;f6QpGB1Yv6J}&aIJ!E0a$L`4 zZP1H_3K^9dceFvj*{BP12(ljtg$gz>oRH$chPs;g8Ic?DfM;rOKJ%-8%B|d@zI~oG zmI;c-H~zf>JV1+irEdvK7dbg3GUbwp3)Ks)rXDL}Z)h``tj!|CceCuN7 zxd(afC6t4H^;7`KKq+#`$cmt4YviZK;g-c19p@1n&=dxEM~~`bb3O}#8i_Z(5SHHQ z0?Nq$$fW^LguWrCk--n(mZ^e)!m#<(w#y*zr!0{7fvRpT367EQeyRR>M4pex{G!KrI`|GM^~mPuu8Lc{@Q%)<56;%?gCGr_ zK#@gr0p;YI=0fY}!8+=5#?IP7WhX=8!3|A~iop+Kil}5uAG14gUv`$r6*l-Ar}Zt`sg z=rJ4hnnXW5Vc&03Qk*|rq$uN|CxJFBHB}~4+ZNjiufNoXyL-_D-Ms)ZxT;3>`6595 ztnX0za{b|r%+mG~b(Xozx~0k^F-ci=2%$`7*S0N=LOOYKEaAnMufJhlwhTFKH7zE7 zt4!un$Oi7bsI+^p7%lv2;M3c)sYvg1LxD`{oz;SB+ElFjT4$Hj5HBZvv)W}p*K=n) zu{!%*wGTrt7oqNQq?18d7A|CbpEFh|2>CHJ;>2I>6*Sn*Gp&3IK8llj@$hFv<;+9W zE`U>1tnef{a@$7q0}-EdIpkSz+}J5}-3c`JG>9DGn_c@{k3rWvf36hMgg#Mlt{58N z1wRys#(_3D2c@fvUP`G1fh{vGo+&T8^}uIb7{sgMY;!rOoCo%lf?~_BCnu1*%8(LC z;mluztivuo@C#y}XS>Qb8Xhe5OXW(@QT4rCy6BZBHz^m_oC@XlVfUUaYs(vXcWYs( zx5=w=VfmDzx*JO8^94n-^qpzB)eJFP=}<jcNFSat9iU#495I@GgRpK`-Kzrpq`jLeNq6 zjkRww8fkiYqXW?BvvlIYICzHKy}8)cT@{C`YTsOiRdujF8Yld|b!()1js1^rG4zNf zxVpN55;jL5LcUq*6BRtTB<3k9TjTa8&s1&Ql3Q~}o*aquMbL};l1lDj5AN~jlz|8x z`@@231;)<+MC;`)6^K<2#%pMUf)Cyb>(;n}Q54=yP>>yh)-?#-S#GfUZqG+~wz|j2 zzUC|lHB_*Nl{_=Po-g-1@geQ_}!cK>|n3ZxL8$D;}Y> zDe6{^p*23=I#$d34ve9WG}%SFpS0(1P@hcmN&P(7pZ<=9CN|k%w7CB)vmfIb*^ z3zri+^1-NMee)^bCaE@G^QIBM&7moALFNqU)3>S(@Oj?Hr;Dj;hwSXc74@Z#5h|0N zZ4QQTlOSK)dzBBe{T>?jv`D`lM2V_eOz&^n=md z@`h;*oSo!BZ){p0E|u#aRc<3^DkJ4Kd!=gQb1rA4rFORa$aZ62+Kx)?#8K! zlxN~OU?GP->jt|>K3=X{NGMa1d6Z`L!|*rWZw;mF6<_!4>#rf0MrxP~kuG!3PZ z^kM4znZd0wY2mISHyZOwa~_V4YWX{+Dm|YgoVDj2+udCD*!_2ZohAF0DS3n@`lvOQoCsUZ&tF4QzsUj}kk`weYBwev1AeWycM|5}(w4C)(6Lh2jl z2sE@}WI35DG&|%wP{@hPrI2-}`jN5eZSzD_@P>HEMskZ-FF%o}rCm2ofBt|>%M_+n zzb%3N5+iv$5eBwy@(7Eh{5X`5|JIZ2R7Joh@r}xJRtr=Y#-M2zq?#2{X#)KVIxuP9 z=uaM@C`D^&_o0C?!4!ih*O!SY*wr5R)&_o7&-=7gisLvDga`)f#tUH@CbdV3)BW#55UJa4y$O45b@6p+65x@2xt3#IZ%1STYVZG@|1VMOICj41J&vx>6_Jl z3x>1^sx8s*l@`FRCV5a!Jt77%6XpXrJ9w0=DZArVk3aD((Fe)@M;LsY!7P(Mi60I7 z27-fsui{um@3t!>RhL1dOUau-13fheK{S(%tZ+!fr4cvo?3}8pxjNUkqRm9`B4lgW zk2*JS$mC_7XJos?7Zt{5SKU`PO%HlO1X@{*R`;GQ+vZ)fZ_h)RT!^Y)Aq=kF6o<}~ zmwmzdd-|D=88j}Z7|}Cwcjqqbn|{*T8Smp~O3bUBHn>Fr4j2fs#*;rLpPuQqeQ0>Y zIp`uBYIq|!{XO((xD$?2u*R89U!g1-z$#=gJQ)RT*xKDY1XDzZAkt-oa3@ct0Iu;J z2B?|QiwG=?$F}f9&l6ySs1MStK}#RZaHf_XN{{>(MVAk4;{JEG%d+%FpQ-VgNqgX< z;_F-8?IRTI^FP>XuYVff@Ho0i+#4iK&-?F|shv00z$8JMN}vfa(QggzonZ+x(GWRv z8DF;+<~p7iI6D-?_%Z44;zJ8}<_0@uf(h}BU3D7`nOi_%Qwus#@(siWuAtckjVX1@ z4YM*8B{ES^&plh4zGmnFN2$%opA}s}xXyh@>Uv5f1s55;29e$0OnLK&DO}f9WazXT z6gRwKs=-3gQ0>LagwsFpb6oGk$>0EZsqrWLSvTv%ag!~31zIc6D!dz_J9b&<)k!ku z6XEgJ^Dd|#dOnKWLWe4sDIsWk zy97dmJuA_Kmw3wHV)7MXp0JJf@L_DzGs1rBdBKG|cM%pT5A%>Iot3%LSf+%f19mOW z5#DG$2VCQ=1~{|`jb#tlT3Dt~HrvI6)p2$PHkU;0f1h6u9v!pb+Bu!n06STtWy_!K z7<$vzr9gTcG`ZNmZ^TgZlM2gJvKA0Sqfc<& zNmFv(M^YAU>2gF&$QLbnFH8y9C@H=`C7R1+3Rt!hekTCi^7yTk9HtYFJTYWa!evqv zsQ8QTtDN}@E4EQU*^Przd~q%p4Vf-#_H)t2OWhRBgK>copMpdN4FtJGcd<&|fv5Bs>v9xw&ZW^ISmyRRZ#rdG)1LPoY>F83sz z`|m^Ag{vj=Cgk1_MSTeZV<&p^vu5Ivl%D>iU8e?_t>*Ej?#Fo#A;%}qy0y!EyRe~u zxD}A6lZ7dhHLw`=iA#vM5N>0#Of(*dXwQq}-_W*@=hO4!c0Xf`;yi?38s*%f%+q>$ z?zq!a-$VYkje5Qj3zcl8p3l4ZXO-i5StIqrcdDF!kpNT{hD+quHsb9b%6iL$wJXpv zxE@istd&*fXdpwpbR;d!VXu|!t4dv7KC|f;jn94Oo}JQrZ$CiVcLF85iL+tkFy28^ zMubvb7#vw*tzt2AU;Ew81U~#Ee?#7(?m`R3DBnfi$9_Q@^hOq`^F+i?AiWoVbYXVM z_H$YP&Vg@79d{o+x2>BJ2jeLS<2j%0hL(>V$z|DO&i8>cVM>2cant@8@h_RQD6oA4C!p$edZ%@JwzKK+*lv^10Rl{W+3G>)cX;Zw$nH1+4r=J4eO=|wU9LGhn9}myz`GGS4kvi)K zi26npLL1}r!UXajA$#Za(s<#R^Tmd3nS23?_c2osLmu$}vK64y#82wHXa7pwagCKdz+WWVv4^kMWm0FEhp85JX9@j( z_vu1Dd0zb=dxRS?ro2_ z`kC?!wC)XhbA}pMkl+gK^{8MATbHrXeXyU&5+}D!dTu}!&$?BA-RZc^L4d>ZvV64k zu&CX`+|S>Q9y@0p3A(k&O`p*;$}Ml4m|*z0P&w}#I$%!8GNsvrQfn>vdBL2POg!%wo{h^B$u*p7l%;tl6+@Efc|e}p<6pa zbnpT}fZFS|$jb&Pi!>m0U!!AiYurNXNR8x4 zYrbIjS>*oVJZd`~L2-WxC5dx}g$9sJgUDLwffK6fb-griQq>|<;sM33^S-@3_C?yg z)3B5q#E-ZoEM{Vz%xD@lW z9g)q`2R84L@jm`LEkN#}!QzKvw;#Kh0|Ab?=*!(a-l7v1m3iFM#iRh2>ILY+Ny_cA zxaD3zh&Bmc1`B_3J{){NXjtdUNF56kN!tbW7=x5ye1h6*mAMzHRr0nPF0>U~oa4j~ zA81W4Tqae@+_dX-ztb?=tnsRySoqNm`Zpyl@8((uKx_;o=2m zxSlQXZb)}aextTUxW8`#t!cdSRLjn(aly*Qg-#b?O)lA=uRV}$sDo84lRY->Ys3y_ zyx0@6oTfV-j5#SUgu-%N%^r58PG)oT5z5K{ZSithx<&i;^S<-o5l+Q{r?x6sN(mq3KQDgZIBi>>#*-A_QX(P6C`Z)CzT7q1s+} z{Dqx_sW(GA>qnExHlK57Dn_t{Me7K3;Tk&L_4yQac5CoHSK#Q}Xhgi;F%@S}H1%7s zm&3K(7umPC1iL(EEx0S%z3ChA5C=X+zQ}9s&Ye^bXeTezTPWP0d(bn6LtQfJt{Q|U z(-+tDeE4!^E*}UihD~ZafI^SU^b~!kP_0?0$NNBA&?~+fe)Um2-7e!(uArC=na>2R;l!22se)abci=f-IDfqmr_WqRdmhhb&wgw#_$jW~AO% zz0s5!cDfMqg+g|8WQ;R*l#uf}s@#^CjP6XdyssAl{eC99?9j-*4+`^tcV=QEmT2Dugd%t__TU1FI14``M}?r77QPemV_ zIswWPC;J0NPvvngO_y+$Npva?8-wsbjLNzP1abi9`zEML&IU7Yc}n#!2H3cg~DKvHRW}`EpnI-AD<#yhSi2->T8K;zeRoGN^g?rc~K? zF9>*@84k~j$TLC5T`1i-ow`TmGA!0~_Z~8ahSxDa9D80oaJWg+&7wgF!6k>vdvjcc z>8tXeMoGF=PY*ne--w}xgY;?3iS0gfXUVyCo%w%IkQ8q)DN{t?Q%Qx1xTc3@QdJ|u~U==gw^7#hgZT~SA*BO6EW#!z8 zB0|Dt;V29VkRfd4c`p27?SWTsnH?{n*#b1)#;437C}n)z*>us{VpA7zRUW%8Vcs2j z;07#uoh%pM>|L)w;!*Vmv0+uHJH_0(Q_BtzctSJl_NvXQTiiR|(4orN4kcQfZEE3; zC>tR^_;><#OZUdMJIS}Qki_DYb?LrAL?58vd>X1#nPDeKEx~?*Pm?gJ; z87uE&=8BUyT#ugj4e0tR6@z_v&!qwTfI*d4sW4o62nC$S@<8{YBW=8&%r#)+Ry_AB z1S+6$B3~?X#9=SII4VI0R$mCX+2~_Bo0+lsaYtl`jtHx8c@ARyMxhOqh`7YdWeTqS zegWBiI-qcylThemF1#rf*MFQ;O5!%`kC6rRyj619L7zwX&o?aS2P@TPmHNYVtaVZ^ zl!FU_=$ys}w2;$LV5n4dQV*1Jl^VVE*mB_wlWn;|EuiM|upz_2t59cHsWW_oF=*?) z+1j4|l)NOB4dr>{L)|fU&ycotmLxLy6KD|MZ)rBM%Z&Z911H_VN>ItJa&24$0GaZB zId`S=98N!QH?5}7R90WDokGBeA-i1GGHM7jsF*al zPkH4Xq)PXK0>>I%xooKmOOY6;y470?^jsDnY468DD0Lk0$GWPJuj4}2na7RvblsUg zZ_(|%ck?YJ?CqZmlb#Qxy;Tz!S#@E~57jsFVA^?TZh`wcsOOz;I@J07c)#KGu`{|b zv$oCk!IFHRD3&2V`-|Or4SD^3cH>dUZ!cqh`Dq2;G4{g+C2hl}GKXF%f*gR7=%!k! z7RK0n=CTJ^Jx0IWM{yB&l-xDC^UeL1DhsEn@_AW-s_x-XRhjpZj3(kby1N2KWw(zX z@IhNHQe2pO4f_oI8NI7KGv~&C`N4H#m#&d+qaU^@OA~bRUF)W*i=?6#S>eDHyzW4{ zH^;1;e$rhn9OED{4g2I$7e>fn-w>e}5ke^rX;|iV%O5XA{~2b>bbre4b!&&l=jGv& zX&+amxPto=F3$EN)A`K!zHL5^nlAT@T7VR?&l(rml7fydoP3=u%SBFJ_?pn3$rCf> zjhbC4BbX891D|`0c7JyzC(yRuMT&N1T%LgzrDrBX-Ap4mT?|EoLU+4FtTZpnhoB}V z5%~c#V{VtbOUcS%Ir6aY3$Oi&Nc75`jT(h(b%q>+{|zUyyd`CJ{abpe=VRWHr^EC*^YkH9_&klh#Ib z?k#;@G%49WAXIlWP>$5%I!;L;70t)Qw?A~Ym52!2d3m!Dp zSEvtr0R*BSu9Ohtd0A4W&rVwKoSuO}*~*&B3>yyh^QxSG(`V@N8s+h&oB6y@$r7~| zntc}#9tIKG?l%(}wg8qci|1ySKc+8;@x%`Ne5o^OH7>f;4v;b+Z}^0ZLq z2Wl&DQ&bW1^RmcAQXl4~kUf?U?DR8xLcUkX zj>D3CCA0_b_XETSxdAUO#NU<$r@f<@cuux&WoV^MG?eG#ImpMek#gan{ZGkBi#&w< zs#DLZ3m+_Que29}6JL5^bA`%wbp}?$Ghf#(O1bmR42FluypDDwS6$dlIIg~pKL8}^ z>F2D^!4@l7Xq8V=^lp5FUq^CWJo7Iq4+1HUBq#a{yY3`E6OTUd-MVmbhdywNTjp?y zwXO>Xsp`tmeGCs}&@Po_>1y(*Oxkr{ZFiZ|u+mR>V*SoB{}~HG$+dpJ+t@{!u3Kx=yM9=iHQsp{ zJXZn;KdgC{oc+_d&rFT^Jv3byRwXxnXnZem%`qZRXd71effQ+Hqu)5C%#M()g`2f2 zIxmEmabr&Zv%AZ8Shb3O69KnDCq(@K(pMy{g&BTbJ`6bbi$={A0Q6R zr;AtJQB-CB2@K$#%nvrz8urnub3iM)TJvov{<^{h&OhHM%IiWw6+N%*kskUVoX^Te z8y3Rz8xnfk_9nN2O~1zh7IbeDz0JeSbkvSMN;Zx)>+s;CKQa*{OD1s|&;Yx^{%Q$UJ%v z#buj``x0+T*pA9N{S2P|aHa5Djp-i7dZ~G50m0~>_l#RrY3DKz#Iy64>}=@(N4emfPhP0Q5w8FFKyM@|(4d3H!9a@mP^8 zg2@NnI4?IWO4g>{SH*8pEtGN^mMU3=8F^siXSujHQ67XGq5LDOI3kSdB$P|!gJ6P7 z@UBqbi4r9*b~@i$3)Mypf+zL$=pHW~gvvLplQEJhHz+T+Ij~biD2RLWo6DNeW8$t% zZz<^xcgyhL8341fb9?%cUB z*-GA7W#4A~B5Pnq=9E#r&ya0aeER!WWPpdE-+e#|)%(%|L&i8mwJ!p2Kub_p57b0w zMV6yn1wu-Bc3XVIi|qn17rgT(JJ@s8&r-|}s>}E4-u-DE!L+7=G?i4oW3@S0K> z(NON0xd(6PtjqP_We-$&>z2m7zZ&l%^dmNvqL5P3F(&%y)D|vXE6}As?Vg|hsPmut zF{E$Aws%mBQ+souOw)b9E#wDkde0(!tWg(P2?0cw32ThKK%*|^AQ>2w*Gm}3zHb$| z+#X?o<2`HJ9_TF5{+Kzh1w7o9xQ8@B^K=YOly@9V;{Nk&294l|4Y3O7NKZAP9 z(C&n=;uN)j24$&vj#%8u+Ky?pd#Y zP}hEw>9J8z(5)9%0pkah*}_{@PHK?vsDv`ukJ_Dgx>hpz&|!N#K6!=)AceHSp2L-% zUbdG)&>*`Wl~{lp7bXQ)NQyjGIAu@SNd^6`JPGx%PS*X}Kfp-O!;qoug=*FtoA0Ye5K@v)4N^7OjDrle`P-_S*Gi)`Fk2uDtA|iN37wRq4@$%Nvo5E+(Y5d zmQTw6s6*K@K{+hczOE6>7!NvfjY^oSF!)X@c@%7uk68n?`Q=XktSW=+Q<_48qwIGfVwmm7Q;Nnie z2&@Qdlh^*xJei0svd6I3@{$#A$rlA?7*sBV|Gcm)f=SqT?h02Pm`|4&FHl$u72E7) zK*dmMFRR;n`$$fgsX?@bcBw$8pr~=V(2NXd#q!(Y)lJXc@CtS30-=d?3{jwEO&TFUN*OgBm1(%|CF-C;TE0l6{dVNbvasE=FMOWrZK$e zol>K{@-pE<_klM#kWDT)s8r^_`UO{54o*(Yh1WIL28#mB-jh8RGVH=jS$Om18g62) zz(PJ(7LIr)#)T)Y4}lpz{e2dYHOyJv|4NzK#X9#52&zxl?$xCm^?|9Cy3eCUA^x$2 z2+;GBaLPLO>yfFpo4ouZd`P{pvDUx!7G}_=EHVZz0=sUe@vcy$67hc7-z`(E3*ooN zC3XUfcZV^l?U|7N2qChFFmo)g;engn2mQy@86Mv`e&@Y&vdB!*KFIn%?%>A^@OCX! z#0y+~yQ`)`hJvZSaznX9S6X~a5Tnqytd^EO+{xwxmAiXGS$5V|?}SI`3MXY#$}oQ`rhUW@NH{#m5jF5g&9#@7x_z9anE^`0WFj%pSG!1yP4yj4uu`{ z9wcHmbD703ki5%Cw(Y!#IC%N2-Z+#(o1VK0K^IR4U8J}N^&qgK@b=lJPj#bh{Dz>#R2{q7=y!K#H5Ar@l zX%Ar&;)Rq0*0=92j2qB&4~BbLVbX}2_cro@DIPs!g6?t|ypqQa&U4=OKO@v1G_lO! zz#6yZn#39xJwpo%ABm~06i;c4_yg^UZ71(Ax!9M#2egWsao>LbVKzc3Y}+O-PW?qQz%SXj~rF7iZ1zpwkj1gH!aAoF-f zpdq<;P;|ei*|vx6IF9a%Y*G?dy5I6X{!v`frj6L#;Y$cQRh&e@WWbgAjC zDz>d>L1vIey(9YyaR(^#P+b99YzMlKV}pX{DEIR(W~=)lH#sttx!aUD%G+N4v?|KO z9auZOumO2{Z1UTNnTGU(P>0&)V8Z=b=+v`ty605o{9ssh7oOVr*a(>46y`Q12hFj| zKO2R5VHQ3$tLB|5al~tbZ+ zKZsm!4x`QtE2a}~n!T7P=eUYH%qoVoEH-KXYxBYdrSF=NETi37e%MnzkWXB`lWaAA z9%<1@3QEg3%svL4@dWBQWFgo_mB!L1Dh6qEE z-O8q5hADE<*+B#S%z=##L5s`= z2SZ@|W+8zo&UvtT*V|l16GtNLRAc7t^akoy!CVC3^kW8B4YFicJ9 z0i*1mVYxdI20-O;tzcbS&-M_T{X;@|wg8nQ{0Q*F?#6 zpWp*R;PFPvMnd^c7V!yjhsOHi%#KGfVmB7HUYb)z-D5d}PI0c;BajVb)yDY-N~pXE zSQb6}D$P)`V5nlHxl(Bqoy1l4LvV|IkiA@EGT61}bmQmQLd5KUvliN^KWp?Q8uqAt zp7F()M($fr&dqgB|Mn1(R~eVXuh&4Cx%)r~@X!b-6)wA?)MdaZy1*(<#knt#VR}*# zd0c!b$H>C)%fi_U1+QIWy{@Kh-<2%L*ffI2*W0iZauHEX+qWndq~&LF$ID3|C2$2U z`+-3Jv)G@NHZR9Voq2__gkA&|%Q(aW$RLL{@9rv(q%epm^7~!_F9APnI9oK zNB=1k@E4xt${#m=WY92lvDb}B$%_l4p))@xcz8x%zT5y}gVW~e4BS)I5L0$sdN?fJ z^yv*#fo=lHRk5t`f?i4rjluN;#SPvR4uZJ0D#0jlJ1{lJeN0|CCmFTr32fb^h#_nMWmgfXX}CgVE;Lo+=qnr|rkKa~Mo z8s?fY*5RHn*NKm|;bEG8&bRpa*e2te1C0&i;^n^t>e~z!o<8zE#q~Q^qysQqM`jqyEmwz^9>zu`2u!MCet+h^tkE2HiI&vQUgk1dI+XT zeDsab%dKHIpfDgzoVL4}?!d|XN?Ssp=Kmp-mGZo$W%{s{0_qu`p`<++Ja1%8P~n(& z%yiv=oA@^|B_VWYnhb)mN86$UD2pHES+}OaIN?jrIjImPjU}VBodCZbRCpN$S}ayb z#I=O?5Mfza3&q*aIV$Jm*CcNR*o68x5I1~Q`ZCBq4dGFd7-{>RW?IjC&EH~Y{loPg zX@MyB&f1G2cct>)miub6WfEQ14Nc4U$mzDygTCKGCyoQUZ_Dn-`KXhEptxTU-EjIw z!b8uY?q4#8IlUOhDNy}?IlvaUX9cb)p+4jF+Y$OVWPsrC7JPqDDD26$YkiG+*J`ZCoh z!$@8^)SMScoML3|g`B;V^In6HJ(n`8w;0Wh$8&zLy&zm~hQsIs)05*O6}nnHFf@xb zFx*(%8{Wj3U9zxJmL6Z&SUV4Tg(wSSYn#X?LEx{CYp&Z#7YmW-wgPg>7DVzd8W4s2 z_aBgKg}Tn-CA#`K4$6QAMlUmxti9CyNbWm}OpJ5M*{AI`CrD7#dr9XV}5w2y~ zeskSW#yC+A>|C2P+sXcViV%4|ti{qU(n~GQOnoqk zj+f2{I~PtdF}wnM!_A7v6ZxwQp}ET>r{>qi8Cj{Zxf6-vetUf4s1j zxB}MLf^$Ee*R3z^tVzC8$;?vm9;lsvb7Dk3@ENKX5mw1Rs@#Ri8zO|)ODZ{4T-lYW z6KAJ2K9kSrU?eX@X{Y9k$Gbky#0{X5dsH%sWnMn|cIajU-q`Q;a-lBN=5M;9`pp`k z`i#*(5K6-&dq0No*C73`Gr$Dwm%~nCfw9xC7A++5aA*27?2A>!u6LN>%a0zM?|p|q z@9J01T%BdFm0mGiMlNjr!YtL-RTri4LOmMa6E<$eyC~~5-X(dBmt9d8ejA&;-|%`4 z?g5eN@?Br{M`#jyyzfd7TLvbjYuu*=?kn(@NtW+-R`^!E@pTb8-S|G6Wo>(&f@LMc zd}$mc4?=R^H7C0Df?;DU--{Imy7e9_Xf0MyL6Na9lyks&i_en2&-2*bqN8Au_gH}e z=?@0Dz(|@o%|BDQ5rxb*Wq8IvwfS`QlVzuzmP$_?)Ht3O8QsFFwO8Aars9Yn}trn*v$Pbt1&aSC^%!tk+u+DlrJo*dO>>6G~!k>{Vl)a3H%R9et(*)M+1MV<% zy0k00p<{WH$q$nmzuPdODvrF+rBM?csmzU30!Lf$6YrVv840cS)f_?hJ{$e(j0mR9?+jg>P>$*n9!$2 zdPcu#Yc7xMp02lNK&pD7`pAFKKnN``PXnacRhCh_zKEo#IKC^#c9*B7C}sBYP{2?y{(GQ zkr;DJKP$ceY23Wo6J4g_MR?_mYLJhUGyeshQC#>XL-%G#ZN(f4iR}!AiWx!WGjfbF z5;|?f(;OYG3JW}5j$r9nP1ff(WguVN1YydEclGOm^H$IZO41$sWx{}qy0P>Kl9Dku z{u}}Xp81QDJ08zvL}fbN9BTn*Qp7avhhN*tV|XbUND0&Bt2_1ZS6DvhM~!vD_auP7 z-GfE0#HTBGfyPO|$J_;7+P6zuY%5p+@v;=k0lnWxjx?=$SX_tp9iD`QVdV3+D|9iO zDr(M$=MbpM6~Z!91N61(r%d55`W({2UF20eC><)r5Qu*h4VFJfo_Qe*-A5{hDdu22 zT5}-5KFJLHm7XH_kCLA?7O5{b*A)vU8+8lqSSHK(9m)}ps%9a4r{snx8vrKj5U52L#U z2ua7a*J}O%jP47|*@bnYU;g-}mA-Nf-p(jfizA{JLChSiJoy#k&)O7y&r-Q&7yeR( z8hu;e|5s{NfvzLBUF6GQlr+{hB+jhyOtD|69#M2$QHcxxRohvQ`xrK~rg)X?UsIA^ zo*(bvtQ(%7CCrz46QgEf(>w`efC+6P*Y{DYl5kByF6|xN9iHb#Z%Yw_7O#t?hEk#Q z*v7nPcbI~^%k><^0p~tF*|eq`%io(L%3{Fa>KOPy8{kT;>!zav+J(u;N}R`|NlPtn zzIS>4lNkqF<{hpyK;^2{n9^bJ?^zW}s14Ct+(@#Bj`T9Z7ty?qhhLpn9V+MP%&x~p z?2J6*)HU>v@wKoccy#IIBV0L$x8DXhI;B5@JU3;po`-)vDDHF$d}Tzx)K=~qLDbC9Or~ZzyDjMZjdl=M1(k#&UZFLAC6N8rL<#a zbK;Yoxj092p*y}j<%^Tbeti5rp71g2{TsOrug*G*Kg|RbUSl?wmna4q{_Y^#{xA7J z;T;g!lm!}B{!G%nyDsEcV`XHX43^|M8s-`NU4?}Ys{*N=HTgt_;5Dg%*rPbizYRS7c-=8`%<3LCHWuD&D!S&E z*|}id2{{{mO7@8UyG}c?86qrPiSC=0cOK%EEs5S?>-;H6P}vQ;-GUNQad~Es*mmuC z5B+nFn3!|epJr^Zu=RCgZk#xN z@OSaV%3xhOqRnO@cz4mySO3;Z_)7HiU{MM=@|zXcdNjT4KFB7zBjllH&im7LYGb1X z0qjH@?&N`~gmd_)uWzHGiM`VPVaR6}^~>nMt;^>v{j8#RbI7i-U6;MYL;3Bj%ynL~ zFCAdcF%Q;odMA1p0mhS%-cYuykQCe_)aCXVMeJTPBQ0t>jf+eQevW8mL%w)+Rt9ZK zXQujsLy|}Yz@j7t?={a0g+2Ia#r)y^yp{T9-G6FN*%60)OVE!cu`bv)1Q`quvx|gZ z6ZTorOULBaBn}=o@sIs8x=O$qdTXz&lAWCsp^&puEt@8q*52KPZmM7W-zOb_F`?ZNy?H>#&1~hzz*L5t?T);1iG!3>-n=VujTo>F_SstCbP$KWVXj-So^~S zE}x&Y$90GNs$p2+IX1qBMRQDgTef%7nk&M%ca=g|q)rnfqD)l3cH)uHyNC%d<{f3j zTQ1Tss5$9^Ee_x{?!-xhQfOZRHiJ6e*eH6z{hR*vllKD`uh z6V$@F{+Hv%3t1|*q@i6)21`BH=xQez-(my9q|tOZ>>cn_v7HJlN6!{hd);-}w@pQ4 zAmZ&`qSCT@{rQGmk=k|hre9J-Yn5NSMm)*SPhd7ch|wRy0SrSfJ39tDNbA-8boS%R zxS;fM@Qb!4$Xs(PW`Pr~Ny8~LUIq86QUW^(W~0-fun44Jm50S)hn4;OWITPz9gI!= z&~bvb3cs@eV1MR=d(8XtayjdqPe2gq$>~daUSUy-`v8Yba(F=hhM!V6T{)_KdxOqA zOci9l!0v25JSahJy|NCA(sHz4c>2a^_YidVT*vcG_JAh)_mnH6i2P4Nav>8^H+O9` zW2dUda^&N^&bfQ`Gj)Q@y<>$PPxd(!`xHORx6_ky3P9RSxFVynpJ*0D(!pZ;)KeCl18Th(vkafoHgvOm-g>NRfHLO8Lt7uJ6`fQSB6KcS%wCpeEZz2P%RAJ(H&|}SfF4W ztBB}<;peH$tz_JG$p_1RQl}gK6&ajp6%v@%+B^Zga(!FVT)&0>@ACrR)f+jb7`f5a>cstw zb5-=cJ`xSDTF;!W(W>|!o_D!xdXrQo5W433stJ)U&|EveG3-~c)JgbOg0dHh426$U zw@46a><{~sWMz8Ic}1`QnbnZqu%sP{X@GMQ@;tCT68L#!XUZM-RMWI3_$T6deETR} z@+HHLW}|jUKhipG8)qP%YCRhpN}_|&ui+6UxFg)yDuP!A)x6c00IdEViA1_8A2@|s z)x2-61W}}PYLXx4phw~s7vrjVLjL%})|*MWQopcZ6>$Ekf}%l6q*Kic#iIG=IY$ch z-{)CbB85acHK)VGI+TyyVstTi2!+60Y7Ck-L_r9eZdELQEV$wimMsw`FPQc(v?>+~ zjZgxCe>_AYjHEn;0@O!~cw;)mfBJo36q#@|{=q~FlxqKCNqHCr%71t(x?fWNVpJMTx?f@+I4_qP>tD?EuL4ZE z?rXV3)#Q9GGE^EwI(ad9D24bWC8-aAr5ca&-;`!{csxuZbq+P*KfM1xSW=$yUu^zw zw-3HN<=1~yVJT4BZV4{GE35Q4Vs* zYm&~V2Hv)A)k_(*q2?5nJDMQB;ejsi8vLX|LdR1RKWsI+CB|w7!ZroCIw<>)6QoC6 zHl{y|jH?QhNm#F%H4=r{baK0 zn???R5Dhck(#0-`ZzVotx4fjXb9rCxv30FIJ-w#>th2ILcOtlB>ii}@FXjSvjAI#Y>4lrSz#>g z>`yExp`Ke-ET#WhMyw~1_^{5;|GcX>PS#*gIj;O)dF`C_Ym z^FiFj4Nz`+EoZ|^{@Anx`g0N0@I~eT0W{+k+L#ew3#gU{se2C~rHf}-z9@rcyx|P9 z#1$*tWdCSbu|cRD zGsww55C=LCE=&*tco3QYUo;z4=fSssG$7-1{-dJ{xXB59G`7etfDb|XUmBeuCu{!e zMgJT9!-3eFJb2F$R90bNmKm`LJaP2=dnueALB7$dCpE~bKKB#@XNUqRsB&TaU~;K6 zz&a!I6mx-;&%6c$ZQm2PzKGbAGhVvLDMC3|D=g6!_6l0B3t7nzIsqp$uaW=tni<+D zUm7y6i19Rw5#y-SuRLMq9KJI3eI0v$zkU&!3zF@zVI|B(?YI-oUKpH1X#RT@Ht5YT z*^i#nO9QSfCw-j3JxOy2BvnDRO*fX63m58RzRqoJ;1O=*VKyH8doTlyX6(}~(4+Zn z<%-$(8c7SU+5$ixdEwY9Y&_cJ&&;S}&@~V8DeoI(rdUsQfYjHnHi2Bh6CjXGwwG+& z)9-4;Jwd@eSPx3}VY86!KwV06$PIReZB;$JfCGCV8Oi?nhk|5eTx|271Nl^2{)Z@I zT&(ps08DI7MSBw0KJblh3ygw)9XMHo(Y+B25Et$TxqI<)*zTVk=J=As4$ z+_yTgh5x<}?>hu+K)ugFU8d;xA-~`~Fkk3zHg)|&cEbApKM55imxXc|zt%|l9it<@ zC!-;@)pJ7^Pj_S}Z7KqcYYjG949+!CL#3ChlhgZy2W_rxPv#jT2uIR*KiI~NzLXq` zpN1Re>rRi>q@>1R#PM%y_IR^?nm^xtL!AnOV<<+NuB&~Kcisa~W{@9SP+!W@9*w|1 z1SM~(Z2!J91ijfI9B<4e?OI|y2^g2HAJ@mi`&@F8BafP-5OwcgJg%P=5arTt@TA&d-As`kGhdx@3P-pe=D$1p4(G7h5tO^h;~IXNURbGa*XEu zwa;fgggcy(>*$*NcV|kJ%(J?z4!pCJkvTT;X!SGn&U1cig3i@l`d!%6!LPo0bG84z z0jy3aYCmKc1vS$8ha<-IrF`)EVvxKk_8K`5uh+_-IsqaKW-%qe0a&YKv0ivBmy9cy0XwR%X9VL0Fo57YvzIw^Mfba?+24umcJF;rB& zGTZ?l>12ihNH_s3`y!2g%0^h6iP5Sf(L3eCkTFxhlY}!d$MC3mZwlx9YxmBfPmWsE zUcK)w*1AA&UH&5VqgyO`5T3A|KD0LY*%uRw)?+j6eb-<)hVbchEwc>@Eo6p%1MzH< zc5Q%)LU7Rjc#99oM*wGpYJeD-^oDsJ*SgFtXW~Z$1x4-SG!;ai-^}GH!j28e#`Ht7 zIy1C;>u7``$oSC>s<*JrGEjP-R zAuZoT1dovw;d{&3z(~F2tW2aRbvcj}chc!*{Hx z2tQNKHik4)&dN-xRM*4Bs8rWc@AAQ1m$OYgT}^ijF@o!Q+G6+rgM^M572!wAtHDU4 z96`~3B~W1UH}igCPU z*Vbmd<)FN5yye99c)a#vsP-znQgLnh(Bd-A;x^9WG0ych&H^&t0yf?PG2Rt5-U2he znx#&0>{x($RDya`fqL{S^|2oHs44ZRHT9@daFMlI{c{sr9zMnX*S{?U!?i@iwIsu} zWWy~K!?l#dwbaA4pXHYulM0S3bmJ`y;$2PREi7tZg=m}@8e8;?U}aw~aPeI?z0e)! zV{@;^A4-LuZM$n98sH%}wJRo>Bh=73$zJ(*JA-EGQEHwRe2p`VTAPV5P6`J_FRCKe zWNp7*mKm(}p66Z>G{&KJy3r>YwT@2?Nur3qEcbR^+vF#!{~)I6EHkBo?j5NTR)Xjx zL#PcuEA~4L+kd<*Lymhms-bugHoJy4IyCB>XYoBPx22BfdR`#o@i$Tr8E=sMa#GgA2L3N<{#P5z>IZHH9_{WD4d3$smX+lPFK}%%)pB6_CioDzRJ+WFFpo=j<0b{81@hC@G+YUU1r4`Dy3=L zlu!5NKtyx(Uq-D)P@QB+N=-Vez};GP3}q)~^DXiu}(D z2(C>xU+DQ;I_~&<(Px7t8T+aXbaofW`slFP4{xd>ih(j1NMyg8ZfntA4$qAsIw5=^ zU!WdfFOWC!yIB2=Al@CI4l*ob<|-z;ua0xCy;6;KUec1m9=@#CHkn2%@w)RwDGU}R zUwib7OEiROf@z(|x?&D<>gVOl|YXj@~h8dF*0Vd5b+zFea z-mk7YrD%#{CZ{g{$T`On{>seg4@tUHu-EJ+z@j-=aMV0YvY=d4a8$36nVdGWd{{rd z@YPk%TG^&(=0lc_D~@@bx)^@QIgb*4hu`fQnJJ~$U!4sF4U$Vozq;yLv)B~9R7P?9AGXO z+uszbI;W4x6|3H#e?9-#Q2lq4e~EVt2tLM`^j1UJYizp(R*p(|>;&vJ#D`tp22P=y zdzZnnRHHKp?<@U)O;jV5dq-*PKB3dc3Qb?+X14)vt%(%DHyPh&EbrB0-p*CsIxo59 z1*C3Lw5*$1^WB0oGV;d;ub1iKKXvaEjF~BwCn!R6UvU-8wW%( z&oOrvSRf;wpR(TO&ft0$)?o)VI@LkN5vsy2_lrkH&-9AEr#&?-O%69zuK;#m<~C4K z5VP+mZBcj`>f3a=56#aDl{~8VI8|>A8zlo(AV&_ciNMp()@2x@XV=LOuZyRJN;S2g zcbf0ZRS7!_>(D=m&JC(l@v9s<$HAt&wU?LI3p*<$e!|zCzORq}m#=URgK^pB3=aqvt)Y=UGAcY%J+St@;X~f^O$D+?ck%jvZz2J3|-Rv|+ZuX4GPV z;{fp-rbuB69(xv{!|=CPQ&Kw!%+*o4Sx#R1O`ht702xZ%3F0XBxf`N>zwnQKkSZ4 z<|6&h_v`hWVhrzM=s#i6lfh{+it1VHIh^8qmrBCM>he_MVDN&Akga#I(Z&Ol`&N*e z!Nl3Pgu6jEF-a2|E1us&l6(JLHaDxci^Q!QZ?{cjmg*-mTJY+?OjY-I5ZyyHcD&^& z7eu-Dr+78t4#OBW{wVYq5$@OTsuRo0-ULE6F$vv8GIe$8BKzZzt}vV@43FFH1QTs} zWD%A^HxbM6H9+7Pc{G83X5pYxdHI9Q$de?!_tPy{@~sbaZPWMqeL98pGpEg6m`^mj za%}GAU&cB2)cK>=!v&|_X4Ww?nU?itjC>o`U%72A>B{F6T1?w`2u$2R4!|aHtFn6X!r9_(W*M;c^?sbMg3*4 zeGol#Kgw2tqU<%+Qdu2l2k@bH6CFJk=P`Xd26H`L|Bj#I=hwSI3-d<*mi^SLEPfWX zzVU;YQU2_f+zTm)>c_SyHv+4TY=9(q@#mA2NGC*6ILrI4qecPtx8wmkuQ4Cu}^yR5-K^y@}NI>Y7#rHH6Q*FN;Gxj!GRJRZ|7E#l3ctY6|ZR&0gsN%e- ztT5_eE%|c zk6-@*uA5l?pv9R-E#1kBd;LDhO-1I52%tI!2Is)# zIcYS%^6-f2xYJx-F!1xV8e}1FB_8_z8=Wa^oU&C4Ny_#I~DH{B>_iD9w z@{Tmtk+?Vfi;W@D8(&nLPY*}AnQZ3%*tH}1BPM4SBs`Hi)1XHp-_>%@F(l^T zPVXIsADN1uy`peR@%TP!<#WcwGlf9D0Qriy#a(pNCexe$G;Nb8Y8{Qv3B8o3w_$n$ znS1TijHD&P2&+{P1%PQpU1nXGh)T*LHNek81PwY=nu2_yb1%WtVc$Cg;C6znenB)g ze(gxw4k?lLGY%ngA0>%D%WXdulZ*3UXf+PUCtaTNoAOv4Uec&*u|yOv*qo_ujshQdYNUZ}6IS{w3d&nJ$Ol z81=-eZs?`Exw4|pRAq-g!LLjA`pP=lH!gIH#`vhg{4mYrB3JXX-Lsfc+d9-#?7L}6 z-P(mI!s{F2m=!P?VB>jOgpK2Q-Ev**PB&%W*;XofkTEy-uSe#`fhh@|L|nl{P9fv+@owH}ru>Y3)O~j*vip?>SRv%M z6E-3@gE&#r`NYU}*nI-dM1acLE&sCh2mHhi2`Oh7j6TMw($`$AD%6w)LV+yetV}}Ens7Uf5%6ic4f&6r z=d_`n?*<%$kIiG???R_5T|*iBW+2e0cd6o@0C^bb&sdQoVBT3ond*9>kUwi2t+4<% z-_P!Q--)6AGMncD0Ndqol|MbF&UxQ9kR?-)HrmzgEQ4>pzqK+zr4wjfUNo?{DQ>QiumS?%6(-R`j>tn$ z_R54Mewc#v+qwT^0x}1Qk=J4!3X$}mv}Cz6bm{*^0}2I(0D%p|n4${#?C8UV@DTOL8GcOX*UrklAe?cas9=8IwOynj3aygXeWm5yDx-k_w;Y$ zin<+4yeZ~ICniA3J3i0zzff;`@LUkoVicq z*`M=cMmnOCnWML5a1dhXWLiiiVGTa>GIE+Z1$Z*?6kR*wB~ghbk>H`dQ0u*DtAIAGsOk0DZ@AA zh3&UbuXgUOX=QrJyT|W(Z-wadPHOQn7CPDs)n&^ZEju%r^>N=%uqWjmzUULbKzz{n zxJtrM=$UmBaUlqo>^AHPd&FpFaGCyRookwO{@|ZvvKc-1xT>WBr4;GjVDUHYo0E5O z2-C9&)38ePHQS1~?{xvzjM=0iVBG*fA~4o{#5E4* z>8LKhB5cbS9fJT}_bcfW4*0Rp{*hY+?ZG75?GND%SXE+1WXbG)$OK-f1*Wk2 z`LNI1qs3YkoNUL-n7&4={X3n3NB}^Wlgkw93zB&~Q(ilNsvSsoR=C$WT*>Vxs(#+X zknCP<2bX0np#1EoV!00th4(Zrc01siV%F^`hH1u%!ADQm!gS<-(PDG2n7pz0vh{bg zP^NGV&EA}iKiyOH$5Q?aY8v}YNLu%An-G(6PFAXi^`-2&-_86h8sPT0u@^Wdp1=mx z&2rOR9w_d?>NPP=TLq^&rie#i$0TGOI9ffzHF zn~C?JNEIITwgh|iknoQUVy8&&>>Z|jF0QS(L@aAJ87o#5rwIM>z}i(c1KtgcEVX;? zx(%cdM4yLxo>m-Fn!|*AMyPb_n+pc&T{#$*{~znd|6UArLQTea48A!o{&J{+H{y`3 zd4`ODePzWv>jcl7y!z8aUE(8m8!1vF5+P;8IUt1TZ8i6Yt4>LEvKaqGmOC4ZGOuMRi&zVe%mL{DZ3B1 zODA5n<-=FMgP~xunDlw*md1N58NZ&$*HFSrN3<8;(jZS-A`J`6xRJ&+ zb+KY8VzzT@Z4QW^ez?%j`cm46&HD>M8cXOBPmQ^0aZsuQpI3xARVnGqRbtL~%wyDS zx~n3NtR=q2UKmIgprYCqx8_Po&WBzKOf*zhBG(1*?Yt43!t`i>DA0f8ofON&BPLWZ z(sF6yW#_uRDu!rPra?Qt*Y6`>Ois+D^SXY(2q^`GB(F9KeiQbjA8D40mvL46$h$6lo1_%1FZTjLtZDBCQ>*Z57WfzCT{k zn2&AJFvEl!{q9x|lh*Ekg|Ijo)f-9&dqtTe(RFliIK}0S2+V$IYow!4nb-W(MY1ofKkeM1c9kC5ft0&Ib62uR`}xH2xO*rQ6DA`p%Vpv9LZP zYDb=$Xx>Rl8tN5bRmZE>XuctM)^M!;OxuX*^%d1tv|-YZ;06G~fcU+13=o&p>jv3- zIo=5OD^KxeIPW;}iU$JVgis*mrNz0%3oFw`zpG4+bg=5=$!#yI2(nC(@7If*SsKn~ z(?F|1s6jJS(CIy5xh|l_1lr%d=7|ccY`#fRDbG6Pg+uJua;pXeSLQdxN3=nD1iLY zIGjKfy-E#72uzXcUk24+lXm@rp)$dvdM1pWZNN&u>wbV2oOY!YPeM{E162bH)FYi{ z4yl*vr1ig4d(|g zqT^2CFyvw1eC6Sra+gsoSIcy0f;kVQ5PytSW1~>%8$~F#sv0|Qww-lPgQekQ5Lek# zl0inYrpH8ND1oG5v@U^Ay{yfJJ*dm8i!fZCUk!383#dCJU?-(&oDUb~CbAsdW2!j9 zi~%RlLUY_MKg-rxHm12rRQEnMAY&!7E=O9i#^b6y*n{G6$M0T-v?z#oO*bMfn3Egs z*A~(2qm5u0j+$i0$hb$B7dVg~zd8=(ISqbl~=9?s63sr}KSNcYrI%2Zg}W&CavrBNWLVtg^q zOI%vKA;M00TqlOJ@eo+Smrl5fHBZ24m=}}D+`wkiUp~y2Z-w5Mwb;hmn6;wT@b(?E zFVF&MuYbXY&VXhH8D7aO|BeCP}fRSP2%Gp2m=cXJ9S#AP#qY76(Qh{3Q1`TPoL8m zm;?haU12QI9UO=Vi(RS!Mq&dQ6CnY1hliknU{dnUqdS))Z+|3M2ApD zSQqu?X`EOENLg*AF>^#Y3arQOvc^C}90-T;j@bK{c~PLTxH_Iyp_{ALp?uX!qURji z?yU+VUimjJ3)NboR}$}r#Y6+`zzN&z5Tbk}lMSsc!yMh|lZI15TofCB}QX>h(XO8e|#DBYhR0 ztIl*anXP60ERGVuJ3e=G1ic^5Pt{igNxbRjV?4y_WLoNEcPfz=DW_hILU_5;eZYdd z<$0ql9>&#^hRZ!g=ZQczfR-vo#UxKuDE&G(X}IOLdkG}s6%3Blan#Q)U0+2aq&zAf z$CR`w)D=P^_cl1KcJX+dRoWc*Z;D!!_%u_j#9jwarS$jyDV(3aA>@VT!1ELlyM+Vn zzb5CT@H7#_+`u(fo7h-kED#4bgfi!PA#x)x&nk6Vic6kgt0?KO?;Pm@D*=wao^NCW z=$9DG3fgsM^<#O0+%req$is6Oan+26;NA+ILer4rL-59uY%_?Zatp!P%s&>6`CCwa z95<$&M%xS!Z^r;gw`fL^#Y0NGZNpVO%OoCyrC=U^P+~8L{ zTs{nCTzWZWv^xnNtyq%3$P!-6x2j4A=?CAnZ_Z?C1a4+cgE>1{vw&jro0RjLn{%7E zi<>ud7)OJ8p%H@l^FIe5H&4cM*Ou@t6`|=babjI7=Qm1 z51th1UoM5IOY!01rt3`e5Yv+sQ0dq`UDXNN(C_Ux2?f~5Wy@+|8C0{g_nX)KptDo+ zNM*ImW?F9gQ|R{4mhD_^i)l&5)Mr({Figos9bI0IbVMeG!sUU% zII*y8EPf2&3BIiTP4R+FmRG<73`ewpotp1>t-3!vtuuARADc`$jnbP54c=X+WpA)D(nS-hi~Te7BD`-LQ0RuXP|= zu2V-O$Y?5W_-SPmW{2O4ofz*RmWf`D=n2+nqRSAvt%nb1DF*F?>_gI}+8 zcPb5oq@N7JrbJ_mMV|z)?DA3-4v#svW{@ns^nsFm3}P#h3@lzmvcYrZgj4lHet7$N zO`Bl3$hc@p(7OR3nim(V|`E8ojRUeym2x_sz_1JV=e}}`*tR0(rI(kD0yE> zLzH2aCl8`4+!20FVQtrQ!E{)KJ5OQk<$d}rEqU+@&POeXbR6mPuKZAfn#c}*8q+~^ zlwD8dEHWOBR2Ym8X1&oQqy@aPIF|o2wiSPzY%F|cU7TUYXFjWgFoVklKT4T;(?uMr z!h_$;`G_kzJsfG9;p;iyIS%^D;+2&aW;vgP2)iq!TsvI;EO@*rQKghRhSCN6DZ! z49Fi$>2T(CDx)|UBXoKJRv=Ry6KUx96Wxg8bn}XXmVSUh&PRUxWx(aMSW$xOSf&<3 zmrfQdMSXy~Y(Y#{&z($%1Sb^bP6R2GXB>|TkI1^Dup*D_HVx(J-%?0x)tIx!vyUv% zdLO8&q!arQ#A>)as%cE;(WImd1JYe?I_NbPy=lnCfk9QuX$NLW*65 zJ1rvaV7d8x_#5_{!gsdf%d$z(Wy`A%>J77wxuWm`yZTXr}Q90-C{ zFjo)@fmqaw6^Easy!`p$@dBcUSTq?zzh965sqc%FDG4u@4H{ zufWA{lrCW^6zYCBFf7E)0FOzqt>t|;lN^=Vnr>NjVRb-72>#QSC!!+vxw|uv9&&MM z7MsjaE`*-F91Y=1O4rLJHH4{BIQ6d-BJ6BzR=m(R4VG^<_~GpIB8?XCQZDjq+1G(s zJ^`k4i5$sWLLjGya4MQ;u|MsweE}cgE9FF$soHd9dM!5X-pHDHEJ}kPun}!-AxnoK ziZX(K`I!ri(Wzb3Z<+*9@7oLc1H4pqWJxzqUs?)4^w~DX5e&hnY^?UdrGU`X)I82+ zc2ORPd%NatC4LqsOVG1b{JyYh6Y7o&I^{eS3o z93d%sKfQckRV7~|QS1%8h|hqQByog!4Q@pJZcGHqgV=N2jm9Dbb&;`|QFMukb!X>m zQUu6h9YA1lQUiv{12Rvx$-cULMS`quYMZ5Vf|5nM?L*r;d?J4A){)}dS%SXPT++V% zWS*TARD6Y5j~i+F^$DjSvWGWA}FS)ps@9U4iF zE-n~Lo-WQjK0#?45Z8<^MjW2#^JU*q?GDd;erVmFFvJOio{$ zGtv>rg+)*4=&G|Ol1M0FF|u(IIMC&S*BABGN0wW8W_#(jBB<52M(Q%6Ns4}am!RE( z^W}~UYGje+g7XtL?`d#6h=%iM#Ewz-qFxN$K`M5T5QVF6o1nXF9IHNOi$*Sp@eZ`_ z*M06)PUI#B1sOb)D2EX8ohpygh>tZs^^;~HOe$$4%r`7Z9G;SDiq^0wDs`pdT@}u} zgKW^VWax|tnN%T|VuXwBua|{y5;EGN_%$*@D|uz^{ya1TM^iFW%ejX73{3fvvHTzImX?s67XzX=fx2A*hqyn z37%$WyOyB$zvd%7J#Lah^yaL{ya^KQu+|d4!GmyLndl4oWk#U>aL_a&ZQyrUTFTLe zD8g`s%js6v1P<(}+darEz;Pbz%f| zaB~=yy$`Dt6;`U6ovW1&Ue05E`2A8mL?67+$D#r$frMy2By{tK(f6JSU>nGi<6@Gz zskqi7iAS8|Y&+b-{rfI4anY0K4JRgqeqin@dQe^@j0OvYdjv+4?JY{uP|gHV^2iF?G!~VWyBYg297C^>NM)6kTi6Ii(ap$}@E0pQs+ULk=$Uq}ussT;6(Nm>O6z@C zydL5^Ce4w!kqxy}CvrLOiB>|#0xiETw`DA5+A54-&eVbwEyOBEq2SZHg5|HKe}zB{ zGksi~uwvEM60mt)wBYH31KdIrl*I2S(ioB|U{?7b{J+=e0*oRu+*BA{LZBLqSqZjw zapSnRE&4`3++P0LCFU~L2M$jM7PW=cLMfr1#R{)-Zhc-r=4rF_Uyd!Blg8WCqwm=Y z8YO%p3>NN-`~#JbrM{~(;vEN(4{eMe;=^UX@fBds8}iD9r&8?51p$KKfW*OL0_$9@ zb8Ppe`a*ZH1c?>L*C+PUB;^lobgNUGsopsfeMX>THLo-Mb;6d=!f1=q_wiv++XZ!1H znu_nupfN10i3Ze~58n`e*5DS86{eWOd9Fu-+Is+7A#=MpnP&aaV#)KCl9!ZW6Mtot zhe(T~X+m8cu#oSf*k)_pG%qKw@Q*pwtIw5lRs%kf+`8rCHtX{IK!R6Z>SAsZm19ax z{gkbw)>{3AfYSOH1)^650UdjbVXceXQ`368oiKn&Iz)B&1|I935+1$@*VvEf zi|g{*4vTn1%a3Y*8wX97wGd%*f+qL@)f6$uGp5H)_gY@X#6`kqA|;f4yaO#)J6$n zu7+o=S-g1VT(a~CfE34y?aJNSWbTQ0iGYG+QoIPum>D9vyb5q(CdBP!4iS!l;rOh93ow@PKqM?SIq_&;csk-MFt=RmIxC1CB?IDVl z!<}tM(KIUyGK!=4kW`S?EEVn*VKH#1|3)Q8&t9^o{%)8PO0*%j9wh$ zyTWgJdAWpzgZvPrC;eh%8e$JtLsxf8=AaIYruC(EuLo}C%zZ5vp->B2s2@uDg;Y?< z%+-X{*)?w<*<6~J5T=j{W-EKhwm>0V^r)*ka#&lFDlCn&#?NWc-|A>^plK3D{xtkJ zg~-GC0?BRpmHiedA-oeIl_!=5O@SBu(>^GTvmg;n%AU}IFSZoMke^@ZUz_M3POUDr z)cp9s?stiY;ULk67-Wjbj!-&*<5%3AsI3+K>2X0sSI!(uFQYN2mVl4#r<16?y+yBLR_5JCPc zS47|Nd=|ib#6Ka9mP$oDIgkm+0XV~f5-ucH49VwPO=YInAERiMu+uol5~Mmo#nUe3 z3^`I#oce@_M!{ib4re$VE!_}%f;1d;^hJ>hvii2ic{wLkLZ<8oV`0nmSJUjz?a)0A zTSPj>hp`fHWj_>IChZXBKwsrZU@h0#VqtJ}py5^cJx5qvt8ZZ+w-A!0o|*0r1w^TF zII*zWdhhH~yz~QwB7~6WBFZB&w#@B@uw!W{RA5rWew{ySTf%yL*6Oiv(w}#aY~410;Cx;10nS zhrr^&-5r8!fDi(~5-dO--}mbGUcIVUT{YF!Gt=|O^vvyh&b{XbEa-Z$hMGGcA8-VR z-tMy#yw2(qCeoiGTeaiA&e)_8DT27?g*F(j7IGOP;+hl#`f^tlVRS1^*#R?+T)3f6 zBz^?V&<;LBL{!FxCyR!`i8bdViogPQ(NrIBcmRx3wFRI0ek$lz(}uR~Pkg+V%cDG( zh4~vUU6q;rtw?wdI>@*zYdu$ZgKUI^sYD#wZa@FX&5;Rh!8urJ@Q}+`HwojMa#WxM z$DzjMVD^W<@L`xvcIrKYgB-=ov&&*qF6^sw)zRFE1;~v$B3#WMF#hbyJ8r=-1*aK% zP{i^J5okjR_-yzBlqeHQlx3JJ5K)>hp(Cy>DdrE(%#CB1~(W@e*l2XkT2&sBNp*sJh$4-lu(B`V* znMxc>T-Kxnedjs(FcaZ5jouxc^AQp>zzJacX0~%pTy3G)-b34Ubc}$$S^|hKV{9z~OR69+KHwXBB17AthjC2`7HrEqCdfpPZ_8)=Dy)n7O!c5SI$ zt4c?M{J7S1Ys~2w2ufW$Eh+~VKHAA(Ju>ww$a;aKtva+7ou07h_Gy{y((;p~&BvSY z3S2@P`ZVXdhP>%IVgz#fYY?W=W3xagstE}f3GrHW5wA!=F67hJdqyq07rQ7zO3cl! z1KbG4#s3JS{(R(VPQ!+GJz4N61jMYjydoSWao4riaDROePg-U0S+#yjOn`%+cTVN z5h}7Suf9@6iH4)(zYf0_acU46kQRA`1=R(8O}8w`l}D=wmfeP_ZmpN8wY^%Z^Cj!PhIANJ6fNdZc;8G+Qc3b9C;gDc0+M`9m@D>!#+q( zUMbwAzAD-Yl0uCMdxcN@USG|nb83okw7HmL3iN3(KkVGQKT7knmF znD4)D!AO2(ifBzefRD=VRXNUQpXds++mJcNrG z7jhu=@;h@_k7I0I4rxj&xd{Wp^Z#AF~e# zDh=O;EQKa=r}Pnu3BTQOVx%$Rp6-#3xP5aFQW>=}q|Ys0SN5set(7%kQ{N>tvQyl3wSce}5;O%!1 zb&+~_<(oB;DQqX#AWjDkWNpdb8L|m7xg3Ngee0Uy>0Oj=5xr%YKaj(Cu0J=^pcsDk zMwXCAVzV+Pe+iQVky>mZg&p94We87pYDpQovcEr=httOGaR>`Jqtf1ovtt@ia?#(}v+AIV!T1<(U(e}mVXT54rf(jg>b z?R{bs$3p|-4rBvz_w;IQBz$P{ouF^J0j>i)y4b6(l(V^F+$0&mQIusBn@IG#(8>OQ z=Y?ZiqWn+;Yu^PwWPoiF$8xn_?-|! z!FJ`rykH59tBLwh+mjO|I!Xj!LZ{zOxw~v(Sg$H`N$IUa&;9&&y{d{Or2z-xhxN&i z&iNxS3B>~E;L#;T+d_lnUWAtArObAxi$M-Cyop4nRVCfghpA}ggBN(Q&cF@X~-y+~9}*&yRsDeL<)ppc%O z1M(cyyM|i3eBe?vNm{#@!o!&#@|w*p{afu&qAM5b%V%{EPn=!S=NSBRzn^w0MRYFZ zt4sW60x{axO~6}pGzST!=6pF>FSE7S+7t|^wKLkIBhsT|P`H1rh+XJpK#mS$=}3YU z$~h)2De3QcFrdDKevpJpfvz%uR_cN9j3DxRGi#Yx|Nhb(Z|FXRqkYvt}0S2 z3ZcmW?VFn&+#-BgYb;7Wp6K2+z zOUD+w0$p~-Q>aFZ@6FRWa^G6gK%nwSBuPu|^tPXxQ;5;|?Ld}%0?&HfH^M|7H#@-= zP`OLehM&5c)^dJ&FP2nG<6ToT!OzpCtL&Gu3`uQRmMkLHn;A9X3wETH0odp~vFS(w z#%+np7)#2iCrou+-MBYKp$v?ijN|MUDR^@Yys|I~?pJUYcDg=daA+)m=JHp=NpZ<3 zMxTuV-fq`U?Y^O6SVY?@_B@Zf4Ar(MzzMK1_v-quf(?h-9MQ)MK^mRN?ptDd&jCMn z5W5X_IUD!CSpE{Z7&LYa?BXdEQ}mQj5tcrvJ-Pk!DbHA~gbDyPyMuA6+?0c%wlrbFcRJpj_dK_6R z)Em83eb_nkUw4CKBRPhihEzp|BJ5=sCl){n08Gm8?oaw&3c>#13w(Hu4M<4jk>|!* zazS+xRq#RO7%-cF;0mEZIc$$t=Y+;X#kKbV&QxO)!P!Ww#lNqFMD>8*miJ~7rU_Ak zssrBGi4+wPkO>!cTMkXeToL$!L*E2sUCgoLI?abl+MQM~%6SVS?9%z}hCQBfNb;;U z2X#=KUPthm{}9olZ%@tZU3;hEYTA+_`0Y;16xo27QeY%A@6jGFJ8VlNBoSUq-w3)f z?fSVpMT8;HUuMnmiOfJ2IrN#(A7XA3ffn{{7|6uikgfA=O?Mw0O9XWTX_27O(zc{i zR5c@Qu?QY7MNtA6)nb`ly0P`}^T|qah3AkuvIKtnDR*1q=1cQa@xNn~L+$5U5mV5& zDt!CIf#4jo$Z4T}p&7WtmK7fp0AxChjLrj{k*q9IQvm@s>0^LGo!-Pu6zLY^Y4$Or z)up&xuEGZtv4AzeCX){n*0+554>os}J1EPRTIDRWf0V)~7iANxfG=2qpe|!}$Ut@0 z_^2(yIY-j=ra&;wVv@Ohsvsq)4=A=xe~6xsTtuyQXj%O+j}ZR@$|DsM8L|>Dx^qkf zdRP4P@DxQXGh@H7{z1b9%M+V9r-j{f2^#TLb~8@auDPH*{>K2y7q=WHpV68?qK)n| z(`~6_Mv-CjvUh}PC$}xVc<2CYdtdyi9l|b`fHiGp1vxM``-@Aq9TkCQ3Xh~%{paY; znz>{s^fi$uJ=b21OZRIwl%C*6>}*pTy^sPdT2)br#nO*dl2v40R2!V`2Ya2}4pf_iw2;y_3bDmgkAY8qwHEIvI+8PdIfBz!df#*NaBgm#>+fHJRl z^*G1z-)<1zvRCCQs94>+$f`5qSzPu#_3jGQ?kp82vs(Kf)JU0z){@k?3K9{+u7d(m z+5EVOMaI|)cj z8D82ehG96Z))b-PO)g7a9%daPw*zleakQvNR@rp-(0|@j)_Z0umFMvIP5aMfVZBqi zmqAz_c>7_-YnJ{gl~D;!xhRo&miM=K&oltP0jCJ7O=Yn$lD@nr(B?;;c5nV~@jifvI_6OF z96@X5A}}EKK9;aE{wu9s&+b+rs|VQ0ipdPZR9dPu!WzRygT$oYoVg#h09-f)rQ{l^ z3F$Jv`?WlUZxpH_ro<0XprW8@X`4DO77Cx=FBwt%_~C;ahjdjFH9J{b5q?`;E|ag&hI{vOdF$EPv?{@A z^?UO>(-=w^7QZRzS!%((sY35Jr>Es)MoY|n3=#7C*aSaH3M=~DiQhjpnrCf3-A6Iy zWl|e_UbkB#YGLNbP|6cC!T=tDTq8znji$bu3eSoJcr73~r!0tK&#S;uq%T>u9~ z&25Mc=2vtXP4la;xf`b4_SDw&W_bRbY5}-xeHkJfCfQ&$TjswZys7=u1pwj%UFw`s zR=J*Wbn>2dHmnrAF(0?a6SS})<*seS2T1aDc~EnOE%=K=o}4(aM8B@CmY@RC@ECE^ zLk#I$aGShTPbGQ}lr(6FRE>j)TofIa@$qM#UWu85tDwNsCF_=MJ>D5Wb^riOj1P`{aim$Ili zbNgvLHBt(LkBHWfMJ<6`yfb?p(=Tk8k0)Z5W#D!ZSA2_s$ZlWI_;P>&O8#loBi60$g}WtN$DO zp+y3Rn*;7;QT232RHOsS$n46h1sAY{1drukI*=8|h&&8>Kq{dTrL+R|s&ZWLQyq%+ z-Gfvr%i8OoSe%y30p(*F9|FG^(#0+Ki0cxUB}&uR@X(6WkrZzw#9OnjMWX`<0$sU@ zDsjJm2pUMm2qZk2sN_OYAL$IYKsm%JGOFY@-EhJv6ivUaxw&vd-L)5MbY;0?O;F%U zKi!bAyB|8M9Hg*w4muO&^hOkxc;>a=$jj$3KLBA3jRf&izti9W@O3N}p&0PG+Ira9mk16LhE5v}tvG~P3ksb}0pa>634tD}} zN#W)Hkbj3trOPpHD|W>PWX;KAol)$)T~F(3`I&3CWu6zsA>T+zaKbR4D-9 z7P1j!{PSl0vnaTfvhUiQzBkrV#^+ck?97G7ww^#cA%9(5&B7mGaz{i9mQ(Fh?fc>% z{z$*fIFCa79pm~@iyNyEdyRRKr&ngImg5{8o|DPZY=g;_LsX4aSzARYYnzzC!(KhD6e(9q; zd#Nop`sX0>C}PbRUcn#SLG><1c2SAPWZSxIT}3X{TvU!xDv}u-CSG2|>8l2#^+Y+p zB4aQ@xrVt}X7KfyAd<8c!~s0n)X8{sFG=zyA^yp+q=rnnpZ%Jl{pCFh;bNc>*=~GQ zj;p!FDO42IEAbY%f#iVA?^FyHBF=F@@}=N8i=kMpc{AXhqAn-BaG!I)Dj>|z5XMz0 z#X6PUfX?mFqs09tI!a+;8#P2@M?yF27NnQ_5Qyqu9Va}XX!u)a`M@c3)s(dCSqSoP zZE-5$h1U5*_L>j3h|ufhwV?z6Uu9m6kmM3LF_lt+v2+RU`h<$;1MZX_oXdJquRY5o zJ$?N5#~RR5V0esMaZH1>zngP(`nnAFmh(IaI-a;yEX892D0E8FRODU43yVSpHZXAe zEZ(3lCDc+FNvaYy_hQudbXzL+$A#nc;w#+@K}0ld3s*OZmfL?EtX1u8gjmHQ^s+g1 zkK!HT|BafF{P0pUc%NfLMCH!p&ap#gS7E9|hJ1yz~Rtj9Z@AoP?adP>USjCyWTn^~l zyz9T(%8D0lTyW=G>bAX0Oe(JUyXS$ujFkjI<;nDh(5vp~6{#-P*PG#WQ#d|Ev`4hE>j)K8=4dXdu$kA`a&X9EK;H+JPe;s2Xq&O-?mat^}jpT<}Ck*KyNVE&CY`bRQz;B%-_Gg6QxXb}*td;dd zR(QX|$5MejOch3O!hs`JaTGRjlv`vJXS=~Sb2uzioQ{B{^-PRg2uJp3f&lYI24>sJ zGVoyodn9&B;n^YB+5fRmgs+SZJpF(l-^sl&J_?xME#j^%CQrONNMGY*}ap}KW$ zjvI`@i+*w2IE=C}n74psAO_TYH~DYIoZ(i&^%GlRy$%M^J}8(4V?=mn26>ltg0)k2 zn-K647ofw4)Xt;jv8aK~knBNWprzgO5?2>HVG;Uu^=mXCP9SBG=tK~Q0FSy3X=mt; z(=uIyJS-8)2Fi-+gW;_H3p#1R;u>nMm=U`ydv6u63AR`nx@gZNgsJ9pdDi>*3(0X_ z8q^FC{fm9IHx%)V8pkX3jVqmzwSYdo+Wc|J!kv;METH15DgciSJ z5r>gkFnA0HPLyX{cGpFM#zn%DD(1E=aYMEEiHEpM3PZFBS|3V#yVh&x{lk{ z5M31=t5kWp4nELhKw3Ekoe!-2l`=@t@kJNz%mEnqTP${Ia%E|ke5aAXg#O6b06yoOMC zm2evp;Nj@BAVf&4eejaVfF2&_@AtsJC)@j(V!Rd$c?t0Vh>6jtn3dS@01FHv8XhH0rwbWLVc3v!jm85@a;;&tydsZmjf+?f?K@WS!54m|@14S8 z9=;8|+};M!5&=+nzo4a>D;P*$9IW(p%ECbdbuJ-13Iz-E1%wVT(y>-7jN>N3`8?*h z@|T(2Hz-^RM?^HVf7mUj3%|<=N7_Cxkqbbi-xXVJrer|})8=83vMtSUSUe_Y9f&2N z!tGSzko%E^A0kS!nn&3DPTDb(Txv5RW7jcv*DAMY@h1vVA4}KN3vTn8>EkpJSVBen z?oT+1h=Fj?eijVVt*rfE>+_akT-%482){*FEG+a1tRS}2f>C?%qheV`CVmC_(6QUW zUIrl^WPBr}Mh5T)e42rJYc?KJB251U`5K)PrJ04HbRLIb%>;CvlA&Q>$A{;tOw@Af zH*eG>$|i!XheF0AR#o%a@x)*nmPL_;BWXYAbHtT^gV!f&rRx=3Tq*zIm*<@bo82%7 z2o_3czo~I%C~Y?MA((dYuH?ex3Nk+-H4Ar*aG<^|2fW}{L*cO~BNH?K);4B#hJnOA z?A+R2{w`iDyAz}$H^)QXm9>P@VmQD;L8ea7*RlZ&7JjNCI|};q*gdmj5)6Fp_AHN8VRMLLGjDCD_0TjVP4upcur# zpeBExz*&6^t5y)h4@WtOyqX;LGTKhe&8Y7GPvU2}%xwOfK*i z{~w{2_gH}bZVUb+;~3f??!rlPj$dCb19-8D<}(Bgihn53XDT+>+ONbKCv7V1+RN%? z&h%3`6&nRfvR@^CIuV5hmD+6!9HahaXSX~%-GjGy8^o-^l09RJim%>9^X0x?ez&Gm zXGs=HKU0zW;nVsxt)Z%g4Hb&l8^pKkd{Clb6!r^my&BjjsVjx=^Bz>+bb7cFeWHOFAjG>dXS&KDro){H-mM zbx78n5^J%bXm6-zc6J=H4|e+V(F9}mImSAxec?Ab8MghWgJ>qEqIYGxK}*P`-T$^A zXJTZR>YOYSjo3zv`9<8ZRX-L*pSq)k4peseG~Wrwq1hs>jP%(e)x|mJ>(NoGiR%G~ z-t-Z8Fz&wKP=GW5D^MLo?(PE0M;6DhWcU5&sU;+a37FhW25^^Zptzo(O8t#hA`z|z zr~|V>{3w=~I(mQ#kS0ntCVxf{Y8F{&6<7x)A5%QT`QLq0p`A-wYp2bIDT3$In!%Wr zq;6wD75fb&PkFfYC3)fh$kW%5{0hS=*=nUdoAPYtTG`VfIlJ?0UL2jrr85C?ZPofT z4NPN-ac$OmG!5Y7Sv4(8tBU=qT!m{{Q4!gAvqVP4%8wW5}~j@8dU;diTqbZ}|NbuH(lLR;0-C zP!$wCPI_GqFh9G#Y&8f%GFKf6MOoscH~i0#fh+=)N-|d)>VTrhMQ>8|GGr+G1++ji zS05UHvcyGi;rKEHl^p;PlFl`R#*lz(>8(%*s78rB$ud}=RB}U@j0X6P;V&m##J(y~ ztACtxGY0Yh=NhgUsNB$pR1-#ftp8lXY2+3Pd>LG|dQ@W9 zxQKkO6Gq>EpOP{LB;l%jnE{jWc|*@xiuq18kHAMEss*_1>Z2x9C$dRknHWui>qlQW z9|4hQ|0QO!=MvRKGHFnItrMaIg|Efga%FCCJtBp0q9Vx0taz#_mZ5bV%MUtra?6x@ zb#%+?x^*5Xgw+{tUqAj&oqTj}C;UN~@2kCImc1m=_T(Robz9#wKaID7M;nipQ&{C_;XiF3 zB2TS8yrE$o%~r9N>TEo{NEM)?kaQ3HzWXnTJg&R3QLOdvv)}gofjN;7V|)6H(D3a*AucnYnjxyrKk-+!(gG>(mZ^K|` z4`o8F$a&L|gVE8(3{?Xe6V4HKj*~F_)Epv`goV9!7QOdDELgkq2$x3km5^UN>g!rc!}qaxB^TFqLKgYhe4&9Udh zlqP4dvjn&1Z};WDMG|~OM3T->9Yh0(bJ$XN!hXK3=gXgN)m=RbldbUU&Evl(siw=_ zJpa_z@b{C;RzrVYz2I|K+P@x*tN*KR^_&I>*gkbu5W<2=dS2}b3mN)Py%3Humzz9E zEqjmcia|p(%TiC4x#>ORYJ&IG@64-2Cp ziTK@Xr_r8|+ZDeSn6?g4*kXz8do_DantI4R8!uPW>3?G~cIJF0j__OMiJE9f0BNh( zF71Cq$Rnn44}_KyRz(66(GE`CZZx~lQ%tde6V56~rgpf^TW6(Slm7WlK>hI&?1Qm)!uRR6- zl?{g8KgNjks|xhx>uq|}iQ^n*^)nzk#~CSr57GdOu$|=LDg2S`)Cmm?AA zCr>o=(8yoK2<&`EWv+DMpPJj;zp*Vf##2AC7dDIeJ*x-(kL_WkVLd(nsv#aOchpn2 zPIiaepD;IZ?X7@CcYiOr#}q)E+$l$AWh7gBK#t$=!q8%(lx8fy@G|L8xm2aE0dLsW z^TTpxcu`UYm#ShmOPE@AoydIGEati%e2hlju zMOyMn>t)uz0$KiPcsC8{FG*{=Wa@b8(J`dn=y-;C-Cd8$*690vQzXwzx_&(};u_thNNzSk52pFS=2MTellXk9C#iMYXu`K?m9i%qjy%mZptqu z!6NAW!?tao+mas}qhV2_9S9nTiTPe@6yJZ9>Fq{!j0e*<-$fnLtriNucLSlt z@&7cCdhxzd>7C~t>KT);`b_we?Mza76Z6-;@&Zw7=m8yBZ!*Z6Q1DEKCR`WCwteKh zab9_Ot~Vld;?j)Pt7H$(lEScwOR76K&Xgn$&wxQPd z!o*0&LGN+Vs_+z&7*F1 z8GXB$8P|FH>Xx01+J@5+gA|^0tNG}?fgI)t`Mo4ApsVaixJd@rG`;f=n`+7K)5t{J zjG6bs!=2PZL5$<_Q@VY7�{>YhAl>h9Yg+=rh~o51EwEtVs@dvR^7;@~tw>8-5#q7A-avN2$LdXOO29DtwyCe1U-I8iws!hrKh#yw{)#sa5o#{=vKHRiOUNYRyD-q5 z`cAiQvFc`7!9jwJZIU{5HMtiSyD(D&>-ryII2_DIfROiJK=BHrI?xi!SChBA2ckIb zCxO*U6*UyhKjl{X*p+H(w#<#M<51Y6v2eWUHY_UPDKRp_=o< zGm6B<_{%R+j%}6Q${X{Mq-2hs`*5lQ?DUK<)5^f@SJo>-rb`_Et|yXe>l!U^vFtAA z^f7<4X*iRzTaUS&kQs|l{K%+Vzr;+-nAUy%&F&YBntQdtd4S2)%klhN=o!ba8Ij0p zNSYecqc!>K_LIeH;O(N(6ENU_U#J8A1^DI1Cl2ZhcZP*C0+!`J_*Ge`qKa1K!$u7= zHx&$ripOkemM3sRQ8f%^HCs@oha2fO*5WKnm9N(5?%Z**#UjKq%AS?0Hg8{|t;lg+ zX6!M-H2m}n`P#*JNm4s)B)fff-CLuRWhhswuX&qb+nUh_RsDFiaeRS#A$IH^A-@Ep zKZFdmdD6#ACq^DKV3PGsQC)u%-2RU8ZW`rfi{Zv*lxE9|m`p!yR4?P}8-TxJB<1DvS5P7MeWR}bK3QHkxdZ8b9-n*}Z5}^w zK!7)NzleG2+>A$f&S99G=hjZ=ZGm5WihMXJ<00EedQH9I)>B8jlLwVkL^Y1rMu91- zz7tlLgWABpk*;>OE^BO+O=GtjX6gYOj3m#4?IK4^Aas{EW-1*2ms-{;L#$JnyO)t` zmpan3PmPI-$V`ks0vwePVmCW}tVASdH+R(JlsOMkD?!u`a-KuB>%-t`p*ki%5M`+mP@$>{`C5z) z|LZk<$&l!PsU29-spU6v15s5n)k8mxc&f2wRCI0fvwVNcN@x60<0I*uB9>C4W3CN1 zC)~7fOZsEp3Y!PntwiUPvUXszHS&LKjy%?_dtqbh{V(fHoFbiZh4b>1yRT z%5sxK%Pss?a6A{EgY4$WZ^A7FO)*rI2gusg2=2(l-nZz`t2e`rlVMfr1h%~q(*Uy< zsPv_svOzG9N4Z4l*@N$xd#Nd0MmNPGC;_W-j ze@`!QLvgf)kE60yg4K3nl|~$~k%3bI?eLn`(6^m+I(Y-dC=MV-pgdga?_``hLtLV6=SUy$SV9KiwK!lAjMrihUyS4nI3z zL-Z*SM$k1ht0Vjp?qO;wo71ba!n2~fy8e}nyBU9mn1#)BE?6*`*Rz__z+SScJW`;Y zIJwh&wCae;?g!#>RUgPZ6Qbc?% ztCrW>Q_`>fdpkoc!baw0^qgz1HwCfTrAV4I+M3kBe}DQZa)mkqCDZho(8+ z37#Q0=UJB!n~NHp`Ckj>;2v^mCt9hU@sOVjua|H$skNk*9#0hN{752-L%2r6s5cjz zz`KWX2>G>QVlu_xHc~`#6@QV@yo%miAp_}cG^;H3RoHSVcdhBtMGMX6re0;yPTZhQ zX$K4{|A|#Rws+_p2d?t*b6}^Q9NopWodJ1A(TGW4y632kX+uDIbSB#7eifdb38O%%J3b1p zSC@T_{dM`hU3gDbeP5Ubz!t~H+$^E*>>~#@3{8DsElZnudg3+lGg$aBx+*s_))V_) zj&q;XWC^{#%2>V#)dn}p1^AwRCcU?!M=gaf8$nKla2Lk!EVYHT{IjyW>eFtI3VWj= zoHu9obC5}!CZ2lJ?>y*-m4;oDoq3DYPqK=Z(vKm`)xI`qb<1SY+8*iF1!HmJ6 z#je68S3dRDHy)~%N2mu)VNBno#wQld;`dEwMDo8n%+9M#j^r+bJ(h0q?0wNcRt9BA zD|d_MGhaCq#ipEn)M!KV3tCpMXV zlexAhhx!BJisDTFZ6I!&&Mww3Xe7pxUqiiNG(FXKI(EYX5mht_i9@kGCZ{-P(>eGh z+F;3VOCSiR;yoo8fBtd8(xzIdvqwSP$j->SN~9oda+f@pb9$Ww8gUT8$;wT1rHoa@ zkNr(D?j$IVS>sskcU1-+YL>I%Jba7E;ovQ^wAKCHRgv&r-R7zB*2l~4L8dy`uU<9W zUkLFkEKnn*Z4hZ$Ak5Q5ypQZ?nMJv1$;nJAg>I?LWfFhHf7lDQ=a6~yqG$M_(%PRe zN9ztkL^^M?kRhuxH*xhYTTRftBUOU?h~T(6n#+sF8mvev)H0r#f3VCMfG5_=t5@)K z=E(Ts+&_<39;q$Xa;>;PXH>iCRQ98{p6A3^yt9z?;8RC;k_mY0U;k@q`J=+j& zS@LOIXNc9x)Id|N&UBN*`Gvj+exLyMs$3&>*90i-=?tj-)~{fp6GxY80ut}PYB2&i&g}m+^GjJ0eNpe>vA%U76lCEl(#S^PMln|9x~`s351(URG<{Y|5jWaM zgI(e)@SZlt39j+l6TJ>l7O@?iyzQrLovtc5!@U|HarjxFo;=(&Ibb?O2zMJKj^vja z>UHH)IvV_-pK7Z`Aym$kTbXFh1>X-k1$*FY3);p#6c%-oft4uGQ#eggT@iArr5;4- z2oqj!R~eVMIG8RBI|uo6u{EL(@ww8*I5Q1#s`j7C*L@YkOP2%mVQR=&4%1&<{g+f4 zw0il{KXSFYr?2t)Y0x(tm2w+!H7mNL^_Q>?Y%g8mPi&a4@bY$_GAWU;iDvy{?L5q! z+xv<=;6+enB0-k%7Tw^1ckYDGFr*8ebBh%Lb7Ku;ln0&`6IBfK6mDI@2-oMMPHB<; z^q5+3y`Z!vG+c{GM9L6J`8d$ccn~Z-NCq5Aw9M}cE}C;CaCNH$bHY-&4K@}t4wDS= z&?@9UkGv7~1ZlA+l;O&cP_-Jl?s8fCTI-`zqR-=SR*prg|1Jpl(&89Yiis(E8~g^# z$soy$z>5J*wcB*)chQyxNR1VSA4IllQs*`SlJQ>VMrKsnN}(xX3ZxDur-H2_?+M} zO%rf;?=ZoU{N3#lGGM>tx%M-4{thLNnsRYH`S(1I^!s`jr>@WRDaz-xh_Gp=h1D|& zOL&%j70D^;SN7A@3EZm80RMY+1E*}AqaP=B;q|){&u2%7a zOI4mDjb=$(wYAAAinT^;l%UO4=xWEC7t#C}mloelSBqHud~vDrn}($xSnJZ%;bGA) zz0{7%Y~Br8jlHl?&9hfr5DP)pI4+o3(_s#v%0){*sevJZllUs-k8Z*jv^L|N&V4IU zK40ly>O&5s6NTU2&@9@bX1Mq`C8>sL&pV73>cvt3Axv9mm1aCNx9a zZ00n=4^b;_Zk{^0IBtD# zMAT}maCu0~qFcPuT8S?uY0AlMWQKZ_@tzGscHC-9SP-|wPQ7ZZ#k7vk%1U$;9`H}L zJeXwa(1`X2nCNe|H=8kH8oW&&2bxggcRlN`rb2m7!~KaY#Z)-4SKK3m=1FyFl34c> z$S$8Tw{wqSZ$E;)<8Kup<#&rDX9H>Kc!<{)Z=*!5Hr?dk)8~mhv=Km%6~7U?M^%tB z`#E3AV5F{@+s|RX@0(4X2+xzy^pI>Nl1OA8_7B(9x>ghvnzPL>mB3baG z8=(5=f1uQ)>qSRyKV|Fb^>DQvXcrq+gQ_-2!9QnI$C@(oRr=IA*`q%|3vjkX>et=#u>Rpu{FV zBM1tb8MIDfOiqJ+Nfm@2?hjGo*b<))?Fzt=IMw8~ph6YlOZT(QQ^fw7XCC6`T9ZI> z2edSbO^Mg&{`T0!8utT|b>}wgPJWkaY8yPN+Vlgq>Z=!Jp!oLisF=twki^$U-Ga4D z@>~pg1lg&4_>K-#<;T<1WcsQ8ZrUG;VtujJTdgGXVqZXOQ*&ERC7)SuhtgU}?$>v9iy z>RwS#-+c)ZG2SGfc;S=;%Yu}T1xlQ?zFnmF^&owGvao8qJc^n+`fNcku*n04#^T_Y zyz60n>5CTY8?Mtyr*1(fLr`;pz0>I-;o3~?oDsgG^2gi-9HmjzLeM1E^*W1)`m}J{ zz`Pmt&!6@0t4Zcn3P{7JX{KqxlL93N<|s!s6G#$uHtRiumaR=mLW~$TpUV>cNJ3zJ z$7yMUj^&(l=V^>eJZtHWxtV8G`mJw8K6)=@;G*dIG+d%r;EdZW;Xp6s<&TbeJC)mS(Tn!;+6;Sg zn()4|ZOk%$i+b`zKdH<}{awR^oH=MIycyjHk)x;{V0lnS#S9$7joWiP|tZ_fs^_1qOZ}{#fXX`cfjxjL^f+K}}ge%1zk$U?P zW+ZU;=H`0(*Q6Pm32nAYT0<+By>7BE)oa33vXOIDVU)c$Yux=pCkds_`#?vOl+aFC$hq#B1NpBU+JDNU;6%W`uwe+a1{ydKG zD>K|s5|sK%%RGEKjve7spf<#*j(&xbMxGhNDE;MZH?9DEk-wBB!2~I+n#wbjx&@a( z)n=A-6O1{Taz?p9;u|$R&6g+9s(mRE#!nshD7GL&I;f^iPsi-zmms)xbS8hmb~w+M zeZ~sJABlFiEdJOijZT#ehbcK-Zu1tNf=U^KHE(8N7X8K_><|0As?ymzqrx~9J;k*J z+(o+;35R^jXyJu8rm7`3>YajL?IoFmgU?>|(=OJbrDi&y^$!)u*t%sLzYEuFlE#1U zC)>tVTm&U;&fx4>brC9<;LN|`W|xeJ@yzMk6NulVHvC)~$A^(DZBkia7b)7n5_xnw zWSDdB*R>AUN7sez0LJ$};eH_38ud{$h*UAPDjxVbD`~UwxU%FCf3d+@12r~o zN3M03FxA+$S~B1Gxj(rCYkVdr$M_;Td{E4T1e4+W41V;q`lE#nI#GZLv@b!S$w7Y! z&843M!I-_Hp+zbhhW>eSLS-VO2pXE}!qez@?JNeL)=08apWy)d)RgL3pAy_h9OI1% z|MIM)ah$%SXg^*qg#p=ajC7K*7+K3s^FdW?9Se1v^uO)zs4MK)$c?u1w+DT$whmWE zFI0O*uP|cZV#Vh&G`$9Lm=|)AD9HYg2Rs(PJ_;@?exkvAIxM z;gIhZe4q)_0;24FMSE=TFJ*T{QgI$%sQM^A(dgki(1jAMqI=)SkikD50g4!%63Nm^ zl5(graZYzq^A^`GwEqtvWrOIa%Ook=O1>vhZcqZ0Ae}VU39uocb$fg9tt^DH?v8u= zCWP-qlr+jyvFv0UQDyO{2vVCJBbP#35dHCW7FgdUScc@VlsH~EIg46mI2mnZ%ln`gk!@|0a>JG2;DmLX6mW!?95}#;9g-a?(by9b zPLbf(xRo1M$Q?*q@;YK*FDNmD)nt2Sn;o#S{DonhaPxhTxb$qS6skerAlKBp9b$xg7z_<)wINT!7*$^Xt?uGwb}X>Vdl(UdeS|A}Px zY1-^OOVbv)1*Z`QB+_xNu{6ksqTE#R^Gt)+XuUR#n|864JMT zkRo27jPqJ4l5qXzAeFjY{M%nF43%l?c;?;?LH`xDGoKl?ccTInd!`8H?rM z{J^cMo;L~%X~^~nD*9lOBU^`qsTHAT;F)O;iwh9zBrP-3FR>Mu<0s9d6)!uqOPr@B zG2F5_B%ZuMA&|rSF3U0;UZTT@vB~RyGT7?>2O1z(z(yD;ON3K^EZVAlA47~;NoR2G z?r0d1~1 z$tmthuGX|8DE0%W0prF)67`*t1d5MzI#L9)BTm$LJNO7bnNZA$4{YL1ns_n-oJ&Oy z7z0XAq7$3P>4Jz)B3T;t>l=Y_tZ>8V!u#pBQpppjy@{xu#0oeXe-bk_i3Tp%jVAF3 z6N`{8uJz5#kY^3F0PzI+rbqz+Sf|R^(h{~sNdRL7FisY(fG{Y@)0#4bOe9pUxp-obn}}_?pd;0I5GeppP6W*x)ezG(`n%8(xOYw-oiw-#14FP%PV^>?haM;q z&DQben3{}jA&c+%$FYeq)OTLiHjYjCAEp5Arovjc52En{V9~N&2T(u>Pn$kz zXRU_c!nToQTQ>nmGZBVRoIuF1LjblP>P7_QszlT3e}JE#=xu4 zy2=r+B>bT!OiYn_kTGeeRqYR<2u+;r`i1+;xY2eh+SDWyNa4xYn5QCnr;>PylA7`= zC$fy%nh@!7~ zVu1+YMi-Gv*<;VXHOG8(LiB_WKgpkqIclJLb&g=O+ydvq9#)i)*M*cOqwMw%H;4BU zTYKrt>JK;;F77CIT8~n(3O{xNVJe@<)6(APOh)I_bhzSvaQR!FSUj~ zsnmh7-5*l}J7L~q?cXaPaRI#@Uxh1t6y}vc{lR*>h;r0_9neg;s&3stQByI)Q&nwA zZJctR{`qg$bHfo8jWD!zL-|fcLQNtxCI*B_Jew@=sG2jAHwlNIRJ@-EtdlbK6G!Nz{QPv#%_PZq8rq+t@)OkWiF3tR z(UYJZXO`(f9H|6RDVm7zKEuS7Vf+779QAeqVOndtxo6P)ZNU!s4M4}IMCkp(`G^-^cL3TDI3y5}O+PFG(=JEy@MQv3D1j`r4JZp4RlhB=Sb1N#cG9dXdtYxO9F?)dpg~Ld8uPyOjgUdcTdUiDaLJad{y*E;O0wHnE0_G}{GWE^+Sp#tsA7 zBOHtlFU1JH2TZ8rHYH9c@hFB+AOv!>6h$K9oNO}jgZZ;og*01?u zPwTi}Oooa&49uD#ri8_&%vSK%NH|MQJJa_7EN2XCV%=F^4Sh59@DN7TXc5>bNao~p z-AOnep8%#!U|T7%rO2qW?D(?RLV>}!j2zB4%N;FD#)lK*1aljMVx)$1N(Pp5pTYO2 z?v#xCa#wzAX;kD`)_;MUBgV1|{s086UWr}QS993exg-KEK}IZMxRxs3FC;|;q$PeM zL4&w3oB-iKsp8HH0Zu_R7rndY9b-eb*&;C@A`k7Pz?-KpJuJqRfpO3dsq(HVVv`4r za#Sxa3@-yXotXvMR&cTLr@=%CM8LYJugKRW6cot#PVT*a9X;~!oBT^lIv;`ve0a5R zu>jqT<*WjxBQt4Z;7r{A?Leb4u^>R>YOHiJ)2+KM zCCL=J*<->OYLsWfud45+{x*oe zCVLPaNUBG;eTbiSK%a=;B3yi!onqy7sgFhA?!Fjy3$R$N5oT{aSbYyKU_ANqh16SK zt~nf#m|-@IAiQt}E?~Um;B)-OrF*h{yp;Nq4`qS470ZEUu#Chw2>3Rb(sp_7BoCq_ zj1VGcI$_58uR?*!71c$jApCYPhT?DQw4-w=B7EoJHS=4@?|y*F2`jmv$-;NIuplg~ z#tXNwUHPE>aH~W6c}J#i$=Jfxsi%&MCTB%#n+P~3*e(pnHwe?kTqAogLjb!Z%fia6 z%eqULP1cIJS-AgzE!)o(iXBP)=I6!FdXj`Sl19fkK*BN?8wg|(tGe*gEQIBSZF(Uy zFDE!H9MVS3HiV=BDjZ`b-#BS<#Sc2&ebK+{M0Y)y(_JD?w{&j7wcfZwzZB5{WaK50 zT2qOOHsgyoY{VxyF#dA9e{%F>^2oW|u5=m({`+$9-u|Nd2Ny0#k$;xrdbwl6*X0K0 z@D0&(cec`xY*cE3(iEeww6^BQ2S*^gl|r)-4s9r?T%%EMDkEZk2$cmN0PpQZiuC2= z!Ubf85(7n2v3>8uzfU*hbYX>#seyRIqBlf|CofI}0wu?V8Sd#atO#Mk>CwgAn2d-8 zQ(}uBiSsrZh2yk8VQ`$f3?kFIKXGd(Zhkqkn=d~nmbdtNBEL-rz28TOkK; zkrs~Rrx3?cP~#giL?sZn;|p?lUT^m;`$1P;8nm;x#}T|X3M>cu#qlqWs!RsJ<4}kD zn*cJRo#O%j-u=C+b~y8wK^hKaJf2>rbjN2iu~D0}>$f~^q*7GqbbEl(oR1TIvViTWxG z<>LzH$>Iii6OIYL(E4w9F~X^3!=fjjlp}_W-(6fVIkc?PmUXFK`99D$#KyyFI>LMA7(G;dhio7v-3q|uad~ry!6d`xS9Dj zpQ7dKeb`O}Q!Mw2}UwGF7duJGD8nq|_?k?*lQC{S7(DgMukr4+u93XlP^UC2J z=zt~FrQ&r=$gl+Bh|Md?-s%g#X}AH`UW2C&pM7H>Uxqr;&wey?v){TqvaWr9zs9Y|DzAlIF8>=7LVPUM_>PzOK8MirLw(^-&$V33Gup<3E zp&B27r(9-!%UVJp_$;5)hIZvfK58Yj}j0zaOVRQF1^HlxTn-su_#_ zUx|i`sszFp#3ZwO@A}Opm=W>%)e+pP8VGhqx(GZ~Xb%-J{3DkVuc4(({Hx<=wKauX zba~Pm@6C~C=ZJ!w;`8zY?$~leejA)Wfu2D8wXZ!Ms15?8q3$qDY%^qNJ5wkHW|JWz z6IDHS?6T3dE8$q6PDL#c>mum_^*reW>!IN>JE}ALXV>ik9l;@?U{~<}mHd3MM(f*( zHQcDMG5uI|yVXNme&6Nq9YmHq7)iQ%fA@_u9(04l!^$3g@>P@ML|tW!CVx@g<1&2% zZ(L|LIFGU?%Z1B7e5)wb?D|H zrNi&5e*Lia<>-wR$=`#|OhGojeKuuFMu*-n`kt(O6vPDlXbOdxsDos1biTI_iMt2i z&iIjU*~iYMN>N}!Kk!b4Af@6VL?t$>6~aco{5@BaZjv(&ll6{d0Q+RcCKY1@xU@xb zxQ!a9F*8S92ZYy7CANO{S9p_$IDgD)f1-H!I;mk`oh zeHJP#O`o}+g|5!#+;US?BItCF*{P-<)ld!9d%v2#R8LOi1*Yn>gFvsZ2G6L@NUT2X z1lC*<+f}Z1OdE}Us z;_LJS@!<7npqpKEFMe0bOHzTM1di|cM&%2BadjXOk?}QZ%txDffCoo~_cS;iluzEK zxWR?r2F)mM*|-42D%`~;lTDQ_e|o2_Sau*@FzTa5vF;L`jIGoCIevEo?(ZiV#YzIE zuD+qWf$lfkDVm(A&7m6vNP?l|9~@^V(K=c5PA;36h8NUDp+Ft*4LPp9!9B_bCxnI# z0mZwtn;v&DuWWeP9B{MTm+Z;be+%CgK7EWqR=C9_&y6VB_|3>qjVA`re+EAeq^iZI z>Bs>V<4%{Y-#cwq;I5Z9|5d(s?Xfg{PJ&X-^MC7NNDQvf%fMoQZqTDjkvrOxZ>?`QMXzc0SeF@)?cW=oaXKrGqtGTDfN**(mj*~<2$?qrg#l&Cwt zzL>OhBo|AtfRS2X!(x71A20KlCTD0B++OCEtnH0OFDi4v_?OC4FPi3bmariDwJky7KNEr79Bdfe&R%+}TbHU3tNz zE9DFD8yEQd$92d`beAp~9@Q0aJNLK%yn3CLuA;<9nq7Qae%EF3LAh8KlJDur&QZDFk{pGc?t<~whI8>QEnW$Q<-6sY{DT{GA4%sc&U3X& zLim`U=Z_p6Sn=7@5Jk{DSkAs3B3X_OKe+F!U(=WDc>~xGzt5kq`H?$6qGrT0G4H&| zpy#RhSNbi~u=;gr3Ka*;6jsQ9HD^2}xl-QPRgLS{C1MoBUsZZZ-|{g^|H57u%}C~o z>I~IuWqc#J@Nf^U$5$FlZWPG>>iTaLqof*e&wvB5hbLQ)(bWwPH{=+3s)ARG^e#8f zl^X%ms*cU2z5`djZ`gW4+Sa6IU+ZI%Z7R-6*3Po#XVji7m{(n0I!XK4rJ`L03|%2N z3Y505t4V(b-=ez^zi!x{y987CgKX`gM2sF@T$2-vffu?l3{q6ui~l!Y{St5#REp0x zPCpp!PG1M#Yk_RY=Xn~;E<}}#p~0zZMqRX0m+fK60o4~C5@8|whw9Jnar@rYG;m|= zOZfswiFLOkrxdytLJQxaOpnX0WFBnXWRw9>jLta#E-V8zlTquUpi}Oy6Gk2b!H5ZX z{6x$&nZ%x>%gURRbU#s1OvGSc-Q%2|i`0$xyuTa1ue*WbFCLIM$+*r@Zi%aV-Qqrv zJ9%)>K!d@#jO$>_4#)z!7)bX5SP&r~Q;yn`$tB=3k^7EU%%`$oKr;m33Ui3SlU&V% zYxUJK$6ev2xq%zhj`u`-DWu+tS+*i3P)ols`krQd;TxS4m)w8MJhnj#M1lulx>CSs zIU6yN%{ffOUGy(ov&wFb9YP|DODV+b%kdqVWtbF!GIPmbAfPxtMC`)?0T#%q+(iFRGWrpu@D?cgI} zI2Y3e0#|lTcXB(GE1%u%#P=zx_uP0C+240J>hDW@p1hk$Ti)9X2WTKR>9ehQofj>4 z(tUP0AgfRN>K>vv*u9diHtBnJ%&7PPuH|K6+iFx^)=cbH$;Y;0@omTOIhjS}5nXp={dEUO zD?R?YPQ>HIKDU3ID_yDn7=200k7xHW`v&(b#RCR=7c<+}>7fme7gQEb=euH6_4CT+#x9v$^o#Z`KMl<|6NY zv2Ueh_M@Y2OJoQd?`s=V_pM(Uz+7k4^>J>N;|4@_1t#v_qc%n7`neIHs()y1y+i|k zu(_cCvCG{JcsxK?6~jnHRYAL-^m%%~o{f9tdHD-ZLG>GKWm;U)mjD#Mt#Pw<8$n@3 zxje#>h5c^EQHY2uHPUmrQCwf%@#5BfwOKLwS?Z}*pX&WApIuqIA|L4nH@Ld576AsI znVrh%@8o*v`^57(0&EN#3R0zO9TbD~6~~|v|D&bp`Y(T9`Y%hy@vC23@0b4jN&YcP zu=9kr9I93Ujcs3er11PFz}0x)@`YRB4U}>!Xk*3xn*Hdsd@7D(t;`6;KoF>@<(qgY zW~tM)TF zbcO|Cec=a&Zy2_51oq+d?*Xm6qJib9H6Sj8*tOX9)jeX$U0blotEoz&nLc(de9Cm( z0)HS&R$Vm36GuuxEh{K#1!(&KrE}O4*ZGZN4nlChdD-P_^kVvIkt<_PBO8;Lbql ztxpv@rivX^`*Wf}#HnfdblN0e0_NYEmKyj9U)=+c*{m2Awb;L<=Tj&{x_|Ig_rrb} zzuBd23#<(Kg^0HO<0fspm@i(t%1xbnDxJ~b7r{*iWGWzI6-wlm1>OX8+yP`<7Uu$2d2kp*Lit@76 zP&fGT%MUCXpvb%RqG~r*wHr&-n%dn}a!Z8GP_}*zZ3^l{X5S9FWG`R4@3L z-HdJi*YdM&Bt@Hf`4#?tRbM@Ev%7h^f>C0Bk*a_2_3j4^fmTlPJ{@xJnT`axuX1oz z1y}9*In6iov3Sui-hHfFbv~-Ygoahh4pjhKD-p;Q_tt)J@WiCK!L8GK_~P5eJ(VAw zKBqjWaFj>sHe1EH#zF9XrT9uSpL0^&fFP7J#7kFy@6oX7taqpZ_cnP9l=jH*Tz&Qf zi_h7^VHvWCQIsaq6lY&~Lw46C@Mu21Dp>xda$3?=!^hdRKP^7-W#JYzU%rO8mlwVQ z7D{P%UAU+7N$Uu5X2k=x9~?NJ2HkKCAXnd8+`9O3(F_YH9~Dg`zbyqE@QACsKlz+v zRC#%@nFiRpP6w)%L#zEDo*PGlRqHEy|KNLkmf8vUr}n3d-!fnNmZghoOw{^y(YEGh zD!m)rG-+V+MxrY^cyDb5JqJ4E%%k8u&eA&2D zr(xiwss=j2ty1mPjF@ZVYweFCuH20UzS?P`a*^&<@|&ZZg$J_D(3IO4OruGjSS z^7r!$0SQsnh;{j3d-3bJLuZ}J1N##f8)-E1DPWo#n9yZ4UAEVy6z83*PHSWvO>}w3 zKi>g+y3(QKB8AJ{%zoUxCrUm1OVuKHwI1#A!R#Jin}631UJetew)pGn8-<;?Oo|tw zKf3z++1$Y5L27!FLr%PU`$qjlu9d5!q$#bkY!;}FkgArqt4k;9$^nqwi|q1-ZG2-! z8~?oXarD4o)uowE@=@qMR{+a@qS%RVy7D2a&9ZB=MAa;D-jk=?2>&qXkS0u@exQFGFTuKYg6Q3x9tyVS|@?e19Bx7Z3M7K6owU&Qnp5^6*0 zY{XzSa?*DeFke=lZ#v^j8VCgm&jX?_C4;sCaY4$v*dZE8}i&6pAdv`xa zkJIuOG(b4N3x2lZwRVEI2JHK+3lRLOFSx;OsHO}5rT76-c_XZ;RTAp|TsxY#W`IDZ zMwC-SPz398Uj|=N84Dc6MwC+nw#0xDcj3>-wAR%RlgL^?~o^X@tOpZ zM#gu;LcT9c7qJSGHySD8`$3mMtK8bSN0gy4+XN}?i$C$(XVT8(BZmxG13yjjXS$Ny zG8?kl2f4rK{=x5r>R3A61H7`ocS8oC`LGbp3(@e_?rZkMcSm9u$6&Z*q}Ki9&yc3f4S8@#RY_~6oc zjRJfdP*z5rpw@?9=`PCZr>kFEFF*bj>ZJb5ZtEglLj%Cf+(_wYw#~Mk>A_X4Vs&n0 zgY295)VZa*HQIqngUDjwyoL?B+4kIsKx!magNUgo5jVn^Y=?#VVzw z%MW-ktNU8qZ#O*s;Edn7-@9sHM$&kPue=#@=9r-tB{H2Ktp*xz$~ZI}=%l+X&noUI z`PzJ4x_*uygfFn}O=~0}wAyyuJjP}lvKr{VK?9uTbv6n%qjK5k)m+l4xq_~$+Dgzf zEL5B3?Y0d!$M`fC@okO)QeCR-2HbO~*fiI-k-51lUYn+7K(}(GgWJ8oA!ff6r76z9 z(8q#pL@i@S-bEDnq~ho<<7gD9=3?Q1xHakl2|y(KN_#}S>I?2eL|&GDmrvUA?3)xc zYYB~DC4uO!Z(v|Vtz2bnUBRTuNcd>&2L9c^zZ-a52?^^|{2q44IvTKc$L0a(I_w(T&_^iTavE%;N;_{c5O4TaY3amH zWilo+l{4kJQUC!p?YHB6*fWeTZ(u?lc=QJDwLh1xc!N?YmDluLp@tv9SxLb(-CkoV z^SL-+>J$MaR^hsCQx!2qlT3Hl*TX=kB5SU|wkVaBRQF6(y#$fS0AVy^~6?`e+ zH0j%Kw*7AsrZ9=}o=ks~;G!UnQu2TT7!(G&uDgQKwsX)}cbnvvQMU4vR7++GpLQje zKA$q0s7wwhtWyEJv%3kIqqW#{7Mq`ylP!nJLL>$h|fn#uMJ3$EmXQVB#ZeZhijx;7C6lPjnL`6Sud z5CQ%s>;qTmNyK>V3tn1bcQ|@d5M+&;0S2lDIjf+)>}mG;FQ#;I%|`Y%`b;xKN;3*g zQ#ZT24%YCWkH&m5)RUY#3DDORvE5;e|DE&WDFvMT}HoarwbSJ&*$go+LKnIe`- zx8Z2yNac&l;v1?3TeoxWGejwyD8-ME1*bJ9Cpkt@Z~@inDW!8{+sn(>*^J6bZNPD5 z$OM`2*O!cSBEghki<>1oKs_tmo>zaPe!`u-oScARaQlK&Ul~}iIc0|wjJ6&++RddehQ0F9O85l1D74(S3KuLggNvqJGkwW9h=784JU}Mju^{vrB<2}1iJ2U;NlC9baTCwEk}SE&xeBUF2jDxkzWGRksGgwggp z@@JFwn&MnU2n}vj*Me_EX>`Y2JVGxhdfQwDp&=x=lES`L_M}5U9qC>}`oLW;boMRx z88R&gkW#XLH{od}x}-UVqS?XLRPmuXkjwDOay7Of1bCgo7xY6i-H#Q>XhCYyrG1Tc zr^~nj&5E{yGXC-fNlF|!x0|VY!s_M!A>#DE8~Aqv|8C&l4g9--|C9`)g?tMDmbeu7s4Tn;kx+uj~F= z$~2KXP2aZtEhw}^`}Dtd<*x(#@49hY0Vu|>=k)LYKL7N?zi;^e^bI;XK634Q`3N@c zl=%BI9v|l8@AddQUC$>{dj6FCL3y&zAB4cS#c5S~+wZrwmY(lV)Pg$4LUUy;q1HaKbHTH0Hgz?OqTe&$LZ*MJffWaKF~c5^!Y>QLw}9?u6S49 z?^5mFZ8MbAlLak)$^N!W7?1tBoj;~jXC52-ef$4Iurb0`QwFqu)*-xfL%eD1HzAWu zzDfB%G5a&tQJ$LJ;*3swX8p(K|J&E_e(U|x`6NQy?d7%jZ+O?BEhpHLNkRnnMQT$~ zyg5H8y(MXKWXY$EbVhTNt&#rOVSTh0FqH*ALoHCV;oBqD7YNdAu4)H0&^eoIG|bW(^W ze0uG*^p=p)czt5SU~o;`TbPE!w?-vB|0RO zF!~6Q*53=G{G{M$}WMyHs3WUW9w17#PV5eXOSNTMLz2Q$bJGCV>+ zc4?)KV&n2*5(c&&Bdll%1R_3_#FL9?gC%_689uuJcoUJii~YHb82kMr*$ z;Gt*-$ay#~A~7zPGtBZf^GSZeRih07J3!jOPWdEwX;LlvBvwpnZjqzv(lZ*I7_wZD z)9JpK3qJx$`&5nZ&M2u}oe?Ola;aj&PT+m)lHEMxidhjPUMx^AWFYwk0*zSFz-;5gl! zOR}aY>?s7;a!k1bt4f?08;KvzaFN(-2$Sv6{N5N(+q4^n#J`&FFVQv>c-T;XJM3l; zY~L*^QblMjjg2pcq+K`e>+qDUWO6pg1j)z+E1Kp2SzOtt8c`QI4@60R{cn2Y+I^i4 zud}~jEL7?##1&1@B1qum^{v}rE!(vwFN4&gz$zmYyPA(c-@ccl`RX^BGJ=kxJw>p` zUcK$)-jrXVT-uiV&27(4X3Rv{Z7=)Q6!w-bnq^nk5VV|*Uj4FfmtBZeWPY)Q3qPAoQB>gmR_R^dm(p_&GsQHQ=NUW_k2|cCI!8!roHa5 zR*ih`u^mMaZ{I`NeDf=!U-jw>%G2~_)v3VCFPgIvRa^EHWdqB6U4UZK7{!A zuyr2!Ak34uUbnaBLP#J;<_{IwlL1@Wcoy2jWV-I~7?N%$SOn@{HtgM|n`77gtBm!} zl6DPSRicJc_I-K*w%py^D<=@0uQ9W_UVcG<#&yOt^zsD(P>?BvZh?cqzF)v8SvUeG zUP1s5bsu&*v&Hxt0>yxOAfVG0a&wV;}M2Gx6toCHt0b<-10)xmE3I;}1 z$=SUH61ZQauJ=KKNW~r!;le=&&~R^t4oHqV*zg)9o!Ct!7KaVU9Pu zh#kB+66za;vq-MxM#uQw_Syu6Ysm%0(!Ku+>Z@-H7?fLp6VW%ZclaEyR}C^Wqo~RQ zxeFXB5jZT~)?t+V{BqNP2A|yp6yER;+j0;YJYtFeK^}~6$M#cDF?6s}vJN^J^|eF_qcnLPUi3{v6!?U2 z8FkNfyqaNxSh3+vAn{FcK2XF$$#p2=4k5k>Qi9(`s1m2Y8TrOCsDTB(Ku9=Q9?uGb zF5#g)p(cR`DeMRl$KklV&KWiMZx6mS3^uGCDc;&ylzb~n-uRnB^x@r*h0h}@iNvsg z`99SsMJRWpD%l2ig0h3O=#4o@_~%^h|BF~sTeLBs-xOYx1i9Rvxj%ONBD3TYKhaP!UZJq|^PEs)VR5BZbH( zRd@!whxrTaV7)u=gU@+hbfhv9qNHtFs162_(3E&e?;Gx8j*vc>o9eu&A4Nh8xJX_& z(ja^&qX>;+35#Vqtx!(ZliX(14>cTt0UB6=X(`Y3vxdPMHrr76|B7y^A@dYzFA1(f z0sufYl82>WrQ*e>K4X3C=bU1Bb~h-YEh;b&2-2-UW(3JBxQ~n=*{~N72Qd(XbQBc2 z8wE1d-L1ojtoD5dP?CJ&nok|eCd9hO;wU(JQ^<|Sm~X0z&bCaqr0S>>jey<~dsX{U zwiU^@0spsoEZYguZfP(ktEd}Nki5LpOagT;N)VGMaTyxqsq5^nwCrq{WL_n;rry-V zXwK$2Qjp+{13s#_*-oF~gwtmSI-$H0XlufvAY!tkNiG6fA&b+sU<~G*(T~!m=u+IY zIXVa>(r88A!t(r$l7Js6TgDRo8cHiG-bH=cz9o~s&0y*mZBCNTpT8T2Xx~492H9e^ z)Q_Dop}To}?FP4EB|#;ck>s^MTynf`4JnHx3Eg}DJ-~&kdOUDoz9@%T(1Aj659k;L z_Dc0xO4M*^h^5pCEL^gTd<9XKmGqQTEOhUcMdFd8%>55a;ikptyONV0Kr?D29PJ46zn%B@97WO+>~oebDPA1V?V7( zX$JbzwlBOMb6U~sRb#73TcQ|Ari4>yj3V&anz=M44|E^YT4P3b&VuG(kP%X~WvKX| z=xW#Q*(wu{Z2Xty#-zhs9-6A;ssoa}FKJErwo+|{jzEc~wVxTo`*V+wmWRAPOQLgVrcw=mKu7wYETJ1HQ&b3tvJz- zFr0lyq&{VmK$J^|MNyL2Wd5(42Dy@~QZ9gTUdlDg(o1<}cR<8e>1-MFG1cSCXnqBr zw%0<5e{4XEOjGZ#QsCC*IWa!G3*X!!cU3%pY3>sMKTD%89Q&oW6H>PoC>besAD6~H zzodtGBRAn{6Rod`y_=MNDSv+H<;|Pv%08B2E*Oby(3cj4FI=4A5*fO%%qoG3Tsic# zZ0k-(k;>n|@Vb@@S1J(#?|OCikh-!gra=o=Xj~jbQ*O~iIek!Js7sVppB{o3bqqui zZAsQX;LJs>YSJbXN>sj;WLasy(0Ny-oylf%sssT=C@qR{CkyD>11$v?X+Sk;qyFI& zBcnuR@@*RYi78akhResu6|OS20jYT+jMzw_i?up;RXanXTSut#oDS3|4i@EWG#!`C zG z$buf0ot44yMEIPD{pLp(npBqQf$b?E&KsK$##D{4M}E;|4AHz5NgPv}4MQgGPHf#H zP(Y>nWcl_Xm3wlxssgQyjVWT;zz6P3hxu}i6Ss#fV!0KCty73tMs;Zs%Q37AiC7L- z$`iGWqK5;5WwBrMa?>5v7Jy<7t6IgUFUn&3Q}Jf9;bNdbWfa({InTCbi>-a*Fbp`S ztrQT>Yk1Qp6ID@9kxorXL6*xmI=9rPto7X+eX>2sknL>Bg?z(G>MVOvlQQ0YqB8Mm zUn*yq3q2{{zJUzooQi|Y5YEd& zD3o(vmO@22(UQ=V*kTz?38^q!Rbt2HP?Zq73pN{$&OT8{mIYkdjA(huiYYxQLn(Yd z`7NADnvaE^WO}HyY{=)Bc2UPcwR)$mi?~J}ZFrI_>+Nw=7}|=|zYL|ZeJ0;_N2yBL zcmwhueD6{`BA(U;8?)gk;q9fC!y2E_WrAG3uB%+WxD$1A{o=3OBv$p)GNPh#22=uQ zn>k$4P$-DsavK5G7Xt$>QRNz#`JyrpQMHd^2(%aXeR}=g1%hPXA%S=q6dibd(Uc^E z(!kfAGZjkM?+}K96s2O7g@b#5)G$QG?sj~IW(tyIENQnmJkNL5XMBwM+a z3QSRYm3nZI(p%L7in%pSfJlUt?P7OVmy%RS!QLv8r}j_ld#3asX}APgY>)-8V<$H1 z#{5g7*(j$J^ZRrb%_bOKL;v#tfFI>c1i)b8B@}MuNP)Y;8qvI~q(Rwasn|myjBVyum0fyN zD9w^I&hyhp6Pyx{@uJ=~Gf`Bgm?NlyFdDt}<{9h0J2opMpzXa@V7XZpA_`wm;L`htG=>kkM+x7;DlxBR^IjjECBTgmu{f*y5E(|c*dibY z6Zqvz4kicWEGtMt|3K48K2a2OV*N3OUXYzIEnoHPz?3b&Uu*%VWMAz6rTDXpeqe}` zhJ?u(REeV0Z&~ruWs-*$Br@9^P?NG->{!c88w~(UBNfmF=B(Jugi1J(7mI^~P~-{@ z1?xuzmbjdtj2s}Sf!&b-*+|gBh*J_V1VpEzAStLK8PK3ukR~9CBbgoam}n5!@ujfm0E$C;5@)vpJQ3^xWwAJ(0gn?r ztS=NLD}O5Il7bLV;0g)?;T{4+Qp7x?E%_BjDe#CS2EryUBLxgEGg}j4DM$m65NbW4 zELoj!#RtzH^+B?G9l-Zq@(W4VG(XVT;iS~`>l~$qnV97ZP-LGzrN!@_D0(pU>l@L| z|51S9{nU9V^>kX=y|$2&ptTb#dFtlzIH`lpDN-guZ@Uew5Zq!XF9cT@B3&AmNC%!u z(cEoc8u$v#l!S#MvWxjFjdK6A=T?;02+3(gSJ-v^-bh9AzNm@}4kEsi*&1)u_fik$+yeDd)ZpF6I+n~fp-zsDjkZ+3s znk{xmh^f#zpwVjomP!YuGg-9k+B`*01y%0Ju}QVD5RJmqHRA2Qa(aEZY|7_j5?~%T^^(-j~ z@(rqzLx$S#yRysk3Af78ZcE=sI)f6nFIq~XLj^r|XTT8WyE(6cq^Lg{UOAx~xixVV z<$zxjD${UVa?)*AiK1|XO81p)yHu>gny7+dzJ{;Ys$}JpOR3B3zr7UoTaDVaB~HS$ z@QUi)vUSQERd4A_S?W@{vdql5@f9^%NTZSrlA^Z^O{ykWBb6~r!3L@o**8_KB0WQqG!A`g zQnD)E?JNW>qLj5eG<>gP)$lW(xvJq;pyVhwb;&8$lJDC8Z<_nU9^J9A+Fwbjg4JQP zir?Gr-CaRdgO^gMBvs2kg{8V*4ku{h^%dHd$)9r*NkRe< z4oi@QY*NLD?JjhABzFzmfAqqkRF=9o6Bq?oDnx9pd4*Bff+{U$ zk0b$s^|m3qRMR5YNdRS}Knn!XEBW|qeXIRasb6t)Ns7|y6uaplx@nn<7p9_0+wyS} z=ZET6Yrc@(JRJ8I0F*#$zY{|WmiBHf9#daIQxdV59)(n(!+KD-LbOqdU(HP=z95MJ zBL&+uM2KC|*0DaLc*_xNV8IF4ZWFH-1Iw-%Sjqm9k4v*s9X2(Y6>!^TbYv@KBF1vK zWx1=G=~d%3m1qmgTc)fKc*d<=HDL)8NW>EL35k^>)AvXM+-E!>bT3L0*Av^OD8p*8IqgPqm@bVWlbbz>01T8Yf92|5MGA3 zZS6AWSRcm)R{Vx4kj?7sP`OV=WZhp4jO$K-!ER$&Wm6AmEKxog5}UDFWKmd^$r7Qd zie||cFcn!QwH3j&566M27m;oTP095NhLS~D|9dh{|Mr5@z7#v=Kz-BD+PED#;=-Dc z8q~5a6uSx4JPg{w`EIzyI+UJyC2*gDT+5Qu#C_YLq|Dqf@mwj+hK~ zgwtLQ2{_+o>R<5r=57twWM1drR>%zr)V^%ivi*Vp zzSLpN%3O@0f+pN7=k9&1_cxy>^&WFnRu7t0BXywwd!2{-GY5cUJiAq94S!QdTxkuf zdQ%V$tqrMo?Hi>BH-NXAl$V}#`qJW$4kFY(A%$@#OuoFf%B#2lbZClOX}8IBSBP%l zFH0$Cc;IC$v#S*_QzmRXGNxN&WcL<&NHeG;Owpd$*|r zT~ud#&j=EqiZE<8;w!JM-^QRu&`2gN3;WP+Y0$AvK(%qUy4Cklb8^C#dpe^pTSMzw z@kJnT&#}9=$7{b^GLyV#bo=}fQ{O%HJ0_xQ^aA)q) zVQ|!tGj`@HC0kXI4+1D5XEZ<4<*{ePRe4$T|IZ&olYf~f4>lZV%=7ZWRZL*=CfyW$ z>ir8FyuWjIVlu&Vd2CBbCG2t;w^xcC1l+h! z-4dF8v&~bs@FoF!hc7PZ&H4uV!oQ&0t9z|BUOj(IJuTMpSu0*5jju^a*aNtQl( zGrUCsPT?ju0s_(a7*5j$(=i22y?ZaKj^w4OSt9`48zHlTwqHK#&5@>w2$c_5>djUs zQAF9a3Ou?YIWO|JOx3olKS3B5k({E8`@n4LPl6`fXC%sq_Lbsro-ZHYR4C<~@m0)G&(MJk=YokS|9)3`+{ZKzDpN{KPjBS~R)gt@4tvZ?l7G`VcB zHxW#ieN|EKb{oP{soqyg4Ji=qh~^Oqb2`c=K|wdO0~Q=`j!P<>dy)v;Q;}ccbENOT zhGzSJ5(fZ@lf7pMbxO(|l_xz8X=l@GL2-V<= z16xRT-%Eg@B3m2mL0A*IQ{meRNmU3PKo(#5y7!;Jczh*3Uk305;O46^GwDG4Pmvm_ zWCGg|#)i=3_9F@sHy-YOMtB**$q+h*KrsY{ArOp@urI*85afo2AvC-Fh>$Fxzyhv7 zhuMz^pF*e<0;3QHg|^cNT%Y}nV0?t$Bg`J5^~h4C-SiPAkDz!2!6WD$LGB1?M*us* z)d6u&I6C9T*b`cgKysq!QUK`*0VnDh8J|w*HUhPotc%b<_A@}%2);*9$$~`Oj|hhb z95}(w2yF(C8L(yUsydeu-i+=BvKy#wAi4qNmW&F+Rsa-(-N1HWKY;VXhG0jez}vaL zMl1uV41_X}$pVlJxEiqQ;G5bVV0s0d7Fh)BLp=d7}G5 zqrKnH?*}_3!pZZK2PY3s9@IFgPd04F-QyhnjFZYaN?1z1(jeoM4ky2C@@pObTpS%l z(t|Eak=E3BNK9|#?V8;oqN*Qxu=+u715ebvXIO7!SvtLL-0J*~*lUf(Z9*4-$`G!q zW|NLII?`yp(R%adTAozzF3&&X(Pm~J^?a{yU+nND)moq=eu^ghdi8M7*0}tldEd0- zue_O<>%qGQ?;7-ecpCRhy7cr!I-u?`DhlHrOwl;oPM!4OI`8NJtc;5b6=)2uG6a*L z(=?c=!7@~u8CJ=8eva2zKa>8;wf2ZqwFu z5#FxADqV0l+M+#ZX16?OYS7e&qcVI{hL6hdQ5ile<8e%;ugeX%8_E4h?niP*l6w-y z_2j-JcjS}Y6^_U7@fbcH!^dMd(RcWCjPDJmCbxS)dFbi}ryE>uaJj+d1{V!}*sQ~w zba;~vZ&JsO_Tf!Byf=qaiU-lfeBp+&8_I4dyP+IW^lm772nxr1gx4#=sgsgPRG=Fy zZm_t)5<@6%u{2-ZU~z-xpmc^CnyYuu-Wr^}Vfr#mUxw++@L3$l41M@44m4`;mSd@I`hBVwWc4vhZpbj|#kZuU9KD!R#x6{Z7}p{`xw^VA~z9Fj8I&2q_?2 zD#&K3nPkxzd@cV>!r;Zp*Z)A}f|36p(aVq_{u9*l?`v(SDnydst*{8m14z%;n!)A& z1b6x08zEfU4*b_PGpOk_a@~L5EB{M3M8xm^;n&=+tMG;Y);;uhM*4qYH2+OENS-ne z;c9N%&24*oXz!t-6oeyj+iq?s|99TN zcFOnv|My?Tw#uB+{|pm3CW|-I`{!t)ZlR&oH~-rI6ujuO;%Z8}G*(YvwJ`*U`$IkrX?HGl8tG}#^ZcbpKE>yKGjoK&E{*kD_xy9u!E;qQ`;Bte@4K6phXwWW; zYRj9mjeE7rqI#QDZU=DUZX?Gr5|DB8UmUf28gt0u?EXkWC{oLol>6`a??CtjWf4J!IKX{)v?{ho2 z9lFBR4c-F28yw9I47WJ!(HL%UxWVZLryHDZa27W(-J-#9*yqjr+(`<$y20fJS8@Z( zEiQL)xxwWImm6GeaJj)ngWl)O`@DIdH}CW2ecrs!oA-J1K5wzloA-J1K6lcMu5K{7 z!Q=*$Cy{;Lyw9Db)SLHt^FD9isB*zRcWzS;F#q1X&ztvo^FD9h=gs@Pd7nGUq6a{K zZ{Fw4`@DT`K-l62iyJI%u*4)*w^-f9>ISPDtZuNn!AgU__k4Jnr~LuLOllrS!v7fl z^1pwBCVBu~_D?oD-`M~0hW}9`dj8XX8^(7U&gj!M+pt*5zZ}*7fqv=y|7WcQAZ(O> zzDZQ)M{5)}`Ol6@^lHz^^|soI{Kz>mehXEA9sd_h9-?-CGqj3|{Vy3$Xg0g^?-7gm z{+u@v+JBf95#RqwqT}7MT)bstD^Fj?-zW6H^h&S=4B7ai0=C&QU9)HuEM_A(9RwK7io+SN;fV~FiK+p8K~I9XSj#4i0>s7=|IgB5!a2Yht< zZ=Ie_%&~*F3_(uUMx>mF&M^tgj_xr}m3gYn&!@I;$e-Ex#?%nzU+v$B8p|$EFAxOm z#RPBNeUD~3xW6VLf%~=7?ScOQFxVm{oD^~D87t@Ho`t(LSA@K`_2xqY$3ZL9 z^6KGqkNck84a)7veDGHD*9FPa>C2b|rt_>Z9koFw0CY<6FR+@2g^(bzR)4)#GuD{o z*8lqRYtIXU!MlyhWRob*P8|1-l)$$B+GMB=-IK_+vr%N^_Yw{7SC!{-u$mw(HcV8p z6LNTeRG*nj*Lt#VVV?aF`&Q|uMnz!7vZs$;btc*fK=OG4KMo8zeOt$Pv+u2bU2GrL z@&l}Fd_g!v^haX!v_iU9vqA^h%X_uw594YS|Jc{;z{E+Bv)*Dt%X<=M{qecIC#0LG zT&P`tU4M;qQXznnattp5)8?6EZkrgUW_oT#f;F^PN8CI-JKdCNX_g7ifA1b0DHUaQ z%-XVewU}YSt#Fo-$XjOzjt5nm*uDfsuwS>B8oDBQ}J{$-h>uvR~GD4THxdHsW7n(WQwca83>f6m(sSKGKr z@oR;rsqJDRkWVQ%u`k0DQoUgb$0ftsAhKYP5`Z}0%QxwwIonEyey;4DpYA#=*1r&n^_IunJPzM0a@NNniNgouaT*~`83J@6fdi8 zokIGF^M0AurgMPMx1PiWlFEItjLGB978SDT?={Ry9A=iYe~;CoQTRyq%H)t!9HO4- zrWpFLtQPXg1-&UY9Z#G`LJHbwz;QkybcJYN(1)jRJ8AJc9Ho*>G>G$!lMN-bkTrlB z*5~3|^X;?>kt%zGRiwvCS3bmt1gHJG5BJyE3v-y-tq4CtiMRm16{up|Tp8v4?6CE3 zqIXsXt=lDwacg7HuD9A4XNxI^RDT-N+-$&{VSsO@>Il`1p|;bHPvfP529k|R-FazG zcYFNGuIa^-omp{}quJlryI~-Re#8x|iqqzn3UV?7WqaHH%GX}3#Cm=rWn#bv9Uq=u z5r@iW6YP`6YyX9}7ncJ-)~+}1zW3$bIB!80S%wgq;QH*DObKC(>^f&dME?zKDk_9xV zdfhX)e43`YOAY#6ExF(P?DQDU8Gq+{>*c#d(H`k#dUlzfUhN#U>b-9?b(m(*?kf(; zQ65DOo~-yX<$+7OE6Kz*rN^5xbR}i~R?eGv=s^K$FuC@c&kq=y%gijKTr)D~Bq5Kw zW^Jq#1ok&&*4dlSkoso6ksjm89#8hhZx)PmKAY!mF_%}LQEiGj;V>s0fVG|-RbYkY zdh;9L@4Jc8Z~8z0*fav(bs3~_!UpHp)tB|u(x28NK9(ITw5I+uqdGk2AtY#?yQMDU zmNhqgdM7j!W0C5>wGi>U9_OBd+;y|H3Hv&|sq|d#FrO@v&tswSr?!W_Z`f5YcVl$7 zUoabAAYV*q^BfOcVU3YUmt4g;+0&|+Tco3H`dv@g*owVbkNw74pjZMu@0K2&zt;2V zqV94`pH#c`z1`)5kM5yF(qfKg#qzz17;MON)syqQr!%iJJ=V22(v!&4~ z^Vi{8Vmya&5sRm$?d=}S%^j@R4wGYW{^n!*Ls)Qi2Ll^)ez@O{&OKNrvXaL$V)VmT zW_x-=2$VedkVW}x!H*{VXrji=udRt#61-orPteD?eoekL+0S~+a6c!;`p!d0v>c9- z=QU11vMxGfAl!9M2=c@dNE_TUwKD_|*%qP#WARly*!dK9g{Yz|P~)2ER45n9O{`oM=Lg(=>67s`8iDWG;U`8wR#yytsSaH}ze? zFmSiO+0rYz@+=(8!IfMD6ZhCyQck2(Gt0;caYBq;;z<(8S@n>Bn+?skb(jeW9l?&9M2D_IsSs|Cvr^xwTvbAJ5oB}rkyusOtq z(jF4S;h;n%=fDIBmV|+o&pfQOiK^(&^QNqz;#xG01!8wSM2vp}Ksq zA_0^m@M**E@gXev)%$qInXtw#X-ru2gq1@oyC+L^R2(=xJ4%ZfSMh1Pi2B6V+rB;h z8sXbjT}&IxIAT`#)W+F#-AAe5pFqP6Ei>$Ww#^i_!fy|5O-ELwr#hC`5w$EUAqyJh6RFdemCv6;3GlJBdN5`QwSj~&e-G(RsvdP{eS9E4L7TMWok`&k}L!<)01`L(_gd#m?dL9q_m zz>8PcH*hsz0FHRQvUj>wy2}Yj2JNsNK8$^j9J(u_ArF^9rdv-_gPCuQ8@yizA`KP> z(r@ICr-pgVw|$hCk@eFfY;Hk#B?=VmgP%K2Bza5%K_^xC*p9OWtHp#H7|8WJee9FR zmL9}3on3!#QpB1NXUxA`9z!kXvmAW(5L^QtXO&GW>S9{Iget0IR5R5|IR|8O7u>FP zyXX2KF-!{53IPk$p&aY1SN1rwWe6vo&Ggu`#Va$}(WTy;&kirK)U--;mM!MC~EyiMQPR1ptF_!)bZ## zQ~#;cqi4qI^2ak@?Yf_0`PX%#ziketyct{EWlb6#bG0hG$WAM}=WN3Hr#bubXqR{N zI^;w`xWt!G(^BX{aQ_KD-89!sKiDc_osZdaY`21o=S#`q`)Tvk5~4Nz!w%lUwGY}4 z-JM6`gdwTET1;9$^B1A7GaY*nO{`a!Eok-PmwCcR_;(yDlf9uZd@jq}p)@BS2ZwN6 z_m)PQ?`Iz)`qB%@#cKr4okN=OEkpBk$(J)*a)mF zG>j$F!-|F$%eK6I7e{w_L^}V|QbBkrD+;siwpuc-b&0I;|0K3eYkBFpnCU?@(tP~wS<99@ zqUdXKSS4rJ-l2FX`#|SIw?0xfC6hj+;_fK;91C7e89miyzPrg(S~ys?O|HXXB(ABO z)iPPR$z*YiT;(05`Qs4;5Xk_~oupNU-x7Ba>ub_0r9IgeporOO2`(7x^*ZB~!&y&k z<6onspK4g8_c)KD`|~}0s%OcYDnqIcvZIupiDy+cp2#QTsdS{dgaqd(zLZX2>U1D# zxUYW@A4eqOa!eNG($@ir?c{+$(IY^YoWSOuCI>1)DV$xNoJrgc>uM1nW4TmXa0;P~ znA3-QvoW%M{qv0fIyr%?_%CmAW?pufFgnZwocjj8oA>vhNM*xxR1n|zt%om(<&;e= zfinFHmK^Pu`MkH^V?)Jq)7?$8<~??{Ri4fajq5dU$&nDlJZD-%aafq(5hno1kEaIR z9rS4U#U0x+pg2D|kM^0q4tq-d1jGF|#^l@AGuE#$vPRUm{=~xVHM+&`vlkqa{BWQX z=HO_!`!?oeG8fa^e-YBdQ-jSh`>Q_`z{TYh$e!xTIGbQHl6gjg%9@b9d|om-rrdH9 ztme2|4LovorZ?PV!+h3R(ZvW{%au)I<75K4Ii_LJ!!;khV&AVvN3Z|NA^b1hfF_u% zV)=E;NyiqOh!EEoOh*cVclrBqY=5%nM98BG$z4IlvUoJ=smt1~2hYB|>95D4r3luy zzJ>geEG0ZPWHKd1aV0?^vW0FjBFqAL-PT3Mo)6VNEqaX3HBlTy@hT2`R@DAtbGi`Iq2!2r^AOLpS2pNqn~U;=J0{YbO|-e z-o<2Q%_H++U$QVVPbY}K$?)9=c99~^!@WA|ZCCiUHPA8NlQJ36j5Riku$w_{l=<)( zZf4F9=TAFf{Y{nmfo(G1$DP0!#=G7Nvl#MXnClUma>oM(-wN%cQLa|F$4bvXpc=%t;baJ)#8AJ`WW8Tf2?o~ z7C+o?AA-x{Nx{6_)oQkyyrE#ghH#BB)bVRKpmB*YM}$T_Bx2$uroMCY8+`^Gb_j!0 zj4H_nFuAZ}6K4ps%Vuub{4AT7WdqVbCIJ5BMsmFIFt5*zA>%UPIZ{IuN%Y`w1mF}q zB;(Yx<#7X^slaf=Jr+68BdMmd+d;S>ZXx5SeDYJGtv-Qc6!q2%qA8kR{zkG>CsR7s z=I)`xkaL8Y8eB#{rEj4B$5)>K<u|(83pdjh8&xcz^5Pm()IBm!yKG$Br0hvKk$2bhIr` z`Z&{sd;a)XJDu&3X0hIfm@nx?XB6EtI9M{Y;N7ofbAfJX-(Fw+f05sq@4w}ry>5z9 z>iw)qfXB$HTSFjjUjy7LQNby0IYo#{hdbKK&$tF-9t`VaVCiwXZ40Lpww+j`ALiMT zgLWrv+Iuteeed*i&Fe_5PCr38w!ZimWCj!}c37-mB*e`A>zN-}$SjY;9M`#hK05CW zxvu+HH@_IbqVvl;70ofjT*~)`ty+`ws zN`IX=Sk?nfAhNGci}S~4hsI@ysj|mkff3n8|LbY=#8bvN@pSAbIwJG?!gt`GX`IK&Q|)pAJKV3ad<(XQmW(=^cuUufp6aS$P&ZpSuhH^?^tDbBlBIU$(oa z*P^ELu8#2<5BHtYmB01)IZ0==LG)*ytTPyKNFBnDdTrk5L_Ff-tlwP!0SG{3JvoB{ ztAf?Q`mM)X;$Y&BgNv}K7i-Hw2S{^qC=dNZ_WlquIlkfQB01_eZh}~EI`e8jFy@?- zGV$04c6%wJP$M~}i%@rejT|dL(Bk!8Xt_)uc{a^Xp6gA1+IWl-Uug3jHRh5Om!Pa>Nk~IFx3%(IQA+gGr2*h zT%%#UkDp#}4%3HdH1kj|GDl+kiPh4p*j)B7&bx9Q!I=HcAcF7yW5uC_#E z=a#0hMPyicU|2A?6}+?f&VEFY#h})s)7ev%%WLkr{hS=OY&hYM9Y<$6+crn^TsC@A zcii*(uHRchPT1AMToas_NkC7}*0{6%qJKdvMz?;*9?b1Jd&+BS3RkM%gVA@rk3@1R z$_I7&p)pr$%+(rm1Ozzq`$QPxWy3|a9OG22UT25?R%_Hhxy}`#26o(Ej%~WUL&_lo zAV*YZjYM9v_N3D=QgbMlXSRAqoiL4{QJvfv^m$IJUY|T@nc-9` zcF9MsdN1a63x(R3%5(69Eh%>Q zF|X|`TcfLOJ>EO-fq(k-4XB?@xFZbcBFT2&((`+v(+to2b)*Xx?9Lhc0p9UBb6;$e zD0j#A-&PjAr}|%3!zJnZJWH?81ENIw@_O`j+Fs-l5)}5X?)&H7kUm36*b1MCHEYirBOE+uJz}6z0U$>3>y~OcjjM>dco%7 z06ZpWjQj6U*9MRrpGeY=Mkezyn{TVM!I+WW1yFJ*a6_${~VgRviCMx^fGZ=CLJ@srhc{GOSJl5d%WJO`JilM&b{uP z?=@t^%=&tq(JQ`oxzOveU3sRTg`5U{zw+NZ^0$9T7taQ{DeVeK9C{S z2_$=3ZwgusfJ5oGv5aRu)r)(MFmX0_dE1TYh7-&3@fBl%F~?;KW1V`1w6m{k`qgC3 z-(?~?<7vSrVvj+8 z1lJk4@eH<=cSrRdgCxD+G}_wlTY1?7YAG|zt-tTZnQ6)MIxE$VZ3w$i08;$(L}Anj zgu=wEwE1Y~u(?P%{_BJ_CZ}J&_ZP_^;~TLH{mfnJ!FjVKI7e+Z9szHDg*l=4|Dg>d z2}KdyfFn%TZ|X-DNV18^`jpyusC)LgVWtRzeWJhiKYJ~D9LMA%*UK%fqWOn6>c^C! z_{Z`dK4$*YnfcCSj>~V!41Ao4!#H5Khani3O*Q=+b%g=!o=$|>(oBqyScBwT38KV+ zxo~wT(X+uY$JsxF4jLHroVpP?H5t00oSD#gf=MMOafSs3CH$NO$68^b z$6Fg^wRB4IgCIHJZ=SQb|JhGrbvQfpXttoG{+UruMtOs6WXIl*e5|s@+F(&!wR_Q> zx$al5pEg=gi_8Viv7pAvAkRb3F|9ms#}hL1I;3Ysf^s<=C^>Z} zJFo|fjGaZy$BI2&9>x@E&&RYp6H`_MQCo@BX>}-XHcSWhnQ*kk5f5d}!s9>#CH*fF z3)eF&!C-R%SR>V?ThQu;eYk}xpO0z*(`!t!NVEOKn})PA+YK6HxCzGPO%tf1b$lPW zFJX=8r-91?LQS{w1O^o2cA!K=3HR?Em7eSmyzCLJ#$41qOuP!~RwUZZMlgFkwnDMJ z=<0`2@lgLIf@>mbc&K_9TLRo!$Mxs8THL1+1eZj+2SeiuBY6yGPRN z-@9C8dhyok2A-{!R2W@exRJ$*zCMQEoEp8*FU;c{WmBlsh|W-cZKINn2*oiGs# z>_i-c0CD;pR@=OpJ64NNRCdbkVfSz6G(^7Bih)^sU+OH?bf%tUtXl3rF)18gUFocYUBiS?5hnxmIlUhF#~v7qT7M3A!qSJ4 zrHm8Q-op_cssXz-hwbao(w4tbz_BwXs_SK@5wse!}s^&Y(tZ%BTiPFjPuRs=jpPL zK>hdhN?cYquItG6*Rf3FF}ZAR8`dvzr^m6M_4Ps`cHTq6R&eHeGK|k;42!v1r{^pG zeM-Q}w=@_EqW5Gy7*-L$BRJ6{i+)Tf{`BGxo+xPEnm zoAZ0S@nsO~vtvSmRaa6vdt*yLTPo-8AM293X?nHiEB;@uJH!KBwBG-J4fu5FH=#!C zxY2hX#6EigoeGLJ8*VZm8p#*o63gEH`8`W$x_7jXg^PR!kA@pzFgTrwKn&KIeZa&z zpJh41{5FiI5#FYbuVqHHv&a$&M85YVrFrZzpE%`rP=uJXaoJb1!|cgHZSpOsu%+Ga zNe{dO64dNwi16vsPoKPpaBDu2ixvi}^3Xue%zDv{15)PQ+!;z?HOj$YsqmAE+K&^^ zgRhQ@&dW>5en6Rw6*{pyl{x)9kS%}q?-fi_`|G}(o|Fn#Zw*Up_{1OCme#_M=-mTr z(Z%r~rTDCr^-JrCr55hnnT`)i=`)zdf^Vs}7#~Z59`_7SqLCIndftvk)K~BO#$>nh zo`B+cF7(P7+UT?q)$~YwVfxy!7&i~UJ_Iu)NsE1{+J2F(i~b;@?aV`G2(IVX8C$2r z{VedOKR38@%vS*O_)J|bao z+`enE=W%w|U={N92*#RL`G1&^$61|mPFr&62C@*B9kRxc2nF%^&v561`7xH(Bg3Ft zjW!z1BYVYYLkzs-BYwE-+-CTbq1o`Lr#!+|eSRC~%)vSKza%4rIlN^CJ=i_@>8!hp z(>7gi`(feZ`rh$Plsti6puRRo6*TMghQ+;Jd*_2@>kS+Cn5atI{Nav5)r**k2nopk zuHN5)ayQHiRj8z$ADxK6KuryIC448#w-bc}EfA#);RMqJ)A z?5tVKy}SgM2Yh*J6%48&B;+e1W{vq8o&n9I?Rm(N7{t}j%UfL*JL5SVuFl7l?P)=e&H zfMOWIMd@)!Vj(MF04bltfU?-W5vm5#?3|W&p1xMwp$$d%?7k(#X|2*O3dMzcy!pX* zy0xwLR4}gye1lz+hYyffL?Xu<4K!*LWn0|;?B{~6j5^$qU|(FuSe^cFSz7(Q(n9#Yob)onKe?!xvLrlN*iWV=&0<*z{w8s2(Xk zWpmaP;T3SrC+2F68OV?j^f#Awc&wK(B8>xu7Uz9)BbOA6A`WAdyz=8^X1i!!%>@h6 z7f>G%Y$pKvk=~I4{K(?ha^9~kVzElaPLwx?6nokGr8Tp`{xZa(pE*eCkrn^L7K6ak zy4%P%$8WC#^_^jK9E7}JhMUE%8%=}xS}>44DY4Qn|LX#X+9#{atmYN zIz+pLJfds{r&|HNf4BdpO7x9k42Ce$8Og@Hr^kaH4|+VBkJl(}lFc)ivT-&9E3m%f z^uc~r2Y9WUMu}~E8Xc_{tVL(zrwH@>H|>PGKc53~OCQVnRI?V*$k#Y5%}kqj^c>br8#1lj za7Wr$=`rz6Z4HUzzUNHMDz9031Zvk`yX3xU+1yy78*zOnYJU z+rpOpS*96@Y*39g9qMhnn5SKWHG$Fy&=Hi-ttN#0@~OU)Yej(UY29@SAl$>+LjM{b`*k?W@exN#S+$Te+D+hb z=)wLSoMelMZ;pSJ0M42pZs@hXr*JBJq&l~-_xs%VfU+Gby~>WSu955p@v4?X_EvDP zy(=_cUVF+D)NozQZ$SRE(_OU97L^iz`xoA8v>X{T*Lg=nUrH z)@30lGz6zz?8jJ|)?Tye^xN*Z{3Rj65hKs_MoR7yC{`%amzC~6*LLE`nibdOsaggJ zb8$vRjt8)GBTRe&T(&ROJA*ke(3t_OZmbdqGJvzmuI;A-PI>^woe8k_@^VJ3HalP1&7c_t03rgg0F6sHFnjnr?v=Z{3{i&dFtYokva>nS#w_Gk~o-c2pE1&tiiXXh*U5DgiE0Jh%F97@b zBMBiCTIHVm{N~^^-t1TOdfYuZ^YOsFIBy+y;0t+X%M!r~i+Ql$^x79UjA(Kmy(EEU z0U%%%pL@4dlSqf$BIK;>iZ?nHgm8&I>H=CH| z^k9eCwKJ+oU&h&euUT&k&eTPmE>(?#74zPhYDYJtdX>rKb@#kZ0uaRL@Bnv(! zzYafGc^s zF$JyMPc%k}6qt<=u#ZY6IV{ecTuV$vz9K+x-Q!Ru`~o0tU#6!T6{R^pwi8OzPlrEB z0rh};EF=ri^BI>!y%Bq(+#mRnh3w@u@Ap1>-W&gV^{d>Y$59Gw8Qs?3i=U4*$Eq<= zc6~g$%IbDcfs5e<1}~9bH=w3he->X4m4A^P1ZB)sPs>neVh0mCn7>cNJZ1t=qLv9_ zqm#~)V}y+I8G&Os#WTjodxFloxaAg-T~<9+xYL~KLdkANkcTf7qkl74yS7A=0FxEV5N`0gXD zDyA3L*EHF&7tbbMC!h$D6mud0I`tb6f0OMzy;EoNtDQiICIXtBF@0d|#f0Q~?UWBP zx)Is`QeNvd_nL#V_oXD?b!d`>Vq29{goj-2H!G42f))OdD;C3%3_sJF@7JoUpc zw;|!YDWV-OX4(v39eSa=60sjdkH_Kpvu#}SV43n=tnGT!d6}Hlfps(0^$u8Aht8{t zkE8XmFjX116W~rVY=*V329IxaJ zA0k8Ye#?)9k!jZ-BH9y-*XxuyDSzGgubIo9{_LIKlA}TG`$Ic;hAIuqe91B&!RGIY z$R3~(;`q+v$}#;uFxI3#k0LolzRhoxId|O|XT$6}oPFRhI9F062LJ%4@^vg#jMQ=s zV1^nPi($clX?-Y&DNs^;Fl##9GsNKjyE<^^q(&Y-ZcMsL?`u2Vd(fxa@2@!+!`5-9 zG6WDe+S5Q7%US(?q~EXY%}B4y{dl+&0yqQnw;CdwN092MrYF-MId zcZQ7T>POtf4bS9AoqiVm9(Fe>$Edd$g-+&R>GFESVic3#>-}3ldpXyWUv&4RGqiq2 z5>Nt?8_HRWkE=7fG#a2k|KOhZN3vXwY=Gs5fsY@8(j-EMUYnASGvHgPVX?pe@cWuE4O=@oNsupaWW)*XurXRRW^d0|IFg+p@seU103~+_E@a z9qz&~7eg}6&N@@POeZC~x3JIzd~W_dN3}jAm;4)QBwY`}Ufa3(+(4IM)^X!wLqZM% z5XHyu>W^-XSGRsprha(PH6ITJLulo%#O3Z@3f0ko_36gti43SioiGSI(>9zm56`!f zzvnoe!AkqN>zGWl`~Zkgk3L%_hA$4A795Wf$0d1(VSjM93URf`nh&uL%YnxCv3an# z%K>+}{95sQiph11x~*nW&DMbI@n)y6=6=RRy%a}oST`y#5R42F=mMCSo+;QO3$ZZ; zi*<>tB7Ve@;vN<}8TE3z`Lt{Cqo$i@Rm*Jfg}_YQKAdD<#3>1OJ41RnC=Kntty(9UXjE?YkMwYGdS&6di1^Ej?aH@3S3U zEv8{Ix^}qw{%3n|K&af4(HKc%ByD_u%AR_7zPqFE=y5;NYCoN`yNGDq_}RN8VCL#| z7%N46pRefMcQYYqq>lGJoy5~)F{y?rAg1~82%Y6;)3EWDvr*$Bk}QJ9B4TcWF&xK4 z0VK&iB(ioe(!nV4dqrUJ56x;E+k7^xn!ffFgvj_Pk$iV@J+_cbFiCtGahg!_#_=-_ z@2usCToKjc>{9Fm^BZM%UIh4J>?ooW#$Cu;zqet()yJR7AG^0Re}8SA089M~gf3~1 zq1#`NgKwz=OY%dhk??pO$}{V0)@FNsyz`j!e(%KVoQe53zO!1N%Fc~8tDW<)eD$`v zs4A@0E_-Z)&#)1zF)it%m=hqfv2-M2?Wwp9>C0A+P&U1luETN+UwAf2M167NgTA{`cXruX0%(II zq{xLiOz}Rlt{3S_kfFU!JF6Z`(5yM@$k-8}I%nC!N`CL<9HAhizalz5h&{!M>9n;Y zpqV{M>CGQbXp1?nI8%OeixlCl_Bw-z)q;SUy=1S^BiJDs1SHv?N{FrlWktyHLw2we zHRU)F#1N!F7ST;M`6O$vM5kEywuGm(dQ93c9SY@iTEcPno3(GrjceWEwSRU>!2==* zhM3(*>Q}Ta+llP95@yZ*b(t-Lo!F^h8+F7k2V7&a)~L)R*8z^8=*nx+SOb7#TI-&e z6Y9h56S>cJ;i~h`NU^SCEokHwNke9DbJzvitgl)msYoDCP}9P~wHOcNyOsO4<^b!G z4neq!pq?g{2XH5|f!cgdgvhFAP*hhU{>#f0(! zo1d<;j*b>|MC29HN?1Q_%Ny5ea4>i1yOVrYSVW}er0?z6cZ^4JH1G+NGJe1d?MUem z=066m;BQZLj%qQZ+%`IGf;(gYsS4fxvU2$&GcQGmw4vninmD~Ik=?dsz6Uc7(0`xd z$g>#nVD%1-7`D_OeLvHF_h8sPH%$EzFJsG)T?Cv&aZX@y%-^uG`r1;D)D}yRVDW|s zeOYv5d>@onFL+&V`N!q*5Dlu;?QtSPvc>K)D{YN+l`t>tReC1y|_G}O2D`YP)oEI3*?nfj) z?rXC?a5?z$x_opm@1gWKR2S1~;IQ_V`6$2Hf?Yct7x3m-(g=bH45dIBq;H+Rh%^4k z7Di`Vj@OAj7uD*g4M0W*Utx!R1dLR#Y=X802D$P*8AFd<>cesUphpj=f-@m!UHZxF zy9o=*|MTRYZH?BszLD95yT+Q?me4#;bvmHS?nxtr4yJ5)$LBKeP3|+_Ikv^_)BT`zVu9W4U+6VdjGQJd#BYl?h+~6vx0vr4`^b|2HO@~S z4O;qgX5xuBCU@Nd9?fEAxOhVrGTz$$u&#Ry5#X1-tMWrY%PvlsToH^U(J(8U4ZE~* zG{;mZZ%NJeySWwffsli|94xsU6X?Ae@|%6>#?gP9iB^B@2A3akrtil#qZGQpb=D7^ z%1KPi*+L_f*I@Ej@q=b&mr1~NUW;B58ENt0B8)p-LD2V(S!WUXb%u36=N6mIo)={=>T17S1qRuJq(Gs5{In?hd#)55?VQLLGdT6c{dro z?w!;X;2+pH;4~WDY2{;p2@Jh_X9xyp&bq&j#FMwL z>`_+pojZPfWL|0w+LkC{h(@$4_~o)@O;Yz-}9+U~(}U~C=A zm*Zba%H9FA%+Fv-#+e4{bv4_pEEBn+-@c9*=4FSJ_oHbV^AdG19sJu7M z1cUoTC2qtN3nb1{>SmUUe-^|0tTUFN9TzPkcRX_4vh|_-2zt1izs`X|P-7?AVj^;M zce+dKakf(o)Q31dW-9&Q7PM9#Q~XH!$@cf8a_?f!Rq{-v_6@b4H}%KE@2&7SA|@Zz zY0Bk2&g9VQM>LMbwv0WLtLayIFm&4ztkl@Fmt!)nvvqR1cS6K<;^4|b@~taW6PIEV zklzP|t&=?&F06=DHj!VSnT+YI9~dNI$Mt}>#W6v996c!bv_ zD~GljS@Zde7mb!I?cG?{HHRq0oK+5Z#YhT$eh4i2elO`xPIDRvKu3x+kWU@wek^A| zY2TvOZ`l)BARX13x^8F3x(XLNz>>%H9(1#bH|5u{du`3eP^9x1Q}VC(O4R=cDFRd( zTSZ5pfQL!P@Wr;}wL@GQv(o12bvuLV*BM{CydTRgA0hOH(tP~{Qg9_7Ez`g|Pg_XDhF{6#Ao^c?ZrISL)>F&WKd1drOV zj~TMBzY)UYDvSneN0=p*;k9y+{m9?rwMQpiCMW)i*PM~g@p}xDaZQG@u7BJ^kB(1w zjk#Qmapk}71cxy##zie+d;RyrO#JF+t(I*yIsf&XEpMCU$#(@~M7$s7-DxQ@Dg(cM zyiSJ^?S{EOWKOe=I9E)BB3{ISJI1s+I}uB?y=u?;4)!Zt*MA_62l_*NtQp4bP|?phYmjq0Ew4gk$Q6nP=|e%bBhG< zSflJHJR|v=+dpxv9p@-tWVdnS8IZgnW{A*{ZCh_=1>AqY`Elm5^{zuztZ1D|+sv=H ze!xmRqGfWx%9QYo_4{?m*#7`Fb_Nw>lEATo1^^)3_ zyq7cJoPRBTucztWUN3*$*55toXa9A2-`_n4Q!P8A`Q7MXI3s%N13tWTgFP^O^l}iG z_s%8IJU!elG06FAkfZ-){z}Z@A;g1G4WFX1esz7G(KvP1eCQvX zDUQ9N_b#Jzw%oX#mom`iFl{!+ zgQF+(yH}~QaLZ>YEQKa5XwbII&)aJm&W`eH#qOgTCjMK5g?6m7|JHo0J!-}$tw-oW zwsoyKCGk3IhD25gGbm zKj1(G^u7#!C*eKpQcn&Zw81;Sv#Wq{#k0iOlDQPEMkpls4t)|12^P-RaI&UMK{V|cSu@Fx~9bLEO5CtW1 z-Atv=-`gd(_m=23KcyLqnqCXqe{Z>aX2$jU4HYw=_rTb|5O{5|arPzp=_7u3FkQnQ zeaCB!aNz`nv75-#V=w|X#et!^%NG2k%>U`dACFC#=MTq9K7l;CA(bN`g#>=~#Pps) ztPEl`OB*}gN|5-Gca#}?|h5~s#`rd$!e!9Ci*F~$aT;NvvFu+ zLhs>SykebIcV6nSy6G(C!*W*s-VWgGH0b4s78X9ZUi7q3D)Vz;VT%0V}?vuk_*!c=JrO8G~vln*>F(~>Zb+xgH-k- z=;k`{{3rjugaz2=evw3TqbfV4;9pWGOynn5^|^AEhfFjL5FePo_i5d4ib&M7%rUDC z@3u!_pIX+4&FQn#AYFly3K8vWsGwQ&qWb~w*(+u=#!8mX(POXQL@YfjdrhERBp(n)$bZXnAA!D_aw9R(`t+ojA*A^r-pP9~d#`2h zv3$q?2BK;qY+(?3A%jLBT&a)br)6o(NBqgy3sP07w~BvgFpS_++Uqxr7|&DhLDe5N zK@2DPTl$aZ$KR*897A()kn82pyv3IN5Mv)@sK=`x|G4oP%w5^w0LOU?N3K)cTd#M2 z^Eq5(b#6H;z3ySIPNcV?pvGpHs{Kw7H%t0$t+nI{)sc1h=6XfLy zf&Mh_h6lO3r>5!i8C_qOg z%=4zf06!5Q-;6QX*wACW`t$$`1d0EdFwTJHSsv8#VqNxLjPm_Se+U5mhpqJU->he+ zJEUYX%xN=ODc=`5jgJP(UinVOv=|FIPjMDk{u(}R`IzQ$WYNX6tuxozaPRkS$hH=* z_G>L8ylSvFl0&vo^hzkv@4yB`!ylRcXqi#yy`8@YH``jc=3}k8VaD*5Wt?jkmW8W&V5eL1zePeCt$Fm<@b&aAU6Jx%k2O(0r0dr#@_q7( z8qN+)F=VuGK^fJ-$*9S7U6U*CDC6ty!tB51IVE>B#r{bLn&5aXulqr9#)j?Zbr?~u z9*7w^{~Ff^<4kexRV38dl@_PpMec}_D>5e0wfrDwx&L+6{(7*`qlzWlM^-!f4qp^Q zPOSGZilckx@CGi`ri-^^NpF8-7ZmjPawxJ0sEcRZi)m;Mj0{x;46?zQ&bB{2m0-}! zkIN&{I#6=LJfVuek~w>-WchjE=9RUd=Am~Y=df46^~ zn=6AQUnxd-c}A}M_>TAH6iF^#kvhN6WFyCepP_`%i!v`Lxu4or>TmN-yP{90?`S{< z!b`7rxV#hgs`CEnfdFznNayqIja}0E>q*_#n4HEC8$;`tSLN)uehfxd*V1d!J01R! zmqpaUUtD0FLG=1=^`6tjvUGavs%3vYepJ+($Q2e!9I)^Btq(razhnb3%yH%#48$KU zpzI24RG{tON8Kufgz{@zbqJf~hbOvacVKKmUV1%K>N~KOi8#c<2{D7o7>su?*3a~k zrE=dt)31L0nmyh3JD2W<P8T(-f`Nfp_JRXdB&hugxjpJ)b#C6v9vrMn@n})A;Q@;B&JBH&| zpVwbb;eR+TCi6FU{dJS%{L9kI6H@y9>^rN<6IGo$elZKl^c+bg!WQ#`I7flN2MQ4z zrXOnl+DY4^^0%}>w9RryHt^keyy;BZ2jR!2;On>NEDW2!#)-6K$IW$CJq3yt5>fgh z5dA(n$zWD=BmF-2_<{enEUEs!IeFJqgK7Ey&L<&d7_kW#nr<#<=jD2GZWKRs4=BHn zJ}Pjrwp=VZ#flb8`6-|ANdLQIdLKkAUJR2~EbV>Y1ApVZlukN6MJ-_?{$QmDrC{6&P?@h*FataecUfoOURcAi zuKYwe+8nJ`q$V43eqx?s|3rFv7-#;$)`UUir*!oudDTc_HF8d|9#+O*$6s52sN)4A z%}?}BHj}G6e~d{ltHh0;g38_UucsY+G3>#$)WbNRuom?p(g29_D9G{WhsKaFSU5fQ z&t`58J*wrbQ)8U%45h92swy3L^QCtEmC_>35U>nh|E?`L%7QB1sKkGNyqmznVQMS zKl9LpX^Q;pU<#IJ3^h5{?e@jAI3dN@)6a^yApP{*y7rDog6F!iXequ4B zUVWC=$ZNfSTW>`^W(H!^&54iUQkawUHt0)qt$bweh4Z5K#rrB9#6N|l$=(( zcjEP%V6$DTMW1Z3#|FCF=-uj__1gCJKO>fIa0ykPO^9G#5=YHp#3`PR>bRt4+PRQojxqQy%ohj0(f4+SgSn_8+}J^J zL*zU>#4Z>3@EtZs{$0xi`Js&7PZ`2unnyRYW_p9Hlwf}Xm zN-`7c>DMi~jbU27?2P8q+v{dL5D!cpM$x?nSPCk#L9ub8A{W@LV^arx7&)F}B}VxM zTHHGw@jIMX#8eK&D@8wh1O#!$nE6ymGi2bo%* z=djO5{QE4g_s+8`raM|_$o0uTKkAd~&eOp_4h)ui)^6q3bIWFd^S)c1-^_3E@^xlq zz2AFCK*`^TUx`Uyr$dfcG_yP_ip3;YFqm$^9$=~^LfkFsUF$W8aM@xm8ybNFgDIcu zkiR+1M2j_kgBsbCOfMeYWXXvC!~*cx(5lJ$;$r33dcfI=tr)?K3TR9zJ?YcDTf-t# z&CQA#RLoooFsAF}gTurgwvi_cYWvBAE>oq$g<(vEGWM_w;|jU-0_?+G&D~oco>~YI z{uBtZSVf5BxP>h&V{0(d#%27(77@5c$Sw2xH_OdEoU;D$_t$2u9K8B>L(ti?^`H_~ zbC;5jOdkte8636Q5^{Y@u#oyS%vOykQ#P;*3x8Z_*rC*zX;2kT!Fp@QA)T&=XQi3Z zmA?yi7#ynb@=51D0&8R9+qfs8(bGmxU%aAU_g(!xYearBie6qnVWsg?^)!khrJ82h z@3g6CoC(w}6ZgSP>juEDaR`UT?rz^@ibm`f_s?3@URq&X64SG-8d-05JoY ze^fL;t8On0&i+1BfBT2I!Z!`ZH~KjY?PMuL+6}!QkHw&B$HUb2W5sRQ{8;&#bbHo+ zJ~We_80BW3tuW6?!P19lPmgUR zUyHkU#`^aNx*Yik%@&kL2`++@8hk2Hyxh~la0^GXIIxgh-|HUfvrz0opopw6@>M?0 zSSaIr0>qKbOIUpu*7V7o6(WIwz4`M;MR5?vYBoMuQOi%9w62yoI*}6|aCEad0Q%vR zX4u&Xb^zliTYj4@zTJrm(>9HU?$j^@H~3t$s`Nlk^{Z!j>{F(^FoI)Uhf!>=Eb|$_3Gb~y2moG5@KIZ5OO$2nWew|@ehfuG_fm%#nE_X5;v)Gy3 z&ZKqbsxwEO`RMLz$D{=-ciiQbUt6nt*vmA)38PkG7(XIrbHMo{Mbwo&;{R}DRM>AE5x_ z?vQ0I^x}r<*x>%{`NV?=b|=t}_I+TDLYs_b6X1$$1>2M#qs+|6^r#l2in7-|eFLXz z-?iSWzRKxacEPUuw{zZ&GrT+!l%$4y(X3!Kiz>~LI&V82R^)leYYZsY$KUxIj%;oI zJ72?e>tbjKD$N4Q6V1Y-Nyf87n`jnDnw6ktt8WB9NAl`;FH^F^FquUlCAbGE-QRtV zXmM~oxye*MHDBYc9v)gnvmD>7O*HGC&4PJI5X}-qvlbI{FJG@oIMTxNkkspkgwQM@ zG;0gZlEUMm@9&2l}^$ zS);#VXkRoXx3`Q_-O-1~z-Utq&ihm@-ek>jDwbW$xav8tl>ydqw}M#O0wu{YIk{Pu zjgkN!8#7Uv2}e1Tz>wQFVhcLScw($4GY#R>aD7Odp{kiR80`hYvpe@<*(nr}0v)Sm zD`wIdB9Ju$&~(7-iTFuIM+CpNkNdrJ3gm*uytVGf*0ZbYWg|tv^zgh<#&z!Jp6u=v zFMYjXjP|u(?mdT1WhSp?!opE%!*WJCz>V%=Nuluoay|zbG?ky*z+}_0Z#z*db zqIMTKxv9CCyM^6u0r>HP$NCO5lEa9*1>J7fCU%JtCh&*XfP2T+87EXd8ef}rZh?$h z9`k>g>FE4sp)M%-o)yg+;%+ws?#M_t z_9MC+47r;jKdS53-Xi%Wq5bkD=#$?(69jPzLa{K!BDCt#R?4?_={4!Ua}RPG1W$UhnR zFe4vk>gnszm`LHd2^I}*s$kgT>h!vW(2ymbRFr{psRT% zc=Gvv(793Hcc0xt?8dOM)&{9TugL?Ou;LkjPzH{D1ZxaNG3fm~Y+g)^GZRk^*zSbw z{(wj8XxTkL9{3gI)Q-KoZ`t>-)?| z#cJ(NuIMEBD(jry^*9@;{asR6zlRNtdf*B#ADF=gTTH_`2h_C~uw@J&%PB3@UqP11 zQbKyvk$?r*r_)l>&o+8F72zjZc$pS*m=(}FujJ#6o)RDQkPjBwg&5^&4I zeD2`FQhAfe^R&XlrE|8vWL*w(&Zy`49&2(PdgqVFgTpHDv~lBSiOzIL2f^r3Yw;Nq zHSN@kB&gsz59}E@>7zT@M=tk!vE>pg1a@Mv7;w#HE61?wW5GRLlEM~}n<{4)l@iBX z*ti30%6{et@(*&EL#F;-(na^x>-9UeWXwr_;HNQ@PM#)btCnB4RYqnv<$zN5h^#>m+DhsFQNn3gh!n^Dm4q?EIOqluTzox7+!oI!%m2~gn z@!oBNk&ZL{!_z(L{cZ+-c_I(8)M)3aeijLv&43M?pgSmd{5&J6yavq70Z2Cd@^amK z9)In!rDhm0{h23b7V2Hr$Q~eED?H*8z8yIZ*{wKBXTh=q*C`#LYBIz*{<>2?lcCL+ zGN)n}B(@bD_X)|K&E%!Qe)D(fZ7~|~ks^KP^kDs_0QVsdIB5(54g@}DhtdPd+LPS6 z@AX(`E>^!ME$)E7nvM)S`ip5;Z=%PTe-@Xl5BEk6)6g)RUN)sjp3nXv-CN?8EoC?M z`WBBiV)#pb5br?$Q5DB*LDd>pc@O(mZs823ke6A*442S&G_XWsJ^@RH20W}j56f4k zn=>GCJuw&BzBT-1X<`{@#PJ#uLA<%@5kUxwGVwW*#Dj3B!$=6xyI~PMo37|c9MH6c z5!EZrG*eImWUxm7@0mGNxp}Yckv4?Ao9|FSgV(Qvo67!12NNk(iNdmWycKg3YoJvn;Ey->3%eS<99&vL^wV1O-0D zc*iY{B=1mRl%9EY9QnKG+x}PD-Hhs*+Jv7l-={Y{KtG%)5F43h@kF{2!5%e^5|L== zq|;jMl;)r86BKk(3zID29=SbggSbFaHF>GWSo2pqA(HSz=Tptl_dS()LzBgxkS?p5 zmL)55Qv3Zdy4IAylOkYoO1{zfNAkR|dn@zOJk)L>SX#Et4UFial%`+X_(e&c|JoT< zc0wWWukL?4TLVoaltYHxJmkrk6m_16G*98fjT2}@O2=w94}YT7+AnRK{%mJ##+adL z^1z1;GZ7M%4A1a0ETgB_%Dv6s_R96_{eIF(J409cb22%(uuQ~^Ec)0PZXYLi zJN08I>5bRAldtX2-6RoBoR?M9$sBIj#6gdX*4D~z*8 z2O}Bt?5siW9WFZ4V-oWFpb(nz-hWZ|0MXp_?fj6$A&rYoG|Ovd(!;^V_oOZ^jN^fG%Y%}=+xq!>+FH3DS15({W=@L6B)6L2KO4pgLJDT}AR7B!LvJ>3ezaKoK zfj>|CHz&6JA8&NG&IC&WNapZ#A36ckw8>gg_v6o9pnMHcTX#G;ZH)$b=Xp3J0(9VghaRT%MItMIHxlwIP%{+dmA zT6#5JG1%1d4e$6XIv@2k+w5Z1_0eQyMD@}8WcUL;UNAK3`$qyzWrSdg-(2Zv_LZUL z`hcikAZUKl{EL0^VG*j+6M;JWsmu{#`LtXgg-?PZ!*`nZ^aZrg2eJOKJ@V;11F zgt+-cKYec9=DSH4Q-5@jcQB4lCp(?&?xYJ+_PcXTUZ*qtY*pm>2{R3Zr)(hI0`)!! z$L|fd{|$HEfk<$;d^bPNfjb@l!8VcM!(@8z1$8$YIBpG~dK)j`PUn$Y&1QMcZ4y%hxr{&>5nr#KHir-_Hp_#CS9{SU*IL^pqZwC5P|} z*hD7{-}~-5mPFHAweX2;}hls_E~@2G+*7Li{t*CoI0H)m%kV>W%)GH z=TQu9e6w^GSyU)?YdYr}t@L zZh3zvg=1cK@aRH9$Rgc0aM6lmp66-monBFGk>&Jdx{_Z2+K0cc=S!Z}Z^d!aZa3J^Uhp>gtm_ynxBlUy8+Bub0Ix z9}M|3&bmRnFr*GfIO;uddgyTqQMSU=ysnU z?s|k}WnMj)iWwtkm1cU+A+YAwZK8zBpj7WXqgM=l3?QAzGN6xPLSYNIPr>wx7EgM4 zuPjQ%y8l1>>Dpv+=6ucn^y((>sN5e@S3+=Nn%3Fu+4w$-u`h=8VVG|emGl>Et`)}q z!^DvAmnHmVF@7OJWEFNIeA_V54QH*mUr6S_bU z*I13|Ik9`!zFB0NY+2?9*+Q_%7Gw+sS^P#%{O?%M&S8>RuWRL+OnF3~ibc>8CX;L_ z$cT#iecW$j?fZItfw*KqwSCR!G)d?#Vf8MK!-yZV#}!cPx%%pjGip5Sv5ayODy>A^ zbSU?S{1HXFU2a3-Qo$)df0zaC%6-^iscKRDMf%|YA;eiC6!2)N*as5S(33glS)y!D za1<_?_nfT9Iwsvz;6su&3$Yx#t|`5fs+p$*J$WK9o7s3Zw@eN^|)2dSgv3`R5< z(HJo>RH)&OUaZHUgVX!_LdTkPGzS#R?e$OBYbsEC0dm`|&^Kp3Aj6bfj){NsY^Zri z1+geUrg@$NHP7Bq0>ks556;V?*UNjM;CNf*?8-&YRy)HhmNH^{kyEcWnGy-n?$ z&PX_X(h0uof88=aZkoStLL9kung>D5*I#aKR*8}+=;dWB5odR1K3xwga^=Uo=il?r zO=qWNXCs3$_d!JaebreCYX5TUg(=S$4+s6waL>klPhvo)H(QJ?y>`Rj11Pcuh}_)^cDVvy_v}{IN(^wxT-z1~HNzn<$vw+}F z{{lPk=f8@uSsAS$1Ji-a0_K2qsO|o#K89S!>xI&riiN5Nv;@PwgK}Pi&-J4P)*35J zH4zG<+Hr}s{CdnM&egQb{adS#H7Htxf1Hr!)s@y^r_{q@*BGkEYO(n#=BnH#`cK%% z&`qd9l#FXulC9)nv4)iN>;0y`?$n7VL5Q}0ZtnTlN~6>m;q1?~dsRDg(9a5@L^z41 zyMOo$JEK&D7=^pWJ!{NydVTIpgDd}aN1zz3e3f700MDw1j$(&dwbqd^ofqmDCBP38 zScms18)qF+uM$$#nT-)9T^wv!!yagEGe25j+~cvS$Jx}lrR=S$aE==-mx4v7UuUM~ zWkPXzQ0p~m+;4nZl$PK1S!Q~)J?m-P3gTpfXSr$8vy~6p$pZ@P&)@TL8s;=b^1QF8 zm_66yIr=}(OI0-iU7v^g{Haz;v@gxSc={mz-*?Vk7 zU)wioaqa6myJIoJ<)ge93wG^&XMc1FOP+IbVYc>^UOsF2fog#DjQp7d9Hi!`_RF0%E+`^VfIJ#Ni3dGn>70N_ zeCBUv@I#ih>l-+(^S&uNls;W|ZwPu#rWNVXgyg)k6n2bs76+xmfgy^E$(IGL>YIZE zR~u0wL`f?Y0_LfX~*$rF>9NzA_{6B^SAg^Tcs7o$Lssb92&@S719 zsx4ey9eNR0z}_jz7Gsww`ls%pkUXet_U?!R9fs&jMztYO^an;kK|8uXR(+Jr(jE~v z?bqwexWBgklGp@VtFyKeXqwfU*c&VM5#^sH^!O#pZ_SC;uz7#l>e)LpJCTsb>MC_(K2!yVI08c-_y?4w0z4t z&g2;|err222QqcCijo{$+N>D@ny4OJhig0fe-dpa38n(cfyu^Dip&XiBlf{O8)(im zRpg&U^1O*)JtuWC3+sA3QcW7F_dq>#axvO<7WT8sJpp$ai(6$$&~{CXp$r<6vAq43 zhtlZNVi;?_b`Fa9Uu*$CcL7+jIJGQ6KHQww<#lqrz~$|FnfzSk%mNJC=!mIV#`}vK zjScibQjp{vzNBU*-psrW7_07GK1>~wgz69umSzPdd?=ZXu?(-MbJW^WH+L%-Q{5gC|{NVgX^0)R0J#Y%md~fNpd~dqHKA|U`k_v;UoW+{`va`ejpV~iF z|HOW*bhgsbN+;24gXL!D&c{5B`Rj~SMpNl1U~hkJX*M(sW|?Tq1&al=ApOX=&qm39 z9r@W_FZ}B=BM4_$wyg%;?iK^8{pd3XE2_cHt}5;LS=h8&s@u%@hkt8D!sOLXT;g{AbSkF{QXr!vF0wj6=jL9b2D0I zoCZ_$%S5d+Lw!8P$4}kcdi`a|`kA2{@q6~0w+-*)E1eUe@6){;5XyN+6P;2S)ZI%F zSX&-%I&(Mb?+rKF>$_jmiqaL)PwgMq;m(-UR!irlQFHVj&N%&4NY;G@>ZX8$#N(DTkh?o|W%8wn}FAom`yCb@!)QI~nO$E;>{(4upwW7$}+#6RkDTK5Lh zhFd|QPXUxQ4+k?c>@b>Sa_cbp^bN#oUNvw31ae2W-0@@gA%AphVR70lDO~OM1JBj} z80Mk#a5(!_4L`_g_C`gcvm6_t^7(z7P$3=+pYxurxJ5~?_WfCr{9g(} zG9^4R(Iu`#0`t0kGRY{2g%dw4^idcE7P#P8K&35j9U`e-0T`E?4Y zRC-nWeVcA|l5rjHciiuq_wDuc@DnBCjakd^95hgWdkR>Tnj75ep;fnWmZl zzAs->KFC>tkJ8;qcjuYaNtgeB?44PfB)g5Q-(xNL5CG@>Z|o-RH->vwm84b=>Z88? zojT5p2pTvU!NEb0$#!8>9mhxxXU+G1QJLdTinPuOhRC=qFSa4|3;1F?jlqa|b9rGJ zuz2w65tnZ@3(g-c%g&hnNyf3yKS$(X0gHgb+TYGCDTMAgI2!1EwE~OS;uXAocx+An z&XdEpb9j83vU$icfkHplwL|?zph|Y>&$NPkV;REw_ofu`C;NUHnr8j+BctbFO#i$x znC9t1G6yh^8Nl&mN_%O!KDjF}j>E!>%NhzNL!OU8m*Teqcbkaic3C&*06IT{*+sX{ zh>4oEUyXbDhYyo=%!u&%Fc~g=e7)Fi@~7{|lZ-x}9QQxAeI;{5oj)WU?QvZiHNWhz zeg09FWUMRs_Wu0c;`))z*H?PwPKR_v?SGL%oYM&#jhm`B!O_ zw^7t{sdCy|Nh9~2jI*Ua891kXA)OOaT^+U;|5R+i0hxFgKN$<{lVP$b_6gEoj?f>a z->>T992E|nZ_!6w4R-EJx!1rTvxAI{dW)@>Nwz`s>Dxaz$oda#mA!AdOpioyDjpJ# z{v+2csK*6+-!#a?)PLMM$hcCNSNUiBMhJ$=ya^tkF8{fNp1PQ>i`m@v(|y>7D`H)P z^_6=T)8{_lGu}EsHKI;0TD>^^vm+E=IQCj(Y+Z@3VUbW|nYCf<6F9f< zB5hnXx^8Qej}5`Kp@6;Ra%a2hm$l{@R*x@+yePyFx;zq2uWE}&!y_cV&Sk9(JMW+U zn2~b251rBRq&9Da&0ppcS-#F?Bk3%jYY;v?=&57I6S>IaZL86&4sku^H5aG)3{NsxHe;0m@9)Z)^sh$a3_ynZT-oQsijv5xd>*r%_KYsDj`Teo$#cX$Y%~u#b zmzRQnc>lygsD3K-vuoVKS)rILgnrgfomUN1Gni#%VHiCJi&IH7irgqNt}qWX+tn&8 zt$x}cuqrAHy>5m^9kr`&SbraaaL^O+b^OSD`&o(9AvQ-O_YXcW0ox=x80 zW^pNLv7O9yG8ew{?bB-gX9jw|fETO!v~&_1@82@L{~M&uf};9lci+$WgC~ z{FHcL{WHkiAX8&l{zSgeSHrXP@jbxnM>;oyQAT=JT^HB(-;+{JO5vkRl>zbYptRkP z?3Z{HcU~oAfTn2cYJ(L&a=6S))+*O!KO02G z*6*bI)X*Y#>uUUUtK#{-s(vjqih5%3yH(VrIasgMvwgd~zk5y?o&)bUL!N1ag?G7_ z!KipPT(A7Id2!6YmQNyn+#r}S@9FTS3g$5|a~sa$Y<^sBQC7yUwIV z=jKy|AG_Mb0YKhpTZ_fhAegXvj3Se@eVwa2HuGNfWQaay~BE zg8kGMr0Z0!E+zPs@KT%1mw&mKyO&SgUY+0TXbd98MR{h*_3PYge?`Jd@i=v*Yn02m zf)Z#y8oyJ?-K9*cn0^SxH9hIO9$5iY&%uv<MdU%*Z-$0 zUh%2GNu^CHX@1JyKZ{4MqAs4Ld#I-D>rx|H_9laPvc&Z-YUQn$nmJpqTx*v&f3I7X z7gZyCGP>_BYA#O zGV!V>6K{8&YPD8P;wLsOE8u~vhDE=-j1fx@4ot{X;{=U-H z-)$d|?Ivgd{G{fq-n^}+k_4l>y=|3qANI$oNxL>VHSxV!b;)-suXs2&F<7V%bA|iy8tP1Q2TZ8bg68o4%{ms0=W=TtOjV4vYyJ>eDXornrM+^M zpT<(vRWPVT(eS#KzPVoJ*X4z5{H%&W-7yn6ai}ilFKj38%u(7(=^CZGou7Yy1UE{< zg#7F?r>QW@Z-w}+U;n)ye?-6%_BvITo~~f2oj57n*=)Ds&3en0t7!Hgx%z2*QS&;y zKR%S>w7`w1?V>|lpFjKiN1L)E9M{k3x|*L1^z!-DCwj2=&Yy3tdb@;Bx)> zZhs$r!I7;DuSIPBbN7F8&XHhM>s|Tf;hfW0CvzXaf>+bet92&DVCjM}w;JnB#jp5{ z5Q`$LJAKyMMt7A;enW;;UAf2e8Ed3PCYR~7#*Z{uaAC*2V>yEbCv8d(=YhV`z&F9B z4P-x_Qr?F%zH8@wulh1R(s+mHcZtQ#P?`^oeC~m^6F}c_$Quw3X117O8W0}w?RD@< zvDg)%o8vvdK3*jbC(pG3ap)gc^|PXB9=*d$1ZQ2}`(0FUNu}4ac@0nD)+lz5)UTu9 z4C>)i-F9As)We>LCH6zL!;y+l#E^b#OIlqxmSdr_ndNLQ*Ly@einDAIc`frK75QN2x1DCbK)VXA�H>Fn-{ZL9(f?2 zq^w_`DG?uC!Ot3qsU!^4ok1N#L?B@fO zi1F`INDBn!!KT^^!ACaEraY>1DQ?v3unb(#v#y9c(!VQv1-*^}6XqDvj=$?wmbe7nF z@xY?681sWk)?U3UbwJCTaEFPL{rxAYfcj} z(w~`F6s_e{R<7O9LmR(oSW*`jt|rKsdRVjNGSQGXPD{eY67JaeFkMXB6u-%i+@JE> z@p`(l>!|xZl!x}a<~o>2uTu z^NI<-+;)3ms1}HjWX(eLxyln7YPE~%$K}C{mwPEYv+0BDm-?cVw2V87!5wPAMbq<7 zPxXaz+!B1N=kWQ&lwf5TK18|k{e8Cy7WJ_%a|&SublkPR|0`Sn{@Yf3puLk3!FvvQ zJhcNTnd;@!T}(nu=k1KCdw>H5>Kq45cDXm%CXGMyzv#;9?B?eXvft#{T!4A#9tf{{ z5Q{!OlWf1AYMSV_DrWiA2SgD!S^;USjYM>ya7v-TJxf*^*h7u|cWYaAwQ2g#XTfbQ z3H&7ITj)n^4AmzJ^=$7MA6t~=TORjyM_Le%q_K;=J~_@bw)NwNN9p=E2XC8*H&AjF z3DyJsr9u1CDrfCavv6{D6Og&lJls|P25TP398s+?k=^2$q0Gx+%IE6NQSSrQqYpxh??v{2$@oo`c&DH2X({|J+ z*u#zHZh$5baitag+X1iJHv1M_EG^SC8YhkhqH}5Ex!z%nzQE&PF*bCTaO6&jP*$*= z(m1EipQrr{l2d&Z@;4)!vfXS85(C1lPXwZyNrkF_gLqLd8> zmACNTgF|Ze61CgoYbc(j>Te8v_av|88BAX>lq zag<70JDqe%vkh#+rQ2OsygPOShmM!M&$sf5K(o%VnSDIy*jZIe=j@6%I)iTyUG~2g z!`4NV+k}mUodLUb)^Oxp(Qz=7JOqmJIvAu~%xs+)-04qFN5~pCe0*p;h%a$nwnNLC z8`7A}7H(xf@*?N9zyPX?9HkAP#OkbM1|^(UtM(2zPi-Tv6JNcf5u8c ze(|dA+Qlb4o0qWXJ2={!@%>*|3==s5i-Xl1oLGyk57UJ-ECw4iZ-Uut`2UuIjyTD z`K~`Q7nc-5cm%YGo4dy@H2Bx1-lj~{hh>S>H!FIj<5JXFzQ=8A>URQxBTCgzj+QM6 zF&!GA=q8;4`L?ZokkMHcIgC7u@(Ba()-jAUfRXedkzO;6T-W&dn2q_~esv!KX(=Ws z&2BP1-+ltvi{BTiKZZXhfwZ^|TU9AhmY+%APz{QB$EIN8~l)9+_%kI%S5%(%6nqikt(; zEguC=1XirONr|4>0=ogUQ=Fcl^mOrd%Nrx3JpB=2PwzbNnbW$PBz)g^;vkgfANuM6uj}UuDU!! z>5o@HhKJvpJ2aKM!*Q@kI;G{na^}`hKiAiF?dfoHpqFQ$VAO(Dy}|F4KauitDVx`f zDBCQ1#MCTu$(mlBE}0!gw7gJ`Kuk-PkUGwNU;jb2c&}oi;+7@9tJ}wy0}XwU(B2~x zbI&^y4e<`PMji5?x}s6=7z(NOk|1uuo5M6(V0zcd$>=8>KD)~NtSkfjcdh$icw`4NN%Fb3Rlyi!<#WVJw)RhD;^Jg>HI?dO2fK&CfuTGKc-(_kP^!Rhjdk+R(!XcTV9p7KW27W+Lq;bIrnS5 zeZ{|kPyq84GKQPyfPbw&6)ox51SY&9Ps!fwLMi!QW28D@o|0frbd{5HYTC$sywnqD zXVcLz^}MUE;l)d?b^I@^8e`Z2s43^=i|*!H%CvaFHY$49nWW*KE_q@Gv(D}Wh?dIo z@ck*Y_%j{sa5LABVgSPtxjxcC>DOk|>WJ>@ZD}F{Dz%$TbDV9)^CdLI>Q8relbtK; z;+)LO>ZCXfKGV$~PERr|Jvt~4-Tm-%vuJBy?-6$<7m zQjI>yMo~cL%(l9@HUwz zDyX*szbv@y{b`TA%eP#wh z0xQ0(y3IhE{L1RwzSxnkx~(yo4A>9+DhP~AMN$ptO1a@ zaU9XZ;rT=s%lT|6%;s7Cr^rXQ6H0$gt^0hD@^r(yolk=mo&5g^byuh+GfFm!w7 zPLEL64_lU?8X*eIESf_^{2ftw{l+N^GNvmW0CdbXbVAS!Tnuda*!L*-JnD8Nr`QdW%M@fX9m&Og#7?e4xl zK#z#Ae$%#E_UbxSUubfxs@U2HQ_{CuVM4<+z5rmuBQr=&W606?) zkUFY_Jw%i}M)cFtn+`R$)w#JJ$0br5Hm}lDeW=sVTMXTLlnw7Qn-$^KeY;*{cFsxf zd9sy*T%By72kg=hHnpv^g|3SB%-Z!)Ur$#H&PbZlPQEo+_$y1hFtgG2w+~c&9TD0j zWsg}~>05`SwIE@frye{-r-W_VZ_myEqj}3Q7uVNYi%|)l`$po!e_|gJ%*Y_Ysms?? z#D}vc5>Br6BK{)CN3heQu{%5f%38`_Owm#)teaYVtJ0127j0*@(x`r4TryH%a(vzH zI|#PwsjKz3$ zYR--BEv2w6ADb{u{y)u}fND-}ghsglWNc4;uI=&zkx#OQlqPx?l} zS81>s}fp8k7a zkAA^2uwfmCnrYJbDi?$}`Ne}3bzsK83bZdee(Cf-jaF@Rbu76hbI9{Zj4_i$qT4NI z_bE!T7|IgbG(~I3j9HgmB0RZ+DCu2gV#$>Y&p)7rq?{EYufCRu@~jf&IWcSU4wJJh z1%#*P{~e?;_atig;f;(Oi4gS*Hp9CCIqm?c9X+pD#Oq?nLh^^K70kPVWrm8XE=EK^Lbg2Zv7-Ky=sre=0|FI zvH(w5eFyt)kqrHciL)LvQzR~PMMYREou0hY(FYk6wWP6;LylZYJ5{bPN1Hg(8oSN~ zyQGE^`o19#pE(dFxP2A>8i8!B`Bnuxo28L(r?0hygMnbXSFZYt+!Pn>BtTMJ7`(LH z2bBE|#|JvGfO9dB-&F20A z`#WJ0%VTxp_0J9aiRfn(Fya!FAla<9qUkYtcF3U|#*uNP|Hn`pF?LY%5K zD9Q`Vt;cSY-9BVbKE#njAd!2X5W`|6Khb1W;k{mE>h$}F<%5*>eW)8U;Djr~5A$$RyFLz@iD5Rj` zz)BenpHN{5ku=F&2i)J^_q+v;)Rq!7{g#_2m3!Z`?j29g_UB(;74cK>i%J9Iw8m7)Bu+P$l`WXXE3Np{p>#VX8%h&6 zon#`yXgi8}K)*mW8-q~}2hP|u08muiR_cOFNWRT+{`cc!Wy%gHqUYJC zGLv-cEB|Ij(z#UTWv=)d`ec;plsfeYN3QDY1{%ga08cWFrQZ9F?;8U!KX@gG6Er+Q zrp>VwpaSzt=rK_9dXXuGQ_hb9JtGr?yDrK0wckBiUib?3dE>?u%6`f2<86bVq->P7 z_aRgO$n*r9I;Y`{eQcP|m|DxNg@9GlcAR-<@6=j)8=vP=n@Ya%lD5FWTyJ{M2a5%QIaQP8uj4|aFpJ=o z2c=s=-gWnM&p-5Kmx;H{cS0wfy;gpqzXN7+OI?H^ci`b@U95=C^f&%$`9estI%#CU zkr(dF@S*o%b{?%0rX1=C+h615TfZ~CMZT`-yPZg9RG9*i#JaRJyy63+a~8J}NmqPN z%8kI~n}zOzcN=0`pZ#be{kgmfXcGWfz*!Nnb{D2S|^(&?`-vJN8{JLIJtmLwx}bQrFyu?%f}m( zGQoC-T3a^zE25NgpBSD)58FtSAI)WNyJamXsU3{#z%(VF~Iu#!n5-YIL=?pDn?VQRk!b&%C6c@Cd=rdyjBqNrkjl2)1jrt zD80t$z!D-V@X~5@3mzwu6&zI9mKNE0GR+zA2%&LB;x}_~0op{VpA65{Dn36|rLyz+=*(fmGo+O8`)U8mJ$_IpO~ zf@Dp`k!cGp@QFh1*voH-3jGC_hZQ+416Mp|9w7UbAa_@B+^wA-S;?jh&(5kBqP)&J zK}N`#=e@lzHy=}T2m!TuZaxyTXwwAZfUQJlZye^RDw`2FrZ$H{)^`CMO7=J^TB z!#*|D@53h<5Fpy_Y6$aOr|;B*kUnvAL?!rbyc*k1!cqC+pDx!XHzBOG!zHMGSKlo_ zr}W@Nte*JdHqs@B*|k2Yp>`*ZGm@A!UBEV#T zjPe;PC>_4bc$Jc>rKy{#ahH&!?1FCsYj1M!3rK8q#_Wp^Zdx#}#Ih8zl$bWx-#nyw zSohqchNnaP{YU)L}z6x-;GFVlh<0lQFi zwlO92n;7q5EfscSsmb&a(GeulepW_?jo7cjHikd%lc6T9zw=Nc!q@eQwuLo_3IDA1?6 z43uC=-8|M$8Jo zn4T3L_NCdrLB+`aX*2dIGu;4*^k?KgAsLd`ky0o+v(hy0%2!zN>Rc}arkn#NWTxyY zVrg5Oh$jK8u_s~1$I03yXIQ50CX=7Wg8bJC38?KwOK`|medHj!D-_IMPZ%a12wy2> z^1naoq?O;e;?r?*;Ifmb{uRF0b^H!DU;%#A7sk}@i2bd{gO2h(6*k3^ONKDp46&5- z5zYl6ZR_LN`ZFlp>)M0fMVtZf{4o{Jeg&Ph%^nJICLoBo2xI9^zcdObXh}6O{i5~o zK20zO8{@?}%H8^090p*dxXcBYCGp+dGJmmX#H8RuVhEcLitW42!bW3wZ)ZXwv0}&< zrAR!_*V!Sq*XqNXq~cDk4J)^&`#6mgym(4y7S4?4PlKl1({HD}_4BBhlX9zBt#Q>X zL`RGlhKBGA7B#oop!Ou&>?`9KqoUCh!0g3MpqMBgOD-fi$yuaj*61QEXK!ms`XKH5 ziR0j+ilL96j%fJbLAu-HENIP%0p=jz-on*3WTDgx`lvo(1p!xgozLzo5<^%L{cu$` z?|Rc?BUu~|m}MFC#am-&yCXdjw|I_o{W5q8MWTwG4QkWA@vtaXU|Bv)_;cV%iw7WM+T z`!heU@5SII!?DC5PaUVz9^@yKuKV>9-99z_vvGd{cfsz(=&2{C4D?>!D%k=tN8vRH z!Y}LLobG}C2Z0d}P(QoFZY?d1VtlGrdg;vvj-$S#h=+tI=xJ-0|(oNx9aVsZ>s+C&Pd;t-4h~}Z-Rsmr8hYGroK@@^$!ld zPZ>1c7$+B)AYXTp_oUlp*EsiD1RlJbz_X+rgN^dAjrCNX3wWzw5WaL-$j*t0F@se2znf z`Lm>*3q@J2M0t#0Rhc-wiopv$Rch$_ zTl;&%P;b2G#K3UngORh``s#3vM_D}cL~`Q0by=WHhi{9X-jc z)0+)}9KGX719*)hmhb&Sa%_n(!vJiT;owf|dG|n(-{uT9#>dIjjyPSEKHv$o;Q?vS zHIVW7NUr*a($!S+1f(BcD;M}w|5aK#dox~CxfB3-ym#h?@4Bj7b>JnLV1s;$S$PrP zd*?Q!bABP`?^g0J*~#g=#o0H#EVrDBHY6hF#4Ga3Rn#zY>$Huvd1etkW;wf*qZNxU z12`zBJ4kKfvWi*00%!$$#|2=Y-2}nFS zfy#@jG6JHf*u{`k4eOzU*(Mc^;Gd4-z0Du{ZJ7Aq$e?~NC0o%&l=442S$x1~kAx=e zzs5xl9=lf8+An{oaz(CAB{z#vo@3qTz05^dCI;C~efhMICp#H^u*MIMX8fSldweg7 zW(Nu}PMYnB#ib1rQ2znaYlKawt4dN5`L4xT1@6Q#{)J@C*2J`T_)Jp$_jGOz7EZ(- zwm0$5D?7AKy+MC0ZtflgJ4T$mH()C%SvEhB0`Z$>^@SvRF8$wj{ zpQODxH-0AJoO7;*39z%XwARhi*EHyfp1&~eJi6jR&Fx(uj#|ZwH`mV$Qsy^d z1KH7*rjz`?!;D@tF_<^1kkn0Dp$Qi;jqI)TJ5%2b^#4SEAJHMlWe_S88pmfaTT%$# ztXE$$-eaY3APKi>y~qGXmb!A0sU<7n>0GhVLz(=(D$m|HDd@~gTY{8Iy%&a25^6GN zHgBoz^@)&Cyrqn((d7f6Be8iscU4LZa5t?0Hu(`CGsp3+>mp`kZ!sqa{gk}#dlGl; zKKK|*v-!LXfVk~7QZxy)By!Ipp{e>e1mvYrERHjO4k=4Z%4X$y#&<;>4l1u%Zn@^S z3Wz*CG5+SDsi2irSEy^FVRtL_1v5Ik!j3;AGE1!-4=Kl+;!;0*KE}l0`gc=SMIrlxE={hb z&a*OY4%aqLw@?JQoKxx@P_?;_*X{n}PD}MP#C(JJ*@D7rN@3ginSxLdK%NddE?moM zT&6RNn`Q83b8bDVFSmd5@cg^~631Gg-&#cpdf>IpqnoJ#QI6qbKH)&v;ZNhe2krDl zyDnlHot&?>90j%>OK;hSp~T|3l!4{iO#|(pI|GC=oN7|qqJnK4Pe$~k2HwRzXQB&k zsz+}#g}4zwvx2{zvYkqYdw=GMw32}lJxv!82bWck0_5uD|#D@(yzqZblAgW2cENj>@FH0d>hj!dP^)%JooU(QTk}5ESjLY4%8O`yP`bx%c63K=R3Q4 zis!lV9UQH2=~md{nG0(Y7$mm+Y+}U;bumFya`3LuhzJl)4;r!V>{#8T0-;ea5L6y2 zO7L4Xv~VoU1{rMRv>DSoNXPpj|JXEp zG1~Vm%E8sClkWM^a%ZYvm<{oIKpIqjEw9mY%N(vkWn^hCc~YXc!+D)1b$r;+tmp2O z)y)hxZ<;r;X^mbY=64Ffc{f!8uySdywZDlt@H~m(&h%6`_?Z=!r~NLQjhi+j6o))d z7H7@4EVptT`TO-He^rnAGd=b*=Mo+tBmae!BvW4B9HI1oA`VWM6S-=CdVn zXKGLztMsyLBVW^mgniO}rzQJJ=iMJR2HIh5>WE8lxK|kVso*~b7d-DmuXM6Z@||kK znHi!}43l{?lDKmgvY;QCCPc?^OxhkDGXR&OM>s}3;PwuwdY4+Gujwg~X)E1*9aZMM zZS}LSKwNijWJ~n!lG7!_JI1z zZ5BJmvBi37X@k`=g+w1y)^$qqx}{^@R0&Q(h|DbkGNl zbgcmztkCuu*9q5AY$U*XbhVCNm&tlVp;!3b(DjHr>=fPkUfwHHZrmrvduueqahN1~ zg=zTb{jN?HEKM7B#ElrjLDOhG^I^=ztK6_r#e13=E57h@yoAb&=+8RoE-8)KUwKCw{!E%3F3K#s21vQ}qf!_9X z>r#4W*@ zGtMVv@gnkheh}>wmY%c*N87L?1rWa-phHP*2^;^Oye;b`UpV;Og?!WJV!r$RXUz{JM=a>;2LQdkFh_!}) z0guwpb3#r=%bu;e`Qhf)0Xj>5g)YQf)P6QOr(QniJwE`?Vu-c2_-CsCwN(6|O9fnF zfe_LK7h*fVFEnsiK}zj`r``cq%BQ7Mv9v{*%4e#OAbxUI7dbt`i8fB6&?ECvv`oW5 zxU4`rytWdu@1usmpv9ObL$w7{fLYxt;4>s_`-cpGU&YSt&)hU#JhG44xq$*bcN|mv z`#h9bMe#kg0%hvS*7BJ%$< zbWOc0_VxH<=x&Vw_9V)T(4`ng5V8NrzR)uwO-dxfo!1!2FHqr0i{0v0f|g6vhX)+rX#wa zZQ(K8>3~f%6nk0-rtU-!o}Stml|#*p9oT7rN)FjH7`js>-jcCJCB9X@gZK}|w^oZk zz+zVZ(c+DfN{PU)9_&AXo-4Y|j_i6R*L&lQYiUajZzb@wIA#OwNqK=^F-oPz>*JuS zo5kBD*Ei#h8)-|8Z%y#DcxF%9liC8mZYz}nAg3>>kMhBkxntP9cUze01v*e;B*E{3qYepDhB#}HV9_*sfqt#;n zi|(_=w+46`J$U9DFwGY{8)i>x2>iOK^x0^=7<6?qe!lLWA<%PEx7lU)NMz-p33GN_ z{QaU;0PogF@Yy`w^}R7GPV_>aH)Z3sP_r>jHYxQzrx{K=O6U@o9cy$jvX(2Y#wx9= zzM!7|EXR8>$7eCecd^88vBV$yVI--u^JSLHom6{#^XV_~w4Z;@1E?|=jX}fyw!Nz8m>z@QQP&)#oy^h32)cp~+ zu&rB2`^vtb^vbWv}zVYzEvT~59 zy>#kZ?2XjhA)`OnRi9bjI1)@Ps*zsfl&M+Gtb=z?1n=m2Q`G?0w%*nVy5LAWM*HP< z?-S2{x6vsgC)NYY6EdjiR2S2zhU+BErk8&M=%(3o5ZX6L@gL#S**0_!GqPS*2i6$5 zJa1vaA{IH#Y(eWZu+3v|u0Y9V+nql@pmw#|T@!Q;1};V{t|cAt~Sy&V|=wLwrm$LSG2piJ6VGIv}_CV z76Q%#UrkQcohN8jeloK~G`Rr5_2;XPr*ot5+B3v0w0!fEdFs8J?$;$rn6DCvw?0HC zT0T+nS~}mHP4TY&cCa;`RiZHubvXZtj>3dGPpqRx0Eu=`lh6PI15Z zLwPB`O0_KCa|U#j$GKj*!U4(3f#3OANR*>2KI?A!)5X|#-+2}};!mE&rfVPQhFqFI z1LQ9Q#Ud8q2Y8JE7R>$pRQTDcX$oMKV?^9nKennAT#~ixee>>%U zy_e-1*{XToUKm;ZZD8|_=~jE;LA%J^4hC;aU!GzKlFHoP(CnJ?zD>2pq(cn!B5TyFJ^mdETo2A7nGC+ z%#M5qC28#XgUTJ0MeNr*8m{dya>KF?ji^ zXRinY^=uojxhmJq#n{8*5RF6I3C?i#ISAJ=3m-vQZ4Fj*1kyd@q4ZU-OFQbN4Ojz4H*DdDIlTe5@cv$E%GOd2c>&9^eyz%B(*90c_o8n1aMdm_C~ z_*Ogwj_TMO?G>S*o=xL57iG`67-Lx6WaH3giIHz!V!ezPDm)^46`flj<}ry2#$I!t ze6d@f=fAQC;e^ zA1_7dM#Oy;00-G(F?(0_|5){ z?s5P18u#n?anjPaW6}$olLBcs)xL4G5=E%b&lol@uKW|!z2@Y&apE-=Fj0Gp$aaGI zz48C&fPC|MAA|6mTgBC$xJ^v1z#tg$XZ$FA_9uNpWva)%%NjqBa0&n?qz(watvUBD zlEk!`u6W>ZWd|uy@#qWH$Gu3Buuyhz6x`V}%EHW}=63-uKP~a#jC;9#K$)GA$Fi6v zy=tWCW`Y3x4TF0@4*2{~%}=WLA_6#u7n-)BBYml^Pz@gRu%xZT10;KS{M-|Omj?`Q z^e58YVBbGRIZ4?RdQa#M&TQy|)^{!KTy9AI1Kfrxs$Z<>gB~6g6<;2x-`lHx$KyJf z#-GNDgg*eH!?I-rrl>4UXriK{=N}cnpl^EGR?%?~Y#DK}(|=Cezdx9ldnA6mN^D53 z;b^_T5V}Vn^27G)j)1gZgD-Idz_>2bUnYhV{>0V7B;f(O(^X< z_sv(8ka8x5af+T|3)K+>>u40l6ySeX7F%{Cd{&D#)3lElupNK zT@*&{ez^Jk`U8^NS95%QR0z*86&n9bwREa%(@L&Zm5LZTPMdVJCu@ zbiK$vtdydA=LsTmOQd|bi|mO+xB!-B`n}ox=()Gml+yxc=|3o2=|KTox60|eo;|Mj zv&EIobxNbcnWU?+QzNDVQ7V$IyHi!B^*ilyOcKspeP!&OSVn~XRzX=Z)`TU3@|Ekr zl%Q$h_lW1N%~NNdW$W!0^o*dbPi3*~baY-oTZ3t{SCxq@GwoE{s9@opw;*gr|Jjf* zLFDx(yZ&74vR3kbzRJnij+Zq;stR>J$5StTTPin=#O8ytU8d#DBD+JNHC(bvByEO9 z%`Pa{wM7eo%J;XpD?1 z0S@5GsJVVgVBKo5=@PEG9&yuAOcJyXz5FlGd=j1-a;{tDrio1TTT|w}{94o^7WIYG zZN6K;z_M|$?KVB=)Cb_Y1)Lk+jBEt!Lv$YCwvBCF9^ib8ZR2Yw8U*c3XM!pzr%N2L zh}tRf7u8|-$QqbDEa-djV^g?EI6+!PUfR^AYpOu-D2cb);(PIfhr-!6ur|=C`{7e{ z2NH|3!{|sUx2;-JH646VUwg~QP|6c5W5Eu{-^pJfXxouSsMr+N!i+m0>Iyu;IqVtw-DG(y zZ&g^VaZ-Y}@;816-?nP|Axu;IjgCs@-AzH(cY$~5Wz23yFR<^*Y6i2qJh({AnMZW3E9LIA8J?Y|+nc-h*24=rkjyk`{-oTACp3Q=Q~Y0ny3)DF>R z)eY36&D042vYNDK4lU}27_zzsPQ{x5!;H9Gv!^gh!#uLkV&1pbFR~uLEpPx}3fB&m z1UcG&z0>*4qfYmUVoa9gwOzAcrJNVzY+y+=1}TaT&)R>{m}|x?q{8#%iE0$WW_iloR$CF&a@i5Uu^D_thlUOUK?*~&z=nG z_Lbja92LodD4>3nY|Q*1f#g8wnBuD{q`>)DlBjNRpi@m1rq>{oV5qdBj(?b~5V`^B!_ zg{&y+ZdlSmakV!EFG&LldC0Y^Hy&T*^(WhV{mwlw@3Fj{W(`+hvSv-b&Ict_&XHaD zJ;}}OyBsoEH_O=Gh2IsB`Fl}}dQmQfk=60r6m_P22#W@nb6~xa!GGLDMlw_`LpJ^{ zP{uHn?2wesNXGjT{JLu?lX5Hi+25qQcA29P&dR*|i>e_KtOwtw?q^;&jv!|xCryAo zN2vmhiAIPV=wj#FWS4#u_<+_#`@MK^Gn)=CIiGQ02(5{3h(4=zU`-XqP6@Sh&f}R~ z6St@trp(3vj}jylpdIkPh>1qnJ19-~2!rf%XL2Rd;;#nOp;=$GxJ9%7Z=%t{MVoRF ze(`@NdGm4Ps`>vW1R6bFrRcTzi~3~idi*KZA+kmJ%agioI5?2tTe(Zb8K?UZ0dOM9 z2BY4`gsCsw0=R8Wm6^&%F#&`h3$qEELC1R2mJ^CYdUgMr^DLIrI!uSv*vd9Mv43y> z{?dnJ^)*W_{Hd$Z_WktFa#r(yoae%yxH>wt3k6tRBcMp^%HqmoN7gZe*Cq|OBCL*|$&o7s2G{-|Zu3jD?RQpCc(OS|6o0#vao z2kJ9c2wT{7&0J(`-X_dw22P9DDsa%RFK13eqLuhHyJfj+71-%ln`Qsu{a?bh9sk3h z-{2qVzTltk7-q%%hX;y%j#j|K)vO!kry2c+xfc{N?}fj~8~a7kv0p&BZSA zd)NrTEF$^+0X9;-tSjNqQe@Ad3IeqKJ7E@Xfzw9Oblf8{vIcny6tq?uY<$|UP>3r! z;DR&F@x+2A?d@!90UfV(e%W-#+YXhmz?9$?p_Q@)rMo# z2{WL1ulna^AB$bwd#-zRMsindGEo+8ntBk>dzdJ;%g5UPbtC*gSz?wyVL(p8nb;)z zKlgI~eHK3NC*4ZC=(fpKzQFZjj;s7arnp{AbFKS9Ta}jzEJwBc31>=Jaal46Svmbhxxzk}`$cr&U$Lww%LmyM*z&gcUhswaV+S7-ho5%%byEql;Ce%T%MQ z27I{#S@-Uq2z_Igq52Zs{B~Y&;3^gG8q6nhNNbI2kH(!`k~EEE@apR|uA_VIw;HY+ zlL&lCY=*KQ-SfQ2O-PvC_99P^BIm3zerh_aRl39V^O$DKi=0fFl!Ap++uxYebXlwP zfa~We&6YQLkTj|C1^s5sYr3UXiskx=quKHyCj*d%vyisEUVb|LMek|_i8bjMc@6Ck zhyNT%T87*ZMnMT_>5allc9WRlVh&!WW(!!uF=WzY3|h zJj(k(rTzRSx8C)Og3UwznOEhn#gILv)MFTWRmXE9hfJHYhT(_MZ>#1}|GcYO;H$uE zP@`T zgOul@A3w{?e`~(=_~A19KSjpluLayEZ4GV{mPSdYL6fnys2+smiZtlf8S z@jUJOVH^C7^G$YTxzMKesZ@;v+A zJ$H>Y_M=P}ltt@Ta*ON1Z1Y4Rsni`lP*cj=j>$+BD zs|CuxMDu-*42`z?Zfj=7^XIFDz$fh21Aflkub2jDw)lzLNiRPQ{(cic7P0eUBxhK~ z=kHL_n|d-z##gqt6Qips8ALwK{q``ep8 zxqB`2JNbKq^sD>ItrF}iY*WlPd6O-faJ-D4re%7c8B`CKM<*0l+UB_Yf4sd1R8vj& zFH8an5UTVd(tGbsC4iLBdlQf<1OWl5BAC#N(m}cb>0OE<&47S{f^-3;DIy&#h>GYv z30R)@d*1cG-(Bn8HG6H6nR6y#&YszO|Ms5AX0vC7tFCCC$Bf0(ew%w!jSr^ozG&>u z620fy=$a?W>NVr>^y2x)-cMuB-+R@#mgRNdv%_^mH19GSuxrw1CgLfd?aP=%XT+Yc z*L`bOQ=9w7Yxh=WiOKh!1-V8QpYmYuy%UmJoi)^;dSFTpGC}n!urF0s2)-8_RZNbk zdv}{C6XE^IoE`%++EX`&=?luZR+8RJj@+h7n4qf^HnEgz8(X89<@IO3HP2485fRWG z!9B7g4!GJ*64xC2ux9=4ls$#VLDHk9OQh2Xg9-TTa>*TumBJ|-GYZbWCgvQ z^6xD65r*A&+O^KPZMpI;^J|D}co!NpBC0hG@hblsqG4(&6M(Cdxg!&xVfw~f7vuy3 z%Ag~KpSw45J)x}5{fG^IzeXtRCp1%ek=*2_A0(zSlJx?62`2T=o(L;- z7LN;SetRwVh4Pdb#3jM|TV$-ZUZ3!6NAVEA2F)&jg5Bvy)_J|`M_1@^ER#7zBgZlW z$CC0lxchRplj6`mpf}*b=oy9)(Z%y{1?2k#hI;oSwq6;mano`l*j31W^fkTc5j#U) zU-3SHr5^Yon~vfmR$cy@Ue@0DO(t+dqGj_D+cWDZ?icpxkfQ(9AJB7Q0?;4OY1!}o zMqf{w;`*z2KM36LyVI*eKz~4|ISKgwn%QLx1^#XX&QaT=G43yLTJ=>IQuF zN|)Ke5G|8vShrd$wS-bcG)1>b1CQwNdD5(iQipmvF^Fzy4wwVnq+ zm`m6fs0K+}MISj1FYGMp9>|Mu9@=|zZ4liK3L*Rq{Y0_Kx+~A&BVtK;l2ZF*gF==p zy;GJfgAm!NhcsOL=M3WVsTG%ed)`DnCQZJ;Y!SA4_8&E*spGH-mQNK%y;kuyiKk>e z&Uk2e=)-^?pyNM1>}0O@mglGoPQRlr#yw;?{VX>8d*REt9%)aNPhL`6j(S3xO3UT; zahgN&a=Xgv0!Dl|c1xOjfeQE`S9WKW~RL4U?JvN=FZj;sgOqSz7jRaIS5oMtctWT8;au&5l1(+wpB zkV3kvq@7Vm7DQrJ`1r)-`+Ir);o{m(b#R8JK7Yvz9OR)2Y&T~|D}J$@?I1dj{Zj^7 zcLHTRezA*(Y|*owvEzR(8?7j?zOGdixU=N{=Q0@&x2Xl=in;&n^83GZ$@lYLx*Yho zE8aZZHZcs$ok+#zWcD}EP+D{IvY1#Z(p~1M45oSL8BS$-dK7k>D zF@eeET9+F4RT6q^Rxd1{htmgEvo%y{-8aEnRf*`EC~@*Ik(8~>&NbV7>iopk5eOo9$?esUeQR#Gqq+JZMB^FV3uzN^nvRJPX0_c`##$T z6020g6SQ1>r}cufcO{k_Ei5U|>BVJJ`-UzXqnK8+MB7IP9BNFy{u64ss=V4+JZ#tQD@-xq*!3m*)p`Yscn>DmFk zd-El$@51d4x}}S5Dv)`~tAR^Vz*?6ej8lZq9&TaWe)8gfInLUWiYm;o1 zao2F4Were_ks#7L*+1jZF~+YT4o>hsR3eC(PzvfvzRCo64%~z~P@F1Ek#Yc3p?lrQ zY-S_rTxA|74ma#Ds&Ld+Q9=gejvYmnfDnXPP??+CgXl_-IU!CJK4EE)#atq{9Bzo< zP{yf3_$;agf}>c;vfR2?>PMc zC56{l{HHJWvfQw0jj#SE(NZ62OAE{tMk|_YMZoxyakcS=v9QS{li>Vag<8EPE5>xT zKgttXnrkFhWGi>cH)Aa@Klk+8&&qFg_F6u2Kfh!?Kh?TgF?h))^F^~zJ<{BMTn+=X zgnWG1ay#$vL$1YJ+1kNNvrw7i2YEc7AGZ0}HE_gp+~i2*$mGcJQBqYy_rhJ$M$hSS z@tsJI3*ik3-H8B1&aiN>H1@1z*xk#s)9lOAoq<%^adU=;muMz=HYTG0ZFsy|0R4Vn zR}1<}Hxd&RwU4MT0{-=pr}Que(2clvc%Rx3b_S;%0XGS!9go~3yia|sAJs0QY{-og zK^~HDQAHEW5fY_t37-$fell=TXa(={doXI+I8q^1!U?D2>jvJJgMq1*HaJ4i!h~VK zH~4+8B#?wiM(d3JRaa6+Ao+!B;!ATY8hO3Oh4?HqikyU5LJH#o*iSX&?##PV92eow zP&pu3zz2B{K8K*=ctaQ2ffVJ`GEMI2x|Ie5S)nd(md_MaO-&V+;gfa{I3 zoh}o;hW1iqvl_{l2@~UNr!TD33LHK-kpaA|1_g{S_6_O@&Z^-*Z4fMS{~-_`QR?Q% zCW=D?vh(({B|90&iR0K3qCNF=Yv%_EVD|u47EVF{E6e%8;iOh0A+8SS09SYS)sJ;x zYT)c6e5d&qH8@iVZEE1(Nc+Ii@rCo(&u9E-Py73p>qAE&unvVdtZv1wi!4LUv@_eHE z1-39A+tZ1Bfvn?Cb~rGvLR&|*Sib4&K18m|dmWbDU5G;q*P}!o?`4avB$7J!XG{64 z&^Pco=)?0+@*rRE3ef~<>qpo$Z392X5PKU14{bI^Wvg`pC-|`s7^spPT;BKLF61u5 zVGvcypc(WHU?q`*`oFq!Qz=21*a`ZKyn=bCfK)O?+QCi6l9GsmiN#1}(H%Ryx_xtz zvtmm18_B6(Fv}tmZdM!lDfJqB3RMF-PuK>%MnO^?kOJ80+N|U2AjuKAMK6&J#tK#( zumZ;eE6&zKC(lF{qq|8Xad-+IHd&aG#{ncUd-P5m$GA!h<=A6Y6cb6lMX1&eClq{% ztt8py@hla%e3Bi1lhp=C7?1KFAf~DH7*nyXjXOaIU^Q+5z={PxEWbWx2v#td`t9bS zwqnxM@&%y~6@T?;xf**LPNWyL0}-r9t$|vM=#_HNL1Y7`CrLh-Ul5iL-YIB5R9yw+ z;K+tP5xn#2tENRA@$v_Lv56Ea--GU?&MWsp$CiaHU|F=>o_jDd%*NfKRiW_n|Ak(8 zVMI@E-u*YTVIpBXo2GJg*oK6(JW?kGz6usAcvQM38uyY#QgV z*<(9a%kQUg;gys)hXWRiLQCdz@sL|FJ?cCqNiFH%gze4t*1~t@jD3tsk*qyKEj^cX zq@!@|8sSBp(yFx76m1?hM!_PoNJVKa_-gc4pI~XpWxI`NZ%B4C(b^0Oz+JmLEN$Le zzhf&UKDAOhS-5xVS!k*G?ErSbq_`y^o|wt+WZ%;f6+-%vW`!45FVJII3Bz22vXscBAii&-kdb>{IcI>vw*ao%NL(x8l6?(O3hht}o z(_N21t$YNLc4Sxp;-KG)6Ngq{JrE12JDA$KF?db+h3r}cY@soQ#o2FxwBIL#qP#2P`- z{rTR@9~{%vo+NURJe@0m0-1m5eGmz(8VEA7(-54>*vDP*8>9Vl*L_XC`Ko%V{K*jY zGt5g)E1DFAv!=iLX~Y{X{ZWgQfN0g`&wRs15JvPs-9{fr%o7Q4N3i? zuy56y6ip31b#l9pEql8T;>gX;k_Y=~EAzUu@|?(a+kznAsZM%rK7K{DRYuH&R`XRK z{p@wuG*r_xPve-*tDAW)eCguJ1+r)VPOs6n&pIkDjOB| zL(JF6&o5Zl1{j{XkX>wqNY6y%cy{aEbD$GJu_=x?UBcj6c3c~uO3k?Dc-F(OyoSfu z7Vy0kP7li*=o)mn;M!Qv#UVoK-qM@@KiB{8x5d zwhKH#UL2ABfd(RH0a;9lrq=r z-*q7|jM@FD#q=7FRdi`(g1bgF{^hWH$1+z zrVz^&!+E+|I9BpB^P_am)66};SRPPkTV;O0oa@8T`C)S6ii*GFnr@8FRmnQ}9*`!6 zT8M~j$)3}Nge4-T0dOw?xN288+Moz;4dAuFZZ(PBlLl;Qq4L8ATIT!xU%xd`Eb8CG zL+Yii0F)xkTleT9K{#kQk@WMm?95vqlYe^_w0N8i=UD&{d&?2B@|S0UM^+-$k^?k{ zimk7XigdbX**8PXWDw%HC5-bHbVFl_7uPhJKH(Y}j7E7n-P7!Y&l}6=iszOx;v1ty zF+Dt;5(x3|^Tyewj2G9Ut*<&}*-wLo^v1~2KfS(Be^ddq-9y3B_j(wN9P8}A-z%1| z7T23111elPftQ{j`jz#*%fh%T@D)dG|GzGd`PZI1Y6XCA;=KHWM4r$Ox=cRJJjcdz zn&_>UhGT^gxp{O^m4q36OjVDO`82N(@J}#R!PL$FY6^@`Sg@eOU+`CxT`}%I`*Sb8 zX~|3D{mTTPr@t;Ns%k)*bs)|DQ3ZTrVA{WRTTt>Z-8%ADe)*Sgi}=@X`(L^(zE8iW zo3}knpSoRZf;}jD+2pEsFjjC&>O4#W9E}6>XnHoyhkkIFkz_^iB5^=(T4?FW&+y00 z|0TTmZ&1Ydr$n2K{Rh!QKqUQ#PVV<6oZ8uL$DC@9!gxc=U(jQAYAb-aW6P9*|z4 zUEzDCpM9WHk!K`VD^%oZV`*cpHWSr2?MSiN;DZ?ZCxNLIjpT|vaTk!f(hgp1S&3=F zz2M8x7XUf{TC6u4`tLo;y`ZGPHy%DiWnMXPxO(tDEg_Sb=IcDkvAFRo*Hi|*G7Ypc zt#2*2_ciQ%U3j(c%w9sMQ!KBWu>P<`Zhr0NF}#w+mUXfoc%uQJ$|nF@h>ze}42c~M zMaCm5Ca;x}9@5+1^a1BT3XlqJ%FLXfy9}po3!u{UD!dMWYrmwWppD@u>_7zrWJT== zRmSZ_6>ys-$Xl6Hq;dGm>h?(7THQ$j79SZ7Bzz223eqPm1l(rwOVCC_YkWw0m1S2J zaGM9A`4oWL!~#8Xn}ALzT3J($BGO%VYbq=9@!kyd*l7YH&<>joe9jXtH*#+nr%QTB zY3c*-)hBTO_n)f%3ScOJQJDn>#!Nq^YQRXAxr$GH(xoDI+EM?d<3Fp&&1scukjgcG zT}5u5UIbe(AxOnL-$xF4-eV#mt;eGNZHoKqnsw$Zfq+MS<1iSpCH4oS!x-EYr!d!5 zpByu4kuej+zf6FaR~S&^nv%Xi)8KoCId0^8l` z9A?=W>^T&oo}vXr#PZ0WeNG3|>dRe{$*i9NcyHSPSb_e?)N$|%m*;?sQDh~LRLn%> zMd){zIhZyGPSbfB5d0%{MHDVt#Q0r8)!_+iLdVh**0c`c@4@nCdyertC(2Q3!by9_ z#vHE^Q9b#OLs5~wptt@yWlMF@NrZb$dIrYs&{i~5M9Gkt-vu2kWa~^0{=8XJ|(;yIH?&If=V{X zV=iMCtN_tF>7&5mi+zpiyIWILQw@j_KBn{sTX|!twoER3=6B$U|8zJnE;eORFjb%x z!UYB3mGDyf3fT(tiU5IQ^yR%U9{Br%z5qDnKhc+(x|6@qm#D`xrv9w0SOMzB9GP~5Ut0W9^M3eyZ!lQx1UxLq528E2RD~Ub0eIzbE zC%%|RiK#K-<5#%V$7Bk?^xI7Eo;SNz&*g*qSNvjIdb<2#VQKkI(U#x%xiBDwt3qS$VTXuyXDN&UY|AoTgdBtPK=;rdU%shYf$-ao=|arw@&C$|hFjEo znzX;*bkgMitg*IJJO=KMA=%-TK|GKOpr=`^SM5HPR4Xj+?3=sxw6lk)`16?X8@aq> zbs(C(ueeRs1H@k(NSJZn(Qem}FTdt*bmScYhd;~?mv|s0f-?-h3#l;{C1RtM!;d1& ziNe5C{d~o)Q%vIXBvxO}9Y_0AFGZ#1&*a@0$-Ob6aC*>@Z4;BX%|4$AM4NHe!Dkg* zFb=RTaSQnoye{wwQwrFY$pPL>?g#GA!(pcZ(-O!-f)qA40oEq(Zt;jNkcZ^we0Z6v z@xRCIz9*AR3tu~}wiFRLK9tXzu2tC;rdfJ+wX3AFhuzIlyx}Jh3=F5hMY(^QwLrRr z`kz!eL-d7Xv-Ur8%CoHk4R1`Uevu9i^QOBHIu@L@{uh~`vYDV<{7($< z;Rsw1Ep2_IP^Pd{=`R9;c}AkM>HSS+kN?mFrI1sVa5-0bT0uhgud+Ufe1dc-=Pbh) zsGsJQx7g<>3`m_Y0ZK=qz>Hs0(j4I!w1&c+bytqV8*3Y~wfZG(9Zx1kq%5lZOi+Wf zItG8+vcmOjY|l-86AY^kSgT6lx%VQIe^DvQ6K>6h-u`P`fH5*bF$q+2kNm#*=mm_L z&^3Ul?TvNSU!;TXU2O(7y>*_^-P^LRy8PVaK%`4%Ib7Rv@a3p(5)Ytd1fu>V(V*1< zfFEIuL;1r?M+E(UnIt|Zth6w1t*f3kncZ@$6nk!BS6KSlbWz6|$eR0;Es-f@+2KYq z_xjb%A^K|{L;&1R9yZtio;-S-B1!t_)=`V~WT7XO-{uU-hjXK z1CGQJiy{N5KcSY1qCnq&`{CaraAGc2IEQo7JW4o%zPrl793=#&$H*lJ+tSy-EPQK4 ztV}(Cw6;STA??+v=c012SU6%!KL;>}wg(fo0SLQ&GQHpCmea#}7dM5^f$yQ@MTbYT zlH|1eNe?7|R~)&U_yi^ZuRy#6JkVbOS|H191UDrkn9HEAg7Z#Zo_-&5>hhQOF&vkf zKE!ZcR`^i%#aDrWC{FUJp7h6O=OU3;Z&d<*Vk5+pY?WbG6k~&JvBcg(d4cA@U680m zr@F}4E8-M}-Me(;N31B!$@f59q76MDq7SLKB=9wnYUbbG9Uj*SsOi8;W+l=E!_O;$ZBt zfdmO3E}a9Nje$1fTQ|P+D0MX?iF?X1X1xb4?B3L6@8oK^x#%NJsi!77&bRWduT03m598&-}M6`u~8r>ILAe-`LtalI-l=&Ody4wZr-L zi<$Wy&z+!q`^qm5*gke7h1lQU3x0gI9%#9HYRB_1=-vW z%>2x^-z{@fZwfkp0xgfPb~NoR9JSOuZm)f;Tu-0`Q2X}+yC1`1*WGSCYzx>cjMmKw1W6-T!{usCIO=#!;E#@aiQUUx_cR; z_ei;HiEVthPI?xG)2Bq^@a5h-0^kwB^*H#bY0Z*;T_G041!zJ9Z5rRnOv@9%>`am2 zM5>KNm5?H6lLo8faDmv|Uv+@h_vCY=22b{XbNv>4nlRL|1Anm#@Za`)FHj`_vAukYhaPJK#GMRbw-yNNfSS z?*$M*7y6xK;{ji^q-_JjDc*SB3rI2Ft$y*w)+W4WSKT$1{e$z{N1Bha=IT@5^)-CA zqWF5ZSUB1D2kB{nV*~G4yb<{1oBzzVLEm~=*nfTe+vVJ!`rUhDOez+He-T%*;yFkZ z(&bc+_C*>qm?^2)TEX3XQ%-(;&EupcAm3((UP1!nfbB*_8S8oAQ%~#v85ng2(k5<@ z3o!dgak#<lXyYaBWsq)d1fXKnr7j&O5(J|X)*@@+UgZj8?j5aS#& zT=ESr4;AFheBH-Q_McSB0qU7Ub$~+g*{HaT3QmbPRNFPse-fP@VJd$2z)>(A*n_~x zM{aOQoe94o;RtwSzP{@r`~RiQNsq;Uqv`5N?7vZiWi|RgN%;CV`JCstqXzzE&HBB+ z<$r3uP6~R7@p!!+aztH{6}exo7Jj*EF#-evc~Y{yjAY+VDt^f(iUz>p2m=XH25bt- z3sFh{0K&j7WGEU= z?w&*ZCtRL2ka%t8B#3ivI)?npPfF~RJ7_cr-2gyq=5AM*^E^$8RfSUA%o^0 zGSZh1GccvkMB>jAWaY&NUL7~OY_qP<+KDV70pDl8Nh_lQFv3`F?|9*kG z;qb!do3Hhq^_@Rke;CI0IR;Fh42*pQ9{j|Z*kI$(g7rX8f(tZFcXhsx^LT^(Xha~x zNq!j@293f6RTi)jC~y}_oItpKROkz=OQ{c598pm+9XA^{t6i09Cfxw?Bw)oFn6OTYvIO3?m+7{1}!ciau2$afans47_6y$_S&Y7D#)q7ZJy~{!K^$GgXVGr=M(T_ zNqs=JP%#wQBG;kDH!?|Fx%7bTr&oK=f0x+n+F@U;S6D>pN6Vex1V$60ITTpy66fTW8QY5;p+8wRqVMuvWKsD zDmL=!F1MZQXe*{Xl*m|pa%G`H?sQq8LYbTbpE8RnE6YLU?eekw1Q~iH;#D!Zt}&x| zak03D-Nf?){S+cPXXV(*$B4x$9-+(!1wBF-gUX2)fyC9x1eME}Ebci5VAd%0m6*Fn zVSc!}Q$b!n zNW11rx->z2>h}DNV(avanYG?d)p;LoSD}*NT8RCOl7RFneH&Tir2h-w1>biFLZl({ z7yiSuYx{_YhTqPF7LC8W!m7>9e=}dEvF*A?vN>pNd;-DxFiq2dMR6?-ah~6+!2fAU zUlX%7cR8zdUerNRUU;>w{LbUqWcRt{((WU3csJ@q)hLe%-@^jrj-j zI2T=4^-ap-`Nuyf?L)akuZs|665#9^e5p^$2gOY29%36 z&vWRLoLpoN`gB43etdp>PDh2oDNERm6IU+&Di8cVSKR$M{iC0B+|k#HO)hqBNda8; z5dQkqc@j|x5qz<=4#h4dE{y=kP>zGgBgP^cZQx3mkg7+eFCyWJ#TYn&m%**p%Hb~7 z3Ec-ygiPIsnCaFGJK-kV#n9|+$F5k?pRg-u^gc_>Jo2O(Kl5x~V(CtSg~Yo@B~N~qm1N?D`TG%8f1^?cNUMyqqLQs!#ciS3hXU8&ow ze^|Z$JMEw({{`*+o4*-FLB&$y$_RK)ek_a|$DFC=NPPGaL23T{i2YQ5_Fv4hExklp ztL-xI`<1_0zwn&mSONDO|5zB;Z}vKKo9_0{-~Hh`ebZNjC8%&4ZCaoC^oios)vhwI z3G5D<_&r93wk7CM2*ruTA|PPM;P(@|IO9!muLFdQu4D7?AbIT z*~xT`CEM$AHRiKq`6~vc1+?ctm%3-)Cy3Qdf*^V9T7yCjWgg@S#~ml(vg0QWJnfIm z9HNqW>STKOBEn$RgMZL>|2*g{SI{I)b(59o@0E2!0W30lPkiZf%2u`-nJ2@vNJD{+ zXV!Fn-$nEgxE&Hmw#hJU$Q!7;IqY;F>y4fR>!(U+Ipyof=)0T-K06jdi923Y#3x^; zIeBLn37Sa~Y{nwdr6}@5hy0Q1I2yo8mJ?*P68{c_7 zf3oAhs*~|PyUXXr)u7l%OaAWYZQ-@94?00v{%#07+t2jBfWWcTprEj&;k0d$HGlyZ z;Tf0s&b1|xwx$wg-M0FY105V26rQslE8v~8Zamo}Muxw;rkv)u4%c9LHqePZwAwU& zTVy5)^yMo%`b&!C9eL)lZ3lti%Dy}iTq%QGw1V3>DJ3G&zsds=SPOh7ZehQz05u%Clo|T{QW5S|kk(-d$2G6wmL&J&EIkT@84}bp5qR_cMGfNrN(d zj>a^k6c{)B;58i&9x#F(_`y43VZdmO;g~&c)bT?vf*<-IUNq`L*B7@(|0mM^MMZ>T zx#e=Zkci(0>*eJp&nBv{k>{1S6T=1k)-6A6e3rLbUjE2x6jhgCMql}w)yVjMYUbQ#VkT&+2o{6pcmL zq*D!Z9gsoHE*M==orJV7m{}P)zieeW@s_W@<73uO&Bf|31cfbV%!W~n{NP|NxP8m7NLxKMCY-HAf(|J#W>!+FC#GSi zG^@pSRZ+!yx(a;L8wJUh$Z zUr5XPJU;$iZ_gE{ML`E%4_WN5g1>j#ZD-mg(I@j8}l_t9vcdxT$>l05qu%&sL;*dfh!EEo!S((!e=p>S6tKbuPkGZmSw>c};B`dn$-WTymm z{xq>c%@01!>tGA&NtHk_D)X7~%XQ!fU4~hwwIgjT9`+a+g4C2rO}ExvDc;?p9q)*D zou`Bb=W@GKQJkG#Ce=P5qM&QOsmbaJ_t#w^}Mego~CG8)y_9jjP$oADvlTq8c{eLJ(snC8gTB53CYB z!c4bzcrf1<5*d6SQS-*pxac1~9RJx~lR{5Wk`oaKV-a%aKHVSWukPkF%ZzTPbP$kW zBw!#UgR@AutA0)15y(t3SrbSfUVSCWI;BNTe~wVK-Q(HzGdpFWvh#iZnjgft9f2Te zT+vcw7FBLxyqDG+f7_v;vIlIP)PcM{fAbkJZss$jPHex>TUdtT!Ve+h$3x z)~5@kPb%Bp=3v(#W@YJNw}?wp0kXaE>D~BjZy89g4Y9s{R2iSzN819^;WjN;C}E8) zU4Y7?^cox7#)Kl7=7-la+=M@Zjn_FU;N_Q_m1oq7D% z`+Hyc+B}P&mIW#DH5CukxHq!o)%5ehyrtxGpWWnpeCM;)D)lZW$2n{q>cm)hI%0I$ zD&uULxdNA+!e}(Dok9+)zS-+fNsH(B4g8nRF&;5K4cvu#6=&NB+lj}c7gXVvF>q3} zJ!vEj7-=Chnvy;2%b} z&)w}f`@$VoThN)FH{B)_(ED$J-SC<k|h>p~n6#&&OU}LUf8%z9?Or8uPz|Xct3T+F4-3gs!f?o1oH&Xf^2&Ffh&!(>MUs zta)lxc*VD}wY;)Vc@DFLW9BlZ@*{e7aOr@b&nEVyP4|cN?4+AO>QXV`OoRIl7N0(6 zI&2y}e|vp)%zDhV?WgPdw_BAjFO+$Q<+N?NGJZP@lr@&$QG>H}vVsEk${ObQ_6sY>_#@ zD+JTNtEeGlKNxpl!uJBu{fL}bsZtNPT>aDsKWPZ0hZ^y{$k+2$%z1^BdNAhVmp-t} zMGJVK?!}m!B{r??E`F{9%l&BsXL!M5fvRsHwTv>i&0v-pdSZMzuXMRScSR=e-h%3} zNFcE#zG?jkmUo%W24hNV_KV&raYZ-N`UhUqkqM4j0Mk!NW6aO`#;&;r)B@G)Y_YGr z+kB_Y@)cpR1GvPHcPRH$Rf+FH8w9DMc0GLj!@+}~h<)Nt82I~+dMqnejl*7TLmzgy+TjyDq!V)CZF^tc~J zlu^vexlh*m(WO9#+Rc+9Tzhc9B68%lYWmLCbntGUbu<{jn`kzX8Q@;CrW2AY>hNe3 z+4+F@>cygd3l|vnQIaJ#8O07#y~kxrKwU;;HgiQ7)E)|kpkzUpz_Uahq;XB=g{a7U z^%4{UlnMSu^o+EEcHA|8J0MBNWS5`A2#Z07r>1We?n%33G6VA;^`^&xx?{LvK+P||ED;F7?1y9iS?^`%9 zKZvB})Fz!9*B1dLj|)N zj;3veiP*7@wQ$<8hUQ&jv~S>p-%+CbI7!zc=wUih=MOx8QqBlsJv#!yIpDNOBJr|-Dd!p`cvMxDbollO=|H(BZ_ejY=sI$x#z zJDYgBUwjMYIlOA~_R#Exo&8*G$L-$HZ)zV)%!myGyzK6F`1gGXPm6g?*Jr$-zc*`E zKS}@epb|G9#wwR9U?W8CU>T`e5m``MabVx8>eAAUtbNn8nD9a!H(cIqfmCwS)}v*O z%(f5TX&nR$G+?<8cn*^5CRmZG4Zxy(09yd z{bv_E#25YHHTZ2GWH6B|gI*83>&UT{?VnVln7*QyD~)_y!ZyBDu||aZT87npLplC; zN)*SpvSlz6pdeoHZ5{hpnKcAO&3Bqr8y`2e3WxYwet2EC{C30f!2=`Emle7-2L#o4 z{@Ks!G1k+G6_+<3&%0W;!9&)-Zx1snodld#Wk0Um*OP}VI@y34eGs*UfFxH|AYdBC zi7t#FoH%pMm_MjU_*jK=S3fi@n-5${nB{2}T$#_ZZd|l&TzqBNo$H_v=yu`}$ST+$ zzYvLGsU1>EIbRu-6lq5uGHc`xKGVw!_D0n%&W7mL2T{$BIL4bNC73*WVKPx#0RXkN zJg83${dZtvURtJ|!l0XbP+yiTxqv&+_~`R5D0H7fMTPsKCo93b%`#WWZWuE5m&2&l ztBHqB$%N9mWdc_2S@%#;EUc{-^AcbUKZLAai`G~$T2zx&PexC!_jm+2wzM_aK;lk* zp`q5?XegHVCu5z8rr79-F^7}TN@yxIHc$%JJc6+_xhJ1jB_@dnGeBfu!6k2*Eb7Us zEg{}X==`Pm7^YwbYEcIpcuRJjxjVzI2#1Yckmas~7$!fpS$Ba9 zA8ZtAwsh<<4;%ysUXDGq8WMt&@m$lB(N}vKb+p${ho7^Y+>;S{4+gh-hwW(YLfev7 zX9v?yc|;GGtL8CGRMfv%d2I*M{^g+j!aC*RgQ?#^Y zJ}qn)M~)3b`}&=Lu0+6PA6C8#;$9_+bYO|gAJ+kr5L;WoM54TQmXdZh*<|7!ePT5+ zj24f`t}I1AXkeMME`1R3IqhtZLGV&#z|!piN2U%5a)uNJ(%f^ogof2*TGeDK)vy%J zM2u7euYIe9T{c}}6ag|C!`!n4J~Z6i$6!t43R zyj$7LcP_g6Il@vd;!*b!MxA7b?GA|`h7^NrnUFCG*CgSW9h|v4TJEY`1)K*fhh`Zg z=Qk$mF7X-Y>`m0yFS`u@9yw33_Bot4V*XIaO=j&znYnfPaEj~4@R&lKa=|ix&2b(b zi(%cYA%G4lD*)a7+Wk!l0xhGTpUY>UPs=}ZMttVX4!_YadfyFr>w zX_}%~X%y)z>CXjx1lMp>{KSAh>0DZ>^k2gGJ8pLP7`8Qp*fS}M-zb1=UZjBAaJt7} zAQ6L99x4^@Esfr(mJ+v|GHW-3S`3p3zl+v229-0zQa+Qg@VK84r1mUiv>4{vVpV#$ zBwfq3HIhvt(+mc#47~_)9;JH47MI>z#FiGQ-F9w_f4Zk{)*56?=DW&_VCD77_(YvU zxW;WFOfoC9#;*?L}Z##eqUbNwGK&4|ia(Q`g1! z)~8;wJdK0tl3w=lXjy;X(Nwqgj(zWA+3WcU+YVBgYQUUHQJry>Oodjhw1l92Hl45l zLFF{&Vr13iAo%=iHZz*K5~~6(+4|vx6YqPj20NbVXgFmL$o6Iwq@?&x9!beU>Qt&` z0deN3M}m~;G3=ENd)Q&9Ahe5&a3R{Y9z~`&5ok2Gj<)(NCKAe+)d?f0QgM3PF7euq z>Al#EU`EXjm|1IyuRZAt+D87HC7dTW>>wj&M3O*Lt3e9dW# zqT>LZIABdTPAq2VtNixEJkve1qw01wkGU&`#{6|v@77DEzzIJaCRmt13eK<99TanS7IpO8iDFJ*vEqo4lkzc8i_- zW29(Ox27mVA6eq-XbnmD6SX(Qwe^))hjM339F&#KijB-Fg;2R98Cp$zBbm|SFQ#gOEsf4|~m8kf$ zOvaT#-N3Ut;zwqDRGA>~sn< z9~4;%&6Zr7JoG$}Y(`cpTrGQD9{$p`Kt$2w&Q)=l^BU2& z2muyknSws^jMm94^6Y(6Omm~+ZfGx!yW__}-Lv;{b85syZ0U`zT&~)-tE{~L zhCBknMmyPZG81*Rtk`;8eCkW0BEh{HSX}NCgU7Z6WX6ekDRLqN$XE<#gxm4UJaNM)vP_j)s5W6uto5Z2&q(r zwb5OyCTp6E?p*J&4{)rn;vgj&foOlzJ~_*YjE&kxDD!Nk$Pm6fNG$S~%nw&q$(MxeLQNnoQWxLD3*+zfGhsdIh%Ov=wo*XQ3nkF|(Sb&q5n5RL3gQIqm=eEuPCt<`ff4^6-vdaCHNe-zs z%fm{$NTfiK|HkU!brItQDx3B0= zr4!Qf>*v>KUAJ~NMk*P{pDOiMSmv`cmOK7TB z)S4K@|6=Pa;Ogj^esOmW6nA%bm*VaeheDyayE_yq?(S|yi&KibySuyJL;F1M``z!} z-#-l5WH#rV&1QCzotcmbLH!27F#xyqMOv>(2Y{h@`YW9OvHMAfBFP4B1rXtY)E?sv z3>Pq5MwIOf3>E;i9RVu`DN`v2b`Df7Y7uJD)u+A2sALV7lUP4_Zk|w3BVQ~cC=f^2 z>>ltP`Xh~2?k6s{@UaaWGbdCah)<7^T0sS*AD;D><83&XmQHS+dwx>;G>vT4KbTuJ zUeU^x%>xI;8yPmLs>)*qOPAwJ?v-8DzD zT?p?x>KryS;ie-^cjRDCc-Go`modz$R7`+%hnVt|J zR~@bLK*zuM-NH%ex6)oUMfFUG;TEGT@?i5Dr>4Y`% zPxj>f`M5n4l>C{OueWU!mwv7Tcl`v@wH0=C#L$Innd0n#QGN&;WhTe3=wk9}nhmuZ z5zAVbO-)<407LbisAv=KC`pBM-EDdOP}CSoHz(4J1i>;Y6$=v0P^ zjY%9JLG$2iul99&OodFAGNcJ=1{}dKWh8qP?q$x4p=-y$ChAq?hnXfMX9g7RWt(G! zePhdc`Vp2yk$HT59h3mw!dk&Q<)CU=2unZyxSb%8hnHaQ7>=DH(TsQUT>>=&FF93K zeTs*@LS@Qf`Wla&UqWt7>1tSRN}7WoT5gPgN@cXZQtoK}*(mrTw-uAWhPNj-|A>2k z_;l*|70xKoCmm?3g2o<}+B5HfRrT-GT4fxcC zFD^QSu~R5h!j6+?`D?nCfw{`?E%n*_@Kc@sX!OHRgZ*ViDtxEjF-Mp_;`#%GTv=dF4%9uar60}f$xOxhtMaxFd&TFZxvAkr4O?g~G9 zIz9OhwEadqRHP>0Wh%BZId~BX#uvvGn$(z#MOKmJjLTwF9hX6sw6k`7Je8V77-O@? z(n*Zzll4TL$fc#4Y8YjuMrvx+9KNIS3wvhWA5)48jb>EKQ^VEb5ofPBjm9W$sEGZKOGkXQwn?Jos0pM-pzt z9OaiS588i~k0cGB8F`-+gtJK}d@RvVu6E`RsOrvgWsSE90HO5lO{Whl1=FA~#W+#Y zGO)ffqJ;iyRcJ)CbTC7$zK3D}sR^p?i(vpcFL^Kn!r}Xk>l;_b8dM4{48EiX;->7j z?Dy1XE>zWw1w8MU|sD7!%>E+uIj{DA`hvWYK>49lv%Vdn# z5)H4#nT~2lv4<}$hXWc7dL^^8EZB`8jq>rLFyMorTu0U<*AWO&^qt=@tSf9Cf^cb; zmG_el2j(!>Ip8^Q;ELQY)qYx*X9lV%beG;J@;|xG)Kysosq#ORs!PtqRUS%dH29a| ziSNM|v87M4b(S00tNrvWSJ%O)6MMhkA-Fy}0A)JUCnJ2&A+SWq`xVQ3b`^JYtUk4h z>QW4~olqYA^vbWqlu%B-NVwf zer#Rydu6kpGpSYXNNU_P)!ibB26Ix z@O>cphIIGL4w!z1k+=}#Jg`UJH(8Oa!Je0_0X+ucm2yJC7o!Xl(9I))3KdocUnlAV z_4Xx{xBF|hMPm9F{yuULvc9*x^+M?Z4Pv@vHR)=F&SrO`tg&|!YbUopE?oEw@G#)W z3f-Ne$xP$JhBWhhH($-VmdoqXoSf@Mt@Wrl8ROH6d`C)~Gc>42TAJNP=h}#@)klB@ zTI`$(FD9VY*;1&c;>noB>!ujX<^E}Ew!Ab=Zp6#w;c04ibu`wQq#R}2rU`Trw%~FR zSVfK41|I{dQ7J@+Rl;tU2`R*qQ4E0vel*tVeIpLPoo8D|c$(dQE|q^zQ?sv=wpJe) zInqr@ZkF3nvYqr$u=Q!Hky|Jvb_~$E;bIR&Gba&<%%VyUn zN}Syxf$(ktkXN)zm2j! z+rK@%+rW2PO?h*!caunSsdqC&U76U`6sx;t?YzFjJCGS1rDeyr~#tw{e+0(i41B6S`&Ns`%$zPCjv?&rD6c~1>L-N8^ z48#s;iMtQdsCi@MGAn7v`l%83jg^`+75T{reccMG4LOKz-}WHpKs2u>Pc*+LUzFT| zTDFkso8qIrvNb4$YO<0cL(o9r?b+<>3N8qbnE@z;da{wKkUy zm_uM?qg1Ozg!d1bRv=f3=Uw##zop~Yerrhd>4vEM2>pa7!WFaZ?_>#K&1_wqVCqLB zz5@bBD>AUJl{&GiT+LyM$GXjm4{)AI%pF2JAyXg*(pF;iJut^; z-jyx+yQZn1fPDg=mi{d7UstP-Gb7TlKfXym%RzFDX7Y>Plb0?XGN+c*UJ!Tm`a8z!z?*aY&yA#ocs{am-;JzE3krN;j$16E_OL=n%OM~$D~ zv+%VeRDBE%0iN~Q)pep76+IP*o&oyH)Pr~!yW#pOM<`0TFA6V zhM)P%^Fo~Jqr=XFRDN_zevM?a8}I9Ban<~QTyFH>FYP&HbuG*9rlN9_?%tmGgb5lO zFfZIoWC|v~g$5k4bXhNcA|Vy>NX!Xq)sd-R+6E%MSd6oVrzu%)!n)I_+D?Fetvt%S zO7DPHf|`W7(&A8~;IU@G&D#GFW}TK5K`1PnG?AJ50bA3vRsLIZLj1OULISAonjm;D%?dlRunyTeu|Xh*B1wEwJIMZnr5kh3B?fe2uiXcpbtyM z#Dxg`5R0kn-Ymnosz={)Z6xDd6RPic-iPDO4E+eD9;o4)Pz;2nYA#10M{XOA9nL0> z9S^nhrla@KGspcmr(2`U&967Jdjj;z>6O$h4G217y1iOO9xWt-i4m0(W4vV>r7zcJ zZ-fWqzD%7~z!^|CJBGEdE&O;*r&pg2M7W(Yh^fKA#?oM{3x-SF&QE=yQFwo_1s{Ok=kCm2h^|8WCeX=ICIWzG|2aoRO3t z><&*wy1CCBs9XPXyeddzT(LY*muXC`Xiw5`9vU=s3$b&vuP&!cOz6Axx7r{NVAO%& zG;Ud#97#MiviU{GfA{J9EPG$;#-u33ja_sG%%Hc@_hq67OOQg4g5*0@SPWSq9Y(+v zOP5X1aPwR!*yt*3XGI zZH-UID;Z%S9xV_}p9{JG+CrM0<`>nz=ea`wgO&OYNtGeX7*mxmeuWargm%fq2|qfR z#PcTrJVthdrZUM4N(%U@MPGKm@Z^nm9@3?W8lp*U^>GX5F{Z1b(+n3g6|ntW$l=C# zxrV!U3#hd!N$K&=HI8~q!`!&Sb5{y3h1QK%1nfpUqbRw%E2aih*pM&qr0JX|C0a& zPFr{y8b&QEK&fJV=h#}b#JjSr%@{P+ASuL)gCNZ7E~#;-BeVZEi4EOdIZ<3fh^gzt zjr2Q%NMm56Mg_^9U{EN<6-!N%1wq6Tq4}Dcbzvv2Bgojgl5@48;pxG;xoQ{y#R;f- zuKHwgule)rZ&lG^Q!`?hQ@EJLNTDy#^8<36)o~x+-9OHOULb;!c38j;LJ24iD2~gg zkdpJA>U6;7rj|dC19;Vqz8e&QM-#H3zz2>o>Qln4wApYTx3TNUNTs} zrD26&PtbLihjM`wrH3d&C1V>}h!Jq4;2Syz9-!_@Boi412U4Sj;2Ao9@h%=mEh9~( zqME8^L8P5JZ@qdEcL=9YN*1S@sa8g$eJ4w5rCFMajUaW@j|UQ%1_I)OV8i4;@s;+| zlXVQ8gBA+Ji@U8sd$d?Kx>eyF=!a{WhPt1_9}cFVfgE2AYk^xh{H~?Fe04DcRM0lQ|DDs}(k@@o!QBF5?*dLN%b+olr0(Ib7be>E5#jXiMQ!m=g4v zW~U^Z<`%a5g?NkX^+sNEp%6fKy9<`zdHGU${*xrYWM|4Hd>u?m3sX`}EmBL|u62Lp z1;_3$#AcTKHIrXvToa}A7WtFvy@_nkpb24IAxXfn{USjK-knF>&fVQm@sPqz+SXOa zkw7qgAq|-SOqHlHYjrz6oMU)GzLmlhGw|F?cmEq=gQt%Gk8%;zDD?;l=qn1s@sF=) z{@+d|gd!;-1$t8aizY;9FsS>#0}wb4akvIL9j|Z|{_)=#g3yG~T>o(Y`4tZ=Mc77V z|D@0aYp`m_c9zXOO@i={d(|-Q;MWGw)I9z&W}1#2c&*`BX=Uj-h_Cx{8dRvDlJyOx zgSat5R1JB89sodPPug%c&>a25F-~xIfa)6k- zrDQZBkHL1lYz)(f?h^v*d(Ax$%5C-CAeZ}2#ryRCo%VfWA$yfDFaZZZPlwlo+c7S% zsHg`jZ18Az?nm;B07s-@=;MO!J#XiIXZc14Z8sBsi=m5^W%UICWWe8Kj*)R}DyGf4 z_u**f11n9k-u1gO)tV_fX70OZaEfODrsxX?0!oshjK==J{6dQB6T}6%(6<13P6NXT z!?;~SRQPv+OiuzNH`^8-E@?`MsGT7km5(dM zdL4m4I0t|JN@UZl}mlsnk?cMxV7`2|auo=~_zzl>F}==&pcf zxa%+a<&Er6&67pEJcp&ZrLCLk8Rf&~16Vzj#Mh*u94CZwGe1+f^Sxun%>ko%wj@C+ zROuocc$=Ng-Z8OPROyl%vT#MpTp_uHQ1YbXKT3u;5lQ8)t#Yl=op34Ejn>QIiqyE` zx=R4^1-1-9anz?hoevO;91}lES~%5i3kO$K7t+3X8kHPEd#ppi7pZeKbT?y+mfC8U z)B@`%EIAEZ%@xf{PZ2*kG!#LP*4gF;vC_-&#*H?y1#=-r$n@w49LUO=8R$XVo8+r8 zWQ0-67YN-*nHlk*Ul}B_GDB85egr#%vcd?chn64VJ9j=G!A>#K<{_=ikg%05WM#>S zJ&2SwGvOhf!cKX)?kq`>A8|A-WJPZ_aMZwB!T9Kc0j|WDQF@>{6Qh;&MW{l+!^`_4 zdmS)-d1f6h4ovhZ1huq>guG+?ZB)~KXhhVh{QCS<=Wg7&)IFIc8V77zpj>yMus(sr z1S&OttO(F{P!3Zp4syaom`w|HG8)P2ZXsu1(UA(O5y#odwhS~YO>p(u)nN;lgVWM_ zKHI~^@hr-BUotT2hg`3gWBqZI!G7R_ganJ@^l_Bfz9dBVxcuikZ@73PKFZBVy5DPL z4lif=qm*mgoQ21?y572ho_2fZr)7+spVn^OR@%Retm$64d^l~jU7N{hEnXGBVJLOU zdp-(r$ZR?adCDJppL?gw??e$vNn;kX}Z||kTvGw2037m zbP;ZWAAL#gXx+Co&~g-B;HqVcqKhPrsI=C#Ge+YQo`n5w=AAstBAb>p_gSX&`yr?x z+mbb}rOlFgt7RjCNEGT17QjFj!Q>#53Dy-;jPZP5yx}0z=VTUH?A;jQLFSXqJ3(xY zKlnirK+$jdAr0X#S`nVGdl~wzKeQtJ3t$0pzu|`vgm*wA$nC3T5ju;#cQTrTe+Xlx zQ6v;)cUFwf*A!kFEC{^rYK)(OMS<0@z`kX{Gsl@Hv$UTVzbEq?ZS|820OHEr;L7|Q zh-&ld4aeJu&iguo=1;=A1D=c!u1Kj*aS2Ttn46}$iEx;ffI7?16BeJxEk2J~d>*y< z4CS$Z$dF)jpT`KhTllzrN~TN1l$thkHa2_m5Y~zMY`wRIdshyP6mdw_tcK!zv@oSVQT`}s||up{+p*xl?39InQBzsVgwGP`<dd?2b1}G3k8G^b#@npR6}5#8wZHju z?Nu2JewhJqqAs0rc8=Kw=gd_Z)|D98=Eg3t`a+BKm3L_;wbKZ$O-d|pt0p==zTv#| zlozZuQwo%DhaKc{A@8_Rjf;j6| zv3tD$19?)V(r_Epfkh~pBCdK8S%4lD1O;Ngp!3MQv_VA}1>)y8BiHbM83=Ums9N3P zPxH5DmK2tS0jwK97_xe; zRi^-0tY_Y>OWLV&3mE=9Gv%17&;(e0fnAO%m0?C07#7fS=`N4lL0#a1~>pBp?!4Dko!N5zg)YZJ<@Ra%Y@aGu$&%;N) zr_kY2F&=FDvjl4it|OdVqVZ}1`+#aywDt~O_zmgpjMIS ziMLVxv^NBL<7X6wd#SW`9{v`g4<6a}|NHEMww!C0gP#)v~{wP z%GE#-uMMJGhU~dei?6vK3Co)mKAmU=J)fe7v6{e)A3MAvi)){z?U&zNKozus_&kb{ zXyi*E6>hgBOdauPun;bZ6-)#1thY!#;s(beaLXyUCgRQD7+fQ3aiPlqU4XObGNK>n zV=N`hu^Ec^8j82*J>mr?#5#(fC`^ajn@wXx;lGIS-X+6@ zgW&?FmTTjz{04*=5)rrWHb@+}B9bDv?@n?z>~B0F2aplwGh=}YD2`4Cx($fHs2g@z z4RBl`9-#w}LStA;A}5d;P*rG6d|xNuSx}vROnISWkXrEH@k!E=(EzXS@%=L`)!JnBv82G$!sLH*IA6rHA= zZeFPLJOHa=5#U`DN3_3fv!TZzsqm{{-z7eIVGIwNUDfgua&GMbM7%0lsTD34YHr^>3~vor z-3^Bsixu|Gi;ETuxCzD3U=cPzZVn2frT>T`YG$8A=1hVXomq{}Oq?3OmX?doOpY49 z_8%hN7Ve61VT$6cbcPeC+uQovIrfBPNn9Ocy)>TVR9&u_VDLtU$-HnZd^ z2K+;=@a(j3!c155*~psptFsE|w!>C}dtBKW1>_{+2BKl;3>CbeRh~mC8;^5L6n;J$ z_lFS!Tj)mY$M&e|jvi=Sf%)^UU}O%@_YI?tyR+d5Uxx`F;uvp|bIsd^m~pex#$Dmh z))DJ9o9a(0aa!P79-C{$lyb?qPeGkkLKVZQ#L}xrHHymWW(FSZYI)Q$N=D%*gB-f4 zHzM!3NYjxe1|K(Hf5u_@w1cg6_H(Yo+dkuaORRNu7&`EB{SQUwui#5L2jX4OpZXcF zdaWRh295>(N7(z1yem41S5O@A9Jm}ytO)a#&}KdF*j3eZe$9a0M`#;Wt)fdxU@ljl zjkghBh@>H?#Ppcf=OVDg8s|msJumw@7keTMhz0(qa_V8@oI6_>r84jpo zxke1B;_~c|rY^S48_gIDsRGGs10T$*1gS`paovz5{fUUAyR*NakK-bC#2D& zUaX$19;!a-wX=A~#l*)prk5kj^|{M!SBLRRPRnM{Ri)-aw}4)CQRE-OMO|_`3~Kh7 zi@wJVTYlKqD44ZQl?p&5vPhGD=!dLBY??8j5a*4bWY_i!m0(ljp$x=YTod-?lU$-Y z^`$~rfjpCZ&&Bb>r<5o4u8Gp83i~Wg_73%h)vUg2PwJ3xHF>l$J_={lGm=&DwupNz zu?n<8{w!b?>ki(FnE^f;iUC_A6O6oQymLF!*8ZcUG_}fPvS%*L*5fZ^Tzu1Ne}cj| z+>1F4DjdLl-!$;h*4>_qpF2IX7-MX1^xM0}3j`pe(z$hTy>F8Eps-7c zeC|ToJ-ArMnCF!Lbl8@sc@#5VYnu%)DIyCme1hJs4fB<_GN==i3f&oBNknercuz!#4u<#!9`L9lPBWC#kBkyz+hijspxrD>d;fA&L0sPC`_h z-bHV;(@=jeyYCm0nd%feo7J%KbdJsnoBlLXmCz!35)~)W0iS%HmmarPZ{v^1(>AUi z#1^mZt&VmUW`}AjlJq|@HvZ7lvl`p;2Pwj*TasKAdz~$UQ})8Vh?97IP@dN61?~&9 zR1x|KwUNa85_zku^Z9_C*1Z|;=924h$99^3ce_MQxiWyUlsfax49}~7L)mUHkGr5$ zrIfc|P^FZ=;82C&<$y}@jg5+%0RNCm$wJBUb>XS*#$|8pSC*IHlDG5AqPb0OC=Y1r zPyaB1=vyvD0_c`4p-RNv`#ca`4p%kt4E}g=lWg{xh3LHm{g8pD>x2(1qe!UCZ%)wP zd?@wJqPdtr?8fh}t4=4Xl_K3O=>XugX$YH)quz#3)wdAXyQv1eN<|SwcBJ|>sLHR;vKb`#R!980!LC1is~N9N45$bRz&&*mGt!42bJ_N z#*vT=-yey4PDwPP=>xMra`PQ%<25bKrc!TSD0F~YZ$JovIDdOoasuBEwbS{%y3jL{ zcDMbs8ivEx_w2ixMYlO1`mS^FW1qRFX=Mc>-8|ymJPR+po4|&yP84D@1@$ z3u@tpc&tDWL?a(yn8A;SDyZbep=hdNVZvl6~@Dk`P)s(HNNJoVUfDec4+l~6vJ%Z19jr)lBQrN%5xl`7{ zu-4zGQ)6>-*`ON?t=ryl^!k)!b#F3O3Pw$NOCT~_7=HO^i3c{lS3HKS)obQu|Is%> zMH2A=b~aeJBJQ{s$!*Tg%O~&W`rB>r2sdDSE8Ty)Q^2GMz zCXczwCLHL!;4#Fkf2I}hlWATqbO9hZ{$gtGWL@@N_MUG3KY#{mncOx8bp~~Ye=s)u z@T(=`EqxgJTOeuoYOtYA&zHR}Ek?m69S`&0Gc`1S9lzqP&RQlOkDT?eK9~ z?LcG01QBBfQ(<-Mhx!C|AdZ%IdY@IdDYn6bG6uGw92sQ^15QQK;ZdWCcMSQKnFy@0 zkU0^2a0@3Y35g_y=@N9P6ALsGN7EcvvEun618{$JAqL$lenAepy&oc-j5nXYh|0MZ z$b=R>F`%js5%G8~xFXFUoECER-j{H6m!Op9QBXEa9!+v=j~`*-%|WI*37gFPLugJ) zCiRB)KYeao2>+cF^IU;MOr@WD*)Yb#chAu2GiCStTZS3DuHlN*UCM!4Z9sjSPYB#4mm-dD@5^ z*j%rrDLxkUv!U?3m66O`VRQ#p3(pf-qGayl!U#}`L57bOWPIhBbL5$6HtCGVqvhmQ zFJ{D;PJ%m>d6`?bZEe0lRAJl2RZDqPRsi2X+}I{Se}sK8>o(viT=wZU{fmVe$Sfez zfOzq@y=okZ&O)3!A>0U8_r%roDFdweyoK{;CApP|>D;Y#EaEnT-i_I?@ThFc2A^5p zTM+LFVsnv_jOo-uTkzH(!4;ne;wThf2J$F`UJ;LnND@9}cM8y4O{A!$ia!C4RjL)b z1Ry;E$XbNaR*1r^W5hTmb8*Qg5Wnw3NnNA=lX3jFeoq_wb?RuxmF?-whlOwOW#Y7B zV+CnbkTTlXosSn18}62RzQ`a;VoYH|0NDJAN?DTD!zhJGup#3wd7MilCx4lW1@5zTQLaFoqsZO$Wv}!3` zQ-hw{uhlO&K6@{}KUTv}Fo^L24sj`^#;>zWA>_c6qI%I9!>=nybp3oQ^S#gKc^zT< zga8Mh#2+~(dSA>4c0PT3RL^T4VY5pU+*i+Oms@l$Q(S{Uxe7oC9717GD!0KndS{O9 z{nm&jC`lzBEQEc9vIt(o0wKeG_G#ov{i=P0)UnKO&0@VVh02=!;>jepNv+R@RcYBY zw`u0L%@xdw#cpHj!;6_f&Zn7Yb@q-CO2@R?O^&Sbib3zRnoWU+Kx?(TB>-llTadMi z)^h1(W9f4RjZc;;W@F2+!JcjxCQU zwcqpe`n0SWg#Rym8LS`$~YgcIfGTAs<9M%qH2KL{}i0?sMQDlVj&74 zV;38NBffutLKUnDo&}or7q}daLUujx67%1MnCfQybl5B){yrMgsIR&I$WM5n^8FV> zU0UZ_MpTxVz=`-fegxEQ0P!w6`!>OUjCIcB#0iS)j405IQoBG9Dbg(f`uQ*d9R zP9O}I7)KyKfGHc{HXn}Sr=CvB`p@;BME7(dv0pik zIg+dyR&-i^PMV2{XO{M=B)Go5l=<<$z0SC&`mFIVx6{h7u6tfE4z1VpjAXr;ps<{0 zC@HpjpO20zDx=6-*{a|!PeXCe5Dk)xLY<(r&|U8HKg`H|(ZxKZ!$A%H7?^`<_(crs ze`NbZnb=4WzQKr+c8~uz%k}@^#6kVV{Qq(SPm22AZT@NAF7ki1`KS3v9n`_*31(KX3crEZ6^s^R&x?qUB3!nY^T^ zSLR|)k=tS+Flk@aCAXBxHi_b9rlu6_?vz~es@WIKZiP*82>zDrgwJ6Jtkc|k>bmJM zSn4|o^4C5Nc+PE;41(gGAu2kRyb0h|7b^zDDvJ%0N=MNtB`accvg!6c$EL6VY$vMM${7?h=972vrEG4hsda8IjUG+|=`Tlfowl!W8xu<%&0At& zKB#2O@Bqst z@_>sIC$?k&3fEhp%nTa{Vx}hTc?mfpiPgttDIpewZ8ALPjzS_x=_EBkvxS|{zu>Cw z`fY1$()1Q}(LV?N{wnSl84lr=JC0{+^!o}r5V${?7c^Ha%dI)oIdRL9du|SWtJxY@DGs{%Al+C`Fa>$iP*vCG_g_B2W?K~*RzpVkm4nU z)XESyv*FVR?I>$9$VtlNG{2S;)YX&IBby~nI+{ysr^X~sI#KH?giSsQe;ob$_BCCO z8ToXL#g-<>fht{WgRa}L9@s?Rl`>zIpo$WsfkaqPRybTM>N@FoBl}nVf*gp#w zyOUP<-@6oa(N=xcIla5ucd=5W!R~?k7Un7{aA+oE+u&1o`Ea?Ok286$55-!s zZ$gm^rLHwgH6YINw6Zhz$jB)bu4Zfg(~EJ9Qw3(rl!lW8exf=#SGmjg?fzV>FW|F} zS)V}(txh^j$$nv1bj9Hf=2e$l29>PTYfKN;#a5g{M)AXc{f;3o9QGaS!w1VB-VLR= z+^IvctwtfkR2o`+us6d-YEU@_mD6dw>`aLqBL(zRU5xl*`!ehmfC=So=~&th_k=lE zoh(4}Wm|lneVB=xBV$7)V1P-E0p5NVnMIjclvKC9q`=(1#FqhM+(}xCj(xmfx4Hh9 z4r6S`%{<}3YunLF;Int$;Pc0Xhl;~o1_d0X>u7MKB+t>+wS=|}s;z84fo9?E-8dAA zKv=_o^L&zRbxPf}`YfHrjm75ro(2c=5c9o3(5bO(UhraC)+itNzYePp7Q4`6G(mx5$J|Q!8?W<_AX*$$Rhkh2*b6T+c)}Z;_OAJ{-!QDzB6S1vijPk-w zQ?bS%QSvJ}F`Rr><3e$hyHYS&8Nxt76u!fRi$_pRJwY}lbxvkG-)4T&mQD(H=7F~N z!MxV`{TB>e)?Zg9>8xyUd}@Je0g_9aFU|g+IONHmTsP$vU|M$4vruVAafgb|}cZTfU$){)i>`TkAGb zN<9j1SvMwOHe)fFnbYF%qudJ1%3;n^`ic$4c*SP(!BB~!tA#{HEhXie8}q$8+~E4x z0=~hbw;f?_?41KYNem+u7X7))-w2t!tgx*q=^6F7bjER$!-CJ3Q zELOQ4n+6sv=!_k?-!Cd1<8^BqZcG=`;IA%Fo4cg{nuVBReETFU|m1=-aGBNPLz zIMQt2w94wygc1gIH((kZ2?SR7<C2uHvr4uf!96sY3@ZcHKNyyIFyx#2szHW?KR_$G==x$ruJje~2`er({V< zkw2*QcjeOjom)FHMXjF)mfD6rol6?&ni)<;xZwwmw^3#9)jJ(Ap*iejRzU(KTjuCuyKmIou?Ib;aj;pcZJ{<2+~7sI%sMtQLA6QwYf>!bA@6bJSBI|O-U zmbT&HihNgnh=w_eHV>wIzJjK^Kp3ZDRTAREqwo$4~;xZDWpUQP}&A(ZvGcxw; z>Ya+4Ry=DAC(9n=z=%i;%L-3B0Bedejg-5ecn@Y5oZhs^+{!Yo$q&`#R!U1gadr&$ zD|OwH*CvC6zF~arHAsv3$ZF|vs)=_+5kRKspCNJE}>ywvZcuf-&Kzl4@nM7j(7EBihPK@Dl1#O*t&W7;VNWZ5RYV()<>9`{gzueTU#;?A9 z(HjF>6mO-8&+C07`y(f;U7JlgA22~{8#BA#Z{@9Hy=UK4K6n@d9IEkmgi$)~}n)y7_Zefa_HBKVu8 z|4D~RG`382(1IM$V?+SetqG%3Agz8BB%)~^GqYIdL?EmRe?iw%a1Wa zW3B=VrbH`VF^N?+bF|L0*>fb$&eroNETTjwUd3lG0T?3nMG7yVQZLmfc!N^VyR;9| zXUKx5-!K|=_SC)AEeQrDf zJB`F;@9x!jWN9WJ&634_fl68h9Yhts5Bof2CZskJZ(eP2V1lWP&sq_U_yBBO3R#fs z*Bzr^D&#DjZ8Mgc>7rrzGUNOTr7EO3N0dY9vX$47$&JC|&!H=O1%~|_l2=H(sJ_Sd zaz~RH+qd`E#7Mx2XikY)r-6`QHsLt(z750A=1XT=>o=Gy+}$`C%4LdW3V;I~k)kXW z=86^yL+0RzwrqVAPfURHl~A>?d%;WLq+4R5(I`U%SYm-dVOe!976P)(@z({r!7ua$EROkWcv z`s-jYqkgRPOp8|xeJ>;Usw10L7L`qtr=Aki6)Hn2MN78oDm#-SPfuYna;%(`Ij7RR z$zWZh4FjQkGvnJ1OC_n;P|ACCAr5I^bS>wi^}|M(>ji64Bi(QPA% zT4H3Cv8E?_poZow zO^kHfSUPt{s@~V^2=-ZU-Ofum@HW(VO*&_wNQ3nd9p7Q5$iTr8z@`(7P4gDz!v>Mb zo&RJ`g?3AwqireOb!z`+hZPA7pL@_AMN5~hAVCT@DKax%P`^z_*cqlWAOa>}+)CTt zrlT}ie0IS{a@)P;8LM$#I@5uUanUWUOMnG(*XgYmrU0qSPYS&jjVL&-#!r1f;D|1v zp`43fqq|q+($)L1r)_~dj(Gz+yS1Ea1ktgB+ru$tsbgwY`VMo|)jc@mRg0-zI{FLW zA}cqCj9Y@lHcuE$i)oq+wpGonStE}tdCA*2IP!|I*pXg^w0Z4dXbsnIH(PHK&%w>e zo1eC*AL1xq*J~&p!`WiMoU%(2OIj`QgN)K(mW3MyAfHaQUCOQC6VrCOs4BmVMJVGY zV?_)-db6g@<+aeOwW|$!(U3@1xEQ`Nw8G@ACNCmRIT~AJ@pud${~Eq!_u?5U5}nsV z$A3ifGBbCazosVkMVh+TnG3^;A*C~a0hU_GQe^ME_u=Xd7}{QE!u&fPA`HVik4SdjZr>|+C5{1 z?5=C3PwB1OJwe_B?w$%Y5u~Ea#UIYj1mSoMHo*|c>$s{qCpV&7QbVUAPRj05z++vy zRf<*kV_H^iS$dT`=ZY*HeIs9pQqsv+b+^crac^!**lDI@?Bh}?r*@H2T^p~d+caOX z(yw->^w|rNs8aJ!&f5XOBKW|`oh~LjB9=TxpN|T zV?ofdzDU1S$;lnsxD3KzGp99=tAA?0+d7xk&`qHS!#>PTIY4ep{TcTZHBZs!AV^na z|5sj_NBF!iVd29h46`j{@l~Ii+omf}&nn9t zYUhD*sq0Wij0kU}CK%ESY!kb+;Ea38=8zjJq!XAD=ahO)9Ob!GH)(39fa^F;3~Jk# zU+f7KcQgukwbPpt5A8|}WXCX-(`Cd>;9N?|dzCm;a1Be33R|rICF;xe3708-9n@yoR4YW6AX{+Wj+CgDWhQBCB)zBbrB;*%18Y=Km2hlCS#90xe$IQ^n7uo%`qIqE1$>=KiBUEjkU_NF>bFZ%oNIRkOKN5~V>DRC`6t5i7hs#P%Ag`p=2 z*-1rd9}I|b%t$Lp)|SZsAY@(Ex@$y$z#V;vmL9OoNmx%_4aAuqa+WO_*_tVoYA{!} zN>yDk)@Xf|TQJJ*pm8^IIOX6}l8Za6QDe7KkxNtMwDsbP?&+(w^BEz*|(Z&Ti}2n0S)P|g{-fh zCPgi=i@UBH$(@?4+?8qv)lPg=%2fudmO{%cI zE7<>XxW4;I8cO?0GUDkZl+416;4L=Xp1LZ}X%ab!He+LA?BtNF-1MkrAw*vlwGY<@ z#Ab{~dtRJVeY7B`nnf*Z?(y1oeNEYX@@Pc%|1kB=(UE*#xOXNKdt%$RjfrjBHYVm| zV%xSo6Weyuu{xR9dV9XV_uh5a*{f=Ib^X<=tLoI*&-2-D>b#vVW?t8<{Djlas8uO3 z1+3C_&t;yy<8?`QvzJw-yOJ|Qg@(TtuFYVAC74)cm8}~CE=d0%rJmJrPD?mX+_8W9 zxU8wX{QN60{v*Q*T{LH(KfLX2okc0io~FDcKQ&kf&c?f!9i8BB7s*(u9W_6D(v4`l5=`#8(TN^b(`gcPsmCWhGe= z1`MNB#jLfLT%vFRluZ>SQvk;O*pcDbcGa%jQ|*j;9v=89ga_kr@%pDiI4lhKuh4!~ z4hm73^i8S)#S_p(v0DUielRCwG@I`@+qHGBy+(}dUL)cme^e>Y9DgS^%bhoWJ zCCwjj>%(1cljbuoK;CY>@9!({!WrLR_&4%P;8gg+dkah_(E`yh*QrZODh|ws^Z?Vh7q~Z;BiEAAC+Q5~3r(B*IEGDc$|<*giwuiw9|w`f?lxw2 z)VCxDKe0freZ{=rOQ^DzBjec#pMKKUyBHKAuw0FOS^q}(t~fAdmZ(TgX=~QGOc`ay zdP{YO_TQt|TT=7UJNEh_V*2k|!+=NfZtQhj%!XL||0&n}E7?f2Fa+y^Z(y|Qh%+Vp zJ!>6UKJ=X$5MJzp7!OG%8HnEvQfRm>K74+IZ%Ym8eV!rOUA6ktWQEgzNCh!z1zkje z&MS6~(7*;UgCg*4{XQfbVje|Wxq7BlDMYvB_}kzdiip@Gf(V=V6bgyh&uFv;6ln4s zm)&f5Dt+#){%1^ucBfK56T;+sr0F`R;4n7&Y~zOMdqGh;0guMxbGbgVs(%J(}`GpBm$&QOgu3 z&;`w!ni4c-&kIwJ8RMSbmx5Pp(J;h&_4|%QEz-`KkN9&1cy^dZbUnGOY@5A#ALS(x z$iV_LgDf}qEPQvbRWXe#f6ivjHD6xj!{Khg4)X2?WE@A5^6~v`ncHY!=)%M+t-2^K zfa$az=69@-&hM%BfEe&hUQGVh6Blpe8D5qZ6E`m0LY(i(+|0{9I77fMlk)7n!a_X= z$zI^ntE33UMzOAx@bxP+PS1(<{EsrXj67S-%h_U&%9dH@t}=$Yhcr+30OZ8nlndqK zF#iWX1>5VtXrlT*-APu%nA7@o;4Eo;4}Ew8PliC+o&vv!Z-;e?l?$B5robAui)Sp0;-#n+o6D}bLcsERZH$Le4)$GpKcBu z#P}#kFwma(f?mg0@V8_{>Q1hP*cljgkG)F>LceWUp+;X$n$q;#>h`x{*)|e(-lKs{nu}yqgZV_7$gP* zaSh>{L{XB*QXX$P=ff2_W5t-GL^h5X57SNXO!`L5KpKtT~)rJh&=cAw>G%d36ygV@qjg;ztD)` zNkMP_ZuNHWuw4w%=vj3PKUSx*@BXw9lk`BM!c6@UrH2dYR7SnJ-huLRP^eUy@XaQa zvdLxM=t#RSmIz)yS2eDSuDRYo00ab*g?F@QPkJ%s*&bQsllE9eTx*n4ozfUv%>_2ee&S?If z1`Yal=2{(88mNp$Mys)Ob&H6UB4dA`YHD2)&G1IKvl=zG=F#v|GvVM-!AiXq?k#aP z(I|6Q=F-V*XYyGT+4r+hhJ}b-!#)1343gJ0+jbSpP{P{t^6{r zlnDD`f)yxG1*4ecLZY|Y19UeSIsHBFsD9$6?ajxNH;MX6(oSY`bCHx*jd!9aY1dL; zNX>0t4KuF4Y>2Q7MGJUF^dm2l(jFN*#XPF8TDtAOFX{0gc?Pl7M>UE!tvG24nfGQE zS)DmTP(Jq`ca~KlTU)tm3OyBGG6Q*8qVr4j4z#Jq0!6Hyy0r<3%P=bg{7XmSW(u=t zv>VT`7*yee_I(v|c&=8n7B!F{Bddhv$szCXEkh<~ufoWtPMCh6P8M5P^201(5Tzro z9inyq;63sGM zt0o+%*!xwJ&#tp*o&fWo9c49p&;aDj!TCQP94D`T4z#;&XD7XAuDe|G%~`-K0Hw2| z8<~W_iHVOS8$|&oW;Lf<`$UMBTmA&=F6xzO^`xc~bYB5UiadBE{@q}tEVuys z_e@JBjXB*458hJ;!(#N!=jtQKDb7aA{=gR>dAm691X%cyXvqVEBS>BNqD$Il;#=hW z2hT*o95aJ51;aH|1%9cLMlfsI1=){xzVHnyB2q2Kqcm38WtQxQv$W&s8O*m$zn$Ol zxLmZD$b3LnlOF{Xl7}+hCYTS-s!ig&sYtB~T-_6K&g*h=N52O=ckx34O^Z1pWo1w| z5Ber?_vJ}p<(XYiYIjKXHNx|S0%y&MC4_%=)a7xnnN9hlc#l*PPA^e@OWBfF!~_IDvv- zn!|!}Fh4g#_RDwpT!_?#^n()B2GuVw*>Ws$;j6mN{MRVhZJ;G*-w)eX@*$l>S_bKq z)NC*TP-?=!^W|kkIu%I{b_Kw-6Ee?0?}Z8pXn7~ADC7vO%Uq4yN3&EdSNr4Jkt?~4 zjmmjyLANki@>s_Uf3+6w9iAJYj>Y3r#pO#e%5kkpjDzXuqxWM4(dh{+G{KYjGcRd%DTSI)VhM*skv<80O*>>n2ToC{ zoAzsy37(*${zH`1x|zXSGvic)^aVP26z}U!C_FVZfo+`SlHE~L1k6DTgMi6A69szJ z6MII;It+B4As~&xj44&l-VJ(Mnb-NmE9PEq|KB%09za4AfZaeRp_}pd$k*L;!Q+{L zh@kQ>lM2Qja80V|IsMh}S_8=l=Y7(U+Z5-h{uj2W9zTp8>XDgw!{(kJLLBbXp}kBs}}TEiK{47g+^2)mR0_O~ z^WOW}&f2tnrc`5`sF7CzkA z#6bO>Ufk7$s#VCE*bhfYp=w^W3M1RN(#1kitg>+;oE`bYR!<53qPy~x3&C#kr>5M~ zN*j&|sT%nn-gWk#RyXUpA(nKlp0^6RFTi5-FzeSem3U1jeB>q#pXJ@w#3k-r9X^RFICvTuNhJ>Np@ z_7e3&HmU;UA?5f6YW*vl{eXEH&EEU;D#lCF7QO7*{`o%~2ul*Pky;ifaxw}6W;W?R zv2Ht8d^buX_9`$f2^FcfN+D>z3Q?i(fq}!AbOUA9ZXv&o~QuWG|S!n{#JaZwku6( z+<0Sb`?5l&57dDRUJLq$sB&fgp^9&vYjwxR2gZGpGQPXacPI&x>9|aFNTY-F>Cok# zD7T>bEMq}I|CRm6Mj|Zk={be`+45}#1R@GnWskH7&yb{!tu1ANICg=S$`XD7IJUdG zgBLA_CPvcYeey7}$R*`(P<2#`9Zfe1_6l>b8>^P}-#bb*S9S840GmEl8pgHTCGj)^ zmE1}LKi>B88)KWGo{r#UFHB1gjtz6O`OM`>1MwB%6}f?WF>)g719_&@_PrECxH3;+ zBCj4@HTG3*q2I5=j|;GLH95jvBu}^}h2ouwY{Q_=V*l)0491pfG4b9cdDYR;J>-;Y|v8`!vmPxZ23bZcx7BD5AQBs{8&U|}K{@AfuX|u&^ zUEE#H?|E|y?Qco#Ta%;vmwAroWB!}`z)l!`$L#D&RJDzT#;EpE)<>IM#51F*8x`3~ zihJHtv4o8YQ%NMMyldJ9mnS_Vy-ZYRYQgV(uYrx0ufJL)L&jMY;QJS`R7Kcpuorqe zV_#G%@}MPE2G}Q3v*%)>)9Fg?Z~rbjy`JiVqW z*Df0bRHYSra(}{#lc0u4-7WbIl3q;fF06W3wH4ZSnYE(Ic4%KD2XTjvt_J`R4!~7^ zN}d01wK%vijy$GY<)>)>v+rkDAlP-u?whp>d?N<7Ixuh8-#y$}(tT}vJ*nZETo!JO zliM60tO+xt!)t4F2HrQ``>f&0aUN5??c`)b?!GDlgc=v%D3@L^=e^^nL)-|7#pR1q zN5*VcNUNp8Dyj}9-Q^x~DG%3jaThtxcsTLOZ(PksoGsO;7o=G1!+hDLV5Z&~LYD9Qd-%~A|P!{yll08xq-IOSJ;7iJ6HeY6F zbikug-kt|&4*9CLZ{DjnPlN}~gdDfdgdeaxYJDYJ4J|O7yrI%AVadxJdwn^c_S3XE zMeJQ#O^f5`KG@(@xqq_kLiZ3 z`S3F73FC{rJN^TIZMomPq>ddGL7mOaCNfpcewK+YoIOqR;atB;cr4tbV%(@|&qFh| zqQiStE8KxL!N^PxZL)MZW2%YquP_?SQmu4a6B^9SAC|Vl4QJB0@3|W*Jxk)U&vxp2 z2MtFJF{?UjR-!~CZG7x{mvi2y#i?)BX2!m_)G92ov}sDsZ#u|8x^vt%A&zyUN+j-+ zshvdDbat8Se=+DNHfJ-)DS-Ifv5S|4{bD85#HFIZK?es*lMylJrI68~g;;OXM#S;& z=1@&5Xf^+ziOZHDeZ>)m!1b(v$X>B2k*q^z2=;{DE|4TFiClK& z(ig4b6G?!W1IDO=r!}Q1k^IWk)*b%h{=3L4l-(}&sc`u6Gba8%A@j#+qjIKknfh5; zl6CofZ%19XlP*h&???zSi%nXOW(RV7a5rI-s@KT}pM+lCQ-ZdWvH@%tE&6T_mT~Bf zbgHKfJjBoE_AhASK}oCqRs}6XaK3r|oG~NCKR1L(^H+!ykBEfe=;z8=EJ}~2f#Mn3 z_!?Ezhj0~{!Npmsmfca@5hQUE-~?3+x_kxsvjpW563p0o7R((Dfyqh(ql#l;u4atCxRpBWiI>l($2lNrwAEtp`rK&tLTSn&N&!Cv~VUs9vFOS5bsS&Soe>af6T)^EFvT+bLniSK3}$jFpw#3|+h z1{Uj;V+NxopGOZ9~{) z17Ikcp$8zZJ+PZy3!))_ZdW(tGAGdQL)6!JhM$ShsOen%Q(ReP4%s1R@WnJ(!)K&Z z@=aE&Cgfe-`~7Hqy(8}tw*`Hr&ekEiA%!AyJMY)bznzg4-Q>j9I!bT3wy48B@i(tq zM>M4)o8qWrW?bjXO#|JNQzA7_Xuq`H+Ik_f83;rKXkOsKAG6yIU!2YYXdcn2=<5ww zWVm(_l6f~WU!W=+6j4@LuY2b|8OtY=E&tkD!>}K6D${g5ZlqfrFDJh##oi)p+Gz}4 z(;T>jWR?7N#3TqpPVW zmI|H}Pf0Uj4DJ0lYpb1Hd{rgJ3T2_$Q4PK_@e1a5iR~8>&61e%&HH=zPQUa+qg85>D-WlUx+rw1xqA^iEAI=;?hdx=97d50WMAX9+; zl5=dY-;~d7+VdAF6iAbGneoMroLEsl6NJIMZz^UXM@zTjE?_a#3 zoX6HQQK?KkUSCsbJapQp)mZt(3xqw~36x!O`PH1-PBBeJ z-F7c4z8EY#W_C~CxtG?l`Q@AfC-?srt2(XOaoAdar7xEgfw=`tSuZYt=P+ksJy zxwt3tcUPMa&AMq>&#Vw_s}g(bI~v%}f{3D58(=PIu%ctQkmC)-NwlM%q`#$-Fg5>(>j+<117F)rgzGX%j|n z(}B4zkWt`J_v>_Lh>x{R+>Y;$nA)FHCkyq!L}-|qsb7?udEAF*^wp}6?d;Z zKN5$BC&Y$QLr~DyN=b*0)X6InOz4o(k?Y9YsWcM$_QvpJIv(_#;=~#xd?@~-yYpL+ zdx+?py%IaM%neoP)A#NVP=uqJwq*V<{l2NhmghsJ{J9`3e zsJs)}su1j#3U0>m`_m;O4CmHnHBWl-+o$;ZVI*`QfSA!jN#4=gPA?l4@f=K~@AcR` z+FS)T4xpGaV(fh+>4dL%sWOS{(m;+iqWw4XIv(Y>#)UNPd_lr(nZ?l2L#Zd=7zNkX zfkIl8Pkvkg(%J8jMq4=wTUl1tNCL~?aWA?{cc7VnE{L#g8*x|6Fc7CCK2heGZjgmJ z;68J=V|=0OkX*|QOrQbz;F)G+AZD=(JCT=+NHU%+n3wX9C7wZ1jK4SMiRjv;BBo#4 zbgJ`{alWEfBkr4sXOPm0jE=buoNr58Yh{{Vx2MTRrO*Qq&|Hg_sBF=2j2@F2m_ms% zWw`(N^ok{s5b(L_mj=I!@9MWSdYcC?OfHtfe^j96T0Q>FfH*cx7?V`{PV6M6`)`;f z%`a1u_%JEZ5g7ATT9p)5b(-7qocL6#f^9JtYn#_aP7kgTGAq z_#iBSr!hCo+^Cm7JIR3A)TeKw9^^$_yorn$?$i|(qg;v_Y!m_iH&~p&OVFrugij%4 zsFZkvR0`7f>56yVSiFHrk(-r^xdwW}*8;t5Qu!J?gRFS^agomqY|aEBuGwgi7>0V* zv}U0P9P2u(E8Fv2F~kdaF7EubMMpaKx=|sM(m66J=(xPj#=Dp#PvXr6 zO_>0sECdH}&jA}l3$&Tjl>Vp-j^&xUI{zRpiek%*V8P1K+hs;&GL3??-$Pjv-YE*& zs;foAO?1oHO0Y}510~5=@E9ROoa~6EuTCF4Ep&P0_|K0m7gjW<=l6MDC%ln-L*Fd_ z6ssmWm>#!*Sjd|}UpLDwOJ4LmOAQXgn+f`x!8*YU8vBRa2*=D*FyaKS9comek^u(p zM_W&6dQJnFW+K{DX;<$fB42}p3Fh?kydtV!-m=BSGf}Fni4m>7?!_|}NTos94$vpV zhA9*K9G<=?r9k3tbPgkxDEsVLw@dz4nmR}E0F|LHQ9M9}DFRVsDIt)5U^j3!X}9m) z(f{}J_0h;LOIx#N$KuBT+;)X@(5TJUi^PnYt3Ew>VKC^)*Vbv*}7A&9&JV6-=G3C_Vu#EK~J91RJk4mj!upU_9d@8h}!VwDw9Vw*K;JVaeT~V%+$wanCU$cq2W| zSKhaorD4oIP>W-fgtf2BBfJ3=rdqc%5VGzlaXde3rdHpwGS2OG8llx+3QnWgAS~fl zY#=un5ga<7IJ-fZc;TBC%qj*Fh}lr+gWGAANSuV)Y94UmkTR;53+wobE?8dduI&Kc z0sGmi(!oh}H)oX9Z}Z#{inqOJLG0c+$A1?Hyp2Wi3v2T>uG)N^j^!v%IU zTTBTNKzdW0%e(+8Vjt*7hDsZgedkA)b_?^v_a^9z+>IlxNQ%SKF->qgy3e3BmvQ?1 z2^GJS>cKdVz=mq78!?a>|Mlt})l9mC4rk21!zyqloTiZ#UZv=;s z>KJBE{O4^H*`pyH<$uI}x*VUJ&o|hA=Ga6KI0j`H7MW!*%$(9c1cnt_NJKRn>|^q= zgP)%PcFun$cORSm)p;kT#s&W1%CV@OH*D#mLI9wOtU_<`yOce55G`7Ey zGJ;l`T2;0EBy@l<@Mr9=kA46U--wFxL|G8!`JQrTl|mYNefCu7SZVkLGG!nnG+?4y z7Qgk<^y?g$Xz%P{P{pezP@cH|lqoC=x3q8&P--Vqlo;L=K|{rE(@p-+U;D?eo^G5M zZbRKV>$hSyLw2)&i>0@X27S`v^t|-I9nXFo*E31Z7AZP5m7Ha=a_*M9ohGz(9d-U;mr{230?(g#Xe{8X^S>r-;FBNQU?I5MpA;5QF z)8siC?+B{W0p_1-3UPPM)c}c3##@u-vJzikprR_5Ylygln;16>jG7(hajl?%2-MKAfA6^+pqfi1z}eyyJ8XitYU!v{!(VT! z+!^g%6dqTcj8d-jZ+RSN^RTft9MSNz{uq$VK;z!DoB0bl19eje zA>;e5nFvK>ZJS}yVd#cG`S{>8SA@ovU1yw4z5$WD^G#XV9IRh-mP;}QLhdmivudP6 zGpE?2(-mBj$>n-KQcbYnFk2NTWifKfR&|#GO7?Nbi_r-&@xfHGW|SCi|B z-YI$x*GD&OOzc0z9qRF%8iayrqO}Qq>4*cPKtdnZjy|$`JU9v`T|T_Ezms$f0=Wts ziE{lt8_wa1povH}@-{FX?oc{)Jy`yxCuu^E;BJ;GI&qVU z31_9R-fD}_$tKsNAk4<&)neiV>SnZ({;TMYqqL62FXq&C(s?+)6ba30@p#H?51GRm zdbvJmv=+g_Hz(kn$+}(5a+`c*^$5(>Cy)cUXt7wBm&vLeJ|4a?H=Jup`+hBgqf%4y zVT2`psN$|Ex!h$pkBK{(l*@c|LR?24A=XuR#UhJ7=pI|F)+gE>$r&BOg6xGaPUF)6 zZ0U05%R&;Q;0d@5uM zcp(}iE3hESZVrRpvW#iWi--KvM?lf>;Ug8z{>aOVzF8dqY?m2b>yT2}z!{zi_*tx5 zSKd3r^|=;fCRouj@;()-CJJr%q>?pZ{`n1{`@>ZLETJo1-OTd31D4w{iyrR26MH z7DV zdE2c*1_%_oQOiU;(9Fc$g_Z~)3a8va2d{g-4q!QW$o}j~hzQe6J&X%7-zQki} zO^HKQ$H@6+>aLYS;-0|Z)8uCemcb~2A#{-bp*`t6}w2Jy0Msm`$CeC8*df zk{G$PrkQh|?dhz&Tl=dn6|y7mf_qd%a90Yw=>}nLt*M$it?p!!J|`kS~|L@&;H6&aJ5oUtklGDj}wyv#5uf&9xlVz<HcHIg*-Db&?AHL;&@pragLn(d(jK|M z+P1c@DNFSJd!>*H8`^K%_=i>j9aK*R^-9HGb~KG8qp4*kyUKN<{dKt|wxHksRBwuG zRqi}PvOm(-8ceQrKp0{Wau+{7kDDYZnhUK16m3ZdHcm)`$gC*eRK?}Kv0y~eu$KOm zVm!A%8nO7dJ=fr=&+A?+dsuj>RzpsC?^J&PYOK}in)lSIX|l&dSbkg0aY)wM47btz zD6>Uv&|z09=8~nJ{AL=Qa-e|-=Qo7sX@}y=Xvu#B^%#XO5>v<7{*z&AB%n6qq09xU zV4uP6E1+6x#(m~4^R6k4!y9n6nnj+i1?%>MZofu%U-frAQv21k7O(1^smwApCsL)_ zfH3=7Xz0P> z1Hs+zhbQLd%9ZdM&(!-Lr~*Z-63` zd(BKF`ay~z!kLYF4S$-<@9;Lf@aq1!f=B|Nj73I{Dy zPXS)Y*NjMs^0BIckvG|GKalt5G1NWV9r3Ft@xNL3zgJ|$-3tOoYKOnvr)QxiqPsR3 zn>I4s9U}*X@F2JOm^IvPu!%Djsg{)2#5ZRSB|GX#iFT4rRDW41y^G7P8Q!{*S=lGl z_B~<^d?@H<@fwuYZU>H2#KDGuFr_qv?gEAplLG?Z>)j}1L+_`?qJRQ5QA4k%(jp>J zqM3FTWy|#(Oi*Whr)2Y`ZeTp=_)FoNx-&5elY(+5KmkqQcq+`oUswSWj z-~o-z-bFi;G%7wBVrR-U6QY_U8oCWiWn;vJ&q6Z}HtmubZuz0--;36)V3YLN$7`aX z{jcipYz|zPI1y99c?nAa?}+r?xh}jfTOIKjM+TiA5`{Jj0~!>44f$tqs_6WuxLB4k zP)3`7Q&W+Uin9!@SFw(<*iuYR|uV@;FCxxRoZV#UD`%Hnm+YrY{k4mv!lNqnSw0(^8&Kg|SV zHG>La*R)4pz68;U*P{II5uKb%;yct=rzLrx5F~{us{PZP#5e3I49b@X@u?LoI!?*H zP2G`~F=a)$5$S6XA8-oB`wk%B<+k6sgpKVXSW!)b%TTF*bMtYjMasa}@)VC{~jDYR$u{4x1@- z<2WarphS)iE76}RY^Ac<7Fo%wtX|Rk#(t#%#SV-2-&`=F@Sh0Ttx~M_`dEBX(dTL%;0!ml_!Z=_l@s-Mfca@;9QLrVpJ zq@?|=vp+O|Kv!yP`46v?t>KOVQFp9rIv)+)O3Nj$f2#{e8>{9P|Z0(WaPB{g^ zW-{xK-YbM(0;#A+8{U_PgrQ^az~~V{lJ<5(&Gnp~rkw3|*h|`rjnwbqqWtPFW_7!u zE`}7Oh%@SSmeWmpIO3{(ZfKkYU0{iFer8<*V8B{9@t5Jm9iBqrbC6u)<-QW9g|@fj zJEx-7W!O@|^reRT{+I5X#OrDnZ6T~Nm^|FZdrTg<+|oe)>2Zh)Q`2XQ+?r&vQ=>k8KXC`zwA)i2r8r{?kd%`qrm+AhM9dKT=WR*VI7hFrm_R#kS zf@ z)4V(Hb|)_aZyF(te++oZ2vaP$A6UpNw_@Gk_$TS?)|1@6CZ#oRRr?JM@sD$yL&U^s zz}jbIuWXq8Ck3<}Hfy;rkK(+r=M6~3ovi$-TbXqwr+S`-jYmFaRb2M>5zr=RkAaX! zb)madAn;t#HlT9f7q*6CDVA?;%DsiHz!km_v+HK+6?~dU6#5T+`-b?1`)IDqj2fyc zNRi{J0!nbHw@k&?CMnluPgVX+t-GA)yzn0s8)lS*G$I~8tLO znC)><`Gdp(iV^aju>axrhI>hTnXb~U(E(n#s|f#%XcE$Y==xrZ-v7AOcrUut9Yd%m zj{l8+<^!5+Ic&L;f7!oPZjm1RCA59y|M+GUrT=KoJeB|B)Z>A8uK&h{@|MZL7ir}i zR7xi28=F?}!^73W7pA3)h2|xg1HFeK^_DXO$*#4WzN|Db>DcZGitnwk6M-qKatWoJpgNAg!OtraM zflei7Ukh*)N2jy-t+=6Bwuf-P~1(rIfNEDnzq7-?^EeF0_+A z7l$`0nb8>IKQ6#pkGm8LhLzb0@jp)NwQ|3>4~>0J>3d-IaiPpeT)DgPmI<<=_=!Em zfG)3I5{L+F{P#%~&A6*qj$`=VzdC7M%ZIh@9k%-aPinc+x!J<_ryvzC=ulV30pj2O zdp`E4veiGty(K63L`<&>S^Xx=A|1C+o<4e|ST#}~uQ(8;#*}~_?SG2o(sXUIJ22n171o9fCs7^P()?pEUkuPdR&@zX1Ow{hn|D`g;q%lv3v!ZNrkBi?ZcD z3wrAoB$NOD1jztFxMo~8LPQJwp)46$6F0jIt@m^Ytki8uU|(q~ zGgzqv@5(o~*UQ0#%UM@ux|&&0w5t4;rlv>iY5)D#4ey9U0uSrb&O=^qr8Dys19ydr zDQuq3xG$mUdeK@AaSJvbE!W))Iw2yS`<^eMC!F)b?l*D(sV&HZLaXGU)%)Dyy=ytd z92&j!6yyrE0Ml5-oWl|+WR$e?v=qgVuMOS=J*vM>oGSh8&IjTN{`^7gq5XN1(nGSw z>s|@SE}xGTd1vhkK-f26od5G(ec4+@!cJh3S($_75nl^XT61Cty~--LBCdXbqs}i; zc3+G4a2T6=ZkDx2RJy+HBWqnZ$lBwW0(aA%h#V2KE` zPZM3noLjkQ*OPXvgWFUou%Y5C@Vb^huB4@CETE{?eU8Mbu`}spfIArQ>lTVD^FyV|!&Av8@-HRA-^8)an0KMD*~ZSE zt08b`Armo9rnDR=Zje^`ssoltKbl5ZKkibl8Y`4V{V?i{86l7e$bx!a&v z4j^(DW&UYkW0`~I&j6^*lm7Tc#W3Z%VvYgNUc*0RrSQHgQTehS6{qn1ebHh5%2f)9 zwF|`zuD$t*>NKAzbjM2SM|m=TC;K$sm0A%{9g_o` z_|{!U@>Ml#WOV}KKk+Cke|4MC^JINTc9N)Qm? zW)U!-f}7r^+P2G|JJix+Q?hFpX0;fs_4Gs0vzzK&smivJ)y~Cm(RvY_ZJuMFDq3c} zJzBY_B7qOq@xn&y>jDS!=~MB>a*shhj{_Sa(SSwYk*T~GA`8fqx|(ioJz((zpI+GR z$?hpV;K#r)I>T%4QQd<}qRI;LNPhrQm&BCp>f!+rxZ|L)Q~smTFZ{0O?tMo+i;JXEm%^tG~=~cJdDr5 zm5&^R!#}~Ln8xb6YVeH>29u-iaLV_M0C&_+dRudKJ2;I3eOfPAaNieMRthWaEaJCT z&l8yNR#fxmr^^uyrY;Z3v`8r0;v8}GuS1#K2xq~keeQbbQXb8m5yG#FByeop{1l?yp_(T-lM+E>-yQef&il@H>=k z=JtY~f;^CSH^(lPQgixVpYm{_zXr2S2ZixnKi_cE280(4fqegmZzgG8KUhmyTVi<9 z^BK~+_GE<%l^CB8WDeg={&99W-RHXqJKx%b1srIaE4V0zKb34 zq-qH|I_;B|+Wzf&JI!1F0bkw~;PkNC%b4>bFC#%$z=L_k{-;;R6!wZ3b#qn0(1DPN zFG{`U>U@s`gN9qaoXFIa=?%Piry^zEbW4nk*2=| zTbZLftDA0DpNX)vaoSZB6H(HlTTvDrQeM^ezZb9<(3UREs4;{LQ z-jx1ojXHDV;F@qZclG*OzbOhlGA9-7jy!t1Di@O4ajPdKd#6TJm*n*F&N^h~#%Q2# z!(>AUxvOUf+V*{f_TSI0k3SO?Hc8Zr#ajg=ZLxQ&rS#{$ODEwHr3HT*=uLx~WYw7; z2AzTf(2ADKgoGNrr+Be*&v_|pchIIB5=pK1-ZWq9MluZ(lBZUCD(1Naj2J$TwL6Ph zg5@NR8)%yK|Ptm73p+owEAv;o4}{OEdL`Fu2_bL4|&!a<1bTv zEq$ouTpm5)){|IyScHqCJQaPx+D$uqlVq6|UDK)Td?>!>9x7?nI?XtmO_jHQ7zCW& zUs_UA5xT^gF(zsRnG)DT^)z?+>qxKyD?1d5jZ+R~=UcbQk#=s@Weq9xFxftL)gml6 z3oFAIN)J@V0&aM!eej}~@5V%UPh*9t{TTo51C&`MmP)gGQq=lViyOK}c$69&%}WZI z-2ZevGd7H1B3sY#R}LP>*}*)Pr0t0YFdB-dpzBXnsf_hQzNqyJkf>`r*i+wA?U6du z(}m}LVVVSo&-!}HJ#9@NGm@%2t$8@0WBk3TNwT|Bqtc&}+7;aJqsC~rU^B8Rf(q^(n>T*N2z zs%3iwurl&cB>{(POk3w_7ESrwjQSvUq7g7^gbzivzYyQYeK|>}MMWsjx=5a(z}$W@ z%}}HZ_2>;`7J{3UaYxatU;brj45+?*T-Ri`oY2)^~IUMWW$kJBA6`eh^sBjjaV^2|eVudWyn*ED&#Y@mx zkW8=SZe6+NJl$#vP{ui|z46txRcobJf>9pt<+nMRhnorTZPnS}b#pQ_>wRPD%l^SU zm(yS^iF6S)aqWkaumYEXMhD|Z`J%R}VNC9|odc+*W=CW`u9$}N)g zPR9}avKf`*3UNT0s?rq@^u0+CE6v5;N!sG!3H9S5&A(71%tKiQQ@XEE zy{||rYgPSQ0SdIh?Q5+X10Tf46Mo6IOXdqb6Y_<8Ja_cvit|NKv`{P0oqF3)JJkL* z{?+nxPyP>_rp7+ZbsLD7mZnv=*JM=RzP}f1VXha;5Q_K9UXzPp^&iv!AED(B$LYg^ zyI%LoM4S6_XRQP9wbR%5WjDMIgDW|6H@8Kv_NU+a1RxFIxZ$S(CL0!M=ilW(_WpbvbwL6{5lzk`Sx8T=#piR82 zrSfS8KWvT(u61XzI$5aXmk8etxgSv!t-)8#B2O}}WcHUs-w2^OZkIR5zRcKvD4p>n z95j)pU*~Hd9(U5V-;kJ2N55FBwQDf>|M`%o+*QtYymXCm+zr)sd^@nqdAgsP?>%#* z1X)s{j?KTkAK*?)6Uo&P0TYu?1xWO8`QKZ*1>)O(a+O2%=TqBReKB}n?BbL^>z@kv zW~n5ft-xX&7J_0o%Zird(ULhcl!M|VItbhFy@XYrR`u5G<6Sm|rjN@=Hs029DpG_x z_W3r?^PKgB_2fazcb-23zu3c*aS6ndhC5Bpq);tkjZ?{>O{w_*nmX&KsJgZfBhoD~ zD&3`yiZnxxfOHHwfYLCCbW2Jj-GT#xz z&cA2vweR~}m!W_z5VG>`l*C6Iea;IZfDr6|{Tza8!kE{3ltj{|^GWrUbaCQ4{&&UQ zJi5ZjXGxv_Trd&Y<41VSRQTXK(SEDV#r=G=1<%si*`UR`+4Y9b$WS&Pv&#zmINGEH z+()IkS#s9QTIoEe6#|dpxy3rY8~}LnX-Ztr9Ee@}?mI;VW;X9UOMfmGIF)qv3aFl_ zvKWHWC0Gb`+bh58g3r@9GOaxC^ zEUqv)Y=bVPF3zPq8^4rI-OOeJYeKoCG$u}i{34_?prYqF| zIuEcje?8PCIqKc6CqKiB#b11sC0Ja$h&EZjM{0<;DM<%xu$?v;qNy$`hQC6a6mwWD z07Vz*p#`90H@+ANBaQBqVw0VI=3rHgjhz**L7^%9jnLbL;8M0L-UbsXc);1iBZTzG zH^2xIqf>~{!I%AEi+H9FAZL^tI6mJtDp@(*cnq>v*9`m;7*pd!Bg`VXuI2>)qdzd7 zAQvVVPI=s!QEw9G!LR3zP0zd=zn-MVR=RB<))_`Y`bVE~KVpc9A(&lNyddsZVBapO zJNCa##`ia}vE#q+?x|xuto;CO;VZUV=<0I;cZ;JzOCh@eG+Dj<@O^O{?}YGW3libBthvxppZ+Ka_jQ37Y+J&yQ^ zI}w73`y9q)Q%Fj{d+zup5&%x8AIs9!oR5EUOw)7zx^q!C*2fIl%wT5Dt z2;F8un%*UAx>TKT0OV7^r zBRNjz6;$PnS&c4tU!&5DM?*vkTTmvR)~-CA*_n|_qf&xW)6bv z8foJYVCNyz*;-MwmsmRn(bhv4`l%^g<#B-IVGj^z2_>?GJNK z%t6y~_%V_SY`ALy&-~0&S0N;wqiJnt&IB<%w@^6C4-XswBNB zSopne>BBwf&iO5LxUQ@zDfupbjGY3UT7E{bD4ExXzQD7yZM7Ld5u(GtfekI!h8Of| zjEuKBICFpNqnNS9NuV6f2|1dh?%*<{*rS~oQX0cDb4Tdo!*Q71 zw9^w-Kd{+kT-<7>1^jdMz>6}BiUcMd8#~=1uN~XoC(?7|cpTku zrGk|4Ycg>%)+ZUx_f&gu3)!5Kzl@MPmCrxnNylR;EF$0JXy>=+V=?4IxV1C$z&moM zC)mG_&BukV=Wz1mE0T4V8LHj-s5@&5z!bgvZ08Lc6TxV{cs4y6)k{-aFvKf+Zm9rt za=oL9t51G6sC0udm3nz<-rA~b)k^1-jc80hhO+KQxpJ31YRiElpuqPsA)3)6xL?1_ zGHTJ_F~&6PSjPH-a>8X9KrZuqLI7g3NA&s z%r}w(mA-PBTI{x8`;0*v_mU$C&O&DI;}iQ;eWMm8>VuZNd>!0k8r#n#5t4Bz&D7G* z)X=G2@$)++5n%|pWx-Zf^qOx%jMac#lKWXv{kplUQY}|Vq*_Qs7DOgsvLfGbKT;co z^cqumfJY5Hu(CuoE0Vh1M*vC$B1*02wjcpEW)!t~VJT}---jtC)bSH?*9QB z-Aa?jB(1vrz2uUF$s7XohODGB6*4iWT-nV55!*nqp5;3~@?pI9?6{+_gvbMxg>Es2 zd)=`dAHthjc*_}cVy5UX0?Y?G7usAIr0gVyId+S3&#jwI2DC11oa`6p$jHYlDwVt9 z#o}~VwDG{+e>4~J5u=8Poi(~WD<9G?*tDz#@@$)r0>Zsl#JPz-!N?tU=`n3u79)&qG3!93QrYi{jkMlD<;02QD1@Zp)-guCHyMn`XRk z527NHV!!$=gwYJBF*Qk@*sImbrQ=645chWDi2pGV?BaG8w7S{Upc zm+J7y26nf4UG^ITc&22GQZ4a|@p2KDGJ*;&SEFazJnjWSrp zb9epxiH6R{6;%J2RjRLQlm8|IGg+zJ&DV5KyuLf#*4bKPwHTXd2ELsEx}E{2E`X%3 zM|`*|oM@1@Ms}{0KTX>qU+A@;!DMLl)LB(Yq~~l@jISd|zbk+t-ujvvwCAXIt=o4_sXp-Qx1zQ}AKLmGPpA^-E!1!jrp$tDfM z@9bi5MzDUT^tXOGIww}Fa(>@dy8X9|R0e#r7s4)akSTnnZCVGc1+ zT4u)GWO!R4JnmakU@A9Hzz`fQVee)>I+K@e7WAjW6|ROISSyN?6M-**Z&)koKUS25 z`Dv_p`ClvR7~Wqqt0Uo~_pK#+CNU{Xb~W=fM|qw~nDqN$VJPc)eoGhd1JRWATgRwy zr)E=(t2>EKOnqmrF1}xHc<>cpV0q15}of!Qlr^ zZL>c|OePn&f8djw&8F~GGVG*4AzFJPgY}Jc>Ds$RbUk|hAH}<;A|}acb;MULOLo+5 zgp(V*`jDUKGK~MRPN+xR$e45>WUVSK4cF$WhQar;yY(**b$6cpK*J(|0rv+~T;|Qz z@=`Sm{}UcIr6w`js!C0f)QBFrVJWO6^!jwALS(RG*h2sRP8cdvXj@5scoN8XW1X)@l#1iPWeU(_j9nZHXGy*53@DT_e5&Xr=5ayZ)K zgB+XNI7F-aqen+^y~U!wu$_dhpAWFCKD&80>T7m@Jjid^ntYFWxZB@B zvy%^vFdy;Mt4MbDTF}67sx037bYC(IM94Z)<~SQ@_prNr78S2G3J7X8tBa&{s>*qWfJZ%msL=; zTd-W}%qU%`FGz$sQj5B~%IdzJQa7U0*_Bwbt(VjKr(Cz_i4Z9i&06&G*IoN45s+eY zm-+B?eLyitivB=9R7$-1`dQS{M65Uv>IA62AI%w_k(I4>EgEhox0b0L1( zk%m#+)yNlW66z{QW9=bq;-2F!x&0Xs!tr_Taf_i__-Cn14tc?+4kE8k_roSSLE>C} zl1;}QDOTa-$)%#iR_m{Iv;%iM9R)UzGuRHR0%E}wW8U@tjaLj&Nws2SjtAR9U!t~a z%9{jfG6Y*WYFQrviSkJk``Y+M-QKbO)@}^Hg(=u3=V;}J9aJ2S>ISY0px5)knqGIH z0rR!qXFkkUtxX5GcGCLn-w`}-Uv3AefsIreM>r|iFEQt5y~AtTKF`d`t}t31kx^9v zjWSBmQyakf7uU>+=d>uF92$!nr|#_ovy632moibBTm?X`6*)=!jm?0vSx^Pr0l{yi zTgVM9MP?7IJ#nchYChK>fupiBVxdPwJ7VY>j%9{=^%%ikYQ z#}9K;=DfAOeDjf#GS8CA+VHz7wm9*qou_xNHP6=op+6(+CO+-=Sng*IRdy5tVu!%B z*_8hJZzhK;2MzB7(jcoYWpn+P`In~ouKu25wJ$$EXp%TKA%%HJy2ACJZ*%^(oi9HX zEY0XqmyI?};d>I-gbK-AZj0^dJMH1$bW;QaBbZMx#%&MneU3E!@0IHGBYXzv=%7KP z;tN0Xvg!*9)t9pEJ0E`6b3_lr858XM`(jljqdkMA7_6jewkumV3lG}v)kdO zN}s2U^FuA@sM+>~zrF6mTD+X23tcTCIJUj`*gC&1&iNbNrd$dO{D%E#`U2u6x%2AVOd80r`SR+Z|NYv~3=H z^32_lEC+OiYHD2qyBy~t|idUG9WCXr~}Lx=Vy6!3!@y60K#|jlpBImm6uYB zFCI!Papsq-Xsa8o&}+Tz;&;sp1daUKjHBBzI;s=*4j9zkCKbt&XOBz@Th(&OZc5Sr zUF7yI7;Cn)NC91FO+>L}#V@PBKeTc#%$=I%Y$0i-Z{QXa4VUf8i&NV|Jp?jo_AhFN z84DP`GTsqZ{>k0rkaPx1S}e>$DlKrlkkV>@^IKVEe{PNU$72FmAn6M}v(BI;ZiYoG zvC;Zy@q@R$%?yzMV6H%%=hytn>ced-`kxtn-_V+G_KE@MrbDiHoGVp3#qwVn=JjXZ zm#k|Gp-aZ=4r%!LeL9hASQ&)SEv|1Nuh7jUedQ4QhsIVZEslI z$M5Vb-aP>`1nD$=&rPO}h?13%?)v(95QdA$Frf1`8rRb_cp51JupZ&k|N!ml7qB_#2^jQ0+I?tDh<-$&?%)L4MPgj9n#(1Fnhq~ zdEe*ve(xXq+j}4TukSe5adF=3iaS_qUF%*qaDD?}zaeM^!3Ug7^IwmkY@pN%ze=54 zJsWuWT>B}95Lsh5Ij#@2G+IV&OYfPlu-3Tyv3F6}c)FON1D@(cz|rA6K+Ik>PE0wk z_>D1YeoVl3P2;bcPjgS{%llRu0r@^<)4xs zWdDnizr_+}39~?;<5M6OKNi2X#aB%W)yR|EleAZ5TrS7{z5by1q%HI#)ubBL+=JbB zy!HZVS=5oXwdSkneGjGE2b2K~vV+Pb4YegQZCootQ@U`rNMp{=gCCXxI*ld8L%eA& zOb#EKs|!I^x{&rBxrF?p>>eo7FPtN4)xC^V7w*z3oM=CyEprTF(=wf*|T1vUfblM4j{kS>JM=r9DxH+Uu7P zR~?*{iZ}O)Gi&*WwKTwYTDI`q`f~ch#)c#wfikD+re5UuGP&%_jm*la&E>p&9v@Kl zrixg${?d6R>Wj)oIE&g`nn$bK=A}B(`6`AR#xr{roJJjd*5}A0H|JB?>1(GO=CeC` z0SWjCZz+G3zp0k^F|wPVpKzwKm)ZN;C1C_)E~&%s-RGLIePnLVq`I{!djwW3kdfLO z`J$Mf%f@Kf^kHptVKlsS=5{qQ)*-g(B+4}qD{(GKv+%~a(x7u$r+<-W6rBpM0`AL6 zGiSA%Pa z#g2a`i&im4mi&BedzjzbK#^yd%ALyn>JKad2^4AKh}q)9|Cp^Es}wQKZcn=TmCTaS zp46UHpEr)x_hy83HvLfwyh77!b10#b2$!a7vw-A5Lu!VE^G?Ae|M}MHc*=9*-~9qy zuy$=eXwOFPT+PJZCJT~Ff2zNz?it&I ziQZ%Voo|L!mc8g(`O3NPOR6TjIn9`_*FN}-6qeAM+bSFk)OhLq29ZvSW$h9Lv7vT9 z#AaEIDbWcE3(%s-w#qQwdTm2u&M7+j4y|P^fIuSP`s)iGD|?Etju@PV$AjxG^&TwOYIQK1pY28TH}&cncv^Up;#5`+|E;8P{Lq-*7zxUw!2Hxsju!$Ywt8>QDEBcBQ%j+= zRhK##!e6`7wS^o7m)d;T26=$^(x1~ldVj@Rd5>i93CATX%b^j3&YFpv(Hv|Kx+-sA znsM@1Mp4Q9F1k~7Wfb`$b~K_<`-Z);Kjxx%N%(-5*@W>SJn?#ZX4c}AA6k7;4>}T) zGIM$B*C= zSZ*ISjveR6(P^tgHJlQ^%eY=7JQ~(MdFz`dQM+<-WbDp`&QJRbA_&we!_<@yPRf>f z0HZN3fWOL+)<`r+U!K<>uj2S=hn?bW30d}e>T?Q zz(LpoQyI4wk0PuUx8B_1J$V}jW^v1zh_`4{g+A^51d@GqLlZzR+*WlhoZh|ekxr$e z%=N7O8{x3VU!>YLE$tZ1fj{A}Wk2wtU#i91z0vWi1KW+{L}rc~$cK34 zhFwgG2a1qaIJSDRx`EB{z~`YGCA>)PLNtE9oWWKmyZJY&l**TDmWtZb_17`Je8G%% z!&Nur72lqm%lr`Ans$3{Ix!{yGW-;~6!)4{v4}n|y(`buZcxC>gHRROq?$Z$%=%pS z%RzHuRuJgvr)JN1M6M~NS{ndjC6hQxe!M?8m0Mm?j!b6TdH6Yv z!HhH<*C#L+C#DkH#WEJ+O2dPeq?pl>vqg#KPBHUZ(#a)~97Y~S9yTrK=Tekn@Oxlx z;Ke%AaV+_ZI^hML1s?DpRGL)EeneUDcigai(7rm3ny8B&}427YCK z)-jn^{;OSp)cb>Kwul-rHzq0Nz{h7uP8^4l*^oyj=svWkf%EQ|ZSS_J{NORy-3pS} zOlpE3V5Eeqw6ESOy^E1b7{DM@{o-dNFzfh=+mR^nh$&Ta0eO)EZf#y>h$euQo5{{` z&CDTIbP!O#?_KXGS}z(xof}5Q6urbLf6V##PIyX~AOfMqr;#L9nwm2!qC<~fqU4L6 zOyo7Z`QTv33T<1l$ipD~R&IkI~cMYqI#O?CNIV=x$KJ@yed9}VjoV1|!)~}0Z zE;ej9)8FO%^8DaUtvx}95tCzG--H`w){d~;bn0-9jrSuu47(@0Y=X7O@kCqX33j-l zHhWGmY{MM+itLZO*(k+hl{qap`GGE*b&QR}kJW#oDuiHjam&gdF9>=%=A#9BIwhDG&|SD-0WR&dALtP)CwS;3UPOw=9^SmU)Y*&B`JUSJDzb$toW&&FlUkK~H7= z#U8jYlz+-*Pa0OtQsFyv7|qg_3rLZ+E5IKuT*mnR?hAk5)Tb<#Znb<6ls{0P*C}>` z2&r5kvV#S6O2kwLQhRI(8>{0;DnZJo*H5;%yO&Q z=P05(AEpRxG(*#*?IW_7hGhoSt&mytCMjlLvo&r3>ye*591kdwwr!r5M55T`zTC!C zoFRG74)fltFeuf?eKN zw8?Odg{Q2Kl%CY&)F=VaL?3&f6~xSOJfe(r9Jqa5dopG;bL+))`f#^lSj@ea-gj!{ zt-v$1VQcWOyaw1p4JGw$#EOK^SAYk3^Q7b`?W-}n%Hpy8PhRJ6@s@W@XMM8aRX0cbbCq+IiGKmj61Em+6x54I zxbNZYNaRiA{h0@Zo`dFg)2->=Kb{7gd5%13;|LIHp+sm zRKEJJ($|iTI7%aCl?MZ4vIn2kVk(%RZ-lwmH>9U!U8{KPz?*37GzR@cXV^d`h6q3G z?>)eamC$S9hqOJh=k2PwQ)Th{DaE7k=UwxuwOc$y!r_5Nuc6Ai>p&t;&%g-d36`zk zZ}A^?ji6aC*B2jOx&`(bQdx3DRwi^#>bn|#Z#1g>k75L=4l3}q1anbQx-$iB$ z`K9yS=dUEuI`N{gKIdf)Gqu40x6;`kufCZ!dJsO+2vJ!6R7(8>5HGD8_D(iL+5L8# z`{RcRLVwK=Ra()&bHYVUbTwLOAL&n+QmnECfL!TROea?4LO_8ytxllHH;&DG?#7iER6{k8 zhhlgjk|1_3)dB7KJ8EClT@PeL4B;Qc?bFftI% z`{ZIZ*IhcTV7VA7ZgewX2R(uGn`uc)~2%(amArp@Y`Xa z#L$bsDhCjxq(*}5;!o6N#EHI7ioA!diqZK94Q)^)U_I?a;g$Dm8L{hw+?F9-TI>Ko z5d5|d>~hqZqU>m9suc7(79+kAS6J4l@M;Sl9Z0k-16E2 zF@f)Exu-L{Is)-)ZlA_LJ2ii)wiRVCP)WmhG62@n`BGk|zrN^`ex=tl`w^i=;3xe6 z6Q3FK1rQ*OjVZ+}OUnuz!E;nOKv&#M+9}H(9-wvB9kS-d3kmw3R;X=b$o+}_)+~Ry zEz!qD6313j`>9NhHKu%lXfOQ>l#Ulv(Vk|T_=WO?RON-Qdg)&tp<6+D1BdM7pZBO8 zBzKQ%HCkM4Ryk{RwU_jLHkuJ=`N zC9YdQF_x;2)v&`q#~KS9>pYx%RT+nsa7gnilva;>VrP!-NHx}JXiHAqwuM{F> zHSuoNf(xG1GNktE_$yB<2sv+wjImN@NoY}xl7Be=R_)H$!Y%jJeoU3;w(@r^C7(+I zA5Q=u(4VhWfV^C}*J{>c!g7%Ok?Mg6!7jf)dq!ya1^-)#(p+5?8*x!%FTQF*&sp^~ z+f9Hava5|LOHw_%d4)ykNpK1#4o6D>ijR@&+l%$+p=XRjTLa(N>Hi{8*V z#LhPs)2$iLU26^K@$rj_ne^#CK_ZR2fxpOCfqqK1S&^)`@aCZaKzu_+@-8 zzrRl0(#N;V^-WiHkmwgY6s4D6uSt#jU;coca!({OG|OnZpgge?$DrmxePZ{E-sJK@ zEE?|w?}Qd#snnz7!NDx%UwnV0%a|ub#1mb_Ti9FJ?s%LJPbxTk5?$r_{vhVWLY6?b zDh?9A()7ZjErFdrK3wcLQo9SI9L*z5IRBRCY4XA^;bHnyL%6Aq zG0$dgQdOhgd_L*vgaVo2aS1!Z36#fUb|>jxHdi%De0RO>X23bLT;L6Ra5EmK3#Cw` zKV7HY28<1>LGsk~@&tRndgTk*ffd*scv6l{UYyP>c#GYv5cPaKqy4LsLIk--d zktxxM@MV5rML&a7xp=mycery57|Ixy{1!4S!Iq5%JRg~=)AxF*yl1naI=YZrI<|$Y z(DnhXlar#&u^EQ{(FeIJn&wq$p2EwTDvwsjw>ZSTjyTvKg|T}+glpp3>EagalQP$E z<3ioVVagjheRe)&HVsG7dh%8*--oJWca9}P01@V?uf489LV`1UMcS76LDF;6}zLV=_G2dX`3`bB>}S#kvSLY={v!r z!DR$hHmg*otK8gN@7B&dLOpmq7H<8y&#D8H|EW1IW=uSh0Z#2|Gv;JIl9QI|Ce!u! z%=H@@PJ68-jUuMg3NzbTc9%%2xz3>^UFjZ3M&eV=N=s8(73{RonrB@54A0xzLXt)d zUOX*k3hHhnN*Zx}Q5+mP%295|81y~Z@#rKdy;b*@d#LqJ_Mn4_S=F8&it0%njO*Iq z?CRO?lH`VKpWGKB*R9X8G~L}jvz$n>n}R=`i-^D4Ko$MXvF1eNE|6j`K8zi-LB$h3 z+;7iLqP@hkpPv@Wp2A$m@H2=fYny|Y?Zbc(OaomJR_4pXw>(;0zUguv!ZfLxV6jF7{Id-^#?#>i$ zY6sFgFsBlHyD0^U8FXbI=J1D9Uren(jAt~i26)Zz7bRmzJuY-WPKl}AHK5ouov<~W z^E0lFaPxy-efgGLbR8NaYJ=CuAPZAbx5>=UUY&7fN_sE6KyK0Kq56x2^W3#%)0$LA zvWWW9-V7v=or|T<%4V%Mq$Q#FIMKAMMutk)1}%XE-8tz=vO1Y)Vs7-)qi9^NcN$uw zj!D0sfyO>xmgg0WZKBp4uj46*JDMnjpQTPgXng(I#5m`LjoKt)(BMu!FHXrN{|vXBNI{zNpi;z! z%oC!cj!P>6xz?3$>{c7W152Ee%0^2-ZUL7v+juP!vT(qy%r%o#CuhziCw{vcDWydR zN$X7@23L^Ka7RIq;D|`%Jt^Kv2b+Da&i3Pv#p8T*bPgIt5P zXI1q?(qcQoW7SEgAG`+g-`m-a&9QuDq;sJT1|Fw_C(C=nyY7_7bwP#Gn!?Gz&N(= z$+z9lRQAKvP^KP=#uCc8u?PpdG|9Do#cy=&F%MR$-Y1`x()K-x&I=PWS$n(0S-Z)X ze0uSStLTlySi{!d)~f=2ZI1k;#&4OwGrsa2rKG#*-nv-jk*)fZ+HVD|NOZEmg;YNA zu)&39z~Ut^n#ZV<5z_42*+Vq(s0Ew~fl{p48sI2#RaR{A0^TlPS8vjL1!jO2iJ3%M z7Oz6ipAk#jWF{U5;+sdUJz%E5F#m5qwp_;63j1-S6mV}agj}BH#$+{rkXOL~1NXTy zX~3~%PV#wQ<67BaObf3SoME{w2wd=%UGSbGfY`6^;5y-o*3L3bNp!~$Sg)L>OnpfG z8h~bpY8TXsmGUrUVIKP&8;wb<#HRK0Ue1;Cs}V=CB9b)E*&Ym6q2YYx$0zAoM&N7G zFJ|>aohD!UN-vqnrGqrSY5P}oM~N)q`5n+qkB8?|1mToPFL zNHH3PHDdk3J=#p2A<%lW=@-XaTgdm()0>C&>i2d_b-vl9YYQ&Z2pb#s@s2+nD*uqNqpd^CV38iaTzqzkF zuzL2J-BVdS0k6;_krqtPo65@QvCWtxmVjy!UF*xSfYL0=11F*-J>GFWbNMol4-c+5 zx(!fG?LzXn@Q^HJ1RuhPNg8wZ`2tH>yk9JQ)W{nLfpw+cBxbwnqQ6Ou_gs$>_@yMK zD!c`@u7Y~Cmhnz=_MVgxPILY~DI+l8+JBqvK<`8TPImoe5UW^Iz_VY$p_G}_=k5<) z$cAm;!6-1mF@c;&{!*y+-CJhQMMdO)^uJZEUtff*eiHt2K=D*`472|$ETH7rk+t;b zx2D}=inGMsC%+|YGhC~G_2E)&$<~M1q8N0IIq;MEEIngcFZ8hFZxJ@%6 zxhpWEp|)TLmC;7E-V-#N4zlOHQkS68!*%noZ|oFXANe9gMm`Pv zu*qrYQoq?@hPCfZk=pF`6nuLMeceBRZpA}B=%O_(Vni$#k?>pesS$;A;s?D%&WA(wGy8B)rytgz9?t0@H+&DXMtIg|Fg3#p8# z%f5rggS$A*NVu^{J4CUUN!+;16S-ALbFnppHpSnPNW-;$xo#nsq!YdS1!Je ze3<97i@_n((!L=VPSo{)Q<)jO4-&GsfD4&*ymE=v7OVw{Hn`dY2mwvp11LJWxnyww zao7hnT!=qS95VC?x||dExWdjfrH#WXc%`r+%}S(E z9u*d-#ik{QB%v~G{#2&@Z=FW=1$d%ct>(Tv+m!Y()=F|ao|}42RCk~s{HNO8Jm=n0 z>i@Pvbk%9S|%{DtvQXPBMZ5n8TU z{@tvZe41KwL-O2%_IWyzbK%{{h1X? zVM&i9!(T^TUFKa#nKPwcNRiWxXv`(lGP@Myp0jdmL`N%q(lX5^pzoihc}!Z+#VN!7 zv`HMp2KFYFJ_oP4xkvwGqPG3->bCJ2X=-u-i9YAKNu1PRi%ssr+Aqgk z4-Xy8fPG}SlxRl!?>P)m3(8*U&Iy?;426$)c89vj^RfGb!r4hmTXBqZZ!~|4kiCS0 zH-i>esfL-BJKN)xhvOP0vE0bI`Tcln7FHd|`L9HmR@U1N=!EC`@ndCNIyzb~&^l-G zl;gb%VmP!9PnObF)I?aLgFonevsyz@%;&=ac@BlP-4(bJQ?CsF<-xHLKZo#3y!KUts z|DaQUv>!HWgqq^5z?I-i!-KyAoq?KIK5Mbj*LT~vnsu!syPWXbQ5gO!j=>WGBaU%j z?+^Of;%d)yI$Ji8#?VRvZFl}PY^5a%`h_yn4SjlwGS@A*;5XRDJ@Fs3bC>hG$gd{9 zmUwQ$)~qyE2JUT30LmfYrwlV*xoF#=--)UDL_dvh<PfN2>=yg{*Do{&f0A^Ws#!6ekC-sp#5%mB{74}lLq3mR{SU*p8Eu(1bNAnmc|P0$<`QI5BO|FHW3@X7Mq2;^T; zQH;`Ef-q@o=nusO-p7J0sf%_zD&p!T#1y2?kdQ_C>1YWV)Ex>X1sauH&Lo)GY1R_u z&rx&4Q3X1ROMwadiBc>+PAq=7l8>4yn#z~UHoOl@xRC;zv5Fm&9kE60(GF2)kh}>N z;-EYlVG+k*rC3n~4PFI)p)T!XUXA8Dsr-a?c@8X}r>-RfmC~JIsLqli58{dShs==TPDO@fr_v3NXE0Y#S^KN7bV_Wg?Qv@W!8vz=+~^a|~grXd-Z@#`JUB%!YU>lj?*Cpe^+Y`J6$yYdgw9|roRkfa<<~2( zaYBJw(141;f=1eiAn8HM_AMWt3Voa<8UMM@*M%yQ|A4FO?1UHb9r{3Y(yGJkAvr>> z;-+;0?C1%IIyVES)DN1O89x^|qQuQ(|F7>;+enUdGYrWls&kjgb|o7IwwGhcjo)(} z?x}e6Q-jAowxBuv;j_7>Zdt^Bj#>2l>vofa(V!>|n+z?{s83O?gZ9~trTN#A-Nx_f z4%!DLS)C5r`z1JDMIRja_DfKW zEcf;FCJzO%V_Sa^x!p|98B*2ar%XgW_Q!ypeo_pLv*9ITDb!=aNP*xmLIN?sU}@b% zz}Os@Lzp-sy)wBwBKmj7A8P%V_9V~6H5lanyEV7&@UHau>w2=OsEKSM@nq1Bhnn8z z?b+x%Ddl9(U+;TyQl?vK7F-bD{t)!9WC^zq(vJY56XcT1j-P2RBVpTVL89DUpB43Ta_3aubi+di#KeAGBYZw9?3xUBRAT=QEKjs z1At%UEeM7S7O?jDJQhn>p%8|(BI1vjC?H$6HW;qWuF~)XW#H< zSiTcA@#4C4p0mRPn%0T$k80|=Be5qJM$s2^*&JMX&%BZyjz*VCOTYR!4rq>)XAnZiKO<@UWN>Qm}zWXc3MHln| zTIo(7RP`uabz#T`AWtS0_k*zPW+Ea+ZvjePQ~hL>IS@wP29&=}lOp^kGlB~iY1;u* zzoyiC*7_DzeIIvF1hNNclDWoxC@Q-TfJxL{Pu!B%z{cVI)Of>1hobRh+mVq3A%;8ezJS5>1u86Xl*Xzy++tNA}i@&hIL!6x=nZy6nI-*JFVV0 zh_=mJ_~2TguE!+;4usCbYv$qKux{s!_v`y5EcoK3j^LPvHxjRT$KIXJX;&idvsAm% z_hs*;VeOXt%2#6V;<=nU--O*u!Z@wtz2IH_2WQZoi#hT1@#-1dVt7@W;M!oc zc;eYJzQtftngD4qqC54(QM``KjvTLk>0nfG3v{rC6Pz6oopHQ_hgP@wgQwDgRczw!9AqkiOj>j}p2)({!{m6Td zf`@~10Y^i~nN24ZFC8us`YD9|gGT|@E6^G_I`m$G|IMXa#wCmARJZR$ ze0yB-p8SIo6TwfRqG`A4@vX!C)W#m(-^Z(O8NHMudU967arz=9 z?rUQUMYsg9V@uPxhOb>}!`MeJ#jS)Nj30*(b7PmDl`oYD)I;2s4O~1xquyJ%94(X_ zKkwvXjM@k$Vl-5Dz2dXCmmWF<%waD+cGVpXhv8{@&ka4@8_ znDe>KfFm@)lz^l6iBTbnv0Jw72^ z65{br#)ho;0i#jdjbG`zm%i!no2*Q7Bi85juAhF)Iqc+^w<-6AB&6e?U6nNOhIMubB)6Si`&Hcaz1R*StZ3>-wBsTSp^ia)-ol;=T?ntEIqi z2SHOF;5YTn{6<#f4nM+)XFISQmjZtr1kHGW?dv54jPl7HCc=q#JFq-nH^#>xjHpb& zXr7$oI9vcx(e_(HX+Dsi_mFX{=M*%eq2M5l5Ww!l3SL5KJCI)Vka4On6ErfR;NXoA zpy>47vUAAU$@$*!P{{awDu-krG0hrz)DcS8#iM2Kht7=;g_cTWJGQ2a1RU=Bu7 z?B6FwjKlf+#1Dk#-lr}#pj_;zqS&XsJv!QhySc*ukCFeCF8voH_i%GZ z;RNuz`;!a_$l(Xjph~HjCT4~x0%B4hS0SppcVQVM)O7m&xjCNWYlQ?OjHeSzo0>WBFu znKPdyFxfF5BQ&`zpCvqbKOZAHnX`Z;KH0GVBS~MoGP4_BoR3xu_?$0C3#iIRs}0oQ z%h3iJ@zLr4E%$E%YKSNV8cIhdg`jN%viMQ}1h8qt(5<79@!J5IAv6f6VjB_;Vq^kk znS#Fl_NNh+ay4Dg43PpL42cU{y^RJzWiE4>O#uk1h%h5A3{!%(CqXP>iO5`mv?1Uy zIwC1Hf8Dk+S^SSQ2x_XE8Nwhc2WY~lWKeH+knQYZjfv9605N-O2PsJF%2^*#&k`^#*4wBFIHaAssT~?a@2she6;F7BEB4TAQd0229S|2M+3;tN2^(6qoXAR z%VUJt!18nBY+wb9**37k+;29pBE~iwSaI%%4XlK5*#`DC_t*wj%7|+kmHx)4VypC>@s+J~6{DT4QZ=Kut@IDZPqs=mjH$NLwT#8KN_C9&w$k zRQ~K_a>N=~3i=P*`OAtxX$nBa|1^an`6Bk6Ch?MH^-`VRQ-0U_Z|z1awa0b-TB4|S zqL6~tj9A)@*6ih7^Ka&`d5A^E@JE5r{*&QR`UCnJH*ZhLA#e(5xgWF?Dg(6G5GL41 z43$HR)s9~!N_49Z$R@}pG6eVm_z++~(Ia#i1}wBi4g+CbclPziijGP-uuAcoT4jbj2Y{v?aq+6l(j#a%Nis7XIszfIviP*mVrcwHqzUnn@sS5Zp&#iG z#^k@=Ht^j5Aj@=bXk6%bb`(S#MuLo>3Uh>V*!~C_im)O^DHZ?3HcmPGSejf;*|}OaupCA+8|~gOR{8T0 zh#H7wG)@B~I114M>5j(ffILPax}e0-IDOEMQOGOM#Auum=zJ7n0wNiUGkrF)8RiTz z2U(9nEI}b-5NlAu7{nITH3oSD+8Bd4fY8SwP9T!?#CA z1G=IkErjlyK8meRs^M}eiV3+Oe@OChB1*Q+1bLxtzVN|c(aIkPK_KAcAFI;9%0vV& zgk5oI{fT3D#*~RK8(Srb4I&&wM^O48#xQh$1WeJ@WfeX?384uB$RSH1{|Qp6fHWFm zF4r9*Lck$(s8R^K`hT_^p4@GRuf_gfY==`8LpHqOa3K4|)g0dK-eq&u zyX#%H$4geGl%QWr8KEC|xuO_}hT3FrlSmYS^wy>fpkK=wp-^6~IL6OIZCbaJJfBbe z+_bM>oHw^6a5^!n)MIG6!`d=9otRZbF*Orl(hVxsB8%FIwT~Hn14Cbf7xfZri5MFL zLtVg&hKaSrjJJWI0pLY&Vl4@ycu;68c+q^qjpx*H#ZT7j9C1x+zK$gob9n4czt1@L zToTXtej#ewyTIr@XhlNsZ|d(?MQ(nhQ&;P^WxtK9u6p?A9ZhuHL2|e&!60S)Je?lI#i?8iy6*QJDgQd zBhBtcR$ndc+`5U*JNb$3Mt0upcr-8FddyHqUQHjo@4Pwi^{bzo+r{%}l9=6#JR4V` z+P~9K@10e5B8l%tMqVu~-JW_ypX|Yp6~_+eyz-Ct#1hYDd%XHrdeOXRsWZ+x*KT$$ z-uI8g_9FYQrnep%pLLGk9K7}O5}V(;J+GB$+;u2L^EO9gt{gl9fW0(&UU+CaXGc0d8*vRRjsb$SVVF_@kbuRYVL2hg0atBR zVnM^WVHD8I71s~Oa0EV9J-RM zUZ-cauyE!zr((KT*aK@$u=}wyl=`30OYNK~vNz{mK;Ey@;oL|qf2U@@fEYgKV%7en zq_hSKnNA5;zZ#4|bI;WJnj!Z&TS*njgEG~_XM@%A$5>~;Su>Gi`@chY2a4e&>R=_ zk+vsL6Qw|@0tfU9n2mVQ5b|VQD5Fk(=wln3K!Bc{4PD@qnf`~r3EVCm(LcY#O$(S! z(Ir=4j8TT+fQoUl0R^@&DW_Hp^*E9g)H)M_4La@UWJc`JZ91Hv@Zj7c=ky;5!rHDnLS2%nDprB?&~MMEfJ;G!>q4{$;n z)n*)XN7e{=6(*i=%Wl%)DE?RoF|;xzrXkJAq9C>7g}zgu#Wd`iPd7ntLCe(gQN!jnHF@N~2LBR0us%6@VX) z(u1ZVl#uQ%NRZiH5VlHdN2MLFi4sxv4n2hjtYAiLt#B*B+dWTl(tQZ6ay&hc6zy{Akgx55&@Lzw)q8g!Q8M$Zx-uJwsm1LKKW~GX7OuDNiYfNO z<8YhBEH~)&*SNXB!#Xuz;_f}A3q$56tQGhFZRJ(g%ndg^;(u8UneDBl;fCu?TO;bk z4vknh?ohucGXrL|`F9kKJL?u&Ove{34Ri~=`(+C7JOt%C|JCcqXh%Rw_cl7{$&BOQ z6TYf?C2VA}AI}BeCY32KBiugy$CG4eH~w1v{o4PS(k8U`=gcZ<#DDj069_$lQhW7PJS{y04WF- z+>{AP#;B9URU%IYoIJ=Q3L}sBqf3!RIAAs{h5!Y4lQ!ZKu^!^ia;X;c@n~%r@2~wG zdoNP**MBSj{e=En*mIq&8}Mh!CbW_nabY1@;*k8{zD(@D*A)K9{XYh)T4&*BQXx!5 z1*agCF!|48t>NPu{6*KD1RK$uS!a>${nt|>kKjBZ6!64Ue*s)k0dKO{9}{L;z}TZ@ z4l}&=rFaq7hUgS}L2LPsUxnGK8A5fCs(C5$RAr7eoM)4bGL?kw{f(0xw3|a9TKaHu zJ+LWn|BUc9y?ajc+zZq1#ll{}wOC-2Cahu#HP?k*Gj*3mOFXbUudhbMsWwmTf{-;s zc#XS#BVBq$Gv9kNwTCdz3p4%&^swOn+rjMG5rV%FXTXgA+pS$VBGrVHQ+FA(8X?^Y zeIaa4wFzn$1gyrw^c8IzM$#+F`Q96!x(M>TFyc9&hdDfS$`-Pp|21Op-y`JzJ%VEA zez}+0D_v||1aqDk0)L-Qf>#)$(TvCpLx5w!;?5QB=|M>X!6AXPk#v_b`a2d{4%%e~ zNIfL#|99m_NIl1I)fLHWFG(h- z1e0=PF>t)_BIj50KbSjF{wHaECVrGuL%J<8>q{hSFDoMm*j3>KaN!$~QlVg=R!~K7 zI#Y#XM4haoTlN{$k>FuJAbr4%1q%{oe6R8#3Bzn$ZZKZ{K3;A*(NLKGW&+}<87V8l8?m$(+Ed7 zDpVR&n$MDMJ@!+B$*$iRDN7X7_;##7tR;%_wi=co5&hVR=S#ROm^s4`>3Ik*6u=7w zF(c!KiyURt4ao`_&bz9s67w3y52L6OueeGLYa>+H>U{uSnRp)LKEMjL6pxmW%pa9z zJP%4A;fjhBkG7EJyo!1sA@9A4V1-SpQcFmA-UM`2yyWdd9#gDX@h3^aOLii{8Ny2r zg$n3eC3LZB09km!c^=!N?;C0tE@5-*7v%Wp?hqzm@h1`#@G?II#&bDjXY#H$u8brh ziRkoedQm2HTcQ4{%cA5!nm(1(nK z4+Jl-4h-$tXMd8kkC>_FlA_6^|CH*({jlOIvz1)mWBtqDAL8)uXnf$;uG-9j$3g-` zwdP|@yGad8x*c9-o7ZO&hcg@S|BJl00ITZz)<)^>O@}nnBHa=a(k&g*l9D0<(jp<9 zo9;#d=~hGFbs?T#dR2E(@^W*A;OYeC z{vJn=Q$&HZJSgCyOGLop1J{X0Ly%!^QCa}bLC6`2g8gt#~1)p|xK34m`gq#A0g68}TFr!5Q zV=fAqu`}Q2T4k95rn5+7%tZ#%bz1u5 zBBh_8Uw9zao|4!du@3PYC2k*YTPjoJ{h2%QajCK@} zS`Dk9Z`yx}DnGVqmMyWv?8i4ByamIaf51{sQfy7e@v4?ZG&g!S)mt?st#;1^OXc+t z*Ie3YuV<#3P?lXCwp!m`j+Hr}0kRNw(TQ1*(Pj6Lm-Mp|WOP}ZH(|&8XAQ5r5BVr9 zAmi+}kQ?k!o3>g246a)GTvWgnTZOq+5fy+b?E~n|kpO3u2*&)iLqOWKDu>=2B{9D< z0ml4|Lrz*k0hq!5T9w_Y1!ErM5Syk|0H(E%r8h@T%%Qi`ZE;8hJ;@iY^y%SVGG6sj8EnheEAC9Ta*l_D(wHgvsT6Bh;cbXnSKiXhQj=R@(O~>zj}+ z3Q6oQ4;>y=r&sz7<7$IgsI^8+>=6)+x#cRg=(xK+nq~W0;k;!Rk7iiz`uI|dz7q@n z%?bN`1G+(bEFQ>yxyytYV}30CM8Da8V-jAOnyLHPokv&^>=Ut&Od;-ZVd1reF%pGf=^<3vNd(%L`SS7SWY zWe)sXG*w#Q&wEBRRE_pj7r>K^7nTb`<*faKi$#DvnqRQD{e7$wgw{_Yz=}i!uyao- zfU|xn>zX5WL%WUwjJ!Hbso1PKQay_IW7-`?IDP--LagUMm2}_&G`)f#ho4dY%{8tfAi!($%PW-Q zRiCrG<>VJMW7{gMl4{h7V?|-=WihH`RwTSbQGUF;Q@G=#j@k4NajkHN6Ef#1yw30g zGiiRK`P+C-%uDdGpQ-tS3&S~ZY2Iy?!>0VHNC)*+8vgZ$sGjLuNv z2WFUn+Zfy9*nBXS7zX7cu%U%){tG1vN-qZCC6QB`2ei68V_r zIRrElWq>IyAovXUT234A$2K#Z*1Hvx_$RKhxMZ?#ctWPKd#AG9Ezbc{TtmH|i`0tJGAThE{eemjkMaXl zWYA)yJ=1)Ud0k5trjwD^_Y5i&T;;UaFcP?o86w#5Z|FzObgFwkq+y_Z7$bJzwk_u} zvB5K1v~GbL#S>8a?k$Fidpe1`Cn2}-IeGN0Kn=Mo_d~eW4mXDfTMvgpXU_;6^yz=p zFf2-{`9iQ_5uKz34VZ(7a9!IR_e4RNGrGrVLEi#z_pa3LC|e2*^6L-RhFysH z3(HERmkRN;DjiK8kY7fT-jEhn6Hv)8p4z1c1~hKXe{1z5cs#gEkOEWkOX0 z#Ka%h#Bac1;8|m5&H$)bk?>#r2_}UeP+fPX0BQ~LpWel>FUY?W`TnUgVCDhcEil;= z`Ymvvfij2Z}AsWSiQ zEP5;8)E2;o;&Gl%GJu8QCjdfwDCZYZp5FhNycQb>dOx(ob&Lb8kI{iBtfbP(_ykZl zJI0kQq6}rZWLinA5G$>oI=qJgIrqG%1zy=7cZRbZ}eL(919q#nV}; zyYDvwQAPQxn6>aO1kY$J+0=ACl%FDS_ACOPRz5ZGAK!rT-AV8D${lt+g8Aw|g^}#%y%C;u;YTMbvhiyJfzN+X-tl-XMgH%LGAnr-Qx}u= zecE&8BstGd)A!veOFTQy^VYioc5P z^8N@8eaA7S$@!~#7cG-vHj{HKc2AMk zP`-;J3Cp(VU2&N+9K?qj5sA|25X|q()d+6tm2K$xXtO4I$hXLMhoW{NxG0}IdLJW@ z>LHNI{Q|I$ooOw~yIy4iuOvXIRg7~^Wt~R*36v(}QUOVhj#@akDK>_$-?KY@V& z>UCiTvh7Xk+0NoX|5TfJOIebQD30aH(Yu2W^>Tu036n|yp-4W?GJ~}}XB`Njtp-gK z^SU^dA7_63l;?DxQ7=U1|3L(rQ2;sLzXNG+Gz`*Sh!mtfK(Kj8Kr--2(&-)cZ!v*x zjbDD~6TTB50C;R}P;LK9g2Zih`61^-`cE7pvP_@iJ22EQ&#umAej)?QWAuBd)0?yx z1N+m{&pwV#ySnB+Y6>OzIy3cjFPr#eoi)E8D{M5ONBn@XKYlQFDsbF?}bJ^8x zTS=nQZE=T)0vg@Wj$=I_hNz>|Z1i9VV$QB(F%^_&|I5p$Vp)i>1Ny=k$#ud8$t1vAMo=_`GWX}0+X^|J^-$K=GM#VO+C#tWtDj`oiopbY zhA&&T1hcNnZg?GBApAG3<)~@u1I#$wfaA`eO&(q3M`_rYeDBq87IZ~--`{Rd)vjsQ zg~IOEOYaP^ev{U^eRz8@Ts$8HAS?&ue`{_a7U*A@8;Aw^7v|<#70YOjOlsr>B+<5e z37WAx10iwMbJxZ!heRb$)lNIH6M6b3T0fx(B{f=) zoWucK!b2;<$7-Q_jT8StxMy6Rynn-^<5 zO-FSO&0CUPYA9a1Oii)jzv?TgyD^9%A(|aVl!3){cC_DMoA~Kk>D$2F&;*+Pc@@E& z74KdKn_GNQd>??$#Pay!YO8>lap%$K^B|8nuTj?^y|_`QAR$0>3?hpgwRd~GH7fJb z<)^=(a%#7VQHwP=CGK81YAdatm4hr!RVOXJccrm4TKLIDoLweh%5|GXP-J~PnYeZ^ zIbjp;4{rS2^f@jKVaE$z!0>YdhCdU4LqD4Uo&YMK_-XfFP_29sQ#27${vHG-xMlRk zhEe_XJKWy&GnESqSlW$ZNv8KHjQ_*@o8R3GwFevlE}-pUY{_2k1Sl;5l+BJYEeomP zFfjoK7yR*v%FO;SFcYd?gtJ6ClS2u!lf*C{0Ke9}@5j^P!L6lSX&0;NJ}sH?rTpZD zxM{6`gV=~^B)vPMdfH}Yw{gI2y}-ROe_7XcK{1prh<8P+_{nQF|JDb293Um?<6RU8 zd^f3odO3Gdg(40frr&i)Vg7OB?!4r+(guoLy0vZt+^h`iy=^I4z| zSfKZEt-~5sTt)@*!A*5~ND8A-Xn}|!xrk_=5lOFK;9mmp_J2XH+aaslnbNY@Xm$Ph zGEXPdGM2hrr3QH{(lv*@{0w=W*R2FAG|HViCHz=rUmbM*diRr^S|>Mcl(00R`tPU{ zvp4)vDrJLv%h6? z!uRX|4!HSImhXt0&jsEbh$QDnePBZWzyvrf?fD%zBmF#F>zI=IBpo;}U{Bm^E-+*< zu(uVHsPnp|pBDgUfA;{|>jzHNzrH}>maKmQ(FeG`o+EqmJdk)Aa?7GMtU~^u9|917 z+tE{EQr^JPI1X==AF@a20vLc;N4VJa>`e}4@Dub8&&Aop6-yE8P(^ZMq*SC_{2#0Q zmyh5-J>8f4Jit9{ZW?_8^(uKT#kktF4ISXqhQ0RQc{xW9W5pz94Sf?)CF5*ukMa*3 zG}4+G1!-x%FRE%jV7(3XuWUX^gEl1q0PT2xS^#nU<A(vfeO8$X`=m0PnNP!@DRr<)f+jUs3l42iXr(YY)D&$5R~Nl zAS4q-3N&u}0RoQvJPne7x;_n6T>+q@Tm#mS&Hp>nB0%_}Gy%NVOc@iZq(15 zPTo=GJ7%h#-HDfQ?xJOkhJYr@w7&>RAGW>6hO#D;U{(cp=(AW|vz30+>bAYL7^S53 zN`#-)ocjIsOkvBF!tD9C+3vro4(HcZQJ4RAL!}yRL^Si3I~Jmy4D=8|`L<9uX%~|8 zeg?Io--PaqK3U7_{nurLurF;AEbx zn#ss{2iZ64EDWqwS=IlsH!uKND5YonDE$_!wymcPiRk9cYH0oc`T^T2@=0rG;lIvw zlbp0J3J~PW6j2Hz%+o9TN(3$vefLTjg9Rvwl zA=#!6fVvqwP={5{+bNn_kcA*e>4rHtLzW=|OnI$?4n`>1f9C0l`aI7#!w(XxRpFRl z=r=v!U;p8CHC>j|Cl^wn&WQkl`R`|By~3{4dY=(7QhB^n^mk$zV01MWL;_Lf0?cQL z>u~e8@MjACdshB41^+!OQ`(C~gM8@J`2G@Z(=%@~HgBiK|IEY>|Mg5PryhpofG1WW z)}fAEgF^Y3a*@$bHzvFgqWnJw@c-}4LHt)_6rTH4%S^k?;2U$Z&n;uwVw63L>7e@w z?N&!1^90&EzEYE?`YW;EVYyuRQ39a7+F}4~s*qo18ephGG|PmLG_GHgcy+=}Aeegg z6++43ZSA3)47oMM0-R4k0}w3B;lk`4k$$tau0B6#cL2e|#bY*=UFbu0iFYHc|_+rt+88yH0c2I}+G#U6V>7(jUraA|n8+Tu1LEev|$; z(Ys8C9z>a2ajYq0Bo`03MA=SvIzXR+R$U>qv5t?=wpx1_ieKg~do41lUSVDtF8NGU z1tj9F@mn|(Rn($s>md}E_-e$77Ivk5GwuKHRC}SuJ^Tk7>V_aCJIk!xi^B{Q|>ifTKXSuzM5YTsyr9*C0fV}V(Bmh4TiKi3x&nw-K6(&N zo=*3qe|1|6O0eNgDR-w*G$p;bQa~D-1d$IroGDUgKO>s=U^Yr!+NxIlr(}`yUWtOp zwUS~OEdmq0ryL+#6Y+3#QSOI|)Q(QKiMg0PxSSjiY|{qsS1RiUZj|D!Loo+n5<43_ zd%yDEsrF%RkUSH>TLMo3N_F`^0RWOY&zv=2+KY=}!tA+k?CyiUak~C$MxZTZbMZf( zfPEWwuWjoXV0CqL;7NhYej?~zDV6d-){b;;gAR;-%Sc0*LH3RyTia`9(B!0F(hWse zYXa_?r~wQcAP-9mNzMXjB$k8f^<4d*s1!%yiw*}ievr*?i3r)NDs^9L8_NNMaz!bS z|6@9^2i`d6iIcAV=PJ!0RC{d5eUQ|{oLhb@mAaL0FMM;Ou zQA{_mBQSB{+`x|i1C?&KA&fn&8%Ev#vJxB_6Hr;eN)Dllhwnob{?knZh!Ju3{lI~c zW3{)EOkP`Y;;#YE=jJHKZ@=!(1>Xxu6S@e0k^mujsq0IC+=2rdVb!~#6>4&C-^di?J=|Jawql-2l}bEox?@0KTc1-Syq-K(I1zDV32=n-F* zeiHZ!9ZVbmHXwvQeoc6=J%H*erQh76eG4Rj7cLI;0|u?Ruy2gy+z~+3dx8$_cn6D; zlR&~-{cp!D74e}y3+oLQDEZsX8|{UqQu|kE42TnA1wa{KfSP?EZM&im_MM*Z3N~wC}Zz%a`Y52+E}bt1L*d*O}m#qdb)#32o;12!B+6_`pW`& zC>My@FC3_N1~y2L4KkkulN)&5>&LtDt040DIr@1Xt8<$#S7xE5Knxgt*{HQZ0+8sV z8y_ts?L78p@GNhC41q_1y!4q&ZB?aIH_hWqjCS`ur1YFsF#%&A# zq~82y;??s>xzr$ z@HniSGK~LWN}9!sua1m}PjDk3*wReVv)1(AKR&R`sWRY416@vBEN6%&J>e=f?BeP! zTIto1@0Gy=wyt8I85XLsQOe5g6L`+U_C_fPOY~2wj(32e${+V94*8A?{nr!cl~?UG5VTwj?WPOF%S>8!inVVd1dT=_C=|44&NE2<-$ z{QhLo0M{6n;mQ_PE1EYE3dch$%6_qEeCixG4~1sS50yqI{Tnzo0*_ zEmVk~@n1`U{I9(ms~`CKhpve6y(t-jq)}4O#9mAO=db~yyVtLT@prC9$m17Umh9%% zxd^4bT<6v+31xmhX2k4mzlR?Tq*rzVP)U1nRQ`B?I|5XqycTQnc>b49aGMrz{4^w# zrLTFtK1<-(=OP3*b>NUggAbME`#ZWRK5kdT;o;4fEEq`V{aHTbr6BtnD0sV8 z;dp+gMUv_m`yF|yj$1_E*nXt3|I?+IaMkk`ZwTk|yhZd^ItBo?AWQwk+^qNWFC`-6 zp8ciQ=<*$7{$M?Xj1baj&j~8~rT6aY@vTGp@HtpqIJOoLDG`h7niU?!kDGCr<4iBN zDZi!!l*``GGSZRN!`LbV>7`y1Lx}KCF_pt^E+AM!Qg(m5grw>Yk^U4<0z7fKP4Q^z zhoJzgl&$jj1whFilJc1vaKRZw`f~xK5SFDbRoCS~&6cOb*u~=evoEGcQR`cG?t{O#UdiBs35Qe0lH^y|5Z-5(2}D`F8y*L4wl1SC3qzXs4blB%!p7L8P0JS7ste|_EgZ{?N2@Fmbfmv$<|~#6PnHix@>^9&TV8z>7u$&E&0^dUc8?D# zrDQ_C&N#an9Hnv6>%Bd4iIv!WQD5q=~9>6vJ+V}D)zi8b2p(Cp?vf!^QG>CaLnoVJSV0=PDQwmnbH)QLH4y7#~iC{+nf5o zq$&jlME=VY;p{$je?R;_M~n-N=8Wn&q@F4WM_$UJ5&4#si|Y?0+WKDtEX3cH^fP?V z>3^*T@+TEsk2t5w7capS-8=U*-7O(M^Y-~bv%-GM;^0H2Tp?g@A^g!2(|KInLf>qS zLLwb-U^wq_I7k0pj6wiWaS5rXOOA{-&W}6>8@T`L!8>ae>vB_ez~7Kr5~*F8UH=Ed z+jnG@>Oe8=yYr$$Xn=xJ$-|!FCwhNG@sjawwmZh1PSL-!{-lEcBD3`$iIDcQW;P6Z zFYqO8EC8Po7car!7YSJC9K%8n_Zz%?AF1B$L+yb*T1UMZv4o9Q=N9> zNBV)+A#pDCqVBDV&7BliMP7!f{jDInY}aX_Ig#JJa=TujKOXpz(4{(Rkoh=%Q7qtUwN|TUM z)7k9d%)&*(5gNwJsovr9)E$BJ-HG+uwr94w0$vYtd3wXGl37*}TgR$v+V*va>On*0 z6VLB{-(Na(|90egMIwJyqB8wT1Lm}0=5#`}w&~=-o6|bEtt_&H5&jhs7awK)`5BYW zi_4XT%dL09qZ|2~+f`M^u9kaYg!yeV@(E3u1b531*w07~*$Gox5OiYGCk1vFUr$u*2pnm#@<*$+0Kujsiz0 zUd|J#tzO7>*pI!r7o3Ni1}B^7>V~a4yObCj2RF6}Dh%H9TzD?e8{{uG`N!6vB{Se% z8v7!yj3kkdgN|N?tgMl}ZX;nFh2PQ%6fb3)&9aPSMZW@Zr;i_yZiM0O$a5V+9sILtew@ zt-MoN({SFWyHXRE$rZdo179$%ygazHznb^aRl0v(x-v+;s;t_VrSIcMlvkK{ygG;z z|2n%BT$LJnNW8W!g`?un{o?DOonUND@#iG3r0K46VV`zJP7SSR9Gfk6&~s8Man*NL zdoDXqjvVL9-=+!_l&4i$qid|1e=j}xaVUJ6f8;RUhbLu4R@TPEHK-$|U9|c}QbQS& zOZ|kSzI68jAJ$gXt5Me_C!nc(mn-kRXpB6`%%(~1=c&%8{hH!478@CoGKphY!MRl` z?!)sh+@Z#*_h=k;O4An1t-wGW%&%nt!eR0 zM|pkckxlp(qCDtSTwK01@xaiDWmRD~QSNfzA|=+-<{~ydH~5t42leWsMDG#H9V(lz z2LX5O6L&|-howm#kCgL&=DSmsGk2$@jq^p!6dOG;nHi%%T@QnEBpFiJ((31t4LUb6 zcF)kaZRCg0oZI)mKP`VCVr-9WGteHIm5^+xPwkU4uukpwKu7QCQLbO%{JFa~JdQ!x zN_Al4EHMo`zSEQEvG^UUG%rp*(|Q_dQ*ei-9$ZN>Dg?eO8=E*=?wccgK{=I~PeR#p z--$nfgUL#~mXB0wE%SJojqrnY4IS*IBTKUt2d2KFLEC65_U+mX~CVLNTFho*y5-U%eNckj~Go46KK z8fDq6bJ^v7-e5mht0jQ1$G{!)p3I#$%XKqipXqobkU1bgMwEy#4|M>4-|AS2#m)uSJbsO@<3>KGMDFXBCBw`1_6>H=O^RE%AX zSG$!#6Dg9nmL%Pv)c2#Y}bEX%9S4dUbnAF{rJMAXY5A>?dsyn2luw^swP*c za~wY~xjuWbYDFZpCnfda%k)fY)%tnSqvW@;r_X18B(xm(xXWyzVs?>N9NMFQ9^q>0 z%?N|zf$oK4=h)ZUtwcZ08>8{`syj_U-m#-=O+Ye-nJif$ z!|r`30NRwPXqt{1QsOsbcU=3vvM~8pBi2F@ek!$_s`z1@4}7S(j4SGSo`)$zZ^$a5 zS-LdGM88X#RIQ@j$0n^6pYA8*gP!Bx#jN-Q_b<#I`ERVIjz}VE7V}ckK`p&pe;XK{ zHvd?aA&GmL%;`-0$azOV3O6qk%rvnttA(&*=G}1e|JorSGaP8FZGoMK(+9B%wp^L6ySlBj#=`@8rQB^h4vd zpKwr&Ao&hiSPPtq-z|^qz*)OP*7)MMGC`nZ+ges);#-$dI3trM9+_3y1GF^~#(+5? z-=S*c9E;^1p;kJQ(D(9(dd0FxQF27|0&)L@^uiWjyS-8H*j1mf8H0uRBhFvc(hFwY{tv1pYwl zxYJ&1M*>}!Q4XnDr)2`;ltMgpE?2>iPg>uHomug?Sj0<;s?#63fC!SOB`|WdYmvtv z40D0{^TmiPm^2iylO-sJw@@>2e8UZS1xHIp$w%wWd;9p%WFF|8y0Gc*lpdY$cg%Lr zcD|Kzn=>N%fu;czxrKGxn|K<}E&#g#j{T&BnS`8k$w&^4)P$4n`ab)mGe-hng${bj~^K_DH|ypc)TrsS8C?1BWXCPw!;99bODSCe3pzfUN|F#y2xHk zVd!VaCUQ{@$a}KV^I(ft14<8y*`W<9LZmi4j1Z8Ws8`v2|BY?nidH&=j-9-3WNeK zrA>Y2FQI6oo=UyRv3H@9+Uf5-*;lM7M*^MM!?fYq9}}5RKhx@q!KXZGkX>j)ahXBh z8ecY9fE}AKsbY@rd?NJf>`3~sHtAqSea2ixS*r|1vV3RoD@kY@E9t#m+e&d-)>vql zR>Qk!0Z*#l@%+$Z7H^&CI&h_Rjxe)$7-0048VzNmJwLD|yP=NLx5s0;q5$*UxC zlW${}4_!du30?ys!J>+c@>f`+xknV=tzwc<)i-8 zIw}`FF6QhxuP-H-j@@Fdu+_NW=88Lisd+g+y}au&zuhw(?K_XOj?0KWbBo zYEomTa5=)TFE)v%N@a#MD#0RmZbWu?U9Q5s6(j7EUO_zGi$~OrEPcuBCtN`rdel4( zxLyxxJ#rFsipvwrDIVoqv3+-p z24Tn-{S{MXeLsy%t+n~pnckWVknY-es?zl!4BEx1m%wE->(pRtI(6xwGNyLSvI#}| zJC&71<4mBbGAK^G*qaNhj@{ zRd2p>2reuq!M->dD{!vz%JeK2!K+DL-!QH5U5C!K+s0^Uz_?5L!M^x)`lEB{BmVhi zKfi`1eRkc|{$;WGF~?ID>g2gI7TClH__dZaT3<737=7ELi-2v{E0n{6P+czJw$miv zB<@h1{;;0H-kdoL;w`N^cE^-=@OPN|iMD6h2)~YLGietZ$JuiAoNO`_1Z{_0IKnE@ z*>*#RGk#w&&nb~I@E=7-4QQTvH}Jv8cO`-4Soo<$I=WDaaE*l}{jb{(Fk7Gyr_t2}##6#iYAP#48l(a4_B8YwHR z2UQiv-pbt;={q5}wr!qJd8_OG+6%K?w3;#@svx2szmnJTBkY24HH!}G9j7PwBHP@# zk&T~ZRzA^H!YQ`Vp;NW@2qDUQnU`M5i!AtXbB~dKr)ef)dk*_yl-%%09?eVICwI8~ z{(&S^q^6x1F+7P)%i@zJElkuicBAILJr@O13dF8mPfBC)PyUU(o4Zd}Wm>rHq|6%N zn$agV%rfqC3V+BJ#9ovn_iGFoXiD3E{m$o+ykmhkLS#Td3OCwxLT>d0p{4FeJOr9; zZA?Q^8Yr}Jf$?e?gUt842StH0XCmx5dcG`<@%I8__*o~{O;N;S1>S_1*uhxjyjzd` zanjp3wvHC-qaep-bKr!+oVCF`&h9w9fJJJAXsbs>Af> z>Qm1N>b@8P0m6IF@iFg5HjtvC@;`63S^V%(c;=|?(@f2R&wV+A?Ih*B{m4OqHk@vI ziVnP-DU?QqD)etj@d}?EOm-zHsKLi%T0;>MWb_hW(EYurBl;c+Z>EaPa_(NxVE4jt zrSB(*n85&RpP5aY;~v{Mbaz)3ma$RqeL(3E)U3c6D#$DRf!KJGOI?Jpxe~Svzi3{G zk=YPTSNPI2=(6|GIXKAOqe~WJ(F<%(<{QJWneWTN|B)!eI+SneWx$)UmwXpsRdpsz zdhjj{=Jq7u+cv!{>JnWhI~~|iY#o~|yNqzKH7o^z<%Khcf=T4RH9K%e_?qA2_Z@QgAw)Um zx)LC9M#$l_iexnlx*wpzek~7(eI=DNJ0o)e0lWPyAH2-bq~LoEx5m< zk9WU1d=|~>-mvZuL8*I9+`~ZCS?_dz3Gu1Z9Cl5BR3E`b!Sc(@nNd%7Q3B;F1DT>AAJCZdAXsfqJ;Xm;Nz}vBd$YFcZ|JidABDImM!&i*VybW<;54$X<=pw zDq0**K-bn)qY{j1Dt9J!C!1NKaT<{}L1^18Z?~JXd18YikcL*$yF6?jie{8TH$7#Iy$mk)&0fRNtRopNoq;0s)I2l;$+#gm(jYB-NMc8ihCcwYRNSnVH8Em8^?5` z_fE|8_TEE|R$FI&wU|gz35RbDo!9cBgWA-5X}(p2Ao*hH(B^4}Pvc$)XRJBx#-@+R z{+((xOGTfV%F9HPm%1b&F?K$&E47FB$;=nru`TGB$RcgBZG9X=2ZTz|x*pBod9J$V zY%S2f$WJ+X0H%G*BiXey=y+Vw__1^zzxeywYms}tulpzpI{F}o;ZwUOU%Q*V zgYzseypXHc&M*+GbTuj^!cJEzT!-6y%$^$x5Tugn?=iF)TUR@BC_Z;L+Shl}`Lc@X zO4!XGbhW2RCN!JC6a9Q{NA@*y{-@Q#ue3JrC%wf8oam~v%w;=tGGt=-UaAMKugFri2KRJ#ZuC=WH4!h< zv*oZ^YCS@yN2tW9?op)EE!x#77=8O-y_eXe_RPiAKeoJMY%T6vJX)hns%U#Kb+K5f zYrpQoV<74{J+j)T@!qv3C~J_>?&@%HxKnJfrc!Wz(cbpx<+`uhSp5L$_wQS8+P-11 z&ptN_qwRRv(8j%Rn9|PL$zln@7;fEUa&jV*jaQIzwVdhjuLZMT?6>z~z{7p;j_!QZ zg|TR#aMayOG$^DC3>8wnO)!p8;$~ZSqoTZ9|fU5-5y&$-;VY7yW&KVR<6| zl|VWB3pSvDL0<7 ztPG7cGiCUtrh4u)GjbiRp%@80`nArC66Q6TR}VEhSeP-%F6W1 zr@*W2Noh3m95EfR6LC&DFG9=_Wln@UTQ~0i*fcd|LyRsqaqE3Kjed8?kxFXZJ%!HG zizl{)alE@(ivBG;2&xL*5BeKEg?jZ5j7eK0P$Wo&;(m1@u%xXdIqTVSf_fdF)6<~v zR}*m>lj{+9)$X?G0vvW}hXOx@7L0ZKR$Ys99dZiZ_cyFa;cXUV++kc#C&5WmU|h@c zk?iA~OdjwPpFrtkW0OR!NC@yj7@ATFC|_aX>2BgX4M1UW6I@G*tgBNVwPZ`0b|gJ= zU0$<=!L3NgAt6qMMCb`_HD!=uUjJSKW9f! zY)4p!#t3P#91F%)G)6zzNh0>Zu6ginc6>A~kh@FPp8nF)PwcHX>1;H(hyBpuIiDh0 z$!?NIHz7>IoKsKD>N=`p59cBgeL{i>dgdo}1I`&Ua9t$dld$?#Xk1f~Mx z*8ON@ZLBjBpNV)4Gip_;8FKstbuwYg_Gty$vSx0OdB2Vy33>ec%4_aq6 zO2xenmLx~&#ASoK5vK+bEyZ9SRVWz1UVchQd`^b6aZ&2Ed&#Q(1mg!+Kg_Jk%P3MF zm}(ZDm#IcHn$!M+j`azuO-##8?ucjc=ONI&RM*VZus z4$)l0AAhh;Kj_>#`?Q^zDfnHQS64te0t#`>J9Ew4^dwC;!&(qx^2ZDw|~$QG(G!%r<3L>r+kvxF>R@;ID^ z%_*Zac#p&GLL0w#D-UiBCu9z4c{2J$sA%%SxD13KZd8HsA!nKG-PjtD={VvjT%f!? zuzAG0!l)0|QP>}x9tV{)TZW901pJ-R7KF|aC^V&}q*2dA`DUo8x_+U&^KSNBcq*dI zoNsjDr%{e1hwxXD@*_uZQ>m6tXrHF1$B|=gt4XVUbO`QV=z929*|b`o0YNjVem+zr zG3r!D@kWV|RCSDeW+azG+)~ZfX>4wxcET$GDyQ#zIB95Qc;>^TZKpx(JDCJwkxV&F zqEEM6p^MwTJ~x(^UFa$gHD^q0i&@O$tgGNBllnkR{6yC5q@9OnJ=^rT(N~YMG|Nmq z=7)&MTJ`F$nBHzAe>n=I9r>)WW+)2ZM0cR@-dwpayJseiIwG}r-PI}TlV;FM+CBKi zm93<~=Et$~Xv(%J)I7e)l$}$`k^KC3X&i>>QWeJqindslgppngg~QA5p}DYz1lUtd zD*L@zkLhfd0-gBD(IQo&g4*2yJaW{tdUU_Coq` zfG&?!g<9XT*9WnN38VM-z;h$=A?_4K?=s%ecBRRzBViU)&c)FQYj?1a?A5$DXuv># z=SA7QMd9mWoRHWmtHo0>8FGR=}4cW!P_XWbAjmEJcKQ!P3@11-jsKK@y8 zUep_QwOHUSo@rkGnPX}vhg9Cc8y^i38+blWLz^%*3NG$Yk7r}K*$B_kcg77dzH1d{ zz~WVCXrAmxemam=IU4SdmD$?oLmfEEq~BA%BwJpvetAx8P{!e3@oq8u6*mkg$or!; z@qn@g*at&Nq5l#dWyAtS8iT%#=wu41+4r7dGO$MFs}?f0W7w6aOFSqa(Jkz8(NcWS z#eB*Q?eWp>_zXylUQSN!ppm!C(rmozE%(Pu^gc869XA$g_U3KFId&=CeTU}6LYyex z_Q+>A0T!LL$eIn2!AaPyT8;11AEj!u?rul%r1w;AwL2Vi`IRItH>_xk%5zvDKZa6t4qxvM4(mcdsDRyizoM z&45-H^5fS}{l}b*b;R5(!epn}4sFJl-Hwu=OgUCFV%aR79`y#ExyC~03>!Uljq?0F zu7$}~x@FQh$?(1Rg*|_m1)aNV58{_qLq2&>GGp^Zoo%PrqxO4GOnuL=$Aqxe{2TJO z1&0=5Qqbi!Xk~rljU@W;__xP-W=pd0D_%)Tmw{JYn!9jG*@JxHr7OUwQgjv)dPVdU z1JgmFnR>n_4)LWnjK^$PcVJI_sC_}?r3Aw6mI^VUdzbZ_tw#sks57Zn2ps#^$GjX5 z&Y&Gq4;#F!&|aqGN&4{L?JD#CF>PyeygNzB3z{uD=EqKH*{Gt3u`vHU67-ZV4_V2( z+>S}P7_Jhh0|tOgHOxHmJNmX+ptO)pXv6&(nUub`=uB%epR9yT+4_7uK(DcdQ&WQ~ z{<$PIY$WtoqT@-V-^+{5b)QVCc!Yj{ZkOJAa3<1N#j(V*wKq{&oE(egf4`N-yHn+?K}vW&X%>E$zGFffiqLR((*Vul-We=~_fVgak2SHh z$h?o4$TBw)vXkwsJ#63v%{}j0^dS!gsX=TnWGL7TkKP$wsW;&w%Lp($Y}G7cFhLU; zT;A-`Qy-9Ed*C)UTc;(;d)t<1ygT9g(FiA6M)gOvAG1$tJ#OSHGcM-oIVi@O>TTSk zh<~T|j-{B?QsDcv=#lxGeYb`py*!6ycRdUa`w#MEkDAJA(GsB4whSHaMr|iQo~N5h z`R3>iSG3ien6hI_wj*++Q0v;?5Vd9gCDTlu*x=64v;GF(tz@U|v@?$4_@%kZgPK~H zq}jAfj!I`xmiQUVvF>C<4DF$}yzxAeu(b~lH+p}_Al*iBX%+bh!C`yOfJ12!4`2s$ zen@(=V6xW{66pGB?l(r1qWzdG)s}K8GF-^~mQeG0zD$G_-@*>Q37s`}uumP<)z>`j zj>Y>!{N`;>m*6X|;*MsL` zdHTSSY}xfi?@mM9)uP7A_FXG8;;?k^ifku!u&eFnMPx=Eb?^&Y6@^|8Q!=-)xuYyP z*E&yG&HUVQ%EQ5P>8X&R&vG7exXxy~E;C@>q}ZzN2UoUcyJl&L<4K>A$R4C1nfoLn zKTVI+$k0Xx5(3|Lo|!eWx5b)s= zP)Uyy9CQ!3?|_VVFQr!A?85bJMv9QYk`=5nJJk-3zNk6T$&168XKW&4+c=Crz^w$< zGEWymqHWVRmtG>sZ^PWh9apjtC!oiWq|fib%89QwB-N)8GE1j60C510AKzt+cQU z1)YUZguX*O%Qe(S6b#JPRfGxLtETQs`+@9aF3MUNm1Z}|;M?sqK+6BJpZ81QJCib+E_Xn8HKY++&v zr0l#bv=|UTPb<UAKSWJg_%vg z{r-{U1!+-Un>K$@_J|j0j9*0sA4MQqCM+LX`XE;ChKEAt_s|p?l9_1jz5_Wzi9ls1 zLJ3SU6bx^+NUI5GUTq20ar`-w!~?W2v|fTqM^m1q4^aOXQC|TR)%X5Q?b0A19fFh+ z0)ljdfYM5Xuqa(i*Afc|2$E7FONo@Uv@{FSOG=Cr7v~O3Yb9z6jqM`V3FKoy8)5Ua1 z`x!VwOs82>D5;l$5Bl*eB9>ek`f(x@(f1epL#*;l$=mYv2%&S}s z6SWs5<$#=;<2**vE$zEpm=yJ?(sOa#P0P%l=x;DF2XMlF(>AwXh^YeRa9=*>d<}a^ z(4A}Hzg2+jC#GL9tUpu3yvdHxTmT}{yg_V$`Ked7BZAFhw9v7eJ(Xd=jfAV7c~@ex zB4CZ!lrwXEix4OLcQ74w$o>Pux}=WyJ(tS1W!a42f6~GY=B~-t=c*?BruK{apUW^k z)|gY7!JcGxL(9b*f9fUEat#D&--f=fxu>8)2d~8oY4rPfiSXvJU=gdzY>V?$*;fzP z|5nw1-xyA-DdNSvLx(*c)?4>SEvZ?@>1{T$Tz_SP*1dxrVloozQNt7G8sSjY>#wDL zBHM@tDJ-s(ofpBvUal?yx1xTaAQ>;g>iZrKhd;gBPVY4wU23fe#=3A%_IUeH!cd4N z5&yx{LCy$|k*lV!@wJ!Z?M(s&ThiMVWUXFV)J`DrCL@n%hkT;R<>7k6K;FMqI)0$Q zb+gbDwkO5m9s}byQn09FJ#qsdM?c$?x{6~1E|Wg$kxv)`#K4uiw%Q`LA>Rg;FO*&-7<&*U$pn^r7pOlB}O@hJdYNBOLByw%9U2IshvkxslKmRbu||>B4_4UZhlH*&Df~tdPXQKY^MKC zIwmtdJ}r|^l{`iD?1(g z0~>GQA&E<|csnT)z!oNKb!3b3GC=D;_x|!}(r0PKtE@uLEbCnCTujuSwNKsE8a1>x zg_9e!-A2bmm(j)Cs7Ze~NdLJt-n}QjBUi{@h|4ABT_L55@cF1c%aDWM!)g0wXlJur z@>F$JtR7U%rF;Xym{?_Swb*OyH0F9jA~Et;=v>_PwN6~YVnJq0)h6lg(R-fehhf|R z8Jb;j9OhYd>mB1q*}2Dkl!>yt{vGz>VAJ{ZPuliSJ6hbw2$HN79RF-cE2-SERmaDm z(OjWQuW`)J=bU}(-r~>~vcCS>A&)I2qwQBTLlZA=qIFI%_Ebl#H9Sxp-_cF1_ciY% zDAOALaoLrEi6vBO!KDFoNEE-f-iB_)s~P$R`_skaaPWfPl_G9DNQ{Xgty^5>`zKap zu7q0l*&Ym+NBVs=A6Uwj3h~aQp<#{FPDGQFYG0UUx+k6RgOpSg@{M@mt3?pA@IqZ+ z>gwjmy~A|KYtF2|mvlus=d89UoLD|2Q~Q(61Y;@0y-4!a4n-v4G33FcM$;e85B3IF z_jXAdTw1z4dh7n)wk}OdEy>&YFiX_dYGO>;)YH5&4x5Y5gJg+1JmJS;oAcgRFq?_f zQ|HV!IWb6Ioe=!mOz=2x@z_r^Tj*$wD%a)JXiXo;qRjW2uftzdvzf z?6!fZ&*wi(4i#jP5NRuj0KT3>-&2Ie&0tBDf(gxbb6CjL8JgVK8b?G~$&ci+wgV4t zrBk9*1^`NpS@Ar8I)lgi`)_O{HZt+l-4%uNR{~IEn6MT*>n@U>N8rH?i}Q*2_1jBI zM?8a3T3r|lSHpJ4(uQBrr()}p$Uf7E2$@3CT>v}6s_>dn3L$iWPZb&b85GWH#+4{~ znp1_N?T2z7a6H3`y zoUEud-HF3DPeS>t`p87=(P|kYPT$h|W}kD8#qs?REA5f6cJ2EdODW>JB9vt z^-)tEC5XEOS^WrScKs6a%OC51%h?mprHMj_b^hExa>C~L&>^;~sHD4vFz0+IIh&YS zz?bP5J}(%qB3(o3ScDFy~A6IB~=;kNXP%kxAdK&Ve;C)jo|>`MmKKXn&VBbqkcvs{;+Ytt%k@$FYwQe< z!{imelXK*5QDoFcexSNhBVm7VK;Fti@WzPI^X|;wmrak!O}^`>Set&gQ-3z#dbmA+ zTO(0oq1*J#(P!?5kE{@R7n7VKO@NEc`m?{b3(GRq-J5XEjR=~Nc8VUQHbMuUoZLx9 z-tyyx_h!?w_|u1+OC3x)dAcWgGAoi#kDI8fau@#fqFWhk|u$gckm zsg=@;8JLtZ<>kxxn_$!F9gcUT1S=;uvQ@&P&&Kt>z)@|bPiQhEsXI1h`w4ok^gd8J z@@NtA^+j$swJ-S$@CwGwLYfrm!K6e^JAMI7KKj!Amg8{yXVV5@ZnZDMaFYC8M3C6u zn#L4O9R=&aj*f!$Dlpig{~Fe6&^LZ8c4Nq!H(yjxU1TCqBm1y)(P^7V`&%%T7)}{= z-3+lkxNgQWo_TYcIQHRw6_TY%$o7i7!b*Ved_9h(aA=V?9{pZBg~>hd1#)Rhcu2Jj zLp0y=ca5yEQX2B7gQ>~x%xFt76la+l@!0ce_goyV^>qx&e@`r;i6Gp2@O12^W8-sL9;`~c(g_(G?yRmgJ zoO`+QmTV6#c)P^d>f?x=;Xlw8IWAFE+CX0FlO}MMmojObP>Lit0jbogF(4&$mu5h(7jC7phTdJAHm55i*GP6!(Gc=Kqv-*Jql3 z^@3S4@91w4EeB3*gK4o)!4SUx0Y!H@JA~!C$x3OHYX=_8VrC#K`ss%*FJLZO7$!7q z6A8Fake69D@8#|?B!+19g~hpfICmGHE)XuhR@|tXqOV+>=K_LupVbx6B_Fdd9G^+9 z#+s@RA#@K&4c7%vv6qCqT9~7UALfmrPE#?_Q4@oWjP(J*57dtKs?e>ZW>=O}2|D9j zv(-{ggP~cLN2F4ezI_Tv^Dj*gatoN5+*K;XaI7omt$gRheUvitO?RTMnq3oNe?0~e zd8<4;B`c{h4tz?o2shw^3hQwr{n0>i7k#2vGuiK7G~xpS@P2a?VSp6H>6^Lvvq*ot zqDWz^mbO`+Ik^Ye4)07I+Gj`b0jlAXe2xUL#rPK=9rch>Toa;orioWM-j02WJ)KL{ z#0U=^bOP=24T0CjGvP+lg3W@1)1%+adAZV=4hI~33~9~C0mq`iqiq1aEReQfU2pf~WPD+|X4xFV($IPZF)ykRUCx z>`vmQM<&PK5MPT+_4dh^+Hi39cE?vXgMI3%vv?Ek$ObVX(sc-#6T|13s%Ev0?n>n6 zrV{*gOQ803Eqw9L^(LZ=iY&F7MX^)j;r@G(nu4VoPg$BR*=Gu5WwX_8b2lBl#8WB3TbDj?j>}WV_rv>P|Y>(v#E5o z6kb>=@{MZ)v2D+j_!rF{e0<~+2gHQSuYKS3Y^AeLid+*z`GyDmI4&7jp3C-|sQRrw zcnB+drgkIEtfw2%an@$YaRL*E~IbdjW68CZ}{Tq)>Re-n*P6dwNe%(TOs&Q*B3 z3F^5%Iap19{a*F^a?CF@qKWhL*>ynAR&l(LZmYC{SJpW!ZT;;_=K>V!o#(W7cKn_- zl&-ivT{0u0)8ukT*h~2JgFA*~Vn(-Y2EN%oW!!uxlr_d%-Ds00bZlxngi=m?df5+d zX5e8p@2@60E;WzSGE9AI{g@`9Lb-JWtgb*eJ!~~S^^(ZM=tbm?22JxqGT{XYwcHZW z_Z(a-`@FGHom65lGf$I9$8}|NMb8}{4%Pk*Z%f(BXisbx4yhsm*D_whz4+&(Tp9(` z1_y#HN<@=;j3NSKIb5JY_2{_y;Ga{)i_@&+h%EJd7kYPx8;yI-OeaPg63d-^p|B#I zpXwdD&+YnN3(sk^)W2hT#{9fN4*Bf_u3yvq=D5^%Au3Fsp5~CwLxc*yso}o=xsJC2h*L|h;t852M1=+SHE%U9Y zXu*4=q(pBreqv?H6V+5D%^a*skDBd~@l2M=9ro-RiXDPciV-msMS^(!HXr{0x-ber z6aMkgQC)zsGw#oyEmZR_UZwT7&N(CO`hGO?6{$F+E*jT7M9h4iIXN(dD_dgT^f`F; z=H;nBmuSNT>12)sq}$rJD0SLoT&`$FOl7JdF zoDgA>^&6ce_ZgmxvxA{l@vtKTWmIK{w$S-wuxj^8Q9udWu*}%lHjMfSML(W?RqK&l zX%;0gP!OyMKks6?qBEH?iDcR#~)MXI8l#wx6~fWImhAAi+!UT60aty`P&N zp>QA4jt8s6jhX?Gd^RbhUI1@v&83Jf{dA^qk!bl{cR-2S4VtLbykFplR=993Yh4TG zl{=PZlj^@ZrN0d7*aEK~psW(}}l0yKNC8tazm`s??EtipGTrSNGP3O4U_ zCqf6*eOT^wm-!Mu$u2eE!q{2i+fXsLPymBgU@ysacTYQdsZ+AHr(DZblNmRPlOQY< ztQr)x8^hnY7(kUAO~A@)XOa(8s(AOP^*-mOq8=To(#Poxra%GL*gF+S3S zbXkI^AHNtCus`7f2h}Isu~1qxd@G3%D-TeaxR;kkMwjyLTCT${V?xYvtls3NM@tqv zF^~!}KtvwpDu8+>RDX-Q{q;pLTi1!`c$$FfSP}iF>qu#wqGtpRywzTWke3|X5@PAcx>9`PhK6BST>!!!(4bW!riu~)qW2Vq)4}EQ} zmIATISC>+oRT#^jg0W-aavc2fr@PoL39Dj`Ibhv^t{i;C5|WDY^Nb;FAG1Ys$t+x9 zvIdfDQw0Nu>&eF4aeUA(1LAiFr5h@Fx8v;3>N2EmyAJsx16`-zSHpw0xHGBJP8DYL zE)3=)dDfV77xUO9=P~ShcGjmX&xtfCeI=khFPV-xO0SDnBik9wi_!NM)W>~IzNXJk zC=N5f@sv?}#B}GP@MCu@o&V@s;0dMERM9ZdMo(L|GI0&q{{V)J-CO?pTP9mSw;LIv z$SW!4fv!p)TVk?o=T?dKGrvlfmwrK)S%?ng!iCbKKlXc#TQbmqPJUk9F^k!Pebbj8 zY3E65U6V#@#MGUY?_zp1OF(W4mmiNU}Cvn2DQX?wBUE0Pz^(dDpVgn zhzhuakheN_(S=kKYOHfbZTxoC&A)eykB0i$P8SbuN&YFzY((Ur=J|lTAwArOkFT7W zum6ZwPmf~<*B7DD#bN0@;vZX<{0}q&2Pb;c@os;aj0`Dr9?&CS7p4D{`1lt(Oj!5g z@Wsx!mcgz)U?xJ%@4eaV%Sp*MBDLTKhgruLg@_pF;B$D#aO3Q~mFv>|xuSsPY!T&2kR$(Mz?4ql(J zHtvYjyvJ$9&evpt!GWi7rTw9)eEQyGP4g?XHMUNO#4R?I85$p%Bjh-U4XQ2w;uqG^ z0dSH0#SLOebg4bQfhzHS34ZSg3A>?G> zctKxn%1w&!ulp1DeSgOnaeP}g5>g;Eew`Tw7l;I#JA^PXOw=>3zVRKPnXo`n$z=xoc= z2QB>lWKvlW_DoX_o4I!kePMHdgy+SeShFkBKc@ASll22@2?EJE%Ed8r)I-FAWIey; z{*QgT0MZNuCR&P8Tja{nPTZFf^IHbdk~t^!uwGzpaPrH8s^Yu??ctwSkjfZ8)=&)xINXshJA%1nhe zT#s(_lWjNs6vC)5KB%M>j>EDBWB2Nq75kL~Qd#2l&PAs(V>?Nh`c1yHm$o+6ZC1yM zu()^K^xE>AM1WMJqmMaR&`{HQLyWA&fM)&HtD|~%QW#10@JgqIbS;#&D^qvdq-pqU zG@O@-I#)U?CXkDuzf64Aes+_{B7ucoS|D<3!?W2Yu5h@!h-$c`{ezWD1M>wUyH_iT z(1^v?N-KW9erZF7AN?upNrCWc_-M$oQ(%+Hje>tc-7$$E*x6PQoRGC7#3?4u-JsxW zaTe)AHshXjvHddX*@(7Hhpc;pL=TeB0_RcAk7IpNaXlI)$_kEB8f{s9-SqFHXCUn z6bH}N=e@-YJIlFhiSGL>%hh;WkgdGRJ-Yawyb@8*e#F1?i#SVvN&2T}uuaCQR5ys! z`StZpf4{agx1wqvphLsFudi*fSo;L;4|?sfT`hO{rN0g}&2n`1+BQ*z#@;FlWw-1| z%KlO0$9;Bnp^yNslk{G0_=JD!4}+|-6|mp6AZm_^#Y|)mkO`V?AV@3+-hXb6XJxKN3(zZ z%D~?v$w`qdRc?`1B3_&(Cg5bf^|va7X8}1Uji2;p!_rgZwWm4M$?Jo*G_LO zJkeg>yVtpIpL`NHk-%zX7f|!24@OMe|Bf##mn7n;c>nS%a8WJf{`P*?XD3Nvmb53! zPIzY8s@<7Uq~(!If?r;(Nf+VzfDRpsy1;v{Qe4Zme;!QzHn*11+it=!2Q|N}H7N30 z6>$SifVCvp4;Vu>M@A-8SsE{-!nUt;6(#X?(eF~2CH8TN%|zP1E2ABg64X{3o3@zV zzerPc3Gf&GLR#&hd;1t!7Ac>L8%}m#a#{rEC;7IlQC4UHMNwo{+FZ8KpBtDiKj-8p z_D(Y2OikRnafo#6FJTBL>E^iP3V)yRi{;_C@y`%j?qHeymI?upZlWNoPCsxpJ9|G8p_+}fe4CRZ!LRLV!as5>_gapAz+!P) zpr2R?uu%-H*Z25X!Jle5th(HBnT9J#v%gSJdZq%;gP_fv%-9URl(KE%oJM(13kdvk zonWp6N19bSYBf!s?vm0n+IsTAGq@brkxsTktATjbNDYb7iTV=i*nOJ&Tm#)!QTE?W z#xV@cX=L}mUo|0Hd*ocY^m&ISmak$Tg;kSiVelIqy>%-*gU?343_O44WznZD9c9`_ z);QB&a+&AKtP>Fk=OE9wp?Aj|`e>cqj#1VYltE4-Vm+qCp?~ffk=HmLWEULo=C1!p z5_5N4+*yq^FXE7wY1fe8OTssPmwogi!;eQj{Ejw~ovLpxV-}FCCocBG5^9 zc*|DrFy`;03ryhq9jSBsi!FU2juQ;VS@I}mi&g_3%uW%9_l$BvO(z(r837nGs2VUP zy4u`18v(E-LRi==eX0K%^&O4by4&237-Uuy`d}((oEt#QsVwq{h$hT)_CC!7&q1Kd z55-j_4==+(iDPxmI&OEspC7I_J@3fFn+s{(X|jj*aBp&yt`@x zU^^6th$pWk5$cdrH$coLUWAr%l2q|7N^S2}%D&gRPdykX^{!I|3 zKK-LImx1N=gIxU$ZHVUH6vXIFeFXR#JeQTx}c|u|-glwhfKTpG@GUO7BV=v zq>|J*U(!Ft;fWX(GGPvz5t?QWTM;s03EL5xW(hkMGRgLsA)d+^;kWo)=Jm}6nSScTDHI0$Gi>A zD)Fe9sT?BnxQ>Oo;LTDFmfwjW4*gCq%Qr^w(&~R*m)*pnN_%6A-no5qRJ!LMsq6^i?|5 z5Otk-ydFqW^>u{qCtZK#n~-RYMVI4eaxZe#GJCCVU z#k|iJ5OtEcUwt6lSC^=RF-RW`%skjnA}2>S*Tw0T47>570}|#Aoi>ix53uDR1~s=; z^Z%BgykIz^|83s(!S(~s>~R0x(3MIzjQI00fbEaM5i0nZg#T9evL zf{Q6rCn5G?v@J>RI`DKJ#PX07lhzqj<-@V_?2}3@XV%Wj#pZPsC*{dRB;XtdLbm5^ z&r9Zdv7k5O=Uot84D`O*()$EEkmV)nN$VA+56?9Pf<~O7c+$x;TN2WAqQVzJ8mYiU ztC2c>H`~hDytLz@s=UeZL*hvACjbKW-U7@|?yMQcA4fs4mn`5!3@lLx&2K1h}Sn9X23KkmJ`&^@6VxY&N+7#w_? zcy{%Z>vOtc{bBBoW;&;&o?G_4zSUU1MY?nUw#KuYmk;H9nwY&vS(gIO_vnPxg{Uq7 zTD2FTRk1`J*q6y?1g3`Ciy(J+y}{dFwS9^?Df4?!ZIo663Pcs=CDYab7t z9~6lEDVL!MyIeG0li_rY_o;bbgYSefWCkrn`n0vK@crq1aqv1q03Qm>kD{@f9zM8i zBGo!_af;a#d$IDl%w-K<^A4(}OGf$TS<+hC-d)`8%eNx=YUV>c9;B4j{!poMFzgUd ziG}8pPzNl!LJB|8V7C9dpp}%Iin75!r{7g!N{sRWD4#|HtCSR>dn~TWzF>U#T{4M@ zmSxv+(;Izyub-V)o!_fD-#GVRa3DKbKY6u%A~HH50_Yqi;v-*?;sQ;{F1ybS=jq{5TNRL6+{kW~mUb^BDjU!nk4;W z7K)CVo5CD$rBzo&6z=%czqQ`(sFchI*&mf+H@@-?+vCrMQ?qF$EpXDFT76Dfmh{c{ zxQd^t7qtkWE;;u-a$~5bqRk zh|t~NDMGNvphbjQdZfi#)3_6HSx1uwIoY6Z92f}YxtY`K&~={m9iMD$79-uuK`pO& z4gg=#_rDgHt?*nIUr}3?6{@%!@ZCd z!4r_5i1EpM_jYMRLm2yDbP#jn&2Z8`%~*5o55v7vYgbRRAGxK%7<&cM5_!Zi>dk#g z6FqA0*%gNHYNxEw*;Er6I@K#{%Fn1K(wSw=B=GfKC5di!ZJD+4HJ(XRt-O5Ci(QWA z%SF&C$K9ENWoWx$#rr18m9wb!{F&-<)OySfl+Fm%Ez)qk9XN4MB{s|LCRSdCrl=Te zj6Qbhog3VeaBXBz9g7Z3mnw*grAOm*nmv`98%M(#be!)|0;|{pciG~FWKE58f}8bi zMw@GLPbQ&z#7@l5L?k0+o?jOBGxL34KoR-9RKoSZ^ z-t(uQIl`<3jlw(_M>PebtpZmv-DYZ~1sT94mV%77!Lv6~jtmA}cF%EC=G<|%7;P8h zA)J<~PC8|RoT8c{XA?5T21Ybt_7brB{TQED`E(3LMtuEa(J*6V9OBe8CtD0<)+Lib zx0~gMxMT#GFNl@li}b&*nKlVwH0ZIkLq*ilhG^lgtqMNbGA|Va9i!#RJhLS#YfVID zjaUmo?nbPUAonWy6ha-qE0bC=^^@Y$XzsDu!?0X8c8-6A-WK?mk)p!b6I=V^#8F{{ zBld`+{tO!*x-?oDUQV zoT9Vh$K}ZvCpMOPW=T&NilKC5kqgtvkYfYJT>9B)g;^$}2e}mV8-X=Z*B)bQptgg| zC2koxdI4YWuti*`u$-}g_)9Qpo6OSwrHp<9MtkI#GSgPJa_J8v@GE^5%=L~kL-#xfyt9d*zp^EkF^e~&pC<#HjFI|oV z?9MbW^wf3d0Gw6o(mf#axry<&6i$)5(s1es%!|QcCev#R=JLn0+WBhxo%H#!Oj^mOe-Kzmd5l3yP{!}Ftw z-ppW7H(x4_0nOY|0lR{dExq5=9KBmWFtl{%*R0n%R%|T_uYCdDgzW&YMTu{cv0@2e zJ^pr_o5o)=k2R5F0^~H@DW?+#=;h69@V;sN;^4SzCnWX|_3w1mMPvlUz%UP+9%y&R zr<>hY>OE)!f&o z?!zNnyS9#;LzPWm`LTap_9TxISm7AxC`{k;c6dyG$Tzhl$b=_&-6)Hz$%@3a4AHCl zio<`<6omE@F<@#~QaC1y_F?DHxiChR2p}e<7tOA>b^$NUg_T!Y-Fvp^jAS;8?mQEP zJu(xyfh5nr7C3J~(uJxxl&(6>h7Kes=NFd}J;vrZ?YIk4JBuZghgbSKZBxMxlHAqy ztk~2f|H=aA3FL7~VE=u`Qd=AmYsl~6l?p~|6Cc4weBLM1y*k^kP6Vyg)&>fy?9Yf* zz*odsUSa6rQ_%45!lfW&ibecMXU1!Rf#_Y!b%6%ZwvP>#&ly2^#J>vMUi1@xq~TDA*|e6G%0SJ4Z=E28P` zkbIaUkU{WT9pG2WBlbpcK1oqwMc@LNc#C)Y`I;mke^-Qu7jz)MdH?^QpYQMWa?^=? zssD~+?fABc;T#mQZB2g0G8UV7p@fRn%wyY=A17F6;`3zP17I5od}8KaNdQJgWriR2 z{>&R;mhhorNOoA(!!0zi%~i+;^E`7cPLwAlamdlZjbwN+0hYiG2gfr+O|*Xlo*flx z(lGq**{oG=$5-&Kh)y9t4MU8*mBz@oMDAE+LT{0gyfEgr8LK}HwB-b1Tm;H+mc9>G zvH~K^IZSyitfwIdOio)6Dg2p07QpPiA^IwAgyaVpDM0}IO~8rnP&`a*P%-JC-IeU2 z2W5I#FO%{n>r?JTi4TJcR{wXadGxhBiUYW*=^M{E%#v*jTLiriIAGCs3;{9o`O*X& zIupk}q3yrNhx=$$A&N}JQ*gDllrPSIFcqugK&jYkGm!Z2pY+)4l8t3+EKgKTrJevv3b`T3X1m9FeSKsr zf8Z^;bU!=|-{KZ!P=7^1Bv~moa?CO8S9Gwd1wt@uU?PpkssHNej_j(L8G(Nyb>y9i z(9Fl>fEwlLL{R=?QK{D8t$}7ec7(`XmWS`MjST zp>FWCg(EI73NDc=PmRhK?4Rffy{massS9Seu6z}|wYd)qz-p+!Xty4Gfs=|W0k~rc z=N1LC|1}xnWEE$clH;yO^9C7sqUYWU!VC8nTh{*3Tq;sh5r8yAp+-_+jdMSPhh?iq z1cShPwt~TR{lLFN0k5j*Yi37m(fosPQ61SarO9`iKl-d{T8d9;Jp8Vn%+U`)*o?aV z0_H0mcXwfoP5UB@ZS>{LMM}iCR)Pu+x4dSm8;(tYT+b8Ow=kg!i2bPPh9%m%;12s= zf4LCqlPw0ln0(|{zU(C0Y5Eeaknr_yLdM%uog-t$Hj|U7tJY)9vjsjoRs@f%Qn^nR zg9zAT9~J#E6z|`kNMd`m{+btdjWwEGr0a$iZ!yf@_GxhGu@;}TsU-0~XcEsC@P5@e zQ~b#T7cj5id7#{{&?d0C&-3x4-R}mzqG(dbX|#toDDbjOr9B8n{j(5T)$feGs3BdF zKfxrk-tga)$e}_fkYp*iuZ3K!DS)~x3CYb{--wK#UUvnNeDE=Av*=bMo?NoLZnK7$ z7{HR6w6s~L3z5_%TSJLxxFP{;>+%!w`83V>T=@e5-?JoXLqVPTi=*MXfW~J}eXz2| zz5GspJxiJU9=rrq3bHsw8RFOPpb9h5Q*$Ouj<;qeCas99tIabh*faX=8`Py=}TDc;_JM>8tdVZu3XGGYP3%Crew! zq57i_@ej94Ipwse`9zjex=odMPEy}uU(AgDyMd&x#>7Jb*@;^^V1P*MVY@fhYOdw}Q{U%T5o`sX2R9+$990&r(8mg-DEl^;fW%6t@`r~im?q6L@e_vrE`1(ctUd9+M5$#7(e z4*VUSzQUMobGC6?7UkjxAE0HJ+%%HY zob2=oxh7l$3&(X4SKI0i_o-laMOw_dY{-}$Ui{ox#7>jAUejx7L*myc3`jpOOUIHq z)BthTu2-3nyonA;TXvrjv6q9Y?{m;&O1(4m+FxD?{?J1CZg}w1E^WM*hnmUW ztG|wUlYFQR;V@Mv(|8ksU2fcV5UqFYFX!pu>_<9yVgn@up%qN`^SDuQqudDY!Zs0KQqb?&TiuN-0?n|+7_m4QylM37 zj~;%}$V7*5v9;A9y!40AwHXgOOyonKE$8op1j*du&9?MsWse&fH-=(r4(9hE&C5Zk zi(9|uWtO1e(~>#I28#+>7S6C#!r*VNTRQc&uK%*s&VK!^P@7Y2$mZ@$B=A{-X^>?I zO16fO&Xld_A?eQl5(=)3b{UuN=8kI&TP+a|^NqTp51Z9! zpx%#UDg+SnK{Vd27K5(_m0O7_CzvJ#^fdY!mp85huDR>M0b#MgG+`Vam~wmL@z5g5 zJGYi^yN;u4jWl@T6$ywx4v*S$yK2qkx3G5NF9{$fV))_KeUUFWtOkpr*9})=6w+>z^z(M)CbgAL!kaqxzIPV(?sDqA!F;{FI{d#M5jkKY>}F$!{EWri5N3 z3VUbASEDKa7QdsK93^Uaq#hTlH7|&E@=C1-WnO%|(xvg#7M10faAerxW2WAB#NEwB03R( z!{GO9?j%{QxBTvVyfFJr5}X6CV*3~?OJPzK53&9~TUdlPsW0@6H6^A!9U!mNv|9F_=CB?J|fDQw~tE%Sm0~~ z-_>nf0EwmG(P|Fo&Q5wwL|MKdgzA084}173DLdJ;c%<=_N=Q9!*}rv)+BA4BC3#hxGi(xUgma&+&gX)I?xo*DyL`AjB@WispPlWqFV@0o zFzl%cv#-p{H&YIx%iC)5*9{8O85(V8?0<;TGxBNn{obv&z_vNq=WLc}|0Urx&l24- zcMCaUIzw5aQN2VN>OVqtvK}@1bQqJz%9Lyl9`t_<^Nxo(I#(hX$C`4y6|tQb1&g1g zQV|iHi5vAgr?*ynCU)G0+Bzey4z;LeXow<@4sksER-D5d1pknZFM(i zOiYg-yp|?MvQ1%_jB(P_FO0Jrflen~9u>M)|0F91mAZ@G!vg@H_bMp;o5Dqd|B1zV zJ~p4uL`cLs&G2?)$xJT|a{zeHRC`a9tAcYrtCPGr)6ec#M#X9)pIO-u`72TcyJ~`; zArIdw9Ov}>s*RUXzVJ_=*5lGB%O$bxxKPPCV@Gew;^vDzA9J%-p*bW;ej@s(wK2LbpG@DoeUA{m-{`w1wyqR2 z3V9N9)@dh*(?U+OR_fBmG;)l4KvdTS_Onk&%bDzm=>3H+{KI*=UG!HP+#z2-eOhj^ z0j{5bCAwMqcdN`~=ftxd^p&Ji7JT}yMgpAou2NET#$+Z0;zTm}3*zELc>3%n-^Y;B z!N01&(v+mm`YIP%imOJZzE&whrnDBGaQ5@5`IY~Ja09zdc_=8s&I}^fy3!wuMKX;i zHrsFmCysoCVV@W_uMZq{!lraq9L9_y&N?g~Fr9VSKR{S%WK`DgyF=XmvB2@(xq)fK zNMa?}qE=YLj8b!>8FBx1$ol3-wnv9)9=$BETP0kl2z%*+eZ&BJ$kt)x`v%QKc9 z2c3_!r~gD#rw9J28o_ zh}TyXoEjZUyme!=ko9ss>uqB*WYKYDNJ|l8YN#ejB>Rrs^|fAtWSjeGcg=i+J8D=x zx#jGYdV<3BRy#_?9tmmwfqN|O#gZnK(I_*J(A1Oja$2eokVP~s+*=AW(vg2`7a-^h zWM)?SEE<97q$68Arlp|3S_9np@9vU}cfBho#`~a0+rv@Hy|?Vw99>}i-hXIM%BKn+ zX!ReEE!Z$frk^;w5YpE2JT-WcyWLevJ1gA8F1^vq)>IhVL;H9IVfHv)ysjH2tkSsP zW~Hl2kQCk{l{Y(Xw7GP(05DmywwYFW}yMxL#bf_dP{)>H1 z7qGLk2D*FcVb8_1{3?6P|6K1l##rv)d*|JA7R2A%)Vw5RUnb8@nD1G9%5N+A>ss;a z-TSQdTBvYC(kIRYFI8VwY2yeW|21k}upa_s+_=quh!AwoYV6o$3_OBQkZQj#Hozyr zC-HgD`+K&_Xr0%3SqD0ZZNz*8q$!X`i`VF3SKyfEr}EV30Wc%N(H4j$#Zh^VBE2~# zdG4ZY6ZnMvnrNYy^iGOf^WLrex;eIj^qMD*q~^U(*7Kk>MhL0hW2|RI`S4VnBb=j9 z6C%6EIr#usWU`y{{VAkI%Xemgm0<#pPv;{qk$IDLz-$^F#uKn5KU8mG#Solg-TgBuICZ!?ue^_ThrQo(LNKB9DL zzna>7Ru-MXvvCm%YZRRR2o3OHCx}9rE(-kwF6D4IjEGF%*Z2%BeR#k1Z<@4&+MqE8`Z=$EH)Xod(&yt2mBK902OFFKX^J3{UxI8=%wxB z8kVOx2@u}qm$XAw`AAps{RlCK;zTVh+`3N2TIDPG40+*7_oZ2}t=mAn3?M*@MP&b2 z+c4l~Wc;G&8X6tUbQ2zYKQQcnNji6xtio`7u-;YpFn>3GZ||V$GJr8!ZNlCSiD87W z*3GuBZXkCNB3wm*!^j&C;BsA~C4d2Om!6ZJ!vzO4Ja=!}otqViQwVxuE^BT4ziYsa znBw=K?J~81t~1Wn8s~l+UT6A29l}S!p$Uo313ZY+i$?<;SV$}+PiHLQ-8rFJ6R=v6 zfyCXt_1e8dEb#6x_s#$|g`BL$OjsUT_8!g8hkL+Wxo!qbnWely=*M=yqG4ydCt#)X zmyPgkUYr^^WeGXJUTPB)jRg;B%m@A1`-w651~8oSq%9bXk`IjoIem6LwI5#ZiQ#Z} z6edBAxCP4IY@9c#x`=65ux0xITb0X!%jbsO$;x!NLwT-(}#;L z?h@PyPH=a32<}0HJHa8q;!cnN!QI_0i%XCNLfGIA!2<*h8#vre~$0CR-{E&1WtN-B^fU%tKjU5TL}9ZWfFrJN3Q+wQvLkq4)lCzZl{4Qy#k* zP(wb*435%YJo~4Ue|{*-e1BF0mWP?2hj)B9IKaFau%N*NZTEIPoHY&UvH#0S@^#si zbYZb1jJ)<_&K~k_e8s!*2A^2L4+ug$iBwBzH%ve;8ru*l@c zW*ZLz$Y@@)%m#}|I{b_GM{#r3`S1qe4;=YHs|PmuL8FJOcZk&O!J_x-MRd;01#lt> zyYxAn%W%l@#a+zfROEu^2w4~M@zL7fL$a?sozBn8%ebYNifj^FaWe0q$POL#BsN`X z^hMlA){nZ)3=-4I23QZS2JGXb41Ti-LP%dnzt!qr}%+kH4khrK$64yN-m zd@3!x`%rJbS**Y81XjVV8${CI%0!(Dzp)U4_3f852HzDcbM)To(^fsjT%1Vl;jBvH z8=zqE8Qp{ZiCxLr2IZ~UY2>n7EzdgBW7W6E_+wJtQ{kqre2e1F&^DG>-9W*TT7(> zAfRpccn5ivAN6kPhh&~bGun*qvRd9W#0`*uit_;d{m0`nL`@e^<3o8)Xw3iLBK)}= zF&pF6^pE$?L799T0_d~n1otYt7_6)MyECTsf3sW8Pc*b{*nd%#m|>2DR?rSuAt;8k zpZ#3+UXmk2H$^~$cB%Bce?_}f5A<2h7NGbV_r87UY=aqh(U6#?2vZ+!LMu(OzBhPP zOF&jp%sMCVqk{hC&jDY87HE_VdG*gId(6+bd!>U^n*B)3$t{>klb)MC-(!LE5xI|ZU${ToEeh4ZA)t7k*_*rejK+FfjBMU#)c#t~dXp zowoN)$S{W6&MmSch)KfyjhG7PziN2aH&4R66xkbH1P3LVFOdRZfq=?*qjc{E_w}kb z3YbE338)nfc%@xtUkVPB316KRxG1;s!c!@98seYk-Z88*5sU+DY^)FCGH<|NRxifo z3{-<~W6Dk^s>PSeN0rDlcKVOz(W#Y5l;etP?}Tw~HmL#}gfIRZMR0x~9zL(cEXmNo!|}$JluCuyQAREUY@qvl-k;w#;3^SvgV29CPF zW&K>lEq?>~rK*`_fl7{(T1HWE?YciRp>b?0+VYPM_Ul-VE<}zlN$;j3Vb(&fR(y%p z!mdbqxq15}Nn>pGs#v;ng=M4Vco0hG`K%jEyYPRn0{?Z__ryB6l>B>a+KpzsIrwPB zrV#EESgPIPc(y*}wpTrQc*pIT7QxP?1ZPgJ`;!bx7U8Pp1a1>Bl%iKPV?Q-#5X^RD zy)jwMo|0YB%G>T9`CWN=CVy#X&{WCX3nOP|s*i%)^39@Av#~0kho2MW{*62As`*Uo z8r_+%mtgj)mo>eC@!_LtBV9ZlZbToO&#z!CWF6QikN$;En&(zDq2xgnU$26L(OoyE0~qwZ z6$Q0Ox#!DI7R1fm-=};=*lPYKqDjWAna*g(5Jr)SerclKEZ^XThVB}A9_+n(-0Xk#;B$%-wo`j(Rcjyy^5Ti8h&Qh2<^R(hB|vv(LYQpOj}jQLt3Vs90W{1KP< zCqDL!R7E3Bg7EN$!Mjw;owsnRVXPQ7H5Ynw@ww!{ZAQ**pWv!%Q;Lf5*cOf9*Y5h? zOe}0h7SjtolCU#`g_bE~k{LQ$R7tCMd2H1eIjlD)p1B-_dcTQ`4GQ+&Ym&Zt2xC~o*%HG!SG88VJf|G$pXiJadu zjy|tVFNRAOWb3NXnOQXdAK_{xvtYU?8dMJF`!(E8DqNDn>9?Ru_QL=yXA&3}N#a`o zox*e$BHZM&L?Oo(pcr3Qg0n}-L9Ej{L@2_5qC?R)(h#J9&IT&B7CMaCqD1CUrHBny z`M*vL3>I%UhNa*AYU5cF?e0fvd&t_{>CFup*e zSz=87eYx<50FBAe_A$iM9xR5%X=SA5m8}q z*6ss0QPP`_RJHZ?5iHmhB&JsGOd8aGmrnEge&2jQzBhJwFn@;~QXtCF$uRdxt|`e2 zdmvRwRh!+tfA|!?qp3|W`dSgH*n56b(c)&ekmab}!!B zqCdAiQ-4G@V{i3(e<`6RyRyr2qT=#jvx#e)4}V{C`y9#C z_E>rtKJ#7`R5a}&y9`U?uyZ4-VSv>1Zd`!jNE3*=%}|>UW`1-F{4T^U@`XGZJ%x&4 zljiaWOey1;3#oyfC)tLO$Z;>qi(f z8aA&*wL-ZI#~ zG01G5s3*&lLTtdQcD87nW9)j5S9~R!QP_8b@-d*WXma#b4c*78_^)gC@*BOPy)RRI zn$Bs)6H70fmGgVZK@7E;{V!d#NJ5Q7AsK+}sTz_YIIvqc*fzZwu$6lb>&ir=y z`DN!X^U<~5%&(ksLjJM}WO7!Lsx!P<+YlKU+s!w!3qEC2XXaPeBQ2qnOMK(nI_zXQ z#zE48s@+EO1L5d5Eb8|NoUvrwI*R@eL8`uMozZaBQ?@n0Q~Z&{?ZRv|h?FNLS)n%@-hzW!DrXeJfuRaTSm7EBiw4>1 zn6EgcBO{3uC=20A0vp4~?PYanLav{qA(nka1v;Gl)(oLomV4eJ?4}Jin0TCFTUCFY zTyJj`GNHxTYdv@C;O%1%3|lGHSL0eVvI%(D(unZRwJBu?ndm1K($==x_SvKPr~S5= z*%&KZX6Tis|3o$hQi`;rB7d)nIt0YtBV;@VBJI5^>WQ*RbX7q~-K97TJEQrrw8y!D zlpHq1!HU`B+laz3qO_+=R{gy&bo7<|EYhyx5I9(A!{`Tq|MC0CV^III|23NF>dll? zA(L;Zbu9WVfVGboxZWdk1vb9}ABJ>#lzYn+jhA$PrR7oGsp~J+;e02)uZ-ZC{9fb3 z*FACVw;_ux;i+@K#U}2B4~^AHR6QpfCbtQ}1PN+mvNr53n|9fLQ_cH2!&`)49X0um zoV8^_9UV2=JnsIzD?9GC;qhv}Yps)(!9a

J1_w{9xfJZcx;3_%^v_1dNFXYk34`pDWBMqz3 zU|buX9GR6^2mRQO z65iEs2`50zW~oo{n$+-WwY!;qoV71|fcFH>fRarKt0m(e;&X zXA0wXT*C@A>p^K#+z~!~IYTr0)bXDPrVU5sUgY21qX|w$!UZVXSh2S;w}(q-KFzar zgxvZtv*e1_cMn0Q*q+uccDg7kc0Xm1s7{;+8jb9mufNrPep&dXTO$i9@N?7= z6|(h%76Zd$(FWyD4=1<(B~th!&ZefAv>o-0@bIbX-_>j@*Rxgzd+9If%9#w%b)ng7 zhY$Sv)TnFUPoiea4}~?mQsB+goTH9(@*<>|%Px=g-Ve0bZ^r zzAKk_UzT~hYaXiSxBR^>Nu24uJww7k_gsZNZP3O0xKgVUyd$cIX=1JOkr| zn{s|v`Fzv$D9Wljn=P_Th6l?BgX>h7L$UQl zn;4KUON;->%zWRg4*#%A#Szm9eS3RdAV4BVM4WOHw`@;iYm<_wRd)VcnyrP)A_WpE3wwHu7F>%0F{4h0X5vtpHhl^*k_%W_l`O>k9?ljoAIEj& zllU@?W|?p@G0FpAw7;Vm;(m&=Q#%^4xP;9$RvbNFiXZEcteoX3_}~1x%dM83M}Obt8k^m= ztgEAUH^aL5c<-^ZeP877b#2vp8H|OU&)$ut;mB{T4Yq2<;gF1|6aMa)<|N?crI4fO zz2Ul&LURi9$E2pF4=YU^fK)~3hp?h8M>|nIL#4TBJm%Uzc?i*I({lzkF9~TsuWuT# z?lw$D%?d*oE{)M*C9=?v6hX2C)>vt!gdJN*4AQIi~p)-D-TtJ1@z1Zoqf(j-z{7R%_Vlx~&592HhpL zi5Iz=-H&&Y0p7T~_yb$34NgS;vYkb8q=w%L;?9btp(dRLE=ArIi>x(~xrm6}=)YW^ zR)gPhqatvkN44Jz#dA|~-uWrsCY#TJGRgjLO=XhxDZC0{U`~ zCDeLm*y{cJCb!~2j(MGJ5z@9_^* zbH`%#Yq1A=sbVc@_=n9EqkY{7#}gMLD?Y)!CB(rT!Em;&7L7CC#Z5_!>Jh7VKt+RGw)o!I^m7*RCC#hHD`3=>se3yjidpM1*y4*ov3bBud!#d$M$+?^_)b zhf91v>SmjaHBqg8XW`o7>Kie>AG+>aXAYdw;0Ll(^pbrAlm8jB|BXO=+1i55$2q>> zxxGx^^KdH0_#-<{43IiJ)j`d#lr^K=H&h%USYbF}$w`w4)Idw-K%Nndg;cz>M&HI^ z|Cn!R|FdB)jyrdAL?6@f%q+rK^+t6iyObOP`_`6wS%Lh-mKRmgiXGXJ$EpQ(s1U4x z(f4(s`;O}Zag^pG-G(}`d#mrsFi^!RI1{s(ZvaHcArbi1!RhQpm75GZw$4i zMdd0-CaMR{+LEGbgeE7ruF*F>-i?kr&n)#mH*^1USVP!h1H9}VYlO)V7OrC^q}}VYs*9O;6au&#@?~Oln=?NQbT$aDY%rw{`k&S;e9G5)jN+2qBX)w3RP)H zn}Low;+Y2;n@9<70OGJs%6B2qB&b9@ph9n%HN z10_lWo?CI|0*es9XVY~b^J3n zuJmUJ$^8=@)c$RrUb+NBgx4B<7p-qq%M&ro2K6BFzBb;1$l1H@!ej_6f|kN;L15cB z370Oe#>bAlrlQrf5Z4|rs>7dkdGXGu?}~9UEQV^2FOwLxShfpQXW1RLekR9Nz&NCE zN&DKU#8>#y5Sob|!G>&`rRKnJh+98S{;IME%6GGcthv7Vs5R$NNLgGXEUJ{%J%`Tg zU@W5C5FT4EYdVL$lCcrX2~#^|mW@IurC1o9TcAL;H#s#-=~P~U`T{}ekt^Rhz_@Y2 zy&vL)S4V^|o;%G|;Y!Q%_PM&SxHUUDH9tT)cG7cDL|J!ct!4Q0cy*RE^J!XiDLp>3 z$7$Z2LMPB8cezJ&vLK_it%v^fs|QSVx}`qVpOxlbaKD=N#mcw7!g%}Xq^Gt z2oUm5x*NKC;~qqnVA?b=z_rT4WAjBrx0+|qNTg(QO13UWav4Or6{C#cJ&T~oGQM>Z zQI8VuZB$+-HoRq@$z{B*!c2h~9)}o71ruHBj_dq%FJST6r93Tp)M&?zNHRUPaiGa< zlp)nYWBm0vi&c(Yq|UmrgYKd`L11(|rn+aj^L*E#T>>|TFVHR0gQ zSbV?Q{GEr)GMSB&^RL%o6j%9y;^(!Hj%40#bg_j3Xd$J{xjBO@#WKg4Uc@6w#EMrz zbXibEJ^TJzY-#v|jEkHrd-kTEZtyK<7ktd6U{wETR92!|#}%%J#PBAGUu!3s`w~LI z;CdM7z%Fd3Fq@?pmJ`Aff2WIg7okuN`TWuvt*2#7KIRrf45B5vGf7{*la#(o1~_9O zTZS32NF*XbvdL}Q8-}iT#E2QgBLWdu%^xLvF#nUG_)_VOw!3FOZoL8fXY%hllu5?*v0x4M%&O(g%xX^ z4t#UeVNz?&m@xfrYU&Bq;S}2!<2t9$jcdpb(aAc2Vt$3!?SYi1*d84pNYIC4r{#Y) zdCecx2vKhb#az+4j=`~QyR(aF=49av2SxA>aAb;?mpp_lY3OF3z(v=qaX(uBO{qP) zyhqop7+CG41%U?cpkP+nkVhWwMI-Yv6PT?S@_PZC5Kg}R?btOChiL14@>c>hmu4>u^bjR#!+4i#px z8k>})GIKNI>nE5{H6Q^%NDTiJ^q9^klwY6QWDimsnb;1wRqgTb+U$at<)Zk+`pBp|u~7*; zyI&r+DZ(F?^eDcv@EbUEHzlgxWE?+>Q7x{ETRxxAq!;@f`0|GAyW9b?ZWbBkEU^Ki znn!}^RzE{@@S8U0kWCg9@~(c9Zwt-X&M`uW@Wr>nC}TCEx&_uiWp4hOaYI$5+T z<|xyYQ4xNJDun_rUuOG^b77~Mm1ArbWBGP&=LlRZf#G{*(dDbKBQkUBTK^G{Xosky zpSGfZp;#=?Tc~3V{D23k6zh({78xyq>)0~;B z)i`bX?0azp%>px4WKuwpjaj8Qojb)-gZo%>sS?*2oeAo zpNmma9%5^%+aOtibW2sUp~%z8mdehA(@EZLr=H8dMsxlv!FF$52sb!mlxE7wmj^6$ zzZ2i#0xheO{bjEOc3bM)6JBJYNI_9n40m*Q^yHpe4tP#@&MgkqP9&g3bkNXPQ4hJ% z+3sc1THj53A<{!CidL1m+^vf-ZNRm7&9M!n*6wG38-b__%6u^`%TC<~!x@Ti6rH69 z=N{zYHbDK9WL}0gcavU-Cm#Gi$QF$ys#iBnW*;iHeklSmV;`j5Uz zKJC+z&~Qb7)deW595;gV&ze+$f~~628)BMy_R+sx(vGuTo_XdzsoTpi~ zRyPcNf?*d3aT~hIe+eRAW+H83Xk>ki>F?R=g;A#}(EC{B{-MwW ztH^{HsR?huV`nQkgL7410246hvDB@)Nf>s4@{$< zX;km7*3W*~_8~qTX{#_ii2^Pp{$wQ}W@@FT%*{9wG4sm%n5T~sgf~JuMokfK8YcCEZJsuGQu1V8p_+y2lT51?VMK6Bu}QHJ zl4Wa*BK4u~I?_ObfIEBKBxwY!1iu_-4`?H+60z$wwhGSP%bn8A72JbAZ_Z^3Zk=l~ z+E@Li6kwc-Vw22(n3n)48+$}mEcOqL-x*~LsGe^|jVL-uz|LhW1$m9sL`0($;FG^b z-vVI`L43Nzo4A{}>r%N8!cCy919<}o3xuT?4ZZ@d0`Bt#F_I|)&<2M_LEOTrk{Gy5 z^>08jVltQwK9exE0~Ur*MhUFFq)VECVJzwBD;b#^Mo3iJ=$#T`Ge7XtL6e3dKm8|{ zYQnI9JOQ#rOJVNBduLWztyIq7+X?o&`VY*IFjFd;i-Z^iWI|AkK0aKx6g)CIx;vwje^RO>+<7(x% zyzfO!uxwPEj(jBgsooU3HC!Ct@8fvCILN(GsH|VpiD$SClEh1k>>G0gwsLEo2SZFD zk2h9GZ^>i!vk;Ssg~CfSA*3tqNJ+e(ffvBn_rG;l`0KQE#%|P_dFN_}g`u)DY?F}U zq%kiY^|dPvRe~ZGNyM7Q(K<6T;!iR(I2S})8uCaVDbWZY8qxoIJ~F@~al8yMcKCm( z5olw}JNBqGa@-flq}dt}c#1~~7?LZa~u83Tn!=|j-2LJ@f(d4#K!D`7W$ z_sbX9MLj>FI(R3G=Q}#yhvyi%${3MD|OMIm5AmQ zx*41q4d47L%sC=%u5KiY5-Ipd_KYw+^2P|<+zVAfI5KvR5{?shvXEc+p^4}Mjmlnk zWnYZZ4x@bRrCgn(L5F}sLnBD`4$&J7H5)h6nG^kN%aV6V7##<@El0T$ zDXUs53k)pcw^rZ*baecMAS_a~7@Aw^s6xp9vz(iQWX*7|v?;o z#$IuaUT)N6=#X+=a|!!jZ|!mG-v67*YIr0m(^3|p?cqErb*XSr0WA>EYo>YSJS8*K zQz;)*$}IuZgFZl-SEQG#ULg?Vs!fd8WVik$aCLAwTGIPRsreg9-`wNvCh}CN!VFBh z(hqlE?RI`Zn8KFkpm2xFFFKa2eIxeCOA~(pObNryqIP1kBc%Y{&UELiebgyc1}26a z#%an|a2!scQf6FsPsw?B)Cv{jV?+|~uN~Nu7h*K*ywB|v|CnaHO1@Zp&oM?>#3)LK zykRUqF^VygerUa3pzT`Nujq>Xx=aPTt^c>a-R#F91B(;1FB$HrLrI^-z9b1B5Q2<- zYxt>kH{V&6EFwOkXN^V|9@O;(UfeO$G;U=5`l$Z2r3Mk`BzPa~LxP1fNE0|#Bc2uC zH={bY*<`ZAFEJ&QgiU1aQi83vg~_kw{V&9}bKH86Z#0~>iA;#PmmIX?By`|See6Sa zEH5gI-l)HG_6!5hq)bxwy+E#O;bH+`v*6bf+LdD*rmhe; za7j$#!^G)TS1U#~kuXn46F-gG?Oc_Lwm9mtmCU1y;Jv=r^+tSDYTjY(n$?uIGv`jI zZ2b`@SYt)<*1=$0Ar}6&D=f5Sq?90uA5i08)WcDD{h_(kbs4EB9;RgY2>7=EL%$B!SX` zPYBk@)N*=iN=;OFKHrHubXI4OWlqcbf&Udin`OLQ6C7K0_b|Tv;&ve`WReW9bg3qs z9R3nb^2<-wXx)en7Fx<=OeTq+#YbviZ#IktzsH7u#g!EzP z_w^4C%wW`%B1mc1o=32hmgVTGNTycBhE^oyX3DC z1@^9E%)p)6@O$ntpK_tGkO|S*ZJs2rIqna<82L%cf9?hqh*!leIs5Up@9`Z}N?BqX zrA?h#4Y?`$19$D24q{`$EM3;oD^uBbZK;=S#&z8&>?h3L7sB%@ z(lZ#iYO@^ugidVyVGh*BHaZyPUk3J{=%eu&=qE((5s|C@@wTjpU6SAtyo&qYf#YFK zCn{2m4>?3uVw|J~mbm-N-pqCQ+WnoBGba!0d$Dz@c}>(7fx_gZ2b!T9%9S<6is08A z7j1Tp&z)frOsSM=tKk(AX-evDE@-0kGQH{HhEULoGR5IO*b+3D3q1U7G)Iq3Zd;rJj9=HP-EWGDn32PuZ$Bzs`Z{b=B#D;ok+3nOr;9m&lG9sCTak$@2wJFv zUXNmKvG{W9h*mWCZ5k~E{eu2=o2twk&3WLA`PpVu3Gr!tGdlZe^mBSwX$9j4xi=rl zf*gopR3u6ZKt%y$37pBO%Yz|OQ;DpPAt}kNk8NMh5f*|(y!$1i? zpU?!LlC@sL!GAd#-aYhZT}FZbis$~3990xXlj{7&*sRUU`iIao{0d6$qAG7P3u{yF7#3MTp}up4uCJ?~w0ghVFe`u)qPGFE3)|<=S)CEamcXwr$tzR=C_B9< zn07M^VoRg1+u7IyK1bIE;P}t1`i)q3GUfPWlYOFlP4D`(K93M=osimiJ|vx&f;iJ! zSQh2Y#dwomsiQrtv?rz&1vuwO2Rf$e+(af5tgqBITt9ggTcSU>bCa|n-;GO7qZ)MM zsH+{cL;M$3eJK8>XpTkiI$B6!J17_PYnfQFd$__{-F?-WNNDwKXBXz)E}?{6wKzT> z_SG(zbBq!ydWx`&`iUiZN|Roa?b}&ddnBqcBYF*Q6a&b%H5zON7h4Ze5p`f@PTuLMCRB)JK<6`zt6!39alRL)drt~_ zmSWMrat|*W^Nn>iz?KHl4^6Srf5A;7dS7 z%C)`@IRKfL0~=>uTYOo)X6V|M19d#_%oxPM*Kc{u3izPoZ@-g6`-S|$ndEMl1c3s9 zVvu|gxXU_Mu%|V7goVuUJf`q3+$$hYaKaVcf-N?(`21I*23@GlBkt#6(q-(nPuxlu zWKt#trd|GwfAC-81rR2{7zuL2Wcx@Ote8VPtt$96Z3n*Wf*)#oy=zNr(faJ~hf{?K zFc1z13_TT-!=D6Qe-?c~!P<_?U#5rbu;i;CD1XiH7Zmw499a10=j2V&qnY_bm_5#y zZ;*~KH!#lARK-^h)ew;*#k8fn?xc*4ol?xyWRO2dkQr)h8$q$YXU11oB%F92S40Vt zHKHZ=8Hm&B=t(c%9C;!$+n#-lM7^trNRG{2%>h};XbTms16tC@amRA{f_g6rXaXDl ztmJ8ET{^{^R@sp>#}AeBme|;*L_!^hM6+qo*U05>$F;|>kC55bE8F-cA;Sob2GIGv zR}FX5a*!E@j;W{}twQ4T0dq!GG3bKD`iCHAG}Z4~w#Yy*$=}=x?FO`?#-&3fGUHZ& zF<}2s_(10UWmV;bPfMJ_yI)l!-)IYT!(-I{*O{O7ZJQA#o6uF);w`ziUGjRzYd6eTafWt!ribY0ias|;QF^7U zw%p`Ww(k!Ibpn&K-g+8p{F=d^8EPnqs5QYGxjK?!Zl+V=cGwfk!lInPYOx}WN9o?b z-{)1Zi+QO0_LBL5e9Rov?zTZyaK=I1QkpC->K`&nF~X`9oim?`>x!x#({)&qR1MrU zl3i`;$E^8ic!>GqA#zhqy&fUbDL#L(|LFTsq>%CabMeiqHn&ZeC!cZ-_kbWwv6&F2 zR*{~D<@}Al6R(vDl=hoSZA^_b1J>81)LgfGVwGSE2&z6Zym`^0#S>1>X3=<(yCvU1 z$f+dTo?nG45+Yje>GzKdcvJ4!IP34mOb?zbg)=Z&1*}_GhMYBZO%W z<2~$?*97I8S=V1CXhB1kBdGhhLQ}8-U+Ezt0aNKE?M>B=6^WJxQ-hIgjlhmag4a=z zYvL;$PVz$;H{9pnSl?fI1^o5X&5{j6=p-642X|$_jCM|ke%1>Ygs@H+-+KR}t@K{b z5QNSX?VtKpbf9%!zL&!!GC()8EFucmZ}l$2B&u8aZ_bbucuAIwh<}G@)2mYQUNuaw zQnqa20*kv?o*aa4yXD*K5lHqYXLMM!&Ocqev0EofoeWkN@-Ao(bW)YJf{n#tCM1?J z+HFN?4ddmplt;~(1 zi}*QXwMk03eXPFDLKXhb1DQ_F@Usqo70WkrPuP(ZJZ$B&-Ch%skFC}^pYPW{HDu@~ z;uF>wwLe;)8c2L0R$&=RT13QDId@jmHH7WfwdR>C@5!Xh%K9Na8ujL60&Q5AK%IXs zeeh!=2yAf|OFM(;&~<->k(Q;p{7%~$J4v%K&oJRl7(*n_=T5V2pV$@T_;=er1C!CM z+XYQfdSR>G55@(`V>Zt$G*Y;i3!&x0#h8y{=GqAn`Hh~R~6N^%Cr0*k7H(~RA&mm|u$?DBW0K&5KeTBr&U8foYD|1w)E0nSJ-RL1w zkKvjp6T&Bvbt8EiW^nx0{*E+b?j1kv$lp34L*K4|fV$L3 zlhjak2NRVML|uY?Rc)`wNi9ffinU~psm;zyk5lU=r>W|%V@vF@rhN$Zni--FFmIdB z_Yqml;O>`HUkJ}+MtMmM&>%WY7H3D3E)b)<`!b${Mr=zjDf^=4jhxB_1O|z-?W?@i z%C*p`HQ$|ipnMa9m^}a_&zs)leFo+JhwqoqwyE`Cnn6T? zPOnJmy!YRka%;jE=~WAo5wq|RmZwF~K>0{Fm?yvmhfsV0TdwFW;i&X|Ahp1ar1(XX z4v@Wgk;fy?Rl@|(%jGeX+WG!d!Xh6PAy>{-=;Wht;@FmHzOxAsBOAat$xyN4U|+ZZah+loG8-I!b}`b$>Ul>=bey z(-!hzhz7T&M-P1voN%E0otYrM)FEe4&wH4t!lK#Mf?1kXkLWzbM&ifyV4WMRcq7xsG z0>CF9+mt^qcK!FhBbH@K@LA1+e?)hxvamw<+v&D@>=0u)@eBHmMnNl#K?bBF0puFX zmeuwvE;`oEb{}6v)!rd{lHXtAW16V<#Vv3HXn~c@k9WJ>NGSshvO%EyjV?|DScwqo z9f!h~;lc0u9Ws^eDWYioI^O~kVJ4ov4k=qL?{Lk1qA}-?^HdHmSeuV}wd{lNXc2~R z!&NnEwXNhz9BrV_0=4KA7XXYNAkZeV^H;WoB}R4NkmEne8Keo` zm}cIP;=ylpinIB!0$5k3h@lMr_?i6K{>Kg>*WY%xboW;;B)xrgZy>hw(lGY6D^le$ zkc#(}o5kVU9v>&Sa*V0EFKlP?vX2{L0qv-PB84|tx836geon-$AF*Kj5|L;9Bh+w$9spRpC#oqpp?jA_Cq~gn@ocR*z zb~IYm+IQwz{)Nir%MDG|EvbzI0@)?WF1I02Eh&%OJ{ar_DoN0gURg4;qQKguuu+^X zGQ-vb)wG9hZY~xDq6FGc|0SH8iDE_c*GzkdASK$`dN7sDHnxJ_{|}(%k&TZqPyr;X z5kRyohlm$=l5@9dfKwUi1i8D~DOL+|w}tmbZB! z;xb1cvifjG8=jxjx866V?Q7mW>cnzzkam9xw=S+Rjqf^#eK(osze~Irxa!dm{w0Fz zbc<$ZP;rnM?Sn``HkUA{_zg8mh{DP}^uLTctvizUvs##Dn5MH4e4H+`fV3m1I*kR)-R3hW8X+r(eJ%IsDcOA zT+w5nOxH3l~=spZQaFz06VB`$fR7u65kxKM8?&h&Smmg_$v^HVA# zBKeQEAf6VYA%px<_)`}Aer*RdI`_sVsE3fDXY;Pb#K0$6_#_>?A*}q+K;}ZYE<$_= z!^?4lNgDV<48leonHjDrqCsmI%~Xn#Elh&-?|vc_Wka+hLUvPDij>u+GNp(oCf$Te zKfroihf(tcKZJ>?@COiR80Hj+dA8ak1_CZ_j_*tMguk*=R#mk9;e@;G%7W`;F$(_R zfCRygSQsCTTQSTzCF^;7StWxyIj($iWo~~~WzmOI& zpv^G2P~2=!eyDJShC__!i%_H&erH~ssh^`FY38-m>fx89?g{+fWzXjCaEY&uq%{`F z7$7%eun@Y3S{((jd_e{~ze_ja(-LhZgZ;*9i6C;dSlTy9**Ax~NeKDLqZ;f`4U@&^ z4=H0bv)Yfl{j~=!k*Or#k8o_<9U%UtokKx+SO@k(Z0@gt3w$W!cNAb{`6>iB0W{zN z3W{)I8bZMF9MW@^(uTT6Vi^E`&c(@^E`r&h!2|lQHKV<@iQV*i6~>};Kd(p>M3LLG z_hu5gneLHCtBY(49bQq-?7}5ChVlJylN`zAf)K0O<-i?0wK+~^LdN6WE0y{=j%$_g zPtx?@-z7_U0et99fEbm`;-Bjz*|+YUZ=Hz)rpEuAT}LWb+m4pZe6V*2KhPl*M>0~C zjHx2~WR>F8$TNJO<=37=91Z`F|sqYTuMdJV;P~Ztn{j7?=%rts%1=*!3aEuM+X-afi%a6()>S!WOQ* zXOdKu^oD@_neo&I7YDOzyUbr?ckkP8NCKoL!m|{MYS`@aNxha+n#D1JrNk2MMGm5h6|Rsx+Up( zs3kc3P(hQ_wXX2r^1_PURU`t@*Z9$lmO$SQk_UB)(zi4wM$QhHmaMDDLzk=(NoU%k z!<7k7Yiv())?YCeDlOXP`ONLig^~Sy45UT*ngN&pzqQmnocQCvLr8+%a;TKB8TT@l}U@4^UyYo|P>H`umpB znERm~lPQikud)2`-*ee|?o)%fJ35%7lQUIxL-{~P_dBtHj@lMD;7v|Gflawku9f2_ z76;*gggC~69$DZ>b)>3kT^!1!+HLb)utT)@4CBmap?5G-XTU9YEkLog_R=67->Hmf z4~7=bmAFSI%VMfqy-N8zxRB4GEtrBeP1YBGG6~N1{TvU`s&I$lqpVW-L_C{@F&9r2n6`vMscTOKPrMk0tXc}R z0I#gqs5=3?q?gog1j&v29;{3b!2RXF=yq0TX>6mWwH9WWlH z@N&qemmHw}VG*y)m!I~T{ywes?-~x>-1^>r5;a_vcs^6}j5ofgOlGxC?=tftw)HG2 zZib77i@w+{ZPzi`l-<tVNFZgVo2N{LpZM@-+{PrD=ZLw2%9m zJ>ZpazpY_crIGrK1LJM9vtPOxJpjSH@`mxG>wO2CSP6V*N%NVtgOww1`%&?kXLcw) zZMjk_;*__@n9*_zHCzjs%QwvwBE5DodU;>d6ddQXHZBVA?*vkISk9 zLQxfa>t#81G6D=r5J!oR<@PP%Ffu9nojFX(+w!VeW@o5di*L0G$sawj!4lj+x0U-P zhlq??wgDreJ_SX!uj{$2j$F5#b~`fOHQI=2xk%c!S@)jUdZ04A4|I;-7vJurmECX# z_OZ8OxltFPM1eW+C4xS#7B&HXoKa4IgFA6;;=6wKnM^cv-(~S><%I9%bv?iY;73MT z5^Xh^g;E(c)dk9rpRXZo5f6;x(sO@}|DN1a66P46%A#P-G$8KleU>kji=B7fqpC7B zUf_-GA<72(^e<;KDDDbi@sVhICkPKPo+3zAbr{EtI8KDyWwWl3j}6uETbe`|&Fl{( zN*Tki{+q_zL6(#JL3Gv>t-PTO8J}mmE_vh2>6xttz(1K`5}Ohe6d5!>7L-CQ`cU>C z)zVqO;!;47SZ|&*RiW90HIA;jUHL{CB$9+3+_KTM1ZMES#WZ9Ta*oV7Op~iDbaik= zQ>P*9voQGDXU;Cq6InIscev~b_7{O0PHX{mwGz0l#xm9W)}cHVHmLcK16q^!Aj79C z-9`ftUSU~tXt^(UW=m2VS#7fnfKEz8Tj^Iegu4wXl{J^K`PJE*?R)%CMgV;0vQfb# z41Vm#f!7Cflq^H$s94pj(l!0;;NG>_GvsJz7tw5l|cB<*+0|mjDn44)U?4 zu&I~8cMtH9rI5dL{(mswOlE!ktQ6K}vVO)HFFO)Dd7RnAf^-8JJldW0@wjgRR=k3u zEj%_n_Lllqq`|iWwW4T^#D7!|BO>qVL$debpP zjBK&SHT8Y_qh~@m3Qr^0-?$zxq$tll_^PwzS?VeF5}^?veUP^TqP=WP{qx<;AU?GN zS1&i{$LGS7!cB!f!9i7eX-y<7%~NKa^toPc(ug_14%eFHcLoH-AW}@8tP=Kidd&;5 z308pFI%Z&|f!l~;rZAy_F>zS&-@Brs-%z}FglS6tePRJKGO1$Q12qYu=+-V6#_0vr z$m3<7;GoD@28E%6Q7hl=szL0(?ax#)S;p1EBX013{>Q~Z4_ zJ1@Nu(E-Q+7%SISeOh0}+ekJ8t0)J?8$u>lws>nX)||MU1`LCGqy!zp^+N!Np@q=) zvh=#|0dKQu<09gP!+jxf5j6f^GK3C#~oyegYn^rNt2XWCl`bdlFBI3-BX=!WJo%ShyNTtDy*uZ#0Ce zRNhtq8fQ!KCmfjSgGM^Z3Mx)Hi%py<1knf0OdxUC3X*uX52`Tt;% zb+^~eV{&i;WH5UzDQi{1#<`JEA2m|woU4cCQGUUgo6!6%@n_cLgG>}Ui!DV;e^Jq@ zE!{V)ajEbMQ6!{RNRFfy!;PN@UQW!ZTb>IWFBosLpi{&uie~NneF4=Nd;4tsLWlv1 z1OV^3*3Y#Mu-9Jotoh72SIl%7G&agbuI`!X z6zaPT#B!NIaNRweFc3!w;uf zw1TDa8X|+U&Sw#8<0EfP!GXY1uT!EV_MTW2Dxr%t013I0HyEw^9#BA2qEVlreR@NW zAP~Ri0w?|y56Rh}m22B%@X?&1sT346wyQf*a%RK1UR*@=2+!+80O%u2QmWOWc+KTC z4Cs{r!wEpBh{k>WS^hVN`>|na-yi+W{kb3mxpU90MXR4#HOv=(oLq}87uNC~-}2HC zAT~}0v>-t835)o+aNq-`x^Nh1-~&f6ZVNVAFw){P{fW>nkgrTI=?I{I$?Oa0jS zp;2qJ-AuyB(vrDeoCy)3J+56P{w2cdE{R9uTqL~q`+1~8#2yEq@!yYqIUW4CCw@lO z>8K&8`$@fsVbH$xHQH*yC$>y>TZ;r!+5PKQ^v#|LP3Q^JBHx(|VWUp75E(`FKCQ{H zPDtD{?|GfU{&!xF_cXI@spbM>szy25wLq~oRXan@L%CLJR)3p zTu@1}xICskX4-#WA<{TQU32Lb+nt4$%r`e;dc93iluU~JsCPs(4%Xe8O!YPd6Zhqd z&t9lY8}2A*vRB_3jou+JXb{~%=uLIrbdR3Znjhb0IUa^K zNEK(U#UTv$7F4%Hw0JHZE`MeqA?8Qc5{LQzEZua6)4=$V9MqPoXDa-fWGxG1-do7Y*kR77KJbIG`;tm1!+9W zg5eFJM9%iPuMLC!(m_oAsQ63{K^HvPnFC~;-NY8Z&&241*(4k}@rw|X4Q#IUR!>wD z?DvmNzu&;4Y^=|TA;}ix=ojx5w=N&VhSqO<{XVDNGLr5M zcv=n6-Ef*y54d3W(-~V0pcvPe_V%V2d6 zsvy|B08~cSO5_o`%D>95B^^y0zn0c^o~Po6uL z_(SHQZ?oPn)@CUmsON?lvzw2*5@QGhYXA+Qhi7pY{bU@gctoa4`n&|y?y5!foqMyD zR+O6WPU{-zDR@(xk)yavHh#1pmipoTYM6dX*!Lu#MShVhUTmtt*jv67nLBHaI6`s5 zS?*POxy-EU3_T0Fd7w(hq$g(g=?+W(Q}(U?{lv6I{_02hpYXrWq}_}ILL7&LR(!}Z z)%-7+c2-5zhU7I1uXrpNb#+U8=&l!*>LVmSYJ#@ zt$56*oUQo<5(z>Rv@ytC-OM4gau)JWlQGFFr;_9GKh+@$Vlq9s%>W1HG_!+b4c`V8 zr?tOT#(M9EPkCKfRXa7&8(Nhf-I8wZXLAo~w8dWK;z3FSP40{!jZ)T-N~X05kD@Rn z*Lss!+Cxn9wKnQC;D*{npbnFd&@uhk?en%aGDJh7-(kQ$Dh+ml z#D!wNsNNZU|5n1mTT7l4$AFyzPYd2DEf4CoBPyeyn!a=BF~-6&yXMsm4c(s#ZZeeP zr*om>3nOD|(Mj1Ze-jwVyFk+DVL7ha@Riz@+uFhrs*6!5M9AG9nwd z;&@v!Z@r#@N~}?X23fp3e7*hS^wOL6o>o}me%u;w^~93^r(;-x(>LHo1PEJs2z=A? zdy2?iK%(TAoii~-&!ev^0}#yAS_3%YQCE}9UVm4RyYitz$Q*$l@TGve;;a{JneS`Y ztg?_0=#1I0cqR?Yi|@G6c!d=Mmhx+1((N5C)83AWiS)TneloAB5Wf3W;0)6^BCXFM zg46tDVqOPeK(MrSz#MaN48o4+YPz%bWxhw@EBfH zyhZ^cK1}1*e>`+!b8mWl+eamW1&y&`AdF`Rq0R}aAOMp4ng)M*`MCMPoty1hfF@eF z<>lageEVkmtkJL`;I{+*vqd%><|)|B-`pFcfZ|IjT=1VSVqLl^0{GV-%x|zCAVRif z*K46u@Ij}SM1djd02ct_-J&GC+w2`Agu8YFF&h}?zt;rIUS1z^J3YQ@TnLiUYzbYC1<9$byoa3Rl!?|-r3H=on_aAs;|f;j&*>@ zFZ@l!51Njd0~1%IKszT_zf;{y*MbehmK6g0{&9ryYq9O+&agFHoL?VB%7sHy&_-tj z2G?AZs{Es_C0~o2m2mc8j?*}u_AWr`Aj=f?+d*3_n>kUVEPKKr9_S>`7}wUJ6(`KO ziPrUX8bO*;uuFT_NnjM-c$2)^!~$jS@oiLV35*6-E*nOV9^ucXy^aRo)Ask9?t<_! zDx`!Vo6IaHB3rVCQu}lfwcmTMbd!LHKma7pJd@DSDa~?{{n$Tyeh$Q-wofS=kUusG zlN*iMC8nv>bB}8~>LB=@`yO|Hfkv-LTQw`cV#(ZW3%aiYy{S6f;F(eNdYyF9?OmU4 zCjWM4;e_zcqKw$A@K1>+3)Wwe5>19|Bn15>$s#hNeUxoi?@#gD%~JFoS%2XmjORII z*5_LlIPN~k-4=VOAE?+y&lao1KfxCx=F_&#Cw@P;v+Q&4%;;3T+1&>_DMR#_4{3(rDG`ucWsh^WuJgH-Q9~Ic5Sel^lwEor%pVg1PX#O$n(;-bO`6PuX_1rUnXn|ppmL*L%QJx`^+n5 zxw0RLH{>#Dy^NPILX9?Qsa}@!!~K;_Rh1W_dHZi3d`wyA)x&1RRCa#S^jOT@SnRU^=Y`Qc7f8k_e0GX&>%A@;)=aD8nwvNRZfZ zYW96HYoCfrUopk%w5C1tbq}$AT1Xt?=fT54m#->y-Oix! z3V-Ry9(kGr73q{(%+!XM$AohsO`sT>YWQ^}EaH)R-8x@;%nhp)qzVRx0azss?(|sV z!~xGl-h!pagzAxd=}(=a(3@RFE1Lyedja$@JiZ>Q2NdmOEK`;3)OwT5cd~=O%U0cJ ze`cA$TPcDP&XH4wIbe`IC|i{hOkuan^^K)+P$}J#H(xr+YT6R@11zRJWLvk>2EV6; zG%3o%6zjHvScfJp$f^@|WF9Q)#QFyb5nSWq_51lkO(adgiS5>qhMd?`WD4aC%4RZ5 zWbabMISmfV40%M3L%nyz0^~Gp<{4%DzU{w^?B#0J<_0H79ux(~VR&`8dp-J)YZ)bSJp= zeYVz#k$JtVxI!A@YyOt4D0-U}8apiuxpU(2Wmp7iICDpk0LeeUBN~tD9qGi+$Q0lp zM!=~;)O2F|$p4%}rOebMdg1kK7GMk^SAG0$@ZtWzA!H6|pW#MsuH?fA*!_|?^<$Pa z8FtMRxfjzNpZPT*tx`OpO9>j5O!xWTe8mI;Bv=aKFe`mI$F*bZTuVx`l`a@6<91?{m~76!qcphogw`Pc8iy`W$}2+1yA!idlI z+Gp3lsKHEOgFgh{3@cUm;Bi69z8m^Uz~C`RGTR9V80A*cTXKm;yDf6JU^vT__N{Yl zMVC&#vWT7)F5H6zJ%0Y9^cwW6T)g%8s|<|j2j#I4vy`%g|A!~)cq8i`PqW~M!#)RI zSx%?b(Zaiok+ue`DXc_7%V~FODfA?tkC63uA=FT{hGgQaQ4#*SLpNFWOC`YQc75jE zwcEMP@p^7w#WZymLgyy3#+ZA-odnyd6!NL=C})i-(n4RU6J0BU(BwbZlzCSWWox!#Vpk3ceoL)!iX>aT>B;XeP*nWbHNK<)+tU^7{ z{4wFDTBwv0vW8q8A>!bfT2gDjDP+gglF4qk(S!w`jV z%`%~tHdOmPjtTQ5N#nXY86{l&-OWShw2{!-`ZV;lYziiz0N7^mlN$fLiQtB45ZsAr zq!~k!AnV+eFPUTBxf@gc_F!~{#8V7=o{Kd8*%P;@Z8CI{d(#lM?*?>6;RfuMMU{h? z(`*9tU3^0U?0ucd@#tEV(S~p~uT$nLzlgKOjkB#~^obOG1Ogxz%V7Xq%~wl%(e$=tQ&*O97>0 zh=pwD2`BPCwxDkZ>l00c1rhSn zgN(-uJUpdYE_ow#+z5^nT1$dNfAj@8wJ}`!K|H|-TH#~)ue|cHTCV6Dd}02td(G@R zSuN^l5)GGKw@_zZV97{#($dN;CDM&_n*ltyLgE{KXM-W<=x&^#sD>n4=?<+hJ&E2jjy07!@DoOF%Bt+ZG z*L(T|D0%+{);4N0#DinM7wGR0w?c*d;jxEW4Y*C1xqZT@FE(b(sRICpbFVQ$3^|WkkG? z=~Bsd3EJM5Yyhg&h{SMyQ6bt&8obXP3$TrFMn`2+p)iEKAZ&p3Ba1AZO%)2W zbzyVVp~T8ZYZtQ!;Z2Fx4oM&Ckkbs87Vy8@Wu46KsmK{=c|x|q(ef261S3! zm)^G(l2N8CPh@O^48Dt$8b^)Wn<`>jX;_<3;u>soMyTna?4}HXPPLqZm`@wsQQ!Q)e(u%-0hV@x2K+(HZbL-P8`jqGBuC$KW}NZ zF(V{es+TUf7VsX%f_a})NAJUQ^(ZrS;`LL5);;>EVh#kIGNGNPaeK*Ut1`gsSZ+bc z@{XJ$;Wztqw_>aGyQ?URz8!J11nR`Rd}7paQX@!mt{&`s%>pdOW;P(Fhp(A;bZ(gX z9nlw_q)^F9UK5%kr18ov0BO!nd9rR){n>9MA~PW057L~K?f^``NOE2z#r~Nb+kDlv z$i-LXNM`do5h29o1LtZPpenK>RYI}tvK|iceXf0&V)lZgeNP|4SrKDsMOOWZ&o1`i z5Yp$`Ybnm7?CTEUu|?*M{s0*Iz2OfzVj8MB=k|_Mu^Ju_{ossKO0IcvuWf_%1~L1k z6@a4}SUBJY4@4$vHLSPxiGCXa5M97Qz8aJ`N)Fwot;3=*Xv7i6499*s8&nQZpRce0 ztp*Dz5eamrSZ&|leSUh?T+dF%NaShAB4>uf&@H#frbXxt*CAd`!8j4-pDgB18j)j& zSZ%tgl{B;U+w;0kM_;yIKs9B(iufDdH%bB(4bXjz9%(BLJ+`9@>w7vWg1UrmRucE- zhw@Yd13~0)f+T+x_j3w`6x)x(H3c|~avgTmPx7Q^9^XiS^i`+|l6ToY^rsvu-z=Ax zwNzj3NuaW>?{H=jBGO=@&ZHzLchvv-d_`5cvH2l-XuDZ?yBxtDXK2$8)qn9oHJk?)?9kqgEfopcXR-VTspJfH=`x}1Orh_QDJ_+@Gr{;9;LLU|g zHba&rvB8K)UAwcsE|4SnWN%}i^e7Iu`gLN;b_SK9sjWD@SAmeoL$7>tFZ$=df-3wU zkrT$XkxW?wy1)u&!duHN^>V6H$@OWzjK@yAVr65hi7 zfTr{%vShUQq#z*iY~+29liXCP&% zzm6$Ng3lxjV)d(yZNtfnWpmFR@PzT@)Gu;&9sS5s7!^@zQ5GV6)L8a)OmA7n>gvgceK8VVdrOf-ub@hJK>sJl=qGhYUxQsbBalfV0HmWAxsm_)_rq9^&P2AI*a z%aHDQdb*)%Y_u%d9T>KlbO|I z_-t_1pwjj-vg1-i#OmQxVJtdcB~JGDhGjdub0;Y%rdK!;8xd@&Z$GEF8@(xcvwofR z!!ai=!(+V4)&au^EPKDIUz@Y%wKRU*iW-qRyWi}aOw(mX9Go|Hy#uJ()QK+$9iCFj z3LicWQOyo;y}hfYfJlrS6BEbgQwB+knhoo93l^q?=TKQuJw^3I+TE~*_v1eINHu*2 zrj!xSu;^`_fW{y{Kh40LEuUly*d$-9eSR2zmusJ!7zjA7^A!m6VpsMu4BY3czn&Ty z=A9iAaVJlklz(nh$AD%n^LC$92FZ!>EvcqRpm_}Eq#B_7M6nNnK8WG;jN`s5;+wjy zgI*s$DfYqB<`bZny&T#*_LG|-ep0;9aa{%&1ehozq%`JW5sQ(FdPIwTeuPc&n5(<} zlnhWNO#wh5o6=KK!oFq2jaa3V{W|0xx6N);N*pvK6KT?K?rRhRNU>_=ycU(_fnEE4 zn7A0%z)n#Nbx1l-$K7k%si)Sbs5K1{nh>QfB zswKxB$Jvu)E!WFcSp;UB_`mIeYc%%u_75+jZNJJ%C0w3)a=RGl&-|Y=PLYK}A&aHu zo4ySn;O19$2LT5M98sggn)*QT3ul~p_%+ORMS=kJp3K9`uvFkH!Bj|TZz!KU)`tLx zzxgDZhsav(cNVev!3F{n!qpI9vodHCBzVcL_`Q(?d1Q;g_NGfRswKZ|U$bi@Y_S$H zkU#D$_P_S+ju5S8lezGem4jx^*zQBvcnIIl#IamME+h0%LD=Mqk zFaxcZ{o0vJZ$1NQ`viIQ39^YgLjc&NfNh2a~V1Hy}RF8XWo@A(IWxo{sQE`0*6N_$Jzs98t_Z`_F;J_D#QURs|#oa;Nna7^! z3;NsCnu9DKoRVf~!e;4_QW0w-D-_Z=L+r>@qcsiW`{@%>oGZ~Y6@oa-&B+L&Bk!X$ z)5{YhzLknJS9r%puR-P&WvVR~R7lr6KuwNiu7r)4+v%m%8BQ*HA3Hb)(xPc$`i2f^ z0B_kga(xV~(zY4Jj`r0zo=?_kbH`<^`lMmoE>K+jY>^nLWVQ)YKOzf3p4|RgF8>~^ z(brg`j=b6{c+%sMXkV9)U6J`{H^1Q>kTFXh{xVBTkd}W#Tu&e$Mouq_jh_Pvf?-*R z3Bi6w?g=RRlE0F`Zta1)WzUw@DeB||0V*7%D5%)x*}wTE?L8hi-i3usKt+xE_L=|( zrM7(t+Pi(j^8l7UcmJ&fwT|r%5(m$8>RzpeC2KfQ6oCO;55da^cZk|Q5dr8HrsKV< z;3?yZ?e-h;xpK$4PChR%%&@b@?l`P14m zb@l`b6Ylo)j#1yt+f={q#d4V80!e-+(4#oQ(Jr3SR-n70DbamM*xPruh^)|^MoDBjy*C!pu35IWwKt@sn#ksx z{d(q9*C|!bmK+;9%=m?;_#Ac?UuT}|zA<8IgGLZm4V9qk^2Y|rTa=?x8*%OG?sNA| z4Q;xrVTnAhGm+FdnB+ZF{Q`F7l2v&`ys=RPV)PqlE31h-4&M6@^o?qSsdXn#b&NPWi5NRYZ}dn_5NzwluiS?$+%j^1c zlM5a%kncHUnAiRWr=+KQ|3DS{tj)wy3{lJ70ow)Eg8Cbm`dtbn#w_|S{vR} zO9uv%6y#!+%-?KVoa?;+q~JDxV2lB~s(-+Vzn#~vXSp85bvYq_dzwA12uu*a ze9A?GK`p;+CWhaAY2canlviADgWa^$aa;W{8x!a-7EXb>@go&d%O z@&d+iud$CAMhEc4@PCGvenMQ^6Cq0+*QI1r*DHt}Pgdh8a_KtqkEh&RQZ7bR*=)ME7+L8jLl#r7ZgH4RFzQoZ06 z-7}Ao8}Hqi6+jjj$t8L?pQV}*y9G)?{jAlhwi)d2rrm`RF{V&lT9uFm1B{RJ$7oFjy zMhgy}%5i5CXRX(=bbYYxEs9PQKy-+3AbkT#K7c8b0(JurD})^7R=4Rjl{g1XpkzOgz5^>)_TP%uvo?zhujWDq3F*wISJuGv?=FD;t1M%l7XjM{_Y-p(w3=M=bx%O}S!9X9 zIWEc3dU^X05aL`u#6Ad0km zF;$cv>yEC`$F>pl2SnOeH|XGjwes=?9bg;HewgqCj$ie*j+8IMC#?SEMNWW`XKD#z zNbq%MvrI_&^R9MqgrU@v!rn}Z&+$}q43ogOWGw83qU07QZqg_Gy@dF$sp_bNL;Nhv zjlb&XQp|TeAVjz6pWK1GWpdo8ooY3z>+7ISzi+1*(6jjn+>IjpA}o$m6<^aE!-QYN zS9%x)F(Z`|Pni6T1^tg4EKh>$;6@2H!=EC6Kzyfs%Yt%-QSy5)p(^FTAfZ~#o>E3U;Qi0~K4er-J2WjL8Ta;Kce za*+NTv3~iN>k1*P<+6!ApNKRVS@`hrF(Q=LC@zyBV75QI*dpP}7u-1V8e)1Wm-O6g z{sM~S5!E2&n1$d7VV_>7mW=5ZhT{yU&|GG$#kZt_>O!k&nUafR(*0Y=z_)Z7gkp?` zLWQUt8DKxOpuEbLmJ`agUP(&VmQDctAn|GQodw zh`fZJ*F)CHEj(k>#19^UB|) zU^c(kb|G+aVsks3r}eDV^Bk14c&aSr>7J?X;D zFRTKRr`R-m)2{02vJF$_E8JZ5u1Bi@<^A?Su{mo6CpnWG8Igp(IC|@D^B3Q)Z6%?m z#H`~yq8eL2n?-%?7_s4b5O98QhQygeLu_cti_kY1YiS;9+{i7rK&>Pg2L?kV_i1VB6yidx48s9# zW&%O5sh~bx@}r(6>9)+Fgi|@=Nc(%I_Kt$X;Cw# z)zU334JS5wV_?DVX`)`amr4u%O>Mt8eC??KMmv1zpuE~ulhv;%?!b-ia+3W|asqoI zwB|gY@9R!TG6!<8VLJZyWW_T%pnN#5i;`nmc9GurP2vAX5(n5=UNo?M zk(PZUg=$_lB#Pp1Kh|E6hv*cNX!_m&mV69gNo=r;{7ZNMF1x!7dW3(k8ktwC#!=8! z)MEfng%_EC@Z!G>SO3W5AlAt({aCDAteJIHF9Q}p&c_+sSEdFhM;Y6$T3UcAaVb88 zY*zPv+WN1IX8%Y~fyl!AIK$8*={g3RgFup4PbdnZ5K5Cjlv8eK&2>HYZ*R_WEt|g+ zQ}6_p2Ek>f0%dsTWf)nQ`7tDU2wj+p7WUb0p)AX zTdz4Y!MFo`XDF&L9Sd@$m$p-22|SdChjM_Lc15)NZL#XBIz}iPtTXM8GNmG6IpMUb zSonJ0Pt&e2ZL&#Q>!TRu_J9nujBa@-!TrRdTTF0{s%t%m9Ab+Fp}1L&>eq+e5WzgT z4MO6LN`j}k`^Ahx!-*md%E^Lrw?}_IP+;dJd`jmoDw^0%t?I^C~DdojGJeNyGhd zDITSZe!67nWu7XEI)gV`8p0I9Rzo$;C@;{Rn_YK9Eeh$57UVX6@uUfG08Fh335}BK z*4-oZx+sY*zK-K9?;+x@rK?VQNBb6^?Kmu^*;%HtB7?_9Ts zKWE(rw_;gPflDkUx80sc8Gm$+?)$t577|bH62BI0@BOVh@nGeK@mY4E&A}VB49bp? zKnm6NfPHgM(>(l3ovdKT8WAwiROf9Vxi|Xja2wS}-C2y{QFOaV%cn;DN!M4^r24o! zvqBBlp2jPAdS)BXZaPf5zaMQ1-o6Hncp94`{9ww@ePGaJLG|uCmE`k!XztCBhJgl6 znj&kHp$5%$jT8t+NGLw>LqqOqO zZF66>3xQno9>@VkeXy^y-iVL7*A->8$JPB)d>*(a{B^I6{v%6_$L`qVkF)lJSMp2? z!BsCgw4)WQquao+S38+1`aQT{hT=_ke2mg}&o`&Tnj@N2+)_6Ta>e@TL#nR#-E+`GeTj5;3c3yLYk5Lr#;*&Cg&k#o%@JBrQNa8F>Rv<#3XQv|gk0 zhS{zXqH)A`rF_Pknr7f8(<(s!U}?@MRg4d7ANZ(#zKc8w?rZtwf40AUve$dQA6V1| zU7{*jWl1%Jh8BP`_$7wWdDmN|~Y~{fSVT7_8 zv>KQa@Cu}@SFr1_>$2Rm+?Y)WO|+mQ1m?R@);MJ7`)}LbkjzvPom&Q`8^9lf?P3Ai z_#c=$U&db~SB}nrxC}1#09SilRDk`??O$YTbQctqi(t}+a5Rqgi|=t3|Kq!wAUjLo zHbx3=VJqbOJEgr}Vdb=+XVh89f$$gz z@g9Tl7|wqC;~f0=xwsyfEiECJZt{n z^h>;zko><5wyUxAkHLmQwY~_((?OPNZnlw@1LIOHujWe@xus1h@Am(OOLD;}xnv?F zTo308f^!kBs0nZ(5nSX72qeJ!1MdFur?N=VbBiD&)`oK4<+I3%2)JE^Y94M_t~&oK z+m4nrh#Gu;@qpbC4+eG@QY*?%7>uy{nXHDi2D(JBIO`6);(B+qEVnc_dQ(s%%MQ{` z2&lL&8*wgj`1=3*Zt>0hA9u_DX1e?yO!Hvdkm-sI-jXK+E&N1b1!f7HD~b(BBEUcg z`wuVozmHuc|2n!ZD*knJT@1Iss4w+|dAB?mbv&%%=vRgl_UyWS1(_r%__DAIdm&@i z7F!i`$HNI{cHM!3Owts5+1RWw2&Ih$K_%Vwkpxux?odG{Sqi=!>_QmC$i{-Nl8#~| z0mr^OLXb(Gg6|bJYab-T#^P=z9s5Ya4g2nBK_*2pf?TR6eQ_N&*5;KAyd#OEYsxVm z|EJYULqxL#2;?gLag4UyMCAJLQt4?W31%z@ykt-rxgM^V0xuI8*OFx!RNHQELTZ`gKI{lFZPJ884!39;Zt=CkC{c(P_*3Fd+4@MHX*Sb?Z{!goy z*8=1bZ06L(Tnd^rZ5TB`NmcjFghk*fyo?)zkB?9AQfa+X4>NWSUNU%&u!k#d!OKLk zwOCmOZ+Q77b6r^h>pXXeCOPmE8WCcDM*@X?cdQ_zQrG69)h~JqtiwJC$};KM7)3X# z9K4~T5UG7szLo@Lc!%qbB(U3eCs(WFBy!nzrwY!gj^@FX(*@_$D89WU0b`?_mxRo_ zvzkSKAPoD{hd4DuC0To{z_bmY&uXtu8KHbKDRm{v1@iTeu>u{`1zfz-2Yd3gmHG=A z>IM=ues(j*PWJ+63p|4E*98A|pL^b+M4&m?B~`ZZVF2EuNaK`(u@et)#O1}7e>ad$|*g~0UyE_StY(4hIClE+LAJ@ z8zZ);3u@C{8&Ez)J$=TnoRdhg)Q#;nhbrodOjQ>gHzsZEU}}BijuQA!5OeDo>(KwQ ziOLBw;6m}sHYUDJZKwHo_6Ex`Z)tv0sZ6RShC|{75&07zh`fYc{!6eNSW3v1zocXD zg)av$>3Dk+(ygPxa?qQQVciX_03Sz|wKB|=aY3B9eTsxM=vLMIa3z?|70(MUr+#3hF!lHtgHf0cPT&l#92z5Gp^9F_9iAR zcE`9{b&BxKQl>k{eG#^{%3$D!CX)SCossUW?6VE}es@9H!IiPbI~S(rFnR`>!v-ST z>3H)%o&Rgm6a1I`3qC-#4*n87kF+5Omj^*8NC7rEmVe5aFJ;aDjAh|WYj|+`hXupm z!R=p2Dd85y_ix)ptFb>nZBcx4jk+{kfx}TtZ%-7rGcGI_M6W1Um>Uu%TfiyrZx$}x zg8_b<&L1GhC4l2;7;Jbveoy*8ro(SkBh_mG+XZ;yimB&MyM+r(YWv#Ng)+D4u5OqC z)+StVaxwWOPgS}Y1oy?7&hS|MFP!%`dkkLTN#~gLEX!teQQrGFg9C?{nXSZ-8aUg% zDgtN+p!5I3L*^g%$iSJ^+E@3;{EcA4uUjFJTvXc@|IZF&BLA}kS@;z=1`hAje&_wHNWcy#` z78hKp+p(4EvW-+=PPw__nlAxNC;_GntvyH$<^mS~3aI$6(u&ElSBU8Etl}*^9dO(L zVhjq9W`wg!ZvX2IMv4~|z`010*_8tTg2-!0QZ3GfSy{@gUglLq2V9Xs3CQavz`r% zzJHPa9Kgrnr7_ZP;?LE^OQ5U%5FSsvI+X|i|1^!z`0hP7ugKuYF2H-BbU#VYvu zQ#8rp{{F(lt=PLQKEW3Kl6!^``EoZ{brc^Ygak%%p(CjE%JshTx2Y(^1 z90Z*@`?ZQnno_mvb~)X`C&w#QZQ-1?=-@SR$I;{6D*qIWi24B1muj~2kTl?w6x9@h zpJov@O=@-`3?Yr9gZ%y6Jlq8+jX(IAPNDmPMFajLaT|U1he8DhIaUlzlA&&R; ze0l-NBN$WkNEd^hrZ~i-GCJ4T+%?ZPH=AQ_O%sZk-mhDAZyROh?Rl1Im`>aC z0ufU?Vly{BsvG#S{!SI}E?{r_(eklh3OYTU z_m=V1r`MVIknUT_`LSwd#5T>nNbRv6XO9h%jg=}@pK#yZYGTK|-tSZ2&$PLxR}S0{ zeFESeAQko^^T*l$&TV%2A+-jc@07S4MP#07 zsh>N2`_QvY<8h=Ua@v09oUI^mh>T`F23KgdDC3uzwOYS6aR;v zbQ+HZC6Vp+uP1EQ0Ym9D^Ivd?j&8+>Cz{S1qN?ZmbGIzD0M@|*ZJ^C~rgGwUN zV*`u1l*ZMIt`D==&+(d9r~V!&4j!vUA9Ld8vGAN03nUuH-VI0G%6+ALUrK0n@h$i$ zBb+K0?1YT%9ZuR~ulGp_eO`PUP&|@a8t4#*dp7pWdoJHESzB^7Dc+?g;>DJ(`eCW| z0O{@0(TrqAiU4u<_cVi&arJElBa8Wg#XaC>F-8}y&W&Qh&sv1X2M3DZDqeM+r)m2N z?b4oo>tB47>a^)n@lC4dQ26J_hn_>PM@GrLx~Bxyy&2<-Ey)8{4cEXVcg^2x9hHcZqICSHe<9)B$NkwbjMS`>v0A6 zNo>j8>3OTKQt$jJZ(*z~TFvTWjm?v*2ugnWBB3ES*rQ~-S-4lXoK*y7!h^QGx?w53 z*-)u{%Eba~sh9Qskb$+?H=~})L3Ro2&nm3Yjb|Cx!m7 zVQu^*vtDLu_vD02Kz-02)w>!E8kl70?%pzzbYzB8UKj&|pK91qKX!S0jeM4jq)MI; zV;qCy$>g~}26ly~0XDdAf<1@|5kD>nT}eKYftGQx1Q(Gcpww*^-NLEELD*b~Qi~z( zO`>q$l;jhX``^rjpD<)};H$edKmJh#*{^WrT3(*-jSsEs9?VL1nJAG@uH7K{2!dq*Id6z>CL<9 ze)v7<&*$?4V=DuP`jY*NPvpE7NLtj-SqcJcmmkc7gWQqhLyuUj6JH$pCFOqR|3EuG1*X4sfXwRDUqjeJC$1Qpgp?He;;il;W07BSxsdX_ zbe3lRAcEqYtxIb4KuP3my7NQNFI=(t+>EX1f1gp3J%?^FyQ92=8eNg>$3}+{n_n1$ zGUMfnX(Eh1;M4_Bx||Bn9XWUJt|Y8<(f#=U^Mw_*lt~S0hHcY^yEij^AJRS>5#2Tw zdn%B4$G$sXFi-oOl;ABk*>_0HBe~nV@pOeF3A*;(Zw2#o!3#y$>E9t=9)Uyor|*!} zM{*uj;82e8SPoJ}*FJIsfM`{8pGOiN+Ji%R-$;V3eRm}sqqAF6uJ)KS5OaASNcm|n z&hoKRaTUXlqpttS3;YJ$j5BG|Hf%+apeL*$usT( z-g)d%Yw@9%@j`Z4h;;v0Z35JR)4mP7_F_fkkHzqa>9NiFMT>#P-8*nIX`2jJ@SJUt zpSI&4n@R8R?TgCGL7&gws(#3_$WP4(R>bt;Fc-<$)_Y(b#X@)1mDJl+9g-9}UVP-Y z>015tqVi%xcxkIt`VEY+7&H}OFK;H|oQz(t(%ljlNo3i7uD+@8Nn37w#o2j z{u=Q7U|o(u8$J2=cNkcw|6tw^HSDj|H)2c@r_P_!m?PdY4JCr5@E;>cOxc z)|AG2$_X_J7@oM!eykV&H9PRUtWhfE)Z)oo?%dtes`Urwq|JM&QEJBxvL}uoT+SPN z#D>nA-UkZp2=($zOjQTkZD6sy9UpIdv3Idc?3K)(y07<4+b*1WH%pb& zxOin$1sZRZ?5&R6Ffzb7&hr0!M)1hL?)rm+_3>jD>lb3Uo$b8MVu#H&ev@o3STg1s zN;Z58R>f{bwplcfo;24~&)7enklnpeU-RDY`kLXJNuL0+HN%E$9M4-6UuHIJ$g|%U z2vrapI@x#Ik}b~UXqeDxJvtlDN_kvSwrwL}wJ0trDnVN5-xabn?s?Gku=Ti^_?yQD zSH}j`xrR`@BhCDD-h@=iicizPM*rDzg-NP$;Aj^~Ga+Mx3uQx%;~Hes2Hmml=CPF3 z)cXm~T^q{y&R^rg9|E5@-1RDm!@xI7OlZIwXH?f-R5^fxZK zbV{eB3J3^DE=Y?Y3kKaO(%mf5N`rJttAKO~EZx1*4brjX63eo`#pijx@B6*pe}C7U z>vNwu6U59t=gizQXH;o@X2Xo?0#s+b@z0ZUZDsFY^L<^eB5tvJm#krl|1O!>P%a5w zFyw2B{kA1fUI^REhk162Uo;f(?_lusSo3dv)OVGlYpG}Rd6*Z@*0JP!_dxTG zif3gJ%%8WvMzm6m%aa4|qrso&POPCo`afyt4C5GR+5X3B!lk>+|CRW+zH6S0C zq!N@PXTL94$ENhqEaVRC6r;)_$UoD5jrcE3j@+E*POQ;|_djXq4ElHCzeT!CIPMF& zNF`t;l>JYk|GUDWvIx4*sb3=+sRZT8h3=!lmB-9MaY&)5PfhFzTM{_hIy6-za`2oy`pBB(y=e7ysGa$6p=P&TcHW{TDn z|Bvjy{2E;!X2rWvd`qUWt;iP2|0iF!fXMwymx6TpNyZ;N?~lZTCx!H)S9i_DzYLGB zC3bdm_e915oJnYN7zL#TE&N_mY?U1Mo32x?_Y~tglQ8Bmic1T+`RP$?Z6EiqqQ$-3 zzfb?%JLxC?L)mgIlpv-Y4&DoRlcwnB#F}#B}eDqs|P_xWB z;_ZwK?KWyUY#*$B`Kx1@>Q<8H%jqZTL=_lngkKW(P_@W-$tkncBS!;_B$(uv>7{J0 z^wsqN6t~Ma#`?&pt=GTV6TKos7*7lvITp~Oj)i;>u^(r0`hgxb2d}uNO+aUND^6oqBj7actA- znNdCLegO7!a!Tv0^gCjpNBS=2Tq=lzmPB(RGGa_k)Lz9E0zIxJo>zN_kJA7Z%?VM zEYQHo+LJcq?agV=W>%!<@OAR?PN;ss^D%CR`(%;>-xIB$^|)PaYRNT!m9F2%c`fDg zVeIi2z2DDdLaTNz*lNFgntQE51!>tST!_W&H|EpvB^uW{)6 z$^UNA?D%Kf@IUv>-??A*W5@q_m7a|ZJL)f0X#+k^ndv5ss$bt)r(pi#EH{s(Vq9F# z;ABK<6c@gN(PIMVj*C5`;O((81Bf?phuC2u zmXCaU#^d+;lbC3Dk6SHM>!e4C*1V>!y|PKI{a1T{RxOkKxuopkaiaShSQ`Ql6)T2_ z_!--99dS77AHWtMy|&5j1MVfMW#alB5cz(n^%MBI=#bOxlv>n&xyBy~e>E?9Fglae z!^NE8VK-O#u@}vFg$$X$iG2kWJznT?r5t@FC3;MhwpEm!;p4E@_-VQrS$=4rnKS*Q z>{ZpX{%m#!LHEHBht+->pK)uF%@(4PSA9sOii|_S=zc7?Lu!(krUloR^u> z-~i};VOg<$OdH$VfRg`s%hICL!t4<6%-53eb-Bwt3bUI0$xZlUp0irIv-eAH933Cs zj*e?xg}VWY`8Xz|dBDd~M z`vEk#)BY0-`)>520jJAsvfVC=D6nNMl8@>R6GZ)>+Glwu_H*T0JNpO~O`vr<7<-BG!a?x^A583Z)Lesi@nQOi>lGvk zYODJ)oII@_S38o;abwY_EdQq5U^nc?e#u*vF&+on2+bw(+LSd|U?NI}RP58wlqk_z z{jItzvMTud#{u5!uh!r&(Ipi4jXlcMu~(+_2+?*#6zoT;deifK6O0mizzre1WHLoI z&ks69)fxZR@95x^BX__bJo_FYvaf&LJI-k^(*5BHwQFPYpa3CNW!#{<0q;jbvdD%U z5d+-8l87<-7h9Nayhs94Sx%V^89L>lhgp!$7q=|}6bdUhOo}J`3h@FY3V)rqutgEJ zbAx+h16P)#SOmpbf-eLcSn^SVVywY{r%FrX0J_XLswQ1lC@CG*+{!1S^1kgzqLIFp zXrjZu?by^(C5Pn2X$jvebvvw0W_UJ-%1B3?#Yma2Tc04|)f{hv~ zuBP1Gw6K?-C-6gW<4%NG+VKPZMM(!oCv-aG!ejQr8spij2;K}fI)qHKu7Is0Jq5?R z<-^Ae)!YQfInM&es5{*F;1xP z$IU{XzV1g)6l@L8xwRNzyEacQdd+-Grw+mFkJ(bTGm=Hk{iU_HM4frx_q_ZADmB?$ zvnZA9rGRn8gsuQ*MiiHGbl74hw`C%+`>D=RlwVoNiH~SvaXvK!>h#6FyUo#Qh&2g~ z;JSp+QMq6ec3eJHU*Voc3Q;_ez6humkR5y&S|Q70gSx^DVbY_`3+2>H%?o{`HuvLlPFn}HSiLx@s;)MFChz|I;(u8M{71~%kHSf{-)FV)1Rxx5L9cC=!N$^ zp?yb(Qz(Q9MRFq6j|Bv|}$teqJ?_&kZ4|M=Oab&&Ayp?mq(6hB? z^arwWl?awELyTS+_d;+`Q>-NyY`4hzr60%=uZe(qct)-9cO=(=_Iro~B5Rg^P!i_8 zbe6(lbb*8$*H^amj6znP%>@L{saA`o9Pq1vRjyShj42eiTPR=pKDJQ`T$(xd=Ibha#WRK z;4$F04}s6a0jnfyi_1S74pQdfKkN`KwaDbT08V>IpzkLRYGpHB4r+x>W}R;O7o`DR zP0Hn7U3$u;E?w6>3s~QECL+3{bb9)5qI8-f6eD%Y;|gzD&U=^RX;D%X=Atk6P!B2C zGY-ndQ5WY|^&QfEd-N+Pvxi~zjx$pXX!nsLGb|74&paRYJh^51e2jiBf29S7o0Z(8 z;B|X-NQUIMD#~B=O6*nyD0&Cw2Il6y?tFug$4-Di-!`;WFL^&CJ06FfQ9hAY2rfPN zcG)b?+#JgO%GiGpmiFhuj5OBGv>`C~%3CxX!Wzs(Ohoi^U3!;@|HN{4(f%rVmIJaZ z)>mCsvWvC@TZifiz6gFj^CbO)IK!b%_D@1zFW_Lwi+PV^NEp!acW zZ*)pZ@loE%DyjYI)r?#>^nI0-p0Ap7_!=f?luuN0ydaUQama~{P-KL6a3{Moz1 zjy<0_NpUck8h+1E(X!T%#)HN~euJy+>wtXhd*V4V2KiWq6+9`@uWx~~8X?a<*Q~Df zoi||03-%kH5z%A^Thn~ae|j{wpg4n*%+s`1!rK^G5m~{ML=4C2-X>|oN{ zl38N#9WUErX2=%ze)366;2+d6bjj5Nb!u&Z1Xp(${aWOSgULX!OK&KCo}7ZPv~Fq9 z&(Z&%pDZ|~H=SNdIp7B-8!SCPZf*!Mcu)dH8(YSBI#=j8)zSnJ@QS8rfhy%VHU|oG zed@9i`)<=%aTm2pK=_P%Y9TRuW13@zy<&15FF%hHD$`N0vVdX ze;QT4CBvV7%^0HKt`mVr72C%T}UlM!=VG z5cw~+B%0T*Je5rEF2WycZSl*X{71%@+tpF8E1wraUn2eJd z$|rPr8NP1g4^y%BmOHNOGu(@vZ$v(t_%*XOPl%H6zkH(}2cI{#sOgZ@IHxqI=4OQZ z>1H~_3SHg6+KQ`x7p<`*ETH~OEn{ea0nd>+q=DDnMJN*V@nVPp*>9O;30Crpdc`jR zaN*4-b!mvI);A)J@_R$(3r15ybu0e;`csmek_eG5FRR|1%o^6gkT65oMT&iv4@m5m z$-$ysj!}T=oj&{?Ec+WXu#E(_ZpCc0_IK%ZSFy5{@x>pY4`!rz&cnme+?9*R7CPxL{dhFqXY}ur22|2K7;BC$E%Ql!DvL^#ym@|K@ zsw-Zox(rbEsj>vQ(LtumL=&#E#s=4<{rUBGnc^(pBvEeHS`$|{ban(~iIzXo-#cTY z4qkiv`gB{vj^$+z@g~TPyN-LbszE+BC@r2zbw6Lfx+PoRPiD*Z=xO%0sZnEIa3%1{ z^K&a8+VMCVXJFIEnxy(BKzRE1ILxkgg;U8zvwrCW?O*w|GR#Xj)tj__h}zd$0iA`t z#;&SZ31so}v0hy&f)8?v_Rn;M-1yM`YQG@uo#R#3Cs_ON(8!K`y4duD8C-A>Htk7j zh5kE#Bh1}iQzIT(UuCUrw-5dbe_1h+UtJ{@^JEYfHG-!V0EWL1`cU6s&7`!u?b)hy zbn@$xeYj+4ZB#N5nDBF%F7ic0y{XciW@C4bC;CFgMC4(ba>W~=o(?vq-OH~R>k^l` zv+W0t@o8ay>Mu{rUksUSA!(&Y9!f|=4pps3@7@M1U7u~AEEPpEkFxO}KHT?g)e<=U zRmwBZ$fIuS^jJ6zzY|woPOOeG6qS)0n#HqcSYCXM*P*FZTmIo=g6-&p+^=dyvp4GXB}` zkKW*acMM~m_P=8b&$a*A|8(!);|o~-Zj53YW9mafCm_xEOr>rW6Yc#V0MEbN=sg~+ zQlg5d=$-Ard%!>`^u8qY^P|+q8zBtzA(?;n{r_PgL7&S&AH&G~_jpO*-}4#czx);m zndamx*AX{Md6C|a`RBqTj5fb!cysYTY#r~P?d|t(W?Wo|9Uq_V9ru@FGF*%upQX#n z8lO)C-!HA(MYmeWHoYlsk?= z56qp6W!uIs3y@$s(Lg7&3roo5bJtv>OMe}wqi^|Ii%uqsCK=x*FY~`ITm0Vbv(nG^ z$b38rrF?c_A2~#WTt2_BgnYheO{}EwwS2z3u#WNjhttPa)=6XX8iO1@$iwY%Q$#v#@%X6__TLgQY7rI2{BXR3SWK2C|Ui2N%oc#Gv%fMEFsG ztQ6hNalyG1J8I-6gAJxQ96wlqaFkwAh}EIcqwtwJb=eohm%^7K2jWBN`G_5R`|SpS zYz0}mK&~vRy>F?c-@gUdJh$D#un9?4vOH+M&wY8VD zFDd7el0%=$5^xOx=jAEBYJthFeWDOx7Gr+SOnQWU<|T*UK@r0F@a6MlR$f`1gbHi` z=eF(G^z%|#`(sWv3YM28icN z1~;h$An~$K_!u-#4Z#a3n(h{pf%F(6SRZg#d1q5)#e$t;z@Q1bSfh3*K?oHd51E!z z1Ew|3$mq*?^b{alIF~!C_sxl;-tWYZC7_9L-x$2K%M8*A4ymHlDQg?|WUKn{vtxOB z+nBGs!?wLLhmfl_VPRF(`Sq>K2jsinfCK?z-ey-7n8Ac(O6so|mJF4aD-~0DJco2w4hNjD( z(^rNdFnZf2I(YepChB+(`X=IdiTh(Lb<)EM?RC+1h1FCaUBT-^gvo(pTDq2>iB^6bnb zJ9)<@Z8qcgKJWvaS;RLsy+wI=DPKup_%?&NGj>q*&H4K2#9LTK4)vZ*Wp)6sZ(@4O z3v>=Em}d1w$*k}1o86LG?`F~$Z_b6^f!>_^h>5j6Vu?%MIoM`>b&5|ogSx)0-v@U- z)w{W68P2$AhuT+B#gc@u5E2oRsSeuO1Rir2jq~CG@quZCa-{HPF6(3NdqC#IYUg7` z%Em$VgcM~9{@57@u)_68#LWg+d4$4?^XTPDQXt~nM{TU1xJIwaa7~Fj=_R?W`ouyJUA@G~(?B1iTtAKUFNLoZjVG%jKg=rD?&rb7u2g15 zcv2Y)KIux5z98|DE@2^=*w1tPFe_Snb7C_qQoEn7=kbc~R}ppM!SNKgEjx?Y$;Wzq z33LyE8i09;rzXtu9Y0)cK&xy&+~xkP=b0bxXYO34Qn@%Re9X`_!T$rksOXEQ z*=P}v8M^)-R9~!mp~ny)vrv!|wyAnxfh4WwhBbCi)OVrpBWNMI`X=GrSCOCn_UxDytOPGX`K!DLkTU~pZT(Z73<8jC$VXX zoH3C;*3nDXJJSLdFZ;mtfHaCe9 z>ZYd^e$>@VWumOE5maTbu6e2&StZP=s$V5cD@pAN^hl3EyAPv}Ej-_b_-WAhg3gG6 zGTJ_Hav&liJJA&QD#=yNIJE-3Tu0i6>F z>qM9l)Ojq-SdSH#S=e&pMITk%e(C(WSCt9n#`tzFdW+&Bws_TxJ|<;-1FgHBXXE}s z1N?OTj=jGOM8%Ly40`KjPp_L$iUYHXIeql_iFgf6Payd+r@7;Kqb2@za>AV@=ks<)vk;)5W*>dD z#eQ=Y0ZSlv@DI^vfo%ABs7u}adFWWMhHm0DylXmIwDI=)CQZ{_DT|7hM?mYX;m1l$ zGaT8{_^6{3_Ldv;?Y!~o0R@RHJAafVuAHS+lzgt@jRO8~t5|Kt40$er(WVXVl}Z9# z8`>}uc*T}^^MYGu;#;iSE(}WwA1d9^80&V(o^{<z~NvJ?Njv?m)!4U9)H1 zirMnA%Lp#HNIxu78(g=(`lIHSWu+Dg1%HbaZo2HCy$ThqVxU9g2>sw?3e)styA)#q zr@o#G??BXkxDs5R9q~0DIo1bCa;CPqi$dNhssD)HPr1WPEe7$jXCpOpeI&LP$|5d4 z_7*ZCC;CoFy8{61^X*Gy;jN34IW<{%ti>GQ8bub!IMfT}g+Wi3+e&l6hv1f(WkLTB5mXmWk#)$9iK{JQxIj$p^epr@cN6^%Ln)q7I&({0?0Q7HA9<*GT~MpLP9dZRvw~4zcSKt zWQBYa(~KogOLvEEKBvw44($W2;1oiV!D-9XLXx5T_of`lC&-~0{wM~lc&g#6E|z&P z+mVOj&x$;4o#S_vc?>$MC~ZIG(rsKVyUS}M+#o;*KnXvJf|hfZ;5uT)aGa?$HAruq zp>&&A>kOC2Czz_GeGV}K8$@@9e|o}yeZnuibelwLNTDqCliv8F((U_NZUj6Mp|!DU zGbzWJ%&|Miss9IZm`8X16_Ur&9jB7V_KRia7zhsc@5B$&#P(k}cbQvBSrxfhNm=)} zT4@ZGH0AAn_T^!2dfh_>GSob8{Rx`uPNY6QyU9>rRaw-dPhf(I*O`l7zhWlNkZ@eeG+U{Xl%SQJ*_)syfdRM0Z(EP;E^%Wy1w{_nYs$p-PvA7 zB_!5VQEr{{?>$$&uukK!Y-JHCb`|r8gTJ?lOUAXai66%(-M^>hagRqb&|M-hiK%UX z#KaO%QH7;5`)>*iYa-zw^(65iyNHnlyy4mRQU-D&X^|z8ltdD+K|k;k0C_6rRR}iT zS=PQjcTYaombgsHA?1-_(UQ1HLYJ`a8W@eBnrH1IMbAtvL2uw^5dmrve7~V`<~cJ+ z`GNGJU7PRO+2ecYo9;f^RHFYaRjRr+gp!gd`xWF*7fEC7FJlzK30o#CRyt5T}Wi>G?u}%LP4}q#<(-yVp zdE{sM`yvlNX#CE7|8a(*636U+4Hdxw)}NYF+Mmbvo_VQ(?0e09q>4u0j^KZd*oyXA zis-cz0OR^vy;Xk_cvz;hG>8_VQmD<~7)|CMHGX4)qcx*5Ajf&P z&nvw>=zmASFW31y-BVuqr3%~!DD40H)*b?Z^(Zbgy~e^cefoyNQ2q4FB^UOkPPAo! zJUoXl`)TMw3a|9%z#%VS=oi=zz-t_|NO(6<^;o(T#u}vQIC)p$N5gB3JDAdbVqf9M zRGI_$^TXuoi4Hxa@LWF~l9!nnY<`_w_w!rRAC6&T6emE4#mNXGqt}tsw3R5?F?eIo zS=2JENjbxcdxq&jKiff( z#dTiq2|p)4#tUj;li|C57%BZ2UgC$sHYIiN8M_Fap)`qGF-A)A_OE|Uixvp_mKokB zyNR|OKa7+#3RxB@PTvP`47aR8Zz)4~bo6B*uN8UbFNIy^`Vz}-EQDlcM?lZc57_HPD_iD1;JV zKv!Yj7K0%(C;G}*K*8%ZL3%k^h)N4iD6t)z##bFWGvE(`pd;tDJ`8SO0Kt91GMO+u zS6@Pcx?Gbxs9@v56!2?1#ux+`Fjq9zXIEer?FBxUc}zyUgduWv=^>Plbty`MjoOl6xW<%oGU#x|@8u*)5isa* zn#w=J30Y|fS*M5@^ohG(VY-*>WKb6jBMVt4Az^*9Ep+~^>|rg{qqhN%-`-9agNW^e z(U-v4Ix6mzJ#3*Wtj{~sssSH$6GMqhRSGY1&zQd~1y7`ZdCImc*STElO+#AQ(CNZF z&P0u7me&FzC3z1kn*1%>Cb1v8{t25T{sAypY;E#4|D295ENVAYJ}+AD=&-yA1KPMM zFEY7bK)uHfZ<`Q%&FUA^i^iVdMGM?@rXqP-=2eP@qP4It!KYf@YENihKOTDujIntN zwJ7RWOH2z6D-wEToi_9|&GPA>MRGrHV!SHc%>wn8AY%8YnpcMzHk)ns{e@;p6OjU% zlL-d`vm}Xs;gsDUBOic`6%>AI>hBod?~&S-x=R?AOlBdDz4t(eto<`ya!BXrnnPMM z@jbGj3h%ZpjC@~7wV=1>IPxJoSmmS*(!08|WTn;=x)|*gcvH_S@Pgivm16#_OTB-M zpO!6>)#Na+46BvoQsSBpLyAejy=`(lmNSmLdHv;%6CV;W*C%GEQCVEoSIA@^0I%Bq`_sESdD13G|!g@*Mfd(=LH@ zWK5AftH@^b@)p?=$V=MvDrAE)8A6eb=1CViPw_`lq<&4_U#j0!D4p2Rc`G@2L|5Bq z`Ui_7l*aQ~Z+&F74at%`*eT}0)@ARfSD8|(dX`YT+Q#LNfia~pRUNztkpx&zY=27< z#W3E{kovax>;1)(x%#9ySt><1H~eUxCemLbXdc{T(@?f#UrC@_DtEZ-FPO&jiW?oc z{tEzNbgVd4O)$;uX}87a{x3byGYzFOXo(|AQQi5Gwp7WIEiO2$pX3;n=e#!5rY$j^6a-m`R|*1?FpUUixL0xloiRt+@pNZS2=t9i2YKwgL-~H@Vo-ZD%e_rV z&0%;f+9oi`%;S?g7kLc64%bTBvS@gxB}Rt#*5iGH(5xT1`i^Qbsr8Euzs#|j1jNXeKv&?{}v zO_|j~iz$;wTui8fDm*&o@OidOEn#Y*S)crf6Q%y8Cb?W@%s`8}(dxbHxeoPl>q%>< z_4F21YGEhMGEJ+XU$K=hcs-KTvL*5@jBWDQi}NS1TO5DWd|LYq<86HtuLy0VDstsf z+tQk$(QAm^rJf}I;$trX?kHVi%ak@XUZ`j?b&ZW?W*9y2O^5BY79+9`rJH_jqmQW4 zB0v}8i%t!zPMxb^oy7PrQsl2nwHm*Be%{UMxVrC=CXgsgn}u#mu|=n*RkzNautDM! zgt4#zx^c}KyVvd^HxHIxw2J<`Di6o^f_iQZ@PJ(b#H8koBN3g7;mRb5ZJZds*f-P%R#g4YBFON}rBZJ3-bP3MeY{RR3`br4@9$M!RyldG z$Q|jny~ilVDWyKrYqKlo`Nr=xh1>3-MTyOT4y&q&d@yic<{80-5N4|Q%4T3zi|;#+ zp96>QG=Dwb)#6mp;$fC^@@!km2_Xc~u?mc^Gp0o2f2=RZ!MKz(IZ6fSlYG4LbtNke zGZCL*F*F|Ql&HX2WmqtK#bL{)OQ~H&olqNBiUZS2GW?Ne^zPY@0W0?jsck~M3iR6< zUm7ii=Tjj>`VlLq^pnLuMf%gxXD3blxnrR;!XaCf&tLsB7F0819{F{g;>TJeN~Mh# zSKyJ8TX3$-msmob z+FfY!Z}KPUd5^gR?>>l}rxE2ngf#d&BYvblcSfwiCl7wiYhM_@DoT$Uoyh!TDnZ%l z)_t`TLIma9Dn5OxMw69OV4bQyDv*_JDgo?t^9kEIBce>qg9`13EfPz@+W4CE-IgU5 z)}3O5!~X2EeH(B>tcG2#sf%Vhe4O#AY%A7RyAT7w+V`s#Tp9-z2RD)j`BE}?y*300 z4LmyeL_5zNre>Py>)LuYC&#SmGjuv}XJ&@F0*-Vt!_HE}`EBr2yH~$0!~cX4Vm*`U zcl{Yij-^hdBWThQh{Oyd6nkP)#bZ((IE*PxNXxfU6F7-!M3_2kBY%!FIx}(-%*sep zVbJD^T1aF#JhPe{tD*NzI*~-XRM`s9k_L7*)sa7YRu;DD{{3c`5U;HfV=v&Q&*oD4 zi6pz%4`b_m36vnFj`)hHt%0&L1ta?AR+B8;vt|+_3{9u<`pm>^Q!vI4uF~|14;|m| zi#KJZ#ZFI{C&skt^%c#3yf0yo=F6|N6yCRte!=12uCZE!y+GlnD#mYGBNg>U;B{Rv zTrHA5Avc)@km0t(M!aT5ooTN z(`uc5eO5MD|75w*i)HX_N^kXoQ#%Rsl!P4y_W~1hgZF_b47u4Oiq*X?e%r_`YCs)6 zNCIZU4eopbyPuVnSN+C~Bve9D$r@|h?p_vgZVqv7HHKRt1nY*0uPDFU?OD)>E=VT1 zVkBm4+)RQ&&RYI5S{$!|f29@OzPE(50xQqCFUM)n8+`aBNoQYBN_YkqMA40!SW-YEU0WKrD+?${{q z7{)Ehk!K(z-z~1dG~qA7^ZneukR}Lk6L7T8;oEMR+K-L5S0q{x-4VwfA+D7a@%}ux zx3`JW7aRv*BUNIK3vs0!aSzB=c^&q@3D$& z+`Y5!IA1H6$KLh|n%%4@13fWGmVBL@y@O2x#b+#{6x)xS=Qr9K&VZ~tVe+GT+`yeU zq9^Obk_*9mLGE6a&Z*l@nqQWE_r?5Z&b(*^wvV0IN@A4uT$!bU_CAGSa7-@P($5j|Kq)qlMI#R=I2V5Z_)>+W< zt?Tt(m1*q`bdAW*2@$k<{zAf`KqYG_G_&4RodqT-(&_dgZ09!-w2}6A#y72HJ%8Kt zvPp~Y2*=iH@3Cjj!(bw3N1CRWuB7?0^Z7ObpAWN}e0Q1ktbHV0*L=fZbgkB@mjVZ- zd7TPn8)DTRZdRc?Lj=$?s!hKK(p5)XZ-l2EfjcLC)TcEI%Pz`;Onja-3*#=zw#V6y zOFK+{A8nz-pH7Gg7ovHZomiB_)tu#zE7YG0u+oxANl}r7G>Sb)x06Xcu&zR?)#ZF5 z5SjJo`ZcODlgm$QCKosGdMSYG@o$TZwEbt?ropYb5B&BFt*U;hy)Ev4A~b9H#qncU z#ZS+~{zJi8AXoZ`#c8S?Gxu^3H6zfSz9V9ZgDa!UA~BV*b0~jR%_FkkL2#B44(@e5 z5F^UWPs06JuhmJmS`paYjIZ|W<0t+$3({1D!CgWzIucb{UQ_fPkQBJ8w1lG06C7%H z#EzzIi3as@HM5R~`1u@B2?8&fSAswk(khoBf%*7-i6B{RJWLn-bq+uCsGKb?OUx5W zeYz_euD|3K%6hZ@$1w@XTj7t{7xUw zPtWf#7oAF}ZF#4ToC;)ZnM#ay{!}&B%fJ`Dn}98Vh0?5^+TScCKD(G+K+n6fjy*1> zy%P^he&rv=G|hX~vqX8OwjJSxGbA)cMm_B61}%S*+zX`{zq7?CnOzVY(#Z@@=%0w* zS!8H)JMXKc5uwqa2Q4sOcvg2n^^7AmV%Q9#TcQ|wu=yu)jD}2t$Y1Qb!tMFd(X*jf zeFx~YebGpyRpl}-JR`KOdd6YuSVL!@KN4yAb&?mJ7HZ?*tW-T?G$lCB2T$v6ts2v# zo*$6Y)wSb~JlH7k9iu760JdvlsN(Qa9lbr1zgI_a`Pig1uo`oUKxYs!;*<3lsNH5K zND}ybJL@Ar4yL1p$25iTs*t@`hj_?ECV^^?s2t(wh{WK>9inymJN568Sg{If&`;bn zP1uyx4m`x&Ss4qFrzGiKOqU`vei-*xM_Y5D{h{HtC3xSvh;E4b^Zc-V0jnvT@^s}8 zV=1`V$*Lm_KN+1Rx8zh&?c{$*!B1vR#^W-qyE@x2_-p5`?kMALA#b9_EanIIZmc8` zBtNwoD-DT~6JGFI#_9|(SAGGSJ@W0c%9y7~4}p=yz|j7>BlT}idqsxN1mJsfDFo!k z6725E-NBC>G#Bz36rCg2sJn^ZYP}cF?+SQZ|MRfnIA=qZ}j(~|`Kk!O{e_GaSqT`qN>HtZ+|XSv{^-wukPuZmCg)QU3HMAKHog2V13 z=^bRv{1Vg?20~4|*y!-P8jir-!m*6MZq`p62NhC2Jy~`s>ttvbX{0W%=p_a(2=ukA zGN`vTcPDcR;d5WZM83?AV!P6>@Ea@U$r!vLHUp*30`i zp=3lZFS!kJuxH79xC105aj};%D#$_{rJsya4#3_ZU1wa6R!d%xZOTX*?vXz1bjO&- z@hd)fp^e9^L?os;Kh9>_`F~MxIY7r`JcOMx=EnZCq0}LAh$K?sUqTS5S~HVLUwYm< zdd|X|UE$U-p-WC{Ra$BYDtcBsBb7SuY{B?fvn-}|$l15zBYF24Oawjd0Vk=?#4l^5J#tyWbVdf4b zKVY44#c%KS3eaN#k>G<=GBv!8 z8Wv?oBeyTrZ*z$N0A9u;r?xH(^g`VJkfNRi5tVw34>b$6>1JRWmV*_5*kjBDK+I#; zBv+DUN%Yl&i#+VsUK!?L#BR%~gWdiZPfNCfs2K;K?^J=)I~&wVpIc655se-B=S>Pj z(0a4d)F8L0Scv*=izl27uf^P27uaiqV39!-bb2$d`g^z!rJN1rA#RjMd4K6_k?VN3 zPei&0p0n~W_T{Nk8&+*+N~#ziN$48T0TONc@?KKEt18Kqw23D$k0+4V#%s~c%GOoz z4g~5-^{=T=gR;9j^ZH%jI7%Jq&I_fjCLFx!7ULX|T)IO2=_*r1cfVxfm0A4%A~|fG ze+t)ivT#L|7LNYjib*pZ@T&ZBZ1!lq`*lxbVHAxj9lwWAeRBTQEfl*0yW@Tb2j&Gc z41KQRG<8qJAv{ah^vVOe)68vEoS`B5)j^_josLUX%^`Kc#3O7+ zgEqVgyTF%gXG@sQ>>NMttmwdida#i<*;SW!t3qeQv@)}EZ4cwNQrx$(cq0aF4mO=_ z2-6JOJn2eyIwv+>64)fRUK&cgIJRH`osC;Cfa;F_1A~QgoPGaG23%5Fsg&W$K8N0ySRoO#L0(EaZ+zaZdqHOM z?w{4Ypa>78F_PB;n91@L%t1Khyb&gk6l5GRT1hXZO>P2Dv8G79WK1sJc&+nx~TYT-fO$-{p;p^v)~>2L=3Y}^C1j2J)bY~gQm2bDGwMHKFt?16rlqb zF%(%ePqX~S4|JS@ZpU!VSHqGKt^%bXBKOml;frU>^IqyrcFY#1tiK!&T5AzgBbljQ z7h>gy=*f5ZFM05#DTPeRGLS_MF`d#rfGxB^nm%#!^fCbObWf06NAB;4F|VS zAm371c*BU~X6;fI52@{a{#-ANZo-E~&taH8G+!z(-#cQ4)2-m zKJC)2_+SyOu#fq=YgF@Bk87L;*OI#psVPei3z{s?vj~8C>ID`2QbjPEq9s3D zq47=JDf~I{sz&?7w4s@_wGI8L!<*ro9ea{}K~?nIFChaSd0+4~ zLMSQUs);>a?UTC116VO*s&rz-w3&5$!sq77Om#A2R#y`fKz}zue-%*;6ht$t6u9z_ zO6({j-ub>3fqfgz13^9&jWV4#eW5pnFpdx}a(J1y9`k)`{8$^bY2W#XH2DoPZ?Tk) zx_3q9iHZ1RJ9+}G;>;cTvC-5y1-(dSLqm_AWqIS>&9vOLCk@|;mI~WltmEHSdU}p6 zAfH{#rvfkL7aUz7!^mgX2Z!|;>Ua8i(W0r}Epg}W^Tt9z0}{OscDL9)|0e7zk7!fI zUv3~yGisCVr;JKkF@l|C+6@dvzPb>l;}2-^l6AgrzSGT5=`>8*{RZ9sKdQb0tgR*5 zIw81IDDJdSoECR0h2m1&-Q7JvffkqI?(XhZptxIcD6TE;@6dbi```b~e3LUfImrjf znb~{owf2eMV~ADfQd+hHLhF@*x=ugWozR{(Hm{nuXgPM)S{Nu~q(NWXmWh=VMMKRG znb3@LtAle^D$TnIc=h}*yslpjN3_7|R*V;PtD})-okxOrLcZaca08Ee2O3dQeIlPiixTs|4 z`65aPYZP6LckM_n+#az#bf>F6!vhBuYxE&ECV+gH$j+Ad^5Hs(^?Sn;EUf;@7^%x6 zZx}2DftdJjJp3FXS5{tCeaMVy2ffobKQRpVM#U4E&Vg@gLAE07H0L)+1Y~Zul92 z*Hj1T5!qrt_DVnZhl~Wq7_p3a`Fo!hfnb3IqQ-#W8Jtx05z?ET0+XbrRFPbJ3Cg$ zd5Mfn0}lJ)vTH;y=L&AQKkM&6Gax|%>RlCbG0KQtBX2S6gx8KF+)`hngBx#hX415O z-*b{iUPdymRiT@_Qu9cyo3}t4AP(wgt^sD^e)KJlN2X zKdMv3_2OQK(j6>>oBc45#8sN@sWQDX!ZN<&D=FUqEe4mJaC#@RI+5qS$CdiZz)`c* z{+U@V{_V@gGWM}`ZRaPY1gB7^$i1MwppQLH4SSb+LB0IA4a{Nl0G_`1vfG?7eoBL( zpiWKJvMRODK-W#hqtr%WIs0~Yn>5AK$_DtAOc=8IZ3{dj%(qJleZ}E4IpB)8b$3n76MLdK({!cD zHN30*?L0fY#f;t*z7^qSA{e2i&n+z(tHb*CG51h_2t?(dB7pFrAJ%P_2K5_@d^Z<> zsCdP!i2!a>6vZZ6q81nd|IO-~t{jq7s4tT!{T`lE_-bl^0|;@@P6LJl$xxk$5~YU{ zCEx?d=~ZeR^h38AlGVIpH;s*Q#@wWsBYSw}6e7>xp)6h+@@5Zr_B?uwElL)$#}>td z(9?cBy1`qfz{T3*U=p-7j5?G=5YNhb7)qqVS^0Vli7;O2-C^j7IrWNke$!^3e2ecP z5FZ1v@m1OKbaN3}5D#!Gv043%9z$Xqt|hz1lJ!eGI3E%RE_d(yk2h2BN9(+IhFk9``}g8Sw_pcS_}d)6S|gsEwgl}e&;ufR4be51d|tg zcI8g)3{!+HktVbct4I^`0INYW?h!+>i{eqr_G?R=zjPo+<+y?b=2WZ`3wNGooadi& zongw(uHVQ>t6>X!!&G5^0ZymcU5?_x$d&!S@8@s@D;#$S4LYgjW%Ht`$?E!w?TxVe zT|N1{(v1kH`;%-u(Jg^QwWXiinlW}6S;6u%&Q7Dx55WUQ9oRUZci=ElP{a^~6+s=n zme|T~0<{Z!&xLd^o&hPe6cOmnlb9c*I;k;!z9PPg%pe(+=E9av|I(GKpk^co^l{+6 z0u>DTi_Fp)F3vTfpD$9f#JGezH%q*zN3j*`DT9 z11=B)PXS?YVQ-!pEVjeHj;@=EpZIP50R#4kLP5NbHiQWB|3zy%DODi;6C`&qP zmoN=@jcw2aI4$N^E6&+ExNdEg!whzw`bp|*SKTZ%(1Tb$J>O`yU;IQ3cr0yP$!eV! zFy(V^8yQ|L!Vxp0(xGtabe@QGYtPHf+Pi9W9(7+ifm=}KwIym%zG)*CA0+bPZbt+r z8Tyr@uxV%RiLhvA78^X7Uo)XgVoXta_I)a!qgj88dNoP{5AljfjU0HyCm!`SwNC5p zy^~#W{nH*E|NhqreFeAs$be;^w4B)^C7Ul+as|EAw>j&s3*V^8R~YD>WJu(CJ2dgS zsvbf5FM>YWb@?+;+*5)?USUS-y$3ED!`ESP7vwM8qCp@0@d5q_vFU&-#m$SXCr?MA zu};}~3|}|cN;``uCw>aD+4*s{^QP7o9#qpgo~evaWZmL^RHU}w_w;_AGwx~nIByHe z@W`}W&l+qze?2&y^Isn?3cliD6g2BQ_v{NR1Nf9di}c=CFUzyCkp_7cTi0OVw4slbXPuL?<)Zk#jfb9?lP;AajSdfg{N%G2F)dV7+Y4z7f2L#~OS{Wh@Y}+Ox7(3NY9k4E3SRUWqqkl9i zVz(u@#u_`g0@MGlHr8$ZU9D@9IpFw&3EQOyqT|6GUL$1+PIzEK5uOlSY(2a(NHZcL zD#r#ts{sXKO&2=Kki1f3x8~^&n=Q-?3>G5}dLq=$?q=Yo1JD`}Y8A4{WXNZ^vXhxm^ zycK(SS6OnN{JgwH?5sdR&;IV0uudx4spQ7KZ|IJX-q0&KMca$TOsy2``%deJgiSv@ z;=z3wQCv}`B`j(+DOyon9(h=oHoA3*w0v4$<8#I>w=nT7E|;S*e5^~ip%K4GMbc@2 z+VK_)mlZx{3nl~fPR3qMn(mIEiEl7MSQN?`nmkq0lZ`yayl7`iBmD}@xX4m(CcE!20YUc|~TwN75+yV9jiUmxROj4~; zZUDvky@rhhAA(O)BRk3Flni_lkyDF4La4kS;!9>Jc|X{f67qh(a3o}bVWl91-tn>d zOIy)#OKh6grOel}l(DqTsoq?rWaV}(N@>lc*+y3@7cr74q@43T=ULYPpg3Z;_R8Dw z9-5s`ZYv3z4!1An8OYF*$F4T<7Q8v5yh|E8EO<_Bb8cw)DHWeT{ z$agey&dw!}6J#1~jRW+PzH=UA1o?(0&jDQknL;wdeLjF@;q{0zt)cbcC3rWYH8X%) z<2UJhf2d~@c81t#18sRL8zb(KevXQXAmJ4vtOKU@MXpOFzM^S+3%k14ENc@bx^`qK zDLQ_0Mpi+Vf}sQ2QAiQdcIBR9c~rKMJq&QFvEzYf{M znfiC~jNQel7fBC^^O+;7jz6Qn{|ClfYx_4}PhIeqn@@!^?HMv?-g8>Pu9=c41c1CW z{dYY^K?A6<1me_h)4EVNlb#KK8Fs^?Y$SBxO*Ti(mQ$LVj| zNG~^l3ETlLy~of2oLJh;E7F?Aj*Z9SyOLvZ%VQOILhU65XYm;lU`w%C#kW0kOWFGm z0GDP>;xIRR{+pkJ>gQ|i%s~2s)Uj*{yg7=62J84*@#uK*`;y{-+UDHqPb3A(ui-0r zlu9H~beuC)oYPg}_B}awEtY4ROYBuQ%lov$TL`cA;4S#4JqmxzZ)Ww0h6fX3u&+dm zEALn1=Fib}K(wlKyL_3|12rxAU@iu^<0y zV|};}=LNyB)X(kjkK=c#RkWP12iG&p4)|w18{cuyI2Gf8hb$5ugSlP*9V+y7NVaG8 zaHkmaHRTPyi8y}SZ7LiU8ntu?kC_X5Bj<%5Z;OPQnlLl?zDyeR>?=o-S?rDN#K!8P zrmfux{I!jTfEt029xOisKN$``i-+AgXir!%^St^zL*nV{d4}38wv;dBJxw<-6atiQEkPv;`zIs9&`vNc0zR>xC8%6sOtbJDaTK152xJYd1y94P5nb@X*b%K+LmR-8 zA9yvfKZ6#{Oc5ZqkXAGhr_d6}0Wypx#3`g*`}w_;1mk{LN|5q& zI)JUfZXM;i3K&BYpt(*J-{n_4!Sak;@`PRe*R*yfm=WOc-O7UxiY~ho9o@A9n{l@0 z=>N|NYYzTHNK7=1sTafG#?&KJ5|J`5(yb38?Pptt;)^etI&gNbi~eHN#JAs_`;M3h zgsp`x3QCjJ%TRZfRK|*J%d<05Y_^~uc64u5Yn)ZWji{CfEJEmFaC=7SPbnRX<#S1aFCD#pGhni`K_idR+rvb>@^T?G zl=7pCEot${u=2STR{3p{dz8-$b{o<}2`zz+{dB4aD@gTG9|z^41>bu!-Vb;$g_kq3 zdH9}u%a$*EyX-74Q5>zJ;s^PNOn$O-^x5aJuMx{404d95%8{Mt_ znHz&PCY(p4La1Wp+HsT1Za!M<)G_m*a;3R!>kJ%l+*%(uB+0TS_3d*28PyS^pSf9k zPE&uEn&<54@#XT-k>vSUMr}(*ms>C|AvHf=xBg_$w_r`_Rqmv$y|*#|$4$c9iY`q4#zfG1x#L=I}p3rA?Tsfuh8qBZTFT0WRc{>(XI|Ru3 zi$s1{YfIoH+D$n+8<};~zTa9K+6n>|g1f|HLy!BZ%?wiCX8p-$HV{A$dSoKB&bg&+ zG>L5{WFqbVUh-9yv(!dyrgAuzWdT6j46=LQmn@%Tw+B3Utz=3$_e^+UC>;%*o(3bZ z@~saf)iuj6(IsJ3((aR59PFkGVzj9gpzJ#dr!k4AF%?e1IX7p@;!W@Q^JvQh_S#M& zn9RzVoo)gR;c%wH2{`sWe;b(JTPlE&cGIF!5yEpPj z(cNz*o^#**#uvZ>?u-OKdAm?x=e|#MGKC4mJ`HQf-pwzJ)pNDwkUC9odngeSqPt$o z#4Yw0Ou#>Q+VaRml*#y%$s8A#H#!3)RgXR$xc<(&-N0H}P7oGF*jPqiM~p@Kw|(6X z(&t%c*?Hao*#0&gf<4>b^_$*_o921vH(t`1xbgK8M~(NB+TEF43>{o_y8mqs0~hrBEEiDO$R^?1L0%*{wAL^7|M$JABHcMd$M$^|o8L_o_{l+mxxV zx<3MgM5L~HS4C|m6c6XrA4Z2ulF4O!!kH8Xm677}e46;-C=KGn*~nK$%E<3N?=Pup zGRi0x2&;*OGlHz(-0NYY_=PX|ex#Fok~WbT$vw|t7qE@S0uGU?X`apXIf%bbM!V*6 zms8_c#5!NSS*TFYNLY3MhqFAsGri-$+E6cUSqHCNCai2JKSTKGpTj#so{ZARzutr^ zKw$O8;bbrkYp=7jR9G&t#+YY4Ne<*$iefpJBHCki-)>@vJtWnT5+JDOjPZM`wIHl- z#M~42jbd+^CuEVVkB1`7ZZy&IJKGw;&vSDx2sK)LpM>ZutTC0Qa{~DH8Il@!C6~zK z1%!)Wscp4_Laa}0RN|~4XNb6~LYd9|USG!ZAirRmmh=G+@JU#uO%jJ$lJrrR7porG zBG6(9{xQM!K}B<1@auXCNj@C0SOhEZhW!Np^PoSC{_it^qzwtx)kI5#!`40Ra%xWY zBRd=WiMeQ%V*pt3M5g%`|*+yqy_t90^-o%5D-fbZ*jToj6`CKPKa~ z9Aj9K@k-ofnE>bjXum$&tCqCP?AtteNBg+lPw~rKvGL#Up%?VOw-;(MSehZUuS+oV zj?RmF31V&2%N{?(OmQrYjJ@Zxc9u7A9aYeKrT$0%jQC*9} zIHC)7i4$!JnN=8xc!UwjR0Zj(@X}d`Z=IHBJ5O&mt13Lt#EuFJNnSDng-<@!e(|XN z5`wERK}o0aUM#fG4C}Am$|n}0nBeQ1P*+3GkvZh){f0~LN3h2aYXdx)vcW9*IFT&6 z01;&vpuQQB@*cKJm>(5NJ_LQ*Qf3YARGgHxVCAJqTh=Dcx8YqwxXb|{1La< z%eq8~;Zw`H3XNLfW#3F5&&Ik& zO6eec=&L?Tr0P??zt5L=n7MZ5Ww#pHGH^M8FFX9!U^XIvQ$2vsLs<%9L-jYe+liK)Mhp8YTyH9v?xg za|T~LbG^^rCRZGj6*Q2DKxA^q$mTHyzLxx^?%h`6-!WA6$4tlu+LSRpj|KxO&%5`Q zDHpF4e0r-0*Sf3YlQGW>?;u_`y(_DO((3rJ{ z-`;ad(s#lF4aVX_U4IKE%5lT*>tP`crCj@_pLd%wz|d(DUAx%xHZ}M+AL_2T&=~JZ zyXO>bPNI(Xp|qut9K?fgdWT36k_DbKLp@hPBoh^< z%j=6hR|pNaTgW7r813UM%jyPsc6fWZy$!gl8jWYJWYvjIzsEy@w-LRy6prA079Z}m zcgxhp$mvFGR$KH*$W-c98_MjO8~;HINOVV>lK1}7^_JUPyq`vBj-KgxqeBL*%bQ2X z6)hd>k3n}t$F_;XdaXT9@V3i;W4R=1Rv8n3iobb?=d;LsU$1*@-Eq3;v)=H~ZF?8E zdL6jxg4nSbN{EG`qiA7job5DOn^504l0g^S_C(=p$<6Wuo^t!B*|uTP(v@s~kcpz? zv(<}tb+ocm^+-j9JRcGU3)iH3P3Y8=ZVhLON?`=@go`)Azy0v-P=Cx2Y$)=qTs*>8 z>L)!c_az@u{+anrwWyq~HK&?aQD56Zyy^#CLyiIOq=mAXIC9yvdcwSdA4%L?v7v;@~=#GAKS{GIx zZ8=s3ZQssH5D}Vhf=aWJIM}9wKKvYip7Ro#i&9ye<)VexW<@_@P`Fsmp5Sa0&>r8RyzjR0yaE9uVImD&|*rBBT@7Kl%mz3mDa;pR;!^U*GQ! zYR(HDfXq73d>^L@t_J=}fW~*;eSDMRbiFNgQmz@hhhBT2V%GdJ*(NX_mNj{ym903vwx7;8{+-MH%tf4OP;pfz*k7&r?t^%bc})ddJU6P` z=2on#eL-lUV>(ije>1zyn{v`h*x=E`&fEadQA)~NyswH6&gcDJ{_8_OZ;kIDsrXwB zLWf!i59n9Covc^9ODA?6QxCFNynC+AtL8j^HGy^$XaKE2&!nl>f$RWEv-^?SpS+yP zjr#v60oY5*Bahml2gN|7(S^8$jHmA-l2;WDcrCnlhAVScT&M3wlk^*c zTY7~NK~>=I@c2YfD`<6xAi~*?v@pGNEa)`&(<143736%bfPo^injH3vE;!7$T9$IS zz2_`qPH*dH;~(5o>?1+w)g59lCTu(Q`KX)7qOTxArx(oG*Ei`uHc}N`z7D-b$s!3QW2ap-rSg?56v&`zER zRT1AoB)tWQ8!cgmztYK|iXZ#z4{jaA%Fe;YG%5N!w4MA4K8qN$G`6WGmR%Fjv3rO2 z=j#k!9DE$9+m$3lfY@=r#MjkhwPfF5tmmrPhc>A|Am#5`UXd3nWTeKfnpwnT^GXX} zWjt8fA&z2_-P>~G;1EbX%B(CuFE{<6UVZ>fggoar^5;J2iU(6|JC`39D|C$N#M6 zjli-FXkd?RPm`CQao~UOIBLT{V>K}X6@?%<^kDWleHmjlB?4xP0p011yeM+^qIgff zuTQpa$o&K*^u?hZ*9N33N?Z8iHJ>pc-7lLb@6#@)SR`;?F5&l*tE0vs_f7X2B`3T$ zO>)A<)WV7Do5x$_@g8HV+v6?5%tyxSsS0ZUb474YVB=K&2^ZxRuS}kk9ni{cI*e zS0ZXR9=E$rU%#BXl3r!?9s{UL0omo3VmIST7{?~RT!CvKxpLLVIJi{T<&z?7X-UH;LZNL*P6!^&(^8Of%In*bcgAe|7r0*j;nU*CGT* z(OvXgz)C@I9<0l#pv#4(tXVsfW1uSb3qQ+EuJUk5oOfn5M)w7L@tu%0K_Mfg&X~B) zII|vgI=+i`OURobRTf=~6oDyn)2NGJmmT80&ACl3bO0)cw4#G}g$_YgkkSbIZBQfn zDG%s*_Pzd*b=(*Jc8se(1>$XzIffd+iLgH`ltw^oi$0xfAHmswpk#(A(s@hcE^rSX z&RCZVJ)c;xizZlUBp1GFr0g>qI`&xU91XjQ^4SjC8q%0s5ntk-#%YLzw=)uRe%|Z@?nu#s zaUqaL^^<<@=6i}x2E>!?-cEGP)v!Iqa{JJ7>u!+hm8kUg*lRjsSX+cmp?JZ3h<15x zL4N)CWAS7C-Ow?!0t^5?H>j$gtxs$$!cssXK`$GNdf$R^7 zrcgE9D@`Fv1KqDtX@^Jy>3T<=ncf((EC_>FLN}!S7p}mp_M{8^OK5?Vm=FEivEO7F zD$7=b-jO$>DSE{8D0Pe#t*W!oM`fO6ZGqtK4mwwk-8m%`F1r+c7C20dp{9!r<#+r`Mxk&UVCulBY zyuR&DvS;vtkz)KhtIAAsr&+BHb02fxR9OzO(oFgGu<#h-jz=)2>Ol%fqdIegk-+iw z6lEO(%IoHhYFk&r-&{8=GPkzjL2oc%-kKv_&a2*M^Q4@)d}fV++dOAn*NkyP^5oL~ z`ew~@@(Sn4jS|6I%6TDu|5DYFY{c(zgfUr@tzj3Sgf@&y1QW@|rjY(VFPmzRU2XQ- zc{efiuHv<_F1;#g=+GjPyLT5mX!OVT_Uw zvJLEqZ)h&5A|=0w44!$TV2%<$}Cqjd+O(RsP0Yj4J z#>yf4aBOpzd*YTRA|3n4cLO+#q9&ZQ<#Qxv5YY1)=Ypr=f}&C)L=n5Q@HdGdaz*gL z3jG3wLKx6Hxr8=B5fDCf& zD4QQr$>EBe_eFTbWVL4Are^oYpmMG~EN==+!5h(~%&dmil~nXLX@-c%GU*ZMxl z+1_-x!*evX*6-L&LzYIxW-kIQ9k-KOe|gcHC*JFjcjmPC|4MA_{i?AMg{y0>=1wv! z^Nus7_#O_84vmmM`r}|w3ij1etzrV?GQWdIvzYn7xNaH`nd^GgaH~k8)y!q zhmpzG`3E!%VcUsvc=F4MKEt$~ zMkBO_gt1ATsfcOlI5uRlG40DTeYU;v104P97)yPmH0QxdRNOr4LD{dz^k^s~(?l?J zf1df^`)r3^g5;IEcJ4;1)n5dY@?@86pIEAd0<2n4m-|f*pxZEPc$rO66~F)*2o&o> zWLGX4&wsb+V$Za?j~qYq+N3yEn;jrtEk?A>W!MNV`y}mGKghg}Nqar0y87xyNH?lo zH2&KADDGfEY{_^$f#IBzg`|TV5?cOnLEAKdyfQ~Fr~1bmUe248w+iPSr8RA{Nad;L z?;OBOM>Ld)WTNd>^6on1l2dsBz_hO=*J;t^sw}43a`e;mOiZcI6Gn~%&D*{o%$8a) zCf#b4f^}OZQbU~7PjqHm5-x0xWSbk^8@-z!;!58&$T8PEgMr_3U}8rAkQ&G!M%`!= zx&4)m0s66%f$c3Hln2~@k!>ToH~N1?0bC|AY}`Y)!++ejt2Js$XBkS^ff8NL&5DHd9X!8RehHZMQ|`Mq*`79mu_%_#dPnJYmJ?2t-_V$uF0j1 z3E3msRF7U&<#oX#GxHZf`!&QEugTDW36gUNxeXn3Hkby(@M7Ji2+`dnLX$0&nbnhbHafU38+qxkW-cvMlrdsO6;Z_S+g zS;FGCi-XCBln<3>2e;{ams8&pI#7{m5V%=t!w>dFH3Kxx$y8mbIesU)VkU(lMIyMMxkKA7mY1D0WeyI#@1L64Eg>sEZV{%Ul25Ce#gR z!o(ai4;*@2&3pWbEZ4abwo*%VnixEiVj}9?xmT%+elz^cFDyoJf8YRTs}9*0{G##e3Ncb{rW)esnFCCm-O3{zHYv4(xdh5kY&>7pRyIjWcI$Bx=!29 z(|bJIDttfOOgX3k_!f2-8!879=yEOgW8j10Y21oP>AHcKYcB%p+s9|cKp`M+m~;2{ zM-J~@5aX$8pYmn+7ARi0^Nz_zn-+j>u-D|Eg5bweaZBiUbR+inzwc=!uE|yHbc3p` z3Vp2qJzmkv?VaW;o*|Rb=>9ZS&_{}79E(0RyT0`}^(#le`8+TvSlr9rlSJoOa9My} z^iLqzVLV*yhtaRori?)+cF6eY|C6M$}w{qKG zjHV{I{F(~~@nr?0Vb~vfFHgkHf?(B7P16i63F!j#ac2^klFF^+3b~1&@YiePzc1&z zUR|wFz8sMjO=ooHND{wfr$`)#3(OeyV+4ll@^6(4Knc`)b*oEaw=0 z-Q)Ns1J|zHZpRuro3@U0xUju_aJwnQ*xdtn>6V~P>V>qHQ0a619S+kMi}>jTO;jshwgx55adbq z+`l(2Cfk1M?_B7{JPrYJ zrh(fhw+qx&R{k($x4vH#LpLyFgW0? z&WN?k0Rl2t_FDs9v(!v`|GvL|3A8ee;|fo_P(bklt0Yp(8}@)o9n!Aq->x3<2^}oY zCA|JVchmuKSQKg+6j(#rlpa*+&^-vqfZNoQdZ+DXj~TMjJI~J^)&{c;igdaR>Y|fH zs~h+|d>;BwLT8%DPX`vA{L)m~#|-;3@Ugw}^yh7n<;P}5GvGFTGsaoX z3>+zey>opMx?_zg!JDr2hk$$-m8UOSfqsaksd+hWXTaSlMHJU0s&|^n#6iJ5Z}-gN za$QdMek;Gh>rbPXT}v)kaS}=d(rLoyF>eAusbF{Auy+mJW6FmRyZE1bYg4yN+cd5J z;ZMH1XgiOrM-ef->lW8mC)2oB&OmR!T=A08j_0%x{VY#eY8mW&G_uitElZ+sd`IjT zN1X}?rpT8o6U}}z(*4QO^c|a&StdW2*C*jEHT`@MHogG7cy|Ao9%4iK;j9rzesfxp zY62SrQ?u=ay8h=2RFa&%4T0u6{Cs|C&+(|;THo8xJtSctELUFeyGi$8ur9AY>u5&0 zQP|U-!NjQV&jcv-XQiv=&UwMJjOHKJtG{`o28lw5w_Yl5(~Au8h~MO|HKH!~Y|iQp zn-c0^b=5r?I0-(+2m0tzM4`cyz84;=9zM3T%hLy1Ks=_Iskd(n1qthuwUPP?tv4JZ z{?|W3cPNA~$nUhRx~psE8@|Q{6c5F5biJC8X%xIkKlW_4-FuYz(tUII0&myr!qudn8OVQA0=onIg4|8eE_ts&vy zY=J5M>E-`KN9|w6J1ixGo^}a|75DDHIeqOKKq03-5O$ww8 zNed~1&z+6SZ!X|sr_2z)XksC?$m~msXWqX2Z^wcLBanz> z_!S`II->>U#;WB;t2J_&y#?dE+uh%XpUwAT;ZY(;WboP1L6P@3<3-$eEO827_ zShYB#XyzR-AAw$d@SRpavv`0GbfB5MNUuNOVD?A{Z|14U5d4c-`OR!EV-S%R@rle6 z*K)suXsB~9hk-VdUjh1$m%sj8$g0m1gj)C9X_(31w1^Z^!@5nbt^ZQaqW|p1B*tr= zPBBgdd+QEfIvwSN@f)}b<@`>SrbsPNJzZj{)%2ELb>;O5qx-kg9c@*?oawY)q_fF^ zKa+l2mlLwOA@Xtvy5`ACZU##^e_M#rFj@liIC#N(c+2RzoVB1`o1^sqL`7l8xdP z-iA%WHj;(jR392WtY?0>9Ot!-+W&`VRt*`D1=IU zWAN_j{_V1^O7hJHpoz+BIkVTlZDjG*N)%v{rsv<=?DoRrH*N(ewf&j^Mk39gG#Hiy zj3v+YhLlde|5I-W5VUVwNPF`h^)5*zn2g%$AoL{cF>l$nriL2mwmQ^~(Z7`8X6I`klpaVQAh()``OW;p9N2F#yv{!DWMDrYqDAeHfLC1M8kbek+CPGl=bpvJ z{B1jMvQ54jPx4Bnsybu#&yQ3xr0!3jKpd7rV#z zV;}DqT4i7C8=hR7Gd#>Mx1Mq&rVuV(kfu;7Bb1*U`T$&U?34U%c4(b5#rj$p$%;WRbp9ZL%h67Tg~3epS6x+rY0vMm57< zYyP!-$K4QmOuMvp@l$bo&Ci#|*w4yA|7`MM%MLHW1tyPpsuxgAVl*VS z!&UGeq4M@8o+K?97C%>@YL>RXksW*JY%}z=CuK#R>#K9j(|7QlbY^)Ow4~1&*{fbw zE^QcS$KSz>$Yx`PskdU~P4MLp@jfY;4*#;*6g8 z3cY-wu3ex=T(Ik8Vw49`2Kg86tt{)KvGB_7qtWTTg)svGffG824|EFZlA}&Mh!tG_ zen=*=(+c(n-Sj7rWzlb+WG#B4Zxev+NY~s!yC>L@4c>p$&hWcChk`VPDLZIlJgcdU`02h=bjC+nAr1T;BBG9@ zFTK2p`}-9#U8!(UpwXb9ycxf`KsOI<1XyGoa1^WxV;OoW^R)@X#*um)_4rckSf%wx z3qZMkKA7=15xwbzE3FG&hleYuFZ=`H{DE<>3chXVL*(;Q^#>Kh~=JcdB*CR2;C@uoy?iIHQPi zXlM|KH9N2Z4ENcsj{vNSi@C*a^NJEL$5VKoY`xp6a4dT)f2@2gOD@7i4uunGmGQ@$ zdNiCx_z-JIZ*aohZ-6T()hza3+c17?!9Cfaz>TbI>?m=|1P0G4%jGwPvg zkx=AYvU4*7RKR!SwQjegd%WKJ>ZPEU^g_MJ@>PRMUZd)zzi}-2dYkle)V63<+(VSp z{bhNCCz0$n+$iErK(tX~oMo`xUQ`|vjR!Nc& zAj3HV>H|EMd13-(0iK#A@LtQy-xmk%;!ZtaAF(bQBww;Wmttg_*}@&_P@mMkv|pV& zy-E;Ugrj2`x1-o#N_)Sj;cw}{i1}toZ8RK{Vb~^5IM8m(C;zBA$#dbg4Hiq?>6-=B zZKgy%viBSMlrSVAX;`R;s5X4=jntnLDMEjQAO*a0G)dI{@*O4G4P1kw01W-X#*AS~ zD7-f?DtN<9-!j%!eBgTxmKwZxFI6A69Le1SvIkz?ze4Jmaqgn$BuzylB<|$@;z@p$ zdFZE3CWJeE>QAU6--Ch>-E04?S%e$|tj9Je?BN0XI)`ulhD@^pqK5`$7n&_JWi}53 zsiE1igJwZ$5F@lq_Rd+57I!AY?r%8$_RM}fwzO_xW8n$fH#~uS%(wno&WJDt;PgjF z-DBBEayw8L7~c0qZ-zK12tr){;nVmRWI~0T+#hZjm;hz_dXNp31CboKr<)LiFEPEu{jbs$ULrCDYPHGQSbh;gc;|W64~^67=+T_oJ;J z^h0Za5ZNFw0#y@-%I{RFf5#3a=X&>`71$zusu%(fbApFB?FTr2|KPm%V$&UN+Z_h< zt!DJz$wmjmK>0lI%XyL-PwW!UC6|+w$v2G@8wIA=8BRqnCXtZ?~j6pKD|{S zR%?H-OdZ`g*WPQLvhJ!?uq|H_nnO?SlUfJ*k=cv*Hf+OXQ&A|Hvlx--`;*M|(kn4SIV ziR=T%UB^9aPRmw`ZtFz7nV2w(v^))|qK@C4)bXull9sxBG3WA=hP0%ilQkbMD2(fP z6w4w$9Z<}8NYs%kmIZ%0kFwy;)xap46)k=lWdWC~0WK1VD|ORVp(md~mZ*bKEQ3zt znPhASg$E5FyM{5F9&uL=~mdw=r2E4+0<$13Y9mb&1F@yS437JxKQseKzYY?3pP)^i7_5?mjqS|(L< zSb413I1~+uTi?R3%-?^eytpVb4c$9Vb0(LIy6_QZL$p~F=JTDeOGMA&IQsU+t~_H`+?^PimjL{vVEY6){S(IMSuIi$?mYU`w0I4`>(n z9-=db^^KbJFD>GhDQ}YespV zId*d&c}#pGTFslnR1%eMk%LM4Vn{{(Qq>M_Y6>6d2F1m03W>og-=acZD3YjJ|KdlK z9_uuLrhx6>e-H>2#J1e`p~C7BAy|7Cs!iz71%G{6@gqdlNUWsDjQwR^E{|wLM<#_l zk)AcQ8axQkLIeN><~eB(@FHj;(DJy*!97W_oxcJA!rd1$#zDn3j$OwM z>bK=gXrFYa^9XgAUp0|0d3juLS;jYAa!9S5PNHEfpN?x|P3Z*^bzpfee1vy|0%qUM6e1WqWMV03||<&@&eZ?YF`z zvoYr!Kd4Q7)@RVB0O=uov4(5R2F*Es5AKoH^Tpu$&B#*~(7(Van0Yyd9osrIN2Lcd zuj)Ua(J12KMUTBBNJwga1#*U&Mv~vbvW@iee6>jRlXQUt0}SHCvc)4xWR(md7JYGQA zm|PovNtwvH6ovh1noGu|raCKbUvQakNnknMA^Q_{p39rpqRf&(>4u)jP^UKqbl&By>!#8X=ai^U809T|$ek5)qA(=p^f78uLjW4S&rO<<#4o zP`M!M7Z$Oc|3lSRg~hQoT_?D^1_&-eg9MjAa2ed)-Q9y{aCZsrHpt)(g9LYXceemP z=e*~;_@BD!{meyoPw(1Qt7@%!-qth_ydRcCA=X103Y4x6*=kT&$$OaF5>sjW6;6cr z39q{FUA)&<1zgh6@HBVCSAJgawNmk!i=po{q}bCrV!hFk%YAZ1jO*}>)m12xUQoOa zYO3NV2LoBvMw9Tsk9?~G2k2RwNHc+8D!S1lN$)sv&z61{b?h1PP&-fVk$Vi>nxk`p zL$*=aKtsEFt4Z#8ixL8&3pzaFL?$Gq(0(Eg;nAQJ_^<1~2s!vu5iQ_K2t5r_FyHC3 z{+JH4+y@pV2vOHjq0ffBFEzQW92}u?zcWMIR7oKTM_DN6R35ar?3&xmP-7|TA+Vi@ z((-d}CadyYo<`;hrm%JeW|9gVR0JXlv^xnOd7D7Q>xB7h!mc6Z-m+FsK7TUzh1??z zp^AydN;0LhLRAJOL$ixHCcoIk;WSDjX;Hy4#RuX^`6B(vr|P4UTN2aa0O`c9C?(YD z6@aJ$O!v7yDTSFoNH_uF4V}{H9^d8Df#c)2RQ@Ptg%Te_O(}yFNe%259Lvet^@(T8CXV12iC1A5l1=0nfg*mMuaVbi z^O2p^i&nksU(Yqi!hAt&z0R3VF{PNevhSP3jM1@_oYDEFO0kj-#MY4IcXUV23$H%Y zem06kVHi)P4Klm9!KamGIr1d_gjI-NP>OJ+0%MHWg8tOLbt8)u?rv(rCI%x#9teXW zEEq+m``xhJm>eXx|M+D48WnLG)`cD*B=P|` z2iU-OL=k1!81nynds&09ajPor&sa=-6U6h6eW+RL?C)RqJR8Is&@Cr7b)qvL*Y~f>?>=e4WF+GHemk)1%il}v30N>)Ti!)LPr_jj zIif$+O6-MTPYQxM1##E~dW#RS>I5b-xeY9C?bqp$?{z(2uxaByv!Oa(qPBg4G7s%) z%uNjGxo^z&<#El?t39Gu@) z^Yysy=$aVmG0^gvKszRB9={E?X5rP<`sCjS{_(R84q(S)Z~LxhwKBX(Z(hK=@En+MULxQhcN{%=uA3Qy&IhmOt$^#ks{sHo-`-`}P6{h(@fHPoY z4C95^eR7(+SR>Nl^GR1RS+w#;y^lcrF1|FO9{{v__Ay0ySl_X#dqiV<(<#n~PdNGKO{G)V6l1TSe2QUFr#xnf@5m%3;Xj_)%;D*PTi}!A~{W)W5J=VCXYw;IG6Q)a_=%HL3wnWN;b1kdAU7p$>iPc;;$oR z^YkyJVmC!dxaD>Cu&ML)rvElwHJPg;)wx#WEE&IHB4ndzQUmqI!`7}V=G=;d;*If; z)!KI(o9iRgd=++LZZ9aSPP2Rja_)3%x75zad%17Cy=Qta=m^2QA_P#Z=h`YS5E?B< zUC!RZwX|>Gj_ImOdY&oZl+noY{h+I}an!_x{rjdBBqJz0GRDVeoiKY4g`?HcKe8Y; zjLkpaLd`xMJtAKWl!lzVcZ%YNG$EMYZLPC8I>Xhh-pxZtdW2QjOxAqrUr57A^FR$* zc=LKH92Od}kgj47V_2-I3;Q&tRk7){LVNQuQf4_#H)@X4>O4UDlahcqr4{eG6uutd zT&cqqqp^C*W#?BJD7J~94a?lM6_n_MU&smq1 z%r2xXp7ysiR!V1BEQrrH!)zjC0pZLhIVeNgWJt($sax?~vZ8o{DSju3n~Wcs&sS|4 z4-9LEaX5N=BG3=VluBBz@XzP8;PWrdwJcoQEP*-MZZpfzzHYYd4S(SQZ z`Rq!&fW*>p<+Hsh_sJ3@Q`<$YAbUP_Br@bj74_Je4~cfqa_ofL;v~=~goM{6+|)+4 z)rgMu6fcbLn>7-m7!^?!zNcu>cHKRTbVR*7+_Pe6t=Dus2bZ1%TZ5iZ@C=?YE`SkK zRyNn}X!D2UQFSzb1t3x=GmrQCWFtBtO|Yl%q=f@Mqyk!7RsbWBI6&sJJ27hk41}DORR+ATX^UWRQFZuAj3QFUg2;()JFdU`GGTe`as?pX=o=T!P_6}hh6(Wq?UO}baU%&aCr zb3dzF$-W^0#~GvRV!pEA->(ZJAig3>JAR2w?qqBehqetc*5)0h&!#e%ClFbMm8ojy zPMPnp6w#qokF3;VmpLQ6=JNASrGurcv>dCsJb#Uu->sZNE#bH4DXQ3EL&)T)gfWp_ zu=}j~Df!sWV`*{CrqP!2YBb8PKbdngt%|bRGDu$W@I#jR?=vHDwZ9}jl`rJ8mhjic z)yFqjEy+&@K2+cJ$Df9R$|`IFA+T6$v>^(#Vo9S=rVCTO30Cm!?Gb0{4TUo*diMfD ze@;JR?1rh9C|g?dDYN}rwz7X%u5RJok8C{IYNCU#$g{!qp*-#8n|QVKpuf+yOv$JT zuo@Qewas5r8gAVE^;HQrp~<26t2VK9Jb}Bs550Z)w1CubRb#*@NqO^}9er>gO`6$s z!WD-28$-&>|2*1H6TQqp?xn%?LD;+_ILn+MaSJ6@3(k#= zyPg#0&XDf*D_d%{r*l+=(idTEAhNt<^vPffd4T5<$cbmZp^j2YkhDp*Sa+H06Nm7A zV!I)ai1H%=>R#t(o08<+#Nzw<<;=(*!%L^w8Y2nuKH9!nPPpqpYK}akfCqfM>?GB<2=iyn(){9TcqiqX35}NAK-6Ce3j-~7mv{9D^zTVrmB1`r7cY_HE$yQQ) z2borFwWK=*wgod%qd}9;OR^=3`nnv+%N3@*sOIc!`6-r{}p9=Y5qKWeD!vbM=;)r4GYCVis<%TUnbn$bn+=j(D zNS!rNxnIp@m5$vqDUZw1pL9r#WU2#f1#F>fcE#e|=@so1{ge!|!4g7=;1pUdj@b%MS6n@Zu4)LBGGEAyvCr7F`j%1DYRiV&`lAlJj*0n5E-b0em&XT2 zde}iMOyrN3Yuad$(T7=DxZW}1gbNMxJJiA-6@uR_ICV$1+6>*DPTfW)znctj7O;#t z_U;!#9{XUMm|pSmUx8dOBLy=JBa5k7ZY7@+R_EcuO@n4QD<$Eol7owF-yS=hKUMm- zzZJ2=r1jqQySBRw3f%me|D3^Hsx3BIgeQnvsf2KMZQH(oH|d@iM4WTAL1YHVH$Mbp=98TrPh#nb^&RcF+);470H=i?gGa<&lb9aPHBI zPL6-}tie0nCtS4*X2ilu>ZRxO;jLk;sFLu@I-8Y^#=|t=u4NODq8pjLeL-2{XO-Q2 zrLw3S%uM<~tLko?lL#km()r9s-ZU$&wwF9R_>nBJD0)8H?xD{Uzns1BS&oLYwt5!f_nHS$(5$XTgh{z_l<4TmW6XR`_B zzqH~_{^x!8J7C_MBN%;Q`Ih1ga~SDU^SpA^SZ2z-Ley9wN2c?;rRu^d+HSj>;bpVW z!NZw_!->tsX5*GX-OvFdZj@%>ZTAwbbnwgWlxtXHs+?Q>1o??e>Asbhvau>q?_0HB z-;sn3ck}bwUYM0%5nNwASMq_=6dv<)N^7xM*{Diusa=ZKi>24<4X?9_Q+o~}`yS^a zW+GW3FKu!Mn!Pn7vMr~CVOA$e<=C?^PV!rB*`LRo79DeU|9wChAIX_ZQ}w9~u(y+$ z%rVaVZ7wx@pFgw1r*6bWz7-=gy(wW44Rj3k#Fc|Q?n$%4z9Xx4Xgpa}2XEV>oiH(C z4YT&@YrXfsT8xQ=c-H(jMV?953lu2!q-E$#9%me3yN*avb9GCD%7F%lb-b&mA0(t?QDprSubs z;j^qIoD8IT#1A;47dir4pS=DC+ApIw9$}!q5O+NV`ct%^ECiA)M61&|W zy3U$34e=BVXM<8S4qMC?>RLU{>jEgl(Cb6MuyE|4G;S#8cQ^bD^pUFh`k1IO`E#og zTw%U~zt1o<%p*lWqK8JPtdQMk<}&9%PHNQ^?1(ym zy6rIX%a}MWc#8aN9bEq(cQ1P6v;g5pLn3WL0eaF`|ImaV%A6mDJQjQN`PRHP_kEDK zRF)%l>)UN^KEu)U*Dm{ZA)GdS*X)cU@BiJn+a-LE)jm;_wKjFO+2E8;&BiOb!lMZf zU$lHlc{xXicS5Buf0zF3+?JMH&5Ape1jpC?#b&b2k^8GSLJ?WsGIk0CT`|minSk2! zx^Wibz<0{LYX58pL&lWTY$qgCvo6K}o8>fePd9)${^EmF%^$08_tzkN+vccraiYR> z=z-n#*VFAii?BxqZ}^U3(hK#I%hO7llQ->xqq}0Q$?0kX4n$Z{^T%b=_lqc^=QX)c zwa}x0U0#w#_QDs*#vdaf{}P!tCZe#Gdl<)!DyYlj<1ObAC!CT_JL(sV2l5MZgcDc- z2z3GF?l)NiW$H-Izc|&EMwm9nz(~Ytw<)LZb8N*CqAR5MmeLkg!-VD=y)^gi(4z7i zzZj`HpI%{YMi*mk;!7?tltFZU+@VeSr`h2B=mp2yb=*AgMcq_(n|<5XzWq&% zm!ZKRQAo(&qaT=4UNQHjXt>jeaLYe(kNhFCwY1;HDUHjm2zh8l+v55nc9=(L+JP*i zjj83A+2#LE0OeM%9537PitZ^FZiw$tzh}Ail=>8J7lmuK zWl3RT>BiXltJ99t2q(fO%16QSA=bQ_dP}Mj9$!fOA}OwWlb&lc!l|CqppCpAKJki; zk@04^Hh-4)sm#6{0+)G~Fut6{mG;Uab7!gVM$NC;h*BJGwYaOuJmbsj-Zdh{%k9F5 zyIt$7g_}8c?Wfr3>~+3$mW<6e&;s-$2hhWD>@D)jAo!r;>RWG=hvq%_T-nP>M3x8+&UvOd*E^_BV=keDg z`%rT#I(wx}{ADJ=X3+{=_F}Cd@aD31hn%bk1koyo())&&K!HMPl_Fpp0{{0PSABo$ zCfbaLq&CL!YLj*qu1s@HMV(Gt3s;MaWf5VlkRC&mk&oy zCT+E*f%-%qpG(nGM$kojADxIY^GtzlQ7*lMghY>#heam7Cs+f{&0bjS5plLqpn(rS zy~fH6l9ju4cS{zSI7QsvRC-|loC$|mFdtJ)XI0%FP|mJi2RmJRr^<)+3eaA$a*e+P|4 zQ1-BftW&9~)EYosm^0R-o@X*~#kg2KJWCh4S0Vl>SxRLrWAmFX6kN=7H#Q7IqU>J@ zbU}LsL5`n(c|q?yKd3ZUBg3R)6Vl_Wli!(odF!Mnm}cr-dApDTK1}bK6IqLPW=dD~ z=T}A_sW=&wl5S?|dDq2KuYBoFW?G@_;5Y9vPm0>!2*~93p*r*86Ewc-U_FcByo(r^ zKT~H6IEU98^C3h1TCiae*z$rVUxo*-_vNfiYKGKF3+2y_|EAV|aM;%iS(4r%X1mAY ztz}kQ0m(eth;2T%k+do%BC4}r36yXwc+c8B zty_v;f}rq~I=4yOlraa|{U0$NcAVlyDc6dsP$+%s{Q8)4%N#Unil9sX7^5X+B|c}& zA-Of`EH4(tAieh7-u%PKCMDmP?zdzuKl(oY8O@34KyO&?n{FeLUgl=Oz|dGQLBr^1 z_@aY&w}Vx5RmmBe5+FYq8)4v>0G?aZ`L%gqXkV`6^|TCbRBbl%FW4qo0k7Cpve$6H zZoDjgyHCgC(~?h+t4^Lvoy<_Zet|J!!UJhqsM0`O+qeoN@z_19flzomDnUU^5-g8z z@cu6afF}c<+14hkkv7OZUd?R9?(==L(x*mL)Mi9-;PV3}i%g!dZDjV+-*gu|!X3w& z%Z-yCTG76L%|=47(AWk%-Jx6zTVF}Rf-PW2r>MH?YJ|Ll8TdVF$1?7wXF-}f^yCHag}KBFr7150NN81T{xWNvHlPkwYQJ7Cvl{c*O_XAh;PmO$ zT{<~fSl~ZxcNo{!E&8$k6u9;3lshxc{E)&wcjnHs^SqlU%&;Dsu!};Mt}Od4#8`rl zc+Po?F1Vp#v4JLBMoER}-nu(FN@tnLeA%MXLzX1p?!3U3+OZa4KJwzMJoY}J!<1dT zyE>HVH!qinD?N5vB^dmR>S)W@sUc}LQb5z3C~(VA!2IEt5q-c?U+4~pwy5*0vb&u5 z#M;4p4B@{H*;yA1O-Q2T$d{vA$M34#7((@YcZ~Yu?+fnBg75EzvK@x4nk2W@qY>MO zR6%jR?C~e9?met@sg~34xgH;1SKanU#HJMn<&}<^>Hzv#w~g_NjYB1qHb6_R?9E-xL@XF-4xAp|1a$C)PF;TBxqAGf= z%)GcKDqMh$7!L{B%KZZ-vb}oX$<5$3GAZ-z8%gaer7;KUVHsQ zzY|c7p}Io_G~WP?JA%XWw5~z0$dmrw5wE@xhySZN{Kk$1N0vKrEX04#6#~_{v0WLm zt{a^alU2mft5Y{kamqe=GD4W?yJvgA`IyLr3j#P@Tm79EEY+#py`Mt_>FZOzN(Vln zQhN1TXqoa^oZ2?1>b19PqvXv1t8DQ8GU0Ga<*x$aoSH{#; z)C=AE;QwS%=<4pz#Qqowv}9Dju8J28yR_&V7zF&&I^G>oHP*@xk}o5|q6%J%Peiou9w zxnkmxGhCQD22y2;hxXm)a;&7u`$!SUrY9X7531mjt~*SAQN#){iQfDXGOt z$Tb%loDtkqdPcS2ih4&Ofp(cB@<0?_fcAzC>wd%REyZ$m#U6Y{lGul+Ata5>khO(3 zU=sbp?+oky(L?^PYk_9Ghd{IyVRy8Eyu$3mLrwE&A*-!jl1s0vF&;hJo1KzYM%5w;NS*dv~Q(*+|za z&)mJ}`(HQ6UFwe?3Py?{BF{`WfVC+yi?YQZ4$cwnKlu-g)a@TJ?bfMmY(k=bbGOh+ z>UQ)pyASt#O+g7m!BID|t{S0vNO?Bo{Sk1Y;|MxDV<_Zj2HmZ2F201w9Q`j}H@E{c z?nDL?RPimI&;u1RcN|Ea0m!KUwY+}oFq0$r`A427WzJ*2$YtJv@-lADxXZ}a(ZhR2 zWQOz(b?q&grWTOslh@DCNQ!F$kIa=mKXh>6p8}5`7&uSJTESlhpRZ>XWgvCXhupTH zi6LEH{x2Chs5#t1*E||7IR3SpxxDGhL~~REGmQGeujCoy*-sw$xjMhqKTg1Xa!o57#$Z(BDngG>NT||B6D+7JnY_v-8w^P)jUuW& zs0Jm3EA_IX*Spk3194%azo_d}2w%A99my2<)?%JVq6>yMEtRmxI_&Qp-E90FuM!() z6V94$9Qb{QVG;^t?@PO#@NSmVQKdXsyi56hej$*$nzXr(BmbuZ=rAt z%!VrIIuy_g?Pk{=61JY83dV#W7Jp7Nf07}@It_WluPXVk`qiG9;!9x?m-q)*QvM9xA@8H1n0}5_)@zQ{?OKd-@|;j+f<7w) zuD^UP)zc}dz?f8#oX{mM(nlPj+MaeXbG|_#^zQk2#J|-qx|6eIyqa*N(6PdajGgQu zM>>iGa34a=l2BNy8VZh~HeWC$azE1*DjB3^(J}03rJ49JZJ!yN?rqCW|M)=#QRd{s zkq`BS80CZ0!U!v34DDX7SAM+~MD*JS!Pk z)D*8ByZ=?XFcL4NBg+hDQ|l$~VkaCa;uC78VeMiX*rZm1-GhCQa6n9=X;Np!`rIlG zOM}K7wG5Dfw?pTVk=XBdPGPw+XwJ;&Q&fl_T^fSI(6nr`RQn!6TWX23zi3v<&;Op^ z)RxAbC)NIwh@tgsGERqX-sBm?-9(Krh+Z+`M4rR!TmwXDWkSyx|1#);Np{F271-=J z2X&kMbapWDm2qS%rx9fl(;nI#3~6MLp;t$E6*`KUNY3ZCu~a|bc1DtdTJSBQmmeSp zx|LW>WoyYvUSE))2@P5kvGINz6TLeg!0A+x{+4C$(ewm2L;y!ahVqiqD|HSQpW-=V zrX!!#RLG1`=X{{PSFv+!F zn%rPB`qHpgMm=Hn?8#>b>-O9dPqmd)H>`{jHEhoPj0GSJR<>i1jtfhhwcGsOFqfTb z8H&!UYkj3+l}KvHRRut2)5#CVQr>_Ie*0Fx{NnBdwal+n zwR(wXo=OF$Fd*6-?Gv(qwb;ZcciuI18e_sDOG>q*)|+d-O-#i^Ud*ZTD7}QMEI`Kw zF>GG4^-$$aRA^+A_jAUGN6W@tWuJH4y=g7rp#8>UN+MJxDHOu?4@}{KjX*!c+*XS$ z5g>Xc`)ruG)uU3{;NCBGd9z0*Y{j-=ji*vEDqREc-k*ls)F@xoR2~@$II8fm{KE?( zJ5JTgQXA+6XVn)82ikTQlC0R+s`XS=A(>*w*kz3m{W1{9%F9`bJff{dm=!7(?I~}z zU{U?bEx)A8D&f@bICqDH0*VHpBx3% zcn5Mm;9c_$XdJ*k_(47q0BuIGrcA)GG*qA@uM@{`$#DV7b5&sApu_s4f|#V*>JM-! z{X{Pz6_x#zJGp%wR8w}pb@I}yog?cNmuC)wpI)zh>}1 z>@U!%EqYl~vKKXFW#G$9=z@E<9_3gvG#E#2b?vvqxms|sz+zjh=0shGMU&6>=1!(T z$n5;cA_CDOLWZ)4+}J`n`7cS*n$Iv&IUk$6 zLX>?53q~iu+y(_f^^Cpx=5&c4lk}IG2Ip`T_SEcsrRw*_la$XRsicvPDgO18=(aY2 zU8mSaa{FU0l+4VL&xl!^4*>VYZ6l?eV|Sm>*TD91spCruJKfq<#w9y5z=6SjDQLf| z*2^iJQriAl-1sy8I4B6{*HC{bX-BIBZL&0ZS=mNdeBIG>mUE|%Me*80>%m;TNM83~>_=-9Kviyzc4U0ON{8-iF8JN*h#GWGXjkwg|Q`aYO|HY~% zhq{#?50@|XxmtO(GWv4+Wd+V0F_7n%Zs|9Zw~Tu!;jT^xsekcM353B-UY;^(o@5)B zq*jsj5^BmON^P93R9af5Xfr33T3?e-szG^HV+O|tR?l&zNX%n|j%D9kd$&9&%Xh^9 z?$D=y8!ru>zT5M@dO!IM`N1ufe&|}WB@?7sl5R=K*hmG+BRoA{2`buR;iF*-G$**y z@y^b&6aq2At8D6nurhu;jj*EHSk4kE_9P9`QzuHvYe%05IHIDY5xBL(KW^$|Q-1l- z3!eslISMi!(>Ff0!5{JaIqfLr#;yL1M_*S>7=8#?*%4b?z>VR2Zb*<(g-MW+UOj9v zyY`w6gaEQTgXlaV$Qyd{i3m$Rd8+PEo56#;BG>TZlXnNtM3_er%S)LI+p+|Q z>s;Hu6&q>MYhg2!D4T*0c7NiZ4X{|b@W@-o3`3h(@HK>7#aI&6+Ql5=M zDQCf)LQbc4e}@JY-WJ9%3Mp6Z28@a~8y1oRWj95gnibwc0XmiIiM#i_atsa4UvM4!OquXxp}bD2p=eD{c#6Y-l&1f=;k>j-&tYi+=ue*TxQp<_ua^?? zVLotL-=EL;DZHC_J!7PUspq6U>-S#f-wjX?R~ms$ssq!o_G>^&xfUI<%571M{t&Cd zB9Sw_uv~3(ivxnU$&rC`6`WsjIHV2}a)EtcFfs0ducqDD+9n{Ae&6m3HF4izCX_rE zlhKyPm=Tf4n-jf_o#}0Op%(UxQlV^_5TUD34rWEf>Q`f=HrxL5_gq{1U+@zXDmwM9 zv|zp}GXmT^-CRa2>-6_;Y`h7%+_21+Ke*7HI2@cxt&j7}O@-zOq}AKI-2Xb|3b`54 zI;H0&(4<_?0M0)TEt=f#F+xxkC5!C8cponHyZPpfy~S~#?kvke##jZn>kM^zytZl* z?Cpu4eZxjYG|_zS*$JyK&nwTIU>}4%f05{qt*>GWoVIS$=k1C%FgR{Go~IpWsb&}< z%52bXM^x;9w+(wd4XgOGze5i98<}>4-L{}j@CS#@SY2Hrjw#_Yh#mHpn<2{N*%}3Y zDhfY2?}rwfrf`=5vj`zcmHrxk?8J@wgLxb^95FGIqLeMecU26|-v1vI=zY>%a&!jf zTes+E-Gukt_RU^f{oi*(4)xGWeP)0DW@mi>X0u#?^Bsbf7oVeo9M?mS43MUn%jUPY zzZU7Xr6k@{3x&iLFBoqK+paXWi#T9^PNI~dmB{2>axoo>QVBgVa{Y}Z5=`I@-#G2m z({*XxM#A2r!(ZI&@{&2mezrjzjn9%ID7|b9#wM47+k1-7(KYw54~p=R*Q|vfq8L(i zs3Tn&duggbr9-P&^rXKBaJm}Ay7j6w zuANG>On+NaP{DaTLTTn5NU}23zdAgjFBUP|XPn(WL)$0u>*>2y^t2qBHbPgcFSD!m z$|9wWPJPzex@DfK08KV^cO0L~>1|wmvsFOy&N|}HzN&Joe|pB7{x`}C{^=V=;=}D* zaF5(v*6>8G>&g8(C4AasyWy>nm3rH#yPd`RRo7rN8&(VW+*l@(&Z}&F=S3daKUp}I ztMn_#ojjC8=gaoFLsYsME8ikFair4x7lOt?KUYpf)hMbRb8bUvBTv*~?2?CrKilBV zKN_dp=>(N5-b#Eg5j$ScqadGS3JHn&b=F^afKr`pg3=;U+N(~$K295xFINusAgv#u zf>-dH3*5)q%40G*Kvw^A7a2*E?w_zt<@a12AiO}%j$J65EB!eKKDaC zmHgg^v@NDZclG#@8{r!^bM7tLTR*oJ7yuJV3I~ZchibBs{nhZ?L0cD&`&6^kdNTM0 zE5*Gt1QsQw_cEu2_!Tzd;J>{4Ar`bDtsCV(D*Pw`h1U9m4rvDC_wm;iUdbARQJBxT zA~{;GD)0x8aDEr2XrF4l4rbEAwP-m}lNZ;*Pq7ofTionMu>bg1ZwG{?H0Y2MTM+KV z>#ly>t@>eRS$|7;Vx(r3>?Z%@f)yaCvL=YV8mgJOM;r&@63n6W!v7Q^%Tj!oDNv4R z1lE34lWd7W4`WzVpGx5tvHp@PsmyJ~V$UMC{UGOKCEJEk$$WBeao;OF7|)(MMCvq{ zD+T}-V@M2v?G+wQoc06T1kke#RPQ+EEqi@usB@`4jtC*fkUR!>*?}(M&gP2;UF+Ut z6nlaaK7+YkgM1jXNUaxW#rypLP$Ns8C!gB%c?-x`LkUk2wak89+s0C&9F{JiivXoL3Rf5f{y zoe4v2jcz9oFC@VptztJB3EYL-$!}tIU<}haFGx-N)wml)YXK2M_1aI--kgJ#M{)L#uc?B8y%QOKaDEb*D9Z->~L7|cwx zk`B*Nvud`cRPY`K^AJ1ZARk0A_Dvsli@Rr0PJwXt<~k(3bJopI?tY#=$Z+acxBOyv z2(Rk2|MuTGt|Fr!g<;1?5MM-GS=`Vhvp|om8`po0pEz#>9QV{(Vs%Gh*!aPtRy;iG zATA+zU1^}su&x@E>J*XCWyJ@oW6>Rs-{PJ)1%)waf74N78t5s6POvMMtrg0{R0EJ~ ziU@M9J=LpNiTP@3Rd@hhf+0?SUS+x#qEYdMM5W4qnj|%(x%l>o{ry{130_Js;%sr3 zZL=8|eOZ4e$J}zPukH5EMuNKzDKOX6T25z?xgPEWH>r#R1nO_cx1m7g63cmd4hzwj zy9FN0S)vy9-Vq~9i_gB?4b^S}pdN%W{Jqj;07q6Ltfj|I{y_{M}leYPGweoXy67KLWr zA7w%#^H+D|6F)fhGjX`Mf5OZWNw*=mT2T6nz|t^xMh|#1|2Ys5ur?lmE24mnC>(Hp!Jq* ztvcX#$4K7Z?o8kw3ALX8Fx$a@f?BE^BmE-NxjxXnGv-6hdv!%PCRlU11fTZ}e*`%I`2oA1g|HZAK7S(q3*1R?43^~qS0_Uo=?*O5SqYtxL4VAY%K zr`hLoh`0aQVQtkAfE+<&T`s3L8rs-cql*U?7)?_}b#khFWvC`gbUBGV<@nQz$J?4m z-e4Wn^zd1G%gP>S?v7M7*1aRX{+)1}t{)lqfJ_)H{pnB*&+4UL#L6O-s*r1l3NavCV6rSME~AZ?|47jd zL)$**NZ%5jJ*Xsy!hx1~hzB*lnX`ZA<|)xx>l`K+Ng0W_Nm*hzeCgh@sXea#ki~pt zu}3(?gN)^VnP)Naaw=P_KU+-L3BOFMtm1{Ak^ky_-}8n)zxB^S_J$4RwglP_ znu2R`+hzOk%li`-Ym1FWbV;*y^pPvio;pXyuk9OWnVZeUVO{+}bprH! zz}`nQgu?k_9E7PVJhnyoRt5P#V`yADJE@AHvBqPmZlhlsYhq=8Un zzdj{$4*=PuG?nRetvHg}&OS+;x>~=sfUi>8NW_&L=9MYQCX{BVi$14#qd0@pYJCa$oY7n~yX z{)w|*JsFtedDVX9>er}va% zBDY0M!{GzPx+umIrkFbEFH7@YbnFOZ0${u%>k}?6cd4{F=kVeuqm4h2GCy0|U*2G*`)o7% z)ASDqUb$FN)%oa4usDLM$Y68PO_DCGOT~4ibz}K0)uoOIqtqP}(bn!H&#P2ZI1`+G z-ho(ezQJe8QiimacxoDEet1xdfLbPX7h$PWP$shD8CUt@oTd|a(SOifTWTrg57`~0 zzOVe|!{}>7UAwz`xb;zOSd5iwf8L2Zc>P$#E$xig?U0S2+NCAEvf^;PEGeu|&QTTk zZAB^;no!V%p!UV<1Qfq}n;s$chl`2ARE8T$D^W5U$C`toqw$vARAFcl&qR={$k%*2HFeTMRnv|Q>Y z)lI)7*{0c(D#m8f*C%Aw|BVyhbMpc66vm8 zg(^}|RGbH&;Bnq@BAd90d(Okicno+lXl-IcqaN85bqa}NniUS~t#VXhsuLO!Pw)&x7r3ZN6y&9-jQSu*)@oxl>W=l( z3ey9+L>ZpY~Y6gSN@Zj5boS@hE+z^N2hF*xh z?d31i`$!Sis^5UFn=tch5r9&JZh3F5qF0sYUdvwS$hO1^G9K+b-L{Y+CMv*5G_8KK zyB=Q$+x>UcH!4|uRKx>ND3_sD_=zf8 zA+sy*cso!nQVa39R8vl=&8NO*?Tj3h9Gm#cK~!0ZDnjlb3BYpX-O{&2yr z2Ha2(%oZ0~S)vKcMloo)=(EgzqS_DtS4T#kZD$ zIHs^Is26}jg|x(h&3?BHHxN|VGyKPDZRMqyNv2x;DYcrs(?(sr?Iova%L*Je2fibf z#sab1rv9OWM*K|;%?plO#qU8;=`jF??U(1d|7%KK{g$~>s(E#UDw)~1p>*O~7CI-2 zm%`M$NewpTKb)o_TFPCD#&?;D-uoV(Cn}%x%Ziw8IO>Cii`vbA_(`FS@+g#+QWImy zkVhJNScVN@3paU^$kMls%@x0XzfpWn!&6N;y_u+r-QAbq|98nf33t{_3*jbgm8T9p&#Vo!);z&+)*oTy~5d z?0@>Ai)2gclKRb!Th2Xin8YiIC1Ze5F-Q9M?e4|0Gg`c0%bcQ(>J_$gHk+$!Mz)O{ z{V>NHK6B>3XBR@iHzueAiXUq07#=xFiNJLZJ;-xvHBO*BV+Y#vvz{JMBjvp!N8)(p z>kbLi_~~v|dWaK(`Ua)VvU7HFAi{%#AMuK^&7I5rqcna^jnttC6?2(;)7Ih>Xy?2n zwV#4WZW_4!EmN{T`2#IF4*@XhMGq#5Z$h^evL9hF2AtF95wAmNM>t#gN_1C;AR&?c z*GIZc!}dxKK(bZAbQ=Qp{5;Rk!|CJ(uJu~n7X6sjFO3dyL!;2;E+h&GlFp`0iu|aq z$S-|m{W5UIACbu3gz%Img~_n{n-AFfQ1p=LywtWmYCQz9mqSr=3cs_&evZ3#r;+eM zZ>J2EZ^f+>(zEatylbMutAE^$-kvsv<$Q5P{;%btiy-Li4%SuU~RRI5Ga5GuB4ce&)`ianM{uJW0P{9g|ug60kW>=2(Zt2Ct&R%HbJ!Q-1hb`%&eTv6u6#3e^r}S?Fmfy7<>~ zA%_netUC((OJ6%qF1}^!ZXigGKJ;bN_p3unQ)H)y@>@F0V|3Hc#UVF~Fo-mrrSrr% zr}c@20<6rg6lsT`MEQ`!aWhG-a}roSzhVORx}O`qXK2#Syjt*cpxbz7I{f~& z?s$U0#G;^do9-y$E>+ZTsMFIHrUWFtYu&JtOA4hggAxgCZehl1D0f$cS3lo6{@3J{ zLq1Wl%KHcA+8NhlD3tZYX0=39lZkO?^5l-!WwfeLhKoRbv8i>n!)F1HJ0S$ilR@-H zEg?Pe;X?Ky{rA?6vyt0ndc@o%d;S4>s1 zHRz&|a7Wow(MD)@H5ZAPH&IU@LT`JkN_jPjWG-0l0> zYzp3HRbxwd4i7C}39a6`K;d|=jKmEV*P~%uspS&yTH2?$Vr4s=S-zATJbmGA#+N2j zW%~sgsoF#H7kgEs`Ppf=`95*wpRVM=X<;uG8Wi17mUnjc>>G{BS>x*WhaDEA`QZk~ zelrL3s)ToCdBhHHZ3=+i7X)$N{WRVIczp=@hkk=!o_UX3x^i9W2xc#$&%p25AZQd; z@>C4Q{~0V4YV`j4PlNQh#Q_nu7>b`_P&4`9!1WH@PF3$zXdmh|Fb)G;JXG8 z`9GIv|5`LV5IQ`F{C^i{2mpjJb|7|m0=d$-#W{M1h(mPm9yt3t**#9g|Lr=)_nt}O z^_&m9dAtt3j8d{84!u#@%g)+esjIek_SaTf>C=ieP9A>M3M2DVRb8KZv7H`88?bt6 zX~$8m-Fdn`UVbGNuRT9v*}c4M>}b&U<-_5uRIw1O%YM=-e06zk>-bfwv&OaRpH<|G zmiMFFw`==N_x`x^b$uN5&EGKyZ(`AoIBwOXdEnEMsW3DyeD#^zWxPjiP zkfSKNsjqmuxi#3TokeH&{34V41&p<~m`aA}mwit1kXDA1d=5$AKBO-6z8T+&&P+Tm zv{77!fPh^c|Bt=54vM3B+l3)O2#^55-GV!W;O?%$Em&}OC%C)2y9M_Ry12V-aCf)e zL-PEdx8C=wuTGtRPt~dWs-A16dwP0zuIZhf?z!(g-%~>0DV_&g+rfuA8T{))AMhhsUl?06_wi_tj~f z?qGj64}J_mxpwS@h;Vf1AE}Os05~gbD77+vl#Be*9}XV&J~_AKNu3FN|GOfXw6dS`3EsVd&L5*Q4B7n5bYb|5WGkK4r6A|E1MNy$n!F<5oS zy_xwZ46K6w6~p&Ym{Z8fbLUyd_acCMwl7#;XZTFM#hWL+@5bx zhkD-p2+VZwjf=b-t5CX}Do+H)++3tkP_i6)sslDVYk;0cC}+ndK;UV0eHv4F%m;oV z9uC-_&IFW}TZKqzlsL4_+C!<2wiVEp#-)O)L_v z&isS%?Uvus9Bd{iwTO(HFTM)-&yXm5VV=kDcUg86Z6E6ne>LMjg->{57UFUDb?ILc zsum}{ARCweoS3%By@xqFM4ng}Sl1=uYG}39tsE3J+Q0tc#lqkj?_thaE+b1)$FD53 zjBs+vJs5W*z8jt2GQ+W~knM=C-mst$rp%H6T`WJ{n1H?4nC4?s!n>}E@^l)=zTln8 z8jkXOMo6QS)vF}eG$j6XB$Zynz@8MRYj6GCj%(90g~6u4yJ`41KdNcO9*k(+5MUg) zY|$(;#SgC;994#HxRut4)38Fj6HRnx&8-A_vJz&5x>NH`blGkCs)`y)E?soH@Zxb> zF^8q?1;u!$ly&Y}0&EVFd0{9X!==cSG_q_kSSOYCa8+{-c3hb}D7m6MU5xWvXgnVy5GfDRz=zQk+N$BD`atJxV}s8NDu%Gc39(pn}Vs2S(9 zsVT=w`N`WXWjR9nOoS)Naz*1_xJv#+`SRXO+O)NMsV};;H!qa-1L2Hwm zFAsu%uQg_tM4luoWm`Li8qT2Boxx5gJqJ?I+o_W$FPga!iQ11wJh_%pLzvoQPQsN$ zupgP=KxYU{WW|+wPa|1{2h-GU+(mEVkczo05G!YYKfSHoF{r>&^5e(faH0FLGxAZY ze6Rk=ut?SU$n`W@d3Wu^^!{l4;th`p&3e^c>#(7QSVB;18o1X(Y0;)32e|l^&DObQ z!do#Smn_5)#gdpB02^o`455wKP!*U%yM8dSy%RguZJhkpeo=Z`kN#-E%Xq!j)x_EX znymfY)>WO|R((Iucc`yxWwN*~3dnZOp?Xr%dEDG-V0N3#k@rKprMd7aI~V`7CQO{J z2{@6nvXuJNva>KDVoS4R&1PNgUPN(B+r%Vtdj8%8$OpLYSG3i&a{9oji<@BkeVz1e zx{--IXG|fd?l;AdlR#d{ib!l4m+v7m>|-*TPCa=WSF%2|+MRk@O2KXCqyc1f9dlhV zY?5=g@sxmxwG-5vBRz&2(z`92(@JsUt+X+V)(XxXN{U)fijIGs^|izhtjR4-UA&X1lQ8 zYISvOVuRfX6LjXD6urWm?>Dr?4C5(C_R=syLpf32S(oZ%>gUF7KBAIynDLn0Fy@qZ zg^Ff%5uG`Kg0;?D11o4fA;u~~s3?Ar$}~cUnV`ZK6SnJ#9{Fny$1=@2xGQ6S zij(ayg(Gip zL6XKfx9T@5G-e-m6Z%(=rr->DUbLJCV626-b-GZRhe(}3l~$8?A8}}-wI%Ga;~Cr| zX|P|GAegytROq<3R8gyYhS*Q4i{L7$zvpS#^Ezvh*I}5Knb|spt0{t1t4+pzo`o4y zSXGFhu_Rc`;+rZ$zEzzeD4R16sTf~j%v5E-9NSd&l&WEH*wYjQ6)WcYmo2$9aU&z1 zRC<`qq33=e*V;-uIPEI=Qq7jxs@RHHctFS_d^xy8&k>NF?Inqfo;;MXlE<8Ef@HXH zn~0js1p~~c-dOUfcKX3wfW*y!+;85@H?q!Dg3>JSFSs+ECXQ2*Z(%FghCx1LKaW3q zJCsjRarOPkKSf_YjuxtJm%2QxBqSNYgM;rbzzx#2S?d?IAZ!47fka7lCsE*s$s17_b&5 zFim98Kf9F9kaJn+A7{fhiTOgyu$HwsnIFLHD8G$A_Y-u0?6T*%r*YZDdrqv~-~uIN z6KQW?Wzur-+4gqru!`%P5sUdQer~8p1kvZB2U)7_{X+%lgyZ_?PCrJyB1_sw$j7>4 z|0}yALXI3<#q^$_D$0t^szuFgN?oh2+(g@pZ8+iuMP10NL$%8_%%pIdQ*+RGaXGlb z)AD4@;?gk=IKlMREPbyb1k5O!wY-%6o#u5`Pc$Z->y4Gn82@to?*`|)&6Q^3YK}~4 zhIxTX%k1*xo*itJNERkScI%YF6eGs!JYow3c$TgA5Xsm=ybE3x@29=5|d6+Mp{%z-&tD_sH{lj~8d_I_{`>mU!2> zjr`yyYtHYJp2hw#eE3J-i%P9pea|oz3QsQT=~K>{DHH^uM;%OQ+p|N-EpZr~{1R&t zj(i!Kf#W})Xc@JOkg??bkrR+S5j~VZNkYTg=Q~=B*bkxPXiK(}J{ZDtCu>%FLcZn@ zp`g{ppz|Acd?}uv#H7PrqR*nusE9!)Ho7Rg1a2^|P7faubVdM*66E(dc+C-=k&X&J z&51SH?FEQ2Iq#fV+OzJqJ@;v$(cwj{>*UMs`1`Z=>@O=(jh{Y){K@cft^`Hu#rD2f z9aCc-+oylzYdD7Q$&()tA&q12F@Odg@!4s49n`9OCLRkLfcjH(=q~hU>th8?=zhk> z!AjH#&begEu=-fY;NG*P(=fDkthegHN%;(&;6R;hwn%(t*r$sB8N|3PK&XzGSkAYw zjBjx4rEbMHqoTg|(uA@-V)Hnri$bhGZvLJ(#!QG0C57x9KaBhsF{L=`>kb)BBYqfY zfc(XQL@D0nt`cCEUWgB!xb42@IJu}`RauG8sZ>rr9rcMqUSjL0^@Gw{&rd-M0t8KF zlbKwh@{Z^biih+alcDz1Y$wE2#ESUD13y)b;f3+iMEke&X-qJ`3{_Z+hm?uYb*iy0 zbqOaiJ#%)VAGPSdJh)t(2G0~78AsQP+7^ZXCn;&AjkK^GFBO&p(yC0#f z7^K2hvOr$&@|vsEBNH!25Y?Cc&aa&U;}nOdGDN039bq2a!wh3|SEdQfOy!<0s$-@J zhxH2Rw~{5bBWpo@?t@DxKy5KDAkoQ{PSxeROTy&`f4wn<#_q-sK5IQZ)4tl-^R!c) z9ht;+E4cpP}kVylSzUa$M342D{xdp$}uii%$D&*P4^nuNeMP2{|T z14r&Bva~`;4JKGsbt)`|M@Z(GYkR#GOpUfq^K8o-`BR5U&~bIy$_`bdAbAyb3jWs) z0|Q6R5OnO$6P5HLs+D}@1fx&$>7BvhDkY^}UW-z8aWmJQR<>0RX}30=JQ*5atpQ z>M3u~3f<_&X67bz&V!c9s*Fgnyx%Sse}s#ZXjE9L(lKc{j$HI+mPiqS98)>;+(6xL zRY|iuUK*tWx8abT-0|8$wT82~@2#DSdLZ&W}pEXiNbz2$S;Js^Xe4UMa+d*SB-P2fp{|!3h^Cx{$D@$r#|K`j5 zy4f^XgA5FgrJJ>%vMG*)Ez$kki?YOEkv9UXi-ckJ^RGdKO|q7}|VoMp`a2D+#T$oQrO{ zbhg|XO^BsiqjMErO2P`a>Y_c2g2!T#(gc^P!SjG@&3#XShN*GwA3qtfEutso*^Ul4 z;hg37k|fndgn4D8ith*T<{xOE(=j&$ckE#un@CN?1gXboRLSGjyt2YnI_t7-GT?WQ z)e7W_eiv4u2W71{en?-=Jb|~^ClWP`N zJ|VN<}##ue!taqtFmZDxFM$l1**}XPXIqQXW5W{s{?((+LmGW zaLkDocN^ei;g5{fmlHb$H1FHowY|9Ps1*Jh)Uib%G`$*&@o8enGhZ)vbV+){M@Y4t zRmld~QhTp77p&!?u7$h9tD$*@UEdqZ8}_tt4v4Ulb<&!f(A(niklZ%1&1yNlxr(_U zDjL^gn|hkUN}%BB%o0iGOC`_Vd*aGipO=}ty`4P5TB6I=J-MD?bM#=aF7J*V)C=1@ z&Ivne`8wDtp5L?j##5}L?f71EM|w76VW{WvYphBL{j&CA!_mSaRNLC%fVVhjA4ce9 zrr=4IMJkhU3AXa9%etkBiuT}kPpKMXlgI-u?dqU!a}GX4R+rg{L#5fSWzjYn)-Fzx z&dej%`N}2)V{NqP_ljrf30yfyE}9|4O|{i=a=<+`(eR=feRD7A?QJLN^2S~?3~hBT zoD*4WrUe|0LkpRB%WmRoINuCBv4{%TucjRp$|($GM#jn3?m%rx%}TWSBObX;tc4f# z=9X$_nv4sE`^6&tHql!?=v7@lhc8U=nNMG65cm{>nB={cp0&))s!X9YQr*svD$#BUH8I~* z8mjVs`}N`lUmttTMa*HbBqG6w)lM*g-Vww#->@BI!*CwB zjRt%rm8>tK>L3q~GVIli5cbx;-A!=T^Eu0Qv}|R);u5MQIW_i5S@C+=Hk!6l+BWiN zw!1=I4r@lLzG~_m=GE7yl8LvQKrR#JhrPQ?9_g~O^kDgn6%a_JKy)=n^?QA?(X^z{ zb=Q^U2Jp(IWLUj8lrgTYP~Y8mHOL%$I|LUiz&aA+5Hs^=V)u(xYE59t4t|gFJ2eJBq~2gyUMjCzqo~^3 zrf=g8b+6&wq0@L>lJ;%*=OXyYgedd<>e`MOU+?e98R0q5S)`qJk=l0?^GBb?dStS& z7CX6*F>7|Lw0RRaDSmo(m)H#lJ9c9r&)sRlYsR57R02YDS*ixn^jUr*7M^E-J^Cyj zj4a}=WT9rM2KFIlE%HqqhvyCtDvr6%yxonZmfBeMIoPC0Asy$N}R^m&>ux z68^-;a?d&xsTRsx|BA$HvfRA)i{Iwj{Ralr4NZy4hk(I!>qT2u4%@Yho9Hi<9@I3d zHQ!BNtfy?_<%_V7X{fiwF4&uNUG?~7*WP+hNGa*pHSIlE+|xNCQmN8!e&WeA| zqk!)tk*(?E?*oTTs>*h@a~ip2uoi@Xi*IL2a)t zUDxJccHiSzK4{vc-NKDdW8#6yS&woknv&oQ!%siELpq;dY@EEchKNr&WmY`{X9y#& zsPQG9J({Jb-pz+TdEuTRxCWo_zMr?sbQ%iW%6c6$lMODf?u=y%5{O1(2AhSv+d?sy zlXsPdr)L=(PCO_ku3S;yTYWgC7x;Al=+FWAqDj0)HMyrp8c!#<7SA(YfyOvzK1!q2 zeHo0+-tUC=@D>Tsz0l@IT*g8ChE4wM^&(RZ=J%E!^R)rUr;PhjRXylr`gO;zAQ|;0 zh|MZo?+S!(;awa5#+3i7w^BP^@Gww+XQ`Io$0`Q!xK(pu#YrsSgCapWQv=wX zsuc*H8QH1fcj755!*v%<|U52yR5u>WHB#Q z32(3pQGyCl0nNk^P0U6jaC;VLcIfDEXvBHL!ga%v{+OBhh$HVUEAJh-i!k^Awe1Dv zzz@Zh2XF}U|B$uA=A5udv`ToKUDJF~> z6z~^D)c^8QkWpz&SScu=6h-fzNW@!K^o8gj{NYbT|3Ji3R@Bq(wZ`Q;oMj`j;#nu7|DzO;tD$VLU7Ve*o>E%jfeyf1tW?oasWnR3%c+^aN14S zoVS*Z=o{WnYZWim-j9+Cs986X*BFprh*I%T6rJi1`SrlB<4J~@hZ><>dZIq?g+3FB@Kff$>h2Z1>pezf!*?WqfiwIs zKfJ>^G$K2^BRLGf?gAVp3Z;3N_H6cNE|jkkQkx2dn`AAtFS!Ktt5 zKL`fvV1Zs=UVnwreoPn+DBv%QaQ=(OG`?PL7kq#4z!&_=U+epef4N}j8Otb8wDL^o zFIX!7KOA1(8$}qm76A5;4LbDme))e2|G!P)oHI;EyGI~(pBJSrur2_+)`fpvBQUT4 z*joTBxCXdgzpS}?xf1P6?YBeji3Gh+NCpa~?r=8$s5#h6{@0&Q`!AIh{?nJg>fLA% zD4KXC^ans~H(AjKqJJRb{x5&|x8L?Y@A)sWzv}%j|EI71Prn`T3Gk}-zheJK_yFMK z>qYRo@ehELIU}zPdHI9)FDxIoJW>DR{nmf+pX|iwEM4#{w*Yfx;U!2U33XuwAN&ZhJB4yWDLn5 zU#b4M%j;f~;nv~W%9HZwg?-;qnjr=`6V4t^&0iHRrxZ5OF+owBO?@s=%@gCwmxOax ziJuk|#MsUYApwWS2htwe(|}ShJvfC$ScIiHimyL<*ztvjpOzlRhclG*#cyUOuIB+N z26V%z6o=V&QXS(+D2$XzE)u|0056q$+E!%VtyE_6Dv-LDUn zzhS6~UW0njlsKsfFuudJYa%g4aPIL<(Qs_VTkvsST5zh&7vWp~G6GJAt209I>;il} zysZHwBqq~hIG5Yf@ObzR`wX|QDM%0Rglbb5f)H4etW7^^w;LPqaL|#xtl4pr2`|;& zMSTMC(IA!d%w}myJk%s%EcbyU&@?Y7FHvH<_I%x1;2SYV2U{Q~QPNd4KZ^<+AxA%} ze0iHLyPgDywS8hv1OQV;PbIqG0(@Rmh>s2gl*mz$EvXBM2zySsV7srJX@it(1yn#H z5KgG1gatC&^*|8tod`}yzzTK>m`dcnBl-8heU0t&3m3uY#(fsovw&*=)vGU&T3g!e zzGqvTL*o$qZp>heoCKoQ8u&Z`KDaad#2Z`3Bl*@T>xtV&M?xJc^@6>fW709ML}$5b z@~bwwKLG*>Lw;=|sjL>TU~yo_PGaUmPnEJEmx}vxfSV^uFOj7`^rOZn0*BsBf3oX> z45&?Kz8}_~^xjBAghl#mBWZ<*{dhU1m8ez!*b7@QEdnW+t>PK9%} zEcJOa==?b(fa9)S>a@bOy4!;2;f6Ty?Oz+`j43g{0iJ8&>`g{Z6Wz< zvggy`%G@CtmUeR0L6$qqt(Wg;>sT`@V>sFceG{$~{A-yK z)Qa>`bgxYvfrNCrjyMnkm#9HT^ggKF_?(b1=WtLk^-|rZaf*t=vs2ZhL{@qP5*q&& z7^qRGNQ*5oD6zXT@P^H!z=hwUg?}%Ts?I2#7FQJr;M7(}oZXT^j!j!~IgJwh8!4U> zmM=qZd52$U=HX>b1X52289|W@!Vw4|BZP@FXaDjVtI9&>^TMU2BE)iLVr3-wX~c{? zx?@T4>K&M5T$hBxqKqik+Pqu{A)32x2zR;freleB(tt1rJuu9FB~S-OB&JF>A3 znhw7qI4v?t>+&LA@^S1Z)d_PRC{r}D9YKojaW@b26q%p8umm~Rj`7OxO@R7>S}ER1 z+;R;F1mD$UFL9TkqxHFeW)93JeQpkw(0T@3+5_}6oS4kPCBTa(lu#AU%$^NYiEz`Ti{ziWrC2y!-@fQYmKwR1z8Dp@ zmJ0Q+nc+jd4;{E=hr)yU;P>{mHn>h4)GJToVSTMCkqqPt6{I7T>;+p=X$*7f*$;5- z2J$G$AJ>Fy-k-@qSuKPFgmXgtCG3iw2{eMZD|<l?NqZoRC{LyPv|^xUz>sWU^Z7IKPlB zu%h5uikZ#Wb7d25fcH-{5OpS1Erh2w#GqzL2^CY{Iw zGYP#9V1H~&9t$WYO+e$aj0sPIt# zS_ste#5#+W>v8W@dMj3SlI7MijNerN$uM20YK2kvA-LC83!AHrV| zKu>Cjw!&ZZPSE|Mif?bfegC*av}%V^Ue{jz+5`QypP_J}uX9nMu%Mq4ya|PfGtmio z41n@$6Prdk3pI|ZW6L84*G$h7P{&Zmm{3ky^If}<`V#8AUvs>KTJEsI&Pf_2?q8;l0~`=vw2x^Ad4#3Ish#zp z7wMq5Q9bt%e<2IP=l=QHaiNHG6>7X3x4WY@_j7$(2aK0lYPx@0dd|O~zyoum`I)D$> zfc}rco%@C(Lyn=RCjElnva#jnL4WvQW+o^CbKx17Wd6Fu!<+m!>qi}HkIH{5@+XT^ ztIEI}=bs`NF25l{A!YOHuWu-yWHq|}OYSuO=lW}g(SOhXTQKI|@;P+~$A)-^9kVe+ zIv!PfZB~J*-2=|otBu@#3 zvOcwj2sPD1b3a#Hz3_kl|4FEzm~FqU|Ej%EW#1ci4BJx>$=zYm`7B@`eH^g7U|aHO zwgH7G@L7@c62C-fA|-=F0UgPM9mjudGDM;+LM&|9V=Ht`QsQ&?PG5p5f~BOG9Ajr_=?CEEAf4`70?m3#Q` zWZ385p(~ARmUAspfZ>)Mve^2lG-3ir`52CK?H!_D`Gt=gm_;?+c``>G)zd>6Qid;E5NRDdSh9`$AbRhi%VxMlh ze>2C{vmZ4XL?qpRn)-Ta-m&v-^+{*^`Sk9t8kU$9aEQ%Fs9?prf?Bh zxbLn3e`Pn&4vh6`=Lr)(@p6-f^6 zZ}z3wxh-_?ber^b1@NMJNR`JpMQWD@cGzj@US&R-Wh`%zquU}upAR8EegDAn1gLmi z%60TxyFgDe{O2&-=VmPrFzSwm5p}av`3i8jD-lXIXMrJ~knK(I=XpEfvkR}-#K0J2 z?ZrZp{?z_jbfcOp=&Hu)_O2JfW)w+D2qmt&%|uDa?n687*Y+U~cZp3c{&r;`rgG;o zXogDfy=KTYin85qck|8!igzHZYGS@(*@Js?SKx5#@(!Q-eqJJD+b@3d|6mm zRpgyty)L607lnCRS7)%(px(2q{4XQ>@-+_x(7bN0fdj z()AH6%ie{~BpXu8mU3{Qk&#C^0VGF&3si`1k3dV%&(INoUvbQ5 zARP1BOK2#JN!a$-Ug{Kd`((Bk;o_s)RJO(_X{csxAcZe0z*E-&i;*_yq$hJ%O14pW zU9Xt;5Ps6%L?)uVsv^8HOWw#(laPSnzV=q1yZ208$aKHHq1`tXoo^`g-szi}mC@xr zoZNCd|JQ8%6RM`ig_&)^44c^vn$yBsrRzKW6U_YYJQL;D->p7UL$K2Msr;qNfoZ?E z+C*2)Bd8~JgwMQ8A+5UUDHDxDrHolp*G4SPX6EsTr2SGSSG$ zA)f2dBr;c;WS@)OWO*~c& zfv_~~_Q)&p#o3V_1sHu+Sq^(W6NZHM9}P4wwgeM2p+C^mteW`R*z-P{a5oc7ZHJh6 z+Y+bpVqDje=dcg+sx&7=G4aY^E0G&&B>uP-66M!^H{IFl$f&I-gk~-S({BG+>P>Qy z54V1?WV-x7l0tFJ!}nx~Yt+|i7>iz*DYk5N2$f(qMxQo`VZICu2?PpehyG*Z9}I}< zt;w*IZDAV!zpp5Jm zGaeblL<_efS=?@)EWGeK`Z`g9-cZoI+>%e<%i?ME*>5;m7O&8-u&)klb5?KOAJh;$ za>Gt_B#$f>>{t+Ek^= zIlL4KCZB857FdKAnlE|hsL|*?CxCo05~^uq<`eM98ep<9kN2ILD4(n11hj5IPtnha z3s=Uow%bb?e9F;93AtgNph7yyHs39>xX#CpT_a6Ev5S|ZVV88~!@!(CQZ?>@E9-=s zv|A%OOs05^DSTxVcK6g40dscORn2DlT9IC!*dT8`UdKiwto+(Kip;n!d?1wvCCxAFE~f=jurwzl%!`1~L z!Sfi=bZ%N9kTjzsEH1;8>5RP0&pf&%q@CXz<#o0Fwu#tHGC-ZTEL45REE{&>f}ZJU z{^-kF(+`h&l3*F_N7mqZiLT*iu_99cQUot`HaBJz7IUzTmFpPBe_ zC8rWn84q{{8q_>3T6{6MAD>WU8SR0+5P~S3391UIu7z8hTUSpO&S$V&G5h7HKV*FK zl^vkmrn^JeSrO1;G`g30ZOYwxTvc5|<`S@?M@?q%hk zRMb}eV7CMD`NI{0toue3>m}Pgz4n$5*lSuZy1O{C_wmc3DEHohn^L6_Y++IaYAaT2 z0@-}cEdkduCtAVeL;KKRJuav!H3`nYU^n3B&f?^TK!DFVCf#=f*t=33MU;&P86h9pfw{;pC9`pwD$qdbroT zMdRhCUY~iYP4%u%#AKi3SbSF)J786=U6uyxCPRYQ2?tKq@jdTZuBI0jcFz}3@0+(& ze*dE5@ZgJS_NdI0eM_tGlQ>;~RCBWd_yykFJIdZRPtonEGs&sE)pnB3fQ^NUs_)HGB9skW;l zoR7>@8%k*=0&(HHGoSD44X&xQgH9+o7C{yOS_nk9O9}xftpT@u5tNJjy!VATzE!8t zHrW(ydJGvw;XRv(60~b#A*O1LYM8FeI2PSYRRq`JgpXoBzXiKX+9WTI%aUT4^W=!` zu1Q7T)Vtmk|L1p5yeqqJ`9anMy|D>9Xf}2=s1qy-ykC#y_>mGY;qV`E96`M9``-jj#pU4!?I(p?3ohgjC;kBEjaTc_{W$L zq-o{E4ar_;nX%_w{cB{%U0&%g~4`4T9r#wMy^Y2N7H?!b;VF9rAybQV2CMs&th@(N>Ibj>8qD=$)1_3H`n z!b&n{A8?2(Gj0Mhda9MWk#yU|^$3}^cdT`_evRdWmp2iwZ#U!HefOmt(#rRB3paD3 z_MV`re4nJMO|L2sopaV*KP4rm=w9Tct76=#oSWR&>O0~l>o}ryne;!Y4;1akx06gc z@bM-(KGxLCjBp-)kckwWYz^fniW7gF+VTeHJXZ*uEIn?)W6a;Si$w#x1>5>*roLG* z8E&JH-n?%Hq^EB?<%_!1_C!lEHsZ}P5QYRrN+2d`>IooFD_p)UihZ7&ADME)I-RZ> zI+iITiz9U{`W}>!laXglm#@caPo4i_a}fR=WsDLGefm_y(SNk3 z4$cZD21!divJ`jI{z|?m(GhtR+Wu zf~=O7ZnTp72}MeU-=4<0YS9U3P~<==HU;q>i4o=`hw3A}g_ z=Y0+BQB*YJ7?*6q3C5e_KHU=gI5S&*(JrseR)uKP#AE=Z(h>fpeMj6F=T^d4=oz=1 zdh^*`&AgRI^zP%jX934(z&$f6)66&QPuwAjSB)>& z?@-R0tel1a^ex})8O-dKZ=IL*28CN&TJmVYnx|2h(#hLJpPxYMFhA|tK<-YK^1Ger z_3`9ZO^?0IsCqP*Bx-`W*xoAU_U(pT@@{3(AN2KSv6VY=qN~$*+)ffZZ8yhY1~HAd))OM7oisUAK?o- zoZ5q}rG+^!9=l)-6ZNI@l!SGnn@cmCZs~ofPf85bN{?IdvO{u}8N1P>=Y5S76McV= zukYMx!m`F@SqlDc%7SF68M-)iJw3Z0srI>XkDZ|E6%;(Y1h~x6d~&ttFLg@Ih@{dF zyF>RF6W!uBiK79<^t_Gp68UtizGgxXcd|7~p#`oz%{&zBo3fpCkU{iQvO3IQD^!4J z!*>@e5+4w249(znn#sVs2Ys@+FbQr5CcwSCVOG>>*|~{$xQQ^qrIZ(uZq}E>@^cmY z;H8=nXYiaUmYQrvY2)9wH^l7l~xa8M2^y{0n)q)I3zNycQLMp{5Md-j4 z*1MgaVeL%mNkVHiwRMt;4_9!%V?L3JmTI)LR<8Pm?jaUSx3pX2pfep-isoa0zX)rN zyfYy7(x!BI%eq^W0zbU8wHT994HAekYR#s(_QMOxTaY;rdj~k3YCrrTC4siLUne(s zKZ}4N6pG}ksEn5`O*QK%IBD5GxWDzVew0XyMU+Mv%YWjjUF*d#!3eO=n@`xe{z_a0 z*d==w3^%LDkCHBgZP@dVwIyJ;l;Rdg6qsnytzQSJu44pNjA?<0QEadbdWJ`~YM(~O z=x1^qWNV*vkme5H(sV#y`}f`LG2kLIsazQt%5oPmlzs${Z9h~nFi4qnjbpMzPK#!+ zkBCC(;i;=pK23_J??pl3-$WL^rF`$$9R27p2)iv1_x@4*{TvalUFuiVi^6xhDYhoL zpsQdP?x0JWw-VAT_1WH08Gvj~XRfrQ=lhM86s5S9Upa|ss%Dbc8O+MY=)qSCV0p~~ z-)UFMdiiHDN&j%0!kR*O#e3L==jj@x-3pJOBs_g$Z68towQ88%v4}6 zUHxR*JD^Nc=(CtrQkKebt+wsiPkGKyg`>s0aQQ{M^m`oqoS5zi<01FD`78s)V__RG5HjLfr-UI;rYx zpk2AC=zfF-v%kTHzMqEdnjl~@LoVO!7+)odpO6(i6EBqw-c>0lSq-`GYb?&_4Ba7% ztKaKj&_2BBrZA>^{F3m&jyr76-M2cxtGXcZYrH!9pl299kb{ z`aC=J&%&B4d(bQtno3)(Fd845-j%=}Q{}a9n`Eivk+lpiC1~n@e%y(w!d($y=maJ9 zxUGq5x4jp;P3m21e4|a_PtqYC+VQZ3mP{S0jvYwUQ6@$ca&tv%wJ>)QG#i>F<8@> z;bxla7E4od2%2+&<`}43)%-gNHD}pEs>WyVp8X>`KjpB#yeEG*KZ_RM`uY`FW z6ns93UR8At6VvH@YW-n|JoF27W_5TWOm!X6uRpeR$trp|LUE9_4~G?kQzM1q!qMv} zq2-5A5^T!#K-xAZhLg)1|Ho)+>6V9?!1?CS>uRBBbwhNRL;(Qa)S9`*#)R>1PRxU* zuHtYh+V-CUW#pf*78Iy5d)y?Yc}A$3-ZM=T9Q3=c3rPz!*F%x6zT#mB5=XHoc*EWRDhm&T5Z@*?V&X}#*JjrW&Ce@|Sn|{NooNaYX^XID zb#^wSw32%|S&u_y>a0)pD~#z!c;hMzqXJr=!_4tbVAe%*X$Ur3seNpiJdfS(b-dJ^ zn9nH$_;ilvjtY!pG89#UD8E<*mn3l5GLZ_a8-X3mW!91fH2c6hj=!TnQe1A!^dEZi z<=^z@XLjgFEQ((HqZy|ZU8i)-P7o;)tRyt4PUn53wgtn-&Bq7u`0r&vOURTXvGv2h zza#`b7!9CAN9;ieJGFu3`SwOwykqQ-t4{!}B1#YIQOJ6qkf zSA>?(tiXHAl*%Iw3ZD{zqC+))>^hHd|NEy14|l=a@VgZgezv<=h2!nvlFk%rlldnE zUsRO+zANYZ-ZSyFPynDt&r>i?8|J9#e+8)_R^D+MeaiJxK+W~0+%s>K0eOUQBfTrA zDop_~Ggs^=>G%M0c^{GE+vBdF>I+nj^-Uy|n$Dbioyh^9Ad0A}mY$3(eS2IJS1DXR z?flPP_p}N;sI>TVC_!D!M3ML(%pfuyL^i>x+OjuH48a!OSYvcdyvZm~OSRKE_s)Q& z?4|w$`IHtdx%t`d@Vm*NA0JT2%}q7Sd<~9%^~>Gp_@^04{*;Y`ZUE^lV|`P;QO!}C zNO^MYqPUmop^Odyr993-q4H$Q`f>?KxIp%Hc}bP_Z@8j1g3_H2V_@(Rx#MqSt)hyx zn&6w_>GMHrujjQi1{&GhlQlNI%SB0HJ1v(lc{*-Z8CyAm1|sySI;F;T* zBwMFzI=9s1%hy{aQZLYJeq8ESz55@%PR})ZR>3X#)EY&vj+$R8?kK_F{8_d0ta=oTWofw9qSY>H&NXMz>8rxEW@(1G7!J=&%;YD4s1&VZuV2{2mq7A7dE4bxL zAyC*;4<4pEK9Us(5j9EC}+-fQ4*zt%DKvO@g74)A9qbJ4jMdB$r!$2Ge23bnmg+h4{enIXc@+vCXK zL|Q#BlPKHBel&JfVf`XU+S`djg8ua1Dgkqey|@?Cl=Pj66Fg#uR@E8V(LSuPqRisV zi!!AOt-J2BZP~>YUc1p?W{{QkM&{Ho>~W4Elp8*sg4-sSLNqg_rOM)t<+_~AI!dp2 z_Mo0Sr9TC6s+7$+!lwL;@hF)499`ktJ;|FK{qw8Vjn=cX63kc}sno|cV`AkJzJ3_1 z2d$|;$3TSpNeIsEWXg&s()te+fQ@j05D{!!A7^ZbtSs_+&Nj;axYm?q>}9+@{4%86 zm%J&N?G)|*Ecg|CBa%$c^tLnv#pnfDDf4vg16#>F67)z$e=cSB%WE6|lSg&-2Duv2 z<-0j2h$$FN!N?Yk8{d@Q8mlzPjU_KXp0gN#U`rh^y+mz+&9DwMj#KvQT?G;??zOIp zzV{CkO^yc?`f(DX)71&B`^Gm<9Ggc=5Pe^^2={PU!@v}&}`Bn~h+bc=DpjHs%onV|EKWrfCnUMaKg{s9?e4a|doukfch^f+4 zL|6s=_pX1<=5LMd3L7|TM{_Q1i`}c+x#1T1Q&;AX9*|+|2PcFAN`Fx8U6VTW@~e4h z{$?u`;Rn$At1DGS0Gv4QlRG<&SD|~&bU9blQ221CVwtGdK+NOCnV6OEUfrqHwmmL6 zcC&?imC;ic$EX_6mR}-k$4mMA-FiF%|;= zzoSwpx_*`64)kw4B!9?;Tfeuxy+Bc^#vc^?c$wL3*B48niTr%f(t&^Aol!ms=`Kdv z<00Ria{g5{qtlk_sXFksN^q!nEsZIUrfQi$;k!0-vI9T0a5}0Vv}L`c<T93>hiTD!#&dpQPYXP-?;SL{gi=Ij|tME183 zW>+B@t-|*BSeaA~s%JMx-L6j4Syt;qZkSFfvLkQYGm5U;$UL~I#jVM;WuH)J*iQ?J z<9Pl%i+mk;-(L8>?2&ghAMGyjl1z`3e;7nl-&7M&!qYp4Z7q-JO?ziX|3y;l*ZBJT zRkv%p7yEl``jw6o^QSdvp6Q2vYqA*&jOp&n@h+_V;BT{@@RDbXrV% zuM6CcgHG`}Q4g{A+uix&na*tQvYoW*0 zS30AE2L`TlqE6LGB~@qWdX(#=z)|I(nyEKibhmn{MKS$l&jR+gqEZy+NJ42u{-uaY zaU4P?ql_HZM%Mu&W8qKpa+Gy)_Wv(hdpZR2z1XK{w8%ItYrRV&VY6afsECCMVL5o` zY)grW5+5()mJJ<=FKWYde;L}7qHuy-R)~{=vF+yJ8x8X77#D?(q>8T;KqbVV zUXXeB>VD4JmtAFeTn)JZ418tS2TAy)Na@dY9BY7iQ3Xbb(c=^u#8r!B~L1BC< z?GX*CZ#PL7+1!8k?8{g?LkkMDV=Gn_s8me9NcRP*6oASgeX3G6R28rsD5? zN}Y;+W|`OW35rwurbyfcohlrjO}&yWRhpn7?&8~ypLzUsAJ7QL~Z(_l(n4PEjE$Y^3;iSjt{`cr5wl$%-RY zNS123?~fRzSyth+aTMZkDspfX>RDnSQhGd;PTGw)pF}APhIh&3l7vVGOE=MP8p_qO zDW3JC9!#sUn=|PCd+QL9XIbq*`0mZGwg!Ftlc?FZi?Vt(AkOHYSG$NxDb|o2 zS6tlnYvflmf()$RERFVFG##N>OTDImE!0uosmlp0$vlGfpLV&bFvbgV$SE-U2Zdt) z-UDjVaTh(D`iqiHL7q2ysVTBSfl5b7@+ffhn$aIN;cv$%73lcJ>EdXT`AAa2x21(I zlTo3HsI-}ghbb?X(-&wmr8a6h-h3_@AQXf8HY!+W`+H}$YP%l3r1BRWuA1FBt7N;V z!Ky4VlSEdf%_GV)U0n(h<0v`woV1;`B{*-}<82z+WlaZa@3zX3(Z}J3Hd%43P^PJP zuthGy64RI3cs}r3efd)Ayl_{EWk?EFp~s^n+V*1GmOtRE@a4z1<*{CtNOUrYSb54B zziZigJdiO$yMoi#9*7vxdcUhUpzcFmbhbyDOpo}nlgybYaBy?Dp1xLlyP@wlNL#G< z@{pL+%{V(*L>ga`CFrylrI6m{an$3p$rej~-C2#S3cqAGZMS~1j}y`tT2)5Aj&J&0 zT(L2Je^QuL0P+-d{L*ecpU%zfFDI}9k#wA&9=8S5oCo(&g4J z2f`Cu8Q)u{rRUxSYUU+RydRg_%YmvcXPuhT-xLQ?i-Q}jS-!NVf~j@8>f(Cvnk{ji zgl)^$m&po$7wa}rWVftUY8 z7|9o!&`x=B;Whf!RS1l_jTdfWLnvkh-DDMdy;pp`sQa!8>=;TkmYo(O0 zxx}&yrq>6*AN)op>UOQorF+}c(VP3NBTaI|Q|RRT&s0}gpy&UonB1>S;NAaZ&I!xH zbcYlXN^}=a%Z{Qw7QP_&w@g~rrgK-(lrNJ}9{i~e{By+X<&*8=v^VTjb@}kicH(vC zOyq4vR46v@2cg_=A5c=l2VYrVsq>!Y9Re zo%HDTb0dJF6z+tcvc27lr-yvyoIamUBVE-wWmTP0+iswL9o0{_^dgLDeV0TccXk-3 zhb_CVMaD&Yyi{y{_dgYsdn08;Q8t;EsOI)IRMw-g)Ao3&Jn(Tn7leX;!H#XFv)hO* zFod%=26Z_mej0Tz)7B3P#;oye->bzRp~#uTe;4g=artuwsIN37v8FRaBqz3~SEqzN z&!AmHUKlN83HGHjecv@xQRxIFSUW1+lN6^6v^T~`F`xT&)#PUU{;Kt;8UT?jV@sMJ zY1ztTjFhea?yHG$9l8J8i?Q_0Sl_S2*YUv1?61kQ%1E>*5l6sR? zr8t&zcAN~)e_>2&`w|<{R=9M!CT4wvra1kuGi;yhpR4iEv2{MzzFH#GhiGy3RsQS- z`hF=U2d515+3USYw^XLl?-VDDS{A=FE)^NM5lJN4HOkEON91~cgeYgsQ`(vIj3P)` zp*_D8eI;gH38(o~e{y-ul!qHn5+h-)6E^E>}ZzTXh~5K^w*mQ{;5pSPb=w!jF>=`T*q zE3{1qqpCZF=0Kuq+8!$ywS6|DPTSBqR9ODgr^JpUsO(wop$!>MUH>znl;zFd4 zm-yseRIw?FM=s|?X_r$7+AagPsF>vSq(QAF4WWFfal^$ToK|#U1^SJVQqKB6DU+%g ztmi<@u6$q>mwwL7^5u9^k_oCqS@wV*pv$lJrrsVaEqRrR zR2;^L{)-_Hj>o(5xD+`$K|RL_x2rbIFkX|c6x9(i>u>5r9S`T_JwK@l_12`PSj9U@ zW4!v5usg$l$8w4)UFpeOD?K+jJCUz-QL2C}Hv3*G^8EcndeD}u_2UN7(^q?oTm>KK z{VgPU#hj7TY85@1rX>Fj znlT`kZW%85R3@|%Z8paorLFUOJNY+XNQc9XJu6>!kfL;s1}zJnv}B~WyJ#Zm=TUqz z(sq>ZEY@}FeB2(_ihuCiZ5i-RwDg9n7`6Dd4{r|FT24IdaJ{ZPDgF1GN@$;IJ?ViA zmq|}_Kf7Y5%LHX9t%QKp(>~6>sQoK*=$pDu(wX1YkMuW2MNL5Jq(V)JK)J-JT}rvJ zInp&fLFGOunU@{Q)zgWhH9Tt{{Wu?Ve!Vd=Y8ZE2^EO+??Lp?GEz{O~Si1?8F4#=E zh6ubuw(-K?K7N4911eP+%~?*xIK~p4UQsgmV=^!2y=^Z(?;nj#XVtHS%xVm)XvxbL z>5?i{!Kg$O9?7If>)9oXC|l+SF-*VyfGl}8tqhLo*Y{T38zpwd2Sg#<%Y$-YJc&Ma zCC~qKam!^V$RKf(iXnmMT#$Qv6`Doh!TQ>J)}d-JCoWJ(l4}XMDCDE`enOJzPlX2% zX+H*gs$B{jKus&p9pqMw>Bq10_a2aa`d6!D>D+kTYl^%($7~wm_S>^NZV|bTi7?Gf zo5MHxp{j5BlrX=QFj$> z>`LkJY+R4#-U3jr$A&69o=wmT!vR(Cq@4(?pyk)d@L&7(FEdUns!{g+4-JOb1T25H z1bu>_uscdZnc`6eh^ITvAQ98X8wO{rInr`@`Z zet6`z*UZBC`yj6_`FlU-`&p*`vaL_E+-?oo-ws^)#~;|G+zcNeDN+OjQhN48ol~-t z-!9A3ERk~CHLOXc_y3AP`_oN%8lOYwMpWmv$8$34;?@HjbXii|i7 zxKW#;w~geCr{MVQs7UvkjrIEzdHaB*yJu^rKb_Lsd+Ubke!uZ_Dv;=1-TxWN`Wvwf ziu(N~_)mJwUt6bRM!8|N@<(*PeVa7*Ei{?dpA33-D%7j!5yRzjPn$CO4g|GXJmEe) z>p-s(zwg4{5>6g{z9-!!b_da91Xe~-OD<6U_y zRYQ8xr6zGZB_b$>lcwE9P7v9;zsbl@PI(-NVfB4uagm5fJ&tn{XBihhqNuSqj74M5 zE3EZpBa~BoP%*^abJOQf^z%!fQpT0LUX}GV*82EjC&L+KCeN`B9mE*Caie{^G>Y*%J){`E8ykGdczo;i{L$bQy(&?~s9|_= zv(Cm~X}Pmspl>Uld|4@<^C^rp(Xn!;zR@qX$J?mYm*r7x5c2S+a4MC42-oDBnu`jf zf9?jgLci2Kr9YwHs2y>!EZ=fYVc{V-nC4y9&s zq<>XWctz_g??{^cH`1pZu2He&nzyC1GpZWSu!uK}^7<+c_qr*1Mcldu9sh7w_t!M6 zA1#NP!=AHBN*EnI^=B3)=g~=w*^fQ1OFB{L@DFykQgwc@ z8sBviPei1|fV=days`&=58WR*e`84wR69PDDg6(w*5{t{M!eVZ2qK{rT*f2xUsokCGVGq6jJ@i zr3dS)0q}T86?%S907K=d&OKC4^*<9Cg_58@)ZR$#@siWNZm!BVk`_M*dlFR68UCi7 zAOSz4@xR#_$_cttMow9pKVD=xvwt;DV)B4*lnv!tCAQOs9{#2sfL1|CAAk8+Qtjw- zpXKF4Q36(QvGLTIDgwP~xJAfm+U*)s_C4}rWjWL~i?+n+aVhjDuJSbF+@Azf1>S!} zG_xYD3_F(N^So`zBt2YIM{Mk;Os-=!ymJ1nutRBF2>*Mn2uX$KfVx$8L!|E~EOGOGVya#HC%#D?T7%q=C-Z>Zjz94GDTM#&A>{!^ z0nPgY{I}55@QFU`&y9+-95N!Www6R_m41;UBsN8OmiNa|wzr&8B4!XM0hLvU+-FP5 z=vN=1)QrKm{)G@9MS4WtQ5%Q4qoBWXeQ>vd_0s`-wAkF17e)%;%^Iuy9;eAUC#ha_ zcxCS2#d5c6QHqy|=@b8ML2Wr+$#DLzk;?^bzfS17fs>z>hTWd1htYjmQl_95@m{mZ zz&GfX!*~8$?t59NV^TH0wMMEq;Ty3+TP&cR8|coM&B!UZZa2B?I+=OD!cXPkh_FnB z{3xLE2g<(m(9;R0DkOEO<Q7P(R3PgOg{!4? z1=VvK&ub>k>2P_9=@{d2TZv97k|*&bM~LW??{ogxmTgFRwv_+)NvrE)bL>_-g&i~` zi*Hn9)yt0{=kYQ*Ez>m7{~#h?MS#v}oc%=_R@JmjEcGE9_S4aiS;kkK52o<#vLO^q z{t^`1Uvw=EEg#3q+^4($L0(9p>Y)Dh!SnSx+UWf4AA|E2M(>UZMcKbWxdepUVyPS3m4A^Ng6B zIpEVN4gUIN!i^9IKt;ydmOWF+hEYPEMC$Px`0|1GE%(ur$ty4U->&*Is3juFFLD;) z%j2b1vW)D7Gy5)kIX{jhI#u!`lRcl-`>x8GX1)R*KYm)N$AwM&O)0i$maWOw$ECZ% zr~jSCrK~#TfSj3Ud0a{&N!dF|YFkv-NI#Q$FAvDOayt5Kd$r6@K6As2qQd$MO2xF> z^}!mVgboFDS;*TV8YuAhcO@v*&5g5%wTRp&!A ztrW*obAj1=D=+b1>3F9;dWJ-BTkodXEor(o0U(FCArUo)Kb)gWBekou~kb!A-WDZhx&_|~#{c7&5~=A0@iu1Q?wa)qHe`E^I3 zz3oLc)r5vlz$qB1-fzcm z7hev=zgH9P90-_RgW7cb9pC#ZnNETuZCI@OByq{9nF;mzFbw%E-!F$M_8gST?%tu^ z_}rDD5QVr#kaT-r@b1so@s}Qu*WuanCA-kmgY7of>Zcrm5Qx%V!nkx!ek0v0c`*tE z=hwWIM77jo&Bp6_%=vMjv2i>kSLaQAhrV5Oh99!i z_P#9qw@_qWc;bQz|9F$E9eK|B?`!;{Li%n#%VcsC4ZG+puB4?mO4oIxo{ow1^H7Fg ze?Vr|Cx*6dFGm@>B41dw(sFKSR4KZD(v;#Di@mmNI37|4&Dlso-K<$o?i{G=Q9H35 z7G}|77`oSX7m#x#{Rkg(7ogYx literal 0 HcmV?d00001 diff --git a/third_party/icu/data/icu_conversion_data.c.gz.ad b/third_party/icu/data/icu_conversion_data.c.gz.ad new file mode 100644 index 0000000000000000000000000000000000000000..d5abb06b8ca21e1e6116ef1732c661c815b1489a GIT binary patch literal 177696 zcmZs>1yGz@w5{8?ySux)OGA(VfyOn^IKf?lyIb%Cg1eJIaCc9F2X}YZ+hp&3?>Y5e zeY3`|)zuWM*P3(8MKB*}hV;~_&QW3>_;Q-?CdXlMR*gElg+~=s(}{R+ZOEgQ8;o&i z(6AbVwkDmXA;gGk+UvP$ zHbtq@@-?a{v+!N2e}>(?MPkM%ezXz$IZSWKKK#DU&|5SAqG`W~KvAQyA)$${*@WG3 z>C#z=e_$Q{!sFWUD2YqkQI^prv78#)(SF3ikD=V7`KY2C!tjYhWv4r6&dB}f$w{|r zTHN2Ch_;H{3trJ|vX%^c<(;weTaA_PM-T1>QTsx(h~ju!w@1vBFVPqwcy~wA+_16+ znlA=zXP#^9_d?GMica6(*lb-(`;oR<|B&qt4LX6dYYj_l3oVMjLm_gh(|u$U0y#f7 zDSCBO+ut3OB+ZT2O+CAoiBZzDQvP^%6$II|b2^;beWORBes zl2^Iktt$xLf$KkEoYJSBt>`k`eH{N54KFh9AaTk;Na9dnQ|coCC1d@^u~=_wb}_p? z{R%9_vGK_di}l39T@tVL2H=9AmsTeU}JYi(p- z&e!v}@KP&}7LZKDY-`VNiVEXRlZHR$V>9z_5{w{~j6P${xmQ{dty%ta>lV0D_fU$A4kqujTBcq~B-PZ=S`-kf-wCKWkg+pPOLGi7c`jK2yP&R$; zYL2C%wO3Z9==Bb!xBkk7Ty#0P@QS;6k@NC3@s{tm&aJ-Oo951qQnv2KsOnAAmz;*n zcydtZ+RABGIoDjaRK`K_Lb;7Bkn1FV_%3065($-IPN}M#gh_Y$tJIpoAL*{{T=!dF z+xlj9i7nDGQsQaggIfAh6U<=geHsTsC38Y6E{~xuA-DLlU-JPIQ^S*)-e=i3#Xp11 zo5^pLII^Ot0cXn|H`=t%v&_Mt1f*N{4G|&sbzTHkKXGY(Z z4b`Z|ZFTdYGkcR;O{!(_gBn=*l>t91)pie1#wXwYuHE+5)u_;!z0~IDMu$!&Xh7jR z0hVknLO9R@91BX%7niDV^QRCwZ!S*bT4fnNHJY?%Inm%}p(lzBPl=|I7139|A3At{ zZoU!$=aV*9H0H!!Qd}ilXX;;e!ehw8Q=NU+@~Qr!yfxBRKVkprZDp7vFZ;vPnDrPh z+nTa!OtCeCd+`0SKQrcezh~a%SaFexS9DO)#Ql>dwEDxiN99iC`?!x;i~?Dx_6$@` zFJscy3wT=^`Ol!{gpfSx?DekX`Sq_-etuV^Ghc_!Ee=lr}5p9*z zJW6@pKAb1slJy(=7)_GlxI-si<+=TXcHW>t!n0U^k(+w z_mA9Ek3T3tHIlB{x*|J6Z_vcS9GFQDN_HsS&iY2&Q7|)v`aG!0mU`RYo}lh{G6Da| z=*u2-M}$tTc8AG>X5GRQ46~|BT=y5|J)uu6qJ}{)&&P+$;*@U-^Y+|J?i>a^(-6!G zo{a^=S&?W}l=l=&KmE9P56n@SmpOT6`0$`@SrY#wBA3v+khHp|k|k!_;vqcPNla|+ z-b0X1BR0V^G+p=xXN6#L#Z|Nd;&j>l9P+5ofC)&JWAuMV(+nin=qGl<2Io z+C9;$BUL-?{Bg#^!oqFTKAw>!-)Sd0wdyew7}})o#W;3E&oehdoHpRXXFC6YBG;eI z%b0p47-p^n|$Je*yIYZ`kE|H%=QWE9vyTHdGRa5W0~n6P83psx1SNT!OWaDhiOCZw7aBk!ipW421eXM zA$U&>>+p{%Y+!LgD7g0^cY5W_W0|m4w7s69{@3SL z_H)j{DTl|yW?FGkqV58gyP6GJX0Sj_rrc~&b1^s|^Gj*eQp=~e7i(!gnti46F=IYg zE+biI8W$tuc=}hh?Ng2HTAK#L(fj>(8Y8tkj*+0at|>F8Dv*rgpJ>hp&$?&ExGWQI z)oG@suffQ~BbIeO`^IY{fvsAMm1=1oFCs|B=9(Wm8ViqVP<@9+v867i)%;m4HOHB} zi$*-IX;BoAg(#^0U9j4&Iv$IrCO@ zZO7MYO{4W``SGJPZ<_-*pxtQDH8wd*Zp-D1gVV{%VN%|kQx84_2GgdWsjknDHbXsn z^D$>(=n4H^?I{C^7k%)aNX988x5XPt*AHCEL;~JMwTsqH7r`Jq&Ciix+6t=ozmP5w zrbW$lW2i|lTw}i3+0k8@H-2#{Py4n;qla|bfdkzIZv;szG?!E^26fWqx#RCc^d4TKmi1ymxvLGGa^oWkm0O{A%M+u`k3hRP57x ziQTXKfgzvdzaf^0c>WQ6WQe~T(VM3{^JKL7dS>$cXw@9%BR94@teC`i>N6FFPUi1r zzZ6w1OA<>S%Go~py+FOKFP>aJqJ_>_;#rR0O=cv;(ms;YU^;}MJG_dzKtP=c)5Tr`|(z7qWCy^Cvg9u}oBbKn8TP|7)xCHS=U zgT!J+{6XgA=lsL^$Kvqw$cu&Y8I1z&sM12lJ+TN?V%QiN;F_N!UPcu)#Y8~40XTu#hbl5pVMtv#1EWO7Oj^(R2xJ52w6xIm!g zmpMtgrVx308NpyYePoyx-s19R&wL&~Zkje{Q27&WMn&KnVO?t6LdQRb%$+4>=!0E% z{~UD8aX?-bE;SpjC~n-L)%hN$wYgDn>3qYqcJhMa*CGxRe#fdoBG&z8Zno683QEW@ z0v9LeaPDJ6ThM0vpM@aj+%nEz7#0s5=K)dsOQa@sH_w!bGL}2vR6gLnmGDh1@7p?R zyhQpsFeIl{AV-*^5;KO0ku-I>ga zzl01rj8K&lzm;to6!=x$Y6iD1A<)1=TM|>4P-4gmz1nT{f zT)(Z*YvORJUB07tBT$4!6p4U){+m;w(EV#(`NT}hy}`Cq%XaA?jjuxJbCt%6&xhk3 ziXpOZ98Q}I_mvLb?+Q`~EDFPLJyu%n<2A2u!eq+7^Eyb`(p5A4^uBYYXS2`<|sXi}g<0kx-JSlT2-{nRGAj%gkw8 zM7XrGz7zjM&cnZ(N?X5Q^X*fQ)I+*yGb@?Uy4b zv2t9o3}Wq3$Z&F|K70ug!1_@n93a$$^}?a60m?@(->-<(c4e>f(NS?zs0+}2r=|UpU z7I<&<2qN5k*a~S}bmg%H=dfOinx*HH2+G49GV&oX!3<^CdHc@4+D z`%{b!AS567C1MSqd_$f;@Yh6AJ$>HGQ5m7T%hn_DLhAT%>Ljw&s+_noxqg1_yy!Eu zcH^MK`uqs>^O-K!WFBMlL{zNRZY8>5B`>qv@M(H?;Lf7OqO4Vh4&{tPmP<*C{HGN2 z)A^V3%8IrQlQ&NsVw5%p715&fR^eAJ3&U+aNoT|%j|UMzUin;J)Om`_7}}uw$hl@^&=ZQ9C^@N|1nwcr`@SYQSCPNbLYzs za?snOhY!r78?qcz)-N9r=8JgD2s1eAkDZ?HLX5;9+p4Yw0CU*5s zk;aohYcWoKu_OomXx{nQa97`y;-m4?F6YH=Pp^TA%zQ#$YHMP--mZFN6KBb4<@D#d zi|_U|RLbw1q+Stu>tkPbk(OTkd$$!ytQm9%LrVGe0qN%-fU`n zL@ZPM(L^BdW0~9?7hYk9{Lu@}qOkz((L$Gf(n58m_%8{goO-W$0`T*~Rr!TYM#p&a zo3_P9cQNs87vJA>dnV1BUUv(TIwz>_ANo_E+%`Tp;a7Fv)xX%CRVk$IA}@}M5iC|k zUcrAP_*|9YHdbM?+JalN;`*+W4{8>+-GIk&&7;c=f0U@w=KJzp2JmF6^K&S#=>O0mtQQ4AI)L*>s8B{_qHB@Q)j-?Ihe}<)p_H>zCsEbr~r_cRF+is z3F<0V0Fbjn?(9PWP?1pK;iH*OW6|E6aX4_n@f&Q1js|C7p`{A&)Qv_s`&^G2@X!o_ z8Dk0njj!}l<5S{qYXNj#fr7wR)G1EpGmaca96hvCNm(ZK{fpl={sNii#~hd7>FSsq z?x21|Vj8O$Fh5Hyc=2r@kz~pW$*f;;!%vr5cv9?hp&wqi2cOByrzi)Y?bY9BVHLDm z`TydS-YQAY@N=K#`ob*t!OW+&>>r$$*Y zl@7?BmGjAEY2u6jjEjWwwc>2w4jfg1O72E>$4$Oj2dz^q{Sa6}{@UmpJDs5>+d2E~ z`_qyue1=(?zzl#}bR$`mnE9U|j z>|5)({#0Fkq3(PXbBRTANTDE}9iN5*sRU`HF(wH3{V|nLk5R|TuIqaCfHs9THDSU= zRF5XA;YS*UpoGp3lNi{E3ivwXl>Otk1o>zTEzhS>d1p9Di{mP~uhd6apiWSF`2}8PA1fuRSW)LbK4(h0_l^wytZsSN=P5KmN|TnX8qpBg4qlx^ zsg%20^)@CY7d=dnP}RI7g|9TGhPxY|h7i?_a|-{mKLTFbA*KTE!zmu^c*#J5{iTd5 zP}(Iw3z0^V%CG>_TfD)(rg;x`N>5(Y5b$V}E&-eg{(+q>+z+?C(MaP4PAl*Q&frO1fRyV_0ouu*U4Hgaj z$j|YUEroV%k{ixX=+uLcigme%LCsLIyGxWV!J}y00)r0+C7{Ib6r<2M*%-~)nIbHn zlCyL2@boZAfkXYo&f0Z5k6&E61sO19TL>LeGn8i2k(DA{dIULG2dJGvgb1NdAi|vx zC*^-7=+-ioN?jWubpqc+vT2{t0-Q0z_V{pL^R1f4Vi*LcI7Ex1S+FA9919t#mpx85c}fw6T&_@Uh|Hk(N~` zHlj}ehg;x}-}AT}0nJkGnU@P`l@gT;t=TnRzueh0F7?GTj5I51tyKheZj0+_CT6b4 zqdNuASQCYYiPgjy7A}gdD`5V-ur@*tgIKGD^V6`rbqp-KmhRLrBCR^j5IHLU+Yb|a zyEKe`pI3{K{Dh%kE#X~YTsGACIQjY2V1UuF8Y3pHOKbV9f-mC+DB7}sJAb-eQQ$6; zg31zp5kRz1OcuL0JLkHG9LARc(V+Y4nK~lp61LQzEBzq8bAsl1&(K8M8b3nQp8+z{VTT7)tykrq?rA8;7pAFgoz&;A;d{i zM>uOfLtU8RmOeDbCFmXfbiywT)Smz=H{X*xqS`puNJ*0~Q!z0`Hv>qx49bV$dn|%v zf*00KIlY0ZGhK-|kK7+NT;+tEZY`x|9c7j&?qLm)pumvt;4V(FZZL`KbHw{mc{gxQ zel)|SJI-ZYFtHQ%NqHP0&mH}y)Q`Js zU5oseRctli=J_{nM$-|XZ?zh`lz#N%`H+sOX?_4P zAod}gdC2$KH&Wu*KY%NkE7-{q79D+^RFq$qGbbFis0-!#U86QwI*O=&5Xe#h zTff5U48w_=hW;sdMh#ORM6^lQ*RN_zdTik+fO7Q{Y#DZM<;Wv_;S{sd{-!Ya@#7?) zx*+2Oej5VTgbn7ElQpbepBI)9-8B>d0U&4iAVX5htx;mviyq^Px#D|jP=RI{@5&xN zMr1n86^{NbauF9sovHh}k@KgF6al#LeIqFcf?KitlK|)2@-&H*xjBS%k^HjAhJ(gGttd03}@tR(%BB|u6MRGP%Y+9 zAX%^N1J(s{A?3Sr%Vmu^Hle1CFwf*5bf*79CJUl13cx37w8;w0-EI(ONyxEXQ&4wq zVt#iSpJ=8f;bLeZn=D-@Is?v}GYUI0pCi-sr=NvZg;F)1jeuMlzTV43woq6wO+<-c zwxMiDHMSvYfP5Pw_WvtUS+*F1u}r)GfxjU1(aEy`ZIDb9LHa?9*q5m}<*-0sI(bH* z6cVv8u4f4&P#o$0El?5(Ss?)w6sCr$`kjUYkp`q5l-XXOD6+`1QRUpIBU|s@s4JV| z-l(Ugyd5G)*foh*pyv|hCYR!#+!!M;tQjIJkxX~2BsnTJ=w&@9IC--}W%HIxsONf6 zag@kijwUnH*n@~YZOxhJIfKWRy=R;IRZ1pPcnsKjrnwDwGmR}n_ASR}y<#tT(bWJ8 zPOdON3G@2)h+G7Y0*l(fLg=4wkzyJXC)0>-o#MU00x5mVB2|r{TY`59xHhc3O)L1O+ zH!%l(F<`jDuN|W1*Oj^>9q4#F4435+vA&(cu?8v(_Wk73rCX5Bq!6-KJFW@&e}j50 zqZJ57v)wQ!)N|#1SsP<<&%eKvL3zZgY|m2n>-FkHPxS?%^Gq6t-P(3#@j4o~nqW0c z)SvXoQY|P$x8y>E#tAN+?07Ie;w^0RGKJ;3!N66Iq%0>CCM*R660-(bkkj+w`ZPBL8kFcPMhV9t^-chu<({m0!M; z8?xvy(|phetX9bTjQ3kAl3qn((aX;11moLG5@PdaQCja&VM*|k|3qO19wF_^Zj(D6 zLJzPz9tN?F8}s^GvQes56mL1bDG!Y#yHB1}rd1^6$k<-ESmfQdf=um0T^<;E=GdkkkDi8rSoSlN}mk}G#F*M^em@zrBF0!4Eln<~=isNyB z!qrvK4s3!cLqGbc4a0Vj=vdq+T-f3Tc@BN0zn^O>gc8Y5Zsyzr4Ksf zeS$6M0}jVq?B`7I863XR>MERZvS*(E^H&X{zM%sN*2V*sS@Y2m+Fj-_{XWO?TU)&B z!k6*QnxI*@Y7_&4Ms~+n)WGG0Hh3=;>)D!*-=h4r&L)_j?&>z$4avi>eOJ@^t8r#(^RddA^+YrdWMEVi}X_&0dvT{>N0a1AV= zmlh`R%wU-iXkj0|$LBf$9ndQ77N1tG6GE{j@eq&7#*%iEP<7wlV=7hT$~`4kYhbYGJ6( z{}nob*)!Mx$_l^>S)|bd@OPjt?IwQ!m*KrgHtm4R3E#HnBCE2j621Y)@iz|_65(}8 z%*SZfD3q z$HzrwK6$9_%D7lR!a&5wRTy?r6UoBoL`T;=s$JjF(&-U6z|C-^g2etHbGqn>vrn-g z&VI^%>P*i6)>m1h9Q%FUx$bpzCq^JeGegn@Yy};~M^Db#E@AtRiUt*3eZfSr}#j(K&GjdvsfY-xr5U{V*R3juZc7+S7R1&NHy z_TrOGfrY$q7DW~nfq|1IXfb`mq6uD<7f0MSi9|pP*hh8f(MKBF?*P{SpevfxCB^?1 zkjq=txsLxjUZ4Vct+kCai+??mjN(B^DV-t)Q112aYjYMmn}jo?bj%qCqm6&(+g`xN zsN0X7_?&LyTxII?C3I@E<)`fE%Ejo578p4K*CMx&> zd3!?x3YbZA5fYaqG(MrEuf*WPjlS)bjpdoZ6aMa8P_n^|+Z>Oapou3aaTe%8%6Nf5 z=>nodFX9bi{^7=MhHb+A1c%-{LE8;WWXWB^jl>Bg$p~Sx+nNiky*OVOt_?K}?;jsU z3tL27&LlC_7vW5bOC~{W>Pi@Lh>R_&M12!YhA4?nxDilGO2SwjTQGt6Mj3L&^?C~; zEN5;eSVG1zIQedEdjl0})kTK&!N(&B_Xk(|ncM+I=oOFRdF@c_TwA!Jt!F_bv$dy7gtR5U(Vfs;&W_T+ z#t?@P8-E%$=oQz|fD}bAHz!fL0Sj~7k}BAP@~n#o!-8Bev%vT_+EOwW>5B<0Q3Tv5 zT)=-6LgfHzYpmb`@B+r!9o>O&z*req(MUWcJf*n1pYy-*M#n8Hxu6jg&1qjV;dLJ} z%UTi|5qRB208l(IoGEaPuxzNh}3CoP=&qsY+YJgqha4Qxq zEPVtoFjFqXRl=cQj@l;e&&g-&u<1FTbU4o7pJ1Dj=BVh++M@&8g1KCJ0Mr)eqZK_7C#41fY~ zN*1LmOJ5^?i9P%@$Q+4E9Dl0&QFx3cJ%))85fwI}ex?%%>l${&j7_xLT;!3bPD%Lj z8^TM>^;C7cGe%3hJIsahG zkj_kjC@b0Y!J76Ecd z>uD2x71HA^M~-lBp~aCkL-d;kC|b&*ZoB}bf5AUU6`VmG><7gn@TBwH@Z7kd1E}VC zf_j38nav%`ZV^_y5qm31?YA6R!u5p~hu5^?i1#?lX*{Rbw9q}A8hRq^=Ob9uAgGd zWhtHF%2kT0aUizjC%mOTq9=pWiBk^ynwl|_83b1hPg9U2$~oOPv~xl!I5;?mFsTyc zh>c5Aw82Rhz9-n6mhYi-3>yXoZw1Mne+A$uMd+B5L4nAoEI?oExI$~u&EhlBsDl^) z&Z@Q|S%Mj>t#{N6!Wth0wb{seaa|n_CL>%Z%RgiToF;*l@IMH+%(2PQ+3{7Y&a_*#tr$ zvQ^H{t+Ka4!~;>sex2>FHC)(!|~zCUF{1I5R)*Owm~qWBkLun6^jNO z{;{!&a1U&j4*+u(bS=uo6jWo4ve1Nj{*LhFN(Yp4@_S_XjMu>-R1Z_akQ|vige8qr%Y(%O zqc|MWyIInfM8hApAR?4!L_=%QNlsJ3CuQsoud_q-dlTBaj)m{-;X8y$<#-u7x9jM! zky+(fnR-`9cHtVh2Q96oPe4dOnC$NUmBrktgRSw`U`m~ihMHvuqUe5u8o^C==l}#u z#EHLtT5yl+pGNwse!gJxphe`4U@zywJD8#$Xq6Tut%zInn6#L*Gd9loK`xGlEVJ+d zU2?n%|Gx1G$^&g^dP0szaRtVP9o8==guP8IBY^9iN-ql0XL>- zK1f6t@Pw2huynNYR6vx(b*5!3agGB0_fUi5Dz;C(%PQnHomDEf554UwwvS%U6D+T` zM@1K49UPIM5Tx}hf~}416YFhHJv@-r+-^xKD?_I@DBjZ$3eis9jTzUHsnHpwUJKdW z?B5{Qg#u9X8c&8~u*Gz_X8LpnFNj#bg$Lk)tmfNt>9L~krHv!>^U|G9dH zt$LSE?V*)OV$&gy|FkcRPp>Jz!Q`xtml8hD0sR}U$iomflUULT$E~m5hYXape5g2UQ>yV7|E16vH0b z#qgkcGqNibz#X9FMr*AlYE>z2UB(7H&tNLT`NCV_;$=?7SQC0Ikf zxN!xX({!!Mhi~3*P^2Tfumd2}M!-dc3gH8^{sUjNm9h)v6YeK3I4~R-1_BR&my$nh zP+Z0KvR7Hf_Nv!VMXWstxPWg>OE~vj_nC+ls%7c%{U73df_sKg8g-2xDmyMLnJmO$ zc|vhQsB^Pq#!xlZ{Zt?~(&x96O6Rx83GagJu|tw_NQwGT$v>@9A(3Nm1#iQE3BiQ6 zIi$#F5tQAqSj=$6Ak#uftV%K^V`U6b4Nz-*02PKL1T9s8TnfR(EOR@^_|H;fDN>>{x@(`CP0Ccj8l3CK4WCF57E?&iyHC{QjA-hPD$;$S*3{X zA)T#x*zAW6)eM*VD=ASO&N4jf8o~Jkb>GZ}iWoMQo|&4Gcjse|g2=HTAN({!8XXcp>hvojEyZkahz^!bH&G-}ZAiE()TJQf^Q!gk0`W zd3Lu35L_aI_=d6lx{*0-3YG?^Hm5+*J{^El&N40*i%)j5_9d8{a31J%9Bm&2-o_g} zdILvyYg3{p+kAiyhRq20;D{?}%`IkrThV2?!9wZ=rNg_9dPQx+Gq(>iWNe$O@MOfd zH0zeb9I%mj2@5(Ncersy3u>KwQemzzoTs*rLF zzpOoZ@|;m)!|YhZ>w+#>424d-CovcbVZ)qRncO#*SSgoy6t)v+LQ32JD!)XS&{j9c z2J}WJ(jWA2i|u7|4U~}j6`Rtfn$o?*IP%m>b3yz?ufaVq^o65Kn~6d_2#VmR%~KO_ir`OCw) zj$4;g;C~Q7U+}rQK#b9MwJoSEBUZs(OWccW;peHwE=2rkf8{4Kj{$qnR^D(AI%ObL zAQj$Bq&iz5WgunkjM_5yAxF>ieu6Dnw2A%~f zE|<(HY#h?JT+0xS39g05_erE8f6;*v`8M!`oYzep!Er1RkP8_cH^l1TTAOW$@porK zF=e=qTyx+sG>d-)_ar0Pn&CS9QI|2xsip(oO)ZP3ImiZdKnnVH8VK}4`h^lH+oTMs7Nz}=?iDe?LC`p~ zjO+xOMhZ3ctA<3!DQggcBgvfYB2EVTqY|C-4bDQ02_LjFz7nlz%U}WAvSvsqS;KN} z+At}9Bi$r5Lopkirqm8r_?rCMYi($p7{Y&SbCf!C z=PD;=0zublzw|dn>#W}2cFN*KnJVNqZ(7K%MJ<16q1rP6wEumdnVyk z>UFZ;s>x{Tn?Eyk)u2!absMpZzGCP}rq9RRg7lCGH?lR?ma}3XM%p|f&JmO>Jxaxan7CIx{fO9!+|!$e>qyx>sua@l-F^6 z>XpZ)c^g9;C=|%4383Yq)e?c?gqj5W^=DSR`97>d>`tN>qL>UObIk4ZZmb-{{+=DU zPQ!HX_ry@NRDhq@6}=LiDLSI0e0@G+^xJe$5_VjV*Y|Xuef5E_D12@c)xqQlF5d*F6G%Hwr*RJ|?(VP{Pruj!~H{ zmg z!k`DL9CxFF^o)D)WDSg4@!GzJ-PLz{vc#n6A&*zNGQ)Xsnkg`iCW!=cy;8f2)tK6u znaOg%j{bga67N;6YMU9HAUVs$n06aCr{(tNE>AObz$v1vF;-uq^bgc+eU+DgHkPCZ zVI>-`y)W+u(;;FOsb@&R8OPd@4EULw;-!QBu0gL&IWgHu{I}kYR>*3ej@HQPq^Ql^ zIV|Y$cQAyx^SHfwKj@IhS9C|o9~$8|%;P{7LA*G$8976+s`MqQeA5o84@@25n>S{~`9;J-%;iQ>AKCy|xGg znbFF4OMc}jWRRoC=v}jDd;q2ZeKE6uuBqb;4!MGgYlCc=^`ZCFPTTl#(^9di*wg-* zCTyXQ_~zp0vEi3lO396m8*GbKS5JzdKuYR$BwFk_@~Dp;k4jXFlV#0*(bum0RZwR; zHLj98+o?$YU%!up2^fw~=mY{gYV@l~uljkbb z8uli{Uot}qOxRxx=!uT%wLd-czBfDEXM`au_8lmmtPgp=8vzbqDlHrFK_T8da&pbe z7wh#|=&A?=We)S93xH$fl56IxREAH;Kb}&}Xfd?vhT}8c$_XLYbIR zc(Q-xgc}lnl(nr1UBG;->4GP*{T~wgJU=PI{LxDK1+ifc9F2B8J1J*5O~476b2w?~ zIr#yBjDejd$ghRUnIyc+w%@j&g>@5vGFa#@T+lD^bBcgKmpgbEb^Q}Gi9sy1TqraH-Def=8Uf}k-2WL-kBER# zIOcE31oRm0a$8AKe*-(~3koDsRFMC7&;wxdxXfTEUlUNf5Wf>-~)$IC1aj+%S^r)4NZTRA{v5;;f^o!MI-5uqTn4vcR>X^dl58! zp&UFPbaZ}i;?L zkQdD1nJ|~F8P@-I$OvKY#4U!x2GFR5YqA4GHTqX`&oBbu@~~)*PA6UszX|k}!_zT{ z)Ql+4EIkHN@dN%t>KAs#4hAY#oidxaL&Ox7NDu&;{tw^xc65ruxHqIyb1=Lh@cjQV0p}lF3)l?JVogpeApL&Vi${2uZ?^q zyIwB%D^~qe1t_|ZlrBBUR6V&6lxz+?X~5RkM2^tN1--kVmN^d&$YdLA@&NSVX34~u zY~Z@_>@KmQo8ZsZQRttNcvuRyE9vv|ejuht$=O$Q-;N z9{8A|VR(<8_k17qr?Y$A8AX`LB4zVDnvh!E8m zxIXS{{BIbg&!ozd%0r$4KT5CRGsA0Z!{=WeehQf!vg0IdP%{RINwHU`#qe__tdqf+-0Cxr31e+mA!Mz0e zE~v4jWKsE=G*gM9gIw*<$xb0i(j-U=@|bc#!q}UMIT*hLq|p$cu!q@IU6`g6!^?#h zx>TF_6dKeTeo4QCqzBP&rzFC1DF{WVRdy}!KJ4Vi3oih73vs=E;u5L}a}i*`rF>#S z_>;dd(s~V8wNbtZ?L?&iGc)y1a;lIFilXY5N0!kyfrTmoeJ=!kH;1kP+eq?<*+Ygp zNdQ_We)i8CxWBtV{@D&f={-`L@~`m?alE~QN7)8jdx<4i;C){karMBNza2Tg89+1t zpeOd1s5;#9i{OqH1yh1)D<&Qrqc7&zUm@v)n=Es|smP4xG?=NZtFB`qv={{=t78vq z7A~&)5z=?%j1nv43@FEUSFKFR(_@jT*NU>PJWRvNpGGGO61kjoT>a33F2+jF2Zrpl zGIwU!klog6)T)YQ5gx#|!Snqtz)5Rdv-{?=PQz%^XUXIc#EqQ*8??rm02|e_POdN| zK;}NvcJ^9)|01gsAHb22h-OqYO&6xN9!j4ie)>xZbO6mN<75{*wdK9dbfcLSkrJGe z96xnL9H+Ybm@_Q*jr2DZ^!|=lvvc?ui-g3n5^9$C{|Td)>JQ83PD~O234kgG7LHDy z9&U^_f)2=r8fkmobSCGlpV#QtWGyGxvCqje=l9~=E50NrR zV`*Uabn+bj(7y8+elfOP20G!YdrG3TfGB%Hdg6`T4d{ILJ6|ziiNw_bc!SoklK!H> zP}-_nGR7@8i6cO$mx-|VokH}xm~{=Vbq>@+WfT+t*CKgFgOc#v zE7gKRP(fiQ>~;#1ek>EG_QFP_kuyI*R73B)9^--aM-QySp3Dn6?=t0;^rt9P?GPPmdbe*dBkj ziios|#KA7I+-b}JX>`t-((;>KED0HGmj5ALxII674H~4t)qsju&hTV)5voleGu78G(8YHmOcP4^8lFTwcQC+V0ob_7 zl)12IbPi0je_2$HQJCKPq1ER9lR^?xGYHb&0Q ztWmvCNvY6Eu~pCuA%are?YU%1r!+abG#T_)At%wLyr=1a9O2-7)Uq$5MQY?lze@7e z@QT#%WB~z)8N_DRpi*EVdQOG9lnUFF`)$f4v9! zjDKEIm`?K@Ct>)R`cFnd*KCYKuj!j10Zv;0r2hwD(F~c705O``{m0s)4D6uPSQt5?HFiseu4cDhB5@qRbkdQ9vl8}%tkq&7@8Yu-N75}sN z_VfMyU(b0R?%bJO7-r_)bI*Cs8QXghY80Iu3c7&Glw1g&PGTL1n<*g#g9wZBd6*YwS;3K3P zu*G{L=d44?(QE+7E-?(giP~VCporxNn715)QDbxhp(9*T)dLLj`l$Tm1J-E#2Xw+w zAL&>yEwfEly$7l`RrLo`D`QyoiJw$*ku}GL%U9|zRa5*$ z^fvx~id&-z-*85o63YNuj~Gf!R^0@qm|Rl4V#!L_9l;9cjtGPaQrGyZQfZ5!tR(?TWQ%ZL0M)xTM5(+m_^hgb ze9bC&at}`zaKr|wToK0!Un2wN0Hv-yojnPtZvf9x4)xr~HdF0+I%% zgQ`|^b=7s^Th)-`JmKe_kHjVWHt@AqF@_trj4r{Wy#WHspc>{VZaEipPeHZ&(Zy)( z5*?AgVu`4&=(5;tWRMMuiq2&R&Mf~0UaXonKJAQ!yb(HQc&>bIUiUi#nmIBac6&q9 zk6LDNO9+1-gho9pQo#_H4Ji=CZfRRcbFOj zm;2IVSiPLyE#|0@u%>u0ki(U6j*8hcxiV#{nuOx{+%vRk?_+A6eK z9lQ%u*c;9{m~{aO$t)U4-de}7%w}|t*ru3~bk{Q}NcVg!tC^In)G5sj zH9cWUQs}e)RQax&{&@P$$rM)m_vc>^`xe6MFcUGkjz93-Z(#Z~FK6u=ZS{sluqfJm?$IqzY0=G_XBB0jbpe#Y-SmRS2A(Y zGI5&qvO{W}J%SdQF z*B7UfgpV%3Ag9DM5{3NE9@8laPVi}yGB_0Bc&0#`QHgTVJ_+WIvrei-A$s+Bzl*%1|Lwkml3Fk-A*=?q2~VR}5&d?efjDBL8BHWaSRd88JVgT^ zx>!y)HRJtJ`e7t-bw~}Y!M57Kii;tyL%oSev*MaCcwslCS3VfC_&~>i2VjT2M zm^5G~)GhvoJ58mzs>0-2dg zI@GqWlyC?b^=H+f6NkO;p8@XphvNo&b784!H<=VW)?mqswB+bJG&ElZZI0(Q94Ka$ zqa%7Y?+k4k3~g!_2yW^+sdB_pv@J%gM>_&Gj{huKXJ%QXp1WCZu^?v8nI(aUD|GaI z%PLRC655>T%#aTH4*Qf71xfu#MG4}&RX#FUPM}1D&Y;+W$EH5?nleXME=@{zVz-VD zRg2iZ)p$j*^0(=PTDpRS;c1_@uzDsC5g!Qv zqTN;F?Fj}_yyx5}3&~HXjK>gn!AfGC9u}V1Qj&s*cKF4VPJbgi26*$(zJ{pj&@z`; zst~N*pG|-Ehm;?F3-*B+Kug z{8&O{%*+|2e%qYgz2bnuzfI-{U9Srrq;63X2$~EW!gCoyU-|XI1hhN!zH1+&s@b-3JXlxC{`CsFG0AKfrpUYwKArA)OPv>|^eT+!Kt;ize%F|&4+0QX}s7JlqL za*iH`J6tOJ1{frQS%ua1M7c9^@LG+Yzx|m)n4|;0Z!`-Pt;_!gf~bc4EpiAALI!hX zif4;IM(Y7aQ}pDw0Zg%EPU5onNSWI;fj5ZwptjX;uIh_6}8Jg1{o$WC;kO6Gmey7 zVc@^;N6rU*=aWFf!%;5xP#_-ubk{w!*(=BapBSBu&k66U?8T8pj4VQl|5J>JnfuK* z46(!|sZS!uJ14;*_5^b&bQ#DWn81oL<1lW6Mayt(rG^ANfn97q|8Rg7SpBu-d;=&HRN%VpBMNEr5hw+v1dzc0Io$y7Tl(T?(tj?v|M}unzhGY%65<%AH=BQa z=WBznXM?TRc*1(c-I9s%@v8Q9o&KP4rRQ2@BU+`-ieoI+F4$2?)yb0Qu^NT(U2S#9 zZ^ObYVZ%ztTD^JGicya${6i}KiErCa)6M9bvflzz7A_5coH5UL$VTMJxMq#M1InFh zvx_OAA&@o7Fycl$fd8yT*mWa(rM|!gqzMNJNVJB441k>-qB|S}P<^bGAU--uhzQ)Q zg+n?kQzP`F(1Y^6MtJ9tSUm5r1k!?;!b~&O*g+dTQjWgCup^iDi-?JN8s(lK1<4lO zlA*7gSm)c)A&_p22*N)`O}V~DAfI)9gz%SwBn`QIgP7-|dK6%~0^}Vg6=h*V7F0UO ze^=0)06CQoZ#4tkc4mVfn+C4dr#Gkp72?}8tldcTR0Xg9D=p3n&qb0_;1I$2fz(^ zCfP>EFZHX7g}?L*3XSiQg(sY!B~Rv6@Q)H9RN2{`Km4zVuEi6k5c zQa;6(%3o-dJy9y$fFkX7#l_7Fo&mBU(U*u z8KE)S!>Q#^$KdGmMg+0|6>buKd1<fulH7ImaXy)rP8NSnnUc=y&0jFy-(8d^2++R#VL%_39 z>~W3nXL_yuJQTpvAf>QPUE&g-)y4BjfNw8lkpo5r<=ioXoH2EwqM3M^sB(Wo7k}gP z-$}_o6JL#Pq~}k8`Ud+eaLF)|QHoT-{M`A!#f@=7H2DRT(kwA5xJE+r8q&oX)BBhi zQxM7pb2fOu{j;`kR9s zKP=Pz8pzJ*f-Dq75G*VRVf?-cTw5cJDGO=4FS|4=oMO&7-Sx?UaS?5eckLz+T8p$zxRl}`Xon? z6N^vHAX@Qd0n;4|vb}mr8`|-qkfjN1C8QI{bw4i(?W| zOovqXEOZ-UOF z`v@C^mH33;goUJN(NZVFjG}KkBDkh|@UbN?62js(M!*Si8|BFfQ5#WUK>6|Rfhk}; zbE2i#ak_p_M(`4_zS2WP1;Q8~R1<61Wtekb61SxFNm$ZXFnL1KSCvlmmvi{g)c32I zZIoOFwoKv4L`pMRPwmulD=rW~cJ>of7!Y1ATMO&ylX-G{ZW5wurLWuiwX?|vzZx(s z_-96-tl_x$P+A^{bcio`?^iyoPwZekmF+9>94iDSQG=&{90S*{tA~|hCQ*h^l{85` zzB8dx-!Wf)MCQBZ;%?!|6tx`4B*YTyjatG5CigK~=a*MEMP*F)YPN%cI1DRsKt?6A z(i^-WuulA`41%EiMD>70X`gUtPTV5yk2TIx?ajN%FILayKzPrB2Y+1OL${42obi5? zMxcOpvs;>%)89A9oek67SgrFqc{E2sccV1=7Z?m;-eEN=T59jjg&>s-VVlH~#~MYS zRpHYNWb1KkQt6fhW<%iuz#0Z|m*Fv14jd#9p+fs!0er+CmXc&wPGLye(zXdbi)=$h z9(t-z>1AD}yx;3+mg#6Jfdp`2(>?aR={+Obc2v(vl(WMy1?-iWdSDg`3|RC7j0Bbp z8w-$VR{090KrMw^fg(AKi6~_9&1htp@W#G?CJUzv_Dh8!jW9Gt*;D);b-8Q`3)%Xn z%yYV7JB3}WQg0>esf{*jCUqQiMx)maY+I#zX4w?awNMY_Lbp{{)91*@n6z=wnJaaR zeXZmy@3ZYl*Lgb4{LK8MzRy_C+4t}EISEC2eAi|A681uL~pG(OYZHv1Zx{IKj0*$9dDu~kq{k3CcfNG4~4 zbK{07gyh93y6VR81f5dIwfnoW$c=qK)_lYd5KfyPaxgQB)v-k0@@UeCQd`pw_U_jF zF$|OLf6^hWv*Dr#O8C7 z3jPoh#(k$`D}Y<jofEqC5ST@Q z53-3)kuhLP_5y1H2PPI$?ug{inuw2V`aX8~lEE}#6!+2DY6mi5;MjsCO(xQM>x*rZn?1t!f zF%?(@`KTX*8E2A;2n^!MIE?JVX$`23_?zWIOb0(oAXu06SU6C<9x`UPU~G(Kh~QXV z)BEGl7$&6u$D#6m_ItrV09NTwkkItAR(f7W9u|0v)1tdU2Nw6LG2HwJs> zk4QSUqn~-i8O+3pJq>rY$azr9^1Dn>$M-e>({m%?lO~-NLZ?QjCVwFu9tUs^2*`AHFgB1k z!U*0NwXka($&f)Ffhfq}gnWRKZ;O!@mYfrY#ZNm0<`UosYHGwWy#Q^LOdOR&335Yu z;LvC$Dw~?RrWk%@aiS|Znpk`{oaQ4iqxni9-K+Uk8Ns^tO{4&B0DkL9n(dj@4)PvZ zuuMXd5n{t@x4{0)(CB+U`Y_3cuUJ`s5Q}Trj~3WbZA!ALE5^PE+bk9D;*PbU*z(DL zK=*u{WCLV`iN_*^m(gS*5ypN-!UqR%S=uEROd?ybfCc1|(Ni?0ti368¯wmj#- zEg*nN5nDcs-1~~!!hNdR+UdPd9CYB;nE-p2*fEDI=pJM}BKda-=PI{rGKKh%)&&HM zOp=7Ros-_7h*83S$W5UBYxpF+gCN!VQ^3={ytjo$0zEq-%$|WbaB~cTlCb93^n~i5 zEuw!~K|OX%Z)Z0*zW07<>C!P8#IfM+|3Hx*Pl5H;JbNx z@Yot5o-T)nS|}e5(H4jw2ZysaSo5vtQL0?*{vxjzhKD=W?pUw|71@v&f3$(xD_zAB zl4h-l8Sgi0jK94O#Hrga231QdZ z5tJWbrOUGA+m&|BVq>+7gaU$UKs9nX!Lv{dRa*5Fm)D+MGSE0WxuhYY_~hMqc55yW zVC4b6BUM|btYmAY6e$p>0oDf3G36U1I>PHIMJxb`i9e&#A4B8yv}Nj2LeY+&%w#x1 zk4tAyN?@eV)f<$nbq-scz^CH3j@#gC5Ja>}=3e|Vbecn+nluH8m|1)CZBl4H-vL}X zQd}uB%jKM4jBLZeHecva37JiT=tq{(2W6Z3V*Af3Xt9Lo{LU)76@2{DF8kZ$5w8y8 zD9SL3t@D_yljOzNlpthsKoZngsHRq|&okD<$O$L%nR^NpIJiO+W8KydbC9PRhcZfGYbftqqu~u(+GzK$hFP<@{niq2uQy(a=?@lG#6TF2ecqx3A0w_vxM7(8N7epoaFGegH~K}!2n$`S3~g5z zVc*puKAEc^2neOoD=F-KBJhg|g43%cu$|N|^!WBvjWl{$>xu^}rq{ zJD6 zkd`q>znF%Ou2ODp%^@e>@13T1uXJj*>!kQ_;ulsqA6NLb-4Couq<1jLb22oA3>nag z{v`XtJF_NkwU6~4h*qAxi-^`S(CSL7S(|&&$!^^7PhKRnJu;tPG`pr=YjDD-^?3h1 zV64?iNw3}$C8FD8FkVX zknGm*1HZ)zc}xP}9@3K=!ZR1}N-_fxJrO1sT7;w#4p zt#rN`^{M+m)v(67@}cqnQHPATFN%3Hh8@>Gr6usFgQUNCBE6?z6dgSP-Haq6Z{WYe z0z%d;4PUPQxNee;Vmf^rUz<}XWB*J1$3~FS5*`n>%Lz2FT5Iu z8As&fd5HG5qp{;@N$kKutx#Q=P@NLa$Bi_@_Jvo|f&#veT0E>MA8EP~JV6z^J5QSU zwO-iBtu*4g$`SPLV+(4}#C#jhP~JsFsmn0?m&N<|bOVE$Nr1V6n&&_;3tO3$~3D5}?A$A^@pKgyDGKK>^)ku@-@?pT_%wwb8+ zG%Bj2PzOMilLU2=bd0kBd3r2jQ!fOFe(3z0AEhx0ldxlim>O}c=yr}ZLIA2N6Vu= z@84p(Hm)yF5+2R#ILDB>{S{>{ig{Vf6;4;bQogM@c7ONdOOi&mzFW&Z zK51A3b~$_f*FPcR>+=)mfyXD9jx_{uar6X0pHN|l1BgR^t0n+imO^^71J`(sw#oqv zOLXi7Npp}ixFn=aGcv~%8h|#jb2{|fjvgaKxYU~vTPza6<^3~_jOtjSFFbtj?p1YN z19W?!fW-PC>FJ^1h@^`-cz3P*A%bV#h>B~r^=QXAfX5bqvLe7qHLEki1cb(VT}!WK zM1>x2iYeLWgsw>}e)y;4FKC><^wLZH>oJp9Y$gQZ5yCAxfZaO;Pn!e0@PCskfW!*H zaI8Kmc<~1)VC^@EOyCP?+MPm46QdTv(=Q|qy4@^NZg_mo{i;7d;twz07uGF@7W74N z`4N$c3sep>68!FBur+6q!oZf2pbpY*QgY%h(~aPN5kIx)J{yj*i}?Yg5{CP+3f<%koyUQf*(j^gn#iVS=QPF zG(QzU4HutKWO8sG>+Y)re1Kp8 zU@!A29tFTWQxCj_4`+RrBp_T7YXmt2Ief%8b+XiiyfUCbOyYbw8nSZuP9Hd7#z!I5#Qv)e&uQ^<rp9qa0p(2-tL$+K zkEEG>`*6o|eyW@I9f7N6(kDE$z8Q%fpN4B5Q5xNC3gYu~tvDq1SnMlGfB(G$e7)J_*;v66e2;(-!-_!HN2T?1F%QSdJAnh=0UVd^lXy+hQtQjuh35Y#W*sDu)xn z-8>s+0oBdL1u_Di;p~}nQYo?v=v1g2F6s*3NG}~)C>55;w_gZx{}RqoVBwQ*f4dz8 zy{J|?2T499Lgc$;*U~br4J}5@kuvz1XGEFljpEKEf;S%Jp(@ME_qA4S%6v-;}T4N*f`EzTF*Mp75gdJ zAz@{jEdvs>sjbd*ApJC;yL6IuoY&t(I2>f#Vb>x-AFBJ*k^@!VYNzQ`(3$StvEWKx z-&Saf*Lm9HuL)&BDXpQG(mjCpA&K1B)|xylTsX<4RGyanc4SI9LR#P=*#ejRY$Oba zwx;^rs`&(YV>%TLrkuNh0V}Q~hDho784pl#z(7wLlbqT~7eh^_sijbUm*6Fah%zTW z&2VYXVDkl;B&pYY8kyU%I4Ng7(wIh6#mnx|sLchmzjRW$g(}?ihupJn@E2KQ}iHMJxrp7w=HBgs1L@Tlwms zY^WfTHdq0ypdNMkPuh7dZb%{D!v6j%?^+Z&sWOl%smkrXgf?hw^Yd*KgIksKlW1tl z2JQZha$LLVl-W*DhvXEsk2b{O?RTqk+PR=rRg103#(kxopsNhlud=+~aR*oz!$?C}~o6!n4mp`n{Yz?T7#K=W{skhIV z7D5+`VLUfdX}?J4-tcp(wx1=qswOf;t3gIchX9qQBpA0AZ4LPYa<1M!)1qR0J!N$k z(lb+BOeATsBhFh&kQm6?d;}Qb9Bs>U*?r0~$t_$#`(ylqb7NY9neDqA0T?1Wg>Pf( zT<<|ch!K4h-k;0gDWNEQco&5dmn$5ld8Mpw&u38Pcj3m)k3#avxTF}7m(l(61{(~e zPaL$tg`-NB+Oz%luKjN`qiH5)b>*IADvwxSo9dtpXwBI)6V|?JWi9*&yk3Rq)H7z* zh4Wkeq+xhBX7`a*W%%+A)2U~Quh_?{*q-*DfIY?DLX|&wnZPUvh{SgfUbp#aM?cw*4W-nIGvq7MIyI{>GI z0%QIhF-{Ww9tF7+1g*Ei$W)s`4HNkmtw|BcI%0X-(relq*d#I*JF+V?@p^PF0^$yl zPzwl}Qh!m+PzmFQD()jT z*a4ij4fRi3l&akYiT&H6!~_zY+!5oh?xVgfIKmB^Xm|p70@vlPjJ>pcgkY~#C&2&vT zh0nn?5PNF*L%gSv9?z*n0nP-auE0ZyxvV=e1iEusRI9(RG(}nq3{1J2!bS;LB#DC=Cd^t$8fAqUKAVMI#xIq;Kls|nPM|6;Gu+*2g`Y{mP z_bsv58&Nglv4;Cha_8Y1qc{w)0ZEC;?L}e$?^J5%iJ7_Y2h8)4+qQiMNi)Cp@7}@3 zp^pSt%-zl4*eH40=eGUwG|lDEv)i}rOULS|)UTEA)c#*PI==&>9&WTNRlCRjP$6Zc zCY7iANN+#Y(Nrh|7e`9edLt%QM#h#M(QCB2^+Ip25PG}`Zv^v^Va=7O`Nx-h6#5C zNc+S5?>K zy(KA-O>Oq@O#gkdr?M&Ypo_58^W0@Lq#vrTw>*zt%H;Gr+sz*zcrN(19bS1jQ!Wh5 zZ`v%{H>8RRyCb7XfV;>^Y zU(fpPv<+n6c6_2f^&Ea7x?HONe)SAs$lKcHSCPH_^y%cRF;()-$izxr-i_F#%-hx^ zy*Pi9Yg0e_#N$4QdMdrp{-+pz@AOP}{}uGxT>D2-QU^BX-0LTjJunf}M*Z4&Cm=Ug z8RU-ex2BmSP}2;+j~W(gIBTA)9l^cw_)cWeTNuwsgWfb|C^m~!5Z8zMrX=-70&www zvgmDgv&pq>6{G}15a|GS3^Fs{fv!eC7A~xU?+RYCY6c0EiIK#qTMkMnn_D^q4fGx4{FRdpvtL_Z+!(2lkHuhg;bZ@vp0-OTM7bG?_C~ zza;Ya8RSUO+u@{1DuQi3E5t>liMQ(%xXAZ(qIhm}8YUpvMp2D3ybJ~$eTykiK`@%4 zD@e?eJgRa!M_DGBDg}sd*BL6pM+u?U@ZcL9I~S8SW9tL{6Of4=Bi}W<{Vd_pKv{f) z2fFTn8w=>OZ5*ML;^P^bQBP%KCATaPVAcZFBrW1&@KVR7=-(svGJ|>K+z%R-36b6p zNsDHOfZ7B~*@Be5CaJo6e(DhTv_>b|wqMwn5UjLu#@gel*^8s@e zE5o~O&CJ!;(6Oy5B$@Nw>W)7AKwfy6GBIymZ>y8(Pp0+LQ38~QXh|HY(2sdP;|;Ta+I~&W2$)}_viHH5#v|` zDoPyl52(bnG7%TJCAlyaYeiHI*Bkg8FL}U6z^rqz9-)WeWX<&yceqB-2wWu5R;{6d ziaA(_Yi%71K0Q0d;Pq)cG&>0Z{%m5!BhT3v(J7tZv2Efy;HUZF)8kXmS)lp74>Z|b{_juaGzY5DnvZqJYO9b z4!k?FecyS&!`mL&|7-%w(7g3iRu_;3g0=@GT|MbUe^Rv~zT>bdDtr>(tqp`;9%`Hn zhYwLA)!xp)CvKkW3lVumGWe-Z#8NN+$Lp>!rCgu^<G<%B5Mc;Qe!$#ptCn>~K9g3uFsj7S4T4QS`IMJNqf**>Qk30<9LOerc!&v z4RF9*anzn?Vl%pie9@nrPR_DB1V*Bzj9+8O+&{-1m?}oy1bC9Hx)C`+=aYm85n^O_ zq>iY+SmJ1={S+s9Zr-tk4QWg!(eoC5>Nv@a?(gS)Xi25nD68Vc9BVnVX0I2n;_Uda zwJHmJfb`M7f6gXyWJzE594t5noqp{?D;L5;z?p{8E!Z*ejTI<-6s*z;sj#5MlwyfN zQ9w)E@GhLZC|JZ9jG!S5G26egxc{c*7%BT=F%?pP`I7k}O>0Xcrxub~n0JAWmt|6aFx@KYOnJwYH z6I133Ru%ik!`k*q^~5tpA`T;<@NHIq!8VoeK4bgr)Jwq6%>L~JU!B*jjni;#hwI8- z7AaR2hmUKx6NQ@1Zp~_Cw%@jB2Wy?6+t7ZT0{4Q@Ow{}9lwuNuk1vu+vdJZTjVfoX z5I6e^6&4qnVN!Tx3$^#gRB(T9hy_Zq=LlWP2cYM~KN6>?Igas0ky9d)i>lGR@rk5p zCWI8nEX+-UEVI9d>g>Ba2^yN${y<@>z53o%ECfjT_M1|7=ZQ3@3Mj0bEU24AMlS06 zoJvAob3_&52_F6GOwTTUK@>N}IIv7#yU*Dg#$nZX@1;{-Prze)ephy{aAOZ)^X)N zD(`-F*fDdOd+{Pe=F84xKK9xp^Bstu1U^*_YME>+e-{N&7J}CM&%@L7uYf8jEZit| zvQVTx3cSy-dejS{g0~1#@qA=hU`JUkq!5a!Nti3@g+M_DK3pMuz}mNwWkoXl0W$}B z5n%BA;%ogMs$(4%N<~g z^E`&kb77X#Ys5@u4irL?^)Bf9+!9fXz;+(3=cMxFm%?M|hbHdo`Nb%e=Ok|1US!Hf zB!v_f^9Nl2FaMUT5j`7GllD0};i3zdVjYTc2f`0_fJ_rFbpb_{_N6$f zQ`RJc(C|Olb`wb162irXu*Qmz8d~bs2qf_|{NZ-{dXH2CB?RQR9^p$p!ed=R7aiRT z<@8N48s%lK-#Uaf+NE-hANEe;s_m4fFmd_jCW@v@g{EIWNTN`>6*3SGZm`$l*l~|C z!*eO=CRdHFlVzPz0E7_`9roy{;a|Cv8b~_K=M%Ir)2yxOTs{LcScS1l>J)Ew=nP>R zPQK7J6Jt(op3rHD-5#lwL}{lyspkT2+hOGDPl)O%TkoZsjbSeO{CCPwgX~Np#Whucpf~9$PBLQ~Q=q(Jjw@@B#C!#(q zLNG&-LqVt+s(9A(gEqGfwWsI`oBI7_H+z$6>wDhp?&-UDO}FVO7x()^m-ctmkC#xf z3ij=K<58U;oKKz!x4)f6I*z5WZ;~$FB9EUaIXK02v?;Dxnv#Dty&h+4d%;yqNRBp9PMMQ^Lh`r z8vkVj!EybA)DgJU=`?*%^Dc&qP!=FFjf#{)VLA3!f_Ks`y(DDlzY7WC3$2f%X1~Dw zb#WV2z$D=CJ?F`w1c^G+FW4SgC{vZ|}mwA4$%f7-!U@F8R+JQw>x|2pzD`9mX?g`5{uzi9^5LwFrjNE5O4S)(e%3fhVy&~CXI2SU66?QlIU71SQ6)p zPbMqsg-$_+3Yd24aj+|fw-?hZTyS|$ImU%nCC|Z5NTg6a{sTalK z!YMy^_q`lMKSd0#mW84q#y)lK?ZF5$fIy`|`%%R>0t9KHmlhxMO)Je# zXJ|XZ-|4=mlf5sEts{-CK8-B{F6F60=PmKjAfOglnp_Oua(MvJe7sJzAJKAQo@SA^ zW)a?lAZiw!Jub?LM?~JmmBdpt$XO$A*3H4iOIP!ZgJCeTtNAKKCzKC-b(XIKK^2G= zGe1x%pg|KxDYV%bf4OOsGJboeHG~*lFApU~tim%0ueUxR78^)I&2NZk?=>vYQ7BRf z1J}yC6NgmcnWp!9;Xi$;(A`@e&)4)iO{HK1;U|u*$VI0FP4b{h9_G?nNh=l#0U$?i zG+7f*Eh^E+W=JU`gcOOA21JQe>buq2Wh(BhPn3u5lgZXd*VqbVx8?;>a-O)p+U_rw zO!9l&CA{<*qa2~Vr6VGBq-ec4R#6#ST<5hgvf)kenhJwA1Cn_1ZO(|DrZo?1N}AnC zGZ;gP|13clYMGJZlf!VF$+c;Gqk#)!_jF|(6wz_`81=X$pCv{0?QH|NN=^$q#F39f ze{#VqE-VMa)Q3th46cxq;dm!QD(b?jG+*$RWMW{>QC-=3y$gJH}~<|YRSZkC$mR1HRzuQ%ZcPsTFjrNv^5^eph0w4e*V(E9x=ILzz*jk zO3_r|UeYf`Upy<~3}<}OS+@3L4sJw?Jk-56Ct4~Tk>2t6y-uqP z^BVXwiJwNt7D1t1k6e0kcFQn=Bj@vQf$e8;^a5j5Um?=jQ0* z>;DIj{-xTFNbtez2S0b&a5gM?qNqYj3~jowz5Gv?7u|JxwD;j?3DQ;ZI{CVqPeu;V z+a#qDE_vd$irEcLUoW#8vLb&AW6GsbKAytqfnyys6}$)(OQ(k6$f6Y>O<{tlq9*Vd zDMk_Y;Yl6SPd|hBV4`H3ppB^4BAlHZaq;vd8lYQ7dO{|J;NS3kfBGlhwKO1DrbsPF z;~%(+{8R#>7@0)nLx`FDWH+HtsE!T_QzbvvmeP_HJ@}}{KmfRkaE33YmNG>tXCcaF zo8{_M9?`|z9wE$%TKi}b5AUz~TWrV@9}|xg@{q9&s%~f|{GH@t-kmV<>TdllW9oGF z(EhlEw-n##Amw}|N&!*>r6I@%q!ix@Kf@CQP*gJe3CJ5H4xA-^{V_HON5L*ixd2j< z_qKTar4WDI@+l%819B&I;RZfCzHOA~vL*d<8~nZXd>NwJbf>N5wjpx#o&ko54>Cb~ z|Ib+hXKfFEf6IUMq=p_2fJUMAVEunYXq#= zEm7$Hm|w#ZipJ1W3s1+i^$XAW0)(Qf`X8Cf4I%c)Yz3-&KHCRVQW=L5I=ahaD{dU1 zne&dR{y697Kni5+RPafXX_vp+J$N5K3t8ju@X5jmkQOMoMJy(%DG2D z<-mF#V2dE`UuS@l;%Gr>bmKtALjf|~mcJD?i|{yYkMj>Ycs@2{5;TGUMB9x`Nl?`h$WKk(Xk}wJ|KfOA+T@9gDJy8^6xIp3J3P zgyG_ulh2nauzCFaskkegAojZkX<+fr<9=Fh-Cor3;z*#9jdF+g>JTf7e>8#or)`|1 zCS&h-)LmSq0e&S1$coNo(P6fpQBcC8o1)ufcNDy9H#)}a_BIH(Jx76SFAma5^2OjK zqT!M>B9pUleC#=$M(Q~-4X z>N$A@H|xYeTz#6q{X~9H_)oT9=32h76WYh2p_AlaBm~pBMDhiBZOeLWN3Y#>anBJ% z3F2n#Rh%wd1((st$5k(@oErxEDTCvtBP0X36|Ft|CcNhkeT`dpabImB{PZ4yL0`2D zsQ_cy|Gxq?zy|-K{^uKi1 z9oaTHJPN8RmB+CSs1#H0FDPhS;Cu~8+qJ8?o1#oI^bns@C^E8?Oz8Y5T+LW z@lUPCB@=sXpS;ISF&E&kc0lO4v7i!2s0EdveV7hi-xhl#x1kHrrGbP(`Peg>a=G6sg<@3;oZ__RZF7y0hUBTA8V|f#H zxybH#pLtTF>?J`HG;ZU6bWszfI{Wq7?P=xxAGCWbT#fX@AGinG!!nU^E%UDW+k+t5*$NV zFue0uq%HH$>1@yQX(RJ~?M*CZcyxl6GW2ebufJr@{rS^_OS{-MdYP%Yc0>G@q#2F- zSm7MkH%HTGP41m~z{g$gO@RtyA$IP^I_hENHftXPD#lC(Mn_zKPTF+q?osF535`_* zRE(QkbH8zIzR}6|{hq1?tO%L&TN&CkS_8X;THRxMoa9;e)I~`9n($gI>ijjDU#(Be zlMUH&hmD8x4K8LECQbxTfV6a*3T#R)&5G_K55`OsTlpT1J1k+1^5^nKlt249BWb0< zW78igWjJK3)cmp4rc&G0jQ26__ypFK30V{Hi{7pIcvy9ZGDxNN(fH?ibReHi)^M48 zN>irsXHh;9`1nfM@y$j5%PiB5vau)PZsXx2Ui6J53|<$G^k=3~)!`#gqpD9ONEw{y zr~HBb$%l^c%$Z5X|C&iN-~WEZiM}lA;$O3oq-ZRh?P*j|c&6oC@qbNVTY$94o<4@e za`b89e@$R})D$=l*vXzG^{)wRdjZD*JIm%S|22W_e_xui_Wyh-NA272a^U0lQr}FY zD&b8)dC^yr6gkqD#a;hv0^1Y7PH^?1u4dp|8y@jz%+q;!prMbc-Hi9{({V9;_unrN zCj~xRne)2i$guQqzKu)&N9J>tDI#`XS!iq_Gz-*S*%x(LSi|*0eEQIJk1D&ZacAF`22$U!~MCA9$&@Tc1c> zXY3}Mb=Yt;;x?<4=tBYidNNrNPB@^nMtZR8a!#6Z1 z{#7T+y7@WbLxg5IHf}P>cKja;aqwKkyY_0NpJnHSJ8l8}ylQ!Nw{h})6I4FZEF$NV zijmu0sA+xAr)TDReOcQ%7fZ+X_xXh^;Amo=t?&lQ0JMe5Rk1eBP(w;_4?CY4IpHY?w6pA+}xjsuzhSX$q)T=w#mU0zGd zaOO&ASZ*PqmkRP|fU5HDRI=DHJMQ!Fx6Rj6(PG?OiR8f@Qs6_X+pgLDGp#9x9}dU^ zs$)eoWXS@MeD9~WG#FO}ToY5Qxh>hlW50At^s?wL&p(?+DTpqVm8!h?EdO}dj&FVp zZIPgrur<}SD&kNHXI6=F&H`fewPdhyUB@0PisRcl5(fRd~n1|Bi<56y(lq zfDVOZk&GGfS@pW_O>`5hiJvw?qrR&O1PYK~@k`SWPj+9eJmq7$31iD=`owD0i?>I3 z_UqS+?iv(G(UYhPRe_hlfeq613zOY8_U=nv-We#V&fSrF2R0;FH?Qt%+>Kcc+`40< z@|m_y>0g}N4Ezeqk@;hwQbRcM2>4Y?IPnA%cKF~H<;8>UGLPQ@^|kqB`H(-vk%t#P z<`XMF)iL+F&(6XR@&z?W-n4D~d^TG|?~axhSx|D*{RiB?cwF`Bm35+h+JR>VMiw|CattAAvg4QiJBAMH%k=zhE2YXC5~sH8UG1(|SK16~P}?p<4xex;7pFdyQguz=b?d98U!>p9d8OzZcN`McEjv_lxoe znS3H;<0c7T#ZFWZ+1(xJ&NfK@g<3G#)HVoy#0t9m$Y{{vY|N9Iv~*lu^-Pufe62BT z(q&69JjjclRLxAf4V$B)ujG0_e&@JnTO6I!tBpxdqXA2vTU?}g3E1~#0(^#qT2@kE zb|Kapb_v)ofe-(G@roEqTeQQm<;mhET}XKF;)_?xm}eru7zShx2@S8LtlEXxYuF`X zf0IGi|Ke3Pl(ue%%n3}n+<=$jpW^AHT2>K)H69^cmYiuxbToX>b`C41NXWp^sxKk!8dnIcJA&bt8lUZ**STRtoPkw z(iSLM93IU7Qvlp?0sNkKeX9V^c7t2M#r6Y_Z|rA!@jpDA$qqbTq@Inj4~En2d*JT$ zH`rfHv7d9^Ui26=xi}N9bxZJ&5IGUPnB-)>xS$!_2`<8B?jT%)TB z#r5;!$uqN&LoF!*eOl_DiZNvpWKEkxknBTfrL%zWBF)9O3o4(tY+^oJ4KC}qYVCN@ z1W)YQ&)U##dwsC;t}@qc_Kp#}N%+*@y_m6ISwe&BW37hTxg`3#jw=V(=~V9|JY3>J zbzKsf)}_8nMYo)}^DKlXRHThMxhC*}CFToNy)_zHM>^qG63b5KEtFJ`9e=OSp9;mg zvYLC#JH;phm#^(=w<|*V>3j%XGV$K2G&Rk^p+3>){ZPxJy&I-+%5)b(l~CS_?doG^ zji-{2=vYZG?$$nSz^+Vc5a)gfE0i^|(D3;p%e6Q5)}vzIn&G@5r`p5mTVdt--tymf1-*--?c-&*&$JX>Htn;fDHrw7bX|GS&Z`|`ul~f;{u$&l8F!c;U{AR$PU)&vO9bY z^PnHAec5N;wqbi84cTel-j8#4Yuc`!JDRovZ*Vr$|2Uhra<||TIE)8@kR8s1xV z%Xr*9XQkc+2c9pl@y>FgpihoLK}0_Yp#)%hPeV62+PI z-d}lg9B_WDAh;-bJ@O=fr@mn?`aT0lBMY3Js?bos=b_B!F|!v`z30=QP3YApaQbvO zHaCXLa>VWjCRIYtLA0}7lw6rt4EN|$e4vd-wUpS9~ zy&Hy$+s?9epIA4V&m~vKdA3gk5-#LL+1>OMYZj2zp>c|P=66D)hpu?rGz>nJH@drQ zr_kvLd1BRxuXgVIQX$YVw(Vp8p{|W5WZ_($;2uo3hVpQ8v!TDM#DGkh0&24qwy~)^ zb-G#lDSt|={o&%tC%DIl);Ki+6GJiHT`Q0cR+Y8W&H49sj|}hX%-Xo;Z%->i zJ(6P{%y!FHH4|iixJD30aCUlQ_!bJBX};f(7FWrWySG6$!E}qM^rtdbFn#9=&}Ey0qu7GR<*GAV7t#AEQ zvT_r8^jE)7vl7vG4>wt78Z3})W4fWi?xMyX*=#S)2jOd@!12IB^Q;Yf9)!2&*W}yA z(mxf_ItLbt5rTrB)<4n*zvwh|f6;a35^=mw$6n8RRI`7EXuAo(TW7|_ea6LY#@TJ= zxV+w?4$(Ghfp^JC&Hl-wIgq0^6y)MDbJ?6dJXYZLYsj_?hi4pCIvfYM9IF`hAC9#N zc#xf_u*sgz{`9E+soBj3HCa-J5<@`^V8-el0_sBo=EM9Y?M2Oj$OgKa2BR8FrVdqYq+Esn6Y*zf2*sqjd7oqbUS~qt8$PLs!=%HVAoO!V}wEqTR_8b5D!fA zIMQ-awZ0tWk9Y-@35rn!HfK;te8J>r>=G#g_+lgOuJLk9sd^5=L2?tDUD|cUSTQ zX9DK6><~n%)!XYHt3dS6TW=i!i?6lr+W1R*^ye@lt&1DgjnQJYx04|JacQju61s<-$hFedOJ_cHK? zb3<8Ad~9jULf1PQy|v8@24j&yo7)Mt)m7i$hi;aky5)Aec{Vmvkl{(_Yc#e3ZmKWH9l5TCl2|0!hsxgv6z05kYF^l!viJ_vMD~ilWIcDKB9t ziQ?Idv^}!%HA%~&g+S7>=pk`wSqzXG*glI{VGZ4CK-xDtlTT@FM|c5W6SGFEG8nq* z+4~Z0-YL56*72@naniYk^Cmbck?dEOvz8gLXqCF8=Wwd65S2*!@>NVQ$hFFk5G&UK z8zyZY-r1ov)Xf$r!#3V$v(@r-6y3i@Bfd(N4p!V`$|=K8vQ*l?SuSYf8+=p&J1c}` ztp?`IX01DHz6uM8eFe!N8h2YMV+9p6@L%O%{Cp~3YN zO{wmXeWZ1B`P(Ax!?<{HI&nHbx(+&odiimqsK%riv)Jd~Nayg&_RAi^eC8913usoy zdt#)1X?rFJ20bqq5Wm!@uQMMKd=h+Od}1X1PnL5=p)4t#*do!7s5m)ShGdc*fNFuV zp$s?eE((_E=-1+-_6G!h8t$ zS31X{jc>C8>}=-|6r=X~S&)^llL2~hv%I_SG3he)9!!umXb&dWzd+`2E?5^(Rdy}yi&P{ zx@r@DzNZWFgN4uq`NLx9b{TBZ`haPlU>$VWfv|D9pg`CUx}cyE3)%u+0pR*VVGrqp zLSf|eL1CH&QTJ)9o5k$1BHRo~XtmnAzPt`hR3l{?X;L6%@+g47Id92X4=+EUw}rWG zfX`XhbvBWj0c;RzXHvaegV2@La-3P352%HqR{1yf+j6yqg<9ncGvo>fG6eVAd|9E+ z^;bth=DnKfx8NYXtQ;+`u(n-CCHX7628U3!BgE zlrArK^v5P&^Lv=g8eoN2!1@NX(Dv%3kciN$SZ%+Uw0ov^$unnR*BvmW5ATI4hAMiq z-$SUGS~EG$F2=$Q8&o#Whw}2$CLX~CUpN*xzM`>V{g&Rol@34<=F5Rqmq)iIaVT^s zlpO~llov;WJw>)RZ2-}|sY`>D%tZE3K;U)vfFHcOSBg{mE5REfNc~!L^@`D7Hd_Fs z6L$)!umGm_UGjk~^Hsc2-UDt93w|WTSOXn`E;U<$?kG7#72deR5wF zo&Nh<5XoVyCol$wjk98(!+x@2U%<9l*GOO;9om|_xh_bLeCtjqmJTKmeu>1?Wec@Q( z$a|us@=EtbndOymMW4wJZG6MKk1wi7I!D zCmg?3vj-)rAs_hf&HgG@`)m6=xvb*I!VgLqc122vUE~HB3)PXyZRBOF|00U!9_jn? zy;sI;^|nlqqeC76gF=elqRMfqoZiOPy-RgvO#YdQI<+o>E@@vh?67Qs#-oy>m%*cA z@-^|NPBwPbDslsuGL7rEz&lZ_n+eHy3r5EiyS59cxf4BVL6oQ{fJqd{#%G&_W5JLiaC!gSqgf^tlrfVBw~vi$7oqF=~iJ|WJr0_ z8kVbOgD+Wd{&4I8p4+p9Ba`PX9H4tG9DtOqcy}0g6yH!lcug^>L*-Jya{>aW974Bl z>gJK}5_J&Pm--PR1@MHV|Z$_*~q^At&DIWlK>r$c;OGETLvjEHWTg)#t587~w zdU8`^3%C-KPUWbe()6j5NBs4E!IrS}#zRJPvB*;u=-5g9SpE27{rF*TePPAsLGF=5 zw2PQOs#{Y4tJ5|XYdx&h@k|-q9Z=}J{erdrC-p{^VoITlKkpuRDxlDH+l{q;tCgVs zsEq)0JZM3CGD-kCg}H*j+X1ZZ+r6xnC#nsJKc_%oxMV|@kSF$g*eyD2&p}IC9t=ox z3KuqiMnpl>k>382C4n69_59_6>I2hnH*)}wnVLlc=}OHag{*p_I#kaC)+~n-@;WVx z3ZjvgMWZb$DnE=Tsw_W@uRpUQ=8Stek60*vpUb+rNCyLjCO}3$IKLLy!w?D^lBdivf$4NVqxdwA{waJv&i6 ztCNh-+h-e6xq|p&%GOHzuaqsP&kDzQ!C9~)HS5EM%hdqwL%xjP&Pn%EQC;WGKAMtVtA7?z|!*|-H$Jxz! z%lVx=^D!WCuY6+!dl*A--AR#v%ri#t(}WYNUN!7J(eOD@uZ0i*#Yzo{Yybh2=~6DO z*w8(MAZ^(%5=DBY=29kcf#UB1YCRdTTCep0WqL7;C-;EW5>bY8hXfu@RdVmq+!8&JJxdiL0#<@ z44d2Q*BU6#u&vvjs`bvWZN!dqr>!>-zEYvEO)f>Rj%z$19j=V9WFV65RtgZwZYvuI zwo3lBkZU90dIQ%^fN%rXL4aNZ*O4UfEl1t~3)m3LKBzg;D4iw--uWMeO-(VlmJO@V}}RbGLFruD4?32m#V!ohlrn_&X) zBG)pP8q&fbH{#~~q%toMX66|0rD_Vo@*sv7OcC}7uxNGRuta=P#?KX%9@^a(=nU%~ zT8+={7ere1el(?hBP>+6owL7WI`Q?&2hJkFSPD6KV{5tQw$A3yxz@P?e8&skMb%3k zL+ByU10dY1^)VI&>tT0h${Y8~@!;LE8IM5rw>vi&kh4MEG z<#(`YLd{ z$xbWV1w+sYtcVo}&Vnw%8Eo7G|DWJY2}P7t(8LvRG$YqVMdgNXh)Tmucsj#FMJ!a1@URr#L&tc;Q*-iV02EFQk7ydvH+ zQF-MEdtP~Sc_ji-H+iLpqJi?m4@D#84}fD|$(sY%3{bb2g7izBATNL6$hDn?0O-GA zSdNPq;*aK1MQX&Qc><7q_PSxatA^4KWfDh&TE+Q9VuboFW0MUaz}>SAkz7H9w+)$0 z;sv)HeiKHkw@tFx+tQV-#QhBb5h##xq!`!$R?Mzw_&kFt{&hO_;d=JHZmb>lo!!LI*J;mG9Yj4=yifz2Rk7fD*VR5xR05Y0XL+4ompyP^#0;LOH|=3-YBn>& z2SM~>C~3(3bl8FMpWyA%#8ucnw*dqhiT;j8H;wGL_*+M}kB~MF5f4yGRTt2Yp{3PC zy%`Hz)3DK!EHLfdkDr`nL4@BOcHot7l|LmF!BD!idHnQYd8GstlHXbKUTtvg*e)%1 z_4lH+c30P1xd{mh{S_sANPxtc?8Ju-1=I}Kk=^6Q=f-ElX9L`+)R8%ku*vB)KO!~O z>udTtb)U!xJZnBQD%D1Nz7Z14H}-%aLQkR{}Otwi#sXaEaHDTO|3>C)gwS z+tOXyxy%MmWaRKb7eH)h`R8h_$ERpaCw2oF0W4Z}D>iqSU4!{&_Es`wIkjpK)(?=~ z$&Lx^S1mf7Sj^R_)SP7tK2dLiG};&DQ^1+|={xd2T%8!dqga@nevcsjU?VJPVf5F` zHfeLRI2&QPKZf3QiN3#2y4xy*@UCb)zXIm#xwbB+k(>N|q)*gzRrb2DqQxzdzfU{; zJ&c&M+|8&9%QNful^}*RZ00j7?n+lv;5Lf*B|I)R~ zm~6mq&ClW8b|aB^rgKR-Q4bvp@IVXz%LF_H1jOmMw5ya1$6w&K31&crTjF0^IyB$eW)g zNuX8pbFgRcMf~2%6}|!fmohj>l12t`pP%+ujH_shgs#N%)~pxVBfyQAr>JKJT?$?C z$9&WPKsM>~j@qG&5APNXWS{4F3b3K7jrgL$0)g=D>2_PHT z5#5^1T@1UAc)sEjfU&2(LN+9Ht)U7BiB`W_E`z1t5fhRD3W`((a16R6v*Ue5B)UPO z9&IF10BkltI~lefmSqXHcjfY^ttJv}P9L2YA{~qpFw~Z4?1qfYfE}_tPmhLc`)!Lc z;5vH9?AjE(de*sZEU4HPs^b~%j)XPlj+p?B1~3c&&2Uhu|FMPEWBc1?Qu0D;+1amS zZou<&7hn1u=ed&)?>~I(*^?$xBl?K<8LvgDMJXATEp{c|fD55s8vzJh1AL*)$*4Dd zblK_zISmrpbR5OT*4x87p8ZIB!Co&-VB&YL)c8Q^lHR-w(k|)E%OLGCO!G-(4*}#X z5y=k#$a+1&pG0y+x<%?rcnu#LQW#PgT*(A?_u)+dP80u@slJI0K>Q}^Jr2?Ku)tt6 zvQzY3G!sh=P;>j?*0v}Zj7x@g^17kREqhbb4P**kvw^Oi>OwiER8N(`|DZMrpj$A; zbB0T5GmZd?q&9wmRY)*aPcG`Z|lV$ua&Q3A9>l`PWSF-^wnZ|tuqfshr= z=uDE2P=F64z_S;UC&Vj90q%@(SJS+7uAu!(d0M>4m-394)Apu;X`FcW&H1J3!`Sam zMlrXF-7=3rLgl+irH{?Zv=*uy(?Y>WrCbhQnQ;@G0>n(pr zaX=4AW3C2{`4UZuflGJjw(59RWo9tU{ynhyHAW-|3x^hLk?ykZlr9?f_bWcArm?n_ zhAqXl^y0&oFsyC8tByHtIpK@oy#wS*%UWs~Yyd%;umb0GORpGJa@s%Qv>5HI1Ob)K zb9a>c?p+V4*FTsZ#1-DsI(hnQ^ykp42b#Xx$;@r7ZqssT!82s%cTWYkgIvCOU*wg zUM^Jk$BKWvzV>g<$TJ1d@h7d04M#f!&~w-n2teEzzkmmb+R7$Ja-&sVjs&gstsDuu z{u#UlN?k*r zUItwVh|TQyvVk4ReqI9c-_|OJvtzH9ZIe=)c#i1y*y}2TZ{@hd#D&3x%RpSAY&~4{ z$`?r%Aas4}9<)^&?`TO`>4&S^>3WM}QgLwBZw3^p!+^C%fP9mNsC%f%XA0hhQOTjn zfcFS%;rX+X7h{X(3Ey9P-@=3JUjMrbjM{pTU1xjN)1E3MCHm=_ahO`N^a3KYGdCtA z1=zxHs!-cFRn`(rUqQ@WjDHo zhpHG`0&zD&{_a*?z5K__Ftv8-|2Pvkh~;0`L>^@QuWKTg50d=H*{_y=^N+J%T?i1_ z{B@mwbpK~Ac{#)Sk`@)zP2^2*Bta7Nm!a86@lMVayQy=TtwA_!=Mft$EVAtv%nVyH zpAhk1y58dcZL$-;F&pO#Rh?;n70)Y|j_~Wf4vlhd%l|lyRQchS z?UoB#tRFjUt+%Wwbf^ZXR)_6Hmin1O*r^t{DZtWc8z0HF)*gjLyPzj-I&X`zTDK>n zx{U;Ex@_ySTB9d#x^8>3)^E3})*p36JBXDaSZ?SkEC2-F4JdTq9$~FNZH;c&oAQST ztqN$!o~xjU>dOt^6t$OALKk(DQ@Ztk^R1FhAX^7KnhmQ_`G2wzRHKGs)!{K$2_Gam z8ltw0O{U86AP)tkN`Ko;y58+NI({;5VYDqlHC9j{36TiHVNb{qp@r3zGrx`klF#p z_YZabtNXuo^uK6XD1KYJ|F_1_&=nxf+Sf(Igsr*o+J&tJ@z6!AMeuw@fNT>fe(PI# z5`tDwd6I{%q4Fd|0LZkn!}Arh1|#t$&r68#=U)JM8wPtq9~2G~q{ohciPN9Ta5?BA zLzc_UrTtWD$H-oA0^r$dpvK1Mg1;2V1?!_JMXmG=+C!K88~9>% z2p3+Ey481~km54G&o;l|fp(eBL_{Y!oZ6ddHJf2(7+`ii5brC`&HZ%^5g@1eJH-X` zPTf*rySlB^#X(ev#t)+SOD&xhrYI%yfcA&W2D*U9_}h`6OslF5eG!z(tXTdgPXvTF z(;4_nixjWyH~{VZ+tdHm{ny1VQNe#>^n~Lo7xi~0>XN}l1lB;+#-IEx*fQidcl#%Q zi{x&hjqR#i#U4*UR2JC&@xrjBkScCucUQS085!oRo5Io8I)Q9IS6esfykiJ&7sdZ? zT=W0J8)(<&->|sDK)Q+{NMnO46G?X==-GtM-wuCY2I8n<_nWfiWuML7?hD+s_rHQ)km{~!V>XEkthofS^)qnQC^nKt`79%SbP2+E$ zId-ftFwK%7)T2f6704gltU?nj&vO&KjwILqw;H!UfW`ljbpdbwKLf}9vuAF$UJhf_ z2Db;WI&U+x)-Se>)gC1gfb0~e{?L46_qYsrB!%GZMArJf*2?;$-zij}804}J6EoU=Xo**K8K~O+x z?sN=(K!U5%7Ll(<`R}3H{o}gdiA*3o2a+IGuzzvu2y5#6Yw{-2?_LJpSPE19K>!!j z$jdexyDbywXrW|T%a3{2t;G>_Jn+%1|A7XOGW};IKyq9SB0h%>kMmzuA6=3^zpFwn zv-WSXJ}lAt{0y)hKxFNE`XiI$yKCdRCCq)11~Z>P{BJ`lk=GA*`-HhLFrEk`?L!CW z?Fl|Urp3_{SLKsyo&-??lrL{fFN(Eo(G%7y_dYajAr!P1byXX3#qeHMh9EMlHJAL) zCHr$0i^Kja!4nWffsrxUMK!}-C1#A>%4f;&KMnX_w)`Jv{eL{FIQ8$XAK*RyO}Jrg zo%L@<0S)~x1R=#Z|EpRL{AvHaxXId9e46Awr*BK|#H`F4M|Yr{<8dpHbERw*QXrw$ z2l7rScw(=ukp)UOR4&UyApg~z2#|95J52&)Sb*%q`EIJD+z+x`mkTUZC6eT`LQ^}g zevcjia3)LsK}l_&;EmJ)WlGSVab&sNEAufr1q85NckB!5?~i3hO$NUj9M^6UY`U-M zzJveDz3=SJadmZy6ZhCxgtB#>Fzbxc2!Jc7TScwYsSbCh5^ zwxZrY7!_JrhPo-au4!)awWXKh?*|KXD;(6f;Vkt(S~+WvAOxUo7_TZ{vRqSy&%5)@I6s#`QiIOJrhvW^g@0VD18DL zOuR@@c}+w;lNR2-C_t7g6N)CvD-nt2%PSGL8e>IJ=y_?De zFy8Z}8*-jOkz=&rpOp2?(Wf9_2PnV%?YjBB-{Bt>ET;mrx;cW&EC-Eb3Ig>7dBg7c zC^z_z&0;6{JvyczqfYMNHV3di*|oK9>dX8$ArF!xhtY%#koycD+{SrUti9Av{aE-@ zuF$5!2LaVF022dRIrD&UB<}!zr-tL_V2UItkl%dISXgQoU+5FH0g>+H-pd7!zq9_J zXDuO<{A`OlMRyypPzj6>1R3^omVq~tN)+5|Zk{H$3gw+D~3bLVN^({Ba7&|=&;2LT1w zzgI8ULIgSxz+qpV4v2Ef4e_#Ov*6~06x*G?415~yuiPtuWR3sr6WFn;y9LXx2@6oS zi6a*xXu_Mfj$-QCIQgikBKh=j<_Vr3MY-GCF1QJf;xpfM$rf>8n&kHGkrWo{TP+X! z@$U@h9MJkNA`0k$G@zfc1Ewz1r?FqLmt`o35|n=}LqT$q$TAeeBVGyva->WGbxnXE zl@PHV>WjVPBLOxNDGq%*bwYhRv=@5MmNpSwq;#}Jjona2p>(pviQUkbc$S%W*A~=0#0>dfT68(sD&e4zeDvXgf2gTVu9rwL+K;h-qp+ko3 z(l>5vlWhGiK>>=M5Qgs*f=I~*P&RPwA?lqD0=yc4@}{7R!2%7%6l7iyD0;f_8sIs8 z*P$cxfKfn4PUaUrh}DZy3mG+nx@yr2ag12QE@2Ej<{$?d_<3Zh%} zOAAA=;x1`fbYW^+wNmXbCCX!YA7Y0_jHs`l+TG6bT3bCf`xz`wN~UhIK3$yrrYCH@ z8bph-CQ)v8zy^C@)mGyGyXYogzr|PR@Waa?OxSfvB3Y7?_SJ^s(vk$H`gqscWBTKL z*%M3QWydfb*G#O+1S;Z_hH#uI_Pbvf{djtRq`ul`n~V+VcTak|btl6{qM(|H@7s@i zuMIY%eswkB-bvE29=n4F-tSNT@(cjB(uI~TZbM^J{Etz-?yl;B%>*>#Xqag)#CJZ3 zGx#M{Yt-TPuf>Wi=)#)SU1oQD7ae6$nGKy*INypbGseCx>w)k@RY%nXX+Q70?Q(Vv zOm}x_koSJU#+Y3T)+6Ti0}hqu8aS zmEu@ahxY{?A}+Qp(pG$W7r>7`Mah2Ox$Sc9rqJ@(m&hlYL7?LVtHf7p2QSaBi~2dU zaJr#_Rdsl7!kaz3V=oW=pGn!}@oG_|;=xwm;B&uN)^5(!X-nW5P1VXjxGv6^690Hs z`knwMb2Z7X{ZcCSkgV@d+OJ-Tl-bRF)8nbStKTOVymzm)FdqMKEU{jfN`Y3B+tBWm zAL_4YkdpJ6`}!75$Ie`jp%FAj4WVf?LiMA8F8?%JQsa`n-^hwWg!`TyMFm%o^G%E% zWME+TEb4Kn5+Mz9yzh3~qt8W6*&!QCt?hOvFUijZsSi%kNEmnQ0^L-)9Ph%LE3kR$ z`ZNN6z=+07Pp0o%pS<%{*lK$&cfs-=FYAeDpB=FzxD9!=SDhNcQ6kzT2d6Bq>_3{ z*Uqr`Of12|h}irS?q#MP1*HdRuhI&DpUuQkeUMTDiFo!ok6a&4mn%k@I`~E?Zbzj{ zlio6;Hj<#Q)@iXdezRzIH4*xDxs{i7%6X!%+d{#LL*I;9j!I!n^a? zAQ3CgAVt>MdZ9IxKunv&d0V2>L>(ndABEhYY&RAdn%;q_R0(&|ZL(#(clKYtzJD$R zKdOB1;ro7J2&9fncUwIiYyR2OjcVKZho-n@`U+|0nT0h!UTHl^Dgb`lS&_~7NYXv4 zFTBnEvkVl+^M%CV1+3nAzJ)<@Hhs~JSKaPe>A6^v^S282xw6(ro8_V(j=sR&(+C-qO^#`hEG84m{E>%h*g{E zY+UF2l74!_JT$#f?mV}sAhUTq+SEJnbQVV)-#Tirrt{@XFExXIfL`v- zl(3pKO4#s=7obpXd1a}XP zF0DR{v)S(lx$x2Wj9faHNeb(+8^UTS{OKTjvjeV*X%h?k5?6k|mR!`K@=ye(x z;b~JFYY#Y+B2pTu!F-V5OlNOaqXut{jj%KFXpdBZQs>!>pFPF5Z%Qo`)^}Wc6Z_q9 zpkF{S6z-0~jN2ZMJC)Me=}t}Jk`k+RKjCC)lxeesvz>psopeI(mf5XqHiNHUkguam zlCwv>R`FC{0T(J^Fs^U~+gAFUfV-Ngy4|oHb-vxCHEd?LWctF}LGD%0-XWpE@GjLc zuBn5}it1DDH5$z2pd`M2Pz$k)Nz)gOp3bbE6jj0U*(?HEBMVG5_ogZV>1CbehbAAU zGlt4(hixY9dim!{y?wf#>MT_yZ>p6%dB;#LjTN{1gf=IN2!3`i+Z^>ez5O>`R9y|8 zq%|r74lI0Wo@1AB+OiOfgJUKo>DO)W^iL%<+7|EJHJd2gQ*wG@4D%MFOz!R$xj<_f z1PuG!*}qiBcHKg=Pl+5>^(|Vsn~oI@D%Kr(sRFXc5$ml{zNRqF3C zg2E&HLl}$OBaK5ChiGt9408ismd zZJ+x%dI)zn#1i#!@s|p!Jm@j5eHu7MGU7#1Yt+WOv2Dmu0A)=n`W5}TTQ__%|MPvu z?3ULl%?hdOd2ebI^Ycw_xQU;uTJkAXcc7!Z7%ZO&6f@s%ee5sszsXylNG zj>aeCCJtm{+*r0gVnXh^efoV)j2$Nq=kIk1HQ5Sm6LD1c66WmBEhCmCgEN$(S2!4w zGGD&+_LC5fM3;WtyU)i9$ThQ6B08E9$pon6Hwhl!@_l?^AS4K(B_an=mmQHU; z?LQfUCa7O?G9a9dMe_(AO`L;GVcf4Ta(2?CXLT7a~^#bUE5^?OV&Dz1SUYf>cdI$CJB` zN4TFAZa6CdpD7@!U6%0l!*dI3R{fyQNLZ}Wc1v|oS7N0^E$HpYaoJ-^p`}<1w+sfd zYuplGO5$LJXzWA@nC(J=^w(%aGe$7f&~ME zbrowlyC?X%Y`*leSYd~{?J`wWIIa8gosNcd{^uD+4k}TdyBaS!z4~w1zJhUjsfRbC z^Nr(I=`_2Yr$XqA@xE{w{87>STN#D!p8LHjUNLm+;Fyi7^@UbZ_}NsZLHUZB|ZN06d9jX8Xoh_o{HY%YwD~hJBXzrvl=i< zcD-I1&u+5HCl}hx@4$K9O^n_M)H&{PiaVQYH|ViXC*LVUyl9 zOzKB65j$lPN@3~|cb}2%5SZXB804}rWnfpkko9hS&@xlD#~H2*NYdSkF1CCTlEX7 z+WKK64sDsn9Nw6ci!Tdq@vVumWmTUSwH9aIo#p#ta=W0cMkRCXGtX!Gm)(!w?>B|u zqDb}9k4GQ8ktM1A0-AYHL5e=u`N;W*{BFR>ea7CJNS{HB3({$WxlAVVfFY$HjSSf6 zUud+Q-H1PujuHEQoex(fGAm&(C~ZPx9rs2fF3_1y!?PT!7&G`){KWr_xqwCugaJ+a=| zCB>izjOVXgt<;Z>!=-xBjV!`dFUAE`_8Tcdf;a9@LSw(h_b!nijynfqJ>AklG2XRf zO2z&*!-(}m2%Yce=T`Fqx8Kzpqh9WsQ0!l}#^`?))$@rDXC<;?dWg$@7E?TU zmXLbZ_-wPqfV1!1--_ScCy`h@<9%iV{UQhEg!uM`Ueizwv&~FkX{kw5H3m{OKYD*78<%Ew2CeC?A~_i`7?C4o5WY zCbV5%^9nqiBsE`0lO!j!s+kRE3xQ4AC&1_6FS08r(!9&HV@w&_oXXuiGngsZ-P5K= ze-@0zITo;d(7B!W1*$8 zP4n*TN3wqK*hcW=Q}eH<_Md}t`a?OaA|xGoy*i7oB{@7Ko?|p%4v9swF~9%!yQeMR~6zWpTd%xncgaUsqSyNz3+S( z7XhWvd{M%^-{NDnUPd~FX}_{${xxg;K?Yj+3r6)p-dGn#t;!N8ykhN9UxBAx_Zizs_);ym7j#GoZg3YnI^OD`W;o4N7#h^e9<9TSx744b{qVeZJj9 z^MY?1lb|<+)&FKf^nRvW_jHAvY5=}`0JYqN9Ip}-YrG!E&uz{uUpL|-ReK~#?EH&+ zp7~U-7J8qraE+?;D6NGn$~=5Nj}~tiE+cLn^yWvB41Vz6X|Ab`oW1tIJu04y#{CY_ z&NHjW7&0O#1~Mt}eEz)*pY#U4WRR_(h)f>6_h(M@?o9fSY*j|rb|$mal2{9wR3Jue zGP^0AWF19hb|=Mb(6!fopsxPdLZ-q~GNY347VQ|*;j_F$8s5J1YCB&w+4xxe(if~; zWY@Et-Ye%%Sru#E$lnHQdIq<%9u0awkI;n`5KDNuGM7C#FE+$zxV`? z()Kx-G{c~|F)DAu_884~)Hw!yU!80V+!{v zw(Z_SLY-F%ac^s;PZ=JqyP%Txzc<2kHWl-r#&2#eEs9*SN1GM6NxJo`g*4`szDgyj z3-$s@olVOUb~AUZwRI_@lJgVwhcrEge zpCxc)8$|zTTb6Lb?`__o&JJ^6FBWVe4liYYar20rysWQ-s6f>b4YuE3MK&|I8 z;|+2ib7~g0K(oR6o#==tdPV=CYmL^})5{i2wm9TnLb3N!cF&dv2q@A>%vLBovkaFU zDD^p}T>L>(f!F4^;nf;@Q1Zt8zD*Z&d-$}%>IA=2cvY(O8U2hT%qZMDsC31wMB?ES z>FH0;Rs>|uI=P|8D2YW1gK>Trb%bkiPC7U5Je8zW2YzVCV{$js{%l}x2HtQde=eNn zOZ$L?VPg6s?;u7n6$PXRKbuOt930OWO{IQNyv-hNO5NIc{M=c6?rEBlbDw(XR<7~g zoxyM{o7+`!u8J6aPMSN9i?JUI`;`PsMM>}p*pZpWc0KVJQGVoA=K9zsa$)q&kMF^Z zyCSMS@<9PLgP}7kUtq~$v)XFD5uL{=MKtr_ydX=|wK*yEYB^4V#{2b;J9F;$5al(| z)~h5t>cd`D@Y(+KSE}_-Br1@Kl5(+lIVwNl98k4VW=OpvVJIDJVi>NU_F#?GhBdWh zcc|0?Vl?*puKzVPjp^g-3H|o`l!=*fr`_kFpPd$;w$Dq87A%-Mt5~T#>QEf=Bqy%O zY>j)%va^AqrKMJDZx%fL_=Z6jZ?5)Jt8n_2XSQ>@q*YW2QnvBl+)*1f$J@r=*P@C~ z0v5H_1CFvYR1I9dGktWTiKm>Y^ywejYL6>WePuXWk4NH>13OtqJ!n(yksEp)Y;Eu0 z{4qa_HsU+(I>|4pdYr5-aDit$@7BkGPrTgcL&au!T8$TAR-@NbCu4?^=sVLMWDypd zQtvoa*@u2OE;()}J?3TJO_b@A*|%XadJ__6&=u1XoH-#m;{Y9Pne&w8_HUOZ>3thR z)Ng0gwWZb|^Yh#7x`Nb43;n#+4@1t3`Cd~t=jeN9@NtzACg_3dLT6=rGAIhRZ&-0J zQge(r4ZPeC{APu@5~$g5a|fdPq8xL8C6H)dG~h`7p*~xTu1r*!QKr*~W9EZRnSrk@ za+8kvL|VQC5U9lPCmnccpf#b6OGST)G`IV(A~EJF3IUAguzRr0roy~y;%3&Ih)m;5 zw!gf-X7m)3S@Teq(_Z`L>J65TMiT+2yV9U^IF78*2?_H(+`)8W3q^a+JN@T0zPTg? zNh-bP&+GJvXU2~t;_;tif^w_;ciDy7hh{V?G%MG%i};G{8Y(u%Yl?BtGDR36b7i`8 z(WE3sP6}@oOO1KtgmkDC$H->mK;|cO{j&C$?F?GU9tl1f+fvF{#wwxeC#z{>rGNeKxs@IxF9^+}!p6kj)H-$0%8yaoigs;IU;7pI z#;RY(16w(={gYY_EnobG@mtfkWcR7QnXyEXqkpc6`tWJWHjsAZ=(z~ffy2zk@^WdO z?uym9H<*~dAeX#mbV~)`H`s?q@3iJ0bH6U~!8_}TOn{}4uOq(iNx@cKg%b>gb zs+moQZIb`_v}A6WO~BD^=a%mCiRX=yPm;;8n^0sQx{Mr$=&Co}uh+Hwp-oHl1bllV zprh;CwOMfp8`1OzA2sf$iD4{5_pG)ulvTw|>UG8@%}Lc+YORAQmzO>ym`86Y!vxn3 zuHy)c@1JoQyTgo^K7YsF)wQrsJn>-<#*N!@z3=PnU!B}}pVBPayjXPaap`%aqqI!Q zo>qW4vxQ?NS>=~{bBYj`q3PSP773FAs+vH0GIlh2%~9eN`W-d9scyRxlK3z4eE?@> z+uzS4-q4qv(L6BH+u+?9L}eZo2P-F-Hap9T^>Y54%Ri(mU>9L1yc^sUBqp$HYwsXU z#L`vU(~hsy%3gIaap+KcAM^a$LxRDJK6d9_0^7!KL0vxL+l_~9%fRpFT<3rKhRo^@sMbK&b*pA*!)Ke(tv+(`rD}4NaUbFhMSv&V0T7a$*An`Va)f^%Z_i_p`zHpZP^fl?VQtD z5HmP72*LKjg=AG`OnDHP9h@`Bb0lNsY~dQ`!u^3g{e>j_c}nW{3&?kGq~gn_kgrN# zF8Z=$VET1b#Mw(A`;4tT2Yyt-;ssoS)l+1(zDNqqpJxO1;CS7sYLf@B$pgT#-0AoQ z=Wq~2A&G@H`NcSy8`~z&>|V&;u{c=oLe@OT>$zfbtN~gSu!u|>-f0qV6BzKiH-VqM ze*_^G5X|ecJ`pBgAQQX0;x+e?{3^doVg(YC0&cLsht(7=5{@4AbOx*%wq^gxV_z; z;r?bD3TR3cNy%9@w|z2k64rJ#g5#4&-7{I2`l-IiGOb)ACQ`9P42;+k!{4~s849jl zNP)<1x>p%XFSbmp#|L`w(x2^*vTqW?HpyU{1-s25Fq)aq;r{j=dOiZSJA_CwF#Fi? z+hm1JIOHKKhwU56HctIvxc7^@uCa&*rfGkxQa;;s+^xHKpS0wn`%L}KrkXa&^0rC> zXVWEYzCQ;r5fGW;AO@X(qzZy#gtn(0lg52XI>vG;!X~WJn`g3F34qu3IsAtz(0_w~ zM5qQGpUlc6oSR_#2ssHE6(jYoVa#wvRx>Du?p+37y$;C8pmbq50qrBEQ~;YgX`BNP zY2aby&qyAHOfHv(>VP7Hb}mswBk)iAz<$YR>;5)$>(IMkD)3;{07-x*=T~z}E_Ajy zr450N{6kH3+LQ>OThlV-tp=%R2ItwE#{D^oSHA&jbdZk0R^!y+g@PiDu>UZ1K*)nt zoYTPkG!SosP@O-9;SW9eX-yz*q7C300iYq}LqZV@dBmM4Xh)acB;y8< zkWUsl9l4_d6y%H$_)XbL&XQ>%;SUr=2qBk~n*b%pb%xCa8xb6~&JtlOB=O3$=9a!h z(kF>LY~uHs{uNFv;S81AKo~Ftg;P{GwViC{hZ8f`dH99L=l?7s7R?=9aGsUe=m9MPU_wgn#@^;mx=At+ahF`zi7Nvqvk>MzZ;G3T*$5dJ zF96HUT$YoSkeG+z>10hNTn=C%0Fgl2AqwYU2v2}o6!8ys2KYx*!|z_v1T<)-F?ykr zKUarLsNWe37^~J&^J&hPhk-L8RT2Z&Aw6}tt5Y6sj&{s5qVPONCghta^exO z1MVm$CbR+4(G&hiUyipdncbj-E%0qU6pSf_tp(n}iqXs2%&-!S=m}L!;EA3v zM6SBcc4*2~$z5nDK7=RrY(hs1(GGub(y4k1{ctMLcC3hC0au(RUX zCNWfW%6~j}$tQ*G(k<@z%#EP>L!poxTW2F3=zac9Y&Mg~lUr}+B#{0w-X?MIstda- z`Q60GbAc%4{bo`0{yXvaFKwasDjDq${ZxSkDwso!`OcZgkj-p)kI{fYEIjK>t=u?C zn}x?u(Z^RiYM@zWak3L2R>^g-_e7oeu2bJRF-%+7 zG#yWw6`VI|ICl(NoPAKZYI*`5Y5VzCS`&HOgw*iWmeJi z?|AyG;B9w>nBh5)hmLvG@uvm*GOIzjYl{y&zB56IZ_D^iNU{2EPBqyI(%d+Z}FC(;TGW zV*$cL&R@3fm+Y^so4K)sGcIOO$%nw}EM^)Qb4cu9xqZ)TUzkD$FFnDsX(T)xHAg~4 zI)M!ku4OpzG-gBO1l@r28zY~QYjl8y@grWCAjbG1rt_vo z%O;bQ4uLZ}ZStY_&pT7&y!(w?m9LsUf1nEe{Q`2LY}<|lS}MX^Id-HT%T5#pWjn+h z#1ryG>>4p0eFE-ac82*Y#Z2U4c1s4#0VHg_V>Jwgdrt0#D>M?nJo+i`Z8#%aU<*W8 zIX+W98Yv&sWgG^j$)!Anz|Mf^=vkQt7D1a_jNzIL5;XP%u^`tY9eRf9Vkj?0exB+I z3y=#iilj+g;4N8imc$KalVb-hX3Uk#ZtrS|?rMn+Qx-td0$)0fiHIaoB;OVoQ(jmt zcFo?xmQy5o9J#UsoM&O%gITcpAw&Hwmpf$C(|?If>@=iT9?dB2%F5h^=@umymfX=R zv59HwMzC&Tv-tdBwZsEBL4HMHc}3wq1=in01RgzmoQ}-QO^jheCMEgQW5c>}$fTB- z0l5eh;;2D()%iRJaO0_OpK!ori%k2>eaTb#%!>zq=ZVMm-&wBlNJ@yHWW9+zv-hvE zeXiEt>6H$g(+uE|aQNVfoMVNp_CuPJZtTjoaz%P-LMzOfa*#duDq( zFgSa9^ttWyVz!sQc}Ci9oQ3glQygT=OB!Q*@bus(fN-`PV8Q;x*gO06U|IC+I?eP> zmj||PzYZ`oK#!^KYW&jzOolyr(TaACRbOGrhw@Q`rLSVdezV> zU(~`ekqKO(!UzoW-J}*~)@j z4I$%LFYNMy@mtKhxiD4Y$Han7>0$kaK?-TV@^HOZ-{W4k^uCQD=Njn;C6w_Dy>C|} zNQ^Idl6mQk){K1Eb(vvl>(6Id_ce%Xl3}Ojvi3y9v9H9U&7=4Fz=?f@;yO-jhCQp8 z-RY!_!@5nkFSxN#!i5>f=W(ab>bhZS9Z&P>`6{@IY~!%zkN$RCr^9^G)#-tn#gpt* zlXzmb?0-pfwU#cb=uhafmg{R)Z;eAr{QfTSsasF@#{2(l3Lz12`*ZbJifSMf!|>9 zdz+M!fI+>tZ|}?AZn;|PR5^aG`@Y@Qy?2Es+~2uFz+mg&3n;M6Kn$f>aHR?Aw8C0W z-1f%<`t?npY%;Elf zp|syed5gT4ELNY;cFEL@^k->&0%y$S#EI+Mt3GnmNB(2g=)Xfiy!(Xy+~Nb1)~tmb zc)gjGk=p11CorLPXA_tSRUQXG!}O&eLX^1|N(WYhR0tyu-nFoMWp&6cmW5QZC*)jc z;NN#~S%L-)$e415=er!IohmOgKZ82rfm44r#`)MFmXOX_90pc|ECLpAJLE}&qks%@ zBR~~iCbk8>evl4LwnAc}sLKU9zAO)sDG9k5xpEe&Iam>Lu?Zf_GQ#_i&bk4=Hxx73 zK(l}~UF&->ZdL_j9YzIgPC4+_Y!u-}fo{0ZHd=C{T{EE}J%bJL0<{7Q1yOdw3B-$C z2m`S-OC~ZgqVaIY#mYLj2Jt`}RxXm51&|;ifk`N$BucOoMA-bD2X^KbtVM=FNqC`I zqYDgR0Rb$5fnh5oU}2V55e-;H96+d`y1BjTdP@ex6Lm=5BtqWxeL#z480zlA1|6rC z{lKk6jde+j4lT}RH`v_GmhUce@3IPa>FN@Tdc0uSdch30^^k{8;G&dV*?0s@yxK`x zdBPo3<|^=3pQh``ZZ`F635*sTxk79im*RYh&bb_FZ#jRg=KK8ugb%`4H@!y}ceXnH zAND2P{c50k8Ob$g^K8=9)wsPsV7t+Ff3j-QKru`XHSvuF#+Ugg2+;872aY^hqwk_I z?MEWy#2LC{gvBSBgfaBY@z7tF6agQ*$jWvX^c%6pQ+FQ?s~6s7XV%AWr~*gztt&oN zX&#-_$K@OK(d&Hl4j)EA70u7EO4t_sV?5#4EqOO}c83w%n6}p$Ht)CT49C{%b=f(d ze4rPqxL^(M8$_@wij}r1z#Kh1^=84)hCVo*Foz)sTZ>9^Uanf_k7{7I+_3O8!aEhT z6fitbfBS`_-hD(BOs<1h(Dsh=hKh*|lSeuI4eKZ;l{1yY{3B5j!D(FT9R z1J(qjvfq{-z;!Gqbo>LS8ZM{qrbN-CNLRjFw!Tr)#}wU2A|F*pBvj)kc>8$55mB`gg;&>2D7FyFXgX{$>HifOxr4^PKWSA0%|$ za$9G8*qIMrZ+3Qj-v3|uQZ-TF(-seAP`L26=0ygtyIwGU>#N|eUhU?X8v438tP3yn z7Qi!Zy=b>VY0tsa-KNgtdnSO_e@x5yTJd#II%##cJRdN*B2-6igNrly07yI&NR*eg-qqF)LJ zMaEb0Ei*Kb{u2%<^K9pC=ofFjor0$hRqFBcDy2gO0?=7)b6&FaSzRiYt?t)i)V?~7 z*TfA=aykmbcU-6@8yq|8&DT#)C^uTaoO0!7=@a4UP*u57Rk_Bepi)iieq+#Y=IxWRy)##4r_Zw|MSe)u zrqpu!PA!qWY;cPz$ytV43v0ob4U52wC3@q>VA1h&c1SOIry5#?8_~h&apw$bS`IZS`YKNMy~#1we5JDEVmVzBJZP*Fg$Brx4_SB zTVz19{H|zjWa978lj1D8;d0*opKkXtoi9=!HPwtS2Iwz$(fi_(& zA~m(L#FU`VajK>W@T5fdFBw6zF^1zKkdcoiX7NE+idw8oWP7Lf= z{Zd`^pXy@QdRe~YbgGx`OHQyVw-;2@nM~vX3B_0r)@2{-c;eY!uXVn?R`{@|?$IBs zU;hmP0#`8%)^=mAU0=|oVSeS!YB>&6ox-%#1_oQj(99@%P$DXVHLF3c(Yf8sCfFMY zNH@~b48N`n&dE9Fcy}7GLD-M++kY-M{JufKs*~$!4~`vCLR3M(iDhMV?GG4haMaQg zBj}fQeM|zC6Ed;+b5~S|HrE@eyAeL~d|x+}5l>bY`Xn0j!bexnRN!LPFEy1G@I5E$ z&Me&fH>Xr`c}z(>7d1*T*J9W$QmTT{rsAXEkg`Nxv!5x)IW03U#sp;YiV4!)4dQ{vPIB z4|B2gD4HSX6d-pDWbeZ;=s-1}r<&x;;veA^O@eegn{DW>IkU!g8&Ap%)uto0t|Rpo z%ENihFceX!W-tmLxLVvBe)2F?zFQdOwBnsrY4w(7vqV(YD<^BV(dV7Cr4Ii=cTVhg z*3Xdt@d@(tPgbSBU4STO_TB=B?6R}w66u@ynj_UI znwE5~cy5_Dya?hw^OB4%(+RPwHCh(n^s-`++(zcEEiAj-nLg;o6D>5`upJq`nOeNa z7bY{}1nJW2wAyYZwALP?Xo<3rURy|%O#*OwYJ&xBs7cP&s(2v;ymAfjBy)Ej6|tA> z(2s1<2G`HFg^MkH`M4y}{EElCi^sHz$LM@+7tbeF28~J{c-{~{ywpUw>RltcoaXWd z)f?P$Fb~$=LrTJF0KL|A8}td-+u3>zHn$CXC*H(V9PY<qMA3O(tcLrnp#OrpQiQnD+9#Vf!dRwnF%igFY&1siG3fZQEiXW&sJj zpfOfUowjmf5i=n9GnhY$$p86aOp+uMITIuFDO|u<-#!IKXk=rZna+Fu>UalL*MfqS z%dh8%Hp-vvKrvRSfGyGDJYFrSk^vV+JL_(zg<89C?Rhl%SKh!7gc=zIv~Z))kYOdwDL=) z6O+GVrqw<;F^vN`rJf=`26;04CMuilGH65a=C#s_UN%H`CU*M5KF`{pzV)FsbJC>4mG^g=tyrs&vL)a&&JBJtDJiqy61dB2qC5MexcY(Y8${c` z?$k*GW^o-$tbc9hBS8Jt&MRh&APxE~A9XGD z{+wMDBcl))u?1r`GoMQ;&C_VZb}U>h+wCb&^~%m44V$(T=I9u4po2*6&FB zv|+AVDz{>^hPEuHu%OG*OKJPwz)%)*X+{VdBQ0%Q@Xlt)qY-CFQ+h%S(CNs|=sl0d zRz84B&%2yyeZ8G7h#q^Ey!d6MMcQhSSy~bijqeYD(#2q1y1@TDdFg+k#{HKC zoN>0u1zwaZU#!nd4&UdJ-|Wl;`SnRGqmcF0J24DGe2d`4nO~2Yv*7q##P*(yQB3S9 z7KP<{vB?6ni7U8Kbemd0J9XsFWx!h-{}CE|9tk^Zq#jNVb$CKiK=U{#_FW4!g{HBgvwT{XYRBrU zW49nz4|y)uGjhh$-IN{M>_Yh2AkI8Hxg`r zz2=j_K6df(w! zq7b_yt_%u&QR(gh0co;HOv|wl@P{_CTST%m z^fG6}TURXhT;-|a?hAWq0MYXA|Jb5r;v}vBs zuJOz6YtcT~vb%UOmhwi{x;AmHq%lU@9MmJ}%)wE1*0bZz1EVuh+z%(#1BWtx4Crne+I6`MlQHMO{ zEGej&MUr8R^8!z@^~t@@Dh~-|>`%^dWvzO@Mc%F@Q6O+bQtTnnHpnU9g8qT746CNK!S;4O&L$u6m;l2~@g-l1?q1D; zUTvN2P4^`mpcsD434Su4yOuU{jOQEf6b1To73%L7umH-6#F51n!BnK)mM_~sw+h>& z-Iyj>x~5iE#|o5Y%i)HJkwy;X@G3;K#{da3TtYKMYT?Yr9hWrovPEKdnEsNFTmf1u z0BM~Dq9?D+I}mMBKsB&ewzydu)%ye0O@vLCqB)6XtH$qD0m30g9E9Rc8>QX0 zcii|0ZR`6#YS6@Br&@k{pk^DoJ3|m-xINis&0V1fI%Ot1;A{u6?Nr9;LbQj4hNQ+} zxG{6Yha04i%ne*6w;vGJta>N;2lNejxxw|S5|1;YjZacMXjxSM*&D^pH$>n?0 zDYMmiPkuF>*wFxAs1^6Q;#9L}D-#6$wL6srPVyePVrMS6G^Yu8c9>2f3t{@GIHfs- zJ@UOhuJ3(@ahPFd?;|rHapzx6dU!0tKMj*WbrQfu%Q*norjYx1wp^n@qSGI(Lw~z~ za!WN_>WC^YX_b|cgpG;PDf2rAvzR_nFm01QVd{83dYlip7-;N)itc!h14Z3Y-&v-v zHZL7OwQneWC~ADb2~%M3+J-D6XUFQc4LvS9UcEUh)vT8Q`(_b&KzqaqD)2V-)9?&B zI*ybR))LT>=}+7uMC2YUw{#M){5Ph4qWBv`e#847)CQ*zcsA@@YzZPhG!@GoKekS$^WtQI0@r-M71RZWs|8+JRGx6xJ@9zowpTBUL2I&8 zirpV1t@FCv;Q`@p1Z`K&;P(&-9QufcZ`n8tMCoV#f@s)RGmN*!m+= zd_wI7Bq(*~gId_yLl0q4xei<`)-df}cg>h3ld6lxro$h&NZOOCZAYN2=?YvjPO+x{ zEjsz{*QLK-KrsZFJ(`nkR!CjKRGC<(x0d?M*dN-wO3F9X?b&dM%A4)y&BO6?^R~&M zGoaTnKOxW9y!Y|Gu|RG>pC(^3?k##OY^K`E;9D`j>|LG2QULhy9Ux%+fC+(tOqJBm z#SsLcu)h}e*D~lc$b@PbT8+?8XY{{%XoiVC(dcgHD!(&kUH+D1nXh(YJDZUIW-q2! zhcIF@tT)4Yvyu!d$;eR_n?rAoI)j`(WOla$5^@z(eVd9t%)9hKaT&VJu z1`us3vYg~ycUo>`=z_1d9%mPUxW?;6!t(r)g7ww#K`L)fFfRr!KrlVVePwu6cr|V@ z>$s+co}wwn+3uhBqPol$EWQiA9GiF#OsQe=A=P>stFzl8QVZv5BXgsj%#A18=8Kcx zk0M2#%DJ6&#W|I910PuP@KSYn3_AE&M}u`rws)vW&bWNoyO=bRxTKS=*rB!V;u{$` zG=jLDLY8fJs;I(2mG^9$753aVvK<*e?3DW?nJW#seABEd4c9?QdV-|LPfOqgu{!*M z!$=ZPiz^Kl=JonU%e!Hl!OxU{3DJ%hd|v-};p4wwW&VBvc?XA4N&#GVqQG(Fcg;5GB>T>ts@Or-?2wm~rvOj`|2zX1q1Srh&!M zfEUxYH759Inr6ccH6|9h>v|rS&aDv@$GE1c>@O&Ma&) zdB4WGua`5kd_$Y}Do~RPe8c*#AT3U7`JIIWAD!3nLQE#Tqhgh!vmb zl3ZK1VK4u!eEE06)%$tZU|!Y+c5ToJY2{j{VC6hZHcvzNtdprcSTT@`n^?~I&hbF-27V?DkgH1e_xU*g*tN<43FzO6gI_00OJiz=&6 zzVD}pC;d6|ot+sOpSgqqO2YH9=ietXu#?PuX1si|*5G{C#gvjg4Pkvy=XC}HVKzra zmO@`<`H?QL$Yxqu=y+vRf1Ox$<~{LmWU1Af2Wp-ajGD&&KURK0?zaZCZ9X*pZ0B7o-^XnyV{YpZTb2RnMaBfE3^vV_r>SUu5 z^3R<#Oy;lrnRjM2Wb;1NsvT#gXIdBy&D08ts-dA`+(bMhKzWWk*SqSC>q#nWDK-^z za~&wWuPC;CQ~a#JVpZ9ljNloDOCzU}DN>(z{$wWz-!pud4b=uKIfPkopi`i}6WHpM zCg_wO=q%jmX;E#7+VWKQrhV`n{g`#_vrKme_R1|glN#0904-?NtrW1)Ued?i?0mvv zd12i*PH2%XDMT)dzg!ftNg%qRD^8{+nc zPrU*>;ZabHPpal)LParzT}aM9lf*9CC+qGoNNG>Q$kHS*Y*x`D6~4FvpL%l5Y}uxv zla%$=Z=X3mmg&*y<*slzv(jK~X*Ug|xhkT^&t3QRW-YFM{kuO`JvRJ=HzV9T#miaD z6=8p{QvB@#NIrOepy$ZixpvwmdCSzWu+THfK-uHNDgPEq>iiP${pSIe>{*AXVdJ_g zqcQdOay_lN3BgZ{#Yw)_WP)N?tj$XU(Ebm7)zPwz)o=kky%08-F5nUg!yP!4bb3(3@V8O-woJYsneN(e4U2ITtAy&J)W6H{VvaB}^aZzAlP zBQJ$|%J7c3N93vC$tOp4%SoJSdy%tz;3zE%@L@UNJ(X(*U?X5$v?SD|V*RV;Mt>mV zO(WosRdT;yKxdsH#!*uJL&JD%gq^%jzdn!Alv%C0up>Y7Xv@!%m7NdyO{Vpw8=B&> z|7#jo150aSK3$aN!BWy=ois~&5KOFz`Gm)n?_(|+$dQ+1c#9Wm!A(L&Ulv)?y2#;q z$m>lN#=t_g6igC6$s#U4Bo$xIXj6>4r-W!32g$w5T>i9du%zyado8OMMhyy+^0}H^ zx8!N}cr;tL%eVqezir;3dcsY6ZQX4JhUZr;MOJ2p2A?Yt-W8|g=az@8+JMz$LbJz` z&eS#L39H4h?Y4`yXg`V$Knb{9Uh~KlmnXRWa3O1+7iL>-YaZ^U!D`-gZC5guYpqwH zmh3G!d6%BlE1+svidZy^)p`xseSt8)nTBiCB8D#RvDW>wK3Q?Sn)4q&cilbPCLKmK z^R9I-csyX+?}?@ZKI1Dty^-mEOy2716F35!RDVs5z-E2tX3|}=np3kbbVI|lnj0G! zK9w#jVRva@m%h(MtEQ`_&77TGtPpbxEzeB(K8?W5Ev4Mup51KA#+zT|BZs60@SGOc ziB;*FpP)$lc6ecc%m4Wzy8nU7_+J;0uiO5SFmc>PMy@=3S9L~buu!62gsq{^P8#U- zTWqKP%W!+KZF#39)%yCm7Wfd9+iu_Dy@E2O#)i8NIUO-n&3a!*nI`uJE zqP14(%myny(%z?FXDb}ZCN|6#qm~}-rt08wzjEi8);y2C@JNX?Cx*7? ztdsKq!m#)du$3jJHLgO(jc^@eXDM-KtKwAz%_}%n&d=r*dLB3@=?1{>c}}?td$jHK z9fWzSK1IFtXcx!pweS}>Fy;NufwusCho!Gz1u1NK36mc=1Jid{#XI>R2Djil3cVl6 z1@pqG;C2OE5c-~|!0XLSFT8Hhc02A@c=U^A^T6X*@=3be&wbJ>NcX4eSV4LYSL&nX zku7cxZi{R6AJ^um`y&;Du(+;(6uSA;8PGdTez#5Rl*nFM7#AAH=_oiQopUilD%W?; zd|lE9fGV%s$R1ws=PJP8Er5=!^kkL^IzI(H$j0Cp-5LI5?6b!;2I=Es_bvh?1TC%9G=YRsB?0C!Bgw)8~e%_DcD~yz;V@O>BtLIY(W*& zAXzfBbs|f*40mw6x|X~pSU73G`~fsVFX=UqPfIae?r?BbywYA!~$ zkn2sUgVXHEOrR$-L7%kT(#>y)51jDU^Fwv*lQsJMK$X{Xe@l_nggy0XRywM8F+CBc zWLzLsYs>dB-!u%V{&Mfm)R&zxM*e{&?4*2O`MDSMdLC)uff6#(c>LrJ4O`2H?pYbA zlYG-a8W*$XX_EKwylE$KC&hCzot~vqUs*c48fARApuA&<4R)f2>)i5j`f)pQ*38fJ z@$-XFV1FPPO+?_JNr@Uz%C;CZT)+*>E#Lx@g3f!DO`eiUG!KF-Y;cBj+^5cb!Uox^ zRIWXh49+fU+H%;mknG6{JV>-}S_?{Dwbh3Bm8$gy$TR)oZJ2RAbevR}DpZHgTcmIj1@Lv@$C>%J5;o(%5 zDiMQQ2Ir(a47fMih5)hOifh&%+$_G&Q|dc3b9K7huaO6OI_qm0%QuRhp4GW5PwiZ=^XkPeo+j@w^0HaI z&vDW9*r8l_r1*FwMatIm@Xt4aKFi*&q7QDs4u3O!yi_cw$e@Tj68i)HY&&Nn~_3 zSqIQ{iJ$#(mFt~ZtzVN7!ljF>4~_RP&t0AfiOS_Ys<>WTH|x<)F4sfGY1`hp;f7hL zdtWcw|CaA1w00{xJ{AagNY{Sj`w!WPkAcvK0g#vLhZB9nb}l3$a`mGlsn>XWMQ^jb z@21aK?^oe`zi|;6An3b|&2AH%Z&}C3qCcg|L!~cJBnmB6j|!M5t0`R$(97;zyx!Eg zDjg{CK?t_+Wv9NYnT$n}5CVt*j|kv`VM^VNJN~(9@%IaWF}wX-ei^;yZdY05C|!}< zHk8}0|K3f4=dyBJymQKQbVI^j_UNuxn}D+?UU&bJyO!Ht0oCL821@~@yJINmo=Bt@ zpkeq$hA~?R&W`)OapMyMn4%)nWXvh+_0)@>mdBfd2~)PxJkb*@PF-bGtVV)~b>=hf zRK{5@L{LB=fOD$;K07N{1dxHGJul1bS~VnSI!?Z&MH<_;YMNyA$AZRNXNNYgQQq$#Hi-km*<hc!-=a%EH?BtvP*|5TP>0Zb=+2QcFRnpR{NlG*GrK{wj#9OJb z-G-Qm7>QREWmRB`LDR)1`XgA;L@~?=&Vh0!gr83ZyR7bZzF|%+X8OK#auYcyKUV1- z;t^~ujs*zD!kSBn`JBo*;T1FqHgWAF1CH6xeuzypEznp*0j&r4OyV(2R)j zu_)z{ezUM8L__zne_R3MqdHDY##gfJKR0NPQZg4LTei8qT07f>Eyy&Wt)M_@jcI{3 zi)thq)J*;W!s%{V@UXeJiWH5nM3H6?DUqLPOQuM*pbS*fjoChr4_h-wNHJ+LLW6G< zketHm%L3p7u0&cO)SfxTb9|G6wn=M;3i`526J<_HG*TTB6kY@r9Uia%08S6I4(PS(#46*_3XWoojW^M&{`ibND8b&4K(pqu|^#KPP2m3a34J&sju)nxhN7NZ;!_ppcxDg{e0rsk=@P9uy=TQSGIU)2@KjJVRsugP8YcNn<^a}1m|9c8 zEm^p!VV7*Kj08SG z@S^}>)OVG8Vc9fa9s*~4GX;;kJryvND3FS4t9AWoLO!~%PZPLns#Z^r~dfoM|Ta<8$5-vfVShl#XS=>)n`%WYk#!SQ^m2J1rYHE|>pq8Y=e`$u-XnNhT{cWY z+s5dANOonr|9nn8EOxF!L}<6RJs*j6H* zyj44!=jT%+uNvCFh4<1WC}1I(zi^?___;j7v{SrKJPx=H@a}?rVtg94;)vw?yoBk( z?41y;Yq$0E#%T_-AIco`&&@`DnG;I8IA#61Pye|U5$>g}y&q=$K<+%Rng=DGVvW z^SpME$|f?|fHD)5g$3bP(^=4x@Mao%xy5LE1>j}pRLeFuuBc#?%?6K?skI4@zpf*7M`)BJ+m8h@PKY zkC9*J+*mZVh}IR6JydCN?NCvGuVV_RhVPnfS74d$SFpfe5wE@fUpd?7g|S`kiZCb0A1l#a<&^cH#rp&v|6~pNy9KbK ztP#XY>~o8G-p-q})WmL9E^QXoZe+mb@uq-vPrQPJZC!0q{BhD2yD;!zPDt?cR^hnt z>hOeoi-xQ1hOqDu%^f5!r-1GS!nC}{!`E^6b`1x@uyXgnSmg=4u-!*{^6{!=TFAMa zrc=(;&A%p+!1o0DMwbM0+bP1jv)RO@w$N<`eI6-04z!UwifY(ES3!iJMZ4E@?A&aNZHW7}B@k@e1c(hFaP!V2oZBWQ2IrG@ zd;R%617>an?hK2Y!?uvbn7_rX$<=e|eYf6eR>4uBb`alYdEN!S+q1nzdq=QU zZVBJTuV>|`g{Q`agM&2WOeRGM5HsbUk_#n!bh31DkZ3sOcQ=nk!`3wIZ(mUN42)rg zU_j%b)N(U7kqp7Qk;2ZFr*K0i|HZ4G=UA#4Y0V4EEke8qa2__p1QdG*aQstM>Ayq3 z>5)j_lp>>%dro$gwR;b+n?P)U?}7s7-#vSEBo=$J*;bOUu2fl+j6Zw26^2%_Xs+aC zTo(f44bIHVO`&bDPaxDp+c2qfCm?A8W9Yn%r@t8wxdq(Iq2N#~t}}}>Y`9=}Um)M= zGjYT24I2mBWbY(Nv{6Enf|^gfOQjJQ+nLAf<{*5YX`WEFCe%6rVxMdVM6;74iP!f0 zT;$9`JUviX*8qVa^dUIIwSR%P$ zc~S};?hG2_H9aLX3JfPdWP3ByZE))ix(ubzPCy7GHCXx>z=j*xFqamRTE78tHlR>m z*$~7!%i8LDp`{6V4~tMii#Fg8w*;x&UT=dxxIufj>n$4%FmvMDU`89HXR~U3tgUd7TDC$1i40V+;Kr8lYUAJw4jHYb(>8z1?!|Za^fXY{D%=bHcFy0x5a7>M z7`Y14PWCn9`Lsnq#LL24`!yf&>MGJ+ z>%IE%i~5B{+WkJ|o)uosO@c>;wgxhZ13vF;2D*1>lQmOxKlKbRCe7(UxKrqMio{O) z5-UC_a7=)mV9JviUeM&qM*WZipGAS)q7d582+_)>aqh6`98*AHvvO*p>m^eQ@?0Vi zi=7>*T-dnr=8V@3R%EK-g9ar#uaMSP}0dEdf; z)jiJXt9lx3M_Bh6y*3zO!6PN+XLdX0FrKOC8K4KJP9Fa zX*Ul0`efrwTaoGzxeoJMUDXL%BF_AB@s-?~Qfr1={FXw|(uuo@^d%N;I%~3b~77;Bm=_}aA z35pFnY;^%^rN3;wV{j&4)b<-&6Wexj$L36&Ol;e>&53Q>wsB8v+xEoC$^U(xs`J*V z^QCL|s{Yc|zkPM@-fLY;4%S`h42hn7kW5+n_d~8U)@YJ1tSoGMH{kmtT6O6_0P0EZd zYrSbJEX03~$b!$FwX`c*OfX%LVey)^%0yDiyq8^F+DRQ!*l8(JZ5;eXc~c)SYkZK~ zkOJLe8BT8kd)>;Ey@OFGSRGn)S?Su=Y$)x6*1yHI z)2EjSHRNvDx$mgnX?%Fd+ZCm3C9%u_6RS;4JJ5MXg!o88R>YY{KcX4oZyp>xWzdq# zDnXy-lt5%!wtxb0O{w|=5$#xucubF}Dw83zTFU%W9gfU6!Zh1T^lgSwllvqx95LD< zmYl#N(AQSE-(L8Jcow3nGp*NJSD~|3+bNtgHI5BKH4%zLtCc7=wG{*z;#7Z8>ik8q zmA35yJ@pb*)~pw2;jGI#WIyx9+r4XZ|H@7lG$O{V!M5QfAdgeuBhNnGe)n<r$WzMeRMXmw99(PY5Bb?E$AEys3yNU#_y-0 zKwy5g_buI=!g0Y3E5c?+@Eazd67*Id-n2dY=#;l!mL+PTe>$)I+(5dcdJA8ujzm;{ zHN?jRB)y7o_>eUB#HHtYBOq_=90V}r@GxbG{gPOfcMXJ&yrr59ge>541x*%k$gvxJ7D7`x$R5lviH7kjhR=# z!yAo{+Kq&F>q!Yx%m!($KN~q4#{f3s~p- zN#?fgAZ16kxG|NaFU-!Tp=_l#3KsBiW|x6{_?0{hzi+dF>2Op8uRWQ+e$%`aRZ_QE zKTyIuOxqH*lEk0*-ZR0W+lg#-i-TJGZeUQc?S2gu^c}q})-66&lpPz3B=_47RKxAq z|F*r93TgpWJ@@4a_#B|&Vn%F3m3S+c((03_NRl^AJiANK+BY7$?oYm9N!pa|&?Ukv zh=%PgA?5k$5%Nk@M)YL@Nz59w^%=YU+?HaBY)b!E%p4^rr5P>01` z;GDc|NIm&x_k)j-ZwW~0b_%M&cN~1*Te{^A?htYIuTtVcf_Px&#fVW0^=~Hwi*Y<7 zY8nR%IAa(24YK~@$mG?|02;N|y#Kh{DU>9uGK)V&PU~bk8QiM1(s^<>rxY<9jAMdAtd-r)e*+!qAe<;>& z{Mn??zQvPtfA2ad2+iHi-T9+4$owHUrMP->s@{fI$^^oz>%U5kRLbcOO zCx#g37LnOnQd>lo_dzBYkx|%fCcKjh3?PnGTT9PCNt5 zm`aJ5uMxp=m}KgU3^&%a53@ZPW}fW-LH#4tJ$L5o>9Ys)(e$Lbi4U*_yiYIY_v-cK zy-{wqo|ajJ=S4)cjJB+!D&7=Z2&Q1woqb!y3;Fk7HhStk~Z@c3g9r zh#;?JQB9?4=fi^Hi1Olrg(ah^l4SAo>%jnFmw>-x?6&I8I3w+0tqITD8#I+^)^Wzo z`O5SUjj}vXs^(>qI8_a{q`8=VS5Rake6?$(;g5)xasS_Ygg1e|H(I(MVU0KZyYC#5 zy{_7zhgSD>$v=I+N`)pt@ni)eM|=7;{T+`L|8G z0=*B$-MdS(JMYNiukcRc78B1(3Np5~S+sL(1BX`y#cxx6m=R~f!hK+RmMI*!ZI5N- zVOoc;GHqPGWK~ks_8);=Z2BlqU|+3xbj)TO+P8*|+``M2Ff{q^2IoqMiQ`egqnAZ0 z*qu|6fKZ@T%C%1MigNc6oJj#go4N{rx@OJttgmfykNje0ZtF zS)>SI*Cp)t^FD={^od<`h2fee2&+oK1+i-uvRd1FVBdVdS zy!!r_^wKKWhZ3hIKke9Y`rY%GqH%?PHYN?SlJ8T4uO<-4J1Nv7uf2GKCTZhI!)dp* z7H`;`E3nNxX~E^P_}1n+{O@Wqs_pW?{OQH6b@NXJ3|@GPTdP|gTGU686k~b29z$l< zpTMreMw(m+_8YxJG+@rPj9i`XK}(|lXx}HGEMDKj`4E9j$_%MHsZy~Hl?~VMQuM+n zjZ}~s4NxKU+4g9Nz%t5~W>Uso!q=sArLJ)-GMqr&i*qJkf|8?giJsn=zT#8YdXX!i z4y;M(6F)SWbqc9(XX772a+NR-q zqb5*qA374fG(Sq@dd-I?c&`0kVLlI@vE1chu)7}g zJ(%I5PpNzJv|OWbBwu&9)pBY4zc&tYOJntB)hf9IO8$lj9JNypH99Zw)$0g8si`QvZk0*xmr23 zZfgh8NK#JOJ-kkmVV3({yudZK8|qh(ea~fztMSn=U4Cv4TG^VkvI%NnRKm@7LMPkY z&C>#HI>sGn1WGR zCoHJqMn0SZvK9!jVI^|Y!Cb)p(d6zl~Rb%G(25`tm z+Wxf9B%*pk#r7xTsrU0evnk=sF*JQw8-Vbul6&Wb@#l+ckb|7cIM5yGbX?tOu^${k zT;*EF^D;78w4NyD8y2e+-Gcz@7)?zei`w?knIIEN8j3W3(D|PC#+evnxa-OW7sJ2 z4lL;$Bc9&h#^q#l0!e;&)_nG`E{7H96~1h(dC4sBPXn?1x5Vr5V1LL=%qT=nlD!C> z@*z_{pg&$d8K{k1vMI{Je8AU`K!OuN)^%T47WOetJ*YFH@NYQ?BV7_sLh>hAsQfZ4 z&O;WNmhYBgy;7UjLnqv%mksCFM*2z9-nn>JFU(-#)(qk==h+UKs$8NsK63sxMq(J>4pr%_x;{8X`FQenubN{^qS9C*%5bDCA8S#>=;IaOG_D(yGc0ASu zW7AK6_b9DQk z?eQo-;f>`Dj>la{`Qd;I)5yi?ggCVozgzi)jlDPEA(KaJAp3a7Rw~C%5Qy$4saUz` zv|{x%lNe_h08Qw_9X`F97D|*SEgLk5&RR8QEO1JLbX&rUlVy}=5mQ1B)_D4JhjT^t z(>M7c1dO|uS6KQfPLL>?%aoAep}*0+bN>jHS0S55*p5i@Lm5F>cz^a`3TR3CrX8vukE}< z6d#o(sx>TCL5-z_jS&*IbpqY7#;_dApi}{ULW-qw)rw8+{DSM5c^mEpP3BEh_lKm) zD&IvPatTV-h5m!|2kOKDL*Sos#t24`5%|z6dw1OAl&VpsM}^x6#x+~*i1cjx{tkCw z&@PMHoH9|fgIs&svkZZTNeLw?D?J2Np=O;fA`D+}+o3Kfm)ot1t(S0{p#Ovi@*Dy~ z*U<-M1H4aEyi-5hx$i4wBx1;oSbO=B_xo>jc=I^2S@XL$2J!xQV#tSy73Hff67upa_ME3z_)jGgFJ7F@su0lW~b=<@1&f%0zo@CCM)5j&-6&ux3D}%xcx>LsfaSbEhz_2#m>aw&SUv@*r9Q1j|@qjRLO92<+$OoZoeAT zZ%gtkX6t`gTE^p8zt#4vHMRtJ6A5h(Wl8uAL~;cgy*!l&mGh^r{up>Q7Ej)X{>tcM z8XXqcFKFTv5V#7#qQ~gQxG08O4{=kePIt=&?(-B%KJUT8Qeo~H#U7m(W@oKSH?gzm z$Ugi6e+A-B^fq0MO(*diOJ0?Hx@vGYK_#9Cf24OA$Qc~LAR(^@x|!`w7EV6v{}G@C zc1g9}J3KkJO4tollC{f*T9Y1+V!b{Dyd@7O+J~tr8FKW`lVewBzv`2Itz})IYgBu) zoLePjax>gYOpV5#>Xm+QOFZ9`MSZ(kZ{YV?fMTl#3xEzrrcY1p7N_l*e&{0k7kg}> zAOBEV$L}(BCSSqG3B4EF>&g_;E0JzED_3aF-%ySurB%}YN+_XQkb!CGuDr}`!=if) z^bt6j{8RgMUgzg`L(AxQk$U#1q;NyY?6ywB!wCWFg+`7a7+C2pk5<8sED1KE8IN#l>RYhg~hy8xcg`5b&k%^7;m8{n|8k+TU6G z+e_NO4C$<;`&cJURSqRj@$xbOx6pX<3BnYbnIX?AyrqQt{F3v({6LJ2M=#vX zmrM0BP$emKe3|@7Rw6Qpr%Zbe96}w#v%jyY>j$ zL-ZDyM|IK=P4h)JUj60Y8?v*4Gqx4EK5flm0r0Cc{9eFdjs-3{Nx1d8NDn&Di7&~L zHAwj|>m{HkW7&ZKW5S2~C(F^d2ICU%|FJ32JCpg2j?Vl|6z8#%oEcDApA4JVy&pJ# zawHn6Bytn{xc3H1#OYIF7q7PCI&!uyqwC6<nk;rf?Z&^{kT#8AIc8@Ry#T zL4};tL{wHDw5YQ`a9lY1a|o;6L%x0H3|noSS(ge}4KR>pOVuI+F*I3f8^Q|-HA*lx z>!p$#$2e%zxk6T0ACR+Xgdf#i?B46&f}`gIai`NtW=MrM{=LspKJF?71;VJW45|pGo3A8YD*4hT9t)=0|Y#u=QIB(@N+Alc<=V zKFT2?x^k_5o7;x&4QKer_DqrsuRM(V8(|e5Y?-N=G43Pg;g%R&BSxEmWo0Ji3@}p` z0mm3i9~<3$Td3IF0S*5TJXB+@ge+D6fq$Pd!m&z~`RM_;GiIt==v#lxJ|&hD<186! z=6UQnqE01MybjnR=v}-Qza^YTvRRTH#>qkrLzYeZGN*hS^uQ&Z1oo>8?=ffH45%0l z5IiauzGP!4O^Y(ytsr94qWm0N$FBVtUt6$8(v4?)>El|_FqYxCI3UGifxx*Et_Hk4 zV*Wu)?p20*%OrPmXS*GntL@nA+cgo|LGkYm*wXD-2_@wxkHsXFG>u?@LEnhg-VT3a zaV5SVxl}Iwn>NVnjD9qqhjk;zgF}RQ;A&^QxDAvGW^dC21p*D^S|Z%JWWc{7dO#-0 zssIjZ6fCE-d4fn*?oD(Zl@gbpG48l%LYd1rIQx^>73@CUIjY|^y%L~I?|+gmd(v}C)*sRRl@fYXivW}CA< zOvZi~c{QfcN?Y>bjO#yn`?Aj`iMLv-R}(fhNuCX$6+G*a$R5utLhdH^-I?bIBee;_ zQ!WlbnMQ?T`Kt`$HF#z%H) z;DTZ0wDU%$qrx~*(oC=SvS1VVLXvZ*>-Fs329mS}MjD*afxKlgT$gt=FD|rGVHi)) z3O_~V{pV;w;Jd|k6J!c?2(i>_6g#YA#xGm7a<;EuR6MMW5_%$;Ijk{5w%J_qnZks~-hZ8(lH%L6H}mPGrK z8;!75f4yZ&CK$B1>KFW)5s4dEM!yRLN*SpuvFt8Z5pY{F#tubsxzTgODAOaw6i~v{%X0z(P4M_r0S}OMsvIhqmYZyW)7G;|$Ji+- zA30E8pUs`;6QZ~z@(&mVzB6sjNAAFd(IDyVK1#QAEKl|o-LA^Ck^RQ!zU#w7ZuyyUyWuVEeNDF`29k%@ojGH=jgsY-CMo9~u*< zIw z^$DIwlUqMQn!n%w%FOa$1XRK1i@mgMP}i*RJci>3b;j=M1ufDk_fFjlaQ^$_FpwS! z)=|~E!Q@W&wt+iF7v1f0mF*~H?Yfn~nYE^OJPkNI-9{Nh0v+|_ENqfA(4TAl9MI3l zAYT_p4M@y$yBK|Ch}O=*&!@{2L$*jrYr+D&H})k&KN(4_o}pUx>!gLY##tLwcNv7z z)DqJ-$9!xT3T^&qXe3#5N}Obx2i~K(W^otDdGK;bae&jnG%65N%-|iGtwH=Y70PyM z&l#sO>)MhpB;DmX1)*)NtyxDN!uTgLvMi=Gb>>SLFpaGnX_|mtNESe7nqv`6&e|oV z)P)+Hl=;%stOqe;*EH7vZnD|n%1dHW8PU-PP7}70Ro<}3CIhDrzGe|vpa=QU_o{nf zpECVLsklO@)pF0K3hFXugL({Z+!$)3AH~rklo@y%Umc!xyR3$^Xp8t*b2*PFXG-Vn zCVjy8cn1|<2y(U>UtS_>3%)(Sfw^yme;vwvhFTf*EF(-{6{f0iT_Xrr-$9vTcEnMU zhqrh3Dd6|*v33bIrs0Zot(i%pp+j8jNl=c;f&u zvgwB?A|f2{Iq3QD`u~0x z|FKkBWG))V(5^GW#&Dw3NaNYB6zB{Xl$b1OT~xsSfK~r3{dR$3L3`aK`sDT$>$U3{ zR{vwBD^Ep>34sY)S6`PLeqB~u5>(W@(n};fBODE`>R=?0}{|J~gU6?I8M))L0RqNg4VigA6$ znhpZujNxV}DFdbcd>`tVO+X67Y^ZnBr!j2PSYNvOxN-h>;uu9AdF5df_zihLU9!h! zJM5K6$L?OU*SyAA*Jg9OI1xiWW;y+r_T$SN`TkY&P$>AN;+792^oik;J8y&rn5fsbts#5gSY z#DL162;S6bnzqHJXUd4JgOV6|D+oWB%s~$EzrBq8U0u+W&cO7|L&|N)JKTT6AoJKV za5abVh2@Qnpn(Uxy7>kC$M%-@aPp9vr#lSUfW)H~k!F{g?LD%mULKiHd7ggD7-$8c z&FVQ%wn^3vknRXS|DZTO;%-v~F5;YV8zo!PtaiKMBF>a@%06OpRtVfZ zTdzGGct*m&!J5T%uq#qurHPw^L5lV+IwHAMkoeh8$Jy$WESPO6Zf1{!% zx+0*j(e(O;Yd5;{HmQ1v70@VHuqr3k;Y`yih6h%W(39zu*-d1(JTcIE)Ri}?4HhP; z&T0g%7+)^ShWWk<$i3VLx`X5m=hRuks;cOAD05UG(@2+2{j4Yu`_S*Jp=QPmE8&w6 zO7BjYIEM{OG<2%h9QhtTzqFfQ?NDaK#IX4QnNvT3lw5^vPA2yWQieQ74ZAZ!#CY62 z9OoIF7T(;V3%STfoR~v8D!1k64?AX*QVk(0cMxM%xYpIxBJI#`=0WdG{g$2}$U@QD zb>Ge>XmSG7AuVgeRH3kHI@Ti3B6~fyDR{b&>(k#=7c?2Qd)1&0_|K5^%>3E`1ejw( zJs=>NXmR?R((R`3-h5g+fsIF-HgsX5sP^i7VT@xBn!A5U3;NkOx9M8kx!jSc>TCfm zi`1%13DigmV&n#DBoU4zVNtR&Bw^1U$K(Xv+|4soGeMQ|I>i}GM*)}|F>Q8sU}#v; z*ezg~?K@3Es;SEwFX0f`37$owrt2h3+=!k_mJ-0(b)Y(*5ae1HC@0%@6Lo!2=tT3Z zm%9)PQhe#)AZbUrGdTu+#4t{{AEYUp2N$+wlX_(J50665Sp^W53P`C@K^u8fX3?l9@t39Y-r7zWB2mqr&nZm5 zAJc`Ir6eu4F_VTi5E^nppLJR_BM&&8liLc%@Bn%B?*Nc60TF{9~{a1eGHC6Vqz&2c@jkL>&?wH^RFCTv#@gUxFg=Kn94c zU_f8_OZW+C-o}m?pcFq`-XUpM(pyL)%~LvG+QX@CL>8+)bisXdVnH%DQo4MJC*()iO7PgizyJN;@1 z7S}yB=p1jEitM-@jC=R7P1&&%LiRZFcUR$75_bK#qrC)76-J@mDTxbT`9JWW&uJ4b zFh(n7ENgUIuAYP(G_Cb9mWeZ!D5CJ{fD-m+?2KsqWX__1wrndZ;eBU&FdjEco$O~; zIYlak8oJ_2NZy^y&oaE*C0A=wJlUawRm7jVu%JvIiN1w)p;mS)t}D1M459w@-X%NU z^V0EFmFC~e_0)47H#L1Vf5smZvS_?KgH2l&)aLfWNCW$H?eCw%d#RU!;1;zVQMFai z9kPRGDH7*3K%Q|Q*znt(*955oj%~J(c+mMmOqt}vrE;_&1%KX zYST>yuWZ6nwvVWOk4c@rHRZ-^<7hpmMSWttw<+%yfbXpx>@Vpe|0h)#SuAg!Dc}0( zxVfclXCU_#o&XME8r?lBX#U{Oja|)4bUwaq4?C^RUY{o~2TjY({FUaD4xTRfmqWM3ym~PJyiz&soRHB$bH4WbMi0!6``Rv_18TU) zLMOA%D1S==a(&WJYI9T83H(CLjhAp~#`(O0NQd<$SD`dGQ~dHpHntzu{C?;g7wvtw zytTFHiJ*e&6{TQPq%n5MDh_~Oyv@eVQ63!+WGhHNk=1@~NQkramQ0(}VdT_$L51EV zH=a+;=fM#eiWm>*lG!s=7?zW{o-esJi4X_6MM)k>a!dO@ax`XoM>x|{PEM6%3~M3^ zstl^H{WTygjQH+M=iau%BGVZ;j!tqG4uXs=IY`U-o9`p$64F+01e(mjR561XrA|+K zWX{(=kdxmcH7zhhZ_wo}h*`|hpc_ZlD->7zUm9+xY|Z+?rft<*j7x7FbSq*&{$aZM zX9RKZk>2yZvDcu){yxVAn}d81LoR428aNAjplt67cl~FotcTXWdb#8YzY3%=DKyGYfa<9XG*b#=R`m>;I_fUjf@ zH}l;D`>h=VnZ(Vyzs8+gO~XC_Y9a=6QN+bDX!s4C#x-&mi}}No`NvBo+j73$+u$?U zmV9D>@Og|ows?M=;#C_K?Fp2<<+2#({9H2|cfuc_jZpp%GjsJIPve^O&mfHdCX$&F zw(_YnJsBWja@n($+rWsMDUdZ&=r1XoD38_J;(7{rFu%I~yu(l4kW|1OYg6myZ?zJpHYMjcozeuu8y<-m#yJ)iMa? zeC|EyLnP~`g6%t~_+v*^My&Z8_%mUVgaP@_ykqdI$LOt%!SERFIKelF?pS>WCc{{P^mL&;O@Bb-GS`*wuY=%m!|cHH}EM8jnuf? zhNDS)~#BVDjSajb#y)Y%b6vMSwK7C>EQxw(IK-JX_ftk z)EjD-%lW^1Cu1WBxmxeME2zrdY8(h$7#{ohb`4#G^OMjPSc~0r!`eErdU2a*VBLmo{j>!jWWTifkwLn%;EX{i>~I)BP1k-d}Ng(ZX?EQ}8n%m%Nj$ z!^L3<{svuPlc7Dc*%~AR5p#o#^ptF==6$%2MmX1yQtv{o%nb(n^S?4+i{vmn_}T{! zvElFzj0=xA$Dwn{2RuvCRl+JW=%Yes9}bpg7W!|})%le291SrM5%MeyLsF)hf9=8Z zob9A2E%T8KCYwqR>ej2KTKGn-y09E3BxJ+0WD69o^K^&KpX zwCFd3&hJT_EozFB^;?#so5?V^z zURQv8EKl6jo{2ic7)+NnCdK2dXYEO*#rdb=CA`_Q3mRd%4Cef=9lT#GM@`u@ePVe( zaZP!!$_1lja&W*lIwp(y@D$BGWi%tT4$Mic`uoJ(=gJ0F4cMkQWkY+s&gv>0nlq#ujPWY<7O z<*S-E>KbGB?o>Dx{|Z`#Er`E|QXo#Xg1NK`oU$cRMbdxk92z)u7l-sR+R)*P1%9vK zAn{~UUzP8GL;^!Z+nO|^BgBhTvZ|zXByr`aM7+CUWrB=$n6d&RQc@Ni?-0%2 zq8dKIRgUejm`_`Zy~qPVqD#}P2|BjrSd~VX_z*`+eC1qO*xX8m<1Ml}Jq_~uq1BuT z=sW~WUhuQlW?xjIImj?KXP08~4+@@+E0zyobDrJvB`h%vt3~P_yDfY2QY&BfRO(uF z%UmMzGBjuFr+DH1E&i?({qP*us{C`y$=Ymi+rWsu1Ar#gf1oc9I1<~X9L@94NZj}f zSaBTc)?6}$rj9Ak@Pm-zy@`U_y)$4tC%SlxVcnRf$`-;0uFy@Kge*Ds7qMUlq)#LW zIehz2`o}>Ir}+W#X?YR10_LnpW!Xg0aEi@jf5DFfyWrdRIx)j;+_AZ)yg6NTIy9TXZs^2q>FlY~aww37iiIqOe>bmf;`57P zJiVCSs|P|-W^-V^|F#+V$e&7WYJ8-gOt+mP%lw`ik@$dcN6v>?;2SQ5zgUNJg6DNn zqGeD7Nys~qsCr0ym-Quh@uAal(e|eZwC7FzT^NMsbHdM%Akc@nyjy+)Pa`?mrLuEb zvOazE!Y>x^W4gaf8xuiw0U&#St;gi(;@Ql~gPnh+;_-%ROW^1w}E7 z9~Fi(i)Drxk&?>}lrET7&VOZV7am3R0f$4uEuqN{J|w{$45tTCa_#onX?%QHw&f<~ zbXj>`SVyD9CIZcxrAto`SnqJK9nEn?lh@5GJ#$aZlQ0|^9+HOsnqHdbBK^uzCmQVB zWGj&}QId)e8gf!f1nrzM)+1)B%AowXRM7t{WDrUCA^lW((-7OTWo%0yq;=aG7-^P? zRQme?7v_$1@o@+?#T->&dfHkF%f3Hg$)c27zZuyb9u{JUAK$~|EE~I|?r|HyFYw2P z9ipEP3`mb+Wz`QwU}!e;#cuOWRjo*>6k3N&wKAeG4BRlhq27p2ebA8{f!zD`-8B*i z^-kB$>#W^wCljyjDCJ!8*N5EH(D-U8GL$-(RVYVysDR+Dv$3yQ7e=UF{|zF{RIN;5 zsB{O3Kkhu7vldc39<(Um$Om&4RJL5JTP5OptUaMZKoojG9Om&yTQXPNsZzw9vzES7~+y5OqQ}W9bMqvA5Kod z?y`e5jFeEqlt{-1yW|)6@x)gUV%!(!=k@Iu>R0XU7w;#NMs2W0Zhn`>OpOaO2RN#q zq7dYtI7R>mLPAglh7y3B#MLw3RD04}`?77#VB3Lx-*RM8<~Klm(i<+e&^0oH#$I`ScJ>T@F(e3xSYiLHgrD;WaxoOGw;Ar;We&HJMz@=(fJGyEl>d>8A z`r1%jmmYZjS(6u`Ouj!>(PbWY_))LD(>~x7ag1vguf=$$eMo;FTR-F0ahrtAHfG3) z*;&7)H2IDVzomXVlEAstRew@lgis)y@-eFZJo4AigvQmNCP9;J?5s5-NL{IFR$Q#F z21`5$22>PoB|9jbn3+6)1nxpkSTYC(Tom`ekPtI@APFi$x_%Y75eYV3za|oCRKGgn zh$Hh-P(5KZq(K$TId?cc+&RsJMVyO46}*uFHr?RQ@2dvY2}bUim%?H;2+iX#3<1#b z;)YeIM(Nn+L+Vr`A}p$rllVjFT1Z|g8A;QQw@!NODVq(d-@cVaRkbc>71rEmJ{n@ZT_?fo)8pue>)phh%Dtr*S-Tqi= z7`$T-F^VD2wA?=Wg`rgVyM6zQ$m9+sr)*&t`$jwFSZ6sC_c<+tx!J%N95jg>v_0cL zvX1sGM;GOKi2e^`(Umwm!ScU(#V^qbe@&u)^Yx8gu~YsU%{No&6{i2o83ID`dBxf{ zZt9U*lS0yMixZPO8_~7jh#`buR-4+g&o9Ur*=MX-SeBNjm;4WO<>@0k&du|Sa%upb zNoV%?MVVt40GH0Cb2DIJLIEH;Y3#wq4#3XJcZItDIf=jySe#G>C{4;hnN?($yZ*X7$6NL^;%_g~j-=@A^?tmXy-wiAe|IesDOV{6GuM z`flP$LWA(zARG>_o&SKClo7$h00u$S5Sw%m1t)Z(Ei?ew22K`S zFQG5(Btul%%P`#114=)NOd|8c78SXeClKcwC{{6}Qg z@E$#;Sv8l{u$ArR_e{Y2-wUTSZ+T595V!sd^(x^h1TB=#HGY#7OQ&;HC~R@-$Ibog z-S3RGhElSv=F9l69lL?$z(VLQjmoz}2X(@uL&q%f|})uj0$48+fR9; zGu-(C76#n8fl8HcMMG!5|7NQ!yTCN^Tfa*F2eKFCW~D_r6@bd5GP}}$vSn+eTArPwAuO^y41I< ze9SOFaJ3~rEaKQd>$MNtC!DeyXWY2E{u8kGPi(+O+TI90CuKVeFh;{x#>pthFFL(v z9&Uy>aWm~=#G13DGZ|mZw2S31hTD$47#ETCJjTL~9r}B*Amd|ng_pRU_C9ja%6n*x zL`tXMC}kceu1^nUgn>QY_pMyt@)fCzYs`ZEKN41#6EqgdoDy7587*$WAHt2;K>wL| zc=F{A9va776ih;iz-<5nC#ry*Fo1$Vgx9Wi*MWi++QkeJO2Ui~VGtf2OY&`LWDt&+ z+5UF)&&$F1OZ@*!ll?*H`r3@ekN=PUjEpLY3fa3?z_c?RjkML3F6?28{jztz`t!dk zC2iZp7QXTPtykYje(O@$w}gF58DcGWIQx@rNBI90>UBc7R*pI$4II2)fLb6Hcij7e z1Now+l8*V3uC}@TBK#t{^%D_sex!mQK@pZy#A??6=4{0K>Up_LXZLo0TJYmKUai9K zgXhtq>+3_D=XUoaLX#mwdxN({2WT+aW$*a&Up%T_>hmk`^afEcJ`0+?C;1nf5Vl5v zn>WFUruE6C8r5h2*XJh8!p-4NmXqUu@svMVq73(F!I}j-)&WwlPwhQluiIZ=!C$#t zBq&E4Dg*kRCI&6B?EY;`noHv2jr9pp>Pqo0_^`!`( zubjlCqt`1zgM&|Kxtl1eng+f7&fWu~Pwh_&(XUUr)rHY%91^roAl<_vVcho9haq&oPM^qqyf&a+|Uy!iZ1z9Yj#2;M8WDTiQu8{I?Wd0 z{&tS@S*{*G9kitb>Z6+jq->ZPdffP`P0?~Usb!8$zM@QVRvuR*4Tce*3%JOKOei4@ z_Wva$R2Jw5)Gp8*h&vN`tLbC36UWjML9qOd6RtFz^pC5zf}uk00uFbhT~HpQBhk!J z9GyDZG-%`zZjd?136rRMnyZn zwQM4-a$NiYwa*UgK#Gxf(zLRtIxq_z>rg|O{uzUa(o4!#CS zi=Wc!Vs}ZZX8?6;UHV6@h-e`eCCIRdD2lsbKDpS;8(PkSV?#>;egT#Vyjj=XxipTf zsD-;BTIa;uU5sNCzgdM6Al@>WEN&kDxN4XdZZ_Wq^UdD4$78Rb2UD%?)`(7ZPAN90 zA|7wLE>J2o%{RYnXn`oB@Lw zRG2u@u);EEx)KLr%^muxr)_T4EB-MZ8nYi)E5Dn-

Ox1NZ)NE#9!3HuYzT$Z@j z50!)PxrzX|-><1Ro~9ZwbFlA3fowAmV@o7jQp6?O?XxBr44ZW1+0|e&^1=p-sh$%O z#o1oIUW&L!N3VUQY6HK1`pS4E1xql@6ev=)M%mPWQ{Y? zzXBp)51WZ%DgLQX`TYu%AjiF~%_kzTsCX85oSc*c{&}(O@K%cSN72Lr7g;4E?k)FW zEtLSiafr4IHvhn1rRc2Qq`0pJ_jMjJJuu1x5lT1pu8YS_3JH#3MG#3!U#Tjx z=MV+Q&^M{znk5Op*mXj4_)#@to6mk>|HZv7MITEf+pX-nD`Z)TH_bOYo3Aj2XVD|Q z_vU+5s5)sOr9ps)1wDnoOYIystDpW%NP8aHj z*R-s5^Ew=D;GvF1We&f8$LgQdtseQR8b-JOsj`>%{!a_d%{rcB3RX|;Up6Pv3C|!| zRZ6?0Gv5mxT27G1jq+#dgg%fVeU=S)b2TrOQCH>XKj)&@ckvtboR9j&SRl!hj6$uY zMcp%pW*0`9kk*3bRJ|_b;3S>NE?Zx`bwgG)&xPTbGJ&@up~NIE&ml7cJDbazLt(ET z!QZK(T{CKTo+QuJtu~G(o03`ZePdt_!BB$6Dw9Js*(BjpPFF)yByBneAOIkW}G&*^6E^d3~*f5;-Rk6BGrw# z)Vm}hj2#+m8^ODe&Z)md7fFuYyiWI$T>7o(4em>lfbzJJ;4Y-z2&Lh z{LD-xh4gJaqYOQSeq1ZgISlgb2j?x<{o=+ zbA)QA*Ya2-enfK8#Il4xRsmdk>&uUlct0|?1Z!x4yGlFG=(F(-dP*0u zQV+@uO>pqzLzQ+P*^GQ%H_YO7sX%D4ab(9c^KKBU!CW>)f6&$2|3$J^c2s%MXRhbd z8*=Z}wbD^AEx9W9moZ!EHJe>8HG;|*4W(t`=^;8aZT?x(R|6;Obt(fQi_~aM-aX%I z$IDwwV7UZQ&_Nw?q`vylw9gN!crACj2MF~^=~e=v6y_ZM+j>C~+1T*D0Yy(DX<99m zk2|~YayfkInaf+uHhU{R#=63kI|tzk;J@>X>1xD=>rv7OzQrhY)SI>`n&i zHuS|CNo&P|9=HlWEe^4#qATl3Dt>ApmmL^{iO(VaLoifKg`UXS*q{f@I5XfV5MeAI zvhTJ@Mae;!mVV12ww-a+QHWEMrg8ACDlKh^xINTD_Knbs9MI#Kc+ZJ#z#ieBA~<8u zY;bGXmpRGkW3anSYry_Z)B3eFS+km7LCYQ`v19WVS05GHp%FETYJyg04p&!4DUc=+ zNs*|L)0{t1m|nGE^z1hxJw{>%E^!~OSxvdX{I^728etBtuX@-7m}yH3u})^Q(vmPd z%MwxPI_O@c`FXGrDH$;YzZFA_CYqRkHs_@1UNTjD0#E=O;+{GbG;|AO;zzV3-v`jy zwRjHM_4xXK?#SO+CZz{H(oL!GTXu`W{po|DzlMSPqnD(Meiwzv+5lJ1yr4c-pgwFrlY&`LMCqGc#O^@m=Qf4V3 zb$^~!W&xX1{d)21JKlxG$aFV{bf5F{0NQ>UakMssbT?)AY+4)X&))9&`T~Mth_o)E z*13*jq@4l!W)eH}QkhcE1ZJ_e{q=^N;Z5pX6N2`rRD>qkOO` zOK#!)HafO-U24Hkhh-zlFLqhfq((*U%1iv{Bai$Uhd}P9P6Dz&h*S_M+Fp-JkCLb4 z>qZ8eyuU-n!_fE1A*nm(QxW$RHMDhQTtm;K;IU&XI3sOT*iq}#OR;^s|2d(mwzPH? zU}qaUwenm<3B9Q;Fqy&pJ39@o#J7T_F7e4lXAfF9?`$nXCNW#OKi%i_G1GkfGX}hR z0aG)hF|>t0z5x&EhBa(%dVe4W>IztCX#7P^*C~x9IQO6NeGl_7ira9w)NmGM!9&Zu z!Ma5Y>+Qt)b<^QD2uV;Oa``+(KD&IVcl&`=fGYO5*)s-y$xxvNazx=_(`HMUTcQ0> zClad+37#8>p_PlLcJeqqPFciq*VunlDO3nk*P-c`vkvq;9z;goI~Ef2`E!n@Zb0=iZAU#4Eg5It7(Rq&fx*Evvm)P8Itfsb0_i zlKo*FVDRr^4Z#MSfN$81`oQS1O5QJDoeyw_7hLE$OYZy)DE?F(lhsD$PkN@;7JQ|S zF1Y?+!PB0}oHk#Q6SHsl znOkZ0$cJ_k#y5sqMUw~w_(Ls|LPjU6|6$DI_lY*+Xmr1(rSlwlzp`*jNn=4ub~Vx? zCcS-1lVIGtG?|c`N3*YxyiEFC`k$b+a%3d<_5p)rY8`)QuKQ8ZC`C^b4g3stV+0!8 zggaAi<4TFYUDizKmn#HW+)h)c*2;rlGK|qnk<#U)7YYQ%u(OJoW;1c|`HW!~>lLGt zdcVR*O`F2C!C>lm_20_aT{j#zgau)4a5a1vm3cJ~5>rGiZUbK`jibUqYJ;1R9Wk+sFMaC!m4{<6{>|w7&#fHJz$#1`+s#GOQHXa|_6h$#Fcq4rg9yqmQC zx>B?pmm1?7$zN4+dVNRjro`}7vXfTTQ_=P?0@BGx zUr%O0O!j5m0`H^!go8!3d`Ee-Ukrk>N)H9*uv2|igqseq>*ONu%3VkZXnZx}WnQg8 zMvaBj-;|Q|p6m8}Bgz^n1kuP0=gxjj-(aaZ`kRc=3G3UB|IXg;ZBxIeLYCt~@IKWl z9^EbGZ!2A&G`juJ<5D~B;7cnJXp7H*Ht||=_-^~I?OHJAbJGqi#uQY2m#dx2sAVP) zB9h2rbFkrm!g&6Op;kg?D;xe9VRWP7>kBb^a|T@_aP`^2!>k<1!qstdbp3im;E%$tCKGlzc zl5&jb2~%B5p+$f$BY-p%Yba*R1K(gg4f1 zetBBu%%{uOPvB**8>E5T`P5DJrS1OV+JPfUhrerLu?L!ZI??%fk725Vs3ZiA>e%J? zRr*o6fvD!$b~3TTPXwAaL#)JC97ilUy3Z)lZjt%8Bp)RQ={Ku%Z}PfvtSahCM#-q+ z^RCq`%NKj5HA^glq7b_g2XYGgZd87m?*r^E4<PpuP!M(QT=p>e7p0siwNKVHE9g zFT^du4_YWiwA+80`V2cW_83Hk`>73{HIRloq%@{IC9p2&i-C|IE}di{EN>ku@6-5gF|!deq&3hJ;>CVSip5C;vUjoN+UUF5BoBhMvUQI9B3{s1Z!F!YC7V!)~N+ zhEnkK%a`hs&}h_A{l0TsC*zQC%D*mHdEmjZl)kv8wlPN|S0V<~1q)q})^_=kQfhXOSi3eBcP2Fk zdx(5DJ0KahfuIO~QZtjLL zSW`}Q+Z4br?#Ak2KzsAk47E|$$<$&Bk%>n_tO)@{D{@K#AY#8aj4qjID`#tgN-E0x zZM1#TI%1t>;PS&pubZ40 zmULfb9u($$ZA@sC7-MlHi5-PD{u_fDZtIB>!6d|UD|+;vVz6VaGCy*7HDqpnKQih9 z6CmsZTcX69BhCGpK>7<3j*p?VxT#98hX0*?$_u?4LsQ_P-YDhTp-4Osfu)8JOCwz#GN}hRsI2l)S62ZuH*Y4QGPYjaW& zZ95@#1cg9!aHrb47wlv8^!)aNEV*_p(VZ~AG?)1OUD*(O>8|YUeRdiJ>^8?8z{_{D z>Umt&gsn2S``*wkxU^FC=4avqUNy=eIbQ%u|1sQR@$vJ17@2q*(DJzO|4)4VZ{*BfS$;a_h%j0ro zce8GL^QynYXsg30c)_s8=BnSO?y`Ahw^_?!-N$jgiVwuW4O+^6B+B_XQw%^-40tH* z`=?{+T==)$sOR*m|FrJ%eq}dC%VEUFaiogxfP?#BDf@;f=jLqM6KUG>;V(lN2oMee zM1TO1AciOqAQ}XS0Rdt`3~?YpJP42g{`WBa_x=BI&i{Ngnh-AT13kYTjD|bE8^rfr z-37V(o{a7sJp1HfubyuoPl38SQdcC7!JvzuxrFr}ohKK;xw+R(C?7;Wn=t&{2WAAH zfvE2nA^HEE#EWot|CN(VUDtN~zi34~0VJ7Ni~P5c=T}}ASZ+mi24u|nUtEjyek|;T z*%#z<%Fk_|5!|c`Y&T-Ny%_R)-1A2o@^k;UGB+*QydvD(gaP_TD0-XSp4|WU{lByK zO&H({aiO|)4lE;rv$>_*%5IR5D26i{9J=C>zjXaN3~>ucLq-$}WQ zg6|>26DX+U-89=z*&2p{VSUsO&MX>Itdp zDX!`nuIf3g>Vc~6p{VYWsNT9Vc;et@q5(Rv0GW7zOd>!g8K8p-kVyx~WCCQe0XjGV znY@5Zfk386phF^%DHF(42xO`NIy3;8I)F_5KyXb;_qSt!4+!830{DR#{6T;K5a1^W z5C~!j0s(?SfDjNM6!ic4%NdFrz!(`e`KBhlbIQwMVnk0zRZ zXfl83{}GS)rl#n8M5guK`G{C+-{}aSYR2ZgT{a(0RA&xmo(ss z2|Y)^DfhEARmA_R8pKrov==q!XL-$M6P5VpX@IEEMV}eJCbNidM4NI1ebDo21Vk42 z!SbW<#P^rk`V6n=r#}jTCJ+X+l7j7g!9(79X+1>Ih-sHC+!^Rc`{uVfGKS@XqPjd5) z4m3D-^HjD(oGoITDP=@UvMChJmCu!9F>>m*ynV)Y+y*>{kQ4}<%If;mw2$eorMMmK z)DG1G`7Cb(G|ZtNF1kJ1`ufTqJfuE7Rq3Pf8Y`yE>^m#j`{?9$T1c^H@l*Y+sDK# zTqceBJ~Z9mZ5d!|g%m%ifj>XFZ`#CF(=PL=bU*#}>o#=XsHnj9U=%8Fp;4dGD_Hom zVgGa|5B1{kuQfusUV{&NkG$PgX15Ie-7$9n)8ByQ;0h41(Z}Gz5c{dGeQOw0D;vPr zDke)NBjxHUDHTr9qoG!i73ck1s9Q?W^{ol^bE9l_8hA1ENtmr^erzAr@X~gsqR;I1 zYbt=6yjTEdeF)_Sr$SK?%N|ql!=bIHBr$1~u@s}O!RT|i?K8G;M zN10NDwPc@)(a3)cs_wCn@Z-!Bv z&jYYYRWgnc?SaAg-0gu)i9E(3koerjAyhbXKR}+me*^Sq+Y9D`B~f%AAuz{&1|kxS zK?JcL6@G!nRZe)g_+p*#ikf=2UGsN7Lux+y z+(o5BDyM|!;z$v;3n$GB-Vt*-T&HFu#SWAFNwoo%os3>z6Pe^%vI!5Z1ulIM?3MfL z(h3d0Rg0GoS|0m=_UTwmm;l00L$RFw`ggoI7R|vCYD59C99mr5k^u4oenWSJfH+&# z51cU((_DYViy$xw?e1C+i4~wi{ zY&A>?NMxK$i~ss;ttRiZqg2xR4D`_yNo9R(HGbNcnY-RH+9!DG*5CB>-m)Wp@<`ge zA1VdV++WEzHePW6>&A}}dBmKl+ji96T&ZIICBme^XD`gHuWT-CAl2lYc)jc?VAyvM z!(EG~DKQ^n_K;6nvj!^p1yXY1Y}7Ds=+mN0gRGN>4$~W1TnkqWSaSQMJ~BJ~lupO+ z_N_h@Wbj{o@OrnC+&`iyQ9Eqd8M7+acXaG>MKyA*4+=S$rQJDW;E8RLO|Nfw;_~CIovBj_RBZc2F6k34TkP8bF(lE77_SW6Ke|g!zOcT7 z9Y0}kc$4W3yE)GjR+WN+b&V#auUL|`&)yOi1dI;!cYu!BtZL>-?;{A2Wc;*GhN=8& zQ$|$?w-^L;8O4ft``=~C$f#s=(hnyzJUyZpvSe?p_4O2+M}}-wejKXC=11ow-&EOi zF6GmE{q}VSK^QFL$}Z3JqyS2ce5<}iY|U#_vqiCNcNV!w#a?$C#k0H-VeM_n!her% z{lJ&AmxTB44d_)fX-HQtNhFQRJmW;9G$vhw%~afv9yicOSF?Uwg-e0yX^1pmx*}%C zL7%3!+Bz3Mj?0B%>>c&taUGN8yoeBd?5}IYLUap`{XnR31wPg22CZ z_J#IS(M5viTgsgGbvG}?<;p+{!aJS3nG`RStvYO5*<$h2YBTR5vxBv(y2=To>fRd# zfolw!0wxXDWr@_h7w@Q9)h?)T`^AnsiSf0y6%<}pMoV^o#LlU1?X4(&x+))+PrHNX zw{^&Oe~gtnuKUOoM%pIRpB@I8k77YSz5?GMIqzY@)Lm>27~o@denY-$c1MzFAF0KE(yu=-cXGFC2fttIv+|BkI9>M77VOkb56M^^-`|#Q z%f`kbmr!Z0ynZ~y)$MRL^y16cX|JeV)!0S;0`PJ~yypG53D%WpK(~AxJsuPmQa!nNqUm6=WfsCZPce|271 znuxRaNva?g9)CQxx;dW5@q zmna|YBq9sNx@ciQW5%Ee%~vs7ogr*ES0-Pmln+S%b(ZtBV&7B72kG-!T!oYq>IY@z zQd<;LbL|JT{ul9|r2fY&4C`Ij!!3vlabGB&Wct!gp3!9oXw(AIGs*rq7snXVGXCMB zDLv?db=3WOy;Cqa)mq|#z8{tc+8ri@P5QF!ZyvH`9DD{{5DSRlaW3Yh$MvvvK;U6a zeW0&)_{okuLHhaoZou%RLMMbiLn!pwDH!q|=j_#U5YleY=&OhCfpsn`D6At93l?TW z_A(kpd%$mG)3y1pdk+s1VkPxVQs8VZH8BhE*^A zU@a{v{Ej-fxc)g3l5EGuoZ{&tr}iYKbFWUXlG5YU&0`N>3ldkI8xLsenGPZ13Dh2NpbGbwhSD1o&V5UVSIKNmaN*G>`(ol`QKxpHc-HS;^3RCC2H+9B>&b z35+FV1{0FLA>aRgf8ip6D%2g*&s=O+U^3HgSGaEceyD5#`j%ORu)2g6Gy^TqlE%F1 zoSzjBw7vfd!F-5va&!#6>IRLb%1k7uBLy}@>1t<|EUD=NzueSCqQ?a~nPeH@@L+4= zJocnKnAMK`$ba9OI~={)lHT(4LHQajTBG2nJ=)=FkvfKE{wy72wb#*1QzK=AU&wy7 zHuM*h`v4=DA#qI|S&H|9gw8(YXA{ddB>F335ckxyOBT!tp98wG5;~-i8hdKgi`-WQ z8>D<2rtw)Y65UK}3s5cjjJY`xxdKZpNf6$_;Mcd_w8K73`#KJ-e%P7|3ynH6LQPwH z!(G#sYp*LOJ=IMgFQ~%fQ8>}bQ*)uift^M z^@?B|BqZ#!rSh!D9#St8(Wr83Z!1Yk)6;X|02lnmawq#keJmyCF8MSO7zV_}{#G7F zHN^p~H0I%J9!^f{=YLWj;5(`&d&1v+NQkcVy@tcsdZROn77ZWb?4L2bvKcd~l#m-GPt^65_;S zm$@LKIkJdVc?n>G9@2c$-#!=L$A zRykJ$ZHJ+o=cj=>TJu2aZqJ^{OY8jXQ1pcjsrBvHHdU%Ul z9QPR$|JJ2%9zkN1sqOZF;(o?rK4|ZI;N!i!o|hk~F4e+GnF)?VHNAwX^c?ZVZElF1 zB94~Wcaas7lNC~2esdAToQ2m#r#IvZI>_`JJuj~Z8dfi%gYyjrhQis@X7JbAll4ws z$c@zRyJgQP%4AuCHuOE#e*p>%wvCU%vCaPYc0j322#b|8zQh>0WBUwAvV=a6?pa#g zbHl@5z6B~K`7Qd)rujKb0S9R+{N5tDo~$mlM93p&5P~j75Kc+&x zh!2EIPdWQNYKCsk(jrwN!Yq_Qw#RCpn(^l1C6@>X`a6>&JXM~kt=DWv@7qqt`rBB}$o$8!c~p;DO<0}+qg^C>)) zPM?dVumIjPSj;5uSFl2g;3;@Euyw`5D(6=@qGUWXtpc`xo2{jBV&0R+DYR? z+BJ2jL1~5s44xmL`ZCYf-q<_kc!i1NJZxQwlgYV&80Q^>1z{e@qGL^u&w?TNAJdFt z;p{p~*$B({EuZMb)*Cdd@G~s}_u=2K0@SrW-N{0b2*q-HUj~O8^?F1Ip!{r@NzNX| z3ySk;6%(xVi&!%yF_yrvE*`m`Xu|VQ+K=O|zrK32bJt^BL#QRlKHlsra2rh&GKD~! zGj!T8nSBB|M1PvKMi%udss*O+M~V%sfmK~ z``nWMajRcYgjCMm!G3W39qe1@q0M>Nw^Owkq9o1PrbGn(PTg*|M$7< z*yvC^pW9;s{%KK^zsJQNqLiy{$CQV%2vV86K6r+QbcWpN8aN@-iN#A@apF0FG^*#9 zT1ZI+0oY;j;V&IWN4P?m$S-Tb6+~MPOHzs2AEEx}T>qHJJO(5lT9mQJ*__My1}a-( ziz_qb2t+3wU+#}L{i-1k$m9oISonyT|yx8Onv-p zKA?Ys($Q&OGe0VAeDZOKBFRtP6YrWMEKX!8t&=LUP|vPwfuYEk2q)IS)fNZJL~+DaxEJ%j5uxB?q<}bBaHY7k z{DwTbPzzy&D+xMH>%KMWtio2NQtmZe*!U3JX#Fzz<)LDeI(aoL;0t+p) zn`;yvR1U?Z3#yAB%FwoFZrFys)F126b|FtaW)b?HxVqJ)D+S}{(2ujioP&uCN)$n+ zbmQW?nGU|diS=x_7Yrfp8_hplT_PC-fA8I>d#Soft!sk&?W8I0Z|-B@!rwp|+74Ag zP=|wIM~0u)j(S4Z56scQzf zaCHW6>u)Jziya85nByvi(`o$hr1)-Ad$6jAG?wY}57dtmn>a0^39W0R&zY}xJn|WGIlxzC z+gHRs%rGux><-v^FoMrDqCzxo@ez}CDH3xDRu+lrpb#Yzk6uz~Gg7)1`2ZPfV2xFk zsv%7CK`&TTV>4$4B`WO?zlbUIwg{G9*Ox*~ojWypIw1RnP;Cb2%U(Un%?^|^UE9hk zd^lAK=aA-DT+)RlFez9J)MgD7iF;1pK`|nM1qoa$9jJoP*iA#q5L;NlOI-nIp`AU4 z0sJ;69g;e;lUC9hh=nYE)kb0UiB2E~sH`91Q}6RDhDHNr2x7QTf&pOl+n^sDp-KHc zpE0fe=lJBmp6z#pK5l4gt_pr;Z~+L*QuU?~i^n`6n4@jq{gvNLwhpXYwI(?|PK4+G z-aW-*!0tx7=TkXpNq>A5gm6H7w1&VBfrTBDy+NFrQuasWe43+!mJ`iVfy9q@Yh%U8YTdQnuZx63vXR)RdXFNmI1N)BM6 zcKIsD>Y{6JpnYuD1Oqk79y%>`9(QVpl5ROOn>x*cpMv#?#>^w~TFPvXgShOem&|#0 zL@~028BrZ}b>QRy6`ZlGhY6nRdzJeM_W(rg1^6CJY5dtaLqJFUnvm9e`KW_n#@ru9 zBF+JWsLYuu^jE|fK9mnrkr7UiAkFs+C-qbII=YZ1{j=-}0Ycw>I@*U*KtLhu(>NZ} z+tvOk+fo~i&?HyKFOiL`$zRa($yod>2TU|vVvlv6gx%L@M^OX4gpHJCPG}#1uC!b7 zUnuMi)H)R0yYqk|r%)=f*Wwp>>NV_ts<`70FOhx3tthEi5!5JwlTi<`Qk(P#KJzRT z+?9MrIj?8VnYI!^^S+tETVywJjVqKYfvomrH#JTS?hM!ERz0vCdf#8^D>*$Vm2*~3 z$d(M-D*CUnAwc4DGQS#oQ~&yx5+C|`^Mw}44{`QRqDq`$hkOg1dqXEl_AK`VX(5z4j+6WcCxQgcp5d;1qG2y zl%Ra;{7m43Wg4ESFTOb>0Y5a-`!QF7d}afA_SYlz*5 zM6_&D6y`S)8zbL8>L=Z@_Fm!piTzc2a^aTk;nHsYi#Pp@C0VjpIzJ|i7B33zrf2&bs?^gtkUz6(*-@;9GF(G znCLH3rCEbNh8SYH5thLj4&;S<(@vw5=jQD+W>3P$fUe?=IzAAfAha@Z+Q58-{Sti+ zuaCy1bVA0kq$pR$nNeQSfUdxJB7tNQy?LxTLLRiZq_|ThrJpqppW7HoYpI?$;As&3 z?H4|f#yx8#R%Z_bmm`xPS%yV~`EfZX-tu5q=+7apurP_<&!n&fSaMRHE|=lI9+DFf zH5};B#GYn+rN3NPnPC_nEknyIWHfMQ*ts`%gf|fW|Y|$ z6xm*1ET4PPeRNuHtnOEGN52=`A>A@ymVNGiO!nq)jQNw}u6%#YTbzxekWpEle z2b2dheYIOwihfE%qByUSDH_$NJK%&)L^4T-lFZEcsmJM=z|=0acKMi|d!E*MF$H%T zu#>9`M<%N$D6)0EPA&3RMGSzVHUjI*-ksyreYb9R3?mgBCpzOeX%*-t%>P9aJGUF9 zhyXrV@8u?BIKVFPxDS@E19nvLASuHwyc!rX`PO208lm*U1F)1_a}axYdKAkkoBDBo zpK*ujIVNOab9m<#%wLkNHFY6~OX`&^?>EYUCZ7CTVnGv#R9$W}%EQ!jbJ;hwsPW)y zcWLDzJ^i)`nM{)VGZ9A^icBe=IGFlzk@}VkuU0lEmn^a)Fql`|CVBpb-R%Imk7dW< z=Og|dm{M8jJZOhN>XvG|5^Ut$hr+&MJ6GaE=(zACh_Jf>ibs;Y zg9a??0ENZXBZKwYDQg+B2y%!nz$Asr+crXIqOO>l_?Swm)w|FbDHaSKemN z^Aj=fZA8x-VJr4eYB(05wWu825eE`Rp?VZywN&>gTcPu*=Bq);Ua1`VTfwiMe_9!= z%@7e%^?BxRv?FXUQ5Oja@zw-$pkR|(T8 z?jPF^UDKY`h5yUQa@=;!0gjuCXKllO>qp(NUxUtZx{lhaZZXt zBSc$tGsdYNn#d|;{(E$|-?(y}Q$m-<@Q=D~V)814?GmK)hCLM;rITI~=Ed<;MnZmX z7Lh|T53qb>+hNvHHtj|jlXk8p^~szxB}4sYQ~NnkKvw~YeEoDLADq3@M*P6W2DNMV z6Om!x1Xu&Cn6kT5Cu#dTX;Zk?ms%7F10elIWJq~YL-7E=qj^%IkHu0K_iWZg*iW@) z4S^g`;2b>EAsTR&um!9G^X{D=KoA_$O8WiSOni^}PtBLj{g+s? zwFK;bImz-_sX$w}lSb^i*&;|f$b1cH6a5o<8O}$^$!){8wy$^BXrG5^?>GHep1vmu zFHaoWlf@+sNXOJ^EWMi(o4SHkESQ3F2INHr7X=pjcW$HLCwXWI*@*oX|8o>E-ml{` z0sm;c*e5CNGcE+GTmwl8W9HUH@P4>)^DR)bv#lKGy`w4Z@Y!!Rx5C!nSL^-`MjK+@ z_o$NE^8HY~DqF}Gw=0I83et21#^c`sNbiWf824KaOEnTCLfhf6^H`K?;PMlqVl7cm z#BAOnkoU7sNeXJdECTQ#(}lf#^#u98C?loNXO`-f;rZ{9OKb#CNhdNRy|Z;rN$QcE zu;!6=wc8-`w)F_QAO!e%{mCs^ueoZLdTQ{;Tlv7KF_Hd)IQ1zlW&2ByP>fUD$PS!0gKn(KA z;js-zUgntdWX0np4;Nu9a8CSnZkMnaG|5T24t_RVtew%#(U^DV0V&A1V+(rPT{ab5 zzUuwfw*6%|>t2i;>F3U4+^;;}F?bhlXrX?>eCAHAa8Si66<|{XBPDxo(39$=-w;N( zECcTBIsG=C{aPLQHrP(s2xmdRc=bMharr!3;cDD_3f&0sAK5gJBKz3^g6L~?xKBQR zS`=;0*Suu!xHCVl(y2CU9R16!_?~yF)scooIgnoxjwQMix9AqQA`1i%7*>384a&ql+2-Rez-m$X(HjLWw&JPHs>7CAhQuO%ANvow^putOuIrpAHPUh_1 zqi15>3UZrVv$WF0;w?E<0XB6K}#Q;Wc!FOP=irsa!-|(eed|I7JP2PA^QWh{1ZW;}? zRiK|ZxRJ0~X>LxQqVbqyB^M%%QJgx?*jVx?b2scG(8b*@{46~6HfFB*;juKOk*362 zcFCma+4Y9;YYxNy>mDVu7q)Xf=ajOl!>}5%CQd^r`GR%{E9*;d!4xWl%{2wmWz##g z%Ud`iGlT_mD|ZbqJN!A&#u5^NjqTB!jvzPH>IAlR=^Ij!{K@eZ4A&qNxK2B9w#iISp#N)xERVH z4%Sz2lDQpj{)~D12y2Q|9IUTpNA^*pL5krn>kfC7m8KB-zti?*=WLRi427&TIw!zp z7{qGAIJ_ycyYbj%{i{?C$}VD)o|q~%COKw!ylJ1pLPqVBBSw}9?SrooSS)|wlPoU>c;lv&&$p7JdL+QK5E87@;NWh-HN?y(kNkoZnTf%j zQbg-ozYH}@AkIQ(g~RWVU-*lBNhR9zhzwCUsz;3@nG&iE>wBZx@IHE(%-9F5X>&|O z_#%p0_20{?_oNOMpH#9WlU9;k(=Fc>*dp&7w+wDy$xMwnlSo4VnsW-tq=V81$< zZ?gUx5n_#F9ot69K0!JCKL9~MzQ1s#tNndMySgv4XhL|ouB~!7+iFWw$(FSe`zN#k zxdkZ3RG2G7LO(~>Ham2OIk@`5Ix8WNpBql?>%#9Fld$vRAhQPmqBkc;KvQ{uf~L^L zCo*WwM)eOHJ@_m>M@q+f4NVyZtt{ZsK-a_cIdV9oC7lrN$fxZB<>B{&yDSv zZ{_13_?Rx}`KDJ9z@2=)p?m07S%lDl=q|(d)ZRC+NUocoH5lF2iwGOJONZ(Zoraju zsK8w}4o$4fHG{gE&=cyANri2S${E*w+4O>t+z!Cs`#M90Cx}+MnHmVUVvsoV)9>AP z`4XtzWYkaW`{Z$LSHfC@*Bb|m4ypy!d0bq{o%B{-@NK3GjWD$(>+zh!^zYRLQ&f^Q3VyP zj)1KA20LHNJx@~WM0#Wru?Eya)+B4=d@4B9q48YW591^d)Fb&)G?+Zn*QOMf;eo^2 z_SzW9nLcZWjW@BGvI!up2C23nq8xQ9lV+o?^=*q%J1C1?=rb{k;v$R3mnmX11SaQ$ zqM;$IUYm~^uX{a%x9x@U&2*H8q<6RjR~;a0q%B@H|;ehQXmbIW?*KZb; z3)SBQ-m75X%2<^49?8Rk5qu&aEp(TUgX3A+trY3jylvmiDzoUV3*D!5j*~vxZ9CwM zC^Ww^IZ<1N$6?wr!HK^7sjUg-y1-FgH4+|Jp(C8zH=TMq?w!Wr$s_r@FylAkvhBpz8$QDSbl*UcyX2sW^n?SOypy zv{gZ1RQK${nHK31rjhsELt82J@Z3vPU?OB};F;lO8#{&tdX={qhgiJDZanos$)KP zGCug7%wAM}`ZosAzxq)x7PhBB|Fn@13-h^vWD)*q5c+dNrtntfMIQO+&_eZFBKDg) z_*uyg>;N&~rs`cyMsj{_v8o!db{EJ**|MulN@vpEH6uoXlM4BQe zogMHIOZPVa9Nhj9M}%3sr7jFCbT1ejw=59?5*S;lQ#J!5n$i^Ds8kL@NT4n%yhC8_VQ%5!B{il0MD8l}P+?`E3&!0y!Sj<< z=WJ#pAWLN3vXS2d(hzxU8ECw=#H?0GiY_Cym(9GOu4mUVcR?bF{Rj_^XPnt^f>wHr zbpL;BmURam=c|E#`_7blxvUYT?iTRNwLvGF2U_K}JTf{~fFRhL20PUH4kwNaEk%uX zdrOL{l2T=~8(dVp_sVQ7Kji6uq{g*CEuFAmww#7&W9@)+?53+QR}Ge{l2fNOaVTB%BlO-@3Jbvm|%qAlSH;!NLuupLJl4H04=9D7Gtip#&7U^x z6DQExP%w>zEe3Vkh=GE1d{8GEEWMPd8w<=J)rIF7G3GyoJDzLR*cL7(G9=WjJ&pX= z^}pVQxRF`T$R%fF3R`WEHZEvo3qd4F zogKO1jl6qC9y=pX-Ny*kx0USaXowVa6H!ezb|cnb`56;nB>3Kl;qL;9jXd?rRQMv$ zwS6Bl-WjR#jGTM!A(;{R zfstutp)pFNb{)WLksQo;?R_GQsL{R|=O{{EJ7LdyT(B=TQqmiF<3DS($nB>T zNX;khGznN!DtxTy($;_a&DMF5uTnp5a5dIz;TcFHzw4wru>+D3|&X zY%Q%?lqm2@LLK9J8O@675wbbATCL3Tm-I0O32j?i*Ic_*xribyAmBxqt{WZe5O3*8`vQ)RsKK)N9*Y;V%}R>ALm27ezWm8b@?@cDpbxqk0c#FB%`FY2j3mG? zaA<3O{WN?ARz7Msq4qCfvz2DqZOY~qS6Edv&!K&br zkWe|eLq=)~C$`mzdENhAETYY6`LSOzAw13V^dw^iEQXJ7n@d)1Uwh}8sZ#ZZw5P2P zRls)JPBb`9L6?lWuRBak=zXv2iSs4sZQbi_eMIw-^7pFNPW5V|xUpb$1E)7aSuf{( zce3U?8TJXdy@+J}Q;@mpJL5-4kqfEo&567>3sKdQ6A-EZWhvpRfY7EQlO4z;Id<{; zoi|NMn;x#p@8~;slKQT^L2eA3T)})RQ14wl#y!22Nr-}s1=mbG(~c7$ z2?7MOS>Je>FLoKMFW+HppNz+ii`zj7Y6nm@5(;Suy$nfgiN%cL~I~tmM6Sh`=?Jz=&QuJc!8|)^*ZqU*PzUaYBuq1UkEqGMtfpHvTXJCAj6Ms((F7y2d%}%JN!t(#17kM(6EfAPZkGE z#U)eh+enJXcr|EbUOObSzxWnwAeOC@`S>S9enUGp*%(s<4V7S0x!#(IO#tt+-J!^- z;fwQc*A8a>j0>f0Y=gkdl0;H)ukHWJjT+HI@ehpp<4!1WbX4cGd;O{1G?PQ789<1x6SC`M z?(id##1dgHaf0UU3< z_d#hcXa?{#MfPtpHdC#rc<^h(bAM#+%PUFp#9PkFM9;5H$s0bL6?L?=@#Q>-v!Dgu z7?ibguY1Re{N2B874<#8h^cG)tFAwlR9zg%PL32(v0*x2Ra#0;$SPI)O80Zcc9r*i z=U1)DsUis$ABB>KsrrCH;{s zq@(VOruK}fXYQ5{7LDjJJ5PJ_Ts$u7{!LI02_U3?3dei;N4t8oPit#vXI+1e{(5KQ z-dg_u(ggc;f^3p0zE1$C+Y0s!(;?20h??AZ#&Ke&k&Q#SC7Gb2#@jn;NaC(=;SnM) zF@AV-3WkGkE`TJ@WNVm6ptLlI0CEEHomgPO0uy<3*m6$ZDYvAiI45!}Y`p$9S3XHj zRBTcODv&^2aCByT=)?#tng)5a+>YlW0@n#K)fPArQ4F}VE*L?pMpTj!wu9A|K7t)=z5E7*y(k(@{061bN~%ELcUe;?q3!&SSRK8Bj>SZR zEXCYl{nbtW5cjCV5lX9Ybr}LuW>U}H#5SixkD=d1N{SQmS-GHquGS5Q`IP}eQqSH< zQ__QoB9_|Ag*O*CtPpXsFf#Bksk{BvO@+C26*f$F-6A1@tm+6TT$4${^B2zs@{UkU z2t0D%8IS~Ey@Qtj>vK=E+f-gJTXtgvu$j=z^~02+TYQA*p{EnUFpjOkuU(X zhaGDv^Vf0mnw@k;Cd19-nOW496n8e4PXV#^yGR{`fvODKN!_ zFEoQYU9acZanKlmytXZCs5Tx(9xH&z#LwXLoM->DqrAIH~~>^*YP74RE~c9Mr4;NVmKV`Xkc?~bx8_ONu1ToT||-o9BSLR~CtwB>O3Ic4Z z`Jh^~j^}~|8Uyd`CDAg=g5{>|`!wBW?)!+|B#XlpD;v$oMbf#^6dk1NCIc4vrJFRm zdTIZgI&`)4fxNHk6+$6K-bAfVc*D0|zZgzD-6*@Kl$wIWKJ*M-7x%G8WXcga84=Tn z^toD^V~z2q2F{GzQ0YboAIuBM9??Azy}A~<=R(BUfU0g5zRIN8EZ=oK=tCLTNjRym+NUAAe@79r8F(MQjl)q zbdl}eogW$vi^QdiHs9;LaIeDMuj6FGVi!#m_So68A*j8J|1~z>Yrq*BbyE{EpILpm ze_ov-x2L5Qe~B$O3SWyrH@Kas2OlhVMNI#wDr&ZF*0W1(Ihuw@XXA#qR+o!uh)Xdg zCG;5eOSaB9p{ze&Pa()jb)73P4-P{E06pvc=$a_33wdS{1O`j!`krJrUGTh%m}dYL zmT3saBDSsd>7t)~v{1XoY57S3BJ19=!Y4Yk>gvG9QZ?i{*4Ozl&O?g`>6fhky1X@G z1LqP=2Slbk|3Ash0E_hi8*4idyb4F>hL=W^i`--fr_m6xff_9nKFxfxn^6TaWPJ%<&|G`-$1M}ZUOzM( z#qfSNTL%=yJM}1OWfIfK4> zs`_={%!zU@)YOwbk{U1uD6w5Abi&H^+X2vk^5sPTIPAL54yxFIfIO^6IK<`4m_e_v zOH-E(I1zr6SvxSOCpLdd@YB0b>IiKjq{M08ETS1+?Mr*k2L&YO%Oi@We5xA4vOFr8 z-Qg9^s5sKhF9)x|&#<42Ujbd(Po5?t_LrxQ?B68eqYk2gmGVvcqpFmP_BgjTkbdSS zc1>g(yfR3x0zk;{7XLQq5bh(px6>^rkzmp!!5uuMvN>!YySD2FM}M$ zRzk(xsQEt7i3?Kxw#7@?y0J;e>U}GLMGJi5xPO|ddyWTrRjC1QkrUc3xZSA=PN32S z$tZJNX#X&|`jkxhD)QUqd+J+QSj!ij-o)+A6dYf{zEuF)=-^5nSCt!J3l3@HP)}69 zGT+<17K7ejxnA!|?AwN8pANPo$%1kq9sx3~t{>y`gn7|qy$E7%;?6(ENdGa!N;?y1 z)*}tXUdIqBZr!BSZzd~IB}Irex8=PyQ$t&FCLb;=$E_QP8nU>!*ZyI`JdG^`W>r&N z)kf@1)a08^FYedI5OYKXjUi@6m)icJRKY0u?enSkB@V{D|CEr3>{HJrw72%a_%s%m zBf8JV-FGXHg6EbvL;R1?_xVj=N>}G5Cx{KwM^51QlFslr5oIufbn(}L#OYm+Jk+5{ zqb0x8g}dWJ7t7D8%~ais^L_Vn>cp;g2Lu)M5{M-D0)MC!@^}A!WHS@9Liztv6|C$v z*D)KF;yshdbz)K#JXeKJC${NKbCJj*GP-;QGF4n zO*|4uLIw<~kjjpTF&$hNDJ9aKZYB{nLmUT}?GaRS*=)%Ub3xn7A>Wf4V(6N$3-x`? z26!jwI`Z~wjQ`gJ&>v6e@H201_f3w~^OPZtZps0tICfG^YCW6c<&0#N6g<80+gC-y zUr}5&G(?5$iin{7zv+T5)uSmsx=(I6U+Q9cWtN81Wct^J~Rbq1( zX+AGf6A(5p+#@W7pKRi!22HyDxvQS^H-Fmvn`s%@+hhMLGBJLKsu^%Q%6^BWvr~ zRF1cj;%d&`xQuQ8(_A!_t$}UlKGxsAh;XYD-T}(srI$s52?PU1a;4YX<`e!u_C79km&{& z6p-%eYX4?nTzqVM9t<8?U-kRwJK1MX!dUsi(!MwOqwf{*-z3&TN+@^Syw+H&G60l_ zduJ`tyX`n}B1AzLMo&iwtd9z`QSgITolYoR@Ja2kVJooP4eVA)mUQD+adqhf0k3uL z2BK2opj6#G1hJH3O9;&c^mQT?D!AXxjSDRl?B*M!qLK$iGXPh_fz^<&SOFTGQj+3S z4$QCs0u`RD?%*EP!jU*1huI|=K>it(sWnaDl{HYH>&LDi2Xt$KR6bN+>(}aiw;yk}tO*rY#5n@YGQC<&jO>k}0l|K{ z=F9L`!~gZ!p7E2duFmGaDTI!=S=i#r0{|*D1)OvNu6^SsMzRrMa}HK9l$D%wnws~_ zzCs$+@W;y`_FUL?i#dV0GI@$R1(UL=lH_bxf1gee%dx;Ce#OiQ_ZLU`s%-E;6kJ^@|hY* zR1E%$$UljPSf0EI9g4$iWXM*8ZU!J88Md3W?M0>GE`6>B<$55hyFqLj0`OM0Hp+7; zm39_1uB&}Tgd6ZQBo(D{Z6psmVu~T481y2=NF7nm%AJHqEAHCp-AX{^;S_!jpL1a?xtS;mXM$3WCw{r1TR;JLp zd>DGh5cTl`?t zxc(kGL;NPw*>afJL#P4{UzKAY3@8DqD9%SALaK10_| z4NNz#)vNq_21qR%-jC9^!ik?vu3-=qxu61l#${Th#!nSco<^RDBKaI-4!INJQ7XKP zvj3}kXE7hE>O&MEOmSyLbAS_fxiMpSZT+drm?aDd^=lJel+6c|B)_|%+BBuk49wZ2 zBI{w*ioLH$V%1I$QuM|u9D%|hY2Pa{3gR0Kiu-SvIs^WS_ceR;Tz0>{P%b9zpG8>* zOVmjcB5_Q@`?wSyJx_}@w|m|&&;V}x^|4^ z78=+0WX)*OoTXedLGi-YWgiCdpk9qNOgZWittQ{a}e1{EeQZR?W!H${B;7{#I{E~<;Dh)XYIidr*w zR5x!bkfXT3zL^e6M>RZcT_G}*$P&7xb1IF)v9YuuV4bbXcbjWNS?lM^dztKub5qK1 zj?LQYLxA7klUEZqUQV2fc&aVmT{>Z*uegvKZMye2IW&q~hh!5fT+P@rP=zY@(4)16 z%ZxG-kMVlRh?OezY)VzE(4@b_vF`l#uSZi^GQvIg+nseer{;b5dk*K;c4$o;VB3B5 zQXYm}08>tdAs54tSuyJNG_1AUwo7(@=j*#qN}cNl3KkUkG{3)aF)(7qy!sy5R zqp!e-K_%J851+u;NrT>VV;oEplCv0DS&XzUMxGVR+$%;%rxDYsi~G7r#NESF(G%@H z^1N7Yzq(|7qu+ndNNb?Go^FT4sjr1(cgzgC7G|m}UKcr`EDgbcjbAMs3 zM&R3rEMT2nl z!A=ez&m_c9({%S88`Z}idD)n|9Pv;1V~xH{-Xqv(9{2-ch5O_T)X7koBNWI;UTeex z86ilP@hv)OEYto^S$?t-(8mbr(M3D<$od;puz9xL6J!AoxiA&Y`HV?;VFy=YEi7x7 z*tP7eH9-nuvK*qZz*7oH?5bRB+eoYLIGTweZCReB%1)i2!?x|m(3Flo1cF}VxtT}G zIT<5Ec1V|z)(du(^qVe|BrIO##rW%-q1P6cq8ylNTo=kn>N$&@D!NJ)@Y|Q4vEX$dp+#x4|*tK$cAS3bn6jE`@ge zgT|k^T@}^hUvVT8Jd}MkWpvH^y9sY^dyU0|k#XB$#%S(KKZiaM1_mJk_qDAQS+A)a zI#y#AIL8QZN70vwa7fjy>Px?d=5SE;KVpR*3+CF&|4%mjky!Qqs(97S&}QzUWVkn! zXa!Fki6^da2F9O+^EkOCCwivVO(w1?LvT~(K;VwNHi)14ON-K7F-U9w?%GSFkQx46 zDanUa_jMynfkND%SahEhI!W7DD$m?PNwHAU;>C6((TtI8#K;>W-#odr&#dVOGOuxG zA31^vb)z#yv?~%rd55sTo|kfO?yE{ZDYWy#e-V_si-_O&LaycK(3XL)`bOFygsPPE zX-gBgu{Bp48M~I8m?OK1k=n#aF=HfA(Ez|fXlS)cu1WI5#TE_KN&1#3U}EH%%+*lW zq#lXjb;LxFgX)rsB6W396DnWc>r1J|a3URM5cPtVZ^3XhSW9*|dEu|U5&CtljZYim zWKXj#*<;%+kSr2LmCT&i7M41-va~@HY=_q92moNo#gz&Ekx&7Ws*r(evK9PTZ(@hb z#=MiK^LDm1#exNA=?{59Mbba=i;Zn|ijVlQo(w(lF@p-Bd8m)h^elY7Ruck*Gq^}L z*DUbru01dNK{qtAJ?VD0x^eH$@h5IG51(_vm?_i?%nRLY&C~ z%~YZcgYRXZ>0IYN+te8q$-&AuNrMC~e)^kbBp=9$ikA$mSII${XbF6X% z$9F-x^H9<=zy-C=+O8=X4DSOgRD&eb%`&L-CiQqOvFC+iGB`TV6 zJhnhvkVXfZbdd4wRuSL%g6yx|asav*rQx@!|Qcex>S41JF&bi zyM@*?H&OWge6S|7OsS0nd4r&8&{tJ1h8RBmkSB0)&olz?Lhq$vMQFf*##!&jeai87 z%X@(s!f2g!fw3#o-4$O=-*@>nll(fQKh{W!Y-wVtqTk2g$ae#~bEtrpdxUL8ab1)l zotFED&_pq;Q+8JbkZ?Ldo!ACO4{dPM%~1T~qwPmL6wdW!(~dFs?qcodt^%`53bKMW zNv5i=wM$LuxATD8=HN`YSA6uF-I!gfadg_(`V{Tn2Fe1_3MQ`Tb0ck*(-Z&uh4PL5 zYs-uULKfqXK?Rb_oz!G&D?9!k2b*V4Of>I4j}td5DA`wVz>+tW%%#1f8YHJIHl&99 zfD{Y(oA2yMI&K(I><3V$Hw$NAD^c>v4#5+Lxr&%HHty2p__zEI0K}m$BUC?YDRn4%Xy1CvS#6%dbjR z58YBxMuS}%vi2q0THz+-=rIdxrj zf8lnsjG!PSM3FuNPvs9njUBAn%;RCO50fzQ$p`}7y0UB&KCka}54}kx_Fg#johofi z?0ZjM(_GY(Kkvqzqvl)}Ns=-Y!jH5IDh6HtcV$D*3iNX)>y<-yfcS5#A#ehnoG?y# z?Td2)zg&BxP(^wy-A4SH)>Z1B3KE^)om&w7?y<{M$X!_`()iWUzGql%F3gA+4*;~G z0EDJHUr2?KMp{uyH}P3FjLd%#r{=^>ef&pEnv<*8uk8SKvLrj1iGR(!+5YSmRXoV8 zKoG@JhuS4Of9{wj%P_CKyqUX>Psvf6h)TC3Ed7P!q?a+v55G)!!UjHeg!=qZMywJL ziQlj46K_f5i*pG}P@AT#o$>~#g`j0bU`7uM7JKngmE$r6&HZ|h)-sqT8L#u5cuhj2 zFcZ$27zUexgyA-JaHXbFd3`>#g0xZw$9q{VyW+LZI{;x@rp%vEZzYdf-gxQZc5U6> zGLVTVFP`Dz#DU_@`XyLQKI|KxHp%niybJM6u$ay5|KD>ES$=KZUlC8|`$^VC11)OI zOQM&ghgKBmfTQ@L;G|s9nxjxNlyJ{5V&t(~>NlYhO3s(LNC$z(BlD~Z zmsDfev*VEdL_1hN5TxJo;#5K>E2CnTRgy*4B({?rv!+uxp`Lj_KAt)B94H5}70%+F zJ>StK9jM3gP?qHOmajOpnPt;{t<#u2p44qp8hTQhNml5d&fm?<_KYTQ9QW>Oh!Z?3AaVroR#^bre9s8JY7K9f@FK> zMPPwjuKI!+;;{=w?6#+~CcPJPvFwbB87Ty$&n#RjgQXMP1A9070nF@OdtvBmy)F=X z3AK@cw7<>tOzhn&>mU~upervGj@}FUt2Hu#qkX(WSdATZY1zc?8!ZqEdT!0 zdz41SP?t}NG*LH9lXwE#TEalbHTcDLsHqy2{>_3XddEP5pVdVn$%O z{0k!$vpfNX3?k#I`U*MGh&unXlz@e35C@IeUL(r&K=N^-K`AttiUzAOOO+|C?5O|b zwvj;h)N;^GSq_|Dxb;nG2K&zl0J~+j(dC8+Tbg+HduENYyueh z{MT+ylZq84PU<^SF_hoUH#YdM{wV~!uAb`wb8XabR;SZv88;v0MQ+D`le^aC1C%DV zHLpIf#R$gAQh12|(5U7&i%d|> ze+M{pf^V{=fv(d)W!lYA2&6^_oNrxopoCstz+D(aX$r@PK_7cVfeHUqphrgg3D;1f z(Jg4itZH9HQ)H$9C?ug~b`l=dg0Cu5Du zE_A&Y)3fNvewjfEEg1>yry1j5RD@V5(8)fdtSlHI z@%h+8iIM)uGx+z=-LHYLBaw?8{Qo%7p#G;$nj~$V%;uW$1Y^@!@RjC?*Q?ck`A!_Z zyh5rCbtGFoNcVniHjB;Nao=Qql$?Q%GB7cqJ>ufjT}V)o>^PU*Lvf-3R`hSOIVj`* zrdEWxG4?K7H7SOcr`!dHlCOwEXESuRC$ZYJosR9Q>;1I{_O?EDyn)4zv*-Fb>&O&* zftixG+BT0W!45I7|+o9h)?pGDS4h2;P=B zLf9#9l4!*6<_MR2EjuIX^Jmofxtj>`5SlVJ(t-1Gzt`&rq*K6&iYS}M4lj(b$TA^% z`}uu1H^a(YL~X24BEvr@5H`0Jx+ZwR&-}-f=-?~uz-Aiy0};sv8D9C`;s+E8Vv&tF z-V}vfcNnQ?An#i?aHukiaboMch*N&Cy#UC`u;1oOxoCZ#Ze-nW(vFS9`9|J;BS*cF zbbknc(Ofq!Ioy*gMY6q6AhGc_XcI601cBO8Mj~ps5>P7gWL4NzCI9;Iwi0U1_!a#D21lP);_v}n0z%}yODuZAqUSvD3sUYMOD44g! zmk}+60CTtIzJI?Jh~jLffvigDM(;Kovybm)o>AN9z#Gcoz*uZtRnMePgp$sAO{EJ;Aex5j#-FlsS5YgYw~A9J!s zi(@OEQ(|MH&yKBRYQ4V2Lr}e$0Abo4tf2Aw=%i5!j(1nC6!OLYDRMHpB4us`#;*J_ zQop7Uo4J*>DH-~i{u`k=V)o>V!4)dyX`}8D9@E)pjSbl#5aBeLKbV5+CS$82Mhcmp zz|4_L#73?uG@v6BeLM%6&)Qkr;5ff$9*@21rGhBGQ^=$pHkV>W3QR>W!B1VY%x}$@ zn)%WUGIFTPMH|{iW@WIJ5i9`cBucg;|8}`X-=6KN*B(W%qrO6l2 z1w|K zI97K~Sv2f%zow^lydgfC!?PV#z*# z>{S8ALD-H`#TgLGA8`Iy`dNch4R<^gv) z#zpr0Jt|GFx}_O?IOv-6!SIln^5ni>X*g9(=}L{IXE|nC*3ZxPuM0K&mg_^xt0&EU zZg|NdoZCFSCpOEm$h5gSgmi}^?^o`tXOiVvkCvVN6flnDw{GcLzP?#rmu23$Aphge z>DVHW%CD_+^PKbbh5OdK-vi>~*AEWQTuV?+Qo&C|hj|j#$#Uzy{fC(oP@nO~JKH}Q zGGuf>UXys*xiaj?dTM9>)~sIncQZCln%eiN3G{r0*QYD4t4n-!{XXY58G8LC*l9oN5T0^f;|7`Rzk$ZC+}u{#oF!ryHyxhM#_-b z9TLhT*E}HBkY2-^*{?k*LNa(p`KHfGS4oeU`>G+S-X|x%6(8!Q?g)VObZ@olm(FwbO^ySOxH}t3 zewwtK9z7I9GFYfeIIcc5kT>{E0M$x`PVZN4w`hf=Yq0sI+jk*sAI~VjxzRbBTLX2X z13PdVx%8Cvq!6@~^!L&kfz(g|S_x9%y)Sy6|_+m0$i^J??{KP!WUQa zx}%?pM?a=y+d&HJJGkZw-^K=CARwN!06;?@$OyXJV(+|oC)$e(U1q5oxt1EQhHBaY zz{IuEwbaz01o|eaX&%<=^~J8QHQr9tTe|s*`3Xh+t{e+Tdl?e)`$o|WMLBM90$j34 zyS`@_)mWo=r0!6X?cuEAztgHh{SEmeg2aQ^^PDKV}s@w5KOPW&Hb!jc}>yIqr zsBB$MtPdpUUS4N*5nTIQ9}6mCXmBJU+gewbZr8{Ag7^tV|2oVTnD0*U1&3km)DTd8 z;=ggu2}A|OO#X??i2)8egeIrK)Zd}wpR76jcMF(Jant=r)z(_rcvSAx$&qk4yy%NI z)TupkYEzzIhf8uxYna-(|4dLm;eWJLI9-eo}Q2;Q_Ar-h@^{qDnIg6 z*2M~GjfLGrEY%|0X=P#wDM027C}^A0K*caWiZLnDtShfONju^rKtN6ZKfJ{|MWJ+J zRn{8ctPE=z!kU4&DKrEN!X_VDY@ZaIL#EBpk{TFKg!dZZDT81eNYjzbDHdC91Na%P z?19$Jki9k3k#w{FF$dpTqpjI0Z=wv;pE^;rrXEK8snBfq*b)@#W%>%?D=>k=j>$0TlIE4HKqn!gleR)aq)O7p=N}YbVYF}Y zBrIou=MKW@!!0%W0YtTcs#dI|k%-8)cc@@;X6nd2V~773nf-(sjl@XOS{GOBzgZ=u5z+-8A*5 zERlk04ake#a$(rX_8Mbd8_6br*k2Q{i)&J%_iYqf1`}P*j{?T#23C|U@NdEu6<5`y zzB!zzWJP=Cxrjvf>E$F-)Acsalu}Tc)pC1KE=CeqrEPe#o!qo*1N7IqVIc)c#l2PQ zGR_p^BFr?uo0IEqKo-L+tYi)>BvCS`KLr4=qH|Yh7pmkztZf{x)obN->4GroIsu2PUco_oS1S*J7S|m zO=Z2cDfMYeZSsr?zAjGZdzh&ckFFCfEEa8%hFm(m>%t&QR|HdtG__Xqa#^+$CK3L{ ziMt*;?UW9eCO$z)AJIh%BXk-%^YvLX0vDy2tn4o z-Bxt$J<-6O7h(<%$hwAEY;QAncXhxt9fa8rtkiKW<5b+wy^W$#mJgXt*&gZmRhUdRW!N1P2MGJBCU}$Qkgd^ExUa)X{2I) z9JpkYYLt5ksoUCCqm77RBUY=V(gNl^j+st*N%;o#`;t1SaPw#L zO7EH3LTXcQOBGGW?dbnB;i@qjCKFxBs56~dWm6}N!tVpwr7+2k#bApfL_%5u5Ia=P zR}g{TK|e&s&B~V;X(PveelspNsb+FLi<0J4Q0~HXRC4s7<8`4sb7L+}(79|7ldk+oxs0OGs~4xa_99Z&O~jOsboTa_Z(|y)C^ytGu>Ngw?AjzNQ{fhT|Mo zfsxc;AmoLbvBMe_C!i5Fh!hgEFj%j(l_YIj($r96+xuvi^ z^L8HNyIq&E_tH^%=OAKoA|{<=S2C_da@mW$W>Lh74adw*>Rl$9nwv!v{?hqlu9-3R zzO59p6gA=3N+8q~=+r5AMeM~|6Dag`tEM02M!w(^%mZ{m)vRNIFk(ldjX7GEQ7I^4 zkkn-@b8C81XT`5jPn;dzQ#BZeaGL<+hf|=^I1n@la=z}lgmF?TRrA(aFNskI7P)>V zl_9~Fa+)+1-Bs1WYkUdp4}sGaNEXAM3rB`9{nQB0!|>Nbz*5yl_!%lCiD( zpT2k}s8|V24WQ+I4|J2JvE`H?g%CkEOo5~a|Bl`bfhZj{sw6(*YD$mEOuSHAJOOFCc`z zZ~$eUqrSZ)dr|-cw)#z0V5FaZ_5Dh*KDRziCf4V3m%POm5|}FBRJNaj<@+0C?HRO1 zh)do;sa%e5d}GhEad5NT;D~np=x|1zECmq+U4*dwte&IAmg^c!Ie9DB4*uU4nDiY@ zOaw)y!0g1Bktaq#=DHKqF|Vr|n(Y_B>q%wp_2|YsRn2Sp0+%AMPn9rp-tpszk-& zN@ij}#k8s6U>l`OO(A|wWZKpZ)Kj4MBP>!S3i~O={svwD)0cg)D~v-yoAF(@B^;xV z`g3Tpc1h5#9VFwlp1)U81Ay})<%5(AapK$;Das)$GfIo>+NpXugmVYyFB{MQyML@9 z={e98*W*fj`}HGb#`+yywQa!INs`rnXh}H-W}?;F5yr*WLcn`p+lOV%uwfd$DP3X6 zQcjfMydAnTGuZ?K0^(JX%ZX1jFDPmscPt@VLSeFJc%P1E+qJlWXU z*w!Z5Y`n2;+qP}nwr$(CZQFnLdEW1T-|ACcHFNH}YR=W&S5M8E>5;beS@qLF&qm3V z7=!XgQXeVlu4>R@!u31TN_Y=nts}=K`G{kL08W8T26n@EW)kN5(;`kdY2cTPv0MQ1 zu=Zrp=Y%2u2d+B~jLcMcoP}=^ROp0gYue^S)BYW>rV5lu32xHzcC~`6rn=014+w{L zmbj71#;;2C;UgI0iEyEp{X|!tq(i53n1y`H0M;bw^$^)is9Zw#|8cjwEaVXf{YK=W zu}y;h10{A}FL$h$ye))k!c^N~7!1wiXW~JwUt;5ovjhBg1$PES0cO*lkgVQr)%QB` z@59eD;omE5pDP&R9g>hGS}1T=lU@H9m+%GSsTn(e9q!Ok@u4ya{cD%H|3m%STD%ToK2bzN^vI=$&(sNaZ82e?F)0w%Mn@zRRL-l(iY@mx} z(+HD05go=^$O@St2+$x~TMv$2;-B!#C-v3qrpJ!bbHg!a6q1Bs96J}&!Hp>!cJ#{2 z2HhI-7K-GQN%yNt{EL+=E(BK*f6`aPG<1Z5;C5n6*f#lDW2fCUKI?6txu)z-nd}`WQlRQS1ax1O)GJQK_O_$8M{5K$lF1%}i_p%8Ql@A0ys!T(aOtyl zLN8N>JY2IP_#T)@%5A>8}AEq)G_l0;YSuPH%;FgBJ6a)RURx z-%?YgtuBfqGsUcr+c0p248FNUdOk|7Ni0*%CNiVo>R*s;uXUB+Y(JfbiZzJWehpCt zReFUx#=7L|z5+9Y3IgyZWfcqW&bz)myX@tbE(R2zb?iaM!t-$vQRnQ_QKHCS;50x| zHT@95{WiR~3n?pvOoez;V&QIRs%AG_>58>sj7JO{ z-f*Pp_*s%&+|X<|jayG|3GQ1UCeB=*G!&=}#kA?g$TE8b|3@OSY!9z~5$|I+IBVZQ zp)3_aZU^?0=7~Oe%}oEA-K?21JIp7=$=ci7ThCvi^|=k-{9aA}$o$7jPP$tj4r+-* zE#U-U9Ec4L9e9o?_6?J~@D8=m;c!UckBqG92A@5DrNhk&DH4Xi6N60pD38K zbjz~gE*_TxjEik2Ij{cK_Z?>W}(r1qsK{LrJ~Mj!1*5!iLz`e0#!FxYhfkJHtUQQwoiW7Wl>Q)6`GH zz^)v@w>Wy#Hnt`8kFt)?Vf_^_mSEnoeO^bBzhN&jiPvV(dfGb}8*H+g2wE2~_=9G3 z4lHt-AJFv{tRqH?i%D03tpKMn9F`c?QD59|G-rfQMK>|ZD%|go-slVuSY9m$rbnOq zB%K?P>s!Ml$V&cxVydH>gKY2*i#?&`q`EBCMxyX4NKPf=#qK%@{&QrTZN(uvAFq*{*`L9P>^r0%r;69%5#=AFQ6TgN$JDRQES1CrL`u#i}aO$_L|3)-A216vmlncs9b z1^aprvvte@588N1aWjXhW80vP<_+q6*W%P}plB09^GgA2-oNbDbC`4RMhPN(bM~3i zuJ3~VP&6zQ1@{`L`7Y53R(EYM@Fw%bH`s}v)Z$GGUQZnR0#KUGV1CceWZ$3#-Hjwk zfp^gE(}gK&fO)h0UL)K@c zs9qk@^=A@zdAq`bSTh`b%mWKtCAZ~xY^$o@wb)I4p0SfrMlI=>42>-jh>y4;E-Zve z<70O{jLsUKWG`mlX>dkG;G$s7Q;6}qlOirb8imIQ3*U8!cUHj9Ob!DaZR}C>?O$xd z%FRxrs#ShsqFFv^Xn!!<0>l4c1&Yo^3*WWBE11^zb>pf%O~q@?L@C^eE4Y$|&u0|G znp+zX!v}To`3*}jFbDv$8nYB&PtPNJ{&}UfK$J|eVLQFmUo?XLQcxcB{2Hr$Ka6o3 zo7}l)5g&V#i|O8TRs$UP1iE8R?kz8})5E57;fegq1F1dYTt0TeQ`FIwDaAFGRM50^9@Wrq2DYLG^wJtC^#g9za%f9$M!5?=pGI|q1SSUk;)@JCi*4kY+@ z7G!A<1XHsc`(7$xo5gq~sGw`HG|Yfm!LQIBfY;toBVh2)2?TQIuQlsoU{F z$EEE#m!I@YNxwS#xhIh|rCfQQ7BzICrZ!J@=dQ%M%3eiLcCg^D!FPQVjE;rAy|Nr# z0=4)I{6xMQc&27EH6i`MOh2SL)?G|0Ie33-ulatlH3p$-HvZke`2c&1EnC&tgcKh< z2~xB9VVV`H&z5M3wG(6EEJ`jtB{{(2nL>6n%|5g)zSK0ihlZzid?KFX%xVv5r$!8qlfm%P zi)QakuXGCP%r$Un$I4McjGrn>5WCx-^Tw=0qe-VWSdz#{ah>l;y#XGy(cFcyh)V*r zu4#m2_O28kJ~EWL!^%V12!gx2zm>p)=wff#-9>-5$(H)f$&*n%L1IUl7SF$C*%bQY zJel}Yiwz;M946xFtPIs%T}?=<(XfeM0Pz_MdIVP0GtETctJE3lngEzGx&k?BK{q)4 z68_KFql7LVhC8}U;ol{1?EXvP-i3HPp*~)q$5VhAz05nWysL#;eF?K=cn&P%^ho`l zVj!DrOeU^zMjzUE=+yzY9(>EB5#?vSAr;*p#t!>L= zNDG4OHkoSFxf0dBs2qYazfc7u`*FiA$`g@F7Q99Otk93lDb&XqVS8#LtkjYD0#QldkOihgO79 z*5HUInsBbj{~ie(fDt}*ZkbNjQv%DN&dNH12^W@Ayq8PT_{+FiM0SUs_-VK_WKj?Reoq~FPk;XsCTKI1?#T5w$t9iPadHVr06_6SL z=;e50#p~&}pp`9gGfj&?5=2J%>!9fc{ymXRvm!o@Io-$V`zt%JU13FsIKMh!CX=mO z4Cj`;`^gtl0f2HmooL^uIl3O039<39mb4vv#G+b3iKZYnMP61B17A9Ce-<|Vf^eMj zJo3F}VsI)#Ut#r5RQ#`m1&ReLU1gr6^BNkL-MWv{_dYsUs-%wPA4IG_l^8&fbS!TA zMRbrmUs)I-XgVMf$aCFdJE(3UZz{s~)V1Ko8=Yrn0p7~_tJ(|*-)r|atkZdDpp zEa~=Fj7BFtO^Qa@9a|`U?Wh&92vsf0prN$_zK#JqWYq0S+;~eMdK0#$ffyb;H1Pmt z^L2#QlyF&oMrli}tvWpIIV6K7*KP*Tj?6|1IH=LIJR^MSKx4cGi)ISFv_}W8&l(C# zsCUXtTbx?FY%Hoz50R4QHC~^iM9`$#{b{~Jv>RKA3K%$6J-(O6Zp?yzbkD~2VLADx zo_ZYYcZ)74rzB~=EEQ%x^(`7s;d?leBP<30Uwlq~m%nYn5Ldq^cxqWMar z%Wjkg%lz8EnvE8XK6qZP&VFphIUI94R#$5?tbPeokt!TD<;t^OT*O8^|6?u<*h8~2 zUosey25>J#qA zgo#S<4;H1eROhk|8d}V=MI&rl27)@WD*wg==$8dQR8D)%&Apeh7B#98J-M%qc;$RjqrD?5qC@|q#;`XP6h$3LX|7asI+(Onud$*pfmk3UwL zKc=C!vGUjAk50dRU0};0R0r|mnkx{TnS3nyo0&>u3ouRn6J=yd?wkpQK}#Qk%nXD7 zl8kW_2Is8dXMR~EpuXD^p1JQ-dCfPj+IJEW^jRh$@=(N5Y4f9)W>lAMaaHZ5#Z9ou z)~nPp$s^{w~pYPV=y-Lun3Yr*uLV@Kt9VXVGrOcIqvgYwP_ie_3LE$N3(chR^R^8Ys?C-gg|6%$0=J^P^mFRt zofyHMePFA8yC|~q_p{Vx<+yn6LREW>J@gTx7xVdQzgx$8skJ(EBT2u{{4AFSX~0Nk zK-@(bGRN zaV;I2&at$=iv(^PCGdZj=sD|0#c(pd zT#YgPIKCt6>xGC2^@Si==-Jcu!BGA^{{6Ohh_=KL%0E=hc;=Cot?2}8EjTVgznWY| zLPG+ep>M}1&+~$5Nkb%?D0ZmJuHu;plHBfPv4{%EW7?0dyXn-^jO~sk%!`}#o#V15 z_^?JVziW=_dM_i&X`P77dnW0(qjT}i$?Ul7rU)o5+pEbMsnsa(kCP>Wh?Tu1!iuvH zVnc#0L?%D+1K^#Gp9xg5bv(6|hZm0JlMKj?42T#($=eRuGs6ME@jry8$OhLLN;%Qj zM8`Jn?{SuML8*NSK&U}@PSt9}r4?OAIA8VbIqr(GZ|cclIAvRv_wQhHk#)r(;VOJ31gN`4pMOf__LN;bF^A=JokiB?LMowh&%!+xG47(LX>K;U>l!585jS6DHo9&`r>-Er zuRnGBgKJX2Y$0WqMvqpQ%9}WxH`2$>GMeWZD_z%(Xh#P6!#X27rMJW^?rEaR(duB4 zG*;zA)P5Dt*SIw4{(Opoi=C|h$39dP_&eJ!8N9tE$~VBheM zTPlZ|QK$zA>w~r~qBwMrzdle8ay0)1&lb1j8l&T4uun3CDQQ`EJqC+E)xlVqD{p>1 zXBTbBVp}tqMt4=(!o$v!5??ka?>3swpvGD&G4$+kdG83+tlLBO#35m_hEN~H4Y@D- z#-|q0Qqvn4p5{kXoWVP~1j1>1rY`C!ajq*%DkkodBz(IB6Ns7e!CQY&&uKD7c8r)h zQBIDw# z^%$dssoE4MDW>*Om}31dpH3RhLP){^AGSozc7Rw8a_*W??U#hy2tH>nzDC1v-;vzu5E^cuQ|6peEi#16?2g#2T@nt%3Onj zDk#`k9@(Kb2#gP@hOy@HmG!ccS)JD#NMnh#B?ppPf0|w-;Ji6%Lg9?EJ+G5`~hd(y+x$KO`r0W`UT&lTy(SJL( zjC8odSmlT5Py?%n`sIe;2M|GIJFG$z#wcvnyt0H!lHk(ComGD=tUYmp0(_rjmQ%3p zWuv(%t>oSUUw#sN?!pcoz0`)Psgg$QR|-u&HtRBiF#{TIZimYtxh_e}k4O+KUv->H z4;(g{N*{*s#(F)V_B>&Z1!kt-rQB63c74b}>=p)J%-IqdH;a43k-lGuxqtyzz{I_T z=i61wK^jaw#$G~vtHu0yq#Y<~)0w*rrYBUPx0N&*#P^8L<7;WcsqU2MGF`Qcw~}zQ zrgO*<*(5ESZe4K@zzi^x1A*^)nw|hTu~{EYZ8Y`>5MStNaUHuztSxXyEmI{y4qccP;AlzmlAf;SJVN- zLjv6;r*qR(dTB)8Up(Wb4|2XJ@jkdbUGGlWGsSN+Wb7KfdAb4`$H|%=Vn`}~VG?>7cwt4YRF)qKj&lf~Jnp75q|mmMXf|E)=56_2d{O6n#HUnV z4_?}0X%l~=6MqQC55uexp*Is*J8C5P==R|u82!Eaou03?=HjveON$|^GwJ$Sy|8{- z*H}v%0~Yf8Ie&$PSWwit@{A>FrcEDfAtlwK+e9fs_eg|f`%FkPS^VBScp}62qm#1X zVsjK*+FlMdg(Slr*PM~H8F+JajSlCm7xwZwCLcfLBTFq=Wr3Mvi43DXc?X55ogx00 zWBfd)pbebPxl2lX`yr_#h-lGYI>IA1w*8@IQSqAj6EL^VLv>E)P(*XD&3(Z6Z5FIU zQqYX|<_mq(^yGm=Iu-0N-rusg;@S|7HRGUh>S?P~eg>J$zNOT7y4A>v=}v`=P`-^%AIO&Tym> zVs^~IS1<`NFe4-!%0_r=d_<=Y1yEML-4Q9`Sp4bEaOv0~t35^5d1bzT6-KLv3t<0q z{ktNuo{5O2KLta?mUIVmbY$hhF&TN9f9nSqw4y$@1@L}X*#N>ABS^US@ zl0SBl^K{+C?Km;wh+5F}X+3$QQ9Z<$)1-_BLkO_g@GlF$*9ZaOS;al+r4Q+`x>U15 zhMb`;QV1qHJRjq3=nVI#+cL#i#ay2VCYY>IlpwXTazEk)u39-?T-DLJd#Av~JJ23F zn9sLrRc0I$-{=qG=;ejj-iOmyf4BN?UF`-KOYKHi1*pbo!P zE%F-9Ca3%Yj;8I%ZHkV#Mx`m+tL_x>#lWkB#7kG9!}!TeS4k?R-@E{xxi50d1CZ>} zK;)%}|ITe8w^lIC?sHNBqi5wRozGh=!7Zm7^`@wNRyLAYtvRJsLZC0pTgk@4e>4L7 zU0|D)ki7wTE=>nED~(ckYK#tTSy6nf=4rT+$waSe$W99M=GUOoxTV6!#K)|^jh2B) z?%JMgas~sFoI^sMs=H6zmcTqy^Gk2g}oE#q;N)M>gOBzexR02^i5IC)_>c;P62ajXTC;UKz+mNJG4 zhTcN3{Knf?wpG^vn|a&w{{E7DsWg5#>?p{}IwZd?1D< z;wdOe-#8D~SA6Uvk*F*#q+=%@pj#pjPfZy*+ph%Rr2EC{lNthXeY71ry`0MSXk3OH zv8*`WFj$ z>A=IS7j~56rtn681~P1xeewu9;y7!>k+&f}V>ENr31hWA4EC1}qc~=YkP&;I+uwZA z>1&JxpYP~AgbuRZvoz!^%kUm|)Strm&HUypQ(OGmp4>=O?XIo(KE`;NdTN+)AZZr- z8`Js6>+BSB?^XB=;z~#q7~={TaT-W=y)o68PQpFnAXl<(_^j4##_|nHRX9u2`{~mD zOUGBO_{UU=qA}1$mX>-D&CzCHN`(*pSpDmk4_Lc5Vv8EapAYtH#5kAO7Q}eT zmYOjeC`}6eD;>8Vg_d2?`In7Y3VSD>UcMzP#t*{t0lME{53jc%2+|{dK4(jzY}b^y^$XuvUnkEQZTGy|ORUp$438}| zy9Qd2h5O!iM+@tqF07vZw%=Ge z3_|caKZ4^>d|4eTHRs2QB@=|>fA zNusP#&@7}R94@ke^+x$XID>NtSL%wQ{VlwWmKh&tf;OCRncP?`qxt>PaeRNDqEGnK??^(= zWB~^Ya&aqCnBc=v!!cl4qne(ypk&*5`~Dab zdL4!S!Ji(i+RqYu?Xf&t<_cRcoVsn(`xQvlUhx1~taF>yvkZ(XhS~UYJs`}0_$t?0 zJ=lZVdVlFrDeTcF{G_%1EVs@W8p47?bJ+&ox%^p&;X@jx;?P(=;=>9$Dpu4!VzYKJ z=m7*@1@Kg25v^gBh`pJ)L|gZU!}(uh06xFpH#fqUBJ zF<_q;woAf&f|Rkf;Iuey6c0G^7zc)IUKu~pI1W?Y@#W5QPbFERZ(+OYSJ|XsC6*fV z)Sqnebn_yjgQP8w>YmNBk9A2;Gp;qcR{@Sn(zYE0Q(P|y{KV=%zhG6i+e4nHUy(m*8x~SDY2Vruc$$nq&~YS#uvhHpKf==Ozch_(Rm$<=lnr? zd`>(=&&HJA?gf5rS|s=9ghQS<3Q)uLe$D04PO=_gU<8NTZbz(%S>Uq}F_z@R-PfxJ zL>S4wXG7V&b~Tm-e9m`f$c zjwP@SMqDzZW`k`50X7YPQYTbc462jg2XAAe?}!VASL7M})eQ9F=JY#ZZG?!EE^#oP zJ#L#{QHLkF0a%do?NfM8mf4GHr<|nj%EZh44`2R1mJ2Yk;sd`Se>MoJakGXJR_>b$ zw97rAAQ&zhk<00-WB#zeZTFQ3It_-TOv|>?Fc}@?VBNY$GVE7{0@D}BpWGsG5j+f; zz`2D{7QxSlr{S8B?KqC!j_OaFQ91BLWj@2Zn!pXOYO`z=XQ#PMJd;!%=YfboH}<)> za$ET=$Uz-6QyvyZQQ!+o%bVg|KL>13dVa0jy?xoaVVqkH0wLp(JMiaEQ<}3?~rXmR|n$6tM@cdsH$|Z$%51lfC;C z4b8hJ27drC>ul07mUxufxe95?Osx>2%oHL z$~p~89-ShFD@8Vgg7Le67 zFasf6m-mMmg)y&ZHLfU_qUE6!40>)3hB)H@-M)g1VUq>aLew`AG&2_j?d=sO%Z(UP ze(fUJi*$0=hL|!+2O?g$aG`#8wjrY6n6N%EfdL{iNPS^|HfFK6+NJu@VV*L~o)#atCWw?;c0AUzpVk7{H2{b!x;sCotNb^btXf*d9#3&ZOgCR4 zZ%9>jxVbfp()>h6qPOa4r`$YM^1%@nob}?S9yqDgsPGf1vYkQnk7lA}d~KuEC5bV~ zg>opJer;_@Ro)3aJc9y&ddTVqew)(4#}WuS@*J` z^TC@NQbE7uaFS+H3oVL9I)gLWN<#XF?Fd64GSPm~aEQfM;@>#ntf=`hf;! zj}Bpr7Db9(i3GHC+OLlRluyEIug@2#88aBQI9n|V3B&|8<`Ny}`=bLyz8M?jahT2t`MjyzB0 zM@XI!z?tvpRz(U$G?Vs0*UXe=86=}QwL~M$ovHbav5{GC60xPBcj1dw$89&Ih%2FX zbuENRk?+A_{TVV{ax_=Z?2Fb#e#Wtt!Ijxscj3>AAUibZlJHh1Qbn|i6{r1-RR`AJ zAK7x;qjJ;m9X4fDCR}d@{9B-fZzh7+fKSX2*c46UL1`aCX?_wO?fr za7c=bMn5PT|3xGxs3~e}mVfp&m*#af*UVQj-5kr;2JsdlD=iOF^fz4*YHnL79$t+9 z_2ObU!WMH3TK=e6dh6+)|M8dse|lNTLIJxFSZs#CzP%_BF9Vm&^pXOVdtTmx z58pjM%yBL4QP$fvMigF(O!|UqY*HuFD;>JADAVafzRYj|1zOfdc-J*&1eMZ`1d+43 z^5Uu~)zGV(Z}-wY=@|48MRJ134+l2Bore*|78r0ycZe(sT`M7IlgZttN(_KaA#I}% zv?;L5vF-`r+Cu{+hofhT+&nXH7cc(GDa@?yH=hYOyDfg^E5epnwjI@L`=T~W{& zcXuz8u^1`zUU4QgBgue<&E3&6_;P0A%%tUh<8#Rhnd*AI?6}8Qj}f=ONO~US;YZjy zLD}0B*|@Q{@h#t4hR>Yt3H<#=u&57(#Lby`cCaXlFsanpvG$~U4m<SJ) z9v00nkOhZsjm@F$v-$G+xxK#Ac@1iMDTOG{F7C3_4NZ$O+L*WMjIF~ie|KHO0-~KN zgge~e09hOSXXPkMFGGVD@|UmSqCUOB&%s-SKNHQ{#_g;h4;;~W;xwSd+Pyw9Ek7&~ z3)}5_D%uT45`QfF52VnIG`{u58te+DX|nyc5kz)b@LuZOxPnl3#a)$EQjH`Pe~mdQ z9&1H2i>+_3q2&p7NZNT#w_;&OR*cTU`f~*y77cneoRQ~N27@t&o>*?+72IpU^c1hV zdOkFnPVC{jiE*H!*D-9cYV%0ZwwtQ^er;ksB6~TlBJ;KHt0?zojO>RMp|>J-j4^D7GrJLdz^|kg`qm#x`^~Y;l1Lf zf+^*&0TWMyNDkBrUB@L%kNb@L^0T@6!jyD12E=J0c>##h-g2w{d;W#;QlIbYE{S?%o-#-6KXKqlah^vmGyG4iMH^wBD{i zPauIXLBb*Jh~CX3jut%pNCtzkT&_HK%$lgImR^m7S&1Yz@YXS{O&+x5US8A|;A3G9 z+QbD8gCKd8SbaQIZ)^411(m4c3n-W?*BtD5+(YcmpI~T$U|*^%_qaKVI?LR($0bGe z=q-dQ#`Fk52PY5uCYjCqT3?G+VR%#e{&Mq1dLcUpaJw9Rwyaz9dSB+TCKl-u&^k6~ ze{t-E7_Qks_l^;(aggcVfH~#$1*fe0ILYqnRu%3Q;07uaoPs=_adfA1Sv}%vE+8oK(6x9N)LTPrCqMMEsF0wyc9S4B3sa&baE8 zZE}x`rOGAuikvM4yjTKTDbL?kS{0@T8i~`w@$$l08$r|PAt*Sj74TS0 zp=3bDjTmZ;2nalWlq&_lEz&3@hPsEpNjYC1y)v!F*Di6GKI?Z{t=S?+TWJ%jwNXVt1A{MP|o{km}Txs}|r~FQa z`r{GAo&K{$(KAa#hEUzW) z@lgZB(79WNNidJpLB2r5i8!!Z=+AH_QJLkhx*|x&0X7X9Y z9XH9c8=q6H<=JQu!cEw30udwNneaKho1#oEUT?;N9^GZ>rt! zKoEo3Ct(h6fNK)+hnJYZ9EN!PIFx9K1mTF91~2LHks|GQ|8aAR%vSvdj+yMyRjKLJJU@wixn=!*MK6wgI< z9*8$0&nF`p_ebn*qII?0G8RT}v-r|cJ=Jt?Q450c@m$+<64qLY9-`1M_d4;KmI%GG z01=Nk%mhl?ef~EbXnm=Ra?AaoopehM_q_Lt_T?dS+S*qL4{bR~h2_9I|1&={zikrZ zPU_|x#=fwPInDOyJ~Tz<;^*k?2t4~<;F{*zS_(Q!^^6MjY^JAhn5q?Ib>EXIunjvl z&{nThxH*`2S=ew~HyS171Z=7yR%M3$+_?2X16%W&T+L=Q9T82fHxKjBIV%0)KN&rP zzkuC$-cp=s6rgB&(6S0E&$>{=^Ul&Zz}-QVO@>A`kBb1yzuuYaHUF>Yk^*yrxGVh%*Bcl-f6;z*iPq<82hje;8j|1E7#6dnf z(a{QtU^_BgDb)yE`@8B_jq*zbFwj(3hF<9wr zbT7+?+;!cjZ-&^##k2{*;Fm>7uzGgY{m}UL!&WVToSX(bM{Qu^;7T`+*KKYpM)a%y z(MwGB`*HS!qc6BiJ{Jr<6tN)Nm8y#lGGYy-H=bvmakl$#Em%CCx zexQcFD%VMB^Z?vGM&4Ori zGF=AQXurM*Pqb`v-@V7W+)Vm504p^;k0pO%z}Fn@#PWJJULg>VgX8Q!phidHOOFw@ ztR)7KiZO6V`M?XHBMlSQ7K7GRK%ebNLYru_I^fqUm__r~-FLDz!)71sj^?o=8JI8? z_Z@>V)L^~sN&b3B3cL^teV>hiHqCka|98tsS$L&lwVn%rbW|f+R!yr0ctybKn2E8> zyn^?%d#COQ`TBz^s#jq5iG&(zq^PDQM=&WFSIuaR^RyXX7lg6^Hkh0UCr#22k=BS7 z#FCdS=51T$-_8}P8&&zWGJH?7)Ei5ai(+jl^KZ>#Oi7vxs9qhTe(Yo!1f&FO4IbvM zV3-&bC1-y%v+Z!Etuu=q3RaR$0&EQb8i8?s3|mb|Y{1@{DPp49stnsQA}o3j(-_HO zhUB+kF3(MaXc>miIJ{{97kh`%e{5hpWjuw!ILzEL@HKb|-!J4jjM6r1Ale*Eoq;&E z`mroGobCr&8#6ez{vz-6IRHv$;6)D>#-=6a6G;yUbJiLI(Ir4<>`sFI3w|#tCB}u3 zFs2?aIwJw)7KUw)L5DzrKRs>O|KK>$_QZ)H)kU;`U46iv*+{s!RoQ=iu*B*R;Y-*{)O;|+ELg#2GahXk(Q**P zELg^NH}m8}Bhw;b6W+(ma+aF<8--<(X!2DcO8p%)9H2K z^ve=j;OMelNx;d{Ye}F3RCER2^xyRwH~XjtqQR442K1Elcaz|L-ZC=~hLLNFLFr1M z$Mz(l4Ajvb_~T_7&ePHVFT(#d#nQf@QPjWKlil;WO|#A5-Ri_M>HQlKnPoUfVz7o9 zLKWKs9oCkcO|`p2cq0G)q>puB1>>qIt?cd?UnS#cSg06N*1IoviU+3zEc}gycp4fM zYJHUtL0^`evH``+q-}m1ah_SnXf?_h(~@OJ%F|g}DOG8ZYnrcSmo?fjvp%3ne(r?^ z(GV{{Dd5%>^S^y$?~OtEa8IK$e_px!A#h!5M>HxUyio` zty9iD8AgjaIALv-$CJwgy(LtJlHK@{WAnH5uRxo55G@XcS)=V?V$lB^0k^6TS=K<7 z>rST&H^44kp)4D1_h&%=fWOM3(Ruw54y85rO&6N%xJNz86f|rN)Mo9Ej}(@iAf_G# zq-ZGZkq+vr-Y@jq?j-nR3H(tA%DOFZw^nb1OCL4LA?7E?ukeJ;VWdwBjxMhJUsGhN zV+8LHeT76n%{uo#TofdJ?-m`e_}BIS6DZqbp^HbSM7$ygN(ks~x|Jnefq)*~6-2}pJ%brW! zy8Igq$zM@{K`i7>&`Yy>gbgz|^4I?#%(SljRpQi2hw4Mr2m@!;5lr_?LcqgN%WeN! zfdW*tOukQrzf8V~fW=ty%L0F$pUI_5v@wCh=zkfAMWf@(4F7QdzYJ7U6iN>jri?em z){zxj`imaGfD+cMB^FSIQ6|QfOz=T+m+I9i_8H@hYM0lYTp8`G{nvm0L+@WXz+ew) zk+tLKZ)WOF(tnzBHuoz9qugLm zwN2aCtR{VT2;3+-_`g*DPhOsjnMLy@1pH6dCEzW%15>xd{3}#UN=8+5K?uUi`T-|@ zQ|;dj@YQJxPf4xN@=v1u*Wpe7CV*6$uVg#BCH}=ZXKZBe`g`amFH8Sm-nXW3q(lA1 zEqTr~XC{TE=@kYoDaR%SrT=8sXV`kiuPM*hOy~bkdHxVNQpcF`+ms7cpbXd3@$F_A z7##lPpoo8CF=b2&hf**=$fg``5&B;x{tw!)@&66h&+o3E4Hm5>rG~kO`zj~i{S#K7 zEDaPd8p}m>KLr>VW8Vl*k0me5Wvar%cp%TcD}-V1Yra*{>E)QBv?yf%s}T9*Z{C7_ z(47v{tpv zN!fHHf6mT9Yqr}>ToB~s4UQJn04i)qrMEG#R$5MBL|~a5Vqcu?Ys}|4pjkju-;BO7 zeZ$NpZdu4O1^h>8En@1G^DLz7;MjT7z+u9E3ElBP+3J@Ct_Q}j7%j!x=^qF7uxc$G z@4kW3(RFow0|A4@ri<$80tNxZ5mhiBsrdd!E!Bf7n?}_atZB7yqjf`q5W}X?MfG$+ ztI~&m!A<|aaD%l?8GD!WEMnUItGX-xA9HU37FGAX4HE)NO9;{iAR?u7hl+}X(jXxy zodQF*qyj34NO!j&0z*nDNT;;a&^-*zcg_qTkB`6q@Bd!k`(4*}?d|r=KKmSI)?Rz9 zd);f#8J=$%-r1%6xVQh9g;(;xmHx8I&;@H=WDLrHx0jR zJAu55>fB0%O;Il%V*s+o51Y~O%e6ahwC3Lw6ROX@(7uI{_}c&ad$b$KLet?b;Vps5 zG#p_^-QfWUoi`EZM+Mh5QsfV-xDK%~C@ELCtg6K^Fwf!N;B2q1#>702|8R@)i!Ot= zCq6bW<&5r_wE|$6@RK?7YsK*}k;;unn>9yq{dv+tnD<(0bQr$gi71ge4=#uL zG!IH(Ie^RJWPT*K?KS`Q&bzS6)X-4{5cIK#&Cretmp`q43_Ao3C^J9V!xSAUzT%0s zLkT*HFO{Ws*$)tFb#=_h8cK1~x{nzKooi*|FR32Fu)#Z$vZ$g}>exfij~GXiXXK>o z8}OY0E~8E~Z88=Ye5rNzMk30-Gi4oi-CYW5G_VK(#Tj1@N6#;Qar}Z-`JFzC z0@IRj18jwm`J%wlPU~G{r)}9d`x!vRcT*F_hED5uYPEyLh8B4=sO#a?H#|YTp8j;k z@LcP$y8CAjzbV#iHElTe`tFlFYsn;_f_i@QVWKIY?V@k)aC=UkdupXD}M!1{ir=4^Irxw(Bx@dSYQyQ97(} zan^`qfwrCVO-(g6CaqaLx7T^>2b9;Ij=l6G!&ad*diqx-9#<594cPQkX8-TA(jE}& z=E?3wf%+g?<(EwPo(1`XAK=)ykbCf87oZn()MHm4mkZwg9{n?VIJ&6f7rUrewm!g4 zTaz{uDR+Iiey!G86SiSB=TG0Fs7xLW8kHmKl3KkrD{&@_L`WLYU23OGpm|A7I5H+D z24)Y9{+-_SvVm_w@txQ3o~Jx}dApCv)?D(aSr~YkE&FZY>P9~Y_X9L4Gy)H7aZKH( zb$=TawjOzYGW{L)6{>bjrjZCoK*$d(xpo^CsJkqGRC%IdV^dyR;X;~oJbZ4>m1^-b zn56hhTa@2)b-g_aumvf%bQxZHo(JrU1TH2kel}-&?TOcHvRR;@dY(oT#y6FGhQT2L zQ+{7E0XX;{{pzcbnSqzNvd4{9)SKdS>QT>n%t_056HJj!T~-Ou^`@8w#sf?QfyZ$X z!7jio=&C2IX7>pGC9^OybL?wgnjZ$7KW^&Hw!gWZ^~o{M)*K zwgqWj{Nnd7Xf@ukBXUHHt8gjb;NZ1bPVWjD)cUAc<_C4Y#Cc@QMU1cU1D5<)$bb)} zwAY>&{FKGYp@n#34?+~A7hQt(+Of;3PUq(;o+%HG>i4$Q3cJg4?}hd{9#Zf~FG|Ur zX6go{N{*llnmOR-FO;dFofJyGG}GPM_?me1P>UaBT#F4*5V^n`hNx{Uk!lVCMgX~x zaavVZqo4Aippl)q`~~!(C>>`_Jr9bz8uX#ZI{wYEg@df=!Ov?#Q}Iw1Lo~F@C?IlT zReYlR_G&@R|E=)g2e2(b%LfZ6-<$q&eR~+E%=sSx@krVLKtxJfXv2(xsnFns_F#`@ zy;1?xYG@M>T3}$NZwX+u0cXjE6g21~Ukl~gG5Retl^G z-vx{}AZ}2~$4CX~03}<2$p+ROIQ7H;HUXt8nevBp2Pyuc_CDVte}Pk}hoyU5o)h2s z{k~-{Eni3f1FHa_@Cd7%9bhLjZ6`mf`4e)q<>Y4VuVy0k2+B^?V_R~92ve)StZS&| zOK>Eb2J+0Ds{I6j?2*9mNX;*sV!?Q-6t1xzqcC8pK^~nbbWjJ07HfMk#!-`RidCq> zfV@PU0*3$_C=s^$z2>)3X5=GZ>PUpM{#ww_5+Nrv(2OAcB_ZWKU0qL4Y^3U)f5@p& zQ;mgr5#KDIyz8taB_%Qt!j_}F_f*#>C*tnELC81998xpDoSncBEa;nc4d(u888ols zm-we(0r>ryu$yK!RHvDAkj5afD(D_JbDCz1=ula-g4Oz%)QmBv^{{w3XL4F-709P@(j4y} z&v^Azw)IpC)sGr2v8Xqg%2L(5b5GUbl?%~;-$bDml;1>Yy5)b>bkj}+tJ{ADbUY{n zc@li;|1J2W0@l4J`vtTMXg~n!#OdG$s5=A|4FMsjqUDc@;SS8y(Fk|_0|GDv6-UR! z!N2+j@M7_=Q1Yit*B$fn#0RV2^Hd4At*R z07N;4_HVq1nlZ~iW=tH-10cPQtqqngXm8H)&JdK{1?LkrE5P#L_xL+Pmn^{6I#ny0 z9dlUwa zSm?W_BwF+B#e_#qzA0^C?Kj=+?coc~j#gUWeg#s@CVk86uQ84yLffoQV0^AE=*<*AuIRGS!dSB?e zFC>WZ{yi&yw>YTv32-g|+d@@?t$hqzF!CE1c>fmnpG5ubpS17=^6apv7f_gmrUYs0 z^Qi?5So?J;z~J-PpEktwl>vr{5M%bAhH1qYz=waaHo4xf-S9e!(Y2`a!Jv)76cdms z=QYcw`{%^LNR*NVd9ocG0W{^{h(cbgQ|JKAY6T#=0KR1&?!7KjD?toHwzp+8M>{T4fVK6-xrpJLZ179tkX z0>q9nLwTacqqRtpy5UJ%AdL=ScTjcVHXiRk6dObiN`U&f(K}$7XkDSOaso+IdL#ly^J#XfBIE8A)0u$Z2>5I zR5>*R=>1>;W&j`<6>VitnHIpzxQ;_pb-lHe$WK%~6zM&GyVHlBZ`UKe2q2e`ycP;K z-fcAnT=DVEs(`ow8E@jliqC09wI4O*yq8Q#3wggfo`j{g=C!#%So$)%#Y+2Rm{wFi zIP8+?t?0jmu>ZGjdET_$kU#Id+1RnA3A=@ccK#233j@aU=NurfVMN+PRy3e^Un{!{ zf}P_v@ZUi@U^VbyJbU;Hx_nS97~oTn8353yP9o<70P9EE{+B7Rpu%k> zq3JjZDa(Oq5(p`49nB^BF4&@)np_}~^UBV#(*Aek$qM``=+Q9Xf(2C6hYLFiP=QeO z|MD%b{Puerw{1-Q_hphE|5onMU5|})AlMcF_Rcc=>iBbLD%~iY<^Mix9WdcA5h4zSy_ie zLAwLex=1LA8t8buAgl{{$Cd8>g6{Ppd0TAIBpEBGXVL*X;mYRjY$pK6k;G+qI??v(vuYXF{K(9HvKzy@j{#W z#DE-80h3VPv%D+6?G6xM0bYa{iSqH%Ouf~-C9-Q60Pv~A0OXa9NL}E6AjZ{8Iz_m! z9W!5q)u__SU_r()-WRAR^{T)4G9<@ei&d{Lu)nx)5~@;{M_%uHp}_Rcg2wrBmAYK= z0@LzeWYH}A$5tMQT1U9S4wS5qK#Ncw`Ksxif&}0)Pc2p;1cKyL=f3w@*YMO?M`9Sj zh>PpTN9U4n@o=Cca~J^g(%>PW#kyh&fB{sjjb?S5=y$w?=~P_@G67)f0)<@wX7?-* zE=Bj@?hUX@9Dx@k*#*4wu|B{gBU492*peWj%M9@7Z-CSVkUB!;jF5wH3lflTj=aAZ zqg+L|j|T1EB+Oq2;=&^5F_3Y_kwL*v>?+2nQt>BXJNmrHTTY`{w~90S+pj}%0=2UA`ZD{AuS3lF zp8ONhWZ##nu`ZiVd2|6ubuj8r+UC^eIig?;m@{f`Fux5-{U~JwK*b0*{ez%wKH`&<5dam*-CBU45{Q^i5PKFlhs6I4 zu>0s_)%mLQ7vq+zJng%_py8vH6`h=m7xagQ1Mgc|Pl^?JKHroal2e;|4vm=g6#D{T z$gX1CDwP0&oTCBC;9XBk9*G`7f`HZw2$y_SNVOD=jFgd;M<7ob=t0%r{poyUK-BaH z=am&mkb|l~l-1sziLAp`0I42*jA+iQ-$KzBr*br*Gm#R<^u@=1P~q*-&=Nc9Q78s% zYxDof)vVu)4WZ)%@e3F#0jVR9+66c-5Wk>8CC+>#8-pTYP@xhsZv0TC(oBio?gU0{gX z`%BkE%_2rleFBvf4{Enw>~LMoO8SN&{KTsOA1h*!`PP zA9BpO&RcYy*H(Ve?hzG8@4kLBqv4rLe%xrWzA45~X9ZE?-4oTn@jBx8JaXjkL0AB6 z=S@Tx(H#%^2#_(=V1^V7k&euaCk8SWJ#A+2l~6DejFFDgM*tgxG&0B-=_o%9kY3jd z5>SGiqRD(l_L_#(9_wW=cZEzw4{4z9tC1Dmm#HU}g{rLynYaWw$U01539#!hFk^bG_@KS{`_Ml~ zF&O{?eTFUGY?4%Gp$5|j_Gl4Lh;JV=G$-WDIv(FYN%j-|Mk@djMf}ydrX< zrIe8qp&}_o`v*bxzXkFr!y)LGd7ZlmRe4XFo>R&{M*xsxt-qlpGL}*vz0BZLno{ejFs$eo@faN@b-5>GY;09yngCu5v6{z0FcV*}Ex^nNi&PsVp}if9VgG|ppQt~`mVwMLkmvYq^G{8k?Y|r@z+iql zBW;@_)hGCS`vAz4$qKL>0i2+O|Hj+kh)@cP0h>K!Ph_DHf z`~ktKVgXz@c zdBD*Pkyrp{LaH5-aLWGA1B*=A%m`I`1eBB$5>U1L7qFLLZ0LCB02pH(+k3#|7tPUR zqP#EA{?h>ilmd#A@hN=F+aw3ZEEmx{j3_{>1ArZn#$o%9G>$c}kwCZI<369k@ZF<* zN{;$-L8W|5qYEJm538sttTWHuYz7z3nMkqMD8WHuW#L&w`) zPG_0^U%Ld{I$*o-iA$KrM-w;uE1!U36VUrF-j=rho3GWd{10~M-%j}7yJi3?`FQ?j zVuR-sxtkLaN+5Ms0w7S( z^nkM0-#LPv0(sk5*ypYu!5=Ugb)@(IW!uc(mAU)xO#LzD`NKo_0{k-Q#!nfr?tesE z|9uM$84}+xx0<570nq-#s+=MgjuKNZv=#rpJOIdfI_8q4!VNx7zvs&KgX#A-#TQ#q zxp!0+9%3%p%k<3Lcl|G^`d`rFJUuY>X&*35ve$q=?5m7oWpDhcT1|}$Y;Zai_~O2f?jti(0KztMQ-xDCpPIc!9dHLH`NIEy z6Fnf=biy44>3W0iw>O@|R3U&Ty^cyrkKoL|seO>``S-x%;MqU-{O_y(BtHN9&YS%U zi{t$YSk&D0zP$BR+^6b-uUjp^Y)0xyMd<u?6DU-y3jjD@Pd1rjMy0oVJjent1e2 zz|MpqBNRvl^W|WGhV;?My?wgzXpO$wiC_@?d-u&)e>?KdUdImlZ;=94qc4eByFY<> z9Tm480g%lC5`%ZM!7Smw7Q%lRc}@HVxK-gepK{Y*P$xv!8;BJEIuBq{{dtU#x26#X zBedf^I0PR5*o70wV}z;=1qrykMu3H4>5V^CPZ6NUTl*+!sWWxGQ*F{uPYQ4h`3g*e z70AM&m?2<)PA~z_NjywiJ`HTcz^tD10#@wtZU#X5-i#-k6p$MnfY=jT7ToNB+}?h)s_?HVU6MGC(At z2QBV#?)K~Mi=LVv$MG6r!flT1{w1P}QfxW0ZIujb+_&G{B~C_1ry`C=uURF*3N5G= zvBTJSc1Gl4dkFeIWjh{+ON1)A(m9Oj@%;;^%)}>Y7L1EZ;9`t&k-rOd+cm9X}S4cmPS>#S5>VW zT5$~x%Mw|Mm2~UbMXUqf*jKH^FZemAORUDpZ(~BN z%IVK)g@)zt6H5lL5U3K4h34gpCjs$E4-?@@N)g~yiEXp4X)YflSGT%yJ!1HMulE;k zi6PG_;_nIhUDtDP3?CDW6JlHApt{k}-jY`0F^^veeshDSG_{DXjuyY3_Vx9*w%1m4 zf%;96{^c{}D(+dQ^GTe<`;N~h?N-E?e5!`~@f;8CMOZl;#_BEwSB2(3x;Jy+?!|ey zUVUg!v^6060n+^=_F%k={n@rdCY8<%b({O#;n1Ml?C8kaU`Xwm&CV@ji*if1(tW>> z7Rxvr3isX7L&8b>>b01PfytG=!&{ZpaX1HQ^efL`U(*ySWhW42QQnqS*V;Us#pW8q zW>uf74s!P@zisAVulll?#FTlxo3mSRHz@UsW*5Qa<+P$y+TovJ9~vEmrxw;jmPP1Y~P8~*@OWv zkf(v3J@6(o>#p|74N-}0%=?d4+f-ZqGuxOw?+wT0rO#)!N49J zW1k>kGUtE`l;d>v*Haoyc3f;D&E4JB+bnln!Z3i(whaS*3$P1TvcCvoYYhMMg##QwHmaOwQIKvelI z%>HmMw+&`HSSh9>jRA2Lg-JmUU1&3V;M-c7eIum;M%h9)XO*tU6x;6G3OqZ!up92Y zyu-!VN7SgbcVRa;E-0Q9Etmdg19)cljYH?cykBzj@T&4fr}rN=cP?4ZS{AGhuw9fI zzOp__+)$pssMlM`$hBJ1Um^^N8+!AN_Unhbm6^Ur#>o-YqmryfLCmJ<*pgW$TjKcG zdPWSsM0G2k{*#GR{=YC+o6B~YxX27t!>+qo>+>7n)1QC*m@w#@I zRZU%avFrbZq|F=7yKXH6D^BqU$Z2YWY-noU?7#f0&SmP7t6N(7TR(=;@hS^0U3LKr zk#BY1dXmYkDHts$Jvnn3HjJwKN$o8y3ECGiZHWvgp7_k5dlNq6cM$vX^?e|n)Qr7F z)vX?*xp1iX6N3@lfByl!P3 zB(usm2D~(1KRn2Lt!VRk-Jx#E88(hE5YhX#DQe%?vn>q$qJya&7!YT{5e$crV1&nTTrdJ{BxyoK z7m+q0;(?fPAnKo(-Ad|p=7wC+4N050;t?{lbr$S9J7#aYN7#?(S{3pb?N~LWuZjyE zMc)es8fQ`3ICPi0jfT~+@{NXQaUH7{^I=`97pqlmU9cA5K(j(ltAp+=0yHjn8da-D zct6VT6~V@(FmiYHOqRX(OZj7U5bLIuM51{G^V@J}eQaL;g~_d+Se-UaGbq@*z2`LY zcqGKpOj$+SX%m_PrB{sY+soC`sTAZMckC#4SX8gryrJXVn^%F?A4_i5M)KK^V6fbO zQN4cihQ4!eK?UA+@!ro*VTIJZ`QWFbMfJhS8_P=1oO>*P(rxTI_LcuwREKTeuys}`tB@IvEw*UWD={n{ zEFW4_U*5do;H(0zkQs^Pv205$G3*&EUszN}Y~FBjR;jFz8I3KrY#T2zgb$YQEvg@G z5~Dd^^QgG0D})htp(?J;amJ(KrS9&TunVc@@1q~s>_+X})zEfQpgXSrr^ihN_r>xY zXC%Nni);~Vew1PQ%g4!3oFO0(K;e<)90d_S7| z8KLmDw7wfQ@(inf^=6ic_~w?YTh*7(Fe{RZrj+`etMf04wHI^V#{SB1w%Sm1zVMpE zbVs2{pT~+7?o=s3QSxhtsWZb*u~sAm@1|50e9m+)hfin5vZ5JyiU_AmPnBJLSA|`D z9MtaJS1}5UN8TmuA@L>J(VS4bPR$C6NX3AX;LgnqqO*6q&$T;qTuYqFX~x{kXk)nY zsIh*gLLoXHYB$yFH17~549`7Vh48u@FpynSg~3) zm)AHQIA->SOV}4DG*yY)G?~91*woxcjFb&!j+8wi9T60$9l?8$x!(K!<#-OO(QXdu z)Rwk_TD#^4>U~9y%O5Ri#c~i{(HBR`c$-Q_qQffTxt!k-UbWSuWwlJBWx?^l8x7WO z!+K{>WhdZPVJ9FBg=g|NBfL5fpmwwg2fgH~2fYGG>lK_a_*L+Fx)qvxcHNt3bUkZ| zAsYL}(&m~LqXTk$wn~j{uH8#<+cvSf#lew&=PTejlVXCCR0m0O(`6-P#-kl+AB5nF zZL!ohyF_@}9M@I&>fKB=+4g&fCO5l<+b1(dER3B^qT*alEkvD7L^CXVS`6fK4#P_L zG@17{4Z5b+ATS7NzoMoXe6b}APqN#Mxv+BhqeZKe2|TT`;-f{qlS!4g;^@-e;qw6311i6WBEmUtAW9oDJs7-1F^-$plq4r9+i}|UiX2)D4oo~krA`V!I7mKCB!1d z@UGDjvr8o(l6HiIE9l_k4Y6q&!`16b!_n(1?tbZ`(!(wMmhgkw`E)tHZs8u+^vMpc z(EHOFkx@@$GjEj2E3-ec9ByH??6GmJQ07a8_6&wjLhbm%pvnnWO3jQk!I?BOBW18m zx2)J9cuv#jRi$#F$CYsDh5I)ldm4FWG-h_;1rw^S^n2y6X{(kHHC4_1tIgU!x0ZqP zPCtLRU(#>PL+Bs)0B1)+wkg0~v1y%e{Rpnvr7eetJJ$*^?i4qzt=}A5kJymC2IHFj z+9CwJ9UrK2NRE3}v!a?Pe2_=evL}glB-8M+dsdJ4%4iv^%q1(9dMU?=J*C9Snea!+ z5~RLlE^H!0PBpMXdHvb}uq+&)@VV}Z9KJL<%L>+jY3HZ>=#KLxif>9pW4xg76mGOO zpT&K5T9GG#jSlnm8Q(PXnrgSTjb7Y$o*QQQu1PO;>7Y;J!YnBy)cO2(iSei7KAKYHirj%0}*;!PY%8_STjxCekq|>T%oyi#tp`|vZ z*;(sFFwm9Hhbr%7An1j7`|oLPBYPG?DP}YBfS!S~Q1^Lf;*jCZfCZR)l3rw^?t3G9_Qc~5b_>5qed^xX z3~3?}`yU0Q^ZY$YuLf09%mH z8Sns*9imKHI8Kk}ruQ6e6VV*$Fpl!=eTI=fpSde@7cxV>C@8qzu1MVRcanmihtmNa zoIuC^UEoCkFoIQy{z*Pwk90Qy@DDFDB)w0;wa&*kogN+x{PP4x*r$t**WT_9aq{2k zZa%pwF+558JUnRxyt&`p752TaR}SMQ-^z#xs4A{?AtWX8$l%v`niMV5-zSbtVZgN& zcxy`7vzj$k%OXo!J4V}lG&6Tg*a_M-De_CS$<5Zajl`zJrw@xwq_@QFrkkIe$jO~I zq%Ce?r0=<0{8(xK$?K$%WYxgA%}JrmC^mu2rB=u8;qSXS?6aXtcGC{D${{O4b4x}0 zDp|G<9+^MBey3-3A$=M66~dB@2ajSN-?(Ep(ci&kqZ3wQA_kxN@IKCI!!)$t$@a%k z$WG2C_{y)0W`AI^n2z=q;+0$0d>Y;U^$%eW9}TkyNH+%V3RLx67g1D-B9nbI{Kd|8 z!QNq@qhR!xv2@jB{+`MQAt_P#oG7PdPqJm8$=h0%%qq;@kYHtd5w|QY+_ZE;mgvMC zA>I9%{^ga8B>k$dT_J8f0_VhfVur_N)NEj5B^oJKfhM8Ag~R)IN6VadDtk16>*Rpz z%7pW36_?j~Tr-(`6qR0)$vw)6?cK=1@v$W78Pdn^R-D;0Rd(q7iM?rF#-+0mC0Kz! z)Wy}YC?VQck~d#dx!unoV!M9bvPb@rvq{gVDAAtj>=s3M@`ikijf7aSX6duoGUkge zCY^nBl6`RA;Y)<;=IVG%UHrWA*ouU0p%r1I8xY*Sw4YX$OBqhr`_iWAD&@^t7Hzwu zc5jIA5W(F{>PsuqG(^M>72%2xV>Rqm)-s%&7SbTjFXGb#xkwv3T>1hd2k~N^hr`x* zGV4D!rYpn5FpS*wyAQ13q*+t30n2f1s#+ZhOK&cf9~_Q#RP77U4Rsd1;%oK%S>@PS z!kQB{;VBCn@1PSqOdCC@f=l$t->2*$M68QcAD(y1J){>N!-VJ2^)#* z7(lG;glq>N+7JmxdL7PaA6Ra+9*knI((hMK+}WcaZa&<>oX~lceweQ9reif~ipYY* z86*4-zI^&ru3Q#ch;zx|ji=sOG6sg_!w{Fq8+%*6(J=ok-N4$ZO+9tc2tZ-MfO`w=mEDPPF=mn~S=1ul0g+UDNw4DXw_ghj?Bi_PKF~mk&2_pa)gmK~+Qz zTg%>CBZ-HM{)b#w=4?d^1)sr5EB1i~Vb)|_wko6jI=?-n+!M)dpxd4;$}xWm1+Az7E*T?hOSzvUmp zw+DY&j0aw7qna+CXvDM;W-?r4s?>DikM$cjr`1`ArS}!K0?r=3QGy~eehxU z5%rGr=G~uE&kdi2Ah0;P5`P4?f3GNQvOEIrAwcL_2Zf z@?@XSagyk=nU^V0 zsq(1la$`XVMn0K~a#4uUl@7VYoZF6`G>xKxd5_JMGX}hXw;Lgu5jYWXIb9St4p-IQ za^YU85zCzio+&-Dp$J(+8zY%Clx+S|Gr zvu??JQ0{(cekX?&vVY&b{}$$wUkPTpP?oa+$-2v)eZxG^CoDh`I}9_7pb@9=S$b$q}}iM&e9oYnZ|I3w~XA$ zdH+jZNMqq_G8tmpn)EGmuHAYFEEoPGFV<9(ImzMdN6ir7-Jll9S#x}98q|is8o;R)kYV~{N zW|q&^g#n6YH<3;vduS6#JAGDcMe5^4;KViHbEpK<0%vcw zAg_Nx%KZo&ELYG1en#Mqs5@Ny{5^Y#EQ+iwuuIW4R&Om@PrA%feT&Q#Mm9*MlBO%@ z)6O!;Hd{e{RXSBpUMA}re|;s(xa_RaeHfdVuH#`76^pb^DWWMu^&Is6D4XAz?MwXi zyxIIqrVJFAY5i|=HQmm+=P#dlRL5qa9(z|N&d8XG4c_gvl(Anu^5)`pc8`*&wCpGA zA({+G+Q1KjK1UnU_etSSHXpiIez5XHX9(RQXnk1m(b39a zuJxh!Ek#QMy6BH(pE}2~yQALY7^W;`bMv#p4W??;GlW`b;4l+%3Ma(1(Sd@5|ci?Hjq z5MrIm?(UvhsB?vki|$&DW_0`v*j=a--sNl)b(#Jd;0V!+b&W7)ICa?jGr$g_7vrjC z%<$)7En{d>xNj}v`w0Wt;&2lYKAPF-7?WE)oo~LRua9uc6n9f+l|G#dhAYWb6bONz zoSUB!o;q2>k|D%MAa}ol#S!@Yt@lRxN!s65apl~Wk?FXL9SI!Wm1*wid{HFiuCMVn zJUlst(b)c=q3Ajrn~Y3z=hAC5TN~EyDc9b{ZP7;o)iEp^t_}Vt9CZ4siHh zLnAdjJOeyT(a^{Y56?^~&ir&M&gA~9;^bnDP7P(kx(NgM;S;kgQ<9DbuOe{-i)#C;>h22>Z>K`*Y1WgcjDht_RpE|VKG!2=TJ`o}A1+~fz zArIFVzCvR7En;FK4CxK1{J~lKeHas?W=O9>w8cq=N}n@~kZIw`J?b^l|%rPLT_@Uaoe$qXACTWS1by{(UJ zbjQSEr=^W-K(R*Cu#)J>Y5NfoH=!&fWVc2X|EJ?ej6 z*)88I%?*A!4IQF_YFb!#K>N)1&Z$)>HAJC7OBzQF#RLy7Gwim=NbXOGh-wA=R63FfiM&}G4uI*x1Sc>MBTMMs#cpAd@d9`xy zv4!{bk}_7NpQAkcn;epSZ8E*PpFePjou5E7$$cHz^Po*NKiH_+HQKjaLECkm4<+4f zjJaTY&)|s_E9`K?bko@O#lFM(%f?%c>S$8Q?uSG7ho0$g7$eLLzJ^+FUF0cIYs_2Q ztKIIB$8~ap!}=WPp>c8R+r`@b%ci26mET7Ae(o}vHhx`hUT}@7u>9mGyjPhI<+#!4 zN2{YULJ^MenR3+;sLTbrmhqZ*E1g_)U3Jt(DS8pE@U7~d%KS=>?B(6LRZqC<{+2-GTz{BCB#J+?4tOLaNXakKB&yE;#gST-C6bAcYW%v(?5!UPhjru)~;4s zL6@l;ae@&(6RyAb+t;_M!7dJ(<=v$T%w>cRVz+kJb=1{mY`=2FO~;{XxjcjjI!e)s z@L6{q6&AIsETe(b|cQLo^ z32XInrV3uXD_c(@8rR2x7kKaW-*;cQDlTMS0~z^McV`4a+K6sem4B|@z9oKXz&{iq z@q||ztVe}m8ZqW;pSJh>;%H~1>PN1p(v{!t-oJORl&S1v=E|4~{9OF}JFJfP`b)QF zL@m%a#&6tBIU727`7zpZ?f!6(0#^PKdtzMd%9D^ye!LE?aa`RTe@WJQRV=fQFTyyM zN|_rQD)g@F4!-GvI*)x1RTjN(f?@hYx%_thR7qVMmI$BpSvNNT5JDaJKE7wp!!y{kz3_I#A~lfx;2oda zi0tIZclS?N{0$4U%$Fy><{BYjb)YI>mi;AHOJ}>4z z3rWk1{+YJAu=aq%qVA3TD$~2Ki8vRkN=Bn;%=R8OJRr*5nhoUIzH~@tPXEGd0#;o* z{A?12U_$dAVK--aVLSorRUur7|GZT?yg%6j?U*GL-eQFAY0C-)7qYI{2rUoWZrZC^ z?wF|V4P9MIow!r}=DVx!C|~gAbSJ4;T*Y_cz{%8CHg#=|rlB{Vhn^MQs{fQR@;PDO zB*ZW%E-J8XCWn{3lz7d?Py&mfiCjzzota~Sero@V)!9i{-NfjmZ1wWu4S}AYWkp}V zS6CVsmrvxsxVu&QR^hzqz*ykpgiQxt(%wUpeR{M*v@M+K!$XYztIw^T!FNAF#l+zI z!y|F~`vSp!%Ls&vd+r2dBOW79W6(ql1ll$5bIc)!cDZ<>g0~I3F{^oX?uFaJcJY*h zoq)DhFyDvuS6~;kjK(q-CfzsD19z|!8F|tclG~UyzdGjxEsdqu3Q;YY=vj2I5fUql zv2lTx(OO!-q>){ALL)WwJU;E^Cnv6_h#&%PPBR2jZuBu9rF%wLHn4Z#bj5Q zAYbhKkYldRd}-&1O-xjq-I8cu>HZVr!_ro{=%~1qmgS-SZ;;g?6T_*6ucer7Hx@l% zOh!OuB)ENFXhnb+b{9CTE2S7Hy$IALcf+Jlq%dD2pz;9S%mbw?K-G}Nv|f4zv3L#C z-cIpo=cJgZq~f~Q$)5ZOX(7IQKs7aHXcg&fl?AI4mA_3~kxddke53zKfMNepS2H-ff?XiLxAaciHGZOyyz0X`(iH6{3HO0a5(Bs1oV z!MQ%UXID7Bd&iwqmp{u~yN{FG3cO=xjQmh;^+Z_+3A0c}Xw2ky64$~|@t^k-7Op-D zHwufn-6VJYIrGQD+i43|l^~C=3*CRusPUY+pz!U??zu}HtU~9)W5}B2-aQX6_<6r* zAz1_Rj!nk4*w;7w9OAHd2?%zQIOtTgG|w+^@Q(PGDw$uA&>vD%v*#GQRJY zSBN$4+RzaVc60`ZL%55rdHIjT3xU9lw1OMpMo&T-;+%?dr6-)YEnqk$>#sr@7LA5l zLzblH-ME*eC*0gN=!<@sd@*d9Uu`YCKp`f0L0olr;(N*jx~(0wZKSQ|z=L+5dB}AK zQ#Y$-e6Fz3TY7uTuDGs;C}7*JRFp{5G4J!lQ)hO|2930%=kNhvX~#bL zng;4S_?O6+?k}O$=9Hh_q3(o(27Ep5=%IGw{EWvB<{5N(kv&6c3O;EK^?1P%_YpGP zRJ!faId?l3MMTx$ubq-F`+0(SiX5+dO)N|-F~-v5c7J(aKlAlW9n)JUr@U%1&+WQ^ zNiRkIdk=I2fmbZAuNqn@)ns@nZqs;XU?^gx;y$bZp5>!#68K z6Thr36|C*-Ly8D?&X}6n*Lh>Mp0gNTSiYfme5>)X4bQd`M_=NU(gj|4Uc|UX2|iE) z_^Qa!!}Ur(T3~};h&{dk?iu*xx0SoV7fFt`esO{Oz))XRsIC$h=UmZq5SDD^Ph#ZTn)h-Ca~pO85VaieZfE}(inrF#QM zUxX5`c8T(K@5_Q~3xpWh;^968?5};E-ok!c{PYH)z8z)rS2a3b@2ePA4Ge{MU*FNo zCei;$*<7c_+#p#%4(w%W!v5&fTBwF=nv$=AZ2Ai*gf{gWwHNwd(K;9Cb3hjAgDd`g zjT{90^yTsS-xsk$%xG~_Zt_x~6{7CHwoqDAb2%+mt0-Ur47TJ<%4T>dx0YxP)= zP4@d*4VLWpB${_-3ovmLy&4~itXB0L@*WQEFOL)qPBEivi|O6!wo59Gj2ehS+X`N6 z#LObbCnb(b-o9ij3p~FvcyU6~mxmlf@l5Jj{56&H5kzmTl&!AEUEx&L&8$}P%-|!y z@;e7;)0xzB5B1f~q;he}T1Ac_6$~pC574RBG1~@JtFmfPr@sEwku0yPSB6Zo%hepl$~+gaKU2?G#Fu+6W7mTE10zW0%O4SU##|(T@)3_};9ve? z$x!9FkG-QwG=L_4!2=Gcia3pjVKxS!PtDb8I1Q5d{~u^n(7mVML{2n{e{u)>%p2kL zk8hmiLsMsE{OoZ&`+y2)uoRrHlUuRC5`W#o0P*f8)D!b3xCD9GPx9cl^e43kK`mGw zCra{_Rkf=gMi1O1r5=J+-G3SPRZ6}ZsyX_Dbt;dX7CgMt?W3H=+_MhN4@wSagdcNQ z9J2a&MBpE(kx?V>Vt*>_V3B?l#H=gWe4s#9=yL)1LOy)arQ zhD$K0MxcEHzDi7^#yp*L?jdQ!UCYgtQg-<%*lw+R@g7dK7}^2)%8}DPf@AJX9!#=l z;p`RI#dF5E8D{F%uY5XQ$Sg>^zS2sa7!<)w{T^{XX!$*r`k3mxu-E>^VCm9E#XE^4B$ez2@~CREY(X#ECMv(_!Oy7i*^PISx8VWl1OOcBIo zK_M=(Qn`)YwUw=Sg`ZWlzX$l?X^{=xKtJk3)-!9QN=*R@H~H>RJm-QKk4(`K3A^X# z+(WGD-f=GO*)8Ml z9i@9CBQP$OJAt3vxzxw1+KL=gEq;AbwbWG&TBaOQD*00WQg%>aI-bhyiTr-w^6=Nb z5sRhh#|W;-=9r%&9ibPv?`A!x`ANsUSluY|Q;HA+5?tNt=EAVv_=SGGT6gYG-9O|ToK1kAcN63h zv3Qm4KGgkY!J?OR=I4A!jHN+jK!DEmhkCILCt;853ChQdtA@;Z?ZyKaj&`R z3}qWP%?WHq1?(gZmxfygH640$&mFBRKoU0SFKg@f$=pi)xI%>wU=t)d2ZlOTJvaQ6 zw8BUQ=XUQUxY3tnJrO-H6so$K`sHkL>QdY~HzNDRBlAA>!Gnv^b`Pe=*jgUDmcRND zY8NrPl&`1;w}}dN`ry5~`(4`n)@^Ax>A-8)oh!z#HQvDnGw61P40CicR9XkZ$K``s zR6b;{YpUjcPHxMHZCElMni|_1syGx6xjKnA9LuK@&TeM=B{;bCqJ6ZuL>U!$*iu=UVZCZ%ey=W4x|Cs;JByz$m&QXD5F{TYkx@>B<~>e(u%7 z-Hbat@?ph_sw#?F?^d_PRl@K4^#l`cX6r(&6`hwfJ5{@ry|*nxnehB_({_b)Oa->8 z!kOdDEPqEw&p2P) zk#6P_Ws2X-3N6){|HI_eZC2_Tdz!Ph{-6MSkAz zj9vVi8;q`Pukv;9(p)5=%UapxsHKJ*TDemL7axRJ_p?aKD=U1Qg)C;2tr>(aFNHnF zsX2^^QuO9mEmN0MUQH`2LpaXf>**wohr}t1`F$~5b6lX=Pj1$fc3+Zk-kvs;=eMUz z%cC_)lWW1-RwBtGjWU~LGtww?P9rd8x;;ebY1lO&uRO~%1^c>tCNN2(*ZBF?f~OGm z^6MOFJMGfQkxAGvE!-T=pVcDrQlE22eNZ46R{E~#=lzr**WK=}n?>(^O?GKwwaf^%}sL$8imL{%dnB%ARpIpk-!|1t14!8 zIpLOtmrH@|EAFR4e0vFfS`j?q3l6>+Ygt#c;#OuEKdNTO`ih9Nxpv@I$@&Ib_z2w5 zDz9v8vM0q3tj{bBOmikg&fV+AIGA-Z;OdA2I*(SZ2^CFwib z5PkA!2mG~*(`{)v_nWsV={TNZiCkR&wNLsrPq^zaGNr#XZ?utXrnJrOEspIpLMHJN zbeGOJskDg&#@}#(kU-FBbEx;3td~1}^W~nsmZe)u31V#USC^uu>ONlbh7uwKh?jgh zC?#$2UmXy@{VG1)SK76?f>;=CV=ls^=F5EoOWp~srqE{|)`li%v< zD)ZNlleyMu5$7`E`tE&8F}=+Jb01$`*PFJYofj-qA4`I(zRFU5^}d|L9Ok^kboHl} z=FcMA$8DUEAMy9x!WL>g%NQJ1To@;6QW+<7fJe#)%8>7pP`NPCP4-|^dHd~9BtJ{# znL+0v65!<9iA{Iw^kSH!j(i=gsg69vcPnPoLWL z`L5xd>xkC!OowN02o>Gfq@bGT!fp)iUAxeI!`zBax3DVx`FP8XA2uu_RwTBo+A!zZ zwu$B4(8j!lx43z}?bID=&X5~lm0M*gBBvYtlU@~M_UrA!`xf`Z-@!;*$t%@+q_OI1 z{q5ed$V<#P%^?CPyIONhAdr3=b*Py*(=A;${K63`{(6Ke6RIPXFX7P)?T}7`3#Xq*GIUAO(el~XbmGS&XA5Fuaq^Fx^j)w2* zk}ShFZ=Pc#GX1GnxN{$!o&sM;AedpGg8(W-*9|4{dO!a}JeysV-oP@O)RLZmho(#{ z&t`OGjOS{=*Y~V~Us_dryU~em4#e$r*N>EYZ5+^1dfPvz8F(i0+Pk50{rp!l`gjqh ztroRfIE;ha*dR9dteE%f24b4mqE;iHY;ICwZZMyWHwxQjium8O>rlBa2PrsYbocReFBlW5F zlb&qF#X80BaaK1eI2{K^axvN(twU|?qU?2Oerw`J`bw*=o8kKfV;ltir~gWfm2jzxUEd#CLs zerlMmEBONMZY0fj2u#5YjxR;{%4~J5+3C?r-W2x;q)l==Q6BYYkA=e5E1{qt8F~)W zHFDk8`{~C=Dv;-MLuS+0Y_|WvHD8uz@7Ku6UO%Gr9i7)r$rJ3HD$HlGz@scNO@S-@ zMCu)}oJQ??_WL^Zn7d5Im*5l(iXV&sKKfYia4$auT^9 z+gE3kT7Qv1njmj$)H`wr%fWYTb;xNRaRb3=VA!(Y!N8QBj8&e#@N_G^gOG*lqdnWO z&P^MKtQS;}y@*xzVAM>OkMiZ8dM7RMI@p(A{xRv`R&v5Uwm1=hDl1@6Up4C-E`BB zqwZ<)4os^rt(A}Pxjnri5YGKQnejH(A2<;)36Gwur`_wjywdK3+J}|s7QACYY>UO| zQXlF7$&Z$;%zp0lX)3lv^TVBah!2>djzR$<;rgH;bz~-L^|Ueio}6sGZDvWSTk>HJ zKTF}*R;r)$>=U=e0{h1tO1ro_p^cgD>=2#vRgPmPA5)VbemHqn2NoKMaAt?wAL14^ z`8suHzq2vUj~*v~|6eVon>?*o`@_y)4x8uehMh9NUzEAoYdp}UV8>XNCg`z{a=WY) z>thEmYYt?cBaUiD9CtjtB!P#gtmZOmw}yLfsEHeYayB(F+-DHP?(&Ib^Gzxb)s$77zo#JL|v&mTPo zn(^_y)xZ9j-xR9_N8qFGI!W|@uveC5*R_uC$L_O1Pvu}sQY^EJQDCp&^W$7yy#8p6 zzGx#|a*dfu3o$bz_!woRSn!NV2i1=Kv{WOoO1u9Ob8@3N-W2udRWKsAWPk|d>m9XM4ouu8p@R($ zirz)w*L^-*?i>(VQ}CXZ6z8|e&T*HcPT%)l$z zJJoiK;5z5Lq?O*!k??rC#Ao)iUemDkan{Bc7xCNQqyF37Qv$9@AfCyRkr&wSaGlHT z{z*2ve!6&V)%?M7Z`*Kk4@+%5IPm`Z(xdV3ezMM|leOh{3r=A`x$ZAMnZ9JVY58k+ zaO|PINY>Bob(Ks$$pZAblQMc`-SnMe2HhN#?2(q{wI<_G|B6A|`p)xn-H3Rg&|En_ zv8UTx4{%23x%%elS{v#9FYpwQl?9(M&b#hrcb(+s^x4~nn{&9bi=T%xo^?wv-rL35 zdN1^&zW(HE4`NiZ9>tXo$9*5*vtDbifA!qilhYe+>w(X$;Va%T)GW{Yt>y>nmFn(^ zan5&sTr+kC1Ae`4=DNpsz5~IS*c#Wgk_=F`SEvtKD!NnxWM;*?q5*)wr@wD@|WOjK>D=@=hG(&~Ahvkqp*6CW~ zzI%xg#cbuO`|9TKg9mUu*-2MI&w={dMv`N<`u31RpD=Q9pRYU2{nS$k@YW#wrM=w; zZk{@rdwjKDIi^^0@2R_7`P-2mn<4&RiO+OYKbsf(60^TTn0@6QMD$LMcOk?ho92k` z*;Bpc+hOO?9L+GwH5k!7NvF8~1$^sUPhaa8)^Gv~39#G<3E2d**Ka-7K~^sOAzslv zeEGIY|6DfV6vmJ{v=q{RCFtMf-WZaK|31V9x5Ony#}-MCda%1SSoksW^Q9+FV8+Pc zyL4r6ij_Z|$m^BiS%&Y?ekTCemz)oLmG0GkI^ln%H$l3aVM;u8aL>ALlIDCPIQQgD zg35`mmg@bBTHfCoIKJl{qld5v>j_IsI^9!yvQDhfFdW>>$~wIA_3E%UUVj%R&)gOk zwG|Fl+s*C_d}t%pTcW%!^qz)-n!_YK%Hog}daC+}9$Z9kN#!xa=&kwNbTXMv`d4$F z#}>RbyXoCWewFf=4kJmy??!iEP9!D3->;?&JL5I=x*y$@t!NO<3bpti`(zjUlpHqj zUAp4(PuKpi;XI^i^-w9KAciYdR|fWZ@-(0*PM-XHMx-4HpT+wzt7xB-M@#)u?T!4V z@I%zYTPix#BxR(k`-~r&@H6tkUCJ__8KWJpgr^g7Mdfxn(Z%H8;M|hdFp~4MfR%XU zAdVN#5fOudK__bi+Lu1^N#i0#0;HgvgBxB7dsQc;6Jy=+-Yt1YSTk*2fHwB7=;gp^ zDH*_GG*EBEB)2fU-B%6;$uiVqo9B5~=${&^FU9qhb?7B;y1}!7BhfYXV7EoT+rVQ? z>*rTJ6wI;tcvH!KPCn}gOC30YwIJ5iNDk%woH)6v)+ck-g@4J`2%cnnr59}>?sQ#s z#v!reiOfVdV61k0+6nBd^kfMsVyvVfm1nop0MBga`K&8vR_fPZ8V~qsSmGG4ITJ!Y zTrCDD%ly&q=P5?)=gx#K-xjErTE}GJcIWT@i^uWzYCsSEDE{+QYa+7bG zuyLiXdwQyqAO0Gc@V$h1|87Bt0p#J-9#8KB8MYKyPoA=s(>N2~)#dh76BlIGzZ$2l z&P^u=@%BM@ml|r@ioY0DoN48oak$n6|A?tJL3)X=)@f*f@s@qa33h)MRu3|`&4&lH z-{o);1m<}Z==z;vi=tn>{e)LhN_FKZdUBQRpoFB=J|AEOaB7Du9FDs#7ajGF^<)oR zNuTH()X|wazT#Z_sbe22C(6pcGC0c>qFFxAY7q^2Zfk^BHd9IrI%z`6a_0dP=i^(a z^yQ~>>OMVb*LmMovb=|5AF7Mmzv5eK4tV_^l}_sG>d#}|(;JK$rHg^>CPoGQ!ym9MPl_QB@hx>*wP-RlcgyFA(-RMn>t zdlunx5#$DCDI{##W`_P}+X@%c66k={O9DOd?A%NJ@HVTd_E4pVKgyjhS><)7#^F00 zVH&?=;tVVO6aTZ{TlmR_*#B_)*E?0XVs9im73ZP;m?xQL>^{D5l9H(=Zw9$jf9aUc zm^jxxpP2)D@^@&t`be_GD})s>eE;&(HF*X-(h|E>FyZ(!yTE?Fxt#LP<3+@G^I{xc zAHsUWvd`0$7D*cMc65_@3dL{dbUdGgmf}@|*<W{zY>s#b)n0(P4D?3TiL@ z;)>Ut52v{6_jA#S?xYlddwM~j#$R6?VcDKt@VNg6>R#WOJKB}(Ph(}*yY=OYcjr_E zkx-KW_;99z@>@<^DND~uhkCld#jB$lG?N|m`%|7=o-4`Ciio-%V}i5_p;`_v@iXSh ze!kq?+=K?mwN zDxZ>Fxn1l$JZksvRNoZ*b217eO>syN zzJoq_3yon3Eeq~9k3-2FhKuV!pX-T@xP(pj;ZJ3crxF^9wz5Zm=FIz7)>-Rk_~x+h zgwChW(CHdquuz^BrGA=b+)DRvr0eGs@hF@5`8!2nJP#B1<34=H(>3Seb?486*NOdC z`krUJNXF`F>C*wd=Mi*iB#qhn@>pLv-Lq#2JnQ9a&>ub#rJ2Wl9na?kxnafXVKn+4 z(v}C~v3d!2Vw;M6cO%g;tK@^2867wq(&SVt#C*o^x;gP)P zxj<~d!wx@YaMdB6@9j!^pUdNg^(&Q9!i<$W=S$}7J(iD% z`&;R0!a;p)ohpgPy)u8Pg+$VFUp=XR>ido!OPp~|8`Vob?BV)q?)336o`kL7RkB+S z=TAOPWhK|XWTEUiY+_wm#D z??!f1NA{^xhOrc>#c;)8y)EyfZkjsOy_=~{Z@+r(ZOJ+EwMx@~CnvQ%sAr_p*X~u4 zL`u}uAD*mX+zASRBm!9{mdDZT2-OJIl8}<%=kQ~;uQ0<$2Ev{d;z9d~oC)Xt<0k(z zoQKxf?0x$>#kI0>^Xu=;Ly!#*g$T^g@spcDJC!P)bpO!njjy0=#~dMg_t zV%)G+{XrDAE756{2|I39cU?)*6OlxWGHl?T2l70*=F%A}r?owlB9nCxHVG}*VEXih zt};5kYI8z^z26_AUCrT?IhHJ@mjqarcfXuIhdvJN+z&p#49Q9VqS{mcLC3u=&-k~V zNNvcIcc^5>-;%#*$FEOl#L(D8+`i;GPmK9D0>C2cDC>;z?ogDlnCmYiAlONGL)l&d>v%kf% zJgYONM40|405F%hNo)Su8fE`xJWheO=cI=@J?u=ALyg8L( z7NYk)aa2EB{c)drZ}PFoQJy6;@&Ela%J$_G{2ctUPBIc%Xo6}6c35_Aeh9zF%rizT zr7`nhuYT48KQItRww*7WCO$AkU?We*Q+;`tWN~2Ob##i)<6<#VKG^NEeN>-dVm8uE zasEyAScCC?7^f>6!Gbbz6!SP7E+6HTrHUs$}EbP(Z=>uRG@qgQj%jh=&XW8=oZcBa&oPijXT2CtMNGp2{_ zx1xvU;GCsh@&e((_5 zXIjhWlBM+6u1r5Z(M8ZRHg<<1ovqIr?l6&>F&HRplOd?zX-kFO@Hq1K>pwjaf{_z8 z*EAI2O;Ur3l=V0LKi2nHst7jaE5cq2?2wc&rF+|+y<8%X-_GjR>~#51L}^E3L;6?z$c%A$X-OWvaf7xPmS|&ch~c=4J{|kvM9dyt1E?e#Y_T)Bt=-7@T6y$6+?3F?R__ z9VyTIr(|!O@GOZ$WFwpF_i{Ow4~f>6wiQe=EzBM%EOxKNlfZ`k<`yA&4kFWlzaqLo zuTwztu#s}wRD$SXByaCReGf6i@5w36fvG(K+HBJCfsrYzvzqX(_NoSZM3tnIkLLqD8^;3Jvz$^k zN2ZK028IRbjm~+5C1)H)ij4@^FlNuI7Mxd35x7-<=lOj7VDg@5ZzlFhZ&A}SE~FQn z$IMyMqhq@{jD|5>nYyK+>o1<3c82(#p1uy(zQk&t4^uN1(!IgV5QhYWC3q?Y4Lerpb^wJ*)BTL2HA+Lb;l6cdM1 zGe?e=ugKP(!$M;AWbyqLfW0MB}L8V({@}MJp-9S;vjY;uk`Oveimbzme7zrN+XOyU&#au;>OT1gyFx%nCxZ~e8r~|A$By%7+-iTGkHFp> zcEhinW6sWNRp#&0#c~Odi$EF91pj0Y_l?6AN4y$1D)bRedD-875HW2^251~ z?K2kbx1c{@AhN@(M-im1yO#pW3 zk3xKpRT+o?;k(8;XyISO2@B!xjVqaQv>fKx_HG2%8_E~<^OcfPp9yTA;O#K4GTiKC z9}(=gt%pP(?^UljLnfl-9>R|a4s}4zT<}DYY1ZA<>)4+?B;%d7l^w21p2O4D$C^6q zN)HC1vb#Ul=&C0uMkC)8Tnks5n@MZM+!7DcWnDJm@IN@C(`Fh0*+_L%I3}t>7(?F2Zg9# zv1C_{#=5^^xgK?hMt?izoUvOd0ZF>@R7;lRn)dET^?jdJBcSSe7|QACInJ&7E10#V z=)6rm&dhtBmg(4WIbvB`4zc}m*!Nsc=jtji4_=+OJ+kRr$wIp));$I9$59t{pw{e1Ig`pi{A^>p znK>Tk4<0hBCPy-|sPTa-e>1wk|E!eI@mU;b`Sv%RP5C5Tvk!y7C4azkEqE61V|4p> zAV8lBa$Qib#QsE7)^o1B{|YRPvbH6 zh&1(KW>5U%sIfodk3y3k3J&o{+-&e+A;^(J*HfdPK7Dr%p6Dr1bn83rb4jl(r(NgX z>hfp`ez2M@^pRMma1CJm*HKn(jUNl6H#J1=p4FZM=a)(+EA{xY~OB{b>{^Ne$-l z76+-X$+>%8r+M=UnnM_UjF4E-(z7dt3(WczJy7z6o~yx$J$=4&;DTpIKet7K4UqK} z3w7RZ2k&=y@B0JWbF}8rfjb3$%+uQcDPPX{_sChN{~ClK;p2RB4fA1ijn#W4hQM`1 zi6xusuM&r7QOHzK?+}FR6)Q9659xo;G}BJ`ELh7&V~iH4ekDoqJPm&Y5{9`gY)k0P zPO7#4jih~gJf5+GgoxT4SnDZ2RVdgW3u#I!V_%*n+X8kFy*v9PJ-(1(2ljM>r?|6n z^REEDGQV9b)5dTm)-LR`61qm#Q+~dVHFrzjQNMG?;`lB$+AuO+NiiN7gnrwcG|-Yq zzTp++Qt5_)UwvB@KJc@7_{RItJx$0>gmzrr?CvrO`4&8pqc{#YU(<;mOA-5)}+4 zceaW?cR{he{n(#h8HW7&mBXXvvt2yk`J}Hs?YeFKO_V5{JNDYmw5Qj<3LXWn=SiXf zmi79VEFQ1E`0JDjMjPquIn5Cb_jXbqe|&^`8M@^f#^6xU>k~^p%{}jP$upIsnE#UA zQ%CMTRU&M+$mEg>jSsY>3U%CpKj(jfIVL4Fw9BGcDX|GbcoGo z7{)0=26El$uH(wdNZDUJRpZ>OSdE-eZy~`xG`OFUG2yt((E0f+rwm+EI+d+0N)ZK_ zVZKxH?B}d#Oe&T!YcKPY-bXr~2A`T#M{oWFeU*o$9oyr~3C+>IL2=yIB&8ARNz3KA z>5nmabPo3^HNZfB#)mDGG5xcO3N0+k|bB2GMk zKOC6+obNIh-RfC-yYc}95Bi0-Q~0K!s*9;y%}fLL^o{AMg0Sus@0~3TldR?h#y=-!Gp-dI1lBniIx) zWE)GjzkR@SG{ERzX;AYyv0)lXnrGBP-@NAOF4?Y}C$S~@YNO^^wG3>tO4;G@sE&~s9JiXDb$7%2 zifl^{lmy+=0v)%-&z|kKK_Khi4;RX<6*>J+Lp*y4O49nv4ffD9V@8Dwsv|2^YC4Qw z4jt#G_jGz`I#%{Ga&pw3Ld;iA6+d#*y{)Dg#*9er?{hFkX4#mT=)6w9IdL?*L2l$u ztlzihEsMZkr|xe3#rf{NSWi7&R_o0#Z^SEW^*(oIlUV6QQ}yhlSF>5qj~_Vio@ef* z<5pV#avbb6U65Ysn46rQWPaJ&_cZs z2ae}!I>vY9BU|-Szlqy&UR&FwW);C2uo6j$AyOvag z&W@(rUkAe4SKiprT3m$qT>01%cMVDBl3ZEyR(d`cMc#j>dNChxA@7jWo4?!X&r$PH z@b&A|3129OHFBieEuUcFLzU!(mQSoo(waLwom>B`U(IX2;Ab?R1UhYM?T_f}bldWA zpDXVU?jd=4J0u^s`*e4of4HrsmMcTD!e_$o)v^Q|?1(*EA#@s^L`Xj$zbKjo9V zNrB&N)VHG4IKNHjOP|DXWP4q%&M#|TKJ>_sw>r;qu>81( zcc=T`?OSpi)Dcui{mu#NH(Gq>`OgEre$I6GBk6r=KC1T4Iq73)AN(EJPcXX;esn^^ zp(}1E$Z>GY{&!zV-4%rNb^7mF-<$a96z7Sozr6j@135RiCrLTPu%1m|t_D5KZQC#7 ztfuTUJtximxQKEXQ(9W6mQ!?Z)K0GWJMiwf-;>{)$F-iV%lLYC$(`{HEWPf_p>Njj z4J}{rg`eu#tIz$$+Sxuk^=wtt`2pqYAo4HkP)+_tEo&BMPJOH_%;#o)=L^bpebhbQ z@hPo*0dV_dN}utUbQ&pL`d0|Y_%a68Y}!_S_CwHKIX^?n0_?p#4~qQ?V@&x*O7<0O zJyo}emrN1aWpQR7MP~-Tft|jC<;!<)+}SCtrTd;1_B2Q2k9I!c5JKumS%q`@+ig%f zk?qdeZ~aWgqdt?($UOs&*4vZk+%;zZ$(dkJyRQf5q_fNOW}nj&ejkpuS))_GlwFv5 zetP5;wCa~@)|da2r8G6}X_Tb46u<2Oi~ZO?diK%zK2YGw_dRvVc|pDm-@|m)|Hzb@ z&WEo#q3PZ(kEb8IS3|L$J%l^~=eZiQQ_#FpA~R>O(vjBL(ORjP52mr_IzEKwmXt}O z+^J5dq?`~`^yz$H=9HoyQIQU`zS0Y2J3k+2UnHV^0l0Ej%Z;>WR?9F_a|TINC+bJ4 zBP^pM@o3s4^XHy=a1>;bMrtJAI&CxOP~>cd{m?+#lfJ50DBE5G_?t{9;u`Oq(#Wzl z?a80ilIi&BXH}Ngtw+8_N}Y!y81u{1IGohe8LCgn6aFm7hSByuO&MXaelpla9B9J` zX(y?QyLe0*k#)x|qye!Lw!AUkD$4A5*igwvos*v)fmZ7K*%J=W2E=)bQzz zDn1-fV^Xi)ImggE5a~Vi#VobO!I0XMxcEBvc4r!DUc|5Q^3BI-$(Gq0CX@B$z#W01 zSmVO`A+4mvs7E!tmG)PjYc?nGkBlW+@K1!bFz<8TEh*;ZNvagNPOvDDQ{=HQ-08M3 zvgk04w^MN!L+|IQvqJwkP&y^OC~vPWG!$ogn=8e|hLt!{(v$tjR?%WY&O^`TN0h33ieQaK3R+K{%#%TgDUw({R23AtTHXz_ifUU-d}$9Glw-k z*_E_J&Yjs|h2lWBqZ;PiC;XBz(rwY9=h<9ie~5Y{aa1)Qk@#fIWiPpcseq*8Tf+u$ zm9rXN-ET`b$F3_*&0!emHT3 zXZM6|r6fa7o&ngU$T6fY1`$(rub(p3FDaywf>>INx4l~(IjCAn=l)oX7pA!(%HX74q2y?sJ{#HJq*xP zJx?h9%Q$bY==8bnFTs9}nGB!d@ORgW`(4_HS}X3&;PhTXXm8jr%s{LQy#>m>-tdW> zv)9fJK92G2ufz#hPFNJdMYRx{)0J%(q~^D(}C%IK0JAFa45IH&*$|oD%I5=_saYZ{UX)e_c5?|CH7K za+=ZJSI)%dc!$)pUlCmElHL8si1Y`slGfZ^ss8k7l4H`f*U`ZQd4S5ne?7;dYxns) znK4-G#~%5tb&fs%W0oI>$wvUqrq$D*@{vztL!vPrec!O-Y;8o6Zwv3=pAI+wnhy+G ziP_&C=*P|YV5SjXgk<&7K-nK(%(TVuh?wt0!O17%;0XRzkCo%UioG~oo6oo_GG8YD zlAJUl9w!~rRs|*Zg}jSG0vgVfxR2@c)5*)`d&=P;{2cfxrr|l$SsRXh2Uk2INjcvX zr>VD`oj=}OV{eF_2tT%M`1FD6U*YKReRzO|I8Sgy&1b}xuXGlX9CgBUoZ|f(_Hsis zX3nY?(fTap$fGNi-xHXWdjsi*7d#1eymlshb)Gytv#Q+EnZucCrbX;&bmAgwDOUmxyhh4`?bBIECb!p~GY zbhv?KfzYx4S@4Wnm2Bvb8BhpKjgYR)a|ACQfiNR`E7(`WBBLP6=qF;I`6TkuRqmL0 zX5nN6HBsBdk&Z0aYQJIJEiI$ok`7_z821P^2>n{*s_ZAVkI1~x>;wY?r-VN*O;Lkv z8gQ})gbsjP@m4}aN@8|uHgueUbRH3s_>L#MYq0xYDY;3ptRwljO!Q}SnDZxlCLX=_ zN$fs7OP=u&@aE4PrE0J+k9cM0l0c*K49A_SB(8n}LdH4Hu-)9H6neCEKGK3j?d^ufbQdHKfd;V%1BH)>*|H#OL<{*a= zM#7EtSMDUzZv@?WPTm~a8Y+hs?!+L^665ay$POP zl#u{}74p_AorxVD{CX}#NPiryi4Nrvq@h2NwIEhi>=ajgY!sOzT|9qb9`+Y+hTX-+WVEG z&a-tkpZ5nL#e(tdbe!eYXAUdQdWh$cFto?9PmHvuUh`a}AyGu;4;Z3HpwQ%a0Nb%k z!nb95{HS0xQ=!lpCrR22{wXnqRt9f;_K6^ogFq{FvgFWk&TA3A`lZUub*SoGqt0c$XxawbGNJJMntk&*i9Y z2JOtr7^X3O{+2^d8qDaBAwnKy2Wy*8#tL;GI2k!blgc?PrzQIkx-&(xeZd7iduE&-@bchJVN!{SN(l?@>?rx!Tfu1$*>PUuY@1vkb4b0CbtFI2c9IkKPCj(F z3G_bp@jQFyK2~&bZ>-aeBe!gp&IWE@#(Ol$f+6ofCNAUk$vXM~m0l7@mbdUaYYw-^e?Z??I=)muSYu#%~gG-oBDkUYNjOYuwl86uLw04{iIsei#LI zlVgu~M(u_#*a&w}zWe;lnx}nbPr)k(k+_@IfnZ_s&}{immLr0*Yk2nc^g#c5zB>!_ zG(T2dcA9-eP!ZkD0-o-`EvF5$gW=ws!!jj$_cRm%+YwvsUFvFP8yub(+-PG#fuazGY#q>v1ffg>U_HM7U6d}JGRyECcn z*>`Y~igb`|C5j)p5j^JV1-)|0r)qg~e9nUjmOMFghFR)~W1}aW?DW2RJTizqlK&sV zLrs(OHvYT5dme^>!!@JFs>&)4C0RbU`7676B~|bzu{rh?1(kBZDi`y)&0RJ7Zsa4= zHXC^G$))E*<3`q#GQ902D}U&#Bf+<)Ek8X5(wT8v0zHi#Ay>n>g_;=s&mTiw(z!;= zQ+O&_ETL~c2;5#LtMBf}2k0jzdDqMJrVxYAwUhy@9&4x`5?P-~C<^oZ%5hu|tBW@2 zQCn%fKU}k*!y4g=&_>!WYL4o80;}(nr-lZ#GZx-6@UV;1 zmA0$w8Issir1qxE3u4$`MOOBD zOpNniPW;Tr@Aj1O4z?=tK0c>F|J@N{rQPE>=s4nwM{*^M;Ov#KP59Q!Igc-Hf!4o^ zrvLf&b2t;HJx7W<>v_?HpGEXK6vgPIW<8zbJg7Du;dO3vPxkc3HWgY5^bGEM_Vy&a z@~(5oF$p?wb7Fo9_9i#u9#c-uS z)$tXKc%=i<_Q`XJZ|phK<@FU%mtQ80^|_+I24&>?&V`+!kd`@5cTqlG_vM@YU4A)h zp6RVT&Clr?yh(cEJ$pWKu&1Mf-LEb=BBx`8*GQ`)$B19~FxsnrZB?fy!PZlp89tKm z;@-}u%-{masrdZz*Uks4dW6x@d}NpYkL#U|`#V9=G1K!TL+#6FYSW{WEx5AddiH8h z4)UHq=QKr^94WUW_l^;|zrxlZ;-Fp4yv4V`+nwER@{=K$yWIivc8*Jx*z&s z&brwuk5m6`uZGfyI6PEumIQNW=u*$hmv~d_xo#uj1zuiv&Rguf{kbmDz+jVpm=uJW zKlvlwrsMXt-F=boR{o}-31?SDCmvWhzGCold8a6Qg(F}8dPXd79o5ph+{(u@RY~@c z*wMh(pCigaPdk)25rm{VbNoM+T2E}xXLGJ8-tM>jH~ZxlL(Y}?mkBu!qx0?c#Ffz# zsZ;XvbB4y3lk4?WAKB^E(Elf3FC`6;+eeYKFQYSCp#qKlmw_hzIpZR0&D54 zoOHNP~(GPx@pGVp}3v3E>Z)i|uo8ZU{4_+mtsFkS?n&nTLv+;FF0)ftoQnIIRj~V{@e^tK$O*gG z=?e5m-}XuL|3)d~8|1+gg(<%ORqf-Nv~wdV<8*1RNtsiUL*^61f(PzntJMw{*7fn& zyLKP|*rs6I<9L|sC1}qv zvf*~JP24X+R6BJ5p@A2{nb#LjJw^`34i5&AS=r%>M&H%F`_IRC(Q?J>9n^%E>tQEn z0{r_4zt3%TY<(gQ&?zy2n^$>E%jh%UjX!K_i(dFU!wcrx7546p#0{9^1OyOCq z%zcT~xztDBfu5-5PsscJ<;A}l-h@$-+WU8P?K^tM2gEZCmp5XGp+T9hc<}3$ChCOA f-M9KJb%DHAzhmrS;(q6*k-C6t1+_7A4W6-~*cM&L literal 0 HcmV?d00001 diff --git a/third_party/icu/data/icu_conversion_data.c.gz.ae b/third_party/icu/data/icu_conversion_data.c.gz.ae new file mode 100644 index 0000000000000000000000000000000000000000..0e54fdb9eaffd814477460f71bc194104c1b247d GIT binary patch literal 177696 zcmV)DK*7I;zNPud7hf#Jr=cwV{=%)^;oo7Ye&}i4?v{=gnoD=)bxp^Xlzuz$`-l(A z?>4b1)Gv~3LjF6_zl9Szzv|QAUc*X$x%~e2!`Q{|z<6<~(bUb!&AR&`mh5+>Z)6vC zjSBhMJjI8^6oi>!xIhMmK)_1DAVWkxP%;Ba0c!rHw!d%NU)`Z(Z*#((Pt`D#^%Hw? zl>=mTz`uWgpQw(@`^vrleqgZRg+jF(R=aR9SZw>Y+caed4|rc=i0-pxK@j?w7U|2X+$X|UqX<>AKzBxoeDtf4S@$rX%sjjJ_ zM|D@-xp;}`ojhmWk$<&~KV@-vpS=p*z`InaE2d8~y~jEo5KG1OGJ|Mq^#;ZWxQZIv zx8AIV%TaPUo^Hb7`ur;SqKYlA^5(3vXgfY%E%2!Nl|5KaMyr$2>ZH3m>8_5NpXzk5 zIUQ^k8rnRImu&Jr9aRw7QKSVDdwyr`9Qf>0@uEY~bZUm%|e<)V@8 zvMSoQYWvY%8Qx;>NeXdy^Eq@s`KjA zH8s`MI8{5HN2FDEp{=soRu2ZN&M{WkQdQ6HS=~*$YWLJus)YO1V}XCk+(s1VUGM#8 z?mtUP;QjZrxsl~;#&R}eIb*S$u~|T)nZFwd#$lH{Oe9SG;xg#?>2FZ`{0b)0;OP+23nB^m6Z+V(8_f{68dLf0fq% zE9L(~0Ja}T{c}kEANqp-G646m+Q$n1vN=D#CU0kzfB$Z61t}MI(frZ93~%Sar~Ie9 zkY8VMhT!7??`~}|D40hE4<}D(rW^Z?9H`2vtP2{rx)Fz-@%fPa!m?!ONR%>0rwqn_=2t zP+d~!o@vJqQ^2fTbvW%7ADRM-FzvS|erd_6647Eg8@=3`wTs)8igxRhH2OXw{>9D5 zsB3cWPQ_h+^sFA+wEdQk-mSA19h%0@nAD|V(@<{0cuUCGLw;#fx@5e&Dh&syv8O{( zvOw^wdq&hklc8}#NE9f9O~ zb`_>qdt`N={N^*-+(i~ z-JKpqGvO0X~vVE@4#$y^RLfO}6UI!}~cb zw@g%CE-$wk;PS`}&ozCCJ8g2R^dPgXcvs4a?IU1Kc&?$?hMil_>R4%K9`|$?O}+^Q z{I9RKu7&70cBNFn9KJ#Olfw z(sWZqXQ=4zX5lEItYcMJvP1*CPq(Af?RS==+!|7vnhK{-e&(G+psg6%Rv&b$u3hqi zi=bo|qBdH9pcK+RRk>9(o-{RVx=C-V-rvctpunzeVje&89d}1_M)^E z2mdW4Rozat3cPTKhay$7n<)08-Ue7-VAU7fHdx+1wCKEVxv@A6*OoU#*vBh7>`hS; zgxjofLj(V@CEy`yw&X#|90Ab`vTOe+)^WX zMcffae8D}?2l%RV@^6kQ)`H*{7ag=OxVX8OfR@lEWK+J)&MSDbE@fu-zpm>v_r3Ud zREYWC=082Q!2FOH$58)ee(&Q=@Y2A$lK?QK!|2W)G(Q@&x!T(!r(Bh=|GKASBh1(p ziN5H%|L$6jzE3T;gZlxxrW#q+RjXIzT%Vrc+<6YM8I_egR8$q5S1wDlyX~7YAAHcq z4=v%TLN)ojDtVVtG_B<(1CH1G3 zh2=|oh*j`NpG)G?R9a_pRc7Dv*&zGzF_y4}sr(arrR&RF?$EE8|KAP|T+vWK7?f7I-IEd*W#-%rSvHU$tQ2o(%2RGqoh~2&(-;*SVZM^4)ng`^uI}B62o!Q zRdZXH;GAc3cC$IXY2p!XNhrfvg^ye0i+GIkqg~pC0a;P^*6Ux=oB-8Jbj0aUbwYBB z%II_b_HYuUc(uEzfgHvBM$-+aNL`P5V>y#Gk+tq*tNTOMVGP0cD@|SUXR3qyG`~@Z zz-)@VEaDb)w9f4`Y&!KC{(G;H=}F(Gb7K33Qp;Xo!SrJzTx43CBKVgpqwrK#mR$_3 z&#q!r)hI4eMX~07z^Q^Foet8%iQ{rwxBNiAB{^(4|FYMNQ^BR0Ki6Cm8~W?A*>G`e z_s~5yTR3hm!Y8#|f|UM{VAJpJ%prlqG{;murTz{0l>E7mrR% z!7eWGOAy{mMbjgdnz#S#d0KMqb*XR32)>l*+lwoF2$ookwRxh7e6Z9)Hw)B!M3|?y zkPku|YG^>!JH*6!9x7$aS&|`_g#I;^7sr$#=(p@p`$p(glM!Z#gS#ci+#>H&;>-s* zaf$X%F@j5s;H@p-)}u@JW~HGHY;wWrlA&yuLqvax1@xnYE}j43WgYO1_2vQ(^p1oy zXaMur+PgDcP-X{k#TzbeL_+{oqan`~?%UIYfXv?0XwayK%vP`WIh=EM`4P9N&=q5s zHn)v=A+I@mN{M>j-OckO;Guugnx23Zw3&&eL%}j$Il%S!3s_>n9vzclptU+pTrG)PWdXzI_{C6eaAl z)o*4^?TKDAS1zl>QS;Jy5Za=kvKyxoH)d!^_ zIz4s!ewO8=V3h0l z&HobUFQkbT{;5;Q?qYS5=xromc=vy;qN(YbI;m~y&mU}k)Rkgoq zuY9p45oq}<6nEpvZajp;vl-tFPERWCx~i(E|k26KT7S?NbvN7C3yJD zhrfLIfx7O091UFGw_LfuT)VPd)v_eV?Aww|SA;HsgHcSbw{>G)NVp1kh;h*luuU#s~C6%KZ@(eE?e4%p}1HJ7& zbLWzP?WMG{AJ?=NxptQA1}hudrPMChcJT6w+*-(Q^f6u43N-OFX0SA3k$jW8WQ^5q zSRX&h?-B(vPLEUr>K)#w126#Ne){pv*&9$60HT3&Cg75tSJ&gY27oJifR_$hxvNkj zwHGmEM!XD4yTT8t2`1_QO_A5(tx9lh>`#V#ugIWlG9^sBe@sV2PE3g!!K zfUS5lZ?w2hPl7310qFUJP6Iye3S}doRJmR*o{LXv^q^@!)hXCTtd5KGuz6kc)x@_o zX(YP1xlbtNpD94axyY7q5i?;oW#_th#E(0`?u$cwgpN)mPSA)8GEbJ|xX^D(;l{@i z!)|n{6ZUfA4?V^D8#CyhDZdz@qf-hlz0e;-aWeuSZ1B1OFm(g-s}qLpHnSA(MU0d$+`>U=mFT6aFkk0i zT@~B(I%imK>M*d2kjG1@ zACRWSh-r6^u+&GNJvh56HPc>4w-^jB?nrOwkI=A4lWpWaRG98XRsub914hiRluahB zmPe}k&F6e`hgwm2_~?xv0y}pxoL}9i0$^D#u1@hI2&Nx9K0R~L^kY7z94#i!7hZ*9 zl|o$H=`BW7f^Tn+na9su5F@!%45_R+rstB*dx}exYRSZX`#w1(6#6r&NiXGZm*D7N zbWuq(oE@6N6mO_vsJ%2=p6v`rB%!S{mO7);he$rCUMK_;zEr9wlm1+M(z(0X76;~6D4@GKW(yJ(U1f`Fb& z$}Jf@D%p~&DEd`7w)K(~SS2hQTCEa1-;B&J04xiRYc?Ldxxt$;llmUv4dXs_@^g4H zTJdF?NGYgf5%3aE9X)r$3dtPm9M2UBEoV@j30|_*E*V}|_`;el`dBXa?eU`!BgSZrYxZm0FtyPn*;QgkSO|=fcEC3{@R|2-(jR{wU_Cnqa1)FdoA9NwF zcENI6@W~)>z}oQ-uLy&|s$#&Lt@Eb=n=n{`hw0T zV>+|{FpiT5#HqmOf@JJBu6Phm(`ayUzYv>PCZ#ymkMq|>VF%m(hvl#Os_s9^buA{- zVfr&LkB!@0mj8#YVL0@qzf07u@)420gx|*McmbkNM{Dabo&HAMmmcclrxuIvS6L~G zm-mf!p0b37w|M!HB|`#Im6+7g#}6rdeNi?9m|adZzDki;(6kmgRyFzANP!E7TpRU8 z{f1+%og0%IVx^@rM&4?TqZ}LOL3*6b<_$l0gOOHUs_dmeF+cmu3$e=g{g`z;yuyzi z1=BmG^sfAK&N|CbFk~?B$tJPKE=Cebu?->71~XxSO-`@!x;v{_9MtI~Hu2Q)kaq=e4V! zJW@S`p!q}9ANrWz@JF3d^n=t~X)0D)E&`^ZHhgIJhU-{;hcL%tS)Esw((7vK zOOrKNzP?wJ1-hvNEzBaHCXX}e4V&OK^`M12+E7tYc}{_K&D7<76kZ5?)2oNT8Blq$ ztD)~m*Nsl~)^YNie-YAk>1LJrw2R}0i=MFBOU(4*Dm6_HCjPy1kO|LIp-#1wLz;=< z^7tRGzc2E4sE@g@em>1IQlJn#3W z^o{ZgbME|B*)bYlviT!WP?X0JVAu4B%jnh5dv*~*j}FzE9&LJb-0vHMU{>iJy(IUN z%d@qSq(jDU)e#u_0Dys5whoaYJVpjc5Qr%M?)}c;#mA^hr0{1n>Q!^>VfCtEmgzLS zy}Szsh$szdBucKk*4(6eXaz;gmO`-pIE`!t^?Uu1QeFauvMTMo4emMOhLA0HGzlt_ zHXLJiG&W5nqxc@uP)T^>Vq6gHDaSW|D~Uoo882jARvi&WCxXgHSTco{-WqJ=*x zHSpBFNTx9_BxRT)N{HX}a1@sKG~dr zBB`WIImF8z;AaF%9xYP%d*F_;mpE2U*9!hl+;CB2v!F{lS}#01-OvL*zsuzo{eNF- z(zNMI;=o~K-6|xMEQueukjkz7<2rqpU7UGXE=2Hu<+3}|S2@X7my_y`ob0O{$NdUn zehQ*-FEI}+tL|vwjP8>9;kS@$h}=bF8dHRs>k^ktIZM_#FSKPZUr%!WkkiM!CAfqm zJZ1##d0pGSp5F=-FXW>+urv&r5&C$XPKL}PL$-w>qs5R*h764^yhsM}n4uv@jiK=# zzSzCciDQLFuu<5MBZ}T3`vYf;`By8X$?qxmk^EKVbt0b)@6wX+CZ;k-(vtaR$$Ycq zWwxYNUs9LsLei3lYc9F}h<{$%!@Obd5r^c@Ox-NuLhsOMZvtA>P)XBGVwQXLEa!Zd znD8ZuVo7(o)(`TPXTOdhTDeAF;>~0gn#A+i556fx5W0cmu^W&z9C-h^H;;*fF=JzHSLnr%>|?{ zxv?}%91Y7w!*(LMUDsQPVq9nw00A)S^b8qLF1GE5T%$^Fd?^B0rRw3Pg3&|XCq!jj z3)9PrbYx3guZv5VHwxFgQEDsc63`aHTnQhbJDMZi-2tZXUan8BBmH4)T8QThlg^TH z;bPzyHm1dAzS)r_PF)KtmvnI|`Eb0;H33>Xx(@)k%Pa!?!gS#AKJJ;-H3SDO@4?*RY;%8Pe-=(qWa(Q6Pelcq^dV5#l6;HEb%nx|4bNLlWki1hy_IZm8Du=%67|V3W0G$QENrb4CPK zxuTTqMrNY>2lI&wIc5If-_ww!B(qh?YD8GMIENU%lv}a58LM7DxcPJ`^1J9q_y9Et zN#3(99vM=T#evQv9-vP6pIh@P6I2mK96-ZXOHw4|pf#i(Lx5%uDz+7W?#AxYp^Vzk z7gE)d*~Kl@6=GVU%3harCTiqj8sgb4cy14_1lR#S0fW>Uek zGEas4r!0OlGaUM1rhsjgfNNr-abP6kemYZ!;oy<(uz@pC%717onyLTHVF4IJ((Itl|DwpW!mo za5-w=r7yoLxY;F1Ye~{t^2~VGE86L zd4J1RHDt#cn5o3r`b`U;ty|`-ed6%i5l@@BbC_lRUrg2k+gsUm3p~NG8@v{(1E{P#aLd(Ly z&=67@PuLEY6K9@tow7&zlue4_3@gk_E+$H=M`u>50_! zm#0XSAS0o=U{w;c-C&ZNsOyDo!<%DSDp-*U@{ZX2MoJ6qAp7T3(e#$f;^oLL4z43Z zG&xY+&J|v_j9RzUX&1y>63s-C5$8y-q6-Z3hZsj^fn{tUd4gRs!kH`y;X$?&Na(H` zFGzamMqxjMO|{R`+_~N4a$A=vD#@73 z`lUV0ld1}Wnl2uW*SzECcU64gS6xgJ_p79_UaMX>FJ6ICB`)Z6`ZvXhm`r!qm(YRn zs!MdiL&ZO8NM&zIp{WC;$gyiWipmRJ?0OPO#l>t>0#cnr^)$S75j&)p+DA*cmr35Z ziR`?w+eLEZ`zOxYE#yRTCX+nsauJJd-+P1aJSUBr{HA5jy0ryVq!URg9+3F>fO&=^ zf1wxM|Dss$Ea$F!darX1i&FNao4r@U$t_W3Qb@#6)Ie22_v&6Z#4HwNB`I?C@NXR2 zw(-;bpkkI(rA>K@C}F7LuIcVV^toiQ`Yh~Gh8vo7OcP|qF32h}J(&zi97!r&2JA#3 z)NiTtij^$5@+=8gOCP^D62uMpRtRdu^t;RHoJXOkk|C;hN_j1ux%PSq$zmaaVAop} zlaB-ud5am&X+$2n#z0W80=c~fU#zOIPPHzyzVxHZV}e!H=m~Zy)l5$oBp@d^obK$U zNn&v%^m?IOJU5YKLe#+(%{xB$z#&Voi!@vhY0M=W&wb#c${NArksn|eLk6V*|EG&f zg3Hvs&#kgc^6Bz)>w0tw=t4TFNGtAHc0t(v0%FdjT$MTvNo*jWxJ%v%0>BMYDA){( z+eB28Jmg5zP!Tr6KxD{=E*DK6hzHvtFB##^Xd)V}A8rrpvV%FZ-=tt@uF~EGR;Y7Y-H=czU6P%Uta-k5oBCf|pv?&qo#!XO$ zwI!RIOmr@*2tB8;*Bjmv;}%YTlFB51IX#vrxr-`$n<%I!+M3!WrhEK{17DW|Usw3C zWaKH)d1CFHQkG1y_fzt-7gH^C(Nxo};NB$OkgK$pkR_9uayj`4w$l~&JE_z3d(&-L z)9C~p+jiI-KWLh#^3qM;Q>K67^c;hJU#V1J;w5)l3&4pcEgZh;C{xGqD7WYXn+7|VkMcd_qy6oCscPfW24ej9h)Wa$lV zFK%x=^xK~t4x$T?OF!mG`9A(+??(=61&S;eK89ov*Fh_j!ga~Be%~}*Z9lS3OI(>M z)WR8ZDfo87f=HvQLyue%VT=qYGEo>Ni=u79WtR#=KxBzH^TGxRqZ1mZ%d-?aU6nBK z32&_xOe@v#!^w>n8s*#r4Ts36%hOkjr)(-0 za>mkig~#05)UPKLf&I#G5qA>=*B6{PyuAWVA!5$XFD?xDEF=R%s<4nJZbJ@OJa+*A zaYl(ya9;NY2qB0QVqg&%w34jiehnh*@~%G-hD~NyBi2}@>i3A&=rbfW+!s%&I`veS ztS}`ybSxPJ=z0i8mz2|ykEA@o^H@oRBAMHk3x=vgvI9vU7Y&h~4f(*ibx)sS;fA;` z|GcrhH%F*RX<;Ot*Y!pjR*-^K17TsIq`vN-5W!s*kU3*XLXb<~BBY#XYQG$u&oenu1Rn*E=P46h)9Ab`Gv-ApEhBA2r; zqsWi7NU@;2jS^lYtGMPqjvd@%v=B@d-f$*)MaI)gz;Y9BS5VCt45)=ATiiz(iK56l zVhAS74*BF}nIjDrR&lwB2HzJ3YSy6}1X52~PZX>%H3|s63Wv7Bt0j5J2E{ro3xOw0 zR_hqm0)R_DM6=v*0IQ30rQi!yeSh1H`;cJQnZTRrQ4Z~uG!#1T4;IvOBJY%Nw3-8oSsn=FgLV0HcfHF}DzK3}Hs` z=2JXtxD1nKEHpWII17x?9=Ey?uWe7WN8(_m?vov^kW3`fcuIa<7|1*{Aof*>n&6E6 zT<(sjf=Ne)6C~C~3{4}WhhIlzDTpeOa*9OaANkpd`9n-jaJh4o=ZgO}^_*O1~!8bC1`TWK$7%R24yO9c?12#MYH1 zOA*(DW?c1;;!VNSQMYN z<<}|xh9D|$cbhOl)LT|s4ypG&NiiuTrMfC6<(RL&MM!6=ZlggrBwe?L1R%O{p@tNz0d99(Qt9+ zmTJ*l0$MH~E&cWz%S}R_Ay{L++?c+g=a)3`k9(M3I=k{x)b*t#-`A9aURj?k4#T_ThZH(byQJJV@#`k@ zh*-z{5n|xE)o9p}j4CDywGh~@TGu-(?ez*BnQ1Zy;8L-x0$(t<(CHRd5{xdm!0xf5?tVs<_f{!ki-?IzBn88rKvGB}V|>#Z*n@BnG>VjD z71wh(_bPq{V?&K5G)xG#2R*+^%9!y2w<|P#UHjN`8Kq36Txi21-~+9+?pU1qq`dv(AmV;g!izHaq!@i&z_LQ#~d()-WA)Q z*AXkQV;IH`r|dP-y;iZMf=q(}fD4AcVT0hGkE1yM#RYDH4;mrkG-MqlZDhS%=(J>gt0Rn3b1B7yD4aras zZ;UG6Ee(Ym4GRREx*XT=y=(yW>x{#Yg+fxjUw{$~Occ)K!^L_~4R7tid^fmp^k|_} zk^R8pDEB5FyM&DZ_PE1N-(t44w{*QtYl>Gdww+sivrdF>I&))T-jsfw{u6pBR1< zc2g5GEpvcp;F`+vHMPG^&%&h7caB%5R7r)bYzJa*VQ6)fMA1*0Y9Fv6Upzw$@ydTU3rZALAq1Y7TkcRq$?sxfnowpSS zT;=>!NOmoS&UNde+XURDOd_*eBE&}T?R^(ANw`m%lwE#Luf4q%3DK`$X5fVjWfvzc zM}M>~m*jU{SN)@tZqPDbNV*q{2ja9EL_@gh+W^qx2JCm|0gM|8aYt?Ys7g_RUL;s> zcKgO6;mmQ-Mt>EBrw_S8u{G?E_L?#`4X1P$`-jf`bc!uG%y_23T&pE5Dq2S=9j7Sw zEz6eVkDDlko2Y_q)e&PWUlVpx!#k6YFb;H)ip|JL}x>FkJZqhCO*4P{McF2b50=ZGdr-<)hngqNzIF%K76P{+Rus!XR^e{6{B^usK+0 zY40DhY&P-4l|p)a&K8oNmb{>iE!O>V1{|{<(=dq9(AISQNc|prRh3>|nUzqc?Q%-L zF2?bKqr7li2~PxnK&7+P(!A=VbuTmlBxgb|{!yaMqE+DKsQ|BH$O^g^ufr~{;Hx>7 zP26oI4uJeT1y{bpCQ>?#I!<&RBOZa|gbkO7C@1Q?5IwF;P9z#AxK2A;+_}LqP%d$K z8Pdu9NGD$2aKTkv@Ta0=@94@qd@fEs`m&{YEhw;xlB(@vd51RUB8I}PBUQ;`tB%mB z;(z?mZ!fAYI^NVO7MTQg@0rUkomTf2@aHZNM|$i}WD{GV*Df&dJa@sgqAM3id((KL zERpOQzBb}XQCv?IOMI$%@ge2Ep*wP zUKOW?eVzSZLRckRSDgx-zTyzfI3>byu8|$9k$=mXSq}K|p>7v$_{2fC3pf2s`u~RW z{G-mk5T6@FXqQ7=uxqo6{8oyB4RnmUyKi{mnNgYyR88kPH`%H7GxKPvSfEz*Xr-mR zBAd==Yw~|=3Jr=wxKYZ4LDyp%uEy55xS_I1Bp^(oict<{n%aMkpoA{M^aP%FwEP1P zr-4V)V7LYWA0x(l;1V@(m4b6@m`(h?t}8p(K$E|z@(lxh%eO~-u(;pb|1SMS>D;uN z=#UCdN`Fp#FFsA1VV;LR8Ev%IXm>&1%OXD;Cr6z$lf9DOtCH@E?2%rtD5C*aB55

l=IjWqCbRj#7^h=@HX%F6NfzC;$Jq&TVjUXE38lLe{xK5>_j20#9PYE z<-fI?7jfXGli~_jo7nHBi20_|-_{Q?nPe+*{`gXg*-Ku<(po#8F`V>7@a%YZ_x_FCe{M^73o4IuL7Gvp9 z!=J!2eqT^`@qk2I$ z&MrLfVgd^sq_VS5zHY(AV5#zQh$$>&6%zZy)>X(m>H1TxNS zQ{5M1HpQ5cTV9EMZc)ynV?oPAgB)6xg4(4J+r7%Od4V`6D`n|4z zn6A~^C0P6x-hmtGuM?Kv=gVAo(b?we!s_fv)hSdLrTT`w& z9x7>W?=JG`EmwqBE;r!){^ev>*{6F~yyGP`_0+kiQ$mx@MD#Q%Ra!95bhu>v@e_hB zX}_eXGuezwh4paFni9_T=kyynypsAu0dPn(yW}*rQlsR4gTW{Ppu(zf&a)DC7>P0p zCgLK3>NWt%iWW&)E_B~ppAZsAC}5*FAB+wfcmP>K6B006P_V7sWpHWYCO1#_!Tq9y^cvlA6Ip~5z_#sexz zo9K?E4V_l!cV~0jUwH=;5v7<0Z#SK6n&epL4x~Ez7(rZem0dttD7_#!UaxQosE)#= zvJf*~Af$zKhBB(k7;9KlNjZo$Zfk|a7sM{#(daa|hEuV_czFGSn<(+XpbT!|rY;3z z6B4Rfm`eH$lo%YS+X*42n)e!RcyoQ=_4Zrh6mU4D>(B^QYO4@IfQ1 z=5r(>sZf+lT>OS{KA4i$_ysgXt5Av|G0u`M;hniIG)Q{xgAw&P+~z{(1DAqoNw&|5 zC@yn=7?PU{oe~9;tbkY$IsM>GK*8xUu1vc1qu?HC;kk0d>7YL{sH(Hr6~B-Tpi58W zY=;LBcul;_FjyS%L9W_Cd$tt#S#|t?2grI+#j+IXMAZPL?LZUbikC}s(k}c6FA_3b zZ15Ls0B&nf?g`TBHgU$S{E#5rA?!&)fzdfKO^{qd#ztk! zZQX}jM&ts#X{Oz>-S9-WI(FmL-gFLuGdX%HIG0N-%Q6~^BU*yxL70uqI1M#( zBSn&z7)dyJ)<}nL^D#>5hpSIatOW8|SDpY@%UohRTn`~m(6uUX4wxfv6!f=I!a>}^ z;x3KHkjevf&Qv5>Cpmxr|FQRNYm%cnv-kNdd4a_F{a-M78y{u z#cE5IT8I)mJ=ZnAlFTGioQii3-!Q}w+lBCY-NlL?F{0wwjBtX3S_HL z0LtsBs2fy~;=s*CI!kofiEqTt(WO1;5-LM`AWe=7nnuvrFs&!Eg;UU83Xd+)(v|U; z(03Npowa#n!4z5IP*$3nEWJpUS|cm+G6+10tw8lKJ|XGIbV**slhm`2+{y)&GAeK) z1|g?4wd+VsdXh8_9_YnNBmYGPBVuX}@@G1V73_kFJB2itX*8WUv84Hf(ot8;)-H%x zd}GnJfum?#bu}PC&m`b6rYi$xkem&ji_Af&5w=fq)+M7&-EbVUnA#@stm(Y-oQBT= zTq;AZS0XKy@JnU%&?>-Xd3-vkUK1^B6sroQEJoGIlo5-JGs-h>{I^DS;724I?i8~i z8_K%8R@qBycV5JF^8TPWhRA}Iu^_9g!YZThR?d?jhiW^ULu1sPL6h6TRF4Q6TEh;9 z_S7^zpn@N{q!cctQ%Xa`6nf@~B2x5l<>O?YrKa;HNug0v!v)f zr_mB-(Z5C>jvV?BM<)YHYNrK=Srw&{O9z5GD7pjHZJi6v`e`6x(s?2KjtCh!>IaFl zRQb+N|D#)tWT}!RZX`~XB&%%{O4VB#YiF!u@ilF+4655-~c`AZZV%(DO2G3vZhmu}L4we>QkJ#R`(Wn`G#K$xY&G zbbwtfU}VxJx7T@f@Jxt6;=!d+Gal*Msla|DT{^l*nG|y1e3;-6ej=h_q(NdYcNWGf z+Eb+-9Y-Q*gY$?I~9>CzW=dFFS~AO-y>3PM=)K}wgGOIKW*bwS`28=7)$kV3L5 z7r8y9)taPw3=Cc6*B>bzoi<7yNtsQjje4pckdKQYBheA%$f77&6h(Fv9$Kj^*kI=C0*WL0CS9E`ElvchT%as$i{eFIxBHThI!K>P~Patxy&tinUyX zUXy~k0v$BbGaN+R5=*2fZC35MjjEvR7bIQCX3$e@@>}P4uuz$`oRu7ru5a@h8KJTc zT`N>fhxFhP!0}-U9Y^UzHl9d3RfeRyu7$}Mkw8$n&4QC-uKFvQd8XEvD(_07TN&d5 zyDcOzrx_aGr=AlDz&DhX%}@>!odUpMRlNI6FryEZXZ z8o};HHm!+%CSD!oVzp>iv1S#>rDQi?-@?GB$mY_50*TfoRqsEDSRbjA=p9#WX5(yM3ghDQo zn@|xSBr6hvxO~$USeUbd?+8C=4C|{xf%`LbRZ`)_vnwZx?nREGn$_5|{4e&Oxiw0}B#*O5YYKsGYNatzl*J zkdp$@L2*FJZn5q0N2JK2R5)Z&B#TY38Kk)pX_7e4h_MvxV_YrtCWz%iN7zcEk!4pN zMt4=qph!YVbcJ+!C@Mzb4A4Du5*nd%=3C#vYUs4Gz#I~+%1gjEB~aqk+jB<+5|ob3 zv{O)~@LRX0L?U75{A#fa=_q&B$!Yb-A`O(?6yBdwq#`?1;q5YHTt{s!mqmbtts=|r zXGFp`alH_II4df@D|-#4vdHQb^l02ui4O|wsr0&s;Zjk@rOiv~S|pY-K{&NsgH>eI zN6ZY>Nk)tIReoZW?@8Sex9xS$R33|Fnl@+pku|v>Gm=-NpI^@{7^{q3 z{^M?V1j}M!Wn^xmgA?|=6ArFeS)z?vgGwumL*b#)0J!ncSuxCl(GK!Jk)2TbB^=m{ zvl*|d=n>&i8ErEeP8_(!8r9G?p2|@L;AlOAjAll3#Ez6mowD3IG^MGh`Ii)@2s=v{ zV?dECU(R(HL@vm#(BIY>LU4hhpf2b348qU^v^qw_X6_sZ0K3tL&o24ShOEDAT2ZfLTBb^I|8*JC=g z>K(0g^rNF29hEzugvx!e0%{M`9;iG}d7$z@MU$tK*V@a%5gE z-2MwUpTf<*aLmKmp+ahk#I4X5l5r;lt~uwLbE+Y|YatX~XuMFUqrriTmvhZI*PK%g zNgHs=WzV_hoNLax=A3KJx#pZ|Ks*ZdJ%wtZLdLQ%+fiXc=R&Zu&|R4o*$(HLbFMk( znscr>=bCe_Ij0)DvkHx-3Jt6Z8TEDGTyxGf=Unrm!QotU&NbIugA50Qqw$3F3Ar0V zhS&KMdppu*BFiNJ8t|fsL|-m)y58p^jq^BJ?{=NoSqafR_enla%6U?6mD+i$pC@M@ zps3xI%Grrx=gHZJHC9KZvlFGBCubkZZ6M0oiE<7^J^N5^15wXT)N>%l+1Irn9hJ@= z)w4(S9F?_e9hJ@=wX;X;1HSs61}B*A8-?+WmCi zE7x|rUHk2{-Jjh0y>h=ls^8zD-``^Rz4Ewsd*0icx4Yn=%5&fE2VdV347NoQrd zmv+CO-Yfs=%tCbUeFne%U3cDYl>je5(7)pZK`F0WKV-?{iDe0Vake1Q<6+}$Wa)g5 zeGN|g6S{R+{n}IZJQ?|4=+%tdOP1?o{yP~;Smr%lAxS4yzQAB88ds$e#b(8{riBeo zTw9E9CmS59ea5;#645j$Hse(@5M%flD2xSyeqzAnOF^%2>P8p}sXrB)zftxt2q{m& zeetE8+gKljzdHHbP8PV6Kj!NX__Ym0KpjYhxsrX-_tV)cX5gkPRYYjS$krL*NtxUQ zM$cu(hoLoFL?K?Oz~@lVGwAFG;k7iFw4E=L7#Jc^_yO>;Q#A}}-iy>^^1#hhdKyCT zIsO*?<~t{z;LRU8!Pi381BVV778zhSGDG>{>O<|&ADl@NR#G0A-!G3asw4RqhX)gE zEz@%(E*P|<#DrUO-en>IeGh~BA#V>*;2(e6Mo|=c znHkib#^+A3)ib}6b9yn6DGU^7g}A;`go3V5MX7bhdw}FMNm^!&2x^S7l2SxABd9Qk z{(CSh>}-+fmDt0S!XnAYIuUUHAsa#yyf7m|XHs1NLZlJQIwW4C+Qm72I@bd9c+{5R zDyLCrnW3VUhxXhNfS2XR)KnWn1C>{b#sKxHxVjcWtdtUcm<%Wst%%moayP=U>PQRC z$j7EIJ{DQPXp2NGJdd)D684|~^GeC4u=0Re89;H(Tuj%Ax9-T1F_nqw7$#C^AB|4q4JrS377v zvEeRm0yMMWe@kVy)(Ce86v|vRxZMy!qN7YhndLz;0E-TU;ao+Tfx#K)5Mdf&?7fy2 z;<&HOFFy_)iL*L$BXi}gWzTx9EIUflQHCETfBv<6}L1mg^CQ^G?qJglq(2Wx6MLjX8VblGjhCF%@W>FCZ&}r|qPnOq%UPqw*5TkygL&?p z4~DKH@!Vm%T(2M36#{`FGV(ShLD)I^O31`iPL({`bBDRk^p#=cV`?dwlVxdHsx+fb zq&*`4Rz1DR4?q!x*ZVEtNk*o!a4n?q)q;yuT=7nxpwlFx^Q}RsGfyL*Q)AK2gGZ95 zjwC~!B&{oyd#7$rT6uZPD(s?S8+U5fIyHA){>O^W(5d{y87n3}*LblG?jZ^xFlml} zjiX{OsR-EBcl28tp<5+sO5p^V#K=bqMdpr|!;Pd=2a zfDmA`lR#1Ko1KTod!6-LXT{2$vQ@UKGTc)exnQcrDWovwZ^6dJ@bMWtTLvdQ6i}X< zrI=)HeGE!m8HN}L^hA<3q=;O9nLCmpsj4&6BsW3CG-MzWN2bV(>r5z_nN~8h0&X*% zq%jQ(ZL(bnCYi4iR%2_|Yur(lVp399;{GURyz?;RL+acuJGV)4llTbK6-}zFCiPI0 zTBS)6%xJ>eMk3k$lO|}~NNKyR^pE3tS*ICdw8My+2cA_G;dUGey{TGZdP4;y0v!tak05RZ!dd(0O7-dI=4g z#-$;DqM=A4a`sQkD3TPiDam6h4aKAzlEK4#H&GsELG?t!7+!oTvdK_D5==L}niv#M z@ygNT7>cniWN1n%$!&YAWf}Hl6@bMGMI~O>u|_#LJu%jCk=gSc zz|qO-xXg$cs=2VsFsLkKX_{D_o^{@Iq;SayzgFd?E1N$U(8*&e6J%+W*U@aJ_3@fR zkpiYlNbO|indD{2S9-{knOGQ)WRdB|YE1b`*96lsJ-2i4%56`>l&0xC!*gM0VBtb> zC4RK<+pY3p2$oDoWp<}$qYuG4Lpf74hkZPQ(`u5jr_PmhgbqDf-h){#EbBPLsx`52 zUF8(vBeI>zlPYmeXZd>K7!6E4Sq(HDwI47cA8s7eEP@<@#7v#jVtUYM)|H1s`S`Me z=gRl1$fByq-vY-lY!wfw{1bHKg(HHPPB=Z$d9*Td);f^4tmHHmdIAX?0C#~`TU`Lg zMPw`)4`elGW)ZNF(5d5;Oy1z36HFtIxmI~6|Io>XfMuNILkAwbXxoAtz~~7^n9c*x zgl1&^oIpY~9>@jO96nP3RLPYQCjcg~28U3Ai|^xBZlc*EvncQd@HvG%5JVW&AXC_s zV3$1M&_$k2-6M+O;w?M7GsyZ?@^h7}Un9@9@CzChgOjUd_9{8Kq8eliOAgCo)URPI ziHZfSx`HEvl^ggAym8Vci#eEn9GF@T87U@%cyYDkz-mN3kQ-vsnPOry7>Wop_>l*_ zga%Cv2O(F~C~SrYVkC%-0CWWGAvrOZdAm?=2F!3m$q03!yzP_O+$RkQag%J~xv7w#JY@4QBv8 z3OUFYDTyYG0v1sz~|J zCU>F9L;n8l3kP=gA(smIW>@bd?*`IW>LkwDXJk3lRT&r^(l+RL)@c@ zRTIl6*v7>>dgRdP(iJ?5 z%T=TiG$44K%v)BFHw7p|!2re8^{$FycgfpDh?1n3t0-91K+-FzCspSudK!H5)RVP@z(TfA0OO^@d-o;2QL6UsuSSebmtLz_O^HMKjk z7*LsCPjL!e46NL7n3iQ-7%Cw53WkPptev%^l1&XISqk7yZ0QwMM(k>Gv9aP-mH3_E>|$pTIIk)qpDcKBWI;Z$3PY@NiLDxts_A7_ zV4@U$Cf_bJ(8L!LpOnck@=10EX~DJ@;GhaWD7K?tKodV?Z8!ph8H zS?7YqXVQ`7BqJ3?o00Ur{5;J9=v^`aaOTTzfr0`vOvn-g*@eeip<{hN)JcS52*nFa-aLl7x zdCD!cZ*M)WaOA&$#&QBD=YN!xCPLc{{GE`e*1U$riDV~$hwc=`IyQ+xUIQt+DRfQBMM+5cpwu)eLqRC%5>%LL4jvZx%^Zl! zz8WrSOjAc~-ulbLb;{I{2zNK(+?|=`I>Q1!>C-!@DsTqAkbRF+nd%%n$RChY5&N| z_t&X%lszWFYNhKK!m))pla+pee=TG`$-DhHfWI!}SLuJZ?o}Rm#vJB!x=e+_53#g$ zQ3uw5+(1L6Rk5ik>pSZkFFGHrjIM=eg?*#qx=@BHgsKbK+(I^&QBl{S@<3G|Lg}S0 zj=Ue1Fyi(l7Pvrfmv}(IrpwzJH|T5|F5ylVdy1Wol!n3llQ&0fi8KZJGiHd?9cfYGP^n?%{8(uT<+3XWoWDF!wX79NWV=sXEbSN0y0 z&4tO8R2Vox6*j;MP1=89=Zp(v2AHY31ztXN zLrl#Wk{47#!EZby8IR|iBom7U{CNTJRj#X5<5i^^s1gCB0AxsSDv@0Ah+S#KjPou*NpiGx*;EAQ1-4cqiU54#^+vZ+<3)MjMiLnod3Zqm zJjfB1A4)#D$_}D{pqYn9r2wP2p~~1^DDM>B%PL}&hvGY**^z*>lvtE?9WsGMd{I*zQ6cl|no>bBJd$&9Dgy}5NrD(P z$X#L!wUKbRVqzN{Do=PAlC>mxv7lBn?=eeckq}j$JB7Dmx+r;KdG0Vqic=@!!pI&^ z?LVGSpURx`P{JD}+su^OlnuwIyfzlWfGZhSr!k~Uxy%G&uJ}3B8BpA7689wjsS>@% zL1RGt8xK`-SCza~B%sM)2kQjNpyw5l9e%zJKqhmh{WwUNfly>mB^SY72U!URrBcq6 z&IVbSM#R6dNM?nH^9WJOK&>)uZB}cnUnqz1TBD4IJu)*Qcc!m)X71Q*Pg%hiRcB|q zsi&msoizqKCBq`c!|UpzO0rSr-YLvzSEOnx(p42jAkaL?nozjT`;Q&^PdV7tv~;LKi8?{3@A%vJ;+7)2@qqX^AVf0x7XFW=aQATcoDhs+tO0xt01fzy=?Gn@{lEzouS6rC{sc%41pDKI_>?@8EAojf%gWL1K-iQe_=f9sG*|jWRh)8Kd7Avl1~8 zJ$BIvz3hagJvc;w0?WF-Mppa63`vDvNUJ1um{e%i8YCm)oK=usW$OiYAX7FW5!7BH9Rl&h$#7$^ zUIXLd1SG)PPsaGfP1WZ`nZZ#>>QZw~p%v7JB5GO#CoMa?$w@HTMT%-lfIVg0h&XM$ z*{K^P7GXpZjIct4z@_4p=INxyxXM-uUx4YKL`mkBWeI9LA|MlSh$BoKIb?K^y>MPE z>M4N`8D&S<=tR$m*g5s;Ey$f!20KdP;7iye>&ufCOdhB5WZNyP%>vB2;82VG_!M zT}r8u`DW3B78lf$rUpG4R6`|(BFYO}p&(wUr|wS$2djibDp3p*I>OhIf&@l?ML3Dn zrD175CG>X*{X-Z+59oqQ{09eA;aQU+6o-N_O(~A2i~*tKv?PW06cmqYDsK9=BAG6A zC^geu)(uKty zT4qEe>h-XOY0)%vRqq{-fm@Jf+yQ7n1E|z8LDQD!ZXzE~7)6W%Z)6>dj!|Kd@t27r z!}<*&Is;}gP|NB-bVBBdvPzWEqHGl#4~U#-prSZXyW0sl3?4dRiB9CA3(x3;E;7}( zP3S}tNl4Fa!HZ+nkr2dVPMs5akYyxhrz_U~Iu;=mMaV_r{Rd4VouZPGR3X`J#`Njb z*;blZ0#6C1U`ZsTl6k6nCN}^el$nnD9mC)uyw+Ft9V@yC|+VA-EOgzet}< zTtj(2GI0$+ITNnXPPENwi{i2%apH72F#@Ad4{4UuQ}(V>_fr0a>k?rE6XIb4M@&eB z3A7;jFqyO3GtydfI;zL4sAJW2>$iiF{JpOf-;1u(6G1A1}w|+lLb_v^T_-)ML8D= zw`QCd-7jijM8RlCFpUfVO&!9X8>mhr9n8|TF}Jc%kfs6{xu*AOf_a)ul^bn~&_GoM zp)UM(O`o7?M4LW-QfGS5X*h zk$D42frZ7$)|zBF3x`qMQy0+11jI$W5iSS_QczS^eOKNK*TY;5Dnc<**;D^)#2k$B zK42Psa6Y5TmZItzmC@0O>*%P&%q~FCm7p0LPm%hfR9R_MSp@~I7gG?VHkjzCqk!N@ zQ;2x2FlRCq;1nwYj-&~zA|f=RurP+8%>}Fz(lgSHG(tt$BKgSR*HbLyL5`di8*lPJ z^A30f5;9^yW4In-L_4vZMw^J%&tYzZ2cC5rN7xLIMhWilL5KlmCW$FBqo6T^6VGY9 zSQ-r-h}OV#ic=kBlw^jj6;-rIpEAgatx$acOKK&Svi>qzDbpilTq%ipHwu844qzn{ zt|6*}-xV!2Kr0p4$}))^Sf)&Aw3@1lnL7}VV<5uAkVK1JX3#K#`50(9VMIhAJd<_8 zMP<$@tk^ic14&{1PV|PiGH+!)fyfKY5I8;1sGds%$F*emK!QMUA5WxYlaprBYHCtW z#v$*3*Ti-wO>ia*?3B?lLTw^r=0v*JNZgoVhyeHQ$s}JBDgGwnB`A6}rojYvHWANa z!p&harOO0sM)%8!v|$sWFeZ{NOfY2=Zm)=^ibI9hf8$?P13p8W^Fb4k*682gh|hGR zM?dX0hTxGxc!fwzl6PSv_M(gk#8r3#M4|F}4Oc`4`A3y10}#PIKxVQMD==LB1O_-6 zUTW20ic4_Kay=}fcR)onrrPxV-nMo-|X4O2(vLp$l z^d~J&fP2{Wzv30}RgLg6&KLtf^R(Vo{8cQ9}j_h7dFta;En& zS>z~@v;@lB_12SxD~)1c6vT{nsWbXbC+CelSVCvzZSsw4b=o#{I`J^8f>L@ta2XF{3k?Q9?HQ53ibqWt$F_H`Rdg)LFKwbzXeHn~{c)Mi$Xv z9c?rxU}TdGIYuz9M0+2jA%W2X!BRN7Fs_dV%MznV(0Bp8Wlu&F(rdFN+NwzEmo2!_ zrjhWQv$%9cTr7!LLxR{#3qdr=1*RmOi0WX{3Yzo%D3}|iZKHf|nO&t^ZZaP;I#?J( zA{nzW8$*j&30=aaL?zBdC+BUyFBBA!IrL?oJ6dyn${bihIvxtrk$EbpNYIj^amkop z(2@*i;A3O(W8X2LnT{1kGCZ(|!j;QV2I1EuLOQd+8`Hn9G89tf4O;`iGS!svj7dTj z#^jP%U2&UenIvm8Q|={zRxYs0RnTOngDwri(Kgf&gHG9`*TJL-!a5}Y)d)a!qS;;d z$V)0GFKvb0<9X04xTod#3r@h8E6s zmX&!M)vL;tKzCGK)-@?TFaHR+4rGb*z;M{CN3-mN3#HnzCIvdVqA*}{#^bH%@lk*c zP4=Ryb-hE!DSJoyd)5T-HW5qwjhri+9j<_JT>w zGs9e7?_5+)o>}WC|8fqU%tms!aAk2OS)5juc}N!eki~*znU7@ogyqP$vf>~T{26Tv zy15AzYpT_RhBcvJJAp_Oy47g6h@;S~AuwtPVhEQLY+AHt4CEnqSs01i$)ea<6dgyF z>x7_an?=xx?#}Db2uKZGWYMOBK&wGxzs>`!$QV8_Ghkrm@@4olVplUTtHBF$?KerI zbi_}>s*=o#DB=>^BBu3VelvaQqO8b8`rM)^SLEzfWYviKJwUIl0%|C!K;%)Ix~?2q zuCz!@fDH!(-B1`TS8^@)s36oV3r)*H*Gf|ZN~-833J8WPg4vX^tq=iIsKX))Cqi37 zwggux3Zkp%32Mm}7;qgf6EGvLMU;!6P9%#-EHHTw34_jAX9l>Fk z*>>naWRFH(M*A+Kv6sNqTBbeS zLkBp>mJD(nLo|EP0(?-q91MRr=yy68g?Z2@d@bimbk1qEg%_Y{InIo&QEVtYOPv?# z_~de|GChi3Q^fq>w^gy2(VB|DG-{A>O5})AQtZxwY%>G7Xkhmaly3mWTlRm0uoDR4 zAV@IawlomJNiH-LEC3=ZBPd;GjD}aPfQB?w3u=lR~fCVjFDhX zhF7$J0#2ikKKbo%m6)Eayb2!WG?oGeLt3|hx(vLe5;dU()C4Hj6&%x^ibjYt388XUv3y&RRy3DUA|iKrkLI#;0)5kn#@zTo(YBcPvas zb0FZ=ms~UvopfPqg*oH_w=o-X=kKmytP@!9L>!o$$PmB|jhzX=bumaAgEW?|X1UB*e zwCx6%!e{XLO|9`BNGK$7mdSh@-Yl$Fp*l7|)`*mzOOaV(QTnYX(o((}kLxq$Uy6eCe zYJTphpQ-X|Q3nqcDe;Gg<&p%$Dx6IJxAMG9wwM(WwT(Xp4^0&W$aVFZz3Y1Gs@GK~ zd2_Jc%7+i$D_yUpq2SF&CtTA6coF`>h;+U>vqC3n`5~NueR;TGFt34f1^U%>99KO{ z@(*0h1IKm3tK%)O|rs7SK~CUPb!^ zJfW$ep3wDV0ZI&wpltMbpc*jYPbScek*moAvSFcU!tYErr=|d%^sTp~I1%aE^?Ow0 z13HnYL40b+rfL$ynnbi(4LTgrdX-asgBMs0O?3%lwHA91&JJu#23ib7jxs{U1|QH^ zrL)T9%`TJ+?_apsjWm+5n0(D5F}Px|DK=Xb;zf*0f!sO|55;k7M726GuFeHA2u=-x zQi|2tSmdFQtb;I2XZ=Q~tLq|bC<-aTDkxTjgjL1`PeI%gC&&GlHQP>A>47Sm0_vO4O+X;jTgW zYY6-znnpS^Nmmy*Hi)2&smnalKT!-YB5a7n6`@>mL3_}_N?bFfUrj*yu@R<`ST9d8 zHW&@KS_BuJNVU$O896Fd9rb?~N=DSoMaBu0MWj6Pq~V7)c~KR&lbJBMCSa~c6R|Ob zPDg!{yXeE@p0-JC7$X9>3bZzBNot9CrtzB6MDjXOx+#>d3zds3&XTYj7@)A?1kkwag&~+|MkC$hFb`$fe z6PHssLPZ^Hw8c?OLI`dPxypo&hCnufa-5;g^$?gtcCkS;ideOafU8`dI~HnWQgo&* z@ICla(GMYM7O0spwaL02Us|~(l;o$fJ>WT-&Z}mN_Stadp|O4S*hSSkTPlLe8XsQt zXw43V0j(_5^%g8TJi4o)n0k6qB|f(EZdLzd*kYiZjWsr{(VcpN2GKN5f_n_qY8mA2 z%waSu%2mjsVvw$*-1ji@(pP2%_=leX37_r}%L7IU>?2Fnx*$3QaUp+^o8ZwV;Ij!k zYr@i+EZ7<~qd*C1-u0m2Sf|x^?!sfGEy+@=WQFv~axu?ZOhENyKxT5xW1qDmGt5hX z#bsW#`MhGT3rg6D!~;(i*lmgf6`s0iCxLKu(ho#{3Dvnokc<7-cCc|dZ4(IVUIg$Z zu@ZsDl{kvU+G*%zw6^~jKF3rZCg?_hQ&9qfA~)tD%ox`N0A?YC_#(IAwWyRl)zvCHRj9Hd&Wb%Hr(rBuN)aeXiQIL~^VeC01 zvP3gtVJ15>=OKyv(&*_duJSUXcNOwck!Q7ma75U?f`Hi})FEVvP&V3}G9FkVV^+vm z$s>YCwxyu;ihA8oJ4|#|XAt599+Bq`vb3gNLf3Fom47U9&*V zlzQ6q6y`LNrJXVX(1b2=9aHupZvq|+2;R_96?U$L$)g8rUxe^g0qi^!9+0A3;BIvR zd(PdyiVLWjU=J=J;!>Ot;mCud%Ru`C8N zm{m6=2OEHX*A@DE8WTI=lF?L<6Kbq-1LeZ%}uEOB5w4XDAt_>1A z+?%SXqLkW;4!mg6c%-K^Fw(rDV&Tn_X;~;(kr7^L!RNHZgY8uUemOP%IOWYO^EyUK zqU{~*${3k!^24#Bt`g^Cge72k{M1v1Sd<7RX(J5s7>0lWUrk<-PR~ZKPg!qIGR%`4 z)&v=wyvd^Si_DE>bYR6osL+iS8UA1Z+Z(|21~Ud&BiuK6R(Xbt2*xrQ(rnVFdopVH zq#f`C4V}omI!XBe6EcEVAtVUQuQ5rwEab?W8?Fr);=zHHW*(FLtpejsN|DM{L5X5= z@H)Yv3DY9fLOK37TEL4iEL~X$A-}`EC(&sp(NQ&V_!Gre9>ptf0FEG^yyX)2?xbBe z3G7gYJZOWtF;7;XEcEECJovE5DsuTY0f0?CNopwxG#mmA!?Moi(K92SgQKxiO`WSm z09QcBga{vr7C8u(D)>rhmJ4Y!;aDz6k6tqChVU{IerH0}sFo5vP1=>wP*{QmzY$>J zj0RP4>i*ON0xuM*_wa2MSJgG0*P?U1iqJX>WUdR6Gr@_pGNe9&=fMdHKb$&_=&{g1 z6Z#43#Lnr{2^n^^+BMoTomj`o`wF)uoFavvnf5l5@FQz6hWjviSuH@Ta%s76EbZ-- zx%$vO9(XJB9z^yZ$13qmMY8}o0?UrHtMLnp&dZj32n2wP%CBP+@a7LgV$OYN*6nT)g{KtL921V#fO zAbsWeR0EZ8Ah56!3Ewt@Q38Y`k!qx0orj0w!gfM#%FV?gB^JnI*Reczeh#E+Hj*CF z6(U%|p=9F4*JLOf1$-THY-9$?^=&1QkbJqQ7*MBii2H<_9vq*k@e#&rQ{$%zgY%|x zI`2bpQWuOAu7&bOq_IdN z92%uQ7lZBM?pKz1ZmW=_oX~(%4qu0aL&#rE$8I`O*yrkyt;=%FEF_^ovjWv>JT}cDS(h=o5h$WAbb(Sc z3J4#Ws#tk+*`a}yc~C@GEMu1)vx}&^Xtj%XckLE9pV0%gCnLKq zxuWdB719IdavT+GCwM96WR^Rlg&(5)0Tl#GCJ8csVF99I0f`1;B4~*YYS5(#HWE@r zK*HsNr@4Y-oU5Tx)j;4W!kLI{xLo60u5r$NI=hezmlGm@RRKOj$EGI=?V@#;3kq?$ z%(>hgJ-ib&&9e(^afwkbb0=mUV7a7ndIc9g#& zHd7^Sa^e4V{#HBz7mQilq}V?4fC@j#8DWx*~@JqX=`47Hfau^pd0UZDi2BM7PKi83C; zEXVc0b#z}=VOU2z9%F2~TulM|FtRqCMge;)R*r}eleJUNoS3aL zQ(;*c9IO~oB3$f1cp5Oij=vSjy;r2=0T^XFWx9>UE+1*YD z*iLJ%PJTupF|H=oXXCWfia1QN-BN;`?O`M=r0Trs7{*L8!(tRB;`Ibo!XQcpR$HwR zSV#@*Luib!8-^rKVdNuvlEnOnau_a!x({@~B;1Qjfmu|OofHbX;tz=vO9G1%5>Nb~ zG!1{zb`ul2J@M*O(NtWJ2VDZW4<*^wZV$s6k)pZlBw9fvu?C6mN=bAf$-O;M|A1!; z=5QHF04{0pBy0*&U6q`&K4qxlGMpk)kf4G~Fz>~KaLDWouqE*$iLB6is(LEK#Q98j*rbIo||jP|&w9Owz_Bn{rbE2c8oqP_Zr% zR(Qc<^6vKW7Z|crPY1|Qb(1fzu>#noKgmq0sr44F?(`Xok|hq)_Jn z<%8AHFkbU`hA^CY$l}Ji#d50Og?EH~QPr_G;TlZ$AQ<<3!#9VNt>LP|0Zgiit#PO3OBG^J`HG<|O|FOmNP z79vx}DpW=Ocsr!3M?u4p8rUwJh475Dx{oXnr9O8s>Ort-(36h|&y6XbY|w#mh>cit zs|t;&Lr2!)tet|R#I8$oO&7ZGd4pUp*4m}N#c}FLc*W%_=km3q%g$;*PI#UIf03Dw z5{(mnF(DTw_{F3j4@@aBZ@G&$!tf{QeCnLYgexuW!xYWHb>QF8skG(zG~YmpPA@xId!0kPxGa ziZ8+~mB&ojA$J{xPvjz8A`7UCvPP$Bcv7EJmW!ZqOqwG(70}!kxYaR-;{#1-}B{3psml+-`X95V& z{G(T_l2xQMb(&>!Jizn-^-mTQusY1r8fONWTo2A{yy#s`kK)D7ag?@`Qqv@S@F1U4 zpJDRn9+Zv8P<+ZdhWp|y3+|?eR!WEbm3~ha5bX9P8R(QUw{9IXHt66EbV5B4WIwD;|4uC^0qeHe< zO8kX$l^ZP@h?ZHfQRM0bN(fsdxuj7`OaO)V8MdJ$1+-3z{FG!HBj#FCBaTfRkwBJ* z?7-y7mI0C1e4-Dg_y#)7n*q%;^6B*Bu8PVj29Jj$^zxlRlz%Y6tS61z$Q{> z@l4iy1Z*-pkXb(qW+{{^#4=GyCr~|6EAmhTC`FEvK~o`iBXbi4g(}2(OgPWuiY`Bd z@JyYjX8@O`aQ?LV5nM8>;1ZeKU{WkV#sr7R|>dXM!1^yI)Jqq&X^5k=c3wbD0wnB2D z7L6<`cZ5p6mdYt!Ig2Z%ILi_MrQMM~W%ezTX~hQs@RYGsYF2ids!XRis*;(Gs+O9L zalNPkiyP5!G$dFFY2W16T;=av1q;-|BviaWcxf@^wK4~`NU;p;wQiVy=WBp1|h zaQZAG$rX~(L9@Ls5$bZ_T1=Jgikv_@rrBt(WQpeZ45yG>`MMOx7q_rC&k|<)tC}+s zJOjbgAJOP!=yWnPdKns_FJpwN2lBRYlR-pyAWMwO1yCvTio&8ZDmvZ%074bqa9qjE zw8krW?(Ay<7$@boN$G2nd7hN^a3IFi2|%@>iiDgPgUcA;%SPJnf(4%+2gXe|qxlcb zp5dWJU;RnX`pNta@*u2G?M;4OAW4mOJ9Pas8syl}8ShQT(n>Vyp|uWz?%|YIX*%RG zzHu4jxJ+YQrZFy)7;d>qe8xJ?sL1Osz{zF2Qv)UH&m+K;B%vh7^R;kh934j*pC}bo zl6DL*lA@9`n6oQvwDfO@n{i_J+lWn-8ol7cZZpxaQgNApHnG&qx;g~`5R92b1Ym%$ zP#o5sJXDr#tlGpJ?*9cMDdk0H;ZZJ#LH0kUU}~!kwO>?~R|m^<0z;iW>*>13?DMH} z{dH+BQy0J{g;eSq@=jDefmkH?`Ye11f02_AHY0(jG#vvdk;|~_EnE(in%CU4QBLJ+ zly|kqbFm&mrRY0`>?x|=RlFxv7j82JOAYZX&2`_@KsQ~8mg6W;sp|oKM$<2-7fs{mhBSnasJc#rp2`qwH!3^>*V6*YMyHUVp5s!R-oNnsTsyt`~|jmi6L&_roG72T*2x=^m; zMtny8Tn}YFE<$An8*t%0?9bGjPCa)D@|wa%OoBO%f^ihl+Z3le?37(jST1KRnJg&f zttXi-B-u2GR=HB?dXmk9Qs1(HtQ)eDX@qrTs3QB6FjpC_BTZc%rr-g9&^LIa=rOL$ z>@Im2=Q_%AO8wW|KtWY3ab4$%=Isi=?ef7Q>g<|yxFPt+1p1%>1#)o(wsS?EL&)v{ zR3{7E;Q^J}QxTXm&%%1C-~jo=8(q~H4r3MN%>bM$@2P%P#l)h736-cxo>uhrB5-AL zcq2yC0}G;!iqeEdq?Abitk_31g%S6>j*7ji4<`27#a81eWC&GG%}4Wq@KxilMT!xD~VYqLcyr-%KMQs ztu&ZccAMxio)$nTK~01h7=c2LXQ0c+j8se(B!rK}@|%n{wlPd6 zTBQip%vqW{;jS(V6{avdOI&fv<+vNLGIO5XQq4LGh>oe&{+4A7nKPw4@|m(A-FY zvV@qCfq~>YYV)XS8-$PmV}%0$t~F8Om(WbXCSG4CB-oO$jaHbck0dDAp1~zblCQ}-WA}WldSdk&DV+!jS zaTEuDU_wb|abPvb2?)?2jzO~%yZ$f2H45<$2D+=XuC6ppt@I_WgzzeX`AS5-62hku ziy)00N1+YUAz-D}tVe0PuiC8vPh`OoktAV=7EPiJ*luv_LLmfG)D-Z9@QcC%Rgk0$ zcF6f6yOldUk4^YQn8aq`5Rsy98&*XH;TGX(=2ZU*yf}jp!aY)ja0SVJ516-B( z8^XBkGAxAXZ6TrSwMvL@QE@y&6@z0@g(Bg7(xRIiA+gsP}2S= z^cM|GlSEC1ZsH3tZIMa&FIQSK$%t~ng~y0XoDCloxnj8NBN*yczPY9p#3tWeG?3#4 z(MNi=NKi0o08OOGM2bv;0>oRTg}UM`ONdl-TvMtx}yPREt#jc|gPm}R<#RH)-bmgNaHvxe`a7JXJX8JrQv5=eP5(_0BD*b`& zqohaqAqKgoJSKmIhr~9yF?d*ON_8JRjpXZkR0@-oMH{bH9@>=b7yoP7 zDpm+nskma7JNa)H`mhL#!g0{vLD}eVT{4_0WunP-)|GDwG87}tv%5ko8;e{emk9Ns9Ysb@+R`%|Qs=x~IWPe`KX)-ewD`Y0yi3h8`ED&go zE}TiP$PP$mv>X*h|KfiSVNxBeDqr>+goG>BS}Jtwag>}U4LjWzs$4kW(JG!rN_Uf< zp1vJ&RmoLq;3^*w`NXgRJ)uW|rUH@xrLQ;X`fbL8b^~EBpaRnDSB}2|$~MZ0Qj<4I zeT)Xnq>h<^$g(yF93^IUDdUUlm?c+I%)`f)>B*$8!-S51K4r!hIkX0R zn4^T&D8%jgd+ogWZ@C%2=wOTu6V0#~kyC_nirdPw1*EQO+c+k+kR;asqhBSpa*IJ!Kvht_8J7 z<_bY8gfGfg4;ua}B6bxgR28k{$FULKY19C`uJ7f+g=& z%3Cm2ST`#VlHP`7LRJACt|F%un|>?p}D zNrEdn>iY9i{}#qFsha-l*d))@IO_igt01-u1c_?5)`Wpx>!l%R>j7mjq+b*vyH7a$$y30bSzfL&O;u>RX`GCJF@kR zIIQdmRj8GmG}ckc>j0NjQpJ^aiU+|*+wGB10xOmUtn(>>N;<~}Ww$0< z{U()R<4(Gc(yPuQ&7$NYqZ+Y4cOqSXMqEbxt@s;hD5wT>x!^yNPAT=k1X9%GF5aH0hSx>aApf5!Yh>=059N5BWbsGZc!)F3kt=8nLkc@H zzq@qkOdn=)O>)LQa(xcyb*5PPuHzt&Pb+d+6vBHriJ2$QK8b08FHB!6mF4_v@)LHM z%(|>mk!C}FDOo8YbCtG&HbR*ZJ2Ls3OpKjFlpxWvuFJM<+qP}n=(1g1wr$(CZQHi( zzID#K!#jARTp1C$_sU6TM6CUPkz;Y+mhJsG=Rei-+XszdPZXR!vZZdIF2Rn*e{I3G;4;aM7oofg}WbE;2i17=h{c+YnCI38Pa>}ba3pw%5_g)8mNLIED-RJsM6w!+lm!g>E%NrrOQU9rc#jA)FS z-vXXu_JYdGpqM{xAx9llzJ>|821W-CEnUaH2gVN_ztwFSSegB9t^3T*viy`3p(_CS`~L(L)2`K+2`T9wErStN$oidjx4 znO6>Vy8>yyQp7u}(!W21++&__r6tku`bd)8V?j>Oc`EXexW1xJR%AHeRYj3?43t`$ zLe?S_B9NtN&xSF0JVGc@XrKvWaSE6o4UL9oJidTdP;}=5kal^m2zzR{;e+)oc>$F- zhHg-7)aa<3R4)>^aDuT8TWAj!Y$VW-^iAgs6{}3X@VGDO5`%Ktgl|sjw*lF(Ty#51 zXz*9D$y_8FRGO-GK6KJ%2{mdhc)GKTh<8}As@x?zak0n-j*4WPVD>@@Xih<4WnEHj z%}@tC75qnl|JX;Y(zKAWab9K@qtr$B2=H>t!oZ@*pH1rkn>9p?Zt^S)IqFu6c}R`R zL+PtpL|Y3dGe&f@Dns_Gx&R}cs2TCeHsZDh#{-1adJG0`pqIg6c+Qn`q`ghjF$(B4 z%ev_C3emrUyIrP3nRA}kJWSgh!`b|0e3&{xk|c)u^N)InJb4bjoFBJiJbN^1Cj^

NL!t~$|7lP59;7WCDrqSD z7|c!Avk@w#k(p>D{EIzPC7SXk?MqZBp-EyDHf?+wIpgHppM=^h5ChC0L);;ngjf#* z%R~{iBAb5>)&^{2jxtPmx}FnqEins0N(DyTU_cnrCk*$jqFP>CMp)G4U<1B2@f34xyw zPH}6fvkgehkYa$)QM0FbXFTxR#i)Se)tQ!(Y2GuQ7l-&{aSuD%s^oZTk#DVxl(iQX z83@x!$p0pfbM!NrXJ3%jnmuSo2qdcp!KlM=$M$1o=7&3;W+0051M`IC3%86+#6z)ORLbSYMwDHl)Kq)Ua zKqhk`iD~o}G%d&EH{@VR*^SXI#?o-#l=zH&aN{hdoGdlo4)5p~SY~3*V=!`Svn0ZF zoW!V)+BYnOj#ht?*!5^qv?sZlHKf5S05+ZsNxmd|Tx8g5wmnORi-JU5l?2)ep!h?#)u{pAS|8GgL zE2SzijNC{qGcC7ou8AX>nh=sldT4=&wd`ub@1I0`e7e%dsO5p^dr;%Zkp6+@lRO$; z=JC*W4kCMiO5b>mNC;p9cGy%rC-pd+iUYI}YuogqhPYv7sosB5LL{8So*FkI_9hTi zagtjP>lNWm$HV+b0e4ZwI`Y>uh!uK_W5l|NG4LoQP%RZ(J;ZI_IT>XwI1Q7IG8lUD z?P!|}c5FCO+k?A4IT~i8bt9@kc|jbwGkXukS&XVPDE!(S6hq4S4gwU+n2bsl0}ku< za>G?KJ?EkdMa7i}`vk(bAnodcRkh-JT_dwYKH1~Fr5fF4T4~8K+4zkUDd<;HYGv(G zgGVSK{;~pSVkydP6DiHM)`?r?KU^MvC%zh>03ku2vo^AIPq(TEDOEQ0SLxn@j8EP5 zZnB1?v836`(J=KSYI34Gy@Z)UCQy~igUT3Ms<&i_daz`vnkARlz=(r%mR&Up{9K#S z7(&?0%0I&7T`;c-^I{f|GoE}v+PGQ=sMc(7&94e+JIcrBQP-J14g`{hMPIb$q{^smq5+hDm$fkL*4%WEr5oNce z%KWsk1xe=HCo~AoiW>~&st0f$g=||MBlu}RUPe{eDW~Zan#Vf;GhmHl3+fd@zijMf zsAvm8*z4s^h~e8t%FMFmMwFKk^fX9dAT5HwVrt2tHkd82+?COiT1i9l<6+wI9{CC* zQ&WZN=0MYFqtTVi*eI2hSlU2nM#SX%__Md0Q)m=Kr7~TnmW_3jy-Hc9EHs{IR%BsSc&h~QA{%lIz##*d_?qp z4e$mE?G~kx7wLw1FVf&XBYc4Ee+u4$dvc(9?xT{{?mJS*m~=yU5=Pf-VL(x{Eu$yD zB6O032ZaIUeIdSfEO5{c&>TQ;cF`U<0h!L}17xhM8cEy*qP!&{F zY8EA@W+TN`t+Y}IV@Y+~BjD+X5J}=>7d6L4m@RAyJ(RcMDU7Rjbf^MaRSp9A%4*QF z#E1qh1qTQf@tfK!^0ZRW~u{X=<`cZfwK zl*#9kDQlo43k$~)z{*?bkqYHgvcHlklC$<-^cu01q z2_l;kCUrvw>K?>mauPt)r?CQjh5akH&LWOm~FS_ zB2TJ|nmCqxm*E*tbfHU4Y_jx`lu?lghNxHANJb|y3|h}qmXM^Xj`|ZC;i@3dNvTFx zD&_M-#F;Eo99|+K2Om%HkUS?rDZOMc#W0jV9R+Ko;nIEkv;zeha@;IIue88`4*Rfx z@HVDfL>n&)+;P3PpiUpcB`k|5tblODc5xVe&MwQ;@B|)3ZN@{T4}#bvAq#b!(pE)e z3+3B*PAOkn0xH*YGT$~AS!98ip+QLJ49cNCb`D`I_h{X3sA-{|asyo9QDaA4zjOH4 z{~~(Mz38Co$bcy2sz|_E3X65z5e70hvC<(9I9;@X_L%P`Dr`@Znumsvt~S(uY${vN z@Cls!eXju{>r(eYaTV$y0E5eZ_jYTslY|g2RM$M76hN{%)|FW?(;*u?!NV!Fr$Q z-flRhbNdzE(iIb-266MX2k<6vn=`%3jC)OAORbefB%+6rToNQ+!^dlO^A4<$+eqYB z<{Ga59`FBSCx;Y}cw$q5p4f#WXIeMU5j~h?EN8>X16AavsXB%OD1aiw6ge%EwUNl| z>?kK6{+8McVKe($B&RC z7F3~`y@z;@pq4|5xtGnSge zVMTGNRvNyzC=BtC@BYw{G-A1GsVm@x-;!oP-YP(#$Pl4oA;?Q&pFmBwlMqNe*?Y0J15fWVSz)bSlMjC$w1K2)VRaM2XeE#ELpOjZl^K9eIsKwODv zdTvacKpQu?>rpI@&2RAxT1W zx?ySp$+b!OaEukFq81?ct)ELAvymTpaXg8uBQJqWD2X+It|*xhLDb+5r$5>gj=uqE z-c5*@j9ZkH7Izj`)g6LoGrc@$j+0VGq)V1)7bJ{j@>nbMb?&H~(SwYix5ZU+*7%z-B3bPn?c~j^F5?PR6`r98ZROc1@w13Kl(Q zP-!Eh>a3d2tJ1+tZr)^O>)xflxLfkULBlI27m*A%7n5tiv#=Kb5ek`DO6CcsAN_CfjhE?yiAm3|R&6s?`%0Gl^pUDmFFDjGgnn3b1rAf+zs zOh!40QvOQf3!f#M*CU(WP*;&27g^+@iMEbKtWBm{Igxd(Xz~e$A!bFUv?R(bBfL1t zxIMMha+y}Tsv}(kWUg?lZP7U>_+D)bHb?$@xTn$(eDdvhFXh|RxAiIchW`}a4=rMx zaF5~POHkUSkz3dFBN97Qd?;p5hfJ`7IVKNztj>7~J^EevK0LtguQ_m^_1t!(I>Wzi zU%Ah*k;rqCADNv;Y)ntoYjEXmE4rO+6b>f#8- z&uvz(>#V`6&YD6VpAXf>qVd-J2U*mc$-NI5q~F#w7QV&)hz*8WyqB*0w5`gf?_DWL zD<262IAM<5QjyDBr_8%MSjbOYx36}qfG^p%d~_Xm8i^Oet(W)P#6`|4FTP%pRx#m9 z7o=F2fJf0CQmwv_;xgp^?H%E{X$IM;n@i~Qe|^j+Zd*0)8^*x6yK|*W4EI^ZCmH^5 zbj(kDCt+{zY_SO2(jMw2_ZF;=zFzCbE_=t*dR~pGcD8P-ap8u&7)$eW9|t@Z#!@TI z-V52}l*Yy;`ZkJ^JW4&lO*2cFb%wZjXzrC1&=AF43K|_oj|r%rxc0iHc~awM+6LKJ z9lK`AC6AKdd|S(z8Q#;)(G_jJzgs7!PS`ByGc#sW_dRyKa_|10QiKIF@-XBB=dVCK z>xbIHS{b8*%oL&BPAMM$_{q?|uIlD!J(M*19xEW7D{(>vM%m3QkITCA?X^^wqq`ei`p*wI$w z-ec@~cTU>hi=TU6hFBBu^Q=Z5`DRDT7j9f_aM-dgdc3UB=TS0lR3pnUxlWt)=0i^! z-559-!(Oqo4uxp}7c#?;6c2b>mG8VAgKn)_X{NZ!ErteDK7_Km%F6IhTKy%LPe51< zVDTI1+%|0u?(o?onI#T*%>BwyOG+#D(3UUW`oQ(}BBKq!d{yk&K|4HXx@<0;kp^ z1br`vI5(}L<`i(LjPsy-Bx62_-7@Hagh!4@wvwG{Tznqk>?b=kpeRTM`fIUebh?H+ zKsERBKJ0nXYCB3b+h6rY$Y(QTdlU)G*UDB)z6S39(1Ki&_;czy*t_B5wD#E6oLG66 zMi1rVQ8j|RJAcCna!X+_*$gk{F>zau-W1G5GT<{LCtDc_;D_lNV% z67Y}&v9ls!Sr30-jXkLHfZR$P;Q{uYfs}o_P<58|_B^D>mQ1TEw+|eh;`n)okuw}G zk$PA1PY0qTg&v$(`|eK1>t<_MUhXR*byf-2q<`I5(2eI!SN5zYH>|%h5i;;jX{tFS z!%CWs+{w-pvDD6b0a2Xxxo%DB1^%C1he_ZHdHLSS{)=Yxez4E41&1!gif5!xl`4_m z$USVxcn_5hWKBdb_zP4>4nAVv4cLr68@^5nw3V11iOG3pDs37(Qj8?HozaM^V~ik@ z=Iwr~Z53w5^vb%)G*+Di(%dT~nk@Z$Q{jz#w@D~rXW;K0o(<(XTS@ey9*1JOc8pc; zfb{k)4%8DMzN1-Z<;L}Ddl(Bjd=p-{yQ$ZO`(xvQl}!eoq{LI;Y%*5+i-gz%nZM4? zI@6;ZYt>IaLzl3`UR!fwtcym8Sf{0d<#*-rncpjQA6ugL6}eg?T6|r0-ItgE*f>M# zkMcJG_GMh$-7k}gjWOdtvVqu$3v^BMOiqPpI^m(XD*77svK#-DpG<*92QQuKeJ`PXnj~-xS zeaec{W4h1JM%rnSzB*hwS7H@7+^ihqv2PD5Al5WA2D%Pi@S8?iE->-EknZRh-d|4G zf)}D;6S%-~&)8eysXZ#ojn8KiJ9Xzi-=jqjhz(+8p!M0%Em2r;M3K0naJ~)tZYFg| z$eheO1nFBKZ3Mh6m7|?=5lo za`eo7Ii6ODs-x5&6u0on@27Cx%z1meAXRc5pWLdwIb%w5- zhR3#EECAa0^yDV1lN0A9kCy3ft)aRr)rjMX1`k(OHyeZhday7bJY-`xCLNuzL-f;T}{lrfKqQkELW<*hV;QIGlX1wCtGk25)?}}l=jFy{s<=Sxynsgm)UbEdc z-vV%UfAal~4|{nu;Cn7Zk+|u;_h`KiUr?6Od!MuGWQ&$PdF+BjEy5*3&xx#@1D~pi zzWx!63b`A_^OO>TZ{56xxA?{lpSV_gK?K}9Kqc+J2fa1j>_*2npPMocT6AP{K?_Bm z*X+!R3;jhnlo-45HLQm&)1Vj4@;SC^gJl8fZW!xIEayLY=XZj>N_A9DPKxin*T|JJ z-T;PKs-sL{N;J{XCQ08JAjx6j@$|G#_c;Wt*>KfYd#9$MAuH^9m&V&|_P}%m;KnOr z8u#R2FYBTv*vXhb1wQHJ3(m9;KNFeF+U9;8GS;C2F2%0fR!=KW4^fXz)bjiVjb z)Z)1%;-5Ho-K4@p5q9!#PwF13Ph^9SMDghP2&~@4*Fyvm>RK%KhoqaV^%*$YSC?{` z+0O9VEGtbh{9;!y{E=bUl;UTv>Gv@lw{r6{5*hV7-hf71d0eN0I4g%K$uH0(HV?PVel$?D}4pAOPAgQM} zIh8xJP7n2TReZ5id@OR>QRVT@*eF{COEq>4gx_|q2OG7!bE(B1LV15W5WcKPxdQf# z=_zE}>Ua;fibavJXPV#pgD7LUXBH;52g-Nj`B`ZftM2n1{bU*E`!0S(uo!X>abb&& z#D(>N%rEd z`2~;K4ktT2O-t`cw>pNrlj9uobN>!!#48HR|1g!oqPFtcbNm(AzH{1^J`3>_X&Q8| zc4cfp)V~n5qOHQ4@i^sf@!(FK9jo!S78hGK;C>zU5^qr;c-g!;Y-c0*U?J2v{^XzA zVjjk=ifE)GKLY$SnQp=85{4kNRb8hKx{COi`Cu#R;5hj_4&h-r9B^zE7pb>u3AV17 z9K%@6mQ9F6wC8;vvYLB~Zt_AuTtkFCT$)}WO6h}U z%)$7vpg$k`j~{wrTlWZDO%SFypR^X^GOS2ek?6|=!}Ax08XdDRJ4O^02Q43>3tWTw zQycv|cyu`mFVHdg8m50~oQJV@I4py{6j8&#G_K^6Ot;yf5{;-hh9oU$uLGLH|*|>G%JG z`uRUi^b~11`u{#_`~K;Jstx^A*Y)2@Eic@6;`iboa-@&iNuP#it4=+$Ox^>B1Le7E z^6|7aSMV6@rKZ040bVM)?_sf!bc|WRw81d2@}?V4zLNx*E&O7u5Nj0MlGUo1k}r1g z$niPkI6&s6*HVb$|INMMvNlDYEUzBl{1fRYDYh2;0nyRV!LAI2;4FTaU*~bv#)yTK zP8+Zq6C24T4f%GHNwlcUt4;mz$gUz8D#lw)C7tL&+@_nqW-45sz$-Q9J(YfbB*7=A zp4)e*6uZVK`1rnk*Z2RI4^)faHeyndC1Zt1V@g9Q6iGT~w9z69{{ws5=(Wg@JDKqh zwh?ow!XKignC9C)<52%3}V!CYhpgc0=9sjS{6ET z=GmN;t)6#kd1wp8;a^5N?SQ7*klTYpCHlXJ^M^$s=9ST?bq3?ikX*3<|Jzv2SL~eT zDkKABfyN~fUMiR`hI4z~+uhjWFOy|c7l#MvYu6S}Iy{2D5+8{J|{qRdG@<2|%BxiPfSk;5$ ze4Fn*w&@qAvjW3XuU4;0mTWX?XZ)~H0TA}kbkD`fK<&-mc4whJc14CRqw=)wY+k>8 z9;x1uZ6hdcn{E3zCk1F_waxBIw-fzMi2MJs%hi6fY+gm$`~)jzerM7Si`S>T)bjr3 zolk}-wZ(j2U=siuO1K!JWp8b?2UKIZQ(MK3d{0;uy%O!lOsM%cU< z{Rsgr+3QH0TSJ@-k~C(;xWP(D7Sk8RqA0TSSG5^vIBN=qJP9J!_kiQ6y?7CY4UGTp z)#J0b6*dikD0M*94BbVIoBL-9)~3ilIL1$Y%ATMl4);+&CcJntP_T8H2^zDZ)dftJ z*o-LSYPS$dfe+B_6I}_f)wXQCBU*=IwN9YtVd zM9f>`t}qs-jYz3W-?%LG>aq>68q6@eMCYdh8ycnKKK%R~0mRwtV=@;Zn&#Ca`WMrb zUb*~Spot8j=)P&2f2(g0kXy9jg)!l~g7$=agrQ3dexuk1WAQ>NL1ppowo0Q!n9>agul6U&-<)=&Eyp zS^hu~IHj69>3_Rv&@c{1+(@GD!P?8FPZHmM&<~)>dsY8eQ#sPaSksG#)l}VpoED8Y z#??CmRbDJxPv36$__Hd$pBsK?cog+0Fi_w?*zQ)@hBokt+{lK;eeB>}!uxDYJXke( z{eyr91PX#PzV@zA=!P+BvgAAbRf`~ggF4NlGTz9u{(zC}Q+=GVfV4+$omf3Xto*5s z{XiTBEun))P@akmfY03YGrn1n^iX}+9Ei2*z77=%#Q05vZS9~x?$n^OKa(knt-Abq zKBW*)%q|$!9(KC{O%A?Br>0}*~kz9VmhbCn?E6|7&aratVzI+2b!jt z<-xBmUsS3b^fj&PUS;v?I0_p3bpxDY9 zFiyBYYP>(NFm*SxD7!i$C5kC?XZ{n;(L*G~ zX5GA1jk?9J#-}^or+VoHT%s|dIQ1yXkUNU<*5Gsn%ttOSFnFyuS~jZ}o$_H*sHu{| zuvxV={znEBX@D@$5Oe|_=#-zUCw^)93hK@IfrqNbJ?#mj&&V}WUl=!;H65Qh@OPVH zx-8cH^F5uwTDieB_#`EI0r-#8{RAOFdpNyqX_uvu6oyJF zo5+3atVS8dxG`}Gm<-#HH`<>u{-!1#4RsBA}=d2;UU z-49T~d~F;$M`Hn_DM(f*^&p~{I(g%y0t2%Sbtev7R|Ct$qsfma>Ma#Y#FVM2%~%); zjy#o+E30f%2&Q5zL`f*`*@}niYR5`C&~B9uboac)F^?2EbPPDMldZgOu?nAIqcKWE zD1_!CpXXt;6hC%rojcQTCCu0QMY(a6nGfa}xim%fq`WnTrT}554GLNR;(9k7&Pu6A zi*s)it+p!XXgMy{7+}ko5B3p(a@HWi)dm*7T+=|tsqT>F2P(gJ7u%Tsh1 zfXJr`v-530{l&r6h@yT#_4^@AVfRuvtBrtz@WD-kzf|%ch|nj|DHCMP=9JWp4?W|G zsEquZ{RWbKpF-CABQk&780$t)(ub8bHRpuz50N}=x9_gP=>o(!p>6054&IF6ltrNh z@MbIcoaWiY!|Gy3WVk}D(#VVPLRa;H!=;y52c;3WCQA)6(#Pgh)`M5Ue9mhdkwzL4 ze3>3NHAd5)g9z%fKV2b7ToTSsYrC6byuUh3WDqWU=}Ks zW|gn&7l1^3b$2o4*FOi$7=bn{62Kb+feIBiQ)kXkG{ivJ{{din15W_&0+vwpM<5Z? z+$+UE;m=NG-20YDm!O0uT)3YdG&mzDhD$w#%eL~(Ek=vJBytjNpTc~ozXxFs}6D0WHHW(V#}MZ0d!^9rL(fiZ$*fJU|hXn$yI!L`29 z@HR&$zpnivAS)$UI2hx9hCBL=pdcyn_e{mHXFC2M2T=z=rFzKE{c#Z!XgdBZ1LQTW zVA;b(lM$GMKL;?`<3*)@N6+(2AV}o2WiXv!olurH4i)n)@m_+@8ej$u_z06a?9~12$ zf^08$8w(Gj22t&*L%@j7=&gzENvcc^l=_u0>qqojq42@m2HjfYvunfD`iE7cuzrE^ z71*3r5-#zJdiyS@Q29y(M$8uHSCOlUvVC)u8H&Z$LmE$r-A43c0>Pbig6e`SWRjsrXy78cc1pXDKC!MNB5h_l1n_&w5s&L_>7WS` z$F}O!5h)!X4KL`q?5ehGh{ed#is2Bd2$q%H0ywLpdEg*7tpe2$*uEe}s}UNSF=cn3 zh^c-3mwxFkBuVXUicj5-ga!=dy-{LEWqATO6CR(0A#)xdmqYQ^$0+!>!{*7KFa8_4 zdTMZUwKc%HPf*-56kn(veP{zo1EPyc)~R!VwO0*l9L(m>-9YQo<*x@&>$^BCth}yD zFnI|B9H3}^T5IuOBA9S--{vonM(Tjb1$*%Jy#xO;Q0y z_OHg6kQ>1s(1`)eNdOL+>gC>{)7*kjh|zFmKx_Lkh&v>m1b5yQP047hST7HbNxe*s zx+W)_Px4Shym8V28`DlS_aox3UJ6Vf3j`rdoU5-mS!FO2Rwe%>;3mgOre*+{cnAj* zF)-W{GbF*2(8H<~r*E@X$3du*3=?=-#sO@TPX!BEQ8c0?xhnllK!vFg`cPsf^tpd* zIJ=OG1vp_XsbnP5L!;K9^qr>(6(Ru}NvgzcVk992c5oJz_XjC4y_C7lUY*J~dK9I~ z24Ocd%m;CG+2rX_G=m-C9!|m}%0aQoN+dZ@8a6i~GJ|dq$GrLqAW0Ez1YJs?A^@&m z8kC?JHW3Es++c{8b{Fx}3V_z1j@LOsXceRfu3=D;Njy`0xZb${2PzZ%;=)tYiVK&z zq}iU7QJPjfSbTE`7?&3uK|FZd;ASGJGmd4On+aqujhNMb^Nl-an3qdJSyxcP~0{zuxd5Dj)Agye%vU7V0n@xX& zz4sI$A*e&iU>aeZpT|{*e}mZ_ZseoY@VTlpwNmxi7DAwr>l&a}1YjHq_@#K#{IHK+$e zq=fIQ92ZRJe?M&tey)7z4JI6w=cLtfjDmAMk@LJ&cT6~%q4 z;JYc?=yomTJ&~qE>1Qe>lX%!cg%)L?WkPooHX~IA;qHJ;m@I}*5T~hy8Ll~GNdZAy z;ukh*J8L=hv~onoFm0Wh$Pz-7xzufZD06ctXH^90Oh`kCwE?E4i^QBn=wtGzLFl8~ zI3(%Z(8@MpM@|M`wjr!bl{R5iX;X`^tAO?cP;oQ#nHyP>{g=!^yhA`j#pkZnuY&3P zV;Baw1`?lCdO`uzNJ;d@1b;S0{x1AM6&Cn)*o~OOftaSrgk_;0YI_@@8#dCMaIZ1$ z7cyG&sNO8;)?D%?x^n3xo0hv&gDl?Y=Cy^loVXL!*~LC13joCzG`6tIwChUc+MFz?`@he z)!J#k@X(U-WewF<`dZ#&smMTT4HeQ#>StL_s#v=aAn#S2 z9$Il<(7#jnENK9(Dcb(^4{}i}W6F)|ElZz*fWqAa9aC1JWQkihUYVh5IujC>I!dE~ zsn*Q+lf|4z?(-np-^p8RxB2ZYGpiGARePll9%so_BBcqmwtP3gNIifz%%d4WwQOZt+cxs(<2y z1_mS{saqK?vVGj*1qNTHyy^y1K#)cV8XL5pC?;b+2>JzB-NHmpdP0lA4h@hwPU^{V zTqrLY8-ci9MkFkHX+c=~XdZjc4E_&bhirsqVQVFG@)o}cAj;rR-Harw z?BtXHo1%r}X;G$R7BK|5Ys|9sj&44d&2#a2X1uGJqzFk)y7C&_MLK1S?zG0e#(JM$Bf2RbGpLJ{fBB0+~9Sf-5pu#HOFBAPMyphbja$Xgq5i zFD&5_`r(Qqg06)fl4yev+}ij9YQ;RW%?gx8TQd(~qJv;!OVtbr62XFWb3puy8~`)I zIFtiDc@PX7C`rgrtC{HRwlxI_nu5L@VyJ{#PW;fApbv1pz0z)%X}i@2mTS0BbJ$Iy$Y-9;CQaWEKc8{|#U1{nE!bdFXOOcC0p7gs!_ z7;74b-y7471*+c0tjq z59l^jC|6^ff3?0}7tH2x5%uB)Zx1|6laCp9@+O)JLOWchNU;1I%OceR&{{=jKP2f! zS|&(9J>JMulfl;AmvBHQG7_3G#S~PaAO_x93y$OmterfPM=@)LHqaOvlB=e9GF=Z= zyRiwz2u&O*qej6HN@Fl>j6_EP|5k~x;IL4wSD$1`dtsNJ2towCN4Dj`DecH_A6435 z2z^PIB&g_LpqfxO$pD89mAdM%wx96#KM#0)wIDKK$3T}bN&fh5{;ePL)deNc`6~O0 zk|C5Y<3=xX>RRw{oBmd;t3t=tJYpVbjI~{ER)>(2Yp=jw&B|jXA%gc={nJ=Sji{44 zu^#BEi%3Z#jiRh(W;J}q3=zaMhMNMXij)Gi3?OYu7*fEfToOgR2@@2e%bln$62Pd; zB-L8b*oG#EeKfh)wx+u32{yC(>(@+nzn%T5*^fb@<$?+W^xQGl)Wdi+K!-wy_9S5a zEwx|&`bHYBT5k%P9>N++7>_6$5^Xr;Kb%DuUPYw}{Mc|Ay*GhO?Mk!0@3~5vxct5U zO2L7`?J`+Ui_7ddZZdScl6*32pv(YV0R&go6%N3Z!XZYpd)DL7#tQWtDU__<(7fA;Bx3ZoG65t5(D^4w;Ok+R|nheb_j@5^{O*0 zS%!9Ih##dYKV3tglD_~U?>r?14-}(g2XJ4#UjL&KDB3M!ak6La6eUwkOO%Q{T#fZ; zv9KU%Wjuj)pRt*8$V@t>XCE+lL07bL*e+VzQ7u|!t#sTZg2L-Cp$DB8ptLK+yQ>3| z=m|jG8WU$?%a_5TZ0QIif~?I^lGvay?{tonh{Y?|3s)hjs*$CLinf)YVkpynsN;=J z$(}HSHISsnlncIrnY1mzqyAC@RvwAXLYnBv0$G2#2f1+*IaFiiNx==X2rqoWHPT(W z_SYcO;&js$eLpB^?(tfWGgknKI(=KhxH7n{#hlKTNj2IU^pmG2&40!bJASM9HQm{} zLYvZ%IQLds>ymZ~sxk}^Az2VeIov353+jwzZu8l&qySKb;W#eetg2ApPZJszV2{(x z)OIFZd!=onGw^b@;ot2iKR!w~Tzw$t)v&}V51tK4SMq+FFx{pY`lsX$t^>;5nFH)( zxqNA9cLpN?8M2H1ny#*M6CVPna6a|ABbQ3)ExC<@f(|;$Fq*O|l&u2#rcUnErUkJR z14K!eVX=0Hp1G@jNmo)!j&Zsoi*eoh37G6QHe>Sj>N$otQ5Qz0q7GS)vays9+9Ib8 z2)nz_xwtDctq;2fX_#Fj-Epk?h$v}HwcPlb#mDv<%jxDutkbcwaYoX}49kq01I0vy zeoT3qC#JGAyQFmiPsFFe;bN$3`cb$&(^rsflERKPnZgm7L(Q^X@?eXmy1YlxWF{Gy z=og2di?1J-LoZ)My{;?>yLd#kd5$oJKQ9DPKu#w{$F%H~8K!35WSQ0?W{}7?oOflj zEsR%a>}ZdrAr_HkU2~CMX|t}HdPQY`W`J%c!Hsxuz#B5>mAu55PZP-^55Qt4>jT+! zyr@A%{HQ&~L$Q#mxIC>H-VFz-+Fy5X{iWl7w(dJT&Zh4=ni%be>>0Rt`>X*9B*h`7 z#i-Z~tT$g$G7{yAJ|hGxAgn93f#Lo#zRmeI%dR>`8%3H=(ekZ{M1@?RBpq(2E{&=C zb_Z>l%`K~QZG}vaMa0kLAl*wv@d+$8@SEw*BVkc4N$IL;d2a+5B1pF|z)t~amZv^D zU#WuvqObrn8BypUmw9$Q6+~PGpb#}*NrGz+wSaD8DIwtc&s%--X^*7k5Q_1U{P14~Zy)8SD!xydS@O@!DQPA-|th7HaY(G`f4Xair= zxo7bSkymPlm9Rlj(rh1R+HP)gZG?t7-z^n5FH_7OlA;4DJl>LdDybB8j z>ARSLs8tcW&13g9O0sX5an-wVIbQ_m=JQpcq$M+ID3rZl6^gX*( ztr&v_kXY6hWc7AIA2hNS{%UdylnA)8NKrv;Wk0NW*wO1ate~s1A}YnJZWggD8KU>s z^UQjgDnGGNCN|sTnGDaK2F6jfcE!iph9?%Wq-`PXO8ywGZm=vY0O(YKQ0C7ga5bJt z6S=KGd^kN^Cly+j8O()Eun#o3P?u(ENqD+<*IE`OHh3Ot6uN9_(Z2n z(b*_ejgt!|`D2s1-PtN}0%6BES1H0|M3N!bNVTbw$b>OYMu`SOyC9aPk*aYZ7bvU* z1EF~hL6Ohr+)cNn?{!#UQS2QhHZHX^0tq0DA(kPEAhXYQxG`2dw}2vWFSR^#AB3L-)&#rZkVNzi=SSQ(I7SR^xp0LV< zG)q$kxzxCsQehNrCZ$^6K|WL=7Ag_0TF94;IsTR~aWVYnimhO@=SYrvVDC0)JUz|^L^dRbdAMUF? z35IMm=K~hqj_$QnjAIcRQlL?Py_$D!3nh0S5;t2CBk6Jb7vY#co7ZbjT;M223mA?> zXlI8X;Vwi=5h1~h99bL~02p*9C6p6B6cA>9d;Px+)l{9vqjZ$a0Qb!o9PMcJhW!V%>U#gVbUS%CP2wSP zI_fyV-3kFO*L|AF*s4UZ!H$MY?oTL))mD|BBhtjsrwH)SFe>R0FRAa2uCFu`_XFf< z@>Euk_o9uh*1uUYT0XTL4!UK>8?8CInVe(-yD6CNTn2PSe|(h~$SGBC5#h~wOeGq^ z7QA1+5mQ-~>19dudL0wEI_`{R2R8(6bFaBQASN0wt%l)Us|lh+S0D3%9`eO&6a{9X z(z~Z(Fh3Gx2LYj9x;$6Pawjpcr~RYoiLJh)`XPT63P<-jgf`DMt|{Ve1Jq@Mj}r)S zuF$B8UF^aN-m{oq1CNHxBgI#ars106(6WX8=ly-S^r;hJ_?6V@JEhWVP?5!O>?PZY z7a4FIgY`PPqNF_^8x3mUbqb%z?`??n_0t$dGoV~KESy*Q_Y1LIlfzAyjMttpcDuwo z#@=2{{7PS8E$U-#x)Hy^7v2jxl;~IsZ;?&oS0AxMW~|)#mo41f`M0PM^Cn)mQF9i) z!h6!Eq1x?d(yO82uD{9Om3p9?J!`ME^?$F`88%}XH+x@{Y4kx~jjL|-KPgvg13npB z^haKmzdSK_WADG2GCR`r=?^~{d;Xs>y$e6_b@J#t$-7%-CwBbP^ubrEN001w^yr(( zl)qSqUirm%!Z*DkSE(j-U8SE*vrZPh>eu>eeBqnkk*idjTK>uC(krtqU%ooI^qtnB zUvgdAbZNTbE3-+z_@cb=%w)q~a-C}VNx4eZ+JoPS-hQoWyUNtsJ!3=fyh*)XQ*7&1 zy{UKAqTZ=Hx&7kV+P!B}>)x5tb`LpkdAaj4(AxcQt@Y8e+H*Z@=k>hV z{AE-7cN3(K9q=v^;EfER2hz_DXb&CmP8Pr$8DJNrj~-|*^M4)s{|)=EqZi^oVgGgb zBK{}rKhJl!{U26YfG?u|UG+Cy^nX|V|J?3!IAPcr1B#RKXP>GF{~dF_!J|qs?_7M< z;f$lu@0zVpF;a~7R%pIPbPx?zdpEeiCUS&#Gu-t)pXr?$cZ&O z@4i!5&z)0vj}v`r!TrPQ$U`zJ?8s9z#GF!|^gC*N0`IW_dRSPBr$}Ve^h#*nJ$l5d zN!KT|!&e}&k#oc`XX&vBmwfW$yp44x(IczLS?!PO&cox;l`wUylsWmvdbsOLezbOR zBl7o^2><8YEFTTPPn(NtL9WkPtN)of`Nw*=?~CCUm{C?XhB^7jdZ+_-n1kzo@9;-# z2u9Y!JzR~%@LkQ)Z-cHs!J(!!z@6w@Frf`cxZa*eWuo?!#slKXa4e;x8FJJ!pwM&! z!|q0RZ=Vl(M^{B_3NkHd!G6V1$KO_qLn^fanKUIme{DL_=qkM8wq@Pu2fP$^BA@h{ zUWt0ZpS#CyOL|)%>vU1p*dSYE18tBEkx}VP<_-^hF;7F?nTCy6D7ART3aI}Fmq2L0 z^Pq_yBYN;fk_|K&ptp#!!e_^(bG_0g$hIl6ZIW!|dZkU2ZBu32WZ5=Z-rw5F$6et2 zGxYZtFzgbH4*F8a_Re8@=lB~si{Mkz7yOg-$FwD+_OT<#lTebvoNv{ zVWs8Ga=FIxD17zT8T7G7KEo9_!o)m`H zTtix+H9!rXxJOA+4Lcgf76nJ9hc(g0H(#c4={5@V)* z0(r6SBk}zg_xkx%bpQEB03*u?@*2W38LNw>Ezge?fkEj-JTsv{etm(fCGoyrmHFx0 zyO?IC`YOk7&P3^jugjfZYJ4~_ z)R8x=1-Yi8NN){?BJQNeAQlm2`Y!B_gU3TGHpWL)8xNYZrW@^o!JCLm74=<1Mr|`P zBuxeu-8GP|4Ig@`a&7F~a_PyWYV786J=D}{&xuLUS@rWzv~oEaZ8YX;sJ~}1m@)1c z3bUtjuPwDPBlRM_Kom6fD??)pUr+=dB^#J-hIpYy14q{ z=+Vkq#;=q$I>#r-ozLLc$JfdX>9B$^#v;(07Sk)dS)}EfMTOm1Ss|AH+A~M$hfkJs z;l%p9#PPVo0sRH4QF;u|qg=!NQflr}A8%DoPDCc1cdlm_ag9~wtFJAvVuHTw(L%A5 z;_0(a*Y%qC?hCS1lU3b+!|^W{m-{?vDP~@?-KTwX^|vcOn?7zU7a7>)R!{4@eQ2q= z8Mms%0y~~PVimr;VAD7+bTd}Q=b!y2pSIgYk-7&-${$I>Y4Ahxb7$|mLo62DDNj5Yo0o+DE;^quj{#FTh4-vufZppjF! zS4tR{wW}mh*o3RDej_C6dVXczIX!=k%d*_Tuq_}lt{&S69*k&(*R`Q<3XA4w6gy5P z>BC;boY7{kv49z~mWG+SfNhhlnYB$Z44Vu+_V?8PboQC{nnU*U&^NsbNxnBi!zL6d zH!}(A*+_Zb@vpxtnfH7z)Bj2%n9pAq=^FT-K86V--+=d!@^L=%$tZ0XAYz-5q$tll)^@zkq${?Nsv zrSh^59!=HwMlbJHVIfe#=Dsk*@Ad^m@Q6+X>5_Z<3kU4);CR4IB_5(;)T+j_1ZF7#SF>GVm`5&9WQp z`BN4N>7#C_#lw1V2Unr?kv1QAXMY)xezC`ZF+WWHpN@nLCH`heGVPnbQ+FCzC8t<#qkp#6-ZVDibJ~cdWph*dt035?64>a6L46%~a8_@r z#LEcHizT-iUl-Xo-%~(1r#^uxfCy(d|o_M5_ag1dvKOUk=S=8JE;E*5nc z#@2rqTY`|3pGY2`HsY!8m-4i~x`SI}n)qE*K*ccw9NgnhHo6hSU2(C*k`T~G# znhj{Me|XweOu*17aJ#?GM*>GeP_pfh10KrjGs*wpUh6;z%gYB79P*B=dfxK~7UIvh zGs%PFf^pm@WDYXd`+4u>0&hP3tUsL_S8H1=_-P|vHhy;W=A+#3ro*v@y=Y50HXcM3 z(x8b^oAiPHF!EP>HJH;A3#<0o`tXgfylHB0lAm|K9>84vFe4v8A4$HS7K%DBO$T6@ zc?Ar*0Jdk95909^AJ89Wmz}mHinj4@AjAN5rjy~fubW8a`1I$g3{y(o`i5=vsK3Gy z*x6Z#+sKAo0GUQ7ev5pQ*sW(%!FP*j#ER90Q;Aiqz@xrQa-%Tw)^T z8jq)A6@&6#1+THsyQ5hwKSGd4$#QhH*#1n#uP;`zzSy#}DKX!71Ls64ZCBg(d-S=C z{pBBs38u8oaQc3}AFoHXT`B%m=RlvOa(wq%_M6u|87jKPA^jX#?eS`V2R!mO6sY5~ z#qbyQqR!W;f1$=Hrg^)jTJx82Z~5aadu{4vBL2pgYBlEU$A2yS%g^3+(mebg*EaCl zXDvW4Z=U(_LQk=7yTzzo`0=$odw#Rq>~f^{(+S>wcx2zN+2geqW*zGdk^cxB^rPn^ zaaZ{UdNeG&@n6e~pz6m)IPx`lu7cOj`jtAdZt6W+5aL}5feO*%F|OenQ$DHhhK}jT z7M=wh+|sEhTsXMLkPKw6xNioCX?O=+rU+%b25z=D@4Ihyz0tyf#!GX%SbUnRYYRq4 z@l$P@Ogl5<=|}HRjQp5M$OeTy91E=m{aoV#TU~X>QopvJmyfd`b|d*^NQC%AOi%s1 zZMP32R|B-vVs@O0-~6%;u6rf!895uI@I8?A77M|cHz1f4vL;0xIC|*Eu4Lp+?kt#J zYtVcXEDLr7yk31FZA@68hX-F_XS3WTZ*S*wkA{)Apx*s}iS2r7e%&7}*op=i8Kcak zXO=$0fyh|gREM_tigRqqVbDTiQ~x*cjFlMD0bF^H9+{2)322;O*s_O}^6?d4=HAsn z{W!=kWv;UA<%4|o04w|BJ*`({r?~}8o8x6azlxo*bBd+($9wfsz*BH+FZVp{@AIvO_GW!GP(G4| zFW_43OuzPCo?hk8PoKqvXwMrNK3d{7OuO5?t0_4FzYnZyHClebE}!=<7??L|F?bVy z(!f!szHm)Hea$br(_TK2U3=Qga0Q4n-wfAl;kkQy;5Z}!>FaKh*LoFK5^-pIH#u)* z@@LD|eTx!_QHwsC>we86VAb)_|NN-`asl^oO*ju8f>x_rMD+X$lOR|UKk%DY^X6E8 zW8a;;WxgBfd)}->qTMZq^WGhf@p#W}czz*@j5be07ldm1FfY6W&i1aKAWLp)wu9LKLOy3Bt=&psrmxi5X= z#eY)3I5b}>UXXiRJ}wqxF~8VaP2==u%p$MPRCW)M(C+z`%Yg~p)5%W0j4o4y!Vc)* zP*CiAb5Ac{73uDt=d5+tXuO-YIU4Y**T(`t&37+}lm!0&U6%ZT~-(n`_zCNs;Qy0-! z`Ls++K}!*mLu1+exdZrocv#CHr+6zb3%XTJtc)-nAEw=}4`X|oNk46)jhBhBZ%uxb z70gF7>DN@beGgRk@9&(S7U=ovM8{y>E%bVgA@6%Edw<=B`*`WherBJaO`n%;jTZtv zY;R^O;@xMv{cdUGPs@yQQQ1-V{2Z_Q&CnYzyZ-Un?!o}CKLP$`y4Jz87>9X%y7{tU ztf(U29j{sAqs(jRe)_Sert+IDVvbGL=De7Ul=e)9Cc?fp>i+!JPYOV>VNB5tUxD>h z-q4=#W(h;TVONAEmcPaUN{-!o@(a^%7LsZ64kqsJT`!GX-zVBpBOiD(QY6>c-g;-4 zm5XmQ@^Ss%V7tD6?Y%!Usg)e*sP*{fG%;~=JL=Yb9Nk`CfSlbYDh9E3wZ3JAn9%e| z%+X_lI#nnfVG|?QX#eP&UcQVKnY%llWcgZDM*dCC9yxlg;Osx+?2NqFdXw*r;=!hU zTIec#apA}Ff^@D|&U;(6d~KIk;gYU-^3r^A9Y*s=ZU#sZH??+`C7l(-(WL8?Y#AXZ?1;M>o#n|_OiaN?b-OS&GUVK zylj-ur^Qw1SoIcs;ycHl(Lc`rqxa)Y@8`aTFsvN>*Lldyna?|!mkROpX0?lrq%!Q2 z`D4K_&M&@`&5Qp){1%d`K_%CR?=|%%ov~Ry-1ZcB*%!n7(~srJuSxCcLJkhSPx+KM zy|z~OT>WXEoctssKaijNLW}eEY2)hqODQJ<3WDI|K#4Asz^;+)?XQ+-~X8FabqtVw;2T6{bDfPF9Gx#9)H2JUWbreGC+GF zpy!kQi-qU`filOz%WS{Ojn~m~pT}YeRyi~4^6O0WKs8StO+H(=xcP6)n|x4A4=D4M zyFOGzuL!m0H=Bun0b+W&-g1{OFFFmxdD$zh>~-Xu{Y4g!jd;A#CxvjCJ#q?40aRYl z*Kbe4qkBK+>vmTHhcu>=iJ4ZfZSkki`|Ji2CV!$*3#Pfeqh|Hb(46Zhgp{>jBDe|= zP&?V|6+(-jyRi7LSUbg!swXk0*Yh5_r$;vi3Y=eMMm+>Jc$$A9Lj0xl%dR0hr1Osh z@^^G3^Pz8DTT6_PX}~jeR2o(5dI{vSVu>X*Xq3-AEv80>E{X>O(-PPTvrDxAS4vJ4pk94_wxFgU7-F(eRyLI1qJ(x!7>Jd z_^>AnlH&YZxlePaL|M>zfR@t9w@6=ZJIWrRQUCbP_J6#*ts zO>U}6lIWkvM8D%fzIVU4HnRxkQL0>Pch*=7v;6^T_BR%o`#a6?kyQVVp_?zSAOE1l z`mh*Z%h)xj-#}oKfAKhcGU94U1X<2F4)-SLsBW>(Q8YIGgD695#R-?GtO_Q}!iN0~ zU>7j|TV{-#!uZa1b@k0&2$osd5BZIs@wK~E^ziQi1dM#?pWLC0EbaUFU%}%3mVvY2 zl5hB+zswEW{PO92jk%q~V@ zH%fWs2m6&zXTh4^bgq`yi{E){jeX&#AKz;G_&I+*K@NH5XlQ(7BER3`e?tMIcy){M z{91^D)mzmz#VV4k7BVSa$i6u)7E;w;y3_qn6tGtR$khM8UBKGGGg#}J+_mu;r9`&* zj~o?+*~{NBcG;_dzwE4k;p-WN=etB<6r*oQDHDPJi$#zB3jDhYgtlqnG6?gs3ar_K z@X1fb`b0^Yjsfxov+zuG=Z#QBi2)W3;p|owClCyJz3Nc4inwCXB&;gbPQi)}Cf|AM zju0yX7O;6$0WXkYoR@(ID-1diSA<9h$SU96f)^F@x~f#f99P_UV1fyms}lo(fFQJ01-=@vF9cBC@G^bX>Cnalk6KkaOodmKjv>a& zbO2=tbhfGtz$c(us|tYSftMMp&cJt3_-UB5BD4m~SbsO+3c+`CYj z{xO&th*wOBw@7HpM?m%(q)Uy8rH>tvI&-io3atR%kEJ0L4~VEMX#z(}%!o51t`dLfID1ICwl zBf_ha^*4PrjwF(jB?-)s-RycoyT;=h+30%yy6Zn3ggu}V8W)E2v6r`U3VvYe>vG@f zQG`#e{t<6sA+>(+7YVsUSB@BKxFcWsyN zBDwt=hTuGDeDlT^?;MlP*I&}-kHgANjZ=asOZHm_M&;>z&(3B#n;C3oC7Z@6+Zk+T zlsz(oO$|1sA$#a!=v(MB9be?rcbR<79Mj#qb>-UcSYX|*X1zgt>+q@hII#F=e*}N`PeW*Z(u&*v1)2vJ zss(op#c%{)=VG#5Oty;wcV=JV1=lg_SiT;4c!@1eZYUbx;xKV6I#wMCn?2AKT##qe z>PpB0Z*aBWEP(X5%@jYJmw|-jF2Ji1@_P=T#ep$e5e7QFN z;(jzPkypXo{N*FlO=-=5>^k02U2w8G2Y36Cg28EMRrF}+!xjOToKJQnc;TSZH#WMz zXz1Xb^>J9f-3na3@cN&i;egcvuYW!af6<%W-m<*=b|wG%vWE}Y-}tTIKK9QI+^qnZ zvZ<{`Vm^=B6q+`L7MnuTrqHx0G;Ip~7dHj4WvJ+Cx$DtgkM4SOZ#wG9T~F?M^5Bzi zba(S8o?dpv&%5Nl3tz2yerW;d?rXtU)(0Apj|E>IYwbIQ>URpg>DOza{PRLT9jliD z0AUNE1RS%jWnFjEx#Rd)4v4&wb)%_)s_R|X)nL)NPc@KLvAAv&RZs&(=dOmc9$1nZ za8=A|xu~GWE2x2@2RR!Ch~FXvcN0$4{QAGgQP;&a&xu`z`s?w;&&D;%k2gV+KAJE#5b(&nNyg z(Yx=`dotrD=KS7ike~Ch;HOdeo2ugPziHE`H+`FE&F#`d)04Z>x=$nz6dtx z{X)5aEfQkePLzngE?C~b4ETe;-s<3gq5wj&%s*mM+(?my#|8}BuT2=SH`dw%c({}u zE`HmzpBbAl85x-jcuex2Nnm7{JeZSfV@?8tlOBGsh-Vr7F(YCozE8?}dx%EKnzVWYOPQ-j#4 zKuv%?vg(*vw!vgUt*!gCM2rlQGuX^vGn36sHZ$4G zW3rveb|#ydY-X~V$z~>-0n5E=V&>w4=2yx(79FdOO~IB}npLK0WtvuuFop~2qNbK~hU6 z*X(l5F4yFOHU$?nyIix&HM?B1%Qd@Pv&%KP9YA~z8q6SM0V!1DX+q`!XL>rp#HgEI& zWfM|fEiS*NP_sB{uDc9aTvjnIT)<_&;$amdB^^-&?Xep`$jAROcxfu%q6bYqkTsIY z9LZ#lWHLxHStOY(lB`D$WRzsGN-~)xne38GhDj#NB$H*5$uvo#Zsy9tQ4PvA=`Y?- z{|yB^y=QCxbshb70V)*#*HqzBpN{u<)lb_QWDJMktl_t2#!pnL|G zTPAPE5MWdN@n(<3FO{#7Fa6;z!NcV8Mgl|sXPyN1W_9^n-GMmu$v6M1^T8Pj8 z!qgLQT;D^hziIX?YEj>W*j;Qz)dT|To4<+%|A#;6B1&J)vX@rqkn>e!!GE5f{H6jB z5Z+9WKmHi7+gd>w`g1j%1$*5T>(jx_az6>LcuS<+~@w<8hFw=DA;`R z@xNGpr0%~yT%Q(TlR{MTd;iG$So!N%0e<-z^#L?feq4Pj~qE8Uk zjsgcs*p1?oeYi*&=fl&foqY3xOY*A$5Ee}M-_5lzSgS|{m?iGkpJ616Z91kJ^Oe?L zSdDbM{4<99i#M-w8(Y%ahsNGEw2ay{(dczpe=@-qidcU(3!y50c|XT|dg(qsY56Li z&sHZi#*JCBYk@P3zu91cF_y1F#C&&90lR5L*Wi}F#a0lg$%j4m16h7gM*fPveG7>yA67aab$A{wzNx;koM;z^xp0-$_yX&Y z%#<{YtW5)DSu5w0R5tHdkb!juOjo$#9>A+UVZ`vLi8}YMy85&J$Kaf1@eY#oSjft| z4fNaJ20;EV$t~z>-=YB>9o|sFh3#xI(%ZfN%Ts3c|CgS6{sZ_`KiQE1;K>@uO4T(I zh)-K=6SMtS+%$>zzIG%%@0oQz7SV0NI*h4%y>ONp#@mnzG^|Ht}YoD^jzsSihdx7O60a^Kr zseAq#_6J7vmkL0zEMKUVVHESOzmzWDI3d8&LIq))P=4}33{aRWpwTzdnf}EEh;sdI z4=@<}{x%5|!tY; zKCX0sKj+^77~^H0eKK{%pV-3*9(=>>hXnpFbC96O{*yfr#!a8d^}h-obTSm9eebPc zM_7>aIKKCVc(fau^!r9=0Ss$0!~X7AH!#?HvyN_!t?)YE`LMsQ0JyW@(}|bfzZKjC z#J*9z@LNqNU@`p(dBINgr${2~IzN*_T$sY-pmS&Y%k4wfKL9=-Fjbg}*E`|@x$>5C z(%)+G{)R69xdO1M{QXS$zpVfW(?8#he?tMPnFQWJoB9&=;r(2uDfK71nKh5zA~N6Z z*D{wqNi2zweU5Pau}8tgW8t{Ko4>iylLGxE2w#8QSl)lOG+&43%O?@!{N!`ZLyfp> z=+CAhAiQD2zV~ut-WVuMx383|lzNN`-*?OdjSN1CGE&mdHzEjL^=F@W+8zCdv0qpy zmREiX1tA*9P^Y{f?msDfVEBGwk_u_Ad{GF2TYqVFfw+WbO5C;afm#|CVAv;h04cM4lr-5ugA(QVE*$};og3(-`X z8_*k@EvTa^{d{j~V5i{=?14|N5)9Izn~j|<<(CIY#JXT#P?m5+?n;LCf%@QPD!0a{ z4<58`ub=osz)LR10=xS$7Vua4(ZYOH;?NpL0{WzV=g$y3C=5PPEWUjH|G5Hyz2D(O z@P=F8!SBoB%hs^F|6UC3vm0*Z^ZD_c#%4|DcNG2izs+AzfEKVnQHak!t6QS?!s}C9 z9LG;vqOSF)3?cW{UmG48Un}m|S1P|-rNff(R%pK0SpK;$2JG%l*~`&G*r zP>+v$_4yA7?#J&*6WVX6;Ru-jqLat#o29#K{!)mrz%NX&KVG08DAeL}ex|2w)EIyR zUEF8(E!N%2F5T}tpdJwgex@G%eXsYe0vPTUQ^;QX_ODtJf384JZ$AmZ$I$+Oull2( z=TPo(8ZQY_WX}%~{*S#ItRns%8{BUdSTy`Qh~f8!uI|QO%8WZgy34HL{zOCf>9eqI zzYDkiw18A&+!Mccg!!3tc7qspAE0nRnR$$amXWaXZ;#Vyy zA)wl7uU^&_RtjwnmvJcbb&vh3AIU&g2cPyZ)7ZlR^dvBy)e%Qah z0HX)Y&kKRUSiUsrJ!A5%rvXyRf5dO)_ZP773mUTjodRIG%HIOL!xiIHtO3~j=f?zJ z_A3FO7~x;;2x|6fTf7T>(9`_VlfVF;b{t4%#AUUto<4JB`(Ce|3>dNPKWe*wYq8mF zp8UCCrb)+7d=U{k>rXi>{%;pR%!PN%?Vo=Poh8E+xBZFWSaj7G@>^hwp2iEl{0j_S zpI>uQ2$b}`ky-AlcG!*_BkET~ABJ}L&g`N*|0mr3;I_C-v1S%Pa41p`9OqC8rAw=Q zr+3R*{0Gg`)<9i~vHZ`pdHr&K_+t}@$b=Fdz2h9Xph>b8iu5ym`R$2_w)cjGAD6WP z%zVExAGkdY(_VZUFl z-R$az`AdoJV>d6!^VaC#LSZJRFCNaZzZ*ad`vt>F^x#J~+xRpT(U%QD(_a2rjrlIn z`lm1PWg~DK`ITr%|3HE9`hk8CQE-N&8lTum{r>l9ci8Mf74kO%i_r-DeL? z>#yQS{}Tm>82&DP?vEAVuF~EYLXfq;Hip`_3#@(n58<=)o&5t8qum$LZ)5uy8-=!G z?{z5u^o*q&jBIrJtFy-6&=p$rzF8X(j(;U4&2KKy-|Gp!)kyz|^6%D&DfH3!dc*st z=gOA_7~<@^Z~XfTxW6htzTe7{o8H;x{7r+l{tf>Fh8e$H0I|ovlD~jV{kjA3_(d08 zJN%UY{9FB<1sD_T-^-j(Liz1Zsa#+GY_`~6DnPyQPvRDS`!TTLU+boqPYor0{YjAV z{>aXToK1iFBaimESu|Ybzlvz|Rp75rd&52cSHeQh{s)8o(K+w)b!q|~UPZEcQ)D46 zM3jRrZHX%VJ%=Cd0J4e*iCWFYmwMbx8L-XHa4d^;@d zH%*l97igacYWTIMjQ4>1OC$4>P>}JXez$A8F~lfN*acxlB`|B?&)S1hsLQve>E zHGC#H@^8X(ylt%CvAhSfxc}K+Q8%ElUUtv&0A}e7pU40pRoK`27U}Z%&l5%jO+#xCs-%AxlEQWR-L=T*)At*S_`|NX=S#GFx{HG0d5lZ# zR`W0Poq*xxx6|@`y8vgtJgpNpf^FtUdoUO_yxd;k=_257*xMj0J$*>(^`{nL{iKmQ?EnvYD}v- zSaPu5U|}Q=EbBWkGc=QKK02$czzRwjZAzmZ4@1KZdnU4T{vhutQLnBLo?@4-fSmM4 z)_}rxY!&pn>wd6QqaV*szE$A}i!U=TxdOmlW#;1_&jX*Dm3eo)&(QmsVgNP1wi3=V z8(>3TndK0$$C~MFBefATDmRslK_6@_@d=pFJkxe>rVo-u0uI4;f58o!(4Z{w^R$wNDW@TATF-3ofK^X1XjGmmoiGrR2H3sZ#^a z48BSf-U)&;oNZSM*}@^*NudTh*7=z_)2D;YJ;uV+5`ApTvgE>C>q=nTkB^M%H`)kWw-QL*buB6u;)5p%gQNe;e+R>`$IP9j_B zir19VW=~Hx`Y=&J>qWIXurqY+UR@*9xM1qk(6_Z~xYCW` zHa^!fgl=5Vn0k>!P7ya77a;=eMpqi+!8~9&y&B4F9|Wr-2xl~-_%@3UtPPUiu+esN z^*DERrRvE0lN4r_t+wXftPV?^o%xprv5XDDMK$_EiG@gdqOhy*RGF^x7_>QWtQaTkYbw19;ZC>3 zJwBwOT6^q|dEK=iC%?iXt++jj`zDTsSb1j}dC>NbkhHNgB2;uDCF9X>j7@MefNNjr zBu*&jJPz*XgL^KxF~claHy<&l@Bz9)ZY@>ss##RF>9)R`vjdOGzIHab_^%Eoe)k-_ zUIm91Jw~`lG|mRWGa1;4b#*44MooH2biTu5vh?8Sbd9p#xCq*o$SCNm3~fj+w7az? zY3DM*(FjIi%HUj7agBKAg601m$4795Yq4TAFUDyVEcANd2%hQ71{&nV?csIPZiPTD zB55qvohB>0z$Xs!?qoZ|ok1Mol%LS#`GenX=t5J&@k)6g-}MWwz6p0CySbsjY`W@@~t0eEXS z6kr4F8kouyB#8GW^&#uCV7P;cjB@d>6-|(9*!NUh`WS=k7{9Q#SW#tAB)c9BYSdOVNFvVQ8uZb&?qzU7jOY_N z=D`*rq5n8eo(}3}I?GO;P4;Yxqre(%%1A0DEM`)}25m?Vi3)n{Mh7b{z}4F%4bbFx z^me_RWbR~o76%aOAygej7p_6)N2njpH8Oj>rT9|hK+x01?-;3PI!bV3ubjtrH!B*v zJTw782qu!VM=OJoaJiBvhgEm@6JNb4oJ@}pBhJpvW7qz192!+tnWP$zQFfdAb+Mi% z=g^Uck+Hts%Pe^4nvv@*hBHkcmyIOeF(v7`jzblu?l)cUYG93;Ym-*X>zq9_-II7_ z)BuCG@TBoTt9Dcp@f+j1wqD=$LD_BBg{B51sL!Ixbqnmfk;V=0!3rEa7VOaWV^NL} zVam+O?C#eqtDH&H@Z>cQBX4Wmp%~KL77jNuhQ=RPVR)yCgr@~14F8+D7^P>}^ z?N`Gbkk+{od!V>&GLxnJiX-Ca7U1p68l)4#=)_*&w~ta6M8I8Fl5b{94~iLYf}1H+ zBW1K9p>I`ivcwFnD@vS5?@0550~S;75?8#2q%c(>?-wDTRz!^x_&VlTB*YeKoke4gkcRXL3+vsWtuB-sW?*02jcEKLcTlpXl0 z#OTf$I|!6@f@u%nw?hBcOAZq1L9B0ZOTE*Te5qv!_zh`VvP82iF--HZh5)`MYZk@J zMaL5{tGcGWxnGejJ{T!n8j(rB0sP4Y5h%uXy0#2#m5B;DLOuC{d~h@&7X(S{gNrs{ zpy;SX<6L763ws#pVoj#hIawlZ7B$O~qqC?QA(SC-XApJ62b3V5+lxqh>;lXiPV{3+ z>S+W#jd}fe3~@9`Zk*#LVG+=n+*a1pQzqiY#_iu^*}bERjSu^8RGb` zYl4BXRT5OPOoeRZn$k!fnK3ZZG~K?$Bp)?b+MvR*2u|#u;acXnaq9a>0$QBH; zOoJN`_?D$PF!S3&ex;&smQtS~8gdXxMwSj_wqy^UfxL+=9&NtblH zsL`APn6%|u(UIcwCLnQwN!a(+D8}^m`6(1qA&Y5QwrmBtaf;?IV!4>I87pFmI6pqh zsLzXmBCds9Mf%T6!6RH60}e84_sLh2!wBRk$^!0NRTx1(=WuG zWV4B^Hj&fz42^V`q?|Z-)T(fOZy6-isIEej+|9h$8t(a`Sx`9NRhiVg4e82;G>$`> zw!;(%p!vCYE$<~Y!t$hJr(tMJ$%)Y8lj`wR)Gw1}uUv*24=VpC&u*h18JO(3BxQ0E zDZ?&}#z|p2dHCKnnW?`U}gDe)F)maREK>~6q`v6Yab*ClZwm8h!M`-)LAN`B9)SYQF$czuuG+a{bBu|doUJr zGY|o~@4dh;19#%LO&XJ_r&bee8>_;|_dvFJtWz%J7}A&0HulwMb(a#r$vU<+piVK??a8mm{Pvi zJ#fT5LT|EVN#8QOBo5wkQKWoZX?r6ePgst@cJLaDhKnlD&|Y2@FpIEutU0YilN4fI zN122q8EClB8<2b4PyvsMx^7{@c9^diKL@#`D%pEc6U(>n1oOzI2^zvU^k3JjkGv(D zK^bUV)<{Henouh30*z9d$Y>*{)VKpRGE6idKvA_x7Hr7!K(-NCs<=hKE)KA53q;20 zb6LTOFjvBy!~;m-Cgmgqn**~JOq;OCBS{~cmpuxb=8<+|ow!@qWE4KwXi-AC(bP2e z!|4*>5~*JklxnixGPX+Vpf2*P9u`r2)HlTS@5d2SXNq;r z(mw|@lW9W78@zxC7?FO&(bgj9CQCUk?x5~9PcB?~*iJ?SEUt7;O(;9$XxTqt1DwPR zCQ-Y2(Za;gZ>yb*!yp9yAlelApiEF<%0M6s&&!)gqj~0^d2}{Gg{<@#2fj5d`pprO zk;mZkX$L51bqcV}v?Q9!57cp5Dcr5j28y({?%E~)s&nIdVTsAwLHu%)a}6N{eprGL zA`jy!l32<%50JesLzs26rgI8CU?qF7ZL$^aOmc_@TGwVGTZgGrRs)ploElvQd4Y*F z7Hwh^Euf45-o!w+ZKHx>&gVaZy9_f>Q$Jn(clKxRb+Q4WsQ-XLUE^-hWy;-2K4N>C zR5~lh3&VLrIo&k?GLR|d#&q6R7+9t|dgX6Y9X|S14pq2Q&@4lS5P-9XSRwJUi_MYE zD_L*qi&#&`DwBlDcDQjTu}D>x5#^HE`Cigo>E4OIgbCm3WFymoYK3f93&?;iHzKEE zM#t5K5~jc%nz^1-O*~sAC`7J#wbS^PRiIEMnj}l*PM#Tb*<9#&*9@B=3K$|mPgNE$ zg%26D!UvChsu_p}NXM5i#o!AX1&i)i7aRcQD=NadLb=ya+?w^_AG)-NXHTh;=y zeWa!9n)Xh46q~TQtVtR-1#rN^cuZKA%QylX5S|d41=(h_dKqB|&V+Db0SIK@_)1uX z^GSzwj#0!T8{No4pxeemtzD*$K;UdM{IqYl@DB(J=&`&Or~;z)kgueLe6()FE$^ z6DRURL)yO0P{PoKWmuMrOV&SOX2Sv(u*fINGS~C~>uSIRqF3$r&d5bz#sXZINiWaC zik@IQjI67^=Ln|0DAqPJ`cbCW4lzj|Adf2z}IxSl|G1!qd6gx=EDb97_ zq+tV-XeUFvrb|Q(`ai?`XA-{XO^my2Ack;%_MrcWb zPWFK|N;Gg9B0zDE8u3B9C+Kt6dxbH|B@WQ?dSE;HQ39dF58Q4LlcL!cb&hM+$<^-S zWN3DPH_MXeS68zPC`*_ikTuL>p%tm*W;pzO2x&^qsup__fSoCGXtH&@ zM!)-5nFly|;lNY2N;C9CRDw)txTX|Lmzx~%QgQ9!gcWX%`$;y1w_P?5Q!zCmfe`Hj zd2wQAqrVL-AlWHx)Pf>xCR{0vNsc)-pF1V2$=!02MkVQALkcdDgvfvL{6a#TBLR#DI$^Jfd3f##A#pSEMRML&C6Sg&2F<;(e zCZFa}Z!KPC3gZ$r0Oo|J(yBvD4IFMbE={cpy~S=OoURCPP1X!k#`Sv;1)K!rCgI>A zP!XnQbOcz{1`7>BoF-p1CMd2?lVh35oo%qK>DMy}svdjV%+HrpyLPyS-B!k;yoB;mdQ zIzYw0`ibEv1WISTiqKHG)W>`Yb{|@McC5} zf2(j7y1*^o4`;1J60p*^;$4nHzFl#23|FNd<(uSuIq4AO zY&7e^dAx>e8_{co^N-w>AU|JL`12VyWo$(FT&h}`<vC=74lXe3ZDhntW9 zA9-noGb`N8HdHn*cOw$OpuSg`yW0@V4DEuDxfnzl+}rhpu?20?1E>?*OLUt~F`j}FEJ(pAQ(Av8@(k?w zHa;*(!u!~l_Wrw|gdui)5;l-EalZftF!6^=JUJmS0!1QuQpBU8>k)w?WPHivjbivC zY|ZiBeDrV`$u`Ngv}UB|n~}#`vM$^*DCw^kuuD?`gF2c5{~#8CDsBN`)QCy#vl&Uxd^``kZ09a_h0VixJ80-E_~yD#Fy***?+010pS z%|de~0z3)g%5^Jdu#*ok>Df)&cM8E%4t@{-I@OaF8gGc-64}l6_wf`)or(lxDP;Dc zNg<9`v5{zH)MP|O)fkUqG9=fZ8;xyP8BYyjkqlaHeJB~**90iXtIeWonEhds& z78!L#1R_yhpq)~^rM9rP9cv`G4<1dBifm9}D&loXv2il8orrrb21X&W`M6I{HsVNsC6u#OZ#YqGS&`MhAa zKr#B*eX|i#v(UX6rZ6&*QFIlCW@=>2QUt=TvCj7;g;$}WdwYiaG57iHMG?kElH?Vk z1sqd_n2HHzY=a>d5)B?JcR{3ez3VIdzeXM$3Y}v_=TNLAqj*GGXBjNsnCgBe#^~ZS z2oB7wK3<5#ZS|&WcX3&7KPX{J69P?q0J-Ug3Bc@$uEb@~m3WK6Rel>a2@@7(u}wI_ zkfH+$e2Dx&ATtSF<`Q}v&4&sS5TLMaaFQI=MgR!yQa76v?t1uo zqlmjH;RCXqQC}}}lNdA=kcV663Q}k~x_B8F7nh*{t5=z&$iavKESeZ~7Zcb>HHx$x zXh0amv`g~K?9)j5(v4w=&o!iXvL}zbf$O$$ z|AY5O5+2m_$)l>p_LAQ2l?IzgufYm6`s2#`Q< z+W%wk&$3`IlpM0O8+de7i}*1;Sm#t?V0PpQ(Y|CR3q9ZFkr@Z$Sy-8;l1D zkL5H!=IOCoZLCdzNK@xn5o@KuH?CB{IIyj7LbEU_6Ri{++6f=2Bk%-C#hKcmmSI1v<(Hl0nt$EM=7clW|CRN z6^>KqRyuQ^+Lw+TbUDS!EH9dP97HLHhSAZn(u}>qERcA)geS+)R1%s?B2EJPwa3k5 zh16UuNO&{hQL*4e2ucGBPJ>OdvO;}$zqC((rPKgGpDwqKv$!39#>OOBUh@Fycr^4D z9EG7MLpl6XGHMcJ-O3i|?_;8532@LfyNgbpjY#QvCSc@A73)m3Wdto? zUI+!-*>0x6-yzD-HVe?+1f zVdRv}G6iioA*%^y?gyw}avY=YC({Lk0<;y(jm?+W(8)M3?A5T?Y0eDIN8DE0P6{%; zf1zTdQhKNKvLgEIZTB!;O0=Q!eHs%}gxazXSY2U;qr^kwDV)`H?Hh{menSjDyQJc^ zdaw|{ZC~#*DxDBL1=4~X01KpRo~QTCh;}q$2^IfZ71AYZOW2Krh+J~^7PvMHdnuv| zv7|uxqBvWS1lz@-NUK}E{BYS@opArWBN|6&O3T2@Mxaq0kj)%DZk*isq{MYlu*;6? zQ(C!)qN51=f^8N1+%$$V!hG+Np9?B|nfGK(B|K;Y>h?-{=@L=MCUPBcj8bLHW6!Nd z9bsn=OPh|)uG7$6D?sixJOW)^*mXh<$^(ZVhU_HEP2W%3UKqNH9#tEf$g&f59lljM z%1AUV;<NE^n(;kSkGvYc5HKus9Sm8(StcD#`5A(F_QX5?=+eM@vwt zm?en?n1-de2P=?c!VUAFU|2IU$HJ^vF-HbSn-D8SCy3&_%Qsj+i&&Na-!PtvR)7Qg zmeR6IbxcixvrP+HqfHv*)zEm@k0gdVv&F$E_blqCI)yX&hXzxR0UG=x2-E$VzZMKB`vL9975{{M}-}`aiBns&1QskHdwg5A7jrgzB$fS zcpShZ=znzP-lT4`Hz3hSyEmViV}IW9yc}7!jO1HZ7c-Okrd`Bq(Y3Tm9OV+FVFWTn ziWCN*z664O((f(rnzjAu-0f0#(Q^>1PO%gX%mt~245mJ{EiGh-KWk4DRjnY(u1KMA zQeM(2rHq~suGe-7-=*6jlDHY{4?1QkfZJL(VikeILVFiZRDHlf8&nIEXA2<*GJ)l- zuvIulUI+WWBm!G_3^5!W2AlFM*s&F0eTd^)mJgX`j!S(K3D}!`fLw{f2unK)4v9@7 zQ%40ZIvKjm67vl_6*LeT?fpZ$x;sIX3F0^qy3Ve?j*8g_L{AExxdVNi7!qs~L|R%K zr+t}lN@LqJaCQy@Z4#{rv$qOiOtP|ou-{MoRsIu?Z5Rh zm(IjU%&h>%6HObG5`q^jLJ=ZDAx#<*;t;Hz)>sJENZ&ybnzj)6`?{G}2gf)IEos6y z9v3-cWm$3(q=dB)v3ZkV>^SI$Y(bkO#ha`Us{;v(95&^^3A>ao#k!C&7Pp zNrgkA*^uznTjD}(pXT&%iAg3w z8ui1{Z^ZpuI;pph_p$zVqWA< z-LVad>9A_C|Bh~$EFWC}JAv`@@jkiJf#<-F1jbBPtM z%aoy+4egWl0EA%?>C0xZ7I()<&`%hqgzB6YVn(}Lh^{+k_{uu0Va<)@=J1mw?{@)h zxLx;^aXpnRK;@PpFmfJO-3nrR2iXA6RCJ4{2Q ziVaXqkwVwX)THKiyG!LB<-y$JE-{1{wF{U~E+YFKU>rXxrwu~Ar? zUtuz8N|%(iLwd2R26jXUxc@(U-CxHj#DeJiIH%v3)cRQBjhN@1?)fmfz6rrWI zbWXka#zszrh##N9qq$2O>#`>JUP6))Er||%Kgkd8VdTlr&muQgp-T%M?5c20_xO^b z^vWomu3ANcNKnx3pj}N1(j+Jol8h~dLBTF2eceSdl4T?d@dr0W%}Ye{jFaJ*4hbjt zSxm8)Ho`_>r>BJ;gL0RoZCrRhugM=9o~7VHk`N}|7NT!ffxK>46>=SlUSRmM{Ycco>j; z0tXD~^T1dgCq+C(tw?};4b-rBHY|+Kk!0cIwDt=7YAT!Ov|M}PL=!c&~`9aaGYmXGBf8fih_6iUi0%rA?7~@*$H{fzdmNtkX-)TtXkN(byXU3YZ|JJhLj(RGN^@cS!ct{nfZt@%8x6AB@p4(L&154u z#WX-}rIh>K1sgq?gvAo6dRW;-9c4-{(=*QZ4Wp2#tgMM=ViVfNl)1_f!@8h-NZ?J+ zc5zSWyyKap;K@*&Uyyg1-Su4XKXQh&Hs_1fWrwqgq8+dW2F6>h z^VX&($Wq@0W6AO=VXNp;jI5o=rJj<0FO%6T=?2&AFn4pCa<`Q=LdQV%}Jh!z$ zN&J?<1+6Cqxd8xZ_p~Wi!Yf7p4HUD%o#tDnE3w-g~8oqS;uAPu|Q9zwF?vVjIz*K5!TPide0ri?t=gHAI0L2e*?GgziU@T^f zC=UwQ=&fV(Xq*~hm&2``>H=ruqV;CWCZTawfl5bV(CcAT$O~Yd3@cz_xVmA22Z+}K znY-z8m&?#Q&-(=+xwef#lNaK2N*kxoN}(5|XztF8zxRpHUN>jEFOs?;CF<1vW3Zi= zO&uM1Pf5|4cxS){dAmopEpfgVu64n{Hp!v{#K)KWeUijy5Rc>J2_&{o@s0x!+Hj(T zW4naDdWe~=?u49YQ~|}m-DV4OBLMm=I8Gd?1Pq;L0~1MOyRY}~nChxS zY^L^^7ItDme5rvTVegV#JH=W&ZE$ersHf!P?%?10KHcEA30l~f1yL^CO|$3X3=P=# z7MiGbghW`Y*j)^JG_evFj1}=|1_OYBdX}+|)ZtBLv`A&*iDGeVE3IfKR(Ff6K za3ChQ3w8QFt8Ok#J&-q^{~5q&O;Fzh&cic6JxdY~(q!3ZaPzk?DHCh5j(TIqU0w&|w3%P10| z%jpHHnRbYLMQdIbACc(5N5nv3WIa-r3I9H`mF-J|O?WkqZmiO;6JhhW1eR41%fR=x zaL>mEvt&r|k{FT6CiXQ~*cm2ZtO}wQ90FzXva|Xt?Exd{Se#aTQzCFa0jEs_*549= z@H0a_6nAbsINP-=re4I24R^t`P^B{cB#-~Tp0~y#4r_$d38#BNwR(K7gTw=1ls@_7 z*lP*k22OPX^erOm1)aND4-=BWCy}Vl;MB8-m`|82ps%qAowvTCfFfC}MWj9<`UV@I z_4qzvuOjr{2=q@8s}!k;;SMyY2W`atC7zB&EVvaSbpI5DuRe{6bC~7EJ&+fwInO8P zIxZD_IE04rQM~%0e*h#-TU?h}&4BodssX=+TYwCUlts88mNAgR)>hQMS7d{NL}6PG z6B-sL4}W1Vo~bsQbQ7GsrzURz%(rR2(@kse<&Q<+tLaqOxCu0~0e?1pIKE_eI0KMR zcxRph%u4ACa9S~S%FsKEK(o2;ZR}D<9N)wHr+6sS zH7)G4!P{^PYM&l9O(KU#>;NJ6hKLp-PCxl9%r zJuxnqbucA-0W;%7lE^R9T*qP>2-XcVYdTpZlR9XG*{{v#Xa?CfO?aV@%(CFx4FtOk zt!HkfxU~`Bo_^X_v$1F^DEb z3pF65aW?1{tS!Kr7(21Tq+Py(U_vEhQ3xm!mSrbQCqjvoBG_}6@TO|Ova;&)cKNK8HdZh+kYF`;yH8^FgjkfA4WgCstXgp-$32i^da5cY zGwqb~!=MGd8N8YXZ>B+fVX(v5SeDPo-7_|}(iW@~`|RRrF@?jzV8UqVz@#6Mt>D}T z5PHsoxObpjU800H7QRgdR22y@{#`_GB80-sACo*5PLclXEs2hWOtnguSZa(1PgRnVF}zFnP1oq41?`hvb}7y zlK&K;7GVY$d~KWGInmR9<(#B+GFu#f)^^Tp?B8_1I5FJ;c-S-<-9nUP?<)(1cIf6F zbMFbQonOS6w~IRxixPf235lF)-okAMNL!bqZB!06KTnb}x%i$X%$=@b2t&y9`5vJ& zP4hjKbHEzlwG9v@_!nhD$Nb>TV|m;o9gpZ_$I9zt$Y*fV%25FF!?F8~Bd~<+F6>xi z8N$l8EX{-p-*lsFHKBOtWj~ei45kO2)VFjBDK`i#491GI?GIX@;1QFEcE(k13pZ#H zQwhv40VBMvXcPbx;1ARGsFQVU6X0nrSSg;e2!x)2?q+R?j|RD{e%P0=e)3??d~T%y z1jn8@bB{d4h{QQt;#_2V91-aTm61W2WWiAc(~&`%WFwq*ZEiehM6premC1ymjo2u( zO}4O-8k!++v!FaJM#U(-@M$9Z*sz*(uJIq%ptQC7 zquE3~dIB900WE}0Xe9y5O29259crrc6qkhER^o^*X|wk!z^apRLD=#*0#>^@HrD9` zg#g#|8}e&3%$>GemvQ>aQWYMxCP=Hok4}KJw)4F_Za29c3?GJR2yhViHnZbV*zt9g zG1u=C1A|^IP+|+Z7$Cigd-diK<2H@qCA?Ju`Rw;VakBHuNL>S6hP^*rm3urG`+woA z2 z=R?E+strU-6YL8HCk3g3`Ym`3viuydG$n`?Qi^y5&PbAG5R7t$lp;=T0Ycm_M(vgI zXC?|!pQIFS-~hV;wgXQn&t;1k1$3GWCeU>%EKIN4^wkzoGO z6D3o63Wv2=252PBh!9WPNle}EBujMZBdxfwSp_x;xOx&H_6epdFP0Y(jvN9ZjD!p6 zjGc0|bP^e%I0_ozK99=y;Z_?`-8HsoXMJgB!QFMU$j;!t76SJ21Dr0~(%oEe9+FuD zRU=Cc&J&0ltp^GRD#tC}cGEmQ5i`z}mmhWw5$lDgN*kt#6q9y~EbrlQ9m35h(cdn{ zqY_kca)3;6QL%nK&5%FEWJZF)B9e*%`aiT84+ji9y3ks#QwaHuX(t23cUUFn@fxwJ z6{%!Kr0ekL9MHl4lCFZc;>jl&EFu%}p4vP)1r1;NL;?zSuX#I`l}9ZVIz)vlGEfnf zv1ZNT`>EQ;NEJdo?gZVI{z7PaB2+!Wl#^_N#RukyH1}Tj53DF)g-Wco?RIEo_$*j> zIKF*H0Q91HJ1?B9#Y{24nEE{KbgnkNlER{tF4!UQ= zqC*(jPjoGt0rxP@Ut8Ki353fQG8N)a(ZTpABKi7})>khKX;1lI*Xu=;I8ARrgL_QV@*8$nSe+zDbd&o0NG_^cod3G z=kSR=jpeo_a`Z^ttBuKdoFI9Zzyw{NFtQD9bke4=jciCy8%aaAA~gYfr(-t&IYi(N zQJ1|^3vj6}XM>ASOyoouk6WyyjH(7CMVKRQi6}(^9~k8Lvtmyr04SwdDv&B;No(Q9 zFGx8>(B*_EjhmCgmsP17mHsD>clke|~rc`-HiO zHza2&JXgxfp2g|f6cgISU6GWWp~>d~%Iu^pd=kf3avm^rj#8lZp}M1LNEU{*hHi=UMRc zZz$_H&xPvgmu*j{#t4o3M3KLpoN5`3D2>xS8wCrNjl4pxQbe)AXpsn(mt|zIxeXMX zy-tNT*;3RZPEG~5X3l+r@tjEMoD9CnHm^G3bG${S4v;Jf{vnTj*|1r~HVOu(MASI& zHbU`cU&jyeR+Hmv4j2YiI6Yb+FIv}j@ow_?Rps%0%`SB5%VnE2>f?Tgq>+*n)C>m@Dx3UJ-7rK1gB{s zicL_l(19}sO>kxY6cp)*dbSl;xcC9Qxt$y0ohgtpS(lRyF>Mj^hF8)P1zb~=sZw62 zNU8=-M5_lbz?jKi*;NA=d}CUQedpL^&g;`L^PUM>H=zp721E_ z5Y$ib7sETAK#c^ul@T|W0{$`rZ9-(65f5mmSj0p)omWg?WCF^gu-xDf4opx-ua*Q384K5&kMGEVl!T%UGfgEt7z}U$a97rrZFh+d!`DM~N5F;X4YuGuvZyMSx z2~V{L${q;@>5}mm(MtUO$aON zq``k*6KfKtapAgQr9{~{nyojyDVNa>LT)C3js7vR6cT6AH~LhiTfYbE$uVq0j^#lI znL4PAS(|bSpP48nVRkx7T_7A{#9>!YZsZTIyaBK@=W<|RRU)!Vd#ujUsZa;U^G3Qn z{^h3WA&lX(2!|@RVklpw)+9uEesElttbSf5QD$aT!@HgyIDitQvni;rEX=@t-Y4%- zioG@hD3l^-$I-s?@(GcKeAJw*`rCMT*n=pft#?cdjy9Zulmh2TP{AI((aAa#!vyW` z3`GjsNn3t}M2ZbHFvo+VyV~k5uzHaRs z#5{Di>LyTXTj0C^+GJtHZZ;x(j~`~F@yT`J_+eRCJvFw>Gq-v4m3U~MFs*4DF2LbD zLu1onFZ|p#AwOqLGzFh=CXB>WRVU3lkZP*Hq&yrJs>1#L#&i$N#tHsobz_UNymj;? zw1omMNnG(aEqd{w9EF@)ezmNa2wele6c24{yk_#;Q%HHTQx|R9WrJ~{Jgj>*mYsbL z^3m!CtfJ=WkfkoTLyvJN`#qv4{W#h05eFIQnLIVL^1Fyn*-x5>R&^|g5C38r4zy8x;KbSOZ7HYsK!G4FO@8AXsFp~e6|y-5JEY# zr#iSW;Z99_U-{l<-HNmy#IrsOw_Gd%5R3w}bOD|5({UP=@B`|k?edK@ffBpS4T=KP z!1uBsF33&}ZrwRHA04pTNIT0X3?G}ooIUfk@SSe!EwUsRvC6xOAb{oNRZafsu%vsb zI{k=USjaBlqk%N>hIN_yhIS!;n{tO@I7E~aUx3IHf}C&@vZTw*A%`rua*WZzJbnV4 zZ1c+nGU=>AEKtQ3)zm!WeP+tf<*s@c==%MPw_Nma1%-DA*Rb5=ScJ<8>$m|r_ceR0 zmI8FBB(CLhKb)%y-kAA8BLJpMhg?57UowcxH8r=3NnG0rCfp(`>|~Li2GW8ZchC30 z%gK;~0J0ft-xjhX%o@6e31pzj$H^NCn3^vC^JZPtB zj;)Y%J3_ujeveqY4RJS}X&Ehqj}Rut)L9Z_8VPQXTS{>_t_<|nCZVX7e#B;6zGL@&~jtk77JjMM^wlk7xnN#W6u8tLTO5oQ<9xSz?N##I|YKRk2;tFVxdj;8;D)!Uk~c zIt~mu(F~;&w&IiVH3j}`%q0^=kn=7KAw;<;8#!NzMf|)wp8ZMCZiSXLm~{PDfa)EEp^4QY7jaXSp1N9ahRn+@G9D8}N(2dBdX6uB8)rgEBp)QURK=JA$MtIrQ;x zVTM|dd9RYROmF9*HQix2|6Rx@(bNH5?HYzWkmFk~*k_}=3MY60hN|q1-c0kxp&Og( z?ckWyN+|6dDYC~h>=!OCpFpo*OBX3Ks4_PIbqPtK08Q5!hNilkx_lSpC2YSPkjo~p zMDWZPoT&vD)+Py9!bXTf1y0xvI0mLq$4hOQ0sy8=nvq2lu~6;6@onRF5J6gDc9+P9^`*`-V$w1 zx~HmFUCs7a+%a-7Dn{3Bs(S3RX5nPEvwBd35mH1PF)4mpl`v{W$DxJ@jZk1>#qi{b zMgU>$dPU%*{#aOI9@!D1qn{?7f|xf*(!gnN(pn5rLWe(w$m1j|(8E}Sux){`LvQl} zAvJV{cA<=HgCCIeOe&Wdf;>f*N8Jl9hm8S9?COp5D#Rph8~`nd8NpU*0wi)ou=8v| z>t^%sHcpJc=}s&ImyJX8D4{4V)NvKuhlV$>>a=?ObQB&8mhU%$W28Lb*1!7>4hu<>DCO5s|P*uKa zcF3Pmp22>3r|xn$(Ig-&F+znE4)qBY?f6}=%uYu6E;w-*ckzvw+#UWbar*9;#k)MAz|slpLCmJIg=#2@J}3}36rleqatlWVVWuHk zHn|>;%YJG*^x4Hds4H~f1efzLF+#%3?f%CUg<|Syrh3rOm3U?aj=jjGr|Fum)N=_g zI*2L9Hpf6YRRF>4z<<D zCXz+P_b>seZD2WTm0-;*e3g61Rn{SV+URj3CMjTGT&bjm8ObxkXgW55{6pT{^ zgJ{VY8x*AMib4YIq&1HVEAGtz)-pzi(${ZAFXtyhZ~*U^NOv4F1}jz9cBz>@X-bcI zS;A^&xEuAmpR_x4O88GYrj%Hi4R@3^(RQ151=c)i|7rH9wRU+=dW73qqCM zSRWd{%^>F?;M{G<{6naN^<}3WvJW*GpPTGMYB>}`GD;@mg_O0yKPtGsg3SvWCODsO zj;WBhF(m;~k__e%A(=7#2?`%odfBM+iq=Vx9~u0=ClPC>UcqrXFQwW$4|OX&GG)A1 zcs{%oFV+>p-r7Af&kB@;bPE-J-{@NUbkh~a%5L{CW6tE8=|0(Zq3(C6zCT0X!|&NM zRpJrYwfqdOrLNb4&%fRSe?clEgJP>iU(^0A?4GhCJ8Yb>9y~A{swvFtI`l`1S9^)H z_I>VwxblQfFj#5Rvit(1GWeJccBPwvd=e0BrFYWJ`{}zt{0s^LEDBVyA=!u&jHu0H zcQkCyfQziflZ}EEl$C6nlO2-ui%Kw{ApMZfX*sVFIn012iITd)I_7vND~MFS6Q_6q zJOQc9xAZqnQ4iRr$lyUz0cASf-2*R;n|h2FOxyG*SX(G6!DEy5ZrvDDWuFb!gTQHL z3@t{B=R^T%sn%}yMG{u;GaX!_ck1l!7_V(UB{-|Y38dErvy}^~Wej)?ms`vj1_&_Q zw6Ug`2&@%}?4&@bcrCD0y1{#)tPD(D*c!gJJBy55V7n|EMh<(gEQwi)Q)cCKosKye z0e2-a0Z$O_!-$&8V4gOe7#;__1!V_l7X0jy{4Z~ULkP#{l`IgI!cV#kcv0UT9oT*tV)W24$paXAe6vkr#gH+?id<~YF&xX!`&hCrN8Db=NAl9 zBH|LB#dVu9-yssI!p(e}RA#%}X-2A#il4p<`A9@})$?kha4z%RdSA~VM^0CNVa^}w zC}(`~H|(?Q<3CXfWo(UsjM_apktnx>q^f8zS)M+(+T*f5yv0o?NF}zR=&lATXqSto zY844Eljnc3?`LU3|NS0b)?G)rcb{~6BH8m#QC5t5WI4AoO(Ib>Ta&$VhNei(j9=0u4ewVwp%gaBa zId!3?CUJvYYdsKFDW1V;9&}5lvR$3w^LU$ois7PQ>dp$|j|FQHB2=)v7U7oI?2I1) zHI*2TK5z1s=3&U^@ftYjM@;KezSAK)O(gKvJ(YrLp=`c>!wH&YW&SN4g+0{!c2~OQ zf+Se5q8}+YaPAsJfnMb<`D-F5cINDk!|*7l7asyL4eh!E*ab{I=ni_r5fwHbu-yZ} zroVz@m7*J165%r<`9q!5*6D+C;ROQ`r2`)f)@~glFXk7a&oaM-!b4po%=Tt%L_fYG{w+LqtJ;p2}1EJIFm zlj&R8uiX1%VX{8n3dzi0&~%5K`5D&1Eja2nSy2&<$_`=o26^KK&-VuDAS=xUnT+Ft z0MpjOU*L~TY1+X>DZar8?q>-m^=wqZ4T)#kD%|>fqpzh#kn5)8yZGs*+b>O6s*X!afUI6VGCocqhhnls%9U&ZT{J0!S>*u$WeLneUnE*PT4L; zOOZiRIhqga?66s=+ogQpFCT@YUElc(HKjuWMR(YyWX2kzX>weIb$MIlu$TGF?~xU4 zlxfBqm@nHhL=9D7gjZ-)GOf!=BS1D9TUZMH39=gw83462$_ zKM1^(8^pgL!VoU8v<0_Vn@u-dE0dVL^*J=s!V`Xj9W#tj;X!ob2E=27R-&3s4>|hC z>UzZ_n3NG=C%bXr3lRql0mLHwEdtBN$x;!-;RM;;j2J)bfkq@wCnEj&igOb|P9V;k z?juIbh?ncO;%)n~zs!(7F-T<)<{rQ2J=4Ong#l_q<+j9UG-015IlKg;VP{~*Z~JpM~ z(xiKehxn)L8%OI?*=^E_yL4SAL8F#lqU~N%t?YVAN15FuiQ=L{k;_utFlLOUr;Uoz zBTY{;JQ%izz!K+B7LRSindP_bQ7%pOaY!>A(pSfRU${)^s$#nn& zS76R3YwEUCr{n4|n?>>478Kf{zN8Jc1=wp|VJ31f?ZAHvBXnlUjz#4`E^;C6uA(_> zrWYxvV8n!CG0+=IMPD!gotR34imS;sq@7Htmq)Qjdb|^mwh44L2(Nm&M;ZzpO?};B z+No|v!h|9GXA4I}z>oGN$`OpLjje3Jr3FMaSh1{!^o!8G>Dcefs_?As4N@!rA?hR1fLvr>Q&xQS50?t ze*r0j&}-O&>-8nLiyoWMYv~9lyi1N{xg?O;t14IlF%{}oX1F&CZsPrwjB*$SAD`o04iTOkgS4AJ3#ij;gs&aBXtHN}PivbWf- z;T{u;QZQpToM?{fV`AzGxhoLhxyp#llWsaPI8gNPVXSYqfL4rhrZj4VnX#*;uwVk4-Iyu@;q$5QS0JBTx|vw~GSw)a8Kr zt*))?W`tyPX_#FSROjWwh67%$3mivMpIZ=oYSSqa&O!CMGF<0u@O@@K1$GRaf?Jwe zn)I@QvffeW&pD@|BJ{Pp%k*dkjk*JWJ!wEf7Nb;Q2Q&`v7KglGK?YIOkn$=I`$?nJ zy%@D^%yxCPMqV)gZJ+AnE@DUDf|{?=E}WM(-LNJIv_6kFGb+mSrF7Mv)TJ)U_&kIs1y!2{*v{jFX&u6G&mdEKbp9SUV>bA_e;8zScDX*L5PxPp(DblVC%bv<3uR+2o z65HLxg@NOsMD{EULcpvoK$_%dxJCX81F>xYuMJ$Z_4u6%Kv+l6q?`TQDF}|ZxZPDo z3t~M(NODwi1Q2*IhG{c0ZB7Xx-?7PM8q%kcSq9_VQ1=t;auru(txaI@|f#RNk<=TA~0&K)Szt0{t4qc(!C(mu0tmq;b)0afS`FmXziJ=07A0Tdge6 z%y8qZG|t_mbMG>peT&KAHwprpglf45l0*kmU5*f$valms7rzni%HU{r^y7vrV#5&t zfKdX!kmt}}>ua5*L(;nFn=FJeBXW(?R)^w_;K12gGxB}n0daUD--CiQaE%>46G=20r=bLT(Ea?;&`gxuvn_sT}eHbA|C@s zH!*KTxgmi5DX};|gchP};PgVv925EwBu!XC7206H8ud1)!wGhPymPI*Rp_W0H9K$1 zKPD6C&fxe$HQ(8>#*aS9bJtE0D*4wf5+ zvLO?uWF2?4nR`4{qkTHK72SU=tb=DO5Kc%rdjdYF4Ds7U875d|!4st0>lv5e|86 z%_8>%gR~F;qFy%&!9JL?7rI#q){++MJ;9KBi!@CW-tu zL1+=vl%y*e(A^=6u53suTC*?)hGYeZmcuIg_cIn}`uLBdCE*gRsnqp{% zDQTEt2iiHnGf{rI;mK(D9-7X6+=!2j}eiJl{0^Y4XVW! zCJ=!-BCKfnq7{#U#=33u;8KTTN~jWmHl``UxuVQRNQM#HsdLq5o0lKcx}d962N4Z6 zouP9{6Kd&3>xyXOx&ii`lxN}#BO$>}p^7`Pn%KCu0N`xT0n`?$LW!bGwlEV)C|{rp9J-B<)b59CIo3Vc5!kGz{uypPx z!bUL9(gg%E^;2=H7bldPUJ9m@iO`!Q2K&^2{pKV^#9LU|QK8%BwK!)ZTrU9WP?m}j zCX7EA7A-B~B^^C&EML>C$!(8)%5W!J-*$_)WJk>}|kdBCMJqQyUAz;$f^E zvn6NAmQJ&I19(PkycS%V8@XmNvzahQQaBb&{sjp=EQ3qZPy^5*`pGzHggIGT4ui*k3D5yceX!G`@)neY zlDIamSm;19+*V=)jtU}*rbVNK%c|d3mX~x-HqjY*p)tC=my;f-196AqBn1CC5q*aF zW*SI?agV41ea4hW6o$dLOZ4M{a-;x{#DNS+a3cI8=&Y5_=0rtn1QZw;^r>bLLJ;Al z!jNT!c=w^8xFH~rvUieZ&Y}Jr26^1}4wc8s*3p47A%rLPFRZo_Maw$WTqrg?sFCr5 zpj=4>VtfceIiHy2Mw?Fe5yF`1?6gD;+qH;WZ}-RnFSoipM<;)PSU`dPxHQtZ_WP4N zJ;2+Z8`V!?ToSv02tmB++pJ8N4NPWUZ4s}+5D$loIuf*+q}9Fz?^foYxJ0o+B}}F@ zfXX&q(rrf!PyUq~FL#A9BN+-}z~l^w43ko%fHm3(V0|a-pa|5f8=1vMAXC~fVH6;- zBhM2ck%#9z%YBo<&?QQ1U_%FUm!$G4rj*bn@|1#tSjNZ?rIDOgv@jNCe~* zT_>lwz+0GA!fm{q>fv>m){Gw36HD_;?PS_QT45E%lEk9kBf5tumUS?0R}m$l)4{6b zWccN}Yg%NwFzs!cqdQ8FAtOhc151}zRt2%)m$m)MOj%@EUJiDD8(Y#z*9I-2(_!FC zyjBl2Y)72%P?RJHw$#8WH-?ZI@XqDVR*toTH);>h)x(>#(1NfO62q>+9pxf?$rQz4 z_{r3;1MYMOszbu0B_DIJ^*&&~4tDYo-Zs=7;-trz8)py-fP zu!m0W-Fcv zxEY)k^S>TqA;xJ4Ck*`WO#2R`S3%}eSFy{k;wH=hGCC>?PpQE)Uo7+u7y21T)gh4moJT*;sB*Dl5Vb z{ChrLeqhG3D))(4)fXsa*=R)kH)uzheq)J-k_3hA?nr1zfrK&ojw9i%4Gt&d!3OAp zNL0UboAR8z9H`uBKgn}d@|sItkzqaKN6F~p8MjL=$XDe~>Q|HAFwnyX4kRy-xi0Ie zn4PhIS4`S<88{QTTa|6oiw9{WX7hRvSfJPdF+ZhpkMKx+k9jKTclmWPK(}~clHH~w zY+YC?3kK5?fno_e=LwdN&DGOO47Iz+4Kfp*xf&Nf?i7J)=^i0=5}?2_8oq8Bbr}Zd zg)LLYlk6w)U7=J#dlGa%E(;6dr2k~o)R1*dn!5z+hWO@kz*y>?=5?&^&Y@;YW`kc? zMo&kDA{0iHXxc7Eyo~^QNOP4IqRhD?;|5?Hy^A(v5=?Gs30G_8Zp5HKE2b-JF9DBA zIL${m$lEf-u1$cDosy>6LZVw!wD9CetB}Pn0Mpz=8vdm=^U5o>QCH5$%t@R%iA$a* z@;u6Ct;L?qRIW)!reYVRaN8KnUb2U3baf_j5);}JeW$_%me>i@GW|ypPh@QWz?NkE z#1fK-FN&m2Mk1q`UISt-(Nl#Pp5Y8xz=94UMJ(Tbd2cuUF-ZG^{16+Ri+Pv|kH+mN z+>=$RoDCvNW?uP-289BH6%OnZ$C42@5`u8TDcZ<3|im7E%(i;fC{i z&E0lVoHi5@$tYbmrGemCW3{07oU!pF?QpzoYM#0fyPG|rLs8DN1j9Kw{;FvyvJmcG4i8`#_(Vw zsLMEI6lqf|T8#QeF%04HZ!FuEudcT2TXJj=9w~drkd8cLZ6IYrACES%y!_2=JBdsc z8(F#s_*=f{T4sSM4@3oFDNgw2NG~Y%-~o+HK{_qi;|m4~2On0hJ2p0*N=j|#s!p2% z*<7f*j%Llh5CaQ-YmZ5@8L}vXv95zH3~z3^s*XcfL?wn*M-q}VL2RhN#hMn2zE8_j z7#ew*B23H?j5Ik-i-P1HbpH}!$;qv)brs&$h8-CJAtPmE1cZzhUc=P_38z8e={i7B z?1vGWuL?07`r~srFWTqBOqy$GubHT!C<-XFGwX@q~%pc`h-mnoO1rTr%_g7U26Hw+5jGm9mq9dWFc@XIozy+?D&hB2|&SqY;nfJm+@J-Gybxx#Fw&})j z);5V)dIV<*B9mTWM>{2@i8_NMfnMCmaKPpo;q^QXj4WMUb>IpAxI{92c%I<7qZ#Gl)9=S%fVUKhb zRGsk~DEVP;60>u)rL+FjUVzrwEOM5o)tJ zP!SH;E}U1E9XU6zuV;3QUzZVNvK+KkEU2>uA1Q3Yod9E+zQsmpBkg^@K+ z1)^h(RXJ>`Jho4}P2)_33R4O^nSYz8Pnjb_`p%GU=Ee_@&n|=!!XH!sK{P;wys#V@ z1Bb7Gv6K{(0@K9C;lbj;dkYy_1S-H#W^kHyNw&^A7RPI0>dmw~5#P}@##P8a<($Nu zb@Dz&vcPD9&$JT^;N%TlqS}u1c#7mxYC;JpimU1Ev~t|MGcX=S`nG+U)Vg8Z65d`W z8=}*kw21}eX(5yyCN>hhjgV;G8jiQLypON`5o=#3vQX1EOQt3>h>`2Xo0LPS!F*z* z2|T{2vQ*SIJJ0)46Q#=XaI?Nn=PJRr2{rBRFU1O!q);AA1MVd=v0$TfTXd)P3R9|5 zQVm~n$*ad=sexpR(37$JzKN(0kry{bFvAxz?;#>35gC#QT}i}u-2x}vpjCn#-YrKV z%@dM8vHBcz$>>39XE9uhR=hJm;F!iBOK`nVe2mL61MFIz@L- zL_`tFg9j&eB7ihSpa|sZ;b^lKArfpb5N#{Dseb_q3_5Xx{>Pxi)_J0{Zs^FA#vkHN zdc7-NUTsJsA-SH1DJyg|ky~BKn^M*!J2ulLskCz0KD)MYzzMe@(PwB8HZ8)Y9l1%u zYT9X9_z_Wzx@s7*01|OJpOP*lT~D^%u_Pshz(C^MmQl+0@;+~ilyA@jckS|BXMp&! zo0d(aGjD2CLxVo?=Sfy5Ck@S+1XNIi@>mX@^`%VBhO|=ze<75%qD!bP2SYqIN z`!><5qa&y}``ND9AB40R_FO#)Bc06$FK(GhJcCJ-AlIjt)B!i0{z!OrvNlJ&)HLa;ufA>}-VB8jLX~m}}tYut~yqpomz*CR*-6 zlFYdq#!mQkG9k5wsT5-Y{#n%vXitF+#z9?hCFAiKhV8LnWW`}{Hq6E86DjtBY*-;1 zJJ;pJ)v=8^Il_fUc#jcQpg`5eHqfe@IHm^STDj3Xgl8;G`B#tzX4#Mp?i**0Wf zwN5mD(^%(AslNk}uivI0(9*Yw11 zZDAy5j>|sHwI0j1<5CP7(%`cVbL~#p(XMzT5mJXSv0{jKc)-aCY#5Oy^jyE+WUIpI z8`*z%!?;wiV@Z+sn6xevPa)sm=tRM@ZH8Y1M6yC@WYjj`q?b@`4_@zzdJ%WgCJt1X z`y?;QGVFoL|2@DVndT`PyHBwjJ@?&r|3@!8V&G&PbBWSeD<} zeuuv;_JRa)1kQG;YAFaJ5_>YVEeY0%1nY!PjAJG;<8>b3u-)SD2xr+2dN>Xl06&5u z+ccm0D77iZ$8zm5nZi zI1pJyL$tl6O#`;SN`zSoMdW*>c8ZQw1(0V&RzI_Ul8v@WVN-%y zWpIIgSljN7p0~$7EU~`r+}EaQSi7(ehY{P8EU!0WF8bhW?b%&gTi5#R_@yBfFrvVe zq>d1Rqfo5cIEL^LZP5ZPzAlCG*42{ECIKaZ_3-K})Uftc0q!rmyu`w@#|GSaPV>84 zf@*akGi|DIq2KA6PP2h#~ zI|^a}>3;CF%G?$p8Fa=TGznKcs;muQ;niC3DJ5DLZ*zj9l>qDauyrl8t5jbsOArPb zoC))(mwkJa#5{5+eST?kmv&{+w`Bd@2S3%Rh!T0BJV9Bi5j?}}{yl73ZFzl7()GYd zxS_Vj`LJyD7U6kY;x`P6V|{cj#&kT?I!!a@e+pu(5-wAN=`P{`l8|gl>4HXDu+(p8 zn`vRO1WQb!vrG<~6z2z4^G)iZZ*?Vs;RVm&xor%mfhEo|Jxj4#hPVKlxOWCaMtQ+G%thefsw-kZ?}aVpvIF! z_Bc6%Q=VFKaM{KaonI;gGI)`AFjCoHHUSZ9a2h*W2V_oA3KJod2z5&Vc}^B4cEqrm znA{LTH=G ztgn425f&H6)K~&lf_dTwB+S5?B%+yy#I_+(Ed>j}wsE(Nu7HDh@GE%nf!r8ojfersnxvtPcQk7w z%CF;<>ynHho7{kEZ$xylIgw&dqz#*pW7`S&r3q;w19fZNa%3341s6?f?_4c--o0T}SYeUkhgA2%o8B_F5X^G){oO)q7f z`RLp}zUgHxzOU!!MnBwCKELS;kJ|sBmVbPMAC@09^8ZXjKfOU*C{;2<1k898NJQ?X z2Mw%adD(%z?Ex?E>>%UhkH8?XRd!}=xdc>&s~StJV}wEkOf0X&!5k2;otUvACnnUr z$8u~LZ~iXC1Yef4$48eqRpmHA_V=A;#~odB(yTLlXO7DL)zVyj{^BLA?GP;)&^8bMk=?ml74Lk znjxAH`z67#;rIh6o*w46w_+X%dz>3OCn{QG8TVj82-K;tA3~>UHYtQ-V-hKvTNBC0 zMFzizN?Qa$IRlyt`6t@cXOE9nIEyN;lyKzJ?(dC*lmiyHya?iU zPU6B5wM$bP1SK3a=p8|BmsMBP!=WYbQg+P5FdQ67vH>kY>n3MBKET_7+@-L4N3?(|tTHbI^>gPvOfIfgl8URbCtSb4v+6dM=^TrE*#_ zsB*I9Vjzvg))3{YyTj-g=+!=rfG}?$r6+(iWKEoiOf((~-s%(}d|{uk!b+eM5Pf4{ zv2)J>FGv%3#6-asqE)y^V&sPw-qBe(g0Bd}AS_?;dXV9C)QPQLmXkpXDQYV>gB~yJ z{_(?WAV)xztI0v$Sthh9nW|&2ghA&f0&METNEsBc6LPXkHixrQz&`+E5JUzAsYNcT zlXqU}eWa~R9jprtXp!SG@|UCqcanM?h`lX9LORYwGrVgALSHvfIL^uxJKU~ZL``|B z!%ec4wX|@o396bfIeO=C+MKu&;$M{@y3TD#;(QC+wN?V3w*WI2IiN|$X+o>j!ETqQ zVa$)6iSV*z+>(KT^Js8lZ#s`Ei>vB9a^!12YgL#Ny6pObkFceni#s`&zFhW*^FNKu zIoi{JejEPpjW>yif6G&YRZz!9yu|?FI|S-my3Pand6m$AlMbydP7A6`L%wj;VbmW> zO$&;24$FB5Kk`=UP{qc&O><*no-cF_GpU*#VN53HpKuEq~*_t07QAzKy=BQ8R13Ahcu;yX!MP3 z?Q;Fn6>}+1@IHt#?XL=)4EnRZX&2qKr}7?$Gd#`RxNF(pn)c~d<}Xb<`ediUZImSy z!Lt+anYp=VNm-7-#EjaMVf?5)uRboV4`W}sO8xVg9)ko|Cb;G~g>_ok||0x^8gI4ZI>%4n>F#n^J0QI#2$w$pD&ut~!IO=#ePWEnRt_m71Cg1y=mSjGt$Yq#E`L(XMDXOj|5+e78n-!jK#e~ zd>9t{g4mhC^SO%$PwlJ&>%Cz#$%zMpvP+V2$fJft;8&D`hPr7$Dh!i-2DX8cLWnJ71VQI#+NBZ} z6wR`}%`+HsGG%T5uV~jE*R@+U#OKL9ZAwTUxQGs0el{1n^&RqAo-X>N=`6AYt)j5219QTShXvelW zo)mE$O5YS8NmYBHMx|PP}oj4Bhpk`jjz(ebaZbS z!lDf2Uxug%DaF+_%%@wOLi}=1lEz)~?{*LR7V9UK&7{uBRvRX1ScDarul8Mx=}v)ppQqgB86Q~SLyg7vz(&P=Bk{EgQD+`i8Z8! zjWpFm|0w-U#HU?$7T5jIDhW=rZ7H0+h9Wr6m@%gQ=+`O@Pey{S_|>eucA0sogfDKq z!#qX9%$Audm~0Hw+=cDFD`?GgQ&BeQ`@otfdY9NvnIJlIdrtbU39xfEgcp44EUV74 zk~VbAD#y_8nBW24X(^{roV|zC#a&(5)&ISO7EBM@w%$n|pXgSZ6gz8DKHRN-!HZ^u z$q9z(YM!%NKnqb73@BP}gVx&=C7FrAv=vS@|Aq=k`v%y|v~eURosD6gm2F@BAZsl^ z%-r~%3XCW}J?-Sl+9Sx80!Fl61Wp%}bb(KI2?M56BcgLvuuMZp9D?Qt*1{P zArdReGQERbvFP)a1=J$5+)3)u_Tr%vZP0%Vygwp*aI93r_PMFKBVkO*vUvcq=uwN7 zh2~*koxAXaw#!Mii*F)0e*LM%xR0kLoWWa zrvIL?l5AnJLDShNVBu6#uI!4=w0XK+ZjezlljnyD^46#O>57<^=o>aIPvW8lrT8B0% z7wioSMZV8aYFMam!Fq2{SVz7EYcXU?ZTEH%#(tX{ryoLosD!cI>dDV~yjwoeBF&Ak zx0gak_&jdOVv4E-L$=)%;B&m#NjyeL)Jn$J<=MmTx`ac_hZ7_sRVf$kBce!PdL=Il{`9|Cbtu>Dw`SL^xu zzLUNv^Nz$SK&)bfwhs*A9tG(NtXxA~>gB)&PN05-{rIs*$5{k|)LGNKMWu)OZv+x9 z-a29pf+r*b6e5Vfqv7(hq42izi$>2bx)|*+O;$R>7^cvnX>yvs5rSA2$Vx$uSt?u! z{EEr`q;wLd6T+`DNDa*SCl2zPwSfQgqlJ-Q^M0txp+jrrX>hzLH;gGUHE5gg=VT@D z%?exdCUEeWuF8mv#QHmA9*iFF`Mb_7>I=ss#Ry3Ec|=m&3=_?{#_Og+7Lqbx8w_;AJDj;uUkF`8Jd zP6>g)5tZpNth#nReDEV;MWh*O5p}Zht|>YuF%oMmm6i=3MmagBH2v~r*K2K+!{Fnr zWbByk{MqFz+O>dow$35ynk-DV+d`}Cy>2QEfV|yT%2lDaswR7t5F^k^kvI3mTh`>Y zk>NiBV^Lu5@d!QnCg_-QaOOVq?+Bw0sZMrVB~^F*32;S2a#%~c+ALRAk*`b~JICv* zXcWnHt`QHA)e)UZ(22mwdRalEau^dcBj3~zdE&jS z-6w1^i~Bzl(m0kEH?{#)BzKvk6+ECWX;@Svt_h8f{H2lEz*#2Ls+)#i+`AdrWR_`Mf5pY|#43&qIhwT9g-8LjvUb?&lklu~s%%0$OI`*e$T9+dX1XR{pll zr36_`g{oR2iABLM$xx6?wcZg0=SRUYLaN%($zke@3IIpWWL#rDG3-9tE(9c(mQ?UA zy7LP(rQvye$h@R`cEIoj8yfhA8uPi?oY$(jEwUA7BiaN8 zEQ94==e+PW>+DG3!Acd&5~6z{N{9?-%6?WQK$0?T(uAaNq!mJnb_(wpVSf1>BXD&*61m{IUaQ{WT%SESuWH|%R_Oq=gzIO;|dJ00;R783z61?2wmKPdO zi9SQ=2|KS8Hn%p4Ubu}gX`FMV=J{FYYAA{TH4IW^>&wxah&@(17=%XgBV<`5b0qkL z8NIiAL{V$^LZ(Q4s&Z5L-bi}q`^knsYlNFhQy@ifKn)kja{MeZ&j};OqqE^f+Q1&d z;(}nkU!?darIApVTSRO|PV~5;Dx$!Mc4@f0;DM0-bP5v)_J>KEEQ0$zoG2TjJD6(&{MQr!d28}48VR=zh@gSsHel=qNYGjURdx)_Km}#F-PO*p z4el6FcZ_Gd-Z;#+yURky=L(Sm`0ytEnY~4f%iCNo*Az1W1_=8iXgtk&R4^sh$c+2I zWpdnvwYZ0YHVl#21_8y~<~f`foK4HDPuo(OLAIgzwPotGz*zJs5JoAiggsg$6T&Ty z)pyPtr5?#j?iB}25KIA<7U&4atj2fKOuE)#PG7^ z`^@9j0m`7?@FfmV&S94V!%?EOnY*|;tn=j=gs1a2m@48_Q$?#n0Ju#V- z(^E25l?`B0USV~&L{yP@n;Q1u<3FC7hfx#ud?X$MqKq}Gm z$5`4oIX4Cy={C65upD647_u>}Nr)X_Q(F?OgvtR_9^8*Zp*v7`7(9tXO81bmeIr6% zoh0XfhEk{i)bPS!5;v*6L&~y9Q8G?mzv^M5Pz)6pM#zC`PeuTz+3`|b+oE{z?%#CZ z;0-p_IxrvDnIjT1$ROR@_IdEB-73R=Kknx|ICjcBp&sY_g8R`6)v&!5)M!w_FgO~Z z5F@FMLFt&_iNKnNLQ7h?$Wj=yKlJn9K-4vdUXI=!Tj7ZV(nb_ot&)--)_{gPcPkgI z-!*5{=$Y)UNJVo>sz`B|@=3=<-iy4~qw~4XdlYOyW~emz_d)^Y)b@&{RSyt6ZX?*) zif1@FWl1V1mWZSy)vuxUYlRRQl^76I-Lay@)Ghd#;6szXwuP=MeOLiWz=$dA=8)>< zu>!wbWc1oKY;G74tAq-6uF}rke&Sixr0W5lbZC_Di~oJBQe(T6Jy*=CvYIzY>_(`U zoX4fD|12P#s!X~rE)}059qPG(L#T8nU?=q^ZWeYCDESTiw$g2(x=V{1;DW(uxxry* zI)PK0WReb@0$wmDFVbNH<>T`iB=u46CQO*zm>US+v&Vy|oYtqjY&hppm&RQ@RGcFt zJb|1sL}`bfB||u--6?F5kyDn}q%QVzwMvgXgk}$3sVA5gL_9)ng0F*!)8(fY?&E>F zRHY&%0ACF!DwyhIC+JrIpoq)5MTl-IQ#oq=q9-MAXibOoZDl|LH3s0iNtbMuhX=B1 z!RZ%?YEk+EvK>#b(SDyKj)H7U{RjJ02TV3)Uj4Ro54zcjU?Ps{*7H3u@`A~rnFRm| zxVM1-QcBG%L;Z28w(!r?`6!QU1t~;C*P4<4l1E?R$cCk8F(@dj^Vm*I}h6hKhy81m9|cJMShcU?D4} zL1d%( zI=>Nrv!Q1dw4TU7x1+Ezsc%Xf)IecmDP()WEHfY?IRSF;=xGLJqi15KNWe%zls?_B z`xL8{OEjPNMy8?Ph|nI?>o_+PT<=hJV+^8zZU`4{yEIqhVu;geInGjKqc-H>gvZSA zXKy;cZgNF}d7K`t@a7SsGgDc|y*t9SA^0a+V2JM4sbJ8GGG#_Ruw#rI;k543YQ`Bl z62V;|fI@|)r=Ej(k(|>u<-%k3#HplVfjG?1&dFY|q|yXySqiqY+Lm)a!8Vp)B}>7D zHYr3ARedWC^1p3oW*}Q-WUqqEl|u$Y=E^uW%5-SREh0%%DoPN{t3XCr3y%TDd#EpE zraXo9H+F(u9)-5^QO*aqm{&#D6e2GWUb+nCa*8y_z*%k6V7^NVbB{F#X7UBvVZz3Y#s9dvLjX@|I-!=zjM`2;zZp$gtrk}Qu1nO z#ez2iJb-I8g(8=51gCUaBg!ds-PCpNKr6ab65LEpBKA~m+)tyU=El*|H4O&))gig& zXU0{BWPh6~%nx5OTPlzOdl1-WGTEKese?NX6fFQzABzz9VmJ>Hg37vWz5rF3aD zN`)7);jXZwamL+BdL8NVyegQ3zkS>0-{Rv&cC#UJ80~xFfd(tyjXgPOi}2ZVyqLBa zQ3EfTGNr&hYc{*lBpw+^=8X*joV9*K<^-#*bI%47bBGZNMIT6Jl=VQ`1Us=gE@V0} z07>ibwuC_{#}UokkkQV6tp;V8h0FrdAWVHYlPiLkck|KOGqM%WxSb(}(##Us+vg|^Z!tu|<7)G@)0OegA? z4!~QW@*=WMpX;fA$f+q62P1zC3pd^}Aam1^0d&vcdlvg2PFop5@>G4S#&6qwvTu~I z1&1da*T+uvOKz3Z)j#WE{0~gH*8b4|Ir?c#wd8H7F9AxZE2Ln(y1Ms8L zuyj`nr#63(?7tvh$I!Yd852F;i6*3*<*6h|;nNOj?n_?@$k>SR&UJX0N2P6HntB;Cp&lQ6=! z(onJXG8J5d*IhmbmG^K@sH~f;PZ6HE5uTLU8@4g;RKQ#rI7{#_ztEIkDnkF@20zRH z4{|rV=<_3X?*EV6AnL@hE-$-&+NK@;*w?!DUnT7c8|Enkpy9Ya_P;7j`SDW~no-G9 zYe)AmsaVQ<`QO^_l9xOB3)xe|CW@dX8L6Su;y-<$nG;nLi1mXF#MQ9~i912@#_~fY z0+$o<%&@#ITa3rkB>B3KvbzNFt{wjd^N#-_QK}~dW1unQkxVH~L2=?147j}kPPZP? z0wzq-F?hQ&$E$lBv%H*092=Y%^&Y|u77D%PSZ}AALa@~t-SSkrx^$1R@pgT_oeJU( z$R|`5Y@7y*=_##Bz+ZL-Z@508Zjb@-h%|neT70gc?UK+$-5^D; zNks6IH-|C@eo(%K~^*-*e+5=~VS8B5vE+DT$2(`h?dSo0i|L@t*U$yXwO3b|VVwzKF;J^j5;Av0k2;Xm*=jV9x3 z!x?BY__xAG;x3Q*2wRhTS!L#DM-twmr)AnNGV`0H(PP8!`euNM7m&LF5^8jgs65Nc zbLKW(c}u+O^Z37W62AJUH1HvFZHTIytcpWNBbjz|d&I8p_sKY@d4zfvkx)l>g$)&M zev91T)~2bHq~UfCKKif)2it|uW2EqDo3zKxUUs_9-r%;&wLSSyY%1q>SH3#u*o(6P zC1>bNxDA~$LkFO;nHA;IRKUD{9DLM)Vi07yfPOOG1i94Q{zHQdC#aef%Nd(Yv1J6w zi9fW`@@ai*n!^@;r#6t^bSyA$GX=QR-^fHwTuiMzL$gp6+jFEtd=xjC4 z){}(>+LYxbTA>zVMsHyWC4MO~2?iI7vB}KLX80cl2G|l%nyV{`$jia$1l~&_=B#li znCLN`r&YL|;Np9rnFd}SH!+`8d=N@D6n32yK202s$*vxgERj!1**eNy82Lgjfjr~;y`5= z$g(EnKyDd0hD;1^QeY?6g$xs7n!07}9I85SZ|bFhZ4JaS@OgkGBW_LK$5Fq>(2|WWrc*1_R#fG6ZZ1lb{;du#E}% zW%B_hYymeO0hjN~c1a%IAbY(ze>$*-%wF$DeFTmIPa>Cv0sb|DqBuOdhIt=zsOOFZ zP}k2|)QnNWF0^4dV&a;Bt!}X}88Pf0p=(uhNok16hUOygy8j;GePO}Fj*&pQ>|u#B zpk+HE=b`=)r(6`;ALfxH(^u0Jl8~9#&lzk90IQAykOCCj6j-E&3di8fY&$(cMztOi zP@mSYn{%|sDO@1W3uEgx?N0)A1b)3nP-(pQuI-&bN-(BwDUuVlgGudJruKQB3?QQb z69R_XWHK7s(Fl%qyn1nFiwQdI^HeTpyiq9K*d7v`1n3Q=wi7p}G`qMkRmsDY|8YHo;oolq7wN)QsYS+?Vlw(ilQOi zVft^1G#6ANF-(Hud>Z|dD4tRz zkReVbkvJxiI7W6WF)0^$@e3pCs4HP(n;@0zC@l&a^G#fm6aq(=q~0aPkBzedw@1QL z5ja~ptK{g9B-;}vTkr`~?F75nr;G3fv8Cdb*dEd-=g!z642KM%TzAW+bTd?k0jc_<(ih$vWx?zsV0k+A(d!m{V&y{caQ@xtVTU zH_TRSbve~tFD|(MH(6tK)-HDtvROYIaIkIn43Cp<2+6a!ll<6SZD(Hw3Ejsz$_St42+q2eE^MH>38%U*5tHL{8+HW zQh4J=YE3JUR8OV?iwf$pC496^G-L;bA15J=V!)*NE6g{cvdTJgG-s1iFEiM2%y-7E z%za}%g@Dqk!~GTj*i`2^-DPO3`Xy%8($)`ATZBUEcb)!K_BBTx?QS#wZ?|sx($hwD zu?qBEZVkm9`QyQ6>SI$2+>9Lr%zK!6!;ZJG@fH+)(9{`Bc*N;0N<9B@V^Rl?Uyo?; zvRFL58&as~M}1ZCG5!MskeXWS|8LgM# z)}X~n78nV9Ru%~P5r9%z&{B%C9c|XoVpGF87uqavm}H@ELxPd|6yWuD=;D!aPqrFl z>o6;GWb|~|kexd}vA*i++W)@mGOl;USoja+e$GB{(~b+)#y6+LXI%O{v~+{Uaf4$LlKF)0c( z>F17Jmyy|K-n7c36=oN2z?fjGRXE&i+x$-NMzYFRJI|avq-DEJ+NK?QX&Dn<5)|n{ z|E{a<*X9{forT!9lKC-lxZvw6`1mUb429;Qg+N*6fPnU@hLyMcP|cG{80X~zc3-nE}vkbnjSr?Jcp;8SF**hUyL z*>fY4!3K!c@TPAdR5$hxTPXP(KEI|7v&pK}rD;Jx(WNPMbxsqX!fj|UyL37-WH7Zm zHc!Htl>b-YKb#9oSF#a^IH^cYTWG6Om+^Ks$$mo?u7S-yoiJBPJHe1ebPzx@u!?L% z%|OE=uRE3u7u3edg;WtADp_8WlE8u-^R2SWplFC*<%Iu$s*`29Y`t@MB~RBbJmEw$ zu_wvI*2H!u+|iD0+s4G2U}AH}wr$(CZJhk@Jn#E`=Ulz7>e}7)NAy2HV5LjxO!aBvNjPA9+~AqCfG=1#|u26m{vT}&)jiOwpAxfvK1 z_AG~>qhGkR<}OkjQW?;wNxqZAKywt1^ByuVIAq#uhNDi4ufD88Pc%!D2U@E^HMR~& zk{Z@P^wK(}cej>(vHN7>uiK|aAw$9`J7EfGvegMU$0}cd(LVLDgwr!spLm;FK+v-B_&n?;fxdGazV!C{ky z1p5~0C&e8NHw6h%G@)hiy0s&-}Wvjmy?g`%JMG12#^wbV>P$m#HlHJGF>`gP$`s zGGQ8wM7D-phc6J0PiauU9Ob6jcGb;=RY-!mfa%e}9^kd`oQuegroL^!r zX#8O%zq2nxM)MPOQ~X*YEp&%+R6;xJ>8KZ5j8c|K9;^dmpoh|CjhcVA?z(4h+|4(f z>Cz3si8a*=arM`?4_jwTK0_|}d3|#;6vHXn|7cLqZX{YOwO#P=a(7r4Jx#+T3nr+m ztd|j0g`*<;G?#QjJw6Od?{P zeNYGl$ULEb6>>`OCBka77QkRf7U|M2>cQXK8#Sf$gNFWfGS@k<-#+X{v|MvV_08Lg zbHIJS@@n{tCz?m&=EB|qR>xati(HeRCr(RG-@FcLeUlWcQ*v^zw0N_-K&6K@BkE_F zB_pm`{fZ|`x7BnRw`!bSkHKM!9T<;;l(6T+;2p3?)ht_q}wKx9)^Mau?@2)zW(O;n_#Zl!-^7%DsLI}j|zAr=}H18;k0;x z%Z@Qj!Y?)*B*-MiwL%B>Jl&r_E)(U!>s9|3nyOGJZI73RnF0Hsi?F;-FE*)0l+t!A z_l_>~P&(o;;6Za8z;ng*TbKlzQa(Dn$fIp{WCmadH?1 zR7F4RQ3f@a!GX}jMB6e8$z;TdxxVgs^44dS47VO_jfG>c0`JZ#)8q->rDn zTQiuF-?Wb-%hxkCMme|jM3gAGf8=gDJHq>r6r=C|@r?@P$|RLlcg;=|{`Tb($>|On zE|a#5GWtm;qoapb$Oh^vw&G_jwqiIptY|vqCcsaOBD3`oO$dA0JOQQ3rohB(S@3N| zUU&;BU0;{krz-Z82#RpuAeL4a?3;-)G^Vw1`p|dh0l1(B@s+UIL*MQ(65OzRgid;W z)sZdjSl{iruCvOU0kX|TaSXGoYvw|SQUxiwHebo%`KT|4V9_`!1N%Tglv*P0FC7m` zw8ysK{!11~5c@8!KyRfiw@tTYfkkw24bl&)tQP@Dc)%Xo)#((*pJ*}Dx1lmwNWrJVn704s-_hRAKj;T zjiuE27S12srsLDoyA(goMDb@L?z^NOKGmB5WhkQ=a1bYWSk3ycLN2>xX^W|jLx!Oj zK)g(y6pGepkkX@==V!xy`wEK3(4CmB@V8OPHBL_s7g%#Ql>KLtm}dxPV@ zf5)^qJ6R4Ml+v&j+mme31q!T+vE`OdOvd5z%`_etv*og}M0v=*t?n=eJnOUep#e=W;;O28a(6T&=5A$N;!fMoNzkU#_Ts{Z;R z8vs|VYm_h*Q}W{;m7%!0$r26SgXi2SGh{7Br6|R2Y=WAT3Kx|x2&$S8$MHVGi3I_#sf$}Bf_X6g3us(#6M5gZF?DmEhk7NDG| zNZKm_?hQYWVprtayrSr~*u=wrQq2s3#j_J}TnYKY(iLqbWL4`6|GOc#@|&HepRWvh zVOc%dy>PztbhhpV3C z&x_wl`6+tt{P2W-#uuTKF+@bWWgCi`$Og}D8HI$|wWX&%5R^6W!25)U@1~up3fkZ} z+`j#T9h38|&|ZRY!{}0a57D(7bu7U0EJ&1Dtv(Su!22k)M(kZ`RV+*8+;xP1tz=)>eaENAa#i_(D50SiqcoCgZkI2+Db;i~fQPatD-& z8^Qm2=oxarzjYo_@xq@#TjSRL^Qmdjeum3VX5&o26iI-+Pa}f(WkndhiQ`S+@B$}6 z9Vin8K2;kndex*UA+(;~zxpZ%PRF%|8i`OU%^bwDow4fV zrxlxNAy!l{MCCm_#cli5c(joa-Y-4nQW6SKQ{_<{UrTNqRB1$R2oF5(;N9I71*82& zX*UGh4q&gn5!1hA&3fT1XpdL<^Q87wX+Tr7B$6FS+Cd^HfzEU2gCHDi|6O#=)>l%= znHJc=F%MzpzykM;@m7@L?W7U!Gb~9I+6Y>Qr^42Tfojs7?&)w#0pbHLkNKHi1X_gs zcI=(H(D5H0tr&C2G1Bb!Pqe*e1ECw&b@kKC#IWTWB!YtQqwW?J8#G+3kzxyA?M}y% zHcnvBT{Of!i%(UOF!q?Ly52tY-ntT!$Rv^(P3w+q?WbGdEo^wl}W~@3E_W=Yy-}qNJwG0R( zNXVTWioON%{OWkhHl6AJLF51nuc_C$V6@mK>2x(uD+}4?e76>(uM+IE2$9Ula3qyZ|jI6R8<$r4Tv~ zANRY;jwZhmS%n1wp6TjLfacG%>;Vb6AT}(tPfU%dpZC99ucfi?yD)5uBm)r`sl(~! z>t=X0FNbz7y)y)lpV`EI^t(jQeW6Ut%Gb=Dil%cc@}aDFP32wo6CNY}nNr*^66!$v z9pVfb{!(;bKG!rzMLLA%SFvkT+@|F6Zbgi~`U*A*Nlmh)oG)7H0#~eHv(aMVZ>+hb z$2W5~{ zwq<6L;2_7xPH#43o`i{p&d`+k?EZ#qa|hr$ANz}wAKyhryJ6nm&Q#Xry^s@y_TXR@ zn$?pI=fUt0sR-d(3Xx?(xgHvaQuIZ#l5j1}0k9*A@D}-}XPQunT#{V@de>+X*kW3uvNvtq*39R~CdA~@C ztFgJZd)wHtz5w2_Ck~!sH@`C~|7Dz5@xNA8v2Qou0zV_Rgm#@8xf`)llSen6CTl?@9nFuwwsu!nB zTZ)%c*?3Zk{f3#pdi;vD7EQHet(UvaL12mAmq)fG#P4DT^#TTvvcge5`_3un@?en@ zk#=#0K(lk{A))%}Q2JrfUU>TqkH3<}$b$>~s*lzoh4NjnD<e1<&Re2j|G#F zM7kwl-$yVnCv*`l6Jc0jqKrS}p4m)7#p;KAL6l=7n&4(h<2%1Ri&( zIq;6nge1u5ast3UF$FELJmrTvkUth7rVZJOon;v_{~8uMwc?kJctRwXm9`yE!GXOL zOpnYj|M|d44k4mI?iajt)uozxgeg2!vOpxdnDsAB=|4^)D+40KDpOLT0=isjB;H~? zJUyAxOQE-<(ukH`#7^>QRzfI~6QW0+VKX^WV0Ie_Au{?Y&&K?zY#X)^D@$aIUJeM5 znT;_9#gV~!iQi%@oa%i;%1~6p5`LWi5smyTAR0Klws@kTCgCI)SgvxVkS0+-B!Xs2 ztQA<&LG@>T6g^itH1y%ykT52i1i#sSbTtwKgwZNV=Z3W=3VzlZ?S_;m9t$p~$aI6y z6ParsycwHy}Qh8n6I9UIdyQ#c2(ORkBSlNI?Ce)Hgq~23xa%k|SPA?f_8RP5%wL z5CK@p&r4kjYPVvr#I%LNM^-oUF8b-XFk)oESfW2B-Ku#~;J5|t4N2EAvT10OQoCp= z`W-kHczU7Jg+0`YG(yPv%UCp0_7payv1sj-3KD*;(pzCXW>5)MU9aux9gQhkp|x-`FP+e zt51F{EASsJkPi0CSBHGh(q*)lTViUMud4r>zo5Kg}BW8CszOKn@^bHl;QNQuuCAQ_$fDJ zyF5~Gs~EVTiblSJ`$vDTN(JF!I6BE@B~mX;!OP%v@FJ(HG71Ew6-_1nIBn_!%3fq8 z*W$Yfm+qocHpq~XlSmh>V!6a#i<4>=O*H)Vd%uWmVC6k(3RZ&@g!RoEBw$nrlL_&ZbX*7(pwfp1HzG@jq6l7a=AeER7<|s z_(xhjbC@|W*x3q7Y(GzbND=>0jErf|y;~K{6ge}Q=v&ify)*xVFr5_h-+1qwoF1uO zP8c{NWf^QW;MfAhdZwT0u$(Z{IAa`+TE^MtzTigDTy_dogJbQ?ucJruQ0>iX3V#;3 zFupyMHXy=DbfpIP+ee0nGjK=9gAMg&!@bVHn0j^M(W2aVtS3u7wx4L$q4;+)|4Lv7}U{ zjCUn{PVgh zO3<|ez7k)HtLL2>_g>hkmB#b>S(;f7|q}n>2Gc`h5VHolbH$ljBqA9!s<(7Nl z%{Ug%Tpl_7@9#u0Y(E!lwU+pF2T>yAdl-pdkVYq{e#ratOn*WOnrBIh|1Eac8YK$H z8YM39*&yVTdYHuz*pg3RwLh=|iCU{4&uyK~QsKBT`tW*LZ4fuh*EoPtHLnAoNzo0w zL^A`S5PlFvTI%2hcSJTBq)g_|@y1Sy_Rvh0LIf$<ik4joksJz7>H_5-&p#-ufP$`_d0VXX2dL{9*H+PYo!%KQ_h5V5zFfIgJrSNX* zlK@CT#HX01>{HH{6UrOsI$FvBpByz6@QQhCN(n2`8M;Q%@Pbt`d#zYxV`276f9ccD zc;=tTEHsfzQL2=e&X7gELECt)<@muc0jdX=S{T#}dW0ChCK*uu+E$fd^X>79_Vm~Y zkk&nl6<*;sen$8GHKZY z(k|!;yug+1?1!t9IAlTDLwbuaep?S2MgAg(IV9K+)fx;N3V)VFX+22y1y<38$WAs3 zWrwD~sLyEMnQ1Ghl^1tmW-xk1;%7%?@;TWm8nc2TiC2+?&lwLa#iOq~afhS0o#>FC zAf~ItL@Wk5##e75KBuy=Kjt&HMx8=e8EP$Fl7qV#4)|! zZ~5AARr)?A0}35_XeIc_v=oG&ouNUoe83okj!sJ@tM8d5P)M2AckI`{dzEGx;L#^R z*j*k*OQinyRsN+hLPQJb%iMyji*n3Vv00VT%saPzXGf97Bw%B*lD2;IzfxZwqZ|33 zGG$?qBQg9rf(JXvx;QMNQX5tjoC-Sg*^#f*-;7E@(Icg$6j`{uRcD19M^ zH&TBS`7`&PA^W=Y^0b>e{of07V0l8c-%PpSXqzcwJp5XXR5P+v?f|^5XNHhm?*7S3 zxTa;ZgE2iUGA`u?!yV81a(jr@7pR}txHY%!=Wk{8))}4O+btf}S7e84&VasaX<#AN zf$tpFyd+hcWpA37kShxSmq-0!&x>J)xMQk$#jhw@d@)9v!Y+AB7Pu#;7K$a{MtqpC9Nx4)m2Z~kx+WmutE-sNv2 z!dYrEDYMMZXF>jf1f?Rr9HVZ`Ib^cNnNjT&BMEkSQMz*hcG(<%-MS<`;xmq+!uK%tz z*0Pg3np$glT7nu&QhJ=ZJ4RuNKd`U<^XGAe`q?oQ*n7uY8r$t(No{#h7M|-qRV1Yr z_fX=YM`2Gndt>=_NO0Y>$TxpHzWw#FP5TBou2x08ZBmwV?Wl+xb;rhx- z@7~;p&CgaQ8dThvH*xP2TGz;;!r%TuAmnK&K1MH7STGYD^{7uKrBCu+Y`6oFnee2i z>xBP$vjVl7u9=cBkX{{Vm7Sh}x_Fu#EM%lBY>$R6C*t~aa!(s|I3s-7VP5)fRpyy^ zFbC{)UCXvab1Kr{nD7so@qd){emQ*VD$721(RFUAhP&MRK5X9M5S|V!rBTVK8>+QEJ;vv1EmS%zniN|i z+Kgiud>yP8b#Rf;V4ClWlZc^`qw>eA`qVDJ!AzpolGN+&md*IX3h~a|?FiCQc-v5Y zzsYU*^y_aP5W&%5z5;#3aU#IbX4Qs@UFl=0iLly{&Y@QR8=%gEejuk)%T~31Xnh-x z!4SpVKGhl`Foi6cHSj#DLfS`_7bJdnma(%Q&Ul&{Ml?VF+OBjYw zZS^)f!0R>X0InT>1hLk$FE|;ZrZq20XO~bJ3II;Vwr{BrLv#)#WCX!|L2rtt${M_2>(Jx0sd*Z-7=iJe&tw1h_oNP*(KK{U&wi9z& zSd#vs`+j;It!B(o1Ek~UqA`5>3v)*Gk$)(TKubQzKwzE*rlwr0Dpjj$W?inBhU4qvs|=#rHnnqB>96|2)aMewl+o22${Y^TzRc7+hR z3^*#R@03*Av%h*@0jbV}9?iM?k7Sug;tkbqt#U~vTJ1}YyRKzXBp2eahmPJ}rIJ~q zS#@x$Co|1w7fIgb`N47M>2Z?o+BesEt#*v5_AzPctOsQR&LIg>tERo8IjF5Gbtg$n+(Ww}~{5m(!_07syfXk)e z^KfB|T^bXSsd)C${3vsd>|mQF{7KUJ@bxphKLG4XMBMhP@JxQ#?!HI z6bWk6_1l(P!zA>+CyM5xnFfq2G2mtQw*=2xgBsUuS-H~j*qX*&^X^(+remgRUIHks zpCD~RrS?S%3$!{#Eb6Ey@lPzm^}wvdd{is$ zy5PY8+_WYhY1s@94c{`n5FregOjm0Rp)$$P;3|s(SCk2Z+RXh4APY3+DW-Q9f@d<5c1u;tH#u`q z)GeJ*$d#P+Br0lsnV{@wPU{ndul5045)2RL^L$BU#QzGU3HVj(6Bq^3m~VsD$1nok z+~06d|&FI*G^9aRL+>6PPX8j0Fk&MHI+(1`JE+(;l28N zYVIz6wdgWB2my>3PB;j^N`zb+#x2i4a8I{|?1wOAgaA|S{6*UDuIPiE?<@b|Yy>2QMf@p?O+!dU>?qUNodl~(F+HH!%%9DT6 z$0?6&v^0XB@D4XsV1!d$0!&d?EI*UW}b_ke((O4AP`BAm71TZo91F*r6S^FGdVE*Gn$ECPCVqyYWN z0&W&-TDNrh<7sJY%+cvDY6Ecr8ghoN9{ZjEVgW)ee_saCE-W~%h=8waMKC^+^Nioj zdCDt4Q7dw|QiruxEkJyHm^i^%PVcYQN5qIRxO7Mf@)a8dc<)cIBhtUy(GNYk#bP)a z>MCQf`YN4^&cI{9{e7U%JbAjv(P}6iO=|4c(USLJub=2LRNNYCc^9orQ$M|!a60r} zd6v}`1Lvnbl2~b{^QY@g<3i85jtJaJ#>4aDmPp9xWtU}5fBXmq4Z)cjb8PjC-06#6VUNKrt{e0$qIc0d zEw))sPj_RwQd3)&VXNEUV+87bBFpXblVv%0%mxn}!D7;%K>B^yC?FD^!f4YsfGq^l zU;gD;5FsE1Vw4KcQgYD`U{i95EmqWKE-Tir8QDIc){sIls(N*R_~5u-E?mu0ibmO! z-yTDL2%|KJF>VA;W@Mgm3Ii!T32m%J!m1oIA@VxNcMij*sBt3fxf|7aFDu?#eYoBH zQJc8g6Ci@pz2p6Sr9JZl5sI$#^kb3h@v_GgMSg6CxB<`9Q+@%M&~F^G?D3?sYSkH5 zNV`Kb4aHGua2H4&o_baDM7G(X^O6_zS3_Z!L$$^_SHwN)@jGcQ>g?Slkvg?Hl*b%Gsj2Le zml{G+>mQaJiWuCLrty@lIO*v8ExKSt*e&?&HN_g#zLn{h+JxbVY9D-8Tq?1bYn9zT zDz=+}Ns6g`dbN$;y5hw<(p~pS4aOy1Mxb^X>mm^zl^JRWueY?iY1Ejd5)scK@Q1Cu z3?8LhAoVIe(@ijcZLk=w$xM{|!|hDV+<7NsLD%aN6KCsqhVo`Dd^qN+D^Boq1h-YV zBv{%IC}p~iZza>uv>Ag0U5cb=ia3_XIcM&Wy<7lY>ZToo5kzx}wiSZ} zZ*S20x|ZxwwNpS(M^eAqmxWO06P`(eHP_S92TUk*m&XK-=Z_H>C=M-!zdLH5M`5Ay z**niwXqU&4V1alF3#?S6#lvOvJUlqgu?C&uZ?*?O(^#5rl9VhBl&AF8cAc(>2^VxH zvA(a~iaL?M%GVKzV0WHYFR4YCoV0A}r=43?-nMo<@{%(u`VAj?LBXbt={{2*AN$`c zoIU3MJ?b|f-?y9ByJn1IC@iq913jlHvT;6=7caM*Kd@cBu-}|4%U)(}d~$xL}&Oe zFD(|Hr3d|mwOWBRoB;WAL`vDUtYo^_)kh$c%OE+LTA6APG49VnaJJeD!aX%QIu#WV zSWs{?t>C<7tQ{u*44&DjN!k3ywd&bLnzkdJZY(>&yNe-A{WkJutY9jGVk;>9Ou_|) zT97JJPV#OKNm`&$p788BYTfi$g*TE2p)lgBJu@>o|H?UpdQy+97bW9;{xWuK#uZF( z+r>FM?o&E)zk5Sc?6H`>#!v3XUy72a;M!JMr0+`zJ{?NJNS`{ZMW*WS{(Jw!*w7sqDBNjyLzfE&I#VKrT}iu^ z4eh_cV$RN6>ZkBU+yEa7=HdEXo?qsthm~%vDm`bLK* z(N0KKT#1wokFLV*>0|DuQCMOzpD!T{xlz-7!It$5lrPAY9ek@0Cc*c!T?(*W+6mX5Oan`MlHZF zn`xVN?X8E*<<3Ui_43ZCm|EpEh9}Irpa>a-e`s7^rntw79GSE5K-w>xe$r#@39tw_ zY%zV3(zN7t86n;99OfasOFQG|1;Pm~LeNX$FC)eRQC1y&orsex(LmHwveO$NG&^IP zsd{%b)^bhh*EmV1&w9J_ZEy_v36}Ee{bO^L)s8-8$xx;u+31Hej7z*ub%;?)UU}KV zlJff!`zE-Hi=s}#GpJW)>&SB3%@|OHl+z`GA_1WiTW*@X^|b*V4`?jsz={kvXX8B+ zc;B3RlTkm_(;?46HeM)1i@3*GW1YA_%G_DHe}Cu!*1YEcj@_x-oFT`!6t9(OJRupp z;*|;F)sW_}hS!D?fvgcj`ErZQ{OXUjY4FE+5tCO-*i`0^Pr$%KBap9}9+X%K z9*Gig?$AkpBwSz`+7?Mn-bpv_?2IR7`FWp2}T7F{-qUqIhnk zdB^;>89}fE5h)K}scZ#v8Hl4m`jcP`UYCE`!g1QcgnlnULVjuHD>*mN)1t{iGf>+j zl0_U<8rRQ`@Z9pz{T}LrZ>zwjzdYajdwc@LmIT$R`CAaxfj*C3!Gr8!)|60Q|LtXc zBPFwiOhiOhB_lIrXdwUO0xpi0&U^M>M4u>_u;2x{3o9Iyd9BFs)lIJhW9lB$TDsmy zr6qQ+I3+6*TO7Fa&ScokTBn7frPg%ciOTaCjF!IPfC~%xn7|-?Ils7sf-phIWc2Gp zGRt7sZ2>lv)Au`>mN5pi&ONWi?@TO^01k329bNTatU|z~5OkJ~6hI_gj`geo3Z>z3 ztRE=>E*MgHpYCgMYX-zwZWr3J$qqBU!Iq|^aga*SU6Rsb-1mlxc6Em#oa{RRa$F8c zx^5$cusJ=Fx~;<3o0&?U`4Yzkj`D3$xZl$!;s}Xz`Ud@@9?BoO>-yNUk!Q(5%xD#2 zD;b_LcFcqy>MI$pUgh<1Cy}%dp`3xCWH`c-ho@AapXmi$B!2$8`JpKJpV@b{_Ox4f zH?_ij@U)aUeMiymf+zMw`%Qdltpmgi2EAck7iLfM&%uf}!)21QyJ~7)o2XdS+PrNR z9mU?-46&BGhO_f2G@4dFsvddP7wX2%uW>wQcEsxTJ9H-jW zjbV>2*_5_!=aRR)C8CNrUYgGFiNMm`wWn{T-6PY=+6qpha39|iL5P%l=|yJSkLmh) zS84m;2_7y@#pQy0$J7~_0{Sh5SBBiHO1EAm9(P~8wh8Cbp1a2*-P&|pe}y?*pI==( z=uh%-PaQ}k>>Y)XHNf1*IcZr{ysbP}_f?hyl)Q$l9crv^>EXCPxho|+)V;B1c*uCi z)y6w90r>TlJm`yi%cW@ia>?wWoAu9 zhei3t$$wRB*$m|Km$$P&s@O$F1<6@vfkhk?Zm&5<|NU!wIbL2Q7G3@(@{tquFB9EY zYoC{*@r}EUVOt_V^xtDH8{CXJIkA92PB*(tI3NTLb#?Ff<}Ug$xS1`KWQ(Rwe|J*|YO;a7^e%R3pH|98`D~ zjAWI93R&H(F2R5RI7tV+tUf_hXR4J#8&Fpoobz4vJKcM0#LpWe-I!{FMH)?JNvVMt zlWLLx8sjUui^N4$LCg36N@{ew_<5?~u|K4|28{{7{WNbKW~UXUm02`r9TYz*=V(A4 zp<4t+Cg7+ni_q*}m59&JMvqlhGfVp4>LZG6r8St8-3IP!!x3WS?3wls!zp4I<&1lf z+ou%T$_;_5;Ypl1XGS!#*b0t)aB;}Wb}luq@c#oy+_QPj2M}_90fCkYh0wxyHJ1&A z(8l=GcYxb%9p26e7XB9$#_D(UiBSI{5gwv50;x~6@(&KISks+}h<*MHM*b04qJQI{ zsw)MruSK#aqW4h*v>amlhVUu^zHNl@z!!Gk(S&sew`0gH(^-LR)R++`2lz-)w>@dKDHsSnA+qsJfg2u1{u zVNlTk|3!p{(-zd1Ms@~lC>G4v+}m2REo@?LZ}`j$-dXm9=( z_5ad>|5Crv@dE*w{~++U*jEBSXm9+F7W~Wqoov27Tc)xQ^MA?zkAJc>AMQE2FfX;0 ztX(*9Kyzx|*JgC`PeEooS`JJ+VE+GKThY#eVu70ghhD>S)jx=+WP|>T{g-THKL1Ky z|0`B(M_O7<4Eq-Ahzb z(M0cG(}PY%C3TbX|7u1$0o{)o-pY8_w}G2ujnK~c_MrtI2I%YmHbCJa-_iWM1yGaV z`wSS2elg;OKk5(%MpNpJ!0H3AV!IO2`)U~e5|DtgAIiSi%tJSrL#)(- zQ9V0j4E=yNbVWNU0(}Vme+B@zoXn%c|6zRKj?tT+P#l8d4`4smbs#xvsZ�+L?6|;o%?&_`B&#wE^RzMXfW$xcI@@Sc9V<#ev==ANIk{NgRDI+&B)oabPGF-Y~K>V z&-;i?adp?ZW`64SyY39r1ogZE)kHwuK|jpOVv-zT;4*D|j!o>dWiLGv){-0H)gaQN zK?T7_YV&tf_b=(`uTL(Pg%Py1m3h1sJ1xhwmI<-Bx>KiyF1JGq%*btP4P&O+ol-d>; z3NjeXI}IntPKpy;c{Bc%qqvdYLLU*okFc$oFO5a^$sicCob8}Fo@;4Npi@sR{$4Gr zAIY!pm&M81X$YUhq4~E0B9=-taE?DRoG%>7%Vsi@%CV!DaVfXP^;AWw%yoy0Dp0e{ zyFEpc{|HvQ^G((L7Ge8`1|vA;<6QtTS#>1BTy!Vr4;7!ECQqi!jB&mcm&Xkbkp^X- zqIwv@UmHL7Qzc^c_ps}Ig^>GUUF0$U@p7Yxf-y3&i~o-S5~*CssmD)4QN!?kxF3VY z8!)B1G^qMXSdvoO7taO{o{{ES=%&^O1wm=ASwV|o9_wCtRZ@mYLqmOW^VE~70XV~j z)o^j>RMbGF0*l{cONEx&+xery9{j3ngEZ#`@y=HRBWKfcXM;2n-=`XkUCFCkV*kFi zBUkZZfZaaakfOcawfXyDYfaLp+MZAd>g#8@`Ha^#yLfA}5~tstXN(Ix(9Qk)kT}8o zzrK((c(d|oH}~t@W1Zkb$eC2QBA~S*LznpGvn_~2bBSMo%c*8p{h>@9IwYeiTG421 zyfO$JH{)&;1xp`6!qGQD$Yu=fV^9kN)!#atk^yI)ZlA5t4oq3<`OwbL&i1pgm#4WI zM1d({I`#metkKOzouxTcK0d?PI1o;aet~lhCFHG)L_$g}@y51$0i@8R`$G)I3zv)R zVEO)Ii>`OsD;(i-xi}OVeO2qh=ly6-ecozsr#lLh7LqT0udiraV!DivaX%xUsy0k@cR{$q zaf1;d0F3H2=xTg6wY}yp$9*?9ImTliHUVKx_Ed=;Ei`Tb6q0P`rb7{bxEtsm$sF@j zrit1|$k{cZD-=l|Gd-`1h=x5LE;6r+(mkC|@7T@dx94rQIb2hl8zS)NSeQ>pI#1ei}cLjU3(HRu4Xok5X`v|rCn;hPqW64^@ z0W7Ha!|vt#AitZI$y>M-m&=FR{Cfo^QFqsBm^(lBy_w1Lx3rqgdA2awbLl?acEZ{D zv^NL+Jt*N!=ltVI;+n7H2;8nbYUby+!xYl~P~!OMk>K59DT3ZGb)Jj)+cr2EJz6eP zqY*Bd+lZd30B=-sWlK-z+}q_~*id?&-#huGO7njOyETH~-023T-@G&jWm0}9Q%M2E zB)J7js51LB<-jzs`Gv-76XV^**x#~olA9Xg^%FNF5#4S}^H zQ@JX9Y4GuwB2)lMU!1z_u98OMnpwI{&B?+_aZ9Y?+91`rf#)`Z$t2Xp*Xu@k`E-^; z(Kz5y3dwTObmi93zA}BwIA^P`;=I^e;j(Sk<45mwW@PUpt9vun4(lRw#<{*>E=@|@ zTIfSCY7J(4OO0EG7r_-*vR6`_$eA`k|GvD7PGTn2 z7YB#>&jS`P)P97=I+w{lnZfe^9Fo($Z`2xY;<+=zHIISej4UtSIxc!>qG~IX&@=b! z`~{Cl$K7zT@t3zg41fy2f%k++tKZoEyG3ZZxse*+;i4F;=uMr0x4O#6b&lkgWV-E) zd8NcrwbW&^uGE#tIe+}N5@YGIvB;7j!&x?2<32&*`|G+zn|3>)9gT;b!ut!EOMLrG z>OlTuH>>`-3-D~5s1sBRWjjG|ySVf-=LZD%P4z{;j2?ba5E;)`IzK{LQ#lcxIWPwpYR2J6m%>Ic6U+ zZ~J`d`YKTQ=zW~PehlHlbe$qZudRJV;%2y>2(xQiT0aA|EUf+L9?{)hXd=VYMS($N znX-9my>1_=a+)~kdY!uu(3sr@90oeK-xf?e{h$sFL2XgeZ^0W|cg|X87@!sLtDbz< zd27+@YhE!x1op1G{juDVd)$t`kAuL=KRtK8@AvLZf9bqj-+Oz;aZos?zMmma*%M-0 zTAmCYarCY|vV3+SqIuD| z^97&Vv3|Da4a|H6U-BdY?-BdUSnYd7ytJH!wKb>xG0!SmVlGIq!p5sqIrIrFUQGX8M4#YVs_yjA<2C1J z8eb|0^7|emNSNRHuJbMES;waEU7$X7O3t%5hH-E3x%r+WwW#v;<&q)e(WOm$m}_D0 z_-AL+{WbKxcN4Y8VIZo(hGd+IX4IZch5?A1f)DAu|JwKc62&+4(OHuiw3I=^teEi5 z4=5nuA~7bl;PBOZUhL*=-5hazy~?+jU%PH+BD@>!UxaVnyWNwBPg-wPwXxG)AM^UY z7=ildf-+@O>CM*;FL5w|?F*D6-!IUf7!X}#2&kLuKUi;xef+3GAn{Ze8P^pV6v}UVGUBO0`!bW zp?jsHN&A4Z>x+fI58EvU&)02qBg=43ljaW3jYOa3fRF`S)S9k2pXi<_J89ATCFO5f zjYQA{{FYNe=l`5jR#^{b6u8SOAW5bH{q3|gf%C7P z;3xOqJGeVOIr5C~MSk~>cSv=3VC?O6o`-z)#+S$3N-u_vrG`{;g)0V$&RMl7ueny= zH3HjC1f;nOQ@04&+f>}zX@>o>7x*xP?y?B64oH-H39t8Kd+k=a3A5Wi)CrOw=p!O% zLh-1UUs0W7Ts-J=y&oMMV-wtY>vzrdnBH2s8l8`>H@ho(_1!R4KPxB?K`4B^p;V0i z6I6~Rvpl}|Rs(^kxR-HQ80wppmWrQ2=P7~ol(~@SfL}gji#k$&dg;57h$^I$W%9)z zrg>ZMn|DBP?^1i^O$b8>e=Pa|oO@Z4fT#k~L(eTp-1am0`W&T{7u~2M*_GCOYUERl z*n?$JuTMk6gMIZij&kTpIY-hDiqIE>_~1M3-PY1^=D~&SCAYdCqmw?ZL1y|utHHo~ zLL)f+`#X#6rA9W`Do*^&k$!!q#@CVc!B{FS>iqE&rH2JuVw3Ef0L)zJ*$&ovkY`uA zBoZhD`{Duu()xOL&5KXZiwU%Vo4AY|Gq8>iLaaVoy;if$!DGda{JYGA~|EPDMP?>!J1eqohX3s|0 ziQyajBTe}tCLpachFeytJFGY@ElzCWjbJI_`!(?W956h0&@P7ntHDHu;TT3;CnxX+ zSRUmSBs1d+2sz#fg)Z`3)RL&H*MSdH6iB?DFKA~O$LC!R#j1C&=jd}!F=uDwi1sS8 zUW%3XNcOMN+S2(PK$b3DgXm0@2@)y_;V-ImQ+5HlDLDYox2{?s zLQeIF&zne;*X7xZV!iB$V;(W-gyBl$bttnJ6gS+!@+;V0?SWoK&HSL5HJ2JCLsIIa zT@_k+W>i}5RwBnCAtic65}&^V=p-o-5RcXUMW+w2li62 zkPD(~j6;!k%_8AYpo94wTnWRvDI&!uF<0p#a=hQ`CuF3Hk}LC`{^8sTjDMC(ZRx!I z8eWwp<61Dr+I4(lg3sKyiXdbz6`;C#Oi_c9zgO&|*tAzT(r<1cFMZGTQzmC zk5>CE7jRwcljV*siWs zzw@`G*VS#{axowr!-UzKtD*nKxv+W9?3Tq~XJg#A79kXPjuuue%5|PvFW@Fi=!Pz6 z2oB<#QXDx=xpDhE=>~@m!;RtD7oWsergB~-rqxno5Q>YKe#Ti8uVmy^hCirr0R`T( zLav&qqmc-ECZv~HmhbM<+sogdTopG`BYRB_eXSbr+oa*C&Ra_qdVZB8Bg*XBbCcq$ z(xv#u5N0aNaBhwPJwn^2lJ{ytM+ z1u*nb!Ko6?UteE~Y^xeTm|i>D0YSAvhG@I7(4PCKRh`tVsr{u7tdq|>OfX$S3;Mn( zu+D&lM~+-v7gOKZ+??N^!&7`cyCms06%;1HwgaBKGIxX_rk|&6ZL*L_Cn$z6p!TF8 z<%)90bupID_3$%}y;R%ksrak*hD@7m(CoU0`1}Aq0`cV>7G4SmLBE zub*9l@V_A|kjj*iktQ*^RG@g~B6BNn+xQ6h_Z;I?$JKPLX{5Da5>9Zff@01b0r z(#hX@{5pRPqZm>T&4u*7x;lwvwHIsqa0FF){xkncw%Z}ldWXvGT<&_$fB7thpIkiW;FwG_RV#}0FMk)ZLH;z=NQXT1KgvxeX6B=2T4NR5k*U+^{5 zIjW!P29QOF{OI?~e{PRb&72*IsNLRru;K%@F}TO=50$}r-J(XPi@kPmvsV zQ6^%{-{4imc^d*^R#st>iy%pkcM+T>?i&fm;5UPyxGJs@IGPpFZw!YjX$9d`|^< zK0vW(h5>d?F5S%a)3FDCi2CjmM@FRcKxY2L2LpI6%Xc^@*ZO!c`j!W;pe1Q=>2=i0 zTeObQAY2=X)b4pWPH$r>z&9?v3v#|3vG&2&ngofZ@+&D8nXbT*_3)CboM`MzhfyS< zE?wdU^n^hZKK0X`dH$jN8N9kOlR@0S)+^}D9OSSlHrxYh^9I{KZRPJWI&I{W2nK*p zhznDofxma78$i_Iyk#njIZ~LDXSROvn+ldn_7<45_rK<3n}O9As4V+j9HI#x!uMgg zo#9IQtpUUYVNj$d&*Tq~ohf*kMtBB@GqT!r#ZF^Hy>sD5Lf6;7LZtbetV?08S}VyO z5w0#rZ8Cmb`W|iqw#!Fq0w~5X^u@}%4{LuM#~ST7L*9QSAFQ<3?itxyy_B3wF&reo z%l~tvnX*>pfs@u65s|v`A>C(4CHA=w!L$%C!DE|{b;x7JXez!c72D|!&nc(hcUhG1 z&v|D`c5&?@M58!tc$J}4!^)4A`F3Fu?Iok@4R#oskkNgX@H23%8yBN zSEi7$s374xtwV?co?zOwu!Dd6(`yHp^7e*RWU^W(e;b{tUw@t1P5sIO(7LS5(7kXV z*Dp~U0Q=hhon-S&^nQ(T?+`1M46*Xj{v=>xc`SsBn-`O5LXE7Z=}xRlJfh$nOX)GM zy|}my?$We^HteU02p6)7)sHn#ysSH*6YLPtitLae=Le3TRlc50Kk+A;9B14^&r79U zzjw^kFnVq+eiTssP$eV_qL2nO0M%9Dr>HXGXmSH2XZR6K-zSj2(Jx$s_P080NPt@i zK=+SxmF-J$45j_Ks~&2x0fKT+?63l!!bO)}l2Rwl8n5bn>*dTtkjQzeOl$VV3=x+6 z*(LN>-LnTvu4s@1j~IIgV%QD_V-{UmC#b|tB3E%w*fU`p^ACRe)ENqWo)!xMy(Ocq zYg=Fq>0NV;Uu_GU{<7-7YAtdz(siH!=;vAK7QQC&BG^6^@M;;#*-gP8gx&|go@3!A z^YjlgdbJ8|b@1;>fQAhz@N9s4)L8JYfm^|ppP__!>q8Woc=y6Fyc3mea_P5)ddEb@ zw^pUycgw8E8V+XhjU4tQtM?a;wg+1jDF7b=g*$l1QGQ8Ez+Gigayxn=R2O-3A(*<^ zr&MjV8X(?C-L9mCWQNY8>wl}GZD?7O=X1<>3&Zn@I|USiSm~>3R&Jt0#p9&>?yQ&0As0!$JdM%pxx#)h>8Gkx z7mz!pf^z@(uF-Fac&)=qo-|osRoJ&3oJzrin*`%Xf#l+7f9qW~PW2xWwSMSHll_dz6HCwkt zSdIEPFiMcS;ToYB4MsA_cC`tYQ+J8hFCq9d68gY7 z*}J{fc*8UYt-h#9CRyNuDiGvqpfK2 zE<@%XN6ZH#@uU)=yP8dsQL&Ok`PBwisOGS`zI#7qQ#J-xo1D1@KecCj@(`8sc7PQ5 z9Qbf(BqYrw_IOHv-R<7P;rzz_I(__PsDAof@}20~S0o1RTUr@~awN`(EF0vcmoq49 zP2>?JrhqNq5|tKxJ-Huor5@9}TXWr+!f{N5@+9?~h=OXqEn;<)iu>+V8#UlX6m&Q3 zmU(TC3E!cGFG>-2a5>IP$LKJB^Xk&PAwl~NS6C#_eKV2%S>*2Y2W8QDfX6DyT~|PP zZXH)~i&7=YP#BjL!-VaHL@S=-T6nGh&*6I&<+TBmy(hnbH|=Tfpa+*8qNMT+u%0{g zosj#YgaQ}nH_OeG2JL{?bkIAS?`5(6_Dfu&BMYaTJ3Fo3eI&n6j2xpLet?rFtQ>wJ zelL+D>yqKD3dz@V{=1MF^|j*Nhadi1M98xu&aQKDVvHSif0K%2wJR>_w?I|Q70{rN z$AgiUteMsmWgq3a^VCYs!KGg^G5I9)2UZKCzGU48rEqKGAH9}n;(9~6 z(qR?v+IqRB?l5}FvKEdX_pmz94ktO2xHjRWzF7&YZ#uhOziB*y$M~06x5ezV4Fo)J zu)4!i*8?uan^C|)Ul!}4!EF2OmSYe16cITmt@TZOkS88`h%Ows4;UX{UHVdAfAz*2k_@0zQ(SNQ;h z1?-F@F%*H(Sp5#GziN0Qzosm_&83+Qms_JeLwv-(%cw|?T?iJ%_vFQBHpWatVl9kH z(%_;>6G(mTlep;uePl<}h*A@%wUtY&o^c%uFGhirZr;y&bDi<3t$0$eKD5pO$8#S_ z6B+>mIcmhlbcV+pbt}l60fmz-N(;J|;tO2*em=h9vHJ+@$;UGaJs6*pqMuhptI$RH zuZa)JUzEKboV*>Db}Nk0Z)cC;cl+aJdUc_M^WpH6By|K!b^lh_NknoIY&hV_MeVQi z_@#Xw>wggH{_RLDWl228Z&U3KD==RLsOrdfGPgz6zxeeQKGHRcG(U#1>d?aFuBNuY zhB>LrOY)SZ(5zrH_*?iE_3eo#DzxL56u+zidZ9Rhu~^%mv081kl}1}9ePpW3O_TM# zhWYdyko3f&5>%To_aTUj&4!I{q>7nwd#9F&_kK_CbFUqspIS^>=cW)czt3er4?6#O z;&;_GM)kuGY5t)Fg!p0oYpqdQb(`8;Oe1MhW%Mf|+Q(N6wZW?6Y^}Y%ABMDp^aEd8 zJCeaKl#$E3RwlP--UTw24k~@~Tw28GdPYCzjj#pzFCu(s0rQADxvjx1U!hjmUM7s)xFz;s#x7bj%I=QLO^M(v5(UDu8Q&!?9~Ez`~#OhgUGm1 zFaq43=kNR5U?*)YP}7_-syutAP4U&b6-2S? z+5F+FJ^k}7FTiWOFv(WSd)4Jxm8Y>k;A-<>We$>`I_c?#o+nnxdaR@F6@2Z=q70Sm z*ZCtYuGp_(BpK{aySa!-@h}=ZW*lx3e-a6@UsT<8)wJ(%lF(7bTC~%?DPQ|`Af6oY z5Dcpxu556V5d<>~5I5aqIC!fm_~|NCvgIg23rGwz@y3!7!_2eeCtKXZd^X)*_X^IyjyX(_>q)&n%nB>*QF zSMu!LUF9R51Z=_Po(38L&3%;Q?`;^JApi;24*85T0DwiCJ~JBb1$r>5?~%k58`{6I z;Yc>#I$uh9F<5i=j6Ib%$B(Aw-NLS3GiVV1x}B+#_<`*`ShzaYL9<4x4*-`;46u^- zE!?=;tK$L9i*EUO*Fi)J4b$liwpqTs^0n!OTzrha#ZASd*m-&_DOl}qH-G(<11wz) z@(wzCBJNPGDzo-twoMl0{zA@FACx^53fpx{T)a8ARk`Ubvbs@VVxE~agyo5q7m9cJ zBmSgCQEpd;6L11NiF4efA~<)8i7D+W`yM<_+!P-_JOg*6)_X`}&7ydHp~2Pzcmm)j z@-6RW214aIQDXzuAA3EJ(N_>v#t;GEVmiW0PGIba&o5yMu5`BF=K#U9Z5J$)#mQSo zQ^UFV1$P0B?x6Opwxv)nr_DX@d{Acp*13S7yPv%U?n9cNb|3Q9$s6km_uzT zjOKG0gplHL&ZT_&T6CtjCvZOguqc!@#*#Seo8s0~(i;CPkf)m=jmsHuvs7nSo!Q8<)A3@-Qy0wE9N*~m}6$6Rrtd|VuxRlVp*7% z)6(jq^zobsGL@p+{<@T7$BYMd@}nCKrthXQ;Yo%GmGDl=8Em}PHHWJ+I_eG~MUGuo z*NX;8N)z|nLGn*H{ELuech}rE?UU9%IAZFTc;d8{z-GFG)(L(4eG3GUD z@qSHu`FPOFO8EP9k(@M(-T3zPG17MJ>&iTTpq(1}mCC5({Uma6^SqD()gFZ!e?}bO z5hUUXlZc)#40DGCv|L|Obemn1eBpaK4YJmN+=m4SS7s!11fU0JURkL?uR@_t)^M1l z*)kVvznEW~!ZvkPh@V-B!w|O4k;bKXrL_q2Z-kc}6|y~8ZP*3vf$)}x_z6n71HK=(j(bsiy{U zOsQm91XeD?w5Rwj9oy}gT!6WUe1#cEYSq+S>*e@TTpcLU^Kgn{`|0XJN2lSvv1j$o z>Y}HM#2krCr6=QebR#nZR*%U_QUB-9r$0u%J`j^oCQeh0+ApME!%qp`!<|3&3WPW1 z)NdhYm)SI@WlGE&>^7l5osnIvKa{GSKp~_Rp==obtBkVmSE}piOnn%;fkqHY~+5alfNKuV@_+T;Ve<=Ve7W zI0J4Xu`(UWm9Sc}^CJlKFo&yq3$Ed}=4i9dENSGUL{)|js@#;iBr!=LAFug-!$#ij zu7tb(s)IMC@g_`_@!SeQ*lp;8V-DMupt`rn>@mU!x^L#=0$>PLGexw=*@*dNE|+f6 zgxoPLtS?bKM}cCsBDcVjxS+Dcj92PXMMjkUIjgxS`(@SDUs9QpvS{f0N7&a!!HW;ZN}xpzISV!d>t%L z{7;nTFOz@Ui)8GBkd8U4+>_LDKJq5#Q!nVw#bv8L$;Re@2m6YPe3M$*(7S`#thMZ?l6)vq}v1E(fm1-14$dIb0DOsO-?%wRp{724#5pfGh(P}5dlgvi#;z2yi@0MtHQm-)=5uWimG^&^ZvV8TbBH%dO0MCa7wWd5E(Xw_md ziTS9=3pF^`q#pPn<<>EFPPjN3I9f8JT^Vo>y8C#8alEdweX(bpb_eiBp@p5HO82#U z(1J8D%PM)y@ir&$hQ+9=k&dFacK&LK^|#w90PTA*RT)bqKTyJ?m_uXXi3{KIUcKkE zwno3)z2lqOA3wwk>mQtbu!9=W!3chbl_Bj$yvZ7o>j_SYJMbT3zXQ~GkM>mzA|>dr zJSYwI9v-Z67M%9=CV&*ykm7G)S+3wk0R0-lt{i)`%2#1G0`v+gLh3R{)Ll1Dk9Rsg zc~NHM!i+pc#UpzNeZwfTle0tbdtLaGse=)EpK|A0HQ)g+DynNAtwz85T8N zt|xlOOh860d7%8m!7tZSBne4((z8#6pW?1H_GLX=yB?X|yrz!Yj-!Fwbj(0giEk>A z*9o!(Z*O6bk^&zD1Qb@b&z1NYfdt;|0VwIS{i=+dpa{SP6bhaQGra5t|FqXR4sqc)7diFHnDOIgLwX*!!Gj?#O;q2A)))Yw8{8Ts2 z>`Tm{f3&zfUoGZ z<}Kjr*9svT(9^bYTR+0t=J?^}d!PZ>jrV2%V<|S`axKQ6Uw!;Dh0fE^xs%hRf>T*f zFx+XNmENXRzNjQIX5@`vH*{Vw)+hnM_nP9z z)MNNXPPfMc-TIA5*li91blk4nvL*1c09H}IACMLR2OguI0QPb>q|9f>7aEVs26JPi z0%`Wo7DlA)rKQwq655s45K-0zwzx5I+QRXQiF65rfttipXN_Z}JwMrd4YA8|q{g(O zHgf$fc!ICt<$Bm=J|_eqcmao65Bv?5nlAc1R<_n-8Gab@Q!l(U1+uMc3Q-?2!#iJAcN1{J` z{`y)@0*QzsF}0=*CGNa$4+7m11ibx&>kJ1|a#79AR z$vHC?PdmN_7SH2Izhtp@PfI9T_Dd*o1uwonuFrLeLOC9gP`tty{bNr1pQ>R-gv#BJHHjjqY z_)#IMd@U>PX%+8?irzgwzZBQb5cMYNl-~%gmUd+Br9YgFpnaSxg<)9_iXpSS+GdZm zkbCL)cySK?Bn*w8Q?mjm%dU2^UiWD6B{<*RL6Eayw?5C?Vc(FGXf5_Z@2>vB?pstD z=wsI+xp=Fqi5;T;+-`4(d)ppD>{2u4ax7;-a1=1rx~ z?Za+#wnz-YYUb%4Aa=|3#d3`BceK}PnP?R;@GYmF8MG9H!qC*wNbv<)GGVL0dBGTn ziVJj<({6qOA8ubhJXumfw#BdlH>fjj`<|W7_JB;4kd-1E^{D&Qf&<7-j7pv~B%zDv zx6lW)y|_WJNcWmBt)ILJR7U+Tg$$NHi%s7FLxS6Ylo7KD$Qv5*>@$w{w{yUAas}um zLC4HI(m&YGOH3CX_HJJLq#Ed*A)?~7+Ai1?#ChqYO?kui@ zf`6>O&y3G_0|!o8|FCPqKM zr`(U965-n4Q+y%1G4_Ti7HgYFaqs933n@BYf2%)O4=UVZzBV3^hgPzRwJa#a?z@^e zxq~E44R)yJt*I1bQ0Qp1Rv-VYT7H%i%s*G5Pb?&J>ap;yla9iX2^MhkG-Fu~R?6hd z_g$yc;OdBS^G$4)XM&{x4EB5YC7vpFTPyR;`0vUb8S#&^v3J}zV?M!Qo(mT8`9WRH z@!P(WeNLbI%FS@@xFBn@=bFGG(dk{}O_LP4QBI}LO;hbn)K)a!5S2$cyzO30zf+k) zl(lW_)@Q@KR0+s%692SHFWmk3{ZqHzu`VQ%zT#p|WxklY+@+Yu{YmtmY2Fp5>viw=&uGNOI=@hEvg6MRdi0X=4<-m;Uz`u8%vn|8lV8 zp=T*k_n_6P%v-ipH1<85A+Ijy)ymIQRhfYDJ)9#iu(#K5Z!eGklB-qOj|%wW&eJ8Y zF89@{tXsQOH2XcAf4E8ms;Vr&@g5G$3+(Up+kdqx2k^L#+a_Wx4xAsOxViM-y7;=) zwjlr%7=?WsdKPmT`j>+MIPnBVDMl|tFWc7IMAk)s0%MPy|K-Kk%bI8ZkVh9^kJ>hb zfK8*YUqjC-E<^uvkVa0triu(i=jVo@iZ#x$rpvy|Fz0hK}RPab~zSeJ4nr-_`$%m()=E!?@T8}AP~ z11FQw3b6>~cLpuFJUTA&Tur17IwK}SUKO%&$&*+naW1nM%o7OA1gmv_gn){%jO0mr zlPWS<4RkbG&Wc=3ln(~8CPPFEu|V=9sL6{;y`LTZHq;I}3nr5l3b8U4sjZNB^*dGX zVsNC6ti?_TM4-7ujvewdIaD~}-0yvT)hlAea6S)2ysPw8oG+Av_2$f8757zo%->bm zBkIB`@1_U7Err3ZfBOn-$}sazPsvAE8Dm5RcF!3c~fEy`!Xu6fYPx!)&K4b zYtdPaCPmxiY#Tdf=fPHXQj%yvLP?t*pICzsE8o-CCqWE0pyxwzF4_&roH|kN_Q-ly{g{5Xo=)t{H+;1eP3U5J6*Z9X=)GQDQf+m zW|}vd=UzaPjo7J}FoxWA=Qxs5-GsFBzLhuEeWrd6Ie&%>k!2tx@GBDM4Ez)U_>Dxp zTYG|}SO-0OcoYKA=C5Z?m%A>1Ttq)KXHp;CR~smw?(I4GXYTd62?`oOJew_AmU)wp zUZl1b4LMBhG}wIRYFZXFaLnTJ7;Ow|Ulua@hnzzjP1WBU`gOT8dJi4naXxZxc5AM< zyv`finf-@zX%=&BUT*p9Xu#!BF&9>msy}xm+_BtjNZn%iC~K@p6`nhiY#jQvv@?6+ zyyxz>?_OB>mm58Sj-NOmxi`Ckmz#~KTZ|l6xEfDWg>NZr{!1Q7#t!}J+nL>R-gENX zcWS{^wkL<{+`La#ys%_cZ&{34jBYQ3^J5_(|D5hiC%aGdJz_FdP(J)oG z@<D^P}DHUz>1B||kSOWv0YI$8Dgj{=kF=-guEivCJiIYt_9JwSH zk!m}-lZ`$MKO-@;s;OsK4vZXo>*R^ply=_#e5dO5qWgCp+jq=Ce}_N$HNWV9(Y8b1 zI|iZJ2z=d3B_peuf|G?v=^iuK3+ZAe`7wWor2Lj&`Oane-vjcY3HjbB@W_}9>086- zv^0rHuE?2}IUGrI@RsoSjF0<~gKk3^C#kc6c8D-h<^xioc7bumvYwom4DJNCv}m0Z?wD zin2y6xx0dFlFBUva1Jbq6l3Zk{JdM%EaK$%=HuMXZ~Vc96+7h3?Te})+$iY#R17BE zbV;>|+$glSUP4zyUz4#MzGJ?|!%RyQV&Jrop29>6wZWG3+N2^I4=|bB>>L(IKQIp+ zz)mD%;#0It6Y-_pGB45~E#g)*N$cbeF^LRS8z>sok37_ll-I)4Vhxc6V6Iz<&6bT1bS>PHG^j57+G9~6Xag^%i&`0L2xvaYz zyO;C~p|aTVZI{DBqP=IT9dS;)TiZm-YG`BK`Y9&7UB_w`a2uR>6U;86P*&{6v1^J` zZ4veULvkxxqz!V<9I2BP^D4i6W|Afpn6Mi|3tTsK8Rup|jy!P@wY zq;{+u@r`mp8E1Bjr}}}g+jRk|4vC6^h}-#Oj{*soL0cnNX(arU)`Zpz-!jv7Et6Ha zk;0d#GaRxG+HOF>btvf?Ioca2AQ&*yHz-slWy^MLkbGv($g7Vrc=($;@0kvUtR@e= z|4Z+WgR?B8y)BZKZ`}>`Bg!6?EK?DYcUd!!REXAsmjmPH(r5p9dfbZFo0`2Pip41k zWH+2)Z}OC1hiXw(4@U0~y&Zq*`jyX0z$LVK_mvi`rK-Sq$uuGh=5f0AOWp~z&+AtC zrXan=uFDHIobN*x#)aaz6}h37EdMvE(sW$xXUR-+Ig-UT-deOX&wnvlzx_~dL%tti zk~>$|RvKp?Xi3PQL9_ULhN=JEhen(9ZJHJAgWVA*r!7-bM&2^^jwOpWJ_PFGY@B93 zlVXzlDB0as_pz&qa}0In`)gv4LpC^~dP*L4B~itHUEyFe8J9h)PkB%J zN-3wE?yyMPGEvC(;O{R^e4VZ7E#y*`qnE?MZs^c0YS&2H7zSP=g+?oB+A#l1Igwns~M=AMDb>82A&Y^&$;3@(R0DCVc0M}&}N%1JrLi)+l=AA>*! zI)aGs_rboZ+d|C;4S8^w} ztSODN&HLn-PV7(}R6YA`*`G&g3nePnQFBE6;Jj=))+z;Z`r&1q`mUT)yk;)e2GeP~ z=-9<2mCB|J4&RXbryAX)PR<(j_;O7KyH|JzcPIg55`hayHD^~8LZ*YYa@PZcat<%6 zls3MhGak8ACSx;VQr;Gtqr>vDTF%k;!^XP%o8?jG#$q)UoGmT;oB2^uoSZF9e+g=7 zZ4v_mS0~2SrOO*H%k9pFtmtsm17Q$XOQLQG|?kXLSpbaGRyXR+v89d zk#mm%ZG+ElPCGViu~^JAJzD=_4K@NuYgBK=UEQX7zV%G@D3JgRmOb~2%FewvP>x!f zjy5ZzgcU|4Jq%Xd+~#^7XF3BB*Yok7pfIZPMn6 zFe{0kSq^pEr>Ak))-aCH>rb0UeYaOjeB`8+8h#G2E~^`M|A&7kGq$_GfGwJlxA!G; z=ZkCl4=a}|O3Tgj95ieN9#2DDvk4Kdj6I9XHv8<Fg8|w%ShirT=k!pgQ5_ARqUQat%%o9k}_MT^vc}}8Pl)b=n(`j zB^i2Uu@gucd4}ovlyCS+ALDczHSC0p3D2lsAl>PosOL}lPvqNGBQ54u{u{WHP2O~A zhL%*bM{7JrzQ-U~B|uihqxkQ**VcwTMf%8^rS0|{L{E_w|7+%0sf;jbAt~m241|CF znJNB>eQg3=kPIi($3U2l%|~X*JAAdN*qa{xNPc%DePm73c6xN9Ye}tm9?2f=D3g?f z$32dgtL5ncwl_HWY-cI1#hPQPFrAx|t1w-d)2S3)nDeSIHW)b!$|P> zaPdx7^ifEAZ496q&!F|wzWukw;^CK@?bxj-N{#640M@d_Kn7Nl-v1!P{?7kmUXs+pb#!P`&0gTDxpGGZlu{`KD%Tx)j*cNtR;2KUD9Q(1d0rTs_r zAa{Swtw{9`*!l-nJqY}TR@~SxTj{HX|M#Vp@n5xNOmDjchyR17%rbX(EYkjhEAlRW zuD{?);J@Gsd#e2}xZ3(JxH3-L=ppl8`xm@2|6h1D<=XWKv533axkgPhdjz8XlA*fT zwTd7^%72EOGQaY&Ro~P@;n*=^g`?U>_Fv`WupZyfzm>Of|7-LwC2jTp82zgS$^W+W zf0SYWZRtPVJSy@2cK3gb{uu++{~gPJl%oGX43q-HKn5=@5a>ft`{*#11AiUGf`w`v zgYD!p+gXT%wj0aF10&k(X_fi}2ra|5@^Xo_4c}3D6%!LuWN*capu_=#R#+I9!|}P6 zJIp+)PH&e%#0^K?8EXyw+C*(VG-g~~n(u%oD>4~LEG*qI@NKxw+X!0lPRH70+EaeE zIzw(r)Oqq)27dBC5)a?|xFmF+quh>9FH*n0p_!k1Zi+n*3#NEUVqb^;Lz3gIKWvV} z8MpYs0j3fwUC{mq?5_O7YC&`Z#$u^2>^_8xo$x%AK1Wv~YVskQpR3wS^EW+DZ-uH} zfU@?;e!f*q%J$srI{7;!$;L{dxsC%BjczEN46GL|O^}Zx$W~kK#jBuF4m!%L?E}U@ zMJCxNo*{4A2C&+U7`XQf_b{+wS+2F|tm$brBDYOR-qQ}#6^DK)%p5|uE{U^6=&8CR zG!ecPQg==SQ%2oO(gwD_h@UsJX|PM1p0IsfF)L1c;d!mRt%togQbNe!HA60Cyf^iW zhU$mtuZTV{h@DmSk(EhQ@Vi1a)RDIX973;eMbMF!JBw0!U zmrnfS*11n@Ua>j$7xD@M+?$*e#Ip$nMx8le6(1F*rRE^cMJF5Aa@;|7$_FgGq+OG- z6FNQzl{}+)1}q=Poas>l<^i<)HW8F_y=Y_n-~L~->s**7mCnO<6EZ~{u2`AfO*|V24e1(^&tG&AA(<2kHLn~?& z*w_0~Vh3KG`#mxpvWx4jL(F=z=}$;e(stat5igy?Abm_x%iNcbMRP9yere}u6z^sU zGSy!EL05-wM2bxRO0~K)))hwHtEWATRKNvITjwh1M*j*^bvs!INBcA;4=o( zRy+0yq&C$$33M*mSXYXd@cgZu%zg4 zJ(D%JH1h(RWAZ&swWPaVCOR916wH4Oix`zWr7lFef>o4G{XB9KlDA$}PA6EHnO7G0 zq`26&PJiCItYB9=s0mmhLf_ap@F^EfK4uzibKNK(gTe85r@&A@kS zYN7773FRQ|H?;B+{`7T&oQb`S8MLo*dV9PYEG-+@j+e669BD|J$}p@{l5w}cq5yD1rQ#DZ{;`&%U-cvypw zimvQ~c(!fmFVjG}*=;K6@SmqtxC)uM2Q6}SJ@-|3WD;k-m?#q}Z$2CMyC(RCSM}Ar zY?t!If>aVYuH33p1~aPZLZuKg;feqAEkx#YB(mG~JR`cW&70rO zWJ!r^F^VPN?x@7p#?_1*^GdE)Ie^{$hn2_7F&D!?;l|5X^M{D;S3^7TSoieLmQ;Mw z_}|-HCE#LN+TdLvimdV@1}b{qEZEhICC?oP)4(;;1RxTnSJ{L$QLUkQ`PydNigKek zdP=^lz-^`KTNLsHOPHZF&>C*c5Vd}pFFJ&BZnbug73ecw=reTzd7i+%aNF3*^)h8* z-GYug%>3xt0GDNzb`EyNcGLjVMT6R*hlJpytGU;o0+I$B9&tA8a7;{vzHoVZA%1BqBuK~0#+gvE~r1bIF! zuDa}Wz)aFGc%M7zDo$jv0$55vuC9av;f?2D+>v>yf8n=RP{(cC_A-y7kPN5Vyy=gy zyMe^zQkka5MZPmVPJ=1iJL7c(j*%=gLLX2Em5N8pRF-ehM2$#g6qUT3mouf)clTZ{ zwECX+L@6VmPe^vbFK_Pyio6?09r9+;5!EJNSbbB`X6N^%P@&&Zo&+ z6|<*+pKnt_KdiPWaxanK)h%&Y_u&@cP}dWCY$4}jxhsZ;*fcT);~l)LPdrZ-`x%p62kFQ}i#WFCkYmJ1x%fokYLRrm%v=upb*ryDW z-i_z~Z9^=Y#4VT%YJg!|i~$K*eUqZMSU!5+ z2Yp_j#-(l4U%;CwT6SeNT&VigAuZt5w3~zXTu8y$ zC1ab_g`f>1YWA~xA`bxBOgNc;B1dGY&OZ5UGB+NeTd97E1B2tk$qx6|jN;|vX3BRO z-_a|1$yztcz=h?|KMmuzi=R21HO3C$PjYOb$3{0cUPDWSH>nv}&*^Q2 zK-3jlR4v6#1-KdjweOEq_(^+?{vT_185_CQEeJcz%*@Qp%$yFV!_3UgOdV!sW@hH3 z!;BqHD$E_4ocG*2-;Cz_H5$oUC0nb!Ro0TqviE+L%7Yn1ZZeD{AswTD=(V2BavVWT zwlVX4+-y6fB;ZU9p7yZj=Q{p238SUb1$|C&mRWb&C6G<`k=QR6E$chy_iXsHY&Q&<=eo-%aqmM+S?VyK=9-HIm8{ z!B!cVF13E#!`oxZAO}&6@v@lBfkU-f{WZxG_zwfgJ$?mO$0nt&Rar}RP9i?Gv_*N> z=~X$6)mYbW{ZUG;3~xT^jqmg2dRr2QnW%V6-mrb1_$yJ6mIz@lfvC?pCREl zG6}~bMk7_%v8*-wHY?bjy4LL^INe&4_<^#wT_zkFUaEk6YPBHBQwm*~0Z|_?+X4$l zdR|D{)?GNz{(@ZPNQcpqxkx?;F>$?~9EGWQfu2`t}5{2Fk zCC}OUmQu&0-ikq$MT0i2f@m?7`%iR6;h=h-`GJ<_E9^XnGJs}e66_(ddA=mww@W!4 z+ANEmEY&$822nF8nS=SOMyi+FHHSP!`uWe+u|);=<|+tuPuG?;hg5dU59((V;_YIh zE_V0*Mj2#)o5f>XMmqFmehtFTnNzFq5G8|K?V&E8>n# zWa$WPg)TQ$jezviKG@xA=FEnsq$gfd0Qy37_^+B>Wr+*~<(c)x@sO{^KDbk2e2&VT zIN^}qVIW3`Tbx^jRm_EgQcgGOe4%?o)b)&wn_mwkW;`4GY`nJSlvi(LN1%$b0Jo#V zZLJYM)u#8hi7N>MdHx}2ahfE>@bXY#&os2n54E>9^2eC~ad<9Q0v0A1ER5LBjkD!7 zdeCBcZK&ha?JCidI^^HZ@*sK$^Ypj)%oNsQ$b*?tdt{w}!u@1|WnV4f_vyH5v#pG% zNky4F2RW+K!Nj}iZHZXDr}U?ZbdR(a!z=?{_*AK(rp7I%1kP9EW~g#-L_oGLI|Vj% zR6ze-1OAgs3ept`Om`P}TQr*HoY+%~X1T9RwmKh0vLt6^A-1(4|Y>~T_e?68%m%c?kE-AKT>veVtio6l~TtOidKPAXvSJu+h=sy z>@b6ACR1nwbFD*v((b-X7_`2k+PQw#-l5P ziTKQ-zfpYKrF+)L9Rf&gWs1=#5-beea}iX9S#S-shdLx>F0qGc2woB!Af>9~mDEOs zA<-_O+bsrZF&h%BR!vy4(- z*q#Wakjr43F3?&|8R|}9dp9X8Hc9gRLANAB)MpZlmG`4DG-U1LTxsfOhoyieGAQaK z0gO*Z-U2b|r)h=R^DNJz_RO9I`2rK{W2xJcG>%Jd!0>@mRp?!Q+u;`CPG_?z`Xqx} zjMUFmnB|egRAOZc7?l%%?iIVWm3U1XW{~j)-5Pb{2`SU5`gPZeyj9Ak>V)CsnT+Ic z!QBmj2KZ4|U}CIEihGa;>vat#iQ!h3Jq^Syewr3$iS7EandIXcs>5!{4fU3`DYp;k2sdFCm6PS zWnwm%jA55Fc2-0uxU01<!`@u|ekJr+gi#U!YJ;gz@io$C{ML6i!-3dn( zQmKvio>FKM;3g7kV?7%b&?V^;n#2nVS>IGa+)(dtZJwtM1H)%3&cvn+QyRqDNWRvq z#*hu?m+f6~S!j-oWpi`%!E#m*vP`jOBaZH2^nh=EB~ULXzw0_QP=mK(_4LE$$ZicB z0SP6XRAJIK$)gIT6_j2E%RKZWHUyIu=eJ+`{dnP+m`{kpNK&Nd2G8<=E3a_;C@%Ah z+Qb@u0t4-_<&Xh^jm^5WuoBW1VFvXjXc)(uwO95Ga+B0I?h|_FY|Z^xpQ2lj_lL0z zCPSwTw^oc9wp+;Zua!}`(~hkl^916CkD42j9?HNgozz$kL!z3DdC901q(ebKDgeo#O-hwVB&X#1@AX{~cBVbIDERxon$+6t~gzT0)9SH(v zm=dSW!A(y#uDB;}sg`6XW&S}0U{M$ckDr(fEmqtYY~U{Oke>VC&n_al-cycklQnCn zuufB8cn)o}8Cr}62@Xj)!8Of@B0JJ4eH$F~p+F810vBCj%1oEdOKrtZMXfT)u5#&+ z`vJb^Q;g(F!KG;`Ir$oZSM}1$&I#(bdPC34;!_=>Zu}jkKDwBrJ;mTUS^oYsh|M)` z)U%(Z2P|2!?9k_T%?im!mCX4kMZCMk5XKHLKjz8kTTouT`MJDVG#DN|ianQ*;USq= z-q3vr;WCry#4>mOSj2_7^-LTsh5ShdYxTJmfy2bK@wziCB9jyYl831qmB?OZsa{wq zCm?ob@lEWwx0{tpjitwXn^Nf2HI)N+R(Pp2vogULJfo;|y=Fo8#VhNCxE~r+z{C33 zmlG4wPVP}h1)EX&{^%qR>>+-%v7Q`UpS3FkgR0J;k$~4;#{ipun~=~!C#9c*I2jJ5 z&gB}nN9sl8aBqt)*S4*tg3Upn8VaM$^#ljiK`4oaY#baO9G|M&5)WS zw~VX2rfr58q9=V@f_uflmZ{+APm*xYhay3Qm)YamLn zxP^(#0x7$tWN0c0+j6IoXkjh#C>dTLFS>a7z{pPuRO3*hq9ItI0X~+@^u*GZthCGQ znuTHaeaSoT^`>^nK%+_dN;tZ1f>um4>n-*#Tf7Omi5-_p>N()dHqD`TCR}_!@u_a; zPCW%_=fr^QgZ>^9g`0kt1)UI#Zm2L2|9q#lZZ-T~nk%#0oriHr)`RR~EO2 zAORNetFovGgkU0kA2vmr5zWNarIf=gi+-3X2Co@=m8OL}H0)u6jq~Q9{$Xnf(1vcQ zp}MTTo>(or&4w*S~HzrqC8}&kX?i1yXK70r?ly zop$1~akOCD%ZI+OqRgTO4WcYpN@8k--HY|qrEvo6*V)CU^TYVx}{$*3@!#j(LtnnpN5uF{JmCa%ry#&IIF%&XVp|t}H90Az0K7 zRe1nA+{Fv-e%wm+)3nIM86Q~~>)%xu^h{@IWtbCd4j?}F9VYH(gk^5tt{O^S#p1HJ zfv8kc&dL{AuJR63Q6O=K`TC!ADam6oELscBob$eIf*Nwh9zDtYoWVQNblHiWGIpC# z2xO~ufYx;idCiQLm7+w}@3uVX-N@$+964h#q{i!GhJHm2MlWZql&NmgY!oR)-IKXJSI}9?0TmVEWQe?ylM9*Q z@TYRMi8)yr<)cfO(ioyc#8x%Gwx|7t*wtx79Zid{Fm`%&-U5bb>D>%VpbgZOBjg&c zVnZVaLYnCu5(Y|tOn)mFVMo6e5KACfizuJd< zCK)+9RK_hA&N=Ke#QAbvyzNH;&2qyWUVpXrCjfw)F>Gn}36nm!3S=%4#3T`w>#E?qY>cZA}$oX40;( z5Ze~=xhz+naNCJkG0vY9kuUO2uK8+0+zsDt8ADLypcWXi5@&KknMH2%s0UL@zTn3Q zo`(f0qN!eS0}1~ulXG_)Vs4U+Y)6ugOd5q1n4CLE6>!K^PW}am%?^84gG}3CjJ!3Z zG>?&HKeSX)X~t&V!=VtQoNLY+#VIp2`eK2j+8VY}es$9-XD#At(rauF!VtFNvU=uC zW?ON%&+J#a7|kciGf=h&N;OYbDorX@w%6pT)N~6%Rz|=a4JKtW;kZWqImag|iub$TKPuqhE;T*#vc=Z$ z6r?qoX2k2ozWIY)Xv)&Z#PUI`xc^=eXYE3oiGRvJ}tJM|82 z6Ph^E9v@hA^_Z%VRcCI&swn9RZh)#ktd6R+1(ta!XJw!~2uM+aeHIm4YnSuX4j#sO z=tBs+=F=QEnD8PUh3?Ru`$D818emv11p}4(`sM zV1^*3`r}9CAE8Kuhk4wmbpGyC6}Zw(G$svo7{~ZW2+$YyLy!UBJe8iZWK3dqLEEEn zwKoHC>+&v}xsXOrad=sV6UJ@)mjwzh)i@=T%!$#I6pOeH97Mp*fWDt9% zD}ecB+KuZ$hZ{afq%k&g#E~4q*6(~z{NHk zeRQ!_kU|&93c(n6*7W!!N{k4lXD?4y$X2W^cXkviACEmh@EjL{;k}V$zO*MQD>=%J zR~Dy)KEqUyOr>J2h7|J1l#ZD(&-$5?cWeC;t;{@p1y#iC}Vr+XTG5{(^P!3nzd zd`}(JLoC5N0Ilri)Xn_YA*>k*uFX&JUGjhr7K<*Yj1$Uk1;npzE>62lCY+B1mT+oH$ zOp~9C!8-eNtx6{GQPK;Mj!|>a$Ynn{mXy`52eIRIHHvW&$IPE05nia|c7u#lG^Ayo zh{n7re?ls0L}{;0rzUB%k3AiwGw!*8tjPg;^ zH_Ng72BIu0ZX^Dj8NfpT)*VH5nba;ivy_%4S3!_ypMWbfqqVg z0?4K+3zcqt86@Q=4D<96j>)Qgd!Gu9A|_!J2Gfgs$uv#*A4I1@$abTVzWqDTpE|D& zp8=F&7HE86T>V~vl+e4PAkck8f!C6D-N{v7)9;pt4Gb*^_caTn1V z%TVy1|7F?JdO=@XpE{q6y6ez2kFXKjaUNm2=~W=$95N!m59m`yw@8kcWhGXk3>;L$ zzlTnj{$^n)3Jik`WnBCF69#*pITCI!U~CiPFg^vDLrdrzEPxZ$v>8r1KzAdlD43KU zEhWpHPAGrYc(WUnI&cO+6b989WL_W1(KJw8T?E9rj1VZ)p7FSq%!w&WH1a28ro2+1 za^5+h=wl-g=BBLUCGLhFlgd3kGgO_q$kkOs zKRe~3fMC9jDs=_g_~B=qsXK%4;FQkXtc{mX#x>ks$GAZ&6@pvg34Ym&A|p+XS4f5O z2YC>6Pn;t`EV?*KUWA7COS9x`#%r{GBkoDqQgc7HpB`6vk(`&GpOkv$WySEw-Xw*w zp%$AxUhNJEG(^P#{TGIAMma_UgHas4WC{=Mj8$6YqjcIf96CafRFF740V2VBfp{6m zGFO-att6c(uep^=lj_UnQPEw-p4No$%38-6(8^$8Ryii3%oM)=8kir;M6k>A?w*a6 zZ1UwbSm^p0)dn3TYeup&;ogrCP26=XDqIgfT!c51yQNZ`HE1=QT3Hnc+-6r)KH2mf zd$e)Bnze=gWu|rDI zBV!nmf#&f#PO6LQ`5F%tE-SY%btidW*RSc&1>n*rVeP96oC76tjYSg|gAJ6o!sq zlF;GPs!B+e*2-K*gP57~l2lZM9k-oRVBRz|s*MS$!J-1Nh-OpT5Ktgk2I696=eT1c zcCqI;5`CttA_)g`)|m=O6W;0m=rxZ4&`w8%B5RXd7)+>PJ~gP+rvzmc$~tRGv*eGf zerh?X%231nMhNY~1?E-ckRKIsM-5Sl^~h05l&V4wd7NFowofyJRN)EAWl*h`4L~l~ zzsE0gisOxmFs$QGYqQ`n50jKFEp{acvdS7?d&CNPYbW+6pJSOGk#qPCex{~IcRG$q z>}z|_9xl!uU;QrUUDqfa(!D#bR%rTaz$Z3cPBx+J$95Y~FH6t=SiwCd+GI1^v1LgH38Fk%~Bn$!IV=!?1l2^`51TkuS*~*@Yvy&LL~$KZBiSwD!6GP zLvv33$Kljcq2^tRX;nl%!SeOfABxCQd3l1zA2JF}V>H^yYgL(@uz}PM94ZydF~mt? z9mqo3`KAq9vvU20je{=^!Db|ENpOjBbZm+1;HG;*yrD-E81hAx12q1Ee9c|=b?adn zl9z>z@?rPfrD%B~lg-5Nst_i0OQh#YfPi(|=;LJS8(a}2ec4Ei`bzx*QvL4=|{6Iz8OI5)~%qQrvrkj{m-oT^zotAPX_vQv6IIrB~Aq zq36lQANutu`sZwMoxw&z#!5*j1D`S`{kI;WHH(X9 zT&8Oct;~-Jbi=o*bUxE2t4&pDDQ!|T>yKA|#?cH>OCDwO@DK(~(>kDzfVPM(0vz<4 zxK}YO*zqu{*5+o^t=xRMg*f)le@#Gp{R^5`adR|1H!!qUJvvG_vNf{jCB+zVwB(k`#eh|q*yu@oZ?*L0- zMjc?7Sbd36hmSH@1fLJM^FyONExL6Cbs~RG&`sVn zNV5EHnl&5_hLii9?E=LlmWZ!PwB`ZZl^W#ROC6jj0n0BUxGtaxUWA@Zi!flJNRil; ztzHy5;K?@s)M>yZ_tc?<5-4SN{f=>izu-%?vBW?WD4Mwt#6xe3j$r|uL+cE zMCw5evxPS%j*xZEdvI?M!L0#B*j-FZ6Z!T0uL zsZc!Vg-?Yox?d1g6ItGZG@Dc#Ro5@UT zz|K>3rX0yqm|u%1nD?QM&-D(RCK2?EWh>42x!JNzJroQ=Z_wdU&IM~vD2If`;~qSd zB)4Sl-F8hG9J7&Q?$ceFib#~6gX$-(VFps8_u%N=j;!!&o1J9su>rO!;T~LsOr>Cg?<+Mj7eM#C+J*g z?hmYAGE+3{?4!6WO@)izcyK>K;zMi>S?buZj92#)we&ap!)(1;*=uoF+5y^3oBGn_ zH^f1aK4RV>`h8suABH(3GnhKbM?e<5*Ve$ljDJ)FPVIPpc+7xxF66DnfbhSHW9m9$ zw=ifiy(LEyqGF_M1k()Vqy!ZP;js>c`n3oFQHtd0I;9a4gCAXX=uIf(mgr`YeY$@Y z^0DUMsEe$=TB=S#uKdz#JdaG;9xPBrp2ef3302?orZn&U!CUh!8UNbSoVW&&IzA3} zRL-6%TT_R>sS7f<_9hU&HUJJ+sWN&1R9MnrDGdfMTQie8lAW6zW2{$11GRWNcSNSW zEt)=;P0i=kZcwRV4Ufx85-chMRNc7ZTy88(%DVpYPMUK(p8M|gOX--5BsS2)O!rI5FkZ$POQ_DqtKsgXIAlr9K8+Yru=IEmr?XuyXP1yAFhZ}0Su zLcjjmUr1ir+ts_viTYDI2?h!rqRiPh8^M^aC{i=Yxt)aaoXf9ykWVZS#Y6Io=wk%u z_|H36)!o(VbXvRrEfz*sig*YJDRvO$hIoJB(8iB9P!TgXVcbwZ%nR5tcuKZ|ugkuR zx45$DmdJjbDPvX#Dfq!)(4lcP7l}Rf>Nt}UK;mOUo)W&0X-Y>U0y|6zq<}8z#A(N! z8yaoXZ3_rbxVlkrT)JEk01O$ey=rq&Q*UV&7eXupL7X_X(-FM3r1rX$0UKCtKRtUu zM)i2Py}XNXHfk&N4Di^KGcruK(+9FD3~ovc_tf7J^$bd25dbb{7keQL>9MLMJb+nK z22AW|>%AEy+77yH+v<>2l;zpcsM4h+Z+a*S9+kd++}ps7rf%VfSJJ#B3? zSlQ=?AVA+-r?&;FYLO>s7L4ab+n$un31D&lTZ8;H@&=SJ=(Kz<8+al%R?dULR$UDcCcQrPmv0Vt%1H#GKqqblv>h3yLSls{DiA69 z-XeN&E}GTNB?ep78$65AMs8g|^h0)U$`8I5MFK>eoky+@TavRm($ zw3o>K(qIMnt`E2IuVqW~+Tc1|9!yPobh1#H6=!hcN)S4Rju#O^&j&=0qL3|z`IyXEx zJ}9=&D%HTNF+*dEzt5JKch#T#58E@OwOlMf2pLgs%C@FzRch#PL=X;lYHm>~an0q2 zuu)%bN0?0XkJ9b?N?aSvSEZtNR_uEEMNUzLi!j5XgV5t5GS*$>d5EKO#GAeBi9B2U zC>5Di1GPEr!HbwpSfaFh7y}NQ9YU_-7$pq&OkIaG3}@X7wNOkhKhlT$8(!29it@Wb zOkgk-Tye8$Ubm~Dw{yg)Qw(6$eD&U|3u3!lfc=yJ6I(zXG$sY@Y6Nx)>pI2S3UZXB z?q`Do+}l7}+H8kN!y8bcN=N1IqcOvgl-!3rKlqK1N=*lxqR-raD%wEdl|o&rs{ zSX^QG1+Y3C1Sg?5{MNrp-uq1O6d~#M$ShcLc3ZyIz?Ui0~kVeDMBI~NOk8VK^zBwOEbOvTFg&gr&zp*-AHZ?BiPy0ErF^=piS%2+jsa}yi7_uwu~3$ml-;kI4{G22 z83}x62*)TS2!$)mt_iHW&`hj-d?YDl7w4G`Jcw}UJAPkN5eXJLWU}Qi~F>(C-2X-xHInYo)GN7*Av36^3pJyKXb%+ z7!JL~wg&zFLRPWR1ziU9)(G2RP^Q^5TBGh9Whqy?A=az_#h7<5ous^-7t7 zSNY3Dm2^GM`t)$mUgh%Dw=QzdVGVHSHWwYM?W<#YzGA-#>{h zvC-{au7uzOvQ;!v&_(5YN(y$0nK>E?BF}6zm}e+MDa}u9Nshb`oU><}9T3J4t*?zl zx*jr)g7no`C(BiXOM34myf}i2S5L7vm^K0Mw*EuOYm9VUbn^0$TZM`nSf}Idt`kVu z&{<6*ERmT0#10&|N1>4kv-)X)51=WbVX^P*4IvaKzKrK^*CY$X^#a>Lg5~ zO37*<+7pXd0x?7n9gJ*3j&o>#>+e%VP?!*+V4p7D~74GYgs;G z5}QuK9K0pGV_{8%hk>R#=|*>%jk)l_l-mm70WP}}s~hLOT`V;iBZAz;)J=LG*Hk%@ ze?l_5ee)3$78b04NH&w*r_dUQjWBxGV>Q|A-0N$FZxP!WH7=yz?rb-dQ%A|!=!7eY zRes3TNJ+J{bvzX5e6*O?$zM6FVXT(&ak&w{%*nj;k2j#RrtE4 zC%M(_!{Ar9NHD8`vPSGza8CACXB-9Wyf%s_*%$caZxVge{bm&jXWKyCTF?_YQheug z9i_rT(PELLTu>8@LMzgBXa#mvg;C_9^_Nk6){*2WjLEc!8hP3~m>;lTG*7p4C)L@H z70x_?5+9o?`E2n9y)f9ZRWDzY`RHq(pNxCl43E+umb|hW`)gaChf%7yV6&emJiWRe zXsPg3VHn2gpHi$AFm3c4#{JxElYKZOI-9$?24%b>WIX79(NGdPReAt6nLfXOw)&Vp z@k0k@XXO-BW#^S6TQl(@2l*P&oAH<@0uZVgIvrF~nmnFt(lFPVp%2YcgiiOO$hVOZ z=ltONOh=QvsrEr+hRL7%Yx@B2Bj`yNkY#)d7I%j%FDAq4@i?zLsczibg@Ci4cBn-r zzfE%rUMzOCo8i!cuuMeJ$2jm!o)ITNP4o@OHc+z`cGJ#)No4#zUVTLHe_JT0gM`h4 zc5-OoZgL|(z$#rt%QPB*i#0TAJ3i3kg}#L@B3FqOAv?6Bbb&C_^;f6qMy`A5u}*U( znk(h@n3G9BmXSZ`cGx~Qk^yfsnEFFMsh*G^Y{|#_T;Z0NLXt<70Knr#lWEW0RPo(#rrR~@8Swbr z3yVxcIdknPKF<~|wQ7wc!AwVFhi zSny*s(842a4}e~_B>_id?8#J8H98cqNcw&EMQrA0&gmv-ISor&gX2CCk%%J@qGs51 z(eduY*>MfUJr&z5Br`B1GnBk0PYxD*jc$f%GvV|rS%|9$Uc)$^E+l^`)$wW5P0F0o zOoA<9>GtBnq~aT={P|M8ho8;qb7=mI;{F72x*;2Q-QJMM`~D+3dcxS?3c`NRTEFJb zM&r+g_gC&8Cx&P}@!PVx+F7bwe`L9>FvWxGWjtfPJ7g*PeMJqWE>oM6M4Oka!^EpY z1l;ujr6YX;tf(wvR8G(VTf47DZl#4*8-vXNvk_b8X?6lr&jK7!G0%z7K)kW=`bEc% zN}vp9@@Z5iLxU_?1E<;q2EuN4_j~dJ9{!>(M*1tjh+Ks4=kZH~*OJZO0#n61XUY@Q z4et&@cozg3Hx{|peP8asV4Uzy$8~|Ysv&`_*5Qs0pPNJ-v_!ghF=ET$kUH%em`Zv= zWf#Ek{CHEfKrMR=KjIP)oRk(kock8qWfH=3^9**$2p!EQ*fsDYql&%QiMTo$?!)$H z6u#G$EITvR2;gVZvKj^ZXyfG_NPaekAvbXS8U#Nt)seoBv_bAiAC~Meahkm&eodWS zRZu9Coi`twA1p$nbkM>>%Pt>P8T4Oh*2WRiHWZsaXQRl%1PZGNmRsRBRg?q2r+xg1 z;?{x>WG1e(`5|!ynVl_h^hpG6B5S~O=t>k**SA%aX@oIA%Xlo#kuzICL}8;;R}7ll zkLNo6S1Ca>rOxD_`39RGt=piZU3f!>W*)N$ctI6t-l|;;PPtEjr$6M-2mXV(BdJ!K z^-oIl5+W#X67L$`OtBj;?z{DYjvwwci>(k56SFPcS|*{PV1PP94$?|@1l-wR_iH4Y z!Ow?CRoHp;hT1U!^fW#Z?=rBT9azAoT`MeSN%`U6ok>gZuvn~ejYa;f+CVj{^8T&FKH5} z8{TCWx&>c$ugsP9T*yw(STJhgJ?btx#zL|al>c&?nCk=F5C2wL-I0=)A6mE3x!}Du zeoD($Ztr!7ikYE1(uQQAk5)Vq;h**BWEOr5O@ZA`s&@*_s@JHU14Te$30tcxwL;5| z(M;%3E8iyrPivG5FD+(f3+9F`91�o!PAS)y*?M**04q@apqB>4a=->#(A4+%FPU z7PXKaSlf%B0}HMK zcQ%%S$^KG)x1Ij!(?Ut#H7F(6+0`$)jj4OW%){gE$s)s&OdyPTAbjRjg&5ss!mEyx z#x8Qgg0Q9jYp}Pc+S)Nj;%CXFwe{Gj}USa*b zv;^uh6U~4Mi_3L*YUidwKfl{u23w7Hiid|6X zqytzKMR>9ruXPgER85`vRri|x)!3B|tLqpryQ*)PcXfqnw%2@Q6GRK-$&r%mqZN$- zy`gP?y5-wgwpYgmp)tV?6UBib{pH-SSpm_&FU%~-RXW26HwhWwg%va9Pl<^1vDyg9 zl&;+v=wv4L%13%$G!YJ}A`zt+@s;^y2bXPpJFvqT&_J^zzZ*+(5i}`zqjctJ@gir8 zly6wR++!-qDoS~pYEw^-FZMNImwFie%=pmx+=4!FQg38n5XzW@kbNsv>w31Q zC%^1J5Q77%xvvLz#btd`HvkU9f*l4cmsFGxb2V`qn3%q0nhVng$dq1Xz-CD`Vp7EU z%*~tSvQ5AAB0jSm9-2+dVdT{pfY5*o5(hQE-CF|gp|Nq-{4vpUsf%&S5IB!ifhCs@ zSH_EI-q#-UyB6&kj~Hw}V&v9nup=J-ZX7|3%PE1xDWOp6Z4E;wUkq%IQkA^BoYnDD?hx*5dNv?bf>sXX6-|gOhtM{IioEzdsIHrIgLbefR$ww&S5Us|0^bqXA>6K*A-)%siBHC(uGpDa%FGs6@Iwx7L{0 zyuqFEnFcvW((`5E%Etmu+WFjz+a4-bd1O=eMj=0V##Zhj%cTj?HlESKtUP}32O}Z> ziLlu@+iXJJQ?roKPl%#*o7er=fQt@Y3t{A``99u!kksGTG1~ErsU*L`4o~suWGPIl zKUO0~o#X@tht3ph7%Aj@9SqAPTy))J8z3j^@9FdDI8V@ETxsJ{4`68|#iiAy3qPbu z0&&r*BkyZn3ENKQmt6%rVf9&>i6wrtMqsc3Q6ing^}oW+*$Eq=ZH=KJUTX?2Gx8Or zgLk&Oa03x0b;v2dbK&FXi?(`U1AE-PR<<>;)EyjqvjQ!5j@Y6m3Xh{K!$jT*LZTr_ z=O$mXWLyNhku~c8P|Aru5F#~ipy~%3`Yu(_5<}- z&uX{Oh*kUsZhYMg62V#J+>r5NoIul|R^+Kjx?B%PyKuY~W%d4Y-j8lSOJ_3-A{HaT z4mvuaq_HVj`K-SzRP^UGz?SvbarmI&uFY-EC;J^tN-sG(N$*P%bD81r99^C%s8F@F zf6YPj@7`p;N^mtTeNOg%*bT;c<9gujE}7K}yA#c$+D}X^_6TGLs3m;pM7HVYle86k zvMnkMtrFkaLu_07(a$q_N*aoq(%i#&mr6wweHRWU(e z*n3RGVfA;bM!FkPahAj;op zr*c?gA@sNDllg3Mj-f-nN*@fl0h5w!kkNU}QGa^BXCS)|xMBB}Qs)QN>|zuRYc`(0 z5<`8m4e<50zWcyo*Id2>(*l1*PJU8Q*6>p`H`<|p0Dsd(?t1SEjBA!L}%!6meHz zfkCQ3mzgQdh`q9qD%-YG3Fx5V<2-^bMZuJ<2|KnAz9J)@$f|yghsMI@(Co`aj@o3R zQuCa+2%-$R`nfnMn*YfW4$pJEIk=&5h@01>D3Vvm*z))rmlU3*!`wkqn3IiTX`$2C zE;K!@OUgw-cz+)dP_d8K^oa{)>qVQzzn%ptLqMdb(aqRbKWfQMWJ$)DI53_W9n_f6 zvlKBs7DRTY%ns4vEADUdp$%niiXf4Zx;8bxA>CTeGeLtEJSf#7;{S(dLLknZC_X?z zb>;OJQV=CX!w=6n$FQ2DOz_(!Inlavx@eP_u)7xH}(CHOD^5Izz5?Jo#+ z-nyKO4Oj&8BFy6T9_&R8pxcdebGzPUy%{ow{d|AYjA=^W(s~j(S;XC@6VqxZy;NAl z@otdLXgbFAB>TIGDYYJ?f>(2xbbMIeA z+7wD*WQ#TLxMqYE=%xrwD)=&N;RdGE4h>iHBb;9q6ww}Kc?Q zYR`P4`^JIlR~e@YKWB>P&3xJqNkpbb$kSd2G(z8jR{UD)@3l?&pW0_R+YvPLrPhKx zo~eWRS0J2YqOj#u?=$n@D%W|c91xS#e#i` zru8j{ME=gYdeqp3a>J7q1A%obnZ`^Kf##nn!}*So^nGC3+-@6d8pdfd4YK7-_4teu z5;XXK@?$F3RDfh%F=AnpRM#Tb)c&n{VRW$(_;pWxwi>wwogQvg=nCUrF*?cyW9c{{ z$m$Ukdwg%z!a9Faa&@1w%0>4fh6v{o;N5cWA;&gfP{zt^}KXK!*u zEL$>{ujhp^epNX(`a_$2^R%}%D?!m;FdZ768vwBcqy)$N-3Ao&1*A@l1tebY`LpfB zlF{*H5x?#CEXTA0&C)#UgrboA2N>7F)M5ouMqbX$nT+#uR^nE9cXHI$WXRSp;53-A zP)wC%cD+H|kQV1Tv=>@%Mve7c2z4f$K$G()7rKR59RdjsrnP`T)K0{EY<#h2&?0yS z53HC0a2{$R^lD%-RL^(F>x<*iZSt8F;gn#s;;TN%`}y;6sB02h(%@O9DU;0?PDZjG zv`T%?$H(P@EDK-i?xjDZ{^rlQl*7ug%bF9FvtgD>;oT_qDc^JN$#AEch5EK>Si&cn zv;U&Y)ISL1?WtA(08~6uON)=oH3gSfxB6q@xF3Il05;KPx)cB)M{jG;@ZgARr2|m! z`EA)>H^q*%mff(2sq+z`B&-0p&6kaybJI7DYQg6r8HLqj@!RL%J&z{%O^0MPz}#V} zTYpfpIZ@A8Gj_h$;T5lIo<(vN+aG&p`Z?ixMLnvG zur&M`Om@V3B1;|dbw`C0isqAQ3otOhN^$4Kx)wC-`s!ad2itmc=fnCA|IS(@sSr3d z1gK&AQXu>aXiI21B;AL*3I78wct_$RZ1T*Ww`N3SltQKr7<&c#F}0@=9;Q?FWfzw? z5cgwS=wRPI2Cj1lUAuBb7sDT)mp~sh5c(Kt%@S#VJCR{uE3eu4o&pG?WB)RFd4MVt z;?i%;c=Ht_#lPSPOESp#`(JMjpM~zYY6SB`4O#KBpQM6Lw}l&!{ky0q9Y+R#9L?;)OCUOV1^URu zWi}Y$FRKpF_p6Tmo6vbh>tVQ3b0+@&2e?wmOacS1fHPn-xO$(#JLoA&I)yo2O9d0V?@fZj z?tEkMW8@|QIKob)er6uD_}KUvXae(iMpECRM~)#_)Pi?*oemrk86_@A z3QLEwKa8O3YkrfGhrcncQAP+x{^%c#KU?aIB%)>`Bo#q}Dj|RVhjE8T{~zFJ$m%S}a)Ji0sAW)fL`{B!$AD$f6>LzC z;bUqJ#5F#H`+ylx4v1oyqd@h9NBj9cwXZ~@si?ih9G&`XZ)?FiQdqnwH8h0Th0ERq z{vo)eZwf316fNVMz%&W-G#QO-8)Eur2MeM0AH7r3Fnf?I^#4;az2vK{)J7}xFv2R} zRIp1~K}{)j&M~Z`Q5A0tW%p~2i^Y)tui@c-!M`*$_`^Nl(8EJ(~R91X`}#{sdXB+`)uc z?G7TKE`a|bfcz;3lC6Sr(Tk?xS?+4$Ay@8Z0?i)>lc0W7Ax@laopRBaWuNlf13RKj z@?*Iy!-%%HQ>0H1hI~Mf+5w)>Ch|lqCWt<@jqgTGEIiMV}$^h z&}n}dU*oLvk2Ci~_fIs#B75=)BRER?r@Ug)wEm-jd6Rs2xvTtj-_kmUv4_V4@t+<)-7z@G>IT6EyR5JvgZ4y^EY^Z37MLxyux*A`K{zsHdU zSVr>GHza3CPOQnqHYXEnV%s>eZQC{`wr!tqV%xUuP0&AeY|>v3cVp~ZKtEve%2h4uru|d)^et_87y(9Vt z75~3AtrUN1$Nb&by zz|=9_)E>sZh=1TwC%z!)9>>364}&+KqK9@Db@aLz2aODhMhvh{wQFN>ud+Uyno89?mdbmoR+2l z9fp~^jaLi7I{kvw-=4EGjeHG!cpM(byn(@J7<4^h?J+s3kB-FxJ^CMJkDh7HSePbyh3v~;IsZC_#Mc8e&N!d`glM2~5}t2FGqhz(*b&d;a&F=w2!k=X<3 z5yxZ^Yxg0HP7xUSh{?3q5^NP&I4gRXRrXALiEqx)Bofth^`Kned5%LfK za8&eB=zMp|PbTFN6EB$K&kRtQbp$k9FXp1-wJmC+2|5t=rWOznFZTvDFYl#F$5r0| zo7bM_J`vBqkue(QAHH9X9ejqPDQxT91QUq2nKIKG)P;%ZNS8*OzzOgY9uKKME6zM5 zqu3T+og=~cgb+wV~b(e%th zJu0HFF!qGJ@tGZfMYGvPe#(e^zk)m;8$3N4@s z*0GJVjhH>bcZT=^WyU+y!dIjOPV7AofwU7vP7$e}d@%|MqVbf4G{M{lnp^Uul2YAu zq>K^VPBAPb;eW(+3MS98&_lXu(S0O!MT#p5UF^3MpBuZ3v;Nl40?} zh6?hz8c>@`s66l0n`Uc~&~u5q!zQi}G)2ob^IyqM7^ltvr^%AB2}$`>>3YfRP4| z@p6f|qqEw}x_9eYnkQ28Hi`jW@YM?5WlL%!l7iAY!_K2(O{39!9W~Hi|!%qH$Uv;aieyl5c zi9AEsdlz-#fN+(%OWN&l4DYL&0{m=*Xh-)7ZyDXIJE_}!@9{aG zZrjG2%&AY?y=-1JpmwyL5(t28&6ZQL2iaT}rUX#@vOX}luOx3Adp!V0h-)~DX3EY2 zrCf03tkX)jF6k^{79}3zsCqgj}lhi6eS~zD0B`d&z6zgyw5JS zHFKGeI$mR_S7);d2RYaIBi5E(?|YX+*Rbf}(m5nuHbO7f#Tc_mfYpFp3t0>fCW$A1!;BoNhq8@h z5CR^A)~3sYOiIomUBGAI`q7VbGG^&J`K6X;D0e-7-)oMDecDcdZm5envUBy32?-?P zH{Fz|%OE%pUFsag{+c|Ws}E$*I8c(cQgS9xGfr6QI$oh1D>-i@NmQhfZX!M<&1PW9 zwmI3=f*9$v42sHM7Wjl4BY@!%lf0g83(@6(d!ZFifdyab;n;PuGA~xNLv5~3fr#dM zkE^EB8jWbPMYt%ARl-+B--K|XQY`wKB+T1ili%kQ#HLy3NtMVC+UDM(2^#}}noFfC zLzI#@e<7UAlwd7arq8j3!{V1Sg%q!;RXxr<3^+pxYwA}zMuwvbT-&J)(euFwK}h^P zNJ+rSgjWWfJJ=}uRtmjpn@Wp9l(a|<>qZ3zK7VYo%Wro<*+E!IC*7{jsOl!RI zB3MCNeoU1UeU&-p!)etPs4)FKMP8xHs-)Tkgzg&znns$r_1tWVeTEoc@fIK<`e9^- zL%c;C%YwLP+)(m}oD-Nft}l-3yck#*9f=o*TOjlEL0h-$IK~TO;Ia%|ln&C4EU&+A zM(N#Q$xq*jIRgGckU3S=A1>P(3clUqf|4#eWEoIHSO86X#&#EKyap>i69`GK;K$HwgPRSXX3i$ zP~I<%x_Yyqw|q_cK+BXJYYxhid`S`lq39Zs2X_VQp$pulai|R9n>-FLp;9l{PE{xy z?xL%yN4_+KRtlpzuG`sw3lH2qbJY>+I@}K2uoW@~UB6jl#VmdT2viOu9=ZBVv8H%$ zg(Og+_9}WWA`eOQWu)%g3wMD<4|xKVMShAk=y*<#$2ILEl%=vZ22dZWURScR1=m|o zL+LHKinjASCFjX`;xELyH`&IZ4Qbf%-`FBR_8`mn0RbaAGfbV;uSNkn=2q4}%}Pp+ zSxl*cNV0$l>)$Q7o(Y38PDg@_PS`;qZ}d#{0?_4B8VSsuSx^=y!~Un(T20EpTSt ze@NC72(I~t_4~~6@r{Jy#_j;0WOo+X=~P%&+N@d4&|+zT9fK&rL3Mb0FO2Fp?&a5L zlaU|P*)>B24(O#xoh|NTH!J%GC_sPxy}AK*RQ|`EfRq z2!2{jWpqyGlXm(w;fE^s+!J}T~GX2>TNt9&2T(%84Z8m(@-ACzW7pLyLG$T&N`xO{3`UeX} zq`3uRKYim$2AgjnAH^HY#&~KI1AiB0JeYtKN90rCIkQ>x&$aQtCvDSD9`@Uk-gYFM zASwW$M@S4TobFVgU8Bs&O`Q8`pZkS8L5xWD*s-c|`>4=(8eQF6#jQZ?1sE+iM>%wxM>FNxQgg10fSsnD0bq~%LC(Mnw zJ@0a$I9=_bN&+i%Y@4EYq}{(*y~62m^2-Z@cE)MY_b=cJg~fbwV0KAY(_GV21<|z& z9XfW&G^CzH_aUo$lW$Ktidj>v`uuA3l(liIKrk$K^waw&>G!dOCNgH{P<;NB1@Ku; zKn9a$|3Hs52a(_GHpp0BJo&laA)QhV4s&jZ3|r@*gn@h2R-(dRU6sQyau!O>*DpV~ z{YlIEtzjoB!#Xw7HXA}ba~ps88p27V3LQOX@Vm*zq?D0gKpeKr;Hw05ta> zA(X-_+%=q87@Qc(rSu?m0rDLsmgfzjsU;qX2Di))rwzmn`Q!CS35KvFGFU^CqiM)o z*@idq5hBj?neLx%v)lj?BTX{6<~lJC!zsO9bE;fr^b^bie8bzO@kaQ`k=9r3lyOQ6 z3s;~~q+Z2Q+I2&6q&nzlW|i>V$9H8K6VT_C zD25zLY33min35=JnE2KxG$fOyW}80prGmh|P0R@$CiZFE)_oNpcwd%hn4!?g#_g-c zs&{yTj7WT|1LDN~u0twb*hghZN>VHW4`$FYgTv&VOc3mxNbZQ!2_@z2BL>?%kWDJ` z5lHhTLbvHK+m~qnTu(3;il;aVe7Vq4?ywxbKtp)W4C{6qShpClsUx`Xq1>!F0z-mn z41>uX*HuLr9}M7GG}a@xJwQ^;Uj32f#A3syjNi>891hwKKMSLJ91(>g%`$w%H>q(i zk;a{7cKJbtx+cVMUK$ zc)P$~jR&1aoL9OiKoiMyS?iV@P7krw9ah2*cZutX;gOx5jdJnPsu$IJt5r>!O9Zd` zjMph<+>&q0eOT@jxjV6?&84HIU3+;dJg9rZzB?BoqtmYPPMi<61`IHD zIFa&x&?6kP-^a}`4R_#W$5YYuG{F@g*k{Pa$euqn$d2Sj>lLjCTCyJ6bk~RfKG)L_ z<;C^1;D8j%h*1135}`PXR-*l3&#Tl^TDb8A;$}`gO^0Pr6Pr>&C<2;1%5TqA?2N-E z4!65LY*-e^D3Lf`?!~T@15A4e6+z{FYxH&Ov6F{>OyEy$qKep{+_>u z+5dce!~6#sve;;_lY(}k3*|2*>*$Yy>FuQj&3Ly}S7Q^tX=mlkHtkBHMe;Xkb{932 zpy=&CPeIa2k$sI@8jdEenafLx!WZ#7Ai>_pYN!JX5~x5NoU(T)6vg!x*<2JGnB~S2 zaQZUmHJ$oYz-+nVRbGw`DN`rWxB{(BJR{kkQeyESI;onPj`e^fl-$?VR0z5&{n-Jb z(=23!`ZNxyt^27^sFH-x;LMS)UQ2lNhO%OClpXJ~@ijyFqt0zqM)-;;6e&iI7$qsW zs9~%Nc^X|ree?=6k{3bg9L$)US!v5ZR>aW7#A@Hw=ZMn}FTz>YP?Uu?Lz!YjK02Q7@z}nV@qZ#Ug>~SUto2=6i)7kwn*?mtB@a3k12!tfJD5K z!%uZAIO6kz1ojWW@N2(Qt%aa0RZ;uuSy)2~mQ#IDG}py3OKWD;smzVN?s>gSLrdN* zb=JCP)4U}rV>-HF>68R^u6+F(Xxq7BF5Emc4Vqj^sgN6RG~tcB0bLzoe!!QzOv~eD zSoLK%cvjM>Mq8+o*~ypsq+ZQzgF@>e{hkt4rAk>@1MQ>z-C{j3bQwRTM3#+#@>LrH zivcVR%~x$Y{z8#{ZHDMVLZIJ!QbJsGT*_OK+aIQi(27R0nYD{n!m=!}{cIX$M&HQn zsP86(+(A3&KsYzFZS=YHS;Yfm4x)7gxKM(T+`6a{IlT-1xGWjcDz5<59KXvfJiOextx4! z=x|&pb#k*w1&3BM4jM*q^G=xR%d8R4glEOoPbw8V-tDrKVN_TiYIAigW^T;T@Eu9X z^1Q9H6%&d;A8MyN+dDAvBB(~OzxTt9SAD$k(*X^ zq@;D`UT2^Z8`eSm!d^eB)c%T+S<*l?&5m|M^3y#3=)kzT4@0moXd*L>H_ob^F694x zN$nI?aqs8}$NbjWX|Hw1UTq%{xIRfvxqn(N(K?{H#^LcrPWLtF&;Pxwf4rQ_m)vz* ziRB8E9+;Dq(4s*`LrZ4z)mO(4Y2hQOvV-kx);U|)-H*g5D4bw~9O=RaeX&tg3Lxw^1&>M`QiP8}B}%ljvbzcSzy@=6*R zVK6S*dZB^+!y;5bMr);>SSM_995rg=y1%4rh~bs2`dc?^c4-4gRr(F)fA67{CphpT zk`!rGWH1F@D-_0V>6}bvV=ja0|^y;h?cpMf`4m2Af654$(Nme zPXx))HPQLb4IvtZKp-K8>4DcIdy5iKGBF?T`gcn{FODdn43Kb!ehmFLDRGSq0eO5JlC+4Tb?yE*wy1>)BlUhMvK8u& z@zW7Ae{|>_W$yd){(?6mTlDt9wVC!2U-MMr!#S1akYBa-yZ`VHrR`Mpa%eu(1Am2Y zN^?TNQSlw@%7b|Ybai{Dv-kSr#r@aYLRmaNt7-+>y&o*e8`hh}=f@^$@Y^eKm9t*g zGxQ%jurfYr8Vi}KmENeBT-5K+ml3N4Pnd?Gz??|3mJNeeFG~uD^3q{t)zuBhO3`4v z_joMC<7GC=DV+Ll@c87B9+0K$*?1pscU?8uekLRD>=WHFT*50&YTqn2fu+*4ZI~E&9J+Pk!_eWX0CxHy#4_)*8Sr*Wft%o_ zq{8o>Y08UISzQ6v?SOIPm%?X`)k9;h*$5~8%O>$R%0{(&LOa1xO5-4}po@5*h!Q@D z!kTJpQn42|ut207xrg3Na~4S&(6#Vuo-7?zquf8=bi#d^Mmvjg^L=R*C=?%0SD1Dd zi~gkReijdifjk;#u$FbYTu09>fKUCM?el{8%Iu^IVVwdLY5Rz)_DNh7zJF0{|Cp}! ziCh)FeVKJaz2yvX%k0ZKa(+JM4Ex$yHb+z?q%Mk&o#Gx?5iK~Oo^pmcW%jkv^|#6B zYg5!#IjA)UsV$0EoRTJzxjmmYp|?A^z1OaNlr;la-g~`Y?bbfhnt|4m@iSJORO=sY z5S{?|DE$55;|Z2DAhcYN#mhDj{Ltugh8gCI^qg<~3}%R3~S;|P35#o+pWt@LbNcWsmj$0@{4 zRhNkF^2KE4#4^MlINy_vF!Wn}fdAXbBK;Ob4;f{T;1i6$u&MiZGGBNUDAMm_*5@fD z7jpazc}37D7?ml0POP!Zfy;SkY`+`0Ats5{Fokm<`ic7?iK<~7gw<$}lvAl1$Sg5_ zmr3QTmVoGj8=MhUpz9UD;0T9Omci*jE2M(Rc0-}{hWSckGAPM=hJS?ZMH5f}S6pnoS#hCL+0+;oB zdwPJ7$j?`an8d*Y%;_$NeE3Nu5ABA@k68lRzVIm99Tz#ApPZQ#MSL?a*)Rhk1u@{P zc$Jmam%hlHUkTh7nT*v&Q;uVvNBg75I{allE2|o7?STf*l98pF!r_{r$nmSKR}OLX z!w`rkypIbcY*V+6z!=1Nvn_Y*WrpUJ#sQ(_Y zw7P%W7Hig~#dy0PV*mWs1^OZTGh#EVJ2?jLq~XHn#PBnUqFblU&45ijR(bfVJ@<3^ z_RQ{Pc)-pTB?KB5-`2Hz8&vcbq>ltqWX>`e8V}#med_qP$-}^GKE9X>AJ6X1@Tj7i zchNp}#diK>SIev59~EkI30`A@%$m;DR3i&sX9Cf0jPC=SMWyCfGV7?KNNL<9G=|;3 zr?B21C!b_<3Von|&4^!K#(?o4&E7KpD`Md3WC>EDk&c%39lcVcLyngq)BjlqJ{Ykr zKw{Xxqx`@x+SdDlZ~Ui^l)D0~#3oxbqfZoxooDODcNF}KrThC8Xd|{u4wB*uV|v#E-jU{k-grejsnk=K-uG%1s#u zCP8`3_~6Cv6$*U#!sa?7)H;XFdJ8;{Pm6&wm{!y4g((uAG(~<=Qu(h4rpQG3{5gFJ zos>z66hz_X?$6pYl!bhJ707dmfO8`a>*N_ zf`6FV*Wnm~f16Jai8X;Cn|ZV@4aa}VX3%_Qp~bp?L4BXm#7s zuKUZ?Yy0<{X_Y3UVU`?m<@@|iIYaLxo1dH*thGIqj(3^O<0Lj+_XnC^!WgCTc_LkS z?m%=|K*$Q8=~+DTt5|)~v!xNQ=ATKGgMYUP<0Dv2mBr8* z__prd`=BY%TFc)*T|K#N2(hW`^kcfPaWGDz&A-?mVSi(9yL3AMkXiTf+8%H2Q? zD+KmW$1HTd!6Y$6)EE)#klX9gsrXELp7A0Q6vzDM$ijgmDW#r&zQkH5yyKnkFgQ5_ zF>Uu$7^i#p8E8H97(OH8_-!aCxBdfq#J|zD1F9_iD@=WOs8Hz z7SUiBmrOfj$l~Ukga|-h#VwA#&KtZ5`tbO0iSqqA%fcL7LiE7kQ#H@zkcexbh0sp{ zw^9|UGK>Us3eA@Ovio@R&H@pZd<`bMLd|NxtocCg3z-VA|0JcP`RvP+Uzz!@XTN+Y zfv@YLdzr$md-IPUky5`l#0TB7 zGGD3wwSVwk1~*1Z?F^+PpXVN^rb8d|}ZHoS< z_0iPb#+a>oqoCkXD?k|-Y5RM$^bWyAGzMWi#IRtUNf)%yLtoZ zt)2O3qwyJkA;og*GY`4k_+F4EuLnl`_u9xu3qp51p8q=kt3Hb3=eYLUPr5&gN?2zR zzf#p@_dextu4pQrRZ=}HpnF*4%ve&tM^S{E*ZGiz5}J;oOJFz8mTNu+VZz#s!<710 zuy}6)&|4G_;f~D~6SgcUOf#tj+);hkA!DrnTA0t$st9>-Awy{D`l)zxyPHV8?JZ~a z_ura0PusS@y?CLJ623+14|5frVLtt#Ab|uds#?^`I->V%o;^u<)R-95Yi)4H`*G!x znfVH_r?*cyXs~p_BT#|9w`2cinZLa-$JoIu`nTmK{+uHA79Az~osqohn~ zG6)BYm+tumQxq{QO_5OIx={I)%Euh)^2=4n;+f+%;UOrU%OP^##ZgB2aT2 zSXXY?uoI1~&67+mXxyLrI}?HAi<_tlZp>IKK;#+zRk$sX;I93V|1wu=qbldf7Qm#+ zB9BeeyX^^J`hw1?crR~$RHmY4ZpIT;B>(#Zx40JgWv>IEXy^!( z-FE;Yz%+|9*NE5PSF8hjtvoBwnuS3Bc3XYLF3Zk1`j>n1_x1Sk4CwmTfo+S z$(>3i=*Nzqp772GwBQ|Bqq%pJqoUCr4lJJ@$6Q98{=XZ^=zLCYgzifm|-+FPM_$V zyi-R8x4ye~DKs=SrRi5D6vNr^K8S<5Mo+FLoyKktYQ#`TtBgzaKD;;dd=Jx|;)rs5 zdsny0^bV$rEVwK1lmDe5D-m)v2_=Jclhbb1-vE{6j-tgHD*Usrd~dHV(3vs}ghBpk zuc@RbJQYx*s}swRQB-*AC1$31ST@CdKG)d*&(`PF-{!DW7q=J_*Z19B^VjxcyL7MJ zjjlwu;DM?`uU*t1n=&XHPEk@D0Dc+nEoS<1-YheX#~v9#9AxeJbaU<6iW-`z3JWu! z$QfI2J^*LL9|MPsMoHfw(5N1lT)w{1UOmZ~$f@I}zU{4Kp`jz^v)Fs$cA=y|32LTk^COK zEuq?MLs;G15@uvg9ARnL-o^6uYnPG=7i@3mLRH(V#fTclIb$5vXb%(Bbi~+Ih}_$y zKU1qmp3M+#$9xQgw+K|-nH0x&>!ZU*Zpw6HBVXCuHFWST0j?tltz~3`Z*`wdcW%vu zPz+#}ui&PUSBpM8uo&iV-UY7T|KPkTSe-kWU0mWKP_6k5%6P{-iLH&7KQG=$8)wik z+Hh@eo(1x!M1>ruEJ8i=5YK{30OcNKQf^0w7(*vT=7lMct`iY6cQ5&dUfXgpK4Ui; zJ8kgKMsBYC2}H_@bvJnXLrGiWJ>n=Vh_WB7Sjnr3cKRb?nkfd~8I!k$`0B8c>-89#afok78iqQiZ<;h`39Kh*&z z@peqP^Zt>;zZAzN9P2drEKcRA)-{wxdm6z;Cg&>|XUyyF^PZZRO9%Zaf`UUSW-<-B z`DA&cSrHh%XAwE@WG)f&oL3!t za2`th)>$Yn@Da_plx!sn4z#eKx6zRBdoxKwXMpUAKNWF4Nwi=XS`9wy#u@c+o9}`4 z0C_c8;%$S`?hsR79)SlztLPmU;RZMp?rZ^3@t?fBtXY!p$Ye^G;XpB)`1UC;;!1;qbq9TzXW z5bv(v&Z#FER2Nhk+LFJO*?>KSKG@HrzPK8XbXH`UB3M^EIT`+2<6!mRDZi#-i_-BM zi4%a%Wt&0EiPy`whIL-F!fWdUOl4nvYp{cou$3%cS8o>N4?q$DT-$gn+i(NpyE5=AHR|RkWL<%WVq^jg18C z`DK`;y~lmV^oILPIN?%Bytv!rFu`EM;bZGq7ERT`3maWYcIwQe!a8@F$#{Ia4{9Xk z6OH4ga&;>nWLb43Z2YQp>zaPqs^Mxt{&s$S58#Q90~~!-#N-<&!ZJXNA`i4lg#fWVi7&0#AW} zC|e*%$nt3xHGem28FjR-l4r+&qbNi#2lQrLKpVD@XoAo z%m>`0H_ED46X69^Z2SKf%`UotfojR{pkc;%fk(zL#Lgz-ShIRj_ebB!$B7QkpQ4 zE}l{K=t)YGXsttkR=ideQ&SY_$SUVu)Mq>R2_cS<$K6uy9m-D5lFBqBg;i75^muG{ zVC+r9&@Ne)ofHdiDPc%e`l_@x&xUfA$+u%yHq5sYZ**RXKGWh(lP1=zMMlP2TMEQV zuG0@_x+GK#^zP;hleqjagOTIiF)i5 zbv(;}MC`xjAj$P^W)oJt+8M};QeNGkQ^=|NzUL&&h zV)1=TILk%fboa@(A@x=!Q2ta!%tX4ER1Nq)3LT|zD)-@wcD7ZL%1g&&-3UMP5xwpb zk8l^~*D27-G}q^Gw@2j2M~2ByjfdA6T`pwE_u|YZf6wgro+`5lAAP(f@NL6l zb4V*BhBP$oR%&mwWN}(K$};_G`Y}Kd(E1sb+ucszL91eggVi>Al)tGqFFvkW2Z?ga z2Z^NLp?Snvsh{2c{q7f>##Fi5h&92;fMD=k)O~geM>Z0{FfTz%F}VNgK;A|MJnyzw zOla=)%VZ!8UGb)K>UF;6{giU7u5OigkrqfcP zw6b;f<2@F6mq+%Ccw#vmzlt}JSfq#Bd5diN$+#Fge&`E*(~jxR!%P*&vu>wlsiJTp z{K!9E_4*Fi21eDou(X7CKaZ_wMLHsoHTG)&9Dj*BY5(G4A&66G4Uxa@>|iIxQm>kw zbbKIjZI&uWJYY&OWnwq%5oe~;smNC6Oengs`}B*%Pd=jcu~WJ)MBtlM>By)^&Rv?b z`7n+rmg|Ch_$-N@CE09$$8L1Lg>R*5wciii&z$abogF!Z4#;>H7~v$9x9v|4sbn4+ z2vK0DesvuDZ5ubRmaA3=)+8xIiEZK%{JXE%UuLh^r>Xj_%zKc2(m`tCAVp67*yQfIoIU<#3sOA7Y7*@&Zg38xO|S2JeP*-c9#V z+c2mEpS^jx-Ss=AYP%RJx%;-!TC%u{q{9>^kzHPeOdp#&GK-F6KW}kmH3)T) zjVFcqs7)8Tax$>y>`IDE7oACb$g1`|$31=A9v(NTkd%AeN>eTgek;|oA-yccHxAcZ zEG(AlegyKOm)#kWpv7@Iq&&B0HpE-?4C2KesTOuhWBo3@8jmqFdr9@I{F^?=GT+K2XJ=(=-#`|mofo&KEyGx;EY3fBhrr)c>(3J zN)i@8L?i9uodbCrOC819>6FFC zzw&cB&}2NUmj0pIZ!=>&5%vWRE;q&UwPKRHAtys>6Qu=Etn>=x%Y{?n9Q{8q^&50WPa%U_xe5L&d>F(KLR*Th! z20|VGv}YPEChWgwh^mOWWrg|Q@0JuKtkI^~QRYKnCuhFW!|7D*ZAx>X-e=~$kZBck z+$U9?G3#H9`Y(~2*;je}_A=B92fv>v1(M6fNjf;m5=0^3T})PqXVOCqr#vd$Jhw(G zB5)mg=`ex)hMH2ZW+x)sIC0lJ7@cYBa?M~DKjI*d^Xz(lJ;IT|bHdXRp*SbzI{U;4 z#b(aqjkBlo#!gR}AbaoI2)7qGu_dK25ekZHuN?hh18qif`>nJd0QYUEGCU=2gCIUU zj1LA`01K4-nj`-@3JK6iVYgYU#ZX^W@ytBfkhym91xO?t7kb+QqGh0jzRtoE#$&=+S%64=yuD$-)qt?l4xwnOes^&=c%jY5q@%${Bq_1mP`jw~ja?jJ)`m67c0R6aI4rWt%0LWM3(oqLxVcjXDS`1Dyt zjn+6n?vHlRN#;JG9s19a=PgV=c!2YVns(FAyUtMPi)0F(9`FOOg;1Lm*3Q4v`ENqB zedOJ$oE8$0&tKbQNG!+fxxtA>4DP%yHuDw8^?h#})Y-rhf`g)YY=@Jw9_^7}JRNY| zClVZ!vdN#$=G=8jLT8o^mksy4QE-&!Hn?8bdW&TbvcwfmGtLfR@24 z_vLEe_)oJJ`&CclT^|+!2U2P%cKJs*_E*=8Tt_I;!kD`9^pa(}uEW&nkvcmq(rgzV@$C7;o62W(qo0dL zE%?YTp|5Ks8!`>J}aD;%~{#vNFNUz?N6BauZ~<=schRHS%|Mr z0BGVh{rAuf&g(>ed9(H z*6<9$(9vHSK`fZN?ds-1J|`5qV?3)};+Q6QXvvhx!&8#@Tjt4FVQmFzVv z+=}$mfWibpvE%0l=Mur!0Jgj8Okc%kn@Iaqeya7yULV)qwbv0U9XK7u_r7N_ou}>T zPfMB9Dv&Kahml&=bGe|HRNFd%@W9P0UMracYUgJ5lrs7O^ zX~c{=@cs%RKq-?xEASl?-F7;(etYS#*s*}_1YvWzBjSt2RP8c_fBi<{$GsK)y-nPW zb2YnBVaAaf$@WH&`XJsy>0iG`rgNi)<;JsWxBJBUF_ZXnanN$!N#bOX*zrkO4BTwN zKOC!kv$P&|<@naer!y$`Ix7A2=`G{Ox{T94U;gARym(Up{zJb2xpRb8Pz81uxzs~9b^-R-8<;OhyDpbWJ1SPxo zYH@I#O(r=GVE7KI6xNiMPZ-Hwq!=UVwe?kv1P99HOJ?%j&V3~wL6+@kJIBy3{En2l zDAd|bE(?V+g%t^-bZ3NlEkN*=a^o?a;JaB8AH;nRJa>SGn&VQHe6#H0E>XgXE3 zE3yVrzjM6{Yi*7o2G&kX)~!ufuVCPwsD+*C#?7l%kBWJesn_09F$R${+G0ME@YS>Y|H2Anmsf5Fr$^Qj5>qewGt}j+I}$=fj~(O9c|X#2E*}D~*Qv zo51WV+DE8E^xMhTb3%n}r)=a#2T)=|Y`d+CT)FeMsvrlX<2@do}o2ApI$# zm?uNyHUL(0QHhjil8>njzN|gBIvwvAHVEfYs)Q=S%E{#vYl_+znZBHxyWK&NNrIm# zL)KsK^KgkqreR@jg5hfY}rwYZ< z%E@fcO+a|oV4ItsFfi&dAg2qRD}wy;DQS|JN0hH;W{a?|7zR|P*(5WE4Q$tC9L7@{ zUBb(31g)(69)o`JI=5+R2`JsnJ^qaK9T`AUSDUq^y8$ceV@K4-@=QgVdFFe9Zt|!a zGIMC}e(6rYTWmR2x3PMZ^cL|60lzFY7BwD>%E9j-YHveieYNJUQ(rNvhngPimZY*- z2>t1mchu>c9`wr+JGp%)W^J) z_9nALy9_9i)^?r;p*v9l@M{HFPU+e;As%B_DlV&mk@xn$dC~SdnuB7@+m3{w)DFdGD{xA1QvIWj-w{^uWy-ZfXW0C@G72|1zGIk zkHNc))qrt$fgRS%_HW0|%91*os2!lajxE*9sa4Ob&kv8|vi$ALORN*S9@+yRJgmjG zQjtA$4o0aa#)nQyhj&_$u~tvx09@w1F&zgc+0;2`DDCE46DZ_x!;6qZSTqXitkGKp zED)EgnwZxBE=taiOe|Uk2RCmiov1El?!M=`nizncZoc~F(uCUx1ZN=`#?|RX5h=1G zWFaq!J5pj34R#A=K`l|9(+?L@6PE8T4>iX)jOT~98LPJI@(|YzXy9*Y^ggS!_ImVO zW1CZU=WO&Y({4C%k#i|$d-glj%#(J!R6FDgORSS6DVN;|Zc~wa@Wei_kL{;9wx&|; zqP?;NZH#w5oYvpK(jqDcZxCT{ik7bZn^IsHTG&igH`7wML#e*CyYea#wJhJYrWj!bDdwW3--0hjw_QXd@z7=(Xv{s2zT%>_@iZsdLy3XXtc>N2!8u}0^(kZ=+ zk?oGi{M1GNw%c+dV0L5J_S88>uk$K;6tv?Gs?e=^I}bOVegL|-OME7b%5X`+81 zNM0$b9qmC*OPp_>bqx?WCuLDg+=dMGKTM?aR;xaD00?9Oa@XK6X&3#Ii(4}U516__ zCC{hg-Qd9cymnQ%b!$=860wr*Uy{5hDLniK=QT(y3df^59rte3jH_dh?M zJh?`4yi;X?S1tC}?ihWMPT(LI6pKoUOf^TWOIKCtzGbK%S04~y?~s!&hbq*8wv!_Y z2){9@#FiQ9GS~cHm6)pEi! z5jO%kTn2hfT^?~pWL-BHYs6M`L>_&_|D*`B@*Xvl%2_VRdtY&(g4JH*(FM*P|0dE~ zHIqyjdIkTKBIL#%Vr*nnB?dqAg_p&Oxc)Yqb>4I7WOjCbdcp!OCo%f!YYqywa*BSD zl*N0^(mNeXd~2863&@qX;Yhkyz^pFYG!LaY$?kiX1(UCBWY!&L-+p(7$Nur1vg(La zqqQ*NlSLDs+`NZhDI7vn@M{L$XrqXu?;#TN`}BO+q>B;_E+w0-7Rir=^PFFK$nqF} zG8un;dt$Ln-JC&026fVmC)BHc_YF$<@``&tftnoWnV2FH@}Pd=G4_@+P`on1nfZ@6UTC?fvhI zn=B;ic91;r;c(Dh`?$`tc9SqobcSW_bQhy=j@Dj1iq78W3p4)GvE#N1n^B`Gg6CwL zw!s4e5B2Vc)uzE0Reib6ja zV;SykEW~V7&t!jPsEqTX(RjaZQR>((okZ5%!O#4+)7uNr{I=?GQaa%l@T435YMfE8 zv-N6s^l%wDCOM6vw2NbZ9&65uofVn(Ctso8ocWmV38paVYT%orncM~2v0ONKJwJbN zo{!6k>3(9sklkMrVi&JzMm{VBDo%t2vfp1j09@AoE3KD#S&ib?n=bfk{c~L2tU4lI z+wIN5R6k|>mXpqDSr^^K^K}~a^JZ9=jv}R7u+`#_lbe|IGpCh{%IB%@0{p;r-e30s zR1wB`dHO>Th~53*_QKO_O)&&lKC{{%xY)BZRX(+dTkk|fYCyd5z+9cTDSi-#AHtC5 zliuL~iQGkcPSW3}RJ+U*tuQ$hkR7Ov#;bn7)1~+h2J%iD^&%1aCwz*reGK>N;+DSe zx!Z}Eg!{uY{#JR}?yUF^JVA|cGwP3*-~$Ng{b{#Tt;30(afKXx+Q}{d!(@jAYCole zFYkW0(ptSQ4}R$PIa1C4EiZI_u_@53E`}QzZ;!?7+r03gSiLMH7>rF|!LCfCS>b7$&VG!2ZTDq}u zTpM4O?zi-Cz6gw_`}~G8%Ib#<5lVjG7>98*-#u99;8Ak5jSJhh%J1~n4`TE!}=96Af3E`!p81b zeHMsAc^^_qc zNeTvqMN*v!TJpPl7Z*_YDzKv7yv)%S@Ah!(>S#G?czCH z*ZjdX?yIo8{%iSoPV`KR9&!m+L2UyM8>c(he17`;PVnFU2Uc<^&yLa!jy1!pBW{z)%rpB;g*i}scSn;r)g%^mtzPPNlLd4 z$5T7`E1BO{taHMa#5_M3!f?TB|CzfILc^n@2%J4nsO&QT?SfE_pfix$4cutxd#33o z7s++%;R&Z+GcK6N1>EIy8GJ}4xN1}jA6|v2I1Sh^(l4$H)ZHtNRdKPF#Hg$IFVnzr ze0Hu=VF-6vm)qAvU^Bh*S&&p8(o&E-7p%F>Xl3K*yLgAX0b}@Tz+6TqU*#m%>oc4W zTl<@w;02x8_~a-SR}91h&9he&fC2^~bM!sVyF0nJKt+jF$(I#^JRYb5r*2GP?G>jw zovV~xa)MyF&`LVj9`>7IF>_k^o{n#RSe4rfahrS@jW?-{^GOPP`(N}JCaZV#!y8U` z&YkvL+WR36&ho&ZoaGQxszw`6-E@BjcOQgPL!WqArBa@6Q51yv^`tw78-d+K=Q(b7 zstmLdb!GpH5Bsz1cJHTyd^g9DKfM^xxB?T#IGl)_VnxdQKH{#^Jmx|#@TPjQZ#GK! zD(@-F_=u+U|1YUzWI?uuT2RuAoMb;#gYPd7&IE(^Q}f{IxR zUM%j(V~>jE9Vs2zap9=B0ZmEE)(uKj^BE>3pwqFjeDV24Py0Zzo46yUYV#^Zor|0H zrFHPow`Kk5e-Cotoy~Jp%eXRaQm6|{AI}5AqY!w5_l$1$OFidZ=i73BoQ}3#Jos=N zzxL0({^Mw`x}t0KGf*} zjC5vHNt?rxzq(x7sk+t=TkB^UC?}eYcEuM4e=znFy>lH}F$o}_BHs0mVzw%uw&67l}O$T$Jjlp_eW39eAyy+C*LSLz0LWH^?1h(19;{Y$kc(44@a6gg2cy%bYm%q>V|KgLq z69&K?x6su4x?QJ2#D72xQ3G{-nAdb@iUNXi)}?>%I7yCZMIfLSvzTHm;Mcdc?r-pe zX%UG|<9;F$+v_rM0?N;qKCmBGVouMq-(jGzrQ%`J`#jP3UpoD+>y+~Itq&5y^$V-P z-1&o$5a!Jp!8^BeK8b~@uGjX{`RAPp_kB%(=w71rouYnRY>bajaE04v&ij3&bS7wE z>-?2~2Z`Z-g8^VS79%bHumsdfMFw6!@uffB6070N3Hsy)$L6|>`061>rTk$NaS{}R zhW^En$I!WM0m9FJAoxTvWCuAPUwZC7EoGx@%{8sKAsQ-L%|HCJsfiFiE z?EaY1H*H$sxq|=UWc&+ybxWV)E}dANw;lmJ65L;!2$lJ~<*uG1+))b9y}-seKq9e- zuT|!EXYHS_BO?C?hI6iwbDCSR0Us?~mwkbTYAosQJ`I$uzyGd*uJttZ3Y}Z<39BBf zve>E&pxNl45Lx1`F39u3D93sRLQGUf+;R**N~01Eg*#IsE>~YqF6WfKaw{j z{!kF2gLS*-hIhrqV*6HNc;1QJh$E;#mHW664p2^Cra;PCc*!roKr20Ox%}WF=hH#9 z*@eFOj@zDA>5V5H)1gbopJ+?n*%hn2NmvrQ>g%p0T*2q@d}{;6K6}N- z?C&SO{tuJIUEXwrMn0cBu0apu)gJlk^@>N;?4y_tMUu?|OSjJgd%-ZBC2}6mg!kJU zZm`jFosDjA#4yLYtWdSvtKkQug4aO>XSKYomh3c&j&`-z+2-NJPv_{P9xUQk;TAEB zH~My8$n(T*G-x%dtqwxHLg7>bZ23~dMcCbF6dx*Ny{zw$_W_26)Cvz9>fMVqV4rVx zjc{@O+;D4=fq72vN^zleiy6Q`U%TKQZa6c}TEFvbbeFK>*2RHxRuuB~uL1w`4x@{+ zdlGTGm0;zN-`Mt1ronI1Q@vEPQ@f8bb;D{a_abSQcm`z4rOb4Y?nCLo2fYmG5ZEHy zo2Lk;<$=L_Lc&pa5dB|<3)KpIbh$+9fWj_5xx1h2IL!f)bcf@Um#L9}JDgHIY)gMA zOdzKQc)disNDIGC)$gORXY3`f z&dZLH^zB$`N{5G;+i(rSF!df75eBuMfcW4JoEFsC>BQHKmCtlT9v-Bh^KWWsUujzIagY)syFvZ)eU3DYz5Hn;jOu76MoWzm0sK&e`Uf{Ns7onn=sq?hiKd zeZF8yIiEY#Lo4Ova=X~`PREz%IKzJKW*S=ND%bU{XBr}@+;N9!LFoE8|L4}6QK&-} z9w~@dpliSB>1I&*Uy>}KD`y&U!h5-AD?&xIfucU{uCZhsR? zzAUj+1t+$@Ij(yEKyL2@cP#g`IGg`UaTf3zT z=UdNPP8zrvu}sxj811kUZfGYr^4-qh012FdJrX%y1^91lq%&g-W*ZzkN}D2Ua+rP= z*?GW9B`9+`Jb!&zo2R^9I5wqvm!%yqa5{8A_-+FdED|RES%P)mmhd&W3)4xct}K4z zJ9#_e#viMNfd}{yib`iw64HK*QT9qBsO~#~ruPaE)GxRv-3>QVy5AQ4@Jr9*V&)MIT7H~kG-$t% zux_ChESsBJk#1ZOYw^?vZzuDV@Ae$p!clJ!Ua)l;>H2vy%Np-ffkpMp$se|ebk^Z? zxt9lyi{5Y}>Gb#l>{%|9gvJsmYVcZ|2i55mmz0p(yz2=UaNf;s!sLgf+I%{{)TPtM zjt%6Ae)?vQ&Ji_`z2=*&_>n~MZ+)l`#i;mAk`F6#?>l>}?Y(ni`#KsKpf5fm*pJJh z<#(U&ulT3`8BppqPyCxESi4!cwenVv#ib%UvL`OIK-PSu1NG;F_HfyN^=%EK8PM0;60CJi*IPYTyDo5fe?wT0 zAqcKQ;eBg=rtsHlP;8`SQ~AR0>pDfb7ya);N1-@;>i^_)$B#?Z$?p1VwB`Sr#Up(kuf-|JeC;0C{i_VeZslmuxstUfJ6$H8*TWTZ zfT^{2ZV+t_AJ}Ei;R=xW<&w^sp8*Vb>(fWNj%z=+VmF}Oe;wu6nS(o*#?TZc`~ZwNIDX zE9P0bN38yk6>e_xFi%76eYv)K-7<+w;om{CCKo6yy+A6ThGkFV`Ij?{aMAw(BZ;1e zHZYCM|7jP`)6@I+7#uq7hs{UE&S}{n?>(QvqrbtOw=)1uRO3T71wWo<{}DM#PP!ty z>GQkA=l-fICN>wj4xged=_CL$cdo&=^!yPDgACDc?XdIq7DLlTBY)nB#QxsqS1m!> z^HVXAE-d6g08jP1o*86my+p~^_H$Mx>1DA@{he|`SN(e1`#YX*Px^6z&-tDHfe(V; zOOWd!sD2#nvJ~s%q-A;@h)ZmTALcfn(Q4)oIaSVozE-zejkBHLze$07U(CNMlPR4E zIGQtMKhk<{CzFj!ThLd61H@x91EV0i@v=XNEvR{ZGSF0|sFUU2{mC|K52sVuCwG&i*FJ-=kgXnp1!VyqR64(se z>%7z3Xn#bWR84(pGxy^tB&CGwBcNQ#Eq1TrR~Oy5oEZFaKIxQGoB^`cj~OlZ7f|Ue zL;g>)!-t*l<|hMa>VpX(Cq}b(=BQRY14p=-KB0MDYL=+f{jjd>l6C}6n5!(mRGM&N zZJyJ(-5Ryan)POgXu*PXFJHAv-du24Sy!Ad%`THme6ek@?)B7(8xM@%pj)DR@{F`} zjE1Y{rxv~a-jhccr7x~193a6cCl2IBZgGKptWP?!~Vd{HU zFR$y?Z=oGu+-@(4^I?wctrr_pI^Y$^Cy$S#p0_RAF)sw*OFOWV%#wyx;RC7FU80@M zd!Jfkv{_J{+QCYIpc32d6Buj;M?RY2JB)QASy-H@$cAwnIBI=1xPWC z@@C+3@wwh88tFm{ugjG0D%fSYKGVV*4gB^5YMHGBxx*DeyjLvOYNu5FyzponDex5j zusVsNO?O)Qslu4-z8C4Szp1(|3*ORYVK0l^E`-yrZ>Z(l!$h(k{@UekB*X0ei$>2< zeQdq-UX%Iv-_p5c7YK?@y|r)WlJDwFw>rI_t~d#Jfr!2S-R38xPhSB;^k&MeTi5a;mQwX!$bo+lhJOP212y z<~ku>*|#qZ!N2Wa67?<)F`BEnsOjMtPia6U|dj*Q}u1j~O zh~5=~OkvCIJK@S4u%XFWE%Vsf^3v7I?Q1w`B9*7Of68``k%3r#zD<=N$w5@D5=B~H zuE$2;72i`_U!K8`Yx&B2q6#L5kHA|iGk1I~wp(i~1VO2K)&sBU9LXCdg}3%> zWpiq72SiNx4W{pYA_kExU)?7wS-)Q4MZ3P2dOY9#E_IwMw^fH8*Yjwzda&nt;)Wdr zcQx>AIgf0HxF2%7d5t zg|&JbwI6<(qn$!c=e+CqvXpJRoHXC&jnm}A!${dYE?k~Q^XqTto0I#vY=jtL?5>=5aG$Ju%PtaE*S5D5Rq+6b?{75 z{AzO2{_PIi+e6>h{TvkpI24JWB0ZyBZht<{hZFnd#QOO}aHD^lgfZXW++eBxBrbec zoB@jrs6kf;rty4(Ji;w>=9HZm~tN(m# z9|`}5Hb}g= zX&_NlLUCwuqO>wstD|Wra3WQrk-Ff3_$p31+3p>|AjOw(!}jtVx20S4ChLEQi$#|Q zH_zb#JL0<6Ga+V-H*W?}rOn*@+B14P^Wgv;ydmP;>k;sWTp2juJ@AJ_j>ze@Kl_o- z+l7A^*OQ-V*!A)$v1jw!%jJx5%dzv408#t#9i1IqBlOBNaMeU!#{B-)a9Rwjp^<*G z*|{Dg)RT*&Z@eZgD$Cx;YvJZDPj>RJeQb%mU;8PS2#yTUo<+iA{`GWOQC=5BV7 zPJA)@zi}O8S)e7AbBcR7wfE3)2u=21xWmR%OB zU~Kd9o~O*5w-4#0Iaok{XEf0p^6M)6c9{TYdC39D^=;HkRP>^>9~4+T$1me^Kz2c`zFip#PHVzGO^PyqX6!0q~{XQIr+RWa!m=_ z+J*a}Y9hMn_~da4f}bNvpd!(b6#$j-(=@?Ke9}k_}_^5kJ%H&H@ojVF-~8T)(dhJsWra)6$JLsSt=d#gC!q&k32R zSAQjw3xncsWYuH4{8gn9E9gUhF|Ws<0Ybj+=7U$1{m|ZyoT_1qew|^J+3zVF+X+Jc zh$);6=FhTo+_SF*A8T4QYWJiI5yVC5MdSS54n(Xbwbxx>!#$mCbr{IjUZg+McH+{B zq(s;7etvGw{|IIA$Cm=y)rI7gof~tlH(qMwsAyMW;C8+L3n-CD@>48hyEfZ3+QaB~ znSZB|dLKChq<_+^f*&tAHLaYt7fZSVCaGoH7pnVlneGp=y`_5MZoRryZeI3;B%TK5 zxm5`KuuG$Q_|NBvSNih_=u7`X5Q zz7Cu4-D9c~npbGf%iPze$i#L?@2~#&=z;N<-XitXf0bCk75WRn@r2K>F3GjMh4@)? zcN`MT;}$KIUSYAqU4mtxjrT3`aJ-3V`n_G+G5+?6p;0&1saJ^Le*Dc8Ua=GncO~9_ zoS07+giMIzhvX<)dh`FtuB!(Qa(~Zk9Mn8rr)Md0y~Euq3*}2E)5-4@rXw?@*&9`-@iA7`!p19m3hn)?AZX>_F@-#0GU(Y3^ zB=huV0~d8)07|Z@6MUsvPQnjWB>eXWb$viowjMr2w*+A$+%bE7r+dX4k@y!! zF6jq<0#jY{#}kovz}0+g?gkI}kT^xuJra>le3JJQvwQZ``|Bj8|GK{E?3{L<=w++p zPLlli;dRfmu*anM4$F290Fd1WCGvbgvIlw zi{AK?J}B)jLIanlMg9C6p$eTqTElk?n#iBAR#yqk{BXhNJwEFD{`hfT|K$_#CAsJ( z{osKCP}3-F)d1Yr#yfv>TQUi*8gr-m(-ud>Rx!+g@YDTp(P>IsqaADIs(IUkn zAPbFB;8y7!Svxs60Z*I#f{zQ6bFj#%{fM)^MBbh-lwuD@>&5yN~b2ggKvDKSh$eg1KL z<}aN)zdz}p@HGFS6Hwy6>|-#PzjRJZpElxv#p!HX9~#jfRI| z+D+wr$IUgBB~sd4azQ4K6H(g+h?(>%7Dq_l)a*o6Z|AcC%J@V zSh-a3lS=^mrA#p@5|eY6Hyau1hd~<512B{7yd8%ocb8j_s;cEX`mqX#wxG*zk%fa9 zKuHIQ+hiXvmlQ>PiXUgV8j#oE#EM-3=q{ofXMtjbXd=WqFC@|a(si`?ohyO8Tb*^fN{KcD z&iXq*tEL}g`Z1CdN*1XeB$kjcu98}kwt&Oo!m~7lb#Nx85PzK$6Z?bdr{nz|Fn22- zQ)IC0yV6n|`#gm!(G<6W;t0o)td<fMm%loDp*9nR@0Ls`$*(cR4g6m!oVMIc>EjF!!2FT*HgW(f9LbLjmA4^CgzLktW2VAsTMpc2Z>#B*ZOz)_g)}%p6tNK+g#U~wD&mH_Df;W zZC&XQ5<7K1&m!WphuOPpB%iPPeZ=z8ym1BADr^e(^+L~+1R-Jm$P~{P^%x%enu2n#@Hz?HbsMb8H9Go(s!Ea8u%Go} zT*4o}2W1iW2@mMe#p_j}IC_Pw`xJ0mekyc7ZOR9wP~`T+F2ITT#rJot#E7n?u&pjW}3qD`#Dr$9L>R$JQPxd7~M%tGHJIgAQoGtgVq!;IHMBcWy03co24`gk;N6+A-|olBEN04c%qHgnK*$zrC4!&b>v%dBl7B! z%xFaBru7WipV1b>l8a1d`YA@xZ@WDkD$|urw^N@&lWSJqTtcyhaZ()&OpXX`(|S~U zMqw-ZRCHuJLyvE7owg0b`c=bF*?#AONR$Pkj9u`bg3U!MLPm^~b+f_tl5x)Nzjy4U zJHxr<&?g%h;JoSjj1*W5+*mP;N7OxsB#YZylR#wesHSS`IM7(riPeU_9O1om2ADz7 zx)_We=+>m>T;31hs<|Fp*Lxb=a)3Xtp491QiQ_A~8LdRwo`%c-6H@+gbG|H))VqvFEmm=ug+!p`mM$qK2u#F*v-AZq%z}(x=NP|@-%e=k=(a^f2bQd>h?e67SVX5 z+hg?s8V+70I9re9TZIxjkP1OwMpW z4#*3H_NSIt=hEdP#&?8xiXBi1-=h835%7y;`y6Wqet>J`n=#(4)cg3ATTysm7Aclv1Q0v?dvB6+j>)^E}bGTOZ(>(}}FrY;A?&qI-XJOdU$ zz0Cytm3aF(Y_}Kox10#O@^P`BE8m}MoA=R2fNHGswxnNJ83=!VwpV13K73F~WG^5o zGHfJ=$l06|=|G8ZfLp6Yn_{w%gZ0n!|`#L3F+m9=F1{< zvT|vCccgUGmkY-8=$92sT<6Pn&h6N6*cR#SnSK)oZ;qX;dqxQzTaN~XcK>A+hT<<)9shXC~oM1W(TWUrt{o7?U8kE0STBv$j%9WB2tB^R9f(#4f- zCOzuymdCA;N>{gB$h`JEQS8o(!4dy3(#xpLPrQ8edXKxMeqJDq<>Uf$7sks07{r&( z>5O^2G>VO8yj7}>k)|>CalFZ^5JgGJjt#y%53D{0ST7FEbi(XXEOEUM*zR|wb~{%M zaeb|vO%<^)#I6520zn{DXX^sz5 z=NJl8u-WUoY?KhN{LpnYD`@AFFS>%72qKSFN2cf>SN z2YeatOvZ2KR21;+d;$4I0f^iJ!_+Ay*74?##2xhx=_GIh22(3h*`?m$+ip0X zI=y#g=6ha{H94CY2YWKsN4`uauRnVkNaKT`KcH;CBC&AQ&pi3(TWWo8rHv;#3|hcz zi0a6VEyw%6$&hk+i)CFNsN|Ou=kp_=CLZ5VeXbYWcQ6GKn!=^6NV&5mn(6+~iG}Qq z2pdw{LrCo=7{`S)P59-9Jw~$4ODZ?&*>9xh+x(=J+}pb=DEHGSpXXHHWx%0#9w=8W zJ>TKt+ZkPAojzydqH-Ms{Ej;Db$-MN>`vdlsEQw7-y$I7&Y#JOvy+W-vOb^DJ`^pw z`!Qbv&H1+2$4Tgfkq}3r@ozI!-d|-pMz_J$KHd%;@(aE++S1Dsp{xEOq%{gzv@|EM z@pdBGTecEr81Bm*07(BjN5f6db3IXje?z#acYl?-F}Rm~G{2!9l-t+!b9$amnCp!( zp|yiRA`D!V-0usH+8ns-v;27S#pCmvlc?POoW()9(u^3@#oIow57%#u>4Tf_19`pQ H>I9&>5Bj5d literal 0 HcmV?d00001 diff --git a/third_party/icu/data/icu_conversion_data.c.gz.af b/third_party/icu/data/icu_conversion_data.c.gz.af new file mode 100644 index 0000000000000000000000000000000000000000..cfbeb165ad3428555276a463a90a1ed2e34740f0 GIT binary patch literal 177696 zcmZs?2UJsC&@QZqg)T*kG%3;%1nDId5fA|ZX@Vfqr1uVqbP=WZ7Ab;A3DSE6ln$W> zr1wsM&_ddc`hMSCcm4l*&f4ef*?T5uGJDT6=VS%&bGrRo9J~7E5%CW;%c_};OC1gk zEGD29A-Jd;(S&}xj+Vq|=|;J-O;jn0X9B>F_~S$jMayuwHtcAH`#XDkFLE=kRBaWJ z2%ltX|9U@f~Z5C zzt!a0-h+Ya#erVMsd5ZB`tQ2*xi#Dx9ouKesk6szjLNe|*3XuAXSrB^W+CC!o8CD! zbCL|Mc$W9Ke(sxo@~K3&LLF7AnMxQKhGt3DCS;VShGS=A-=rKpQR)P??tN?tRD0-g?OEX(>xyOW{Rx?d&6drvVd@ z(`(Q-4uHew`c#&Q7A+1NbAfcu_tCIZ(WB>4lG@uLmij(+;Mr(4`yzGp^~3|ZE7E6E zCFk=xnr4scYn40crb1+pzG$?a9}AxKByV_qKC103B=_iJ@EFCzu(#?j5s#w~GIAx9 zXF=u}v*;9!-JlYUF#)%_j&lhf>f=kl01);?kEQYbdK>IBL8;&Dt)^i_1u?$XWWfx< z4;27L9*K$PPi@>o)t$@Sg2rl(j*BETCaA{STXz~HDmERZGmmU7C?rFGxYs_3l$j4F zKkg2AEG#neGDGUXZ;2?72zHRny+Lh9l64Ij%71eeztt#tgQ+%?olq_dTO`6hh+N z4CZm@Yi-8I;yq0vm{9XbD&0?#hcB%$%U12}td;)opbH$C&U;Do%X|&`8T3shmaF)F z!JGv+3h6R>r@bv^8(yc_C5=OQzO7Z${7L2H3%Vbg6(pC~Z3t-+)wyos2Ohg=9DLD< z2oIM2zBrco)yFcwqoIMnRe@ZSH;RGcR=kZ`+Y`k-+scV_CQZq)tn{OPBfznLLfOqN zf4OntDTaj76kYuOE`FqqsIO0pF_$bsV#aYn=Dtv9P-KLGcO~lwodRW9kfNNT#-IrGsX06oN7Q2$<5Kn0v#1UY@b7- z1OgMCzxsehyA-)$;U~Z|bMDUp@WX8)!#-c^MqAkXN(WoBcjnEwdsMuZgR>{MxUdOl z2gy6)qJzWatEV4}Abuqdl61CmR!R9^wWyy6up4pD=Z7fLKD#83@EtszkSn-n72LXX zVlJ08=z;V03_>T~I0A1pt&97gIXs@@gz*1LpHD!dX+lD@%sqH*cAnr3gQo$CSKA}X zgTr4n0&KCdx@SGSkApU6{U)Q{^T^b5IgnQ%-n@wmaF53?$zoS^lvKf0m>&`%MwUsx z%sMN%TUwrkHoIin)@Yi}A6W*mh4g5h`-R79PE_iEiwF0G+uHS}1A{1Qw{8Vb3~XY6 z)xchRTcS&065ocbunYFHEk&64@Lw6eXK3)I7Ar@kfYumgG-Z<-DbAv!>@6Ay!|*!;<8)sBBUt1WEL3yq|a=BWPGGM?lM7y|-JPD9N;Gk@`^sj-6J zFq28PDxO@@&jC<~gYH2&t$D5}&zqNrgWqk0x=;ye9Osg0uzI)k*0(-DEy62V?w4zY zSBwPol1muQf*!ekN~6506KXHdZvG>0=8byr5(AyH;FnFh%~orM0v+8s1&1iWWc`3U zZOp@vN3M_id#(h7IqzlXB!SyRmE6V27)*+`HGcSqk!}_2V-L1?@A1CQmV&Y|xasQp zHTjoiP?A=Z{BXNfyYKC=c-Ox&Td;NP`i(-izAYfQuhA#dm!>F6msaau-BMxDN9pij-x$co2OS6CH!(u*lsmY@`3we5n#EYul(lM?pi>eZ>8Tm4LGxU_ojqxv~;!sjk0!yW3F{hs*58T9Ji>O&FUBgb}z zx4;l)ym8JudBzRq$8+(&Ux>wIC4)ZsB=6n!&@_1D{o~H#LHapTBlh4y8>C(e}o~)-NbAJ0>=CUL#@GaXRbmg-5In~nRF$gPG>#EMW z#odYY?NO8ot=+__>)s_k!O(Eu zpzU+z6hp0{(P49R-{uqLLnqA*b<*F?uy{r|I|+*$qE0Smes@(XUE>kaN;=p*aFE^7 zeD9R?%hu+t!r6{CX@MPezeD!^l)$M}BI((3laF0ryv&u9_@CeYy^h=tQ68=4F|d5h z9GK+5H779}aD>IWb(yNbDX&O}xi6`gf)*C!VRJ6i_azVp9QWSj)CStIK1Xb+In4HT z4f8+Lzfv)uCN8=d_L?X)Om1F{axx842oL!7`m~nw7Sp2!vT`HmS2VHRt+}woeq`ZA zd&QtRf>#+DwIC;Z|D3r))tNm^@9_R^w9 z9(zd8LZ$cE;*+0vOfyZBJXbEhm~V|?@)k7Mi}d|NS<0~Hp6*^(^KOZ(2ky7W!r^V{ zo4klmK7eG~fRfhPn4%o)mFq|QH^WlkQT4k(L3dsA9;2}o_L*K8Taq*tp%=GRXpSarustG3UAbI^c50tR`!~5Reeh=) z01$&4l39JO99JWQ!H-fEnPKso@~xUd4Ft9!(c<3bhVhy0l! z1W$NrZ>s5frAWuAerx*2xhAitBKOvELOp>p9M1GHjs+!Qx*QhGt6pPIIMW&Nq_oto~MB|D^!eG)C-ph1NGBK<0yMDQpO? z@78H4G1nU17Q_-UL%cfA8_meW9ef2R9gm8T#PD44XUjG^3khG`l!Pt{K$DNAF3k2x z5yuepPGj|Eyr3R9+w08hJIRRbv&hWwi#v}RAd$yt{dS!vpFwK{9|I6K`V|g@0)vN} zJ$t@OWH@M`WL<#yrHv088HO8Rkq0ubRq>nK`+c@J%BG^1E&OX!FgaoU+s?=XVIoJ! zEaFZ$Gk+k$uDQ>5lfep&ogjjQ19V8qn4~P$sx$2TvEpoN=R?=p&b}5a7{*NAgb()1 zJNlIU3PeeKh6@;h+G%}{GIp_UHyU<9rnSm<-(2YY%y8VAjBBv&d4eeHq8xHKQV=PQ zuRNp|>J|~>F6C=B#E6fCGhIqai2|Fa(h);5aNLpJKwVh1Li;sFf8$m`waI%;isE95 z2AwfJw2~kdz8P^40(S2btzHVwjKL53_)yxvo;I{1MzebqPV0l2#V^s5Ng&cCw|Vt< z+f(k|x_P^9xrdzGrx5e4n0N=I+s&w@HwI?7vH_c0LhWX5Be$urn@#{WsVir079T&) zA=oa^_I|1l7u=rS>przBCDh?-i#<2@2hJ9{;m}R=BKrXrmGMzq7{huqh4(5l-ay(} zBYhWFyJVxz%iK)0YjM)PQYXjQ7CZ>O86FrS&Z(=`RBT~9?D+|?n289QLbvKV0Tv%% zbDp^9dR6X??4-{xCwFgQSE!0vh+UX*89C2QmTDKx0l5aC&EfP%@f7dJzaol~JK0U0 zqcmZLZBbaJ3~J8 zmY{$$ys9JbZ`>l&0T`-C`@==r2F1H{?1#6uZHcR7yN4UM?=m?P5eS(+Ulg$3L(#AF ztQBLkTAFEV$bLUIS}hzuS#~IRj($r8O5aPbnipYaz{KnlfUviJ}UI(`jO{rx1dCE7;85> zvmOy^vj@>D>kUAoJhR)5(?_%mE!IWw=wS(~7+y4dsSY8tBO!zRq>6Fl4D)i{w8dv? z($rX-_inIH`+vPaH<2u0<>hQqWvNFB)V~!N(Ahpm3r{aZY;hv`7gsFVvrYL0;*6gR zIvLeQ4Cj-7XTH4;hF&=bgb*7Qn*Jvqc zAO8#0KOPTFjmH;k-g9Uo4Um{Lf3I?d!KEk>dLO( zF%FyU+n)1AvNy6Yz;^Jh_+&Y6U99tuyxs5a72n~3O|Aa%WZMhI3d`gL;cq9&5P9CM zU60(&iZB-|+4-OlY{Md8gRvpVju+{>*wj}y8L#vx$p1~o{qI)D7d8O`)!ya(=MS@0 zS`^}}GLvt+NS<1oM>WE!JWg-%;p5I7tVo~3@7uf`V2>#CH(oF*IFKpIi#GKz(P853 zHdkp3347Cj`ay1tcRnvD#&m)A7WdRgt-Yw0Z9~~?;%3BIi`C_EOvR-NPxz0!-KD*f zDQd|2bV?)|2)JsSs4NJOays%ED(F-=IgCjYUQGV}Ht$kMu2|>1@a$wovzmQyTFoXg z_thfrD>#$nLJm`EP;9#Vz$;= zBJ9T(Qn&f?foqAvk?VfchW>K14O~k6-2{@D`@q8VD_NXt-wWT1yj|%KsiH7Bs$&wzGy%s6qd2e`rA-0hD85@a?FbJEq zh(KLl1S~T8Ak_QA8)Nhn$3M4$cYf8bJ7XEvon8 z?fi+I@6$LZ8CIlxWFwWrR{!Scp*&O(jNA+{Ggv0EK#un=3zhN9Z@r}%dR*tg*j@iLr(9g~I+-bGb1Qa=dGGl`g`ID;rP6V8xA-l^ zO+NUOr?v0ZN6{80%0OnWkW*Ivek|ZPHxlh3GY7qa4@SMZ31b} z-)DdOmb#d?%@&DdJPP0zDFk|Ko!;C$+w{r)D9NB6VA<)WDR(Ti?B=!j#7&>*AcLR$?7o&X+H2dA;#LcbU?ouF)Eu!C@e}R&4S*}{v^QMk-1>VV3k3OB% zUK}7J!znm6H9Vnm3r8!G&S4wWxwZPB8isf?$dU+Jggk6e3b|{4*V@@YoLzOzbUshYx`=+DBBJvvO6Lj4i$(^o z<-GKE^&q_6^6_GgpDhdbgHaqxVRjJ+KKAh6b0s^)qFnfnSa7TUm#%8K3Wk+ifxCw% zqviM&twZ!z40EXjV+AO!^#Uc9IS7DFZ+=Rjb$h@Pj?MI1j{!qtl`jg%F#8`<@8G~| zkOKWy!9>Fg#V36TytH`>;0y!_!huWQ(Y9W$A=&bR41>|j0%fi;e17PPE_^~0EU>S7 z^Jp2I9){7~i^H#wW?Yyjr#~oJBae@NxZO<#b(3k!uI~iNp0$l<6j7PLMWm)+6c4m+ zY~Oi)(n{iDvmY)0{m!JP$Wr>d-E0x$s?+;{Pzz}(@am7Fb01YxHyc-6<9WJVhw+;^ z&q9y>x@mI;w&eOalq>f4)28b|C^?>Y3|0;c;iH{($enM~B&M8hQ;L}YeE2Ue_I4mf z=>^C1oV5yb)`&T;Jtlb5DZr|Haw;Qr76vAH)p+l#fHhvAr-WHz)d#0|`wZH$c+2+k z<;O;+WvK9XQnLYkwe23UNeuztXKt8$jXm^5>R8H>~Y*^rOd5 zlKl)bt-@5M?#xw$ro)WL00;Rp$4^YY-TdVSe;1siH67u&_a3%h6Z4MIuQ&Fr0{&+0 z;jM+Tu;*C~2pPrY0%xD7mq>*Xe#;6gjqxM&T{rT*S$NCkZ~LgT%xq(Xg`E6gW?AVP z%mo9bqjEnjqiMK-0n^vwN$!kg<#i|RDnENHTHpL>;Ne4&(+YuVF?ZRhA_*#@$`DGI z*$kWV?p_(IpF-*LKE1D{MV?*e=(WXX)=EqS)%u6vD?0!{R|ShmB{!m>oeiaT z%UO#Qm*69%nuXQyAa|cnZHufuT-7U^np;5Q*Ze|v-?95p3Ez`ad)fTPcyXNBD)T~? z*66(G^loofw5g(UP4e_9Cidgih^-NEP{YV=^4>b>($BOsBJcK(^~U@#k)iy`^Z7yM zhp$Lty8ypo&g?D_sAqBK&BK`?y!KtQ7ipkYua<}KyE=Fvl^6O#1g5dKU1tk{fX)}s z#qr)7%RE4SJn$F3&@{&izu|&!&P9I`?Y9KSGERU-*DPfwp{o-(+MGNN~!i`Cw`zhzT3qb)UlKtZ2Tw#bU-yLCbD0L8D`9#S^V>eJtS8Hh{RJ3o30U`#D%$0Z6FV)uHICKNLh;$k@%+Aw4qsPz5aR6x6Jh+Fb8tOT8cgUwd!KSOeY$U*{~SBc^>oI@Hk7@&+MRnZdH8g z_QkDNW;^aHl;Sqz3@M`r)hFnuom0Neb-Av#r}|kNdRm>^L{b9?@?v3auD4_*7!yCJ zuptsMRf-?2*>4DSSfd9W(Vv**r40$PgPkO*nsgFR0#%AMkMA--T6!gfECv?pEL#e{ z=J@#^7*^a??7-xJQhRDBkPxA~4>m5^lgFNT{UMosTlI+gQy(co*&##AyqsqUjM)$t zC#F%?^t%=HaskQ)fFW#uUxH0d+YdiJT|pw+ib`HY^wR!Ts#v0lMo@ z@wgt#JBsao!$h%`o56J$;SP9dNpgS!D5&#@ic$>FPu_T`dm>Z+6+h#B)A%{CjQAOd z)o)6dR*2iR4WBp^9u9@-<;d*dRcjn>2pNk#3@>-P?t=}E-uvEy5(m2CJZcSJvF-$` zB3b~mUeU)|BeRIE-b-YYDyFa)iU>djZME%=K}$n6froK5nT(z-ey6SYy*~Pyi-XkK zEbgKw`@55dVT-4hJo5omw z4f)MlfzzEEG2Ox!+RNKO*Zk24vA1}14lY}8ZWq=Uj+&JzSz#f@SLf{Pj3@#}8_8?4 z{eB90@w=UFTI`){j$)uPfHA+*V$2A032wWuvx3IX5RH&BDam~E6AqfJHuW8_k`eY; z7;>(^pf(2)G3orpFz!tA>5c8ogmf5iE>zD_;rI6B*kitmwrfvxQEyIur{}c@ow6Ui z9BjU+{c7UC@J3t6!!5BbF;QjHv|nS~thsCaV(yV_y>#msz7N*^(+cu^Vvs$?;#2mQ z(Tg%kZbC9G0{zFx_BT=kq~Y>7m`L+_Rf<)wXk}R~d6{m)z|89|Zvz^={we~V!_Ibt zu+2{&_5_@_e$=M?TLA~=QHmXRevF5JDr{Te&{%IQF?>8-@$?L z#{xpN-pQ)yart;5SS0I9=nq(xH~1u@<1&QX#CC)`Sp9Z9&P3Ske7}azTyVaMc+Z9&5H6F66Ifx;_2VyPRG+x0?Xqet8PZr~Co{ z@aph-i%7IzCP00&uqCr7#VXkOlwB3jRRkS`(Xf}Mauc>Q4uJ`l6_n@L#f(N?K+m{ZOV~W+a4C=rlQR}tlY1zkEFu8s2>wXW( zfoy-{D1*O{J%b)Bi;v31^L}&H9*Xi6cYogRUNN8)H>*A;M!@L+2_2cldO}Cq>KXO< zl&)6Smw*}1vu&tP4GH(PIKAm+Jsqu@W`LLwoJK%(oTl|K)6UPgCgxQ#{}=K_hZS2! zU(Wt+B(iD2EQ+Hmfe;%T#$B-l>r?77@-lL_yc&V7-%!f2I*p^qcoW-co9Xrt^0`-p zY})tZ<4_Kx1sUZ_p)ceu1fchSKIl1+jz6yZ$zTiRLzl;=To5gBD#lhLUH}JzP?x#{ zhidu={>6cv(uz`+Q}Q+GZ(|F#*l%dX4XB%o{SW5i$p;(4_Wl~NAQO$PhEVkheE=Rs zI5=MG8zX)YU&*0c=bSb_rt|pB?R^4CRq@T=27ndQl#rDgzWv8GgwIx+k+N+rwW%g& ziX!hKOpC3yObLmfv)4O=M8k%Dy`aOs8J@y)s+WuXL9kq6Z&fAXI~`JY77Iqhd#m~g znAZt56M=}A{dMlIgQ-9BGukI-9UV?A``eq3Ul`{-=tb`Izrm<4tnLDshM9Hvx@2rb z*T3R#SF`uG$IKNgl8HxPpPzJLyu?$m&(C}?;QT%2law#ye!V}cq-_9ygsL-Cwt$Eb z>lb!Qi7-N4latGl(u@1#Nn(X>zQ-G$&8fNSm1sr4GTMTYiC&JHFn`fptK^Sws)U$T z(I$q>KPx_ZG|oCmZo62xBV=nr_!ca0wh%7Lo{QHWtVdG=q-?*HWJIpFeT z`hzpU-<&ma%?#CUOULE?K*O`LY6y*NSrw=*>*O}n1hkIB@pke5Mbm?8G7t8hwW745 z5SWZVF;o2)J}Hg!y9YLn5bFYRzvREGv4PyjqeTze+pheoM&NRiqNlvS{itDFbihGu z)WK-jG!px5%lJ*;g$kgPv+F;oB1A?Nz6hWVXCVLX!!j~Np)5(c_hInQM!oNU!$k<0 zFK0Cfd$p|Et+r~pE=TxZxSTzm72hN;%hfKi>HLu_>g;*$DZWX!f)-Rr9@oF&`@(;@ z{|$Zcq(rWM`@pu@-rwnXhurPGoAJDxCBDzS|34^*nRvE#T>f$KubZ(UQ?~bd_PJb{ zT$ut3qW74SUN-u3k*R^6W{eL0|b;T2}szmfKn*VcYpc8qQ0J;aJ}s zBrZ%C21?pONzmG(g!_bANl1M#NuHkImXwIm)_!A*5RA;JS;(7@)POEaHQ0B&GLocM<#ukB-1WV^& z=FX@+KU=)rl6ZLU-XD{Mi9SEiFRQYN#RmQ)ls&?h#JAFFM*m%|1WDEJnBMkKCey!Z z|4-^4VPX4*&f~J@@8bCjKhn0@8128?aZl_&G|p*W#W|4%B-r-Vxj3wz2Q{z;twi~Yws z@6&mI@`N~a&p*n&lSN|r=LoR)t&LgvWdgst&1R#&fVY{pop$5(CGg63%TZtU+-hiQ z2<+9tk!DM#2V@4F^$xOpdN=iAl|EF{=vaRzC?HOsun zmH7Y5KW})8_x!NmU89i050%pX@!{Uj6l`U@-X`O_zhSjXwW^~vI9oiA-g+@EbK`Rg zL6BQy?EY ze{WZFMs<{G63h0lOR&=qyoJu+irxVdC^QAz;zjo7bo=c8GYc~(t@jx4;ymb&nZJ5` zXpX(;0z{oj@G_SX(^T1~{~`T9EV3f!trVplW!x5^iv3&Fa1j`iEw&9?{V2VCzI2)m zsu(8=&$Pt`&+#239xGV3*i*m{UK*Y+${l~(Gq!~apiehh&6;c)Kai)fr2pTgY13Fw zUeD4H-)OrfY|;CNXsaymV81oHP4o8AzS2Ob@x-%YUJTed5O&!ArNC6Iz@D)h^xFZn z;?7?wwn@sB3vDz1k5zeBj-%I3!0>9vcq z^0pQeruNBfvQy(dH{LVBS+?uY5NZWZ#w@Mq`oadcyErpQym<11$8LTfg6e}|GY6Sn zbx>cXXEHhpoyFvBT2z17jP~p=B`yLFu%72Cls11E(5(|_a58In{+HaJ^)*G z{?Q+BK~pKuuJqOZ!{Y7C^w1~KHwml$e~Gc7Az#PmpP~QgG6Gw?g&$HV$EQJrS7zx3 z77sK>{WkW!1hb}A-0sdsSN8b?7i~BH2w{O$(hmD*G)C8)7wx><7aaS^U@;?UD{cE< zium?FVNvw^q1R8y{0vO~wlq2)*uKVHdeP!ninaoF4&{CP^JE3aBB1qh@#+$L-)Y%P zHphht?;~2tAB`(!m46_Q{U79JM4etu((gaCz|o2QD>{E#HSAQG`Ak>H?isQAmQl7C zM$ZefpZ@Lb!Zv}uIC{pXOoXNPkLiL-MiQ-(m)k_oW&0Wksl%)x{-Qo~4!dL+2jS-Y zEpyXW7LcqJ#GACOu@mIBbKdFh$QFkNAgBc{mUiB>@#)Ejuz+pPNNxTHfj9Q=f42xEIZ zaFWAZuotSrKzI4shBsv|QbmgHGJk|Rk(Ju}@!!)7wX%9dvA8T}HCI^NEn%{me`Ae^ z0xnKD&aL-v^L}va`4Z-i&X+5v{6A{Me|k3Rgj1fEo%-WHla~2!v+9K!$QF!IOUM=2 zslWJhl^XYT=1=2IYzg0c|MZ`bF#LC9#PAOu zw=uOBG1i+CUxC1FgU$3I5wn7EDS$9Mtk52NS=yvi#Elzk8WxGPB$f4$>+cg0-b=6c z^!{G6^V)M@FcdmbjZ|H~Il1LaUuG#ZEb`!mw2Q6$iOA4CW(1Y5?~4%_)|HWwO)eCd3Dk*e=Xm4mZPx=?f$MVd_Pq8a=ebFP*Sev>JE;CQw?%rR`4!9vi9F_s66QisPe&QNp7 zy&3s;Md*6}{LFPj5Ra1K?NNGkqu`;~E40Q8V{dn3LqHt+Y6I|&dWZjr=nI99Zkha^ z_Aqy#fgAq*RFv|X-OA>L7~b)6F94=H9qn}!gLJ|N0hMdUV!6vzIOe?k9~Qn#SMI2(~gkh z$Kr|L%;Oq1Jji0Z!R)~A@Vq~!F2f|Y>l;8kd9xs|piaMb%2G-a;{0x5a32rrUobz} zD3BVzcUXg~>0`k)bmEOJ7D@39CxN)gS8Esqc1Vu(C$GUcy6kw^z^e?e5(S`z1KXQo zbS%JvG4V4=FR^0tet7JT+Xnls%XVW7w91Kc2k)Ex!xABmCyO@?=>gz_=0Pc-46{{G z1S7_>^3bo38&u}r1Fl5(<(dR;VAgK%oYUjQE!Octk%2Bcb@-A1j9VngIT9N$rH8vf z_~4JQtA6WnFdfL~(v0#Hk#(Vt$1)iD<5(#{0kI&HWz=dcRK9Bwd#HWF91WWOiCP3a z)j)%p4;BR)zxYGm->>hXhyehk=CFO%@8+I^t@B#x5wS4pUN2Ff^4C3%-ulny)ew8X zkJFNw49Ekqqe-Kx zA?gB~@ArSN9_xvovd(6E%b)IF!wsl?zyRTK<@E`VJnZZnYRi>ZrjSEz9P zkpo74KN;r5y;jj}V@k=1bCl8#xDi%NB1_Z9YAKw1$Z-1f>!WI4{5D?KwO${8iK;c4B3y}#Q z9(i(3%I}ztb;q~?i~2+ z3{G}{=)+$ZgTePe?F--KEw3EbLO>vnAL8vK?Mh3gmUx^a%5M(kc*z{lp1&!zhq?iF zJ3&Q(7Y{ed-t{vOrCvm}BGPe0`Nxqe>ts<+gWFEPKagl}{e{=K8GANXCZZp7ww{zc z>@#7Ozk%+*pBCth`fzi_6p@}mwaa?V1y}QZX^CeT8pl4-e#)ikNK6Do}&2Wdj;FFf_;rb-&}cMS^rx3!cam)VSP4wT?qn2^NF|??YP@Y zYk9wDIYh$9)2oYv7MKH;3)hri;gC6M$3l?260egrG5eZ_b_t;hLf0}bwp zEtl2Tt#Z$)Y3%6K^K>}}=N06Fa9P!pqs@XE%KY`ofmInP;mng)W23(m6@gRnCri5+ zuja2s&pe6@$4Ul3Y-LYZO*guBJQrereJMguW9@|~%leg%Hr!d&x$z;vlUX5jRDwD( zPMUtLaqwikBB^#m{0DpO(l@^9dyOV2M2+zl<;=KAWdHVugZFB!_z-iPY7R& zCsN8ST+2y!{|m|`%Io~Pz^^YC@DEtFZ6~Ae>;(4_8M>FBFRsJADDrG?XiY+JZ}J5$ z4$%mGcKQl=ql^!FAiVgs8^s|fyG=Z>v72QKdJH)6LfT@Op}YWpz^y`-$#Z}>-5XgO zUA8Ej`RF^J{F8h<-qXhK=`u*^NYNGfie6JmQ%k=DCm)UhlcV!H*Skeg=mLn3Lr z1-HvsAIj|dPD3_-%MU{Q=HnR3*p>k}B!qDm36SK)Un1b3lqC=oMkE|>dx$3w59hcB zh&o2u2jen>Hja)oPQX|s?8qwaVZeb9+0pphSviWMl za?FZ@6tALfp^K9LF3Z+0fU6gdvVVGMyXRVPNb#=m_-WbtC3ftzZ2NN38|&Bvgc923 z5Q+Krm#&K<^X)wsW!qgJ%9%^-<+Xx4$H@Hry=z*74hQj9hpz@opP zjmKBZwvQ%{FIN_dfo8Cj}YE~PY`t`%XqPNHrrr@J}00;KPE5*b;?%*qz#r( zfM(@c3d2y^hm$85$H0)f*7ldc3P>^Bczm~P{S-U4Tef|w&;@;Iu_)38&04V3MnKZe z&$SSc;&WpH8LeZnV`%1A!0>c8rJ#1pY%B_0;`Tn^_c=ROs=zHJpxtW2bP*H`1)^>= z9W(c0qdXTy&jl8hjDgph7ZVPccP!SK_xh!#Q{8F>kIdAAuUVjOv>t!*zYpAf?=~X1 z_}RE!5AxHE^@t*Po%!M^mf`?J7&sJMTrzGSg-k5Ut}tI2kh+rQMl7`75O;B-V+?5K z-44#jPEju^nY4#i1#?+BKz)8nF{ZhR3N5}hx%^}6{W|mQ0V&Qj9N`ng-=J>%I{x&* z7DG_^@t+Ecf^Jp}?|l!p{-YuWG9dbMljcHyv=YB#ZhTs2W*(4| zPIL1Xf_7M=x_hdD25Rl#WvTa=*n%{k800AIW_utCmQ*}dn86a=<~e_+ZzT;jEw%d+ zWyKW(yQ|=(mbZoh9oPlhZUjWWLPfG&`8;Ts-4ntl+<~^3Y2yg9I<`Qc3Hz z`%HZM+3ma^2x5QUdvc-hm?kgB?VP1eVj14zArcdvy)@dTY^4RdJswf1tBoYJ+ItDK zkFuAhi93=+-#+|IwIz6R@-|s%R|ykM_e zIuSmc#kBl`VNu4B?xPc)TMnS3pWZ?8^q<0pNt(sCdiZd49%q;{l)PQN!67bFLIht< zmrQg?s^EFYyX#>i4@beBlm3VPOk6p%2|m0)$yJvszR+VJ1wh%^ zufJ34V3q(NH+#WO)fIi;ZHJ5pz6obLViH{HxVKPFy44E-Il&+D0P)UNYcMWT@;ph? zkIpEG>YZmg;v{x7gWMUKqRFq9jarVQQ!V$MU63EB5a>sw@^0YrqkfW(86bUX*f(4K z=aGs6-ue%AQT8RD`7Z?{L%0^?KK(Ksq2^zgW9S!pt*T2edsOaa`lDie-G09xu>3oJ z>$!Kv7}UNgv&dOX<+5=kJi(M9P5pvuL=o< zKRdQcsuX_3Z1l3InyH?DU%({<>T)+|*pqC1xuo&bC{ki10C6W)d0qO0Pn6zDxgIaO+{XR*38{*HLQ+@;i+6umw8~>j zmirvlZ-ms8?MPJ&60*XkQ})ZvXvDAyIoFI#MX9JiE4?GSN=nRk8i5(#CQ~4yyY>8@ zQs`Pl60L@gG=lnbBa&T(`iczc8XwYISbsuTY(ki3BDdfJ%WY0Wz59<7h?GfJ;6xsT z@!Nk*Jsq-1G;~e0eVgcRo9M&#{EC&LaQe_6W874I83j7EO1p_Hf@E3>In8PPJ5 z4|zeus716YQ{JZ=!261uDN!`h<_}C9IAbg&N}ZaN#RK+&6_d+(Otq_58(*r*c_jS*H9( zV^ECwS7IZ|mZz7IIe?_|{-=V1`!?T^@Z68SiUTn|vLNmb-wehQUOvXJ(GLUtk&m^8 zuQ8~A4DnTu|6D-jY1P`P!r7?8L>7HV>x_*Uozq zcQJ_o7_DZV#?0>)p0^mn@spG%S6)iKh0~Mn~s{Ib=N|;pNnx;*l9{*(t0v6y+6MA zQa@YN$bLb8l_NaEOEzP?BPVWd-=;%c>|3dhzlT1Kgu^;ZT>d$?MqFcWOvZJZ9NN6c z?_=fP$DTyTmPeZ^bu8Tok4;hv>rw-D8A$u($bWPc%_-%McuGV-d)cXW)~R;fsW#lH*59et-D%s|sn*`9*4(L9->FvJspi_LX4I(` zRlhg+QB8sUG}z|tjy$_up+Stv(9@nezAj^=_%5A_iPTX!O4Hd1ce(Poc!NqJU;@)* z_sDtZsYHy~l#Ee4@}_Q*Aq-u_e2IGID@-CwtUwZ|Xrf!4XQR*;STA0^v&8xYCx5|# z8o~tdbPOUkF~qxUWj!davP$Pb-jY|_U=`&O`S3Wg+Vgy>{fBabpKon#L*|NJ#NDUA zdTFYp3tz`T0vr>r8@^Q105iQb->S{?fURcdOSt00o0{Fz48nC1R$a^M1r&eXx~OC|v3Y-N|5>0?3I^uhhPhJN++ z$-6q$xi-vUIkzZS>+%aFH&m@nU4+U$nU0zmW?wp4`?!>se9{~<5o3rtqQM-B=szq(?@-y33^_KCy5AGC%$u4|Iu)dxouhc=ayrXOGGh(Ab1C-R-fE z$hd*JN{-1)1)2|PlWK;64I^J77Y0*fi{kj1?$6!|yw7k?OwT5Cnor7G=N_P(>yuB> zch3YE7un;gjwp-1JB1#bvBp{cE#Wim+V3(qpYW1wpuTRdPS0ObAL5dG1lBo%1pp*p z`UCIpwvnd#iuFBBOyv|*Gy1URYf)bF{$x(`%=~HKgZm1p6L~d-B^JNEsJDeU6&Mop z2GpZyb79UUo9DUF#gTdPfyY}j!y5Elf}hGC9iuZBp?02GeL)*AuLasKwYt@MdfNMY z*K4w2_C~KveQXU``2A>{FTL$2bA|Vut#~^YygdrH=#um5VEf)dxvsiCb>eNZ<7MJ) zlpNSxWSMxr>`VKegUL*l-1VqIXU*x?@O#ekEN!{tWreD~Q|ZGeVzj+03t*uo*|!<4 zFw?2;4nC5xni|Zsf!&3oY)u#58@BcooFV2kVvxPHd~&;5#=CoO zb8}~Yo4%GMd#1j-H#k2$w_honYgkgV<2{ub7dDd#&&{>5Kk6$hS=woJe=5pMG(*+R zmw@|9BWBcLay^{xqn^&zaKwR-ac;o82f~77sWRs`Z|-9MUI?#-Yp#v&ru0g1>(k#K zv@T21%Z-lVNr3c^e*6JCbf!q+LddhRIx!AoyWt@icP#gN?yshshBf*_Q%;^0rlt3x zX?rh?TuhOZNd|;Muk5UR0=b!2aaul(gwyg@i5ZXdw<-y;LX9_0Of#YOw%_uQ{8w*U zYDy+M^UMs=+tI)dmZv#POeSh}`o9U5RhIw6`%TT5Zq^|;b8~!+e!uANneLNdnCU3? zxt&_`zHyM+EmOXO%#6my)BhMw98AhvFkN zH~1bsa-*nSxO$hOzB63mP|1|>HEmOO((QbH4Q*v-Qcr{1Igh_x)u%|BPB~-?rh2Vm^E%IFsQA83=C=n3 zsX>@GwamB;Pu9>2Qy!kf%(s#T_L(sb;tyRm&ir_a3DVvh{y2@WEMkeWsq&XKxmm=Y z)E!akx;w+nYgB~Nqkby+*XPJYUCHXPUa^&y3A<)5z$t#!fk=b40#vX>Yuc_6kQJiE&VZgir~_?J&`^?Cs~A@^<-MkwIjR1< z+;1`(6KNsjy(;KoTTuS2{53cCAd$)&1JXA0tqtSO0b&cDL$T0wlFDa$bF?OzKl(XX zK2R=>4MhSUoVa&(64Ueeiz+x~-|Mh}ie3#Px&7?j>w?^8hp$(zCKQlhqkyVVj=9^e zk&K-Y@A{Z#AKPc%`$2q_#|tR7$?Fx;%#iu@L2_HKO`5KEYG0a7LD{onocE4yG0x_H zo3*dbVONr~$vlC|y-W_VXsaE*QF7~6fXxwP-=aZepjYnIx$i%D*!CcTeiZ9^u z)mWu@^^-mUb?5N_m*ptv*j3P%7A%VJLYz3!kgJ5k_1Csy!h)+{?y~V^Q^iSu<@SEP zm|*wQo9C({)sCw&$)E2lYb_AUKpO8`fix$<8N=E4m371Nh!tkHHPZy1Hz(85H;h>a zf1#Rx%HEusKO@N6m3(j8E@_B6``?y&R^6ustKbUdx%44K2N+f5sTfd1poHTE&8p08gbK#UPm?hS_l6v9c53_g5;;8D*P{%R*1;baW^j5?DY6D z`u|q+SFRJKoRf-p!XM1h*1ThrA(Z^LKl9OWD(i6beri zPyQ=uUsd!XZ*P-|@+6cf#S#6}xMFl{6c)*ALr-{02ylF7xyH%j#CqOahrG9Zp^m%M zXzA`X4N*Kp@tKy=ecK3~-!5@c+S<SI`G8yaSa6@WC9>P z*EEDCoY45(XMR91b89Z)c%P<|vN?{+4`*xMDD|FDWmm`zYY*Hik6sA;3}tnun*H@f z|3|k)aX2kR(!Z{wtYWNAui3rDT>7P5NBW(>OKA>=AfnJhTYCA2A>fSKs~qzGhpo2& ziz;Z}hXLu1r9m2&?nb(E>5x`H5b0DTms+H|OF}?cLZxHrZb7Ay?vfDkJK+2Ne*f?P zeb>F$?96kXGiUeAoSAv3x6mha!!@Z)MRv&6>*Rsc|Yt!);K)^%8 zT^~Gps!Lf6C8Oacf1HLxFq|R>rf8|7#aFK33y5Z?7RCu04)54N04F(Qo(>~Xozmc1 z^NDq>D4uA{^NIP#TmgsNRrV1F9;-f?B;jP7+yVw8fvtSxz8&@qXs`H!v9SE?S#3Pm znIjZU!R`vSyql0C4E`FvY$)iKBfrTAdli=_bL%C`6{At?leb4lXWiAbsJ2P)z zU-GSx?`wxcO*Qy5w`bYY%`t;n$y`B#u7L_B|9EB!HNGTk<}TGxk7%EVT;&ypP@nv zJJ*CMF|5*>Vd6mJ_`}R;o-2RmYsfUUTQ6)nnHWJ)!~AX?jKdV5si|h-7)Hd4dp((7 z7uO<2!aDv(g-QKyJhRp@C$yigHDs!H^vlgx@-C^?w#nG)h; z#ZXo;p-hBvJ>9X9n|%tNbOfWG;d9rjG9_ND15olBDJ1om$C_;noO)08(wV_(?)!$4 zxx#1i@LSCa6jP_ii9LxuHL9TxKw&gv!g3-uz~mTDADv~cRx?j-WOdhC%j!{C|Grbz zG;QixR==t3Fq%7-1!OY51*QM+RPT~`Pi-Q(z+pDe@OM#U-R^*oGVwcS2j*`DkCL4y z5US0I&jknnHJ{4OiJ3hrWs*V67n*BLHIk;2Mjgr4TOC%}mO&h7#iv&r_Qx~|=!qf2 zQ0;7}qA4Vmh;!AiJ0+15*X)o<)YML-I)|EkOn>4!e2Ld9pm6J~oagU*r^q}W?sMb9 z4s2SZ?God{d~907?GN&Fq!J%iK>FJSilihzoIQkJ-3p4vWl~3uaAjcYKCOWt&gA+r^yn6S_e-0^Gph!+-X1Q9(Fuy;Zmkb!{&b0EHs7k>*c#B`*g$7G zol`}3_|nj%5+V7l@o_|@b>k0~C*r=P>Q7L)IX_j+)aebYbN_H0tY(pmEMSw{Dm+Vk zP5iD(qHcGjBH1@%(DBse+nD-VY7b|S#hKY###eHfV{yO#GS7d*#|qJN$11ggn-0lH zqUD}zVzZ0TYNStPXR5ob4>z4`EL?fmBP&gS{^TSJh>0zWxx3_uc@|)YccUw9eA~V? z=3sviiCSh;*Nn!t*ptmTQzL-K0fE1|vWkvJU~3~p^jo2NFh~yBhA+4a3H=cT&~gQL z&0zNNXHYv9F8R6IBR^TE7$fQ9pT)`%O`h24hNE{=Y8G3Ig}LLR^G(vjA+Az_qIsFr z<0G*S_GEl^%1X%zHhA`ie9c|^>S!n!qoa5gru5<;rX8+iCG1!El02D$R<>@RxEHg! z%ovcKRww3GXG>a$`Zl}d;W#oxRZ$I!CDA7ZpOYmTzBKwcctbODbiN~%uLeo>9qxZ& z`oS(Y&09O8Pg`azH!w%?kJz|_$ARgmGuy$#*)!zVNf$sp7fEvd*LL~|KL(xlv&n_LfzcvB{EZqAJINc3avP&H zx;-8Z4S7xRA+lQi77^){?c9uFp8A9=uEdnD@!iYs{1^d9BE32x>L@Vp&ODGaEa%+xOCY{ZsRa z*(i!gl*Z$kjeIG5M*W1$&=F8i2|cf!su(@M!O5awadfMpJP|M;Eb@{XYU+NYIZv;H z7NRcHd+oh`n*|*(l^eqjHA-j*Ehl~ck=|ZOoDm zgk!QC$d)dP-xn5Ini_pQX^%F^mPT1xvxkS12~B~fk1$DA6@>xoIdeiHA!LdwX{kzZ zYxf#_ZKt-_{u9Jwx_-rxCsU{QlR&uA=+sX+@bn3)1LOPUL+y^mT;Hz;n$u0wk-l4h zJSC5^BvE74ziE$GJOPZS@AXbVhQIrcL{iYmm5)Xid% zT|QkC-8FUV zxAe{Mz-zYgTAQ&95IW0jEjo?RO6Tx~s7rr#cxs%+!+PvSC!EU;mXgqpEhb@$*5$Q;{lfN9mBzWd6-vDBb^gqEZ^LTVWpZnpWt?m1jWg%{fEzKPH}_jg zek)0AUj9P^@A{iOS?ZZDue&o}cH;0k$#0^za%H2{-E|DXfg>ML_&^fAQ{<4OHR#;( zU*FFG?9$CE`T2r^zc`N|+efyMY!WL6IcPP4s)T$lE9Soajv|Q(cL1bd!BM{_E#CXaxNOpa>>@6yB{xLo$i&iAhum zcyRhzU!ge8HYK`dBP|I};Har4c7*`j6f~yZlURVs!_2qT+d;y&R8V@HI)97HuRrIj z*b|2kkf(;H<uS^XwcqV)^KP7F#{Od7Ud%RV5=zCj!Z?+o# zs72a)?e~3}=Nh9L>O-&$^;%af;n7~yLV#~-5<<4aguX>mVZ^!w!^^+q(W zO-Bz?AgT!7<~wyh{4Oz6efq^Sk+^14zAqU_UAR%6il*Z#eVMt_ ziR*1|s!d-vI>tO)ds@Kt{7o zl8tCN?X&P-ug-xclO^T@qui-f(9d#uc`OexUba;aOZj*)Qq&LhYQT;s3ElqUSVUzD z|K@_SSMMW-BKrY=wfUJlrmt)D00;ou# zd4LSBLI8X7CuXdD-K$LJr>r-4mgCelTe1IIC!$K(JQROyI9o!6Yz4sSyoa^n@=X4I z{!2=_7T*ZA9$)|d8Z%b5hX1~q(x`7iIx&$iNJzBrdn zqTfCh0SJQOhdyG)KPfTaR1n~1J6i?Ux3md1a@C}LSu2)y3RLdD^2d8B{6I^n{e^Pr`yZ&7-bUB$m2c*zjdk{=(%&DK2Dq2_ADx{ zRDkH?c2j4}omoX}^EaZ^ZPas1u{uY6WxmPyUfCf&>mx1hn~Q!YE(<=Qcg?1da7%|b za?%rAgt9kcWvq8~)GFV&m<I#NKrLiU0vaIw-Cf_I-wIVGzg1+l4Jb`Y6B@q#6`VqOWt>5X3V3s)@(s8RRkeQ(NV0E~Tum zL|xvva|heB@s?ULFnk@aV4|~r4z%VfUjy)#IGV4lKC-dCx9QP zF2-e066rbHU*erCFVz1oD-~ljTA5=qHgb=hA{s&Qvm#bwudhTHi!y2|6?vNyiU9wV zVhZ2x>k`JofWA}Gmeh-~NB#p%2BuO>5x#}ly^lhK;cDTQipCi$3L8OO2~uR zzaG<=v)jF|0)>}==^Vz9AF;d#Gb&NCW2@<^!)%j`H>IyWA*|=u5<-&nLAjo+NgF9$ zDItgy2f6@v&hJG@UK8$Bo(#D`n4bkYKd z+gc*TA8UhUT=7aC1&w)0mBB_Tja&k?xDoE|t{{iC!ITnTPVVds@gLmh4dCapD_DVs zl3;6tMJEqA8)B@a)shP{NU-(5sto&z9l7Pt!;?rEA?Gnsamr)U13EN}mN}%hcm1tm zXP)9E?DSlacI*Z;Gv3~I;};5+kS8cN!FQy5nA?vK!F=?%RQQ08NDX$_Zn1)uV+W!X zQ7Uisr3NDm{cTvg+&sNAL?E{wZmheWG!4IT{Q~EqU=s-W>FUWvsf>}W%Fg2|j zDL?KCp1c!#$*?u_pcaCnO&_tt)!45#7%O-~mogX!mPvPB7-#M)d4JB@>Hn~!coHQH zkGI{nE)elZ`?@D+n6qPPV0Dwx9<2qFtK53K{$R&{?LHGzY)SKLt&tp~UF;`sLUYti zfGz>sM~Hw$cW2E6Wp3&olQ;c8t}l$a?9NFt?&e z4?<)7wXti-oz%$2`aA94l1E4H;)grk@f^Iw0#=X1&qTieS!8SPrqKTGMiJ7*h)ZSz z_lx8@BzhuaMWJOJSZBd8!{PUibj%$40-Lm}9%&rqLa_7xOA05LOSXD|r0O>LD<>$m zVzfWZH>^nR*a>(#4%`kPUX9)rUf++Towh!r8)BFP0EJbAWu4VJ6l6$^LEYrV9G}TI z7Vw3izP$d}qSL+z|8}#wQ?wS67QY)v3PG%9)IF6Tiw`^D(%81#h%$uF;hi89{c`>Yq*N?>yA|0m0|r?o zmtb}luYHXUT$;k&1Muyy zp}g|#_|!jd(M&n9pP^ZE4w%V(&H8nGo7~3Yzid$;p%G3LFT zeCR%PJIM0C2%5M%{$wLjF8<0L;jQO@aFGkDe|`F6^w*>8^xEBQGFL7Tx67_`MZulL zN>wA$X@1!8{A4ezcX=3oehCkf8?Jb5tTb9-W*l_b8`X=9%v?#I890pJSd1)YbU6(; zPKG`srFlt(YO|t>90w$2difuLDy>a=+c|#i#Ep6#CvCUb|qrF5qB8v#*;$W^mi3x`yEP!RSmS5A=doExl!hKG`38-Y#e z%{*JasnU&+6LZ+Om;KHCiw%;{CpkIy-o&9apv1%|>8mhw|@oNxRliNYJxc+&GNs6( zY1>Hnv~9YZtk?^ud4CA*cM1qDxHMPL(FtgA+~s8o^YKxff%IzqisunQ=xFGu77N~= zq={AqY8bVN#;g|g8}UB;IwL8=mYC%~dmT3ZSfois*E;8*d=dP8Psm(?R-5HxUd& zC0tmeR z7@)g&>Ek`X!+hDYd|>Ng{9#|-gFLB4-Oe}v$1H>7&R3uZUS$SRqznVsWN2!MJ$P@i zj$?-3#kyq8^xZtT3A1>im64}VhkgheiYnw=*n~4`@#5>tT;AvJ--joW73nnK_zAp7 z4@n|Zr!rX-7V!D>mB+{T)7R>+-*pC_j8asEW%91Pdzss)-F|VA5+j`Ts?uf zV^D7(XAyvwR{gJJPiTXGBQ0Ru93lO{cp)u&mUxYQ3b1GhY@3T{$69$4mMTIcOipNQ zX%;UQ@D!=d7i>377H$j89*~12K#SEo#8WfDffF;qZu7;2_*fYCy|O+D0&5lOCP7BL zR9RZsLEa%3RFe}DDAy6tj~1pCBg%vV5}n6~;q{i5q~D3&ey?h z&gZSh9NZq7A}!@qHjY~g^#N{c8$ZWLzdICe8j&C!SSbTefCGTac1zE>E1 z`vQ|<&RoL$al~-9n?HrqVfU;}BNeW3*gLTVxLSERY4|%pAc4z`st~?-z7fE{q!edo z+o*{v{A-ni-<+jSI2~VSDH21jd&BkxD=iPXg6EFgr-v*>B8K}Q#dJJ$-5bx!zY(M- zz6(hxCN}+whDn(<1qT;#v2}0soNoeaJvnNWLhMW_c{sc6>>8`yy-ATVxQ^T#hFs89 z3wCpNy5aCGpuB5Aqo|hb_I79_rKpzazHn%~!=1}_pUYpp5nN!xohw|G5HgR9cnUoU z%k)mTOXdye!AT?MCOv`lC=d~ZoRJQ_2JaPCMbXMwSTeC)!O> z6A?ZdncY00Ikq#lGm6NnekZFn3A`&y&~%J7J9*Ga#B`Lk@>@^2;J4VE_s27ixARxE zlgeinj>VITXS&q5S?55lI08o1Ex9ZYgp&TH9(bEvi(BOcp9$ARnR{FGTvVOr2Jg>jW7C2evHa_ce5T)E23I0o zWe_p9>NJ{Dv#b_*`3}$XW@)XtvYngfxY$;zX#bBB$g=uex=GKvx-x<}s(~JFH zm;Az*TIuOK)zAJTWxJP(XY%~d_Zd}i>E58%9KVHj>F}Tzxd3p2FN1h?0E%4+n5&9c zUt`kCKa+=PdgIyB#C(9-p*gwD$F|;>w8c16+|QJOBIt*e1SJ| z(Yu7a!g&>d?2Z%ll**9o`_*eJN}M^g-lB?VVB@gHgW4ZxIP(1krRF0Vc*D%_~whI!)Gkh}qJ_ zYGL*Bl0T-^j4%%jjQ~p4+Rtp648- zEM+_w&Au8%U3Oj*&AyDqZ=jC@ZB#?c@}?KJ5@UsUsPl}o3dTkQKWgqKF`%> zF*~zuQr~BZ{Vng*>((EAKUjWA=CVPvvwnu&e$0;@94ssyMto^mIWS+oqzRDR`H~z61TBjnMe1gi zKYIpRx%M7BU)KJ0z~kF8yf}6Z=yQ;ldZ?^0XBZ>B4c|!?GaETMpjkFHo!9>Lu?}uX z_IU_lWICVs_!WQW3dM7C244yVnQFhoqizqgsZP$JTv}p@s#SPIHzoeF1+|T;Q<{KE zF6Y$nk4bQ}@Q)sFw8(=#o-li6w|Rs(nu&9lZHvthm-){UU92cP9e~gVXKOxeC?`KL zUsz_G-EVL^Gig$grPx7Td7K6XJoY<3G@n=>+eR>4HdDSpa}ac73i&kUQeyv_BTe%p z56zj%n<{>`e9EuLJ|?z;Ryx<=$lyR|!Ld=_v-AAt(twPOG%yqtf`*l%L-ike4tX(# z9763kcnB821ea?pZ{9bnZ~@?~vSvRNx))(9Nrm5k7SLpnFF=#{R%))$wjKMUa4-??Fy{iktZo5}*bwx=z*YMLgfzSYtgYmaa7ni^M|omvalb`*%zs;*bUn)Vu7ErlxLJQ@BhFkHRe#))qMx^0AWMGvV}xX@=cfTlB3NyTor+_XbI0Oy%Xaw=-1cSy zyc>PkS0w0Zq(+0?CVGPJgsu4s=q3oZ*|@ba*`U*8Y=&%xKx>_4M#zIn4jm>V$4yf} z`x9};>J@oh%23~n7k)CFA0;1o>2AGTt?_5elP_tKcaE{dul27!U-fekRt*G9$_x2a zz-G@LlB5fYEpi5h7lUbQfFju_2Gc8HvBOk##b8s3yEXz4X{2e!46?2U%u*#j?odY} zO~IR9UqcwuEMqyD+$YQmw3kNO$&%?2xGALqhf+x)rv@qSi@+PJssb79EeOMu>LR}l zR-s`@!M8>JDssArD{C+ki-aCFgi~f9dc{^C zGG&AKQ^C{vO^4y}ynH4Ze%bu)%OoP1rl111UkX;%kQIpa2?rm00{Ql%8@N9Z1WOa} zV(ZG41p`BChAL?>s3AfE zMCgNw0aZXCudX1YpLdT%22R`sj|PvvL^uGj{|9-Q^kU41XO7*8;?WBv4(}!Vg+q_4MMC%)DZvmWT@IL*uCscFVBc$)=FTN)N>69x-$-S9o+!fS~nvR4}J(KlLocf7Xnbk14Fa_yRGK)kx<8(xg0^i zFwLBsNw*DNd{d|2IuzzEKPX#;d=^W>(T7C}q)+5npuY&ehwbMDFm&~%A7dfdAde9? z0DV3na##jUpq2N*{MPpP59-M-kQdwjj(rh^MF(bJCob=``8p{W;x{@gew!ZE|3&_?f!fOV%Q>p-KmhT&; zw!!-DgCWs0jqSvMRQ)+Pu~1RD4Hq;BVN{65Wf4djAY7Cm#AQilI)AWZdBUkMSohg~7+W+6iPYaC*Q8u^UCT=0aY+DE-(W z&&q_S^7w`M(u><-jw?3EUBmC-IX7LBh+lFm$D`qG^LNr?LfN-aSDk`i9i^wWwLtQDfVZ9AW{ ztghnlWww_TFIk&KC)=VA53)M4kCz%IglyVB=!@#MM;~elb%Z!vhoi#KwigwTSYL}y zwtVmuJxlvZF1?#(y?pJ@R<1wYgCFcEsMmj}}o1|SV-@wt#g@xb*PJ^Is| zy=Nw5!*EpJGc15*!yk5R0yZLhUQWb1je4x~Ddm3Sxd(g`z!Z;&r+C=iXv6s$YtI!u9VOeilPv&u{$IHUYiVN9QePR&1F*snAq7wauo!f4 z4&xYo`QBd_LC!t778l}vGJ$xnv|@~U3k;|845xDpr*VC@>qt-W z*3~_v$7_eL!@jCD%UTP?Zd`U*5*;307*nISD;*_~*v(8iyy=Al6l7!@}vLWY_ zl!NJ3@z$K2^YDNS2P!EOb+y|Q(csu|_vxuGo}KQ}^}FJQ(yes|c>SlXI$kEjE*^CT zc=$v)Z#*a{l>|-xY@M_nej9zFpR-|fZU@?a!QS-c@0&F4mfyiXrJYp{TjudEm#(E6 z4b&zq+-s`*o*Cr`>hOzsP-Wf-jPs4>3kL0r!yJZnzeO8}FsAz+P<2=AjkKfs;rB z5cvoFeK&;mmF5-K{Qp4Gq00eF9&TFda`FAuyz{)XPUif?TVVPoy4g_nw|bQ*|MWe;O!OeJn)fKa`I4_)RS6Q|x&}tbq+T$@ia4R(Idnt*8IN zW*P?yjB0zt?z>yG2iPf49AiCZ159YFhx*0O0@XPdyRTILAp9-o35we9!7gq4wd@Ap zU^vt7+tMpaMP>?gvuMvI9>cQkmyf+>ga&69v}Rl9_o`B!oWRRk^mSaz^2W5crkO7H z$h!DVrRm8t-v5EMer3gMbL1d`WoMM%1e<)uC7(K35X_&*ODWm&Z^C))uEfi;qeQ{8 z=0+41qiPOao2o8|=~EKpLAVpe#{`=z{$ylM51f=Y;6Zro7$YYYcA6`0G78;`ym%0v zM0qh+=BV2O)#@zW?vwHy6Po1g~S%jMZ0BfP+QEg1@wq9kQ})+YQ@e<>-3{~g$DK-LqsISV_Uzd zRfdRuPeXM%Dby3Nv3P3Xe}x_LgamR@J;#LI>rO4M-o_EOSDe*DZ@iZ(&ta?iw9&ay z#3}hT70S?V=GR1%&OQ1C*H*sjFAjBU`-$P;H3j3TBUF^wGw|HO$gwk=&0O%`Ay#1YnhRgvP@Z)xZ){$>{MvtP3>5uL>+r21woZEL|Fw zeFcBMlH1rE68^KGh4`%ZN8rVc@-Jd?)YbO9{TpTrE6O-gp;de2?3XnY*1BY9&hY4z zi5q0<0~$G3no;q;Ki%(p@nAfJ7if!XJGEGRD_=_cU!mWJ^`1bX$4?9Ocb0Ym82#P1 zZwkxKT(d8w&@z2j%7G4n?s{w+k}-rqTJXfQq5g~`BsO`NXNnnR2q`7ne`!m&={?5I z(eeRgv%K!I@3QZM zCDmWyl#`p-`4wKgJ!eyxyfEZx$A)j9RAD}a&J&YY*nx-IVPa2FzcWsdC4R>Iq*TQn z`ikh4pmH#8fFkmjs79p$Zd(^3FO9Zb@F7{q7tGd@#S6sXW~UM= zkGvaEm_`TZ%z>hNBW8~hJrAQh5iG{U9Ccm5g$>qkckbKU;3jYq@k^WaajjOF%ha*26 z_)VIX-SwapSENQPBI|?njZ*lln2sY&nUJT_2f)a~Gdc+cc7+;l3daEK7l? zA*LK?u>@-7V-}RQdCoZA2!L8+wt*I_j>jizMG2Ev-jAp|hTc-~gz;UvcdY_5ue?9e zP>hKs>KS7e`D*Ka4B*-I-k8adg>+Iuk$~ul;<;b7CCB^7gvX4pqqcRc<2V)Wg{{09 zybBId+YCjKQICxs+SNVHH~#F78oRe=c`n4p;MN@8O5fF!D5w%JJlQquu>;@h-RD60 zxQAT2=wE!Y+KIQ@M^g7*DaYg?t9zA6V4uB~r@_U<V;eJ9Jvb(2ZL*&#;qN3QL&k&EAH2Yg)_%VQ{&cx*uhS(%&N!# zQVFZL6v4LNp{W@=+5H!|#vchKZ)xTU(%EgisR~F+y*uU1clhM&zVYZ|Kmp)v4Ls3D z%u8ymJJ8%2$w8l_1LhxUX}+DgrqC8cC7caP;Q*24+UvMcU+4B`>$H;6ivFzUdmY*$ zv146FEB#%rVX7e^#=G>UR)?_H&E3zD8HVI_O$Q1-++KvVyr-S&iwA>Pe7)LiWF0HW zl`e|PEkU+q*9h{xb`#&@=J*MJ)WC~|;C!0dmocUA%pY$~;ab^nH0lZ4)fpcNC<(3n zh36b`IAtvBXNYpGiI3b==8y#4X7(+&T*#m^GB+p!D(MaHcx`` z)*O_7ofi+U-<}t~#&}O8g>|&~qA=Y5X-*I=s$a?mG7-Cj~sRvWVt_J=bN95ikV})$uH_ zrz>@FsoU!c>4(dQ$bS&GP0nKxQy7t|)!_Xjl;qjkM`X#DUyW6VGz=>2ey_F68`ZTLP^z#Io? zChy11gCSvFjW**r?J-O?9Z`RY>54_6@5zGl9_pDXefuO4*2`C`JK~=wv70lr$7*0nsFqV?aeT;IlvzkurG(!E%SVvmOJ&RD4DPH%_ z+N}fL2@uzpEn;uW)I{}bYr6^j3#usXh|RJ~gqE+Wx-Wq$T(nKHU7lohtjheAZPss% zRu|6i)I=bzG5$d)Ib20(>&Zg8*oV5RwD!Qfn|Ei4(YQ@mdAL?kXLi*QweAu12*0d*?h+IQmwZx?UiwJF`0%W>Pb&u!isbnib*bm&9|- z((F-B)s;0miql+FxZC2gRC2FB@3PPJGme~sZjcn4B;prKPC-sOg*PDMAZ*fdNEo(Z zd85RPVbPLV3hwgj8N>H8@A;76e;t00ALfsCyb1bzw}1O-7GFM^$|fr|sz_UJBTtR} zY`5lYOZ8xDz`smGf_B3H@7SIXE_NF~ZOs?vjmGhz;PV!Zk&?Ru;;b46nT5Bu{g^#A z>;9e2@0UIYlA5tglopKDB0K$7e3k#=2Z`OAqB0#ipQ1OsptD{nlkz2;8?-;6fpUM% z2wgBmq0|Vi-_*eDQp6oJU^8DT*w-w`#gseE>95Z3Cibr6n`94f3A-czlMdtdyJFNY zIT?utxeANo26+kx;xl;)7veXxs)k=7;rQO$+rAD+-72b5$9a5tm{zls+IRD>r~1Dp zKl#O#Z+mOnYyR1!z{JW#@HE*j$}oz>PA*uVGtyg#(wY6V-rYVgKWsL_FhYqn+7#Us z%}}_)vhey?Z4_nuZ*sTcFxsiomqp<#s6?t){%Av6Sz}FG_r-C8- z*P0KWvXZ=u$y8so0>o5^z6Km;cN2RAqaBC=pKmoO{~fiUtryCH#9A`| z*THrjr{C!%Ehx#peJEdmRnBGqt@gQPrD^_s45Nxxaal+Z)?GF z2>=4t>bvAFD(nk!dX2OtZj$OdsEHop>Z=|p3Sz5*tAYnl06Gi@fz}%uOs{vSz`hWv zCox%^JHX(g#J&L2laPDAxc}5VNw1N)#7%98ou*>z`r6!lA?VP%zPm=``SaD}sv++?Od2l32G;WidIX z9QrR`O|L+OugDR>I`@NUcoPEyd=hMKlbn|Q0!ku;z@l13Z92d&6!zH5MRNwu#3MM4 z!0_vS)Vhkibc{6q(T~h$^oeeW&Of_3QtQi`gB<*P0NME2$PcjD!o9-n$9m{g9svgP z^jPLG062g+QD?w8F}HpNfc5{NdZ80Ap1Xm8Ww$-2_|dlLmPgYIuT@nkMo=|1_8eaD zp?&f;y1cJ;(6xLuLve!cK>3g=A15&LAwW0*3pRoZpyapgurxDxE(UM4QVfv&U};ba ziGwt`l$(nhR8up~xkj0bD&9x%689qdycO1)3*8qxx5luz0M{xI*P0a(5?1tHY4)}< z!dtuc3PFb2$~Z$>TaD?X)WAu;1;bRK_x#@sdc4ABEcq;3o2j2I{;78jr?2RQiAHAagvo>uuaJlCV9WCvwBR;sC?0qDnCC@+fL8#k zlerze2Bl?62OsZP2Tav!t(rmrv>#(_rTCk1f&ApoQ1O=WCpmuy=1s|xuxbP%@PjWU z@pa6ERnHyc6?s=j)Sc25?}1Mdr>r{`xEi@vsh|-PWE`?40J%9(G!9+UgWOnv?Y3

STra>~Zc)qJh%NB7fnoOJ4n(VQC6A>Xxi7uk0u?W{nM4R+%#qU{(}E5#eene?~9 z_PX?nzX=Z%8VDw(QGXNeE5Hf5mf+F$^NNDbMU*0wxY;ivw1m1@vusEX{u>#EN9&tL zxVBkFxQ<#j#8>{X+PHJNJc@FqY;Tj^?go=AY?_%({Y* z(20uo@`?9CSm;X0sO&Lloi-Il1Ijbw>GW~&fKR;%F7Pq@RX8NA$uDHE7-{UWNj|I> z!RXS69XUYF=o-~pL-<~JtCnW0j;LKGC8ub_8!ouej z%>?>bc$!4C`ig&>OxZK^Zc6+iq3od(<@7=v5~Y}o#SKU3iMn#n4ZO67pvq{4x|94$ zgp{O;%4iQ#;@w%ShbHm9S>&0)1kp+Z09dvAk@h1Ig#wCWN?S5#^atEGi9o;buUhs> z&uh+W-m{4S0BidnEO5ieL&yJT@@n+6rwCwye>Mr|G7$=duB%7|?iY;HF4^uBr$~pM zGI7WV1a!C1UZ}kT_gK9PIx?@BA3jF3iv5N87#ecM6f#6aP*B;vh$>`&derz1O8y!X zdS7zOT%Pk;1*jJzG{$Fgf0X5L>A96!Uz$*7{$#ZS_l zQ^$*`%f^xcaBvP}(&7?bP903eU+^<-JgIhDw^*$J<}ZEiDnvauEHc3(B;o!=?4BnU zt9>m9THjvDLu}s#(!)kha~I>rg=)7Zi&c*ngs@MfTepk8_dT9o{A#qgQ-!HZ1xe4@ zj^~eG=m2kP^d5OrMZ=}CcQGjIci?_0LkXf3v+tEtc$RVHc^1y}m{=mO7nwWmg}Y+R zvkydtFBP|`h?i_dYYX0|k9zL#W!PqaSlH=b!CY^EL4O_J0tIW4%6xQb1TL#ggT%L0VQ zFo-d*?x;3b9$jGOD#5$GN0S zTiZwEiPA4iqsv2QIZx8WqW&GZ-Mk#b!An_j+>cL zbcVoiNe&~ zIl#v!fcp;j(S0@=nTgm=q+#{)-3({JVNt*5aTbBheaN0jr8UaE+S+g z5#grL$UrZCAunUT9f|YD?E?jmR++~1j-IZ$v937>OykFWROYA!E#@5zcgXH|%#;K0 zHu+NL_cvMhikZA@f+GEM3D9pFfYKuyG|=3ez&u*&F;P$Hr(g&2rSFskG}VTC-5ur4 zWGun;LbTjkD#N`VK$0g*Lw{;2r6F)cyUkqd{#=qN4{wX=>fu@MrsnFFW}a6X8b78y zK3Qa?q_LKiU^zNUn9X@Ln8&4;7MJY2Xp-Vh-@2CqIop?6R?lfnPfk5gIc=T7Oh9kU z&E8KdT&EOljH?`QeCZjcl&n4z^sdYZ7X^oLt~VmD74KCEiNsrUHdpZTPEnq~Grzxw zBRd8O8zq82lRc8jl{}JCzst0ED&4H?o*tH#&G)CyOs*=ww=ErxVaAfV?p@JjuZdb) z+szU^I~aNQ1uMiN(>aKiEo%*Jyz+%c&XwTfi+Wi%`{mTHXRO||rrk;X|La`t%{nsHCZwvA6k z%AWczjoT$PX$o0#ufM-Dr@X}wlQLgop|_nj|1NNFTW27ZUOM=$?iWTPr((4qr;z|* z6XW5eZ+e$ulVV+CbGn&v)bkA1bnNXZq0Fx7xay{T`SSa<2A|ztE43tjL?_#IU}z}x z=NZ>H+gz^KzOUvZ%3GVut?o0|YF*KH-kr6ur$P1t7^T#+p3nNey(pj-9Q%Kmdgti6 zqV@Z`Nn;xg8rx~CHfZdm(TR=5W@DQtw(S$Mv2ELS@;klve&6wq^^CpO#`$-jwVypd zbMALJ8jdX$=dFC6i4<(vID$7nF1+;5RjTVwP}xbdr00|>Es+mFq#>Ym+R+*e>+}Rv}n1N`5X!;?m6lp^R`S3Y;?72f@1H{;Dg(Q?V=k0*%y2Nlv&^xh@g;mY~ zpaC>1G~4D}k^<1I2S|NO3_4B%q$W=5Jo>xv_cBR<=yhp{2Uze%-MK4FwNYQYcO|@q z6(?wumug&yH%z{%2-e~gIy6GPgakEtR~ETl1vP~dZFEQ29C7D-v@rvULBhK>Y4t3O zT7u6S`-!CKRr*ZsHdQQ1`-p^d6*Y}nbNLO9Omch!=nJ<>1LKebd8$^Yc;*sx+>z-6 zm2;r?=Frhv{PWr?vZV#g1uy*@zk@b^W*!>`hXdLu8S9{(m@GjA&OLF}lU+QCZxeUi zck)X6o4(t6zago8#_Dy6Iw9t|`Am0p8rehhQZBRp99Lm8Wl@^Fda4Qlj@)>YfvQ^T z;nS5cDUzW|beY@QXuQH69|u&oH(zj3G2~GmQ0s|zU>l!5my37ZI_pkIf+f0qtw6_*1b6%pRk7+G-a1v$rm(|%8HchM>k7`7Oml1OnF@mz zRhg!+qhG9R;N!aqSt{I)nD1sH6RXF`EBsGd0uCNVAfM6XPJy!3RIf(&{a@fKK=A1@ z-SaXv>$~9!9gslWznkaWJt#mNm}lTE_yp6=3i;5PeTVjV)AIU!RQ*K-wZ)er2A7(+ zSV(LH6&eLQXj-g%g_!O3QAtOl4QPP_1};z1Z=NEY8c2mfip2i6UqErS{~9 z3vuJS5lm1Uayx}wlm$~c$<>ebZ^0|OZkS6p^U;6+j9TZvF?NP+($11rrYV@00zn=j z59OyXFoR??oYaOo#)2Sh6Ng=ozC#_dd%YWQ;-hI>kw>*SGH3 zrjt6{J}+ipC+FjT$fP&iLe+UoH5MO@&d#_ijnO%pfD5#CVZA5Z&+JF^X+1 zS|%IyT4QxtpsLHc5<_AFPz`0Is(3uBqWXY?<`5N?)b#$GJ z!?~n=|8aLx3O(HE$J-qZk&NJV9|EkF^zo?^6Dp+y;C&4kgTehPZX(&{F+Zp>qA{uT zw+8CrFJEzY#$~thzlvP~FVQBFUUa)GAn6$GFZPh94V>wEE`ld@_3$sWwHtE*|~eULVx|S^_jo z{xQCJ!`{uaWA;plOZ3!${O7J2>#}=>f^$^G&aVW(Zr)> zM#feHQh&a#*vXVb4@_<6yY(+_lRTcT@<2bBDiI0S6Q4?TGwk_AAaD_n!B4rd^SMUUI8lsL*{Zdus!4s4RPGZ*e8=2f1n=n|8S*S!0R@>+=zACYvCC@9(=Y_OjYR{q z)<80xtjbXNU&|=!u#1~Q_?f&nrP!yV0?aDsK%a)gg80@738)Ey53)qGm0Ag(3`2I! zu}Gp7lM4rhBEU_dF#cR?mB1S-dZnHpKu>Yehb@*(XAuCt5zf5ca=2N$6GdG zIrGT9wafHP`1d9QEzt4)Q&!geg@Ny~&dEbl&dOl(O708C zPNNJFGxRHhxR7GpWaL0?V=AJ1;B&9{ zUzseV_cpLt7D^9{)*bp|nVYhtTUw@q+!C@zEt3(7%qtcq?uy*&V^zv82%<~)XNQLR z&e-|p3fKjWEcl+SsCoqr8){m!)HYQrADT4y2UlusI_^6H^8&G3bH8mkm@o6gHK|sg zv$}{B+!eY{T+rijUEm~_6%d##q+Ut0%im5}Uub&%NxkVO94OT>QWzYmN5%Gt=hElj>TDIP7{dykPS4WA2nelcidtVQ)@Z)I7FKsN8#N9_NjA$}c zcPm1gs4g!!J7+LV3%#OK8#zpgL>TBXb(E*x^DO5ss%0Ou)^ZDUcQCg6B$irSrRcon zVg9Sq;LI%jJXBgtkB&?kJvD5xuN4>Rrv`$Jm?s@17M)GaTkf$|b*>35IVqB*g|{7q zZcvW~`nW=9IGvp$Wk)bL`5_{ph3#T`D?Zd&EQS8WFIXaY954$E=e1YlVk+$uQ+>2f z3NvYBx65pOarog52O*yCLocr}itm zGGDtwm4_e=%}CY}#}TetX=#-S7;)A6{9*?OrPu2*fjgsl#d9{c zYr}LD&)$~%4rJ+2iY{(jMvod((K*665}N`^#5x2LCaj^BSea`R4zwy{Mkmdgh)mjg`;=Wb-_1oMxr=kjSMJr6-6}NKHH+M|I zeeFO^>m=B`=G+Y5>q{5NWcdN&6oJ<-@CQWwBrGI27KOh(4`SpQbg9#@<1$1rb}QRl z&SR|Zu`8;`ViJs4A1FJ@JV$U^RTxlZMfzQWe%$QagHG&3@!2?+28#wd2L-M2X)E;t znw^sIhxo%{u@>02ZD@&eh9X1~*~PD5hF7l7tvMj)sio0n?&aVhLx(xqS6uM0OTreX zPt_U)BdP|c(hyko?QXZw`y%pzhwY%AnrnoPX)ck#t1bCVQLsYeINw;^~Y%P&yXnkuBYQ}^R^NKA_7_KEp{k- z zA=CPC>9Kc>UIg2u)@(_JT$osP4NKxy#wgFJI@7{808qiAfUCy@E@(szXd|Sv`ja!& z%e@&8D5P6s->3C_hccvS{EX7C$xlzFj_-0VPaQ_{tFMC0HqlKL8@|7i%%WXEz~BF) zIS<_;vyww&*lfsb4M?4ey;UA!C`Lk0rhz^jrvB?b^ zarq|P*bSQ%e+?5Qu^DY#7;k2j;J7iz`ac^FWJC9HakGA8!?@mCKlD@^ku}cx^&nnr zvvy?J0DT#IZ@Y${_#rM^WcZbFr0@Oxos*0BOtGXWf+4@We|j6N7+ySMTs#&uQ(EIf z)hTe7;LmzOtMQ+;Bkf1^Rq=VqiBH{05PYS$;7xaG*-Li8D(Zf4bm665B^G z;pir{`VaSML%^WRw-%@H2>a>ftEUn|la4|3*09ffW#fhy>fL~AC74%KjOSy?e2CrZ| zy+h`8kSf;?kVfd+Qd*!x_thMd`S3#auL}OI^;>x=t(Y-570*ZFE~pl0i;c#IX z>}+Y`c~LDE6|vnmIt_8r1-SwL4I3ICv3}v;;kGR3MyDs7oel@<_J$ zM|*jyQdv=%n^0sxkKC0vmQ5AW4JcTM)5P&y~` zQ3XqlZ`qiRHGOnoZoKtJP;5Xg$JZIj@|(@L18=I8;_(<=J=y}+6_qQhGfGZxm6a=Q z{!#MP124e^!HW%;EQVYdZ+iAxzVQb*FuW_(S~FSbEK!0I$mXcIN;p3JYU&$naV_3@HVr7ds$6_lJC~6)w&bQ{sSCo{7MKMI zJZPhaYPih|j|LTvdn)*8B{WxqwK0k_ay|yln~G+p+>5@-sqyaQ$ve2c1E*Wmy@)HZ z=Lo7k+n5qFp5_wJA506U081~@&1f^Ed#^Q^n}$06TqognF*LFe5DCa`T-}V*#%mNP z%P;Wh^{+^_oIK32T(QziMu}Yww-r5241xJ7 z3I6q|^%>D#x`f6~iwDk1XU&V^_)>cfRVJb~hB-5+I<(PkKMgn&nnddb;T6}UV_`o{ zIE$S9KDaDqD2{P>>kiQ!mbzQXDqhOsc3~An)l#%sVFkn=jQ{&(eObd&ZZ?jeX7e9? zp*cX}o)_d4BNWHIHl@(gtN$FM5D$i6EcHNFRu%!|5tOq9?(tDxsAR+2VIz@(Yw3O% z8d{L}#AGKV3^;EA%vuTNKf~V69h9@!wl1wIPaE8{|C5bqSt(5#EJ4jVdO80E zbzvqVjL0Qk>%&v|CjYXa(H|g-6Gy#<(FMpd*MIHQgHyKSe87dzpVUHbW&1^_{!f&0#xd#sMHJ-Rdz!n)jKN9lAS`I{@fi|ltqP~Uzn%A zKkLmEgIKICfr%EE5#D>itM8ycSd-Gah`EOK-KrSVqn@e`)+n(x@eiQ9P{Za?112lR z5|UKEQ@#G-YJ>SXmcm#>IRU)m0KocyO?1^6hQvJMo}`9K0SKE zo!PpR%v$9SPj$A=%=#e*>R`>~(*K5}K;L*o5K)o?_UNewKpk@-bNgEZszOQ{f`hXD zyj4npHFtFOgB3m?pP^qV2$z@Hx#Q+~FdF~^c`)#GYuW6#MmVNX# z%`YS#A46BOX3I`|4Es#4Mon_5trxLoTWjuT{YSqSpLEpHBGHnpubip<-&;S7!_Xw=>6k zfoWIk5fI`D`VFQ;4^B=U!p~0A+9&Vu3prgtA6S8qdHM9XK_+ zhPJYeP$l_tPd)XyyO=WB+ zTg0-7pb|E&v+AI@pUTT=CX3Fh`-K9HCACQ{~%g`ZR0khiBgeH} zVpd37e8dr3_?gjaJVdUTD49L8q&BOG^7`*a#VYsyTVc6)Xj!4RoWx*b&*fp$Q5Z}0 zwmzmizve_Q1#xCNrMqnQ`{;t`P6hU|L&xXRyZuko5@XoBAg!zijQO|{A0=wOcDrl` z{j-ysb)R^u*l3_H`>CTvrE6LK7M{)YD0C3W$;3K%%Up{5+wcBPK1u)#tcJ3i`nbyWp-k6Fw78HdoXJ-mfGHN_u`H6dgU8v`S*PC z195uYmbfAEkPyzrIMC_?v=k}A7ca>O7e^svyl%jK!Q=@L+9mIm@G@<24Co|(71{a< z_woG^wnqrs7)iJ}BFz8_uv*G(^OUmLpfkCS0xT}Fon6LewJTc7oLm~tE9ckdT3R(-xJU%1 zRDa^?LNCg@oZ0<(~3Ioo&}gtW#Oxw|~^NhO=mGvt5URCD_xD7}~$v$|4_5 zcvn>iTAj};HP=-~m{r_I6ARX&HP<)z&d6wR74bRR%L-i*Ln~X>ub6kj`4D*I+2Tbl z+!0yi8Zz7Ur+UNFeiN66q%}CBTh@&e<#dYn`FEccm!x;tZkrlpec?UA<*QiwNjh|H+{Q2!`_0wx4$YY<3zo4 zCxhy!GE~{Oh7aaP+)$Z+I!yI=I{h{1U?$-FqLg#*y>56Ut314uB*xe!ow_duhci8C zfHflkI3qrFM0IX?nf_MKgZV$KB%-W%NTQx8w?W#@rFVI) z+L^0-4cQVr)gEf079Hvm$EXR9(e67XGs9JE4OuqU=GRp7xSa|(Cv(Q_ZVg!%yJdRW zj{T$~dPx?L>~%L)7E>A<24+tFi~6H7lZk`a=Nf4sABi2P>MrhdmM%v%=KDap6&f+%cEdoz~6V1OV0k z#a>grQuS-%UbV74eWkD!Y#(&hR2BYK{En}{M$s&k1^uc>H|YZzebf6?U)$T zR9+s-+dNzNA<btiPpUDXL$Ywj~E9qEu0fUTn^TU^tt&IPzSqpHm^7%~T|dDYz`DX0>hjM|3PP-b?$-`%lQ8LZ`uiRVFBD)(4J{ z;PH>>X}8haF(Qm=H^u+_567JxyNijrUmG?FE8A@RSIU!@jz&2Po#w*wXk_p;x8Vpq z<9;KR2DTkt8EUMHhY>R63L&r(igwb-V8G*SQ!q;=d?4B;&%1A~TfT|_twy0vjaVqX zCQ_=3DMfwJx}s1O7%v^!JXpge>aN44KCNabP;9 zJkW1rA(DpGD(co>(Bqdn{~8V9yvMk>P{r|yO)1TfvP38zhMmfL%P4LVioNw5A(PE| z0qxcl(7r@|mPVFWXmwy6FjM55!;L(KP0*XlC6cYl+!2vTbU8dfBhP08rR#2Yn5WDw z;jykJaq*RCqJ}eB~ z-NPNc>LvPyxRHy4_eXqAwN`+3^iU9igIXJdF~xC`yP7C|XzQt-!_eha-#Xc2l5;W^ zK0>G1D~GFnGwWCD(cl%414AV2tem{mpe7>{C|uTNxE`@K20wib7l@sQ>YFjRop$Lh z<8NCSJR0xKdz4hw6K4$F9M?sJuiUmjzF380UQoMEWH7RJznK4`v^Eb7l?)%z^c&HH zv{_f4d~XXlIUu_1o?{$!#ekj>qEMya+l;qwXUk`pC{!rXe74U}TgSuTd@|t@q$kpL zlO5|q)YY@vQ!OJ9x8~_Vg%Fwf#g41BciB=>t119RJPwT5)jo%@IL0O7a9T@Q6t4Cm zZyn%VSYK*LcAQtJx$%0;H37yUH$gCwOYIw!lo*-*Nyh|=Gbu@KaOWm_^I`Bn0o z<$zUXI-59R;p4RXR7Xtk=QF|&xzRh{48ah79%%dD&sfR^?J_w{B?vmL3Vm2rMXmyY zd$bG_med*BHF*ymAsz^=HwW=IYtI)c_c5Di!z{8>JIV}%z} zNXwMmTli(;NeU~B4rD6~PWUlmm@u$U02%#7_Ha}1>n!i?Jd^ZEsDaq4`&J=K!?R3< z;EY~j$&{(ZE%1w1p>6s@!(#}S-AId$21c0qo?j}Y-{QU(BL)q{o{7v#$M9IancsD8 zTfp@-0yGsCmKD3sB?h>Ncl&0k*g1&y1DVH|DO7BQyWyd!u?U@g7ZiBM28fi;mR9J_ z=h*Ry4mAb8;T6?g_Ik4d8ry{@$8K*w*_sP7)z6n&tLJ;j+(l)tcm!nG)`S;V-T zkMNN-BAS?FD}>Xva2vl5Uf;-_dZ6OVbIr+U6f;Yv`qa5x9Qza=%aAv4zTI+gR*}Gu zCqFw;mH14iPOG)*nc6PpH2hR%a3&Z;HjrfrF>!W_t7=L&(N}twX~qOM$KE?y>j05i zM_DizaZ7u+af|%t7wZDR!oAg2q$p|~QNMKY4HT)Vzp9aM6RMneVPi|8HbxhxQj5J1 zTiTZAWIUk(5({QnPk}qXt?hVQ4&L2#_kq?yWE$tAaK(dw`GY&zlT5{d2~Ghln+E=y z!(A--iKm1X0yx{=KO(b;Q%q2;XuHBOh(DO1Kxn%X5bhmltHQR3-HcF8%~#xhfLp#A z=!WKY>Slc>$YeF|mJWm8JrMf(7wUMtI5<&vdGidVu)4qC=@EpD*)?*i6z~pbHrF1< zK7Rq&;F`d0Ya8mDk-UGi9Y(S)=7#QUN1Fat?xY@El*&JL>tybuM5)|SM|thofd*yE zbx4JzC!Y)rRVAw`>p~DVW@z;f4iJvNY&&PYz1dqs)08#@$>=5;(`KbgI;doN(18tQ zKG%Ip;IndB4I?0gy}Sd}?UgFCvC)O?y_Ra69PB788Y`@L#4#zj&y2khKYSfX&Ny$J zkR3?wW`C;3>=kDM(zJ%p@J$9iEx(QuzKW=ZRziv!<%N7L99KT+9p_j9zV2qyfpo6XXuIZcXbx|w1U`ysj;#Y z326B__{q@-FNOX(`{edV?>g&YO$u=`r+0grb(=M}*72gX@~!u}hPLWVxLaksd+kJS z^4FdvgptATLb@jHI|5vW=XdBD_ZHSb4`d^gZc{!xs3UkHq0yD+rK1b_0fY{)6oSV2 zO8U+oWUgzyoUFM&9ZAy}Q>`s*npDC)Ntf(#S#a+Wp#_9) zDYM&r4;_mWw@BCR&yIxHD!#zE`9mlHc1M#WK?uL^7329r!cGlcbeK`Pn8N+lOJ&|h z=i@J1okW*HZAKzk7B!z4|D*o>|*3)7<3>@FxD=MKK@ybA@h;DQ!XCqR({g)}2{rEr=#Ib!)(9gBCPd zn}`xUZ4s}b*$LMO=Q;*1>%LiXeRPLC3ojTS77z@EPzu3k(1h#DrcM@nZ|q}epZu{* zsZ1$@>8jrZ23B!h&c^Ig_0@CoS;UG0SPaqDO_fijvj6ZZ8XaslYaHr}l3dm8^Ubot~xHO0az%jB>Rb9FcvteOPw6 ztckH&(`&TcneS=8b6;vKD7+|!RP!HVbN^9K^{F@xMGnM+DA?F z3XZ++S9)Y0PBm|kWOsZRj64}PZ8bTQ6$~6TNH0#|!d%n(yf(FKp|laK2^@kdbN{_t ze(-i3MZ=Mow3TV<$lokL5Jx=$(VonXEq5CZe5au2*!M8K_7*%}b+hW|<|@RqbN}60 zL27=WVNWjUBZxHMmIN=VU|a z)(hw*>a&>6ryr~Ph7>6D-EwO0mb;eCM=tgvd%znUp_!T0@6*(Iqy$Vn%RhhnO9^d>G{!m-h@_Eszb#Ls-+nCNt_WL&mjwkPoh7o!FwTyAgLEg5w|XmU zM94R-uzP!Lx{t3g0EGrjF!%MKA&M{{Vb}cIML6Tb1(D(e;o%({bEs*YE~YGSYN~-? z2cF?TYdMqWvUAzPN8vKIpQOp}dW;FRnX-%RDuOk-7?ou+gi2;MOEDJRQsol6b^Ymp z83nbkaUd2%IM`8G8R`J8vxNWyr8xzziiq&ia85@t?fP4=msP6fWE!8#)a*auCnY$Q zuLAn!3$itW_E=i~>E8r9@%;C;_zRWmD;t{t&$g5YB|PAm#0y3Dw8jsKLYh55#7$NBqOO#1ddLB-H#QcUq22E!&HXp6TF2Nzcf zYG$Q`ZOUYUp^mO;y|QWk;tYC)s5m)nv)&9QJ~j-GaU*4;&3@}sFm*(C<4w1M(dc@| zv9zX>?MmU0H)_`K-Re3X3C6d*YByFvoR0rfrB(}Vjh0B9 z@Slzw_;g}HZ*&L%d7q^{)a&$T^mYcntf%RipASwPO-{6*AIWf@t4@&qJfceXEgR>g z3KwzuJ4{k-yzcusAcemfNLzVnX+cvw?6lr>D1T3Mp5C934$--_TUY}m9ViU?y7)_Z zZKxStL`BU2M&oFwP-NxAYtSsI*#f8cbn7(|DvY{|8>9SNPm8-C15^o>2dmt(tzl8c zckP~4QK5+hMgq)TUMZz>(m`P^063D(_Gbyz#T%*o-yH@QMDt7ifC*X^9q4|g*H_y> zs|lxm{xhz2kC|1KMrWUd&@t9S^x&jS?-4DkRpZ#&3%kI<&D$9QuKBI0kSDX*QmZ1a zWN>$QlzWXA?ZCGfRn&`QHNG?+0yvvK>v{Bb7ouTnJDe9h=tk&R#tvJKrxn#(!5C^w zLBhL3^k3Q{d)qA%C`1T|)(&r2;(ad}43|h6;$}5@mD2d4T5n}3*F27P_M&>e1r8M) z8?8~&a^jYhe#D6{@R0<1@xRV{lx(?%mHh0Q1_Xir37X1DZ;*M>yhfW6i5`*pBEJaP zadNkNly6%lWbT3)yOZ=*{^Vg5V(V?urf%LKLOkLpC4aBSSrRde4nNy^4BGjEV%esz zlkW1*87z3tGBr#(wdA!fp{5sKxJ|J8GxpJ4XdDim2wKlO7x&om-Oim^OSyS1AcLZc z`Q%3;{BO-qQ6pmR^7xPKz(4pfFM517pl)6@&wOfU-nlLA6mzz&jIo^cwS$d%@!jf) z1{(A+em-JzLD!lC2#pT%=}bwg>iNRmgx{6Xqpu>L?(ap}FJdzv7;YVgeNLfK1;Y!& z$C?~DyhGf{ao*&{b3WF|PCD_uN}bof+aOgY(OEZa>!=-!@w!1I$3Kyt_6O*NdMp=- zug==jJB;E~X-oSHRMH(u403Ob{PA$jR%ZQW{YrYSeG43Iy2iRqS}@4F>frA{vM(`_ zDmyPJqxX=$aT>-v(MmbU>I-AM=_pE{1t7K&@j>OHg=~9$pH7E_lrJv6#8~H1lpOtW zG<&FKjv-F`YZmN%Hk3OeRYsdE)&C~C)zlMB3cE|cX$0f@$8a?guAClg+a@qqIF{Zg zJ14fPGqtKSx6f5emQvT}Us)HkH6F;Ck==Y#xxQ25q)GIAEBezRvPijGz&4l{7g7+{ z|AD1|q)CW|0v8NUNuj>Qe1?Ck_XkTY%4YJ##ho5xaeO`pjalnl4OLLYmrz_)t$XW8 z&dMV#XB=Rwm*r%TuG9R<*jg`kCQouYVjNy?_l=)+ps8wNw$Y$jShneRIS&Cx(+<#> z0#=tn4M{jC8x!N4a{&=Mxmpv-PA2H4(pGxO4}5lJ_;@;wl0d&U2O6xNfWdcDRT~Aq<-zh{A+xBxd_v&t4^iNfxYRTdfo#B5U zo#~i~^r#;t`%3%@s3l9%FN0GZRwzKCq*0;WDNV?c!e0!oO_)lE#Z!X90j9-*eX!r>K<$dkRp9I4VK?{2CmNtQ?d+ z|NrtjVcp-IQ2L3W&`k5QGyGdbbym9+4xGIgf3}?;24k{E+pt)F^npP85i9G)f(-w{ zCg42OFN~RIOCmlI0%}>a9@n<=bKK5pS%>lL#Pr|_YYi?_?E=x`Rc26be@>jiPB-K4 zqLx@GlTQbWRKX=a56IS{E*_?6jF~g7aHg6->buhl%)`DWiZwCng+KRYq!T%X;xv*$ zI&5M7l27?^K(>fVH?Dr-2&16AJ+y;%_aWhay&7cXoki~V^fByvs;!W$rr+Yyl7>}2 zkh$&l(z?;Bgll|9;BJDa8tSu1+fPAeQyPRq6^@W`I^@XyE0qZp6Cz$xIRsa6!fc`+Nyzo!`cP$d8 zNoXvHABD7Kt30tx9$4S<)iNtlQ}ea4b^e!3@Xyd4oXXL-s5ePsg~ka-5I`K`4A&Y! zBrG4qcZS+Q=*jasE$8pzUOgM8B%So3+y%CLUhd}q8}Nzo-plRH&tNKdy*3;N*9zwr zzjU<53Fp6Dewb{pmhi>;qKhNZ{6N9uS8j|_QO;uV7b+vb`{oY%u z%U8GB>-&dqw4#1SLBWOidhxhBUJid67owEvm)e4w81Y+Dt%m?oiO7tjGpOh4CFSm0 z#)UAvr&A&~EM-*E%ysf*BS$!ZtBJ72D~#TlRGwsB|9Un?VR8W9MIE_FUml7ofU(}w zqSZMjc>AZxZdq{ryn`~aNUDU4A(HVrr*{v5~*Gxl3wXT*SiV>4lAyCpeUK5sc zKiB`>sgd65_?#4l-r$M}C^Hvs7hnmaka0uNXfa$yj8>Y)Z#^v6)rZdSFyqK?-i$E9 zr$p>fll6B$+S`xc(O7NT`~1wzl$VJ!G0@A9m`#VEl=H9qMPqTL$=ulYq+w9XFQ^8x zQf$RDRcqYF7k4&{tuV9)XGrwjVKETds5WZKG;_+oFoF@iK|KJBumL*eNb|@ac{U>o zVw_ksXl__-;i+o4U@>lnZGCjK@DTMrVWxpWW+M{9?Q2c#pt8Tqr3L`ih>JbEf6~YS z`zk7ouZl4j#^~!@3hbF9cQV;XzHEK{G%nZ)li2KiB7~(kh{-*z?m8RJ3lq&JrYh39 zq~}WRhL#tpbP7RnxKcXJm0&YRfQE=95+!Rwp902zCY2O_ovfAq&)-pjOwB#M!~YRO zh}~mpr}oEroGy_t)sL7h+l}>2mBYvWHNF5EuBdThGTUnznj+&~4H9JHmBYDlqDr^p zFaf=sVz)do&8iv3M$6BOG^ZK8jBpZ;$C?1;*z%t99GGXV=l`!}4`Q+f=2taRCIDlW95X*=ft9p3+`l& z3^nWRRk5B<<9SMym4^mjvrixhG31dyd=_UTv_O95A*mmvfHs)M?Auxs#CSoo?%$t% z+pEoOE>6S9CVHIL>)WroLiiSh`FU~{4+4>KmAt_TAr?_l)8H4gxc*W?9LgLl}TySasUxWrY zxEj1t_L{Y>Tq*|CDS{%4>_>j?B}EQ15+t5*pltoIEpA;pK|@*o=Nt@s3-rm2`%-J~ z|6>ove;64ikpPK~3vbrB3OJ%=Kwq*2#w}E?dsW(?=rZ%LkxLs1io)jbk#pl(+v0Wx zWK2{E%O6flO5&R=ove=~W%t@j(z#S}bs_!p(htZ3{WiAA6QxiD1T@cvPx&SLP+9U#o})iO{g|^(R$GkcpMd&ra$YH3Y)XJ zDlkoB;{>=|>y;uO2u=w4cpq!SXozYe&~$D`PfjL4ARjs27u zSkGj#cQgpPsV~hxKwN=BDliSDJAG_}bI;cZaXm_c8%T^_530S4M?Nd;9!Lnduk}*-1C?f^&43d%yJ8>o8x*?V)RM1%1@sEn!o`IAf2(h%We_D>}s=) z*r71cnr=px-a$Fo`gl?{17yx%y`a82o0!&OKSQ%R8%dy#rtuthBIw%{U=2P@3$09R zsmV(ykPfcSA1>P~cR!K4%lu=ZQ_WDJW-X9Ka^B*ETiI#^GU?Z@JI~#-$G#K%(N3?U zKHRm19L&X^gc8hp&Jr}D$Q`l85(G|(F)D@F$}wRxI91%0Tx@N90l}zho3#ch{ zP9`~9-{gQlLs4lY39Ga3@2cWKuuLzUq3mNvAwyG2(*jfkcZ2FYh~#D?ybX!C3!-}5 za=OT5OiitQ+t~PGoi84oa^uWLuyBJFZdm+k4NTs1tAcWKk8SP;O`$ zf`Ij!@CAqkVU^0jVx##gaH$W{(cZ~}|K|ORDWjx zop79RMscgahQnxW)KY?85y_G|AmbG7W)ap(woVCX)Tk}pA zPM5WLTcwLk*))bX4Pp*^rIxB}ns2zJvod*f^gtbfX8u@agvXlaWD{yBl>vsF>IpQc|{x>bci z{T-oBHCvWW>Oc}Fr=DQ0l?UjUBV|W$!lB%c4~cN&pp+~bPDYVNm<}nbZ@*InZ90Bb zMPo#c(O$C!{ml*n;_Ny%?U2Ck?~*9fxG~dHiJ?G>i?GL%O2 z)h)p{I>^s*gcJuCU1;h|H@B@FK4W_Po*gBSo=6IL(NVC7-lL7)s+|F0B?pY?r+RS` zbXzf)eKxPX4w|XNMHr{EgJ@+VxT!tz@Yav&893z!?c1L&v%>8`x>Yp%-A}Z`4$avO zHy8Ux55^KEY)KB6m0NiY7NSyrh{%QRb-`PT0qAvZRR3!jCF?FbF`Sw$#CPaII6QQQ1#iv!P*#NFbrJ0&ql({_XHYj6{P+uhSWE4VeL{X6jxc zCnREDY?^o^SI73?ao6!r4l-u%_HzYVLzkJ$jYfxVG+u}={GtHqg$u+7im!aW@;Z

l|W1M>1Gy*MYM{b(aw;_Q83-N z^7^Q1PK0Xo#Rb)~MbD5e zFJwBQBO5xzSO#R$HreC|32?e*c|F9NX|0@5md7fgcE)bmm=U;7+1xUy;mOzGq4hI2 zkb$A6w7D!()ey{Q#l?pvU%#P_{>c3`h3{#Nt_YN~fV)%BJPGY(%j;w@g4Pg;R&s^) zOoPj=OgW;O^9M?YzE`PW7OaQvR*X?>2xxaM-Y~#o?7Xu2H+dBYjL0c~7H`s{1!tKNx;->dc)u~ZQWmiq0_68Cgvr?{u zcH)S}_+NoeJdU@{Xu-Htf)8#DCOp%C=>wW=aiS<7tO}Z0JyHAP>^W#X$>JDcIYe^>Q0dW`d!w5Sl zi8q^XMMekqlct)L0L30Dp*4)nIti1kTU*wnBXOcr3F(cGRsy^pmxf?$C9jfBWb+}c znu(57ea>68LF~5Jb8PR|qPwFuh_L&KMmgZ zK@(P#R|DTamC&U^h=krj&Y@NvY~1W0hCTC-(cZ9s<+l4)L_ntuQU4H(61(#IAW_ql z*VOu%T9`3;%5Vtk%`+YA*MBd6F}tlEwJ1m_o)`pVJE-K^{`$)DS8x-=cRyHvr{F39 ze+Gt8xvR5+!*QrOQ52@7n}|hg(Lj06t%j4eX<<~B0V|&%7XgVW{C6ioAnds&CpMqP z?^O`ju`vQ)AHy%8*Wf!npdf%qo(h(08b5U*JR;<4s_$E&!(GIldYU1Qa6_RIwv`yr za-Z2^`IQJLF-Y`h3=!-n0X2v^deWFYuoH+G_=M7}FV)|ItJQ!+~IdF=nCof6E|74AMlYh%10)La@LZqxDh*jU{G6uv$m`S5wgv zbP_BRk_8{Fk;D>_72X)Dmmx?8Bo+iq+!S@NkQ57)0ZbdB4#pU@*HorDqEw+c9>zGMX7+~~H%<+aV@KV-daN};{{fv)|ZwYcDEUjEq_VE_IL$Pf% zAl0jDSum0MI?Xx9LkKbP(9M3(AApE1XFaXK3z>kcwX^YqQ z@_FLFLhL4E&;4yL)KgX5*yrHw;rwpPcMeT7g<^N;eY6H6@5OAOY<`Yp`zR!O@_ojjP)Ka&c+%d+s?L3i67`N zny?TURg+eeVV7%8zVHM-;g4bv1mZ~d5!sdi^69r(ZE1$-H^d}&$uZwmAz}IQH{U^|s`ENXBA1PI!^V(g~cEZVO+@`Dbi#4miy>mUq2g-Z2S}X zniq5hHkIZIkpY+0m;(oo{$xMU^7kFh{6uhFG<>~TIzky%kHq}!BZzqOHzEVQe^HpX(0jYiq9_TaESk=kt zEPmHpHrsM}-tBT>!q*@D1&NTS$@B10nn_P^>~i3ihW;bW@k{+=>duMBydxOWk>x3_ znmb-?R-+vZInv1%wcf-M(4{6^00+JKfdz zDER#~j~lz(vU16=!IiUqfffB4+6^dO0~y>&Oa_A9SB)+m|c zJ?nmXjJC#fC|n-=2e)kOtVW(FMkK~;4&NQS7AeQmSY+uh2smYfE~pYmik^W7qcV&o z@MHDu)bcCfrOE{rsRZ&tb6hf+I;XbtZw1tDUx-w<#Z60?bSuW!!ao=z$9*w%-Lg;6aetw3 zy+ER1t#2t#7Xh`&6e?URfu1YuGlVe&kbevB`e8D`sE+2Xo4*kLEx#oT;_L2&#AZrI zM^jhTnt=AyEs#DK49NX3&y^^b-i&t`prn@GgDs=&q5MqhiBxr@U|yQH31*&zm`gURikOWgNnY5O z$;z<(4sPZ*7cCOBbhez?n9v=SC-tpdX;-o|B#a}tNUs|#gcrGr4eXjLA`G7VQ&5lC zsUqXZvM7-0dTI<%8aWwmT84suV8(?yDCQcN@R5w04>kCFkO`|Zl!(upiip*}lnk|Z z89aBpnf>~-?nD7nVzdAM`-c>BN@2oh*ZY6f`;uPIhWH(S5r z6sSx*jSq20qZzb!Y+0||Jy6)AZMD!2=Ze^_QXmIN`5{M4x*hq-=d}{FSnI?O0Vdtr zx=c6ln$&x%(~dM#!B6)A7|79TEZh-*=)h>u*_)&s?XB>re)SZA!IWQaJW0u*EqYg&XdOjH@}&o3Hg4l9VntAX6uiiqPKwQ|q(}3YFi8k` zO^IH!8jIYUwY#I$?R{XAl{1cn60t?Ctn%O^N3>1+N3_>*@@gNw{Wl>ng4=?)v+!g7 z$FK+b@o}NmRUJqwVI=-VrPbtj<1FEPY`=Rs5{!)y+-)~3+dLu12oKN>L1VTdsm!3} z!#j@DfQ6HfW@Un2E6aLeRC>!S*P*e&&aD2H(t*Yc8UG?VD`SGjx!3Zz(N##A%`V!# ziR`K!Zf}PsD~OynHtN_#+_)=x<>^BHY(v`9QaqX{DaBk|>UjMUcLRg3c9?8nUW!-) z5qM3ch*nh%gU5@XkCJ7+FE8@X-0CdG=eZ1NJ1kp$;J3%v8WQnPbP+nNB(^e(5?vPCG0xMu2(jG^ti>%S6g!+V92uj&jJ_44dQfL{9NlN{+%_CxM+#U-Dv zuA91;9Uw|KP|PSR$1BPoPJ)3#fC#RxO=ep|QoDhJaOiQP6c-)G*c(SYrpyiXak>=b zC(k!X(uA*dE2}y!b*c0c9>B-9Uqs=N%-&iWW;=~Jh^Pd1q(-ax7JW(kDeN2`)Uh>x znQR2cFL9-S4o-t<j*LX0F$1LMaFsi}vIMdx3ka&*kbuEd6YQU{8 z+WvcH`6>Jyh7JgGnkRqR;f1Wjv0Dw^_pPj?#TEg9+QT1yA9j;z>p2Hop1Z42htF9` z3j1WM=}8@;u)MP!i>A`*-j;j7b{+S`;dWXM-x$JG`D@OGYXww-cw| z8HFL2t*49~Y6JQgmVkQPFWwrI%x{+@(Sg$|_9ttM22{M{NA`>dF@%m>E zZpWGN7%}LTN;`spYZBtBVS^jz&I|j&L#tOG3$ak7bUtzQ z>Z9yBh4I?AN=7M(9!S`xt+t1d^8rYCZA%bKiy&7PJ>$N$0d6^N;W!r`02+FeW`AP_ zQ!ez$EC0o`RU!>Y*+{AaE`cP-*ELL87uWxA3|8%F*|cuZpyt5-&f2DSsz!NjW5E2; z9XNh4;~PFSnUY&lrvAW}d7C0s##>*BZEd$te4eKJMDpxn_Xxn$&4=O>OTE)SZBSV{ zj={J_NEeL2=hUvq84p!|9QB6qh(PtR&Z1YMCZHlh18Q|zl;mG|KhUGx^A?NgY{%Zc z^fsc(fQhen(YhUyu&WuDCUlgak4=`F#Zzs2kG|x6k)h@iNcy$c7*mq6TVPpIe25Na zP{i|t@?uYdaB3joV$AcczvBc z@v68idH+mA*33^haHEBqsX!K0s#Fi?Y77hOK7mKtT{#LK!*m8k1n3~lniabUqhG)r zod1R&4BfK$Wh~Y0illKwLDQgov>zQJQ3tTusB5`*4*l=|!dT;L(B=8!)Q)g;Xu}l< zM~!K=YxypGnKMGGi-idMMJW z2y@aX`hq%BaWBB}Ch9!>O2$G81*=`}3GHt*DEC-pCSjC|{&ZPCW8jQapWHbi!IL=f zxnd`n{4gZ8Hj4A}l?S{gNO_0dys`z@lyR@;)qqq`hafPdhT8MbYGKL)<&v{y@{?p4 z8peahQ6?b|rowQ$h(4NM#34*z`^QW|D$~?@`*GLrm6Dg<(3#M~Fly#nr}y(`dZJ$> z6BA#={eJl)y4Xjd*0XTbSWcQ|aj<&=_C+X1;8%?b#ApX4s;s;$zvz}!Nk>B|;L*j! ze`(IwepIhUodj1Grn1qW{(`N6Ek}*~h_g8#y+{K6c$Y8V0=nm;^!#?}*+S%!PoImC z;WPUOy|D+5`K0XwcN4q_r(@1_QfF$+6&H}poi7X_VCIvwOGU#juAyXfC2~sSUFuU} zWDy}5A*5F4xbk3(y2F|FiY;@uvD3>F1FnpWAt5#{VT$fy&K0j(aR zGCUvX>lEyKuNI%MnXnt~?1&(qhYV=9QL&23(PsT-UMHqDWsWJ+qp&!lqofBbyRcJhG9ogeGg<8aJ2zKTt=5fp25zzX8(^ndkqnvD zBnb(W^g{^87_7p03yq2<+^{zB1(VMv@ARg?4JeY%M3$)!XT^z?l<+^r6{CfX_-qXJ z{`4{+SEH6x%R^phR4wEdx?nq2^cV7H0~_*oXpG16uZ7=GhU3Q0jhPkHfHv0zX!s@D z5GD((=$hlCCWDHmW~|4wE=nVbWB0g}tWsaM1i%QW+{g&b1ozDnsI|evhkxeBB8|f* zH&ULX=9m43UdbtPjRh}@ozBmok39?8h!_bAZ+yR$JA_l;-}S36In*-par-^#;|cXXac*L(yB<8{Zi zIVg5rhXiHN{;dZndNq=+2@7AVsVFV70{5AOy9jwTDhhiivOs3;4>X}G`!{MVs1-_> zEAJfd_9KX~8)N`)Y;9oh9pb1xP86Hm`$o4M+SHyUDm4c3J;D`Esqv{PpEi-axCfxn znB!vX?k6Z!ZR23TTAB(r7mF8z0nW ze=--w{bMqK?E4S;Qx!#0!FF5kc?=VUtc$quc>*$ro#1KFUO7_O08Cc}J%;}bf)HA0 zc%Cx%LubVFWu;m zY!-}Z;{)u=+#2p25s3XMh4hmhuf}Q5{p^HXq^s~!xKKWPq(+G9ZX|d$-$Sr^s$1d* zMsr?EEik+g66X)e2Eay;TR3x{qE8Ov zmF`w7@x(s)RI9Ak7MvxrhUmwyJzj z?4y3)q|!HUTgci_>Rd*8NBE>`)Fh?P*1}cr2jW27Z|AJg3Peha^jl~OoC5wx(0hoxC+c0PZ6n2+)!{;JGjdnRyF`V zToug|LPAoIY63P&{>2}^8#|9F08*_>$ij?HgohVh9haj$<}`{^MPi64|9*Pa>?#wU z4OdO8mLgA3^-Z(wTEed{AFfWNWHMMw3kBV~0S?arL|3i-Itsyn<7l_g{$ylP9$S%I zRNO9g8VroV&L=6mPh5Ez5lSK$vKpacNACN0j8Huw&j(Z2mxA3#ZWR*jLG(j*fs!5d zp9<=Y({MnmxY~8h0jB zUZxZxc!AI+#ER##Hgvy8Us288e(!$1i{9Bp!LmIt;~&M~YGzT+V{cB63AIiT45Z3! z)lNeok>r-6`(D!|2}KQCuADn%#DhjzZp7#kOdn7ovG=|wE7g1-pb8slB`cy1PD!QY z0UPKd!55@e(6Pf3TeLUqC0tf}mpjeP6Z@`!2?(-KKs`8lv2hRa2+$8Dpl~OoJ=V8Q zlxWc9`N3F3zgOf6q*(;Dw*2pTAH<8egZ-%0LC`g#uUDPO%SMR1Ao^m42gi?L(QCTv z>nSggsu0vepeh7FX>#%cnmPtnTLGliXq)a^3ylIK^ZoA|pp6ua`PUY(Q4Cd>&S5Gx zMf3oQxu}sZQzyD~4%^R(O;=hu4+JKJT8bSaIL)M3DABcdL|BGkHUDp9+>9dYj(HlD z5OG|vQ0GB)z%Cr0Y*sQd@osNXxTMajRS1Rpi@9RrN%#m{lF;}E96_v$5LapmV8QTO zC7EJv$$uk>jZJ>G-ZARdoXLRc;cWpu9PKD2#@8~`V9npR|2-NCE^BuFpR7Hr7v1g5 zp54h9_KqiHI13pLoPUTZpe@nOiIWl_#hPCRcAfihy&}x(0VM~4lj>HqZk~TEC0JD3cEAy^kb_BqquiymQ+MnZ(hfm7Ai% zl*}@{g4LWel)7kR)r3t|R`jO6W*h&Ts7o-fctGh+bK9Y~hh!90)w-^V&{o|o{S&;w zhW2O@^|3{QaIl4WMKHrx-!JPp3B`NGc%im*4t5~_-Y%cC$sF}*h}_6RnohXOO!IT^ zmuY4ApsZHzGl!n33LV-{X{TtK>EfL_=~I8n74C7M%u=H$2ZYOob@f}B4SJ0jZHh<= z?~;Vu3|4me?Ir6d${*)YnfCq#Djvmx3D9U+#W>r5?IuJH+J`_2qQr?=cBOf0B;$$_)|}4rstG!ro~kM2a8o7qo!5IaQPC%zwn4mi0w1N zX+;KGPyF)E{574})64;f`d&Zjp(hyB$I~V;WQRH%A*JiNhSPYo9?(j`b|+^Elel$I zmenLlMh}TQac74noxr$TkEw<_uYcqBBeej>eD&=0m-z4KI!Rabl$2bOx(Mi{@lSh}EY-xNncCsZDK@Un2CbCzAEZAG`2}5t`D7zRy%b_#2`DE6IyM$OL$rU*Nj*JlhULoPio=U zEQp7FN0_>96{fh$cw9q(e1d)elsWt*sYjUPESluhI@ojOlKq`)PjfZkHLqEB&Q^TrxaE3WD#s0(~r%J6+%Cz zGZOR`>e#GAw+&mzj~-d^i}dIh=`LtQ=#WIy7-OHP!t@iZO^p$*5^}@1<_-1NWl7Ca z(HObslp=R8zXXPcSdGy07!@;?uR5RizT&XcamAXK@A&KbBR@tiKH+QevRN%-5o3W8 zr3PPqL8WYP&B4-;8rF^Yd*AlLWF7=x=R>j=jkLCAu{GL+3sBN~%i|2q|Kmo>AUcnO z+6;jYji2K;>lC%#uM=Z|-CH30Iob* z0-k6uS1M!H3E@STdJd2}@ewe26m5d8C8ju+-W%DXU>5>ULNY*#xPyEkO?aH#Lm>o)S$K4`@aLT9|&0>EqJb`M4*asTnM+z*XtWPjmHa zuV4mF=yVSgvKk+hr2;P7xYQJEYDK4`5RDv{3nca!9!`d76C%i%MuPnZtQaXwKnATO zGN@qiFv%w3g)q1e7=Ph_%TjYp_L1~q&k?pS;Nw+~zn5WJ5eFo!#tP=Vptvwt8$k=m z1%Lf|DT|9gn?^GHqo}GDD`=6&TU-l{Ys8#JVYq#_$NKhopGDK?6`Nu7q8T*Y=s~y2z2J)|TB%O&1;fFw=TPQk6~IvbR%k zy1em|IKRD)X~0Wj^=*r%5Jz$8W}_jBL_~i%&pr*vj=J}XgdB{-GfZ0_E2IK1Mtz6X zL}g&go#URgY1K7xj^4|bDgI|y)P_6A7|+}vXeWMLdIXOc$W`WaK!qy8f++2g@&qeB z@N2PSB*}p#yU#74(1+$3_iB$MYM_NnnGA&SDMAQOnIb}g%n6!GY}`HIht9A>dpQV` z*bQ9L0-rPit}N+Re>WzH|FNJx`%1HTYatSKamEfS7^ndKaZo*m%=S`I%xZft`EC6fknMAG>95V~nqFp<~hMv^fs z>eKartfzSB8$S6@_Z^nHgC}sR)e01v9X$95KSd--Q2cvUQlr?Iy8yOU0U%U{{kmI4 zqJE^a*A71f%&AWz-wFWfeO)_;e(PF?Mi<}LlXA_$ZAV-n(_ri^CxXsA^Gq1WdC8vIgAszwdP#iMKZ|lD$HysuBe{$ z*NYu&i9oB$4kFE2P7;WH&PWO|>ls&@f0MZk5IfFX8nM`pkdqM7R8}EC!%E(lCv!Nl zKaA&sDH3ho3H0zKBpa4e>!4dC8HK~9RV(Sin9gzuo+bZQK0- zBdFjlPV}^&y&j$GqZm6mEdg+i9^k_%-W-QVRt*ytDua!xLjwgn<#=yr{i;Z@=CeGz z%0KPl^i$=iqI~EKE5uMb;yFeKd>p_stEi=Y{eDTd)J158Oqcs)yC|UAZu8Kw%uQfO zcW@+TRK$c-1Rz3;hA}@qFHGSQ@jKdVsNy_BYFPD$vvD%)@U3E!z_u{>=i-)BM;OKv zkw+5?_F)CVcI=;;^*oqm6fe^uU_hgpW^WX+_h>cMvBU^`BkI>O${XK*sM_^?1a)yVma5&R9hnE(a9;a4mcD8S@zlIs{#n0erkVU9!M z=Y|MLc;Tk~xgVh+>@HA_D^cOc$&m?LP)>Tai>n{Dlx4oMz>Xc_P#C26RBke-wt6jS z&Yuej5zLxavNIKm=_kLnsdW`WbE_lr1a>$GOm|%N)==XrNY_&g_($!V7P-$nB*f=jb>K!REi*`?UR9(8hCx_8Lhfh34jMK88Sq(08o zzwH)Rj`rzkNIJ%Vcm1wKTLj462_P111-i{Sm-VpOCnZZVpcIDps(L~s|FfD~^nf0; zVkFB1Mh}_9)eigP?C>S8RqZZm*aN{x+gIXSnpxYa8g5Vu>^p-0T_YjVt|{3Pb{ovt zA5@j!rg=KRq52S$=pZxG8w+(-KihAo}Q6x*wxvlJavk@7Vk%-zx1+YIDh9h(p z&7?eu$gIchdUFGjuZCZCH<_44flM843JYum3VLCr1kJ%+>rYYJhqzadbpt4?=5sjUr}ozcJzXZNSoYxSF}O{&kK$`m+Ljk(VRQ& zZY>iEwIX47P+-8Af0&hgtLF9D>*eX2UAMzS%A@|bHe0GjP4GfR0r@yY;UK0)a#Ze7 zTYQb;X-74HuJ83%b1nQw0o?+j>iouDX`B11PEcXsWg8AzQG2_BLc*j>t}AAjv;s^8 zsA)axQ2RbCH=!Dm+n>Q$UibE^Gk1%jT_pHtG&%z^Q2 z5O0`CDb*9WiQMpB?b9TYGy|z`q}z${xXuODn+yi6uH3&?4ox#|oc&!YlP+?LR~;v# zE}`1kP~n#`d_*FEb%2Qxzb&JEz(z{jfPrf(94&q_Q)O3gSeazjHw5teZ$s=jnC2Z) zHB7-DJj-po4tkn}aVWbxVu#NGK%kqm&nw+jVDU>r=X?32(c(*jH$wRgZwdJJ#%+nw zC)|XQ^M{FU_vD0;#Y1X@D+-Q22J+_=y5^AQR|I2M6p=KeTPc{o9&atioF0DjIl-8APaM{BJ;sb0}SC4mMM>j z3T*W!i~0<1=hdn3SARw%dZmF3JqV-u=#Oz|@*<_i5t~EJr=r$7_+4zLu1!?k8xkI@n*RlXsVViqhD9v?P6lTu7R~ev6O-Hwl`WmPan3Az=-m;p zgkxsSYB^#q$clyH%QGADl4S>Iq4wHPC!QSR4li&lhSj(pOnBnEURNvW$y9|%y)lkv zaKxFFI>h6KIjLh&89z_Z$eXCbM#j) z@yR0jhKF+)9fv)(cUYR8*0s&<=2sl>ThEWwzvv$5ItKV~$i-_oU9r?(>%im%TF^2=O%*-x_`7`%kM zlqvbP9`*UwOUqO*Jd9KX(voaqeb9Em-Epo9PjntAdcQ$TC)uU>!5H}GdUSQ_a;DR) zC*tiX)THMuR5U*u2t)E~#V!bU;syVioDd)6GeMYd+)^1Khy;q17S7ns45%Vf6f|5; zEG-%zg2y?jy*+*7m|r3+67TPtewX()Ovuq;z_G`Eg8QK0Ge_~6DTyo)YhFbr@)iiD z4pB}4N~6R4&31mnm2k$Gy>vmGFd~?Pl?580)nMA+h@~-l@6oqZsHvLLR6JsRcFcPN zJmQF7WvUnZsv-0 z_-VuohR8$*;eTML6&MO6YkO`;FGWD?&(8=b0ltk}$^=|bE10Yy*#H-{{pzRAv~yvS z3`_$8CBE@+pv|P&)00^ZI2v+{iqz@hmO3T zZ*cG9w!>a~R)fUG5dQ|F8o_vEBzk52bh}ru@M9LvBT;Eb0SLUKAn~$$d5kfASFET#LWjb#w*UwV4(-nq#kY4ssG|HNCall;D8c1O}Rw%hxEEecfMrpzpa(1Qr zlq1QY{k9g|R|9$cYVaAyQ(a0_=u&MC)a@=5Ii*mZJB^;sr`F*f(&tZV1fEo<;7;2@ zP8qeKw~1-mnB^b@9vVC1C)O$yCW_JjRtQ2^vvxMqe`Gv-9S}MSV9_xAuv78w$|QB< zD_-Xj^8U^DE3dPVk6(dfi2_W4Y{uM1CR=_bSg^J1w+``Dv=TSr6JAdzQJ3|-?ZhbJ zegG<*<3KYqHEN}7lTC-ZdZzZDyJT79KKuf@t&hge?DsX<f%xJ9J%!FNJ5;TtSKe&^Rz0NS1G*B5UCx zz%xQJcbe8m-!2DYnKh5%fOX+l=`2TlsWnGDu*HuQmJ7}pqqmhV_{-#7O@2|EfuqVC z#mEXQn`(zkkTam{hChB=64WW=Bn*{0NC_kt#0T^q>;!Sh5T%P{?;9%4#};7@G6V@# zes@&;M`g3okViJGIM^DEvq@#@S|~+rof(zY_z#{eP^e+P9QX2;iASNph| zZdtZd%3&#%O35R2Pm|zSuEbipd;iI1wp<60__N@~vg6IR{h&tHsFs1n&61l{h+BJEq;LzKzzXO2}T=dF~q<(&s2Il5M ztixYP-CDA6tUNI?#0X36ltrmDrSG6b{XNIJ5@iNaS`4yZvt}e!msjo``j1 zu&W00zp3^ZP?W)ZQmOy72+($}X8yE=MQLH2B5Gu}3KNS18K#T$=JPGMKYpa2l7L8} z$j_<$N3er_qwA1$e4yna+MI)QK^h=cAe#S0*K80DQ9C?S^j@kUMUZTe@1YEVz?hOt z%>6+(;Gg|}Opz^hyGP;wVXR+3dSMIYecIvi2qxhG(oe8IIuTv0S8|Goga)pMz6RtIbZ#t1TTbj z(}?zM4DzHMC!Eb!vA82sMmOQUhZK9RF8Tk@m<$UF9TY78#I-+5Z{`;kH(Lc5t2C6` zs2`Qb`lSMC&bEl$9TVEiq&v!_Hz$fN4=X2%8VLO0{)+(=xBVFISHN~ssZE@sOD7f2 z_VOy71r5fZzqC_zIpi4eN{Zv>Vx(YgD~FPcnhIy%2MZSpRcvNyDD6` zK7S*9`dkzTsxllz!>^QTKs~;NZgCZD*%<%JF}bH3^htT1$YxDk(YM?hnKOEqrVjmH zjK6i9O1~tp@_+4wDIsb9^YEgeRB)jw^5^4!J?3mV)YkuWx#0Z&`bQgoiILMe8->5o zZD{Rw!#!!V{*L%R{C;}j^#W^R^p8+u0CrRo{u7eHOL^6g&28IaTVLm z%(pLx22wJ5%!gg$I{Tl=`ka4i!v*0ie-&-yy>oO~AOq?Ss>Euu-A4A(TDU7k;;I?fd|1HnV#3UommE=;Wi=yDsVb}(ah*L4QB|=2 znexuZsuf?|o~w;zvwuXxrzr(a#2o9Wx?<$jLkH+;**YI<4#Xrr4Ly7Q_cjXs(*8iF zdy{`Ib%ZR9$!}qt<@b{*BduZ{ltdMZ$IC(^r}V9Ka`I(fYr=PPvXeW*KC2ln?WOEy zJ@+A_J}7!*h?|`p98`$faF4m*L9pG|m8YVO@_E~1GkVR>v1`6V=I%KDN^6Dmv1ZvD z<6Z-W#pfaTnM@l>zEwxe$S^eMBoV2rZnG$s)0ZUp%Mu6p6eqPKK`XMwjUW!?PE_FfOos2p}V(cy?ooY#>9RQMa7u2PHk5%x$_X>)21*31e5CWC`r`-Uo02pj6GHAhP#RJHt5BXqC=YciXjtJuwhBx0ZC!SoL>#GCnysnXW~5E#)Lvb5nKy&r zaL;Oj#e{X->EIKq?M=X9fc)x*qA`!m{R&K{SI?wcm3Q0US)9rJHM3;j8<*&jU4@7b z(N?5PBW9HkRz9a)r7D=A&QiCE!l34~`WNM<(#e&aKt};nqOv?e_!2l(isLCGBTBGJ&urvVt5Gzs*fefa!Xc z{e9Jt%KIvGM-GxhMs5&Q;0`m2b-PbZN#B75-^f_H1Oe)-j)uVcyoYkh%$aaAdEGdF zV?!bgP=PvQid-J_?GF1+8{4?Jzjw=%o6m78h#Hsjo469wQlfS&N4ukKQcjb4{A&w) z2lOsW_3)UsWPT|QEa?PSP97ijaUY!7 z^!qY7Ks1WfD2zCByyU6!eP&%c`hZy3H^r6?Jf*U z&r~zn_t!<(47JWhX?c!caF)_XGtAcppNnqS`?p;Cui_UlWhTd;+dRu}ie*)Ax}BDm z7bRxp+p^f;tI@_&i&(Ifn>0W%GDk&P8r){N7c?vG}1`wiSmnia(0HnxX3oiOXPh zurO?rYk?Ij>6ZmxS#HDcZ}Gjo_!7A!L=RR;kQM2*&Jp(N-wa^cIrn|ASs+#&V<^1N zTQ>O5&EMRdK=w>R7Dfos$inhmKj#D}NXZ*I*O-;Pf25s+`*R?LS55d8UbB7K%?58n z|JDOS2AyrAtdC#7VFm=5Aaa{XhZa5@-~VM$_Vcp$LPu`m{moob2ye6>70M{u~XT3+ZBT^#aWgz({|aCM5I{N zx8E0kjdde)U+?0_I^+{3`qmsQA|_KYd92aWr>`=-2g}B*M{{t3MiqZbD1yiFoXA?M zru3(1Hc3{vw?b-TjVUfN&}}}w)&F1S(=umrmnnA#giT)N-6AesN@y#BR(aF5$tELe z5LEKv(I`4^iLK7g(>3%G`rX>7DgAK6>|%}(&I=KlA|m!?$Ug%otMiOi3Z&ONa_C{N zZ7AL035qv29@6)x8JE>w)s-6dTKW>@Sr6w-K3&+|1K-q)-q$_+O^Q}6eM-C)*DELh zdBT*V+;*N+)3caKkR1Vsmt<&+Wm-R(nZNiy6RIrT|YjyrO$4N3D8nZ=l2UpF=raZ_*gZR zYC2Bj@M2Xm6y(3zCvSmIn=|B}MD8_8)!R1NFktl$P9seK<Wp6nd9Ehl@|y-m#~UUV~$<-eu;vA)aKVa zmhHbmM5F!n)BKateaqRM+U1pBaA-8}{)9Jk;ns}R?!T@=XdR|?#pn{~r9i!LKd^W9 zomcGOl<$~sSSQ99`;Sb0%$d8mJO8c|ek9OXzqPVc+3bDy)M$FNtGiTH-g1ccXGdoS zNSm{lXs47hPXsFoq;8f_PyHFI(bIx^d1&-S{Qn0;K)b)HUrZFY(C8&Xb>-sU%6s0( z0DN>L{r(C-j@Zs)Izgdkp5c3!w;NpV)6q&Fq`}MQm7vGiWIoyI5c4-$cFu?{3PCHY z?&m=Zoz*2tr$Y)+32EZ<)cbx=aE=bEXMw~5{2p@`cS7ZjPJLT-sRPpR>7#MzDs-Z> z<=2*ZYpfUpI%LMRT$S>Mc4xi+?Kp26KxCN42Kp3nJS>K9!C5=I6(MhnEf+`Z1Bb*hn6{AYWYkLR-9TKNU}P>q$eo{Ohx%Ui&E7)m^L@>i1|q&s8pf!xe){ zyn{LzB($@x)y_9`Topld7L%maaFD5;jt>Ij|uF@vX!&l zn_sYm1pf0)mxk$w-VdXxyU5|fFqq#h2?K3_rARoc{Ae9B)7r`bs34oyJg;mN@z z4vZA17S617`9zNQ5dHAO25*Ryfx$9atv`tCk@)K0;Vc@uZeCrMz|*rOjbs$Rz&70nmw&Lj4`6eT^Eh~-->(5Yzpwbq(8 z8Pm+pmcIO*Ll=~hm=avd8^ycwU2grtRM zVm}PL+vGZPCSGa)4^|60K7CV8LIbNZcJ-y5awkYucI~{rVKvo)^@VTfRc{;cTJt7F zCGmEQc1`h;fB)WiE*l`r#)Y{08`E}{_&P0GwF^a$)#_S<5$_dwb;L&ggLh|`0Tk-& zQ0}{}H(N_rNUu<{s+Sus)H%?kGNRt_J7YwSj~8LA2S$BkBF64r49)BA{(ZcC)PUjl z?#QXfWs1VAeBEBz;Fm6G?ftOw@5c@3b9a-%85fqDo%4BDyunOi5p8?E4gR}^RDzrX z*RV*Mzm$?Y?7_fMjfZ-Gv1;Z~Vjj^I0^-mJ9NlH35*;x;A9%N1(f%KQWQNz5dTZaIC!HCw{ZP)+0e*J5 z;Ek9r!P2GBROxnUh?pr7&~7ve49?H;Ik2!<%K%WHdxns$N@tpNId@ik;XelN5FPkreXwND&($GQBvn;2VE4mWU8l-aWvZ~N-qZ)rN z?oz!jA#GeV!Fiqbp}ACCkS%q{LH>{1?JP+=`{;~(r9Ae-_lBX_(iwI&wMk82{jTb`HKpRn)HH_D{C@xD zw*9~g{QC`KCZoSj zdT=>L^TfJpz^I9!u(dhD7m0=D7@`r<_+Y%VC4aW87-LI*e?@qYG>7+5qwkouGQ<3i_oPHgv`i%Oi#Xd{ni z@k5ror=!}Ew}D0n2Bb~9JJ6Aa(cyAcqB8v8WD3Dy8VVKRv*}NX-;_L^)<4S(rR<$M zQ;frn7H$Uz-2sQ26a#V016WOtiOYqWjc;Tt8Y`}}ZuKS(f_l2S@?;s>R5!xRtp?y5 zokF-bKG7m%KDu;WxY4Gs_s+)%!({3^2KJnp0CTJD7i%x$(;5>;SCgy&Mx|}oLy1(a z{sg&=yWC3*Py+ol;~rC8SX_~^*g1OW(5v=^*6tGGn5A;eY>JI)^bVu4cNqf3VeF}- z6VtTp2w3&HbwoKLL=&5D!byC6E>3=Hoip92VYysZ4wDBI9IW~VqrQGxRyQPh z%hJAJh2aP{vW|tmT7}EC>me#oWPMW6cbjaD zQ64mRja9E@|2wZb6Jg*GYEqRQAl5}GA$Cz&B&q9vi z$l;xC@CS?Q$=;+=r+5FDnE#>SoLhijMhhZjWNJ3`WaOZ?p;?`5vlU`F{k7~%T?XNFUMD?$N6ug|iuG}*& z*L>TC)rioxARHzk&1qCLZ`dIcB!WPK5kEA4}`uhKw1%Q<=!k8&^?@YU=A1?FEs?Ca4ARc~;qOMH<&I*k-Qn0234mm;PUy2xzgCnF=Bd4u8`=6%$jOU7t7?v9FRp}H7H&~8#2uDTRYB;;}g|xe%rS=LC!gRI*YYh`O+|Chdlr)tSAyM1AdqHKt-vjABbz_yF+s>ut>E2iMPZ0LzS0$%kcjQ2N z&TGT4<|8LXEjbd>Ao2r5N^2K-E9R<{*7g;*qwA2u?&e40W*v zk5qEqINH$%=a2@uwle`x`#Ebp%5F}Bqt3Bg>LN}$Ag8Ms2(FA%z41&N_NXU!wFK8FYUcG?%$r*~b2g~JiL`dZQWqLMkL)Z`Ej zdf+8gmugiO+B&L9-AHS}MC?cRgftEn1_ZoNlTfqZB$!W1`KoNp^o$NNW?X2N*iaR|5NQ!aEW)L&azraP??Y2lB@kkPH@B49nxQTv+Yz%XCsA`je0 z)TJoZ@a(TZv+aw$%@VYkds@M>%m6rYJyWik#1xvwn{2<=Y7>^tnJzfY(0(2Y=^^NN zHLaoLpM{Bk6TCtBvsn8|7G(f|&qd#&ia2*vbrZqn+U z(tDTZDI`8{4_|i zJtk<@P#CQ(g@_8D(bnas^Nh4Yxk1k4ckUF+km#o?dWNR7oaO+<(_SraPi{`_6Jx|L z_F_##cIGZOe=OBqn5+=SVTGl7VqsKa+RgSx-^D!ZMygD>WNPEflau1sR9`$EL@nk8 z_<&^hNtgo1jbv%enBr7IJOwEj=NRf?c%s@>WtoyU?X;JNjFa8KBvW%#XM$s9hu)JJ zmwk~=RSOH9Drh_1jU~M55@qFSMKbtE%NoqpN_SQg`ZO#tccT zS8-ERRxrj7PDkzRiI3iMUJZX#&FbWW=Jk_`K<UXkUWJ}ZKjhThJY zH8&V`Y`G*4v{>j#;5$eEPEhl**5DZz z5&Vz2!7jDB{M@5VCBZJ`_qfezaA3Sd6AP&ONx+5L1u1Su^0KIHKNnJ4MYm~^iv5gX z@v@=M|ZP}X2Dx5s~d{7%VD_62I&ke|8Fjtpjv z)n-X%b6|Rx-<9tDMz2$BZCP`v4wF--xRRICik_}hR$lWcque}iB{%3e1xY^cc}nj0 zZT#*HXo*cyz|JxIt0s+M$*{!7>3Jhq^U8DmCH2_f86)+rO&8W8C9;g#F^jE=X0NiQ z5Vvb+0+%~e>3ymWox{nK^dS7NyOgStCB`p3M{Oyw(7nJ4mzl?{62O6ZxO!=AAeTJm zb-uYve{cPlZeVWAI(O&29~C08`R7O8d4LwoOt)^0%dY8IdGZvIAq`skkh?ykLQQSDe)xU^jvmb~)VBq2<#(K6D}blI5mzsqV+58n=@`K4weNMEA+zfky?j z=ek~1jbW6SLLo1xoWA^X!S84{>nJ6q>2~ve@=>SLB$(eg{J&jBdSy3RN{$;X4(W_J z#-}u)+m+WqSQCO#1G`_?yyU-as0{peb~E5YM4iU#UO6}1c*ms>c4LU(P5P*LJE!wl zIdSGio<*r{0=rquePygP)5{DQ-Xt4kga~x_^<+t5)uk5&yuP*8O9Ms;eUfy(xXY{C z+pbs?ahF)unIK6?dNVU#lRvQRFrGyrH(sHbL-rX_%uM=Glpz0@Pk9?c5;Gz+NdSuY zG2iH8K|lP!UqssNE+Y&lqr12&YTc<_89FdW+$K|qB}H2NtJx8-Ws@9xkBYjn*78uP z+w&to9Qh@f`lP?=gXU{<|wGDkk2-j;nP(d>g^C6kYsE`ZZ z(bC9t>dL}4dQacaJO5Dw+S;aD=_h5B-!@=5085P<1|z3V;h+j1wCA4HRmkzIR3Y5$Bt{xAcOXw zLH0WEvYliDsxzJ^izKp0O)ZrX^0&I??@)7h-x z z$`Rt4c!A${CFFKU++h|QG9P%WM*|MA4s z?6RyfASnEmHzDttb^rAqRvYqy&tDa0>%y#leRd2_osLtcqVjTry-`&#F~p=k39__z zr*~8g_%P^hf;e2_C=XoKD&rQAN`)+VU6F~_vhM3lljX#I&F}h6H95LXnc?;zr#c!& zr($($*tJUdHq4s~9i$EC+om7K;+-Ml2i-PP`p;pH^R zI6+qC*THxXHV)ott|pqcVc}B(Ctl^J2*t$Jodo(-YNadBv?vsBmEp-*LYz4=*)Ln} zA7a8xT-ubc?F#W|a087D&P3hqxkMJuZlJ8Xb??<_mr5PDBSe7GjqLUBDo>F7=HQb( zm~*IjJ1j@7yg){6+&Xo6#% z;qFwW3Sv7nmxj$_D}&QS`bYi4g3R84tZ{O|No$%1y2iaqcoHwFcrs(ltTv{&$utUC z5^z+lm85v}gOdqY?Vu`Sa+}!!=2fcWY`GTbn?v8#qkbGa?@%H8RfHjv9ZE!LWgo^Y zr4#Cs{$sd0QQkMj=1f(=i=*fWiu8mght`P_{}E;D%Lep?G=r>C{h;g+Wq{d>M%$xl zOGwtrle4pDVkY|E>s0BPM|hREu;zPSkErpcd%#}1nE4;3qP#8c8<96iyFJz+7g9!? zxb}YI;ps=Qsw!CbBKVzBkpk1)g|{7tvMns0=hkOcuiPN=ZB3BL0DdQ`GY3|LE>eH} zs=WB5Z9ZHK-D=2^Yo?B#xI$QtRc3x~IbSwl^^%JUcdqRBfawzBFSjfj@m7}P z8D**2EiuzM>%ha5$n$zHIZU^@o_@R^*_sa%ZAUh)NKiFVczByyd6^h?CU9a+ydc!6{*;rTYr*Lh;{Jl=*@WQcG#v8hCQkmkA$ zPkGRSnMl)0U588HwQG&C%uY$-HeLjFdDKNMz{}OjC>MG0Z%sFIYFVM@;c^Gzb?O zra4AOL(Z%O&ch@2z~|GxbkyH6{?AUsNoLaZw{9o(!u8BIcXsjQ#+$1yZztCl$bLOh zDC0*a{h1%RBAAompebU;GgC30!w~8JG$*=rgR&?qD_&m-&n&W-;j*e5#oU?)52dnR z62;)c(~4)kjICx#F85^Zj8{?QxC+0>j7PtmUyO~X0`_1b1wAJ7*Lk5KVBF`0gz6CbUlfk`YValg^ZvJTWCacygoYG?v^xhq)$;zELdlo>?%!nQ=Cs4m;MWGOELg1&u=Tg~^ml807VT;q=XLfms5bQt!6-yo_hYnR{1b zj*;h661R}=6$dQ3ehfYt&dP^w$+!D%I|}iKhINF=p7PT)_gt@lBVL&%z^idumeVJ@d^2Xby}v@gghGRmK1YcH<&dtBVyKCx zZrNOnv?!|Ydd%2RlmjbI9B!kNKjp((sHE->QX=w$sXq-;dlIbdLNW*%zH4U#8PJhcD7ACeGWq z$p9PwDSH})wQp=LBD0*>imrCsC8wp2%&aP}FZdxE{0I`G zwf#kO468Fu8M?1ZuFro!th(^sJ4kZjwK?fMDsZYR?e$y)eB=F-K~TSvbkWjrd~aM# zlyZ>?D#)3mDP?DzV!-bH(WY|lYY&_|wKJR+Vvnk!sL(*qr1kJ4vCR9^mx+dKZV_>T)fZ6lY$t z4Be7xq?i_ItI~&-@EozHBS8!^1xRaTxiaj2W4VlIz7+KNnwn%9cJXPkm#|!t#~Z%G zJ|yZxFQS*omXxchZ`3w<2Sx@OGDa%Ox^$JoobMsB%H4 zGp486%{7!o=xt#si|l5V#Bf+o#V#eRTd`=7>9GubT2c@0LUTtifZs_4O9!umR=dbb z=r1G1>LC6+{NQ-i{S}XD3WDp-u&Wuz=#pqT>>?y>J7yi8VyJu^0yG^wlnrh#7TKP2 zzO-d1Ox83(_4BPeo$tUj-5yJM(N|@h9WQRKx?5h9oH>?kz!&g?yfUl}d(=TkE$-8X zVU;0MhszW1zjn%3UD4KX2Zh<*9|vqy&$7$q+S6(CLs^Cq`ZiW+m+SVtj7eK?x%snq z7-xA-U_q<#Qn+j!fJ`N+!}{n0#ZZt#0=GPDrS$O%^0Y8{rmZ1COminrb5vtdCTuE1 zAyIV;IyDa)kKT)oV-Dt?an`ASs?b@PL|9rYcb;K={MO?ZhsL!ueKGhVg^U?9LO5*3S!tn%|Y-WSz>VI;Zs_ zHR-Ci97``wK}z`vP?zfSuNGlxz;v9J1e7s6OoFIlhB045v`OrfI-LQ2JK4@Z8Xx)A zoBGrzC={b_h?dckR_fL~{uJP9_nbQ96S|amHa>a@d^C~1$8W=VIi&3@4y_Q*n%@_f z-QjQWu%vZMcQAk)U4_$#^oRERMsIgO^qRDex{HfY5Xwt)Lh|i-3NYcOH~yaQj}&IM zSyPi>FJ~vYiSm=O+f8<7Q1cwLxeYxYZ=4i+UpktyhKz|nVI7Ja?yc8L2?g`X3h!T% z*30$Y8&Jvp%}ikAD;I&T^!7UhxCYL43D9A30o(l}p(b?|Z(>)O)ALw+WQ(UW4~>(& zrSs4*J@%774F3!&DlGvudAQwMS(wn|c;`oAmy0OBxGN|G-ggU#Gk@}g}BCp3jm z_vRE9S)Rq~kM1pn9zJyTGAqP|hx)hHACPNV{j?ZXHz)AEiVaU7Cm(Uw0amIj{dS-+pGRsHOp4n$u7mi?O&G&G( zcb(hw(kjcEn!4}4%By`jTUQq0=rG&uv`0Q0ID@6{oy2Iicy-o=G!#+pXx-Xoac|86 zd^1Z)###9RYZZ=$}8isZsnBLHrD!{#Xs|>PxqzH(llDWu3u~pwRT7l%|Vq*0o<76C+SByv1n# zy>aL>!Qc$d@7fjwXfdK|7AZ%9Go(X5<^f}1<9GeWeLEl5%w%JDJ6j9hF4biN=Al!H z6ohw|vq-@%*Zf9&#~|#B?Y2D`WRR9am(FySOdy`q0Njr=x0C4mjVo)ervrrchBtPcixXIF!WYYvW>faN{w=p_d}D-@P+F>vmW1h1UV-|(yG$oUB@UyUTkvgq&HYl z4vMP6mSH=cQ#+tp!9_LHMcRh)-B76gmo=WyC5_^6Kc@Kq?fQUDaoeHpx29bK?$6>CS(wqBY#jm}_szxu&eHF(34Sq%7fbdk*i; z5vJ7_fVRH)JjEH(wf;k;0_A&cygjR5S{XlC^^3qs+Mqu*S~mJCF}nJ2k)qJ8l*u=f zxSR-6FI^O$B(BbMNbOfpiVLf>;d#|&IQW#zHOSNV+o{1wVx}!wB~LFuQ2Ih*soa z>lCjroZD{KoCaRYLNn6qp&=Qnc@6p-9Uu?b2R*s(($*wNV{O&Zyqz_F?cPu15(-P? zUpDBPAY=mx>10}?8G&5&WII|KhJ)mO8hjl}i_gb)>L#s0Nh@2}&Z^~0NKC~w2@Q^Z zNTLo#Mp6KOm1D%MP76F+XQ&KhX5;wA!;KQo#M3;ff|rCKarA~t7|(Qhp>em9xJ3`` z>EDj;z+rJCQM7TdXB8GDx&oqcp?V>@v+*0T7`yAj1)^Wmc4GvCFE6UK=Cyzk#iPBt z?C>e&YRt(6<&>6o6O011dbE8V39u-TRgsp(uBzc*&n*G*dNAaGQH}z;xgBgatvXdE01JLZ*oqv{@?}2?Ei5$^pTHWc$TcP zO_WbxlvZ|I`2d4*CYPCdJvm8x?Q~~lM&R%GpOKkK`FnJE4r`u*7ZS4HC4)&zJiMA& z#PmZp#Nt|6L09R>9+~@!8J8?VRo-Ngbi9gL=B8iEb5Cu&UnI!MAA91vw7s;6OR2Hu z^59ja$wTHl>1yeEVt2r{-MR$f=9o zf(~v=S+|kKaKU{Z+VAVV)ox-}UgC2fhojwrc)ZYr+!9Qw_tz;pA58VrU|x1g{86Sr z;u7{t28%Pi>7H?k^Ku5d4p0p~FWt?|nYr;=#Yigab`d`^RPBVa=SDw}9BqU}Zsp%l zKhcO{`U`p>(E;=mr2#Lr(?znr$)L*F%V8oY^Jku7m8xHFog|JACc}XE>F?amKCuJI zXNA^1mvcj{vuoy#5*b~Zof7)(X*7q68Q1lgchl?ocT*{)!Gp z8JX@bO2G7Mtis*D$4({4uF6B1!p2ns!?G z9md7i%eSKW;pv)hGS3kM!&l(t67xzOMa*x~(^*k-qN}CnRa(iVOZssgVv?0Jf^jW# zy$&CjLGyk%>7_SIF5*6vlGkoW9Lv6hr@}cdpAIX$SGM}IXi>wu){#04+C3sUgr%Ki zGC75qmGV61_WgW9%r&e%-^z}fx5@)vp>9qM2K-54DB*dEz+*7wn+zVKt6|$(8gczT zh-^L$K8;u}G$L29gi_3pz<6`Zh!J&yS+JA)h8ghgA5+}gMbVub&_a3vjc$%%zw6z0 zOJACz8<2?i=rx9hj>%n_AHf3J&f47RI35QT)#g_o5N%xHK0*&~mZsXJJV!LmiB!WX zep8UX5OwGe`3<8ngR-oWX}ET*Vhqb%9;7_!{U0o(LTOvVPs z1N1Hw?YtW?0$;|BSH8x`aI%XTbZBG24#y;FrV`_=@P(&5W3bM54Y~mA1azO;>2{1a z-31Bo<(kLeLEuqn#;mYCuR=H(LOh*hlz+GrMpA9gG>qwaRDFIl z8S2q27mC90tc;-s1-{YhkzH{LR*akKF!hIZURn16cr z>xL&+Qg)bG8xUwOXAnmb>rq{!?qt!Jr0Z-&yfD6eco{72gwtPJPF}qy=S8{O%^epiBvO;T(T68oH6zaPK~jL-cZd z+{{kCu$8jgZMWRuk>$j=-H?fej!t^kkvE+7TmZwI^y_F zjIw=R0IET8Bg0OnZ-*9$xl|h`8!slZ(ZYO*8zI;bJ2qDln zoh4)5aaBJmPSHV8*$R)DwWy+z0#NQ)OCqC9%F1sM%4Gw3gAVt*qoRuZdOFFelzf-) zISs=NrJ%{~9^3O2o3M5npWjQ)!8CNnwOB2L+RLDQd=X|qc_ngu7iaKg*VOR8TH|fa zrghnf7tka3vgC|WCuTY~8Yrq~Gy2jXuoe?_5HG6Q(g@bKu?lVM)C_(nW`T!HlcI56%KE(2&%d4Z-9HwD z{jS7z+kl+^w=x{NHZt38tM&o!B5f=zG<9R0W5p*)y>`Y?aA5AEyJ`{p7@2hY!-@1G z>}?ten(ThJ#3_+H_#NZ_Uf7<=(dD9%{Obsjmt$f^P?^M!PQk>P)BXFL>G0oBKlE#0 z7ZAu_{WN9q;ZmNrDf^d+gb@^Q;F2SmA`KSDs)W&9pVm-df8ZiOo#}T?he)q+MSv7_ zjZ`KvBW`kQaL-dVVl>W5RpqaPl)rNc8NM0J$vh>T_~i^KA2;X{;ccQTj^hwFfR|yr zMCb?IP5G-$ZGG36x<+&DaU+>IMX;OXZ_Gf&L+jRgk?QF-Vg1AM!u#0#$hrBjRF9nD z*){GK8W;H+k%Cl>{1G`4gDuBPt-DsE-u>%51HstWgMGtL7*TXrAiJas+19UCP)0-j zZtlB}8c^=~ZVdG{66Q++SpBnjQ@aY&_hsVDBOwL-^5(pmA)bVub%~~3^vkF~9N))h zBMnb>|5PPfD~uHC^P^bN%LWWOxnQ`wT3*?g*Fed!p5YwlG$A#eG*3Iks(NoeKQBYx znQ&;^8?C6R{<$O$-)p8la7OuRZ1vSEeYj;wEaGE7XPE?-Rsud3|E~%#Uxlep_yhw_{@*pv_jAv@I_` zp7ctF_*Iqla>ioK_QL3uOw zd4{G3nQ-OkzC@Z#8P#eWK1I9=AB~=2au{Ir$;I-m+>$5wL2U)a$x{9^?Y*h~;j!YB zIZe`j$Pk)~OD>&zekb)U<)>r!jD6)L^Crm&D;3OM)N%N@(&#?LJA3Bon02eH9pTFg z@xUc`WEJxQo#N_}bc9IPE8Vr5Dm=OEVj{0($lB*A3CSeJKW2kFIg0;8ih8r5dQ29I)>6lUIB(! zyz)KFc_AViugpEuJMo0oot>hAm8*-ZMUyo#R}qBpUnK0#A(ZaCRi6H3YFN@E~b9G2(5_zv}(fls2_CU(P zA6~0?6EG}Xl!Rj4HA`JaML^kKc{ld5XLcJB_Ok^n^(rXjKeV5qbRr~PG#f;%R z7n;%qVi#5fYF(RSn{bg0JYb}>Fm%BqHtl;;Jm@rBRIL`+~6^9jnp zd1-@Y!+A^6UJeg=9*j{XlAT1DNdh1zP@qOXbz2-#7X8#MJyv^;U?N87Q|Zibs>Slh zP+0lBn5x8|vsSLl?ni2BDkfjl$l+yp7IXF*TqB7mqdTlHsFUWk$gMOCv{K`Gkc;A) zDVrLT7dA%n$XEG*DnpHVZp*9XRc%Vh5O-x&(P;4PBPSMv?v;ShG7Pt3>X^bKI zzhnRB2KsP3R9jyeK-_gPky}W-K)h(gR3<0IynCsTSEF*RqR0vejd=(1*t_~9i}f;r zID+MA1~}|VhCP}=#$78)S$1aRGr9ETuzWh}N~Gu8dSGuS@v;FG2NzNqlLk2Dru|0V zz@-6`jenSQdf@AY7dJDJU1Xcc(ipIw#y87UQlD<#jN9su!{l*6F(zKtgFsucnzbNar1w8TV(i@kqMK|_%-=J42P*0Pm_&S@ zof1)g^jB*2eY*h#nWS5B(JucJGT_pHO70I`MyPcNQVCZISR2#z?m^cpzS8q$wA5|4 z6v@9iLoWKSuqBGUlVDocN-KJ|1V6<`n ziq-sIY`_}`id)uB9VU;*ov&9s}H2G-n zD1}j=w-~LSv)>+v&D<_lXF3dx53|mxKISB6Q$yYvL*t#6lxFIDUFH>r2g7zQi(LBw zjl{I@Qf4i7f;?ZlNjuZ&GD>gljq32*e9(w7=j$fEp`*SEexUd@r)w1}%0snJD^wMW zN477jO{wr?s-8VFLt*qbLu_aX1fN~TT-IKRUO8|f^G(ddFw+QuY6FvCSBjko&q@Za zwnTm>pBT!&M#{5c{*WKdx5-A=($gy3@8-5Tr#(~R=-|gKin$(|LSX6I1`SN1KOE}= zG5OX(i$o`UZWzmaA(!OKLSahlK3)BlHPGxa+jF>z zpM&Rh+D|zj5F4+P7C^c#t_uMz21PhD`#V<~r@>c3jAhSgDT@-&M5Utced z-jLs(-OC2J83*dXEy@;beT&aMZp}5TsBloPZr{j(f7^gJ<|ME1JNLCB6Mh+ft!5~>(C=hHrG|3O=4YP=IkEo5%-GWix}e9y8@tE;Lq9xqe~k!Oz)xv& zE$EYHNTNh*)!uSgU3WGr>gh=Oosc;mGOxQ6_h_~H@wsKEqFi~bF zn5l++kav?wtwK}P3u`DHz5hh^+pB-P{ohz*D2`m5(6rn&DReK=jfQZgBKE_w5Y{^4 zpG|{xmbN zsl{!|Ukl8?)EymKZgv#Yoc*dm#p}(nnrhwOy*S+R{Vu=n&c8@JI2wiNG(Km`;T_FO zRS4rEZ`<^TYI0DU-L=f0%TIav{7SLZmd_a$H#(It7WH@H#6L&R7I$)nk3iRI=Ca=z z7ynQ2_^(zgE^N88B!dzR?3+pE-GLfW(4rE`S&HR*0^P|CDRMsgBX$NX0lH&C21 z{aU?F<^C^?<=<>T5BLwVn$i$ASsX%s-iEi)*k+rlr!L`QORdc(>GJTi=%qTVnx ztiyO!0QC5e+CZ5uqAF`Fv5tjSrY=mJYBvO$%?=vD@O+tAb^ga&KK&zK9^U7b@oTDP z2tc1EQ>Fc)Nhw(@_w@BTQzpF3!AR>Y^Fd9}Jrh>{HB@{(tF}6sjaj`3u3sgoO6=M-q7`WVeo?Q4J~pxwsP4uhl+l>>LickT=f2vYEBE=lhK4!C^BER*RAbE;&IZ+rd6D@0yOPUcSBVwnDnmcBykKg?ub=d zCmhUnorf9jDo%~ITv?9&q}4s8m~6^`9EX~XL1LnjvT96M+#l2+tW0Nx9+L9?h5Hnf z9p7N<%q@Q?9TFz5??o$<)_upLeX_OtZC7Wi0^tYBGZOhHatUfln|FFWk?hL)b9^3E zH!S>d#iG(S?4NNuzFA?;5`9ut0t4jR)N3JMpFEVpuF=91$Oxoti`k1gaI4Cmyf8k3 z^bWj$3bq#(L8vL_`xBj5=E}q)=L;e)(!0BPq=#ZZgKx9kFzuuw#~d9Ph9>TlIz3Pn z-T1OjC*mmHa$XTPAW)As8StHdj{M{|xl?Yt{Z<3!MV+K=|6Ok7MarEgKxfAQ4 zThn1Xuo#nZsy|hd_pfe%JN2up0k2Y1CYFnfBxJ5vr4(L{i4L0+1uz!t!x}RmH=vh+ z?hJFbQktACU?o5I3ZL5tH@bZi8(lhi*$(D)dzE2lINZHBD&=b#h#KWpxbpE%&r6jW zyGkPittC6zwsZGP;W4}Hm1Vz90$03Lf1nXfI(BD0JC>cH;hy>@mE#{bps;L1)xby>QB7?!X&p)YLY4 zTznV|N)OR*8nbR2Fh=Dvd*fO?Y8|-G(kP4+k1xBV&^BG8(nw;)^doO#0$6D1jjIwT zv&T3ebH6ZY6q83D#z@WMT)A{xD1_SicCiwQp#D~>-M_8@-I0D*^n-MHo4nR12grBA_SkIM)4!)~r7yRbX&arxlzwkKEW7%N=cRvB1?09XhWvli*^G+2In>jOOyW6L{F{M~ zd`{=%9FoPa%D*t4f4KoG9v*WQhjVc> z+U(5q;)fxOWhMVah8k+r$lgwI^2}v-Qv*6tNu`GM+G)r;mGFx1??o>s!1O1oAaVzZ zwdI#Y^x?AX8>PPwjdaXJX!%8!7~--(XQJq%en2jn``R_SNsONR|rnWJ`D2NyaVQt?ze=Dh{l-+nd-ww2|p7^E-*r z=o&4r**U8^UDuEKmecdH;KAE2p)(Iw-{$PqgN8KGX32AtBVrv{ba`d^PMjJ&=>D|j zaUWZj`{MqhogE=Dj1t2H( z#sLv(M!eK8YB@n)CtWqZ+Lnf>dw(1PQkPypyB#O)YKEB>)|sCh$Ez;W89gn(N6#-% zS<{lWdO$^6yF*euIO4jo{)zdGj$@s)E@61@7PBuVy~}!X1WX`hlw^)PThrCruGsE5 z@*o~f>IyV=)Bw) zkcBQ^CWsoA7nZ_dpP^A$>6b_J5p^M)hQMNIA5a`p6747p`F1jgE`{4r592#hUKCEF zQC?NruL-yDh}$UKMlEa1Q{xGRv5NZQd1=AF((qz1+xmiIRwOx~E=|M1t@H*XWb#E! zzKr{C|AU|7{`nG(CNJ?k)-%L39WaKfGA6`R=M}0-J9k~T3p7>t9!)!1b`-4>dvS(Etynssrsf3# z&#|K8s_3{XblJNaACIb{6R7C)F;umIWF9a@hss^8!=~=ridP;Dr>KSNA>Cqvny9Mt zr05)ZK~X$Q{-~hFW9b8(4bLp4y`O%-v+IP4YdSnMQVhNuc;n$_)I@>p#d+l~x_0Ny z#nbQI=BjDu9-kwZXQtBD&uDQ6wD9~aIzNk!$s$weO<*j~(4sTcoC?GAIZo7F%aD$1 zT`kH!{b|Z!2ZMZIvA}h|O5W zx_!-?SA#PBSPK=zYV4AVFXNdl@(?z=M?90?XsFVvapHPBl~q;(&pH^_;uFwT=D^dq z=ro?tnyz+5x*yimNNB@}Y}YJi+vb5>br`?clst^94&Rpz%ihZ$mFD9uS^k4}S4s*d z9aP5G>_{vFq5BVte?NUn3RTiUwF6%m-mjVE1bszG(b!5`8uH-(?g~&cn4ye`Pyl$f z=rx|Zq!e(hyqP~hPdl(|iVIcZ!iBqZZR_|bM*REK(uhM!Da&}mKxT_-#1n!N%UhI) zC|a2)rqu~5Q9xRi|_HsP^}qKll+s^(;JpSS))q4n2nL! z_J=k-g_0MWiPFkJJL%kcWvouY%PuLc*!|ot{3BHc#fz6R&qr6ik@5sf;(BdZ)d7*`}#Z4@bzc^qRTXN33`#l^oZnH{asghV; zQBBaq!b%jBT23O(2b+m9jfx8{5I;Cp_ktE%UPWvQJ;&O{9~=`Dj7%d@kDIDoZqLze z4DV^s+&GO6Nofp)l2ViG(H#ZA<{kfbo8Z380a~fUAWxyloFbxg7&mb?S~WjP#eTu zEaMrnhUd2dk*9IfY20)gzn~-@(M`wmS|a>_!l?7}Jl+ea;t{@0sM7%)I9{HwGbVx` zC`!iz_kax^xZlWVb85hX6Q3wUf|f2Q$-W!xok#a+Y-9O&I^U(bi9$tm;|I}CpAVdT z{7|bFHME#7h7m6)D}F#(2Q*3IkDqW6|EGldZ#CfYeVl%B=jkW^e)>G`o6h?E)N?pe z-i{5YaFsb^`T-8%dTN|*p)TXYNnGu7nLby1esP319fjN45U24xzM^rLa31FgYKdx` z22`_|^K-eJ*m`7tGV_lc&}&PAz}pGb?R}%)8_)H$>zRxonVcx2hWGe^2va4#yu$W* z`f_imbUXMLxh{KNtwa3EiZ zCLh#87$v=Cfc#NVZz1WK*I|Zof!^=|1Hd$v6At;R3;lkA;gEOoCapTlodP#x zQK?g;2M=}2>6Xaeaa7MG!OQ;lbnw@u(Ir-Rb~~Nj>ZLfn^LlbfZ;t9?_a&XW=kfg_ z%Oubiw)1U|hrQI}7n7PAwsW_2d*AI_f_dA32d~p%JK>85?OI!v4F`%IJTJ8ZwJbjX zi%BwI;D>GWz$4KTB)I9bpFSU!kSE^(O)Xt4W6PHa|4mf23xWj${VhhoTpCrU@;lOR zXG-DGeH_>3-b3U0?sT48yd04X=KoHA!E^o}C(?_7XAS&7b6Z`)8{ulg_6M^MF#^4ssxR<6Zj|So{-DCsn-i4bQjd>Kj3Q&-LDda_K=xrDSWt` z>8ih}s&{?#9yyKEe}Fg2Rp&;OvtFUbHw|T58&U|HIi|K_7?XQY2xCsROGR2p`aVyn z3S+Dk$X=l;e6Tm}{{47jO4HCP>lQ2tNykD zrN$G6>%TD;97X>DOWZ_kZd=KLvhJa%x%01An}V>HynE4*((22T;XiQ&pn&|3rf|NV zt~;|gO3X(pbms&649#;9DIFi6djDp>$M&DFJ4)bMDV>*Lu+E3+E#Fa1)%e;rR;Rh?vwuS| zP2+FzfU=g(C~Nrxlr<4;s}sr^$4Ox`NXUFeLED*NDNDZT{*p*2WLov+bz(1MmaMUs z6*Uyqj$>=@$42QVNa(R*>u$`Hg3T+mxbx1Vu=6WuIgc;DrWWwVKYw^rJb6Rvlvj>Z zzrZ@hlXuWOmw@t$rKJed=lBK{{sQZi9S&pkI%zPtW zY~J|6=B1D`pAm6dhIoBWOx=q2rWI!tc#f2jJ`fZr4FyRpQ{v9vIUur7{L5+%22 zSj|tLgI1^sJ)OLg)w4)bL-pR&d#eXDw(~G3k-V*5&7n~$5k!h3QNl%(U=bxZut4Wb zM3ekb)UdwBJ%x&2NJd3LQ2$!!^P3pnrpj?-R@029yG#4tlM1)Dp%6N^mlyrc>A& zzoM!r6vT@_n>j&uuDSRXe9m#|agAx}g zlmaXnndRk4%<6YHMjrHX#@$g>2hz=CxBRHic&$+o&N$`yeneR6&zubxvJfEJ?dqk~6desZ%uiAm2JgrbNn}&x!3~StusG zK?P-~xisy(1^jy8ohDMkS#DuE!hf84qWl+4>ZNjH)t@XVD@)SFjtUcFJsNs?o(a`LJtUk3}J zTqDlahZ?rik15K`H5Gz}5Z3)fF=nRSQUvvy>942_Sy0Cgpcs=Rwj_xWX(a{96w@%G zh8M$O+%di%#_=Ds&M8pr*!4?D%p&*iALdbbjO94}dcc#AnkEK8f#(`W6l0Ra7^9$M zhY&>`F=1d{$9(mVna>o6Z4fCTSiH2w6zN);v^kT$OY&x4hEx4{II>>+ z!<07k2*ik-ylu_Tt3|QmEt)jjwoPbiZa{A58Lu?Z!nx}fe-kClG^WtN zTa|0JD%b2`oRc6B)M(6*=e&K+6ZO`*^A;ZXN-euLZttmu0>O7lisHj@8@4k$rd;u_ zSCXX|lO@K?=~exiN+{a=%C4gD^D=>Nz3A>@StudB+c06pIwdA8b_nOwXX!Q&6_lE0 zl!Sa9V<_h_GH2i+a642%Z38OhDPybGn)C^(|8L|OZO3>xwJG1tsjI`JHnJqV@t)aG zQc}73fs$!oMsbED&XCq+Jt%mxB%`%5>3MY^Az7MVzkfa>91>EIgk(QRdZ&En5Q=Wn zr&1Uy5{8t$w=PL}-Kl#@M{kqzYG@@6h?Iy+cn)q2kT8&>QOBTafgI|^WjOu|)}gsUf4jGVI{U+|1 z{BG1Ye@z|DYb@b)!}rK;!eJXYq=Z%^QYqWXtM-g{ zq$9iY9~}9-0x6&Pcg_)}rTY(7-D{ipl|S=JbV5P?1O-J9>)304Ae5JySNtFdhk-bJ zFlk-WZKmkbGM76`dY|=_oXUGhQ9`+6);3A-LEI-!!{R-l=pIlc{u|#I0wcwL`;kkN z_O?jmpN1LBic+RdYq|!!Lu=GH(;!3M5sIz^MOT7Xs7(@1a4neAB46BrYk~&X;O-jSEd+N6?(UG_9wfK~4<0PIyGwA_!QF;=ha~4b_qq4` z_5G>t-d$B)tJdCY@9r6TnRO{P_P+sET-tJR!Gin_l z0sDb&VzDQqnER{SS>d5PdZZuXD?$421gEV_^UGb2+A@9?-CZDyBI8)=nB*atD{Ej!`f~JXk&?xm^S4P&X#{3*tUQxrZxF?UXB@ zDdOJ0xGx=Mci-r|jEQ&(>^^&6%ICYf`$?!)E=y4>t#D0T?$B!_&f=Hk+}*T<*laBN zb|3U{s@0-zz}JDn^9lB$a1(NYb$#=6yyIS1EP;LV0;%({D@f}-t0I?it|XkoaYvz` z)TPhaxF2GmM^}4+m)R{MTd#a5BrZMF^L9&EmAVoSYJ4H8ZBn8=2->bPFh0t@C%kwi zux}{q6N*6inUh_3OvbLVMIec5ozv^%drZClZIAUfF_qbnkGwg> zbL8%aE* z1h6TlqkU7zX_H!q$+DJ(tB`Qcg$AE|7gG+z><(%Sy{D-{dZ@okEiuF1+PNZgge4xQ z*FFK^Fu2nxhw7t>GWwv;F9ZoFTas+W#EIacB!(i>6ch46-NlA}?1&}yG|HQg@7iPU zBdv)yiRY@3KhKO1N({AS|Tyl@ZyA}CmJHGOm!)t}^#fU6dg1J6BGAC}Cna_jb&F`2MSNhmE z3>8#e()}H(w=ur2<>GF-C5G81UK4!9CvP!eb$x;_X_jfxQ$8H8L&kI_VK5zubQBx0 z!D8(%DX5mzkLhqo`UJ2DGQ^XIZ1E&#z;pOR}XSP+F z%GjNr<1Y=55CKuEB^fE?&2N(rmw60wYLd~FwK)qG;h@~mjzYrDs8fqQInfwa2YriB zss?!~@T$xdY81-m4`peh#VPV#jq8#z-3TQl?Euma3ymj9qkg1)$uM-4-AhBSqfNW* zu^p_V>#lo1oodKm&!DzpgT*=XY+!dBpkUKM5@L%`uTFGQJvROk?B#7g-Pm_%2Y^TKy*h%(J+31I+XVbAH9*vL^y}B9$h)PzLN-*oX4|%O9W+$ zYEhMl|Azvio1+8~_R(gBM}_%*EN2Fms=WaD+w@UmEE*EPzt3hPXr2@Y1J`^XIItZ1Po`X#~&O&4NXhSMB*pS z`sfFEv96w*Wj4>BEpp|SD{NWogJ1eDWoK?4O5lekf`<$Cx&;vN2#>dMc8Jq1i}ON`uzbw5$v181#Sfa%lk+3QEcFqLRtIZ zb?d9gs8t7Q6SjwjK@rX#mU9dzZ)T>`-Wdb)!P(88FO381sL_sMIxXg?@WmGT{kQga z#IeB?8~yB{_4u>BX-S$~3gU@)v?J0d?xhv6Vi{)I2>6Jpa4xxG+`9_)V;d<+j9BYE z96iJ3g|XM#%WRU@92Y6!EXGJaAyzPl6K0RxX!4QtoR8 zKDW=fzFLSuHaoVdtTE8M^}AW4NoqiS*Y$vFFOU_pUkoiJP}oJRj%=N(tlIx@;c<*JWw5YoF0(+AY>76s2400S&W3sfBBPr3+q>}NkTll z56h>$M61qyf->JK2wEVMoCQOt8zjVL=TV*{#rdYEv0~%SephbB!^CoWYz51l7mSt zpU>Cs(IT8>X=)fP$Ke%@BcrwXB!=t^yWaENn?4B{dgs2$X0%}DC91j~DqrksnpsOq z0oz$U{1v|%3(D1QIonU4DGu{6F4%G{J^jyoW@Z}*lxbDzr$Rq0rXkS28BJ2H3=MZo zL&yHcp(2ePSZ}?pDBjq&0BLWbvO`g)+~XyQ`8|_v^HwvU0*U&2Yr3{G2v$;gOigwE zwQv$3#>#lhn@D9$4enQcQ8;m=oa30fAN5pp7FwQe674O$WclSOrVl|8GO2u@FU!_x zCDgZ0Zs@`gkN1#TV_Uh6cX$NV^3~_CY_TAZ$mbIj;g2Gd*Rx$-m7k4~43$9dN7SqP!q2HH%rL(G#GYZ>Y%NPv?lSJ^51LdD4s z2SKP?Nnd~9JbxXdUby(`)5a|~)Up;L2B}RKY2lQs>?86S1VgO9f5y)D6cTfWM!cbv zWAb`hb35!3l1|M_KvgQt-$R~Tt-Zsmv4bzPqv0UE7hDtr8x^uw2uU!R&m^smSo%Dw z^T#HpTq$HND{09i3XSy$R9*OEn0-6u=;o=y?Zl*zruXjPICX&brm>9ZL-PG8=R!yR zm#Je@nWCRYu!s%5?T-$zrT|vsHaC-Mnp&k3WLQ<3GL{mucp3G!Y=)@$+T@mWIGz*W z)gz+0Djv*@d&L|YaLs1FuWU(k1h2sM;C@6MS79H-Vze&j6qaK92|eV{H>UX{s3h$j zy>^O?+|h8%>E+M{f~VhE!M&4yYx^xZ7eus(?zIMf%@$(Ce5`n0!@W;u0KHq-CSEIpI#z=Qo zGO*qBXr1`{Zd2CLfeOj=hPN#@!lP4?E`T_hHrb%(^|v}L98BSyM}saQvv#d%r7i=$ z3_hG^G+OEG0UC^f2pBfGpD0!%Q{w%=%N+^YTw*q%y_R-1T>_CTZ{9|fdc>F03FU96rFMBSd>GQK z7r3rRoV;2YDt!W~16hLiUl9sFQNXy7%3@P-gE+Xmr>EW~S`&PbgLnDi^G=I_G^b31 z(*~Nn_Xl}+xiqz%~9Q5gPqBWV>F!_Shn6z)8E7I4Y=%ydo6=+Uo z@r+iUf~~8tf@`@wR+dbasB~N;mw5BH>SBSF(qruO=Xq$%rx2hR*%s>cZhE7~J3j<_ zt+0g6pW%0rg7TB+**UA?+#1schyfS7j`ABmC>dmMD(_w`#dc-;D$XZ~osSUcVcOh< zvLE9sV~u%lknZnt06jv_?;!gO+$1qo-bdRjz8;DObZpL%>Ag^KwZ=kBQkd3yG zDLgT>ZXq9J^9a()wedf{k!u^|zi+J$NMDgwMaCJb)W`!QQQ~g`So>Kd=G+mT$htMd zzsm|8e70bsn$>FhMD|7h8@G^)0^PM9((Yy@p^Cci7q2D^SR9?bnLf`@>85H~U3J;@ zDa8pIR(UohV(c6j%l#pHlypwroZ>$Ah>s5tk{S$!n}m5e-YL`K4awaenXG0bTd&d| z1A=Z0$5`c%GH-bG)G*MvDQc5xnrenvO+P#66Az^)^LzXxrGcHK_jFdOCc0)jNMrVz zP)qF#nI%a~dHX^%r7#lT)n)=DE`A@WnviSS1V$E~1d z&B?9yDokfwS`!jALDKo@wW{PFDi}RH_DpYs$Gwt=T?NBzHH?=es5T;GyJ*z0B&dw? z5-^I!-P0LSx*;WbAhm;4#<25vZqf8akui#3I~r?bq%_#ZhH-}HTYwY$Bh1W*A)66q z?)taa$l(%%v?N>@l3$=dPW7^7y`dfzx<}-+`q1Vgn0sL9X6S2g@pKG_L!rOB|736M zOgm4+PSOWl5iZdTs&kQjpUK>(CPauXNh24Uv2<))p@&E8VW#IR`%bMPDeNl@shnl1 zVyk?dnI41LgJ`Bu!@Rnv=zjL#uvVp&udxa!93N|^MJ?+Dnf^PeGTqKkPgFo!8&kie zQDwYZ_R&<)`O5SnWxXwf?~iJzzx_$4{$DGH>gv`TC zcE1r}jXNR4Wcz2OIIb%M9c1*l@h5VHPPy|Mag%vQLX%U?(Y?JJR;UgCSl`T1aMis1 zs9{;TweI*UD=&A&$T%i3XZxduS3U$Kq`ZY}SBU__r7m8M(@}1g%HAS;@#WXg*?sbm z%Dmm@0@H^PLjJpj?Mk#E8EZrx=j5xPl;#J0=!kj+tfLI>Cka-kW%S@^<%HHzVy2_d z;UB~@gbb92f)e83k#&nHb(LlALS*w)9_6VVZ4{6SD`_bew}tW)=^l zQ5`t_{_>wQ7f$!h2;D{n(6Wb~X5}@9OV!muvrMP*a&z_EmMR}*;y{d-9R8w$Z*|3V z2#lc3Wa&&^0leCI)<@}%rDKm<+NcpT-WZE5JvAsyDaD&Er5dF`>F1S1!uP}J$?~|a z^@iTkE)JRHGl{hnWHG53lH2JJx;1TZ2)S2>(1aaOI~5}nRjO>}_0?3^;&Y5yI?l5T zag+w}%>6)qy>$sYuHARUyT#;FzJHECyvLk;igM19=I0iB-m<|96zTWq3MA?7aehb} zeJ854!I|N?jU;Hwnehqd@Tw_uayE9$jCQiVkJVlF$|9-v400&1k91n&c7Dl(xBVXA z^=hwI|5P48;b)reFQouZwH_pUa$z7b=phuv&46jLQmrIpaE$PMC5FbkCkN04NJ2#7e!%LW512(3!R;v}(}XOCdJkKLl%rv%dUMx5 zU{_AY1!)8I3#JNj{RB}BRfMqI&w1Kke>}%xhW&t!GJ%=@+=#{v&|{a`SdMMsx*4xn zK2hC9ZeS`1JAs&kEy5h2578PYJ+{+Rm64i+P3mMUtXSS3-p%8fiuMMJvhHSDUibn&0*H=+q z)Pl8HlH(D3&~Qa+5)(rD>R@chIrx9EAuRSIqfY+07KH%AA;YYT#V0Vv=y((G`x-LpQx9#J zig*;%ztAS*Ucs_jg~>o9zCe4oXmG?xA83IR5#JwKY%#?S(7r17;u889LW76A&2ApYUWa1=@zw)$h321Gi^iT&VKoCN1uD&_xDT+fN zCe@b^Ow7_A&$F0SgUF0S_)etn@t2|(czT==LHnE%zuAKDg1my-50xm6%40j(p!*8~ z%u$GhrQeyD_+PeODEf`#V;GnzhPb@|DfZBEDn0l*+UXw8aGbVcOM*7oH$P7W$MQ-m%4GTVCZkPRC2w$>D*~eFKyEFLn9<6Wwnt`Ve2= zaG_Qsf@%EYnF&|58JBs+F6Ss-@BVt`iw|=B52wroGd7;*+naY&^f%ZbyvX9~vLzpp zhRFyw z+w{Xc#-IGRsrHpDgA~E+*}VRP>$jZDLLeBgT;cr7Q@J)o=^IVdYNWqN(jn?;Arvsu zKfP$AA%)0Po4SZfr!p~wReRV?q@@9-s0W^4S=NhmOo5-o&_P4r0sogP4d^#v`r`kP z^r9!yl;8!miK#uXBN+Y_5b!22=K;z8)6^SCJoNVbw)pnatkb;+B?Ps&F;-6j*>5!g zKGH9yGzey4Duq94)G_aQ3Y29)5PKW_Tr9uHd)E4?V?i=?c;fW>hZ<8RP` zwcqeNSY-a<`NNhDUI2BNg18Ng^kNDLu2QANgc^9E!?y%~<>W;h#mKGNBW`xy67Rf? zF8U*uFV2d4tTow|;eU?vTP=OT6u;u=`9~OkYs%j~PgY>!{!8><6y=ZTQ&Nk{f3TSF z!#?Jod}*qI)Km7M07#9I0r0|9QlW@)p8ngGKyZ(pb4?-!!RD&}Pn3q2&tZ@S9!!*SI z47$O#z41Z%g9sc=Uf{%) zZE3Xk5?NkhNP{Nxx`%iv;amfFx_ykz_e-!@()S&cF|5J$v#Z5n_JYSkc_rDE+ z>p9GU&c^Dt#G_)}>EDLq@Zu%G&Vg6;7YuI4*OyHF;*9^wek9ZcOtn?mZ%Kp#Z@XAV zhV+`uveNH)= zX~3zUh<(gI*@k^wW^C<5?n~n+%sbhJR73rH#j|<}r9Z#^kEbZsRbrJ6hTmAt^s@g~ ze)FDe!(X~3_ho1b$N!yLYtIoEZ;c=I3r?v{wj^iN{;5b{s_g#B)BnbqCOG)PG4;E) z`c328zcjv}WPti7%3hvvT;@j&R*-+3k^W1XIz4-d`ad>&u!ELfM0{?{Wb!66*{ z>D5bB^&c70@*FC-NCLk8r@;CzE`JwTztMeXw-gS$3I5&u^20cQ4}bJfxwi2M`d_yA z|9_!YW|yFW|8MYW$b4ac2nmDvX_1ZNts#)fp9)v^Ce$8LN|HfNsy6H>TO}dwg z5?p10qxNMg?!nT?qCc6S6kISAll?73UTVmH^T6Li3o11VOZ1;A1YD1tx26+W>GgfM zU*x%Tadc94bdq({(JMetzRN^cU zxX(4eO*Ky|vJsjRS(|d2Igj4rx1JBK}V+=5ww}Ff@w*Yib)3$PR-)3*v34p0Q)@z9$)rq&$SDNXt8V~Ga4cmIiF)l za)v$_vHJ7VpgHNInYQ?7+EPNkLEEM$;&@u;P%i>K-7zDP#-APx5Vq_}@Xq@%q@9a> zD%$KdJn-H!i<`ockQcQ}6tznb)sOpWZ;nGo1*zz?jVdFZZ;zNQspPvxiOdOUsR8lz z+{@G7PI8ik|mUocFAAh+>T4Bw9_4{?# zy^%QTwoXcRDHe*aZ*W&o@d^Y@z2+&i7nE zkk4J`qtLos*s`44GP0q6fe1e>E*F8DQPoH(CW7{Q2w^%mEL#w=y~i6%>J45*$4N% zgc@&-Bk7swSFcjV^?#^+PRuqiWLxc``@R`#ao~WI(>|be&z)0(-gJ6naV%c(ip{>$ z_f)vO2T(aL3vqdk`}P8V5S`ClN2>XefA@NNy&KLql1Dq;%%*4e0NxBF+sD&S_3U+X zD$t%-E!O-owz3iY)hR9~8`cN^J~PR9wPSh^Bs#U2<1?&TyZ?0Yte1gkx1&=P;d%gwx2j*a?e;!^;Y&Y=VsdL}l=Slrj`x z@p2Dr#2toB^2qDKrqZYV(;db;uLqhci}qY19A8%8rMps+hfL$&?x9%dNica5+*@zB zWbx#Z_QhU>BtU>Ilo(Xa)v!5~VVsfN$W)NDh!QeyWT8~B0un@C?jrm)g`e;R(jtod zx%2P1|29eFQDRZW(}s)4P|A|4V1*_iB>hhl&LZ9)lSHm?#8cQyFPztye@zuz7ep~H zrfqG)e+M?hpZkI>skko|$~=iI1}9zVJ2P@w(_i49387@MRLHxFj`R^%X!0c7aw(H2 zb0{&XRI3;e*dVFGX=Nm2uw=266T%WCz=J7`lK+p14)b_E60Rwip|(wvN=52R^Tg8= z7D`*&kbofjxK6>Sr{7=;kg%WB?4yK(m5*Qe1dw5zR7ED*dizpN=wuBB7=}-*i}&+N zhwRlBArfb!wGeyH*NiKQQfvpHk00U@uxx~MvYkZiiCz5Dprml#BWp$RIEuEYicQ)S z6w~){6bVx@a%{}Tej#LxT;&GxPUtXN5eZCkB&wU3Ju&zIC_cJ^L@_=(3E4ohXYBx7 z$zfb_*oG|&NL3tRt+1pZuNi}s9p+@2zqAr6x)pt+TM%qTX*J1=Ykg~>O#v4mWh#^JnF-T8Yv>Ez5nhk1|CU-S!k%U7XrS=OD50S2p$`Q;4G0!?wpC;m2mBp*RGLOO9}yWc#0=EN+FGuR7dQ3~-SlaKEhLpI7nxZKRrCu8fak`Vh)mR<_}*VAY2?PZuo z5vp#Y2*?ueF}iSCj!e5c*MQdK@?@h-2u6Vj$YSqv2&hw$VhCEEnNM`tM`Zbb>_##z z)kXu`+<(-q7zotdTe^tqW}&bkJr=d%YrX4kC2UaYJ8ivVeJw0)ogKC2eQv_$ciMNFxKM(r1M6|R zO|1Wx$Pe1s8|Z(=@>sqDZ~I(C;8hjcE^GS8OxP2d!fx+zMLYp=Cbq3mt2`~(@upa7 zzcc8vXsprJpze=2U>nS@?vJ)%ISl@= zi|+2>_;gy|xf+_@LGn|ZpeNFRxweXVs=D(WpV`9k>B0?fsSK{)PEqd()ugd^dxfcm zD)pkGQx4%;FIW!2edN$FwDlD8m^haa<2enk!Gq=ekx|dK*=K^c(%!SVf1mv-_^G8K zWi9eU-J8$QKeC>KNy1mN9FjBPX8b3vF=7(T@Oeph@#AinK7DSUJ~Cu^Rwjc$g~owH zbBjh0JeU`x@StmHKfyO(JO_+dkRP<#1l&4IeU*w4R zWH;U_9b;s6n$0K1Vvt7F54M>W6Xfa8KB0TyG{q6=*0Ys|-1cPE5h>ug1oc;b`e#Do zrpZJ?!6qI~lFX*bbi&pqK28$ihsoS*YOc|dSTx}NMC|hu2fr}iRld0J1M)OX_dZJ$ zTK7J)JT^*xHwzz1emjc-*s`_H67#&t(aH0D<)1UYsQhl82f1mO>3x=HwCR0j1#J05 z=TY#*^AtxX2j573x4^^jG|c5bODx)@MAkoN`<3$8jQQPce2n?+YzkFTXv6zeQO`ph zoosyN`Q3aEWz#T#{i+x=z<#A7cKI|K5%JulK|1xXcb{!te!cUz{c1=^Tt8_<2>f9c zWV?EW?YvN^*RJ5qfr`@}Hoa%HXYcvkS$E&S=bPoFe?K~NJ>qDdTzi(0b}*uAVa|`d zmi8((ohSB^y)2zZkd>6%k(9fdl>3sDoAnL1;~VbkH{6$RxLL{Y&AIrm1un_BS;@H_ z$+@e^xi87NSt+<3Dez;vNH-O{`2;Mul#J-;+ktz?)u$3JH;W<^l+1 zvzxEqlR`hCHshSVA~o=TApQ7DbLy&O)-qy_3=JvAo0lY5KWOcl$wELjG@!sm)yay?n3Y;nm>* zrM@S2hYzU^gB_Lov7%MEi+c&4$*ks5q^`xT#lA=irb)*pVFKX)9$pjqE?o24Ukw<_ zmkPP{JNWv{jHpeLRTzQ>F5|^ogt}x5vJavI;c^%IPtGf1V{0Y}yEony$7WfS0%(+2 zZGMajEr|j;46dvBF92HX+EU%c$f~^_XjOG}%amOr|ug!{T^Hztv zqlt>_z6yFpC3_r7d@o0Uz;(u+%pUmYnpu=}DYmFK?|I0lB)I1}Wp)U5fs~RxHl^Q3 z0NZKlU*mkq9-mUPqTYN7fbUv&Pv+%#y8qj#px00W;JRMkl>q^L&R@3Rf9=T{LO`LG zv*Y2$v{MM;_pHg`Sydc| z)A?HByA7-W>{6#KH6uRSZTjnn{r%GYI02pP53@y514`f2QPYL4ODms}fYWTHHzQ3? zT(%GDX^N#5Zr4h~g2O}5j1xJ!r8=`km5@5rv&A1Zq&-KJS|gY-C)h~r!XVw6?NH!zjzITu(_$ODTj=7^XS$ZzG<3Q!%yHV`EX-wPfF@(Qt?)^tPOPKU$?O~Zueg? z@OOWsA^ey^g;TmfAg0iAsQlPgq$nZ3(GJ44L-XwR!u8OC%GdiQ%-|cbLa<~dN}|$8 zt3o%g)dNQ-g}is2-{HQp$F=2pY@|Vq1bfYM_@~bbZCXhq2PGG_qxynQweF5dikhyC zt!S(H^{vLs`bJImS*=mat2K%7t*AF#pIb9@iQaNIv@7sWd_Q7dTCJ;vYQ6Rb2^9jr zEv4IOejhtEImG>Ow6p&V;QrdU<_*e2@YcHd1a0jnH85X4`Lik-(bu=W!F#WGB0t6G ztMbCyiTI%zhlU{Yk@(>kLXCHom(o9;9`t?qX3X9(!s5Bj`1(%$L|k9kDO-NdUgUQ{`p#WvqPVywHUETgSxyFtMsaw@@E4llZBOMr#QVm zs~awq;z`AN^|?YXe$@4ngdnT96@1u3Gm@Sei^>$EkG!iV9*dC}yMcwP4f-dR^j7}d z@9slc%al@YF|UI>ZtCR$Y0Dn16iA#=$Hy(AG_Qhhdz_TWdbc>tzrrGGuS zJaqc??6w$1TP)vYYhs=o)xeH_vRxSwvb5f9O4RWM+ zXi{LnU%c`wo%@>QjXGkx*hdv}JAFU9eC0*po+@h!yQ&(LU_(h6h6E-b=ybr#-DyHX;#PstLtj*zC-K$Q`{I27B zBv75>*SoaF98_M5wOD%<$dKPkTH_+0Uwn+gO+pp*P%pl@pK^Zgl|b(u_y`0FQLO=1 zmH^s#2R2g?8;V_F*&nwRr^2#_cF?R~*oL0H7VUtIi)-kfJ;JwaKNBUUHm|(GPF%s; zGm+f)NcJTl|D1vbvW=io$vewU{%SHRA+_x!qc8P)@AF}I?g_BUo@LxfWEr0 zqgj?9Fh~!?aD60D6x)6B+RWlrIPL#b=6tEKOnTZ+S3O!;1bggl<}ZxLF;=;UEStI- z3mdJ(aDCLUlTg!6(uK+u(W*x^2D}y;WfRS!Oy}!F!34Xfa{3+DIWaxr*iLW=g2ZNP0d%*Iu_l0OOM6Ow*Lh`UVCS3XRBW3x%8I4a#4s8mT~Rw9LO_4xTY9VU~ABuNgcH@O1J zmRKA1{VB%-l{2p8hd`(}pPQ%Pwe?syr|~761OfLqx0v(j(13*&gij%m9x6eNS zAcG0o%YNlnpN(&M%HAN+qLQ|8D2*!;e#+o{*E86RiYsytAn3e}R0s%AY}C$3<^OI! z`pE>=Uvw|L>FVv*`)?lZ+jyg)v8`75v!$H@Q@57e6;Uzhc4H#c1Ouu+qVx5G3P@u< zjQg#wPc~-eMz|Q(&4@rrQ^wKOVppnLLOl}3aG(?H_be3;clh-i#F1B>-gDsy4WK|7 zymLZOGHko}oTV_`R|tcBtr{_I*Lx>M_>KK6LBvFhvLP%hwgZ0gD!;+8|K>CLQ?7+t z|7UxfulEW5ojd)gO6CZ8UNjG9e5WRRa#{x zWh&fCn)TEl3EJmy*O^GPAX7K_vL*J_J>TSi_Nk z!OBIRBwTe`Yr~iI%XTwt>W=pSnk;iu`x@mzzu%3)=O)nSS_bHNDu5@*nVPEL4*N}= zEai)4j4(5f1mAaaspEt-dv6c7n8QJi`w_2737R`{Wx+u3Uq8GAd#g<*Z#9%?l1XNi zweFafA>L23$5XkWby#32Rc;>WEqCJ2e3HEXV2Dkr`+@vU_`7wjeO+si_ zTgW4LXQ|$_)ge7o+fBKmL2D2zJy?3kvqv@s625dUKq003Ao`}Lv`W=lvqZF5cTaXq zi;h1ZaV#C)l+F&w2Ln@-l4W?8*`2?!7u$!2_?F-&Ac}0WpjSo1;Y@-^;Lb6(uYB#a zzE7&N8x|avwC+fVaXqo5UO!}f*3UlL)L<6wY4JomZ7ryKKC2U%=y`PqfBsPN6uGBK zp;g8=^KnP#)$xM@O8a`QCY?Li_}Zm%IidfruYptfo7Ym{ zMUldgo7ob7Q;*$9!tS0cc3;h`?Lfc*YR)w_isyjjMQu@l5C`Bh_8n;lPR2ZBQ!}Ji zrwPLwSvC^cQ!3@yUK*K^2i|AN{&vcK0uVM2RV&dKosCxiG&EWNL*|P19D%fs*5NfK zX9DqUZ*QV{ee?*G0D6U>H#v3%JbRqJr*#>w=Jt;$vAo&Jidp-y_@&47{V`M`Df^>_ z1!c7^uB!$+w(~W|^|AAUS@Ku+6-!t_&$q&n0lx9ux^Vf0te&j=8aKpel+c$nHd!7v z063p5^~r{4fMcz14e-jRmmB%#v*45yPdkZTgKA|kbBnFpDK712zNg)jA)4Bi2lL^r zt#d2=y{969?`AH(f~$?j_ugnu>~+m{MY=z}p*OUi?++_1X~DxG2pDYNUxUBpo5M0L z>8zp_XDA()!YzTqM9`vY*y4rt5m3!>`hLl6j9 zQ5~0lKvrNIxF~-BdG7VLa0Z56IYZHe#qD5B5 z$EP>?As4+RVe&$MUhhdnUgmcYWx?*qI~o^MW2L)7HFkEOd4o`p;jy)oW=9ZH-qno5 zxr6)le&QGEz}AmCPxLlp4=iDLQ>223GV{ARtH#`;c5?BL*DQJNZ3vdT0o+3V(w5^3Ea{9Of&=^*7}a+Vq2vW! zH0kDnb&PS{yu1V)o=E}yG*dbPcTsoA*!?WNjltRTHBI;TbJD9Qj?ca)E*S35pdHXO zx~?an%dw>~Q&(E*j7UI%NKs(yOV(0X=J0L87FGAd?IrLY%f=J^!Ux`A5Z}2b=%BU9 z`Ep|_FZGDp@L7KVq;$4t8ol0Iu3J9w2-8KcbBl&k_?kn$rFVSdwT~swp+{`@Cmotq z69+lX76D@6QX0N9E}r-ufy2^+i}1z$vnV1)&kYB0W~Gw=fQw+$Gq$oK_=9Q7Do==C zGw~3jpz(YxIyp;Z)2RJfx|HiO(wwzmDBY~@cbL24rI?)){n*b2XS9}|_ef#uto!>3 z0V;BFYvmw$vt{YCP1uGh1aocp6ULK+z8o{16Q=$=-JOeXsl%1YhF445r0$(SW|=%W zo^v5gjHx zoTQnVx@I+gc~$8Y+a9EnRFcNx16McK-79|Gm3$%fhFQImosPG8AUgQswz5$1lh4JV zf2MWzl(@ONJ4SEYZoO4C<6O4Au7u8$G2u35*v8c&_TZ`;G-TYl&#=H@8_VVlYnB}X(3jHhKEPZGmLb`p=dzI_-M z2YIY>GwT{&3I{Z;A4JP(UoXfMwzHe~p(#364Vu{z9nGf$-YSSZEYYO9K&{=R)h}Ic zoKKMoANbSMeL=fd`{Hqs2O@ISj83f>iZLQ!aID@6Pk&qemey+vgznz{koH!FWf&d= zMDcors$6hMFrLn*ruXh-uHDXBs`xHbr?ZF3!rb-K)$KJL#rk>QN$5s3fIsVS9XqPo z-_PGVWmWhd9v&NM8^7gy^}A29%yVh_rrcup0qA=vsuTmWB~gdg{0RKbgQAS%hS87{ zY(R_4y4!ve{~h(@>U6geN4|38A;M_?50YfVS0)}&Y6+J^IjKgLGc|hkx?{9E*#iT= z{P$?S1WevZc9n)sxb3j!%J=0NEw0-#_kZHqMTCm|noJlTA4G-;x5gTs7O=OP^TyE} zNrFM5YOY5tN>}iF?n5i7y{yoev+b{8iL>oAWfbVb5>(IZ#1A*A3#>tebCogc=D89T z^E_$kf{Y;^Ket1r)8dOhxWPjzag)YfRZSc=N*rLA?k#+#R%+Yxx;c54%DHL|ICb<} zMJZkl9A&E%_Z`>y>aeQA*Kxb&d*~i9>YNX$hu)Qn;<)!x{+W!5%*|_HZLW zq5M3N3p6}4RFN8@?2_g6aULE=F}Q%{c>F%iM|?bWHNbW$bf1<0U?OY7bg^Zj%CLdz|?J|eAM~T<%xvokz%{-ytmf zPNSxV&OMvoPwO@?^<)C|7Coai80$M+PmR}!X*ggVKv?4T#(M|Brg9iT-LV$LjqmIM zQ{P-X*~}4560K$-S3dj*O)-dd{Ww)lVFC zZm0=t@wcEjs+l$jHiEfa+e4dr$hCS+W9YZrj+rzG)xWxUnr$QL9a`DQJtgj7&jJC2 z)*(U_WFPzZ;dqy)kq?V<``6tcq8sHWq2^faFe*yQ2$+5RoxK}^+X&R7fbE;B z;MG3zF*)4+{>5NMHJod+G%tIcg#$$kxyI>e{r6&D@S`F>IjZ!*Fdi04VlX*?YUpW@ z%c3IT&Q+;3FMjdTwl@*h$3;EedQ183>iEsS#nhi9LPKe3j9*%;o@YTDrlo(XIQtpg zRQq}Ii|8F;j36&QrPKMdM*--P!CVaAewugiZE*2-%r4A?u_#u)`|}3F$b}6o_(F=R z+Kdu*7xtF)IaauoX)bf?tKxD+n0iZdSj*`E29ycLUow)OG@Vz15B7Q*pRz-^H8)?4 zm@3aqL7<9lqkO61)g_d$$PPxX4S2%8Me&|6rG<6w%$eIg{)LIrW0axW3f`KgbTT7<;!@%eB~q-<<0$?T?}V$I4m9WYTnP9&T)Kx4~g>cXt?IaCdiiXK)?d z-QC^Y<;LCJ-CcH`_xZkWceDR)e#w>esZOOUxw=x_b?TgHgYJ+_vh~m`vA9xP5b}WIen4_SwEJ~~h4Rm4oUnrat9O-rX@J?E$xwG3!MicLC+&5& z=e*R!cXb+>-1||xW8Q52tMQg6)M4?DM0-WO&o86a?f4J24>n!D<%mYpM>R@2Sq~#p z$Kdt?)cDv#q!U{OcWyE9LYS?>s1SqbbJi07Nk=jQ94L{UgmUXw+yeQmMfJWI<3(<3 zNBoID^@x_bE_8_?LKAv;Xt7ryU|3#>RNR}AE7=zHh%@J-5D=y+&KDCzuKGUWNBg_J z-LYKaz=y+bU*3>IMKeWdZXb?Q>Jlg|KRk!rwhjVm6s273ZOve zXcWk-WBsqM!?c2TluHjs>uQ{&t(#Z>l^Cy@W2Mi(RK7CE(yzmuU0-Wqs{`!i9HKk8R}!%oH~IC%#n9fR1pFRWcX zC$TG^hcQ2El9SeJtj3J*18N0EM`(?Ow#%c`$a5uNhFfg?+qamb>T^w{6@Vel5^&Jr z-iOnsd#^P7YCs^O6qx50?&0q6h2n?NcbStw5slu0MnPlHQHl9*OvTf7B#RXS<$fyN zWgk!Nl`E+iLB`d$WgPrN(;j86P}$sP5XLQg38mfA;Q6*|)3qA{dWhlqOV*!pE>(z7 z+irK%Mtiy7sSoaa^WEImSXeqF?xrj0U7J$6N|Ll{MNcXCXyxI9@455{sk3FlevN7w z0=8h2o*7WO<}vW{L5<%5__?IzrCjzP;;&&mt*P?VH7v5n8<5Gfe|bH^)n}}G1{*&9 z3-R9MZdtU3L~5C>-Ju+;_6(S&G9kDg5&PX&i})KMT#?Km`2f9>&Vcy%k-q?(V`C(}mr-)4F&+%Kud>26TYmhKwLB2a_5M~j!NrHQ5n=^Oy zukM>1Rmd2ZZ4v6_^$0c%Y~(7dhGH?m5Y`53c2N{ zzds(_C2zRj6^lc6ifh&x;(g4N=?`>uwgS&EpNX_CU5f1)aIrLjf#NWYm?!I`dNv3S z+}{RK)~4vvO&1WPuof&g$y?oS=f@ual5CcBcSYy>bO_T`I>9W=3-dlQq;sPX0HDn9 zgOX zs3V%g?pm(};FcBi9Su*W)+3HT9%E;3Gs^|X1>tI+ZIjmnkiAvSKEQ}Oa0}_-$Agh9 zAKn7&+OJUSzE4^(5EJa@Ilt)rKb-DX4wSfs$%0l3N~VM<5r@YQVQ>uQ1-uwxo&8`4WU>T?J*9h2ZKa_sJ^1;9};Wc zWFyto`s6KsLu!@Z1@-3yF!ZoOd69ihLeFM0NN>j~h%kPjSbIweW!K-H6q_lShjv!uJs&8;B81o)##5Tr{Y+$9PP)szIOy>(Z+A#%6|EZ5Ir5y9p3v< z-_MwEAO`dg9Z>lWfeQcQM}UcWe#IAYYX?l+hZ_L7N$t?kuoYF~?Bc)X-7Scka_ zEH9#C*(yw*+vK*0RLRD9Qoczt`6OHdOt~ioPsh@_Q7bW&Q~~?A&`}gQ_d0EN8%I+8 z!AC&00)g4Ej_#9 zW8`U4&pUlXtrwBTAyAZp>57JHyxYxqbLJr+y)Ba68yEnNa`+Z2&}@msj=oJD5MmYR zjuL~tEJM=dKa7>B9h0i@%EW5n!D>ZBko27MDh1+9Bfx8j4%a@W2$B5q+w)dp+; zQGz9rJi-{u$2ZbILkn-IOOa17Dqy>`-qyA+bp_HrKYn(d?r`G|fWxETW!jnh3o6LO za72=lMsFKEF<5<&ON07YkW|KhRH~g+?U-g}jrM8meIY@yloP94kKTmz6f!$&l4MlQ z$Ut&yi<5IurTpmN^3|k(LOh=nE_qFf@}Yp(W9vbsiY1tPzxqfXfx#_>g>tSp6Y2;W>_JsHmyZwX_DOSYv$;$fZ&J?06Df=EuTaDwG! z?{I>?sP;n(u^tivJaR1HaUOu_<~1g7jwk>Sg_X8YEd^JKPf|{EU(P;#qlkbt^~hY4 zZDzj^S%%9_?E-_x;ZfMdA_K#LmHU31_AE|r2|^1YHX6oQa^%Zc`X}{$&f&#_E$K)T z4J!APG1BbQo*|) zEP(i@|E~+%nUDD_3{n`go9Z4>jxI%%4P&^}uGZ$N6d> zaSVBjA1P$a!03dKIfrImOyhB*=@@`+HHYuT?MbW`fL3qD=6VQhTU2tva52H@lan9* z76uoiC8Q0Tj!GnigCY)yl%P1b9{i1>c~aG0wSa&fa`ZyCI_HVLU5UAz!{$|1RI6@z#UvW%fI_z4i;?9T`O8m8~ubz zA47>n%E`OAF?FLYSEcW4x*DVF6a$Afyp&&?r@Yw%Vsm89D9^aXbnMC&(H7R{{wWZ4 z+&w5M5)Q96H-PEBC3t}-UoMmgCKj(#-|2>?2c$Q*-W$}`Kn#D*5;~lArshDLXandT zJw*wcryy35o0I{<$a{h4hovbcSxlZ=d>Lti;HEom+l9{m2E|vq`eKdIGk@fkq?ru= zglqA^Can^isWjSbDCFlrgKPqkCgy3Ty)t#0I2`oc2r9HGS7U~`kFx&#jUP*;zGKA8 z#$U(8>&-TIVh?_yR)8V;;1A^eFHjvUqpDtit^n?~oY*-}r&=Px43P-~4?#8SZ}mYn zYu6t0&QoF(ghbU9g%S>cm|74Uuw9gGlqRaz{>#cQ(dG&WZ*H&Lz)3*yJK6vnDV*@EU{+7$M#hlLVQ#@z~kn{7A(_Cm>{1zPyQZ7~!Zg?>XTOI(?c`)M4arX(MH*8I46k@oUBee1p&Tri5p)+3f!C?J3 zg$EL@CvpZ!O5ITRl;i_oD%UQpdj6G*(@6SCor|D5+efV(2jJ@^hQ9z5-u}RKmzPvJ z{<{P6aZ4o$))Kw+L{%jLQl0*2@0Sa@jvwwyNV+w+uvpcjF6qvrB>M5tZbA_Ao!e%0 z22IsYf$j;myBjcx7Z`x*G=t}{ctnD@9{eVZE!2WChqWFhTS0=^X5h<)*QS+ z*B8#IZz0`*RYB&#J|)b6QVYfL=AT-A|Li3&9L^cWn!MZ8SjI-utTv=1Nqd9)Eg(DI zP7aYetRTe_MlPUBwR)nEzy{? zlB${mkHY8Zsk+;j;p+6EljjU3dvf!krXj=xn;3e$korC*)rSnY zKxXJGAo(*FD(W5zWaq-I1&df`*CM93pC&Wb|oc2=5w{F z$2iq(>^lWCvWFnguBx9}Nd^cc&0Y%!-Z z-(M25tsPK=XCVDnukRVw*GIz`!1p#;k0Ab0q~!R_p z@I)f1P6;b1jSU@^b^#p$tF_%a=^$Nv*usgOX4$hAghov5D){)4CCH955FflKU;4}w zEt>RXzv=zf;?J4gLAa-)<7FoYSbe&stlFBtuUGV0$5VBYZ|~Q8$+^}FBxvxa9sZKL zeZ3&T$cCk~I+Vh((gGO{{@oKYcfR?yWh3CiAt|bR!5AZmHnvrrxM+KwzI?Brch|g1 z@=dx{Iq;b!niDa^srj`wKdPI5cB@{~jVC3hjg$_0j#N4f*(bYe{kxY<`vivZs> zw+$)bOs52ec&YjWJ&3|?{-Yio?b(zunN{n8vVzVkdV0MsGiXKqegUaRp?9Kha4)^H zk(h5-JH)$5*Sh3QUz}qzXxSKP20NjwOExJayR7u8lLyz^j&*tc+j{8P(WqP6)))Kf%=5A^Z7Z+~OG12whB-tur9gE!2CazbOAdcA@W$9k0 z1=s<%n`4OgS#V^eV0pQA}M};G*|eQ{^{Z3JQ^Dv}x@P zOJg-uepFTj<~MhmKJpcm@~1R6t$2P4Eh0P{^XqDDT@_-$R3m-+-~3~*zk707Vst7l zFT2mbL_9*nFbXZMQV?6Hb(Ai}tfsg@?=AH7Cb;Tl?vB0ufFEA9ToW(wht6hz7^^?+ zycrWI^LBKtzkHV{*n*&F?8X7ZV~I#Bm>4_CP7kiXecq=}wg?SpUu>pc2L+4wl%vNk z9${nYxj!(rxEtm?>#=SXTD8p7o8s9-uGc|*e@&HY@~r@=*It$pV%rVRJ|upu-@}=5 zvcC>cN*QI6nK|UM32uly_kHhD%`UPLXbNXbO@g%QY#sIlvcUr4J%|J#9jWyBlU9Rv zKGrO^YGqz@Oc~8hCrnTX2w*%B+H>)LwQdF%=61HrB;jc?b<6Mv8ejYeR1 z-oNf{SBU_V1hfI&3SN=j?)-_TYY1^7o+rBg(mE zYIDo{wU7{i%EJ#> zcdE!_7Qrvm?7p9rn9skPB`HpS-Z9@U*E}Z)__eH|*?j~H!35pCR#Vx$8sW=U^J{|c zI#m+3Ua+G_a6|(-`GBZ@JOH2riqTrp;=>g(>u}i<^lgb$m@Zj0UGQ0zv{OkOgNkRs z2J4CCqo9JHr_(vF#juOgEal;b=4-azHYg+nXA)dXL&^ed@%clQdiYm&GmZ|Zq)HeP z9nmb$4tQyU+I|trze9*$m}xY27YF5e->wl45Wl3pM6%5R3(^Snv!#WkwlQH11=Q}{ z4sg|e*`VGhE0R&8H#O*auu%dfk;~O7`FUh#NLFs_45Xwt1KveHiYc1 zd%KbT5OpIp*0KXn&CFuCS*=f=268XZ2n#ef2A;)`(2-Krlf|@yO0Wxzp6#ljP(ha;Z+uy5XSRTrYvp65U{J;xiPt%m z??Je1jM^~z#ge~^H-Whpi0Ug4sU7$abVCT@%ia(tvjM11`Y>{58I9uyjmK~vr&v>a z(bCLRay8B@Z&Le5UYX1yi|jI(jk2oGc1c`ee@TB9YlbE{s=NL?kHd0>O>K|&aILz$ zrLYrP2BOF0sLuUnZa~;*ZIe}{JAC!ru7pEszxUV}>1-bz4|k~KC;vm6a156&CD0ld4|xo5ZZ&L ziWaFv8;>&s7gB7e(wkX-r~rCMVbBj zC9_uRd_;<}pOM{Hy%LZPBMb=7`w7I=<%7R)_Ue*>|GU10A`i+J)&hPtem( zK5(b5(@-|64Ke(f&#PLgWxyjUAT=|ZQ9FeLz%1WDp7r+8JBVc+^6|dR_4~m2&pmW7 z!_UX)O6C=gSysW@DuBEf>4T$zH`HIy&7HU1Z*W>!inHq>ncNG@SlyXUT71-3*G!i# zo`sx&e{o%>!~{ zeNg3hau7Gi-+A2drpkD|w6;EDd`IY4vJQbLwCb(KK!=;n297VH{2Lm}a4nH9Bbdw4 zx=pl{iEaHj#fKRJce|iXYiC}{`@SvGH@*5CXq*QI5OYC+&9ykF2hOhW2sZ*YxiOGz zfo^DHq+@p2_y$dkcy?EtZ8fRBwfM{6k+d2qjL%9_4^1aVr?FPG<%aZ@iR8;val1uT zyGGTLMSD4j>dr&)HYU((U2pwCJF&xJmtI;#2;F_Trh}W^C4VvS=i>wTh)DH^Ss}Ao zRmz@VNQ-ZlvICQH=al{5TK;gKD4b?(+=WibnwHo*f0_h}HBILZd@mR0BHq>8uZ5Eh z+Rl*cBh8u%8n|0cTw`MbCz^b|ktMo#<*I$(MH2Qh2(087bhAR2YyaXAHZut0pR}Zw zJNnMjh`4?|$NQPl0kb=Z7Je1Rf)**9l|_%_`E zf{|HrHheLx;6MVU-c6Szl6QDPnUL4H-xYr+Tc12;6C<)7E?pD5C@|>`0Amekq%l5uw+A_cJIFtX(B&t1+Fu& zbz>^K;od5~eX@S++`iOmi>aZq@f*pLKflZ4lUJ|mT>|2z4sxB-?qD@qXv>{*T7np^F@H|!EPUO{~)?P6#aN1koQsDUQF?OzBdo~XUB8$=UveAulS&U$qEINMZodZu{M_Cps~CdarII*uiF@Va5lOujA87pdcF zOeRlpJbw!Mtoy65BqJ|k2B%80TffjYr|YKX)0n~3q>JQ|(HZ;F3^~=MZ}F?Vkn}n7 zdUru+`9Gt1&(5ASD5H9Q>fd4niwnWK z#sXxfv>okCpDO`<0=XPX0DoVuC_{&a*t@YY7hb2-w5U24tvt}YK-EHm45Ldj+Plso_9ZJDULc&USE%^ zY3xdgR$|&#abtrB-vA@)+Fcbt=%{jD9M&a&@P@&6K`(Nh`4Ac@8eA&Y8X5!nt)f>c z@vTYUG7gy!lnjJ)>^Ed_fv9N!Ynk8X)W{w0REJsbMDB0(ZLPIxKW3f27D*%*`&?At zRRI|~)b(vS*b(zJ5Yyhfh2{Qj#l#JoPsx-JhKIVMPk@;YhaVm6S-nlkmWGgcFg;G0 z9~s45-k%V3-^kkxS#ErXo4(2ixn1EKnfjsOHUgM1aYmjhFAuGYJ1h#`A?{YB8aYzD zD@pz*fD8MmSo~Yh;oV-J*|=4-{>M9>esVvL~5PV;8MxSZKi!(hUd7O1uN z9)C?G%wRbATTPjqcnrGc-*+BROb ztxwOD{QEn(ONDc{uz{Hz?k&dQMAm}&M_0t{)BNf9YfQPc>n0vlPpajB5=nLGeJ>W@ z@RW2m261{PqA{lKhHLUhT|v^SO9^afdH z%)!|9!i9Hk=m_?d%&>{0nMYa(-Q);Tc(?%W1SuLh_^D6bd2{@kh!=$B& z4$(Om);nzJxdeb#_oCJVq~nj_j;+iZb!_p+)2aEyha?h2*t zs{3$WV?{6}dsTiX3@Qyr0WoqEr#4;Nm~irAh&j;1EXQ3#H!Q_65m*RqQ)qaUBoh_? zU%CsDUYtum?m;R0uq<31A)<&)RXixTbjY0r9T46F0F znfE7ov3r-s()V&A4}P@>F11`PaWwci z2l|oP9^psbG;&fTBW)WE7N`Ux_8sIdW4{6|KQjAb?Vdjl>05uaOpm;zPqYE*CVEgkDXX4_f=9$^>?nLsy+X^w3yL3vfRQp53^5drxe$<&m*&b8*9@+PP-swAkhqe!GObDVyLDgVO7fPAb zE!`l0mW8k|w=9Qm;pW1p$>*$ox2GV}cUSSM@5%69$U)dV?Hgt@b~8*j6t411)jG{veO7J?nQRN zq2CHa7{BhSR~&r&VY#7ZfS?nCz{7f^{04AXHe2w^$>pA(5xGzJuRh1!tp`smREQs1 z+4C>2mt1-4woGu}`$KQFO$J==D6>7A2Zof*)urC~2y~Gi-Mrk=Ok}SSyUzA>nPAVa zWjRkQK=QO&t22V?X{GkQd1whnD^?n!a%t{_t zKD%%U+^J!awD6^03t{?T?b&u^rnqU?hnL0R^&!iH3|CK=aedR9AFJ0v<$Qwo4U1ZP ztqpYuRh-_}TmzBNo4cX~pod3gs=qcv4ZMCn=4Y^~6P!gx$P|^_N;Rawc4YU<`9k05 zf{Cq(+wD!@?(=0pv25>mrH6>dg!oahuYrQPghQ3fqwZo=U27PhAcmC&ti@RNECVNL z&+y$A;twc!PtvFEFou4h3H~B>fJZGE7>V57>Xh1oy^Z+0q%vEsIEx@ql~eT}_qn;rUYexd_o(w| zr#5EeHa=ux8CWKbpsA!^ECH4b%)$x7Gx|Tt));6^!Xg+WrJ|i00vd5EDj9E5j*~Ogt`M;BVVIWc}yqNrMRkV2Y-C9Fk+ZVoxgj5Y$*WG+zgtw5fuff>01Cr}T)bf--;^VnH&+3T+$Z{=G>H)`n$ zu!K*M)Zb$JJ$jj@dK%bnoM?N7G2fD*?H@xI8*2Uqhpka*W+!Q(R9_E1@0^_J{ACju zV}+l1(=6+#Z7{JUVnnCMVB>gusbcHB&QOueKF7lnGfpDZ<~RTT?&KPDO6Q~I!t7Ns zh(<>Ie(BdDt0#_uykX zTKxG7zfUq|ykpPJ6w3|nYyT(*r0@61t(SH)PfuaP#_#012$LR8_ACvJRjM%C)vnZP z(}x~C$gvU&?4PNJpbrD+GOvMi2=cNzL{)r^l+I7~Y2$ddmSdR@aC5A9bELSakpzmw z%j%ZATJ20l<~t{ONSLEWP~Cb<&RrnujJDSj;~&j`kIVqde0$EwUe!B$5Td|zKdeX6 z>32K(V|lP(sULT2H~D6V!my)ei4Hz{%suwn8ODLZjfm`ypHG(BnioSCqry;C*6obz zEe$Px0k}%Zq!R#1qU4eE_`0GCib zy3cpiM)V^bPxEx@Q;iM70NMbiW8|Wj7Y&(2@~kf{26T)r?-wv1na#u8CN7M5@opJa zQFdG#;v7TO*LCBiqwi7nOP`Kn-jYWK^}RQr*6;(EMy0V8UlZDJ>6#bHtoAiPo0@Qg zU-&MECsD=KJ2MwU>cP^@yOq{htP%B1t$UT^O<~1a@Y{KQETR{dsR`6@cJVsKYUujP z^UhZ6Pwn6hKtxG{#)F?MSNF9e=tkqS7KY5fALF^BEw@v?_0! zE)CdIz;6+QA^tA=WSeZ0cHuATGN?z^rPmoTq*xXeJLuo9i;awi>msCFw$|lJfBP6E zQm0EcVQGXwnhL8>YnMdRDJ!lZ*~;%_8(}FaX#c_)ZF9f?ZvjTbgszhEUNGe~@xWgH zsea`kGBtAkLN#wSU|16!AdMAfj(jGHv>+Yx-1$zJYV*TMbMc(sF$fkc4KQA!;}EqU z0vh-!D)RQWCf-ag^!Ib&x*q>V!_N^WIDcZM1|JG&PdA(ThmupQ%g6;ljI=kYSyk5E#hPiGJEkqc+tqaD=*W^mv{Rf7GnQw6zCxXo7U&&(AosiV11&OzYylh6n7vHnV#AbM3<~z<{O* zo)rCTn--`>=ODAulHzfq36XTAoO!bb;c}7-sk$L-2g<;5s{rI;3h-K5qx}HW2VHN!WSo%}|W!k@X>%IUIt9RQba17vfsAOvVD!Ta% z$??#m3vdC(-+$v-^CL*ZoetPY{bKct=d#yrCbzf1M7W)0MgWJyGa6_)cG)Fr_jj0< z>b56xKE;K-Q6v200}Z(@@@H-*lVd1bJOCn4v~4eLsima{)AZ^1n`1&hb5Ea{eGGjF zCjZOq(f2-y=Qvhi^_!Fl3Em^ln2Y3 zyJkH*z!O(7e}UVX#u}X_Gq`oKuX~4}U~aL-+wdfcmQMAPDu6g~tOGVqGN&~9pf-z> z+U;-UGK+xc`=EzN*FuD%iOTyc>ftF6N;^s%Jm5nAZ{oh-a(c!9# zZba?@c#HFtRZSVjzpWLqtiJK7le4}}g@YQ7QX0pRrni0=di3$zV%o+90!{$Q)H|VU z3Q+K03GMtDnr&4cLsbsF=!4w@sSR{ohI`N2ii!((NA4 z;t$BRn-%cE9~5KtvM5Xyc5`Gysk-cC+jvcf!?xM;PSMKJf6P_x4Ttjmsml_N=g1?1*JmKZcgA1&}7d#?~Olz$}+ky28Q0XqAW|Lab~C_ z!LTo)9zizt)UWiAk>_eUY~g*oVWM?lqQ1RqtjPF{K?+zRHQYD{<@S?#TsKj$WE9V2s8F%cr}21MDWu`qRyze*V?bn+oAz0MN6agE{j@MwvC{Dk6sHanTwN`v?J!l z+}($pCeQFc7mXJllmYi7?Qw%P>I{Z$WDv?{?zK8VQ8h!A4D!|=S7LSwQj|3y)wrLn zsTc?wbtISTH{vZyr7!MbGenLiRCwb_RUQO^g0~77=EQSW#MzDdom~dpL2gz2PuF#k z7@Dh{pI8$|p?bURc#}3A9fot>_!;}KEAqDTVnyh3?)OL6dn#PmE#IGjaoN@e4*X3* zhd2?a8PN$yo`CTy-L~2#g|yF)8<~>_Jg>(5BNxuTOvxpSlr}0gG>vel2@zrT;m%)_ zA^>&hdP|~k?#u%SP2MUUbbQ$j)v9!KoDBzbl^(0SkY$Xo5%|O-=M@`h8nk0Z4MZ#sg5 z@Ir?CJocu-gt?15MpB^$(EIUDiG4QWqHRXSn*H-NzxUDe!ZQ>Dor-UEof@waB=o=Y z%k=l5O;?i#voG&7KSQvWtKWcbKQ+0jWfKf5D092o*60-|+mFZzxNTA)vt4ASjZ%s; z1ZAuf9VM@i-QlZd{;C%Wnst&-fWEyh$B+`_lL%EGng{K0^WSbp6O$#YZr(B51K@Y^ zM4^zRV%HRjL1ACd)aJW^zitlRqCKTqoGT^f7}U5xVDg^vtTX=6h{|0fRS3mYdDhd$ z<1)69U){J&U%+uNER}uzFXcB&->|MGv-FZVwK>GH1AtDBa1ayESN9WPdtUrM<6+nek<3jyko^5=iB5$lH(n%$;O}yA<1SHXCe7 z+U;yPVXze(vbFNiAPoQ7rs5|{DrR6{JJGiD;Rs8$*v+ zx3%NoG53L?rkPo3Y7FmEPZgF13gym^(Oo~o6U44(TE%SQSw&RyY7rv_qV}qTNUn|g znL9ioTaUD1^U95?3x1XF2URbC=mbocT7KYV9y_#4DA-_IUk3W-U$5)brY3xOfm0I zFd~{RH-tF6dKEfS#Z;?zFCYchEhh%0CmYZoYZU6!6~_g->u0}M*EFt5o$;Ig)(2e{ zZgL;(=Zl*Jiu#O9tMizTO`weR476R|87560WG89Mr2UfIDv!OyH*DZn=S`BgDL~F_kF)AZ7cz4#cqTr-4cA#kf)8VeRqac>BjEdhGf<)lY4D z<~Eqc>Cdziu&UeUU%gF5&}&q}U1j+fw-Pqa734}e>is6G*LSMY+}2?AROVvzw_h4l zTs_(VoLBqoK-e+|-o9^=R7oU#8h)uIwpG>}U#JX~A1;qtG!mo)PIXx)2Mm@p?8Z;S zON3hf13)-e$~A8ktj`PPWX=9VkM^FQd|RuvRk1HNMqN0QCM2Mn;=bKK?a~qF`e+q3 zzo3(jmb6bSTZ!8nw96Y-Rhe{Qkr6Ns^h_ucV0c*W;8ReRwt7}9Ap5OFm?)N%xf)W- z?F)r2S)(|=N%?BF!kv%w@Z9ISZ~sWzBQZ5d!r7rjbh6(jLgYE;lBo6kjsiEP$4Ad6 z>%Z1BV&v7z0{$9`UDz~?x}2=Dw)H`lypab_xJy(0mep>01cWKV>+FQVxu?wk<#Q?vQg^m z5hjqqlSw}mHpG#n-oR7_-efAe+{-^qk!z&4fGfDZCO!$o4)qQ_g>5-2cxWz%dy2b! zgNx0KgS9==1gf}Gk$s*=6M)`;-T<%Y5k%0QB>>tED7tKD&A>0FogXs(L@1F;)=Hfp;nXLNC-TW%fpyYD>=Ji!(XB>S@$jPa#!zhVspX8^rx9sCHY?PNS z7q-j_+F^6Cd&FLkSj;4esELlDQfomD zd9|x+^FH~ik5@z>X48b>YdsyAXmN#ye( zvk~Dl=cQ}{H^D$3HU*92%hIW6poiI9S~=3_FY5yc^2ETQvdouZVaIM(kqpW$yu zAu-V|J@3bIzWs(L^ctV?%leF#blIT#?8R6zZ9SZfzlV3o#lmxd_YkuUn)dU?oW`Kf zr~UZr_h+5pz(@j0A^&3=V&r49eqbq9*u%10yj*C)5Ye=Y`J6)kjTy!te8?FVNz8A& zUn6#OM^3R%EYnZ?7=M#0GML%u#676wUv<6~r|L&hpys_!Tg6E|p+k3QW761C(!dVm zbLJ`4{1b9%Z}5|lA*af^3@>R0P*1O+v)xvlMs#&VIZh+|kTvRNLJR{owwCX;i4t=q zo3(PIzaKfDGepfoX&RED@G8E0j&X8Su4XaiNk7S?0M6%Fzflq@A<0kijY%>LxWieG zQ#$(moUJBvk{3cK?hU^tUaeQoVzW}t>;5R6>0G~wpg9?3cN#U;avAi}icCPGYK-ON z^1xohY}drzeMrWESEo+$KA76P-PbPZf?WXatkF=~*nmhFT=gQ+Ht2H=_(c}cbr8lW z3r(O9o=LSi{%VHo$kRdos4?2Mu5NmvblbPz=If(Bk>=AqW5eiQu0we5J(P~z&BB%m zi?U6Dj*nGmztcC<{dv3~*a`U@0Sac7(bRoo*Y)f?UO`}>Phu4CLiep_o478-_5qLb zV0QEiTqo+Q|C_|YIJ!+FAR?siD#94v0NTV6f30R%2=-AX^39&*TPho&Vy?4m4}AU2 zZt%wb&n1Je#iBI*lOz};krjzO8?w>J@P_u-(kK~u+%cc!0%7p}@5Dq^2NR1BhZBRx zl}RZ}QJ2wK!vx$nHJSXXMyEa)BCO-zX$GV?YdLzWNsfDE*8=&*8A6LHR1Td!y{%z( zi-=BGATx&w?$U-@uBN<=&~51iDnz{ft3$%PPnxv-BxeR!Nz*r0E%L8iT`(c~0wnl($3}%dmai^ai9K?e&(?A)>H@^eYnKw-dYWfbRARDep{m_huZ7D*iK!I{|AzpF;hZQLUN?NqA|ae>qbH{FGdGXNCyC8 z5eJ#UKNQiMzIw+pBmYZ1QoR=^>_0I6H^Aij*uNnE7ay&^IdT5d_(f}g@E@Y+j|h)A zY>*t06w}+<*r^Sw=#Pm1p-3DW*M}7Z&o8<<#K^foSM-d+3pcnMMi^>$Pu$B(`hvX( zZ=SUbZ?Q7iIr8lLa@09Xc*ZrdkBELDDU6dBJCvJapwA}{4#AB4Chp;c{wD5`b;4%i zF6Ux;Kqu$w7m!*+vy}tlXQ#-pm*X_M*l)+>FZorfX+Pc3&UH^ZS&rxQ{BF+owTjkU zDtR=HGtx3JX;L!_K}Gm9i|P4*BH{*RdUR2bZrpjovwNFlesj9ZM#iirf^_{2~pHNBKvf5M2X`dJa+Qw#;ld9^R5(~?zJK+S? z{rMGh2ACkbePC!jH@ejvNFOM`VW++CIh&-lY*1>>x74dhs6Kk)xn~gW?yUI zFT=X<0%F^HXT1ipw2$-u$oprdkTW4))WGo4ytiSSn=RULjSXvsrPfl9i;-n*<>s$S ziqolC6bP=abg&LK-WdbcM-z+Mzt8=@C225gtGkmD?bz;%;jj2$xgRG92jxGmF1oN;{pAkq(35c|ci+EsA4l{KreI z=BE;h@~f4}Zk9=v=8Nus&GWC9W*5-^YuNu8`+vz)-LhghZ{vT<$7X)dzkcGk)c!(K z`~Tr5!IW-Zexmh0pPkHtK^IaO! zJra!2+(=SCp>hF|98w#x}MMxfl8VWri^)I@U z_aDTj#6tz+B7YGYzdfMY|I5JtpgZ~VAIAUX@#yiuYTs%fci?~ce~m=v@D~F*gQ~r1 z$Q+XEn+LtxWDHO9+NCF3L($=#JpnJ=u;}X2k0B?Z~3ORP0`D;ieu(52|2 zv)USYJp?a}0KEUj-d8|H`LF8&0*W*!oq}{ocej*;bVzr1gCMDdARQtt-Cfe%-7s`_ zGj{-W>wo|EK6~$V_PyuabJojx*UWsue8a=@K5zWKSy0(JzMh&>^FlaWd0`QJ$oz8m z>tLgKA60#ea9%R40SlwA8=3{GvbZ1v;qnk9&a*qWJw>cc=bWPv|1)<&n-C zpaVl4r}T($*#7eHW12R2P|Ujp5eP=#!yJgF*l&!%&@l7!S1rh+ z#Dnlf7M5PP;`oI;*ta6)u1i0`9zxi_8bR1pXsT+pgEti^Lvt}HEdTC2fG72@J`<@0 z)s~K{{_5&WV^u(J*dvwsv26v5p9}WupUnQ zVZZr&HdfCh)-Lh&Z8RGBl$U2~C-m!E|9&OaR41d_7Sumn{VHfh%i{>8)_FfqMK$%& zsvGhG34TyS>l_rpM2GrHc^5Wmhx&g-j?BtSFTH2K#?1S@n_vZiJ>bt20(4e38W^6XSZi?az5K*F+wQZ|)?MDfP*KHvaD} zsc2jzNZq6%812%qFyy<7XyKmYnPi7zpN?y~F5ms|dz@qObF-OddFW21Upjw@I;~8CA%)Ns zlNb4`Qo{`)Il9TLs7!OlPbpcrlE)zF#KH8OVsBVnUL^MA;PA)v*jwpDDk)@CuCbV_ zT$Zl9s(h=6zDuVP&?UJ2!FB$Ap_jnNR}4@usdV3-0{<@_|~mPx*%Kn!fSVb zHa)Cg!Nb+51_GVn9q8_0_G2%K3o2OuW>R2P>s&MlP#{P>a`nWtRCmOUGCxUoN%0T> zL%Y;2r2Xl3$7aAdvhH}PCvi@dSsZx&$boP!3DQ&3CYcX#DtCaYWt)xr6F>nKbOb?S z>SpdS92EPl4(Ict;Eg**@wkHE{-Sa#o3I#P;~#l6KKB-{(*ygb9--}~LAfG$8~EBmKAO@aVQc%gJ*Tz&1NP*i{9*fOG* z>lSdQy7BeYo@9bZ?pCo&2>c2jkr+y-$bhvwVScCUf9xfcqz4{$R?th$Zv#X>|AS_UY{lm)8%zr*lLz zC%oO`_JX~vAVd0T zu6M~BHr>5VB~9IBcQ&Bp?Yp_MJ!EUM@{ab74!X0=uF7_h`GYLD9?LVhp>6EJCWtV5 zM`g}f;qZTzO(ot{!lg}BBL#U@>JO&EKD6|ra(_vwKK`M<*}fHyJAD(ejbQO}vWaGC zbHYw@NOKH6<}r8rdnf|YQjl{%@b${PMexnao<;FpkSpCfIc&RHBKis?$#hFhh_QkF z3ZPh(oCPOoX884%m@fL|{mVdEAF5=UVKoV;MUN&~Ye*r?0g8Jc(ybXwQ?l?t#FUXP z2$cwmd!lk;u_}Y|;UHoKbL-rg`7A>04j%t1Ckfs}wVK@2s*MWc(0+5O%3J9z<|@BCJmX||8+w8!!?1Pe)_9xQrS|tz4ED3 zi{wZM4T3qH zYMnxk-(58bO(NdWU%N#lMv$VTdAmjMJMs0;RtfY8T60tG)zWP}={A)+Q$a$liw5!2 zZLqr+OEGov>z6{kg72?5Zz<}ZVkEg5rQ5ndUA%YvQpK$b7rL|KWi!}$vuX^>Kr%Fpz&}BrbK{KV}>ObA}WqDL-elA;U zoOcadXIxVaTX!627;>;$mb2|*j@#&7cH7jjJaoM!p!1c_7Qq)Q1-Ht(!UEyyHo?2> zX`I8k?zyW9JeCI!On&+E?0@+^wN&J|wl_zcWT zO5}Tli$QZqZNbwlI?;uQ1!?fLIZ!@_r?lnuyGw3yf7eftKH5*1{`H!Q5QL}_0rbu} zJrJVq>`&#=Ex_)JO6y%Qb((83(yv{8-4?oS26y`~q-b0q zh^yp>qrTq3P}!0}a{;Ze`{|Y``bNh_ah7`cW^tBpOO`7=Rg0(!wQS>YDYa}M z{rxxO(w_%9vr%#Rhp{jFhyU+pbAP$cACFR~K%c?gXYvZFw;)j48x{r~Qc1JA z+%eF`OIm?fk@I~|$O_*#hNMe))juD3eD%rW5(Q{I^3F5Liu|_jnVI*`k(ii+*VR?? z!hA^dc?=o{Tc7pPWFp5j|C+SGsW8wNjEeaVZs4mEo`$8-+Jj$Z3-qDi zaUKX`%=4T&eix>o29c!Z$iNIv9iM#D(KSaEJveUFj$;ox-w5&T+A4 zLypfYzngHL&Oc2OtxQh-n()kdZ<~Ig)bJl{`8}ljVQo=1p*A=yAa!<^82VS%#RuIQ zC}Z3_%LZS2OdH;V>*gwhavvi;D+kZccU4JP5iwCL48#sNNP`wAJ zGv?bO$nRzN;kn0G0}-0{&e7x%-ZQ0J+hqm4Dw+i#hfh9 z=0k~>{Ap5EmeH)Eq`wfVmSE3fpP;-L$EIX-K|Ccu;sf} z;DrBoyP3Z472cL2?ro`yt(`3x8`R>t6aUR>n!Q4|2FeQT?sQv~Rsx#ppK#d!ta$pj zz<64ISE{%VvY<+q$`+`z^1GwyE#_R<%Khu!v?TkJGomN2=R>ZVqt41@yF{*De|nu) zg|ru#-KDboPr$C0i>+^#w5u;HDJ+SBXGoiLbwaeaIPACZdMlFt9DppKyy0%l;nqV3 z!GDj(Ag=s*07zAV5PKJ<|GT_=$CLk`2HBrd_h%6JSLDI2#rJuMpM?2W-GRSFn4MS= zY-Il_a}}-!%2?93g}9DV?>P@JC`(4iGjSG|q1u6ZLa;wzv34%#K2X1F&8g3V+=ofP znhWdalg*MwmZZU~YYhmOk{hF#xIkZR~CIUJIzvblVlb>v{`zA1zDF z(OQ#ePoKKo<|_Z0bvSgov3Rp9sB^zW3_3w!ZY2s=m7Vz#m&E zn$-4IiOVns7I$$MG~V;56Xhxf;&~+9*|tj-sC|O__-a4k0tMJE?bo7+X2HtxWwhdGdxi+7H<)Y`vNep%9iIL)jTIGYac{! zvFG8|wFYS3BDJOWl;1mga;;n9?Vb-eqBgheU`6i?=pF|E)fIznPZ$`P;E=j-9?}1> zlWobBiF z)`|b+Pdssn{aw7>H?E#i*(X|7ftu`{gQ}9#pkXlQ7|&k^Eq=*_E&Pn3ZtRC-=R?k? zqRvF*R)}sQG24=f8&+&jd7JnaWf_2sVkatkCSy}ZHiq;Pnl`ALMR;qV0DEuecik*z?RzWZDZguXt=>*0-rJcqf_3l`G$Z7djx$|InnY&;g;_}Gq1j%sTz~W zUuk1{7eVVG{J>!pn^+-c0JG#NzTQODBYe-pj(9BURfh8I6S0PXeTee7-tTv zKIGGbbfRWf+xRP{zo+E>7jnNw=6`RTp173z1LXd?f${es*NJy!pQ-RB#;0DA-L)A% zITPZ0pJVblZ@{PQ8^ZagiczI`ptbAo;n6^9bKg_Wxl5y$kK{P~h*p+J40ht5;a211qqKDhFQGV$^|N+ILHjWp`M*-` z|Di@a&5*l~n73oNcXhq2mPD1UU)|r9a-BF4Tx8y_(2^y-&9z2xFIt2nK;7JTHzq(V z_ZoPeO_8l;M>wyS*Qo+p%y9dg{$y_Q^aQ}^0`M{}WMJYsFLqC`nI16HH0GJ3jjr(D-#0 ztE(M-zt-=5f!L3Zhs9m1O(W2QUHjI!cd&Q#Q+F&^%E+w?#rG&o;^i6f(JQKrdu3fW zFR8Fzpc2+3_Z{s=J^GWG%ztYo1&^xiPf8yD14<@;BWCYPr~RSB_S@2m+w}?mV8Qrm z`;~gDkXJnyr~|TuIs@Xw($tIcSK?mvUyA4zp*+XLuWr}H@Pg`p-`6kPuT0=3|3XN{ zBNqIi0=3y=5ALk&``qu{AUXDJsqSa0d+Guj^}SnyaLSFf51H@;XnnV4YT*z0FQ~u= zszCgbApdyy55DS{455qMFZWBqZLq(W~95Ds`B%Cd4=n;{Dk+|u275-Hkz zDTF%h2C-WU+iK%9-TB;txU&={oJ7L&tT7nAZ9I5kpD5IH z=7@ohsEu`N7W2o_5O2#p=cqqBz>d!p@xgg79g7#~h)Atk-T#l@`inv$Yv_>s6|{F= z-rp;}aS85jWcF^*5mcpbT!Pd_M2SM zkbyNEhCnOz6r={-1(s|W0=m?bml_}qyEX)IQsT)e$%U9X1hG=$$x027gf$$37%lN+ zq@I_6B^!chE%AIo4G@Q28=PH#TR4holJLVB8!aD(I{E&1dbh#YXy!0<>zPh2!zlX? zH$3=tl`y19Y4d$)?{^1RTC%cBc)P=e<(O`{wyg&#%ZG9Mh`En#FfpuZvzxJmuwl98 z2g|IZ8jmioyJnqUWLGYN?$d62j_01W>}<1Y~Mk~7$el^d^O6WOytWn5fDPniUoq%I<%q%11Jqs%URtGs&| zn>hAfk4_;{TaQp7QcDkcRMq8p_RM$~+faZ1b)<&g!s|$Ny@3&zrK#1C&+#EP?C##3 zR3)AC*;GZ&CfT#x`KaWzkvskr@J!uF$Kw{7;I9-?}-7`nqe3R?ovWTi+st z2~soAQgsd;j*9wu0req{Otjv-*no|zD_9qo)&oEsDjLifmoo<>#2mAo^IYhhV}|By zbsf|@q&sI^jy1;a_0}-Bq?qnMO|V)i8C~#5yS}oh-S>EO2K)@RqUFAT$N;5J+6VUt zhyx85oXjH})zC9E(1Gl7)-#7>B`eEH9`_d0ppK&RK~3I#>K98{12&sh%pAjbc}%H5 z8UwVY<@Ut(Y1=oMi&S;)TN2wIahn1f8#>p;t6z6JDlf0&IU4~}v1pvHO%9$1t&-;h zuGKh)H@I7-NDXFN1gP-7tW&yU1hx(IoP;I|V|;Vtato}OQFkY&e5yq4;s7}D&U z^S8$S7PPs_!2o#BXUoIGph->Zjn8n*K&cK5JFYQH`GL%5o-Ma##RKdiF*a*)_hoip z{#F`W?ecQpW)9r^d%jE_|Eq^et*$VoH$C`4?}`!fd*5JEQi{s>rq>L2}dUGcB z&`uAyEFIZurYCXII!)!^Fjiutqn9mwgSsirt7|tIsp0d*$ z`!G&vJXqbvz+{JftJq3Pg!o8GL%O{JfV(M8ujc*?pWpyo{|4C2P5IP(8FBBMjJIqz zoV_+ZlSQmh6BQ3WR{Bi`F}IB{5blOV|E+M5%ejhUv)|me^CpMm0jLeT*hE)^)a6rR z14D?2to9I!y*-=S*-;-Ut#9d}@LC&99iJkI1EwU&@JOb?Q9pVUkFbmN?7OeP+1k(z zVyDNa&Whsk`E&br+29=n!0W{F5f$hl=;@b^4fT;27OW7BeAX(PYr|>?nxKnFk%#6H z#;T1TKZ)>9m0qQRV=pm{=Y8}rbsnF6j}vhWTXWrw8Xo|t+;ns8c~06NHq7;nWgi*K zLoNk>whIW8I|Y0S#b{uz-_r^?nrCA`PSq*OaJ7OG(bE&Nu)=7C8eH|2EW9~9TreFA znsNmic12!CtwHqYISO+?bG}AUx?C*7gT_aN{>lCx9d9G*Zt@y+s;~=G;i@dc;t1Od zsLWE-Ug2YssFiJ~V8EK@PSQOktHWpuzJQzLy{8~vXR?js1aoWG6r0FqQikRO!ko4 z8|ng-==a?w1?(ZEPLbh;%z%%L+dNF zr9tmbOAQaPlARTEBE?@ytq4y*^l0jsWx-$iLHOJH<%%$#Xi5kyC`uRslr`D$z}gQv zuR2o=7BiCoDdrtk6CYW!pQDR(9oJI3O+@0Vo=9lY!&E&OKS;lH8oI8@|3YFxj$~%P zCQm1aDOC0pl7%B=i?isky*K}29!BuMN@UPVhxOThyC}tuD9KK_z}^To!(#pr|5qQu z$Lk+~JQ5x(us!Cn&2gxmJ@tU_LzLIz6SlE@sElk{x}m&w0|ga^uT&YysIlbH;2#G; z+X+2jdGqSEph$XM>+o3g1=h{$rb6QiZZLZoQf)%R_itycDjtlIaB_8Y7k_Urc- ztrm;3@i@J}5IAo*c3h&0U`hyhl=Jw(`m?e-W{Gb90Ba{6vx-0-$g5qXP?}>N16$Dd zs|7G@Nz`rSSWoQqlxK~M57W(O;g826;&kT+&7kV^ZsT){kzc=kmM@pkJV>d&5|nWT z*gCx|MksN4GC(t!8XYmRXczi-kvNWb*xvDyD6Um*PM-&1Yz>%N_q$DG5u+2ffCOPSx;h5Ik}@B$DYhfOXG09J7;MX3(Z zKLDhI09^3yqY}7fu6jfsHGP~{zS}&+)U;Pt1Ht85ZO9&JYLGV_DETqu`3aN_EJB*Y zsrBcfrleR0&)hIaYUb5^`DS{WXVo!g>pkDTC?myuk*e{yQKKJV#%b+N2-1qp((w5CRD5(f{hov=S4yG1cfPhDCjwG-h0{}$C z&EtAC)jF?rRwpD7_a#{}rZ2tQK#rp%7jaT~+u`HEO@od5XEvS}F01~}%=8?SO7kkc z8ojcQc-5UXpqCxC3M|yzHC8=c(-WGERzJn*?5x!8Dz%dsWTFe`=@x>k^YGl%g&hl* zhHSONsW5AUCj?u$n?ib%ubPeH7&M2}JLu}kZbyfQIC)HUB0kXvu@Lv z3ioO%BZt;oNNte9z|R6^dafT@+tBK#HeNB}rDsb7(BVq*f|HPr>J*6MVTFPMsZ?NSI?XDMWa-l>@Vco$ytyz1z zN%U^vF24Ek+OO$R)rccJn?Hhm zr_WmF&1|;@c+_52`CRkin``0}wXPX7xo}$Sn;OJ$aej?7_k=R}g0N2e{+gzoF8M1G zEM}k52}N;oh_qDrJ+dpzoku98q6n3IsFj zmlAE8!zYb$R}N@QC(JkdlJ=D+$>Z!M)6Yqy*Pz()vWSD!qcN{DDuAP2YT|IyBj0`( zCj`iC5&5YH@VP!d32kKeD0~iqls4xo-bri4wOF=U44Kkm#I2tCWmWb1&$d<8=G zx&6i9x#)D@`*?YK=g-&=lLv6pzcdjuS=V8%I|*0Cg&j)H6PZ4Xuw$b)o^XOyVD|?9 zsN#&M?BY^%v96mom8^XVsBtZ_Jk>SY>UcX5p5{y{t}8uSS&Qqe@rVZSp0SdNqpybi zz-A@q7@the;2WI_&NuLVZ8@orLQ;7Zd4AY2fhRCBE3Q!440 zYhCzMuN=oGv*lUvRjViQR87}YIu2}=J!+E^d%UyBN6g{+@WH0h8++Y6`Gg#%+s@+@ zCk4419hz4w@%A=0$x~Of;{G&oGin6GaZ{`_(?JU>3UDmt1{IZ`S4cpRcY2Tlt9<|vJ%I->|xKBWRETbH;-gaW7~Mw zhf>qnG#UyDYTE5x&w}F$S2%K|7oXM?M0!Zb$gWU@JS-2;VII14x>;{c6I~iCGXIQ5AL_6zzJW?e%C--x$0j1Ha|m^^~jOP30~qs zcj`h7Zu73UNk?zIVNCfSF-wxC?Y}V+EXy}C+`0%#3N+Xkdsw~%NN-}FSlwuVj?^5* zUY~u6BZF17-H*r1Wh$81{jp<1f^@T~T ztJKjQs2097`Yh|k6C!m(pd#${X1Ivgmy4od4Z?R!Z&NZPt02M2p{lhWC1QNK;q z(Qc|I7aB%C^(L0u>-yk*7~(Bmaa9-81J(bjnj8pWR z{G^yjE{}5W9lXaX#T|yr<_PSPlLxMkC`=FsrX}BRr~6sgNAG#t)s1h33txoO`X#i? z>B~g`!Qy6{!r=J^7`EE=y%4Xlj!pj(EPD&2*!yUl~Xs~{om8$uHIQH4;rC^NGn46*SS zgrLuRii{W`)vef^>skaqF>U0R;-C91BBpBjiS5&;Y4C@PaWv$v;kaoWlR1!(ftJBxs`^10GqH;wBx$@*iO(AVP zc}*dtBaU5CTi-5>+8H6msjX_tNwC~#0WEGu_S9okoCPrVFPx*58_T6eNwNEGi@NI^ z2}>W?Ut{BcE!WDHwzK`_0~!0KA!Fu3zh0)M1_t%Wy3-uyD-R!d?M+EWgD^04;e5w@ zA@_zji8yao)mHtZ5Qps{NxkBHIh21hC|d{z?x_k%tvfoYwZF!)m@3;oszF%)c14QF z7yP9oz|{+0W3XHuB7WDibF`4o4!==8Was&&K9Fqw8mf3QQKrfgM-y`K7$1FLfWx9c z(4(%`VYePA{b4ij{g9Ns;s-F*`PMZFJdEuFi*|)Ay-l|~eogyBpUkcS^_dYVnP|!l zh{uNrdHZX;2wdHb+{+W@Q*vWweMc8G;0-M2sov^Ml``}CpLl2HDGO{|-!^FoQ}1O6XPef-EoNT6-c6?pJb)HFdjq?L>34UmD%i31-hyD6e}G64zFM9oFfJ z6g9S46mD31gCjbU&Dd&jka9#qpAbG9CGxdWijI;D!|Pu2hMyBJ9^RDsVGwIs#J9L? zw+p#eLDPJilbVm#X05cMEf>9LUFBj7x*<^6MU)Ss#3*|R&s8}Z)+7Xm7(9F|?v;pj z#^w$HM?%@HJDhb2Svwnw{0-iT;?79!oOol>ogFEQ{5G_m&`3eN;5JcdcA7 zr>`|;DBd?5uzYx<=3!ZD{h+^#UOJw9o~-NuI_Rb5@p3`sn5=(O&^L39Of2I@vQa_E zEp^lV6BS)7w02bc`gH=vYw*Ys?t~}Wq;|7IO})ixIIOxWlintIF>A9!q`@N@n_>&- ztQmFN4=P5~^}^)VrgqM+Q|$W$_xFU=nPqY{xRWZBEFN#!aXPajy1^+Q#wKUvB69cD(m3(Hts~nPAdj1xY6`;_VdiXiuvkU zSM#alP4dp=veME+GI|Kr)TX3nf>gG4z;izvR;!n98*}L}^>3R%#lHDR77L zvM4p?4)Vi8hDV|4qE`?bl6G(cQRQ53YP0lqlBUKFjvF81m8s7;`m~KLnQ?F-o`|<^ zxO<3soj=;qNLht_RHIB=C4lH0c)oI)jv6jXuCd=LHhUS>AM@r*$0s8KV28nl8N_)N zVL%G)-qBUz+=*KyDlNwY6rfvh!;3`wRmBddU8)GEecU#tz13t^qg5Z5kDV(u^=p^= z0Ll1X2I^`(__lDV?s{`4xxT)6U}M6=j-*z_3IXx$cU4v>%uG%J9^#=zp{iiT-SJSvuEA8CuA$~%^5I5~9Q zabkA=WWp6bDReSdpKCwTgz`@<9G4=*jh?}d2G5ibLqYN&;W;Wg;^E3( z{W+y3=f|$W$v6}3!(iXb>)>?L_Fk_d{XwUNYTB1hspa*RpZK>2F6mh`#!}>7#r6*M zO!|)zaaVTqclx=wIcyN2wA3p*Sdptqjcslt9rbqc`9+i-e(KO*wC2VSY|U8|bYcTkQQM*up6bNPYpn?F^AL4|C8Uf6nb?gTcuw&reLo z^xy@YTkkybm&XHWlIUBU{_3JiIBsZ*-uW8sCS!^5@!HfPQ&8jQVyn9oo?pb)Lv=rZ zc|B+z`H{+M4d1s{_Ys_m_=t5AD@STE zdto#6Oi8(*%^LSE4(5QLMF zg_EvWqXX&IH>wA1Z%WyP|n0 zWeAT|U@96fRs)+|AqSH`I@|@J6`x@4791HxXeD!F*%1V?*z;Bdvkb;@)KIF z!?X-iUrGyJgwVwEFmBtphfO5{Py37gql=)iaE`qCxc7Q^5{ltTYi&$U+b>|#7FHb8 zF8dgz_00HjW}JtfpcjW+=Avs#Rn?W}u_xA(9bnCFa1peO?z*+l*mYf6&h|Mth3B~u zELz$tyxDgJzPy$oJ4(rc9qiv#6vRlSKI(hG!g+Kl6ppCHheU~U1r(4kFe>uNP_x$u zvg8ybw~^>ZPt%v@%My8)w=pVQ(E zsbywh*z&8)1yQ%t%d{Zhln}P4npHqF_El@gaBG+mre*o?nfL3*D5xbp(WxqZq{g$J zco1(wl~SD+>Ds58aXb%vp0-p-KNnl!8-EyQspsen7$^zSqY%q|M!b;s<%7CVm>ZdE zmxbiBgq2`}9^9T-j$|eWb`#UE)@vAiWh~|)m-beq^np<1_RcuSYw5I_4j4-t?!G5+ zPQVBuzE|9xKC+wYv60}CML9Ceg2-@MVno3Wa?@S_2uR-Q5Ot1LrG!20VG^a`c3So2hrO^U5+=jp;7E-=NeL&BR+=GHtIEc?qyt-og#+WA*c8~N!>vzj zDWnIv5&-7x#r;kc;&61#XbZWJJ&sEq?EPvyB9dYXG2cvq%)6iZuzNhO@G5o@-$%23 zmHP4)=w9&&ybZ4v9aoctYR&ew<~*sI-7HFrq&Y)i1ku{44sgv^>K0Q zVwrTSusF>%%fz$%z_PXRPyvWVTq!3+wNI%GSYhZwIZqN?Gbh?b zs(dUfBAK4653)oZ!RH;;68F)5?wCSiShnsFKS$I!YVlo87z&P8_Y10hxps(mIizDl zjfM+TR`hi>9cUxTiL9tr(B)Dutn5c{C}(1{--ehxmOx>fn8Muh)CUuu$z9+ZOD{_b z`^QJx;?l+Q5IgNpBw0@@Bl9>%e5rxv7@f#Ty6S4cn?Ug@z9D8Qm6wnBv-Mep!~FY? zbg(Wu)Vph6P<~Z~R>O-ceu)GeFlx6aBZx4}iH>|8;zm!z_tMlmz7^WESLjn@sqH`q zg&f^F-*G@TeWJ<6^4oob))Z-Pk|FFFgyIS9mn2$~wXpd~`|ftdKK7fd*SRa}p2`s5477m<4Pa%SRp!^unPG*_n5|9@BXF4i3k_k<>2% zN6K8|bSf9-#9Qrwq3&t4wi6sNJS4VX&&K+WRqq=?-=obFBBbFSlQsnT#%pRgSL$c4 z(SrId7rX%};WyT~Z>V>+*H?6rDniCR1H#*)t5HaX4hxTzVoc~OusR|W3%oQma7({a*T_ZjTlJk**Zj#R~YS3Vhrr&XpO z6GWG8lGm^yT?#N4McvL9i!kFfaFfOmLoSBbW>QqrV>Ng|gF5?M@4%J%FeddS$)U&y z7!;o@_XN@xk0l~{4c=k3z42v_m>%pK-ljLiA@BvUoYgUhNc`=ip_Crp1MhRO^nyZ9 zQS`aS4^8Cl#Oid@ompcL43;@bIJ+hcl@7zuPmUm1^r;6A5H&o2%zWq`A)zFsH80+e z@~=^OKKSPNzTX7N#!VLrp%>b~O_HXTA-((H;s`jq5-tPasG?MxdYTx+Z(Yh;M>95F z{3OLgnZy#`<|U`ghHuKvGaNT)^oPlcq|Z!dn>zSRS#m04?cuzcW6g@h z77}qUdG?c97|gw|%a6|)Czu*7r&X&e4Qy>JjQFb(R3rjCP2x=&MNwZZj$g@UUj|dt zppyYy7VW$0zS(0Ogk3)n?6g+Kl(S>kfc|)UgV2biMAcCi>X!J3)0myE@t&i>RKlQ$ zy~llt@Lc-(31=NcqqtPBge{=0s9w4ge_mr0wpXq`=HAJM=#SD(KzlJTh2%RD7~R-W z3A3ONR6&Y1fC1(23Kez|vEoB(qHtsWJM=_)M-sFBuk)FI)HS;^J% zf-7NT1UsZSor05kF;eD4YeHCGvIU0wHzPTnD5;zt3DEF$G4&uplOsb@;;B;3c&R1E2%hE4Y-)a;tls&!+$2qcP-zuyTg4bo=DBEP;*4jN+4gLMB`2DV(AGxW zaVj|+r#mkMGk7aqNlq-uyv92TUsmM}-z<=ev9X-uM#xSVV@_Zl4$fZyb*pnBrB(Ep5-&QI07Q ziq#?`tgp~*G20l9V*K4UDGo!k8y#q*>_4gosaE%mxJF2e?4@rWuUy2QjkQ@|WZ5;H)<}3F{JfZESVdJ^ zZ@p0U(6Hd-@_Q%_?9}uSfoJQXCt_4fACwUm#BPwI{QMNMRr_sV@@PD2#Hneh5NCuw zka*6Wq^=H_LDMy#mB~zbFuNyATetczVtGZ+R5wK+dOc#&p}BOgHmJ)osNf;%CXiZq zMUWAvwgD}LNe`r#xu&#|J|kX9V>Py?gQ$WeP9^cn*7S9^#&b~R|LmjVo2`8iy~%lY zfT#v=VMyI~ZN=@)BdSC!!i%dJ6j9to38q4MDGm$?k7@vYFX50YNmyU%yqWbT-v$#q zrZAOJ|^T<)qF8I7ifd^cC5!slDp)U`7^xTdBl1Os%HPq9>Kt#Or|!_x3uw0 z$jEzp=@xarBG;T4R*;nU4>cxhP*a88nNaAq`%r3n-AeQE@hp&VRe$?4FO*iSzgn4( zuK(v73Jy|~^BdkN2!U{z^-f(ED9A*WF`EY5EEL_fIHl@1-*HpaI9wM?Gii1WaiD+a zqXuhkk_gv0zPb;XN2LxKU7OKQfv;Ox$d}?Um2tEI<%Id8)ke%8H7no8oUz3TAWAU@ zmpmSu-*FzjfmBZIp11DonUsP!y^3maifLkJlU^%Im@ZRDJ#P?q(A98Olt}mhhE!x&A6J8*r=@b0~mtD$j>QxAL5% zR{2Ai_u+N-5ggrPZ0+jwXrDO_dj`F8_bmaxty-KnBru=lKK? z22j2tTXJS<`}Ev#Mg2im3K1Y8Ie<5!(et@#lB!g;QwkI_a^lR~oJJ!7tXOZu>-eZI zeJ-NZ{O@B7P^q9Nv*}*NWSs_8R(+tQXP;2|1iHt|0mcb=-=XtrDt+v`MakkJJn^jP z;o^kL%dU4NZCJTJ`$xra@Ub#N*^b&QOJE!dHQJ01ltqetxi}oX7a$RGz#qiQgDZEv zW?;Q?arNjb(6TqnV@lul*!O(*(PCG8xBy?;#<9W#g%XF~FTh&~*4755jkq(WRm78Y z6kmWsUe}y@Oh-?xz#kWZd#;p-5K@g^Rg-gFF1E#A`O)dMY8NbISg!w@D5~@>!XZl7 zL70yHdMphoJqox+1~h^MEt1G@4JQ7wPv66zyh~S5c9_tGHB3(x)IQ7~@AB7Xwt7jc ztKZb{!ldon;ZRv`oani#V~v?pdjpgi%8{LRq!fFLHcQ|T7}~rUu_2*?7@+Rbpmui+ zC3^a6fJrlDP6i%8uJh>B&Zk`mm~MRFs*y(4xsK zMWTLW`rLd}42s63WR3&4DS0A)R8 z$NL4$TiN$x`Bz<+s@%2~o;Rau%5)CLnjiGIb68kdnOM{qb>0D#R^MX`DJzV)DY%(k zRb4+rU^<@3-J#P(?(Obe%*^84Q>tiuvxZfLK_zpp?|N3&6d`p{zVI?Obb!d>h3>j9 zg%I2GFjhVLq|01*_Do3LIg*S@bVtIsP`tz)vn9duuV^YXG3uIf{Zil|ssdbOu~hX| zxzr-GjANYH1)24&5_QDU{1%OAOM4++o&p0CE^G@~`MK7E4um9v?@Kv}v*Nku1`U`1@Eb1Bfu-)k0zl7-9c z3h~wJu2(daa7L~WADRagkS29m*$FRu?Xe#8ylQ1SnkXTOvo7c<2MQsZH-gZrc_pB*YN;kAmzmGhp3+RINOLBYheN^22YL%eh`Q$3z!x^Nh! z$RpTkyub=A4_!n-{*;=6*$`MT@VJPaFk!yk#y4EAXG+3ZbA37c7(j{NIena=Pu|&M3ip1B1(+xP=sflyGE7c%R@~9%kwq|F zUlG%ESRKm4wZh2bMr@}Bz~SCx(2x&AkIGkQ4wcc5*EEJ{QP2ihD3Wi{dxH{nVlmk z-NTj5z`M|Ywv0Mm+Wrzgwr0GJeNW@@@_dq3xJv6M^RV;$7#ztuoAIRlM{kd$A@(BW zYci1MB+JkGWt36yibdYN)@LC&A!k+1rn#DWjA-IrPss@v#IE9+hTMr^%ubW(*A1AU z-*|ChNjTUx{WfSuU1+GA@r8>^aC%q1`Q+l&x!+851@LGoB>!LloMm+LUF1rDaV_1^ z^^h0zyu=(n?OZIm0#&Mj1&X%ml=V>$iBIrN0aLhkD1HAs^Y}TxISw!_j&K;WQ%<7* zL>bX`4E(i*OOz87nUYVh<1f<*MfXtv@EYT73Vb5qU6&M*tUXEsgN^cE)%;UFZ?f!k z+DFf>Me5tVMM6i4@ppX!SW~Et&^DsG#u_dYlrc zx3i}!b7V*_`r)-=hZi#j^hsZhPs>uw+z_bqWfn=SQjtL%Hoq``A(6?SQ+l)HjSt*f zs*3AAUT>qFdXcAew%;Qt%rFtgaM@qW?}^@R_Uenh?d#*xCD3=PoMOCX-xocN6;e_4 zoCXN4^-FVJ<{sbx$G_GkEm6RgneLAsE5fN z?E;ha?6^`_8cjid6K;zE+z)$qq|*_JG}`fQKs}vwYdj%DfY$AtTOo}il00zveF&PooRqIUhdz9WamB7$R@Ch2YxU|S zhZDKz04hgc>7vA4EGz8jXIHyn6r%E67F$)DbdeuOt+REpdg3P_xzfyam{Qb$AM_%O zSA6!j*lQ{8&-MjmqpP}_OCzsC^c}?*RE;zfQ(mve;2mxf#;ct@2-@us-FibPy>cPF zawct#kmO#D(QQl4BQix4Z#)Rm(QNnT@+0^aF*d4)yK~)^0+Y$Y&E;xFoQv(deVYl{ zFCjdgBZ;%qEnd~*IdPvaH%DiWtY}qRRODK?QD?S4ri$9JC|hsh5B(py-T^j~FlrmF zZM)stw(Y0pQ*CYATidp6V{2oJ?Nf7WcWdL{SIPIiU;gCEo$H)Ta!*b&*UV(jbp|Hx zkaj%Wopxr1U`!kB`RJ){o9R4vo9`VW>5v#qVD5;LFLG2B%8>2z-u7rGd{J__9Ng!S ztIy+DRo3`5RlP)c)1HQe3;U~>CBf;Dg8l`|?<101Sh^an{dU1=6zbZyql-JjZ;Hw$+H&4XULFHLl*J!Au67HgA9UX*`Uxenb$Cx`4GqWK2J0E+u}~h_wyjTlyiNx5l4`>P(os? z4Qc`ZKIv!Y&HOQo&tY>)qiTLzadlD;ucFZYd+eV8SpjCRJ9_o1RgNjM{YKKWGVCSb z)xI$+GariAZsTwbfNn&c88xzB^Q&H0#;U?S)3lAx9qWV=yk9rkFKEsc*k{iCAW?UqB>mQT8;aayGG;N*4AQT*=%kQj@c~>|; z4-T2HpFOOke(`R)%{4yQ=NyyW!Sz-y}8jbUK!MKSOkwv51oMohEh5lxzkswIGCW+D-69 za{FWKNjwFXe3sG9v@$GbfV+}H-pAO!Q;WR1c-W}Onj?t7ZsK|VVj$SdjIyy=Uletu zhX2`#2Xb38PT9uybBhwP`B0_bgj zH^-DB1(X+1BU+~H(wt5RHl^2*6cV91!8JLvGF=&VFzXXgay3V79>z`I@kTEypcZK0 zMg*q}$mDhhFS~Nq4BQAVb?g5IGVg0!Jld7*=f>Ve-K{(j8C!~cy4$$%2Vov3R1@NU zdW-{C%>G^Az4KGZD6dJG5}9rx`RT5nyce2UN>H${@AHfv$L$U#b*Ms=RfIns+apqQ zvLwB?EHcBEGyK$#$^6mJ1m1!b>1>4s;-x)mk`w$MnZgp5SWXL_4J}B~4j7>PYCMM8 zwj707dP_|qMZ27bQ9iYk`&U?mI5WQ1!=~HDpJawKla`%un}Z${RumS|T9pR4?WuY6 zqv0W-hNBOIj2@gP4-3KjHz0!aw&x+!vrBnj1@?Xv(AX2XBg9wlI#lfIz&ZEn49!*~ zelts#xGk}BZ?{;2x%iuq%ZQX~bWGzEEF6P3hM{L0i5ovnIa1)oMSWzx@S)gS5bR7ephC?|-`BJ(0r4-FIyXlTCkMByd6wUnRDy^O1PK3#F+562qEmn5UrO=j#-JuT%}4Smf>0aLoD>qQ z){p9Ouo4I&wU2szte@Ixn69yb(FP^_Xqk5!PR`ZPQBX=Vc|1HdMFsHGxj?Cd4 znaMl9lMLmVpUTe+EBltt_3;(uEH7xCfvL)*B=IP~XV1O2a*q-kt3NE&edmwsJ&b+f zGBmkgltE@@CTH$B$?ImmMFsyi6yZnqn$G(d=}qBU=Ze8|^6BD~a#HTR?K4Z^{@cX< zc=~x_-h)$Fmu+yjot4nUh1&I^H|ZC_>8Q($sga5;!dxO@?NT;jX1A3B z;AEec1AYZLv(w%_dl&+6nb}E?R4W>wjPL)wNs82QR(zy?(mq3Yt>L3-xh-=L`R1fr zdkOMEy36@DW2}Fn+!#)Y0E|l_cygcRdHk=+$A0bgExU}l$FQk=@WB6%GJ63KEz4+J%@YBytdumJH9%j{$Q)oeyyc)^2?!K^hhZhl(CLMa~eU67xmIX67{dP3~b zfh(JmEW21U`qk!$M!Lv>V7nI^p-7m6&dlnobPlWn7>cKvavt#&J?hb*XV=WEmo%=f z+#&({K`5)MY+7oa9ENA1eR4_1or3(W2z&{rKEi%0ypiY-2kMo~9=ca(LqgFpD?r8p z{B_HYbR+2!>P1JCw0@JnmNrN2K{=yOsp_?&@sAl{*JSdHcUBTg;Dx{s9x3ZFZV9DTzjfgOb=Rvl2E3`KE&HT3|Q+G&jRqxOC6EVPI)SecY7|nF}0a zNP0*pYV{x(9US!L_PFZ@wQrSMWTN7ibA-Q7^9@Ga(X}v^2e6tWrBn8&P`6mH&U8bi z;XR-Y_%qwr0K7;WPo`77taL`@lfLv!?PXV_0lqU@+D?e*j=`;lWxt*K`uEauC(AiD zO_CwVw__vc_07&VxrOq%unBetPZ5k^K&v;ar%WLPY#$wI#u5Hd6}rLjdP7(X1$!#^ zKb;kKIettN+QXAO7S{p%Ji=2*Epat*WB8oVHT{EW5w-HfV)1ayM?BxJcV0G#@e+9% z7Is(}OeJcjk6R6aHSbEr8RrpB7yMC%tcOH0n2%eNye})i9ydVXMqHj%5ed}Vj?Md~ zF0;Sr$BSySryy_Ny2?^(>2l16Dv%h0#faexwn zm>&c$$zv!5apIX{;v4tu&3ybGnlfwd1kp>gOTg=~I(j$Br|jvqme!{R;o_{@DS%h* zxKr(*;m@r1&p^|^|NbH>lT`cLGfoP%7?brYYAcn?mr!rFpHrh!zbU@Nn z$FUpo6MVT&r;5(pkHNtvy6MWs?8%(x#oVMh_rjB6g=%L@mcRnrj)C}`Z-7bHw0!__ z?ICuJGffH_cs62^r6u14hdN3dpejcHvU2ZLldWRExp`Xd9l0>PP>B4&7l)y7=0Gn* zQ`}W$!u=59Pb;ciSvO+1vR>4$$m(<|vW8_N_r*nNh8zE#^^J#6n!`UAO*s2lWm778 zztOmJT9!LzI#)0(R?fFs>P@>M-C>qxi*ojM;LAPLEH(YNd3MYgPN@ed!JT6>fL<}Y zj2Lj|#OiO>l6r8;dAY1%y- ziUHGGsVxv2)fy_?vhKl%tA%x_4FAL0fmVE{-OQ~%+5S~mLJmm|t^PKMt~prw-l?120_V2C`FLgLYN6MS6(?4+FvEQ*^k70rZHjy zK6~qL_MlD)X%*Q{ZtX9hE_(cZ?q4{Xj}sQ7o0ppm6n4VlYbSWvzSq!GNTrsaEk(I2FF1pc4csgEa`^&G3l>(qWs7_Rf3#3iS{e!}kT*H)rwi{SMq8tEBx!)Jc26@v6;w7k7(EeJ<%zS(S%u zVxJlLUxtz#EY%O1nx5)*8=PZdo9$y-(<0T;iNrRsp=)H*V@r;32`>XtseM}-!1gF zn8gI5IzDrlbPdBV`?vMiZpVnzr|7cWq_*52{#30?wM%KLah#9Q`M}C1j=0pkiUU?I z7^_xF?9voN#yJvm*WZv3`PPzz~YD>>h_{Ci8wtbG7H50HLn$7O9L$%|^P z(dFnxX4>v)p$yyS)tvSoVm!k|GZ+BQ8IRxh?RGB7ie)Un&5oHU(VbTx!fYn_<$4^Qp^}To@V7{*w;wRDnTHf(;ib zKYS|pPPt_lVJja?bwbTbZ42@-4Ja8Gc!;G51y?$H{;~3EF1+!Y+C|Z)qFR4GR1na6H5x3(rmt$gT{H{V z4jDSh{u@w}TWd4sN6(<1Vr}5NM0U7wwPtM`2*@_Q!St&*>qptXS9odWQ|CN2Ze~nV z(IRijCYPSgg=xSzc_4m*B<{45x$SFRgLbXe$= zYl}V|p%&X4t2Vq35W9i>dJDl%4A5)_nUkemSxMtY!T41h{4x(5cmI6-+mHOI%;|-K zs$`u)D*tV}Xj8&F^p$%q8>wMs*KEBr0K2vw)6xwlO3?3mWsk&lG`>hnNX{qjDH`sx_+ManhMOp@)TOfIT1sMgG8 z3;KF^f9v~%u*&a!T}6s~C@dBHNkW(WTrUh&&UH8R`+epJPKA80L}y+__;gbJO*Y|z zSZL-DUCE0BUx5VUa!YuEY4sLfjm0=Glw=|s=Z>GLBr`;x3aQ@RciJ-#WA=@mAw43h z$>H$CVD64aFUI3dC!BE5*p<}zI)P*LX0&%bb8GPAY57^CGg*i{Tjn>K1JBFm-GU#M zHY3sS@0ma3C|!eRWuW)J5Zq8rH3$#Rb(-gj?%$V3xB<7~R{jfxf^(9>%_wV_>5fY}P&!aNAZi+iUK#w`*0MJwfa?d?7o5am} zteV&9c{D__}HW#ccD~WTl;tE&{!{E@{0AL#)c$RVgVP|s#&5W z-7av+?=xVCZ8hni_Y_9}UM~5owqh#3(K^fyUA*<(UGEX~`|r5ik;l#xL#?VA^PLn? zjIwIG%MdgafIdS*MKnNb<0XRPi_|*tAX=HXLUX@~Tm2H@wWcCsqI(l%nB2t;8N`G` z>r!BT$Fevo=dy#!F|2~nm+k&z^VO3Z69s$-vU%!ZXLLvV?-wHq)}DPaPr}4tD3Vn% z-p{;sw6Kgs2mo>I=#&a;LTAMZF=INXI5N_-Hxrat-{PUNi%2*HQ)tix3DZTg^mU@8 zIH?t#btG)aY%$s-r%(Y)=ntO_3somW6ZG^Ye4Q+jzjxqc7o`{5IK`BwZA{GjI5Ja- zIbCJ{jAs-<=`nhWi5fM2Q6HW4H!4#LiV}wqeKYGewCgE~O>z$3-9vCWgi)7j(y`g; z(shl&pkpdpMQ?dqqF>$=AK^7G?d=~&{6xu;C^G)Y0`MP`h z=7W+-j4SsXgzOv{SLz&#>C-?raGQ7UJO?bsbuIs62kCb%XQtT_>~fcY zQ0v<$dP=-*B)%7m-WURpES9hrd4ud)$++130ttIr7F8cF2*jjBPx;?yRvCD# zN43K2*@E(#TFvTIRfQNPK8QP-tppZ0YD+Vs0_cO4o0`<1S$V^n2Y2RffF?{q3_DUY zCMC}qm1{3J^ss=#SQ5ELUN$9x(cw01Whq^Qap} zzX(V%xtq_7{y4x7aFbX7`FOErzPMZUgffykpmUR?GO5#BD);q7<}^(7a(#~7?N~GO zW{VA27?4xJL3QZxMV6vC3u1w_<?qZ-PTQt?NGd+zGW zk+YeV)C=GOrZGrv@K-^F6B8!=wKh~47Tk6nwOV`K-*x@TvDF1pqDBq9nM*B=?Vsew zbX_f%y;6p3w`{s=o*&sXl5|4#tl*5_PL8d z_1%|xVDnoKm7E3MvrCG^rKwBs;tm>J#N~=Lcw}z)pTDz3@K1eim9=^+*8ucmnz1id zisrXViz+7I`RNc1Z}q+oquawHe8l+p`G`Bn-3t(~nHq4P}snM#fHyu)@K< zPeza|(pK$q>6Mg|tELcx-|A-k_8&{z(Q+IvFFIB7Jjv>Qb!m_t`PaT#j-J87IgBV{ zlKmuxw%u}S=}K8l&mtrymxq#EoL+yX@hlfPwj;QjhOHc)yA&(BCtx;LWW&5*5)Z5` zTRqU%^&d!b$w5DzAT=l5$&1-_y3{UXqj#ih<_NEN;L4jQMIvXD(`k%Cs+w7)uP zzlq>8pEUVpV^N0+@4iu2Gdo!jBA2S3Tu$%gEDfdf_b32E_*j3O(fJ-0rTph zrc}RTWeR*x)=1)j)%X3Ji(g2iCO)thEe&eF?4-+WNY4UZQ$s5Ev}r-5-w4y%yyO4W zMd~3ZF3Koe?&j;FZN3ma9iwfpQW_XQaT2xPD9c&>srY`vv2j@MY7%w+X6~vV>nk6u z&?4X?|2bGRVrGmdh%$Srs+%DO3??D4oz?G%j~h3NOr$!t$%;6`iLG&l@lgMuaMn&m8D%~pXxc=uT4AW3_JtONKLrhZ*u zz1k_N-(o46P`TQKzL!^=-ERi(Ei(}#=#k4lXelSu$}R;f#rR`Q8kR6>PE`b;RahFL zr_fUjMyBBcq^cbWy+*mwu{sC@5|xo}u6Ols#F-R66#Vms!l*o+%kL^M!} zSE^w}T628c8^fN0Jh*^rfBg1c<5I$aNuTmmc4=tPc5~Y98yjuYuV{5r1<|$z%?)Jd zOnUC9j_o&ofbL77MWxyg?NWZst|#>tGdgK^2!qF;&QEgQ0lgGdwbHlvnw-zIwM6WE zd+TyqHqo20M6hX4W+hi)hDnrd!D=x^7xHcbs4&g_HAIwjK#afAgVk%|{$^!jEeHWE zCYrqoZUB%5j;i`C@SPrAUrrHn4S#Go^>_ox{xBkF3L&^{oF9K&hdsS{5`)5)mQq{c@&qCKt9^!Di;S-6syWvU zuAsMk&#~SGoXhu~a$&D|i#|Wao5-eoQn|sbxc6k0n>j)seKWTvUdAE*>2@_3`F+5& z@ru7{s=K%JU!Ntw8+{aW;2L3b{M^*fgzl5=k#DNO?z9o9T;aXpU(e&hUaK^Rt_5#p zC)f0$zKv^ZsW~jefSh~|8fW8!&PBbnTsh`odRNDaS_`A@T00Hy7a$)MY^nT{jewBu z_^|X#yw#BOA*fOJp={13&k}dMB|)F_>slq<+Ku+KVy&(Mu7srhD4laaSR8|dzUKFk zDH$z0B)oWvCe+FS%1vvu#B+rmhJ+kM%3#-y98buUIa7G~tM^8z#_x)CvFGuUpKB?4 zxx<&ya}9FU(Y!mW;vc4O!sHVm=H>|h1;(f1lvE*4I{n^flw?1bY{w6r zD>-6F&J)?Otu=uFSxHH@3#`~h_vN|Tn~WCk8l+eQsyb6{cvswQIi7(Qb?G*y5^F-` zpKlRJ4s84+M|3qyo%}whO^KYzF6$Zy{N_gy{PcpGDFA02>zq2rB|-pmU3F8;gA*zj zd*E0SDhEhN3SAw%aq$AK?Ab}lhncayp&MNRZkzXM&mkyLeTaAMNr%A6UTd*Hd{R0N z)sx}u*IB{eryya-b-J?^%P2&~o4nZ`wa(}_#=?w0!U>|Z?8}(m`Y8n=mIPs*Q&UI^ zeIj;n`}t5YW_P+%3FH!VH>CpXc_eINUC(6&7~$8Y)2LD+FCZi7C9eVmLfyW`E+&f-)UABP2{%|p;mCZ@cNv2l1znn-_kse+=(#o|iA(FcS!e2(7NPddH6>N|<3?&~ojjXV|XV!3*o6rnNJ3_;+PQr+g@Xn+yOSO;bRkoHmw?hCK#Rbx}97ab7bU z;+P__cnp2R2REY z9sTO3Oh`mcE<(5q>yYpvCCYNHCCYT4o}Hn(3HZ6MyS43>eKtluB=PN z4Df%i1C1pe02RDwdS5B68V#@a{*9E&kv?2l1z#f~`zPNnJ7*kOaX`hX ze_nEouqZyWoV`h_cm-_bHC{&Z0#1+olW@NfMUc>%`=yW_vzdG*wbU@sde&aIxOVs+d&GUT#^yo^+<7c~NSjdbq;Vgj-d} zZ?AM34S_DEn}XjSID2e*iM6E@I*vc9`DsVJ#6=NZ{6il!sTKxe7qpaB*z_rVm2T@! zq{#Ipw8~{}4B0+v99T;1Q!A~f#Mj)d&{Z#V9cN+E%ua#_RJ`pLb21qvc@;}#M$x1m z9p1TA;mG6HTjUe1C1c}c{h-@REZzy6-czVni4jyYyZYH|FV$}tLYKO$@T#brG^b2J zQ8IO$!zKr)MhJv3w;)vIO{d@AiAJvmn-6>Q64zOWiYf9`?Jlh5ug*=+P|7R%+1&2F z9-!);nv{*2Qs2H#kDWxsilTG(7uP%$CfskiEGnuxcWB)Gfs2_Ryi`HD(Y zmhxH{c$0u0E;E#a1;5Hjr04v2XIx78c--R95%|=y<)HgqZuh`ss-Scf5l3H9aor0$ zrZ5*r^Xh}HGToXJ@i;8|qLt=kNI&f*ZjKFo-j!N_E$pa%ebzeNQvXxQ9jtc)-pit1 zqy(|vZb)V=YDqtF)MO(H){AZY=P{xgJIMu)-vU*5bZq^>vx+)j8Yq4P)k#Zf>L3I_ zu8{_EO-9~w_n}hy@XM==NMk=;NS#PKY}JWqd8S@gWE&D5z`!xk^}98@{D8 zS8c`UN&|MwLP;Mkh)1Cwr84Q(;%sWvM6D2MsC=xjT~FCEXct$Xh9t2$jOeDMPCK&H z?fHao_NdU_O*I)Ky!eSF@|IOX09clFA)eW<{8M zE5v%M64zg?Fe&encqSDfOP>IaP!ia&ogaA~RaMSA(vPVL|8R|VAYtX~D7#jY zHkNya+Cr5&^azG&so(2sj^4GO?8q5ugkaDf1WdBv>3OmiyW8hS$0Zd%|Ih8g1({% zD+&|?s&^(I$yAUK37d21%r0kFeN>FBvoiu|erOD!0*T722#j`1uwb#xOBNS&P4B+4 z2EQ5dKc~CcW#}@)OHx0$7?m&viAeU8rJsh z;DaAxzHRNh4c@kp2b-F=k#tKmGCs;H$IoS-Mkr@!k;<{zw*ja-J!s#GI+sJ7>Dqf2 zkpOCfSm?8YZ*{g4sCTlsp*8v?@_?z+*cc)PC%xY1*c5* z!mho5jur9X&khj`UP(QsEK-0_pWY+d z9=f!=4&CT+y3#O-(h~aXMNLw{rH)(-HJpx@brb$g%?>8Q8o(Cc$1O;cq3q3!zKt=m zGc%#Pc-I1|ZEv@TF>gfmv5ig~-wNS-43xE^lP0mM>g8@-P?M&dnH_FN3T?A;LPrKP zZj60l=_0E%Mde84q=U;?8|}LN(6^~tShS{*Uu$#r z+=8{qg}Lqq&HZftOp+vL3>0{ct2kY0#H>)1?IdHWc)xG#<$w0_xp8fqz%*wItT;4=6&ry9ST7SeWmA#$% z&|BZICYtjBf}@fWrQ5yGZu#y&-lBnm?Nn%EDY0*B;8sJOY^DyVoY4RNoUpDD88Ec1 z3bWHMqcv+OT8x^9y#Q>p!=jSCTE3$=CorZ?VBA;mTXfkqJ#tO0v0pD0zMZQ}+5nH0az9j51_uE}aetga%P&;Hdv8IldA z!s%d-2}WX}VI`YZ9tC!YZ?qOkqbR2dJ{~e0{42dpT;kHSdxm6X6=eJ{on z&8TaXe-zr{syq@1oDaQas0S1s9drYGKmWr1sKTh$8|%n58n4@F;ExH=Z3$GYiGtCb zSU7wmE!{Hz(i=K3w2$YaZ>YN|xlEjNv|+gk-D_H7!RU}3?FQWAJaAUccD~;W7wLx2 z1!RnXHGhO^-)>oIEmtF8VlP-d8WHOM1yTLB)NBd=HaK2gjV;+%nKGXy)0sTe zb_?;Mu~Gobvn)cXahnd1pKL|!?alm6N*(!B%zwr(h zuHc`?>(EX6AeR8JYA6$=_0xDWytrmOyxE!SryiZ_#s7BOsU9(eCd*8Lct$iR#6`&H zDJsQTbrYR8(mSqg&_}8N=*an$G~%_@=y+hQc(~`lD&y_@G8Ygy`DcG>grIFc0uL>; zO(}^77Z&AsQ}qB9%2cUT9FfMy)w}7{1Z|OADdM}p)ZGdBW-+>MdujufPzG+arPyb4 zW+oTC&MNicT%DmdpX?5)J7_666Rau*2OLHXDOks*Oh)mVvFmD^O#0m?tdDBjl{~*0 zmczN~C}E9otGN*~HP*(Ol;pzi9qAZYj6*7N^s67$Xd-lj!yT^l(J;xA6%AhU{0_m) zv1yDd9kPFkGsIMX#~b-ffoY%CDjcEcmf7@42d6k<5_>>O^%a^%neD+|lOJr_&%3?#68it|PjfNj|&Sx@av;4m1%jK4Tb={!{w2WxBiu z@U1?_54#}5uBaomxHs)hUGyNi)Bo=|G)d)pZRS%}SgopU&3d;I_Li0y^;Er3B2r7) zWVH8|JLM>3D)wD8VlrdYx-)H9fuH=yP~%&VMUmk{p)wUmjjxC7JaCk5N`P8GzMA5W zVPKOqjXklkP`*!#YOrOYfzS5JX_P)b+f1q6j1{4r>s$6hBShVv&K=<|suyVLGm*jO z}_Ml+OIZQeljm}@eVw6i?UL3Nq)=kCMX2F%6RGXa@xb}S7-=xelms;+}?|@)by>xQyP#S!QM6*JWp2j z7ya^;2cpmqL63Y1c#A^j{XjI%~eYAy^Nd6pojBf=3+#tVVns6wUH*q=iVT`j`hU_ z%_?NF&JEF5Va5IQYGtIx@MT@DQm27qP|#?q{(flroXvGG&QWaZeA)6)3kR0bEp`A4 z_M|*PP*EENo)-V_=7s`3<8|qsP1j~o+2$oem3Jl^AiLO7p%IPM?4v4Q>a)9KesgSM^Og1=X>7pbqt+>}iLBvu4T2 z+bPcPhR#_ZOCF0P45+>(jS&Y_r+R9!tYA)c@uV+Le{r1bNZsRJ0?FBoo2WCqDTt{w zKPMtrh`~}RQ~)f~?Q&EHJ2wh7yJ_pGT|Rph>#_;51KL z?Bc+cuM-Lu8QHaHI;UD4JqqPtZv2WARbx2KPt>b43Q!@DsWT(J(bqI6FFIdZGpG8+ zehU9=q82|cNtn|he|u`ttUahCEiJE)qq`i)c>*D-HhSeXR^UU3sv8z4c!F`!c6!Jl zSOlne_TpNKl66`sFI^heM1Favu5X^BpBjnDGU0#}b&htdP@lkT5rJ+yvGC)x>IT0` z*(D1YS!;Q=nXxH5{f_;OIM(NPYOBvw45gSV#F?19y2SxI`ZJZ29GvS?7Z1Q3zCyiWF)sy^I0lPA!g}YV;JFG<$8!w6=}rm~`gTPgW*z`BfOX zwn^z!299G7T~zpVFo4>*Q(kVWLu$`eWEyc9Y^_NEeKa%6@Qd6v6$X{%DCu46FLobS z)LyV0HLBoA_rIMzt$6{XxI%4Yj469FRn6W}A;ssWa)XP7omKp^!mkTyR*9WNRkXCs zzVk0^EgXZTVWBno_G55 z7d5R%kBn|3LeH$iK}04}X8hHC%*a zex(R=K{>>jl0g;$((V*(?jusXXF9iN27vJB}X#PiX;fC zM2{pet~@u46kUXqjALCiC%1trmL>y+7BOM|CP!m}_??d2r6?IA+6pHb!`f<2eg!u| z9MnmH^u&(t3AJY!FBQ6LNWB*?x``f1Vtg_!c@HOt2(i2S^>r_}sUF(i!OK-Dm*GfX^*CC9=b+1E4%@%}*;T60 zda%%1tjKz>$Xcq{da&49ti*b-#9FGnqKTxngC^!HV90-d30SXQR1qXwILqO4?px`i2a5yM90u&ty z3XTE=M}vZ6K+&q6r6!NGqe|JN~b#RhWg zapeYuYnAJ1wFal_X^mU78)>ZuXfB5TfbPP~5ZC+f1QYL~6Yu`5|Aqg9yKc#TLg)Uk z4gb$g+Fi(|53EXM1$c{cCAvayRfiLBraWn%o`>K>EQsv!`QtyF+;&xogpIu>VQDBn zX7;ha-jI`R7)b;^`Z+QKBVer8d`{PVrRv}%KMsOwNMt;9PIVzDruod^rJ5g2!9Nrh zzA>ftm*F(yec%)rL6Ciwq=g-I6u#gv_uGizP=7TdaV%L#zg4lKCI)t+=(dq6{59z^ z$bO{QjY^nS@8-O$uog+59KG$d!ogM84ZdgUcywjpO(=_duldlaPS<TZjdoF#I;gal*Z9hY%*+~V5 zH;$Cs-%{)Ym0c1Ea`COzYQBI89&pLT5KT2P*M1*N$dn6;X&m%T{9mUGvig^YA(Y+6 zI)q*w{$8G1rfe6EZGInAk^LXx{SuHb&%&QGhMy?N_X%2EI;Tz|2wOsiQ?%HrtMzojj_o5r^7iE&$g-+GRlk<#1;)#SQIfQXADo zJ(i6(E!)|@GwddEI zuV9PnkyqBfd3*+JObQupyf&}wE?L-)27wqeQK?hnkQhRLl}qe68jg9qHlN|4SK5w7 z7uyKUNX>r)arH?r@f)?j0nbMTdcw~F<73H6~O z3q;y#*>9O@20H6rDvdgMo6D%QO2+rG?Ex|st`(=BRSC1WS+dZLN$8Y8-js-BUmID= z;R0PXt(H;9I~-XV=qUb{a}LCW9U||`A~t`HE&Fsg1Iu&ffAm@j?q5>2#j1vBPpb1LG++sd_=1_4!%j1chC6uff+jvY43Vx4NJEKKT zEzdaGvD@Q{ug*Y_M=tArs$3$b!lkyB7pwE#9r&7Qc&8!!a~*E->KlA+<D|H1KauRx+Q;v@&+cYnob(wMyHpYm&*c&}zw zqJUowB6tFI2J)QXEJ}a%$QK;;yHxtae!=#g?ARnSsy=#6BTv3gAo9QBsZJGnFz4^7L8>s)_;}`<|HHHJ z%=gqSR+oV{{smuv^tZqBqMg}R(UBO92?+ zS3jB{$58M92F@Y9JzNA0+7SPuaYU>n@C50oTn{6!k{9szV-e+_x_@olyj|~Kwu*gm zUC(iusDY)}5UqG%C5cx%V2`hW(r}vqd2TW5t&_9)&Ok^L2HjcZ>{TZeu=gT4aZ(XQ zS*$1_# z0pUK?uUor!aS`B)NmWs)N-N$W~D)y~?Y_d3!qox2mvUfvQIyZ+e2_Q^cunldmq(u}+N9;APU z$_LrUe8f*iZCl6GDI*c>K-Mhi<`9$3pXrJZ1_5-TpEHTfSh9hb!BWTgQ`xV7Tx`eewVL@+1_ky|*dj*B(uE2&Oo3Yqq4Y)5H0G>5q*C+wXv+2fMkyvW z&*ac3(EYzvg&`&m`1R{Q1t&pZRM7#2$4HcGoPnspNf3xJ|4#k;zhTkjRPh2R{y%_l z#1r*@gD!K2GW$PpfO*7YQ})AzEwSZ76YIYu&zQ`XS+?N{H-@Oewa2i>OBTwu;m9j* z-#z}=uZ|!Q@?h`=W}J}75etVwjD*P1spmiA6f)&_9=`1Zuatd2z_uNfw@c6D4fQyp z&Se7r#mECde`n<1n`N`R0LDb?%k+N&gmx+H-1vfM)2f)$uQ*M%l=Lz(c*FuaV!`|} z+bd9fFCqAV{HLe9hI?MtjlA$K|6K__!+I#bf13MW;(K>DC6$QypM~fdI@NeD`A_XN z-1Q3EUy32q{%^6}*9<#~=9FyVu)_a18zr>~&u4T8$5PQ4xh$nAk2Jc*9T2RUK0 z*HhsOacVN~pQN~q+j%2)WNbOsENvPp)hNX&naCdE$YUbNW|kZ&q|j`psj;f;(vx^p z`|9laxGrO~_$6ks$!x~ZEtN27z#&@b60F$U6#~T1jzO2xB~tU5KnmHf0OZ`*_nq+l zzrP1qR9UtHFI56MeCdi(pT+`OzIu<28?66uPWJy)BCQPw$DwBNI9#4Tt?mE2`L4_@ zi&=elir|_vacXm??N|n*Cb6C+1a*;4Wz>`ul;E{y|B?|4WYftyzhM zPL-;xV`>7!%Inm8yl{Hx8y7L$<7qGcFYwJ+oYgJN0)ekQ(~2LT0)122Mti8?uK-B7 zuy?b(2S&371tk}tTV8sVun%oEI3PhC%iqsj)#%O0_;_r0n`5J1U{#9L))Knx2kj5p z=-r?wV`Uk_Fb? zB+?!d-1no|QvFlOnIl|ELTv}RwKG^)BT;N}g2--W#)zj0fdn^wqBX>YK?{^t=QtzZ zdvJ~l81T)8;~?Pd2B~ZDE8rO3Z&DV@yG46T#7sguDEGe0gz5)&JUpf3ViPwl&20OR z7n`8vcN5qZ!CxI;l?`7C9-9D)<9LB$6m7u^Y@2k5YY9I#b7$%!0zh8av(6O7HsWeN&dKk22oc6e=k3l>zIZ`%96$PzM7Wx1 z0I34c1U;I3XD8vM*~XgoXg?0`(Js&Np*zmk30#jsR@uz4K z%gk=-^P}A@OAPBe0ZTZCU|J506~RufKw(LHgOpJ+L8a-aRrZIKOwB8ERvQTGaA27t z4i4x){dU>0(X4n|JL_i!(-ayc)_aS|m zFANrZ0>_9UTLeQ9IHwKk!UwAlJ4F?`W%?CbB&&dX{@vOC%K#jA;Lqh3E!O?K99?tM z>&bfd8SruMO{%24U89tnyXewq|7m~9jw4vsuxPY&<#DltKLq*H^pcS zN-MEnBP4?3<`6=pV8}iB$_du>0fFE+$|6o^vohC^4#>qxYc;RC)~m*{zUVnyIp?%@ zL*N}HvBRtv)2uuTy0~W0di|rVp%qRp;0pm(#1AJ%t=GtXcP=?*3D!4TLrGJx+dF3c z6V3v~%w7#1p^vSo8GWGUZ_20?eeQweJpJ+y;yy}Zet)ASQl#io=1b?L15<|R-z-{$ z6}>kVdW_qSj8pKxM!fb7oh!t$mrme5GVDbULr|G4d(c%ZXQ|zmk~`2VVDFQ8#Vy#7 z)IBG}|Aym@`~QCcCqUT0M%U+mhAvVuTt`X4nZ*KKNaZjbFbibr(?y^9RzZnUJduVU zTANX;J4z}CE92bGqFEli*<7C)3Xn&HC5?y!X2G$VB}5xQMIva8ec*dWm<4`NNddm_ zMf>dL;{GC1s=+BrAB965SSRNJAIOT^?`O-Vj_14ckjM}XL#}KrDz?*xQQv|)V^Z;E zB`}pB+<+^ya^mZXS01_Ly>reAdntTe19$~JzP>1Y%|kmFZz{xIT88YWTI6f15ZF;G z_O|I}kzK0nTHoOPsJ|Oq$+j$w(*_x|b{K|9c>hMJ`{U!;;b>Y}{EC_J0QL_e(s_f)>}!5+rn*5a64G91rvp0$+{ z4f=4f-AcTob+$ea%JM@CaVS{jc2BsaOCcP*2Hcs8Hjv0BolDr%Kv zMKfrZq|S*Y-794*M^Z%ootn}~@$^_4S4IE|flH_4Ef>1C zxvWu4 zD}&w|KWxANgc}C>T*%Nn)4KmwdL-rWMPGj+H3x^8&0_@s6LoN{y10udar&}9a;wCI>+9uuN8+aZgTr_ErBVXH(!OBIM|1; zD+>pDv&{iLRY?zQ9`RVB;?WcI6ck=cn~j4<>sVQcA5O^k-=8$lt*!Lq8mayvco)f_ zIw0Lp^0xgr!Z-m4|M(#Lo_)|?aOfw)DyFX*&Uhpa4(Y}g&c@N z6yD-66g-16ET}qLeuFek`mAk#5%d#B1_Ho~I+Whl=BR(G3y=&#i~B~A+QDnm^+3n& zG`Fr!*os;KA^&r7aDsW5Q~0if_Rab-?u}kh%FBpHxMn^DIos@M!>AV+PKT(G(1yL z3A)`xTIOzo;^wRM^WUE}5F{}I3fLHcz9M`#1<^xK_0k|F_#qyUJ}les&s+>RjlazC zp$91AR0U%bB}F^soK;_#z890zr&>fodZ$yldb!rmQ5KezH)PB_5+XSLwz zMxkrzPi)@xAi;QQA(W>Yy6IW`5Whe6QSmdjL;=iy&j@x6u5L5uvk)K}C`L%Lbf5PG~qe zFC4ya4WYT;g#6RoeUdtix1Ig_U~W=C70@{hp2dkp!lsL)i&D4npr6V2&nf`uryEW> zxvul?EAh2TH)s8)8%IziNH0Tm6$rfAH+`VH7g;arfyK(~J|jD!-w=#g^#iyAcW!dL z09Jrroc>L`^D8w)L;7+neKb;00ZP+tY7feRb_v>V=#T|~XH&Bj4XJnC;4*-zU?R$-0nedx;r-AqnSLnR@Ldcn&nGaV#M9X~Hz zzuJnEKiNMa3;6$FTvL6?0kh)(F*pPINOUGBlvfh+bIW#PsZfnC>`F4 zJXGcszL1f^&(B|I=XOR{B#}@4l<{fbR+BhO)tRV$F(BeySka03hC4PU|?M6XHVSxzKw8OZep*S{?BVE`B(n&8%!_S)c_Q zYzd{@?SVw7?xPsQ{xXB){3h{m@R_ct5vbKmyr%8!)I8GIXXM#Y&jA#onMSNa5~_vC?j zf)vCJ1TUjRwI|xyy1(61ZZu47+-n704R!?{8xYaRw62oQkmd%BGa#wUHyQ#-{e(}8 z3{J`%zFQ=1rjCD=1p*vanEThXCP383qy)v8-& zW>7s(rI9+B7Vpxu;C(lcwaI{K)+#Q`88}JaMLOk=Mvh(wp4i2onfu3d_YX1m zH3!1n@liN>%pjwW(bM7QV-`17)nejn?28&z_pqDpouLiFc1)-Yy&4{v^u|Myn{6_x zV~X6>pB#N$=$IDL!L~~urMQT^NRV7dTTMK^IEgnNGcy2R34r{8Yq|Eg>A3;O4~Zj0 z)TpxM;zsuNp39c4cr4cpO+EBI^mX5SR~14fpD@Z*^A8luKH9xO77a5BJb_9+zl^(R z1*R=3N{x>vU>dJi1CUVF=oNkhTeIR0Vftl1MDW*#RPB%Om2LZ|@rP8>25!K01#T2- z_*_P8>~5B!v3-C#Y`AhWQs3n!%pQ6{E3-|A$MVtI*MJznEUFbQS{=MzpqWB}G51Kc zI4&6tGR@2LD$yb8rs;Pi+O;C|GzeVod%JyxH$$lH{FUfr@@1$o2)2kxRBasW)j#uY z7nMV=+?m#d{_Ew}sXnTxRFf7dHa;)gt8DLeP6h~&1CIy$2;C(-?|{#6UcDTBYSU26 zn3<@!ua{FO+KOYt_WoD#yo)K}Qz#1mtor!D+QMQ&(acw*Xcqnj@JOxf?CQxcemJLS z-B`R&2ZCJ$$X;SNeJkuJ)k4IRlN(`c{JQ41ki^O)plCPQE`NW(kC;g9`}GKANO-iU z+hD9*#`EZ~+f3C|gKrep>Tdb;A@iO}c0s8Ybsndm<3np{VZrCW%y_O%Xt!kBp|n72 zJZ`b+C8zUSa`Lhzh#<7R)@GpM#o1*0jHex`Xm4%U zw_YChVwq62W8Y^+^-~jCsTm(l!;4x9rW73-7^AUo_w0>-D(80rc05Ey$;P)OqKfX5 z$KQyCH{rb{+<}hrDdbK%DSV(WNL%GT2|f1brm&q_|Ly;+3HFS7*YU7myq3csBB_AXNXi|iux zgX*Gv2^MN<_!{EiXhl3UNGYk^xbDc^|MVzt+NLe0RTItOw-pf)B z-BAPgdG+9R0Pb@g{VAZeR{D=>>exFkPsi#xaH3>}fR1qT+%7l*Z zeJ<3+K6RHOr%r+OMJe^=)*=ZcyMjk$!szTZj2(dPJB**1r|`MIFwOwUgdn3{Q*IU^ zsm+uDBXaQJ&)8}UYGDk1&W`778S4(jDLk!|uGU=~8qe$zLi&}}XG6sa6;C4i^f+=T_6<@>KB><|P1zd% zVS?UbbGlI zB!-D{!pcguwyA|OzUzzv3X!3iC=MZzObA!K{B}$V!84@g>10f|{9Bi& zAo`;g1*nz{QN!rijpNk_{L5?DB7Fb25tOd4=P$O5UWBeBU&Z~3-=N+xS*01j`aW%+ zoy*`K4n)QI=5HyK$jdfMKLp^>uOZEyXwb9Wo>;snS-eclq=D8oFVqYsZ|Pjp>rRhP zrpDJL3L5H5!-7u~Y9Ttol}!O(Tz_xqca7Dudc613to7j zef9^@D)rkT&RoJQEHwVT#`M!I3DGdt`lo=T^)#c^Lsxm1Z-9@74Xo3hy?TnQuYSkr z=HBCk)%kG5A*C&hX&k{k^~b{f>o&uNtJm`6SVIYq>xT^v5|lO;6)eiHt?`xI*@TbN z6|M*Fl64pB&EX2C5Y(i&ovXBxLJeOPz7?{~@5BCaatmzG>tf+_ z-&{$AQITepx|q)JsQ;SJ-kvU5iGT(v{mfM158E2g+$77K*F7-V%r)@zZ!59BF=Bf7Mr5U^0p zNM0UqP)eCMiDFLdLWLeu3v>8adyZ2RpM&J-OAUOR81s zSV=|{&dHWa%c>9G-~-}_dvZ5S7u9LUFqO|?>I%PnY%>u&(SF2IDbvihdd$lB>@Az_ zdRMNt*U|YpdD1d_Lj{e2rV1jF3=8a9ShZ1mO>$(#I`Yi$HXvMcN zpFRsPe0+ZZ_J`P}v3Us?f7MB zouI?9w>0A*mHlK&gj48e1ex*NCVg(-nsb1VvlBXg&CU$~^DfrSEbh8wy#6bL>9@IY z5OTInPI|b`Y;;_2LnCRScC+dI<^<82#rwA&de-vrn`$s{@95#a!FPvoj8K~OjP~sc zAx~2e^sjk;jDD^z8^dEL>=SncDRSZ5xGbOS(YP0z>^fgeqBeOsaTqjwwZV>{>>&1! zG;AYgu|owV%$2hQ+uZ8HiR@XGv;;=nq%kVhoSD^>t-A4Vg_dDfC-KF1!My~eg_z@jv_SCa%;gccF*12r`0OmlxRNNF=IM}q)B zxpj5{Dupo7sx?Owr0h4eibB<-Ol6@o;mnkqW(sQ~z5YZ%UR3VdIHQ=1kgQPo`RIHLA$TNDad2+4ENr4BYzcsKzLH4#LNP+LUvmI&E9f|!E-rAfDkL~-MrE+)Q9KxR z95m9Z*CK$^-5e~Rmz_S(MY!7(X&)+#hVODjo-P1mn9*cQHWww2{zkZsiU=Kb3zzL# z*3znn646X%qYp2ZyO0l$hW1>mea&Hk)lk=by${d3RFRgc{t$0?SAfXcgY_>7B@Ox} zMR3W5TxJkHEKz6|f~r@rE*+w$CCNR}h@%NT7NpY$>2#=GTn;E1hI(OLXKfN5V_+^<(ksRr7JWYEBK(k%tAC*4HFo%ghoy?4}ks884iLvf}UoZk4LH3Ii(6Cq2xlO(7 zp9)K^3R5Lm!mluOOxC z?w5`BGP@MWQvBWB-ogfdd*}BNSkrwwjpuhIq+oq9+XCdQmCT)`9;E^h!MaP~EZR_5TscI4F`N}5(z81>i zk>}|`Zann{cx74p_=?xj*nDl&f(Tk83sch;Mb)GVXHwNQ+<%8Y{`Y+%Hq?D0t+e4xgpA;@gZt;X`|afZVh{D} zMBwBL^V&mXkPFNHq^`l9Vcd~>o=f&8eY>z2)G$ke=YboqUzp_X!9f-f8LA!`L?%2F z2Bg&rA&0)-H%`BR!@63F{VTJnSaL4Ek2Mu;ucl7cwp8oX(62ljeBqTPT^ob|rdmNh zF3eyYBgVN)@x+HCV=1(DdV5^%GSf73!0ZoGEU1dqJ1@6dUi5#>>KIRQx8S+#${sgizaZzI~qLs-TN;IO_Ic_IY5tuL?GO^i>(Ol(`cxk57 zL;Ns*j&z7XG<%Nmefthi-z}|BQ=M>x&3-@O_rTZ=nX-ey2o|GLZN%s`BRKW-6PE5> z#lit1I_V7$a}%CHWIclEeF$*Wfs}{G5wf=rePs<`vS>{8Z7@Fv)KiGJo^6CG{Y>QJPjhIvBzm3oTy3RhbwQ#QA)%)#6rYvY?B(y8 zmB&4{p2I7@NQD8)ZEO`a$nGQ0qz@^eFBF+q@|*Q0Yg$5M3e|(oRH?|Q>7e~k1;^I6^sjw6 zb4)W{HKd?kUekvzMM_w zsU1U_Y{6&ZpMFr1Kd z`HjlJYYeMAJWXVKTd^&wjRE8=GdjZLb{0v@n*l&~4UEd4>r0KNl;_Q2R9wpL*d!6- zC0%&`>Bc2x8C~S{Sx0H2^h;h#wu+mCct#{^sn0-4Ehr#o(y`r+&7u45X8|NAn+(GN zcM#|&ayr$hwRc+|DSk#}#pMR^ZKg~&Dn1MR^0YlK-$iLsG8CE?I2Oeaa_~qe}aM_85D5>{Wqs zIa^^OPMBOT0Rq~|yxV%Jg`aT2qYE*$V_3^qr~@=J&aD(Q((Nq{akrZ?pu!V8Y-o|m z(C>0a8*7WmElJB6oxzP|8D{S;>+N@GjIrOd(J~K~C8R;K+0He}zt7kak%+}|Z|8R! zK6fgp3j=;Z;?^%1yL9+JA6ka2RF6X{;DCDF*o{uDT+;aJ-mDQT4Zgc3$qZi0iAG$> z2}mtA7A=vghbq!iMhPe^_VjDNuEaE3@lMH%=~vDe*+G97vLCcWa{1q^F0{@4WI8#j zvW^hiFj`RMonOFRd2IDY8^@G$u=RmV@^B%%H=wmpBL-_1Uqkvia0lZIo|Gn;C^t+* z5b#@36@t?Q#F zA87vaQcneCg{v7jC1nl8tw)abZ`0%c9nk?+4|gr$5_LwakeTXkc|wuR=ejK#`hGbY zNX$bw+@D>x9o?}$H*9s4#L`IN@H={O@V~Or>`GfN7RT@f;Z#L|wj7u6Z)Wxos1vyRCG%%GO=;wDe_e?(+Q!ar)#5`*bkG>EWtCTs**t z512Wl{lK#*=Lb0~jfO$wxgE6hL)vZ=zEb`hPJnI=Zj#$dhpTA@g1lrn-RseYp>n0s zbb04e^Yj<{k`=`HAj7PcYrtB!3NdL+YA%A;B1|%a8D>G4M@g}PyR`CJ-O|bh_{2c) z7m1V@5^&kqZjhm+T_fy*^SeMfawBTA7VrHPzeKa|wd=CYsbeG@vx%oaNUZhPg?ng; z3)jw&hib(*cg4B75!qTbv0J=DG+8GFNuxZ;>ViJVgt(TyZW^O2h6Y6A%*GUytlsSv zXFn?B?a|>{9BCFMB<+lzR(5i_>Yy$SD5%@PTdeb!dTy?l(}&(@AL&pom}f{)XC-+8 zv{z%3My3Kgizm8AxTII}kY4GOc|mp8ZJl>)HbJQ}cbli6HK0NVlT*wOcSAqVoHSis zqdjpPhzKTxpB{ir!;mxlp1?kGt8vz66|eJ6zJQgzV6MjTq?_Q^+>&JD&v0}qvzDm?E!r0Oj|=~Hwd!f|q>l9hXKC~`VcRW&!{3L*vc#Y> znX=%~T`jJ2s+3ADiy&b2j&`vrp|yA@IwqE?)5a$lD`=zpd+qhTAL8!nXHt&ZDVwEX zFV=aG>594KRs#$^+56>auvC_5I0&XI{;()y8p;glN&Cqd=|5Es6n z+_VvboH)pjT%-!XuhVszg&^^37H?N1$292jHdp)HMS_KVbZilH0ly6rh?X7{ieApR z=|p1D!!$$YvDlH8+qExM2^h3*EW)_H!f4V*_!MAWrXZ=AeMHz(TczFaECU|>8sD_7 zbPbw^IE$VAS9}yE;cy?OJFam}9I>D>e14|#c?OVf48YG!;RBV0QXvt2ar!n7OFEI% zREf%(e1+ZPeGI;x<2?4?SWqOA9 zANEmhs|s2xULGu8r{MXxN?#7sa_cwm=9VfSr`L47!UFNr7z16i|Wu0^_%Wm6s#I;pXVn=D96g?j=H}2;x z#>>V=76D4tX}j7<&DI{quQ(Iuyuo2@(@(Nkzm-Oy5%jL|*acc5DbPrvv`k{5ZTH%u zFw-wWbY7U5x;7-DM*2wBI8SnPqk&;D9!mFqLtTi>tLyZoSt}4=eu!T^Mz+jwVhM6y z3xXjM4oaI6WilbxBb8tcN{Ir|2fjDc#zN@uS^610&o|{Oj;Ju*)W5mR# zuRA7eS*og>m%Sx_XF&thZu#Kz3#JSn@kWRdt>tjxWskP0kw;MdlQGzbfF9XLFOOcL zrA*-5RltXRt~s_O-P1Qfp^)YR6QCs zsOX%yVy?g;aYhAaBx4ampN?aRBdjl6dj&NKOVITT4$T)r)La?T6lrfh^&S9$=bVT^ zpKft0e*O0^IVC+a8>G+;GW5d(Pc$bEt_H>2&4~H;O$CcaV}kOaxhQ-U^vI)tN^CaT ziI9BvW<+&&XN)e=3rFY;@MU2BOr1e(c(w17pw(mh9W!=boITLCm zyUQnb{G<}>d8Yur8KlTAC-;j7F=P%MhywWx?Z&LEh8_EPxr%eo)arA(T#q zpl0d@{6ig0_G{T+n`Pw;3jTO&-meL4gga~kkSM&ThT)~Q$ziIf^YJ4ze^B)Q z=TK29Kt;WUd9fIy`kcCmL79AH5tF)lyktrv;i~P_OmsFwpAVt&%ns<1lqMrJvUajW*Bb8bt*`MLMu4EOHe1ufc+(s- z-NB^Z?d;=R-M)O#JLEOdvO-FXKI}doyR-RhkqtH3-u%q5;(Y79>qK)({b9y8?U(Q^($-xLhl?*uCqF zHI$$P1I<2Z2|uHQ5TN>Hs@~U*^zIk`V2VR1YmxQ<2A(HCWbR&eEs@gR8V~!<;KXq; zNpetq&dTnn`J(zN1a^b>eC9>@ty-ln4*_eJu)Gz1eMWH<^7CP`5}}|EU!y`MA;hbD z*DX0+@G(>~&&lFVATKIGC*e&9Syu^Ukic?Z=5@qXpLJ=>yD#-@jG)a~8|UJQ4!>{# zz@2zNJzI_i&I%oiqQ5srRf5;Gd;hqBt0F|x7nbg33Z zNL6}FPmLF7BhjE$YXR^Y)-zgRcSm&0vfvig0Fj6JZE{l>g#2K|K;NvxcPzSwa>tO! z94Bayos+P`2%YA#HY`P(AjT11K#R8j?YfP;=uw#D{oAnNasReTE4vs)k0`Fa0c5zEUJU)gFjAw{2~1 z52B;3f>M(DMB6NgaHf?O4s*ctBD-`FpU|)|vx|EV4!lHWUf!Fy^P=&u)`xefkyO&T zsuu}ereDzsoFceP(`j+A!}U^-YetI<-ih3OOA0I1eaaDNm&-=mwW!#owX{W$AT8c3 zY7cHCquSGE&`sd4($5dwqqa!a!(ltWxzB>;5L6so;d3mfs!^%cUzd$i?lk=v?k#Rc zl(v!=W31g#_280Jr#T#PT^^*>hq?XEPzknsGMRC*x9Zn=#Iz!yO?@Yb;x_4BVTm%V z7@zU|CtP$5GouxoU8Dyd9=5fVfv*q?RWRfgB9ICuKE_8|CLY$Q)issWZ|>TL3?W-C z6l)*?f^t&>;-VDzPqJfUtRNgql<53+(E@4yid?7Xj8tkut~m>ET>4^4lzEwgjBwMl zfjdfx7LtKxhL&1@orfYR>jCSH7Vs);GYXQ17DSBKjk|o29+7vnt=Fo}y@j>XX?JOZ z%d2KvRk?~L_3jlMGI+dsQaAJP64({pgxGJ5!h;!#WFP}^j*Jvesw5ZdatidOFc-7v z_1R8j1lyCGh#sW4&cs9oY4;5s1Dvu7tPclqi&z!R+LqIg8N8gVs!UHIsIKB8AT$8L zP>R(CKsSE~kvFarPXTr!+DR1e>ps%9kW~Ma_Yy0J)UOHoU=_<$RB{e0Wh{#W*?>8r zZn7=ks-jLeR`s9@>-mwmB`uH*i#^qJETToa{A)ORxh7ogh)mU2yy*duE((n#Vt%;G zF~W60(6%@(dL$(+!Kpv^_dnMxj1UHICo1~~75F0<3wUaSx}1aepDI8tUeL|7N;eaP z-7!HDk;1d%qDQp+PJDyK+f6h)$CI|zw?Zo14x6DHJlS;zk3r8(q`RF>J9NO*di(CW zJExW@m2NOW(iKbQv3I|A)i1GUzubOKixIsjPCk4G1*IyrlQEy2ykL%R)9Ea2@(5z~^Pwr{cGQi1kIh@~a-Z2={GZEy62Zgay2efElx|NSLEdP*mWvpbbFL49S~ zH~7jY*c0l=OEAfi7H!s&u6I(DIS{ciMRP^}y(smDN`sb~CBbwqXHeAT0%{>O0IDazy($Ivfzl3|}de}lQQP4WQ z;o-!&08IJogtg=5PIRC#Vf_@%o7O>Pyyktj(m5QW@}(b~UG+ZP!cEA8qD_>$dXQys@tYT^@op((1<@Q zm+%60X$|wNipXPXt6f3^lAJ1u8BR+meH21n>9Ze16NSZSidE-|sVPBLK4FP5QQVtX zMF6*xz%Hk$9>vVwQTi*AA(w4JExG067G37V(=bcX@d7OYSIb~C|mWUUPi=>B+%mdD3R9XkQn~7*DQ>S|{E}yrdJ`6^X z!^}R!91IrEV;#X~dx?61wr$1#k#sXM()vKJadujWy6NeW&8sUHr)CG1( zZLsw@85;{nc0;cnGqrV)S%Lv(c&%Zq#@8_HAfzJ_lXnyep@O{87-ZQ-m~Bc>ZkK`>n>IE;I1Pi}Gs#Sj9defHJ|Vw7egCTY zh9KT_2Gdw0uCKYOQnXb?u|pW;_G;aOTxxAb)7wpTk((}IF?$LDLDB~S7~n@6a-xkL zdR$^S?h6#1uz5m~bgla zVB+3lH-q!pF%9|VDi5eI5+wi)mBw1u6XBoc>6VZ^LB7%%gte!gFiDESwmtsxNEaA> zaLNDoPvDOYoF+2vy`CXTi>_|8i}tkl6_U*wq-Dj04|(rBlg%%EdzL0`qAR2i^-I}P z%m~^Y>w8-n0If&>(Mv}N-?Bx9YQz`8C!?Jqz#Kkeu*i17Ho7p(Ow>%<0;d995+jSr zbXCr{8H`s))bbf0w!YOjyfnqEH7*KGuZzV|sV+K9TlBQV2x;BDLMUE?BTK5Vtm}<( zGJ5TFRU!jGY^g0hr%C-&Sy`w5u|5Ao10nbKi25U~tN_JXOTue;(PQZB@h(Po?ZBaB z=nOo*39_I`afG4s4xQVkW)K}CgB%(eYpfx?4%Bi?Jf9H51>*kDih)P`;#Hd(IKNCI z#(_u_3fFqm0`g!+U-{w)6LZfM%Z*%3=IOT&)G7}E6%@bqugXmm$m;&h5qjy zBLxN420nOUkw)S{Pu`;8ehz&1aAcSnzH*F!?+*-7`0&OXti0G`RfTOX%Ue{(7rY0%mZIa_)`Hd8;ZgHSJ49Jg5WxM74%;pAJP1Ab9qU$mWPCGE=*%qdDef3xMV4xIpnZNg&a zN=H^*r>zk1@b=XJI3!+;L%I?1fH|fX(YA(}VX;`whcV7AI+gn-{lRH7b58mW{E=JK zIKPvCZ?o~E!kYs83Wfiwcb^SscQ#Pz6E3ZXR+s&iplE>pVYOq{b}Tj)7oo~*T$xF_ zZZ<6KZ08b#MW0b4@!`Iu!z316Q*l71N2lo7&L!cM_8C56&6h?(cDgcF=GDAQ#%UoH zu|#S4_%t{uK}oJuIjCRpGdW?jmEor(!*J%Wu)Jyil7v?mKbi>}MC~G~)W#XSQJljQ z6;o<9w@JUfmUw6*C|RHN&b}DlV+hhxhhksi$W)$ZMvzK*?qQZitt~w@*l|To7ju;wGKYnsHJHCye*GPa#v;MuE z@(e3W*kENeIGuDzVK6zdUiAiVC1`O5RDpUW6x!fnc87mjc;N{u)(SPnI6M!TR8KDb z#O$p)F6iYf$}akc@&9CJcU?ik*qu!Z-b<()Q8ni=fbj+`DHjux2p7BR=zoMk=_=BK z_POF4RQ6tCFnV-Ilg~*{5Z{oKCz7F{7~@!6K{l&Ow0MmaqpX%Sh>t?a?(Y^>^Cm1@ zAkPfyEROcDIzw%Cj3HG+fo7qoq@I%oP5%-+(Um0D)e!<{D`Z?EW)O%bIseK=PwiZ7 z?^AJ9+B`W;{|#WwC0q76fD=VYeaN--hU4hAh z0G>PwHbVDs%68?>0JCX^}V@Qyp8kqO$I_VYw4U$I^jAINu{*(!{*U<=|@x}|Dx76%$`{BMheg2pmeyo)4lE9>_q&JTTnZt zolp4mLtkfj#uC2t%^}<6ktdK#R+SoR7e_EvPr<7x2ea#lVPbgrUs6WdwRXaGyO9;7 zWI71rLek+X2M7}Z3Lz^uBUS*TJ&hd*y(UV`!(|l*cG-0PP3%?5+RW*8<30|QGpq0Y zm0wJfw1NJGmViR^n7pZ7;F}5+8`}rtY2AAcMRxacIKeC3nMW<^D^IwTOy{wmvi+zf zm}1D5LC+P4owXPY#NKWK4S5&i-D;!(irI_DcSbdNnpX^|W=F8ykG@7?su0hYS}c4- z2}3qY8O~M0(ffZ-iMmPgx1>VjbN?S9! zIOT8H?4ZLMiKRR2?cfL4z+b7iRm_no%U8Lyuw;1WIu<{R#Xq3JW@$v%2j9jMa@}`6 zHJ!lDq6$unyD+LVGjT%=Xd=*axTp^8LC?VCx55%N`n8+<%>BNW@J%uykk=wdtX`uu zOi6a*X#1O$F*1QD@!(mQ-U#82wzUWPSd4jYY~+5^8sfIA;aJ)w7H<|`LIAzPsBeBr z)3U;LU}+co1Uk_S+Hr5l3@56yI9ObjE&Q(OM%Oy~;kKb{W@ZWnf{<(S9MT8C1xd+i zZ4+FOnT-imDlS(&V-Y{-ej^QB0tv=0bfL3SFdofAOB)5_CZ@2=J2Y4{Z=Ef5FY<}j z6Ct3h`xj}Ki&eGLPjO%Ou*n+tY-siz{lStcs`~9bpwvR;KY%Wlgjiy2FlqHLeNTnG zqI+zhG`ZW}Nl;2TJ!Sop2HIvY9&VlNEvqlu?Ki8mE}_BArK35!$A5RDvO6v7Nl@Hq zd;hbCh{SgXBXYN>mPMa%teBIJ&4kxe(=Eg8yClN>`xZ5lpi~?+teRHEPT>E(Rx3k1W#JQyZfLlHM%nuZ8c# zq!n2G7uaqe2(4Yu5EQ&(BxV@JcC4psVWIen9~JDv(O)ZJZTZ#fTzNEZC~_2KrHopB zwWGJb$VRto<@@i?8a(sr0Z2kB(2fapdDF6L1@`~ZZ&SJ=qGf&qbeUPa@y5V77ndtd z)ONHiC?%9K+T8&@JWt_+20e z*NZeFTbt0fK3M9*V?3#L!?zmv%tiuNg$e08gV8%|$T~f}J7XvJbgoJ-e6e?`#E48* z;oIy#4Z>%^emzgj^8E#mI7=x^-WrX_ea08C^n0V%Jo9+;=K3eVf3wdp{W(k_ zFi{ITahD+eZe;{jiLyrWT(_;#oroI);0;A`OMFyK*OIch6I6?Eh{N_7QU=VS)S!T~4(ehX^~ zfmv9x+kss3PW78o=llF;^Y;Fm4MeH~>tZI*8TnjdODB4L&oAYj&Q9otZVA3P^IQ}p zIz&a6V|~QT2r$A*6RR@l;X1NwN|L$)!UBKEzcW@(S8ZGfSwbcHN*wDN?BphhX`n6M zx^9ALiS=sRXBCyv) zx9sa8ET!OiC$I^%F-AqApPuxR2`8pA-)PD4GG7lTCxt2HG;sFV>BZSiohu)9Bd1}m z@TLwR!&KO&K*UC!itYTN^S9uBxO0d#)uH~g2mTigL~t|x^A4klp2|TDl?uXh4dMt) zf~A$X%rj_qvKmyEC=g~hrr z)(s##Co6)~G0VJOE>28FshX;cNNV!(C0mv3s zn0t0|z6U=j0Got1k2f-U<62S6yr|~ldx^U( zh1!3`0``v@VAt^UkFSB;zN#bt4ud%hqKCkLWf&`8E4?tCZt1|esS{#o}de%`wbqd zA_|-Owf8rx>+R`(GELs|nVzY;oJcHyvo_ybFp!yAI0QGAwPhhtUrCm6_1M_W@zukP=o6Vm4+U ztyxT*l6Mqso$DJdXmq3dT;O>Tg8CMX5@rejT!YLR!z^B*7h6>N zCGg!yo@x)?*Wsy0;*Nt+P;s=PU1LXH6Xq6na0@aDl~}i0glT(P7tG1NZ@gPM4>WP| zZ0FZHCs9XQkou@-s2qYA=A`cH@LTiC-yB~+1x2^yu*Z%kM1aDh@LCWPf9`oo=DAvx zGgMpY=bAcRWRI4(#Us)$uhAGCt}C7epmZLHxfCygr^3GC5_P|ZN_Q60gZQlD=1%b3 zt!#3SBa|y*KG|55A#oK-0W<+xy139vPOjBUjS~8` zgSHM~6KMf$xzKdc33pw9enqy^?_7OI_%5zZ)Q@>+ngukRH5x5nkP{psu#)!N(eqn- z{NpJ|ZenJw#HPbCjU<i+FG`QgyqAX}d@FRUpV|%-Y5+#*7LSNO*wzfR(jAA3ox!%R1FjLKLLV)T7Lc%3`VlCM_cmiS9MQ6O5$3U>w+s*oS6G3? z(*6S9q=RuTS{SAT_~=mMiBppEGTNAaLX|6e&!5pJI?>Cs)^l|^&?&5EbPI+5PlZ`f zM(8Om($T>(+E>+Mdl8kvHDl7HVzU1jZb6^XKV;HzLa*IgPmcl@Kpc1!Ak3&FfRotc zMr9b|-Y0EynUwl92By#84NDenky%``$C%7jY^R!A-oj8#t3QtS?U1VL1S9slLxlUT z$C0`Nak#6?jcX}$`)6*a+@kel^iI->w?9|LwpWh|x;&JMqB)riZUWp6>Qd40Jo6>E zR74BQs=+&9)#*+o(|Hdq1~51x-3Bo0;cX5_z|F8y94yR1GuTs45LHKlCaE6kR+W_B(HgBnRTQevO8 z^^hYlzKOw5F^bwY#e`COEPy@qfZ8SHvb88E{USn4QOxea`0q|*t?s8neN^1Wt^cqU zuG6e-eh9HH)**d`0Ugq2hHzBcz?d*X8qm@zI&EqN%!vvrk6(qNDAknpi>$K8*j<1> z!I|RMebr#8qE-b1ufu#`zyf2ykYEJsf#9%E(*{Er$}L8&is=zp4xGXc}nfe-~Cvd*HL2(L-LAho)Ekb}mtCZ`N)rg?e;)4Rnvk z2YueqXJavZC5~w%L~GH% zhk43+e-x9s2Zb!+!mW8WhFXIXxGk{XmK!ObtIw;}ygcO7JedyTKvXW;gASU_4q9f8 z99b>2dj~BV(dPB7gNQL?F4JH@ql6^jV342ZY18o_B$!GIpt#BCF{m~&)iTjvR&_SH z1BT79Ax2dHnE};KsHilHy`d((Q_!s&cPoyLpRADcOVo z%9yJ}LmMqq95yHpvs}a}cHnBG1p7E0A#?ya-+-jbxImFB4T{soE>qOTY%nSIic#yU zJ6J9=q9~xQLaMP#(k3Tv5~#_qo{&Sn9fw6)L$U9GT+Ce|b9C22h1$f(8_ByF?69-t z54M8(MnQ@4BtP{Bx(F~O7B?@t@k5Yh6|6v1<2?JpXYkavLy;-q1nZ)&M*n=Rp&X>L zU6d~z`cS2rx+vNQ*+*EXjts+)xhYY9OSL93vTqqbx=R^Kq_OXUObE@5(z@v zn=$oo3ZjGgVo`r+d_P_qwok6^@a-o z0=5l&pDqNrx8B`4*Uf?~?VC5>ECwrdgzgBU;?|u>4b0MhUzMnc65!~92w#yx({|6s zmy}}$NeQ#7uUMZtTf22^4uRP`NbquM700ilLe=Led>H=F&yd>1i|29`j|x$4R9Y^@ z;=*P|{$t!nm9W$us<>SH=77-Q*fJ%hEi`RsP->M=fu(OdA5SNg1`mW8x3n&7SYxuc zh%g__po*eF#NvPqC_$d6m5Id)+CRYv^_7cJpEt4~CG3wE4|22E7tlZ;=)NJTUji`> z0TF6Q(H7{?Gqv9_eykF71q%2~6?maGZ;2wNZYdu4V-01QV5z0WEgh8O)aK%7WTKyp zKQ*1IH|!lm+{mmR#mr2ho`Kem7@a`R!s6zsl2`W!_J|l!n7SX_JDk81IlA|X2KTh!0BioEh$pI*JecNw@zm1Ua)(Q*#~D)79PV-@g3H1dJN^x8ClwZf zSlV_L%RUlF0Sp31#u_T&>F17u!}0sl+eet6j8m8{C%}@sCNHfzi%YDYp3g;Jsuk=g zc_k8!_I&;_r_c8;=sc?3`Q`;x6$lw4#nyOyM;GmFqDd`_rv^ktY0auP+!=D(xe(D3f9GG@^Wdg61JrB zYjhSLSws_+vXOYTL|L5&`OtnQFBVv;;x*dKsTQ<2+kfcGKe@(9Z#6)*Fw2O{s72Zr zMU+%l(M90i+pcR_sg`Dkr??M2IsOYaXuNk8SR&cM`{ZM_&*Bi5y;iFq`TMWI! zBQq#r>!}V7n%l`D)u)9TO|u?riMrQGEixkm7k72g%iCNVx~J6*3di z>YVFNLSAZ@%q^W{(^G7$ZS;RFAPK;pcJ6>q3@k++otw~U#hZ)m+hw3y&4sb)q!;{U6E8UQ-8Yt4;ASg zC>K`8{XZGmv6yI0UHI(2#`U4Evh&GVpz~u3qww3z-R^o(r+=m~l?(WK**kD9=H*nm z&=+NUw#RNdP+!x0j1RoGRVrK0f223nc(byoCL0Td% z>BQ}?L$EE>lX(zKe$lst%b@!?9gdS(oy3(zgg&NFZp6f@@tE|t+JM%(oU-3OOTH* z^@By2dzU~q-Y{z~Bu;zXJ~yK55NO3?j-VI@CK=p8>iW`^ey$4i4Z}D=nKq>f&DjC} zBsILaDCA_Vl@Pwy_l_kKP*sWV`Wt4~vqL|HQ2MU(jV~+lA0_@zSq;Jrj!okeMXZeCF4VH9UtnU zy=nTM5vfFqZftl3R8(Ae_hQAC6IPkyN?U&mSd^3ag&nvH1KzU^1;AtP>-T2 zsFtZqtS(kkW+70#q@5bYI?Y7F416O<5!%Y>;nmIkp&?k(p+gl6wTr)GAs$5SbG!Hp z_I%P)8Wn+l`~H)b-8agiWs)QfrB4?P+>wjLLg)S2aqUX+rdod$BO^tv7NNf><$NJ; zGX+HP4U-<`VsmaTEY(BY#^dHH{hUtNt0xGLGlEO$?xwB#uy~?3wQ%J0i^uiVb&Mf* zg!PK#%(YGskDME^(qtyMSmoFC3O4nU^Y&LB$6AaF<~CS-wK} z6u3WY5sZ6~R7~>tvr)j{!LnePp~bokq`1}QZn>M?3SSO#MSzov#|H!MtU{a#yL)+o zyz-zHypr)|@RS0JTU(*&cKeIiVFa%Jx5;T8DJ-%Y!4gG*ak;RTpb4>WpQlHwLDZZ> z`(q+aljKOJWWq-n${AIs6}3UYSfQ3ZbOJL8YeD5sN(Gtkm)WPH6%tItD-Wh@g!;bv zRZ^m12}bgmCLIKpys^5!L(x;6%saH)SpAU=?Sd7QFpX{?2Mwzm386@C6d43P$#wCF zrc$C#$OFD#r-uwi(E3H)dS9s(zP#+vVagGuMl5 ze~5ldi!H5$6uVT@VW~@AN zxXO}v<;FF!WIZw3f?1Cof`U>-3F33a$&@P^H0?*wqYJu~VZmLtsn9E9;G2Rx^U!Cw zb>8;bis9T``alB83kSrENl*+8K8*!-ERo%)XmP@Q8U6ahQV{=*v0(7f4Xbp`sI(ge zdYPBMolj(iC`3Q{BVW~-H$c*?=jPRmh^%uUm?qHe+H@9qWDQSr?!K)4D zb;91oZ$RaVUaph+^`IQ<4+0O_Q6q6jLBugWTE7;^2qQ2`2qOqlct8E|blG6mh>i5| zEp8eSfAjJZib%2k&DnR!(VC1kyLpx<+9yK|Nk!(&@Se*+pSPA>??l1sgEV zYUVE~wM$gmd@{DS>!~i6od|D$qwj(0w81JHp|8iWR$u#I)D_O4K%Uzk%Cz~KKt~x` zWQ*XWwZCvX#y%X_l7no6YnS`EI9RJF?R631D>~J}4>t6`%}yZ3c70=%>fpJbd!IZu zs}g&}Fhpnq-lnnlpGY4)IBl<3%&JH6-K)jUVnF93fCPD#k>=zwKo9CmUU>wSr+aCE zxYSCX_&-6`a$T73iI%CFoT0_rPjm)7|5OVH6sbU*WQrINvVC1?{ao!9CsT-NIaGg` zlXq~3vm@3-W?0j?IDzOB${sxz-1JP{ik})@Wp}~VQy+nQndEb3$D0&{Pv^f5!vdKx zIVb3Y$^2i>?c^!gs!JY!{r9tgEqeslO{*q7Lo(y$X4QGMQebJLwd8bq=peiX6KYT1 zd(~BPQ96_nW{ZMiOyXC%pUi?F18(l(mE@CRj5&Swg8{R09A($UMU8`2fC@lX(WDA5 zTu+llj}v6B;WgNpr4yOGVSTDyB^Eh7dJD!lSGCeSJe>=btrs9=_a)&4mR|M{TFuL} zP{UT>0mVQ*Zp5YK@EW!`v%_jySen z3&g*LGn>MEyE_kqwDlN%g0x||6UCqzB8tD*59g_kRmCi8H#~8iW?|3 zD0D|TLwJrxN%IWFPSS2pR+GxJG!qS$=)HsT=z0Y^k=H==laiiu z|8#Eg`m1^S?ZoOqaB&&7&L}S-76>vDM-;?3E zG(l#B8YkCQvz>>Ufpi@i$v&btp*qlYH|SXe#%7MJ*+h?xhoA}7wsm{A zv26|Tu!}^RbmFb>K~N4H??~>;G$G`yCE(Oret2SAMl)`dzE6*}P#00Db~!i>Gyn}9 zBKVgyAkjJ~z#p%>Lce)zpV47XSHsIe`uEeHij(M9VMJMp-eUec5sLVci?IEo!Pa$p zTD#(oW$XN#BGQW~zypm!OKci|3>`EU6(yGuy*@tfNSX{B+W1#0a24w(st2MiwGzgp zyen%fKJ4i;sj~MB)wn1uxW9^e9LWs`Zy`k zD*-wY=UMI((G=+Plac32iyuKw$`y?IocxDh1X_e*uvD6?-DD)FUu4h_-iPqnheuw& zOn;KuAQTIW&6*x&;?*uUiKsA;S6*X@W2gz7kFDnGG!@wVy#?Ks;(d!|s|5M+fjd<1 zp+*+8juw*J{BvQIb9In#_c+%qy*<0&TS@B#7nnlw7PdtNwnPOEBW^ybuLtt@9r&$f`q(YSK zQDViTnOB@TDUy6xvK5GBWa4F`g6QXM_huNB zl8YD!?7=7FldW~Mlq!qFfe9#T(E2M%9z*|~m@#f9c{!uBy)+;bI1p-+oKGj`)B6ei zy=N)-Jm0UA;9ofqv`vd-y32+)Ep(ou!EgV6>-j;ZYaj-}eNb*7c;0n-_W9gCT*$OG zul*o#SBoxN=IxhP>5U&OL4qZqSaO!RvgnRQb{Qypx5V0*@BwgtUlb8y2%!`Gh-#M` zf107_$Tv8sC41$Jp;7jAQ*^p9T&n@)5$a!-EruJ=;0{>3p}{u`vN^+V#;C5h=h-f9 zbfRrl{v4}OTr5mL!V)8dA*&}|?S|jOlsy zw6JLjjzwb_%~*Ee7>m zyFT!!`HHD>XTGkikF$)O{f9DIY)J4>dsNP^KgVFl1=O@RAdfgNJAQV1Q;__lSxgWT zm&i-hXb9|y)vCIF{R-ajK|0Zt8bm#?*eW{Pji^KS2&zFx7j64p15%-e>mx)~S5g6J zB}2Uu^Nh)(5{iM)`}BiF!+>rkY0gYalxVDwXbtqk=N1_*Dt!(VGK5x5w&S(07X}{1 zluG#uSQ>57Wr%%%82SUYY|>jKk)qxf1ZX9L-!8mHi9lmtH0*JA^a0BXEmCd7zz7JM zR9qgYK$iD{4`+b4%{^j#ypQebhvYL%H%^e-?_8%xgKTFAiQt=YUZ6248xbPeA`CPb zph;Ie!+gJha_$qH)(wAhl0DUoV~Hif!h|)8VvwK{swC(O zbTDP7rpx>S1W!ScFP7HYKe>cT>b?fJ!nLVBP;vxA1NI&&@AIFVp9N|`mQg@ur@k1i zyJRPkXArKuG;@ci@v9K?N)O@s4z)ngod!<=PH0Ywn`H@kqz!UtSvBJ^^XPBC# z{FxAWdvU?YNF!Cg99*?_$(`s9<67FZJsIjN4<_iQu|C;phvrW_$%R_$?$pvhS1Q2; zs-_ceF~TB%e{);2&PnShf~P?quMH3|AuxbuZIslh32%{UYyn1CtNBbopsyWp)&_y? z0fKKrPA1CDh5QVVGBXmd6>p+RKVpE$nh#RL6Onv@Rxc^ERudMe+EPJVkBayoT z-K(&7LS^Tcjbu-Bb> zCg>jzUV7eXMQNRb<1#bFcd7$XA~IqF6X}E|TcDg6R7VU6OgC@N<|w|Ht)@n+hZkSB zYRhKnq$L4eiy_eAU@J`_`W+lt++WwKj2BVP9{%30 zZSX_tMrOs2oMy_A-TT`??o+5GDD^!S=37~`j~q_jsYKzaXx0~8^w5mL_)YC_Yz@g@ zEux-k#w>?E8?0@3p|kS0=6A)3ZczSXamt$)fRQe8vss!n= zuN_n)7Ml5lGp#Xr79Y+8pCBy;>R=J5cXv3GgJ+SW5CTY_uR2KgiHhcLwE}NDS*<19 z^tpm6%KMAIkTLr`#v3?_Jk*GMNsdz7T?ZA##qaHcxor!k&^T|3beGPYb)zPf8JTN> zK}I}(_yCnfZvn{7+Uz2sq&I?~A8HbzOhrnFd?y#(p0GzN?zQt>xht3B>0(Ng9I34JwBQ_SD=1J5RMTg^;VkID4_gm`v9&htrIQR#O(Xq?nw$Y3*t` z&tTJ&i6!O$RK^rec+4WRcRY*%{edJhGnwkX7%?zY+ zk_6e)?x2g%WRtwP`s&((muc6aUT7HQ7p=hC2^vI3lV3KsFQsTQS{y1|5T%7ruP&VP zFiiy*S3zOdczr;SID;)%3B(zNe~a^=3}|`2uX-Z0UG;-#YaQNj`w#_jPy$)QH{AOe zaM0L}lWpA}cMGVx>APe=ST^0P;Rqoh7Dv4Tf2#tIEhi5kt_-!;`s!EuEz@vgw1LKa zEeP$F-8D8cEdgZ}b?S$)9GykYv9~aj{k2R2lepo9Knu6+gP1G9XtXlUcLu)CmDpG0 zjmnpWt;zRyr5Sl5_mJSurNlG^fs@j`NoretW<52zcT4hpP<%K+!<&(|$`2OKq(ANk|2ESs24iF`x+ys(>*ytv?0+lmnM#OXzvbhDjWsi-h zEQH(QKU>xXbiv)Wu#?15z?_&`rB>JW=Q?8jMPm{hI{i$hjku{tZK(t^a9sVzwk*)H*u$rG{CgsA44)Cpy0e*(~ zFeB(-rL41Y6f}6cr^7AY8Qk*vYGdoSjK?0?5AA~6JN6T z%cF)KDkJ*#X7XXu=eWxz@pIaA?X-Dl_Ou*P||ff9i!z1>l2s`B4#9s6+@Kyp#szu6?24zE7L~NGMeKS}LV1*2tvL!rzsag?uaUi4x zh57ZvvC$-(D6XP&Ngs=6i3<;Pe+{(4s#CttkVLG6Rfn>%;BGHL5oi`9f*|tg_gp<4 z1yaasfr1kyCEe}ab@9Tw4i@9ClV*!uTs=3ZwCZ=HO5lRna=_LuhjoT%F|(yH+gc9& zLVTGwdD%2O;}%W{Hp6EuT`YWNagk=gJXwAf;*EsL%$AkKHOC2Tn|0zB1Dga(W9EKM zNevdUiz?fa7jFpdUmJJXAScJ?u)A>Oe#J`}(Mb-P$2oMFagOqRdQc%Z7lEZcjNGMP z++gm-D@D?dIYS-Nu|;?D)`a^cseJWf2*l

yCcg9a3PfoQZUpvGM6jS_8_n{HeY` zT0&^hCBlWYZ#WEzNUj45Cc+9O$ChTNCrQ>WT@?#7YKS=SS+Jb%2>*I;3UNhMik)M>&9F4W!HT&%SOo(xbk#J*=_|Q5SpV zAHn5L-bH`wEqU#IVW+%2#TB%6`Na2iZ64mSq|q!FUY8=NVV(LauAK@IF1<>F07TSW2Hcrsc}Onb>n&S0t_;;gKag zKvs~zDeCXR4w_XSCiS_H@lT&4(&<=*z|=%su8LRuItY1)x315AuwS^ki|WM}VSt>B z#*^A*=$)cT01+b3!A9Ocy(zD&!9RaMN*1yxuP80gl z=eP(pmSb`mhRhr~oc>7>I6xuzcomq{Ej-9_|>gOSnQj2Zw#ztOm$ zGzm*WWn*81j7`UL3S&Ad6G4HS`?pqQnLm6`AYhlEmN8ti<#ugatz@c4!*L%TcUF$-GPm_#;|W z;Pjkf$;S4mkY?NxkD;DSYp-}OJa_gMp+7FTSZR@F1JQ}9=*j*gQ$)X99WLA-%C}5b zx+>MBRs{q^aY98)fyL(QHLwKRHxEVdtHQ+$a z;KR(368dLv@DSG69c#}q7^Jpv<>TSfgPid-IlC67dYB3=-6*N7qsDj@{u>g67J3V! zZe$0Adq7?YWbFmghhzPkoB{ln=c-y_qvFyCY6Y1l<-<<;P^My2p}CxX;O$aexA(9% zuFfeQUA!L4GVa|6hY};Qg5Y#KB5(2 zjDDs=nYB+0%jQ!Bw-|Mqtq3ttJ9Tif>=5bbdA{EHKmZl#h~$QIOLknF#o|If&q

=;i#UO|3se9+{Ho@U$p;$)tl1RXx zFJhTgzk(>wkEX_!^^|Re?B~8$1-1m=NqOqJ*a|<4N&7l0E zwcRTX`d=9i*Nzk4zlw+qVY=)lODe|RhUMqI+xsbM?9wkSfGf6-L2sro;PMLQ6vZoe zoqYbCqt8ftLQ9|I6#OC$3l2FX#rn=_Wp`Z+hK&O2jHhP zC6t@rFI!A|6|87R#hOW1U9>0}1XL;9&rxyP1F8@K!oNTj5HD&7~KQv?!i-yWt`_gs1iy354gEk(K( zKNbF{E~%x}wE)?R_B$F4vj$;q`pgfNx_R_!_*x~sX5f(!H?HNr&xLtN9vPyP#6#uj z>oEnu9{cn@ecTvAPPS$opDXUW()SYtnv_w>*iSWE3y{*qdY4oS+CVW~P+jq}%XYpH zsmHm<)5gi_hsX?NGNJC45CfbXFq*0exSGRw5ae3^_G^$s;PsA8KX*5{*VdQ2FxbZg zzHEK>*t?5&Yf=gU%8A!Hk&5H5x@M+1$()d&2k*JJ)UUF|1~ZT|YecbQ)3hvJe|pj< z2(&|tzus|!B#0O_GJ3T-9QC>x=1PafNoyKi`|7Av`XEM>eIms&`zP2?U^XE5%jKjL zFH)Mk>|Y?6E4JDMnJ4yDS$WwLc&YwhKp~*Y{>oD14-Ee;VRKcCZZVeto!mmfHApH6 zsSl*UuKbzUqD}Ks&TWqEim|V$86ovc>Xoht^25i#dS`*`^v)A zym>?iLkZdfsnXKZwKt>puf1!LBoV>TXi&bsCgJR+N#7=6DARpi5I}Z55Su)o4kW(8 z9;CbNZ>ITMUYlFc{D|z2OCD@pun!lOqTp0<{1I}OUXOE zy2-{-=x*8{hfS2?=|T!|8e-V@s{4zH1AQgW7*IxuumMmNMYkH1iPet3L7!HobM4pF z5T*$P?lgHDH=6Md+a`HF?9ab$z&l)ajyLmdEX&z|;4fs(R-_j0(^cEkD=y&0g^<<= zVEu*~s7_ z6x*%ubl7(49;ECd>KFY)o=D++lUQmE)Dj{n}kZ;^~>s zX7{T_FqY{m>_d|+DNlop<}3&{!;q0ZB2AqZS&jEG10l_QEfrF+E8SMwvNKocF&cT zcT5YF6PPNY=DjPY-=+DZAsi!VIjU*6{N-jorFobNzw!ezcOZBG6s$fB2MxDA5-5PURU#l8{2kcTMgRAws~S3Cv0Qew#~*)8l2d+ zjmAlv^q%&6e)ryI*0X2L?7h!lXV0v8-vxp8?C_olJZ50(KufQzm9~zt>m4T|5lHe& zQSL+?>){BS#?!(x!=P_ug98aAvtr#fMAnwCNkYA2Mp(<9nxi6Mjxc1^kk1yP#yvDC zXLnG`qmd)q4uy{RN5qGL1B+)aRnla}fraRP`tMg=vftwW{aQH-CmJ#sGn#XhZ^5C;9pKNCNcWP72fCTzN)B1Rl2gZ^wr7`^Y}<_xi2l z!FH~n`S~+Q!`kn#xK0#Tfd}kq;iiycE@rT;Dl9%EJ}fb8ZD^)%_Jvh|Re*ZKC4s`J z^+(JVyyxa`B(y<=(lR72%nnQ!8Dz=>*?BTNhS0Penav&lK)HDlKJ=u}7LHs^g1ldz zW&E0kCDcFfOIISb)uC&Z?Vf9Jem>;ze^)NP-26TFh?Y!^<=jq>tBn& z-CI_1T5vlG`dksEh@ZjANlK3azWS)D06R`l5(p`|)cwleA><&ED4@h1!p@Wy^D2cj z{3_pJ6iX50{~J`itG;#YDb=yg>J)w#w}_a)%An!YGvn>z|)N4-70#7a3>JdaWzMcEj}qKzFNtY6gHt@vlLzR=bGKF^9(-|a#c>Td7D^>YfL;%FH>V9 zNvYAlOK)euBQt$|YYrP!W0{5=l{zYZao(eim0Z~Se0G+(A8vP0u=q&5snt0aEz=7$ zIDu&mXQSe~ZGw8PZDN>pQ&57S;2wEe@xPc_6gS6kDVm! zd`%0^f0$N~oov_t4-9oLoO6cXd%&>#w_DsZUq_5!e)IRe+U`P@bKMsGqs}GU;%PY> zPBZ$r4M@9~WmT)!uad;kDAnLd z2m(9*ddOb5_b-p%AXbEppaWN%S>0<^%GOMlJ68UxQGj!>de*pQLRU{vG9+o#fT?W< zgGU?yRu`R-1k0Hz5M+tj?RT0cRevFk``bWQq1N-xtohqQ*P(W`XlTQkturcM1?IOv zuERMHx7l|JV4m>SuR}OOciX2q=#DY0G2Fa4->mmL!uOK?cK@gJkUpn6*CZI0EZU<~ z2gP(2>xg1JyYJ|USl=OQGsC{l=2IZVem$M(<6AOlS$X(z`6?_M9nYesDMT}$T=K-* zrMS9~isG-BIQ5l)(U9&5^ihgCS0%*7Z`-WUp!xeT!IUBP`;ptYL;Pne`QG5_TIdq# zs{1ilS-auHB4X(9+Jmvs2jj07#p4R=40PZ$kyo^vfZDKjVL2pXfvr-5^#r;(cSUG{fL0j`5CVjAIiAxip1j+^=`Y1945hA;QphA z=pZRY(;3LV&L>Z=gd*rw0{CZQuF$|xrEBMI#_NIF6LeU5K#EuLXDUB1Z#g%3u&3+xnV^(fP{X`JMqy!#D z8+!IsI?#P>t-9b#k*e*l?wg%~%oG}!0Jk_xQzQwa%>cJLy_ddD=xemDuWaOy+oa!~ z4Dp4q}C&sTK+{+!7c+wzI#m~;{GZ_xCpymdL3q(^WoWZ2I7osr?ahtoQ`xzLf zWv)~hVTw2&b#8oo1Fl9*f_i4AMCpEO_oxT~elqFn9B6G7%s1c2H~F(_Bbud&E*=|n zohz1@@>;vNg7m7MHtosFT{ER_l|q7gsO4Fk7^i1K8(F2@H`se2JIh(n>29a`zYd?x zwrl#vvZ?_qZ@ffw@n~v~+NbDbtXY2cmWX-PuOoe<(HDuLT$L;15!Y3E;j=l%Y-dYU znFTsh?n{iKz-)y6Ce6;ctX5BQ%Ob%WT-}>19JrRbLYYj9|L#n*$mE%xMncp`4E?PnfF+U-35zVF9rjp%U;qtw8J<_Shjn4XwG;)DBQO{f^>+>UY_IN<6Q> zx?6v``EX*__dQ_x5LP(>shWJP&#KD5VZt1`6g8XIe@)nisl3Gi3k6Ta7P=O-ix*-& zJ(u?!H(jDdZmJ>E)1w;)5xo>8USW6>y`Ss=5%&d4?_<2e6RaCMkNF^NR~uz(S(whS zdz!Uz#8^n|W)u2!Mug~ir+zUed8ND}C80e34G`E%lL9Q`7pMQ#Z*O$GmNUjw{FSiy zl3?}QX5AYjcRL!1lq0xei_dT8w0|@e4eIq zf%#Ry&H7d-)rumdxph;U4~A>Ks|S!_M%cjmV<4(b!_sFI8n_=pRr)w!wyR6Ca}J}# zw7}wOuDzdzA*5)P#dp6 z6ui&INl0L(S8_a@EL>TE|A96c#d1-)ugpBn>dw2@g|Bi7EQ^Mfy+&?TdzwQvk;Pfn z_N(Qa87Pt56;z(Z3^JqYaOP$DLULwCe$fJH=&=!d-W$NCE!LVi0b|5!t&1dyzKR`n zCtc=k(lSbZsyxqwL68>C!rJs_{8M21&mTHBR6s>D+$2Us^;`)PX3+dS){-V@`JTLk zo?;jUnPgHfX_H)H0_aQ z3y|@-On=k?#hbGDvTG}w47GzXA+9;QG-sI-dch||*!NW&c`6%A;B5LUdWSv+q7|W- zy!)a2Rq1@|CnBN)<{vg5VK*`(l@St}f~C2b^U3kPrsAM1(*ulzMzZhv7c$WySvr!= za=tN9Oa)bhV^Jn!{FX>sX)kB>L+*1D;sWt3u;!{bEh!BcT9VxQNOH1G|On&+XXJ1h(~^6G~uy=?@s z5z)ZZ=ogbCR0@=zg;WakQV=SnV!+swHo46#iRi)Go6{lfDZr}e2dXQov6ZRMoA;!^|1?w;R8r`+hF2qHy%KBjNQ=l2yjo|q6 z14)z<`17w?ny2pSmVQgY3ks<28i}tY_|2|W)STtfxzv)4{z7NclAms7Kai8vH6p&n zi86RkQOk;|V1a1=RMx`W9$w50;-`2EXIu}%+AFYEI^92IFTkzA1Dp$%4XHOSGK4rV zX6(z$&5oIc$OW;_D<=0R-Wm8EuqFKI>2xU_WyChz@Y55B7%m}lktcrH9Ao6l%dB)( zQBl#8(xUmC`_-D|yd=X)PCvOe=s z$I?|Q2>4M|%IhVTK5RDBC9SAhm#8tJ@oqCb_6S@j1V~AxxV+4_8yXl-xZZVLQW{&( zGqhz|mSX*`k_oP=(mIzL!(w=F+aEa8NricMl+z4oUMM|kShTK2iayR9H}i8*u(0Ip zr2S3>B{BRJagZ5&iYcuK#L?coMjK*9{B$Ot$b&LDH^b~!nok4lpa0%e-dEV@{h^I# zg=a+~iJN;=V8Yx~4fAqfhMGem0T4l=t$x6#K~(JX;lsLARP`V2hirl(nJq(Qj!Me> zS&A77Wym~l%o839C*fRA>%@s+5_tqvgqX7+$N-fhv;cupG6{%S+F*-^{@@ZBORPn` zy>*Q3&=ua|?-V&Q9Cd6qrL(H1vR0bIFYOv#cikC2_}>!{_hHaHXrm3?Le;;g9#_8_ z*%AVC;8C>-%BcwoQJKCQTkuA^dTKkMX?}#S%#!<+wXei`9wWXKGq_~G%s_)!O>8_v zwCYc+^G{^_h&#`zmBU~R;8F!d^Cj$LrnZKQrtl)6?L=cyK^!mSWOr;?stVRV4-Kj_ z>dBFXg$oF{(g$<6@m1;m_w=ujXOOZGjUQD0h%8X8B%Fsk7xOiCx@&9EDK%m9aQ@#Q=hPqZ8Ch^-cw2{4!1$Q*2?4DN(t7vbN6M1kVTC5*{A&z zuQ;p>B|xQue{%Y>qOeN&4LljCMlAb;1C|bdG8dh0WHJ@(Q)HszTJ?@H<-ot>VU^Z}Zrrq;Ii2jZ2cxwaWrD&vJ1()iR%rZiS$1k~*tQ1ca2sJpIO=0p0 zfFWg$hXWt9F3xI)Eq&80#RB3$y%$bzEqYR#RM!k}+!%fCBup6Zox_nJa7}M_`ALHcU@z0|NGCf_TEEKCGPH8K} z+h94e82{zECNJq;Sx~EP5OH;iCojE5&DTc_>21MlI~hJa_OkJ$lJ6lJP1aFa3BU8( zFol4+(KN19g(w@~lXiz=d4c7J-woNYWjx-3Z%zB=Hj2B;y zeGyVp*eb%beo_N+Nqz0=Dqk-*sIdiqzOUphOnYzugRgIy zrT6a+@*np;@lL`&JI7!B`uUMG`LcT^ct(>WI1$3SHwGd9wr~JvWoCsawJQ|1Zte15$;)^RNiEe__hy;E*|=I5BR#8yqU5PSQ z*uGFuJj${J9K83Qo*brDY~_qE3(h4|CU2yyh8sTRkHZ>t74`^(!IC3dPCnUSkdZ&4 zH_Zv&EMfv;0tJO)IEn{lIR9h6R7(`v6lBg7E*P8vm!z8zhmKXY)fy2ZU~RVPnq%Q{buC&Ex4+ zK6ELgKY}V(dU^+gHHR@j(Gne%g?gR!%(Quu=^t#$AiYpzQMXghCyhDFcjT30JxVWF zO+QF;@+FdfchNIYuc995e=0aTP^Aup6p$BNv??b+U9hDWK}9VpsH6_KWv0uj(<+O5 zC6jcrYf`O^13A_x{z<8gW^R&N!6h?)4P%Ug-NF9s= z=T$dRo276{fEZjLLAtR&xJo^$LT7>XNAAne(#@G_;^W__w)XqFKE5InuahTyMaR*# zTmqAo%hB(R8dwlez{FWO$zB4$ZdZr?S`f364kbaj0?knRfue|N8F_^omhg$EOfnV& zja)o&tew+ffRwX8y_1(dYuY0l+Vei-c`Rzo;^0x|l$ zQ{fjTP^jd(qY6b&Y?XZIQdPlXL-ZKQ^}BPT2AYUhKhb85UoJY;*1nD7Ph3 z-ad(|=9YC^w)oRM@#oFfB21A!4xRH9dWvdkBnD;6ZwtRwMKa2+v}+|?sSR~pGS%Ya zYo(rM(!39dQq;+@*Ie{WEu+N92u{;{OZ2AW2{f5nxz2=7tHhq{V*p+2AslF1r##1tCQ@nCX&_ zLsh95@gT)gLTBJ@_)Mr_YU@2@AY~zSzD8#B5+PS%9k|`_UY7tWopxhgcLA;qr#()Q z?T8>Buph@S-iYjM=8x5_IOE|wjpPQF%PjaSMjev_U<$Hsm9tlHX{Bf4TkIFys2BqG#>`%!^Dv-ClYiri#PmCF-;(#DrG)>Rs9%xEIqh0YNmwIr}THl&TWj^_h zMm!@uA~`Yp?%{T-pBpFFZJnZtCU^<2E_Hb>mwi;8ev@`76s4Bu{zA)0%Sp>=W(Z7h z{?|s&)eH1lJqliHUr|#v8Y(DNqJ-L7J}&O?N|g^OOH8~LM2bZ9Hzc0~pLu9pRIX?= z%(x|H(f`qCe&xQB7$`IWh%^DS5~0<3*xwlVnl)u9hc?SnnM5G*O`N~Cv3@`# zMZ`(2k{_SY^n(I;tVWrm(U!nv9H*v_vaKhHP~#VoZBo{{$H?dCZq#N|5grs+ewr)E zuYZx&5|JA~EJ2ddKdmS=59$il22upU^NTFb!SeW?J9R|1a%-3T=@}?J8RQKTpl=HL zRLS*W!3W*Moclf{U$n$XGJguXwQx$sA&1@Tzh>5|&AXk>&s75^UNh-4o8`W{SG;S+)V%5`T?Gas1$#h} z;Dek4aPf`Ql%a$P5DMaDXM1rw2mM2~Ghm(%DVjq_pxPh@Z0{G+LZ<`Y+@Z-GPgY==+HCPLZ_pY)}V80Ul&>&5mY%0XBp(lid zI4VgJ{yhXNsYJg0YY>)HD&N4tK8+%o2vh+U%>;G^r*s}xu~LM=*3yOo-qBUp2U<5< zOU4D#hd=tsHrD=E_` z0Me;mkNo;lVg~EZb%o`4_x(SFF8rtwf;EvZDNvlm5!egSqtj0Y%rE<&1=c=pe0aLE zx9HdtsGftic>bQ-S=nJ!qS4We*P6Pw1l19>8vb4zRGYGWq_KvwF-#Td&a1xFB4E00 z+W%!(nc@2ax>u@|Rcz>yRk0`cH)H0TcTOBWwLsRCJ2_e>s^wVoSWT!0wG4BZaxW&g z2pP0CeghsfkXMxq+6sSVd=WDLSUhwfS&6?0yD*XzmCMYmh!i|2DG)d_3n>Lpg4@Oc zdc*cQzoE_W`}}82lK7?dcpC^Ht_ZS#%LkzdwGZ@{8SbC0; zk#8yr1UEBLvknmOA8?(m_$TroAqD_Xfr&8S6h;n(fS4KB+WO0iiCfI@^HaIdSfyMu zgGkbc+P_RXx<1~&$EKyZp*BCBJrO$HjMj5S>Hz%eCIgR~NylqbDGe)jy$(I0^D86&I zEx6G4A(T|eMSR+pW`msyzb(DY5A302jvLSDf8qShh703pld5F#vcQ~|d8qdN8+b4y z{jall`TslvD8-IeVPTm6l7bTdY%dm0@q-oNKD-BYV;fa>A2e zG7ypZ*V7cGgOD&HV1kROAck||2)`Zyg_!SmrvelO`wCJpDh{vFN}n-wI4U)7{z{(- z6MKt6A<|NxDRexlI4}8XpBa-fukxzDIdqzR@=efAxv@qrV@*w50z&!t>leMl+8=ji zrRlH5jn8NR_K6u-Z=hn$ZFK^_R~R@>X;tn^1+@T6caGjLY4?Ze1!}$b!e3KWiSzvX zfTMux99oK^J^GLRgF3p-a$!CC3JW+I9En>a5p-YNsQP787+@=)MzkKEVsd{-#I79L z_VTAsuxkQdmamYVy#fMENv{~36hzK?d01)6XNsuVUBakvy{TCM9?)LdoXWUdy<7#< z^8#Hs9fk$wbs=9*n)#*%dGijvhfuW>0XnIlNHEyR>^{X?8j3@7A>3=ds7e2>QSHSJ z4FmbTkmG)S&P`k=MS@bgXgn|yoh4t`$;X#shQdLDZmeNnhp>lk2D5xRl^>EmfOy&0 zHAOJX?fY5+*5RCpLH5F}HLA48k;0D9J*dBVAvT1apa)P&1k?X8)O*rZ}X8)u6 z-ua#{cUptC(R@2oDv?EY*5$1KQGrs9?oJ#1&TNbAz~j7Ilp)nTIX_m|4(=1x0=Y0& znzeps_>t}u0W^`nPl45BYxogT5|6(lsiH@E*bjjr$7s2No>x2VLH|?mbHCPo4x%@} zLVa1js{^)!dO3M~sD*Btr+f8;#-n-*dA#t%3`=ue%r6Fn{PwF3@}zBjmhT~yq!TSG zOXd2nfvEh`YT~vPCarmwm#m}hcPd7CEQHjkD$@$GZ(S16>;eoE*J~vES_WQU80{rs zlQFMfzW?NDzUH^u2%5fr*{tlSRQTG#D5NUby0Cm_xcS~q8+Yn@b$%!&GJ`VoJUdd` zq2Lx;Nv|_6;ol^8yVKfkZYLkLC+5JYIx|;Lhwa&z`vfv|iG}4!< z-|D*hen)Yo5qN>tGz9BJHL#`}1Z&BuLDrwsOEaRPzvqG>NXaMiGWT<}&WDQ44y=-p zy{j;+lFbm1*F^vj+G0n4M1Rq2SuzcsfR5 zUCgDhTIn>XH)it?YVGLQDC^MvXz5at((&zSB*o-B^ArrlX_lsTTA%^9kS3#+Ja>1C zuiDB475BHsSd?kBbcmIP3|A=3tFM;Yl`Lhk5gA5pI+I=j@D{dE%g2Zi>H?M@wJ9~=}hhMiG8g53ynXAMh-YWwH|r2i(BDxIKC*8>I71KB7;W@9uhaaJ zd!~95ae?nAg?XwfZB&ih6fD)J*HmF}?KX=Z1IDa$Ja@Ql_3&GzA)j2bWvw1@;m6TM zl=zKn+lgt2R&lV=n5i%4?s9fIFccHmb_itmZuO86(tX+7w)EAuVK~3kSdMNLy`<;a zR7aVbqA?4Jb8Bv3`&_kM+Sc@&lS*fEyB6bGrLG?L)tYH=I@O}vEy5nVU)C|%qyMUx zA=rB?Hn-6-c&_v$ngbR4m+JG!xrQUtF|l5tHhqAFuGoi+5wV@b617Iq94}V-q}WPA zfjTkF+VlIZ9JxJNl5Wjh9}c$esE#`VknG}toCSpug)v$ZQL4ybw|0l%fJ89c)NkBH z<&58co^JgfMKsO<$)z(Tahw~zo%FyRV;$1Yp;Ynl_`o+xmQoC@jr<-MkIX73JR)b# zn$w)@TYbLyH^Z?U-~!KmkP@uO@x1dg_1Y|JG$Q(Ob)sX*+6p< z1YMH127#5Ybqaw928I$6lLL2kQ`(e&9)B}^uRRF9u*qAytWpqobmuCdyO+;jv&PI!YcpgO(HQ9H0QUu(%T_*hK%12YJw5sm1BU=}zzx zz)2=hwvrg~AlI2@+f0AEL>tEb9B;7ZZ($xuJxs6uhT$K#bHS+dX)0*yDdt$$hZSlt z#T`odaBwWEHG+jdj>I+AbZV5-6Z&(ipN2$a^a9`f^bv#x9)AlVjH;WpjF5ogSwiWn zLfipGneE8wnsbMHb}eaH(zd9c{4(BMEkWhqhFP9jnZQ=wU1F=2QQh%K!0N*`{Q|V+ zq$>v+0%S+p>9Ci=rZnC7OHIF?tk4Pw7EYD+nojgV<@HwOese@!Ca=nd@O(7$2wc!Q z8<~=E-cwxiAZT;L_~AxDICd=N{Y7UeOMcV3pP-m{!RrNpf?FVZ9?SnXbN>FeNTOV{qWzyF2*haoaJ_Xjb8nNFd`OAR=oAA5SGw6 zsGhtxs}NSu1vTWa&OBz7gsv0rqwqVqTCN6@y9Ay5{JOHN^tLke{Sl)+LkD%n0$M$3 zKjz}(wo7?j%8{jG%T}T*v&K|mBN~xn)6s8?sz_P#1bgEcpB#~MVc;ej7Mcqi?me*U zE%_TKK-V3%-p;nyb0s~KQrn~T)Tu)7lc;(C9a(_XszIMZ1Qv)??B0B8WFPRwIKaUkl^w$NQt$b?Hb>2|# zs4PJ>wCqTmQep$lcYMFLPYF77n^jdKU0yF9qHZF;4tr4HC}#x_b|SwCJXoK#Vo-GN zbjuRCDO{9)n4S8LIyNb2j=m@24*%Nv`+2v;o$)LF(*aN)Q`P}nfb2z0e)6q?3&77{ zyNn}{*@su#_=W1Xd~gUvCkZYuMtuOp6mR22GKLAH&7l@tROD!Xh)W8F4aT0GZfWdW z&BaZg;lO8YWP~rfEEFK)mV=L$d@6*@V_gAeYNLLnf%m1(&ww6DK1QS5F|0t+e+@b? z8aWn;IBR71sKHr%CH)gGZCuQ#sp;*CVPUnS9U1t}(ToRp_`f^L3N0~{rlxl*8^c@w zn*-l=6Z{IETlvKMWu7=`WcZ+g!^=*dOz)}n9+r7Yegk6zgW9@Wo-C!(bNRtx&oDGm z3GnPmG7>Pm!4E{6stihR=%;XtC*YJrcx8rC7AND|>%)Xc+Ynq4 zfF1eu%k22FKG^}+m}&1SxG%&n!yP)v0PibvIW zuaYERViuw017l3$@fZj4G_yp=s68k8UXFR2CE=Unvo)cw?yx@s@g1_RMK})Z@xbyE zSQ3?|6Vv{{Hdr7o_=yj*Q<{4AM=^|vhMM1ey0O#5#OK8RrMNz&C^oHFHAy>C>=;cI z`V_TWmALkrX1kyxO(7dhr)qCeFAqqMZ%$KuWRvipyyLO1P%zZR^WG~GCW1Na@Y~b@ z<|M-A6+63Q_Wkab$X=fn%PVC(h!J>!F^K$;X?J0xwr~h@j6jgKo zG@opPbAlS+`1yk&;h|U0vJI4FHK#VF;2HvrJw!apZQ25e+4eKGnQlKCTf{L=U}@O8L`|LafpLc1D>&)@Z5xi6i~oZ1S$O zp;YeKhrFS2a?gAT_S!dL$@uK$SgD4j2flG4EP~{o4BjO9b0lzh@PcQ}@04#6Bp-PZ zAC=S>nK7Q!z7VP5)i3&nvVZ!hpOAXr?7Njd&br`9&)B?^`kqMhNy$r+MQugU=7%QW ztGg=t*}XIJh3b-FSE!vlBL5WHQp)Ue?y@cv!FqkOIZQ}x`Zte)6bxO_;#q`6^u^=C zAMneF6NrE~_nMrkIgOZFyw1${Us1Q??xPx-dTE~B$cqFvHsvK7`EH|ItZX1w<8_*V zPU;&#_ykS{cZx?%g5oO=tn?)~Q~^0br?RcQP2IF&c&SKF!Jeo2;bged1pu)CD+-9= z>}lRsL$m-yVL@ra_zN(Vb0H80Bi%Y*G(9Wm*P@&|>P2o;S{;*Qi>mZ0Bms*QWv^>o> zFOo8@{AS(#Ud&cdC>TH@NKyDTB=9OYu&3-neW*ZK^A`;qo@4i;Dh~ICk z75eoq(Qc3uGni#OyGEZETp)>Lix^6ajY;Y`;bvlV6mtzH_WA=R+pP3pl0OS}qdnyR z2$eI9L6-1n_~Fl>tF!me)7Nna<8Vx|K4sdJQ)czVl9j`BFieOay0NNRW=E(uPKVxv z`)REC%uQ@0+|sZW#Wp3?OoqRtqxZ%?o5hVMT`#FKdylHQuf`KGf|{9veNI^F^hYR} z-ATWuH#_iqFa)`EesMq?94AqHxgI}hsA^c@kVMGP8Z05IRWg}f2|U;Q_h1YwhPM@N z5!*?N2uTZaE7llhTW2v0E4TbbnGdJr7b5}M@Fr?;C)E|qv$wT#=oH$@=fXA6&1ee( z%1jE`I2>T>I|N%HJAZd`X3!byDtP>o580zDLJ%J=QJA67s3B+w?OtH!3}gv!LW)@9 znVqB0sUyl>jsSii{6kM?;_S--`tU9xu|}rvivvrmZ&Jg134S4IqH+P}u3DlYXMRXT z;wHtjmd7cBEYLG4A0#ttw9ZsdtpstXsIIZVrcrk7o|l+%#QawSoC$w|q!OOq4lcb6 zWuN!tt%U)2o3=89sp(W!KLP3Y?3|;LlT+W4DE%+Ri~w{upVbGN zfZ{N#kDnO&P#+Y#QK0sV2hf;<#35xLLeAL{tx|~ z<5vTgp8RK(TLdbAvChM3oc|XU_16-gyoxn^}{@`P33)L-Zm?5As_V}+=1 zuyd&4|I)@a_{5)eo-OQ#7S!=Q&FK;Ay{6OV6v?RZ#NsrGD*JVV!HYHhv6@&uhB(F^ z`?F=ytUYVnC5hd<8})n)MHZnWmykE!A)%c&y_S#+7w2sby8u_Jyjxb4!;b_=CHxdr zaZADduI%%txWWhKZ_{R?qkTVH3*4P4d--B@Nu*dKD}dDYR47tN(bvp+V{o ze+ivW-{rAr3nu(x99ovP`)vN~O;~H#sCI;p(KkEp^pg`@z51H8E$ej*8<>$Fll%_?I7Out4Qge)Pu`A#yVI-Dw@v^kjC z*Z8nbb1XYUaq7W_5dheMkPqPdpZFx8y;;|s(e$h4A)0;i+#}|5xlPl&qwZZRF2kk)^$mj59fkSZQ-aEMns5tkt zXn+{WvK(De^;vxP=U@SjE#WmS+Rjh=*PS21$5^p)&kVZ+A!%5OvTgJ}PBH_~h`m%B zu{WFpqX-|UK1F@;#M{*lHdFz;FBIXPDL(upc}+gXAQKB6iQi>6<3l0b!V#h)B;ws* zK=DBO;WJ>W=n>W5E%O5fSc zSDP*6|BQWFSMJaLsNg;S?W2O{y#B{Us@K%2!etZZr68n5I^i_iAwD@taEJjXNKeZl zrFKFFkdf5ENe6}?!QX3AWZ^s*GzgV8q~Pq|!zC>A^=ShVRDP)3*7)FRrS(?(3V0}S zW-?dR8Y`}~V+Rm&c5u?VnaiKKy!SGp&4OKG{4v2~jTl}#1*e)>vGX{7D^rDh8e~2A zs->FIiILG>PA?z_{1?bb`CZ+)Vy|#d73@!~hs!im3eK8lXiOnN%NFTTl5l$YD@tKH zxUyofiQ_1S!}!ZEHOEm92SxPmR8BY3c4!NRqU1nN+uo>~V+k>#VU>F4bcvoe*S&}s zd}Ba$l^n9#yfXf`4u@+XDYiQ)x;tR2J&RUI_?XbXT+k2=w~^bOBMaMt;nr}OC#2RE z$;X=L#6WbpJGk~MBH#(@DpHN7)JA?$+Z%Y=T2cxQrg}@oN9u6E0&e0-=h^-F(z8lh zA|!WFw45JzYG?nooT! zkQmp=rJyNVsdbF24MLAIyS;fw{Q<741;VszW=9$w9ig^kiw#?@LaW~|Z*yCc*fMwH z8Je#Z3MXbioe9;liO5p|RF;r!GY0yi0@(N*Hw^t(jG`eq;l>E)P5YuD*>6G9Y~&Cf z1SckcAAsv4Kq&GAy}$+w6GEG#q;oh8LlEgTx8dVN*fE~z}!Lr6`PkX6t3BQVtGisfoOBFO~hhi6^(X6HCm!K6XvF@Kg3i=^_EljB^CQvM3=laSi_5aKKD~#@#OG?2IV%F#Ll? zk!_BGPrHtf8RV+B1qD9qK|llV9YWrE8|&9IfUj?uJaRvl5aiv4zEi9?AZmU!fi^&7 zXqCRg2@ogMTC5?QJWVL+<}|tcZZg9NnVNEFV|B@W<;M#Z$a5v$szP`bjqq)5A}tutz#!)?p|-r1aP}X(+x=EHx`Og|I8hrzxP-hem-f^#k*!gGQsjy!N&9HB=b!AQEZT6AH$N9h-EDo{yVG(;o~8qCi69$aC}zVv>J@n1#;# z`)PN@fDX(z$maG_(yN#_w6q|yYQIVW!lnS=%-4|=0?+4IVdUbWRxmJBeMkgAw5@mm zumePHFSyCXImk$^-d|?&7LEpq916l{@h^uJGYbdY5EhCd1pSRLDh2)9mrt`91i*Pc z_F2H+e(7IU0XX&lv6B>g!)PARh&D>q21Q@%R#e&F#`23 zq|n?An$GNc20EP(b1zT}od^Bmk@2f+HRz>@;yy?U8A(V`MY4>FMB%RnxG8YFu3u1l zFw6tYrN2*HpL4-nfosOycSYO25lnuiyh+?lFky5>QQC0Zua z+X>BXo8h!oJ`bNwb&B^;S3ki_n4BId)i?Hq?0HaB>Bs}UCBe`4$Dz4#TSvp^U>g+uWZ zc2>7TRDF}HTUFF5!j2`OhDH_ByIGuf<<+0*#2c~>e|#6phz;Wr%7l&mkujbH0S*(j zjVpiBzh(R9(EED}lu9k_#O74?6c$upcmNe#EP-%gme&g`j6XWG5K1|_@PI#j5qgT= z$KD+g?+JIr_(bw>jMuSKCQ^7eO)d0fYdjIU3NboT?f(Qz*q^^}yoPbk1+h-bTLcs( z98sO9L(>$mJ9Xm8)D90{gFs$Q!f}W|Z%L42d`}+{0g6t{2xpK|3I07o%#nQjex6!;uhPK`Rz za6LYEYCZ78z1w+_!uq{zT=+yG7Os|LiWNFyq~o@xb0UOQ@DPC6-Eu0mAr$6o zXJZ->QKMu!Us!RZKFJ~(Ug}t{_2+VEStaF#1tT@c8=B|q&o>~m-8UgT7{=?X;sOJh za$zOVX*Qtl`=7@S;gdt=DTk%iZvt6LmT?)52{X0Y>@isgtMxt0)8c|T{1vUGymyAk z!jIO7ou4$7`x%wH2N4eY&t<)51s|<9WiDldms9o|<*>jJT_>_(%PHLZwJYLf6~1b3 zV2B(yi;-8u?r+8f4vcC5_{fXitW^H0t&Rgp;yl}1kKu;b%jm9gVPn`)b-U|^_{dt| zgI~!vv~Z9F0vUJ8t*bA^)!H@yZrhWzRS8EsYDKnj@p%TFCDhcwuI8<0j?Hb>4S`!+ z3zXk&L47__92jIp{U3pSOQeE9VTj|{fafzx5_B9(; zv$A$}Q?a58gH*ui_M(4~tbPq5a4AZgmbJ6k)oTWk>ezfPJ$Q~W$aO8n(|<-yv5BEB z{o^4XcRj==@T>_?=h}-svSUIYwj{4DN1Nndzcw-!>ceV@!ti_% z;I0-_Q!ZomUE!>V%3t=QB%qvhDas&oR0BHGClWQ_BV|QIzMc}qc$Z*Z*x_PbQ6Q8O z<=d$2K?Lrc9jC#-;00(yoA{;CP(0hYBU|^aZ~I}uxS)cXBRK@g`H-xIjiE#BlR<$y z{{LFJB1}WcV^M`0cjuJ+C|2P2U}HDR@>+Xg%d`A5b3_sUE$KqLDgn%iR*hTS!2BM_ zUvO8%eU9Lu-;0lZ+wRsi7%=aszddfY>k>B+AfzDpp09U$gZ`ks|I%QD$T`s*gYf45 z-;^L&`7t4?pKCLgO4Fkfrq;BVF z+!%|CS;d$LHR6Q{%7)Qi*JO~eUV!&3%&FMWzQwWvZFA01BsQlmpcwN07?~uud2ZPS zec-zdoAn6@@^#W6Z)1!{cR;w&-+xNS2&WVdQHBLWo(X@@x--9xg-Cu>b9@IFmI+2} zIElTtQj~4|nx7~<2Z0gSu2~$HF4XqfP<^1-RUO6CYWy#|6aClP^eg(?m`Q9azZ`KY zRS=@Gac`qt44NMlzH$L2ygS*2#K|0c3!_^P{o8xP6#(P4LW5s4=B6Zuy)5RY)C)g4P5Q0>ml+?hpVcDBxcH&*)-OxA++Z#C>mE`P) zyQANKyZL%Tt)jG@{+gbn{;Zed>VGsD3b{!5AASVQ;(RM$m@-rw>MvIgpJU3w&cTL8 z69^cw36a4gC@0%43oO?c)^YJIUlq_P!fldy`8!o)+6M9$sFW$~JMOpgvsgF~4h{;s zy0iH)=QucR9nFOmwlPoU;61xMKYIRP)j znZh%NEj{AZ+s!@o5g_#nnA%SxKC25Fv5%>6B|c4*!6celG#YqR1~wR5{^QF=yXZ zy6exR%)pHYBVH|-kR5!ancaz^6LQer=2<=2cX%3Z)ugWrbg1N$fmR%1nJ-4tT3V_HqV31baN z7JCA@H^!gv)v*z#zxr_*=>K8+)Nb&6Z%$fS;gMUB+cIf0YY>GoF?fU*h3(;beG>Q( zVb?NgGT_~;%etk?dNnyQqGB0(U=bghwov{0V2if7N7x7=&eV>h(_-JXvl46=qr|hx zgK6C&{U=gwsM^0tU4|x}((gIrKjn$S<5=a~JE zeqFpFB72fj-{-^ttRmq@I7&%>OyiCaU;S>uN)0n{;%iNkwdE0}Q;POihXf=Xk_l-Q z_-1Oo9SM%STWVh0ueFfSv}CwrNx8or(#$cJ*Sl=%r6r$kba)&D@FXTC!aT!9q5IbJ zYp*@%1~WF-J5H;3nBf>=T~*4wc$>`njzH);ewG2m{IG*4HtWoDw?@dlGJjuM#_m=A zhE;5y0)k2ISNRDuorDwZusv=6z1xSx!NSG$6VdACl3RT`e5|$53HB4`7Yoi`W zCj+Cv_S@RH0V5ebm`a^U2l_7lMJdo@gK49>!bDfTUzain7Ujuv1tg)?bSt2jM+_4r$SKJI3|Q*inZJI64nE*ltQV{j>FKI>D#)*FLm(b8`r#3uepeIw7t|$px*rUL~Ln zLmRJ1ikbD$KTG7>W2pP<9F|jBE<{3jROn_}K-QRn`^%-g=EAR8Vx#1{7*2oo(45q* zF#dy_R!0snaf#ry0#O6?g;~k)r$S+ScDXtnOf5g4FohOTv4P%FOB-5zkoi0F*Q%hV zJscXk=${q8xGAjPY7f>Qt=vDqJ7{qtGi(+n0XNA%AJ6W-+a6xNLUa=OeM|c!vV4p1 z6tX}1J24F3ZN*)2V1{`g_K{nw4T3NR)CFF%yy5O8qaa|xnG;@Qb-p<&Llx7PBFS36 z4irm{;rVWm4AIUfql>X}q1^_q8TFETbs^`6qB8}?v$Q0d2=skD(>s+!yx#BLF@#|h zG{elRfjE(T#~_{x75<6p!+V!UOQ^SWL$X4%d5qm(dAi<+sQG61jQu&=J-(fBi*B$J z8D>S;6Yd;!h=+9r#tZJs`n4|1Bf>e+sI7jR-xI_b_7c`0VZ9V1qDN zfB2SlBE$QUzd}*a?|&hNi{IJ)4WX_C8xt`|oK@VP92+HBAMvFaJ-#7%1;--9ZquI< zkGOF)hFfoYj=^=J)Z-}eqR%%Zr3gP2E31~ zF3!yB@OOcKy%}|NlDxT9{B*yeC>p+h`!{%QOEz*8l+XI_;lzmX&)ZG+vN6&E-~ahJ ziX9lpLE5bF0ptW$I22`y)&`h+`XK-@LHrk;-!STsV=%CS;#q%uJyJ}N){@kbl)Y6! z;_&apd>xf$xA~QlhL~@y|CN+125MNmReH@O*rZ(6q7M3DBo9d=BYbU!mp%zCkNm3; zy!IF%TazPV&rs?Zw$g|)-bgatC^^kHbYg5O{Q?n5N}kuDGY0$buHO}7=hl%_hWw`+ zolJ%P-O9t-Sp1t#jr?5+D${&SE(zbCWFRJ&m=CKfHw!*yEd)i%n3tj}20yCX?rqS_ zu`foW=G~oIV#;P+5eLkxpW^iXB5Sv5?BJ_zz};u&_|*O53&gH+uXD zj1;Iqu>qwCXt<7{f-`=oDh;aq{>wNgrQcK3iT4a8=G9LLsLiveaX;(rt1)tU!Li_` zevF@f3`TA}4yFZ6^OLyN7UA4&NmPHs!EPEke}FyK-+pIMC~yKpae@E$0$Zjx75poi z9yln4KuU+J=w!^*b*kjeMxV6dtn?lGwUuO2#Arfgi@{XK;-z|^vOad_IwgDFMo0t; zy&)`ZE!@Yf25AY| z#LP8sNSusnbPMVfZXY}hvP-Ne?~DQ^Z}6AI4krRXw(2fxIB_i*I)xjE*Ic7u;8f** zn9wz!cXo-a7a{RB3)KKO1Tmp{WGk?6TBJu7eQkG4&qJI%o8OocBc1d?iFmym_@agN zd>h9JF(O5`xs&#VouX?Kh?|u>D)9;*5e`e~f~ z!9HP-JdS!x`xLT%i~3bHaOi}E0B!Y)Upe0C7m?GB&#G-W%6O`H>H*WL=1+?oL1ey> zO9tLP_N-)x7DWFT+Zwqh&5}^N1cxpe82*b*__tguvD$9{6mIm}LP&y~_lQ=WXS^HR zEWyq+f9H(a@K)E|P#8m<$FAjY*s7O0wd$9tRW(8Ce_}!Z$~oXx%g5l+pNmSA`)b`u zbg&HuH6SR);1%1GL@_FFsL&`Nr@Lh%u_&(u?l;Lp*MldWMAlPK3xO@2!e;T^{Y6wT z)-8>&zh2natXYT~%-LGmCi)17KRyWqhMFYFNq^k$ zLv*aR9sEcB9K^wA(GG3e1Z06&eQ^OX(n}|mV}jUS!HI9A{!u{Q<^Hx56aIzS;{5q@ z=f*b0Y(S(EMjDB^sNBy&pMi*W`23&{9+mFF7c|J841%|+s~Ubr5=s(_45Bz*{YsOcwcL>`=Nw&VNe)n}qq%huacc{DdD)fx-us(24 z)_8(sFetC|Ltw-ZptyNTFkbhYE zVMPC^IR*(j!I~UgzRzws79Bxp7l}q=>dDS~JAy3{a5Mm8kN0;-FCA)}srA<+^9O>32Z=ou|BJhQCK1|8A#z6S?22u*7FQQjO?ai(p9j z?X!Nq{iU^j0c=LE#%-Uk0t2xZ7Pd%bqG#5+QJ*fx>f<7>Fb{PK+eAc>3roppD*xHu zJF#UhueGiBL+9<0N$o-K%0<9jO}yT_cfAzipy^PkiSy8J6KCqLRM?&O3GUdJn140| zKN%Z~suV(|c_|Bc47%6w6E7J9O8Q6GcN`g0nSuQnS25-s z+J#3@%N}I)@lm5cE+Kz;frvRF2)~7jO}`NmYNPj;*71+90h97u5&3Ug$KU&cIILk% zMdaT>(SD0cduh;JK__k)2KIStCKs*nrD?%ByP6SF5$e_nX&Ujl zzXk8yw~Z|NXg-}(p9Fw?8lRIOE%%Tkyh4XDpAra5MGB-6-j4`RokpKSO~&1(XA~ajM$6W zQrR@B3Ij|w{eD=nzodq`Vr)!=AWqh9zp`-{vJhd`xBf@_;$QvAaZsxC`(KJdqd^5Z zMLkT|tA#y(J@$c&v{W=YU_W}FtLi}PNd%I@rg)+U6>r{mrN$TL|*}q{}{^u>#P$ zU)i~T?t?r)-rV~1z%{yj)W51wG*|}SD?nPka`Yg&6+S-K2LU=K=)Hg?BNpb;GnVj2 z?9ZZ2kU93P5w|x(l$vs43r_`JTLfNyH}&4GETb|~_d6$Ew_6s}H-r>%{rc`c2eR$I z5_8`?lm$V4#KlG*EVr-hXs8u;Cre90q?aLdq9bvdV~wI(Pn6G}>C?fD95&o7vfBo0 zJ69uE_aj>LiuJC?wmDye?!obd*U!tDCqfDV7+1Oqlo$bOtvaqgj?~La49iL-Z3WXb zHDXIs9>XQet&mORCm-}jpSbs+(~YO|js&2}mKBK)JlWP3hB95>1>RWxNDuMm>e zrIqX_jYfn7bEoe|i~Et^CZeQnTcd+ZQ;n1*wslqkuFZdV@Pp!G`SoIT5}C6Z}q!vmtBR}D3d;Lot# zxv02$Jg_SU9_Iwox_xMMyBRX1!|qzUuU$W4IwI zY(wAd_H1v}v|Hm$-;9{Shj=lf#<8~p{Epx3&EimvhFqKGv_LD~tnY@nabVg~uO}f(Np+`&{?7bWIJP2l$%Fp18}e+L z)0Vtz`{sTO^J!SzhhuEzT|_QrZ))h7sbRA_zLZpikw$YezU`adG0ctOZch%jm3KqA zl-sGHcczA|?nzQoLp<0Q(29%7s3>(?>gm+bt8D`klB#8yQXcFB;gDQPXwj3YVVnD` zl++;)_QPB`sAzF8CaD>(oD3{pX;S=?NeM)asXYOgHpH33p|@Q zi;sb6G)@__?*2P^yY8Jkp_u^R`OvIwnkdqAN8iG9S_?qLykD*hOi}uK3qbt5U!IF^ z5vZ+upabkz-~v;e9%2FbG4EI8;#-_vXaOjl_bb6IhzA09d{u#!Q)qS|Y@H)tHQ?zK znlA|3;K*0~pkPv@Up z2N!$uVX8BI5C0EYi&~$eTA#gI zshu>xdb=a~<|Y?zS>`qsAavip0{wpFbFnHVbekIb8(rParKD_~{B9c%ldvtz{6W{Qcl?H}$w{`<+o{3sPJcic zT7iDB8**i8*pjSn`(}C!^I*993kTG~3))#r@*i{!_Hyd>=BQecIpjfq*bVs&!pXBV zvgu^*B=XL8XM63%4AA*dTW90eX>|(pdusGq+wK_F>vtF32~DSQdk)N?@yxh&hbc@) zwD4ZP9peLp%!lTNXNS6{H61!OdvgShzU4)BCk}Lp) z^P$Bq?ZqSw7WN+-p}=Pjg3~&JQ3H-mp}B$J^p0TEftb^1At3LMO983aCP%&+K)Y$Q zED-kRBVSEmdtKVA zj+D%R-|Lhtf&XQe{!bJ3f5R%}K1t_2Nf!w4?@U@i6fNaiu9dCihOb$su3`S3^s&D^ z{cqR}<+~9u-+zDKR>*_bz~I@){{`ItVZi;LlN5T3UP8q{nDX=xOTgy5Un6wGGQH3e za5(SRd$DtE4%7R~)3rtn| zdn>@}1-~8_->P(TD**9=pmUwlXCSB!Dp>v*V{(6F%>NUXYrai+zRhVq*HO0Yngw({ zX(`upJ!u6|wEjy@h4umax5)bc$_mJSha9H;_xIg=Xj>_Wu)O{oaJ^Pk^lT zTAy2%ai9y(mW4Xkg>8V^$u?BJZCR+dSN>81m2VG+14vI!qyWu-$+dpJ&iS+U&o?IgRMgUN&>MdCLb12RG93rq<+*b>Ahqq=#aA#kQ10{*SwQag-Pr|< zJP>p8vHL975e)-`Epgmrm zDT%Np&GJX%(rmxW^izw5puK=}c7%3BzeDSz~T$nO92p={L_ahi1fc_Jw5J z2_3yqNx_Ku(Jf^0pvC)Vvixnwdo;vm0x2HxYCBgOn@jf9B~-?Q8anK%h=?gy+mj$e z#G9run&tJjkGGAG6F*21;Ac+U!Yz%tasShT^i$D01_X%DvTXj0^Kf}u@Cr&q3jE2| zH%oovw4rPm2oKh?^vzORC!82!w)CRwcy3I6(1}SZ4CcThHxgMX1fBE#9V@&myJ-~9 zYy{8ZFx>Gr(XnYq`co>#IfQ{2+ys<-upPUFYyjxzJ$i~0Bppl?{YP53IT(ZmlQAQ| zpRH^0A-oRdJ)>|$GLD}_;dE6~fNzhWc2&E>*onT*TpOrZW9Tv z*>iQg*#}H|>xRB1Xo)2o6`f8JR4QT?357|Py`KydDZ8Br6DVt#=oRnBQ!4!|VXmrg zDq*b3Xxd|-dTSD>^Gv&3z0QcO%U2M%PR9^ceTU6{t7j6}3`taw%ox2$^bdZ#d;GO3 zTtWJ*WAXh3U+&^#Krjc{*rzkd`GMJPx=$FoeQC2a_cPkO_qzr+yF%@aqHhW@K@Kb} zq3?#xE_1UtrG4sS{rtjaB3wA*zwTwX-nSL~2tFCRjdpuxxiXN9zIQErx*B~PA-$(b znTNnBKNkVmzf7#8CyJy!r(biVO||iOLmVF;`<~b#G+%CfuKgl-5#%ya5UyqJpYLzH zKdE`Bs6XFlw>&@D;&no-N8R4^+v!7&oT<_tGd0s87wCxnNUjcN3wWH5W?8GG+`PSQ zmmWgsh=PDop`<-Ha_-%@vwK_9ySq@oGKcY0Z2!NgMoeGSUiL+W9@(=#9qQW-eLmE{ zpMwL)#S9q(kXsTSbX)b==82#H0lLp#C08;Q8E4eYxlRL;1o7z(@?Z+#>P|}`D^t&g z2Vg*<$RwGCh)F9$JKd*Dg-dS+1O%%r(+Xa6hNv1Dc$NZ*pWaAS|A z`B$PjY)nW4ODn4|Nt`jX_KP^me4hO$-M_zOTfhXyiLM?1HQEiuWJKi?IAr|4^B3wn&xpM?~&Fga9uF z@0eo_`{Zgm0euVkeztKkaG4cUHoZQ-TZs1Ll!&CeU-)5OOsF*VZ1iSSmh0#!b5o@& zufPXtqkpj^G*7sBQPl=nPT2P>)DbegwhbMINH~jK>}PrnKgconw$-dWs`Xy!t-&(k zsg({g5}C-m3_x8}K^n?1gAd_ACzAw>LyB8l&DVkt2W`+l^XXwi0;y|GW0?u5F#YAJ zt=4VkVY(;5hDk(X^WaKK1Een~)cOc|e*L3Q2*NF?YsXde5%CS=2mSZw=9xD4+lt}E zi5%sEM1o!f^K68=8s+0ZUR`PUwKxPoP4(SYn^I>^> z)Rq%cb>+9#&)CWWz?t=x%S7XgyCt#DPY=LrpE|BDz0d5vd;**Mm@egq8W1)^!c&_E z(wkpCkzGC~gAq2*H}J9k7tv;0Z9;-GKF*%cTIiJ{5K=3N)KDKt8HB2RUC2vVX*3xK zUbN58;&@4dVK<#w^Cq2k@~-E$!mmxg!;cnFPwZxw6SRZJ={0s4__o>nP83VGc|axh z)IaBp`L~6L$S9dPMy;QIY9+6MJ;m;{MChjWdvM^Uwo?>0MwG(zoDlB=zzni_NSgK( zP~&6g9=nHZd=|9&eB<$;XDIu@ewG$333Ybpd#DN(GLDO8W2c=`?@SPEfo5 zI{QaN=D(?213oK6wg`3}Wz^J{o2YsC)~a`hx-z@Lz+TKQJj?V>lGcX$&NTMK;~YWH z>Gy-hqk@{VNYYp&K;JOazmvKoBKTRd_D}8EyOqBUTlnEg{Lw@*C1AqP!v7kSztc$B zNnFPooLMV29<3ptl0BcwWe3)d4~VK0R97#XfC<-3zwSGze^QbEW3}=pz?NKo2EoTq zC0IST7V&8t+Uc>_iI%vqb>Vo(UC=*^^N)4SG`YS!XI(ae$ioS-squ7DYFJm$Uv!}> zsNeT_**y~>`yc9kAZE`wvx#)IJa8-UF$D%<5fv1lwV`>;n-dUKK#Uzr6SHX6g=Q_< zAO&R^RaXzv(Qz7Ec?<}^>`I@9Q=gMh97MA0;BPO*aL@FJ10e$FhzKa^Zq~%?L?e(5 zo-dF+xEG7;M~@qKPb?jW5I=x%Kdn)(PPH0w+Z}m&SNlO(gtXB24tWBUe@{x$Y*BGK z;;;9jyq5+q>uJ~Y&)lB?O`sv%@6yMBZT_LyLh&xq4$(Hz7SSfr#``So;R%tCN#=C1BVbaOFoI6sIm>9+c8zKMVZAAZX$ogtHqYQV&>qvwPgZ3)moz zxcH%3Z=-5&RoSfBJUn z(uaHfr|-fb9@v1KPamPr;`eh8&YB1vM}<@Ws~O|i-#@_(+g#qNa+d5rs2CeA3^p9L z1ET|Cr<+@w2#&I`+1>UTKCg~)F?An4e7>)cwRWEGVXRnEFiUVXb^ljVp|PC2mHzKZ z%YSb0chj4c|Jn+Ak>$Vl^{46oV230s)w$0uvZ(iM(VJEpbJ>KzNgu@f-l+arERCbk zG=Z2J8i#9fa7O1UR7VXH3roj}%$3zs|Hk$0HVP=8)vfiIo%3pfPQA8qae4>cj=3=cdN&n6z5ZI|s< zkYnVh!87+$MjYgTd7pGNTwWBr*_8~MownrW=vF z;=f-!-OgT`QxE;Zk&j@Wo-TBG1@)lSt9-?(2-nT;FgUAVlC6MqQP;iZW~^|GQaA&mY__L z8KdXz^JyO3l#t9(xLO=_VRi=Lu@J;Mm+~Fy+|_m>b(kdIiQ(dkZP4Kxvn~xgBcuff z#kNdYCceIl*5a#z#eKiBH?v6;32XC7j@5YP);ML;$6^EMr8PU~|>M;jBY>7u^o>LyT*;PodPQ z#~f$Jiyu4KUf=;h%R2d}hi@CptipL2s@B)b9ylT7bn~>OQUypw| zOko%a?F;1MV>qQhs!wrzHf7EM#we{~e~|-w)N%3K53ju0k)7Hita4&Z&U?lDsC|+= zp)Tb5;ztRZt3?;sd-u8GlWz7Hz?!2E^lCu-{mtl0GK|Ce05w!xkrbVa@D*07dv0e# zP2h#ABu$d}H6z^K>b=~JW%Z3#r8jQnyGeVzaL!E;$Onda+fz^T`YPUrMj*@nl=*5# zCy(7FVXy81QR2$6XM&$(;ha0y9pl~#yJEkd&#$+!Ez5gxV7Hs+FJ^qCrbbE>8*v{- zb)??ki50}z;GT5b-q3E9ZZo~^KNYEl+!C*_hw?o)_wcqS1q{)koAJX^Ua6k&W3H+3jfk- zEL9~XouIhD?60pVVgxe4gBdGSEOv=2dD5JVE0`*CH2R%zbtd$joSZu$cQM6x60k@u zi<{%CYDI`t>Jca50VUHOdn_zxM^p^>*(JVsCoIx65?N;ikne;)H7=f#=R*gm_Jbx< z-=%_G9W}U=D7ju5_rFT$%ELg@lwtAV#*~k4dTOsW%@N1PSyC@*cHn_W7M-$$$0(1o z8wStw4)GRj@vZ$}^^RQ^V2Y_bo;n~pSXjleq<0k*;@^7ygt4PeB0#2R$^J}5N%O+& z*KEGiF}M+UTAm>PA<@(5k?I^tY%R;vP*g0o%70)R?`C@F*n%q3S7dNntJWtb zXTJ7D;kXsu2oVW=Ekm7`?9yh8mnJR<%M9r4370PY;8#6M-L z9@h{1hK%4ZJ~>W#ye8M(^dCb;JAxYTD{EARxN+}@@K&W%(8l$)Z<&&lW>Mu*F<7Sx z+7a3z0xy_&UfsRqSRmh>s^5T(RK&j&2uUS97t&NmgdST69&GF?;Uu(b_Gd&(+gyCI zuZp&^t7hBc(XC>_O(#AQEb`+ygm(oX5Wj@VJNcpZ6Ib9; zSbH*+g$ju3sOl*EEUTqm3cGFJa^>S~k6P0n}f76Qs&|GUs%@?u9o^+^p9LS9?E6Qc7zgtQlLy;xo{ia7CFV zTdZg04_{Sg5cZXtcnf%S8J+{Um3wU^P5HdNYUHWixezB8TQm5|V~ad{IM6M)rcq2N zrs4z7;p86(pa;biY(@4+Cp1_T2{09lf1qur%bI)OvABRW;#NYH+YTai-S}wiJior% zU|=~oHJg5?^Wq2fM-d*1lZ^EcdoMW}{{9W!bAMdnHvo6yReKf`5{%Q7jY_n2Uf~Ux zNW69!MzqRZ3uVe>aj|Q*>W@U{`EBH%s`q`FK!#UjMYfxAiZgPk*dhdm`k;?CFyU) zRKawwWnABH6SG9|FqhLL;Tpb-%lFIs(J!U_LLp<~*n@z--$cX3@X%$ z1xEBUJTz9b4zG|dG-j|4u1Ha95de|6giM?HS>iWO0;N2J<`J^m7>Klhc(`e}PLWJ_ zcsT#6Fc?N4>ykmESi@h$5&G5u^_Rtanf#dtWZ|+#uV>pfmd@lQ*1V zf`sQ|r(w21zMSy6p!K zEVL!A`%%`nw@I$i$Idg|cq~>~2O9J>nf5HUQ|tSvX4)LO35snF+LVaY3WDeE#=)aK z=i{C2fz8ry`a(K*!Hl+ByD;hgT^t&eP_l#fq1g3Ck93iebg*_+ zr;Lx4sUo`Up;h|d3OY^k;QE=M!#ZV8XDXLuG+Qo+yDA3;(&S~fVhP& z$~1X@(D8B@Zh~31cU=()MW_93@f^og_xMQ*Vgx`qqp0bu9hQhxRNMUnQP$uP^nipY zo^d?$I8_I)gZ)5{;r?~TFt{%(mR#JLh3L_yUC>Us`2qkLt&w4f%VIn5wITPa=$8KZV3Wf`lNi)cP>@f7~Bf@s@4Z*!!&IWx4bd#K@8^@_q;LgxeNiX@Jg*&bkI8ow_FCM_vM9F*{F3TG~z(s%H`9XO!{1tCO-Ae zk=RZKlSdxOEi^4FD@vNJDnUHyI6Q(lL;;KvLoF*PNMrwwn3@KBURlO=CYR8TiJ2$juw%$`_hU)v}P zoBwB)BSmn{)ORVKklaRIIOIz7KBHVmxIPo3HyY^yKkJNNDM=41vcvcoTH3Y4s27x~ zv8wZ^ql1O4g~DZNY|;YxD%K@wbVV1HL(Xdhb*~YiI`%J*s^S^$PQ0fgfj5mTT6f!N zFXKgDiOk~psWRDbPq@wWHgV@aMKU+mSsF;MjVjnc9X@0Tgx?3mt)>TrzHT;5DG7io z{`_2JfOc7}{QD)jzY`-;XvF0S(Skr(WRlJn+;_p$>B;U~R{AZhu^f6k>NOLF1wCK} zR6=5|2s(p7`OV$>sO)|PTav431=}Yr*ET`jo%UScm^Fr6SVXHD{%8awPt;4U)tT)_ zP!iGZ)uV3TlALkLckj0T>eW(9Yuqq_Z5>#G6J6DTEf%x{$Kuy+x|Mhzg>s~6n-xra zZMCkDN>-6{z5(0$-C0^@$HLP;P;+d07$X|2X<+HT>Edh5BI4d(={r;cm>D-m5gDB-M~5@i#%Kj{S@#SQ zxJI{B&9xP+1oa=Bw%?fet(OBaRNK##XLWA1G#y-`%q!&#S7h;V=UCjdOs?qU4HxBS0(b`u+QcVg zmQ!&DJ_%0UMMV&C~3mM&H3=jXo z4p~RCFOZqNyPEv$A7NPNQs_jd#AEhFvnZ4UmCNGej3vLmkyOybijI3&)rW_~sDk`? z9ShgF68FZ7C6&Z^jhr@dpU`8hIjU;uP2SfrFQ!_ZQ$<%hfcSvVpTd;8Jh^9?9|`tI zobteiuE9xYh}Wx>{HeF!a(7zhQ?u9;2iDCu<@BuBe)bf;rhg~rkThve`dJkulJPzh ze12rE-8?_O_pqV-g@Z+6SmvM;aV`i6wJ7 zZn*{cA@A!z+=y?n!#~*rw5=R=bKOu;3$1y-dv89|EqhU%3oK>0@nw-UV3y5{^s~Y~ zGRiL2Y=1>$nu{L6#)%t=(`SC7I7L`YV3b)7l$q2YoF5wp*0ERMY*fCEBgK!*)8hT5 zP#qoO%6p*zKYg=PthVEM$P6n{72)-R?eG)o=b0916HB+8rAhaj-cudt7qBsNvoKtm zQ3o@j4QzwS<+w~Qj2$;D?VZE&?p=x$pz~kk zOw8fk9P@jTrI&*r35&J5RPICrEeq{Xamrp)=T%2kg(IX{C2)aNtisFQ8e>g)q89l0-egWTTkdjbx|Q=(x(QCHa74l~@d z*IDoLY^|2QrkLZGm@Wugm5==hru|vCoM**?&8bx#IrP_pSFa zR-l(q?u+9J4f}T$#qM^YIBQRxIHpi=7zywg`M83hPk#+X2G?1QVHu{-n<6I}B{Ii*hO&%x^D_en z7FGm_E>%KM#c5q_1|wXJMEe`A*^GA`W2_U6I)a>0kgy+CRW3U51Eb)JtC$UbDujqi z&3mip?Xu_Ty`;6wtxhcG$NE!Y~GXz+iuucS%X z#@d-}n(;nd9vNCFe2Pfuet4U(e_V0YVyg8am%rC5!9C#$N%uTUon#0W@(a!^YO>gg zxFbxs+{YMtz#_#3O`tSKk>EgeP?hnDNpt6ygfh?Gd5%&-@Y!rv4SMX9$2l2ei?MNu zfCww)9*uE+y(W0NgWvRR0b111P@q!D$sQ#|^GEJ;B!w@cuP~ao?b>e&LwnZ-V;67V z9UD-+)3NNt%owq;+9(ql(viv^D9kum(Kyjx%L`p^!9}l_KDzN_km4}j(8)(M&fQ0( zHcRwJFX2IGokFEsf|Q^il9m>D!nHznBm2IX77Yyq9}Qy32c+iCYYc8 zN+G$`+j9jyu(5U!8obp#MjCkd3%Q}jm)_>wa52&%V25*$`f;k6GaF&OHdnUkS|a;N z{vEPZ#^6>Uh0)t3)eX&$;vNZA1N*Yh3qQS=>IU-Iexw2sU4^ziQE{}{o4@ZS{FsVj ztffRg>K6r8k48cLR2VALEn-5QA9S1&QtNz;iU8Bz_l?vR%i`n6=F)icKj}(ETfa<{ z&aGR9S1`3+XBFcwSkSnqHgHeFpk6i1fEjD4t!AQR{P3~*SgNujRV&AkxYfyh%p*cv z+q`Ngkmv$$7#pc^^mnmn=J8%WQ1lp$Ml6i`C|<8H5)$KM*OLQKWlNRDVdrf z%D!Qq|I9bx`AB%$>;HLi%XM0E0loG6<(R|^O&44-PTh{rb2k-n&2?OUV6}2oljB$> z<1y*6B}m7w(Y-~fhl~fneNKl$0CZJj-_apXS>jAuIqklqPp_K? zp!^=7(71`0dnB7uYw16E{CHter3p5)Eyn@Pnu*P={z!`>(xh>kr6+m_OBXy)YO97RH|}lK~6q?TFY`9{y^x z4%B1cc_?g#X82Bf*4V$2f)&4KuG*Feblj?{OJHtJ!I;1~O?XUXF1n;ygVJeRBn@;wE}I6j=yPQNb@I2+Pm?^m4i#@Ww7&CLRr?x@Q+ zGjFS=^nb}ZNFI1&MG^92>_rRTs3<{3Y~L5MjMlM|JF)j|(G;gz;(?JPX>AednpyH3 zrBPuN>TTZXv3Wuc*$ItkctDQ2dwBUP)vk%rX3NJxU;$+ME1(G@zcmOG7df&~z1Uj1 zvI^<-fp9ijowtIim-bTf=Y7tE3!+E?`Pu+fwRoe?bmMu#fG`LBR`j9NBQ00WN9g8W z^_U?GpNN@17>ytORe`vS;YEmnMi3TR)6ks>LxgRh*WxZqD$wOsxX!(GqW41Hv`Z04 zXWuH3a3L?x^c~5E@*c0;^(6Eqg1> zmQT2(TAuF#-9RqX_lak42va2r5Q?xCzZs zGOgC6zd<#^GKAVQTV)-4jKFiLNNF@e$0Fue-1k{y3z4q2cHc6H{65Y|2ObpKL~yGfMd~*sPjP%5V6b{Kt%R?DW?AFhALv$Xf(6PpejS!;O z=&1CZx;}7oO_+aua1AUriY}4@L*GqIiaZ>EU zp0Vzz&ZRd^;Mp59`dtc)@+F#7ZQk4lJp>|h$mYsyKT!3m7Nq0|*|j!y&tU0=-z?#c z3+Gs=!hY!&Kflxr&a(@U7tTy=qrf$xqpJo~Dm{;X7tVk3`FAxFhd|j#4f& z{FXUJJ?Pm7B~J);1>k51quOsAL1tMeD3@t|93-$f>`t)hnn*#eroI!^6+0w2(#Qqu zv2y*{cG9H94Wb{iIz$2j5CC{Y&{JlV5;8%>%4rJJr4K^OOQ;7(>C>yFZ2Tet1{jFc zvZJDtR@2!{CTK_HUIkHPDJYJIzkkD zqnkUPsql+##gfE@KdFM|WR^{)nx7b{mD{`}H&vy6I>!Tfk?Ruou~G#nPrl#)i1U53 z(Sp3`kIV3IkHKfokuTEJ?JL`y*YB{Brpc+uS0SSAM%J|Mp6Ya!6cPq@3|hUPJt9Asx^g+v@vvpeM__!dFThGI>Hu zjnkbv)QlLQO_!3_#5B}HEU$R6V7Kencc?LB=roD>pNo=deJ?iSgSgsm#|LGLH`bae z^7}_5RK{iwHzI65PNdw@+yCInQ^!m<9&qr89#Hg=9*c08!-Y&56JY^Z?@3(=VJJG>*8rCjtoeE~h_=wP0wn0)1p^=DScmNudz$Lt> zocs{i8@1QY{E?eF<|ADK#h;6}r4oqbxMT+be80%SIUb$Gexz6ArpC!K7SBDFfrmi(vtpHqZqs8#@Y0gKW(wSQgeM`}@u~gh>H$D zj5TdCPi2XsK+cCbSm>*SvRAwld!F^9#k#5{li07Z(sYUt=>;}ly?rCqo*RWdvg?N; zbw&Dkc@WbjNJ1G)hq|qV<*cT%qb5{i-K;2i1nY_FUhA@c*LG}$qDjIgJT1y5UbURA zY;mc7Gs%5l*wt1yNNM+gKFU7!@Z3Hwp=y*JSF;yuxr{xZF+Nh|7>JzsHInYR6Wv&> zAvbdL>K3rAB3YqjipR0h$#rEoXfqU9Zbz!9h1(l9qy-4?t=vJ~{aM@~Z-i?h(^AZ+ zogoq*5mSD{d0!cUkO$ypQp+G1E0xCr%}gMZ@j8rm66cH=+8|G;z3PUB^Rj4 zW7pFj>1H2)wU0ws$`xe&TD@~_V_a!R*lsk*>%RTrg#WGK-m)Y8Za`M{FF~4ObpM3E z=?tAvv!S6UNVp(Sw`HiMzUP~ZZrOx zw+UQy@ zy9*68<;>_MIV;9}Vn0x!UVeE=MsFajwbqnP?nc5c@&7ROjlq$9U)QlZwrv}oOst7* zXJT6u+qOD3Cz+TN+qTV#CduT@{Ql2}x7MjseQwq5s_we?VDGipLa}3Z8FWzi!J?;` z;u;S}{v=)0nAQ$q+i>kUvp6tjGALu%t+y|5(^!=f&pvoK>4! z{-w(hq`DpTSLZAq(=B0SFaPBw{(FF#Q=GL1pjD{UEoCZaQe@7Kjtlvj(a6i!B;{9y z-g-{%0Lt#hPdzJRhE;IFtF_0mRFL3+BAS@6m@r0Cqo)6WSUwvtA0ReE1c5A3r>*iS zmyK*34ic-^NBze^X$7Cs{&W@*9&8bqBPL1ssqC3QpWaxVGSuMzA@P9LNM?;G%9Nh> zaZrS8z+}nhSpq7^kN}zYL<-NCXhvH(wU5Cx&O&py_tv_^$Fd8vR$v|mPfKSDzPw>S z5D5LyX!N64=V{Efk*Djo@@ldW5dnQ!8%k!Za|+17@;X)~oQW(lCucAic6JkP1Rn%s z_9VU#WEx7Qyi?^2nv?jDz4A2rQ~a+x{Y}}+-l85IsbE$@z_6jUr#OI)%{P=BYbiNw zEK^lYx8)xl3lAMALBjgI+Q(w^RlTS$#B}xbAx88;B-ZEQ(L1aGUnvqkUbYLol z+Jktp43xCjCglkasZc9oo~|rON)h))&j+#wosCq#4~ezRrR8ZI4x2|`m)4))aGd&O4S%B*R^8NI62la0azd;BcE}Y)`Lk(kxgMB@ zg`vo`KGD*!Ln|}l5uoVn->HRGPxTJ#?tOW)&MH0^%kTbiiP^Uqw!3Ky83#s-`r?^( zDs^8)9nBpzuupI|US`0?J(E!S_spZM>)T;rO53YPobS#Ly^>ytJfV-V&VJ7JvtBqJ zIUY?Icx}4UNw+C+e!)DSEn)qFwsxHqrZc&_l=m6LMtpTaPMmF061NZ|n$#M8W6w*7 z+eb@tdjG4-Zfvu^Qp^iIgi%BO9UoEw;y(mU8Q7MasFSb`2dD8fdFY(ET9*YXKz1#w znI~ILu?e)un-fi%Ql2=@onjBY{60vYP66%~AAI{HmuG;L?+Dq4MNagc5(h`XL>4y3(qL7% z&Nze%wJT@XcIi!czV(Ke$&%y_OFc<|bb=MR+*U|Wjw(A1+?j#0MV;v>e`OT*k)nb# z)(Vcdg(QzZ9J;j*NV)})vm?w^*Qmb{+P`n4i5nbi5Dm!Id$jgM1BU@8-R||=>$jWY z5UsMs5=8=+)6kmkILB7lzX0j6&pAfuhi_?GDISkhz?CF$s!b7-zP+Jqt2yw%1D^RQMF2I12zxSzlz0@lq^zgg|7so|Lj zTfvqPrD#OtzZE7#KsU8MEg6nr&Sj}B$z5>$MQ4T&1!>I%ns&ZmkYS|?MSv?QOa2w$ zs^4;m92A$@U&_9cMuR;4TRypy^5H`J-JiBe`?Fth3FBX^74VkFF0Xe8mB`yOofBML zDqS|IVf~n-W&L=Yx#+prl6yR$Azivw z<<)g8GsY5^tB_a=hZ>()>Svc6N-yiJP?F8EYW|25R_nMyi-ZO)dcBII=rk*q9h-b<6riXlFo5^GgCb}dbb1pp#UM(* zu3D*Th4;O@+$~>*_#7-G7zJ6-a~HC8XD28a;G8qba;CmlB7i#v+3~x6IOC$`Q*P7I zv9}K#7WS0P46f2r%oR_@`Iav>my(M8+?FS*rfKuujcmml?zv)5V5 zHCy~A#CSNw`1YV$gW3gS>CS0#)dZCjn=+lz+keI}bnGt5%B}%zRP3_mK|ZIh>WJg*o}W1v|6LQvBXMItQx4#frDb@gEPLyz+fh*eESW z4kDTbWV_QLvO;Q$$_=L*9w0JHO-0DBI!h{mT^U!}c<>Ddy{u$(_444`rd8V>2oceL zwi`=J)A+`Qf%_qVl3}@r-HYCLXK&&B?Unowh_cpyJY5+_YQ+lOcxSX*SNpV|%`t7? zRsx6SOe56(WfO9VOXAYssnIf6u=iN~)Q2Y9d@68ko%9vVaM-E`GK`XS=m)gVfXnlB zkH}7Izzjg# zYd}Y!cPR2X(Gq{Pht`#=`8CQp-B}oQ3i7j@hVeMYC+GPzgI3s#5Nn-r}gh2vOj1y+n#ff0A) z87rF7P#i-c)GTjINJ9~jieM;>2DX9d%Aqu=(Z%p*IxAjXrDvHH!VP9=^|1`Y9~fAK zQLgF_6d4pPOE1CQEkq#4Pp~v9Fnq9fEesAt#s+Gg7!?EuVo5Ir3d2GqzQo$>3f z{_bD54ORIfoFL7IHZNs>QFGJV09kr&3Mynnf7zLtiNiS`S7L7S{ZVa85;xf)TUsaV z>ok+q%eJ_x2%0V3;j?brYEVj72DdHQ+Otf7Md_F-@Z?ci0feqc|Huj~Zll-w{AH!c z8QyBqo+gl`a}zMfkbi41we+l*%o3x;zLt|BDinsb+;7k$BM9$6+5R?=hK4mVg}|bY zDUBB6iC1l98t02rP2MZM2`xCeAS2oBu!caxVe|c-|4mMnvUK7?^>VbZ{NzGg`%Z`%^<$Vcat?q1R|LuO=qc5;m5jQ>j zF*hgu9@{;+-a2ju5)QpwJXf9aMiFuZ5MSKBLfTjwAx%~_n3*k^A-sASKb4Eg^TE?^ zJr$1A(Chvxo}F3A0^^{=XCwgeMLvBq(XZ4KOSQ@1QVk7W)Moo@#3$zIW=kAPkd9Mn zkSw5wRd`BF70k8ns8X53n_Ri9blFhC06JTiI`KUltG2a_VHgg9or@MLUY% z`J%?aeC+c2sFn?aK0*5stALDu!3a*LfHYW9z{`bM7PhxgAIbT~BY;pDY3dJ6Ue``# z^9S%}_2J!5%`|PtxS;)1{^oKpcF0k|EPcXNZEZt@t`*pN4K?icRAgS8t9{ZRmGkX# z%bKrlmVSyP;JzCIXL>ozwC5QX<@F5?b--?NQJXC0oco-pf7)Pigzh+zUR?aB!C?x6b&(s@${YLdcf78XFy*ajoTT zkMPE=cNu(BF}LyMA((wL5=^=_%M$VYX8Rk!WXBp8pa%GmWt}r&*w=Y8V0$1DUH}My z%fnLSE^s-!?GtMmn~=`_mIyTS0iMI3XaM(;2a6O6bnp^bRr%0x@9y8+_L7d98Jo^u z-u<|1SPXbB;mMVZ)2r?)QT*>bKj8>o1-*)hVzKas-LP(EN;g#R#+|_6*=WA|3lC9XWwp z&Tc#o0_h1oO`CW!<0ciwn9{044c_I@Bq#aNp3gp86_7D;7KaW&a(!+@R$npASY30! zU`7JZTD6@5t{evAk!LW~Z^(SymnACFN+LNgg2Ag5n-+y-WAAub7u0`RRUw82KH}k`%L0&^W@0|%#)Hj(O_jyI zFl!c$6jp_yFffC?umU}xU?icsKIl1yavq*N1Ai)XNsO-?uruqcI%H4y!qE*`U;bQ>z=-EGEzaWeS0Ceq{`=C(VtasuS%}B#CI&!#~qhWgJYizTDEDwNx6kp zwQN@%wD%uz@XDQn!_Gom)aF&kx;Mk`DTVWm#{0=Eck$;WA0ik@J+Sv1xdrO;^m0)n2wn7dG% zOQC9Ad0u=cgIE1IkcyX%Jq^AZWI=P+;bpbmrcLlQt$-vil7}-?2?LBye%;-FbJrX~qRo)f}`1 zbH$MmDU0NrV5K-TH)I4y@xj2Vd!@Z#C=tq+cbW>(KZ;a0GmwjgAKk2Vh zk&NS}+JwPh(4EkN0MuyZKL*4`G%dNVZnLe>rtLSu6=C^_E?ikLm{Q6io6E^(*hTUf z0RWrIs$YE4NKNCy9f&Do!WD=qqry+BgeTHIjogLs3@f0IPOn1QWmi1}R!m!gkBQ!a ztXf&u3W-qwGnu3lNX!Y8yr(dc+o&tI_K!ICkFo4pHgbkgh3@$A(klKaR@_SEEkhbg z9R^TMr!HlkRwnx}&#b+*QrS2b8pMA8WP~2Hj5w^M;vvI&6CQPN)-cOg$*#K&LPR*1 zO5;#-kxo_>Ik|Dx#FZ{8*8`)}r5euM(AQ_nfk&Nj5R1wnV3ewq)0x(}ckX~YW^r85 z0^f1g2|v+8DI#KAR`P7RRi9PC6gW{D55u_C$nFXc6h0%5mi>Q%$2liqNd0!0;93q! z_VsVAPGY>IdhU%(zh~d;5DAZ`8yAAP?w@4OTYDH{Lq?e|3ybt!o1)^_XO*deGGfE! zOSel|ZV0Q&$iOiXDZY~vD5Bf|_>iJ5Ufd?u${z`p&7L-kje42)d05eMHMyqE-?*|ht3V8g)Ru}}hFr~sHQII+YbsiH*Y4oFymCAuPmCg0m z4VM;2i@cf+I5djm9h6S#c#0^Uq|CB#KfdRxpI7TxEc)e`oiplMtOCcE>c}HP%pZVN zIDMmhhM+>pO(oi3G&02^3ReEHL-imG#kKMuf>+99aQauaf!Wv`Rxd$V04a&N5%(_S3AZ+IteRUYN zIbzyK(R>l}5C>MN2D4jm8w^Aq429dmd!i5OnAN_Hvt6A_X?rVqYWm1=Als%PmexZc%Pa9ziGKy6tzXWeCs<2uv;bH~jVIpi)X#b-s%>r^(sWEQb>5aVsBs*A zi&V|de0Qw9_;O2vv(*OhHL}b|^))fUpk5kh(Grb~GYw>^i*Bz;)n36}77v*N!iImo z+ELl>Ivg9Yy4hX5F1F){0E3FZx`dY~+!H_S4c-2b9yye%(=V%ApUH=4o;t2w_APFY z%2L5(n*j*Q>mVQ8K;7s=WTr+arOx?1cpj;-HlQKog_S`3=wNB$2VN-2SEluyCMv3_ znLED$87nH>q|&BkMLuaOzw~5QY~TG}S7Qb_JEWeg(pHj)k@R)>CGohCc4s>C!BbWmkR6pgR!sCNz>gQPM~z`58+{n zrc{?(L`J_l^5EZ$v&W*J%jIyn%VOn0v2=N+>1}gplxueCaM^*gpN>)kbYSqR+U&Kp z^vE%mhj%Jc9gy;!ajvkEP57ac6|(ZDvbu!16HDdg%U`TrVahvk3K4J|&|$pPj%zI5 zxPXsfal9vMcAlNZE;S!xV>_PIQn}D5P)<8rjc0!$%O;XnG+hn87R~{Fx4rYWek5j8 zg6#(4#fz~IcR4syj=e`j;whqk7`VWDBj6H$HobwSsYoWlw2txgovRCRQMuCO86pHm zVg$y#Hcrmps^4kJhf^x!auovyu0=1IN6;DUA{pt7o#c8_BB`h=sSany0Fty%l zI~L^qYTF88%-$g%rj#`pjbC7j;Ej*skCp>JgghE3BSS zVMOeSUqj7ncIRsc&hfsEcEokQt8X$+6VlCLeBl$zjK-oHDnU_<>mNVfo{gvZ!>XT} zhzk=H42|Ndb?^$VN05U{sO4njxZ}7p2o5+94ZO>8&jEN9VN{v&J5TKvGot~S_=8bj zOOB=*#T6E`+(3H-@6_#x)U?yFf|klqmKC!q`$X9~3%h-P1<_jbdeLD5K_2674oh|% z22_=*uSKku&gWs+K+@x_Rplna=7eL4%4TWQ!(Y7Z@ITq!y`4~LqcL(F3o#U2`y&;e zGuUZLCnCp#ivSA9nFPRHahjZykPvo7GCoVIrn15@otf&UDM5HR3j9HF_3UzMv;Le6 zRvP>@agJ>1ttz?9rg~X*hWW$dV;Y)=ICz)B0y9Qe9f%6Va(31+gk&%Z$XG;PQAd+& zs-s`YQ}|8aZ%Ac7n63}N)Yz52BB=09I^|rp)@WAldpCEDW!e`y$#XKlfd||9IX0Bz z!(onjVwOh+N((AAGPAMNE@!nesej2cV}=M77b6~Z>%7)rl^ciVb!qTmL>9(}wIVFO zg~%#FYKG)B2!QOR#n5*nr?Vj;EcfVnwfK$mjJ`!F%?OGmC&au9;6>iyjWp{FusBH? zl!`+~g0|Z8RqA#;(|Ykw6*v(J>iemIb#%RHKM5gzCtA4K2hB?_?u+{)kO?K0mFwi>I5^dK7-uscSIc)$q9iR6hs*mj0e`*t5{&?S;~eD zg!02LW7pI2FXN;w4cFwuadV)^fNE(|y#6)*OZ|v9%+E+A33DaWtoEmXaoGDadL|XS z&1aD6=gI$hJ^XVlVPapqvOVO2mmu!}Z2j4DW7)bFW!59F;G|yt0-6Y&-N8>uc2*W$ z@ie9b01kzv(4t}&pYvc4`(cbtGxDOTB>|EEh<#=C8D+Qyxx4HD}ke;<7NaT`v|_OdeEM2yfr|Jwi>$HyH> z;s7Jkpg4CzBI8t)1<|iswD7W$jWXD}Phs~Se@vECF5D6WlvfA|gRo>U!{~UFD1%o< zWURC>hVrNAk=5qNaZLguu%d8TO?DzK`+49XK9evjweVC$_)9qET8z~!JHe;jG4@4Q z+m5hk9Se~g(qpI^8FXUF!wkpTNG?W@)td6-kRwIHF^68$bgFs1i*hBZB9x)d)+GWd z0!IEMfWttg#psp%WHC4c<(6cGMr{7kAm%CXO#{N(cH4?sW!LU@Soq>LtY}A=;?%S& z;Vhiuwhx_ZH7akTzr5oc7C8eDURb#bWL)RD=be>Wl$m*&)*Y^b2jz9Fvwh;Q7+tIl zE{{$1qCdCgzAI3;WhNOj*iLwEo0Wp=nHJco9gDJl7O%&BM!;}Sluz{F#Kna3q#goY zB=-9Rb76T`j*aqXaPP0k%}2Vw0eqP;w6xW+bw>K{I&U$UbKrn{@jsNO*?ZYFGMJhr z5=X=mM@%_t*oZfm%E*2z)UQz*QW!0ytrbQb6#Ul`EfO0K6`LuZ(r$6*OV@In_+9bY z7jycxZBIF}jR&`L6B;lcU?l2ltu`!{Wn&oG8W*}T?-5qQjC*!*>P8MdeqjZ8Tg@@u zHVh7Hyjn9BB(8N$}23#zmxv@I?QUMrRnbD&jr8(JV{7 zF6AJW+y`6`X(}adZBGQs*YC8W_#sK22Ngskx9;=~1ezNfr7E|!ixa!RX$4C(Ap!(Z z;NgNXiYQ`W?6AOfT(?$jnpk2=GaPa}o~&6jj;HuFElUUE3Sp0Zvl2v2V+LZk#le=o z-%ednGqjoq{}>Xml9(FD&8^lNGo&z{ zd+QO$;qxKC+hGVK{H+7OmWd(fTGVK#_A@prOji+5%~4Wd$N59w;AC+a93@YrRpmWUD@t4e#(OyVXX#xq zF_=l;!u>n3mIXb&&*Pa*27<%c=CL^Bx~0*puC|vTjPMtn<3|TUOWZn(t7W+b6BFW3 zno}OV&0FRxSRCZPNi*Qrtvd7&BlIZ`!6w2A863zHT*ge-9c&Pw1cu52cAHg!3I&51 z`-Nssw`Wy)|5LiZ?Djap%2aLj5ccP+g(C>=$loyqgID2}{*zoMVf5v?l*AHT!11OR zfTHzC+>D(zp>A;AGyYhRb19yy=fJ10Hr&UlvAbs@qfbJV zL5L;9Bdwe)F~hlTmD!=X*6}*`X3C960oF?Y)97{+`I0VqNkuNAmZQ)QEvjhrxPyWC zO`kJxrSdgGfKu?=uw$E(Td2!55c3RYH~0YHH$OtaEmJ=iIpWO_D~rh3+_7+_(jj|U zjV#jKE!kBsV1oU44V{CMW`6Zr|C&|k)BrinWJBYhH#~-MJNA`lax8*MLYR)4{d8Wyjieb4hnYct;RIu3yHV zcl^$sJtXFUsnbOF<(V-0c!Njy?)2W?5 z-mKVUUufWaBWoRd={z`5G7x&vW^dVl9!PtWjgY7J zzjqU0RQytLsH1`;Jvyj_A7psIPyI?_3nfX?qp<$`K@%}yc+C^n474Y3wqi{43{|d5 z@?u3mIim)iAI!bPey-_t^IhaL4Wn#m(OdHUzaKmbqv+;D^ypSoq_?d8#v!&uR6z%Ny zl7kf(+(>ayN=*m=*J@ATn~vCt2&f4bnDxnLCRt?33nw648BJ0^y%Yw6g&X2 zx|S#QTB?}B)0YujeBqS^GC3pgR#FM}DEjALw!a^;~U=}r|VA$>1d;E(J$^K41VVN}~?-9=v%k7k#iS?~80l9_M z7W~ZWebi78h3i?6`y>2L*Rr;1ZBdn59a{Svp$s z6B>2mH>8^5WASLf_OGsPyiUAi-&#^6;fj&rq^5em(s8c#HXaZW8)CYvaA(d{C2ZVTwm+$6ML~&ottKux& zEABTx2|U7GylEHWX;T%09qu|86Y&bcxetFWiF2{}uQNm>@$oYYBJrZIIoROAXkhYi z158Mxz`=&b(xZC{EVbJh9ls@ekXpg3u_}qsuuvy!_!;JE!OemX%-IDXSnv^`8XJYw z>=`X~IQ1ZPvV#2*M$|S{b^)-92YC3-m!B;R%bgG|4f1T7% z7tOLlQ;+WC5LU^ift4mCb-ICk7*sMubPH8qb;2D(4+_Omt^*VH#3WB$cxIy3r)8?p z!Kk6pjGE}rspCqt_!5c|3iyGY`?gD%lOVR6PvWf3K>XbU`6Md^`b8+6tCnu!G(Z0I zV*`t{n*sLbeP%(dEemU@Y2SSflchTy9>6g1UcZ)UZ4vbs^Tka>Gb)`Ch}2w1w`HX% z{8C(gzgFU&zNTxfK&F=<40!O2VQn2FL#tO5|dd89h zMA_Qd1ve*@`t&5FvA`l+o|F)FLOx(n5e4tnxEZSbGw%aCF5g&C5hE{9^b`h5DNk;V ze4!_GUoE*T=43hkVGL)UA;EL-%?~09C=+DrX0>f-3uOR3yf-~f6p8&RaXgt%dG@Jp zBx%JOlRDh_Sy=n85=ir+#As%wb;oF?zkE;<015{uVW;PEg_PoP?2Nw$485CJPwN~& zlmD$Afk?FgKL^lG>$IQag;LrQIt3CphNeQI{VHnwvZ66k<}46hy%fZ2w7A~wSAA$- zu+$@@+C{B5iG#Mna#OL0;)q#Ae|ajGiZsy!F^l-Kaq7Kti{$el^M!Eqqio4%3sPeE z)U0xcTZ1J!gkny)378gs5b&VipqOiR#pqCXc-f;l$FT5+*je8_t>s7O=Kk&h%^K0{ zRgS;gR9CxclB0MVMgOW-(NGGJIF#1)l()xHM~mTc1q4#FAq~BhB*RIix^y(7gM}CW z_`3EI@dLufq4Uz`H{~8)Hg|rq$Qs?#xTqQRT&RaNx)>EBTtk_!Bb;CUK2jfNZna|Y zHp{19JETuLw6i_r(>kCZdN4ZA<`(ixctl-cuF`!fAEb?#K^0B`6b~V;qU5&RmU}2N zS5qk>3Bsk2){{h2K7-@N^Gk|*^z2@m-|q4N24Oj$q43`}?zBI~`99d{(|#g4pyal?l+B$&?8?F2jVG zM06mUpe**crZK!aeErej$euU)!#*7VxuNnnP_({>n7+gIA%8+J8Vr-45gG1(X#5@1 zOf$Xy27#^ZH`LC7o!4*|LuJu?6eD3#4K!U{t&T(#(a3A~UA>go@EoIV!N_mAi3gb0 z@*0}^p`JTZ$f1$juo3fM&d6(8fX7f%jbW=z;EcR-h6@YCs7TkciD(YfvU%=uf7AD| zlPA!fXP>AJ%v*jAl_^-PZCb(6~e5EHF*>l2 zO6}`(&!CC}i;eTp8UEbCW~0o6RMN?O2g_^Z#1}oTtv9vjjIGHWsdSMoWX2rPyG!8WYH3liGU6YlhZ&e@ zP7A5+M}3zxdX{D*fn)U)=uhkm<`t_Erz%7WMJrPOfjvc=~J`E-ECVVbn zI@;L{rD}2cQQY3sB!<(^(Wt&Nl_;Nl>Ek|N6oZINe$z9T>yzTS1Q_yKi12HdC_^6} z+|=`5^zvna2lu?76IKMpXLQL=I_sErqaiAOJljP}B;zh#-NzXgFoi#bpKTl4hxz;Z?$sKvO6X^WAkAcFq;0qax_AU6 zum0t*ayJAhMXlzWFX=g?Hpz%LZ?&d0gi*?+VV$P4MQ#xrLa0s$K5ouCnBLDSlt&KR zsQ1?}gCJOUJ#HVzsn;Fj&35Dz$c-Z(GGHH6)wFnk0#qSk2z?I@O9=k6pu$C`>hK~h zium@Kr^eF2SYzo=?qmbFFrTO6$X1M@(Y1%)fpPeUg)|J57ADyVR9TU~|6HVWG8Fg_ zBD%8903e)*Uj)4TXu5J}SUpp*A-L=hmiBZYdm5aH^ICtXSI1X_VFv|nS%;0qN@#8s zekOR&b1??G-3s-ht9H6jb)jtN(=CWdPivNZj45-pyh^b_nDi)ZU(H!`cWNU$tOZX{ zAk=B-@l$Ugle>VDWPk`43@YgzI99jbg+7%{pYGQ6btlEiIe#W#*`;RY4J1l4g7zF% z)HBKq^&J~_YV`8$XxELZj`yzNNlpsa90Uei2WNd;fCPqWLSL;cSMKv>JmqH$POr6r z-t|`Bl7VY*8`ED#*ZhX>O+F=T5iDcUE0~QYBn?9)4BL7<3V3l zgEED;FnQVJ4&Thm?jvZEjy|g@Gk(^$pE@)F;=SHQ^Y|E;;cONpG5#1peT^L~?qQog zhKs-DJYx!lUPV~aQTCI(KsPieMj~iznIa6vB4i!*tMPLphG7wj%qfMc6AV}PP+L3% z%2d@rY%MR;80~x{tc?4jM8gdZIyN{n;)x&9@?30B|D_3P=Tay6#_0F;M^D9 zN_4eVFv0k&Hd^uo<^EzHKDh$pP#{}^s)!#%keSefCaF1Gh2N;bsW7xwCcE^siYrHw zVImkOtkJo$T#vYvCm^&m3Ead^cS^L%Ihs*=s1+qVGrO#Px;5Exk5H(TwQ;B9 zyLa;UKwcENt*f1*gZmi_ zvEJ8US1KQ_N3|{Ww8okkf7FBg%9bRNH#%9V*&a{j#`JxWEeAt^&Gr?r0 zaOY7h1v5B?TTAMpCVQ=PF^7=nB;ZE_LdC4EYBrxgD7~msCvCJfoc~@jvXhd{>TIy` zROgdL2x-$A<;LX)h9Jl{e&C97dw@yTWOjafhQ$2EVTQ)04=!Q< z^Xg&&7=?#bdVkd`jW*pwyVM_6+ae-yQ$jJ@r`C8=!5R64QgmKYWs*pCVn)Uk$5L)j zC+W>KLNiyv$POza!!Sy7hm=UOe2&ME=9=MD2_l>I?pz}7>|9)H*PeQ=0wty&$a5TG zS>;g1@N!6Yvrr7iz4Lp+@7$f8^JwVvU^+3L0S zlro}8Q`nBUXjTM7!$3YX`*BdjDep#(FrlOsHq@x1eNujI%bSu^5bKI8Bct<`Fu;<$ zLrt+~ctDwfmjw`0!btAZ>}7g=_(VqWYb=Fd#wU#$xYe943S>`}n*2;B_tvuZ$jj5+ zg%XDaClG&##%{n4EGgy*B!u|{y#+}zt z_Ofa7?BB(mWkZAEj~1C5k9HMh5>6gimYyqtcK{PTadhufJkvu&sGJCTga^Yt+Bi8N zxyFCAmDA^HR$i)@EnrR_TGk3kw=kSLYc3tuip_fuhPYblgclUMIKNV2>mg&B^s2hg zq!1NTj_H3*nI1p*x+}j+_Y3ARHi_1QO8M%babrxyQ?NaRK{0I(D~|`@5AP4}2QJhk zxI9!q2TS(1SH|u%1~3=YI)~@}7DTCdb>X3j{|ng+UT9|^CNqNtC zTVssz?q+~6(=mryV`Sh@h89t>oDu=cMzz+>z*>eD9tMM{iJ4qXbX06YEJOATZ2{l1 zr24HBCqjm8u_fk0C44M3r;Ke1E+-i+DJ|(qkuVe|=JBf_Y%H}J`>xbb+na$h-`q-X zAkd*U5Ca8A9S;kuBvBwZAE&qYHv_tmbbB~o*W;#8%6H2*L0C2)+5SV zuM%vS;0~iv4po;xjG3EUP9@;n)Wl6rKt&Y|?p>)aI%x(K;n==9KJr0?1KsEprp~pw zYE6&5$6=lbyuFbFE&w0&6%oti1XV0@rE*#NxKOZOkXi%QzPfYh9gL=7fpp#t8!UGi z##tNr< z6Jj?mIw%8#4(CCIdX?H9wvx|&;Z|z?IhYucA>Bw7<|V&XNa5%0^3cDaIqes?EhH5f@zJWz&x;Okalq*l=ZCwWX@2YR{j__cdnn38w zfw-J%vv_eZjz4pJtw%fZ!qjwhUgJs@p*U?Z{vs`Gn|5 zCpI=_jr~K??TPqigy|8$2!^|elBm)mDBvH;1>t1i5`yVH0{8=#yK%sb8M>|<5!Jho z1m9JaPCH?Dl)09AP39*|+`2M8thcIO?Af-NjykfAYDZ)zZJM(TCTyYyf1Bv&LZNZb zuY3P6c4w<&I%*#5)RivEozn$sKg9t7&rV$8JX3@c>A zt`iuEk-UtwV-Fd3+hmBG6U?~AzTY&-(;ax{zbDfE8q36lH#JpbEjO&*+*wI&@XHHF zDgmFZ8eW$X8?it^C)Mg38_C|s8SnhfD`Y?~VZ1?I@`xvcGyQ}Ikr{bb6?gz~`2p$F z^Ar9@+pGOdNR~7X#_3!b+Pkp)3~U^+<8R?y>D?1>DJ0aN<%^ukD~xzs;z=0D5cH0P z+;v}Egwys`%=A@*qf^D@bw(kRS`Y7hMqgz}+kTpoqugP=z6#2vQ>u0qqS3()3Lq}E zJU_X(se}{`soWHCPq{VZKGI8XihSizn(Ix81v?#wwqUIc?e1GGK3iJgARf1gpu9nv zs$7{+eeg&vI_puOv^=X=kuX8{&!ayzZ3f!z)6r)f>iH#qXx|nifx;8(jMJ;a#oyXD z^uojPdJqf{Z7unAxdW1p+^Hrnp)g#(DQntq-1lu-5oT|vZ#3w(>el~5BmA7SIW6(; z1LZG+$UlJlp~;6$Ee`d-?bI|gRO9>^O#;b*p@}a zfgga>3uE-TNH`arfg6G0K4{QSQ}y=gwkqOTpyJhWQKPUgCk`3X{T&-wX+9I|QQB*+ zN=pLDpO^`!cywpYSc$$9fFVl`D7%^m;1F0#LW0>Dl|zb=;Cc(k1;sU!$;4B~+x(zC z*HQ~jt!+(#qT*N^?1!&T)Yo;lm`OS<;sW=QPt^Pt2wv&0fX_v zq6BA_x_iH!LDgBY)^tS!v(}J(u6O^r%}cQ_A4QTY1DVscB5e+LGRMS37tKV z1#0Nr6N{$@RG=7bOAS8D)}&L`%BEB$mtbNE&p3sqb2Z%FV)S2M2%HcU8Jlu0RQfF2 z&)=OcS7u*S<+hn3$3(=UHL#Jpmz=Lux7###u~@Xlz^Cfvwb&?8f&JNWo@~y-qJ{6; zsi6+>djJPZ2{5>s14LU9zVDpCvb>`%LSY6M;dK5MJq@0NOr`kq{j%dx`ch_5{gjW} z#oKOEq8>jnO$^U6oPEZx?V!?!7W&d_*2GkrA?Md!X9jbv?EI&mH+@j=%a5DVozk7T zRQs`RHhRNNrS4aj?ev@f0SBsyDcf0jlj`jg8&s`K|3Ti3tjP+edhhwz5W3|bW+TII zn#_3o9cyD5+ysD{1tovRe$-ij*N5>7{|sj6g{cR(1tX}L!sGPsrLzXZ4?qS%o`3;s zrX$(?L6x6_gX}926mGC=@EPIi?+X8(6UchyUoejA5z(e_up_~X;!0*osRvM5d^2>k zLHLYvSE@VgJy=w*RIu#n;rpo4lj7cmE_D68u)x=`8iYoZ5LESR&y&LSHQbqU>a?F)KQ49Fg^vj^d2s5 zhMJe&fB)o$RzDqpz&#)9#;x?`76DMdbYNueoD?`F7r-EXF`|VqGl=%c4J~^oF2SGU z^Xq{IC-DFmLr@!TvmOs~Cpt!|jMOxUp2MvY$_n8UHPj1ALBUU4Ts!|!>2PMJhO{Ds z8jL?Nc4!pA#3i~+3rSK?9r#LK7mN>bKnzW$VxnPWFo_L#d|h*4LuMxqU4nRl#2+1{ zJhY^i`riy=fIm7b9Vh9`nN*ZW7<*tzDMdj?Vb<$WjNx|Jtm9@+m*Rt(`d8x9bh9J_ zoNw11VKuJ7#FeP=wJab3*#x5?&TYQTD1c5$muh7qy(C zy-szS!hKAA%B=2KRw~Lli8E&Lwks+0Yz>nWfp~P#*`*%}`Sk#dF_}4%_bisfW*I@H z`73%(QnJlOHL~z%ANBMx>GZKu-ZY)5F)`}fAa@QdohG(xa*3*2X-otig~?b%20QuZ z$M2k;HH6n0+FkLS)YQo$ZTiTfNZ}Z)S8Hr}+&+iY@Lg)xi(=j_s#y6qU!}!g=SWqZ z>%CCf6T54WY~2K=Uz+e67ZO|v<%V)>e$5C(5L)lgq`a`FKKY#?9(-3jWFo8+Nan`-eD3=@-8vv+~p#14*6PCzm-|M%jZd=q&{bl4yL+T4AWe- zbW$Cb$7B61b|>K{mrYTIYBUrmN>U2{i{-s_)W~mkBSR}EdB)nvL9T{YYV75Ky)g!8 z&j;`lU5VsK!;%uNhNh!FZ;KHQHnvvk@PjiH5snqszBm<%E@2%b3;-kegN~t9)S)&@ z3hpN`)xMN_gAo>42V_42_tVh}({8{tg9eZt>!=t0PTEYvs(*@Z!!4CxBw_aA)zyt_ zv#P07(v*tyBHj3c;2e0BGKvIcv-c>nY(Xw40s~vVQ&3{=*B8aCsYR% zL!sD-Wo?CCY+0I4pXP0E`j~ETA_ku(6IMFfkkvBgPqqIJgX)#@oV86$d?cIn76AX? zc!{9Y7U-U*4)?U2l1|POd4r)*<_=X(lqewn@{UXy(eJdOdUPZDo?q?W>e(>2_bWB5 zjR84H)b^1A*5kFv<%IrTMamdO)RJ}FW$==zF=0|hP@JcHzoxjX%@;=0DTt8$1$W$e zsCrZzRU7AF6#@kjz#zW;dec&3iEcTIHxSC6tz_Yr$8Ft2A5cOLQ(;XBDgYo7eJ%vS zgq*2g)WA|UENyPGf~jG02@K?W53y4)|A9(<05kh=FZI0@9QC`sYsQ8qLe*S(TpH>j2ozL8&ttr zqgps)&KQbRqF_6nA(SZPVe2TySG3pKOd>%OHDG@)Ty@ce~dU+$>w|8G%qU0 zZI^9_b+am2Rf?(HVoh8$4i+nkao=|-k+T!Wer$l*6KZyTyf}oGt(e0g6*fP^J_5+)$PgJOI?RrG)$s zP@bQ0ZZ>hE7Hd{%R@QESNHCQx<;NEQN+m7?Z}`RZLc7CiuKt`T#6h!srX^2CB0huy zg&)(N2+D-ak1ld5^1apn5ty;Bhmz5hv+Ap@H&C<3{L(G>NgXgRs?_1L(ul|T6A`%M zB~P8_AFlC;sb{+862fa0JF#-y9s2)0`k7&>NB~mDS<(? zYKHf#!09JES{s_e3j6G-jmSN4{6l}=Prxa9M7m(jX*j4-Qe=)LAoyM>jTq!8l)8M` ziSC8vg{92~PkSGCo;6^s9WBY(-2$OePc_f1hIZ3wL=3wQ9Jyd`592$-Y1%0<@H$dv z$bJ+NZqeX|tr`)qBH>kW0ub%}Yt)c}R<}_tK$5LI9+dwj#T#wOW*(yK95G=&1ZMzR zFM_KO00A6CrOAU_B^fWmP;a!`rM#F{we%}X?!;5XK9H#y^N_GuvJE#vE{H9hlfV6n zOfA`gZK_DdaR5qqu_Db?fGP>Z+40jzCHL`7Zf>}|d{z5!<(FVoOq2=wDUiB$4!7_= zzhfG8ls{$5%Wur?Q!}8eeDZtc8t*c zm+BiaoojaBIgILg2|EFAu1f4uFURoG=%7ip(BSW{(A}~1B`{Wr(S;lm?d8e8OpW`b z(;OHs7m>_9N@+yLb7S#}<(`ziMcmn~qgs%$&i-5zsf2h4otzSw*t*qb5ZEZXHjYq|MYu7GBEa`cU z+{W4pV?--5a+SDZu0^}VF2B(r^!8Pj-2@L@X5Dm%elMZ9CSb#edgnNeCcVx&jPHzOlUX5>8_N}_9_EFp@vXBI{gitdWfl$c)owl-LN!-s^;tIo+>?pZCoZr4skv|z5c=q`7rb|`soOf5l1VjXnGR*VVR&yiaCd>!gDI zgVe_CPr`2Xk^0}F7aGAk1-TzGK$>#pRPQ9~@3f4~@4j*^yE&v~ye=yZ)~CQ!{c1G6 zQlSmR8}6>I=<9w-)akL>)7C)Z2kvrfwF?4Z*TSD8MJUP&gN+2IWso@drp1tym0!fU zmM9XWaw*7&$ti5V1~0bLe?+S0oo0oTDm~IZ&j#qMRh6VL1*M~_>JFMJn6b$PjXX<}@a1qOlvLK=>D`br7j^%AK?jl&PqWJ{X+&;w*f-FfPcb<<`X@G<`*{M#j2|Lp4R^DgD_5(1;Di{i)Xdqs zAqq-fnqTX6d%bOy=O|p7=rNFrg0r_qTDgmg3YQ9(Tta9HZ8>gez#Plr%2g*=$^S3A2BQ)uXmCo&~6U06UXIPm>7%X{W1hLDp4O`=XWxlHJ^=U(k-=&*Yu z>f~X+6^o|lF2o;{9QW0?AdqSJ``^Fyoe`Z|uVwKrA8BehdhID-cjX$t#p;L4{JfNs zl4BmrswTBx<=n(^tg=m7Il(;p5mv&Sg|NtsNwU?El#$Zk%O>vwP+RE+>9%iA{mnq% z5+?J4*hXkk>Yy*4E}4bir?e3~&KoQ9i_7KJ^@q}p4T{P7s(d@51sgeq;e0um|FD>(6mGQabfRSHnb`Fvs!{3svwZfx>OQUJ2kN=sV43XSZ9O>Za@nd?Kf|pye@2b3Ga*-T!v&-)hG**~y`DowYhC z*qn`buS)Tz#DxE5Pf?>Hr+vs_;-X?&^S!Rvggi&=KR@>0D#cE@_Vf9SSIH|Q$ZVCK z49F}M`?4=cu38U#9Uh+92p*e?F&aSKm<$}aBG|OvE#cf%lJ_nV7%2J>8A#^npch(h zWw-dFB2oEnT0lI8twemEXyPlxx3a*1c)h9-SBclpn8BzWL+_k#Xv~C9{x>|~>97+@ zxYpDIj#-D0%TszbyER{ccUp9Up*1 zd=McRB(g(PqenC4!E<)XN;z4Y%4Vf|K`YC)_HUnpdMj>z$n-eBj~5wo;i* zYTpdN^l@8n1&@`yInU%bcnL;}PwMixo!*%)3)aG^vR8Ew0`bA&k#$0}hZ`gq2zb2G z!K(oXi{s?UU>lBfV3SKfq5&wmhIftaIH<%SPyipuR_<$I1=6o;NNpJjDC7?z=$7iQ zai0VSv@V(_3a5L35{qVL>0 z6qYq&#Yzbi>aUcm)2HPS_D(PBSJ2$1s_oDI^ZFWm!kNjeK(wvZq!!U+x7kY|>obbjvQ%EtdcXH9{(g~u z8^2(Y2kS!a|)5-5KT%%1}-~Ub}&o7z4*-6>!EUl}Q|JFe2!}ppG zo2RmnO z-9ijZTWoqE5}@f!);SnUtZB^Bes#F7UL_>^hz7C@-JYJc85RBllV}p}WGS6KbNkaE zTtuk2gjr{}$4q(y^U9AX0p~RTATe}p{Jy*|cdkv5h3u^Vyctg@mP%G``?XpP1waM7 zS0`4uJ#+Hc^4LGrXl{y#Mr05H?^##ifu#2b7H+XR1L2ixayvb?#L25tk%cQftRnIj{>w#?xX>b6IS?}{PN`5>h2B5Vp+ zo|zh-hqxpf-?0AxZds!#Y3_6<;yj?@z1n3(oTetlx14{fk2tj)0S#?Sxq9zVDql?w zH0BuZpnNuup2$-owZg|Y9%JLju;+zEn7AsXy`!V}aUQTA1P#jxY2orAR|`;;twYwr zt9Y#`?BgV#x)%29F@rDC=@~Ej3 z0jEDz*sSo`MG>j-j{s*`j!HUP>NaF)VDuEpg`PB;VE4|mXZ?h|vAJZlatxacIXh$! z1EZFWcfW*N959V8m;|FqjvPIh03$^%PYMx8&_+sLBNC_(OJ$cTUWxoGJwhELO9M^z zFrKeit28OiHF>V=+o1yD3WVwcsjEt*91+`quAPt&YfhC=u*%#aDDlbuBa`jt#0I7RX8HjqR0`pm8W`IwJ2eu@b#FOJ3NM;c>(P` zqJIRM;--Uc48Qb@9~>Wb)I9nxg+%Sz&83=l;+-0n2djk+1L}t^vYCXOyg*>k)iCjv z(Wg=X&BAY6;;HJ!Rhnf$MDCX7j!(ys{+&^H3(PUa-+oM=*X9CX5uSmp)w$h@MJqzfo&B!l(VJ|*p>Y3B& zGr&LX(w3nPSekLCe!sN2Bry#|6goZ9D#eJJfj6(GRw~NUU`Ng+)9-Lx7_6W8jiBC( zrKVFCFhkt;QSNy)T*u1v$Ebkf`*}_A60E+(S3O?QCVs|DpF>PE?DOt5r$hjY<9e)k zAHvr-9UIE)V2Le%FM;O46hcV$Ia52}B7&cRzI51+>jJQ)=~gZ|nQV;w{&!ZF(_s#O z<>9s8CmCypbmzR&oJjvtmbYCxLSdH~Up!_hb`wlI8#}Hj_=@?fW4r10F*8N~I8RYt z!48!Vr@dcw9x6=o@ZQ!~5M=2PmLmIT9o=`(oHdwun}@N{9!x+bj+VGKEW?C)9MP8~ z=tP+{ZTk_aE@@xj2{j=}8!BCyd14!CUQLJqC2UT@db-)_J zPe9VWyyQc{pON1p(zyd%{AnY<#qg20*pyCxkt|u-YDS6DT);R5i%i{_Px*?A=Tg#+ z>TRwg%|8f73|Rv|U(Z-d4h zG~rQ{3Z-RSzj&`EWVc=Ek4dnP_y42tqdW48_9+ip14G0*dq9+7OJs@feP-^PxLcHSvp%m;HlVN)%VDkt zetCQI2WurF%SU$8w}x|CJkO<`OFW0m2tj7DSeiuN zGG7{2V#53-oIsB`7Ct2_W+#Dtq`Z`lMqEbl3R*#;E3Sw2wPz%d!U_;Gv8?J=J`}4u>6$%k7(>3y&ut1fL*$P5(k7}bOBXlEgVdA%&4(|!_8y@X?Y3xt|!Fi zb(Z=$WXH$(&!QHuQaqp?%JNGPLn2l3!AMLp%^-_FVZjuOhRDS!$(jkOXafe-u!-wF z7Pvf?S!!DHU!<8f8I$INkNLui=kU{fQWQ#Q*?%#m3uNqv{u?i{eL2UG!9n&=YQRmw zyvp3+angt3No9yZPBTS}x1v^ibo9*Krd$)t7KECIFdE2xwlvwhj2-u1Vn*HUH(YOC zUdN6DZ#bdKNdFci;I(S2d zzQ1w9w)nuc&C&NtbvcCLpaJE+;#~kdTxOfB0N(rzCBvP(Okw;hw5qT|H=G(Mn@j-o zy=xUBpx$eW@^8y+;lgp<$o}9I4T%nbS{!W}E=xd-#}N`gs1AuBt{_w-2U5VNEsFqK z&1!_fDf-Zj9Pn2i08%eE0C6zSK9%(y02_V;;1&Q-4=NM;tvH^yth45%=m3OWetFWU zN)wL>zTfMhyoxSfV`FWSRu}!_%kqGU|qqYW?O&q-XKYIyr!Oo-YbcYc+ zl>ggFzkeIbi>3pp4&jG45H)YcAjw^F3P)IQZ!PhEJD`7$b2PI`ErtILqbR&*ybQx+ z-OCmPtO1DKn9I8u?tK>|f+`82q+ zc0o?cQo4xAX`WuVS&RLbBL(N6h~f_s%WKn96^3V*Fo|oWxci|UA%zMCMD>C<29)tl z_BoW5jPa@EFJEw58#X950_nGng$>yUm(@AXAsW7HCK8CwprU9cKS{^MDiJmbWj`In zu(_{;r_Ne^Mnm2|C3k10x};O%tk%2U@p>F2uv^mS1y?Fv z7*E=}2%!GxGD7XlOCsI)%;vt^e$(QH_=zZaH`jPakb7kJ1{mfCoV-W=1UzJJHBny3#T=0%&gMO=I z;g{i~0H&7U>|p*9+u(;5u|eSlVD0v`<6X3iOG_KS_cy5%zsZvK+bSPxZ zUP7X{S{U3-z;r&;jbo(dexYO3b=wc^BV^v(;zxb^wZwD;Ikn>3GWM+q(mH+{_4iFb zyM4GbAV~!12|AIpEzLXKyO1VCU$n2oKUX03>U1cK*I7lRM}~-$q%%4wxJ>Qi;I@on z8Zd@mH*BQprzZOT^q*>n;=h0F;SEn1On)9EtO&yE#y}X%pI+#R|3$WlxOh zt-4n{@2Tt*l}YK9;&aM8%!*wWU(HNBcv>+oiKsabRo88i+;4SGD_f6L?<7^nm*;%I z`t&aXvAk4riJYM*B={AdcPr|I=DboXJ4uyos2|qxa^}$Dg;2Q&dBotOITti)>l|Iy z%AI`d6*PNBSCaS485f#tnPm~nma}Lf6xrN#!824sGB}F-_#JO9RdPSM+V9NwmIrGf zl}4w^ywIM!nN>1P z;92)m!na4pfRf+#cOWIF0HVXB}iJo`K)|ll$ymgEpTph$bvKZSzi?84~y}B zxubfe>2r8PzwzsT^LhDiARRFK9Wq z1lI0Oz*g0y$tkL$Q$Auxs&%XHwgsF*dBtuO`8JUG%UQp&ZpEm&bWQwQIZ8vwzK_K3 zsJ!+ z;9cNSiJMcMzM;H8Ny~d){SvRtqYm-!C{G-^>?kK3_Vt?W7~_5bF7Jh%=u4I{#Z+_f z;UJThTM9ikQT~nw=?LrR-~DpDe*WM0A~YvgwUS2DTnjiJ{1%APB4 ztdD4mWS`zkvis;dK6@M?a>rwXnNNTg#K=tGOcH~WT<-xcww(u?*8 zeEPTFmjstsNp_EL!en5a7^7D*Z5E<^K^w2Bsf-rYKN*~(G{!9K zbO!}>;X2Q~t}05?48ij+ozOgo!MV7m>xt z&(k(;RvU>(l9qgm)}(rnm^%HA?y8Y~*_BWQ>Zm5pCmeb2h}kdprO8e+g10SFGKV>$ zoz~fWSC3D+t!O!gSFDLE3KgAEZHGvcHEte5jNdO9x0CE;`S<@onfNL|AIXj%5)v+s z{t|nlC~KE(-6N6<9RNF`?)n93tE^-cOL78LP0aM++GF<{?_^{b5J2)qyh z=tE_)w%*_ZN&c)>0!7xvYO~uNmb^yk$!&$DVf^;UhXxW3E!T2mh5cIX=0!HM4`b@> z3J(eot2xXYN&-1%Jx5Uy&YHpo`;J9IWIp+f@*S}Ag#ILaF)u1-^=>|*3ZG_!&eiLc zYH+pJJE%U8R7{ITMmtZc$}sUtB=)UIrmMVpT9;w&-axxRO@{089*7!>w556IG- z-fOuQ&CiLm;WVi`nM~0zm(OV{)4t9+iA)i8!#F_F%g{KHi!;8ObLR1FCF?S1UvJ6i zq-44vN%oF~=T+5!Tb%y=MN#tNE0%sEU9$FmDagdVXAq5EIee&JMP@R=p7CpQ!jgHpz81<(HO_lwMs!Z-{UQpKA;$J zeY&`xT>XB_X5>bDj~1|X_Sv1hP^Q3?!E>gPnq57tjeQ@t z&ANaU{GOflHDPLL!c!{ammU=?dX7NS@F(y!7*XmplGe?{d@=kr9=%G>?IVmGiyKg0 z8J(3?Rvt~-Mbb~`8O~?5RjGz(pkCC?G}EL{t1`bV6)@@jg$UzDsN-X9#`tC ze(-|>d}W;Dr9RcLW?3xfT2T$OawDTtZ1d%;*t0gP8f-nhD~iq(Sc%LC)?~I^Emswu z5IVhp{W*`klX9-l5bU&UOFjEX^n#v+O^+xYX7hn(an5&1?0QwixA;dHHPbFE5(;d5 z(o+1{yC!88v)+jz8MTg$BK$SgOL(LC5nLYPFxM0B1L+}t!*Jm5t3#w){iDLR{jvIL zo?2&32~1?8l*tp$$IAzlK%OLe_(T^`qcjCHzT9q~9?n-^$FQqkse zQ+6r*T{Ikj(`PlZ78bzIwjsD+zfe@4K}C{`RN#NtG$XZJEyQocTW|?qK>3Wv1}Ax?S0(hesj@nXM^eBjD=DabFPp*3GFC zrbe&ear^?T{rIA+wB27q$&jroKSLE9|u_qDu~ex_dY+JYU{|$Bw-PY5OcYskchL)CG~aOix<= ziw9D`EH!o0hG$+H35-HBDhCqz~)nj$dVLAn@r2*DfPx$)Q6I4@A719v(*ik|m z1)C7bmf>LqAC$F)PzVpjjjq@_+`V%9IuPLy{WG=YZQ)E{2XuYUwej>4l=VhBz4z!gpyQ`HzOOY)UhnDP%AVSK+WXZ)%cD zJT67Ey;T^-wz*b<88T3U1n3-{J}$3@xf-jUdvGV)ctQek+plrlx$?FZMD}u=AS)g@fPC96#u7YZif%T*t>^HoWTZ~ zBAgIOZ=ur$rgp1sr$+zooMD~`p-9!6d-Qv}MQEw$D@qrN)?@W8ED&V%wa9hhyOLqB z!C^K_8`UV(+dz7Pk2mpZu!8D20c!TFTb`GqA^W!ii$L8~u2#_1GK=<)0ZJ{C>m;Fe zCOddOs?`dac8b}Ei>CLNZn?M6Po)?|8RAwhn*0iBv1Q%zc$yWy7Q;(s(&+}~g-L>} zkisN++kiBxlukQ!MG?0-w7yd;k;)}qvsd9J1{Ho5Rhd39AiBxZ)A=%uLgjdYfCUJ& z3%R_Ik;lT4n;iaeh#-ZacZg-2g6~uD@hn)3!yKcjpi@cTwhA*2nhv=q3>W)UyoP~eb-bV*n2EISX`8)c_*`Yo1o`byj;OHzWmzR6?-zMwpneQ7 z$?T?y4yNEOv!X~+x%g+jitp-=v0y8fH^LwO6tX(%JmN7({`&n~xn&|X~-stW0WdhhNWYO*iyeNJ0nqCWKrMeM*ttoosBzlQ0SacL5=B*lxv_=0w(R;9P z6>u#22X2%ebEn=!IC;&*A)BUHri4X1-wvCOWLhpUr~Dy9)p);d6Jm zdqVd+Dj2B_VfiRo+utw&Gb-F6;(FccyQKNvJJrXQnP2b+hC*r~@9bW`0vTRpxg3Q* znqE*nnfdV)*s?pzRIwxT`z|{)ns+AplUArPzwQO>sfsCemMne6%c**F6B4gaB;PR) znDyT1SH+23T$a54sfv8h-$X@z<_pZeOqA#0pK{}$fZ?&UTYL$}>Q`DFYEqP=ZgA#6 zdOvYK+4)aJAt&6g_ZT*nU&fVc0AVTokb;7&C5&{%8M0CnU#DUXp4M>|AluOjJ6nCNQ-nMiPy7+kM~G zs?a^oJZs-Tb|II)O>N)F)<{p29xoYx*~uJz{SiV2>sN)R)P=nAnr|3TeI0Sd?dT{O zO$Hj2BiOt7Gc(OPd{x*K(}g2jx725uPz35ATOFF_jPT-VinUul2t>{rZShvCR3BGW zuiW^W&QW;5{6$;8{3%Z@R0hYsu%Cg@tDW&yl%F;jrl7zjIsSh?-4#zb6+bs_GW$B> zuCA3BzIaCl8W*Bl!Hd@i#UcPu-A>u99RT))uZ5C&pVrjyiRz&C?_%m1)rl9 zBN4-PLC;53*X(Hzen$-b-5 zf;v1}wNBImD-e#m=5Q=|3!Pha{NYOo8flg2z_1ld*#hr2XBi;O=LK*nFO^;wN!*VE4EM94R#NjCyfiyzc9 zl!^$S)v(O4M8BRR^B!({+ai4kL+r}hqCJ4Eke^QJdJp@?yQMd$-J<~F2`GSk6>DWz zy6?m#rnmT0pnr%DaU|sTIOlb}B3$9sB=F&zpU!}Gu!ph$u17}^FE!y`<9=iRu{5l` z_P*xS{coHwzjtE?LePoSf+;gdu*pmcEmVgrj%U;Ll~9Dl+?ES3de(5@+_DSBZzYS@ zKhqs7%wMX&qk8XAM(i9lT%|VBwv6F&v<#QTP}MRMMW3!spi9=KK?&n{?ceBqnIT3g zgG+9l)-97}JjXc>h0_*z1l@DjL!ogngHqK4`7@4W9?u=6MbE#q?Q3dNn0zd}dE93p zB)93NdMSZGJh0=zaBzFq?ZvVEaVf(&?ie29i@gpfdS|=+ar)f4@Ca86^d*mVvi^7{ zo=P7bD9<|C9L{d*ut(@8vCM8tNs(((b$EE`6x*z>a~plm@i(3#O37~vO>1Q1@%3=R z=`rF?p_FXrNs(`5m7rrMOjRsSDpM6#6)z1kcF7N2llZCGg|von2Jiy>Mm0s}iL~v^ z7dk03i?Fs8!Hpm79jvhv=Hru;eEToy3UYHq%2>lGf@#jx);0ouneA|P(M?#cv4Rlh zNW7rDYC2rnAX%{NnI|}N!19J8J)X{>jxwZl%XF`^E;8-|sQ=u#de(sx?$g%L7ONK#x_C1n+PHX?_w(E?M)) zkYKYlcfy3gMN6CxGiKFygWqz%S#E*02~uBX9W)Ln|MccnGn9h$d@B_WZ|1MP)r4^G zn5}YTrB=#gl0N%Y1Zv#UM_$IRk0=m#U~DuiAb+!JkTbOCc9`%^`@!DWF)U*Ct#|LI z4qSvcO6mAFqoM7G1Z_y${IfbNv zc$#n2L){EnB^(p!Ua$8aUf{)J+G+j>Wjz&tNsZKK#4SUcvKE=#g8VpOxs|cspqs3m zhpi6gXXlsV`(?0x(r!o0O%YxCz3NNzz!X7rlsMi^w4YcI<`Llas4nanpw~3_qndl9 zgCNwGrD;nqwF5rEg21Pd{pB{x5saK|FZ=P?!Z&7 zX$*av3}!1G5$C{}pM_J?l=vxJ0QB*j7c2N1LnQ+A2#O4fEO4E^j&uY9$5_sJtkQ@J z+xrZi!AYoM@$z2IDh;s509Id-;sa^e@R*v%Arp|dSc-|QoaH?DZ97%A2F$`-YxgF@ zr6C*k55AJVWw$V5)V!d=gB2F;f>qxM@ckjHf0i&1<-a!-g$wO)(A33}dqkdC31$I; zD6+&qM!aUR;!z_Zv{Fm67k} zEYJQu2#tvqyoNzZsyC-?NH>kpsCH>6E z*yFd0qd#?JE`X0XaaM}+zwv33N_yIreMGc7o$S_9);EFYOBjUsEp!+YDBxD>{RyB( zC4T_wp}zu;Rw=#N2~2ladoXyQs1t1Us_d-Z;nU3g{!4)qXUy{-_ifVtlI0jSniLPA zrQt$;#L!KxsASqSPwe#+J)I)~*n+AN_RJ8zk>aj=XfY!$yQ)u}!@d+f!lB_8kGEv} zJjUr@f2GAY#2JPcu#%!b7TNVi3Tb1{Usqnnk@UlI=x9q}%Bj$9x`0@pkz=oX&{X!_ zFS6@co*vy@@>mID8_VU(w#~-%9LdUg&9NJbs$R|u$1~6MlZjP=nEtiRpqvDUnSt2=h&~`jGHWP^Jm4gwLbij z4c1WHR@t8FQ{r&R3WS$6`z6T~$pi^@N6wvfR_=v} z9Oi&{uka$yw?YYTzM}-xb)M|0cc02qiu_I^hEkKTO_ACfxa+MrBJB~$VkS^^065}E zb$15BL5#U$p-i_(TDKLf=BEJBpl))k<~3(p1|;ocQc?5OGT%a4T;SxFOnzIwd^LiV zrYdlz){A|EpmlE$biz358`K$ude3j!mB_ zRe*$!@f}R?9ai{)DcR8CghH8W@-k6ltXKS1g*5&v%btmwgy@CLh`)L>J&3_KnZ?AK z7maCKfS6SlbQo$NG(}D}wf)JUZAv~sS)C}-rSOapLSh|`j-F%b8{zYueCUmLiTXk1 zeJ%DY3G0Y9^m_xG75mW*9bjXgB!{nE6z}>0PCO;WX6VHN$lmt`kqn?%!#XXz-e;ey za39twVV9=A0C;H{=G~JFo$(Me?Yn@op_CLl;7@Ru=!JS@?43z#qoi6Ab9!K!zy0Ru z6KZJ7-{Q7G-dNek!l&q7N#~fs#6L+v^^t)31k<#zBwh^mTS!pp6##?|qKz{>i&{{% zp5k;%P+Rphb3ZT&Sy>A%=O;SKqp}(80+JH6CMIdu$iqYjfWCK5B!z>LIiQk@vTYnPT^24 z8{`H8Q*>dppa*cty5J(?KLtaXhNDdSYhzl_Ff4K~?i>SJ82rRiee`Raf|32g{^BXm z8_J0}cskAlunpK|$dYRkjy7hA;f#?m|LU)*&voVQvhCRSAJkQ6oEwsTnqI^HTr1)J z*47zvZiu#|2k4|oQ>8$kUcJK1Ly#-5h!2%%8o4~8@DLan)1aaZ9kQ-l#wIa96pEUT zaY zzv8^gjLtKshmse)B|M%WqF4Jiq?pW{0sI%FKOlcKeIaf)j0i%^zqL#l59u>A<*b>K zGR?rzme#+Rh{J`<4Zh%pI$9@AVmD+;2)rG&hJ>%UqNhsZ?(v2iS|^TUgY}FN_y8Om zIw$0RxZ4nI0ZE0!a`Lp0NDZb#V3g-@;fGOD)bxq1r2QF8BPKmjfGw>+#WKJVK$2aE zuv|v>IL6~ot3@Uz8yautI*6U1Ym5+gG6?vhWRmn-{3lX-BtaUy#h&J9^%_I}Ml$?B z$pCKx?^U~}YBkb}W7jBI?hAY~%Zv}DZ?MV5%&l33{fv>{qP#_Egm-0zfzqCQ2s~n; z&(#YtvYZpX(>V&Q3vWbeM3E$j)C5oi>s`OT z{#D~qzbf6m_i;_=BKa=dq=t&}p}|BygEQjwE9w@+ljX%~0{1n16uCETYDn~??Habx z)kTH_@b&IVzpCX=d@xqa%iVly5ROYJnZ1``i0Qa=EM^&_rtw^@M088Rb4=CUvD67I zgdE?mayKqVC!>^IXKIP^wMv>n62J>u0b9*jHq_6YwFow`?5irS0xB*NN*r1J(X%2h ztIHSPmQxb#)MKxfAVxwE60Ira0TeecM-3SSC_H?;403*IuWqIBEra8Dt3-ZU))=%y zU->bjP92Z=cIsmgD?!ny3(#xfb(!{Cw~p6k9^OmxoYHj>E>@`>%W@syu6uSf^`Xa_ zWTTTXhG5?1yBcLnb`Po*q4T(+{3Vk0cc=s^LLEvd;_P|`?(CwA%2@2St=JGI{kfb5&9 z)jm?$1kwNrzYZ)Nrd<(L=+HV0Y$6BPG1e#nwBmdj>mJo5p})V&6IH9;^z z5GRx6;}$lg)G3rrVWz>Lcq2&ohl7R)H#0A;iBQ{)&>o9PbczpNVyGDHf$W*P9sWT6 z4VWTM4zHOV4c@Sx{&9s#TcL-9texFwuK+-sDKGfXGnt4a^%Fu8O4#?6Z0O&YZ~YQB zeN3NmN7ojiaAcSDS%#pl1FFNMX*GEjKigtfpZ;52hcQZoI;3TUvmrK?5B~pq=O76o z3-MK(>GyDF)@RgzuD9p0ueaiPS1)Rk@mV*TEIuQ`iky?kv^^#5?x@pRZ?fQ#C3LEs zd#|)}`OcVfVE;ir9Nf^@^-K&I7WN1Vg`!#xRWeM0ea(T@h4r({0!mk&%$=_i8*n@W z#sQ<3uAfZg7kN#+I>*@XaDwC;z#0qNTOhwZ0NQUcN+s8i)}DI{6*%taMrlnAqo&_Q zn6kOdZIx9g4^fI7fs}Q0Y`yo?>1-Pk2hKu73;l*8a!c%jL|Bz)gE*o7Y)*Yv$|dCG zsFdA^Z>Zyzs`UH>pI<%e78Si6x#3G=kn#r$x>;&ySF3wuTk}GTQ<~G(%r?^#7btO1 z9_tV4w-;{rweUq>F-7rWHh#fQKcDo{dMYL*d50=`C%h;uBitfX(rTth*k*b1-h?o% z$xJ?0lV-1S-F6MLr)sthD?auZN>%t+qv`%63QPwYku;^E80oc-Ck>k~G9lgt>1VQI z#lzQfJ;X~1LU6PKSUgZf)bjDhqMSb>!&B}7W8_T4U8*x!bC&t}I+;vZ#5msEwE%&q z5`;BCoG%vXZE0n-T8nRWhP|E4vJf+O%|5fWPKqDmHM>1|%pcO*G}k$&^15`7KOdJQ zIVaFQ`zcBZOOa%iQ884~f80;g^&zykdcPr}P(@#O8el~!l0)HvstCON+00!cBgs#p z9__gnMV5ESTXJ|I45?G8O*qGX7i%Dy!Dsz=SVN@^ZjP}uB@{(1`oo|N9t#PvxQ#V0d-);$1F^_*H9SRRoApNCNA!{01XGzDZ?H?15 zRk2|R`9GX}1z1(h^YEqf(nw2pN{7;t(%mT_-61L6k|HG`-6>ttNQ08nDHjkBkn}qj z)ZhF5zVG>-=l^HVvv+oOX6{+e-95XrGqVbO-(Hwed^8;7XJjv!6HR)wyOYHR_A37Ylh|JBBh$Ej1k5K8kg5d*E`-~W-Y^^ zG#n+NM=l!yRzq5=a>~1F@|qvlm1%lEMb$q=b2*|nutb`dnV^a&_D4p3|6{4nUB$#8Bxa9)ajEnzO4+| zDn#0n@hiQHf-joxG!NkYP)Z_O6!Z8?XaYBh@rFLD9#$he^z4bOq>|;R)2_Hl`?TSwQQUQm(J@E zW3DX2GuT}Mfj6?5&)Ur%IBz_lbK%f2AB?}EVmy4HNNb0W9^pACCogQI+R)DQi7}Ox zhr^g_s~#}sz46Ug9Ucy=r##Uwf$4T*;F-eBW2G4*wzf*i+6pN909o6Rt zFN$Tl)xAOo>LlM#CXmO^S)y5>(P@!d(ykQlD%V6~474sYNpr7Ajn~=SGCz)}%akM}5$uocbubfqkJ6PC1H45cjQ4jEZ z7-Ew~V{2*Qs2+!P?=4mi_kIXZxxCWlEs&7Ir&iB{>kpLp92k@*Vecx?n8f=`ZrumJ zXLW$XBu=ILgCao+n-=UQnH-aPm{YlNzW6Q~>jp=))03@YvM|HeX`IFJXY+;lHPg7$ z%3;LclqF7nY-%csQnygARwITN%c|<}7)TI}NNU;Ovc0jCeTj}y+)e4mpXWK{>|`-5 zdI^!VqcP5xER^h5N^v@uDr?_ztMvMMnie5quwj!j8~z^48CmeQg6(ki)7~>Pqoe_E zhv__(M5%>29*UbC(xR8{&t`-^d~zYQ-s9A4VtGd5J0;~t1w=eX;O#RVh%$FfjA5bM z4WtT<>nl>{_%Y*ym0~j1Bp$j?@DIX1=Kd5C3ES#>;4qC;A`{HHu)qxIdPzG$;txDTY0v8G|K$_ z$i??>hs*d*buf8FMX zH`d@xfA=Oz_b@qLcPzPniLmyUS9ES~HNZgN71$R^R*6VNFM7p-Y)l%hC9_<9PGT6% zugY&HV|Ylt(^BKnV>Ft}|5R)p%O6?*g#GK0dNm>k@idD`(32(fow72`9RF%%n$Der z?{D5(IpUIB=URQ#Cm%iZpzkvNeY#=$;>bQ_7JnUCDMN=^BH=W}&b}dp_a`q2IS|BA zY=oGmVst!FDx_;4zc9!ng*F8)BQ5(`{g+C3(RvflT1sA<;PIre$C<%>*hi2LW5no- zjfSg?YuLN?4p!O>>#@E`@A7f-ob;({d9j=LHwwRolrA_XVGcifyjXKm|701?3@wBq zwkgWYuy}23;>pm3&eAF66hFR&s6dkR;!`P)_=zcujn(4FXUQUrtgoy&HiO;s9M4u?jDe4cLIcr@xgGgD%A+we!_q`%a%OsVJc^@%cO^LtAPzXDbukvZVP9V_cJg(|n3n!a;vf_ zvfPFr<(hjL9PY^+>M(|XW69G8ot?XVhn1MhZi#GsP1LaY91F86zEAB#&|N4k^_*XwZn*32%IndSxroqrN^F=j~W*d7eklGMTny*5L!K%rwyVB@S|J}im~ zhEP$q$PBRI9DAicW&nIW*7aG|_32_HC&L9Z)>|i^O3`V8pUCtn&MSwRK^&t`P_rjA zh+o7-ouE>1vj&kohi~GE8$eA1VNlr+f=O!v1T$7!VK+eaZ~;=Eq5e9E%5X(6(?vRZ z2%tauJgAcnB2-1v3%Xl^0UP7qpm8yXdNp)G2E4)xGNMo+%C8L6je!CADT2Gof54(n zHl`||HW#+#KcJAhaX&rff8+llKp*`Ny#!Pm8B-PTS=5PvKbr#!Fth(I8;&kTq+^Sx zNuanhbfFkikPLij0s?_aOmi0DN@Z(hTwK2@f$D&u%T7_n8QB$jF8CzlFdT?-Aeo$_ z#b;dyJ0SYMC_c$R3=?8!@*@@yMgTPk8l@Mi6#_vUclNCZ!?dvEkD?of)YTNVy|JqT zKAFwBDvB7Ai|Eq4H+PxzD_;_ZEw(C(8X~8@n~U7S@R2RDDvBxt6V(9)38NoaPT!|! zTRz&n$Lv5*;y=kNA~y6Rfy>mRww^`TM|^D`Ss}Qm8U+N=Bdv&9ke8~)BwkW6i$TOl zr81}-oWS!0Jg>fpcY*43KorCpu7K-Ru0(=A@>{Qc5PvuZmLlt*NU(gEW2#mF1v&Fz zqC}%Batg-kB&EJ!@TXST0TV2iA49*?{R%@C**E+s&o>Lh=u+=Iu>L3!eyj8OHxx*Z z5il0N94~_}u0mVgolTh=a-%YLewrq!P9&>NJQaS-o`@8}6$1U8!YOzIzf<6sn)o>R zwN8c@f#rC^hJn7=vV;2t?yr~FBvdq#`k_Ur6KMv1x`8~>@uN)IYDkROb5*G7EAxD7 zSeh_sYv2KaY2mQ>ZRIOKbMWxTnmfs@{Qy6HZ&39%$JsHeqd>h=Iq^Fa?3gyv#?30$ z*ZopUl#75oPgg)4ZCd)4ShSq91u*PdCpuzUTEy>S3Pbd1Vm3xF8eRRgSua=}G+S(n zboUb;`>qlagOj}`gYTlYqUc=dl9pZr%ljO`b_jJAquMon$6mp;p`b26fxDq#Jte&c zhPQg2xSx$L9|m}KsldL)eHsiVRCobO*S1k8t1n;)P_V71`=G1#p!1{)aKp;D+rT0G zz-ikRt}-G31Ae0$l>f!Jm`B?>hStqcNGoUPBt?V;sx?Vg904ZatRN07J4b|ZP;gVK z^D;#i&wyol&Fv}i?aw#X^$CNY40D~ z@7=e>lW|v$XKGt51;ZXOe=iv&7x87Vu3FHS>1-YzD}|V758wiysg?ryr7HEhK~VX$ zF*DDs++b?dLr<=*%wpbWzJ_q`4&KC79-|cRNS5C;l z0X{J-WC>Qh-x)Dwq{%>X2mN0M%ij==kk?i8!cBK#RHVN(4H<@%lPVCk5yepz>UZWj zxT&I)$evY{ht37()skn_xuKuAxQ7@Gv~Afoc&MH%`f~K*4GME;2$X7i`yC5q_I?TC zVlq|ECwr3uLf*AOV>&n-T8tE#baGdZi9Y3b$oU#y_5O^q{SFmZYfDt15y}ukmadRZl?Z3WM#1XjNKUm4+9FT z3t`9_+Q#sg+XMuB*V+hUX2CZBIEU!H^$KA0EwRj^CLKk-wTM)}SwtNtsbE~E6t462 zprVi}CQq9K&n|^JJdvAftpLc;liH=7x%jo2U*wt9wXYG+_-D|-^g-m(7b&N&qL!@I zbqr0~{pK3e@5Wf1a8}+oq*&(?5y09@`4x2)Af`ncOe{&fkS&34%b`FN>4kyVx^PvKwKBiibIkga({2T`NL}G;pkfN`h;6K+v97dDvWE#Q4(&9 zX2sT=Rj-~?%O0$=Ug0G-W6LFxWjCu5tz5s!U?ZfMOb25QNQ|k=r`k1zJV(}59%o-o zKSoL9rw?u8d2+Y=(s}EJp6k$1IplNmB?*2Y-|%HYY=xJ#zCs=HZb8`b5$+e4uw28d zTrojv3!;6V#B56R?5nVbor-kMdH0f3eG3FHtj|U5Yd{95# z>a{{$KL9)K>e0H`jgyf4=tFF)jvub?vbUviu>-E;;d*G%(6)+z1k#JB2dxy-v<1*z znoyD|T#`Y3l8qijfod^DC>p50zr+M><1k3_x}`X0GyxLeqUlB)tPXnU}apeL>{{FDa z7QW^%BKldnWcNpWuPGFaFGR&siq7;>ji1c~Q}_;T4)<)+hkHfg!tn*G#gF26^__~E z<%VS@5nHG;WkqT{Hf^U)f^2$)k)AvQ+{oeG{>YyWymQ*ttbxW+qS%w@=Lce%&EwZlp8O#f}Je|2%qZS-`mDN7`D?~8o zDtzd*)6VwloaxF@v6}9QDN^g7p5%PAf>q#H49vRcdZiQ zy=<$cruR3VTJNZ-r7zuRZd8u85hrTI>TI!CNhCj=%riDk(E}#(Q+t{1JJ2a_{oImU zduT!qn;~_vPTn4z!D}n>o$FYIB7Po4>wU%5xD-m-H;F)WjctjS0usDxqbQkT@pG&~ zhOn}#iVubY^yzid+)Fb$N*=0h+Dwk$-V{6=P`Dl_sEbrcjZ04mHjl|aaMrTGwW*G% zpY74L9SUERRJ5g{=Z(kJvox@dbWPNJq!tsa#}kz_EBFp3x-hnVk46pab8&?0+YTJ6 z`Isk$x_Y+BO2jJMCt^EQU5YfDJ#L=H_T;AVJ&#UNS&xMQvwmXGv^)C)aS3tN(r@+< z$@Kv@p%SuVemRo_3JNr1hv}RWd6P5Mx8uYlb$zX>K||;^c*dXwSYs{{;(?7GYp*tRdKD)3-A<#pxz zZdu!zY=N!hgmq6DmJL>(E*)pwMzA$b%+d_1_B5_nUYn%8Ba;sQy+B%Ot zz4|<5q1c%iaDYZhW=V7HUaaspk&|+?_WEVqC zznMZ~Y}n&Fkgha(*=^QTc#jy;J(KK2ck#21ecXYgYoPAPSuT)&_%R^ z&8q1twEfOV;xl9m5i=qcPi%nC(uno4_EvWYCPnXO3h?Ej|7DCMS~e%}POcfmw^366 z)mQ}W9T#<8#+vo?aFI@aI*+xvYepU7Xlk!!tl(O4*Yq$m&-Xqa(DzYH3Za0e2#eb1 z1f~+q(nRQ^?u=-J8DK<9W?Gd%#M<&MD;o|0gsPvfN{}=#1Kw|VqB1a0l%PNZ8Mm@w z@P1L&R5&(LR<5LT^_JP}(L+9EUB2q_kUQWmrw)1X(w+eajA~Vi{0MIxkt;U>c3u+3 zPHnKmnZ0j(?8KMW7E?t$ogqgdX*?9>k<)cJ?n}c!dIKkOZf25i+bflC{F4`+>Q89u zAomj44P|)w88A5%x8|>z>x2u<_mJ7G0b#?W(;MF_EKabeu)ZBWAJ=v_m95e=drEs? zW%eqsbvQ7j4RQ+{O`e|RzY0zE79*}p4Ilro%1vGJg(oMnJaGn2>&tMX(&vW6uq&a zTv5d#PC*YmICz`BX8H`GGNXG|M0i@3gL+XXCTtg{M6xlZDN$FjJqQtEmID6=iFi@> z8M!Q;&qg!EXPzR9cc2h_{GwC`@A!q9;XK#UBRp-Q$oJ?WDxrZ464i;taf9kT<~R(g z&i0C{&$|93magz9%E=eFyLAz+^OAygRGGZ_M0RT#TlaBB<^sDuHzRcCKzf9XdW29s zaHi`}opBcHQ0;N1xlPjuJApg*LZwLO9Vh_TxCPnFucUnt>~0m~wF4?xA(ZJKbUS98 zNh@B!Vli=x1LvptN8l01i&KkRNy5hjki#L=ya=+LzvK5pD1!CwIf*;b1xrBLNzA8O zVF~fE4nSB%T(EAMPR)^H*eSKlY)BNV{2Qb-yZA(9Aj7Bq)~skVbrCi%1vetvW2EsU z&8O&UUW7LB#`BA_D0~@{*F$1P9CaNK{Dzt*>QZcuKlAEcsv*TWX<9}Qjgn%(;+`eM z(w(YiIG<>Zv`N&P@#a&Ph1kcLsGO!b`2mZq07daO5PFiu^X4j<3(dP-9)7K>_$sUAeVxPrW$oQ9pm73e61Rk7H<;g|V# z+A)+QV-?YozY>I-#0Bx&2fz4@Y3w2zWd;^J>hy@=Vj57r3kd3lAoR#{mzBR8JnG0C zL^$ebm<6*RGrtCZgr&uAjvUtbF^6@?E4;bvB&W(0qasLhUG2mzT!?fHBdoET^1zXS z+9F(#G=t*dAH;nfr*UXecNfYUK=DuFiRb?pv7GH-#1SJd(#SCO1IIA91IJD>jK*#l z5B`BPoH7jriVac|ps<30w741|l%WN(0~u4yuc1=th`0p6FRoH_X^4_32F%BP0~WH* zz`_npQN+dg3nd^+07=xi5^>Q41J!->`mOdX9JPjYI0VK z{}Xfi29u_CfkCl8O}_?QzVa6wp8+|?4_z1ndR!pS16%hGy@Cs-Plu3m!T3a^l8CFO zqixUq9Fv)6-ePG3QU;THOUx!2uD3Dpc~`aQgh$OBzl@AAGNdOHuJcGb!Y{qFSRyz?zs=8jU%r}`{ktZ^LxE14*9W}Aq29yVJ z@VN;tP(nSZ&Wi^3!v<-fyCF2-kbPo?kTb?3Aqoqm9!S7jGOSuSQ+HUblcWrZVT|xO zn7=id|FQ7``c)M* zzzAU#5gf1CF-L}?E#?PR(RP%b7%`^afUg5D7PLTj2k}iLnSR%M+FT-eNmbIR2p-A+ z%c0Amr{QqZG5~)Mc*2=-U^O&g9RN4=!T?2>C!n3AamJjnmPo1CcL5KG#;GLmQlHUn zB@E{vzqHmqqL#QomQa6-^(4V(H&QUw&)vW{K>(Y1RSB2~Dt5`ON3s*G$%p<-plmnK zZch^a9`DC_QWol&NV!Bax6JtwFw3501=5e`OrwyNke2)Nl@QiRZ4zNh6ZIXD$N8N! z%t-nSJ7>`dy}j@o6W(oLCNd*bWwivl66bbdfmz~>D+!5K(1my0HtflE>z|%a$!r*q zqT|__JxJMkIK}ZZ&^7P^H1x4(cXADFA(TNAiwf-+9kF(hPKIoQB$^oLw*muPgjnAK z9|yr?v8_xY>>|KDG=SxP*#M)S^coRsf*ou!iGui}0UjAE2vC=~^~|{q69kZ=TDl2V zII&d3SB}YaTW)M~9iI?+edp#hsecOmq^)el<~sx09>@_4ta6evbrb1$C48Bj$t0+m zjein5?LrepolMHOZn$Af{hS14cA!D3nWv^aULq#Rnw>NMb8%ZBCEt^Nt0#)NsB{yf zGJ6h+kIXc;AY6(p>lqxeDC^oB7uj%t&1k00I%Mcy1SdB2r zFB6)!dDzq<+0F>+6|Jy$)|wO-uJJAm`pK4T#?}bW$^(j%wE||DLo+-!NG&uCVXUXi z0F7BRTmYct)CC49!~nN14rCa^vaJIGkBeJ@SabIs+V}VF1V8*+=U6@&?jkJkH**`Y zXF;8aMCb>E&p5&jzPGMOwnZW!?8$Pd-9-tL=dc9^ji@hpxjbT)sQF=N)v4mP1z72f zF5<{Yurxm;WJ2{gut2gfPuXZTS}o!@VRN`Pc^c2odkvYkazWXLWa3E~m>Z;TRH0_K zF{23?xDtat#$xC=2xG9=tHH!`ro8$~JEuym-!)w65z zBw^%A<@uG1Rs_Uc0>8quA|H%1O`%Yu@WvTy_g1m!1e#QTOxWmE4Guxv!+3*VCaBF- zWo|4MkRfq|!XP|DRrr8U*#|aqtUffAcDvs5)f(sc4zu$Nz z!(Hy~I$*GcGC!(}p-=Fj@`pLvGF|y7nds26k(w}ow}$@Ql0it~Vw301g3A%7&hT`= zEBPdt@5C~$xm;f|s}Fd-*jzm_Te2DEOT+2EsvdZSiABqR&5vi8rrNYVfi2fg zvW^O#W}}qWVfeDbvfpopVB{O{rHJcgS=sg(>E5;nDibYEkg#Yv|9sB`9X^IrQpEdb zq|etF5>sMbi+{#%@k!V6=kLgEOO+Gau(;BcPo-v#LV^{>rDA!V>*CK<-;xm>){&iV z4Yq^S0QU#TL#X4! zeVR~JORB)iXO`SW&BsQ6)GU4{fd8iefKX``PpVM9?7WkRN@$>hB!ZMw5ip6b+`skAL&o$Q1ENi}M7uVDxO-tLncfgS`1 z;y@1oVM}Kv-El!XR`p*CoqPSdFd7I5(ia9&8?8?i?%6@TyVco&rVlA$?ytipkwSs% zgYiA^4ii`f0qY>Bb@zR0<@dC^p91`z1OQWj1eI`eJSw6P87ay;5CRiv;EXZIKtY5+ z^FF{FFD@E3LB$a8EYbF2LKnrb5vyRQeS1&Kn`oshpLov@)Q6Ce~t z!A?Jg`4+a(8g(_5%vKe3JoWTqH}~3A#jlw`Q;=kPiNP5Ye zKf)O#Ww8w@(TDhJHFp94!Ditf5g7t<7^^a)Lh4x{Uca?nw|A0^bRr*sw@ZV&2 z6^4J9{*c{?da3eP8Zi3&ezN!NHsvAuqCVsmHkbHc+kKWW0kg$x!2U_oB%FJu!opMG zU8*csiP2u+UDGVrzleCH<^lSDw>&QD4d9$0hc^3^cb6*NHC~~4fX=`?k%|to$cYF9 zza4yaq-SkubH3Bm!pWzkpg&HP?(SW?4lzCfuXg7>Bh$}O-@N5lAcf2fDt|lZ?Dh|A zDy&VEL<*3|SIF(ME7|x&aY|$>95mYS8O~cp*YdEQNo#@C@O6f+?rU|z1VSeGPH*PWiTNkQ1W-N~(%jA&*&%5rd9ieXJt`>1hpUWA6Xa8kOg?@usG zawS-PL}~M12=P3L(|AE5u^@UeNb(u}IeceZQb*;vg43|RdH+?xA(`Yi3|Lnrn=e5> zb&J{gC3S)xYkt!1c_YmAr6PZZf?yLm`JR?tw%6a4ETqA#ptbcBOZ@U6jziF#`dHeg zOKF)K)e3KX2aOaI*+28Ky<410Wxd|-FfLxuJP(725jcF95#yn`PJQ?yaC)Kw8fxUX zo^1Fx@v6~PiGmQCvkjXYzzu08PYrP62b@hu_wdGTGUsQrM1WJX^Ay!W^}uC*CqeppzI0tVk-NURyD-ohZ3a zq&M#<82l(O9K5NMt%%y1P7dSD%FB2Lq14hP_*xuw08h}z9A=I~H$kko`J6K-pRC5f zr^v8{NjWL^soReg+2qu z$I)a?eEY}YE5mzO6@tN~7^@{I-p}8BaH<3Mlwur}NVI;#suYBROhTOpd_Mcj)cwJ= zN;mUkS}#V2nleF(5pEeQ^oVk@ox?-S3P-*8Iq{qUsh>?OoPJd3zn~Ru{DPP$jRxYa z-jI@wN#N~SmD=W!t1zH19G?q7`dZJ-J(h=V4H&c*iH+xFQ|QZA`eY$ZR~*YCTwMs_ zRbuoqSK^Il(g7}%(8*-V2`a+;3psXy#>Zw!9O@sHvxSm_8H@~)D%pkzh8VKQ(hq-* zz4kb&a!xNhnsKgQRX)FdO9Io6>u^#*;1;a-9-)Fy);=WrPVB=LKASsTDnNFT7i}IPKBzGH-ZtIGHnDg!dS6rlE zUaq1=Erv%?;w%dcx+K@Vp*^sLI?n>aieUmw9E>vq_W^g8B^XC6G79KMF^I}7SIUr@ zUCAD!gc8O1eMt|Zo3?DX44HP;y1|@r@;D9RLNU^t?zpT|hFV;IpD?xQRlMi8#2ReY ze#)EKLt+b2yV%o!*#~WRYFDk(cf+uGEr}10%w>x;i`|bsod~LB%m`Ig!qlfJ@!g!u ze48F#AG)>r-*$AYgwpgVoat@UJJGgH?Eo%VgTR)hTXXSA*dBQRZYRnLydx|>5(u}a zsS#HD#p5~TBJOr1y?%^xIy$sC!FWwA5J3Q1CwK+XQ3%xXL{;ZSVORyo9>f>mo#-PO zz)mLUA(Nw}iKFP$8mm6cys#HL2-qZtyE`IO#RhSM$~~2Dwj>m4u=m1F+edvyYL!7y z6A7d^E>okK-Pelvcbm<&z-&vF<7!fNuD|6t(11v{f;HqWAdE^T;TSk8tW2-b!Oh3~ zd2y2jRbw<@3G?Sokz$pe=VleQD_`R<&`x<33L$|4ZM2uP2TeFgPJ&t>`nxuk{aj@N zVF|ofy{Bu0Jw{z22C-lfXy^lN5#Z3Sap;LapWtIH1ROGfkg=c!5bc#N{+=AY+!=U| z&wM$~SK@46QPCA-<(hk6`i(AEaWg;w<*GgX&+=Y!@ISXgc~y{)F~^KBF53EG0v77d z1vO>4()!9ENPs8gy-^GB@-xhhNh*K{SubPzrbvs5pjRaBtu-$6lDLLb+Flzl`y^(B2LtkbJzkUog*?h)0af_fD2`2t8posH$s$m++ajBAvXfat#? zY&+{K*80HIAKDzUIeJ|q0XL}o9S|}&lVQ%od^z4%8o*nU=@o`<=V(D&c#kGKa%}-^F_~e*PmHwbn zm`-@73Xja8_!*!bRf_xCpbC)jtHvU#K_uF%R}iEc@}e8jAMc0nBw(6n5q?UZqEfzk zwIiJhR0AR0xmJ-YGWVe2fMG!s7=K3>=)?N^%>eM11&}!4M*$UWErNB!;Bt3!>%R0L z7$A%i2H=s#uwm_*cXugK;O2mGYZgld-KZ(`>wKijaobPPkE z(c#SzCh&Ch0ey_ALF?}y8)-Y`#NqP;5AzP{zns<`)R%3;H?a^8b8<@}qzQA+WBx{B zE$BZU5D8%KX%B}4nE!vge+2!9^vNxZ)w5nNApB+xL= z_%p!S^{)pc`-DX~M^!qa%K*bAA&qDdg4H+({4l!U&Xb^$bcUZ7#b{Be`2I=F0SBOI6*+|6KqCw-3i1n7&k!^v=s|uMPB#7{-w*|7oi0Ns4l`n z2vEsHHu!FF3hQ|ThxH-5`$5@Tq^79{Il&o1VD5oe z=YcH`$e)MfOYalsVl}KF6Vr$V(OQkegWqL~;rS4$iv=-TIU;~-vb!E|?*t+mjZ>hv zqYq_w5pnYeA-)(l1{2qb1@T$6g!?HYu}5+bi-ig7a3GOwWp|Uhz75LTm>;;6Iy-Ms zY;=AbgyHgC&>L+F_+MZMPXC(_!$sNf{Q4EsM<`dqO$@~0Ej-D@GUquk;s%TFa3GT* zn$TXk5e8!f#c4ZCDDc*F#6zjfe*s$~pCgo!9h$bu(Z$2WJbO=#v0k#vlKu=v+*|L` zg(=DI!si|cMARFPWV8B?;Lu?$kZRra^&#Ty(c9!mWF&`i3Jl_5u^>S!MM2R~Z^s9Fs1$hOK@wJR$lwP#_%Y+|A@)U7&wza|D@PPCXHFO04^v!0Tk$Z3ohlSE z-kfea*V*8iYNuZ4Cr0{rK^M&5NR0IHsDBaxWfLeT`|o^}QrsMT!7}mJsN`_SrQesL9+5LWJZ$e~_+dJVMS33D=;n1! z`!ViCsJ=M+zg{lK2g{W3tyc{$#SiA|cNpM?<9xl&_aG!7(O^ZZ?hw}7j73yE)!x6b15&(d$=6~%mMMez#hx3;*BxDBE@s1} zj5FM#sdEIOS;Tdr^SVj}ycUIcMb&eFs1b0aiC&t7;l$C4ZW|3*?j}$Mpl82ppMzU)ZABDo*!TP`8 z_;xyJxe;duH9kEb_Wk)FIFi{`^HK0|;Ts3p8R***a8mjyvz5Ia*gPF<9)^lQ#WaL zoN(SwV|-ildD9Kd2iOzahtpM0Y0oeDHde@w0h<$Ei>!y&@#%rbH7BsANDts)R?+v~ zE%*lqjk%&)+$UgrWvf28_P5nI)FQVQ{)9^Y*In$8VIY18klsNJ#xoLqOPC?CK{DqP zjX1CZJGnraoNfkcqFJQ~3urDYIN>IMdFo1xx0fg$Te?o(pulqYo03$3qNaq#tmqwFRZKSF@uVnn+R+=ku?&OhXvZDsh z2MI1I8CN8l^yqv8!rvbn_5?(PDg53%hpLTa{?N2+EYfbX33v=PQQv>a|Dz@z5F+G! zFaXrW-UI(#rrtLES3Q)_8!GEJ{cnKwzsdtm|8qfX0uBi$p&5LGUWw4myIWULz`M_GqEq@1Y0zoNb7$+5-A8IH<(96Hx$oopAT@w+sMs|E#~O zVE7OHZ^HL~`2RBg&jpcQwaSkQIWN&}8$h$0kqTVopN!i#VD=LPX>jxCzZwUSwru<( z>kw#G`^$RAf0vO7944H^0-G4E)Y||&;3YlkJqTpL*Tnw=Py46xKlqe?G2P(}|IPHj z7S^)iBQuKREKIX40nOG&DwufRcGib^Tn7fBklFoXT#@=qbVqrY!3mHjoKQluBRbq) zzXuo6*qdZvlf#E%8>k+*5FM2s1Tx?|;!a-u#nb+&{0|LeJo z@&zsXEhaufMVxjBQgcyXDld$0XWkKK0oZJbV+(Ay#$kaWeI+i+r$|?9`C= zM^%KFQGc`C1-eeg?HT)@;8{dlG!~*s2^JND(-z(kC#yQAq6#Oud?&v}UnYn+;)5Fm z`RM^2!w5Bp3g;tUN>_v6cT{|3#1VU_{;WnYl)ym_5VVg{QB7=@UUcRF_c1r=0xjuY)eduiwbQ?FbyJEMP0~AM6YT@UUA1BdUQGDTsu87NIN@MNy;g>xh`;s(j$L( z>B6eYmT^{!|FHj1>*v_l&goJ35%bXYvGjh{%;@2Z%o#=gU(&reoBr6F^aEM`DqSb< zJ#iua*pMudhjKcX-!R*_kNGUc)>*7F@>6Pml+=aKv%T@}6m*ORy-}VKST7rF{d~hx zy|{;17O1&}NLvHMH?||6bmrVR*K@z_i(Q^QA`$X4>$j}X!>iA^C^t9>x4npA?Wo@5 z`*2-g4mT(2ZUI69&iyLDt7PmIb5s7Uk+k+3bL9NSCw0?Tv`lzG5Tzzh+K?ngc*RC!1}Y}VryE&3XJ=MmKtcKYe{;FlT7uW`RQ zi9b;+>W}!rAO9}pPYLCeKj$GpBKa~yM(LeKHaVRZ=%2W;Prpm=P38$hO$}t(V#9p6 z$#}BKAnWd{0DF8C?Q>F<$a$r4N@xRt(6>PPeJFe^{9#-#r9%pZjwqxvuPZf_^oe|;8!oP>7lhq4PEtd~9>>3^f@bQ}BX`l$t8 zQsfWKof$R#c1!Zt9-+#Y2!&5tw^zIhrv%z!jiTeOy{;SWe}&#&sBXJb4Atf#TnNxC z@2&eZKdhROx(@b1_;YnF06u0xtljRE}D9cC$aLmL-#g@&|^zK6_D4^C1WiP$(Xqo`XyHwK94)5pZ#=WO-g zQKvnczUEJ_Ok5Bj!Q>Hva*pj9-Bk`iVa;Fs>35=RgJWwio}>ef{19kEEIh?cU`gN!IO1EuuFhN*;IN_HRbYr@$((BBWvXFz8VTy`)QrDq%Dta zo!kC_Ja{@vp&~6$qdCl zVH}#4r3q0!Sw%@^S3H%FPPK+KmZApduHxB`7e~eb;rGDdpo-2z>`%=d)6_ae$l#m z5;{iG-jzKXiru>dL7p(*=!r+dxzY?x}yt(vDmPhXrKsvIAZ z9WS|RpDl1xI~6NEpF$O?Xny)ix-FUk624ae`?|yj&s`t?HSD3rk>Tu3$xr;OCMuk^ zsw<$l(#Eue=BOJiHsS&-L z27yBQ?aR^B+aE>jex{nlZPoL(9dyXGgjN}t%^rx^&M!$12Maqq7OFO3p^G34`>_blJ`9 ze6aCc;A!7~xZ=Dxvt24M!_eUUc7bQ~P}}6#URQx&&>p2-Kg0lTsRG;KJg&iJc!6hU zpZL@r)4Qdw%>Zt>BHrPAyuk*t!1HsT_{JU6zooCk0JW11Y}|laC66?l4-c7ybuWc! z`4M0c*~zA0+`v>NPcWO04w*#svw`cVdu24=$!@pO!24aE}%++L+|9I<29{e*O@mdwmuN05-flbvU!f%yCX2wfzJ za_NJj`(lXOO|iq|=AFrt-6=im1%W~&mIhw@r|8<*3XkjdO5dWL@=Q4lb9Ci0?qr#C_6Av$ME4g}SA^LUB`DKw1wPj<4 zpXs+i(JrhI|LmjF zDufjK^mG2#JprxTvc1aGBpKm)Hp9UO{vA;Uoc%ZZFG}?h2Op;(B|6dZ9iM`c+dsG* z{rKsk_NKZ-NB(@lyxUxG_U4v&txK)afic~d_9%TnUxULT?k+3gWgTA)45^NetcoUAxUB~CMg3*P z_P!g|lYm8|G47u82DPrmWyKq3e|eOb*Qa_zN;ike_D_0a{VuaoqD4|+{KiBrImCqM zr=3-FM>S6M!juYT8*93J^sZljJN3T~{gL-;2N8jD{Za|Bd$iqquJ@5j|Z-COBodD7W{$9?Bv&-a60Q za;W&`?=Ha>hgULyB_Rd6ZZPqC*?pUKUUTkXcD>xRdN#hZaQ#}j{ai#xTjVS#GBRF0 z!oNn^{(MY28L~WT@U%7|rSJy%C_NWKQh#Ifx$Wzdj>X)Cqmy+#zX})sruAD^zJX^K zwS*$cpF5gpzuZ_h3_L%p7lDYdSYMB@|Jt>^I_SJ)9NJgE3P@EsO({3(yJ!>E8C$(b zP*T9)YM~>)vY7u_xOv;Wy6Z2$Hgi-z|1-elYQj%9X<-KSyV2B2$5!a!?)k5>aAumb z8BNZ)5RsqfmlNc--$U2Ex{rRYh9QeI@ACK^|ESbDPQmii4y$zFZ@;`%AZwSpR1T}e z_S5dcT;4ioId(8SpOLRJB6v$)zUSY+a@8$FF?K7(n9_gqdhe+A-L)6duZ!lg!)rTJGVx&rub&%ThxPwz9l12`xh^DJUi_X(R{Tr$$u{}&(WjHM6aSNz`R^_A ziOYM@hk|SEuilFAEc@#YTrU+V-F6w7CytVXzg=Wi%8vai7inEju^*_~trROGpZm4U zYW~E?uY#bX4MCNAxn^p{VXxEYEM3R`7aq0Q`c=bxVz&Phew=wcI@ad#&typ4O|8@~ zT>(A4_SkEOwD+=9@wLb3|A(`;0E(k){zij)Ah>UEOK_J3LU6Z0umHh>ySv-s?hYZi zJHaKmOK^8*VHduAp7;IVx^?fZy7hh4Q|CLZRMNC zX6;eGN{`vttv2pXfBG^)%7j)!R7HOkSY)~AT|qbj-wb|;be-KpuDyq2v6a(Bw*IVy z*+Ki7udaF-Vp~PJ9q>Q-=sxD%8m@w>rT%6=nE|@MVJL+@kKer?7eu|u0DvsOOAFBZ zStawn?8^{8#^&By^vS`tlpbl|=NnR+vJ`&sy10059FIA~wqDZpy> zVW`6F^QvP|;fA2rOUuU>MS4{M56^-3{LM9cVe~rVt$+CV?lbD@vSfK<$4e=}hGWop zhKr$zNs;u!Z*9c~u~S^7VY0t@zX=}re7 zMhtrgWQOyO;RT47<*Q1LvUnnhyO2Ws4RGsT=+x(ypGn-sPGpW$fbH>YvM3Lw#wy@c z_nJQA$_h(_Bt%Fv`46DUPQ}Y~M-WgY)HQpiX&l{tS>H62@O`T|U9a9lRC5Y^z*cb+ z)p2E@CjU#%IJe{XuuaJUfd2J$(k3ET`K1-%E}_SXDBSOC0di+Np@((mb^iONSQ3q$ z>$KJ7aYr!+?LDRIa5i&4%SRctk<~knDrM*s_c__Y-IY>&jooQ`TIxE|vVOuk-B)4T zE{yti?sob5yB09()CD|rx<+~o^6wt&B;eOgyI42IZF|SPg<(5}wOha$YqbdJ(mDL9 zzG3^=4-j%wdru2SbbE(8H#_CUIV=15_4f3?lAe~a6AtVE~l1W&`QI zz2n8gup5KiEntV$PMDPc9DYcD(>?s45XRfDA;q?LfGsvVFtXeNZdmPrr2H4~2l|^G zW0@f2iS`~BqL%aPg~EUnR{M3Ns`idMi%rky9_f^X_714UW*5eqTfj4`ohWI`1^k2l zX4lv%2>JXV&yO*6hvOI|O%_E^Alkok^L-bv1Mc~L2{^d|t6od{+%P_sJLE7Q1WkA_ zXO}K`RAgg&9^|{9=6j6iR-7*%hF#Z&-PeXa)t0{0hQZf`A=ibW*Ok7l3&X9u*Qf_2 zmu~}Ne9LIPj%mFZ>AWoHyvpdjj_JG@ z>AfuIy~^lM0A{Ju)7WPw*mow%P!nt%qAQ|jMVpdWMzd?EJJ=R&mlx$xjux8uR&>y6)~RJUuG_l4gpGxMz9R_JL$ z*7uo};7%6tu~<*b{rP^nzgd8l;k>`Xug3z7x#y93ah4Km-32~=FQHr<8t2iY6mE{u z5EiQp+YC2B8xz$G@1563gJ*`Powb{?wWw_fAP9JO;PN5`Yx zZJWHz2m75)cAwWU+)6G^FS6QQ|5|u5=GJ+SUUR6INJ2$(8_H3JJKbLvZ;U3xL;&~R zOE*Rr6>`8caD$oRWY{c6b~#C#E2TMg&un{!aJYHTFInF=G&+%puXbck#{gFk~=}sz7q8Ftf^emb3fr^1u64$#*I8M`&ox;URSx) zeUjW(@OQL+h2T4X?~<1|aWH%3ZT(>!(^N21S^DP41o(D; zw$)oOMY;6)NClWZQc44MrUsx~9|;2~hDv82or!ZNwiE9P5+C`)m2Hn$s~hyzJmQPU zag!zL9Gz+9wjCApY1zUI<<<~`dSthqRn${iI>$oR_oyz=rJY+Rl*I|6X_Zm9v%hIl zFvyk}()!Ya`>eU7AL)YVsTx8d7*fU-2@;9#b-v@s5H_Nv{HaDaavfl8pD;N_|Kcji z)XwX?ucaxiW{E4GsGYGpLz}j6lr`aRqJl29ZHcRFBlKslIet}E8jR`vdoD;&=uB zNTsZWqS}vVtMQF8F`E}YIq;5^#VO1UM2=KRwMh!!>N;Aj_L~%;>L2Fs)VA6VDLmUE zv3(32tigg?#QC2ldvPvVqKhTOTMHRng?OFSFd(jbkmR6ueYxn?_2WZp=!~+1UfVAA znBag1ltjY9b^BAbD|I7RqN|YK5A&2WHvQ~?Sj>4d>m5wlto}s)S*sziOctpBzfEPw zm3K+*XhSJn=Tk236j~y8!Fz zI8MCHawku3;uToBCZ+Ge!z69O=GDoFWw#yGPR9xL`nIkDOIk|0*X3-}jJq=51WZTV zSgTthdn~0at;4gW*~+@Rem7eR;l4^5j(*iyCA(KjL@vOyuMG!%IoMvn(@E>&+*?ZDby3lcQ}rL~*iRCcB65CU33Cy| zegN~BXj$a}S@&4?PtG&A7Zokoc0#H3GWSI}c2LJn{J!HgsLqr7(vn^zY2EY2F)rh1gyRwNba!|&1iSWP;UpkvvN~5X*C|>Ca$Q(- zvC3}vyu<$9XfJnT77Ds(4H{S!368is01SoSTqM>$cM{jyx+8~QP`~hVd(W%)7IcYke#6LMN(GF?rw2zkuELWL(IviRO^nl3-}9_?IHh4 z->-mU1@bR|ivykR69^ULNikdWv!3Pw_SM44pl_%9#pfSh&r?^Kqu1wjQ>6PPTVQHW zh~h_X1D1?&!%CGa@&1Q)kb&AU3bVdQmFjsnJ9k&q{oK89IaH`O?@J%bOi8xJBSb+6g4@sg@sBqiq??w z;(+>ER%P3NmtJHP>GJ%QX5NzBEU&hQeHkFb15J1WWX|-xT;_c9yo)n8_*-odwpq~J z)onX1YLg=V-gCJ*8|$^~Q2L?9$464qDwW+=(mxWaSgrFz)U$jeeQux4MN75f9QfJ^ zjAyv}X=}NWE$&-x*KKUt;VU!t7@2(aMD=lp<;&nnSVMHRHc2&LhO_Jc_n^|)V+u@D z;Ny-iO!J^2+U_e!^~Efwn)=J`27~ENV%?Rq&uoN&o`~KoxoRg(eLXnuhhG4rJ^jaszUP@*wcrgHesum zvW1<+JF%{Bzc%bUk-8qF`vb{$9}kZ9Ax=HA;fv0_{YN-SE7H7 zXag_qnsuZ>eZ#fAfx-X>qDP#zA&eMlgdLbA4Zp)$ElRp{0spLT*!}-9rvGCsUs-In z{6CE8mj4>d9z^^X*ZYMqJKFv)(!_u5YT;Lk;GS3iusrw2?o@;;?7tQ#b`f{4c7HDy zYuH|r6`ybTgk|Wkl}2Ru(^>~aWP`WAUTx3@6%XmYz3?+{e}Q08kDG*l7S96gm}s0a z_+%}}OUb{iE57WJDJ@v2?N3xT8!u)d>WO~LR)!z-vk&sYRRN+gf;V6s2$j+A<~?Lg zmf>knnM>(fd807XyKD*6LXYz;3j^+0f`>|7d=DsQ9`|`$0<()#Vyy!ZI8X4DV(44c z)YHf4ib0M@Ol+?gX-@_Ia2&jgcigXKLL@^XIq08nXwiV(pGpnz-t593@we#Stq+6( zWZ5Md!~}fV_4u)@K^bH_DQ~da@BiH{c%myuqQ#7g53;_`+Ozw zcHkt8Hu7gnro`3I9pr7qcDoHY<@1%U4?NrzgUQ{-dvonqD%o9Z7N^5 zbzwk?biHRj0*?rDoui(yUj^F+*!G9Rz4%@(5?Sxt?#*B2Vx#Y5WG;m}wk2Yt0ptc( z4>)W}lfQ2QL`UOmDUWD!q_w=qVV4t?9+VJ;LJz(ysjEUkIl4y519z)1PQq(h!UxK( zA1cDV#AOsJ;+%Gd9oj#ezz;+9rhfG(pW%$FDPvKE%q{GX`~TULKi(J4k-Wg93*x*ac$kKoxp@qH@&@hs{XMrIGJ5Rf!CJ(Q*@VwQ|z^)Pf#DEz>gdehD#@NB!-gm)?Z=izjo0)$ZSa7+D8M0`XoAX9u>u$A_ zp-06XdVhq*{_hFHDvv+*Yd~PS7lhwaVRLwXT|iORL)|V!^sSRQTtCHwrR>o?JFDpp z?>@I{9B4?fcNpWn-C>N~5r8ilym$zx{x*{DFu%nhz};1TpnM4U*4YA>7WKWFf8>+c zVjxI4pE=JkLIhYkMOcK_e&5(loHlF3Px35QO!94&;ulK(T13z%7=qjJAO)0ew2Zn z(L)Kdm@vVoo@;mJqF2_j9qA*U(;!Dr-t+t%;H7X5`Y0voF~a1|Y1fI9UhZp*Fm|&! zdh*7eN7g6xT)&8eE6J4eoOICS$+Wz3&JfYpHhK3X@)!5_lu(MR$2CCi&`p5!bS-7> z1GE(YLg^ZJd;n0!oP!dcZxkP-r(yHz7sggmvUPwYNV;%H|N6*YsMDrYy`lf8>tg!+ ziqTtmRfMt_B;7t#cnP+C@NVz|u808WCe7Oq3J<{+58g?_xo0_`g2fwCQ8|Oj)J{st zYU#qo!reRTTTlE>$`}Js!bAP^v+Aucx2LGw-$|(p>FMX`Lnxz%C|T!4fimAc6ns?& z8(g{h9vbHEeGZxgN%_$+A9Q~+^@1?=4Rr@OUV-XFe913gi3D|D_sL$En}#k@Sftc* z=v|)VH_~`tp>Kz0vnDpvm3+sR_H~B}O%b<$;VtY1jxum+lw8hhl{0lQ&HH#aroG;$ z@V^|Ubl2Ybonetkj2g}y4PEL5J{`&5%|gLGZA0Bn9Ivrtr?L<3#nn))LrS*a7u^aE z?zIxnD(o8h3X;?WG8)nBuVX1zNTfHg*sxaz{>h_D+mO7&#J5rylL`KZiSPcGNqzl? zxqAGU3HuN8@DIc5NWJxB3PZ}SA-h=-gG&+L%=4?jiz_l`5CBoH+MQ0^bi187@jE6n zO5Z~nKr|fn?^cH|;HSgcq!!nY%&rVDNAJMTI&laq$}SSxE>WVs*c*@DHzQ;(jC%tZ z3t_e|QC#Sdi?E)pDet`7L+oJ)3UbU#T@`4faH7LEWqrqdE`4#b&I=MPcoQh(^yTQg z6)5-0%s^EB?27B}_=$PBi0@#g2lPYZ(4;$8b#C$fBZ=MqUHu#-z=KOP2>?1UyYnl? z_k(s=%fpCCkdk zFR#BV39Q-8)mo4zUmLRsJi^5+i~RIQBbOfJ4?(Gy={4J2MgtpeecvRF9(j!hT5vUl zZ`5fmsn&B%yNh1GkvTPf{i5U!9()a?FLly;d%||#lWwwIEr3#xZAtQE85n(4UrBI- z?WA^kp?Ua?j;`Cg&vG!AGMh_ufQ0on`W#j*Ed6c#or@N9mf!_8cgRr)g?%@e8^Lnj z4Qu}TscuK+;u8BJoFUjz`GFO_UGWuF^LyQ2jOeRY-vGvR;zM3PGO<*ZK<7u8eP2W6 zL~)0{sV5pMLDHeO8=YIN{Idi`Y37IQHTQ&w)iqJb$(AMyHrh7HHKJ_v#-17OMyO)<*HBMf)DG`Do9@_@ND~RsQ!UW*(KYwKiX4<3& zhE7hT{T^Spk=x-fb@Y$E?@jrAiI#m{8k)ozeng@JGM7GL>WG-tViCXD|Hq94N&4pPo{(S$gDb`S6~xO(Zf_dgRL5SvWHGK*%?Vd{Szt@X#Hf z0+g_`3`3E^bJ(-T)6vU?d*sDFHnuZPPk(96po|>B@CfV`{IjG!r<#4 zDVWg{7?}e~IH;e#QN8uX7s)L={;w1UM2q#)C#wI#5?!@945SY4$fy66{sT|=odCKE z^TJoC=M~=O{|;@uwYNyUAK4jwb@l-&#m={ioJ1#^%P`EA?*Zkr6YE?mevec;E_IGx zT&r0TEh?us4q)ft?LaZTa$rkc%oA#c-Wtk}Y{l5z{A#oo$bjn#1S} zcte5tVUO;XHY)d(0CacK+^qiFHF9pE=k;rE2|BN4O?jXqfeGldZ?afqhQ~QB*b&@@ zILnv3KrAj)O)4$X{IPaavSQ+C>5 z^-(?85BzgXw=2mWGhzMP^0#H+KZMpBY#8W^wiRp-k3^G}R&0CyX>nRWpC3l-x576{ zL<&R-rA}@3q4uGM{~_e+hzXM4FoNYZ|1UXD;QvGauH*kFZ+(FitRN+4WCPhtIJ-r zk43B~_Xr^454Pe>SOOop1d-zT(6;#2um-yVk~LB^QYBJ4#ur|AsX(Tm$t4Vl6vnXw zRK(U8T6S5ad<|PHY=Qqg*^Gb4b8oH%tU3Rm8};wa5@ZO*t57n8NBigHh$s(u&{9m< zuo%j(z|=G~bSY=XRV8z-DVn1#q37TkDVJ4IOt8Y$%!G+x{&ilCeMFi79@$D+6PxMY z1f9gCjQ$1MX8K+p*(zCQo7vt3gT%*-{zckudN^L$8rfhQ8lME?RQ%@v%MVc+pH6X& zQ*qjZ`PIAj7K@_{F--~9J%cbax&EV|{~xi}lz`t#|5!byOJsg5+gxvmV4X@B!Ew^3Y~{z9 z>0rSmmC|>|QlyK&z0|dEMhLB#6Y(^#ET*&h# z;O5Z$1)r5mUyfQtDQ+KTNc;rXm4}{=#6&6fN+lr9p1BieUPr^g?8(wZurqzG(#|Z0 zx1hhW^aT{d8vH8a@3fW^`i*vx*UfQ{@YnFbMeD=_l1-JlvJ;zQV^-AE8cH%*mLo-a zRR3BD{^+@nnav+0+blxYjNMj)KLp=dy`1Ks_XeF9bDH#6b#}OSsQGep==F9Nmi=nN zwkuCtnRvg}I8@m=p^>N8i2O>LH6EY$$XYNqd0AfTB|rGYt;E8iM8=_n!J+ibu5@y# z7m_=fg{QHiV)-eX*Qf}Zw6CtC+&U>R<1V+#q*ywO9NmyPWBtVDs#+4JOz2|POjT$= zNiT)0kZT<&#EC^{$;#!0e#E9+Klyi5C%xQW#ys6Jx)-Xf40y(NMj@oIGeOX2 z|CdIRToZT^RI0}jeIu$Kf!zf$H(#h)WAago5z-fb<}SOIkDEJz*a0oUWlZX3z*F$0 zhwlZLE*BP8lVC}L9XJKou!+R|Ux75R{aBbTaP4|aYphfkxPHAg_0}#KQmrcS$|1(O zH2ZG%2-(UajqT^-jpz|y#%_C>U2C{K@rX|n=qu1n)ni?$IiW)qyittOC>?z|{vi38xebnIL3abRJzMGAg$Wsj!xGj6jzbdVTj?#OvqZG+PfHwJ= z*XD}`Q7nk%rZR#b>Emn57$4u;gP_nCZYZOQ&OwSY;rIH+!2aaeavXIZ!uyj+W)Xf#+=QRPI@Ne;P zEh8w_;1F%BnTPuj@uSZMfly^o-Y4i7IUV%A7aV^Qi#fN?d&7Aae6`4~DpV`+P3-S0 z&;48Fium#8-lQ9|^33}~=RZt~++nvR z;1G~{VJs}kriRJ7A)c~IM11U`!oo;Ye<$$3n>*6vz%c00ph}!zzc0mET~UK$KNW5o z@M2Q(r70O36fRzt~XB-Cx4_#*WKXDYti0={w_~=vnDptvtW9tg3cPEtEfz}&o6Q5DC1DEHF z6)S&|H2Eq*b`7k(_lfE(@D&fL7bou9abn1Z5|&zhOuL-oS8q*ih4ujnTc4BN^$_YC z2-HxmS7p>G{KFaQ1UtpT;rESjOy&&eeLmnpC!&#;ktIqZKHw%g!@sc}uk;QYA|PSH zOzxf)nrX|rO^Y|pePAzCvZc55LDe~@NeRr8vY z!RnZAK9VI*=Cu`0Eo0!BKQ6N)_x?1Rd0xJk-kJFQAwAKkE<3hp*&UZh)Wg>Pu|@;W zV3nU=9H{@!i!@fS$Xv!5LEfz7XBkxaV6E(rA=%7Itpt8F@oLBnf52^Rqla|Yy9{J; z9cq6U2i!nZ)>yI4q-|7KY#_F5>AssD;{m}kg{t&K&?3$L{pO1f+t}iD)H#$o4jJ_a z{?r7MQHzak+6Ep``}JOk22B8v+LfIwpz05rg(LKgaJ)r0F1$a-W%~=&xy_}}8~3s% zI%HdR%;q|8q`SaL?#X402pM#9xWwM?%g$O1Ln)1WKkeximdLB+3UX!OPv?>Uy34QL z#x{zQj!4428UnF{*5Fh0y*eCEtZ=PY9o)7LVHG>Ob5^`hdOdJBv*y?e`-|ksa@u-( zmlpb!9Oz~EOHs+{Z*ZLBFMbm+d#e;SYr0{kW8HYgDy{Y5tS`QWOQwH3dlMl|PseZU zRwG*t*ZoIkPNyEC{N%LLN=<=cb4`yIL1?pWWk4I@8{=*jEgJZ??@44S;Y_#EV(@Di zm(wQ~n(l}!B$=yfqt}&OmrDnioh|zFaWzDApiKS>t!hvd({Oi)r2iCOoG1&wEQ<|i z8W`;$1o#-?^)8x#@%?&yG~*~99@%@PU|(lP?M*`ZPDh7GdqrcZ^zFqajx0lJ1*i-$ zzKmwDg!qzfW0H8bXpi|OB$kLdE^4Pc|bTMe(pbq)WW zY4pyxBu2SMQUc<;mM%4n{Iv;g$ZXX_eVN_{+j$yU#Kc@MSYfNg7%Ot45Ki6XhvW-J z1RZf3yG^~beuOMTVL*;%B`mEi!&>bx!5*m0kvsa9rf~XEEEB0O#jaGe?WK}G^h9AW zeqgN^T}Jg z`cdJq_C1Df+^{e&<`LD`Mh4a-J%iysQ?JZAzyy0Ib?@|!YZ`6Jer|4>zTGC|{S8)j z7|Nu8a8D}|uJ0)MqrJcDseqY(VP0Udsu|8&R)81Jb*D>_gwcqOpe>is&s{Pr-C)Eb z1OZ(eQU;@|vt4D|79v}Q{;W9ZDEws`7n=k|5*Jo_w(;2ZyMatVe~+M(h2vB2K zOUA)tY4P>stgDB#>6~}^x~0QLR)A{<^l{#B1_AZgrZY>9^+gswiU3YkaYTZ zzu2S<~{s(_s2p-FEQT#c&~JlAQcsKP>M#11Mgo6X55-e5%(@DFG+MyzIM z`52nXhqUmXo>eFy2WW)T^(HHSXr+Gp;g-o?fKPQq=Tu#xt@C74Vwgo`tytIZw#b(r zF|Hk}%dJQZ{V_68Xe5GA$&q@V=M0arzCT7h1_vIOL?73KWQpe-%_*7%q#zkIuBawu zsv6f*gnhq@^`L@fJQFrV;}Y_DGnNFC)q0J}@RDobAyUG44KZPfjuZA7lH|6*zLn|# zpYz_LEK{Rlc;gA0%me!X(v?|L<1jxT`L6gGfo6otciC(LSl(*>lWnRxX{$dK{!&2h z!u;`c7wSd!YVc`)uSTDj^Jy!vDEIC`%lL{MT8M zVSW(4kD-5_@^7UQo$uFb{>XzD=)%U7#HP+v%gEoOv9e4=G{|hreFW=G&@QWCNJ<;nvv=u|o9EsemNS&4W3_#Ewf-bUwK21i^i}+VPnrm z$@*ZlLd8Vv;q`b+XybY5EP3g97K&B#-@agY){5t&dhEc>8y*;53L{Gy%Qb#!-5d@k zntn9-LVZZ((IJ5D2yOBSCDtl@@2aYvs3tQlRGKO%k!>(q5#!AbDHk7y)s5qdm6?sQ z_-1PtEq82D&qi1GhV6h;cQ$AGYYKdgV{+yp9-U^HOwV5lCUdk(v16_k1gY-?n9QItb`jx*`@@RA{Rf!F zasC_2?WOL_XU!@?I&en>c#zzWf`l=v&PQl$-X$QY zPSQ%s(uV!;#U?I{{3*gUY{c-fIanv?J=G=;X)}vWX)=An-$teiDdRP}ogR+* zZqi?Rm;5rOm-rD1G@)=JIMIM=u{&@N{<76Wa}w!pRoWoX!j)sW=mVbTM@Az!1*37Z z{q>ky6g~E)ens?;csbOPude)1E0rMrAP4lY8bOQ<9(#Pt05QM<0|CG+`Ggb zg5+cmcg1>1KC=KBWxe#NT&~;tk&r0fJ0;U+v}CZ2J)rmh6{i82pUJc1%D)lZ3vGefB)wI zdYZuB^aUaMs)TmPfc?jL}3kG^l5aCTBt(npzCV=CBp z-dJfo=BwCSloTRF6PKO6gn;JgQ;Z|=Jo5QdEMwI|dM$0X>jG1wiu~qEcxysHh(Z~f zsQ%kG@z$RUpx>|gVKsP}now$GoSTWgT_eXd)f5XQv)*|%{7@0lEHfyTN1^=0LWGnD zHy}h3J4``t>YBiMAh{Jej@6H29Dd1Rw4R8_XPQV<>!rGWQbLF0syhWt3C=SbR5U5Py$@ zt}f#tFH+9>S%)LERb{|CptNjB2eFlQMfC`)xCPVlG(5iSA2Qr6f1(nqny6O{ zJ#uhTKW(jtTD5HQ3JW-VAViYMSTF~PMDFZ%syc-7ZIKKd6HmZaaHrwutkRdb94uO0 z{VrJAo1+HJjxzZYVX+}}RL$0EpC13uZ$z+WMrBuJ;;lF%6_^vf7#d6cg88zF6O&j4 z6y7H+e?4yL5d!@#!k1*J<7!r&pz4E?dtU4Bn9Rm{A4sCRvRzd=&j%Zl(ZQRcv*Bb> z3z8aVBS>qsAQ^q%yeD?Fm2*_wmXdP@I6+gBRF;@##^0t`i8s8{P}Dg>SLZ}@`n!!6 z#akp`-7u8+3mFypi)Pn%FTvhPUk|G3dHjShv z`je)~N{0U)=Z~ZbPm?p+{ZD1ObnV9>3SW!M(4dIUn3AO-k;7EDIhIrdZLpF#>Vbcp z5~=dk`WDcNgF6FesgybEu@%ovaVRbS=z(m7St*q0i<1n}n@0F&K6`26@9`;_sDRn? z(~1!$^H|Lc36O_tTrqx(GEx0L3u}^CTjQkZ0linvMrU9!be6Hr=-2HX+&LeFz*bj( z@2dY%EJxr}`IiOhmi7z${Rhr~pwMmc$RB)f89f&om1G7%5n3UtuMA%=2Pbd1(M{uG{Q1Y zdV_*6COW47c+R?PjamL2qpE^sG||we`RgU4E{+0toJ6X{Sy`UxNhFAk>qDu|QzECN z*P(Z@ntRv9sOk74&L^b~TMtPj@(?Ff@ndQr+zzG*8s!E&ffDuK9LIsxc`i%h%A&C^ zSl*0(tb)W`Nqn2$-^PsU{U9zV@x;=w?+a@T7dDY?4R8#=3~uO~RvNXXv0zE?6sjVe z+T@(%vvbEOe6f|!M$wfl2!C|UiBv}3W;((LEfyhe*^pZV<%UqUudkbm3)2xtOAeMK z;U`xNG5)ei+Xc!@2~LtyG7KBj&q%%#P??N5>hB}W31q_a18U1Wp^;a7#=)=(*jBMw zQOWPX&`B#p`Bjl@5NOdB1+(j8Fvgz)xMP}^FTb;irT z5z$F)F7V9%E%H)hWg>H9f!7yX^j&d-TZ~B!W$C{Mf{v%ZYFH?5N6u0#?DN_vCg!yL zWy3OQ0oE-(Q@b`f4p~|ieGYKLpK~wu>D1Ya8v2`Kej+y!Xnie2y(tjr2?aJX4!+`F z>*F9;;q-}aDI=Un=06w}@0#kWc8Sfo9)5--?t9&m~Pt@fJ0aZNP3hwQ{-y$&SR`mIT)!f zO_szHvw(Stc?5!&3yGCzXkldQ#Vv%{Tp=+TCUgq6;R|m$;U755d1&q5Rx; zm`J9#%Sz7m^Ad4l#BimwA!=Fe8)3f5F13T7kfW1EW0IwfH+dJ7pv7h7_IEgIFK*<= zfzL!HdlzQJyt+`u{c z+sfA@Fmsfgkhp2+lFTz8awY(lcYpmI6jNKeq#>>3w0bQ;|C6+LJ072s(FAS|3S(JO z^Lso5!a@Q`Gi?*F*v&2Lr_o9pDw7ae8NVplB+|xkpIa=#+xK>vrMj>MN7B!JIv+A*po4pdwx3=>SIcZ-9jN^YqCW#rSb2igLWH$)A{S{1) zjd>pTWTSYJfeXo@wI3iQr6>;3=>;3`iSH3tK zY}9V9r`)=6qVCYYoX~q*5!~Sx>GV`NLTL&}MuzP^2B#Hrx`{iZLcRLm>k@cvhHN}i zV)VZizbx})zbw2_r@9;Ll$YQA0uZQ3&{ zsc|JP_@KYUD&jL!13M@gLRJK;T$~b0Lv0+p)1qMoQ@irGg)H9^-`p%ag=Nk6`mEbx3y!SzG)W@fM@Y>D+t8YV$xQY}5aya8{v$GQrZq-$P zX!OU9@YsI3f`63`Vh=H}tzqacJ^DiAY(jr_<=nhzq~JKV+o%ZiQCc+N&Bh%xS;Lli4jK(I=!E-X;dLolyjEhY{HUAU|FQte#siW$U^BFX$Plt=zOinW?X z+FQkxe6vx+y~kJLwK!$8DkD8OI+`s}XOdS)Mw=+}tf=9fr!B@cNUmT}l#NUG zu4F|YwfF^-unO^|dhFgJ*2j=&WEX|h*P?ntbswvI30Z$_k3V7H13hk1KR1k#vDW*=6fsZd))1&v~CXzUe z2N`oR`O~(TIPlt%xa(q^)k)Mh8k+n1SZt%nJ*Jt%Zy6|qliu1m>QiEBgPsYdvQi3|O#)R~K63@vJ>Xz_z< zYwT=NO+^%{dP=#NKdbRnBZu5d>49v4ovkAcTXdT<_oTCo&bWUNl9<;!8wV8R2gSIRw3t3`L-*YVTlpKY%+3!()Y?-^AWjL;gQF{|xL9A3DY0WATU=+awfaw=v;{L{<0qW! zKA2Pst0!tntCh)Z>))!mE1MfSk=ISD_rdF+JW(8F8vjX*o@X}Z5`hp4E2G@Wn%4x*L`c@*61ZGPI$KZgY}G~J z##v^8MoGe!BPMRI3%o-3pt|K537y7-pc4P~rg3!W$9VuswnzC$^hf^G_`kCh0f&c@ zPnPiYy3K#D2krC7i;lOds`2e**)Z%HsD2s0QJ*n2c{@r#T!P zOj*N3xNiIA$aMJZlWET1B9|JL&l$&cbGaniPjSX!WoEdux>T{VLc)g*EqFrpX!OjX zkg^wmN=K%ng!Q8x8{IEbb0nP@5z8=l z_RwmP__)+4Q3g|D5q;)Tj&OzFVG<&<1v+jUiD@C3<1X(RPB+1>fjb?6)p8;^3Bx=| zgfURB!~p}5vgngnMR1FleAb*Uw*$BS-+*_WQy6N?3ypmULjq~m3c8`R%^d_Yx*pJ| zz!A#v_dNWK@qbNs3J35SUY`y|06mfyX?OQAi5k;9C@IVUR}a+_>B-tKfVqns4e@e2NC#tDIY zUFlNt(U#L7R|_BYVyg=GRBNmiUwr@e1Pz3sKujcG(i*t*h5nKIm=x}0 zb|6V5kxh||*mi6ceoO1!inCzG+-m@%0(r#9Ej?Q8o=VQxC57bxt%8P!{5$jwuz2;C zC}ai9HJ3a%(B>HapO0B+R0|Eyqj?AhwXF$17-K0|xzCf{(JA*cn{|EacM;8!`4#=0 z2q!JP)n$(&y7Znbxe_$Ww7sJwv< z%5X;hsPUn>`NUu8pFhV}QxJ90C4W!C8>j6MKuHz4zHT~7Q}a!c=nGHNx~XEUapq>G z7qr<9$y$JKFnWx+Kdn^DJD?lXaQAX6pdIeFzL7sqmI)51@RmZzso%Hd%tq3AkeT_E5 zBaB?P2`qZ^n_7dwZGXf$S&D#-Iuc8gpiU}=I=SLeHe}uTe*u9&e!m0`5|M-wHSu_; z!S9%K#-<9#P;TYCmXee+gt`Q`s6fLioe5Pd$+tD-+_-?XDA5hF7cYjE%l;4melq9> zsXWR~+01*D?{s4)cSpftS;S;8oO^*oWnpb3VW0%!@dPUDqqZjwRi~U#z~L%1rSpK+ zrVY<2O!7B!WS*KOX;9TLcmj$Hr`Xa3DR(sAH6F}Y7T2GM2-5^V<8tNlA3-#L?3Qvg zn^d3_E@cE6gAs~Ei~jY{QGg~ZyYryos7e`H=yMPi#iy_}1lq_frX`2l6U}{vFOcfw zFqdSKtnLtzsAqOZ`pTf~8sx9k8v5}~gm9t_^9c*ag)Y!pp3lyK0tb43^IT!f*>wh=CmN+tkX8?u5c z04TvlUQ)5jN3+v41busuZ>90N3vzLAR|mlf5^pCdwko@{8s740RE#XO!kHUHG!+Pz z(vdQz4qDenn^Dr*5vMTnrkWHC5Y`|`6g=i!MhJxJ=sreJ5@p1x05iHB16c*6pbfXR zji_*$@us>wNq;@v4{%72+#UbP4VsBjdyqkK@o}TZ70M8(SEXlu10{q((8kTvK7(}g z3+2#V>R`c^Jm!Ed{q4TJgD&J05VJT)@~%OG3-2em(l9k0{n#{=pmlLacd914BHshe z)XO?~*t2w7PR5PPHcGlhZ3T21INa^+aVIttjnpLtd3B2X_pnULh$%^OfD`P3q)re) z>8%6NLBbGWp4xhocQTYp(pl5(Kaw*E`7IvOX;Kr9N{V1j8ah) z2okUg6s+ulf40Y@j-P~crU*^(ElC_nvTRy(tQ6CUif`};lS)^pBenf%doT1S*FxJl z(>xHRCX%~{6^Y&;nKDe|Iij*%-b8-+^1Ra`58_~Z7CRy&rK#rT9}K@2-6-OxGbs__ znB?2Zs7qP=plu?m0z644xgD4Npv%Oh=;q5)QW1@MjDD9EH{xmvNMMPFf^(WQR!LWh zw5qWw5SW49A}XbbFeExr1{%;N(Xu$*@=Tz@WV-O7RMxc640@0x+Ctr#9Z&``v@@_g zN}{QZY7oocmWI}pe^oNfSh0)nHtF)`7+`!dU8V(TfG{9op)RXOmIB6Piz=5X;y3t3 z$f(BR*Nl*kDXejiO=9p4AdO5oQ3lBtnV43AaHc#JU1sk{V-D_9saJ{^9tq`Df?<;2 zwl3c(*ONqiwj!o0$&e2B*`}R9Wn-kTqL}!|3xHxXttsRiC}4Jxg)TbEol6yzA%9H8 zU&x=EuZ_&UdqN1PA&Eo}^37O3`kmv5*C;6lK%%rg3n+plL+)A#{B$?uS+!==r4He! z%P2-@EXJg`vA@tVAbUK_*6}TOoZO{Y3mkfny2PoenA9=TyPl_dig|OV663qGEK@X_ zz^xzrWS0>xDa-{Z2gD5$4@l}OGN;%A%!ce@g)(%Nl$sP@P9PgsmLFSV_sVVM)*5)}cVk<(EE5dQWTbPK+-fi}R2~g-dkruc+V-jFCL$uawDZB?JHY!e zXcMP^4jkZ=CU>W9TbKqaEiS-N8!{{qo+8kGf)G7P1mM0Bcx#AGV(AJD;DulwopFXV z*}vSPI%#ulvKhMqX-MWJzn46~s|s}wk;z%OBAJSB;I@fTYvsLyJp>y=|J*|@7$w@R zfVj+?SkAb}-vD%C5)HTwN4j`5>>uIrOlE713b!t}Q6|@G&|BMfM3D(nK4lDvReU`2 z5F^x)3)OhRN9Lfg+|c0b$Y3Nzz6oK7*9qrC@L+cN)xkS5lLa1rJ4zVBnH^Gd7TS^> z7abSZ)G}51Q-JFokQT+0Qjt5wM9_>!WztJm4BeR}P00Z`N%f^Ts`zFoc~tfhK047c zj7SQne3rM4MWXjdR5-T#yOJ?wwEZM&t;7{@^Bde!QM*i@4df|0P(%lb7}0jA%5WaO z%0feRl@eJH!Y(6-GEv=9q+ZLM2x1D7&8{R5!c5DQTV4et%CxkQJ(@%Ym+2pV>hu>V zyg7KgbmiqRO&d;Xfk~iKOX#YeK^}&1pmGrfy!j-WA~j3S`Ae{66rA6+j`nX`Arl#L z^4?3q&E}GP1e(n-&-_-fuCUgG;KPG91-*ofE{Q`6h$O_qQXSRZkXihGJ5)T%mNIcF z;mggd4H`iiM`Qz^K}VW6PFd`?XjY=rK!UCj@#50Y5H&^wB$CNVMEa&5iSX!>c$7?O z{go(bt|TK8w-lB~PvF4Q2okK~4(iFnmxNCRqZq9^D;hBg!W&*D6g=?RSpGzofX>7@ zilG+nAXsGu+A@@XDp=YGq60h06+<2dyzWpFK+L09NSV)7D(qjzTdb>i<$zHx> zf~S_l9<&Q=k4ob#kpW@_1yWs@(oYe0zDS#vv^4|I+mzU!MJA=9ArSp={|18lqM|aZ zBvF(>LN~!!aO=G!>=OT@Gg5;pd-?TKYX;Dj_NL4T(34;LqQNeM)J2oTJ|^0aX%+?B zLso`8!vzHaIU<6S{M94^I4$%^A$N&e2u68nu4j~zXX4aGT>qwDgl2{)yoILg^cJe+ zi_#Ak8S%tXYGG%pXwK}@BZa2pdq;V`Fm@VoeWyiKJ*vv|=Z5+?7O)9<0*e zMP7Rqeoc%mp1QVSa8bMgsk^YLG9?pwhQN`)R2G%+Yt6loU{9bYIwPDSoyww8R0SF; z$-P1)1Xjd-r3TzU&D#h2V#*>;P%#6;K)|JK8bx;xkYe)!#$IC7cf7y$vp z;E=D6++l=eGOV7C29<_)XLa| z03+#uC$cMGGQ(O&`U9HGq=FU-yNUtU=XM!*>H!x%L9Q0A9`8O{;B@9;pKSitnHQ9NiT-Zf?r@I~`2KuD^#b!>@4Xh3&C5^j}s zGfl;|f_fHSn7lYB2Jqx$3WrqMQPP63j}g@oB26FC5j06zB-XAyCOj%-XQi|uSgmAo z-}{~S4ufu?qmG0!ll!d@|3Ui*B1Op+&kqKo=~WtsF6$a%?J~_>d=-B>-GU*vH~E9o z!HpA4^IQyKRiuWANw4L-&>QoVLk3N`e9dY=iBRZjB(5V12rk?%VP(xK7t~x zU{%G+Axme7e`)xVUV~o&BY{CRco|YLz8#3O5TgPpF??Z3aOd`s?g^PiGmB>G=JR2D zEOI+j9DE7nxro(Lh@?>H^<0QCRR}DOdsDgnghgv$0*DS?Brv`zE{|ci(?X^o7 z8v1=mg*=LWcYZgh#p$nXfC`yH?xaY{Dq^b&F@V^BoUf8xoRYgmMODc9tjI6Iw3Ii^ z4X?~sDaMs_2wJVm<*SUGV$f+J#0AU=ybBpx+jR5kMVKk{wy%{=9m*cbTQwDdUCVKY zN?(e_6to;}kr-}91&U0zWa{FA*`=@I1kVfZiUx*86ebXGO8f|ifl(d?GMEbovJDP| z{8}`jva`UYlGs88?iFf!eRyL8YW8!`O`%*muDtfh z9F%usT~p6pJyzFtd(Z9$rH1wFWT6y;7j$m~=t7OPdei8mQ7f@4Zz;tla{gFO(~+W& z*iz+>g%3jWyK;yU-a{Os zn}Q;743%by@QN4_%|!~HEc~sqPhnBMq7WIW@|0~i(|a&SsZC{+m)?=Dnq@LCQQIL> zy3bJ{%~7lAj7HRA#&^_nP}7*De?|kcJ`2s7+d@k4EL~HUvLQ#`XX4O=tHFY5cXxe597Fb z+824KytbB}9ls`mSu4OvZk=-H;Hmak4KF2qb~9pPy(Uech(S=bC|iLC>J?TOOzCSu z=1x76>LuA2k`osVEBRyw4L-V7eQ3sf(wl#;fk+ccisUPnEAz;AFTpPe9($H%J zbsqyfCT10o5<*d-MmZ`yX_8+#9lD25x#%v@6-g_7J3VeY?TuroJH=oO$2On3NGCO= zR6k8hG4iWNgsHUn6Fmw)59;SVy+IN(P{Gp4fD4Q?uV zun3)%s68Aj6I+1CB?~Jjd_^@43Q_nnub2Zo1B*^JUW9p^-OB+J#3@u{c8;DKWiS$Z z&>wM7-HuP3SV9m2vZRW`r*#dDrZxGMYt7J1)YorSBON|A;79=MmifD=4Tsg7wsR_z z6O)FcV1`wuRQH7d>c(JpIBJkFJ4(^VbO8Vi)F+GqVyeZFQI>R9;U(JFsZZnf5c9XN?pe!3!Yn&_Yw3LWPT?7RVI~o@Mtd8vCHwoGDAqm zEIOu#sNlt2q3NDO$gaq!jVL8JqoGn1ju?}T2lkj4ObKftbcvL;nTos+sD=N+_VL2| z1fTVRGL42t{M(|VG|<6(W_mQvtpDIa3$T6rm*6=H^@;5*ISaL@*fbhB-PMK{j(wc# z+ewYWlweiK#R#U~-3%(sZIg%5;T$T%zbdbA9Dikj9m*6FnW<1k=!n$h>Lq2<;P<=$ z?F-@GF)&!(nNf~v(NiihttMY-g?S}0XvuAo4jon$bRnxMOYzy0R9^Isi9~Kj2n1df zv4kF}0kS&$EwMeQhay!#k#?~!;cjY<_ew6U*d-+t znh3GCf0dCQ4XOgak|zn#?PAiG#GY!b#25AkNIvBaMAC!o!BWx!Kq1N+C9J5!#so9U z6^x3V&mEc3iNAS%CxQSHARkB@9Tcs@^|ix8hWfH;VKdOp3jNjG4?EKi*d5SGXlXj0 zvh8*fqKU<0SYhrlT7NUNIO`r>ytV}-! zj$Pml{93gUo~=lHe6Qc5S7$ZPRpcL4yU3$N|GP0UbQR<0S!H8f{s?p};YUSLD`jkB zl_dC^sF3cc3>!(tQTSgj48GH*OS2kZdRL8ex_hpJZX)zBsI(IpwOB(K| zI!OU1)vJ?MUV=`S1r6P0%Ps_Jp}GJ*MtFgWl8HC1rf`+q4?Vq!eG~Ocbrw`}MV2Uc zsz(BXm35gbOxi5R!2(b{8Pm#Jrnx-aHpp+Y>~s8)pTuFfEl#bl05HX6}m?T~hFbo-F z68eNZF<{IujL*VlSLy;V%dpy7N6H5ro+R?Be5FU>i-2rRzUfUN&W$t`j=NE>S+D**8)jsg z+>#d_N^-|}a`gMLiXbB)^%Qm?LG1)h3toMm9%)a8?ZevP#FlJm--= zdt7^^YijKBW@BCT#2=HsWRcAJnMELMCNST4p{He5QC7s`^tTcd6jdy&sGNj_HI#(C zum&Em6CtC6{9>p=!AZPUAa7;HaMA^rVVN>bfn%rwa|T3!Yr`uXq3J6cJh!~C3#3PY%eY>%UIS}UcagOSS-l6^ z>sWE9fL%Zt5urLAsi^J9s~Owt813^u%ODHC%8pj`izxs5=tB0Wl4?#gP4N|Luh)mhAgM{T2yD~SM>zJSJWVgIs_hc)Q*n! zO;SBX53xbzPJ~yjl_16m@C&&1{$AblIXt3NPKf&CoJ|EkZG4JV&`V(({yvBKWX zyc-$P1tTAckII~fg3lBN0XYn$y4T%NMbK$mIVlF1{<5FyJ&bjwBb zAnkQ5SOJa*gr7i^@MK(}JlM?WPvL+$90hA&uhRE3&8jk1g#M-mk7~OY6^T*mLFw>Z z=f@Wok&06K!k6Hc3-ycA4k{HOt~Z_kAgDl!=k}O`Po-3mj%%MBWsG7Dw;-rY8WNWW z(Ft&2t6bObfMA!J6XkpJ1~dQ* z(A4RG^NB#anKwqJ`k2APhV@gl;E{}(ql<%=Bj95cm5Hz)$CUxv?^aDC0iIMI@it=S z@fy@O8(_3N0mN};-NSE>)u?)r`qy4N$MH%^s1)4AZo>XP zp+lOa6BEMWgcN9ow8>-Gqs)pOCc=eZ*O*9`JQ#Ta^ zHmD3Fc;XN>Jx%k&0+cF}o-1M&^%8PPRlgVJgihwm9OJ&DUJ;`)lXHDY_IqSiNQ8IzWgeg!g{7oxf0 zwfY>$(Isg)Ggg|u2XsZCc64$Fgtu@--=3Tl1cEQxB`lg)G||eJ*dn71CxZ_sgARpy zLez>vN@1WltRer1m;5D)eq_bUigJ~Zr$pswsvIdA%QQU_n```0&OMC-Z)~*d89RUM zngx~-8p~j7gGB~A9_n%M;Rb6A)^KYli@F;c>rYkyF;mtEzDB*K_kw^at{fBp>j-UF zl^{)aP(x7-_;UGrX~8OsC5NQJeGvslRuvSKgzSYxWO~iKNg7y3!M&abk!wNT^%WP8s=D&O_w=wzV1hY&#C}sii=qS+zSJg>^(Lnc#>S0P}G+8j|(&V+)h3MENF?2_}X9y#I zP(e?2#7&_jFieHorJ?H7@J68B+Fi+w#Qmit)sa8tu6Eu8yC;{Ebl397rlKsPC?xI5 z!jdmxM>jUEt=m+eH7eT{z>bjoUJ4CRild>k;K*E2;|Vm1B^|V=llD5;|DnTFjzc9H zyz&y_RSoo{2AV4NtXEYCVX3ZpLW>&JXSjDZA*y(qE_;>pECYj@0^DQ|wO8eWr;f@| zit)qbtLL4S4y&@=q@XdFFHwyBzOr;*l`=6kTn-Bom0_pn*#L1q2nmCcUa4C}ScB zFhg0~%P$v?#uVSXHmbdPQn21r11;Dd5;<^`wf!zgNYX^jWW3;aAc;C5OYy8nV;LjhtyU zX`Nq0jfri4y)^zP51U5n+bCW($%*Z~@*pxr4(a*8x@7#z6XsB8f#2~bz=@Ran(8$X@}ciLN5X_0QUjs^4~FgPsComcpqYq50K#(HXN^-@V%Mk3ZlC0BnB-K5 zSB)gS32Le*pdvlLV80JRr4uoOi6o@~5hFDiTKGEWNUnu&J_ zxg|7WpvZKo-2%T>u6oG6mC~Q`VMV!XbD{Y8MdNst3|pxvE0jZ{lG+g#B)=W$-G|Kv zidt^!si~k)ZesV(s_5CQhc}?-q1uT-pGt(zIV6XY1SDJ`!9w~65CK24wJ05xY%_Rm z`kt@UEJbOSV2fcvs*+@^!E0ARX+0faUD{_P|>8fJmm`A3NjxJ9WjY^VO}|_SoLk@x2n(9y1r!n#?}Ulyr5zsR;A~eSQ-Hx zBYDZY!x_OJvL<}_MW}*%_f-|F-Vkk-bhfb9@g|^_z^@jET!ubZS;RDUMX?`|ZVZKLhtCh(cC@-Bbb9cuJ+HK$04? zAWdgY=(eR%u_FH?d!?qfQz9vxx*~B{KA$VS7!pb?WD1dWJfE(V+Gxs2!zYM^l>l!zn+<5dG|s@T*)(-XAe4XCHC2D&PC_1sml ztD@)A>OO*@5ezjjG=iaqh8h@ZV5ouN4NMi7Dlk=GvH;ek%8|Lsk+}+)xFTy$(-&Q@ zY=G8n25j?H4aBO~rUs%arl40<6ki6F!Me0%=d8L0Dqz}mYTSO&b!0lv5mzH>@}jQ4 zs7WmXFR``8q%i`c$|6xE7FKZvRSW@Mlj_f|FUs4qRP)t8PrPEW3Abh`wPg!ZaXMB?+3;>rP6xbrNrowt`AVS+wsf$lh-;eg8D@r-45W{Au7%1AiL$ z)4-nw{xtBXfjI&@yk_|{+!;MqXGIcnH#1QkW;??#^+gQaa& zly-r38^90#`3DL{fJ+Dcy~6wJzdr^3e^H=#fvPvYMY>*x7nphzy=sitZ(he&*l|du z9f1hKG*PP@G8ra#JzIwv-t0&#scca86ZSJ|jyJqlIiBR5q=XmbGRXd2@lyUk-%df| zBR8`$*$@D0{Z9ViY5ugKzh58+%}WC6cJSxZoBY#uzVFDJGPt9#$Zf7~X8eVl-1dE6 z-x~s}+VF`f-fAHMwE#tsWDxRdPFmZCvVW-ChgWH;UVQVF@&4fh{VDK&sK8oQi=;*z zZw2rh4k6t4%u(e%zHjO87pS)3tS1Zg^Z_`bs!l}eK_}N&JRmSeSKqMf&8vdAS>fvE zr2TkMjb|T0{tZV7&S?##l({c4(8PpzjbpNClIHZgb5e;qGtnbOwlkhA7ud+->yb!Z-q*(QxRGc~JO>d)<Jy+X%OlrNL7LR zdV%5K{w%=j#|3IX2>yV#dhE`U$KF?qv5)tP$wBy0cr$$nc4lzkj;MQ*sA9M#GtlE- zk=JA*02Z#vs>nWGlbKQSxF+2t@fC43CM+ZFMm+R@#~(>Iym@g+CgEctOk2n<^|WZN-W+6%ZZSnAA1Am18?aPC>IcA>Kfqr#f-F@USLc zK(?&J@+TH_firs6Pf1w}ncLL%O~UO^2?Gtfi0~bT;@Z|ilAi>kXh#b!mVDgG7-Wg_ zjb4}m7ieLOt}xyvUEHAuup^Wx5i5_)D`lR;2)v&}x+omglu4Vh?NjEKEDCR9W^BWn zNrfANnK)93qVEZyLYfO3A*do&8{w9M$B9f9#cl_2hv@rGYU>+&)gXd^9YBWH(1*PR zro?_b`WkmcC;E3mhd4LDvUoED2nzO68l`pk(S_1>xcpLYmJ#56wCiGWh7Bpd}K*$?RZHNd{geC?!*W>23 zy^3oOVQRX;cX%CJ$j7gFB*u(&ZKTTcFWUn+=m-g+aDCCEqmR5{?+HWW1JJ@GSz|mT zw_XvPoy~n014Upu0UNxP-HCb7h_LXAqC-IcXxxFew<9MUDhYWHdqOIcGdD zxW#k*VKD?p6(~$FfG;^52N@Seq(xl7z(nBgZWJVqGp-lhj@HO1u~9K zND5uXhUB58pjCPU5jgF(VJ8W<);*F~iOfVCLHqDLOADa%qSJL-Jz=gMmwA z4u+~h+y5YLNbbsTrcIiN@Zm6+3vzfgp1bi!QvrZqo_L8Am6Al%B?$`1 zG?mKDC=tNvNl%7ZiCzX-df{8f-6X_sQJYs_z>teeD0t0xp=r)WNMaUB7 zZ9pOP7RK<)I7`nVZ+EgCbW7T5Uk3!ewYS1-ErT!oGe9ba95#&oKsRI>-qbys1`!^M%p=?%zq07bnM9}<$keW(=XtV>|NAcZ3}EcgMB zStR&pp9*UeU3V45qhd2z`WswEiGcVFaJSB^bKG~$qzC3UYcL%r9g7V4f%*pW3V3Wg zxe6GMz{W}B_Bv#UL_1YRzfkU3FG+d^3U@){$(Ut`!Uf$<#ubBehIngn`fudWenIpw zOc>*Qe^Co)qADjUL6k_KZI6j(Gk= z_64|$RC5R;2m)(D(pc^oLqq6V8-VCpJA=uT*(X75bmkf!5cA#W8B`!Z=;BBuSW>gL z$|qxr(IhqJi)vV?U~IrSlCVSJGxQ zdTrY%Q@KIn-H6GOsD3WY55}8iDid^tGIV0{4YnUBEIhlOIM%iR^RFTW8dIx^7487F zB%hOkTbM}m05OiPsTc>M0`?8SGH{s6^xQoFm%v3Eu<{N!6{_zlh#70L2@#O+gvhTm zBu*>8Wy1Yo*8@Pvl9Hbzr6F0l!K;8@FqplIN4#ZPiO0F{ggv!AGL^TWGBivpCGRM;m zGG}#=d^pi(5C|OdIRvXonQ*;B3UY#*@;-ml@1VX#_yJ2GMc4}$9@Rn!KnLr~uydve zR`%9pN2^0n;9cFIz7fq!!c^wouyyB^YMGn?nwm6`5p)mSV3c%#!G!8<*^Wvc))0C9y> z4i_o8wMHQtG>!`UwrEEoQU?aY6cq_H9dZU99k~V|T5wAvDK@FHU2gcFkC(`C-9ferfTO`yPOIeJmt?!Y%q&}L?t zZUrUHm3Z`wVfS0+Fqbt~2)4>tOQMEb_1 zRj8LubO6G6i}FOLPS>FWZ7;nujpY?4fOnU%=S}M#$eQ(Q+sq~(*u+eYOUL$x&1{fb zga^!^C3tXR{bjeys&>pDZvkTJt{j{=D?ldCthDUdN(KyV#G ztCigjVO@eK7p*5S|L|@C?JL^_k&#F{OUVL;in9o*q-&`44HWrGt3i9+=(qbp?NV#MIknJ;AL5=880vxb98`^D-hw7vsM8FrX#i_BV32dN1|$dVLDL=#f7k?w@~nuAsYgu^8#0gi zYi3QfDG>Mu6ASFh*?3HU73+Ewq%y`6lGWp8rLI4qvj&OeWzvFj_CVanbG|+26YRU# zBHVcZ-(&wKBjoo&nhbq?@9*Gw=3jE}0^0l7y)njBHu_eIphb;LZ1l(d-VJFLg$={o zYU^FsE{h~tSw*K85XGVa!{6DU$TX!ag373vOr-H+d#c4Lk~c{2(Qa!F)9r>UfmImc5mEJg(i$l#)3u+ zULR5Tg~-$11y}ec>@=(xnQ4M2NJd5yI+`$W!Duh#xA1OZ#>mw0%{Mb+7_sgUA%x@F z2zLiB*cNEU$P&vcw*~>amhwV$t-q7JpO~Lk;A@D`NKWBt$5oPo%+%^a-Il15$irl+ z7AN(7C|P+}GSnYg1`PbtC`shEo9!_M=w~BHopE53){AW9l<0K4w~SqG?4HXEg2%;$ z@gkGIh#+_!oF&NJpu-gIOXLQD*+LtZqz{#b&-3%)1oaBZxTI)D3Ps zT6IbFAhHe`FL(qpdJsNm*XMvP2J zBX-5FMS<5w&_XFEp-?5CMdfDI5-M+>=&e~-NE^v^M9@ZJj|ZAEY#)-r4nb3Z^A?%Wkwn*WV*zpOfJLj z?j)|wHO^VAD&1{Z!NkeTRi~ng9WQq$F{_M9pq?d_xX64W*`v4xs@Ldb>XDTRG)??q zD!PT;W`UD!Gy?<=r!bM;$&nh=4#bX;N+rCNPAJoAiYY|wBo%+Mx5-L!K3EoZs6akrI86730OF}BVMvQL|- zLgsSllj($Mr!9*wZZP|TB!b|g%QskZI0!+hfNBkfih@30bi=aj*tXsb1WgfR$40ex zlDF_^)MKG1Z0v-NvSc0Wt8&1W+x{(6$$%ISb;=a7H|5acRx`6pl(#ZH(0L}%q*zQE z8MqbsXEq6#Y$Fn87dl0lYNs&%A`cEez#if5t3_eC@VF@2d4}uDz!bAaFfzg>oNubX zft*~3=4qygVt4&OHq|1{0z6F$bZnKrwh;GfR2wz<8}`#UQ8o@3AoOGNH(V;HY*8GB z2e#ilIJ0@<-|}qD)k6VR2~{sXBhC9?Z!BC$Uv-u;iFx>UV-=fv*P+&^bUE*EovD=g zJNs5lMbA25h`lLZiT7YxoL6t_Lxwa_78}^dUT*3G_&unl&`?ZazwUg+g9F)j34oR}2`z*S12cBG-L zQNotlwHGD`2kq>0iugefAACAsTn^~Eyge5*^{ntb$($*IA&0{0@J#E6-9G53Wg04| zfQ|}#DKx-_slC!EDHj5c7swxl-YN8b!U#IkCUQSA>!}}d65=4|Dqoox`+DbF@CWkB zTBV~_B;7MTTP3kQRwkzCD`*rwf(X*-n9LE#YJ_xBlqxln4i5MS!lY54Dyv-HLLV<> z#wnB&Du@KnhOwD1<~@MsGE*ZBLc{DE5LuDRG=eI^83_zLC?iZ^PzvD6m^4r_;d7%S z6o}@eQG;x7p~|x0o{zM;6+B{Rl!lfPW*X4SDF1s<%P7`57VR1V;}?cx!?fWS4yb07 zL;s5jw8t}qXQPXf5lc-fwjBiBDAq}zYpRfuyGMste3tU zNAC|m_qZDGX_f4G{mx+&sEPeMjzVD}>y-61j=nk?LDgXmDefng*6*vMarC}@xh7#8 zldxzn1R4LGoimI>-;tiKST;~p7)@%CT4$YNlGOF7_(_BR8J$wHQVEc>y6#~5Z3Bmx zxnDOBV2*n(k;taDBpBbLjHA$}A|Zwp9+V^$kr?y}LW`_rKZtBjVNF&FK3OT~0>Y?# zj7-cC-rN0~q7M=VtjOE3@dn|uz=Rcf2_X>^OTg2+?eE0m(ZAE`p#Dde6BrA{dnu7nBXwI>=|I15U}W@syo zzFIzEq^XSoTh=wLA6asP-FzUYnbzC{dJ1>41aiu_q6BKnK*~Ih#(kS6(oqYrrldnI z@duP8m33y);z-kCNfW7S^8M*tlASNeLWOoLuwFw8P46u=P{X1}2?VwfCZFycbo)(p z^(-DW%W12~H?8X2z7=1t*H@xNooPJO>l?#b6MFvCZ#Hr~mo<-`%gC&*!w2Cy z&vSoaZo-+C->{bigAwIDa|(y8;AeYBwO%s!1yvf9ZgCdgJ@|V#a9j0M(7}!GZjWZv zG*#SL)qi3XwIzKxne+Dzg5(iuy-q=gvK~)U$h7mURjM;@z$VNnF(AzqSv!Q$nLW}% zYahHPp>W2Y>C5{=!6xgEuqDtITzyF0Wb|{(j0>*Ab2j^5d@}na4eP8Ej`{sHbp?Hm zW3$FTSVh?|Ml4?+WJ1n{SDlo~1RTUyH$^I5$&<*3hvZAmGO_CR39hziiQs(o$a@Wb z^1&j$YU+#6J&rIHE>}OG3h&PnGJqoP*!CZst)izND6Y=P>F#W!+jxxoSQKNP>NQKD z0!)iPU(x4>+)jMR|mTuwC0Ss92#sR$nV@~iiucffQ;6sRp`%AFJds25*ENh zpL{}k2ES1V$;zDDTtZd{*E<&?$+PRrk$(E0zbsGn*x>13QcMK^p99rxS(Z@ur%u)& z`3VDRoaX@|BCUL@5hf7ZOdkj!1n7K&cW*@1K2&Kz$YoEcwTlb(J!$y~AAMF06c9?8dHVXbW^cp3?4^Ev| z_6XxDhVE?p4gjQjE(q73@O%sYLQT17$P?^lsX1yst~tBs$>Z)}r-D3lv-whP3ycYu z$zXIVNHvaD>>6R2dffnLOeO8IT~pDG68a!0koFPXQh3Ac3#kuG0pX#Xu~6_LKVErP z0Xw1*SujSn)Vg!rE~IGEyq+8Y8Pnw57?C&5hJMXGR{TlhyT)3{O37-;_mc7%hE*MZ zX{$8>xy_l;F!G(BBZT+>^#a0&-u3ecJBW%K)MbH$_JKT|-S=+K#y#EX+v{Dwd8z5k zty*h-%!wn+g!`CoHw=UfsvGo)=Pyj7`Es4biY<;79uJ=>SxYQ5zHS(2L1ikz$uf{B z>qO3o7}k<9{*CW?!-c=-{(0pY++Q;{x&}|Q z25SFBb{Ve`o(&^eawGUO1&`JEg0D3bGF7brN$AC{DJ*Ap_A6&0MQ?>O`)l@}#rMYY zMI$`xeRnwwV3GLS9A_sw0(i*;Kq9Fib2h-$=H7_i2Rmi*3;4Vazhy61An~#4y;@it zvSXryE0DGq`2zegENtZ6?^bZ=%P`)k&tSu5<{b5D?%lrlR}(bk**AA~s;(h`_w6(! zI`W5G`al(Ra)mgHFUfr}6kV7;E=lPAp!m@a;~}nwtRn0rCYbi2%&SiGf#VQT39r*S;NG!M#Bb_8#92tNs(FINHkMmID+3@2}8~zU-^Au;r zitXd4fMDVnWg3z*#9t2z__wr+ELt`H(tGC#&DbvLM@jC3P(JB(SK}i-kUG=ZSHu$G z6nGju4;~M9sBAV}Pflp%Y_b~L-T4l=X%aTmwf*SQdZ9DE#as44UbbVrysCS z-D;1h;lhPc9Pc?MpkXq*ESHr6vDp0^xGMW;Sx(h6Oj9YRTvYz(UOpCFe zda$19+eLRnf=96Kpe=1^RUMo(X142R`4B6}rS9!adwT--SW%FypTAOF-5{_~S&6l$ zQf^h94DioooDg{``~UE|CuEe-+=L#OPb>V)4bBP0N}dE$lzz&A&i1Q@3tl-n%9G|l z&`zHCN4pkYuYNLje)KAO?h`|Q0}e}vMceV@;Pvk`+nhz7giy3#7cfs~NV~|@Lal%c zqa@cmcb6`Hs!!W)vfMl57D(iWOims47pjYwGk{E_t^}&*k{`$QBtbrCu&R~|-a-$E zhhiTu0bu7>uZPT@48f!Y8w+s%ojY=vqZvJo>;O~NPec71eIwlhoCWJg-v8Tzf)ULN zbZfETE_Ade-Vi6@d1lf5beI+E?j#Qx1PjM3{qW~XAy4X2p6WcSuA;ug_Qi|yq?Ym) z|G5Q6M$`se<#y4=<4znnnjEd>dcG%Ga#vRcYLw*iJ)4pDnpFR&$M;}{tbifplz`Y_ zC0+OT=e_l>ODuB!pxoG*FEeB{2tJRw^Ss#%KX=SM+_ltbF&;{NB_=giGiaE44oU%C z2ka(f3s5Q!#G)$&k~(OLv}HR=1fyE(nn<2xKk^zFyR3F3)i+kGl#e7RgN`aP;4GuK zxyg>L68%QfZKH&tx7>UkBDj-oFn7WXC*3lTl{4U{*orn*q$z_j>2Fq|;&wyiXby@m z%7XDn&bK4Y6I!SDz4AXdY@5Zr`~JK&K#VBEREaADnT%iwmx5l#bfm48w=4VH_wTxC zGQq5-Z5=3SkM$Sr?04qt!1S=C<+>4Pa!$Ebsy31k)g~8H*i=kaTtR+vZYWB7w|+vf zq57mjOX*r%i01XkH|) z@GJo-b}!;oV51gqTy(X+0)=KKIQeewgd$OU_^L8CsbxVoFG=t2tzWy&mivqH^(cc6 zciDGnBl)mn{^RrFzS^kk&~pz-#w&@b2W194h~LTxHhSzX(vdGYQHxHUnqzw{BGp@v zx&zv0X_B+2dmF^~h@#2aSY{jO{xw+UhlUW3hga7eS7R!mV~_AN$V}O$t-`eGk3*4F zWu=CvA(|CdeqXe{xKZs@b|VdK zxeAf$&esfShUz9=D^5SO+qPE!k^oBC&`$I)B65Mz<2xc5dNEz74qO^@jHq}LrC_MG z46(fp%5W*%T-Y>5@L!V+jn&xtdMuf#BKr1AT!jAx&;r)$M<4Du)Ze-GrorFj7)e6C@qa_^I?1U@?RI2C@jYYRbK zNEf%U9oVd@y&EGA&L_81aNn?2Lof$MlHFg!uZl|T8%(or^5=`B+)7$ApA)|6SFF@~ z)Mc$9(@<{A>2o}Z`?j-QJ6zE_&IevUsNK-AHQzria DxAJ6r literal 0 HcmV?d00001 diff --git a/third_party/icu/data/icu_conversion_data.c.gz.ag b/third_party/icu/data/icu_conversion_data.c.gz.ag new file mode 100644 index 0000000000000000000000000000000000000000..bde20b6da6253d866f87fcadc7e6c3571bd64d44 GIT binary patch literal 177696 zcmV(nK=Qw)Y~Vf{F1?1xlz~`SrF9}G=>#an6vieorg1@eG+-bur19ihRu;nzD-17!19{Ug8uDNDVKtv zYPIY)C$?9SFWQ)X0;`XXfmW!U);Q)Wi66#3tTK;;nn%CdG3|7ii6^y+wf0tq%Qi@i z;2|`z>pIzt06E40!Te$Su?5>(LCf_+4Blw>M3|qB9Nb_|MQrXEAzi`WJZ)bpO)ir= zxI+ne)Ib@>2M-ZaQ=5P1XX@Hb6%U;eTr#n}>w5S$7xAd&vK(GLNW*U7%{A_YkTHTo z4YDa{4w^WQ7M~usu9XK$!kJpM+X4R#!`9GF)f%+Cw`B}db*QSk-|+U=30oXtZ8Afm zc3eB19-}e3=}KOl?&=@2hTSG`WiCHA{yezABrbDI@r}RtyrnU9C|p3tn)N4+soyr7 z^@)3bY=;P=3jS5c$V%R2gR_MosUKOBej2=q=xCQdGD4p za9Y3~r81~GR}X#*HTzb^lBcp*?!6SC1Fp%a|1tRzvWqs+Ixc<6L|h4fxlw~zcaH1S zT&qvxJQW!v@%c!f`&Y+({WDIZ&(CGXZGYO8-=2dNKDW|zm-Ppo^1yB$*zn`|E}YT1 z3>rTlJ2ZHc-x(9m$T;0{ej$9=)F=X`r$YH8@fii* zmY`(++jGn2O+^5V0eutyXV-;3?h|1#G22{Es(9g#GMDsD%A;|@0fW~6(+&(f zt3Ne~QMQ2jVJCs>od}-YINSgs2f!>7J@UEX-vH@U=;sl_QGtF7dC@P-WW}}wPd>pY zZQG zlXzsJf11p2?8xMgQmWKyb-Vfd5}G<(%SLzQM#afuGAruRm~O-*CxEDy?kqEqw6SoN z7I#yV4Tv>NjkY!=WZ}Vkcqz&(x@F^kE1b2**!Ok^<$#**yxDMsFy=;8S!4EWBRm6X zJ!?KtNzBCf%GX zE4a^gK&q-i&ckGugh0J*tE2(VWz)x9U@N7IcY;cku``GR>zkpsId*+n;%>R#b^FoK zL(OucKJ}&(1R9d>knD#nL+=A{r*`fGn{@8X7hG!U=f^ya45JvE%A&cpiYk>kud0b- zpvBkyOYQ#H#B31Z&wLeGhT(@=neu`Mgl&CT7skWDprT#E z*Q9~yDxhI{u)hnmcx5RY2I!i3(``F)#Qm19U~KK)Xwq)gxpU?tTz9i+k1u!3xZ7jC znmgKr;o;TM{iekkqG@YA3sgpn)0RzVJrPV%nFb0_#0o~BUuBzuUrP09F3PVFFiR~j z0k!k4YBFo>yNXVfn4{|NV4Q@}Yl8mL9ty1zjU6C=Wx6tIfH)4g&0>dw^PyH`eW`6U zXmS9XF#uTGJzx(!U`1wz5xs3wCN;3ZzL#cy;;X=D9>?Ovz%2X{3rR|0vZi!aE6#P< zRoLVw{_VHbzwLlob_xOQ!Dm@$;m5{n#Y7y}EtZ=NRWJZ160JmQ}>2im1lAZ%K zvI4H~k(~H<8Z(QmkrjM-hkVmz`Ki@0!7D5b^E@4}tcWm;vRF;-_d2|IN8=?Cz&tL6 zUV(iUTpFD$EVCa|eY8da5nAI{ZGsio(y~D9KDf5=9$fmRDV#9Gbu|Y`qeD7rcr+gE zu|F90e3fm5X=TT;?1ee16OCU*@{8$lwJN)WJeHpK zMR6WYc7I1Z`<>9hJkgiz)S-H4^d*>wrcA6~Txy;Ukf>3v-S>8~+04AAtN3=DMUl7# zjt<5+j8zO0zqT?Ka7G?@oZV5^08|3w9&#-!l%4D;qdZ0Z-9cd;ozujV!;fS&Ko%8<9@g(UuRFhk7ZEj0e|%K_xXtUy3HfDYPkM{{?m!Q$gt(`GOtP!(@%5Ql2fZjKaiyP zKW)$3_;Pj6+;;d_?h;OABKZ(pwNR#yH&bWFw8*u~`+%yqZSo4P@zU517y#q27`1eH zbQ`Xos^Z5rSSIn=%a$q;?ot)vAg2+wd0LK+HSTG(v0AJ1phFsmE7zh~ zb@|$$iDSRON9#-k#{B3m@$|PFAST@VCqf}c`cUjN@)-qy&dYdTr+GWptF5_q4mg?X zM8KL?in8&-&m5OJ{m<^|GO$b!JEYaCf>KvpCc(O$u*XTmjfjwjPX}!EmwB(}UHz*r zKTgKa7s=>;|H3@^^d;`?Z>S{m%H4{z@?>xQ*>Fd?UQHj{02{Q~^uw7vfbl^rYj;mQ zS9DjpJ~6AI90w;wGI<)z@X@Turq+QiV^?($)@H<$jVMo-+-@6|S4UsA0j?Hquoftk z)+d_L$=DqvTw@?Q6K>2#Vfu^B^RpE>278*{R&QPzxj&t;Nxd1A!RCD_esw@q3r-s0 z9TU;Lw2fR;9`kuKO=#C&3__cAF9wCu;VPvoXx_9$EE{(_u_;dKA#p|MFzM|G=R@)q|kK(?};YdM#fICazHYv;c6vLUvBY=_8ZUWbE~mKCUxINSGg6Wd2>8 zi*JIh=Xg4yI4;mhV`^}&O6EPVYy%@Q*KOLqnDKaOcPI(Y*H(o_AoeESE+ z{jtVrinf;0ed>xhgN9@+ClN7MZxNrme=Iw#IzRA-7+QK7n)W~A-14`B|8@fWZ>)V;y7CPB($;k;V94j` zF7-1_U3&HE>ororZ64iJ||0!vTc^(Rtjp_ZbaoTz?03m*%NM=CtSX=9_{BSrYK zqtO=0U9*vP@#FW0+sY>&JIbk8#=iC+z11+=DW050dOYi6PhR9LoMNv&j>_a+1s9Cn zjkC6!H~IQ0xzy3dqk|=U!=`XG=hzuAM8iz*ICb$0n(XB# zz~j}pus%P{ed}wO^W-aKnbn>Ky8qRB)2S+MxP?cth`v6E*nc4nA8Y{wchRPpDfAjG zV-3Md>tq)gnM!?pEN-Q~ej9Oj2VHfx=X4f!)-}Nd z8OD{4N7Qs7d_Y;%YdqSq|I@4Jd4i>Ej(4DFTYE>mF#ug5m3$LYwoYwI9Q@%AY7SkJo_>Ez4~C$tWE_Nz^QYpFd?F>=_~&q^ee zY4?MIL67q}tT$h=k0T=JNaJ)tr9*dsy}33%toZn`4*zEb+AXs5UIczE?RgLWMo-U2 z`c$TiGAzFbWTo7=^U6P6)6&l~+b@q1SJZWX-p2T;pNYy}U#72q9O*Ci;*5Vj@ACibXi;Enq%MWsoqn{R z4Q6~no_;)@L63!e%-<}|36Oo1wsSP^u!?Te9|YG(mjjBhD%o_O`w4!@=-+Tj%V&tY zFrRo$;#C@Ch8U!huRdu|Tw9xzwAzRYqF_^$MFE6?cG1;Q&{@J zDUL@8wVxbT`%R23g>V>X)Tm7*+9yqhYp;TZ|v z8^JyrxBEsxKo45weMmHv7H;qob#0qt_2xmyPCvwj1JJ<4w|PLq#CYtnK}3mTt3B>@ zJ_xsFUy@#nr=@@T8#wAOqEFp7?{Mb|kiJT1KgZXXJav^=~jMqJS$V$;uySL+&MTV$uM znW`L5IYCUUy2*>dGM0UR%roiv^bqJXefk+i+T)V>UxVZFmrLKyU^0z|36s(L;ZVQy zOE8OU&V0S^F7)t2{6I4w9}m~@7uJ+d0icB0skZ4J__R+PA+O z7u56VfcNz@<8do~O?S#V3XG!Y{WR~Vqj3l~Kbxh;0^=h^_<0*&KGFGbxGLNH7rmdu^!8c-B~K`M$p##rGVRu-;R9)7nUH(}j- zf`J(f0H6f($$bh2U4YUmy!LPpZ4CIf(h~%}AZnfL&WumS(ML9c@WtbUV^Tp#5wO~~ zxk}~H$xC}JR?;_Sfep?b$xlgz|bY7rwN! zw#}YTbSDq88uJro$j52D73kSf2BZ{SAXzSmq$5a;K0FQMmQ&2w;zw(jt?JZu=}RXZ z7%P%bRfkzD6`63Zs!Vn?K2WTDvR0gmjAarW@RR}nj_~@H6+ArZdZtt7{W=q0YvdavNYYno z)`{6g_za`3Hrv3s!4slJx{l&UBbs?l=qz8gaB8I=-Z@!GNp^p;60Sam&5?nLCfF#| zMS&pF`qgPNxrEv++0Fuj3ovIcfedX0@|C&514?xb1x#Mj=64z(mNAYq7p`LHgqu;D zkJ9>p?qDP(VXCg)t+IK=a+0a2_dUWu`ex|-MgduHSY9n3=!@efuN7(`zFg#QcLDVG zu5*Abp-W|i2@cem^TCXw@U=ecDSE$Jnc&ve87{3+FO3(X{EtVXevEO(KC^{ZtdqN3 zgG^k;iea2Lv1Ts5avo@}6@lBm?i7u?B(5;S%CjJG4BP<-(~3%*P&HNe`v*mIz~0%f zcz0ubACjc_q?t;@lw0kiNd!#)eqDXwRKUed*U0fz5vo1dOw=;QW)AEO3$a*x^u zhDjQamM4muLvH*8i4YRcwmD`$QPd6_MqiH0bCMDhI-b_#0q~Lw378<>4H{4o?yf|s zXX65xxDz(Z|O)A1B%20uj}xOUuJlc#d4*2)d44tDl4SGnJ0ZH12j_+{XD zO%T5d2B<>oYO17OsWcAkc-^IwejnGn2=0V?$1EA?fFJArqwSRDA=ohOG1Q2g|wDHnDQ@hl~AKTI0c+qtP3|}gnj>y1L1C-A~u*nZ*K>F1amw_Zn6 z&Q>V@V%yE9%BnCK;2hyWZ_cY!P{JMnem^fqBPAAF)A`bo*nO+wg71RtFep+CvT0(G z=8y)$7;S~K)mv<4b|lsZ-2hWs9NZ^zMAiD}Zbkrmtmv9`W6Uo`3Z|+k$$v2R`?yRx z!In2Kf_7TMw#*oSY1j^Fk>)iOWlvbGc9%B_MuY6(3(oU0ytZI?*eE8h-N197H@WGc zAV6K8xuoSE@vM%3_t?BPS9a&zvYyC96`wo;WHu#@V$x*NSB8X%y z3O46p^0Xpcgv#^}brW)-KgWKU%iU^Qt4_4FUarA+{#P+>$nX#9a-|aUwT4wU+^*Fk zG}*2T>B$CxzVdZdP-K?gG`FSwH^Aw01?#6-^KMK)n%JVZ31Uke$XlC+$d`1^Nb-mP0=Mk+#TJNzI zzdez+0?>Kv(bP9Mz(f6;J}vBde(Ll5e0XMN-K&FcTo>*95Aj}6yzIqqMQXWHZJ1Y| zuhM$NWHzYJN_%`JJg)BNwtSP?jq3m2lAK0z19v>KF^Wxvvr}U61ffD+))Ela4H(6myrmD=nj`7;*tuc5o+dE1|r4-j+Sl z^{J=@2^RL#p#C%nn561v*z_-i%HS=G`J|Qlf@QQMebVdpDU4(lnJ33~W(KPZY=sFy zP3g=8Kb;u{m;kIXk4Qn(6?gNFpE}1`J3Y=W*y#j#4>my_Jnr;bVV2&R77VFAnePz^ zl`zsTf~H2J(H-syc_|z#M7K&{%PyEb-=Z0A9HZX=Ba^_}Tb9IPMn1nJec)m*_OXp>7It_qdohT1NzCj4>XhM*0EYyk2g0BYhTWkwAPiPb-tH^J_ z-FUJ9?pyJU<8X;V!iz29pfK_nM#ceUAh_s-)#$dDQaz9D!T_0wSblWufzmKTw_S$d z5Zcn>8pk)%lc!Q`t|U(!|$5 z+*XMBkjTOLHT=HE)~#>_Q#IkVkELIPZC}7b2>-cX0u$73(jz!B;T6*NBXY!fI!4^4M^u=t& z>n?2Rs?oj^ix>{-jD;Ep3QPWI_bUujCna7Bk{%)Pp%K*aTcXmPFuv zI)+dxUiLy;@XV74WHCI=`f+*OZ7HivIf40Lqw_6Ph;G4Yr?JiH6mH@;7v>-$OIzk@ z_zMhaE^}N|Hs7LmOZw5?5{xvqR~388RKsi7lY((O@rRcg+{))E>D4IQDmPAk<$!t> z@LswB16KkQw37(f+k(EYVTmko;VCx@Jot8E_ixVZH&hXqt;LF5{5}y%s~fQ`uTlg$ z>k>NZT|rTo5Z>pJ{sNAdP@kV}rzIkfyjTuC z8Zf};f*va?&r&T%=FUS0-d9dIW>9Ho*tAc7EuKOPDzz%*W<(ohX&NeXIagYC8I)J+ zZ5m#_j3YR6`Pn^vTDUc!^mLGGzF9;2o69B+|D0{;E|xN76JwHOK0b-L@cxA? zrj1EUCPRAU@l+p#%`(aVpYn3N;1=_2&S~Y~tScn^c4+GCFW+(yFxpfMRyu&z<7Cuk zx#`-NP8gE!V4UYAj)tt>)ynS0dX)BPJgDXC5@C(!Bw+N@)j8Jo?rTR0+4WFkb8b7; zm-39eZ6rKwSPqJ#?7owV!$QG6z6z#YXR*N&wh^e_P8|bcGgDUM@-p?|Aa-_WjF8-6 z(ZYOM?|#gDALl0r@XE>#Qdb(j`rw%38#(xHN$F+{0UE(#7g}O4=ota>l9sZSn?u@M zNSouVQEutcGj^ogimiuxVE;>3>bWxRZE0yA_0dnuIXJnKFyN2H>SsvS60h_fOCe{l zhuQzQGe7xl|2Zr9VcZH z!ha1oEuaeohA_AkASdCGKJ0(3$(_VZxQ)>_&^aAx1-83>JTSVoZuvm$i+(r0!g*Te z{*~Woel5IE*C>Cg!>gd;1Qr-XTVl$sm7W1Uf`{n72i}ZKD))$lh?(CA*gW})lx3f_ zz>Ps`!WOi>JaBQJz#n}#;NlMP@lkn0(3s%?v5v0T!YYY|)6RCCCaEpH1NSv`3>|n+ zc8A7(Q@qwo{)~ki7Z?t?cia|4P+?BirCxb8QYYPh809hOUQf*&lHbvM&n&eV-Exk? zuV+TQZc{fG+D0#@evXG4kG5)n2`%oqErlxg)3P!kAQLKDS_rRQycj5eu;CRj=!F^= zxyP+#Fwk>ZEo48#{(=%~9DO+N;Gc(Z+X=TamCE~6`qQ6Obkr&bZI1Lrb3Zo4R9uy4 zqtI@?CTL$v3d_3K?mC}-sO!%1iF>HMnPPPEKbUweXKi}#)X23UG4cN zPriUtc!&K)O!sNJ?$KO7rdnUsqE-o4+Bd2-hX?Mro-oVMLslZO*AtJGljb$5jJ5)_7go5zYDR=j4Telj1G);g6V_Tr#K1x{VLE<&v2Aj~io+U;>Zo&~H0 z2Ro-v93QCCT{9@Z;`HccKH&5%K=_|Mb>>s&w?Wv_ug4kGE{s#vFPFKFLkbQDovM-? z($&`d*ce^$ua|k8Py6l(kL}|_j&xPV^&w<-!KyswGmdpBzOyhWcX4|jw|~5!jW0(A zp;~1KH1Or%gCI-&57foTmA!i-quWS6(eFHq@*<3#XR_QZp?#I<6 z-&o}QevSUiAE)jt4xR8t%0f{%_>}P2P>J`MD9in?o~&B*3i{17(jN5bi zy(tWf_QCXH2A`$POj6JFYrBn?&>^*A-pVNq{mTK36Oj|b{l{ZG%g;9Oh(9}XGar3TZQAk_LjG+_|KS3) zDyN`nQz|sOjZr6OD@m<8oArHvuzuYQ$GNRmGfgwE8L#?5=zA_;`=~KaL&~(ylof|w z+X=c|GUIR0S0P{f{WI21?>cDEBJrmdYx4}<5ui8 zy@i5)p>GqI=MU0v+4SO5@Cbvd43{A@LgApm6-Ns(_Q*MnKKL9jWmy@wyw}iRFWwAg z=ZYSz@t6hLRb$s86=|Wn0DUhLgt-2mJe=KQAM4@ORhka)7SYklqXBcq9fUJOV^@&S z<^|FIf+@g!T8q)Fi1M|k`Vhxb7J}<*Yhyw$(--j94Ja}R_-N8lWBV8qZ-M+dpa`Au z3M4Aw$&D6>|#3o0ClWCD@LR5AX=SFF`%x)}X+-IB{gxBGh@{l1~Q?pNPspmeU4h z*&^F0VqVwGneNE(5o4I)j3Agb`7XrGwAwDUL!hMwsV9KcG1I-6=-uav83ndyqF~7JiKgf`cWI+Zua+C6aA~ykM*+yoebtvApyhmyi**XGO?%GKMWWX9RPd=piyb|)qU&fRw0AZY6%&26>C zl=OwMA^0!%^&`9#m_0*-V2=%{zRFP$j$>075u;S5<5XP#CNK27vWL}UDeLi&^{9O) zFpgG6+j-9(h+TZcT+d8mobH=NzwUYMWVL9^uIazD@c1Ms)AA|uBb|bBL6k`wcjIqc z!oxU0oYikqP za2Z%`hbrslHq{(gJJZ2>QtXx!6E;>j>$5h)pYVuWSbZ^D90ve5L_!*0*K#IV<{qX9 zQp$t#%Q!-uim+1F+^_z*9_`qbO&k0GYex$!%Wc2!l$Y;|0TSw3-;-+rJaqCRrPr9G zupFs+U+}1Ka%x=dA1DoAlz0e9C<|#^Xegs9MV~)N?z)Um^CS3nRLokSu-k8#DB(PS zO=N~U7l@Q6bbl9!D+CJ8N8Fgm$saydQs4{(8@tAHh!SxtH;OqK+nD$|2S{Wet(ePSF;GvxTM`j z)y+8~#TmxVaXz*^olL{$Hd(YTkBX1un#Y~6slwx~3a4bgJpw!zPXTKF^K9pmB|T~5 z=5pPi;uFR(3Th!#kZuw4-hC7Lk;>h7I=s9P8de^68)1=8^WEDu4udKTzAt5AmJB8v znhbvnOJXAp-bWh8)pW_I*TF}c(y%1Rfv<;BzgopAoXYg@YWpUjbA${hPXxNvnm^0mcH1jx3 zbbHYI)Ye_cWO+wp`PRw(G^g|kx_l}V{JMT*z>iH(eh-nh*IV>SnM+eKZcqo*wq4q| zd(iG5Dg%e-El7eEY@et*pG^zqD!dcz=ks~$x{qszbue$l^+ru`-J?Fme;H7t#Vae! z)hj1I`~5GU0T+MPRXbrK9hu=+1l zCg<^emyb%KP%o9(0t0hhif_%dN?F zcY`nB++pdu0IBpFX|F4%NF_ry7392|0(w0;OgF|@wR?=w&*!gs*b$Qw!DnmY>$Vhk zeKS(o!M>bl<%}sr+L%iiYqY=BB4DG9@U|5yWr;GHlw9peWE?Bo43m8Q5Y6}chn!82 zGu*EVu{r1ddVAC-wuPBd>H1%VvM%UM=jM7N9|x^;Uul-gXXNyKKQ8Rpe!{NtpTC0o ze){W~Nb&7S!>R(OjogpD*1)0mV_yj$v=V`6%-?pXsQv0G_Bi<7)@~fcu6;6&)|<}o z_Xy62&j)54Gt&FQehcwx`1}oi`7q_xdiAF~^wmq|l^tMIffRj2L56nq!i5bUk3Jrs zZXCJ&;`zl~tk&keOuKo} zo`yC0sxt1==6v4|DQS4pWZ!1N|9bq$A5Sjiyk%Ya^6_VUo+3bMZO-iwAc3I>p52o1u9yGdNg;Yl4QZHW|}b3W;t$`A<{4`{19` zGE*#sRfNKyzfae}-eZP;w9zQ`e@GG;LhiOdqhzR&c_IH{=CxL%VCjX57Qc zAbEd?&QY9k=UWjw+F{E@boyX26)J!e*nHYWJB`20v@& zDX*cxa{EQTvV224AJ5v9ypYaIa9H_DIF<{U@7)mzh?z0~!*bFV7z%bdLB5xK9Y;qv z2V%O7ywSqEdZR9L+qLt8+_15tlMY&?DjUdC7Lo$QxEgoE;BU&pr-v+SUSIRs*R;&r z%su_pGtm8yhEp-MdF3do=|^3CEKqNYvnj7m(XsuAK8|^ve{fw}-Ub3ctdGncck9)t z`~&Ze!*KPt38xn6BlsCG6qXhpkb3e!n{;oiw3W64d#=0h{k2`{>UYqI$j8gKxJ4hm z7nir;^EfG-WGP-VuGkYWbU1!}+A|JI;(M%Fhu{t5~bM zafa7>)|Ib0OL^N4ahurGw)fkjq+`MbOZ4}T9<&_WuiT?BAKR&lIp)LG_)jf+eU7mFy+*$8@wTqF=CdTEn?i8Tm7nN}M|va6q;7gf7OnEe*d@X`KUMk&6 zI{^f^24*iKx>T4*K8~>bmBxw?b9bRaRFiVbpfnHqSSXU*Mr;@5K?7?zmnw5zj}E>r zG|YNKDbq%Kh}4CUULnNJNVA_*X-|uJ)f$%NVtpx9%D4JZpuqd~=~o*ha4jaGvOjFb zdA=yu_KDjJwnrW7q%Zrazfjyqq-TPbhjj)`kK}I?_uPWEgI`#q7GQJR*v>NMeNt=$ zw<#fQHs8L5g8C;!)`^gbdC-K?oe+bbrMGAJwpa69>=JO9&W9GD?QFy4mD9h+E=+9k zk(3Sns^TRUhIj*6_3bq;n2JoNPzcl&T(ND0O$A7`#6#!?a%J%Gd-HUpg6xk+9P)Pa z)DvV{zM&i$N@-(UfaE8vGh^uXuZr^(WBV$tff5{%jHSOeJUd0{nN%gPs|rupfiWr zSK42K>yLTE*nnnZnS{M`Q9@27E8F+kJO%!h^Uf?`DT>SajKRrtGoYvQ8T)>r^84wW zJWfYGxo5-mvoV*-+RGt++#erwne`NAHc#%PBfbSNq)WK;&BOBqJT5DIWtdYIM9)Ct zW@MyZKvw_?l8-B#OIsC%+I^LB*^eO#pGUg?)Y|C+{rJQ!D;u20^#xCJw-95?yxQYt z^VHC%>_mO}5zSNZLOy#&Q=Z?Ql8+z`K`VKPPF?0{vPeHn+jF$c97W3V`oJ{>1l!Ct zk1r}DF3CpYk>@AFewpjK{f|O@6&m5=x=`k3gpE5nBnhP+xe^yio$*RT+N9=ttQ(&a z`ZYc3*<1TF8A*z&O6k0opZ?^RhP)x|`H&WX(Y`WSJ`3fQAWojHEp<0_4%-fA>X3c| z^#9Rfx4t~Q{V$UtuSjQ_c^vb&GwJY|p<@5x`aiBDa_X5!tDZH&+@e^#hm2?6K1CtN zUJhp3oUArn^t$4^dpHsd!%-GV#wCq~8w7p6h#LnOV);xW%JS+gb(J!4DE?KwZjV<3 z?8i28y$P_T(6Q$8UH@VT$yi?3WhG-Gn|CC}@ZfJaZjhT}B8_6R^8`&*`c8W5ao}n_ z?_mE!UMu8Ttwfb>nGz_Cl;%5CK4x1$!~J|7p)SNOxI`GVAQG3t7VkWkkKyQKz$Xo) zu3uhZH7&aa(|>H^PCI&9-*svA#@k`W9Pc+@t^<;mmQKy#tX*cThx097CA-ly#8h&e zyvAv1$D<$;N(5#pJ}&y>g_l0UGg%xPsP-`teVZ@@ooqM`FmiVes`uwsxo-W*#pL_5 zGyGC4g6AIsuY`xvPn?7hq3}=!a^&v-G9FYKT<4B1id;0MI!4h(drDhE(|EX`0xUFh z@u76wnT+4y9e2w8yKJ;$9J_2B3pBS4dUE6Yf`bOTf-5OdO+e;Du+R^8(7Z4_`_pXE zcx^5LaNhe<&m7(8 zyLQ^5JHNo^uI*;-p5w9C9~x-FGpG2m6h5#`ds z0wE%Q4GnkShU9;3!4#6m{)Pi_ZR3h%l%4C@R}KoRNkG_b7glg_WXqq-iRszTGS^dR z;I}hRw)-#m{gT1|U0#f$!iR|<@iis)pmSAQo^_qTfTgK;U>5+_Q!C}#1)n4gxKr-QJv>W zNTBm41NP;7k9vyg?!np7c}j$x53~?!?4q=7$FowS_n`IeN4_`smX&c!@^iG&PHoI` z!r<@ux(rk)#^UE*qJ|rPlEF=7_gVhxh_w^k#JA(h z^|NvcJj&lqHS?nadDC0_cO`5bdZSp_BMl==of|W_riVne1-Mj|iphFFmk+oulm+Wo z4`pXP6q;wT?PkBv%mv#|TA@ny&4|h(L=3NM*vU)+yzxkCjK0O^t`YGGfOh}6<$KAN z{L1{O*KN-_-!GR1L6E<@`1$6__GLe?a&Ex)=@EgdLSnTj1DG>+@XoyA%?sG+de`%x z`Km>ndfgRt-zd>QC4kBqH+03ZK3dP0BPq0%9*;tc^nmI1xR)#e{FbP!*HtQy6ADqs zm0}WUyi{Ss-0E?6>Kw<|$5HPP|CnWQXfP?e(UTyt94Vi-v*-+)nRp+^h}#(IZHSt# zHG;WNN#RSg@&sJ#otN?GvL8izTB4csZu2}vnVz)~rwU#u4GC&JIwvkbh`C(J#<9v4 z8-wgy%UC}1seR?Dh;u|R&$Q?*T(dSo z&!atap?QxAq=n(KjS3TWrghWL`FMd)u9th?rQeC8{&MU6d6?2xh4gmMbh#wACPY6e zvmAk(+xZ4>2w=sNa_N4UXdV86gBrDK4}_~gBq-@I(F#X;pL1GF)mjOkbg2{SG`jU| zt9I_vVAoG6ui>E>|AO8dEw=Z^Tu$~!HwG|CjXM3`0d}yz z(w-1L|AQRe?-dwbNsY%BiiX*Q{D%be@e0H7ZEy3O_V4MhbBckPb?lVB;8zDAu4+6n zA^3quuTLKwf+0?}h?g?c+4}|@w)vO=1ZlS~r|-myRpN^cZqrgenW5%5o%_KJI=Fkw zDSP;6_dPY^8t&-+JZqamAUvY$BGF174!~8fMQ~k5(v9|9B~F?bQ4!M)J#cR)PKq(D8e1eEMM=KGd+9KHVoLe=IVm^LlvZDX!&T zLl=Il0E~e@XvD+R{?!lj{nLhiJ8#cg(eDcV8=mGrQ~)gW2e{h#@iuVxgn!-R{D%tw z+W2n(i~qa;x%;c;Exwg{NVz`k(@WR{)b}PB9jElTOlr%pKk;`#j(=9O!?$@PWi9A~ z;Ui0v^6Ngx^j)q*jT}?5-4Tswpq#-mafwI_hbs$#ZD%}K7}GKXM940N@J}si2`uK7Hj7flnKp_~ z-B%2IDYD1VLo|WGFn^~ZL>pF+{-4&F2Jy0Y24Q4ITV%>g36Acfz_&q#S6RltOC;qiH%C@bENyy zVEco;4Yoas8|6Clg)==dUi4stO^@j%O%bD6J;c7TRA#U82orpw8SRiA8^yf#EIof_ z4&ed;Cx$I%7|t88ySZ(rcxG>R8Oo3FEcJ+VsPu+1P4oviib^9p&o(lRU8rp9yCBf$ z355VU&mSG$Po7HBir;MjXUC@YMeM<8@&KC?y+;r7yX6}>)Y&J!=eenHN|e+|?C=%UW*(-eYQnYr;g2I*>2xJb1+3Ziol-7UqGT z>*1Y+1y5f1BM*jXjmhAKqeyQH>i_8}#)2O94vJQQ9kFex!&!sFcSnJu zI@uGaK_vw)qCu7%=>+jXdEgpFvpLzjSHkh5e08I%>!M!$f9?Yqr#?%L)Em0SI;A7C z2**=kXEL`z_&x8jvU4Gd<>GqyTuwn~=QQ?s4;iw%NYM@rL=5VHyYoNWO@}$E zq^r57E_ZZu8~>n~IIZ!=*-4yKv$H+;1s%oGF^Rw3S>!tn+8jgaX<>UEE44Ml((_Q72h)(3e7JYw%IddJ zH4x#zP$1=b6M!Bwl2rI)5$hY>D0K-^5y>6@oRy^K(?>H3xur;HVTR5QoI6KXF2_L+ zzJTRK*C49n7*YvW5R&6oK&q6%k>q|wlDnCB|rb5}O4_F>VO(5`<;J_RdpG%l|>;;5|n93sjoH}FNZbs#9!h0AqU>fb{@uSH<})QJnnu`oa8 zhJlDxzGd;jZZCDQ97M2y%pv>mB>z zTJdLq$)R)CsW_oH9^ifObS67@;(RvuiBr5{fCx?C|IQeLvRvYxvpsQW$D^B(_Q@Ny z`SR1jt$*H0@~&HR)evElaYg3O4$9O?Y?S+OAA>D*c`)5`H$Apz-G1SRF{=apJhj&0 zW|k@P+4V{l))t#J)0O*Dw91}9_&=IEu!SpQtdQVUYODk}C5zQ*AA?RBbRq3yrgR-A zGLg#=k`MJ=0T*7xBA0_5F64welqd5vi>IfZ&^OF}inu>)tO$8qS+5`SVyxzBkx8bz z^`2UcmFccj4U>~n?s5Z~*{m=vE{toW^eKaZnQD*okfFWd4eSFE%RnIkoF_ApbsXcG z+_dBdqmo2YVmP~+7IMLl>5S9G&x62 z%-i8NQJ~Y9B$DwNO}oPN#MuAsI2tBY0z4vo{^w@bp22U)+V>x{*@P=-cfUMZJgE(> z&_m&dbH$fi@A)xLXk!u=6rqRAvDFz7y>t4srI<+>Iq`U~*&5xQpcudY&4a98^2We$qrJI#{x#Sx$C^&NF3M2uMr3(| z+QG-%Fm+>DNNs5C>gDLC=yIL3=+|5>Cp5r$nuJle&EoF#^E8RqJ=l@qfiZ>^(Vyl? z3BBm#fX7>`ET{0awbp*maN`?mo6mMGI6d9Pnyb^m&97=gZ$GJ!{4nAVb{#OgZJeT{ za&?_-<fHpPF@#{DsqM-}ViF>N0S~N(%V2?ExQztBhDVWHos#)n z<&&BG+>~g;puU>Z-&P0!RHn~BK6WE3HTt9;W<7QGHti)vxv#pD-I%V5u2iX^6wfpi z#s--PP0!54J*0Z%*L)cwAuwwuw9XTH^zX8%lGK$2yt6J<_$W@8apo7u2KHCbWj}#z zfRlUhW-9%9x}XWN$7@d=)p{BkfFqF2(t(0jtx`@UpZ+SURXpsgpyl!?>6M^@_6UU< z9Lz(>tK;BP&(M$wQ_LX5otK$fi|x)BXs4m}X`o}RL5nA)(^v?C@LfKUZZOvrId}M^Ejd~kiyX}L)prH|-)(r(PkU0n;6h2}8bZ(z{?{ZvQZgCar zU0H0z9rk=JOuCQ#(UY?Nbz;NfikV9pbM5%lWxdaw)7}R+!d0jsI;7wd8rl75I!%LD zSdynd+<;#UtjF8HX7k)SUyW>)?vFF}ti5$moKLqdii8k61W0faNYLOCbZ`g+34tKN z-Q9I?f`~xjySqCK?rsBv&dlNWo&D{5_rB*=ol|wHR;~4{?tXgxGt+C;`*!zx zV1E_c9Z;wuh|ab}8e^%3wcxz=4&Ly7c+mclr>{~FgN>)c+?2i9Ue>&1Z|V2fb~Dpg zy~_PgW7~A)oPsm#Y2nj(j%RL=`X|)(xr~34#`D4G#dkTH`d0*~VIih7A|7 zuUdgvQF@X36RutZ{da;X{d7@z@P`UBj^VX;oNdjnb|(wczVgcg$cQA;H;b{?=x4@pY7Z~*U0(BO-akIdnS6U*ZN6bmWl2htoexMnw~_ix(S;j zi}x2U9~HAa>~y0&cfQ27Vz!l)VX;6VD;nKr_7lRyEYJ$JJ%Ph{TGJ8%s|$HSt4seY zeDSm5#S&{JD_@-^)r~t_!hJ(gv4Yr*pDEt&Ep=nR*$+-Ab=%cvII(c;{-G}UYrj%1 zIyPR4@)$kVZE>-+RNvnVuL3JL_g^2Wi&AH=d1WQGWT?04jwjp@sxG!5&veKG|9~r`b}=KG!w<8w2gwSh;7r=ES@}Fq9X+ zsmZ_k`!j@DnV#&};pNevRQ1Q1jXcinm|LnA73XPI*2jrKiN$+Eg1Q>NBk16SW64Ki zc2}RtL~Y+gKLE2iR>Yb_t9jhLS{R}wIHR4%ZDw(ot#)I6HwWsZ{ZGOjx@4}pVL&Da&=j(%0qjm*k^ z%B`6~eeUVv23I9fIk|$c-7n88$L`4Q_-rL#@NaO|V;cNezON8tMn!7u=zCelY^6kU z_cNLalcXOwUPqzs0j=xV7)$Z1y|nUi-#`0WNTz{Z;sz+Gh#PLr|GHDgd=ocazPDjB z4ziw#m`u6F*$dgiz&YV-cmqwyPs!d52fKt?VOM*xg<7CV3CL8NJ5B@ zeB6#lv=vy>MAc0IW;;JNdur=_@Dm;jsBeRHJlHhGtN)OF*0LBM4-5I_7_Omn-wmo= zmVBpTlewMkgmpl*rzId_*2a6wOs?yBjKk{Wq=#Gnu}0dDXh^@?IOYDOK@3}FB$T$T zAx=cN8=RfxBZ4tdUM|`b2Z_6XX*oa$=kREz{`0Kw-9@PHeq~H7t9poMz3gK#jWQ~B zO*iY^LAfJFjQ0%yYHFGfSP)8;c*u1!sjr%xfPL3%2DoWNR?<%YXzvWyL()_7cPewO zdcSvidmai&rWd&4y%+h-#2j&0AWPXYCwkMJ>T?s-!>F98X5JWxZ)mGH;S^;8+w(*5 z=2`HCEb=z$Zu+4@t9~tk{#JMqmHtZF?RFV=yZ-n?`u?;?r#-5#CGa5a=%b%z8@#pBixmxV%YAiJf-dRn;U)e<~mQ(Gs8A;(wVx^BruQ$vP zsUEE0bfA^cf{+3#U4>&Xkqx<+Aj6(56`6SOpy>+uS3{N*e+-0^VGEG`no_4d?&(4Z z&Hc)XvUO1@o|D)RffHQ&F%wqN_~V-pN&JbhFU#-COgURv<#p`d^y`W8UA0JV_$dB3 z^wgPyflCVXYoYVlJa>ovUXjxxc$ery_iHpG$MuPXUrM%}IUMX2W>u;}i%G9&k~4wY zsEkFPK1JTwXjLW(c}54))S0+%az9VJI%8{cyGQ!o^-Zo#p*oR$>(MYPs%+lglpEe@ zXqfuV=?ZMGYh}Obf{p9w^`&G5H0s~ICQI;* zG%yOuii;vQOH=w=67Sr&R9XpmilrSSM3@}x|5m^HoD&5#COa^mdItNu6*=URO=O~F z13N<`RS1n+ga}kBrmf}AKd;VsfmFG>cpkXPe2*egqB`i05RVxe=|kn8X2r=+R=GWYR5ejHx4A&c`gQ#+ zU~aZ3XNIh>v&TBRt;EGH$1+c%R&?+!mr=&~i|9M;$ zYRvsgQ$avy^dPtrywBZ(xvsZ!PETcPFlhPHp2IymWqXA?Wg7=3XD3hn`yO4I&b?EI zZR$b)gr(7Q`FGkGYMZm)42sT=Cay3>ZV;WyTnDWKoBx+{t&Gq_a!M3upc4TajH z>#WGY$ps!)K^t0Tn+{)NNL>T^a7Eq4z=^^Mm+CN_t}Bv9j}b zcSy2}sv*W+es8XulAJ~X7v>j0;lxlm%7qujh^-+cW1m2YVgbvYiRz(l^rSA~>sgy{ ztS&wPW->CuuMB^G<4BW$yq1c-M|+^?f1T#EBa&}Wb*GAi0>%bYyhfpyqfj(OCEU?{ z>ZB8@3SG~zOWe~<1;(e5ZK9_PvLL{+k0S+ip8ia%=T zKq;mGgMkX|cA|nc&kG4pg@^83o?0jNx>hM4VO3Oq;~W zq!gfTc0<2Ubj9kNOdnQfkHlU@q;`wGc6dJ~s5M4aDXL2W&+ueHh72EK{n_`^vx1Pi zcMUqe&)IP^!UdETNc~NMs}1#E278-3 zka(PHtjiUzGCyMcRgMfk3Rl1*d>Q0<5pL}qGHkzFs%2O0l$y;{W)X0{LlocPQvrM+ zYEC_J>;oHj?buIF?{NzKiD%d`%}JBg<`M?lxY|34&?c*Pwnb@=!!-83ua3*w zjlhqdOZ%kEJDv+BjlI_awqD;HoDHLitl52WNwNznOuq0+E46PfvNmK^-?VPFe5e~Z zEXaguiWM80J1nShS{l2ZM$p&AY2VY|b_3}l;-9WN~Y?&FXZ4<>Z zm@uM}a5$PvFFSrGu;*gxhvc%m zo8A^`im-TDUEOsLYwZHYD$MRqt-AO`i$$ox_9yYu-=B*sdWmu*IUL{@^kJ9%uJbpq z z++MwdA-2Sa9?v$_hWq6tCx1>SVeRppu9>CXgtl99(+EP8f1vXnL5w8x7Ou#`I>*kd z@Ba4VFP?4MNcXD^J9mB`jC!B;r$j}(kk^#pphg?(BJ-{?=vZln%gih#Z>L2bi!o32 z8TWWp$BDzP*^zvzn07o7PW@^_b|0Rocmh~~OFY3?M>I!F?FdI?6`gL@!i3?QV?`Jm z+@QDyg9YlNrW`J^GqU_D96tN`mi{{NlC$S8wd!>Q-{)Sq92!ye+P>Gj1Ez1wfp#6= zD9ro1843!yrsK_1(YQpG{A=G|&Hkj0mIEIx7qkEM7|>KumCSHf{Y#szP8n8&O`0HJ zb!BYI{@v+!7Ah@|AsXh-S7rs`rqqpu%OT}eP_L|8BD+aGg`KxW@f*i2RWxKG^i)Py zEvOTPq0Un#P3j&L4_3#;RF-2K)&lEPd2{43`gzTHH3w6w#)=fpKsm{^n)lYL#Wja6 z$I6$dHmN`Tq6(cAoB+p^sa4q5o6+v;ll$8DI@EWQ-q5oP30ykihhmz-tsS_)Hdo3j z^}CNBcFw6t!8lXh8Z^O{75y8XFC&hES-Pz&pzm%58k;hX-KrpicHn!7fK8UQS10pv z@$|o^K8ljfBu|4#db&TU=}<(LFUg1#ye!JJm3DGz#iwzD;Pw2KqUGTfL=C5M5pFtVQkh$x@wE6KS?}|%RB34tFApU58kVxqD&u2m2 zwNzpbLX2AIOha#DN+%l@6F+3V$IM41sE#&xE&4uelg>k;9+T4TcZg4IA)0WEJxk-$ z151tOCJ={Mo8cL&J--RR`?Sh!%}f6M;Y@LM%f+Z-l(!H1c))M>Wi2Qe#QBY^Uv-kV zpJ8s)qd13pcVM8u;;UdSeF^PMVpH1^$^B!Q2-;eup+U35STo?ceF2lT=#E9_Q(wbl zjs6ijtDbwlIv^iwN@--pD`b;4<{)vhbZ+{N&0IeKSV=XHMg3ZSHA3WMJDPF&U@%^AfN=13QZ-i;!$Ao+?I(`Gl<;l}DWz9n zO6KLXrL3x8>_foh~EEe)hC?*7om1wP2FKmiNw}URngrkD5@S%ISpe=GZzWFQi zdB#5K(n+ORN`r=HlMmBq(n0e8gg7FTD3e}#!fB@?BCtLQw;oMg!W&LubC0}!{nf}2 zsmW;6_KU=W1HEFMG6q-X_H$Ta(dLOS-s+h}eV>Tty`x9G_`SG4(iO_n7487@yZnB~ z9_!LWL4w5eCTjuo!SNqFg{~TE+&xX(Eo0t@?@Uqtgr`jyy%YupW)zRm4=NB~ZwKe* zbN=#`qliXT3!mthT)1)Gy7CfC_bm?m48~*-U$Ah&&0uJlqkjLAsHpv0chs|WBeT*S zxzo?hS$nyrjD}u>1&t!~E-GJfl$cTdFNMRE)zo*}80A_;IlT1QB=Z@KG-Y zc0{ASvqf@a2&yKtjGGyK_SVWH|5mAS4~D$)AeXt~td2w5uFnFU-U;fmF9dyMo7~qq zai|`wPs0+-QAT?J8tl=BNZ4iG=|ntGDig#?dkKeP zeY$N%Gnz~Adpz*$=Uh+$fjn!>(Eu^vHbhqA+hMv2BqQgpavMqB3N7 z6Gn2@e?Pl%7~>~^?|I4`ZV$Rej$`aD`<^^x%M=E`rZLYO$f~pYm5Fzl)VC4b?d;&u zM$G8gpWRH>av0`f1hw$1I!Ia7V8tcRJhrYXZ{r#I!(2EBVDNq=dGVVwz>L_Kh+R~E-P_0>NTB4u`3BF#Ql)2ia;x3A%)>iPpl*EJeT?uFJsARk>p#~0zw0F zP?q|m8gJU>)$6WbrJe(G!tagk(kVk^Z+5BTF&RQSJU_!23N6QtA>!HnZ)$W9nkXv+oJi7oP7p(UG zME0qB=;hq+A)zzXkm4@o?R$ZkKE#q!_wIVKbmR&wcmC9EX7_bUSs9FpWH_Y$xSFbp z96s`@JYsv}JEM@2a=m(qGwJRD>$e_MJbW|8u(v|KcrgBjVFUK9Wfz(A$Nm0^{+4U_ z&RF3UYB`)(M%l?#^u=H8!2E^S4_FItZiJaA(rMcUQXDwdslb0+&Tz`G0+;mhX0J;f z?9K-mi)D306!6VfzfUqej88urhOoh+H~HgU*G|vfk2=T;>mQQr<$P=od(P-ZwVM-R zjWC*WhdkeH<*1LdA)-ElWmmKn(kk-9vC|$ zeG9LN4rID;Vl}~U7@_fDSBMH0RpOv{pqJq}{z|@3@0odYQb9Koyd{6c(#mo+&y;Z- z1|MjKeX_;y$I6A%37d4>ISL_<&_*J(Vo${L0-<_8dy7rfi;jOemAw!8Hb~scoI=~3 zGocH6b3c!RyzA^n@Qv+l@rnjtQMa z*KD_2!Iv5iGWnuJC{u!071_PHAqEVzz3lRHJ6(j1I zTW&l$KoT8C`24z}oPmpX3leV6FktnPkFl_bk%_5n8GQF;o1L)zy}JA((*-yf0f)TY#XrGJ!yRM&r^|J!Z5QYK1LPf9_m??2Ihw4FaF9FBw!r2mZi zNBs+pc#ZK$UxtJ)%Z6&bOxQ-ze(M~xYMi#sCT#FHbryhVc_}i{O z?FtAkmBNWM#hYjLiGdlK>&jO0!@)kDlO5j;&Oc5#rch7&$vyVQqg_4!G-{T^(*Mk) zBeKaKjmjJi<&|IMLxDd!U#WS{&316W1x8qG=sx@7)NSM__MeLsT22*^g=hy$HLfXp z7kf}UX|;3~vsgFENA_2YyVCO#GTI-^(lWx$VDeFv0Kc6rB6#)Y1q)642ANuT%f6mu!=kw;?~NM{Er1e3EKGp)HOM`w8hH1HV;l zN6j%tz}s-oU|Pxt%-RMFi;<}LGwzspt>k73b^U+--u zZoOVU586B2Jf$ie>?guKE^(YEA?OQx?e7^-PL(_wut=Fc+PTc!h-MiIcb!WW0fxXVtiaQWAKM3^%pZMY$DyC9K=wa-_CvoB~@Z>dy|NP zh+v6q*!6z79f4*n{?VkN$bB&ceJ;1#&1E45u>|u!*{`6Ys!5iG>uLnaM{?vEho zu-tAxmxWlwJ{6oM}%_#$jW zr`;Z>WDNWUMPx&2RJe4C+gRtD^?MwI3~hZue48e0!q&yTr2}&wf^D(RqTaF>wiGkQ zD6Tffi$A|FeO419e-qgUcZ;&N>t34Uwq=nB1KBE<(I2VT+wDLv;C=!&&jb^EH$Mq# z#q0^)EA)G#O_Zp|)I&5C>Lp!ctp1i3ZVlGs^DSLRP^kPg`*V6E2@$>8>Sm!jgX&q( z;X8uyA>1QdWV^yeF%7(;b!}#=OUjR`9KgzBXd`Rp_-O+TP zKP69%=*;)AIA&6x`8@JXAU)au?@vM5 zDh!V%WXC#+-nKrJ0~v#sUl-*N?Qa|mM}HG;h3s(6frze`>Jbq~y5JFO{hp-Y3k}l*+GQKJ!a|(l($ZDNkXkvzVP)HK}yxwtDzv7>GfvNWJ zCIN@0F3U~)q-#%lCvXTO0f+ewz(xEdVu>Vyg;_(bES*)Y%;)gQtz}`JzWbZ)B)Jnh z6s^3%zgV#HwecS}W{4_5h^8smP4*;aDC0}WKQ1!?;7aTyU8V ziKv!=S=nnk>r!e*ivUSpkvBu5YRvjfD!YIgp($wDlw}n3jLjqKkJ7py)-p%H=&heI z_p+l2HdZU)K_L166cRa^H#48#%>PoqPi%{8V*`|3WDzOdBryo++|XN4B$1j-Pf= zB_gNmeTR7bop4w^3B6*(d-*uE_u&)B(i8YRPESslK2fj09@9@(R=M|zG3k?k(>`Rx ziqR^{Spvp33XSzXa)Ki0SL6;_u)}YpFI$J;-=;lB-Ven66>%)jD~bHgeyBF{b`Hif z21)+&Syz4dJu$By%bG^IhUxY!k>`h=Lb~Z~jm38IXP93v>rMS?uEJgDsE$(c$^Y;# zA$dyyB>QLWyFk2Y57nd)5L^`ElKuR5wh*EH6I<-HZjAKu;OvhxKG4Z z6WONli8OUsN?Gr>V}f45vdo4{LndPLW31_>&vI!JSG_JMP-%xhV~AL3{I#s8YpMi4 zQg|$3*wSdLU=O5B=bQIyOmuBulD65o?P~gDQ~u%sBYH*4=i8j0_FQbW`(Y8FyeldC zughY$z_Cr&mP1<7t@O9E{tu@N*K-5c9|5pH$%_+yCB=o$ucV7{KGXZ0T%20gtc?}H zdF{IpE$BjzJNE^XauOfsJ!|zpw!}JGiYW!s@#ti3tl)~8HGCxc)oa2`N@^e&Hc6?( zR`I7>0`PJib6!vLg8^0;9_2;?hj;fKx=+jW`=Czw>)sO76Wl?7Z+-PFNS9(>ukt66MlU`&w@l=a5!^~NmqC&xAafPe7K z$?=sj`^`V{FZ}OP!p-e|a$~Z8!J=Emh4L)6u1`j-z6$Em?wyb)Zx_Sk;Vx|3PrwxS zKzFw~0RQA?>T-J5*U22P$Yr^|cCdUc`jH<&HlKt1g#F0vXNnw>eR#m&b8j5g9`hL- zf%!w|*v#QbxC0UgW1p1Z%Z0uT2JFs0;_bxSI#~8woNMGYXDDhoH6f4r+lC!Pae}FL1 zIBqK~eRV8T%Upx;?y_~elHn7Br$8gRMgk6T+QwZH=IXHS$!Byngd>A{UNt*}&$!#-O!fUc0R!go44aFfkBv9fa#cDFRX@2vD%3`&Q6K7glp2>?+{{NJt*3jl z<=a5|#}R4uL&(eRYfySYKiVg7!SG;fBd+i9?xx~(&gA_U`=UKJH~S)cCBB=bpe-uD zwp+ED;}lF#Q5e^aE;E?~B^%i;fm`l(c9dno3$A-zosU z8zMLY_kPmnQ;cf)87AoCP$MEZ7N8}XYu2nkI?HghY4Awx4*WXYJ&}WQ7LIX1#HC!mz4Wf>4mAz-8|7&n2s3! zi}41P*e`QUZvLr1uU4L;p(zmsXqwY}e0<0wP`|@^2e-!f@q0u`qlO>%XAhaUWR*75R&5iIXLE$T{2ThZz#@a~f>f}nXg$Z;*qSn991%&YF0eyZx4tH?+3W?=Sjwc*=?5mNwB{+2~G@pZM)^!Fnu764R z7B%i$VeC;Ra?af>^+UKqSDw2n=>Vk)cR$MLhNAWIbPe^Fd`o9XP@a;H$459 zBb@SIeojSJl$@k@QfBjF8nI@}Kweqb1oO1wt1l&r6@2L0XM$O<7V}PCFnOHv#n`0< z2Al@%T~ZQ!l5SJ1^r&}qfe?Sbv87z$!Vz^>f{DU3Tf$2}2YJ z1R)ahQ-`jHkF9JlT?<9~Ep0t8Uz42j_th&{GPTEh6Zup99xfX%kp=K4l7ru4$dni0 zRnMqAV=8>>FrvjuReaC{1EG@leYk_y{WzlI1;gFefh<_^l%WF)iFn2jH%C>%u6`h@ zP^;vj37u$jyJNzQ|a4iw{I}?Ze$cg?8 zBsSm8o%>_4bz>-lEv(v-f)-V$;vm<%BO^t0Mwsy3m-iLF^tf$AiRwgT*Om;`$KJ0@ zp;5txQZbB15*wu0KUFLSRA`jqUPqjK9W5n{$zJ~@a4x|~PA6roCd%EaLP$X#=Ksfy z$JXC&!$yZ70-5|A{FOt5KS{mYaA1B)2)NXVKy0&bc+-4McrXBiy?6K9W5akhv8Nvv z>d!WA#p^~yyRGrl1}glqlNRlicb<7dcu5bXSK12i*N0UWe|$Kfk?LYi@oU;6=ZDee zUrrmy_kOdZd8WE5!(to3?H$wc%_AwNdbgs25Xy_zlql$WB(HmOlW&t7s134fw%dAj z-K6`4(9h*j$pVRTBTN@mM=-?uj?3a9ONWT|f7^*2=H-0iwCE5$O)&mHoxeKMCrq5} zc%a%_%8Pr|vI7RS{h+VeISX0rP7_%O*Bced&xQ)d5HUKx@SnbZk_xdL{z!18gnsZb zlUji(oagJw(|06pb7Mq-v>|@~ma%r^&(H7D7=J0gA21^~AtJ2e__KI0%1u=j{&G<& z+=f<~Z<;pkXPn~90KJn5w@4NLpIgf*BDty*`bDXDn;dz*xDROuzZHWA>4!{+EURi9 zk_2uXQm>>kZSs}*2nz&e2I%BO5K4P%jK6`xUmqXyisSdfQofBw5A0GKJM$}uDsFff z&}#5a7Uxkdse;7oJsG*pGxPvNY%EYYp|w?lnLGD%neTgl62WOhSMC`!$@Si%_C-+f zv^^@YMn6A1BflN`*l~5J32uknxi5h+e=CYA{85@tn-)IcX%$rbf1vW8vAr~Q5nAXK zOn6jCnSC7E@^8ovZ5ie{;~H&NuxERMSBL*EC}E8dHvP9b`u6di5)gcFW;vxSn}9k*|p zym4Rj0M^_+=r?SD!nyA0z_3Reoa}5@Yl2UIvja<`2zi3WdK~rv3w3YiEN9WP2(cYX zRuep%1{wU?PGa>k^xF+*gs=E5lyc^*5bD_iuHgQ&ns@gClEAfPU z_7YHF+ALGsLSVI6rows_^rOyNZTR8;tZNOU)}~&CmFC%hH`Z!6(x+iQd1pgIHfM)J zEi5e6t;HD^G9RoETeYG0*k4Q3GkjB3ZO$Zs`2*iR; znulrMl1K49X^6lF*0CvrGsWvyl(ez#r&FSrB_MM^=`%~LvD+PYrqnH|G#`JUzK%&V z&bx-lw+p8dFHUJ+)HS~hY<>}V!WO*F7HIJQ*CqX|6UH38{{6ojc_ZKkQZxYdNvsXT zj+&v(_xyNUyyI&?Ey-=`n@lux!8OFewu=v(i0u5|!a;o-j+=}?JSm!nSX=`)TV%0y ztE`TghCxg-!z63@MkAMGOxTg?kq*9-Atnf^3TAtgIKi>UF zvqvaQX!VM9)wo_gZMbTsiEGhnd8~HwAECgo=Y1MI-ebu`JCpAUI@PzB7VZq^yOT=$kv1-#@^Zr&6ByR}{g!U! zxP7`CxCf#_!sB4Iz>82|U#IA`=u-ak&+zh7S%12S!@Hm>3hIQI6bEpUy%78{Z?KC@ zyt|J1`}B)Pf|dwdZ-YsAy<0!p{jzWZP}2@;y!Ra<#IU9Hof{d$O%6$3^WP&wt_78r zuKwEzv5Adar-p_{POtl{P3g$;Ar1bwcc_Ei9OG~mi>o9%hmW%LLBV7FUFd`{F9_BA zS3dbK0amV?&7tPY2BP0Z{fhk7*FZ%y?%zcV24R@(c)uE=V;2A~{=eqy{r-L^hf~ki zB!eN=FO*0xB_2HVLq>UEBAI~QX38<}{%6cDZlv!u9>(5){djrewvFcz)*`;Y(+qTc zQEuEmbS2}asPiXP%)%@$@a1fw^zdQa-lsi^`7w#dqOq*U5-yG=tcx=jKN0U$_8Q$& zSYgu4p_ORA4`1mIk>foyW(P5G9xX?9=g-0VNAF)Y7`-Q!%euvs#$v=Z8XavvhP^2H zdK<9G+;#l|PT_N&V6B@cpD8aF)rEDv&^(z}WYLEmTVZ;ht2S7B-(iYE$7OPYk@qrD zF^+m~bjiV}b~B}@+*n=X<9DsEo|v^=gzT>e86#v1cI?vUi3&cGU^xGr*0&B>?h$9q z3ENU93(v1FZs@5!@5b0wG?>+gcu5=FYK53sH9;AkTO{=R?hj+?&F&qdn#U6QNEj#F zu-4e=*VIhhTUWdG$WyxF;+_NpzN3M>k2PDauBtp5nUgzJLfYWuoo3`d}A`hOg zPP2}C8@4GFG#Y;gHpMVv&#F8yK3RQ02XyBXIr zs#f!(v4z)}HT5F-dgin`u*0r|hjgK`8o@>R;QybQaHj@X>hE``DYlPEs zny)=d*}G_iQ>Bv0rC3STqWXQMGEljyN1->7qc%LEHA)p49wCA^&LFYN{mz@D>~%h4 zwZ?`OfM()gmYWQ!<(E3R0HA6Wh_Zg2ZANzZy=4h1WiG)NV!4W0dMDMJSWuf1!D?tt zMB+_Wzq_#`01f|jr!b%wQXax|c>Mh~wf-RUZWanqHnG9iH+G19&7@F#mDDwlIDb*K z=CSoQ;syy6n4!jAT=2Vd_SLF z=zGqdHEguK#r+|YJvvRpyMTgSrUs#KNo%A;(Ri478SK8?w@`|^j;?oeAP*Pu{w^4mOP zPJ91C=BBS2sKDw%_Bw&r$pQ=ehh%dbDCR{~PRLjgAnfe%`rz@JB6BM-SU)L~6(liD z-yTJO-@#h*j1xXQeTLMPOhn|pfx{{MA|^jhqKo!g_$?zu?U3!;u$wi_D0(E59rAt) zhID*DHo$@ZbO90YfBDKl|ZhE%?vbqmc#4FBMs*wZY+;r;9Dh9qsH=#?xw&seOKe9K$?$jkt;e_G=6t?CFnBwPJ6#sLIDQ zIra3{w{LJJN9x-h&igIx^Cfuptl};6Ez2x<8XFnDT`yI)|0*JzkS&AX@hj4$`aY;8 zBqU|RwZ5gkNKg(QGM69$6J18mTI@pfJ+8MXH=N3DJtKyQ%YW0WCGY=HzB;HC0zQ?o zsP`iv(XDQs;=TTCFA zbkHkA0KeErCQuyr=f(no!uO)oaGF?z+-pRY5YPjv6pL`2*XrALv0e7QnD;(yie2WT z5g_aJQ-9(4eRw-jewB}GagL3yB{ukAv%G(xd{$v@jJ7^nI=`b9dZ6(^@e~s{FJht% z30rT|z`u8HPD}m!u^F0;<5Ozwv|N|S)95bD(zR)kRTcl~E^)S04?b*bjq$s7b;xsW zhI{uTr!!EYrz^P1`ps`^-nikf^HvPQUr9P?-EY}vL^q1}QkmK9h3r9JQgyOfC$Wpm zf!gR;daZ`qEEz2=w!SCDT=esVdc(<&t``>AFBbA?>DzJl^xHO==ncHFu|3F$Dnu|a zTB5KPG6T`{`kvV+i=(m|qt$`4{F69QA*x?HhLinIm>Abh6*^$V#OGY}2F=egT4J#l z%EVF6o(5Wf?Ra<^K~608=z&DiAd#_1z~{X!=Oq+`X+m6 zugihyr+_sx&!&gRYNbyg`zvTPVALix732(shWNFOgAeZRBt-T1_a3@`@%u^?CgZ65(gs zm+av{c)H!%Dik&O!%1yZjnXP6HuNxt;eFYo5zl~S+;I2%3*p>=%@Aq#rfuAU^=m92+Im(0XQ-m~*N-x+&gMdicAUw@u_ z*H6;OehDdVV=I^VUlGZkh(|}4Y){1b`5oOSVp;ic^Ak~5Kgs=x*xbf;_C$2|)m?px zc=i+#CuCvuDWV+H>IXp<>`f4zT=!aLf2o9I2%`=;2U1wdVYc5xAp&rUshCK&#pf8I!r$$_8VuC;Qf%)7<9Lxm3oupp^m?(= zR|oY3>AUAC_B8yX8{BIEhtzKKHs5uguyLP<4oiH{9V}p)5&%6x2MhS@LNJw_??>v# zGj^|1>_I}K2Tug|)M@iJc+vD@zb9f#xL?52Ez{ZBFAe$_S@!sP&$DQ}4SM@W7`Y{g zbj(0Di~)7_9=-DgU|WJ91i~?6oS?-aL;3xX8ToaQMcYnF<>(*3I0<{A!m1SYMX3^- z9Q8<1-+Ybos(o8pt#%!rHM6GcBC9sVzr$urrmklD(|;K*O}XJ!;r_mAe8?p3LnP$s zRMcLOK3fWMIOY0@fs}3z)h|y$7N=YzZATvqy}Q}xhmkwD5Bg=v$ef+v@%JO5qvZ@n z@W$1Pr;OS$|A0h@Y0S5Kb&Kw#)?4>JD#gwK=K70Z4@V_86rz5nV#)&H- zOKH3x-hz>Hv8}$prh!NOtAK%n_NUotwy)NnCM0mWw||-gQvhE76bt>$Whtho_nt&z z?!#Poi|@L0n=7q7jYph)#sFmR8daU)5*G00?#CAJp(DiSNPM-8<$4XN?b{F8n?+Y& zyMP5iZx4l50S?q__c8Tn>HWz|@w)IdkL@3U>u2rz4v+0I*mp_hE-_LM2ml3AlyuNB zC}Fby(UkDv1bt0N=L~|`=H9ScPvCRr&}vU}ySm}E1~_l+zJB_td50bOK6KpR&p*Jz z&oatxPMp_-bk87rZSGC0_2a&0F0J<9vyA?YY9#V@4fuHW2I>6}jkr1Yemt9BzW(vl zvNtb^xS>Hl#3JGSNC5TiMf+1Szg^qIJcXa>Lte%(PlImPx&fl+UC6e0I?(4?HDV-t z!y1DP-1Me--f{udu$Gjw%DwY+;s#lD;|rV2M64G+hx?&{u&3J{n#U6kz^5qehYPXW zqWIe_=7%3RzGs1$=j~N<0}E-y0dI_$E!z1Z;6oVA-O}FB^~%RjdV-}xIv~G7L(%WS z`aft!+F;P5#>+=f`xLQuhbrY?;QZ(8Me~rS+gW)!3NgCW-2I9)+7;m>`Fz-S%iD!f z@Xa3}t0myJ7U*bp&)G6<^b4RfJ$%{)k(h{=Nv{Kq6*)3E0hW0`)O*b}8a~y4e)c|Mx&nXU#p5u=wHx@0|P*rt)}7u|AOn3ZOiEMUgu-IMt0vP`Tm*_40<+mBS~qa(wW$S;g~ z6cmuVSnBYPZ5{nrgJv0t3ZjUdtZq?3pre)%6o05(b9ng%7LW4E7@a2Bmg2Fmb|37D z&d$p2NcBq7t!>BO_pBEsOC>U-^rLvwiLSltWELGuUI9?V84z5<5YV4)L#Cjq<(=apZy#a{VG57 z&u0H`b<=Y|c>D+VHjh=`M?gSA*R@c`UcU47HFwngO#P%3`3INz{3pEa&DS_({`eZF zG2ATcCMZ7xtHUL|vOGU)z(3j){MCGzmVhF3dIiT{V^y zn7?nDGUKj%(Yy%%p-DRbLyNh6(MpCVPaPvYHu<+-2F$k`%-$DA4Lkm8rpyr^PZMA9 z7j1@5s}sh*RiJ_ZB%KA_(ub3_S9Z6DD zNQl=<6W)aIuSfmra6R8Fo#JHC9l5&%1H9>*_HhL(dv_1(4zU4YL%)9$UKydCeF zwyw>4{`g0a^dJ7mjoVi)GuHyk8~RtTxg$PZ6W@N{Ydd;&9D5RyJuS9ta2{R;6piPB?yxzD5e`+nuW{=-RP>2>yk*WXg`6K9nkH_^ti zcL!Vn=`fAIs=BJJ6Md(nM}~7X>-M^;-cL<~R-vabxhKk7NoM|f)%iW@&A?2rIdkZc z2k5N`)t!I3uejstbp2XiLiltxBtiWPe|28oy+$oGyP)K=m)SsX3;yrI4mQ9I87}SU zwEps~bZ%n&w@2&8N0A_)WydJ#kD)$AL|RDx`IcKeNBJ0)0HNy>iMzAvk35RXLEKJ$nUF#5MXc!q#|s`hv2D6h0O8=9>p6KtjHWvh zsrku`3|=RzOn}4IFqrCDyyNGuZa{0bK9&SqVdvA?bq_oOPi^s`IE|OFa;G0jz%x65 zCx-b&B&Dq_R@T%ek(`3FPx*>=Ei6Rflfe|5qyAdR@%LLVY9I-i5ZFIUFgM^5W3A$k zz<&?-O4?8lo&ulp$NAo!9XSCj_cL7GW%{sY`|xAl&{vL#FI!#DqxGWk!QMC zrR9r8v!{;aQWKZ^qQ&Pdimcc*T7G@ec>f*!UAl0p<$t2X2-oFN|2z7-bkP*Yf1=IN zA-;}yS~L|pXEAPtXx);@wJuL+Ul#W>K`Q&Scnaz+K$-Y2_zmPPY;Qi~Zm#|DceRrKPlEVH=&eyJUO?1(Zj4v9Q_j|6gJ@WP7l$%(-GXPCMQmaq`}Xh=1%4D#PtccseA zLtb?4aw_`S{T)~%rC>}F4s<4Rg}Tvms!q#g6%1jfbSQEmaz!#w#me+U+8=lF&F#=Z zkF(aklr57Y*eF#_@?oYxLbW*z?0&j0RhKd}d9n;ymQk z|HAV80|UsY+ya}UxCD6)UY&b;;SmJH4u_Q{Rs6iCianib>T6@#o-WI|HSRQyTVs09 zRj}}uPL{!B0bIA1tX9reO(#<@$<(;y)vV*tfA;S%a^QwLzGU+v;|b$1W4~VVq#432 z#?pwPm^?YcEI~#PeNe&hP?)VeiB9AZm{jg)^70`ulAX8Iaj!FK?bC;3f1GGD#l{F_ z)8I;=|Il#~s~xa%=BXWsaOSBWsI(SA|FLc**NHarG62CYdzpZwmaA|w%9p+DKqJdl zgId9RW1PhLwz@`n1_!sC#D)j2oWw>4@La^kwtRj9L(^&(0)x|H|7R2Xru3riO@^1O zF3a*iEodaStoNc{draOk{fwxM_h`ksn1U`m|EnXp{LARx4I2M{0{&;iYD51R|8{_2SUtsyOuEvenFiufeW4p zf88OXJb@nz?Y_W9tMdE>-*@dDL*;qd%g;~)pM=PO7OTvV8j|j;fsu0TeZ3Z|!BZXz z1pfC~{jYPU7aCqpKy2$(v)33t(%BN9OzbrK8O8e%09SCp_#k#Br0iS(5(QkI`QU<^ z;)0pteilFqka~{Pm5$WWh(5&WYlx%oj;j9tE~OB;tDhwm>V7d#gf+BE#a}_RXjZT^ z$mAeq7O$Tbqn{Qd9rFRA$9gyV;cju*JW{tTdJocfh)q!KJi+VQ#p_yKb-WO}!U?bY zKJ35{7>CgnM()uie(`p<0?L&Ti3~=9Mz4wqL%+iuq>)w{HI75eBjp!yJ2No~-Gpj9 z71ATOT5NE{`waWH74sxaF!Fo0hN1`lg?CTW`k-AXwo^dgb3{!2%Lk|T8QxRUcYy-^ zHxt!Z14aVV)+h92=rubsWDAryQt+o7p7^w9*2=Ziz<(xYf3Uw`FVA3Z;7tTb%FGmQ zjJ#?O7xS9js8|&Pw8?YOJIs|eTA?D3l?FkFPPpWX;U@9We!%Q6uH_;_y{#ziC zzKV#Y>Z8yF>hOLI{`P=UP+BTGM0Hr-CI~~eTR^GaaS`;DyFP3$Ug0@&cYeb*k5J85 zHI=r|idx&M#ASZNmt}@{94h$DQ9jIP#xNSRxHhV))Qy2YjtAJtdOVzGeo`?q`h44u z87m^iX6pVqiITe~7pu^aN&1Ydeo>Wa47G22%jT8Ra2mr*S9)#b-2n||pE2DLf1KxBsH`rZl zXnM0XJ8^7AVbB9kUX6l&#C4h)iq&alUGUbC(;< zo=aFeX{| zQZM3n(MXvHy+X|G#D@*6%B(y|0UKro6TQ|YiBEp-KBggmcZ~l?Hem*pf&iz}Q;=ro z#4SW}^SC|7de8VahM(8!HMq+`;7A9Tix!SmQGBKKw|;T7PCauBQMuxtI8~DK^fqtVu_Mt2s_~mX+T*7e}u} zKw|rtc3_~BR(tKFZ(gZo8fHuC7EKTFSnO#iHZU82ddGj-1H1-1)9@)E%4m=@+Xz1v z`5Oil^k96rW?c6k$BpVOw=4L>mz-P`qs)8^4{|Zb0j@60zoOIL7^N?bdYO9g{SXd>-Zaek1rJm&;*Ir zmal3=`~F+Esw0TX9rgKWu-T`nlSYNEx@UdUe9-nzp}S@S1zR`u)$%xT)I8Q=E?Po` znTl>043^Mv>cKWu-zDVF4Zj=r!`b>`-xdDz&DR)t?)g`k+a9m#A2zY7wWQr!M}mDf z0`WA<%tJ&=TxGg}i{BDrXM9jrZEyITg6rbO(mP(1oa`;n1D z$bj!$q(|eWpSI^s+IsyH`QYwp;KzFHdm({1chmZzqFu8}=)C80x)I&nDDs6ySFl4| zgPnUk_6Z~G$Wpb5CoRpG*gCvM5Jynhj>goI&UC7aQpiRpMIQ`P!Wh_lt88!KRzor{ z6~5XmBvl1a%YG7=oa_ZfIdsWzvnSe9sg?iq>ry$B9Wg*C9~iotbb2X&i18c&6J zhFYd&78cxNO(y`Ci9EEm2US2y#5ZoRUgf3_TpYYh5H**O)SjeBtoQO6mCbxL7{(ck zrcrCcHRX3`z@&B`POKnCcTeEs0RE#kIU{@Ha->OH9Q*up{dcNblD+zFFq!Rl=vu@# z?n8R*sa!ksAz81i&Vr?wf6xX906t}hNdDs@a1DOT_#$?Rj90u=QEBWhflDWy;s>b$ z(zO}W_^4A`ey@Bjm4RGenuk+SV5XOvaUbBql2R&ND}sjmcD)>fpxXj%l>_p^yte>S ze7wIi5xrtV;>e%Rs0Sy24DtHKBBeg7L_$7evp~j?Ow=xQTm;+1=$-tQ(oKU z<(ubdKlI<6jdkmC6~U92!EfPS!M9W_L-ik<82UkVqW%>v>k2c&^-`d2Nxr$cxT{ng ziC0hUw$YX3IxOx^+4&g_Qv(_`_;-cx$5ce>Bey-=MM3(Vk-FtqIM>W^aEuU1MR0aP z=#+^GC6y9bJ%oE|tF!%Y)7qhMW#PaPkblY*y2Ad%YWk?(wJDD6G*&lRSxAQ_o7PS8 zG(`{f$`C&xH#X*3`*WZvn0((GjxoJw1HUr5Kpa8}ZSNvC;Y5y?335s~iQq{rAg7I4 z4izr3VaqbR5|PuOU?eCLCgo)KoDN`;+qIn0$dYU9V#!`=kE0e^q2A)Xs0jg|pmB<$ zK0^D;SZ{HlGVhA%D6Sq&|5X0SR{UTv55Onm!!pI@Xv&0h}JJ~ z7_`&ftKScS86)H)0v3J<5XTW{W6 zZ~{8Aigtu&IW@s7o2DBwl?zGzQgM+9dnj zOrlz8jTyS3)bE6=CNn+=Sj3NE+N~u^ymz1&4G^m>dqR2gUXykWOx?tznzQe#!dk{LYRV|fM;ueS%Cd|l{G`KTb*oV;Hg&Rx_JK1gKd zA2mO7Z&xaV)lbw?Agi{aF*VIoP~Saqxc?!oj%`DP@f^2XP@Lje-eZ>USKs+bXY6~p zQ(zm`TJD+BUIt<4%>e}d(I9MQvi#Ez8S7`#u6}&QZx-c0jySO?p>H+Hw#vp8sdK^J zXzTqh`cKs2zvx9B?d_K^@0*liqXsdw1JtG{*j9X|?tKsSc3U$GUb^ez`@ zd5DM+V?{7qF&~6#Lrhu?rOT`CiN`_(71w_;yp1(&fy*$4-CO)3ReDsHu(v9LMVGEA z+ow;sK~O8&RU?WuhDdn<7M@d(Z?OMcIPgL$V@A;{7^tcgv0*345Y=f5 zsFMAu3j9}Q@j3r0PPLy#;2W61gD_)IBH4{#(J`FI_oD7ROEF|E;&ugc&j!!aD0QAN z!R7FWI_q*g2{P5}`u zpU};)W(?NpH1y18A+T4%Q*h(+&KYUT&s(*kUnS{(rzF7SF$*Q=p6Qu;Q>j24alz`l zmaJ4ftUxq^p44tM&$$%Llx6+Z3)>5+8M@v(LpT%!QiVgxVT-2B2;s-hoJi$9adQgo zz3YDDX`;T@%dg@b8*dR$SilZbc+>z6K3sVTU;0%YVv7`ligYQ`s*k0PK`-3Me;NKl zikf+<^iJyaYK#1$6D@#+syqZ|WC>_Gp@B|BO0@*w<$=*Yh<*2Vr`?4AskZ1Lfbp<9`4u>Es9c!qfh#LA z0#9hggF$OWIqzeni+$OTk*q5*ufbO0zrJTrU*cOBWks=>4v}d4NDi2U@sL7fU;qxj zN^h^58PX%ognXkCI2i~U)#+2M$_aAuRIdu}veW$7@&zzN-T>UVa49>=`+ijO4fP7n zd&a6dqB94==*Xyco;t0i}Eo7$T^6vRmeEzKc9BqN7B zYh;GXw4FQIr$+E8akk>0vj#0<>E&5!6?o^1lX#KB!$(Z$Vn}(i|8%D0tDI>$oy{># zGG|Y(NIZ7LBaKZNxL9^Z)f1f6>ycYec*8K-u@^K~Las@m;$$IJhAJA%KvRfgHbrOW zVY0q;&9v$;Q9$@LaY=_U8|hkc2Q`C5bq2+kwVFunl%ZJ4-bCC;Gb9wxy3Fa^uGk@pcu=RQ9(2+@&^4p)Y2Nn#~UZ*$w=}KKfTm6@2uD- zfh4srN3}u?|5(v3*vl||{J`;q2)b_7lLe%=y4iXt8jBxz)$s#9X@fOcntn8$7H4Jz zp{??JCDl(sYH$x33*viLJ)SRxheJW^go*()o^aY_c&E4NscnW*USi{K@(8PIs$w)0 zJSRX;I8_`flQ)@W(z&n+L1sJa)sgOVeGwFgaI^NmO&(q*U7iXlzGPGJE^XcwDi7roB?&=jsLyn1BDk_(Qq<1wy86jXp1Kpbl+&$3;( zcgHu~DacG4VL+WPsdWNHH?Gr_CRgR*J3nLQd9Rj*ukbWnx1*J{E27TmPkhXfexqMQU}kWa8FB%-}h{&UY}H5cq`3YK{Zhrf_o zhzv5y(wDhLxFikq9xfL2L4-0E#ql#@JjBc*;Omdu#}flC)UFBEM+D;>Q}D44^9yFF zQ`w~Jd(jhB!nruno}fmGVeW_S5s<5Ub zn6=G3y3odo>ID!9tp>BfNi7HBqI0ycqQC88FAIIN z0EVgn~n`?Wd7R z8@Y233eJdDN?0xASdO_6o>EJyrD`%roYM6eRr;*o-V+=vIytg3onHWbMDi({Qncap zTojRkz@J6=M1}1riY5Y)T4Rh&T~{0nRQ>S{_`4_kx+3^*Bly`~ca&T(*xJN%P%U&K zMv{){!KJn>0b>J$FU6yQS<(Xv;PDPjxn9SAjdKen>e!M(%CLN-swu-^CYTk-+m%U& znKY8^M;@yxz$Ot0zFiRB(aW;AWfi&|>M#fGO{5b4mefqOqOab(WKdnQ!ZzLTCLFJO zNSt3#O(YqxpDmE#>Kr1d-hH4Nut*l~?C1Hq?VPN#ILzyn4mP%Ak~RaKt6ATXjz#M5 zVQHw{8Sw5Vy6+z~RKgzGaoDA)HAW;*4AIY5gRPZco9`lMhDIIAvZ6htt1Ntc1C1T+ zLh4GKB^&T1@SF8uH&4qnWP>~) zt+8AjIz+3I?vR&U z=_y+&#d`==a2%t&)ncsaAweuyutk&20#AuS!C#(!(d>IV$qa`KI z@dYQrq`5ZIRW;H}c39Z!4R5pUzu04lIqbwy=PLUDxUvKUs~q>$W(CVwSvmZ}yM1o> zegD0s3cs(yIg~4Fky$FzRC$qiB9y>qrQa4G`;^j?O6(rbAn{8xGI|dYBjlMpCL7%m zW2GpkpYwvD3&Pl|h>~9lgI^t(PytHBC8qEyM7+e}|7e*!I7K2Vv{POKoFD^Ql8xH? zA#rnU4WAR$2Efp6iY9ZNbk1VAu;{`fvx=D~q;m^9sI2*!cu{1qTo^)uj*3*D!U>rT zLMj(2n{&h*qRmUb77SQu!j(jBW~0FwVOP0yG;<%B zlzi!`r)j;4y|!8xf%Jk|xM45UAag18VV0EZ1Y?R$2~C%Mm<3IWEN)pJ(DXRWl<34N zG!9<)>;!Z_yr~<4-#0kX1t$Jtsa3`-h_;D2eJ}rr#VOMS^903rErUurgyq{*%s7r9tvCSYu`!K|<7GS3!vd&fQJ%GKjB|BPJwaH{l1_D9iyhKBT!b`_p#UOHAb4e6=BF~o~OcnRJ zVh$IwoyzwUsQA zPb~2ec7txt{+Jw><)luuQxn!ssGWoCH2v zuQ%xnADz!S359HCID; zm8@7UK5BV9ed3|-h1=@@Yu9dDhUZ)TUNJCXl1_17!BS|d5LX(r@PAq*8&l|NSgV#tX+H)jhDX;zW<;ocQ zz2Web5JEHyx|+o9EI$U1)7i%p=i$c?(n&RJp757$Q0OeUzgm@J0w(I2R2ng>GRI>9rY#1jd|n%2ngaXoL($31I4b z7=(uAB{+4pIR?2Yv3)@?To7-A!GaHTvpEL$j`_)n=#UtcXsXI?U(mlfo^KS;800Fp z})9;O%{OQ!uiynJct~G=qo@Cp7heJ`BOhBbtCzMWHTU{#~|^Df3WyhsrK!T&2_< zcdDqbHZJy7-vnj7K@-cu;pTE;^EbWQmUW$lJ?v?3#7Yc?XK73#lSNi3{Q#3srjqyI zCxu(U6_T}QsPOMfi@q`=x`@ZiGAWkV&?Fp_6Adj$G6>!HSX%yWv{xI)KB+@|c`8C6 zm5qlkVJ=0hA3`rvm$UX;K|sk{m|GUHOA#n)$tXb?BuejQs<^p_X$kpYtgw))>S&ACXC3}Y#qTBh^L~e=OaPSDSY%k0xUozv z@KhvN>!_~j5V=FWv=0f3e2z6`aM)vASj@PwfT20Yj;AIJTLPoP6vW1!2ri*bue1JK z*~G{(xcynYv8?kaOuWSUi@g|tM1KX4yP*5HAavfu|(L&2XQ&ImRI-QKW zzkFXxOIVZ8R&LNQN2wqfH8WYyd*BNw(T)KaSv~JpIVvq$u)B&vgaY5PIH9fs1FHie zga}|g*G&a^;pF^K|tqKjzw3Bq}L_Cl^5nKh1(e~+^VpZD{O}zpU}_Yt30ueUJXx%dV~A4aP=vn^gwm zrIJvD5Pk&}xiE4kCm8Le(tFya9^yg!2QyjzvtCL;+Wcn*vQCJsno;nl;s(c={BEzv zGQGT3?5v&JG9s^ohc(CD1M8;K;R(-GT*?ah8FKuu2PV>0H~`#{FC&MH%SQ>_qW*C! z>wEL})Qka2go_d_Qh9#P0AhRHm9aQsDh5T4h@465CWb*0m+Hu44bm>8Y zUr1=C8KAZmbESZQy`@h`unZLV%<5D;aSXfjT!cB5@^ll(&xtp z3;nw5@|kNe&<|SNsAWp^DV8OlPaw21PU9BwU|2`mrLaZ9@BM}ldvG6AlF*7DZ+={> z2u5JBWCdlL0MaOj%?AqI?sswsMz-fh{xY$^l?ug&8ns+}c@;J@9sv6({Edh071XjW zQ4J_!*~jwGoD`~8au9g%xTV@BSYD8gaMjA_?q}Zt1 z?T%V)W#$cs#;Od34@KNY;ieU??uMT)VrvNxaS5TY<#c3@R+N}-d1YU*B*b-XYP=?dT@YrmmAUM* z1rw@NpdANbGPy(!dWup0>z$ut zF^qQsH|E*6fv2ga>SQt_V$JLVxQPc!6+{71#zcmtYZEar_u;fo(l3e+$plpxP3&;z zmB^2j|7yqkcqO(lVGL0;|Er)?*QK~6Jq}IAaTZioteD@4GxQ3em0HV{_N!tef@2-$ zVAl%0*EG9hQ?2T63}hkom5AY>2U5=l<7gX2;#w9s&hf$c%ePc|;k`luWlAu}y~yv{ zyl@iaN=#E!>AT6``u<>o+`*w*ShdGEJy&W4mnoVD;j4zUsOLbOvNiXqQNc1vh`hLr zCg3{W0L}%Y^h>7w5-7z%+@ z1;HLbP*58-Mn=Cm?IZ`&riW%$rM!A?NGuHJRADS#|D}4Ov_>WO<1aa$NS~vKaSU!G z!n1TqZiAun$rL!PKhXKXt=UPorRSY&$|b6Igzh>gZq+Tg>ZPw5!-HBL#?@Fl^xvyB z-bCqt6ln}?saxP{VZYVaOnqg*TUB^F*_@%Os{{d=vxn-s1@75}5h~6&3SO6(8s$eUnN-=Cn;UEl7;!=2g%_pq_Yb zsK;0H$G@Ba6`7j;9l_%|Kqg4GC#XJVTi=QYnlRk3=ENefmJcN!y z;F}>K-q)BfDo84ijnNXfnGJ~&0-{onF_ikmxDTqwE)zz{^F;?|`73T4n?7F$3=5HO+gU36jOdh>!#CLVn)D%;-`V|Zi!x+z{tM? zv4zdN+$1B|B{o~;)VA@$7PWcX=DLeE-^(suhMkDLH}OAWfe2-Msa_34 z-MI;}7_W)A7EbupJYBL7Hh2n|(0VAxMX90zCZF8R(m>p_qk>cWSi%xlZCVEc2>KT% zMO~k??lAKnu>CZ@mO7yHGPAbA;2^gfq1}mi#o<`+W(UlMub5Vdq);=ESQ*AP?XNg3o^@-ppnB&*s}h=PJcP8aPXM}N$ZncG$H5gkF71vazl35~)YgjA`WE6p@M zEXtzpqPt3#6|Daxc$9m0F0L2pUmpWw1v(uDR@Tt4x0`R1UKqz50|Nc9vMRe}ir*XN zLT9ab93GG+vS+e{erfD|_(r(el>rA-;47j*7TXA`T*bR778_P#g0ifadbS8V3PXWQ z1*LPdv2y%;SsGW^D`j(#qi5zH9lhI9v%gS8K`!;RXQ2?9zu+1Z=)yYrn#?Lprmp1wRL|^Gw zl0R8RNGgSAq7k2!reBW)C$v*}3l(Niw7o2ozvoweu?7o%Xqi-ViNOoq0 zYwcI~I&-b?!i=59NczPf!pLf2PKpJziul8yB_oK=z0g!>7m-n#614G<6^f}qQ-|1e z^Gq-wCr?S#%O@ks+(8#;B}t#8j`gPmQ{LN=!4VB#l(|24T#e0tL<5v>y~CjF-3F4* z4IFG@=An4_2y`aPxM1Yb%n7kvV~zrt-?|YJn!?eAV89NLLB+-u?bfc6ED4o9=~aT4 zRZRWb#@XSkL4C1!R`T)(QZPTNX~TkB+l*am+OfjR&HeUkGyCop_O|}I@I(59)LFi? zoq+jHhX(x28~5>jz{JB|Ts)kbZ$JCd9d9fc%?KV?_N>)yZKkRL?X%u~=UJD<9FpmGOETacpnkgDWxGeX;785L%e zU{YNNOxo)Za+I+-R+^leCEZMCRZz!acV-#}dP(Q{<;-3PU34=SI+{QCiLt+7a{iP~ zDYy1d0YpbTTZW>^8Q-!me3$Cb6<>tY#pk9rfQ)cL(`=nui^B?@qZUDKl;=E^6iqJc z%@U66kWBk|6H^vqr%}F4<%gKcN5RMYw?jbCXt+l$$5h z>}ASdv9Rz*uWVq+rn5dz7UhJwSLiP)kq+pF)e@*1HWRFBX6>1JY_LTQC94RJzRM>R zEm)cw@sEqgtdXyupW_WDBDq0NuOY$Sf;r3Zj%Lf9|A-f5wm+C^wO)Ev-lKB<*6+n3 zS&N!~^mfRez!YCCt-%KR*Y}3eYL#R9D_F~2BRu}}QE$7(CQpa8t6gNG$lVDv`zT7R zYMFy?4r;Xe>mvo|RYYunKPoiJ;m9iiOT(jYV_zPdSH zM(Y!cXrwxK$a8CITkT(#kt3EJGONj6%*{z+S99Y~)FM1rCE-^cu2%sxHTvs{Dl{OP zvZz?&mB3`@knuBr8P<)tP*sJtRpm?ye*O8w;S0nlo>2$my62{>&_wa7}?Re~1$L=D;%ETVgtkftd6(p7zlEzEzR$T%ClRZ+*NO{}5I?Xhs3m%19yb zfy>3;6drom=sar#QY9!5Dg`@Q=3o+V&lOHo9X>NM%e9*Fy&>c-^KFP&U!){K#ux-O6xI|B{kbDQ1AE%i$tXQSRLfOG91~2y*;e*^`DT!4RyHQQ zIP~7^P;(#x&XAR6`&34cBXioR{cQnPg#tx``vwH6Lf(@4(#EE_2knq!EwyexDW58f zI|A`mVk1ULs&uA{YNlm@(-M{j8LzY{8xJ;S=Rh<(W=0>w#q*d&W@x8XX>jJhlT=RF zot}wYUn|i6ibgWCm5}GO!xzIPeG%>m=x1cKl8Gi)H&z#GXd|)cUHej0&5M_y{CDNh z!_7oo@%+|GvaM-b&p>pyvnAdcvTNDu1c*TF0&3i{1Ol)~(@q~T5@T`cI@Q0R zyC&AD6};U|%oPnx##<>;iVcM)#6Gia-It?Luk_}v2Mm}eNis1?wE@fRncO(pvx}sK z6PW~FfjUJj=%}hyH20TcTKUEO*azVlS-}MVKzfqHBJo205L*v8hm(b$2WMvW)4S}k zyp7o;*zf;!#HOb8W8|s>I60rnqi^MKe&Fh@j zr0IN03cmJ+CrOL)8;gb0gB;OT*8tuj51&r{$TdX4twWiOCK{{r60#0h(>BBJ3OeI# zX)Qd;FLfeT!~hYi~wGJF0HgG;YAS4@R><@f1IB zGz^|=nEMZ`#4I$t*l5kaeKOm}l85j;xtN#+LsaOjNf!~iCl@P9AwU#xA(jzODxWwn zJ#(H`?GXHaW`$ls(|0MYCXT5dkF9PgunnzC1qtL`h+5C4#ezmRLL?ar%nookG?Q&m zmqs=h8D?Ht=fsn}Z1+MyN;zC#-kBH9i(z<=l1@|cDmoVx&}_1W)tLZHKB@dO73Id0 zzAD%%B*}v>_md@RFbzj_bEfpLD5U&Gxf}?6?I%>KVVCnAj zUhp+tS--@})PDs}tH6bA5@B=~njS65IuWO4xgJB0XXs`@#L`cpl9uYVDkFK5mFXG+;D2}dc z7aiPTaCaYU@WI^&g1ZbZ2?P?{-CYti8QdKL0fJisA!q`@B?)eU;BY5--|zj;_x<;r zTlZGob84ohdavHSyLQ(z&swW{_p`{ne;9E604a}Y5yZdu5tF7lXPMLweM$Q=F+@~Y zR?m#%2TxerTBrd3{LUH0oNb5sQRIiGDagjRifhIk9&9gd)~(mdXn?u~(8f{kmeiNZ z`*P(%PVB&ZzR{_(2!Kl>bPwl)PnK>!g@Dv2nLu24(kIjCDWGNs>){yZhP(H6U$9kR zY{O!6Pou zY3NtOHDGco*%=aLcWT{9 zA-Uu$Y){cVIjJ+=tGxK`CWo&9-G$*L&pa2PfR~nGT@&c5wNM&|ck? zZ+W~lp|Gpe5;E-O6!y_f2Se%PW&+;V-UwO3dYn)`hlD;_}z~VzzO76ia z>3L+EG;?Nsly|>IP{YTM3Zy;m>n{OD3oLym-v%n}#S&s5tz3>E`V6|nuv^~-!hDyF z#fmBC9e%3$Q_zB%Ex)k)O0p^6d|egIWmRp}84!M!c$m=fD9Xy%WOWCoqhtSzVUuL^ zd<(zNu~uzU-c{OAh54W@g%0p+24bP2)mm}E0d0Shwn`Rk$2~Fe&4!29D8;ADm&tiW z_{rvli1-yv6sP&BBhuW3bzohfD0%J96H$LTl>@fG7jk?*l{6z}v*)N(sq}kXB4=r~ zY(1Q{E-g{?q5@x7{$z7D%gq$6WIL_(C6;ms;t2kXJ8xM!<-Vm^6`L!6J6=!`X|b;1?M^6)R-%k5Fe-n-UUahF+U1k{nS2E@WWi| znWb1}MT#x==dJr)J+%sN`pXo@tY1 z&wR_=i1sfW(#vkhO~m6ZV{HbX&93+Z(laj^mBT^Tc(}$36iUwbsiH{i`MVY^;;2+Q9 z_x1M0Ms={xR?lag*$*hBS1EkZF{2oYm^GNA-)(fsMSJ%qtl&w6e&1TL6)R4k`g8pw z_By4hbgYib8+0(O`)-T6Rv9@7f6P}o{~njaX~7!PeGb{tdS>1#iMr;^uHGT~ElaG4 zpB3|qU?Mbw%j!kFfh{%NFFIahV%f>Ep~pq}Z-eOzKmzrPI=GIOdd^8gSH2$)dA_NR2pL=Ex!3m( z;!0K&NSBx`I%qmCY7>hx$B))8Y*9%AGEVp&R!*Mw_6+u3rsDOiuYNq+bb3F*?!H^$ zie~7rM=nitc`JfldlIBcUO>Q}-~SXxMZpJJ#LB74PpyLnG8(r*1g(0AG zGGx%4L_lWV%5S-%l}9ll-C8Fr%%d%ZcoiT--~_96UA@bWYF99;Kqn=@Z;}h59 zc4lV@AgabxbD%)MH6yD@jN{y%)NusmHf+h#g}j?Q=)J(%5E8>)bP9fVHUM!FRrT$ayQ9<%;zX;^fu?dXJgUR`ggQi&Qcq?=E9!K389qaXn=o0LiCL<-jUxA{sGjv#=?ep!_3a zyRl07<*HP}RqhuQPwMN5+?Or!S(=4 zu+Lo`x!$a6JS*<>ddZVhP_aByhyfSZnF(y(px&#- zu(9|%=!o)s%lC10$`tYUW5q(~sZ)RL?p|4)DoKE+Z&3@CxmQRHitHA7oQ$soRidI_ z+jXnUG!;#N-k|{P2D()DUYIh#F$6yu=LgS+oTx)h=P%?mutDvNesK@ZTlkUF(5k+j zrSQ~RtUj7nVmQ48%0l!O!1+=D0uRm0t2bhSM2Ad z7(>#lF>96!LbtkD4lWXQM+ituQ-?#=ULg0dROzhVOdSkuWu#I43;- zK9&2uSBqZjRl+BK_(}6)@g;O}eyW1_hQT_n)io+{RJxiG6xwn6{St?En`=+M^G8Xq zFYVWO0Q7WKz64JL?RK<(`Hw_h*ai1XHzhli(AUTqZ-;|uqD_%2X{|PEldMF-*U|g+ zlz?Sz3mo|7!v+KOxBIq+D?%ph06Gg+bj`WGKGs%|cdYVeGzkgr#H zKQ)%8-Q~vODCk`_UZAo)^NbQTK&}XEBEt7)yXbxzDDZPpJ;Jy2yz%{A%U+ewQ_1OP z!kn2Cc6E(9)A~W;ng=7vWQ_nIM4NjDKvF88bX7G;F)4IWQw`t{)4fB9BHr5b6Nw&X2NcFWXY#s7juZQQud*%LzvBUw+vt{(X5 zJB#KpCX|j-MB_a@{_QQ+t-y&EOrfwERP`Mzh)bvLBVk~zso9+^rO}F}U|{AC)B|Q= z?wPz1FbciteZI@pb)DVyKS($ji=_B@sXy}hOC15~nhE6|1{3y$vi+2j za*I_9zdor-sTb$^%5Fjkz?Cg(7gy{e-G)&m-y8`tHDc64r2)ovB^+ z@tNYs(?^g#%|2uuhY)iw=3Nevw2ff^gZX)oCi3JZLtda@L4MfY4*yrqQz}1$79ojR zp&Ya#rqZIVg9@$qb|NI$wM~&uCkGt@c<54f`04te|Og|1c< zxN(Db(P76Mk9))HD>jG+J&F7tx$GrJA)9(ZT$*v*!j2Suww1E<>r22_Lw5bz{wNun zU&GyBhh1Us4(y68_R4fw;V$hRbvlBuu&H-W9L%ZnLTf;hHzZ!Jh!7K8hld8;z^S(# zR(~#%v~YS^L!CfZL$&6#S9MR$E&Pgwll@9Gy`zK8P;{?$xW%o`(6+|S&E>1m=Ospu zQo%uW3o5t&GDRn!Jarn>;IM#oulC&&*ARUSHdYy@+ z+_!7zWA@Ig9{jM>d2WR7V(s~+t+F=&tylYPV@^orGAR^kiLtPT%&ou}WL|z7NDh4kF$&twcr(2-(h}V%iUG z-);5yvPqB$AMBBIqim^$Aqxqyv-y-X6Wm5i;E-%7c0RYURQ(Zlo9U-PCe+WEyT*i$_>t}!#do{6+>=h0^}-e@99GM(ive3@7!cX%Q+XbYO|rEr zjJ?Z(i#=mfs&w#b5!WizzihHl$ln2ELPyHTsquJCeJ3k=<)T;$I^^AR*R6P@ zYqVOT&2U!v3~6#yZMf>Pdwg8Y^;DNs%$C;3Qmpsz+Yy>0{S#f{%!II%u+N0DWvp*S z4c6E_HH@rOvLe-Y=_cac;_kwNYaM#5K4o|3HYsT69(HdPe2ITaHP2xy@-M_ z^(ltOp%P=Uxp)w*v-GQ>W?-^8`(d#WLZ>%kCE5Gmqh?^S5kuLsprSk3!6p4emQ-5Jl!N2QxycS*^V)eZCg2#Ew{P^5(m&~q;+Cf9TSLX*0`~UdCO;5( zgD?AFm=VmFCVzSmM7(t(wD`}l2!5@}lchvhVZr|yo5#pHs~#u?k${M7W?UoVviKoPkJ5M_!~p{lTFH_E6f zu+WscLi!+oDIy6`hck|7ZO!i#y5;|tN#ZDCC4$ZtDfY*ilKq7!A}9+5vrM)V5?^6edKR!Az&u|7q)n`7!lY~y3zd8)DN#T%||es zJE#w_iK^%2hFRv~LA5T_?~FQwSw;wD&emx(OPKKle8Uq1%_e`X6p!e{U$4`va7o;l zS3{h4;?)7NFO4%2O^j)|zr^Fp?LWW@BEta6(jKOfyeq)!qAJq5%wcB_RdGoNT+50V zJfD$k10QPvyk<2W@196W{6-b#`gEKqt=~*q8w6|u*fdrw2$<;;Ew~*}DJ^~~ep5E^ z)S26cYEVvr1U*~3HxikR2K;{tR)dWh5O1Optv~aRvt{?#XGtzV|J}Z0Ly|NO1K>Nj zyYKbkqsvg>TDznE4L)lyIZqLeiTU5qa{mL4t?pSJo*OtDkF9ouq4u|)qj zgq8o7+*U4LU+RAyq(E35_M4`7)LHT?>>0CF!D)Y3D~+pwM&e(C(?44oeVK}&u0JgH z53v5%r1hJ_5NK-s4XpJ)z``oqT>?VqHiY^qrIHcA8h6^22PeUD$;x*CAWQ2nSW8~0 zJ_UWn-&?fSUk%62$t@++iN|#U=u}JrBZZ$US0-GX#OI$s_WranZ$cxHVC(3yFA4D` z8<|LH{u~O}jm?{A-$~6QD47uxrS-%?IYMyOvgc>bnB{)iZf5xa)F4dwex`b}7W7|3IvJGo$}vH8#B{+{fQzTfQ!9RPtLLudpguA6%U3S$C6 zG#C>87a^U1vlGasqW&hkzo?A7?h;18Yj*;<%mQHlmx=fc7^2rvXjW}Z7j3c}$I+P9Da^X5rJnR%%cF)2+!YSEO zN*G;yFh{V4v6Qc-!eHSJ3gE5zD4NdH|2THze{DR~&WPuXC{Tp+|8qA0l zQV5S|+JfLOigV4cv)`3mJw_W7Al`%{yc_f1JiOUeHwSka4wZ}w27^cfsQ(5T*-(>UWawz@B3@LxNFZj@o zVQ9#Z>^^G%{r=k8=gkLXj=xYUiTk}s$@4!4lb(;jF_FX%W(`HS#lOavo{#+337dSW zXi@<_(CK47h&S3u9l#|&UbWyWO2MwA zG)U0HDksun&7SHlfK^T4_38f(x~2@W{{&r>crsXJ&WNCkVLjE>$YURrY%$bW4e=%& zsf+l#O|P;Uy{U?`2ALI~+n6_?>5TuMb#(L-Lz<-&rG54@%v1bFS%jPQS^;3UB5$H& zr@@S3)Q3gwajf`B8S_)|Q!)%T92iYP#q*G|X#}hNj=UzicaF`d60*@fCja-q>i>bl z{)bEP_q4zC{k}90W`9!{P2FxEKK?s@_q~%4B$D5F8G!McD*iVnuXSH3$mh@AnAzm{CUvJ2EFSIL9{mswPga0RN~{$~7}m{M$iW^GP(FX||#s`@gi4 z-C6#4$V?#$EafJr0I;aZR`8pe&_+v)yM)c1R>!c&?O+;YYO#rOQsKh80!B!;v7HY z-XGB1>=7M59qhq+n5LJdM)^iohXxU;3aNU+$A=2j!4m5NMw3dZg3E+iDMV|*e5iuR z_vz6btC6^L2w5=%kuhY$7l61BZ`pcr^S|1%hybjJjFm`hwh0LR3vIu{vp;Sz4o1|+ zu>&TAGEtO*!)z~75OMwA>0=J8iR^d!_=ns6q>re2Er|3Hd!6J!^mjJ-J85+B__zF# zF}rHWIdTTG{C9}2jEKY#StO6v3=#{?FWJ@ijVR4qeshdVUf37^8oSBoo+Yk}*`x=6v{!9GbL<8EYLGD0!n^-u%aB*yekeja{DRG1HPMX{UsUfF<~ zHN)$P&Yj*oEkyQ+NFj&9-Xl|tFzqho;dk$B{mvf$ft4@IvP*1#s9?e)KoklaOM@ht zjxS6em6*+#Zc7;ug~Mj|+jI$&$0SzH*8ZJB<~Imn&KoPPM`7_pv#sUv5GkblKqQUf z2otsp6w{_3NF4t=g=A)9X%s;WW+P8RT$Zgp^p&A0;zRuSVZ_ouuIg{!=&_D1=KUc5 zAMrXmB5aBv#MN?dKt`e&U&DN$+h032Ai%kttVTP&>;XjgnYYCLSA^A1`Uhzk0q%?m zBYpL#+M|dp^efUbYTvK>|}*0^uQ@L{JryUAN|ylU)f6 zvT=Vq&HtW0W>*ZUMgDe>P^RpXZ>$uh-t7OcqpvJj71?j+_;>U*TO`<@OCMcC2@AZ; zrrUiUt09)V(6}LpEq&Al<9oQfRq;!d4#+kT`l+DonB_&K^w{L%tQ_;$l+s5X_cpyT z3#O=Di?>xncJt)Mu1zy#x(hP3o*BkF$_>a7&@kHKV{4MnHQe8TS8r6UhLdF<634rG zZ@VJvkx?7^pS_4ajkv=!Xj)U>)l+l2Nvb(~bEM)TQw=t3!L}7DYJ_5Op*mW2Lo{XB ztU9l}!aQSoV|W;B8tEfHFd5(JZrND3HFuA0{it}2*Mmc}#`V(5pq1+x5HvsuLM%L- zKS(8)7^!l9MU0SX`?ba45{QXe_^^2FGqCzD;87{|Dj#WK1}R}ikY$p=brP>>gQ2;< z=*MBw?Ux_sKg5>A%FaK58Gv6T!Zk7#kE}kA;$*OU`zH7sc(ddSvB?W57m5mRJGZy- z-)eueD|}Y$X7>Y**VtZ=)BM7YFL4&AHUkWsp>Uf7n$K<l zn8Xm73*$+Tz$-K_6mL@B1)T~9l|wG~r+n8slbKi@o6LC;T)=uO6&^v!ArPGkiaC1H ztHN5$#LYX7>(=JP;fd%xiDjcFubpRUX0-bzIw-ol8qX+Ll1<6>-UdF0F$409Ya(Bq zh-brpNB5Q5dGULRI&YOwFb|Wix`6z1ia4V?({phAT5}h=knAfPp`S>N{Pxw^L~c+@ zn(h6l2}N#bF-2i8p5!%pQp41hzh{Vx(1BZx_$PTMCkr{{nB40yuXU8hNGbNf%;-IC zN%xm?nsQ-=X=<7vzn})gP10{O)gO+~I^Jt9=T(oc>v#7!w+qF}(I!EySM;goJ@HI$ z3L4GA0)Q>Q0#dz9TA?W9%O*;C-^7DRP+QY9N=fvl<`k-JDn2bxa)0{I(IrFBqi zu{c@RrXj!byS2?hmtyn%3rcw-&-J#f3;nVNgC+?>JOu-LhRhG&s>FS1PBCb&DeauwcanQrjtUj-8oH)$VG z;pgKiysCZ7o>z_%5BiMJys8CO-a+Tje$2=KH8^#2+^^Dy?&M5fq`#QqzJ!J8DpdAh zKQgOdNM1DYiWXL3>Q)J2EPEr^D7L;AxyVWVWAS8Z;&7AtCQo3+li*$6msEi%6wjlF zMa#S6=K~Kr)S)u{5+$lA1f5%f_wTZH6bkO$HlKPX=e!2bu)lfsWvqM4>Tb#^!4kQF zqR$eoI4zj>`Yrtdzs=*3mm&TAcdPAvDKhQWOoQ;+ciui_y$qGrDJ{mGaz|Yczp7c_TETY9OWFY?fSM8f!`9&URVb%%@mBDmK1eBEUr)B3>P#cOjFtU3Dfj$n zq-fL#phM1&{p4jK19>@pj2)2Y*EJ5zsjTb9E10T)J&{eQLuu=?``NcWW)9DbHDjix z2(}-0qeol(>RG(Xv2L*p@yB}mj_&NL;vE}rvoQpM7&;9Ul_|!!6R&Hb>$tbUn2Wna z1mp82nd1!OUN`(58ovnjGqD9+3PNLfxfmRtP-B&(K2O6-vLy^1zw#uCTQPq|B;wD4 zS?peZ%%&$XN{jMwTOY3<1C5yS>zuVZLqc9Ln8^HQ^`tm!g3&VB4c(2Rwz=#-gOK2%i9C^PZcNQr2{~ ztLWab^@?;aRO*Dt=XCvv%CNQVhPTYpfXJQ4%gAHRF2ha?VwiTDLffIxGgNCH;<5MI z;i}j;p@HbHF|!@pX34)-9l6sLk4Ow_uXQOs)LOiY0hYG?#5MRnz%SeHaf4Wg#<3sI zkLo`=DvJ_tN=Z>sJ%*d@B%^>=S1uhpxFVBf3m!MecGz04t66oH)d$awBG;l1e=r6` zJPJ#D9ERm3akKJE!88v4Od(cb%6g7NiNRy8x5VElELyjbBI^3oG~* ze9l|>k~m!Fow1ED!NPdEYpx0(1Q2zD`cm1<;r2;Y9A~j zdMOmyCffdgRK%BR*!uOLR4&Z@E*YCpk1$bp2pLvP*-dr=5ivNObttZ2< zj1eKR#K!K22--e>%&g!8S5vy;F}9UP>PNJ-{RI?|Cq3{sITuIez_Ey0jhP77X>55A zKT~QW#@P%MLtI#CU=KOy9au4fs?gpgCTa_Q@2H}s9l|`!guB(}t2}6M_Oj@xa4&1w zA|9*BMH`KTj7^1$d-DA_epLg4+vd>|jW1X*hYaqo$O1-S2Ai@+#_Gss!qSsOA_KLlIkR)6s!jF7E8f5Xab5c- zuLW<5L(dwoO!x~f0BEzJ#ZsSZ!6orZ>Ke0r1)$8qD|>xYSzox^mC%q885~_OwkptxM+s12vbQHWQnCteg4>5L&!ag&g!MGTr?nD$FmHdOb;Hjarq0q1yYi% zONf`DxM~|MlxEWy#3a&+Bq&}JowR@K?!sc@i8fJDC|dJ)gvpZ`_=1@(Hj~(&dX=YD zQvi8jHePuLsL9Ki`9X3PZb9BJgW<<6!27A29e#CXxkq5k z#qHvk5DC}BkmMK*tmOssswAU6(RoUS)BR%(; zKWl{QlOz%vrK!M3BqKh&XL~Erid<)-1Ls&ca&kY1hhgv7niKIg|cr(#}jWxZJn0h&}y!UuS7$tTO+-xrsc|Sm^$J>P2;u}d=B?1ME zMI=4djW^W=mrZuCoBI|H>s6*DT2Ry3nC*LiavL}UrCc)d+B7WEK*%oy8!OPOSr`IS zQ27{7R<@Of`{fao23DfVxT4yTq$rpUgLb$T#>*0B&55F)gx@~83oqqWq97(k9k4mG zMFWj)nHsw{{t9bt!PP((#%}>P7E4tyGNJhF3Tms7q7l4@=*fVG^O3{6aJ{!NmgaeB z!<*lXQ~g_drh(`Yc`2B;%KR$_DqDNG?MdHeFGmzpNCG^At*=Rki%4h zD}LTYla?_xW$F!WSSk<`<#A$cYgKN!4J~{)6xT5G3O@T{zLDb;wMmH>i?K=xe0;I4 z;Q7teZcvTnz{?KKc!uSIKfDAnmiKw(Fc?8+76vw;M}1}gt`YZc)^0d5Rz6lOoOgP$ zCiQFb+?~QhVZHW)ghG2nEh~yd2Vh?9Q7GHGq*f{!pFrBy z-cy&L)H?5-aRNmj?bo6CpH@bplpLy`!tGWFFjp?&((8kH!xe$CpMd1_0&dzQ-c6oP z;n&n@F$@R{b82tEcHkz2l*|H-d7RI zTFS_1paP`Dp>Uo=v+Wp4c5BBmaC&*2`DLMb@7}7vXVim%Xvu1RbsByKF^XH*up+pu=`n-vqs=wcF72C$hveA+iLa32 zc+`r+yf(82@kKhMBgO+S!-%oRisg0F-<_$d$Onc`Qr{nTwNi5MhLm6MZeqkr$}())n$4VpLIad!%26F&Jh8iGhI>oABjkNRN&l9@$`BOK%@xgpUD5VT8FDwxhOOj>iwfF9q7huu z^~}97mo1r{$N7qu`dRIxhq!?2-2lH_(6peYWLU{uj>Dq|E`fteOAb-w7o*VH{zl3i zXCf`ax6OvyTr6+NzDv4G;mpN%mzhLUfBh-jU~VJyi*xh@u&zQGcjx=dAim^-ja0+Z z`-4yw%BV;2!|(uqDi-5Os_?)d{AtyulCU%v`>!8wG^~$fu=^Agd?Hfx<(UsQ_&+hz zHHo)kM3GTBlcy2J-Hmru3VoGdD?MBbWKN82nq-=K-9$=PfnqyeDGRe^_!<`eP4x@% zo40(i+eQ~l#pnQI(Gn~mzuj^*A)YUm*RcmKTq4w0)PdvrSe;boz#v%y8SZA;CAfy_G$fQrYaIo-^LZaT!*0mG}cCc=^Zym1K&R~Bx2jm?RN zZ(XUkK!)L0X4dDt6n;P|Kum(}-m7IHyRhA(JcU%@n}#oE=XD@KicV7rjS1ItD`wD? z5UZd{C-rf`(b7P!IPP`PXN6B55Y1huDwOoE35ngv&2YFRFQgao%cFh6Ww&a);>) zN{FRHOSrKu=7z0Cs7HlCT{A6ucMOrGMtMslr_Z|MDx!}mSYNus#S&5#Wh+7L`5D?I z`O(DeIuVbuJV+V5qKL$%n6;JMcT>)IAx~6jXJo&0-#h8(OpDR2pvA}`pOFsMHe5hy z#!+vOk-rc!8G*262m1#Z*iG!%GRK~7#(%=kma3q!ki(r!Z6wb{F9kz7ey#L4He=km z-F4p^S($F5TXY679JT3vrR$tKz#-N73RvZAzCW|9fzT`qzInxj@6nw%OwR&Wek!p- zzoob=8H z20Lq|sP3UyWQxc891K{VdrgK1^+}M3xwou@*SXp`n9@f|4B$NrEnn% zj~<8M`&;AaIbq(5FTGUBjZ@TN!o2P^UN65zw4!x{11W~We~p9{RTVCyylIgC9<`D| zsiJHa(wiC{*sj^ar~@X*^x`szgkQ_+6QMtwrX}2d4X(nHwJ!5JjPnfpF2GUu!J8n3 zV@B~?`*qwNmAp3c`2^B0+Q_aH!fX&~s8-)&vq}~~XNSQ@NRa88Y%LY4cUk{vfDRq= z>0zgA^=9QyMq?865^2k9oTM|Jnc|pm7T~M=_Y0Dd6pSq@z!Ozw{nN@3a@Eb7Crqq2l)Yo-RW&61BiW&N(sM)o}-wU@tpB zMmYJUqlz(ltd&8F5ff>63C`)usoEvk`;MTcoN}D&NM?-dm~X0+U}XEw&|5Q@7?a%b zVep;W&{=`osy%gDi>fXZLFkIG>~{#pCD za|fo(`hqc?M}ZMrpBJN_Y;7lH3lOMNMiQtsP;X+@!$u z8$%~zw)m+ilSxJN%Txx#-8TI6?5SJ#=c$7Xq@>LVaoQUHzE>AlDni80xwP1GHa*OX zikr(8k`A0X0DY|rS9SmKcqIMP3MPWIIk{y9=+nc9tE$W{{4|uF(Gg~}I*es0YMiv3 zOfhxO2ZLVc_lJnZ87mJXN zS$j`^=}Sf$7bj+WJ*d#-?MYA}^W??7B19q#r0$y2$5xHB-Uj*|){pCFbj+g^RzDaz+sUh2*wyAH zv-@(=;`>%7#!jFK?6WTi2g!)z{zB5cB@*`!VYSa5Au$DZkyL*3 zgc8Ib@_+SblJvW)5NsAvlp4td@0wfG3^{RqP0xsocynQ-do+){;xxJ8Z;XWE#q^7x z6M(JDRK&>qLe#pnh$xU4G%7Fs5&TZ2qI_BGE6(H5DXeOl@A-X?nG+g*dT75t+}0`^ z8cnL>BpC~3O}d6wb=zL#2wRn&za)YGl%D>4C0 zGN8ok{&wv9dKa;tKhZH+g<&43J{_h4l>5GG*rJ!R^v--Iyh>Y^1Rc*+WU^V`ccK$t zzk$KJ#{2#&Ofpx@SnV^q4af#6e70I zbb|!+UJR2S)?wy%DW7r+MfFptz${RLy>e%(EBnfsupsh>p>mV4E%*!cLJWnExSbe^ zZ?Cx+XN5ouOc4d#NP>lfB*|o@qmg#8Z)mnkbdT1$2*>fn4JlzT=I3;lkJAOVWeHaY znMFX(g5P@tjA$Q`<0WgpllvTf?9AXAS4NgDatrWo*l143|N|Rkg*^kz}4J**#@Fa$nZ#_DDqH=#cPf zG;HChv(~3*W4aBWEm|ApkL&cim#q`l{36u*q;}Ln+tfB9+P^5kl*YHPj%0(hd}J57 z2Qrr!xee>x9$91NBE}kYk~#$yu`P~r{&%GE;V&@y+XO0F!v^mZ{H#RzAnOWBanD+` zpD545>@M@63bAr<5NSp6+Wa(mzmocy%xI(;ykt-X_PtNqQu4uIQNeL`ob~6~;MV)vcK-VT1xR9ofu&e{)y=+6(WU$Ips|)is)<9I7>7R z*4USo(TuRs)f7twjGW}+#QG_GKw~ynio0x``$>4P;7NgUvhV@oL`}(SQOun zrE)aoa9(d*GCP0iz~DLWV@ToUH^8$-syyWm5B)VoGMzI%H*66EiCz8}1P=341l#Z< zJ&)j|OhH@=j1&C&b^*XMM7^gw5m| z#s5t<>tyaqQ_PKOG{fZab=W8$Ns{~|!9bGxJW7DA&3$smFtj0IQWf~{W$4S-RWhg5 zCD6nvxs--`sypd+zpBvZ*>+Qsllh8Iybwjtllc%u&z%A-Xsn_)$2KknsI308+&Omw@r$WQ zFOPnK{D%K5cWzG3&N~n{wlQcgK`B~(5;j;53WW1c(^78l~kl)0uWumP2d5Ck&{X-MPo>J3 zuNgQ7eWqAKwgetjM8U15UPGx7r}H{R#HCDhHwr_l-rwb)DC?cthEz-pE5y21J!uwf z3{Ws;dMv{29E=4I)UU6v)1 ziq}`1K$*bfu)8B9fy=IALBvyY`X=W}IF1Tfk7eOH0vJQjQgu_a4QF1<74f>Pg|gUZ ziRsvj8;gjb`Z&&c$``~G#8p@5M_>zcK${ESV1T9)yJbO((HqRgb6OXKg(a9SkKx|b zB4skjcVy?*+T`eqizK;C&B_ZE7fmyIyj^(+)3Dx7$8`{CQw}f8$|Jeqlv}(Z$(kr(?h?tYP}L8?!^%BBRd4 zsMRz~^rKS&YA92zz1;#1{6l^pm^eG|W0Pa2XiiF_GmZ zp>VRZ7iEk>A1j@;Tw7A67Uz`F`wGJr2A^;>cm8YOFVAp^uQ&pYwKcUrdXj1rYbUIJ zI`N@f2suQjp?sj06|ktp27O8tHV8`O;i@TG*Ug@+O_qgZCO-q?jtwjA1VjO_U-Jb9 zw*Z;`)W!z&!TU92%~5#&dVA)t1y3SbuUrrdpvnkw&CI6sdLIYU7-(g?LfrJNft%IP zgF9oMk-juvQU-g@qTw*N zVTVE=qMiT=$A-A4Pvh_JG#{p~AExo@DjyC#ZZ$;hl#}6xsF8u@^OiMwj{^0_M54pV zzUz>`Db}4{=z+*zOH+@nF;tc8{={%RNwK&%d40srKYxo={?O9b-G$WuDfPJEMjm5- z<{s{zSS>p#q&JIsZI<``L-#=!mLFkVVm!adjdGF)p*sCmTT*p5fw(8#?M-cydp*qD z@`z6;w5P{DG0-i95L<&l=*6X<1E@=Gi9^zZxia*S{y4lxRADx$Kh0&Y8mPJh5;&)x zlbqGRHWIm5q&8nLdHCk}us~f=)3IyRnQ8bkeurMR#6RFeq6k&EL&$ZoYST2w1U<{t z_v5p>)PmB0?8y7g=I@ETVN7d&$VLwD?PkIIIKFVDWUG;J_?R z=wao*9}`V2JhIths!+|HXcG7ae>{uOJ7Jvb;I@%cRxPlWDstL#gykP)n0ue%as0~;!;$G#@ z7OkTut)l^Po>qbVCM^xQ0pUGUPlMzi(&{O>47s}zIe^?hpDOtm)vQuXjF(Z-+sF{? zE2HZHmc6Z`HO~(=rsLiQcMi zF)sWMs}6q`s!xK6_&wT^OgmD1MJ%TEUn6CF8Qc=)7t@4l5ogQ{D!k*PCtV-ej@b`0D@rc!+(UI|B6BS3L!{M>q&12*PjBENq;+L}O>BtgaX|`XEWZGW zG6e%JYCP5q62r2)MF*{><83%=Lfgn-THKO$m1&3^4WC~(4V1Hx_o+1|LIANw&Sa@$ zjC)qO)AJxeG>OJ)FXLu>iJhqDjwCuFlcbh0De$&9m}e%f7-t*Jq9z+`;wB;pnt8W8 z>pS3eTNf$2}XklBz8i5&if{>UUuLx`%!uE7m-i;s;9 zOUZK@1?0Opct;iQU?DShSJ;>Bd#&h&lhCjxe4;-*{!@L}kIoYQ09`33-N`hFVD#M?aU7Htx)y z78Y;`QRg^GZO}U!k0LaU#7tQh`VXv~1yv(4k|Nz_A>FRF@9SbdxmTJ_O|?ze~UvMiY*E|kq@w$~5Akio{G4%6|2v?OX?Pm*z-)gbO= z^+K_a8zDzPP?fTA$2d4%CcB}ClR2g^h5Fu2s-|WAp*G!4KHam#64ql)#4zh^;S|-iJ2*%q$Ar^UIV@+F&)(-^;6)bsBY_Ub35*x|5I5^vZMV32X){o zckn*LeI?vN)RvPOmvp(O#PcnU6!@x?P&F!C&vH(aZMJS$T*=Fe-uN#jfw6KTmGdsw z6{^1O()bT5nhr+|pS?NP>!w^rr#=ano9eobm1|hJK3iQUU%6hD^TCV{-L;VsHHzO& zX01fImOcP+$zC~0d;qO2(hzE7piDtR>P|c3ad=NDu$>2LYyR@InCv6m;Mfio==r}E zri0*Q%E0AGC#ht+jSJSeGO#qT<_XEKRK`}8K~Y+LMz%^OVJcEf#fY}&CtV1YYM+Th zJ5O(G>v3|idV7-Fpw;xPOLovBs>)dOd=jahD}rMAUXJ4bEo)!WiQP1CuxbsTvEvN; zq(+?+_qLF2dV(al*tE4IM?rK)#N(h#GK88&Z^L%Tk#DTpwXP!!i?}K2s-asH!FJMj zSm08SaI#A{xe|OyyK*zDsOH^&+tEVNb(H;FrDHy0Zi!5v&GM3r)1T zO$c>mRt^H;5yLtO#R?hvqqLmCL!58(X{k{i8aMe$JXjk!(9coV7aWfIe2$7or|*|a zG1<9bQsNrb{4lW>m~*BBZzMPy?d7@&NZTaBjuJ#Vf4Mx4koIxHIsdp!r?w|qBxnJH zpW0%)*S} zPzZmt_i_?_1!(AF>j8j8ybUD>Fa1|Yra8_ zDCR{It{rQz-XvNSZ&sVDJTyT+d7lo1Nh8!W2NBvIccRdGO43Xgu8YyyY6ii5qjr88 zOlm!brwQZ<7(`Q49y}L{Jy>5bSZw1i!wRZFPy68FS=Ls`ib|2KsI~BeDATbD{j<{$ z64^vM^Ry_?M`a}QvQD%NEAe#-9#=7&aSFN#4l5CaK>645Bwm`V3iwb@4K26h{VsRd zy6E|yJfpJK9EFFNh9US!n=p0WqMu+aT|RBOB#1#Sr6Ak3BI?J`vrI!0@i#^eh`QuK zhVMnCmv~>a)3eZTPhz}DJ~xT(Fy+K2&CG?0G!~JJ`Dho~uW^#-@IV-I)FK5+Z&u1> zr8TTchLD_Y^@>>9TR#O5^rkqMo{Z4s7@7H!uXxRJY}%Hw(h{`M>EM;jz!j?3rMF3H zw@F5~A-6w6GD+_8mN3+!$-S4KfeY&-<~Kj3IeDokKSq$3L`WG~RpE411bJSjSTpCyA+3j(zt6jmKr1wd>k_eM{ zFeB}|3`miOGznobcvD_nJl&2yr0u#If$1lB+B>5BH2Zzh8)=bxg_2K0zxxl~wb>Em z`+1aU3LlQy{aB!frxUAkx|9FwJR7d1-OJNR!m-yFO{I%q8v4B% zf}sX)qHh)U;C8e81RzxuQppG`X|!n=i#Q*SMHGIw$FsBIm?FJO<3wsY@ziVSd>%Fo zzFgM5lefB^yVR;=W7<~v^21Iw{KL}$k^+?J0rp{&&2n2X5<@MfQjiTk$U#{o+$9o5 zxL0Wf&h7N`aJ=#o1{%JgFvi)Ie@943p2JY_qDJC_>hIyN4zd*NXS_Kh|ma?a_78zpA*c&@eIC?-VD z;Y>1g)jAza!#bl;#fff*$*B45@{(xW|LC)(lNl}Mv922`ErMvo?;t2gbvmE!ZB#7I zWuu~1_p1iO*WXBRxoQ}6zT|R%WGGM;rD24(e%ky=adIh-o+fi+{5xeCAUMB}F4jpU zjIz{obp0Q2lSl*jG@lrx$UlKC+>MehZ!M>)|5jRGULMc4#upyRdg^dw`V3KhlBh1P zg+id%=VzN{3Dacvmw_|?fY)aHQTfI^g+0M0Au|H^@ai+*UX^u=C!z54$JYIN3~1&^ zv{FRskuS_XTlE!1JU^a!NqZ7}Dvv@fmp2n4_qJUJb^x&ZY5%kP&^SyPn>mtpj=8N9 zr=9xi_Ij#PP+XEE0^OVCjpH};tmVHj(#9=r92*l2>e#+4NruX0<1uJg)Vn}7iOm!V zqKxWM;Sxzv(rsioj40J}t6akC?Hk$}ulZ|RMKvi(p&dy0!JO`)nZG<<^bU*`0=7Tx zbGAt7?s>?PFC$F_jBiPG5(KGU!hQ95lO>9!y2A!|@ zHQ<+v`Rxr>R%|v!o!mk)=~6by_j6R`tN*O-_gnYv*sY$2J<|o+Xh7bqUXStl*m>MH zwzzq|VMR%+{#Qp>o*_NaO_RETjg)e|c<>Z9F&TowMe8vsTqfmYvpSK7_FAOe>gg9D zaB(DClR-=thkxLB(XcuvoJ~esXX+LcRSvZwTu6t>sl_FpLMHHqjF2GJPhMdgiis4R z&nCp6!)HkA&g#aiAZKmGq0NI`Xxw>T{G9paM{n|$*s?p!=S7B4xFyT9HBWTU_xN-y z<<7Fx?fP~*sZSO8T3)PUrN74aFriymIR^tGib4#Ej)OWmUUGL@yB>vhQk$2wcZda* zR_v>VY|^EzJsEZAmw1wB_nY14eA)#)xv_S7Pkeu^6un(`JRB%UJqb$XazeS@?PL!9 zCKy6*bj91(QpucW{bbm2yAc`F@EeEgIQbCIgG*y}sj_Z&V#&1K-2SIP0Jn1|n4)%K zmdDf9hThV5JL7ilx7$RXEu5U)r8T;bu|Da%H24oFrkk&;qnot?H)_~=kX$B#;d?NW zG?!M%h>0k_X%-f}ARic-Pm>)Qm`i2zNq=2WoIT#67&EpMumfh+u6c2+)KFw{ztIpH zC7p6cOjsLHQ?lR=iV}ulJcP$9$1P=-`J_y$76UqDY9#-lmoU*l0RN8kIUhNCX*EkQ zr`qMUxLFXsb1JnHc`N5(rm~@A zH7H;AFrft{W|=U5ag}-Mb>tiUI`{TYz4mgHk1-12Oyy9)|Ch+XTyLrD0uj2}6xL&> zqn`@kO>s1ql*#fijhE4J!X3KtdK7MGITw(=k`H>ujX0$e4srxda-@sVH(hUd@!P30 z3g@N=PP_{svN^f(OonI1$qREvT^INEp3>gmW0L}0&soEXFg;h8QCUx=aDiB^iLtI^ z@!L03J%6u?M2Y<-Mijf(fM|GSEb&^MSA7&-EK6fdgYnwWW&AJKFq30Wios$HQ0bj0nJ)6`AZ z;Y1c_&UrFDl`|q}^4m7*kv-#n3pbS1U7d}4}xy1j?KL@DHG1WM0kkqePp ze35l0Qqhi;aI20|6No3a&A_%swOT(yjXuy;8&TpJSIn@PKD#24s0+TGGlq~x?uLCNdE&Oqumt$ z@uMc+Ds@pm`75%!mDLP1t@CMpk@<=RW+Y6NK|S&{{EesMDL!Nl@i3{%CM}xvN$c?& zjl4eS(@8q*M)6yQZj(Z8rQY8B*EYw#*K8*`^RCDiGkS`=lSyjQBtcj`i7IJjQ;76U zm|0&QtXH_&UQ8M`(SPK8J*n|;XHOdcL<{Sa4&dR8V!#bg3eILyS4=u~)iIWqYEni_ z%7sZ!vS#ax2z<2^*ekWEAFeaW_lw&#2B_%u^s~#S&A_5Mmqp!Kg^kR1E_6XD-sj z9C$cf3YXuMRGI#^M%os#1{4obqn71Rh%taaW) zzmx&!agv(y8ADR&rdE#Ll$+!0p=O}WZM_A*Z1!=JtDG}y5Htp^*#XxwOeNgy@L6dDNk-d*>@>!QfUtK+GRQeGrgEM&LJoRnWt*1wg)MH zeR<)V$~mken57(Fjn<_uD^;oIPl;iFJh_D}Uv&}-2g&jTasyA%W_6!lbj|&(oCAIJ zEb!FtrsC>SRYqlKx#uHY*)^Tx^N|XhU5A_2w#tb!@T5J;8g4{CWqzhC>s1DrsAt|UDzb5Je)*G!TgW_`P26v>Jo7DriIhL6*Zv3%ZYkN(!{`c&xZWuTziaL@~$q2Aw2d4J~xp48|m6 zqKs|w&78E;N1+K%*M+cAeIV$(mZ8rmgE;Iusv2^*VYl?)j6!36k4l&Q3FjPM==}R* z^bsK{)%Df1rat*6V?4)uRPBy=-Cj5~P7XW=E{wFF1!xM7(HvnJa+jr#(;Ib@pD!b$ zl?e`YWvT^~83k*qtbaLgbL>#OvX<8?pUzWGqxm6p zWJvti3<4s3r>m}bDW1t7mrS~D6{m1=(Tzbtqjr_%o7ZvEKA&a7e-=)hj@QC69p`qp zCsVJ|8f}t9RfUCt%im;|xC&;|<(id;i`~)b^D_|5a=A=>I}Sy;-P-95 zyL6;wh3As?5}Bnx_>-q^mSCoE9;zlIT3l8ilZ`8n?KlrTz2(mqQt`tn{o&=X8N@v- zvl-S@7KWtvZL>*EvNDe4Ny5pYT{i110TCIsa(+}ORq_FtEZoKSxB4q(!-iY;u1z( z!m97*5#|GS(k8cOGBmUtJ4uVr!lF!kbv2~9q}s_eD67Zb&KU+Bp7CebWDs#_= zU?owGEF;Urm{q_ZU#!9lXwTEo{+{c|Lwz3WJz;tai6YFw%BSNismVtV;dJV<9%o8$ z$Zq0uai~c?$#ieBWeHL1C~1*?K&kJZhBPa!`jqt;i4oNdLhVWVIy5DBc$#biwOzuR zsd}px?#W~Wb{c|4QDH6KpJ^?=OoSGem6+vx+^MWMoT3Aj-?@Tg zTP9MtfTuYm9572>z?Zn*d3;|kqgW}xOlPVsTRsujr(sUvz__uTTT{7)43-P*uE{40 zH?4Bu8%rf!ZPb;U+D`TLUg%TLqRM5iTp6UlCfV(~iIB@8XBP86<`OHWv6M6EgN59_ zaTj*>ekUI=%&iB}J)NcFhFBjLk2tseS<5@+ucmX(4oYKoSK`&$ibIu##KyIuVXV-M zS0ptT+UfLA)9S@=EJx!!7(hf^OhYGYN?c8}2T#+j_1KlZ?snokJ=o#vT{50oSk4kV zdp6ZAe5!^1?!V1cYBa~m5?5qx8{G38CZ}9U2fjw7@+x5`(Zlfc11*vVDfEIxxW!FX|DPO2cnO-%25{HPxPYM zWUd67{IZOcJD(uacvs|cm>hzt9ap)SH)h(pOrA!zn^0^9>bTVX+2mTMH~|xR@RhYO z`0$BV(^fl0LGSA*u7%l!uBsEx1!N=VI82;n9G-ftTjg!08pbpf6_$ciTslbSzTct% zUOoA5HilPb@t6C^NLJ~cqaS>tF=0J$BzS356B;)?En$eSMUO@sJ8A8(h^8x*De0mD zk*1h(?Os%SV&vRxYJJgXmX^{xfiC99or=IC6sTx)Etn!_zN{>X^*6~?m)FvK`liks zQ+GyK*Hsjv4RXfFP*47L19_w4<8{F7)A%O-5U;b|=q4v|ejA2Mv+RI$Svu`rL?GX_9DVlI9gTqGs z-1C5tcYZT5u4Nb!YQ3G2sS$?dC%Q!=_%Mjyt4!N*2vB<@UTyRF+~TcW5!CYe9c}zC zs_mH+OOs|gYW0T%yQgm(7%r={}IV(3CI4M{nY#bnWYpC~E zOwi5u$85Rj-G48P%Fo11su=xKFF2?41O<)09+MqvVp(rwgwajUU>mAm9rE>%nY5>e zCgX>3sisx5WhJ_%Je3#nEhUp_sO`GEvW##JuD;k*btG|mzRK%1R5P7nMmJUkYD=lU zJ~ZZcMLn$Y7fVwH>p4%0uL+4g%@2U$DRKF}&nK7KRym6RQ92sEj6WogE^XBJ8~KAm zXp@Ynfb}_4+V&|X(N14BgD9$Wm>0^V?UQ-6oq268Ws%X)v$uJlu(j^17Du;HlNo3f z=UkP30^y!AK*S}44)+r6N%fipuqqq-x#g_FHJuHU#!eOHsg3HS6uI={&4(&hDKom= zOIK8J;%H7hp=Qa8vgDJj^rR%!LtOMpPjE}u&F2qXESEmxQ@=vuq~ejPg!kcI@}Y&T zb~WjBL6J{SR!&>#=o@qJPQlh+$jtuKb>97nkJw1rt*4dN)n(n?CN!&OB&aFi#BS=H zU`qo0D8x(OuimcZJo%)cx>|di^4J*G_=Lf(rs0fG9uee}tElGRL}oHqHQ-Dg-g6!8 zwmRDx`P4UA8kqZ4ay~cm$#ez=K3QcUqR~}aY17#}yg1jB;#SGVuCDj#xpKy!Jh_+V zp6EADQJ1pk(zQISl-nm^pGB^4^eF?XAEZl(@-Q%C4GmsZQ-sCQ2jovWldCT^Wst*2 zcMTRN_j(u1(oPeU55>x#6e(8}o=Y|I_(JjZCoRV&NoSp)>dLXk^l+h&R$Q0qK$4kS2pUOleoWrKKIm-il# zxM{y%-&g;gGHzagvX2SdWTm5_RF-PE21mkEa}%u7|AS)c#DP1T;D(dF)=_Iz8dzeJh43PzEJqmoadAx()W z97_X(I2tqGw2t%9SFw_q6)ru5{*`gJ$%#egk?G=5g}!%<=?ZtHkV2C0j+4*BIsUJ* z=BkC|^y$nsMOTSJHSYPLD&w6~sOO9s`ZO2Ao_f;PbDL|tKM}Po-!m5Wu3lr3s@dqk zWaO@i1&x%j_JqFC7V6U0Q_7wIBRQGt$mLJ;h*Yt@An8GVlPbJdl{)`_ebj->be5-7uR%+Dx5aQgu)9op$zhB_~JmlZjn!>2XM;V-M*%as#%u zW)dCiJ~EEVV==!HqpK`HPo2f=+4y7IJ%3E}N(W zMqVl9WxB52ksd|&!<$|XbY_h5%ki3ah@C-=n^l14$w!%%wERF0QRy#;0wC!htLjt3 zZ#r*A1<-uaxWo>0AUr?eT4N+X)p-U4a%AazA#aj*YC4P8L~}YJ z#@oXTc#^vg?sCfOla$ zLj`rBB=*d;nD5Ww{gyKoBFaLq6Aa3d&M>wWU&ij}uu0{RL@sMXn($!KaBR|MZfxr9 zvNbZ}%b^eo5W=5Gry3@LwV-|FRaN$`try(s1%nmQuqsrZP!ooM_>5S?e9z1=8$U(FvbV`6^CvL=m! zQxSr+S>&*I3*~&v6Rq+g)=x@>waKS6@WpX) z=_HWf8qK?ZwR$szsCJIv#$u_qmHdQJWEj+>iL?u<;rPqM25?{(P;VPn%|iwt=dJ;;hfsn{ahIH zCs!12bltcv8x5IFMy~Kd0T7wD&Ia4uw*n{sEaydMdlN!UyMkAkcd2X`P-eV1J zqJ{BAc)mDJ#|T+kl&s|=B5_sMQ?m(tEtI_j10tJ!Uev)j6Ken_-%1oCJo zx5Oq(1r3oehD!_MD8o_jZ|ou+O>}l>I_$gDWCVP(r>r+N%`uPg>?h6=MPb8W4Ac!L zF?A~Y0)^#SL`8I8Ihgec`J5NU3Q^K|cZx0N!4%0z_d%2YUvF4OlW5^6no-x#T9q4VkI|)UJqSt)iKleP z8ZvLp=A5jSE>H2|X<{eY5Al`ZOcMzob+>1rnG}ezDwTeQA&=!V{E%BdrFrD@olmkG z2kB}bH`Wus?fHHKp8xyA2)Cc)auqY>CN0ZBgRaT~Gi@ANKG z!RJ;Ubi6br34N#CjE)CoBK#&D%<@F*o)j#e(8CI>W4o_@bt|VMhpVfyhNRbjv3can z8)&y|vSVub8SDC*Nm&{4;iN@DhCUByBdMu&3bk$Wm8hAe-*n;S5PWs_XG&E~;`5UN z_1WBM(*8#4q?*C(2Q-02=Y}n7wP&NK4jdG)AE+dkB_m~ zNd`5zRSvawosnvQKRfdL^4)1P?`KBJnANM8MKd%^swNgxFL3zUm}rrA80s zTWVruxX_l{vZHSf@P3j)Wq7j(rzt)s$hFg`ZiH_*_C6;!*Q;#qQqfF-PmG{AtiIFw zNEtfP)NDUax(~I>rID~)Wo9dx@<1 z^^=0Np`F2K^?Hx#Zr@mUNam-PyVOSa*E;EBOFkt@dy2oQGsN++ydBd{iCSUpYt zeq&28RYaR&H~mSlhL2({eCDCvAB;(vT_u*_PY~nvMiYVbCmmJ&aHhAP$Fe-}moROU zOR*#|eIDtgD`aXViXXmHxEmY(YX>0O0%FL88rHN&y_?2G1B)eo`9;|agBZjTD1!T;mS6}nv| z#XWACmDCx`sJ?HS`+)T`%{f+*=4xLJvnO4CnYbgB=sNAHCS_F{j#u8 z!0|C~k^QMV3e~9_CpE1=VQkzC>oT~)9W-?tD7>8t)^l08yIqF{Z`uWbn z;z5P{tK{=$QfvV^A1vu2M4bMTw!p9Z@^RG}Cn^2gt*Pk%RJr|I%`X?Qo};*h)=m|- zY7a5^e3@7+uP1P9b(D)PqT6V{ztL=Rx=rf)&3^aa);Ppw3yc=mkL8@l@ts7Hpu~h1 zDLHrl*+BJn9@92x-RZDz=eCqME#z6Cr?aR9@V;O7rmLp&WRMu~yzHeOyHidvww){6 z+I0y~W!#6&->y&@Nk4le+VlWt+|`Eza#vZ2i82D%e!I8lP>sN{!g{buE9j)Vy(|78nVq(_Oq4uo7gq#x2yZn9*zGhwH-96>`;}=(>*0P z)OMvw-3N2K8QsML8YK1DnA_IA39NWIPKIjC;oCQ-;K`=^cG6uQzv8x^>)Aa;Z_)IZ z*0}r5zTK&(mU!||+*6C=rW$WG^e#EPd|?f@&oCMFW(7MoP`1dY^l8CvM@9?S&3ah7 zCT0Hms!mOSuj6m|Pg_3XG>_BP0{nxa3oH@k(2(4?6!#eG@OtzJc;^Vtd8 z&4UBCfu)f3|E2>CI-K+j;M1{?hnpJZ#eB& zZ_oS!&Gy?pzjeJmL*2klhK|vf9`XOn8jXF0ETZVZbiZsH8;M00n3498-k|(Q0$c`@CW$JF z!65Q%$ZZNn%ijr7={_;=e0yG3ySv41r~m4|E{1(8jNC%8e6BR*3fXl`76QxZX>8|8 z*{q=v0$q^yR6rZQskVVh!(-!SFm%-k%k++$6gw+#R>7phx*Rbi>a&wbVm(`2+%G;; zU|yu+n-p0{hQ}eE{Eak?>*l(je;0~SH9YR4l`Ne1=GBk%+JsuqWHXc)xMg`QEJEyY zxIq6W6`DGLu_Y#!`e`H2*M4)h@5XsmlfJVV%>T7;rVi8>+Jn^>LJ`lv}gAi1LX}+Y=9CrtU zY&+`e_khnAtZBlE`&Ut$7xf>n+a%4qsG3ZBT2y!&U9Qe=67cT+v-XX>sLqMfXwc7nK&UY%K zJ-!G`W#AdxK3t@;{WM3N)VaP?Yt99plxq)b?sB7qELr9r zBimDjP*a6<_m#SNy0Hbaa?0$Wy3TB3n^#Ldz7OG)Lfl6aU`p*U77n4jtOV;H@GiVO z4sps(XxnEm$s1nLW@=_jF+&9$rWeJ}HJCUMNm~ zZ+W_)dtn1os+o&9C=}BBXC*_m#-wD^uRbC{h$0?eSh~0gI=1oE>}8GdC)Jna>>CH! z0?luo@VZZcvFPz=TPx0}st@aiQ`TGYf_W3`L zFJv2^H0=eFr+dD;+|>4qNB|8`ZV_4lpz_USxidXpPziZwF2|qC2y;EYFAQiXEY<_9 z?;mVEv$V6A+2fgm$xfmW54YhC*C0{@+;0lLbIG9qGyXiWXfyB-wsQY?bJLAbX z1MN~fYj;*og#|a)Eva)BND4z$9B(jNW6E4TLttQ?N0Dj6ZZz1aH&mlXpz*a&9wlb? zFphEVWRuDH)sFbx`eB;&T;pcT?Ul<~xvZwH4n24a#Z%8CgSdxLDLt^dC#xe4?~CV3CjL)L{;KBA?iolz$(KGrbs z{WS*BT|W(jF2xr&+kP$^os+L_EU!gmI(w1|om6=S2UL4P3FFw?Q$;iSuaaDLNm&F> z?p-t86PDdR;uPB%0fEoHKdpt`Pmvy|QMMsVD=R0Z!A*&zgMKXGh1k+?ldcmG*?MhyvDZPbstrt5AdiO+`U)}HX~Neul(Rv?+qCZWvc zLSf_+Pvzc1xa`Sy4e8L%Y&_rBwe#IyL;iM* zzQ0flU1Fv0_gwtFsd0azk^Uriy6u1Q0Ju#8m)T`(TDNOW>g6WVt{{)#oQCj3E32Km z!5iY!MA9@I&Dtfty4C5&dolgV+H>XH=cN2P+c%I_PT&$ZnfoOP;}tY!2k79=8o`;V zJf+`<)xShJe3yJ9*`cmV3->5rRo>fq(HitZ7pZ5~!zYklt3N&oyAu0F*1rqbTbEAU z$M=PGOXEb0-FW4Nd&TSM6)H>B~*fr{3g^ znJSL6t=+7U$6^LVu_XLT@`lz{kqLs%d zNjX3HtZ~yIJn_0`KI(|ae4Q94#K0YY(&sk6(Xz&WPzFA2QNBDF@gVbhpZ7gI5#SIW z&((i%V=l+r-@Y{G%YE!hebX5N_t#svv*>xJ*AAl$$qe0iEWXj;t@&>jEqQ*aofT zNnG};e)L+;p1QbgOfDOa-P1o=xLHqb;00!*c>%g&Q%ID*etlgdiz`*Z3BvPKK5^_?njcYq?Ino}2cR z#aY|A@6Z1=o{9FfmW*m|OiD!`dfI`CQj&weKY0M{>wsS5MYDGZn_bW8e=RIiCWCw9 zFSDu6Z9C6%<5DSyt&X4jId11BFerH<8!g=svul~!_uS1?!29`!a+85^S-q7yGOtds z8aI>yiZ24eSR3Q~LKR+U;?FYiXLYd{0h+BW1cNsJtQ@2ls_Co$CJE%pJhO9cA;gCp z1f~z*xm*ahR88E9r2XMR;Dknfmb1s)DADwU0jxPTLAM*O!LYfg<{d>kLZO@Iu1x+3 zZGX?%)|*AHjBD*`kn`f^>Bt(@oU5tWu8er2LZlnQN)hSI$c$K=t5MNaqNWdNKMoto zyDA2_9D|H^e2QbV_W8k~eMtq`Mhj>9I!3BamXjpQt?>r(leGPQ4?GUp;@aOkBo$W( za*N!o?g{96a?6WgB|19p8k>hP#sA0Nm!&z5>&(7uEq)NddH)-`W<*e=Y&nPO>XX^` z_t#xX%O=4=1PEfBi$~ST5`8_0v&&`LMo^zI8sz1(ABG-0Q0hbnsx2Nxfg&(*Ze+y= zH|aC9Ddg4YBdAE!W{!GWw%FiNb1mz6ME51ak!;y3DcwfOzw_bLZZEy^ul^%6-A>A)63x6F(QIOb zB;iVL)TCr-VzdB7&P1sSX9ky+4gedSB?K+T+bfRq*rlJcX13gQ?A1NS$~sP88fd$e z^D(iRZd#U~nGY`iCwHHxadOkq+Jk{+ACng^2cs>w0IO%>WObzkk*t*Ed5TK^ZMwvm z*l1g?5~hoT=~@LE)hWtylVEDFFIbi~zn%OnJYA&D$@__C+^k-37Wp|J*YmaAsmHMg zMtt+dQqaU``{GB5#^VVd=JmBQP&OXxq>t$iYRp=Ru|0dQ}@2FRC_Fcb zsGJ{q5I<%6bLE$>akkbjiq9@yx|?y4T`6v80n{YGUxp!$@(WUD@t~AO32~p^ zeBndg7ku67D-Ife~yvrn-5iE?!e&Nx;*sg;e)6Q5pM=2*HxHz^kwM#UF zUN70pj&(7!9v=2p822|JefVIX`2%|?*|ARRI_>K`%rSd>6h|zK=n48hZ#7d8ayCt3 z`v<8GRxag*Ex5Q&cw9KGLYmG*YDrlfhu!fD`7C4heG)Cr<0CV*9GM=MRXMSr(+1en zxXSOhb0peO4<5Blz8GxR5&TyHUw2;KJUy=ixRP>XJgXs`Imb__lgm8jNak#~9&(cO*AG@>o~1FN#rJ6@5tcuoFFJ|wxFZA>gGo_g1mD2b((LDs*9pCnq@ zw04i9aiJnm{Aw8B_m+gzl!wyp0=h81u!$hFlA-$><9d0&39 z!%BV{J6>24)G&pX_LxU7%4fZjRjZuRvc0Bqu(YRcCO+<`-{?_TC}b{_A`5D<<8FF3 zyS)&J*K#?rr`}Jqj;A(fsqynd)o`BhXlasJ&t-&dFb}@ zR^2F$d_Zs0t$sg8FH=m@rJ|o@ffQX1;+n3#Dre>L!3K-rbY0uHgd@n(RXCj@IcAaI z?t{HWLE;M+x>LoJ&M%HuUoI)*u%8R+R_kvMq~gKSdHk1`6J;#O$s*eLYA9A)?elzY z01D*BXT5mWKztcUV?7>1i&NdcSfzP751iu0C+8M$02bzPTb@+|<0T%d`%LD#DlB+7 zpAeMj6$$ayxrRaJpO2wqbIkFM%5&T#sYQ~$6>>WBrzO5VM^3W}%Ze-aD^C&^TY3C= zMy;k$4)F!$qi9biD4iHBsf8#<6WqkLY82K{E;qW=Fp~P9NNu#cUP8*XQ2F|}{B%kF zyA|kKtnHu5vQ`f`_bD`rr^aizwjP!|wMd9pBr_dZG(|@8!#Qo@A?<-LwEWBjoKG0> zSC&8EPf$_(nu#+sv4NEPj4T){qS=ggZA(ZUi<1&?Rrb{qjl|T%Q8K{m7lbY*2GPXM zi7-ebAr{KSQCV5uqD^<*%@!_=$L5OwGFG3d-Z7LK&ilf|7n<17qhg!^p-i0DR_%$z zWYWVn<>{xTFmYB^AKv}Kuq~CBEhrwz9ymqP>*p{AkYK0KW;%teJ#!r|cPHPrw2|#%Ivh?tFHl?FwwG&NA+va&^J~6j9DJW|*d#DfvYtn^ zAKgk2++2x%6j*R`@n&SPT(Z39h46jQp~#<@arrgV;%3pQ@Fh19J`3H)bjZ(H9n!W){QDlXo_T3K|lsu-{%O)Q~=HwF`T>7EXwp$GOKZlExe_ z1qs{6;@h*p?`bPhmbY9?#kYDKNAYrFaxtj+Y&Pk3KD~Is>&$StCL-4xI-W13XHKV{ z!);*|YkD>p0z;!5$`^jki^+`+Y1vgYUP73q(3SXRh%g6L{%-iAr|xBP=kb$kDApi- zbmYYLjJdPrq#bz*_gZ)sH5?-==S_7~0*CoQJU!G&mvuZn3dU?nt{8~PU_{wAQ2q+SFHw7msX99{4)2myk-y9W2*?ykWT+(~eU;O_437J?;c(7_?W27&}a zV1g51U~qdgU485RsJeB#nJKOcOLNXfoyVdn>ZXox@Xp8j z!|E!G&=V=4xPXV3TyOkDf+X$z?R(#B520T=ZtzO<6!V%%vy6_u%y)i}bKHN{u(CDJ z@%48xxsKuZs`a?3N0I*)o%2F5jlio&|DQaXz}hX0l!GCQ??6o_@3l?8%|!e5t!Jdy8Z2Fxr8|O~{zAR|1>-91A5=>Ti9 zQRba%;G%>NS7i9)b8zekd1VSv=@(Az)v7qbgq|`)zwdV$L8{YuDAAHfr%(!%R)&}L zX{5b|JSqng!Yd6A9^f=Qfs6w|>V3cABDM|BdPE3VDwy?SMY+}f2C=v!peJLPk_5vN0C%9RFkrMw~F z_^DR^;$2qA%mgkPFlrnfB^?zVFOocCv%$!Dx~EuapDp{$h$WsOhNl;U*63&Y) zc)8W-vsS?Z^4HGOgiaP@EaV^4va!n2`P=5P0Pe=&HZuZ#(0|NdRii$Id@ibeOcA(tpP0dGsSJMbN`BCE#- z4ihqtmB7acD`9_>mv3|U-z)eT&}nD;NtaqAHX64rL{=)rNv-VzQ3A|jl3N##2Rsj! z?32%#Wr6~@EDN{He&41PUZ3NxUFB@ZkKu? z7VVWBPX_?E!`G9i9bR)WfzV&ZTQ&5tPfo{#pVAgYh)`F{&7W+2SFy}Z(mvd*V_6JX+MR|v@ukk8>^_g;-N_!;G^ERhm zOV?*ZUe7iy)hgyI+aJ_jkCsy)BN1T&*FLS*Pqb_%SV_M2-yCqXUdCmmC-4%g2 zmMoRj_czacZvDI9a_tJmT>8wNNU!0Z4|T^8sALKF3momZ(KN1jKrSZRjfDHj%o^c- z(J7uW1s(GjEXE+@r4>%_f7rUF+}PRi@u>q|q3w^VcJg$qmAYQ@G->-vluHIMRrAN_ z)Gc+ekyXnON6A~So=);ii4<_E1BjK8iJL25`kUP^Zgyh5dQGzlVJW=t4Zd@RuM0HY zxifu{(r+ivQ>Ao=fBBG{?Rz;HdAph8-B9=g1J&BQRJMYzdprv?;qPWz<|`=l?BiLT zJkJdisKU5MT#E7W=GplE%V**bJ8s(_E3F=U?&~#XjGqaC=9~y7`xLlXkv{LO2$PJo z|0K&q^^y9s@;U@9iZN0qedq?Bp81Ol-eBL7pDx3l0})Ogf~W24h3EN6Q3NZKV1HCS zY0=JPxc40t-|?u5c_HU1)WVYG{Ay~%Jou1y1 zkdl_xgB3D{T6TFJdX-XIQgR3dGEVz|3zVz+Q$hh19*avG+U>cC^ zC4t2IB^i5?uvdVB&47T{>~J2hn5~F4f*IilFWU-&i_>-3_G{Ehbeve0X~?2`Csi1S zV<$(!CUuPvR|DpH0REQ6$X1vtZ}XJ?4*TS14kz7Yu7$?tKb>a*l(X6Z8_GD#b*Ii} zMm_z?_=1kquG5QRu|g*AxKQePetDTmed(Y(iH=rZC!zzI{>kjnlyKQH|M=|%^CFc{ zJ=3M)d-L4UV9xK$hja-Bg=M;CBxdBS%6I0T`=`jg-x)M72b%pmY=5tWuQ~;XcJe*T zt+m!{oFMK^YpdlNM0dpvq;8nM9YFr#Yuld8IcaxKr!F{W82_$l_|N8%awCW#{TD6O z_+tJj7m1VEQrkDKr9qvo6CAEJ;;h-gG12K`R}Z93_cu!VpH%&3jl{Ov)Koarc=j{! z=z}U$6YbQmqElF;ztE6zKDcyFx-x*|B~LgZa+yuHIX{A!LtV!on91{^?L;nbi0Hnj z1bFSXwU&uoQ+N;w5Q*ay^W21+`1_oO;h)qTLtlmVk^&p6X5oeJ@ zGV>nw$f3X_goL4rvH+5n-f-r;!RPZTC_UYF=L2G|>+`A-r|NdyqDsma;!;?jIUKuB zGKGznwxY1*`YH>yg?7Ft1!Ru~bhuLV)_3gt#nu&4cRu^-b`AaBpQy7Rb!U*+;{*m>WL)?!G>hEczIokSMbvn-bH8%6X&lA9b%?w;L7|jys6;uQTU;)BU-Q1_ z$fk=Twd_WIzW#HQD9_lpcV_M~$6vI#Bn-`G2-gYT8eiZMhYhba@SrI4TlHK(B7F?@ z^rMDcwT&i5h&;Ene6@rN6;G91H-p|dc~pBfrTm~8!<5fe^#pd1Tpr2JG+~SMFB)Cg z@YFWFt$+Pu&j^7@0#l?K*!6O71)V({&aED$B1Zbvs1=)f`|%4(f&;cHtL5S$2OOp( zg9YZ+<}Y*PCzPK8oCR4-?K9{9 zenB<)nWn0C2ARse_jj69WgAJ%b{UShUAlFQL$coGOZoEfO{**3d{q>W-`uo?gM?~7 zz2{OO6j)P5D(pHHHmBygiZjyVFtn*Erl3lF&rZ5)eP{(LJ zRWgXq=h)FN%~!5gP(o4)|z0Zjv2B%v2PoX=}5K zrt7Im@9sM{Yd?5ya|%Y)l1sjA*}Ha(O8t|!PyFh;m9ic0^B75ZhIc+(Szh(7fpDB6 z5q|r{qoznalIBj^6tB~n3Rk6xy_nDXIh^c|FD(hoQIa}2ayN+JczOK-ZB}!u-t;uC zj;!){Rq-{8#n=W~f2Q`%B=KXL1cToaP&-FGdPfd7=S*%cPFmOqx2SZC;6}9*>am>J zaF9+2ym6tMR3B6cT$x!_*z&JVzrNWuP9II-AWp-!-cV69G-g*1tcoke3ioYZ-6X+Gm>^D1}Xw|A@Tyf>C?~AXi15x})oP8{81=0{5kYF$hQyt~QBH;#@AMW)t?Qf1h>scajx3(~oC>G+?@o#H zTxf)!1BN)ixMnB2#*ncaRE$L$ruycdFn_Gh3s50)hQ3C1Bldd#^Ex!jk;R!zDF3T1WUn1#vxzNMk8;JSnsNV!ESJUKN6`=2_bfOho6M=Eyzwg{$eTYyWJP|n^J~H{E_1LkV0yI5kI$&igE@1u zZDA*KWP0lykIGBJa9c-wD(blncuK2qaVb-y$}No}0CQg_@}e78!;6e*ossjiun$$F z#zxBkJGt^mfXe2?vg^vk-Je0}HW9Ol0uON>wZuD}chh>*kL3Xu=_=XbWpd8nM1~31 znR^5(m6l)d2AQg;1V`VlK6E#ltcpi|rynUYk)=;5>nY6z%9(n9#!h7Br3%ye_-w0f zo)!bOV%a7-y}I>Gf1~?DcRuH?wfN2veS+sb8E@s6jNzn#Z4Z+;$v+n<<2hk#&5o%e zf^jkGbCX^(K?($MN`I;#qfIFHDAbH4@1Du4&a{qpQ2PiDPn6NjGmL$vMZA-Jb|YCQ zHFz^76*wx&8aG6C=Ld>Bxog?KktgubU=zSwYI$a!i1AjMX(PzNX){4t$6UHDjf@>@6d zl|}l}`M?jcIEfdVEUjO3Dt+b5KiP?`F)N#Ty8$g3oYr!5x1(`eJ)X%`dEt9w^Ss~g zDZnf)TS-eEJaw;;?^^$XHwm1;)4BD<)RQ>c^qf@t(^}7M#%8yLXV!S9>brC-O-edM zM%+c6c`c(D#hMmkg3@`?-ee}lXGVEo8LO_=#nKt~O5fC%KBloX5+Nw(xM-juspf4bw7Qm-ThZTMlp3x<~Iv+Ek&AHe%7 z)dP$3P^aO$@AKoU*=?&VIl~9wlZ5GKa;TWaFVDsG>VPV zQy=>fX9>KBX2T5bCbDyEP|Z8GOAX~>#Oz&N;T#GzCTu63gWtyexblI~{V453bMBj} zifs+vjhau)HOeaBx1v$^u51xRn4-`nvyIblc50qoRp$>~!C)yXzB+@2Jd+pcvN3e2 zeAtRRW}BEw7oMK&*(AWfV#fN0h^P1KgqaEc+BCyAGZVt(p2Z?U$$rSW{JfwagqopT}`EZq!NX_Jocnhr$+j|F8m=*c{c!Yj53xG3;aiY5S zDBaL%1fnyrx7eVMU1?ye&A?ty$@(Sa*P0ejckLl{aL!wlyx#X!p@VZt2(b!RNGX?t zR!IspIc=?Xs?2Twi+izLeCU2y{?dTm7RufVFIa`CBCwmN%a zzxt^Y0P2`(-^8AGsZvp>fc4ttG{T1zY&{$DCW}q^SVpQD_^H)d$Z*H z=ydU?OrSZD1eyM~9g}g4;WQJm;~geLz5Ls5q@8a??`z#3=RGl-ruayi{^*ImOx2wb zUI@ci!whZ`rtUNEsOK_0KDpU^Dbl`kZ31xEAzl^={U~llI~vcU1Vo)6$Bg?mhTO&s97*nr>Nn(S^q!NPL^VKd|xXv^%y_Zlqa@a`)A8=e%pV+x?B| zW8dpgme;*FfmB(g;iTSW=`F(&H4HoNTP;Wr0xE50rIR05)bf6{=nZD+&5O)eKX^uL z$4=Tgy_u%&b2#fHG|qFa^M^BmN7xTqjjTcUVo_}LcIde9;Wx|hG5DalP&Ps>w^`Fk zVP={>+047M;Jn3PUbXL*x~%;y@LMP)e%XiUS4PiG?nYeWB%8I#7i@^F68vewp<-IH zm(kjl4+ezHpl+>)VNDd&1($H-{Oc7)Qgwg)z3z7t?K_u&Ocwy@$% zO)IOr?2MPv!8$w%yJCtu#u$qkT)qI433sV5-$9Q!P6G?GzSPMJERV}vX(7kT3LzJXJyyn_>w(eSAb`P|aDh=<&u(gX|n&TsC`?<12dQOp?dz+Co zNngLzY}ksr-hp4&a?MG*>e%yb*S6GkkkhMDl~zR1K=}O^jvMK3?wDO6^IN>j+4!`e zqrQ?z*WOZb^Wqvo+!X>w7yD=SZ#65FZ zCkt)eI`}h1z4M$!nxLaV{Ba~WX4|OpjO;+$&ID)zpyp(+@ij*GMP3f*G`ez+JsG$- z-Z6?5`4jItmNhMVN;%!dV({Ii15*5*p)h8Cd-5KRhhI@)Fz{p>*YN1pc=cO0dKA|s zQnvki7NT}T{=g2CUoRGLw~OP-{pHeXgAsoFX;Z#ss^FIhdGBxfom_Mr70eU^o5N&Z zf~B2@%*C;vYadM`Q&Qx~i1LelOuJoueq=|`+~La!M1!1qFvdHF%k&p5O1|_%=6M zn*8nnZ<fT>uADnIE^uIu6Ax^R$4@TSvqM@)+XCk<% z%f6Z75#Fy5S!A8>6h~{1Bp?*^H5pQOYS}h?ss5G1;x|ZgtiN9+RRX(m+#67gC6_=u z%_GHjRq~uO;6wl9fOp&J5PI95{hN$o`V)7#!qHC9&0Q}`%g$~68Kpj_cI(+lEf{j> zW|)=1bdG!l(x998tS zg|W%K0;RoLz@ZWD{v3G{t!AbvRm+|8Zl!3!o!sQ;gD0|d+C%)3E@!Zc?#(Zf7xUt# zvdTNIM-ydv%q^k(U-=eFSgx3)>HNm0Ug07($upJ7GRLp|Hr13Z=e0~J3OtgjsE_vvEc_Yf4bD6KyJDk1W;MVigfYFs=A3PQ+PPXMkq zD>&2SDf`j!I$rmU%SikQ(_f&)O_-v~K7HK;f^}wj&a|edSDfcP#d;;ay(s*4Bq&Fc z!OeD>`O2%wvLQy(e@TmgP295r;z_XZ!Cl{XBLmUC0Rib0eMD!s=i))lhq>o*YP&q( z(WSX5;~_aj(*u{vdLFx`N;F|BQ}&x*C<6h$hs);S$HQsyoFxyoPwx>;oZW}70bsz@ zm)z%N#A=#JlOJ7U2Et_;%|kX~TLWJUl7DS5G-R7hpk6Y(dM_dKIkI_}_H5($cW!5fs8)F&iI*oy|l z$GaB+sR6H1bVQVhvnL|n7`eNuuSgm}zsrq(BafQ>$2}Q>X1}p!F_z ze(eO%IXvuL2gzZhQ-q|%H)>&b>U#f=EkU@~zUIfojI|QuC1MhXRLPBk0%yzOC_Ca3 z>%YV$3Zi;Xmc@Q63Vgr+MJAB|ysmt~Yt&kdL@o8$IVoPG9j1&tl02R@J$-nrbNOqF zZmR?A#joqMBNEB*LPs98L3=7mP8 zl#3TT3!K5_w7yHkA7pH11x;PY*GaTkvf1?ZDcSnM(xpjjc-fPRP@^i#^{Fb2&x>F|VmRhYjWo6WriBjb}TwEi(moQZ6d;nh#? zN>_tWr_9gswRB32f_Y!YYVc}5v*>hog}Zdn_i}UQWwJY*@OHM;+i`q!^HND`c*lw( zpwdkJiaK}6Hh+WD9zRPVx=chab#biD~IUV4GJ#WH#4#mi3*+Eq=XKeI+U9qHpXCRwGa(owJQB`@0hG)+IXV%Fjz?_(j09-H>~!l$One! zekz7BLoiV-PqK9%o$zO+ZKkx=eN6HVK7a!vtn7!{-!dR1fjs-&i6Z)n1S3jV2{x~C z^|Ihg_R(dA;EBD&}Cke36`! zfqg@{(g;*0H6;rfG<^M>89)TF{wN(9zgmZd3>Ln7PSaon72{o#0FgGcB>)#ct;2OF zlAe>oZ6gL<2BW|=g1_nD)h9lWIVGZa8 z#vRWnZLAAI>?`&Z@}_F0d+eFfqIF34tOoT3<~-@4uUI_8J4%QVbx^8sFphrCDgjB6 zJE0J@3Qz4t&g=OUSJn-_u>V4xlFK4BYBiKbk~B|zWx-U~ZY||R9?Bq5rd}ni@&!>G z-)-k=(1I9H&KTjNy`%j?^g`#4s z#hA+=;xp!G4Md>pkI(2h_vl30OS_28&R#$ zA;iWH%E1|mpkc(|DUCr@G!~L+rR;Uhl(x~$)wRu(x1oeX)K=P*!Glrr53^(2Q24~` zzeF1g0j}_`V5Kn#TA*|d;uN5lxZ04Dar}3Fo(orNOFTIIUKoF^@x?$gxUdXVfJO|wkSBW}^fZS*F+le#HS)bfcR-jOu%U_!N zFNQG}<=B-Dlar5B3pj{_Uq^@XNiVP_4>aL}?Hs!R)Foe8C*ictk!aJZzSwugtNWyS zc8ZkFTc%dAX-Ip{dJeNmQ7AkW`S9c*bLfh%n%ttSgzJQ3`G}<4Wod~Z1)~iKR{w*q zg(gHAZrCYC-DlQQIF^kDSM_hvkMGc;A5Y%)FK;?TAq+9fvTDzAkpA&NXhDoU&b*^C z*v=BT^_gsIigfE54Ne9LpEDqu7C&v#tLqWNK_6rUkfkx zGOfV~az^NDm1tB^wOtDU_B$)wNIFIe9;h!tN&Lr`BhuK*5WyzCR8EPo8^5wuk zEiluwvrRLKO}HZIp*>KH{?{#0l)K_9Zq0%B0v`P+B~S;0y5Ez74OLC%qgTRF)nC)> zfQsQp{#Pz2BT%USs>j@1#BoPTKs==aISnF}Mm1E0JO`1=qf!DQEW@zU9D=exPQHd> zlqn9>_f(!3oG*)t(0TH=90h0<7qS7qFxq_BB@X^H>e;#CDmr_ts49;BIO^DCd(wR3 zG3wZKd(sRh`s;A!VfNBb&iA(-4N(I0Y!&h>geDJm>r~HN?s8~Mj3Pp0M)|i~3tS{S zoU|-DG?q%OQK+v*IY1o|ITgaa1aT!##a0G}+jrF)Sm7vVFTBmb=7VF9RwJ|gH=HuU zfBgbyE&Y|ul1Pd+1QkJz*&F~r38mmUqZl`B7%Hw>jya4-SVZ;Y-UvsfRYS1=py0!Z zghkXyR4%m`3mFW2+Z-5?-=Y30u6AW1gM%NI!#xxMB@!Aymteh@1KI$IY(P=GAJ|zB z;Amo=T<>2`gvMB?b+ew!iRZ8W!p#KgD~fDw(_XJ^TG3R5IJ<|wrH~&Wjx#c zyS?Uvou`rqmfQOiGB>v-$P!5;=i+U>gxpF0c0_EiJe|1$ULc7f% z9#^553K9CBM7uT%a6)L%1l7i1bE0?7%wPmHj8b_uA%U&#lcxnYx8iYXrhUXx1 zc|~JYP%M{Lq%@bl1j9-|db(1pvoB1pc;VYP-R^hDwcTa(0*is1X)wE#g`A2 zSjGdLJ)c3-o#SUINIdUiVEGOs26t%)Dl&VrJONqGNq+W(J+#AuT(rPG{037_d9GD|o zKAHJsnq{a+a2548X~4F4O^>K|&cY#{(b=m0$p6jTFL6KqUP?btVI zc@}Z7zZ=6P!*Za1rp_$f+lff_+)-&plta6lL@&Yjjyr03MasaJ4*rb!Mapw8mHio& zr!Akb`gFx!t|JUH?Bz0u$3I~{@$@RY78p*iuuFrm3&8Lk1gd1b3X0Xzpi0infuHRC ziPWdd_KY0;iIk`5_KXPB4k9A}!)XwiG}ECc zoM);8L|=wsr8#tEf82Z+h9*(#mtjxSD=b9!q(DVIZ9g8B8eW)G)_P_lz7Xh%sUn_* z(B;9@q(bP@Oy%Y#&4Kau{IpsN*b+8C${lUT~&;zP4qIjA*HR`K=3sQ8|NvkkOgQ%pL4nN^MQ)M9AG7Kxvu_^l#=EJaq{|og-YakTAP;O+{ z({qI3iYF!NP@KsCRkAoxWiaa*i59Pd7RFnr#`wcdUSEK?LmcuvncP;u9 znOpu%s9tf2fh$3*AYG;N0$J;mv#%z3Oza1Nr7_frW-z6?=F`>8ZXuI!JW z55o@r^y)6wKq#hHc4634cZA`Jy+ppARCOp$Wavq)5Nv+JPf7$}I1Q#`=s%R^FL_q7 z%Kgob@zq_*_SE@$V%1&f_S6}AVk_DQssv{tjCnuhfLou)2wicYnKCp(?uo5pJI7Y} zyV8M&h``y~L#r4W*Dua|$nV zW1u&}6o6?YAXoW2^P-ynM4~~r{EqVp4h>$7#9Rg*AIp#`o|%t(BNUn-$Cv{U@mX_R z2PU9V7P`)XCdkm`szQzyAOauugaNkBfhNe7IoyL0WQ?n@Ck(sLbq+K^hAvlW83}k` zPZ*#h@Q-2nOMiQuujEhe__xRZyD|8GA$r>fjX7Bk>C7R~r#f)hEb;$s?Grqzn&~I{ zSl7swb%cQZ^VrMk6<#<2LnjsE^*L?6vxMa*Q2;igfYylH){?+h1^JY>J3b$^4s%2_^v%;1Y1Ze| zsr)E5$irJ;6>`jt7WcqbDW55r0C@0ceuE*HbF{uy-5W%PH=1+Db?E$If5|1;{(D48h9 z3$aWaql>NL+RG9P%G5)?=@M2fr6xH1|AOg1!g0$9~gXqC(nUO1F8L(Gwf zl4EXF*dNZOvEkRgw+}&wT?osc)^^#|3@n6D6vZmY3nTXE)Q~N@kb%NE4Fj$y3I}-3 zo@iC7+>~L;0=pP$2Ig1CWnS`&go$iq27C?n#d^pOUmKj`1dTgh4tR@{zvfsEJO>W+kTlqr3T zQ@wGDC>8gnxFbZTW+|(8lV0jJu8ji{+jTm%3H^k`dRnnjMa_yAtpbm>1jmAt7YCSm zG5pt&Z@NB7O4|M;eAporDfegy6(!Kr=ViQ{ZlAfN<~TsW`4w||sf>|%`@Qs<8Ize_ z>SyQ`$N484h2ZLk&x%a~$sJB38nQHcLKb*x!QY?d~;`UMdHMz83d-wpN(I6CUQ=A2sN_8UA@2MSKDH^D^_a&v~dVPeFa zTa&k9jCgUgN4%uNBnEh$4af))0a|gK<1nL&cd6OR80sP-U0>L1mzM~2UnzgT!BgGU zk27n+X^3>q4Yr^yXp?QBOT~pJ0yM*#SjwAlG}NQ4SMVGxX~!hL5COdSM`RG$<$lQ^ za?16?j%NPj2-b$!hh-R35{Y6Q6B21o1L%hx^?UzE6N+KXQ&UI-0uh}(?K5IJbJ}M_ zbgut#^iLZdk<9E{f_er^Z(MZTRvdT#WW$)4$du!HV~ixmV>H~Bw9rev<$2j+VHd=_MydUKLB&=|iYV+P-CJ4)cIe57NF@=^4jfeU z)FvEd^wfWjV*lgl|I!v~$2$!QddrlD!t^?3Gy1l&?@>zhlibVld7~eBf+zP8X! z!YxRUoc++(e;`VYJJ}@o1#S?B7+wbkL_qqw!)> zm(vevbJZn16E7V_kVh`Ozzrv`jh-;cu#_O91Z!ybj)M? zx_-x+ta^6OOt)3R7=I49Y5`)}~KQwoVJ*Q;7R2(rx+3RD+JnmlL-lS|gvlEWYU z$T26kZlqk?7I?PY);!@9a&-FJDd00`64Z0JaYbM^>=}@othsfiYd2g@F1l@@$mRVN zaJW0N3EtZsKCSolv^u}?glu~726=)!(kXkc3KD?Q@#yaqzv_z46<+k9CM97j+HK^# z61vHYb^oIz4yWDfo6gywu_S9^-a{DJeb#z2|IG{1Ox=AqW*&Rv*paoyJfM#~)4Vr% z6p?c-ypiNt&i7?Kz?r}v`FvKL0kB(fjv1Yd>L@1pdS~L=ZHgn*W21faR%P_I%cJJa zD;J+4K8Nnu?vGioz0Lw`t_?>5zOqw)$k}0=RMfooHG%&49rXox>Lg|BC7DhXXbE?* zj0Q~1s^;D{QmkcIJ!Y0?i!kXVJMZ)4`a@prSYru9ws;4L_}6;yB8!(D78WO&GAcKP zGcHLJ9G0L7jE7Nr4qySyk!?Z>z>BDuQ!##TW^9 zRLL08?$1iurtxqSJy7CoMSNeW;O_Qg-*18O>wQb!XL0QY(mUNdv%5To=d^JT(V+T# z7R>>tl*p26x^j_Ydn%D=@>x)7Ho2dI4Sj18nFa?0ib!Uh#Aq33c(MRZxS-gi5m0P{ z0|@me`lu@!ruis-qn>}A*zE|uWgyjHEe5h+QB*f7wlRsgy#-__ayFbuLGv|%5XWJIUWh6-EA3ldT~6@B}*LWtU} zQ>+MwW8XK7{ld4aag@>^NfBi=sd{d@4TO4m4BG3Zb4VKT&7e%?ixQ$D)j#qb`nIt1 z62XBY8L}@4UHQmG?`NEFy-y?I1+ zNN?B2Lj@dG7bpS3O)DZIT}kdB3MLH$(<2FH+@X;0R*A9aVd6iQl+HM1-lSShYS^kc zEM7=|N^U%Mse9XxAI)&D+ph4*wMt+POIg_j3@ul2jlxK@<6B^AVl_?W5#5%fhJ+^j*G`t?6c zi3rRsY{y0)fP8h};oo%q>lEY_ypN@cGzSi4;B){jpU+s6AQevCPSMGm z1)bPj+)4Cuba2(Q`cDRAl+P5 z9wl>yANhisvQSOx3fX0z)wb%Fm)O>&+LsYIhzY*#)gRrtRf!NQ5}~#M`|=)s1P{%$gIH#qY{XkZ7LrQ6>`-CxYd=ben5$I<*E!K=7YelN1$>C^g}Kc#H8=eD{^XD6n#5 zx=_jLlzwiE`rvOt5}F0V9SIHO_U*49MUXK}j;31je2Ccf0fmCyNAN*J3Dsfh>sP%9 zgrT5XFEpS5rWhvI+zrRu@&rZJsq0=s0S_^rf@NP7Yr_%xjOj$KwBz?;x`zDrks`90 zXIGf|Uld;&;*Nr12a-n9TL_XBSM`(xZ@^O-A(k~a(CQHqj1lgTulj3pBpi7X@r zW1D4>nhpNiO9@rBM1R9STy%jaqsb|P;A9mrij-4 zEHI2$%Q0HmDL6qqsmWx?T2~t~_U2sefm#+qZc71g+dXm4IDfyUIy#usI|t!@+49}| z<}D`QLJw+I#9=$bCBO7#ZE~5Vs`bPb56T$*>g)U^ApkN4rcVAnn5gM{x zRUDX2yd_!Dk#5x&lUE=w@L<_KRia4l)o6Km9A$FJ(BvoIGH}ibAtK%L{ZvvKQcWa{ zu~GUShh)Vy5y&pV5@m9rq;`&_6wgctm}Fiz8B9astHu{5QS?dQDA|3NBH2?^jEdYS z4ouN6Ar)u!OWV!y0QEcnsE#Dy@_KA0akQL7W(wa_Qz|Yg{h)RcYVE`KY~%NQyY{6(Fe1mt6Ny&HozE!KL$Q# zk#}}U1rY>^`G*1D3Ktc=%?73V`iTK5lngjgv{P8!#T0*9bYLaf7mNZYGA7`a`sPCi zLmgT0It~<9MKL|~3(%z$cykm;c9-4V}y%5wj6mB*QCSmGZQGPE*@;x!+x}cU@ zgvzgiA{lQlVN{Cv5!Cs{>_GqLwf>tTdu8z=v@H@)>h}Op?Le58Cer04sCIquM}Q|; z+#a+*1*N7odX3_*mD5jH@nFLkc~ELIT9Tsg_bO25t2e%4X>KA^Ze7&%cIFIGphb)I zkE3!5Z^dNb8E2L42&Mf-t;8wV5IUEH#*r$K%HLKm&t%j(hg+P1-wCD&Q`t-Sip71P zN+vx%_q9$b12w;WvyUqfG&)@vnTh|N20K5S=~X2daZ9lZ7v67aqlIE@P(k!lQ-TrU z7{Hkze9uUP&|OZnk;P}J#UHP|lBq(IWt6^hY~z>Fz!C*Jn_)&cR76XyNbmKwIq=LR8TE((R0>)O z5Fzqr%g<>loeYCfwgr)w|%rpgMo#0WC`$32B)zv7?gDFx6Mo)dJ8GxoTeEVac z$dZcl%n;>v5(<4t3VhQW34*pa@Bb(~iV|&^)o^3h?g&zI-d}d*W_=HK;q-2)dNgCt zF?tx~F0$R&sXn>fv3wA6T!{*{0BpD!+?}C5>P9+HhgXmFfq%F@014S8Xj@I9A~w@1 zlT9U``8{kHR&j`nPEdP8jz5m`&GeavRk=O*HEpf99e=SZHF@A0dgLW|-ml$}A8cn* zg#3d_xFonV9mueO%bj>E5h3&GF_D^jeTwho@zZR*_oJp*Ri7AM5ZOX98iY4z3h}Yr z%`bhLgEZL#DT^%`2Clc~P=l87;E-fZqk6lsS@IE(>UOazJ zu5X!CPbKST9erRc`WQV; z#QhOT5$ndnC2Q=xY9$aeK39Hog2))i@G?wuOQ5X$kK)WH7xv#qMHd?h zr)<)2zr@Ci&lb8HNQENh-qiKqzLa!MzS}s>KHsR(HZ48E$6yW6P^mu^ie|$7AH2N< za2?CCEhuJYw3wNhnVFfHS+ba!$&xH)28)@=LQ9q`ib!AtsUOU-exx(Y@7XH>GRG?u{d#1sGXBu=H55^v`S+CUcO}>GD>exr8 z$4@HziZt=%?NIrrOWrGX%&kC1&E^ZkTU@}?OhofY`a5Fl zL$X+3_?Jb7i>_yacoT@^c8stS{~Ddo>cxI;>;#ZgZ#XgW`mewy!$VJbw(-+&-TS(Y@Qo*OAiWeqqi?W|RgY&2x6R^ulyTvKT+xd&r>y7}dZ~y6=6bKi8 zhyneO!}l}J6Dc5%xPs^S1J?Jan4x2s;qURmZ3aEvc5GuPqlf-$Qv?!m623k7g7ze} zn>7mEyhO)4o=XXzuO6ss5zTzdO2zHGtZg*DFNS|x`4nJ@t*i?y99*i!2)$*imNv$m zy!G&Xp!+RN{b4oUnt#_V$t%Z{zEu1;?}~e@LB63XS+uC}ah^M(Auhj2rO*w@3|~}h zL~)VH)(u}&4^l~i;h_nMZo2sPx}df<^YqGcs3`r~E~jPwP%g~VK0{t$Z+@OzX^$WW z{{EYh&dRrWcba{5i-gj%yZN&FYp{E_v5fwkgDuw3*BgXiR?zqJ(kI8#glJQ92D}dM zD(AEECR_|lE(7y(OOJBdy{DptFY5@u?VE`fIyS+YN=M9YF1g_g#s;wzQ3MLc&Bi- z;29eOrrzNui2Nf512Ps!22M1RtDp9g%=ws>p6jiUzmk;C6FW1PV(pdC{h9{bpNLO8 z{(W5hsn&<2zF}Df=h<+^+W;&6T5F7{>hoe>6f)KHJA0R{$Z%?M9j;$=6j)7JvrbyR z;-@qT;LH)M4XHre*vl29c__5la(og^jJyHyu#91$ zdUp>&2a<$eaom%YEkyHJ;@`8qEPCT~NJjOQv!7OB`ig%(`ew8l0pBn$&pP%6 z-^H8D0sAZ_=)h_V%XKR0&U!~93i>cDABC{Lj#X!~H_zBww~y9iov_3SwlyeKxH&1s z>$5k93eV&*AyFOu@M!0U^ONZWDHu{Rcw``zAu4x`_ z=B{b(Zf9&ayFPVqixi6Xf z+hrPN;WsZEWxakg3$hb?3T-`sT|A$_3txuxkC@m8zrOEe*G%xQo!BqlXX(;v*Qc72 zc9v(gAmgT4&>EYHJfjU6!5BB`PHK0XgT1T|eOeOsvn>{AM#}ShCd3WbhW$Qq?K6DZ zjz4(#K=fnx8pP5h&P2$L7fcjk@1BN9+U(JYNTPX$L9N6mFaaP)9IQf zFt&ClG8?t@;&066V1XM`gR~mw@#dJrnT_h*|WB0OcWc+kCPy6e<C?@h^-E@pk*>oZtPkQ@9v}7{*w^-@32)ZxqtjClTHYUn z8FJGvb#!rQX^gxcbflhXy8K=8efcQQLbz3L&$~wsdFb}dY^wDeuxJlsZ4NIg9zsh zLoGQPDjd(0t}q<`wBQg-T`Sj1J72Lu13dCZ{k1NipK}k^lMNx?_RPrY%|(|0duoqP zKI{Z`P&R8;ft0lk9+s1x=CuvQ4G%RTIp|e}u=?`*-Ap?lr{xh*tqLN3qCF=xu65z} zupbs22Q&7=DKM6$@3SaOKFBM;UP&|X+B_YiCZvucVS9rQ?GXpxSu!)IDNtb0325-znk7I_hV_9%4Sf_NY<#CX+6N}h{<6XZFG5J_p573fa#QJe15}Ii+HsW?Erl9bE zjmV803$Z=E>~aZhtFxT3ZJV1klbY?Aggh2euBgh9>BB#h+-Ez@V|(D=cF4M)6%wC+|8k452%E)Z~c_L%T;4c zvLNwgN3kRcU5jD;C0P`!`s>S-$*0lhKr2;BnoYZc*GT6=vWBThoYz4(HfRtim6~vY zWrvypgHnO9+T<5|dWZnhr7;lBmS#vNBvYbM2+n1oqQe0a3E8=m9q{bIWvT3(L9NVu zOREGdEdiz?zob*JWK*(|gQ8zA3vGhe#G`t$kG&}K$%KVts5SmJh@NwbDxX74@KEQg zzvg3Gm}Vn?FI);sE(D(W-dbVX&t~?;Us%BD=Vz=n%8md}qjL|=ol?24Tr?@Kr z5LZGJgI5JlmMlhHkW*!q8WP<_S&&wxl&aIOdGC@lUn6JHAZNkO>h5OFY`f!ppdn{@ z6Cr2Ocd+P5`W~2k4U4q9J-=cML9mX&;t-wLGBmkjZ0`3Ki^0+qhSvAgLT7%4iqn-t zYhFQ+b6YuHHh+F6@k%ew=|^dW@NB@8Arv=K+-JmyI?o(h3^^pW%c?<=@1}Ij9J-)C zoL*~?IYG;MM4&a{n*Uq($9vf~S*?-_J(EmE4kOGaJO{2Sh8G}$y=Q%P!wgU;w{l(% zx$H_ji*Tpq87Frvi|~oJD6eu}o|uo(RAxd)u5kc7c0HF5OVPyX0X@z_uKI8OE0Po3I^W;WHYp+AOc zZk5k&CSOZCp`1oV_gguWrDEb)E4HhGjxK%PH|MCI@v)z&i`}t)S?j%}r&{N&)425b zokk`6T{N`EWOd8%q=OR(Z8@*n_o&HcfY2{*Bxg%UWWY0cF%C%<_;Y~#b&Z%S%j5GL-Y~!LZj&xg??9&;qH`E)jZP#Z0_Ad{r zE)ig$^KX`z&y5FU7XQ1lX+UE#^SKO~Z?@?r0tR(2%k*;o-G-0XGaSGG-^#tsPnyF%}I>_5Q((#ZC^f@#|va(+oroiG@e-- zUi|3DmFqT@53uI{b|lZd?Z3imhEr}vB3E?rz<3RK!%A>CYI1@q)9J-lC7as0KZ^H*7W1$Wkw zLA}p&hAB|0;&TJSQN5#tcJU=j6pf|v@HtZ$@^~vvR!?u0{){GV4aUo#CN*bh5lk}9 z9Z;5k^PghVk3*teE(v5+ISZUleXXvZ@?q9bIz4CDhMgCU}X(o zgRz154^}rb<5~@EOf6hY3(Y^G(7?+(K&90G%y`-LHL-iDIM8X*3(fVFz^YwZRvf7Y{|+7xT;9`V#PU=s4PzV3%v{>j&>SjGK);=LEj zOX!#XI@I#EwQ46)U4!gqhxlkWUi)Q0Bi8CU>ch(%297x_@^veM8Jj2pfp4)<4jmr^ z^D$t7F`da5bR@`Kcg6Wjrs>d8yTHzw8fKek;r7pDdVa)uEiGGCB{Q%vZ_PSHmy)Q(g&Q)Jpq)D>l$nq}Wj08Oee?B~enE z!bf98w>H7q=E@2sCd6e5Jd$Zu7O0~ojt2X3^QNZ9mDst{-~B*j@YYN-pI?kkkvTL{ zt4(`oV1<+~E^h1@Gz^!(-oPF>iEOmmxbZPGfKn05S1qeS=>S9wJS) zE1-Qtt6nh3DuM49s>Mkg^FHaLbkSu0S`E3sm$JFTvfQRCg)R9M`uL;mmvx(`mK~PI zxVaAG=ed=cwCAOo_{KM#W?!rytmR2m%ej4z=`^~y^_%I*@GEzj$hdx-%XN0Ou}j^o zQ$t~=18t*Wb{)*dKFq)`TNMxUxIT48&udpbO;=0$!-+n1|9`WFt0n$8l6%|TdS5(= zv*mGtSfSw&Aqzp2EqUBs9U!WJ!LP$WdI+()aRI0i){tV6nEZc zTeEZiK*_4g1-FvsmaErB?guX09yKu>zcpG{riw1`=k`zuo=Ro>A}-J zCzbe-g!26}o)gm7QpE;|RR3hehXV=9`Q{gC#8Gi8=pOv%@~8t<&Zrh@L6CF^!Sb1uC;hHb1{~ zJ>~{dq*0kJQ4`?o*6HaL3-mW%Anm@xB4?hMC!V2=$2K=qc)t@2&A?plj$ZH1n!>dQ zOOkGGcKJREF*GKew+j=4ni6jwM8@FF;_?!LTqB!4C07%YxB?v_F|}{ojqp4eBAv4E zq7>S-?Eoouf4!@_eN3nXZpEiuBb3Q{(;VmS&%eI*oSa11wEen79yi&SY#gkBSi$u@ zNz?qQ){;fyQpT6XDe>zx@CNxr9M{d`Yl5i*mnzui0?ghfw|EZs>%>za7UviHAI0&j zFAN9Y)~HlB9EUTlcHX~*jCp;=`SBiDc$pJk+7~57@VrWNw1AOQ=sU4mpug|;`J^OX zsrJs#QZ$b2ijyrPKQKmh#X3p&* zO@~KnllbPtQmu$o?TNfXvcA?#n@!!?{D`~^O1KLJu9O3mJLvlP zY%kTUlY2kV&iBbI&zP6r19Yl@tvx>a6ZY#+N?`Qn!*46wS(=Vqkxrc^RbcX zX}+=fQ7ioRja!#{pMOa@^m{Qnxv)LvA{~(&GIl`9Jw_(@plGRB7>Xr)y67*Xu($0YhQ^ zu%z_-9Wzp$zjr9Tu*cNX@>9$x&9J4RZGwxB8(j6$zRG9;98u}T$L%+|>0evgUG>sy zj^|wTC_eJ2O_^!899Nxodzqhhhi%o#fAV%Zz81GRyJC8Zs|J)N-yC*^yAcJN$mQ*) z1o>EX>c(1A^V;J9ZeN|q?3{?-k>k~-_Km4C%aJoVl&jK5>^hM-)q_*&))s!+5vd!? z(igVHo_nRCnvq|tMW@v({;)5Wr5|XsM_;Xo)VC~XE*n#yv^ePorGf0o0-BWtqd!XL z4bA5_#u0tRFeVFWMiJJb${(5+t{R+CH@2i*fphc{*uN&75!va9IviWHOd`^8u{Xzy@A{Nk|(6VXPaM4A23v1vU}%0rUWX zfM7suGMx6Hy~iK}>);1szy~7+1Z#v?Tqp*|_+OzA;=nN`$jSf4?VyV7&5Jl&dAFam zBDhGONV?$IcF!U9afeGcgoW>)ScaF0d|S| zM1vy1P{k-wHc9)Xz>=ZJBYz;t5@(PKCqhvm%adowGG-bw_n8Kb5u;@~GH(KQ{-9~d z5@i$&QH7{Ll}6kl0}g}%@%$ASr6}@b*?)!htIjG0m1}2Z-UE3@56w#<*gKl?T&%e4iar3+3$s^fLu@nuau1XEDtJ74a)t0Zcm>sy>Llx+JMuyY zOE%N8C-NbytS9m%YqV#mZxh-O4@}G=;1G#12c*QL*A*^f3V2?~&6Z)L*E!>lgO4{H@$mqXd0|CG6vNt_-4*b1@i0{qk%~m3 zh0z3SQ%Saj_a&Ah94+No*>dp`RT7noM5={R2}Z~Mx`Hq;laOt&f0C34DINtL1r`NB z=Y+mme}V72AW!#7p6DDiTiaoFaL(jto6OSPlcPWWpV&Rfxqn6gFhOEzvI13j}?V3e70!;eFf%pky7@IVJ z_?f)jf%pYuScvqO1MwUAkpuBN`IUoHwl%;;1l}Ci9m8+HZ&(0jJP@IZm~KVL(k4cd zF<*+71ozGnQgXw}h6f(9GU7v)Z1=i;%0K{{2+li@WyFUmQR#D~&4>e^Z!_e^qD2fU zXzX+4%#eqizx>eF{ovHTC4`^?mVQ^s40-5zCL?Zg+5M2iygKf(hAEhNBO`9Q4@GTd z%)u-H{RoE6!7dRQaJ9~egSQEO{PeqJ>D*Tr)b(GA`j0x*xQP6h{`u1de_g!S?~g8c zl>p|gfdb*Ahu%(e)J1i5d}yVN$qM>g^LS%q!ZBw5Lq{J%mmVN4ATC&d21OoQg*!4R z3}aOpQ6JVOXJ2b1Og&+S5qC1Hs1o&uDGh=KLxm!@WCs%|7tdEMRIE^Y49Bl@ zWQ!{Lw-Bye{GTHKMacYLiu_gok8nKUKkNUGjmKo^&dSvskZZUm)3lGFukJ+IJBx9& zG5?`}j`{Dx@z#GA`LDw5kNVHqTknv__aApdJWvKF8EU1eajI-*(*LU>Lq+{RC^F!d zVUZ0FJN&_jkGlwQ&DZKszy90`aqzdb1L)oq{J*T=|I`Cd=^&oE$Y74TCfHUO)7!Fk5?Pf5U;wx2e*IeFnLXal(ax`~#U zq=v@_JMv!DC{b ze^p7cR{d|1zgzm>%^uIv-Ic4kBG>Rru52GuTic;_a8BiD`~S+WuyT8gcr2uDphmbzRV{|`<5PobOSzl;1A;SZw! ztpAtr|3KqzfBIx~=bx*@|NB~Nj>t9si_q=&zl$u${Ucles0yG@|J2Wgq^03U_mNz$_Qz;Q0s$uVDjKTjz^&3cYpOh^V1@qmN4hC!2c zWA1hnUdF6vP!M1^8#sVQ#iuYAkkVG*4o<`Z5{t7t25VM0_B$Ya{Qi1#RKy6zT<2SI zf4$fJav$hMj5oH)PEz^FOW9ai*^L#CGw0>j0aqEsBM^Fzpj>gO9jWYoA-%_<{LqUY z#sH@5z9xiaQjwGSek|3NM?b*G(zl+b_(AhpdN}`rw37Gr`SCa6akav!-(9nlAVlY> zphY7ioD7KwdlJ}QP2w49IH_qF?Mj}B+o_e4#R|LAPhWPV?5o`JF-Rf=M z()7?h#++^sc{@yaOWB%vv9TO)Fl9ny4;AWZ>0;7`L5SA%V5itXQs#lB&iVc3?U2J% z?>JKCC2@YP0;NQ11?y>fePM+exU;vn?>P^4B(&ZT)ptx$B_T!I$_*|BnWlBB_>y^8 zIdiwsQA6eEr}J#TLNM@|#vCo(*gI-hhhvI%{GKu01Z*R?&;1OxhKJ4sq@gqMBqkO; zFGM93sG;oz++lYJ$+f||f^ zCHom=SngF+SbhuNU6wP)V>H~4i2rY^RX52qSdwTwFIjID!q{=k8WG#fmKHtN>0-2nwK*cSVQ7mlKi^2zv-yANtgsM zZik97s-%}FCXYnC>AYba)y%=+K2FG(UV<})?E%q-Du|SH%IXa=g*S4dy2xc16Z#vD z$8V@Fv^+d|z$)dgNjX^DR`2$jMdAdd5iupUB-dExZ^ysu0;z+1z;iko96}PlP*vcL z9M!!KFh2jxq17-oVw3tx+y&Q6Ergc2_}0&@XBH%>@4~Dw3wK@Yw+cfT1*Mf-*PB<^ z>YR9VzfA@2?$avH^jlV-y;(IyyEC3Hh7xhq8Pjvd-3uAau48HdpTSe)$u6P`FTINW zj0wlX-!1=`$uaWD_u=|O(MW)*(3Oy!E9%MvG76MH;E#TRPvodlzL*r#$dvk%k8zGt z(w>e+Na$CAjwz6YP8F@^P>M?v5K{?-o)?PZTADE6K zJGf7y{{a;W?ZKuGW6fA68i>ZV#{UbU5w+$y$n{cYkw^?>7_NtR&{ZfB4W8%~A+Jq7 zXDaW`c|V{0?e}Oe=Dc<>(UEXW9X+dcf@9!o59+k|qjqo!njFqKEa768)k4hkCc0Ng zFM61iBB}(|u&UYFT-(93ht4iFM7KGVOfM|h*Xa#tJa%qQ?m_CC)ZT2lPT?filJk*K zLypd#7xo{2CWY8Qt+xaHMRQ42eFi1-->cVc&RrP-mxQ8{#xTh=fr56(XF=M%qHEkE zMTum7!)jn#@uh7A{wc;}{xYI^ zD=%_}326_L5mzlkVbajsfzpc0O@n(QLN<_TQq${F4a>YNSI*3e_!}8*iMXwTIZrnl zlqzs6fgt5bkWI9}67tbF zB=0&Dt@jZF_nva(*P`W-efk)jUJlr9oP(iC6T=F@c1E+*7?$Jnp;Xhwabj&V?SaG` z+Ru~7i4;s?WK&)fY*Bh#xip;F5P6TE7b?u)V>Niw;!wWBmV%zQ>f6eBjE)ApuOdL2 zQ9Q)n!=%|RI?%#nly^OZw%+|^<)4O{BRQ-aSPrIDvB}A22%R~Y!@Foz)$Fe>mbEUZ z>JN13z{nEKjAKx-woiHFgf7Hp5mV7O=^K+r8(33Tv+E`*p3%Et%Z!?Sic!Bl`7t=z zdHm)xeA%Gwa4bqcSajmOaUQ4x*==L=wWpo636T1X38{gW7$?1|d!gTJ{kc>N(t*M0 z(j+Pf-N>e!rsj0Y@WQ8r8VE-SPEN_eT`XW3T^^XsihGaYdbTgoah2(Z} zOFZ3dt_~;UYnFle{LkzCJs0e(gM(r<&pgmDszVJ_3VYQ z=h%}W$x`9X#CwK%ddoq+Or2J?D@N_pn-*yj!62R)4v*HhAN z6LRO32Yloz1`9KlW%El7szLkIuJsoX#^p2e#{N&cn-7;E0f-}o%hu;J@R<f0LmtGq#@opil&2 zcJvM>3wH$V#>bfJ}5|Cn}; ztYBt9gFf1(Y5y3&aZ_sP2D6FQ{{c2xg7n9sC{_vZE&?2ie`AFLK$x|2Dq=-AQ~w4s zo4*;q_4X$!==?K13gA&|d-tcHM&{LiKYqZyzznm79;6^@lxbN$MP!taw9KC6EOZjz z{#csW60?Y1{bzlmrS42?o|Djab)K8hanLY$EG*7EQbC@R*mgAlS>`xu_#bQ;8r(cL z5de-1d@KN6Mth%Z`QR1;z8Y7XgZ0do55{DZwqNcn2RRKpGCotxNJu6w$1vt2(af0D zd&ba04gV{(p{zZi4;gfB10G_NE>#E_v=ag8b#1%?bZmyYlkK z{Rh%ZS$p7#kXUeOZJWY@6Ph4iU#N!Bna0DgWsg?xSKk8>0S|;blo5O3-oN6tvhLQi zU+lkMcGsjo+avIkUaa^ah?)`b87g4fS&$+46b8jTAd(6$%TS?2DKUz&!fm4lp8nv39Z+pDNlYnG`KoZm#-Gry%dixNJgG7B3*3eR zsQ*Cj!0l5-q*y-BicMumh%|aMXivo40%d1JqPVpD8s&#^Pq&jm`;{P;(`QmMh7cGhGOi2VJtj? zFX9_~hYn6?O(!+8&KZ0}0`oA|Ubd#eE96 z8daMD0gkZDbwZcqMcV0BoQo}VF+~|uklIplCJ)GRGr{lu&I@4N0Om~uNPJ9WA5;k| ziv30|QoJ16-D0!k0*>&t~k&$@s@AgZLf_ewz7<4qJ)@{MD z(o}OL_iA_vmDVkLU2*YTwl2Ts^~bDABa^%?}=lm zlHj;B;U<5n&qk)-;pr|B}dtwOE(Rdl?T#QuChVnbTDWied00A+icNkc) z2*P9-UIv`j*WI=T76kCV(d`g= zimb82L2B5be1_<@PIZ7sYIwJGD`XpB`QY?S4TtgsgHqMu09#Zq8um5}_=g^RIAD=v ze{6~NN8-t+;KKi9|1#+xdK#<$HXz&%keKWRW@)JSv&<7+a)V)LE0PVdq$`q*v7#%I z-(pQyq9Wj_T2gs=cNCG}B3lx@amQNJka3kANuW8B97*6gP@G86Ig*@gh}nyF#Fgzy zcg3~+LIMb}l3Z*G28#~lqqS1 zrDk%#{!Q>VbxF2y@(Hm}Ts_QmHKLeKYgZ$9BNt zaT3?ix3+C;1^K|%(A9Mf=YS*N3bcuS3{3A5z>NLmZ2Nr9%Uy=FI1RP20+krWslzbC zIFd0zOKdRL{r%`;E)W+t@~%pCuk6gH^}ZKWFm8isC*Zi$^ktgXrX5F7e(G{BOpWhV zTy;jXx({SqvzVqRhG=t25UF2s%}u`~Suw*!hBK%V0RkIAidu`IykeDU(1(=)---#U zvif3f4k#f%I;58;fjGbf9Lcbj@=RChqN_iIjWVlb=BCEI zYf%r#oHIA^lD`6P+PhL47tw?Q4#<0;=9^zP>%0{RlR_`?NtY}q4Kjm*b9<6Cc8nOI=7IloqMs!%zEwpC{bihYcj z+;&TBKS@{}yI(jauGi79NarARq?35dAl6c(_0S-9CeuBO|3plD&T<{nFLlk*IwPZ; z{<~B+>Pl60CDZw8C)L>AEI(79q7tFqDz?0>gm` z%Y@NPk*P4fj1OMCdA7DoSD>;8u=NQn5<+&Ht|=o2=`Nv!%b?M05nXzNHc%*+LLWB! zxl`eBdu**ei%wV3>((W>ASg?;hp{mv0AUS4d0o<7I%sL1%#^7Fk20Jzk`WT#h8I}+ zGYVv>p%VhRk{5Zbx{{N~NO6ufqA3M&3T$4nvKB5hA|XA6mSWP#YYwV^$ZbGM*O?T|P@uR$5^K!` z5%d6Vl$^5QM4)1HV9Qf?usCrmYO!edS3fWcjG?(NcJ$U_zDJUY{l=t{MtwNL)SXa$ z^~m5ZWt_*H$6#LO6L9fr8Brw{4XU2x=xa$ECXqpOC_1K#JJ4#93b4>to0+Oet38Ys zCJdU54SS24->g1$SK>9sP_qe<#WThFu*7B)mlfC*eKw@L?Qoqd;nKnZVl`!k5j!tA z$f0}(m7f3d#x472vWE>MKQ8#SdOnA-o6x=O0J<1>JWeme1J-p9!b9L?)sn7&^Q^a9 zdaYXdNuSF7oZW;Hq|!FhOkG(Z$m#{fwD`zg-;JX-srLs_V0i(HTB-7aZ=rOZ@(`xR z<&fHrsp!WANZxCN#4$)*~hH=bN#dakl zi^FQxy8ufu@?0Zo+OA7fZc(fV`jz2>fkp^{wDEp!d8&fap{)D)I~+;cfBky1XXPMQ zs$lMwKb8>Nm=5mZ&1K^-^)ywXc8UVaUiDI4Pu?tws5n#+tL7zQ-A=?8uKxtC`Nt1H z0<^KO{AHAUpdp`ilTa*6u40(H{ny|yTfLoJ&@uUUW7JGL>09kD_P}zk&5_;giL>O) zzXm4|cMVNi2~@htl*9GR6tUIJQ^7WH+R*&vqI$@$mPPrqMlDbk)&O~#8Dzo>p*lJ} zC#7Ux;$1v+hj)HVU`#+ra3uxKPJF6>x*37#vv^)C7Zff_P#+ln{y>e3a98GU3&OK= zvpDJzuPu=ZJ)ld+0s-h{7&zIK?l^8cmB_R|L>U*X{lW) z8L0pOm?f2m6>?xnqa5j`O561a!`Qx69358nB^;fAA`J*sqw^G z$0m5j#9=QAQj4-nvBqKxDtdxJyoOae`Rjc183lRA>$-OtXS*9a-~e)t6=8-2~R%;X646Ys+y2sS;l5ZAVjS zmlSQr-7EIkTvB>z6yGwm!M2~A@8t$6Ba<4qsmd4?cGVH$8`=!ahe}jQBz8E7a=Ja? zgg{1BCpF3e#`DUs328MR=?|Y!m?`d zVDmbFTA@NK0eJtFfs4K9mje=S4;f&D75UjpQZG*xE~Kvx>XJ24JyVdHiFZ)J;9}-t zNWF0LInuDn`d}E5@US{)L6tA{4Mt`;A4sgze}y7!DUxp#yL(^Cx-o*YH00Z5D^GnSc*{9LF@Lq9fJ*-DOf6m;Q7KS$LmPDtVsgm1<+ou0hsXzs_ z5lEpn^K-(%Uuv!hL_(5C{jnjZZ2RQ*Ks%MD z8YxkcqAtqU-#`2W%Y)|A!AO;+hFn0Q`;|4Sp_nPw2B8;CY*ZlRXi(K~3G}6rM0nQCr%&gS9fI!Y{0d};K2Z<&{Qr_IofeK z?_TGjaY-=1B;hyc7nbX-oJgvF3?c0WvJF8_7H!ahvU{mKOWfUTN2RTxLK zPr*UA3`saeLi7~}LzIzQHOTGa&5wk%IMAe&a?N;@3yhX930JBwt2i)CCu!)kFtF-Q z>SI+&THw;dKF}~?cFqi7+RjCz+^p3W7~oBzNaa%TD@8r0#mG36_9h4VX;tY~2Mtmj zG24l=sNRWQxl3e@>$@vxw|I}8nZ)rd!RIx4`529Szg{rk+z?_{WKS-)F!d~Ik z7HJn0w+D?tf#lHR{EzA{r9*6xq*buhP!jZ8RknhX658zLm$g++g75IQ?=bULKVAk; z137L=I})iNF-I!4LV*vCQ7#)oepQavxWqIk<2!Gz-)IRSAjCfa_lF(4671tn$$m|w z1JM!fkE;q^8$c#VR3p5$YI?Q=Y%zW_ux}-KR8jDXKTM3G`!STorKb#*NAg6UYvx!d z^o$_f%Nae&A!Xs4;E?3*nalhDpDFv&tqA)42P@Z70Bd=R5=N?BRU?q;ZT13VV?P`k z36R=42fvEy>UwO+1`c=pjG%f?dd>Ps#}A*vPo4hx<-H-@QUq&7hz>Ks9Ay9#xK>>m z4oZL;9n`Ni@<`Sm{hKy1i);?4jOYt4Oi-ltWy$lCfuW@Hn=n%=2(EfRzalLN1|MavXdc~Htv7a0 zn}Ff)c`0b6v1aNu8x4@)JC(w8SMHq0Tm>=GJAI-bZAqSnQTa-FO@-5>mpJPxH#n#`uuOjo*3u>$ijBMqL zx5)oR)mH|^wKdTK!QF$)V1v6$m|+;4Ffh2w;O>?LcgWxp+&yS;cL~8QxI^%egb+wx z?!Di8Rj=y&I=@cs+NXPUueG{!9D^AD__@#AQ#u&0i?^~ZRz#Xsa z1JRVtrGks1U=-HchHpnE$#Dh8MtbB+V@V$Fy(rG!5F7qm9b==*ExT(&%XY5o&*pck;*gqerI7epWd~x(Y&X!Z#(CBH3mH_XI&4 zQh3!WR^D2KUWY$nSTx?3sk$`brPqkou!B}8x+iFKpvlIqo`F`WE5IA>K+h!Wv<^lQ zST3xxatdq2wPeAjK%9PnD)=jw$t-tUM;p#x9}s!9ne4w*)= z=}uen4vR75LriVS!E=>Y&G7_7wGAOx7y~;7h0H4aBAOrg9SzoW+1meVU3s~TMf6-A z>l`X&l(`f4V3-<4PWumLU&S<*MolzQBk5K7OK9w)N_TZzy5fA*S_x5dNW9O1ONEtI zb3bxQ9E{2XTtv*$czRXB7QJ6doV1~did1d9gJ0CCf;OsM(Qs#HtG^?bdf_rr-Cj7VwRKM&-C^&s7*Z`v z}mtaMTihUoz>1-4x4T{cgXqo38dD2kwgL!oo#}t zZV%gtlQ^!e+-W=61tc)l9LsQFTP$P8p^z^v45oyL+o~KrH?cm`D06mE7=AxX0&Ha) z1*A;royV}p=alJ9*u#DLi4c8eznt336VsoH8DVqD*PV4@-c_ZM$!bhStb5VD7h)oJL1BAjCwVu49wS zI!_dwer=6BG!30yq~4B(VMg$WVUn?m#CEAZ)z|YT!v11vg-IcNTR3Q+3hUX*3RA4* z1c}SjhgklOah$~b)ApjvY))$MJ$+LArNn58p5^$ zW~GVRA4pJ+K6$ZwPA6L@*^mGXBgO?;R+w>LcPQ#5;-i!Qv}UD3yw(U-O>5y0eT29m`Jc?gjO3Ee23F5!Aqk$w`B{G(s2hlrn~Ow?833gj#~Gf$k_YXED1?W`WNU)y02!JhA}~25AL1=$RHMVS zalN8E1{x!cbv6H+u}6+-X$-I$NICTZ$*MAbq?rN^IAj=?kAG{g^Fxqa>bM8#ey%sF z3bKSue*Qj%>!4Doiic|n1d~mKJWnoVQ$?mU&HJC?(kMN`zufJK#HDqQyO z-j=Z+j?-TPD0^lAVn0VeONA#j;B#RB$x&mfslJ6bWI;INXYEv!gmPUCps3@H28)H2 zV6vki-3jI{=mE{dQ8A!CxgEOKSV@jTk|CGxTCwoQl_av;ML2%2xX4}4TMx9Y7*T`l zeID0>t7sHyQ;STXxUGt`bDCQ-a=pEBK_4qA?5~VPXicokz`ID|PNN0k`U(VZ-Kvz! zwOdXUD|o)F;m2MkuPZo!+ianA zq>k{|!5U=`iarBH-fEy{PvZ67=Y;;6mJ(e8!c4_i?@3^!+&7w7JAZpTG2~(Jn*&|# zN-hVT^p>hPb>`(z6!dSz5T`H{sA$69x{u;WX%p}Z)5VqCQ+&R7xjF4STKpVP(V`8z*B`?}n3@y>PK<*1x= zXE6EW&bT28^aMLG{oLnP) zXyUh{t^aGNq15K9&^u@=(O6?QuwG`n6OB?of+k60FP;gU9?p21ZA@Fssts0vmH1#BVc|%C9@Q}twrNHX-W@s zKB*%UrcBBDGHs6zFnHu`c-7W=j<6XTmrB-R`OGn>S(}YbfYuz8y#{D@bfvL|OOBd| z+ON1T6;utmcFt%Ml+dt$6g%}xLjJ!7zpQo2y4U)YyZF+*Ea>}ujPFek1&Jbe19g2yC$sqf>Gm){!* z^<7CIlc5C|FKq(e^g6T3;B9Wmwb5%9olk=X=;Ybvwy!%7AEumROlq*LL&63+xoWs@ za{&Gz6x&elNqbD5lF8DhO85@H^($7Y1Fdw~i6w#d$wKXrFezL`lwz6#?{{J{3Iz-; z^Ojrg&b3j{9U9Qv%P`N{)SA#QL5bs{X!>I!=!4JE{?4EHLFTr2ru10%$X>EK>lRUo z7x;#<3C%*+oB9n)(!Uxw>=5~L)EUKL@+$;@4*`I&cOtCxm`Ov!<7Kt+Xed8a4!3ck zAHGdrU0EeYvrKcuUDNzCco!}}15E^edqs{D^B@OBjVVT_5lf%~QNsnl_z~oFyr+H> zf^t?!|4>9hLIO*cttR)Fi?Q5TMQx8EvXKU^l@P_16m>2CZ5Z)OwkAfUL|_}>0FKRA zDJd{^U~EDBVzrP?#nFh^+8hIp8@2AaVSSZQ6E}m^h6c^QpJJ}(YqHkhDWXfjt@B*$ z_v*d1j@-~DoITxn_<3$=VPmNd=P6o>w?~>D729sk(S*2c%TR{*u5wESq)POc&;9=t zljMfUA~7>(=P>;+|lQ<=6w*gH0*vT-+6Ocx$tWn>x#J zR-}c=U5Z7nMR#Qp`eDSsgwX7NQUfY`VfySWt$!GD<$s+^!;L)usSH^EeL*_F%Bu;; zg-EDOsr@TDU=33-%2k*6Yg_uqzl5IpEeD(3$>Xnw>Hkw1#OvJk;#3ePG=1c%bNoka zpd0^pCFwu9gX&ibe_c)FbIlGOG`m36wX30&|E{pRQc!6F^mP;XN80%PzTe>N4Szoy znP1=z#C!WBx62 z8rvq_SAv!TD-zX(#Z(~~FW0PEUH2k}ZtK^jX2z2%iNo%uJ$DtdWpmEwII`pu=JH0| z{e1MB7n^}v)L+(jey>uUS3f%*ER7J-N(yVY+x$Vw0IM7s_sc+k@tMB1FFT!IU5kUeIDG4#dLZaTy?yHF`z<6)Pi7lRt0fF@ z!_DhDKjKYzWuf`8^mdlGsU=W76dRw$+W(NU>2rE0k{jWYrR&&7lK8L10kfZ8Bfa%b z(~;D=HatUWC>ugU=(}+`H`4Xzld=cw)IBMrSN)t?>a=s;LM_iu!CoFIh=I2uRTU!s z3Mt#kU0hN>7sIlLi+!RVd4SJJ3~l+2@`lEw9#8R%$_%O%@(VqVeoHVEgkvZ!4GC+~cf4x3U^G;a7N@4Y5V7~sS#sA=9Pmb z6kjN_mAH3fzVrizMb!g(goAGh>qGWyf=iSyFG~OP5qbS*3PEYE5fv~NT%a44I@Sr1 zB!Qvz?3+a_FlTwtbZ=JKrUuzmQ0Dnv2HUtF?zqo)G5LxuaMDfdoW=T7-*eNVdKQhM zB}r=t61TTC4`rhh4f6xhcAuIDbD7>kINDx8Y~YEuX;AG=%g}e+Z39`iW_=V$$J@p{ zQdA!89Rvei-T`I9yG_bHO`{=A#|150%mkeFJnue0wL5o7_5Kk`bYL=fu9U4Lgd>cszTTgYiozfwaSCi4^(JKEYSac?!sb=3g?Lq?erjmO)WB!Xg zG-+O`ex||Jh#C2w+R}u=1;_)o+!}=tmrm6|jQDEaz?EW=%n&lJSlClY9SUfp|2ai! zZA}pakix4AiI%RtQoaJH(J`|Ke@d``qo?^*zbQy=x-jafW|XIX!yrqZ zQW`m%hrH^Q7e}P3S=+aaJdT$Q(>nwDQdPVO?m=#Pr60l}l7nH*9B*9jl%(bh$!x?HVkXjZM z@K(oW@jb|tYP6TrkF!Pa3X9%XmC1{-GlkYqoFSr}6cU`BCBenX^NdGH;0&KQ(Z#4_ z2ii#;qx)c1=l1UUpM>yCKEpxYa$vP_?o=GRekLo8`Bx0xXX0A^%>(@^9vR%U`jahb zQ;_;i=83dsJr$_m4`V7yUpzg>@KlI&$)p=|TowOAh_v#dm{70E9F`%N%v?hdTEUmr z9rD$8_$Kn&B)26sRWJDZ^?M1M6K@dO1nH2O!JZk)&p6iRyPz=T93iyG(i95XEJbXw zht6+43ASR>It1)>{y;BW^nm&C1}sdB8350yB?2EV)z zOCdfr-jc8{Cc0UA^6H1BxX(zYN7CBcif3JoG7DG_i{(jO_YQ7c zxuTsH(fp#`=&P3JRfrry@iC@1YJWULeh9@!qTEYw63zOg6aM}<1@ogwS`;h#qjd9(00TLTRI5`CQc(6)z4B$PBs*S-kF0I zZ7Lo#-EytHC#BOg3chaBbQ`VVRUNQvEJQqIxyZTKF z5oU7<5?ZN0ze=%Y6hZDGr)&-$WMVbqzggsHDE@$qfE&A`R>T%Xiny$-%K!X zf>5uDPF+N~10g1wLt<*F_?buI!@Ok>TK&?*6IOi*#w1e$DJgcp4e#Pq#K6s6S#a?j zda0V8)ZdAahL#MouNP}smw)^yC{zfMNDzcS3@Ebs4V3;`ZAcY*50qF)Sf7o-0~Ih{ z%1Zle&q$9q{Jn6QZ|J}OProQ{$wRKwK%d?1S^BF5CsWFYd5X?n4UDFxVvuGXqK)~y zW?F(9Cc=5H-5eYF-4yLdag)l_sOQuH9UcKk+)Fu9)n-g1RZ+ILZ&_O%j{QP^!O($R zANrYvPT?P7wlCATwEc;xMhj`< zkvX$>tGj3KG%d&M2o1OxwzLUCUD&&8pQD!aL4V7|seN>BIdzFwC-PH`jf=6R>e=e} zJgA8$&3o3;u%L~Vo<8sb7q*8o^?T+sR=p(8&%|RJgnqiIuCMk2(>V~cF`Zhi&U&O( z>w3J%NJQaBh(h}|>-8lmziA(_-GA^%zSta-UhooL=v5rKo4^=RG2Q zE;0k@E?m$!fr?&i-(=K&q;zu7pE}QKH0_uveq{mi2 z`Zc3%kwC9dMm@L{d6jA4A04tqY*UkJgl?_YM1>5`>Y6V>1SfhZ-TC~ino6-pHMDn$u63!NYmM-W4@to!#D6!YA-4kc**DV z>+gE7WxUW5y@+dl)daN9z3km{7&`=*^LD9mo*Klnt!(T0&Lhw+q%5xrDnrk|=%dz( zS8$qNBo98I8nXGEg{#CoXlK{c3?lici%hLb(k;!ia}kg#mywq8230}B5}{9u4B9p3 z(GvgF&rCvcoILEr2A}O!Q8!bH)j2O2?M<{FNeoHAa7&F0LQH6Ls)1p!T`?1ZhA*(w zq72Zp+TOolV*EBG`1u@!4E4q|>GdKE)}~v7Khiv9hSdJSUbeiO2A~vF(UYwm6`m<^ zDLvURz17KAmk{=igFirIgGx#IRZNrfkEChq!_b*I^D7RUb;p_ZP;>7C3eQ{l=&T+8T(1qtyL zHqQXh5-Az7FPAm;)Dc1QnBu?-Kp&Cxt0BD2RR=kFT#Y_?<-VX*B&1o`$cnsUYYw+= z{)C{-KBYr~l z3|EZxXVJ^$Xl6iq&>iTaDb!o%acgEmZ|#Dh%g6uif^8B+Hqj?6*)rdR$2dh1qPt=sOt=d#1h6K7*z zUWtZ#<3}E`YOwl$(k9BfLtVtX`$|5_(;(_eos!0oi21z;wVyUpA=?gZF-O)9qThze z#rG}h4I}%1aVI1xL`)srl-12j0*$<_OD&M!Km#a72o}$Vd(I2Q@C0bG$F5eydZQ?f z+A}F&>p+HSaUkCv%ERRKS{GW;1KU+{*B9vJUyL~gDDDufh;|p7rXiGR*!`p_vLkb* z`lUlQ&UgK?khnBFFEqaIB(EWkQ300iz3(v3n{MJsxpxP5Uaf0{J<4OzI0RE@1U)aZ zVI187U@O_%@Cml#KlFk2h&iLqFr^IcTNs(OEjSbHPJO}HEU%!XB&~oQNN?G|Pli6chjE1~FbNq*PXT^tvus2};^98~TWg5#!%lxP!XR+EP4Di=&!t$qyyP`YU)!qg>S)EFg#CDgo`D4pU4NSD;ha;o@;HJHgIR?GW7-!%nYLZZAumo+g zd}H?S*Ra%9ZZ5GIOK_mD@xJ_}@l(F*;e#MDZc45vwio(S#gGJ_svdZ)5Mj;c_fpihqzk<~uz_y411CADZ;%Y*`}TOFg<*Vj&#xZVzBk zZJjfbv>7krXvJ-MKI{YTIh+ya$-EK=&taW?&VwEFu z44RVeMNL?cO($oiK@n#ReBWwn7C<28{WRl74xJ65mZ9ePOqw_=wJEAc?O|G*-0&3} zH|AvwpTk5{zea))HU}LvyZXc}m*-XCCUMk|Zl~7h{uw#D4jqHNi*>gs6`J2)v*Ef4 zgN=|jIa!%Zol8y!mLV^+oBYJtp=#k}@l4m{j%j%9OVSHD4TiMoX3mj@NQZJmiY*l1 z*D$G%RHqSXt|^yTO`jqcC8q?i844*Twma8DGg#`K7Ov;gRzHuk4RI+yZ1);Gwa(vp zjmPH>Gh!+9%=z{8)$YWHmADzwlybXk<|EuX$HqgJ7s5ja`0bXkxn=dGRO?6YxwB2 zqx{aQ)tly!-3u@2t>$qk;VyEQb<&jWjmzDsGkFj9xsDZ_d~t54M@Zj>k?Px<7L`~Y zs`uf$5Z~;=N;}U1fsle?b8N5DZyLpwUB$LO-#D$zqq)?2hLxkRM-@M^0;~i_-s%iB z2!DtvE~VPBi-_ExwXn1s*h=ZAisU8fUw5h6e8a1y)UdkU zj#m3Gg@t%mfC;cVB-RRmpOaKiLl=TlP?P5j6$vB{9*v6pR5=QOU zA^@xcxp9Yy-iD9OB)m^oNw>+fRNW4Xm&he4t1rDrhp?tlvMu1i9G*v0`5{ecxyiK> z<{{|nWoi~Lp~CK}G7(aS5nMvBc9dw}dW=VSGCcLBIhxuOAtPrKK*?Ao9PJw!>!nmm zF+vQdw6`9mmQoO0*F*l%Mt_E1p?*7~l7mTIE#F4_Ocdu%j?J#1aFfR)p+X2|RMzIy z=QB(GCUqIWZg_ruH!?A7vsQkPu}|+8OIgPvP9vI#Vik9YmK{xB%qOmWKm03M69*Wr z0jP}&&o@rUZ_+*e&Es(Z;V(_tQ^lrfKpzboC}Df2KbG*?6N7ft=aN@0$6A9mnEoruDp z36lBZ(O7|^FlWsS@~yr;A(3k7h5?RPT~I~eB!i4v_e8#~cI6joiJ`N(QP)b;(+M?s z(+_A~$d?N$b9*-)w%SCTmy=Z0x+YjX@1Xhiz2k(ZFeEA?pU^xXtjk}@E~RAYvg%&D zS;a}wo(@Qv7LMI*s&-XKfBi6Nv;w+W9&IW7R_=v|O6gnvRlYmiz2T_KyZhB$w$VzC zkNcyP^Z)oGd4h_M?(+7%_z;al0q~{7cK`gGhy*^jGfksFnJRKY8R9Cm`d({YTe+Cq zrvfJfR|2nAi$ocNfn^87$dv?xy$YkO+6sQULrQ zlAmMXXwtA0iabnRid2`bCu2q@A!`b=>{~M?Bcl4-A^9ihxCEs*RtbP7s^7A0mFOe^z zDfh34JJ~0@_qE-|nQ>Zyd#)@4EQ3|r@nSp$H@vwY2Sm6}#ONdO=degzg6|6wV7KadRMfH+;JAn}*_v-o?#V-@Y>)mRC&t z`X&7N>FfC2e+Jm5;2WA#E_?zhVa^rCal>Z7t!`Q1X|PMW61!+R?_@`cka|{t$O9js zm%=LV64yo+f8>w?9`mU2{V*Azc^^6Nm6sh{S!+Eglx@B}#aiSLan4kx;27T?7y~vv zDA&B2gp3x*SfUB%P}}$83EyF=@{%sJrlJ7OxRl>m32Z?;JfqNalJ1l>VN^x9>~!Pu z%Ni{MTZ2D1qAYPY*@WF`Hlx^4_E;7IsS;PE=Q3L9^(-=t|Crgl-26n02q8M1WfVBn ze7==By`aLOL$|5S#yN6kBRn3h?a7tuZF};tO0r zxR3up&STgEOxri$%X}`W7BMJ=T?4~NosKd&lPbVeP-^A9Ldky&0E+bieFDw<@#!%< z1$0u(%nJo&Av-=&V|fkH%kkIOaG00{G>)2Wu-0aVhW=i>X+!qGvB;y%N?5SGJ*!RL z>;-K2A_1*wt-&}6ziF3Lt)>|;^*k_2&$C3@6>XEY8C;^r3A>7r@RrKWwqxIf0=!v$ z`VkN~=hd_k-DyzVY#`~Q*n_s7qeT}}Z-Jzw>bV-@%XMdcQisg^0k%9}=#{cK2W7U!Uo+oc>BksAAuSod1&B}~i1r})Djm$O zT+Y3pgM0uV5cugy=nTp-aFBew_6WBpK4Ywg9V6`RP&}c{ByTSFH9@K0N1aI6^mdUZ z9lSd2*KL0=vi^6Aa`QQ%YEtZ6IwrX6Z9G^y7!Hvl@1%L7n9ZT+I!8(h*z7`wTE7vu zFNaVx`A%6Z#;-428)VA)Sxj%8hZ)>jy8bTO1Ko2y>lj?XsD_W?;iyBc(>PfSbc#g! zC${J5X0pbq^Aoi}Va_&&q38jN;sJ|S#XX1>-GcHhV_o?S^`?=NHjTBoDj95PH(}VG zRbJS*EsC(q!i3Usc5jm^&}bO(eEl;EzIX)l=5YC>YoCWV{iJ3YA25Gd$AM<(xkTe0 zsqR4vKNg$gThRG0YO`+{-u1;%Q-{5@Nm+Lrq2*f|zpGOkG3e1jGqSj*Tbq0&{2TU# z1}5Jx5UXT33eQ)Ya-gvJiIpdIx^r?sh+$0RVydE27EM>Rh*%4qzQg_q$yO|dGyX^z?wDXt z@?r=ai4kaByx1Z?f7a5$B}49=F*LEw^z~HjB>_j7I=~z>T~i8cGDPJ~_e|4{l^<`< z6p6Y5-)e3J(Y$h5*P=M_4fA51dmu5q%F5t)md@)QHVg|~n+4LuG4g#;F=|3u-c|5yY(r{6F*Q}m zyMuR+7KI$nr4oN@$ZNHML-Sx}w8l|On6lA&>4*;B12C09QhSn@j7}KM34LxuBsW$B zOY(^OuRWfCh}e^JTREeYB&Ox}sGni(S&=lGo7(3#uJ~?9I&bNZR|CRN^NAV2e9rfU zl<7ZtXf<01td1F^{VFQ4p=aI}LIk4+!p|_a1!9XxgL7kf(Blvm=#nq+#f!dB-#MKMTE2NX@2D6Q1kljcr*l3&-M1194r0ffEp zcy-GTqD1}aN!h8t$;+g=#UPcrAI)Q6|G}WbeXkN2y2-)h1$O?5&%1W|1#UvrKfcIM z-pX!2Ls4sHVbg1ZRJBg_2DkmUtb;zwEfV!}ph;+t35N6R2nZ5y$m^>oBhzH&^k zZhxncqu5BZz@ddQiEWK|*4{5V5TT=WU@mpCEEX}l^BfL(s?{z|hvkkIVZwd8 z&`Ek(C1)oBX^u81#AD_B*t0%O*cem^_dzvb9eK?v-Nn=`{M!b-dD#+AjCSd=o! z*q!4Zs%9|2q>g=wHXqAnAO^d?np0f0=tF1k62a89#NnCoM8|uaio^Vnf`5WCBdUe! z{vm0#>uAkQOgubgF?KIkSt@&HIqlaJMY}_)4|)RbxQmaqEOyN)2cu2%u7dr1YKMR( znL^Y#04L5&Q{4O7GU2dDK&24HVfgE19BicCRs;pvIZw4q+-+p*X2kd(+4`>FID1c=F^|CvC>lGcw` zrK6RCh|$VTVP+=`^AuXJc~`}j$gRTiN(oLzo3Aj6vg^+{s&%UCa7E2Yl97Nx=K@+T z?QJnYigef5gI9dE;V(X z%9G&|Qjm~`2gX;88oManh__`~>JA%F|Kw!pCM{vA1LCS2@&m2|2E`MM=_cJO?!+dG zGl3D^sG^mF*sSP82_i|ZC^o?`@6mSdP%5n4vXR724sr zjze_O!QeYiHQtZhX)>~NDq51%Hcqya246&iw$vq^coX{92bZQRt1O4*5pX$GbGUCw>VL0IB1%=RZjsZdDy zyS=vhdJRwE8YBwBCQUUyX?nm#I3(j)#E1HUG@{dFfvk>;TcwOXK<@K(2+_l?vV+4j zQfK{9Lox4qMJek;)I;{TtMvY^D8uFT+mxSz=`R;=4gAW;eSY#AGK}S0&jE-QfbW>IUkdJy{H9I* zpAjbO_GaF#BAHfGXhLHi?8E+jd_^s6;itTR5YXd2r{Q^^-L0tR~?&WUps_$DeZ%Nyc)~=$)!m;?pDl8*Cr0IJ8$|7Y4tud5Jtt#A=Rr<(R!`n3WPh!ZtnA z>1hdQ=iLXXT^~wIS&BYtEFTm$s`SfLw;E!S=ns%Pm` zhBi?}MacvJd~klhYYp5tb~LSN1U|@|YU1xNBPCX&xCV4G8 zMckqdJ`_5O{+gsyRHv^ye7zFT(v|c*`^#%bZuOuGvTM5=Kh!t#ewypZXtF5h4E%)SygX%kL|j;#G0iQV4<5 zz}RxK-$Lb;)A<}1FW~m`CMs|Z$)%Hw=cgKZ_SP%(hYq=xEQZ3;_@3fiB8ZSV)46XZ zzQOPl=6?Ece9ZaB9Kvtb==-0&TeqXf-mshsPyT_O{oeA|gv1TEQMs`gn=s1c>S*A6 z+M$RGLNA`9l+>l%FlVg^6>3lMK(IGel268I(qdkjI;^285CrmI97P{r56!59$$y9k zM$QY14U`j!Cr~Phx34g<$4(eDk3AobHQ`LN4GdD`IW1>Pe6UGvshV9SZ7Mdy>|mWR zSwJ-G2X1W~uz&JL3uSnhJl_Ol&%p<$kT~ECEk9xBYMr1(Jqur?t@zB>6_lz&f^X^? z_jBf*@}bCUM(F<<%qa&d%C2~qb1Ed2DSOrZstMG=R&?5KqoDxC?Cdc%L5o)Hb>x=b z>NYcn9rK&65^CJpCTzQ2I?S7t4#BJ^h^5HW*x^_EUoJK*B|p`9Q_{!pM>y-02pKy+ zevL?umP%yfy3o$O!2n_u8g_Wr`bZxNS^eScVULt31PE*k=o|F&1u`Gr4*jV}6?tG` z3+#Ws?z6{q4*9?atEfy3OOnopRF{Fys{#v;LhabMsZ*ryGT}CO)}@KbwZ4d!5E(+y z#8Na!{ctVu3uc#x)DU-V@aMNc^t5ij62{fXg{eLeYCV&H43DXYD>c8<@YUdevywLo zhVj*L?VfA|yT&s~_p%eO5!_#q68}y+Q$Qit5mm%}5p!4H?v)6(=3F)zh~YYHF9?qO?T3p@j6)D@6;BfLlUSCzZ*gb3a7 zn@TNbPeEe#QcCg3)sOb+)0c(ZaV4gPt%HbOkHuP2L|~(uNZujJJ~|XDkWnx5?qfu) zjF84>fl=0r^-MGciPjoWWr+pa*&Ok_}6y@mflEEX7`jHM?KL7U|5F zCpcGVhfAO>!@VkvXfM`HyGnm*u}J|sPC?8Umtv$fqLZdk2%I53!4N92ZL0T3aR2g%utlW?IHQ91Z@Ev1mhu0AjveS*XMka)*6XyboUWikU}XB;NWW)r{=# zJ+g?m)0>&+I@g8@eaIR2oz`~+`~)?&xrg4gC-^=JEEKs+L)jQH_S0Bnav7H;gnpI7 z$A_4RHyF^g^aOZ4bTc0UJC47&Pd%3}Z zudgntJ0FSF^cyTDolN>XcOM>viMvElVXxWZNIzBgEuBzd^aU+iGOZgQZv2{;cK_Vh zAG=Hc$Owo~k+Qi{j~DlC`%#4Nr{el`JgF;?OBDMQDon)rpR@DlR4&AGo^-W}qPl{@ z^T45t1JT`q2TWQUsRRJy{f3j*d=wn;cj`QAoD?d)J3dVFk@h+ka+A#zTqK(6LQWf} z!t1$wCF-W;HG2sQ(#Ny>LQHtWjhiuAAWTQ&d%Gj*QNh)ND;z;FwoVN}T;DvbE-QQH zPZ8uF1O+Qhio#!(97@P6(31=*wfJU{3wh#fA~@yOP7D|J;d=I*+zSH04fW$$lKNy< z2x|U&T7?7f+t1{4iK$?%blOyT-{3cCC+7?jlWsFe>lBP|VZx z@XaKikJ>3eK=eOomoI2kF~Oy@kh*{8sz_98v*F;f27{u2Pt1JBnVX z>kos_+rCnru5&T<>RR=8hQe*CY)<#r08($clN??R7AW|%qdQ*K;e$_=lfD)Y0 zKR-gX@7ms0_c3?J0%$k-pJ~wp)d}L{vyXAXOUVlG3$9ppx4$mcwcx zB+1j}SN!oGw1**Ilp>_erTLI~lKMizqC^zETV4fCo(D)vy5QCyeex#4W)rpI9YkJ*%tns=*1Q`vHWllqQVg@k#=?9JCVdL zNue8?p3;5^UJFAD4bi<6Ba@tS3a0v)eJ)U?bifG@L+eS{@M@_|y8B+$GUcYc5<)C6 zZNo^>6Tx6aa04OTfF<***yi9jaQ}hVbh2H4nB8WnBckXK69}QcV8-JyfhTGuF(q+bVo0AQ>Q?Lsr=kieMMB*`5k@F<90j58II7D@bv1l4oFCC^Hq(m>L7iSgpU>~`NndRqq)Nr^pfmi zK$zDSSGOYaS>$qS^NF+@G4>e2S3IntB!lCjXkC&&G>EAd6^^=T6;@a&jNOZmWn}ml zrtn`wM4Gb6Xl#xNzu(9k&rTQU5mFF07hLKzKv zGhvaMc6Rrrc^p@f1t3|Er9wiO>zeXTXhRT1we- zj%a1SddPPLq(W0fpxLoDazr;c_cPah}LLc>}}pVdPqZ^bk&Dm=Km7SqZQ7y@*I|r^zh%9Y?&neA?GzGZy@C z7jDwT7#ogi<|>?W>b8Y@`nkz8MjJ>ZYTa_swu3?{MN_{6$J&I?;U17(!ivuVn(pZ; zk*l%PdAA-_IyPzoeE|<=>tGe!yZZ z?qZTiKZV(xp>1AutIR}1l7E;(D4g0O-=YLK6sH9cOrQL*zWTgOr{dvy^~BSQJ#+n= z|3hkP23kz`MQ!zEz3iLUqO!aA)~I0YA{G-=@pGLifuw4O@MPb&m_J+R07q~Lmik(C zdp2n&UTIoE$+`K*)%Lr&WEy7EpJa@ps!t0C5Ce}A{R<{24;o(RTvTMV%AC)e)ETk4 z*h`R~8zuP5HU*pJ#>uCN$1OvncxRuQJgL;@_GO+W6Yy24u(RESi|q5c1s{wf6UReC z2*1{iH?12Lyr^W?CEAFiI+@?J_LU~QLeX; zP`gO=s1v!SE!(sLvzAKL_V_m*S?X*e01CwD7k%gRz1xk2&>@S}H0hucy)j9Q9V|+3 zclFnmF-kX7H0xA=v_n%@*T`&ZMPM8`X{*D96(N3}LU33d|G5eCnr1 zi&jdOq7)mPCSSp^@+dtn9upjBAfX5FFfwg|;Zo0Ebc;&qf1Do(TA$B-D9bb|y~2r` zXR|M@lOrY+#p;_Xq49)1D#}mL*l<+9g-cody(Pr*;!>l(aS0kK$k_a-F`> zjlmq-h%@WotycO*V14=Qz~|tdr8b$~wEOssJ=?ch$YyyZ&Fy4$#1XBUrZ9EL67QB8 z-Bj#6H-aEooMO?4V)~0e)>_E#=@Oo76>V5nySl|FY>YaZI6YT;g_SDPx(@+=SoSjj z`R*EMWZC00efg47ruys4hXtY#MwCAUD2PhOyHT~<(eT)+cY|rJkfs*C>BSIsaxO~Yi80!2FyQM zzQ=(}<>A`hLee2T`a+o2aSM4uLo+lp-7Sq=MZ z&C-sEJ9qWRwj!rgtJO(lAL>1pK>*dS5oJT`A%~p-9@4! zv6KyXPcbkA@@FLH`+{k!sNhAI!wZ2lvs=L6L*ltmw$HiDl!c7!!jgCYeFEuPxnuYv zzl%gFlk~(j;b_d|ib5vEr>=hU_BXFl9#QVyP*|J#aKGBxcIRJp)%#obB#KUpK^^;} z+kw5LN6K7G2Jxh*bhc5`mAS^zwuMbp>Y4pc@Dv?q>qBf0Rdh|>`p555e%tU0VR-^m zYIU>S+<7&ye?&-Az0mpFLxg z{u9S1pdGkf%k~|t7SPewue~603H~`i2Y>ad2>ga=o9d@kNsYAEC>PrJ1oik=feDer zJt}x>#f;fu(`6b;S;X$j!U&qtkQM2T7>Qp1=>jfpLFc{W`+x zCkpw49x_iBMnA<{7{(o@19Cw{@T0j3HS>2In~_Z~?C!yMY>*@;Ws&fg?rQnFy~$e+2Yik&ZuQLA)9+f9)){_^#V^$C*q6eHB;A!b zdrmGd{cq}9kg%n*AFL?J0Qchpb^7t~Us%m@y1DXNm1$48D%c07$t4-{4MY`x{wU6J^V`6HF)zTaT0ZdTf114wdc_C#=keOm_-e z4bi>RMPBpt1}ahqT2so7D>uo0Q9{WN72IrC!kdV}Xt=mlw^

xTr1Y{%hYHb1%${x=5v-L%O_J;a}#S8STNRDpPIWHQ}w&F zeYGNey0NwKnBD=4sq4Y6+Dykah)hk*%gsYA->q8P8v7>u9(Y`$Rc$$Hd1~HnECI^o zWFsFQyVQ8ifHhnPd=uZ4YAQ1hH{90mFl;dSEt~9poz0x>?77xQmLXQY^@{m8Fe>q; zkg>Zd)=afNuqE3RmIGEYuzGu1y(XPum0^sr+W6eK+?WF~xGD?YfpZ;l)xx^(aQk(j za9(##^8V+S`|RGA-guwTPlj5$Z{SkE8#ofAgzrakk;y0r@;P`3U&53g6DbK*_}luM z2NT09!Uq8Tus58A9E2)EO+nv9r=!{<0+Ex*4HzbYM$V#?f@SA2`5|=%I#b6{ zj#1dum(&=>0CqX&GdKVbut#%_a(1&3tR}2uoJoAQV2ZF&*j+qZvJ%L(!(!eMoXj*DCy)%6f4M*Ka&LHIyc4Mj762f|lm=>ZOC!Hl623yi85{G=A z{2Kf{1ynCJLL;*n+yGczlEDdf0(_^r%=ff|U?y7toOT_9#oEeA+d!XB zM*$At4wD0R({@Z7<2lpL@`5cS%xuGaPVYs1Pn?J!M|cCSur|anj)0BBbi@cT4d_dl zU)aYuEB*}8OP0`|FtMyC#vJNW@^JEZGKb{B?E%;FAk0408RTzd5LJw+$9TcPY{pyx z0^)-3CNQ7Y23tWKa6+IG`0j?#_sDSMP{1g}p>`l&Bix8=v<2IMJBvMwo{v(Yaxj}Q z73h&bx!i(o2VVhn5A1zhZ^ABO2B|F>MIB8&PA&vm<|;T|Rn&8owq$5ZkcNV<^afEx zq!TJ|c5GYxMUsr#o_>#3Ny!6m(GN;9N-1$aE(Nn4H3*r5Sb#W=vV(WwIc^1R8cvIg z!VO1{LCTQ>(X%la0BOxfJJ52B6VnFk#75zQ*wffMm}c0O1U_XDbsZ&z9811Poy{1} zsG!9H<#aFQ4|y6zNJ9bRFu=MBc%nW)I&I4D#*JdnV&l2dyxveBk7oV+A7mf3GrT4) z5f0)OU}m63pk`xd6Abuh>>&tdDiF7kBGgLca#aB7qx$-L+PhhvS-zQp zLBZE0oxA3VS}!a65JwTYZvoA!ynxu{c>Z_*u$XH7HB$XYP5^= zdkmM2Y||BEsp)}L<#_32J1Q*Kjf)H)4JpQ>h8?=;Y6CbU=PP8&hw2<~7%$cJhEewx z{cZgy?HeUQo**O1OJQJKr$|>mQ2wVfsOzj(cQ_gp7EEblTm-8tSq1Ssrh+!*f(&ocLBz>1Z) z+j&NT!Fsx=b%)8{Uagb(66utTq1r8*c}d& zk5H%5N@;Y+W<^ujwDHW<;2FHYOsB7;lA*X*0L8^uj1N_ZdVpz(FC+9JF!1@xeCB=KM)D1Q<&Yj0{kL;2L2Q_6H|?8f^S01C7!?yLbpO~ zL%)Xi{Yz8`nS%a}y+&wDJ_k0`nbc@sn ztzE!8$7#VKFbC64QhU%OfGrtC>qb3DxkgE)xfp}k>)C6V>u9qnQp#F-98l<$%m!eU zw_(f!2;K?GDRMKinEZ|Wh}r<2y(KgprH<5+@|rGScL74^LH1YnWzJ{rRbC~J$twom z&}{x!L380sK_V}X^^!K2mc)>-`m%omshY=K!EMfM&-uw3%nZ;A=@vSjHJG!Hdjec+ zI!+<)qM)}ZMzmf)=jCvYbKdYy2zrD4@Djh3;DX2zrIRqD_em^brSP|4j&Qs9s>CDl zM9qjg5QUYDkYK@~Ac;C1wM4QpdUwq2m|f9Nqmbel(IwGx@hwrjK*5>MYQ?g#nLHY> z(3#u|+`jzXd?)t>o5U=k&Lx)Nq}b+!o#e3;CV4Vx59tzgiLR2Y#3#goq@$$wBrSOX zwGwbcA4oOC60#R?L%XQApqq4_gaEsB4DBWDEB!IEnl0yuxr@1-xQT26V+hSeTfscb z!SK3rsT>>YGxIouM1KXu<09GwY75GEfERrvO9)NS$AB;#1vcSDp)p`J@cE|%Z2r@} z>)sNt$os_;hH$_#AKN$Cz24rzst*_@+MlYy%84o?gecVyQ#=L|WA_wEkkvg1^as=2oq&Umz!`ekjuwPT4qFLF2)OHTBQy zEVaH`cl{#S59LSIdj(atqY*D}qIjUVsC=zjqOMhcRX0;VQjJmzz>1pz8H>Y~fNhw4 zFPx$qOua2~JKMe9v(@8p4|lf*Bg!W?!jtb=<2mhJ8aNW6p?|=Y_Gsi!xFIqaIT5uQ z-5Yxie}TA&^po_S?4|Uf9;KMbTPb^KC1C9^P!CfcQ=c;WaN6+?3N{J|f@^%NV5iV1 zo*=0JfXxWeYjMZuLow;mn?yz+dIM;RBLJ`aJeGqQVq6BV;U89S4ubm;vaXxCyLe9C zAl^_;gn5Mdf&G)G6BI*Fi_IIvQLtT{N4)ENF8?EU8TT=78aT}N@*c6vfRcNNp=HK! zZt=1NEBFfT9j=4t<9CBA;3DBJ5k0C%QYcA?Y9Vs)&+)|k1wu_!Ao`*tMYM|lk%!@T z;gk41d6Re^e!Xy;xHf85)G=W!?+w^G$8vu1YGI{d7Joel#qxudX+C2(V=*(E^N_z% zR3mN{bz5|u*N!!uaf!Ks6NC;+8|E~|U#5{&&58r7x{x7d%w_H5tl%#Z77G?~3K*}c zNZNY37EH}%>M*JpjO)YyXI@1$(-`o5K~A>djSwMB;tgh3vo3IuV3=OVF9VYDPL7Gw zma`mw5{u5J|Dd~=U3pVP`QnHmhj)_mk<*;3V)tNGp+2CbQn%51f<=zO_({7# zS1>DBCCp0N9O_0|0&@TuphvTJvd*$f+4-E=kS@N)Nnu}xXIsL4$z_1adJgX^XCG%V zcLVPfZz1;@`wvr2?*u*t5!AZ1v`$nBIe~;DiHNUoztH1RNX!%bUy_4-fE-Plhfl;L zAaf!k!V5y@0~`IWK%Yn%DgkpEZAJD)?2Tk1D5yB7t7-J5j-oJe(EF*8k`*+^n3I} z40H&=5sZHQ0^JbpOl?@#)5tVWG!so#I)*w~xm+nwxs_$gJf%$WT`@&DO+{4SRIOE> zSIknTs##pHT zp}nMAY@B5tXf82zFrLx#bU0mi{Q$#EgIV8NAJpB^C&1mQlR>E4slBROYgi2g{|GpO zYRpxpeA7IjdUvqVZA{xw+eOC@*Ic*B$*>QyTr&5!nr%;Q7E4=G2mMpcWc3&IAf4CP z!NP@`-xFt+JKmLN_nY$|JnJ%Dwe+>~Tz}o=?z7I@_GtSG`+B?GHrY1Ip6V|3i-UXp z`QFZOm)_)Q34Xe3fQ@SKeQ_^wd0al%UDt58-m}PmHuNGgD^eLO@)fxMb4ne%0r$Hb z?DT^@!<;|ht~kLOvixUOn0%Ja&f`9QFd^{WJK0^}Y_OlS*V^Uwwf26tTb3!7wboSI zed{kv2>_e;_IZwLjz)VsTdIX^T5ecv_-67zmj0`C8lX#at?#W{tO!fCDaLffGSxBP zeZ)ic#DmLqt#={#8FZmFk#-SjNC=jOLC6V62Vyn2O1O+agjdVm!u`M*#s0w(a_a^AMXiLLd9PS9u$X;hB(N88i}?4z^i~Ni zE)nN2`ypG-z6Jj2_1tazi~ItvhS{F3qRyf{WSnOY<$dC3^DlB&fr%lLgJFCn#}h9S z_@r~>Bjh0@1)&E21^k0Bz5}a7 z<1t`}$LA5Y6O8zH{7xJfw+}l66OZB}=0+YP{-WF9{}4J7Zs59LAA?P01a=ltr|6jAN{EtW)#}lv*$#*MXz21vCQB!LyBJpJg^@i~zIs zMOqboJ!=|wJ#1tQvLCV*Fbf!m=__d@%10uFbdth_c4Q58 z30Y700NpS*<_P8@W-G>wK8U`6Zi~)@7IQI}l+J`-MNEjP$X$pRVM0g~7!_z2h=6+} z-}lOs@16*aZ8&RK zrDyB=fxT?Cd5z_Qr5T`sS6FLpckE~ELv8UEzUc}$(AMc&7=nh6MuVx|eBVN|UA7fl zgWxPlGyOCV1JAMq5bY1GU#xW7acc)_oNc*%oMWW(y6dcal>3G=%|05S?;`LX`dn7` zWls!P?wdFdIW9RaI;K1NIu<$Xj!DjQj;=O=IUVkFY{Lzc%~D`*;Tq`4@r-q~bX>Fr z!DdxyZM12epS@@O1ARB$<<2XPN=G~AMdvoxdAG|=cQ0{vaHKl+IQBax!(&ritvnsQ z)7@2ep0(bb470eY=5f|Db`S$PH`^CkYt1g>I(>rfl5V+yYVKxTWgTO_YuK;9rGKN# z(jM003=!jN6WiRx%mt536GJ=g6!j+cFwJbuRrPUjFX@!`lvRp+#S_IIpLh0Z|*Vd<_ayfShL zVL)QgztLuN8k&r1i;P3wK+Z+oLp?;rp;w_Ts8dL1!~pv*V*=9x!O8I$d?)Na^jhS{h%HirJcU|| z8Hk@moI$L{55+!3S-`hB1JMgL2s?r>h_snhN>mcI5Y+e-d_69iFqGVx_MWyHasgXO zGf27MMZZPGFgCGXvOBXGEH+rxc7j=IDihD<@vH)>=(6yFU=$!zYrxoao}=Ym71V?M zH3INO5+A|CbCk?K%*~u5e3RgYpq}57zn!P%++!VPs+nah1hyuxy25% zKXN(&6#5jnhQ4xWU>b_BX0T(pP5EsFL_rPzy?`t_26pNU(L8Z%G$M9*?4H=>aev~5 z#UBP2!;*x8#LbBXiF?7x&@6s)T%Wjoarffh#)aap#Vw2N7lV(%0)!bKEsn;;T#davV&%HFlCC&prokcUmwrE(Isb58HZs zlD)6Br+K1jHryM#nU`Cbwp6>+X14f@Lj7dz4{fR;$=t=-*%q+PvyZf0F*6{oK3F$i z`$IEa%h8t`rWmg2R%*&r3}qjMR3THn)ebPMH42Sm^?r>@<%21~AZ3QCP}5fTMYl{_ zpq47dP#NA=JyUH|f7g80mTMORRQ;Ihl=8EpouZ%OoU#_G#)X=vs<0v;KPPV^=g6Na zD%A6|_cTJaMY&p;psbe{%g)NMU=#K!_bN`v$HCuI$t`k>B1Umcp;A_>4yZP$HmC)f zPpVdmA+k%dj*5edWO)c~n@?pj*-_aknOkmD_EEi2%#@Fm2jrU-R}`1wXJ1kG(|!a< z`#5bESgV+2o@Gh5_5!f)BalYigGu!SNZ`|8A#1bymRI2)5-bn42v!H=fu+HTp`GEK zh_>hkOfyUw3XN(5CrWqx2Et#`6xwk{1$`KmLEcMDAPQdL zlIT6@bHU?eU^eA+hYocs$kWYaFc`6nS==m67yb>wJ-(XzgVTX?j(v(<#Ubz)3j2va ziyw$?2ucBgTF9BpnazF0n=Qx!XC+0{Rj3f0;CBGO>ml}c?m@v+(PhzMVG+NCGmy!l zFQrYPGnnsS>V1>Fp7n&elPO{jXY^w%0h`lZP9b>74zSm;YM3~dhGk+$ad*NkFjtr< z8ZN3A#)EIBn75s$67&#vi^4^vgY~imJoftpgN4t<<0aatb>c^&KcbhSb3&3p#%s+( z@doljye)i!V6UJ^*c$#NJB2E~m)Des))E{!C%ET1H`z1UyVzFt zA`Xh%j7Jc3g1g@-{wm&I-co^D_)zpp%##SBuSOq@u8}N~@S>S9onwXZ?-Gg=D-(Xk zZ;4wQTNBehW_R>^$+IYxxJGOeOGRS@E4WA4e71(w$Qr@v%g+$f1s^!$nS1E_=zz_I zeqA}+&0WPi!HwfMnWLeP*n--Ib{u?KH0DdjP5K@hnf8U6N9{&APIi;|)CTHoDw9$~ zs=%8u=TS7&GBgeM1W(4_0|ab5G8ExNO2fy(ogxFl`L0L&jHSxa=JPY-WR7YYXrcf_9IVyv* zgC}5Wa6ZHeCr8G^p#5_=F|;;#F6asN2a{4oh!7@(ABHZ5ZiK!CBmPysp1v`DL~v9{ z6WSZv9PA&^`riippoj?uY`%Q&OwR#W*!kcq@b`day2HmPmB zO=w#UE9~=3qrtrR*(fo+F*X?b8r+6vrtYT0hLO70nm_7&>P$_Fb_d829x2z!Zb`38 zdx3R+hb&P(R(?eOQ$A6SlVwY<)eos30-c~PvO)3=iVccO@+#R;`48m*b&Te+=Dtn{ znUh55ov$^0H(b&u>4zK6nY-Gt&O(O(WIPKkCDsgEo$U=|)Tpk9t{tuu&M}S{dn=m& zR4snUi}dig-T7{jyPF&7?f_=ph@;L)^OV4St><9)-tH@rRk$dEjehm1!22n`HM zf`I@o@XgoC_tMurP#Ag}{u3!c6h^Y)*;x_X9U2jiiad%8L)=HsL?>cyqOs@_sQ5>s z&V!nQig}KbBdWqXL)SvL!iBKG93QF#&*@qJLH~Ha%-6=Z7jX9uuiy8^NA>DmDo1Dg z3>(ij3{KuHj5aMXIpC(9X;`lBuYaPSVE6c4P-Sb%_i0qB(iExHG zhbDzrBI-~$=(KYQyYTO@H!!y_<1mxZr%*MhM;JeT4=I|wjC6qbp1>js$taqLSr0V@ z^jg@**lC+(GL%OCNbXDB!Ms76U1O4U%ZZl32+#!vOJMc-m zoN<=@j@JiH1}ndmH-PKqlyDF6`U{Q<9K2^R$;$)H%K>g4dl#b>T|y^;x%o7$HPsEi z?N#6fA4A4ao|8M0caq0JU)D(}q1-0#C3mI-s7D!N*<-jVyaw=CF5(FVIH6RKC?N8y zxf8j5?j7E89)a7Ay^_%Ya5n0)LZ+;74s5}EvkxR`JQKL~Hd zeZw}xEI}7xq}bK?u7ox)@%V$DiN1$U#&*T8A!HIXxbE0AOdN)Vc?MPPa3lkbg_XG5 zxCfZF$oAp$V3FnruZGS7xTFih9x0EMMk*rP!v_QJe0u*Nuxy_}K1P>gI$(;ywRZxg zfz5R>>TARho*P+&kR!MVOZaP83iIdBfaw^H8;5;>-h$eOdW>F({e(-vqwojtg+w>0 zmTV#SBb$i*2p{pM2|I~{h`EG%+!yRuY!3bw0ZHsn7>AeQXao`YJycr?T7Sl8<_cCC z)U$L>D*HS0JfniKg(U^Edk!N8CLp^>eTXfHtBDsq@fxu3buyHi|R+i^41BsI1K8zj}DG!HcmHC)xL z)?&4p+CuFC&1>~9NUnLI5B^eJ4iS>M3WGYLJ7*kb@mtk!%VStuS*Ke2S*4Z)YmA-b zYVJAXIqhb?CadS|^m)feNN=(*-XJGVd| zsF(A#E5ZHC)x|a5_1k^kyTZ5K_tAIOKOqJ+Znr&6lT4e9uk}l{ zYV{xuU1!(b(H>GEEdainRam&7l44E6--D5^PjIUb2$j?p7eA~KQBQ7tf= zvG;Hkd>yVWek0*6X$kc_?LVrI?4*36zhd3v41-+i2Y!`st7wX79o*5D3C9XV0;Dh- zD!XQaBi#G!tx%OE!A-7_zm&Iv%i=}zd3*=&41XE_6gQ89rJ)0Pi1y6J~AXRJc{BfACdsLC6n&T%^zG zX1FWd}-= z8sQA2;I~2-bs|oMbz^Y=49diw#5O`I;w`QT;T#Ar?om!cN@W}U4Q(F%Kc<`&!`=vq zlJVe4C2-JeI;#yMmzqafNvI$+kh)P0L0UwH_9M~A?+7CT553xN$csoDA{YsUU7;Dl z6#rQ74sV686WkOP9=_Y^taMb_$}K-kEsZt$fzY+*8@55%u$69y`iQbpIY?vC#pnyQ zoz&Y@E!AvIHB92VYYMfqv|OE9{{hlgLNmd%%8+7sX3&FIbFuLUR3j5$*JOnuUB62A zpRO2e!XNbk<6!eMQ;IPMjN~4p3VP*5hW)yM+Habv@JO}VURn}NDKm7%dKbJerx;}V z2f8QPJzA`;r!EW{yubSI#?7YRpccfLuNhYvmKr`Ax0n}Max4P@^VQvQ%$jW*ZWWq; z0Z?I=;i-`WzgKJ7WEl=!daw1B?TS6lVX;@(E%toJG)Gg%BzwLs&+4>{wPxCkAW?8x z|Fft}XG|SoKJn7THOh2Pbsj^CDcvweldrrhPnHwpo8++yx}uf5zVW;i2|}NH^+)UG z*LJSSs=iU3T1Ri-H;k=MuQS(3Yc|y0uI*j>qGoo@h?=1_H*0XUb87h2v9N@Hvewx^ zmNk`qlhUP!8siiurAW=uUXDDvnkkTwuikC?Em>B7p!Q zfsj2<#yR<(KjZs)Udhkrm)x&&?sM+zbG@&(&e7)>j@0p#_Dl!SG0*;&Eye!AS?tO2 zr$H(HZQh;UEB-Ms1Ge}t`qF%f{>P9FK7qV#LHIPpfPC;Th~Q`hdUD_plmYz#?SO5l zf3SyU|nNxe%uhV%nopsitMa?|;n1>5=G00C|s-zL~3sN$gjBPW4-4zbej z<$VTncy?YcZ$5AbNZ=3hf8~j|#mG*O3>@O^2aa&75!df6ZV`_lc!;nRAZHk7DkmP% z?J(I7nAJ#E*LwOm+AJE8#wCA5)I}9|HR%t^3~DQNJM9Fb8F|D$!U;hVy*Wr?N#t_) z!-d7dA>lhwt7u5{RQv{ER#pjD^ZC3&-W0x`pU1azZS3)!CqS_vPjHzxjh7FsMb4Nj zIcvFpAjHe}ECTY3Yy|cLjhtuf$*h0rRrHTc2fLih=8`#P4vX8(*#WHJmkQnsUI`wE zwn~nN6-q^6;;pp2dO=wF0qthavgP*kVz4(vA^+10i|=4v7$_VJD;$@1MWJkl#M*F_7Mow}bJ3RMdn6Oiwh zpDebX&#L2&Y2N?#JnyK|%iMam-1^bD#Ek++44lwqxx$@43_Y}7@wfRT?)4^1PqmH; zCc4rbXZ^|c&%JrtC-&SxlSkQgq4`!%qir_hiF}MSgZRR^Nv9uRc@JT7;6L1Fk*n9! zzTiHf6rgH+)9wEnGd(O6(!Z}b+CD*>Y&iiQ0da#lDw~R>JlNLR+i2e4M-h8{c-@`; zh1LqsX|POB>B$=U&WSd)XjW;_nuQ8pbF5APM+ctjF>UH*wDPT5ZaC}Lk=EdL855LW zbx}-drZQEID%nNCs{IF{pTTR|>Dqlrzm*r+p9~H;S{|5df9;C#f8!qV8sYiQvDR(w zbni;I1m}Ye+SYo*ICn(b30dynFuya4P!)rhY-OC(@I9Q7o?_iR9o5+iwYx?mD!1L1 zSWg$8LVpb5&8MMestlKbzr#|{*Af20UdC?$9*K*oS|XiY#I({kQ*{14@I1;Vd@*)4 zyF*yP-bbz^WaG-<5dlACKfer65L!^Blx*5Q-(rj1or|ZCO5tdChi?`B8l0}9=w6^h z(Te`&&iL*->NNdqW2C|6Ugm}R4Xw_$$vq+3JG}=w^&Pa%&uXmpn3klRRsXnodV5Ag zPSdBZP&NDW?554NA6pivUv`!#4T|H9xh-rJ>T_R9ShGj5y0fP1plWaXRE48`gd(ps zNwdp67WKD#T(7&?(KNlaxEGJq@l5)Vp0aL{zSOvTh-kX3o2a?#2!L0I%FW}Ut+3f! zZi(v2)Esr)@;C=kU8DMpj+bbew?%KYZX&voxhh}u^_v!WM3`{|9A*PYi7u{6O^mX%sd zR+IX%Phf~RmA!)Wh_H-*OuU3yixIj;6QM>-6EYbLg;|B(h(`xN*G-fH>1%aR^GHXq zJJ1B;IaVBFKE@dE;PQxavWH&6PNIiWF4HB{J@lnvZL)`ei&MhC8fuKp2`vxT3(DFa?)KiXwHRPk}q!XI$~-CoU@OG&>uA+C9&agr+hcP==v};5%!U z?;ual4`E7O(Ev(7#r|eGh#Oz27#$a@u>+FN8xd2mEiq z_54Ci{wby0nfhm@u$E6H;is`i_~REuvwZoZD%MX5%IxPG#4wk)wt z!=^K%NP2%gA|peRKyoqu3E>o1AZrcHr$0m8!0lq3W6&sj>C1V^;`bpZ`R{2Ix>IyG zx;x4x6w(eOciBtHBKg7a!<;ZmIPD~}j8;J^r0UtT`8Aw8;%3xz>~KOUCfuuZEc2wG z-{SV5ce+P8%Td|bE$%B0I{bz}p}izufNTyCmzwI155<9(O)TSNQ@weK<;38Dfn1jbN^ob}raR7nWqy~h)|q9Im~UIpx?o=s1bQ>Q zX})%Ey7x~|;X3Wrqd#L;qVB>PcntJyfJ3gJPah;csz9(!%jBW(^5yV*!K%xusIIPp%i#tglhMFRI({@<`Rz2C%KQiBq%jrQvDO3;XNy?>g$NpTJK`>nh*< zR41)nQ+@wsVa>EAfBp8lCH0sZ@~gs{{tx3n>}r@>zxKnG277Hp{RDNZ=4e~Wr|hQm zx=XLBDz8>wX((vPZM)K$*8?K-VyddN@k#CD#y4#@TFUBoeJoT|v^KVE?r2o<70;EX z{>AEXEfgg}Za8-Z*uF8=(e_?vkws~kX~Fo5N$n&aE)2cay~QwPsM@r^X-3tNo0v1j zlfwL4s8?9BiS#T7si#`lXQk06IY4G@Mf^B+-zZ$NF}J|6msd3-C+~N@gXZh zW&_czr(CZ%DN-GsCKd1=F~+cOhZKcpN@OC9Y-`LZ>1%#1ki`1{Tw|Z0Od`)_D}iCG z#|$QW94nG(Wm%Yw=R-g zXCXC`S1<12y#u`j;?`JCk)2kP9<;(-|6)mrSBxq-Bt+2s@|`IAy$Iu{`7j zvxszmxCehIP-O#3Q3A8jq99!<{$LBq4VZG zW14o(K$&^*U|MfM|CXUR_p-o$z%|zUroJuERoN*45Wvt~pYqp(*bQ=Bb&{#5@zd|?t+Yr(?8{Cala;rVCYpJK$^S$G$$uf9o zV41PNF1MdGDs=1tsht=o^YKhLZJmbOuhN|C%hS9W@ah|MFZ*^NTEbPDT5azj)l_5q zWCMrp_l@b!?UtxZ)JOW~58P7ERp%RjwYUaWY4TNl%1oWlwcNVPI94~NU!=cjx@m~e3f0Z3PpT+YLHANbgvjWBrKv}1NaCKU>d9TZdcV-x zwWmy@K`Z#kxyB|i|8BKA*En4EHurr{=6xSn54#*nOND)tD-`*@`!AI5dk8HE^!Ubu zA6z5B1o!}w{+9b-$cIz4eE-w(tOTz-HaRBKaeu8V(_E! z2gB9Ad!7C5kK1N;%;>nO5Pd?wSHH=ru{L#lPFKI^ZPWw2cCuPNIUyE`VRN>t-jKzCp7vBC4^$Hw>C zOigM3VI4|;bYQqvYGfi5K$`I%?T7vn(`)-&<8OmFWZ>L8V}p5>(QGPmvOQN}BmN!s zqkj=RhrpudQ!IEBDjV~bc!ca=pIpf|hAazl z3FdME{tEFzp@+MM`H~*T&g2#X+eJR{C7^<}ocoMdATEf}B^3`l96MI(LOKPO^RIJE zfLXjNZ5QAMOGREDNh>fVj6w-8FSh-Q8W&p1Qlcx2c}G_tf3po3?Sq-Cgc`|B5?v z=Q}(5Lk5ztuym<+ftvWtHV?T3C7Y{ggO z?XmS(4LODyhfDC~g2|l0a4eFCe&GeUCG2AM6>bgp2d6!ECGWK0KHdk*=R>%j97K-A z)3GFM6t-L76vzn^5hgXl6QWx5EHaL=krn(rc74`Orisy*wSryFS;SRwyYS|q>8Ou4 zi}xGpiQeST5gfwH@m7L${Ds(J;uO&g-+}W;KJg6OjV~hJ<4v$_SZ`u9t`~g8=aZ+& z-*iK1iZVTJSxg3Z-C*a*QSEEhY%AIqP} zPe&cdC2n)>MdTWu_@EpNhv?srSaEn~xsp7HH_VTB(3lj`+Z-Gf?BxQO#4n`9A-2|bl98YNar zH_Lm*5{fC38qo~l7;-7OiaJ6}#oFE~NJex6)PQbR6NI=3PWx zJPB_MKY^dl?ZI7y?nSGR8qNt;Vzh108fX{#6NEzjf*U-~oD)54{~&+1Z@#C#d$egBrmhV8l?s#LW>+e>p(ivMD3k2Rfg0}@ z_gj0GwW+0*X}ht$>7L=N)}iWH^QtC8``XyZa@Jg6{G!{a?x%KZ`x|x|9%~~ty=z{n z-|C7DOHFO8K6}JI-bUIE+HX4#y85_Uxd-^J1XqWafpdFgU*uVICbNR!iFS%ki9n1+ zoYR~t)*rYDXC`L~cPuxCo5wxN8N}(qnGFwxGdVZd3GA!vp74Ln3g&UvMfOsjRqzCR z&A-RXK(4dbvJl3qP;>t_|LEY1(D>lKz@$)lXsf>$xT5E-#V)z?k3Gw=%u!@Bn{S!& z%*#!P;fcPdafB(+7_Z~&>Kc0**J+!o-d3-zR#anE;wrRyd3BeXO!a&18ht0-A2q32 zqMffvQWe(RRWH)c(W*4B^pCAVXA`H#b=RBa8{zHdIpZ4QZ0yW;m3r2A4|?kdMnyI- z&v7;)-MQ;H3hp=dMdm%mU)BruGIm4wAGje`jMm~m;KyLK36#>4T>Lp2=H&Y~nYSLQf-hVsUr~Ysj}EJ$Xs!YV<350R78rhb#u~RStI`GMIk{tlg7X5jKr~ z2>HwrvWLTuU>>^;o59w@C)nG#>rsW^3F=1Tz-zh3Jtifa}DwG`U+0tsnjH4A(exxu-Qay zaz1sPmeL%mBW0%_iSorIl4G);G2<0hMEqU^YhKZE!wzTCfLWVf*=g1SCEig8`x)M>S}yKqrXjXY)2AL9Uo-%*h~=WWyK#`NlX^{ckE*_v<121dme%~y9yS)4c9>S0i_Dnun3kcf1}9S; zqq`l-a!TOdtQ!oNaT00+IijtilIUXS zBC{X6FZUtmKAXvX!9B{E#~H=>hi&4#M6$UUyNtDxiL)NT-QY^5k#UGYG1i4y{`ECuFSmd4hWR+D9b<&mY_ zUgUn{jq^K#S0h)U3}!>-HO5m&7F`mZ#puRZ1qm5i2EqEyTF3gp+6cd6&*%EsqnUM? zk5~>S#K?=@k0`;leh)7Q1tZ7cOt53WjI9i(^b_?1^|Rh#^vRh@ts+Lv5 zDi>9*s(e&&v0`K8jjEdJmo@!V%T-rZiJDW|LTx|YcYR0WCnLvLZrE*(vrl$bxMJLu zPK#rMV~M?yt*0%~p>ZyErMYvxa{`USk0UFhDrh@vH~S{LD+fWoaeuKN!sFlvu$VKE z`;}LS4JRQgk&YE!r~6WKiI>@!C#C{;}s$-WELj}9?mRh z#Ih#BVb&nl8~8al$UA@{=quzUw6oW2ww_& zi_Sbtzq-15b&WzT*1EJ^b=&l@2D1*)-PQKd z&eb(AhRuU*q+_4+zFX++YM2m28PLF7FoO49QI9)wT>5#agG~~udd}D%$w%xFqk&aL~}&*w3u8W;$zopZM7UmVcD@io3Hb+d0uuZ09-JI{UlExOaPd_?Lo5 zdBIV^C!vo~A=AyQVr8)FaEdw0z}`KHm(1^oJ;mijBJqg`lAkCSRYtv`y3+ANfiRD@ z(~CrZ#C;^~Bqqr*=~Kxx@hefWa0{&^TjN{#2HprHmb;(J;nhKh@mmS*^ZTPT&&Tb= zoyGNX^AM7^gXiX<{3U{VSZypKI4B?lxZqy_j+q2Y1ziMJ1-Y02UrDT_;)Rr`u4syI z7yX=)lb5kt{2bn6k%`T;^qTxa~m+h z93+P~6IBAN*nl+UDB$k^+24hKgqnu3LuW%V;cz$_y~BI~cV&m*vFst751bdA=bSta z!8yRG;CAOV^vNk~yUY=?ke=!j;5IK1jw$jneM2 zg|h!-$7HLdv&HL#M`@Iwx&yK0D<<24M`XjM?vMtxSbuI5Vh`s$z6_UeAstEvuGC0BQ< z_EtYq9oD4imgzSdx*6{ouNhky#~U{4PwE-QYSUiJT${-ea2<6&akuk4_U!Yf`1=M5 zz+Hbi#0}pGbqOg$%|e?);;=aKEn3282X@_9_+K~|>{Z>^wO})I4AaMSvAV)ucof*H znxhMNf002*N8~lK5U|Qb&Mx*lxE@TxcJO&%GT;>Pn zTxJ#H4P!n-2tAFwh%}8>MOcv*;Sr&%;1>T9?;rOl_ZasuH{$8zo$uS?j}KJ&I(urI z5A8jzeav%Btxc;;XUzH5Z}tl3a(8Xdf9}n$9nPc9*)ERT>N@8-;yUSSW9Y^jdcj$d|}owl{J zuD47yuQJ^?eK7a5{!>i5>U(B-oB7`PDt)g2B6bPX304GehvtXzNayHl#y%KmFYYH~ zEpHibGH(Kp;1wYz@Tq-8oV)@2XZ#GoHcSZ|iL2yQ>Kr{s_)M54oJFsr|I%@EYjQ4r z6l;pz5flq1;VoM2X@K z;tArBV!8N-$R^q$?k4#n=_-9GZ6MnryC~Z(J0fc=8!ZV5Z_-2Q7+OY;q3_efgr9^Z zbYp5A*@L8rf3a-LD-a5vqRn_exJs@b@Q_sQbmS(l9(s#67J0@=WuInE1S@SbtaJ(-=tX0rDHM`jPZ9nh9L*dI8Hk!I*$z8L$74aNW98AL}SiLm28 z@FKh?@rRTNuZmWQpGXR&jisx^=S7D^!$mAniSV<~FO-TF3(wPc=()nWq73mC30syI zvrIlr-ZI80og=v*iWg3&vw+&ULN65+0wjUtALK51Bl#OyhIG3)SyV+Yq-uzh_yz2^ zU>Cm>U5M5~ukc{^G#P3HWka>0dQcZ|WjYfVY8n9e6Q9gu`{S z+pq(01+0Pxu(LT+x$lsDJTLDwn#KQtwn9gtpV8mwXmlm-2+|UfAyMusr3u0|e#cZYYmZ>7H_&_2(DyF%B)pCjj^xzH5ob@Y1l z3{(Quhps|>nNrp><{W@z7n$FfN@gsh5hH{7mD!uch3CQt;NGy7ISXnT`55{ySQ*$F zFb0~3D#F@G%cwNkEb=HM54Q5R^*(kVchzz>u`{iuCXw-^zN2n{wwZRKHdDJ@b6b5t z)whOKeY0|5MW6D`WmRRZDq2-uuQXTbYeuWHGz_gpdrN22HPfYNpJ{GrvNTWCCiOp> z)!GF8Q3GtsvHZ0ycQkNra87gL&T7Y2N5HPK=R2-Bx40@?BDc-W_cr!E^t|%)^se%9 zyaG?EyO}%3v)KFB_c|aC4~~i$dPX~D74tjm2rOn#V0+k8xubw4=peX_Y49rIC&{DE zlhdfd!iA!~qGZuH(E(A4Xe)h+EFr!S!w4&0i?~8e18T-W%*FQcU!wo=$Q?XNyh%R&$OX zM6+px&ZAe+L5iS{(y?>_*`4e|st7;620IK`%r#ywB$t!Mj%Pn)pXapVzT{5j_2LtJ z7t(?o$9W9bXH_w9MnA@2W)|xU>i|56-Hy|rdz6cEA#Nr2IPwNLhD<^V5gg_6xAE5q zJXjNgB6pK#$PZ)=wUDBzsU(YBLf)bd(HeS^Fjv$;ERr0N2&GSDT@}lei1NE4O=(bs z^nvGtVmR@13eYw`*)hA77S;4m)3lzb)c9(NW8Vpp&xv$nBXgEcmpbCJ`M zQc~(%;fI)IZYo)m_&<)l{kKRi{*TugEQbQo6RJUdh#xyQL?}x|Z**xK*{h zrjzQjYNUFjMynO*tMm^HHsfD&wyoN(cWiTcJ#qf`fhD2!5e4HZ%L3nqP4HJX%-O&m z#x}DT0Y`iZIt69%n+nndFVNb&x7>d@9L^c`zwBq6v&a|8<}r4~~=!0xh`-c6TM%gA)X4IH1v=t(4wOLJDTo3b5nWB43X3$=~D z3w`vr^ELNf@QOWeTte3+XESG}RBF|>X~{Qn;7)^ zwT5TL)@GCWmie~1mSw5sx23r?(dxBKvOWNw#Wjo15(`lAymOGdn|GA|N}wi?ALtcm z9vB{IA1L*I@;?lm3B8Csh(3(202nccbq=@|Cpj{n2JOH@bSRVKgjv~jG_ZQvO`=)&5!+NIhL8n1eUdaSxs{X^Yd-AKJly-%H>UZ85L z+M#}^{h_dJay>Pa+CXwi9yx_b#CKp(tPsBi9+^x$BfgTe z$?^CTfs?OCPlLVjH1d}Bo}UBUg-iH$Vh*{7+D4}eQ$$O}T5(-*3sG<3CHfWJR5($1 zNjO0Si}~V-qV~dmfK5o~W7Kc57rBd^LA|0xU>$u2I_WPqM6i-C7tF;rV*Sz%EtEYr`ADdxU=F|KU6M z=fPibVpSl~*(@j#9Kqu8QTPJj$e=7etp937n*|>ykN^PfBQ6VY_ z_|XsUMn>JXc+vmG(dOY4j-zWd~ zz?NXMP&Dj=cCiZJBka2z1NSjtldq7+yvO`ljEyZ9%ojWq>=Lx+x8Mc2Y-AYnFLIH$ zlD`|9MD!xF$gbo}>M^}Q7!sC?+~Ok1S4pKA4a9?&;xz~Gccmuwd{sn=-!PCL_fmQy= zz60KN-u1wJk^08@{J#BujsFpFj@tSE_#Xv!1y_ZXku+!><2)mS@e<01jsRW}!&uL} z1lQ%P1el)6?az&H4s%v<9&sS<5{`n+f*-RMvgR;fLcHjYaL+I|{4>-t+%X~ptfEQe zMd%jLBbS3SgUsNCfFNM>_4f{N_jlUtN_$V+Nb3U2Uh{erW)$fS8i9IKP5o+F)yc}j z%Dk!=pd&|DYpd|;Q1!i$t_ApU zi0na(!T$SOJ>q`jH0Ru6bJ%m>3-ChrC*YS}f+xeX;7)LJ)&Pb%`aQZ5 z+R13cn!{eq{fQ`egOCQ?g`9Jo7hDt4o=5V`$O2?8cMoSNC&clBtNn~N7i_`$;8XGT zIE~-Laxf(pi!m`7_8t3#4PHa{}pW!#fU$PCrc(vpUIxb z$mHK*7RPYpT@_ZvImIM^2ptp=`Eu}VTpAn`To7>j*gk{zzW20ep=-B2U}Jc_q(EJ!RqYoXV?J7plkB%upRxU)Jcf z^Yu}~4pVarVsqP{IYsX29@5vuul0`zmWMN;6O0ASAoB)b7>S%X?nA_g&Jh#{o(S|H z^Qntp#2~>rG|C$ToaY9-Tr`J2g|9=0qP5XQ=o{0LO^~7T&^iS zntnk~5WW-c6*_4y-G+Ka3dj%me*h;QVBfF|><<4PZzi`4PGRbEP;f|~$h*v) z=h8a4&hC!dj!q7p{iSW1wbJ6U?63?o)23Wwn(@7{n|Y_Twd1sl;c4l)?bf(zxg5^H zF1^d`>hId=%yzP!zZ?aQJd&gr6fs6X zQ=&5?Ga@sh9UwWB5hWtV(CeTt@GP(N_|LEu1(i3)IZj_0NYS&HfdPewYnO^VROi`%7R$_T3%RH)<(8g z_U4W)4${H1_p{~Ncs7oeu)HuoGJiBLwh*>>hu-V!Hy3UfHIrJH<7^RrD3gNp>Jdkjtt2 zAZ2+<52HKNC+T~_>Efpn20(;z$puL_$rwqlv`DsAu2*=Jhhqz4f5uLU9jx@p+49LT zYT0So7TE;Z8JSNel-a~GQ5oHlT0@+{@8H9U5HXnOfxY0L=3n7A=4;VP^d8y+T?4dF zYh(Z-2MT#1I|*J6KAWF#9Cs$K7~GRf`P#H0F9ZGsoAd(>HijxD9mIM9ArF()WF@r^T>V(lYSB$` zigdWFDCVYOU#u|xO8koWHSq}vhZ0&Pq{e@b#bZ-qSH!wvYvO7pyh`9Cbc>s*6e~u{ z>&rXJt7DGIswFAn3Bp?dCDszniAF?S!hq-EJun-8Bia*W@)B-$kP&_w!Bs-TZ6lW8O+$CYl2Foe_X(b|=huA@)n~N>GONA=-c* z%?t7)`GdScZX_$oA5;z4mAZ)Liava_H8W@@x8W-&DH+ovTySn-Vw0Lh{VlTCMtoMNqyKBlYcQccqQx>h>(Ixg9d+c!9B0Tut!u?jeVT^-LH^PRI?SKVGuTi+g#epCk6hHi&e zhRD#1;Ll*~P=2UGxO1c{NKs!$euO>1V^#1WJkdHPZRWzS*|l zy3+E-yvsbmY%onT9WmZA1oWi7Myu2G&;-Ii`B z=dFE|t(Nta1+#{&b#0BUEi9$xzUIEB!^Yi4#Pr^@&fL+m!kTG6?fB{pyLx->dOQ0a zfyxjk+MKbLHI<#kX~NmUX~$J_Zz3jMK7Rx7JD(6|hzK#A8YKKAx+U%+*)KUNK_zz4 zFwt94ZSi+emS~Nzv9L(kL;OneR%(^bkTRv`CAeg$L?-(dvq*kVUM>GBzb!u{KOncu zzblR^<6|$x){W~B_d7N#wwtoKVy?W0e3AU1{F}VBVvVA=Vz0bg%mnE{@l{bD(Ia7R z;ZR{Uy_`}Jk8ukBhONZj3sAv${#+lRz+*_^-Zew^dn6}&2RFJCA~ z5LEJ|0+S#X-$;xhndERHml#SkCdLvlSpC)1ary?`ioQy*CH4@yo9z1s}tPu zljDk%JY{d?_E>JbD4~8raoj280y#(4Uc6LTSLmSi)EcrIk%uGr4{Q@Q80(E01m*l+ zXb1En?_XpNHHS&$%U;m@&)UEL78;8tm0T`;RVJ^vR{2l$VO5&8RNumc zTP}fCR>bzfHpsrwA$QgDbnuz{H-baM!y-4LCI$*$V{^ENx!1TYkUt2AcaC=s)uAH+ zS}fxwqqq2-u?F}Y+=TrW6!AOqC7|b%fHeec&W|~;fAAv&7kCwA^bpZIaeYa1$zsVa z$#TgO2`PcaB|;0`kX}s908aWXY$-nlISs#J^Ayt^<00cbW2|wgVK#Ud*v7WzJS%PgZ5KP5IL!7u`)T`QM}c#W>z%9A zg}EK>EN@reYX9TFqF|#?o$$kO{Yb6ouIQD>sc=!~S@3KiJrEaI9FPTf1fK<`1}6oM z`5A$4fwjS-AyK4Dv>tR0x&w`8v}M&}kKvHqQQY?2Pnz%HdFIcy;{9YJzH~JyG?gazteEX@Y8VCs5Qk{DlAsZ z2aDXAV?AM!Tc%sCSUl!srWoUK&_iLGx|tx8%P?2pLx*eQ)k>8Vbh?hJovIHYSJ|lM zX`6xnGs5uMXf*lE5^H1INPsRcotbWvXOw>wz=~f`SI|5g4>GUWprz0RzRDiYZG%{m zc$C2p0d=wpP2&FtQjlK6AtFFbCXbMt$P5x9SCKV9iS8%(ptqPJ>?PVN!bB~FT#&UM zCOzbH%1c{>?L<~lu4KG4L)IT)#~{TXr8ur{{PF~CVlZiBa+Bn9Nodlq#Og$Q;*Z2J ziMJDC5?aNdiEXN^R%n3M|E>@!?<(RI`SLj+$80LwBYQ4wC;22&3b#>{$a=&gyd|-g zjF3jqB~7L?MN1?u>3rE-SzXyLfLkvmDI6TJz*xv+S)x_i3vosS%Y>?W(<^3jrEwObCD zON@W@KF#NviB(@KT2=TeE?2rM1Qp#Xo>g3^&{S-%3{;-2dRD!zCS7$vJwMZRh4@|~hGbCF zLAvxuI9NoA*NZQTSBtNUW(()hr>UOgEPRdN589RYg6m+t*xFTl^7 zVi8zISl$BW-q+mOG{dypTxoGzAK3ca_4XLYVS9>wx*c^4cT9Ec1Ui1Q)9W1Wn&rv` zZt7L%dq-!x&U)K=*B0j(=RD#nb^q~5yf;0UJz6g(zz^*RS41G_63D-=fbME$WG84? z%m%*5VW2pA2ONHj@1kdu^QvvSWx1)Xv5|2#c$KGt7kx8f<2^_hl! z#x!$t%Q{P(WwjYM-!aWJ^)@Xq{WJ|WpEY+i=a?kMd4^*KzOkdxX?$TWwr1JW92cEW z-B-O^eNtZ=Z;fZOXQjKZtJo=X%>>%3iF34LtmD1ohohHczWo$XCmpR~i`2|9|1!U} zlvy5{ubaZA7r+A^ZI)TCTF2X;IgU8zxmX^Zm+hb7ZydNC+!S6EUBYO}ItbU|IJiYf zXXHG$6*7buU+$2(|=b(#lhvHHLfqNZ8wI>zi8?r56>po&3VJDgZ z-MO4TNUx-~QgI|kG{o;?3-FIX_g50ls5`3M`(ud8qmBQL9#KFsDq!uhTvNW60|{G*fGH?6JBJfl1=H`_hTInjQ>nrS(0;ut^bX}w67qB&kOtg2pRugd&NM+IDwSaGm|s!*0U zC_hqhxvB#6g|k2!x5!Y(w84Db^4!|UE^(&2A9`kaQvsu>42}vJgOfrR!fhk|up&|~ zdI#FdXwA3_!heZCx;{8 z_&`SAip(d&^BVS8vqcu&|C;)DhU*9CI?kG)FIA{Ptva9`QxSv}@a=2**9 zi^@FTRBB|HCYpk#XXf3O4c0!urTk>gwG!4UOM&H^WfAbM1!lJSm+89ktifV6q?qXCz|(~6#$*yS=iRYRtEtwB2w6mzDcRc1LR!l z89h!|OL!6_o?qx}AuW7O@1qCO#Z)_L2Vfp+$Trj&>LF#P?t(r4wfK`{ztk?9AQvkr z#Y_1=^6~OE^8PVjrS~OcCCQTA;@={N@C`kkYED+*?XZ3Vqu?D-0>=bz1f78MI7Coi z@KI2SEyk@_B9?`nz!qRn1?>d?31D0SG{yl!PcEjL08JzmM(J|;61|x|N1N&8!nLAY zF(u9u?Gv>T?-XAa2gSLPo02)w-oPO_A9FxHTftQ>2Axg2QXD%Zc6jV>WoPAZISYBLC95?=5!j+`c$zxLLruENA%UqQ)K7C=@*VItT^W@J-f07y`*Cer$ zrX-F`oSSG!SQLLY)~0A9KPZcrmWdmQ^Tf}^4Dmn0Y`QzO8l*jg!I4DvrKEHN8UpWU zE_IIFL9_?+I0b?pg1-DzG@jQ45pnOcZLEWgf1*1gbtCL>NpN(iIJ`bGIx+=xlio#K zK)w6?jeI{mx$gVURlBjn%WNmQ@a^_*7;u znOy8C>Ri0Acw2E+30k_pw7j%l*{ZUNvXSL)%i}A`D*~1LYDSGx)ln_ceAD=}oj@kH z+>md~FgLK=wG6dp+y2;;_9EL}TNwE5o$QNjNw%>dy{v14tujkri^-y~e6j4b7Tdnt zWsdvy#&)KiVb|NLY&Euy_Vtc*m%zQr{lGK9x7(iv_T0CDdgWdVKsl=3QHk->W4XtL&0!uvDy9Zjv zSx#8KSr6EgoV{Fw+=D&KygPgxKk85S4*@;Lzy1w@E`Z&JT?D z5BFtxz3#8BN5EYj??|@au=1=?u)CkIoVK_v|5$%nt$_arY|rfV9NQe9f&UtHOmd!g z&U5~8gzRwKlb-*^by&gO28NyT!(^WmubABBlbP z-1y3H&G5xgW_V@zW$12vWqfHQjg|Vnx`A4SMxws0BGtuewFc2;>w6pO8P^z(7$e5( zrlV%3<+5#xgLJKS&-3)~Uh<6%utL|u6QTuB67wRfH+v7KA$J?s$8C$e=6(j9Tn}d_ zR{_$673gt3SMZHLfgeQYqdMLz-XxGDuf=kSBjiqM0gVU?X*s56b*1tkb zyh}pJY&tisUuS#ri#;Ar1eW%mUbB#LMyTV^=A9#r%?5B{uOx(NAGl;R{+vJtJp;wE2aA=4bJ? zfcX~#`!#zcdptakmCcGU+k*y~2YLpb1nsLAP&~uLxX1j(Y{DGCP(V#1oKR|Dx$l{0 zqWiw9r7PD7IS$(zSwDa-;!)#mFkj?0O*20*pD?>kFO5ltM!LS5ovH`bMU@)>M=+J| zFXxr7DtlYHvgAS0*T0e9oeH!++x=|%v&GM>pId(R{#E$P`3ot?EEr$#zM%8(Q@{Pc z8~thXXT=}U-|>I@gR}7O(8BwL;-XJQ86^)uRuU+$S2?X}Xf?lPbxnEAdew6EQH?}b zpiegHj5keN%yLT-&^rdulI&w@V;XAuXnJgBgQ=25wif^!_SeX zu%n-YaE@?2ayRkT_a_E>hM@4F@Q?6%kT2$kPKUC?W#PGzf`~bCEz%^iIeaB_3?v`v z{xQCL-iw|K?n&;2?mF(5u3}dY_YZfj=b{HdpJ#?=3vdDcI7ZrkS=(5GrrD-rreWsk zmL=BFHls~vTWzCl4_>YO``#M&@niAXaa9KYI$d4ev235fDJh z77MoXZM=iXFm8}-gSW$X;UVmg>`rVu$Wfm_cfcIp*}wq5JEME98>r_Rwi&t@519@C?3rdhX0A4O1(^S@^`32+V~|Vj9`3#7 z`{s8A;1DO=4$KFBfkrTg!3}^*Xh!Oz5Pu`Tr(h@6f~X;b)OVT|<%nmAU7~fOk)rvc z-J<297b3A_s4OI(727@jZNjjmtH~KD%#>5fvB~*Ky_4vqw4_$abyAzA`!Yt?YF(R^ z70&uM`&9O*?8e!%vK!RNsXM&h!1|H;-5WG(@U?!|`pfI}uG_cHi|oAY9@&!Y!&xP@ zOKSNu&u0$G`K-pFG`jtrzNjTzMDKC`EXK6;(rNU;!nhFiK~vSRxSZN!{P+%(QMvpF2%VGk6|5U^n%7f z$&8zzy*z^P19UP6Ma#nLLT`il!Slg0!3V%|zXdY?mcgg~0-(cH&U`y+RU1#}E4BZr zSAcoLnHB3Ra%*afO2b9Y0?FH01NhFFU?9{a*5A@#m>uhkq~r{^jS) zzmH3&R_WBG27`5mOW~UxJRA86?E~HHj_@*0Bi;c)Z(=VwkNk%iL5wD8@-ZI3Uja77 zBC?2m_&F>Nt0QosThKS?M07WA1TW0n#81WFka5EPl4CJ@6rEzdaTDW*gS=^f+*w7g zj47_BMiZ9>7ZDX}LNqBj+uO%|!qvnDyIuoTo$D!cp94uoH=o5@=wB9^9j+Jlg`S2} zqsB-d&`RGH3ItWbrom_a8G-E~eQ1AZU04?NLi3nYnYS3*7;NAawPVttOa2?oZGf&O zqcJnU+QRLKTKF>j0Pz9%R*#5GB8;!WuL@@Jy0Fy@YZMLt|HQ8T?=1WO*wz2o)&JPl z|Jc?4*wz2o)&JPl|Jc?4TXxme7j!_h4M z(z#``D>AA#)_ktfsA|-ARV%AcR5`24s*tMDl|L(%lz%DhUFs=4R`$AVOWDKH<|X@z z_(h)Ltcu??wC=Mh)3M8YBy=A1hCS?ju7|q}G-R3VQ>=kZJJd1yI3x=s_^0~k1*otP z@-sfO(%Ez2;jD42AoD&$1)TxkEYOgd#EOB7*d4fML3ZOq%*aDzDe@J3`cvw5sq-lNbFGURlWOnIF0Zva^S|tz`bX+|vX*Cy z>rBZ)vb)v)+OS_kLBk&n2R6RYhDaf7O24O^ zbbkpe=B~^HCR`x-6GhLsrSUD}(S)Ok@+5guo#YXzEz>0#{LJ;W2G#yi`zd%`c($ZY z=Q{aWL$eNLpQ`OkyOM&Xs1vUyfZzp_8?ey{pHzhG^_#NYFO zUo7DNO8K$$oB4Cu$KIdW-v)d;^fmYEu`l#z)%$U;_r1_QZ}}X1KIz4YSNGly|Je6S z@we7LO1~}mGV`jJYf9COk}E|UJwSwfT_UE&`ZWm=3C}-=0fJb z%n+*?9AMI{j-b0>19x?Rvkb9=xw(Z$h^(KgZck)@Gkkw)Q# zp)H|aAz7$0*dbUeK>1#~$GgrrrrAbYH(QpQ|C&cy=9$}@-&okTCAKo#3i}7gI9H0h zzx#@NwR^8C;*58Vb$xTa2eazzz_?(W;OpRr(1=h{s8bjPeUPcpBE})6A3n~Vf=t0!lZJBgRbnY7!CC9F8k1a2+=c*Ls6%itob2;a*8 z#``D`MQ#_?fii2mXsqyGp)}f?up)I^Mp4%CEPIwb`+7G+PFmir+=1Ojca?OL=U&f0 z+`XjedvRjf)E?J+9526FmQ|`P+E@6X`{IJ_1^hyH;fW$aiKQf>#8fype|EQJU2?NZ zv({#rGM8oSOq-f|GIdfKGGljE|1PI88`IsX^OEkwJ&Jh{ofu_}Bt@JPWby~_2J;MD z6sIig3cWXV2iXIpG9SJPTZpy>SpEJU0C2@o(W;G;)_jw#z$;p^+0>{Yvt zICAWDR-46Wc?mPmepa)^W*Kk$%kH<690%>IZAsSkmW|e}_Hsulkfc6agq`2?4o#-& zk~{=A(qQQld76r*S)qBT`Y!)Vn%B`sk|%isYBIevOO8=0)ec>;v5%?LlwsOndSc2k zy*B)&s}Q#1$GcG^@Bin&D2};(+tZEXu|>hEd5Dv_&4aTYHKx0 zb-3E1DpxnEJJh?>NX=EvQqp9wjGH zH`2$4<%YG;H8eE6oc5Ti0M^1RlA1V-{0p^%zJr<0JWc0QeZ*Jzzi~e7YwSwwEX*`i z7IGx|Aua}g8e56MVGg3tq4=PX&p{7@t-1?67?Xm&h#rX{WB!5J{|r<;@(dy+v^a1Y z+zdY73m+{=L;r~_#{R&*#oYxC%&*A7h#8?D!4aVa2ruF)f{j=fIv5-joEln+h(!)U zF;HLOR+=763z+=x11J4myhQgXM}obp)8M}3vj&eLoS`kDjX*)$8hRJ{Bb1LgiEKeO zfG#x^cZV>D+)Qnthck~e4>6B0hSKUN2is#M;;Y*i{239 zjM*G>EG8v(LG1dt!wDyo_NKf{9h`b2;eA6^u;7#A~CVx%-Ii~^L*T4+$OvZSB;s5o`+hAu^7DDj<$5 z!kRD+;8YC8+`ultS#TcwNiHqxy{cTpVi0LjjWCNd9fzv z>#3@|_f>ChzBauY_v-emN3T=f_J5!J5mhy(YFSmwN6g3VAFg~z_*nLl_3_}xCsiGv z7Jc6MdG_b%&tpCpf9dfh`SZ0;S3kvl`lHGZ|L4f3hA(Tr8LA7u_x&;I=hL6xV1vY{ zE&uuIr=s>=-Rg#)O~1D^w8o0L?e`@PsX(zn)n6mltkDkFNwh}re-6+X)kidsv_thr zjY+W6xM^8s+iZX7sCD*t4RinMS?77+dFJ`eJ;jN(Gb~R`gN;RoWZ36kGR)F*br|hx z%>hl5W|H=h4rjm`Ul~uDw5I*01K^Z>0Q9nG;DRMsSK6l7lbi`|ERcv6cyoM*{MUme zi0;UDu%(R#%}sg88=8sufS@B+BiSeoDj(C19f!X|hyw=932LOD{$k+ z@}6-y-0567w}Ll||4GmhUKAN8>kK9pyA_v^iLia^;(W&Uk=oR4LtpT_4SwcDZA}WCe>Y|RMk!bUQZ!?1$ zMV$*C%z9!SVH$W?pyR^r#FgT90u}2nDh??{tU#Vb)*eH;3KGW^%@IE_0aNd)ywpJG=ybD{#Jc zhd%(#+uDeP$m+&>*}b94ai1ycYgYP$rneU&ae@f8mMvoB6Z&oA?O9KtU^? z28^F}?sBf5GnG@$Y2x$-evgi`nsbgVV@0qqVKT-!xJP2CKJpV%A~RbU@>w5hS{R(}wewu!`p}Wy#oNbzCT4cIuI%6^$qm7>pDucyv z4HU;K^@~7jS)uEvE!AvPT~x%%cS*B4UP)>s-z8E>bjK(l$x%D%B@-o^+AoUTZK2j1 zEo++(G*vWCub){v@<(dTjBnB}q|ZmH*i~~sO5Rtz%YHlX&5GBLU(I=y^4k8U=G~kR zbF038)_vVw{i$aB_dVYo-}8Q?{}}t@@ej!l>5tYQYkz$GKK%Q#?`_}fzxVn6s%ACx zCL?P;*R+0*|5;FLt;N=TtbJNr1)JuRbmi29|;^y`m$pPsD`F5pMeMO%R z`mM#Fhc?=`gM)_YXtQ5*JaQJhuDBk%`*@wcrGbM%S?C6GB5YqPuvFX|+!efwFrGAm z>?TjAjHFxyF63t7Y9fk6COszZAod_W0>Z~J{001C!Wg0v2yaC4C2}_9JcUm^Ol_g2 z!bYYq?AzW`bLf?f<4gi;2ImXUB)AswFtSo;7wwIDF8WItDXfW{8i|e+MDm1cQATuP zG$Fb%`ee-3n4}nU^vmc`(PyHbh}wmG;S$hFr$$T(r@{T#9AScYjWe<}QXknF>5LpL z93wg(bw7GeOk+%5?914Jakt}CalSZN+@ZJ!v6(SzqNt*9;jPFak#{1>BeEm5h5y5^ z<`TFUzz1@YMGITS7)&=&733Zy88MZ#i1e70MGgS3ZVxpVxVb&a0b(U#GCm5Ig>_=` zu+y+WGcVym`F zE$7XnI~PLFCDG5%ULls-{!k=AC%R1qY9x$`}5+sb2#VO_2Ad1vDdO$95UCz-Omg1Sp2_v6y6H%b6^G)b4Gxki^UnnQE`ym zp`3PhPtI|86k->0+Bto=S2*8UpP9cia_QTtKgpj+Ye{F|6**1h5&tGG2Fk)mDx0(JFcsPn(+t8}xp*VWIInXnh0DC0^OcYN+R-$9d% z6JKk)+xD=nppD!b){@h534SeYUe}!1@~#En+OsvYjnW>JBuO7hD`ZtNsytu5Mb1&U z6+%^Sbx@t9xdh&<7SJor)b-TO(4Et@>$3DO^^Jxc(~QnU^B!}JSzt-9w43*tFPqaX z4$D$&ytTpd$a=`$*LlOW(DTDLHkg6vhTMj_f%z480@oLpi{rz6`3c*JJ&21Wv=IHI z6iNzhETe(>Z`gcL3=e0IWOrrDf#zDk$^+udJLUq$U-VyLCUBB=g{q)DqO?)6s9tc~ zH8I#>9bs`SE{nq|V_gL5>q5>aE{os9xARR}w9gVvdHz&>3Y>jC%)er#I~XSX0AnjP3Mz+Ahi! z(gt43_8C za$7&!EE~=q?pW-654FQP&jasH-#@-MaA!UBqJ1JC$5-H63Yv#^zFOaTpV~JfKnaCH zUW5c?$NU21b2ENFXw>M$ol)hzoSwN^A!FTOoEIhKnI!KBGe@6Vi_8 zg=h=?6B+=z$)kZ9{{a74-y<*ETjkm5>E+S8`?znouDWix2(Ed~BTlWe#<|AX0~8MT zonM?(XP9G$T@EZJlJ%nnVO?&$XMJJqZM$Tj?u-B*=U#8I&+Pl%f7PE57!(YKJ^;sT z4EixpwuWKfU}JGxp?l%Nv)KWJ%i}l`PL315DeG7AjodR8> zjp<}Q23Lh0<^l<{66)XNZN$0w*Vx}NiD(q6AJT$IL#mKzaMDwerN}-=GeVE}6D@sD3Cv@+oo$as@n-8xc1{g3$J0XP|pv zwLce>aE<=K{_Q>{5WO0_8ZXIL;=AAz`Ahv8-!@;F@2U4EXlPfuraI3%N*o6JN&5x6 z-hS5pm+gf0g{91L5LBZI%QtYlXIuZUq=8q@WSIwCE~<5dWr1b3g>J!E?pSVE58Eg( zSxvJiIo3G;b*bE)p55M8-VxqGp7CzA>p#~Yu1MEfXMyt)RAyUTCqQN2$9>TK+)eYm z0ItY?zOH_}{{T$A%6t^xa<9&F!}Et{g6FE6<(lW1XaB{fusku>be;j<@niib?Oe@d zm=(>^TvoTJ2&!mhhN4XVTJ~LbOJNe;P>38ZM>v4uM!v_Nz)NVKn-r8bC*o}_f z?hNmG-$Wq3ivs*Wtsfl_27e7jBbFitBjlkwp;e*A;N0MIAguKYeGIk)uJ{-Gvb^`) zzrp=|+NpLPf@cZoN%ubRsr>5$`oOKA5}d)aP@mAJu#@o*2``Ca(iBP`S~z1Na}v1X zPC#e*lskyGowpMFDR($I>^>|{*kB-&y=SvIM>uG%6I`9?yu*BSxFY;b_=xcJ;o%X6 zh$!Jh(SYcgG0K>4F%>bFqPs-Di^_=F2zBh*$T1P+;ZcHA{wdx7-dk=X$HRWk_Oi!w zsyOl7a&8j$4ad(O#CEVAu-38SS@XhqQr7e3D(?&LX72zm)w>(i;>DgNo?OpU_eS??(6+DjTD%#)^}flz9p0^;W9~1m z9j>Xalg@RH-S!=}LvYfn%_#Fo;AL(%?lm0MAJtvce$Y77zo^M-f%>ibmFA}QnC^jo z6DU+&dbNI-!DP4(e4mi%zs@3awAl+&+bfpdwphnKrw5o{r@fPW;r{%<3}7h5AxEry5(>~&>E>`_hwWI1*73R~wpL%{d z@~y7s$PZj?MBVASQ*~eKYU{@|_HH`cG@*HHOG`_S)+uc@;;-#jB#%0lN?%BcvP9WF z=?UPge{26CCWsHV4QNY(s`h@{IyfgDiwnie#VguZND4c)OOMDeDJ#_9G&gnG#t8E_ zYnh|Q<@0p$xqS#Y4SGX9#y-S-!~ZfzMJZu=xJro#f5to#zeV2l@XB&V;u|ToA@YjfoOPiKC>^ zLu1@A`JijZC9X*7o@_}zlsY&4zl@iem035tV7p?vMR!}@?QYJT+|HaKISX>0&PwJEx{>r2mkzlpKWU@sQpORsO&}J2fZy@_`+Bm`?^jE~G(9BRJ z0*CQI74;akA*A!)@f6!n>DS12N_~nenqwX>|xq>y19cTAJH7OV7*W6|;flA=OSdj5KaC_jYsvE(F^<(({YIAL2}?&esR- zgoOcSpwjc%QRb+0_4WSnzX_?4!%(jf69Z$sGLOak)i=lA!~X^D;3~LhYrL<6O7wQZ z4boGhia;mFQ^%3_;NN4qKqa;pTS%fMHWZHddS8^BfGs;YgmpA}F4?7>`1vLnF;G3`6+uc|0UmI|QCZjiCt|GSth`v?c zectt6n%Cl%J5nuAj2Z*Qh%pf?1$LFa9(+j!mJ~DFG)B8iv0i>ifmc3KVe}h2uR~VD z5?zCAN88lK+dn_mG**jhgg6liySdh;*hF%pYpe#c;`044&hGIhr^=}y@V>0HS~ z@u0SgZNtRd#SBT2oS_!0isUaP{PwCgV@qKRvh_quY}1|k-)hf(-&nKlhoBBq|DtYZ z?ckr4KeKA9zN@|lKX-nneKCIdSbe6})3g*=EsI(fH{AF!rCRp2@0Z^`-~IaA_kVxx zuj}2orlqW%Cf(X0mXs>y0r7pVu36XH*b3*^OuYxVlo#|i!&Xy<^_f%b8R=c>wmXXK zbfBWlv31&s&flEdV6zlqGugko26(-2FV*>{27iNDGZ~cHClSxUXZQ+BArE18<$a1c zAYw(Cgxtt=f_vOwInOy+JY+au*h@4&YH!@%Ns^>KNgq$JAi|I)f-Y|40?-Yb25`j3pKUAlHd zbu(wb>iRM#KJRJXKY0;(?7XVH8Tm5{VvDAi&F^)(&z3%z-u51Q%1|Xag_L||?u%S< z{-V59xhL`}3o5&JbdN18Es~Y&DSup^T(-B=T3TNETTx2>rEVj;rgi-*dv4d4d54Ri zl~K#)7uR-wl)o=`c(+itEL)SEn|&mUlJO?Be`;ImsMPt%8xnI87RB$4I}_6)`Xlmc zID@ZbZx8Fw+{?f*zQQhkC~ZD@0e%iT3}FsD40Hs~hgyPTfX{O@UJr7hB5e)+x4NdIkUm7m{jQT#l`ry|w zUl&)O{vPuK|1+k3V$;y(oy{*=mrKscW-8FiO65Q5N7~8SNcA2iN?D*7D8DU#qncwl zZxPva7LFy`yup~SQv)qRs>{(kfVe*vGFK!Tnf9CRBD{y5nJD0%n`?1e*F#@>((bc0 z+OvSL@Al3O+zI_28Xa)>!hO4dygu3U!E5qLJgePDT}NGcuD=|)j=L@r=zhxGM9)?4 zU2lqeuPwK8kMWQ(20Hn0%L-=zk`9ubO_r;sY{T!`b?}qRv<}Trl}bTWhO0knzw56V zs$qk>OT|+zQtnkx)2Eu7Y#!(i3LIBmN#4ueYWG9OOUQ_rY(8fxvC~}}JqLYwUzH=R zGt8jU#lj}KT9a#fU>ymbzI*P0{yo8T|6A8dV5RdNGW$8lJ-0P@8%XUmNQ1iOZ}C(^ zpK!!I${XP;^ynRXY^SWp!SQYbADrIsRl7&^N-;n=QQcMd2pqkQz*I@mRqM)i$2CPN zmHdcIEt{(-R-MvV^_iwzLqIiA_DB-f-dEgLoFvJH&uOM`zPvJw|^{+p2Nu zvDWb}yl+=}EZ#4eP~P_jJ!Y%DgqzmO?h61_EHW6JMonHfX6oaj0yo02&@ZC2{%)S9&8 z>D@EayT;~K=2hh^=(eb9_b$gX?x!Ei)MtO{cCqWptnul*)GH}BQX|rJ8E>;{vMyxg zrg>7LQibVTvbJ|Uk@GH(-FhEq^a>~85yZv`g9ZLsJmV65}v*! zX>^=5>b+=Uj3gmEg`L(d1D!Q3>t#ksDlX|_;-#c6NoNutCKsi*W+>A)rano!72gs~ z6@H6&AnX=>H9A5RC%C{GO81lJ5f205WCvk0ZXue9xD-GIx(3qVK2-RZ`-l6h{aXUc zKw03tca`(0^@tg7{$ZxrMmRsak?u|Q!IoK_6OD~}qVbz~z2lkZiT_1VgO~|Udv*}* z*ZKVZiI7Tm7kL%C4O}84@OaEUM63UdXN~)YN9oi1C;3S3IQu5E6}Bnk4TBB)4TT1! zZj=_G396{-KAMRz!@i;YtUF|gHC-{CFv@iWkU&-;(@3%MajGY}2}ZZ!p6-r%zQWgW zNK9xeZ{H#7qWq-1q&lbpp^kQia;oeeoF5QgBoitRXs3dE6{Q`moGHx~ziIAlYHRr^ z9@5cWx?O^9lY=hoNWH0UTRp4sXj5a;jmA$6j~c%fV&xbuM<-Q(QY6T7J07+_5-)0}O6SV!F8c=W1)#$tUFxqR#9Gpa*K`BidBk*W+M(C(N;H&m`1Z;t5|8>u6&_F)+xIHD_R_|Z_DWQ4D4AgMc4dlHLG4KXD zGoE*+UlF?-UgNczab-a&>V`=g_k3@ zd&|5=JxeMjjKi!!tPQP0zQMR~nYbzxKa}EM;!!#KIi@<4&OA2?h#^czv|VZc2lOJt zT#wv6e5AmDAO$fVNkF~`Ee$3I6uvXQDS@2`8k&#kjkyV%`#11<4)#IVQr$1q`VugwAGZWQgm{DefVPtv%Zg$*u~)F)gjpF0jLnQ@rV!Fc zkb*6Oa{eJs|F9&+1x7IJ8s{8uKEIaVKYUe0g&>QQ#y-gXgFj2~cSLK{y!cS!+XP$8 zbx}d2mOq_)kJml2K6*vmZ*ka|aM8zzqk;uMA8Y3^!`DP)McfsX@&AM^cPqCO_Tpl` zBN7!I6?-H8Y~qcCVX^cmPvlj|S4$D{qvypwh#MULIPP*xQq)f2QsIcGirB39y7=mZ zFNyYqesRO1SBYMWc0_-VDT~Wa(xqWC7Nkx}8W|rO-5#+(uvE|*Q4sYq+9BdabZ`-@ zef0B`KS(=?qlf{*GW<(SE-D4|v_C`P!BL?3n;trY*oh&$b$`W5>2is>{g@=LW4UCl_@7pJi?n5ROP^*$ zbC1?H?JXTUrFpVc*$>J6Rzu_L`j@psYen^wn>Mv5TRK~Yx7=u4SU0w|vX0!?r@4Pi zdCTz@XUi|m{ToW^I-oYDH{58v*xX;dNIFh7Tk;mvqXQahYLT_i>zbRY#M%x+2STE2 zThp4+@}LRVRMM<#{jGhNWL&$mEla$qy}o^ayIOKZ)}TmHeN|U$BlO>NsoG)cWaz&i zsgLSPjV-1vogYo(3@j~6HC7>2q^rxdZe4|Oa%X*Ktx0RB(+$-9qE7@@eYS<D=7P}Wn!C=kg*7SKP1y=0H%j^w9= z9}2(9_i^GmZq9cumH##Ti)en_%%ow-w-Q=oVnpQdMcgRPeC|TQpOK%0CnNiWf8e1x zRwmGEs9mVtX|riX)DL7gDF8ghHNbz{MyjRM(=IY_EG7F1+Z(21;OL{NgD5{KGFpH! z0NBlG>?>h|=np8PDaWYO>15{JuzL0?pmx4s&1U{f7t=hn{q#~Mk26)^1%k{D9**s% z$%qSa^RRZDiMWi?mntD|Bz(hQQF}rm|6$)9U%VgVPx0Y^YVaSNSR=g0d`AL1LnDz& z)IG?e$wt}(!@M7zw{7{>MYf4fg_{Z-v@4$8?sbksR*KbNd+hk^eCxjE+W}t0OdrBy zcU8IWx=y(N@Er|Bpfj*?j2F2fFx;JLtLgl$lW6&58|E75T?h);wXoYg=mK)OW1_Rk zC2Hrdnm<8>eVC7>$ zPj(vd6!i=fhOZ#iP&0vZolA3*SCUAiX3}d)EaM{Qa)c1{Ssz7YVMy?bt7g+UxxCTg z@j{y@DQbf7Mfh|6KJI;>d~IXT=amRb!(BiHJ{mD2A|m2oggtVHh#l>XnH`6V`xcE5 z;Ts~Si9SRRh~vfYjeQaoF8mttcZ4>A zC(Ms7jSow@l@yowSL|uwYo42Vkvg2bo|Hr`qQ0f^XsMKB(hhtf=69gPO$?>M7F_5< zxbHbWfNJWXS#4fn{nsAn8s)hF{JC%LT1S!fy~(XVsjJf0KwtZ(B>_C)hsirR$~3GX$q2HVfYi zy%Em@0)B7qWp+4c9sKPJ!3u$me~;(oBy-H1m4v zlEYKYDPxn@CrF}`BJOcp*oEvW);uHC`g`9r8 z&+qBsJ>k>&YXk9+k@yR;IrJc4gIQ<0YX_uN^>?G;ozf2UbdyKvT4C>QyJNrRnCCd} z9PNqnUUj9~D=ddiafVX;EklJl&T-Lm+TR?A2sC+K*cW$(>q|8bl}uHs-k`mt?_u~; zrvQE=UVg7b(D6cgS#FX~mlNb&WTuW%>2Wy+W>DjGdyR=^llh;{GGl>$hIXy$kbDtL zAWbR-e3u5z0reYsWk-YLlcZYGL%Lak(NyS;=os2psyhn5Oaz%uv(-U8()`>y%I30c z?sV!U>fe>8VKSptAJlCzl>paO2ebjU>Al{nn+9b32=g2Z%UWpp*?F_`kvZ0Cw#7SV zx?g)I_%HerJvt}RvCq2DyuiHA`qm-zH2Urb8iJLz?E};(HcajH*VW z5#NI|0v_Kx|1Lxh?igiJ*m*8Va7gecuby?EfuQ%Ny`bG;Ok(xnMhIG9cM;FGa(c4s z!sdl3Ss0+8R&w^T55aD3Mp!W`iOu6Y;qHJcr9j~2X*gK+E1=z8WvmO!FW^QT5!J+8h}{vhEs8Jv79KA+!Iuh0NWe&D8vh%3!Dx9 zhvb}efPz1i#3k;-|A)tsE>jNEra_Xffl*IiL_ZHa;{TX0SjRZk+_&5&_F3jJ zS_b7Zu@L_Vw}QYTb%A`9x#TqRCqg3b1m+C-F^Y}~M|DAu0oLLjWMPOCm;f8yAHFBS zk?6Su7dZ+XK|{#nhy{2FHX5hKPa)+|Pcwe9nmA5wCT|p{otZ};NK2>nr~OJhOsyp6 z6L;f^fiM0LJqJh|&+#haYSIqkAVNHT0d_w6FVtV?)7UTgm&8KyI?4lTBXttRNjgGm zA@(QSz+WUhCVqw7r+v7GnD^)zXeD|wZVE|D9YUW(dra<4NWmNk-SzE;$&1ZJb+2-t zcmLxa>b~y0WV3ZHG4OPQw0P|Wwg;X;!R}SIVoE_q5f< zbDbwmiwsWfEmc5P0y?WwNg{Ao=E(V~C7Ms#c%4qeR{bLv$oI*^6?7#=#Z*mGUQ(nf zW+~PwU#gIr4y{(d)G$V0u6wDysbK^0L$Ar#A2*ye9y1MuyeX+^HcUSH0JUr%&!Y5x|o3t#%)Dl-t%=$Jk=* z9J|zB>x_5dfNuAjZN4pFt+aJGI^FfqqeuIDf{J6Be{-M!VZj7Q&lo>gEug%KUVq_D+Om{;Nq;5oNZgyaKe12psn_fr~SS^&c|`{8bC%4MR_VK^p}s)5pwF?5*4sevW`ASkDu19)-0tR6u4AXEIqX zwuN(xJtf^Kb||26~~OD zeIpLT2|*pc4U58!#~&uT$?4Q>l&z#R!de^)_ZW8*KZo#@aGg+zzlD{eThNEG8z9Z( z5mdk{j>JrWr=V9uZ+E*X_~0)}`x+hGUk`u9rTEzuq^=JIOQMWw&hyx7}82 zjm_(b_89yx5F>%ha~HJ<=|-GDlF>^s`PeyFC8ij?2f+@`3G@lp0AKA|pcb@{aXyW^ zm;0-`%)^JuXLv}DprJzO>sT?QnLordV-`Z*QAer7??5#K8~pM9-M%;8JYQsBW6&Kq>65uY=xWY}%$f?*Lem>lwdo&Y zcl|4Eq~2>Jn5S5%wv9kBooaPgv>7^*UvfEKu>*6Jr|KdX8vu5|Qi``(Ogx!Llwn@D)*{3s;e|64N;vBs)`EL5mi?; zSu2MP$VTHzP-9K+Tx9xYL_M3^aWc$3bRv2QWQvW4{q+?>KFLoSK)Onh;Jd<3`YkRKUr8JZbim#; z3w17S3L`b_3u`RDFgh>ke5yWebvh&cWa{wbK8e|hk;&<)&1t7H3Elb@_=_KvbuWKZ z7E`vfbVeznEUbJ^k2^ij_oVgsS{5jy^%&aob}v$&o)v8s)qPOC80CSIg5u|eYYS%; z%_vSQIaxZY{9JkOGEwQr5@AW8XmZhuqKFb@>EN=1CFY{NMaK$f7A(wP-~D1q?;iFZ z@5{E7n2H`3Z0v^0x}P4OmX>laaa?>iAO#JM&yVv*)kl=^uX4t-PBXVKr$UNY29r-m zQ;R9#kQs81)=t+mKZV_6<}$Vbb(Bn>LLbRM10!`DWXt_Yqfw?2M?+Tke9S`>4*3a@ z1_^*YQHv3CAe(Ecui9JdogLVY8iiYgb7Rh;3COMf-LCbvDo75RVLAki$n4H+E7oCg z+_GP?wt?bv8!#isch;Cw!4EdfY%v)@clo+=iFuP{ly#`JzooA8v}vVr2)M|UP|qGW z%1nAQ(Ix{IRl4nf?TK}~d6S7}Dm3*otuw~ypJaotnp!TjKB-!;!NPo6gt)NNgZ6A*c5HEsy03EUG}QWHr> z(vk*}M^N5UQ9yk~(SD^4rzJ4%GHb)`ur9G|Of+2%*?og>|6yKY*1+8GG;RZa2jpZZ zhy#fp!f4_y$P8}8lw;pwZ)0;n^9!OC1R8NXSQeUzPKOyEoxG7)hAT(e147Swdv8ld zr_k6>e_pp$-^2LIgy}qNbnDh>aB7jNTq#%lqWrAvHPl^`{zkHs&;=+d}PgJ6Ii;C4R|-j=3Gt z(vWNlu%fCoi?ru;B%{DwV~ukxcIA7s@!*<_1>zrE`{7uXSc2Rzvr z)KOF=a!3dVgz85KCo%;c2RXgHacl8|3AH2_-N!xxg#W)F*YsQ1Ohz7U9w;q;fgDm2 zNk_a*#?U>?v8;Wp7*=waoN<@_mi7d0{c47v@qkgw_?!7N%+HGCRB!{_FI)>p2-AX2 z_J5p6=pus=H(*DyC;DB?pcrvS$i^EY4cNGCDW4MBzJ|gVs?fnb3%3p-1$R_LnZ%~D2lDc{F0l++TuS;|0w_7 zV@1!?aAvT2>3gl|y|eeI-Zy(i^!!+Mr6j&sTsW$5Y|-Q5s^U|{cS=;HHKiG)k4ntN z#YLkFw&fILkIvF(WTmf3bthLP%}?5x^iL8p|r@#+M#y=eOs=1q|Q@?DcOo6GBS86&r1vC zBa~ZJ2bI5cXt!x~=WX*n;O4!yd^Gp#ykHz>xT#;O7Z^guNFeHsuza<4wTIi4Kp}qb1Twnw zs=dJ`wYfp*X|)amC3kP@C0mW-wC8>BE2<0TEt-Sc7Rn6t^Cf$BxM;4|&aTcx=O*Vk z*EY{$-&t>h`=+znG1|_wy|=boyV&N~9#}V7k}TIPrB;uXY<~rM_@B0Iwu!b*NbBfs zU+6PavXM_^bH9d z^5}fHUwk;0f4*`5}A=jYvm60;iKK7zJsI z62t&h4k&&PqOK#KAzDBKcsfKy`cb1WcQ8G$47{4~Cvhfe6D0sy<^d9pXvQxCt?y!N z1_lFKx9f-if{9{d#^F;S!@ED6^VJwUYF_BD|AXhSE61_UT5Fb>yI9}ZXSlw&CORiO zPTO79Tb4G)OQ2V*2d=#p!9^X! zEFdT-_nBIb0y5CI@q%1BH;3~NJBGc0)g$a*hLf%V(pxW>a$Qt`hLCK7RE2PH&Z zNIgOJ68{3NTnOkG#|SNi)5JdHb5stK#m$UZDdI=tVoHH7`Y2ixjf&YF-92iUa9o60 zP#FF)vNozPraUG!Mgq!xPE@Dpc+|J(J25X}B#^1FIQ~t-)}%)%+%!hozEo;zWXhJL zf<#f`gG5RaK8XqJ(S6C=QkbdPX?rqqvIb;Y(|%8Vn_^1doz#(VDxMR6H!dUYAb4d- z$=g%*rTCI8@uBFd$W#1!_HgBndBC^RyrRz$AA-hs;j57#(vPY4}3nqS_^@Y{TWiuH$o=fVNka8_wNl>1?Tu* zx|cZEwp-={#I@A6Ob$qF4Pb!RB-@-B)Ng-Pj9f6`TfXN_dY({pqSG$(*q zzE$_DzMsCQ{EB=)|k=~UGWxHesd4w9Ne`$>9 z6n9d9zrEEIW10+9-@C>OhG*cCo1)J$ZZ=P}|Kl=vM!*EVz<<)0=ez0?_`g9?dVk+C z-y7do|FK{cBpud;I6)K4`*yk4xe8oXS0T_B&o!_CKxu`p-`Bi0=E2Nr^o>G&ntMaU>R&`I6q)JgN zmfnygO5!9r5*Fw!OC@JJI;8)|w#zdW=jHQc3h8rcxpa$kjeNM0qO6hckbi_pIa4tQ zlp<4LniA0FXf@h;9Y^;{^G)N@cGvGPWSbb46^=hVDc}kTc{98XuI>)J9buQ*L^gqy z4q5eqPO&B1vBN#p*Bsa%`UP~wJ43sIr-I!>yMwKM0;E%&@XrcN34MZ1bvTBIS&42& zeM4=B{Pxe-Be*ZPSm>&Q#14{;JdU=Ak;E8FYp2jC1IRy#Q}7G1hauT%Hg*>F9pNSw z$!rWe#x`;k&>5zm<|{V^;8{F+p5AV!lnN9kL^gw-(qQ7wIo52s5-BD zrCtnm?O6p!{!DgR7AZqZH%o4cscp+!^V@RUkg>smhqxlK5ss-R*L#Q4r2P^-A zl*TilAY8AUt4dLCQ8AT66orZug;udvy-@d7KhR(RXW%?D#*$(4Irwh6XPmI2 zZ382Qd7rVGewezE+@JIWG|>C;Vo1%*CvGCDNy(Izko{`Jm0|B--eaQ)V=HdQz9uXE9N%AFM^Ji(&hLw11n$Wba^qU^lZ9AcNyK z9$j!u(3Q{Ok~!IsD*GBXWz7*q!Z=}xaJg_^!nqRo-X5u*hO+>xw8 zRwJut*bYV;vxZ&5{l@v1eUbGWOfHsFEW{e(7}5nIyl~jR(eseWh@FUBR6OP(mV|$a zAA)~_J&rzr+=uuTaXvH+G9$3T(!gTB#XmLB?4x^R&OVN{wr$pJ3oLGn zySux)EMA-frAS-qnkH@B-969!{15MkcjjT4{lvg9SJI!{*Lfa?DqH(WSEn1LC8?(? z_W(B6Cy+xiNgJeM=^}7n30DN=P`O4rNxDP2SQeDSR3i0dO@H8ji`AR~-WIo}K>tYp zT}RYy);`lrP{%0e$wTrp%2!H@5~5vVoMEZ8Vu0Uno#O|%AEw*-0ypeh$3VN>ns1$D z9SEGdQ1=4gnBaWK4Cot(GdMah&94EZ-8kPVUCTtW9lUq#?*Tj|<<=Jqb2#Ggwa< zGpRaKHE9F+8aQJVk$cdg;9UwC_6xMrGiZY;sbn+hKj1H|!lzvDV%hPxSm)~tR}7`WaExwg6PGdjW`Fh7pKAjr8ky~ z`;PgDwqx3Ghd|OriM@_~gj@$-8p8OJJtB9ftHF8BHQgHuyoL;beun_?H)LQ49U%Ed zfOPiSBXflu9|5D+<@n*!cwYEPkV7yxObHc2mI6!Q%Fv1+4V*HTh3*3X@UqZUz(q+5 z{^{T7Gl9NRxi2CRAD9Q)QS(4@dW;_ts0ccs5accJQ@j_kAMpY9JhTw7Qo_NRXCCNM zWW!&h+p&1SgW3qr1=x5J{uF)|K}%Rp_<;ko12hKx8od#76Dz}iA_Xar!M!+xoJ9JY zXvPl&J;4}U5onyJV_m@0mW^pgjY1AWl)@*&*8|o|95RTQi`atbM63W-uQOnC>H==q z?U=vOJyA)BR45^o8M24Ag+xINL=Ss|{DeA-YzA%t73@4z2cDkqgf4}42FLjWo^$SQ z?&aG%PW{@$x z?0+7Z9vmJrgpP%5!QG+T(4pX3whmd3;2>#89-IS-3hoVT4GaRdMr!anbUJb$su2(( z1|p9m=E4WVs-Z0Kq|giccPK9q=lj!J;SB>xCtk1#x)QMgxfIz8aRzWs=7CRd9I*pyOe}10Zrkc7fX{Ih5I@ng8b#p^+$j{(fUAg z@Si{;xc4s%wuJV8fAd9+|=)~X@4 zWfssUGmz9j#gp1Kja%xcHe74gwp|oh1;+&k1T%$$x-N_Nh>IjVsa>4WIjG}G8@$!n zf^41Es%bZgYQ!y)`x3eMn)s6Bi4-HdDnlt4;C@*ozYEBQ8>E}1yQK&@U&Yo*jF&+k zc8#gjG}=^UEH>xc)`NF&zpK^#kEh6M_7?amLGrOU7zv4o-$2Do}eIyn+i9C`D z3xlvR!1EN~g?OuZy*Om%YQ{x|oxufL{bS*!{EoO42?_BfaqRe4@qfol<7USdga7}H zO^6NxwmBy9V&wd&Qa(DaJn2B{^wgr1lSzvBg|Xvf2E>SCYGN^Q%VHT$XFLrzlDnL>1Z2KO5XR%`u>-IN z0HJLhu+?^AyUT52dLKe!E)a8 z$Z$)$Ky^}iO!ZOKuFO#$SB#TmWlU*>*xq?pv_-`3?C3n&^|UJm^qdpBmUqq)jqAJ( z_VR}$DlvoaF%y7SK;_u&{O-Uz#E!KtuBQN;gAV|^^Od1_5D4h3H9%uv zZ1`{_8}k+m$H{Ql2q}QK^n|D(&!fF$@R;X-J#PhTUl@T5d%@5BB|A!X|4BW-M#oRyHBbi9%a%Lsq?$@xw!*4_#jON6|N3V-|7eV2r zvuCji*}K>dRuYR1@)2L@Dw>&kf$|^N?!E_oc@HgzK8^l2J&hhqLx3%LG~*YbPz<1p z=w2Gw1ry6~-!ML41=)q^!fNrKK`T&5$j6Z|XmCxFAs)gvf$h@_I1TzH(BH@KHo5z| z1^_2Rm*;9=L2#j;hsXQf0S{Uw<#KG#*% zwLv@)@K504tztNMvREc86tuVB7t9jG3D}|@(k8_!)n33N!l?JC<^kJptlX^VruwPe zrevuWs}8GHsnax<)w|WRfnjNz@rP-(WtGiqKk4}4_~88Fc@+2;+|ex%1n}0PVF*Y~ zK=0Rr-M$}UfDVwsmtjq<_zBX zC_%I;Ix=QdtUu-(UlGNLCdXvN6c&x)SUPmY=&^^89=#vV5&@lxWr#7f{o zfTr9_UXdzK*Jre6Jj~dik(JSz_A0q7enzY$=4Ol_W`ErD#NyP{%$Tg68TjXyea$t&;RWWKzwGy& zXiqliV(xSEUA^sD<{HBf!vVuhU<4HEDm33!@0C3i5E&d?_3nU^h*8R4itY***t?gi z%wT8b(RkGD$|H)e@~N^xlK!2q1S{H3w{B_qvl-JOZfoxtDPoB_JB$LQz#=FYgmqAb zJ491D(V}cYp!IaimgbvH|1`=QSj`LDMz>eA4s99T>}*`u(4)Rb-I_Xa{p5yq_226@ z)M4rt*T&awt!=6;tiv@-Z`#=Wu!Ym!yCY0ES2$8MylbvxrSySxn9L0l*)J7oGK@sh zB?jh|{hh-k?-XOTrwoy%deeT8JG*K2n>PTCfGben zutB_?kOtf#GYH{$6@CCIpE8enj8;uYGJ4Pe;q$*#&&XC!1Kr+-Qdrme{Un2daE!Jh>~3wCw4beo?Y1dXIo8PqIV&Xk;6 z*&i|?Y2A`{CGnEpB|S>vr)^0;ku94gbh1Bnws2~qy{`5HQ@nq zB*jam(@%nq@_hO?>PPYeqMq;=SSy!-Ovg?<5r@ZcQR5H}*dkbekUo0aSF@62~D zb+GLBEN_fa`l*@)fTr?7RsgQ*yQCp`uKJ`_sGFwWqs!NfQDWr_L0>dgx=K0|+@)Q@ z6&*hXtd5=nOnZG>&vvfh0cfXP67KHo-kBpj(g7E379|NGqD|uY(gl)Lonr+%ThXnf zT7EPwX)bR6Q?$2>CBD|xy=z<-zUxk>x>MBou`|5uV5ddcx8qv-j<$QP{aYqA|7eD` zjTRmjLuJKa+w)GkR2q_aC8uRXdAYPunl0(m87Ukf(E5vAdcjYM6S#=q(SoY9j zbzPd>8kDX@e*oCR0;U|0F14E$8f3aty5rh#aGqafw%YDGc#aKrwlmzl-Sf-)%2(&l z4ek!E3m|+sZYdy0v^thJjyM>uTi%LbHY^(A^_Z&Bbz@L`Y@!i1&- z7YCO676nRunyx>Sg9##OH*n8* zTfz@>ud|;pRm}CA{=99RD`7?Sa2lJ^NV*N!R5nUE$h{xI?m~+Yi=c7A{l5P^A6-kG z5?8>x7PuL6f|$S--z|5AgX!oE-el)mx0tsWGjvMj6FE%bmLHYv2SkOQ(i`Fuu}?fp zj1u<)Jr%y{fqFI|H!(CW&0NhF#d}Fym!iYbHnP2;V_26#)}f39OhJZhndEEN?XH93 zFXA*wZ`mjLQkhsh8l2x1bizeLg-lU;*8(X`8Ky2)MJle!c1vGMo=PvvXDPy!@v1`Q zA^8t^3+Tj*R!mmXGy`M4?=G7L9 zWw_;+@ju-_&1`kF<}d9Mz`ePy+o2_Z1pEl)Y2_-An&K#{HF1U%<4{wB1!22uIc45y zxn|9?U9k@boRlWVN$U{9Cbd95RdTTFqU5?VPnTy{VNikgK1KUVeMI$5m8J=3N;S9C z<3SQ^mwdW3S!@#SXg}AsM6gaI6-#7a!Tsrs{GxQa7~i!@yi4{}u?4Vow35S}UxcfK z{|O64ja_l_d&&;wJ>?(De#&#oO2uF~RlY)jQVCUU>P^~xpp7$Lenwm)47I;*t8On9 z?UlY$j8iUGpcPK}OZjr;d+kxMH(y|Bv=mv0<{PFU@IT%*{>}{l&J6$G$qW~FjAeI^4iC=z|AE@Z67+mphMd`PW?`OYHsW5yVP?q}ou9hqb$LXighK@7Z3Ei3cNBKwB;~SxiWLfnf~=aVI;T3LY0;OM zFy__fE!HN7){XET@m=wb_cVfpMzejebA%_w-v-im*Ml*k?;%Gh2eKXd5!fo*pzXj| z)eUwZmI9v%-vXb3$U^Zk!*COb(Fa z(-~NLJ%vU(grA080p2_~*x`f{(g>1@c!!iriJ<;NrO*b@vl!3m%~TqtiyTQ!qrISd zz>{YkkwzX*B{D9s_VCb=V~Ri0+cSezxyT9CCab85!A^ik>OGj?ag^5EU;@(1O= z%$Mb(@{i`#=EoMM6#0wFi&huei;8+(?;TS-tM|iRe}bovuHvtKhL?m7>^*42KuG@$ zeK!`r?nUgGS`<@+Eb7}MtnhPpVSaM=>jletd@CB-Got6!qQW9p(d42JMcsRT?s=r= zA4Ma2EbF1_F|Ehpf~@>$-J){$WRJ{RoBgia((cm>MBM=sAXk(9E^~4E>hyvvNzRGf z@Z4iL2Xk6;XgQUc+taV6>C+tP-7`05oyZxJH!A;Bw@o?!WwWyDGAS9fG)wZM#5Zv+ z{*ed^FE#vX#IGnzbaw0>aI$kI_IpfbbQr%rDk-Wiq9(i{{2Aa8or*Ms+qi`s4(k+S zEv=Y(k-C*$$LJ3B`9<_vY7F@h@gi|2kxod*3Xx6VTR9nYq$*$^paked=osh&@C^S9 z{t9sg-UButdJ}R#G%)zsf5mslTjK6(mzyH>lQde@MZg^oC{fCR3iJQ6Ie^y|s%%o; zSG`j4RJT-5G%9VgW|nG_93_#7LW1{g-`kdVxJ9|0=YI=Nj2af9rmB24*OxnBLJ_JyuU|5?{p$I^9Z9)gpo z{^|;CFT-vl(Kr@R`pyCBVzYjkL1U;feAW*Fg!OlZP3Amn9U!%-tUqlwTO!C#JM4QM zCH6ViQ)Z!YiUF-J)U5*K#3=1qrBLeaG6JK-cHwN%Kb=*bye_QRE=Egci2HO2MfZhY zJBD@8gq+STUHQNreoz=ESk($^Zf!_t7}IEKoY26jyIC{l*Ndt?mF%B{AH?tPDrDu} z&)+`1|CIB2?x)C)6W>R?U-`b_{m&2omGQn5f5lde|9Y+bW%;)9^;>^mBPvuRYR%my6!przYP|M{F5A!u&JGH7({&s5uMAzLOdH=x0S&uf z{^(FLL=Bk;lfrcfEUE(C7qm93OFtx10A@#F?li(W_&kW029-s7sL#BZ?!ZMA@V2_~p^%;B@F3AI8T;bpcQA zOWrn~nU@~X9<@C7Vf+tZyiJWi8pn?v6@4RWeB_vji11SGJI*}LFOHh~n3o)mj2IUo zh^UK-i@e2rMH|iXhDS!;;yz?V(eJVDv!)R_&`myXpcZ+AaEQbMKkQY6#gJ{@C|D=e z6s6(!rG12Ka&+oMQU+mSyp6Qp;;*$F&!jhA-K6T$;*jhNb=ve-i z)I;%!DP!{&^glWDNQp4NcY0*Gz8P{ZPC@9Pm2F7X7&u_&(6DDbhn?P-=@CEe#Tz(Zuiqj>7{ws`(+Pl?Kdl* zmL8itK6!bHI(bSeC3{*PfbZs3W<_Ql%sG~WNnepPDY-dge*V~k30ZxUW+d$aZ>TxR zU*mw(Hdd1Gcf#5D#Dudk@7W8;Z-~_B@R0*{S_YvL%O9%GYQrL62^R9^V=aUkXpzyI2^qla}QMofq27R-F&AJ3kh|^ z`M7~d67)Q1DusE^yB@fkJvY2!&}Et&RCs4QVk{As({`KtlW&oa;XY!GG%V85jDLf- zI-a!>5Jgh}UGgK?#XPoF=`N^xX@}S_K8jE2_+?t8oucZege%%*E@^M6O6E~nv~Pd| zCsSLZ^BZ=Vu+|N(5#FOtrg?#GxK^kCYMySLY5CjS!{W9!*rz!cJD!*~YQvTDluZ3> z>l0h0X}BgRn#v^G?bX|A5Vb>p(JKG_wyb>3*X>mg8#XsK*2Y(tRJK;)s$W(X{8;n- z#dpRJ+0O^nGk!s;9{p(h(W5fEnqCKM%4@n`d+~esGQ~SY*@Ukm@Dy6|O;qvn=iJ&C z4XO3rs@8qo@_E$P^HuK}PqiIyo7Q}&;cf$>`F6{V=A_1k`eO|fo4zz3Z>BZE>h{;5 zYs>0;we)M3x2D9p>E^2 zgg*$ygr|55zJs(0usbhN2>9NJCD2F6Z-i$Q0{JjD4D|u_7tL? zVjmMvkw+8SkrSbNAY#x|f&jC|2Z$XJjoU&DBOb%nAv_=ru^M#42B423M}}TFFWI)a zF9iR9WCSC9Jnsa5LZ~rR6zUP2;@jxlZ#`>`a19IOK;HYuI(D0e8mmkm^B40)kVijh zx@aBg(1C10HRyX(IdZ(WLly9)h|!Q?-W{$wceU?y5E5$kzwz&e($H<#{&+L~HR$BM zKAB9zcAASr#GI2S8J%>o>I7rGI%99K^0L!L;r(h?ab8QsDbvPS}%3NiY4^h?lb zB(tut3nOmue?&F%PV#odTu6z{Se-E@{b0(er2EOs)AF)L=1B|p6}>L(o|m0HD;t}K zD%jKgZccNmIq6>7uv}hYL@!uzN%6_vcY97N%ISThWZOW-fSf)by1z(oOV}Krn7lbF zroi2EdvQsh@?NzCuevF6Z*)J<`;U_75^CS;#UpyddX*MUF6_uVp1mn8B}tx8ndC`Z zlJh3Ny!(y3RoUt3+f$WkdozY*97x?3pBTYmZlJ$nc1BJ~IGyw)Ntmoi{K6l=#c(Es zC&kQ3;3aH}dm7gVoE--vAB0OevsiB!S1HelzsQoX8s7D&chS6<0sQS?cQ`WYK2YHt`$tHlj>Ppy92_^6f0{owMiK3c>K-wd z+rTbn_NHDVL1`ygKCXs`BNlR77l+ReqqZVnf!6wukP0#z z+A}!UTjM1KmIf9F$^xqc=K{XK1kjLTbo9=u!yHf56GIT?U=3A^v6X zC)mm4kJJFEKP~}SBU5ZRG+=lH1{}&Pg^2?(`4?;-DM`#%07jhwJ zmij;<=&bu=NLlk4sSQXsszv7N`AN4td_x;bE9Glph zXD_#xTO|6MT7u4@TVUXrHk-#d0}ea1C~**(X_=jY|y0pbMF*{;HTIJ0?w+ z3zaW5jXIL{r!1#)i)fJKqAE_`Pdh%JZaKJMXoRX(H6)>z#Eqzsi61{*hBzT2t0o*HX~(PeW;qt7dFdoX{(l zbr!abYg9GPZmR&)3{g`-Ev-uMlU}{5)?L@97E;Zqdi!f#{m$m2?H>f>Hc8{ex;b?l zn~5FYK+07m2=92;HA0oAzpnpJ`$1i-onujZ*Fmqt-UNkine&AAN#MEni;ZG#+Tu{l06qe*|m~Vl3o>m+9g-mG0pn?N0EP+WT1r&h-!&`ZDS* zY)t5Ba4vK*vJL$ZeG&0AbkpA)9F2&>=+KdfeL=KujSmsH=HKGkZ=Vcu`EiZ{k0rPQ zb`65^-vr-)EbmW{N$Kv$Hul!PHP3U01(5z5&fS2$RBmdt9+DxZmM0&5#oNRuY1CaLktv$9*VLS>4s z$sBFFWbV>ER{c=L8^<}OI%~{l)Vt)d+K2WP-UN^;edKUC{_yk(WI|Js*~r3Bv3s=h zo_9JFj@THQ=x%i^^q7NWxDla3Tto~8y|#ROFKS;#6?G(WG$5}wknYp(GAbz%xFe_p z!WzHmetcuHCWD);NH0t58Z#CFMeVeZ)<+flG)Pn;?o?z+VZT?e8TlPydlK zvOBWiM)re*rxDQb`+${!bxq=L~>qsj5W;$M!PY( zk;Vo?s&=}vOP;U%M|;lv*s%=c%?Y+-Gt~0RZgU*5JkUhS-z)O<6bsE-Z@z2%4>U%G z>-U;FT=)F9y;*j+`I2pVU@o!@c|UXnG>2cis(h=0%HZgrB)A(^kC{de(wBz8=r-a- z+-O2H^)|yvuO;2X?!`T(>}OVRrg2ZO|6o369HtAypb@8I82lZqN=gGQDngLpNp$gZ z*xl(VbUR~NSRcj!+6KxYikEKUOo>X3-Wu7B<6(3#YS{3|G5myx3RZ6>hEo{sju1uI zcx@az>nU>+w^!WF)XL;f{Lw60Sbg~Ogi}c)W01V%tm~}hoYPzmx18O^N?>o{_K#W= zM^1=~>Ew)~t|s0f(;0)97RF&(A6ghAnt7E4Sf`9s%2)g@oRoZ;`v6z~!nxTL8sP%f zz?sPF7WSA}iEY7ElE}0}w6T;cgpJq|tc|darVV?^XeHf1&qQoQwO}QtJJS*@N`T#i=MS+hG9Ckf(^zq<=*bvmW z)Shk*w|;Oh3*HZ{09VUn-h&|*x{X*zn2Y$&wbvLlavTnK$aUJ*!^qY(>#9vAo7k3M zer8-_IqgITHOPOk_tAG?F8^OX&`?JC&?6A90($ptZz}X7av~B9lL!9xT@EIqmy?KO z7^aW^pzXLl*Ix^BBSylQfn0AYAn=?Iy$j6_j0N3*eGoME1+51wpD~({iy%Rza4u#C z_9lu8QFs^m&%@-X)8Kj+7ufHS`BuYw;PoUYz6y2<(4gN!bO;IJ5M-n0h5eEJx$_X% z;lFZb*e+S-g3e)Oa4M7&EOu%Q99@Fxt(^i`Q<3H^`q@UheUJO4d#c@F&}cU61Qw>} zKYzA2+g1oTGHWeH$4J*LTbh0(NW1n_YqSMggJPWYsH9R#Rvj@sHy0UZE8dCvcC776 zP@=U->O={wV{!*y{7g1eR@fzMKi+C<|IqbCS*iV{>erRhN@yC~@^8n$|RswOf=RFw}~0BhH6R?s1-;p_{NA zcoKXXB*oeqO=R(P@y`~KYqNSIM;T-|qS8rH)SQp!XU(wQN;OS`EorWV0 zyZ_%%kw4u24OqpFS=f%z4uLff^khpd8=cj@49L6CHeaQio((6(^C6dz-4;xo@pOt9ceph|7FXzxQwyp zagJ!;r=UL6?ys~h(%=;w)kiR8r*4YcqHro*YOF5Ruv|Y$%~sHrSbeUY?uYRz?lZ1WmN4y8#SsW#9GF2S`A?m?e`XL+|+H&}bR?S4l<<{#@@;N9$d6zl=r66))1 zbmH9*|2JqUMgUl@Cqpj**JD4j1sB9gQ6s?9NjnT=fiO@U8?yj88G*$Nrtu=6(diKu zhMm|6={ikoNzd%EIfmEEUFd zL-L(CHoq-$0>5X>+h}bhKl~}z6A?=MCr6j}EPH0slc>#+z2p8#xs+a;PESov+!+5Q zp>Gl~@m0)paGJj;_IApd%!ted$!&ZLr<&>DzTr=dEr?DHzsTCj*hPOB=3`dVn(^)E z3|w#8E$-e3D5sIcMjQ{mfRtkbly+Jy=?EqnI3|+Oxrpb1S?-&zub!;HY$yy~8Vovm z7zgPUroqm(z(H6k|=7 zpgjXEzD2=j`}rJ)?fcFN}K>KA--BIDkBeEsa4Z?u{xbg4`AN2vD`2|UQmd0kr6i`8WDU<+mbxkCFLr<0O}dQZk7|i- zjX1?xzz~IHb8c|Qa};4mC~2hqjQdf&6W+zrIl~DVNIUW#ell$?vp(zu6-FF~e**4> zpHOyKKj;@|Bcd11LR^901aJ2q1J9`ILQGU23?2C^wA}a0^Vz2fu>D(I5RkPW9}MBH z(ZlJxu@wJ(%VFbq+acdN$Zm)#SRYs$_zcpz{Q|?hIG4#e$R~rsz%y$ltjenbSM6%g zZ0HCS4T*#v4#mQ(s6n{l*fK<5%?m(|h_a6-qMa?BUO8QffHZdaVN zOjB*f*k&1r%Ad5iHpVt@7d#YA7joLqwV{=wfC$pyQJ$B^Z|b0GtBz=%XI*6-W;tvcVEkd|3CJ%IdZ^)!$z~V$ z4u>xKpE}-}SVok!)O{x43_+m>A=#llfi2!z#}1QUHAgCylq+wTM!0%A>-2A>(}Xa= z)sEP%ev;9Wsp2q@c&h7qB|hBgY&W!gY{d)zlSZn?tLMwFb`25!DeUO%E#B0%Rb&!O zYj112)Ed@ut1-4_z>mw{|NWKJI=gGRI6<(v{#xbr@7$lQ)hBC3wJU1=`xRentuJVr z)a0s{)TI9M{W@RIYGyWd)+l~(D)v>J`0>7SR`rmYhc%pun$As$^2OgsyO%L8r6qgcQ7`yp%L=PJ`pq7QtD^IKfsu2C;dfV9=0Q_ zoPLUWhBl9}l)4@N4!scLCaG9Ec}z}nSRwT`sf^?&DM^V7P7hE57 zA;N%Ij4dWf!DiM;dV%kOJCDCY{uw6b-sNR+j)$$K{i6P&rjh}R1uMi&rFO8>xmHGh z;y|PTG~!_)MQ9Ja7G*)shgZTj!w&%RYZD{}!iSb124ObflCaCb^;_#@_;IlN*jCag zik@V}aZxki7opAo&l~0m`HsS$5PDHxlf5`6A{}A}mbuG_WwsXb ziZUK}MRzk+g63H}>jO9!92e2XnnDj!*E71#J~QdfUPC%I-jicZA#Q+~EW?a5L3tH+2FoWwphotC7WtdUdSm ztd$o6o(*4Lu8vnarJSzkLKg7rJnF>DF*>g~WE)_MH(k>dC>VgmenI`yv{f_{ zXtL_>XtOk>+GO(q*U-S%Ajy|(tJCV0Ng9@EtHbJM+$Ww7t)&ETX?KzW2?BL09T z!jm!kNXO`L%u{SW$eI4b$_r})CN&>5jNXG;9i9^RBY_+{j<=b%4qpQ-Uz5;WKorZx zUBoa@Eih4NX&}p|^!5q<3*UaJ$8VV`NVF;^2W z(@3NWf_9GniXp*pMwhGEEK3x!+9mDJ+8bLNn>(9lcBBI8<1N__k)wS=+mrUcMNcGO zl*4s@=@zTpvTf2<$v=`05}e$l{$Si^rCVR<4=XN9Fw)2JzUm3OQoRls4c6<5716Q; zMXIt)p-`MuSD0VsJtjd-)w8s)0@9fDki=fmyZKG?y|9W79 z?=RnFV7f|hhX6+g;VDL}#l>Q+@RfnN?kUb#N2OiqE`c0J76NKeDf~ICAAAjTOYjIF z==DW1F(XmWp>rV5!ILKeSj$eLH(;%}A((-%Twv209Qfgha}KqS^<=`cF!PX;Lpwab z<%z%LiNEEE|5xRSzptqO@2{vUoXefR09%IU&Iw(B{Q=DlT?#ORMo0HmwkrSVR>WS?U?CBLh=ze;LzZD zH^%Y6?9rXk1hj)}6i6=m9b!(P$T7)Ss%ZfCM2~`PECpW7b*8`cVokRCpyH*JBj2fc zZX9a4WJVeqRkx+Rx)h>Coi`=7mD##-LoY+FHcSbV%e@@ulVz$ zU&cI(<0bA)o|{~n5Qz0ee~(U$!z4~gZcIgF?9P0dsZQ^i%1t&W{+XDU^fCQWzOAUV zM{!=yjHKlLiD?Ot1WxjrjDb00b9Q8INM$GXjD5s^6@}yL`4O>c3GoT=m|fvhS>&*- zOceVRdmvW{+J%3z@6lsP=P}WUi_qDy3~VgpA#Ymvc=lrISllt>J-`^DA?9NTkg4?j z^uMV)Nz(`yaSfO=7zvI=&ZDCk*D3dKhv6pxBexJz1g}ArW9JYClgbEZ(0@beAlLHP z1@oUkzzKhmrsIzyE(JQ=4R*I>s*US0hR8@bMul05&O}5&q`p>nrn|^HI+z8U41Wk& z?BlwDKixat>+%lwe+}-1u7zd?YdsSkBdpJ?gIynj^+*?XHqMVa0&Vqg^klkXoNt^P zJUatdAgciL@^#?3uh_HO@sIV2^?+l$tH?>TF4ON*PgK9ve9`9Uq5uak+pICW^k-BX zWDg_{q$UMk$Fi(&Ty->CiVcUfYc)DRwp*nBt|`;u^)t1tD!jrCJeEAkRdKE~L7A*Q zrSqyqvU*8B#c0iL?P0A;`#}H5cv&Bze9)EM@u97wUDxSX%-26R6&n|*FN+Vf>zdX! zLYl@jPi!j|2_!S*cjeWxplqdLy7I8fqpP)=U1*Qcfio}H?=UJ#gWxgOPx zdyX-WvW=+3&nFBfKce4d)v=#34YaMKw^$770Qh@H09SNDMszYlnmfZ~4B0 z^WMY0b-}-&YWO)s6KoT76|4mD5_uPO1P#HrP}b77P#W z<4C2LJBVp;JG3~|J2(W=2a$y-z|KcO;n$(Bpeoo~>tLyuyw3`yt&cz_{niYqH0+l+A5lwswK~+)G|7_tx@>s z(ufLnJEIrPN}V6phu6x#6~~D`8J!fdg?oT~i>VGXF$G)=Ul?1*$MKTFs>llok8wM3 z??{`NCwQqmA#)9-3ikk=hdd8E1AB_fTE>#g&eljUan|A6)UUm6YUL9o)Z$Z^R!%ks_k$ z&ol4ez7N4_#7PVrvlh`6eC+K2-Flu~ZLYF3I?{X@p^2eF|7kbYwGL#`_xaBSz63LZ z{e71KF{8peBe(<_1%He*V7n0x;L^}N;gKO7aH?hmU&F$2H^>(#QN)d?QP2yJZII6a zm$%4Q6dH`Qp`GxY0Nvfw@zM@+R=Kjh`~9iD0^p`%=_SA&ep6R($go^-P&`XKtDGmz zN=<*oY3X&zUy7v$jjgw%(K1p;QC^kbl71H%1UAtpMQ_t$N2xv2)S{NiDka;cm*fIf zvhIbhS=|e~ub!4NWP?D4Yk+dAK44h~I6{@iRE<#Dw{w!{x%9lg(EgwE8L)-EwePh> znzw3}$i7N;Dta68?L6mj+jOH>vqrsA^QZ2Rajx~T?Y-riDZ|7zbu%&zSVOt7$hyh7 z(fcrPB6!p{$JqwH^G{so01caicn-S(sSQ1b{2BV_DX|~14z(|FJN#LpnBaM@)@gIx zaeeVPfB{?P?%^C~Q<$fk-4>30v}1-N5u}WIgKxn|D^SG9r*_FjY{^K~OKq_xNqJK? zTAC=W6(dE=_NZ2M>(h>&V!ZT8*F*ukWlhVC_U=MvXMJb2uy@<^hF{h8pTjFJRIREm z{za@k&~UfqNLyL+{F*-Bx;|6Ocl|)ru5A3#q;J?z^QjV2dE{r-_l;lm<#)gDuYJ(+ zp)I#XUAMGK@#CMLyQ}0i^BcxBl{BR_{8igrGq%2>>5n#?aF@87s+Wmn|JTaYFHqFh|*~Jnuj^XLc~Nf7Y7p-0Vr2?DPkz#I(=p zmoko}4NH8B0rg`Ha7jOa zrNM8(*{BBGAQFU_itY$q0mL_^e-R`OwTN(ul1g1gwi0kS32Ht(9s&;`p)BMwY#A_Q zw4%2$MnJe4Gio8%npnQq+@aDIq+d2iT_OCuMjDe8F=I$AG(Xkz^=!>M;YOxq1cck z2#38vN0Dl%6Q}|bl2}TpBdo?(VRFzu^kG~J@h0Uz>N(1C(kjra4&wF@Jj4#dD(p(s z08}w{6yX-3fFL5KP|XY|dmQ^t*c$3uLJ}qlJq*`O8OJt<9|&(?OBe@eg;WliMHr3q z;)Z}u%yaroW*YM&?Jem)LJV;qSwOu(ok#9LwBc9cTK;j z-6vgt-9haajY3UU^^)_YJZY9(p}eYQXg;Y_vZ-AZ5v{X_G*R8lu*o#W_)dF4HB>o7 zZtsfk;I$2C8!TKPMXA!%mC7i^F!>01v?397C0@!4WfsXk$q8wlj4h|h$1AjIobfR5 zdxben)^u~8G0teVBzRInwU82U5(W=$fKEqf;CG+_a1S2>O+r2dw8W=K3XA{&gd@mV zSQGpPd?-u}iGYrW*CUzeA?Rc1bnHs}R>Bg(Mf@bp8dz^&)bNKMqpuMgh$8G%WC<`V zj1DdJUvsZ^7=Te@sOyC1ooBoYW_xN(G%Pk4Jie?SgTF=Js)I4_edPZnXv4rnlc}e<-jCE4r3RP+hiGT>aGMs-|90t8U$8gN}!F0#;$rx*xsvDv!G{DS$ zYlXeS?sMGtu7JFT_l6DfyBt{i9cQd>c8CQV0NVk~5^bUWP!lvBG*;5@`bga+9OY)l`KYoQARZoe6_ z7P%e$2z?2y!HgoP$WqEnVj;Q{Vhzj*Ek?BAhLTa_QrukdyY~pUNP|iJ36Z!yxc3Ai zr5~NmI8EWF!h1H`>e>aMA4>hA8E_O83TdzRR$Y8)=XmkTCy2E(yP9{PnB;Fhq9*;lwV+#j6w+?Bl3g8O(MET0eIdU6ms z7Ei~Luu<40zW7Z0GIcE`9#qGkIgQlZC z-Ynj4q$hflKSyu~FUMO6*6|l&i-}W2GkgcmBl*NLY&X7$c#k*1wqd=A(YRjl6`xO@ zCV$fnr76nvxMeXJbRK>OYl%(5q{KqfKu|<~a9qbX;;FUkFiKmLkO54ky#x6`S#Jy3vm7|np z^6RonF)AJ);t0J$RFq5i5uOuAq}LSzWhXf$DW&d^33Lx?J>eqGlWdYjEGG0+wrG@C zDcvmZ8A~XpNNPkggk#91FJl*}nOn`tH%r z8IDFSw)?c>qb1MS&-mT+*O*~Y=(p>J8XsA1nN@}?-ECDZ)k95PZ5z#dEmz-9drosx zciphqw8nhY*iQdiSEl`=Uai`%a;ul?<{P%_cBoR-25m3RWp$N)qe*9MZ7eXnHas^7 zO^2*wZ8yxD%*(BRY(Jb2y$$@U{P%)%LK^o2iRJU5^fPa7ugei8IeTWLTjKWjDGM0_Di?`?g0N}Br+Gk^O;!mV5DKB zR^&`{Q}jaQQdG%Y0ndb2Gab;V$hP2;fXesEJHkIAaM6Fvx4?tA8+zvWFNIe{o(5{X zU)*o)S=OeOR;KO7`lfq^vs#C$W6i6Y4DD-UBg2WK`s6wc(_U?;GzvU|e+F)NtISr^$$c~-#_ z>^1)$F9W&GUduult3u8F+x(-0GeYBo`vQ|f<)N+qUf_zJyB53T&Oi1n$1+Ee&1}A9 z$}=xFA%-XVp2iWTL}R>;ud8e9XgCm4YBJUDwQKa9bbr*O zW{GybCP`ITb634cJ4dV1ywX3m3Y|@y9@kxOmT!c&o9B#ch_kUX-&N{a<2~rD9~c$c zz&yv)f+wgOi39KD9``V>3wnc>iZ15|_!7ZXfk05h zFA;1J^hIYQ$=sP79`_cvGeV;s1t%~!hT)qCCDnwSNqnOEiCjVsy^~BM`;vUpi7&?* z^S5)`upy)_{)fIJyiEG|%h39Qsr-7}jjTb8KF~Dg49+#g%j+vRji*wRgoRWNuEJ&$ zwaNL^by`YusE(AKek95lmq?Due#VSfSe0Xx0~L7+gS<^lsWdFnOY)^Q>37*P#r)Xj ziiDV$7@xGEbdT(wbd9LKaG`L$XrFkvC_^}0G*~=F@}KNbOp};iF(+j=B`on0kyDsQ z?ZY2pb+IXe$AZ}yg}1@^*lEEYjD_vz_YsizYzzh!?KrAIV+A@vJU^Sa83}U5Tr>QE zkr?^p+u<(oCI&9}6Z}nm7rpy@hkgI}X84c#)_RQYJ?=#=t}D^i*7?i6-u}cUv@A1? zFh8_*bsl!zwnr=%&E1WIbe+^|s(V!RtsGx*qq4N-kM^*!$h5<>(p+T5jK{PLZ8bQV z>S**rO?=I%nk@APO+&`1EcR)9 zD(`&%>d?6GicpteLU4TOaOitrqdyd|MrN{%oIK3N+!;F1b2fgnMTGT2E|wxX8G^? z9t8#l^8;Gn2H*OCI}rAnyf3^zJm=jtu0rP}$56*7dtF518XDvjy<32V~=Lm zWj$Ijh~DhW4U3sInF-WS>cLt zS2`_@4UQ%DMz)@|M2E(?+?D3e_09=24nK~ph^nCNtljLJ?5-RH`NsXleh80)AHZVH zMDAB!AvTGuu-Uv4dNuif4!=9lF5d#-O za=AM8Tlh4*0zS=J4Yx(qd@oc`=8Si$bV&STGJpJm_RUPiz1FQZ1D4IK$q$^_0&_C?N9j)4>8CL&y90S`j2 zAuo|}=x+X2{v%#K@*J&$8L@7MY@x**r-i>{sNQ&7njhDoTCem6eOf97{sQc6idWBFbt_6N$i*P1Ai9C!?#^wuj z=qH59?ZocPVliJa)-uDaQ}9Vx$Q}w)Obj|1ULDBw_w|=}cX&2+9?f`xf}y2EPUG_eV$0wOH%YcGYdu#~RE!M0Z!) zM>|*7z!)|Uwvmo~&iihmw~uGCdyj|o_XvFP@9`@Fn*)ZxKCpVXdz*VzUWI>CKoi;n z?PP6-8^Y~anarns-1WSSggE_&}(818C@bk!n2q&Bn zH2W|3v;B*`3*0*A4+qE5%Ra}hbNqGob`Ej;W4mf5OpA;y46Ak7+OL|S+H2Y=+DlrN zKF@H(a8=(-uP`u79<#{W#pbYYa;$Z{aEx=@aC~(w_h8;MUnl<(|0f^Im*Z>c{~b_; zzd((dJ6IC7l(U-Ch@Hx+V76ziW6ogQgAPHn8Dp6^^EK2JS_G|%4vc(?jD@7Elk5-h zG^UNwlwpd@4Q~rI4VDB>1=B)hp*oSD5hPMSwA6pk^W6nIHMX$jrYYXAM|Vdn)3(;0 z(AyjR?vUD?iwj$%8{(bn1DHO9T$+rz&U zJjx4>3O)&aj0%}yd69@59Kcr)Wd3H5y3$LA;1Oy3UJIMSSsiuxGKoS1o%o~B^58EM0G_| zguCeHl$^Yb)#B&y9wSG1d(dC#6niJpwy4?hpZh1!Q^gw6z&f%Co?-(25sAL5_o-{@cJ z>*{IYd}*y~-eBCLzXAG>KI&aHM9uLUhHA7bsA{7=t6EocrFwn!&uV*hzv@+02dk2+ zJ5_tDAE^#&QgqAo8x7rz_l(z!EsWz08}%pk3}dxvuVt>y{ZRt1-!q=AfzMm8d(TbWg=%6`yE^lCSg1Hy+iO`xEWjrj* z19L93it&aqpCN>vMqWgkMyn#MNQ>}@P*!k@e~I^xdz5>Odzc&X^zqL3ZSltks(hV2 zHO`0jp4L9*xu({pRi-oMeCs!Rg>$*Pw&y?hX4ekqQRi$I$8B|;a~*M=bTx7nIUl-a zdv1DP`vwPY2AhVKg{ngGa7}0uxI$NGWaMJh2URg&0!@+5S;G0qadED3hO!}+gwX?9 z6zv)zB99}g$f3yfNXKX|XcL3O+{!40+Cwv=%OlsqWuahjTCjI;L$EAp2^Ivm2FC?P z`rdoeJy+bjT-_Z77QXS2_NwYw_2a62RWqv^Rtc(BRraZfFYi#+thBJWvdB^-DEV1> zx*Vz`s>W7tuQ^mB1LrZ-XVp$sd-W3yscWV0Wq4ucQCcadc~P zE`AhiiroG#3RLW@eh$rv_srY@t-L5)$5|hte^$j2=VZr-unY z2}|h4)H<>UNfG~I*_c-#6g)+n@qTcXTs_bssod$vOTkc?g;4DU(p?~>e>?bx9|AS``9f>5u zj{m@m@SemUQYO4AS|xrWDU>#rt`eUY9TE)}u|y@p&qBXYDq1W&Pv4>E3hRn8#9Jh6 zSzgRC`7n9Q7^8HKzma7~w~LcSRrErthB%2| zz>W)c@k`N#Xf5;#?*+07sMg19F8dFwJ(Ff!gHAz{p+-39)Lj7uk+0fCr#kkm5+n8h=WV9Op zHA`)!cFHMtFYp%m-uorN1)(3|=1~IL3Eg6RXF8dQtW9tm&M>YLIfDpz8+g}&7b8SC zTsON7I{;U}DtG`pn=_UB4%x@^@=l{!{2yp5bQJm-{f&-BSMrV^EfE2A z-(GN%-vg~i@_>id2|mm0$oK}Sp>0rEv|DsrWKZ~9sDJQ`pXZp0_pOf?7sChW-my z2DS!_f##u#ur|^%DvdUaJPOH!t^94hkKM;zwVX}tOlzr0WIUpr+jl+RavWwR+ZN)%~krE(dsM>Lu=9A(%E#)bSc_rnj4xd%~Q2W z{f}m~HbH;X0Go0we{IVh4V)XC)10`o+OgFUu&eC(j%&^>t_qjPZFBRzjlB;&uRJ}y ztGpbqz?153=Fagf_Wt#~4#>lUqauc$(T-Wg{LVT8i`f&{9`;o3D9{sh5ZuNzcop%J zfG9<@l|Dt55MPL4gcYwvTp^}`ZpJ~(#rE-EqW|$ufm$Kw zZ9}&Ul7Vg+NBSs*aFMX3u!3$Ovs0&wj{0&uPVd$(_pU z#V7bKqy;yQ^BAtrs$$@bevHA)EY=s+0eBF*9j8C{C>P~I+)D0oM~ILhU3 z--^7??J7aPcjg;+`WNiDm0db3C>nZ2U6Jl=4>d1ym71B-8c-bdetr$j3Qp}ea zQhqE(6mwYCPG*z-l$;Q)rcCg`ol@DoBDefW>DrQdC09%CmYyiBy0?7=Mw5DwV2ugc9+HUZn~6OMy3;P;P@;?Pa<(#nzNGKljNbpKbZJarVXbueJrYGd9}Z-`>|&&+^bz&(z!4 z#Gu!&H9Rx6Hk-`1%(u<8EK4oFEzPZoRp=P$TBB&H>lrBuB>6pdI)}(BbGARDfPZ4s+{r z&u~LXIqw~B2zr-)jsKaqm%E>RnI&Z&g_=ZPM`lL3Afs3w{V&o#vNIx%90(lx4x_o)-q z3sh}YJJb)gKXli18r?y?N6#{N_5BU0reBuUwqCZV?U;k-x(e=(7$?g~x$<1!oY$Ri zT#G$sFWVRK<@P{Id8Paxn4H)_B#=zXKwcwai3-9&uBV1l8%Qq6Bc~9F_zozRBa?|| z#8+}QIUau^aPsx&X|Ok*M&9z?^K*c^a0%Z|%pvzs+vrqbifE}=E3PYUA?hu>M8Be& z3MUFL2`7kPF<(4U)Lz&Rs0j&ujQUOXB6pE9s8^H-tfTLsPx^}u5v=6P1#_{@m_V={ zJ;Ixd7&vX%VOBTz4jbi2fJ#`#Y0N#q>CR@uCs^sAl6zTa;O(4EV3(@owc!onJwm_o z|L`6B^Wd*Iu_}=0Y!;LVj$rZlD0~5a8ssy>aFt*qKat;>Kb3!+&jF6(Y+OP%rM6S6 zs1Owd`f9oGjcA~BbWDo8yL_VjnLJbRMy`*^mS0koE3;#Hu`iS=W!Km>%470fF?V8i z#BgJ>V<5Rgk*S=dyr<};II1XC+)yl3bXLSF4$A|wu9A_$U*t_Z3riO)<&WgA0?55ls+}ZBtPNsdKHP3R%e8M!)xYsb%a7S;{j#uBUzEHWnVm7D|{Yq{X zXBHXsUMn>JXc+vmG(dOY4j z-zWd~z?NXMP&Dj=cCiZJBka2z1NSjdldq7+yvO`ljEyZ9%ojWq>=Lx+x8Mc2Y-AYn zFLIH$lD`|9MD!xF$gbo}>M^}Q7!sC?+~Ok1S4pKA4a9?&;xz~Gccmuwd{sn=-!PCL_ zfmQy=z60KN-u1wJk^08@{J#BujsFpFj@tSE_#Xv!1y_ZXku+!><2)mS@e<01jsRT| z!&uL}1lQ%P1T{UC+n*cZ9OkUzJmNsyB^(8t1wUpjWX)l|gm}>(;hte`_-CkPxMM^H zR7I1>i_k65k6aGU3^Ick0)l|i*WWw9-QQ`mEA2gPBdrT8d(G=jm{FuRXawp_HTA1y zRVOP8EAy&iKp#1}T3dxzhpO+@bXVb;6#Zu-ZJuGC0*-y=@#d$dr>3BJf;G-I(sswT z&3@6LbsV>owguMy);MbqtJIccXE>4^9qlYzjM1<+xj)eka8Fo&>a!B5$nxr2CP(SyLFFF}X$V+9IqAf8C}pq~pN z5h=PXYAT*5wu;Lo<GO0y;cH>3 za4o1GhsYkp82qmQ=C?xEq5aWV-XrcePIJyJHitb2z5p*|e*%8#C3rGC3+@CrXANMO zqu-+|p`DC2tU2t(+@FYoHwbCKUC24ddBHUy?Rg~6j4VLra`$kSazY$0xZ2NXbHNs@ z4?Y!dkJI=~EC*9!u^1DRVc)S&_z+?qahX8L7UXo2qMFf9gmUp*@n6vvQH=Prc(P=& z^qK5=j71}rYwz}7Uzn+ugp;>PwCzrs0DzL@*7#fwbhxI0ANrmCQ0QJF#A}gfInnVtoXw!WJDI zeI6MQwL#AqVn!QC6`dLFAJK-s2A2lM1Q!IHKDN)`z3)BkS?Jnr4_Mlpo0@JIjmAqR zk$H`&+HhNUL(^A%P1RLZx2A4YgNn~(8%mp%{8PNUcwWh~Qcqd9Jg4$%)rIP@H8WI) z)t5Co?RlXHJoOx`*^N@oW7fg5}{%=mcW{GswIF6h@FBi?>PvPs(p=fP%5xRyy3X3PWLi&01-3S$0^4nQ2q5G0phi*v-7t+S+m2#qhNB+;(eR zwOkJ8V3*$IcJ+7dbY?r*&R>oK#~x>_JK$OE+vy(#T-E=A*5JF~tYD|$wqSgyAha($ zFLEkc4K-s7hQ>$hL_Lv>5iGh3I>0!`*v>e@xW=dedOZv}Gd7$94+P%ld?vy=!s^ee z&lE96KvSYKBQqj1q8%VPlo2H&#?b4aFYqj|CXny%=fk|a+#csU2j!SjY_>m{YrgEQ?5l9y?)oYv5|E^eyvu43r1*g7rd4;boEj&=}@WxIKr57?F=a zt#9P@MkDA!0h>5LekEUnefAeMj&3g8E@~#(COt3xPvRH1krYYpN_L8CgsbQ)l#}d0 zjv$v)_d&|?lpaQRrccuMgww@OB@9pz$|V;h-6UfqxzZxpTDe}~Q67#hjQtrqC3djV zCuhqi$EaneWm{wuWM^bPnNVgE%S2^#M`{gm2ET(3Cql$vq6hYZf0}=V-0&d3CD3~@`}Mdxs<;hZO1!}R3V#rS?C6IHX4tz&>Zv? z$XNCWE@C(EImBlIkRIe=LXXeH`{V2Imv|ZQFW96X^k8GCV$wmZClK;5Sxr__`@q$Y z6|ENC6sJgs%Zg%dD)z++rWRYGd~_gFkOC3Z!uE4C)CR>G?UPC~c1 znM$!@w7kB&qr5uin5gd~OpXi9l{%}_KQ-~4T3HGM%Uat4O z`!Ue62dw=plg(ZeYJP7TVax=((;}6m`fcT|%1)I7Dhy@v(r(4VqT_{+3I`Vni|-WA zE|HYxmX0gEQQD!bQ~9}yHdO;_n3@l|mBula4)!fhyK9Mii`(R`@%;2&_RS4+4(CPF zq5aS)#zp36R%3WGdjwYw2#c2JTXa9)#NW-ohCb%4FqG*|RN)J0rdJVCTjIENljT_isdGw>9wJJaW6=?YTR*O4D#5Ac}P!M{N~qzpfbOpe}-WJMlEih%CB z%NW4)GUu@d0F9W*jA!~8a~WTO^K>D4JK7w&0eyj_(6Y#i(B{A_U$JL{=a7f*sd0^T zKC*AN?YFM9yfN=G4=@`{(@aNww5+gZ+D|*aI>WBs zp1a=8en+4(#ECX%tYuAQXK|Wvws6{U)!dtiiI>mc0Q}A;#2F$&Os573KZ$OMyGZs+ zj!IC8T{KMeR#aR3U6dtSBWx@z67~?klDw5#r8A^V>3In*87h&h~s&5G@&Y_6Cq?;&3#KPdkuZ>?CP=&jf*?-nybdQf~- z)JOD4*jqSMSWPddRK#PP!oOiFvG)R0FrGgb-M~xb#qsL#TJZKE@mx0NFS{S-ICllF z3f;>W3K9gBe5t@Bh{ZP&V@M`BoX91H5{-$m1PoSxHFcc6LARo>QY?x=-p2DWJ-T%5vu!7u;*h)g{$mDqfXJ zOMVx>D{fhwRW#-Aq~B=4*k2VtkN))jEdOOMX#QvI-+v2@MUzS{m%b{KS6r+7r~0re zO`@z^}YbjNteIL{bs9BP;iJ_WY1tvSz1+ke}|jwTMXJFkC-UE4nLsC2}fU6nYjs8%Pht1r`Tn!5zV8 z!KuMXfn$C~;9FpA@MuUB=@P96orCT`;~8yP_1I%LBzF|IJ@*sm4Z9_*2X6EfMrpK1 zWKH-|cx)sR`5yTgP768w`$3kO>ze1Jol3_~>wI&Ji8NN}S^5pSe{_8P9lc7QY`Aam z8?GB5Lo59$U7YTjw!TiH@2BsrzpL-6U#rd3{8X=2FH_If9M^8sozw3$+%fz#oHc4q zF_sF8)$+k2x8_(+Smc)JmMa#Id6_B3cpPvjOj9=#WO5ni>U-#LZM<5kk^-jdsM@Le z0CJU$YM!tS+eU)w^3s{tWJEKBv62Mc7Vc73E6COEYBsLG2i%*rOE3^^ISipiK-WjZAKmd@c!1`juFn zXixl+I41FSLQF!d_%pFhmDLIj==HxVgvz^$ctyT^4#+W^%J#^fOWR34iIl=^)FiSV zu?TNTtR*9)5wN7mbf##D#3h|Cn=7j;`vvOOOUX(}TPXv0juqm^B3RT;xSLi22Xz@e zTKEVgr8z>L@FjH~@T7Oa>g|Uq@dx;J{5Kv4x{rr=ZR`w&6YmH$c!d(GzK|4YMLWg$ z;uew&S@)R1vM16HvaFb%vRK(1*-zkHTEulkEohu>E0l|diYAN7gmdW$WK;a5V4Prv zfFZD;QeGt|f!!Sb!wRwv!Ca7j_~4I#PVup3vnDg&GCF{i@C`U`G9NHHKn*G#PXTG6V)S8=)0T_LFGR`IOjLWQPcdu5>VY}K>sbv5a#1L`T- z`??}M*Jw80HigXJt+(yFohG-=cRtt?tRH?f7HYysWk#4Y;eDL$$Tj35lF!@4U(&B1 zppO3s9sdtH{vUMwKj`>>(DDBVbc{{Jrvb@j!$WutwgZ3x@q%o@OhE$X!n}ALGE5ec zN#u0mHeQ4^!Ukb2a0yY1I122wbi5GXOT>^2YB~t;{|E<*Nb!2{Me%C!b-jJ+b$SOJev9v-XOr`)ZMtQ- zsjab*aW#0Cr$O9zNcT#2S=UJi>H6uabffi|hJD5~b92i&OPpo388_cC%{28kEinBw z4K|-OcQxmjB*uA$V+OvlqtR)6VJ^02+0z^solo6Yy<2@!UmI_YXR~LeyRWO*DRa#P z_Er<;XvbK`d&dt)FUNfQDPRC~w2CcKGspbP{Mu4xd1$_F3Y%Vlm~FIKX1Qt|Z-3@E z;+*GVd30X3e}=zt;C66Rcu{l-qbchkT#Mu279pLH^W0X*5MGpb486#&6ojxUyftx? zNTr?w-0Kd-r3Qk4FotSRD#$lvTVROzh=qimXacOg<@7;%CB2o3BQc^Oeji(ae+1TQ zCDDw!BOD`kiPuT)NbAK6ms{m?6dRNceF)f}*mC#Fsc>wTbikgXT2n}?9 zx|(`RN$A0JJpG%xPUTV=7xdoXu0+cHlvzB3@! zM%GAX7Gn=%F|#(a5IP)9j~S2eZ~3SuCR3nU`d*_rFp#( zHFYtcG;IMu&KTVRZA;A&Fk13THCEGHyGh$rS6ly7uhD5BDbhJMD2K!&=^y)EgM3l^QVnO)*I3J}q0uHSC9x5&HDSMB5b^}e&d zTp!_k<~s%A_ndHgWC+xeIUb-X5NiW`i`{{f!x3A0y+N2UZrP|iv>mALZ(!D4uAd}7Qob~6?jJDVDq?i)p>T9)}Xp?#Na zw5^kE3$T}Q^Fou;^uYYgJl4F#Txo_ZXUz9aBf%)vLVXjxQ+G$#QtvSon%0{qn)jL& zpgO&?u&sx!VtZp}T{r6O=${wF!VM$sqD`RQjNi;{FrPD-tK`Y}n*=mA7L$Q{`yeoR z^Qi!}S;!GOD3zr@2tq_7)(lTwoh$hp)rdYrJ9@FajrzR=l1TKJmYM-QZn zsdm&3(A=#d+fZkyhm@VV3)-sJ;!l$OQoC$|T&$oJFXjKp$IIKu`^S8h-j|G(BujRS ze~TQ#H}rI>Iaz_X!}b{Yu`KKawg7u7Xeanj z0OJZ^;~XILAWM0+suSRm*j=*v$<<9SUG z5%)gZ#yZIOC%Pk2H^L5=1V@L8!|NlXBU1qW`7YuD2C2{A$oIpO>%Q+??$FzB+FADX z)@By3*=H7-7aO(uO}aGgIW?@>SUsz1S>=$5Pi6L!$;F}a=_Z%_S`zk;sXPTOAKdpQteGmo>rs#$FSQt*wn%7 zFn=-2%&Se!!R~zARANdto6Y5xhE}s>fh8X7-2*M-EGI1AtOx8#&R(uT?!lgA-W@)U zAN8mEhX9oPuYW_JOOOirLodR<@a%ADs8u*8@+MLf`5Ya~*uxaF*0O5DHh46<0lN}@ z4X=VLSXr#|%oWTR0CMv)%Ai)zf6)h#A>rS_`GN8N;l3=d*ZtM?2n1N;9m)0^R-QEq zcJ~vO(-yboAL}ox6<90*+cSGT$2P}j5OGBvlbq+B^PGPiA$wi>T-#79W$9_|Vcuvy zZ9ZkTn*Ufztxaudwj;LH0E2&Sx7ZlA3~O^s#8hCE8($f&8NL|G46h8o4Bd^dj4zF( zu~NTRH&Cn4NYuAgq`FwG)*!lUeQ!fO;~L`;W5jsfbkyv$T((VdkgnD4d7d8LOTMuI zR_I!ILbL!%VqRqRX7AxNd4fGsp7O2Y5mfc zrCm#_PUEMir#DIOojxvoZu++Ln)I0&bmo&xU*@_@L&lMeyo?bU3o`OE0vT5`Th^*m z>s#iE%*@O!8K(3->21^RrwvaVnKmHpc`BAlrhZEu2YzaqRyWO?S}XNKa<3#j@pAn8 z*j0*NF~6i%iB0@a^i$YX_=46^&&U}7=y@Ta`B}UzV8qbCe$5`q9uLoBWwRp8_5jlL zK+m9)0C0T)#WPHdd(2g01=KXc38e;>`<{6wy6?MMx^kV6Ur0e_!T5sr1)YDN`tASS=uewJEB=W7j{n;qoP~df7Tzxu z7kw(qD0u+l=seX>z!ylL8EmRpj5rDOmgdLL68 z(@@h#(_=Fm4DKzmy#Tdgzb(hs&=v!ZueQr}qrKb?JNh{Y=Lpv$cN1@Ye`2s_2nrtx z{|K)K2uFVCbSOJq7M>d^h?pbSB26Nj!&gGbK)9UlALF~{z392%p5$KWuH$~`Dt7g7 z|8VDeE_y)e^UUyU0Y>W|$4L7xYa2_@G~0B{G|W8Rvcx*tX0+*St8KKc-0HRtvYBjK zKn1(xSmay`2GJV2{jMTl5Z`iL0LVP%neP?)I|d8G1<^FdZ00T&3h@1BEGz3D_%b{J zZUc{mVYm;gHY=B9WZq^DV)cgiv!`&|@E)TQ0RhayV!?L4jdu_k#tpJ<@OJnvJcRv` z-HB}nsLB)Q4j6Sl8~ETG>}~6L>|z3p|BlUMEw@%%_gd#!cUXs58PHtELd zYJ+hcr)GdgtIk(9RR2-I>i^X3GnXAoRLCycyde64ZG03HM5BFa4ee=5laEKFb2S%{JKqHvL;07Rw zF(dU+h`*8FQ?L_jLDY~z>N`z~a>TR5F3~#CNYQ-JZqahl3z1kdR2GuYitQf%Hep!O z)#Qv6X3DAL*yQ}A-br*)T2iazI;l<5eHo)`wXV&|3TOSBeJXoYcH`_>*$wLC)E!=L zVEst_?hTqX_*%bf{pIy~*X>*9MRs0xk8DZy;jEI{CAIvS=Q9Uoax;UOqO3RBTk5>5 zv#SnOCpTM}RamPeQ<1qXqjm-_qfW;53}xoiOi$+COiAXnjLzx(((0u3P3xG}C2emS zE4^KMpY*Zm-ZV`rk~%l#WU@JFYf|H+#YtU~ZzR7=b|veQ7bVM+(~{RE-%TEnd^o8j z@xO#F@h9T8#8t;uE0+MB@kVi0k*vI}92zT(6U8xNOBL-EOXQtn2FTt^$4W~i>&0cl zzQTdRyYvNW4jIIcVR3@>Xg2RPm*U)p$FPnvdO>5LWX4S}ATomS17PoiqGjQAp|`>O z;Q8R0-~$kA-U3)v%ivRg0qDb3&U`y+RU1#}E4BZrSAdc3nHB3Ra%*aib*ZVWu3KKW=))hwugTwMei`-A`1bzG z=}%)GuX)6M^x@&vhoc@2dD!6Lp$AzHpafO2b9Y0?FH01NhFFU?9{a*5A@#m>uhkq~r{^jS)zmH3&R_WBG27`5mOW~UxJRA86 z?E~X49pPo1M!W-p-o##V9{CS3f*4KG)MoCC6g+C_2S@<0i%r2a*2%xU-5}8B<(MjV3M&E+Q({glJN5wzrS_ zgsX`QcD)8&b*`t(eGY`h-Fy~rp?_IucDP>H7kU~_jT$3;z~IleP#~xZHVr=W&j@S} z=|lTN>%y|A7n;YM%Dlzc#$W?8z8#YW<2t{=XdJ-d8I73%))sC@)WVnH2Z#?K8hAux z5@CD|epN7&*M+TSSfgmTt1sw$XiGHn4QF-rwR=^Ys}rg}Rh_6DSuwhNOxcIh_oZ{o zW>;iXZ>;%TqfynU@2XZ-pQv(Hl~o~Cqbq+_EGhp|+PlJ5=_ekhGfCoM7e6EMP3}A9h_9@mtrXA`SeH@Yn68uyB^8!>@2>BVG zS?TP#@Nm{RR*-q0p@PnU@#2QeBvuSu#O}a73)}@CVn!Y!OOdbO-!7OK^`WfDieMvO zS9gZX=PL0m^|kg30%R~NxIM7c-`w|){|C?!4??}doN)ip@o0OdBf2+woEgJ{;oZmp z-f;GIHprZKgTXjMGBTc*%B{m|jc><}BI}S<#3;!+aVaI0q$&r-aAkiJ_-W&kdL>V* zm7U$I_LbV#>V9e{YYaD1*Pl|iOPxpApKD#rm{farc6qJcng3tG7IBaE33@ufSr~%Z$=1;-A%TZCnN!L>i}XfYqegxwVj{JLI!{qaTc%4g_?hc#4XXX4_EYe_@N7w)&UNy$ zhGregK2_V7b|nQ%Q72wa$V-A#hNnzU>y=rWnV3muy3>cGrYC<*_!c`e_CtJG0yi-> z*`9JXIg;=y&aP-7`zq={ZzuPHaB30%GWRm;5p*beJaQBCtdm2mAmv}-3wmxi8{2n- zDEGQu3-In!wy$7+k=pKA6s83ILiLWSHs!5LprUZ$zs2KARuvB~$}V*Lq5fVioLh{S zrj=bQ9ba;<_;~T%qAE}k4*WV&HTCSWAD%GZv(y^`kMRo*cbY<>ixLa`(9|Dw|tI0pY-Cyt9x&Uf9(6E_*?5A zrQa5OnfY1yx#ZKX&qKboEm&Jf6`PB((psQv$tcgSxLVap6{m(YgY>6N;{c>N8;s%P znD$uC*{8ZB-phfO5i4Y6@>r)B4;cS36PaqL7lc7EU^Hkm^pbIt`Ih;dxsdrUGsJ2J z2beUgBM2~U;I0mEmLYa9T$3hfCD@0iAk)}?m>8ofx;XkV+9ujQvNWV8w=aE#8zfoVgKM5=Sp$+cVBU@ zcJFmXobj%)u5YgQU|^LU7#D04d>#A{8WBngbqb>(6rKt#VjN=n;p5yX=p?KmfRySI z%?K%3O-#g3;tdD^WK$nnJnj>FdwTkN<^%X- zkLR~9F0jiN=dn2dwM{W~*X673SA8y@RZ>&gT7}1#W(%OyC13_&YwLB0zaKU z*8K7pV1IJ{_AT06e!r%nI#K&VPw3n0cIY0cJ6B`XQh*#@P(J`@n+L!v>FQx1GOnZ9 zqrGW*;%pCsMa)kJ26)%HoURma#Q$&TN_a`61H%Bu*BhcTL1)2b0Q6r&D$rm2_56Jp zL~@DW*gE0^)tR12j>EITNF9$nNQ@)uVm(neH-}rm>nb=0o@*EE#%5qE1y}g%1jYOi zs1HW||FQSiL2;&m+a}ujw!0fn2o4D(BoIQ}-IIyCJMoFTyH9*##FL4;01@1wY1-Z0 z_xV-rzgu6`?tXR7sY6lJRL@M3O4IM7_kCTN1T*d?A&vZqx{49Yn#``|o)xSY-U3yU z{`{TXKb&G-gmAa02E;RVM@$es6-lFf@r6mdQghSSraRK*nfE#yvXZ+z%pTHtTqj9q zdG@`oC%g5?t;p-pb7rr5z0MXq=$W3c%{|=xMYokXyK}hRJ>Adc@_JZ%2zr>hFYdaa z^O}t8^!)Tq>87+bsr!;=CY?{3mJCllkUl8mLRx)_CuvE-JT0>Rf^K=3fBQgVKa89#1G$a%qoEjJrSQS_w zzz2HypZn(c=L9vO%rGtt3#A2y`M3EIf$5;484Q?$uwa$1#*6hG^0`4lwInzXHU-3m z%iztRAn_o$Hc$|l5a=JU1xTSY;a!L{^h_uUvk#oh9YLhSS^{rC4c-dMv`7$U$n^L3 z5AaX-sokfXS&kZ;-D61_BCj#4VsPFjv+JE6C(qU2 z-Qdb|Jg}ZMcQ>8{V2#^4j-G8;XQ&4|j@8D$z%KctekcfE()3pSEW=s@(r`>aUw~wAC7=nyCp+Ob*f*sOMJRy#K9|68zGJjMSWY!;>+ zo7IlZYR6`^W3$?^S?$=Yc5D{NQMF^U{+HOS_LH~vlehMhxAv2_|5s1mn9U7hRvTRw zBmX5^FH4o3m&Qr6+P;Ad(;Dghwu-j@)qfXCKeerprpUW0F&eIZwQ;`52fz(yj8;RA zVU*#KfndC9TnCZ?H!V+Xdz|~-b3AiEapt4@E$O z{U3X&y@$)@ig8_VO>wXGF_bgmryk3BKkfGLLEo` zgW!S!dlqsiNMyQ^!%&IH>&P)E0_tCo8<~Tsg? zSLg}Av`&Q&gUtz7hDL{%!F;egFeYqG_;_e&Xl8gdECxOlK}Gxq_eHZq$w5=#d+>Z9 z!-w;Xb;dh7xeT70ep~1a%oW}d-U=Y)JHy|?d%|5|=iyE8I#3%;f*xas5*kT$6h7@N z?F8)N#qKA+}3=$`h3dnG(l)8zQ#5l=J zVdbzNaPD&xd38J|zqjBy{~iAd{}%5ncvU2LFFYO5IqL6dSM>JiGtr4L%VM^~o{T@A za47L((y*kW#M6oQI?hkXOW2bzHz6?r)8TmRzUa!RE+CmVGxAjA{>XiiFCza$(xZ^k zKcc@x-Hc#}4hVkn`tZK+(EJjfpIgW!@=o%e@-FZp;SsItuP_mm|9)0|2115;h26ByFJ%p_;&P|3z;P_feU+Qg(YiBnu<}yyMNsN5ntT1m{EMZ_)SS$>589Qe=hhL^>h5sykEV3 zb^Lj^^iFAP>7J4R_%)|W>wazgV<^k17*IL3>TT5@kUnaEHvj+d*`!t%RDG&aR6ncP zP*>HksHv_wM$B$~E^$hEie;)n8nI@hc9c${H3BBrV2x3IO7luPLVwzr076g?tn2OD z9Uq<5u0igRo~ho=-WT3?-g%xGE~JBMePbGC%r$fb0nnR<`FgevrQM)8rfJYj)1J^l z2DI^$@w`cEI%+xw=w&YeQ9cT=+T(5O>@yq*u6Pd`(4m+4viv6k_d-2j-QeFqtTPIP z^b5ki@Lbpr7zMro&O~SsT~V#*iI~UOSit2y2Pk+CNmIxV$kQlys5H8O;bh!kzGSvC zU$8{%zgc8J`Me7#p9#!2fY$Sj`HNM|9?U7?P`E?5Gr8jdE5pd$30VFUIPcgjb|Xv} zy5;}oJ?1^)UmuzT*C5ZMY7krCYY=?^gX<0mU&07=)OC!l)M`Q>tQOji-H1O#@RF05 z1;kmz6Q~#I6(fuDfc1qk3VQ^h20R^E_!0ab@;zc@ z=zk0yjex~?&BgI>10xW5&;s;)#Qo4u?_%d=+bru3`zLR&@DKQW_>ItSf3mN)JIR&^ zCit(`6YdM3wLQ_(;t<&A)>xa+QRc+C&$yETlN|4U?4IJ0dIa8cK1Rq0)4+a(Zut9p z4|t#Y7W&_ZexptS{^mNAB=pGF>|NjyI5d`D)@o;pKP@~l+%0g%gLAL#ZGjZJp(CuP2QiSs{4y17mxmH(9PkX=?Z zsS`n1v{utY$&{awG35i~^W^(x40{7iPi(9QAC_0**VRK$Ju z2+KC@GWj>@E@i&4i$mZVYM*ZMX$v*0wd3@MO>=FJ?C-4C%xPw>Im^U0;*4KSxwb8? z&Ay`G$j^F}% zlq^A3-i8*l8zbvA^-r4fK)C&}WMUJyuCVS#W7igTYh~;2mi&h4ReydvN{9Zu_-oDY zo_|Q?2dnPX9R=Zt`DFz^B%f(tcbA~bS67$R7^*gvefkOed9*a6c=I>I*E_`r%J0{` zZ|Gd7saXC?S+cZr&o4#U{Hn1veQHvwCY9Hgjj8-zGok_Cvb!xy-NVdrY_PEn3zP;a zLEfU-Wwm>gV02_Lh=~c11qe*&gSXzZF7Od~j^Lw?pjT2{L<=#Q+>Jt^UuO&Wc>++J z;KlG`g&&0%K)j(wcrN;F!qJqIsR>Ds;%cLAMs19%PMV+5v-8i+37sExs_T^7$(`Pm zQjv5rWoM_VoS}J*c_rOPcRiU^*~yUE(n*!|HTz(eD>IbdH={7KbLQl9ZtDGHQp)Gl z%W20`h9ta*IxJB0rVD#TZ;W4^FeQQ4p*B7tVRXWcxGB-<$e&TmV%s_(l13*tC5jRj z#CM6$?;uUs1QuQmaoJHV{Po<9yz#s$&U5xy&Su^bAurMxks&a$hSQVjXIVaeOXT*L zjWLU(ev8%#s=1}CF@Q4kkkOBw$y*?315t&kG&m3Nq4O8AI*0y!|p z2+ydKnAe#Z^vPtv=D<2pb74DzKSH07)iuxr zJBpqU$nFbqlW|Shomeqos^T$okVC@Kz}euWFgctad=MBPzJp4`ZN$BUOvq7iT-X^x zB3@%6$z`+&v?el|^dGT;xR&q>u+;pRBltSfP3lA1c|fFILlhE1_`O6gshPM2w;JjP z^~8-N-Xi7zW=|5$!i00ja&NN=Y3qm`0YPO5zLq+cYY`q4R&k|(QQM8iqi}!=$A=$8 z8cKb}n8{9If1tl1KO_R0Bc%xxe&$iSlk9}m1O*<2y8}H(pM)y0DS+uT756tPIXJ?3 z)2ucAvaEAW^tE}PyAthDmb1nI`gH9!^&3?;%~8X5YpJ8ke%&0eAFS!AW@<8ZclDn@ zf%1^<3kV=n)IAhYz(SOvP^zwKnA%TjwS20CDyFw~mnCR=09({((_7s|^riRwSzY)Tn_6KULHh-Ho>Q(w8-1=VE9US5WFM02-}2xf~F#fu+QP|uycqS zR@8?W( zr+YTIYi%QSyJba^{xV2&$?zIP_GF5DS+4Yu`arm;=6jbj_`*QYkzY6vz=Z@kr5)MRh@E?FUkB=&lI<>&Hn%{y_5vNH&Y z!?n8rh2*Pxok|X7kq2$3r8Cs?EZ=iN}F(BY};vBbKvHsa%4x%4oCoDbO5bg^%!{gy`A(n56C(qaCKu7Z$hHGu=-z8K;z%ss@Za4swo#lnW$ zOK|Naxb_lUdkL<+1lL}IYcIjIm*Cnf-+@bupVQXZPpK=a%x?__@MZQEvdD`XW5WKgtm> zr%?NokRfoFAKui@*~J`w0rdS>is@Q?j1&3}99Vfv=6!P816H7cq{o6Bmc|0t6uzI)mPd z>Wch|IE-YX)*{y--=a37XP}eNKT*d}4*+avA2J$QiC`kGW(sySPKf8@#{kzW z65Abn9_zyH!u7-dji&<&e6afu3^L5tqt!&E4wR?wcJD13;oU$PC#-ittI;Ab5X76mkV}0FsSNLQY2(0+(Gm z@b+EC763!Yed0Wln>2=uCocu2=Q*TE(qdw7LM^TaI}6xc0Coi02jxTifFJ$|A{H)% z6~fQMYv35bH77wpG=$j&#RD_SE+`#Sk9mzn;Cm8I5~IjVK+>cuZ58bgbsE(Q+}%S# zswRtB&!n*?ux_zv>}Tv=oX4DaZZk06AKi$;j@gm?Kb zc|CaxxobFK_FtR`?so2c?lvxrH-y*BB?H@aD|;ilktuqK!bfpwr(U!*V8N9kwj zM;W>q-Npr`C8iao2c}CVi!sXh)1Wd~4R-;1f1Q2>K;rh-4b204R z<&p}CR1(!Twrx-wsjXHrMY65+y4ceaZZ2xt)Of6+fBo#*xz%GTlgj7(k^aK}JXOLh zSzIjn-v3+X*CAgDKfnI8@KfSv$CvVN3x6ywsrae;eW0wgd{V{13TH)^%9P3pm9Hx$ zmD0-Q%1xEOD@IkUsc5OFt>{9X`SCvz3t47xpSHG<;0YI?xHScRX z)(0CDOt8@Oa>&KAdY=POs9`?mYFr=QQ| zUmZLil7)-lQvf8ZKbi!6fo@~m*h%=&1P@_0aSZV~VKaU^ZUYX1C*WV>_TqZsUV*I5 z8O$}zO6+)?5w{ADBitlp60Z=sq?4p3QW5~?3;Z8s_?~G1}wnTl58XI*f@=Zjmh$~tJ ztN}@a8GJH`nl=hd;8WugHVgGav(P0RCmJ7dCGvUH!sz zU&N$EZ;T{G@I?=W!-bCp1%gb$F8;sVGB%cd4H!|*GsyJy)L|48NkQm^m*JA|EAX%J z>4YEwMmk8!CjAATj3Dklb~+{!N=Lg;UC^`9ICL^99XS)x4ZbOC5B3RG0{FuWf0{qr zkM{TR#d9wuqWd z*-feS_`o?BuS9IlJ=LC$Vl?8@*Q%P!lMwW`l>_fbj?j*DQW^($+^1Tx;Y?; z)~ZX_f7I6-vP^T#9V`benqy{#{gH6d%3sLKOvL~ z>kQw8C_+tz&Orm9Y={fy%Tlx%eH;>En{WYqA~BIXfm%mQHj9$^Exs!a$p2m zq!R@AUfwnC08UT#V=y~?VgARMNw-sXQ{u?`iMR2iaZ%uF26j6p4?_n=urgEtu>?K@ zRvMZgDEIDjcW@rJ&9TT$NZ^Y)3gF@r;A?6z%r_n|J}{OW_Zy!Bhtn(_aKC6CXclSK zsrRU|RVdY9B}wsFc1K#=7SUEFX_4^S7PMV&+tOAcvA3>k9oO2W)hIqBULig!E)FR&MdN*7_T-v4<}Ma1_&HCs8m#Ya1@?jV`F6;`cdm4O2Wy9K-WR@o z{(t?kz*F?rhxAAIS^gaVYJjHu=CAf&@vHr#gT!z+?1M=V4%A;zZ;S$DxepT6FLm$ff!sGj)!}KYrs1QKZvtP-$*LbO;Q}`BC$8IJ8>0&f}aDONzB)zCe%BPf!<;dl5(Bb+D1J#_%-YavdFV1#ux-=zZ`-a0W>8Hik#T zZo$&w-GBu!7kLi(994~;2u*?_AU8T1m@L1ed!TdC0o(aaNx54 zm5=Ew@$U2X@#;POJVow1?jkqVy~K6OrFE6NHoAHN5Z!avFBi#0ckXq_ft4I@E4IRH zYi-YLA8dW?HyyKG0$?;d3COJoKd?)s!8^mBz(Sz%!3mK4H@%xZ0k_MYXGt&xWu^+Yi`RIes|f-31<`ceeMGx7=&+e)ZbC_k2nI zcRqu6hxfDhuCRY(mXPpF+O|p6jmATIe3sALQjSaffgv zI14TxpG4>ZAn!G#<&^94Y|iq{i#gK)Lh%#p7jrRlA+YI?nSB^f0IJ$WzX0&FcAAU!8o=)zpn?-m?m@ai z*o9k+`HWtK>VQNb2EwhdWVi~R3`TkqJRjZ0fu^#?QiQc>jEpqin2bo7THePi2yH_>_~8KbUk&eJZA3! z-zVQ_-%#%)kJ|m8dyiY_-sH-0-2^MM9qw}ghThL}-1FW;_I?0{-T(ZZ0+_%tfPU)f zC;Hd=blxKG9`9uD9S_63#JR-rmtA3fV<|UZ0tUX<`cmy;&2#`vnyXDP+ainrtwK(b?CN>f7R<0ut8|L2j@* zfDDR4Q^QfP)v#eOdH8X7eYiffIP^Z08}1V>4mAaD2Uhyiea}4e!2EsDrFNYF?-Jab z;(Os&1-1nB!G|FwaOBNLlp-&nr(<4XKjOsr8N_~MK6N>58t_A%19|?p?4g|9oOM7W z@R*gw?8orZhcP-ZzcZPvQ!FIg1^i|yoReH6U%~&vAI;yw=L-yiNYTrP!BKOgmC=8q z`$yl5%82?FnHsqjtYbF`#|sMhk-Q}C1qSAWtHP!C3i{)Srx?j-#i68L@zUkiCYT%znXo#rgyc z!5+>^4wK_xYuO~=zIw>1;V`+qxpCZ7?lJC1?pbbM?k>)Db{T6ei_QAXY+{aQU0@}! zniyK}6wM`HBlgD+#L_Vq^bwR0c>=gB=K>ERBHSTl30?{kg8KrzfXq)1bOyqKDqvo$ z3=R+9fX#;=gg=Jgg_~fbVa2dx@K^8^@ZYfSVRYCL5{9RTqhSN#oe=*ZE`!xq8+<9e z2mmTi2ge5v`=!1~KAi87cO@7(`+J$55?4CNrB^%myFR;R9HfXGo!&E^U+%r`neOwh&CUak zz4jAeq*YrGmNDktrrpLvhSU1fy1UvR8mIa%H9^f&e^r0dJkXxez0hw1AYr#&tsiMH z8J+{vb=dTuIoA?p@d0$%ZEIh9jPsex>z?Ml=$q!}2f7C50GoI$>>^AKTLoW$xPbhF zx`*aLlcCX&3avvcAOx^^Pr#i6D^eJ6@OHqH@E7n~2tA3l#38_|JA&Mgyo{7e^x=Qv z9$<-B1eS{Jh5d{rf>&aU5K{@&LjM5=_A%%L#Dp@?5DJDo3OMW|VCzEM;6p#sH`k+e zy>Z4n3mqDp$FkXc&Xj5DY{~<=g#SUYEjFebCIVmH1pQ`xv2K_4y2hn8s$x|<)g*ui zR;!}bHq~kHDnj{0K2XM&wn|2{9&TxDn%9`s;H}HAMOP~;;>&OU`TBd;?_R${zp{Vx ze>;DAOU0#sK<(=qD1LP>X)Z1;URpe>7*le+eW1>rCj{G^|s_y+iK|tDNfcwc366@?XBcbYo!<~ zKHf68B>}8zpSNrV!k9w^$MwWwqcBMo29Qa0~k`7zA@gv0s~=}!ZrSWU`|*bqy_JL ze>!_Q@4E;1Dg$4_YWPTy@17DI?~{40zTf_ZfnI@MU=A(;^K7~Ab4ZEYjV;2z#i_6q zLL6x#;UMNaDg&&<4x#VkzLN8q9BvtRKHE!OieClUP!2>YoR64~x{oeGv(Qd-7)KyK zCv_rZ5Z)2z5`DPAm?h|?;Jlz5<_Z4sH~PBy3j&*h&hT{P->5tA9YLIbz3;GZi;wKH zdgRU|>l>rSKs2IESZj_$<){Vvh#YI8g=reE-LKdpKcT=VU#U>~t>$~ybjvDToosK* z%=$-FrRDWy5#^%FN8l9bKy6CnU*ho+t7K{0Fewk52sF#oZL_5NWJS`&k}2Y$E!SH{ zig$~tl3Y1eEmq~qKT5c*B`wCL?oIIKb4@V~k89^uU#{3%zN?Z~gR1>dv#)ws)%~jU z>XHi8@6b>4Px3G0ui~;x)!v5Hz*@YbX=Pne<%}}f?*YH&{e1F!Ud7X@qcwf&H#YTb zB};difV1;-*7ktPva zxS98iJ(cyImCk|lxuQN1OCt}(-bj!n^h@}WSdh9eEjI0PDn8ASzB==4#*8$3iaO|OG;(x+l)?~5uGiWpE`Za zitF;W%fDR&U6@@;y3FZ1Hzy`{cF(1KF815e57pPv>tIhrkF4&*uC(kA*@Uhux~$JW z*X4fB{cdgDV!G$&%6c3ucwNx3=b?OCenI}c+{CUoJCEs<-05oO;!YpCoXmUIlhku* zUUj!uT@PoE>Kx9LWok0BGf$-xQ@!z@J7VV2;_U|Z-)xG6Lqn5s_)t-&VnJ}!r)pmCVhP$tOr z7h>*0Q<3{Z=Y17!GqAUJb-njD!X6<%A#cL3gbcoPSBZ52m}9pXr|IK0aOEv|qZ~Y} zvH?<=WQ1g(WOnQK7FbJ7^V%j)y}6cN%K>O%LEYebLj$w%ue$41krk85j{hG2du7?h zis(vARdnr?h7pbX8b36zmHZ={t3WF6EB{r$(oWY3)d!UbWsYKq{E_^NYN6qZHNvj5 zvaFewzl~jWYT!+k>az4sZG>imTBeq0WZFNv>)=pTxVtbJ~5{-Nk*?neBYy#sm0ufd}Wk<9p&u z^c=Ehn-3aK7^A^Xo^LI51-*B?39bg~9aE-Zk#;lqlY45Nnktn_fm8C;#o7w}T|*hb znC@3`lq-~n)U)(SmIk{Q><)6AciaiSTfQ>SOXo*h0}uuMW9{LfxVL(b`!W6!&=jN_ zRJs^|V=dEUn_k$)0QJf<&yc{uP)gvd`@D0Slk1c@{&7C@*g}s`!_Z^|DSS83seYs6lF2VtfoVfm=0Ql{ooC z8Bg9%h19P#A%TwOIKb2Xv>mt4vZ2jH{btQ8%?SN7(-N87A% z>Cv`HZj66_XcJ;P)PUWKS%Xp|Z=o+jIhc32C#0FQ)eJYih{_~u@c9@VbORenbdg4q z|0bOv?8mLcqH!hohlKxtaTipwCTg*(dQF2|sw28n2*RZ%r0ajw(COk&!CXvaf^?AyfG<{}i=j)x$r%y`ZB;8IdN)n{#Qop8`r(a9W zPWC27CW%sZr0?!@F6&ztX14>~zjp7`-I+7J3%=8wRB2LNa%yT)M!(MDEOqBw8T^!0 z3FBgIk>4YxL`&lNiOl5AsmSzM=^s;jBtZ$+JKRjjNVwGDWyjo<=2T_M)}%KH596Am zNTNT27oyHlccKIlvAk=H5tIO73GO6t{qDtXhn6F0u$w_duv0J@%!i7=+Q6tlSzt#{ z8SELn;#==}YddAZSSl?<`)Jos58SiOG0ZyOJjGbA#~J@vwm9E;-vmB{G_biqh{g;d z13G^oFeP*&`~-dny$itnM`JLkXRzkLCGSR0kyq*02d4S)o><2=iw)eSOfn2J95r+| zD0O4CFil8BQuotL0cY4n+Ml`;h8WXr(>bGDm!l5J`^z*^w0xrKjc&5hV|b=}tX`_{ zx1A7UTMAnD$}*It%A2a=8bGVo7Aj}To`LbhDR;<3%46D@Kr)2Tj#JK+W{STwnj2c0 zs>H+Fx=D9SkS%fmM?Y0-s@YY`s6X9M-%wOvTKB5{Pvgtx6!DPO6O!H1kMgC;)j+B= zTs2eqj{*bE&(|wftJZ2Ydb2)4w@EWp<&a(!e`%R5!OE{F(^M8^tLn8{tt^xMkp7a@ z$?=K^1yS{nZi|Uwdaj$P7ASVK#kTfqwYIHPj@PnuQgx{!UY6bVvh|gCMJq|VSY9hn zl%JEmlJOP2RrTs5?LmF5Nd#JM1{d1>!-;oll^2pNgSfLB}a zBPkO4SK3|DWqdw%B5EUSQ+PA{3(5_pK_v)oI5Duwt8xu=&U7kWT|5XNcBDC@97@N( z0L?Pe{mRqJj}HzG5n+?ySonwV>QKj^!hgv>Be)MnMsiVoQ4atlqda^+go8~$72qxr zRD?&wk0cT4KE57v9_oet0{qrJ2!|=BS#LR?Sx4wvvVgD+c-ynESfZ2Gg*}};gRzY= zhWG_X#Xx8p`W2`sKg1M4B4`2h3>pOuf>vRMXoWF5Ka3 zygm9}M2=9)oy~s6=_ag=DvX^Mi;m_;6bnxCmT_)_eLju9QIIZp!t2RB0@wo0Y%`z{ zh`CN7A}TVbDDHBHqWF<9lt{1ej$oA_QN)c}67wQJ2Gm0#OH{;Q5DfWV>1(U$*9z2Nz)R>#KlCl3YPI!^O^-YksqU+5gb7q z8^$i^aNc{|O?S`r z(EJ&p^Dr)aXP6T_=-c6DI~O@Ap7Xwt|775C;AJ2&u-A)rVV%caf4eR2Lf`#RDZD%4 zTR73b%Vo1YHRFv9hUw;Uwkh`e))dob-5~WK#X9L)iMsWSq)Xdc$tLl?&GIH`(}t#g zjj+aE&0kuZ+V)Ai$dY80lIP8a`USNgt4CBv)J|{M)}(AQH;rm4s$X6+q56Iep}t?^ zpr(SRvrVq1zZwVC^{8nBYhy}XQT_GCLE;tCiLwQfuRx(NxURe!Uj4qNv7tn)Z8NmN zB)XQ3&8bZ<8lZ+Ajk@M}ts^B9TU{;b;%%+9tw&qclH0O6MWX7rx=bt3|IsCBN2)u5 z{r4;NXo`WvU4Yt7Xrt)WIYME94z1CXC(TJi4g z{+EIHzzgp{*A)9q>l4dt+aj06rwC?;#s{8zMeZjKy=|p!uKl31%Kf+Jf_t$0glB=T z%x?r^u^GMvy%1A}$pk0p&(O0GHQ|$?Um*_Mi~0}Sf-eU`(Jdr85r@YTc$8B5N9Gvz z7;Yl}1pf}#&x&JtSQTs%_c#An#M0Qg2_ri`jBk#Pjv#<;cO+{mdpYQYmx|5{`|*Ep zkSrSwNJdE+q;BK|r{xLWWHhg=sGGy8AlpQtRl+D zLF!<7AtRZ2n?98Cf;f(NhBTW(pgp12GS{=ctPhL@w5Jp?*-Ji3$)|BxGkHEB`rONb zm>#kWw;Wo6c0eZF8sY$wgs>I+2Zcr)42J_J{g3@|0aPH-4*~hye_+HK?K|T?72Fpd z16LxRA#;$Ka7S>Yuh{j--qp6kKEbdLR3zRWw zewf$cE^$9`pY!bTpAHL!h#;Qs80zx+M?+6jE^nK@{eFVyl^C zRyJoGKTc$iNQj&)`oMqBJ^<_?2FG&Z4#m8QA9I}yX;u%yllT{nQNMEq!}xXWXlKpSr0sj4CevL$7S~t+Z>|@oOaQ) z3v`Y84?r47)vZ-?l_XhNt67X{+a_BkUn@T=e!)Rx`KY%P!sYP%#osCcKop&6nsP)?Scq|0DeYcu$_A9#g4JNFsK$Lh$ku-0;9+YdqREjDMD?P6kvY-NVsjJrB=t$2mRQ|kUmPu_HgaHO zYV?=5oR0h?OX7r%TjC{A34&+r_7v9tgA~^P=Wu8LXZ{ON&z2q{A^-1h`CW1(_Ij1u*`Ax0mmnUl*tj#)TEIzu=AG7XZ0{ z^tHP8y56}4d5~bBX!X{4Or1oJ-!d<= zn%S4QRlL~(vv7uR99Tf<`1!mRP7qwGT;^Cf72J=4HKJ~UC?1dJ0jB8_0PnPenF%5d zyP2n0Q(Sn z`PLobwfP@}9>6*xIwKy!Kft)*jp1eRLGW!b23!rNAS;l&kfq4IsABXQj0Zg(k&bwa zv>{;dRpD3Q8Z0kd5iAV&+nMw2%=vcad^>Z#ojFgBWDpsv=vMlAW*uuJyBFKQzQp;! zUc~y!Y+@XstEoN62MIddUsw_5GzP+@;U^RFh_47^@N}F9(+{%@;>Ejd^LUnp(m+@{EbW{|3kV@d_aH^z>zRk2z-1-3=MM`S_;Ul3or!$=(Q9eAvfUn zV7o#BR0SB>i$SEfYalgnE5Hi|gW>QC#Ann%bO1FI1kOsK&(J4m6$p`?fh3s0xXy$h zBpn4#+Xsx(cCYS#$((nca>sc`fe6So|NQ_47+VJfgn^vE{s1e;3YCYO z;l04DD+1QXVdy95LFgou5rlg^2nTW?%8i~bjv6yJ+7`hlR zcB#PiEJB?_okZoJ(oiAf4sa;Yj6fj@fr)n|q7<G=bFvfb8tGo+IRI9lQcr)E95|kbnnwb6cMo95zG1vayaJne8@ zvCghrrw{9o700W}@f3U7JYPMz z-T}Tezr_DMuqH?j^$&l8pF*vNu48^-cj7kV=Hupo=x8I>0Dfv^SUYYk;Ul3V@eoN! zF#xaR1KPjz6^xIJ4vb~=UG#c-1ic3U<6R_mCA}av5$BV$s4uAmDxdO=bce7FHxb(p zJj-jb_1J&$iKO@B)08HviG~1P*ddGy%xDlcpTn98Ag+~63-btT5c>~%Am~e4UF6L7xa~kbY@4^0XCYuinp0RnV-wkb5;P(btGdkJq7G`is+k| z4eVdQXIsj5@U!^exg9x&S+PtheH~3sxk|o6`iFRo&;la3N%$h%XY4l266hTogRVpN zM$q98fysV8EDM$jONM0tXEjKjz^L#;h(2J&n~#WtPYn+XPW4xMXD5+EgLb>lsOzggpx*!-u`&Ab;KJpmkz;AG((HU^p}Wyj;$`~_K`5D=S!Z?CoXvy_-dm{?{%2s6U%J)F~AOI;$@e5ca> z*gDZX(Rf?GLbn10vcDR<#_MLW<*~(SK4;!w-f#Y9E;Lt|Dota}|CxJRCRpojR~%U` zqWg$@i)X7h7wmmL`Hux21%LmKn;NzN_AUGfMDm(pnXm`2Q3ww5BVq*@$?hV`5kiCr zo(00MWrz_-3~D9n3~B^QjaUZD4gCx}198@aKE1D7KoV4g_#`Ks6M7rq`}4d#-S3?L zIFjsLtO#?9fvLZ$?W-B9&QZ@*C#nyto-5BNM#+(~CvEE_BU|^2rQ$qEUfZKKqf{%O zu1rzE)h6{*jYX5AiBo@6JyxZtJ}3>!wW{svDD6cZ%1~gcw`{R>vhT4kvXen5`k>8c zRah_E9@-B$Bo3z2;v~4U0T}kHYp8ph8*&p}@y;A)fop@i-t*1J4^9X(;aYevgcMPM zynte%=b&BSsCydDg6~6oPEwJjlqxEYcAL70Hi5AYln-K=vzTX?aR7sPjoM84Lzzr5 zlanY#ltok+%|=;CIt60q&v4hkSi#2~!XGBalKGTbR1ZzSSkK4>b%pK>D?`H=%78Jb zjM*RtTF%U7QP^mXkH7UG5zC?RTDmdz3d!kR|Lbq>6e+jE&eJ z$`G#MNAaF;)3{SO0uYLes>EXgCC+&4@h2JH#qPGW@@ALC76= z?LY3dyKXrDbuI)@;ta5Zm)OnrpSGoz4IrS5Ft`o3jnb4K%n=Kor*hzGVFmpqkjk@r*FS4>sz1xLF?tyKG3XVKRiQ!O&9 z)^^C@c13tgd>aG1Ljw2*qzUyLWdLRe6gYIB3?l67F3 z1VJ~@H_;!VTi68v4|{>=B|auO$TgH9)Csib^kt0wj2K2X7&|x8F48v8j)2)^1N}I? zg|?ZRKyiW-pAGn{*a!?A+KTRqwxP07|02||p5fxaPj4?zH_sh6%k|yCaNM)!*b{BP zK}cm5aCw(m3N1G*bn6)FC`*RvjUmG@Lf=KF)$Y`N)b}tNj8BcvjY+0W0CwKZ9Bp=+ z=9yoEGmD2Nk4XeZ#oPAr&OsooTI7@aF8hZ2y7{I6BXNuOm-mhDM&NVkb@)yAZ!kuz zK>iC(Ev`aboC@EEZ~;`OZs3W)?0g=Rj=6yeU|Vs;xQY1x2oDLraVIgS(f5%Y#6@5t z{uWvq#=sD;E#YUOF`=U&R_ILNjQ_BosmiVVOF2{7qO4R7R%R)WDNiY*l!c0(iX+N5>PmoxS82{_U0S5htsSF_H&mN? zScY4|mPPw3(7}I8Fe{zHu*i#PSE17gSqiK_6M$nP(ad? z?vW2uR!~pS4$!n5t;Fh}d{Fcq1TgV)53j~S6dBT^1;ev~TCc$ID zIzfMdNN}F-;|^p`WmHq2kW)wr#7%^$gk6Ns_%`eh@b~$Iyn)ycp9Gr`Hiptbfd>Yg z8V-PP^B4OQ1$fc~jMt(bT2!^6M$D8eiUiwW486Q+o5et=Vneb!+tRbm%&` z#`&g(&efJlwyln0_j2!Lzdz6rd=jb+?}!YFCSpOxUiJ&lQSLRKg)(JUO+$x3nP54;4{G)!8zf2@jhg}bRsP7#$hG0YFKhc@m9j8Xe9q7+Z2St zN5~b3qOan*;xIW^xlnnO+^N`0^v8>3C!_<=nMek*5Gg}8N`4V#2!{!3dAm7}Sy?PS zb1MUj4~euuf06EMq<0o-~R-ViDZ9tw;MYzcG(l)<^dQ1D>L2nNQJLnC3velxr`vLULD z_h4*h{tQVYHH_noqtI6*F}JYJbNcYsLrw3_&*R5=M?j{2lgHqH%}eI8In@BSm$GUY z!o>5~M=^e^E;=MOI!-}XF(7_7S_S8k3z3otD{?k030s4CfvLVckISugf8?6wT;bSn z-(o|ogxO-07{1Z>(^2g|wl%aZZBNo|(S5EnwWGSI?sxt84%C=terlE57ugrU-vN7} z{g&;PEn=VNOaW{8tFE2ybF_&*;l^AmoO#X^XRcG}>gs0FUFp$owkyk-<1{!AxR~w@ z?rC&8eUV;6Q=UxkYG0FoZE${gW%R@N7YPZ-N0pGhp3nLeGF@(QxAUh7=ZFrAPKe%! zW{5SCWaJaH3zjRt1K|%Cae)}9Sf+3)T0x+3n5w3xQ(9^!MNtRIxx_5_9qfv9qZG%q zvT5@7@f2JwcVdOuWssknWo7b7_%PyEVk@B~BKSgLkiwzZN%mJZDATA0rJg*d*h5s~ zM)@51&+;kwUwAWK4)x*<%;D@(?pORe!1Jboz4%IA1@A83CKv?>oL#U&P{#k3x0)B_ z1>oI26=h0xfM{+ZItrDemys+)4dNLVLLs%t6ZBK*GU)}W2pfzQV=~zw`JcE-QKooJ z>>!d9PZdREk@7ZmCy7!$OZqB_ry7^+Os+_t4;3LBfZ*cf@72wkFH^jlKh#;t$*Mz8 zLu4unmirTv(x^e?Ktd<)Bfl!s$R=ZRrC*}c5rO1OQKeuApW;cladtPhg*Ap_P4}Zv)V4T zpViILFX}jAxBwFM<>rKCw=L5laQWP~J%q2=j|B&WOriWpYb=BD19Jr{!m5N0qYE#E ze?wpul}cVn{*;(u=hFv0hcF}+qJ*$O*dTaMI6;&pULfuSOr)1+m1vWAHj)Ys3cE3h zj3JwbHAyAdQ|RRe%0HLimCwT;;QMipoG%|PyN5}zN9b=*C$0lhl8#&z|1K=yw{X%} zlM;<^Kb{6gqK$zyzNKE1hwm9r_oBzpo$eaIjoTbv$8N`GcDZeWwTJbgHOIckIgCE# zW%`HsulS7Ko?hBB-D~#xym=t@eaFM~)YC8MZ#^1c*uOEjCp0^p8u=~aj68@ej*N-y zjHH5G_&{t~{A5B0Sm1OMiu^F*uoVIa!=f*qd3) z0n$9m&SUpu5zKtXf<#GtQG8J%o1tQ)C!}#}^e+IgZ-+O9kB6oNk-%P`-}8W$(Z%kw zu39Gt`in2@>9$9f8D_J|V9Ymm7pP)LV|&qmB&# zR=;-Kb#^#Ea1C{5(mQC37P_anCb)zyo>S_$YrkoKZ2!t3b*0i~&wX#HccdqszT&P2 zWN$ZJPJ7+M+}B;pTqSTun+Zk<)gF#-7)&+R2iJr~hg-u-BYmS?V{77hi~`mS&M2Nx zU==)uZhf0@k|-`ZEa6HIVgF)(!F{$~HdmgB?;-|~JC#2xe*>w@aI%rSM($DQAw=P# z%!6fP`Pf?7A94+TOa7UBto%p$?|8A|7RiK)&`O>qbI1a6g0hh+SDBOj>LZ#4O`T=| zh&F;MuBs@>K%Ij4nR(P{DoEj!OF4HYd5rUGXXLMR9+zNz^9# zU6d=@1ZSUNf~jEc!Q-#wcIB*v@1~BE!Y>gv!EbW4c$a9T@RXokuwB?&v{kf3lqzC_ zea1b}KJmAbb4VpxDt#&y!sO+M)Qpy(dFW=e25o`)1s4m!8LU9ogwY^67XSvT!`fsA z;N8z5HWHT=Y08;YW76g10~$Q_V(PloO{rS#ciN%a?y0pJRFkGzr}1j^DLu9Kv^;H2 zN{L#L{JE;1YP3q1^gX2`(-iaYDyS0W(oE?GfRbC#31}|j5^ocIBup2O0A|17K7+m~ zjrR`k3os-($w_0+W^Dnyt(eh|8DZwK{$ieC7!vmrA1Cqw>Fo^ycxH4D+?#3x{J=wB zHT1HFAeNxW9L7LZ+xE7NX`A}i(xPh4X~G*%G~8^M-iS9{ZCdh% zY@Pt9V`X!8%b3=Rx5L{%(X)(?I@enZ9NF$29=G>vu-CBp^!~cQg> z;n*qcF7_CEj#XmYur}UZ4~a735%Hm-mtr2V5-*h(%Fba=q>IorWEAv%=R~gr zHQZK~3g$IqV`fEian z=7dG>0zB9&dOVsP8yg=7Thzbe&tv}RsHh?GI)X;kv77Ot#I<FiY^nKTuQ&|Dl zGWJyH5i?k+tPrz|`7cbK&L*xTG8vVOe*jTj6JHnI9$p-5@^AHj2S$N6%Chv~cY zeMnvF={)H`Aj+zbtG{!wquD;$J{d6oeSrQyv~9MJcC2$|xKGjldSc#5{%e7;03x+T zd4QpovlnuE^9JyC@J8|t{L2EH@VIy@%sc;-o|eX?#jQu2OPPSQN(VZ|llW8x-037>)MxAv1{bF3Am9&YK5}QPW zZj%;ZENmvk8_baQm(G!L;Of`OPRJ|eL*PhNd z8h$FF?Wb)@IjLTu;!&d%tMNWKEjPZ0tbdAKq4PiyZ7#SY@1}qRt!WCgbgovC8_Y9YW zjDZaQId6o{bEB?_jwWj=z!XctfMo*w*47%{=>B$-|AH{zHHV;EMW|&4V0O?_Fd{gXvY)(8LuZ=&B^@!4;gMo{_ z3EpKMxkpXcIhWg$Y?!qjLQ=MN{MhBbyI#uKKUfE?|yT(#6$ezKZuNshM;r{j@B<;-&a;7~b=9Ty#b z`x;x4^#tfpShgIni}G5^%(OE@w10L~?DA%8gF8>t{=kHYC>gQ%>Df{V_+LFJ)A@jO2ya)SsFf(_#c4~LvTLqT906U?2arTci^HFb$}oz-nXw~DT!uJv7XUEE!scPZ#{MVq7@ntEC@5KNbh zaMrI)#?{x7Q`ynq~%x~vtlW-Zt@J`YtpM+ zPL)x8sCuYdHRO77gpvs}$G3`Vf8ctgsA~Ylzlkryvv46^Bl}r82)zbZ?-WFh zUPpJKp<~g;SaPf(bjw@jR9T02zC=^tIC2B+g`7rE=>w?&Uh<8sAC3_wVvpjuVlbIb zjZd0R{i%FJ^-lVT(om&T9gH#^ig$^@a#TJ7R}nM7ZK4G)lh4BjqBWAal0_1x#351& z+jv@TCg&A90v>04Sce2Tk2#-mg6t*iBG!FoHmnW*hR0>rb!Il>eeegl9LtKn39k-Y z!p89Euqa#-6n8}3AAqLWM}E_E+0#VKA~i@UpPk)j9JgS)#s9NdaK6e*>yad(?Za`*is z?ppVUk6Br)uB4Fc%zoeJePj>p*{$1}sT62f$Dv)P&RGG)hMT42YjIeBnn+8wHy(wz1WT!k!Us2SZ^ z^VunEKWi@Q53>*Z4rf01Ft3W&mb-zIk1pe%;b-%9@>m#$Ux4o6E#-gYwLrI{{rKZ} z2JUy>BJ3>o2X87(RHa0(l&9i(y!&VybSf(4=VL~`fIkRaw|JX*N&Mm1Cc#h4hK}Y= zMRU=koC%yMoD{Z`d6m(MafNx4)tv3*JVtNw2J&*a3)p=)MSPFQEuJBsAnl}h5tScn zjDDx`sK%-)6}MzHVzzj&2od^(Y*8-WUwBa*mflhZRb3SVNrm7(7K`^5Y~;K7moONk z@t5%pf=tm^u}ZpC(I<+poGz&s%@&TwR$yxcNBJ|*j+|Z0Tg-jjXM&$1udo|lg6|dn zi`QbacudY&))ls!C1H){#Bx#?y%|f{``ES2dgK%>F4BRp1v`?(1Vj!bHhBwNQ@n6s zXdu(S$lKU6&NbWF%nf_aIzLmc6vvlpX zA9W1FK;1>{9sMoiGV?miF;hpwTYaUjShH4rK<&}2(l0XZ(C<_yX^gtQ+H0Cx!)CMI z)Xr37d~1AZ6q*m)CfILVwpdo#itRsLPkc=RYXT1mJbaSu9ehn(4Hc1Ji7{k-_&wz~ zbuH~2d<5CXSjHk)jhGMNgRl-!(OX9@hWAEZhb55?&^qWjWgvYr{F+`w?@a$iiK8x| zFQTH6L*b_3^zgaJmdNGs)rgAv4}A`OE!7E)4R0rw2i5*JzR`iv!7G8|{v}?fr>S>- z;A&`1_(ibZ_s#R(kzs3TZEN0PYHWUJyr6TcyVSp_Pu0COHM3r@6q&y1H){rJJi0-~ zUB+j+aDBh}H=6hQGUEz!dz;@8c1*Hk_Ct<)u0!tr?zW!6{_DhAawT}|2=5OUMCMSd zDc(re$n-EoS&E!RYH7vv7RVfAI%5Ju&d6h2K!zf{k$LnH^fcr)91Guo`_TWRR#Q*X zuD~l;HtuuuE$1OCm3a$ZPh(QnkgWpS1LKI<OW%PKR|_}tLPG};_ziqUiQS*AXw4Z4=<_jT**lyzvWxRzbFs;+x|n&zW! zouR9~Sc7So>lSI_)g|>0G)r~!b!zP!!!w)E)xza}(UW^Thl2Hg>p+2PirM-ez!cFOm=}j48b~@)VM~*h&vjql>!F$OL zv2wUf{ttcvzcFt=3iJO667iY*U1&6qM4NJK%s#Am_FDFL_CfYvR!8PCuvR&YA$MoZC|oWsnoh!7q{e@bV;4Pgpwpr3-bGd8l7+~;f$Ga9^?hm0ev?(EyFB=#y! zkR##F;BvY3oO13q?f~{YW&&dl!eZQIbYtS|F5FY72Ss^X_$omQY!3g6V4%n?MDSf$ zGByC?U@qP&v^i%7qdg2Uvv|e$ec?4D#q70+Y~Afn#}?;$=PTz#=WXYA_bM;yOZImS zEDwC~)BHL9Hi17ub?6(^oVt@Hfu+b=q#2wrT`tsykGkE_amYR*tGFt5J#IW?Y-bv1QQOj74Sdf7aF7-l>$~ph=FW6Yah5q) z&JM0Y?(v>IzTSZq;8_+imUvEnjtHq9YAr1lZitj2E5VvhWhHRBpf7j|ejNV`pTNEd z+=5EMJ3$XTM#vTB;SPMMs94-z(otfT9GAY3%oM*7l?k`uI;hR3~9bq9mUrR>?7<_ zRuuCgav$DL-$C0>odNE?O;8#{hbBY4Ax~r%R07S6bP0bCw~w3-KMK7hqsdO>Z1NnT z3SRQd{R{km{LH}Iz~;aTe-CeK*K1prWs_;I;Wp?C{WZJm`SmC3De7@*Lfu|-LA{~= zdfmplUv-YUfpu$Y57j2rb*=N&JyjpkChAujHXD1I9-3~NTAL;rHychHD5g5|KI;Oz z*%@>n^E~%-^gi?M_az1f1dG63e~n~>?vvd~6`4(LCB-3exG+*q=}7xTi=v;W=fZd3 zo^U$dLLE=_Q{A*4bRT^z<2$PrdkO0=b11V5^DQ$U?DjZhH~fL#h>p=6;P(#G_tCTI z4e4RpI$AcZoJyxIp}e6ipnjq*pw?2}Q5I2z(2MY^aLY(-m=6vWCRS+B$VFJ(nyamKm07L4j&HR3U`V0g|<)->c5lCIgtPDrT#8@ESBOeLC4m&56mCT` z#XDK5bcZ-WREy^e>iMU6m(i2l-JA+`K0BR#o%M>j2H^i^Fas{8b)w>wo6s3(8q^HR zgc?ANpzh$224R%#)Xp>~wLLYFGA5E68bkaH2m+PqnyYTa6aj?iAw#%Q-_o~X~(Q|dR?k5w;HXRDX1OVti_rbeTw)|zw+J=Pgoug+t&{c$WA|{T~AoVhQ;()GESWp`B-P!8JwT&w(PO&uk1hU zaqQKsqs%r;88gCI!+ggQaJ-yO+|!)i>^f#1m~Fb!FHpNs3Ly=&9jc7me_t+e_99I9@skBhdKT_Pq~|UxxSsgRsPk1 zHbGDDC9#{l75W;!6v>6ALvJIuBIlrTs4;W{8bFoO3aIlzCA&f`q^hVUU~C z8iT%oevsaeuA|O{+Jrxo{}DC8e}krAE3!JI3%7|#BiZ4nq=IN0=-_+iIq6P!wQx{v z6=sp?w4sZBur6CSMVF@Ap}nU$s2)&HtGiQ^U){fIYh`U^+v>J8w`wf4hWc@u3@t@x z)!o(G_1XGFU4iztHbeVDW7aIzuGPgFjv49Z9P3~EDrXbdCf7_C&sFFA*BNxE9VeVO zUEAE%Zjs0C;rN>So_ODQ`}o%Q5Fgi@p2;9i@xm z$*>om!5GW3vpaL|p;}%o{};v*T*77wh6(dU14Id;iK2s|MA5(a8LXWDjX#oauN9E{wE^WpBq!`69}{~5F7G+d zMb8Y+6L%wbiL2UWbA5Ksu`jdsurSPLj4KQ;^hNqLhP#HQhNt=g`dhje+FEs^y2P4Z z)wxyAE7q4cD!)U3N~t>LlJZu)DfGDVGFzzFcr($)7dmmOKvjv6}tiJJ>xurAm`xo zZ~=0GIi3BIbCet4-bUTLB7SddnBX2hSGZLeFKiBK=W@X@!7{;4&|Q|{d+-XuN-Tx% zL8o(WQ=U4j)`$Wew z$9MY@`#C%A7~~jWZ)AO9Ze;FfYGE`O)*A~x&QbO#_Bl3}eS>*~k;OR2AemLH53J$r2b`OnudIEH1MoGPlzI$m5qTS) z6JY@EXjSCD@SyOnusD2>JV@*yP@;X{g!iK>&+*>Y-&$)9n9iCem^?<7@smEJd#_ue z`=s@0Mr$T$Dl|Vey)?}i zd&GX+$#UNS_lMj?a|zse?n2is*E{zzuf+%Z!`>6#roJw|f4zt|;86jLbHa1ey};eZ zv&GxjzbB|C`$SGqVrXw@-RXMzTsQ<@Kyn$qnH8)G&QDaq-^q`~r~)H)lOM&e<~y;C zf)RpE7z1Nr)A@0{ooECt;oSw#Oyd{uzhm>TNxbJ=7stRp3wq;O=6lvhP7Z29ukv>A z=VN;X+wmk}qG*L!C(aVL7WETe#oyp9g;RuAg_A{eF-JT_)JZrHpa}_nT<{0$i|xi{ z3*HDsU?2SebJAaQICnKi!CioEMY-G^?4zs&Oe4}B4$*qj@55|F0#L$Aq&edt(hG*^ zr)VjllKW`q=sS=tpi9-T+OvkUp0dAliaAcsCGfAf&|1K>ZRM78kD@WWvAiX`vw$}n z$y0MTbK*GdI5RjWIS5zJoyU`4Ed@ISYXqc#0QhQ^@SSLgbeud<(MvH!QJ_duzEc?F znTo5*Dph6_E9#X>t?CiAPIX+dTYg`@Q_hfQ${~ePnWmbmdZ_HHJf56KweZ|n{)15M$s;EdsH;JjkjFyDjE^OAOsx|K2l8XvKQ8%9ba5Hu!2h9^cg zN9w|T!`{p1x5&y)VyK<`H-{c``k%TvSKCEzf$@a>_i! zw9h!hc;8^sP0~E5yIixQdLF0|1IzE0rIp2%-uXMQxb#m!@sg6RWvLYnt2);t)vc`0 zRM)9t?NnWfeuc50d6RXPW0Cug*Xt|se+m2uZX>eENXQTErWMhT!VeH5;~7AcZ=EH&RNP4XSbRt_Q946@-$8<_}~i z{WCR(T2I*oVc|AJCBPS{o>FH6$9ek;`!jo{eVDD24YKp>R%?~zrp0cVW6ieZ*zY>_ zxp5|#l}(IWhcybI=$Ys4IaN?ZtCfQ}a$KFRmRCpCJ*@Ag=4lfRUro4W zwq-iF_FE=dUYK8)3Cm<#w0(^IzJ0snic{x2>A>tuY=dmkw%#_WJ>Efa#yh(>X!aai zPn*WJ-%fF?aSV0VI;i>;H`|I ztO@Kx?33(r_6SZCSBVbc#bLeimqJK{iLQxSiWiD);wnj%^pNbCY?v%oHcBRw9gs{E zjlo}H_xNl1LM&D=6mKew7OHR?HVM0fO%@mg3-ErzcA{INO`=#4A)Fz~7F)&JCC#NJ z(nMK_v|e&dyi4>nWoUX@%T{ z5qLiRGCd#u0(XY5(x=hq(!0`I(FRj2ksp!O&@M`Q+I)B!;}=uO8p>?K$VVU2SMxj(xhWXNR^{hDh`4gB2dT_zvO-8iLz`-OHm!3h2IxM2?k&j`7?RLP$qXK`!=%; zLxynSFfD;*q;{pfrU~f+dRN+Kuq*75VUd^N!4W%DKoL{gL+Z$!$e^%}{7$SO#uH0| zEIn^b?T+*FZW zzPM~p*~0RI3U6hoDyQZ~?d7@&^|RGSG}p9x-6BK8xYOLq%CvhN1ul_imKXE42FbDgb{~CGoCO_?D^ax?sKkzyBE#kT|pu4MRtTWo>jtZ!pddmaHezg z>=En+?4|5=oUv#OpMmYfxB^Hp6f5I%v9AEjwZzBaukgvj55j#y7tX-j3*KN{>=W-l zP$wRvg=i{zAMnF-7?t!yYQIP+=*L5e;lWbhN>83!=VG{eIU6{;I`xj%_L;UCtJ}KM zI?{rhb4|&nkEWiMUAA`4vu=vFjrX2M>rQt&UBlc4x5qumy~~y9f?dCzMb5phC{NJ4 z*1sz-HW)+vN7#rD#9X2)v7Lw^i^%<a81aTThtbGx=pf}HWe4Rb z_ zy&IqnjdqKcrdzMCHy*K&)-_h9^{@4nO>JvtZ|i8~+~&lbEXP25uAOB^Y<%l0%Tvo| z%Q7q99^*8)-n-|!JGxSx_Z+_+O`SUdzw36ib3C>$w9f%E+9<~Y=Q9`G(+;p2tNkkj zU4m7?Jfaa9A6gk61dXSTpm#!8OcV1nKQRe<++Kej%ZNiclrF zEa@p3FUggb%GN6kO0VijR7upYsOeF|RDK1lm?qcA&dRpQCd%i>OjyagA026XP98X^bzMEh4XvQ2? z8Mr4`aCWdevd%JVnOj*I>`m->>=-tUox^^^-ox3;y@KB6&F6pR0|gE{!Z+~d@CNZV z@LuyO0e=Bw0WgD&7nEU6{zg6%JA&0=HG=)%(@zks72OdhN=L~`<#&|(ql7WnWB!R* z7ZV$MIJRwUQp}GiUQ}Y#KT+ zfMe$D;oM|DW36VTu@gbx84ZwTFTRCWg8t^d;Z~yk`R#$e`3n1t{lso#o3R?~r=T8m zrS75yqEDjk;s)Z$qI}_ee3IY__Ju#2mxy)(*zXFvnE4v6qACDi(=C)p4k71|6Nz2{ zlee9xhkFpH79SnU9Tj%3?Gc#69+*=tJuTj~ZeI8V&p607tw$`L3R#x3-^dD z2b09lkQZ>w8saa(BUPcN;c1Zv;f(N;a4Eoj4=97FKI%f+V1N(4WdY?oV4f~V z?nPQbx1n#46j~Ynhuj*R>o48>$o zzzp>|y|%YDs7cno)E?Dc*B2X}8RJbajav_6%4Ss{fB(>bK>aBgPr51tJXnkT^ zVPEg)x)gFyRv?-*-Ox1F(~wva8$-rUyO zT45Pr8DKtQ+GApxKbkjKx>*0Qr8&+zzq>;2e%=SZZUJYohD0K*DC=o6;0&Y%vJL6T z&@k>W&8!ogO@Qxw&OgTw^JfW$3crZ%in~h=NRCO^5{GD{=)I_c_=hM%v`*MuSSsu- zej|A=wMl17snSalo@9hXCM%RLRXkMGDZVT2Db6SkDjbR*%44dSsLN4V(Ve6JL}f(v zRJBqrQ1n(TRUA?jD%vU6Df=n+DSFB$OAm=}i29433i}C12nDb5%EMQSP~QyTB1F13vNmNIrPKdT{lhMr@Volg zv0uJlRlglYt%}$GJzrufomzga;!UNZ`ex1Ix+CgjU4@~wnPo|! z6U+uo`dxGdCy{xU{)TcQGB#Wk>JfTMHXwfo|Me4|Gp;1(T}RNq!FB^6if5MT=KH3@ zriG>`(+J}{unMrLgC);~JN`Jt&K6FKBhPWx@yuD|n(zMLu5hCsrzgYL!@oB0EVz_t zMm7vR2{jId(mT8^bTu>~91i~oe-0&+&cFenW#+mU zx^S1u`OCJ*A~$2ES_93nNxxXnG2Az(4GG3a#(?pb5i+(loY6&5tv2f?tHPFJJ7rZ^XIZaXy_S_` zx#^@a+e9_@G(%>$ae<+?o~Mh^sMJ!RwI9>C)SrN^vRT8@WrP1`wDGISWcFJmw&wOR zpt`(vrFqQWv4OFmR{Vx~07t+ipn1)shiEP6H{eN(_DmZyhE3s+U^-dDj_3RbRFJ;> z!~7tB8g>-hf~8^*wg#&Qljs3H2lz=6g?&Z;icnD-Ap>ZwM=&q;QsBd_!j2-FC|5E` znkpLvYR6FJUX?g{K+LLGT^tcVCZR>b#dvo7@3^`+N8Hc2@p1QJ<*{vJ&PBCU)hV@L z*8iatsvam~lqVGPfyd~7fBv@u|676ot-$|Q;D0Od|MLpu`|JFifWd#kpX=xQ3;bt* zHk1=e2@i+5P$vQ16rydS--SCPIS3c=1C72fEr+riS`9@ax5A@C4svv8Z^%v}7~)0$YBYZ<#I_t;V;e=cXJ}Pg9Ypo4JYkkx67uw=S{^9lP!0>|O2K0QbSO z#kRv{wR zilzgGa}Qvze~U+oFkw&pjzEJQ#1;q&@QK27;c1|Fe#0|`xbQ8$A0L932|5aP0_3p{ zYcDt_cp`8J9^h%Bx8g6715$@xd5Inz$dpBygPjj@uP5kHfi*xu3Z;=rW!SjYBigQ|J=(1-B#jKQ5i81T)4# zz5!c?w*WJdP#D3h@T>S%{333_R|(gPa>W91o@l?Qy?B@SnwSvhO72MJOZx$a#;-|-riCYk7j9nUYA82CO8DvGgojC+sW?)e9}eS-rND4Bz6p|1yjU$1lwtc zD2pRI!&zZCR8EW|%R(E&+-+Cr^g*A-}TtBfa1sg@?z2i6g`Onb3io@BzTbccbL*{(sXzrjoC=P?Y)?RP#;@Id+adSOeJdeGT z{d)q*faQB1Od#$NW6A8$-0-!?dZ-K_dmI`a*#*3hT=GU4omv6V%2v|9{L+NN> zKWyz{nP%x{9dFfIhFcHXdf8vv##;Tr#k|~jPA}En(dOy2`o+dQreWsJ7N_N#MP^xR zZe{LgK4~sDCs-_&Dr-}l#k#~AV~w^Bu}-v}vKHD7I^tb@-9tUYyeoY>{YZcvNC^xF zKF7a-O~LMjfDDkYLjKUaPzBjGloNgzt`C2WOrY$gifQX<4d{0IIJgO1Lw`$OL$9V~ z&@NH`p}qnRi~yw)Y76}rc^n=d`a>)VP6~|jXZU=c@9w97U7h4iaNM@BY!PcEcn@c- z9_wP;Z<`I^|De6V(a5>o`4#Z55$9CbCD%e%v6FOUITqMQ*aX%-mfn`lma~>K7MrEm zT48HxPqrVmuLT~>mkz6)Vo$ZTvWCq?CWYyZ@uu;cvC{a)_}kda^v3ksgqdm#`}9L} zO07h5PmO8HG#V{apK0i4Y-CzzI%*1=Zkdl+T-Iy$=}yeO*0a#t+jrGJAxI-{h9*af zpm^#PT0eL%(v-2C;b(MUzGZwx79w6`7efiug@4#5ISg(gXEKLiFJkLiZ&*`-lDrCYh_%elDC;zTEtn@20RjXA3z-{J7Ux|Jl zZHs;pjYn^dnh>RkI;)zbS_+=+t}=kxltWDSyKR?b8`z7{QtSZhJ2PG6IY)h0RjZPYx)G=vH(vu`{a_i)Q$t#m@Cf6l% zQc_Y{r1VRfn6e;cdrE!EoK!sRd73|MLz*%5Xlh>S=+q^tCsKo{H`3aqH%u=~`zI|e zZCk22Wp7G{lt;;e2iM7>?{8* zwMp#aC!$}%9>Q0+UQmF|25R#wF3!ndZ3Dh!Bm5Q~15ctaq-D~=)K0)b=7kEN)4+Z8 z3W}kaDG#Z?sV%63DN3ki7$K8_tNaDtDV|5}Htt*(yo`dHNM&zV1gKh28U7ynZ%`a9|GAn+*pJEG)KiMaGjX=?dn zpp^ux8r96K9Z|=rUt3>QzfrwPb4)AI7a0;vdea^AHjBaxRhAKk~!bM?A_-438cx&i7c^oJoDS`3+hrTP`%buy8d{0BqYj>Hux96uP*L%ea zN}qSOcN<^ymUX#poZV#C+t=E0dzH;&8)`S(w}A?F)w$HQ z%$@9M>It|@0ef-ReHr*UQSTz3Fwli42^B?>Df6hiX>8yvEuh(Gi|NC2k$dC6?jbV+s(ErTb*s6ZJ}+aZMcnMdt%*ht+y@(yyJQMRL5P% zB*$|*VxJA3S#I5C*=_cl2Aj?q*+!dwxz4TS>gH=-X<%)o_MEPt{~1=2 zJ_u^hOv`agouvn;`R8p9?JJ!_-5Sp*-%WpEz#XKMNT?&oZG3}9Q%BO90G80gY|Mr@ zn>l^ByU^DBdW;bKz;RKIc&^wj+8`PuS|r*dS|xfV5=%zNNX6W!UNP@uN5(C}4eUS{u1N#>D^@&@JUfwW6$L(>>(M4Bk$UFNoiuN&@eC}^0Q zsmdrxFHcjZZBK2G%1UjRx+7JU_9D%jwl7VRHZ!$b%E07?$pezRBzI5VmrP6Pn9@IG zLW(b0o5W07ka#-568~>}^Y~@)-4kvnyiRZ@=o6MEC=!wrHY7Yq7@Tk zqPIoYMb)X61Dx?rc|n<=x~CcuC5#qDQ=%%Aos`QJ-QPu^)?0=Exkn#QJ4WdXjfWB_cYu3&H03AoWDbo~hBlDzi4(*n;vDf9 zaPD`3=HG^R5h(IuUbX9lgKg88P8n)+|7rdKIgmNk8>@5c8)#POo>>mLdk|YA8Dzk_ z$5#+g67R{9;1MtGZeR=PX&~>jR^7e6rY@^0tMpT`@%OYJbH0uJY0A$l0yULX#O}1$DZ+myf*+%(fAqs{k)54G}@5sW^ZG^V^3l4VU1>mSX($r zymwf%aFFDXge;ikLa zx@q2AZ>8rVP%?V@t-g}LN^)MPQOHle2qi^K;r_stzMTvbYN91k5SSg@K^n*d`)Is;d)|^vv7%P>Usu@!~u4;Vcr;3ji3o7STr`Bz* z|5~qA*J~cASJ$1Yb=6kZGHb`x{Hk7F^{t{`g}35(<=e__l}{>KmG3X*lzPiDs{hpE z`mg3R=WgFo@)B^ic;OQaFJmR}a8Thhv>{Xn)Ftwalm%l0GXe{Pf{+mUU!VD3pZQ;( z`Cp&;U!VD3pZR~;X9AS3v=n#&eH3jXji5fFsG)O^kkXVIPm|M2;m(W;jJ8Za)53hh zT*3SfGQnm_V<;p157Eru!;|XvyUV>R{OtnVAVy>mJAx|$t^A7vKf%2EnCu%uLW9VY zkxo=+WMAYYRZgSR_b>;uM!`E^AiS}LflNmNa}q0w(U8@Sw*x)K+`wGJA1m1)t`JBi zNva`ohU{-FCwXFg--MaznVEeXTyJnQ>q}Evb9ysP#j1d^F3Ou_{yCaC!zavuES)O$RpRHvQRjNb}1rem5J~XjR6rRDWDRiOM(1Ow#M( zmcpkXm(-0{#p1Cu(Mt3Zmx8W9pK?2)zj%*%B-hKDhE%{8nElx8*d|U_{uIGB!AU`y zFj;&+JX@p`w!}XQT=*afP5wY;23~(i@m$#_dPPi|7#&=-*KoWl`=NjW0?J0CPax#1e zaHG>m8X*Y$<0rhgUCkYPfS!HJp#vWGGxqObj*;3QT9xKlL%wEbZTqTrW>rO?ti6w+2$qs za_Xy7ZyvrM^?AUzvch&hD+-r^l)nD<&QI%c>fzH7lYVH;$lvS?>0k179A_FN8vJkdN95 zy{6ouzNfyV=2Op8Nm@2NNX2PgfQoAecXbe1$#j5(Q8KqJcRxFkITJ3XqLkXmvdHI1 z`$(toitx&Cvrs;{jqFRx$Qq(EkscKI-+Csw&pBt>$Jw@8S6TjA##k3xI$7RXVf%7> zrTrhrC+9?WqGyojx@WCtpF8Y|aZhj;x<7){E*zXlv?tyYpUBZavnGgjdiP;!r>hf9Vm~G%XN;cFl z+?F`$7yJ5n2YD9(2hLvaAAfXkw?Eozb)C0QH}}$?P(P~uS~a)4zNFyK{GUUAqyc7O z*3S<=)jwUodKU$Mxqhzu9VkMJbN&t}-CFgizNsco_sYOGbkgtCKh|`sL+hl#cX?U! z82J0Vz`>KE83}arhT6TlJLc!EPQDF3G=K*O`_{W%?nGZWaGty#S|099G16Z$nzCiw zZrp1i)o_zp&Hl~V$k~rV7=!-@-N64O=!Vb1Ch{^tu8@Tt;!osfp?%mMMh>Hh)q{Hx zyw<_pgU&`*bFXtYaLYI(s1GJiV~&k=p3{(b3(FHGh(}6(%MK}*sxE*$O*h4Q**ED2 zd9-R{R1xr~Y>XZjbvLRuHkh;|eM4rOti@UGtop`Ro0^+7XmPW7@1{eVR5z_}ezoPk zR_)sSXq((&LdUBe59QqGkk#JMW=HG$trlc&%$BwGw?5oP-pn+)j8DcRw~c#;IZ}Ps$t0dP#Go zW(if?0l&oWf^Oynz*!muqXceEbA*3^`)MyohcqT_Ko3|M?j4Q`jUoGydE`=(O?D*i z1t$}eL;7&z2rB}G8=>EX0HT#ga0({9qu_Uv&lbEP;}IaWK@yK_A4!C42N9Z|M#CXc?M z_I!N=Og_1_dG!snJpB^=L+y|H)3q&Xx>mQSehqvzrrNA}x<;e(7~5L9TH9MQt^Zh` zSesd2na`QpnzveVteLiKdkgzn+f&H{380+Yiu(Y)A9Gw(J{GaUh0=VIe! z!*9Jtr_edHIl8~P8r@bMMSofU6mTnp_3^s1n)3RAb%ScpR4=O>Um>ZOU9q9^L{+ou zVAaaXCFOfc(@T7RCzimaqVkeTaSdJ z(|gcs@Z5DxcBQ-DIeu7MS`S&j+t0X8x<0wud7a*P?@{k4-$H*|;!T)OZ9^wOcIqP1 zgdt`f;H03d`6KY=co|>Mqw#Zik5M&9olj%ySp7Mp(Hj0{L5AQMUxpH_myB}=0l$Lt z;c4`V)GW#%+CwCcaSXmjN9cQKC#f>v6>diBL$y(Sv|M@`?JR8&okPC}SkTGTKa}H8 zT4X`!81Quy#0!EK=FxV-ZQ-Br8{`h$jW&jo3r&vv3=fRVg@Vv!NCYj4>NB)OY-&HnR*Gkg&8M%G1EgL698N8Uy@MOs2f zC}osl;LuA)ZZrFEO3`AzLU2g1SFj)J$1CDO=wog#?gez%|IG(sPq0#cK5sm5$<5&U zIoaG0cNm((qhPhz3PHMXpD06|Ex94RCQFkS%RP!t%6p1uic^XU@;Bh8>&oY<1JO-m z|B3U)t&KYvmli)aepSN0q$4TY(+bja)32u;NV}RkEv0SBrj#iuX(^24JqcUle#W){ zI@5%h{V`i(w#3|z`4)r6(&FC7y^cK}EsWZx{37oxe=UaiHs}XC%-E{sz6kC zqF%(TkKGlM8$DB1EpI2=511x}Oe`NFSIfRgTZ6Q)TzFlu92?7D$$N$J&^g=$&U0oe za+p>G*~4|=Sm-n54eb#;8`%#Zq;Ci7J}mM%oDb>;1Eg;m)J!Un8m1UK87CW08mo+1 zrUKJnb2IB?TeAIMd!b$KNODx!ci1o3GaMerLT940*zw4@*WKNF-8V1rlNc7xgql(| zP_NU+Acv9eNOMGnl);~18@vZmG0RvaJB^#h8-^7N?&5QVR1ohRBx)k67v>AIg)Kl> z^Q~YmcA7s1G_fPROQ@Rrh+DzULW968TY`!38azQL5sHN!glC1LMDxU-Btlt<%q_Em z1x$Ro&&<4>i#>->WOCnaCS?ysz?x?Gb|mskHs zy-~YWKMq8h+nWo`11!fZ*FY3WW|i7n*d5?|9_}zZo;!0~JzUdVh+E-V;C<(-^1lt- z4{jkY5(&Uq`7}r)q6smPO)La-xu?xscd{!qA z#BR(GGkzfV5f8G1v49!RYRHna?yySOZq9q|MD#7HMbD#&=rL|5ZfkBHcP#fX2>C7t z!S?3t=d6*eY;e^wU5wY@z@0nvW3y+#zS-XWUW2#LyWHCmkdybkU%aRn_iT38gBUg2`N07> z7d!uRzHoMNop(?2DuD%id$28GCuWkD$)r%9a5(ZF96>OY_80`@`@^r{cw{~D3z2}; ztO1A9973!}J<=8#4DX?9X(_bTR07yx8&aB5?m%HsIHe?yI3HN)Cw*RDy1&AA$UD)K z>`{T%T;m$#GCD6i`v6P$I3V3zby8h%u5GS7_j^y0FUL;{ObYA|6b8(JHvwnhYA~I6 z7BmOe1zrVq1a}c5Ni@U>D=WKAASsixh;{!$W^2pV;r+FYdh;B z%g$;KjuUFft>YG<^Z2`giE$vd8p8!z!ApFw@VziwlqQ-cJc#$jr{LG{-*`RVQ#e}m zL|iDDC0!|*e77nM}!|cAzo>X;28^{De7$6UvB2j4+=+;&`d?sIX$DA2&2;#uy!>(lyefo;K; z!GXa(ff0V4?}2ZVPvu+T&Gw%6w)L&^9rl&_y88F{pZj@%7a$V&fM`N8$lU;@bs)IJ z;-E2bJ+LV-GH}^1^v(9nc8_+c9Z&3qw&Nf&@YwXpFiSrUfSYOh3%W8bQyZ(vRClO< zRrjOrW}U0HS*@+6OYP9Q@_M^^j^?W7re>(-mwL0>qSgVBzfIc>91BsbS)!3@UaC*1 zUFv0;e}OcmQ}0!i^?|yBwNW*?s+vl5MRdi^@&V-+%CkV=Vr>Oh>8li0C)S8+C)PDn z7ibP@$LNmg9fqLsAJbmb7Sm%BV(wsmZ>9mE3UM%;WlqR#@pSQL23HZIz#&@EAzA1* INee~&9~=pQ{r~^~ literal 0 HcmV?d00001 diff --git a/tensorflow/core/kernels/fuzzing/corpus/decode_wav/fce08de222896ac3a20657a3b4f42d5b6c54a96a b/tensorflow/core/kernels/fuzzing/corpus/decode_wav/fce08de222896ac3a20657a3b4f42d5b6c54a96a new file mode 100644 index 0000000000000000000000000000000000000000..35a99bc97d93c9beeea6a917453685fabf853ab2 GIT binary patch literal 69522 zcmeI(S8!9;x(D#Bz11bFSk;z`+;PD+7zkjD=`DnWP(lq56AYo30KwEy0yvaV0|8S^ zaSp|lU=k7_V2F(|HZHg~NtUc?^)784?#wwa_u;;sJ2^Ao+7D~(?_tfiezRuw?1%rS z6UU9)xrBgxHge|JB`Y^bIS7If-alh*BFO$(1O!F?r}FRd;lCF6zYG7GKMe!||EmPA zAOHd&00JNY0w4eaAOHd&00JNY0w4eaAOHd&00JNY0w4eaAOHd&00JNY0w4eaAOHd& z00JNY0w4eaAOHd&00JNY0w4eaAOHd&00JNY0w4eaAOHd&00JNY0w4eaAOHd&00JNY z0w4eaAOHd&00JNY0w4eaAOHd&00JNY0w4eaAOHd&00JNY0w4eaAOHd&00JNY0w4ea z{}}wZMWqS>Y3`??u(rh+h(-2b`0;@ z*15CY*z$REQ`7lILBrL$n5wbm1!dp7>C>3gxwb35WqZw~ipyo8uNo^)RJT{JtGfLr zuG&&FxPDH(vqn~x{Kos{Y>nuhu)d>8SswE8z{}(1e^h)?IkD*H(gKx$Bx|w{NSj&9le*Elx*PAmh;E6gzi$m@~Xy_%*AHno04| zvRLih{rn#VX9aRWyl|rMv5+mQ6fFo%iuA@6ru>{bJ*gp@6;2BKK7t)RDZV{vYwFST z#cA7;$0n}sBaEw+rbN=Cqmq{oT=RiC-Iq`nnInY2}Qn3m+uekZ_ zyG%P{DRT+)Iei3W2+=@XLCa>zIpJIf>lemQ`dP*W_A5bGXptm6t{y9mKwf4KG9rh->Ig zEDd`w`$bS5qnDaR-i8?orRZdGDpSryc@Kl4Np=1bkIi=r{gyI~Hk4LIi6`AhzejHQ zQ@n`xHcp{j<<1LPD7hz+vp=McCGJNPu{G3M_J@3pz%7gy6$$9PB6fg5p%*Y#a)d%c z$adih_I&DaGKpTrF69Y?K2evrPW(!oCH_-5QP9K7;~fyyMNW&qpHL9DE^22OH^e8@ z2ol8QVX}ym;Y-8jg*8axLXU{{@z)1u^O{AE!!xAQ!<$6&csO?pzf*igvQJ_O$rM)# zp9#A7ExgQNjB}Nh%n|bg!YRUDZWd!JaZ(_|`_h%>Y4#5e?DMyJK605HyB%3Jwu#$o zGj>??)=INgSEYz;&uU%QQQbX6=P^k2rRq49Lmtw3zfIKkMcb_oMmJUGF$Jt+t#QUn zn!!rGqE_*f`jv67t=eX@Am$*WMSoqJt|`$bn)02K{4a2pH_`f5t5@~Wh>Sb!7I%)9 z=B0X!&UAaDCChxlOta^BtjHni7FHqK!B7zWzD!q*YotFGE2sLIQuZkJ(V(&P8DuuL zp70bcqikn=9o)jXMHdo3CL~~q)B~&^c(KBhVyWb4=t|K*K@)FLu$L>~4HZ^~CB>G< zvty=9wg+!!R8gLj=TJgv-_SGY=cs&g1KJ)~?N9frJR|ULu>F)U>Ow5Ur*ZMTWqv`x z6j+TE5Tw|ALZ&~%=kuTQ&-HMfIgTUlR{vbw;6|+i{fV9@Dzdt-cCTTKNn%Rx9j0HV z{Z6xAb3}cor${r`aKJdVH$xMr(8%OUrgpLZf+5~;T9={Ou1IJTHOrdsHP0FuJ@l?c*gl6Rr~; z-tBm{?@tWX`~Tjqo?p)9+LBct;PJ-xZ7NBuXHDP z=ef36$D2Mi4lv#|)|n{QtYuQ%QmuzHJl=1#-XsY^(61H0Yp9BUnK>_6CoUHt<;p&t`60_XigycZdbet;Gc zcTnHb|DY?$*O9xv13n2ZMLt7oh(**F%pm4WayPmOTY?eL6UZFGC0eFXB8d<(*gI)= zNaKhqY#wntEjZXI=n6g=B&8-|Ly@=sYuAe;r^EC^>oI19pA5YGC) z!&#>_e(iWG2g#uPPF(EIw9n}+(bT?=*SVGA-V&?ave`IW-=*oN{$BY&9@4!>bH8_z z`Mil_sOz~V&+1gRFYmY_yQb==D>Y;pGPD8}D$8l*HXN_}t?7*7mNrdWtQ^<>kM%owI|r(ZJX`24vE{~%SWD} z6R=;Go(6fCZLBdolGqzsC>bkB537uLD9w#} z5PdIJlCUT7%fym+e~de-EGjvcnlLZ1E}59RFYQ5^y6>>$&_q+hjD&%Gp7gyn$eLL) zG<)E%)IN!05>n!k_>jbnspI-j>%TiKFIk*0EasN2Z1{`(EJt37ZK0@M`xg`$v|0mI=RIYXn&;a7x=7t^eLs`NmG;KYcf=DZ9Nzwy?XXyHbHG)+-mN4)?fq)fSVJ;%>9kO>6bL4aLSJv)`(9p7PZZ zo?s|a>7HUsFkd$3*cW+2F$w)3^EPcM@mk=jZ=rXScdfsZ5KMVZ|C8xuQW;*#I+BG!Enbo6t@xhM5WJK1h(;ynzu)&G@;XWjiHcSPx&s#iS8x$7 z#zT;q#3|$@7!&`(rL``!tg-N|uN@2h&(Md&IoO9tJI)Q{ypO6DxLLjlfiA*vv>6G* zuX{2%P3yl+}Mec)gpKBhF~>U%g*6c^IJn3 zBNoKOA)D9p_w^%q4OE=;7f8M2)=0 z3%z692i^BP%e;^AO5zEsn7WDBi5GiX-Ai2(o7(i+TxU=5rUqsOhWJjoXwJtJqz(wgh;HIWS|XT=F?Ls!?1{e&e!2f#eYS^=!NW`IFUhH$)6B@ zLUtj~{7z4%H#0DSWTiN;{(hconEinbb-Z@=^BnXgdk5Q-jUv5FTcEw7t1+aSFW5Qm zRqhRr6DE~rjPjKHvTU|;wLxPYZm%;>(Q#B4yRXX2+6^t%_CJ-wjVtUWwlrgdTBfLw zeJelT-O`h&`$hLoo%KHYeoD?)j8`mCj8*N_`^}rJA(o2XWKElVWXGKL`|`8;A-3C& zdk&@jk!`;*Uu$R=@2wl3)PXf~?XYi8&U^?qq7w*FzhWK1=RjRSgx z2AZL?H`B7svDI_ae;hyRUF>Ljf9LNw&*0Or5aNBb5UCCnBQpX|+@ozrER$@jTy|f2 zAR0gG(K@X5>(1wHyXSY0!Zp+}-KsRrH@VCqwyE}o_5{1mKK%VHm|`L8DBbfryW7RG zDLoIg*_uA8D~hS|1bKBAsa@C{*{E*3+d8a^A-~f(t3}+fv7xZ}gEnDDO-E(h@TLW| zZz^rCCRhCQdP8OQo1p6a+M$I@$4&IMyNeHzNgra3=iky;2O4rX<{a` zhOpVemxYqhVUn*yb3&p+W5S+>ot5}Q>%)GG`aR)D%JIGlNyV}6q*tU{V(XG})3Y-w zG7>V1`#1E@?C(r#NqL)eJY{$P+QE~DH4iKQumA6d1AqVsfB*=900@8p2!H?xfB*=9 z00@8p2!H?xfB*=900@8p2!H?xfB*=900@8p2!H?xfB*=900{iO1hBtXJJ1aXfB*=9 z00@8p2!H?xfB*=900@8p2!H?xfB*=900@8p2!H?xfB*=900@A<-&^1k;qToabO!<; l00JNY0w4eaAOHd&00JNY0w4eaAOHd&00JNY0{?FU{{;0JG~xgN literal 0 HcmV?d00001 diff --git a/tensorflow/core/kernels/fuzzing/corpus/string_split/4c01a1504da9de2216894743ecc44424 b/tensorflow/core/kernels/fuzzing/corpus/string_split/4c01a1504da9de2216894743ecc44424 new file mode 100644 index 00000000000..eb84b9e610c --- /dev/null +++ b/tensorflow/core/kernels/fuzzing/corpus/string_split/4c01a1504da9de2216894743ecc44424 @@ -0,0 +1 @@ +./,abcd.efgh/abcd,efgh.abcd/efgh,abcd.efgh/a \ No newline at end of file diff --git a/tensorflow/core/kernels/fuzzing/corpus/string_split/5bf16424630b5afbcffe711fb9834440 b/tensorflow/core/kernels/fuzzing/corpus/string_split/5bf16424630b5afbcffe711fb9834440 new file mode 100644 index 00000000000..4cd522da7bf --- /dev/null +++ b/tensorflow/core/kernels/fuzzing/corpus/string_split/5bf16424630b5afbcffe711fb9834440 @@ -0,0 +1 @@ +.ab.cd.ef.gh.ab.cd.ef.gh.ab.cd.ef.gh.ab.cd \ No newline at end of file diff --git a/tensorflow/core/kernels/fuzzing/corpus/string_split/a7185605aef0a8fd682fcb4656e4a736 b/tensorflow/core/kernels/fuzzing/corpus/string_split/a7185605aef0a8fd682fcb4656e4a736 new file mode 100644 index 00000000000..03cfb6256f3 --- /dev/null +++ b/tensorflow/core/kernels/fuzzing/corpus/string_split/a7185605aef0a8fd682fcb4656e4a736 @@ -0,0 +1 @@ +./, abcde.fghab/cdefg,habcd efgha.bcdef/ghabc \ No newline at end of file diff --git a/tensorflow/core/kernels/fuzzing/corpus/string_split/d5606def44fdbb9385dd764612069db0 b/tensorflow/core/kernels/fuzzing/corpus/string_split/d5606def44fdbb9385dd764612069db0 new file mode 100644 index 0000000000000000000000000000000000000000..304b0d66fe08fd1a29827488727702dd9b9bce3e GIT binary patch literal 42 ScmZQbOiE5kO-s)pNCN;-W)Kzt literal 0 HcmV?d00001 diff --git a/tensorflow/core/kernels/fuzzing/corpus/string_split/dbac766f3160de65894bf5153f478146 b/tensorflow/core/kernels/fuzzing/corpus/string_split/dbac766f3160de65894bf5153f478146 new file mode 100644 index 00000000000..a8740444aa4 --- /dev/null +++ b/tensorflow/core/kernels/fuzzing/corpus/string_split/dbac766f3160de65894bf5153f478146 @@ -0,0 +1 @@ +./, ?abcdef.ghabcd/efghab,cdefgh abcdef?ghabcd \ No newline at end of file diff --git a/tensorflow/core/kernels/fuzzing/corpus/string_split/e85ff62f6d457666f54a37a19a115a24 b/tensorflow/core/kernels/fuzzing/corpus/string_split/e85ff62f6d457666f54a37a19a115a24 new file mode 100644 index 00000000000..47d551466a4 --- /dev/null +++ b/tensorflow/core/kernels/fuzzing/corpus/string_split/e85ff62f6d457666f54a37a19a115a24 @@ -0,0 +1 @@ +./abc.def/gha.bcd/efg.hab/cde.fgh/abc.def/g \ No newline at end of file diff --git a/tensorflow/core/kernels/fuzzing/corpus/string_split_v2/00fd47bf73afcb72e7ed51bffd5f5fec b/tensorflow/core/kernels/fuzzing/corpus/string_split_v2/00fd47bf73afcb72e7ed51bffd5f5fec new file mode 100644 index 00000000000..f1410e184b2 --- /dev/null +++ b/tensorflow/core/kernels/fuzzing/corpus/string_split_v2/00fd47bf73afcb72e7ed51bffd5f5fec @@ -0,0 +1 @@ +./abc./de./fg./ha./bc./de./fg./ha./bc./de./ \ No newline at end of file diff --git a/tensorflow/core/kernels/fuzzing/corpus/string_split_v2/14908973e6720513a5f37676cb9fcc29 b/tensorflow/core/kernels/fuzzing/corpus/string_split_v2/14908973e6720513a5f37676cb9fcc29 new file mode 100644 index 00000000000..e118d2d351b --- /dev/null +++ b/tensorflow/core/kernels/fuzzing/corpus/string_split_v2/14908973e6720513a5f37676cb9fcc29 @@ -0,0 +1 @@ +./, abcde./, fg./, ha./, bc./, de./, fg./, ha \ No newline at end of file diff --git a/tensorflow/core/kernels/fuzzing/corpus/string_split_v2/2779ba7c4d23eee9f79efa3660084c5d b/tensorflow/core/kernels/fuzzing/corpus/string_split_v2/2779ba7c4d23eee9f79efa3660084c5d new file mode 100644 index 00000000000..9a6c8091974 --- /dev/null +++ b/tensorflow/core/kernels/fuzzing/corpus/string_split_v2/2779ba7c4d23eee9f79efa3660084c5d @@ -0,0 +1 @@ +./,abcd./,ef./,gh./,ab./,cd./,ef./,gh./,ab./ \ No newline at end of file diff --git a/tensorflow/core/kernels/fuzzing/corpus/string_split_v2/5bf16424630b5afbcffe711fb9834440 b/tensorflow/core/kernels/fuzzing/corpus/string_split_v2/5bf16424630b5afbcffe711fb9834440 new file mode 100644 index 00000000000..4cd522da7bf --- /dev/null +++ b/tensorflow/core/kernels/fuzzing/corpus/string_split_v2/5bf16424630b5afbcffe711fb9834440 @@ -0,0 +1 @@ +.ab.cd.ef.gh.ab.cd.ef.gh.ab.cd.ef.gh.ab.cd \ No newline at end of file diff --git a/tensorflow/core/kernels/fuzzing/corpus/string_split_v2/89734a96b93275e495a9498b806fafe1 b/tensorflow/core/kernels/fuzzing/corpus/string_split_v2/89734a96b93275e495a9498b806fafe1 new file mode 100644 index 00000000000..5301a91d8e4 --- /dev/null +++ b/tensorflow/core/kernels/fuzzing/corpus/string_split_v2/89734a96b93275e495a9498b806fafe1 @@ -0,0 +1 @@ +./, ?abcdef./, ?gh./, ?ab./, ?cd./, ?ef./, ?gh \ No newline at end of file diff --git a/tensorflow/core/kernels/fuzzing/corpus/string_split_v2/d5606def44fdbb9385dd764612069db0 b/tensorflow/core/kernels/fuzzing/corpus/string_split_v2/d5606def44fdbb9385dd764612069db0 new file mode 100644 index 0000000000000000000000000000000000000000..304b0d66fe08fd1a29827488727702dd9b9bce3e GIT binary patch literal 42 ScmZQbOiE5kO-s)pNCN;-W)Kzt literal 0 HcmV?d00001 diff --git a/tensorflow/core/kernels/fuzzing/corpus/string_to_number/2db83ea58639b6d7d585fa12e3947a82 b/tensorflow/core/kernels/fuzzing/corpus/string_to_number/2db83ea58639b6d7d585fa12e3947a82 new file mode 100644 index 00000000000..3de80927d57 --- /dev/null +++ b/tensorflow/core/kernels/fuzzing/corpus/string_to_number/2db83ea58639b6d7d585fa12e3947a82 @@ -0,0 +1 @@ +6.023e+23 diff --git a/tensorflow/core/kernels/fuzzing/corpus/string_to_number/36b4a931886b941dc41180050d12ca94 b/tensorflow/core/kernels/fuzzing/corpus/string_to_number/36b4a931886b941dc41180050d12ca94 new file mode 100644 index 00000000000..d531129b283 --- /dev/null +++ b/tensorflow/core/kernels/fuzzing/corpus/string_to_number/36b4a931886b941dc41180050d12ca94 @@ -0,0 +1 @@ +6.023e-23 diff --git a/tensorflow/core/kernels/fuzzing/corpus/string_to_number/50a2fabfdd276f573ff97ace8b11c5f4 b/tensorflow/core/kernels/fuzzing/corpus/string_to_number/50a2fabfdd276f573ff97ace8b11c5f4 new file mode 100644 index 00000000000..d81cc0710eb --- /dev/null +++ b/tensorflow/core/kernels/fuzzing/corpus/string_to_number/50a2fabfdd276f573ff97ace8b11c5f4 @@ -0,0 +1 @@ +42 diff --git a/tensorflow/core/kernels/fuzzing/corpus/string_to_number/62edb2a1eee34b001652cd86584becf2 b/tensorflow/core/kernels/fuzzing/corpus/string_to_number/62edb2a1eee34b001652cd86584becf2 new file mode 100644 index 00000000000..72f88139d0f --- /dev/null +++ b/tensorflow/core/kernels/fuzzing/corpus/string_to_number/62edb2a1eee34b001652cd86584becf2 @@ -0,0 +1 @@ +0xabcdef diff --git a/tensorflow/core/kernels/fuzzing/corpus/string_to_number/90013d1ec28c46a5c00574e60c70b6fc b/tensorflow/core/kernels/fuzzing/corpus/string_to_number/90013d1ec28c46a5c00574e60c70b6fc new file mode 100644 index 00000000000..c1113b83e8f --- /dev/null +++ b/tensorflow/core/kernels/fuzzing/corpus/string_to_number/90013d1ec28c46a5c00574e60c70b6fc @@ -0,0 +1 @@ +3.14159265359 diff --git a/tensorflow/core/kernels/fuzzing/corpus/string_to_number/94f3e3cee6957ce5815326d6788c85f4 b/tensorflow/core/kernels/fuzzing/corpus/string_to_number/94f3e3cee6957ce5815326d6788c85f4 new file mode 100644 index 00000000000..320aa3f00ee --- /dev/null +++ b/tensorflow/core/kernels/fuzzing/corpus/string_to_number/94f3e3cee6957ce5815326d6788c85f4 @@ -0,0 +1 @@ +0.69314718056 diff --git a/tensorflow/core/kernels/fuzzing/corpus/string_to_number/96f547bc04bb913da0bc08915238ebd8 b/tensorflow/core/kernels/fuzzing/corpus/string_to_number/96f547bc04bb913da0bc08915238ebd8 new file mode 100644 index 00000000000..51b7b732f69 --- /dev/null +++ b/tensorflow/core/kernels/fuzzing/corpus/string_to_number/96f547bc04bb913da0bc08915238ebd8 @@ -0,0 +1 @@ +6.023e23 diff --git a/tensorflow/core/kernels/fuzzing/corpus/string_to_number/d3a903d18fc11e1f35c572ad4da690ed b/tensorflow/core/kernels/fuzzing/corpus/string_to_number/d3a903d18fc11e1f35c572ad4da690ed new file mode 100644 index 00000000000..9a0be0764b6 --- /dev/null +++ b/tensorflow/core/kernels/fuzzing/corpus/string_to_number/d3a903d18fc11e1f35c572ad4da690ed @@ -0,0 +1 @@ +1.61803 diff --git a/tensorflow/core/kernels/fuzzing/corpus/string_to_number/e3b629c92af44260c189deb32d6f06f3 b/tensorflow/core/kernels/fuzzing/corpus/string_to_number/e3b629c92af44260c189deb32d6f06f3 new file mode 100644 index 00000000000..6a0e60d48b1 --- /dev/null +++ b/tensorflow/core/kernels/fuzzing/corpus/string_to_number/e3b629c92af44260c189deb32d6f06f3 @@ -0,0 +1 @@ +-42 diff --git a/tensorflow/core/kernels/fuzzing/corpus/string_to_number/f03eecf3bcfe4967a1888156a3115c8d b/tensorflow/core/kernels/fuzzing/corpus/string_to_number/f03eecf3bcfe4967a1888156a3115c8d new file mode 100644 index 00000000000..ea9cd255bc7 --- /dev/null +++ b/tensorflow/core/kernels/fuzzing/corpus/string_to_number/f03eecf3bcfe4967a1888156a3115c8d @@ -0,0 +1 @@ +6.023E+23 diff --git a/tensorflow/core/kernels/fuzzing/corpus/string_to_number/fa54ca9186f77122ae2a82684a062e16 b/tensorflow/core/kernels/fuzzing/corpus/string_to_number/fa54ca9186f77122ae2a82684a062e16 new file mode 100644 index 00000000000..00f1e2ed8ff --- /dev/null +++ b/tensorflow/core/kernels/fuzzing/corpus/string_to_number/fa54ca9186f77122ae2a82684a062e16 @@ -0,0 +1 @@ +2.71828182846 diff --git a/tensorflow/core/kernels/fuzzing/dictionaries/decode_json_example.dict b/tensorflow/core/kernels/fuzzing/dictionaries/decode_json_example.dict new file mode 100644 index 00000000000..5fe4ca23d1f --- /dev/null +++ b/tensorflow/core/kernels/fuzzing/dictionaries/decode_json_example.dict @@ -0,0 +1,6 @@ +"features" +"feature" +"bytes_list" +"float_list" +"int64_list" +"value" diff --git a/tensorflow/core/kernels/fuzzing/dictionaries/decode_png.dict b/tensorflow/core/kernels/fuzzing/dictionaries/decode_png.dict new file mode 100644 index 00000000000..d795ae7f71f --- /dev/null +++ b/tensorflow/core/kernels/fuzzing/dictionaries/decode_png.dict @@ -0,0 +1,50 @@ +header_87a="87a" +header_89a="89a" +header_gif="GIF" +header_jfif="JFIF\x00" +header_jfxx="JFXX\x00" +header_png="\x89PNG\x0d\x0a\x1a\x0a" +marker_2c="," +marker_3b=";" +section_2101="!\x01\x12" +section_21f9="!\xf9\x04" +section_21fe="!\xfe" +section_21ff="!\xff\x11" +section_IDAT="IDAT" +section_IEND="IEND" +section_IHDR="IHDR" +section_PLTE="PLTE" +section_bKGD="bKGD" +section_cHRM="cHRM" +section_fRAc="fRAc" +section_ffc0="\xff\xc0" +section_ffc2="\xff\xc2" +section_ffc4="\xff\xc4" +section_ffd0="\xff\xd0" +section_ffd8="\xff\xd8" +section_ffd9="\xff\xd9" +section_ffda="\xff\xda" +section_ffdb="\xff\xdb" +section_ffdd="\xff\xdd" +section_ffe0="\xff\xe0" +section_ffe1="\xff\xe1" +section_fffe="\xff\xfe" +section_gAMA="gAMA" +section_gIFg="gIFg" +section_gIFt="gIFt" +section_gIFx="gIFx" +section_hIST="hIST" +section_iCCP="iCCP" +section_iTXt="iTXt" +section_oFFs="oFFs" +section_pCAL="pCAL" +section_pHYs="pHYs" +section_sBIT="sBIT" +section_sCAL="sCAL" +section_sPLT="sPLT" +section_sRGB="sRGB" +section_sTER="sTER" +section_tEXt="tEXt" +section_tIME="tIME" +section_tRNS="tRNS" +section_zTXt="zTXt" diff --git a/tensorflow/core/kernels/fuzzing/dictionaries/decode_wav.dict b/tensorflow/core/kernels/fuzzing/dictionaries/decode_wav.dict new file mode 100644 index 00000000000..eab65386ce3 --- /dev/null +++ b/tensorflow/core/kernels/fuzzing/dictionaries/decode_wav.dict @@ -0,0 +1,4 @@ +header_RIFF="RIFF" +header_WAVE="WAVE" +section_fmt="fmt " +section_data="data" diff --git a/tensorflow/core/kernels/fuzzing/encode_base64_fuzz.cc b/tensorflow/core/kernels/fuzzing/encode_base64_fuzz.cc index a8f07f4bad3..b8d779fb138 100644 --- a/tensorflow/core/kernels/fuzzing/encode_base64_fuzz.cc +++ b/tensorflow/core/kernels/fuzzing/encode_base64_fuzz.cc @@ -19,7 +19,7 @@ limitations under the License. namespace tensorflow { namespace fuzzing { -class FuzzEncodeBase64 : public FuzzSession { +class FuzzEncodeBase64 : public FuzzStringInputOp { SINGLE_INPUT_OP_BUILDER(DT_STRING, EncodeBase64); }; diff --git a/tensorflow/core/kernels/fuzzing/fuzz_session.h b/tensorflow/core/kernels/fuzzing/fuzz_session.h index 9777be1ae8a..57d562ddf43 100644 --- a/tensorflow/core/kernels/fuzzing/fuzz_session.h +++ b/tensorflow/core/kernels/fuzzing/fuzz_session.h @@ -72,11 +72,11 @@ class FuzzSession { // By convention, the graph should have inputs named "input1", ... // "inputN", and one output node, named "output". // Users of FuzzSession should override this method to create their graph. - virtual void BuildGraph(const Scope& scope) {} + virtual void BuildGraph(const Scope& scope) = 0; // Implements the logic that converts an opaque byte buffer // from the fuzzer to Tensor inputs to the graph. Users must override. - virtual void FuzzImpl(const uint8_t* data, size_t size) {} + virtual void FuzzImpl(const uint8_t* data, size_t size) = 0; // Initializes the FuzzSession. Not safe for multithreading. // Separate init function because the call to virtual BuildGraphDef diff --git a/tensorflow/core/kernels/fuzzing/string_split_fuzz.cc b/tensorflow/core/kernels/fuzzing/string_split_fuzz.cc index 87a548a999c..2564f8ed030 100644 --- a/tensorflow/core/kernels/fuzzing/string_split_fuzz.cc +++ b/tensorflow/core/kernels/fuzzing/string_split_fuzz.cc @@ -37,8 +37,7 @@ class FuzzStringSplit : public FuzzSession { // The spec for split is that the delimeter should be 0 or 1 characters. // Naturally, fuzz it with something larger. (This omits the possibility // of handing it a > int32_max size string, which should be tested for in - // an - // explicit test). + // an explicit test). size_t delim_len = static_cast(data[0]); if (delim_len > size) { delim_len = size - 1; diff --git a/tensorflow/core/kernels/fuzzing/tf_ops_fuzz_target_lib.bzl b/tensorflow/core/kernels/fuzzing/tf_ops_fuzz_target_lib.bzl index f752b59568a..e9322133590 100644 --- a/tensorflow/core/kernels/fuzzing/tf_ops_fuzz_target_lib.bzl +++ b/tensorflow/core/kernels/fuzzing/tf_ops_fuzz_target_lib.bzl @@ -1,13 +1,25 @@ """Fuzzing template for TensorFlow ops.""" def tf_ops_fuzz_target_lib(name): - native.cc_library( - name = name + "_fuzz_lib", - srcs = [name + "_fuzz.cc"], - deps = [ - "//tensorflow/core/kernels/fuzzing:fuzz_session", - "//tensorflow/cc:cc_ops", - ], - tags = ["no_windows"], - alwayslink = 1, - ) + native.cc_library( + name = name + "_fuzz_lib", + srcs = [name + "_fuzz.cc"], + deps = [ + "//tensorflow/core/kernels/fuzzing:fuzz_session", + "//tensorflow/cc:cc_ops", + ], + tags = ["no_windows"], + alwayslink = 1, + ) + +def tf_oss_fuzz_corpus(name): + native.filegroup( + name = name + "_corpus", + srcs = native.glob(["corpus/" + name + "/*"]), + ) + +def tf_oss_fuzz_dict(name): + native.filegroup( + name = name + "_dict", + srcs = native.glob(["dictionaries/" + name + ".dict"]), + ) diff --git a/tensorflow/core/kernels/list_kernels.cc b/tensorflow/core/kernels/list_kernels.cc index 5f244b1b10f..42fad1d4b05 100644 --- a/tensorflow/core/kernels/list_kernels.cc +++ b/tensorflow/core/kernels/list_kernels.cc @@ -483,9 +483,19 @@ REGISTER_KERNEL_BUILDER(Name("TensorListGetItem").Device(DEVICE_CPU), #if GOOGLE_CUDA -REGISTER_KERNEL_BUILDER( - Name("TensorListGetItem").Device(DEVICE_GPU).HostMemory("index"), - TensorListGetItem); +#define REGISTER_TENSOR_LIST_GET_ITEM_GPU(T) \ + REGISTER_KERNEL_BUILDER(Name("TensorListGetItem") \ + .TypeConstraint("element_dtype") \ + .Device(DEVICE_GPU) \ + .HostMemory("index"), \ + TensorListGetItem); + +TF_CALL_GPU_NUMBER_TYPES(REGISTER_TENSOR_LIST_GET_ITEM_GPU); +TF_CALL_complex64(REGISTER_TENSOR_LIST_GET_ITEM_GPU); +TF_CALL_complex128(REGISTER_TENSOR_LIST_GET_ITEM_GPU); +TF_CALL_int64(REGISTER_TENSOR_LIST_GET_ITEM_GPU); +REGISTER_TENSOR_LIST_GET_ITEM_GPU(bfloat16) +#undef REGISTER_TENSOR_LIST_GET_ITEM_GPU #endif // GOOGLE_CUDA @@ -537,9 +547,19 @@ REGISTER_KERNEL_BUILDER(Name("TensorListSetItem").Device(DEVICE_CPU), #if GOOGLE_CUDA -REGISTER_KERNEL_BUILDER( - Name("TensorListSetItem").Device(DEVICE_GPU).HostMemory("index"), - TensorListSetItem); +#define REGISTER_TENSOR_LIST_SET_ITEM_GPU(T) \ + REGISTER_KERNEL_BUILDER(Name("TensorListSetItem") \ + .TypeConstraint("element_dtype") \ + .Device(DEVICE_GPU) \ + .HostMemory("index"), \ + TensorListSetItem); + +TF_CALL_GPU_NUMBER_TYPES(REGISTER_TENSOR_LIST_SET_ITEM_GPU); +TF_CALL_complex64(REGISTER_TENSOR_LIST_SET_ITEM_GPU); +TF_CALL_complex128(REGISTER_TENSOR_LIST_SET_ITEM_GPU); +TF_CALL_int64(REGISTER_TENSOR_LIST_SET_ITEM_GPU); +REGISTER_TENSOR_LIST_SET_ITEM_GPU(bfloat16) +#undef REGISTER_TENSOR_LIST_SET_ITEM_GPU #endif // GOOGLE_CUDA @@ -660,7 +680,11 @@ REGISTER_TENSOR_LIST_PUSH_BACK_BATCH_CPU(bfloat16); REGISTER_KERNEL_BUILDER(Name("TensorListGather") \ .TypeConstraint("element_dtype") \ .Device(DEVICE_CPU), \ - TensorListGather) + TensorListGather) \ + REGISTER_KERNEL_BUILDER(Name("TensorListConcat") \ + .TypeConstraint("element_dtype") \ + .Device(DEVICE_CPU), \ + TensorListConcat) TF_CALL_POD_STRING_TYPES(REGISTER_TENSOR_LIST_STACK_CPU); REGISTER_TENSOR_LIST_STACK_CPU(quint8); @@ -680,7 +704,11 @@ REGISTER_TENSOR_LIST_STACK_CPU(bfloat16); REGISTER_KERNEL_BUILDER(Name("TensorListScatter") \ .TypeConstraint("element_dtype") \ .Device(DEVICE_CPU), \ - TensorListScatter) + TensorListScatter) \ + REGISTER_KERNEL_BUILDER(Name("TensorListSplit") \ + .TypeConstraint("element_dtype") \ + .Device(DEVICE_CPU), \ + TensorListSplit) TF_CALL_POD_STRING_TYPES(REGISTER_TENSOR_LIST_FROM_TENSOR_CPU); REGISTER_TENSOR_LIST_FROM_TENSOR_CPU(quint8); diff --git a/tensorflow/core/kernels/list_kernels.cu.cc b/tensorflow/core/kernels/list_kernels.cu.cc index a00bf700ca2..23f552642ca 100644 --- a/tensorflow/core/kernels/list_kernels.cu.cc +++ b/tensorflow/core/kernels/list_kernels.cu.cc @@ -45,7 +45,12 @@ typedef Eigen::GpuDevice GPUDevice; .TypeConstraint("element_dtype") \ .Device(DEVICE_GPU) \ .HostMemory("indices"), \ - TensorListGather) + TensorListGather) \ + REGISTER_KERNEL_BUILDER(Name("TensorListConcat") \ + .TypeConstraint("element_dtype") \ + .Device(DEVICE_GPU) \ + .HostMemory("lengths"), \ + TensorListConcat) TF_CALL_GPU_NUMBER_TYPES(REGISTER_TENSOR_LIST_STACK_GPU); REGISTER_TENSOR_LIST_STACK_GPU(bfloat16); @@ -82,7 +87,13 @@ REGISTER_TENSOR_LIST_PUSH_BACK_BATCH_GPU(bool); .Device(DEVICE_GPU) \ .HostMemory("element_shape") \ .HostMemory("indices"), \ - TensorListScatter) + TensorListScatter) \ + REGISTER_KERNEL_BUILDER(Name("TensorListSplit") \ + .TypeConstraint("element_dtype") \ + .Device(DEVICE_GPU) \ + .HostMemory("element_shape") \ + .HostMemory("lengths"), \ + TensorListSplit) TF_CALL_GPU_NUMBER_TYPES(REGISTER_TENSOR_LIST_FROM_TENSOR_GPU); REGISTER_TENSOR_LIST_FROM_TENSOR_GPU(bfloat16); diff --git a/tensorflow/core/kernels/list_kernels.h b/tensorflow/core/kernels/list_kernels.h index c2591f53141..686679474c4 100644 --- a/tensorflow/core/kernels/list_kernels.h +++ b/tensorflow/core/kernels/list_kernels.h @@ -30,6 +30,8 @@ limitations under the License. #include "tensorflow/core/kernels/concat_lib.h" #include "tensorflow/core/lib/core/coding.h" #include "tensorflow/core/lib/core/errors.h" +#include "tensorflow/core/lib/gtl/array_slice.h" +#include "tensorflow/core/util/tensor_ops_util.h" #include "tensorflow/core/util/util.h" namespace tensorflow { @@ -76,26 +78,30 @@ class TensorListStack : public OpKernel { ~TensorListStack() {} void Compute(OpKernelContext* c) override { - const TensorList* l = c->input(0).scalar()().get(); - OP_REQUIRES(c, l != nullptr, + const TensorList* tensor_list = + c->input(0).scalar()().get(); + OP_REQUIRES(c, tensor_list != nullptr, errors::InvalidArgument( "Input handle is not a list. Saw: '", c->input(0).scalar()().DebugString(), "'")); - OP_REQUIRES(c, element_dtype_ == l->element_dtype, - errors::InvalidArgument("Invalid data types; op elements ", - DataTypeString(element_dtype_), - " but list elements ", - DataTypeString(l->element_dtype))); - OP_REQUIRES(c, !l->tensors.empty() || l->element_shape.IsFullyDefined(), - errors::InvalidArgument("Tried to stack elements of a empty ", - "list with non-fully-defined shape: ", - l->element_shape.DebugString())); + OP_REQUIRES( + c, element_dtype_ == tensor_list->element_dtype, + errors::InvalidArgument( + "Invalid data types; op elements ", DataTypeString(element_dtype_), + " but list elements ", DataTypeString(tensor_list->element_dtype))); + OP_REQUIRES( + c, + !tensor_list->tensors.empty() || + tensor_list->element_shape.IsFullyDefined(), + errors::InvalidArgument("Tried to stack elements of a empty ", + "list with non-fully-defined shape: ", + tensor_list->element_shape.DebugString())); if (num_elements_ != -1) { - OP_REQUIRES(c, l->tensors.size() == num_elements_, - errors::InvalidArgument("Operation expected a list with ", - num_elements_, - " elements but got a list with ", - l->tensors.size(), " elements.")); + OP_REQUIRES(c, tensor_list->tensors.size() == num_elements_, + errors::InvalidArgument( + "Operation expected a list with ", num_elements_, + " elements but got a list with ", + tensor_list->tensors.size(), " elements.")); } // Compute the shape of the output tensor. // If `element_shape` is fully-defined it gets used. It is assumed that all @@ -104,11 +110,11 @@ class TensorListStack : public OpKernel { // tensor is used and it is checked that all other tensors have the same // shape. TensorShape resulting_shape; - if (!l->element_shape.AsTensorShape(&resulting_shape)) { - const Tensor& t = l->tensors[0]; + if (!tensor_list->element_shape.AsTensorShape(&resulting_shape)) { + const Tensor& t = tensor_list->tensors[0]; resulting_shape = t.shape(); - for (int i = 1; i < l->tensors.size(); ++i) { - const Tensor& t = l->tensors[i]; + for (int i = 1; i < tensor_list->tensors.size(); ++i) { + const Tensor& t = tensor_list->tensors[i]; OP_REQUIRES(c, t.shape() == resulting_shape, errors::InvalidArgument( "Tried to stack tensors with unequal shapes: ", @@ -116,7 +122,7 @@ class TensorListStack : public OpKernel { t.shape().DebugString())); } } - resulting_shape.InsertDim(0, l->tensors.size()); + resulting_shape.InsertDim(0, tensor_list->tensors.size()); Tensor* output; OP_REQUIRES_OK(c, c->allocate_output(0, resulting_shape, &output)); if (output->NumElements() == 0) { @@ -124,8 +130,8 @@ class TensorListStack : public OpKernel { } ConstMatrixVector inputs_flat; - inputs_flat.reserve(l->tensors.size()); - for (const auto& t : l->tensors) { + inputs_flat.reserve(tensor_list->tensors.size()); + for (const auto& t : tensor_list->tensors) { inputs_flat.emplace_back(new typename TTypes::ConstMatrix( t.shaped({1, t.NumElements()}))); } @@ -145,6 +151,200 @@ class TensorListStack : public OpKernel { DataType element_dtype_; }; +template +class TensorListConcat : public OpKernel { + public: + using ConstMatrixVector = + std::vector::ConstMatrix>>; + explicit TensorListConcat(OpKernelConstruction* c) : OpKernel(c) { + OP_REQUIRES_OK(c, c->GetAttr("element_dtype", &element_dtype_)); + } + + ~TensorListConcat() {} + + void Compute(OpKernelContext* c) override { + // Check that the input Variant tensor is indeed a TensorList and has the + // correct element type. + const TensorList* tensor_list = + c->input(0).scalar()().get(); + OP_REQUIRES(c, tensor_list != nullptr, + errors::InvalidArgument( + "Input handle is not a list. Saw: '", + c->input(0).scalar()().DebugString(), "'")); + OP_REQUIRES( + c, element_dtype_ == tensor_list->element_dtype, + errors::InvalidArgument( + "Invalid data types; op elements ", DataTypeString(element_dtype_), + " but list elements ", DataTypeString(tensor_list->element_dtype))); + // If the TensorList is empty, its element_shape must be fully defined + // except for the first dimension. + PartialTensorShape shape_except_first_dim; + if (!tensor_list->element_shape.unknown_rank()) { + OP_REQUIRES(c, tensor_list->element_shape.dims() >= 1, + errors::InvalidArgument( + "Concat requires elements to be at least vectors, ", + "found scalars instead.")); + shape_except_first_dim = PartialTensorShape( + gtl::ArraySlice(tensor_list->element_shape.dim_sizes()) + .subspan(1)); + } + OP_REQUIRES(c, + !tensor_list->tensors.empty() || + shape_except_first_dim.IsFullyDefined(), + errors::InvalidArgument( + "All except the first dimension must be fully defined ", + "when concating an empty tensor list. element_shape: ", + tensor_list->element_shape.DebugString())); + // 1. Compute the shape of the output tensor. + // If `shape_except_first_dim` is fully-defined we just prepend the leading + // dim to it. Otherwise we use the shape of the first element tensor and + // check to make sure shapes of all tensors are compatible. + TensorShape output_shape; + if (!shape_except_first_dim.AsTensorShape(&output_shape)) { + const Tensor& element_tensor = tensor_list->tensors[0]; + OP_REQUIRES( + c, TensorShapeUtils::IsVectorOrHigher(element_tensor.shape()), + errors::InvalidArgument("Concat saw a scalar shape at index ", 0, + " but requires at least vectors.")); + output_shape = + TensorShape(gtl::ArraySlice(element_tensor.shape().dim_sizes()) + .subspan(1)); + for (int i = 1; i < tensor_list->tensors.size(); ++i) { + const Tensor& element_tensor = tensor_list->tensors[i]; + OP_REQUIRES( + c, TensorShapeUtils::IsVectorOrHigher(element_tensor.shape()), + errors::InvalidArgument("Concat saw a scalar shape at index ", i, + " but requires at least vectors.")); + TensorShape actual_shape( + gtl::ArraySlice(element_tensor.shape().dim_sizes()) + .subspan(1)); + OP_REQUIRES(c, actual_shape.dim_sizes() == output_shape.dim_sizes(), + errors::InvalidArgument( + "Tried to concat tensors with unequal shapes: ", + output_shape.DebugString(), " vs ", + actual_shape.DebugString())); + } + } + // 2. Build the lengths_tensor and leading dim of the output tensor by + // iterating over all element tensors. + Tensor* lengths_tensor = nullptr; + OP_REQUIRES_OK( + c, + c->allocate_output( + 1, TensorShape({static_cast(tensor_list->tensors.size())}), + &lengths_tensor)); + auto lengths_tensor_vec = lengths_tensor->vec(); + int64 leading_dim = 0; + for (size_t i = 0; i < tensor_list->tensors.size(); i++) { + int64 dim = tensor_list->tensors[i].shape().dim_size(0); + leading_dim += dim; + lengths_tensor_vec(i) = dim; + } + output_shape.InsertDim(0, leading_dim); + Tensor* output; + // 3. Allocate the output tensor and fill it up with the concated element + // tensors. + OP_REQUIRES_OK(c, c->allocate_output(0, output_shape, &output)); + if (output->NumElements() == 0) { + return; + } + + ConstMatrixVector inputs_flat; + inputs_flat.reserve(tensor_list->tensors.size()); + for (const auto& element_tensor : tensor_list->tensors) { + inputs_flat.emplace_back(new typename TTypes::ConstMatrix( + element_tensor.shaped({1, element_tensor.NumElements()}))); + } + auto output_flat = output->shaped({1, output->NumElements()}); + +#if GOOGLE_CUDA + if (std::is_same::value) { + ConcatGPU(c, inputs_flat, output, &output_flat); + return; + } +#endif // GOOGLE_CUDA + ConcatCPU(c->device(), inputs_flat, &output_flat); + } + + private: + DataType element_dtype_; +}; + +template +class TensorListSplit : public OpKernel { + public: + TensorListSplit(OpKernelConstruction* c) : OpKernel(c) {} + + void Compute(OpKernelContext* c) override { + Tensor* output_tensor; + AllocatorAttributes attr; + attr.set_on_host(true); + OP_REQUIRES_OK(c, c->allocate_output(0, {}, &output_tensor, attr)); + PartialTensorShape element_shape; + OP_REQUIRES_OK(c, TensorShapeFromTensor(c->input(1), &element_shape)); + OP_REQUIRES(c, element_shape.unknown_rank() || element_shape.dims() >= 1, + errors::InvalidArgument( + "TensorListSplit requires element_shape to be at least of ", + "rank 1, but saw: ", element_shape.DebugString())); + TensorList output_list; + const Tensor& input_tensor = c->input(0); + output_list.element_dtype = input_tensor.dtype(); + OP_REQUIRES(c, TensorShapeUtils::IsVectorOrHigher(input_tensor.shape()), + errors::InvalidArgument( + "Tensor must be at least a vector, but saw shape: ", + input_tensor.shape().DebugString())); + TensorShape tensor_shape_without_first_dim(input_tensor.shape()); + tensor_shape_without_first_dim.RemoveDim(0); + PartialTensorShape element_shape_without_first_dim; + if (!element_shape.unknown_rank()) { + element_shape_without_first_dim = + PartialTensorShape(element_shape.dim_sizes()); + element_shape_without_first_dim.RemoveDim(0); + } + OP_REQUIRES(c, + element_shape_without_first_dim.IsCompatibleWith( + tensor_shape_without_first_dim), + errors::InvalidArgument( + "tensor shape ", input_tensor.shape().DebugString(), + " is not compatible with element_shape ", + element_shape.DebugString())); + output_list.element_shape = element_shape; + const Tensor& lengths = c->input(2); + OP_REQUIRES(c, TensorShapeUtils::IsVector(lengths.shape()), + errors::InvalidArgument( + "Expected lengths to be a vector, received shape: ", + lengths.shape().DebugString())); + output_list.tensors.reserve(lengths.shape().dim_size(0)); + int64 start = 0; + int64 end = 0; + for (int i = 0; i < lengths.shape().dim_size(0); ++i) { + int64 length = lengths.vec()(i); + OP_REQUIRES( + c, length >= 0, + errors::InvalidArgument("Invalid value in lengths: ", length)); + end = start + length; + OP_REQUIRES(c, end <= input_tensor.shape().dim_size(0), + errors::InvalidArgument("Attempting to slice [", start, ", ", + end, "] from tensor with length ", + input_tensor.shape().dim_size(0))); + Tensor tmp = input_tensor.Slice(start, end); + start = end; + // TODO(apassos) maybe not always align; but weird compiler bugs seem to + // prevent this. + Tensor aligned; + OP_REQUIRES_OK(c, c->allocate_temp(tmp.dtype(), tmp.shape(), &aligned)); + aligned.flat().device(c->eigen_device()) = + tmp.unaligned_flat(); + output_list.tensors.emplace_back(aligned); + } + OP_REQUIRES(c, end == input_tensor.shape().dim_size(0), + errors::InvalidArgument( + "Unused values in tensor. Length of tensor: ", + input_tensor.shape().dim_size(0), " Values used: ", end)); + output_tensor->scalar()() = std::move(output_list); + } +}; + template class TensorListGather : public OpKernel { public: @@ -155,22 +355,25 @@ class TensorListGather : public OpKernel { } void Compute(OpKernelContext* c) override { - const TensorList* l = c->input(0).scalar()().get(); - OP_REQUIRES(c, l != nullptr, + const TensorList* tensor_list = + c->input(0).scalar()().get(); + OP_REQUIRES(c, tensor_list != nullptr, errors::InvalidArgument( "Input handle is not a list. Saw: '", c->input(0).scalar()().DebugString(), "'")); - OP_REQUIRES(c, element_dtype_ == l->element_dtype, - errors::InvalidArgument("Invalid data types; op elements ", - DataTypeString(element_dtype_), - " but list elements ", - DataTypeString(l->element_dtype))); + OP_REQUIRES( + c, element_dtype_ == tensor_list->element_dtype, + errors::InvalidArgument( + "Invalid data types; op elements ", DataTypeString(element_dtype_), + " but list elements ", DataTypeString(tensor_list->element_dtype))); Tensor indices = c->input(1); - OP_REQUIRES(c, - indices.NumElements() > 0 || l->element_shape.IsFullyDefined(), - errors::InvalidArgument("Tried to gather 0-elements from " - "a list with non-fully-defined shape: ", - l->element_shape.DebugString())); + OP_REQUIRES( + c, + indices.NumElements() > 0 || + tensor_list->element_shape.IsFullyDefined(), + errors::InvalidArgument("Tried to gather 0-elements from " + "a list with non-fully-defined shape: ", + tensor_list->element_shape.DebugString())); // Compute the shape of the output tensor. // If `element_shape` is fully-defined it gets used. It is assumed that all // requested tensors have the same shape. @@ -178,17 +381,17 @@ class TensorListGather : public OpKernel { // tensor is used and it is checked that all other tensors have the same // shape. TensorShape resulting_shape; - if (!l->element_shape.AsTensorShape(&resulting_shape)) { + if (!tensor_list->element_shape.AsTensorShape(&resulting_shape)) { const int i = indices.flat()(0); OP_REQUIRES( - c, i < l->tensors.size(), + c, i < tensor_list->tensors.size(), errors::InvalidArgument("Index ", i, " out o range; list only has ", - l->tensors.size(), " elements.")); - const Tensor& t = l->tensors[i]; + tensor_list->tensors.size(), " elements.")); + const Tensor& t = tensor_list->tensors[i]; resulting_shape = t.shape(); for (int index = 1; index < indices.NumElements(); ++index) { const int i = indices.flat()(index); - const Tensor& t = l->tensors[i]; + const Tensor& t = tensor_list->tensors[i]; OP_REQUIRES(c, t.shape() == resulting_shape, errors::InvalidArgument( "Tried to gather elements with unequal shapes: ", @@ -204,14 +407,14 @@ class TensorListGather : public OpKernel { } ConstMatrixVector inputs_flat; - inputs_flat.reserve(l->tensors.size()); + inputs_flat.reserve(tensor_list->tensors.size()); for (int index = 0; index < indices.NumElements(); ++index) { const int i = indices.flat()(index); OP_REQUIRES( - c, i < l->tensors.size(), + c, i < tensor_list->tensors.size(), errors::InvalidArgument("Index ", i, " out o range; list only has ", - l->tensors.size(), " elements.")); - const Tensor& t = l->tensors[i]; + tensor_list->tensors.size(), " elements.")); + const Tensor& t = tensor_list->tensors[i]; inputs_flat.emplace_back(new typename TTypes::ConstMatrix( t.shaped({1, t.NumElements()}))); } @@ -289,13 +492,13 @@ class TensorListScatter : public OpKernel { PartialTensorShape element_shape; OP_REQUIRES_OK(c, TensorShapeFromTensor(c->input(2), &element_shape)); TensorList output_list; - const Tensor& t = c->input(0); - output_list.element_dtype = t.dtype(); - OP_REQUIRES(c, TensorShapeUtils::IsVectorOrHigher(t.shape()), + const Tensor& input_tensor = c->input(0); + output_list.element_dtype = input_tensor.dtype(); + OP_REQUIRES(c, TensorShapeUtils::IsVectorOrHigher(input_tensor.shape()), errors::InvalidArgument( "Tensor must be at least a vector, but saw shape: ", - t.shape().DebugString())); - TensorShape output_shape(t.shape()); + input_tensor.shape().DebugString())); + TensorShape output_shape(input_tensor.shape()); output_shape.RemoveDim(0); OP_REQUIRES(c, element_shape.IsCompatibleWith(output_shape), errors::InvalidArgument( @@ -305,11 +508,11 @@ class TensorListScatter : public OpKernel { output_list.tensors.reserve(indices.NumElements()); for (int index = 0; index < indices.NumElements(); ++index) { const int i = indices.flat()(index); - OP_REQUIRES(c, i < t.shape().dim_size(0), - errors::InvalidArgument("Trying to scatter index ", i, - " from tensor with ", - t.shape().dim_size(0), " rows.")); - Tensor tmp = t.Slice(i, i + 1); + OP_REQUIRES(c, i < input_tensor.shape().dim_size(0), + errors::InvalidArgument( + "Trying to scatter index ", i, " from tensor with ", + input_tensor.shape().dim_size(0), " rows.")); + Tensor tmp = input_tensor.Slice(i, i + 1); TensorShape tmp_shape = tmp.shape(); tmp_shape.RemoveDim(0); OP_REQUIRES(c, tmp.CopyFrom(tmp, tmp_shape), @@ -357,40 +560,10 @@ Status TensorListBinaryAdd(OpKernelContext* c, const TensorList& a, for (int i = 0; i < a.tensors.size(); ++i) { const Tensor& a_tensor = a.tensors[i]; const Tensor& b_tensor = b.tensors[i]; - if (a_tensor.dtype() == DT_INVALID) { - out->tensors.push_back(b_tensor); - continue; - } - if (b_tensor.dtype() == DT_INVALID) { - out->tensors.push_back(a_tensor); - continue; - } - if (a_tensor.shape() != b_tensor.shape()) { - // TODO(apassos) support broadcasting additions here? - return errors::InvalidArgument( - "Trying to add two tensors with incompatible element shapes. " - "One is ", - a_tensor.shape().DebugString(), " and the other is ", - b_tensor.shape().DebugString(), " in position ", i); - } Tensor out_tensor; TF_RETURN_IF_ERROR( - c->allocate_temp(a_tensor.dtype(), a_tensor.shape(), &out_tensor)); + BinaryAddTensors(c, a_tensor, b_tensor, &out_tensor)); out->tensors.push_back(out_tensor); - switch (out_tensor.dtype()) { -#define DTYPE_CASE(dtype) \ - case DataTypeToEnum::value: \ - out_tensor.flat().device(c->eigen_device()) = \ - a_tensor.flat() + b_tensor.flat(); \ - break; - - TF_CALL_NUMBER_TYPES(DTYPE_CASE) - -#undef DTYPE_CASE - default: - return errors::InvalidArgument("Trying to add unsupported dtype ", - out_tensor.dtype()); - } } return Status::OK(); } @@ -403,46 +576,7 @@ Status TensorListZerosLike(OpKernelContext* c, const TensorList& x, y->tensors.reserve(x.tensors.size()); for (const Tensor& t : x.tensors) { Tensor out_tensor; - AllocatorAttributes attr; - if (t.dtype() == DT_VARIANT) { - attr.set_on_host(true); - } - TF_RETURN_IF_ERROR( - c->allocate_temp(t.dtype(), t.shape(), &out_tensor, attr)); - switch (out_tensor.dtype()) { -#define DTYPE_CASE(dtype) \ - case DataTypeToEnum::value: \ - out_tensor.flat().device(c->eigen_device()) = \ - out_tensor.flat().constant(dtype(0)); \ - break; - - TF_CALL_POD_TYPES(DTYPE_CASE) - -#undef DTYPE_CASE - - case DT_INVALID: { - // Uninitialized tensor in the TensorList. - out_tensor = Tensor(DT_INVALID); - break; - } - case DataTypeToEnum::value: { - const TensorList* inner_x = t.scalar()().get(); - if (inner_x == nullptr) { - return errors::InvalidArgument("Input handle is not a list. Saw: '", - t.scalar()().DebugString(), - "'"); - } - TensorList inner_y; - TF_RETURN_IF_ERROR(TensorListZerosLike(c, *inner_x, &inner_y)); - out_tensor.scalar()() = std::move(inner_y); - break; - } - - default: - return errors::InvalidArgument( - "Trying to compute zeros_like for unsupported dtype ", - DataTypeString(out_tensor.dtype())); - } + TF_RETURN_IF_ERROR(ZerosLikeTensor(c, t, &out_tensor)); y->tensors.emplace_back(out_tensor); } return Status::OK(); diff --git a/tensorflow/core/kernels/maxpooling_op_gpu.cu.cc b/tensorflow/core/kernels/maxpooling_op_gpu.cu.cc index 0c7a236b2ff..89ffe6494e2 100644 --- a/tensorflow/core/kernels/maxpooling_op_gpu.cu.cc +++ b/tensorflow/core/kernels/maxpooling_op_gpu.cu.cc @@ -384,6 +384,8 @@ bool MaxPoolForwardNoMask_NCHW_VECT_C::operator()( int32* top_data, const Eigen::GpuDevice& d) { const int kThreadsPerBlock = 1024; const int output_size = batch * channels * pooled_height * pooled_width; + if (output_size == 0) + return true; MaxPoolForwardNoMaskKernel_NCHW_VECT_C<<< (output_size + kThreadsPerBlock - 1) / kThreadsPerBlock, kThreadsPerBlock, 0, d.stream()>>>(output_size, bottom_data, height, width, channels, @@ -402,6 +404,8 @@ bool MaxPoolForwardWithOptionalArgmax::operator()( int64* mask, const Eigen::GpuDevice& d, bool propagate_nans) { const int kThreadsPerBlock = 1024; const int output_size = batch * channels * pooled_height * pooled_width; + if (output_size == 0) + return true; if (propagate_nans) { MaxPoolForwardNHWC <<<(output_size + kThreadsPerBlock - 1) / kThreadsPerBlock, @@ -430,6 +434,8 @@ bool MaxPoolBackwardNoMask::operator()( const int kThreadsPerBlock = 1024; const int bottom_size = batch * channels * height * width; + if (bottom_size == 0) + return true; SetZero<<<(bottom_size + kThreadsPerBlock - 1) / kThreadsPerBlock, kThreadsPerBlock, 0, d.stream()>>>(bottom_size, bottom_diff); @@ -449,6 +455,8 @@ bool MaxPoolBackwardWithArgmax::operator()( const int64* mask, const int top_offset, const int bottom_offset, T* bottom_diff, const Eigen::GpuDevice& d) { const int kThreadsPerBlock = 1024; + if (input_size == 0) + return true; SetZero<<<(input_size + kThreadsPerBlock - 1) / kThreadsPerBlock, kThreadsPerBlock, 0, d.stream()>>>(input_size, bottom_diff); MaxPoolBackward<<<(output_size + kThreadsPerBlock - 1) / kThreadsPerBlock, @@ -466,6 +474,8 @@ bool MaxPoolGradBackwardNoMask::operator()( const int pad_l, const T* top_diff, T* bottom_diff, const Eigen::GpuDevice& d) { const int num_kernels = batch * channels * pooled_height * pooled_width; + if (num_kernels == 0) + return true; CudaLaunchConfig config = GetCudaLaunchConfig(num_kernels, d); if (data_format == FORMAT_NHWC) { @@ -489,6 +499,8 @@ bool MaxPoolGradBackwardWithArgmax::operator()( const int output_size, const int input_size, const T* top_diff, const int64* mask, const int top_offset, const int bottom_offset, T* bottom_diff, const Eigen::GpuDevice& d) { + if (input_size == 0) + return true; CudaLaunchConfig config = GetCudaLaunchConfig(output_size, d); MaxPoolGradBackward<<>>(output_size, top_diff, mask, top_offset, diff --git a/tensorflow/core/kernels/mkl_avgpooling_op.cc b/tensorflow/core/kernels/mkl_avgpooling_op.cc index 2409f7e9dc2..939cbd6f96a 100644 --- a/tensorflow/core/kernels/mkl_avgpooling_op.cc +++ b/tensorflow/core/kernels/mkl_avgpooling_op.cc @@ -357,22 +357,21 @@ class MklAvgPoolingGradOp : public OpKernel { if (!outbackprop_in_mkl_format) { // For avgpooling, tensor_in_shape should have 1 dimension, and 4 // elements. - OP_REQUIRES( - context, - tensor_in_shape.dims() == 1 && tensor_in_shape.NumElements() == 4, - errors::InvalidArgument("original input shape must be " - "1-dimensional and 4 elements")); + OP_REQUIRES(context, tensor_in_shape.dims() == 1 && + tensor_in_shape.NumElements() == 4, + errors::InvalidArgument("original input shape must be " + "1-dimensional and 4 elements")); // For avgpooling, out_backprop should have 4 dimensions. - OP_REQUIRES(context, out_backprop.dims() == 4, - errors::InvalidArgument("out_backprop must be " - "4-dimensional")); + OP_REQUIRES( + context, out_backprop.dims() == 4, + errors::InvalidArgument("out_backprop must be 4-dimensional")); } else { // Input in MKL format. // For avgpooling, out_backprop should have 4 dimensions. - OP_REQUIRES(context, out_backprop_shape.GetDimension() == 4, - errors::InvalidArgument("out_backprop must be " - "4-dimensional")); + OP_REQUIRES( + context, out_backprop_shape.GetDimension() == 4, + errors::InvalidArgument("out_backprop must be 4-dimensional")); } // TODO(inteltf): Get outbackprop layout. @@ -484,9 +483,9 @@ class MklAvgPoolingOp : public MklPoolingForwardOpBase { dnn_shape_input.IsMklTensor() ? dnn_shape_input.GetSizesAsMklDnnDims() : is_pool2d ? TFShapeToMklDnnDimsInNCHW(input_tensor.shape(), - this->data_format_tf_) - : TFShapeToMklDnnDimsInNCDHW(input_tensor.shape(), - this->data_format_tf_); + this->data_format_tf_) + : TFShapeToMklDnnDimsInNCDHW(input_tensor.shape(), + this->data_format_tf_); memory::desc input_md = dnn_shape_input.IsMklTensor() ? dnn_shape_input.GetMklLayout() : memory::desc(src_dims, MklDnnType(), @@ -494,9 +493,17 @@ class MklAvgPoolingOp : public MklPoolingForwardOpBase { // Get an average pooling primitive from the op pool MklPoolingFwdPrimitive* pooling_fwd = nullptr; + prop_kind pooling_prop_kind; + bool int8_forward_inference = + std::is_same::value || std::is_same::value; + if (int8_forward_inference) + pooling_prop_kind = prop_kind::forward_inference; + else + pooling_prop_kind = prop_kind::forward_training; MklPoolingParams fwdParams(src_dims, output_dims_mkl_order, filter_dims, strides, padding_left, padding_right, - algorithm::pooling_avg_exclude_padding); + algorithm::pooling_avg_exclude_padding, + pooling_prop_kind); pooling_fwd = MklPoolingFwdPrimitiveFactory::Get(fwdParams); // allocate output tensor @@ -523,10 +530,30 @@ class MklAvgPoolingOp : public MklPoolingForwardOpBase { // execute pooling pooling_fwd->Execute(src_data, dst_data); + + // Pass min, max from input to output + if (int8_forward_inference) { + const Tensor& min_input_t = MklGetInput(context, 1); + const Tensor& max_input_t = MklGetInput(context, 2); + const float min_input = min_input_t.flat()(0); + const float max_input = max_input_t.flat()(0); + + Tensor* output_min = nullptr; + Tensor* output_max = nullptr; + MklDnnShape output_min_mkl_shape, output_max_mkl_shape; + output_min_mkl_shape.SetMklTensor(false); + output_max_mkl_shape.SetMklTensor(false); + AllocateOutputSetMklShape(context, 1, &output_min, {}, + output_min_mkl_shape); + AllocateOutputSetMklShape(context, 2, &output_max, {}, + output_max_mkl_shape); + output_min->flat()(0) = min_input; + output_max->flat()(0) = max_input; + } } catch (mkldnn::error& e) { - string error_msg = "Status: " + std::to_string(e.status) + - ", message: " + string(e.message) + ", in file " + - string(__FILE__) + ":" + std::to_string(__LINE__); + string error_msg = "Status: " + std::to_string(e.status) + ", message: " + + string(e.message) + ", in file " + string(__FILE__) + + ":" + std::to_string(__LINE__); OP_REQUIRES_OK( context, errors::Aborted("Operation received an exception:", error_msg)); @@ -576,24 +603,26 @@ class MklAvgPoolingGradOp : public MklPoolingBackwardOpBase { orig_input_mkl_shape.IsMklTensor() ? orig_input_mkl_shape.GetSizesAsMklDnnDims() : is_pool2d ? TFShapeToMklDnnDimsInNCHW(orig_input_shape, - this->data_format_tf_) - : TFShapeToMklDnnDimsInNCDHW(orig_input_shape, - this->data_format_tf_); + this->data_format_tf_) + : TFShapeToMklDnnDimsInNCDHW(orig_input_shape, + this->data_format_tf_); memory::dims diff_dst_dims = grad_mkl_shape.IsMklTensor() ? grad_mkl_shape.GetSizesAsMklDnnDims() : is_pool2d ? TFShapeToMklDnnDimsInNCHW(grad_tensor.shape(), - this->data_format_tf_) - : TFShapeToMklDnnDimsInNCDHW(grad_tensor.shape(), - this->data_format_tf_); + this->data_format_tf_) + : TFShapeToMklDnnDimsInNCDHW(grad_tensor.shape(), + this->data_format_tf_); memory::dims output_dims_mkl_order; this->GetOutputDims(pool_params, &output_dims_mkl_order); - MklPoolingParams bwdParams(orig_input_dims_mkl_order, - output_dims_mkl_order, filter_dims, strides, - padding_left, padding_right, - algorithm::pooling_avg_exclude_padding); + // Pass prop_kind::forward_training to create a forward primitive + // that is used in the backward pass + MklPoolingParams bwdParams( + orig_input_dims_mkl_order, output_dims_mkl_order, filter_dims, + strides, padding_left, padding_right, + algorithm::pooling_avg_exclude_padding, prop_kind::forward_training); MklPoolingBwdPrimitive* pooling_bwd = MklPoolingBwdPrimitiveFactory::Get(bwdParams); @@ -624,9 +653,9 @@ class MklAvgPoolingGradOp : public MklPoolingBackwardOpBase { // execute pooling op pooling_bwd->Execute(diff_dst_data, diff_src_data); } catch (mkldnn::error& e) { - string error_msg = "Status: " + std::to_string(e.status) + - ", message: " + string(e.message) + ", in file " + - string(__FILE__) + ":" + std::to_string(__LINE__); + string error_msg = "Status: " + std::to_string(e.status) + ", message: " + + string(e.message) + ", in file " + string(__FILE__) + + ":" + std::to_string(__LINE__); OP_REQUIRES_OK(context, errors::Aborted("Compute received an exception:", error_msg)); } @@ -645,28 +674,26 @@ class MklAvgPoolingGradOp : public MklPoolingBackwardOpBase { const MklDnnShape& original_input_mkl_shape, const MklDnnShape& input_gradient_mkl_shape) { if (!original_input_mkl_shape.IsMklTensor()) { - OP_REQUIRES( - context, - tensor_in_shape.dims() == 1 && tensor_in_shape.NumElements() == 4, - errors::InvalidArgument("original input shape must be " - "1-dimensional and 4 elements")); + OP_REQUIRES(context, tensor_in_shape.dims() == 1 && + tensor_in_shape.NumElements() == 4, + errors::InvalidArgument("original input shape must be " + "1-dimensional and 4 elements")); } else { - OP_REQUIRES(context, - original_input_mkl_shape.GetDimension() == 1 && - original_input_mkl_shape.DimSize(0) == 4, + OP_REQUIRES(context, original_input_mkl_shape.GetDimension() == 1 && + original_input_mkl_shape.DimSize(0) == 4, errors::InvalidArgument("original input shape must be " "1-dimensional and 4 elements")); } if (!input_gradient_mkl_shape.IsMklTensor()) { // For avgpooling, input_gradient_diff_dst should have 4 dimensions. - OP_REQUIRES(context, input_gradient_tensor.dims() == 4, - errors::InvalidArgument("Gradient shape must be " - "4-dimensional")); + OP_REQUIRES( + context, input_gradient_tensor.dims() == 4, + errors::InvalidArgument("Gradient shape must be 4-dimensional")); } else { - OP_REQUIRES(context, input_gradient_mkl_shape.GetDimension() == 4, - errors::InvalidArgument("Gradient shape must be " - "4-dimensional")); + OP_REQUIRES( + context, input_gradient_mkl_shape.GetDimension() == 4, + errors::InvalidArgument("Gradient shape must be 4-dimensional")); } } }; // MklAvgPoolingGradOp @@ -691,6 +718,18 @@ REGISTER_KERNEL_BUILDER(Name("_MklAvgPool") .Label(mkl_op_registry::kMklOpLabel), MklAvgPoolingOp); +REGISTER_KERNEL_BUILDER(Name("_MklQuantizedAvgPool") + .Device(DEVICE_CPU) + .TypeConstraint("T") + .Label(mkl_op_registry::kMklQuantizedOpLabel), + MklAvgPoolingOp); + +REGISTER_KERNEL_BUILDER(Name("_MklQuantizedAvgPool") + .Device(DEVICE_CPU) + .TypeConstraint("T") + .Label(mkl_op_registry::kMklQuantizedOpLabel), + MklAvgPoolingOp); + REGISTER_KERNEL_BUILDER(Name("_MklAvgPoolGrad") .Device(DEVICE_CPU) .TypeConstraint("T") diff --git a/tensorflow/core/kernels/mkl_conv_ops.cc b/tensorflow/core/kernels/mkl_conv_ops.cc index 14d134e2d0c..75f08956b4c 100644 --- a/tensorflow/core/kernels/mkl_conv_ops.cc +++ b/tensorflow/core/kernels/mkl_conv_ops.cc @@ -465,19 +465,18 @@ class MklConvOp : public OpKernel { filter.shape().DebugString())); for (int i = 0; i < 3; i++) { - OP_REQUIRES( - context, - FastBoundsCheck(filter.dim_size(i), std::numeric_limits::max()), - errors::InvalidArgument("filter too large")); + OP_REQUIRES(context, FastBoundsCheck(filter.dim_size(i), + std::numeric_limits::max()), + errors::InvalidArgument("filter too large")); } const int64 input_depth = input_in_mkl_format ? GetMklTensorDim(mkl_context.input_shape, 'C') : GetTensorDim(input, data_format_, 'C'); - OP_REQUIRES(context, input_depth == filter.dim_size(2), - errors::InvalidArgument( - "input and filter must have the same depth: ", input_depth, - " vs ", filter.dim_size(2))); + OP_REQUIRES( + context, input_depth == filter.dim_size(2), + errors::InvalidArgument("input and filter must have the same depth: ", + input_depth, " vs ", filter.dim_size(2))); // The last dimension for filter is out_depth. const int out_depth = static_cast(filter.dim_size(3)); @@ -486,10 +485,9 @@ class MklConvOp : public OpKernel { const int64 input_rows_raw = input_in_mkl_format ? GetMklTensorDim(mkl_context.input_shape, 'H') : GetTensorDim(input, data_format_, 'H'); - OP_REQUIRES( - context, - FastBoundsCheck(input_rows_raw, std::numeric_limits::max()), - errors::InvalidArgument("Input rows too large")); + OP_REQUIRES(context, FastBoundsCheck(input_rows_raw, + std::numeric_limits::max()), + errors::InvalidArgument("Input rows too large")); const int input_rows = static_cast(input_rows_raw); const int filter_rows = static_cast(filter.dim_size(0)); @@ -498,10 +496,9 @@ class MklConvOp : public OpKernel { const int64 input_cols_raw = input_in_mkl_format ? GetMklTensorDim(mkl_context.input_shape, 'W') : GetTensorDim(input, data_format_, 'W'); - OP_REQUIRES( - context, - FastBoundsCheck(input_cols_raw, std::numeric_limits::max()), - errors::InvalidArgument("Input cols too large")); + OP_REQUIRES(context, FastBoundsCheck(input_cols_raw, + std::numeric_limits::max()), + errors::InvalidArgument("Input cols too large")); const int input_cols = static_cast(input_cols_raw); const int filter_cols = static_cast(filter.dim_size(1)); @@ -509,10 +506,9 @@ class MklConvOp : public OpKernel { const int64 input_batch_raw = input_in_mkl_format ? GetMklTensorDim(mkl_context.input_shape, 'N') : GetTensorDim(input, data_format_, 'N'); - OP_REQUIRES( - context, - FastBoundsCheck(input_batch_raw, std::numeric_limits::max()), - errors::InvalidArgument("batch is too large")); + OP_REQUIRES(context, FastBoundsCheck(input_batch_raw, + std::numeric_limits::max()), + errors::InvalidArgument("batch is too large")); const int batch = static_cast(input_batch_raw); // For now we take the stride from the second and third dimensions only (we @@ -893,17 +889,15 @@ class MklConvOp : public OpKernel { OP_REQUIRES(context, dilations_.size() == 5, errors::InvalidArgument("Dilation rates field must " "specify 5 dimensions")); - OP_REQUIRES(context, - (GetTensorDim(dilations_, data_format_, 'N') == 1 && - GetTensorDim(dilations_, data_format_, 'C') == 1), + OP_REQUIRES(context, (GetTensorDim(dilations_, data_format_, 'N') == 1 && + GetTensorDim(dilations_, data_format_, 'C') == 1), errors::InvalidArgument( "Current implementation does not yet support " "dilations rates in the batch and depth dimensions.")); OP_REQUIRES( - context, - (GetTensorDim(dilations_, data_format_, '0') > 0 && - GetTensorDim(dilations_, data_format_, '1') > 0 && - GetTensorDim(dilations_, data_format_, '2') > 0), + context, (GetTensorDim(dilations_, data_format_, '0') > 0 && + GetTensorDim(dilations_, data_format_, '1') > 0 && + GetTensorDim(dilations_, data_format_, '2') > 0), errors::InvalidArgument("Dilated rates should be larger than 0.")); } } @@ -1067,8 +1061,14 @@ class MklConvOp : public OpKernel { Tfilter* filter_data = nullptr; if (filter_md.data.format != conv_fwd->GetFilterMemoryFormat()) { filter.SetUsrMem(filter_md, &filter_tensor); - filter.CheckReorderToOpMem(conv_fwd_pd.get()->weights_primitive_desc(), - filter.GetTensorBuffer(filter_out_tensor)); + if (filter_out_tensor == nullptr) { + filter.CheckReorderToOpMem( + conv_fwd_pd.get()->weights_primitive_desc()); + } else { + filter.CheckReorderToOpMem( + conv_fwd_pd.get()->weights_primitive_desc(), + filter.GetTensorBuffer(filter_out_tensor)); + } filter_data = static_cast(filter.GetOpMem().get_data_handle()); } else { @@ -1468,7 +1468,7 @@ class MklQuantizedConv2DSumReluOp {"sum", {scale_summand / scale_output}}); else params.post_op_params.push_back( - {"sum", {2.0 * scale_summand / scale_output}}); + {"sum", {2.0f * scale_summand / scale_output}}); } else { params.post_op_params.push_back({"sum", {1.0}}); } @@ -1533,8 +1533,8 @@ class MklQuantizedConv2DSumReluOp const float max_filter = context->input(5 + bias_index_offset).flat()(0); - reorder_sum_scale = 255.0 * 127.0 / - (std::max(std::abs(max_input), std::abs(min_input)) * + reorder_sum_scale = + 255.0 * 127.0 / (std::max(std::abs(max_input), std::abs(min_input)) * std::max(std::abs(max_filter), std::abs(min_filter))); std::vector scales; scales.push_back(reorder_sum_scale); diff --git a/tensorflow/core/kernels/mkl_lrn_op.cc b/tensorflow/core/kernels/mkl_lrn_op.cc index 22ff4cd80fe..407ce5d653d 100644 --- a/tensorflow/core/kernels/mkl_lrn_op.cc +++ b/tensorflow/core/kernels/mkl_lrn_op.cc @@ -22,32 +22,26 @@ limitations under the License. #define EIGEN_USE_THREADS #include -#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor" +#include "mkldnn.hpp" #include "tensorflow/core/framework/op_kernel.h" #include "tensorflow/core/framework/register_types.h" #include "tensorflow/core/framework/tensor.h" #include "tensorflow/core/kernels/bounds_check.h" #include "tensorflow/core/kernels/ops_util.h" #include "tensorflow/core/lib/core/errors.h" +#include "tensorflow/core/util/mkl_util.h" #include "tensorflow/core/util/tensor_format.h" +#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor" #if !defined(IS_MOBILE_PLATFORM) #include "tensorflow/core/util/work_sharder.h" #endif -#ifndef INTEL_MKL_ML_ONLY -#include "mkldnn.hpp" using mkldnn::lrn_across_channels; using mkldnn::lrn_backward; using mkldnn::lrn_forward; using mkldnn::prop_kind; using mkldnn::stream; -#else -#include "mkl_dnn.h" -#include "mkl_dnn_types.h" -#endif - -#include "tensorflow/core/util/mkl_util.h" namespace tensorflow { @@ -69,8 +63,6 @@ void GetBandMatrix(int depth, int depth_radius, } // namespace -#ifdef INTEL_MKL_ML_ONLY - template class MklLRNOp : public OpKernel { public: @@ -79,675 +71,10 @@ class MklLRNOp : public OpKernel { explicit MklLRNOp(OpKernelConstruction* context) : OpKernel(context) { int64 depth_radius64; OP_REQUIRES_OK(context, context->GetAttr("depth_radius", &depth_radius64)); - OP_REQUIRES( - context, - FastBoundsCheck(depth_radius64, std::numeric_limits::max()), - errors::InvalidArgument("depth_radius = ", depth_radius64, - " larger than int max")); - depth_radius_ = static_cast(depth_radius64); - - OP_REQUIRES_OK(context, context->GetAttr("bias", &bias_)); - OP_REQUIRES_OK(context, context->GetAttr("alpha", &alpha_)); - OP_REQUIRES_OK(context, context->GetAttr("beta", &beta_)); - workspace_enabled_ = false; - OP_REQUIRES_OK(context, - context->GetAttr("workspace_enabled", &workspace_enabled_)); - } - - void Compute(OpKernelContext* context) override { - MklLRNOpContext mkl_context; - - const Tensor& input = MklGetInput(context, 0); - GetMklShape(context, 0, &mkl_context.input_shape); - bool input_in_mkl_format = mkl_context.input_shape.IsMklTensor(); - - // Sanity checks - mkl_context.in_dims = input_in_mkl_format - ? mkl_context.input_shape.GetDimension() - : input.dims(); - OP_REQUIRES(context, mkl_context.in_dims == 4, - errors::InvalidArgument("input must be 4-dimensional")); - OP_REQUIRES( - context, - FastBoundsCheck(input.NumElements(), std::numeric_limits::max()), - errors::InvalidArgument("argument to LRN too large")); - - if (!input_in_mkl_format) { - mkl_context.MklDefaultToEigen(context, depth_radius_, bias_, alpha_, - beta_, input); - return; - } - - if (input_in_mkl_format) { - // MKL supports normalization over channel dimension only - if (mkl_context.input_shape.tf_dim_idx(mkl_context.in_dims - 1) == - MklDims::C) { - mkl_context.lt_input = - static_cast(mkl_context.input_shape.GetCurLayout()); - workspace_enabled_ = true; - } else { - Tensor converted_tensor = - ConvertMklToTF(context, input, mkl_context.input_shape); - mkl_context.MklDefaultToEigen(context, depth_radius_, bias_, alpha_, - beta_, converted_tensor); - return; - } - } - - int kernel_size = 2 * depth_radius_ + 1; - - CHECK_EQ(dnnLRNCreateForward_F32( - &mkl_context.lrn_fwd, NULL, mkl_context.lt_input, kernel_size, - static_cast(alpha_ * kernel_size), beta_, bias_), - E_SUCCESS); - - // Allocate output tensor and shape - Tensor* output = nullptr; - Tensor* workspace = nullptr; - - // Convert Inputs if needed - Tensor mkl_tmp_input_buf_tensor; - mkl_context.MklPrepareLRNInputs(context, &mkl_tmp_input_buf_tensor); - - // Allocate Layer Outputs - mkl_context.MklAllocateOutputs(context, &output, &workspace, - workspace_enabled_); - - Tensor mkl_tmp_workspace_buf_tensor; - mkl_context.MklPrepareLRNOutputs(context, output, workspace, - &mkl_tmp_workspace_buf_tensor, - workspace_enabled_); - - // Execute LRN. - CHECK_EQ(dnnExecute_F32(mkl_context.lrn_fwd, mkl_context.lrn_res), - E_SUCCESS); - - // Release MKL resources. - mkl_context.MklCleanup(); - } - - private: - typedef struct { - size_t in_dims; - size_t in_sizes[4]; - size_t in_strides[4]; - size_t out_sizes[4]; - size_t out_strides[4]; - MklShape input_shape; - dnnPrimitive_t lrn_fwd = nullptr; - dnnPrimitive_t convert_input = nullptr; - dnnLayout_t lt_input = nullptr; - dnnLayout_t lt_internal_input = nullptr; - dnnLayout_t lt_internal_workspace = nullptr; - dnnLayout_t lt_internal_output = nullptr; - void* lrn_res[dnnResourceNumber]; - - // Convert Inputs if needed - void MklPrepareLRNInputs(OpKernelContext* context, - Tensor* mkl_tmp_input_buf_tensor) { - const Tensor& input = MklGetInput(context, 0); - void* mkl_buf_input = - const_cast(static_cast(input.flat().data())); - - CHECK_EQ(dnnLayoutCreateFromPrimitive_F32(<_internal_input, lrn_fwd, - dnnResourceSrc), - E_SUCCESS); - - void* mkl_buf_convert_input = nullptr; - bool mkl_convert_input = false; - mkl_convert_input = !dnnLayoutCompare_F32(lt_internal_input, lt_input); - - if (mkl_convert_input) { - CHECK_EQ(dnnConversionCreate_F32(&convert_input, lt_input, - lt_internal_input), - E_SUCCESS); - AllocTmpBuffer(context, mkl_tmp_input_buf_tensor, lt_internal_input, - &mkl_buf_convert_input); - CHECK_EQ(dnnConversionExecute_F32(convert_input, mkl_buf_input, - mkl_buf_convert_input), - E_SUCCESS); - dnnDelete_F32(convert_input); - } - - lrn_res[dnnResourceSrc] = - (mkl_convert_input) ? mkl_buf_convert_input : mkl_buf_input; - } - - // Allocate Layer Outputs - void MklAllocateOutputs(OpKernelContext* context, Tensor** output, - Tensor** workspace, bool workspace_enabled_) { - TensorShape mkl_output_tf_shape; /* First tensor */ - MklShape mkl_output_mkl_shape; /* Second tensor */ - - mkl_output_mkl_shape.SetMklTensor(true); - mkl_output_mkl_shape.SetMklLayout(lrn_fwd, dnnResourceDst); - mkl_output_mkl_shape.SetTfLayout(in_dims, input_shape.GetSizes(), - input_shape.GetStrides()); - mkl_output_mkl_shape.SetTfDimOrder(in_dims, - input_shape.GetTfToMklDimMap()); - mkl_output_tf_shape.AddDim( - dnnLayoutGetMemorySize_F32( - static_cast(mkl_output_mkl_shape.GetMklLayout())) / - sizeof(T)); - AllocateOutputSetMklShape(context, 0, output, - mkl_output_tf_shape /* First tensor */, - mkl_output_mkl_shape /* Second Tensor */); - - if (workspace_enabled_) { - TensorShape mkl_workspace_tf_shape; /* First tensor */ - MklShape mkl_workspace_mkl_shape; /* Second tensor */ - mkl_workspace_mkl_shape.SetMklTensor(false); - mkl_workspace_mkl_shape.SetMklLayout(lrn_fwd, dnnResourceWorkspace); - // Assumes workspace has same TF layout and TF dim order as input - mkl_workspace_mkl_shape.SetTfLayout(in_dims, input_shape.GetSizes(), - input_shape.GetStrides()); - mkl_workspace_mkl_shape.SetTfDimOrder(in_dims, - input_shape.GetTfToMklDimMap()); - mkl_workspace_tf_shape.AddDim( - dnnLayoutGetMemorySize_F32(static_cast( - mkl_workspace_mkl_shape.GetMklLayout())) / - sizeof(T)); - AllocateOutputSetMklShape(context, 1, workspace, - mkl_workspace_tf_shape /* First tensor */, - mkl_workspace_mkl_shape /* Second Tensor */); - } - } - - void MklPrepareLRNOutputs(OpKernelContext* context, Tensor* output, - Tensor* workspace, - Tensor* mkl_tmp_workspace_buf_tensor, - bool workspace_enabled_) { - CHECK_EQ(dnnLayoutCreateFromPrimitive_F32(<_internal_workspace, lrn_fwd, - dnnResourceWorkspace), - E_SUCCESS); - - CHECK_EQ(dnnLayoutCreateFromPrimitive_F32(<_internal_output, lrn_fwd, - dnnResourceDst), - E_SUCCESS); - - void* mkl_buf_output = - const_cast(static_cast(output->flat().data())); - lrn_res[dnnResourceDst] = mkl_buf_output; - - void* mkl_buf_workspace = nullptr; - if (workspace_enabled_) { - mkl_buf_workspace = const_cast( - static_cast(workspace->flat().data())); - } else { - AllocTmpBuffer(context, mkl_tmp_workspace_buf_tensor, - lt_internal_workspace, &mkl_buf_workspace); - } - lrn_res[dnnResourceWorkspace] = mkl_buf_workspace; - } - - // Fallback implementation - Taken from lrn_op.cc - // TODO(inteltf) Check if we can use EigenLRNOp directly instead of making a - // copy. - void MklDefaultToEigen(OpKernelContext* context, int depth_radius_, - float bias_, float alpha_, float beta_, - const Tensor& input) { - const int batch = static_cast(input.dim_size(0)); - const int rows = static_cast(input.dim_size(1)); - const int cols = static_cast(input.dim_size(2)); - const int depth = static_cast(input.dim_size(3)); - const int nodes = cols * rows; - - auto in_shaped = input.shaped({nodes * batch, depth}); - // Multiplying the input with the band matrix has the effect of reducing - // the - // correct patch along the depth. - Eigen::Tensor multiplier(depth, depth); - GetBandMatrix(depth, depth_radius_, &multiplier); - - Tensor *output, *workspace; - MklShape mkl_output_mkl_shape, mkl_workspace_mkl_shape; - mkl_output_mkl_shape.SetMklTensor(false); - mkl_output_mkl_shape.SetDimensions(4); - AllocateOutputSetMklShape(context, 0, &output, input.shape(), - mkl_output_mkl_shape); - - mkl_workspace_mkl_shape.SetMklTensor(false); - mkl_workspace_mkl_shape.SetDimensions(4); - AllocateOutputSetMklShape(context, 1, &workspace, input.shape(), - mkl_workspace_mkl_shape); - - auto out_shaped = output->shaped({nodes * batch, depth}); - Eigen::array dims = {{DimPair(1, 0)}}; - auto tmp = in_shaped.square().contract(multiplier, dims) * alpha_ + bias_; - if (beta_ == T(1)) { - out_shaped.device(context->eigen_cpu_device()) = - in_shaped * tmp.inverse(); - } else if (beta_ == T(0.5)) { - out_shaped.device(context->eigen_cpu_device()) = - in_shaped * tmp.rsqrt(); - } else { - out_shaped.device(context->eigen_cpu_device()) = - in_shaped * (tmp.log() * -beta_).exp(); - } - } - - // Release MKL resources. - void MklCleanup() { - dnnDelete_F32(lrn_fwd); - dnnLayoutDelete_F32(lt_internal_input); - dnnLayoutDelete_F32(lt_internal_workspace); - dnnLayoutDelete_F32(lt_internal_output); - } - } MklLRNOpContext; - - typedef typename Eigen::Tensor::DimensionPair DimPair; - - bool workspace_enabled_; - int depth_radius_; - float bias_; - float alpha_; - float beta_; -}; - -template -class MklLRNGradOp : public OpKernel { - public: - explicit MklLRNGradOp(OpKernelConstruction* context) : OpKernel(context) { - int64 depth_radius64; - OP_REQUIRES_OK(context, context->GetAttr("depth_radius", &depth_radius64)); - OP_REQUIRES( - context, - FastBoundsCheck(depth_radius64, std::numeric_limits::max()), - errors::InvalidArgument("depth_radius = ", depth_radius64, - " larger than int max")); - depth_radius_ = static_cast(depth_radius64); - OP_REQUIRES_OK(context, context->GetAttr("bias", &bias_)); - OP_REQUIRES_OK(context, context->GetAttr("alpha", &alpha_)); - OP_REQUIRES_OK(context, context->GetAttr("beta", &beta_)); - workspace_enabled_ = false; - OP_REQUIRES_OK(context, - context->GetAttr("workspace_enabled", &workspace_enabled_)); - } - - void Compute(OpKernelContext* context) override { - MklLRNGradOpContext mkl_context; - mkl_context.depth_radius_ = depth_radius_; - mkl_context.bias_ = bias_; - mkl_context.alpha_ = alpha_; - mkl_context.beta_ = beta_; - - const Tensor& in_grads = MklGetInput(context, 0); - const Tensor& in_image = MklGetInput(context, 1); - const Tensor& out_image = MklGetInput(context, 2); - - GetMklShape(context, 0, &mkl_context.ingrad_shape); - GetMklShape(context, 1, &mkl_context.inimage_shape); - GetMklShape(context, 2, &mkl_context.outimage_shape); - - bool ingrad_in_mkl_format = mkl_context.ingrad_shape.IsMklTensor(); - bool inimage_in_mkl_format = mkl_context.inimage_shape.IsMklTensor(); - bool outimage_in_mkl_format = mkl_context.outimage_shape.IsMklTensor(); - - mkl_context.in_dims = inimage_in_mkl_format - ? mkl_context.inimage_shape.GetDimension() - : in_image.dims(); - OP_REQUIRES(context, mkl_context.in_dims == 4, - errors::InvalidArgument("input images must be 4-dimensional")); - - if (!workspace_enabled_) { - mkl_context.MklDefaultToEigen(context); - return; - } - - if (ingrad_in_mkl_format || inimage_in_mkl_format) { - const MklShape* tmp_mkl_shape = (ingrad_in_mkl_format) - ? &mkl_context.ingrad_shape - : &mkl_context.inimage_shape; - if (tmp_mkl_shape->tf_dim_idx(mkl_context.in_dims - 1) != MklDims::C) { - // Fallback to eigen - mkl_context.MklDefaultToEigen(context); - return; - } else { // MKL supports normalization over channel dimension only - for (int i = 0; i < mkl_context.in_dims; i++) { - mkl_context.in_sizes[i] = mkl_context.out_sizes[i] = - tmp_mkl_shape->GetSizes()[i]; - mkl_context.in_strides[i] = mkl_context.out_strides[i] = - tmp_mkl_shape->GetStrides()[i]; - } - } - } else { - // Fallback to eigen - mkl_context.MklDefaultToEigen(context); - return; - } - - // Dimensions check for sanity purpose - if (ingrad_in_mkl_format) { - OP_REQUIRES( - context, mkl_context.ingrad_shape.GetDimension() == 4, - errors::InvalidArgument("input gradient must be 4-dimensional")); - } else { - OP_REQUIRES( - context, in_grads.dims() == 4, - errors::InvalidArgument("input gradient must be 4-dimensional")); - } - - if (outimage_in_mkl_format) { - OP_REQUIRES( - context, mkl_context.outimage_shape.GetDimension() == 4, - errors::InvalidArgument("Output image must be 4-dimensional")); - } else { - OP_REQUIRES( - context, out_image.dims() == 4, - errors::InvalidArgument("Output image must be 4-dimensional")); - } - - // Prepare mkl input layout - mkl_context.MklPrepareLRNInputsLayouts(context); - int ksize = 2 * depth_radius_ + 1; - - CHECK_EQ(dnnLRNCreateBackward_F32( - &mkl_context.lrn_bwd, NULL, mkl_context.lt_input, - mkl_context.lt_output, ksize, - static_cast(alpha_ * ksize), beta_, bias_), - E_SUCCESS); - - // Allocate output tensor and shape. - TensorShape mkl_output_tf_shape; /* First tensor */ - MklShape mkl_output_mkl_shape; /* Second tensor */ - mkl_output_mkl_shape.SetMklTensor(true); - CHECK_NE(mkl_context.lrn_bwd, nullptr); - mkl_output_mkl_shape.SetMklLayout(mkl_context.lrn_bwd, dnnResourceDiffSrc); - mkl_output_mkl_shape.SetTfLayout(mkl_context.in_dims, mkl_context.out_sizes, - mkl_context.out_strides); - if (ingrad_in_mkl_format) { - mkl_output_mkl_shape.SetTfDimOrder( - mkl_context.in_dims, mkl_context.ingrad_shape.GetTfToMklDimMap()); - } else { - mkl_output_mkl_shape.SetTfDimOrder( - mkl_context.in_dims, mkl_context.inimage_shape.GetTfToMklDimMap()); - } - mkl_output_tf_shape.AddDim( - dnnLayoutGetMemorySize_F32( - static_cast(mkl_output_mkl_shape.GetMklLayout())) / - sizeof(T)); - Tensor* output = nullptr; - AllocateOutputSetMklShape(context, 0, &output, mkl_output_tf_shape, - mkl_output_mkl_shape); - - // Get pointers to output data. - void* user_output = - const_cast(static_cast(output->flat().data())); - - Tensor mkl_tmp_input_buf_tensor, mkl_tmp_image_buf_tensor, - mkl_tmp_outimage_buf_tensor; - // Convert Inputs if needed - mkl_context.MklPrepareLRNGradInput(context, &mkl_tmp_input_buf_tensor, - &mkl_tmp_image_buf_tensor, - &mkl_tmp_outimage_buf_tensor); - - // We do not do any conversion for output. But we simply emit it - // in MKL format. - mkl_context.res_lrn_bwd[dnnResourceDiffSrc] = user_output; - // Execute LRN backward using dnnExecute - CHECK_EQ(dnnExecute_F32(mkl_context.lrn_bwd, mkl_context.res_lrn_bwd), - E_SUCCESS); - // Release MKL resources. - mkl_context.Mklcleanup(); - } - - private: - typedef struct { - int depth_radius_; - float bias_; - float alpha_; - float beta_; - size_t in_dims; - size_t in_sizes[4]; - size_t in_strides[4]; - size_t out_sizes[4]; - size_t out_strides[4]; - MklShape ingrad_shape, inimage_shape, outimage_shape; - dnnPrimitive_t lrn_bwd = nullptr; - dnnPrimitive_t convert_input = nullptr; - dnnLayout_t lt_input = nullptr; - dnnLayout_t lt_output = nullptr; - dnnLayout_t lt_bdw_input = nullptr; - dnnLayout_t lt_workspace = nullptr; - dnnLayout_t lt_internal_input = nullptr; - void* res_lrn_bwd[dnnResourceNumber]; - - // prepare mkl input - void MklPrepareLRNInputsLayouts(OpKernelContext* context) { - bool ingrad_in_mkl_format = ingrad_shape.IsMklTensor(); - bool inimage_in_mkl_format = inimage_shape.IsMklTensor(); - if (!ingrad_in_mkl_format) { - CHECK_EQ(dnnLayoutCreate_F32(<_input, in_dims, in_sizes, in_strides), - E_SUCCESS); - } else { - lt_input = static_cast(ingrad_shape.GetCurLayout()); - } - - if (!inimage_in_mkl_format) { - CHECK_EQ( - dnnLayoutCreate_F32(<_output, in_dims, out_sizes, out_strides), - E_SUCCESS); - } else { - lt_output = static_cast(inimage_shape.GetCurLayout()); - } - } - - // convert input if needed - void MklPrepareLRNGradInput(OpKernelContext* context, - Tensor* mkl_tmp_input_buf_tensor, - Tensor* mkl_tmp_image_buf_tensor, - Tensor* mkl_tmp_outimage_buf_tensor) { - const Tensor& in_grads = MklGetInput(context, 0); - const Tensor& in_image = MklGetInput(context, 1); - const Tensor& workspace = MklGetInput( - context, - 3); /*Worskpsace is enabled, get the buffer to the workspace */ - - void* user_input = const_cast( - static_cast(in_grads.flat().data())); - void* user_fwd_input = const_cast( - static_cast(in_image.flat().data())); - void* workspace_buffer = const_cast( - static_cast(workspace.flat().data())); - - CHECK_EQ(dnnLayoutCreateFromPrimitive_F32(<_workspace, lrn_bwd, - dnnResourceWorkspace), - E_SUCCESS); - CHECK_EQ(dnnLayoutCreateFromPrimitive_F32(<_bdw_input, lrn_bwd, - dnnResourceDiffDst), - E_SUCCESS); - CHECK_EQ(dnnLayoutCreateFromPrimitive_F32(<_internal_input, lrn_bwd, - dnnResourceSrc), - E_SUCCESS); - - bool ingrad_in_mkl_format = ingrad_shape.IsMklTensor(); - if (ingrad_in_mkl_format) { - if (!dnnLayoutCompare_F32(lt_bdw_input, lt_input)) { - AllocTmpBuffer(context, mkl_tmp_input_buf_tensor, lt_bdw_input, - &res_lrn_bwd[dnnResourceDiffDst]); - ingrad_shape.GetConvertedFlatData(lt_bdw_input, user_input, - res_lrn_bwd[dnnResourceDiffDst]); - } else { - res_lrn_bwd[dnnResourceDiffDst] = user_input; - } - } else { - if (!dnnLayoutCompare_F32(lt_bdw_input, lt_input)) { - CHECK_EQ( - dnnConversionCreate_F32(&convert_input, lt_input, lt_bdw_input), - E_SUCCESS); - - AllocTmpBuffer(context, mkl_tmp_input_buf_tensor, lt_bdw_input, - &res_lrn_bwd[dnnResourceDiffDst]); - CHECK_EQ(dnnConversionExecute_F32(convert_input, user_input, - res_lrn_bwd[dnnResourceDiffDst]), - E_SUCCESS); - dnnDelete_F32(convert_input); - } else { - res_lrn_bwd[dnnResourceDiffDst] = user_input; - } - } - - bool inimage_in_mkl_format = inimage_shape.IsMklTensor(); - if (inimage_in_mkl_format) { - if (!dnnLayoutCompare_F32( - lt_internal_input, - static_cast(inimage_shape.GetCurLayout()))) { - AllocTmpBuffer(context, mkl_tmp_image_buf_tensor, lt_internal_input, - &res_lrn_bwd[dnnResourceSrc]); - ingrad_shape.GetConvertedFlatData(lt_internal_input, user_fwd_input, - res_lrn_bwd[dnnResourceSrc]); - } else { - res_lrn_bwd[dnnResourceSrc] = user_fwd_input; - } - } else { - if (!dnnLayoutCompare_F32( - lt_internal_input, - static_cast(inimage_shape.GetCurLayout()))) { - CHECK_EQ(dnnConversionCreate_F32( - &convert_input, - static_cast(inimage_shape.GetCurLayout()), - lt_internal_input), - E_SUCCESS); - - AllocTmpBuffer(context, mkl_tmp_image_buf_tensor, lt_internal_input, - &res_lrn_bwd[dnnResourceSrc]); - CHECK_EQ(dnnConversionExecute_F32(convert_input, user_fwd_input, - res_lrn_bwd[dnnResourceSrc]), - E_SUCCESS); - dnnDelete_F32(convert_input); - } else { - res_lrn_bwd[dnnResourceSrc] = user_fwd_input; - } - } - - res_lrn_bwd[dnnResourceWorkspace] = workspace_buffer; - } - - // Fallback implementation - Taken from lrn_op.cc - // TODO(intelft) Check if we can use EigenLRNOp directly instead of making a - // copy. - void MklDefaultToEigen(OpKernelContext* context) { - Tensor in_grads; - Tensor in_image; - Tensor out_image; - - GetMklShape(context, 0, &ingrad_shape); - GetMklShape(context, 1, &inimage_shape); - GetMklShape(context, 2, &outimage_shape); - - if (ingrad_shape.IsMklTensor()) { - in_grads = - ConvertMklToTF(context, MklGetInput(context, 0), ingrad_shape); - } else { - in_grads = MklGetInput(context, 0); - } - - if (inimage_shape.IsMklTensor()) { - in_image = - ConvertMklToTF(context, MklGetInput(context, 1), inimage_shape); - } else { - in_image = MklGetInput(context, 1); - } - - if (outimage_shape.IsMklTensor()) { - out_image = - ConvertMklToTF(context, MklGetInput(context, 2), outimage_shape); - } else { - out_image = MklGetInput(context, 2); - } - - const int64 batch = static_cast(in_grads.dim_size(0)); - const int64 rows = static_cast(in_grads.dim_size(1)); - const int64 cols = static_cast(in_grads.dim_size(2)); - const int64 depth = static_cast(in_grads.dim_size(3)); - const auto nodes = cols * rows; - - auto grads_shaped = in_grads.shaped({nodes * batch, depth}); - - auto in_shaped = in_image.shaped({nodes * batch, depth}); - auto activations = out_image.shaped({nodes * batch, depth}); - - Tensor* output; - MklShape mkl_output_mkl_shape; - mkl_output_mkl_shape.SetMklTensor(false); - mkl_output_mkl_shape.SetDimensions(4); - AllocateOutputSetMklShape(context, 0, &output, in_grads.shape(), - mkl_output_mkl_shape); - - auto out_shaped = output->shaped({nodes * batch, depth}); - out_shaped.setZero(); - auto shard = [this, activations, in_shaped, grads_shaped, out_shaped, - depth](int64 begin, int64 end) { - for (int64 i = begin; i < end; ++i) { - for (int64 j = 0; j < depth; ++j) { - int64 depth_begin = std::max(0, j - depth_radius_); - int64 depth_end = std::min(depth, j + depth_radius_ + 1); - - T norm(0); - for (int64 k = depth_begin; k < depth_end; ++k) { - norm += in_shaped(i, k) * in_shaped(i, k); - } - norm = alpha_ * norm + bias_; - DCHECK_GT(norm, T(1e-6)); - for (int64 k = depth_begin; k < depth_end; ++k) { - T dyi = T(-2) * alpha_ * beta_ * in_shaped(i, k) * - activations(i, j) / norm; - if (k == j) { - dyi += Eigen::numext::pow(norm, -beta_); - } - dyi *= grads_shaped(i, j); - const_cast::Tensor&>(out_shaped)(i, k) += - dyi; - } - } - } - }; - auto worker_threads = - *(context->device()->tensorflow_cpu_worker_threads()); - Shard(worker_threads.num_threads, worker_threads.workers, nodes * batch, - depth * depth, shard); - } - - // release mkl resources - void Mklcleanup() { - bool ingrad_in_mkl_format = ingrad_shape.IsMklTensor(); - bool inimage_in_mkl_format = inimage_shape.IsMklTensor(); - if (!ingrad_in_mkl_format) { - CHECK_EQ(dnnLayoutDelete_F32(lt_input), E_SUCCESS); - } - - if (!inimage_in_mkl_format) { - CHECK_EQ(dnnLayoutDelete_F32(lt_output), E_SUCCESS); - } - dnnDelete_F32(lrn_bwd); - dnnLayoutDelete_F32(lt_bdw_input); - dnnLayoutDelete_F32(lt_workspace); - } - } MklLRNGradOpContext; - - typedef typename Eigen::Tensor::DimensionPair DimPair; - bool workspace_enabled_; - int depth_radius_; - float bias_; - float alpha_; - float beta_; -}; - -#else - -template -class MklLRNOp : public OpKernel { - public: - ~MklLRNOp() {} - - explicit MklLRNOp(OpKernelConstruction* context) : OpKernel(context) { - int64 depth_radius64; - OP_REQUIRES_OK(context, context->GetAttr("depth_radius", &depth_radius64)); - OP_REQUIRES( - context, - FastBoundsCheck(depth_radius64, std::numeric_limits::max()), - errors::InvalidArgument("depth_radius = ", depth_radius64, - " larger than int max")); + OP_REQUIRES(context, FastBoundsCheck(depth_radius64, + std::numeric_limits::max()), + errors::InvalidArgument("depth_radius = ", depth_radius64, + " larger than int max")); depth_radius_ = static_cast(depth_radius64); OP_REQUIRES_OK(context, context->GetAttr("bias", &bias_)); @@ -833,9 +160,9 @@ class MklLRNOp : public OpKernel { PrepareAndExecuteNet(lrn_prim_desc, &src_dnn_data, &dst_dnn_data, &workspace_dnn_data); } catch (mkldnn::error& e) { - string error_msg = "Status: " + std::to_string(e.status) + - ", message: " + string(e.message) + ", in file " + - string(__FILE__) + ":" + std::to_string(__LINE__); + string error_msg = "Status: " + std::to_string(e.status) + ", message: " + + string(e.message) + ", in file " + string(__FILE__) + + ":" + std::to_string(__LINE__); OP_REQUIRES_OK( context, errors::Aborted("Operation received an exception:", error_msg)); @@ -847,7 +174,6 @@ class MklLRNOp : public OpKernel { MklDnnData* src_dnn_data, MklDnnData* dst_dnn_data, MklDnnData* wksp_dnn_data = nullptr) { - // Check for input reorder src_dnn_data->CheckReorderToOpMem(lrn_fwd_desc.src_primitive_desc()); @@ -965,16 +291,14 @@ class MklLRNOp : public OpKernel { if (src_dnn_shape.IsMklTensor()) { OP_REQUIRES(context, src_dnn_shape.GetDimension() == 4, errors::InvalidArgument("input must be 4-dimensional")); - OP_REQUIRES(context, - FastBoundsCheck(src_tensor.NumElements(), - std::numeric_limits::max()), + OP_REQUIRES(context, FastBoundsCheck(src_tensor.NumElements(), + std::numeric_limits::max()), errors::InvalidArgument("argument to LRN too large")); } else { OP_REQUIRES(context, src_tensor.dims() == 4, errors::InvalidArgument("input must be 4-dimensional")); - OP_REQUIRES(context, - FastBoundsCheck(src_tensor.NumElements(), - std::numeric_limits::max()), + OP_REQUIRES(context, FastBoundsCheck(src_tensor.NumElements(), + std::numeric_limits::max()), errors::InvalidArgument("argument to LRN too large")); } } @@ -994,11 +318,10 @@ class MklLRNGradOp : public OpKernel { explicit MklLRNGradOp(OpKernelConstruction* context) : OpKernel(context) { int64 depth_radius64; OP_REQUIRES_OK(context, context->GetAttr("depth_radius", &depth_radius64)); - OP_REQUIRES( - context, - FastBoundsCheck(depth_radius64, std::numeric_limits::max()), - errors::InvalidArgument("depth_radius = ", depth_radius64, - " larger than int max")); + OP_REQUIRES(context, FastBoundsCheck(depth_radius64, + std::numeric_limits::max()), + errors::InvalidArgument("depth_radius = ", depth_radius64, + " larger than int max")); depth_radius_ = static_cast(depth_radius64); OP_REQUIRES_OK(context, context->GetAttr("bias", &bias_)); OP_REQUIRES_OK(context, context->GetAttr("alpha", &alpha_)); @@ -1105,9 +428,9 @@ class MklLRNGradOp : public OpKernel { memory::primitive_desc(target_diff_dst_md, cpu_engine), &workspace_dnn_data); } catch (mkldnn::error& e) { - string error_msg = "Status: " + std::to_string(e.status) + - ", message: " + string(e.message) + ", in file " + - string(__FILE__) + ":" + std::to_string(__LINE__); + string error_msg = "Status: " + std::to_string(e.status) + ", message: " + + string(e.message) + ", in file " + string(__FILE__) + + ":" + std::to_string(__LINE__); OP_REQUIRES_OK( context, errors::Aborted("Operation received an exception:", error_msg)); @@ -1160,7 +483,6 @@ class MklLRNGradOp : public OpKernel { MklDnnData* output_diff_src, const memory::primitive_desc& target_diff_dst_pd, const MklDnnData* workspace_dnn_data = nullptr) { - // Check for input reordering on the diff dst input input_gradient_diff_dst->CheckReorderToOpMem( lrn_bkwd_desc.diff_dst_primitive_desc()); @@ -1345,8 +667,6 @@ class MklLRNGradOp : public OpKernel { float beta_; }; -#endif // INTEL_MKL_ML_ONLY - #define REGISTER_MKL_LRN_CPU(T) \ REGISTER_KERNEL_BUILDER(Name("_MklLRN") \ .Device(DEVICE_CPU) \ diff --git a/tensorflow/core/kernels/mkl_maxpooling_op.cc b/tensorflow/core/kernels/mkl_maxpooling_op.cc index 256d48f4d5d..0697251c7dc 100644 --- a/tensorflow/core/kernels/mkl_maxpooling_op.cc +++ b/tensorflow/core/kernels/mkl_maxpooling_op.cc @@ -399,19 +399,18 @@ class MklMaxPoolingGradOp : public OpKernel { if (workspace_enabled == false) { if (convert_input != nullptr) { if (input_in_mkl_format == false) { - CHECK_EQ(dnnConversionExecute_F32( - convert_input, - const_cast(static_cast( - tensor_in.flat().data())), - input_buf), - E_SUCCESS); + CHECK_EQ( + dnnConversionExecute_F32( + convert_input, const_cast(static_cast( + tensor_in.flat().data())), + input_buf), + E_SUCCESS); CHECK_EQ(dnnDelete_F32(convert_input), E_SUCCESS); convert_input = nullptr; } else { input_shape.GetConvertedFlatData( - lt_input_prim, - const_cast( - static_cast(tensor_in.flat().data())), + lt_input_prim, const_cast(static_cast( + tensor_in.flat().data())), input_buf); } pooling_resfwd[dnnResourceSrc] = input_buf; @@ -456,9 +455,8 @@ class MklMaxPoolingGradOp : public OpKernel { CHECK_EQ(dnnDelete_F32(convert_outbackprop), E_SUCCESS); } else { output_backprop_shape.GetConvertedFlatData( - lt_outbackprop_prim, - const_cast( - static_cast(out_backprop.flat().data())), + lt_outbackprop_prim, const_cast(static_cast( + out_backprop.flat().data())), outbackprop_buf); } pooling_res[dnnResourceDiffDst] = outbackprop_buf; @@ -520,7 +518,6 @@ class MklMaxPoolingOp : public MklPoolingForwardOpBase { MklDnnData dnn_data_input(&cpu_engine); MklDnnData dnn_data_output(&cpu_engine); - MklDnnData dnn_data_wksp(&cpu_engine); // initialize variables for the pooling op MklPoolParameters pool_params; @@ -550,13 +547,13 @@ class MklMaxPoolingOp : public MklPoolingForwardOpBase { dnn_shape_input.IsMklTensor() ? dnn_shape_input.GetMklLayout() : is_pool2d ? memory::desc( - TFShapeToMklDnnDimsInNCHW(input_tensor_shape, - this->data_format_tf_), - MklDnnType(), this->data_format_mkldnn_) - : memory::desc( - TFShapeToMklDnnDimsInNCDHW( - input_tensor_shape, this->data_format_tf_), - MklDnnType(), this->data_format_mkldnn_); + TFShapeToMklDnnDimsInNCHW( + input_tensor_shape, this->data_format_tf_), + MklDnnType(), this->data_format_mkldnn_) + : memory::desc( + TFShapeToMklDnnDimsInNCDHW( + input_tensor_shape, this->data_format_tf_), + MklDnnType(), this->data_format_mkldnn_); // Get src/filter/stride/padding information memory::dims src_dims = @@ -564,17 +561,24 @@ class MklMaxPoolingOp : public MklPoolingForwardOpBase { ? dnn_shape_input.GetSizesAsMklDnnDims() : is_pool2d ? TFShapeToMklDnnDimsInNCHW(input_tensor.shape(), this->data_format_tf_) - : TFShapeToMklDnnDimsInNCDHW(input_tensor.shape(), - this->data_format_tf_); + : TFShapeToMklDnnDimsInNCDHW(input_tensor.shape(), + this->data_format_tf_); memory::dims filter_dims, strides, padding_left, padding_right; this->PoolParamsToDims(&pool_params, &filter_dims, &strides, &padding_left, &padding_right, is_pool2d); // Get a pooling op from the cached pool MklPoolingFwdPrimitive* pooling_fwd = nullptr; + prop_kind pooling_prop_kind; + bool int8_forward_inference = + std::is_same::value || std::is_same::value; + if (int8_forward_inference) + pooling_prop_kind = prop_kind::forward_inference; + else + pooling_prop_kind = prop_kind::forward_training; MklPoolingParams fwdParams(src_dims, output_dims_mkl_order, filter_dims, strides, padding_left, padding_right, - algorithm::pooling_max); + algorithm::pooling_max, pooling_prop_kind); pooling_fwd = MklPoolingFwdPrimitiveFactory::Get(fwdParams); // allocate output tensor @@ -586,10 +590,6 @@ class MklMaxPoolingOp : public MklPoolingForwardOpBase { pooling_fwd->GetDstMemoryFormat(), output_tensor); - AllocateWorkspaceTensor(context, *(pooling_fwd->GetPoolingFwdPd()), - &dnn_data_wksp); - OP_REQUIRES_OK(context, context->status()); - // check wehther we need to reorder src const T* src_data = input_tensor.flat().data(); if (input_md.data.format != pooling_fwd->GetSrcMemoryFormat()) { @@ -603,14 +603,43 @@ class MklMaxPoolingOp : public MklPoolingForwardOpBase { } T* dst_data = output_tensor->flat().data(); - void* ws_data = dnn_data_wksp.GetOpMem().get_data_handle(); - // execute pooling op - pooling_fwd->Execute(src_data, dst_data, ws_data); + if (int8_forward_inference) { + // Execute pooling op + pooling_fwd->Execute(src_data, dst_data); + + // pass min, max from input to output + const Tensor& min_input_t = MklGetInput(context, 1); + const Tensor& max_input_t = MklGetInput(context, 2); + const float min_input = min_input_t.flat()(0); + const float max_input = max_input_t.flat()(0); + + Tensor* output_min = nullptr; + Tensor* output_max = nullptr; + MklDnnShape output_min_mkl_shape, output_max_mkl_shape; + output_min_mkl_shape.SetMklTensor(false); + output_max_mkl_shape.SetMklTensor(false); + AllocateOutputSetMklShape(context, 1, &output_min, {}, + output_min_mkl_shape); + AllocateOutputSetMklShape(context, 2, &output_max, {}, + output_max_mkl_shape); + output_min->flat()(0) = min_input; + output_max->flat()(0) = max_input; + } else { + MklDnnData dnn_data_wksp(&cpu_engine); + AllocateWorkspaceTensor(context, *(pooling_fwd->GetPoolingFwdPd()), + &dnn_data_wksp); + OP_REQUIRES_OK(context, context->status()); + T* ws_data = + static_cast(dnn_data_wksp.GetOpMem().get_data_handle()); + + // execute pooling op + pooling_fwd->Execute(src_data, dst_data, ws_data); + } } catch (mkldnn::error& e) { - string error_msg = "Status: " + std::to_string(e.status) + - ", message: " + string(e.message) + ", in file " + - string(__FILE__) + ":" + std::to_string(__LINE__); + string error_msg = "Status: " + std::to_string(e.status) + ", message: " + + string(e.message) + ", in file " + string(__FILE__) + + ":" + std::to_string(__LINE__); OP_REQUIRES_OK(context, errors::Aborted("Compute received an exception:", error_msg)); } @@ -684,24 +713,25 @@ class MklMaxPoolingGradOp : public MklPoolingBackwardOpBase { orig_input_mkl_shape.IsMklTensor() ? orig_input_mkl_shape.GetSizesAsMklDnnDims() : is_pool2d ? TFShapeToMklDnnDimsInNCHW(orig_input_shape, - this->data_format_tf_) - : TFShapeToMklDnnDimsInNCDHW(orig_input_shape, - this->data_format_tf_); + this->data_format_tf_) + : TFShapeToMklDnnDimsInNCDHW(orig_input_shape, + this->data_format_tf_); memory::dims diff_dst_dims = grad_mkl_shape.IsMklTensor() ? grad_mkl_shape.GetSizesAsMklDnnDims() : is_pool2d ? TFShapeToMklDnnDimsInNCHW(grad_tensor.shape(), - this->data_format_tf_) - : TFShapeToMklDnnDimsInNCDHW(grad_tensor.shape(), - this->data_format_tf_); + this->data_format_tf_) + : TFShapeToMklDnnDimsInNCDHW(grad_tensor.shape(), + this->data_format_tf_); memory::dims output_dims_mkl_order; this->GetOutputDims(pool_params, &output_dims_mkl_order); MklPoolingParams bwdParams( orig_input_dims_mkl_order, output_dims_mkl_order, filter_dims, - strides, padding_left, padding_right, algorithm::pooling_max); + strides, padding_left, padding_right, algorithm::pooling_max, + prop_kind::forward_training); MklPoolingBwdPrimitive* pooling_bwd = MklPoolingBwdPrimitiveFactory::Get(bwdParams); @@ -751,9 +781,9 @@ class MklMaxPoolingGradOp : public MklPoolingBackwardOpBase { // execute pooling pooling_bwd->Execute(diff_dst_data, diff_src_data, ws_data); } catch (mkldnn::error& e) { - string error_msg = "Status:" + std::to_string(e.status) + - ", message: " + string(e.message) + ". in file " + - string(__FILE__) + ":" + std::to_string(__LINE__); + string error_msg = "Status:" + std::to_string(e.status) + ", message: " + + string(e.message) + ". in file " + string(__FILE__) + + ":" + std::to_string(__LINE__); OP_REQUIRES_OK(context, errors::Aborted("Compute received an exception:", error_msg)); } @@ -788,39 +818,38 @@ class MklMaxPoolingGradOp : public MklPoolingBackwardOpBase { const MklDnnShape& workspace_mkl_shape) { if (!orig_input_mkl_shape.IsMklTensor()) { OP_REQUIRES(context, orig_input_tensor.dims() == 4, - errors::InvalidArgument("Original input shape must be " - "4-dimensional")); + errors::InvalidArgument( + "Original input shape must be 4-dimensional")); } else { OP_REQUIRES(context, orig_input_mkl_shape.GetDimension() == 4, - errors::InvalidArgument("Original input shape must be " - "4-dimensional")); + errors::InvalidArgument( + "Original input shape must be 4-dimensional")); } if (!orig_output_mkl_shape.IsMklTensor()) { - OP_REQUIRES(context, orig_output_tensor.dims() == 4, - errors::InvalidArgument("Original output must be " - "4-dimensional")); + OP_REQUIRES( + context, orig_output_tensor.dims() == 4, + errors::InvalidArgument("Original output must be 4-dimensional")); } else { - OP_REQUIRES(context, orig_output_mkl_shape.GetDimension() == 4, - errors::InvalidArgument("Original output must be " - "4-dimensional")); + OP_REQUIRES( + context, orig_output_mkl_shape.GetDimension() == 4, + errors::InvalidArgument("Original output must be 4-dimensional")); } if (!grad_mkl_shape.IsMklTensor()) { OP_REQUIRES(context, grad_tensor.dims() == 4, errors::InvalidArgument("Gradient must be 4-dimensional")); } else { OP_REQUIRES(context, grad_mkl_shape.GetDimension() == 4, - errors::InvalidArgument("Gradient must be " - "4-dimensional")); + errors::InvalidArgument("Gradient must be 4-dimensional")); } if (this->workspace_enabled_) { // The workspace should not be an MKL tensor OP_REQUIRES(context, workspace_mkl_shape.IsMklTensor() == false, - errors::InvalidArgument("Workspace tensor should not" - " be an MKL Tensor.")); + errors::InvalidArgument( + "Workspace tensor should not be an MKL Tensor.")); // It should only have one dimension - OP_REQUIRES(context, workspace_tensor.dims() == 1, - errors::InvalidArgument("Workspace tensor must be " - "1-dimensional")); + OP_REQUIRES( + context, workspace_tensor.dims() == 1, + errors::InvalidArgument("Workspace tensor must be 1-dimensional")); } else { OP_REQUIRES( context, this->workspace_enabled_, @@ -852,6 +881,18 @@ REGISTER_KERNEL_BUILDER(Name("_MklMaxPool") .Label(mkl_op_registry::kMklOpLabel), MklMaxPoolingOp); +REGISTER_KERNEL_BUILDER(Name("_MklQuantizedMaxPool") + .Device(DEVICE_CPU) + .TypeConstraint("T") + .Label(mkl_op_registry::kMklQuantizedOpLabel), + MklMaxPoolingOp); + +REGISTER_KERNEL_BUILDER(Name("_MklQuantizedMaxPool") + .Device(DEVICE_CPU) + .TypeConstraint("T") + .Label(mkl_op_registry::kMklQuantizedOpLabel), + MklMaxPoolingOp); + REGISTER_KERNEL_BUILDER(Name("_MklMaxPoolGrad") .Device(DEVICE_CPU) .TypeConstraint("T") diff --git a/tensorflow/core/kernels/mkl_pooling_ops_common.cc b/tensorflow/core/kernels/mkl_pooling_ops_common.cc index 5398e6113f5..dc84d3941e7 100644 --- a/tensorflow/core/kernels/mkl_pooling_ops_common.cc +++ b/tensorflow/core/kernels/mkl_pooling_ops_common.cc @@ -41,28 +41,33 @@ void MklPoolingFwdPrimitive::Setup(const MklPoolingParams& fwdParams) { << "Pooling algorithm kind is not supported"; context_.alg_kind = fwdParams.alg_kind; + context_.prop_kind = fwdParams.prop_kind; + // create memory desc // FIXME: Pooling doesn't expose to get the src_primitive_desc, // so src format is currently hard-coded. // A utility function is used to do this, // which may be broken with future CPU architectures bool is_2d = (fwdParams.src_dims.size() == 4); - context_.src_md.reset( - new memory::desc({fwdParams.src_dims}, MklDnnType(), - get_desired_format(fwdParams.src_dims[1], is_2d))); + if (std::is_same::value || std::is_same::value) + context_.src_fmt = is_2d ? memory::format::nhwc : memory::format::ndhwc; + else + context_.src_fmt = get_desired_format(fwdParams.src_dims[1], is_2d); + + context_.src_md.reset(new memory::desc({fwdParams.src_dims}, MklDnnType(), + context_.src_fmt)); context_.dst_md.reset(new memory::desc({fwdParams.dst_dims}, MklDnnType(), memory::format::any)); // create a pooling descriptor context_.fwd_desc.reset(new pooling_forward::desc( - prop_kind::forward_training, fwdParams.alg_kind, *context_.src_md, + fwdParams.prop_kind, fwdParams.alg_kind, *context_.src_md, *context_.dst_md, fwdParams.strides, fwdParams.filter_dims, fwdParams.padding_left, fwdParams.padding_right, padding_kind::zero)); context_.fwd_pd.reset( new pooling_forward::primitive_desc(*context_.fwd_desc, cpu_engine_)); // store expected primitive format - context_.src_fmt = get_desired_format(fwdParams.src_dims[1], is_2d); context_.dst_fmt = static_cast( context_.fwd_pd.get()->dst_primitive_desc().desc().data.format); @@ -74,7 +79,8 @@ void MklPoolingFwdPrimitive::Setup(const MklPoolingParams& fwdParams) { new memory(context_.fwd_pd.get()->dst_primitive_desc(), DummyData)); // for max pooling, need to return workspace(ws) for backward computing - if (fwdParams.alg_kind == pooling_max) { + if (fwdParams.alg_kind == pooling_max && + fwdParams.prop_kind == prop_kind::forward_training) { auto ws_pd = context_.fwd_pd.get()->workspace_primitive_desc().desc().data; // store workspace's dims and format to create workspace tensor context_.ws_fmt = static_cast(ws_pd.format); @@ -101,7 +107,9 @@ void MklPoolingFwdPrimitive::Execute(const T* src_data, T* dst_data, context_.src_mem->set_data_handle( static_cast(const_cast(src_data))); context_.dst_mem->set_data_handle(static_cast(dst_data)); - if (context_.alg_kind == pooling_max) { // max pooling must have ws + if (context_.alg_kind == pooling_max && + context_.prop_kind == + prop_kind::forward_training) { // max pooling must have ws DCHECK(ws_data != nullptr); context_.ws_mem->set_data_handle(ws_data); } @@ -110,13 +118,17 @@ void MklPoolingFwdPrimitive::Execute(const T* src_data, T* dst_data, // set back data handle context_.src_mem->set_data_handle(DummyData); context_.dst_mem->set_data_handle(DummyData); - if (context_.alg_kind == pooling_max) { // max pooling must have ws + if (context_.alg_kind == pooling_max && + context_.prop_kind == + prop_kind::forward_training) { // max pooling must have ws DCHECK(ws_data != nullptr); context_.ws_mem->set_data_handle(DummyData); } } template class MklPoolingFwdPrimitive; +template class MklPoolingFwdPrimitive; +template class MklPoolingFwdPrimitive; template void MklPoolingBwdPrimitive::Setup(const MklPoolingParams& bwdParams) { @@ -143,7 +155,7 @@ void MklPoolingBwdPrimitive::Setup(const MklPoolingParams& bwdParams) { // create a forward primitive, // which will be used as a hint for creating backward primitive context_.fwd_desc.reset(new pooling_forward::desc( - prop_kind::forward_training, bwdParams.alg_kind, *context_.diff_src_md, + bwdParams.prop_kind, bwdParams.alg_kind, *context_.diff_src_md, *context_.diff_dst_md, bwdParams.strides, bwdParams.filter_dims, bwdParams.padding_left, bwdParams.padding_right, padding_kind::zero)); context_.fwd_pd.reset( diff --git a/tensorflow/core/kernels/mkl_pooling_ops_common.h b/tensorflow/core/kernels/mkl_pooling_ops_common.h index 49f799d7ba2..8a60c3be91f 100644 --- a/tensorflow/core/kernels/mkl_pooling_ops_common.h +++ b/tensorflow/core/kernels/mkl_pooling_ops_common.h @@ -18,8 +18,8 @@ limitations under the License. #ifdef INTEL_MKL #include -#include #include +#include #include "tensorflow/core/util/mkl_util.h" #include "tensorflow/core/util/padding.h" @@ -50,18 +50,20 @@ struct MklPoolingParams { memory::dims padding_left; memory::dims padding_right; mkldnn::algorithm alg_kind; + mkldnn::prop_kind prop_kind; MklPoolingParams(memory::dims src_dims, memory::dims dst_dims, memory::dims filter_dims, memory::dims strides, memory::dims padding_left, memory::dims padding_right, - mkldnn::algorithm alg_kind) + mkldnn::algorithm alg_kind, mkldnn::prop_kind prop_kind) : src_dims(src_dims), dst_dims(dst_dims), filter_dims(filter_dims), strides(strides), padding_left(padding_left), padding_right(padding_right), - alg_kind(alg_kind) {} + alg_kind(alg_kind), + prop_kind(prop_kind) {} }; template @@ -97,6 +99,9 @@ class MklPoolingFwdPrimitive : public MklPrimitive { // algorithm mkldnn::algorithm alg_kind; + // Kind of propagation, forward or backward + mkldnn::prop_kind prop_kind; + // expected memory format memory::format src_fmt; memory::format dst_fmt; @@ -187,6 +192,7 @@ class MklPoolingFwdPrimitiveFactory : public MklPrimitiveFactory { key_creator.AddAsKey(fwdParams.padding_left); key_creator.AddAsKey(fwdParams.padding_right); key_creator.AddAsKey(static_cast(fwdParams.alg_kind)); + key_creator.AddAsKey(static_cast(fwdParams.prop_kind)); return key_creator.GetKey(); } @@ -443,7 +449,12 @@ class MklPoolingOpBase : public OpKernel { explicit MklPoolingOpBase(OpKernelConstruction* context) : OpKernel(context), workspace_enabled_(false) { string data_format; - OP_REQUIRES_OK(context, context->GetAttr("data_format", &data_format)); + if (std::is_same::value || std::is_same::value) { + // current quantized convolution doesn't have data_format attribute. + data_format = "NHWC"; + } else { + OP_REQUIRES_OK(context, context->GetAttr("data_format", &data_format)); + } OP_REQUIRES(context, FormatFromString(data_format, &this->data_format_tf_), errors::InvalidArgument("Invalid data format")); OP_REQUIRES_OK(context, context->GetAttr("ksize", &this->ksize_)); @@ -461,7 +472,7 @@ class MklPoolingOpBase : public OpKernel { bool is_pool2d = (this->ksize_.size() == 4); this->data_format_mkldnn_ = is_pool2d ? TFDataFormatToMklDnnDataFormat(this->data_format_tf_) - : TFDataFormatToMklDnn3DDataFormat(this->data_format_tf_); + : TFDataFormatToMklDnn3DDataFormat(this->data_format_tf_); // We may not get this attribute for this node if it does not go through // graph rewrite pass. So we do not check for error while retrieving this @@ -655,10 +666,10 @@ class MklPoolingForwardOpBase : public MklPoolingOpBase { OP_REQUIRES(context, input_tensor.dims() == 4 || input_tensor.dims() == 5, errors::InvalidArgument("Input must be 4 or 5-dimensional")); } else { - OP_REQUIRES(context, input_mkl_shape.GetDimension() == 4 || - input_mkl_shape.GetDimension() == 5, - errors::InvalidArgument("Input shape must be " - "4 or 5-dimensional")); + OP_REQUIRES( + context, input_mkl_shape.GetDimension() == 4 || + input_mkl_shape.GetDimension() == 5, + errors::InvalidArgument("Input shape must be 4 or 5-dimensional")); } } // .Input("value: T") diff --git a/tensorflow/core/kernels/mkl_quantized_pooling_ops_test.cc b/tensorflow/core/kernels/mkl_quantized_pooling_ops_test.cc new file mode 100644 index 00000000000..7c1e32d6e35 --- /dev/null +++ b/tensorflow/core/kernels/mkl_quantized_pooling_ops_test.cc @@ -0,0 +1,201 @@ +/* Copyright 2015 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#ifdef INTEL_MKL +#define EIGEN_USE_THREADS + +#include "tensorflow/core/framework/allocator.h" +#include "tensorflow/core/framework/fake_input.h" +#include "tensorflow/core/framework/node_def_builder.h" +#include "tensorflow/core/framework/op_kernel.h" +#include "tensorflow/core/framework/tensor.h" +#include "tensorflow/core/framework/tensor_testutil.h" +#include "tensorflow/core/framework/types.h" +#include "tensorflow/core/framework/types.pb.h" +#include "tensorflow/core/kernels/ops_testutil.h" +#include "tensorflow/core/kernels/ops_util.h" +#include "tensorflow/core/kernels/quantization_utils.h" +#include "tensorflow/core/lib/core/status_test_util.h" +#include "tensorflow/core/platform/test.h" + +namespace tensorflow { + +// Helper class for converting MKL tensors to TF tensors and comparing to +// expected values + +static const uint8 dummy_tensor[] = {0, 0, 0, 0, 0, 0, 0, 0}; +static const TensorShape dummy_shape({8}); + +class ConvMklToTF : public OpsTestBase { + public: + template + void ConvertMKL2TF(DataType dtype, const Tensor& first, const Tensor& second, + Tensor& output) { + // Create an MKL to TF conversion node and execute it + TF_EXPECT_OK(NodeDefBuilder("mkl_to_tf_op", "_MklToTf") + .Input(FakeInput(dtype)) // Input + .Input(FakeInput(DT_UINT8)) // Mkl second tensor + .Attr("T", dtype) + .Attr("_kernel", "MklOp") + .Finalize(node_def())); + TF_EXPECT_OK(InitOp()); + AddInputFromArray(first.shape(), first.flat()); + AddInputFromArray(second.shape(), second.flat()); + TF_ASSERT_OK(RunOpKernel()); + + output = *GetOutput(0); + } + void TestBody(){}; +}; + +class QuantizedPoolingTest : public OpsTestBase {}; + +TEST_F(QuantizedPoolingTest, SmallAveragePooling) { + const int ksize = 2; + const int stride = 2; + TF_ASSERT_OK(NodeDefBuilder("quantized_avg_pool_op", "_MklQuantizedAvgPool") + .Input(FakeInput(DT_QUINT8)) + .Input(FakeInput(DT_FLOAT)) + .Input(FakeInput(DT_FLOAT)) + .Input(FakeInput(DT_UINT8)) // MKl second tensor + .Input(FakeInput(DT_UINT8)) // MKl second tensor + .Input(FakeInput(DT_UINT8)) // MKl second tensor + .Attr("T", DataTypeToEnum::v()) + .Attr("ksize", {1, ksize, ksize, 1}) + .Attr("strides", {1, stride, stride, 1}) + .Attr("padding", "SAME") + .Attr("_kernel", "QuantizedMklOp") + .Finalize(node_def())); + TF_ASSERT_OK(InitOp()); + const float input_min = 0.0f; + const float input_max = 255.0f; + const int input_height = 4; + const int input_width = 4; + const int input_channels = 2; + Tensor input_float(DT_FLOAT, {1, input_height, input_width, input_channels}); + test::FillValues( + &input_float, + {1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, + 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32}); + Tensor input_quantized = + FloatTensorToQuantized(input_float, input_min, input_max); + + const int expected_width = input_width / stride; + const int expected_height = input_height / stride; + + // The input pools we are averaging. (NHWC input, quantized.) + // 0th channel 1st channel + // 1 3 | 5 7 2 4 | 6 8 + // 9 11 | 13 15 10 12 | 14 16 + // ------------- ------------- + // 17 19 | 21 23 18 20 | 22 24 + // 25 27 | 29 31 26 28 | 30 32 + Tensor expected_float(DT_FLOAT, + {1, expected_height, expected_width, input_channels}); + test::FillValues(&expected_float, {6, 7, 10, 11, 22, 23, 26, 27}); + + AddInputFromArray(input_quantized.shape(), + input_quantized.flat()); + AddInputFromArray(TensorShape({1}), {input_min}); + AddInputFromArray(TensorShape({1}), {input_max}); + AddInputFromArray(dummy_shape, dummy_tensor); + AddInputFromArray(dummy_shape, dummy_tensor); + AddInputFromArray(dummy_shape, dummy_tensor); + + TF_ASSERT_OK(RunOpKernel()); + + const Tensor& output = *GetOutput(0); + const Tensor& mkl_shape_tensor = *GetOutput(3); + ConvMklToTF conv_comp; + Tensor output_quantized; + conv_comp.ConvertMKL2TF(DT_QUINT8, output, mkl_shape_tensor, + output_quantized); + + const float output_min = GetOutput(1)->flat()(0); + const float output_max = GetOutput(2)->flat()(0); + Tensor output_float = + QuantizedTensorToFloat(output_quantized, output_min, output_max); + + test::ExpectTensorNear(expected_float, output_float, 0.2); +} + +TEST_F(QuantizedPoolingTest, SmallMaxPooling) { + const int ksize = 2; + const int stride = 2; + TF_ASSERT_OK(NodeDefBuilder("quantized_max_pool_op", "_MklQuantizedMaxPool") + .Input(FakeInput(DT_QUINT8)) + .Input(FakeInput(DT_FLOAT)) + .Input(FakeInput(DT_FLOAT)) + .Input(FakeInput(DT_UINT8)) // MKl second tensor + .Input(FakeInput(DT_UINT8)) // MKl second tensor + .Input(FakeInput(DT_UINT8)) // MKl second tensor + .Attr("T", DataTypeToEnum::v()) + .Attr("ksize", {1, ksize, ksize, 1}) + .Attr("strides", {1, stride, stride, 1}) + .Attr("padding", "SAME") + .Attr("_kernel", "QuantizedMklOp") + .Finalize(node_def())); + TF_ASSERT_OK(InitOp()); + const float input_min = 0.0f; + const float input_max = 255.0f; + const int input_height = 4; + const int input_width = 4; + const int input_channels = 2; + Tensor input_float(DT_FLOAT, {1, input_height, input_width, input_channels}); + test::FillValues( + &input_float, + {1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, + 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32}); + Tensor input_quantized = + FloatTensorToQuantized(input_float, input_min, input_max); + const int expected_width = input_width / stride; + const int expected_height = input_height / stride; + + // The max is computed from these input pools. (NHWC input, quantized.) + // 0th channel 1st channel + // 1 3 | 5 7 2 4 | 6 8 + // 9 11 | 13 15 10 12 | 14 16 + // ------------- ------------- + // 17 19 | 21 23 18 20 | 22 24 + // 25 27 | 29 31 26 28 | 30 32 + + Tensor expected_float(DT_FLOAT, + {1, expected_height, expected_width, input_channels}); + test::FillValues(&expected_float, {11, 12, 15, 16, 27, 28, 31, 32}); + AddInputFromArray(input_quantized.shape(), + input_quantized.flat()); + AddInputFromArray(TensorShape({1}), {input_min}); + AddInputFromArray(TensorShape({1}), {input_max}); + AddInputFromArray(dummy_shape, dummy_tensor); + AddInputFromArray(dummy_shape, dummy_tensor); + AddInputFromArray(dummy_shape, dummy_tensor); + + TF_ASSERT_OK(RunOpKernel()); + + const Tensor& output = *GetOutput(0); + const Tensor& mkl_shape_tensor = *GetOutput(3); + ConvMklToTF conv_comp; + Tensor output_quantized; + conv_comp.ConvertMKL2TF(DT_QUINT8, output, mkl_shape_tensor, + output_quantized); + + const float output_min = GetOutput(1)->flat()(0); + const float output_max = GetOutput(2)->flat()(0); + Tensor output_float = + QuantizedTensorToFloat(output_quantized, output_min, output_max); + + test::ExpectTensorNear(expected_float, output_float, 0.2); +} +} // namespace tensorflow +#endif diff --git a/tensorflow/core/kernels/mkl_softmax_op.cc b/tensorflow/core/kernels/mkl_softmax_op.cc index cfab529662f..3bf17bc449b 100644 --- a/tensorflow/core/kernels/mkl_softmax_op.cc +++ b/tensorflow/core/kernels/mkl_softmax_op.cc @@ -56,7 +56,7 @@ class MklSoftmaxOp : public OpKernel { MklDnnShape src_mkl_shape; GetMklShape(context, src_idx, &src_mkl_shape); - // src_dims is the dimenstion of src_tensor + // src_dims is the dimension of src_tensor // dim of the dst will also be same as src_dims auto src_tf_shape = src_mkl_shape.IsMklTensor() ? src_mkl_shape.GetTfShape() @@ -68,7 +68,7 @@ class MklSoftmaxOp : public OpKernel { // Here "x" data format in MKL is used for 1 dim tensor, "nc" for 2 dim tensor, // "tnc" for 3 dim tensor, "nchw" for 4 dim tensor, and "ncdhw" for 5 dim tensor. // Each of the simbols has the following meaning: - // n = batch, c = channels, t = sequence lenght, h = height, + // n = batch, c = channels, t = sequence length, h = height, // w = width, d = depth switch (input_dims) { case 1: diff --git a/tensorflow/core/kernels/partitioned_function_ops.cc b/tensorflow/core/kernels/partitioned_function_ops.cc index 71e506e5e6f..30a8be141cb 100644 --- a/tensorflow/core/kernels/partitioned_function_ops.cc +++ b/tensorflow/core/kernels/partitioned_function_ops.cc @@ -28,6 +28,7 @@ limitations under the License. #include "tensorflow/core/grappler/grappler_item.h" #include "tensorflow/core/grappler/optimizers/meta_optimizer.h" #include "tensorflow/core/grappler/utils/functions.h" +#include "tensorflow/core/protobuf/config.pb.h" #include "tensorflow/core/protobuf/rewriter_config.pb.h" #include "tensorflow/core/util/ptr_util.h" #include "tensorflow/core/util/reffed_status_callback.h" @@ -50,12 +51,29 @@ class PartitionedCallOp : public AsyncOpKernel { public: explicit PartitionedCallOp(OpKernelConstruction* ctx) : AsyncOpKernel(ctx) { OP_REQUIRES_OK(ctx, ctx->GetAttr("f", &func_)); - string rewriter_config_serialized; - OP_REQUIRES_OK(ctx, ctx->GetAttr("config", &rewriter_config_serialized)); + string deprecated_config_serialized; + OP_REQUIRES_OK(ctx, ctx->GetAttr("config", &deprecated_config_serialized)); + string config_proto_serialized; + OP_REQUIRES_OK(ctx, ctx->GetAttr("config_proto", &config_proto_serialized)); OP_REQUIRES( - ctx, rewriter_config_.ParseFromString(rewriter_config_serialized), - errors::InvalidArgument("Unable to parse rewriter_config string as " - "tensorflow::RewriterConfig proto.")); + ctx, + deprecated_config_serialized.empty() || config_proto_serialized.empty(), + errors::InvalidArgument("Provided both 'config' and 'config_proto' but " + "only one should be provided. Note the " + "'config' option is deprecated.")); + if (!deprecated_config_serialized.empty()) { + OP_REQUIRES(ctx, + config_proto_.mutable_graph_options() + ->mutable_rewrite_options() + ->ParseFromString(deprecated_config_serialized), + errors::InvalidArgument("Unable to parse config string as " + "tensorflow::RewriteOptions proto.")); + } else { + OP_REQUIRES( + ctx, config_proto_.ParseFromString(config_proto_serialized), + errors::InvalidArgument("Unable to parse config_proto string as " + "tensorflow::ConfigProto proto.")); + } OP_REQUIRES_OK(ctx, ctx->GetAttr("executor_type", &executor_type_)); } @@ -435,7 +453,7 @@ class PartitionedCallOp : public AsyncOpKernel { }, rendez, std::move(done), std::placeholders::_1); auto* refcounted_done = new ReffedStatusCallback(std::move(callback)); - for (int i = 1; i < handles->size(); ++i) { + for (int i = 0; i < handles->size(); ++i) { refcounted_done->Ref(); } @@ -489,6 +507,7 @@ class PartitionedCallOp : public AsyncOpKernel { }); } } + refcounted_done->Unref(); } string UniquifyFunctionName(const FunctionLibraryDefinition* function_library, @@ -506,12 +525,18 @@ class PartitionedCallOp : public AsyncOpKernel { FunctionLibraryDefinition* flib, const DeviceSet& device_set, Device* cpu_device, std::unique_ptr* graph) { - if (!tensorflow::grappler::MetaOptimizerEnabled(rewriter_config_)) { + if (!tensorflow::grappler::MetaOptimizerEnabled(config_proto_)) { return Status::OK(); } tensorflow::grappler::GrapplerItem item; + // Add all available devices so that inlined function can be placed. + for (const Device* d : device_set.devices()) { + Status added_device = item.AddDevice(d->name()); + if (!added_device.ok()) VLOG(3) << added_device.error_message(); + } + // Add fetches so that the graph can be pruned. for (Node* node : ret_nodes) { item.fetch.push_back(node->name()); @@ -530,7 +555,7 @@ class PartitionedCallOp : public AsyncOpKernel { // TODO(nareshmodi): Consider adding and using the more generic GraphOptions // proto (which also contain the OptimizerOptions). TF_RETURN_IF_ERROR(tensorflow::grappler::RunMetaOptimizer( - item, rewriter_config_, cpu_device, &cluster, &out_graph)); + item, config_proto_, cpu_device, &cluster, &out_graph)); std::unique_ptr optimized_graph(new Graph(OpRegistry::Global())); TF_RETURN_IF_ERROR(ConvertGraphDefToGraph( @@ -562,7 +587,7 @@ class PartitionedCallOp : public AsyncOpKernel { } NameAttrList func_; - RewriterConfig rewriter_config_; + ConfigProto config_proto_; string executor_type_; // Contains maps from device names to handles of function partitions, keyed by // FunctionLibraryRuntime pointers. (Because this kernel may be instantiated diff --git a/tensorflow/core/kernels/quantize_and_dequantize_op.cc b/tensorflow/core/kernels/quantize_and_dequantize_op.cc index dadc15b69ee..f13341e0afe 100644 --- a/tensorflow/core/kernels/quantize_and_dequantize_op.cc +++ b/tensorflow/core/kernels/quantize_and_dequantize_op.cc @@ -49,6 +49,21 @@ class QuantizeAndDequantizeV2Op : public OpKernel { errors::InvalidArgument("num_bits is out of range: ", num_bits_, " with signed_input_ ", signed_input_)); OP_REQUIRES_OK(ctx, ctx->GetAttr("range_given", &range_given_)); + + string round_mode_string; + OP_REQUIRES_OK(ctx, ctx->GetAttr("round_mode", &round_mode_string)); + OP_REQUIRES( + ctx, + (round_mode_string == "HALF_UP" || round_mode_string == "HALF_TO_EVEN"), + errors::InvalidArgument("Round mode string must be " + "'HALF_UP' or " + "'HALF_TO_EVEN', is '" + + round_mode_string + "'")); + if (round_mode_string == "HALF_UP") { + round_mode_ = ROUND_HALF_UP; + } else if (round_mode_string == "HALF_TO_EVEN") { + round_mode_ = ROUND_HALF_TO_EVEN; + } } void Compute(OpKernelContext* ctx) override { @@ -76,13 +91,15 @@ class QuantizeAndDequantizeV2Op : public OpKernel { functor::QuantizeAndDequantizeOneScaleFunctor f; f(ctx->eigen_device(), input.flat(), signed_input_, num_bits_, - range_given_, &input_min_tensor, &input_max_tensor, output->flat()); + range_given_, &input_min_tensor, &input_max_tensor, round_mode_, + output->flat()); } private: bool signed_input_; int num_bits_; bool range_given_; + QuantizerRoundMode round_mode_; }; // Simulate quantization precision loss in a float tensor by: @@ -135,7 +152,8 @@ class QuantizeAndDequantizeV3Op : public OpKernel { functor::QuantizeAndDequantizeOneScaleFunctor f; f(ctx->eigen_device(), input.flat(), signed_input_, num_bits_val, - range_given_, &input_min_tensor, &input_max_tensor, output->flat()); + range_given_, &input_min_tensor, &input_max_tensor, ROUND_HALF_TO_EVEN, + output->flat()); } private: @@ -180,7 +198,7 @@ class QuantizeAndDequantizeOp : public OpKernel { functor::QuantizeAndDequantizeOneScaleFunctor functor; functor(ctx->eigen_device(), input.flat(), signed_input_, num_bits_, range_given_, &input_min_tensor, &input_max_tensor, - output->flat()); + ROUND_HALF_TO_EVEN, output->flat()); } private: @@ -198,10 +216,11 @@ struct QuantizeAndDequantizeOneScaleFunctor { void operator()(const CPUDevice& d, typename TTypes::ConstVec input, const bool signed_input, const int num_bits, const bool range_given, Tensor* input_min_tensor, - Tensor* input_max_tensor, typename TTypes::Vec out) { + Tensor* input_max_tensor, QuantizerRoundMode round_mode, + typename TTypes::Vec out) { QuantizeAndDequantizeOneScaleImpl::Compute( d, input, signed_input, num_bits, range_given, input_min_tensor, - input_max_tensor, out); + input_max_tensor, round_mode, out); } }; } // namespace functor diff --git a/tensorflow/core/kernels/quantize_and_dequantize_op.h b/tensorflow/core/kernels/quantize_and_dequantize_op.h index 6b0c5e5a466..a495e8b71fe 100644 --- a/tensorflow/core/kernels/quantize_and_dequantize_op.h +++ b/tensorflow/core/kernels/quantize_and_dequantize_op.h @@ -22,6 +22,20 @@ limitations under the License. #include "tensorflow/core/kernels/cwise_ops.h" namespace tensorflow { + +enum QuantizerRoundMode { + // Round half up: if the fraction of y is exactly 0.5, then + // round(y) = y + 0.5 + // E.g., -5.5 gets rounded to -5, -5.4 goes to -5, + // 5.4 goes to 5, and 5.5 goes to 6. + ROUND_HALF_UP, + // Round half to even: if the fraction of y is exactly 0.5, then round(y) is + // the nearest even integer to y. + // E.g., 23.5 gets rounded to 24, 24.5 gets rounded to 24, while -23.5 becomes + // -24, and -24.5 gets rounded to 24. + ROUND_HALF_TO_EVEN, +}; + namespace functor { // TODO(pauldonnelly): 'signed_input' should really be called 'signed_output'. @@ -31,15 +45,69 @@ struct QuantizeAndDequantizeOneScaleFunctor { void operator()(const Device& d, typename TTypes::ConstVec input, bool signed_input, int num_bits, bool range_given, Tensor* input_min_tensor, Tensor* input_max_tensor, - typename TTypes::Vec out); + QuantizerRoundMode round_mode, typename TTypes::Vec out); }; +// The implementation below runs on both CPU and GPU. +template +void ClampScaleAndRound(const Device& d, typename TTypes::ConstVec input, + T min_range, T max_range, T scale, T inverse_scale, + Func round_func, typename TTypes::Vec out) { + out.device(d) = (input.cwiseMin(max_range).cwiseMax(min_range) * scale) + .unaryExpr(round_func) * + inverse_scale; +} + +// The implementation below runs on both CPU and GPU. +template +void ClampScaleAndRound(const Device& d, typename TTypes::ConstVec input, + T min_range, T max_range, T scale, T inverse_scale, + QuantizerRoundMode round_mode, + typename TTypes::Vec out) { + switch (round_mode) { + case ROUND_HALF_TO_EVEN: + ClampScaleAndRound(d, input, min_range, max_range, scale, inverse_scale, + Eigen::internal::scalar_round_op_google(), out); + break; + case ROUND_HALF_UP: + ClampScaleAndRound(d, input, min_range, max_range, scale, inverse_scale, + Eigen::internal::scalar_round_up_op(), out); + break; + } +} + +// The implementation below runs on both CPU and GPU. +template +void ScaleAndRound(const Device& d, typename TTypes::ConstVec input, T scale, + T inverse_scale, Func round_func, + typename TTypes::Vec out) { + out.device(d) = (input * scale).unaryExpr(round_func) * inverse_scale; +} + +// The implementation below runs on both CPU and GPU. +template +void ScaleAndRound(const Device& d, typename TTypes::ConstVec input, T scale, + T inverse_scale, QuantizerRoundMode round_mode, + typename TTypes::Vec out) { + switch (round_mode) { + case ROUND_HALF_TO_EVEN: + ScaleAndRound(d, input, scale, inverse_scale, + Eigen::internal::scalar_round_op_google(), out); + break; + case ROUND_HALF_UP: + ScaleAndRound(d, input, scale, inverse_scale, + Eigen::internal::scalar_round_up_op(), out); + break; + } +} + // The implementation below runs on both CPU and GPU. template struct QuantizeAndDequantizeOneScaleImpl { static void Compute(const Device& d, typename TTypes::ConstVec input, bool signed_input, int num_bits, bool range_given, Tensor* input_min_tensor, Tensor* input_max_tensor, + QuantizerRoundMode round_mode, typename TTypes::Vec out) { T min_range; T max_range; @@ -89,15 +157,10 @@ struct QuantizeAndDequantizeOneScaleImpl { // The semantics of the op does not guarantee to clamp to the specified // min_range and max_range - because we may have changed either min_range // or max_range. - out.device(d) = - (input.cwiseMin(max_range).cwiseMax(min_range) * scale) - .unaryExpr(Eigen::internal::scalar_round_op_google()) * - inverse_scale; + ClampScaleAndRound(d, input, min_range, max_range, scale, inverse_scale, + round_mode, out); } else { - out.device(d) = - (input * scale) - .unaryExpr(Eigen::internal::scalar_round_op_google()) * - inverse_scale; + ScaleAndRound(d, input, scale, inverse_scale, round_mode, out); } } }; diff --git a/tensorflow/core/kernels/quantize_and_dequantize_op_gpu.cu.cc b/tensorflow/core/kernels/quantize_and_dequantize_op_gpu.cu.cc index 61c79cf6959..5745e418f36 100644 --- a/tensorflow/core/kernels/quantize_and_dequantize_op_gpu.cu.cc +++ b/tensorflow/core/kernels/quantize_and_dequantize_op_gpu.cu.cc @@ -32,10 +32,10 @@ struct QuantizeAndDequantizeOneScaleFunctor { void operator()(const GPUDevice& d, typename TTypes::ConstVec input, bool signed_input, int num_bits, bool range_given, Tensor* input_min_tensor, Tensor* input_max_tensor, - typename TTypes::Vec out) { + QuantizerRoundMode round_mode, typename TTypes::Vec out) { QuantizeAndDequantizeOneScaleImpl::Compute( d, input, signed_input, num_bits, range_given, input_min_tensor, - input_max_tensor, out); + input_max_tensor, round_mode, out); } }; } // end namespace functor diff --git a/tensorflow/core/kernels/quantize_and_dequantize_op_test.cc b/tensorflow/core/kernels/quantize_and_dequantize_op_test.cc index cddabf8a99a..b9e015c96b5 100644 --- a/tensorflow/core/kernels/quantize_and_dequantize_op_test.cc +++ b/tensorflow/core/kernels/quantize_and_dequantize_op_test.cc @@ -101,17 +101,51 @@ TEST_F(QuantizeAndDequantizeTest, Convert_1D_tensor_with_int8) { .Attr("range_given", false) .Finalize(node_def())); TF_ASSERT_OK(InitOp()); - AddInputFromArray(TensorShape({6}), {-1, -0.5, 0, 0.3, 0.8, 0.555}); + AddInputFromArray(TensorShape({7}), + {-1, -0.5, 0, 0.3, 0.8, 0.555, 0.50390625}); AddInputFromArray(TensorShape({}), {0.0}); // Min AddInputFromArray(TensorShape({}), {0.0}); // Max - // With int8, the tensor is quantized to {-128, -64, 0, 38, 102, 71}. + // With int8, the tensor is quantized to {-128, -64, 0, 38, 102, 71, 64}. // Scale is: 1/127 - // Then it is dequantized to {-1, -0.5, 0, 38.0/128, 102.0/128, 71.0/128} + // Then it is dequantized to {-1, -0.5, 0, 38.0/128, 102.0/128, 71.0/128, 0.5} TF_ASSERT_OK(RunOpKernel()); - Tensor expected(allocator(), DT_FLOAT, TensorShape({6})); - test::FillValues(&expected, - {-1, -0.5, 0, 38.0 / 128, 102.0 / 128, 71.0 / 128}); + Tensor expected(allocator(), DT_FLOAT, TensorShape({7})); + test::FillValues( + &expected, {-1, -0.5, 0, 38.0 / 128, 102.0 / 128, 71.0 / 128, 0.5}); + test::ExpectTensorNear(expected, *GetOutput(0), 1e-5); + + // Ensure that the inputs haven't been changed. + EXPECT_EQ(inputs_[1]->scalar()(), 0.0); + EXPECT_EQ(inputs_[2]->scalar()(), 0.0); +} + +// Convert a 1D tensor with signed 8 bits and round_mode half_up. +TEST_F(QuantizeAndDequantizeTest, Convert_1D_tensor_with_int8_round_half_up) { + TF_ASSERT_OK( + NodeDefBuilder("quantize_and_dequantize_op", "QuantizeAndDequantizeV2") + .Input(FakeInput(DT_FLOAT)) + .Input(FakeInput(DT_FLOAT)) + .Input(FakeInput(DT_FLOAT)) + .Attr("signed_input", true) + .Attr("num_bits", 8) + .Attr("range_given", false) + .Attr("round_mode", "HALF_UP") + .Finalize(node_def())); + TF_ASSERT_OK(InitOp()); + AddInputFromArray(TensorShape({7}), + {-1, -0.5, 0, 0.3, 0.8, 0.555, 0.50390625}); + AddInputFromArray(TensorShape({}), {0.0}); // Min + AddInputFromArray(TensorShape({}), {0.0}); // Max + + // With int8, the tensor is quantized to {-128, -64, 0, 38, 102, 71, 65}. + // Scale is: 1/127 + // Then it is dequantized to {-1, -0.5, 0, 38.0/128, 102.0/128, 71.0/128, + // 65.0 /128} + TF_ASSERT_OK(RunOpKernel()); + Tensor expected(allocator(), DT_FLOAT, TensorShape({7})); + test::FillValues(&expected, {-1, -0.5, 0, 38.0 / 128, 102.0 / 128, + 71.0 / 128, 65.0 / 128}); test::ExpectTensorNear(expected, *GetOutput(0), 1e-5); // Ensure that the inputs haven't been changed. @@ -162,7 +196,7 @@ TEST_F(QuantizeAndDequantizeTest, Convert_1D_tensor_with_int4) { .Attr("range_given", false) .Finalize(node_def())); TF_ASSERT_OK(InitOp()); - AddInputFromArray(TensorShape({6}), {-1, -0.5, 0, 0.3, 0.8, 0.555}); + AddInputFromArray(TensorShape({6}), {-1, -0.5, 0, 0.3125, 0.8, 0.555}); AddInputFromArray(TensorShape({}), {0.0}); // Min AddInputFromArray(TensorShape({}), {0.0}); // Max @@ -178,6 +212,35 @@ TEST_F(QuantizeAndDequantizeTest, Convert_1D_tensor_with_int4) { EXPECT_EQ(inputs_[2]->scalar()(), 0.0); } +// Convert a 1D tensor with signed 4 bits and round_mode hafl_up. +TEST_F(QuantizeAndDequantizeTest, Convert_1D_tensor_with_int4_round_half_up) { + TF_ASSERT_OK( + NodeDefBuilder("quantize_and_dequantize_op", "QuantizeAndDequantizeV2") + .Input(FakeInput(DT_FLOAT)) + .Input(FakeInput(DT_FLOAT)) + .Input(FakeInput(DT_FLOAT)) + .Attr("signed_input", true) + .Attr("num_bits", 4) + .Attr("range_given", false) + .Attr("round_mode", "HALF_UP") + .Finalize(node_def())); + TF_ASSERT_OK(InitOp()); + AddInputFromArray(TensorShape({6}), {-1, -0.5, 0, 0.3125, 0.8, 0.555}); + AddInputFromArray(TensorShape({}), {0.0}); // Min + AddInputFromArray(TensorShape({}), {0.0}); // Max + + // With int4, the tensor is quantized to {-8, -4, 0, 3, 6, 4}. + // Scale is: 1/8 + TF_ASSERT_OK(RunOpKernel()); + Tensor expected(allocator(), DT_FLOAT, TensorShape({6})); + test::FillValues(&expected, {-1, -0.5, 0, 0.375, 0.75, 0.5}); + test::ExpectTensorNear(expected, *GetOutput(0), 1e-5); + + // Ensure that the inputs haven't been changed. + EXPECT_EQ(inputs_[1]->scalar()(), 0.0); + EXPECT_EQ(inputs_[2]->scalar()(), 0.0); +} + // Convert a 1D tensor with signed 4 bits. TEST_F(QuantizeAndDequantizeTest, Convert_1D_tensor_with_int4_V3) { TF_ASSERT_OK( @@ -237,6 +300,38 @@ TEST_F(QuantizeAndDequantizeTest, Convert_2D_tensor_with_int8_range_given) { test::ExpectTensorNear(expected, *GetOutput(0), 1e-5); } +// Convert a 2D tensor with signed 8 bits, given range and round_mode half_up. +TEST_F(QuantizeAndDequantizeTest, + Convert_2D_tensor_with_int8_range_given_round_half_up) { + TF_ASSERT_OK( + NodeDefBuilder("quantize_and_dequantize_op", "QuantizeAndDequantizeV2") + .Input(FakeInput(DT_FLOAT)) + .Input(FakeInput(DT_FLOAT)) + .Input(FakeInput(DT_FLOAT)) + .Attr("signed_input", true) + .Attr("num_bits", 8) + .Attr("range_given", true) + .Attr("round_mode", "HALF_UP") + .Finalize(node_def())); + TF_ASSERT_OK(InitOp()); + // Note that the last two values are saturated. + AddInputFromArray(TensorShape({2, 4}), + {-0.8, -0.5, 0, 0.3, 0.8, 0.555, -2, 33}); + AddInputFromArray(TensorShape({}), {-1.0}); // Min + AddInputFromArray(TensorShape({}), {1.0}); // Max + + // Note that the range is given as [-1, 1]. + // With int8, the tensor is quantized to {-102, -63, 0, 38, 102, 70, -128, + // 127}. + // Scale is: 1/127 + TF_ASSERT_OK(RunOpKernel()); + Tensor expected(allocator(), DT_FLOAT, TensorShape({2, 4})); + test::FillValues( + &expected, {-102.0 / 127, -63.0 / 127, 0, 38.0 / 127, 102.0 / 127, + 70.0 / 127, -128.0 / 127, 1}); + test::ExpectTensorNear(expected, *GetOutput(0), 1e-5); +} + // Convert a 2D tensor with signed 8 bits with given range. TEST_F(QuantizeAndDequantizeTest, Convert_2D_tensor_with_int8_range_given_V3) { TF_ASSERT_OK( @@ -293,6 +388,33 @@ TEST_F(QuantizeAndDequantizeTest, Convert_4D_tensor_with_uint8_range_given) { test::ExpectTensorNear(expected, *GetOutput(0), 1e-5); } +// Convert a 4D tensor with unsigned 8 bits, given range and round_mode half_up. +TEST_F(QuantizeAndDequantizeTest, + Convert_4D_tensor_with_uint8_range_given_round_half_up) { + TF_ASSERT_OK( + NodeDefBuilder("quantize_and_dequantize_op", "QuantizeAndDequantizeV2") + .Input(FakeInput(DT_FLOAT)) + .Input(FakeInput(DT_FLOAT)) + .Input(FakeInput(DT_FLOAT)) + .Attr("signed_input", false) + .Attr("num_bits", 8) + .Attr("range_given", true) + .Attr("round_mode", "HALF_UP") + .Finalize(node_def())); + TF_ASSERT_OK(InitOp()); + AddInputFromArray(TensorShape({2, 2, 1, 1}), {-0.5, 0, 0.3, 0.8}); + AddInputFromArray(TensorShape({}), {0.0}); // Min + AddInputFromArray(TensorShape({}), {1.0}); // Max + + // Note that the range is given as [0, 1]. + // With int8, the tensor is quantized to {0, 0, 77, 204} + // Scale is: 1/255 + TF_ASSERT_OK(RunOpKernel()); + Tensor expected(allocator(), DT_FLOAT, TensorShape({2, 2, 1, 1})); + test::FillValues(&expected, {0, 0, 77.0 / 255, 204.0 / 255}); + test::ExpectTensorNear(expected, *GetOutput(0), 1e-5); +} + // Convert a 4D tensor with unsigned 8 bits with given range. TEST_F(QuantizeAndDequantizeTest, Convert_4D_tensor_with_uint8_range_given_V3) { TF_ASSERT_OK( diff --git a/tensorflow/core/kernels/quantized_resize_bilinear_op_test.cc b/tensorflow/core/kernels/quantized_resize_bilinear_op_test.cc index e6133415d0f..6fc48945923 100644 --- a/tensorflow/core/kernels/quantized_resize_bilinear_op_test.cc +++ b/tensorflow/core/kernels/quantized_resize_bilinear_op_test.cc @@ -273,7 +273,7 @@ void TestResizeBilinearOneDim() { << expected_val << ", " << resized_image_val; } - // Value testing with reference implemenatation + // Value testing with reference implementation CheckTensorValue(image_quantized_tensor.flat().data(), outputs.at(0).flat().data(), /*batch_size=*/1, diff --git a/tensorflow/core/kernels/ragged_gather_op.cc b/tensorflow/core/kernels/ragged_gather_op.cc index b2a342f6378..903a97a9601 100644 --- a/tensorflow/core/kernels/ragged_gather_op.cc +++ b/tensorflow/core/kernels/ragged_gather_op.cc @@ -236,8 +236,10 @@ class RaggedGatherOpBase : public OpKernel { values_shape.set_dim(0, num_values); TF_RETURN_IF_ERROR( context->allocate_output(values_index, values_shape, &values_out)); - int64 value_size = params_dense_values_in.NumElements() / - params_dense_values_in.dim_size(0); + const int64 num_elements = params_dense_values_in.NumElements(); + const int64 value_size = + num_elements == 0 ? 0 + : (num_elements / params_dense_values_in.dim_size(0)); CallWriteValueSlices(params_dense_values_in, value_slices, value_size, values_out); return ::tensorflow::Status::OK(); diff --git a/tensorflow/core/kernels/scan_ops_gpu.cu.cc b/tensorflow/core/kernels/scan_ops_gpu.cu.cc index ed6c6affce5..ed66c02dc58 100644 --- a/tensorflow/core/kernels/scan_ops_gpu.cu.cc +++ b/tensorflow/core/kernels/scan_ops_gpu.cu.cc @@ -1,4 +1,4 @@ -/* Copyright 2016 The TensorFlow Authors. All Rights Reserved. +/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -17,8 +17,20 @@ limitations under the License. #define EIGEN_USE_GPU +#if CUDA_VERSION >= 9000 +#define CUB_USE_COOPERATIVE_GROUPS +#endif // CUDA_VERSION >= 9000 + +#include "third_party/cub/block/block_load.cuh" +#include "third_party/cub/block/block_scan.cuh" +#include "third_party/cub/block/block_store.cuh" +#include "third_party/cub/iterator/counting_input_iterator.cuh" +#include "third_party/cub/iterator/transform_input_iterator.cuh" +#include "cuda/include/cuComplex.h" #include "tensorflow/core/framework/numeric_types.h" #include "tensorflow/core/framework/register_types.h" +#include "tensorflow/core/util/permutation_input_iterator.h" +#include "tensorflow/core/util/permutation_output_iterator.h" #include "tensorflow/core/kernels/scan_ops.h" @@ -27,6 +39,258 @@ namespace tensorflow { typedef Eigen::GpuDevice GPUDevice; typedef Eigen::Index Index; +namespace functor { + +// Map a contiguous range to the actual memory locations depending on which +// axis the scan is taking place over and whether or not reversed. +struct MapIndexToLocation { + __host__ __device__ MapIndexToLocation(int dimx, int dimy, int dimz, + bool reverse = false) + : dimx_(dimx), dimy_(dimy), dimz_(dimz), reverse_(reverse) {} + + __host__ __device__ int operator()(int id) const { + if (dimx_ == 1) { + int row = id % dimy_; + int col = id / dimy_; + + if (reverse_) return (dimy_ - row - 1) * dimz_ + col; + + return row * dimz_ + col; + } else if (dimz_ == 1) { + if (reverse_) { + int row = id / dimy_; + int col = id % dimy_; + return row * dimy_ + (dimy_ - col - 1); + } + return id; + } else { + int col = id % dimy_; + int tmp = id / dimy_; + + int row1 = id / (dimy_ * dimz_); + int col1 = tmp % dimz_; + + if (reverse_) + return row1 * dimy_ * dimz_ + (dimy_ - col - 1) * dimz_ + col1; + + return row1 * dimy_ * dimz_ + col * dimz_ + col1; + } + } + + int dimx_; + int dimy_; + int dimz_; + bool reverse_; +}; + +template +struct BlockPrefixCallbackOp { + // Running prefix + T running_total_; + Op op_; + + __device__ BlockPrefixCallbackOp(T running_total, Op op) + : running_total_(running_total), op_(op) {} + + // Callback operator to be entered by the first warp of threads in the block. + // tid 0 is responsible for returning a value for seeding the block-wide scan. + __device__ T operator()(T block_aggregate) { + T old_prefix = running_total_; + running_total_ = op_(old_prefix, block_aggregate); + return old_prefix; + } +}; + +template +struct Sum { + __host__ __device__ T operator()(const T& a, const T& b) const { + return a + b; + } +}; + +template +struct Prod { + __host__ __device__ T operator()(const T& a, const T& b) const { + return a * b; + } +}; + +template +struct IsSum { + constexpr static bool value = + (std::is_same>::value || + std::is_same>::value); +}; + +template +struct IsProd { + constexpr static bool value = + (std::is_same>::value || + std::is_same>::value); +}; + +template +struct IdentityValue { + static_assert(IsSum::value || IsProd::value, + "IdentityValue not yet defined for this type."); + + template + __host__ __device__ U operator()( + typename std::enable_if::value, U>::type t = U(0)) { + return t; + } + + template + __host__ __device__ U operator()( + typename std::enable_if::value, U>::type t = U(1)) { + return t; + } +}; + +// Each block is mapped to one sequence. A contiguous range is mapped to the +// appropriate locations in memory by the permutation iterators. This is +// ideal for 1-D and row based scans. Column scans would be better if they +// did a block load and then locally transposed. CUB's device wide scan is not +// used in the large 1D case, even though it would be more efficient, because +// it is not deterministic. +template +__global__ void scan_kernel(const T* in, T* out, int dimx, int dimy, int dimz, + bool exclusive, bool reverse, Op op) { + typedef cub::BlockLoad + BlockLoad; + typedef cub::BlockStore + BlockStore; + typedef cub::BlockScan BlockScan; + + // Allocate aliased shared memory for BlockLoad, BlockStore, and BlockScan + __shared__ union { + typename BlockLoad::TempStorage load; + typename BlockScan::TempStorage scan; + typename BlockStore::TempStorage store; + } temp_storage; + + int problem_length = dimy; + + // Initialize running total + BlockPrefixCallbackOp prefix_op(IdentityValue()(), op); + + MapIndexToLocation map_op(dimx, dimy, dimz, reverse); + int block_start = problem_length * blockIdx.x; + // Have the block iterate over segments of items + for (int block_offset = block_start; + block_offset < block_start + problem_length; + block_offset += BlockDim * ItemsPerThread) { + int valid_items = min(BlockDim * ItemsPerThread, + problem_length - (block_offset % problem_length)); + + // first construct a counting iterator that has the desired start point + typedef cub::TransformInputIterator> + MapIterType; + + cub::CountingInputIterator counting_iter(block_offset); + + // Next map the iterator to the actual locations in memory + MapIterType map_iter(counting_iter, map_op); + + PermutationInputIterator permutein_iter(in, + map_iter); + PermutationOutputIterator permuteout_iter(out, + map_iter); + + // Load a segment of consecutive items that are blocked across threads + T thread_data[ItemsPerThread]; + BlockLoad(temp_storage.load).Load(permutein_iter, thread_data, valid_items); + __syncthreads(); + + // Collectively compute the block-wide scan + if (exclusive) { + BlockScan(temp_storage.scan) + .ExclusiveScan(thread_data, thread_data, op, prefix_op); + } else { + BlockScan(temp_storage.scan) + .InclusiveScan(thread_data, thread_data, op, prefix_op); + } + __syncthreads(); + + // Store scanned items to output segment + BlockStore(temp_storage.store) + .Store(permuteout_iter, thread_data, valid_items); + __syncthreads(); + } +} + +template +void LaunchScan(const GPUDevice& d, typename TTypes::ConstTensor in, + typename TTypes::Tensor out, Op op, const bool reverse, + const bool exclusive) { + const int items_per_thread = 4; + + int dimx = in.dimension(0); + int dimy = in.dimension(1); + int dimz = in.dimension(2); + int num_blocks = dimx * dimz; + + int ideal_block_size = dimy / items_per_thread; + + // There seems to be a bug when the type is not float and block_size 1024. + // Launch on the smallest power of 2 block size that we can. + if (ideal_block_size >= 1024 && std::is_same::value) { + const int block_size = 1024; + scan_kernel + <<>>( + in.data(), out.data(), dimx, dimy, dimz, exclusive, reverse, op); + } else if (ideal_block_size >= 512) { + const int block_size = 512; + scan_kernel + <<>>( + in.data(), out.data(), dimx, dimy, dimz, exclusive, reverse, op); + } else if (ideal_block_size >= 256) { + const int block_size = 256; + scan_kernel + <<>>( + in.data(), out.data(), dimx, dimy, dimz, exclusive, reverse, op); + } else if (ideal_block_size >= 128) { + const int block_size = 128; + scan_kernel + <<>>( + in.data(), out.data(), dimx, dimy, dimz, exclusive, reverse, op); + } else if (ideal_block_size >= 64) { + const int block_size = 64; + scan_kernel + <<>>( + in.data(), out.data(), dimx, dimy, dimz, exclusive, reverse, op); + } else { + const int block_size = 32; + scan_kernel + <<>>( + in.data(), out.data(), dimx, dimy, dimz, exclusive, reverse, op); + } +} + +template +struct Scan, T> { + void operator()(const GPUDevice& d, typename TTypes::ConstTensor in, + typename TTypes::Tensor out, + const Eigen::internal::SumReducer& reducer, + const bool reverse, const bool exclusive) { + LaunchScan>(d, in, out, Sum(), reverse, exclusive); + } +}; + +template +struct Scan, T> { + void operator()(const GPUDevice& d, typename TTypes::ConstTensor in, + typename TTypes::Tensor out, + const Eigen::internal::ProdReducer& reducer, + const bool reverse, const bool exclusive) { + LaunchScan>(d, in, out, Prod(), reverse, exclusive); + } +}; + +} // namespace functor + #define DEFINE(REDUCER, T) template struct functor::Scan; #define DEFINE_FOR_ALL_REDUCERS(T) \ diff --git a/tensorflow/core/kernels/scan_ops_test.cc b/tensorflow/core/kernels/scan_ops_test.cc new file mode 100644 index 00000000000..588b606a99b --- /dev/null +++ b/tensorflow/core/kernels/scan_ops_test.cc @@ -0,0 +1,146 @@ +/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/core/common_runtime/kernel_benchmark_testlib.h" +#include "tensorflow/core/framework/tensor.h" +#include "tensorflow/core/framework/types.h" +#include "tensorflow/core/platform/test.h" +#include "tensorflow/core/platform/test_benchmark.h" + +namespace tensorflow { + +template +static Graph* LargeOneDCumsum(int num_x, bool reverse = false) { + auto* g = new Graph(OpRegistry::Global()); + Tensor data(DataTypeToEnum::value, TensorShape({num_x})); + data.flat().setRandom(); + Tensor axes(DT_INT32, TensorShape({})); + axes.flat()(0) = 0; + test::graph::Cumsum(g, test::graph::Constant(g, data), + test::graph::Constant(g, axes)); + return g; +} + +static Graph* ColCumsum(int num_x, int num_y, bool reverse = false) { + auto* g = new Graph(OpRegistry::Global()); + Tensor data(DT_FLOAT, TensorShape({num_x, num_y})); + data.flat().setRandom(); + Tensor axes(DT_INT32, TensorShape({})); + axes.flat()(0) = 0; + test::graph::Cumsum(g, test::graph::Constant(g, data), + test::graph::Constant(g, axes)); + return g; +} + +static Graph* RowCumsum(int num_x, int num_y, bool reverse = false) { + auto* g = new Graph(OpRegistry::Global()); + Tensor data(DT_FLOAT, TensorShape({num_x, num_y})); + data.flat().setRandom(); + Tensor axes(DT_INT32, TensorShape({})); + axes.flat()(0) = 1; + test::graph::Cumsum(g, test::graph::Constant(g, data), + test::graph::Constant(g, axes)); + return g; +} + +static Graph* ThreeDYCumsum(int num_y, int num_z, bool reverse = false) { + auto* g = new Graph(OpRegistry::Global()); + Tensor data(DT_FLOAT, TensorShape({32, num_y, num_z})); + data.flat().setRandom(); + Tensor axes(DT_INT32, TensorShape({})); + axes.flat()(0) = 1; + test::graph::Cumsum(g, test::graph::Constant(g, data), + test::graph::Constant(g, axes)); + return g; +} + +template +static void LargeOneDimensional(int iters, const string& device, int num_x, + bool reverse = false) { + testing::ItemsProcessed(static_cast(iters) * num_x); + testing::BytesProcessed(static_cast(iters) * num_x * sizeof(T)); + test::Benchmark(device, LargeOneDCumsum(num_x, reverse)).Run(iters); +} + +static void DoRowCumsum(int iters, const string& device, int num_x, int num_y, + bool reverse = false) { + testing::ItemsProcessed(static_cast(iters) * num_x * num_y); + testing::BytesProcessed(static_cast(iters) * num_x * num_y * + sizeof(float)); + test::Benchmark(device, RowCumsum(num_x, num_y, reverse)).Run(iters); +} + +static void DoColCumsum(int iters, const string& device, int num_x, int num_y, + bool reverse = false) { + testing::ItemsProcessed(static_cast(iters) * num_x * num_y); + testing::BytesProcessed(static_cast(iters) * num_x * num_y * + sizeof(float)); + test::Benchmark(device, ColCumsum(num_x, num_y, reverse)).Run(iters); +} + +static void Do3DYCumsum(int iters, const string& device, int num_x, int num_y, + bool reverse = false) { + testing::ItemsProcessed(static_cast(iters) * num_x * num_y); + testing::BytesProcessed(static_cast(iters) * num_x * num_y * + sizeof(float)); + test::Benchmark(device, ThreeDYCumsum(num_x, num_y, reverse)).Run(iters); +} + +static void BM_OneDCumsumGPU(int iters, int num_x) { + LargeOneDimensional(iters, "gpu", num_x); +} +BENCHMARK(BM_OneDCumsumGPU)->Range(1, 1 << 21); + +static void BM_OneDCumsumGPUHalf(int iters, int num_x) { + LargeOneDimensional(iters, "gpu", num_x); +} +BENCHMARK(BM_OneDCumsumGPUHalf)->Range(1, 1 << 21); + +static void BM_Sum2DRowCumsumGPU(int iters, int num_x, int num_y) { + DoRowCumsum(iters, "gpu", num_x, num_y); +} +BENCHMARK(BM_Sum2DRowCumsumGPU)->RangePair(1, 8192, 1, 8192); + +static void BM_Sum2DColumnCumsumGPU(int iters, int num_x, int num_y) { + DoColCumsum(iters, "gpu", num_x, num_y); +} +BENCHMARK(BM_Sum2DColumnCumsumGPU)->RangePair(1, 8192, 1, 8192); + +static void BM_Sum3DYCumsumGPU(int iters, int num_x, int num_y) { + Do3DYCumsum(iters, "gpu", num_x, num_y); +} +BENCHMARK(BM_Sum3DYCumsumGPU)->RangePair(64, 4096, 64, 4096); + +static void BM_OneDCumsumGPU_reverse(int iters, int num_x) { + LargeOneDimensional(iters, "gpu", num_x, true); +} +BENCHMARK(BM_OneDCumsumGPU_reverse)->Range(1, 1 << 21); + +static void BM_Sum2DRowCumsumGPU_reverse(int iters, int num_x, int num_y) { + DoRowCumsum(iters, "gpu", num_x, num_y, true); +} +BENCHMARK(BM_Sum2DRowCumsumGPU_reverse)->RangePair(1, 8192, 1, 8192); + +static void BM_Sum2DColumnCumsumGPU_reverse(int iters, int num_x, int num_y) { + DoColCumsum(iters, "gpu", num_x, num_y, true); +} +BENCHMARK(BM_Sum2DColumnCumsumGPU_reverse)->RangePair(1, 8192, 1, 8192); + +static void BM_Sum3DYCumsumGPU_reverse(int iters, int num_x, int num_y) { + Do3DYCumsum(iters, "gpu", num_x, num_y, true); +} +BENCHMARK(BM_Sum3DYCumsumGPU_reverse)->RangePair(32, 2048, 32, 2048); + +} // end namespace tensorflow diff --git a/tensorflow/core/kernels/scatter_nd_op.cc b/tensorflow/core/kernels/scatter_nd_op.cc index fd54c6d6d75..63bb793fdcb 100644 --- a/tensorflow/core/kernels/scatter_nd_op.cc +++ b/tensorflow/core/kernels/scatter_nd_op.cc @@ -29,6 +29,7 @@ limitations under the License. #include "tensorflow/core/kernels/bounds_check.h" #include "tensorflow/core/kernels/dense_update_functor.h" #include "tensorflow/core/kernels/fill_functor.h" +#include "tensorflow/core/kernels/inplace_ops_functor.h" #include "tensorflow/core/kernels/training_op_helpers.h" #include "tensorflow/core/kernels/variable_ops.h" #include "tensorflow/core/lib/strings/str_util.h" @@ -121,6 +122,90 @@ class ScatterNdOp : public OpKernel { } }; +template +class TensorScatterOp : public OpKernel { + public: + explicit TensorScatterOp(OpKernelConstruction* c) : OpKernel(c) { + const DataType dt = DataTypeToEnum::v(); + const DataType index_t = DataTypeToEnum::v(); + OP_REQUIRES_OK(c, c->MatchSignature({dt, index_t, dt}, {dt})); + } + + void Compute(OpKernelContext* c) override { + const Tensor& input = c->input(0); + const Tensor& indices = c->input(1); + const Tensor& updates = c->input(2); + + OP_REQUIRES(c, indices.shape().dims() >= 1, + errors::InvalidArgument( + "Indices shape must have rank at least one. Found:", + indices.shape().DebugString())); + OP_REQUIRES(c, updates.shape().dims() >= 1, + errors::InvalidArgument( + "Updates shape must have rank at least one. Found:", + updates.shape().DebugString())); + + TensorShape shape = input.shape(); + + OP_REQUIRES( + c, + (shape.num_elements() > 0 || (indices.shape().num_elements() == 0 && + updates.shape().num_elements() == 0)), + errors::InvalidArgument( + "Indices and updates specified for empty output shape")); + + const int64 outer_dims = indices.shape().dims() - 1; + + for (int i = 0; i < outer_dims; ++i) { + OP_REQUIRES(c, indices.shape().dim_size(i) == updates.shape().dim_size(i), + errors::InvalidArgument( + "Outer dimensions of indices and update must match. " + "Indices shape: ", + indices.shape().DebugString(), + ", updates shape:", updates.shape().DebugString())); + } + + const int64 ix = indices.shape().dim_size(outer_dims); + OP_REQUIRES( + c, updates.shape().dims() - outer_dims == shape.dims() - ix, + errors::InvalidArgument("Inner dimensions of output shape must match " + "inner dimensions of updates shape. Output: ", + shape.DebugString(), + " updates: ", updates.shape().DebugString())); + for (int i = 0; i + outer_dims < updates.shape().dims(); ++i) { + OP_REQUIRES( + c, updates.shape().dim_size(i + outer_dims) == shape.dim_size(ix + i), + errors::InvalidArgument( + "The inner ", shape.dims() - ix, + " dimensions of output.shape=", shape.DebugString(), + " must match the inner ", updates.shape().dims() - outer_dims, + " dimensions of updates.shape=", updates.shape().DebugString())); + } + + std::unique_ptr forwarded_input = c->forward_input( + 2, 0, input.dtype(), shape, DEVICE_MEMORY, AllocatorAttributes()); + + if (forwarded_input == nullptr) { + // We were not able to forward the input, so we deep copy the tensor and + // set the output. + Tensor* out; + OP_REQUIRES_OK(c, c->allocate_output(0, input.shape(), &out)); + + OP_REQUIRES_OK(c, tensorflow::functor::DoCopy(c->eigen_device(), + input, out)); + OP_REQUIRES_OK(c, + functor::DoScatterNd( + c, indices, updates, shape, out, false /*allocate*/)); + } else { + // Output forwarded, so simply perform the scatter. + OP_REQUIRES_OK(c, functor::DoScatterNd( + c, indices, updates, shape, forwarded_input.get(), + false /*allocate*/)); + } + } +}; + template class ScatterNdUpdateOp : public OpKernel { @@ -282,6 +367,56 @@ TF_CALL_bool(REGISTER_SCATTER_ND_ADD_SUB_CPU); TF_CALL_bool(REGISTER_SCATTER_ND_UPDATE_CPU); TF_CALL_bool(REGISTER_SCATTER_ND_CPU); +#define REGISTER_SCATTER_ND_TENSOR_UPDATE_TYPE_INDEX_TYPE(type, index_type, \ + dev) \ + REGISTER_KERNEL_BUILDER(Name("TensorScatterUpdate") \ + .Device(DEVICE_##dev) \ + .TypeConstraint("T") \ + .TypeConstraint("Tindices"), \ + TensorScatterOp) + +#define REGISTER_SCATTER_ND_TENSOR_ADD_TYPE_INDEX_TYPE(type, index_type, dev) \ + REGISTER_KERNEL_BUILDER(Name("TensorScatterAdd") \ + .Device(DEVICE_##dev) \ + .TypeConstraint("T") \ + .TypeConstraint("Tindices"), \ + TensorScatterOp) + +#define REGISTER_SCATTER_ND_TENSOR_SUB_TYPE_INDEX_TYPE(type, index_type, dev) \ + REGISTER_KERNEL_BUILDER(Name("TensorScatterSub") \ + .Device(DEVICE_##dev) \ + .TypeConstraint("T") \ + .TypeConstraint("Tindices"), \ + TensorScatterOp) + +#define REGISTER_SCATTER_ND_TENSOR_UPDATE_CPU(type) \ + REGISTER_SCATTER_ND_TENSOR_UPDATE_TYPE_INDEX_TYPE(type, int32, CPU); \ + REGISTER_SCATTER_ND_TENSOR_UPDATE_TYPE_INDEX_TYPE(type, int64, CPU); + +#define REGISTER_SCATTER_ND_TENSOR_ADD_CPU(type) \ + REGISTER_SCATTER_ND_TENSOR_ADD_TYPE_INDEX_TYPE(type, int32, CPU); \ + REGISTER_SCATTER_ND_TENSOR_ADD_TYPE_INDEX_TYPE(type, int64, CPU); + +#define REGISTER_SCATTER_ND_TENSOR_SUB_CPU(type) \ + REGISTER_SCATTER_ND_TENSOR_SUB_TYPE_INDEX_TYPE(type, int32, CPU); \ + REGISTER_SCATTER_ND_TENSOR_SUB_TYPE_INDEX_TYPE(type, int64, CPU); + +#define REGISTER_SCATTER_ND_TENSOR_CPU(type) \ + REGISTER_SCATTER_ND_TENSOR_UPDATE_CPU(type); \ + REGISTER_SCATTER_ND_TENSOR_ADD_CPU(type); \ + REGISTER_SCATTER_ND_TENSOR_SUB_CPU(type); + +// Register TensorScatterUpdate/Add/Sub for all number types. +TF_CALL_NUMBER_TYPES(REGISTER_SCATTER_ND_TENSOR_CPU); +// Register only TensorScatterUpdate for string/bool types as well. +TF_CALL_string(REGISTER_SCATTER_ND_TENSOR_UPDATE_CPU); +TF_CALL_bool(REGISTER_SCATTER_ND_TENSOR_UPDATE_CPU); + +#undef REGISTER_SCATTER_ND_TENSOR_CPU + // Registers GPU kernels. #if GOOGLE_CUDA @@ -319,6 +454,25 @@ TF_CALL_GPU_NUMBER_TYPES_NO_HALF(REGISTER_SCATTER_ND_UPDATE_SYCL); #undef REGISTER_SCATTER_ND_UPDATE_SYCL #endif // TENSORFLOW_USE_SYCL +#define REGISTER_SCATTER_ND_TENSOR_UPDATE_GPU(type) \ + REGISTER_SCATTER_ND_TENSOR_UPDATE_TYPE_INDEX_TYPE(type, int32, GPU); \ + REGISTER_SCATTER_ND_TENSOR_UPDATE_TYPE_INDEX_TYPE(type, int64, GPU); + +#define REGISTER_SCATTER_ND_TENSOR_ADD_GPU(type) \ + REGISTER_SCATTER_ND_TENSOR_ADD_TYPE_INDEX_TYPE(type, int32, GPU); \ + REGISTER_SCATTER_ND_TENSOR_ADD_TYPE_INDEX_TYPE(type, int64, GPU); + +#define REGISTER_SCATTER_ND_TENSOR_SUB_GPU(type) \ + REGISTER_SCATTER_ND_TENSOR_SUB_TYPE_INDEX_TYPE(type, int32, GPU); \ + REGISTER_SCATTER_ND_TENSOR_SUB_TYPE_INDEX_TYPE(type, int64, GPU); + +#define REGISTER_SCATTER_ND_TENSOR_GPU(type) \ + REGISTER_SCATTER_ND_TENSOR_ADD_GPU(type); \ + REGISTER_SCATTER_ND_TENSOR_UPDATE_GPU(type); \ + REGISTER_SCATTER_ND_TENSOR_SUB_GPU(type); + +TF_CALL_GPU_NUMBER_TYPES_NO_HALF(REGISTER_SCATTER_ND_TENSOR_GPU); + #undef REGISTER_SCATTER_ND_ADD #undef REGISTER_SCATTER_ND_ADD_SUB #undef REGISTER_SCATTER_ND_ADD_SUB_CPU @@ -328,6 +482,16 @@ TF_CALL_GPU_NUMBER_TYPES_NO_HALF(REGISTER_SCATTER_ND_UPDATE_SYCL); #undef REGISTER_SCATTER_ND_UPDATE_GPU #undef REGISTER_SCATTER_ND_KERNEL #undef REGISTER_SCATTER_ND_KERNEL_INDEX +#undef REGISTER_SCATTER_ND_TENSOR_TYPE_INDEX_TYPE +#undef REGISTER_SCATTER_ND_TENSOR_CPU +#undef REGISTER_SCATTER_ND_TENSOR_GPU +#undef REGISTER_SCATTER_ND_TENSOR_UPDATE_TYPE_INDEX_TYPE +#undef REGISTER_SCATTER_ND_TENSOR_ADD_TYPE_INDEX_TYPE +#undef REGISTER_SCATTER_ND_TENSOR_SUB_TYPE_INDEX_TYPE +#undef REGISTER_SCATTER_ND_TENSOR_UPDATE_GPU +#undef REGISTER_SCATTER_ND_TENSOR_ADD_GPU +#undef REGISTER_SCATTER_ND_TENSOR_SUB_GPU +#undef REGISTER_SCATTER_ND_TENSOR_GPU #endif // GOOGLE_CUDA diff --git a/tensorflow/core/kernels/stage_op.cc b/tensorflow/core/kernels/stage_op.cc index 73a02a34cf2..c91bdc43cf4 100644 --- a/tensorflow/core/kernels/stage_op.cc +++ b/tensorflow/core/kernels/stage_op.cc @@ -151,7 +151,7 @@ class Buffer : public ResourceBase { } // Are there a limit number of elements or a memory limit - // configued on this buffer? + // configured on this buffer? bool IsBounded() const { return capacity_ > 0 || memory_limit_ > 0; } bool IsCapacityFull() const { return buf_.size() >= capacity_; } diff --git a/tensorflow/core/kernels/tensor_array_ops.cc b/tensorflow/core/kernels/tensor_array_ops.cc index a97a71b344d..aa85f546a81 100644 --- a/tensorflow/core/kernels/tensor_array_ops.cc +++ b/tensorflow/core/kernels/tensor_array_ops.cc @@ -352,9 +352,9 @@ class TensorArrayGradOp : public TensorArrayCreationOp { } const auto key = strings::StrCat(output_handle(0), output_handle(1)); - auto creator = [this, key, tensor_array, array_size, marked_size, - element_shape, shape_to_prepend, tensor_array_output_handle, - output_handle](TensorArray** ret) -> Status { + auto creator = [key, tensor_array, array_size, marked_size, element_shape, + shape_to_prepend, + tensor_array_output_handle](TensorArray** ret) -> Status { *ret = new TensorArray( key, tensor_array->ElemType(), *tensor_array_output_handle, array_size, element_shape, tensor_array->HasIdenticalElementShapes(), diff --git a/tensorflow/core/kernels/tensor_forest/BUILD b/tensorflow/core/kernels/tensor_forest/BUILD new file mode 100644 index 00000000000..df035506f76 --- /dev/null +++ b/tensorflow/core/kernels/tensor_forest/BUILD @@ -0,0 +1,53 @@ +# Description: +# OpKernels for tensor forest ops. + +package( + default_visibility = ["//tensorflow:internal"], +) + +licenses(["notice"]) # Apache 2.0 + +load("//tensorflow:tensorflow.bzl", "tf_kernel_library") + +cc_library( + name = "resources", + srcs = ["resources.cc"], + hdrs = ["resources.h"], + deps = [ + "//tensorflow/core:framework", + "//tensorflow/core:lib", + "//tensorflow/core/kernels/boosted_trees:boosted_trees_proto_cc", + ], +) + +tf_kernel_library( + name = "resource_ops", + srcs = ["resource_ops.cc"], + deps = [ + ":resources", + "//tensorflow/core:framework", + "//tensorflow/core:lib", + "//tensorflow/core:tensor_forest_ops_op_lib", + "//tensorflow/core/kernels/boosted_trees:boosted_trees_proto_cc", + ], +) + +tf_kernel_library( + name = "prediction_ops", + srcs = ["prediction_ops.cc"], + deps = [ + ":resources", + "//tensorflow/core:framework", + "//tensorflow/core:lib", + "//tensorflow/core:tensor_forest_ops_op_lib", + "//tensorflow/core/kernels/boosted_trees:boosted_trees_proto_cc", + ], +) + +tf_kernel_library( + name = "tensor_forest_ops", + deps = [ + ":prediction_ops", + ":resource_ops", + ], +) diff --git a/tensorflow/core/kernels/tensor_forest/prediction_ops.cc b/tensorflow/core/kernels/tensor_forest/prediction_ops.cc new file mode 100644 index 00000000000..877dfbc020d --- /dev/null +++ b/tensorflow/core/kernels/tensor_forest/prediction_ops.cc @@ -0,0 +1,93 @@ +/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#include "tensorflow/core/framework/op_kernel.h" +#include "tensorflow/core/framework/resource_mgr.h" +#include "tensorflow/core/framework/tensor.h" +#include "tensorflow/core/framework/tensor_shape.h" +#include "tensorflow/core/framework/tensor_types.h" +#include "tensorflow/core/kernels/tensor_forest/resources.h" +#include "tensorflow/core/platform/thread_annotations.h" +#include "tensorflow/core/util/work_sharder.h" + +namespace tensorflow { + +class TensorForestTreePredictOp : public OpKernel { + public: + explicit TensorForestTreePredictOp(OpKernelConstruction* context) + : OpKernel(context) { + OP_REQUIRES_OK(context, + context->GetAttr("logits_dimension", &logits_dimension_)); + } + + void Compute(OpKernelContext* context) override { + TensorForestTreeResource* decision_tree_resource; + OP_REQUIRES_OK(context, LookupResource(context, HandleFromInput(context, 0), + &decision_tree_resource)); + mutex_lock l(*decision_tree_resource->get_mutex()); + core::ScopedUnref unref_me(decision_tree_resource); + + const Tensor* dense_features_t = nullptr; + OP_REQUIRES_OK(context, + context->input("dense_features", &dense_features_t)); + + auto dense_features = dense_features_t->matrix(); + const int32 batch_size = dense_features_t->dim_size(0); + + Tensor* output_predictions = nullptr; + OP_REQUIRES_OK(context, + context->allocate_output(0, {batch_size, logits_dimension_}, + &output_predictions)); + auto out = output_predictions->matrix(); + + if (decision_tree_resource->get_size() <= 0) { + out.setZero(); + return; + } + auto* worker_threads = context->device()->tensorflow_cpu_worker_threads(); + const int32 num_threads = worker_threads->num_threads; + + // TODO(yupbank): This was from contrib version. + // This cost would probably depend on the depth of the tree we have. + // We will need to run it on a number of trees of diff depth + // and see the num of cpu cycles + const int64 cost_per_traverse = 500; + auto traverse = [this, &out, &dense_features, decision_tree_resource, + batch_size](int64 start, int64 end) { + CHECK_LE(start, end) << "Start exceeding End"; + CHECK_LE(end, batch_size) << "End exceeding batch size"; + for (int example_id = start; example_id < end; ++example_id) { + const int32 leaf_id = + decision_tree_resource->TraverseTree(example_id, &dense_features); + set_output_value(example_id, leaf_id, decision_tree_resource, &out); + }; + }; + Shard(num_threads, worker_threads->workers, batch_size, cost_per_traverse, + traverse); + }; + + void set_output_value(const int32 example_id, const int32 leaf_id, + const TensorForestTreeResource* decision_tree_resource, + TTypes::Matrix* out) const { + for (int j = 0; j < logits_dimension_; ++j) { + const float logit = decision_tree_resource->get_prediction(leaf_id, j); + (*out)(example_id, j) = logit; + } + }; + + private: + int32 logits_dimension_; +}; + +REGISTER_KERNEL_BUILDER(Name("TensorForestTreePredict").Device(DEVICE_CPU), + TensorForestTreePredictOp); + +} // namespace tensorflow diff --git a/tensorflow/core/kernels/tensor_forest/resource_ops.cc b/tensorflow/core/kernels/tensor_forest/resource_ops.cc new file mode 100644 index 00000000000..aba97ea9900 --- /dev/null +++ b/tensorflow/core/kernels/tensor_forest/resource_ops.cc @@ -0,0 +1,135 @@ +/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#include "tensorflow/core/framework/op_kernel.h" +#include "tensorflow/core/framework/resource_mgr.h" +#include "tensorflow/core/framework/tensor.h" +#include "tensorflow/core/framework/tensor_shape.h" +#include "tensorflow/core/framework/tensor_types.h" +#include "tensorflow/core/kernels/tensor_forest/resources.h" + +namespace tensorflow { + +class TensorForestCreateTreeVariableOp : public OpKernel { + public: + explicit TensorForestCreateTreeVariableOp(OpKernelConstruction* context) + : OpKernel(context){}; + + void Compute(OpKernelContext* context) override { + const Tensor* tree_config_t; + OP_REQUIRES_OK(context, context->input("tree_config", &tree_config_t)); + + auto* const result = new TensorForestTreeResource(); + + if (!result->InitFromSerialized(tree_config_t->scalar()())) { + result->Unref(); + OP_REQUIRES(context, false, + errors::InvalidArgument("Unable to parse tree config.")); + } + + // Only create one, if one does not exist already. Report status for all + // other exceptions. + auto status = CreateResource(context, HandleFromInput(context, 0), result); + if (!status.ok() && status.code() != tensorflow::error::ALREADY_EXISTS) { + OP_REQUIRES(context, false, status); + } + } +}; + +// Op for serializing a model. +class TensorForestTreeSerializeOp : public OpKernel { + public: + explicit TensorForestTreeSerializeOp(OpKernelConstruction* context) + : OpKernel(context) {} + + void Compute(OpKernelContext* context) override { + TensorForestTreeResource* decision_tree_resource; + OP_REQUIRES_OK(context, LookupResource(context, HandleFromInput(context, 0), + &decision_tree_resource)); + mutex_lock l(*decision_tree_resource->get_mutex()); + core::ScopedUnref unref_me(decision_tree_resource); + Tensor* output_config_t = nullptr; + OP_REQUIRES_OK( + context, context->allocate_output(0, TensorShape(), &output_config_t)); + output_config_t->scalar()() = + decision_tree_resource->decision_tree().SerializeAsString(); + } +}; + +// Op for deserializing a tree variable from a checkpoint. +class TensorForestTreeDeserializeOp : public OpKernel { + public: + explicit TensorForestTreeDeserializeOp(OpKernelConstruction* context) + : OpKernel(context) {} + void Compute(OpKernelContext* context) override { + TensorForestTreeResource* decision_tree_resource; + OP_REQUIRES_OK(context, LookupResource(context, HandleFromInput(context, 0), + &decision_tree_resource)); + + mutex_lock l(*decision_tree_resource->get_mutex()); + core::ScopedUnref unref_me(decision_tree_resource); + + const Tensor* tree_config_t; + OP_REQUIRES_OK(context, context->input("tree_config", &tree_config_t)); + + // Deallocate all the previous objects on the resource. + decision_tree_resource->Reset(); + + if (!decision_tree_resource->InitFromSerialized( + tree_config_t->scalar()())) { + OP_REQUIRES(context, false, + errors::InvalidArgument("Unable to parse tree config.")); + } + } +}; + +// Op for getting tree size. +class TensorForestTreeSizeOp : public OpKernel { + public: + explicit TensorForestTreeSizeOp(OpKernelConstruction* context) + : OpKernel(context) {} + + void Compute(OpKernelContext* context) override { + TensorForestTreeResource* decision_tree_resource; + OP_REQUIRES_OK(context, LookupResource(context, HandleFromInput(context, 0), + &decision_tree_resource)); + mutex_lock l(*decision_tree_resource->get_mutex()); + core::ScopedUnref unref_me(decision_tree_resource); + Tensor* output_t = nullptr; + OP_REQUIRES_OK(context, + context->allocate_output(0, TensorShape(), &output_t)); + output_t->scalar()() = decision_tree_resource->get_size(); + } +}; + +REGISTER_RESOURCE_HANDLE_KERNEL(TensorForestTreeResource); + +REGISTER_KERNEL_BUILDER( + Name("TensorForestTreeIsInitializedOp").Device(DEVICE_CPU), + IsResourceInitialized); + +REGISTER_KERNEL_BUILDER( + Name("TensorForestCreateTreeVariable").Device(DEVICE_CPU), + TensorForestCreateTreeVariableOp); + +REGISTER_KERNEL_BUILDER(Name("TensorForestTreeSerialize").Device(DEVICE_CPU), + TensorForestTreeSerializeOp); + +REGISTER_KERNEL_BUILDER(Name("TensorForestTreeDeserialize").Device(DEVICE_CPU), + TensorForestTreeDeserializeOp); + +REGISTER_KERNEL_BUILDER(Name("TensorForestTreeSize").Device(DEVICE_CPU), + TensorForestTreeSizeOp); + +} // namespace tensorflow diff --git a/tensorflow/core/kernels/tensor_forest/resources.cc b/tensorflow/core/kernels/tensor_forest/resources.cc new file mode 100644 index 00000000000..9f8ceb96200 --- /dev/null +++ b/tensorflow/core/kernels/tensor_forest/resources.cc @@ -0,0 +1,59 @@ +/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/core/kernels/tensor_forest/resources.h" +#include "tensorflow/core/kernels/boosted_trees/boosted_trees.pb.h" +#include "tensorflow/core/platform/protobuf.h" + +namespace tensorflow { + +const float TensorForestTreeResource::get_prediction( + const int32 id, const int32 dimension_id) const { + return decision_tree_->nodes(id).leaf().vector().value(dimension_id); +}; + +const int32 TensorForestTreeResource::TraverseTree( + const int32 example_id, + const TTypes::ConstMatrix* dense_data) const { + using boosted_trees::Node; + using boosted_trees::Tree; + int32 current_id = 0; + while (true) { + const Node& current = decision_tree_->nodes(current_id); + if (current.has_leaf()) { + return current_id; + }; + DCHECK_EQ(current.node_case(), Node::kDenseSplit); + const auto& split = current.dense_split(); + + if ((*dense_data)(example_id, split.feature_id()) <= split.threshold()) { + current_id = split.left_id(); + } else { + current_id = split.right_id(); + } + } +}; + +bool TensorForestTreeResource::InitFromSerialized(const string& serialized) { + return ParseProtoUnlimited(decision_tree_, serialized); +} + +void TensorForestTreeResource::Reset() { + arena_.Reset(); + CHECK_EQ(0, arena_.SpaceAllocated()); + decision_tree_ = protobuf::Arena::CreateMessage(&arena_); +} + +} // namespace tensorflow diff --git a/tensorflow/core/kernels/tensor_forest/resources.h b/tensorflow/core/kernels/tensor_forest/resources.h new file mode 100644 index 00000000000..34f61b2c0c5 --- /dev/null +++ b/tensorflow/core/kernels/tensor_forest/resources.h @@ -0,0 +1,63 @@ +/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_CORE_KERNELS_TENSOR_FOREST_RESOURCES_H_ +#define TENSORFLOW_CORE_KERNELS_TENSOR_FOREST_RESOURCES_H_ + +#include "tensorflow/core/framework/resource_mgr.h" +#include "tensorflow/core/framework/tensor.h" +#include "tensorflow/core/kernels/boosted_trees/boosted_trees.pb.h" +#include "tensorflow/core/lib/strings/strcat.h" +#include "tensorflow/core/platform/mutex.h" +#include "tensorflow/core/platform/protobuf.h" + +namespace tensorflow { + +// Keep a tree ensemble in memory for efficient evaluation and mutation. +class TensorForestTreeResource : public ResourceBase { + public: + TensorForestTreeResource() + : decision_tree_( + protobuf::Arena::CreateMessage(&arena_)){}; + + string DebugString() override { + return strings::StrCat("TensorForestTree[size=", get_size(), "]"); + } + + mutex* get_mutex() { return &mu_; } + + bool InitFromSerialized(const string& serialized); + + // Resets the resource and frees the proto. + // Caller needs to hold the mutex lock while calling this. + void Reset(); + + const int32 get_size() const { return decision_tree_->nodes_size(); } + + const boosted_trees::Tree& decision_tree() const { return *decision_tree_; } + + const float get_prediction(const int32 id, const int32 dimension_id) const; + + const int32 TraverseTree(const int32 example_id, + const TTypes::ConstMatrix* dense_data) const; + + protected: + mutex mu_; + protobuf::Arena arena_; + boosted_trees::Tree* decision_tree_; +}; + +} // namespace tensorflow +#endif // TENSORFLOW_CORE_KERNELS_TENSOR_FOREST_RESOURCES_H_ diff --git a/tensorflow/core/kernels/training_ops.cc b/tensorflow/core/kernels/training_ops.cc index acf162deec9..6504ad1b09c 100644 --- a/tensorflow/core/kernels/training_ops.cc +++ b/tensorflow/core/kernels/training_ops.cc @@ -283,6 +283,22 @@ struct ApplyMomentum { } }; +template +struct ApplyKerasMomentum { + void operator()(const CPUDevice& d, typename TTypes::Flat var, + typename TTypes::Flat accum, + typename TTypes::ConstScalar lr, + typename TTypes::ConstFlat grad, + typename TTypes::ConstScalar momentum, bool use_nesterov) { + accum.device(d) = accum * momentum() - grad * lr(); + if (use_nesterov) { + var.device(d) += (accum * momentum() - grad * lr()); + } else { + var.device(d) += accum; + } + } +}; + template struct ApplyAdamNonCuda { void operator()(const Device& d, typename TTypes::Flat var, @@ -331,6 +347,28 @@ struct ApplyAdamSYCL { template struct ApplyAdam : ApplyAdamNonCuda {}; +template +struct ApplyAdamWithAmsgrad { + void operator()(const CPUDevice& d, typename TTypes::Flat var, + typename TTypes::Flat m, typename TTypes::Flat v, + typename TTypes::Flat vhat, + typename TTypes::ConstScalar beta1_power, + typename TTypes::ConstScalar beta2_power, + typename TTypes::ConstScalar lr, + typename TTypes::ConstScalar beta1, + typename TTypes::ConstScalar beta2, + typename TTypes::ConstScalar epsilon, + typename TTypes::ConstFlat grad) { + const T alpha = lr() * Eigen::numext::sqrt(T(1) - beta2_power()) / + (T(1) - beta1_power()); + + m.device(d) += (grad - m) * (T(1) - beta1()); + v.device(d) += (grad.square() - v) * (T(1) - beta2()); + vhat.device(d) = vhat.cwiseMax(v); + var.device(d) -= (m * alpha) / (vhat.sqrt() + epsilon()); + } +}; + template struct ApplyAdaMaxNonCuda { void operator()(const Device& d, typename TTypes::Flat var, @@ -2525,6 +2563,217 @@ TF_CALL_double(REGISTER_CPU_KERNELS); #undef REGISTER_CPU_KERNELS #undef REGISTER_KERNELS +template +class ApplyKerasMomentumOp : public OpKernel { + public: + explicit ApplyKerasMomentumOp(OpKernelConstruction* ctx) : OpKernel(ctx) { + OP_REQUIRES_OK(ctx, ctx->GetAttr("use_locking", &use_exclusive_lock_)); + OP_REQUIRES_OK(ctx, ctx->GetAttr("use_nesterov", &use_nesterov_)); + } + + void Compute(OpKernelContext* ctx) override { + auto locks = + MaybeLockVariableInputMutexesInOrder(ctx, use_exclusive_lock_, {0, 1}); + + Tensor var; + OP_REQUIRES_OK(ctx, GetInputTensorFromVariable( + ctx, 0, use_exclusive_lock_, false, &var)); + Tensor accum; + OP_REQUIRES_OK(ctx, GetInputTensorFromVariable( + ctx, 1, use_exclusive_lock_, false, &accum)); + OP_REQUIRES( + ctx, var.IsInitialized(), + errors::FailedPrecondition( + "Attempting to use uninitialized variables: ", requested_input(0))); + OP_REQUIRES( + ctx, accum.IsInitialized(), + errors::FailedPrecondition( + "Attempting to use uninitialized variables: ", requested_input(1))); + const Tensor& lr = ctx->input(2); + OP_REQUIRES(ctx, TensorShapeUtils::IsScalar(lr.shape()), + errors::InvalidArgument("lr is not a scalar: ", + lr.shape().DebugString())); + const Tensor& grad = ctx->input(3); + OP_REQUIRES( + ctx, var.shape().IsSameSize(accum.shape()), + errors::InvalidArgument("var and accum do not have the same shape", + var.shape().DebugString(), " ", + accum.shape().DebugString())); + OP_REQUIRES( + ctx, var.shape().IsSameSize(grad.shape()), + errors::InvalidArgument("var and grad do not have the same shape", + var.shape().DebugString(), " ", + grad.shape().DebugString())); + + const Tensor& momentum = ctx->input(4); + OP_REQUIRES(ctx, TensorShapeUtils::IsScalar(momentum.shape()), + errors::InvalidArgument("momentum is not a scalar: ", + momentum.shape().DebugString())); + + const Device& device = ctx->template eigen_device(); + functor::ApplyKerasMomentum()( + device, var.flat(), accum.flat(), lr.scalar(), grad.flat(), + momentum.scalar(), use_nesterov_); + MaybeForwardRefInputToRefOutput(ctx, 0, 0); + } + + private: + bool use_exclusive_lock_; + bool use_nesterov_; +}; + +#define REGISTER_KERNELS(D, T) \ + REGISTER_KERNEL_BUILDER(Name("ResourceApplyKerasMomentum") \ + .Device(DEVICE_##D) \ + .HostMemory("var") \ + .HostMemory("accum") \ + .TypeConstraint("T"), \ + ApplyKerasMomentumOp); +#define REGISTER_CPU_KERNELS(T) REGISTER_KERNELS(CPU, T); + +TF_CALL_half(REGISTER_CPU_KERNELS); +TF_CALL_bfloat16(REGISTER_CPU_KERNELS); +TF_CALL_float(REGISTER_CPU_KERNELS); +TF_CALL_double(REGISTER_CPU_KERNELS); + +#if GOOGLE_CUDA +// Forward declarations of the functor specializations for GPU. +namespace functor { +#define DECLARE_GPU_SPEC(T) \ + template <> \ + void ApplyKerasMomentum::operator()( \ + const GPUDevice& d, typename TTypes::Flat var, \ + typename TTypes::Flat accum, typename TTypes::ConstScalar lr, \ + typename TTypes::ConstFlat grad, \ + typename TTypes::ConstScalar momentum, bool use_nesterov); \ + extern template struct ApplyKerasMomentum; +DECLARE_GPU_SPEC(Eigen::half); +DECLARE_GPU_SPEC(float); +DECLARE_GPU_SPEC(double); +#undef DECLARE_GPU_SPEC +} // namespace functor + +REGISTER_KERNELS(GPU, Eigen::half); +REGISTER_KERNELS(GPU, float); +REGISTER_KERNELS(GPU, double); +#endif +#undef REGISTER_CPU_KERNELS +#undef REGISTER_KERNELS + +// Note, this op works on cpu only. +template +class SparseApplyKerasMomentumOp : public OpKernel { + public: + explicit SparseApplyKerasMomentumOp(OpKernelConstruction* ctx) + : OpKernel(ctx) { + OP_REQUIRES_OK(ctx, ctx->GetAttr("use_locking", &use_exclusive_lock_)); + OP_REQUIRES_OK(ctx, ctx->GetAttr("use_nesterov", &use_nesterov_)); + } + + void Compute(OpKernelContext* ctx) override NO_THREAD_SAFETY_ANALYSIS { + auto locks = + MaybeLockVariableInputMutexesInOrder(ctx, use_exclusive_lock_, {0, 1}); + + Tensor var; + OP_REQUIRES_OK(ctx, GetInputTensorFromVariable( + ctx, 0, use_exclusive_lock_, true, &var)); + Tensor accum; + OP_REQUIRES_OK(ctx, GetInputTensorFromVariable( + ctx, 1, use_exclusive_lock_, true, &accum)); + OP_REQUIRES( + ctx, var.IsInitialized(), + errors::FailedPrecondition( + "Attempting to use uninitialized variables: ", requested_input(0))); + OP_REQUIRES( + ctx, accum.IsInitialized(), + errors::FailedPrecondition( + "Attempting to use uninitialized variables: ", requested_input(1))); + OP_REQUIRES( + ctx, var.shape().IsSameSize(accum.shape()), + errors::InvalidArgument("var and accum do not have the same shape", + var.shape().DebugString(), " ", + accum.shape().DebugString())); + OP_REQUIRES(ctx, TensorShapeUtils::IsVectorOrHigher(var.shape()), + errors::InvalidArgument("var must be at least 1 dimensional")); + + const Tensor& lr = ctx->input(2); + OP_REQUIRES(ctx, TensorShapeUtils::IsScalar(lr.shape()), + errors::InvalidArgument("lr is not a scalar : ", + lr.shape().DebugString())); + const Tensor& grad = ctx->input(3); + const Tensor& indices = ctx->input(4); + OP_REQUIRES(ctx, TensorShapeUtils::IsVector(indices.shape()), + errors::InvalidArgument("indices must be one-dimensional")); + + for (int d = 1; d < var.dims(); d++) { + OP_REQUIRES(ctx, var.dim_size(d) == grad.dim_size(d), + errors::InvalidArgument(strings::StrCat( + "var and grad must match in dimension ", d))); + } + const Tindex N = indices.dim_size(0); + OP_REQUIRES( + ctx, grad.dim_size(0) == N, + errors::InvalidArgument( + "grad must be the same size as indices in the first dimension.")); + + const Tensor& momentum = ctx->input(5); + OP_REQUIRES(ctx, TensorShapeUtils::IsScalar(momentum.shape()), + errors::InvalidArgument("momentum is not a scalar: ", + momentum.shape().DebugString())); + + if (N > 0) { + const Tindex first_dim_size = var.dim_size(0); + auto indices_vec = indices.vec(); + auto var_flat = var.flat_outer_dims(); + auto accum_flat = accum.flat_outer_dims(); + auto grad_flat = grad.flat_outer_dims(); + T lr_scalar = lr.scalar()(); + T momentum_scalar = momentum.scalar()(); + + for (Tindex i = 0; i < N; i++) { + const Tindex index = internal::SubtleMustCopy(indices_vec(i)); + OP_REQUIRES(ctx, FastBoundsCheck(index, first_dim_size), + errors::InvalidArgument( + strings::StrCat("Index ", index, " at offset ", i, + " in indices is out of range"))); + auto a = accum_flat.template chip<0>(index); + auto g = grad_flat.template chip<0>(i); + auto v = var_flat.template chip<0>(index); + a = a * a.constant(momentum_scalar) - g * g.constant(lr_scalar); + if (use_nesterov_) { + v += a * a.constant(momentum_scalar) - g * g.constant(lr_scalar); + } else { + v += a; + } + } + } + + MaybeForwardRefInputToRefOutput(ctx, 0, 0); + } + + private: + bool use_exclusive_lock_; + bool use_nesterov_; +}; + +#define REGISTER_KERNELS(T, Tindices) \ + REGISTER_KERNEL_BUILDER(Name("ResourceSparseApplyKerasMomentum") \ + .Device(DEVICE_CPU) \ + .TypeConstraint("T") \ + .TypeConstraint("Tindices"), \ + SparseApplyKerasMomentumOp); +#define REGISTER_CPU_KERNELS(T) \ + REGISTER_KERNELS(T, int32); \ + REGISTER_KERNELS(T, int64); + +TF_CALL_half(REGISTER_CPU_KERNELS); +TF_CALL_bfloat16(REGISTER_CPU_KERNELS); +TF_CALL_float(REGISTER_CPU_KERNELS); +TF_CALL_double(REGISTER_CPU_KERNELS); + +#undef REGISTER_CPU_KERNELS +#undef REGISTER_KERNELS + template class ApplyAdamOp : public OpKernel { public: @@ -2786,6 +3035,147 @@ REGISTER_KERNELS(GPU, double); #undef REGISTER_CPU_KERNELS #undef REGISTER_KERNELS +template +class ApplyAdamWithAmsgradOp : public OpKernel { + public: + explicit ApplyAdamWithAmsgradOp(OpKernelConstruction* ctx) : OpKernel(ctx) { + OP_REQUIRES_OK(ctx, ctx->GetAttr("use_locking", &use_exclusive_lock_)); + } + + void Compute(OpKernelContext* ctx) override { + auto locks = MaybeLockVariableInputMutexesInOrder(ctx, use_exclusive_lock_, + {0, 1, 2}); + + Tensor var; + OP_REQUIRES_OK(ctx, GetInputTensorFromVariable( + ctx, 0, use_exclusive_lock_, false, &var)); + Tensor m; + OP_REQUIRES_OK(ctx, GetInputTensorFromVariable( + ctx, 1, use_exclusive_lock_, false, &m)); + Tensor v; + OP_REQUIRES_OK(ctx, GetInputTensorFromVariable( + ctx, 2, use_exclusive_lock_, false, &v)); + Tensor vhat; + OP_REQUIRES_OK(ctx, GetInputTensorFromVariable( + ctx, 3, use_exclusive_lock_, false, &vhat)); + OP_REQUIRES( + ctx, var.IsInitialized(), + errors::FailedPrecondition( + "Attempting to use uninitialized variables: ", requested_input(0))); + OP_REQUIRES( + ctx, m.IsInitialized(), + errors::FailedPrecondition( + "Attempting to use uninitialized variables: ", requested_input(1))); + OP_REQUIRES( + ctx, v.IsInitialized(), + errors::FailedPrecondition( + "Attempting to use uninitialized variables: ", requested_input(2))); + OP_REQUIRES( + ctx, vhat.IsInitialized(), + errors::FailedPrecondition( + "Attempting to use uninitialized variables: ", requested_input(2))); + + const Tensor& beta1_power = ctx->input(4); + const Tensor& beta2_power = ctx->input(5); + const Tensor& lr = ctx->input(6); + const Tensor& beta1 = ctx->input(7); + const Tensor& beta2 = ctx->input(8); + const Tensor& epsilon = ctx->input(9); + + OP_REQUIRES(ctx, TensorShapeUtils::IsScalar(beta1_power.shape()), + errors::InvalidArgument("beta1_power is not a scalar: ", + beta1_power.shape().DebugString())); + OP_REQUIRES(ctx, TensorShapeUtils::IsScalar(beta2_power.shape()), + errors::InvalidArgument("beta2_power is not a scalar: ", + beta2_power.shape().DebugString())); + OP_REQUIRES(ctx, TensorShapeUtils::IsScalar(lr.shape()), + errors::InvalidArgument("lr is not a scalar : ", + lr.shape().DebugString())); + OP_REQUIRES(ctx, TensorShapeUtils::IsScalar(beta1.shape()), + errors::InvalidArgument("beta1 is not a scalar: ", + beta1.shape().DebugString())); + OP_REQUIRES(ctx, TensorShapeUtils::IsScalar(beta2.shape()), + errors::InvalidArgument("beta2 is not a scalar: ", + beta2.shape().DebugString())); + OP_REQUIRES(ctx, TensorShapeUtils::IsScalar(epsilon.shape()), + errors::InvalidArgument("epsilon is not a scalar: ", + epsilon.shape().DebugString())); + + const Tensor& grad = ctx->input(10); + OP_REQUIRES(ctx, var.shape().IsSameSize(m.shape()), + errors::InvalidArgument("var and m do not have the same shape", + var.shape().DebugString(), " ", + m.shape().DebugString())); + OP_REQUIRES(ctx, var.shape().IsSameSize(v.shape()), + errors::InvalidArgument("var and v do not have the same shape", + var.shape().DebugString(), " ", + v.shape().DebugString())); + OP_REQUIRES( + ctx, var.shape().IsSameSize(grad.shape()), + errors::InvalidArgument("var and grad do not have the same shape", + var.shape().DebugString(), " ", + grad.shape().DebugString())); + + const Device& device = ctx->template eigen_device(); + functor::ApplyAdamWithAmsgrad()( + device, var.flat(), m.flat(), v.flat(), vhat.flat(), + beta1_power.scalar(), beta2_power.scalar(), lr.scalar(), + beta1.scalar(), beta2.scalar(), epsilon.scalar(), + grad.flat()); + + MaybeForwardRefInputToRefOutput(ctx, 0, 0); + } + + private: + bool use_exclusive_lock_; +}; + +#define REGISTER_KERNELS(D, T) \ + REGISTER_KERNEL_BUILDER(Name("ResourceApplyAdamWithAmsgrad") \ + .HostMemory("var") \ + .HostMemory("m") \ + .HostMemory("v") \ + .HostMemory("vhat") \ + .Device(DEVICE_##D) \ + .TypeConstraint("T"), \ + ApplyAdamWithAmsgradOp); +#define REGISTER_CPU_KERNELS(T) REGISTER_KERNELS(CPU, T); + +TF_CALL_half(REGISTER_CPU_KERNELS); +TF_CALL_bfloat16(REGISTER_CPU_KERNELS); +TF_CALL_float(REGISTER_CPU_KERNELS); +TF_CALL_double(REGISTER_CPU_KERNELS); + +#if GOOGLE_CUDA +// Forward declarations of the functor specializations for GPU. +namespace functor { +#define DECLARE_GPU_SPEC(T) \ + template <> \ + void ApplyAdamWithAmsgrad::operator()( \ + const GPUDevice& d, typename TTypes::Flat var, \ + typename TTypes::Flat m, typename TTypes::Flat v, \ + typename TTypes::Flat vhat, \ + typename TTypes::ConstScalar beta1_power, \ + typename TTypes::ConstScalar beta2_power, \ + typename TTypes::ConstScalar lr, \ + typename TTypes::ConstScalar beta1, \ + typename TTypes::ConstScalar beta2, \ + typename TTypes::ConstScalar epsilon, \ + typename TTypes::ConstFlat grad); \ + extern template struct ApplyAdamWithAmsgrad; +DECLARE_GPU_SPEC(Eigen::half); +DECLARE_GPU_SPEC(float); +DECLARE_GPU_SPEC(double); +#undef DECLARE_GPU_SPEC +} // namespace functor + +REGISTER_KERNELS(GPU, Eigen::half); +REGISTER_KERNELS(GPU, float); +REGISTER_KERNELS(GPU, double); +#endif +#undef REGISTER_CPU_KERNELS +#undef REGISTER_KERNELS + template class ApplyAdaMaxOp : public OpKernel { public: diff --git a/tensorflow/core/kernels/training_ops.h b/tensorflow/core/kernels/training_ops.h index e10a4cb1254..054f07350e6 100644 --- a/tensorflow/core/kernels/training_ops.h +++ b/tensorflow/core/kernels/training_ops.h @@ -126,6 +126,15 @@ struct ApplyMomentum { typename TTypes::ConstScalar momentum, bool use_nesterov); }; +template +struct ApplyKerasMomentum { + void operator()(const Device& d, typename TTypes::Flat var, + typename TTypes::Flat accum, + typename TTypes::ConstScalar lr, + typename TTypes::ConstFlat grad, + typename TTypes::ConstScalar momentum, bool use_nesterov); +}; + template struct ApplyAdam { void operator()(const Device& d, typename TTypes::Flat var, @@ -139,6 +148,20 @@ struct ApplyAdam { typename TTypes::ConstFlat grad, bool use_nesterov); }; +template +struct ApplyAdamWithAmsgrad { + void operator()(const Device& d, typename TTypes::Flat var, + typename TTypes::Flat m, typename TTypes::Flat v, + typename TTypes::Flat vhat, + typename TTypes::ConstScalar beta1_power, + typename TTypes::ConstScalar beta2_power, + typename TTypes::ConstScalar lr, + typename TTypes::ConstScalar beta1, + typename TTypes::ConstScalar beta2, + typename TTypes::ConstScalar epsilon, + typename TTypes::ConstFlat grad); +}; + template struct ApplyAdaMax { void operator()(const Device& d, typename TTypes::Flat var, diff --git a/tensorflow/core/kernels/training_ops_gpu.cu.cc b/tensorflow/core/kernels/training_ops_gpu.cu.cc index 4bd32592db1..f45b9ffca7c 100644 --- a/tensorflow/core/kernels/training_ops_gpu.cu.cc +++ b/tensorflow/core/kernels/training_ops_gpu.cu.cc @@ -101,6 +101,27 @@ struct ApplyMomentum { } }; +template +struct ApplyKerasMomentum { + void operator()(const GPUDevice& d, typename TTypes::Flat var, + typename TTypes::Flat accum, + typename TTypes::ConstScalar lr, + typename TTypes::ConstFlat grad, + typename TTypes::ConstScalar momentum, bool use_nesterov) { + Eigen::array::Tensor::Index, 1> bcast; + bcast[0] = grad.dimension(0); + Eigen::Sizes<1> single; + accum.device(d) = (accum * momentum.reshape(single).broadcast(bcast) - + grad * lr.reshape(single).broadcast(bcast)); + if (use_nesterov) { + var.device(d) += (accum * momentum.reshape(single).broadcast(bcast) - + grad * lr.reshape(single).broadcast(bcast)); + } else { + var.device(d) += accum; + } + } +}; + template struct ApplyAdam { void operator()(const GPUDevice& d, typename TTypes::Flat var, @@ -144,6 +165,39 @@ struct ApplyAdam { } }; +template +struct ApplyAdamWithAmsgrad { + void operator()(const GPUDevice& d, typename TTypes::Flat var, + typename TTypes::Flat m, typename TTypes::Flat v, + typename TTypes::Flat vhat, + typename TTypes::ConstScalar beta1_power, + typename TTypes::ConstScalar beta2_power, + typename TTypes::ConstScalar lr, + typename TTypes::ConstScalar beta1, + typename TTypes::ConstScalar beta2, + typename TTypes::ConstScalar epsilon, + typename TTypes::ConstFlat grad) { + Eigen::array::Tensor::Index, 1> bcast; + bcast[0] = grad.dimension(0); + Eigen::Sizes<1> single; + const auto one = static_cast(1.0); + m.device(d) = + m + (beta1.constant(one) - beta1).reshape(single).broadcast(bcast) * + (grad - m); + v.device(d) = + v + (beta2.constant(one) - beta2).reshape(single).broadcast(bcast) * + (grad.square() - v); + vhat.device(d) = vhat.cwiseMax(v); + + var.device(d) -= (lr * (beta2_power.constant(one) - beta2_power).sqrt() / + (beta1_power.constant(one) - beta1_power)) + .reshape(single) + .broadcast(bcast) * + m / + (epsilon.reshape(single).broadcast(bcast) + vhat.sqrt()); + } +}; + template struct ApplyAdaMax { void operator()(const GPUDevice& d, typename TTypes::Flat var, @@ -302,10 +356,18 @@ template struct functor::ApplyMomentum; template struct functor::ApplyMomentum; template struct functor::ApplyMomentum; +template struct functor::ApplyKerasMomentum; +template struct functor::ApplyKerasMomentum; +template struct functor::ApplyKerasMomentum; + template struct functor::ApplyAdam; template struct functor::ApplyAdam; template struct functor::ApplyAdam; +template struct functor::ApplyAdamWithAmsgrad; +template struct functor::ApplyAdamWithAmsgrad; +template struct functor::ApplyAdamWithAmsgrad; + template struct functor::ApplyAdaMax; template struct functor::ApplyAdaMax; template struct functor::ApplyAdaMax; diff --git a/tensorflow/core/kernels/training_ops_test.cc b/tensorflow/core/kernels/training_ops_test.cc index 2dcc4a500e6..1ec57b45221 100644 --- a/tensorflow/core/kernels/training_ops_test.cc +++ b/tensorflow/core/kernels/training_ops_test.cc @@ -151,6 +151,40 @@ static void BM_Momentum(int iters, int params) { } BENCHMARK(BM_Momentum)->Arg(128 << 10)->Arg(256 << 10); +static void KerasMomentum(int32 n, Graph** init_g, Graph** train_g) { + TensorShape shape({n}); + { + Graph* g = new Graph(OpRegistry::Global()); + auto var = Var(g, n); + auto accum = Var(g, n); + auto zero = Zeros(g, n); + test::graph::Assign(g, var, zero); + test::graph::Assign(g, accum, zero); + *init_g = g; + } + { + Graph* g = new Graph(OpRegistry::Global()); + auto var = Var(g, n); + auto accum = Var(g, n); + auto lr = Scalar(g, 0.01); + auto grad = Random(g, n); + auto mom = Scalar(g, 0.01); + test::graph::Multi(g, "ApplyKerasMomentum", {var, accum, lr, grad, mom}); + *train_g = g; + } +} + +static void BM_KerasMomentum(int iters, int params) { + const int64 tot = static_cast(iters) * params; + testing::ItemsProcessed(tot); + testing::BytesProcessed(tot * sizeof(float)); + Graph* init; + Graph* train; + KerasMomentum(params, &init, &train); + test::Benchmark("cpu", train, GetOptions(), init).Run(iters); +} +BENCHMARK(BM_KerasMomentum)->Arg(128 << 10)->Arg(256 << 10); + static void Adam(int32 n, Graph** init_g, Graph** train_g) { TensorShape shape({n}); { @@ -194,6 +228,50 @@ static void BM_Adam(int iters, int params) { } BENCHMARK(BM_Adam)->Arg(128 << 10)->Arg(256 << 10); +static void AdamWithAmsgrad(int32 n, Graph** init_g, Graph** train_g) { + TensorShape shape({n}); + { + Graph* g = new Graph(OpRegistry::Global()); + auto var = Var(g, n); + auto m = Var(g, n); + auto v = Var(g, n); + auto zero = Zeros(g, n); + test::graph::Assign(g, var, zero); + test::graph::Assign(g, m, zero); + test::graph::Assign(g, v, zero); + *init_g = g; + } + { + Graph* g = new Graph(OpRegistry::Global()); + auto var = Var(g, n); + auto m = Var(g, n); + auto v = Var(g, n); + auto vhat = Var(g, n); + auto beta1_power = Scalar(g, 0.9); + auto beta2_power = Scalar(g, 0.99); + auto lr = Scalar(g, 0.01); + auto beta1 = Scalar(g, 0.9); + auto beta2 = Scalar(g, 0.99); + auto epsilon = Scalar(g, 1e-8); + auto grad = Random(g, n); + test::graph::Multi(g, "ApplyAdamWithAmsgrad", + {var, m, v, vhat, beta1_power, beta2_power, lr, beta1, + beta2, epsilon, grad}); + *train_g = g; + } +} + +static void BM_AdamWithAmsgrad(int iters, int params) { + const int64 tot = static_cast(iters) * params; + testing::ItemsProcessed(tot); + testing::BytesProcessed(tot * sizeof(float)); + Graph* init; + Graph* train; + AdamWithAmsgrad(params, &init, &train); + test::Benchmark("cpu", train, GetOptions(), init).Run(iters); +} +BENCHMARK(BM_AdamWithAmsgrad)->Arg(128 << 10)->Arg(256 << 10); + static void RMSProp(int32 n, Graph** init_g, Graph** train_g) { TensorShape shape({n}); { diff --git a/tensorflow/core/kernels/unicode_ops.cc b/tensorflow/core/kernels/unicode_ops.cc index dd4415711b1..6c4ed1eaaf2 100644 --- a/tensorflow/core/kernels/unicode_ops.cc +++ b/tensorflow/core/kernels/unicode_ops.cc @@ -13,9 +13,17 @@ See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ +#include +#include +#include #include #include +#include +#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor" +#include "unicode/appendable.h" // TF:icu +#include "unicode/schriter.h" // TF:icu +#include "unicode/uchar.h" // TF:icu #include "unicode/ucnv.h" // TF:icu #include "unicode/ucnv_err.h" // TF:icu #include "unicode/umachine.h" // TF:icu @@ -23,15 +31,57 @@ limitations under the License. #include "unicode/unistr.h" // TF:icu #include "unicode/uset.h" // TF:icu #include "unicode/utypes.h" // TF:icu +#include "tensorflow/core/framework/kernel_def_builder.h" +#include "tensorflow/core/framework/op.h" #include "tensorflow/core/framework/op_kernel.h" +#include "tensorflow/core/framework/register_types.h" #include "tensorflow/core/framework/tensor.h" +#include "tensorflow/core/framework/tensor_shape.h" +#include "tensorflow/core/framework/tensor_types.h" +#include "tensorflow/core/framework/types.h" +#include "tensorflow/core/kernels/bounds_check.h" #include "tensorflow/core/kernels/string_util.h" #include "tensorflow/core/lib/core/errors.h" #include "tensorflow/core/lib/core/status.h" +#include "tensorflow/core/lib/core/stringpiece.h" +#include "tensorflow/core/platform/types.h" #include "tensorflow/core/util/bcast.h" #include "tensorflow/core/util/ptr_util.h" namespace tensorflow { +namespace { + +void Encode(const UnicodeEncoding encoding, const icu::UnicodeString& in, + string* out) { + if (encoding == UnicodeEncoding::UTF8) { + out->clear(); + in.toUTF8String(*out); + } else if (encoding == UnicodeEncoding::UTF16BE) { + // TODO(gbillock): consider using the + // extract(char *dest, int32_t destCapacity, UConverter *cnv) + // for UTF16/32 + out->clear(); // subtle: must come before reserve() + out->reserve(2 * in.length() + 1); + const char16_t* buf = in.getBuffer(); + for (int i = 0; i < in.length(); ++i) { + // Emit big-endian encoding for UTF-16 always. + out->push_back((buf[i] & 0xFF00) >> 8); + out->push_back(buf[i] & 0x00FF); + } + } else if (encoding == UnicodeEncoding::UTF32BE) { + out->clear(); // subtle: must come before reserve() + out->reserve(4 * in.countChar32() + 1); + icu::StringCharacterIterator it(in); + UChar32 ch; + while (it.hasNext()) { + ch = it.next32PostInc(); + out->push_back((ch & 0xFF000000) >> 24); + out->push_back((ch & 0x00FF0000) >> 16); + out->push_back((ch & 0x0000FF00) >> 8); + out->push_back((ch & 0x000000FF)); + } + } +} // This error callback is only useful for finding illegal encoding errors when // we want to be strict -- otherwise illegal encodings are replaced on read @@ -146,40 +196,66 @@ class WrappedConverter { string name_; }; +struct ErrorOptions { + UChar32 subst = 0xFFFD; + bool elide_replacement = false; + bool replace_control_chars = false; + bool error_on_malformatting = false; +}; + +Status GetErrorOptions(OpKernelConstruction* ctx, ErrorOptions* out) { + *out = ErrorOptions(); + + string error_policy; + TF_RETURN_IF_ERROR(ctx->GetAttr("errors", &error_policy)); + + if (error_policy == "replace") { + out->elide_replacement = false; + } else if (error_policy == "ignore") { + out->elide_replacement = true; + } else if (error_policy == "strict") { + out->error_on_malformatting = true; + } else { + return errors::InvalidArgument( + "errors policy must be one of 'strict', 'replace', or 'ignore'"); + } + + int32 replacement_char; + TF_RETURN_IF_ERROR(ctx->GetAttr("replacement_char", &replacement_char)); + + if (replacement_char >= UCHAR_MIN_VALUE && + replacement_char <= UCHAR_MAX_VALUE) { + out->subst = replacement_char; + } else { + return errors::InvalidArgument( + "replacement_char out of unicode codepoint range"); + } + + if (ctx->HasAttr("replace_control_characters")) { + TF_RETURN_IF_ERROR(ctx->GetAttr("replace_control_characters", + &(out->replace_control_chars))); + } + + return Status::OK(); +} + +inline bool ShouldHandleFormatError(const ErrorOptions& error_options, + UChar32 ch, bool format_error) { + return ((error_options.replace_control_chars && ch <= 0x1F) || format_error); +} + +} // namespace + class UnicodeTranscodeOp : public OpKernel { public: explicit UnicodeTranscodeOp(OpKernelConstruction* ctx) : OpKernel(ctx) { - string error_policy; - OP_REQUIRES_OK(ctx, ctx->GetAttr("errors", &error_policy)); - if (error_policy == "replace") { - elide_replacement_ = false; - } else if (error_policy == "ignore") { - elide_replacement_ = true; - } else if (error_policy == "strict") { - error_on_malformatting_ = true; - } else { - ctx->CtxFailure(errors::InvalidArgument( - "errors policy must be one of 'strict', 'replace', or 'ignore'")); - } - - int32 replacement_char; - OP_REQUIRES_OK(ctx, ctx->GetAttr("replacement_char", &replacement_char)); - if (replacement_char >= UCHAR_MIN_VALUE && - replacement_char <= UCHAR_MAX_VALUE) { - subst_ = replacement_char; - } else { - ctx->CtxFailure(errors::InvalidArgument( - "replacement_char out of unicode codepoint range")); - } + OP_REQUIRES_OK(ctx, GetErrorOptions(ctx, &error_options_)); string output_encoding; OP_REQUIRES_OK(ctx, ctx->GetAttr("output_encoding", &output_encoding)); OP_REQUIRES_OK(ctx, ParseUnicodeEncoding(output_encoding, &output_encoding_)); - OP_REQUIRES_OK(ctx, ctx->GetAttr("replace_control_characters", - &replace_control_chars_)); - OP_REQUIRES_OK(ctx, ctx->GetAttr("input_encoding", &input_encoding_)); // Make a temporary UConverter to ensure it will create without error // at execution time (and to warm any data caches the converter needs). @@ -228,7 +304,7 @@ class UnicodeTranscodeOp : public OpKernel { Transcode(&(output_flat(i)), input_encoder->converter_, &found_any_format_error); } - if (error_on_malformatting_ && found_any_format_error) { + if (error_options_.error_on_malformatting && found_any_format_error) { ctx->CtxFailure( errors::InvalidArgument("Invalid formatting on input string")); } @@ -240,12 +316,12 @@ class UnicodeTranscodeOp : public OpKernel { // out-of-range inputs. void TranslateCodepoints(icu::UnicodeString* s, bool* found_any_format_error, UChar32 ch, int src_bytes, bool format_error) { - if ((replace_control_chars_ && ch <= 0x1F) || format_error) { + if (ShouldHandleFormatError(error_options_, ch, format_error)) { *found_any_format_error = true; - if (elide_replacement_) { + if (error_options_.elide_replacement) { return; } else { - ch = subst_; + ch = error_options_.subst; } } s->append(ch); @@ -263,45 +339,202 @@ class UnicodeTranscodeOp : public OpKernel { found_any_format_error, std::placeholders::_1, std::placeholders::_2, std::placeholders::_3)); - if (output_encoding_ == UnicodeEncoding::UTF8) { - s->clear(); - source.toUTF8String(*s); - } else if (output_encoding_ == UnicodeEncoding::UTF16BE) { - // TODO(gbillock): consider using the - // extract(char *dest, int32_t destCapacity, UConverter *cnv) - // for UTF16/32 - s->clear(); // subtle: must come before reserve() - s->reserve(2 * source.length() + 1); - const char16_t* buf = source.getBuffer(); - for (int i = 0; i < source.length(); ++i) { - // Emit big-endian encoding for UTF-16 always. - s->push_back((buf[i] & 0xFF00) >> 8); - s->push_back(buf[i] & 0x00FF); - } - } else if (output_encoding_ == UnicodeEncoding::UTF32BE) { - s->clear(); // subtle: must come before reserve() - s->reserve(4 * source.countChar32() + 1); - for (int i = 0; i < source.countChar32(); ++i) { - // Emit big-endian encoding for UTF-32 always. - UChar32 ch = source.char32At(i); - s->push_back((ch & 0xFF000000) >> 24); - s->push_back((ch & 0x00FF0000) >> 16); - s->push_back((ch & 0x0000FF00) >> 8); - s->push_back((ch & 0x000000FF)); - } - } + Encode(output_encoding_, source, s); } - UChar32 subst_ = 0xFFFD; - bool elide_replacement_ = false; - bool replace_control_chars_ = false; - bool error_on_malformatting_ = false; - string input_encoding_; + ErrorOptions error_options_; UnicodeEncoding output_encoding_ = UnicodeEncoding::UTF8; }; REGISTER_KERNEL_BUILDER(Name("UnicodeTranscode").Device(DEVICE_CPU), UnicodeTranscodeOp); +class UnicodeDecodeWithOffsetsOp : public OpKernel { + public: + explicit UnicodeDecodeWithOffsetsOp(OpKernelConstruction* ctx) + : OpKernel(ctx) { + OP_REQUIRES_OK(ctx, GetErrorOptions(ctx, &error_options_)); + OP_REQUIRES_OK(ctx, ctx->GetAttr("input_encoding", &input_encoding_)); + // Make a temporary UConverter to ensure it will create without error + // at execution time (and to warm any data caches the converter needs). + // This instance is not used. + std::unique_ptr input_encoder = + absl::make_unique(); + input_encoder->init(input_encoding_); + OP_REQUIRES(ctx, input_encoder->converter_, + errors::InvalidArgument( + "Could not create converter for input encoding: " + + input_encoding_)); + } + + void Decode(OpKernelContext* ctx, std::vector* char_values, + std::vector* offset_values, int* string_length, + int64* next_row_split, UChar32 char_value, int char_length, + bool found_any_format_error) { + if (error_options_.error_on_malformatting && found_any_format_error) { + ctx->CtxFailure( + errors::InvalidArgument("Invalid formatting on input string")); + } + UChar32 decoded_value = char_value; + if (ShouldHandleFormatError(error_options_, char_value, + found_any_format_error)) { + if (error_options_.elide_replacement) { + return; + } else { + decoded_value = error_options_.subst; + } + } + + // Emit the char value. + char_values->push_back(decoded_value); + + // Emit the byte offset + offset_values->push_back(*string_length); + *string_length += char_length; + *next_row_split += 1; + } + + void Compute(OpKernelContext* ctx) override { + const Tensor* input_tensor; + OP_REQUIRES_OK(ctx, ctx->input("input", &input_tensor)); + + // Go through all the strings in `input`. + const auto& input_vec = input_tensor->flat(); + + std::unique_ptr input_encoder = + absl::make_unique(); + input_encoder->init(input_encoding_); + OP_REQUIRES(ctx, input_encoder->converter_, + errors::InvalidArgument( + "Could not create converter for input encoding: " + + input_encoding_)); + + std::vector char_values; + std::vector offset_values; + + Tensor* output_row_splits; + OP_REQUIRES_OK(ctx, ctx->allocate_output("row_splits", + {input_tensor->NumElements() + 1}, + &output_row_splits)); + auto out_row_splits = output_row_splits->vec(); + + int row_split_index = 0; + int64 next_row_split = 0; + for (int i = 0; i < input_vec.size(); ++i) { + const string& input = input_vec(i); + // Convert input strings into unicode values. Output to a list of + // char_values, record row splits and char_to_byte_starts, which are all + // the fields needed to construct a RaggedTensor. + out_row_splits(row_split_index) = next_row_split; + row_split_index++; + int string_length = 0; + IterateUnicodeString( + input, input_encoder->converter_, + std::bind(&UnicodeDecodeWithOffsetsOp::Decode, this, ctx, + &char_values, &offset_values, &string_length, + &next_row_split, std::placeholders::_1, + std::placeholders::_2, std::placeholders::_3)); + } + out_row_splits(row_split_index) = next_row_split; + + DCHECK(offset_values.size() == char_values.size()); + Tensor* output_char_values; + OP_REQUIRES_OK( + ctx, ctx->allocate_output("char_values", + {static_cast(char_values.size())}, + &output_char_values)); + Tensor* output_offset_values; + OP_REQUIRES_OK( + ctx, ctx->allocate_output("char_to_byte_starts", + {static_cast(offset_values.size())}, + &output_offset_values)); + auto out_char_values = output_char_values->vec(); + auto out_offset_values = output_offset_values->vec(); + + // Load output tensors from intermediate value arrays. + for (int i = 0; i < char_values.size(); ++i) { + out_char_values(i) = static_cast(char_values[i]); + out_offset_values(i) = offset_values[i]; + } + } + + private: + string input_encoding_; + ErrorOptions error_options_; +}; + +REGISTER_KERNEL_BUILDER(Name("UnicodeDecodeWithOffsets").Device(DEVICE_CPU), + UnicodeDecodeWithOffsetsOp); + +class UnicodeEncodeOp : public OpKernel { + public: + explicit UnicodeEncodeOp(OpKernelConstruction* ctx) : OpKernel(ctx) { + string encoding_tmp; + OP_REQUIRES_OK(ctx, ctx->GetAttr("output_encoding", &encoding_tmp)); + OP_REQUIRES_OK(ctx, ParseUnicodeEncoding(encoding_tmp, &encoding_)); + OP_REQUIRES_OK(ctx, GetErrorOptions(ctx, &error_options_)); + } + + /** + * Encodes Unicode codepoints into the desired string representation. + * + * We lose a dimension while encoding, since a series of integer codepoints is + * encoded into a single string. + * + * This accepts two input tensors: a rank 1 tensor of code point values and + * a single rank 1 tensor of splits which determine where each string begins + * and ends from the provided code points. + */ + void Compute(OpKernelContext* context) override { + // Get inputs + const Tensor& input_tensor = context->input(0); + const auto input_tensor_flat = input_tensor.flat(); + const Tensor& input_splits = context->input(1); + const auto input_splits_flat = input_splits.flat(); + + // Since we limit to a 2-D input (inner_values of rank 1 and a single splits + // tensor), our output dimension will be 1 with it's size equal to the + // number of splits (outer dimension or ragged tensor). + TensorShape output_shape({input_splits.dim_size(0) - 1}); + Tensor* output_tensor; + OP_REQUIRES_OK(context, context->allocate_output("output", output_shape, + &output_tensor)); + auto output_tensor_flat = output_tensor->flat(); + + // Use a single index over the flattened input values tensor. + int idx = 0; + // Loop through our split dimension to create a new string at each split. + for (int i = 1; i < input_splits_flat.size(); ++i) { + icu::UnicodeString unicode_string; + icu::UnicodeStringAppendable appendable_unicode_string(unicode_string); + for (; idx < input_splits_flat(i); ++idx) { + int32 code_point = input_tensor_flat(idx); + // Check for invalid code point + if (code_point > UCHAR_MAX_VALUE || code_point < UCHAR_MIN_VALUE) { + if (error_options_.error_on_malformatting) { + context->CtxFailure(errors::InvalidArgument( + "Code point value out of valid Unicode range.")); + return; + } else if (!error_options_.elide_replacement) { + code_point = error_options_.subst; + } + } + appendable_unicode_string.appendCodePoint(code_point); + } + // Encode our string and save in the output. + string result; + Encode(encoding_, unicode_string, &result); + output_tensor_flat(i - 1) = result; + } + } + + private: + UnicodeEncoding encoding_; + ErrorOptions error_options_; +}; + +REGISTER_KERNEL_BUILDER(Name("UnicodeEncode").Device(DEVICE_CPU), + UnicodeEncodeOp); + } // namespace tensorflow diff --git a/tensorflow/core/lib/core/threadpool.cc b/tensorflow/core/lib/core/threadpool.cc index 42689a6c3b3..e929ff45a1f 100644 --- a/tensorflow/core/lib/core/threadpool.cc +++ b/tensorflow/core/lib/core/threadpool.cc @@ -22,6 +22,7 @@ limitations under the License. #include "tensorflow/core/platform/denormal.h" #include "tensorflow/core/platform/logging.h" #include "tensorflow/core/platform/mutex.h" +#include "tensorflow/core/platform/numa.h" #include "tensorflow/core/platform/setround.h" #include "tensorflow/core/platform/tracing.h" #include "tensorflow/core/platform/types.h" @@ -54,6 +55,9 @@ struct EigenEnvironment { port::ScopedFlushDenormal flush; // Set the processor rounding mode to ROUND TO NEAREST. port::ScopedSetRound round(FE_TONEAREST); + if (thread_options_.numa_node != port::kNUMANoAffinity) { + port::NUMASetThreadNodeAffinity(thread_options_.numa_node); + } f(); }); } @@ -83,35 +87,38 @@ struct EigenEnvironment { struct ThreadPool::Impl : Eigen::ThreadPoolTempl { Impl(Env* env, const ThreadOptions& thread_options, const string& name, - int num_threads, bool low_latency_hint) + int num_threads, bool low_latency_hint, Eigen::Allocator* allocator) : Eigen::ThreadPoolTempl( num_threads, low_latency_hint, - EigenEnvironment(env, thread_options, name)) {} + EigenEnvironment(env, thread_options, name)), + allocator_(allocator) {} void ParallelFor(int64 total, int64 cost_per_unit, std::function fn) { CHECK_GE(total, 0); CHECK_EQ(total, (int64)(Eigen::Index)total); - Eigen::ThreadPoolDevice device(this, this->NumThreads()); + Eigen::ThreadPoolDevice device(this, this->NumThreads(), allocator_); device.parallelFor( total, Eigen::TensorOpCost(0, 0, cost_per_unit), [&fn](Eigen::Index first, Eigen::Index last) { fn(first, last); }); } + + Eigen::Allocator* allocator_; }; ThreadPool::ThreadPool(Env* env, const string& name, int num_threads) - : ThreadPool(env, ThreadOptions(), name, num_threads, true) {} + : ThreadPool(env, ThreadOptions(), name, num_threads, true, nullptr) {} ThreadPool::ThreadPool(Env* env, const ThreadOptions& thread_options, const string& name, int num_threads) - : ThreadPool(env, thread_options, name, num_threads, true) {} + : ThreadPool(env, thread_options, name, num_threads, true, nullptr) {} ThreadPool::ThreadPool(Env* env, const ThreadOptions& thread_options, const string& name, int num_threads, - bool low_latency_hint) { + bool low_latency_hint, Eigen::Allocator* allocator) { CHECK_GE(num_threads, 1); impl_.reset(new ThreadPool::Impl(env, thread_options, "tf_" + name, - num_threads, low_latency_hint)); + num_threads, low_latency_hint, allocator)); } ThreadPool::~ThreadPool() {} diff --git a/tensorflow/core/lib/core/threadpool.h b/tensorflow/core/lib/core/threadpool.h index 3da7dcb6328..90c9f294472 100644 --- a/tensorflow/core/lib/core/threadpool.h +++ b/tensorflow/core/lib/core/threadpool.h @@ -22,6 +22,9 @@ limitations under the License. #include "tensorflow/core/platform/macros.h" #include "tensorflow/core/platform/types.h" +namespace Eigen { +class Allocator; +} // namespace Eigen namespace tensorflow { namespace thread { @@ -37,7 +40,8 @@ class ThreadPool { // // REQUIRES: num_threads > 0 ThreadPool(Env* env, const ThreadOptions& thread_options, const string& name, - int num_threads, bool low_latency_hint); + int num_threads, bool low_latency_hint, + Eigen::Allocator* allocator = nullptr); // Constructs a pool for low-latency ops that contains "num_threads" threads // with specified "name". env->StartThread() is used to create individual diff --git a/tensorflow/core/lib/png/png_io.cc b/tensorflow/core/lib/png/png_io.cc index bc52180265c..e8dbcb97b94 100644 --- a/tensorflow/core/lib/png/png_io.cc +++ b/tensorflow/core/lib/png/png_io.cc @@ -92,7 +92,11 @@ void StringReader(png_structp png_ptr, png_bytep data, png_size_t length) { DecodeContext* const ctx = absl::bit_cast(png_get_io_ptr(png_ptr)); if (static_cast(ctx->data_left) < length) { - memset(data, 0, length); + // Don't zero out the data buffer as it has been lazily allocated (copy on + // write) and zeroing it out here can produce an OOM. Since the buffer is + // only used for reading data from the image, this doesn't result in any + // data leak, so it is safe to just leave the buffer be as it is and just + // exit with error. png_error(png_ptr, "More bytes requested to read than available"); } else { memcpy(data, ctx->data, length); diff --git a/tensorflow/core/nccl/BUILD b/tensorflow/core/nccl/BUILD index 50d9a2e8daa..4be33b2a0cf 100644 --- a/tensorflow/core/nccl/BUILD +++ b/tensorflow/core/nccl/BUILD @@ -11,6 +11,10 @@ exports_files(["LICENSE"]) load("//tensorflow:tensorflow.bzl", "tf_cuda_cc_test") load("//tensorflow:tensorflow.bzl", "tf_copts") load("@local_config_cuda//cuda:build_defs.bzl", "if_cuda") +load( + "//tensorflow/core:platform/default/build_config_root.bzl", + "tf_cuda_tests_tags", +) cc_library( name = "nccl_lib", @@ -34,27 +38,17 @@ cc_library( tf_cuda_cc_test( name = "nccl_manager_test", size = "medium", - srcs = if_cuda( - [ - "nccl_manager_test.cc", - ], - [], - ), - # Disabled on jenkins until errors finding nvmlShutdown are found. - tags = [ - "manual", - "multi_gpu", - "no_oss", - "noguitar", - "notap", + srcs = ["nccl_manager_test.cc"], + tags = tf_cuda_tests_tags() + [ + "no_cuda_on_cpu_tap", # TODO(b/120284216): re-enable multi_gpu ], - deps = - if_cuda([ - ":nccl_lib", - "@local_config_nccl//:nccl", - "//tensorflow/core:cuda", - "//tensorflow/core:test", - "//tensorflow/core:test_main", - "//tensorflow/core:testlib", - ]), + deps = [ + "//tensorflow/core:test", + "//tensorflow/core:test_main", + "//tensorflow/core:testlib", + ] + if_cuda([ + ":nccl_lib", + "@local_config_nccl//:nccl", + "//tensorflow/core:cuda", + ]), ) diff --git a/tensorflow/core/nccl/nccl_manager.cc b/tensorflow/core/nccl/nccl_manager.cc index f8e8c752227..df49bf1b976 100644 --- a/tensorflow/core/nccl/nccl_manager.cc +++ b/tensorflow/core/nccl/nccl_manager.cc @@ -24,6 +24,22 @@ limitations under the License. namespace tensorflow { +#define NCCL_RETURN_IF_ERROR(...) \ + do { \ + ncclResult_t nccl_status = (__VA_ARGS__); \ + if (nccl_status != ncclSuccess) { \ + return errors::Internal(ncclGetErrorString(nccl_status)); \ + } \ + } while (0) + +#define CUDA_RETURN_IF_ERROR(...) \ + do { \ + cudaError_t cuda_status = (__VA_ARGS__); \ + if (cuda_status != cudaSuccess) { \ + return errors::Internal(cudaGetErrorString(cuda_status)); \ + } \ + } while (0) + using se::cuda::ScopedActivateExecutorContext; // Contains data for a single stream used for nccl communication; this includes @@ -177,8 +193,8 @@ NcclManager* NcclManager::instance() { return instance; } -NcclManager::Communicator* NcclManager::GetCommunicator( - NcclManager::Collective* collective) { +Status NcclManager::GetCommunicator(NcclManager::Collective* collective, + NcclManager::Communicator** communicator) { // Sort by executor to make ordering of executors deterministic. std::sort(collective->participants.begin(), collective->participants.end(), [](const std::unique_ptr& a, @@ -217,7 +233,10 @@ NcclManager::Communicator* NcclManager::GetCommunicator( break; } } - if (i == num_devices) return comm.get(); + if (i == num_devices) { + *communicator = comm.get(); + return Status::OK(); + } } } @@ -264,37 +283,36 @@ NcclManager::Communicator* NcclManager::GetCommunicator( // NCCL2 prevents InitAll for more communicators than devices (but doesn't // check that device ids are unique). Work around it by initializing each // rank individually. - cudaGetDeviceCount(&device_count); + CUDA_RETURN_IF_ERROR(cudaGetDeviceCount(&device_count)); #endif std::vector nccl_comms(num_devices); if (num_devices <= device_count) { - auto result = - ncclCommInitAll(nccl_comms.data(), num_devices, devices.data()); - CHECK_EQ(result, ncclSuccess) << ncclGetErrorString(result); + NCCL_RETURN_IF_ERROR( + ncclCommInitAll(nccl_comms.data(), num_devices, devices.data())); } else { int savedDevice = 0; - CHECK_EQ(cudaGetDevice(&savedDevice), cudaSuccess); + CUDA_RETURN_IF_ERROR(cudaGetDevice(&savedDevice)); ncclUniqueId commId; - ncclGetUniqueId(&commId); + NCCL_RETURN_IF_ERROR(ncclGetUniqueId(&commId)); #if NCCL_MAJOR >= 2 - CHECK_EQ(ncclGroupStart(), ncclSuccess); + NCCL_RETURN_IF_ERROR(ncclGroupStart()); #endif for (int rank = 0; rank < num_devices; ++rank) { - cudaSetDevice(devices[rank]); - auto result = - ncclCommInitRank(nccl_comms.data() + rank, num_devices, commId, rank); - CHECK_EQ(result, ncclSuccess) << ncclGetErrorString(result); + CUDA_RETURN_IF_ERROR(cudaSetDevice(devices[rank])); + NCCL_RETURN_IF_ERROR(ncclCommInitRank(nccl_comms.data() + rank, + num_devices, commId, rank)); } #if NCCL_MAJOR >= 2 - CHECK_EQ(ncclGroupEnd(), ncclSuccess); + NCCL_RETURN_IF_ERROR(ncclGroupEnd()); #endif - cudaSetDevice(savedDevice); + CUDA_RETURN_IF_ERROR(cudaSetDevice(savedDevice)); } for (int rank = 0; rank < num_devices; ++rank) { members[rank].nccl_comm = nccl_comms[rank]; } communicators_.emplace_back(new Communicator(std::move(members))); - return communicators_.back().get(); + *communicator = communicators_.back().get(); + return Status::OK(); } void NcclManager::AddToAllReduce(int num_devices, const string& key, @@ -400,10 +418,18 @@ void NcclManager::AddParticipant(int num_devices, const string& key, void NcclManager::RunCollective(const string& key, Collective* collective) { static mutex collective_mu(LINKER_INITIALIZED); - auto* communicator = GetCommunicator(collective); - collective->communicator = communicator; - const int size = communicator->num_devices; + Communicator* communicator = nullptr; + const int size = static_cast(collective->participants.size()); + Status s = GetCommunicator(collective, &communicator); + if (!s.ok()) { + for (int i = 0; i < size; ++i) { + collective->participants[i]->done_callback(s); + } + delete collective; + return; + } + collective->communicator = communicator; for (int rank = 0; rank < size; ++rank) { Participant* p = collective->participants[rank].get(); NcclStream* nccl_stream = communicator->members[rank].nccl_stream; diff --git a/tensorflow/core/nccl/nccl_manager.h b/tensorflow/core/nccl/nccl_manager.h index 76b49101d47..5da4fe5554d 100644 --- a/tensorflow/core/nccl/nccl_manager.h +++ b/tensorflow/core/nccl/nccl_manager.h @@ -103,7 +103,13 @@ class NcclManager { struct NcclStream; struct Participant; - Communicator* GetCommunicator(Collective* collective); + // Gets the `Communicator` object that will be used to enqueue NCCL kernels + // for `collective`, and returns it via `communicator`. + // + // This may involve creating CUDA streams and NCCL initialization. If a NCCL + // or CUDA error occurs in the process, this returns an INTERNAL error with + // the corresponding NCCL/CUDA error string. + Status GetCommunicator(Collective* collective, Communicator** communicator); void AddParticipant(int num_devices, const string& key, std::unique_ptr participant, diff --git a/tensorflow/core/nccl/nccl_manager_test.cc b/tensorflow/core/nccl/nccl_manager_test.cc index dbc07865f0b..f43103e120b 100644 --- a/tensorflow/core/nccl/nccl_manager_test.cc +++ b/tensorflow/core/nccl/nccl_manager_test.cc @@ -28,8 +28,8 @@ limitations under the License. namespace tensorflow { -static std::vector GetGPUDevices() { - std::vector devices; +static std::vector> GetGPUDevices() { + std::vector> devices; SessionOptions session_options; session_options.config.mutable_gpu_options() ->set_per_process_gpu_memory_fraction(0.1); @@ -37,12 +37,12 @@ static std::vector GetGPUDevices() { Status s = DeviceFactory::GetFactory(DEVICE_GPU) ->AddDevices(session_options, "", &devices); TF_CHECK_OK(s); - std::vector gpus; - for (Device* d : devices) { - if (d->device_type() == "GPU") { - gpus.push_back(static_cast(d)); - } else { - delete d; + std::vector> gpus; + for (std::unique_ptr& device : devices) { + if (device->device_type() == "GPU") { + // If `device_type()` is GPU, this `Device` is guaranteed to be a + // `BaseGPUDevice`, which is a subclass of `Device`. + gpus.emplace_back(static_cast(device.release())); } } return gpus; @@ -64,16 +64,14 @@ class NcclManagerTest : public ::testing::Test { }; static void SetUpTestCase() { - setenv("NCCL_DEBUG", "INFO", 1 /* replace */); - devices_ = new std::vector(GetGPUDevices()); - CHECK(!devices_->empty()); + setenv("NCCL_DEBUG", "WARN", 1 /* replace */); + devices_ = new std::vector>(GetGPUDevices()); LOG(ERROR) << "Running test with " << devices_->size() << " gpus"; } - static void TearDownTestCase() { - for (auto device : *devices_) delete device; - delete devices_; - } + static int32 NumGPUs() { return static_cast(devices_->size()); } + + static void TearDownTestCase() { delete devices_; } TestCase* MakeTestCase(int num_ranks, ncclRedOp_t reduction_op, TensorShape shape, float value_offset) { @@ -153,7 +151,7 @@ class NcclManagerTest : public ::testing::Test { stream->ThenMemcpy(out_cpu.flat().data(), out_gpu_mem, out_cpu.TotalBytes()); SE_ASSERT_OK(stream->BlockHostUntilDone()); - test::ExpectTensorNear(test_case->expected, out_cpu, 0.01); + test::ExpectClose(test_case->expected, out_cpu); } } @@ -166,7 +164,7 @@ class NcclManagerTest : public ::testing::Test { } static BaseGPUDevice* GetDevice(size_t rank) { - return devices_->at(rank % devices_->size()); + return devices_->at(rank % devices_->size()).get(); } private: @@ -181,13 +179,14 @@ class NcclManagerTest : public ::testing::Test { } private: - static std::vector* devices_; + static std::vector>* devices_; static const DataType data_type_; static const Scalar max_; }; template -std::vector* NcclManagerTest::devices_ = nullptr; +std::vector>* NcclManagerTest::devices_ = + nullptr; template const DataType NcclManagerTest::data_type_ = DataTypeToEnum::value; @@ -195,13 +194,13 @@ template const Scalar NcclManagerTest::max_ = Eigen::NumTraits::highest(); -// Instantiate tests for float and half. -using TypeList = ::testing::Types; +// Instantiate tests for float and double. +using TypeList = ::testing::Types; TYPED_TEST_CASE(NcclManagerTest, TypeList); // Test basic sum reduction. TYPED_TEST(NcclManagerTest, BasicSumReduction) { - const int num_ranks = 3; + const int num_ranks = this->NumGPUs(); for (int op = 0; op < 4; ++op) { ncclRedOp_t reduction_op = static_cast(op); @@ -230,10 +229,10 @@ TYPED_TEST(NcclManagerTest, BasicSumReduction) { // To test the higher settings, increase num_ranks, // num_collectives_per_iteration and time_limit_micros. TYPED_TEST(NcclManagerTest, MultipleCallers) { - const int num_ranks = 1; // 2; - const int num_collectives_per_iteration = 1; // 1000; + const int num_ranks = this->NumGPUs(); + const int num_collectives_per_iteration = 10; // 1000; const int num_threads = 3; - const int time_limit_micros = 1; // 60 * 30 * 1000 * 1000; + const int time_limit_micros = 100; // 60 * 30 * 1000 * 1000; int64 start = Env::Default()->NowMicros(); srand(Env::Default()->NowMicros()); diff --git a/tensorflow/core/ops/array_ops.cc b/tensorflow/core/ops/array_ops.cc index f55562ec99d..281e2996ed7 100644 --- a/tensorflow/core/ops/array_ops.cc +++ b/tensorflow/core/ops/array_ops.cc @@ -2743,6 +2743,9 @@ REGISTER_OP("QuantizeAndDequantizeV2") .Attr("range_given: bool = false") .Output("output: T") .Attr("T: {bfloat16, half, float, double}") + .Attr( + "round_mode: {'HALF_TO_EVEN', 'HALF_UP'} = " + "'HALF_TO_EVEN'") .SetShapeFn([](InferenceContext* c) { ShapeHandle unused; TF_RETURN_IF_ERROR(c->WithRank(c->input(1), 0, &unused)); @@ -2878,14 +2881,9 @@ REGISTER_OP("QuantizedInstanceNorm") namespace { -Status ScatterNdShape(InferenceContext* c) { - ShapeHandle indices_shape; - TF_RETURN_IF_ERROR(c->WithRankAtLeast(c->input(0), 1, &indices_shape)); - ShapeHandle updates_shape; - TF_RETURN_IF_ERROR(c->WithRankAtLeast(c->input(1), 1, &updates_shape)); - ShapeHandle output_shape; - TF_RETURN_IF_ERROR(c->MakeShapeFromShapeTensor(2, &output_shape)); - +Status ScatterNdShapeHelper(InferenceContext* c, ShapeHandle indices_shape, + ShapeHandle updates_shape, + ShapeHandle output_shape) { if (c->Value(c->NumElements(output_shape)) == 0 && (c->Value(c->NumElements(indices_shape)) > 0 || c->Value(c->NumElements(updates_shape)) > 0)) { @@ -2940,6 +2938,26 @@ Status ScatterNdShape(InferenceContext* c) { return Status::OK(); } +Status ScatterNdShape(InferenceContext* c) { + ShapeHandle indices_shape; + TF_RETURN_IF_ERROR(c->WithRankAtLeast(c->input(0), 1, &indices_shape)); + ShapeHandle updates_shape; + TF_RETURN_IF_ERROR(c->WithRankAtLeast(c->input(1), 1, &updates_shape)); + ShapeHandle output_shape; + TF_RETURN_IF_ERROR(c->MakeShapeFromShapeTensor(2, &output_shape)); + return ScatterNdShapeHelper(c, indices_shape, updates_shape, output_shape); +} + +Status ScatterNdTensorShape(InferenceContext* c) { + ShapeHandle output_shape; + TF_RETURN_IF_ERROR(c->WithRankAtLeast(c->input(0), 1, &output_shape)); + ShapeHandle indices_shape; + TF_RETURN_IF_ERROR(c->WithRankAtLeast(c->input(1), 1, &indices_shape)); + ShapeHandle updates_shape; + TF_RETURN_IF_ERROR(c->WithRankAtLeast(c->input(2), 1, &updates_shape)); + return ScatterNdShapeHelper(c, indices_shape, updates_shape, output_shape); +} + } // namespace REGISTER_OP("UpperBound") @@ -2979,6 +2997,33 @@ REGISTER_OP("ScatterNd") .Attr("Tindices: {int32, int64}") .SetShapeFn(ScatterNdShape); +REGISTER_OP("TensorScatterUpdate") + .Input("tensor: T") + .Input("indices: Tindices") + .Input("updates: T") + .Output("output: T") + .Attr("T: type") + .Attr("Tindices: {int32, int64}") + .SetShapeFn(ScatterNdTensorShape); + +REGISTER_OP("TensorScatterAdd") + .Input("tensor: T") + .Input("indices: Tindices") + .Input("updates: T") + .Output("output: T") + .Attr("T: type") + .Attr("Tindices: {int32, int64}") + .SetShapeFn(ScatterNdTensorShape); + +REGISTER_OP("TensorScatterSub") + .Input("tensor: T") + .Input("indices: Tindices") + .Input("updates: T") + .Output("output: T") + .Attr("T: type") + .Attr("Tindices: {int32, int64}") + .SetShapeFn(ScatterNdTensorShape); + REGISTER_OP("ScatterNdNonAliasingAdd") .Input("input: T") .Input("indices: Tindices") diff --git a/tensorflow/core/ops/compat/ops_history.v1.pbtxt b/tensorflow/core/ops/compat/ops_history.v1.pbtxt index bfcc92dcb0f..8022c390c62 100644 --- a/tensorflow/core/ops/compat/ops_history.v1.pbtxt +++ b/tensorflow/core/ops/compat/ops_history.v1.pbtxt @@ -12076,33 +12076,6 @@ op { type: "list(float)" } } -op { - name: "BytesProducedStatsDataset" - input_arg { - name: "input_dataset" - type: DT_VARIANT - } - input_arg { - name: "tag" - type: DT_STRING - } - output_arg { - name: "handle" - type: DT_VARIANT - } - attr { - name: "output_types" - type: "list(type)" - has_minimum: true - minimum: 1 - } - attr { - name: "output_shapes" - type: "list(shape)" - has_minimum: true - minimum: 1 - } -} op { name: "CTCBeamSearchDecoder" input_arg { @@ -17326,21 +17299,6 @@ op { minimum: 1 } } -op { - name: "DatasetToTFRecord" - input_arg { - name: "input_dataset" - type: DT_VARIANT - } - input_arg { - name: "filename" - type: DT_STRING - } - input_arg { - name: "compression_type" - type: DT_STRING - } -} op { name: "DebugGradientIdentity" input_arg { @@ -18487,69 +18445,6 @@ op { } } } -op { - name: "DenseToSparseBatchDataset" - input_arg { - name: "input_dataset" - type: DT_VARIANT - } - input_arg { - name: "batch_size" - type: DT_INT64 - } - input_arg { - name: "row_shape" - type: DT_INT64 - } - output_arg { - name: "handle" - type: DT_VARIANT - } - attr { - name: "output_types" - type: "list(type)" - has_minimum: true - minimum: 1 - } - attr { - name: "output_shapes" - type: "list(shape)" - has_minimum: true - minimum: 1 - } - is_stateful: true -} -op { - name: "DenseToSparseBatchDataset" - input_arg { - name: "input_dataset" - type: DT_VARIANT - } - input_arg { - name: "batch_size" - type: DT_INT64 - } - input_arg { - name: "row_shape" - type: DT_INT64 - } - output_arg { - name: "handle" - type: DT_VARIANT - } - attr { - name: "output_types" - type: "list(type)" - has_minimum: true - minimum: 1 - } - attr { - name: "output_shapes" - type: "list(shape)" - has_minimum: true - minimum: 1 - } -} op { name: "DenseToSparseSetOperation" input_arg { @@ -21153,24 +21048,6 @@ op { type: DT_STRING } } -op { - name: "EnqueueInQueueDataset" - input_arg { - name: "queue" - type: DT_VARIANT - } - input_arg { - name: "components" - type_list_attr: "Tcomponents" - } - attr { - name: "Tcomponents" - type: "list(type)" - has_minimum: true - minimum: 1 - } - is_stateful: true -} op { name: "EnsureShape" input_arg { @@ -21626,6 +21503,33 @@ op { minimum: 1 } } +op { + name: "ExperimentalBytesProducedStatsDataset" + input_arg { + name: "input_dataset" + type: DT_VARIANT + } + input_arg { + name: "tag" + type: DT_STRING + } + output_arg { + name: "handle" + type: DT_VARIANT + } + attr { + name: "output_types" + type: "list(type)" + has_minimum: true + minimum: 1 + } + attr { + name: "output_shapes" + type: "list(shape)" + has_minimum: true + minimum: 1 + } +} op { name: "ExperimentalCSVDataset" input_arg { @@ -21691,6 +21595,84 @@ op { } is_stateful: true } +op { + name: "ExperimentalDatasetToTFRecord" + input_arg { + name: "input_dataset" + type: DT_VARIANT + } + input_arg { + name: "filename" + type: DT_STRING + } + input_arg { + name: "compression_type" + type: DT_STRING + } +} +op { + name: "ExperimentalDenseToSparseBatchDataset" + input_arg { + name: "input_dataset" + type: DT_VARIANT + } + input_arg { + name: "batch_size" + type: DT_INT64 + } + input_arg { + name: "row_shape" + type: DT_INT64 + } + output_arg { + name: "handle" + type: DT_VARIANT + } + attr { + name: "output_types" + type: "list(type)" + has_minimum: true + minimum: 1 + } + attr { + name: "output_shapes" + type: "list(shape)" + has_minimum: true + minimum: 1 + } + is_stateful: true +} +op { + name: "ExperimentalDenseToSparseBatchDataset" + input_arg { + name: "input_dataset" + type: DT_VARIANT + } + input_arg { + name: "batch_size" + type: DT_INT64 + } + input_arg { + name: "row_shape" + type: DT_INT64 + } + output_arg { + name: "handle" + type: DT_VARIANT + } + attr { + name: "output_types" + type: "list(type)" + has_minimum: true + minimum: 1 + } + attr { + name: "output_shapes" + type: "list(shape)" + has_minimum: true + minimum: 1 + } +} op { name: "ExperimentalDirectedInterleaveDataset" input_arg { @@ -21726,50 +21708,66 @@ op { } } op { - name: "ExperimentalFunctionBufferingResource" + name: "ExperimentalGroupByReducerDataset" input_arg { - name: "string_arg" - type: DT_STRING + name: "input_dataset" + type: DT_VARIANT } input_arg { - name: "target_device" - type: DT_STRING + name: "key_func_other_arguments" + type_list_attr: "Tkey_func_other_arguments" + } + input_arg { + name: "init_func_other_arguments" + type_list_attr: "Tinit_func_other_arguments" + } + input_arg { + name: "reduce_func_other_arguments" + type_list_attr: "Treduce_func_other_arguments" + } + input_arg { + name: "finalize_func_other_arguments" + type_list_attr: "Tfinalize_func_other_arguments" } output_arg { - name: "resource" - type: DT_RESOURCE + name: "handle" + type: DT_VARIANT } attr { - name: "shared_name" - type: "string" - } - attr { - name: "container" - type: "string" - } - attr { - name: "f" + name: "key_func" type: "func" } attr { - name: "buffer_size" - type: "int" + name: "init_func" + type: "func" } attr { - name: "output_types" + name: "reduce_func" + type: "func" + } + attr { + name: "finalize_func" + type: "func" + } + attr { + name: "Tkey_func_other_arguments" type: "list(type)" + has_minimum: true } - is_stateful: true -} -op { - name: "ExperimentalFunctionBufferingResourceGetNext" - input_arg { - name: "function_buffer_resource" - type: DT_RESOURCE + attr { + name: "Tinit_func_other_arguments" + type: "list(type)" + has_minimum: true } - output_arg { - name: "output" - type_list_attr: "output_types" + attr { + name: "Treduce_func_other_arguments" + type: "list(type)" + has_minimum: true + } + attr { + name: "Tfinalize_func_other_arguments" + type: "list(type)" + has_minimum: true } attr { name: "output_types" @@ -21777,16 +21775,139 @@ op { has_minimum: true minimum: 1 } + attr { + name: "output_shapes" + type: "list(shape)" + has_minimum: true + minimum: 1 + } is_stateful: true } op { - name: "ExperimentalFunctionBufferingResourceReset" + name: "ExperimentalGroupByWindowDataset" input_arg { - name: "function_buffer_resource" - type: DT_RESOURCE + name: "input_dataset" + type: DT_VARIANT + } + input_arg { + name: "key_func_other_arguments" + type_list_attr: "Tkey_func_other_arguments" + } + input_arg { + name: "reduce_func_other_arguments" + type_list_attr: "Treduce_func_other_arguments" + } + input_arg { + name: "window_size_func_other_arguments" + type_list_attr: "Twindow_size_func_other_arguments" + } + output_arg { + name: "handle" + type: DT_VARIANT + } + attr { + name: "key_func" + type: "func" + } + attr { + name: "reduce_func" + type: "func" + } + attr { + name: "window_size_func" + type: "func" + } + attr { + name: "Tkey_func_other_arguments" + type: "list(type)" + has_minimum: true + } + attr { + name: "Treduce_func_other_arguments" + type: "list(type)" + has_minimum: true + } + attr { + name: "Twindow_size_func_other_arguments" + type: "list(type)" + has_minimum: true + } + attr { + name: "output_types" + type: "list(type)" + has_minimum: true + minimum: 1 + } + attr { + name: "output_shapes" + type: "list(shape)" + has_minimum: true + minimum: 1 } is_stateful: true } +op { + name: "ExperimentalGroupByWindowDataset" + input_arg { + name: "input_dataset" + type: DT_VARIANT + } + input_arg { + name: "key_func_other_arguments" + type_list_attr: "Tkey_func_other_arguments" + } + input_arg { + name: "reduce_func_other_arguments" + type_list_attr: "Treduce_func_other_arguments" + } + input_arg { + name: "window_size_func_other_arguments" + type_list_attr: "Twindow_size_func_other_arguments" + } + output_arg { + name: "handle" + type: DT_VARIANT + } + attr { + name: "key_func" + type: "func" + } + attr { + name: "reduce_func" + type: "func" + } + attr { + name: "window_size_func" + type: "func" + } + attr { + name: "Tkey_func_other_arguments" + type: "list(type)" + has_minimum: true + } + attr { + name: "Treduce_func_other_arguments" + type: "list(type)" + has_minimum: true + } + attr { + name: "Twindow_size_func_other_arguments" + type: "list(type)" + has_minimum: true + } + attr { + name: "output_types" + type: "list(type)" + has_minimum: true + minimum: 1 + } + attr { + name: "output_shapes" + type: "list(shape)" + has_minimum: true + minimum: 1 + } +} op { name: "ExperimentalIdentityIndexedDataset" input_arg { @@ -21898,6 +22019,136 @@ op { } is_stateful: true } +op { + name: "ExperimentalLatencyStatsDataset" + input_arg { + name: "input_dataset" + type: DT_VARIANT + } + input_arg { + name: "tag" + type: DT_STRING + } + output_arg { + name: "handle" + type: DT_VARIANT + } + attr { + name: "output_types" + type: "list(type)" + has_minimum: true + minimum: 1 + } + attr { + name: "output_shapes" + type: "list(shape)" + has_minimum: true + minimum: 1 + } +} +op { + name: "ExperimentalMapAndBatchDataset" + input_arg { + name: "input_dataset" + type: DT_VARIANT + } + input_arg { + name: "other_arguments" + type_list_attr: "Targuments" + } + input_arg { + name: "batch_size" + type: DT_INT64 + } + input_arg { + name: "num_parallel_calls" + type: DT_INT64 + } + input_arg { + name: "drop_remainder" + type: DT_BOOL + } + output_arg { + name: "handle" + type: DT_VARIANT + } + attr { + name: "f" + type: "func" + } + attr { + name: "Targuments" + type: "list(type)" + has_minimum: true + } + attr { + name: "output_types" + type: "list(type)" + has_minimum: true + minimum: 1 + } + attr { + name: "output_shapes" + type: "list(shape)" + has_minimum: true + minimum: 1 + } +} +op { + name: "ExperimentalMapDataset" + input_arg { + name: "input_dataset" + type: DT_VARIANT + } + input_arg { + name: "other_arguments" + type_list_attr: "Targuments" + } + output_arg { + name: "handle" + type: DT_VARIANT + } + attr { + name: "f" + type: "func" + } + attr { + name: "Targuments" + type: "list(type)" + has_minimum: true + } + attr { + name: "output_types" + type: "list(type)" + has_minimum: true + minimum: 1 + } + attr { + name: "output_shapes" + type: "list(shape)" + has_minimum: true + minimum: 1 + } + attr { + name: "use_inter_op_parallelism" + type: "bool" + default_value { + b: true + } + } +} +op { + name: "ExperimentalMatchingFilesDataset" + input_arg { + name: "patterns" + type: DT_STRING + } + output_arg { + name: "handle" + type: DT_VARIANT + } + is_stateful: true +} op { name: "ExperimentalMaterializedIndexDatasetHandle" output_arg { @@ -21926,6 +22177,33 @@ op { } is_stateful: true } +op { + name: "ExperimentalMaxIntraOpParallelismDataset" + input_arg { + name: "input_dataset" + type: DT_VARIANT + } + input_arg { + name: "max_intra_op_parallelism" + type: DT_INT64 + } + output_arg { + name: "handle" + type: DT_VARIANT + } + attr { + name: "output_types" + type: "list(type)" + has_minimum: true + minimum: 1 + } + attr { + name: "output_shapes" + type: "list(shape)" + has_minimum: true + minimum: 1 + } +} op { name: "ExperimentalNonSerializableDataset" input_arg { @@ -21997,6 +22275,346 @@ op { minimum: 1 } } +op { + name: "ExperimentalParallelInterleaveDataset" + input_arg { + name: "input_dataset" + type: DT_VARIANT + } + input_arg { + name: "other_arguments" + type_list_attr: "Targuments" + } + input_arg { + name: "cycle_length" + type: DT_INT64 + } + input_arg { + name: "block_length" + type: DT_INT64 + } + input_arg { + name: "sloppy" + type: DT_BOOL + } + input_arg { + name: "buffer_output_elements" + type: DT_INT64 + } + input_arg { + name: "prefetch_input_elements" + type: DT_INT64 + } + output_arg { + name: "handle" + type: DT_VARIANT + } + attr { + name: "f" + type: "func" + } + attr { + name: "Targuments" + type: "list(type)" + has_minimum: true + } + attr { + name: "output_types" + type: "list(type)" + has_minimum: true + minimum: 1 + } + attr { + name: "output_shapes" + type: "list(shape)" + has_minimum: true + minimum: 1 + } +} +op { + name: "ExperimentalParseExampleDataset" + input_arg { + name: "input_dataset" + type: DT_VARIANT + } + input_arg { + name: "num_parallel_calls" + type: DT_INT64 + } + input_arg { + name: "dense_defaults" + type_list_attr: "Tdense" + } + output_arg { + name: "handle" + type: DT_VARIANT + } + attr { + name: "sparse_keys" + type: "list(string)" + has_minimum: true + } + attr { + name: "dense_keys" + type: "list(string)" + has_minimum: true + } + attr { + name: "sparse_types" + type: "list(type)" + has_minimum: true + allowed_values { + list { + type: DT_FLOAT + type: DT_INT64 + type: DT_STRING + } + } + } + attr { + name: "Tdense" + type: "list(type)" + has_minimum: true + allowed_values { + list { + type: DT_FLOAT + type: DT_INT64 + type: DT_STRING + } + } + } + attr { + name: "dense_shapes" + type: "list(shape)" + has_minimum: true + } + attr { + name: "output_types" + type: "list(type)" + has_minimum: true + minimum: 1 + } + attr { + name: "output_shapes" + type: "list(shape)" + has_minimum: true + minimum: 1 + } +} +op { + name: "ExperimentalParseExampleDataset" + input_arg { + name: "input_dataset" + type: DT_VARIANT + } + input_arg { + name: "num_parallel_calls" + type: DT_INT64 + } + input_arg { + name: "dense_defaults" + type_list_attr: "Tdense" + } + output_arg { + name: "handle" + type: DT_VARIANT + } + attr { + name: "sparse_keys" + type: "list(string)" + has_minimum: true + } + attr { + name: "dense_keys" + type: "list(string)" + has_minimum: true + } + attr { + name: "sparse_types" + type: "list(type)" + has_minimum: true + allowed_values { + list { + type: DT_FLOAT + type: DT_INT64 + type: DT_STRING + } + } + } + attr { + name: "Tdense" + type: "list(type)" + has_minimum: true + allowed_values { + list { + type: DT_FLOAT + type: DT_INT64 + type: DT_STRING + } + } + } + attr { + name: "dense_shapes" + type: "list(shape)" + has_minimum: true + } + attr { + name: "output_types" + type: "list(type)" + has_minimum: true + minimum: 1 + } + attr { + name: "output_shapes" + type: "list(shape)" + has_minimum: true + minimum: 1 + } + attr { + name: "sloppy" + type: "bool" + default_value { + b: false + } + } +} +op { + name: "ExperimentalPrivateThreadPoolDataset" + input_arg { + name: "input_dataset" + type: DT_VARIANT + } + input_arg { + name: "num_threads" + type: DT_INT64 + } + output_arg { + name: "handle" + type: DT_VARIANT + } + attr { + name: "output_types" + type: "list(type)" + has_minimum: true + minimum: 1 + } + attr { + name: "output_shapes" + type: "list(shape)" + has_minimum: true + minimum: 1 + } +} +op { + name: "ExperimentalRandomDataset" + input_arg { + name: "seed" + type: DT_INT64 + } + input_arg { + name: "seed2" + type: DT_INT64 + } + output_arg { + name: "handle" + type: DT_VARIANT + } + attr { + name: "output_types" + type: "list(type)" + has_minimum: true + minimum: 1 + } + attr { + name: "output_shapes" + type: "list(shape)" + has_minimum: true + minimum: 1 + } + is_stateful: true +} +op { + name: "ExperimentalScanDataset" + input_arg { + name: "input_dataset" + type: DT_VARIANT + } + input_arg { + name: "initial_state" + type_list_attr: "Tstate" + } + input_arg { + name: "other_arguments" + type_list_attr: "Targuments" + } + output_arg { + name: "handle" + type: DT_VARIANT + } + attr { + name: "f" + type: "func" + } + attr { + name: "Tstate" + type: "list(type)" + has_minimum: true + minimum: 1 + } + attr { + name: "Targuments" + type: "list(type)" + has_minimum: true + } + attr { + name: "output_types" + type: "list(type)" + has_minimum: true + minimum: 1 + } + attr { + name: "output_shapes" + type: "list(shape)" + has_minimum: true + minimum: 1 + } +} +op { + name: "ExperimentalSetStatsAggregatorDataset" + input_arg { + name: "input_dataset" + type: DT_VARIANT + } + input_arg { + name: "stats_aggregator" + type: DT_RESOURCE + } + input_arg { + name: "tag" + type: DT_STRING + } + input_arg { + name: "counter_prefix" + type: DT_STRING + } + output_arg { + name: "handle" + type: DT_VARIANT + } + attr { + name: "output_types" + type: "list(type)" + has_minimum: true + minimum: 1 + } + attr { + name: "output_shapes" + type: "list(shape)" + has_minimum: true + minimum: 1 + } + is_stateful: true +} op { name: "ExperimentalSleepDataset" input_arg { @@ -22024,6 +22642,107 @@ op { minimum: 1 } } +op { + name: "ExperimentalSlidingWindowDataset" + input_arg { + name: "input_dataset" + type: DT_VARIANT + } + input_arg { + name: "window_size" + type: DT_INT64 + } + input_arg { + name: "window_shift" + type: DT_INT64 + } + input_arg { + name: "window_stride" + type: DT_INT64 + } + output_arg { + name: "handle" + type: DT_VARIANT + } + attr { + name: "output_types" + type: "list(type)" + has_minimum: true + minimum: 1 + } + attr { + name: "output_shapes" + type: "list(shape)" + has_minimum: true + minimum: 1 + } +} +op { + name: "ExperimentalSqlDataset" + input_arg { + name: "driver_name" + type: DT_STRING + } + input_arg { + name: "data_source_name" + type: DT_STRING + } + input_arg { + name: "query" + type: DT_STRING + } + output_arg { + name: "handle" + type: DT_VARIANT + } + attr { + name: "output_types" + type: "list(type)" + has_minimum: true + minimum: 1 + } + attr { + name: "output_shapes" + type: "list(shape)" + has_minimum: true + minimum: 1 + } + is_stateful: true +} +op { + name: "ExperimentalStatsAggregatorHandle" + output_arg { + name: "handle" + type: DT_RESOURCE + } + attr { + name: "container" + type: "string" + default_value { + s: "" + } + } + attr { + name: "shared_name" + type: "string" + default_value { + s: "" + } + } + is_stateful: true +} +op { + name: "ExperimentalStatsAggregatorSummary" + input_arg { + name: "iterator" + type: DT_RESOURCE + } + output_arg { + name: "summary" + type: DT_STRING + } + is_stateful: true +} op { name: "ExperimentalThreadPoolDataset" input_arg { @@ -22089,6 +22808,29 @@ op { } is_stateful: true } +op { + name: "ExperimentalUnbatchDataset" + input_arg { + name: "input_dataset" + type: DT_VARIANT + } + output_arg { + name: "handle" + type: DT_VARIANT + } + attr { + name: "output_types" + type: "list(type)" + has_minimum: true + minimum: 1 + } + attr { + name: "output_shapes" + type: "list(shape)" + has_minimum: true + minimum: 1 + } +} op { name: "ExperimentalUniqueDataset" input_arg { @@ -26313,207 +27055,6 @@ op { } } } -op { - name: "GroupByReducerDataset" - input_arg { - name: "input_dataset" - type: DT_VARIANT - } - input_arg { - name: "key_func_other_arguments" - type_list_attr: "Tkey_func_other_arguments" - } - input_arg { - name: "init_func_other_arguments" - type_list_attr: "Tinit_func_other_arguments" - } - input_arg { - name: "reduce_func_other_arguments" - type_list_attr: "Treduce_func_other_arguments" - } - input_arg { - name: "finalize_func_other_arguments" - type_list_attr: "Tfinalize_func_other_arguments" - } - output_arg { - name: "handle" - type: DT_VARIANT - } - attr { - name: "key_func" - type: "func" - } - attr { - name: "init_func" - type: "func" - } - attr { - name: "reduce_func" - type: "func" - } - attr { - name: "finalize_func" - type: "func" - } - attr { - name: "Tkey_func_other_arguments" - type: "list(type)" - has_minimum: true - } - attr { - name: "Tinit_func_other_arguments" - type: "list(type)" - has_minimum: true - } - attr { - name: "Treduce_func_other_arguments" - type: "list(type)" - has_minimum: true - } - attr { - name: "Tfinalize_func_other_arguments" - type: "list(type)" - has_minimum: true - } - attr { - name: "output_types" - type: "list(type)" - has_minimum: true - minimum: 1 - } - attr { - name: "output_shapes" - type: "list(shape)" - has_minimum: true - minimum: 1 - } - is_stateful: true -} -op { - name: "GroupByWindowDataset" - input_arg { - name: "input_dataset" - type: DT_VARIANT - } - input_arg { - name: "key_func_other_arguments" - type_list_attr: "Tkey_func_other_arguments" - } - input_arg { - name: "reduce_func_other_arguments" - type_list_attr: "Treduce_func_other_arguments" - } - input_arg { - name: "window_size_func_other_arguments" - type_list_attr: "Twindow_size_func_other_arguments" - } - output_arg { - name: "handle" - type: DT_VARIANT - } - attr { - name: "key_func" - type: "func" - } - attr { - name: "reduce_func" - type: "func" - } - attr { - name: "window_size_func" - type: "func" - } - attr { - name: "Tkey_func_other_arguments" - type: "list(type)" - has_minimum: true - } - attr { - name: "Treduce_func_other_arguments" - type: "list(type)" - has_minimum: true - } - attr { - name: "Twindow_size_func_other_arguments" - type: "list(type)" - has_minimum: true - } - attr { - name: "output_types" - type: "list(type)" - has_minimum: true - minimum: 1 - } - attr { - name: "output_shapes" - type: "list(shape)" - has_minimum: true - minimum: 1 - } - is_stateful: true -} -op { - name: "GroupByWindowDataset" - input_arg { - name: "input_dataset" - type: DT_VARIANT - } - input_arg { - name: "key_func_other_arguments" - type_list_attr: "Tkey_func_other_arguments" - } - input_arg { - name: "reduce_func_other_arguments" - type_list_attr: "Treduce_func_other_arguments" - } - input_arg { - name: "window_size_func_other_arguments" - type_list_attr: "Twindow_size_func_other_arguments" - } - output_arg { - name: "handle" - type: DT_VARIANT - } - attr { - name: "key_func" - type: "func" - } - attr { - name: "reduce_func" - type: "func" - } - attr { - name: "window_size_func" - type: "func" - } - attr { - name: "Tkey_func_other_arguments" - type: "list(type)" - has_minimum: true - } - attr { - name: "Treduce_func_other_arguments" - type: "list(type)" - has_minimum: true - } - attr { - name: "Twindow_size_func_other_arguments" - type: "list(type)" - has_minimum: true - } - attr { - name: "output_types" - type: "list(type)" - has_minimum: true - minimum: 1 - } - attr { - name: "output_shapes" - type: "list(shape)" - has_minimum: true - minimum: 1 - } -} op { name: "GuaranteeConst" input_arg { @@ -29151,33 +29692,6 @@ op { } } } -op { - name: "LatencyStatsDataset" - input_arg { - name: "input_dataset" - type: DT_VARIANT - } - input_arg { - name: "tag" - type: DT_STRING - } - output_arg { - name: "handle" - type: DT_VARIANT - } - attr { - name: "output_types" - type: "list(type)" - has_minimum: true - minimum: 1 - } - attr { - name: "output_shapes" - type: "list(shape)" - has_minimum: true - minimum: 1 - } -} op { name: "LeakyRelu" input_arg { @@ -30635,102 +31149,6 @@ op { } is_stateful: true } -op { - name: "MapAndBatchDataset" - input_arg { - name: "input_dataset" - type: DT_VARIANT - } - input_arg { - name: "other_arguments" - type_list_attr: "Targuments" - } - input_arg { - name: "batch_size" - type: DT_INT64 - } - input_arg { - name: "num_parallel_batches" - type: DT_INT64 - } - input_arg { - name: "drop_remainder" - type: DT_BOOL - } - output_arg { - name: "handle" - type: DT_VARIANT - } - attr { - name: "f" - type: "func" - } - attr { - name: "Targuments" - type: "list(type)" - has_minimum: true - } - attr { - name: "output_types" - type: "list(type)" - has_minimum: true - minimum: 1 - } - attr { - name: "output_shapes" - type: "list(shape)" - has_minimum: true - minimum: 1 - } -} -op { - name: "MapAndBatchDatasetV2" - input_arg { - name: "input_dataset" - type: DT_VARIANT - } - input_arg { - name: "other_arguments" - type_list_attr: "Targuments" - } - input_arg { - name: "batch_size" - type: DT_INT64 - } - input_arg { - name: "num_parallel_calls" - type: DT_INT64 - } - input_arg { - name: "drop_remainder" - type: DT_BOOL - } - output_arg { - name: "handle" - type: DT_VARIANT - } - attr { - name: "f" - type: "func" - } - attr { - name: "Targuments" - type: "list(type)" - has_minimum: true - } - attr { - name: "output_types" - type: "list(type)" - has_minimum: true - minimum: 1 - } - attr { - name: "output_shapes" - type: "list(shape)" - has_minimum: true - minimum: 1 - } -} op { name: "MapClear" attr { @@ -31447,18 +31865,6 @@ op { type: DT_STRING } } -op { - name: "MatchingFilesDataset" - input_arg { - name: "patterns" - type: DT_STRING - } - output_arg { - name: "handle" - type: DT_VARIANT - } - is_stateful: true -} op { name: "MatrixBandPart" input_arg { @@ -39011,62 +39417,6 @@ op { type: "type" } } -op { - name: "ParallelInterleaveDataset" - input_arg { - name: "input_dataset" - type: DT_VARIANT - } - input_arg { - name: "other_arguments" - type_list_attr: "Targuments" - } - input_arg { - name: "cycle_length" - type: DT_INT64 - } - input_arg { - name: "block_length" - type: DT_INT64 - } - input_arg { - name: "sloppy" - type: DT_BOOL - } - input_arg { - name: "buffer_output_elements" - type: DT_INT64 - } - input_arg { - name: "prefetch_input_elements" - type: DT_INT64 - } - output_arg { - name: "handle" - type: DT_VARIANT - } - attr { - name: "f" - type: "func" - } - attr { - name: "Targuments" - type: "list(type)" - has_minimum: true - } - attr { - name: "output_types" - type: "list(type)" - has_minimum: true - minimum: 1 - } - attr { - name: "output_shapes" - type: "list(shape)" - has_minimum: true - minimum: 1 - } -} op { name: "ParallelInterleaveDatasetV2" input_arg { @@ -39561,153 +39911,6 @@ op { has_minimum: true } } -op { - name: "ParseExampleDataset" - input_arg { - name: "input_dataset" - type: DT_VARIANT - } - input_arg { - name: "num_parallel_calls" - type: DT_INT64 - } - input_arg { - name: "dense_defaults" - type_list_attr: "Tdense" - } - output_arg { - name: "handle" - type: DT_VARIANT - } - attr { - name: "sparse_keys" - type: "list(string)" - has_minimum: true - } - attr { - name: "dense_keys" - type: "list(string)" - has_minimum: true - } - attr { - name: "sparse_types" - type: "list(type)" - has_minimum: true - allowed_values { - list { - type: DT_FLOAT - type: DT_INT64 - type: DT_STRING - } - } - } - attr { - name: "Tdense" - type: "list(type)" - has_minimum: true - allowed_values { - list { - type: DT_FLOAT - type: DT_INT64 - type: DT_STRING - } - } - } - attr { - name: "dense_shapes" - type: "list(shape)" - has_minimum: true - } - attr { - name: "output_types" - type: "list(type)" - has_minimum: true - minimum: 1 - } - attr { - name: "output_shapes" - type: "list(shape)" - has_minimum: true - minimum: 1 - } -} -op { - name: "ParseExampleDataset" - input_arg { - name: "input_dataset" - type: DT_VARIANT - } - input_arg { - name: "num_parallel_calls" - type: DT_INT64 - } - input_arg { - name: "dense_defaults" - type_list_attr: "Tdense" - } - output_arg { - name: "handle" - type: DT_VARIANT - } - attr { - name: "sparse_keys" - type: "list(string)" - has_minimum: true - } - attr { - name: "dense_keys" - type: "list(string)" - has_minimum: true - } - attr { - name: "sparse_types" - type: "list(type)" - has_minimum: true - allowed_values { - list { - type: DT_FLOAT - type: DT_INT64 - type: DT_STRING - } - } - } - attr { - name: "Tdense" - type: "list(type)" - has_minimum: true - allowed_values { - list { - type: DT_FLOAT - type: DT_INT64 - type: DT_STRING - } - } - } - attr { - name: "dense_shapes" - type: "list(shape)" - has_minimum: true - } - attr { - name: "output_types" - type: "list(type)" - has_minimum: true - minimum: 1 - } - attr { - name: "output_shapes" - type: "list(shape)" - has_minimum: true - minimum: 1 - } - attr { - name: "sloppy" - type: "bool" - default_value { - b: false - } - } -} op { name: "ParseSequenceExample" input_arg { @@ -40276,6 +40479,52 @@ op { } } } +op { + name: "PartitionedCall" + input_arg { + name: "args" + type_list_attr: "Tin" + } + output_arg { + name: "output" + type_list_attr: "Tout" + } + attr { + name: "Tin" + type: "list(type)" + has_minimum: true + } + attr { + name: "Tout" + type: "list(type)" + has_minimum: true + } + attr { + name: "f" + type: "func" + } + attr { + name: "config" + type: "string" + default_value { + s: "" + } + } + attr { + name: "config_proto" + type: "string" + default_value { + s: "" + } + } + attr { + name: "executor_type" + type: "string" + default_value { + s: "" + } + } +} op { name: "Placeholder" output_arg { @@ -40591,48 +40840,6 @@ op { minimum: 1 } } -op { - name: "PrependFromQueueAndPaddedBatchDataset" - input_arg { - name: "input_dataset" - type: DT_VARIANT - } - input_arg { - name: "batch_size" - type: DT_INT64 - } - input_arg { - name: "padded_shapes" - type: DT_INT64 - number_attr: "N" - } - input_arg { - name: "padding_values" - type_list_attr: "Toutput_types" - } - output_arg { - name: "handle" - type: DT_VARIANT - } - attr { - name: "Toutput_types" - type: "list(type)" - has_minimum: true - minimum: 1 - } - attr { - name: "output_shapes" - type: "list(shape)" - has_minimum: true - minimum: 1 - } - attr { - name: "N" - type: "int" - has_minimum: true - minimum: 1 - } -} op { name: "PreventGradient" input_arg { @@ -41643,6 +41850,71 @@ op { } } } +op { + name: "QuantizeAndDequantizeV2" + input_arg { + name: "input" + type_attr: "T" + } + input_arg { + name: "input_min" + type_attr: "T" + } + input_arg { + name: "input_max" + type_attr: "T" + } + output_arg { + name: "output" + type_attr: "T" + } + attr { + name: "signed_input" + type: "bool" + default_value { + b: true + } + } + attr { + name: "num_bits" + type: "int" + default_value { + i: 8 + } + } + attr { + name: "range_given" + type: "bool" + default_value { + b: false + } + } + attr { + name: "T" + type: "type" + allowed_values { + list { + type: DT_BFLOAT16 + type: DT_HALF + type: DT_FLOAT + type: DT_DOUBLE + } + } + } + attr { + name: "round_mode" + type: "string" + default_value { + s: "HALF_TO_EVEN" + } + allowed_values { + list { + s: "HALF_TO_EVEN" + s: "HALF_UP" + } + } + } +} op { name: "QuantizeAndDequantizeV3" input_arg { @@ -44844,34 +45116,6 @@ op { } is_stateful: true } -op { - name: "RandomDataset" - input_arg { - name: "seed" - type: DT_INT64 - } - input_arg { - name: "seed2" - type: DT_INT64 - } - output_arg { - name: "handle" - type: DT_VARIANT - } - attr { - name: "output_types" - type: "list(type)" - has_minimum: true - minimum: 1 - } - attr { - name: "output_shapes" - type: "list(shape)" - has_minimum: true - minimum: 1 - } - is_stateful: true -} op { name: "RandomGamma" input_arg { @@ -49256,6 +49500,86 @@ op { } is_stateful: true } +op { + name: "ResourceApplyAdamWithAmsgrad" + input_arg { + name: "var" + type: DT_RESOURCE + } + input_arg { + name: "m" + type: DT_RESOURCE + } + input_arg { + name: "v" + type: DT_RESOURCE + } + input_arg { + name: "vhat" + type: DT_RESOURCE + } + input_arg { + name: "beta1_power" + type_attr: "T" + } + input_arg { + name: "beta2_power" + type_attr: "T" + } + input_arg { + name: "lr" + type_attr: "T" + } + input_arg { + name: "beta1" + type_attr: "T" + } + input_arg { + name: "beta2" + type_attr: "T" + } + input_arg { + name: "epsilon" + type_attr: "T" + } + input_arg { + name: "grad" + type_attr: "T" + } + attr { + name: "T" + type: "type" + allowed_values { + list { + type: DT_FLOAT + type: DT_DOUBLE + type: DT_INT32 + type: DT_UINT8 + type: DT_INT16 + type: DT_INT8 + type: DT_COMPLEX64 + type: DT_INT64 + type: DT_QINT8 + type: DT_QUINT8 + type: DT_QINT32 + type: DT_BFLOAT16 + type: DT_UINT16 + type: DT_COMPLEX128 + type: DT_HALF + type: DT_UINT32 + type: DT_UINT64 + } + } + } + attr { + name: "use_locking" + type: "bool" + default_value { + b: false + } + } + is_stateful: true +} op { name: "ResourceApplyAddSign" input_arg { @@ -50471,6 +50795,69 @@ op { } is_stateful: true } +op { + name: "ResourceApplyKerasMomentum" + input_arg { + name: "var" + type: DT_RESOURCE + } + input_arg { + name: "accum" + type: DT_RESOURCE + } + input_arg { + name: "lr" + type_attr: "T" + } + input_arg { + name: "grad" + type_attr: "T" + } + input_arg { + name: "momentum" + type_attr: "T" + } + attr { + name: "T" + type: "type" + allowed_values { + list { + type: DT_FLOAT + type: DT_DOUBLE + type: DT_INT32 + type: DT_UINT8 + type: DT_INT16 + type: DT_INT8 + type: DT_COMPLEX64 + type: DT_INT64 + type: DT_QINT8 + type: DT_QUINT8 + type: DT_QINT32 + type: DT_BFLOAT16 + type: DT_UINT16 + type: DT_COMPLEX128 + type: DT_HALF + type: DT_UINT32 + type: DT_UINT64 + } + } + } + attr { + name: "use_locking" + type: "bool" + default_value { + b: false + } + } + attr { + name: "use_nesterov" + type: "bool" + default_value { + b: false + } + } + is_stateful: true +} op { name: "ResourceApplyMomentum" input_arg { @@ -54377,6 +54764,83 @@ op { } is_stateful: true } +op { + name: "ResourceSparseApplyKerasMomentum" + input_arg { + name: "var" + type: DT_RESOURCE + } + input_arg { + name: "accum" + type: DT_RESOURCE + } + input_arg { + name: "lr" + type_attr: "T" + } + input_arg { + name: "grad" + type_attr: "T" + } + input_arg { + name: "indices" + type_attr: "Tindices" + } + input_arg { + name: "momentum" + type_attr: "T" + } + attr { + name: "T" + type: "type" + allowed_values { + list { + type: DT_FLOAT + type: DT_DOUBLE + type: DT_INT32 + type: DT_UINT8 + type: DT_INT16 + type: DT_INT8 + type: DT_COMPLEX64 + type: DT_INT64 + type: DT_QINT8 + type: DT_QUINT8 + type: DT_QINT32 + type: DT_BFLOAT16 + type: DT_UINT16 + type: DT_COMPLEX128 + type: DT_HALF + type: DT_UINT32 + type: DT_UINT64 + } + } + } + attr { + name: "Tindices" + type: "type" + allowed_values { + list { + type: DT_INT32 + type: DT_INT64 + } + } + } + attr { + name: "use_locking" + type: "bool" + default_value { + b: false + } + } + attr { + name: "use_nesterov" + type: "bool" + default_value { + b: false + } + } + is_stateful: true +} op { name: "ResourceSparseApplyMomentum" input_arg { @@ -57111,52 +57575,6 @@ op { } } } -op { - name: "ScanDataset" - input_arg { - name: "input_dataset" - type: DT_VARIANT - } - input_arg { - name: "initial_state" - type_list_attr: "Tstate" - } - input_arg { - name: "other_arguments" - type_list_attr: "Targuments" - } - output_arg { - name: "handle" - type: DT_VARIANT - } - attr { - name: "f" - type: "func" - } - attr { - name: "Tstate" - type: "list(type)" - has_minimum: true - minimum: 1 - } - attr { - name: "Targuments" - type: "list(type)" - has_minimum: true - } - attr { - name: "output_types" - type: "list(type)" - has_minimum: true - minimum: 1 - } - attr { - name: "output_shapes" - type: "list(shape)" - has_minimum: true - minimum: 1 - } -} op { name: "ScatterAdd" input_arg { @@ -60889,42 +61307,6 @@ op { } } } -op { - name: "SetStatsAggregatorDataset" - input_arg { - name: "input_dataset" - type: DT_VARIANT - } - input_arg { - name: "stats_aggregator" - type: DT_RESOURCE - } - input_arg { - name: "tag" - type: DT_STRING - } - input_arg { - name: "counter_prefix" - type: DT_STRING - } - output_arg { - name: "handle" - type: DT_VARIANT - } - attr { - name: "output_types" - type: "list(type)" - has_minimum: true - minimum: 1 - } - attr { - name: "output_shapes" - type: "list(shape)" - has_minimum: true - minimum: 1 - } - is_stateful: true -} op { name: "Shape" input_arg { @@ -61771,41 +62153,6 @@ op { } } } -op { - name: "SlideDataset" - input_arg { - name: "input_dataset" - type: DT_VARIANT - } - input_arg { - name: "window_size" - type: DT_INT64 - } - input_arg { - name: "window_shift" - type: DT_INT64 - } - input_arg { - name: "window_stride" - type: DT_INT64 - } - output_arg { - name: "handle" - type: DT_VARIANT - } - attr { - name: "output_types" - type: "list(type)" - has_minimum: true - minimum: 1 - } - attr { - name: "output_shapes" - type: "list(shape)" - has_minimum: true - minimum: 1 - } -} op { name: "Snapshot" input_arg { @@ -70964,38 +71311,6 @@ op { } } } -op { - name: "SqlDataset" - input_arg { - name: "driver_name" - type: DT_STRING - } - input_arg { - name: "data_source_name" - type: DT_STRING - } - input_arg { - name: "query" - type: DT_STRING - } - output_arg { - name: "handle" - type: DT_VARIANT - } - attr { - name: "output_types" - type: "list(type)" - has_minimum: true - minimum: 1 - } - attr { - name: "output_shapes" - type: "list(shape)" - has_minimum: true - minimum: 1 - } - is_stateful: true -} op { name: "Sqrt" input_arg { @@ -71827,6 +72142,53 @@ op { } is_stateful: true } +op { + name: "StatefulPartitionedCall" + input_arg { + name: "args" + type_list_attr: "Tin" + } + output_arg { + name: "output" + type_list_attr: "Tout" + } + attr { + name: "Tin" + type: "list(type)" + has_minimum: true + } + attr { + name: "Tout" + type: "list(type)" + has_minimum: true + } + attr { + name: "f" + type: "func" + } + attr { + name: "config" + type: "string" + default_value { + s: "" + } + } + attr { + name: "config_proto" + type: "string" + default_value { + s: "" + } + } + attr { + name: "executor_type" + type: "string" + default_value { + s: "" + } + } + is_stateful: true +} op { name: "StatelessIf" input_arg { @@ -72509,40 +72871,6 @@ op { } } } -op { - name: "StatsAggregatorHandle" - output_arg { - name: "handle" - type: DT_RESOURCE - } - attr { - name: "container" - type: "string" - default_value { - s: "" - } - } - attr { - name: "shared_name" - type: "string" - default_value { - s: "" - } - } - is_stateful: true -} -op { - name: "StatsAggregatorSummary" - input_arg { - name: "iterator" - type: DT_RESOURCE - } - output_arg { - name: "summary" - type: DT_STRING - } - is_stateful: true -} op { name: "StopGradient" input_arg { @@ -75378,6 +75706,127 @@ op { } is_stateful: true } +op { + name: "TensorForestCreateTreeVariable" + input_arg { + name: "tree_handle" + type: DT_RESOURCE + } + input_arg { + name: "tree_config" + type: DT_STRING + } + is_stateful: true +} +op { + name: "TensorForestTreeDeserialize" + input_arg { + name: "tree_handle" + type: DT_RESOURCE + } + input_arg { + name: "tree_config" + type: DT_STRING + } + is_stateful: true +} +op { + name: "TensorForestTreeIsInitializedOp" + input_arg { + name: "tree_handle" + type: DT_RESOURCE + } + output_arg { + name: "is_initialized" + type: DT_BOOL + } + is_stateful: true +} +op { + name: "TensorForestTreePredict" + input_arg { + name: "tree_handle" + type: DT_RESOURCE + } + input_arg { + name: "dense_features" + type: DT_FLOAT + } + output_arg { + name: "logits" + type: DT_FLOAT + } + attr { + name: "logits_dimension" + type: "int" + } + is_stateful: true +} +op { + name: "TensorForestTreeResourceHandleOp" + output_arg { + name: "resource" + type: DT_RESOURCE + } + attr { + name: "container" + type: "string" + default_value { + s: "" + } + } + attr { + name: "shared_name" + type: "string" + default_value { + s: "" + } + } + is_stateful: true +} +op { + name: "TensorForestTreeSerialize" + input_arg { + name: "tree_handle" + type: DT_RESOURCE + } + output_arg { + name: "tree_config" + type: DT_STRING + } + is_stateful: true +} +op { + name: "TensorForestTreeSize" + input_arg { + name: "tree_handle" + type: DT_RESOURCE + } + output_arg { + name: "tree_size" + type: DT_INT32 + } + is_stateful: true +} +op { + name: "TensorListConcat" + input_arg { + name: "input_handle" + type: DT_VARIANT + } + output_arg { + name: "tensor" + type_attr: "element_dtype" + } + output_arg { + name: "lengths" + type: DT_INT64 + } + attr { + name: "element_dtype" + type: "type" + } +} op { name: "TensorListConcatLists" input_arg { @@ -75638,6 +76087,39 @@ op { type: "type" } } +op { + name: "TensorListSplit" + input_arg { + name: "tensor" + type_attr: "element_dtype" + } + input_arg { + name: "element_shape" + type_attr: "shape_type" + } + input_arg { + name: "lengths" + type: DT_INT64 + } + output_arg { + name: "output_handle" + type: DT_VARIANT + } + attr { + name: "element_dtype" + type: "type" + } + attr { + name: "shape_type" + type: "type" + allowed_values { + list { + type: DT_INT32 + type: DT_INT64 + } + } + } +} op { name: "TensorListStack" input_arg { @@ -75660,6 +76142,105 @@ op { } } } +op { + name: "TensorScatterAdd" + input_arg { + name: "tensor" + type_attr: "T" + } + input_arg { + name: "indices" + type_attr: "Tindices" + } + input_arg { + name: "updates" + type_attr: "T" + } + output_arg { + name: "output" + type_attr: "T" + } + attr { + name: "T" + type: "type" + } + attr { + name: "Tindices" + type: "type" + allowed_values { + list { + type: DT_INT32 + type: DT_INT64 + } + } + } +} +op { + name: "TensorScatterSub" + input_arg { + name: "tensor" + type_attr: "T" + } + input_arg { + name: "indices" + type_attr: "Tindices" + } + input_arg { + name: "updates" + type_attr: "T" + } + output_arg { + name: "output" + type_attr: "T" + } + attr { + name: "T" + type: "type" + } + attr { + name: "Tindices" + type: "type" + allowed_values { + list { + type: DT_INT32 + type: DT_INT64 + } + } + } +} +op { + name: "TensorScatterUpdate" + input_arg { + name: "tensor" + type_attr: "T" + } + input_arg { + name: "indices" + type_attr: "Tindices" + } + input_arg { + name: "updates" + type_attr: "T" + } + output_arg { + name: "output" + type_attr: "T" + } + attr { + name: "T" + type: "type" + } + attr { + name: "Tindices" + type: "type" + allowed_values { + list { + type: DT_INT32 + type: DT_INT64 + } + } + } +} op { name: "TensorSliceDataset" input_arg { @@ -76810,29 +77391,6 @@ op { type: "type" } } -op { - name: "UnbatchDataset" - input_arg { - name: "input_dataset" - type: DT_VARIANT - } - output_arg { - name: "handle" - type: DT_VARIANT - } - attr { - name: "output_types" - type: "list(type)" - has_minimum: true - minimum: 1 - } - attr { - name: "output_shapes" - type: "list(shape)" - has_minimum: true - minimum: 1 - } -} op { name: "UnbatchGrad" input_arg { @@ -76874,6 +77432,104 @@ op { type: "type" } } +op { + name: "UnicodeDecodeWithOffsets" + input_arg { + name: "input" + type: DT_STRING + } + output_arg { + name: "row_splits" + type: DT_INT64 + } + output_arg { + name: "char_values" + type: DT_INT32 + } + output_arg { + name: "char_to_byte_starts" + type: DT_INT64 + } + attr { + name: "input_encoding" + type: "string" + } + attr { + name: "errors" + type: "string" + default_value { + s: "replace" + } + allowed_values { + list { + s: "strict" + s: "replace" + s: "ignore" + } + } + } + attr { + name: "replacement_char" + type: "int" + default_value { + i: 65533 + } + } + attr { + name: "replace_control_characters" + type: "bool" + default_value { + b: false + } + } +} +op { + name: "UnicodeEncode" + input_arg { + name: "input_values" + type: DT_INT32 + } + input_arg { + name: "input_splits" + type: DT_INT64 + } + output_arg { + name: "output" + type: DT_STRING + } + attr { + name: "errors" + type: "string" + default_value { + s: "replace" + } + allowed_values { + list { + s: "ignore" + s: "replace" + s: "strict" + } + } + } + attr { + name: "output_encoding" + type: "string" + allowed_values { + list { + s: "UTF-8" + s: "UTF-16-BE" + s: "UTF-32-BE" + } + } + } + attr { + name: "replacement_char" + type: "int" + default_value { + i: 65533 + } + } +} op { name: "UnicodeScript" input_arg { diff --git a/tensorflow/core/ops/dataset_ops.cc b/tensorflow/core/ops/dataset_ops.cc index 8402f250f9f..b2ea637464a 100644 --- a/tensorflow/core/ops/dataset_ops.cc +++ b/tensorflow/core/ops/dataset_ops.cc @@ -83,13 +83,6 @@ REGISTER_OP("GeneratorDataset") // stateful to inhibit constant folding. .SetShapeFn(shape_inference::ScalarShape); -REGISTER_OP("UnbatchDataset") - .Input("input_dataset: variant") - .Output("handle: variant") - .Attr("output_types: list(type) >= 1") - .Attr("output_shapes: list(shape) >= 1") - .SetShapeFn(shape_inference::ScalarShape); - REGISTER_OP("ZipDataset") .Input("input_datasets: N * variant") .Output("handle: variant") @@ -142,57 +135,6 @@ REGISTER_OP("SkipDataset") return shape_inference::ScalarShape(c); }); -REGISTER_OP("BytesProducedStatsDataset") - .Input("input_dataset: variant") - .Input("tag: string") - .Output("handle: variant") - .Attr("output_types: list(type) >= 1") - .Attr("output_shapes: list(shape) >= 1") - .SetShapeFn([](shape_inference::InferenceContext* c) { - shape_inference::ShapeHandle tag_shape; - TF_RETURN_IF_ERROR(c->WithRank(c->input(1), 0, &tag_shape)); - return shape_inference::ScalarShape(c); - }); - -REGISTER_OP("LatencyStatsDataset") - .Input("input_dataset: variant") - .Input("tag: string") - .Output("handle: variant") - .Attr("output_types: list(type) >= 1") - .Attr("output_shapes: list(shape) >= 1") - .SetShapeFn([](shape_inference::InferenceContext* c) { - shape_inference::ShapeHandle tag_shape; - TF_RETURN_IF_ERROR(c->WithRank(c->input(1), 0, &tag_shape)); - return shape_inference::ScalarShape(c); - }); - -REGISTER_OP("ParseExampleDataset") - .Input("input_dataset: variant") - .Input("num_parallel_calls: int64") - .Input("dense_defaults: Tdense") - .Output("handle: variant") - .Attr("sparse_keys: list(string) >= 0") - .Attr("dense_keys: list(string) >= 0") - .Attr("sparse_types: list({float,int64,string}) >= 0") - .Attr("Tdense: list({float,int64,string}) >= 0") - .Attr("dense_shapes: list(shape) >= 0") - .Attr("output_types: list(type) >= 1") - .Attr("output_shapes: list(shape) >= 1") // Output components will be - // sorted by key (dense_keys and - // sparse_keys combined) here. - .Attr("sloppy: bool = false") - .SetShapeFn(shape_inference::ScalarShape); - -REGISTER_OP("SetStatsAggregatorDataset") - .Input("input_dataset: variant") - .Input("stats_aggregator: resource") - .Input("tag: string") - .Input("counter_prefix: string") - .Output("handle: variant") - .Attr("output_types: list(type) >= 1") - .Attr("output_shapes: list(shape) >= 1") - .SetShapeFn(shape_inference::ScalarShape); - REGISTER_OP("MapDataset") .Input("input_dataset: variant") .Input("other_arguments: Targuments") @@ -217,58 +159,6 @@ REGISTER_OP("ParallelMapDataset") .Attr("sloppy: bool = false") .SetShapeFn(shape_inference::ScalarShape); -REGISTER_OP("MapAndBatchDataset") - .Input("input_dataset: variant") - .Input("other_arguments: Targuments") - .Input("batch_size: int64") - .Input("num_parallel_batches: int64") - .Input("drop_remainder: bool") - .Output("handle: variant") - .Attr("f: func") - .Attr("Targuments: list(type) >= 0") - .Attr("output_types: list(type) >= 1") - .Attr("output_shapes: list(shape) >= 1") - .SetShapeFn([](shape_inference::InferenceContext* c) { - // Use index from the end to retrieve the Input shapes, - // so that to avoid guessing the length of "other_arguments". - // batch_size, num_parallel_batches, and drop_remainder are 0-D scalars. - shape_inference::ShapeHandle unused; - TF_RETURN_IF_ERROR( - c->WithRank(c->input(c->num_inputs() - 3), 0, &unused)); - TF_RETURN_IF_ERROR( - c->WithRank(c->input(c->num_inputs() - 2), 0, &unused)); - TF_RETURN_IF_ERROR( - c->WithRank(c->input(c->num_inputs() - 1), 0, &unused)); - - return shape_inference::ScalarShape(c); - }); - -REGISTER_OP("MapAndBatchDatasetV2") - .Input("input_dataset: variant") - .Input("other_arguments: Targuments") - .Input("batch_size: int64") - .Input("num_parallel_calls: int64") - .Input("drop_remainder: bool") - .Output("handle: variant") - .Attr("f: func") - .Attr("Targuments: list(type) >= 0") - .Attr("output_types: list(type) >= 1") - .Attr("output_shapes: list(shape) >= 1") - .SetShapeFn([](shape_inference::InferenceContext* c) { - // Use index from the end to retrieve the Input shapes, - // so that to avoid guessing the length of "other_arguments". - // batch_size, num_parallel_calls, and drop_remainder are 0-D scalars. - shape_inference::ShapeHandle unused; - TF_RETURN_IF_ERROR( - c->WithRank(c->input(c->num_inputs() - 3), 0, &unused)); - TF_RETURN_IF_ERROR( - c->WithRank(c->input(c->num_inputs() - 2), 0, &unused)); - TF_RETURN_IF_ERROR( - c->WithRank(c->input(c->num_inputs() - 1), 0, &unused)); - - return shape_inference::ScalarShape(c); - }); - REGISTER_OP("PrefetchDataset") .Input("input_dataset: variant") .Input("buffer_size: int64") @@ -282,18 +172,6 @@ REGISTER_OP("PrefetchDataset") return shape_inference::ScalarShape(c); }); -REGISTER_OP("ScanDataset") - .Input("input_dataset: variant") - .Input("initial_state: Tstate") - .Input("other_arguments: Targuments") - .Output("handle: variant") - .Attr("f: func") - .Attr("Tstate: list(type) >= 1") - .Attr("Targuments: list(type) >= 0") - .Attr("output_types: list(type) >= 1") - .Attr("output_shapes: list(shape) >= 1") - .SetShapeFn(shape_inference::ScalarShape); - REGISTER_OP("FlatMapDataset") .Input("input_dataset: variant") .Input("other_arguments: Targuments") @@ -316,21 +194,6 @@ REGISTER_OP("InterleaveDataset") .Attr("output_shapes: list(shape) >= 1") .SetShapeFn(shape_inference::ScalarShape); -REGISTER_OP("ParallelInterleaveDataset") - .Input("input_dataset: variant") - .Input("other_arguments: Targuments") - .Input("cycle_length: int64") - .Input("block_length: int64") - .Input("sloppy: bool") - .Input("buffer_output_elements: int64") - .Input("prefetch_input_elements: int64") - .Output("handle: variant") - .Attr("f: func") - .Attr("Targuments: list(type) >= 0") - .Attr("output_types: list(type) >= 1") - .Attr("output_shapes: list(shape) >= 1") - .SetShapeFn(shape_inference::ScalarShape); - REGISTER_OP("ParallelInterleaveDatasetV2") .Input("input_dataset: variant") .Input("other_arguments: Targuments") @@ -345,43 +208,6 @@ REGISTER_OP("ParallelInterleaveDatasetV2") .Attr("sloppy: bool = false") .SetShapeFn(shape_inference::ScalarShape); -REGISTER_OP("GroupByReducerDataset") - .Input("input_dataset: variant") - .Input("key_func_other_arguments: Tkey_func_other_arguments") - .Input("init_func_other_arguments: Tinit_func_other_arguments") - .Input("reduce_func_other_arguments: Treduce_func_other_arguments") - .Input("finalize_func_other_arguments: Tfinalize_func_other_arguments") - .Output("handle: variant") - .Attr("key_func: func") - .Attr("init_func: func") - .Attr("reduce_func: func") - .Attr("finalize_func: func") - .Attr("Tkey_func_other_arguments: list(type) >= 0") - .Attr("Tinit_func_other_arguments: list(type) >= 0") - .Attr("Treduce_func_other_arguments: list(type) >= 0") - .Attr("Tfinalize_func_other_arguments: list(type) >= 0") - .Attr("output_types: list(type) >= 1") - .Attr("output_shapes: list(shape) >= 1") - .SetIsStateful() - .SetShapeFn(shape_inference::ScalarShape); - -REGISTER_OP("GroupByWindowDataset") - .Input("input_dataset: variant") - .Input("key_func_other_arguments: Tkey_func_other_arguments") - .Input("reduce_func_other_arguments: Treduce_func_other_arguments") - .Input( - "window_size_func_other_arguments: Twindow_size_func_other_arguments") - .Output("handle: variant") - .Attr("key_func: func") - .Attr("reduce_func: func") - .Attr("window_size_func: func") - .Attr("Tkey_func_other_arguments: list(type) >= 0") - .Attr("Treduce_func_other_arguments: list(type) >= 0") - .Attr("Twindow_size_func_other_arguments: list(type) >= 0") - .Attr("output_types: list(type) >= 1") - .Attr("output_shapes: list(shape) >= 1") - .SetShapeFn(shape_inference::ScalarShape); - REGISTER_OP("FilterDataset") .Input("input_dataset: variant") .Input("other_arguments: Targuments") @@ -447,23 +273,6 @@ REGISTER_OP("BatchDatasetV2") return shape_inference::ScalarShape(c); }); -REGISTER_OP("SlideDataset") - .Input("input_dataset: variant") - .Input("window_size: int64") - .Input("window_shift: int64") - .Input("window_stride: int64") - .Output("handle: variant") - .Attr("output_types: list(type) >= 1") - .Attr("output_shapes: list(shape) >= 1") - .SetShapeFn([](shape_inference::InferenceContext* c) { - shape_inference::ShapeHandle unused; - // window_size, window_shift, and window_stride should be scalars. - TF_RETURN_IF_ERROR(c->WithRank(c->input(1), 0, &unused)); - TF_RETURN_IF_ERROR(c->WithRank(c->input(2), 0, &unused)); - TF_RETURN_IF_ERROR(c->WithRank(c->input(3), 0, &unused)); - return shape_inference::ScalarShape(c); - }); - // TODO(mrry): Validate that `padded_shapes` are all vectors, the lengths of // `output_types` and `output_shapes` are `N` the `output_shapes` are (as far as // possible to tell statically) compatible with `padded_shapes`, and that @@ -504,22 +313,6 @@ REGISTER_OP("PaddedBatchDatasetV2") return shape_inference::ScalarShape(c); }); -REGISTER_OP("DenseToSparseBatchDataset") - .Input("input_dataset: variant") - .Input("batch_size: int64") - .Input("row_shape: int64") - .Output("handle: variant") - .Attr("output_types: list(type) >= 1") - .Attr("output_shapes: list(shape) >= 1") - .SetShapeFn([](shape_inference::InferenceContext* c) { - shape_inference::ShapeHandle unused; - // batch_size should be a scalar. - TF_RETURN_IF_ERROR(c->WithRank(c->input(1), 0, &unused)); - // row_shape should be a 1-D vector. - TF_RETURN_IF_ERROR(c->WithRank(c->input(2), 1, &unused)); - return shape_inference::ScalarShape(c); - }); - REGISTER_OP("RangeDataset") .Input("start: int64") .Input("stop: int64") @@ -538,22 +331,6 @@ REGISTER_OP("RangeDataset") return shape_inference::ScalarShape(c); }); -REGISTER_OP("RandomDataset") - .Input("seed: int64") - .Input("seed2: int64") - .Output("handle: variant") - .Attr("output_types: list(type) >= 1") - .Attr("output_shapes: list(shape) >= 1") - .SetIsStateful() // TODO(b/65524810): Source dataset ops must be marked - // stateful to inhibit constant folding. - .SetShapeFn([](shape_inference::InferenceContext* c) { - shape_inference::ShapeHandle unused; - // buffer_size, seed, and seed2 should be scalars. - TF_RETURN_IF_ERROR(c->WithRank(c->input(0), 0, &unused)); - TF_RETURN_IF_ERROR(c->WithRank(c->input(1), 0, &unused)); - return shape_inference::ScalarShape(c); - }); - REGISTER_OP("ShuffleDataset") .Input("input_dataset: variant") .Input("buffer_size: int64") @@ -622,36 +399,6 @@ REGISTER_OP("TextLineDataset") return shape_inference::ScalarShape(c); }); -REGISTER_OP("MatchingFilesDataset") - .Input("patterns: string") - .Output("handle: variant") - .SetIsStateful() // TODO(b/65524810): Source dataset ops must be marked - // stateful to inhibit constant folding. - .SetShapeFn([](shape_inference::InferenceContext* c) { - shape_inference::ShapeHandle unused; - // `patterns` must be a scalar or a vector. - TF_RETURN_IF_ERROR(c->WithRankAtMost(c->input(0), 1, &unused)); - return shape_inference::ScalarShape(c); - }); - -REGISTER_OP("SqlDataset") - .Input("driver_name: string") - .Input("data_source_name: string") - .Input("query: string") - .Output("handle: variant") - .Attr("output_types: list(type) >= 1") - .Attr("output_shapes: list(shape) >= 1") - .SetIsStateful() // TODO(b/65524810): Source dataset ops must be marked - // stateful to inhibit constant folding. - .SetShapeFn([](shape_inference::InferenceContext* c) { - shape_inference::ShapeHandle unused; - // driver_name, data_source_name, and query should be scalars. - TF_RETURN_IF_ERROR(c->WithRank(c->input(0), 0, &unused)); - TF_RETURN_IF_ERROR(c->WithRank(c->input(1), 0, &unused)); - TF_RETURN_IF_ERROR(c->WithRank(c->input(2), 0, &unused)); - return shape_inference::ScalarShape(c); - }); - REGISTER_OP("FixedLengthRecordDataset") .Input("filenames: string") .Input("header_bytes: int64") @@ -838,53 +585,6 @@ REGISTER_OP("DeserializeIterator") .Input("serialized: variant") .SetShapeFn(shape_inference::NoOutputs); -REGISTER_OP("StatsAggregatorHandle") - .Output("handle: resource") - .SetShapeFn(shape_inference::ScalarShape) - .Attr("container: string = ''") - .Attr("shared_name: string = ''"); - -REGISTER_OP("StatsAggregatorSummary") - .Input("iterator: resource") - .Output("summary: string") - .SetShapeFn(shape_inference::ScalarShape); - -REGISTER_OP("PrependFromQueueAndPaddedBatchDataset") - .Input("input_dataset: variant") - .Input("batch_size: int64") - .Input("padded_shapes: N * int64") - .Input("padding_values: Toutput_types") - .Output("handle: variant") - .Attr("Toutput_types: list(type) >= 1") - .Attr("output_shapes: list(shape) >= 1") - .Attr("N: int >= 1") - // TODO(ebrevdo): Validate that `padded_shapes` are all vectors, the lengths - // of `Toutput_types` and `output_shapes` are `N`, that the - // length of `output_types` is `N`, the `output_shapes` are - // (as far as possible to tell statically) compatible with `padded_shapes`, - // and that `padding_values` are all scalars. - .SetShapeFn([](shape_inference::InferenceContext* c) { - shape_inference::ShapeHandle unused; - // batch_size should be a scalar. - TF_RETURN_IF_ERROR(c->WithRank(c->input(1), 0, &unused)); - return shape_inference::ScalarShape(c); - }); - -REGISTER_OP("EnqueueInQueueDataset") - .Input("queue: variant") - .Input("components: Tcomponents") - .Attr("Tcomponents: list(type) >= 1") - .SetIsStateful() // To avoid CSE on multiple calls to Enqueue. - // TODO(ebrevdo): SetShapeFn to test input dtypes and shapes by - // reading from queue handle (is that even possible?). - .SetShapeFn(shape_inference::NoOutputs); - -REGISTER_OP("DatasetToTFRecord") - .Input("input_dataset: variant") - .Input("filename: string") - .Input("compression_type: string") - .SetShapeFn(shape_inference::NoOutputs); - REGISTER_OP("DatasetToGraph") .Input("input_dataset: variant") .Output("graph: string") diff --git a/tensorflow/core/ops/experimental_dataset_ops.cc b/tensorflow/core/ops/experimental_dataset_ops.cc index 9733cf27768..07110341188 100644 --- a/tensorflow/core/ops/experimental_dataset_ops.cc +++ b/tensorflow/core/ops/experimental_dataset_ops.cc @@ -17,14 +17,17 @@ limitations under the License. namespace tensorflow { -REGISTER_OP("ExperimentalDirectedInterleaveDataset") - .Input("selector_input_dataset: variant") - .Input("data_input_datasets: N * variant") +REGISTER_OP("ExperimentalBytesProducedStatsDataset") + .Input("input_dataset: variant") + .Input("tag: string") .Output("handle: variant") .Attr("output_types: list(type) >= 1") .Attr("output_shapes: list(shape) >= 1") - .Attr("N: int >= 1") - .SetShapeFn(shape_inference::ScalarShape); + .SetShapeFn([](shape_inference::InferenceContext* c) { + shape_inference::ShapeHandle tag_shape; + TF_RETURN_IF_ERROR(c->WithRank(c->input(1), 0, &tag_shape)); + return shape_inference::ScalarShape(c); + }); REGISTER_OP("ExperimentalCSVDataset") .Input("filenames: string") @@ -68,6 +71,79 @@ REGISTER_OP("ExperimentalCSVDataset") return shape_inference::ScalarShape(c); }); +REGISTER_OP("ExperimentalDatasetCardinality") + .Input("input_dataset: variant") + .Output("cardinality: int64") + .SetShapeFn(shape_inference::ScalarShape); + +REGISTER_OP("ExperimentalDatasetToTFRecord") + .Input("input_dataset: variant") + .Input("filename: string") + .Input("compression_type: string") + .SetShapeFn(shape_inference::NoOutputs); + +REGISTER_OP("ExperimentalDenseToSparseBatchDataset") + .Input("input_dataset: variant") + .Input("batch_size: int64") + .Input("row_shape: int64") + .Output("handle: variant") + .Attr("output_types: list(type) >= 1") + .Attr("output_shapes: list(shape) >= 1") + .SetShapeFn([](shape_inference::InferenceContext* c) { + shape_inference::ShapeHandle unused; + // batch_size should be a scalar. + TF_RETURN_IF_ERROR(c->WithRank(c->input(1), 0, &unused)); + // row_shape should be a 1-D vector. + TF_RETURN_IF_ERROR(c->WithRank(c->input(2), 1, &unused)); + return shape_inference::ScalarShape(c); + }); + +REGISTER_OP("ExperimentalDirectedInterleaveDataset") + .Input("selector_input_dataset: variant") + .Input("data_input_datasets: N * variant") + .Output("handle: variant") + .Attr("output_types: list(type) >= 1") + .Attr("output_shapes: list(shape) >= 1") + .Attr("N: int >= 1") + .SetShapeFn(shape_inference::ScalarShape); + +REGISTER_OP("ExperimentalGroupByReducerDataset") + .Input("input_dataset: variant") + .Input("key_func_other_arguments: Tkey_func_other_arguments") + .Input("init_func_other_arguments: Tinit_func_other_arguments") + .Input("reduce_func_other_arguments: Treduce_func_other_arguments") + .Input("finalize_func_other_arguments: Tfinalize_func_other_arguments") + .Output("handle: variant") + .Attr("key_func: func") + .Attr("init_func: func") + .Attr("reduce_func: func") + .Attr("finalize_func: func") + .Attr("Tkey_func_other_arguments: list(type) >= 0") + .Attr("Tinit_func_other_arguments: list(type) >= 0") + .Attr("Treduce_func_other_arguments: list(type) >= 0") + .Attr("Tfinalize_func_other_arguments: list(type) >= 0") + .Attr("output_types: list(type) >= 1") + .Attr("output_shapes: list(shape) >= 1") + .SetIsStateful() + .SetShapeFn(shape_inference::ScalarShape); + +REGISTER_OP("ExperimentalGroupByWindowDataset") + .Input("input_dataset: variant") + .Input("key_func_other_arguments: Tkey_func_other_arguments") + .Input("reduce_func_other_arguments: Treduce_func_other_arguments") + .Input( + "window_size_func_other_arguments: Twindow_size_func_other_arguments") + .Output("handle: variant") + .Attr("key_func: func") + .Attr("reduce_func: func") + .Attr("window_size_func: func") + .Attr("Tkey_func_other_arguments: list(type) >= 0") + .Attr("Treduce_func_other_arguments: list(type) >= 0") + .Attr("Twindow_size_func_other_arguments: list(type) >= 0") + .Attr("output_types: list(type) >= 1") + .Attr("output_shapes: list(shape) >= 1") + .SetShapeFn(shape_inference::ScalarShape); + REGISTER_OP("ExperimentalIgnoreErrorsDataset") .Input("input_dataset: variant") .Output("handle: variant") @@ -75,6 +151,44 @@ REGISTER_OP("ExperimentalIgnoreErrorsDataset") .Attr("output_shapes: list(shape) >= 1") .SetShapeFn(shape_inference::ScalarShape); +REGISTER_OP("ExperimentalLatencyStatsDataset") + .Input("input_dataset: variant") + .Input("tag: string") + .Output("handle: variant") + .Attr("output_types: list(type) >= 1") + .Attr("output_shapes: list(shape) >= 1") + .SetShapeFn([](shape_inference::InferenceContext* c) { + shape_inference::ShapeHandle tag_shape; + TF_RETURN_IF_ERROR(c->WithRank(c->input(1), 0, &tag_shape)); + return shape_inference::ScalarShape(c); + }); + +REGISTER_OP("ExperimentalMapAndBatchDataset") + .Input("input_dataset: variant") + .Input("other_arguments: Targuments") + .Input("batch_size: int64") + .Input("num_parallel_calls: int64") + .Input("drop_remainder: bool") + .Output("handle: variant") + .Attr("f: func") + .Attr("Targuments: list(type) >= 0") + .Attr("output_types: list(type) >= 1") + .Attr("output_shapes: list(shape) >= 1") + .SetShapeFn([](shape_inference::InferenceContext* c) { + // Use index from the end to retrieve the Input shapes, + // so that to avoid guessing the length of "other_arguments". + // batch_size, num_parallel_calls, and drop_remainder are 0-D scalars. + shape_inference::ShapeHandle unused; + TF_RETURN_IF_ERROR( + c->WithRank(c->input(c->num_inputs() - 3), 0, &unused)); + TF_RETURN_IF_ERROR( + c->WithRank(c->input(c->num_inputs() - 2), 0, &unused)); + TF_RETURN_IF_ERROR( + c->WithRank(c->input(c->num_inputs() - 1), 0, &unused)); + + return shape_inference::ScalarShape(c); + }); + REGISTER_OP("ExperimentalMapDataset") .Input("input_dataset: variant") .Input("other_arguments: Targuments") @@ -86,6 +200,18 @@ REGISTER_OP("ExperimentalMapDataset") .Attr("use_inter_op_parallelism: bool = true") .SetShapeFn(shape_inference::ScalarShape); +REGISTER_OP("ExperimentalMatchingFilesDataset") + .Input("patterns: string") + .Output("handle: variant") + .SetIsStateful() // TODO(b/65524810): Source dataset ops must be marked + // stateful to inhibit constant folding. + .SetShapeFn([](shape_inference::InferenceContext* c) { + shape_inference::ShapeHandle unused; + // `patterns` must be a scalar or a vector. + TF_RETURN_IF_ERROR(c->WithRankAtMost(c->input(0), 1, &unused)); + return shape_inference::ScalarShape(c); + }); + REGISTER_OP("ExperimentalNonSerializableDataset") .Input("input_dataset: variant") .Output("handle: variant") @@ -93,6 +219,76 @@ REGISTER_OP("ExperimentalNonSerializableDataset") .Attr("output_shapes: list(shape) >= 1") .SetShapeFn(shape_inference::ScalarShape); +REGISTER_OP("ExperimentalParallelInterleaveDataset") + .Input("input_dataset: variant") + .Input("other_arguments: Targuments") + .Input("cycle_length: int64") + .Input("block_length: int64") + .Input("sloppy: bool") + .Input("buffer_output_elements: int64") + .Input("prefetch_input_elements: int64") + .Output("handle: variant") + .Attr("f: func") + .Attr("Targuments: list(type) >= 0") + .Attr("output_types: list(type) >= 1") + .Attr("output_shapes: list(shape) >= 1") + .SetShapeFn(shape_inference::ScalarShape); + +REGISTER_OP("ExperimentalParseExampleDataset") + .Input("input_dataset: variant") + .Input("num_parallel_calls: int64") + .Input("dense_defaults: Tdense") + .Output("handle: variant") + .Attr("sparse_keys: list(string) >= 0") + .Attr("dense_keys: list(string) >= 0") + .Attr("sparse_types: list({float,int64,string}) >= 0") + .Attr("Tdense: list({float,int64,string}) >= 0") + .Attr("dense_shapes: list(shape) >= 0") + .Attr("output_types: list(type) >= 1") + .Attr("output_shapes: list(shape) >= 1") // Output components will be + // sorted by key (dense_keys and + // sparse_keys combined) here. + .Attr("sloppy: bool = false") + .SetShapeFn(shape_inference::ScalarShape); + +REGISTER_OP("ExperimentalRandomDataset") + .Input("seed: int64") + .Input("seed2: int64") + .Output("handle: variant") + .Attr("output_types: list(type) >= 1") + .Attr("output_shapes: list(shape) >= 1") + .SetIsStateful() // TODO(b/65524810): Source dataset ops must be marked + // stateful to inhibit constant folding. + .SetShapeFn([](shape_inference::InferenceContext* c) { + shape_inference::ShapeHandle unused; + // buffer_size, seed, and seed2 should be scalars. + TF_RETURN_IF_ERROR(c->WithRank(c->input(0), 0, &unused)); + TF_RETURN_IF_ERROR(c->WithRank(c->input(1), 0, &unused)); + return shape_inference::ScalarShape(c); + }); + +REGISTER_OP("ExperimentalScanDataset") + .Input("input_dataset: variant") + .Input("initial_state: Tstate") + .Input("other_arguments: Targuments") + .Output("handle: variant") + .Attr("f: func") + .Attr("Tstate: list(type) >= 1") + .Attr("Targuments: list(type) >= 0") + .Attr("output_types: list(type) >= 1") + .Attr("output_shapes: list(shape) >= 1") + .SetShapeFn(shape_inference::ScalarShape); + +REGISTER_OP("ExperimentalSetStatsAggregatorDataset") + .Input("input_dataset: variant") + .Input("stats_aggregator: resource") + .Input("tag: string") + .Input("counter_prefix: string") + .Output("handle: variant") + .Attr("output_types: list(type) >= 1") + .Attr("output_shapes: list(shape) >= 1") + .SetShapeFn(shape_inference::ScalarShape); + REGISTER_OP("ExperimentalSleepDataset") .Input("input_dataset: variant") .Input("sleep_microseconds: int64") @@ -107,6 +303,59 @@ REGISTER_OP("ExperimentalSleepDataset") return shape_inference::ScalarShape(c); }); +REGISTER_OP("ExperimentalSlidingWindowDataset") + .Input("input_dataset: variant") + .Input("window_size: int64") + .Input("window_shift: int64") + .Input("window_stride: int64") + .Output("handle: variant") + .Attr("output_types: list(type) >= 1") + .Attr("output_shapes: list(shape) >= 1") + .SetShapeFn([](shape_inference::InferenceContext* c) { + shape_inference::ShapeHandle unused; + // window_size, window_shift, and window_stride should be scalars. + TF_RETURN_IF_ERROR(c->WithRank(c->input(1), 0, &unused)); + TF_RETURN_IF_ERROR(c->WithRank(c->input(2), 0, &unused)); + TF_RETURN_IF_ERROR(c->WithRank(c->input(3), 0, &unused)); + return shape_inference::ScalarShape(c); + }); + +REGISTER_OP("ExperimentalSqlDataset") + .Input("driver_name: string") + .Input("data_source_name: string") + .Input("query: string") + .Output("handle: variant") + .Attr("output_types: list(type) >= 1") + .Attr("output_shapes: list(shape) >= 1") + .SetIsStateful() // TODO(b/65524810): Source dataset ops must be marked + // stateful to inhibit constant folding. + .SetShapeFn([](shape_inference::InferenceContext* c) { + shape_inference::ShapeHandle unused; + // driver_name, data_source_name, and query should be scalars. + TF_RETURN_IF_ERROR(c->WithRank(c->input(0), 0, &unused)); + TF_RETURN_IF_ERROR(c->WithRank(c->input(1), 0, &unused)); + TF_RETURN_IF_ERROR(c->WithRank(c->input(2), 0, &unused)); + return shape_inference::ScalarShape(c); + }); + +REGISTER_OP("ExperimentalStatsAggregatorHandle") + .Output("handle: resource") + .SetShapeFn(shape_inference::ScalarShape) + .Attr("container: string = ''") + .Attr("shared_name: string = ''"); + +REGISTER_OP("ExperimentalStatsAggregatorSummary") + .Input("iterator: resource") + .Output("summary: string") + .SetShapeFn(shape_inference::ScalarShape); + +REGISTER_OP("ExperimentalUnbatchDataset") + .Input("input_dataset: variant") + .Output("handle: variant") + .Attr("output_types: list(type) >= 1") + .Attr("output_shapes: list(shape) >= 1") + .SetShapeFn(shape_inference::ScalarShape); + REGISTER_OP("ExperimentalUniqueDataset") .Input("input_dataset: variant") .Output("handle: variant") @@ -119,26 +368,21 @@ REGISTER_OP("ExperimentalIteratorGetDevice") .Output("device: string") .SetShapeFn(shape_inference::ScalarShape); -REGISTER_OP("ExperimentalFunctionBufferingResource") - .Input("string_arg: string") - .Input("target_device: string") - .Output("resource: resource") - .Attr("shared_name: string") - .Attr("container: string") - .Attr("f: func") - .Attr("buffer_size: int") - .Attr("output_types: list(type)") - .SetShapeFn(shape_inference::UnknownShape); +REGISTER_OP("ExperimentalMaxIntraOpParallelismDataset") + .Input("input_dataset: variant") + .Input("max_intra_op_parallelism: int64") + .Output("handle: variant") + .Attr("output_types: list(type) >= 1") + .Attr("output_shapes: list(shape) >= 1") + .SetShapeFn(shape_inference::ScalarShape); -REGISTER_OP("ExperimentalFunctionBufferingResourceGetNext") - .Input("function_buffer_resource: resource") - .Attr("output_types: list(type)") - .Output("output: output_types") - .SetShapeFn(shape_inference::UnknownShape); - -REGISTER_OP("ExperimentalFunctionBufferingResourceReset") - .Input("function_buffer_resource: resource") - .SetShapeFn(shape_inference::UnknownShape); +REGISTER_OP("ExperimentalPrivateThreadPoolDataset") + .Input("input_dataset: variant") + .Input("num_threads: int64") + .Output("handle: variant") + .Attr("output_types: list(type) >= 1") + .Attr("output_shapes: list(shape) >= 1") + .SetShapeFn(shape_inference::ScalarShape); REGISTER_OP("ExperimentalThreadPoolDataset") .Input("input_dataset: variant") diff --git a/tensorflow/core/ops/functional_ops.cc b/tensorflow/core/ops/functional_ops.cc index ee14a851eb9..5e0bdd888ce 100644 --- a/tensorflow/core/ops/functional_ops.cc +++ b/tensorflow/core/ops/functional_ops.cc @@ -226,6 +226,7 @@ REGISTER_OP("PartitionedCall") .Attr("Tout: list(type) >= 0") .Attr("f: func") .Attr("config: string = ''") + .Attr("config_proto: string = ''") .Attr("executor_type: string = ''") .SetShapeFn(shape_inference::UnknownShape); @@ -235,7 +236,8 @@ REGISTER_OP("StatefulPartitionedCall") .Attr("Tin: list(type) >= 0") .Attr("Tout: list(type) >= 0") .Attr("f: func") - .Attr("config: string = ''") + .Attr("config: string = ''") // Deprecated in favor of config_proto + .Attr("config_proto: string = ''") .Attr("executor_type: string = ''") .SetIsStateful() .SetShapeFn(shape_inference::UnknownShape); diff --git a/tensorflow/core/ops/list_ops.cc b/tensorflow/core/ops/list_ops.cc index 88d6d14c306..01ebcd15439 100644 --- a/tensorflow/core/ops/list_ops.cc +++ b/tensorflow/core/ops/list_ops.cc @@ -28,13 +28,14 @@ REGISTER_OP("EmptyTensorList") .Attr("shape_type: {int32, int64}") .SetShapeFn([](shape_inference::InferenceContext* c) { c->set_output(0, c->Scalar()); - DataType t; - TF_RETURN_IF_ERROR(c->GetAttr("element_dtype", &t)); - shape_inference::ShapeHandle s; - TF_RETURN_IF_ERROR( - c->MakeShapeFromShapeTensorTreatScalarAsUnknownShape(0, &s)); + DataType element_dtype; + TF_RETURN_IF_ERROR(c->GetAttr("element_dtype", &element_dtype)); + shape_inference::ShapeHandle element_shape; + TF_RETURN_IF_ERROR(c->MakeShapeFromShapeTensorTreatScalarAsUnknownShape( + 0, &element_shape)); c->set_output_handle_shapes_and_types( - 0, std::vector{{s, t}}); + 0, std::vector{ + {element_shape, element_dtype}}); return Status::OK(); }); @@ -45,9 +46,9 @@ REGISTER_OP("TensorListPushBack") .Attr("element_dtype: type") .SetShapeFn([](shape_inference::InferenceContext* c) { c->set_output(0, c->Scalar()); - DataType t; - TF_RETURN_IF_ERROR(c->GetAttr("element_dtype", &t)); - shape_inference::ShapeHandle s = c->UnknownShape(); + DataType element_dtype; + TF_RETURN_IF_ERROR(c->GetAttr("element_dtype", &element_dtype)); + shape_inference::ShapeHandle element_shape = c->UnknownShape(); auto* handle_data = c->input_handle_shapes_and_types(0); if (handle_data != nullptr && handle_data->size() != 1) { @@ -57,18 +58,21 @@ REGISTER_OP("TensorListPushBack") if (handle_data != nullptr) { const shape_inference::ShapeAndType& list_shape_type = (*handle_data)[0]; - if (list_shape_type.dtype != t) { + if (list_shape_type.dtype != element_dtype) { return errors::InvalidArgument( "Trying to push to list with wrong element dtype. List has type ", DataTypeString(list_shape_type.dtype), - " but trying to push element with type ", DataTypeString(t)); + " but trying to push element with type ", + DataTypeString(element_dtype)); } shape_inference::ShapeHandle ignored; - TF_RETURN_IF_ERROR(c->Merge(s, list_shape_type.shape, &ignored)); - s = list_shape_type.shape; + TF_RETURN_IF_ERROR( + c->Merge(element_shape, list_shape_type.shape, &ignored)); + element_shape = list_shape_type.shape; } c->set_output_handle_shapes_and_types( - 0, std::vector{{s, t}}); + 0, std::vector{ + {element_shape, element_dtype}}); return Status::OK(); }); @@ -89,9 +93,9 @@ REGISTER_OP("TensorListPushBackBatch") c->set_output(0, input_handles); - DataType t; - TF_RETURN_IF_ERROR(c->GetAttr("element_dtype", &t)); - shape_inference::ShapeHandle s = c->UnknownShape(); + DataType element_dtype; + TF_RETURN_IF_ERROR(c->GetAttr("element_dtype", &element_dtype)); + shape_inference::ShapeHandle element_shape = c->UnknownShape(); auto* handle_data = c->input_handle_shapes_and_types(0); if (handle_data != nullptr && handle_data->size() != 1) { @@ -101,18 +105,21 @@ REGISTER_OP("TensorListPushBackBatch") if (handle_data != nullptr) { const shape_inference::ShapeAndType& list_shape_type = (*handle_data)[0]; - if (list_shape_type.dtype != t) { + if (list_shape_type.dtype != element_dtype) { return errors::InvalidArgument( "Trying to push to list with wrong element dtype. List has type ", DataTypeString(list_shape_type.dtype), - " but trying to push element with type ", DataTypeString(t)); + " but trying to push element with type ", + DataTypeString(element_dtype)); } shape_inference::ShapeHandle ignored; - TF_RETURN_IF_ERROR(c->Merge(s, list_shape_type.shape, &ignored)); - s = list_shape_type.shape; + TF_RETURN_IF_ERROR( + c->Merge(element_shape, list_shape_type.shape, &ignored)); + element_shape = list_shape_type.shape; } c->set_output_handle_shapes_and_types( - 0, std::vector{{s, t}}); + 0, std::vector{ + {element_shape, element_dtype}}); return Status::OK(); }); @@ -127,9 +134,9 @@ REGISTER_OP("TensorListPopBack") .Output("tensor: element_dtype") .Attr("element_dtype: type") .SetShapeFn([](shape_inference::InferenceContext* c) { - DataType t; - TF_RETURN_IF_ERROR(c->GetAttr("element_dtype", &t)); - shape_inference::ShapeHandle s = c->UnknownShape(); + DataType element_dtype; + TF_RETURN_IF_ERROR(c->GetAttr("element_dtype", &element_dtype)); + shape_inference::ShapeHandle tensor_shape = c->UnknownShape(); auto* handle_data = c->input_handle_shapes_and_types(0); if (handle_data != nullptr && handle_data->size() != 1) { return errors::InvalidArgument( @@ -138,19 +145,21 @@ REGISTER_OP("TensorListPopBack") if (handle_data != nullptr) { const shape_inference::ShapeAndType& list_shape_type = (*handle_data)[0]; - if (list_shape_type.dtype != t) { + if (list_shape_type.dtype != element_dtype) { return errors::InvalidArgument( "Trying to read from list with wrong element dtype. List has " "type ", DataTypeString(list_shape_type.dtype), - " but trying to push element with type ", DataTypeString(t)); + " but trying to push element with type ", + DataTypeString(element_dtype)); } shape_inference::ShapeHandle ignored; - TF_RETURN_IF_ERROR(c->Merge(s, list_shape_type.shape, &ignored)); + TF_RETURN_IF_ERROR( + c->Merge(tensor_shape, list_shape_type.shape, &ignored)); c->set_output_handle_shapes_and_types(0, *handle_data); - s = list_shape_type.shape; + tensor_shape = list_shape_type.shape; } - c->set_output(1, s); + c->set_output(1, tensor_shape); c->set_output(0, c->Scalar()); return Status::OK(); }); @@ -161,9 +170,9 @@ REGISTER_OP("TensorListStack") .Attr("element_dtype: type") .Attr("num_elements: int = -1") .SetShapeFn([](shape_inference::InferenceContext* c) { - DataType t; - TF_RETURN_IF_ERROR(c->GetAttr("element_dtype", &t)); - shape_inference::ShapeHandle s = c->UnknownShape(); + DataType element_dtype; + TF_RETURN_IF_ERROR(c->GetAttr("element_dtype", &element_dtype)); + shape_inference::ShapeHandle element_shape = c->UnknownShape(); auto* handle_data = c->input_handle_shapes_and_types(0); if (handle_data != nullptr && handle_data->size() != 1) { return errors::InvalidArgument( @@ -172,16 +181,17 @@ REGISTER_OP("TensorListStack") if (handle_data != nullptr) { const shape_inference::ShapeAndType& list_shape_type = (*handle_data)[0]; - if (list_shape_type.dtype != t) { + if (list_shape_type.dtype != element_dtype) { return errors::InvalidArgument( "Trying to read from list with wrong element dtype. List has " "type ", DataTypeString(list_shape_type.dtype), " but expectec type ", - DataTypeString(t)); + DataTypeString(element_dtype)); } shape_inference::ShapeHandle ignored; - TF_RETURN_IF_ERROR(c->Merge(s, list_shape_type.shape, &ignored)); - s = list_shape_type.shape; + TF_RETURN_IF_ERROR( + c->Merge(element_shape, list_shape_type.shape, &ignored)); + element_shape = list_shape_type.shape; } int expected_num_elements = -1; TF_RETURN_IF_ERROR(c->GetAttr("num_elements", &expected_num_elements)); @@ -192,11 +202,88 @@ REGISTER_OP("TensorListStack") num_elements = c->MakeShape({expected_num_elements}); } shape_inference::ShapeHandle result; - TF_RETURN_IF_ERROR(c->Concatenate(num_elements, s, &result)); + TF_RETURN_IF_ERROR(c->Concatenate(num_elements, element_shape, &result)); c->set_output(0, result); return Status::OK(); }); +REGISTER_OP("TensorListConcat") + .Input("input_handle: variant") + .Output("tensor: element_dtype") + .Output("lengths: int64") + .Attr("element_dtype: type") + .SetShapeFn([](shape_inference::InferenceContext* c) { + DataType element_dtype; + TF_RETURN_IF_ERROR(c->GetAttr("element_dtype", &element_dtype)); + shape_inference::ShapeHandle element_shape = c->UnknownShape(); + auto* handle_data = c->input_handle_shapes_and_types(0); + if (handle_data != nullptr && handle_data->size() != 1) { + return errors::InvalidArgument( + "Trying to read from list with wrong variant data."); + } + if (handle_data != nullptr) { + const shape_inference::ShapeAndType& list_shape_type = + (*handle_data)[0]; + if (list_shape_type.dtype != element_dtype) { + return errors::InvalidArgument( + "Trying to read from list with wrong element dtype. List has " + "type ", + DataTypeString(list_shape_type.dtype), " but expected type ", + DataTypeString(element_dtype)); + } + shape_inference::ShapeHandle ignored; + TF_RETURN_IF_ERROR( + c->Merge(element_shape, list_shape_type.shape, &ignored)); + element_shape = list_shape_type.shape; + } + if (c->RankKnown(element_shape)) { + shape_inference::ShapeHandle result; + TF_RETURN_IF_ERROR(c->Subshape(element_shape, 1, &result)); + TF_RETURN_IF_ERROR( + c->Concatenate(c->MakeShape({c->UnknownDim()}), result, &result)); + c->set_output(0, result); + } else { + c->set_output(0, c->UnknownShape()); + } + c->set_output(1, c->MakeShape({c->UnknownDim()})); + return Status::OK(); + }); + +REGISTER_OP("TensorListSplit") + .Input("tensor: element_dtype") + .Input("element_shape: shape_type") + .Input("lengths: int64") + .Output("output_handle: variant") + .Attr("element_dtype: type") + .Attr("shape_type: {int32, int64}") + .SetShapeFn([](shape_inference::InferenceContext* c) { + c->set_output(0, c->Scalar()); + DataType element_dtype; + TF_RETURN_IF_ERROR(c->GetAttr("element_dtype", &element_dtype)); + shape_inference::ShapeHandle tensor_shape = c->input(0); + shape_inference::ShapeHandle ignored; + // Check that tensor is at least a vector. + TF_RETURN_IF_ERROR(c->WithRankAtLeast(tensor_shape, 1, &ignored)); + // Check that lengths is a vector. + TF_RETURN_IF_ERROR(c->WithRank(c->input(2), 1, &ignored)); + shape_inference::ShapeHandle element_shape_from_tensor_shape; + TF_RETURN_IF_ERROR( + c->Subshape(tensor_shape, 1, &element_shape_from_tensor_shape)); + TF_RETURN_IF_ERROR(c->Concatenate(c->MakeShape({c->UnknownDim()}), + element_shape_from_tensor_shape, + &element_shape_from_tensor_shape)); + shape_inference::ShapeHandle element_shape; + TF_RETURN_IF_ERROR(c->MakeShapeFromShapeTensorTreatScalarAsUnknownShape( + 1, &element_shape)); + TF_RETURN_IF_ERROR(c->Merge(element_shape_from_tensor_shape, + element_shape, + &element_shape_from_tensor_shape)); + c->set_output_handle_shapes_and_types( + 0, std::vector{ + {element_shape, element_dtype}}); + return Status::OK(); + }); + REGISTER_OP("TensorListFromTensor") .Input("tensor: element_dtype") .Input("element_shape: shape_type") @@ -205,17 +292,20 @@ REGISTER_OP("TensorListFromTensor") .Attr("shape_type: {int32, int64}") .SetShapeFn([](shape_inference::InferenceContext* c) { c->set_output(0, c->Scalar()); - DataType t; - TF_RETURN_IF_ERROR(c->GetAttr("element_dtype", &t)); - shape_inference::ShapeHandle s = c->input(0); - shape_inference::ShapeHandle o; - TF_RETURN_IF_ERROR(c->Subshape(s, 1, &o)); + DataType element_dtype; + TF_RETURN_IF_ERROR(c->GetAttr("element_dtype", &element_dtype)); + shape_inference::ShapeHandle tensor_shape = c->input(0); + shape_inference::ShapeHandle tensor_shape_except_first_dim; + TF_RETURN_IF_ERROR( + c->Subshape(tensor_shape, 1, &tensor_shape_except_first_dim)); shape_inference::ShapeHandle element_shape; TF_RETURN_IF_ERROR(c->MakeShapeFromShapeTensorTreatScalarAsUnknownShape( 1, &element_shape)); - TF_RETURN_IF_ERROR(c->Merge(o, element_shape, &o)); + TF_RETURN_IF_ERROR(c->Merge(tensor_shape_except_first_dim, element_shape, + &tensor_shape_except_first_dim)); c->set_output_handle_shapes_and_types( - 0, std::vector{{element_shape, t}}); + 0, std::vector{ + {element_shape, element_dtype}}); return Status::OK(); }); @@ -241,13 +331,14 @@ REGISTER_OP("TensorListReserve") .Attr("shape_type: {int32, int64}") .SetShapeFn([](shape_inference::InferenceContext* c) { c->set_output(0, c->Scalar()); - shape_inference::ShapeHandle s; - TF_RETURN_IF_ERROR( - c->MakeShapeFromShapeTensorTreatScalarAsUnknownShape(0, &s)); - DataType t; - TF_RETURN_IF_ERROR(c->GetAttr("element_dtype", &t)); + shape_inference::ShapeHandle element_shape; + TF_RETURN_IF_ERROR(c->MakeShapeFromShapeTensorTreatScalarAsUnknownShape( + 0, &element_shape)); + DataType element_dtype; + TF_RETURN_IF_ERROR(c->GetAttr("element_dtype", &element_dtype)); c->set_output_handle_shapes_and_types( - 0, std::vector{{s, t}}); + 0, std::vector{ + {element_shape, element_dtype}}); return Status::OK(); }); @@ -257,17 +348,17 @@ REGISTER_OP("TensorListGetItem") .Output("item: element_dtype") .Attr("element_dtype: type") .SetShapeFn([](shape_inference::InferenceContext* c) { - DataType t; - TF_RETURN_IF_ERROR(c->GetAttr("element_dtype", &t)); + DataType element_dtype; + TF_RETURN_IF_ERROR(c->GetAttr("element_dtype", &element_dtype)); auto* handle_data = c->input_handle_shapes_and_types(0); shape_inference::ShapeHandle element_shape = c->UnknownShape(); if (handle_data != nullptr) { const shape_inference::ShapeAndType& list_shape_type = (*handle_data)[0]; element_shape = list_shape_type.shape; - if (list_shape_type.dtype != t) { + if (list_shape_type.dtype != element_dtype) { return errors::InvalidArgument("Expected list with element dtype ", - DataTypeString(t), + DataTypeString(element_dtype), " but got list with element dtype ", DataTypeString(list_shape_type.dtype)); } @@ -283,17 +374,19 @@ REGISTER_OP("TensorListSetItem") .Output("output_handle: variant") .Attr("element_dtype: type") .SetShapeFn([](shape_inference::InferenceContext* c) { - DataType t; - TF_RETURN_IF_ERROR(c->GetAttr("element_dtype", &t)); + DataType element_dtype; + TF_RETURN_IF_ERROR(c->GetAttr("element_dtype", &element_dtype)); auto* handle_data = c->input_handle_shapes_and_types(0); c->set_output(0, c->Scalar()); if (handle_data == nullptr) { - c->set_output_handle_shapes_and_types(0, {{c->UnknownShape(), t}}); + c->set_output_handle_shapes_and_types( + 0, {{c->UnknownShape(), element_dtype}}); return Status::OK(); } const shape_inference::ShapeAndType& list_shape_type = (*handle_data)[0]; - shape_inference::ShapeHandle s = c->input(2); - TF_RETURN_IF_ERROR(c->Merge(s, list_shape_type.shape, &s)); + shape_inference::ShapeHandle item_shape = c->input(2); + TF_RETURN_IF_ERROR( + c->Merge(item_shape, list_shape_type.shape, &item_shape)); c->set_output_handle_shapes_and_types(0, *handle_data); return Status::OK(); }); @@ -304,17 +397,17 @@ REGISTER_OP("TensorListGather") .Output("values: element_dtype") .Attr("element_dtype: type") .SetShapeFn([](shape_inference::InferenceContext* c) { - DataType t; - TF_RETURN_IF_ERROR(c->GetAttr("element_dtype", &t)); + DataType element_dtype; + TF_RETURN_IF_ERROR(c->GetAttr("element_dtype", &element_dtype)); auto* handle_data = c->input_handle_shapes_and_types(0); shape_inference::ShapeHandle element_shape = c->UnknownShape(); if (handle_data != nullptr) { const shape_inference::ShapeAndType& list_shape_type = (*handle_data)[0]; element_shape = list_shape_type.shape; - if (list_shape_type.dtype != t) { + if (list_shape_type.dtype != element_dtype) { return errors::InvalidArgument("Expected list with element dtype ", - DataTypeString(t), + DataTypeString(element_dtype), " but got list with element dtype ", DataTypeString(list_shape_type.dtype)); } @@ -333,12 +426,13 @@ REGISTER_OP("TensorListScatter") .Attr("element_dtype: type") .Attr("shape_type: {int32, int64}") .SetShapeFn([](shape_inference::InferenceContext* c) { - DataType t; - TF_RETURN_IF_ERROR(c->GetAttr("element_dtype", &t)); - shape_inference::ShapeHandle s; - TF_RETURN_IF_ERROR( - c->MakeShapeFromShapeTensorTreatScalarAsUnknownShape(2, &s)); - c->set_output_handle_shapes_and_types(0, {{s, t}}); + DataType element_dtype; + TF_RETURN_IF_ERROR(c->GetAttr("element_dtype", &element_dtype)); + shape_inference::ShapeHandle element_shape; + TF_RETURN_IF_ERROR(c->MakeShapeFromShapeTensorTreatScalarAsUnknownShape( + 2, &element_shape)); + c->set_output_handle_shapes_and_types(0, + {{element_shape, element_dtype}}); c->set_output(0, c->Scalar()); return Status::OK(); }); @@ -354,28 +448,29 @@ REGISTER_OP("TensorListConcatLists") TF_RETURN_IF_ERROR(c->Merge(input_a, input_b, &input_a)); c->set_output(0, input_a); - DataType t; - TF_RETURN_IF_ERROR(c->GetAttr("element_dtype", &t)); + DataType element_dtype; + TF_RETURN_IF_ERROR(c->GetAttr("element_dtype", &element_dtype)); auto* handle_data_a = c->input_handle_shapes_and_types(0); auto* handle_data_b = c->input_handle_shapes_and_types(1); if (handle_data_a == nullptr && handle_data_b == nullptr) { - c->set_output_handle_shapes_and_types(0, {{c->UnknownShape(), t}}); + c->set_output_handle_shapes_and_types( + 0, {{c->UnknownShape(), element_dtype}}); return Status::OK(); } shape_inference::ShapeAndType list_shape_type_a = (handle_data_a) ? handle_data_a->at(0) : handle_data_b->at(0); const shape_inference::ShapeAndType& list_shape_type_b = (handle_data_b) ? handle_data_b->at(0) : handle_data_a->at(0); - if (list_shape_type_a.dtype != t) { + if (list_shape_type_a.dtype != element_dtype) { return errors::InvalidArgument("input_a.type != element_dtype: ", DataTypeString(list_shape_type_a.dtype), - " vs. ", DataTypeString(t)); + " vs. ", DataTypeString(element_dtype)); } - if (list_shape_type_b.dtype != t) { + if (list_shape_type_b.dtype != element_dtype) { return errors::InvalidArgument("input_b.type != element_dtype: ", DataTypeString(list_shape_type_b.dtype), - " vs. ", DataTypeString(t)); + " vs. ", DataTypeString(element_dtype)); } TF_RETURN_IF_ERROR(c->Merge(list_shape_type_a.shape, list_shape_type_b.shape, diff --git a/tensorflow/core/ops/nn_ops.cc b/tensorflow/core/ops/nn_ops.cc index 4dfd95b0191..7ff9c469438 100644 --- a/tensorflow/core/ops/nn_ops.cc +++ b/tensorflow/core/ops/nn_ops.cc @@ -327,6 +327,9 @@ REGISTER_OP("_FusedConv2D") .Attr(GetConvnetDataFormatAttrString()) .Attr("dilations: list(int) = [1, 1, 1, 1]") .Attr("fused_ops: list(string) = []") + // Attributes for the FusedBatchNorm ------------------------------------ // + .Attr("epsilon: float = 0.0001") + // ---------------------------------------------------------------------- // .SetShapeFn(shape_inference::Conv2DShape) .Doc(R"doc( *NOTE*: Do not invoke this operator directly in Python. Grappler is @@ -2190,11 +2193,7 @@ REGISTER_OP("_MklLRN") .Input("input: T") .Input("mkl_input: uint8") .Output("output: T") -#ifdef INTEL_MKL_ML_ONLY - .Output("workspace: T") -#else .Output("workspace: uint8") -#endif .Output("mkl_output: uint8") .Output("mkl_workspace: uint8") .Attr("depth_radius: int = 5") @@ -2218,11 +2217,7 @@ REGISTER_OP("_MklLRNGrad") .Input("input_grads: T") .Input("input_image: T") .Input("output_image: T") -#ifdef INTEL_MKL_ML_ONLY - .Input("workspace: T") -#else .Input("workspace: uint8") -#endif .Input("mkl_input_grads: uint8") .Input("mkl_input_image: uint8") .Input("mkl_output_image: uint8") diff --git a/tensorflow/core/ops/ops.pbtxt b/tensorflow/core/ops/ops.pbtxt index 430212ee1d1..a9c712e138c 100644 --- a/tensorflow/core/ops/ops.pbtxt +++ b/tensorflow/core/ops/ops.pbtxt @@ -4944,33 +4944,6 @@ op { type: "list(float)" } } -op { - name: "BytesProducedStatsDataset" - input_arg { - name: "input_dataset" - type: DT_VARIANT - } - input_arg { - name: "tag" - type: DT_STRING - } - output_arg { - name: "handle" - type: DT_VARIANT - } - attr { - name: "output_types" - type: "list(type)" - has_minimum: true - minimum: 1 - } - attr { - name: "output_shapes" - type: "list(shape)" - has_minimum: true - minimum: 1 - } -} op { name: "CTCBeamSearchDecoder" input_arg { @@ -7915,21 +7888,6 @@ op { minimum: 1 } } -op { - name: "DatasetToTFRecord" - input_arg { - name: "input_dataset" - type: DT_VARIANT - } - input_arg { - name: "filename" - type: DT_STRING - } - input_arg { - name: "compression_type" - type: DT_STRING - } -} op { name: "DebugGradientIdentity" input_arg { @@ -8599,37 +8557,6 @@ op { } } } -op { - name: "DenseToSparseBatchDataset" - input_arg { - name: "input_dataset" - type: DT_VARIANT - } - input_arg { - name: "batch_size" - type: DT_INT64 - } - input_arg { - name: "row_shape" - type: DT_INT64 - } - output_arg { - name: "handle" - type: DT_VARIANT - } - attr { - name: "output_types" - type: "list(type)" - has_minimum: true - minimum: 1 - } - attr { - name: "output_shapes" - type: "list(shape)" - has_minimum: true - minimum: 1 - } -} op { name: "DenseToSparseSetOperation" input_arg { @@ -9834,24 +9761,6 @@ op { type: DT_STRING } } -op { - name: "EnqueueInQueueDataset" - input_arg { - name: "queue" - type: DT_VARIANT - } - input_arg { - name: "components" - type_list_attr: "Tcomponents" - } - attr { - name: "Tcomponents" - type: "list(type)" - has_minimum: true - minimum: 1 - } - is_stateful: true -} op { name: "EnsureShape" input_arg { @@ -10089,6 +9998,33 @@ op { minimum: 1 } } +op { + name: "ExperimentalBytesProducedStatsDataset" + input_arg { + name: "input_dataset" + type: DT_VARIANT + } + input_arg { + name: "tag" + type: DT_STRING + } + output_arg { + name: "handle" + type: DT_VARIANT + } + attr { + name: "output_types" + type: "list(type)" + has_minimum: true + minimum: 1 + } + attr { + name: "output_shapes" + type: "list(shape)" + has_minimum: true + minimum: 1 + } +} op { name: "ExperimentalCSVDataset" input_arg { @@ -10154,6 +10090,52 @@ op { } is_stateful: true } +op { + name: "ExperimentalDatasetToTFRecord" + input_arg { + name: "input_dataset" + type: DT_VARIANT + } + input_arg { + name: "filename" + type: DT_STRING + } + input_arg { + name: "compression_type" + type: DT_STRING + } +} +op { + name: "ExperimentalDenseToSparseBatchDataset" + input_arg { + name: "input_dataset" + type: DT_VARIANT + } + input_arg { + name: "batch_size" + type: DT_INT64 + } + input_arg { + name: "row_shape" + type: DT_INT64 + } + output_arg { + name: "handle" + type: DT_VARIANT + } + attr { + name: "output_types" + type: "list(type)" + has_minimum: true + minimum: 1 + } + attr { + name: "output_shapes" + type: "list(shape)" + has_minimum: true + minimum: 1 + } +} op { name: "ExperimentalDirectedInterleaveDataset" input_arg { @@ -10189,50 +10171,66 @@ op { } } op { - name: "ExperimentalFunctionBufferingResource" + name: "ExperimentalGroupByReducerDataset" input_arg { - name: "string_arg" - type: DT_STRING + name: "input_dataset" + type: DT_VARIANT } input_arg { - name: "target_device" - type: DT_STRING + name: "key_func_other_arguments" + type_list_attr: "Tkey_func_other_arguments" + } + input_arg { + name: "init_func_other_arguments" + type_list_attr: "Tinit_func_other_arguments" + } + input_arg { + name: "reduce_func_other_arguments" + type_list_attr: "Treduce_func_other_arguments" + } + input_arg { + name: "finalize_func_other_arguments" + type_list_attr: "Tfinalize_func_other_arguments" } output_arg { - name: "resource" - type: DT_RESOURCE + name: "handle" + type: DT_VARIANT } attr { - name: "shared_name" - type: "string" - } - attr { - name: "container" - type: "string" - } - attr { - name: "f" + name: "key_func" type: "func" } attr { - name: "buffer_size" - type: "int" + name: "init_func" + type: "func" } attr { - name: "output_types" + name: "reduce_func" + type: "func" + } + attr { + name: "finalize_func" + type: "func" + } + attr { + name: "Tkey_func_other_arguments" type: "list(type)" + has_minimum: true } - is_stateful: true -} -op { - name: "ExperimentalFunctionBufferingResourceGetNext" - input_arg { - name: "function_buffer_resource" - type: DT_RESOURCE + attr { + name: "Tinit_func_other_arguments" + type: "list(type)" + has_minimum: true } - output_arg { - name: "output" - type_list_attr: "output_types" + attr { + name: "Treduce_func_other_arguments" + type: "list(type)" + has_minimum: true + } + attr { + name: "Tfinalize_func_other_arguments" + type: "list(type)" + has_minimum: true } attr { name: "output_types" @@ -10240,15 +10238,75 @@ op { has_minimum: true minimum: 1 } + attr { + name: "output_shapes" + type: "list(shape)" + has_minimum: true + minimum: 1 + } is_stateful: true } op { - name: "ExperimentalFunctionBufferingResourceReset" + name: "ExperimentalGroupByWindowDataset" input_arg { - name: "function_buffer_resource" - type: DT_RESOURCE + name: "input_dataset" + type: DT_VARIANT + } + input_arg { + name: "key_func_other_arguments" + type_list_attr: "Tkey_func_other_arguments" + } + input_arg { + name: "reduce_func_other_arguments" + type_list_attr: "Treduce_func_other_arguments" + } + input_arg { + name: "window_size_func_other_arguments" + type_list_attr: "Twindow_size_func_other_arguments" + } + output_arg { + name: "handle" + type: DT_VARIANT + } + attr { + name: "key_func" + type: "func" + } + attr { + name: "reduce_func" + type: "func" + } + attr { + name: "window_size_func" + type: "func" + } + attr { + name: "Tkey_func_other_arguments" + type: "list(type)" + has_minimum: true + } + attr { + name: "Treduce_func_other_arguments" + type: "list(type)" + has_minimum: true + } + attr { + name: "Twindow_size_func_other_arguments" + type: "list(type)" + has_minimum: true + } + attr { + name: "output_types" + type: "list(type)" + has_minimum: true + minimum: 1 + } + attr { + name: "output_shapes" + type: "list(shape)" + has_minimum: true + minimum: 1 } - is_stateful: true } op { name: "ExperimentalIdentityIndexedDataset" @@ -10361,6 +10419,136 @@ op { } is_stateful: true } +op { + name: "ExperimentalLatencyStatsDataset" + input_arg { + name: "input_dataset" + type: DT_VARIANT + } + input_arg { + name: "tag" + type: DT_STRING + } + output_arg { + name: "handle" + type: DT_VARIANT + } + attr { + name: "output_types" + type: "list(type)" + has_minimum: true + minimum: 1 + } + attr { + name: "output_shapes" + type: "list(shape)" + has_minimum: true + minimum: 1 + } +} +op { + name: "ExperimentalMapAndBatchDataset" + input_arg { + name: "input_dataset" + type: DT_VARIANT + } + input_arg { + name: "other_arguments" + type_list_attr: "Targuments" + } + input_arg { + name: "batch_size" + type: DT_INT64 + } + input_arg { + name: "num_parallel_calls" + type: DT_INT64 + } + input_arg { + name: "drop_remainder" + type: DT_BOOL + } + output_arg { + name: "handle" + type: DT_VARIANT + } + attr { + name: "f" + type: "func" + } + attr { + name: "Targuments" + type: "list(type)" + has_minimum: true + } + attr { + name: "output_types" + type: "list(type)" + has_minimum: true + minimum: 1 + } + attr { + name: "output_shapes" + type: "list(shape)" + has_minimum: true + minimum: 1 + } +} +op { + name: "ExperimentalMapDataset" + input_arg { + name: "input_dataset" + type: DT_VARIANT + } + input_arg { + name: "other_arguments" + type_list_attr: "Targuments" + } + output_arg { + name: "handle" + type: DT_VARIANT + } + attr { + name: "f" + type: "func" + } + attr { + name: "Targuments" + type: "list(type)" + has_minimum: true + } + attr { + name: "output_types" + type: "list(type)" + has_minimum: true + minimum: 1 + } + attr { + name: "output_shapes" + type: "list(shape)" + has_minimum: true + minimum: 1 + } + attr { + name: "use_inter_op_parallelism" + type: "bool" + default_value { + b: true + } + } +} +op { + name: "ExperimentalMatchingFilesDataset" + input_arg { + name: "patterns" + type: DT_STRING + } + output_arg { + name: "handle" + type: DT_VARIANT + } + is_stateful: true +} op { name: "ExperimentalMaterializedIndexDatasetHandle" output_arg { @@ -10389,6 +10577,33 @@ op { } is_stateful: true } +op { + name: "ExperimentalMaxIntraOpParallelismDataset" + input_arg { + name: "input_dataset" + type: DT_VARIANT + } + input_arg { + name: "max_intra_op_parallelism" + type: DT_INT64 + } + output_arg { + name: "handle" + type: DT_VARIANT + } + attr { + name: "output_types" + type: "list(type)" + has_minimum: true + minimum: 1 + } + attr { + name: "output_shapes" + type: "list(shape)" + has_minimum: true + minimum: 1 + } +} op { name: "ExperimentalNonSerializableDataset" input_arg { @@ -10460,6 +10675,276 @@ op { minimum: 1 } } +op { + name: "ExperimentalParallelInterleaveDataset" + input_arg { + name: "input_dataset" + type: DT_VARIANT + } + input_arg { + name: "other_arguments" + type_list_attr: "Targuments" + } + input_arg { + name: "cycle_length" + type: DT_INT64 + } + input_arg { + name: "block_length" + type: DT_INT64 + } + input_arg { + name: "sloppy" + type: DT_BOOL + } + input_arg { + name: "buffer_output_elements" + type: DT_INT64 + } + input_arg { + name: "prefetch_input_elements" + type: DT_INT64 + } + output_arg { + name: "handle" + type: DT_VARIANT + } + attr { + name: "f" + type: "func" + } + attr { + name: "Targuments" + type: "list(type)" + has_minimum: true + } + attr { + name: "output_types" + type: "list(type)" + has_minimum: true + minimum: 1 + } + attr { + name: "output_shapes" + type: "list(shape)" + has_minimum: true + minimum: 1 + } +} +op { + name: "ExperimentalParseExampleDataset" + input_arg { + name: "input_dataset" + type: DT_VARIANT + } + input_arg { + name: "num_parallel_calls" + type: DT_INT64 + } + input_arg { + name: "dense_defaults" + type_list_attr: "Tdense" + } + output_arg { + name: "handle" + type: DT_VARIANT + } + attr { + name: "sparse_keys" + type: "list(string)" + has_minimum: true + } + attr { + name: "dense_keys" + type: "list(string)" + has_minimum: true + } + attr { + name: "sparse_types" + type: "list(type)" + has_minimum: true + allowed_values { + list { + type: DT_FLOAT + type: DT_INT64 + type: DT_STRING + } + } + } + attr { + name: "Tdense" + type: "list(type)" + has_minimum: true + allowed_values { + list { + type: DT_FLOAT + type: DT_INT64 + type: DT_STRING + } + } + } + attr { + name: "dense_shapes" + type: "list(shape)" + has_minimum: true + } + attr { + name: "output_types" + type: "list(type)" + has_minimum: true + minimum: 1 + } + attr { + name: "output_shapes" + type: "list(shape)" + has_minimum: true + minimum: 1 + } + attr { + name: "sloppy" + type: "bool" + default_value { + b: false + } + } +} +op { + name: "ExperimentalPrivateThreadPoolDataset" + input_arg { + name: "input_dataset" + type: DT_VARIANT + } + input_arg { + name: "num_threads" + type: DT_INT64 + } + output_arg { + name: "handle" + type: DT_VARIANT + } + attr { + name: "output_types" + type: "list(type)" + has_minimum: true + minimum: 1 + } + attr { + name: "output_shapes" + type: "list(shape)" + has_minimum: true + minimum: 1 + } +} +op { + name: "ExperimentalRandomDataset" + input_arg { + name: "seed" + type: DT_INT64 + } + input_arg { + name: "seed2" + type: DT_INT64 + } + output_arg { + name: "handle" + type: DT_VARIANT + } + attr { + name: "output_types" + type: "list(type)" + has_minimum: true + minimum: 1 + } + attr { + name: "output_shapes" + type: "list(shape)" + has_minimum: true + minimum: 1 + } + is_stateful: true +} +op { + name: "ExperimentalScanDataset" + input_arg { + name: "input_dataset" + type: DT_VARIANT + } + input_arg { + name: "initial_state" + type_list_attr: "Tstate" + } + input_arg { + name: "other_arguments" + type_list_attr: "Targuments" + } + output_arg { + name: "handle" + type: DT_VARIANT + } + attr { + name: "f" + type: "func" + } + attr { + name: "Tstate" + type: "list(type)" + has_minimum: true + minimum: 1 + } + attr { + name: "Targuments" + type: "list(type)" + has_minimum: true + } + attr { + name: "output_types" + type: "list(type)" + has_minimum: true + minimum: 1 + } + attr { + name: "output_shapes" + type: "list(shape)" + has_minimum: true + minimum: 1 + } +} +op { + name: "ExperimentalSetStatsAggregatorDataset" + input_arg { + name: "input_dataset" + type: DT_VARIANT + } + input_arg { + name: "stats_aggregator" + type: DT_RESOURCE + } + input_arg { + name: "tag" + type: DT_STRING + } + input_arg { + name: "counter_prefix" + type: DT_STRING + } + output_arg { + name: "handle" + type: DT_VARIANT + } + attr { + name: "output_types" + type: "list(type)" + has_minimum: true + minimum: 1 + } + attr { + name: "output_shapes" + type: "list(shape)" + has_minimum: true + minimum: 1 + } + is_stateful: true +} op { name: "ExperimentalSleepDataset" input_arg { @@ -10487,6 +10972,107 @@ op { minimum: 1 } } +op { + name: "ExperimentalSlidingWindowDataset" + input_arg { + name: "input_dataset" + type: DT_VARIANT + } + input_arg { + name: "window_size" + type: DT_INT64 + } + input_arg { + name: "window_shift" + type: DT_INT64 + } + input_arg { + name: "window_stride" + type: DT_INT64 + } + output_arg { + name: "handle" + type: DT_VARIANT + } + attr { + name: "output_types" + type: "list(type)" + has_minimum: true + minimum: 1 + } + attr { + name: "output_shapes" + type: "list(shape)" + has_minimum: true + minimum: 1 + } +} +op { + name: "ExperimentalSqlDataset" + input_arg { + name: "driver_name" + type: DT_STRING + } + input_arg { + name: "data_source_name" + type: DT_STRING + } + input_arg { + name: "query" + type: DT_STRING + } + output_arg { + name: "handle" + type: DT_VARIANT + } + attr { + name: "output_types" + type: "list(type)" + has_minimum: true + minimum: 1 + } + attr { + name: "output_shapes" + type: "list(shape)" + has_minimum: true + minimum: 1 + } + is_stateful: true +} +op { + name: "ExperimentalStatsAggregatorHandle" + output_arg { + name: "handle" + type: DT_RESOURCE + } + attr { + name: "container" + type: "string" + default_value { + s: "" + } + } + attr { + name: "shared_name" + type: "string" + default_value { + s: "" + } + } + is_stateful: true +} +op { + name: "ExperimentalStatsAggregatorSummary" + input_arg { + name: "iterator" + type: DT_RESOURCE + } + output_arg { + name: "summary" + type: DT_STRING + } + is_stateful: true +} op { name: "ExperimentalThreadPoolDataset" input_arg { @@ -10552,6 +11138,29 @@ op { } is_stateful: true } +op { + name: "ExperimentalUnbatchDataset" + input_arg { + name: "input_dataset" + type: DT_VARIANT + } + output_arg { + name: "handle" + type: DT_VARIANT + } + attr { + name: "output_types" + type: "list(type)" + has_minimum: true + minimum: 1 + } + attr { + name: "output_shapes" + type: "list(shape)" + has_minimum: true + minimum: 1 + } +} op { name: "ExperimentalUniqueDataset" input_arg { @@ -12744,144 +13353,6 @@ op { } } } -op { - name: "GroupByReducerDataset" - input_arg { - name: "input_dataset" - type: DT_VARIANT - } - input_arg { - name: "key_func_other_arguments" - type_list_attr: "Tkey_func_other_arguments" - } - input_arg { - name: "init_func_other_arguments" - type_list_attr: "Tinit_func_other_arguments" - } - input_arg { - name: "reduce_func_other_arguments" - type_list_attr: "Treduce_func_other_arguments" - } - input_arg { - name: "finalize_func_other_arguments" - type_list_attr: "Tfinalize_func_other_arguments" - } - output_arg { - name: "handle" - type: DT_VARIANT - } - attr { - name: "key_func" - type: "func" - } - attr { - name: "init_func" - type: "func" - } - attr { - name: "reduce_func" - type: "func" - } - attr { - name: "finalize_func" - type: "func" - } - attr { - name: "Tkey_func_other_arguments" - type: "list(type)" - has_minimum: true - } - attr { - name: "Tinit_func_other_arguments" - type: "list(type)" - has_minimum: true - } - attr { - name: "Treduce_func_other_arguments" - type: "list(type)" - has_minimum: true - } - attr { - name: "Tfinalize_func_other_arguments" - type: "list(type)" - has_minimum: true - } - attr { - name: "output_types" - type: "list(type)" - has_minimum: true - minimum: 1 - } - attr { - name: "output_shapes" - type: "list(shape)" - has_minimum: true - minimum: 1 - } - is_stateful: true -} -op { - name: "GroupByWindowDataset" - input_arg { - name: "input_dataset" - type: DT_VARIANT - } - input_arg { - name: "key_func_other_arguments" - type_list_attr: "Tkey_func_other_arguments" - } - input_arg { - name: "reduce_func_other_arguments" - type_list_attr: "Treduce_func_other_arguments" - } - input_arg { - name: "window_size_func_other_arguments" - type_list_attr: "Twindow_size_func_other_arguments" - } - output_arg { - name: "handle" - type: DT_VARIANT - } - attr { - name: "key_func" - type: "func" - } - attr { - name: "reduce_func" - type: "func" - } - attr { - name: "window_size_func" - type: "func" - } - attr { - name: "Tkey_func_other_arguments" - type: "list(type)" - has_minimum: true - } - attr { - name: "Treduce_func_other_arguments" - type: "list(type)" - has_minimum: true - } - attr { - name: "Twindow_size_func_other_arguments" - type: "list(type)" - has_minimum: true - } - attr { - name: "output_types" - type: "list(type)" - has_minimum: true - minimum: 1 - } - attr { - name: "output_shapes" - type: "list(shape)" - has_minimum: true - minimum: 1 - } -} op { name: "GuaranteeConst" input_arg { @@ -14422,33 +14893,6 @@ op { } } } -op { - name: "LatencyStatsDataset" - input_arg { - name: "input_dataset" - type: DT_VARIANT - } - input_arg { - name: "tag" - type: DT_STRING - } - output_arg { - name: "handle" - type: DT_VARIANT - } - attr { - name: "output_types" - type: "list(type)" - has_minimum: true - minimum: 1 - } - attr { - name: "output_shapes" - type: "list(shape)" - has_minimum: true - minimum: 1 - } -} op { name: "LeakyRelu" input_arg { @@ -15319,102 +15763,6 @@ op { } is_stateful: true } -op { - name: "MapAndBatchDataset" - input_arg { - name: "input_dataset" - type: DT_VARIANT - } - input_arg { - name: "other_arguments" - type_list_attr: "Targuments" - } - input_arg { - name: "batch_size" - type: DT_INT64 - } - input_arg { - name: "num_parallel_batches" - type: DT_INT64 - } - input_arg { - name: "drop_remainder" - type: DT_BOOL - } - output_arg { - name: "handle" - type: DT_VARIANT - } - attr { - name: "f" - type: "func" - } - attr { - name: "Targuments" - type: "list(type)" - has_minimum: true - } - attr { - name: "output_types" - type: "list(type)" - has_minimum: true - minimum: 1 - } - attr { - name: "output_shapes" - type: "list(shape)" - has_minimum: true - minimum: 1 - } -} -op { - name: "MapAndBatchDatasetV2" - input_arg { - name: "input_dataset" - type: DT_VARIANT - } - input_arg { - name: "other_arguments" - type_list_attr: "Targuments" - } - input_arg { - name: "batch_size" - type: DT_INT64 - } - input_arg { - name: "num_parallel_calls" - type: DT_INT64 - } - input_arg { - name: "drop_remainder" - type: DT_BOOL - } - output_arg { - name: "handle" - type: DT_VARIANT - } - attr { - name: "f" - type: "func" - } - attr { - name: "Targuments" - type: "list(type)" - has_minimum: true - } - attr { - name: "output_types" - type: "list(type)" - has_minimum: true - minimum: 1 - } - attr { - name: "output_shapes" - type: "list(shape)" - has_minimum: true - minimum: 1 - } -} op { name: "MapClear" attr { @@ -15894,18 +16242,6 @@ op { type: DT_STRING } } -op { - name: "MatchingFilesDataset" - input_arg { - name: "patterns" - type: DT_STRING - } - output_arg { - name: "handle" - type: DT_VARIANT - } - is_stateful: true -} op { name: "MatrixBandPart" input_arg { @@ -19449,62 +19785,6 @@ op { type: "type" } } -op { - name: "ParallelInterleaveDataset" - input_arg { - name: "input_dataset" - type: DT_VARIANT - } - input_arg { - name: "other_arguments" - type_list_attr: "Targuments" - } - input_arg { - name: "cycle_length" - type: DT_INT64 - } - input_arg { - name: "block_length" - type: DT_INT64 - } - input_arg { - name: "sloppy" - type: DT_BOOL - } - input_arg { - name: "buffer_output_elements" - type: DT_INT64 - } - input_arg { - name: "prefetch_input_elements" - type: DT_INT64 - } - output_arg { - name: "handle" - type: DT_VARIANT - } - attr { - name: "f" - type: "func" - } - attr { - name: "Targuments" - type: "list(type)" - has_minimum: true - } - attr { - name: "output_types" - type: "list(type)" - has_minimum: true - minimum: 1 - } - attr { - name: "output_shapes" - type: "list(shape)" - has_minimum: true - minimum: 1 - } -} op { name: "ParallelInterleaveDatasetV2" input_arg { @@ -19760,83 +20040,6 @@ op { has_minimum: true } } -op { - name: "ParseExampleDataset" - input_arg { - name: "input_dataset" - type: DT_VARIANT - } - input_arg { - name: "num_parallel_calls" - type: DT_INT64 - } - input_arg { - name: "dense_defaults" - type_list_attr: "Tdense" - } - output_arg { - name: "handle" - type: DT_VARIANT - } - attr { - name: "sparse_keys" - type: "list(string)" - has_minimum: true - } - attr { - name: "dense_keys" - type: "list(string)" - has_minimum: true - } - attr { - name: "sparse_types" - type: "list(type)" - has_minimum: true - allowed_values { - list { - type: DT_FLOAT - type: DT_INT64 - type: DT_STRING - } - } - } - attr { - name: "Tdense" - type: "list(type)" - has_minimum: true - allowed_values { - list { - type: DT_FLOAT - type: DT_INT64 - type: DT_STRING - } - } - } - attr { - name: "dense_shapes" - type: "list(shape)" - has_minimum: true - } - attr { - name: "output_types" - type: "list(type)" - has_minimum: true - minimum: 1 - } - attr { - name: "output_shapes" - type: "list(shape)" - has_minimum: true - minimum: 1 - } - attr { - name: "sloppy" - type: "bool" - default_value { - b: false - } - } -} op { name: "ParseSequenceExample" input_arg { @@ -20340,6 +20543,13 @@ op { s: "" } } + attr { + name: "config_proto" + type: "string" + default_value { + s: "" + } + } attr { name: "executor_type" type: "string" @@ -20516,48 +20726,6 @@ op { minimum: 1 } } -op { - name: "PrependFromQueueAndPaddedBatchDataset" - input_arg { - name: "input_dataset" - type: DT_VARIANT - } - input_arg { - name: "batch_size" - type: DT_INT64 - } - input_arg { - name: "padded_shapes" - type: DT_INT64 - number_attr: "N" - } - input_arg { - name: "padding_values" - type_list_attr: "Toutput_types" - } - output_arg { - name: "handle" - type: DT_VARIANT - } - attr { - name: "Toutput_types" - type: "list(type)" - has_minimum: true - minimum: 1 - } - attr { - name: "output_shapes" - type: "list(shape)" - has_minimum: true - minimum: 1 - } - attr { - name: "N" - type: "int" - has_minimum: true - minimum: 1 - } -} op { name: "PreventGradient" input_arg { @@ -20986,6 +21154,19 @@ op { } } } + attr { + name: "round_mode" + type: "string" + default_value { + s: "HALF_TO_EVEN" + } + allowed_values { + list { + s: "HALF_TO_EVEN" + s: "HALF_UP" + } + } + } } op { name: "QuantizeAndDequantizeV3" @@ -22816,34 +22997,6 @@ op { } is_stateful: true } -op { - name: "RandomDataset" - input_arg { - name: "seed" - type: DT_INT64 - } - input_arg { - name: "seed2" - type: DT_INT64 - } - output_arg { - name: "handle" - type: DT_VARIANT - } - attr { - name: "output_types" - type: "list(type)" - has_minimum: true - minimum: 1 - } - attr { - name: "output_shapes" - type: "list(shape)" - has_minimum: true - minimum: 1 - } - is_stateful: true -} op { name: "RandomGamma" input_arg { @@ -25095,6 +25248,86 @@ op { } is_stateful: true } +op { + name: "ResourceApplyAdamWithAmsgrad" + input_arg { + name: "var" + type: DT_RESOURCE + } + input_arg { + name: "m" + type: DT_RESOURCE + } + input_arg { + name: "v" + type: DT_RESOURCE + } + input_arg { + name: "vhat" + type: DT_RESOURCE + } + input_arg { + name: "beta1_power" + type_attr: "T" + } + input_arg { + name: "beta2_power" + type_attr: "T" + } + input_arg { + name: "lr" + type_attr: "T" + } + input_arg { + name: "beta1" + type_attr: "T" + } + input_arg { + name: "beta2" + type_attr: "T" + } + input_arg { + name: "epsilon" + type_attr: "T" + } + input_arg { + name: "grad" + type_attr: "T" + } + attr { + name: "T" + type: "type" + allowed_values { + list { + type: DT_FLOAT + type: DT_DOUBLE + type: DT_INT32 + type: DT_UINT8 + type: DT_INT16 + type: DT_INT8 + type: DT_COMPLEX64 + type: DT_INT64 + type: DT_QINT8 + type: DT_QUINT8 + type: DT_QINT32 + type: DT_BFLOAT16 + type: DT_UINT16 + type: DT_COMPLEX128 + type: DT_HALF + type: DT_UINT32 + type: DT_UINT64 + } + } + } + attr { + name: "use_locking" + type: "bool" + default_value { + b: false + } + } + is_stateful: true +} op { name: "ResourceApplyAddSign" input_arg { @@ -25419,6 +25652,69 @@ op { } is_stateful: true } +op { + name: "ResourceApplyKerasMomentum" + input_arg { + name: "var" + type: DT_RESOURCE + } + input_arg { + name: "accum" + type: DT_RESOURCE + } + input_arg { + name: "lr" + type_attr: "T" + } + input_arg { + name: "grad" + type_attr: "T" + } + input_arg { + name: "momentum" + type_attr: "T" + } + attr { + name: "T" + type: "type" + allowed_values { + list { + type: DT_FLOAT + type: DT_DOUBLE + type: DT_INT32 + type: DT_UINT8 + type: DT_INT16 + type: DT_INT8 + type: DT_COMPLEX64 + type: DT_INT64 + type: DT_QINT8 + type: DT_QUINT8 + type: DT_QINT32 + type: DT_BFLOAT16 + type: DT_UINT16 + type: DT_COMPLEX128 + type: DT_HALF + type: DT_UINT32 + type: DT_UINT64 + } + } + } + attr { + name: "use_locking" + type: "bool" + default_value { + b: false + } + } + attr { + name: "use_nesterov" + type: "bool" + default_value { + b: false + } + } + is_stateful: true +} op { name: "ResourceApplyMomentum" input_arg { @@ -26690,6 +26986,83 @@ op { } is_stateful: true } +op { + name: "ResourceSparseApplyKerasMomentum" + input_arg { + name: "var" + type: DT_RESOURCE + } + input_arg { + name: "accum" + type: DT_RESOURCE + } + input_arg { + name: "lr" + type_attr: "T" + } + input_arg { + name: "grad" + type_attr: "T" + } + input_arg { + name: "indices" + type_attr: "Tindices" + } + input_arg { + name: "momentum" + type_attr: "T" + } + attr { + name: "T" + type: "type" + allowed_values { + list { + type: DT_FLOAT + type: DT_DOUBLE + type: DT_INT32 + type: DT_UINT8 + type: DT_INT16 + type: DT_INT8 + type: DT_COMPLEX64 + type: DT_INT64 + type: DT_QINT8 + type: DT_QUINT8 + type: DT_QINT32 + type: DT_BFLOAT16 + type: DT_UINT16 + type: DT_COMPLEX128 + type: DT_HALF + type: DT_UINT32 + type: DT_UINT64 + } + } + } + attr { + name: "Tindices" + type: "type" + allowed_values { + list { + type: DT_INT32 + type: DT_INT64 + } + } + } + attr { + name: "use_locking" + type: "bool" + default_value { + b: false + } + } + attr { + name: "use_nesterov" + type: "bool" + default_value { + b: false + } + } + is_stateful: true +} op { name: "ResourceSparseApplyMomentum" input_arg { @@ -27789,52 +28162,6 @@ op { } } } -op { - name: "ScanDataset" - input_arg { - name: "input_dataset" - type: DT_VARIANT - } - input_arg { - name: "initial_state" - type_list_attr: "Tstate" - } - input_arg { - name: "other_arguments" - type_list_attr: "Targuments" - } - output_arg { - name: "handle" - type: DT_VARIANT - } - attr { - name: "f" - type: "func" - } - attr { - name: "Tstate" - type: "list(type)" - has_minimum: true - minimum: 1 - } - attr { - name: "Targuments" - type: "list(type)" - has_minimum: true - } - attr { - name: "output_types" - type: "list(type)" - has_minimum: true - minimum: 1 - } - attr { - name: "output_shapes" - type: "list(shape)" - has_minimum: true - minimum: 1 - } -} op { name: "ScatterAdd" input_arg { @@ -29272,42 +29599,6 @@ op { } } } -op { - name: "SetStatsAggregatorDataset" - input_arg { - name: "input_dataset" - type: DT_VARIANT - } - input_arg { - name: "stats_aggregator" - type: DT_RESOURCE - } - input_arg { - name: "tag" - type: DT_STRING - } - input_arg { - name: "counter_prefix" - type: DT_STRING - } - output_arg { - name: "handle" - type: DT_VARIANT - } - attr { - name: "output_types" - type: "list(type)" - has_minimum: true - minimum: 1 - } - attr { - name: "output_shapes" - type: "list(shape)" - has_minimum: true - minimum: 1 - } - is_stateful: true -} op { name: "Shape" input_arg { @@ -29771,41 +30062,6 @@ op { } } } -op { - name: "SlideDataset" - input_arg { - name: "input_dataset" - type: DT_VARIANT - } - input_arg { - name: "window_size" - type: DT_INT64 - } - input_arg { - name: "window_shift" - type: DT_INT64 - } - input_arg { - name: "window_stride" - type: DT_INT64 - } - output_arg { - name: "handle" - type: DT_VARIANT - } - attr { - name: "output_types" - type: "list(type)" - has_minimum: true - minimum: 1 - } - attr { - name: "output_shapes" - type: "list(shape)" - has_minimum: true - minimum: 1 - } -} op { name: "Snapshot" input_arg { @@ -32996,38 +33252,6 @@ op { } } } -op { - name: "SqlDataset" - input_arg { - name: "driver_name" - type: DT_STRING - } - input_arg { - name: "data_source_name" - type: DT_STRING - } - input_arg { - name: "query" - type: DT_STRING - } - output_arg { - name: "handle" - type: DT_VARIANT - } - attr { - name: "output_types" - type: "list(type)" - has_minimum: true - minimum: 1 - } - attr { - name: "output_shapes" - type: "list(shape)" - has_minimum: true - minimum: 1 - } - is_stateful: true -} op { name: "Sqrt" input_arg { @@ -33513,6 +33737,13 @@ op { s: "" } } + attr { + name: "config_proto" + type: "string" + default_value { + s: "" + } + } attr { name: "executor_type" type: "string" @@ -33913,40 +34144,6 @@ op { } } } -op { - name: "StatsAggregatorHandle" - output_arg { - name: "handle" - type: DT_RESOURCE - } - attr { - name: "container" - type: "string" - default_value { - s: "" - } - } - attr { - name: "shared_name" - type: "string" - default_value { - s: "" - } - } - is_stateful: true -} -op { - name: "StatsAggregatorSummary" - input_arg { - name: "iterator" - type: DT_RESOURCE - } - output_arg { - name: "summary" - type: DT_STRING - } - is_stateful: true -} op { name: "StopGradient" input_arg { @@ -35923,6 +36120,127 @@ op { } is_stateful: true } +op { + name: "TensorForestCreateTreeVariable" + input_arg { + name: "tree_handle" + type: DT_RESOURCE + } + input_arg { + name: "tree_config" + type: DT_STRING + } + is_stateful: true +} +op { + name: "TensorForestTreeDeserialize" + input_arg { + name: "tree_handle" + type: DT_RESOURCE + } + input_arg { + name: "tree_config" + type: DT_STRING + } + is_stateful: true +} +op { + name: "TensorForestTreeIsInitializedOp" + input_arg { + name: "tree_handle" + type: DT_RESOURCE + } + output_arg { + name: "is_initialized" + type: DT_BOOL + } + is_stateful: true +} +op { + name: "TensorForestTreePredict" + input_arg { + name: "tree_handle" + type: DT_RESOURCE + } + input_arg { + name: "dense_features" + type: DT_FLOAT + } + output_arg { + name: "logits" + type: DT_FLOAT + } + attr { + name: "logits_dimension" + type: "int" + } + is_stateful: true +} +op { + name: "TensorForestTreeResourceHandleOp" + output_arg { + name: "resource" + type: DT_RESOURCE + } + attr { + name: "container" + type: "string" + default_value { + s: "" + } + } + attr { + name: "shared_name" + type: "string" + default_value { + s: "" + } + } + is_stateful: true +} +op { + name: "TensorForestTreeSerialize" + input_arg { + name: "tree_handle" + type: DT_RESOURCE + } + output_arg { + name: "tree_config" + type: DT_STRING + } + is_stateful: true +} +op { + name: "TensorForestTreeSize" + input_arg { + name: "tree_handle" + type: DT_RESOURCE + } + output_arg { + name: "tree_size" + type: DT_INT32 + } + is_stateful: true +} +op { + name: "TensorListConcat" + input_arg { + name: "input_handle" + type: DT_VARIANT + } + output_arg { + name: "tensor" + type_attr: "element_dtype" + } + output_arg { + name: "lengths" + type: DT_INT64 + } + attr { + name: "element_dtype" + type: "type" + } +} op { name: "TensorListConcatLists" input_arg { @@ -36183,6 +36501,39 @@ op { type: "type" } } +op { + name: "TensorListSplit" + input_arg { + name: "tensor" + type_attr: "element_dtype" + } + input_arg { + name: "element_shape" + type_attr: "shape_type" + } + input_arg { + name: "lengths" + type: DT_INT64 + } + output_arg { + name: "output_handle" + type: DT_VARIANT + } + attr { + name: "element_dtype" + type: "type" + } + attr { + name: "shape_type" + type: "type" + allowed_values { + list { + type: DT_INT32 + type: DT_INT64 + } + } + } +} op { name: "TensorListStack" input_arg { @@ -36205,6 +36556,105 @@ op { } } } +op { + name: "TensorScatterAdd" + input_arg { + name: "tensor" + type_attr: "T" + } + input_arg { + name: "indices" + type_attr: "Tindices" + } + input_arg { + name: "updates" + type_attr: "T" + } + output_arg { + name: "output" + type_attr: "T" + } + attr { + name: "T" + type: "type" + } + attr { + name: "Tindices" + type: "type" + allowed_values { + list { + type: DT_INT32 + type: DT_INT64 + } + } + } +} +op { + name: "TensorScatterSub" + input_arg { + name: "tensor" + type_attr: "T" + } + input_arg { + name: "indices" + type_attr: "Tindices" + } + input_arg { + name: "updates" + type_attr: "T" + } + output_arg { + name: "output" + type_attr: "T" + } + attr { + name: "T" + type: "type" + } + attr { + name: "Tindices" + type: "type" + allowed_values { + list { + type: DT_INT32 + type: DT_INT64 + } + } + } +} +op { + name: "TensorScatterUpdate" + input_arg { + name: "tensor" + type_attr: "T" + } + input_arg { + name: "indices" + type_attr: "Tindices" + } + input_arg { + name: "updates" + type_attr: "T" + } + output_arg { + name: "output" + type_attr: "T" + } + attr { + name: "T" + type: "type" + } + attr { + name: "Tindices" + type: "type" + allowed_values { + list { + type: DT_INT32 + type: DT_INT64 + } + } + } +} op { name: "TensorSliceDataset" input_arg { @@ -36822,29 +37272,6 @@ op { type: "type" } } -op { - name: "UnbatchDataset" - input_arg { - name: "input_dataset" - type: DT_VARIANT - } - output_arg { - name: "handle" - type: DT_VARIANT - } - attr { - name: "output_types" - type: "list(type)" - has_minimum: true - minimum: 1 - } - attr { - name: "output_shapes" - type: "list(shape)" - has_minimum: true - minimum: 1 - } -} op { name: "UnbatchGrad" input_arg { @@ -36886,6 +37313,104 @@ op { type: "type" } } +op { + name: "UnicodeDecodeWithOffsets" + input_arg { + name: "input" + type: DT_STRING + } + output_arg { + name: "row_splits" + type: DT_INT64 + } + output_arg { + name: "char_values" + type: DT_INT32 + } + output_arg { + name: "char_to_byte_starts" + type: DT_INT64 + } + attr { + name: "input_encoding" + type: "string" + } + attr { + name: "errors" + type: "string" + default_value { + s: "replace" + } + allowed_values { + list { + s: "strict" + s: "replace" + s: "ignore" + } + } + } + attr { + name: "replacement_char" + type: "int" + default_value { + i: 65533 + } + } + attr { + name: "replace_control_characters" + type: "bool" + default_value { + b: false + } + } +} +op { + name: "UnicodeEncode" + input_arg { + name: "input_values" + type: DT_INT32 + } + input_arg { + name: "input_splits" + type: DT_INT64 + } + output_arg { + name: "output" + type: DT_STRING + } + attr { + name: "errors" + type: "string" + default_value { + s: "replace" + } + allowed_values { + list { + s: "ignore" + s: "replace" + s: "strict" + } + } + } + attr { + name: "output_encoding" + type: "string" + allowed_values { + list { + s: "UTF-8" + s: "UTF-16-BE" + s: "UTF-32-BE" + } + } + } + attr { + name: "replacement_char" + type: "int" + default_value { + i: 65533 + } + } +} op { name: "UnicodeScript" input_arg { diff --git a/tensorflow/core/ops/sparse_ops.cc b/tensorflow/core/ops/sparse_ops.cc index bc0cb2095da..de08a107845 100644 --- a/tensorflow/core/ops/sparse_ops.cc +++ b/tensorflow/core/ops/sparse_ops.cc @@ -401,7 +401,7 @@ REGISTER_OP("SparseReduceMax") .Attr("keep_dims: bool = False") .Output("output: T") .Attr("T: realnumbertype") - .SetShapeFn(shape_inference::UnknownShape); + .SetShapeFn(shape_inference::SparseReduceShapeFn); REGISTER_OP("SparseReduceMaxSparse") .Input("input_indices: int64") @@ -423,7 +423,7 @@ REGISTER_OP("SparseReduceSum") .Attr("keep_dims: bool = False") .Output("output: T") .Attr("T: numbertype") - .SetShapeFn(shape_inference::UnknownShape); + .SetShapeFn(shape_inference::SparseReduceShapeFn); REGISTER_OP("SparseReduceSumSparse") .Input("input_indices: int64") diff --git a/tensorflow/core/ops/sparse_ops_test.cc b/tensorflow/core/ops/sparse_ops_test.cc index 6a9b5ce4d31..00283c59932 100644 --- a/tensorflow/core/ops/sparse_ops_test.cc +++ b/tensorflow/core/ops/sparse_ops_test.cc @@ -133,6 +133,13 @@ TEST(SparseOpsTest, SparseToDense_ShapeFn) { TEST(SparseOpsTest, SparseReduceSum_ShapeFn) { ShapeInferenceTestOp op("SparseReduceSum"); + TF_ASSERT_OK(NodeDefBuilder("test", "SparseReduceSum") + .Input({"input_indices", 0, DT_INT64}) + .Input({"input_values", 1, DT_INT64}) + .Input({"input_shape", 2, DT_INT64}) + .Input({"reduction_axes", 3, DT_INT32}) + .Attr("keep_dims", false) + .Finalize(&op.node_def)); // Shape fn always yields unknown. INFER_OK(op, "?;?;?;?", "?"); diff --git a/tensorflow/core/ops/string_ops.cc b/tensorflow/core/ops/string_ops.cc index 352253135c4..8ea74f1d43e 100644 --- a/tensorflow/core/ops/string_ops.cc +++ b/tensorflow/core/ops/string_ops.cc @@ -13,13 +13,24 @@ See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ +#include +#include + #include "absl/strings/str_split.h" #include "tensorflow/core/framework/common_shape_fns.h" #include "tensorflow/core/framework/op.h" #include "tensorflow/core/framework/shape_inference.h" +#include "tensorflow/core/lib/core/errors.h" +#include "tensorflow/core/lib/core/status.h" +#include "tensorflow/core/lib/strings/strcat.h" +#include "tensorflow/core/platform/types.h" namespace tensorflow { +namespace shape_inference { +class InferenceContext; +} // namespace shape_inference + using shape_inference::DimensionHandle; using shape_inference::InferenceContext; using shape_inference::ShapeHandle; @@ -250,6 +261,31 @@ REGISTER_OP("UnicodeScript") .Output("output: int32") .SetShapeFn(shape_inference::UnchangedShape); +REGISTER_OP("UnicodeEncode") + .Input("input_values: int32") + .Input("input_splits: int64") + .Attr("errors: {'ignore', 'replace', 'strict'} = 'replace'") + .Attr("output_encoding: {'UTF-8', 'UTF-16-BE', 'UTF-32-BE'}") + .Attr("replacement_char: int = 65533") // 0xFFFD unicode replacement char + .Output("output: string") + .SetShapeFn([](InferenceContext* c) { + // Check rank of inner values + ShapeHandle input_inner_values_shape = c->input(0); + ShapeHandle unused; + TF_RETURN_IF_ERROR(c->WithRank(input_inner_values_shape, 1, &unused)); + + // Check rank of input_splits + ShapeHandle splits_shape = c->input(1); + TF_RETURN_IF_ERROR(c->WithRank(splits_shape, 1, &unused)); + + // Output shape is a 1-D tensor with size equal to number of splits. + std::vector dims(1); + TF_RETURN_IF_ERROR(c->Subtract(c->Dim(splits_shape, 0), 1, &dims[0])); + c->set_output(0, c->MakeShape(dims)); + + return Status::OK(); + }); + REGISTER_OP("UnicodeTranscode") .Input("input: string") .Output("output: string") @@ -259,4 +295,28 @@ REGISTER_OP("UnicodeTranscode") .Attr("replacement_char: int = 65533") // 0xFFFD unicode replacement char .Attr("replace_control_characters: bool = false") .SetShapeFn(shape_inference::UnchangedShape); + +REGISTER_OP("UnicodeDecodeWithOffsets") + .Input("input: string") + .Output("row_splits: int64") + .Output("char_values: int32") + .Output("char_to_byte_starts: int64") + .Attr("input_encoding: string") + .Attr("errors: {'strict', 'replace', 'ignore'} = 'replace'") + .Attr("replacement_char: int = 65533") // 0xFFFD unicode replacement char + .Attr("replace_control_characters: bool = false") + .SetShapeFn([](InferenceContext* c) { + // row_splits.shape == [input.size() + 1] + DimensionHandle num_row_splits; + DimensionHandle input_size = c->NumElements(c->input(0)); + TF_RETURN_IF_ERROR(c->Add(input_size, 1, &num_row_splits)); + c->set_output(0, c->Vector(num_row_splits)); + + // char_values.shape == offset_values.shape == [num_chars] + DimensionHandle num_chars = c->UnknownDim(); + c->set_output(1, c->Vector(num_chars)); + c->set_output(2, c->Vector(num_chars)); + return Status::OK(); + }); + } // namespace tensorflow diff --git a/tensorflow/core/ops/tensor_forest_ops.cc b/tensorflow/core/ops/tensor_forest_ops.cc new file mode 100644 index 00000000000..b4b6ba318e9 --- /dev/null +++ b/tensorflow/core/ops/tensor_forest_ops.cc @@ -0,0 +1,79 @@ +/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include + +#include "tensorflow/core/framework/common_shape_fns.h" +#include "tensorflow/core/framework/op.h" +#include "tensorflow/core/framework/resource_mgr.h" +#include "tensorflow/core/framework/shape_inference.h" + +namespace tensorflow { + +REGISTER_RESOURCE_HANDLE_OP(TensorForestTreeResource); + +REGISTER_OP("TensorForestTreeIsInitializedOp") + .Input("tree_handle: resource") + .Output("is_initialized: bool") + .SetShapeFn([](shape_inference::InferenceContext* c) { + shape_inference::ShapeHandle unused_input; + TF_RETURN_IF_ERROR(c->WithRank(c->input(0), 0, &unused_input)); + c->set_output(0, c->Scalar()); + return Status::OK(); + }); + +REGISTER_OP("TensorForestCreateTreeVariable") + .Input("tree_handle: resource") + .Input("tree_config: string") + .SetShapeFn(tensorflow::shape_inference::NoOutputs); + +REGISTER_OP("TensorForestTreeSerialize") + .Input("tree_handle: resource") + .Output("tree_config: string") + .SetShapeFn(tensorflow::shape_inference::ScalarShape); + +REGISTER_OP("TensorForestTreeDeserialize") + .Input("tree_handle: resource") + .Input("tree_config: string") + .SetShapeFn([](shape_inference::InferenceContext* c) { + shape_inference::ShapeHandle unused_input; + TF_RETURN_IF_ERROR(c->WithRank(c->input(0), 0, &unused_input)); + return Status::OK(); + }); + +REGISTER_OP("TensorForestTreeSize") + .Input("tree_handle: resource") + .Output("tree_size: int32") + .SetShapeFn(tensorflow::shape_inference::ScalarShape); + +REGISTER_OP("TensorForestTreePredict") + .Attr("logits_dimension: int") + .Input("tree_handle: resource") + .Input("dense_features: float") + .Output("logits: float") + .SetShapeFn([](tensorflow::shape_inference::InferenceContext* c) { + shape_inference::ShapeHandle shape_handle; + shape_inference::DimensionHandle batch_size = c->UnknownDim(); + + TF_RETURN_IF_ERROR(c->WithRank(c->input(1), 2, &shape_handle)); + + batch_size = c->Dim(shape_handle, 0); + + int logits_dimension; + TF_RETURN_IF_ERROR(c->GetAttr("logits_dimension", &logits_dimension)); + c->set_output(0, c->Matrix(batch_size, logits_dimension)); + return Status::OK(); + }); +} // namespace tensorflow diff --git a/tensorflow/core/ops/training_ops.cc b/tensorflow/core/ops/training_ops.cc index 94ff092a85d..995ed42d53d 100644 --- a/tensorflow/core/ops/training_ops.cc +++ b/tensorflow/core/ops/training_ops.cc @@ -685,6 +685,34 @@ REGISTER_OP("ResourceSparseApplyMomentum") return ApplyMomentumShapeFn(c, true /* sparse */); }); +REGISTER_OP("ResourceApplyKerasMomentum") + .Input("var: resource") + .Input("accum: resource") + .Input("lr: T") + .Input("grad: T") + .Input("momentum: T") + .Attr("T: numbertype") + .Attr("use_locking: bool = false") + .Attr("use_nesterov: bool = false") + .SetShapeFn([](InferenceContext* c) { + return ApplyMomentumShapeFn(c, false /* sparse */); + }); + +REGISTER_OP("ResourceSparseApplyKerasMomentum") + .Input("var: resource") + .Input("accum: resource") + .Input("lr: T") + .Input("grad: T") + .Input("indices: Tindices") + .Input("momentum: T") + .Attr("T: numbertype") + .Attr("Tindices: {int32, int64}") + .Attr("use_locking: bool = false") + .Attr("use_nesterov: bool = false") + .SetShapeFn([](InferenceContext* c) { + return ApplyMomentumShapeFn(c, true /* sparse */); + }); + static Status ApplyAdamShapeFn(InferenceContext* c, bool sparse) { ShapeHandle unused; ShapeHandle s = ShapeOrHandleShape(c, 0); // var @@ -741,6 +769,44 @@ REGISTER_OP("ResourceApplyAdam") return ApplyAdamShapeFn(c, false /* sparse */); }); +static Status ApplyAdamWithAmsgradShapeFn(InferenceContext* c, bool sparse) { + ShapeHandle unused; + ShapeHandle s = ShapeOrHandleShape(c, 0); // var + TF_RETURN_IF_ERROR(c->Merge(s, ShapeOrHandleShape(c, 1), &s)); // m + TF_RETURN_IF_ERROR(c->Merge(s, ShapeOrHandleShape(c, 2), &s)); // v + TF_RETURN_IF_ERROR(c->Merge(s, ShapeOrHandleShape(c, 3), &s)); // vhat + TF_RETURN_IF_ERROR(c->WithRank(c->input(4), 0, &unused)); // beta1_power + TF_RETURN_IF_ERROR(c->WithRank(c->input(5), 0, &unused)); // beta2_power + TF_RETURN_IF_ERROR(c->WithRank(c->input(6), 0, &unused)); // lr + TF_RETURN_IF_ERROR(c->WithRank(c->input(7), 0, &unused)); // beta1 + TF_RETURN_IF_ERROR(c->WithRank(c->input(8), 0, &unused)); // beta2 + TF_RETURN_IF_ERROR(c->WithRank(c->input(9), 0, &unused)); // epsilon + TF_RETURN_IF_ERROR( + HandleGradAndIndicesInputs(c, sparse, 10 /* grad_idx */, &s)); + if (c->num_outputs() > 0) { + c->set_output(0, s); + } + return Status::OK(); +} + +REGISTER_OP("ResourceApplyAdamWithAmsgrad") + .Input("var: resource") + .Input("m: resource") + .Input("v: resource") + .Input("vhat: resource") + .Input("beta1_power: T") + .Input("beta2_power: T") + .Input("lr: T") + .Input("beta1: T") + .Input("beta2: T") + .Input("epsilon: T") + .Input("grad: T") + .Attr("T: numbertype") + .Attr("use_locking: bool = false") + .SetShapeFn([](InferenceContext* c) { + return ApplyAdamWithAmsgradShapeFn(c, false /* sparse */); + }); + static Status ApplyAdaMaxShapeFn(InferenceContext* c, bool sparse) { ShapeHandle unused; ShapeHandle s = ShapeOrHandleShape(c, 0); // var diff --git a/tensorflow/core/platform/cpu_feature_guard.cc b/tensorflow/core/platform/cpu_feature_guard.cc index 9d00aa7b7fe..2efe0c0876e 100644 --- a/tensorflow/core/platform/cpu_feature_guard.cc +++ b/tensorflow/core/platform/cpu_feature_guard.cc @@ -41,7 +41,7 @@ void CheckFeatureOrDie(CPUFeature feature, const string& feature_name) { } } -// Check if CPU feature is inclued in the TensorFlow binary. +// Check if CPU feature is included in the TensorFlow binary. void CheckIfFeatureUnused(CPUFeature feature, const string& feature_name, string& missing_instructions) { if (TestCPUFeature(feature)) { diff --git a/tensorflow/core/platform/default/build_config.bzl b/tensorflow/core/platform/default/build_config.bzl index 3a4415f229b..04287151301 100644 --- a/tensorflow/core/platform/default/build_config.bzl +++ b/tensorflow/core/platform/default/build_config.bzl @@ -543,6 +543,9 @@ def tf_additional_proto_srcs(): def tf_additional_human_readable_json_deps(): return [] +def tf_additional_logger_deps(): + return [] + def tf_additional_all_protos(): return ["//tensorflow/core:protos_all"] diff --git a/tensorflow/core/platform/default/device_tracer.cc b/tensorflow/core/platform/default/device_tracer.cc index cf8b477b83d..8351362e056 100644 --- a/tensorflow/core/platform/default/device_tracer.cc +++ b/tensorflow/core/platform/default/device_tracer.cc @@ -297,19 +297,16 @@ CUPTIManager *GetCUPTIManager() { // for the duration of the CUPTI API callback. TF_STATIC_THREAD_LOCAL_POD(const char *, tls_current_annotation); -class DeviceTracerImpl : public DeviceTracer, - public CUPTIClient, - public tracing::TraceCollector { +class TraceCollectorImpl : public tracing::TraceCollector { public: - DeviceTracerImpl(CUPTIManager *cupti_manager); - ~DeviceTracerImpl() override; + TraceCollectorImpl() { tracing::SetTraceCollector(this); } - // DeviceTracer interface: - Status Start() override; - Status Stop() override; - Status Collect(StepStatsCollector *collector) override; + ~TraceCollectorImpl() override { + DCHECK(!active_trace_session_) + << "Unexpected active trace session detected. "; + } - // tracing::TraceCollector interface: + // Note the method can be called after a call to Stop(). virtual std::unique_ptr CreateAnnotationHandle( StringPiece name_part1, StringPiece name_part2) const { struct Impl : public tracing::TraceCollector::Handle { @@ -332,8 +329,7 @@ class DeviceTracerImpl : public DeviceTracer, } bool IsEnabledForAnnotations() const override { - // We are always enabled for 'Annotations'. - return true; + return active_trace_session_.load(std::memory_order_relaxed); } bool IsEnabledForActivities(bool is_expensive) const override { @@ -341,6 +337,36 @@ class DeviceTracerImpl : public DeviceTracer, return false; } + void Start() { + DCHECK(!active_trace_session_) + << "Unexpected active trace session detected. "; + active_trace_session_ = true; + } + + void Stop() { + DCHECK(active_trace_session_) << "No active trace session detected. "; + active_trace_session_ = false; + } + + private: + std::atomic active_trace_session_; +}; + +TraceCollectorImpl *GlobalDefaultTraceCollector() { + static auto *instance = new TraceCollectorImpl(); + return instance; +} + +class DeviceTracerImpl : public DeviceTracer, public CUPTIClient { + public: + DeviceTracerImpl(CUPTIManager *cupti_manager); + ~DeviceTracerImpl() override; + + // DeviceTracer interface: + Status Start() override; + Status Stop() override; + Status Collect(StepStatsCollector *collector) override; + protected: // This callback is used exclusively by CUPTIManager. friend class CUPTIManager; @@ -430,7 +456,7 @@ Status DeviceTracerImpl::Start() { } // Register as a TraceEngine to receive ScopedAnnotations. - tracing::SetTraceCollector(this); + GlobalDefaultTraceCollector()->Start(); // Intercept launch and memcpy calls to capture the Op name annotation. // TODO(pbar) Add callbacks for memcpy variants. @@ -478,7 +504,8 @@ Status DeviceTracerImpl::Stop() { return Status::OK(); } CUPTI_CALL(Unsubscribe(subscriber_)); - tracing::SetTraceCollector(nullptr); + GlobalDefaultTraceCollector()->Stop(); + TF_RETURN_IF_ERROR(cupti_manager_->DisableTrace()); end_walltime_us_ = NowInUsec(); CUPTI_CALL(GetTimestamp(&end_timestamp_)); diff --git a/tensorflow/core/platform/default/logger.cc b/tensorflow/core/platform/default/logger.cc new file mode 100644 index 00000000000..54b1a1a67ca --- /dev/null +++ b/tensorflow/core/platform/default/logger.cc @@ -0,0 +1,34 @@ +/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/core/platform/logger.h" + +#include "tensorflow/core/platform/logging.h" + +namespace tensorflow { + +Logger* Logger::Singleton() { + class DefaultLogger : public Logger { + private: + void DoLogProto(google::protobuf::Any* proto) override { + VLOG(2) << proto->ShortDebugString(); + } + void DoFlush() override {} + }; + static Logger* instance = new DefaultLogger(); + return instance; +} + +} // namespace tensorflow diff --git a/tensorflow/core/platform/default/logging.cc b/tensorflow/core/platform/default/logging.cc index 34db4901067..26bd8542fd7 100644 --- a/tensorflow/core/platform/default/logging.cc +++ b/tensorflow/core/platform/default/logging.cc @@ -21,18 +21,18 @@ limitations under the License. #include #include #include -#include #endif #include +#include #include +#include +#include + namespace tensorflow { namespace internal { -LogMessage::LogMessage(const char* fname, int line, int severity) - : fname_(fname), line_(line), severity_(severity) {} - #if defined(PLATFORM_POSIX_ANDROID) void LogMessage::GenerateLogMessage() { int android_log_level; @@ -94,55 +94,156 @@ void LogMessage::GenerateLogMessage() { namespace { +int ParseInteger(const char* str, size_t size) { + // Ideally we would use env_var / safe_strto64, but it is + // hard to use here without pulling in a lot of dependencies, + // so we use std:istringstream instead + string integer_str(str, size); + std::istringstream ss(integer_str); + int level = 0; + ss >> level; + return level; +} + // Parse log level (int64) from environment variable (char*) int64 LogLevelStrToInt(const char* tf_env_var_val) { if (tf_env_var_val == nullptr) { return 0; } + return ParseInteger(tf_env_var_val, strlen(tf_env_var_val)); +} - // Ideally we would use env_var / safe_strto64, but it is - // hard to use here without pulling in a lot of dependencies, - // so we use std:istringstream instead - string min_log_level(tf_env_var_val); - std::istringstream ss(min_log_level); - int64 level; - if (!(ss >> level)) { - // Invalid vlog level setting, set level to default (0) - level = 0; +// Using StringPiece breaks Windows build. +struct StringData { + struct Hasher { + size_t operator()(const StringData& sdata) const { + // For dependency reasons, we cannot use hash.h here. Use DBJHash instead. + size_t hash = 5381; + const char* data = sdata.data; + for (const char* top = data + sdata.size; data < top; ++data) { + hash = ((hash << 5) + hash) + (*data); + } + return hash; + } + }; + + StringData() = default; + StringData(const char* data, size_t size) : data(data), size(size) {} + + bool operator==(const StringData& rhs) const { + return size == rhs.size && memcmp(data, rhs.data, size) == 0; } - return level; + const char* data = nullptr; + size_t size = 0; +}; + +using VmoduleMap = std::unordered_map; + +// Returns a mapping from module name to VLOG level, derived from the +// TF_CPP_VMOUDLE environment variable; ownership is transferred to the caller. +VmoduleMap* VmodulesMapFromEnv() { + // The value of the env var is supposed to be of the form: + // "foo=1,bar=2,baz=3" + const char* env = getenv("TF_CPP_VMODULE"); + if (env == nullptr) { + // If there is no TF_CPP_VMODULE configuration (most common case), return + // nullptr so that the ShouldVlogModule() API can fast bail out of it. + return nullptr; + } + // The memory returned by getenv() can be invalidated by following getenv() or + // setenv() calls. And since we keep references to it in the VmoduleMap in + // form of StringData objects, make a copy of it. + const char* env_data = strdup(env); + VmoduleMap* result = new VmoduleMap(); + while (true) { + const char* eq = strchr(env_data, '='); + if (eq == nullptr) { + break; + } + const char* after_eq = eq + 1; + + // Comma either points at the next comma delimiter, or at a null terminator. + // We check that the integer we parse ends at this delimiter. + const char* comma = strchr(after_eq, ','); + const char* new_env_data; + if (comma == nullptr) { + comma = strchr(after_eq, '\0'); + new_env_data = comma; + } else { + new_env_data = comma + 1; + } + (*result)[StringData(env_data, eq - env_data)] = + ParseInteger(after_eq, comma - after_eq); + env_data = new_env_data; + } + return result; } } // namespace int64 MinLogLevelFromEnv() { - const char* tf_env_var_val = getenv("TF_CPP_MIN_LOG_LEVEL"); - return LogLevelStrToInt(tf_env_var_val); -} - -int64 MinVLogLevelFromEnv() { - const char* tf_env_var_val = getenv("TF_CPP_MIN_VLOG_LEVEL"); - return LogLevelStrToInt(tf_env_var_val); -} - -LogMessage::~LogMessage() { - // Read the min log level once during the first call to logging. - static int64 min_log_level = MinLogLevelFromEnv(); - if (TF_PREDICT_TRUE(severity_ >= min_log_level)) GenerateLogMessage(); -} - -int64 LogMessage::MinVLogLevel() { // We don't want to print logs during fuzzing as that would slow fuzzing down // by almost 2x. So, if we are in fuzzing mode (not just running a test), we - // return maximum value so that nothing is actually printed + // return a value so that nothing is actually printed. Since LOG uses >= + // (see ~LogMessage in this file) to see if log messages need to be printed, + // the value we're interested on to disable printing is the maximum severity. // See also http://llvm.org/docs/LibFuzzer.html#fuzzer-friendly-build-mode #ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION return tensorflow::NUM_SEVERITIES; #else + const char* tf_env_var_val = getenv("TF_CPP_MIN_LOG_LEVEL"); + return LogLevelStrToInt(tf_env_var_val); +#endif +} + +int64 MinVLogLevelFromEnv() { + // We don't want to print logs during fuzzing as that would slow fuzzing down + // by almost 2x. So, if we are in fuzzing mode (not just running a test), we + // return a value so that nothing is actually printed. Since VLOG uses <= + // (see VLOG_IS_ON in logging.h) to see if log messages need to be printed, + // the value we're interested on to disable printing is 0. + // See also http://llvm.org/docs/LibFuzzer.html#fuzzer-friendly-build-mode +#ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION + return 0; +#else + const char* tf_env_var_val = getenv("TF_CPP_MIN_VLOG_LEVEL"); + return LogLevelStrToInt(tf_env_var_val); +#endif +} + +LogMessage::LogMessage(const char* fname, int line, int severity) + : fname_(fname), line_(line), severity_(severity) {} + +LogMessage::~LogMessage() { + // Read the min log level once during the first call to logging. + static int64 min_log_level = MinLogLevelFromEnv(); + if (severity_ >= min_log_level) { + GenerateLogMessage(); + } +} + +int64 LogMessage::MinVLogLevel() { static int64 min_vlog_level = MinVLogLevelFromEnv(); return min_vlog_level; -#endif +} + +bool LogMessage::VmoduleActivated(const char* fname, int level) { + if (level <= MinVLogLevel()) { + return true; + } + static VmoduleMap* vmodules = VmodulesMapFromEnv(); + if (TF_PREDICT_TRUE(vmodules == nullptr)) { + return false; + } + const char* last_slash = strrchr(fname, '/'); + const char* module_start = last_slash == nullptr ? fname : last_slash + 1; + const char* dot_after = strchr(module_start, '.'); + const char* module_limit = + dot_after == nullptr ? strchr(fname, '\0') : dot_after; + StringData module(module_start, module_limit - module_start); + auto it = vmodules->find(module); + return it != vmodules->end() && it->second >= level; } LogMessageFatal::LogMessageFatal(const char* file, int line) diff --git a/tensorflow/core/platform/default/logging.h b/tensorflow/core/platform/default/logging.h index 08a692fff75..bb8735ed325 100644 --- a/tensorflow/core/platform/default/logging.h +++ b/tensorflow/core/platform/default/logging.h @@ -46,6 +46,17 @@ class LogMessage : public std::basic_ostringstream { // but VLOG(3) will not. Defaults to 0. static int64 MinVLogLevel(); + // Returns whether VLOG level lvl is activated for the file fname. + // + // E.g. if the environment variable TF_CPP_VMODULE contains foo=3 and fname is + // foo.cc and lvl is <= 3, this will return true. It will also return true if + // the level is lower or equal to TF_CPP_MIN_VLOG_LEVEL (default zero). + // + // It is expected that the result of this query will be cached in the VLOG-ing + // call site to avoid repeated lookups. This routine performs a hash-map + // access against the VLOG-ing specification provided by the env var. + static bool VmoduleActivated(const char* fname, int level); + protected: void GenerateLogMessage(); @@ -55,6 +66,13 @@ class LogMessage : public std::basic_ostringstream { int severity_; }; +// Uses the lower operator & precedence to voidify a LogMessage reference, so +// that the ternary VLOG() implementation is balanced, type wise. +struct Voidifier { + template + void operator&(const T&)const {} +}; + // LogMessageFatal ensures the process will exit in failure after // logging this message. class LogMessageFatal : public LogMessage { @@ -77,18 +95,30 @@ class LogMessageFatal : public LogMessage { #define LOG(severity) _TF_LOG_##severity #ifdef IS_MOBILE_PLATFORM + // Turn VLOG off when under mobile devices for considerations of binary size. #define VLOG_IS_ON(lvl) ((lvl) <= 0) + #else -// Otherwise, Set TF_CPP_MIN_VLOG_LEVEL environment to update minimum log level -// of VLOG -#define VLOG_IS_ON(lvl) \ - ((lvl) <= ::tensorflow::internal::LogMessage::MinVLogLevel()) + +// Otherwise, set TF_CPP_MIN_VLOG_LEVEL environment to update minimum log level +// of VLOG, or TF_CPP_VMODULE to set the minimum log level for individual +// translation units. +#define VLOG_IS_ON(lvl) \ + (([](int level, const char* fname) { \ + static const bool vmodule_activated = \ + ::tensorflow::internal::LogMessage::VmoduleActivated(fname, level); \ + return vmodule_activated; \ + })(lvl, __FILE__)) + #endif -#define VLOG(lvl) \ - if (TF_PREDICT_FALSE(VLOG_IS_ON(lvl))) \ - ::tensorflow::internal::LogMessage(__FILE__, __LINE__, tensorflow::INFO) +#define VLOG(level) \ + TF_PREDICT_TRUE(!VLOG_IS_ON(level)) \ + ? (void)0 \ + : ::tensorflow::internal::Voidifier() & \ + ::tensorflow::internal::LogMessage(__FILE__, __LINE__, \ + tensorflow::INFO) // CHECK dies with a fatal error if condition is not true. It is *not* // controlled by NDEBUG, so the check will be executed regardless of diff --git a/tensorflow/core/platform/env.h b/tensorflow/core/platform/env.h index 5732271f150..7374fccdc2c 100644 --- a/tensorflow/core/platform/env.h +++ b/tensorflow/core/platform/env.h @@ -28,6 +28,7 @@ limitations under the License. #include "tensorflow/core/platform/file_system.h" #include "tensorflow/core/platform/macros.h" #include "tensorflow/core/platform/mutex.h" +#include "tensorflow/core/platform/numa.h" #include "tensorflow/core/platform/protobuf.h" #include "tensorflow/core/platform/types.h" @@ -395,6 +396,7 @@ struct ThreadOptions { size_t stack_size = 0; // 0: use system default value /// Guard area size to use near thread stacks to use (in bytes) size_t guard_size = 0; // 0: use system default value + int numa_node = port::kNUMANoAffinity; }; /// A utility routine: copy contents of `src` in file system `src_fs` diff --git a/tensorflow/core/platform/logger.h b/tensorflow/core/platform/logger.h new file mode 100644 index 00000000000..5d304bea63a --- /dev/null +++ b/tensorflow/core/platform/logger.h @@ -0,0 +1,51 @@ +/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_CORE_PLATFORM_LOGGER_H_ +#define TENSORFLOW_CORE_PLATFORM_LOGGER_H_ + +#include "google/protobuf/any.pb.h" +#include "tensorflow/core/platform/protobuf.h" + +namespace tensorflow { + +// Abstract logging interface. Contrary to logging.h, this class describes an +// interface, not a concrete logging mechanism. This is useful when we want to +// log anything to a non-local place, e.g. a database. +class Logger { + public: + static Logger* Singleton(); + + virtual ~Logger() = default; + + // Logs a typed proto. + template + void LogProto(const ProtoType& proto) { + google::protobuf::Any any; + any.PackFrom(proto); + DoLogProto(&any); + } + + // Flushes any pending log. Blocks until everything is flushed. + void Flush() { DoFlush(); } + + private: + virtual void DoLogProto(google::protobuf::Any* proto) = 0; + virtual void DoFlush() = 0; +}; + +} // namespace tensorflow + +#endif // TENSORFLOW_CORE_PLATFORM_LOGGER_H_ diff --git a/tensorflow/core/platform/numa_test.cc b/tensorflow/core/platform/numa_test.cc index 8b39ecd59cb..91789efd1ee 100644 --- a/tensorflow/core/platform/numa_test.cc +++ b/tensorflow/core/platform/numa_test.cc @@ -44,7 +44,7 @@ TEST(Numa, Malloc) { TEST(Numa, SetNodeAffinity) { // NOTE(tucker): This test is not reliable when executed under tap because - // the virtual machine may not have access to all of the availble NUMA + // the virtual machine may not have access to all of the available NUMA // nodes. Not sure what to do about that. EXPECT_EQ(-1, port::NUMAGetThreadNodeAffinity()); if (port::NUMAEnabled()) { diff --git a/tensorflow/core/platform/platform_strings.cc b/tensorflow/core/platform/platform_strings.cc new file mode 100644 index 00000000000..c1852633d59 --- /dev/null +++ b/tensorflow/core/platform/platform_strings.cc @@ -0,0 +1,64 @@ +/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/core/platform/platform_strings.h" + +#include +#include + +#include +#include + +#include "tensorflow/core/lib/core/status.h" + +namespace tensorflow { + +int GetPlatformStrings(const std::string& path, + std::vector* found) { + int result; + FILE* ifp = fopen(path.c_str(), "rb"); + if (ifp != nullptr) { + static const char prefix[] = TF_PLAT_STR_MAGIC_PREFIX_; + int first_char = prefix[1]; + int last_char = -1; + int c; + while ((c = getc(ifp)) != EOF) { + if (c == first_char && last_char == 0) { + int i = 2; + while (prefix[i] != 0 && (c = getc(ifp)) == prefix[i]) { + i++; + } + if (prefix[i] == 0) { + std::string str; + while ((c = getc(ifp)) != EOF && c != 0) { + str.push_back(c); + } + if (!str.empty()) { + found->push_back(str); + } + } + } + last_char = c; + } + + result = (ferror(ifp) == 0) ? 0 : errno; + fclose(ifp); + } else { + result = errno; + } + return result; +} + +} // namespace tensorflow diff --git a/tensorflow/core/platform/platform_strings.h b/tensorflow/core/platform/platform_strings.h new file mode 100644 index 00000000000..5b1dbd130e0 --- /dev/null +++ b/tensorflow/core/platform/platform_strings.h @@ -0,0 +1,364 @@ +/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_CORE_PLATFORM_PLATFORM_STRINGS_H_ +#define TENSORFLOW_CORE_PLATFORM_PLATFORM_STRINGS_H_ + +// This header defines the macro TF_PLATFORM_STRINGS() which should be used +// once in each dynamically loadable TensorFlow module. It embeds static +// strings into the compilation unit that allow TensorFlow to determine what +// compilation options were in effect when the compilation unit was built. All +// compilation units within the same dynamically loadable library should be +// built with the same options (or at least, the strings should be embedded in +// the compilation unit built with the most restrictive options). + +// The platform strings embedded into a binary may be retrieved with the +// GetPlatformStrings function. + +// Rationale: +// We wish to load only those libraries that this CPU can execute. For +// example, we should not load a library compiled with avx256 instructions on a +// CPU that cannot execute them. +// +// One might think that one could dlopen() the library, and call a routine that +// would return which cpu type it was compiled for. Alas, this does not work, +// because at dlopen() time, a library containing C++ will execute constructors +// of class variables with static storage class. Even code that looks +// innocuous may use optional platform-specific instructions. For example, +// the fastest way to zero a region of memory might use optional instructions. +// +// One might think one could run a tool such as "objdump" to read flags from +// the libraries' headers, or perhaps disassemble each library to look for +// particular instructions. Unfortunately, the desired flags are not present +// in the headers, and disassembly can be prohibitively slow ("objdump -d" is +// very slow, for example). Moreover, a tool to examine the library may not +// be present on the system unless the user has installed special packages (for +// example, on Windows). +// +// Instead, we adopt a crude but straightforward solution: We require +// developers to use the macro TF_PLATFORM_STRINGS() in their library, to +// embed the compilation options as constant strings. The compiler's +// predefined macros pick which strings are included. We then search for the +// strings in the files, and then dlopen() only those libraries that have or +// lack strings as needed. +// +// We adopt the approach of placing in the binary a fairly raw copy of the +// predefined macros, rather than trying to interpret them in complex ways at +// compile time. This allows the loading binary to alter its interpretation of +// the strings without library developers having to recompile. + +#include + +#include +#include + +// Aside from the header guard, the internal macros defined here have the form: +// TF_PLAT_STR_* + +// If a macro is removed from the list of tested macros, the major version in +// the following version number should be incremented, and the minor version +// set to zero. Otherwise, if a macro is added to the list of tested macros, +// the minor number should be incremented. +#define TF_PLAT_STR_VERSION_ "1.0" + +// Prefix of each option string indicator in the binary. +// After the prefix, such strings have the form: +// [A-Za-z_0-9]= +// followed by a terminating nul. To simplify searching, this prefix is all +// ASCII, starts with a nul, and contains no character twice. +#define TF_PLAT_STR_MAGIC_PREFIX_ "\0S\\s\":^p*L}" + +// A helper macro for TF_PLAT_STR_AS_STR_(). +#define TF_PLAT_STR_STR_1_(x) #x + +// Yield a constant string corresponding to x, after macro expansion. +#define TF_PLAT_STR_AS_STR_(x) TF_PLAT_STR_STR_1_(x) + +// An empty definition to make lists more uniform. +#define TF_PLAT_STR_TERMINATOR_ + +// TF_PLAT_STR_(x) introduces a constant string indicating whether a +// particular compilation option has been turned on. +// +// In gcc and clang, we might imagine using something like +// #define TF_PLAT_STR_(x) \ +// (sizeof (#x) != sizeof (TF_PLAT_STR_AS_STR_ (x))? \ +// TF_PLAT_STR_MAGIC_PREFIX_ #x "=" TF_PLAT_STR_AS_STR_ (x) : \ +// TF_PLAT_STR_MAGIC_PREFIX_ #x "=0"), +// but some compilers (notably MSVC) place both "foo" and "bar" in the binary +// when presented with +// (true? "foo" : "bar") +// so we must use #if to select the strings we need, which is rather verbose. +#define TF_PLAT_STR_(x) TF_PLAT_STR_MAGIC_PREFIX_ #x "=" TF_PLAT_STR_AS_STR_(x) + +// Include the #if machinery that sets the macros used below. +// platform_strings_computed.h can be generated by filtering this header file +// through: +// awk ' +// header == "" { print; } +// /\*\// && header == "" { +// print "// Generated from platform_strings.h."; +// print ""; +// print "#ifndef TENSORFLOW_CORE_PLATFORM_PLATFORM_STRINGS_COMPUTED_H_"; +// print "#define TENSORFLOW_CORE_PLATFORM_PLATFORM_STRINGS_COMPUTED_H_"; +// print ""; +// header = 1; +// } +// /^#define TF_PLAT_STR_LIST_[a-zA-Z0-9_]*\(\) *\\$/ { active = 1; } +// /TF_PLAT_STR_TERMINATOR_/ { active = 0; } +// /^ *TF_PLAT_STR_[A-Za-z0-9_]* *\\$/ && active { +// x = $0; +// sub(/^ *TF_PLAT_STR_/, "", x); +// sub(/ *\\$/, "", x); +// printf ("#if defined(%s)\n", x); +// printf ("#define TF_PLAT_STR_%s TF_PLAT_STR_(%s)\n", x, x); +// printf ("#else\n"); +// printf ("#define TF_PLAT_STR_%s\n", x); +// printf ("#endif\n"); +// } +// END { +// print ""; +// print "#endif // TENSORFLOW_CORE_PLATFORM_PLATFORM_STRINGS_COMPUTED_H_"; +// }' +#include "tensorflow/core/platform/platform_strings_computed.h" + +// clang-format butchers the following lines. +// clang-format off + +// x86_64 and x86_32 optional features. +#define TF_PLAT_STR_LIST___x86_64__() \ + TF_PLAT_STR__M_IX86_FP \ + TF_PLAT_STR__NO_PREFETCHW \ + TF_PLAT_STR___3dNOW_A__ \ + TF_PLAT_STR___3dNOW__ \ + TF_PLAT_STR___ABM__ \ + TF_PLAT_STR___ADX__ \ + TF_PLAT_STR___AES__ \ + TF_PLAT_STR___AVX2__ \ + TF_PLAT_STR___AVX512BW__ \ + TF_PLAT_STR___AVX512CD__ \ + TF_PLAT_STR___AVX512DQ__ \ + TF_PLAT_STR___AVX512ER__ \ + TF_PLAT_STR___AVX512F__ \ + TF_PLAT_STR___AVX512IFMA__ \ + TF_PLAT_STR___AVX512PF__ \ + TF_PLAT_STR___AVX512VBMI__ \ + TF_PLAT_STR___AVX512VL__ \ + TF_PLAT_STR___AVX__ \ + TF_PLAT_STR___BMI2__ \ + TF_PLAT_STR___BMI__ \ + TF_PLAT_STR___CLFLUSHOPT__ \ + TF_PLAT_STR___CLZERO__ \ + TF_PLAT_STR___F16C__ \ + TF_PLAT_STR___FMA4__ \ + TF_PLAT_STR___FMA__ \ + TF_PLAT_STR___FP_FAST_FMA \ + TF_PLAT_STR___FP_FAST_FMAF \ + TF_PLAT_STR___FSGSBASE__ \ + TF_PLAT_STR___FXSR__ \ + TF_PLAT_STR___LWP__ \ + TF_PLAT_STR___LZCNT__ \ + TF_PLAT_STR___MMX__ \ + TF_PLAT_STR___MWAITX__ \ + TF_PLAT_STR___PCLMUL__ \ + TF_PLAT_STR___PKU__ \ + TF_PLAT_STR___POPCNT__ \ + TF_PLAT_STR___PRFCHW__ \ + TF_PLAT_STR___RDRND__ \ + TF_PLAT_STR___RDSEED__ \ + TF_PLAT_STR___RTM__ \ + TF_PLAT_STR___SHA__ \ + TF_PLAT_STR___SSE2_MATH__ \ + TF_PLAT_STR___SSE2__ \ + TF_PLAT_STR___SSE_MATH__ \ + TF_PLAT_STR___SSE__ \ + TF_PLAT_STR___SSE3__ \ + TF_PLAT_STR___SSE4A__ \ + TF_PLAT_STR___SSE4_1__ \ + TF_PLAT_STR___SSE4_2__ \ + TF_PLAT_STR___SSSE3__ \ + TF_PLAT_STR___TBM__ \ + TF_PLAT_STR___XOP__ \ + TF_PLAT_STR___XSAVEC__ \ + TF_PLAT_STR___XSAVEOPT__ \ + TF_PLAT_STR___XSAVES__ \ + TF_PLAT_STR___XSAVE__ \ + TF_PLAT_STR_TERMINATOR_ + +// PowerPC (64- and 32-bit) optional features. +#define TF_PLAT_STR_LIST___powerpc64__() \ + TF_PLAT_STR__SOFT_DOUBLE \ + TF_PLAT_STR__SOFT_FLOAT \ + TF_PLAT_STR___ALTIVEC__ \ + TF_PLAT_STR___APPLE_ALTIVEC__ \ + TF_PLAT_STR___CRYPTO__ \ + TF_PLAT_STR___FLOAT128_HARDWARE__ \ + TF_PLAT_STR___FLOAT128_TYPE__ \ + TF_PLAT_STR___FP_FAST_FMA \ + TF_PLAT_STR___FP_FAST_FMAF \ + TF_PLAT_STR___HTM__ \ + TF_PLAT_STR___NO_FPRS__ \ + TF_PLAT_STR___NO_LWSYNC__ \ + TF_PLAT_STR___POWER8_VECTOR__ \ + TF_PLAT_STR___POWER9_VECTOR__ \ + TF_PLAT_STR___PPC405__ \ + TF_PLAT_STR___QUAD_MEMORY_ATOMIC__ \ + TF_PLAT_STR___RECIPF__ \ + TF_PLAT_STR___RECIP_PRECISION__ \ + TF_PLAT_STR___RECIP__ \ + TF_PLAT_STR___RSQRTEF__ \ + TF_PLAT_STR___RSQRTE__ \ + TF_PLAT_STR___TM_FENCE__ \ + TF_PLAT_STR___UPPER_REGS_DF__ \ + TF_PLAT_STR___UPPER_REGS_SF__ \ + TF_PLAT_STR___VEC__ \ + TF_PLAT_STR___VSX__ \ + TF_PLAT_STR_TERMINATOR_ + +// aarch64 and 32-bit arm optional features +#define TF_PLAT_STR_LIST___aarch64__() \ + TF_PLAT_STR___ARM_ARCH \ + TF_PLAT_STR___ARM_FEATURE_CLZ \ + TF_PLAT_STR___ARM_FEATURE_CRC32 \ + TF_PLAT_STR___ARM_FEATURE_CRC32 \ + TF_PLAT_STR___ARM_FEATURE_CRYPTO \ + TF_PLAT_STR___ARM_FEATURE_DIRECTED_ROUNDING \ + TF_PLAT_STR___ARM_FEATURE_DSP \ + TF_PLAT_STR___ARM_FEATURE_FMA \ + TF_PLAT_STR___ARM_FEATURE_IDIV \ + TF_PLAT_STR___ARM_FEATURE_LDREX \ + TF_PLAT_STR___ARM_FEATURE_NUMERIC_MAXMIN \ + TF_PLAT_STR___ARM_FEATURE_QBIT \ + TF_PLAT_STR___ARM_FEATURE_QRDMX \ + TF_PLAT_STR___ARM_FEATURE_SAT \ + TF_PLAT_STR___ARM_FEATURE_SIMD32 \ + TF_PLAT_STR___ARM_FEATURE_UNALIGNED \ + TF_PLAT_STR___ARM_FP \ + TF_PLAT_STR___ARM_NEON_FP \ + TF_PLAT_STR___ARM_NEON__ \ + TF_PLAT_STR___ARM_WMMX \ + TF_PLAT_STR___IWMMXT2__ \ + TF_PLAT_STR___IWMMXT__ \ + TF_PLAT_STR___VFP_FP__ \ + TF_PLAT_STR_TERMINATOR_ + +// Generic features, including indication of architecture and OS. +// The _M_* macros are defined by Visual Studio. +// It doesn't define __LITTLE_ENDIAN__ or __BYTE_ORDER__; +// Windows is assumed to be little endian. +#define TF_PLAT_STR_LIST___generic__() \ + TF_PLAT_STR_TARGET_IPHONE_SIMULATOR \ + TF_PLAT_STR_TARGET_OS_IOS \ + TF_PLAT_STR_TARGET_OS_IPHONE \ + TF_PLAT_STR__MSC_VER \ + TF_PLAT_STR__M_ARM \ + TF_PLAT_STR__M_ARM64 \ + TF_PLAT_STR__M_ARM_ARMV7VE \ + TF_PLAT_STR__M_ARM_FP \ + TF_PLAT_STR__M_IX86 \ + TF_PLAT_STR__M_X64 \ + TF_PLAT_STR__WIN32 \ + TF_PLAT_STR__WIN64 \ + TF_PLAT_STR___ANDROID__ \ + TF_PLAT_STR___APPLE__ \ + TF_PLAT_STR___BYTE_ORDER__ \ + TF_PLAT_STR___CYGWIN__ \ + TF_PLAT_STR___FreeBSD__ \ + TF_PLAT_STR___LITTLE_ENDIAN__ \ + TF_PLAT_STR___NetBSD__ \ + TF_PLAT_STR___OpenBSD__ \ + TF_PLAT_STR_____MSYS__ \ + TF_PLAT_STR___aarch64__ \ + TF_PLAT_STR___alpha__ \ + TF_PLAT_STR___arm__ \ + TF_PLAT_STR___i386__ \ + TF_PLAT_STR___i686__ \ + TF_PLAT_STR___ia64__ \ + TF_PLAT_STR___linux__ \ + TF_PLAT_STR___mips32__ \ + TF_PLAT_STR___mips64__ \ + TF_PLAT_STR___powerpc64__ \ + TF_PLAT_STR___powerpc__ \ + TF_PLAT_STR___riscv___ \ + TF_PLAT_STR___s390x__ \ + TF_PLAT_STR___sparc64__ \ + TF_PLAT_STR___sparc__ \ + TF_PLAT_STR___x86_64__ \ + TF_PLAT_STR_TERMINATOR_ + +#if !defined(__x86_64__) && !defined(_M_X64) && \ + !defined(__i386__) && !defined(_M_IX86) +#undef TF_PLAT_STR_LIST___x86_64__ +#define TF_PLAT_STR_LIST___x86_64__() +#endif +#if !defined(__powerpc64__) && !defined(__powerpc__) +#undef TF_PLAT_STR_LIST___powerpc64__ +#define TF_PLAT_STR_LIST___powerpc64__() +#endif +#if !defined(__aarch64__) && !defined(_M_ARM64) && \ + !defined(__arm__) && !defined(_M_ARM) +#undef TF_PLAT_STR_LIST___aarch64__ +#define TF_PLAT_STR_LIST___aarch64__() +#endif + +// Macro to be used in each dynamically loadable library. +// +// The BSS global variable tf_cpu_option_global and the class +// instance tf_cpu_option_avoid_omit_class are needed to prevent +// compilers/linkers such as clang from omitting the static variable +// tf_cpu_option[], which would otherwise appear to be unused. We cannot make +// tf_cpu_option[] global, because we then might get multiply-defined symbols +// if TF_PLAT_STR() is used twice in the same library. +// (tf_cpu_option_global doesn't see such errors because it is +// defined in BSS, so multiple definitions are combined by the linker.) gcc's +// __attribute__((used)) is insufficient because it seems to be ignored by +// linkers. +#define TF_PLATFORM_STRINGS() \ + static const char tf_cpu_option[] = \ + TF_PLAT_STR_MAGIC_PREFIX_ "TF_PLAT_STR_VERSION=" TF_PLAT_STR_VERSION_ \ + TF_PLAT_STR_LIST___x86_64__() \ + TF_PLAT_STR_LIST___powerpc64__() \ + TF_PLAT_STR_LIST___aarch64__() \ + TF_PLAT_STR_LIST___generic__() \ + ; \ + const char *tf_cpu_option_global; \ + namespace { \ + class TFCPUOptionHelper { \ + public: \ + TFCPUOptionHelper() { \ + /* Compilers/linkers remove unused variables aggressively. The */ \ + /* following gyrations subvert most such optimizations. */ \ + tf_cpu_option_global = tf_cpu_option; \ + /* Nothing is printed because the string starts with a nul. */ \ + printf("%s", tf_cpu_option); \ + } \ + } tf_cpu_option_avoid_omit_class; \ + } /* anonymous namespace */ +// clang-format on + +namespace tensorflow { + +class Status; + +// Retrieves the platform strings from the file at the given path and appends +// them to the given vector. If the returned int is non-zero, an error occurred +// reading the file and vector may or may not be modified. The returned error +// code is suitable for use with strerror(). +int GetPlatformStrings(const std::string& path, + std::vector* found); + +} // namespace tensorflow + +#endif // TENSORFLOW_CORE_PLATFORM_PLATFORM_STRINGS_H_ diff --git a/tensorflow/core/platform/platform_strings_computed.h b/tensorflow/core/platform/platform_strings_computed.h new file mode 100644 index 00000000000..6a17f3bfc3a --- /dev/null +++ b/tensorflow/core/platform/platform_strings_computed.h @@ -0,0 +1,735 @@ +/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +// Generated from platform_strings.h. + +#ifndef TENSORFLOW_CORE_PLATFORM_PLATFORM_STRINGS_COMPUTED_H_ +#define TENSORFLOW_CORE_PLATFORM_PLATFORM_STRINGS_COMPUTED_H_ + +#if defined(_M_IX86_FP) +#define TF_PLAT_STR__M_IX86_FP TF_PLAT_STR_(_M_IX86_FP) +#else +#define TF_PLAT_STR__M_IX86_FP +#endif +#if defined(_NO_PREFETCHW) +#define TF_PLAT_STR__NO_PREFETCHW TF_PLAT_STR_(_NO_PREFETCHW) +#else +#define TF_PLAT_STR__NO_PREFETCHW +#endif +#if defined(__3dNOW_A__) +#define TF_PLAT_STR___3dNOW_A__ TF_PLAT_STR_(__3dNOW_A__) +#else +#define TF_PLAT_STR___3dNOW_A__ +#endif +#if defined(__3dNOW__) +#define TF_PLAT_STR___3dNOW__ TF_PLAT_STR_(__3dNOW__) +#else +#define TF_PLAT_STR___3dNOW__ +#endif +#if defined(__ABM__) +#define TF_PLAT_STR___ABM__ TF_PLAT_STR_(__ABM__) +#else +#define TF_PLAT_STR___ABM__ +#endif +#if defined(__ADX__) +#define TF_PLAT_STR___ADX__ TF_PLAT_STR_(__ADX__) +#else +#define TF_PLAT_STR___ADX__ +#endif +#if defined(__AES__) +#define TF_PLAT_STR___AES__ TF_PLAT_STR_(__AES__) +#else +#define TF_PLAT_STR___AES__ +#endif +#if defined(__AVX2__) +#define TF_PLAT_STR___AVX2__ TF_PLAT_STR_(__AVX2__) +#else +#define TF_PLAT_STR___AVX2__ +#endif +#if defined(__AVX512BW__) +#define TF_PLAT_STR___AVX512BW__ TF_PLAT_STR_(__AVX512BW__) +#else +#define TF_PLAT_STR___AVX512BW__ +#endif +#if defined(__AVX512CD__) +#define TF_PLAT_STR___AVX512CD__ TF_PLAT_STR_(__AVX512CD__) +#else +#define TF_PLAT_STR___AVX512CD__ +#endif +#if defined(__AVX512DQ__) +#define TF_PLAT_STR___AVX512DQ__ TF_PLAT_STR_(__AVX512DQ__) +#else +#define TF_PLAT_STR___AVX512DQ__ +#endif +#if defined(__AVX512ER__) +#define TF_PLAT_STR___AVX512ER__ TF_PLAT_STR_(__AVX512ER__) +#else +#define TF_PLAT_STR___AVX512ER__ +#endif +#if defined(__AVX512F__) +#define TF_PLAT_STR___AVX512F__ TF_PLAT_STR_(__AVX512F__) +#else +#define TF_PLAT_STR___AVX512F__ +#endif +#if defined(__AVX512IFMA__) +#define TF_PLAT_STR___AVX512IFMA__ TF_PLAT_STR_(__AVX512IFMA__) +#else +#define TF_PLAT_STR___AVX512IFMA__ +#endif +#if defined(__AVX512PF__) +#define TF_PLAT_STR___AVX512PF__ TF_PLAT_STR_(__AVX512PF__) +#else +#define TF_PLAT_STR___AVX512PF__ +#endif +#if defined(__AVX512VBMI__) +#define TF_PLAT_STR___AVX512VBMI__ TF_PLAT_STR_(__AVX512VBMI__) +#else +#define TF_PLAT_STR___AVX512VBMI__ +#endif +#if defined(__AVX512VL__) +#define TF_PLAT_STR___AVX512VL__ TF_PLAT_STR_(__AVX512VL__) +#else +#define TF_PLAT_STR___AVX512VL__ +#endif +#if defined(__AVX__) +#define TF_PLAT_STR___AVX__ TF_PLAT_STR_(__AVX__) +#else +#define TF_PLAT_STR___AVX__ +#endif +#if defined(__BMI2__) +#define TF_PLAT_STR___BMI2__ TF_PLAT_STR_(__BMI2__) +#else +#define TF_PLAT_STR___BMI2__ +#endif +#if defined(__BMI__) +#define TF_PLAT_STR___BMI__ TF_PLAT_STR_(__BMI__) +#else +#define TF_PLAT_STR___BMI__ +#endif +#if defined(__CLFLUSHOPT__) +#define TF_PLAT_STR___CLFLUSHOPT__ TF_PLAT_STR_(__CLFLUSHOPT__) +#else +#define TF_PLAT_STR___CLFLUSHOPT__ +#endif +#if defined(__CLZERO__) +#define TF_PLAT_STR___CLZERO__ TF_PLAT_STR_(__CLZERO__) +#else +#define TF_PLAT_STR___CLZERO__ +#endif +#if defined(__F16C__) +#define TF_PLAT_STR___F16C__ TF_PLAT_STR_(__F16C__) +#else +#define TF_PLAT_STR___F16C__ +#endif +#if defined(__FMA4__) +#define TF_PLAT_STR___FMA4__ TF_PLAT_STR_(__FMA4__) +#else +#define TF_PLAT_STR___FMA4__ +#endif +#if defined(__FMA__) +#define TF_PLAT_STR___FMA__ TF_PLAT_STR_(__FMA__) +#else +#define TF_PLAT_STR___FMA__ +#endif +#if defined(__FP_FAST_FMA) +#define TF_PLAT_STR___FP_FAST_FMA TF_PLAT_STR_(__FP_FAST_FMA) +#else +#define TF_PLAT_STR___FP_FAST_FMA +#endif +#if defined(__FP_FAST_FMAF) +#define TF_PLAT_STR___FP_FAST_FMAF TF_PLAT_STR_(__FP_FAST_FMAF) +#else +#define TF_PLAT_STR___FP_FAST_FMAF +#endif +#if defined(__FSGSBASE__) +#define TF_PLAT_STR___FSGSBASE__ TF_PLAT_STR_(__FSGSBASE__) +#else +#define TF_PLAT_STR___FSGSBASE__ +#endif +#if defined(__FXSR__) +#define TF_PLAT_STR___FXSR__ TF_PLAT_STR_(__FXSR__) +#else +#define TF_PLAT_STR___FXSR__ +#endif +#if defined(__LWP__) +#define TF_PLAT_STR___LWP__ TF_PLAT_STR_(__LWP__) +#else +#define TF_PLAT_STR___LWP__ +#endif +#if defined(__LZCNT__) +#define TF_PLAT_STR___LZCNT__ TF_PLAT_STR_(__LZCNT__) +#else +#define TF_PLAT_STR___LZCNT__ +#endif +#if defined(__MMX__) +#define TF_PLAT_STR___MMX__ TF_PLAT_STR_(__MMX__) +#else +#define TF_PLAT_STR___MMX__ +#endif +#if defined(__MWAITX__) +#define TF_PLAT_STR___MWAITX__ TF_PLAT_STR_(__MWAITX__) +#else +#define TF_PLAT_STR___MWAITX__ +#endif +#if defined(__PCLMUL__) +#define TF_PLAT_STR___PCLMUL__ TF_PLAT_STR_(__PCLMUL__) +#else +#define TF_PLAT_STR___PCLMUL__ +#endif +#if defined(__PKU__) +#define TF_PLAT_STR___PKU__ TF_PLAT_STR_(__PKU__) +#else +#define TF_PLAT_STR___PKU__ +#endif +#if defined(__POPCNT__) +#define TF_PLAT_STR___POPCNT__ TF_PLAT_STR_(__POPCNT__) +#else +#define TF_PLAT_STR___POPCNT__ +#endif +#if defined(__PRFCHW__) +#define TF_PLAT_STR___PRFCHW__ TF_PLAT_STR_(__PRFCHW__) +#else +#define TF_PLAT_STR___PRFCHW__ +#endif +#if defined(__RDRND__) +#define TF_PLAT_STR___RDRND__ TF_PLAT_STR_(__RDRND__) +#else +#define TF_PLAT_STR___RDRND__ +#endif +#if defined(__RDSEED__) +#define TF_PLAT_STR___RDSEED__ TF_PLAT_STR_(__RDSEED__) +#else +#define TF_PLAT_STR___RDSEED__ +#endif +#if defined(__RTM__) +#define TF_PLAT_STR___RTM__ TF_PLAT_STR_(__RTM__) +#else +#define TF_PLAT_STR___RTM__ +#endif +#if defined(__SHA__) +#define TF_PLAT_STR___SHA__ TF_PLAT_STR_(__SHA__) +#else +#define TF_PLAT_STR___SHA__ +#endif +#if defined(__SSE2_MATH__) +#define TF_PLAT_STR___SSE2_MATH__ TF_PLAT_STR_(__SSE2_MATH__) +#else +#define TF_PLAT_STR___SSE2_MATH__ +#endif +#if defined(__SSE2__) +#define TF_PLAT_STR___SSE2__ TF_PLAT_STR_(__SSE2__) +#else +#define TF_PLAT_STR___SSE2__ +#endif +#if defined(__SSE_MATH__) +#define TF_PLAT_STR___SSE_MATH__ TF_PLAT_STR_(__SSE_MATH__) +#else +#define TF_PLAT_STR___SSE_MATH__ +#endif +#if defined(__SSE__) +#define TF_PLAT_STR___SSE__ TF_PLAT_STR_(__SSE__) +#else +#define TF_PLAT_STR___SSE__ +#endif +#if defined(__SSE3__) +#define TF_PLAT_STR___SSE3__ TF_PLAT_STR_(__SSE3__) +#else +#define TF_PLAT_STR___SSE3__ +#endif +#if defined(__SSE4A__) +#define TF_PLAT_STR___SSE4A__ TF_PLAT_STR_(__SSE4A__) +#else +#define TF_PLAT_STR___SSE4A__ +#endif +#if defined(__SSE4_1__) +#define TF_PLAT_STR___SSE4_1__ TF_PLAT_STR_(__SSE4_1__) +#else +#define TF_PLAT_STR___SSE4_1__ +#endif +#if defined(__SSE4_2__) +#define TF_PLAT_STR___SSE4_2__ TF_PLAT_STR_(__SSE4_2__) +#else +#define TF_PLAT_STR___SSE4_2__ +#endif +#if defined(__SSSE3__) +#define TF_PLAT_STR___SSSE3__ TF_PLAT_STR_(__SSSE3__) +#else +#define TF_PLAT_STR___SSSE3__ +#endif +#if defined(__TBM__) +#define TF_PLAT_STR___TBM__ TF_PLAT_STR_(__TBM__) +#else +#define TF_PLAT_STR___TBM__ +#endif +#if defined(__XOP__) +#define TF_PLAT_STR___XOP__ TF_PLAT_STR_(__XOP__) +#else +#define TF_PLAT_STR___XOP__ +#endif +#if defined(__XSAVEC__) +#define TF_PLAT_STR___XSAVEC__ TF_PLAT_STR_(__XSAVEC__) +#else +#define TF_PLAT_STR___XSAVEC__ +#endif +#if defined(__XSAVEOPT__) +#define TF_PLAT_STR___XSAVEOPT__ TF_PLAT_STR_(__XSAVEOPT__) +#else +#define TF_PLAT_STR___XSAVEOPT__ +#endif +#if defined(__XSAVES__) +#define TF_PLAT_STR___XSAVES__ TF_PLAT_STR_(__XSAVES__) +#else +#define TF_PLAT_STR___XSAVES__ +#endif +#if defined(__XSAVE__) +#define TF_PLAT_STR___XSAVE__ TF_PLAT_STR_(__XSAVE__) +#else +#define TF_PLAT_STR___XSAVE__ +#endif +#if defined(_SOFT_DOUBLE) +#define TF_PLAT_STR__SOFT_DOUBLE TF_PLAT_STR_(_SOFT_DOUBLE) +#else +#define TF_PLAT_STR__SOFT_DOUBLE +#endif +#if defined(_SOFT_FLOAT) +#define TF_PLAT_STR__SOFT_FLOAT TF_PLAT_STR_(_SOFT_FLOAT) +#else +#define TF_PLAT_STR__SOFT_FLOAT +#endif +#if defined(__ALTIVEC__) +#define TF_PLAT_STR___ALTIVEC__ TF_PLAT_STR_(__ALTIVEC__) +#else +#define TF_PLAT_STR___ALTIVEC__ +#endif +#if defined(__APPLE_ALTIVEC__) +#define TF_PLAT_STR___APPLE_ALTIVEC__ TF_PLAT_STR_(__APPLE_ALTIVEC__) +#else +#define TF_PLAT_STR___APPLE_ALTIVEC__ +#endif +#if defined(__CRYPTO__) +#define TF_PLAT_STR___CRYPTO__ TF_PLAT_STR_(__CRYPTO__) +#else +#define TF_PLAT_STR___CRYPTO__ +#endif +#if defined(__FLOAT128_HARDWARE__) +#define TF_PLAT_STR___FLOAT128_HARDWARE__ TF_PLAT_STR_(__FLOAT128_HARDWARE__) +#else +#define TF_PLAT_STR___FLOAT128_HARDWARE__ +#endif +#if defined(__FLOAT128_TYPE__) +#define TF_PLAT_STR___FLOAT128_TYPE__ TF_PLAT_STR_(__FLOAT128_TYPE__) +#else +#define TF_PLAT_STR___FLOAT128_TYPE__ +#endif +#if defined(__FP_FAST_FMA) +#define TF_PLAT_STR___FP_FAST_FMA TF_PLAT_STR_(__FP_FAST_FMA) +#else +#define TF_PLAT_STR___FP_FAST_FMA +#endif +#if defined(__FP_FAST_FMAF) +#define TF_PLAT_STR___FP_FAST_FMAF TF_PLAT_STR_(__FP_FAST_FMAF) +#else +#define TF_PLAT_STR___FP_FAST_FMAF +#endif +#if defined(__HTM__) +#define TF_PLAT_STR___HTM__ TF_PLAT_STR_(__HTM__) +#else +#define TF_PLAT_STR___HTM__ +#endif +#if defined(__NO_FPRS__) +#define TF_PLAT_STR___NO_FPRS__ TF_PLAT_STR_(__NO_FPRS__) +#else +#define TF_PLAT_STR___NO_FPRS__ +#endif +#if defined(__NO_LWSYNC__) +#define TF_PLAT_STR___NO_LWSYNC__ TF_PLAT_STR_(__NO_LWSYNC__) +#else +#define TF_PLAT_STR___NO_LWSYNC__ +#endif +#if defined(__POWER8_VECTOR__) +#define TF_PLAT_STR___POWER8_VECTOR__ TF_PLAT_STR_(__POWER8_VECTOR__) +#else +#define TF_PLAT_STR___POWER8_VECTOR__ +#endif +#if defined(__POWER9_VECTOR__) +#define TF_PLAT_STR___POWER9_VECTOR__ TF_PLAT_STR_(__POWER9_VECTOR__) +#else +#define TF_PLAT_STR___POWER9_VECTOR__ +#endif +#if defined(__PPC405__) +#define TF_PLAT_STR___PPC405__ TF_PLAT_STR_(__PPC405__) +#else +#define TF_PLAT_STR___PPC405__ +#endif +#if defined(__QUAD_MEMORY_ATOMIC__) +#define TF_PLAT_STR___QUAD_MEMORY_ATOMIC__ TF_PLAT_STR_(__QUAD_MEMORY_ATOMIC__) +#else +#define TF_PLAT_STR___QUAD_MEMORY_ATOMIC__ +#endif +#if defined(__RECIPF__) +#define TF_PLAT_STR___RECIPF__ TF_PLAT_STR_(__RECIPF__) +#else +#define TF_PLAT_STR___RECIPF__ +#endif +#if defined(__RECIP_PRECISION__) +#define TF_PLAT_STR___RECIP_PRECISION__ TF_PLAT_STR_(__RECIP_PRECISION__) +#else +#define TF_PLAT_STR___RECIP_PRECISION__ +#endif +#if defined(__RECIP__) +#define TF_PLAT_STR___RECIP__ TF_PLAT_STR_(__RECIP__) +#else +#define TF_PLAT_STR___RECIP__ +#endif +#if defined(__RSQRTEF__) +#define TF_PLAT_STR___RSQRTEF__ TF_PLAT_STR_(__RSQRTEF__) +#else +#define TF_PLAT_STR___RSQRTEF__ +#endif +#if defined(__RSQRTE__) +#define TF_PLAT_STR___RSQRTE__ TF_PLAT_STR_(__RSQRTE__) +#else +#define TF_PLAT_STR___RSQRTE__ +#endif +#if defined(__TM_FENCE__) +#define TF_PLAT_STR___TM_FENCE__ TF_PLAT_STR_(__TM_FENCE__) +#else +#define TF_PLAT_STR___TM_FENCE__ +#endif +#if defined(__UPPER_REGS_DF__) +#define TF_PLAT_STR___UPPER_REGS_DF__ TF_PLAT_STR_(__UPPER_REGS_DF__) +#else +#define TF_PLAT_STR___UPPER_REGS_DF__ +#endif +#if defined(__UPPER_REGS_SF__) +#define TF_PLAT_STR___UPPER_REGS_SF__ TF_PLAT_STR_(__UPPER_REGS_SF__) +#else +#define TF_PLAT_STR___UPPER_REGS_SF__ +#endif +#if defined(__VEC__) +#define TF_PLAT_STR___VEC__ TF_PLAT_STR_(__VEC__) +#else +#define TF_PLAT_STR___VEC__ +#endif +#if defined(__VSX__) +#define TF_PLAT_STR___VSX__ TF_PLAT_STR_(__VSX__) +#else +#define TF_PLAT_STR___VSX__ +#endif +#if defined(__ARM_ARCH) +#define TF_PLAT_STR___ARM_ARCH TF_PLAT_STR_(__ARM_ARCH) +#else +#define TF_PLAT_STR___ARM_ARCH +#endif +#if defined(__ARM_FEATURE_CLZ) +#define TF_PLAT_STR___ARM_FEATURE_CLZ TF_PLAT_STR_(__ARM_FEATURE_CLZ) +#else +#define TF_PLAT_STR___ARM_FEATURE_CLZ +#endif +#if defined(__ARM_FEATURE_CRC32) +#define TF_PLAT_STR___ARM_FEATURE_CRC32 TF_PLAT_STR_(__ARM_FEATURE_CRC32) +#else +#define TF_PLAT_STR___ARM_FEATURE_CRC32 +#endif +#if defined(__ARM_FEATURE_CRC32) +#define TF_PLAT_STR___ARM_FEATURE_CRC32 TF_PLAT_STR_(__ARM_FEATURE_CRC32) +#else +#define TF_PLAT_STR___ARM_FEATURE_CRC32 +#endif +#if defined(__ARM_FEATURE_CRYPTO) +#define TF_PLAT_STR___ARM_FEATURE_CRYPTO TF_PLAT_STR_(__ARM_FEATURE_CRYPTO) +#else +#define TF_PLAT_STR___ARM_FEATURE_CRYPTO +#endif +#if defined(__ARM_FEATURE_DIRECTED_ROUNDING) +#define TF_PLAT_STR___ARM_FEATURE_DIRECTED_ROUNDING \ + TF_PLAT_STR_(__ARM_FEATURE_DIRECTED_ROUNDING) +#else +#define TF_PLAT_STR___ARM_FEATURE_DIRECTED_ROUNDING +#endif +#if defined(__ARM_FEATURE_DSP) +#define TF_PLAT_STR___ARM_FEATURE_DSP TF_PLAT_STR_(__ARM_FEATURE_DSP) +#else +#define TF_PLAT_STR___ARM_FEATURE_DSP +#endif +#if defined(__ARM_FEATURE_FMA) +#define TF_PLAT_STR___ARM_FEATURE_FMA TF_PLAT_STR_(__ARM_FEATURE_FMA) +#else +#define TF_PLAT_STR___ARM_FEATURE_FMA +#endif +#if defined(__ARM_FEATURE_IDIV) +#define TF_PLAT_STR___ARM_FEATURE_IDIV TF_PLAT_STR_(__ARM_FEATURE_IDIV) +#else +#define TF_PLAT_STR___ARM_FEATURE_IDIV +#endif +#if defined(__ARM_FEATURE_LDREX) +#define TF_PLAT_STR___ARM_FEATURE_LDREX TF_PLAT_STR_(__ARM_FEATURE_LDREX) +#else +#define TF_PLAT_STR___ARM_FEATURE_LDREX +#endif +#if defined(__ARM_FEATURE_NUMERIC_MAXMIN) +#define TF_PLAT_STR___ARM_FEATURE_NUMERIC_MAXMIN \ + TF_PLAT_STR_(__ARM_FEATURE_NUMERIC_MAXMIN) +#else +#define TF_PLAT_STR___ARM_FEATURE_NUMERIC_MAXMIN +#endif +#if defined(__ARM_FEATURE_QBIT) +#define TF_PLAT_STR___ARM_FEATURE_QBIT TF_PLAT_STR_(__ARM_FEATURE_QBIT) +#else +#define TF_PLAT_STR___ARM_FEATURE_QBIT +#endif +#if defined(__ARM_FEATURE_QRDMX) +#define TF_PLAT_STR___ARM_FEATURE_QRDMX TF_PLAT_STR_(__ARM_FEATURE_QRDMX) +#else +#define TF_PLAT_STR___ARM_FEATURE_QRDMX +#endif +#if defined(__ARM_FEATURE_SAT) +#define TF_PLAT_STR___ARM_FEATURE_SAT TF_PLAT_STR_(__ARM_FEATURE_SAT) +#else +#define TF_PLAT_STR___ARM_FEATURE_SAT +#endif +#if defined(__ARM_FEATURE_SIMD32) +#define TF_PLAT_STR___ARM_FEATURE_SIMD32 TF_PLAT_STR_(__ARM_FEATURE_SIMD32) +#else +#define TF_PLAT_STR___ARM_FEATURE_SIMD32 +#endif +#if defined(__ARM_FEATURE_UNALIGNED) +#define TF_PLAT_STR___ARM_FEATURE_UNALIGNED \ + TF_PLAT_STR_(__ARM_FEATURE_UNALIGNED) +#else +#define TF_PLAT_STR___ARM_FEATURE_UNALIGNED +#endif +#if defined(__ARM_FP) +#define TF_PLAT_STR___ARM_FP TF_PLAT_STR_(__ARM_FP) +#else +#define TF_PLAT_STR___ARM_FP +#endif +#if defined(__ARM_NEON_FP) +#define TF_PLAT_STR___ARM_NEON_FP TF_PLAT_STR_(__ARM_NEON_FP) +#else +#define TF_PLAT_STR___ARM_NEON_FP +#endif +#if defined(__ARM_NEON__) +#define TF_PLAT_STR___ARM_NEON__ TF_PLAT_STR_(__ARM_NEON__) +#else +#define TF_PLAT_STR___ARM_NEON__ +#endif +#if defined(__ARM_WMMX) +#define TF_PLAT_STR___ARM_WMMX TF_PLAT_STR_(__ARM_WMMX) +#else +#define TF_PLAT_STR___ARM_WMMX +#endif +#if defined(__IWMMXT2__) +#define TF_PLAT_STR___IWMMXT2__ TF_PLAT_STR_(__IWMMXT2__) +#else +#define TF_PLAT_STR___IWMMXT2__ +#endif +#if defined(__IWMMXT__) +#define TF_PLAT_STR___IWMMXT__ TF_PLAT_STR_(__IWMMXT__) +#else +#define TF_PLAT_STR___IWMMXT__ +#endif +#if defined(__VFP_FP__) +#define TF_PLAT_STR___VFP_FP__ TF_PLAT_STR_(__VFP_FP__) +#else +#define TF_PLAT_STR___VFP_FP__ +#endif +#if defined(TARGET_IPHONE_SIMULATOR) +#define TF_PLAT_STR_TARGET_IPHONE_SIMULATOR \ + TF_PLAT_STR_(TARGET_IPHONE_SIMULATOR) +#else +#define TF_PLAT_STR_TARGET_IPHONE_SIMULATOR +#endif +#if defined(TARGET_OS_IOS) +#define TF_PLAT_STR_TARGET_OS_IOS TF_PLAT_STR_(TARGET_OS_IOS) +#else +#define TF_PLAT_STR_TARGET_OS_IOS +#endif +#if defined(TARGET_OS_IPHONE) +#define TF_PLAT_STR_TARGET_OS_IPHONE TF_PLAT_STR_(TARGET_OS_IPHONE) +#else +#define TF_PLAT_STR_TARGET_OS_IPHONE +#endif +#if defined(_MSC_VER) +#define TF_PLAT_STR__MSC_VER TF_PLAT_STR_(_MSC_VER) +#else +#define TF_PLAT_STR__MSC_VER +#endif +#if defined(_M_ARM) +#define TF_PLAT_STR__M_ARM TF_PLAT_STR_(_M_ARM) +#else +#define TF_PLAT_STR__M_ARM +#endif +#if defined(_M_ARM64) +#define TF_PLAT_STR__M_ARM64 TF_PLAT_STR_(_M_ARM64) +#else +#define TF_PLAT_STR__M_ARM64 +#endif +#if defined(_M_ARM_ARMV7VE) +#define TF_PLAT_STR__M_ARM_ARMV7VE TF_PLAT_STR_(_M_ARM_ARMV7VE) +#else +#define TF_PLAT_STR__M_ARM_ARMV7VE +#endif +#if defined(_M_ARM_FP) +#define TF_PLAT_STR__M_ARM_FP TF_PLAT_STR_(_M_ARM_FP) +#else +#define TF_PLAT_STR__M_ARM_FP +#endif +#if defined(_M_IX86) +#define TF_PLAT_STR__M_IX86 TF_PLAT_STR_(_M_IX86) +#else +#define TF_PLAT_STR__M_IX86 +#endif +#if defined(_M_X64) +#define TF_PLAT_STR__M_X64 TF_PLAT_STR_(_M_X64) +#else +#define TF_PLAT_STR__M_X64 +#endif +#if defined(_WIN32) +#define TF_PLAT_STR__WIN32 TF_PLAT_STR_(_WIN32) +#else +#define TF_PLAT_STR__WIN32 +#endif +#if defined(_WIN64) +#define TF_PLAT_STR__WIN64 TF_PLAT_STR_(_WIN64) +#else +#define TF_PLAT_STR__WIN64 +#endif +#if defined(__ANDROID__) +#define TF_PLAT_STR___ANDROID__ TF_PLAT_STR_(__ANDROID__) +#else +#define TF_PLAT_STR___ANDROID__ +#endif +#if defined(__APPLE__) +#define TF_PLAT_STR___APPLE__ TF_PLAT_STR_(__APPLE__) +#else +#define TF_PLAT_STR___APPLE__ +#endif +#if defined(__BYTE_ORDER__) +#define TF_PLAT_STR___BYTE_ORDER__ TF_PLAT_STR_(__BYTE_ORDER__) +#else +#define TF_PLAT_STR___BYTE_ORDER__ +#endif +#if defined(__CYGWIN__) +#define TF_PLAT_STR___CYGWIN__ TF_PLAT_STR_(__CYGWIN__) +#else +#define TF_PLAT_STR___CYGWIN__ +#endif +#if defined(__FreeBSD__) +#define TF_PLAT_STR___FreeBSD__ TF_PLAT_STR_(__FreeBSD__) +#else +#define TF_PLAT_STR___FreeBSD__ +#endif +#if defined(__LITTLE_ENDIAN__) +#define TF_PLAT_STR___LITTLE_ENDIAN__ TF_PLAT_STR_(__LITTLE_ENDIAN__) +#else +#define TF_PLAT_STR___LITTLE_ENDIAN__ +#endif +#if defined(__NetBSD__) +#define TF_PLAT_STR___NetBSD__ TF_PLAT_STR_(__NetBSD__) +#else +#define TF_PLAT_STR___NetBSD__ +#endif +#if defined(__OpenBSD__) +#define TF_PLAT_STR___OpenBSD__ TF_PLAT_STR_(__OpenBSD__) +#else +#define TF_PLAT_STR___OpenBSD__ +#endif +#if defined(____MSYS__) +#define TF_PLAT_STR_____MSYS__ TF_PLAT_STR_(____MSYS__) +#else +#define TF_PLAT_STR_____MSYS__ +#endif +#if defined(__aarch64__) +#define TF_PLAT_STR___aarch64__ TF_PLAT_STR_(__aarch64__) +#else +#define TF_PLAT_STR___aarch64__ +#endif +#if defined(__alpha__) +#define TF_PLAT_STR___alpha__ TF_PLAT_STR_(__alpha__) +#else +#define TF_PLAT_STR___alpha__ +#endif +#if defined(__arm__) +#define TF_PLAT_STR___arm__ TF_PLAT_STR_(__arm__) +#else +#define TF_PLAT_STR___arm__ +#endif +#if defined(__i386__) +#define TF_PLAT_STR___i386__ TF_PLAT_STR_(__i386__) +#else +#define TF_PLAT_STR___i386__ +#endif +#if defined(__i686__) +#define TF_PLAT_STR___i686__ TF_PLAT_STR_(__i686__) +#else +#define TF_PLAT_STR___i686__ +#endif +#if defined(__ia64__) +#define TF_PLAT_STR___ia64__ TF_PLAT_STR_(__ia64__) +#else +#define TF_PLAT_STR___ia64__ +#endif +#if defined(__linux__) +#define TF_PLAT_STR___linux__ TF_PLAT_STR_(__linux__) +#else +#define TF_PLAT_STR___linux__ +#endif +#if defined(__mips32__) +#define TF_PLAT_STR___mips32__ TF_PLAT_STR_(__mips32__) +#else +#define TF_PLAT_STR___mips32__ +#endif +#if defined(__mips64__) +#define TF_PLAT_STR___mips64__ TF_PLAT_STR_(__mips64__) +#else +#define TF_PLAT_STR___mips64__ +#endif +#if defined(__powerpc64__) +#define TF_PLAT_STR___powerpc64__ TF_PLAT_STR_(__powerpc64__) +#else +#define TF_PLAT_STR___powerpc64__ +#endif +#if defined(__powerpc__) +#define TF_PLAT_STR___powerpc__ TF_PLAT_STR_(__powerpc__) +#else +#define TF_PLAT_STR___powerpc__ +#endif +#if defined(__riscv___) +#define TF_PLAT_STR___riscv___ TF_PLAT_STR_(__riscv___) +#else +#define TF_PLAT_STR___riscv___ +#endif +#if defined(__s390x__) +#define TF_PLAT_STR___s390x__ TF_PLAT_STR_(__s390x__) +#else +#define TF_PLAT_STR___s390x__ +#endif +#if defined(__sparc64__) +#define TF_PLAT_STR___sparc64__ TF_PLAT_STR_(__sparc64__) +#else +#define TF_PLAT_STR___sparc64__ +#endif +#if defined(__sparc__) +#define TF_PLAT_STR___sparc__ TF_PLAT_STR_(__sparc__) +#else +#define TF_PLAT_STR___sparc__ +#endif +#if defined(__x86_64__) +#define TF_PLAT_STR___x86_64__ TF_PLAT_STR_(__x86_64__) +#else +#define TF_PLAT_STR___x86_64__ +#endif + +#endif // TENSORFLOW_CORE_PLATFORM_PLATFORM_STRINGS_COMPUTED_H_ diff --git a/tensorflow/core/platform/platform_strings_test.cc b/tensorflow/core/platform/platform_strings_test.cc new file mode 100644 index 00000000000..5251f10d412 --- /dev/null +++ b/tensorflow/core/platform/platform_strings_test.cc @@ -0,0 +1,146 @@ +/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +// Test for the platform_strings.h header file. + +#include +#include +#include +#include + +#include +#include + +#include "tensorflow/core/lib/io/path.h" +#include "tensorflow/core/lib/strings/str_util.h" +#include "tensorflow/core/platform/env.h" +#include "tensorflow/core/platform/init_main.h" +#include "tensorflow/core/platform/logging.h" +#include "tensorflow/core/platform/platform_strings.h" + +// Embed the platform strings in this binary. +TF_PLATFORM_STRINGS() + +// A vector of strings. +typedef std::vector string_vec; + +// Append to *found the strings within the named file with the platform_strings +// magic prefix, and return true; or return false on error. + +// Print the platform strings embedded in the binary file_name and return 0, +// on on error return 2. +static int PrintStrings(const std::string file_name) { + int rc = 0; + string_vec str; + if (!tensorflow::GetPlatformStrings(file_name, &str)) { + for (int i = 0; i != str.size(); i++) { + printf("%s\n", str[i].c_str()); + } + } else { + perror(file_name.c_str()); + rc = 2; + } + return rc; +} + +// Return whether str[] conatins a string with prefix "macro_name="; if so, +// set *pvalue to the suffix. +static bool GetValue(const string_vec &str, const std::string ¯o_name, + std::string *pvalue) { + std::string nam_eq = macro_name + "="; + int i = 0; + while (i != str.size() && !tensorflow::str_util::StartsWith(str[i], nam_eq)) { + i++; + } + bool found = (i != str.size()); + if (found) { + *pvalue = str[i].substr(nam_eq.size()); + } + return found; +} + +// If macro_name[] is not equal to value[], check that str[] contains the +// string "macro_name=value". Otherwise, check that str[] does not contain any +// string starting with macro_name=". +static void CheckStr(const string_vec &str, const std::string ¯o_name, + const std::string &value) { + std::string value_from_str; + if (GetValue(str, macro_name, &value_from_str)) { + if (value != value_from_str) { + // Output everything found, to aid debugging. + LOG(ERROR) << "===== value=" << value + << " value_from_str=" << value_from_str; + for (int i = 0; i != str.size(); i++) { + LOG(ERROR) << "% " << str[i]; + } + LOG(ERROR) << "====="; + } + CHECK_EQ(value, value_from_str) << " " << macro_name << ": bad value"; + } else { + // If the string is not found, we expect value to be macro_name. + if (value != macro_name) { + // Output everything found, to aid debugging. + LOG(ERROR) << "===== value=" << value << " macro_name=" << macro_name; + for (int i = 0; i != str.size(); i++) { + LOG(ERROR) << "% " << str[i]; + } + LOG(ERROR) << "====="; + } + CHECK_EQ(value, macro_name) << " " << macro_name << ": not found in binary"; + } +} + +// Helper for AS_STR(), below, to perform macro expansion. +#define AS_STR_1_(x) #x + +// Yield x after macro expansion as a nul-terminated constant string. +#define AS_STR(x) AS_STR_1_(x) + +// Run the test, and return 0 on success, 2 otherwise. +static int RunTest(const std::string &binary_name) { + int rc = 0; + string_vec str; + + if (!tensorflow::GetPlatformStrings(binary_name, &str)) { + CheckStr(str, "__linux__", AS_STR(__linux__)); + CheckStr(str, "_WIN32", AS_STR(_WIN32)); + CheckStr(str, "__APPLE__", AS_STR(__APPLE__)); + CheckStr(str, "__x86_64__", AS_STR(__x86_64__)); + CheckStr(str, "__aarch64__", AS_STR(__aarch64__)); + CheckStr(str, "__powerpc64__", AS_STR(__powerpc64__)); + CheckStr(str, "TF_PLAT_STR_VERSION", TF_PLAT_STR_VERSION_); + } else { + perror(binary_name.c_str()); + rc = 2; + } + + return rc; +} + +int main(int argc, char *argv[]) { + tensorflow::Env *env = tensorflow::Env::Default(); + static const char usage[] = "usage: platform_strings_test [file...]"; + int rc = 0; + tensorflow::port::InitMain(usage, &argc, &argv); + if (argc == 1) { + printf("rc=%d\n", PrintStrings(env->GetExecutablePath())); + rc = RunTest(env->GetExecutablePath()); + } else { + for (int argn = 1; argn != argc; argn++) { + rc |= PrintStrings(argv[argn]); + } + } + return rc; +} diff --git a/tensorflow/core/platform/posix/posix_file_system.cc b/tensorflow/core/platform/posix/posix_file_system.cc index c7afab9583c..fc48cab5646 100644 --- a/tensorflow/core/platform/posix/posix_file_system.cc +++ b/tensorflow/core/platform/posix/posix_file_system.cc @@ -240,11 +240,14 @@ Status PosixFileSystem::DeleteFile(const string& fname) { } Status PosixFileSystem::CreateDir(const string& name) { - Status result; - if (mkdir(TranslateName(name).c_str(), 0755) != 0) { - result = IOError(name, errno); + string translated = TranslateName(name); + if (translated.empty()) { + return errors::AlreadyExists(name); } - return result; + if (mkdir(translated.c_str(), 0755) != 0) { + return IOError(name, errno); + } + return Status::OK(); } Status PosixFileSystem::DeleteDir(const string& name) { diff --git a/tensorflow/core/platform/regexp.h b/tensorflow/core/platform/regexp.h index a4eedf30454..ca9ca1e2442 100644 --- a/tensorflow/core/platform/regexp.h +++ b/tensorflow/core/platform/regexp.h @@ -16,6 +16,7 @@ limitations under the License. #ifndef TENSORFLOW_PLATFORM_REGEXP_H_ #define TENSORFLOW_PLATFORM_REGEXP_H_ +#include "absl/strings/string_view.h" #include "tensorflow/core/platform/platform.h" #include "tensorflow/core/platform/types.h" @@ -23,7 +24,7 @@ limitations under the License. defined(GOOGLE_RE2) #include "tensorflow/core/platform/google/build_config/re2.h" namespace tensorflow { -typedef ::StringPiece RegexpStringPiece; +typedef absl::string_view RegexpStringPiece; } // namespace tensorflow #else diff --git a/tensorflow/core/platform/windows/windows_file_system.cc b/tensorflow/core/platform/windows/windows_file_system.cc index 6cf79634d7a..993b9906b1c 100644 --- a/tensorflow/core/platform/windows/windows_file_system.cc +++ b/tensorflow/core/platform/windows/windows_file_system.cc @@ -439,6 +439,9 @@ Status WindowsFileSystem::DeleteFile(const string& fname) { Status WindowsFileSystem::CreateDir(const string& name) { Status result; std::wstring ws_name = Utf8ToWideChar(name); + if (ws_name.empty()) { + return errors::AlreadyExists(name); + } if (_wmkdir(ws_name.c_str()) != 0) { result = IOError("Failed to create a directory: " + name, errno); } diff --git a/tensorflow/core/profiler/internal/tfprof_code.cc b/tensorflow/core/profiler/internal/tfprof_code.cc index 744e1e95deb..0c26855a43e 100644 --- a/tensorflow/core/profiler/internal/tfprof_code.cc +++ b/tensorflow/core/profiler/internal/tfprof_code.cc @@ -183,7 +183,7 @@ class Samples { // This method adds the statistics of graph nodes created by the python // call. void Add(const CodeNode* node, const std::vector& location_ids) { - // displayed leaf might not be true leaf. Retrive the true leaves for + // displayed leaf might not be true leaf. Retrieve the true leaves for // stats. std::vector all_leaf = FetchAllLeaf(node); CHECK(!all_leaf.empty()) << node->name(); diff --git a/tensorflow/core/profiler/internal/tfprof_node.cc b/tensorflow/core/profiler/internal/tfprof_node.cc index 86cb20de7bb..8796234be0c 100644 --- a/tensorflow/core/profiler/internal/tfprof_node.cc +++ b/tensorflow/core/profiler/internal/tfprof_node.cc @@ -151,7 +151,7 @@ void ExecStep::AddMemoryStats(const string& dev, } // TODO(xpan): Make this more accurate: - // High level: Memory tracking is suspicous and requires large scale + // High level: Memory tracking is suspicious and requires large scale // clean up. // Investigte the memory usage difference between CPU/GPU with OpViewTest. // diff --git a/tensorflow/core/protobuf/config.proto b/tensorflow/core/protobuf/config.proto index 4689af06afe..b3dc5dccc02 100644 --- a/tensorflow/core/protobuf/config.proto +++ b/tensorflow/core/protobuf/config.proto @@ -291,6 +291,13 @@ message RPCOptions { // transport for client-master communication that avoids the RPC // stack. This option is primarily for used testing the RPC stack. bool use_rpc_for_inprocess_master = 1; + + // The compression algorithm to be used. One of "deflate", "gzip". + string compression_algorithm = 2; + + // If compression_algorithm is set, the compression level to be used. + // From 0 (no compression), up to 3. + int32 compression_level = 3; }; // Session configuration parameters. @@ -413,6 +420,11 @@ message ConfigProto { // Any positive value sets the max chunk size. 0 defaults to 4096. // Any negative value indicates no max, i.e. one chunk only. int32 recv_buf_max_chunk = 4; + + // If true, and supported by the platform, the runtime will attempt to + // use NUMA affinity where applicable. One consequence will be the + // existence of as many CPU devices as there are available NUMA nodes. + bool use_numa_affinity = 5; }; Experimental experimental = 16; diff --git a/tensorflow/core/protobuf/master.proto b/tensorflow/core/protobuf/master.proto index 03022875e64..c104463c51c 100644 --- a/tensorflow/core/protobuf/master.proto +++ b/tensorflow/core/protobuf/master.proto @@ -224,7 +224,7 @@ message CloseSessionResponse { message ResetRequest { // A list of container names, which may be empty. // - // If 'container' is not empty, releases resoures in the given + // If 'container' is not empty, releases resources in the given // containers in all devices. // // If 'container' is empty, releases resources in the default diff --git a/tensorflow/core/protobuf/rewriter_config.proto b/tensorflow/core/protobuf/rewriter_config.proto index d68f2735365..515d673828e 100644 --- a/tensorflow/core/protobuf/rewriter_config.proto +++ b/tensorflow/core/protobuf/rewriter_config.proto @@ -38,7 +38,7 @@ message RewriterConfig { } // Enum controlling the number of times to run optimizers. The default is to - // run them once. + // run them twice. enum NumIterationsType { DEFAULT_NUM_ITERS = 0; ONE = 1; diff --git a/tensorflow/core/util/dump_graph.cc b/tensorflow/core/util/dump_graph.cc new file mode 100644 index 00000000000..523d37ecc24 --- /dev/null +++ b/tensorflow/core/util/dump_graph.cc @@ -0,0 +1,131 @@ +/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +// Helper functions for dumping Graphs, GraphDefs, and FunctionDefs to files for +// debugging. + +#include "tensorflow/core/util/dump_graph.h" + +#include "absl/strings/str_cat.h" +#include "tensorflow/core/lib/strings/proto_serialization.h" +#include "tensorflow/core/platform/env.h" +#include "tensorflow/core/platform/mutex.h" + +namespace tensorflow { + +namespace { + +struct NameCounts { + mutex counts_mutex; + std::unordered_map counts; +}; + +string MakeUniqueFilename(string name) { + static NameCounts& instance = *new NameCounts; + + // Remove illegal characters from `name`. + for (int i = 0; i < name.size(); ++i) { + char ch = name[i]; + if (ch == '/' || ch == '[' || ch == ']' || ch == '*' || ch == '?') { + name[i] = '_'; + } + } + + int count; + { + mutex_lock lock(instance.counts_mutex); + count = instance.counts[name]++; + } + + string filename = name; + if (count > 0) { + absl::StrAppend(&filename, "_", count); + } + absl::StrAppend(&filename, ".pbtxt"); + return filename; +} + +#if defined(TENSORFLOW_LITE_PROTOS) +Status WriteToFile(const string& filepath, + const ::tensorflow::protobuf::MessageLite& proto) { + string s; + if (!SerializeToStringDeterministic(proto, &s)) { + return errors::Internal("Failed to serialize proto to string."); + } + return WriteStringToFile(Env::Default(), filepath, s); +} +#else +Status WriteToFile(const string& filepath, + const ::tensorflow::protobuf::Message& proto) { + return WriteTextProto(Env::Default(), filepath, proto); +} +#endif + +template +string WriteTextProtoToUniqueFile(Env* env, const string& name, + const char* proto_type, T& proto, + const string& dirname) { + const char* dir = nullptr; + if (!dirname.empty()) { + dir = dirname.c_str(); + } else { + dir = getenv("TF_DUMP_GRAPH_PREFIX"); + } + if (!dir) { + return "(TF_DUMP_GRAPH_PREFIX not specified)"; + } + Status status = env->RecursivelyCreateDir(dir); + if (!status.ok()) { + LOG(WARNING) << "Failed to create " << dir << " for dumping " << proto_type + << ": " << status; + return "(unavailable)"; + } + string filepath = absl::StrCat(dir, "/", MakeUniqueFilename(name)); + status = WriteToFile(filepath, proto); + if (!status.ok()) { + LOG(WARNING) << "Failed to dump " << proto_type << " to file: " << filepath + << " : " << status; + return "(unavailable)"; + } + LOG(INFO) << "Dumped " << proto_type << " to " << filepath; + return filepath; +} + +} // anonymous namespace + +string DumpGraphDefToFile(const string& name, GraphDef const& graph_def, + const string& dirname) { + return WriteTextProtoToUniqueFile(Env::Default(), name, "GraphDef", graph_def, + dirname); +} + +string DumpGraphToFile(const string& name, Graph const& graph, + const FunctionLibraryDefinition* flib_def, + const string& dirname) { + GraphDef graph_def; + graph.ToGraphDef(&graph_def); + if (flib_def) { + *graph_def.mutable_library() = flib_def->ToProto(); + } + return DumpGraphDefToFile(name, graph_def, dirname); +} + +string DumpFunctionDefToFile(const string& name, FunctionDef const& fdef, + const string& dirname) { + return WriteTextProtoToUniqueFile(Env::Default(), name, "FunctionDef", fdef, + dirname); +} + +} // namespace tensorflow diff --git a/tensorflow/core/util/dump_graph.h b/tensorflow/core/util/dump_graph.h new file mode 100644 index 00000000000..03dc807a2b3 --- /dev/null +++ b/tensorflow/core/util/dump_graph.h @@ -0,0 +1,52 @@ +/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +// Helper functions for dumping Graphs, GraphDefs, and FunctionDefs to files for +// debugging. + +#ifndef TENSORFLOW_CORE_UTIL_DUMP_GRAPH_H_ +#define TENSORFLOW_CORE_UTIL_DUMP_GRAPH_H_ + +#include "tensorflow/core/framework/function.h" +#include "tensorflow/core/framework/graph.pb.h" +#include "tensorflow/core/graph/graph.h" + +namespace tensorflow { + +// Dumps 'graph_def' to a file, as a GraphDef text proto. Returns the file name +// chosen. +// +// Automatically picks a file name. Prefixes 'name' with the value of the +// TF_DUMP_GRAPH_PREFIX environment variable if 'dirname' is empty, and suffixes +// 'name' with ".pbtxt" to form a name. If a graph has already been dumped by +// this process with the same name, suffixes with "_n.pbtxt", where 'n' is a +// sequence number. +string DumpGraphDefToFile(const string& name, GraphDef const& graph_def, + const string& dirname = ""); + +// Similar to DumpGraphDefToFile, but builds the GraphDef to dump from a 'graph' +// and an optional function library 'flib_def'. Returns the file name chosen. +string DumpGraphToFile(const string& name, Graph const& graph, + const FunctionLibraryDefinition* flib_def = nullptr, + const string& dirname = ""); + +// Similar to DumpGraphDefToFile, but dumps a function as a FunctionDef text +// proto. Returns the file name chosen. +string DumpFunctionDefToFile(const string& name, FunctionDef const& fdef, + const string& dirname = ""); + +} // namespace tensorflow + +#endif // TENSORFLOW_CORE_UTIL_DUMP_GRAPH_H_ diff --git a/tensorflow/core/util/dump_graph_test.cc b/tensorflow/core/util/dump_graph_test.cc new file mode 100644 index 00000000000..d01c1c5a029 --- /dev/null +++ b/tensorflow/core/util/dump_graph_test.cc @@ -0,0 +1,62 @@ +/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/core/util/dump_graph.h" +#include "tensorflow/core/graph/graph.h" +#include "tensorflow/core/graph/node_builder.h" +#include "tensorflow/core/lib/io/path.h" +#include "tensorflow/core/lib/strings/proto_serialization.h" +#include "tensorflow/core/platform/env.h" +#include "tensorflow/core/platform/test.h" + +namespace tensorflow { +namespace { + +TEST(DumpGraph, DumpGraphToFileSuccess) { + Graph graph(OpRegistry::Global()); + Node* node; + TF_CHECK_OK(NodeBuilder("A", "NoOp").Finalize(&graph, &node)); + + setenv("TF_DUMP_GRAPH_PREFIX", testing::TmpDir().c_str(), 1); + string ret = DumpGraphToFile("graph", graph); + EXPECT_EQ(ret, io::JoinPath(testing::TmpDir(), "graph.pbtxt")); + ret = DumpGraphToFile("graph", graph); + EXPECT_EQ(ret, io::JoinPath(testing::TmpDir(), "graph_1.pbtxt")); + + GraphDef gdef; + TF_CHECK_OK(ReadTextProto( + Env::Default(), io::JoinPath(testing::TmpDir(), "graph.pbtxt"), &gdef)); + string read, written; + gdef.AppendToString(&read); + graph.ToGraphDefDebug().AppendToString(&written); + EXPECT_EQ(read, written); +} + +TEST(DumpGraph, DumpGraphToFileNoEnvPrefix) { + Graph graph(OpRegistry::Global()); + unsetenv("TF_DUMP_GRAPH_PREFIX"); + string ret = DumpGraphToFile("graph", graph); + EXPECT_EQ(ret, "(TF_DUMP_GRAPH_PREFIX not specified)"); +} + +TEST(DumpGraph, DumpFunctionDefToFileSuccess) { + FunctionDef fdef; + setenv("TF_DUMP_GRAPH_PREFIX", testing::TmpDir().c_str(), 1); + string ret = DumpFunctionDefToFile("function", fdef); + EXPECT_EQ(ret, io::JoinPath(testing::TmpDir(), "function.pbtxt")); +} + +} // namespace +} // namespace tensorflow diff --git a/tensorflow/core/util/mkl_util.h b/tensorflow/core/util/mkl_util.h index b7a6e0b6902..928807458ac 100644 --- a/tensorflow/core/util/mkl_util.h +++ b/tensorflow/core/util/mkl_util.h @@ -1644,6 +1644,9 @@ class MklDnnData { cpu_engine_(e) {} ~MklDnnData() { + if (allocated_buffer_ != nullptr) { + cpu_allocator()->DeallocateRaw(allocated_buffer_); + } cpu_engine_ = nullptr; // We don't own this. delete (user_memory_); delete (reorder_memory_); diff --git a/tensorflow/core/util/permutation_input_iterator.h b/tensorflow/core/util/permutation_input_iterator.h index f6375b25157..649318ebf3b 100644 --- a/tensorflow/core/util/permutation_input_iterator.h +++ b/tensorflow/core/util/permutation_input_iterator.h @@ -13,8 +13,8 @@ See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ -#ifndef TENSORFLOW_UTIL_PERMUTATION_INPUT_ITERATOR_H_ -#define TENSORFLOW_UTIL_PERMUTATION_INPUT_ITERATOR_H_ +#ifndef TENSORFLOW_CORE_UTIL_PERMUTATION_INPUT_ITERATOR_H_ +#define TENSORFLOW_CORE_UTIL_PERMUTATION_INPUT_ITERATOR_H_ #include #include @@ -131,4 +131,4 @@ class PermutationInputIterator { } // end namespace tensorflow -#endif // TENSORFLOW_UTIL_PERMUTATION_INPUT_ITERATOR_H_ +#endif // TENSORFLOW_CORE_UTIL_PERMUTATION_INPUT_ITERATOR_H_ diff --git a/tensorflow/core/util/permutation_output_iterator.h b/tensorflow/core/util/permutation_output_iterator.h new file mode 100644 index 00000000000..638c0f45458 --- /dev/null +++ b/tensorflow/core/util/permutation_output_iterator.h @@ -0,0 +1,129 @@ +/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_CORE_UTIL_PERMUTATION_OUTPUT_ITERATOR_H_ +#define TENSORFLOW_CORE_UTIL_PERMUTATION_OUTPUT_ITERATOR_H_ + +#include +#include + +namespace tensorflow { + +template +class PermutationOutputIterator { + public: + // Required iterator traits + typedef PermutationOutputIterator self_type; ///< My own type + typedef OffsetT difference_type; ///< Type to express the result of + ///< subtracting one iterator from another + typedef ValueType + value_type; ///< The type of the element the iterator can point to + typedef ValueType* pointer; ///< The type of a pointer to an element the + ///< iterator can point to + typedef ValueType& reference; ///< The type of a reference to an element the + ///< iterator can point to + + typedef std::random_access_iterator_tag + iterator_category; ///< The iterator category + + private: + OutputIteratorT output_itr; + IndexIteratorT index_itr; + + public: + /// Constructor + __host__ __device__ __forceinline__ PermutationOutputIterator( + OutputIteratorT output_itr, ///< Input iterator to wrap + IndexIteratorT index_itr) ///< Conversion functor to wrap + : output_itr(output_itr), index_itr(index_itr) {} + + /// Postfix increment + __host__ __device__ __forceinline__ self_type operator++(int) { + self_type retval = *this; + index_itr++; + return retval; + } + + /// Prefix increment + __host__ __device__ __forceinline__ self_type operator++() { + index_itr++; + return *this; + } + + /// Indirection + __host__ __device__ __forceinline__ reference operator*() const { + return output_itr[*index_itr]; + } + + /// Addition + template + __host__ __device__ __forceinline__ self_type operator+(Distance n) const { + self_type retval(output_itr, index_itr + n); + return retval; + } + + /// Addition assignment + template + __host__ __device__ __forceinline__ self_type& operator+=(Distance n) { + index_itr += n; + return *this; + } + + /// Subtraction + template + __host__ __device__ __forceinline__ self_type operator-(Distance n) const { + self_type retval(output_itr, index_itr - n); + return retval; + } + + /// Subtraction assignment + template + __host__ __device__ __forceinline__ self_type& operator-=(Distance n) { + index_itr -= n; + return *this; + } + + /// Distance + __host__ __device__ __forceinline__ difference_type + operator-(self_type other) const { + return index_itr - other.index_itr; + } + + /// Array subscript + template + __host__ __device__ __forceinline__ reference operator[](Distance n) const { + return output_itr[index_itr[n]]; + } + + /// Equal to + __host__ __device__ __forceinline__ bool operator==(const self_type& rhs) { + return (index_itr == rhs.index_itr && output_itr == rhs.output_itr); + } + + /// Not equal to + __host__ __device__ __forceinline__ bool operator!=(const self_type& rhs) { + return !(*this == rhs); + } + + /// ostream operator + friend std::ostream& operator<<(std::ostream& os, const self_type& itr) { + return os; + } +}; + +} // end namespace tensorflow + +#endif // TENSORFLOW_CORE_UTIL_PERMUTATION_OUTPUT_ITERATOR_H_ diff --git a/tensorflow/core/util/sparse/sparse_tensor.h b/tensorflow/core/util/sparse/sparse_tensor.h index b9ca8ab395b..89c163aa513 100644 --- a/tensorflow/core/util/sparse/sparse_tensor.h +++ b/tensorflow/core/util/sparse/sparse_tensor.h @@ -238,15 +238,6 @@ class SparseTensor { static Status Split(const SparseTensor& tensor, const int split_dim, const int num_split, std::vector* result); - template - ABSL_DEPRECATED( - "Use the form of Split() that takes an output pointer and returns a " - "status instead.") - static std::vector Split(const SparseTensor& tensor, - const int split_dim, - const int num_split, - Status* status = nullptr); - // Slice() will slice the input SparseTensor into a SparseTensor based on // specified start and size. Both start and size are 1-D array with each // element of the array representing one dimension. The start is the start @@ -578,10 +569,9 @@ SparseTensor SparseTensor::Concat( } template -std::vector SparseTensor::Split(const SparseTensor& input_tensor, - const int split_dim, - const int num_split, - Status* status /* = nullptr */) { +Status SparseTensor::Split(const SparseTensor& input_tensor, + const int split_dim, const int num_split, + std::vector* result) { std::vector output_indices; std::vector output_values; std::vector output_shapes; @@ -601,17 +591,15 @@ std::vector SparseTensor::Split(const SparseTensor& input_tensor, const int split_dim_size = input_tensor.shape()[split_dim]; const int split_size = split_dim_size / num_split; - if (!(num_split > 0 && num_split <= split_dim_size) && status != nullptr) { - *status = Status(error::INVALID_ARGUMENT, - strings::StrCat("num_split must be in the interval (0, ", - split_dim_size, "]")); - return {}; + if (!(num_split > 0 && num_split <= split_dim_size)) { + return Status(error::INVALID_ARGUMENT, + strings::StrCat("num_split must be in the interval (0, ", + split_dim_size, "]")); } if (!(split_dim >= 0 && split_dim < num_dim)) { - *status = Status( + return Status( error::INVALID_ARGUMENT, strings::StrCat("num_dim must be in the interval [0, ", num_dim, ")")); - return {}; } const int residual = split_dim_size % num_split; @@ -649,28 +637,18 @@ std::vector SparseTensor::Split(const SparseTensor& input_tensor, } } - std::vector output_tensors; - output_tensors.reserve(num_split); + result->clear(); + result->reserve(num_split); for (int i = 0; i < num_split; ++i) { SparseTensor tensor; Status create_status = Create(output_indices[i], output_values[i], output_shapes[i], &tensor); - if (!create_status.ok() && status != nullptr) { - *status = create_status; - return {}; + if (!create_status.ok()) { + return create_status; } - output_tensors.push_back(std::move(tensor)); + result->push_back(std::move(tensor)); } - return output_tensors; -} - -template -Status SparseTensor::Split(const SparseTensor& input_tensor, - const int split_dim, const int num_split, - std::vector* result) { - Status status; - *result = Split(input_tensor, split_dim, num_split, &status); - return status; + return Status::OK(); } template diff --git a/tensorflow/core/util/stats_calculator.cc b/tensorflow/core/util/stats_calculator.cc index eb077546501..bce650f2456 100644 --- a/tensorflow/core/util/stats_calculator.cc +++ b/tensorflow/core/util/stats_calculator.cc @@ -53,7 +53,7 @@ std::string StatsCalculator::HeaderString(const std::string& title) const { << " ==============================" << std::endl; InitField(stream, 24) << "[node type]"; - InitField(stream, 9) << "[start]"; + InitField(stream, 17) << "[start]"; InitField(stream, 9) << "[first]"; InitField(stream, 9) << "[avg ms]"; InitField(stream, 8) << "[%]"; @@ -77,7 +77,7 @@ std::string StatsCalculator::ColumnString(const Detail& detail, std::stringstream stream; InitField(stream, 24) << detail.type; - InitField(stream, 9) << start_ms; + InitField(stream, 17) << start_ms; InitField(stream, 9) << first_time_ms; InitField(stream, 9) << avg_time_ms; InitField(stream, 7) << percentage << "%"; diff --git a/tensorflow/core/util/strided_slice_op.cc b/tensorflow/core/util/strided_slice_op.cc index ad8a44a5184..55688e58084 100644 --- a/tensorflow/core/util/strided_slice_op.cc +++ b/tensorflow/core/util/strided_slice_op.cc @@ -83,10 +83,17 @@ static Status TF_MUST_USE_RESULT BuildDenseSpec( { int full_index = 0; - const auto& strides_flat = sparse.strides_tensor.flat(); + const T* const strides_flat = sparse.strides_tensor.vec().data(); dense->begin_valid = sparse.begin_tensor != nullptr; dense->end_valid = sparse.end_tensor != nullptr; + const T* const begin_flat = sparse.begin_tensor != nullptr + ? sparse.begin_tensor->vec().data() + : nullptr; + const T* const end_flat = sparse.end_tensor != nullptr + ? sparse.end_tensor->vec().data() + : nullptr; + for (int i = 0; i < sparse.dims; i++) { if ((1 << i) & sparse.ellipsis_mask) { // Expand the ellipsis into the appropriate indices @@ -112,16 +119,14 @@ static Status TF_MUST_USE_RESULT BuildDenseSpec( } // Gather slicing spec into appropriate index - if (sparse.begin_tensor != nullptr) { - const auto& begin_flat = sparse.begin_tensor->flat(); - dense->begin[full_index] = internal::SubtleMustCopy(begin_flat(i)); + if (begin_flat != nullptr) { + dense->begin[full_index] = internal::SubtleMustCopy(begin_flat[i]); } - if (sparse.end_tensor != nullptr) { - const auto& end_flat = sparse.end_tensor->flat(); - dense->end[full_index] = internal::SubtleMustCopy(end_flat(i)); + if (end_flat != nullptr) { + dense->end[full_index] = internal::SubtleMustCopy(end_flat[i]); } dense->strides[full_index] = - internal::SubtleMustCopy(strides_flat(i)); + internal::SubtleMustCopy(strides_flat[i]); if (sparse.begin_mask & (1 << i)) { dense->begin_mask |= (1 << full_index); } diff --git a/tensorflow/core/util/tensor_bundle/tensor_bundle.cc b/tensorflow/core/util/tensor_bundle/tensor_bundle.cc index 2dcb57a1f9b..3709ee5ae30 100644 --- a/tensorflow/core/util/tensor_bundle/tensor_bundle.cc +++ b/tensorflow/core/util/tensor_bundle/tensor_bundle.cc @@ -785,7 +785,7 @@ Status BundleReader::GetBundleEntryProto(StringPiece key, TF_RETURN_IF_ERROR( ParseEntryProto(iter_->key(), iter_->value(), &entry_copy)); if (!TensorShape::IsValid(entry_copy.shape())) { - return errors::DataLoss("Invaid tensor shape: ", key, " ", + return errors::DataLoss("Invalid tensor shape: ", key, " ", ProtoShortDebugString(entry_copy.shape())); } @@ -895,7 +895,7 @@ Status BundleReader::ReadCurrent(Tensor* val) { BundleEntryProto entry; TF_RETURN_IF_ERROR(ParseEntryProto(iter_->key(), iter_->value(), &entry)); if (!TensorShape::IsValid(entry.shape())) { - return errors::DataLoss("Invaid tensor shape: ", iter_->key(), " ", + return errors::DataLoss("Invalid tensor shape: ", iter_->key(), " ", ProtoShortDebugString(entry.shape())); } diff --git a/tensorflow/core/util/tensor_format.h b/tensorflow/core/util/tensor_format.h index b0c349dd907..a296fb447e2 100644 --- a/tensorflow/core/util/tensor_format.h +++ b/tensorflow/core/util/tensor_format.h @@ -498,7 +498,8 @@ inline TensorShape ShapeFromFormat(TensorFormat format, int64 N, dim_sizes[GetTensorBatchDimIndex(dims, format)] = N; for (int dim = 0; static_cast(dim) < spatial.size(); dim++) { auto dim_size = spatial[dim]; - if (format == FORMAT_NHWC_VECT_W && dim == spatial.size() - 1) { + if (format == FORMAT_NHWC_VECT_W && + static_cast(dim) == spatial.size() - 1) { CHECK_EQ(0, dim_size % 4) << "FORMAT_NHWC_VECT_W requires W to be a multiple of 4, but W=" << dim_size; diff --git a/tensorflow/core/util/tensor_ops_util.h b/tensorflow/core/util/tensor_ops_util.h new file mode 100644 index 00000000000..615f088a9b9 --- /dev/null +++ b/tensorflow/core/util/tensor_ops_util.h @@ -0,0 +1,128 @@ +/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#ifndef TENSORFLOW_CORE_UTIL_TENSOR_OPS_UTIL_H_ +#define TENSORFLOW_CORE_UTIL_TENSOR_OPS_UTIL_H_ + +#define EIGEN_USE_THREADS + +#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor" +#include "tensorflow/core/framework/op_kernel.h" +#include "tensorflow/core/framework/register_types.h" +#include "tensorflow/core/framework/tensor.h" +#include "tensorflow/core/framework/types.h" +#include "tensorflow/core/framework/variant_op_registry.h" +#include "tensorflow/core/lib/core/status.h" + +namespace tensorflow { + +typedef Eigen::ThreadPoolDevice CPUDevice; +typedef Eigen::GpuDevice GPUDevice; + +template +Status ZerosLikeTensor(OpKernelContext* ctx, const Tensor& x, Tensor* out) { + AllocatorAttributes attr; + if (x.dtype() == DT_VARIANT) { + attr.set_on_host(true); + } + TF_RETURN_IF_ERROR(ctx->allocate_temp(x.dtype(), x.shape(), out, attr)); + + switch (out->dtype()) { +#define DTYPE_CASE(dtype) \ + case DataTypeToEnum::value: \ + /* TODO(skyewm): use SetZeroFunctor like in ZerosLikeOp? */ \ + out->flat().device(ctx->eigen_device()) = \ + out->flat().constant(dtype(0)); \ + break; + + TF_CALL_POD_TYPES(DTYPE_CASE) +#undef DTYPE_CASE + + case DT_INVALID: { + *out = Tensor(DT_INVALID); + break; + } + case DataTypeToEnum::value: { + Variant* out_variant = out->scalar().data(); + TF_RETURN_IF_ERROR( + UnaryOpVariant(ctx, ZEROS_LIKE_VARIANT_UNARY_OP, + x.scalar()(), out_variant)); + break; + } + default: + return errors::InvalidArgument( + "Trying to compute zeros_like for unsupported dtype ", + DataTypeString(out->dtype())); + } + return Status::OK(); +} + +template +Status BinaryAddTensors(OpKernelContext* ctx, const Tensor& a, const Tensor& b, + Tensor* out) { + if (a.dtype() == DT_INVALID) { + *out = b; + return Status::OK(); + } + if (b.dtype() == DT_INVALID) { + *out = a; + return Status::OK(); + } + if (a.dtype() != b.dtype()) { + return errors::InvalidArgument( + "Trying to add two tensors with incompatible element types. ", + "One is ", DataTypeString(a.dtype()), " and the other is ", + DataTypeString(b.dtype())); + } + if (a.shape() != b.shape()) { + // TODO(apassos) support broadcasting additions here? + return errors::InvalidArgument( + "Trying to add two tensors with incompatible element shapes. ", + "One is ", a.shape().DebugString(), " and the other is ", + b.shape().DebugString()); + } + + AllocatorAttributes attr; + if (a.dtype() == DT_VARIANT) { + attr.set_on_host(true); + } + TF_RETURN_IF_ERROR(ctx->allocate_temp(a.dtype(), a.shape(), out, attr)); + + switch (out->dtype()) { +#define DTYPE_CASE(dtype) \ + case DataTypeToEnum::value: \ + out->flat().device(ctx->eigen_device()) = \ + a.flat() + b.flat(); \ + break; + + TF_CALL_NUMBER_TYPES(DTYPE_CASE) +#undef DTYPE_CASE + + case DataTypeToEnum::value: { + Variant* out_variant = out->scalar().data(); + TF_RETURN_IF_ERROR(BinaryOpVariants( + ctx, ADD_VARIANT_BINARY_OP, a.scalar()(), + b.scalar()(), out_variant)); + break; + } + default: + return errors::InvalidArgument("Trying to add unsupported dtype ", + out->dtype()); + } + return Status::OK(); +} + +} // namespace tensorflow + +#endif // TENSORFLOW_CORE_UTIL_TENSOR_OPS_UTIL_H_ diff --git a/tensorflow/examples/adding_an_op/fact_test.py b/tensorflow/examples/adding_an_op/fact_test.py index 11163e7ba5c..46beaebe0cc 100644 --- a/tensorflow/examples/adding_an_op/fact_test.py +++ b/tensorflow/examples/adding_an_op/fact_test.py @@ -19,10 +19,12 @@ from __future__ import division from __future__ import print_function import tensorflow as tf +from tensorflow.python.framework import test_util class FactTest(tf.test.TestCase): + @test_util.run_deprecated_v1 def test(self): with self.cached_session(): print(tf.user_ops.my_fact().eval()) diff --git a/tensorflow/examples/adding_an_op/zero_out_1_test.py b/tensorflow/examples/adding_an_op/zero_out_1_test.py index 342d3a020cc..459ac2dc279 100644 --- a/tensorflow/examples/adding_an_op/zero_out_1_test.py +++ b/tensorflow/examples/adding_an_op/zero_out_1_test.py @@ -23,10 +23,12 @@ import os.path import tensorflow as tf from tensorflow.examples.adding_an_op import zero_out_op_1 +from tensorflow.python.framework import test_util class ZeroOut1Test(tf.test.TestCase): + @test_util.run_deprecated_v1 def test(self): with self.cached_session(): result = zero_out_op_1.zero_out([5, 4, 3, 2, 1]) diff --git a/tensorflow/examples/adding_an_op/zero_out_2_test.py b/tensorflow/examples/adding_an_op/zero_out_2_test.py index 45045978176..650fd9546b5 100644 --- a/tensorflow/examples/adding_an_op/zero_out_2_test.py +++ b/tensorflow/examples/adding_an_op/zero_out_2_test.py @@ -24,20 +24,24 @@ import tensorflow as tf from tensorflow.examples.adding_an_op import zero_out_grad_2 # pylint: disable=unused-import from tensorflow.examples.adding_an_op import zero_out_op_2 +from tensorflow.python.framework import test_util class ZeroOut2Test(tf.test.TestCase): + @test_util.run_deprecated_v1 def test(self): with self.cached_session(): result = zero_out_op_2.zero_out([5, 4, 3, 2, 1]) self.assertAllEqual(result.eval(), [5, 0, 0, 0, 0]) + @test_util.run_deprecated_v1 def test_2d(self): with self.cached_session(): result = zero_out_op_2.zero_out([[6, 5, 4], [3, 2, 1]]) self.assertAllEqual(result.eval(), [[6, 0, 0], [0, 0, 0]]) + @test_util.run_deprecated_v1 def test_grad(self): with self.cached_session(): shape = (5,) @@ -46,6 +50,7 @@ class ZeroOut2Test(tf.test.TestCase): err = tf.test.compute_gradient_error(x, shape, y, shape) self.assertLess(err, 1e-4) + @test_util.run_deprecated_v1 def test_grad_2d(self): with self.cached_session(): shape = (2, 3) diff --git a/tensorflow/examples/adding_an_op/zero_out_3_test.py b/tensorflow/examples/adding_an_op/zero_out_3_test.py index 15d62495aae..8cbe2b6793a 100644 --- a/tensorflow/examples/adding_an_op/zero_out_3_test.py +++ b/tensorflow/examples/adding_an_op/zero_out_3_test.py @@ -21,31 +21,36 @@ from __future__ import print_function import tensorflow as tf from tensorflow.examples.adding_an_op import zero_out_op_3 +from tensorflow.python.framework import test_util class ZeroOut3Test(tf.test.TestCase): + @test_util.run_deprecated_v1 def test(self): with self.cached_session(): result = zero_out_op_3.zero_out([5, 4, 3, 2, 1]) self.assertAllEqual(result.eval(), [5, 0, 0, 0, 0]) + @test_util.run_deprecated_v1 def testAttr(self): with self.cached_session(): result = zero_out_op_3.zero_out([5, 4, 3, 2, 1], preserve_index=3) self.assertAllEqual(result.eval(), [0, 0, 0, 2, 0]) + @test_util.run_deprecated_v1 def testNegative(self): with self.cached_session(): result = zero_out_op_3.zero_out([5, 4, 3, 2, 1], preserve_index=-1) with self.assertRaisesOpError("Need preserve_index >= 0, got -1"): - result.eval() + self.evaluate(result) + @test_util.run_deprecated_v1 def testLarge(self): with self.cached_session(): result = zero_out_op_3.zero_out([5, 4, 3, 2, 1], preserve_index=17) with self.assertRaisesOpError("preserve_index out of range"): - result.eval() + self.evaluate(result) if __name__ == "__main__": diff --git a/tensorflow/examples/autograph/integration_tests/BUILD b/tensorflow/examples/autograph/integration_tests/BUILD index d20c17b63b9..2a4a0f75e7a 100644 --- a/tensorflow/examples/autograph/integration_tests/BUILD +++ b/tensorflow/examples/autograph/integration_tests/BUILD @@ -22,7 +22,6 @@ py_test( "keras_test.py", ], srcs_version = "PY2AND3", - tags = ["no_windows"], deps = [ "//tensorflow:tensorflow_py", ], @@ -34,7 +33,6 @@ py_test( "list_literals_test.py", ], srcs_version = "PY2AND3", - tags = ["no_windows"], deps = [ "//tensorflow:tensorflow_py", ], diff --git a/tensorflow/examples/autograph/integration_tests/keras_test.py b/tensorflow/examples/autograph/integration_tests/keras_test.py index dca7c07b470..3fe33df920d 100644 --- a/tensorflow/examples/autograph/integration_tests/keras_test.py +++ b/tensorflow/examples/autograph/integration_tests/keras_test.py @@ -21,6 +21,7 @@ from __future__ import print_function import tensorflow as tf from tensorflow.python import autograph +from tensorflow.python.framework import test_util class MinimalKeras(tf.keras.Model): @@ -84,6 +85,7 @@ class KerasTest(tf.test.TestCase): model = ModelWithStaticConditional(True) self.assertEqual(model.call(), 25) + @test_util.run_deprecated_v1 def test_recursive_true(self): with self.assertRaisesRegexp(NotImplementedError, 'Object conversion is not yet supported.'): @@ -93,10 +95,10 @@ class KerasTest(tf.test.TestCase): init = tf.global_variables_initializer() with tf.Session() as sess: - sess.run(init) + self.evaluate(init) sample_input = tf.random_uniform((1, 10, 10, 1)) output = model(sample_input) # pylint: disable=not-callable - self.assertEqual(sess.run(output).shape, (1, 3)) + self.assertEqual(self.evaluate(output).shape, (1, 3)) if __name__ == '__main__': diff --git a/tensorflow/examples/autograph/integration_tests/list_literals_test.py b/tensorflow/examples/autograph/integration_tests/list_literals_test.py index 917f5ff9d84..e85d4abcfc9 100644 --- a/tensorflow/examples/autograph/integration_tests/list_literals_test.py +++ b/tensorflow/examples/autograph/integration_tests/list_literals_test.py @@ -34,7 +34,7 @@ class ListLiteralsTest(tf.test.TestCase): result = converted() with self.cached_session() as sess: - self.assertAllEqual(sess.run(result), [1, 2, 3]) + self.assertAllEqual(self.evaluate(result), [1, 2, 3]) if __name__ == '__main__': diff --git a/tensorflow/examples/get_started/regression/custom_regression.py b/tensorflow/examples/get_started/regression/custom_regression.py index 2e34362c5ce..7b7cbb78666 100644 --- a/tensorflow/examples/get_started/regression/custom_regression.py +++ b/tensorflow/examples/get_started/regression/custom_regression.py @@ -100,12 +100,11 @@ def main(argv): # that the examples are well mixed. train.shuffle(1000).batch(128) # Repeat forever - .repeat().make_one_shot_iterator().get_next()) + .repeat()) # Build the validation input_fn. def input_test(): - return (test.shuffle(1000).batch(128) - .make_one_shot_iterator().get_next()) + return test.shuffle(1000).batch(128) # The first way assigns a unique weight to each category. To do this you must # specify the category's vocabulary (values outside this specification will diff --git a/tensorflow/examples/get_started/regression/dnn_regression.py b/tensorflow/examples/get_started/regression/dnn_regression.py index 951c93b52e7..94669a5082b 100644 --- a/tensorflow/examples/get_started/regression/dnn_regression.py +++ b/tensorflow/examples/get_started/regression/dnn_regression.py @@ -45,12 +45,11 @@ def main(argv): # that the examples are well mixed. train.shuffle(1000).batch(128) # Repeat forever - .repeat().make_one_shot_iterator().get_next()) + .repeat()) # Build the validation input_fn. def input_test(): - return (test.shuffle(1000).batch(128) - .make_one_shot_iterator().get_next()) + return test.shuffle(1000).batch(128) # The first way assigns a unique weight to each category. To do this you must # specify the category's vocabulary (values outside this specification will diff --git a/tensorflow/examples/get_started/regression/linear_regression_categorical.py b/tensorflow/examples/get_started/regression/linear_regression_categorical.py index e2ad415fbcb..5312272a959 100644 --- a/tensorflow/examples/get_started/regression/linear_regression_categorical.py +++ b/tensorflow/examples/get_started/regression/linear_regression_categorical.py @@ -45,12 +45,11 @@ def main(argv): # that the examples are well mixed. train.shuffle(1000).batch(128) # Repeat forever - .repeat().make_one_shot_iterator().get_next()) + .repeat()) # Build the validation input_fn. def input_test(): - return (test.shuffle(1000).batch(128) - .make_one_shot_iterator().get_next()) + return test.shuffle(1000).batch(128) # The following code demonstrates two of the ways that `feature_columns` can # be used to build a model with categorical inputs. diff --git a/tensorflow/examples/how_tos/reading_data/fully_connected_reader.py b/tensorflow/examples/how_tos/reading_data/fully_connected_reader.py index 74022474486..5c52a2c8461 100644 --- a/tensorflow/examples/how_tos/reading_data/fully_connected_reader.py +++ b/tensorflow/examples/how_tos/reading_data/fully_connected_reader.py @@ -126,7 +126,7 @@ def inputs(train, batch_size, num_epochs): dataset = dataset.repeat(num_epochs) dataset = dataset.batch(batch_size) - iterator = dataset.make_one_shot_iterator() + iterator = tf.compat.v1.data.make_one_shot_iterator(dataset) return iterator.get_next() diff --git a/tensorflow/examples/learn/iris_custom_decay_dnn.py b/tensorflow/examples/learn/iris_custom_decay_dnn.py index 4a219694d10..73bf20fada4 100644 --- a/tensorflow/examples/learn/iris_custom_decay_dnn.py +++ b/tensorflow/examples/learn/iris_custom_decay_dnn.py @@ -76,12 +76,12 @@ def main(unused_argv): classifier = tf.estimator.Estimator(model_fn=my_model) # Train. - train_input_fn = tf.estimator.inputs.numpy_input_fn( + train_input_fn = tf.compat.v1.estimator.inputs.numpy_input_fn( x={X_FEATURE: x_train}, y=y_train, num_epochs=None, shuffle=True) classifier.train(input_fn=train_input_fn, steps=1000) # Predict. - test_input_fn = tf.estimator.inputs.numpy_input_fn( + test_input_fn = tf.compat.v1.estimator.inputs.numpy_input_fn( x={X_FEATURE: x_test}, y=y_test, num_epochs=1, shuffle=False) predictions = classifier.predict(input_fn=test_input_fn) y_predicted = np.array(list(p['class'] for p in predictions)) diff --git a/tensorflow/examples/learn/iris_custom_model.py b/tensorflow/examples/learn/iris_custom_model.py index c6bdb86ba52..bf34d72ba07 100644 --- a/tensorflow/examples/learn/iris_custom_model.py +++ b/tensorflow/examples/learn/iris_custom_model.py @@ -73,12 +73,12 @@ def main(unused_argv): classifier = tf.estimator.Estimator(model_fn=my_model) # Train. - train_input_fn = tf.estimator.inputs.numpy_input_fn( + train_input_fn = tf.compat.v1.estimator.inputs.numpy_input_fn( x={X_FEATURE: x_train}, y=y_train, num_epochs=None, shuffle=True) classifier.train(input_fn=train_input_fn, steps=1000) # Predict. - test_input_fn = tf.estimator.inputs.numpy_input_fn( + test_input_fn = tf.compat.v1.estimator.inputs.numpy_input_fn( x={X_FEATURE: x_test}, y=y_test, num_epochs=1, shuffle=False) predictions = classifier.predict(input_fn=test_input_fn) y_predicted = np.array(list(p['class'] for p in predictions)) diff --git a/tensorflow/examples/speech_commands/freeze_test.py b/tensorflow/examples/speech_commands/freeze_test.py index 0c7ca9bc011..9ed9050035b 100644 --- a/tensorflow/examples/speech_commands/freeze_test.py +++ b/tensorflow/examples/speech_commands/freeze_test.py @@ -19,11 +19,13 @@ from __future__ import division from __future__ import print_function from tensorflow.examples.speech_commands import freeze +from tensorflow.python.framework import test_util from tensorflow.python.platform import test class FreezeTest(test.TestCase): + @test_util.run_deprecated_v1 def testCreateInferenceGraphWithMfcc(self): with self.cached_session() as sess: freeze.create_inference_graph( @@ -43,6 +45,7 @@ class FreezeTest(test.TestCase): ops = [node.op for node in sess.graph_def.node] self.assertEqual(1, ops.count('Mfcc')) + @test_util.run_deprecated_v1 def testCreateInferenceGraphWithoutMfcc(self): with self.cached_session() as sess: freeze.create_inference_graph( @@ -62,6 +65,7 @@ class FreezeTest(test.TestCase): ops = [node.op for node in sess.graph_def.node] self.assertEqual(0, ops.count('Mfcc')) + @test_util.run_deprecated_v1 def testFeatureBinCount(self): with self.cached_session() as sess: freeze.create_inference_graph( diff --git a/tensorflow/examples/speech_commands/input_data_test.py b/tensorflow/examples/speech_commands/input_data_test.py index b766ba6de0d..9269bb6c0bc 100644 --- a/tensorflow/examples/speech_commands/input_data_test.py +++ b/tensorflow/examples/speech_commands/input_data_test.py @@ -26,6 +26,7 @@ import tensorflow as tf from tensorflow.contrib.framework.python.ops import audio_ops as contrib_audio from tensorflow.examples.speech_commands import input_data from tensorflow.examples.speech_commands import models +from tensorflow.python.framework import test_util from tensorflow.python.platform import test @@ -35,7 +36,7 @@ class InputDataTest(test.TestCase): with self.cached_session() as sess: sample_data = tf.zeros([32000, 2]) wav_encoder = contrib_audio.encode_wav(sample_data, 16000) - wav_data = sess.run(wav_encoder) + wav_data = self.evaluate(wav_encoder) return wav_data def _saveTestWavFile(self, filename, wav_data): @@ -96,6 +97,7 @@ class InputDataTest(test.TestCase): input_data.which_set("foo_nohash_0.wav", 10, 10), input_data.which_set("foo_nohash_1.wav", 10, 10)) + @test_util.run_deprecated_v1 def testPrepareDataIndex(self): tmp_dir = self.get_temp_dir() self._saveWavFolders(tmp_dir, ["a", "b", "c"], 100) @@ -125,6 +127,7 @@ class InputDataTest(test.TestCase): 10, self._model_settings(), tmp_dir) self.assertTrue("Expected to find" in str(e.exception)) + @test_util.run_deprecated_v1 def testPrepareBackgroundData(self): tmp_dir = self.get_temp_dir() background_dir = os.path.join(tmp_dir, "_background_noise_") @@ -156,6 +159,7 @@ class InputDataTest(test.TestCase): self.assertIsNotNone(loaded_data) self.assertEqual(16000, len(loaded_data)) + @test_util.run_deprecated_v1 def testPrepareProcessingGraph(self): tmp_dir = self.get_temp_dir() wav_dir = os.path.join(tmp_dir, "wavs") @@ -186,15 +190,19 @@ class InputDataTest(test.TestCase): self.assertIsNotNone(audio_processor.background_volume_placeholder_) self.assertIsNotNone(audio_processor.output_) + @test_util.run_deprecated_v1 def testGetDataAverage(self): self._runGetDataTest("average", 10) + @test_util.run_deprecated_v1 def testGetDataAverageLongWindow(self): self._runGetDataTest("average", 30) + @test_util.run_deprecated_v1 def testGetDataMfcc(self): self._runGetDataTest("mfcc", 30) + @test_util.run_deprecated_v1 def testGetUnprocessedData(self): tmp_dir = self.get_temp_dir() wav_dir = os.path.join(tmp_dir, "wavs") @@ -216,6 +224,7 @@ class InputDataTest(test.TestCase): self.assertEqual(10, len(result_data)) self.assertEqual(10, len(result_labels)) + @test_util.run_deprecated_v1 def testGetFeaturesForWav(self): tmp_dir = self.get_temp_dir() wav_dir = os.path.join(tmp_dir, "wavs") diff --git a/tensorflow/examples/speech_commands/label_wav_test.py b/tensorflow/examples/speech_commands/label_wav_test.py index f0af2a47987..77a88f98e16 100644 --- a/tensorflow/examples/speech_commands/label_wav_test.py +++ b/tensorflow/examples/speech_commands/label_wav_test.py @@ -33,7 +33,7 @@ class LabelWavTest(test.TestCase): with self.cached_session() as sess: sample_data = tf.zeros([1000, 2]) wav_encoder = contrib_audio.encode_wav(sample_data, 16000) - wav_data = sess.run(wav_encoder) + wav_data = self.evaluate(wav_encoder) return wav_data def _saveTestWavFile(self, filename, wav_data): diff --git a/tensorflow/examples/speech_commands/models_test.py b/tensorflow/examples/speech_commands/models_test.py index 04478c09626..cb9304eab8d 100644 --- a/tensorflow/examples/speech_commands/models_test.py +++ b/tensorflow/examples/speech_commands/models_test.py @@ -21,6 +21,7 @@ from __future__ import print_function import tensorflow as tf from tensorflow.examples.speech_commands import models +from tensorflow.python.framework import test_util from tensorflow.python.platform import test @@ -47,6 +48,7 @@ class ModelsTest(test.TestCase): feature_bin_count=40, preprocess="mfcc")) + @test_util.run_deprecated_v1 def testCreateModelConvTraining(self): model_settings = self._modelSettings() with self.cached_session() as sess: @@ -58,6 +60,7 @@ class ModelsTest(test.TestCase): self.assertIsNotNone(sess.graph.get_tensor_by_name(logits.name)) self.assertIsNotNone(sess.graph.get_tensor_by_name(dropout_prob.name)) + @test_util.run_deprecated_v1 def testCreateModelConvInference(self): model_settings = self._modelSettings() with self.cached_session() as sess: @@ -67,6 +70,7 @@ class ModelsTest(test.TestCase): self.assertIsNotNone(logits) self.assertIsNotNone(sess.graph.get_tensor_by_name(logits.name)) + @test_util.run_deprecated_v1 def testCreateModelLowLatencyConvTraining(self): model_settings = self._modelSettings() with self.cached_session() as sess: @@ -78,6 +82,7 @@ class ModelsTest(test.TestCase): self.assertIsNotNone(sess.graph.get_tensor_by_name(logits.name)) self.assertIsNotNone(sess.graph.get_tensor_by_name(dropout_prob.name)) + @test_util.run_deprecated_v1 def testCreateModelFullyConnectedTraining(self): model_settings = self._modelSettings() with self.cached_session() as sess: @@ -98,6 +103,7 @@ class ModelsTest(test.TestCase): "bad_architecture", True) self.assertTrue("not recognized" in str(e.exception)) + @test_util.run_deprecated_v1 def testCreateModelTinyConvTraining(self): model_settings = self._modelSettings() with self.cached_session() as sess: diff --git a/tensorflow/examples/speech_commands/wav_to_features_test.py b/tensorflow/examples/speech_commands/wav_to_features_test.py index 87f29876939..6234490b267 100644 --- a/tensorflow/examples/speech_commands/wav_to_features_test.py +++ b/tensorflow/examples/speech_commands/wav_to_features_test.py @@ -24,6 +24,7 @@ import tensorflow as tf from tensorflow.contrib.framework.python.ops import audio_ops as contrib_audio from tensorflow.examples.speech_commands import wav_to_features +from tensorflow.python.framework import test_util from tensorflow.python.platform import test @@ -33,7 +34,7 @@ class WavToFeaturesTest(test.TestCase): with self.cached_session() as sess: sample_data = tf.zeros([32000, 2]) wav_encoder = contrib_audio.encode_wav(sample_data, 16000) - wav_data = sess.run(wav_encoder) + wav_data = self.evaluate(wav_encoder) return wav_data def _saveTestWavFile(self, filename, wav_data): @@ -49,6 +50,7 @@ class WavToFeaturesTest(test.TestCase): file_path = os.path.join(dir_name, "some_audio_%d.wav" % i) self._saveTestWavFile(file_path, wav_data) + @test_util.run_deprecated_v1 def testWavToFeatures(self): tmp_dir = self.get_temp_dir() wav_dir = os.path.join(tmp_dir, "wavs") diff --git a/tensorflow/examples/tutorials/layers/cnn_mnist.py b/tensorflow/examples/tutorials/layers/cnn_mnist.py index 1e8d7d05e1c..670e929236f 100644 --- a/tensorflow/examples/tutorials/layers/cnn_mnist.py +++ b/tensorflow/examples/tutorials/layers/cnn_mnist.py @@ -134,7 +134,7 @@ def main(unused_argv): tensors=tensors_to_log, every_n_iter=50) # Train the model - train_input_fn = tf.estimator.inputs.numpy_input_fn( + train_input_fn = tf.compat.v1.estimator.inputs.numpy_input_fn( x={"x": train_data}, y=train_labels, batch_size=100, @@ -146,11 +146,8 @@ def main(unused_argv): hooks=[logging_hook]) # Evaluate the model and print results - eval_input_fn = tf.estimator.inputs.numpy_input_fn( - x={"x": eval_data}, - y=eval_labels, - num_epochs=1, - shuffle=False) + eval_input_fn = tf.compat.v1.estimator.inputs.numpy_input_fn( + x={"x": eval_data}, y=eval_labels, num_epochs=1, shuffle=False) eval_results = mnist_classifier.evaluate(input_fn=eval_input_fn) print(eval_results) diff --git a/tensorflow/go/README.md b/tensorflow/go/README.md index 3989f9b25a4..f53f6fc9894 100644 --- a/tensorflow/go/README.md +++ b/tensorflow/go/README.md @@ -23,7 +23,7 @@ from source. - [bazel](https://www.bazel.build/versions/master/docs/install.html) - Environment to build TensorFlow from source code - ([Linux of macOS](https://www.tensorflow.org/install/source)). + ([Linux or macOS](https://www.tensorflow.org/install/source)). If you don't need GPU support, then try the following: ```sh diff --git a/tensorflow/go/graph.go b/tensorflow/go/graph.go index 67e42aa961b..6ff41ca9169 100644 --- a/tensorflow/go/graph.go +++ b/tensorflow/go/graph.go @@ -112,9 +112,17 @@ func (g *Graph) ImportWithOptions(def []byte, options GraphImportOptions) error C.TF_ImportGraphDefOptionsSetPrefix(opts, cprefix) if len(options.Device) != 0 { - cdev := C.CString(options.Device) - defer C.free(unsafe.Pointer(cdev)) - C.TF_ImportGraphDefOptionsSetDefaultDevice(opts, cdev) + // TODO(ashankar): Remove this error and uncomment below + // when a release of the C library which includes + // https://github.com/tensorflow/tensorflow/commit/e0af5ac53e5a8ad9b07cdd5738c0a8e12f938c4e + // has been made. + // See https://github.com/tensorflow/tensorflow/issues/23257 + return fmt.Errorf("GraphImportOptions.Device is only supported with the TensorFlow C library versions after 1.12 (or built from master). See https://github.com/tensorflow/tensorflow/issues/23257") + /* + cdev := C.CString(options.Device) + defer C.free(unsafe.Pointer(cdev)) + C.TF_ImportGraphDefOptionsSetDefaultDevice(opts, cdev) + */ } buf := C.TF_NewBuffer() @@ -174,6 +182,68 @@ func (g *Graph) Operations() []Operation { return ops } +// AddGradients adds operations to compute the partial derivatives of the sum of tensors in y +// with respect to tensors in x, i.e., d(y[0] + y[1] + ...) / d x[0], d(y[0] + y[1] + ... ) / d x[1] etc. +// +// prefix, if non-empty, is the name prefix used for all operations added to the graph to compute +// these gradients. +func (g *Graph) AddGradients(prefix string, y []Output, x []Output, dx []Output) ([]Output, error) { + var ( + cprefix *C.char + + cy = make([]C.TF_Output, len(y)) + cx = make([]C.TF_Output, len(x)) + cdx = make([]C.TF_Output, len(dx)) + cdy = make([]C.TF_Output, len(x)) + + pcy *C.TF_Output + pcx *C.TF_Output + pcdx *C.TF_Output + pcdy *C.TF_Output + + status = newStatus() + ) + + if len(y) > 0 { + pcy = &cy[0] + for i, o := range y { + cy[i] = o.c() + } + } + if len(x) > 0 { + pcx = &cx[0] + for i, o := range x { + cx[i] = o.c() + } + pcdy = &cdy[0] + } + if len(dx) > 0 { + pcdx = &cdx[0] + for i, o := range dx { + cdx[i] = o.c() + } + } + + // If prefix is "", the C.TF_AddGradientsWithPrefix need cprefix to be nil but not "" + if len(prefix) != 0 { + cprefix = C.CString(prefix) + defer C.free(unsafe.Pointer(cprefix)) + } + + C.TF_AddGradientsWithPrefix(g.c, cprefix, pcy, C.int(len(y)), pcx, C.int(len(x)), pcdx, status.c, pcdy) + + if err := status.Err(); err != nil { + return nil, err + } + dy := make([]Output, len(x)) + for i, co := range cdy { + op := &Operation{co.oper, g} + dy[i] = Output{op, int(co.index)} + } + + return dy, nil +} + // OpSpec is the specification of an Operation to be added to a Graph // (using Graph.AddOperation). type OpSpec struct { diff --git a/tensorflow/go/graph_test.go b/tensorflow/go/graph_test.go index b8d65c54f69..067c7db5c3c 100644 --- a/tensorflow/go/graph_test.go +++ b/tensorflow/go/graph_test.go @@ -19,6 +19,7 @@ package tensorflow import ( "bytes" "fmt" + "strings" "testing" ) @@ -80,3 +81,260 @@ func TestGraphWriteToAndImport(t *testing.T) { t.Error(err) } } + +func TestGraphAddGradients(t *testing.T) { + g := NewGraph() + x1, err := Placeholder(g, "x1", Float) + if err != nil { + t.Fatal(err) + } + x2, err := Placeholder(g, "x2", Float) + if err != nil { + t.Fatal(err) + } + op0, err := g.AddOperation(OpSpec{ + Type: "Square", + Name: "y0", + Input: []Input{x1}, + }) + if err != nil { + t.Fatal(err) + } + y0 := op0.Output(0) + op1, err := g.AddOperation(OpSpec{ + Type: "Square", + Name: "y1", + Input: []Input{y0}, + }) + if err != nil { + t.Fatal(err) + } + y1 := op1.Output(0) + op2, err := g.AddOperation(OpSpec{ + Type: "AddN", + Input: []Input{OutputList([]Output{y0, x2})}, + }) + if err != nil { + t.Fatal(err) + } + y2 := op2.Output(0) + + grads0, err := g.AddGradients("", []Output{y1}, []Output{x1}, nil) + if err != nil { + t.Fatal(err) + } + if len(grads0) != 1 { + t.Fatal(len(grads0)) + } + if grads0[0].DataType() != Float { + t.Fatalf("Got DataType %v, wanted %v", grads0[0].DataType(), Float) + } + + grads1, err := g.AddGradients("", []Output{y2}, []Output{x1, x2}, nil) + if err != nil { + t.Fatal(err) + } + if len(grads1) != 2 { + t.Fatal(len(grads1)) + } + if grads1[0].DataType() != Float { + t.Fatalf("Got DataType %v, wanted %v", grads1[0].DataType(), Float) + } + if grads1[1].DataType() != Float { + t.Fatalf("Got DataType %v, wanted %v", grads1[1].DataType(), Float) + } + + sess, err := NewSession(g, nil) + if err != nil { + t.Fatal(err) + } + + c1, _ := NewTensor(float32(3.0)) + c2, _ := NewTensor(float32(2.0)) + outputs, err := sess.Run( + map[Output]*Tensor{x1: c1, x2: c2}, + []Output{grads0[0], grads1[0], grads1[1]}, + nil) + if err != nil { + t.Fatal(err) + } + if len(outputs) != 3 { + t.Fatal(len(outputs)) + } + if outputs[0].Value().(float32) != 108.0 { + t.Fatalf("Got %v, wanted float 108.0", outputs[0].Value()) + } + if outputs[1].Value().(float32) != 6.0 { + t.Fatalf("Got %v, wanted float 6.0", outputs[1].Value()) + } + if outputs[2].Value().(float32) != 1.0 { + t.Fatalf("Got %v, wanted float 1.0", outputs[2].Value()) + } +} + +func TestGraphAddGradientsSums(t *testing.T) { + g := NewGraph() + x, err := Placeholder(g, "x", Float) + if err != nil { + t.Fatal(err) + } + op0, err := g.AddOperation(OpSpec{ + Type: "Square", + Name: "y0", + Input: []Input{x}, + }) + if err != nil { + t.Fatal(err) + } + y0 := op0.Output(0) + op1, err := g.AddOperation(OpSpec{ + Type: "Square", + Name: "y1", + Input: []Input{y0}, + }) + y1 := op1.Output(0) + + grad, err := g.AddGradients("", []Output{y0, y1}, []Output{x}, nil) + if err != nil { + t.Fatal(err) + } + if len(grad) != 1 { + t.Fatal(len(grad)) + } + if grad[0].DataType() != Float { + t.Fatalf("Got DataType %v, wanted %v", grad[0].DataType(), Float) + } + + sess, err := NewSession(g, nil) + if err != nil { + t.Fatal(err) + } + + c, _ := NewTensor(float32(3.0)) + outputs, err := sess.Run( + map[Output]*Tensor{x: c}, + []Output{grad[0]}, + nil) + if err != nil { + t.Fatal(err) + } + if outputs[0].Value().(float32) != 114.0 { + t.Fatalf("Got %v, wanted float 114.0", outputs[0].Value()) + } +} + +func TestGraphAddGradientsWithInitialValues(t *testing.T) { + g := NewGraph() + x, err := Placeholder(g, "x", Float) + op0, err := g.AddOperation(OpSpec{ + Type: "Square", + Name: "y0", + Input: []Input{x}, + }) + if err != nil { + t.Fatal(err) + } + y0 := op0.Output(0) + op1, err := g.AddOperation(OpSpec{ + Type: "Square", + Name: "y1", + Input: []Input{y0}, + }) + if err != nil { + t.Fatal(err) + } + y1 := op1.Output(0) + + grads0, err := g.AddGradients("", []Output{y1}, []Output{y0}, nil) + if err != nil { + t.Fatal(err) + } + if len(grads0) != 1 { + t.Fatal(len(grads0)) + } + if grads0[0].DataType() != Float { + t.Fatalf("Got DataType %v, wanted %v", grads0[0].DataType(), Float) + } + + grads1, err := g.AddGradients("", []Output{y0}, []Output{x}, []Output{grads0[0]}) + if err != nil { + t.Fatal(err) + } + if len(grads1) != 1 { + t.Fatal(len(grads1)) + } + if grads1[0].DataType() != Float { + t.Fatalf("Got DataType %v, wanted %v", grads1[0].DataType(), Float) + } + + sess, err := NewSession(g, nil) + if err != nil { + t.Fatal(err) + } + + c, _ := NewTensor(float32(3.0)) + outputs, err := sess.Run( + map[Output]*Tensor{x: c}, + []Output{grads1[0]}, + nil) + if err != nil { + t.Fatal(err) + } + if outputs[0].Value().(float32) != 108.0 { + t.Fatalf("Got %v, wanted float 108.0", outputs[0].Value()) + } +} + +func TestGraphValidateGradientsNames(t *testing.T) { + g := NewGraph() + x, err := Placeholder(g, "x", Float) + if err != nil { + t.Fatal(err) + } + op0, err := g.AddOperation(OpSpec{ + Type: "Square", + Name: "y0", + Input: []Input{x}, + }) + if err != nil { + t.Fatal(err) + } + y0 := op0.Output(0) + + grads0, err := g.AddGradients("", []Output{y0}, []Output{x}, nil) + if err != nil { + t.Fatal(err) + } + if !strings.HasPrefix(grads0[0].Op.Name(), "gradients/") { + t.Fatalf("Got name %v, wanted started with gradients/", grads0[0].Op.Name()) + } + + grads1, err := g.AddGradients("", []Output{y0}, []Output{x}, nil) + if err != nil { + t.Fatal(err) + } + if !strings.HasPrefix(grads1[0].Op.Name(), "gradients_1/") { + t.Fatalf("Got name %v, wanted started with gradients_1/", grads1[0].Op.Name()) + } + + grads2, err := g.AddGradients("more_gradients", []Output{y0}, []Output{x}, nil) + if err != nil { + t.Fatal(err) + } + if !strings.HasPrefix(grads2[0].Op.Name(), "more_gradients/") { + t.Fatalf("Got name %v, wanted started with more_gradients/", grads2[0].Op.Name()) + } + + grads3, err := g.AddGradients("even_more_gradients", []Output{y0}, []Output{x}, nil) + if err != nil { + t.Fatal(err) + } + if !strings.HasPrefix(grads3[0].Op.Name(), "even_more_gradients/") { + t.Fatalf("Got name %v, wanted started with even_more_gradients/", grads3[0].Op.Name()) + } + + _, err = g.AddGradients("even_more_gradients", []Output{y0}, []Output{x}, nil) + if err == nil { + t.Error("AddGradients should have failed if gradients name is already existing") + } +} diff --git a/tensorflow/go/op/gradients.go b/tensorflow/go/op/gradients.go new file mode 100644 index 00000000000..9f892e1da6c --- /dev/null +++ b/tensorflow/go/op/gradients.go @@ -0,0 +1,50 @@ +/* +Copyright 2016 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package op + +import ( + "fmt" + + tf "github.com/tensorflow/tensorflow/tensorflow/go" +) + + +// Gradients adds gradients computation ops to the graph according to scope. +// +// Arguments: +// y: output of the function to derive +// x: inputs of the function for which partial derivatives are computed +// dx: if not null, the partial derivatives of some loss function L w.r.t. y +// +// return the partial derivatives +func Gradients(scope *Scope, y []tf.Output, x []tf.Output, dx ...tf.Output) (output []tf.Output) { + if len(scope.controlDependencies) > 0 { + scope.UpdateErr("Gradients", fmt.Errorf("Gradients does not currently support control dependencies (via Scope.WithControlDependencies).")) + return + } + if scope.device != "" { + scope.UpdateErr("Gradients", fmt.Errorf("Gradients does not currently support device annotations (via Scope.WithDevice).")) + return + } + + var err error + if output, err = scope.graph.AddGradients(scope.opName("Gradients"), y, x, dx); err != nil { + scope.UpdateErr("Gradients", err) + return + } + return output +} diff --git a/tensorflow/go/op/gradients_test.go b/tensorflow/go/op/gradients_test.go new file mode 100644 index 00000000000..3d1d57b77ea --- /dev/null +++ b/tensorflow/go/op/gradients_test.go @@ -0,0 +1,246 @@ +/* +Copyright 2016 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package op + +import ( + "strings" + "testing" + + tf "github.com/tensorflow/tensorflow/tensorflow/go" +) + +func TestAddGradients(t *testing.T) { + var ( + s = NewScope() + x1 = Placeholder(s.SubScope("x1"), tf.Float) + x2 = Placeholder(s.SubScope("x2"), tf.Float) + y0 = Square(s.SubScope("y0"), x1) + y1 = Square(s.SubScope("y1"), y0) + y2 = AddN(s.SubScope("y2"), []tf.Output{y0, x2}) + ) + + grads0 := Gradients(s, []tf.Output{y1}, []tf.Output{x1}) + if err := s.Err(); err != nil { + t.Fatal(err) + } + if len(grads0) != 1 { + t.Fatal(len(grads0)) + } + if grads0[0].DataType() != tf.Float { + t.Fatalf("Got DataType %v, wanted %v", grads0[0].DataType(), tf.Float) + } + + sub := s.SubScope("sub") + grads1 := Gradients(sub, []tf.Output{y2}, []tf.Output{x1, x2}) + if err := sub.Err(); err != nil { + t.Fatal(err) + } + if len(grads1) != 2 { + t.Fatal(len(grads1)) + } + if grads1[0].DataType() != tf.Float { + t.Fatalf("Got DataType %v, wanted %v", grads1[0].DataType(), tf.Float) + } + if grads1[1].DataType() != tf.Float { + t.Fatalf("Got DataType %v, wanted %v", grads1[1].DataType(), tf.Float) + } + + graph, err := sub.Finalize() + if err != nil { + t.Fatal(err) + } + sess, err := tf.NewSession(graph, nil) + if err != nil { + t.Fatal(err) + } + + c1, _ := tf.NewTensor(float32(3.0)) + c2, _ := tf.NewTensor(float32(3.0)) + outputs, err := sess.Run( + map[tf.Output]*tf.Tensor{x1: c1, x2: c2}, + []tf.Output{grads0[0], grads1[0], grads1[1]}, + nil) + if err != nil { + t.Fatal(err) + } + if len(outputs) != 3 { + t.Fatal(len(outputs)) + } + if outputs[0].Value().(float32) != 108.0 { + t.Fatalf("Got %v, wanted float 108.0", outputs[0].Value()) + } + if outputs[1].Value().(float32) != 6.0 { + t.Fatalf("Got %v, wanted float 6.0", outputs[1].Value()) + } + if outputs[2].Value().(float32) != 1.0 { + t.Fatalf("Got %v, wanted float 1.0", outputs[2].Value()) + } +} + +func TestAddGradientsSums(t *testing.T) { + var ( + s = NewScope() + x = Placeholder(s.SubScope("x"), tf.Float) + y0 = Square(s.SubScope("y0"), x) + y1 = Square(s.SubScope("y1"), y0) + ) + + grad := Gradients(s, []tf.Output{y0, y1}, []tf.Output{x}) + if err := s.Err(); err != nil { + t.Fatal(err) + } + if len(grad) != 1 { + t.Fatal(len(grad)) + } + if grad[0].DataType() != tf.Float { + t.Fatalf("Got DataType %v, wanted %v", grad[0].DataType(), tf.Float) + } + + graph, err := s.Finalize() + if err != nil { + t.Fatal(err) + } + sess, err := tf.NewSession(graph, nil) + if err != nil { + t.Fatal(err) + } + + c, _ := tf.NewTensor(float32(3.0)) + outputs, err := sess.Run( + map[tf.Output]*tf.Tensor{x: c}, + []tf.Output{grad[0]}, + nil) + if err != nil { + t.Fatal(err) + } + if outputs[0].Value().(float32) != 114.0 { + t.Fatalf("Got %v, wanted float 114.0", outputs[0].Value()) + } +} + +func TestAddGradientsWithInitialValues(t *testing.T) { + var ( + s = NewScope() + x = Placeholder(s.SubScope("x1"), tf.Float) + y0 = Square(s.SubScope("y0"), x) + y1 = Square(s.SubScope("y1"), y0) + ) + + grads0 := Gradients(s, []tf.Output{y1}, []tf.Output{y0}) + if err := s.Err(); err != nil { + t.Fatal(err) + } + if len(grads0) != 1 { + t.Fatal(len(grads0)) + } + if grads0[0].DataType() != tf.Float { + t.Fatalf("Got DataType %v, wanted %v", grads0[0].DataType(), tf.Float) + } + + sub := s.SubScope("sub") + grads1 := Gradients(sub, []tf.Output{y0}, []tf.Output{x}, grads0[0]) + if err := sub.Err(); err != nil { + t.Fatal(err) + } + if len(grads1) != 1 { + t.Fatal(len(grads1)) + } + if grads1[0].DataType() != tf.Float { + t.Fatalf("Got DataType %v, wanted %v", grads1[0].DataType(), tf.Float) + } + + graph, err := sub.Finalize() + if err != nil { + t.Fatal(err) + } + sess, err := tf.NewSession(graph, nil) + if err != nil { + t.Fatal(err) + } + + c, _ := tf.NewTensor(float32(3.0)) + outputs, err := sess.Run( + map[tf.Output]*tf.Tensor{x: c}, + []tf.Output{grads1[0]}, + nil) + if err != nil { + t.Fatal(err) + } + if outputs[0].Value().(float32) != 108.0 { + t.Fatalf("Got %v, wanted float 108.0", outputs[0].Value()) + } +} + +func TestValidateGradientsNames(t *testing.T) { + var ( + s = NewScope() + x = Placeholder(s.SubScope("x"), tf.Float) + y0 = Square(s.SubScope("y0"), x) + ) + + grads0 := Gradients(s, []tf.Output{y0}, []tf.Output{x}) + if err := s.Err(); err != nil { + t.Fatal(err) + } + if !strings.HasPrefix(grads0[0].Op.Name(), "Gradients/") { + t.Fatalf("Got name %v, wanted started with Gradients/", grads0[0].Op.Name()) + } + + sub := s.SubScope("sub") + grads1 := Gradients(sub, []tf.Output{y0}, []tf.Output{x}) + if err := s.Err(); err != nil { + t.Fatal(err) + } + if !strings.HasPrefix(grads1[0].Op.Name(), "sub/Gradients/") { + t.Fatalf("Got name %v, wanted started with sub/Gradients/", grads1[0].Op.Name()) + } + + Gradients(sub, []tf.Output{y0}, []tf.Output{x}) + if err := s.Err(); err == nil { + t.Error("Gradients should have failed if executed more than once for scope of the same namespace") + } +} + +func TestAddGradientsWithControlDependencies(t *testing.T) { + var ( + s = NewScope() + zero = Const(s.SubScope("zero"), int32(0)) + x = Placeholder(s.SubScope("x"), tf.Float) + y0 = Square(s.SubScope("y0"), x) + variable = VarHandleOp(s, tf.Int32, tf.ScalarShape()) + init = AssignVariableOp(s, variable, zero) + readDeps = []*tf.Operation{init} + ) + s = s.WithControlDependencies(readDeps...) + Gradients(s, []tf.Output{y0}, []tf.Output{x}) + if err := s.Err(); err == nil { + t.Error("Gradients should have failed when control dependencies are set") + } +} + +func TestAddGradientsWithDevice(t *testing.T) { + var ( + s = NewScope() + x = Placeholder(s.SubScope("x"), tf.Float) + y0 = Square(s.SubScope("y0"), x) + ) + s = s.WithDevice("/device:GPU:0") + Gradients(s, []tf.Output{y0}, []tf.Output{x}) + if err := s.Err(); err == nil { + t.Error("Gradients should have failed when device is set") + } +} diff --git a/tensorflow/go/op/wrappers.go b/tensorflow/go/op/wrappers.go index baf43f84f8e..5cd06299b96 100644 --- a/tensorflow/go/op/wrappers.go +++ b/tensorflow/go/op/wrappers.go @@ -463,6 +463,14 @@ func QuantizeAndDequantizeV2RangeGiven(value bool) QuantizeAndDequantizeV2Attr { } } +// QuantizeAndDequantizeV2RoundMode sets the optional round_mode attribute to value. +// If not specified, defaults to "HALF_TO_EVEN" +func QuantizeAndDequantizeV2RoundMode(value string) QuantizeAndDequantizeV2Attr { + return func(m optionalAttr) { + m["round_mode"] = value + } +} + // Quantizes then dequantizes a tensor. // // This op simulates the precision loss from the quantized forward pass by: @@ -3487,30 +3495,6 @@ func BoostedTreesQuantileStreamResourceFlush(scope *Scope, quantile_stream_resou return scope.AddOperation(opspec) } -// Add the quantile summaries to each quantile stream resource. -// -// An op that adds a list of quantile summaries to a quantile stream resource. Each -// summary Tensor is rank 2, containing summaries (value, weight, min_rank, max_rank) -// for a single feature. -// -// Arguments: -// quantile_stream_resource_handle: resource handle referring to a QuantileStreamResource. -// summaries: string; List of Rank 2 Tensor each containing the summaries for a single feature. -// -// Returns the created operation. -func BoostedTreesQuantileStreamResourceAddSummaries(scope *Scope, quantile_stream_resource_handle tf.Output, summaries []tf.Output) (o *tf.Operation) { - if scope.Err() != nil { - return - } - opspec := tf.OpSpec{ - Type: "BoostedTreesQuantileStreamResourceAddSummaries", - Input: []tf.Input{ - quantile_stream_resource_handle, tf.OutputList(summaries), - }, - } - return scope.AddOperation(opspec) -} - // Makes the summary of quantiles for the batch. // // An op that takes a list of tensors and outputs the quantile summaries for each tensor. @@ -5547,6 +5531,63 @@ func OrderedMapPeek(scope *Scope, key tf.Output, indices tf.Output, dtypes []tf. return values } +// MapIncompleteSizeAttr is an optional argument to MapIncompleteSize. +type MapIncompleteSizeAttr func(optionalAttr) + +// MapIncompleteSizeCapacity sets the optional capacity attribute to value. +// If not specified, defaults to 0 +// +// REQUIRES: value >= 0 +func MapIncompleteSizeCapacity(value int64) MapIncompleteSizeAttr { + return func(m optionalAttr) { + m["capacity"] = value + } +} + +// MapIncompleteSizeMemoryLimit sets the optional memory_limit attribute to value. +// If not specified, defaults to 0 +// +// REQUIRES: value >= 0 +func MapIncompleteSizeMemoryLimit(value int64) MapIncompleteSizeAttr { + return func(m optionalAttr) { + m["memory_limit"] = value + } +} + +// MapIncompleteSizeContainer sets the optional container attribute to value. +// If not specified, defaults to "" +func MapIncompleteSizeContainer(value string) MapIncompleteSizeAttr { + return func(m optionalAttr) { + m["container"] = value + } +} + +// MapIncompleteSizeSharedName sets the optional shared_name attribute to value. +// If not specified, defaults to "" +func MapIncompleteSizeSharedName(value string) MapIncompleteSizeAttr { + return func(m optionalAttr) { + m["shared_name"] = value + } +} + +// Op returns the number of incomplete elements in the underlying container. +func MapIncompleteSize(scope *Scope, dtypes []tf.DataType, optional ...MapIncompleteSizeAttr) (size tf.Output) { + if scope.Err() != nil { + return + } + attrs := map[string]interface{}{"dtypes": dtypes} + for _, a := range optional { + a(attrs) + } + opspec := tf.OpSpec{ + Type: "MapIncompleteSize", + + Attrs: attrs, + } + op := scope.AddOperation(opspec) + return op.Output(0) +} + // Compute the regularized incomplete beta integral \\(I_x(a, b)\\). // // The regularized incomplete beta integral is defined as: @@ -5612,34 +5653,6 @@ func Atan2(scope *Scope, y tf.Output, x tf.Output) (z tf.Output) { return op.Output(0) } -// Creates a dataset that passes a sliding window over `input_dataset`. -// -// Arguments: -// -// window_size: A scalar representing the number of elements in the -// sliding window. -// window_shift: A scalar representing the steps moving the sliding window -// forward in one iteration. It must be positive. -// window_stride: A scalar representing the stride of the input elements of the sliding window. -// It must be positive. -// -// -func SlideDataset(scope *Scope, input_dataset tf.Output, window_size tf.Output, window_shift tf.Output, window_stride tf.Output, output_types []tf.DataType, output_shapes []tf.Shape) (handle tf.Output) { - if scope.Err() != nil { - return - } - attrs := map[string]interface{}{"output_types": output_types, "output_shapes": output_shapes} - opspec := tf.OpSpec{ - Type: "SlideDataset", - Input: []tf.Input{ - input_dataset, window_size, window_shift, window_stride, - }, - Attrs: attrs, - } - op := scope.AddOperation(opspec) - return op.Output(0) -} - // EditDistanceAttr is an optional argument to EditDistance. type EditDistanceAttr func(optionalAttr) @@ -6279,6 +6292,71 @@ func Asin(scope *Scope, x tf.Output) (y tf.Output) { return op.Output(0) } +// SparseToDenseAttr is an optional argument to SparseToDense. +type SparseToDenseAttr func(optionalAttr) + +// SparseToDenseValidateIndices sets the optional validate_indices attribute to value. +// +// value: If true, indices are checked to make sure they are sorted in +// lexicographic order and that there are no repeats. +// If not specified, defaults to true +func SparseToDenseValidateIndices(value bool) SparseToDenseAttr { + return func(m optionalAttr) { + m["validate_indices"] = value + } +} + +// Converts a sparse representation into a dense tensor. +// +// Builds an array `dense` with shape `output_shape` such that +// +// ``` +// # If sparse_indices is scalar +// dense[i] = (i == sparse_indices ? sparse_values : default_value) +// +// # If sparse_indices is a vector, then for each i +// dense[sparse_indices[i]] = sparse_values[i] +// +// # If sparse_indices is an n by d matrix, then for each i in [0, n) +// dense[sparse_indices[i][0], ..., sparse_indices[i][d-1]] = sparse_values[i] +// ``` +// +// All other values in `dense` are set to `default_value`. If `sparse_values` is a +// scalar, all sparse indices are set to this single value. +// +// Indices should be sorted in lexicographic order, and indices must not +// contain any repeats. If `validate_indices` is true, these properties +// are checked during execution. +// +// Arguments: +// sparse_indices: 0-D, 1-D, or 2-D. `sparse_indices[i]` contains the complete +// index where `sparse_values[i]` will be placed. +// output_shape: 1-D. Shape of the dense output tensor. +// sparse_values: 1-D. Values corresponding to each row of `sparse_indices`, +// or a scalar value to be used for all sparse indices. +// default_value: Scalar value to set for indices not specified in +// `sparse_indices`. +// +// Returns Dense output tensor of shape `output_shape`. +func SparseToDense(scope *Scope, sparse_indices tf.Output, output_shape tf.Output, sparse_values tf.Output, default_value tf.Output, optional ...SparseToDenseAttr) (dense tf.Output) { + if scope.Err() != nil { + return + } + attrs := map[string]interface{}{} + for _, a := range optional { + a(attrs) + } + opspec := tf.OpSpec{ + Type: "SparseToDense", + Input: []tf.Input{ + sparse_indices, output_shape, sparse_values, default_value, + }, + Attrs: attrs, + } + op := scope.AddOperation(opspec) + return op.Output(0) +} + // Computes the sum along sparse segments of a tensor. // // Like `SparseSegmentSum`, but allows missing ids in `segment_ids`. If an id is @@ -6370,21 +6448,6 @@ func Sin(scope *Scope, x tf.Output) (y tf.Output) { return op.Output(0) } -// Computes the complementary error function of `x` element-wise. -func Erfc(scope *Scope, x tf.Output) (y tf.Output) { - if scope.Err() != nil { - return - } - opspec := tf.OpSpec{ - Type: "Erfc", - Input: []tf.Input{ - x, - }, - } - op := scope.AddOperation(opspec) - return op.Output(0) -} - // Computes Psi, the derivative of Lgamma (the log of the absolute value of // // `Gamma(x)`), element-wise. @@ -6829,61 +6892,6 @@ func Reciprocal(scope *Scope, x tf.Output) (y tf.Output) { return op.Output(0) } -// ParseExampleDatasetAttr is an optional argument to ParseExampleDataset. -type ParseExampleDatasetAttr func(optionalAttr) - -// ParseExampleDatasetSloppy sets the optional sloppy attribute to value. -// If not specified, defaults to false -func ParseExampleDatasetSloppy(value bool) ParseExampleDatasetAttr { - return func(m optionalAttr) { - m["sloppy"] = value - } -} - -// Transforms `input_dataset` containing `Example` protos as vectors of DT_STRING into a dataset of `Tensor` or `SparseTensor` objects representing the parsed features. -// -// Arguments: -// -// -// dense_defaults: A dict mapping string keys to `Tensor`s. -// The keys of the dict must match the dense_keys of the feature. -// sparse_keys: A list of string keys in the examples features. -// The results for these keys will be returned as `SparseTensor` objects. -// dense_keys: A list of Ndense string Tensors (scalars). -// The keys expected in the Examples features associated with dense values. -// sparse_types: A list of `DTypes` of the same length as `sparse_keys`. -// Only `tf.float32` (`FloatList`), `tf.int64` (`Int64List`), -// and `tf.string` (`BytesList`) are supported. -// dense_shapes: List of tuples with the same length as `dense_keys`. -// The shape of the data for each dense feature referenced by `dense_keys`. -// Required for any input tensors identified by `dense_keys`. Must be -// either fully defined, or may contain an unknown first dimension. -// An unknown first dimension means the feature is treated as having -// a variable number of blocks, and the output shape along this dimension -// is considered unknown at graph build time. Padding is applied for -// minibatch elements smaller than the maximum number of blocks for the -// given feature along this dimension. -// output_types: The type list for the return values. -// output_shapes: The list of shapes being produced. -func ParseExampleDataset(scope *Scope, input_dataset tf.Output, num_parallel_calls tf.Output, dense_defaults []tf.Output, sparse_keys []string, dense_keys []string, sparse_types []tf.DataType, dense_shapes []tf.Shape, output_types []tf.DataType, output_shapes []tf.Shape, optional ...ParseExampleDatasetAttr) (handle tf.Output) { - if scope.Err() != nil { - return - } - attrs := map[string]interface{}{"sparse_keys": sparse_keys, "dense_keys": dense_keys, "sparse_types": sparse_types, "dense_shapes": dense_shapes, "output_types": output_types, "output_shapes": output_shapes} - for _, a := range optional { - a(attrs) - } - opspec := tf.OpSpec{ - Type: "ParseExampleDataset", - Input: []tf.Input{ - input_dataset, num_parallel_calls, tf.OutputList(dense_defaults), - }, - Attrs: attrs, - } - op := scope.AddOperation(opspec) - return op.Output(0) -} - // Returns a batched matrix tensor with new batched diagonal values. // // Given `input` and `diagonal`, this operation returns a tensor with the @@ -7956,28 +7964,6 @@ func LeakyRelu(scope *Scope, features tf.Output, optional ...LeakyReluAttr) (act return op.Output(0) } -// Writes the given dataset to the given file using the TFRecord format. -// -// Arguments: -// input_dataset: A variant tensor representing the dataset to write. -// filename: A scalar string tensor representing the filename to use. -// compression_type: A scalar string tensor containing either (i) the empty string (no -// compression), (ii) "ZLIB", or (iii) "GZIP". -// -// Returns the created operation. -func DatasetToTFRecord(scope *Scope, input_dataset tf.Output, filename tf.Output, compression_type tf.Output) (o *tf.Operation) { - if scope.Err() != nil { - return - } - opspec := tf.OpSpec{ - Type: "DatasetToTFRecord", - Input: []tf.Input{ - input_dataset, filename, compression_type, - }, - } - return scope.AddOperation(opspec) -} - // Computes rectified linear 6: `min(max(features, 0), 6)`. func Relu6(scope *Scope, features tf.Output) (activations tf.Output) { if scope.Err() != nil { @@ -9441,6 +9427,178 @@ func BesselI1e(scope *Scope, x tf.Output) (y tf.Output) { return op.Output(0) } +// MapClearAttr is an optional argument to MapClear. +type MapClearAttr func(optionalAttr) + +// MapClearCapacity sets the optional capacity attribute to value. +// If not specified, defaults to 0 +// +// REQUIRES: value >= 0 +func MapClearCapacity(value int64) MapClearAttr { + return func(m optionalAttr) { + m["capacity"] = value + } +} + +// MapClearMemoryLimit sets the optional memory_limit attribute to value. +// If not specified, defaults to 0 +// +// REQUIRES: value >= 0 +func MapClearMemoryLimit(value int64) MapClearAttr { + return func(m optionalAttr) { + m["memory_limit"] = value + } +} + +// MapClearContainer sets the optional container attribute to value. +// If not specified, defaults to "" +func MapClearContainer(value string) MapClearAttr { + return func(m optionalAttr) { + m["container"] = value + } +} + +// MapClearSharedName sets the optional shared_name attribute to value. +// If not specified, defaults to "" +func MapClearSharedName(value string) MapClearAttr { + return func(m optionalAttr) { + m["shared_name"] = value + } +} + +// Op removes all elements in the underlying container. +// +// Returns the created operation. +func MapClear(scope *Scope, dtypes []tf.DataType, optional ...MapClearAttr) (o *tf.Operation) { + if scope.Err() != nil { + return + } + attrs := map[string]interface{}{"dtypes": dtypes} + for _, a := range optional { + a(attrs) + } + opspec := tf.OpSpec{ + Type: "MapClear", + + Attrs: attrs, + } + return scope.AddOperation(opspec) +} + +// DecodeCSVAttr is an optional argument to DecodeCSV. +type DecodeCSVAttr func(optionalAttr) + +// DecodeCSVFieldDelim sets the optional field_delim attribute to value. +// +// value: char delimiter to separate fields in a record. +// If not specified, defaults to "," +func DecodeCSVFieldDelim(value string) DecodeCSVAttr { + return func(m optionalAttr) { + m["field_delim"] = value + } +} + +// DecodeCSVUseQuoteDelim sets the optional use_quote_delim attribute to value. +// +// value: If false, treats double quotation marks as regular +// characters inside of the string fields (ignoring RFC 4180, Section 2, +// Bullet 5). +// If not specified, defaults to true +func DecodeCSVUseQuoteDelim(value bool) DecodeCSVAttr { + return func(m optionalAttr) { + m["use_quote_delim"] = value + } +} + +// DecodeCSVNaValue sets the optional na_value attribute to value. +// +// value: Additional string to recognize as NA/NaN. +// If not specified, defaults to "" +func DecodeCSVNaValue(value string) DecodeCSVAttr { + return func(m optionalAttr) { + m["na_value"] = value + } +} + +// DecodeCSVSelectCols sets the optional select_cols attribute to value. +// If not specified, defaults to <> +func DecodeCSVSelectCols(value []int64) DecodeCSVAttr { + return func(m optionalAttr) { + m["select_cols"] = value + } +} + +// Convert CSV records to tensors. Each column maps to one tensor. +// +// RFC 4180 format is expected for the CSV records. +// (https://tools.ietf.org/html/rfc4180) +// Note that we allow leading and trailing spaces with int or float field. +// +// Arguments: +// records: Each string is a record/row in the csv and all records should have +// the same format. +// record_defaults: One tensor per column of the input record, with either a +// scalar default value for that column or an empty vector if the column is +// required. +// +// Returns Each tensor will have the same shape as records. +func DecodeCSV(scope *Scope, records tf.Output, record_defaults []tf.Output, optional ...DecodeCSVAttr) (output []tf.Output) { + if scope.Err() != nil { + return + } + attrs := map[string]interface{}{} + for _, a := range optional { + a(attrs) + } + opspec := tf.OpSpec{ + Type: "DecodeCSV", + Input: []tf.Input{ + records, tf.OutputList(record_defaults), + }, + Attrs: attrs, + } + op := scope.AddOperation(opspec) + if scope.Err() != nil { + return + } + var idx int + var err error + if output, idx, err = makeOutputList(op, idx, "output"); err != nil { + scope.UpdateErr("DecodeCSV", err) + return + } + return output +} + +// Convert JSON-encoded Example records to binary protocol buffer strings. +// +// This op translates a tensor containing Example records, encoded using +// the [standard JSON +// mapping](https://developers.google.com/protocol-buffers/docs/proto3#json), +// into a tensor containing the same records encoded as binary protocol +// buffers. The resulting tensor can then be fed to any of the other +// Example-parsing ops. +// +// Arguments: +// json_examples: Each string is a JSON object serialized according to the JSON +// mapping of the Example proto. +// +// Returns Each string is a binary Example protocol buffer corresponding +// to the respective element of `json_examples`. +func DecodeJSONExample(scope *Scope, json_examples tf.Output) (binary_examples tf.Output) { + if scope.Err() != nil { + return + } + opspec := tf.OpSpec{ + Type: "DecodeJSONExample", + Input: []tf.Input{ + json_examples, + }, + } + op := scope.AddOperation(opspec) + return op.Output(0) +} + // Transforms a Tensor into a serialized TensorProto proto. // // Arguments: @@ -10499,6 +10657,31 @@ func ParseExample(scope *Scope, serialized tf.Output, names tf.Output, sparse_ke return sparse_indices, sparse_values, sparse_shapes, dense_values } +// Compute the pairwise cross product. +// +// `a` and `b` must be the same shape; they can either be simple 3-element vectors, +// or any shape where the innermost dimension is 3. In the latter case, each pair +// of corresponding 3-element vectors is cross-multiplied independently. +// +// Arguments: +// a: A tensor containing 3-element vectors. +// b: Another tensor, of same type and shape as `a`. +// +// Returns Pairwise cross product of the vectors in `a` and `b`. +func Cross(scope *Scope, a tf.Output, b tf.Output) (product tf.Output) { + if scope.Err() != nil { + return + } + opspec := tf.OpSpec{ + Type: "Cross", + Input: []tf.Input{ + a, b, + }, + } + op := scope.AddOperation(opspec) + return op.Output(0) +} + // CudnnRNNAttr is an optional argument to CudnnRNN. type CudnnRNNAttr func(optionalAttr) @@ -11417,66 +11600,6 @@ func RandomStandardNormal(scope *Scope, shape tf.Output, dtype tf.DataType, opti return op.Output(0) } -// RandomUniformIntAttr is an optional argument to RandomUniformInt. -type RandomUniformIntAttr func(optionalAttr) - -// RandomUniformIntSeed sets the optional seed attribute to value. -// -// value: If either `seed` or `seed2` are set to be non-zero, the random number -// generator is seeded by the given seed. Otherwise, it is seeded by a -// random seed. -// If not specified, defaults to 0 -func RandomUniformIntSeed(value int64) RandomUniformIntAttr { - return func(m optionalAttr) { - m["seed"] = value - } -} - -// RandomUniformIntSeed2 sets the optional seed2 attribute to value. -// -// value: A second seed to avoid seed collision. -// If not specified, defaults to 0 -func RandomUniformIntSeed2(value int64) RandomUniformIntAttr { - return func(m optionalAttr) { - m["seed2"] = value - } -} - -// Outputs random integers from a uniform distribution. -// -// The generated values are uniform integers in the range `[minval, maxval)`. -// The lower bound `minval` is included in the range, while the upper bound -// `maxval` is excluded. -// -// The random integers are slightly biased unless `maxval - minval` is an exact -// power of two. The bias is small for values of `maxval - minval` significantly -// smaller than the range of the output (either `2^32` or `2^64`). -// -// Arguments: -// shape: The shape of the output tensor. -// minval: 0-D. Inclusive lower bound on the generated integers. -// maxval: 0-D. Exclusive upper bound on the generated integers. -// -// Returns A tensor of the specified shape filled with uniform random integers. -func RandomUniformInt(scope *Scope, shape tf.Output, minval tf.Output, maxval tf.Output, optional ...RandomUniformIntAttr) (output tf.Output) { - if scope.Err() != nil { - return - } - attrs := map[string]interface{}{} - for _, a := range optional { - a(attrs) - } - opspec := tf.OpSpec{ - Type: "RandomUniformInt", - Input: []tf.Input{ - shape, minval, maxval, - }, - Attrs: attrs, - } - op := scope.AddOperation(opspec) - return op.Output(0) -} - // FusedResizeAndPadConv2DAttr is an optional argument to FusedResizeAndPadConv2D. type FusedResizeAndPadConv2DAttr func(optionalAttr) @@ -13530,6 +13653,39 @@ func StatelessRandomNormal(scope *Scope, shape tf.Output, seed tf.Output, option return op.Output(0) } +// Computes the complementary error function of `x` element-wise. +func Erfc(scope *Scope, x tf.Output) (y tf.Output) { + if scope.Err() != nil { + return + } + opspec := tf.OpSpec{ + Type: "Erfc", + Input: []tf.Input{ + x, + }, + } + op := scope.AddOperation(opspec) + return op.Output(0) +} + +// Returns the number of tensors in the input tensor list. +// +// input_handle: the input list +// length: the number of tensors in the list +func TensorListLength(scope *Scope, input_handle tf.Output) (length tf.Output) { + if scope.Err() != nil { + return + } + opspec := tf.OpSpec{ + Type: "TensorListLength", + Input: []tf.Input{ + input_handle, + }, + } + op := scope.AddOperation(opspec) + return op.Output(0) +} + // Determine the script codes of a given tensor of Unicode integer code points. // // This operation converts Unicode code points to script codes corresponding to @@ -13747,6 +13903,126 @@ func ResourceApplyMomentum(scope *Scope, var_ tf.Output, accum tf.Output, lr tf. return scope.AddOperation(opspec) } +// SubstrAttr is an optional argument to Substr. +type SubstrAttr func(optionalAttr) + +// SubstrUnit sets the optional unit attribute to value. +// +// value: The unit that is used to create the substring. One of: `"BYTE"` (for +// defining position and length by bytes) or `"UTF8_CHAR"` (for the UTF-8 +// encoded Unicode code points). The default is `"BYTE"`. Results are undefined if +// `unit=UTF8_CHAR` and the `input` strings do not contain structurally valid +// UTF-8. +// If not specified, defaults to "BYTE" +func SubstrUnit(value string) SubstrAttr { + return func(m optionalAttr) { + m["unit"] = value + } +} + +// Return substrings from `Tensor` of strings. +// +// For each string in the input `Tensor`, creates a substring starting at index +// `pos` with a total length of `len`. +// +// If `len` defines a substring that would extend beyond the length of the input +// string, then as many characters as possible are used. +// +// A negative `pos` indicates distance within the string backwards from the end. +// +// If `pos` specifies an index which is out of range for any of the input strings, +// then an `InvalidArgumentError` is thrown. +// +// `pos` and `len` must have the same shape, otherwise a `ValueError` is thrown on +// Op creation. +// +// *NOTE*: `Substr` supports broadcasting up to two dimensions. More about +// broadcasting +// [here](http://docs.scipy.org/doc/numpy/user/basics.broadcasting.html) +// +// --- +// +// Examples +// +// Using scalar `pos` and `len`: +// +// ```python +// input = [b'Hello', b'World'] +// position = 1 +// length = 3 +// +// output = [b'ell', b'orl'] +// ``` +// +// Using `pos` and `len` with same shape as `input`: +// +// ```python +// input = [[b'ten', b'eleven', b'twelve'], +// [b'thirteen', b'fourteen', b'fifteen'], +// [b'sixteen', b'seventeen', b'eighteen']] +// position = [[1, 2, 3], +// [1, 2, 3], +// [1, 2, 3]] +// length = [[2, 3, 4], +// [4, 3, 2], +// [5, 5, 5]] +// +// output = [[b'en', b'eve', b'lve'], +// [b'hirt', b'urt', b'te'], +// [b'ixtee', b'vente', b'hteen']] +// ``` +// +// Broadcasting `pos` and `len` onto `input`: +// +// ``` +// input = [[b'ten', b'eleven', b'twelve'], +// [b'thirteen', b'fourteen', b'fifteen'], +// [b'sixteen', b'seventeen', b'eighteen'], +// [b'nineteen', b'twenty', b'twentyone']] +// position = [1, 2, 3] +// length = [1, 2, 3] +// +// output = [[b'e', b'ev', b'lve'], +// [b'h', b'ur', b'tee'], +// [b'i', b've', b'hte'], +// [b'i', b'en', b'nty']] +// ``` +// +// Broadcasting `input` onto `pos` and `len`: +// +// ``` +// input = b'thirteen' +// position = [1, 5, 7] +// length = [3, 2, 1] +// +// output = [b'hir', b'ee', b'n'] +// ``` +// +// Arguments: +// input: Tensor of strings +// pos: Scalar defining the position of first character in each substring +// len: Scalar defining the number of characters to include in each substring +// +// Returns Tensor of substrings +func Substr(scope *Scope, input tf.Output, pos tf.Output, len tf.Output, optional ...SubstrAttr) (output tf.Output) { + if scope.Err() != nil { + return + } + attrs := map[string]interface{}{} + for _, a := range optional { + a(attrs) + } + opspec := tf.OpSpec{ + Type: "Substr", + Input: []tf.Input{ + input, pos, len, + }, + Attrs: attrs, + } + op := scope.AddOperation(opspec) + return op.Output(0) +} + // Exits the current frame to its parent frame. // // Exit makes its input `data` available to the parent frame. @@ -14946,75 +15222,6 @@ func ResourceApplyAdagrad(scope *Scope, var_ tf.Output, accum tf.Output, lr tf.O return scope.AddOperation(opspec) } -// Return the shape of s0 op s1 with broadcast. -// -// Given `s0` and `s1`, tensors that represent shapes, compute `r0`, the -// broadcasted shape. `s0`, `s1` and `r0` are all integer vectors. -func BroadcastArgs(scope *Scope, s0 tf.Output, s1 tf.Output) (r0 tf.Output) { - if scope.Err() != nil { - return - } - opspec := tf.OpSpec{ - Type: "BroadcastArgs", - Input: []tf.Input{ - s0, s1, - }, - } - op := scope.AddOperation(opspec) - return op.Output(0) -} - -// DataFormatDimMapAttr is an optional argument to DataFormatDimMap. -type DataFormatDimMapAttr func(optionalAttr) - -// DataFormatDimMapSrcFormat sets the optional src_format attribute to value. -// -// value: source data format. -// If not specified, defaults to "NHWC" -func DataFormatDimMapSrcFormat(value string) DataFormatDimMapAttr { - return func(m optionalAttr) { - m["src_format"] = value - } -} - -// DataFormatDimMapDstFormat sets the optional dst_format attribute to value. -// -// value: destination data format. -// If not specified, defaults to "NCHW" -func DataFormatDimMapDstFormat(value string) DataFormatDimMapAttr { - return func(m optionalAttr) { - m["dst_format"] = value - } -} - -// Returns the dimension index in the destination data format given the one in -// -// the source data format. -// -// Arguments: -// x: A Tensor with each element as a dimension index in source data format. -// Must be in the range [-4, 4). -// -// Returns A Tensor with each element as a dimension index in destination data format. -func DataFormatDimMap(scope *Scope, x tf.Output, optional ...DataFormatDimMapAttr) (y tf.Output) { - if scope.Err() != nil { - return - } - attrs := map[string]interface{}{} - for _, a := range optional { - a(attrs) - } - opspec := tf.OpSpec{ - Type: "DataFormatDimMap", - Input: []tf.Input{ - x, - }, - Attrs: attrs, - } - op := scope.AddOperation(opspec) - return op.Output(0) -} - // Retrieves the tree ensemble resource stamp token, number of trees and growing statistics. // // Arguments: @@ -16696,6 +16903,124 @@ func SparseReduceSum(scope *Scope, input_indices tf.Output, input_values tf.Outp return op.Output(0) } +// SparseTensorDenseMatMulAttr is an optional argument to SparseTensorDenseMatMul. +type SparseTensorDenseMatMulAttr func(optionalAttr) + +// SparseTensorDenseMatMulAdjointA sets the optional adjoint_a attribute to value. +// +// value: Use the adjoint of A in the matrix multiply. If A is complex, this +// is transpose(conj(A)). Otherwise it's transpose(A). +// If not specified, defaults to false +func SparseTensorDenseMatMulAdjointA(value bool) SparseTensorDenseMatMulAttr { + return func(m optionalAttr) { + m["adjoint_a"] = value + } +} + +// SparseTensorDenseMatMulAdjointB sets the optional adjoint_b attribute to value. +// +// value: Use the adjoint of B in the matrix multiply. If B is complex, this +// is transpose(conj(B)). Otherwise it's transpose(B). +// If not specified, defaults to false +func SparseTensorDenseMatMulAdjointB(value bool) SparseTensorDenseMatMulAttr { + return func(m optionalAttr) { + m["adjoint_b"] = value + } +} + +// Multiply SparseTensor (of rank 2) "A" by dense matrix "B". +// +// No validity checking is performed on the indices of A. However, the following +// input format is recommended for optimal behavior: +// +// if adjoint_a == false: +// A should be sorted in lexicographically increasing order. Use SparseReorder +// if you're not sure. +// if adjoint_a == true: +// A should be sorted in order of increasing dimension 1 (i.e., "column major" +// order instead of "row major" order). +// +// Arguments: +// a_indices: 2-D. The `indices` of the `SparseTensor`, size `[nnz, 2]` Matrix. +// a_values: 1-D. The `values` of the `SparseTensor`, size `[nnz]` Vector. +// a_shape: 1-D. The `shape` of the `SparseTensor`, size `[2]` Vector. +// b: 2-D. A dense Matrix. +func SparseTensorDenseMatMul(scope *Scope, a_indices tf.Output, a_values tf.Output, a_shape tf.Output, b tf.Output, optional ...SparseTensorDenseMatMulAttr) (product tf.Output) { + if scope.Err() != nil { + return + } + attrs := map[string]interface{}{} + for _, a := range optional { + a(attrs) + } + opspec := tf.OpSpec{ + Type: "SparseTensorDenseMatMul", + Input: []tf.Input{ + a_indices, a_values, a_shape, b, + }, + Attrs: attrs, + } + op := scope.AddOperation(opspec) + return op.Output(0) +} + +// ResourceApplyRMSPropAttr is an optional argument to ResourceApplyRMSProp. +type ResourceApplyRMSPropAttr func(optionalAttr) + +// ResourceApplyRMSPropUseLocking sets the optional use_locking attribute to value. +// +// value: If `True`, updating of the var, ms, and mom tensors is protected +// by a lock; otherwise the behavior is undefined, but may exhibit less +// contention. +// If not specified, defaults to false +func ResourceApplyRMSPropUseLocking(value bool) ResourceApplyRMSPropAttr { + return func(m optionalAttr) { + m["use_locking"] = value + } +} + +// Update '*var' according to the RMSProp algorithm. +// +// Note that in dense implementation of this algorithm, ms and mom will +// update even if the grad is zero, but in this sparse implementation, ms +// and mom will not update in iterations during which the grad is zero. +// +// mean_square = decay * mean_square + (1-decay) * gradient ** 2 +// Delta = learning_rate * gradient / sqrt(mean_square + epsilon) +// +// ms <- rho * ms_{t-1} + (1-rho) * grad * grad +// mom <- momentum * mom_{t-1} + lr * grad / sqrt(ms + epsilon) +// var <- var - mom +// +// Arguments: +// var_: Should be from a Variable(). +// ms: Should be from a Variable(). +// mom: Should be from a Variable(). +// lr: Scaling factor. Must be a scalar. +// rho: Decay rate. Must be a scalar. +// +// epsilon: Ridge term. Must be a scalar. +// grad: The gradient. +// +// Returns the created operation. +func ResourceApplyRMSProp(scope *Scope, var_ tf.Output, ms tf.Output, mom tf.Output, lr tf.Output, rho tf.Output, momentum tf.Output, epsilon tf.Output, grad tf.Output, optional ...ResourceApplyRMSPropAttr) (o *tf.Operation) { + if scope.Err() != nil { + return + } + attrs := map[string]interface{}{} + for _, a := range optional { + a(attrs) + } + opspec := tf.OpSpec{ + Type: "ResourceApplyRMSProp", + Input: []tf.Input{ + var_, ms, mom, lr, rho, momentum, epsilon, grad, + }, + Attrs: attrs, + } + return scope.AddOperation(opspec) +} + // Returns immutable tensor from memory region. // // The current implementation memmaps the tensor from a file. @@ -18017,119 +18342,6 @@ func ResourceApplyCenteredRMSProp(scope *Scope, var_ tf.Output, mg tf.Output, ms return scope.AddOperation(opspec) } -// Computes the gradient for the inverse of `x` wrt its input. -// -// Specifically, `grad = -dy * y*y`, where `y = 1/x`, and `dy` -// is the corresponding input gradient. -func ReciprocalGrad(scope *Scope, y tf.Output, dy tf.Output) (z tf.Output) { - if scope.Err() != nil { - return - } - opspec := tf.OpSpec{ - Type: "ReciprocalGrad", - Input: []tf.Input{ - y, dy, - }, - } - op := scope.AddOperation(opspec) - return op.Output(0) -} - -// Returns the min of x and y (i.e. x < y ? x : y) element-wise. -// -// *NOTE*: `Minimum` supports broadcasting. More about broadcasting -// [here](http://docs.scipy.org/doc/numpy/user/basics.broadcasting.html) -func Minimum(scope *Scope, x tf.Output, y tf.Output) (z tf.Output) { - if scope.Err() != nil { - return - } - opspec := tf.OpSpec{ - Type: "Minimum", - Input: []tf.Input{ - x, y, - }, - } - op := scope.AddOperation(opspec) - return op.Output(0) -} - -// MfccAttr is an optional argument to Mfcc. -type MfccAttr func(optionalAttr) - -// MfccUpperFrequencyLimit sets the optional upper_frequency_limit attribute to value. -// -// value: The highest frequency to use when calculating the -// ceptstrum. -// If not specified, defaults to 4000 -func MfccUpperFrequencyLimit(value float32) MfccAttr { - return func(m optionalAttr) { - m["upper_frequency_limit"] = value - } -} - -// MfccLowerFrequencyLimit sets the optional lower_frequency_limit attribute to value. -// -// value: The lowest frequency to use when calculating the -// ceptstrum. -// If not specified, defaults to 20 -func MfccLowerFrequencyLimit(value float32) MfccAttr { - return func(m optionalAttr) { - m["lower_frequency_limit"] = value - } -} - -// MfccFilterbankChannelCount sets the optional filterbank_channel_count attribute to value. -// -// value: Resolution of the Mel bank used internally. -// If not specified, defaults to 40 -func MfccFilterbankChannelCount(value int64) MfccAttr { - return func(m optionalAttr) { - m["filterbank_channel_count"] = value - } -} - -// MfccDctCoefficientCount sets the optional dct_coefficient_count attribute to value. -// -// value: How many output channels to produce per time slice. -// If not specified, defaults to 13 -func MfccDctCoefficientCount(value int64) MfccAttr { - return func(m optionalAttr) { - m["dct_coefficient_count"] = value - } -} - -// Transforms a spectrogram into a form that's useful for speech recognition. -// -// Mel Frequency Cepstral Coefficients are a way of representing audio data that's -// been effective as an input feature for machine learning. They are created by -// taking the spectrum of a spectrogram (a 'cepstrum'), and discarding some of the -// higher frequencies that are less significant to the human ear. They have a long -// history in the speech recognition world, and https://en.wikipedia.org/wiki/Mel-frequency_cepstrum -// is a good resource to learn more. -// -// Arguments: -// spectrogram: Typically produced by the Spectrogram op, with magnitude_squared -// set to true. -// sample_rate: How many samples per second the source audio used. -func Mfcc(scope *Scope, spectrogram tf.Output, sample_rate tf.Output, optional ...MfccAttr) (output tf.Output) { - if scope.Err() != nil { - return - } - attrs := map[string]interface{}{} - for _, a := range optional { - a(attrs) - } - opspec := tf.OpSpec{ - Type: "Mfcc", - Input: []tf.Input{ - spectrogram, sample_rate, - }, - Attrs: attrs, - } - op := scope.AddOperation(opspec) - return op.Output(0) -} - // AudioSummaryAttr is an optional argument to AudioSummary. type AudioSummaryAttr func(optionalAttr) @@ -18240,23 +18452,6 @@ func Qr(scope *Scope, input tf.Output, optional ...QrAttr) (q tf.Output, r tf.Ou return op.Output(0), op.Output(1) } -// Records the bytes size of each element of `input_dataset` in a StatsAggregator. -func BytesProducedStatsDataset(scope *Scope, input_dataset tf.Output, tag tf.Output, output_types []tf.DataType, output_shapes []tf.Shape) (handle tf.Output) { - if scope.Err() != nil { - return - } - attrs := map[string]interface{}{"output_types": output_types, "output_shapes": output_shapes} - opspec := tf.OpSpec{ - Type: "BytesProducedStatsDataset", - Input: []tf.Input{ - input_dataset, tag, - }, - Attrs: attrs, - } - op := scope.AddOperation(opspec) - return op.Output(0) -} - // Check if the input matches the regex pattern. // // The input is a string tensor of any shape. The pattern is the @@ -18372,6 +18567,29 @@ func RFFT(scope *Scope, input tf.Output, fft_length tf.Output) (output tf.Output return op.Output(0) } +// Adds a value to the current value of a variable. +// +// Any ReadVariableOp with a control dependency on this op is guaranteed to +// see the incremented value or a subsequent newer one. +// +// Arguments: +// resource: handle to the resource in which to store the variable. +// value: the value by which the variable will be incremented. +// +// Returns the created operation. +func AssignAddVariableOp(scope *Scope, resource tf.Output, value tf.Output) (o *tf.Operation) { + if scope.Err() != nil { + return + } + opspec := tf.OpSpec{ + Type: "AssignAddVariableOp", + Input: []tf.Input{ + resource, value, + }, + } + return scope.AddOperation(opspec) +} + // QuantizedReluAttr is an optional argument to QuantizedRelu. type QuantizedReluAttr func(optionalAttr) @@ -19161,149 +19379,6 @@ func TensorArrayV2(scope *Scope, size tf.Output, dtype tf.DataType, optional ... return op.Output(0) } -// DecodeCSVAttr is an optional argument to DecodeCSV. -type DecodeCSVAttr func(optionalAttr) - -// DecodeCSVFieldDelim sets the optional field_delim attribute to value. -// -// value: char delimiter to separate fields in a record. -// If not specified, defaults to "," -func DecodeCSVFieldDelim(value string) DecodeCSVAttr { - return func(m optionalAttr) { - m["field_delim"] = value - } -} - -// DecodeCSVUseQuoteDelim sets the optional use_quote_delim attribute to value. -// -// value: If false, treats double quotation marks as regular -// characters inside of the string fields (ignoring RFC 4180, Section 2, -// Bullet 5). -// If not specified, defaults to true -func DecodeCSVUseQuoteDelim(value bool) DecodeCSVAttr { - return func(m optionalAttr) { - m["use_quote_delim"] = value - } -} - -// DecodeCSVNaValue sets the optional na_value attribute to value. -// -// value: Additional string to recognize as NA/NaN. -// If not specified, defaults to "" -func DecodeCSVNaValue(value string) DecodeCSVAttr { - return func(m optionalAttr) { - m["na_value"] = value - } -} - -// DecodeCSVSelectCols sets the optional select_cols attribute to value. -// If not specified, defaults to <> -func DecodeCSVSelectCols(value []int64) DecodeCSVAttr { - return func(m optionalAttr) { - m["select_cols"] = value - } -} - -// Convert CSV records to tensors. Each column maps to one tensor. -// -// RFC 4180 format is expected for the CSV records. -// (https://tools.ietf.org/html/rfc4180) -// Note that we allow leading and trailing spaces with int or float field. -// -// Arguments: -// records: Each string is a record/row in the csv and all records should have -// the same format. -// record_defaults: One tensor per column of the input record, with either a -// scalar default value for that column or an empty vector if the column is -// required. -// -// Returns Each tensor will have the same shape as records. -func DecodeCSV(scope *Scope, records tf.Output, record_defaults []tf.Output, optional ...DecodeCSVAttr) (output []tf.Output) { - if scope.Err() != nil { - return - } - attrs := map[string]interface{}{} - for _, a := range optional { - a(attrs) - } - opspec := tf.OpSpec{ - Type: "DecodeCSV", - Input: []tf.Input{ - records, tf.OutputList(record_defaults), - }, - Attrs: attrs, - } - op := scope.AddOperation(opspec) - if scope.Err() != nil { - return - } - var idx int - var err error - if output, idx, err = makeOutputList(op, idx, "output"); err != nil { - scope.UpdateErr("DecodeCSV", err) - return - } - return output -} - -// MapClearAttr is an optional argument to MapClear. -type MapClearAttr func(optionalAttr) - -// MapClearCapacity sets the optional capacity attribute to value. -// If not specified, defaults to 0 -// -// REQUIRES: value >= 0 -func MapClearCapacity(value int64) MapClearAttr { - return func(m optionalAttr) { - m["capacity"] = value - } -} - -// MapClearMemoryLimit sets the optional memory_limit attribute to value. -// If not specified, defaults to 0 -// -// REQUIRES: value >= 0 -func MapClearMemoryLimit(value int64) MapClearAttr { - return func(m optionalAttr) { - m["memory_limit"] = value - } -} - -// MapClearContainer sets the optional container attribute to value. -// If not specified, defaults to "" -func MapClearContainer(value string) MapClearAttr { - return func(m optionalAttr) { - m["container"] = value - } -} - -// MapClearSharedName sets the optional shared_name attribute to value. -// If not specified, defaults to "" -func MapClearSharedName(value string) MapClearAttr { - return func(m optionalAttr) { - m["shared_name"] = value - } -} - -// Op removes all elements in the underlying container. -// -// Returns the created operation. -func MapClear(scope *Scope, dtypes []tf.DataType, optional ...MapClearAttr) (o *tf.Operation) { - if scope.Err() != nil { - return - } - attrs := map[string]interface{}{"dtypes": dtypes} - for _, a := range optional { - a(attrs) - } - opspec := tf.OpSpec{ - Type: "MapClear", - - Attrs: attrs, - } - return scope.AddOperation(opspec) -} - // ThreadUnsafeUnigramCandidateSamplerAttr is an optional argument to ThreadUnsafeUnigramCandidateSampler. type ThreadUnsafeUnigramCandidateSamplerAttr func(optionalAttr) @@ -19523,6 +19598,119 @@ func MutableDenseHashTableV2(scope *Scope, empty_key tf.Output, deleted_key tf.O return op.Output(0) } +// Computes the gradient for the inverse of `x` wrt its input. +// +// Specifically, `grad = -dy * y*y`, where `y = 1/x`, and `dy` +// is the corresponding input gradient. +func ReciprocalGrad(scope *Scope, y tf.Output, dy tf.Output) (z tf.Output) { + if scope.Err() != nil { + return + } + opspec := tf.OpSpec{ + Type: "ReciprocalGrad", + Input: []tf.Input{ + y, dy, + }, + } + op := scope.AddOperation(opspec) + return op.Output(0) +} + +// Returns the min of x and y (i.e. x < y ? x : y) element-wise. +// +// *NOTE*: `Minimum` supports broadcasting. More about broadcasting +// [here](http://docs.scipy.org/doc/numpy/user/basics.broadcasting.html) +func Minimum(scope *Scope, x tf.Output, y tf.Output) (z tf.Output) { + if scope.Err() != nil { + return + } + opspec := tf.OpSpec{ + Type: "Minimum", + Input: []tf.Input{ + x, y, + }, + } + op := scope.AddOperation(opspec) + return op.Output(0) +} + +// MfccAttr is an optional argument to Mfcc. +type MfccAttr func(optionalAttr) + +// MfccUpperFrequencyLimit sets the optional upper_frequency_limit attribute to value. +// +// value: The highest frequency to use when calculating the +// ceptstrum. +// If not specified, defaults to 4000 +func MfccUpperFrequencyLimit(value float32) MfccAttr { + return func(m optionalAttr) { + m["upper_frequency_limit"] = value + } +} + +// MfccLowerFrequencyLimit sets the optional lower_frequency_limit attribute to value. +// +// value: The lowest frequency to use when calculating the +// ceptstrum. +// If not specified, defaults to 20 +func MfccLowerFrequencyLimit(value float32) MfccAttr { + return func(m optionalAttr) { + m["lower_frequency_limit"] = value + } +} + +// MfccFilterbankChannelCount sets the optional filterbank_channel_count attribute to value. +// +// value: Resolution of the Mel bank used internally. +// If not specified, defaults to 40 +func MfccFilterbankChannelCount(value int64) MfccAttr { + return func(m optionalAttr) { + m["filterbank_channel_count"] = value + } +} + +// MfccDctCoefficientCount sets the optional dct_coefficient_count attribute to value. +// +// value: How many output channels to produce per time slice. +// If not specified, defaults to 13 +func MfccDctCoefficientCount(value int64) MfccAttr { + return func(m optionalAttr) { + m["dct_coefficient_count"] = value + } +} + +// Transforms a spectrogram into a form that's useful for speech recognition. +// +// Mel Frequency Cepstral Coefficients are a way of representing audio data that's +// been effective as an input feature for machine learning. They are created by +// taking the spectrum of a spectrogram (a 'cepstrum'), and discarding some of the +// higher frequencies that are less significant to the human ear. They have a long +// history in the speech recognition world, and https://en.wikipedia.org/wiki/Mel-frequency_cepstrum +// is a good resource to learn more. +// +// Arguments: +// spectrogram: Typically produced by the Spectrogram op, with magnitude_squared +// set to true. +// sample_rate: How many samples per second the source audio used. +func Mfcc(scope *Scope, spectrogram tf.Output, sample_rate tf.Output, optional ...MfccAttr) (output tf.Output) { + if scope.Err() != nil { + return + } + attrs := map[string]interface{}{} + for _, a := range optional { + a(attrs) + } + opspec := tf.OpSpec{ + Type: "Mfcc", + Input: []tf.Input{ + spectrogram, sample_rate, + }, + Attrs: attrs, + } + op := scope.AddOperation(opspec) + return op.Output(0) +} + // Compute the Hurwitz zeta function \\(\zeta(x, q)\\). // // The Hurwitz zeta function is defined as: @@ -19627,63 +19815,6 @@ func IFFT2D(scope *Scope, input tf.Output) (output tf.Output) { return op.Output(0) } -// ResourceApplyRMSPropAttr is an optional argument to ResourceApplyRMSProp. -type ResourceApplyRMSPropAttr func(optionalAttr) - -// ResourceApplyRMSPropUseLocking sets the optional use_locking attribute to value. -// -// value: If `True`, updating of the var, ms, and mom tensors is protected -// by a lock; otherwise the behavior is undefined, but may exhibit less -// contention. -// If not specified, defaults to false -func ResourceApplyRMSPropUseLocking(value bool) ResourceApplyRMSPropAttr { - return func(m optionalAttr) { - m["use_locking"] = value - } -} - -// Update '*var' according to the RMSProp algorithm. -// -// Note that in dense implementation of this algorithm, ms and mom will -// update even if the grad is zero, but in this sparse implementation, ms -// and mom will not update in iterations during which the grad is zero. -// -// mean_square = decay * mean_square + (1-decay) * gradient ** 2 -// Delta = learning_rate * gradient / sqrt(mean_square + epsilon) -// -// ms <- rho * ms_{t-1} + (1-rho) * grad * grad -// mom <- momentum * mom_{t-1} + lr * grad / sqrt(ms + epsilon) -// var <- var - mom -// -// Arguments: -// var_: Should be from a Variable(). -// ms: Should be from a Variable(). -// mom: Should be from a Variable(). -// lr: Scaling factor. Must be a scalar. -// rho: Decay rate. Must be a scalar. -// -// epsilon: Ridge term. Must be a scalar. -// grad: The gradient. -// -// Returns the created operation. -func ResourceApplyRMSProp(scope *Scope, var_ tf.Output, ms tf.Output, mom tf.Output, lr tf.Output, rho tf.Output, momentum tf.Output, epsilon tf.Output, grad tf.Output, optional ...ResourceApplyRMSPropAttr) (o *tf.Operation) { - if scope.Err() != nil { - return - } - attrs := map[string]interface{}{} - for _, a := range optional { - a(attrs) - } - opspec := tf.OpSpec{ - Type: "ResourceApplyRMSProp", - Input: []tf.Input{ - var_, ms, mom, lr, rho, momentum, epsilon, grad, - }, - Attrs: attrs, - } - return scope.AddOperation(opspec) -} - // Returns element-wise remainder of division. This emulates C semantics in that // // the result here is consistent with a truncating divide. E.g. `truncate(x / y) * @@ -20760,67 +20891,6 @@ func AddSparseToTensorsMap(scope *Scope, sparse_indices tf.Output, sparse_values return op.Output(0) } -// SparseTensorDenseMatMulAttr is an optional argument to SparseTensorDenseMatMul. -type SparseTensorDenseMatMulAttr func(optionalAttr) - -// SparseTensorDenseMatMulAdjointA sets the optional adjoint_a attribute to value. -// -// value: Use the adjoint of A in the matrix multiply. If A is complex, this -// is transpose(conj(A)). Otherwise it's transpose(A). -// If not specified, defaults to false -func SparseTensorDenseMatMulAdjointA(value bool) SparseTensorDenseMatMulAttr { - return func(m optionalAttr) { - m["adjoint_a"] = value - } -} - -// SparseTensorDenseMatMulAdjointB sets the optional adjoint_b attribute to value. -// -// value: Use the adjoint of B in the matrix multiply. If B is complex, this -// is transpose(conj(B)). Otherwise it's transpose(B). -// If not specified, defaults to false -func SparseTensorDenseMatMulAdjointB(value bool) SparseTensorDenseMatMulAttr { - return func(m optionalAttr) { - m["adjoint_b"] = value - } -} - -// Multiply SparseTensor (of rank 2) "A" by dense matrix "B". -// -// No validity checking is performed on the indices of A. However, the following -// input format is recommended for optimal behavior: -// -// if adjoint_a == false: -// A should be sorted in lexicographically increasing order. Use SparseReorder -// if you're not sure. -// if adjoint_a == true: -// A should be sorted in order of increasing dimension 1 (i.e., "column major" -// order instead of "row major" order). -// -// Arguments: -// a_indices: 2-D. The `indices` of the `SparseTensor`, size `[nnz, 2]` Matrix. -// a_values: 1-D. The `values` of the `SparseTensor`, size `[nnz]` Vector. -// a_shape: 1-D. The `shape` of the `SparseTensor`, size `[2]` Vector. -// b: 2-D. A dense Matrix. -func SparseTensorDenseMatMul(scope *Scope, a_indices tf.Output, a_values tf.Output, a_shape tf.Output, b tf.Output, optional ...SparseTensorDenseMatMulAttr) (product tf.Output) { - if scope.Err() != nil { - return - } - attrs := map[string]interface{}{} - for _, a := range optional { - a(attrs) - } - opspec := tf.OpSpec{ - Type: "SparseTensorDenseMatMul", - Input: []tf.Input{ - a_indices, a_values, a_shape, b, - }, - Attrs: attrs, - } - op := scope.AddOperation(opspec) - return op.Output(0) -} - // Deserialize and concatenate `SparseTensors` from a serialized minibatch. // // The input `serialized_sparse` must be a string matrix of shape `[N x 3]` where @@ -20903,151 +20973,6 @@ func BitwiseAnd(scope *Scope, x tf.Output, y tf.Output) (z tf.Output) { return op.Output(0) } -// SubstrAttr is an optional argument to Substr. -type SubstrAttr func(optionalAttr) - -// SubstrUnit sets the optional unit attribute to value. -// -// value: The unit that is used to create the substring. One of: `"BYTE"` (for -// defining position and length by bytes) or `"UTF8_CHAR"` (for the UTF-8 -// encoded Unicode code points). The default is `"BYTE"`. Results are undefined if -// `unit=UTF8_CHAR` and the `input` strings do not contain structurally valid -// UTF-8. -// If not specified, defaults to "BYTE" -func SubstrUnit(value string) SubstrAttr { - return func(m optionalAttr) { - m["unit"] = value - } -} - -// Return substrings from `Tensor` of strings. -// -// For each string in the input `Tensor`, creates a substring starting at index -// `pos` with a total length of `len`. -// -// If `len` defines a substring that would extend beyond the length of the input -// string, then as many characters as possible are used. -// -// A negative `pos` indicates distance within the string backwards from the end. -// -// If `pos` specifies an index which is out of range for any of the input strings, -// then an `InvalidArgumentError` is thrown. -// -// `pos` and `len` must have the same shape, otherwise a `ValueError` is thrown on -// Op creation. -// -// *NOTE*: `Substr` supports broadcasting up to two dimensions. More about -// broadcasting -// [here](http://docs.scipy.org/doc/numpy/user/basics.broadcasting.html) -// -// --- -// -// Examples -// -// Using scalar `pos` and `len`: -// -// ```python -// input = [b'Hello', b'World'] -// position = 1 -// length = 3 -// -// output = [b'ell', b'orl'] -// ``` -// -// Using `pos` and `len` with same shape as `input`: -// -// ```python -// input = [[b'ten', b'eleven', b'twelve'], -// [b'thirteen', b'fourteen', b'fifteen'], -// [b'sixteen', b'seventeen', b'eighteen']] -// position = [[1, 2, 3], -// [1, 2, 3], -// [1, 2, 3]] -// length = [[2, 3, 4], -// [4, 3, 2], -// [5, 5, 5]] -// -// output = [[b'en', b'eve', b'lve'], -// [b'hirt', b'urt', b'te'], -// [b'ixtee', b'vente', b'hteen']] -// ``` -// -// Broadcasting `pos` and `len` onto `input`: -// -// ``` -// input = [[b'ten', b'eleven', b'twelve'], -// [b'thirteen', b'fourteen', b'fifteen'], -// [b'sixteen', b'seventeen', b'eighteen'], -// [b'nineteen', b'twenty', b'twentyone']] -// position = [1, 2, 3] -// length = [1, 2, 3] -// -// output = [[b'e', b'ev', b'lve'], -// [b'h', b'ur', b'tee'], -// [b'i', b've', b'hte'], -// [b'i', b'en', b'nty']] -// ``` -// -// Broadcasting `input` onto `pos` and `len`: -// -// ``` -// input = b'thirteen' -// position = [1, 5, 7] -// length = [3, 2, 1] -// -// output = [b'hir', b'ee', b'n'] -// ``` -// -// Arguments: -// input: Tensor of strings -// pos: Scalar defining the position of first character in each substring -// len: Scalar defining the number of characters to include in each substring -// -// Returns Tensor of substrings -func Substr(scope *Scope, input tf.Output, pos tf.Output, len tf.Output, optional ...SubstrAttr) (output tf.Output) { - if scope.Err() != nil { - return - } - attrs := map[string]interface{}{} - for _, a := range optional { - a(attrs) - } - opspec := tf.OpSpec{ - Type: "Substr", - Input: []tf.Input{ - input, pos, len, - }, - Attrs: attrs, - } - op := scope.AddOperation(opspec) - return op.Output(0) -} - -// Creates a Dataset that returns pseudorandom numbers. -// -// Arguments: -// seed: A scalar seed for the random number generator. If either seed or -// seed2 is set to be non-zero, the random number generator is seeded -// by the given seed. Otherwise, a random seed is used. -// seed2: A second scalar seed to avoid seed collision. -// -// -func RandomDataset(scope *Scope, seed tf.Output, seed2 tf.Output, output_types []tf.DataType, output_shapes []tf.Shape) (handle tf.Output) { - if scope.Err() != nil { - return - } - attrs := map[string]interface{}{"output_types": output_types, "output_shapes": output_shapes} - opspec := tf.OpSpec{ - Type: "RandomDataset", - Input: []tf.Input{ - seed, seed2, - }, - Attrs: attrs, - } - op := scope.AddOperation(opspec) - return op.Output(0) -} - // Inverse real-valued fast Fourier transform. // // Computes the inverse 1-dimensional discrete Fourier transform of a real-valued @@ -22230,6 +22155,34 @@ func MatrixTriangularSolve(scope *Scope, matrix tf.Output, rhs tf.Output, option return op.Output(0) } +// Saves tensors in V2 checkpoint format. +// +// By default, saves the named tensors in full. If the caller wishes to save +// specific slices of full tensors, "shape_and_slices" should be non-empty strings +// and correspondingly well-formed. +// +// Arguments: +// prefix: Must have a single element. The prefix of the V2 checkpoint to which we +// write the tensors. +// tensor_names: shape {N}. The names of the tensors to be saved. +// shape_and_slices: shape {N}. The slice specs of the tensors to be saved. +// Empty strings indicate that they are non-partitioned tensors. +// tensors: `N` tensors to save. +// +// Returns the created operation. +func SaveV2(scope *Scope, prefix tf.Output, tensor_names tf.Output, shape_and_slices tf.Output, tensors []tf.Output) (o *tf.Operation) { + if scope.Err() != nil { + return + } + opspec := tf.OpSpec{ + Type: "SaveV2", + Input: []tf.Input{ + prefix, tensor_names, shape_and_slices, tf.OutputList(tensors), + }, + } + return scope.AddOperation(opspec) +} + // UnicodeTranscodeAttr is an optional argument to UnicodeTranscode. type UnicodeTranscodeAttr func(optionalAttr) @@ -25011,6 +24964,75 @@ func Cumsum(scope *Scope, x tf.Output, axis tf.Output, optional ...CumsumAttr) ( return op.Output(0) } +// Return the shape of s0 op s1 with broadcast. +// +// Given `s0` and `s1`, tensors that represent shapes, compute `r0`, the +// broadcasted shape. `s0`, `s1` and `r0` are all integer vectors. +func BroadcastArgs(scope *Scope, s0 tf.Output, s1 tf.Output) (r0 tf.Output) { + if scope.Err() != nil { + return + } + opspec := tf.OpSpec{ + Type: "BroadcastArgs", + Input: []tf.Input{ + s0, s1, + }, + } + op := scope.AddOperation(opspec) + return op.Output(0) +} + +// DataFormatDimMapAttr is an optional argument to DataFormatDimMap. +type DataFormatDimMapAttr func(optionalAttr) + +// DataFormatDimMapSrcFormat sets the optional src_format attribute to value. +// +// value: source data format. +// If not specified, defaults to "NHWC" +func DataFormatDimMapSrcFormat(value string) DataFormatDimMapAttr { + return func(m optionalAttr) { + m["src_format"] = value + } +} + +// DataFormatDimMapDstFormat sets the optional dst_format attribute to value. +// +// value: destination data format. +// If not specified, defaults to "NCHW" +func DataFormatDimMapDstFormat(value string) DataFormatDimMapAttr { + return func(m optionalAttr) { + m["dst_format"] = value + } +} + +// Returns the dimension index in the destination data format given the one in +// +// the source data format. +// +// Arguments: +// x: A Tensor with each element as a dimension index in source data format. +// Must be in the range [-4, 4). +// +// Returns A Tensor with each element as a dimension index in destination data format. +func DataFormatDimMap(scope *Scope, x tf.Output, optional ...DataFormatDimMapAttr) (y tf.Output) { + if scope.Err() != nil { + return + } + attrs := map[string]interface{}{} + for _, a := range optional { + a(attrs) + } + opspec := tf.OpSpec{ + Type: "DataFormatDimMap", + Input: []tf.Input{ + x, + }, + Attrs: attrs, + } + op := scope.AddOperation(opspec) + return op.Output(0) +} + // CumprodAttr is an optional argument to Cumprod. type CumprodAttr func(optionalAttr) @@ -26322,24 +26344,6 @@ func TensorArrayReadV3(scope *Scope, handle tf.Output, index tf.Output, flow_in return op.Output(0) } -// Computes the gradient for the tanh of `x` wrt its input. -// -// Specifically, `grad = dy * (1 - y*y)`, where `y = tanh(x)`, and `dy` -// is the corresponding input gradient. -func TanhGrad(scope *Scope, y tf.Output, dy tf.Output) (z tf.Output) { - if scope.Err() != nil { - return - } - opspec := tf.OpSpec{ - Type: "TanhGrad", - Input: []tf.Input{ - y, dy, - }, - } - op := scope.AddOperation(opspec) - return op.Output(0) -} - // Reduces sparse updates into the variable referenced by `resource` using the `max` operation. // // This operation computes @@ -26381,6 +26385,24 @@ func ResourceScatterMax(scope *Scope, resource tf.Output, indices tf.Output, upd return scope.AddOperation(opspec) } +// Computes the gradient for the tanh of `x` wrt its input. +// +// Specifically, `grad = dy * (1 - y*y)`, where `y = tanh(x)`, and `dy` +// is the corresponding input gradient. +func TanhGrad(scope *Scope, y tf.Output, dy tf.Output) (z tf.Output) { + if scope.Err() != nil { + return + } + opspec := tf.OpSpec{ + Type: "TanhGrad", + Input: []tf.Input{ + y, dy, + }, + } + op := scope.AddOperation(opspec) + return op.Output(0) +} + // Outputs a `Summary` protocol buffer with scalar values. // // The input `tags` and `values` must have the same shape. The generated summary @@ -26669,24 +26691,6 @@ func MergeSummary(scope *Scope, inputs []tf.Output) (summary tf.Output) { return op.Output(0) } -// Returns the number of tensors in the input tensor list. -// -// input_handle: the input list -// length: the number of tensors in the list -func TensorListLength(scope *Scope, input_handle tf.Output) (length tf.Output) { - if scope.Err() != nil { - return - } - opspec := tf.OpSpec{ - Type: "TensorListLength", - Input: []tf.Input{ - input_handle, - }, - } - op := scope.AddOperation(opspec) - return op.Output(0) -} - // The shape of the elements of the given list, as a tensor. // // input_handle: the list @@ -27361,35 +27365,6 @@ func MatrixSquareRoot(scope *Scope, input tf.Output) (output tf.Output) { return op.Output(0) } -// Convert JSON-encoded Example records to binary protocol buffer strings. -// -// This op translates a tensor containing Example records, encoded using -// the [standard JSON -// mapping](https://developers.google.com/protocol-buffers/docs/proto3#json), -// into a tensor containing the same records encoded as binary protocol -// buffers. The resulting tensor can then be fed to any of the other -// Example-parsing ops. -// -// Arguments: -// json_examples: Each string is a JSON object serialized according to the JSON -// mapping of the Example proto. -// -// Returns Each string is a binary Example protocol buffer corresponding -// to the respective element of `json_examples`. -func DecodeJSONExample(scope *Scope, json_examples tf.Output) (binary_examples tf.Output) { - if scope.Err() != nil { - return - } - opspec := tf.OpSpec{ - Type: "DecodeJSONExample", - Input: []tf.Input{ - json_examples, - }, - } - op := scope.AddOperation(opspec) - return op.Output(0) -} - // SvdAttr is an optional argument to Svd. type SvdAttr func(optionalAttr) @@ -28426,6 +28401,49 @@ func ReaderReadUpToV2(scope *Scope, reader_handle tf.Output, queue_handle tf.Out return op.Output(0), op.Output(1) } +// Adds v into specified rows of x. +// +// Computes y = x; y[i, :] += v; return y. +// +// Arguments: +// x: A `Tensor` of type T. +// i: A vector. Indices into the left-most dimension of `x`. +// v: A `Tensor` of type T. Same dimension sizes as x except the first dimension, which must be the same as i's size. +// +// Returns A `Tensor` of type T. An alias of `x`. The content of `y` is undefined if there are duplicates in `i`. +func InplaceAdd(scope *Scope, x tf.Output, i tf.Output, v tf.Output) (y tf.Output) { + if scope.Err() != nil { + return + } + opspec := tf.OpSpec{ + Type: "InplaceAdd", + Input: []tf.Input{ + x, i, v, + }, + } + op := scope.AddOperation(opspec) + return op.Output(0) +} + +// Restore a Reader to its initial clean state. +// +// Arguments: +// reader_handle: Handle to a Reader. +// +// Returns the created operation. +func ReaderResetV2(scope *Scope, reader_handle tf.Output) (o *tf.Operation) { + if scope.Err() != nil { + return + } + opspec := tf.OpSpec{ + Type: "ReaderResetV2", + Input: []tf.Input{ + reader_handle, + }, + } + return scope.AddOperation(opspec) +} + // BatchAttr is an optional argument to Batch. type BatchAttr func(optionalAttr) @@ -29711,71 +29729,6 @@ func CropAndResizeGradBoxes(scope *Scope, grads tf.Output, image tf.Output, boxe return op.Output(0) } -// Saves tensors in V2 checkpoint format. -// -// By default, saves the named tensors in full. If the caller wishes to save -// specific slices of full tensors, "shape_and_slices" should be non-empty strings -// and correspondingly well-formed. -// -// Arguments: -// prefix: Must have a single element. The prefix of the V2 checkpoint to which we -// write the tensors. -// tensor_names: shape {N}. The names of the tensors to be saved. -// shape_and_slices: shape {N}. The slice specs of the tensors to be saved. -// Empty strings indicate that they are non-partitioned tensors. -// tensors: `N` tensors to save. -// -// Returns the created operation. -func SaveV2(scope *Scope, prefix tf.Output, tensor_names tf.Output, shape_and_slices tf.Output, tensors []tf.Output) (o *tf.Operation) { - if scope.Err() != nil { - return - } - opspec := tf.OpSpec{ - Type: "SaveV2", - Input: []tf.Input{ - prefix, tensor_names, shape_and_slices, tf.OutputList(tensors), - }, - } - return scope.AddOperation(opspec) -} - -// StatsAggregatorHandleAttr is an optional argument to StatsAggregatorHandle. -type StatsAggregatorHandleAttr func(optionalAttr) - -// StatsAggregatorHandleContainer sets the optional container attribute to value. -// If not specified, defaults to "" -func StatsAggregatorHandleContainer(value string) StatsAggregatorHandleAttr { - return func(m optionalAttr) { - m["container"] = value - } -} - -// StatsAggregatorHandleSharedName sets the optional shared_name attribute to value. -// If not specified, defaults to "" -func StatsAggregatorHandleSharedName(value string) StatsAggregatorHandleAttr { - return func(m optionalAttr) { - m["shared_name"] = value - } -} - -// Creates a statistics manager resource. -func StatsAggregatorHandle(scope *Scope, optional ...StatsAggregatorHandleAttr) (handle tf.Output) { - if scope.Err() != nil { - return - } - attrs := map[string]interface{}{} - for _, a := range optional { - a(attrs) - } - opspec := tf.OpSpec{ - Type: "StatsAggregatorHandle", - - Attrs: attrs, - } - op := scope.AddOperation(opspec) - return op.Output(0) -} - // Greedily selects a subset of bounding boxes in descending order of score, // // pruning away boxes that have high intersection-over-union (IOU) overlap @@ -29955,6 +29908,46 @@ func FakeParam(scope *Scope, dtype tf.DataType, shape tf.Shape) (output tf.Outpu return op.Output(0) } +// Computes the gradient for the inverse of `x` wrt its input. +// +// Specifically, `grad = -dy * y*y`, where `y = 1/x`, and `dy` +// is the corresponding input gradient. +func InvGrad(scope *Scope, y tf.Output, dy tf.Output) (z tf.Output) { + if scope.Err() != nil { + return + } + opspec := tf.OpSpec{ + Type: "InvGrad", + Input: []tf.Input{ + y, dy, + }, + } + op := scope.AddOperation(opspec) + return op.Output(0) +} + +// List of the given size with empty elements. +// +// element_shape: the shape of the future elements of the list +// num_elements: the number of elements to reserve +// handle: the output list +// element_dtype: the desired type of elements in the list. +func TensorListReserve(scope *Scope, element_shape tf.Output, num_elements tf.Output, element_dtype tf.DataType) (handle tf.Output) { + if scope.Err() != nil { + return + } + attrs := map[string]interface{}{"element_dtype": element_dtype} + opspec := tf.OpSpec{ + Type: "TensorListReserve", + Input: []tf.Input{ + element_shape, num_elements, + }, + Attrs: attrs, + } + op := scope.AddOperation(opspec) + return op.Output(0) +} + // A substitute for `InterleaveDataset` on a fixed list of `N` datasets. // // Arguments: @@ -29980,276 +29973,48 @@ func ExperimentalDirectedInterleaveDataset(scope *Scope, selector_input_dataset return op.Output(0) } -// Gets the next element from a FunctionBufferingResource. -// -// Arguments: -// function_buffer_resource: The FunctionBufferingResource handle. -// output_types: The type list for the return values. -// -// Returns A list of return values. -func ExperimentalFunctionBufferingResourceGetNext(scope *Scope, function_buffer_resource tf.Output, output_types []tf.DataType) (output []tf.Output) { - if scope.Err() != nil { - return - } - attrs := map[string]interface{}{"output_types": output_types} - opspec := tf.OpSpec{ - Type: "ExperimentalFunctionBufferingResourceGetNext", - Input: []tf.Input{ - function_buffer_resource, - }, - Attrs: attrs, - } - op := scope.AddOperation(opspec) - if scope.Err() != nil { - return - } - var idx int - var err error - if output, idx, err = makeOutputList(op, idx, "output"); err != nil { - scope.UpdateErr("ExperimentalFunctionBufferingResourceGetNext", err) - return - } - return output -} +// RandomUniformIntAttr is an optional argument to RandomUniformInt. +type RandomUniformIntAttr func(optionalAttr) -// Adds v into specified rows of x. +// RandomUniformIntSeed sets the optional seed attribute to value. // -// Computes y = x; y[i, :] += v; return y. -// -// Arguments: -// x: A `Tensor` of type T. -// i: A vector. Indices into the left-most dimension of `x`. -// v: A `Tensor` of type T. Same dimension sizes as x except the first dimension, which must be the same as i's size. -// -// Returns A `Tensor` of type T. An alias of `x`. The content of `y` is undefined if there are duplicates in `i`. -func InplaceAdd(scope *Scope, x tf.Output, i tf.Output, v tf.Output) (y tf.Output) { - if scope.Err() != nil { - return - } - opspec := tf.OpSpec{ - Type: "InplaceAdd", - Input: []tf.Input{ - x, i, v, - }, - } - op := scope.AddOperation(opspec) - return op.Output(0) -} - -// Restore a Reader to its initial clean state. -// -// Arguments: -// reader_handle: Handle to a Reader. -// -// Returns the created operation. -func ReaderResetV2(scope *Scope, reader_handle tf.Output) (o *tf.Operation) { - if scope.Err() != nil { - return - } - opspec := tf.OpSpec{ - Type: "ReaderResetV2", - Input: []tf.Input{ - reader_handle, - }, - } - return scope.AddOperation(opspec) -} - -// A dataset that splits the elements of its input into multiple elements. -func UnbatchDataset(scope *Scope, input_dataset tf.Output, output_types []tf.DataType, output_shapes []tf.Shape) (handle tf.Output) { - if scope.Err() != nil { - return - } - attrs := map[string]interface{}{"output_types": output_types, "output_shapes": output_shapes} - opspec := tf.OpSpec{ - Type: "UnbatchDataset", - Input: []tf.Input{ - input_dataset, - }, - Attrs: attrs, - } - op := scope.AddOperation(opspec) - return op.Output(0) -} - -// OrderedMapStageAttr is an optional argument to OrderedMapStage. -type OrderedMapStageAttr func(optionalAttr) - -// OrderedMapStageCapacity sets the optional capacity attribute to value. -// -// value: Maximum number of elements in the Staging Area. If > 0, inserts -// on the container will block when the capacity is reached. +// value: If either `seed` or `seed2` are set to be non-zero, the random number +// generator is seeded by the given seed. Otherwise, it is seeded by a +// random seed. // If not specified, defaults to 0 -// -// REQUIRES: value >= 0 -func OrderedMapStageCapacity(value int64) OrderedMapStageAttr { +func RandomUniformIntSeed(value int64) RandomUniformIntAttr { return func(m optionalAttr) { - m["capacity"] = value + m["seed"] = value } } -// OrderedMapStageMemoryLimit sets the optional memory_limit attribute to value. +// RandomUniformIntSeed2 sets the optional seed2 attribute to value. +// +// value: A second seed to avoid seed collision. // If not specified, defaults to 0 -// -// REQUIRES: value >= 0 -func OrderedMapStageMemoryLimit(value int64) OrderedMapStageAttr { +func RandomUniformIntSeed2(value int64) RandomUniformIntAttr { return func(m optionalAttr) { - m["memory_limit"] = value + m["seed2"] = value } } -// OrderedMapStageContainer sets the optional container attribute to value. +// Outputs random integers from a uniform distribution. // -// value: If non-empty, this queue is placed in the given container. Otherwise, -// a default container is used. -// If not specified, defaults to "" -func OrderedMapStageContainer(value string) OrderedMapStageAttr { - return func(m optionalAttr) { - m["container"] = value - } -} - -// OrderedMapStageSharedName sets the optional shared_name attribute to value. +// The generated values are uniform integers in the range `[minval, maxval)`. +// The lower bound `minval` is included in the range, while the upper bound +// `maxval` is excluded. // -// value: It is necessary to match this name to the matching Unstage Op. -// If not specified, defaults to "" -func OrderedMapStageSharedName(value string) OrderedMapStageAttr { - return func(m optionalAttr) { - m["shared_name"] = value - } -} - -// Stage (key, values) in the underlying container which behaves like a ordered -// -// associative container. Elements are ordered by key. +// The random integers are slightly biased unless `maxval - minval` is an exact +// power of two. The bias is small for values of `maxval - minval` significantly +// smaller than the range of the output (either `2^32` or `2^64`). // // Arguments: -// key: int64 +// shape: The shape of the output tensor. +// minval: 0-D. Inclusive lower bound on the generated integers. +// maxval: 0-D. Exclusive upper bound on the generated integers. // -// values: a list of tensors -// dtypes A list of data types that inserted values should adhere to. -// -// -// Returns the created operation. -func OrderedMapStage(scope *Scope, key tf.Output, indices tf.Output, values []tf.Output, dtypes []tf.DataType, optional ...OrderedMapStageAttr) (o *tf.Operation) { - if scope.Err() != nil { - return - } - attrs := map[string]interface{}{"dtypes": dtypes} - for _, a := range optional { - a(attrs) - } - opspec := tf.OpSpec{ - Type: "OrderedMapStage", - Input: []tf.Input{ - key, indices, tf.OutputList(values), - }, - Attrs: attrs, - } - return scope.AddOperation(opspec) -} - -// RpcAttr is an optional argument to Rpc. -type RpcAttr func(optionalAttr) - -// RpcProtocol sets the optional protocol attribute to value. -// -// value: RPC protocol to use. Empty string means use the default protocol. -// Options include 'grpc'. -// If not specified, defaults to "" -func RpcProtocol(value string) RpcAttr { - return func(m optionalAttr) { - m["protocol"] = value - } -} - -// RpcFailFast sets the optional fail_fast attribute to value. -// -// value: `boolean`. If `true` (default), then failures to connect -// (i.e., the server does not immediately respond) cause an RPC failure. -// If not specified, defaults to true -func RpcFailFast(value bool) RpcAttr { - return func(m optionalAttr) { - m["fail_fast"] = value - } -} - -// RpcTimeoutInMs sets the optional timeout_in_ms attribute to value. -// -// value: `int`. If `0` (default), then the kernel will run the RPC -// request and only time out if the RPC deadline passes or the session times out. -// If this value is greater than `0`, then the op will raise an exception if -// the RPC takes longer than `timeout_in_ms`. -// If not specified, defaults to 0 -func RpcTimeoutInMs(value int64) RpcAttr { - return func(m optionalAttr) { - m["timeout_in_ms"] = value - } -} - -// Perform batches of RPC requests. -// -// This op asynchronously performs either a single RPC request, or a batch -// of requests. RPC requests are defined by three main parameters: -// -// - `address` (the host+port or BNS address of the request) -// - `method` (the RPC method name for the request) -// - `request` (the serialized proto string, or vector of strings, -// of the RPC request argument). -// -// For example, if you have an RPC service running on port localhost:2345, -// and its interface is configured with the following proto declaration: -// -// ``` -// service MyService { -// rpc MyMethod(MyRequestProto) returns (MyResponseProto) { -// } -// }; -// ``` -// -// then call this op with arguments: -// -// ``` -// address = "localhost:2345" -// method = "MyService/MyMethod" -// ``` -// -// The `request` tensor is a string tensor representing serialized `MyRequestProto` -// strings; and the output string tensor `response` will have the same shape -// and contain (upon successful completion) corresponding serialized -// `MyResponseProto` strings. -// -// For example, to send a single, empty, `MyRequestProto`, call -// this op with `request = ""`. To send 5 **parallel** empty requests, -// call this op with `request = ["", "", "", "", ""]`. -// -// More generally, one can create a batch of `MyRequestProto` serialized protos -// from regular batched tensors using the `encode_proto` op, and convert -// the response `MyResponseProto` serialized protos to batched tensors -// using the `decode_proto` op. -// -// **NOTE** Working with serialized proto strings is faster than instantiating -// actual proto objects in memory, so no performance degradation is expected -// compared to writing custom kernels for this workflow. -// -// If the connection fails or the remote worker returns an error -// status, the op reraises this exception locally. -// -// See the `TryRpc` op if you prefer to handle RPC failures manually in the graph. -// -// Arguments: -// address: `0-D` or `1-D`. The address (i.e. host_name:port) of the RPC server. -// If this tensor has more than 1 element, then multiple parallel rpc requests -// are sent. This argument broadcasts with `method` and `request`. -// method: `0-D` or `1-D`. The method address on the RPC server. -// If this tensor has more than 1 element, then multiple parallel rpc requests -// are sent. This argument broadcasts with `address` and `request`. -// request: `0-D` or `1-D`. Serialized proto strings: the rpc request argument. -// If this tensor has more than 1 element, then multiple parallel rpc requests -// are sent. This argument broadcasts with `address` and `method`. -// -// Returns Same shape as `request`. Serialized proto strings: the rpc responses. -func Rpc(scope *Scope, address tf.Output, method tf.Output, request tf.Output, optional ...RpcAttr) (response tf.Output) { +// Returns A tensor of the specified shape filled with uniform random integers. +func RandomUniformInt(scope *Scope, shape tf.Output, minval tf.Output, maxval tf.Output, optional ...RandomUniformIntAttr) (output tf.Output) { if scope.Err() != nil { return } @@ -30258,9 +30023,9 @@ func Rpc(scope *Scope, address tf.Output, method tf.Output, request tf.Output, o a(attrs) } opspec := tf.OpSpec{ - Type: "Rpc", + Type: "RandomUniformInt", Input: []tf.Input{ - address, method, request, + shape, minval, maxval, }, Attrs: attrs, } @@ -30268,59 +30033,25 @@ func Rpc(scope *Scope, address tf.Output, method tf.Output, request tf.Output, o return op.Output(0) } -// StackPushV2Attr is an optional argument to StackPushV2. -type StackPushV2Attr func(optionalAttr) - -// StackPushV2SwapMemory sets the optional swap_memory attribute to value. +// Add the quantile summaries to each quantile stream resource. // -// value: Swap `elem` to CPU. Default to false. -// If not specified, defaults to false -func StackPushV2SwapMemory(value bool) StackPushV2Attr { - return func(m optionalAttr) { - m["swap_memory"] = value - } -} - -// Push an element onto the stack. +// An op that adds a list of quantile summaries to a quantile stream resource. Each +// summary Tensor is rank 2, containing summaries (value, weight, min_rank, max_rank) +// for a single feature. // // Arguments: -// handle: The handle to a stack. -// elem: The tensor to be pushed onto the stack. -// -// Returns The same tensor as the input 'elem'. -func StackPushV2(scope *Scope, handle tf.Output, elem tf.Output, optional ...StackPushV2Attr) (output tf.Output) { - if scope.Err() != nil { - return - } - attrs := map[string]interface{}{} - for _, a := range optional { - a(attrs) - } - opspec := tf.OpSpec{ - Type: "StackPushV2", - Input: []tf.Input{ - handle, elem, - }, - Attrs: attrs, - } - op := scope.AddOperation(opspec) - return op.Output(0) -} - -// Resets the FunctionBufferingResource. -// -// Arguments: -// function_buffer_resource: The FunctionBufferingResource handle. +// quantile_stream_resource_handle: resource handle referring to a QuantileStreamResource. +// summaries: string; List of Rank 2 Tensor each containing the summaries for a single feature. // // Returns the created operation. -func ExperimentalFunctionBufferingResourceReset(scope *Scope, function_buffer_resource tf.Output) (o *tf.Operation) { +func BoostedTreesQuantileStreamResourceAddSummaries(scope *Scope, quantile_stream_resource_handle tf.Output, summaries []tf.Output) (o *tf.Operation) { if scope.Err() != nil { return } opspec := tf.OpSpec{ - Type: "ExperimentalFunctionBufferingResourceReset", + Type: "BoostedTreesQuantileStreamResourceAddSummaries", Input: []tf.Input{ - function_buffer_resource, + quantile_stream_resource_handle, tf.OutputList(summaries), }, } return scope.AddOperation(opspec) @@ -30615,168 +30346,6 @@ func ConcatenateDataset(scope *Scope, input_dataset tf.Output, another_dataset t return op.Output(0) } -// Adds a value to the current value of a variable. -// -// Any ReadVariableOp with a control dependency on this op is guaranteed to -// see the incremented value or a subsequent newer one. -// -// Arguments: -// resource: handle to the resource in which to store the variable. -// value: the value by which the variable will be incremented. -// -// Returns the created operation. -func AssignAddVariableOp(scope *Scope, resource tf.Output, value tf.Output) (o *tf.Operation) { - if scope.Err() != nil { - return - } - opspec := tf.OpSpec{ - Type: "AssignAddVariableOp", - Input: []tf.Input{ - resource, value, - }, - } - return scope.AddOperation(opspec) -} - -// Records the latency of producing `input_dataset` elements in a StatsAggregator. -func LatencyStatsDataset(scope *Scope, input_dataset tf.Output, tag tf.Output, output_types []tf.DataType, output_shapes []tf.Shape) (handle tf.Output) { - if scope.Err() != nil { - return - } - attrs := map[string]interface{}{"output_types": output_types, "output_shapes": output_shapes} - opspec := tf.OpSpec{ - Type: "LatencyStatsDataset", - Input: []tf.Input{ - input_dataset, tag, - }, - Attrs: attrs, - } - op := scope.AddOperation(opspec) - return op.Output(0) -} - -// MapSizeAttr is an optional argument to MapSize. -type MapSizeAttr func(optionalAttr) - -// MapSizeCapacity sets the optional capacity attribute to value. -// If not specified, defaults to 0 -// -// REQUIRES: value >= 0 -func MapSizeCapacity(value int64) MapSizeAttr { - return func(m optionalAttr) { - m["capacity"] = value - } -} - -// MapSizeMemoryLimit sets the optional memory_limit attribute to value. -// If not specified, defaults to 0 -// -// REQUIRES: value >= 0 -func MapSizeMemoryLimit(value int64) MapSizeAttr { - return func(m optionalAttr) { - m["memory_limit"] = value - } -} - -// MapSizeContainer sets the optional container attribute to value. -// If not specified, defaults to "" -func MapSizeContainer(value string) MapSizeAttr { - return func(m optionalAttr) { - m["container"] = value - } -} - -// MapSizeSharedName sets the optional shared_name attribute to value. -// If not specified, defaults to "" -func MapSizeSharedName(value string) MapSizeAttr { - return func(m optionalAttr) { - m["shared_name"] = value - } -} - -// Op returns the number of elements in the underlying container. -func MapSize(scope *Scope, dtypes []tf.DataType, optional ...MapSizeAttr) (size tf.Output) { - if scope.Err() != nil { - return - } - attrs := map[string]interface{}{"dtypes": dtypes} - for _, a := range optional { - a(attrs) - } - opspec := tf.OpSpec{ - Type: "MapSize", - - Attrs: attrs, - } - op := scope.AddOperation(opspec) - return op.Output(0) -} - -// SparseToDenseAttr is an optional argument to SparseToDense. -type SparseToDenseAttr func(optionalAttr) - -// SparseToDenseValidateIndices sets the optional validate_indices attribute to value. -// -// value: If true, indices are checked to make sure they are sorted in -// lexicographic order and that there are no repeats. -// If not specified, defaults to true -func SparseToDenseValidateIndices(value bool) SparseToDenseAttr { - return func(m optionalAttr) { - m["validate_indices"] = value - } -} - -// Converts a sparse representation into a dense tensor. -// -// Builds an array `dense` with shape `output_shape` such that -// -// ``` -// # If sparse_indices is scalar -// dense[i] = (i == sparse_indices ? sparse_values : default_value) -// -// # If sparse_indices is a vector, then for each i -// dense[sparse_indices[i]] = sparse_values[i] -// -// # If sparse_indices is an n by d matrix, then for each i in [0, n) -// dense[sparse_indices[i][0], ..., sparse_indices[i][d-1]] = sparse_values[i] -// ``` -// -// All other values in `dense` are set to `default_value`. If `sparse_values` is a -// scalar, all sparse indices are set to this single value. -// -// Indices should be sorted in lexicographic order, and indices must not -// contain any repeats. If `validate_indices` is true, these properties -// are checked during execution. -// -// Arguments: -// sparse_indices: 0-D, 1-D, or 2-D. `sparse_indices[i]` contains the complete -// index where `sparse_values[i]` will be placed. -// output_shape: 1-D. Shape of the dense output tensor. -// sparse_values: 1-D. Values corresponding to each row of `sparse_indices`, -// or a scalar value to be used for all sparse indices. -// default_value: Scalar value to set for indices not specified in -// `sparse_indices`. -// -// Returns Dense output tensor of shape `output_shape`. -func SparseToDense(scope *Scope, sparse_indices tf.Output, output_shape tf.Output, sparse_values tf.Output, default_value tf.Output, optional ...SparseToDenseAttr) (dense tf.Output) { - if scope.Err() != nil { - return - } - attrs := map[string]interface{}{} - for _, a := range optional { - a(attrs) - } - opspec := tf.OpSpec{ - Type: "SparseToDense", - Input: []tf.Input{ - sparse_indices, output_shape, sparse_values, default_value, - }, - Attrs: attrs, - } - op := scope.AddOperation(opspec) - return op.Output(0) -} - // Computes the grayscale dilation of 4-D `input` and 3-D `filter` tensors. // // The `input` tensor has shape `[batch, in_height, in_width, depth]` and the @@ -30910,52 +30479,6 @@ func PaddedBatchDataset(scope *Scope, input_dataset tf.Output, batch_size tf.Out return op.Output(0) } -// Creates a dataset that batches input elements into a SparseTensor. -// -// Arguments: -// input_dataset: A handle to an input dataset. Must have a single component. -// batch_size: A scalar representing the number of elements to accumulate in a -// batch. -// row_shape: A vector representing the dense shape of each row in the produced -// SparseTensor. The shape may be partially specified, using `-1` to indicate -// that a particular dimension should use the maximum size of all batch elements. -// -// -func DenseToSparseBatchDataset(scope *Scope, input_dataset tf.Output, batch_size tf.Output, row_shape tf.Output, output_types []tf.DataType, output_shapes []tf.Shape) (handle tf.Output) { - if scope.Err() != nil { - return - } - attrs := map[string]interface{}{"output_types": output_types, "output_shapes": output_shapes} - opspec := tf.OpSpec{ - Type: "DenseToSparseBatchDataset", - Input: []tf.Input{ - input_dataset, batch_size, row_shape, - }, - Attrs: attrs, - } - op := scope.AddOperation(opspec) - return op.Output(0) -} - -// Deprecated. Use TensorArrayGradV3 -// -// DEPRECATED at GraphDef version 26: Use TensorArrayGradV3 -func TensorArrayGradV2(scope *Scope, handle tf.Output, flow_in tf.Output, source string) (grad_handle tf.Output) { - if scope.Err() != nil { - return - } - attrs := map[string]interface{}{"source": source} - opspec := tf.OpSpec{ - Type: "TensorArrayGradV2", - Input: []tf.Input{ - handle, flow_in, - }, - Attrs: attrs, - } - op := scope.AddOperation(opspec) - return op.Output(0) -} - // Creates a dataset that shuffles and repeats elements from `input_dataset` // // pseudorandomly. @@ -31561,46 +31084,6 @@ func FIFOQueueV2(scope *Scope, component_types []tf.DataType, optional ...FIFOQu return op.Output(0) } -// Produces a summary of any statistics recorded by the given statistics manager. -func StatsAggregatorSummary(scope *Scope, iterator tf.Output) (summary tf.Output) { - if scope.Err() != nil { - return - } - opspec := tf.OpSpec{ - Type: "StatsAggregatorSummary", - Input: []tf.Input{ - iterator, - }, - } - op := scope.AddOperation(opspec) - return op.Output(0) -} - -// Compute the pairwise cross product. -// -// `a` and `b` must be the same shape; they can either be simple 3-element vectors, -// or any shape where the innermost dimension is 3. In the latter case, each pair -// of corresponding 3-element vectors is cross-multiplied independently. -// -// Arguments: -// a: A tensor containing 3-element vectors. -// b: Another tensor, of same type and shape as `a`. -// -// Returns Pairwise cross product of the vectors in `a` and `b`. -func Cross(scope *Scope, a tf.Output, b tf.Output) (product tf.Output) { - if scope.Err() != nil { - return - } - opspec := tf.OpSpec{ - Type: "Cross", - Input: []tf.Input{ - a, b, - }, - } - op := scope.AddOperation(opspec) - return op.Output(0) -} - // Constructs an Optional variant from a tuple of tensors. func OptionalFromValue(scope *Scope, components []tf.Output) (optional tf.Output) { if scope.Err() != nil { @@ -31769,30 +31252,6 @@ func OptionalHasValue(scope *Scope, optional tf.Output) (has_value tf.Output) { return op.Output(0) } -// Creates a dataset that executes a SQL query and emits rows of the result set. -// -// Arguments: -// driver_name: The database type. Currently, the only supported type is 'sqlite'. -// data_source_name: A connection string to connect to the database. -// query: A SQL query to execute. -// -// -func SqlDataset(scope *Scope, driver_name tf.Output, data_source_name tf.Output, query tf.Output, output_types []tf.DataType, output_shapes []tf.Shape) (handle tf.Output) { - if scope.Err() != nil { - return - } - attrs := map[string]interface{}{"output_types": output_types, "output_shapes": output_shapes} - opspec := tf.OpSpec{ - Type: "SqlDataset", - Input: []tf.Input{ - driver_name, data_source_name, query, - }, - Attrs: attrs, - } - op := scope.AddOperation(opspec) - return op.Output(0) -} - // Returns the value stored in an Optional variant or raises an error if none exists. func OptionalGetValue(scope *Scope, optional tf.Output, output_types []tf.DataType, output_shapes []tf.Shape) (components []tf.Output) { if scope.Err() != nil { @@ -32208,46 +31667,6 @@ func ParallelDynamicStitch(scope *Scope, indices []tf.Output, data []tf.Output) return op.Output(0) } -// Computes the gradient for the inverse of `x` wrt its input. -// -// Specifically, `grad = -dy * y*y`, where `y = 1/x`, and `dy` -// is the corresponding input gradient. -func InvGrad(scope *Scope, y tf.Output, dy tf.Output) (z tf.Output) { - if scope.Err() != nil { - return - } - opspec := tf.OpSpec{ - Type: "InvGrad", - Input: []tf.Input{ - y, dy, - }, - } - op := scope.AddOperation(opspec) - return op.Output(0) -} - -// List of the given size with empty elements. -// -// element_shape: the shape of the future elements of the list -// num_elements: the number of elements to reserve -// handle: the output list -// element_dtype: the desired type of elements in the list. -func TensorListReserve(scope *Scope, element_shape tf.Output, num_elements tf.Output, element_dtype tf.DataType) (handle tf.Output) { - if scope.Err() != nil { - return - } - attrs := map[string]interface{}{"element_dtype": element_dtype} - opspec := tf.OpSpec{ - Type: "TensorListReserve", - Input: []tf.Input{ - element_shape, num_elements, - }, - Attrs: attrs, - } - op := scope.AddOperation(opspec) - return op.Output(0) -} - // PriorityQueueV2Attr is an optional argument to PriorityQueueV2. type PriorityQueueV2Attr func(optionalAttr) @@ -32786,6 +32205,241 @@ func StackV2(scope *Scope, max_size tf.Output, elem_type tf.DataType, optional . return op.Output(0) } +// OrderedMapStageAttr is an optional argument to OrderedMapStage. +type OrderedMapStageAttr func(optionalAttr) + +// OrderedMapStageCapacity sets the optional capacity attribute to value. +// +// value: Maximum number of elements in the Staging Area. If > 0, inserts +// on the container will block when the capacity is reached. +// If not specified, defaults to 0 +// +// REQUIRES: value >= 0 +func OrderedMapStageCapacity(value int64) OrderedMapStageAttr { + return func(m optionalAttr) { + m["capacity"] = value + } +} + +// OrderedMapStageMemoryLimit sets the optional memory_limit attribute to value. +// If not specified, defaults to 0 +// +// REQUIRES: value >= 0 +func OrderedMapStageMemoryLimit(value int64) OrderedMapStageAttr { + return func(m optionalAttr) { + m["memory_limit"] = value + } +} + +// OrderedMapStageContainer sets the optional container attribute to value. +// +// value: If non-empty, this queue is placed in the given container. Otherwise, +// a default container is used. +// If not specified, defaults to "" +func OrderedMapStageContainer(value string) OrderedMapStageAttr { + return func(m optionalAttr) { + m["container"] = value + } +} + +// OrderedMapStageSharedName sets the optional shared_name attribute to value. +// +// value: It is necessary to match this name to the matching Unstage Op. +// If not specified, defaults to "" +func OrderedMapStageSharedName(value string) OrderedMapStageAttr { + return func(m optionalAttr) { + m["shared_name"] = value + } +} + +// Stage (key, values) in the underlying container which behaves like a ordered +// +// associative container. Elements are ordered by key. +// +// Arguments: +// key: int64 +// +// values: a list of tensors +// dtypes A list of data types that inserted values should adhere to. +// +// +// Returns the created operation. +func OrderedMapStage(scope *Scope, key tf.Output, indices tf.Output, values []tf.Output, dtypes []tf.DataType, optional ...OrderedMapStageAttr) (o *tf.Operation) { + if scope.Err() != nil { + return + } + attrs := map[string]interface{}{"dtypes": dtypes} + for _, a := range optional { + a(attrs) + } + opspec := tf.OpSpec{ + Type: "OrderedMapStage", + Input: []tf.Input{ + key, indices, tf.OutputList(values), + }, + Attrs: attrs, + } + return scope.AddOperation(opspec) +} + +// RpcAttr is an optional argument to Rpc. +type RpcAttr func(optionalAttr) + +// RpcProtocol sets the optional protocol attribute to value. +// +// value: RPC protocol to use. Empty string means use the default protocol. +// Options include 'grpc'. +// If not specified, defaults to "" +func RpcProtocol(value string) RpcAttr { + return func(m optionalAttr) { + m["protocol"] = value + } +} + +// RpcFailFast sets the optional fail_fast attribute to value. +// +// value: `boolean`. If `true` (default), then failures to connect +// (i.e., the server does not immediately respond) cause an RPC failure. +// If not specified, defaults to true +func RpcFailFast(value bool) RpcAttr { + return func(m optionalAttr) { + m["fail_fast"] = value + } +} + +// RpcTimeoutInMs sets the optional timeout_in_ms attribute to value. +// +// value: `int`. If `0` (default), then the kernel will run the RPC +// request and only time out if the RPC deadline passes or the session times out. +// If this value is greater than `0`, then the op will raise an exception if +// the RPC takes longer than `timeout_in_ms`. +// If not specified, defaults to 0 +func RpcTimeoutInMs(value int64) RpcAttr { + return func(m optionalAttr) { + m["timeout_in_ms"] = value + } +} + +// Perform batches of RPC requests. +// +// This op asynchronously performs either a single RPC request, or a batch +// of requests. RPC requests are defined by three main parameters: +// +// - `address` (the host+port or BNS address of the request) +// - `method` (the RPC method name for the request) +// - `request` (the serialized proto string, or vector of strings, +// of the RPC request argument). +// +// For example, if you have an RPC service running on port localhost:2345, +// and its interface is configured with the following proto declaration: +// +// ``` +// service MyService { +// rpc MyMethod(MyRequestProto) returns (MyResponseProto) { +// } +// }; +// ``` +// +// then call this op with arguments: +// +// ``` +// address = "localhost:2345" +// method = "MyService/MyMethod" +// ``` +// +// The `request` tensor is a string tensor representing serialized `MyRequestProto` +// strings; and the output string tensor `response` will have the same shape +// and contain (upon successful completion) corresponding serialized +// `MyResponseProto` strings. +// +// For example, to send a single, empty, `MyRequestProto`, call +// this op with `request = ""`. To send 5 **parallel** empty requests, +// call this op with `request = ["", "", "", "", ""]`. +// +// More generally, one can create a batch of `MyRequestProto` serialized protos +// from regular batched tensors using the `encode_proto` op, and convert +// the response `MyResponseProto` serialized protos to batched tensors +// using the `decode_proto` op. +// +// **NOTE** Working with serialized proto strings is faster than instantiating +// actual proto objects in memory, so no performance degradation is expected +// compared to writing custom kernels for this workflow. +// +// If the connection fails or the remote worker returns an error +// status, the op reraises this exception locally. +// +// See the `TryRpc` op if you prefer to handle RPC failures manually in the graph. +// +// Arguments: +// address: `0-D` or `1-D`. The address (i.e. host_name:port) of the RPC server. +// If this tensor has more than 1 element, then multiple parallel rpc requests +// are sent. This argument broadcasts with `method` and `request`. +// method: `0-D` or `1-D`. The method address on the RPC server. +// If this tensor has more than 1 element, then multiple parallel rpc requests +// are sent. This argument broadcasts with `address` and `request`. +// request: `0-D` or `1-D`. Serialized proto strings: the rpc request argument. +// If this tensor has more than 1 element, then multiple parallel rpc requests +// are sent. This argument broadcasts with `address` and `method`. +// +// Returns Same shape as `request`. Serialized proto strings: the rpc responses. +func Rpc(scope *Scope, address tf.Output, method tf.Output, request tf.Output, optional ...RpcAttr) (response tf.Output) { + if scope.Err() != nil { + return + } + attrs := map[string]interface{}{} + for _, a := range optional { + a(attrs) + } + opspec := tf.OpSpec{ + Type: "Rpc", + Input: []tf.Input{ + address, method, request, + }, + Attrs: attrs, + } + op := scope.AddOperation(opspec) + return op.Output(0) +} + +// StackPushV2Attr is an optional argument to StackPushV2. +type StackPushV2Attr func(optionalAttr) + +// StackPushV2SwapMemory sets the optional swap_memory attribute to value. +// +// value: Swap `elem` to CPU. Default to false. +// If not specified, defaults to false +func StackPushV2SwapMemory(value bool) StackPushV2Attr { + return func(m optionalAttr) { + m["swap_memory"] = value + } +} + +// Push an element onto the stack. +// +// Arguments: +// handle: The handle to a stack. +// elem: The tensor to be pushed onto the stack. +// +// Returns The same tensor as the input 'elem'. +func StackPushV2(scope *Scope, handle tf.Output, elem tf.Output, optional ...StackPushV2Attr) (output tf.Output) { + if scope.Err() != nil { + return + } + attrs := map[string]interface{}{} + for _, a := range optional { + a(attrs) + } + opspec := tf.OpSpec{ + Type: "StackPushV2", + Input: []tf.Input{ + handle, elem, + }, + Attrs: attrs, + } + op := scope.AddOperation(opspec) + return op.Output(0) +} + // FusedBatchNormGradV2Attr is an optional argument to FusedBatchNormGradV2. type FusedBatchNormGradV2Attr func(optionalAttr) @@ -33169,6 +32823,25 @@ func TensorArraySizeV3(scope *Scope, handle tf.Output, flow_in tf.Output) (size return op.Output(0) } +// Deprecated. Use TensorArrayGradV3 +// +// DEPRECATED at GraphDef version 26: Use TensorArrayGradV3 +func TensorArrayGradV2(scope *Scope, handle tf.Output, flow_in tf.Output, source string) (grad_handle tf.Output) { + if scope.Err() != nil { + return + } + attrs := map[string]interface{}{"source": source} + opspec := tf.OpSpec{ + Type: "TensorArrayGradV2", + Input: []tf.Input{ + handle, flow_in, + }, + Attrs: attrs, + } + op := scope.AddOperation(opspec) + return op.Output(0) +} + // SparseReduceMaxAttr is an optional argument to SparseReduceMax. type SparseReduceMaxAttr func(optionalAttr) @@ -33941,47 +33614,47 @@ func MapUnstage(scope *Scope, key tf.Output, indices tf.Output, dtypes []tf.Data return values } -// MapIncompleteSizeAttr is an optional argument to MapIncompleteSize. -type MapIncompleteSizeAttr func(optionalAttr) +// MapSizeAttr is an optional argument to MapSize. +type MapSizeAttr func(optionalAttr) -// MapIncompleteSizeCapacity sets the optional capacity attribute to value. +// MapSizeCapacity sets the optional capacity attribute to value. // If not specified, defaults to 0 // // REQUIRES: value >= 0 -func MapIncompleteSizeCapacity(value int64) MapIncompleteSizeAttr { +func MapSizeCapacity(value int64) MapSizeAttr { return func(m optionalAttr) { m["capacity"] = value } } -// MapIncompleteSizeMemoryLimit sets the optional memory_limit attribute to value. +// MapSizeMemoryLimit sets the optional memory_limit attribute to value. // If not specified, defaults to 0 // // REQUIRES: value >= 0 -func MapIncompleteSizeMemoryLimit(value int64) MapIncompleteSizeAttr { +func MapSizeMemoryLimit(value int64) MapSizeAttr { return func(m optionalAttr) { m["memory_limit"] = value } } -// MapIncompleteSizeContainer sets the optional container attribute to value. +// MapSizeContainer sets the optional container attribute to value. // If not specified, defaults to "" -func MapIncompleteSizeContainer(value string) MapIncompleteSizeAttr { +func MapSizeContainer(value string) MapSizeAttr { return func(m optionalAttr) { m["container"] = value } } -// MapIncompleteSizeSharedName sets the optional shared_name attribute to value. +// MapSizeSharedName sets the optional shared_name attribute to value. // If not specified, defaults to "" -func MapIncompleteSizeSharedName(value string) MapIncompleteSizeAttr { +func MapSizeSharedName(value string) MapSizeAttr { return func(m optionalAttr) { m["shared_name"] = value } } -// Op returns the number of incomplete elements in the underlying container. -func MapIncompleteSize(scope *Scope, dtypes []tf.DataType, optional ...MapIncompleteSizeAttr) (size tf.Output) { +// Op returns the number of elements in the underlying container. +func MapSize(scope *Scope, dtypes []tf.DataType, optional ...MapSizeAttr) (size tf.Output) { if scope.Err() != nil { return } @@ -33990,7 +33663,7 @@ func MapIncompleteSize(scope *Scope, dtypes []tf.DataType, optional ...MapIncomp a(attrs) } opspec := tf.OpSpec{ - Type: "MapIncompleteSize", + Type: "MapSize", Attrs: attrs, } diff --git a/tensorflow/java/README.md b/tensorflow/java/README.md index 2fa81ed88f6..64152907454 100644 --- a/tensorflow/java/README.md +++ b/tensorflow/java/README.md @@ -1,7 +1,7 @@ # TensorFlow for Java > *WARNING*: The TensorFlow Java API is not currently covered by the TensorFlow -> [API stability guarantees](https://www.tensorflow.org/guide/version_semantics). +> [API stability guarantees](https://www.tensorflow.org/guide/version_compat). > > For using TensorFlow on Android refer instead to > [contrib/android](https://www.tensorflow.org/code/tensorflow/contrib/android), diff --git a/tensorflow/java/src/test/java/org/tensorflow/TensorTest.java b/tensorflow/java/src/test/java/org/tensorflow/TensorTest.java index 1bd00a763dd..3229cce2776 100644 --- a/tensorflow/java/src/test/java/org/tensorflow/TensorTest.java +++ b/tensorflow/java/src/test/java/org/tensorflow/TensorTest.java @@ -21,6 +21,7 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; +import java.nio.Buffer; import java.nio.ByteBuffer; import java.nio.ByteOrder; import java.nio.DoubleBuffer; @@ -100,7 +101,7 @@ public class TensorTest { : ByteOrder.LITTLE_ENDIAN) .asDoubleBuffer() .put(doubles); - buf.flip(); + flipBuffer(buf); try (Tensor t = Tensor.create(new long[] {doubles.length}, buf)) { double[] actual = new double[doubles.length]; assertArrayEquals(doubles, t.copyTo(actual), EPSILON); @@ -179,30 +180,30 @@ public class TensorTest { { ByteBuffer bbuf = ByteBuffer.allocate(1024).order(ByteOrder.nativeOrder()); - bbuf.clear(); // FLOAT + clearBuffer(bbuf); // FLOAT tfloats.writeTo(bbuf); assertEquals(tfloats.numBytes(), bbuf.position()); - bbuf.flip(); + flipBuffer(bbuf); assertEquals(floats[0], bbuf.asFloatBuffer().get(0), EPSILON); - bbuf.clear(); // DOUBLE + clearBuffer(bbuf); // DOUBLE tdoubles.writeTo(bbuf); assertEquals(tdoubles.numBytes(), bbuf.position()); - bbuf.flip(); + flipBuffer(bbuf); assertEquals(doubles[0], bbuf.asDoubleBuffer().get(0), EPSILON); - bbuf.clear(); // INT32 + clearBuffer(bbuf); // INT32 tints.writeTo(bbuf); assertEquals(tints.numBytes(), bbuf.position()); - bbuf.flip(); + flipBuffer(bbuf); assertEquals(ints[0], bbuf.asIntBuffer().get(0)); - bbuf.clear(); // INT64 + clearBuffer(bbuf); // INT64 tlongs.writeTo(bbuf); assertEquals(tlongs.numBytes(), bbuf.position()); - bbuf.flip(); + flipBuffer(bbuf); assertEquals(longs[0], bbuf.asLongBuffer().get(0)); - bbuf.clear(); // BOOL + clearBuffer(bbuf); // BOOL tbools.writeTo(bbuf); assertEquals(tbools.numBytes(), bbuf.position()); - bbuf.flip(); + flipBuffer(bbuf); assertEquals(bools[0], bbuf.get(0) != 0); } @@ -254,7 +255,7 @@ public class TensorTest { : ByteOrder.LITTLE_ENDIAN) .asDoubleBuffer(); tdoubles.writeTo(foreignBuf); - foreignBuf.flip(); + flipBuffer(foreignBuf); double[] actual = new double[foreignBuf.remaining()]; foreignBuf.get(actual); assertArrayEquals(doubles, actual, EPSILON); @@ -547,4 +548,25 @@ public class TensorTest { // expected. } } + + // Workaround for cross compiliation + // (e.g., javac -source 1.9 -target 1.8). + // + // In Java 8 and prior, subclasses of java.nio.Buffer (e.g., java.nio.DoubleBuffer) inherited the + // "flip()" and "clear()" methods from java.nio.Buffer resulting in the signature: + // Buffer flip(); + // In Java 9 these subclasses had their own methods like: + // DoubleBuffer flip(); + // As a result, compiling for 1.9 source for a target of JDK 1.8 would result in errors at runtime + // like: + // + // java.lang.NoSuchMethodError: java.nio.DoubleBuffer.flip()Ljava/nio/DoubleBuffer + private static void flipBuffer(Buffer buf) { + buf.flip(); + } + + // See comment for flipBuffer() + private static void clearBuffer(Buffer buf) { + buf.clear(); + } } diff --git a/tensorflow/lite/BUILD b/tensorflow/lite/BUILD index be84fc5db1f..bb2c53b8c9e 100644 --- a/tensorflow/lite/BUILD +++ b/tensorflow/lite/BUILD @@ -131,6 +131,7 @@ cc_library( name = "framework", srcs = [ "allocation.cc", + "core/subgraph.cc", "graph_info.cc", "interpreter.cc", "model.cc", @@ -155,6 +156,7 @@ cc_library( "allocation.h", "context.h", "context_util.h", + "core/subgraph.h", "error_reporter.h", "graph_info.h", "interpreter.h", diff --git a/tensorflow/lite/build_def.bzl b/tensorflow/lite/build_def.bzl index bc98dc57bc5..5eaf7194949 100644 --- a/tensorflow/lite/build_def.bzl +++ b/tensorflow/lite/build_def.bzl @@ -221,6 +221,7 @@ def json_to_tflite(name, src, out): # generated_test_models_failing(). def generated_test_models(): return [ + "abs", "add", "arg_min_max", "avg_pool", @@ -236,18 +237,21 @@ def generated_test_models(): "equal", "exp", "expand_dims", + "fill", "floor", "floor_div", "floor_mod", "fully_connected", "fused_batch_norm", "gather", + "gather_buggy", "global_batch_norm", "greater", "greater_equal", "sum", "l2norm", "l2_pool", + "leaky_relu", "less", "less_equal", "local_response_norm", @@ -261,6 +265,7 @@ def generated_test_models(): "maximum", "mean", "minimum", + "mirror_pad", "mul", "neg", "not_equal", @@ -268,6 +273,7 @@ def generated_test_models(): "pack", "pad", "padv2", + "placeholder_with_default", "prelu", "pow", "range", @@ -290,17 +296,21 @@ def generated_test_models(): "space_to_depth", "sparse_to_dense", "split", + "splitv", "sqrt", "square", + "squared_difference", "squeeze", "strided_slice", "strided_slice_1d_exhaustive", + "strided_slice_buggy", "sub", "tile", "topk", "transpose", "transpose_conv", "unpack", + "unroll_batch_matmul", "where", "zeros_like", ] diff --git a/tensorflow/lite/builtin_ops.h b/tensorflow/lite/builtin_ops.h index b8c05f57bb5..f97d3ac4bf0 100644 --- a/tensorflow/lite/builtin_ops.h +++ b/tensorflow/lite/builtin_ops.h @@ -123,6 +123,11 @@ typedef enum { kTfLiteBuiltinFloorMod = 95, kTfLiteBuiltinRange = 96, kTfLiteBuiltinResizeNearestNeighbor = 97, + kTfLiteBuiltinLeakyRelu = 98, + kTfLiteBuiltinSquaredDifference = 99, + kTfLiteBuiltinMirrorPad = 100, + kTfLiteBuiltinAbs = 101, + kTfLiteBuiltinSplitV = 102, } TfLiteBuiltinOperator; #ifdef __cplusplus diff --git a/tensorflow/lite/c/builtin_op_data.h b/tensorflow/lite/c/builtin_op_data.h index 855983d60df..6a5a027a9dc 100644 --- a/tensorflow/lite/c/builtin_op_data.h +++ b/tensorflow/lite/c/builtin_op_data.h @@ -35,11 +35,21 @@ typedef enum { kTfLitePaddingValid, } TfLitePadding; +typedef enum { + kTfLiteMirrorPaddingUnknown = 0, + kTfLiteMirrorPaddingReflect, + kTfLiteMirrorPaddingSymmetric, +} TfLiteMirrorPaddingMode; + typedef struct { int width; int height; } TfLitePaddingValues; +typedef struct { + TfLiteMirrorPaddingMode mode; +} TfLiteMirrorPaddingParams; + // Possible fused activation functions. // TODO(aselle): rename to TfLiteActivation typedef enum { @@ -267,6 +277,10 @@ typedef struct { int num_splits; } TfLiteSplitParams; +typedef struct { + int num_splits; +} TfLiteSplitVParams; + typedef struct { // TODO(ahentz): We can't have dynamic data in this struct, at least not yet. // For now we will fix the maximum possible number of dimensions. @@ -328,6 +342,10 @@ typedef struct { int axis; } TfLiteUnpackParams; +typedef struct { + float alpha; +} TfLiteLeakyReluParams; + #ifdef __cplusplus } // extern "C" #endif // __cplusplus diff --git a/tensorflow/lite/c/builtin_op_data_test.cc b/tensorflow/lite/c/builtin_op_data_test.cc index 0e33dcd8c84..4ce7c481e1c 100644 --- a/tensorflow/lite/c/builtin_op_data_test.cc +++ b/tensorflow/lite/c/builtin_op_data_test.cc @@ -63,6 +63,7 @@ TEST(IntArray, CanCompileStructs) { TfLiteTransposeParams transpose_params; TfLiteReducerParams reducer_params; TfLiteSplitParams split_params; + TfLiteSplitVParams split_v_params; TfLiteSqueezeParams squeeze_params; TfLiteStridedSliceParams strided_slice_params; TfLiteArgMaxParams arg_max_params; diff --git a/tensorflow/lite/c/c_api_internal.c b/tensorflow/lite/c/c_api_internal.c index b131f067746..2923dbad4ef 100644 --- a/tensorflow/lite/c/c_api_internal.c +++ b/tensorflow/lite/c/c_api_internal.c @@ -59,7 +59,7 @@ void TfLiteIntArrayPrint(const char* s, TfLiteIntArray* a) { printf("]\n"); } -TfLiteIntArray* TfLiteIntArrayCopy(TfLiteIntArray* src) { +TfLiteIntArray* TfLiteIntArrayCopy(const TfLiteIntArray* src) { if (!src) return NULL; TfLiteIntArray* ret = TfLiteIntArrayCreate(src->size); if (ret) { @@ -125,6 +125,8 @@ const char* TfLiteTypeGetName(TfLiteType type) { return "INT32"; case kTfLiteUInt8: return "UINT8"; + case kTfLiteInt8: + return "INT8"; case kTfLiteInt64: return "INT64"; case kTfLiteBool: @@ -137,3 +139,14 @@ const char* TfLiteTypeGetName(TfLiteType type) { return "Unknown type"; } +TfLiteDelegate TfLiteDelegateCreate() { + TfLiteDelegate d = { + .data_ = NULL, + .Prepare = NULL, + .CopyFromBufferHandle = NULL, + .CopyToBufferHandle = NULL, + .FreeBufferHandle = NULL, + .flags = kTfLiteDelegateFlagsNone, + }; + return d; +} diff --git a/tensorflow/lite/c/c_api_internal.h b/tensorflow/lite/c/c_api_internal.h index e05fd19936e..1cd84eff5c4 100644 --- a/tensorflow/lite/c/c_api_internal.h +++ b/tensorflow/lite/c/c_api_internal.h @@ -96,7 +96,7 @@ int TfLiteIntArrayEqualsArray(TfLiteIntArray* a, int b_size, int b_data[]); // Create a copy of an array passed as `src`. // You are expected to free memory with TfLiteIntArrayFree -TfLiteIntArray* TfLiteIntArrayCopy(TfLiteIntArray* src); +TfLiteIntArray* TfLiteIntArrayCopy(const TfLiteIntArray* src); // Free memory of array `v`. void TfLiteIntArrayFree(TfLiteIntArray* v); @@ -179,6 +179,7 @@ typedef enum { kTfLiteBool = 6, kTfLiteInt16 = 7, kTfLiteComplex64 = 8, + kTfLiteInt8 = 9, } TfLiteType; // Return the name of a given type, for error reporting purposes. @@ -203,6 +204,7 @@ typedef union { bool* b; int16_t* i16; TfLiteComplex64* c64; + int8_t* int8; } TfLitePtrUnion; // Memory allocation strategies. kTfLiteMmapRo is for read-only memory-mapped @@ -486,19 +488,20 @@ typedef struct _TfLiteDelegate { // delegated subgraphs of the original graph. TfLiteStatus (*Prepare)(TfLiteContext* context, TfLiteDelegate* delegate); - // Copy the data from delegate buffer handle to raw memory. - // This can be null if the delegate doesn't use its own buffer. + // Copy the data from delegate buffer handle into raw memory of the given + // 'tensor'. This cannot be null. The delegate is allowed to allocate the raw + // bytes as long as it follows the rules for kTfLiteDynamic tensors. TfLiteStatus (*CopyFromBufferHandle)(TfLiteContext* context, TfLiteDelegate* delegate, TfLiteBufferHandle buffer_handle, - void* data, size_t size); + TfLiteTensor* tensor); - // Copy the data from raw memory to delegate buffer handle. - // This can be null if the delegate doesn't use its own buffer. + // Copy the data from raw memory of the given 'tensor' to delegate buffer + // handle. This can be null if the delegate doesn't use its own buffer. TfLiteStatus (*CopyToBufferHandle)(TfLiteContext* context, TfLiteDelegate* delegate, TfLiteBufferHandle buffer_handle, - void* data, size_t size); + TfLiteTensor* tensor); // Free the Delegate Buffer Handle. Note: This only frees the handle, but // this doesn't release the underlying resource (e.g. textures). The @@ -511,6 +514,10 @@ typedef struct _TfLiteDelegate { int64_t flags; } TfLiteDelegate; +// Build a 'null' delegate, with all the fields properly set to their default +// values. +TfLiteDelegate TfLiteDelegateCreate(); + // WARNING: This is an experimental interface that is subject to change. // // Currently, TfLiteDelegateParams has to be allocated in a way that it's diff --git a/tensorflow/lite/c/c_api_internal_test.cc b/tensorflow/lite/c/c_api_internal_test.cc index e21823c41f0..acf0dfc5be8 100644 --- a/tensorflow/lite/c/c_api_internal_test.cc +++ b/tensorflow/lite/c/c_api_internal_test.cc @@ -74,6 +74,7 @@ TEST(Types, TestTypeNames) { EXPECT_EQ(type_name(kTfLiteInt16), "INT16"); EXPECT_EQ(type_name(kTfLiteInt32), "INT32"); EXPECT_EQ(type_name(kTfLiteUInt8), "UINT8"); + EXPECT_EQ(type_name(kTfLiteInt8), "INT8"); EXPECT_EQ(type_name(kTfLiteInt64), "INT64"); EXPECT_EQ(type_name(kTfLiteBool), "BOOL"); EXPECT_EQ(type_name(kTfLiteComplex64), "COMPLEX64"); diff --git a/tensorflow/lite/core/api/flatbuffer_conversions.cc b/tensorflow/lite/core/api/flatbuffer_conversions.cc index 8cd3faabb72..c00a0a3a546 100644 --- a/tensorflow/lite/core/api/flatbuffer_conversions.cc +++ b/tensorflow/lite/core/api/flatbuffer_conversions.cc @@ -61,6 +61,9 @@ TfLiteStatus ConvertTensorType(TensorType tensor_type, TfLiteType* type, case TensorType_UINT8: *type = kTfLiteUInt8; break; + case TensorType_INT8: + *type = kTfLiteInt8; + break; case TensorType_INT64: *type = kTfLiteInt64; break; @@ -503,6 +506,14 @@ TfLiteStatus ParseOpData(const Operator* op, BuiltinOperator op_type, *builtin_data = reinterpret_cast(params); break; } + case BuiltinOperator_SPLIT_V: { + auto* params = allocator->AllocatePOD(); + if (auto* schema_params = op->builtin_options_as_SplitVOptions()) { + params->num_splits = schema_params->num_splits(); + } + *builtin_data = reinterpret_cast(params); + break; + } case BuiltinOperator_SQUEEZE: { auto* params = allocator->AllocatePOD(); if (auto* schema_params = op->builtin_options_as_SqueezeOptions()) { @@ -617,8 +628,31 @@ TfLiteStatus ParseOpData(const Operator* op, BuiltinOperator op_type, *builtin_data = reinterpret_cast(params); break; } + case BuiltinOperator_LEAKY_RELU: { + TfLiteLeakyReluParams* params = + allocator->AllocatePOD(); + if (auto* leaky_relu_params = op->builtin_options_as_LeakyReluOptions()) { + params->alpha = leaky_relu_params->alpha(); + } + *builtin_data = reinterpret_cast(params); + break; + } + case BuiltinOperator_MIRROR_PAD: { + TfLiteMirrorPaddingParams* params = + allocator->AllocatePOD(); + auto* mirror_pad_params = op->builtin_options_as_MirrorPadOptions(); + if (mirror_pad_params != nullptr) { + params->mode = + mirror_pad_params->mode() == tflite::MirrorPadMode_REFLECT + ? TfLiteMirrorPaddingMode::kTfLiteMirrorPaddingReflect + : TfLiteMirrorPaddingMode::kTfLiteMirrorPaddingSymmetric; + } + *builtin_data = reinterpret_cast(params); + break; + } // Below are the ops with no builtin_data strcture. + case BuiltinOperator_ABS: case BuiltinOperator_BATCH_TO_SPACE_ND: // TODO(aselle): Implement call in BuiltinOptions, but nullptrs are // ok for now, since there is no call implementation either. @@ -668,6 +702,7 @@ TfLiteStatus ParseOpData(const Operator* op, BuiltinOperator op_type, case BuiltinOperator_FILL: case BuiltinOperator_FLOOR_MOD: case BuiltinOperator_RANGE: + case BuiltinOperator_SQUARED_DIFFERENCE: break; } return kTfLiteOk; diff --git a/tensorflow/lite/core/subgraph.cc b/tensorflow/lite/core/subgraph.cc new file mode 100644 index 00000000000..90361faeae3 --- /dev/null +++ b/tensorflow/lite/core/subgraph.cc @@ -0,0 +1,970 @@ +/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/lite/core/subgraph.h" +#include "tensorflow/lite/arena_planner.h" +#include "tensorflow/lite/c/c_api_internal.h" +#include "tensorflow/lite/context_util.h" +#include "tensorflow/lite/graph_info.h" +#include "tensorflow/lite/nnapi_delegate.h" +#include "tensorflow/lite/schema/schema_generated.h" + +namespace tflite { + +namespace { +TfLiteStatus ReportOpError(TfLiteContext* context, const TfLiteNode& node, + const TfLiteRegistration& registration, + int node_index, const char* message) { + context->ReportError( + context, "Node number %d (%s) %s.\n", node_index, + registration.custom_name + ? registration.custom_name + : EnumNameBuiltinOperator( + static_cast(registration.builtin_code)), + message); + return kTfLiteError; +} + +// Stub method which returns kTfLiteError when the function is forbidden. +// We're registrating this function to several different function to save +// compiled binary size. Please note the restrictions: +// * The type of first parameter have to be `TfLiteContext*`. +// * All paramteters must be trivailly destructible. (E.g. No C++ class) +TfLiteStatus ForbiddenContextFunction(TfLiteContext* context, ...) { + context->ReportError(context, + "The function is forbidden if not calling in delegate."); + return kTfLiteError; +} + +// Set the ForbiddenContextFunction to a compatible function pointer. +template +void SetForbiddenContextFunction(FunctionType* func) { + *func = reinterpret_cast(ForbiddenContextFunction); +} + +// Returns true if at least one tensor in the given list is kTfLiteDynamic. +template +bool HasDynamicTensorImpl(const TfLiteContext& context, + const TensorIntArray& int_array) { + for (int i : int_array) { + const TfLiteTensor& tensor = context.tensors[i]; + if (tensor.allocation_type == kTfLiteDynamic) { + return true; + } + } + return false; +} + +bool HasDynamicTensor(const TfLiteContext& context, + const TfLiteIntArray* int_array) { + return HasDynamicTensorImpl(context, TfLiteIntArrayView{int_array}); +} + +} // namespace + +// A trivial implementation of GraphInfo around the Interpreter. +// NOTE: this interpreter info represents the subset of the +// graph that is executed according to execution plan. Thus, +// the indices are execution plan indices rather than raw node +// indices. +class InterpreterInfo : public GraphInfo { + public: + explicit InterpreterInfo(Subgraph* subgraph) : subgraph_(subgraph) {} + + size_t num_tensors() const override { return subgraph_->tensors().size(); } + TfLiteTensor* tensor(size_t index) override { + return &subgraph_->tensors()[index]; + } + size_t num_nodes() const override { + return subgraph_->execution_plan().size(); + } + const TfLiteNode& node(size_t index) const override { + int node_index = subgraph_->execution_plan()[index]; + return subgraph_->nodes_and_registration()[node_index].first; + } + const std::vector& inputs() const override { + return subgraph_->inputs(); + } + const std::vector& outputs() const override { + return subgraph_->outputs(); + } + const std::vector& variables() const override { + return subgraph_->variables(); + } + + public: + Subgraph* subgraph_; +}; + +Subgraph::Subgraph(ErrorReporter* error_reporter, + TfLiteExternalContext** external_contexts, + std::vector>* subgraphs) + : context_(&owned_context_), + error_reporter_(error_reporter), + next_execution_plan_index_to_prepare_(0), + external_contexts_(external_contexts), + subgraphs_(subgraphs) { + context_->impl_ = static_cast(this); + context_->ResizeTensor = ResizeTensor; + context_->ReportError = ReportErrorC; + context_->AddTensors = AddTensors; + context_->tensors = nullptr; + context_->tensors_size = 0; + context_->allow_fp32_relax_to_fp16 = false; + context_->recommended_num_threads = -1; + context_->GetExternalContext = GetExternalContext; + context_->SetExternalContext = SetExternalContext; + + // Reserve some space for the tensors to avoid excessive resizing. + tensors_.reserve(kTensorsReservedCapacity); + nodes_and_registration().reserve(kTensorsReservedCapacity); + // Invalid to call these these except from TfLiteDelegate + SwitchToKernelContext(); +} + +Subgraph::~Subgraph() { + for (auto& node_and_reg : nodes_and_registration_) { + TfLiteNode& node = node_and_reg.first; + TfLiteIntArrayFree(node.inputs); + TfLiteIntArrayFree(node.outputs); + TfLiteIntArrayFree(node.temporaries); + if (node.builtin_data) free(node.builtin_data); + OpFree(node_and_reg.second, node.user_data); + node.builtin_data = nullptr; + } + + for (size_t i = 0; i < context_->tensors_size; i++) { + TfLiteTensor* tensor = &context_->tensors[i]; + if (tensor->buffer_handle != kTfLiteNullBufferHandle && + tensor->delegate->FreeBufferHandle != nullptr) { + tensor->delegate->FreeBufferHandle(context_, tensor->delegate, + &tensor->buffer_handle); + } + TfLiteTensorFree(tensor); + } +} + +TfLiteStatus Subgraph::ReplaceNodeSubsetsWithDelegateKernels( + TfLiteContext* context, TfLiteRegistration registration, + const TfLiteIntArray* nodes_to_replace, TfLiteDelegate* delegate) { + return static_cast(context->impl_) + ->ReplaceNodeSubsetsWithDelegateKernels(registration, nodes_to_replace, + delegate); +} + +namespace { + +// Copy a std::vector to an existing TfLiteIntArray. +// This is a low-level data manipulation function, and it's caller's +// responsibility to ensure TfLiteIntArray has enough size. +void CopyVectorToTfLiteIntArray(const std::vector& vec, + TfLiteIntArray* arr) { + arr->size = vec.size(); + memcpy(arr->data, vec.data(), sizeof(int) * arr->size); +} + +// This function allocates a continuous memory space that contains a +// TfLiteDelegateParams followed by a several TfLiteIntArray. +// When calling `free` at TfLiteDelegateParams*, all the allocated space +// will be freed together. +// +// +-----------------------------------+ +// | TfLiteDelegateParams | +// | TfLiteDelegate* delegate; | +// | TfLiteIntArray* nodes_to_replace; |--\ +// | TfLiteIntArray* input_tensors; |--+--\ +// | TfLiteIntArray* output_tensors; |--+--+--\ +// +-----------------------------------+ | | | +// | TfLiteIntArray (variable size) |<-/ | | +// +-----------------------------------+ | | +// | TfLiteIntArray (variable size) |<----/ | +// +-----------------------------------+ | +// | TfLiteIntArray (variable size) |<-------/ +// +-----------------------------------+ +TfLiteDelegateParams* CreateDelegateParams(TfLiteDelegate* delegate, + const NodeSubset& node_subset) { + // Step 1: Calculate the allocation size. + int allocation_size = sizeof(TfLiteDelegateParams); + + int nodes_to_replace_size = + TfLiteIntArrayGetSizeInBytes(node_subset.nodes.size()); + allocation_size += nodes_to_replace_size; + + int input_tensors_size = + TfLiteIntArrayGetSizeInBytes(node_subset.input_tensors.size()); + allocation_size += input_tensors_size; + + int output_tensors_size = + TfLiteIntArrayGetSizeInBytes(node_subset.output_tensors.size()); + allocation_size += output_tensors_size; + + // Step 2: Allocate the memory. + // Use `char*` for conveniently step through the allocated space by bytes. + char* allocation = reinterpret_cast(malloc(allocation_size)); + + // Step 3: Fill all data structures structures. + TfLiteDelegateParams* params = + reinterpret_cast(allocation); + params->delegate = delegate; + allocation += sizeof(TfLiteDelegateParams); + + params->nodes_to_replace = reinterpret_cast(allocation); + CopyVectorToTfLiteIntArray(node_subset.nodes, params->nodes_to_replace); + allocation += nodes_to_replace_size; + + params->input_tensors = reinterpret_cast(allocation); + CopyVectorToTfLiteIntArray(node_subset.input_tensors, params->input_tensors); + allocation += input_tensors_size; + + params->output_tensors = reinterpret_cast(allocation); + CopyVectorToTfLiteIntArray(node_subset.output_tensors, + params->output_tensors); + allocation += output_tensors_size; + + return params; +} + +} // namespace + +TfLiteStatus Subgraph::ReplaceNodeSubsetsWithDelegateKernels( + TfLiteRegistration registration, const TfLiteIntArray* nodes_to_replace, + TfLiteDelegate* delegate) { + // Annotate the registration as DELEGATE op. + registration.builtin_code = BuiltinOperator_DELEGATE; + + // Analyze the graph to find all independent node_subsets that are either + // fully not-this-delegate or this-delegate computation. + InterpreterInfo info(this); + std::vector node_subsets; + PartitionGraphIntoIndependentNodeSubsets(&info, nodes_to_replace, + &node_subsets); + + execution_plan_.clear(); + + for (auto& node_subset : node_subsets) { + // Subsets calimed by the delegate should have a "macro" op created, the + // other node_subsets (kTfNonPartition) just have their nodes added back to + // the execution plan. + switch (node_subset.type) { + case NodeSubset::kTfNonPartition: + for (auto it = node_subset.nodes.begin(); it != node_subset.nodes.end(); + ++it) { + execution_plan_.push_back(*it); + } + break; + case NodeSubset::kTfPartition: { + int node_index; + + TfLiteDelegateParams* params = + CreateDelegateParams(delegate, node_subset); + TF_LITE_ENSURE_STATUS(AddNodeWithParameters( + node_subset.input_tensors, node_subset.output_tensors, nullptr, 0, + params, ®istration, &node_index)); + + // Initialize the output tensors's delegate-related fields. + for (int tensor_index : node_subset.output_tensors) { + TfLiteTensor* tensor = &tensors_[tensor_index]; + TF_LITE_ENSURE(context_, tensor->delegate == nullptr || + tensor->delegate == delegate); + tensor->delegate = delegate; + } + + // Associate the node with the delegate. + TfLiteNode* node = &nodes_and_registration_[node_index].first; + node->delegate = delegate; + } break; + case NodeSubset::kTfUnexplored: + return kTfLiteError; + break; + } + } + return kTfLiteOk; +} + +TfLiteExternalContext* Subgraph::GetExternalContext( + TfLiteExternalContextType type) { + if (type >= 0 && type < kTfLiteMaxExternalContexts) { + return external_contexts_[type]; + } + return nullptr; +} + +TfLiteExternalContext* Subgraph::GetExternalContext( + struct TfLiteContext* context, TfLiteExternalContextType type) { + return static_cast(context->impl_)->GetExternalContext(type); +} + +void Subgraph::SetExternalContext(TfLiteExternalContextType type, + TfLiteExternalContext* ctx) { + if (type >= 0 && type < kTfLiteMaxExternalContexts) { + external_contexts_[type] = ctx; + } +} + +void Subgraph::SetExternalContext(struct TfLiteContext* context, + TfLiteExternalContextType type, + TfLiteExternalContext* ctx) { + return static_cast(context->impl_)->SetExternalContext(type, ctx); +} + +// Gets an TfLiteIntArray* representing the execution plan. The interpreter owns +// this memory and it is only guaranteed to exist during the invocation of the +// delegate prepare. +TfLiteStatus Subgraph::GetExecutionPlan(TfLiteIntArray** execution_plan) { + // TODO(aselle): Do not make a copy here + plan_cache_.reset(TfLiteIntArrayCreate(execution_plan_.size())); + *execution_plan = plan_cache_.get(); + static_assert(sizeof(plan_cache_->data[0]) == sizeof(execution_plan_[0]), + "TfLiteIntArray and execution_plan do not contain same type."); + std::memcpy(plan_cache_->data, execution_plan_.data(), + sizeof(plan_cache_->data[0]) * execution_plan_.size()); + return kTfLiteOk; +} + +// WARNING: This is an experimental interface that is subject to change. +// Entry point for C node plugin API to get the execution plan +TfLiteStatus Subgraph::GetExecutionPlan(struct TfLiteContext* context, + TfLiteIntArray** execution_plan) { + return static_cast(context->impl_) + ->GetExecutionPlan(execution_plan); +} + +TfLiteStatus Subgraph::SetInputs(std::vector inputs) { + TF_LITE_ENSURE_OK(&context_, + CheckTensorIndices("inputs", inputs.data(), inputs.size())); + inputs_ = std::move(inputs); + return kTfLiteOk; +} + +TfLiteStatus Subgraph::SetOutputs(std::vector outputs) { + TF_LITE_ENSURE_OK( + &context_, CheckTensorIndices("outputs", outputs.data(), outputs.size())); + outputs_ = std::move(outputs); + return kTfLiteOk; +} + +TfLiteStatus Subgraph::SetVariables(std::vector variables) { + TF_LITE_ENSURE_OK(&context_, CheckTensorIndices("variables", variables.data(), + variables.size())); + variables_ = std::move(variables); + return kTfLiteOk; +} + +TfLiteStatus Subgraph::CheckTensorIndices(const char* label, const int* indices, + int length) { + // Making sure kOptionalTensor is not re-defined to something other than -1. + static_assert(kOptionalTensor == -1, "kOptionalTensor should be defined -1"); + + for (int i = 0; i < length; i++) { + int index = indices[i]; + // Continue if index == kOptionalTensor before additional comparisons below, + // size_t(-1) is always >= context_tensors_size. + if (index == kOptionalTensor) { + continue; + } + if (index < 0 || static_cast(index) >= context_->tensors_size) { + ReportError("Invalid tensor index %d in %s\n", index, label); + consistent_ = false; + return kTfLiteError; + } + } + return kTfLiteOk; +} + +TfLiteStatus Subgraph::BytesRequired(TfLiteType type, const int* dims, + size_t dims_size, size_t* bytes) { + // TODO(aselle): Check for overflow here using overflow.h in TensorFlow + // MultiplyWithoutOverflow. + TF_LITE_ENSURE(context_, bytes != nullptr); + size_t count = 1; + for (int k = 0; k < dims_size; k++) count *= dims[k]; + switch (type) { + case kTfLiteFloat32: + *bytes = sizeof(float) * count; + break; + case kTfLiteInt16: + *bytes = sizeof(int16_t) * count; + break; + case kTfLiteInt32: + *bytes = sizeof(int32_t) * count; + break; + case kTfLiteUInt8: + *bytes = sizeof(uint8_t) * count; + break; + case kTfLiteInt64: + *bytes = sizeof(int64_t) * count; + break; + case kTfLiteBool: + *bytes = sizeof(bool) * count; + break; + case kTfLiteComplex64: + *bytes = sizeof(std::complex) * count; + break; + case kTfLiteInt8: + *bytes = sizeof(int8_t) * count; + break; + default: + ReportError( + "Only float32, int8, int16, int32, int64, uint8, bool, complex64 " + "supported currently."); + return kTfLiteError; + } + return kTfLiteOk; +} + +TfLiteStatus Subgraph::AllocateTensors() { + if (!consistent_) { + ReportError("AllocateTensors() called on inconsistent model."); + return kTfLiteError; + } + + // Explicit (re)allocation is necessary if nodes have been changed or tensors + // have been resized. For inputs marked as dynamic, we can't short-circuit the + // allocation as the client may have done the resize manually. + if (state_ != kStateUninvokable && + !HasDynamicTensorImpl(*context_, inputs())) { + return kTfLiteOk; + } + + next_execution_plan_index_to_prepare_ = 0; + if (memory_planner_) { + TF_LITE_ENSURE_STATUS(memory_planner_->ResetAllocations()); + } + + TF_LITE_ENSURE_STATUS(PrepareOpsAndTensors()); + + state_ = kStateInvokable; + + // Reset the variable tensors to zero after (re)allocating the tensors. + // Developers shouldn't rely on the side effect of this function to reset + // variable tesnsors. They should call `ResetVariableTensors` directly + // instead. + ResetVariableTensors(); + + return kTfLiteOk; +} + +// TODO(ycling): Support non-zero default values. +TfLiteStatus Subgraph::ResetVariableTensors() { + for (auto& tensor : tensors_) { + if (!tensor.is_variable) { + continue; + } + + // Variable tensors have to be `kTfLiteArenaRwPersistent`, and must be + // allocated after the initial `PrepareOpsAndTensors()` is called. + TF_LITE_ENSURE_EQ(context_, tensor.allocation_type, + kTfLiteArenaRwPersistent); + TF_LITE_ENSURE(context_, tensor.data.raw != nullptr); + + memset(tensor.data.raw, 0, tensor.bytes); + } + return kTfLiteOk; +} + +TfLiteStatus Subgraph::AddNodeWithParameters( + const std::vector& inputs, const std::vector& outputs, + const char* init_data, size_t init_data_size, void* builtin_data, + const TfLiteRegistration* registration, int* node_index) { + if (state_ == kStateInvokableAndImmutable) { + ReportError("AddNodeWithParameters is disallowed when graph is immutable."); + return kTfLiteError; + } + state_ = kStateUninvokable; + + std::unique_ptr builtin_data_deleter(builtin_data, + free); + + TF_LITE_ENSURE_OK(context_, CheckTensorIndices("node inputs", inputs.data(), + inputs.size())); + TF_LITE_ENSURE_OK( + &context_, + CheckTensorIndices("node outputs", outputs.data(), outputs.size())); + + int new_node_index = nodes_and_registration_.size(); + if (node_index) *node_index = new_node_index; + nodes_and_registration_.resize(nodes_and_registration_.size() + 1); + auto& node_and_reg = nodes_and_registration_.back(); + TfLiteNode& node = node_and_reg.first; + if (node.inputs) TfLiteIntArrayFree(node.inputs); + if (node.outputs) TfLiteIntArrayFree(node.outputs); + if (node.temporaries) TfLiteIntArrayFree(node.temporaries); + + // NOTE, here we are not using move semantics yet, since our internal + // representation isn't std::vector, but in the future we would like to avoid + // copies, so we want the interface to take r-value references now. + node.inputs = ConvertVectorToTfLiteIntArray(inputs); + node.outputs = ConvertVectorToTfLiteIntArray(outputs); + node.temporaries = TfLiteIntArrayCreate(0); + if (init_data) { + node.user_data = OpInit(*registration, init_data, init_data_size); + } else { + node.user_data = + OpInit(*registration, + reinterpret_cast(builtin_data_deleter.get()), 0); + } + + node.builtin_data = builtin_data_deleter.release(); + // TODO(ycling): Filling `custom_initial_data` and `custom_initial_data_size` + // properly for nodes generated by ReplaceNodeSubsetsWithDelegateKernels. + + if (registration->builtin_code == BuiltinOperator_CUSTOM) { + // When it's a CUSTOM op, the `custom_options` field in the Flatbuffer + // `Operator` table is passed in. + node.custom_initial_data = init_data; + node.custom_initial_data_size = init_data_size; + } else { + node.custom_initial_data = nullptr; + node.custom_initial_data_size = 0; + } + + node.delegate = nullptr; + node_and_reg.second = *registration; + execution_plan_.push_back(new_node_index); + return kTfLiteOk; +} + +TfLiteStatus Subgraph::ResizeInputTensor(int tensor_index, + const std::vector& dims) { + if (state_ == kStateInvokableAndImmutable) { + ReportError("ResizeInputTensor is disallowed when graph is immutable."); + return kTfLiteError; + } + + // TODO(aselle): All bounds checks can be implemented as one-sided bounds + // checks by casting to unsigned for efficiency. Profile before doing this. + TF_LITE_ENSURE(context_, + tensor_index < context_->tensors_size && tensor_index >= 0); + TfLiteTensor* tensor = &context_->tensors[tensor_index]; + + // Short-circuit the state change if the dimensions don't change, avoiding + // unnecessary (re)allocations. + if (EqualArrayAndTfLiteIntArray(tensor->dims, dims.size(), dims.data())) { + return kTfLiteOk; + } + + state_ = kStateUninvokable; + return ResizeTensorImpl(tensor, ConvertVectorToTfLiteIntArray(dims)); +} + +TfLiteStatus Subgraph::PrepareOpsStartingAt( + int first_execution_plan_index, int* last_execution_plan_index_prepared) { + if (first_execution_plan_index == 0) { + has_dynamic_tensors_ = false; + } + for (int execution_plan_index = first_execution_plan_index; + execution_plan_index < execution_plan_.size(); execution_plan_index++) { + int node_index = execution_plan_[execution_plan_index]; + TfLiteNode& node = nodes_and_registration_[node_index].first; + const TfLiteRegistration& registration = + nodes_and_registration_[node_index].second; + EnsureTensorsVectorCapacity(); + if (OpPrepare(registration, &node) == kTfLiteError) { + return ReportOpError(context_, node, registration, node_index, + "failed to prepare"); + } + + *last_execution_plan_index_prepared = execution_plan_index; + + // Discontinue if the node has dynamic outputs. Note that we don't + // stop for dynamic temporary tensors since they won't affect the + // sizes of other tensors in the graph. + if (HasDynamicTensor(*context_, node.outputs)) { + has_dynamic_tensors_ = true; + return kTfLiteOk; + } + } + return kTfLiteOk; +} + +TfLiteStatus Subgraph::PrepareOpsAndTensors() { + if (!memory_planner_) { + memory_planner_.reset(new ArenaPlanner( + context_, std::unique_ptr(new InterpreterInfo(this)), + /*preserve_inputs=*/true, /*preserve_intermediates*/ false)); + memory_planner_->PlanAllocations(); + } + + int last_exec_plan_index_prepared = 0; + + TF_LITE_ENSURE_STATUS(PrepareOpsStartingAt( + next_execution_plan_index_to_prepare_, &last_exec_plan_index_prepared)); + TF_LITE_ENSURE_STATUS(memory_planner_->ExecuteAllocations( + next_execution_plan_index_to_prepare_, last_exec_plan_index_prepared)); + + next_execution_plan_index_to_prepare_ = last_exec_plan_index_prepared + 1; + return kTfLiteOk; +} + +TfLiteStatus Subgraph::Invoke() { + if (!consistent_) { + ReportError("Invoke called on model that is not consistent."); + return kTfLiteError; + } + + TfLiteStatus status = kTfLiteOk; + if (state_ == kStateUninvokable) { + ReportError("Invoke called on model that is not ready."); + return kTfLiteError; + } + + if (nnapi_delegate_) { + if (next_execution_plan_index_to_prepare_ == execution_plan_.size()) { + TF_LITE_ENSURE_OK(context_, nnapi_delegate_->Invoke(this)); + return kTfLiteOk; + } else { + // TODO(aselle): In the future, we would like this to be an + // automatic tflite CPU fallback. + ReportError( + "NNAPI was requested, but dependent sized tensors " + "being used.\n"); + return kTfLiteError; + } + } + + // Invocations are always done in node order. + // Note that calling Invoke repeatedly will cause the original memory plan to + // be reused, unless either ResizeInputTensor() or AllocateTensors() has been + // called. + for (int execution_plan_index = 0; + execution_plan_index < execution_plan_.size(); execution_plan_index++) { + if (execution_plan_index == next_execution_plan_index_to_prepare_) { + TF_LITE_ENSURE_STATUS(PrepareOpsAndTensors()); + TF_LITE_ENSURE(context_, next_execution_plan_index_to_prepare_ >= + execution_plan_index); + } + int node_index = execution_plan_[execution_plan_index]; + TfLiteNode& node = nodes_and_registration_[node_index].first; + const TfLiteRegistration& registration = + nodes_and_registration_[node_index].second; + SCOPED_OPERATOR_PROFILE(profiler_, node_index); + + // TODO(ycling): This is an extra loop through inputs to check if the data + // need to be copied from Delegate buffer to raw memory, which is often not + // needed. We may want to cache this in prepare to know if this needs to be + // done for a node or not. + for (int i = 0; i < node.inputs->size; ++i) { + int tensor_index = node.inputs->data[i]; + if (tensor_index == kOptionalTensor) { + continue; + } + TfLiteTensor* tensor = &tensors_[tensor_index]; + if (tensor->delegate && tensor->delegate != node.delegate && + tensor->data_is_stale) { + EnsureTensorDataIsReadable(tensor_index); + } + } + + EnsureTensorsVectorCapacity(); + tensor_resized_since_op_invoke_ = false; + if (OpInvoke(registration, &node) == kTfLiteError) { + status = ReportOpError(context_, node, registration, node_index, + "failed to invoke"); + } + + // Force execution prep for downstream ops if the latest op triggered the + // resize of a dynamic tensor. + if (tensor_resized_since_op_invoke_ && + HasDynamicTensor(*context_, node.outputs)) { + next_execution_plan_index_to_prepare_ = execution_plan_index + 1; + } + } + + return status; +} + +TfLiteStatus Subgraph::ResizeTensor(TfLiteContext* context, + TfLiteTensor* tensor, + TfLiteIntArray* new_size) { + // Note here that context->impl_ is recovering the this pointer for an + // instance of Interpreter to call into the member function ResizeTensorImpl + // (this function is static). + return static_cast(context->impl_) + ->ResizeTensorImpl(tensor, new_size); +} + +void Subgraph::ReportErrorImpl(const char* format, va_list args) { + error_reporter_->Report(format, args); +} + +void Subgraph::ReportErrorC(TfLiteContext* context, const char* format, ...) { + va_list args; + va_start(args, format); + auto* f = static_cast(context->impl_); + // Note here that context->impl_ is recovering the this pointer for an + // instance of Subgraph to call into the member function ReportErrorImpl + // (this function is static). + f->ReportErrorImpl(format, args); + va_end(args); +} + +// Entry point for C node plugin API to report an error. +void Subgraph::ReportError(const char* format, ...) { + va_list args; + va_start(args, format); + auto* f = static_cast(context_->impl_); + // Note here that context->impl_ is recovering the this pointer for an + // instance of Subgraph to call into the member function ReportErrorImpl + // (this function is static). + f->ReportErrorImpl(format, args); + va_end(args); +} + +TfLiteStatus Subgraph::AddTensors(int tensors_to_add, + int* first_new_tensor_index) { + const size_t base_index = tensors_.size(); + if (first_new_tensor_index) *first_new_tensor_index = base_index; + tensors_.resize(tensors_.size() + tensors_to_add); + for (size_t i = base_index; i < tensors_.size(); i++) { + memset(&tensors_[i], 0, sizeof(tensors_[i])); + tensors_[i].buffer_handle = kTfLiteNullBufferHandle; + } + context_->tensors = tensors_.data(); + context_->tensors_size = tensors_.size(); + return kTfLiteOk; +} + +TfLiteStatus Subgraph::AddTensors(TfLiteContext* context, int tensors_to_add, + int* first_new_tensor_index) { + // Note here that context->impl_ is recovering the this pointer for an + // instance of Interpreter to call into the member function AddTensors + // (this function is static). + return static_cast(context->impl_) + ->AddTensors(tensors_to_add, first_new_tensor_index); +} + +TfLiteStatus Subgraph::GetNodeAndRegistration( + int node_index, TfLiteNode** node, TfLiteRegistration** registration) { + TF_LITE_ENSURE(context_, node_index >= 0); + auto nodes_size = nodes_and_registration_.size(); + TF_LITE_ENSURE(context_, static_cast(node_index) < nodes_size); + TF_LITE_ENSURE(context_, node != nullptr && registration != nullptr); + auto& node_and_reg = nodes_and_registration_[node_index]; + *node = &node_and_reg.first; + *registration = &node_and_reg.second; + return kTfLiteOk; +} + +TfLiteStatus Subgraph::GetNodeAndRegistration( + struct TfLiteContext* context, int node_index, TfLiteNode** node, + TfLiteRegistration** registration) { + return static_cast(context->impl_) + ->GetNodeAndRegistration(node_index, node, registration); +} + +TfLiteStatus Subgraph::SetTensorParametersReadOnly( + int tensor_index, TfLiteType type, const char* name, const size_t rank, + const int* dims, TfLiteQuantizationParams quantization, const char* buffer, + size_t bytes, const Allocation* allocation) { + if (state_ == kStateInvokableAndImmutable) { + ReportError( + "SetTensorParametersReadOnly is disallowed when graph is immutable."); + return kTfLiteError; + } + + TF_LITE_ENSURE(context_, + tensor_index < context_->tensors_size && tensor_index >= 0); + // For most tensors we know exactly how much memory is necessary so we can + // ensure the buffer is large enough. However, we need to skip string tensors + // because their sizes change with the contents of the individual strings. + if (type != kTfLiteString) { + size_t required_bytes; + TF_LITE_ENSURE_OK(context_, + BytesRequired(type, dims, rank, &required_bytes)); + TF_LITE_ENSURE_EQ(context_, required_bytes, bytes); + } + + TfLiteTensor& tensor = context_->tensors[tensor_index]; + if (type == tensor.type && + EqualArrayAndTfLiteIntArray(tensor.dims, rank, dims)) { + // Fast path which does not invalidate the invokable property. + TfLiteTensorDataFree(&tensor); + tensor.data.raw = const_cast(buffer); + if (!tensor.dims) tensor.dims = ConvertArrayToTfLiteIntArray(rank, dims); + tensor.params = quantization; + tensor.allocation_type = kTfLiteMmapRo; + tensor.allocation = allocation; + } else { + state_ = kStateUninvokable; + TfLiteTensorReset(type, name, ConvertArrayToTfLiteIntArray(rank, dims), + quantization, const_cast(buffer), bytes, + kTfLiteMmapRo, allocation, false, &tensor); + } + return kTfLiteOk; +} + +// Set description of inputs/outputs/data/fptrs for node `node_index`. +// This variant assumes an external buffer has been allocated of size +// bytes. The lifetime of buffer must be ensured to be greater or equal +// to Interpreter. +TfLiteStatus Subgraph::SetTensorParametersReadWrite( + int tensor_index, TfLiteType type, const char* name, const size_t rank, + const int* dims, TfLiteQuantizationParams quantization, bool is_variable) { + if (state_ == kStateInvokableAndImmutable) { + ReportError( + "SetTensorParametersReadWrite is disallowed when graph is immutable."); + return kTfLiteError; + } + TF_LITE_ENSURE(context_, + tensor_index < context_->tensors_size && tensor_index >= 0); + size_t required_bytes = 0; + if (type != kTfLiteString) { + // These types will be allocated in our arena so we need to record how + // many bytes we will need based on the dimensions. String tensors are + // allocated dynamically and we can't know ahead of time how much space + // they will require. + TF_LITE_ENSURE_OK(context_, + BytesRequired(type, dims, rank, &required_bytes)); + } + + TfLiteAllocationType allocation_type = kTfLiteArenaRw; + if (type == kTfLiteString) { + if (is_variable) { + // We don't have a real use case for string variable tensor. + ReportError("String variable tensor isn't supported."); + return kTfLiteError; + } + allocation_type = kTfLiteDynamic; + } else if (is_variable) { + allocation_type = kTfLiteArenaRwPersistent; + } + + TfLiteTensorReset(type, name, ConvertArrayToTfLiteIntArray(rank, dims), + quantization, + /*buffer=*/nullptr, required_bytes, allocation_type, + nullptr, is_variable, &context_->tensors[tensor_index]); + return kTfLiteOk; +} + +TfLiteStatus Subgraph::SetExecutionPlan(const std::vector& new_plan) { + for (int node_index : new_plan) { + TF_LITE_ENSURE(context_, node_index >= 0 && + node_index < nodes_and_registration_.size()); + } + execution_plan_ = new_plan; + return kTfLiteOk; +} + +TfLiteStatus Subgraph::ResizeTensorImpl(TfLiteTensor* tensor, + TfLiteIntArray* new_size) { + // Note that in theory we could resize kTfLiteArenaRwPersistent tensors too. + if (tensor->allocation_type == kTfLiteArenaRw || + tensor->allocation_type == kTfLiteDynamic || + tensor->allocation_type == kTfLiteArenaRwPersistent) { + tensor_resized_since_op_invoke_ |= + TfLiteIntArrayEqual(tensor->dims, new_size) == 0; + if (tensor->type != kTfLiteString) { + size_t bytesRequired; + TfLiteStatus status = BytesRequired(tensor->type, new_size->data, + new_size->size, &bytesRequired); + if (status != kTfLiteOk) { + TfLiteIntArrayFree(new_size); + return kTfLiteError; + } + + // Realloc space for kTfLiteDynamic tensors. + TfLiteTensorRealloc(bytesRequired, tensor); + tensor->bytes = bytesRequired; + } + if (tensor->dims) TfLiteIntArrayFree(tensor->dims); + tensor->dims = new_size; + + if (tensor->allocation_type != kTfLiteDynamic) { + tensor->data.raw = nullptr; + } + } else { + // kTfLiteMmapRo tensors are stored in the flatbuffer and are therefore + // of fixed size. + TfLiteIntArrayFree(new_size); + ReportError("Attempting to resize a fixed-size tensor."); + return kTfLiteError; + } + return kTfLiteOk; +} + +void Subgraph::UseNNAPI(bool enable) { + // TODO(aselle): This is a workaround for finding if NNAPI exists. + // We also need to make sure getLibraryHandle() is renamed to be NNAPI + // prefixed. + if (!NNAPIDelegate::IsSupported()) enable = false; + if (!enable) { + nnapi_delegate_.reset(); + } else if (!nnapi_delegate_) { + nnapi_delegate_.reset(new NNAPIDelegate); + } +} + +void Subgraph::SwitchToDelegateContext() { + context_->GetNodeAndRegistration = GetNodeAndRegistration; + context_->ReplaceNodeSubsetsWithDelegateKernels = + ReplaceNodeSubsetsWithDelegateKernels; + context_->GetExecutionPlan = GetExecutionPlan; +} + +void Subgraph::SwitchToKernelContext() { + context_->GetNodeAndRegistration = [](struct TfLiteContext* context, + int node_index, TfLiteNode** node, + TfLiteRegistration** registration) { + return ForbiddenContextFunction(context); + }; + context_->ReplaceNodeSubsetsWithDelegateKernels = + [](TfLiteContext* context, TfLiteRegistration registration, + const TfLiteIntArray* nodes_to_replace, TfLiteDelegate* delegate) { + return ForbiddenContextFunction(context); + }; + context_->GetExecutionPlan = [](struct TfLiteContext* context, + TfLiteIntArray**) { + return ForbiddenContextFunction(context); + }; +} + +TfLiteStatus Subgraph::ModifyGraphWithDelegate(TfLiteDelegate* delegate) { + if (!(delegate->flags & kTfLiteDelegateFlagsAllowDynamicTensors)) { + int last_execution_plan_index_prepared; + TF_LITE_ENSURE_OK(&context_, PrepareOpsStartingAt( + 0, &last_execution_plan_index_prepared)); + if (has_dynamic_tensors_) { + ReportError( + "Attempting to use a delegate that only supports static-sized " + "tensors with a graph that has dynamic-sized tensors."); + return kTfLiteError; + } + } + + // TODO(aselle): Consider if it is worth storing pointers to delegates. + // Setup additional context interface. + SwitchToDelegateContext(); + + TfLiteStatus status = delegate->Prepare(context_, delegate); + + // Remove additional context info. + SwitchToKernelContext(); + + TF_LITE_ENSURE_OK(context_, status); + + if (!(delegate->flags & kTfLiteDelegateFlagsAllowDynamicTensors)) { + // Reset the state to force tensor/op reallocation. + state_ = kStateUninvokable; + TF_LITE_ENSURE_OK(context_, AllocateTensors()); + TF_LITE_ENSURE_EQ(context_, state_, kStateInvokable); + // After using a delegate which doesn't support dynamic tensors, make the + // entire graph immutable. + state_ = kStateInvokableAndImmutable; + } + + return status; +} + +} // namespace tflite diff --git a/tensorflow/lite/core/subgraph.h b/tensorflow/lite/core/subgraph.h new file mode 100644 index 00000000000..2a7c3a7c322 --- /dev/null +++ b/tensorflow/lite/core/subgraph.h @@ -0,0 +1,501 @@ +/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#ifndef TENSORFLOW_LITE_CORE_SUBGRAPH_H_ +#define TENSORFLOW_LITE_CORE_SUBGRAPH_H_ + +#include +#include + +#include "tensorflow/lite/allocation.h" +#include "tensorflow/lite/c/c_api_internal.h" +#include "tensorflow/lite/memory_planner.h" +#include "tensorflow/lite/profiling/profiler.h" +#include "tensorflow/lite/util.h" + +namespace tflite { + +// Forward declare since NNAPIDelegate uses Interpreter. +class NNAPIDelegate; + +class Subgraph { + public: + friend class Interpreter; + + Subgraph(ErrorReporter* error_reporter, + TfLiteExternalContext** external_contexts, + std::vector>* subgraphs); + + Subgraph(const Subgraph&) = delete; + + // Subgraphs should be movable but not copyable. + Subgraph(Subgraph&&) = default; + Subgraph& operator=(const Subgraph&) = delete; + virtual ~Subgraph(); + + // Provide a list of tensor indexes that are inputs to the model. + // Each index is bound check and this modifies the consistent_ flag of the + // interpreter. + TfLiteStatus SetInputs(std::vector inputs); + + // Provide a list of tensor indexes that are outputs to the model + // Each index is bound check and this modifies the consistent_ flag of the + // interpreter. + TfLiteStatus SetOutputs(std::vector outputs); + + // Provide a list of tensor indexes that are variable tensors. + // Each index is bound check and this modifies the consistent_ flag of the + // interpreter. + TfLiteStatus SetVariables(std::vector variables); + + // Adds a node with the given parameters and returns the index of the new + // node in `node_index` (optionally). Interpreter will take ownership of + // `builtin_data` and destroy it with `free`. Ownership of 'init_data' + // remains with the caller. + TfLiteStatus AddNodeWithParameters(const std::vector& inputs, + const std::vector& outputs, + const char* init_data, + size_t init_data_size, void* builtin_data, + const TfLiteRegistration* registration, + int* node_index); + + // Adds `tensors_to_add` tensors, preserving pre-existing Tensor entries. + // The value pointed to by `first_new_tensor_index` will be set to the + // index of the first new tensor if `first_new_tensor_index` is non-null. + TfLiteStatus AddTensors(int tensors_to_add, int* first_new_tensor_index); + + // Set description of inputs/outputs/data/fptrs for node `node_index`. + // This variant assumes an external buffer has been allocated of size + // bytes. The lifetime of buffer must be ensured to be greater or equal + // to Interpreter. + TfLiteStatus SetTensorParametersReadOnly( + int tensor_index, TfLiteType type, const char* name, const size_t rank, + const int* dims, TfLiteQuantizationParams quantization, + const char* buffer, size_t bytes, const Allocation* allocation); + + // Set description of inputs/outputs/data/fptrs for node `node_index`. + // This variant assumes an external buffer has been allocated of size + // bytes. The lifetime of buffer must be ensured to be greater or equal + // to Interpreter. + TfLiteStatus SetTensorParametersReadWrite( + int tensor_index, TfLiteType type, const char* name, const size_t rank, + const int* dims, TfLiteQuantizationParams quantization, bool is_variable); + + // WARNING: Experimental interface, subject to change + // Overrides execution plan. This bounds checks indices sent in. + TfLiteStatus SetExecutionPlan(const std::vector& new_plan); + + // Get a mutable tensor data structure. + // TODO(aselle): Create a safe ArrayHandle interface to avoid exposing this + // read/write access to structure + TfLiteTensor* tensor(int tensor_index) { + if (tensor_index < 0 || + static_cast(tensor_index) >= context_->tensors_size) { + return nullptr; + } + return &context_->tensors[tensor_index]; + } + + // Get an immutable tensor data structure. + const TfLiteTensor* tensor(int tensor_index) const { + if (tensor_index < 0 || + static_cast(tensor_index) >= context_->tensors_size) { + return nullptr; + } + return &context_->tensors[tensor_index]; + } + + // Read only access to list of inputs. + std::vector& inputs() { return inputs_; } + + // Read only access to list of inputs. + const std::vector& inputs() const { return inputs_; } + + // Read only access to list of outputs. + std::vector& outputs() { return outputs_; } + + // Read only access to list of outputs. + const std::vector& outputs() const { return outputs_; } + + // Read only access to list of variable tensors. + std::vector& variables() { return variables_; } + + // Read only access to list of variable tensors. + const std::vector& variables() const { return variables_; } + + size_t tensors_size() const { return tensors_.size(); } + + // Return the number of ops in the model. + size_t nodes_size() const { return nodes_and_registration_.size(); } + + // Read only access to list of variable tensors. + std::vector& execution_plan() { return execution_plan_; } + + // Read only access to list of variable tensors. + const std::vector& execution_plan() const { return execution_plan_; } + + // Mutable form of tensors (TEMPORARY for refactor). + // TODO(b/119495520): remove when refactoring complete. + std::vector& tensors() { return tensors_; } + // Mutable form of tensors (TEMPORARY for refactor). + // TODO(b/119495520): remove when refactoring complete. + std::vector>& + nodes_and_registration() { + return nodes_and_registration_; + } + + const std::vector>& + nodes_and_registration() const { + return nodes_and_registration_; + } + + // Get a pointer to an operation and registration data structure if in bounds. + const std::pair* node_and_registration( + int node_index) const { + if (node_index < 0 || static_cast(node_index) >= nodes_size()) + return nullptr; + return &nodes_and_registration_[node_index]; + } + + // Change the dimensionality of a given tensor. Note, this is only acceptable + // for tensor indices that are inputs. + // Returns status of failure or success. + // TODO(aselle): Consider implementing ArraySlice equivalent to make this + // more adept at accepting data without an extra copy. Use absl::ArraySlice + // if our partners determine that dependency is acceptable. + TfLiteStatus ResizeInputTensor(int tensor_index, + const std::vector& dims); + + // Update allocations for all tensors. This will redim dependent tensors using + // the input tensor dimensionality as given. This is relatively expensive. + // If you know that your sizes are not changing, you need not call this. + // Returns status of success or failure. + TfLiteStatus AllocateTensors(); + + // Invoke the subgraph (run the whole graph in dependency order). + // + // NOTE: It is possible that the interpreter is not in a ready state + // to evaluate (i.e. if a ResizeTensor() has been performed without an + // AllocateTensors(). + // Returns status of success or failure. + TfLiteStatus Invoke(); + + // Entry point for C node plugin API to report an error. + void ReportError(const char* format, ...); + + void UseNNAPI(bool enable); + + // Return the subgraph specific context. + TfLiteContext* context() { return context_; } + + // Set the value of an external context. + void SetExternalContext(TfLiteExternalContextType type, + TfLiteExternalContext* ctx); + // Get the half precision flag. + // WARNING: This is an experimental API and subject to change. + bool GetAllowFp16PrecisionForFp32() const { + return context_->allow_fp32_relax_to_fp16; + } + + // Ensure the data in `tensor.data` is readable. In case delegate is used, + // it might require to copy the data from delegate buffer to raw memory. + // WARNING: This is an experimental API and subject to change. + // TODO(b/119495520): make this private when refactoring complete. + TfLiteStatus EnsureTensorDataIsReadable(int tensor_index) { + TfLiteTensor* t = &tensors_[tensor_index]; + TF_LITE_ENSURE(context_, t != nullptr); + if (t->data_is_stale) { + TF_LITE_ENSURE(context_, t->delegate != nullptr); + TF_LITE_ENSURE(context_, t->buffer_handle != kTfLiteNullBufferHandle); + TF_LITE_ENSURE(context_, t->delegate->CopyFromBufferHandle != nullptr); + // TODO(b/120420546): we must add a test that exercise this code. + TF_LITE_ENSURE_STATUS(t->delegate->CopyFromBufferHandle( + context_, t->delegate, t->buffer_handle, t)); + t->data_is_stale = false; + } + return kTfLiteOk; + } + + // The default capacity of `tensors_` vector. + static constexpr int kTensorsReservedCapacity = 128; + // The capacity headroom of `tensors_` vector before calling ops' + // `prepare` and `invoke` function. In these functions, it's guaranteed + // allocating up to `kTensorsCapacityHeadroom` more tensors won't invalidate + // pointers to existing tensors. + static constexpr int kTensorsCapacityHeadroom = 16; + + // Reset all variable tensors to the default value. + // If a variable tensor doesn't have a buffer, reset it to zero. + // TODO(b/115961645): Implement - If a variable tensor has a buffer, reset it + // to the value of the buffer. + // WARNING: This is an experimental API and subject to change. + TfLiteStatus ResetVariableTensors(); + + void SetProfiler(profiling::Profiler* profiler) { profiler_ = profiler; } + + profiling::Profiler* GetProfiler() { return profiler_; } + + // Returns a pointer to vector of subgraphs. + // WARNING: This is an experimental API and subject to change. + std::vector>* GetSubgraphs() { return subgraphs_; } + + // True if all tensors in the graph has static size after calling + // `AllocateTensors` function. + // Before `AllocateTensors` is called, this will always return true; + bool HasDynamicTensors() { return has_dynamic_tensors_; } + + private: + // Prevent 'context_' from accessing functions that are only available to + // delegated kernels. + void SwitchToKernelContext(); + + // Add delegate-only functions to 'context_'. + void SwitchToDelegateContext(); + + // Give 'op_reg' a chance to initialize itself using the contents of + // 'buffer'. + void* OpInit(const TfLiteRegistration& op_reg, const char* buffer, + size_t length) { + if (op_reg.init == nullptr) return nullptr; + return op_reg.init(context_, buffer, length); + } + + // Let 'op_reg' release any memory it might have allocated via 'OpInit'. + void OpFree(const TfLiteRegistration& op_reg, void* buffer) { + if (op_reg.free == nullptr) return; + if (buffer) { + op_reg.free(context_, buffer); + } + } + + // Prepare the given 'node' for execution. + TfLiteStatus OpPrepare(const TfLiteRegistration& op_reg, TfLiteNode* node) { + if (op_reg.prepare == nullptr) return kTfLiteOk; + return op_reg.prepare(context_, node); + } + + // Invoke the operator represented by 'node'. + TfLiteStatus OpInvoke(const TfLiteRegistration& op_reg, TfLiteNode* node) { + if (op_reg.invoke == nullptr) return kTfLiteError; + return op_reg.invoke(context_, node); + } + + // Call OpPrepare() for as many ops as possible, allocating memory for their + // tensors. If an op containing dynamic tensors is found, preparation will be + // postponed until this function is called again. This allows the interpreter + // to wait until Invoke() to resolve the sizes of dynamic tensors. + TfLiteStatus PrepareOpsAndTensors(); + + // Call OpPrepare() for all ops starting at 'first_node'. Stop when a + // dynamic tensors is found or all ops have been prepared. Fill + // 'last_node_prepared' with the id of the op containing dynamic tensors, or + // the last in the graph. + TfLiteStatus PrepareOpsStartingAt(int first_execution_plan_index, + int* last_execution_plan_index_prepared); + + // Tensors needed by the interpreter. Use `AddTensors` to add more blank + // tensor entries. Note, `tensors_.data()` needs to be synchronized to the + // `context_` whenever this std::vector is reallocated. Currently this + // only happens in `AddTensors()`. + std::vector tensors_; + + // Check if an array of tensor indices are valid with respect to the Tensor + // array. + // NOTE: this changes consistent_ to be false if indices are out of bounds. + TfLiteStatus CheckTensorIndices(const char* label, const int* indices, + int length); + + // Compute the number of bytes required to represent a tensor with dimensions + // specified by the array dims (of length dims_size). Returns the status code + // and bytes. + TfLiteStatus BytesRequired(TfLiteType type, const int* dims, size_t dims_size, + size_t* bytes); + + // Request an tensor be resized implementation. If the given tensor is of + // type kTfLiteDynamic it will also be allocated new memory. + TfLiteStatus ResizeTensorImpl(TfLiteTensor* tensor, TfLiteIntArray* new_size); + + // Report a detailed error string (will be printed to stderr). + // TODO(aselle): allow user of class to provide alternative destinations. + void ReportErrorImpl(const char* format, va_list args); + + // Entry point for C node plugin API to request an tensor be resized. + static TfLiteStatus ResizeTensor(TfLiteContext* context, TfLiteTensor* tensor, + TfLiteIntArray* new_size); + // Entry point for C node plugin API to report an error. + static void ReportErrorC(TfLiteContext* context, const char* format, ...); + + // Entry point for C node plugin API to add new tensors. + static TfLiteStatus AddTensors(TfLiteContext* context, int tensors_to_add, + int* first_new_tensor_index); + + // WARNING: This is an experimental API and subject to change. + // Entry point for C API ReplaceNodeSubsetsWithDelegateKernels + static TfLiteStatus ReplaceNodeSubsetsWithDelegateKernels( + TfLiteContext* context, TfLiteRegistration registration, + const TfLiteIntArray* nodes_to_replace, TfLiteDelegate* delegate); + + // Update the execution graph to replace some of the nodes with stub + // nodes. Specifically any node index that has `nodes[index]==1` will be + // slated for replacement with a delegate kernel specified by registration. + // Ownership of 'nodes_to_replace' and 'delegate' remains with the caller. + // WARNING: This is an experimental interface that is subject to change. + TfLiteStatus ReplaceNodeSubsetsWithDelegateKernels( + TfLiteRegistration registration, const TfLiteIntArray* nodes_to_replace, + TfLiteDelegate* delegate); + + // WARNING: This is an experimental interface that is subject to change. + // Gets the internal pointer to a TensorFlow lite node by node_index. + TfLiteStatus GetNodeAndRegistration(int node_index, TfLiteNode** node, + TfLiteRegistration** registration); + + // WARNING: This is an experimental interface that is subject to change. + // Entry point for C node plugin API to get a node by index. + static TfLiteStatus GetNodeAndRegistration(struct TfLiteContext*, + int node_index, TfLiteNode** node, + TfLiteRegistration** registration); + + // WARNING: This is an experimental interface that is subject to change. + // Gets an TfLiteIntArray* representing the execution plan. The interpreter + // owns this memory and it is only guaranteed to exist during the invocation + // of the delegate prepare. + TfLiteStatus GetExecutionPlan(TfLiteIntArray** execution_plan); + + // WARNING: This is an experimental interface that is subject to change. + // Entry point for C node plugin API to get the execution plan. + static TfLiteStatus GetExecutionPlan(struct TfLiteContext* context, + TfLiteIntArray** execution_plan); + + // Retrieve an existing external context by type. + TfLiteExternalContext* GetExternalContext(TfLiteExternalContextType type); + static TfLiteExternalContext* GetExternalContext( + struct TfLiteContext* context, TfLiteExternalContextType type); + + // Set the value of an external context. + static void SetExternalContext(struct TfLiteContext* context, + TfLiteExternalContextType type, + TfLiteExternalContext* ctx); + + // Allow a delegate to look at the graph and modify the graph to handle + // parts of the graph themselves. After this is called, the graph may + // contain new nodes that replace 1 more nodes. + // WARNING: This is an experimental API and subject to change. + TfLiteStatus ModifyGraphWithDelegate(TfLiteDelegate* delegate); + + // Ensures that `tensors_` has at least `kTensorsCapacityHeadroom` extra + // capacity. Calling this function may invalidate existing pointers to + // tensors. After calling this function, adding `kTensorsCapacityHeadroom` + // more tensors won't invalidate the pointer to existing tensors. + void EnsureTensorsVectorCapacity() { + const size_t required_capacity = tensors_.size() + kTensorsCapacityHeadroom; + if (required_capacity > tensors_.capacity()) { + tensors_.reserve(required_capacity); + context_->tensors = tensors_.data(); + } + } + + // The state of the Interpreter. + enum State { + // The interpreter isn't ready to be invoked. + // `AllocateTensor` need to be called to enter an invokable state. + kStateUninvokable = 0, + // The interpreter is ready to be invoked. + kStateInvokable, + // The interpreter is ready to be invoked, and graph can't be further + // modified. The interpreter will enter this state when calling + // `ModifyGraphWithDelegate` with `allow_dynamic_tensors=false`. + kStateInvokableAndImmutable, + }; + State state_ = kStateUninvokable; + + // A pure C data structure used to communicate with the pure C plugin + // interface. To avoid copying tensor metadata, this is also the definitive + // structure to store tensors. + // TODO(b/119495520): Get rid of owned and just make context_ a instance. + TfLiteContext owned_context_; + TfLiteContext* context_; + + // Node inputs/outputs are stored in TfLiteNode and TfLiteRegistration stores + // function pointers to actual implementation. + std::vector> + nodes_and_registration_; + + // Whether the model is consistent. That is to say if the inputs and outputs + // of every node and the global inputs and outputs are valid indexes into + // the tensor array. + bool consistent_ = true; + + // Array of indices representing the tensors that are inputs to the + // interpreter. + std::vector inputs_; + + // Array of indices representing the tensors that are outputs to the + // interpreter. + std::vector outputs_; + + // Array of indices representing the tensors that are variable tensors. + std::vector variables_; + + // The error reporter delegate that tflite will forward queries errors to. + ErrorReporter* error_reporter_; + + // Index of the next node to prepare. + // During Invoke(), Interpreter will allocate input tensors first, which are + // known to be fixed size. Then it will allocate outputs from nodes as many + // as possible. When there is a node that produces dynamic sized tensor. + // Interpreter will stop allocating tensors, set the value of next allocate + // node id, and execute the node to generate the output tensor before continue + // to allocate successors. This process repeats until all nodes are executed. + // NOTE: this relies on the order of nodes that is in topological order. + int next_execution_plan_index_to_prepare_; + + // WARNING: This is an experimental interface that is subject to change. + // This is a list of node indices (to index into nodes_and_registration). + // This represents a valid topological sort (dependency ordered) execution + // plan. In particular, it is valid for this ordering to contain only a + // subset of the node indices. + std::vector execution_plan_; + + // In the future, we'd like a TfLiteIntArray compatible representation. + // TODO(aselle): replace execution_plan_ with this. + std::unique_ptr plan_cache_; + + // Whether to delegate to NN API + std::unique_ptr nnapi_delegate_; + + std::unique_ptr memory_planner_; + + // Tracking bit for whether a tensor was resized in the course of an op + // invocation. This is a useful hint to ensure that dynamic tensor outputs + // trigger downstream reallocation after op invocation. + bool tensor_resized_since_op_invoke_ = false; + + // External contexts (kTfLiteMaxExternalContexts). + TfLiteExternalContext** external_contexts_; + + // Profiler for this interpreter instance. + profiling::Profiler* profiler_ = nullptr; + + // A pointer to vector of subgraphs. The vector is owned by the interpreter. + std::vector>* subgraphs_ = nullptr; + + // True if all tensors in the graph has static size after calling + // `PrepareOpsStartingAt` function (which is called by the `AllocateTensors` + // public function). + // The value is invalid before `PrepareOpStartingAt` is called. + bool has_dynamic_tensors_ = true; +}; + +} // namespace tflite +#endif // TENSORFLOW_LITE_CORE_SUBGRAPH_H_ diff --git a/tensorflow/lite/delegates/flex/BUILD b/tensorflow/lite/delegates/flex/BUILD index 222a043a88e..75083bf95a1 100644 --- a/tensorflow/lite/delegates/flex/BUILD +++ b/tensorflow/lite/delegates/flex/BUILD @@ -83,8 +83,10 @@ cc_library( ":delegate_data", ":kernel", ":util", + "@com_google_absl//absl/strings:strings", "//tensorflow/lite/c:c_api_internal", "//tensorflow/lite:kernel_api", + "//tensorflow/lite:string_util", "//tensorflow/lite:util", ] + select({ "//tensorflow:android": [ @@ -116,6 +118,7 @@ cc_library( hdrs = ["delegate_data.h"], deps = [ ":buffer_map", + "@com_google_absl//absl/memory", "//tensorflow/core/common_runtime/eager:context", ] + select({ "//tensorflow:android": [ diff --git a/tensorflow/lite/delegates/flex/buffer_map.cc b/tensorflow/lite/delegates/flex/buffer_map.cc index 9a6c5e74a7b..262ca9e0897 100644 --- a/tensorflow/lite/delegates/flex/buffer_map.cc +++ b/tensorflow/lite/delegates/flex/buffer_map.cc @@ -93,6 +93,11 @@ class TfLiteTensorBuffer : public BaseTfLiteTensorBuffer { class StringTfLiteTensorBuffer : public BaseTfLiteTensorBuffer { public: explicit StringTfLiteTensorBuffer(const TfLiteTensor* tensor) { + if (tensor->data.raw == nullptr) { + num_strings_ = 0; + data_ = nullptr; + return; + } num_strings_ = GetStringCount(tensor->data.raw); data_ = tensorflow::cpu_allocator()->Allocate(num_strings_); diff --git a/tensorflow/lite/delegates/flex/delegate.cc b/tensorflow/lite/delegates/flex/delegate.cc index 4fc2d82b494..ca7314fbaee 100644 --- a/tensorflow/lite/delegates/flex/delegate.cc +++ b/tensorflow/lite/delegates/flex/delegate.cc @@ -16,12 +16,14 @@ limitations under the License. #include +#include "absl/strings/str_cat.h" +#include "tensorflow/core/lib/core/status.h" #include "tensorflow/lite/context_util.h" #include "tensorflow/lite/delegates/flex/buffer_map.h" #include "tensorflow/lite/delegates/flex/kernel.h" #include "tensorflow/lite/delegates/flex/util.h" +#include "tensorflow/lite/string_util.h" #include "tensorflow/lite/util.h" -#include "tensorflow/core/lib/core/status.h" namespace tflite { namespace flex { @@ -57,8 +59,8 @@ TfLiteStatus Prepare(TfLiteContext* context, TfLiteDelegate* delegate) { TfLiteStatus CopyFromBufferHandle(TfLiteContext* context, TfLiteDelegate* delegate, - TfLiteBufferHandle buffer_handle, void* data, - size_t size) { + TfLiteBufferHandle buffer_handle, + TfLiteTensor* output) { BufferMap* buffer_map = reinterpret_cast(delegate->data_)->GetBufferMap(context); @@ -68,15 +70,38 @@ TfLiteStatus CopyFromBufferHandle(TfLiteContext* context, } tensorflow::Tensor t = buffer_map->GetTensor(buffer_handle); + + if (output->type == kTfLiteString) { + if (t.dtype() != tensorflow::DT_STRING) { + context->ReportError(context, + "Inconsistent type for TF string tensor index %d.", + buffer_handle); + return kTfLiteError; + } + DynamicBuffer dynamic_buffer; + + auto tf_data = t.flat(); + for (int i = 0; i < t.NumElements(); ++i) { + dynamic_buffer.AddString(tf_data(i).data(), tf_data(i).size()); + } + + dynamic_buffer.WriteToTensor(output, /*new_shape=*/nullptr); + return kTfLiteOk; + } + tensorflow::StringPiece t_data = t.tensor_data(); - if (size != t_data.size()) { - context->ReportError( - context, "Not enough space to store TensorFlow's aligned buffer."); + if (output->bytes != t_data.size()) { + context->ReportError(context, + absl::StrCat("The given ", output->bytes, + " bytes are not enough to store " + "TensorFlow's aligned buffer of size ", + t_data.size(), " bytes.") + .c_str()); return kTfLiteError; } - memcpy(data, t_data.data(), t_data.size()); + memcpy(output->data.raw, t_data.data(), t_data.size()); return kTfLiteOk; } @@ -104,14 +129,13 @@ std::unique_ptr FlexDelegate::Create() { } FlexDelegate::FlexDelegate(std::unique_ptr delegate_data) - : TfLiteDelegate{ - /*data_=*/delegate_data.get(), - /*nullptr,*/ &flex::delegate::Prepare, - /*CopyFromBufferHandle=*/&flex::delegate::CopyFromBufferHandle, - /*CopyToBufferHandle=*/nullptr, - /*FreeBufferHandle=*/nullptr, - /*flags=*/kTfLiteDelegateFlagsAllowDynamicTensors}, - delegate_data_(std::move(delegate_data)) {} + : TfLiteDelegate(TfLiteDelegateCreate()), + delegate_data_(std::move(delegate_data)) { + data_ = delegate_data_.get(); + Prepare = &flex::delegate::Prepare; + CopyFromBufferHandle = &flex::delegate::CopyFromBufferHandle; + flags = kTfLiteDelegateFlagsAllowDynamicTensors; +} FlexDelegate::~FlexDelegate() {} diff --git a/tensorflow/lite/delegates/flex/delegate_data.cc b/tensorflow/lite/delegates/flex/delegate_data.cc index b62479a4480..1483a530388 100644 --- a/tensorflow/lite/delegates/flex/delegate_data.cc +++ b/tensorflow/lite/delegates/flex/delegate_data.cc @@ -14,20 +14,21 @@ limitations under the License. ==============================================================================*/ #include "tensorflow/lite/delegates/flex/delegate_data.h" +#include "absl/memory/memory.h" #include "tensorflow/core/common_runtime/device_factory.h" #include "tensorflow/core/lib/core/status.h" namespace tflite { namespace flex { tensorflow::Status DelegateData::Create(std::unique_ptr* data) { - std::vector devices; + std::vector> devices; TF_RETURN_IF_ERROR(tensorflow::DeviceFactory::AddDevices( tensorflow::SessionOptions(), "/job:localhost/replica:0/task:0", &devices)); - std::unique_ptr device_mgr( - new tensorflow::DeviceMgr(devices)); + std::unique_ptr device_mgr = + absl::make_unique(std::move(devices)); // Note that Rendezvous is ref-counted so it will be automatically deleted. tensorflow::Rendezvous* rendezvous = new tensorflow::IntraProcessRendezvous(device_mgr.get()); diff --git a/tensorflow/lite/delegates/flex/delegate_test.cc b/tensorflow/lite/delegates/flex/delegate_test.cc index e13029d9a51..1b2f476f03f 100644 --- a/tensorflow/lite/delegates/flex/delegate_test.cc +++ b/tensorflow/lite/delegates/flex/delegate_test.cc @@ -93,6 +93,25 @@ TEST_F(DelegateTest, NonFloatTypeInference) { ASSERT_EQ(GetType(2), kTfLiteInt32); } +TEST_F(DelegateTest, StringInference) { + AddTensors(3, {0, 1}, {2}, kTfLiteString, {2}); + + AddTfOp(testing::kAdd, {0, 1}, {2}); + + ConfigureDelegate(); + + SetShape(0, {2, 2}); + SetStringValues(0, {"1", "2", "3", "4"}); + SetShape(1, {2, 2}); + SetStringValues(1, {"4", "3", "2", "1"}); + + ASSERT_TRUE(Invoke()); + + ASSERT_THAT(GetShape(2), ElementsAre(2, 2)); + ASSERT_THAT(GetStringValues(2), ElementsAre("14", "23", "32", "41")); + ASSERT_EQ(GetType(2), kTfLiteString); +} + TEST_F(DelegateTest, MixedGraph) { AddTensors(9, {0, 3}, {8}, kTfLiteFloat32, {3}); diff --git a/tensorflow/lite/delegates/flex/kernel.cc b/tensorflow/lite/delegates/flex/kernel.cc index c4fe142dff1..02da1d1a224 100644 --- a/tensorflow/lite/delegates/flex/kernel.cc +++ b/tensorflow/lite/delegates/flex/kernel.cc @@ -15,6 +15,12 @@ limitations under the License. #include "tensorflow/lite/delegates/flex/kernel.h" #include "flatbuffers/flexbuffers.h" // TF:flatbuffers +#include "tensorflow/core/common_runtime/eager/context.h" +#include "tensorflow/core/common_runtime/eager/execute.h" +#include "tensorflow/core/common_runtime/eager/tensor_handle.h" +#include "tensorflow/core/framework/node_def.pb.h" +#include "tensorflow/core/framework/node_def_util.h" +#include "tensorflow/core/lib/core/errors.h" #include "tensorflow/lite/builtin_ops.h" #include "tensorflow/lite/c/c_api_internal.h" #include "tensorflow/lite/context_util.h" @@ -22,11 +28,6 @@ limitations under the License. #include "tensorflow/lite/delegates/flex/util.h" #include "tensorflow/lite/kernels/kernel_util.h" #include "tensorflow/lite/string.h" -#include "tensorflow/core/common_runtime/eager/context.h" -#include "tensorflow/core/common_runtime/eager/execute.h" -#include "tensorflow/core/common_runtime/eager/tensor_handle.h" -#include "tensorflow/core/framework/node_def.pb.h" -#include "tensorflow/core/framework/node_def_util.h" // Note: this is part of TF Lite's Flex delegation code which is to be // completed soon. @@ -78,11 +79,18 @@ tensorflow::Status ExecuteFlexOp(tensorflow::EagerContext* eager_context, const std::vector& inputs, const std::vector& outputs) { const tensorflow::AttrTypeMap* attr_types; + bool is_function = false; TF_RETURN_WITH_CONTEXT_IF_ERROR( - tensorflow::AttrTypeMapForOp(op_name.c_str(), &attr_types), + tensorflow::AttrTypeMapForOp(op_name.c_str(), &attr_types, &is_function), " (while processing attributes of '", op_name, "')"); - - tensorflow::EagerOperation op(eager_context, op_name.c_str(), attr_types); + if (is_function) { + return tensorflow::errors::NotFound( + "Operation '", op_name, + "' is not registered. (while processing attributes of '", op_name, + "')"); + } + tensorflow::EagerOperation op(eager_context, op_name.c_str(), + /*is_function=*/false, attr_types); for (const auto& attr : nodedef.attr()) { op.MutableAttrs()->Set(attr.first, attr.second); } diff --git a/tensorflow/lite/delegates/flex/kernel_test.cc b/tensorflow/lite/delegates/flex/kernel_test.cc index f55759594df..efb7300b0bd 100644 --- a/tensorflow/lite/delegates/flex/kernel_test.cc +++ b/tensorflow/lite/delegates/flex/kernel_test.cc @@ -59,12 +59,12 @@ class KernelTest : public testing::FlexModelTest { delegate_.CopyFromBufferHandle = [](TfLiteContext* context, TfLiteDelegate* delegate, TfLiteBufferHandle buffer_handle, - void* data, size_t size) { + TfLiteTensor* output) { auto* delegate_data = reinterpret_cast(delegate->data_); tensorflow::StringPiece values = delegate_data->GetBufferMap(context) ->GetTensor(buffer_handle) .tensor_data(); - memcpy(data, values.data(), values.size()); + memcpy(output->data.raw, values.data(), values.size()); return kTfLiteOk; }; CHECK(interpreter_->ModifyGraphWithDelegate(&delegate_) == kTfLiteOk); diff --git a/tensorflow/lite/delegates/flex/test_util.cc b/tensorflow/lite/delegates/flex/test_util.cc index 08feb349e6d..aa24675a7b1 100644 --- a/tensorflow/lite/delegates/flex/test_util.cc +++ b/tensorflow/lite/delegates/flex/test_util.cc @@ -25,6 +25,29 @@ namespace testing { bool FlexModelTest::Invoke() { return interpreter_->Invoke() == kTfLiteOk; } +void FlexModelTest::SetStringValues(int tensor_index, + const std::vector& values) { + DynamicBuffer dynamic_buffer; + for (const string& s : values) { + dynamic_buffer.AddString(s.data(), s.size()); + } + dynamic_buffer.WriteToTensor(interpreter_->tensor(tensor_index), + /*new_shape=*/nullptr); +} + +std::vector FlexModelTest::GetStringValues(int tensor_index) const { + std::vector result; + + TfLiteTensor* tensor = interpreter_->tensor(tensor_index); + auto num_strings = GetStringCount(tensor->data.raw); + for (size_t i = 0; i < num_strings; ++i) { + auto ref = GetString(tensor->data.raw, i); + result.push_back(string(ref.str, ref.len)); + } + + return result; +} + void FlexModelTest::SetShape(int tensor_index, const std::vector& values) { ASSERT_EQ(interpreter_->ResizeInputTensor(tensor_index, values), kTfLiteOk); ASSERT_EQ(interpreter_->AllocateTensors(), kTfLiteOk); @@ -95,12 +118,22 @@ void FlexModelTest::AddTfOp(TfOpType op, const std::vector& inputs, return " attr{ key: '" + key + "' value {" + value + "}}"; }; - // Crude type attribution, will need fleshing out as more tests are added. - // TODO(b/113613439): Use nodedef string utilities to properly handle - // all types. - string type_attribute = attr("T", "type: DT_FLOAT"); - if (interpreter_->tensor(inputs[0])->type == kTfLiteInt32) { - type_attribute = attr("T", "type: DT_INT32"); + string type_attribute; + switch (interpreter_->tensor(inputs[0])->type) { + case kTfLiteInt32: + type_attribute = attr("T", "type: DT_INT32"); + break; + case kTfLiteFloat32: + type_attribute = attr("T", "type: DT_FLOAT"); + break; + case kTfLiteString: + type_attribute = attr("T", "type: DT_STRING"); + break; + default: + // TODO(b/113613439): Use nodedef string utilities to properly handle all + // types. + LOG(FATAL) << "Type not supported"; + break; } if (op == kUnpack) { diff --git a/tensorflow/lite/delegates/flex/test_util.h b/tensorflow/lite/delegates/flex/test_util.h index 4d3f5ad0968..2cc2dc30e92 100644 --- a/tensorflow/lite/delegates/flex/test_util.h +++ b/tensorflow/lite/delegates/flex/test_util.h @@ -63,11 +63,13 @@ class FlexModelTest : public ::testing::Test { void SetValues(int tensor_index, const std::vector& values) { SetTypedValues(tensor_index, values); } + void SetStringValues(int tensor_index, const std::vector& values); // Returns the tensor's values at the given index. std::vector GetValues(int tensor_index) { return GetTypedValues(tensor_index); } + std::vector GetStringValues(int tensor_index) const; // Sets the tensor's shape at the given index. void SetShape(int tensor_index, const std::vector& values); diff --git a/tensorflow/lite/delegates/flex/util.cc b/tensorflow/lite/delegates/flex/util.cc index c786ffa1a21..c995b360f9d 100644 --- a/tensorflow/lite/delegates/flex/util.cc +++ b/tensorflow/lite/delegates/flex/util.cc @@ -66,6 +66,8 @@ TF_DataType GetTensorFlowDataType(TfLiteType type) { return TF_INT32; case kTfLiteUInt8: return TF_UINT8; + case kTfLiteInt8: + return TF_INT8; case kTfLiteInt64: return TF_INT64; case kTfLiteComplex64: @@ -87,6 +89,8 @@ TfLiteType GetTensorFlowLiteType(TF_DataType type) { return kTfLiteInt32; case TF_UINT8: return kTfLiteUInt8; + case TF_INT8: + return kTfLiteInt8; case TF_INT64: return kTfLiteInt64; case TF_COMPLEX64: diff --git a/tensorflow/lite/delegates/nnapi/nnapi_delegate.cc b/tensorflow/lite/delegates/nnapi/nnapi_delegate.cc index 9690c659211..4fe07004a82 100644 --- a/tensorflow/lite/delegates/nnapi/nnapi_delegate.cc +++ b/tensorflow/lite/delegates/nnapi/nnapi_delegate.cc @@ -1141,7 +1141,7 @@ class NNAPIDelegateKernel { TfLiteDelegate* NnApiDelegate() { static TfLiteDelegate delegate = { .data_ = nullptr, - .flags = kTfLiteDelegateFlagsAllowDynamicTensors, + .flags = kTfLiteDelegateFlagsNone, .Prepare = [](TfLiteContext* context, TfLiteDelegate* delegate) -> TfLiteStatus { // Do not check nodes_ if NN API is unavailable. diff --git a/tensorflow/lite/delegates/nnapi/nnapi_delegate_test.cc b/tensorflow/lite/delegates/nnapi/nnapi_delegate_test.cc index 84a0a6a1d1c..ca48af0c952 100644 --- a/tensorflow/lite/delegates/nnapi/nnapi_delegate_test.cc +++ b/tensorflow/lite/delegates/nnapi/nnapi_delegate_test.cc @@ -34,6 +34,11 @@ class SingleOpModelWithNNAPI : public SingleOpModel { interpreter->ModifyGraphWithDelegate(NnApiDelegate()); }); } + + TfLiteStatus ResizeInputTensor(int tensor_index, + const std::vector& dims) { + return interpreter_->ResizeInputTensor(tensor_index, dims); + } }; class FloatAddOpModel : public SingleOpModelWithNNAPI { @@ -97,6 +102,17 @@ TEST(NNAPIDelegate, AddWithRelu) { EXPECT_THAT(m.GetOutput(), ElementsAreArray({0.0, 0.4, 1.0, 1.3})); } +// Verify that resize attempts fail. +// TODO(b/113110851): Verify success after the delegate supports resizing. +TEST(NNAPIDelegate, ResizeFails) { + FloatAddOpModel m({TensorType_FLOAT32, {1, 2, 2, 1}}, + {TensorType_FLOAT32, {1, 2, 2, 1}}, + {TensorType_FLOAT32, {}}, ActivationFunctionType_NONE); + m.PopulateTensor(m.input1(), {-2.0, 0.2, 0.7, 0.8}); + m.PopulateTensor(m.input2(), {0.1, 0.2, 0.3, 0.5}); + EXPECT_EQ(m.ResizeInputTensor(m.input1(), {1, 3, 3, 1}), kTfLiteError); +} + class FloatMulOpModel : public SingleOpModelWithNNAPI { public: FloatMulOpModel(const TensorData& input1, const TensorData& input2, diff --git a/tensorflow/lite/examples/ios/camera/CameraExampleViewController.h b/tensorflow/lite/examples/ios/camera/CameraExampleViewController.h index 6bc94e95022..fb5800e86d3 100644 --- a/tensorflow/lite/examples/ios/camera/CameraExampleViewController.h +++ b/tensorflow/lite/examples/ios/camera/CameraExampleViewController.h @@ -17,8 +17,8 @@ #include -#include "tensorflow/lite/kernels/register.h" -#include "tensorflow/lite/model.h" +#include "tensorflow/contrib/lite/kernels/register.h" +#include "tensorflow/contrib/lite/model.h" @interface CameraExampleViewController : UIViewController { diff --git a/tensorflow/lite/examples/ios/camera/CameraExampleViewController.mm b/tensorflow/lite/examples/ios/camera/CameraExampleViewController.mm index 1e6725592b0..996cff26162 100644 --- a/tensorflow/lite/examples/ios/camera/CameraExampleViewController.mm +++ b/tensorflow/lite/examples/ios/camera/CameraExampleViewController.mm @@ -23,10 +23,10 @@ #include #include -#include "tensorflow/lite/kernels/register.h" -#include "tensorflow/lite/model.h" -#include "tensorflow/lite/string_util.h" -#include "tensorflow/lite/op_resolver.h" +#include "tensorflow/contrib/lite/kernels/register.h" +#include "tensorflow/contrib/lite/model.h" +#include "tensorflow/contrib/lite/string_util.h" +#include "tensorflow/contrib/lite/op_resolver.h" #define LOG(x) std::cerr diff --git a/tensorflow/lite/examples/ios/camera/Podfile b/tensorflow/lite/examples/ios/camera/Podfile index f460693122a..96a0d234265 100644 --- a/tensorflow/lite/examples/ios/camera/Podfile +++ b/tensorflow/lite/examples/ios/camera/Podfile @@ -2,4 +2,4 @@ platform :ios, '8.0' inhibit_all_warnings! target 'tflite_camera_example' - pod 'TensorFlowLite', '1.10.1' + pod 'TensorFlowLite', '1.12.0' diff --git a/tensorflow/lite/examples/ios/simple/Podfile b/tensorflow/lite/examples/ios/simple/Podfile index ddb77088d9f..931b72c1f5e 100644 --- a/tensorflow/lite/examples/ios/simple/Podfile +++ b/tensorflow/lite/examples/ios/simple/Podfile @@ -2,4 +2,4 @@ platform :ios, '8.0' inhibit_all_warnings! target 'tflite_simple_example' - pod 'TensorFlowLite', '1.10.1' + pod 'TensorFlowLite', '1.12.0' diff --git a/tensorflow/lite/examples/ios/simple/RunModelViewController.mm b/tensorflow/lite/examples/ios/simple/RunModelViewController.mm index e5764944f66..650c73f7322 100644 --- a/tensorflow/lite/examples/ios/simple/RunModelViewController.mm +++ b/tensorflow/lite/examples/ios/simple/RunModelViewController.mm @@ -22,10 +22,10 @@ #include #include -#include "tensorflow/lite/kernels/register.h" -#include "tensorflow/lite/model.h" -#include "tensorflow/lite/string_util.h" -#include "tensorflow/lite/op_resolver.h" +#include "tensorflow/contrib/lite/kernels/register.h" +#include "tensorflow/contrib/lite/model.h" +#include "tensorflow/contrib/lite/string_util.h" +#include "tensorflow/contrib/lite/op_resolver.h" #include "ios_image_load.h" diff --git a/tensorflow/lite/experimental/examples/lstm/unidirectional_sequence_lstm_test.py b/tensorflow/lite/experimental/examples/lstm/unidirectional_sequence_lstm_test.py index eeb48d12311..9c00d0501ab 100644 --- a/tensorflow/lite/experimental/examples/lstm/unidirectional_sequence_lstm_test.py +++ b/tensorflow/lite/experimental/examples/lstm/unidirectional_sequence_lstm_test.py @@ -111,7 +111,7 @@ class UnidirectionalSequenceLstmTest(test_util.TensorFlowTestCase): # Initialize variables init = tf.global_variables_initializer() - sess.run(init) + self.evaluate(init) for _ in range(TRAIN_STEPS): batch_x, batch_y = self.mnist.train.next_batch( batch_size=self.batch_size, shuffle=False) diff --git a/tensorflow/lite/experimental/micro/README.md b/tensorflow/lite/experimental/micro/README.md index 673daed74c4..cc2a62cb8a9 100644 --- a/tensorflow/lite/experimental/micro/README.md +++ b/tensorflow/lite/experimental/micro/README.md @@ -126,3 +126,45 @@ debug logs here, along with the magic string `~~~ALL TESTS PASSED~~~`. This is the exact same code as before, just compiled and run on the STM32F103 rather than your desktop. We hope that the simplicity of this testing approach will help make adding support for new platforms as easy as possible. + +## Building for Apollo3 + +Follow these steps to get the pushbutton yes/no example working on Apollo 3: + +1. Make sure to run the "Getting Started" section before performing the following steps +2. Download Apollo3-SDK-2018.08.13 and place in tensorflow/lite/experimental/micro/tools/make/downloads +3. Copy and prepare files by running tensorflow/lite/experimental/micro/tools/make/targets/apollo3evb/prep_apollo3_files.sh from the toplevel of git +4. Install [Segger JLink tools](https://www.segger.com/downloads/jlink/) +5. Make sure the [GNU Arm Embedded Toolchain (gcc-arm-none-eabi-7-2018-q2-update)](https://developer.arm.com/open-source/gnu-toolchain/gnu-rm/downloads) is installed to tensorflow/lite/experimental/micro/tools/make/downloads + 1. Confirm directory is in $PATH +6. Compile the project with the following command: make -f tensorflow/lite/experimental/micro/tools/make/Makefile TARGET=apollo3evb pushbutton_cmsis_speech_test_bin +7. Connect the Apollo3 EVB (with mic shield) to the computer and power it on +8. Start the GDB server in a new terminal with the following command: JLinkGDBServer -select USB -device AMA3B1KK-KBR -endian little -if SWD -speed 1000 -noir -noLocalhostOnly + 1. The command has run successfully if you see the message "Waiting for GDB connection" +9. Back in the original terminal, run the program via the debugger + 1. Navigate to tensorflow/lite/experimental/micro/examples/micro_speech/apollo3 + 2. Start gdb by entering the following command: arm-none-eabi-gdb + 3. Run the command script by entering the following command: source pushbutton_cmsis_scores.cmd. This script does the following: + 1. Load the binary created in step 6 + 2. Set a breakpoint after inference scores have been computed + 3. Tell the debugger what variables should be printed out at this breakpoint + 4. Begin program execution + 5. Press Ctrl+c to exit + 4. Press BTN2. An LED will flash for 1 second. Speak your utterance during this one second + 5. The debugger will print out four numbers. They are the probabilites for 1) no speech, 2) unknown speech, 3) yes, 4) no + 6. The EVB LEDs will indicate detection. + 1. LED0 (rightmost LED) - ON when capturing 1sec of audio + 2. LED1 - ON when detecting silence + 3. LED2 - ON when detecting UNKNOWN utterance + 4. LED3 - ON when detecting YES utterance + 5. LED4 (leftmost LED) - ON when detecting NO utterance + +### Additional Apollo3 Instructions + +To flash a part with JFlash Lite, do the following: +1. At the command line: JFlashLiteExe +2. Device = AMA3B1KK-KBR +3. Interface = SWD at 1000 kHz +4. Data file = tensorflow/lite/experimental/micro/tools/make/gen/apollo3evb_cortex-m4/bin/pushbutton_cmsis_speech_test.bin +5. Prog Addr = 0x0000C000 + diff --git a/tensorflow/lite/experimental/micro/examples/micro_speech/.gitignore b/tensorflow/lite/experimental/micro/examples/micro_speech/.gitignore new file mode 100644 index 00000000000..d8dd7532abc --- /dev/null +++ b/tensorflow/lite/experimental/micro/examples/micro_speech/.gitignore @@ -0,0 +1 @@ +*.wav diff --git a/tensorflow/lite/experimental/micro/examples/micro_speech/BUILD b/tensorflow/lite/experimental/micro/examples/micro_speech/BUILD index 07fb8764113..799b2e5a5dd 100644 --- a/tensorflow/lite/experimental/micro/examples/micro_speech/BUILD +++ b/tensorflow/lite/experimental/micro/examples/micro_speech/BUILD @@ -10,18 +10,46 @@ load( "tflite_micro_cc_test", ) +cc_library( + name = "model_settings", + srcs = [ + "model_settings.cc", + ], + hdrs = [ + "model_settings.h", + ], +) + +cc_library( + name = "tiny_conv_model_data", + srcs = [ + "tiny_conv_model_data.cc", + ], + hdrs = [ + "tiny_conv_model_data.h", + ], +) + +cc_library( + name = "features_test_data", + srcs = [ + "no_features_data.cc", + "yes_features_data.cc", + ], + hdrs = [ + "no_features_data.h", + "yes_features_data.h", + ], +) + tflite_micro_cc_test( name = "micro_speech_test", srcs = [ "micro_speech_test.cc", - "no_features_data.cc", - "no_features_data.h", - "tiny_conv_model_data.cc", - "tiny_conv_model_data.h", - "yes_features_data.cc", - "yes_features_data.h", ], deps = [ + ":features_test_data", + ":tiny_conv_model_data", "//tensorflow/lite:schema_fbs_version", "//tensorflow/lite/experimental/micro:micro_framework", "//tensorflow/lite/experimental/micro/kernels:all_ops_resolver", @@ -31,46 +59,185 @@ tflite_micro_cc_test( ], ) +cc_library( + name = "preprocessor_test_data", + srcs = [ + "no_30ms_sample_data.cc", + "no_power_spectrum_data.cc", + "yes_30ms_sample_data.cc", + "yes_power_spectrum_data.cc", + ], + hdrs = [ + "no_30ms_sample_data.h", + "no_power_spectrum_data.h", + "yes_30ms_sample_data.h", + "yes_power_spectrum_data.h", + ], +) + +cc_library( + name = "preprocessor_reference", + srcs = [ + "preprocessor.cc", + ], + hdrs = [ + "preprocessor.h", + ], + deps = [ + ":model_settings", + "//tensorflow/lite/c:c_api_internal", + "//tensorflow/lite/experimental/micro:micro_framework", + ], +) + tflite_micro_cc_test( name = "preprocessor_reference_test", srcs = [ - "no_30ms_sample_data.cc", - "no_30ms_sample_data.h", - "no_power_spectrum_data.cc", - "no_power_spectrum_data.h", - "preprocessor.cc", - "preprocessor.h", "preprocessor_test.cc", - "yes_30ms_sample_data.cc", - "yes_30ms_sample_data.h", - "yes_power_spectrum_data.cc", - "yes_power_spectrum_data.h", ], deps = [ + ":model_settings", + ":preprocessor_reference", + ":preprocessor_test_data", "//tensorflow/lite/c:c_api_internal", "//tensorflow/lite/experimental/micro:micro_framework", "//tensorflow/lite/experimental/micro/testing:micro_test", ], ) +cc_library( + name = "preprocessor_fixed", + srcs = [ + "fixed_point/preprocessor.cc", + ], + hdrs = [ + "preprocessor.h", + ], + deps = [ + ":model_settings", + "//tensorflow/lite/c:c_api_internal", + "//tensorflow/lite/experimental/micro:micro_framework", + ], +) + tflite_micro_cc_test( name = "preprocessor_fixed_test", srcs = [ - "fixed_point/preprocessor.cc", - "no_30ms_sample_data.cc", - "no_30ms_sample_data.h", - "no_power_spectrum_data.cc", - "no_power_spectrum_data.h", - "preprocessor.h", "preprocessor_test.cc", - "yes_30ms_sample_data.cc", - "yes_30ms_sample_data.h", - "yes_power_spectrum_data.cc", - "yes_power_spectrum_data.h", ], deps = [ + ":model_settings", + ":preprocessor_fixed", + ":preprocessor_test_data", "//tensorflow/lite/c:c_api_internal", "//tensorflow/lite/experimental/micro:micro_framework", "//tensorflow/lite/experimental/micro/testing:micro_test", ], ) + +cc_library( + name = "audio_provider", + srcs = [ + "audio_provider.cc", + ], + hdrs = [ + "audio_provider.h", + ], + deps = [ + ":model_settings", + "//tensorflow/lite/c:c_api_internal", + "//tensorflow/lite/experimental/micro:micro_framework", + ], +) + +tflite_micro_cc_test( + name = "audio_provider_test", + srcs = [ + "audio_provider_test.cc", + ], + deps = [ + ":audio_provider", + ":model_settings", + "//tensorflow/lite/c:c_api_internal", + "//tensorflow/lite/experimental/micro:micro_framework", + "//tensorflow/lite/experimental/micro/testing:micro_test", + ], +) + +cc_library( + name = "feature_provider", + srcs = [ + "feature_provider.cc", + ], + hdrs = [ + "feature_provider.h", + ], + deps = [ + ":audio_provider", + ":model_settings", + ":preprocessor_reference", + ":timer", + "//tensorflow/lite/c:c_api_internal", + "//tensorflow/lite/experimental/micro:micro_framework", + ], +) + +tflite_micro_cc_test( + name = "feature_provider_test", + srcs = [ + "feature_provider_test.cc", + ], + deps = [ + ":audio_provider", + ":feature_provider", + ":model_settings", + ":timer", + "//tensorflow/lite/c:c_api_internal", + "//tensorflow/lite/experimental/micro:micro_framework", + "//tensorflow/lite/experimental/micro/testing:micro_test", + ], +) + +cc_library( + name = "timer", + srcs = [ + "timer.cc", + ], + hdrs = [ + "timer.h", + ], +) + +tflite_micro_cc_test( + name = "timer_test", + srcs = [ + "timer_test.cc", + ], + deps = [ + ":timer", + "//tensorflow/lite/c:c_api_internal", + "//tensorflow/lite/experimental/micro:micro_framework", + "//tensorflow/lite/experimental/micro/testing:micro_test", + ], +) + +cc_binary( + name = "micro_speech", + srcs = [ + "main.cc", + ], + deps = [ + ":audio_provider", + ":feature_provider", + ":features_test_data", + ":model_settings", + ":preprocessor_reference", + ":timer", + ":tiny_conv_model_data", + "//tensorflow/lite:schema_fbs_version", + "//tensorflow/lite/experimental/micro:micro_framework", + "//tensorflow/lite/experimental/micro/kernels:all_ops_resolver", + "//tensorflow/lite/experimental/micro/kernels:micro_ops", + "//tensorflow/lite/schema:schema_fbs", + ], +) diff --git a/tensorflow/lite/experimental/micro/examples/micro_speech/CMSIS/README.md b/tensorflow/lite/experimental/micro/examples/micro_speech/CMSIS/README.md new file mode 100644 index 00000000000..fde48374c8e --- /dev/null +++ b/tensorflow/lite/experimental/micro/examples/micro_speech/CMSIS/README.md @@ -0,0 +1,10 @@ +# Description of files + +* **arm_cmplx_mag_squared_q10p6.c**: Modified version of the ARM CMSIS function [arm_cmplx_mag_squared.c](http://arm-software.github.io/CMSIS_5/DSP/html/group__cmplx__mag__squared.html#ga45537f576102d960d467eb722b8431f2). The modification is that we have changed the amount of right-shift to make sure our data is in the correct range. We redistribute because the original content was created with the Apache 2.0 license. +* **arm_cmplx_mag_squared_q10p6.h**: Header file for arm_cmplx_mag_squared_q10p6.c +* **create_constants.py**: Python file used to create hanning.cc, hanning.h, sin_1k.cc, and sin_1k.h +* **hanning.cc**: Precomputed [Hann window](https://en.wikipedia.org/wiki/Hann_function) for use in the preprocessor. This file is created in ../create_constants.py +* **hanning.h**: Header file fro hanning.cc +* **preprocessor.cc**: CMSIS version of the preprocessor +* **sin_1k.cc**: A 1 kHZ sinusoid used for comparing the CMSIS preprocessor with the Micro-Lite fixed_point preprocessor +* **sin_1k.h**: Header file for sin_1k.cc diff --git a/tensorflow/lite/experimental/micro/examples/micro_speech/CMSIS/arm_cmplx_mag_squared_q10p6.c b/tensorflow/lite/experimental/micro/examples/micro_speech/CMSIS/arm_cmplx_mag_squared_q10p6.c new file mode 100644 index 00000000000..b050f6048d5 --- /dev/null +++ b/tensorflow/lite/experimental/micro/examples/micro_speech/CMSIS/arm_cmplx_mag_squared_q10p6.c @@ -0,0 +1,141 @@ +/* This file is a modification of the ARM CMSIS library file arm_cmplx_mag_squared_q15.c + * We have retained the original copyright and header information, in + * accordance with the Apache 2.0 license terms. + */ + +/* ---------------------------------------------------------------------- + * Project: CMSIS DSP Library + * Title: arm_cmplx_mag_squared_q15.c + * Description: Q15 complex magnitude squared + * + * $Date: 27. January 2017 + * $Revision: V.1.5.1 + * + * Target Processor: Cortex-M cores + * -------------------------------------------------------------------- */ +/* + * Copyright (C) 2010-2017 ARM Limited or its affiliates. All rights reserved. + * + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the License); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an AS IS BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "arm_math.h" + +/** + * @ingroup groupCmplxMath + */ + +/** + * @addtogroup cmplx_mag_squared + * @{ + */ + +/** + * @brief Q15 complex magnitude squared + * @param *pSrc points to the complex input vector + * @param *pDst points to the real output vector + * @param numSamples number of complex samples in the input vector + * @return none. + * + * Scaling and Overflow Behavior: + * \par + * The function implements 1.15 by 1.15 multiplications and finally output is converted into 3.13 format. + */ + +void arm_cmplx_mag_squared_q10p6( + q15_t * pSrc, + q15_t * pDst, + uint32_t numSamples) +{ + q31_t acc0, acc1; /* Accumulators */ + +#if defined (ARM_MATH_DSP) + + /* Run the below code for Cortex-M4 and Cortex-M3 */ + uint32_t blkCnt; /* loop counter */ + q31_t in1, in2, in3, in4; + q31_t acc2, acc3; + + /*loop Unrolling */ + blkCnt = numSamples >> 2U; + + /* First part of the processing with loop unrolling. Compute 4 outputs at a time. + ** a second loop below computes the remaining 1 to 3 samples. */ + while (blkCnt > 0U) + { + /* C[0] = (A[0] * A[0] + A[1] * A[1]) */ + in1 = *__SIMD32(pSrc)++; + in2 = *__SIMD32(pSrc)++; + in3 = *__SIMD32(pSrc)++; + in4 = *__SIMD32(pSrc)++; + + acc0 = __SMUAD(in1, in1); + acc1 = __SMUAD(in2, in2); + acc2 = __SMUAD(in3, in3); + acc3 = __SMUAD(in4, in4); + + /* store the result in 3.13 format in the destination buffer. */ + *pDst++ = (q15_t) (acc0 >> 6); + *pDst++ = (q15_t) (acc1 >> 6); + *pDst++ = (q15_t) (acc2 >> 6); + *pDst++ = (q15_t) (acc3 >> 6); + + /* Decrement the loop counter */ + blkCnt--; + } + + /* If the numSamples is not a multiple of 4, compute any remaining output samples here. + ** No loop unrolling is used. */ + blkCnt = numSamples % 0x4U; + + while (blkCnt > 0U) + { + /* C[0] = (A[0] * A[0] + A[1] * A[1]) */ + in1 = *__SIMD32(pSrc)++; + acc0 = __SMUAD(in1, in1); + + /* store the result in 3.13 format in the destination buffer. */ + *pDst++ = (q15_t) (acc0 >> 6); + + /* Decrement the loop counter */ + blkCnt--; + } + +#else + + /* Run the below code for Cortex-M0 */ + q15_t real, imag; /* Temporary variables to store real and imaginary values */ + + while (numSamples > 0U) + { + /* out = ((real * real) + (imag * imag)) */ + real = *pSrc++; + imag = *pSrc++; + acc0 = (real * real); + acc1 = (imag * imag); + /* store the result in 3.13 format in the destination buffer. */ + *pDst++ = (q15_t) (((q63_t) acc0 + acc1) >> 6); + + /* Decrement the loop counter */ + numSamples--; + } + +#endif /* #if defined (ARM_MATH_DSP) */ + +} + +/** + * @} end of cmplx_mag_squared group + */ diff --git a/tensorflow/lite/experimental/micro/examples/micro_speech/CMSIS/arm_cmplx_mag_squared_q10p6.h b/tensorflow/lite/experimental/micro/examples/micro_speech/CMSIS/arm_cmplx_mag_squared_q10p6.h new file mode 100644 index 00000000000..24144615cc6 --- /dev/null +++ b/tensorflow/lite/experimental/micro/examples/micro_speech/CMSIS/arm_cmplx_mag_squared_q10p6.h @@ -0,0 +1,33 @@ +/* This file is a modification of the ARM CMSIS library file arm_math.h + * We have retained the original copyright and header information, in + * accordance with the Apache 2.0 license terms. + */ + +/****************************************************************************** + * @file arm_math.h + * @brief Public header file for CMSIS DSP LibraryU + * @version V1.5.3 + * @date 10. January 2018 + ******************************************************************************/ +/* + * Copyright (C) 2010-2017 ARM Limited or its affiliates. All rights reserved. + * + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the License); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an AS IS BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +void arm_cmplx_mag_squared_q10p6( + q15_t * pSrc, + q15_t * pDst, + uint32_t numSamples); diff --git a/tensorflow/lite/experimental/micro/examples/micro_speech/CMSIS/create_constants.py b/tensorflow/lite/experimental/micro/examples/micro_speech/CMSIS/create_constants.py new file mode 100755 index 00000000000..c7cf8bf61be --- /dev/null +++ b/tensorflow/lite/experimental/micro/examples/micro_speech/CMSIS/create_constants.py @@ -0,0 +1,62 @@ +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the License); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an AS IS BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import soundfile as sf +import numpy as np + +def to_cc(x, varname, directory='', scale_factor=1): + x = (x/np.max(np.abs(x)))*32768*scale_factor + x[x>32767] = 32767 + x[x<-32768] = -32768 + x = x.astype(int) + x = [str(v) if i%10!=0 else '\n '+str(v) for i,v in enumerate(x)] + + cmsis_path = "tensorflow/lite/experimental/micro/examples/micro_speech/CMSIS" + xstr = '#include "{}/{}.h"\n\n'.format(cmsis_path, varname) + xstr += 'const int g_{}_size = {};\n'.format(varname, len(x)) + xstr += 'const int16_t g_{}[{}] = {{{}}};\n'.format(varname, len(x), ', '.join(x)) + + with open(directory+varname+'.cc','w') as f: + f.write(xstr) + +def to_h(x, varname, directory=''): + tf_prepend = 'TENSORFLOW_LITE_EXPERIMENTAL_MICRO_EXAMPLES_MICRO_SPEECH_' + xstr = '#ifndef {}{}_H_\n'.format(tf_prepend, varname.upper()) + xstr += '#define {}{}_H_\n\n'.format(tf_prepend, varname.upper()) + xstr += '#include \n\n' + xstr += 'extern const int g_{}_size;\n'.format(varname) + xstr += 'extern const int16_t g_{}[];\n\n'.format(varname) + xstr += '#endif' + + with open(directory+varname+'.h','w') as f: + f.write(xstr) + +#x = sf.read('yes_f2e59fea_nohash_1.wav')[0] +#to_cc(x, 'yes_waveform') +#to_h(x, 'yes_waveform') +# +#x = sf.read('no_f9643d42_nohash_4.wav')[0] +#to_cc(x, 'no_waveform') +#to_h(x, 'no_waveform') + + +# 30ms of data @ 16 kHz = 480 samples +hann = np.hanning(int(16000*0.03)) # Window 30ms of data +to_cc(hann, 'hanning', directory='./') +to_h(hann, 'hanning', directory='./') + +t = np.arange(16000.*0.03)/16000. +sin1k = np.sin(2*np.pi*1000*t) # Factor of 10 because micro preprocessing overflows otherwise +to_cc(sin1k, 'sin_1k', directory='./', scale_factor=0.1) +to_h(sin1k, 'sin_1k', directory='./') diff --git a/tensorflow/lite/experimental/micro/examples/micro_speech/CMSIS/hann.h b/tensorflow/lite/experimental/micro/examples/micro_speech/CMSIS/hann.h deleted file mode 100644 index b610f79190b..00000000000 --- a/tensorflow/lite/experimental/micro/examples/micro_speech/CMSIS/hann.h +++ /dev/null @@ -1 +0,0 @@ -q15_t hann[480] = {0, 1, 5, 12, 22, 35, 50, 69, 90, 114, 140, 170, 202, 237, 275, 316, 359, 405, 454, 506, 560, 617, 677, 739, 805, 873, 943, 1016, 1092, 1171, 1252, 1335, 1422, 1511, 1602, 1696, 1793, 1892, 1993, 2097, 2203, 2312, 2424, 2537, 2653, 2772, 2893, 3016, 3141, 3269, 3399, 3531, 3665, 3802, 3941, 4081, 4224, 4370, 4517, 4666, 4817, 4970, 5125, 5283, 5442, 5603, 5765, 5930, 6096, 6264, 6434, 6606, 6779, 6954, 7131, 7309, 7488, 7670, 7852, 8037, 8222, 8409, 8598, 8788, 8979, 9171, 9364, 9559, 9755, 9952, 10151, 10350, 10550, 10751, 10954, 11157, 11361, 11566, 11772, 11979, 12186, 12394, 12603, 12812, 13022, 13233, 13444, 13656, 13868, 14080, 14293, 14507, 14720, 14934, 15148, 15363, 15577, 15792, 16007, 16222, 16437, 16652, 16866, 17081, 17296, 17510, 17725, 17939, 18153, 18366, 18579, 18792, 19004, 19216, 19428, 19639, 19849, 20059, 20268, 20476, 20684, 20891, 21097, 21302, 21507, 21711, 21913, 22115, 22316, 22516, 22715, 22912, 23109, 23304, 23498, 23691, 23883, 24074, 24263, 24450, 24637, 24822, 25005, 25187, 25368, 25547, 25724, 25900, 26074, 26246, 26417, 26586, 26753, 26919, 27082, 27244, 27404, 27562, 27718, 27873, 28025, 28175, 28323, 28469, 28613, 28755, 28895, 29033, 29168, 29301, 29433, 29561, 29688, 29812, 29934, 30054, 30171, 30286, 30398, 30508, 30616, 30721, 30824, 30924, 31022, 31117, 31210, 31300, 31388, 31473, 31555, 31635, 31712, 31787, 31858, 31928, 31994, 32058, 32119, 32178, 32233, 32286, 32337, 32384, 32429, 32471, 32510, 32547, 32580, 32611, 32639, 32665, 32687, 32707, 32724, 32738, 32749, 32758, 32763, 32766, 32766, 32763, 32758, 32749, 32738, 32724, 32707, 32687, 32665, 32639, 32611, 32580, 32547, 32510, 32471, 32429, 32384, 32337, 32286, 32233, 32178, 32119, 32058, 31994, 31928, 31858, 31787, 31712, 31635, 31555, 31473, 31388, 31300, 31210, 31117, 31022, 30924, 30824, 30721, 30616, 30508, 30398, 30286, 30171, 30054, 29934, 29812, 29688, 29561, 29433, 29301, 29168, 29033, 28895, 28755, 28613, 28469, 28323, 28175, 28025, 27873, 27718, 27562, 27404, 27244, 27082, 26919, 26753, 26586, 26417, 26246, 26074, 25900, 25724, 25547, 25368, 25187, 25005, 24822, 24637, 24450, 24263, 24074, 23883, 23691, 23498, 23304, 23109, 22912, 22715, 22516, 22316, 22115, 21913, 21711, 21507, 21302, 21097, 20891, 20684, 20476, 20268, 20059, 19849, 19639, 19428, 19216, 19004, 18792, 18579, 18366, 18153, 17939, 17725, 17510, 17296, 17081, 16866, 16652, 16437, 16222, 16007, 15792, 15577, 15363, 15148, 14934, 14720, 14507, 14293, 14080, 13868, 13656, 13444, 13233, 13022, 12812, 12603, 12394, 12186, 11979, 11772, 11566, 11361, 11157, 10954, 10751, 10550, 10350, 10151, 9952, 9755, 9559, 9364, 9171, 8979, 8788, 8598, 8409, 8222, 8037, 7852, 7670, 7488, 7309, 7131, 6954, 6779, 6606, 6434, 6264, 6096, 5930, 5765, 5603, 5442, 5283, 5125, 4970, 4817, 4666, 4517, 4370, 4224, 4081, 3941, 3802, 3665, 3531, 3399, 3269, 3141, 3016, 2893, 2772, 2653, 2537, 2424, 2312, 2203, 2097, 1993, 1892, 1793, 1696, 1602, 1511, 1422, 1335, 1252, 1171, 1092, 1016, 943, 873, 805, 739, 677, 617, 560, 506, 454, 405, 359, 316, 275, 237, 202, 170, 140, 114, 90, 69, 50, 35, 22, 12, 5, 1, 0}; diff --git a/tensorflow/lite/experimental/micro/examples/micro_speech/CMSIS/hanning.cc b/tensorflow/lite/experimental/micro/examples/micro_speech/CMSIS/hanning.cc new file mode 100644 index 00000000000..32aa5b2b7ee --- /dev/null +++ b/tensorflow/lite/experimental/micro/examples/micro_speech/CMSIS/hanning.cc @@ -0,0 +1,67 @@ +/* SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the License); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an AS IS BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "tensorflow/lite/experimental/micro/examples/micro_speech/CMSIS/hanning.h" + +const int g_hanning_size = 480; +const int16_t g_hanning[480] = { + 0, 1, 5, 12, 22, 35, 50, 69, 90, 114, + 140, 170, 202, 237, 275, 316, 359, 405, 454, 506, + 560, 617, 677, 740, 805, 873, 943, 1016, 1092, 1171, + 1252, 1336, 1422, 1511, 1602, 1696, 1793, 1892, 1993, 2097, + 2204, 2312, 2424, 2537, 2653, 2772, 2893, 3016, 3141, 3269, + 3399, 3531, 3665, 3802, 3941, 4082, 4225, 4370, 4517, 4666, + 4817, 4971, 5126, 5283, 5442, 5603, 5765, 5930, 6096, 6265, + 6435, 6606, 6779, 6954, 7131, 7309, 7489, 7670, 7853, 8037, + 8223, 8410, 8598, 8788, 8979, 9171, 9365, 9560, 9756, 9953, + 10151, 10350, 10551, 10752, 10954, 11157, 11362, 11567, 11772, 11979, + 12186, 12395, 12603, 12813, 13023, 13233, 13445, 13656, 13868, 14081, + 14294, 14507, 14721, 14935, 15149, 15363, 15578, 15793, 16008, 16222, + 16437, 16652, 16867, 17082, 17297, 17511, 17725, 17939, 18153, 18367, + 18580, 18793, 19005, 19217, 19428, 19639, 19850, 20059, 20269, 20477, + 20685, 20892, 21098, 21303, 21508, 21712, 21914, 22116, 22317, 22517, + 22716, 22913, 23110, 23305, 23499, 23692, 23884, 24075, 24264, 24451, + 24638, 24823, 25006, 25188, 25369, 25548, 25725, 25901, 26075, 26247, + 26418, 26587, 26754, 26920, 27083, 27245, 27405, 27563, 27719, 27874, + 28026, 28176, 28324, 28470, 28614, 28756, 28896, 29034, 29169, 29303, + 29434, 29563, 29689, 29813, 29935, 30055, 30172, 30287, 30400, 30510, + 30617, 30723, 30825, 30926, 31023, 31119, 31211, 31301, 31389, 31474, + 31556, 31636, 31713, 31788, 31860, 31929, 31996, 32059, 32121, 32179, + 32235, 32288, 32338, 32386, 32430, 32472, 32512, 32548, 32582, 32613, + 32641, 32666, 32689, 32708, 32725, 32739, 32751, 32759, 32765, 32767, + 32767, 32765, 32759, 32751, 32739, 32725, 32708, 32689, 32666, 32641, + 32613, 32582, 32548, 32512, 32472, 32430, 32386, 32338, 32288, 32235, + 32179, 32121, 32059, 31996, 31929, 31860, 31788, 31713, 31636, 31556, + 31474, 31389, 31301, 31211, 31119, 31023, 30926, 30825, 30723, 30617, + 30510, 30400, 30287, 30172, 30055, 29935, 29813, 29689, 29563, 29434, + 29303, 29169, 29034, 28896, 28756, 28614, 28470, 28324, 28176, 28026, + 27874, 27719, 27563, 27405, 27245, 27083, 26920, 26754, 26587, 26418, + 26247, 26075, 25901, 25725, 25548, 25369, 25188, 25006, 24823, 24638, + 24451, 24264, 24075, 23884, 23692, 23499, 23305, 23110, 22913, 22716, + 22517, 22317, 22116, 21914, 21712, 21508, 21303, 21098, 20892, 20685, + 20477, 20269, 20059, 19850, 19639, 19428, 19217, 19005, 18793, 18580, + 18367, 18153, 17939, 17725, 17511, 17297, 17082, 16867, 16652, 16437, + 16222, 16008, 15793, 15578, 15363, 15149, 14935, 14721, 14507, 14294, + 14081, 13868, 13656, 13445, 13233, 13023, 12813, 12603, 12395, 12186, + 11979, 11772, 11567, 11362, 11157, 10954, 10752, 10551, 10350, 10151, + 9953, 9756, 9560, 9365, 9171, 8979, 8788, 8598, 8410, 8223, + 8037, 7853, 7670, 7489, 7309, 7131, 6954, 6779, 6606, 6435, + 6265, 6096, 5930, 5765, 5603, 5442, 5283, 5126, 4971, 4817, + 4666, 4517, 4370, 4225, 4082, 3941, 3802, 3665, 3531, 3399, + 3269, 3141, 3016, 2893, 2772, 2653, 2537, 2424, 2312, 2204, + 2097, 1993, 1892, 1793, 1696, 1602, 1511, 1422, 1336, 1252, + 1171, 1092, 1016, 943, 873, 805, 740, 677, 617, 560, + 506, 454, 405, 359, 316, 275, 237, 202, 170, 140, + 114, 90, 69, 50, 35, 22, 12, 5, 1, 0}; diff --git a/tensorflow/lite/experimental/micro/examples/micro_speech/CMSIS/hanning.h b/tensorflow/lite/experimental/micro/examples/micro_speech/CMSIS/hanning.h new file mode 100644 index 00000000000..0982f33c484 --- /dev/null +++ b/tensorflow/lite/experimental/micro/examples/micro_speech/CMSIS/hanning.h @@ -0,0 +1,24 @@ +/* SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the License); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an AS IS BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef TENSORFLOW_LITE_EXPERIMENTAL_MICRO_EXAMPLES_MICRO_SPEECH_HANNING_H_ +#define TENSORFLOW_LITE_EXPERIMENTAL_MICRO_EXAMPLES_MICRO_SPEECH_HANNING_H_ + +#include + +extern const int g_hanning_size; +extern const int16_t g_hanning[]; + +#endif diff --git a/tensorflow/lite/experimental/micro/examples/micro_speech/CMSIS/no_power_spectrum_data.h b/tensorflow/lite/experimental/micro/examples/micro_speech/CMSIS/no_power_spectrum_data.h deleted file mode 100644 index c19566bfb64..00000000000 --- a/tensorflow/lite/experimental/micro/examples/micro_speech/CMSIS/no_power_spectrum_data.h +++ /dev/null @@ -1,29 +0,0 @@ -/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -// This data was extracted from the larger feature data held in -// no_features_data.cc and consists of the 29th spectrogram slice of 43 values. -// This is the expected result of running the sample data in -// no_30ms_sample_data.cc through through the preprocessing pipeline. - -#ifndef TENSORFLOW_CONTRIB_LITE_EXPERIMENTAL_MICRO_EXAMPLES_MICRO_SPEECH_NO_POWER_SPECTRUM_DATA_H_ -#define TENSORFLOW_CONTRIB_LITE_EXPERIMENTAL_MICRO_EXAMPLES_MICRO_SPEECH_NO_POWER_SPECTRUM_DATA_H_ - -#include - -constexpr int g_no_power_spectrum_data_size = 43; -extern const uint8_t g_no_power_spectrum_data[]; - -#endif // TENSORFLOW_CONTRIB_LITE_EXPERIMENTAL_MICRO_EXAMPLES_MICRO_SPEECH_NO_POWER_SPECTRUM_DATA_H_ diff --git a/tensorflow/lite/experimental/micro/examples/micro_speech/CMSIS/preprocessor.cc b/tensorflow/lite/experimental/micro/examples/micro_speech/CMSIS/preprocessor.cc index 24119cbdda4..6bc3c4cb775 100644 --- a/tensorflow/lite/experimental/micro/examples/micro_speech/CMSIS/preprocessor.cc +++ b/tensorflow/lite/experimental/micro/examples/micro_speech/CMSIS/preprocessor.cc @@ -18,11 +18,17 @@ extern "C" { #define IFFT_FLAG_R 0 #define BIT_REVERSE_FLAG 1 #define FFT_SIZE 512 + #define FFT_SIZE_DIV2 256 #include - #include "tensorflow/lite/experimental/micro/examples/micro_speech/CMSIS/hann.h" + #include "tensorflow/lite/experimental/micro/examples/micro_speech/CMSIS/hanning.h" + #include "tensorflow/lite/experimental/micro/examples/micro_speech/CMSIS/arm_cmplx_mag_squared_q10p6.h" } - #include "tensorflow/lite/experimental/micro/examples/micro_speech/CMSIS/preprocessor.h" +#include "tensorflow/lite/experimental/micro/examples/micro_speech/preprocessor.h" + +void quantize(q15_t* bufA, q15_t* bufB, uint8_t* output); + + q15_t bufA[FFT_SIZE]; q15_t bufB[FFT_SIZE]; @@ -52,25 +58,49 @@ TfLiteStatus Preprocess(tflite::ErrorReporter* error_reporter, return kTfLiteError; } - arm_mult_q15((q15_t *) input, hann, bufB, 512); + // 30ms at 16 kHz = 480 samples + // We want to pad the rest of the 512-sample buffer with zeros + arm_mult_q15((q15_t *) input, g_hanning, bufB, 480); + int i; + for(i=480; i<512; i++) { + bufB[i] = 0; + } // Should move init code outside of Preprocess() function arm_math_status = arm_rfft_init_q15(&S_arm_fft, FFT_SIZE, IFFT_FLAG_R, BIT_REVERSE_FLAG); arm_rfft_q15(&S_arm_fft, bufB, bufA); - arm_shift_q15(bufA, 5, bufB, FFT_SIZE); - arm_cmplx_mag_squared_q15(bufB, bufA, 256); - arm_shift_q15(bufA, 1, bufB, 256); + // The rfft function packs data as follows: + // {real[0], real[N/2], real[1], imag[1], ..., real[N/2-1], imag[N/2-1]} + // Below we pack as follows: + // {real[0], 0, real[1], imag[1], ..., real[N/2-1], imag[N/2-1, real[N/2], 0} + bufA[FFT_SIZE_DIV2] = bufA[1]; + bufA[FFT_SIZE_DIV2 + 1] = 0; + bufA[1] = 0; + arm_cmplx_mag_squared_q10p6(bufA, bufB, FFT_SIZE_DIV2 + 1); + quantize(bufA, bufB, output); + + return kTfLiteOk; +} + +void quantize(q15_t* bufA, q15_t* bufB, uint8_t* output) { int i; for (i=0; i<42; i++) { arm_mean_q15(bufB+6*i, 6, bufA+i); } - arm_mean_q15(bufB+252, 4, bufA+42); + arm_mean_q15(bufB+252, 5, bufA+42); for (i=0; i<43; i++) { output[i] = (uint8_t) (bufA[i] >> 5); } +} +TfLiteStatus Preprocess_1sec(tflite::ErrorReporter* error_reporter, + const int16_t* input, uint8_t* output) { + int i; + for(i=0; i<49; i++) { + Preprocess(error_reporter, input+i*320, 480, 43, output+i*43); + } return kTfLiteOk; } diff --git a/tensorflow/lite/experimental/micro/examples/micro_speech/CMSIS/preprocessor_test.cc b/tensorflow/lite/experimental/micro/examples/micro_speech/CMSIS/preprocessor_test.cc deleted file mode 100644 index 5986fb49135..00000000000 --- a/tensorflow/lite/experimental/micro/examples/micro_speech/CMSIS/preprocessor_test.cc +++ /dev/null @@ -1,63 +0,0 @@ -/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -#include "tensorflow/lite/experimental/micro/examples/micro_speech/CMSIS/preprocessor.h" -#include "tensorflow/lite/c/c_api_internal.h" -#include "tensorflow/lite/experimental/micro/examples/micro_speech/no_30ms_sample_data.h" -#include "tensorflow/lite/experimental/micro/examples/micro_speech/CMSIS/no_power_spectrum_data.h" -#include "tensorflow/lite/experimental/micro/examples/micro_speech/yes_30ms_sample_data.h" -#include "tensorflow/lite/experimental/micro/examples/micro_speech/CMSIS/yes_power_spectrum_data.h" -#include "tensorflow/lite/experimental/micro/micro_error_reporter.h" -#include "tensorflow/lite/experimental/micro/testing/micro_test.h" - -TF_LITE_MICRO_TESTS_BEGIN - -TF_LITE_MICRO_TEST(TestPreprocessor) { - tflite::MicroErrorReporter micro_error_reporter; - tflite::ErrorReporter* error_reporter = µ_error_reporter; - - uint8_t yes_calculated_data[g_yes_power_spectrum_data_size]; - TfLiteStatus yes_status = Preprocess( - error_reporter, g_yes_30ms_sample_data, g_yes_30ms_sample_data_size, - g_yes_power_spectrum_data_size, yes_calculated_data); - TF_LITE_MICRO_EXPECT_EQ(kTfLiteOk, yes_status); - - for (int i = 0; i < g_yes_power_spectrum_data_size; ++i) { - TF_LITE_MICRO_EXPECT_EQ(g_yes_power_spectrum_data[i], - yes_calculated_data[i]); - if (g_yes_power_spectrum_data[i] != yes_calculated_data[i]) { - error_reporter->Report("Expected value %d but found %d", - g_yes_power_spectrum_data[i], - yes_calculated_data[i]); - } - } - - uint8_t no_calculated_data[g_yes_power_spectrum_data_size]; - TfLiteStatus no_status = Preprocess( - error_reporter, g_no_30ms_sample_data, g_no_30ms_sample_data_size, - g_no_power_spectrum_data_size, no_calculated_data); - TF_LITE_MICRO_EXPECT_EQ(kTfLiteOk, no_status); - - for (int i = 0; i < g_no_power_spectrum_data_size; ++i) { - TF_LITE_MICRO_EXPECT_EQ(g_no_power_spectrum_data[i], no_calculated_data[i]); - if (g_no_power_spectrum_data[i] != no_calculated_data[i]) { - error_reporter->Report("Expected value %d but found %d", - g_no_power_spectrum_data[i], - no_calculated_data[i]); - } - } -} - -TF_LITE_MICRO_TESTS_END diff --git a/tensorflow/lite/experimental/micro/examples/micro_speech/CMSIS/sin_1k.cc b/tensorflow/lite/experimental/micro/examples/micro_speech/CMSIS/sin_1k.cc new file mode 100644 index 00000000000..be66e5f5480 --- /dev/null +++ b/tensorflow/lite/experimental/micro/examples/micro_speech/CMSIS/sin_1k.cc @@ -0,0 +1,67 @@ +/* SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the License); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an AS IS BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "tensorflow/lite/experimental/micro/examples/micro_speech/CMSIS/sin_1k.h" + +const int g_sin_1k_size = 480; +const int16_t g_sin_1k[480] = { + 0, 1253, 2317, 3027, 3276, 3027, 2317, 1253, 0, -1253, + -2317, -3027, -3276, -3027, -2317, -1253, 0, 1253, 2317, 3027, + 3276, 3027, 2317, 1253, 0, -1253, -2317, -3027, -3276, -3027, + -2317, -1253, 0, 1253, 2317, 3027, 3276, 3027, 2317, 1253, + 0, -1253, -2317, -3027, -3276, -3027, -2317, -1253, 0, 1253, + 2317, 3027, 3276, 3027, 2317, 1253, 0, -1253, -2317, -3027, + -3276, -3027, -2317, -1253, 0, 1253, 2317, 3027, 3276, 3027, + 2317, 1253, 0, -1253, -2317, -3027, -3276, -3027, -2317, -1253, + 0, 1253, 2317, 3027, 3276, 3027, 2317, 1253, 0, -1253, + -2317, -3027, -3276, -3027, -2317, -1253, 0, 1253, 2317, 3027, + 3276, 3027, 2317, 1253, 0, -1253, -2317, -3027, -3276, -3027, + -2317, -1253, 0, 1253, 2317, 3027, 3276, 3027, 2317, 1253, + 0, -1253, -2317, -3027, -3276, -3027, -2317, -1253, 0, 1253, + 2317, 3027, 3276, 3027, 2317, 1253, 0, -1253, -2317, -3027, + -3276, -3027, -2317, -1253, 0, 1253, 2317, 3027, 3276, 3027, + 2317, 1253, 0, -1253, -2317, -3027, -3276, -3027, -2317, -1253, + 0, 1253, 2317, 3027, 3276, 3027, 2317, 1253, 0, -1253, + -2317, -3027, -3276, -3027, -2317, -1253, 0, 1253, 2317, 3027, + 3276, 3027, 2317, 1253, 0, -1253, -2317, -3027, -3276, -3027, + -2317, -1253, 0, 1253, 2317, 3027, 3276, 3027, 2317, 1253, + 0, -1253, -2317, -3027, -3276, -3027, -2317, -1253, 0, 1253, + 2317, 3027, 3276, 3027, 2317, 1253, 0, -1253, -2317, -3027, + -3276, -3027, -2317, -1253, 0, 1253, 2317, 3027, 3276, 3027, + 2317, 1253, 0, -1253, -2317, -3027, -3276, -3027, -2317, -1253, + 0, 1253, 2317, 3027, 3276, 3027, 2317, 1253, 0, -1253, + -2317, -3027, -3276, -3027, -2317, -1253, 0, 1253, 2317, 3027, + 3276, 3027, 2317, 1253, 0, -1253, -2317, -3027, -3276, -3027, + -2317, -1253, 0, 1253, 2317, 3027, 3276, 3027, 2317, 1253, + 0, -1253, -2317, -3027, -3276, -3027, -2317, -1253, 0, 1253, + 2317, 3027, 3276, 3027, 2317, 1253, 0, -1253, -2317, -3027, + -3276, -3027, -2317, -1253, 0, 1253, 2317, 3027, 3276, 3027, + 2317, 1253, 0, -1253, -2317, -3027, -3276, -3027, -2317, -1253, + 0, 1253, 2317, 3027, 3276, 3027, 2317, 1253, 0, -1253, + -2317, -3027, -3276, -3027, -2317, -1253, 0, 1253, 2317, 3027, + 3276, 3027, 2317, 1253, 0, -1253, -2317, -3027, -3276, -3027, + -2317, -1253, 0, 1253, 2317, 3027, 3276, 3027, 2317, 1253, + 0, -1253, -2317, -3027, -3276, -3027, -2317, -1253, 0, 1253, + 2317, 3027, 3276, 3027, 2317, 1253, 0, -1253, -2317, -3027, + -3276, -3027, -2317, -1253, 0, 1253, 2317, 3027, 3276, 3027, + 2317, 1253, 0, -1253, -2317, -3027, -3276, -3027, -2317, -1253, + 0, 1253, 2317, 3027, 3276, 3027, 2317, 1253, 0, -1253, + -2317, -3027, -3276, -3027, -2317, -1253, 0, 1253, 2317, 3027, + 3276, 3027, 2317, 1253, 0, -1253, -2317, -3027, -3276, -3027, + -2317, -1253, 0, 1253, 2317, 3027, 3276, 3027, 2317, 1253, + 0, -1253, -2317, -3027, -3276, -3027, -2317, -1253, 0, 1253, + 2317, 3027, 3276, 3027, 2317, 1253, 0, -1253, -2317, -3027, + -3276, -3027, -2317, -1253, 0, 1253, 2317, 3027, 3276, 3027, + 2317, 1253, 0, -1253, -2317, -3027, -3276, -3027, -2317, -1253}; diff --git a/tensorflow/lite/experimental/micro/examples/micro_speech/CMSIS/sin_1k.h b/tensorflow/lite/experimental/micro/examples/micro_speech/CMSIS/sin_1k.h new file mode 100644 index 00000000000..645e262aa11 --- /dev/null +++ b/tensorflow/lite/experimental/micro/examples/micro_speech/CMSIS/sin_1k.h @@ -0,0 +1,24 @@ +/* SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the License); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an AS IS BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef TENSORFLOW_LITE_EXPERIMENTAL_MICRO_EXAMPLES_MICRO_SPEECH_SIN_1K_H_ +#define TENSORFLOW_LITE_EXPERIMENTAL_MICRO_EXAMPLES_MICRO_SPEECH_SIN_1K_H_ + +#include + +extern const int g_sin_1k_size; +extern const int16_t g_sin_1k[]; + +#endif diff --git a/tensorflow/lite/experimental/micro/examples/micro_speech/CMSIS/yes_power_spectrum_data.h b/tensorflow/lite/experimental/micro/examples/micro_speech/CMSIS/yes_power_spectrum_data.h deleted file mode 100644 index b02853f2ea7..00000000000 --- a/tensorflow/lite/experimental/micro/examples/micro_speech/CMSIS/yes_power_spectrum_data.h +++ /dev/null @@ -1,29 +0,0 @@ -/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -// This data was extracted from the larger feature data held in -// no_features_data.cc and consists of the 26th spectrogram slice of 43 values. -// This is the expected result of running the sample data in -// yes_30ms_sample_data.cc through through the preprocessing pipeline. - -#ifndef TENSORFLOW_CONTRIB_LITE_EXPERIMENTAL_MICRO_EXAMPLES_MICRO_SPEECH_YES_POWER_SPECTRUM_DATA_H_ -#define TENSORFLOW_CONTRIB_LITE_EXPERIMENTAL_MICRO_EXAMPLES_MICRO_SPEECH_YES_POWER_SPECTRUM_DATA_H_ - -#include - -constexpr int g_yes_power_spectrum_data_size = 43; -extern const uint8_t g_yes_power_spectrum_data[]; - -#endif // TENSORFLOW_CONTRIB_LITE_EXPERIMENTAL_MICRO_EXAMPLES_MICRO_SPEECH_YES_POWER_SPECTRUM_DATA_H_ diff --git a/tensorflow/lite/experimental/micro/examples/micro_speech/apollo3/.gitignore b/tensorflow/lite/experimental/micro/examples/micro_speech/apollo3/.gitignore new file mode 100644 index 00000000000..cb8d4d02c41 --- /dev/null +++ b/tensorflow/lite/experimental/micro/examples/micro_speech/apollo3/.gitignore @@ -0,0 +1,4 @@ +captured_data.txt +captured_data.wav +cmsis_*.txt +micro_*.txt diff --git a/tensorflow/lite/experimental/micro/examples/micro_speech/apollo3/README.md b/tensorflow/lite/experimental/micro/examples/micro_speech/apollo3/README.md new file mode 100644 index 00000000000..967b8335011 --- /dev/null +++ b/tensorflow/lite/experimental/micro/examples/micro_speech/apollo3/README.md @@ -0,0 +1,72 @@ +# TODO + +* preprocessor_cmsis_test_bin + +# Description of Apollo3 Makefile targets + +* **pushbutton_cmsis_speech_test_bin**: + * When users press BTN2 on the Apollo3 EVK, 1 second of audio is captured. + * Then the audio is sent to the CMSIS version of the preprocessor and into the neural net + * To print out the neural net's inference scores, run GDB and source pushbutton\_cmsis\_scores.cmd + * To save the captured audio to a text file (captured\_data.txt), run GDB and source pushbutton\_cmsis\_voice.cmd + * Setup python + * sudo apt install python-pip + * sudo apt install python-tk + * pip install numpy + * pip install matplotlib + * pip install pysoundfile + * python captured_data_to_wav.py + * captured\_data.txt can be turned into a \*.wav file using captured\_data\_to\_wav.py by executing "python captured\_data\_to\_wav.py" +* **preprocessor_1k_cmsis_test_bin**: + * Sends a 1 kHz sine wave to the CMSIS fixed\_point version of the preprocessor + * **This test should be compiled with the -O0 option.** Otherwise, the breakpoints will not be reached + * In tensorflow/lite/experimental/micro/tools/make/targets/apollo3evb_makefile.inc change "-O3" to "-O0" on line 47 + * **DO NOT FORGET TO REVERT CHANGE AFTER EXPERIMENT** + * In future, enhance scripts to handle automatically, NOT manually! + * Clean project by running "make -f tensorflow/lite/experimental/micro/tools/make/Makefile clean" + * Compile BIN by running "make -f tensorflow/lite/experimental/micro/tools/make/Makefile TARGET=apollo3evb preprocessor_1k_cmsis_test_bin" + * Run with the preprocessor\_1k\_cmsis\_test.cmd GDB command file + * Produces four text files corresponding to outputs from the CMSIS fixed\_point version of this algorithm: + * cmsis_windowed_input.txt: the sinusoid after multiplying elementwise with a Hann window + * cmsis_dft.txt: the DFT of the windowed sinusoid + * cmsis_power.txt: the magnitude squared of the DFT + * cmsis_power_avg.txt: the 6-bin average of the magnitude squared of the DFT + * Run both verisons of the 1KHz pre-processor test and then compare. + * These files can be plotted with "python compare\_1k.py" + * Also prints out the number of cycles the code took to execute (using the DWT->CYCCNT register) +* **preprocessor_1k_micro_test_bin** + * Sends a 1 kHz sine wave to the Micro-Lite fixed\_point version of the preprocessor + * **This test should be compiled with the -O0 option.** Otherwise, the breakpoints will not be reached + * Run with the preprocessor\_1k\_micro\_test.cmd GDB command file + * Produces four text files corresponding to outputs from the Micro-Lite version of this algorithm: + * micro_windowed_input.txt: the sinusoid after multiplying elementwise with a Hann window + * micro_dft.txt: the DFT of the windowed sinusoid + * micro_power.txt: the magnitude squared of the DFT + * micro_power_avg.txt: the 6-bin average of the magnitude squared of the DFT + * Run both verisons of the 1KHz pre-processor test and then compare. + * These files can be plotted with "python compare\_1k.py" + * Also prints out the number of cycles the code took to execute (using the DWT->CYCCNT register) + +# Description of files + +* **.gitignore**: Git should ignore \*.txt and \*.wav files that result from experiments run in this directory +* **apollo3.h**: Apollo 3 version of the [CMSIS Device Header File (device.h)](https://www.keil.com/pack/doc/CMSIS/Core/html/device_h_pg.html). Available in the [Ambiq Keil Pack](http://s3.ambiqmicro.com/pack/AmbiqMicro.Apollo_DFP.1.1.0.pack). +* **captured\_data\_to\_wav.py**: Python script that parses a text file containing data dumped from GDB (specifically the verilog format) and creates a \*.wav file using [PySoundFile](https://pysoundfile.readthedocs.io/en/0.9.0/). +* **compare\_1k.py**: This script compares the intermediate variables and final outputs of the micro-lite fixed-point preprocessor function and the CMSIS version of this function. The stimulus provided to each preprocessor is the same: a 1 kHz sinusoid. +* **get\_yesno\_data.cmd**: A GDB command file that runs preprocessor_test (where TARGET=apollo3evb) and dumps the calculated data for the "yes" and "no" input wavfeorms to text files +* **\_main.c**: Point of entry for the micro_speech test +* **preprocessor_1k.cc**: A version of preprocessor.cc where a 1 kHz sinusoid is provided as input to the preprocessor +* **preprocessor_1k_cmsis_test.cmd**: GDB command file for the CMSIS preprocessor 1 kHz test +* **preprocessor_1k_micro_test.cmd**: GDB command file for the Micro-Lite preprocessor 1 kHz test +* **preprocessor_test.cmd**: GDB command file for the preprocessor test +* **pushbutton_cmsis_scores.cmd**: GDB command file that runs pushbutton_cmsis_speech_test_bin. It adds a breakpoint immediately after the scores are reported and prints out each score. Then it continues code execution. +* **pushbutton_cmsis_voice.cmd**: GDB command file that runs pushbutton_cmsis_speech_test_bin. Dumps the recorded 1 second of audio to captured_data.txt, which can then be processed by the python file captured_data_to_wav.py. +* **pushbutton_main.c**: Source file containing program point of entry \_main() for the pushbutton\_\* tests. Contains Interrupt Service Routines for PDM data capture and pushbuttons. Calls the main() function of pushbutton_test.cc +* **pushbutton_test.cc**: Source file containing main() function for the pushbutton\_\* tests. main() calls the preprocessor function and the neural net inference function. +* **system_apollo3.c**: Apollo 3 version of the [CMSIS System Configuration File system\_\.c](https://www.keil.com/pack/doc/CMSIS/Core/html/system_c_pg.html). Available in the [Ambiq Keil Pack](http://s3.ambiqmicro.com/pack/AmbiqMicro.Apollo_DFP.1.1.0.pack). +* **system_apollo3.h**: Apollo 3 version of the [CMSIS System Configuration File system\_\.h](https://www.keil.com/pack/doc/CMSIS/Core/html/system_c_pg.html). Available in the [Ambiq Keil Pack](http://s3.ambiqmicro.com/pack/AmbiqMicro.Apollo_DFP.1.1.0.pack). + + +# FFT scaling +See https://github.com/ARM-software/CMSIS_5/issues/220 +>And as @xizhizhang pointed, I think there may be an error on the internal downscaling, or at least on the documentation. It looks like during the fft computation, the downscaling factor reach 2**-9 for a 512 rfft operation, being the output in Q10.22, instead the documented 2**-8 and Q9.23. diff --git a/tensorflow/lite/experimental/micro/tools/make/targets/apollo3evb/_main.c b/tensorflow/lite/experimental/micro/examples/micro_speech/apollo3/_main.c similarity index 100% rename from tensorflow/lite/experimental/micro/tools/make/targets/apollo3evb/_main.c rename to tensorflow/lite/experimental/micro/examples/micro_speech/apollo3/_main.c diff --git a/tensorflow/lite/experimental/micro/examples/micro_speech/apollo3/apollo3.h b/tensorflow/lite/experimental/micro/examples/micro_speech/apollo3/apollo3.h new file mode 100755 index 00000000000..af22270e32e --- /dev/null +++ b/tensorflow/lite/experimental/micro/examples/micro_speech/apollo3/apollo3.h @@ -0,0 +1,23332 @@ +/* + * Copyright (C) 2015-2017, Ambiq Micro + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * 3. Neither the name of the copyright holder nor the names of itscontributors may be used to endorse + * or promote products derived from thissoftware without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * + * @file apollo3.h + * @brief CMSIS HeaderFile + * @version 1.0 + * @date 10. August 2018 + * @note Generated by SVDConv V3.3.18 on Friday, 10.08.2018 16:52:09 + * from File 'apollo3.svd', + * last modified on Friday, 10.08.2018 20:01:31 + */ + + + +/** @addtogroup Ambiq Micro + * @{ + */ + + +/** @addtogroup apollo3 + * @{ + */ + + +#ifndef APOLLO3_H +#define APOLLO3_H + +#ifdef __cplusplus +extern "C" { +#endif + + +/** @addtogroup Configuration_of_CMSIS + * @{ + */ + + + +/* =========================================================================================================================== */ +/* ================ Interrupt Number Definition ================ */ +/* =========================================================================================================================== */ + +typedef enum { +/* ======================================= ARM Cortex-M4 Specific Interrupt Numbers ======================================== */ + Reset_IRQn = -15, /*!< -15 Reset Vector, invoked on Power up and warm reset */ + NonMaskableInt_IRQn = -14, /*!< -14 Non maskable Interrupt, cannot be stopped or preempted */ + HardFault_IRQn = -13, /*!< -13 Hard Fault, all classes of Fault */ + MemoryManagement_IRQn = -12, /*!< -12 Memory Management, MPU mismatch, including Access Violation + and No Match */ + BusFault_IRQn = -11, /*!< -11 Bus Fault, Pre-Fetch-, Memory Access Fault, other address/memory + related Fault */ + UsageFault_IRQn = -10, /*!< -10 Usage Fault, i.e. Undef Instruction, Illegal State Transition */ + SVCall_IRQn = -5, /*!< -5 System Service Call via SVC instruction */ + DebugMonitor_IRQn = -4, /*!< -4 Debug Monitor */ + PendSV_IRQn = -2, /*!< -2 Pendable request for system service */ + SysTick_IRQn = -1, /*!< -1 System Tick Timer */ +/* ========================================== apollo3 Specific Interrupt Numbers =========================================== */ + BROWNOUT_IRQn = 0, /*!< 0 BROWNOUT */ + WDT_IRQn = 1, /*!< 1 WDT */ + RTC_IRQn = 2, /*!< 2 RTC */ + VCOMP_IRQn = 3, /*!< 3 VCOMP */ + IOSLAVE_IRQn = 4, /*!< 4 IOSLAVE */ + IOSLAVEACC_IRQn = 5, /*!< 5 IOSLAVEACC */ + IOMSTR0_IRQn = 6, /*!< 6 IOMSTR0 */ + IOMSTR1_IRQn = 7, /*!< 7 IOMSTR1 */ + IOMSTR2_IRQn = 8, /*!< 8 IOMSTR2 */ + IOMSTR3_IRQn = 9, /*!< 9 IOMSTR3 */ + IOMSTR4_IRQn = 10, /*!< 10 IOMSTR4 */ + IOMSTR5_IRQn = 11, /*!< 11 IOMSTR5 */ + BLE_IRQn = 12, /*!< 12 BLE */ + GPIO_IRQn = 13, /*!< 13 GPIO */ + CTIMER_IRQn = 14, /*!< 14 CTIMER */ + UART0_IRQn = 15, /*!< 15 UART0 */ + UART1_IRQn = 16, /*!< 16 UART1 */ + SCARD_IRQn = 17, /*!< 17 SCARD */ + ADC_IRQn = 18, /*!< 18 ADC */ + PDM_IRQn = 19, /*!< 19 PDM */ + MSPI_IRQn = 20, /*!< 20 MSPI */ + STIMER_IRQn = 22, /*!< 22 STIMER */ + STIMER_CMPR0_IRQn = 23, /*!< 23 STIMER_CMPR0 */ + STIMER_CMPR1_IRQn = 24, /*!< 24 STIMER_CMPR1 */ + STIMER_CMPR2_IRQn = 25, /*!< 25 STIMER_CMPR2 */ + STIMER_CMPR3_IRQn = 26, /*!< 26 STIMER_CMPR3 */ + STIMER_CMPR4_IRQn = 27, /*!< 27 STIMER_CMPR4 */ + STIMER_CMPR5_IRQn = 28, /*!< 28 STIMER_CMPR5 */ + STIMER_CMPR6_IRQn = 29, /*!< 29 STIMER_CMPR6 */ + STIMER_CMPR7_IRQn = 30, /*!< 30 STIMER_CMPR7 */ + CLKGEN_IRQn = 31 /*!< 31 CLKGEN */ +} IRQn_Type; + + + +/* =========================================================================================================================== */ +/* ================ Processor and Core Peripheral Section ================ */ +/* =========================================================================================================================== */ + +/* =========================== Configuration of the ARM Cortex-M4 Processor and Core Peripherals =========================== */ +#define __CM4_REV 0x0100U /*!< CM4 Core Revision */ +#define __NVIC_PRIO_BITS 3 /*!< Number of Bits used for Priority Levels */ +#define __Vendor_SysTickConfig 0 /*!< Set to 1 if different SysTick Config is used */ +#define __MPU_PRESENT 1 /*!< MPU present or not */ +#define __FPU_PRESENT 1 /*!< FPU present or not */ + + +/** @} */ /* End of group Configuration_of_CMSIS */ + +#include "core_cm4.h" /*!< ARM Cortex-M4 processor and core peripherals */ +#include "system_apollo3.h" /*!< apollo3 System */ + +#ifndef __IM /*!< Fallback for older CMSIS versions */ + #define __IM __I +#endif +#ifndef __OM /*!< Fallback for older CMSIS versions */ + #define __OM __O +#endif +#ifndef __IOM /*!< Fallback for older CMSIS versions */ + #define __IOM __IO +#endif + + +/* ======================================== Start of section using anonymous unions ======================================== */ +#if defined (__CC_ARM) + #pragma push + #pragma anon_unions +#elif defined (__ICCARM__) + #pragma language=extended +#elif defined(__ARMCC_VERSION) && (__ARMCC_VERSION >= 6010050) + #pragma clang diagnostic push + #pragma clang diagnostic ignored "-Wc11-extensions" + #pragma clang diagnostic ignored "-Wreserved-id-macro" + #pragma clang diagnostic ignored "-Wgnu-anonymous-struct" + #pragma clang diagnostic ignored "-Wnested-anon-types" +#elif defined (__GNUC__) + /* anonymous unions are enabled by default */ +#elif defined (__TMS470__) + /* anonymous unions are enabled by default */ +#elif defined (__TASKING__) + #pragma warning 586 +#elif defined (__CSMC__) + /* anonymous unions are enabled by default */ +#else + #warning Not supported compiler type +#endif + + +/* =========================================================================================================================== */ +/* ================ Device Specific Peripheral Section ================ */ +/* =========================================================================================================================== */ + + +/** @addtogroup Device_Peripheral_peripherals + * @{ + */ + + + +/* =========================================================================================================================== */ +/* ================ ADC ================ */ +/* =========================================================================================================================== */ + + +/** + * @brief Analog Digital Converter Control (ADC) + */ + +typedef struct { /*!< (@ 0x50010000) ADC Structure */ + + union { + __IOM uint32_t CFG; /*!< (@ 0x00000000) Configuration Register */ + + struct { + __IOM uint32_t ADCEN : 1; /*!< [0..0] This bit enables the ADC module. While the ADC is enabled, + the ADCCFG and SLOT Configuration regsiter settings must + remain stable and unchanged. All configuration register + settings, slot configuration settings and window comparison + settings should be written prior to setting the ADCEN bit + to '1'. */ + __IM uint32_t : 1; + __IOM uint32_t RPTEN : 1; /*!< [2..2] This bit enables Repeating Scan Mode. */ + __IOM uint32_t LPMODE : 1; /*!< [3..3] Select power mode to enter between active scans. */ + __IOM uint32_t CKMODE : 1; /*!< [4..4] Clock mode register */ + __IM uint32_t : 3; + __IOM uint32_t REFSEL : 2; /*!< [9..8] Select the ADC reference voltage. */ + __IM uint32_t : 2; + __IOM uint32_t DFIFORDEN : 1; /*!< [12..12] Destructive FIFO Read Enable. Setting this will enable + FIFO pop upon reading the FIFOPR register. */ + __IM uint32_t : 3; + __IOM uint32_t TRIGSEL : 3; /*!< [18..16] Select the ADC trigger source. */ + __IOM uint32_t TRIGPOL : 1; /*!< [19..19] This bit selects the ADC trigger polarity for external + off chip triggers. */ + __IM uint32_t : 4; + __IOM uint32_t CLKSEL : 2; /*!< [25..24] Select the source and frequency for the ADC clock. + All values not enumerated below are undefined. */ + } CFG_b; + } ; + + union { + __IOM uint32_t STAT; /*!< (@ 0x00000004) ADC Power Status */ + + struct { + __IOM uint32_t PWDSTAT : 1; /*!< [0..0] Indicates the power-status of the ADC. */ + } STAT_b; + } ; + + union { + __IOM uint32_t SWT; /*!< (@ 0x00000008) Software trigger */ + + struct { + __IOM uint32_t SWT : 8; /*!< [7..0] Writing 0x37 to this register generates a software trigger. */ + } SWT_b; + } ; + + union { + __IOM uint32_t SL0CFG; /*!< (@ 0x0000000C) Slot 0 Configuration Register */ + + struct { + __IOM uint32_t SLEN0 : 1; /*!< [0..0] This bit enables slot 0 for ADC conversions. */ + __IOM uint32_t WCEN0 : 1; /*!< [1..1] This bit enables the window compare function for slot + 0. */ + __IM uint32_t : 6; + __IOM uint32_t CHSEL0 : 4; /*!< [11..8] Select one of the 14 channel inputs for this slot. */ + __IM uint32_t : 4; + __IOM uint32_t PRMODE0 : 2; /*!< [17..16] Set the Precision Mode For Slot. */ + __IM uint32_t : 6; + __IOM uint32_t ADSEL0 : 3; /*!< [26..24] Select the number of measurements to average in the + accumulate divide module for this slot. */ + } SL0CFG_b; + } ; + + union { + __IOM uint32_t SL1CFG; /*!< (@ 0x00000010) Slot 1 Configuration Register */ + + struct { + __IOM uint32_t SLEN1 : 1; /*!< [0..0] This bit enables slot 1 for ADC conversions. */ + __IOM uint32_t WCEN1 : 1; /*!< [1..1] This bit enables the window compare function for slot + 1. */ + __IM uint32_t : 6; + __IOM uint32_t CHSEL1 : 4; /*!< [11..8] Select one of the 14 channel inputs for this slot. */ + __IM uint32_t : 4; + __IOM uint32_t PRMODE1 : 2; /*!< [17..16] Set the Precision Mode For Slot. */ + __IM uint32_t : 6; + __IOM uint32_t ADSEL1 : 3; /*!< [26..24] Select the number of measurements to average in the + accumulate divide module for this slot. */ + } SL1CFG_b; + } ; + + union { + __IOM uint32_t SL2CFG; /*!< (@ 0x00000014) Slot 2 Configuration Register */ + + struct { + __IOM uint32_t SLEN2 : 1; /*!< [0..0] This bit enables slot 2 for ADC conversions. */ + __IOM uint32_t WCEN2 : 1; /*!< [1..1] This bit enables the window compare function for slot + 2. */ + __IM uint32_t : 6; + __IOM uint32_t CHSEL2 : 4; /*!< [11..8] Select one of the 14 channel inputs for this slot. */ + __IM uint32_t : 4; + __IOM uint32_t PRMODE2 : 2; /*!< [17..16] Set the Precision Mode For Slot. */ + __IM uint32_t : 6; + __IOM uint32_t ADSEL2 : 3; /*!< [26..24] Select the number of measurements to average in the + accumulate divide module for this slot. */ + } SL2CFG_b; + } ; + + union { + __IOM uint32_t SL3CFG; /*!< (@ 0x00000018) Slot 3 Configuration Register */ + + struct { + __IOM uint32_t SLEN3 : 1; /*!< [0..0] This bit enables slot 3 for ADC conversions. */ + __IOM uint32_t WCEN3 : 1; /*!< [1..1] This bit enables the window compare function for slot + 3. */ + __IM uint32_t : 6; + __IOM uint32_t CHSEL3 : 4; /*!< [11..8] Select one of the 14 channel inputs for this slot. */ + __IM uint32_t : 4; + __IOM uint32_t PRMODE3 : 2; /*!< [17..16] Set the Precision Mode For Slot. */ + __IM uint32_t : 6; + __IOM uint32_t ADSEL3 : 3; /*!< [26..24] Select the number of measurements to average in the + accumulate divide module for this slot. */ + } SL3CFG_b; + } ; + + union { + __IOM uint32_t SL4CFG; /*!< (@ 0x0000001C) Slot 4 Configuration Register */ + + struct { + __IOM uint32_t SLEN4 : 1; /*!< [0..0] This bit enables slot 4 for ADC conversions. */ + __IOM uint32_t WCEN4 : 1; /*!< [1..1] This bit enables the window compare function for slot + 4. */ + __IM uint32_t : 6; + __IOM uint32_t CHSEL4 : 4; /*!< [11..8] Select one of the 14 channel inputs for this slot. */ + __IM uint32_t : 4; + __IOM uint32_t PRMODE4 : 2; /*!< [17..16] Set the Precision Mode For Slot. */ + __IM uint32_t : 6; + __IOM uint32_t ADSEL4 : 3; /*!< [26..24] Select the number of measurements to average in the + accumulate divide module for this slot. */ + } SL4CFG_b; + } ; + + union { + __IOM uint32_t SL5CFG; /*!< (@ 0x00000020) Slot 5 Configuration Register */ + + struct { + __IOM uint32_t SLEN5 : 1; /*!< [0..0] This bit enables slot 5 for ADC conversions. */ + __IOM uint32_t WCEN5 : 1; /*!< [1..1] This bit enables the window compare function for slot + 5. */ + __IM uint32_t : 6; + __IOM uint32_t CHSEL5 : 4; /*!< [11..8] Select one of the 14 channel inputs for this slot. */ + __IM uint32_t : 4; + __IOM uint32_t PRMODE5 : 2; /*!< [17..16] Set the Precision Mode For Slot. */ + __IM uint32_t : 6; + __IOM uint32_t ADSEL5 : 3; /*!< [26..24] Select number of measurements to average in the accumulate + divide module for this slot. */ + } SL5CFG_b; + } ; + + union { + __IOM uint32_t SL6CFG; /*!< (@ 0x00000024) Slot 6 Configuration Register */ + + struct { + __IOM uint32_t SLEN6 : 1; /*!< [0..0] This bit enables slot 6 for ADC conversions. */ + __IOM uint32_t WCEN6 : 1; /*!< [1..1] This bit enables the window compare function for slot + 6. */ + __IM uint32_t : 6; + __IOM uint32_t CHSEL6 : 4; /*!< [11..8] Select one of the 14 channel inputs for this slot. */ + __IM uint32_t : 4; + __IOM uint32_t PRMODE6 : 2; /*!< [17..16] Set the Precision Mode For Slot. */ + __IM uint32_t : 6; + __IOM uint32_t ADSEL6 : 3; /*!< [26..24] Select the number of measurements to average in the + accumulate divide module for this slot. */ + } SL6CFG_b; + } ; + + union { + __IOM uint32_t SL7CFG; /*!< (@ 0x00000028) Slot 7 Configuration Register */ + + struct { + __IOM uint32_t SLEN7 : 1; /*!< [0..0] This bit enables slot 7 for ADC conversions. */ + __IOM uint32_t WCEN7 : 1; /*!< [1..1] This bit enables the window compare function for slot + 7. */ + __IM uint32_t : 6; + __IOM uint32_t CHSEL7 : 4; /*!< [11..8] Select one of the 14 channel inputs for this slot. */ + __IM uint32_t : 4; + __IOM uint32_t PRMODE7 : 2; /*!< [17..16] Set the Precision Mode For Slot. */ + __IM uint32_t : 6; + __IOM uint32_t ADSEL7 : 3; /*!< [26..24] Select the number of measurements to average in the + accumulate divide module for this slot. */ + } SL7CFG_b; + } ; + + union { + __IOM uint32_t WULIM; /*!< (@ 0x0000002C) Window Comparator Upper Limits Register */ + + struct { + __IOM uint32_t ULIM : 20; /*!< [19..0] Sets the upper limit for the window comparator. */ + } WULIM_b; + } ; + + union { + __IOM uint32_t WLLIM; /*!< (@ 0x00000030) Window Comparator Lower Limits Register */ + + struct { + __IOM uint32_t LLIM : 20; /*!< [19..0] Sets the lower limit for the window comparator. */ + } WLLIM_b; + } ; + + union { + __IOM uint32_t SCWLIM; /*!< (@ 0x00000034) Scale Window Comparator Limits */ + + struct { + __IOM uint32_t SCWLIMEN : 1; /*!< [0..0] Scale the window limits compare values per precision + mode. When set to 0x0 (default), the values in the 20-bit + limits registers will compare directly with the FIFO values + regardless of the precision mode the slot is configured + to. When set to 0x1, the compare values will be divided + by the difference in precision bits while performing the + window limit comparisons. */ + } SCWLIM_b; + } ; + + union { + __IOM uint32_t FIFO; /*!< (@ 0x00000038) FIFO Data and Valid Count Register */ + + struct { + __IOM uint32_t DATA : 20; /*!< [19..0] Oldest data in the FIFO. */ + __IOM uint32_t COUNT : 8; /*!< [27..20] Number of valid entries in the ADC FIFO. */ + __IOM uint32_t SLOTNUM : 3; /*!< [30..28] Slot number associated with this FIFO data. */ + __IOM uint32_t RSVD : 1; /*!< [31..31] RESERVED. */ + } FIFO_b; + } ; + + union { + __IOM uint32_t FIFOPR; /*!< (@ 0x0000003C) FIFO Data and Valid Count Register */ + + struct { + __IOM uint32_t DATA : 20; /*!< [19..0] Oldest data in the FIFO. */ + __IOM uint32_t COUNT : 8; /*!< [27..20] Number of valid entries in the ADC FIFO. */ + __IOM uint32_t SLOTNUMPR : 3; /*!< [30..28] Slot number associated with this FIFO data. */ + __IOM uint32_t RSVDPR : 1; /*!< [31..31] RESERVED. */ + } FIFOPR_b; + } ; + __IM uint32_t RESERVED[112]; + + union { + __IOM uint32_t INTEN; /*!< (@ 0x00000200) ADC Interrupt registers: Enable */ + + struct { + __IOM uint32_t CNVCMP : 1; /*!< [0..0] ADC conversion complete interrupt. */ + __IOM uint32_t SCNCMP : 1; /*!< [1..1] ADC scan complete interrupt. */ + __IOM uint32_t FIFOOVR1 : 1; /*!< [2..2] FIFO 75 percent full interrupt. */ + __IOM uint32_t FIFOOVR2 : 1; /*!< [3..3] FIFO 100 percent full interrupt. */ + __IOM uint32_t WCEXC : 1; /*!< [4..4] Window comparator voltage excursion interrupt. */ + __IOM uint32_t WCINC : 1; /*!< [5..5] Window comparator voltage incursion interrupt. */ + __IOM uint32_t DCMP : 1; /*!< [6..6] DMA Transfer Complete */ + __IOM uint32_t DERR : 1; /*!< [7..7] DMA Error Condition */ + } INTEN_b; + } ; + + union { + __IOM uint32_t INTSTAT; /*!< (@ 0x00000204) ADC Interrupt registers: Status */ + + struct { + __IOM uint32_t CNVCMP : 1; /*!< [0..0] ADC conversion complete interrupt. */ + __IOM uint32_t SCNCMP : 1; /*!< [1..1] ADC scan complete interrupt. */ + __IOM uint32_t FIFOOVR1 : 1; /*!< [2..2] FIFO 75 percent full interrupt. */ + __IOM uint32_t FIFOOVR2 : 1; /*!< [3..3] FIFO 100 percent full interrupt. */ + __IOM uint32_t WCEXC : 1; /*!< [4..4] Window comparator voltage excursion interrupt. */ + __IOM uint32_t WCINC : 1; /*!< [5..5] Window comparator voltage incursion interrupt. */ + __IOM uint32_t DCMP : 1; /*!< [6..6] DMA Transfer Complete */ + __IOM uint32_t DERR : 1; /*!< [7..7] DMA Error Condition */ + } INTSTAT_b; + } ; + + union { + __IOM uint32_t INTCLR; /*!< (@ 0x00000208) ADC Interrupt registers: Clear */ + + struct { + __IOM uint32_t CNVCMP : 1; /*!< [0..0] ADC conversion complete interrupt. */ + __IOM uint32_t SCNCMP : 1; /*!< [1..1] ADC scan complete interrupt. */ + __IOM uint32_t FIFOOVR1 : 1; /*!< [2..2] FIFO 75 percent full interrupt. */ + __IOM uint32_t FIFOOVR2 : 1; /*!< [3..3] FIFO 100 percent full interrupt. */ + __IOM uint32_t WCEXC : 1; /*!< [4..4] Window comparator voltage excursion interrupt. */ + __IOM uint32_t WCINC : 1; /*!< [5..5] Window comparator voltage incursion interrupt. */ + __IOM uint32_t DCMP : 1; /*!< [6..6] DMA Transfer Complete */ + __IOM uint32_t DERR : 1; /*!< [7..7] DMA Error Condition */ + } INTCLR_b; + } ; + + union { + __IOM uint32_t INTSET; /*!< (@ 0x0000020C) ADC Interrupt registers: Set */ + + struct { + __IOM uint32_t CNVCMP : 1; /*!< [0..0] ADC conversion complete interrupt. */ + __IOM uint32_t SCNCMP : 1; /*!< [1..1] ADC scan complete interrupt. */ + __IOM uint32_t FIFOOVR1 : 1; /*!< [2..2] FIFO 75 percent full interrupt. */ + __IOM uint32_t FIFOOVR2 : 1; /*!< [3..3] FIFO 100 percent full interrupt. */ + __IOM uint32_t WCEXC : 1; /*!< [4..4] Window comparator voltage excursion interrupt. */ + __IOM uint32_t WCINC : 1; /*!< [5..5] Window comparator voltage incursion interrupt. */ + __IOM uint32_t DCMP : 1; /*!< [6..6] DMA Transfer Complete */ + __IOM uint32_t DERR : 1; /*!< [7..7] DMA Error Condition */ + } INTSET_b; + } ; + __IM uint32_t RESERVED1[12]; + + union { + __IOM uint32_t DMATRIGEN; /*!< (@ 0x00000240) DMA Trigger Enable Register */ + + struct { + __IOM uint32_t DFIFO75 : 1; /*!< [0..0] Trigger DMA upon FIFO 75 percent Full */ + __IOM uint32_t DFIFOFULL : 1; /*!< [1..1] Trigger DMA upon FIFO 100 percent Full */ + } DMATRIGEN_b; + } ; + + union { + __IOM uint32_t DMATRIGSTAT; /*!< (@ 0x00000244) DMA Trigger Status Register */ + + struct { + __IOM uint32_t D75STAT : 1; /*!< [0..0] Triggered DMA from FIFO 75 percent Full */ + __IOM uint32_t DFULLSTAT : 1; /*!< [1..1] Triggered DMA from FIFO 100 percent Full */ + } DMATRIGSTAT_b; + } ; + __IM uint32_t RESERVED2[14]; + + union { + __IOM uint32_t DMACFG; /*!< (@ 0x00000280) DMA Configuration Register */ + + struct { + __IOM uint32_t DMAEN : 1; /*!< [0..0] DMA Enable */ + __IM uint32_t : 1; + __IOM uint32_t DMADIR : 1; /*!< [2..2] Direction */ + __IM uint32_t : 5; + __IOM uint32_t DMAPRI : 1; /*!< [8..8] Sets the Priority of the DMA request */ + __IOM uint32_t DMADYNPRI : 1; /*!< [9..9] Enables dynamic priority based on FIFO fullness. When + FIFO is full, priority is automatically set to HIGH. Otherwise, + DMAPRI is used. */ + __IM uint32_t : 6; + __IOM uint32_t DMAHONSTAT : 1; /*!< [16..16] Halt New ADC conversions until DMA Status DMAERR and + DMACPL Cleared. */ + __IOM uint32_t DMAMSK : 1; /*!< [17..17] Mask the FIFOCNT and SLOTNUM when transferring FIFO + contents to memory */ + __IOM uint32_t DPWROFF : 1; /*!< [18..18] Power Off the ADC System upon DMACPL. */ + } DMACFG_b; + } ; + __IM uint32_t RESERVED3; + + union { + __IOM uint32_t DMATOTCOUNT; /*!< (@ 0x00000288) DMA Total Transfer Count */ + + struct { + __IM uint32_t : 2; + __IOM uint32_t TOTCOUNT : 16; /*!< [17..2] Total Transfer Count */ + } DMATOTCOUNT_b; + } ; + + union { + __IOM uint32_t DMATARGADDR; /*!< (@ 0x0000028C) DMA Target Address Register */ + + struct { + __IOM uint32_t LTARGADDR : 19; /*!< [18..0] DMA Target Address */ + __IOM uint32_t UTARGADDR : 13; /*!< [31..19] SRAM Target */ + } DMATARGADDR_b; + } ; + + union { + __IOM uint32_t DMASTAT; /*!< (@ 0x00000290) DMA Status Register */ + + struct { + __IOM uint32_t DMATIP : 1; /*!< [0..0] DMA Transfer In Progress */ + __IOM uint32_t DMACPL : 1; /*!< [1..1] DMA Transfer Complete */ + __IOM uint32_t DMAERR : 1; /*!< [2..2] DMA Error */ + } DMASTAT_b; + } ; +} ADC_Type; /*!< Size = 660 (0x294) */ + + + +/* =========================================================================================================================== */ +/* ================ APBDMA ================ */ +/* =========================================================================================================================== */ + + +/** + * @brief APB DMA Register Interfaces (APBDMA) + */ + +typedef struct { /*!< (@ 0x40011000) APBDMA Structure */ + + union { + __IOM uint32_t BBVALUE; /*!< (@ 0x00000000) Control Register */ + + struct { + __IOM uint32_t DATAOUT : 8; /*!< [7..0] Data Output Values */ + __IM uint32_t : 8; + __IOM uint32_t PIN : 8; /*!< [23..16] PIO values */ + } BBVALUE_b; + } ; + + union { + __IOM uint32_t BBSETCLEAR; /*!< (@ 0x00000004) Set/Clear Register */ + + struct { + __IOM uint32_t SET : 8; /*!< [7..0] Write 1 to Set PIO value (set hier priority than clear + if both bit set) */ + __IM uint32_t : 8; + __IOM uint32_t CLEAR : 8; /*!< [23..16] Write 1 to Clear PIO value */ + } BBSETCLEAR_b; + } ; + + union { + __IOM uint32_t BBINPUT; /*!< (@ 0x00000008) PIO Input Values */ + + struct { + __IOM uint32_t DATAIN : 8; /*!< [7..0] PIO values */ + } BBINPUT_b; + } ; + __IM uint32_t RESERVED[5]; + + union { + __IOM uint32_t DEBUGDATA; /*!< (@ 0x00000020) PIO Input Values */ + + struct { + __IOM uint32_t DEBUGDATA : 32; /*!< [31..0] Debug Data */ + } DEBUGDATA_b; + } ; + __IM uint32_t RESERVED1[7]; + + union { + __IOM uint32_t DEBUG; /*!< (@ 0x00000040) PIO Input Values */ + + struct { + __IOM uint32_t DEBUGEN : 4; /*!< [3..0] Debug Enable */ + } DEBUG_b; + } ; +} APBDMA_Type; /*!< Size = 68 (0x44) */ + + + +/* =========================================================================================================================== */ +/* ================ BLEIF ================ */ +/* =========================================================================================================================== */ + + +/** + * @brief BLE Interface (BLEIF) + */ + +typedef struct { /*!< (@ 0x5000C000) BLEIF Structure */ + + union { + __IOM uint32_t FIFO; /*!< (@ 0x00000000) FIFO Access Port */ + + struct { + __IOM uint32_t FIFO : 32; /*!< [31..0] FIFO direct access. Only locations 0 - 3F will return + valid information. */ + } FIFO_b; + } ; + __IM uint32_t RESERVED[63]; + + union { + __IOM uint32_t FIFOPTR; /*!< (@ 0x00000100) FIFO size and remaining slots open values */ + + struct { + __IOM uint32_t FIFO0SIZ : 8; /*!< [7..0] The number of valid data bytes currently in the FIFO + 0 (written by MCU, read by interface) */ + __IOM uint32_t FIFO0REM : 8; /*!< [15..8] The number of remaining data bytes slots currently in + FIFO 0 (written by MCU, read by interface) */ + __IOM uint32_t FIFO1SIZ : 8; /*!< [23..16] The number of valid data bytes currently in FIFO 1 + (written by interface, read by MCU) */ + __IOM uint32_t FIFO1REM : 8; /*!< [31..24] The number of remaining data bytes slots currently + in FIFO 1 (written by interface, read by MCU) */ + } FIFOPTR_b; + } ; + + union { + __IOM uint32_t FIFOTHR; /*!< (@ 0x00000104) FIFO Threshold Configuration */ + + struct { + __IOM uint32_t FIFORTHR : 6; /*!< [5..0] FIFO read threshold in bytes. A value of 0 will disable + the read FIFO level from activating the threshold interrupt. + If this field is non-zero, it will trigger a threshold + interrupt when the read fifo contains FIFORTHR valid bytes + of data, as indicated by the FIFO1SIZ field. This is intended + to signal when a data transfer of FIFORTHR bytes can be + done from the IOM module to the host via the read fifo + to support large IOM read operations. */ + __IM uint32_t : 2; + __IOM uint32_t FIFOWTHR : 6; /*!< [13..8] FIFO write threshold in bytes. A value of 0 will disable + the write FIFO level from activating the threshold interrupt. + If this field is non-zero, it will trigger a threshold + interrupt when the write fifo contains FIFOWTHR free bytes, + as indicated by the FIFO0REM field. This is intended to + signal when a transfer of FIFOWTHR bytes can be done from + the host to the IOM write fifo to support large IOM write + operations. */ + } FIFOTHR_b; + } ; + + union { + __IOM uint32_t FIFOPOP; /*!< (@ 0x00000108) FIFO POP register */ + + struct { + __IOM uint32_t FIFODOUT : 32; /*!< [31..0] This register will return the read data indicated by + the current read pointer on reads. If the POPWR control + bit in the FIFOCTRL register is reset (0), the fifo read + pointer will be advanced by one word as a result of the + read.If the POPWR bit is set (1), the fifo read pointer + will only be advanced after a write operation to this register. + The write data is ignored for this register.If less than + a even word multiple is available, and the command is completed, + the module will return the word containing */ + } FIFOPOP_b; + } ; + + union { + __IOM uint32_t FIFOPUSH; /*!< (@ 0x0000010C) FIFO PUSH register */ + + struct { + __IOM uint32_t FIFODIN : 32; /*!< [31..0] This register is used to write the FIFORAM in FIFO mode + and will cause a push event to occur to the next open slot + within the FIFORAM. Writing to this register will cause + the write point to increment by 1 word(4 bytes). */ + } FIFOPUSH_b; + } ; + + union { + __IOM uint32_t FIFOCTRL; /*!< (@ 0x00000110) FIFO Control Register */ + + struct { + __IOM uint32_t POPWR : 1; /*!< [0..0] Selects the mode in which 'pop' events are done for the + fifo read operations. A value of '1' will prevent a pop + event on a read operation, and will require a write to + the FIFOPOP register to create a pop event.A value of '0' + in this register will allow a pop event to occur on the + read of the FIFOPOP register, and may cause inadvertant + fifo pops when used in a debugging mode. */ + __IOM uint32_t FIFORSTN : 1; /*!< [1..1] Active low manual reset of the fifo. Write to 0 to reset + fifo, and then write to 1 to remove the reset. */ + } FIFOCTRL_b; + } ; + + union { + __IOM uint32_t FIFOLOC; /*!< (@ 0x00000114) FIFO Pointers */ + + struct { + __IOM uint32_t FIFOWPTR : 4; /*!< [3..0] Current FIFO write pointer. Value is the index into the + outgoing FIFO (FIFO0), which is used during write operations + to external devices. */ + __IM uint32_t : 4; + __IOM uint32_t FIFORPTR : 4; /*!< [11..8] Current FIFO read pointer. Used to index into the incoming + FIFO (FIFO1), which is used to store read data returned + from external devices during a read operation. */ + } FIFOLOC_b; + } ; + __IM uint32_t RESERVED1[58]; + + union { + __IOM uint32_t CLKCFG; /*!< (@ 0x00000200) I/O Clock Configuration */ + + struct { + __IOM uint32_t IOCLKEN : 1; /*!< [0..0] Enable for the interface clock. Must be enabled prior + to executing any IO operations. */ + __IM uint32_t : 7; + __IOM uint32_t FSEL : 3; /*!< [10..8] Select the input clock frequency. */ + __IOM uint32_t CLK32KEN : 1; /*!< [11..11] Enable for the 32Khz clock to the BLE module */ + __IOM uint32_t DIV3 : 1; /*!< [12..12] Enable of the divide by 3 of the source IOCLK. */ + } CLKCFG_b; + } ; + __IM uint32_t RESERVED2[2]; + + union { + __IOM uint32_t CMD; /*!< (@ 0x0000020C) Command and offset Register */ + + struct { + __IOM uint32_t CMD : 5; /*!< [4..0] Command for submodule. */ + __IOM uint32_t OFFSETCNT : 2; /*!< [6..5] Number of offset bytes to use for the command - 0, 1, + 2, 3 are valid selections. The second (byte 1) and third + byte (byte 2) are read from the OFFSETHI register, and + the low order byte is pulled from this register in the + OFFSETLO field.Offset bytes are transmitted highest byte + first. EG if offsetcnt == 3, OFFSETHI[15:8] will be transmitted + first, then OFFSETHI[7:0] then OFFSETLO.If offsetcnt == + 2, OFFSETHI[7:0] will be transmitted, then OFFSETLO.If + offsetcnt == 1, only OFFSETLO will be transmitted. */ + __IOM uint32_t CONT : 1; /*!< [7..7] Contine to hold the bus after the current transaction + if set to a 1 with a new command issued. */ + __IOM uint32_t TSIZE : 12; /*!< [19..8] Defines the transaction size in bytes. The offset transfer + is not included in this size. */ + __IOM uint32_t CMDSEL : 2; /*!< [21..20] Command Specific selection information */ + __IM uint32_t : 2; + __IOM uint32_t OFFSETLO : 8; /*!< [31..24] This register holds the low order byte of offset to + be used in the transaction. The number of offset bytes + to use is set with bits 1:0 of the command. Offset bytes + are transferred starting from the highest byte first. */ + } CMD_b; + } ; + + union { + __IOM uint32_t CMDRPT; /*!< (@ 0x00000210) Command Repeat Register */ + + struct { + __IOM uint32_t CMDRPT : 5; /*!< [4..0] Count of number of times to repeat the next command. */ + } CMDRPT_b; + } ; + + union { + __IOM uint32_t OFFSETHI; /*!< (@ 0x00000214) High order offset bytes */ + + struct { + __IOM uint32_t OFFSETHI : 16; /*!< [15..0] Holds the high order bytes of the 2 or 3 byte offset + phase of a transaction. */ + } OFFSETHI_b; + } ; + + union { + __IOM uint32_t CMDSTAT; /*!< (@ 0x00000218) Command status */ + + struct { + __IOM uint32_t CCMD : 5; /*!< [4..0] current command that is being executed */ + __IOM uint32_t CMDSTAT : 3; /*!< [7..5] The current status of the command execution. */ + __IOM uint32_t CTSIZE : 12; /*!< [19..8] The current number of bytes still to be transferred + with this command. This field will count down to zero. */ + } CMDSTAT_b; + } ; + __IM uint32_t RESERVED3; + + union { + __IOM uint32_t INTEN; /*!< (@ 0x00000220) IO Master Interrupts: Enable */ + + struct { + __IOM uint32_t CMDCMP : 1; /*!< [0..0] Command Complete interrupt. Asserted when the current + operation has completed. For repeated commands, this will + only be asserted when the final repeated command is completed. */ + __IOM uint32_t THR : 1; /*!< [1..1] FIFO Threshold interrupt. For write operations, asserted + when the number of free bytes in the write FIFO equals + or exceeds the WTHR field.For read operations, asserted + when the number of valid bytes in the read FIFO equals + of exceeds the value set in the RTHR field. */ + __IOM uint32_t FUNDFL : 1; /*!< [2..2] Read FIFO Underflow interrupt. Asserted when a pop operation + is done to a empty read FIFO. */ + __IOM uint32_t FOVFL : 1; /*!< [3..3] Write FIFO Overflow interrupt. This occurs when software + tries to write to a full fifo. The current operation does + not stop. */ + __IOM uint32_t B2MST : 1; /*!< [4..4] B2M State change interrupt. Asserted on any change in + the B2M_STATE signal from the BLE Core. */ + __IOM uint32_t IACC : 1; /*!< [5..5] illegal FIFO access interrupt. Asserted when there is + a overflow or underflow event */ + __IOM uint32_t ICMD : 1; /*!< [6..6] illegal command interrupt. Asserted when a command is + written when an active command is in progress. */ + __IOM uint32_t BLECIRQ : 1; /*!< [7..7] BLE Core IRQ signal. Asserted when the BLE_IRQ signal + from the BLE Core is asserted, indicating the availability + of read data from the BLE Core. */ + __IOM uint32_t BLECSSTAT : 1; /*!< [8..8] BLE Core SPI Status interrupt. Asserted when the SPI_STATUS + signal from the BLE Core is asserted, indicating that SPI + writes can be done to the BLE Core.Transfers to the BLE + Core should only be done when this signal is high. */ + __IOM uint32_t DCMP : 1; /*!< [9..9] DMA Complete. Processing of the DMA operation has completed + and the DMA submodule is returned into the idle state */ + __IOM uint32_t DERR : 1; /*!< [10..10] DMA Error encountered during the processing of the + DMA command. The DMA error could occur when the memory + access specified in the DMA operation is not available + or incorrectly specified. */ + __IOM uint32_t CQPAUSED : 1; /*!< [11..11] Command queue is paused due to an active event enabled + in the PAUSEEN register. The interrupt is posted when the + event is enabled within the PAUSEEN register, the mask + is active in the CQIRQMASK field and the event occurs. */ + __IOM uint32_t CQUPD : 1; /*!< [12..12] Command queue write operation executed a register write + with the register address bit 0 set to 1. The low address + bits in the CQ address fields are unused and bit 0 can + be used to trigger an interrupt to indicate when this register + write is performed by the CQ operation. */ + __IOM uint32_t CQERR : 1; /*!< [13..13] Command queue error during processing. When an error + occurs, the system will stop processing and halt operations + to allow software to take recovery actions */ + __IOM uint32_t B2MSLEEP : 1; /*!< [14..14] The B2M_STATE from the BLE Core transitioned into the + sleep state */ + __IOM uint32_t B2MACTIVE : 1; /*!< [15..15] The B2M_STATE from the BLE Core transitioned into the + active state */ + __IOM uint32_t B2MSHUTDN : 1; /*!< [16..16] The B2M_STATE from the BLE Core transitioned into shutdown + state */ + } INTEN_b; + } ; + + union { + __IOM uint32_t INTSTAT; /*!< (@ 0x00000224) IO Master Interrupts: Status */ + + struct { + __IOM uint32_t CMDCMP : 1; /*!< [0..0] Command Complete interrupt. Asserted when the current + operation has completed. For repeated commands, this will + only be asserted when the final repeated command is completed. */ + __IOM uint32_t THR : 1; /*!< [1..1] FIFO Threshold interrupt. For write operations, asserted + when the number of free bytes in the write FIFO equals + or exceeds the WTHR field.For read operations, asserted + when the number of valid bytes in the read FIFO equals + of exceeds the value set in the RTHR field. */ + __IOM uint32_t FUNDFL : 1; /*!< [2..2] Read FIFO Underflow interrupt. Asserted when a pop operation + is done to a empty read FIFO. */ + __IOM uint32_t FOVFL : 1; /*!< [3..3] Write FIFO Overflow interrupt. This occurs when software + tries to write to a full fifo. The current operation does + not stop. */ + __IOM uint32_t B2MST : 1; /*!< [4..4] B2M State change interrupt. Asserted on any change in + the B2M_STATE signal from the BLE Core. */ + __IOM uint32_t IACC : 1; /*!< [5..5] illegal FIFO access interrupt. Asserted when there is + a overflow or underflow event */ + __IOM uint32_t ICMD : 1; /*!< [6..6] illegal command interrupt. Asserted when a command is + written when an active command is in progress. */ + __IOM uint32_t BLECIRQ : 1; /*!< [7..7] BLE Core IRQ signal. Asserted when the BLE_IRQ signal + from the BLE Core is asserted, indicating the availability + of read data from the BLE Core. */ + __IOM uint32_t BLECSSTAT : 1; /*!< [8..8] BLE Core SPI Status interrupt. Asserted when the SPI_STATUS + signal from the BLE Core is asserted, indicating that SPI + writes can be done to the BLE Core.Transfers to the BLE + Core should only be done when this signal is high. */ + __IOM uint32_t DCMP : 1; /*!< [9..9] DMA Complete. Processing of the DMA operation has completed + and the DMA submodule is returned into the idle state */ + __IOM uint32_t DERR : 1; /*!< [10..10] DMA Error encountered during the processing of the + DMA command. The DMA error could occur when the memory + access specified in the DMA operation is not available + or incorrectly specified. */ + __IOM uint32_t CQPAUSED : 1; /*!< [11..11] Command queue is paused due to an active event enabled + in the PAUSEEN register. The interrupt is posted when the + event is enabled within the PAUSEEN register, the mask + is active in the CQIRQMASK field and the event occurs. */ + __IOM uint32_t CQUPD : 1; /*!< [12..12] Command queue write operation executed a register write + with the register address bit 0 set to 1. The low address + bits in the CQ address fields are unused and bit 0 can + be used to trigger an interrupt to indicate when this register + write is performed by the CQ operation. */ + __IOM uint32_t CQERR : 1; /*!< [13..13] Command queue error during processing. When an error + occurs, the system will stop processing and halt operations + to allow software to take recovery actions */ + __IOM uint32_t B2MSLEEP : 1; /*!< [14..14] The B2M_STATE from the BLE Core transitioned into the + sleep state */ + __IOM uint32_t B2MACTIVE : 1; /*!< [15..15] The B2M_STATE from the BLE Core transitioned into the + active state */ + __IOM uint32_t B2MSHUTDN : 1; /*!< [16..16] The B2M_STATE from the BLE Core transitioned into shutdown + state */ + } INTSTAT_b; + } ; + + union { + __IOM uint32_t INTCLR; /*!< (@ 0x00000228) IO Master Interrupts: Clear */ + + struct { + __IOM uint32_t CMDCMP : 1; /*!< [0..0] Command Complete interrupt. Asserted when the current + operation has completed. For repeated commands, this will + only be asserted when the final repeated command is completed. */ + __IOM uint32_t THR : 1; /*!< [1..1] FIFO Threshold interrupt. For write operations, asserted + when the number of free bytes in the write FIFO equals + or exceeds the WTHR field.For read operations, asserted + when the number of valid bytes in the read FIFO equals + of exceeds the value set in the RTHR field. */ + __IOM uint32_t FUNDFL : 1; /*!< [2..2] Read FIFO Underflow interrupt. Asserted when a pop operation + is done to a empty read FIFO. */ + __IOM uint32_t FOVFL : 1; /*!< [3..3] Write FIFO Overflow interrupt. This occurs when software + tries to write to a full fifo. The current operation does + not stop. */ + __IOM uint32_t B2MST : 1; /*!< [4..4] B2M State change interrupt. Asserted on any change in + the B2M_STATE signal from the BLE Core. */ + __IOM uint32_t IACC : 1; /*!< [5..5] illegal FIFO access interrupt. Asserted when there is + a overflow or underflow event */ + __IOM uint32_t ICMD : 1; /*!< [6..6] illegal command interrupt. Asserted when a command is + written when an active command is in progress. */ + __IOM uint32_t BLECIRQ : 1; /*!< [7..7] BLE Core IRQ signal. Asserted when the BLE_IRQ signal + from the BLE Core is asserted, indicating the availability + of read data from the BLE Core. */ + __IOM uint32_t BLECSSTAT : 1; /*!< [8..8] BLE Core SPI Status interrupt. Asserted when the SPI_STATUS + signal from the BLE Core is asserted, indicating that SPI + writes can be done to the BLE Core.Transfers to the BLE + Core should only be done when this signal is high. */ + __IOM uint32_t DCMP : 1; /*!< [9..9] DMA Complete. Processing of the DMA operation has completed + and the DMA submodule is returned into the idle state */ + __IOM uint32_t DERR : 1; /*!< [10..10] DMA Error encountered during the processing of the + DMA command. The DMA error could occur when the memory + access specified in the DMA operation is not available + or incorrectly specified. */ + __IOM uint32_t CQPAUSED : 1; /*!< [11..11] Command queue is paused due to an active event enabled + in the PAUSEEN register. The interrupt is posted when the + event is enabled within the PAUSEEN register, the mask + is active in the CQIRQMASK field and the event occurs. */ + __IOM uint32_t CQUPD : 1; /*!< [12..12] Command queue write operation executed a register write + with the register address bit 0 set to 1. The low address + bits in the CQ address fields are unused and bit 0 can + be used to trigger an interrupt to indicate when this register + write is performed by the CQ operation. */ + __IOM uint32_t CQERR : 1; /*!< [13..13] Command queue error during processing. When an error + occurs, the system will stop processing and halt operations + to allow software to take recovery actions */ + __IOM uint32_t B2MSLEEP : 1; /*!< [14..14] The B2M_STATE from the BLE Core transitioned into the + sleep state */ + __IOM uint32_t B2MACTIVE : 1; /*!< [15..15] The B2M_STATE from the BLE Core transitioned into the + active state */ + __IOM uint32_t B2MSHUTDN : 1; /*!< [16..16] The B2M_STATE from the BLE Core transitioned into shutdown + state */ + } INTCLR_b; + } ; + + union { + __IOM uint32_t INTSET; /*!< (@ 0x0000022C) IO Master Interrupts: Set */ + + struct { + __IOM uint32_t CMDCMP : 1; /*!< [0..0] Command Complete interrupt. Asserted when the current + operation has completed. For repeated commands, this will + only be asserted when the final repeated command is completed. */ + __IOM uint32_t THR : 1; /*!< [1..1] FIFO Threshold interrupt. For write operations, asserted + when the number of free bytes in the write FIFO equals + or exceeds the WTHR field.For read operations, asserted + when the number of valid bytes in the read FIFO equals + of exceeds the value set in the RTHR field. */ + __IOM uint32_t FUNDFL : 1; /*!< [2..2] Read FIFO Underflow interrupt. Asserted when a pop operation + is done to a empty read FIFO. */ + __IOM uint32_t FOVFL : 1; /*!< [3..3] Write FIFO Overflow interrupt. This occurs when software + tries to write to a full fifo. The current operation does + not stop. */ + __IOM uint32_t B2MST : 1; /*!< [4..4] B2M State change interrupt. Asserted on any change in + the B2M_STATE signal from the BLE Core. */ + __IOM uint32_t IACC : 1; /*!< [5..5] illegal FIFO access interrupt. Asserted when there is + a overflow or underflow event */ + __IOM uint32_t ICMD : 1; /*!< [6..6] illegal command interrupt. Asserted when a command is + written when an active command is in progress. */ + __IOM uint32_t BLECIRQ : 1; /*!< [7..7] BLE Core IRQ signal. Asserted when the BLE_IRQ signal + from the BLE Core is asserted, indicating the availability + of read data from the BLE Core. */ + __IOM uint32_t BLECSSTAT : 1; /*!< [8..8] BLE Core SPI Status interrupt. Asserted when the SPI_STATUS + signal from the BLE Core is asserted, indicating that SPI + writes can be done to the BLE Core.Transfers to the BLE + Core should only be done when this signal is high. */ + __IOM uint32_t DCMP : 1; /*!< [9..9] DMA Complete. Processing of the DMA operation has completed + and the DMA submodule is returned into the idle state */ + __IOM uint32_t DERR : 1; /*!< [10..10] DMA Error encountered during the processing of the + DMA command. The DMA error could occur when the memory + access specified in the DMA operation is not available + or incorrectly specified. */ + __IOM uint32_t CQPAUSED : 1; /*!< [11..11] Command queue is paused due to an active event enabled + in the PAUSEEN register. The interrupt is posted when the + event is enabled within the PAUSEEN register, the mask + is active in the CQIRQMASK field and the event occurs. */ + __IOM uint32_t CQUPD : 1; /*!< [12..12] Command queue write operation executed a register write + with the register address bit 0 set to 1. The low address + bits in the CQ address fields are unused and bit 0 can + be used to trigger an interrupt to indicate when this register + write is performed by the CQ operation. */ + __IOM uint32_t CQERR : 1; /*!< [13..13] Command queue error during processing. When an error + occurs, the system will stop processing and halt operations + to allow software to take recovery actions */ + __IOM uint32_t B2MSLEEP : 1; /*!< [14..14] The B2M_STATE from the BLE Core transitioned into the + sleep state */ + __IOM uint32_t B2MACTIVE : 1; /*!< [15..15] The B2M_STATE from the BLE Core transitioned into the + active state */ + __IOM uint32_t B2MSHUTDN : 1; /*!< [16..16] The B2M_STATE from the BLE Core transitioned into shutdown + state */ + } INTSET_b; + } ; + + union { + __IOM uint32_t DMATRIGEN; /*!< (@ 0x00000230) DMA Trigger Enable Register */ + + struct { + __IOM uint32_t DCMDCMPEN : 1; /*!< [0..0] Trigger DMA upon command complete. Enables the trigger + of the DMA when a command is completed. When this event + is triggered, the number of words transferred will be the + lesser of the remaining TOTCOUNT bytes, or the number of + bytes in the FIFO when the command completed. If this is + disabled, and the number of bytes in the FIFO is equal + or greater than the TOTCOUNT bytes, a transfer of TOTCOUNT + bytes will be done to ensure read data is stored when the + DMA is completed. */ + __IOM uint32_t DTHREN : 1; /*!< [1..1] Trigger DMA upon THR level reached. For M2P DMA operations + (IOM writes), the trigger will assert when the write FIFO + has (WTHR/4) number of words free in the write FIFO, and + will transfer (WTHR/4) number of wordsor, if the number + of words left to transfer is less than the WTHR value, + will transfer the remaining byte count.For P2M DMA operations, + the trigger will assert when the read FIFO has (RTHR/4) + words available in the read FIFO, and will transfer (RTHR/4) + words to SRAM. This trigger will NOT asser */ + } DMATRIGEN_b; + } ; + + union { + __IOM uint32_t DMATRIGSTAT; /*!< (@ 0x00000234) DMA Trigger Status Register */ + + struct { + __IOM uint32_t DCMDCMP : 1; /*!< [0..0] Triggered DMA from Command complete event. Bit is read + only and can be cleared by disabling the DCMDCMP trigger + enable or by disabling DMA. */ + __IOM uint32_t DTHR : 1; /*!< [1..1] Triggered DMA from THR event. Bit is read only and can + be cleared by disabling the DTHR trigger enable or by disabling + DMA. */ + __IOM uint32_t DTOTCMP : 1; /*!< [2..2] DMA triggered when DCMDCMP = 0, and the amount of data + in the FIFO was enough to complete the DMA operation (greater + than or equal to current TOTCOUNT) when the command completed. + This trigger is default active when the DCMDCMP trigger + isdisabled and there is enough data in the FIFO to complete + the DMA operation. */ + } DMATRIGSTAT_b; + } ; + + union { + __IOM uint32_t DMACFG; /*!< (@ 0x00000238) DMA Configuration Register */ + + struct { + __IOM uint32_t DMAEN : 1; /*!< [0..0] DMA Enable. Setting this bit to EN will start the DMA + operation. This should be the last DMA related register + set prior to issuing the command */ + __IOM uint32_t DMADIR : 1; /*!< [1..1] Direction */ + __IM uint32_t : 6; + __IOM uint32_t DMAPRI : 1; /*!< [8..8] Sets the Priority of the DMA request */ + __IOM uint32_t DPWROFF : 1; /*!< [9..9] Power off module after DMA is complete. If this bit is + active, the module will request to power off the supply + it is attached to. If there are other units still requiring + power from the same domain, power down will not be performed. */ + } DMACFG_b; + } ; + + union { + __IOM uint32_t DMATOTCOUNT; /*!< (@ 0x0000023C) DMA Total Transfer Count */ + + struct { + __IOM uint32_t TOTCOUNT : 12; /*!< [11..0] Triggered DMA from Command complete event occured. Bit + is read only and can be cleared by disabling the DTHR trigger + enable or by disabling DMA. */ + } DMATOTCOUNT_b; + } ; + + union { + __IOM uint32_t DMATARGADDR; /*!< (@ 0x00000240) DMA Target Address Register */ + + struct { + __IOM uint32_t TARGADDR : 20; /*!< [19..0] Bits [19:0] of the target byte address for source of + DMA (either read or write). The address can be any byte + alignment, and does not have to be word aligned. In cases + of non-word aligned addresses, the DMA logic will take + care for ensuring only the target bytes are read/written. */ + __IM uint32_t : 8; + __IOM uint32_t TARGADDR28 : 1; /*!< [28..28] Bit 28 of the target byte address for source of DMA + (either read or write). In cases of non-word aligned addresses, + the DMA logic will take care for ensuring only the target + bytes are read/written.Setting to '1' will select the SRAM. + Setting to '0' will select the flash */ + } DMATARGADDR_b; + } ; + + union { + __IOM uint32_t DMASTAT; /*!< (@ 0x00000244) DMA Status Register */ + + struct { + __IOM uint32_t DMATIP : 1; /*!< [0..0] DMA Transfer In Progress indicator. 1 will indicate that + a DMA transfer is active. The DMA transfer may be waiting + on data, transferring data, or waiting for priority.All + of these will be indicated with a 1. A 0 will indicate + that the DMA is fully complete and no further transactions + will be done. This bit is read only. */ + __IOM uint32_t DMACPL : 1; /*!< [1..1] DMA Transfer Complete. This signals the end of the DMA + operation. This bit can be cleared by writing to 0. */ + __IOM uint32_t DMAERR : 1; /*!< [2..2] DMA Error. This active high bit signals that an error + was encountered during the DMA operation. */ + } DMASTAT_b; + } ; + + union { + __IOM uint32_t CQCFG; /*!< (@ 0x00000248) Command Queue Configuration Register */ + + struct { + __IOM uint32_t CQEN : 1; /*!< [0..0] Command queue enable. When set, will enable the processing + of the command queue and fetches of address/data pairs + will proceed from the word address within the CQADDR register. + Can be disabledusing a CQ executed write to this bit as + well. */ + __IOM uint32_t CQPRI : 1; /*!< [1..1] Sets the Priority of the command queue dma request. */ + } CQCFG_b; + } ; + + union { + __IOM uint32_t CQADDR; /*!< (@ 0x0000024C) CQ Target Read Address Register */ + + struct { + __IM uint32_t : 2; + __IOM uint32_t CQADDR : 18; /*!< [19..2] Bits 19:2 of target byte address for source of CQ (read + only). The buffer must be aligned on a word boundary */ + __IM uint32_t : 8; + __IOM uint32_t CQADDR28 : 1; /*!< [28..28] Bit 28 of target byte address for source of CQ (read + only). Used to denote Flash (0) or SRAM (1) access */ + } CQADDR_b; + } ; + + union { + __IOM uint32_t CQSTAT; /*!< (@ 0x00000250) Command Queue Status Register */ + + struct { + __IOM uint32_t CQTIP : 1; /*!< [0..0] Command queue Transfer In Progress indicator. 1 will + indicate that a CQ transfer is active and this will remain + active even when paused waiting for external event. */ + __IOM uint32_t CQPAUSED : 1; /*!< [1..1] Command queue operation is currently paused. */ + __IOM uint32_t CQERR : 1; /*!< [2..2] Command queue processing Error. This active high bit + signals that an error was encountered during the CQ operation. */ + } CQSTAT_b; + } ; + + union { + __IOM uint32_t CQFLAGS; /*!< (@ 0x00000254) Command Queue Flag Register */ + + struct { + __IOM uint32_t CQFLAGS : 16; /*!< [15..0] Current flag status (read-only). Bits [7:0] are software + controllable and bits [15:8] are hardware status. */ + __IOM uint32_t CQIRQMASK : 16; /*!< [31..16] Provides for a per-bit mask of the flags used to invoke + an interrupt. A '1' in the bit position will enable the + pause event to trigger the interrupt, if the CQWT_int interrupt + is enabled.Bits definitions are the same as CQPAUSE */ + } CQFLAGS_b; + } ; + + union { + __IOM uint32_t CQSETCLEAR; /*!< (@ 0x00000258) Command Queue Flag Set/Clear Register */ + + struct { + __IOM uint32_t CQFSET : 8; /*!< [7..0] Set CQFlag status bits. Will set to 1 the value of any + SWFLAG with a '1' in the corresponding bit position of + this field */ + __IOM uint32_t CQFTGL : 8; /*!< [15..8] Toggle the indicated bit. Will toggle the value of any + SWFLAG with a '1' in the corresponding bit position of + this field */ + __IOM uint32_t CQFCLR : 8; /*!< [23..16] Clear CQFlag status bits. Will clear to 0 any SWFLAG + with a '1' in the corresponding bit position of this field */ + } CQSETCLEAR_b; + } ; + + union { + __IOM uint32_t CQPAUSEEN; /*!< (@ 0x0000025C) Command Queue Pause Enable Register */ + + struct { + __IOM uint32_t CQPEN : 16; /*!< [15..0] Enables the specified event to pause command processing + when active */ + } CQPAUSEEN_b; + } ; + + union { + __IOM uint32_t CQCURIDX; /*!< (@ 0x00000260) IOM Command Queue current index value . Compared + to the CQENDIDX reg contents to generate + the IDXEQ Pause event for command queue */ + + struct { + __IOM uint32_t CQCURIDX : 8; /*!< [7..0] Holds 8 bits of data that will be compared with the CQENDIX + register field. If the values match, the IDXEQ pause event + will be activated, which will cause the pausing of command + quue operation if the IDXEQ bit is enabled in CQPAUSEEN. */ + } CQCURIDX_b; + } ; + + union { + __IOM uint32_t CQENDIDX; /*!< (@ 0x00000264) IOM Command Queue current index value . Compared + to the CQCURIDX reg contents to generate + the IDXEQ Pause event for command queue */ + + struct { + __IOM uint32_t CQENDIDX : 8; /*!< [7..0] Holds 8 bits of data that will be compared with the CQCURIX + register field. If the values match, the IDXEQ pause event + will be activated, which will cause the pausing of command + quue operation if the IDXEQ bit is enabled in CQPAUSEEN. */ + } CQENDIDX_b; + } ; + + union { + __IOM uint32_t STATUS; /*!< (@ 0x00000268) IOM Module Status Register */ + + struct { + __IOM uint32_t ERR : 1; /*!< [0..0] Bit has been deprecated. Please refer to the other error + indicators. This will always return 0. */ + __IOM uint32_t CMDACT : 1; /*!< [1..1] Indicates if the active I/O Command is currently processing + a transaction, or command is complete, but the FIFO pointers + are still syncronizing internally. This bit will go high + atthe start of the transaction, and will go low when the + command is complete, and the data and pointers within the + FIFO have been syncronized. */ + __IOM uint32_t IDLEST : 1; /*!< [2..2] indicates if the active I/O state machine is IDLE. Note + - The state machine could be in idle state due to holdoffs + from data availability, or as the command gets propagated + into the logic from the registers. */ + } STATUS_b; + } ; + __IM uint32_t RESERVED4[37]; + + union { + __IOM uint32_t MSPICFG; /*!< (@ 0x00000300) SPI module master configuration */ + + struct { + __IOM uint32_t SPOL : 1; /*!< [0..0] This bit selects SPI polarity. */ + __IOM uint32_t SPHA : 1; /*!< [1..1] Selects the SPI phase; When 1, will shift the sampling + edge by 1/2 clock. */ + __IOM uint32_t FULLDUP : 1; /*!< [2..2] Full Duplex mode. Capture read data during writes operations */ + __IM uint32_t : 13; + __IOM uint32_t WTFC : 1; /*!< [16..16] Enables flow control of new write transactions based + on the SPI_STATUS signal from the BLE Core. */ + __IOM uint32_t RDFC : 1; /*!< [17..17] Enables flow control of new read transactions based + on the SPI_STATUS signal from the BLE Core. */ + __IM uint32_t : 3; + __IOM uint32_t WTFCPOL : 1; /*!< [21..21] Selects the write flow control signal polarity. The + transfers are halted when the selected flow control signal + is OPPOSITE polarity of this bit. (For example: WTFCPOL + = 0 will allow a SPI_STATUS=1 to pause transfers). */ + __IOM uint32_t RDFCPOL : 1; /*!< [22..22] Selects the read flow control signal polarity. When + set, the clock will be held low until the flow control + is de-asserted. */ + __IOM uint32_t SPILSB : 1; /*!< [23..23] Selects data transfer as MSB first (0) or LSB first + (1) for the data portion of the SPI transaction. The offset + bytes are always transmitted MSB first. */ + __IOM uint32_t DINDLY : 3; /*!< [26..24] Delay tap to use for the input signal (MISO). This + gives more hold time on the input data. */ + __IOM uint32_t DOUTDLY : 3; /*!< [29..27] Delay tap to use for the output signal (MOSI). This + give more hold time on the output data. */ + __IOM uint32_t MSPIRST : 1; /*!< [30..30] Bit is deprecated. setting it will have no effect. */ + } MSPICFG_b; + } ; + + union { + __IOM uint32_t BLECFG; /*!< (@ 0x00000304) BLE Core Control */ + + struct { + __IOM uint32_t PWRSMEN : 1; /*!< [0..0] Enable the power state machine for automatic sequencing + and control of power states of the BLE Core module. */ + __IOM uint32_t BLERSTN : 1; /*!< [1..1] Reset line to the BLE Core. This will reset the BLE core + when asserted ('0') and must be written to '1' prior to + performing any BTLE related operations to the core. */ + __IOM uint32_t WAKEUPCTL : 2; /*!< [3..2] WAKE signal override. Controls the source of the WAKE + signal to the BLE Core. */ + __IOM uint32_t DCDCFLGCTL : 2; /*!< [5..4] DCDCFLG signal override. The value of this field will + be sent to the BLE Core when the PWRSM is off. Otherwise, + the value is supplied from internal logic. */ + __IOM uint32_t BLEHREQCTL : 2; /*!< [7..6] BLEH power on request override. The value of this field + will be sent to the BLE Core when the PWRSM is off. Otherwise, + the value is supplied from internal logic. */ + __IOM uint32_t WT4ACTOFF : 1; /*!< [8..8] Debug control of BLEIF power state machine. Allows transition + into the active state in the BLEIF state without waiting + for dcdc req from BLE Core. */ + __IOM uint32_t MCUFRCSLP : 1; /*!< [9..9] Force power state machine to go to the sleep state. Intended + for debug only. Has no effect on the actual BLE Core state, + only the state of the BLEIF interface state machine. */ + __IOM uint32_t FRCCLK : 1; /*!< [10..10] Force the clock in the BLEIF to be always running */ + __IOM uint32_t STAYASLEEP : 1; /*!< [11..11] Set to prevent the BLE power control module from waking + up the BLE Core after going into power down. To be used + for graceful shutdown, set by software prior to powering + off and will allow assertion of reset from sleep state. */ + __IOM uint32_t PWRISOCTL : 2; /*!< [13..12] Configuration of BLEH isolation control for power related + signals. */ + __IOM uint32_t SPIISOCTL : 2; /*!< [15..14] Configuration of BLEH isolation controls for SPI related + signals. */ + } BLECFG_b; + } ; + + union { + __IOM uint32_t PWRCMD; /*!< (@ 0x00000308) BLE Power command interface */ + + struct { + __IOM uint32_t WAKEREQ : 1; /*!< [0..0] Wake request from the MCU. When asserted (1), the BLE + Interface logic will assert the wakeup request signal to + the BLE Core. Only recognized when in the sleep state */ + __IOM uint32_t RESTART : 1; /*!< [1..1] Restart the BLE Core after going into the shutdown state. + Only valid when in the shutdown state. */ + } PWRCMD_b; + } ; + + union { + __IOM uint32_t BSTATUS; /*!< (@ 0x0000030C) BLE Core status */ + + struct { + __IOM uint32_t B2MSTATE : 3; /*!< [2..0] State of the BLE Core logic. */ + __IOM uint32_t SPISTATUS : 1; /*!< [3..3] Value of the SPISTATUS signal from the BLE Core. The + signal is asserted when the BLE Core is able to accept + write data via the SPI interface. Data should be transmitted + to theBLE core only when this signal is 1. The hardware + will automatically wait for this signal prior to performing + a write operation if flow control is active. */ + __IOM uint32_t DCDCREQ : 1; /*!< [4..4] Value of the DCDCREQ signal from the BLE Core. The DCDCREQ + signal is sent from the core to the BLEIF module when the + BLE core requires BLEH power to be active. When activated, + this isindicated by DCDCFLAG going to 1. */ + __IOM uint32_t DCDCFLAG : 1; /*!< [5..5] Value of the DCDCFLAG signal to the BLE Core. The DCDCFLAG + is a signal to the BLE Core indicating that the BLEH ppower + is active. */ + __IOM uint32_t WAKEUP : 1; /*!< [6..6] Value of the WAKEUP signal to the BLE Core . The WAKEUP + signals is sent from the BLEIF to the BLECORE to request + the BLE Core transition from sleep state to active state. */ + __IOM uint32_t BLEIRQ : 1; /*!< [7..7] Status of the BLEIRQ signal from the BLE Core. A value + of 1 idicates that read data is available in the core and + a read operation needs to be performed. */ + __IOM uint32_t PWRST : 3; /*!< [10..8] Current status of the power state machine */ + __IOM uint32_t BLEHACK : 1; /*!< [11..11] Value of the BLEHACK signal from the power control + unit. If the signal is '1', the BLEH power is active and + ready for use. */ + __IOM uint32_t BLEHREQ : 1; /*!< [12..12] Value of the BLEHREQ signal to the power control unit. + The BLEHREQ signal is sent from the BLEIF module to the + power control module to request the BLEH power up. When + the BLEHACK signal is asserted,BLEH power is stable and + ready for use. */ + } BSTATUS_b; + } ; + __IM uint32_t RESERVED5[64]; + + union { + __IOM uint32_t BLEDBG; /*!< (@ 0x00000410) BLEIF Master Debug Register */ + + struct { + __IOM uint32_t DBGEN : 1; /*!< [0..0] Debug Enable. Setting this bit will enable the update + of data within this register, otherwise it is clock gated + for power savings */ + __IOM uint32_t IOCLKON : 1; /*!< [1..1] IOCLK debug clock control. Enable IO_CLK to be active + when this bit is '1'. Otherwise, the clock is controlled + with gating from the logic as needed. */ + __IOM uint32_t APBCLKON : 1; /*!< [2..2] APBCLK debug clock control. Enable APB_CLK to be active + when this bit is '1'. Otherwise, the clock is controlled + with gating from the logic as needed. */ + __IOM uint32_t DBGDATA : 29; /*!< [31..3] Debug data */ + } BLEDBG_b; + } ; +} BLEIF_Type; /*!< Size = 1044 (0x414) */ + + + +/* =========================================================================================================================== */ +/* ================ CACHECTRL ================ */ +/* =========================================================================================================================== */ + + +/** + * @brief Flash Cache Controller (CACHECTRL) + */ + +typedef struct { /*!< (@ 0x40018000) CACHECTRL Structure */ + + union { + __IOM uint32_t CACHECFG; /*!< (@ 0x00000000) Flash Cache Control Register */ + + struct { + __IOM uint32_t ENABLE : 1; /*!< [0..0] Enables the flash cache controller and enables power + to the cache SRAMs. The ICACHE_ENABLE and DCACHE_ENABLE + should be set to enable caching for each type of access. */ + __IOM uint32_t LRU : 1; /*!< [1..1] Sets the cache repleacment policy. 0=LRR (least recently + replaced), 1=LRU (least recently used). LRR minimizes writes + to the TAG SRAM. */ + __IOM uint32_t ENABLE_NC0 : 1; /*!< [2..2] Enable Non-cacheable region 0. See NCR0 registers to + define the region. */ + __IOM uint32_t ENABLE_NC1 : 1; /*!< [3..3] Enable Non-cacheable region 1. See NCR1 registers to + define the region. */ + __IOM uint32_t CONFIG : 4; /*!< [7..4] Sets the cache configuration */ + __IOM uint32_t ICACHE_ENABLE : 1; /*!< [8..8] Enable Flash Instruction Caching */ + __IOM uint32_t DCACHE_ENABLE : 1; /*!< [9..9] Enable Flash Data Caching. */ + __IOM uint32_t CACHE_CLKGATE : 1; /*!< [10..10] Enable clock gating of cache TAG RAM. Software should + enable this bit for optimal power efficiency. */ + __IOM uint32_t CACHE_LS : 1; /*!< [11..11] Enable LS (light sleep) of cache RAMs. Software should + DISABLE this bit since cache activity is too high to benefit + from LS usage. */ + __IM uint32_t : 8; + __IOM uint32_t DATA_CLKGATE : 1; /*!< [20..20] Enable aggressive clock gating of entire data array. + This bit should be set to 1 for optimal power efficiency. */ + __IM uint32_t : 3; + __IOM uint32_t ENABLE_MONITOR : 1; /*!< [24..24] Enable Cache Monitoring Stats. Cache monitoring consumes + additional power and should only be enabled when profiling + code and counters will increment when this bit is set. + Counter values will be retained when this is set to 0, + allowing software to enable/disable counting for multiple + code segments. */ + } CACHECFG_b; + } ; + + union { + __IOM uint32_t FLASHCFG; /*!< (@ 0x00000004) Flash Control Register */ + + struct { + __IOM uint32_t RD_WAIT : 4; /*!< [3..0] Sets read waitstates for normal (fast) operation. A value + of 1 is recommended. */ + __IOM uint32_t SEDELAY : 3; /*!< [6..4] Sets SE delay (flash address setup). A value of 5 is + recommended. */ + __IM uint32_t : 1; + __IOM uint32_t LPM_RD_WAIT : 4; /*!< [11..8] Sets flash waitstates when in LPM Mode 2 (RD_WAIT in + LPM mode 2 only) */ + __IOM uint32_t LPMMODE : 2; /*!< [13..12] Controls flash low power modes (control of LPM pin). */ + } FLASHCFG_b; + } ; + + union { + __IOM uint32_t CTRL; /*!< (@ 0x00000008) Cache Control */ + + struct { + __IOM uint32_t INVALIDATE : 1; /*!< [0..0] Writing a 1 to this bitfield invalidates the flash cache + contents. */ + __IOM uint32_t RESET_STAT : 1; /*!< [1..1] Reset Cache Statistics. When written to a 1, the cache + monitor counters will be cleared. The monitor counters + can be reset only when the CACHECFG.ENABLE_MONITOR bit + is set. */ + __IOM uint32_t CACHE_READY : 1; /*!< [2..2] Cache Ready Status (enabled and not processing an invalidate + operation) */ + __IM uint32_t : 1; + __IOM uint32_t FLASH0_SLM_STATUS : 1; /*!< [4..4] Flash Sleep Mode Status. 1 indicates that flash0 is in + sleep mode, 0 indicates flash0 is in normal mode. */ + __IOM uint32_t FLASH0_SLM_DISABLE : 1; /*!< [5..5] Disable Flash Sleep Mode. Write 1 to wake flash0 from + sleep mode (reading the array will also automatically wake + it). */ + __IOM uint32_t FLASH0_SLM_ENABLE : 1; /*!< [6..6] Enable Flash Sleep Mode. Write to 1 to put flash 0 into + sleep mode. NOTE: there is a 5us latency after waking flash + until the first access will be returned. */ + __IM uint32_t : 1; + __IOM uint32_t FLASH1_SLM_STATUS : 1; /*!< [8..8] Flash Sleep Mode Status. 1 indicates that flash1 is in + sleep mode, 0 indicates flash1 is in normal mode. */ + __IOM uint32_t FLASH1_SLM_DISABLE : 1; /*!< [9..9] Disable Flash Sleep Mode. Write 1 to wake flash1 from + sleep mode (reading the array will also automatically wake + it). */ + __IOM uint32_t FLASH1_SLM_ENABLE : 1; /*!< [10..10] Enable Flash Sleep Mode. Write to 1 to put flash 1 + into sleep mode. NOTE: there is a 5us latency after waking + flash until the first access will be returned. */ + } CTRL_b; + } ; + __IM uint32_t RESERVED; + + union { + __IOM uint32_t NCR0START; /*!< (@ 0x00000010) Flash Cache Noncachable Region 0 Start */ + + struct { + __IM uint32_t : 4; + __IOM uint32_t ADDR : 23; /*!< [26..4] Start address for non-cacheable region 0 */ + } NCR0START_b; + } ; + + union { + __IOM uint32_t NCR0END; /*!< (@ 0x00000014) Flash Cache Noncachable Region 0 End */ + + struct { + __IM uint32_t : 4; + __IOM uint32_t ADDR : 23; /*!< [26..4] End address for non-cacheable region 0 */ + } NCR0END_b; + } ; + + union { + __IOM uint32_t NCR1START; /*!< (@ 0x00000018) Flash Cache Noncachable Region 1 Start */ + + struct { + __IM uint32_t : 4; + __IOM uint32_t ADDR : 23; /*!< [26..4] Start address for non-cacheable region 1 */ + } NCR1START_b; + } ; + + union { + __IOM uint32_t NCR1END; /*!< (@ 0x0000001C) Flash Cache Noncachable Region 1 End */ + + struct { + __IM uint32_t : 4; + __IOM uint32_t ADDR : 23; /*!< [26..4] End address for non-cacheable region 1 */ + } NCR1END_b; + } ; + __IM uint32_t RESERVED1[8]; + + union { + __IOM uint32_t DMON0; /*!< (@ 0x00000040) Data Cache Total Accesses */ + + struct { + __IOM uint32_t DACCESS_COUNT : 32; /*!< [31..0] Total accesses to data cache. All performance metrics + should be relative to the number of accesses performed. */ + } DMON0_b; + } ; + + union { + __IOM uint32_t DMON1; /*!< (@ 0x00000044) Data Cache Tag Lookups */ + + struct { + __IOM uint32_t DLOOKUP_COUNT : 32; /*!< [31..0] Total tag lookups from data cache. */ + } DMON1_b; + } ; + + union { + __IOM uint32_t DMON2; /*!< (@ 0x00000048) Data Cache Hits */ + + struct { + __IOM uint32_t DHIT_COUNT : 32; /*!< [31..0] Cache hits from lookup operations. */ + } DMON2_b; + } ; + + union { + __IOM uint32_t DMON3; /*!< (@ 0x0000004C) Data Cache Line Hits */ + + struct { + __IOM uint32_t DLINE_COUNT : 32; /*!< [31..0] Cache hits from line cache */ + } DMON3_b; + } ; + + union { + __IOM uint32_t IMON0; /*!< (@ 0x00000050) Instruction Cache Total Accesses */ + + struct { + __IOM uint32_t IACCESS_COUNT : 32; /*!< [31..0] Total accesses to Instruction cache */ + } IMON0_b; + } ; + + union { + __IOM uint32_t IMON1; /*!< (@ 0x00000054) Instruction Cache Tag Lookups */ + + struct { + __IOM uint32_t ILOOKUP_COUNT : 32; /*!< [31..0] Total tag lookups from Instruction cache */ + } IMON1_b; + } ; + + union { + __IOM uint32_t IMON2; /*!< (@ 0x00000058) Instruction Cache Hits */ + + struct { + __IOM uint32_t IHIT_COUNT : 32; /*!< [31..0] Cache hits from lookup operations */ + } IMON2_b; + } ; + + union { + __IOM uint32_t IMON3; /*!< (@ 0x0000005C) Instruction Cache Line Hits */ + + struct { + __IOM uint32_t ILINE_COUNT : 32; /*!< [31..0] Cache hits from line cache */ + } IMON3_b; + } ; +} CACHECTRL_Type; /*!< Size = 96 (0x60) */ + + + +/* =========================================================================================================================== */ +/* ================ CLKGEN ================ */ +/* =========================================================================================================================== */ + + +/** + * @brief Clock Generator (CLKGEN) + */ + +typedef struct { /*!< (@ 0x40004000) CLKGEN Structure */ + + union { + __IOM uint32_t CALXT; /*!< (@ 0x00000000) XT Oscillator Control */ + + struct { + __IOM uint32_t CALXT : 11; /*!< [10..0] XT Oscillator calibration value. This register will + enable the hardware to increase or decrease the number + of cycles in a 16KHz clock derived from the original 32KHz + version. The most significant bit is the sign. A '1' is + a reduction, and a '0' is an addition. This calibration + value will add or reduce the number of cycles programmed + here across a 32 second interval. The maximum value that + is effective is from -1024 to 1023. */ + } CALXT_b; + } ; + + union { + __IOM uint32_t CALRC; /*!< (@ 0x00000004) RC Oscillator Control */ + + struct { + __IOM uint32_t CALRC : 18; /*!< [17..0] LFRC Oscillator calibration value. This register will + enable the hardware to increase or decrease the number + of cycles in a 512 Hz clock derived from the original 1024 + version. The most significant bit is the sign. A '1' is + a reduction, and a '0' is an addition. This calibration + value will add or reduce the number of cycles programmed + here across a 32 second interval. The range is from -131072 + (decimal) to 131071 (decimal). This register is normally + used in conjuction with ACALCTR register. The CAL */ + } CALRC_b; + } ; + + union { + __IOM uint32_t ACALCTR; /*!< (@ 0x00000008) Autocalibration Counter */ + + struct { + __IOM uint32_t ACALCTR : 24; /*!< [23..0] Autocalibration Counter result. Bits 17 down to 0 of + this is feed directly to the CALRC register if ACAL register + in OCTRL register is set to 1024SEC or 512SEC. */ + } ACALCTR_b; + } ; + + union { + __IOM uint32_t OCTRL; /*!< (@ 0x0000000C) Oscillator Control */ + + struct { + __IOM uint32_t STOPXT : 1; /*!< [0..0] Stop the XT Oscillator to the RTC */ + __IOM uint32_t STOPRC : 1; /*!< [1..1] Stop the LFRC Oscillator to the RTC */ + __IM uint32_t : 4; + __IOM uint32_t FOS : 1; /*!< [6..6] Oscillator switch on failure function. If this is set, + then LFRC clock source will switch from XT to RC. */ + __IOM uint32_t OSEL : 1; /*!< [7..7] Selects the RTC oscillator (1 => LFRC, 0 => XT) */ + __IOM uint32_t ACAL : 3; /*!< [10..8] Autocalibration control. This selects the source to + be used in the autocalibration flow. This flow can also + be used to measure an internal clock against an external + clock source, with the external clock normally used as + the reference. */ + } OCTRL_b; + } ; + + union { + __IOM uint32_t CLKOUT; /*!< (@ 0x00000010) CLKOUT Frequency Select */ + + struct { + __IOM uint32_t CKSEL : 6; /*!< [5..0] CLKOUT signal select */ + __IM uint32_t : 1; + __IOM uint32_t CKEN : 1; /*!< [7..7] Enable the CLKOUT signal */ + } CLKOUT_b; + } ; + + union { + __IOM uint32_t CLKKEY; /*!< (@ 0x00000014) Key Register for Clock Control Register */ + + struct { + __IOM uint32_t CLKKEY : 32; /*!< [31..0] Key register value. */ + } CLKKEY_b; + } ; + + union { + __IOM uint32_t CCTRL; /*!< (@ 0x00000018) HFRC Clock Control */ + + struct { + __IOM uint32_t CORESEL : 1; /*!< [0..0] Core Clock divisor */ + } CCTRL_b; + } ; + + union { + __IOM uint32_t STATUS; /*!< (@ 0x0000001C) Clock Generator Status */ + + struct { + __IOM uint32_t OMODE : 1; /*!< [0..0] Current RTC oscillator (1 => LFRC, 0 => XT). After an + RTC oscillator change, it may take up to 2 seconds for + this field to reflect the new oscillator. */ + __IOM uint32_t OSCF : 1; /*!< [1..1] XT Oscillator is enabled but not oscillating */ + } STATUS_b; + } ; + + union { + __IOM uint32_t HFADJ; /*!< (@ 0x00000020) HFRC Adjustment */ + + struct { + __IOM uint32_t HFADJEN : 1; /*!< [0..0] HFRC adjustment control */ + __IOM uint32_t HFADJCK : 3; /*!< [3..1] Repeat period for HFRC adjustment */ + __IM uint32_t : 4; + __IOM uint32_t HFXTADJ : 12; /*!< [19..8] Target HFRC adjustment value. */ + __IOM uint32_t HFWARMUP : 1; /*!< [20..20] XT warmup period for HFRC adjustment */ + __IOM uint32_t HFADJGAIN : 3; /*!< [23..21] Gain control for HFRC adjustment */ + } HFADJ_b; + } ; + __IM uint32_t RESERVED; + + union { + __IOM uint32_t CLOCKENSTAT; /*!< (@ 0x00000028) Clock Enable Status */ + + struct { + __IOM uint32_t CLOCKENSTAT : 32; /*!< [31..0] Clock enable status */ + } CLOCKENSTAT_b; + } ; + + union { + __IOM uint32_t CLOCKEN2STAT; /*!< (@ 0x0000002C) Clock Enable Status */ + + struct { + __IOM uint32_t CLOCKEN2STAT : 32; /*!< [31..0] Clock enable status 2 */ + } CLOCKEN2STAT_b; + } ; + + union { + __IOM uint32_t CLOCKEN3STAT; /*!< (@ 0x00000030) Clock Enable Status */ + + struct { + __IOM uint32_t CLOCKEN3STAT : 32; /*!< [31..0] Clock enable status 3 */ + } CLOCKEN3STAT_b; + } ; + + union { + __IOM uint32_t FREQCTRL; /*!< (@ 0x00000034) HFRC Frequency Control register */ + + struct { + __IOM uint32_t BURSTREQ : 1; /*!< [0..0] Frequency Burst Enable Request */ + __IOM uint32_t BURSTACK : 1; /*!< [1..1] Frequency Burst Request Acknowledge. Frequency burst + requested is always acknowledged whether burst is granted + or not depending on feature enable. */ + __IOM uint32_t BURSTSTATUS : 1; /*!< [2..2] This represents frequency burst status. */ + } FREQCTRL_b; + } ; + __IM uint32_t RESERVED1; + + union { + __IOM uint32_t BLEBUCKTONADJ; /*!< (@ 0x0000003C) BLE BUCK TON ADJUST */ + + struct { + __IOM uint32_t TONLOWTHRESHOLD : 10; /*!< [9..0] TON ADJUST LOW THRESHOLD. Suggested values are #A(94KHz) + #15(47KHz) #53(12Khz) #14D(3Khz) */ + __IOM uint32_t TONHIGHTHRESHOLD : 10; /*!< [19..10] TON ADJUST HIGH THRESHOLD. Suggested values are #15(94KHz) + #2A(47Khz) #A6(12Khz) #29A(3Khz) */ + __IOM uint32_t TONADJUSTPERIOD : 2; /*!< [21..20] TON ADJUST PERIOD */ + __IOM uint32_t TONADJUSTEN : 1; /*!< [22..22] TON ADJUST ENABLE */ + __IOM uint32_t ZEROLENDETECTTRIM : 4; /*!< [26..23] BLEBUCK ZERO LENGTH DETECT TRIM */ + __IOM uint32_t ZEROLENDETECTEN : 1; /*!< [27..27] BLEBUCK ZERO LENGTH DETECT ENABLE */ + } BLEBUCKTONADJ_b; + } ; + __IM uint32_t RESERVED2[48]; + + union { + __IOM uint32_t INTRPTEN; /*!< (@ 0x00000100) CLKGEN Interrupt Register: Enable */ + + struct { + __IOM uint32_t ACF : 1; /*!< [0..0] Autocalibration Fail interrupt */ + __IOM uint32_t ACC : 1; /*!< [1..1] Autocalibration Complete interrupt */ + __IOM uint32_t OF : 1; /*!< [2..2] XT Oscillator Fail interrupt */ + } INTRPTEN_b; + } ; + + union { + __IOM uint32_t INTRPTSTAT; /*!< (@ 0x00000104) CLKGEN Interrupt Register: Status */ + + struct { + __IOM uint32_t ACF : 1; /*!< [0..0] Autocalibration Fail interrupt */ + __IOM uint32_t ACC : 1; /*!< [1..1] Autocalibration Complete interrupt */ + __IOM uint32_t OF : 1; /*!< [2..2] XT Oscillator Fail interrupt */ + } INTRPTSTAT_b; + } ; + + union { + __IOM uint32_t INTRPTCLR; /*!< (@ 0x00000108) CLKGEN Interrupt Register: Clear */ + + struct { + __IOM uint32_t ACF : 1; /*!< [0..0] Autocalibration Fail interrupt */ + __IOM uint32_t ACC : 1; /*!< [1..1] Autocalibration Complete interrupt */ + __IOM uint32_t OF : 1; /*!< [2..2] XT Oscillator Fail interrupt */ + } INTRPTCLR_b; + } ; + + union { + __IOM uint32_t INTRPTSET; /*!< (@ 0x0000010C) CLKGEN Interrupt Register: Set */ + + struct { + __IOM uint32_t ACF : 1; /*!< [0..0] Autocalibration Fail interrupt */ + __IOM uint32_t ACC : 1; /*!< [1..1] Autocalibration Complete interrupt */ + __IOM uint32_t OF : 1; /*!< [2..2] XT Oscillator Fail interrupt */ + } INTRPTSET_b; + } ; +} CLKGEN_Type; /*!< Size = 272 (0x110) */ + + + +/* =========================================================================================================================== */ +/* ================ CTIMER ================ */ +/* =========================================================================================================================== */ + + +/** + * @brief Counter/Timer (CTIMER) + */ + +typedef struct { /*!< (@ 0x40008000) CTIMER Structure */ + + union { + __IOM uint32_t TMR0; /*!< (@ 0x00000000) Counter/Timer Register */ + + struct { + __IOM uint32_t CTTMRA0 : 16; /*!< [15..0] Counter/Timer A0. */ + __IOM uint32_t CTTMRB0 : 16; /*!< [31..16] Counter/Timer B0. */ + } TMR0_b; + } ; + + union { + __IOM uint32_t CMPRA0; /*!< (@ 0x00000004) Counter/Timer A0 Compare Registers */ + + struct { + __IOM uint32_t CMPR0A0 : 16; /*!< [15..0] Counter/Timer A0 Compare Register 0. Holds the lower + limit for timer half A. */ + __IOM uint32_t CMPR1A0 : 16; /*!< [31..16] Counter/Timer A0 Compare Register 1. Holds the upper + limit for timer half A. */ + } CMPRA0_b; + } ; + + union { + __IOM uint32_t CMPRB0; /*!< (@ 0x00000008) Counter/Timer B0 Compare Registers */ + + struct { + __IOM uint32_t CMPR0B0 : 16; /*!< [15..0] Counter/Timer B0 Compare Register 0. Holds the lower + limit for timer half B. */ + __IOM uint32_t CMPR1B0 : 16; /*!< [31..16] Counter/Timer B0 Compare Register 1. Holds the upper + limit for timer half B. */ + } CMPRB0_b; + } ; + + union { + __IOM uint32_t CTRL0; /*!< (@ 0x0000000C) Counter/Timer Control */ + + struct { + __IOM uint32_t TMRA0EN : 1; /*!< [0..0] Counter/Timer A0 Enable bit. */ + __IOM uint32_t TMRA0CLK : 5; /*!< [5..1] Counter/Timer A0 Clock Select. */ + __IOM uint32_t TMRA0FN : 3; /*!< [8..6] Counter/Timer A0 Function Select. */ + __IOM uint32_t TMRA0IE0 : 1; /*!< [9..9] Counter/Timer A0 Interrupt Enable bit based on COMPR0. */ + __IOM uint32_t TMRA0IE1 : 1; /*!< [10..10] Counter/Timer A0 Interrupt Enable bit based on COMPR1. */ + __IOM uint32_t TMRA0CLR : 1; /*!< [11..11] Counter/Timer A0 Clear bit. */ + __IOM uint32_t TMRA0POL : 1; /*!< [12..12] Counter/Timer A0 output polarity. */ + __IM uint32_t : 3; + __IOM uint32_t TMRB0EN : 1; /*!< [16..16] Counter/Timer B0 Enable bit. */ + __IOM uint32_t TMRB0CLK : 5; /*!< [21..17] Counter/Timer B0 Clock Select. */ + __IOM uint32_t TMRB0FN : 3; /*!< [24..22] Counter/Timer B0 Function Select. */ + __IOM uint32_t TMRB0IE0 : 1; /*!< [25..25] Counter/Timer B0 Interrupt Enable bit for COMPR0. */ + __IOM uint32_t TMRB0IE1 : 1; /*!< [26..26] Counter/Timer B0 Interrupt Enable bit for COMPR1. */ + __IOM uint32_t TMRB0CLR : 1; /*!< [27..27] Counter/Timer B0 Clear bit. */ + __IOM uint32_t TMRB0POL : 1; /*!< [28..28] Counter/Timer B0 output polarity. */ + __IM uint32_t : 2; + __IOM uint32_t CTLINK0 : 1; /*!< [31..31] Counter/Timer A0/B0 Link bit. */ + } CTRL0_b; + } ; + __IM uint32_t RESERVED; + + union { + __IOM uint32_t CMPRAUXA0; /*!< (@ 0x00000014) Counter/Timer A0 Compare Registers */ + + struct { + __IOM uint32_t CMPR2A0 : 16; /*!< [15..0] Counter/Timer A0 Compare Register 2. Holds the lower + limit for timer half A. */ + __IOM uint32_t CMPR3A0 : 16; /*!< [31..16] Counter/Timer A0 Compare Register 3. Holds the upper + limit for timer half A. */ + } CMPRAUXA0_b; + } ; + + union { + __IOM uint32_t CMPRAUXB0; /*!< (@ 0x00000018) Counter/Timer B0 Compare Registers */ + + struct { + __IOM uint32_t CMPR2B0 : 16; /*!< [15..0] Counter/Timer B0 Compare Register 2. Holds the lower + limit for timer half B. */ + __IOM uint32_t CMPR3B0 : 16; /*!< [31..16] Counter/Timer B0 Compare Register 3. Holds the upper + limit for timer half B. */ + } CMPRAUXB0_b; + } ; + + union { + __IOM uint32_t AUX0; /*!< (@ 0x0000001C) Counter/Timer Auxiliary */ + + struct { + __IOM uint32_t TMRA0LMT : 7; /*!< [6..0] Counter/Timer A0 Pattern Limit Count. */ + __IOM uint32_t TMRA0TRIG : 4; /*!< [10..7] Counter/Timer A0 Trigger Select. */ + __IOM uint32_t TMRA0NOSYNC : 1; /*!< [11..11] Source clock synchronization control. */ + __IOM uint32_t TMRA0TINV : 1; /*!< [12..12] Counter/Timer A0 Invert on trigger. */ + __IOM uint32_t TMRA0POL23 : 1; /*!< [13..13] Counter/Timer A0 Upper output polarity */ + __IOM uint32_t TMRA0EN23 : 1; /*!< [14..14] Counter/Timer A0 Upper compare enable. */ + __IM uint32_t : 1; + __IOM uint32_t TMRB0LMT : 6; /*!< [21..16] Counter/Timer B0 Pattern Limit Count. */ + __IM uint32_t : 1; + __IOM uint32_t TMRB0TRIG : 4; /*!< [26..23] Counter/Timer B0 Trigger Select. */ + __IOM uint32_t TMRB0NOSYNC : 1; /*!< [27..27] Source clock synchronization control. */ + __IOM uint32_t TMRB0TINV : 1; /*!< [28..28] Counter/Timer B0 Invert on trigger. */ + __IOM uint32_t TMRB0POL23 : 1; /*!< [29..29] Upper output polarity */ + __IOM uint32_t TMRB0EN23 : 1; /*!< [30..30] Counter/Timer B0 Upper compare enable. */ + } AUX0_b; + } ; + + union { + __IOM uint32_t TMR1; /*!< (@ 0x00000020) Counter/Timer Register */ + + struct { + __IOM uint32_t CTTMRA1 : 16; /*!< [15..0] Counter/Timer A1. */ + __IOM uint32_t CTTMRB1 : 16; /*!< [31..16] Counter/Timer B1. */ + } TMR1_b; + } ; + + union { + __IOM uint32_t CMPRA1; /*!< (@ 0x00000024) Counter/Timer A1 Compare Registers */ + + struct { + __IOM uint32_t CMPR0A1 : 16; /*!< [15..0] Counter/Timer A1 Compare Register 0. */ + __IOM uint32_t CMPR1A1 : 16; /*!< [31..16] Counter/Timer A1 Compare Register 1. */ + } CMPRA1_b; + } ; + + union { + __IOM uint32_t CMPRB1; /*!< (@ 0x00000028) Counter/Timer B1 Compare Registers */ + + struct { + __IOM uint32_t CMPR0B1 : 16; /*!< [15..0] Counter/Timer B1 Compare Register 0. */ + __IOM uint32_t CMPR1B1 : 16; /*!< [31..16] Counter/Timer B1 Compare Register 1. */ + } CMPRB1_b; + } ; + + union { + __IOM uint32_t CTRL1; /*!< (@ 0x0000002C) Counter/Timer Control */ + + struct { + __IOM uint32_t TMRA1EN : 1; /*!< [0..0] Counter/Timer A1 Enable bit. */ + __IOM uint32_t TMRA1CLK : 5; /*!< [5..1] Counter/Timer A1 Clock Select. */ + __IOM uint32_t TMRA1FN : 3; /*!< [8..6] Counter/Timer A1 Function Select. */ + __IOM uint32_t TMRA1IE0 : 1; /*!< [9..9] Counter/Timer A1 Interrupt Enable bit based on COMPR0. */ + __IOM uint32_t TMRA1IE1 : 1; /*!< [10..10] Counter/Timer A1 Interrupt Enable bit based on COMPR1. */ + __IOM uint32_t TMRA1CLR : 1; /*!< [11..11] Counter/Timer A1 Clear bit. */ + __IOM uint32_t TMRA1POL : 1; /*!< [12..12] Counter/Timer A1 output polarity. */ + __IM uint32_t : 3; + __IOM uint32_t TMRB1EN : 1; /*!< [16..16] Counter/Timer B1 Enable bit. */ + __IOM uint32_t TMRB1CLK : 5; /*!< [21..17] Counter/Timer B1 Clock Select. */ + __IOM uint32_t TMRB1FN : 3; /*!< [24..22] Counter/Timer B1 Function Select. */ + __IOM uint32_t TMRB1IE0 : 1; /*!< [25..25] Counter/Timer B1 Interrupt Enable bit for COMPR0. */ + __IOM uint32_t TMRB1IE1 : 1; /*!< [26..26] Counter/Timer B1 Interrupt Enable bit for COMPR1. */ + __IOM uint32_t TMRB1CLR : 1; /*!< [27..27] Counter/Timer B1 Clear bit. */ + __IOM uint32_t TMRB1POL : 1; /*!< [28..28] Counter/Timer B1 output polarity. */ + __IM uint32_t : 2; + __IOM uint32_t CTLINK1 : 1; /*!< [31..31] Counter/Timer A1/B1 Link bit. */ + } CTRL1_b; + } ; + __IM uint32_t RESERVED1; + + union { + __IOM uint32_t CMPRAUXA1; /*!< (@ 0x00000034) Counter/Timer A1 Compare Registers */ + + struct { + __IOM uint32_t CMPR2A1 : 16; /*!< [15..0] Counter/Timer A1 Compare Register 2. Holds the lower + limit for timer half A. */ + __IOM uint32_t CMPR3A1 : 16; /*!< [31..16] Counter/Timer A1 Compare Register 3. Holds the upper + limit for timer half A. */ + } CMPRAUXA1_b; + } ; + + union { + __IOM uint32_t CMPRAUXB1; /*!< (@ 0x00000038) Counter/Timer B1 Compare Registers */ + + struct { + __IOM uint32_t CMPR2B1 : 16; /*!< [15..0] Counter/Timer B1 Compare Register 2. Holds the lower + limit for timer half B. */ + __IOM uint32_t CMPR3B1 : 16; /*!< [31..16] Counter/Timer B1 Compare Register 3. Holds the upper + limit for timer half B. */ + } CMPRAUXB1_b; + } ; + + union { + __IOM uint32_t AUX1; /*!< (@ 0x0000003C) Counter/Timer Auxiliary */ + + struct { + __IOM uint32_t TMRA1LMT : 7; /*!< [6..0] Counter/Timer A1 Pattern Limit Count. */ + __IOM uint32_t TMRA1TRIG : 4; /*!< [10..7] Counter/Timer A1 Trigger Select. */ + __IOM uint32_t TMRA1NOSYNC : 1; /*!< [11..11] Source clock synchronization control. */ + __IOM uint32_t TMRA1TINV : 1; /*!< [12..12] Counter/Timer A1 Invert on trigger. */ + __IOM uint32_t TMRA1POL23 : 1; /*!< [13..13] Counter/Timer A1 Upper output polarity */ + __IOM uint32_t TMRA1EN23 : 1; /*!< [14..14] Counter/Timer A1 Upper compare enable. */ + __IM uint32_t : 1; + __IOM uint32_t TMRB1LMT : 6; /*!< [21..16] Counter/Timer B1 Pattern Limit Count. */ + __IM uint32_t : 1; + __IOM uint32_t TMRB1TRIG : 4; /*!< [26..23] Counter/Timer B1 Trigger Select. */ + __IOM uint32_t TMRB1NOSYNC : 1; /*!< [27..27] Source clock synchronization control. */ + __IOM uint32_t TMRB1TINV : 1; /*!< [28..28] Counter/Timer B1 Invert on trigger. */ + __IOM uint32_t TMRB1POL23 : 1; /*!< [29..29] Upper output polarity */ + __IOM uint32_t TMRB1EN23 : 1; /*!< [30..30] Counter/Timer B1 Upper compare enable. */ + } AUX1_b; + } ; + + union { + __IOM uint32_t TMR2; /*!< (@ 0x00000040) Counter/Timer Register */ + + struct { + __IOM uint32_t CTTMRA2 : 16; /*!< [15..0] Counter/Timer A2. */ + __IOM uint32_t CTTMRB2 : 16; /*!< [31..16] Counter/Timer B2. */ + } TMR2_b; + } ; + + union { + __IOM uint32_t CMPRA2; /*!< (@ 0x00000044) Counter/Timer A2 Compare Registers */ + + struct { + __IOM uint32_t CMPR0A2 : 16; /*!< [15..0] Counter/Timer A2 Compare Register 0. */ + __IOM uint32_t CMPR1A2 : 16; /*!< [31..16] Counter/Timer A2 Compare Register 1. */ + } CMPRA2_b; + } ; + + union { + __IOM uint32_t CMPRB2; /*!< (@ 0x00000048) Counter/Timer B2 Compare Registers */ + + struct { + __IOM uint32_t CMPR0B2 : 16; /*!< [15..0] Counter/Timer B2 Compare Register 0. */ + __IOM uint32_t CMPR1B2 : 16; /*!< [31..16] Counter/Timer B2 Compare Register 1. */ + } CMPRB2_b; + } ; + + union { + __IOM uint32_t CTRL2; /*!< (@ 0x0000004C) Counter/Timer Control */ + + struct { + __IOM uint32_t TMRA2EN : 1; /*!< [0..0] Counter/Timer A2 Enable bit. */ + __IOM uint32_t TMRA2CLK : 5; /*!< [5..1] Counter/Timer A2 Clock Select. */ + __IOM uint32_t TMRA2FN : 3; /*!< [8..6] Counter/Timer A2 Function Select. */ + __IOM uint32_t TMRA2IE0 : 1; /*!< [9..9] Counter/Timer A2 Interrupt Enable bit based on COMPR0. */ + __IOM uint32_t TMRA2IE1 : 1; /*!< [10..10] Counter/Timer A2 Interrupt Enable bit based on COMPR1. */ + __IOM uint32_t TMRA2CLR : 1; /*!< [11..11] Counter/Timer A2 Clear bit. */ + __IOM uint32_t TMRA2POL : 1; /*!< [12..12] Counter/Timer A2 output polarity. */ + __IM uint32_t : 3; + __IOM uint32_t TMRB2EN : 1; /*!< [16..16] Counter/Timer B2 Enable bit. */ + __IOM uint32_t TMRB2CLK : 5; /*!< [21..17] Counter/Timer B2 Clock Select. */ + __IOM uint32_t TMRB2FN : 3; /*!< [24..22] Counter/Timer B2 Function Select. */ + __IOM uint32_t TMRB2IE0 : 1; /*!< [25..25] Counter/Timer B2 Interrupt Enable bit for COMPR0. */ + __IOM uint32_t TMRB2IE1 : 1; /*!< [26..26] Counter/Timer B2 Interrupt Enable bit for COMPR1. */ + __IOM uint32_t TMRB2CLR : 1; /*!< [27..27] Counter/Timer B2 Clear bit. */ + __IOM uint32_t TMRB2POL : 1; /*!< [28..28] Counter/Timer B2 output polarity. */ + __IM uint32_t : 2; + __IOM uint32_t CTLINK2 : 1; /*!< [31..31] Counter/Timer A2/B2 Link bit. */ + } CTRL2_b; + } ; + __IM uint32_t RESERVED2; + + union { + __IOM uint32_t CMPRAUXA2; /*!< (@ 0x00000054) Counter/Timer A2 Compare Registers */ + + struct { + __IOM uint32_t CMPR2A2 : 16; /*!< [15..0] Counter/Timer A2 Compare Register 2. Holds the lower + limit for timer half A. */ + __IOM uint32_t CMPR3A2 : 16; /*!< [31..16] Counter/Timer A2 Compare Register 3. Holds the upper + limit for timer half A. */ + } CMPRAUXA2_b; + } ; + + union { + __IOM uint32_t CMPRAUXB2; /*!< (@ 0x00000058) Counter/Timer B2 Compare Registers */ + + struct { + __IOM uint32_t CMPR2B2 : 16; /*!< [15..0] Counter/Timer B2 Compare Register 2. Holds the lower + limit for timer half B. */ + __IOM uint32_t CMPR3B2 : 16; /*!< [31..16] Counter/Timer B2 Compare Register 3. Holds the upper + limit for timer half B. */ + } CMPRAUXB2_b; + } ; + + union { + __IOM uint32_t AUX2; /*!< (@ 0x0000005C) Counter/Timer Auxiliary */ + + struct { + __IOM uint32_t TMRA2LMT : 7; /*!< [6..0] Counter/Timer A2 Pattern Limit Count. */ + __IOM uint32_t TMRA2TRIG : 4; /*!< [10..7] Counter/Timer A2 Trigger Select. */ + __IOM uint32_t TMRA2NOSYNC : 1; /*!< [11..11] Source clock synchronization control. */ + __IOM uint32_t TMRA2TINV : 1; /*!< [12..12] Counter/Timer A2 Invert on trigger. */ + __IOM uint32_t TMRA2POL23 : 1; /*!< [13..13] Counter/Timer A2 Upper output polarity */ + __IOM uint32_t TMRA2EN23 : 1; /*!< [14..14] Counter/Timer A2 Upper compare enable. */ + __IM uint32_t : 1; + __IOM uint32_t TMRB2LMT : 6; /*!< [21..16] Counter/Timer B2 Pattern Limit Count. */ + __IM uint32_t : 1; + __IOM uint32_t TMRB2TRIG : 4; /*!< [26..23] Counter/Timer B2 Trigger Select. */ + __IOM uint32_t TMRB2NOSYNC : 1; /*!< [27..27] Source clock synchronization control. */ + __IOM uint32_t TMRB2TINV : 1; /*!< [28..28] Counter/Timer B2 Invert on trigger. */ + __IOM uint32_t TMRB2POL23 : 1; /*!< [29..29] Upper output polarity */ + __IOM uint32_t TMRB2EN23 : 1; /*!< [30..30] Counter/Timer B2 Upper compare enable. */ + } AUX2_b; + } ; + + union { + __IOM uint32_t TMR3; /*!< (@ 0x00000060) Counter/Timer Register */ + + struct { + __IOM uint32_t CTTMRA3 : 16; /*!< [15..0] Counter/Timer A3. */ + __IOM uint32_t CTTMRB3 : 16; /*!< [31..16] Counter/Timer B3. */ + } TMR3_b; + } ; + + union { + __IOM uint32_t CMPRA3; /*!< (@ 0x00000064) Counter/Timer A3 Compare Registers */ + + struct { + __IOM uint32_t CMPR0A3 : 16; /*!< [15..0] Counter/Timer A3 Compare Register 0. */ + __IOM uint32_t CMPR1A3 : 16; /*!< [31..16] Counter/Timer A3 Compare Register 1. */ + } CMPRA3_b; + } ; + + union { + __IOM uint32_t CMPRB3; /*!< (@ 0x00000068) Counter/Timer B3 Compare Registers */ + + struct { + __IOM uint32_t CMPR0B3 : 16; /*!< [15..0] Counter/Timer B3 Compare Register 0. */ + __IOM uint32_t CMPR1B3 : 16; /*!< [31..16] Counter/Timer B3 Compare Register 1. */ + } CMPRB3_b; + } ; + + union { + __IOM uint32_t CTRL3; /*!< (@ 0x0000006C) Counter/Timer Control */ + + struct { + __IOM uint32_t TMRA3EN : 1; /*!< [0..0] Counter/Timer A3 Enable bit. */ + __IOM uint32_t TMRA3CLK : 5; /*!< [5..1] Counter/Timer A3 Clock Select. */ + __IOM uint32_t TMRA3FN : 3; /*!< [8..6] Counter/Timer A3 Function Select. */ + __IOM uint32_t TMRA3IE0 : 1; /*!< [9..9] Counter/Timer A3 Interrupt Enable bit based on COMPR0. */ + __IOM uint32_t TMRA3IE1 : 1; /*!< [10..10] Counter/Timer A3 Interrupt Enable bit based on COMPR1. */ + __IOM uint32_t TMRA3CLR : 1; /*!< [11..11] Counter/Timer A3 Clear bit. */ + __IOM uint32_t TMRA3POL : 1; /*!< [12..12] Counter/Timer A3 output polarity. */ + __IM uint32_t : 2; + __IOM uint32_t ADCEN : 1; /*!< [15..15] Special Timer A3 enable for ADC function. */ + __IOM uint32_t TMRB3EN : 1; /*!< [16..16] Counter/Timer B3 Enable bit. */ + __IOM uint32_t TMRB3CLK : 5; /*!< [21..17] Counter/Timer B3 Clock Select. */ + __IOM uint32_t TMRB3FN : 3; /*!< [24..22] Counter/Timer B3 Function Select. */ + __IOM uint32_t TMRB3IE0 : 1; /*!< [25..25] Counter/Timer B3 Interrupt Enable bit for COMPR0. */ + __IOM uint32_t TMRB3IE1 : 1; /*!< [26..26] Counter/Timer B3 Interrupt Enable bit for COMPR1. */ + __IOM uint32_t TMRB3CLR : 1; /*!< [27..27] Counter/Timer B3 Clear bit. */ + __IOM uint32_t TMRB3POL : 1; /*!< [28..28] Counter/Timer B3 output polarity. */ + __IM uint32_t : 2; + __IOM uint32_t CTLINK3 : 1; /*!< [31..31] Counter/Timer A3/B3 Link bit. */ + } CTRL3_b; + } ; + __IM uint32_t RESERVED3; + + union { + __IOM uint32_t CMPRAUXA3; /*!< (@ 0x00000074) Counter/Timer A3 Compare Registers */ + + struct { + __IOM uint32_t CMPR2A3 : 16; /*!< [15..0] Counter/Timer A3 Compare Register 2. Holds the lower + limit for timer half A. */ + __IOM uint32_t CMPR3A3 : 16; /*!< [31..16] Counter/Timer A3 Compare Register 3. Holds the upper + limit for timer half A. */ + } CMPRAUXA3_b; + } ; + + union { + __IOM uint32_t CMPRAUXB3; /*!< (@ 0x00000078) Counter/Timer B3 Compare Registers */ + + struct { + __IOM uint32_t CMPR2B3 : 16; /*!< [15..0] Counter/Timer B3 Compare Register 2. Holds the lower + limit for timer half B. */ + __IOM uint32_t CMPR3B3 : 16; /*!< [31..16] Counter/Timer B3 Compare Register 3. Holds the upper + limit for timer half B. */ + } CMPRAUXB3_b; + } ; + + union { + __IOM uint32_t AUX3; /*!< (@ 0x0000007C) Counter/Timer Auxiliary */ + + struct { + __IOM uint32_t TMRA3LMT : 7; /*!< [6..0] Counter/Timer A3 Pattern Limit Count. */ + __IOM uint32_t TMRA3TRIG : 4; /*!< [10..7] Counter/Timer A3 Trigger Select. */ + __IOM uint32_t TMRA3NOSYNC : 1; /*!< [11..11] Source clock synchronization control. */ + __IOM uint32_t TMRA3TINV : 1; /*!< [12..12] Counter/Timer A3 Invert on trigger. */ + __IOM uint32_t TMRA3POL23 : 1; /*!< [13..13] Counter/Timer A3 Upper output polarity */ + __IOM uint32_t TMRA3EN23 : 1; /*!< [14..14] Counter/Timer A3 Upper compare enable. */ + __IM uint32_t : 1; + __IOM uint32_t TMRB3LMT : 6; /*!< [21..16] Counter/Timer B3 Pattern Limit Count. */ + __IM uint32_t : 1; + __IOM uint32_t TMRB3TRIG : 4; /*!< [26..23] Counter/Timer B3 Trigger Select. */ + __IOM uint32_t TMRB3NOSYNC : 1; /*!< [27..27] Source clock synchronization control. */ + __IOM uint32_t TMRB3TINV : 1; /*!< [28..28] Counter/Timer B3 Invert on trigger. */ + __IOM uint32_t TMRB3POL23 : 1; /*!< [29..29] Upper output polarity */ + __IOM uint32_t TMRB3EN23 : 1; /*!< [30..30] Counter/Timer B3 Upper compare enable. */ + } AUX3_b; + } ; + + union { + __IOM uint32_t TMR4; /*!< (@ 0x00000080) Counter/Timer Register */ + + struct { + __IOM uint32_t CTTMRA4 : 16; /*!< [15..0] Counter/Timer A4. */ + __IOM uint32_t CTTMRB4 : 16; /*!< [31..16] Counter/Timer B4. */ + } TMR4_b; + } ; + + union { + __IOM uint32_t CMPRA4; /*!< (@ 0x00000084) Counter/Timer A4 Compare Registers */ + + struct { + __IOM uint32_t CMPR0A4 : 16; /*!< [15..0] Counter/Timer A4 Compare Register 0. Holds the lower + limit for timer half A. */ + __IOM uint32_t CMPR1A4 : 16; /*!< [31..16] Counter/Timer A4 Compare Register 1. Holds the upper + limit for timer half A. */ + } CMPRA4_b; + } ; + + union { + __IOM uint32_t CMPRB4; /*!< (@ 0x00000088) Counter/Timer B4 Compare Registers */ + + struct { + __IOM uint32_t CMPR0B4 : 16; /*!< [15..0] Counter/Timer B4 Compare Register 0. Holds the lower + limit for timer half B. */ + __IOM uint32_t CMPR1B4 : 16; /*!< [31..16] Counter/Timer B4 Compare Register 1. Holds the upper + limit for timer half B. */ + } CMPRB4_b; + } ; + + union { + __IOM uint32_t CTRL4; /*!< (@ 0x0000008C) Counter/Timer Control */ + + struct { + __IOM uint32_t TMRA4EN : 1; /*!< [0..0] Counter/Timer A4 Enable bit. */ + __IOM uint32_t TMRA4CLK : 5; /*!< [5..1] Counter/Timer A4 Clock Select. */ + __IOM uint32_t TMRA4FN : 3; /*!< [8..6] Counter/Timer A4 Function Select. */ + __IOM uint32_t TMRA4IE0 : 1; /*!< [9..9] Counter/Timer A4 Interrupt Enable bit based on COMPR0. */ + __IOM uint32_t TMRA4IE1 : 1; /*!< [10..10] Counter/Timer A4 Interrupt Enable bit based on COMPR1. */ + __IOM uint32_t TMRA4CLR : 1; /*!< [11..11] Counter/Timer A4 Clear bit. */ + __IOM uint32_t TMRA4POL : 1; /*!< [12..12] Counter/Timer A4 output polarity. */ + __IM uint32_t : 3; + __IOM uint32_t TMRB4EN : 1; /*!< [16..16] Counter/Timer B4 Enable bit. */ + __IOM uint32_t TMRB4CLK : 5; /*!< [21..17] Counter/Timer B4 Clock Select. */ + __IOM uint32_t TMRB4FN : 3; /*!< [24..22] Counter/Timer B4 Function Select. */ + __IOM uint32_t TMRB4IE0 : 1; /*!< [25..25] Counter/Timer B4 Interrupt Enable bit for COMPR0. */ + __IOM uint32_t TMRB4IE1 : 1; /*!< [26..26] Counter/Timer B4 Interrupt Enable bit for COMPR1. */ + __IOM uint32_t TMRB4CLR : 1; /*!< [27..27] Counter/Timer B4 Clear bit. */ + __IOM uint32_t TMRB4POL : 1; /*!< [28..28] Counter/Timer B4 output polarity. */ + __IM uint32_t : 2; + __IOM uint32_t CTLINK4 : 1; /*!< [31..31] Counter/Timer A4/B4 Link bit. */ + } CTRL4_b; + } ; + __IM uint32_t RESERVED4; + + union { + __IOM uint32_t CMPRAUXA4; /*!< (@ 0x00000094) Counter/Timer A4 Compare Registers */ + + struct { + __IOM uint32_t CMPR2A4 : 16; /*!< [15..0] Counter/Timer A4 Compare Register 2. Holds the lower + limit for timer half A. */ + __IOM uint32_t CMPR3A4 : 16; /*!< [31..16] Counter/Timer A4 Compare Register 3. Holds the upper + limit for timer half A. */ + } CMPRAUXA4_b; + } ; + + union { + __IOM uint32_t CMPRAUXB4; /*!< (@ 0x00000098) Counter/Timer B4 Compare Registers */ + + struct { + __IOM uint32_t CMPR2B4 : 16; /*!< [15..0] Counter/Timer B4 Compare Register 2. Holds the lower + limit for timer half B. */ + __IOM uint32_t CMPR3B4 : 16; /*!< [31..16] Counter/Timer B4 Compare Register 3. Holds the upper + limit for timer half B. */ + } CMPRAUXB4_b; + } ; + + union { + __IOM uint32_t AUX4; /*!< (@ 0x0000009C) Counter/Timer Auxiliary */ + + struct { + __IOM uint32_t TMRA4LMT : 7; /*!< [6..0] Counter/Timer A4 Pattern Limit Count. */ + __IOM uint32_t TMRA4TRIG : 4; /*!< [10..7] Counter/Timer A4 Trigger Select. */ + __IOM uint32_t TMRA4NOSYNC : 1; /*!< [11..11] Source clock synchronization control. */ + __IOM uint32_t TMRA4TINV : 1; /*!< [12..12] Counter/Timer A4 Invert on trigger. */ + __IOM uint32_t TMRA4POL23 : 1; /*!< [13..13] Counter/Timer A4 Upper output polarity */ + __IOM uint32_t TMRA4EN23 : 1; /*!< [14..14] Counter/Timer A4 Upper compare enable. */ + __IM uint32_t : 1; + __IOM uint32_t TMRB4LMT : 6; /*!< [21..16] Counter/Timer B4 Pattern Limit Count. */ + __IM uint32_t : 1; + __IOM uint32_t TMRB4TRIG : 4; /*!< [26..23] Counter/Timer B4 Trigger Select. */ + __IOM uint32_t TMRB4NOSYNC : 1; /*!< [27..27] Source clock synchronization control. */ + __IOM uint32_t TMRB4TINV : 1; /*!< [28..28] Counter/Timer B4 Invert on trigger. */ + __IOM uint32_t TMRB4POL23 : 1; /*!< [29..29] Upper output polarity */ + __IOM uint32_t TMRB4EN23 : 1; /*!< [30..30] Counter/Timer B4 Upper compare enable. */ + } AUX4_b; + } ; + + union { + __IOM uint32_t TMR5; /*!< (@ 0x000000A0) Counter/Timer Register */ + + struct { + __IOM uint32_t CTTMRA5 : 16; /*!< [15..0] Counter/Timer A5. */ + __IOM uint32_t CTTMRB5 : 16; /*!< [31..16] Counter/Timer B5. */ + } TMR5_b; + } ; + + union { + __IOM uint32_t CMPRA5; /*!< (@ 0x000000A4) Counter/Timer A5 Compare Registers */ + + struct { + __IOM uint32_t CMPR0A5 : 16; /*!< [15..0] Counter/Timer A5 Compare Register 0. */ + __IOM uint32_t CMPR1A5 : 16; /*!< [31..16] Counter/Timer A5 Compare Register 1. */ + } CMPRA5_b; + } ; + + union { + __IOM uint32_t CMPRB5; /*!< (@ 0x000000A8) Counter/Timer B5 Compare Registers */ + + struct { + __IOM uint32_t CMPR0B5 : 16; /*!< [15..0] Counter/Timer B5 Compare Register 0. */ + __IOM uint32_t CMPR1B5 : 16; /*!< [31..16] Counter/Timer B5 Compare Register 1. */ + } CMPRB5_b; + } ; + + union { + __IOM uint32_t CTRL5; /*!< (@ 0x000000AC) Counter/Timer Control */ + + struct { + __IOM uint32_t TMRA5EN : 1; /*!< [0..0] Counter/Timer A5 Enable bit. */ + __IOM uint32_t TMRA5CLK : 5; /*!< [5..1] Counter/Timer A5 Clock Select. */ + __IOM uint32_t TMRA5FN : 3; /*!< [8..6] Counter/Timer A5 Function Select. */ + __IOM uint32_t TMRA5IE0 : 1; /*!< [9..9] Counter/Timer A5 Interrupt Enable bit based on COMPR0. */ + __IOM uint32_t TMRA5IE1 : 1; /*!< [10..10] Counter/Timer A5 Interrupt Enable bit based on COMPR1. */ + __IOM uint32_t TMRA5CLR : 1; /*!< [11..11] Counter/Timer A5 Clear bit. */ + __IOM uint32_t TMRA5POL : 1; /*!< [12..12] Counter/Timer A5 output polarity. */ + __IM uint32_t : 3; + __IOM uint32_t TMRB5EN : 1; /*!< [16..16] Counter/Timer B5 Enable bit. */ + __IOM uint32_t TMRB5CLK : 5; /*!< [21..17] Counter/Timer B5 Clock Select. */ + __IOM uint32_t TMRB5FN : 3; /*!< [24..22] Counter/Timer B5 Function Select. */ + __IOM uint32_t TMRB5IE0 : 1; /*!< [25..25] Counter/Timer B5 Interrupt Enable bit for COMPR0. */ + __IOM uint32_t TMRB5IE1 : 1; /*!< [26..26] Counter/Timer B5 Interrupt Enable bit for COMPR1. */ + __IOM uint32_t TMRB5CLR : 1; /*!< [27..27] Counter/Timer B5 Clear bit. */ + __IOM uint32_t TMRB5POL : 1; /*!< [28..28] Counter/Timer B5 output polarity. */ + __IM uint32_t : 2; + __IOM uint32_t CTLINK5 : 1; /*!< [31..31] Counter/Timer A5/B5 Link bit. */ + } CTRL5_b; + } ; + __IM uint32_t RESERVED5; + + union { + __IOM uint32_t CMPRAUXA5; /*!< (@ 0x000000B4) Counter/Timer A5 Compare Registers */ + + struct { + __IOM uint32_t CMPR2A5 : 16; /*!< [15..0] Counter/Timer A5 Compare Register 2. Holds the lower + limit for timer half A. */ + __IOM uint32_t CMPR3A5 : 16; /*!< [31..16] Counter/Timer A5 Compare Register 3. Holds the upper + limit for timer half A. */ + } CMPRAUXA5_b; + } ; + + union { + __IOM uint32_t CMPRAUXB5; /*!< (@ 0x000000B8) Counter/Timer B5 Compare Registers */ + + struct { + __IOM uint32_t CMPR2B5 : 16; /*!< [15..0] Counter/Timer B5 Compare Register 2. Holds the lower + limit for timer half B. */ + __IOM uint32_t CMPR3B5 : 16; /*!< [31..16] Counter/Timer B5 Compare Register 3. Holds the upper + limit for timer half B. */ + } CMPRAUXB5_b; + } ; + + union { + __IOM uint32_t AUX5; /*!< (@ 0x000000BC) Counter/Timer Auxiliary */ + + struct { + __IOM uint32_t TMRA5LMT : 7; /*!< [6..0] Counter/Timer A5 Pattern Limit Count. */ + __IOM uint32_t TMRA5TRIG : 4; /*!< [10..7] Counter/Timer A5 Trigger Select. */ + __IOM uint32_t TMRA5NOSYNC : 1; /*!< [11..11] Source clock synchronization control. */ + __IOM uint32_t TMRA5TINV : 1; /*!< [12..12] Counter/Timer A5 Invert on trigger. */ + __IOM uint32_t TMRA5POL23 : 1; /*!< [13..13] Counter/Timer A5 Upper output polarity */ + __IOM uint32_t TMRA5EN23 : 1; /*!< [14..14] Counter/Timer A5 Upper compare enable. */ + __IM uint32_t : 1; + __IOM uint32_t TMRB5LMT : 6; /*!< [21..16] Counter/Timer B5 Pattern Limit Count. */ + __IM uint32_t : 1; + __IOM uint32_t TMRB5TRIG : 4; /*!< [26..23] Counter/Timer B5 Trigger Select. */ + __IOM uint32_t TMRB5NOSYNC : 1; /*!< [27..27] Source clock synchronization control. */ + __IOM uint32_t TMRB5TINV : 1; /*!< [28..28] Counter/Timer B5 Invert on trigger. */ + __IOM uint32_t TMRB5POL23 : 1; /*!< [29..29] Upper output polarity */ + __IOM uint32_t TMRB5EN23 : 1; /*!< [30..30] Counter/Timer B5 Upper compare enable. */ + } AUX5_b; + } ; + + union { + __IOM uint32_t TMR6; /*!< (@ 0x000000C0) Counter/Timer Register */ + + struct { + __IOM uint32_t CTTMRA6 : 16; /*!< [15..0] Counter/Timer A6. */ + __IOM uint32_t CTTMRB6 : 16; /*!< [31..16] Counter/Timer B6. */ + } TMR6_b; + } ; + + union { + __IOM uint32_t CMPRA6; /*!< (@ 0x000000C4) Counter/Timer A6 Compare Registers */ + + struct { + __IOM uint32_t CMPR0A6 : 16; /*!< [15..0] Counter/Timer A6 Compare Register 0. */ + __IOM uint32_t CMPR1A6 : 16; /*!< [31..16] Counter/Timer A6 Compare Register 1. */ + } CMPRA6_b; + } ; + + union { + __IOM uint32_t CMPRB6; /*!< (@ 0x000000C8) Counter/Timer B6 Compare Registers */ + + struct { + __IOM uint32_t CMPR0B6 : 16; /*!< [15..0] Counter/Timer B6 Compare Register 0. */ + __IOM uint32_t CMPR1B6 : 16; /*!< [31..16] Counter/Timer B6 Compare Register 1. */ + } CMPRB6_b; + } ; + + union { + __IOM uint32_t CTRL6; /*!< (@ 0x000000CC) Counter/Timer Control */ + + struct { + __IOM uint32_t TMRA6EN : 1; /*!< [0..0] Counter/Timer A6 Enable bit. */ + __IOM uint32_t TMRA6CLK : 5; /*!< [5..1] Counter/Timer A6 Clock Select. */ + __IOM uint32_t TMRA6FN : 3; /*!< [8..6] Counter/Timer A6 Function Select. */ + __IOM uint32_t TMRA6IE0 : 1; /*!< [9..9] Counter/Timer A6 Interrupt Enable bit based on COMPR0. */ + __IOM uint32_t TMRA6IE1 : 1; /*!< [10..10] Counter/Timer A6 Interrupt Enable bit based on COMPR1. */ + __IOM uint32_t TMRA6CLR : 1; /*!< [11..11] Counter/Timer A6 Clear bit. */ + __IOM uint32_t TMRA6POL : 1; /*!< [12..12] Counter/Timer A6 output polarity. */ + __IM uint32_t : 3; + __IOM uint32_t TMRB6EN : 1; /*!< [16..16] Counter/Timer B6 Enable bit. */ + __IOM uint32_t TMRB6CLK : 5; /*!< [21..17] Counter/Timer B6 Clock Select. */ + __IOM uint32_t TMRB6FN : 3; /*!< [24..22] Counter/Timer B6 Function Select. */ + __IOM uint32_t TMRB6IE0 : 1; /*!< [25..25] Counter/Timer B6 Interrupt Enable bit for COMPR0. */ + __IOM uint32_t TMRB6IE1 : 1; /*!< [26..26] Counter/Timer B6 Interrupt Enable bit for COMPR1. */ + __IOM uint32_t TMRB6CLR : 1; /*!< [27..27] Counter/Timer B6 Clear bit. */ + __IOM uint32_t TMRB6POL : 1; /*!< [28..28] Counter/Timer B6 output polarity. */ + __IM uint32_t : 2; + __IOM uint32_t CTLINK6 : 1; /*!< [31..31] Counter/Timer A6/B6 Link bit. */ + } CTRL6_b; + } ; + __IM uint32_t RESERVED6; + + union { + __IOM uint32_t CMPRAUXA6; /*!< (@ 0x000000D4) Counter/Timer A6 Compare Registers */ + + struct { + __IOM uint32_t CMPR2A6 : 16; /*!< [15..0] Counter/Timer A6 Compare Register 2. Holds the lower + limit for timer half A. */ + __IOM uint32_t CMPR3A6 : 16; /*!< [31..16] Counter/Timer A6 Compare Register 3. Holds the upper + limit for timer half A. */ + } CMPRAUXA6_b; + } ; + + union { + __IOM uint32_t CMPRAUXB6; /*!< (@ 0x000000D8) Counter/Timer B6 Compare Registers */ + + struct { + __IOM uint32_t CMPR2B6 : 16; /*!< [15..0] Counter/Timer B6 Compare Register 2. Holds the lower + limit for timer half B. */ + __IOM uint32_t CMPR3B6 : 16; /*!< [31..16] Counter/Timer B6 Compare Register 3. Holds the upper + limit for timer half B. */ + } CMPRAUXB6_b; + } ; + + union { + __IOM uint32_t AUX6; /*!< (@ 0x000000DC) Counter/Timer Auxiliary */ + + struct { + __IOM uint32_t TMRA6LMT : 7; /*!< [6..0] Counter/Timer A6 Pattern Limit Count. */ + __IOM uint32_t TMRA6TRIG : 4; /*!< [10..7] Counter/Timer A6 Trigger Select. */ + __IOM uint32_t TMRA6NOSYNC : 1; /*!< [11..11] Source clock synchronization control. */ + __IOM uint32_t TMRA6TINV : 1; /*!< [12..12] Counter/Timer A6 Invert on trigger. */ + __IOM uint32_t TMRA6POL23 : 1; /*!< [13..13] Counter/Timer A6 Upper output polarity */ + __IOM uint32_t TMRA6EN23 : 1; /*!< [14..14] Counter/Timer A6 Upper compare enable. */ + __IM uint32_t : 1; + __IOM uint32_t TMRB6LMT : 6; /*!< [21..16] Counter/Timer B6 Pattern Limit Count. */ + __IM uint32_t : 1; + __IOM uint32_t TMRB6TRIG : 4; /*!< [26..23] Counter/Timer B6 Trigger Select. */ + __IOM uint32_t TMRB6NOSYNC : 1; /*!< [27..27] Source clock synchronization control. */ + __IOM uint32_t TMRB6TINV : 1; /*!< [28..28] Counter/Timer B6 Invert on trigger. */ + __IOM uint32_t TMRB6POL23 : 1; /*!< [29..29] Upper output polarity */ + __IOM uint32_t TMRB6EN23 : 1; /*!< [30..30] Counter/Timer B6 Upper compare enable. */ + } AUX6_b; + } ; + + union { + __IOM uint32_t TMR7; /*!< (@ 0x000000E0) Counter/Timer Register */ + + struct { + __IOM uint32_t CTTMRA7 : 16; /*!< [15..0] Counter/Timer A7. */ + __IOM uint32_t CTTMRB7 : 16; /*!< [31..16] Counter/Timer B7. */ + } TMR7_b; + } ; + + union { + __IOM uint32_t CMPRA7; /*!< (@ 0x000000E4) Counter/Timer A7 Compare Registers */ + + struct { + __IOM uint32_t CMPR0A7 : 16; /*!< [15..0] Counter/Timer A7 Compare Register 0. */ + __IOM uint32_t CMPR1A7 : 16; /*!< [31..16] Counter/Timer A7 Compare Register 1. */ + } CMPRA7_b; + } ; + + union { + __IOM uint32_t CMPRB7; /*!< (@ 0x000000E8) Counter/Timer B7 Compare Registers */ + + struct { + __IOM uint32_t CMPR0B7 : 16; /*!< [15..0] Counter/Timer B3 Compare Register 0. */ + __IOM uint32_t CMPR1B7 : 16; /*!< [31..16] Counter/Timer B3 Compare Register 1. */ + } CMPRB7_b; + } ; + + union { + __IOM uint32_t CTRL7; /*!< (@ 0x000000EC) Counter/Timer Control */ + + struct { + __IOM uint32_t TMRA7EN : 1; /*!< [0..0] Counter/Timer A7 Enable bit. */ + __IOM uint32_t TMRA7CLK : 5; /*!< [5..1] Counter/Timer A7 Clock Select. */ + __IOM uint32_t TMRA7FN : 3; /*!< [8..6] Counter/Timer A7 Function Select. */ + __IOM uint32_t TMRA7IE0 : 1; /*!< [9..9] Counter/Timer A7 Interrupt Enable bit based on COMPR0. */ + __IOM uint32_t TMRA7IE1 : 1; /*!< [10..10] Counter/Timer A7 Interrupt Enable bit based on COMPR1. */ + __IOM uint32_t TMRA7CLR : 1; /*!< [11..11] Counter/Timer A7 Clear bit. */ + __IOM uint32_t TMRA7POL : 1; /*!< [12..12] Counter/Timer A7 output polarity. */ + __IM uint32_t : 3; + __IOM uint32_t TMRB7EN : 1; /*!< [16..16] Counter/Timer B7 Enable bit. */ + __IOM uint32_t TMRB7CLK : 5; /*!< [21..17] Counter/Timer B7 Clock Select. */ + __IOM uint32_t TMRB7FN : 3; /*!< [24..22] Counter/Timer B7 Function Select. */ + __IOM uint32_t TMRB7IE0 : 1; /*!< [25..25] Counter/Timer B7 Interrupt Enable bit for COMPR0. */ + __IOM uint32_t TMRB7IE1 : 1; /*!< [26..26] Counter/Timer B7 Interrupt Enable bit for COMPR1. */ + __IOM uint32_t TMRB7CLR : 1; /*!< [27..27] Counter/Timer B7 Clear bit. */ + __IOM uint32_t TMRB7POL : 1; /*!< [28..28] Counter/Timer B7 output polarity. */ + __IM uint32_t : 2; + __IOM uint32_t CTLINK7 : 1; /*!< [31..31] Counter/Timer A7/B7 Link bit. */ + } CTRL7_b; + } ; + __IM uint32_t RESERVED7; + + union { + __IOM uint32_t CMPRAUXA7; /*!< (@ 0x000000F4) Counter/Timer A7 Compare Registers */ + + struct { + __IOM uint32_t CMPR2A7 : 16; /*!< [15..0] Counter/Timer A7 Compare Register 2. Holds the lower + limit for timer half A. */ + __IOM uint32_t CMPR3A7 : 16; /*!< [31..16] Counter/Timer A7 Compare Register 3. Holds the upper + limit for timer half A. */ + } CMPRAUXA7_b; + } ; + + union { + __IOM uint32_t CMPRAUXB7; /*!< (@ 0x000000F8) Counter/Timer B7 Compare Registers */ + + struct { + __IOM uint32_t CMPR2B7 : 16; /*!< [15..0] Counter/Timer B7 Compare Register 2. Holds the lower + limit for timer half B. */ + __IOM uint32_t CMPR3B7 : 16; /*!< [31..16] Counter/Timer B7 Compare Register 3. Holds the upper + limit for timer half B. */ + } CMPRAUXB7_b; + } ; + + union { + __IOM uint32_t AUX7; /*!< (@ 0x000000FC) Counter/Timer Auxiliary */ + + struct { + __IOM uint32_t TMRA7LMT : 7; /*!< [6..0] Counter/Timer A7 Pattern Limit Count. */ + __IOM uint32_t TMRA7TRIG : 4; /*!< [10..7] Counter/Timer A7 Trigger Select. */ + __IOM uint32_t TMRA7NOSYNC : 1; /*!< [11..11] Source clock synchronization control. */ + __IOM uint32_t TMRA7TINV : 1; /*!< [12..12] Counter/Timer A7 Invert on trigger. */ + __IOM uint32_t TMRA7POL23 : 1; /*!< [13..13] Counter/Timer A7 Upper output polarity */ + __IOM uint32_t TMRA7EN23 : 1; /*!< [14..14] Counter/Timer A7 Upper compare enable. */ + __IM uint32_t : 1; + __IOM uint32_t TMRB7LMT : 6; /*!< [21..16] Counter/Timer B7 Pattern Limit Count. */ + __IM uint32_t : 1; + __IOM uint32_t TMRB7TRIG : 4; /*!< [26..23] Counter/Timer B7 Trigger Select. */ + __IOM uint32_t TMRB7NOSYNC : 1; /*!< [27..27] Source clock synchronization control. */ + __IOM uint32_t TMRB7TINV : 1; /*!< [28..28] Counter/Timer B7 Invert on trigger. */ + __IOM uint32_t TMRB7POL23 : 1; /*!< [29..29] Upper output polarity */ + __IOM uint32_t TMRB7EN23 : 1; /*!< [30..30] Counter/Timer B7 Upper compare enable. */ + } AUX7_b; + } ; + + union { + __IOM uint32_t GLOBEN; /*!< (@ 0x00000100) Counter/Timer Global Enable */ + + struct { + __IOM uint32_t ENA0 : 1; /*!< [0..0] Alternate enable for A0 */ + __IOM uint32_t ENB0 : 1; /*!< [1..1] Alternate enable for B0 */ + __IOM uint32_t ENA1 : 1; /*!< [2..2] Alternate enable for A1 */ + __IOM uint32_t ENB1 : 1; /*!< [3..3] Alternate enable for B1 */ + __IOM uint32_t ENA2 : 1; /*!< [4..4] Alternate enable for A2 */ + __IOM uint32_t ENB2 : 1; /*!< [5..5] Alternate enable for B2 */ + __IOM uint32_t ENA3 : 1; /*!< [6..6] Alternate enable for A3 */ + __IOM uint32_t ENB3 : 1; /*!< [7..7] Alternate enable for B3. */ + __IOM uint32_t ENA4 : 1; /*!< [8..8] Alternate enable for A4 */ + __IOM uint32_t ENB4 : 1; /*!< [9..9] Alternate enable for B4 */ + __IOM uint32_t ENA5 : 1; /*!< [10..10] Alternate enable for A5 */ + __IOM uint32_t ENB5 : 1; /*!< [11..11] Alternate enable for B5 */ + __IOM uint32_t ENA6 : 1; /*!< [12..12] Alternate enable for A6 */ + __IOM uint32_t ENB6 : 1; /*!< [13..13] Alternate enable for B6 */ + __IOM uint32_t ENA7 : 1; /*!< [14..14] Alternate enable for A7 */ + __IOM uint32_t ENB7 : 1; /*!< [15..15] Alternate enable for B7. */ + } GLOBEN_b; + } ; + + union { + __IOM uint32_t OUTCFG0; /*!< (@ 0x00000104) Counter/Timer Output Config 0 */ + + struct { + __IOM uint32_t CFG0 : 3; /*!< [2..0] Pad output 0 configuration */ + __IOM uint32_t CFG1 : 3; /*!< [5..3] Pad output 1 configuration */ + __IOM uint32_t CFG2 : 3; /*!< [8..6] Pad output 2 configuration */ + __IOM uint32_t CFG3 : 3; /*!< [11..9] Pad output 3 configuration */ + __IOM uint32_t CFG4 : 3; /*!< [14..12] Pad output 4 configuration */ + __IM uint32_t : 1; + __IOM uint32_t CFG5 : 3; /*!< [18..16] Pad output 5 configuration */ + __IOM uint32_t CFG6 : 3; /*!< [21..19] Pad output 6 configuration */ + __IOM uint32_t CFG7 : 3; /*!< [24..22] Pad output 7 configuration */ + __IOM uint32_t CFG8 : 3; /*!< [27..25] Pad output 8 configuration */ + __IOM uint32_t CFG9 : 3; /*!< [30..28] Pad output 9 configuration */ + } OUTCFG0_b; + } ; + + union { + __IOM uint32_t OUTCFG1; /*!< (@ 0x00000108) Counter/Timer Output Config 1 */ + + struct { + __IOM uint32_t CFG10 : 3; /*!< [2..0] Pad output 10 configuration */ + __IOM uint32_t CFG11 : 3; /*!< [5..3] Pad output 11 configuration */ + __IOM uint32_t CFG12 : 3; /*!< [8..6] Pad output 12 configuration */ + __IOM uint32_t CFG13 : 3; /*!< [11..9] Pad output 13 configuration */ + __IOM uint32_t CFG14 : 3; /*!< [14..12] Pad output 14 configuration */ + __IM uint32_t : 1; + __IOM uint32_t CFG15 : 3; /*!< [18..16] Pad output 15 configuration */ + __IOM uint32_t CFG16 : 3; /*!< [21..19] Pad output 16 configuration */ + __IOM uint32_t CFG17 : 3; /*!< [24..22] Pad output 17 configuration */ + __IOM uint32_t CFG18 : 3; /*!< [27..25] Pad output 18 configuration */ + __IOM uint32_t CFG19 : 3; /*!< [30..28] Pad output 19 configuration */ + } OUTCFG1_b; + } ; + + union { + __IOM uint32_t OUTCFG2; /*!< (@ 0x0000010C) Counter/Timer Output Config 2 */ + + struct { + __IOM uint32_t CFG20 : 3; /*!< [2..0] Pad output 20 configuration */ + __IOM uint32_t CFG21 : 3; /*!< [5..3] Pad output 21 configuration */ + __IOM uint32_t CFG22 : 3; /*!< [8..6] Pad output 22 configuration */ + __IOM uint32_t CFG23 : 3; /*!< [11..9] Pad output 23 configuration */ + __IOM uint32_t CFG24 : 3; /*!< [14..12] Pad output 24 configuration */ + __IM uint32_t : 1; + __IOM uint32_t CFG25 : 3; /*!< [18..16] Pad output 25 configuration */ + __IOM uint32_t CFG26 : 3; /*!< [21..19] Pad output 26 configuration */ + __IOM uint32_t CFG27 : 3; /*!< [24..22] Pad output 27 configuration */ + __IOM uint32_t CFG28 : 3; /*!< [27..25] Pad output 28 configuration */ + __IOM uint32_t CFG29 : 3; /*!< [30..28] Pad output 29 configuration */ + } OUTCFG2_b; + } ; + __IM uint32_t RESERVED8; + + union { + __IOM uint32_t OUTCFG3; /*!< (@ 0x00000114) Counter/Timer Output Config 3 */ + + struct { + __IOM uint32_t CFG30 : 3; /*!< [2..0] Pad output 30 configuration */ + __IOM uint32_t CFG31 : 3; /*!< [5..3] Pad output 31 configuration */ + } OUTCFG3_b; + } ; + + union { + __IOM uint32_t INCFG; /*!< (@ 0x00000118) Counter/Timer Input Config */ + + struct { + __IOM uint32_t CFGA0 : 1; /*!< [0..0] CTIMER A0 input configuration */ + __IOM uint32_t CFGB0 : 1; /*!< [1..1] CTIMER B0 input configuration */ + __IOM uint32_t CFGA1 : 1; /*!< [2..2] CTIMER A1 input configuration */ + __IOM uint32_t CFGB1 : 1; /*!< [3..3] CTIMER B1 input configuration */ + __IOM uint32_t CFGA2 : 1; /*!< [4..4] CTIMER A2 input configuration */ + __IOM uint32_t CFGB2 : 1; /*!< [5..5] CTIMER B2 input configuration */ + __IOM uint32_t CFGA3 : 1; /*!< [6..6] CTIMER A3 input configuration */ + __IOM uint32_t CFGB3 : 1; /*!< [7..7] CTIMER B3 input configuration */ + __IOM uint32_t CFGA4 : 1; /*!< [8..8] CTIMER A4 input configuration */ + __IOM uint32_t CFGB4 : 1; /*!< [9..9] CTIMER B4 input configuration */ + __IOM uint32_t CFGA5 : 1; /*!< [10..10] CTIMER A5 input configuration */ + __IOM uint32_t CFGB5 : 1; /*!< [11..11] CTIMER B5 input configuration */ + __IOM uint32_t CFGA6 : 1; /*!< [12..12] CTIMER A6 input configuration */ + __IOM uint32_t CFGB6 : 1; /*!< [13..13] CTIMER B6 input configuration */ + __IOM uint32_t CFGA7 : 1; /*!< [14..14] CTIMER A7 input configuration */ + __IOM uint32_t CFGB7 : 1; /*!< [15..15] CTIMER B7 input configuration */ + } INCFG_b; + } ; + __IM uint32_t RESERVED9[9]; + + union { + __IOM uint32_t STCFG; /*!< (@ 0x00000140) Configuration Register */ + + struct { + __IOM uint32_t CLKSEL : 4; /*!< [3..0] Selects an appropriate clock source and divider to use + for the System Timer clock. */ + __IM uint32_t : 4; + __IOM uint32_t COMPARE_A_EN : 1; /*!< [8..8] Selects whether compare is enabled for the corresponding + SCMPR register. If compare is enabled, the interrupt status + is set once the comparision is met. */ + __IOM uint32_t COMPARE_B_EN : 1; /*!< [9..9] Selects whether compare is enabled for the corresponding + SCMPR register. If compare is enabled, the interrupt status + is set once the comparision is met. */ + __IOM uint32_t COMPARE_C_EN : 1; /*!< [10..10] Selects whether compare is enabled for the corresponding + SCMPR register. If compare is enabled, the interrupt status + is set once the comparision is met. */ + __IOM uint32_t COMPARE_D_EN : 1; /*!< [11..11] Selects whether compare is enabled for the corresponding + SCMPR register. If compare is enabled, the interrupt status + is set once the comparision is met. */ + __IOM uint32_t COMPARE_E_EN : 1; /*!< [12..12] Selects whether compare is enabled for the corresponding + SCMPR register. If compare is enabled, the interrupt status + is set once the comparision is met. */ + __IOM uint32_t COMPARE_F_EN : 1; /*!< [13..13] Selects whether compare is enabled for the corresponding + SCMPR register. If compare is enabled, the interrupt status + is set once the comparision is met. */ + __IOM uint32_t COMPARE_G_EN : 1; /*!< [14..14] Selects whether compare is enabled for the corresponding + SCMPR register. If compare is enabled, the interrupt status + is set once the comparision is met. */ + __IOM uint32_t COMPARE_H_EN : 1; /*!< [15..15] Selects whether compare is enabled for the corresponding + SCMPR register. If compare is enabled, the interrupt status + is set once the comparision is met. */ + __IM uint32_t : 14; + __IOM uint32_t CLEAR : 1; /*!< [30..30] Set this bit to one to clear the System Timer register. + If this bit is set to '1', the system timer register will + stay cleared. It needs to be set to '0' for the system + timer to start running. */ + __IOM uint32_t FREEZE : 1; /*!< [31..31] Set this bit to one to freeze the clock input to the + COUNTER register. Once frozen, the value can be safely + written from the MCU. Unfreeze to resume. */ + } STCFG_b; + } ; + + union { + __IOM uint32_t STTMR; /*!< (@ 0x00000144) System Timer Count Register (Real Time Counter) */ + + struct { + __IOM uint32_t STTMR : 32; /*!< [31..0] Value of the 32-bit counter as it ticks over. */ + } STTMR_b; + } ; + + union { + __IOM uint32_t CAPTURECONTROL; /*!< (@ 0x00000148) Capture Control Register */ + + struct { + __IOM uint32_t CAPTURE0 : 1; /*!< [0..0] Selects whether capture is enabled for the specified + capture register. */ + __IOM uint32_t CAPTURE1 : 1; /*!< [1..1] Selects whether capture is enabled for the specified + capture register. */ + __IOM uint32_t CAPTURE2 : 1; /*!< [2..2] Selects whether capture is enabled for the specified + capture register. */ + __IOM uint32_t CAPTURE3 : 1; /*!< [3..3] Selects whether capture is enabled for the specified + capture register. */ + } CAPTURECONTROL_b; + } ; + __IM uint32_t RESERVED10; + + union { + __IOM uint32_t SCMPR0; /*!< (@ 0x00000150) Compare Register A */ + + struct { + __IOM uint32_t SCMPR0 : 32; /*!< [31..0] Compare this value to the value in the COUNTER register + according to the match criterion, as selected in the COMPARE_A_EN + bit in the REG_CTIMER_STCGF register. */ + } SCMPR0_b; + } ; + + union { + __IOM uint32_t SCMPR1; /*!< (@ 0x00000154) Compare Register B */ + + struct { + __IOM uint32_t SCMPR1 : 32; /*!< [31..0] Compare this value to the value in the COUNTER register + according to the match criterion, as selected in the COMPARE_B_EN + bit in the REG_CTIMER_STCGF register. */ + } SCMPR1_b; + } ; + + union { + __IOM uint32_t SCMPR2; /*!< (@ 0x00000158) Compare Register C */ + + struct { + __IOM uint32_t SCMPR2 : 32; /*!< [31..0] Compare this value to the value in the COUNTER register + according to the match criterion, as selected in the COMPARE_C_EN + bit in the REG_CTIMER_STCGF register. */ + } SCMPR2_b; + } ; + + union { + __IOM uint32_t SCMPR3; /*!< (@ 0x0000015C) Compare Register D */ + + struct { + __IOM uint32_t SCMPR3 : 32; /*!< [31..0] Compare this value to the value in the COUNTER register + according to the match criterion, as selected in the COMPARE_D_EN + bit in the REG_CTIMER_STCGF register. */ + } SCMPR3_b; + } ; + + union { + __IOM uint32_t SCMPR4; /*!< (@ 0x00000160) Compare Register E */ + + struct { + __IOM uint32_t SCMPR4 : 32; /*!< [31..0] Compare this value to the value in the COUNTER register + according to the match criterion, as selected in the COMPARE_E_EN + bit in the REG_CTIMER_STCGF register. */ + } SCMPR4_b; + } ; + + union { + __IOM uint32_t SCMPR5; /*!< (@ 0x00000164) Compare Register F */ + + struct { + __IOM uint32_t SCMPR5 : 32; /*!< [31..0] Compare this value to the value in the COUNTER register + according to the match criterion, as selected in the COMPARE_F_EN + bit in the REG_CTIMER_STCGF register. */ + } SCMPR5_b; + } ; + + union { + __IOM uint32_t SCMPR6; /*!< (@ 0x00000168) Compare Register G */ + + struct { + __IOM uint32_t SCMPR6 : 32; /*!< [31..0] Compare this value to the value in the COUNTER register + according to the match criterion, as selected in the COMPARE_G_EN + bit in the REG_CTIMER_STCGF register. */ + } SCMPR6_b; + } ; + + union { + __IOM uint32_t SCMPR7; /*!< (@ 0x0000016C) Compare Register H */ + + struct { + __IOM uint32_t SCMPR7 : 32; /*!< [31..0] Compare this value to the value in the COUNTER register + according to the match criterion, as selected in the COMPARE_H_EN + bit in the REG_CTIMER_STCGF register. */ + } SCMPR7_b; + } ; + __IM uint32_t RESERVED11[28]; + + union { + __IOM uint32_t SCAPT0; /*!< (@ 0x000001E0) Capture Register A */ + + struct { + __IOM uint32_t SCAPT0 : 32; /*!< [31..0] Whenever the event is detected, the value in the COUNTER + is copied into this register and the corresponding interrupt + status bit is set. */ + } SCAPT0_b; + } ; + + union { + __IOM uint32_t SCAPT1; /*!< (@ 0x000001E4) Capture Register B */ + + struct { + __IOM uint32_t SCAPT1 : 32; /*!< [31..0] Whenever the event is detected, the value in the COUNTER + is copied into this register and the corresponding interrupt + status bit is set. */ + } SCAPT1_b; + } ; + + union { + __IOM uint32_t SCAPT2; /*!< (@ 0x000001E8) Capture Register C */ + + struct { + __IOM uint32_t SCAPT2 : 32; /*!< [31..0] Whenever the event is detected, the value in the COUNTER + is copied into this register and the corresponding interrupt + status bit is set. */ + } SCAPT2_b; + } ; + + union { + __IOM uint32_t SCAPT3; /*!< (@ 0x000001EC) Capture Register D */ + + struct { + __IOM uint32_t SCAPT3 : 32; /*!< [31..0] Whenever the event is detected, the value in the COUNTER + is copied into this register and the corresponding interrupt + status bit is set. */ + } SCAPT3_b; + } ; + + union { + __IOM uint32_t SNVR0; /*!< (@ 0x000001F0) System Timer NVRAM_A Register */ + + struct { + __IOM uint32_t SNVR0 : 32; /*!< [31..0] Value of the 32-bit counter as it ticks over. */ + } SNVR0_b; + } ; + + union { + __IOM uint32_t SNVR1; /*!< (@ 0x000001F4) System Timer NVRAM_B Register */ + + struct { + __IOM uint32_t SNVR1 : 32; /*!< [31..0] Value of the 32-bit counter as it ticks over. */ + } SNVR1_b; + } ; + + union { + __IOM uint32_t SNVR2; /*!< (@ 0x000001F8) System Timer NVRAM_C Register */ + + struct { + __IOM uint32_t SNVR2 : 32; /*!< [31..0] Value of the 32-bit counter as it ticks over. */ + } SNVR2_b; + } ; + + union { + __IOM uint32_t SNVR3; /*!< (@ 0x000001FC) System Timer NVRAM_D Register */ + + struct { + __IOM uint32_t SNVR3 : 32; /*!< [31..0] Value of the 32-bit counter as it ticks over. */ + } SNVR3_b; + } ; + + union { + __IOM uint32_t INTEN; /*!< (@ 0x00000200) Counter/Timer Interrupts: Enable */ + + struct { + __IOM uint32_t CTMRA0C0INT : 1; /*!< [0..0] Counter/Timer A0 interrupt based on COMPR0. */ + __IOM uint32_t CTMRB0C0INT : 1; /*!< [1..1] Counter/Timer B0 interrupt based on COMPR0. */ + __IOM uint32_t CTMRA1C0INT : 1; /*!< [2..2] Counter/Timer A1 interrupt based on COMPR0. */ + __IOM uint32_t CTMRB1C0INT : 1; /*!< [3..3] Counter/Timer B1 interrupt based on COMPR0. */ + __IOM uint32_t CTMRA2C0INT : 1; /*!< [4..4] Counter/Timer A2 interrupt based on COMPR0. */ + __IOM uint32_t CTMRB2C0INT : 1; /*!< [5..5] Counter/Timer B2 interrupt based on COMPR0. */ + __IOM uint32_t CTMRA3C0INT : 1; /*!< [6..6] Counter/Timer A3 interrupt based on COMPR0. */ + __IOM uint32_t CTMRB3C0INT : 1; /*!< [7..7] Counter/Timer B3 interrupt based on COMPR0. */ + __IOM uint32_t CTMRA4C0INT : 1; /*!< [8..8] Counter/Timer A4 interrupt based on COMPR0. */ + __IOM uint32_t CTMRB4C0INT : 1; /*!< [9..9] Counter/Timer B4 interrupt based on COMPR0. */ + __IOM uint32_t CTMRA5C0INT : 1; /*!< [10..10] Counter/Timer A5 interrupt based on COMPR0. */ + __IOM uint32_t CTMRB5C0INT : 1; /*!< [11..11] Counter/Timer B5 interrupt based on COMPR0. */ + __IOM uint32_t CTMRA6C0INT : 1; /*!< [12..12] Counter/Timer A6 interrupt based on COMPR0. */ + __IOM uint32_t CTMRB6C0INT : 1; /*!< [13..13] Counter/Timer B6 interrupt based on COMPR0. */ + __IOM uint32_t CTMRA7C0INT : 1; /*!< [14..14] Counter/Timer A7 interrupt based on COMPR0. */ + __IOM uint32_t CTMRB7C0INT : 1; /*!< [15..15] Counter/Timer B7 interrupt based on COMPR0. */ + __IOM uint32_t CTMRA0C1INT : 1; /*!< [16..16] Counter/Timer A0 interrupt based on COMPR1. */ + __IOM uint32_t CTMRB0C1INT : 1; /*!< [17..17] Counter/Timer B0 interrupt based on COMPR1. */ + __IOM uint32_t CTMRA1C1INT : 1; /*!< [18..18] Counter/Timer A1 interrupt based on COMPR1. */ + __IOM uint32_t CTMRB1C1INT : 1; /*!< [19..19] Counter/Timer B1 interrupt based on COMPR1. */ + __IOM uint32_t CTMRA2C1INT : 1; /*!< [20..20] Counter/Timer A2 interrupt based on COMPR1. */ + __IOM uint32_t CTMRB2C1INT : 1; /*!< [21..21] Counter/Timer B2 interrupt based on COMPR1. */ + __IOM uint32_t CTMRA3C1INT : 1; /*!< [22..22] Counter/Timer A3 interrupt based on COMPR1. */ + __IOM uint32_t CTMRB3C1INT : 1; /*!< [23..23] Counter/Timer B3 interrupt based on COMPR1. */ + __IOM uint32_t CTMRA4C1INT : 1; /*!< [24..24] Counter/Timer A4 interrupt based on COMPR1. */ + __IOM uint32_t CTMRB4C1INT : 1; /*!< [25..25] Counter/Timer B4 interrupt based on COMPR1. */ + __IOM uint32_t CTMRA5C1INT : 1; /*!< [26..26] Counter/Timer A5 interrupt based on COMPR1. */ + __IOM uint32_t CTMRB5C1INT : 1; /*!< [27..27] Counter/Timer B5 interrupt based on COMPR1. */ + __IOM uint32_t CTMRA6C1INT : 1; /*!< [28..28] Counter/Timer A6 interrupt based on COMPR1. */ + __IOM uint32_t CTMRB6C1INT : 1; /*!< [29..29] Counter/Timer B6 interrupt based on COMPR1. */ + __IOM uint32_t CTMRA7C1INT : 1; /*!< [30..30] Counter/Timer A7 interrupt based on COMPR1. */ + __IOM uint32_t CTMRB7C1INT : 1; /*!< [31..31] Counter/Timer B7 interrupt based on COMPR1. */ + } INTEN_b; + } ; + + union { + __IOM uint32_t INTSTAT; /*!< (@ 0x00000204) Counter/Timer Interrupts: Status */ + + struct { + __IOM uint32_t CTMRA0C0INT : 1; /*!< [0..0] Counter/Timer A0 interrupt based on COMPR0. */ + __IOM uint32_t CTMRB0C0INT : 1; /*!< [1..1] Counter/Timer B0 interrupt based on COMPR0. */ + __IOM uint32_t CTMRA1C0INT : 1; /*!< [2..2] Counter/Timer A1 interrupt based on COMPR0. */ + __IOM uint32_t CTMRB1C0INT : 1; /*!< [3..3] Counter/Timer B1 interrupt based on COMPR0. */ + __IOM uint32_t CTMRA2C0INT : 1; /*!< [4..4] Counter/Timer A2 interrupt based on COMPR0. */ + __IOM uint32_t CTMRB2C0INT : 1; /*!< [5..5] Counter/Timer B2 interrupt based on COMPR0. */ + __IOM uint32_t CTMRA3C0INT : 1; /*!< [6..6] Counter/Timer A3 interrupt based on COMPR0. */ + __IOM uint32_t CTMRB3C0INT : 1; /*!< [7..7] Counter/Timer B3 interrupt based on COMPR0. */ + __IOM uint32_t CTMRA4C0INT : 1; /*!< [8..8] Counter/Timer A4 interrupt based on COMPR0. */ + __IOM uint32_t CTMRB4C0INT : 1; /*!< [9..9] Counter/Timer B4 interrupt based on COMPR0. */ + __IOM uint32_t CTMRA5C0INT : 1; /*!< [10..10] Counter/Timer A5 interrupt based on COMPR0. */ + __IOM uint32_t CTMRB5C0INT : 1; /*!< [11..11] Counter/Timer B5 interrupt based on COMPR0. */ + __IOM uint32_t CTMRA6C0INT : 1; /*!< [12..12] Counter/Timer A6 interrupt based on COMPR0. */ + __IOM uint32_t CTMRB6C0INT : 1; /*!< [13..13] Counter/Timer B6 interrupt based on COMPR0. */ + __IOM uint32_t CTMRA7C0INT : 1; /*!< [14..14] Counter/Timer A7 interrupt based on COMPR0. */ + __IOM uint32_t CTMRB7C0INT : 1; /*!< [15..15] Counter/Timer B7 interrupt based on COMPR0. */ + __IOM uint32_t CTMRA0C1INT : 1; /*!< [16..16] Counter/Timer A0 interrupt based on COMPR1. */ + __IOM uint32_t CTMRB0C1INT : 1; /*!< [17..17] Counter/Timer B0 interrupt based on COMPR1. */ + __IOM uint32_t CTMRA1C1INT : 1; /*!< [18..18] Counter/Timer A1 interrupt based on COMPR1. */ + __IOM uint32_t CTMRB1C1INT : 1; /*!< [19..19] Counter/Timer B1 interrupt based on COMPR1. */ + __IOM uint32_t CTMRA2C1INT : 1; /*!< [20..20] Counter/Timer A2 interrupt based on COMPR1. */ + __IOM uint32_t CTMRB2C1INT : 1; /*!< [21..21] Counter/Timer B2 interrupt based on COMPR1. */ + __IOM uint32_t CTMRA3C1INT : 1; /*!< [22..22] Counter/Timer A3 interrupt based on COMPR1. */ + __IOM uint32_t CTMRB3C1INT : 1; /*!< [23..23] Counter/Timer B3 interrupt based on COMPR1. */ + __IOM uint32_t CTMRA4C1INT : 1; /*!< [24..24] Counter/Timer A4 interrupt based on COMPR1. */ + __IOM uint32_t CTMRB4C1INT : 1; /*!< [25..25] Counter/Timer B4 interrupt based on COMPR1. */ + __IOM uint32_t CTMRA5C1INT : 1; /*!< [26..26] Counter/Timer A5 interrupt based on COMPR1. */ + __IOM uint32_t CTMRB5C1INT : 1; /*!< [27..27] Counter/Timer B5 interrupt based on COMPR1. */ + __IOM uint32_t CTMRA6C1INT : 1; /*!< [28..28] Counter/Timer A6 interrupt based on COMPR1. */ + __IOM uint32_t CTMRB6C1INT : 1; /*!< [29..29] Counter/Timer B6 interrupt based on COMPR1. */ + __IOM uint32_t CTMRA7C1INT : 1; /*!< [30..30] Counter/Timer A7 interrupt based on COMPR1. */ + __IOM uint32_t CTMRB7C1INT : 1; /*!< [31..31] Counter/Timer B7 interrupt based on COMPR1. */ + } INTSTAT_b; + } ; + + union { + __IOM uint32_t INTCLR; /*!< (@ 0x00000208) Counter/Timer Interrupts: Clear */ + + struct { + __IOM uint32_t CTMRA0C0INT : 1; /*!< [0..0] Counter/Timer A0 interrupt based on COMPR0. */ + __IOM uint32_t CTMRB0C0INT : 1; /*!< [1..1] Counter/Timer B0 interrupt based on COMPR0. */ + __IOM uint32_t CTMRA1C0INT : 1; /*!< [2..2] Counter/Timer A1 interrupt based on COMPR0. */ + __IOM uint32_t CTMRB1C0INT : 1; /*!< [3..3] Counter/Timer B1 interrupt based on COMPR0. */ + __IOM uint32_t CTMRA2C0INT : 1; /*!< [4..4] Counter/Timer A2 interrupt based on COMPR0. */ + __IOM uint32_t CTMRB2C0INT : 1; /*!< [5..5] Counter/Timer B2 interrupt based on COMPR0. */ + __IOM uint32_t CTMRA3C0INT : 1; /*!< [6..6] Counter/Timer A3 interrupt based on COMPR0. */ + __IOM uint32_t CTMRB3C0INT : 1; /*!< [7..7] Counter/Timer B3 interrupt based on COMPR0. */ + __IOM uint32_t CTMRA4C0INT : 1; /*!< [8..8] Counter/Timer A4 interrupt based on COMPR0. */ + __IOM uint32_t CTMRB4C0INT : 1; /*!< [9..9] Counter/Timer B4 interrupt based on COMPR0. */ + __IOM uint32_t CTMRA5C0INT : 1; /*!< [10..10] Counter/Timer A5 interrupt based on COMPR0. */ + __IOM uint32_t CTMRB5C0INT : 1; /*!< [11..11] Counter/Timer B5 interrupt based on COMPR0. */ + __IOM uint32_t CTMRA6C0INT : 1; /*!< [12..12] Counter/Timer A6 interrupt based on COMPR0. */ + __IOM uint32_t CTMRB6C0INT : 1; /*!< [13..13] Counter/Timer B6 interrupt based on COMPR0. */ + __IOM uint32_t CTMRA7C0INT : 1; /*!< [14..14] Counter/Timer A7 interrupt based on COMPR0. */ + __IOM uint32_t CTMRB7C0INT : 1; /*!< [15..15] Counter/Timer B7 interrupt based on COMPR0. */ + __IOM uint32_t CTMRA0C1INT : 1; /*!< [16..16] Counter/Timer A0 interrupt based on COMPR1. */ + __IOM uint32_t CTMRB0C1INT : 1; /*!< [17..17] Counter/Timer B0 interrupt based on COMPR1. */ + __IOM uint32_t CTMRA1C1INT : 1; /*!< [18..18] Counter/Timer A1 interrupt based on COMPR1. */ + __IOM uint32_t CTMRB1C1INT : 1; /*!< [19..19] Counter/Timer B1 interrupt based on COMPR1. */ + __IOM uint32_t CTMRA2C1INT : 1; /*!< [20..20] Counter/Timer A2 interrupt based on COMPR1. */ + __IOM uint32_t CTMRB2C1INT : 1; /*!< [21..21] Counter/Timer B2 interrupt based on COMPR1. */ + __IOM uint32_t CTMRA3C1INT : 1; /*!< [22..22] Counter/Timer A3 interrupt based on COMPR1. */ + __IOM uint32_t CTMRB3C1INT : 1; /*!< [23..23] Counter/Timer B3 interrupt based on COMPR1. */ + __IOM uint32_t CTMRA4C1INT : 1; /*!< [24..24] Counter/Timer A4 interrupt based on COMPR1. */ + __IOM uint32_t CTMRB4C1INT : 1; /*!< [25..25] Counter/Timer B4 interrupt based on COMPR1. */ + __IOM uint32_t CTMRA5C1INT : 1; /*!< [26..26] Counter/Timer A5 interrupt based on COMPR1. */ + __IOM uint32_t CTMRB5C1INT : 1; /*!< [27..27] Counter/Timer B5 interrupt based on COMPR1. */ + __IOM uint32_t CTMRA6C1INT : 1; /*!< [28..28] Counter/Timer A6 interrupt based on COMPR1. */ + __IOM uint32_t CTMRB6C1INT : 1; /*!< [29..29] Counter/Timer B6 interrupt based on COMPR1. */ + __IOM uint32_t CTMRA7C1INT : 1; /*!< [30..30] Counter/Timer A7 interrupt based on COMPR1. */ + __IOM uint32_t CTMRB7C1INT : 1; /*!< [31..31] Counter/Timer B7 interrupt based on COMPR1. */ + } INTCLR_b; + } ; + + union { + __IOM uint32_t INTSET; /*!< (@ 0x0000020C) Counter/Timer Interrupts: Set */ + + struct { + __IOM uint32_t CTMRA0C0INT : 1; /*!< [0..0] Counter/Timer A0 interrupt based on COMPR0. */ + __IOM uint32_t CTMRB0C0INT : 1; /*!< [1..1] Counter/Timer B0 interrupt based on COMPR0. */ + __IOM uint32_t CTMRA1C0INT : 1; /*!< [2..2] Counter/Timer A1 interrupt based on COMPR0. */ + __IOM uint32_t CTMRB1C0INT : 1; /*!< [3..3] Counter/Timer B1 interrupt based on COMPR0. */ + __IOM uint32_t CTMRA2C0INT : 1; /*!< [4..4] Counter/Timer A2 interrupt based on COMPR0. */ + __IOM uint32_t CTMRB2C0INT : 1; /*!< [5..5] Counter/Timer B2 interrupt based on COMPR0. */ + __IOM uint32_t CTMRA3C0INT : 1; /*!< [6..6] Counter/Timer A3 interrupt based on COMPR0. */ + __IOM uint32_t CTMRB3C0INT : 1; /*!< [7..7] Counter/Timer B3 interrupt based on COMPR0. */ + __IOM uint32_t CTMRA4C0INT : 1; /*!< [8..8] Counter/Timer A4 interrupt based on COMPR0. */ + __IOM uint32_t CTMRB4C0INT : 1; /*!< [9..9] Counter/Timer B4 interrupt based on COMPR0. */ + __IOM uint32_t CTMRA5C0INT : 1; /*!< [10..10] Counter/Timer A5 interrupt based on COMPR0. */ + __IOM uint32_t CTMRB5C0INT : 1; /*!< [11..11] Counter/Timer B5 interrupt based on COMPR0. */ + __IOM uint32_t CTMRA6C0INT : 1; /*!< [12..12] Counter/Timer A6 interrupt based on COMPR0. */ + __IOM uint32_t CTMRB6C0INT : 1; /*!< [13..13] Counter/Timer B6 interrupt based on COMPR0. */ + __IOM uint32_t CTMRA7C0INT : 1; /*!< [14..14] Counter/Timer A7 interrupt based on COMPR0. */ + __IOM uint32_t CTMRB7C0INT : 1; /*!< [15..15] Counter/Timer B7 interrupt based on COMPR0. */ + __IOM uint32_t CTMRA0C1INT : 1; /*!< [16..16] Counter/Timer A0 interrupt based on COMPR1. */ + __IOM uint32_t CTMRB0C1INT : 1; /*!< [17..17] Counter/Timer B0 interrupt based on COMPR1. */ + __IOM uint32_t CTMRA1C1INT : 1; /*!< [18..18] Counter/Timer A1 interrupt based on COMPR1. */ + __IOM uint32_t CTMRB1C1INT : 1; /*!< [19..19] Counter/Timer B1 interrupt based on COMPR1. */ + __IOM uint32_t CTMRA2C1INT : 1; /*!< [20..20] Counter/Timer A2 interrupt based on COMPR1. */ + __IOM uint32_t CTMRB2C1INT : 1; /*!< [21..21] Counter/Timer B2 interrupt based on COMPR1. */ + __IOM uint32_t CTMRA3C1INT : 1; /*!< [22..22] Counter/Timer A3 interrupt based on COMPR1. */ + __IOM uint32_t CTMRB3C1INT : 1; /*!< [23..23] Counter/Timer B3 interrupt based on COMPR1. */ + __IOM uint32_t CTMRA4C1INT : 1; /*!< [24..24] Counter/Timer A4 interrupt based on COMPR1. */ + __IOM uint32_t CTMRB4C1INT : 1; /*!< [25..25] Counter/Timer B4 interrupt based on COMPR1. */ + __IOM uint32_t CTMRA5C1INT : 1; /*!< [26..26] Counter/Timer A5 interrupt based on COMPR1. */ + __IOM uint32_t CTMRB5C1INT : 1; /*!< [27..27] Counter/Timer B5 interrupt based on COMPR1. */ + __IOM uint32_t CTMRA6C1INT : 1; /*!< [28..28] Counter/Timer A6 interrupt based on COMPR1. */ + __IOM uint32_t CTMRB6C1INT : 1; /*!< [29..29] Counter/Timer B6 interrupt based on COMPR1. */ + __IOM uint32_t CTMRA7C1INT : 1; /*!< [30..30] Counter/Timer A7 interrupt based on COMPR1. */ + __IOM uint32_t CTMRB7C1INT : 1; /*!< [31..31] Counter/Timer B7 interrupt based on COMPR1. */ + } INTSET_b; + } ; + __IM uint32_t RESERVED12[60]; + + union { + __IOM uint32_t STMINTEN; /*!< (@ 0x00000300) STIMER Interrupt registers: Enable */ + + struct { + __IOM uint32_t COMPAREA : 1; /*!< [0..0] COUNTER is greater than or equal to COMPARE register + A. */ + __IOM uint32_t COMPAREB : 1; /*!< [1..1] COUNTER is greater than or equal to COMPARE register + B. */ + __IOM uint32_t COMPAREC : 1; /*!< [2..2] COUNTER is greater than or equal to COMPARE register + C. */ + __IOM uint32_t COMPARED : 1; /*!< [3..3] COUNTER is greater than or equal to COMPARE register + D. */ + __IOM uint32_t COMPAREE : 1; /*!< [4..4] COUNTER is greater than or equal to COMPARE register + E. */ + __IOM uint32_t COMPAREF : 1; /*!< [5..5] COUNTER is greater than or equal to COMPARE register + F. */ + __IOM uint32_t COMPAREG : 1; /*!< [6..6] COUNTER is greater than or equal to COMPARE register + G. */ + __IOM uint32_t COMPAREH : 1; /*!< [7..7] COUNTER is greater than or equal to COMPARE register + H. */ + __IOM uint32_t OVERFLOW : 1; /*!< [8..8] COUNTER over flowed from 0xFFFFFFFF back to 0x00000000. */ + __IOM uint32_t CAPTUREA : 1; /*!< [9..9] CAPTURE register A has grabbed the value in the counter */ + __IOM uint32_t CAPTUREB : 1; /*!< [10..10] CAPTURE register B has grabbed the value in the counter */ + __IOM uint32_t CAPTUREC : 1; /*!< [11..11] CAPTURE register C has grabbed the value in the counter */ + __IOM uint32_t CAPTURED : 1; /*!< [12..12] CAPTURE register D has grabbed the value in the counter */ + } STMINTEN_b; + } ; + + union { + __IOM uint32_t STMINTSTAT; /*!< (@ 0x00000304) STIMER Interrupt registers: Status */ + + struct { + __IOM uint32_t COMPAREA : 1; /*!< [0..0] COUNTER is greater than or equal to COMPARE register + A. */ + __IOM uint32_t COMPAREB : 1; /*!< [1..1] COUNTER is greater than or equal to COMPARE register + B. */ + __IOM uint32_t COMPAREC : 1; /*!< [2..2] COUNTER is greater than or equal to COMPARE register + C. */ + __IOM uint32_t COMPARED : 1; /*!< [3..3] COUNTER is greater than or equal to COMPARE register + D. */ + __IOM uint32_t COMPAREE : 1; /*!< [4..4] COUNTER is greater than or equal to COMPARE register + E. */ + __IOM uint32_t COMPAREF : 1; /*!< [5..5] COUNTER is greater than or equal to COMPARE register + F. */ + __IOM uint32_t COMPAREG : 1; /*!< [6..6] COUNTER is greater than or equal to COMPARE register + G. */ + __IOM uint32_t COMPAREH : 1; /*!< [7..7] COUNTER is greater than or equal to COMPARE register + H. */ + __IOM uint32_t OVERFLOW : 1; /*!< [8..8] COUNTER over flowed from 0xFFFFFFFF back to 0x00000000. */ + __IOM uint32_t CAPTUREA : 1; /*!< [9..9] CAPTURE register A has grabbed the value in the counter */ + __IOM uint32_t CAPTUREB : 1; /*!< [10..10] CAPTURE register B has grabbed the value in the counter */ + __IOM uint32_t CAPTUREC : 1; /*!< [11..11] CAPTURE register C has grabbed the value in the counter */ + __IOM uint32_t CAPTURED : 1; /*!< [12..12] CAPTURE register D has grabbed the value in the counter */ + } STMINTSTAT_b; + } ; + + union { + __IOM uint32_t STMINTCLR; /*!< (@ 0x00000308) STIMER Interrupt registers: Clear */ + + struct { + __IOM uint32_t COMPAREA : 1; /*!< [0..0] COUNTER is greater than or equal to COMPARE register + A. */ + __IOM uint32_t COMPAREB : 1; /*!< [1..1] COUNTER is greater than or equal to COMPARE register + B. */ + __IOM uint32_t COMPAREC : 1; /*!< [2..2] COUNTER is greater than or equal to COMPARE register + C. */ + __IOM uint32_t COMPARED : 1; /*!< [3..3] COUNTER is greater than or equal to COMPARE register + D. */ + __IOM uint32_t COMPAREE : 1; /*!< [4..4] COUNTER is greater than or equal to COMPARE register + E. */ + __IOM uint32_t COMPAREF : 1; /*!< [5..5] COUNTER is greater than or equal to COMPARE register + F. */ + __IOM uint32_t COMPAREG : 1; /*!< [6..6] COUNTER is greater than or equal to COMPARE register + G. */ + __IOM uint32_t COMPAREH : 1; /*!< [7..7] COUNTER is greater than or equal to COMPARE register + H. */ + __IOM uint32_t OVERFLOW : 1; /*!< [8..8] COUNTER over flowed from 0xFFFFFFFF back to 0x00000000. */ + __IOM uint32_t CAPTUREA : 1; /*!< [9..9] CAPTURE register A has grabbed the value in the counter */ + __IOM uint32_t CAPTUREB : 1; /*!< [10..10] CAPTURE register B has grabbed the value in the counter */ + __IOM uint32_t CAPTUREC : 1; /*!< [11..11] CAPTURE register C has grabbed the value in the counter */ + __IOM uint32_t CAPTURED : 1; /*!< [12..12] CAPTURE register D has grabbed the value in the counter */ + } STMINTCLR_b; + } ; + + union { + __IOM uint32_t STMINTSET; /*!< (@ 0x0000030C) STIMER Interrupt registers: Set */ + + struct { + __IOM uint32_t COMPAREA : 1; /*!< [0..0] COUNTER is greater than or equal to COMPARE register + A. */ + __IOM uint32_t COMPAREB : 1; /*!< [1..1] COUNTER is greater than or equal to COMPARE register + B. */ + __IOM uint32_t COMPAREC : 1; /*!< [2..2] COUNTER is greater than or equal to COMPARE register + C. */ + __IOM uint32_t COMPARED : 1; /*!< [3..3] COUNTER is greater than or equal to COMPARE register + D. */ + __IOM uint32_t COMPAREE : 1; /*!< [4..4] COUNTER is greater than or equal to COMPARE register + E. */ + __IOM uint32_t COMPAREF : 1; /*!< [5..5] COUNTER is greater than or equal to COMPARE register + F. */ + __IOM uint32_t COMPAREG : 1; /*!< [6..6] COUNTER is greater than or equal to COMPARE register + G. */ + __IOM uint32_t COMPAREH : 1; /*!< [7..7] COUNTER is greater than or equal to COMPARE register + H. */ + __IOM uint32_t OVERFLOW : 1; /*!< [8..8] COUNTER over flowed from 0xFFFFFFFF back to 0x00000000. */ + __IOM uint32_t CAPTUREA : 1; /*!< [9..9] CAPTURE register A has grabbed the value in the counter */ + __IOM uint32_t CAPTUREB : 1; /*!< [10..10] CAPTURE register B has grabbed the value in the counter */ + __IOM uint32_t CAPTUREC : 1; /*!< [11..11] CAPTURE register C has grabbed the value in the counter */ + __IOM uint32_t CAPTURED : 1; /*!< [12..12] CAPTURE register D has grabbed the value in the counter */ + } STMINTSET_b; + } ; +} CTIMER_Type; /*!< Size = 784 (0x310) */ + + + +/* =========================================================================================================================== */ +/* ================ GPIO ================ */ +/* =========================================================================================================================== */ + + +/** + * @brief General Purpose IO (GPIO) + */ + +typedef struct { /*!< (@ 0x40010000) GPIO Structure */ + + union { + __IOM uint32_t PADREGA; /*!< (@ 0x00000000) Pad Configuration Register A (Pads 0-3) */ + + struct { + __IOM uint32_t PAD0PULL : 1; /*!< [0..0] Pad 0 pullup enable */ + __IOM uint32_t PAD0INPEN : 1; /*!< [1..1] Pad 0 input enable */ + __IOM uint32_t PAD0STRNG : 1; /*!< [2..2] Pad 0 drive strength */ + __IOM uint32_t PAD0FNCSEL : 3; /*!< [5..3] Pad 0 function select */ + __IOM uint32_t PAD0RSEL : 2; /*!< [7..6] Pad 0 pullup resistor selection. */ + __IOM uint32_t PAD1PULL : 1; /*!< [8..8] Pad 1 pullup enable */ + __IOM uint32_t PAD1INPEN : 1; /*!< [9..9] Pad 1 input enable */ + __IOM uint32_t PAD1STRNG : 1; /*!< [10..10] Pad 1 drive strength */ + __IOM uint32_t PAD1FNCSEL : 3; /*!< [13..11] Pad 1 function select */ + __IOM uint32_t PAD1RSEL : 2; /*!< [15..14] Pad 1 pullup resistor selection. */ + __IOM uint32_t PAD2PULL : 1; /*!< [16..16] Pad 2 pullup enable */ + __IOM uint32_t PAD2INPEN : 1; /*!< [17..17] Pad 2 input enable */ + __IOM uint32_t PAD2STRNG : 1; /*!< [18..18] Pad 2 drive strength */ + __IOM uint32_t PAD2FNCSEL : 3; /*!< [21..19] Pad 2 function select */ + __IM uint32_t : 2; + __IOM uint32_t PAD3PULL : 1; /*!< [24..24] Pad 3 pullup enable */ + __IOM uint32_t PAD3INPEN : 1; /*!< [25..25] Pad 3 input enable. */ + __IOM uint32_t PAD3STRNG : 1; /*!< [26..26] Pad 3 drive strength. */ + __IOM uint32_t PAD3FNCSEL : 3; /*!< [29..27] Pad 3 function select */ + __IOM uint32_t PAD3PWRUP : 1; /*!< [30..30] Pad 3 VDD power switch enable */ + } PADREGA_b; + } ; + + union { + __IOM uint32_t PADREGB; /*!< (@ 0x00000004) Pad Configuration Register B (Pads 4-7) */ + + struct { + __IOM uint32_t PAD4PULL : 1; /*!< [0..0] Pad 4 pullup enable */ + __IOM uint32_t PAD4INPEN : 1; /*!< [1..1] Pad 4 input enable */ + __IOM uint32_t PAD4STRNG : 1; /*!< [2..2] Pad 4 drive strength */ + __IOM uint32_t PAD4FNCSEL : 3; /*!< [5..3] Pad 4 function select */ + __IM uint32_t : 2; + __IOM uint32_t PAD5PULL : 1; /*!< [8..8] Pad 5 pullup enable */ + __IOM uint32_t PAD5INPEN : 1; /*!< [9..9] Pad 5 input enable */ + __IOM uint32_t PAD5STRNG : 1; /*!< [10..10] Pad 5 drive strength */ + __IOM uint32_t PAD5FNCSEL : 3; /*!< [13..11] Pad 5 function select */ + __IOM uint32_t PAD5RSEL : 2; /*!< [15..14] Pad 5 pullup resistor selection. */ + __IOM uint32_t PAD6PULL : 1; /*!< [16..16] Pad 6 pullup enable */ + __IOM uint32_t PAD6INPEN : 1; /*!< [17..17] Pad 6 input enable */ + __IOM uint32_t PAD6STRNG : 1; /*!< [18..18] Pad 6 drive strength */ + __IOM uint32_t PAD6FNCSEL : 3; /*!< [21..19] Pad 6 function select */ + __IOM uint32_t PAD6RSEL : 2; /*!< [23..22] Pad 6 pullup resistor selection. */ + __IOM uint32_t PAD7PULL : 1; /*!< [24..24] Pad 7 pullup enable */ + __IOM uint32_t PAD7INPEN : 1; /*!< [25..25] Pad 7 input enable */ + __IOM uint32_t PAD7STRNG : 1; /*!< [26..26] Pad 7 drive strength */ + __IOM uint32_t PAD7FNCSEL : 3; /*!< [29..27] Pad 7 function select */ + } PADREGB_b; + } ; + + union { + __IOM uint32_t PADREGC; /*!< (@ 0x00000008) Pad Configuration Register C (Pads 8-11) */ + + struct { + __IOM uint32_t PAD8PULL : 1; /*!< [0..0] Pad 8 pullup enable */ + __IOM uint32_t PAD8INPEN : 1; /*!< [1..1] Pad 8 input enable */ + __IOM uint32_t PAD8STRNG : 1; /*!< [2..2] Pad 8 drive strength */ + __IOM uint32_t PAD8FNCSEL : 3; /*!< [5..3] Pad 8 function select */ + __IOM uint32_t PAD8RSEL : 2; /*!< [7..6] Pad 8 pullup resistor selection. */ + __IOM uint32_t PAD9PULL : 1; /*!< [8..8] Pad 9 pullup enable */ + __IOM uint32_t PAD9INPEN : 1; /*!< [9..9] Pad 9 input enable */ + __IOM uint32_t PAD9STRNG : 1; /*!< [10..10] Pad 9 drive strength */ + __IOM uint32_t PAD9FNCSEL : 3; /*!< [13..11] Pad 9 function select */ + __IOM uint32_t PAD9RSEL : 2; /*!< [15..14] Pad 9 pullup resistor selection */ + __IOM uint32_t PAD10PULL : 1; /*!< [16..16] Pad 10 pullup enable */ + __IOM uint32_t PAD10INPEN : 1; /*!< [17..17] Pad 10 input enable */ + __IOM uint32_t PAD10STRNG : 1; /*!< [18..18] Pad 10 drive strength */ + __IOM uint32_t PAD10FNCSEL : 3; /*!< [21..19] Pad 10 function select */ + __IM uint32_t : 2; + __IOM uint32_t PAD11PULL : 1; /*!< [24..24] Pad 11 pullup enable */ + __IOM uint32_t PAD11INPEN : 1; /*!< [25..25] Pad 11 input enable */ + __IOM uint32_t PAD11STRNG : 1; /*!< [26..26] Pad 11 drive strength */ + __IOM uint32_t PAD11FNCSEL : 3; /*!< [29..27] Pad 11 function select */ + } PADREGC_b; + } ; + + union { + __IOM uint32_t PADREGD; /*!< (@ 0x0000000C) Pad Configuration Register D (Pads 12-15) */ + + struct { + __IOM uint32_t PAD12PULL : 1; /*!< [0..0] Pad 12 pullup enable */ + __IOM uint32_t PAD12INPEN : 1; /*!< [1..1] Pad 12 input enable */ + __IOM uint32_t PAD12STRNG : 1; /*!< [2..2] Pad 12 drive strength */ + __IOM uint32_t PAD12FNCSEL : 3; /*!< [5..3] Pad 12 function select */ + __IM uint32_t : 2; + __IOM uint32_t PAD13PULL : 1; /*!< [8..8] Pad 13 pullup enable */ + __IOM uint32_t PAD13INPEN : 1; /*!< [9..9] Pad 13 input enable */ + __IOM uint32_t PAD13STRNG : 1; /*!< [10..10] Pad 13 drive strength */ + __IOM uint32_t PAD13FNCSEL : 3; /*!< [13..11] Pad 13 function select */ + __IM uint32_t : 2; + __IOM uint32_t PAD14PULL : 1; /*!< [16..16] Pad 14 pullup enable */ + __IOM uint32_t PAD14INPEN : 1; /*!< [17..17] Pad 14 input enable */ + __IOM uint32_t PAD14STRNG : 1; /*!< [18..18] Pad 14 drive strength */ + __IOM uint32_t PAD14FNCSEL : 3; /*!< [21..19] Pad 14 function select */ + __IM uint32_t : 2; + __IOM uint32_t PAD15PULL : 1; /*!< [24..24] Pad 15 pullup enable */ + __IOM uint32_t PAD15INPEN : 1; /*!< [25..25] Pad 15 input enable */ + __IOM uint32_t PAD15STRNG : 1; /*!< [26..26] Pad 15 drive strength */ + __IOM uint32_t PAD15FNCSEL : 3; /*!< [29..27] Pad 15 function select */ + } PADREGD_b; + } ; + + union { + __IOM uint32_t PADREGE; /*!< (@ 0x00000010) Pad Configuration Register E (Pads 16-19) */ + + struct { + __IOM uint32_t PAD16PULL : 1; /*!< [0..0] Pad 16 pullup enable */ + __IOM uint32_t PAD16INPEN : 1; /*!< [1..1] Pad 16 input enable */ + __IOM uint32_t PAD16STRNG : 1; /*!< [2..2] Pad 16 drive strength */ + __IOM uint32_t PAD16FNCSEL : 3; /*!< [5..3] Pad 16 function select */ + __IM uint32_t : 2; + __IOM uint32_t PAD17PULL : 1; /*!< [8..8] Pad 17 pullup enable */ + __IOM uint32_t PAD17INPEN : 1; /*!< [9..9] Pad 17 input enable */ + __IOM uint32_t PAD17STRNG : 1; /*!< [10..10] Pad 17 drive strength */ + __IOM uint32_t PAD17FNCSEL : 3; /*!< [13..11] Pad 17 function select */ + __IM uint32_t : 2; + __IOM uint32_t PAD18PULL : 1; /*!< [16..16] Pad 18 pullup enable */ + __IOM uint32_t PAD18INPEN : 1; /*!< [17..17] Pad 18 input enable */ + __IOM uint32_t PAD18STRNG : 1; /*!< [18..18] Pad 18 drive strength */ + __IOM uint32_t PAD18FNCSEL : 3; /*!< [21..19] Pad 18 function select */ + __IM uint32_t : 2; + __IOM uint32_t PAD19PULL : 1; /*!< [24..24] Pad 19 pullup enable */ + __IOM uint32_t PAD19INPEN : 1; /*!< [25..25] Pad 19 input enable */ + __IOM uint32_t PAD19STRNG : 1; /*!< [26..26] Pad 19 drive strength */ + __IOM uint32_t PAD19FNCSEL : 3; /*!< [29..27] Pad 19 function select */ + } PADREGE_b; + } ; + + union { + __IOM uint32_t PADREGF; /*!< (@ 0x00000014) Pad Configuration Register F (Pads 20-23) */ + + struct { + __IOM uint32_t PAD20PULL : 1; /*!< [0..0] Pad 20 pulldown enable */ + __IOM uint32_t PAD20INPEN : 1; /*!< [1..1] Pad 20 input enable */ + __IOM uint32_t PAD20STRNG : 1; /*!< [2..2] Pad 20 drive strength */ + __IOM uint32_t PAD20FNCSEL : 3; /*!< [5..3] Pad 20 function select */ + __IM uint32_t : 2; + __IOM uint32_t PAD21PULL : 1; /*!< [8..8] Pad 21 pullup enable */ + __IOM uint32_t PAD21INPEN : 1; /*!< [9..9] Pad 21 input enable */ + __IOM uint32_t PAD21STRNG : 1; /*!< [10..10] Pad 21 drive strength */ + __IOM uint32_t PAD21FNCSEL : 3; /*!< [13..11] Pad 21 function select */ + __IM uint32_t : 2; + __IOM uint32_t PAD22PULL : 1; /*!< [16..16] Pad 22 pullup enable */ + __IOM uint32_t PAD22INPEN : 1; /*!< [17..17] Pad 22 input enable */ + __IOM uint32_t PAD22STRNG : 1; /*!< [18..18] Pad 22 drive strength */ + __IOM uint32_t PAD22FNCSEL : 3; /*!< [21..19] Pad 22 function select */ + __IM uint32_t : 2; + __IOM uint32_t PAD23PULL : 1; /*!< [24..24] Pad 23 pullup enable */ + __IOM uint32_t PAD23INPEN : 1; /*!< [25..25] Pad 23 input enable */ + __IOM uint32_t PAD23STRNG : 1; /*!< [26..26] Pad 23 drive strength */ + __IOM uint32_t PAD23FNCSEL : 3; /*!< [29..27] Pad 23 function select */ + } PADREGF_b; + } ; + + union { + __IOM uint32_t PADREGG; /*!< (@ 0x00000018) Pad Configuration Register G (Pads 24-27) */ + + struct { + __IOM uint32_t PAD24PULL : 1; /*!< [0..0] Pad 24 pullup enable */ + __IOM uint32_t PAD24INPEN : 1; /*!< [1..1] Pad 24 input enable */ + __IOM uint32_t PAD24STRNG : 1; /*!< [2..2] Pad 24 drive strength */ + __IOM uint32_t PAD24FNCSEL : 3; /*!< [5..3] Pad 24 function select */ + __IM uint32_t : 2; + __IOM uint32_t PAD25PULL : 1; /*!< [8..8] Pad 25 pullup enable */ + __IOM uint32_t PAD25INPEN : 1; /*!< [9..9] Pad 25 input enable */ + __IOM uint32_t PAD25STRNG : 1; /*!< [10..10] Pad 25 drive strength */ + __IOM uint32_t PAD25FNCSEL : 3; /*!< [13..11] Pad 25 function select */ + __IOM uint32_t PAD25RSEL : 2; /*!< [15..14] Pad 25 pullup resistor selection. */ + __IOM uint32_t PAD26PULL : 1; /*!< [16..16] Pad 26 pullup enable */ + __IOM uint32_t PAD26INPEN : 1; /*!< [17..17] Pad 26 input enable */ + __IOM uint32_t PAD26STRNG : 1; /*!< [18..18] Pad 26 drive strength */ + __IOM uint32_t PAD26FNCSEL : 3; /*!< [21..19] Pad 26 function select */ + __IM uint32_t : 2; + __IOM uint32_t PAD27PULL : 1; /*!< [24..24] Pad 27 pullup enable */ + __IOM uint32_t PAD27INPEN : 1; /*!< [25..25] Pad 27 input enable */ + __IOM uint32_t PAD27STRNG : 1; /*!< [26..26] Pad 27 drive strength */ + __IOM uint32_t PAD27FNCSEL : 3; /*!< [29..27] Pad 27 function select */ + __IOM uint32_t PAD27RSEL : 2; /*!< [31..30] Pad 27 pullup resistor selection. */ + } PADREGG_b; + } ; + + union { + __IOM uint32_t PADREGH; /*!< (@ 0x0000001C) Pad Configuration Register H (Pads 28-31) */ + + struct { + __IOM uint32_t PAD28PULL : 1; /*!< [0..0] Pad 28 pullup enable */ + __IOM uint32_t PAD28INPEN : 1; /*!< [1..1] Pad 28 input enable */ + __IOM uint32_t PAD28STRNG : 1; /*!< [2..2] Pad 28 drive strength */ + __IOM uint32_t PAD28FNCSEL : 3; /*!< [5..3] Pad 28 function select */ + __IM uint32_t : 2; + __IOM uint32_t PAD29PULL : 1; /*!< [8..8] Pad 29 pullup enable */ + __IOM uint32_t PAD29INPEN : 1; /*!< [9..9] Pad 29 input enable */ + __IOM uint32_t PAD29STRNG : 1; /*!< [10..10] Pad 29 drive strength */ + __IOM uint32_t PAD29FNCSEL : 3; /*!< [13..11] Pad 29 function select */ + __IM uint32_t : 2; + __IOM uint32_t PAD30PULL : 1; /*!< [16..16] Pad 30 pullup enable */ + __IOM uint32_t PAD30INPEN : 1; /*!< [17..17] Pad 30 input enable */ + __IOM uint32_t PAD30STRNG : 1; /*!< [18..18] Pad 30 drive strength */ + __IOM uint32_t PAD30FNCSEL : 3; /*!< [21..19] Pad 30 function select */ + __IM uint32_t : 2; + __IOM uint32_t PAD31PULL : 1; /*!< [24..24] Pad 31 pullup enable */ + __IOM uint32_t PAD31INPEN : 1; /*!< [25..25] Pad 31 input enable */ + __IOM uint32_t PAD31STRNG : 1; /*!< [26..26] Pad 31 drive strength */ + __IOM uint32_t PAD31FNCSEL : 3; /*!< [29..27] Pad 31 function select */ + } PADREGH_b; + } ; + + union { + __IOM uint32_t PADREGI; /*!< (@ 0x00000020) Pad Configuration Register I (Pads 32-25) */ + + struct { + __IOM uint32_t PAD32PULL : 1; /*!< [0..0] Pad 32 pullup enable */ + __IOM uint32_t PAD32INPEN : 1; /*!< [1..1] Pad 32 input enable */ + __IOM uint32_t PAD32STRNG : 1; /*!< [2..2] Pad 32 drive strength */ + __IOM uint32_t PAD32FNCSEL : 3; /*!< [5..3] Pad 32 function select */ + __IM uint32_t : 2; + __IOM uint32_t PAD33PULL : 1; /*!< [8..8] Pad 33 pullup enable */ + __IOM uint32_t PAD33INPEN : 1; /*!< [9..9] Pad 33 input enable */ + __IOM uint32_t PAD33STRNG : 1; /*!< [10..10] Pad 33 drive strength */ + __IOM uint32_t PAD33FNCSEL : 3; /*!< [13..11] Pad 33 function select */ + __IM uint32_t : 2; + __IOM uint32_t PAD34PULL : 1; /*!< [16..16] Pad 34 pullup enable */ + __IOM uint32_t PAD34INPEN : 1; /*!< [17..17] Pad 34 input enable */ + __IOM uint32_t PAD34STRNG : 1; /*!< [18..18] Pad 34 drive strength */ + __IOM uint32_t PAD34FNCSEL : 3; /*!< [21..19] Pad 34 function select */ + __IM uint32_t : 2; + __IOM uint32_t PAD35PULL : 1; /*!< [24..24] Pad 35 pullup enable */ + __IOM uint32_t PAD35INPEN : 1; /*!< [25..25] Pad 35 input enable */ + __IOM uint32_t PAD35STRNG : 1; /*!< [26..26] Pad 35 drive strength */ + __IOM uint32_t PAD35FNCSEL : 3; /*!< [29..27] Pad 35 function select */ + } PADREGI_b; + } ; + + union { + __IOM uint32_t PADREGJ; /*!< (@ 0x00000024) Pad Configuration Register J (Pads 36-39) */ + + struct { + __IOM uint32_t PAD36PULL : 1; /*!< [0..0] Pad 36 pullup enable */ + __IOM uint32_t PAD36INPEN : 1; /*!< [1..1] Pad 36 input enable */ + __IOM uint32_t PAD36STRNG : 1; /*!< [2..2] Pad 36 drive strength */ + __IOM uint32_t PAD36FNCSEL : 3; /*!< [5..3] Pad 36 function select */ + __IOM uint32_t PAD36PWRUP : 1; /*!< [6..6] Pad 36 VDD power switch enable */ + __IM uint32_t : 1; + __IOM uint32_t PAD37PULL : 1; /*!< [8..8] Pad 37 pullup enable */ + __IOM uint32_t PAD37INPEN : 1; /*!< [9..9] Pad 37 input enable */ + __IOM uint32_t PAD37STRNG : 1; /*!< [10..10] Pad 37 drive strength */ + __IOM uint32_t PAD37FNCSEL : 3; /*!< [13..11] Pad 37 function select */ + __IM uint32_t : 1; + __IOM uint32_t PAD37PWRDN : 1; /*!< [15..15] Pad 37 VSS power switch enable */ + __IOM uint32_t PAD38PULL : 1; /*!< [16..16] Pad 38 pullup enable */ + __IOM uint32_t PAD38INPEN : 1; /*!< [17..17] Pad 38 input enable */ + __IOM uint32_t PAD38STRNG : 1; /*!< [18..18] Pad 38 drive strength */ + __IOM uint32_t PAD38FNCSEL : 3; /*!< [21..19] Pad 38 function select */ + __IM uint32_t : 2; + __IOM uint32_t PAD39PULL : 1; /*!< [24..24] Pad 39 pullup enable */ + __IOM uint32_t PAD39INPEN : 1; /*!< [25..25] Pad 39 input enable */ + __IOM uint32_t PAD39STRNG : 1; /*!< [26..26] Pad 39 drive strength */ + __IOM uint32_t PAD39FNCSEL : 3; /*!< [29..27] Pad 39 function select */ + __IOM uint32_t PAD39RSEL : 2; /*!< [31..30] Pad 39 pullup resistor selection. */ + } PADREGJ_b; + } ; + + union { + __IOM uint32_t PADREGK; /*!< (@ 0x00000028) Pad Configuration Register K (Pads 40-43) */ + + struct { + __IOM uint32_t PAD40PULL : 1; /*!< [0..0] Pad 40 pullup enable */ + __IOM uint32_t PAD40INPEN : 1; /*!< [1..1] Pad 40 input enable */ + __IOM uint32_t PAD40STRNG : 1; /*!< [2..2] Pad 40 drive strength */ + __IOM uint32_t PAD40FNCSEL : 3; /*!< [5..3] Pad 40 function select */ + __IOM uint32_t PAD40RSEL : 2; /*!< [7..6] Pad 40 pullup resistor selection. */ + __IOM uint32_t PAD41PULL : 1; /*!< [8..8] Pad 41 pullup enable */ + __IOM uint32_t PAD41INPEN : 1; /*!< [9..9] Pad 41 input enable */ + __IOM uint32_t PAD41STRNG : 1; /*!< [10..10] Pad 41 drive strength */ + __IOM uint32_t PAD41FNCSEL : 3; /*!< [13..11] Pad 41 function select */ + __IM uint32_t : 1; + __IOM uint32_t PAD41PWRDN : 1; /*!< [15..15] Pad 41 power switch enable */ + __IOM uint32_t PAD42PULL : 1; /*!< [16..16] Pad 42 pullup enable */ + __IOM uint32_t PAD42INPEN : 1; /*!< [17..17] Pad 42 input enable */ + __IOM uint32_t PAD42STRNG : 1; /*!< [18..18] Pad 42 drive strength */ + __IOM uint32_t PAD42FNCSEL : 3; /*!< [21..19] Pad 42 function select */ + __IOM uint32_t PAD42RSEL : 2; /*!< [23..22] Pad 42 pullup resistor selection. */ + __IOM uint32_t PAD43PULL : 1; /*!< [24..24] Pad 43 pullup enable */ + __IOM uint32_t PAD43INPEN : 1; /*!< [25..25] Pad 43 input enable */ + __IOM uint32_t PAD43STRNG : 1; /*!< [26..26] Pad 43 drive strength */ + __IOM uint32_t PAD43FNCSEL : 3; /*!< [29..27] Pad 43 function select */ + __IOM uint32_t PAD43RSEL : 2; /*!< [31..30] Pad 43 pullup resistor selection. */ + } PADREGK_b; + } ; + + union { + __IOM uint32_t PADREGL; /*!< (@ 0x0000002C) Pad Configuration Register L (Pads 44-47) */ + + struct { + __IOM uint32_t PAD44PULL : 1; /*!< [0..0] Pad 44 pullup enable */ + __IOM uint32_t PAD44INPEN : 1; /*!< [1..1] Pad 44 input enable */ + __IOM uint32_t PAD44STRNG : 1; /*!< [2..2] Pad 44 drive strength */ + __IOM uint32_t PAD44FNCSEL : 3; /*!< [5..3] Pad 44 function select */ + __IM uint32_t : 2; + __IOM uint32_t PAD45PULL : 1; /*!< [8..8] Pad 45 pullup enable */ + __IOM uint32_t PAD45INPEN : 1; /*!< [9..9] Pad 45 input enable */ + __IOM uint32_t PAD45STRNG : 1; /*!< [10..10] Pad 45 drive strength */ + __IOM uint32_t PAD45FNCSEL : 3; /*!< [13..11] Pad 45 function select */ + __IM uint32_t : 2; + __IOM uint32_t PAD46PULL : 1; /*!< [16..16] Pad 46 pullup enable */ + __IOM uint32_t PAD46INPEN : 1; /*!< [17..17] Pad 46 input enable */ + __IOM uint32_t PAD46STRNG : 1; /*!< [18..18] Pad 46 drive strength */ + __IOM uint32_t PAD46FNCSEL : 3; /*!< [21..19] Pad 46 function select */ + __IM uint32_t : 2; + __IOM uint32_t PAD47PULL : 1; /*!< [24..24] Pad 47 pullup enable */ + __IOM uint32_t PAD47INPEN : 1; /*!< [25..25] Pad 47 input enable */ + __IOM uint32_t PAD47STRNG : 1; /*!< [26..26] Pad 47 drive strength */ + __IOM uint32_t PAD47FNCSEL : 3; /*!< [29..27] Pad 47 function select */ + } PADREGL_b; + } ; + + union { + __IOM uint32_t PADREGM; /*!< (@ 0x00000030) Pad Configuration Register M (Pads 47-48) */ + + struct { + __IOM uint32_t PAD48PULL : 1; /*!< [0..0] Pad 48 pullup enable */ + __IOM uint32_t PAD48INPEN : 1; /*!< [1..1] Pad 48 input enable */ + __IOM uint32_t PAD48STRNG : 1; /*!< [2..2] Pad 48 drive strength */ + __IOM uint32_t PAD48FNCSEL : 3; /*!< [5..3] Pad 48 function select */ + __IOM uint32_t PAD48RSEL : 2; /*!< [7..6] Pad 48 pullup resistor selection. */ + __IOM uint32_t PAD49PULL : 1; /*!< [8..8] Pad 49 pullup enable */ + __IOM uint32_t PAD49INPEN : 1; /*!< [9..9] Pad 49 input enable */ + __IOM uint32_t PAD49STRNG : 1; /*!< [10..10] Pad 49 drive strength */ + __IOM uint32_t PAD49FNCSEL : 3; /*!< [13..11] Pad 49 function select */ + __IOM uint32_t PAD49RSEL : 2; /*!< [15..14] Pad 49 pullup resistor selection. */ + } PADREGM_b; + } ; + __IM uint32_t RESERVED[3]; + + union { + __IOM uint32_t CFGA; /*!< (@ 0x00000040) GPIO Configuration Register A (Pads 0-7) */ + + struct { + __IOM uint32_t GPIO0INCFG : 1; /*!< [0..0] GPIO0 input enable. */ + __IOM uint32_t GPIO0OUTCFG : 2; /*!< [2..1] GPIO0 output configuration. */ + __IOM uint32_t GPIO0INTD : 1; /*!< [3..3] GPIO0 interrupt direction. */ + __IOM uint32_t GPIO1INCFG : 1; /*!< [4..4] GPIO1 input enable. */ + __IOM uint32_t GPIO1OUTCFG : 2; /*!< [6..5] GPIO1 output configuration. */ + __IOM uint32_t GPIO1INTD : 1; /*!< [7..7] GPIO1 interrupt direction. */ + __IOM uint32_t GPIO2INCFG : 1; /*!< [8..8] GPIO2 input enable. */ + __IOM uint32_t GPIO2OUTCFG : 2; /*!< [10..9] GPIO2 output configuration. */ + __IOM uint32_t GPIO2INTD : 1; /*!< [11..11] GPIO2 interrupt direction. */ + __IOM uint32_t GPIO3INCFG : 1; /*!< [12..12] GPIO3 input enable. */ + __IOM uint32_t GPIO3OUTCFG : 2; /*!< [14..13] GPIO3 output configuration. */ + __IOM uint32_t GPIO3INTD : 1; /*!< [15..15] GPIO3 interrupt direction. */ + __IOM uint32_t GPIO4INCFG : 1; /*!< [16..16] GPIO4 input enable. */ + __IOM uint32_t GPIO4OUTCFG : 2; /*!< [18..17] GPIO4 output configuration. */ + __IOM uint32_t GPIO4INTD : 1; /*!< [19..19] GPIO4 interrupt direction. */ + __IOM uint32_t GPIO5INCFG : 1; /*!< [20..20] GPIO5 input enable. */ + __IOM uint32_t GPIO5OUTCFG : 2; /*!< [22..21] GPIO5 output configuration. */ + __IOM uint32_t GPIO5INTD : 1; /*!< [23..23] GPIO5 interrupt direction. */ + __IOM uint32_t GPIO6INCFG : 1; /*!< [24..24] GPIO6 input enable. */ + __IOM uint32_t GPIO6OUTCFG : 2; /*!< [26..25] GPIO6 output configuration. */ + __IOM uint32_t GPIO6INTD : 1; /*!< [27..27] GPIO6 interrupt direction. */ + __IOM uint32_t GPIO7INCFG : 1; /*!< [28..28] GPIO7 input enable. */ + __IOM uint32_t GPIO7OUTCFG : 2; /*!< [30..29] GPIO7 output configuration. */ + __IOM uint32_t GPIO7INTD : 1; /*!< [31..31] GPIO7 interrupt direction, nCE polarity. */ + } CFGA_b; + } ; + + union { + __IOM uint32_t CFGB; /*!< (@ 0x00000044) GPIO Configuration Register B (Pads 8-15) */ + + struct { + __IOM uint32_t GPIO8INCFG : 1; /*!< [0..0] GPIO8 input enable. */ + __IOM uint32_t GPIO8OUTCFG : 2; /*!< [2..1] GPIO8 output configuration. */ + __IOM uint32_t GPIO8INTD : 1; /*!< [3..3] GPIO8 interrupt direction. */ + __IOM uint32_t GPIO9INCFG : 1; /*!< [4..4] GPIO9 input enable. */ + __IOM uint32_t GPIO9OUTCFG : 2; /*!< [6..5] GPIO9 output configuration. */ + __IOM uint32_t GPIO9INTD : 1; /*!< [7..7] GPIO9 interrupt direction. */ + __IOM uint32_t GPIO10INCFG : 1; /*!< [8..8] GPIO10 input enable. */ + __IOM uint32_t GPIO10OUTCFG : 2; /*!< [10..9] GPIO10 output configuration. */ + __IOM uint32_t GPIO10INTD : 1; /*!< [11..11] GPIO10 interrupt direction. */ + __IOM uint32_t GPIO11INCFG : 1; /*!< [12..12] GPIO11 input enable. */ + __IOM uint32_t GPIO11OUTCFG : 2; /*!< [14..13] GPIO11 output configuration. */ + __IOM uint32_t GPIO11INTD : 1; /*!< [15..15] GPIO11 interrupt direction. */ + __IOM uint32_t GPIO12INCFG : 1; /*!< [16..16] GPIO12 input enable. */ + __IOM uint32_t GPIO12OUTCFG : 2; /*!< [18..17] GPIO12 output configuration. */ + __IOM uint32_t GPIO12INTD : 1; /*!< [19..19] GPIO12 interrupt direction. */ + __IOM uint32_t GPIO13INCFG : 1; /*!< [20..20] GPIO13 input enable. */ + __IOM uint32_t GPIO13OUTCFG : 2; /*!< [22..21] GPIO13 output configuration. */ + __IOM uint32_t GPIO13INTD : 1; /*!< [23..23] GPIO13 interrupt direction. */ + __IOM uint32_t GPIO14INCFG : 1; /*!< [24..24] GPIO14 input enable. */ + __IOM uint32_t GPIO14OUTCFG : 2; /*!< [26..25] GPIO14 output configuration. */ + __IOM uint32_t GPIO14INTD : 1; /*!< [27..27] GPIO14 interrupt direction. */ + __IOM uint32_t GPIO15INCFG : 1; /*!< [28..28] GPIO15 input enable. */ + __IOM uint32_t GPIO15OUTCFG : 2; /*!< [30..29] GPIO15 output configuration. */ + __IOM uint32_t GPIO15INTD : 1; /*!< [31..31] GPIO15 interrupt direction. */ + } CFGB_b; + } ; + + union { + __IOM uint32_t CFGC; /*!< (@ 0x00000048) GPIO Configuration Register C (Pads 16-23) */ + + struct { + __IOM uint32_t GPIO16INCFG : 1; /*!< [0..0] GPIO16 input enable. */ + __IOM uint32_t GPIO16OUTCFG : 2; /*!< [2..1] GPIO16 output configuration. */ + __IOM uint32_t GPIO16INTD : 1; /*!< [3..3] GPIO16 interrupt direction. */ + __IOM uint32_t GPIO17INCFG : 1; /*!< [4..4] GPIO17 input enable. */ + __IOM uint32_t GPIO17OUTCFG : 2; /*!< [6..5] GPIO17 output configuration. */ + __IOM uint32_t GPIO17INTD : 1; /*!< [7..7] GPIO17 interrupt direction. */ + __IOM uint32_t GPIO18INCFG : 1; /*!< [8..8] GPIO18 input enable. */ + __IOM uint32_t GPIO18OUTCFG : 2; /*!< [10..9] GPIO18 output configuration. */ + __IOM uint32_t GPIO18INTD : 1; /*!< [11..11] GPIO18 interrupt direction. */ + __IOM uint32_t GPIO19INCFG : 1; /*!< [12..12] GPIO19 input enable. */ + __IOM uint32_t GPIO19OUTCFG : 2; /*!< [14..13] GPIO19 output configuration. */ + __IOM uint32_t GPIO19INTD : 1; /*!< [15..15] GPIO19 interrupt direction. */ + __IOM uint32_t GPIO20INCFG : 1; /*!< [16..16] GPIO20 input enable. */ + __IOM uint32_t GPIO20OUTCFG : 2; /*!< [18..17] GPIO20 output configuration. */ + __IOM uint32_t GPIO20INTD : 1; /*!< [19..19] GPIO20 interrupt direction. */ + __IOM uint32_t GPIO21INCFG : 1; /*!< [20..20] GPIO21 input enable. */ + __IOM uint32_t GPIO21OUTCFG : 2; /*!< [22..21] GPIO21 output configuration. */ + __IOM uint32_t GPIO21INTD : 1; /*!< [23..23] GPIO21 interrupt direction. */ + __IOM uint32_t GPIO22INCFG : 1; /*!< [24..24] GPIO22 input enable. */ + __IOM uint32_t GPIO22OUTCFG : 2; /*!< [26..25] GPIO22 output configuration. */ + __IOM uint32_t GPIO22INTD : 1; /*!< [27..27] GPIO22 interrupt direction. */ + __IOM uint32_t GPIO23INCFG : 1; /*!< [28..28] GPIO23 input enable. */ + __IOM uint32_t GPIO23OUTCFG : 2; /*!< [30..29] GPIO23 output configuration. */ + __IOM uint32_t GPIO23INTD : 1; /*!< [31..31] GPIO23 interrupt direction. */ + } CFGC_b; + } ; + + union { + __IOM uint32_t CFGD; /*!< (@ 0x0000004C) GPIO Configuration Register D (Pads 24-31) */ + + struct { + __IOM uint32_t GPIO24INCFG : 1; /*!< [0..0] GPIO24 input enable. */ + __IOM uint32_t GPIO24OUTCFG : 2; /*!< [2..1] GPIO24 output configuration. */ + __IOM uint32_t GPIO24INTD : 1; /*!< [3..3] GPIO24 interrupt direction. */ + __IOM uint32_t GPIO25INCFG : 1; /*!< [4..4] GPIO25 input enable. */ + __IOM uint32_t GPIO25OUTCFG : 2; /*!< [6..5] GPIO25 output configuration. */ + __IOM uint32_t GPIO25INTD : 1; /*!< [7..7] GPIO25 interrupt direction. */ + __IOM uint32_t GPIO26INCFG : 1; /*!< [8..8] GPIO26 input enable. */ + __IOM uint32_t GPIO26OUTCFG : 2; /*!< [10..9] GPIO26 output configuration. */ + __IOM uint32_t GPIO26INTD : 1; /*!< [11..11] GPIO26 interrupt direction. */ + __IOM uint32_t GPIO27INCFG : 1; /*!< [12..12] GPIO27 input enable. */ + __IOM uint32_t GPIO27OUTCFG : 2; /*!< [14..13] GPIO27 output configuration. */ + __IOM uint32_t GPIO27INTD : 1; /*!< [15..15] GPIO27 interrupt direction. */ + __IOM uint32_t GPIO28INCFG : 1; /*!< [16..16] GPIO28 input enable. */ + __IOM uint32_t GPIO28OUTCFG : 2; /*!< [18..17] GPIO28 output configuration. */ + __IOM uint32_t GPIO28INTD : 1; /*!< [19..19] GPIO28 interrupt direction. */ + __IOM uint32_t GPIO29INCFG : 1; /*!< [20..20] GPIO29 input enable. */ + __IOM uint32_t GPIO29OUTCFG : 2; /*!< [22..21] GPIO29 output configuration. */ + __IOM uint32_t GPIO29INTD : 1; /*!< [23..23] GPIO29 interrupt direction. */ + __IOM uint32_t GPIO30INCFG : 1; /*!< [24..24] GPIO30 input enable. */ + __IOM uint32_t GPIO30OUTCFG : 2; /*!< [26..25] GPIO30 output configuration. */ + __IOM uint32_t GPIO30INTD : 1; /*!< [27..27] GPIO30 interrupt direction. */ + __IOM uint32_t GPIO31INCFG : 1; /*!< [28..28] GPIO31 input enable. */ + __IOM uint32_t GPIO31OUTCFG : 2; /*!< [30..29] GPIO31 output configuration. */ + __IOM uint32_t GPIO31INTD : 1; /*!< [31..31] GPIO31 interrupt direction. */ + } CFGD_b; + } ; + + union { + __IOM uint32_t CFGE; /*!< (@ 0x00000050) GPIO Configuration Register E (Pads 32-39) */ + + struct { + __IOM uint32_t GPIO32INCFG : 1; /*!< [0..0] GPIO32 input enable. */ + __IOM uint32_t GPIO32OUTCFG : 2; /*!< [2..1] GPIO32 output configuration. */ + __IOM uint32_t GPIO32INTD : 1; /*!< [3..3] GPIO32 interrupt direction. */ + __IOM uint32_t GPIO33INCFG : 1; /*!< [4..4] GPIO33 input enable. */ + __IOM uint32_t GPIO33OUTCFG : 2; /*!< [6..5] GPIO33 output configuration. */ + __IOM uint32_t GPIO33INTD : 1; /*!< [7..7] GPIO33 interrupt direction. */ + __IOM uint32_t GPIO34INCFG : 1; /*!< [8..8] GPIO34 input enable. */ + __IOM uint32_t GPIO34OUTCFG : 2; /*!< [10..9] GPIO34 output configuration. */ + __IOM uint32_t GPIO34INTD : 1; /*!< [11..11] GPIO34 interrupt direction. */ + __IOM uint32_t GPIO35INCFG : 1; /*!< [12..12] GPIO35 input enable. */ + __IOM uint32_t GPIO35OUTCFG : 2; /*!< [14..13] GPIO35 output configuration. */ + __IOM uint32_t GPIO35INTD : 1; /*!< [15..15] GPIO35 interrupt direction. */ + __IOM uint32_t GPIO36INCFG : 1; /*!< [16..16] GPIO36 input enable. */ + __IOM uint32_t GPIO36OUTCFG : 2; /*!< [18..17] GPIO36 output configuration. */ + __IOM uint32_t GPIO36INTD : 1; /*!< [19..19] GPIO36 interrupt direction. */ + __IOM uint32_t GPIO37INCFG : 1; /*!< [20..20] GPIO37 input enable. */ + __IOM uint32_t GPIO37OUTCFG : 2; /*!< [22..21] GPIO37 output configuration. */ + __IOM uint32_t GPIO37INTD : 1; /*!< [23..23] GPIO37 interrupt direction. */ + __IOM uint32_t GPIO38INCFG : 1; /*!< [24..24] GPIO38 input enable. */ + __IOM uint32_t GPIO38OUTCFG : 2; /*!< [26..25] GPIO38 output configuration. */ + __IOM uint32_t GPIO38INTD : 1; /*!< [27..27] GPIO38 interrupt direction. */ + __IOM uint32_t GPIO39INCFG : 1; /*!< [28..28] GPIO39 input enable. */ + __IOM uint32_t GPIO39OUTCFG : 2; /*!< [30..29] GPIO39 output configuration. */ + __IOM uint32_t GPIO39INTD : 1; /*!< [31..31] GPIO39 interrupt direction. */ + } CFGE_b; + } ; + + union { + __IOM uint32_t CFGF; /*!< (@ 0x00000054) GPIO Configuration Register F (Pads 40 -47) */ + + struct { + __IOM uint32_t GPIO40INCFG : 1; /*!< [0..0] GPIO40 input enable. */ + __IOM uint32_t GPIO40OUTCFG : 2; /*!< [2..1] GPIO40 output configuration. */ + __IOM uint32_t GPIO40INTD : 1; /*!< [3..3] GPIO40 interrupt direction. */ + __IOM uint32_t GPIO41INCFG : 1; /*!< [4..4] GPIO41 input enable. */ + __IOM uint32_t GPIO41OUTCFG : 2; /*!< [6..5] GPIO41 output configuration. */ + __IOM uint32_t GPIO41INTD : 1; /*!< [7..7] GPIO41 interrupt direction. */ + __IOM uint32_t GPIO42INCFG : 1; /*!< [8..8] GPIO42 input enable. */ + __IOM uint32_t GPIO42OUTCFG : 2; /*!< [10..9] GPIO42 output configuration. */ + __IOM uint32_t GPIO42INTD : 1; /*!< [11..11] GPIO42 interrupt direction. */ + __IOM uint32_t GPIO43INCFG : 1; /*!< [12..12] GPIO43 input enable. */ + __IOM uint32_t GPIO43OUTCFG : 2; /*!< [14..13] GPIO43 output configuration. */ + __IOM uint32_t GPIO43INTD : 1; /*!< [15..15] GPIO43 interrupt direction. */ + __IOM uint32_t GPIO44INCFG : 1; /*!< [16..16] GPIO44 input enable. */ + __IOM uint32_t GPIO44OUTCFG : 2; /*!< [18..17] GPIO44 output configuration. */ + __IOM uint32_t GPIO44INTD : 1; /*!< [19..19] GPIO44 interrupt direction. */ + __IOM uint32_t GPIO45INCFG : 1; /*!< [20..20] GPIO45 input enable. */ + __IOM uint32_t GPIO45OUTCFG : 2; /*!< [22..21] GPIO45 output configuration. */ + __IOM uint32_t GPIO45INTD : 1; /*!< [23..23] GPIO45 interrupt direction. */ + __IOM uint32_t GPIO46INCFG : 1; /*!< [24..24] GPIO46 input enable. */ + __IOM uint32_t GPIO46OUTCFG : 2; /*!< [26..25] GPIO46 output configuration. */ + __IOM uint32_t GPIO46INTD : 1; /*!< [27..27] GPIO46 interrupt direction. */ + __IOM uint32_t GPIO47INCFG : 1; /*!< [28..28] GPIO47 input enable. */ + __IOM uint32_t GPIO47OUTCFG : 2; /*!< [30..29] GPIO47 output configuration. */ + __IOM uint32_t GPIO47INTD : 1; /*!< [31..31] GPIO47 interrupt direction. */ + } CFGF_b; + } ; + + union { + __IOM uint32_t CFGG; /*!< (@ 0x00000058) GPIO Configuration Register G (Pads 48-49) */ + + struct { + __IOM uint32_t GPIO48INCFG : 1; /*!< [0..0] GPIO48 input enable. */ + __IOM uint32_t GPIO48OUTCFG : 2; /*!< [2..1] GPIO48 output configuration. */ + __IOM uint32_t GPIO48INTD : 1; /*!< [3..3] GPIO48 interrupt direction. */ + __IOM uint32_t GPIO49INCFG : 1; /*!< [4..4] GPIO49 input enable. */ + __IOM uint32_t GPIO49OUTCFG : 2; /*!< [6..5] GPIO49 output configuration. */ + __IOM uint32_t GPIO49INTD : 1; /*!< [7..7] GPIO49 interrupt direction. */ + } CFGG_b; + } ; + __IM uint32_t RESERVED1; + + union { + __IOM uint32_t PADKEY; /*!< (@ 0x00000060) Key Register for all pad configuration registers */ + + struct { + __IOM uint32_t PADKEY : 32; /*!< [31..0] Key register value. */ + } PADKEY_b; + } ; + __IM uint32_t RESERVED2[7]; + + union { + __IOM uint32_t RDA; /*!< (@ 0x00000080) GPIO Input Register A */ + + struct { + __IOM uint32_t RDA : 32; /*!< [31..0] GPIO31-0 read data. */ + } RDA_b; + } ; + + union { + __IOM uint32_t RDB; /*!< (@ 0x00000084) GPIO Input Register B */ + + struct { + __IOM uint32_t RDB : 18; /*!< [17..0] GPIO49-32 read data. */ + } RDB_b; + } ; + + union { + __IOM uint32_t WTA; /*!< (@ 0x00000088) GPIO Output Register A */ + + struct { + __IOM uint32_t WTA : 32; /*!< [31..0] GPIO31-0 write data. */ + } WTA_b; + } ; + + union { + __IOM uint32_t WTB; /*!< (@ 0x0000008C) GPIO Output Register B */ + + struct { + __IOM uint32_t WTB : 18; /*!< [17..0] GPIO49-32 write data. */ + } WTB_b; + } ; + + union { + __IOM uint32_t WTSA; /*!< (@ 0x00000090) GPIO Output Register A Set */ + + struct { + __IOM uint32_t WTSA : 32; /*!< [31..0] Set the GPIO31-0 write data. */ + } WTSA_b; + } ; + + union { + __IOM uint32_t WTSB; /*!< (@ 0x00000094) GPIO Output Register B Set */ + + struct { + __IOM uint32_t WTSB : 18; /*!< [17..0] Set the GPIO49-32 write data. */ + } WTSB_b; + } ; + + union { + __IOM uint32_t WTCA; /*!< (@ 0x00000098) GPIO Output Register A Clear */ + + struct { + __IOM uint32_t WTCA : 32; /*!< [31..0] Clear the GPIO31-0 write data. */ + } WTCA_b; + } ; + + union { + __IOM uint32_t WTCB; /*!< (@ 0x0000009C) GPIO Output Register B Clear */ + + struct { + __IOM uint32_t WTCB : 18; /*!< [17..0] Clear the GPIO49-32 write data. */ + } WTCB_b; + } ; + + union { + __IOM uint32_t ENA; /*!< (@ 0x000000A0) GPIO Enable Register A */ + + struct { + __IOM uint32_t ENA : 32; /*!< [31..0] GPIO31-0 output enables */ + } ENA_b; + } ; + + union { + __IOM uint32_t ENB; /*!< (@ 0x000000A4) GPIO Enable Register B */ + + struct { + __IOM uint32_t ENB : 18; /*!< [17..0] GPIO49-32 output enables */ + } ENB_b; + } ; + + union { + __IOM uint32_t ENSA; /*!< (@ 0x000000A8) GPIO Enable Register A Set */ + + struct { + __IOM uint32_t ENSA : 32; /*!< [31..0] Set the GPIO31-0 output enables */ + } ENSA_b; + } ; + + union { + __IOM uint32_t ENSB; /*!< (@ 0x000000AC) GPIO Enable Register B Set */ + + struct { + __IOM uint32_t ENSB : 18; /*!< [17..0] Set the GPIO49-32 output enables */ + } ENSB_b; + } ; + __IM uint32_t RESERVED3; + + union { + __IOM uint32_t ENCA; /*!< (@ 0x000000B4) GPIO Enable Register A Clear */ + + struct { + __IOM uint32_t ENCA : 32; /*!< [31..0] Clear the GPIO31-0 output enables */ + } ENCA_b; + } ; + + union { + __IOM uint32_t ENCB; /*!< (@ 0x000000B8) GPIO Enable Register B Clear */ + + struct { + __IOM uint32_t ENCB : 18; /*!< [17..0] Clear the GPIO49-32 output enables */ + } ENCB_b; + } ; + + union { + __IOM uint32_t STMRCAP; /*!< (@ 0x000000BC) STIMER Capture Control */ + + struct { + __IOM uint32_t STSEL0 : 6; /*!< [5..0] STIMER Capture 0 Select. */ + __IOM uint32_t STPOL0 : 1; /*!< [6..6] STIMER Capture 0 Polarity. */ + __IM uint32_t : 1; + __IOM uint32_t STSEL1 : 6; /*!< [13..8] STIMER Capture 1 Select. */ + __IOM uint32_t STPOL1 : 1; /*!< [14..14] STIMER Capture 1 Polarity. */ + __IM uint32_t : 1; + __IOM uint32_t STSEL2 : 6; /*!< [21..16] STIMER Capture 2 Select. */ + __IOM uint32_t STPOL2 : 1; /*!< [22..22] STIMER Capture 2 Polarity. */ + __IM uint32_t : 1; + __IOM uint32_t STSEL3 : 6; /*!< [29..24] STIMER Capture 3 Select. */ + __IOM uint32_t STPOL3 : 1; /*!< [30..30] STIMER Capture 3 Polarity. */ + } STMRCAP_b; + } ; + + union { + __IOM uint32_t IOM0IRQ; /*!< (@ 0x000000C0) IOM0 Flow Control IRQ Select */ + + struct { + __IOM uint32_t IOM0IRQ : 6; /*!< [5..0] IOMSTR0 IRQ pad select. */ + } IOM0IRQ_b; + } ; + + union { + __IOM uint32_t IOM1IRQ; /*!< (@ 0x000000C4) IOM1 Flow Control IRQ Select */ + + struct { + __IOM uint32_t IOM1IRQ : 6; /*!< [5..0] IOMSTR1 IRQ pad select. */ + } IOM1IRQ_b; + } ; + + union { + __IOM uint32_t IOM2IRQ; /*!< (@ 0x000000C8) IOM2 Flow Control IRQ Select */ + + struct { + __IOM uint32_t IOM2IRQ : 6; /*!< [5..0] IOMSTR2 IRQ pad select. */ + } IOM2IRQ_b; + } ; + + union { + __IOM uint32_t IOM3IRQ; /*!< (@ 0x000000CC) IOM3 Flow Control IRQ Select */ + + struct { + __IOM uint32_t IOM3IRQ : 6; /*!< [5..0] IOMSTR3 IRQ pad select. */ + } IOM3IRQ_b; + } ; + + union { + __IOM uint32_t IOM4IRQ; /*!< (@ 0x000000D0) IOM4 Flow Control IRQ Select */ + + struct { + __IOM uint32_t IOM4IRQ : 6; /*!< [5..0] IOMSTR4 IRQ pad select. */ + } IOM4IRQ_b; + } ; + + union { + __IOM uint32_t IOM5IRQ; /*!< (@ 0x000000D4) IOM5 Flow Control IRQ Select */ + + struct { + __IOM uint32_t IOM5IRQ : 6; /*!< [5..0] IOMSTR5 IRQ pad select. */ + } IOM5IRQ_b; + } ; + + union { + __IOM uint32_t BLEIFIRQ; /*!< (@ 0x000000D8) BLEIF Flow Control IRQ Select */ + + struct { + __IOM uint32_t BLEIFIRQ : 6; /*!< [5..0] BLEIF IRQ pad select. */ + } BLEIFIRQ_b; + } ; + + union { + __IOM uint32_t GPIOOBS; /*!< (@ 0x000000DC) GPIO Observation Mode Sample register */ + + struct { + __IOM uint32_t OBS_DATA : 16; /*!< [15..0] Sample of the data output on the GPIO observation port. + May have async sampling issues, as the data is not synronized + to the read operation. Intended for debug purposes only */ + } GPIOOBS_b; + } ; + + union { + __IOM uint32_t ALTPADCFGA; /*!< (@ 0x000000E0) Alternate Pad Configuration reg0 (Pads 3,2,1,0) */ + + struct { + __IOM uint32_t PAD0_DS1 : 1; /*!< [0..0] Pad 0 high order drive strength selection. Used in conjunction + with PAD0STRNG field to set the pad drive strength. */ + __IM uint32_t : 3; + __IOM uint32_t PAD0_SR : 1; /*!< [4..4] Pad 0 slew rate selection. */ + __IM uint32_t : 3; + __IOM uint32_t PAD1_DS1 : 1; /*!< [8..8] Pad 1 high order drive strength selection. Used in conjunction + with PAD1STRNG field to set the pad drive strength. */ + __IM uint32_t : 3; + __IOM uint32_t PAD1_SR : 1; /*!< [12..12] Pad 1 slew rate selection. */ + __IM uint32_t : 3; + __IOM uint32_t PAD2_DS1 : 1; /*!< [16..16] Pad 2 high order drive strength selection. Used in + conjunction with PAD2STRNG field to set the pad drive strength. */ + __IM uint32_t : 3; + __IOM uint32_t PAD2_SR : 1; /*!< [20..20] Pad 2 slew rate selection. */ + __IM uint32_t : 3; + __IOM uint32_t PAD3_DS1 : 1; /*!< [24..24] Pad 3 high order drive strength selection. Used in + conjunction with PAD3STRNG field to set the pad drive strength. */ + __IM uint32_t : 3; + __IOM uint32_t PAD3_SR : 1; /*!< [28..28] Pad 3 slew rate selection. */ + } ALTPADCFGA_b; + } ; + + union { + __IOM uint32_t ALTPADCFGB; /*!< (@ 0x000000E4) Alternate Pad Configuration reg1 (Pads 7,6,5,4) */ + + struct { + __IOM uint32_t PAD4_DS1 : 1; /*!< [0..0] Pad 4 high order drive strength selection. Used in conjunction + with PAD4STRNG field to set the pad drive strength. */ + __IM uint32_t : 3; + __IOM uint32_t PAD4_SR : 1; /*!< [4..4] Pad 4 slew rate selection. */ + __IM uint32_t : 3; + __IOM uint32_t PAD5_DS1 : 1; /*!< [8..8] Pad 5 high order drive strength selection. Used in conjunction + with PAD5STRNG field to set the pad drive strength. */ + __IM uint32_t : 3; + __IOM uint32_t PAD5_SR : 1; /*!< [12..12] Pad 5 slew rate selection. */ + __IM uint32_t : 3; + __IOM uint32_t PAD6_DS1 : 1; /*!< [16..16] Pad 6 high order drive strength selection. Used in + conjunction with PAD6STRNG field to set the pad drive strength. */ + __IM uint32_t : 3; + __IOM uint32_t PAD6_SR : 1; /*!< [20..20] Pad 6 slew rate selection. */ + __IM uint32_t : 3; + __IOM uint32_t PAD7_DS1 : 1; /*!< [24..24] Pad 7 high order drive strength selection. Used in + conjunction with PAD7STRNG field to set the pad drive strength. */ + __IM uint32_t : 3; + __IOM uint32_t PAD7_SR : 1; /*!< [28..28] Pad 7 slew rate selection. */ + } ALTPADCFGB_b; + } ; + + union { + __IOM uint32_t ALTPADCFGC; /*!< (@ 0x000000E8) Alternate Pad Configuration reg2 (Pads 11,10,9,8) */ + + struct { + __IOM uint32_t PAD8_DS1 : 1; /*!< [0..0] Pad 8 high order drive strength selection. Used in conjunction + with PAD8STRNG field to set the pad drive strength. */ + __IM uint32_t : 3; + __IOM uint32_t PAD8_SR : 1; /*!< [4..4] Pad 8 slew rate selection. */ + __IM uint32_t : 3; + __IOM uint32_t PAD9_DS1 : 1; /*!< [8..8] Pad 9 high order drive strength selection. Used in conjunction + with PAD9STRNG field to set the pad drive strength. */ + __IM uint32_t : 3; + __IOM uint32_t PAD9_SR : 1; /*!< [12..12] Pad 9 slew rate selection. */ + __IM uint32_t : 3; + __IOM uint32_t PAD10_DS1 : 1; /*!< [16..16] Pad 10 high order drive strength selection. Used in + conjunction with PAD10STRNG field to set the pad drive + strength. */ + __IM uint32_t : 3; + __IOM uint32_t PAD10_SR : 1; /*!< [20..20] Pad 10 slew rate selection. */ + __IM uint32_t : 3; + __IOM uint32_t PAD11_DS1 : 1; /*!< [24..24] Pad 11 high order drive strength selection. Used in + conjunction with PAD11STRNG field to set the pad drive + strength. */ + __IM uint32_t : 3; + __IOM uint32_t PAD11_SR : 1; /*!< [28..28] Pad 11 slew rate selection. */ + } ALTPADCFGC_b; + } ; + + union { + __IOM uint32_t ALTPADCFGD; /*!< (@ 0x000000EC) Alternate Pad Configuration reg3 (Pads 15,14,13,12) */ + + struct { + __IOM uint32_t PAD12_DS1 : 1; /*!< [0..0] Pad 12 high order drive strength selection. Used in conjunction + with PAD12STRNG field to set the pad drive strength. */ + __IM uint32_t : 3; + __IOM uint32_t PAD12_SR : 1; /*!< [4..4] Pad 12 slew rate selection. */ + __IM uint32_t : 3; + __IOM uint32_t PAD13_DS1 : 1; /*!< [8..8] Pad 13 high order drive strength selection. Used in conjunction + with PAD13STRNG field to set the pad drive strength. */ + __IM uint32_t : 3; + __IOM uint32_t PAD13_SR : 1; /*!< [12..12] Pad 13 slew rate selection. */ + __IM uint32_t : 3; + __IOM uint32_t PAD14_DS1 : 1; /*!< [16..16] Pad 14 high order drive strength selection. Used in + conjunction with PAD14STRNG field to set the pad drive + strength. */ + __IM uint32_t : 3; + __IOM uint32_t PAD14_SR : 1; /*!< [20..20] Pad 14 slew rate selection. */ + __IM uint32_t : 3; + __IOM uint32_t PAD15_DS1 : 1; /*!< [24..24] Pad 15 high order drive strength selection. Used in + conjunction with PAD15STRNG field to set the pad drive + strength. */ + __IM uint32_t : 3; + __IOM uint32_t PAD15_SR : 1; /*!< [28..28] Pad 15 slew rate selection. */ + } ALTPADCFGD_b; + } ; + + union { + __IOM uint32_t ALTPADCFGE; /*!< (@ 0x000000F0) Alternate Pad Configuration reg4 (Pads 19,18,17,16) */ + + struct { + __IOM uint32_t PAD16_DS1 : 1; /*!< [0..0] Pad 16 high order drive strength selection. Used in conjunction + with PAD16STRNG field to set the pad drive strength. */ + __IM uint32_t : 3; + __IOM uint32_t PAD16_SR : 1; /*!< [4..4] Pad 16 slew rate selection. */ + __IM uint32_t : 3; + __IOM uint32_t PAD17_DS1 : 1; /*!< [8..8] Pad 17 high order drive strength selection. Used in conjunction + with PAD17STRNG field to set the pad drive strength. */ + __IM uint32_t : 3; + __IOM uint32_t PAD17_SR : 1; /*!< [12..12] Pad 17 slew rate selection. */ + __IM uint32_t : 3; + __IOM uint32_t PAD18_DS1 : 1; /*!< [16..16] Pad 18 high order drive strength selection. Used in + conjunction with PAD18STRNG field to set the pad drive + strength. */ + __IM uint32_t : 3; + __IOM uint32_t PAD18_SR : 1; /*!< [20..20] Pad 18 slew rate selection. */ + __IM uint32_t : 3; + __IOM uint32_t PAD19_DS1 : 1; /*!< [24..24] Pad 19 high order drive strength selection. Used in + conjunction with PAD19STRNG field to set the pad drive + strength. */ + __IM uint32_t : 3; + __IOM uint32_t PAD19_SR : 1; /*!< [28..28] Pad 19 slew rate selection. */ + } ALTPADCFGE_b; + } ; + + union { + __IOM uint32_t ALTPADCFGF; /*!< (@ 0x000000F4) Alternate Pad Configuration reg5 (Pads 23,22,21,20) */ + + struct { + __IOM uint32_t PAD20_DS1 : 1; /*!< [0..0] Pad 20 high order drive strength selection. Used in conjunction + with PAD20STRNG field to set the pad drive strength. */ + __IM uint32_t : 3; + __IOM uint32_t PAD20_SR : 1; /*!< [4..4] Pad 20 slew rate selection. */ + __IM uint32_t : 3; + __IOM uint32_t PAD21_DS1 : 1; /*!< [8..8] Pad 21 high order drive strength selection. Used in conjunction + with PAD21STRNG field to set the pad drive strength. */ + __IM uint32_t : 3; + __IOM uint32_t PAD21_SR : 1; /*!< [12..12] Pad 21 slew rate selection. */ + __IM uint32_t : 3; + __IOM uint32_t PAD22_DS1 : 1; /*!< [16..16] Pad 22 high order drive strength selection. Used in + conjunction with PAD22STRNG field to set the pad drive + strength. */ + __IM uint32_t : 3; + __IOM uint32_t PAD22_SR : 1; /*!< [20..20] Pad 22 slew rate selection. */ + __IM uint32_t : 3; + __IOM uint32_t PAD23_DS1 : 1; /*!< [24..24] Pad 23 high order drive strength selection. Used in + conjunction with PAD23STRNG field to set the pad drive + strength. */ + __IM uint32_t : 3; + __IOM uint32_t PAD23_SR : 1; /*!< [28..28] Pad 23 slew rate selection. */ + } ALTPADCFGF_b; + } ; + + union { + __IOM uint32_t ALTPADCFGG; /*!< (@ 0x000000F8) Alternate Pad Configuration reg6 (Pads 27,26,25,24) */ + + struct { + __IOM uint32_t PAD24_DS1 : 1; /*!< [0..0] Pad 24 high order drive strength selection. Used in conjunction + with PAD24STRNG field to set the pad drive strength. */ + __IM uint32_t : 3; + __IOM uint32_t PAD24_SR : 1; /*!< [4..4] Pad 24 slew rate selection. */ + __IM uint32_t : 3; + __IOM uint32_t PAD25_DS1 : 1; /*!< [8..8] Pad 25 high order drive strength selection. Used in conjunction + with PAD25STRNG field to set the pad drive strength. */ + __IM uint32_t : 3; + __IOM uint32_t PAD25_SR : 1; /*!< [12..12] Pad 25 slew rate selection. */ + __IM uint32_t : 3; + __IOM uint32_t PAD26_DS1 : 1; /*!< [16..16] Pad 26 high order drive strength selection. Used in + conjunction with PAD26STRNG field to set the pad drive + strength. */ + __IM uint32_t : 3; + __IOM uint32_t PAD26_SR : 1; /*!< [20..20] Pad 26 slew rate selection. */ + __IM uint32_t : 3; + __IOM uint32_t PAD27_DS1 : 1; /*!< [24..24] Pad 27 high order drive strength selection. Used in + conjunction with PAD27STRNG field to set the pad drive + strength. */ + __IM uint32_t : 3; + __IOM uint32_t PAD27_SR : 1; /*!< [28..28] Pad 27 slew rate selection. */ + } ALTPADCFGG_b; + } ; + + union { + __IOM uint32_t ALTPADCFGH; /*!< (@ 0x000000FC) Alternate Pad Configuration reg7 (Pads 31,30,29,28) */ + + struct { + __IOM uint32_t PAD28_DS1 : 1; /*!< [0..0] Pad 28 high order drive strength selection. Used in conjunction + with PAD28STRNG field to set the pad drive strength. */ + __IM uint32_t : 3; + __IOM uint32_t PAD28_SR : 1; /*!< [4..4] Pad 28 slew rate selection. */ + __IM uint32_t : 3; + __IOM uint32_t PAD29_DS1 : 1; /*!< [8..8] Pad 29 high order drive strength selection. Used in conjunction + with PAD29STRNG field to set the pad drive strength. */ + __IM uint32_t : 3; + __IOM uint32_t PAD29_SR : 1; /*!< [12..12] Pad 29 slew rate selection. */ + __IM uint32_t : 3; + __IOM uint32_t PAD30_DS1 : 1; /*!< [16..16] Pad 30 high order drive strength selection. Used in + conjunction with PAD30STRNG field to set the pad drive + strength. */ + __IM uint32_t : 3; + __IOM uint32_t PAD30_SR : 1; /*!< [20..20] Pad 30 slew rate selection. */ + __IM uint32_t : 3; + __IOM uint32_t PAD31_DS1 : 1; /*!< [24..24] Pad 31 high order drive strength selection. Used in + conjunction with PAD31STRNG field to set the pad drive + strength. */ + __IM uint32_t : 3; + __IOM uint32_t PAD31_SR : 1; /*!< [28..28] Pad 31 slew rate selection. */ + } ALTPADCFGH_b; + } ; + + union { + __IOM uint32_t ALTPADCFGI; /*!< (@ 0x00000100) Alternate Pad Configuration reg8 (Pads 35,34,33,32) */ + + struct { + __IOM uint32_t PAD32_DS1 : 1; /*!< [0..0] Pad 32 high order drive strength selection. Used in conjunction + with PAD32STRNG field to set the pad drive strength. */ + __IM uint32_t : 3; + __IOM uint32_t PAD32_SR : 1; /*!< [4..4] Pad 32 slew rate selection. */ + __IM uint32_t : 3; + __IOM uint32_t PAD33_DS1 : 1; /*!< [8..8] Pad 33 high order drive strength selection. Used in conjunction + with PAD33STRNG field to set the pad drive strength. */ + __IM uint32_t : 3; + __IOM uint32_t PAD33_SR : 1; /*!< [12..12] Pad 33 slew rate selection. */ + __IM uint32_t : 3; + __IOM uint32_t PAD34_DS1 : 1; /*!< [16..16] Pad 34 high order drive strength selection. Used in + conjunction with PAD34STRNG field to set the pad drive + strength. */ + __IM uint32_t : 3; + __IOM uint32_t PAD34_SR : 1; /*!< [20..20] Pad 34 slew rate selection. */ + __IM uint32_t : 3; + __IOM uint32_t PAD35_DS1 : 1; /*!< [24..24] Pad 35 high order drive strength selection. Used in + conjunction with PAD35STRNG field to set the pad drive + strength. */ + __IM uint32_t : 3; + __IOM uint32_t PAD35_SR : 1; /*!< [28..28] Pad 35 slew rate selection. */ + } ALTPADCFGI_b; + } ; + + union { + __IOM uint32_t ALTPADCFGJ; /*!< (@ 0x00000104) Alternate Pad Configuration reg9 (Pads 39,38,37,36) */ + + struct { + __IOM uint32_t PAD36_DS1 : 1; /*!< [0..0] Pad 36 high order drive strength selection. Used in conjunction + with PAD36STRNG field to set the pad drive strength. */ + __IM uint32_t : 3; + __IOM uint32_t PAD36_SR : 1; /*!< [4..4] Pad 36 slew rate selection. */ + __IM uint32_t : 3; + __IOM uint32_t PAD37_DS1 : 1; /*!< [8..8] Pad 37 high order drive strength selection. Used in conjunction + with PAD37STRNG field to set the pad drive strength. */ + __IM uint32_t : 3; + __IOM uint32_t PAD37_SR : 1; /*!< [12..12] Pad 37 slew rate selection. */ + __IM uint32_t : 3; + __IOM uint32_t PAD38_DS1 : 1; /*!< [16..16] Pad 38 high order drive strength selection. Used in + conjunction with PAD38STRNG field to set the pad drive + strength. */ + __IM uint32_t : 3; + __IOM uint32_t PAD38_SR : 1; /*!< [20..20] Pad 38 slew rate selection. */ + __IM uint32_t : 3; + __IOM uint32_t PAD39_DS1 : 1; /*!< [24..24] Pad 39 high order drive strength selection. Used in + conjunction with PAD39STRNG field to set the pad drive + strength. */ + __IM uint32_t : 3; + __IOM uint32_t PAD39_SR : 1; /*!< [28..28] Pad 39 slew rate selection. */ + } ALTPADCFGJ_b; + } ; + + union { + __IOM uint32_t ALTPADCFGK; /*!< (@ 0x00000108) Alternate Pad Configuration reg10 (Pads 43,42,41,40) */ + + struct { + __IOM uint32_t PAD40_DS1 : 1; /*!< [0..0] Pad 40 high order drive strength selection. Used in conjunction + with PAD40STRNG field to set the pad drive strength. */ + __IM uint32_t : 3; + __IOM uint32_t PAD40_SR : 1; /*!< [4..4] Pad 40 slew rate selection. */ + __IM uint32_t : 3; + __IOM uint32_t PAD41_DS1 : 1; /*!< [8..8] Pad 41 high order drive strength selection. Used in conjunction + with PAD41STRNG field to set the pad drive strength. */ + __IM uint32_t : 3; + __IOM uint32_t PAD41_SR : 1; /*!< [12..12] Pad 41 slew rate selection. */ + __IM uint32_t : 3; + __IOM uint32_t PAD42_DS1 : 1; /*!< [16..16] Pad 42 high order drive strength selection. Used in + conjunction with PAD42STRNG field to set the pad drive + strength. */ + __IM uint32_t : 3; + __IOM uint32_t PAD42_SR : 1; /*!< [20..20] Pad 42 slew rate selection. */ + __IM uint32_t : 3; + __IOM uint32_t PAD43_DS1 : 1; /*!< [24..24] Pad 43 high order drive strength selection. Used in + conjunction with PAD43STRNG field to set the pad drive + strength. */ + __IM uint32_t : 3; + __IOM uint32_t PAD43_SR : 1; /*!< [28..28] Pad 43 slew rate selection. */ + } ALTPADCFGK_b; + } ; + + union { + __IOM uint32_t ALTPADCFGL; /*!< (@ 0x0000010C) Alternate Pad Configuration reg11 (Pads 47,46,45,44) */ + + struct { + __IOM uint32_t PAD44_DS1 : 1; /*!< [0..0] Pad 44 high order drive strength selection. Used in conjunction + with PAD44STRNG field to set the pad drive strength. */ + __IM uint32_t : 3; + __IOM uint32_t PAD44_SR : 1; /*!< [4..4] Pad 44 slew rate selection. */ + __IM uint32_t : 3; + __IOM uint32_t PAD45_DS1 : 1; /*!< [8..8] Pad 45 high order drive strength selection. Used in conjunction + with PAD45STRNG field to set the pad drive strength. */ + __IM uint32_t : 3; + __IOM uint32_t PAD45_SR : 1; /*!< [12..12] Pad 45 slew rate selection. */ + __IM uint32_t : 3; + __IOM uint32_t PAD46_DS1 : 1; /*!< [16..16] Pad 46 high order drive strength selection. Used in + conjunction with PAD46STRNG field to set the pad drive + strength. */ + __IM uint32_t : 3; + __IOM uint32_t PAD46_SR : 1; /*!< [20..20] Pad 46 slew rate selection. */ + __IM uint32_t : 3; + __IOM uint32_t PAD47_DS1 : 1; /*!< [24..24] Pad 47 high order drive strength selection. Used in + conjunction with PAD47STRNG field to set the pad drive + strength. */ + __IM uint32_t : 3; + __IOM uint32_t PAD47_SR : 1; /*!< [28..28] Pad 47 slew rate selection. */ + } ALTPADCFGL_b; + } ; + + union { + __IOM uint32_t ALTPADCFGM; /*!< (@ 0x00000110) Alternate Pad Configuration reg12 (Pads 49,48) */ + + struct { + __IOM uint32_t PAD48_DS1 : 1; /*!< [0..0] Pad 48 high order drive strength selection. Used in conjunction + with PAD48STRNG field to set the pad drive strength. */ + __IM uint32_t : 3; + __IOM uint32_t PAD48_SR : 1; /*!< [4..4] Pad 48 slew rate selection. */ + __IM uint32_t : 3; + __IOM uint32_t PAD49_DS1 : 1; /*!< [8..8] Pad 49 high order drive strength selection. Used in conjunction + with PAD49STRNG field to set the pad drive strength. */ + __IM uint32_t : 3; + __IOM uint32_t PAD49_SR : 1; /*!< [12..12] Pad 49 slew rate selection. */ + } ALTPADCFGM_b; + } ; + + union { + __IOM uint32_t SCDET; /*!< (@ 0x00000114) SCARD Card Detect select */ + + struct { + __IOM uint32_t SCDET : 6; /*!< [5..0] SCARD card detect pad select. */ + } SCDET_b; + } ; + + union { + __IOM uint32_t CTENCFG; /*!< (@ 0x00000118) Counter/Timer Enable Config */ + + struct { + __IOM uint32_t EN0 : 1; /*!< [0..0] CT0 Enable */ + __IOM uint32_t EN1 : 1; /*!< [1..1] CT1 Enable */ + __IOM uint32_t EN2 : 1; /*!< [2..2] CT2 Enable */ + __IOM uint32_t EN3 : 1; /*!< [3..3] CT3 Enable */ + __IOM uint32_t EN4 : 1; /*!< [4..4] CT4 Enable */ + __IOM uint32_t EN5 : 1; /*!< [5..5] CT5 Enable */ + __IOM uint32_t EN6 : 1; /*!< [6..6] CT6 Enable */ + __IOM uint32_t EN7 : 1; /*!< [7..7] CT7 Enable */ + __IOM uint32_t EN8 : 1; /*!< [8..8] CT8 Enable */ + __IOM uint32_t EN9 : 1; /*!< [9..9] CT9 Enable */ + __IOM uint32_t EN10 : 1; /*!< [10..10] CT10 Enable */ + __IOM uint32_t EN11 : 1; /*!< [11..11] CT11 Enable */ + __IOM uint32_t EN12 : 1; /*!< [12..12] CT12 Enable */ + __IOM uint32_t EN13 : 1; /*!< [13..13] CT13 Enable */ + __IOM uint32_t EN14 : 1; /*!< [14..14] CT14 Enable */ + __IOM uint32_t EN15 : 1; /*!< [15..15] CT15 Enable */ + __IOM uint32_t EN16 : 1; /*!< [16..16] CT16 Enable */ + __IOM uint32_t EN17 : 1; /*!< [17..17] CT17 Enable */ + __IOM uint32_t EN18 : 1; /*!< [18..18] CT18 Enable */ + __IOM uint32_t EN19 : 1; /*!< [19..19] CT19 Enable */ + __IOM uint32_t EN20 : 1; /*!< [20..20] CT20 Enable */ + __IOM uint32_t EN21 : 1; /*!< [21..21] CT21 Enable */ + __IOM uint32_t EN22 : 1; /*!< [22..22] CT22 Enable */ + __IOM uint32_t EN23 : 1; /*!< [23..23] CT23 Enable */ + __IOM uint32_t EN24 : 1; /*!< [24..24] CT24 Enable */ + __IOM uint32_t EN25 : 1; /*!< [25..25] CT25 Enable */ + __IOM uint32_t EN26 : 1; /*!< [26..26] CT26 Enable */ + __IOM uint32_t EN27 : 1; /*!< [27..27] CT27 Enable */ + __IOM uint32_t EN28 : 1; /*!< [28..28] CT28 Enable */ + __IOM uint32_t EN29 : 1; /*!< [29..29] CT29 Enable */ + __IOM uint32_t EN30 : 1; /*!< [30..30] CT30 Enable */ + __IOM uint32_t EN31 : 1; /*!< [31..31] CT31 Enable */ + } CTENCFG_b; + } ; + __IM uint32_t RESERVED4[57]; + + union { + __IOM uint32_t INT0EN; /*!< (@ 0x00000200) GPIO Interrupt Registers 31-0: Enable */ + + struct { + __IOM uint32_t GPIO0 : 1; /*!< [0..0] GPIO0 interrupt. */ + __IOM uint32_t GPIO1 : 1; /*!< [1..1] GPIO1 interrupt. */ + __IOM uint32_t GPIO2 : 1; /*!< [2..2] GPIO2 interrupt. */ + __IOM uint32_t GPIO3 : 1; /*!< [3..3] GPIO3 interrupt. */ + __IOM uint32_t GPIO4 : 1; /*!< [4..4] GPIO4 interrupt. */ + __IOM uint32_t GPIO5 : 1; /*!< [5..5] GPIO5 interrupt. */ + __IOM uint32_t GPIO6 : 1; /*!< [6..6] GPIO6 interrupt. */ + __IOM uint32_t GPIO7 : 1; /*!< [7..7] GPIO7 interrupt. */ + __IOM uint32_t GPIO8 : 1; /*!< [8..8] GPIO8 interrupt. */ + __IOM uint32_t GPIO9 : 1; /*!< [9..9] GPIO9 interrupt. */ + __IOM uint32_t GPIO10 : 1; /*!< [10..10] GPIO10 interrupt. */ + __IOM uint32_t GPIO11 : 1; /*!< [11..11] GPIO11 interrupt. */ + __IOM uint32_t GPIO12 : 1; /*!< [12..12] GPIO12 interrupt. */ + __IOM uint32_t GPIO13 : 1; /*!< [13..13] GPIO13 interrupt. */ + __IOM uint32_t GPIO14 : 1; /*!< [14..14] GPIO14 interrupt. */ + __IOM uint32_t GPIO15 : 1; /*!< [15..15] GPIO15 interrupt. */ + __IOM uint32_t GPIO16 : 1; /*!< [16..16] GPIO16 interrupt. */ + __IOM uint32_t GPIO17 : 1; /*!< [17..17] GPIO17 interrupt. */ + __IOM uint32_t GPIO18 : 1; /*!< [18..18] GPIO18interrupt. */ + __IOM uint32_t GPIO19 : 1; /*!< [19..19] GPIO19 interrupt. */ + __IOM uint32_t GPIO20 : 1; /*!< [20..20] GPIO20 interrupt. */ + __IOM uint32_t GPIO21 : 1; /*!< [21..21] GPIO21 interrupt. */ + __IOM uint32_t GPIO22 : 1; /*!< [22..22] GPIO22 interrupt. */ + __IOM uint32_t GPIO23 : 1; /*!< [23..23] GPIO23 interrupt. */ + __IOM uint32_t GPIO24 : 1; /*!< [24..24] GPIO24 interrupt. */ + __IOM uint32_t GPIO25 : 1; /*!< [25..25] GPIO25 interrupt. */ + __IOM uint32_t GPIO26 : 1; /*!< [26..26] GPIO26 interrupt. */ + __IOM uint32_t GPIO27 : 1; /*!< [27..27] GPIO27 interrupt. */ + __IOM uint32_t GPIO28 : 1; /*!< [28..28] GPIO28 interrupt. */ + __IOM uint32_t GPIO29 : 1; /*!< [29..29] GPIO29 interrupt. */ + __IOM uint32_t GPIO30 : 1; /*!< [30..30] GPIO30 interrupt. */ + __IOM uint32_t GPIO31 : 1; /*!< [31..31] GPIO31 interrupt. */ + } INT0EN_b; + } ; + + union { + __IOM uint32_t INT0STAT; /*!< (@ 0x00000204) GPIO Interrupt Registers 31-0: Status */ + + struct { + __IOM uint32_t GPIO0 : 1; /*!< [0..0] GPIO0 interrupt. */ + __IOM uint32_t GPIO1 : 1; /*!< [1..1] GPIO1 interrupt. */ + __IOM uint32_t GPIO2 : 1; /*!< [2..2] GPIO2 interrupt. */ + __IOM uint32_t GPIO3 : 1; /*!< [3..3] GPIO3 interrupt. */ + __IOM uint32_t GPIO4 : 1; /*!< [4..4] GPIO4 interrupt. */ + __IOM uint32_t GPIO5 : 1; /*!< [5..5] GPIO5 interrupt. */ + __IOM uint32_t GPIO6 : 1; /*!< [6..6] GPIO6 interrupt. */ + __IOM uint32_t GPIO7 : 1; /*!< [7..7] GPIO7 interrupt. */ + __IOM uint32_t GPIO8 : 1; /*!< [8..8] GPIO8 interrupt. */ + __IOM uint32_t GPIO9 : 1; /*!< [9..9] GPIO9 interrupt. */ + __IOM uint32_t GPIO10 : 1; /*!< [10..10] GPIO10 interrupt. */ + __IOM uint32_t GPIO11 : 1; /*!< [11..11] GPIO11 interrupt. */ + __IOM uint32_t GPIO12 : 1; /*!< [12..12] GPIO12 interrupt. */ + __IOM uint32_t GPIO13 : 1; /*!< [13..13] GPIO13 interrupt. */ + __IOM uint32_t GPIO14 : 1; /*!< [14..14] GPIO14 interrupt. */ + __IOM uint32_t GPIO15 : 1; /*!< [15..15] GPIO15 interrupt. */ + __IOM uint32_t GPIO16 : 1; /*!< [16..16] GPIO16 interrupt. */ + __IOM uint32_t GPIO17 : 1; /*!< [17..17] GPIO17 interrupt. */ + __IOM uint32_t GPIO18 : 1; /*!< [18..18] GPIO18interrupt. */ + __IOM uint32_t GPIO19 : 1; /*!< [19..19] GPIO19 interrupt. */ + __IOM uint32_t GPIO20 : 1; /*!< [20..20] GPIO20 interrupt. */ + __IOM uint32_t GPIO21 : 1; /*!< [21..21] GPIO21 interrupt. */ + __IOM uint32_t GPIO22 : 1; /*!< [22..22] GPIO22 interrupt. */ + __IOM uint32_t GPIO23 : 1; /*!< [23..23] GPIO23 interrupt. */ + __IOM uint32_t GPIO24 : 1; /*!< [24..24] GPIO24 interrupt. */ + __IOM uint32_t GPIO25 : 1; /*!< [25..25] GPIO25 interrupt. */ + __IOM uint32_t GPIO26 : 1; /*!< [26..26] GPIO26 interrupt. */ + __IOM uint32_t GPIO27 : 1; /*!< [27..27] GPIO27 interrupt. */ + __IOM uint32_t GPIO28 : 1; /*!< [28..28] GPIO28 interrupt. */ + __IOM uint32_t GPIO29 : 1; /*!< [29..29] GPIO29 interrupt. */ + __IOM uint32_t GPIO30 : 1; /*!< [30..30] GPIO30 interrupt. */ + __IOM uint32_t GPIO31 : 1; /*!< [31..31] GPIO31 interrupt. */ + } INT0STAT_b; + } ; + + union { + __IOM uint32_t INT0CLR; /*!< (@ 0x00000208) GPIO Interrupt Registers 31-0: Clear */ + + struct { + __IOM uint32_t GPIO0 : 1; /*!< [0..0] GPIO0 interrupt. */ + __IOM uint32_t GPIO1 : 1; /*!< [1..1] GPIO1 interrupt. */ + __IOM uint32_t GPIO2 : 1; /*!< [2..2] GPIO2 interrupt. */ + __IOM uint32_t GPIO3 : 1; /*!< [3..3] GPIO3 interrupt. */ + __IOM uint32_t GPIO4 : 1; /*!< [4..4] GPIO4 interrupt. */ + __IOM uint32_t GPIO5 : 1; /*!< [5..5] GPIO5 interrupt. */ + __IOM uint32_t GPIO6 : 1; /*!< [6..6] GPIO6 interrupt. */ + __IOM uint32_t GPIO7 : 1; /*!< [7..7] GPIO7 interrupt. */ + __IOM uint32_t GPIO8 : 1; /*!< [8..8] GPIO8 interrupt. */ + __IOM uint32_t GPIO9 : 1; /*!< [9..9] GPIO9 interrupt. */ + __IOM uint32_t GPIO10 : 1; /*!< [10..10] GPIO10 interrupt. */ + __IOM uint32_t GPIO11 : 1; /*!< [11..11] GPIO11 interrupt. */ + __IOM uint32_t GPIO12 : 1; /*!< [12..12] GPIO12 interrupt. */ + __IOM uint32_t GPIO13 : 1; /*!< [13..13] GPIO13 interrupt. */ + __IOM uint32_t GPIO14 : 1; /*!< [14..14] GPIO14 interrupt. */ + __IOM uint32_t GPIO15 : 1; /*!< [15..15] GPIO15 interrupt. */ + __IOM uint32_t GPIO16 : 1; /*!< [16..16] GPIO16 interrupt. */ + __IOM uint32_t GPIO17 : 1; /*!< [17..17] GPIO17 interrupt. */ + __IOM uint32_t GPIO18 : 1; /*!< [18..18] GPIO18interrupt. */ + __IOM uint32_t GPIO19 : 1; /*!< [19..19] GPIO19 interrupt. */ + __IOM uint32_t GPIO20 : 1; /*!< [20..20] GPIO20 interrupt. */ + __IOM uint32_t GPIO21 : 1; /*!< [21..21] GPIO21 interrupt. */ + __IOM uint32_t GPIO22 : 1; /*!< [22..22] GPIO22 interrupt. */ + __IOM uint32_t GPIO23 : 1; /*!< [23..23] GPIO23 interrupt. */ + __IOM uint32_t GPIO24 : 1; /*!< [24..24] GPIO24 interrupt. */ + __IOM uint32_t GPIO25 : 1; /*!< [25..25] GPIO25 interrupt. */ + __IOM uint32_t GPIO26 : 1; /*!< [26..26] GPIO26 interrupt. */ + __IOM uint32_t GPIO27 : 1; /*!< [27..27] GPIO27 interrupt. */ + __IOM uint32_t GPIO28 : 1; /*!< [28..28] GPIO28 interrupt. */ + __IOM uint32_t GPIO29 : 1; /*!< [29..29] GPIO29 interrupt. */ + __IOM uint32_t GPIO30 : 1; /*!< [30..30] GPIO30 interrupt. */ + __IOM uint32_t GPIO31 : 1; /*!< [31..31] GPIO31 interrupt. */ + } INT0CLR_b; + } ; + + union { + __IOM uint32_t INT0SET; /*!< (@ 0x0000020C) GPIO Interrupt Registers 31-0: Set */ + + struct { + __IOM uint32_t GPIO0 : 1; /*!< [0..0] GPIO0 interrupt. */ + __IOM uint32_t GPIO1 : 1; /*!< [1..1] GPIO1 interrupt. */ + __IOM uint32_t GPIO2 : 1; /*!< [2..2] GPIO2 interrupt. */ + __IOM uint32_t GPIO3 : 1; /*!< [3..3] GPIO3 interrupt. */ + __IOM uint32_t GPIO4 : 1; /*!< [4..4] GPIO4 interrupt. */ + __IOM uint32_t GPIO5 : 1; /*!< [5..5] GPIO5 interrupt. */ + __IOM uint32_t GPIO6 : 1; /*!< [6..6] GPIO6 interrupt. */ + __IOM uint32_t GPIO7 : 1; /*!< [7..7] GPIO7 interrupt. */ + __IOM uint32_t GPIO8 : 1; /*!< [8..8] GPIO8 interrupt. */ + __IOM uint32_t GPIO9 : 1; /*!< [9..9] GPIO9 interrupt. */ + __IOM uint32_t GPIO10 : 1; /*!< [10..10] GPIO10 interrupt. */ + __IOM uint32_t GPIO11 : 1; /*!< [11..11] GPIO11 interrupt. */ + __IOM uint32_t GPIO12 : 1; /*!< [12..12] GPIO12 interrupt. */ + __IOM uint32_t GPIO13 : 1; /*!< [13..13] GPIO13 interrupt. */ + __IOM uint32_t GPIO14 : 1; /*!< [14..14] GPIO14 interrupt. */ + __IOM uint32_t GPIO15 : 1; /*!< [15..15] GPIO15 interrupt. */ + __IOM uint32_t GPIO16 : 1; /*!< [16..16] GPIO16 interrupt. */ + __IOM uint32_t GPIO17 : 1; /*!< [17..17] GPIO17 interrupt. */ + __IOM uint32_t GPIO18 : 1; /*!< [18..18] GPIO18interrupt. */ + __IOM uint32_t GPIO19 : 1; /*!< [19..19] GPIO19 interrupt. */ + __IOM uint32_t GPIO20 : 1; /*!< [20..20] GPIO20 interrupt. */ + __IOM uint32_t GPIO21 : 1; /*!< [21..21] GPIO21 interrupt. */ + __IOM uint32_t GPIO22 : 1; /*!< [22..22] GPIO22 interrupt. */ + __IOM uint32_t GPIO23 : 1; /*!< [23..23] GPIO23 interrupt. */ + __IOM uint32_t GPIO24 : 1; /*!< [24..24] GPIO24 interrupt. */ + __IOM uint32_t GPIO25 : 1; /*!< [25..25] GPIO25 interrupt. */ + __IOM uint32_t GPIO26 : 1; /*!< [26..26] GPIO26 interrupt. */ + __IOM uint32_t GPIO27 : 1; /*!< [27..27] GPIO27 interrupt. */ + __IOM uint32_t GPIO28 : 1; /*!< [28..28] GPIO28 interrupt. */ + __IOM uint32_t GPIO29 : 1; /*!< [29..29] GPIO29 interrupt. */ + __IOM uint32_t GPIO30 : 1; /*!< [30..30] GPIO30 interrupt. */ + __IOM uint32_t GPIO31 : 1; /*!< [31..31] GPIO31 interrupt. */ + } INT0SET_b; + } ; + + union { + __IOM uint32_t INT1EN; /*!< (@ 0x00000210) GPIO Interrupt Registers 49-32: Enable */ + + struct { + __IOM uint32_t GPIO32 : 1; /*!< [0..0] GPIO32 interrupt. */ + __IOM uint32_t GPIO33 : 1; /*!< [1..1] GPIO33 interrupt. */ + __IOM uint32_t GPIO34 : 1; /*!< [2..2] GPIO34 interrupt. */ + __IOM uint32_t GPIO35 : 1; /*!< [3..3] GPIO35 interrupt. */ + __IOM uint32_t GPIO36 : 1; /*!< [4..4] GPIO36 interrupt. */ + __IOM uint32_t GPIO37 : 1; /*!< [5..5] GPIO37 interrupt. */ + __IOM uint32_t GPIO38 : 1; /*!< [6..6] GPIO38 interrupt. */ + __IOM uint32_t GPIO39 : 1; /*!< [7..7] GPIO39 interrupt. */ + __IOM uint32_t GPIO40 : 1; /*!< [8..8] GPIO40 interrupt. */ + __IOM uint32_t GPIO41 : 1; /*!< [9..9] GPIO41 interrupt. */ + __IOM uint32_t GPIO42 : 1; /*!< [10..10] GPIO42 interrupt. */ + __IOM uint32_t GPIO43 : 1; /*!< [11..11] GPIO43 interrupt. */ + __IOM uint32_t GPIO44 : 1; /*!< [12..12] GPIO44 interrupt. */ + __IOM uint32_t GPIO45 : 1; /*!< [13..13] GPIO45 interrupt. */ + __IOM uint32_t GPIO46 : 1; /*!< [14..14] GPIO46 interrupt. */ + __IOM uint32_t GPIO47 : 1; /*!< [15..15] GPIO47 interrupt. */ + __IOM uint32_t GPIO48 : 1; /*!< [16..16] GPIO48 interrupt. */ + __IOM uint32_t GPIO49 : 1; /*!< [17..17] GPIO49 interrupt. */ + } INT1EN_b; + } ; + + union { + __IOM uint32_t INT1STAT; /*!< (@ 0x00000214) GPIO Interrupt Registers 49-32: Status */ + + struct { + __IOM uint32_t GPIO32 : 1; /*!< [0..0] GPIO32 interrupt. */ + __IOM uint32_t GPIO33 : 1; /*!< [1..1] GPIO33 interrupt. */ + __IOM uint32_t GPIO34 : 1; /*!< [2..2] GPIO34 interrupt. */ + __IOM uint32_t GPIO35 : 1; /*!< [3..3] GPIO35 interrupt. */ + __IOM uint32_t GPIO36 : 1; /*!< [4..4] GPIO36 interrupt. */ + __IOM uint32_t GPIO37 : 1; /*!< [5..5] GPIO37 interrupt. */ + __IOM uint32_t GPIO38 : 1; /*!< [6..6] GPIO38 interrupt. */ + __IOM uint32_t GPIO39 : 1; /*!< [7..7] GPIO39 interrupt. */ + __IOM uint32_t GPIO40 : 1; /*!< [8..8] GPIO40 interrupt. */ + __IOM uint32_t GPIO41 : 1; /*!< [9..9] GPIO41 interrupt. */ + __IOM uint32_t GPIO42 : 1; /*!< [10..10] GPIO42 interrupt. */ + __IOM uint32_t GPIO43 : 1; /*!< [11..11] GPIO43 interrupt. */ + __IOM uint32_t GPIO44 : 1; /*!< [12..12] GPIO44 interrupt. */ + __IOM uint32_t GPIO45 : 1; /*!< [13..13] GPIO45 interrupt. */ + __IOM uint32_t GPIO46 : 1; /*!< [14..14] GPIO46 interrupt. */ + __IOM uint32_t GPIO47 : 1; /*!< [15..15] GPIO47 interrupt. */ + __IOM uint32_t GPIO48 : 1; /*!< [16..16] GPIO48 interrupt. */ + __IOM uint32_t GPIO49 : 1; /*!< [17..17] GPIO49 interrupt. */ + } INT1STAT_b; + } ; + + union { + __IOM uint32_t INT1CLR; /*!< (@ 0x00000218) GPIO Interrupt Registers 49-32: Clear */ + + struct { + __IOM uint32_t GPIO32 : 1; /*!< [0..0] GPIO32 interrupt. */ + __IOM uint32_t GPIO33 : 1; /*!< [1..1] GPIO33 interrupt. */ + __IOM uint32_t GPIO34 : 1; /*!< [2..2] GPIO34 interrupt. */ + __IOM uint32_t GPIO35 : 1; /*!< [3..3] GPIO35 interrupt. */ + __IOM uint32_t GPIO36 : 1; /*!< [4..4] GPIO36 interrupt. */ + __IOM uint32_t GPIO37 : 1; /*!< [5..5] GPIO37 interrupt. */ + __IOM uint32_t GPIO38 : 1; /*!< [6..6] GPIO38 interrupt. */ + __IOM uint32_t GPIO39 : 1; /*!< [7..7] GPIO39 interrupt. */ + __IOM uint32_t GPIO40 : 1; /*!< [8..8] GPIO40 interrupt. */ + __IOM uint32_t GPIO41 : 1; /*!< [9..9] GPIO41 interrupt. */ + __IOM uint32_t GPIO42 : 1; /*!< [10..10] GPIO42 interrupt. */ + __IOM uint32_t GPIO43 : 1; /*!< [11..11] GPIO43 interrupt. */ + __IOM uint32_t GPIO44 : 1; /*!< [12..12] GPIO44 interrupt. */ + __IOM uint32_t GPIO45 : 1; /*!< [13..13] GPIO45 interrupt. */ + __IOM uint32_t GPIO46 : 1; /*!< [14..14] GPIO46 interrupt. */ + __IOM uint32_t GPIO47 : 1; /*!< [15..15] GPIO47 interrupt. */ + __IOM uint32_t GPIO48 : 1; /*!< [16..16] GPIO48 interrupt. */ + __IOM uint32_t GPIO49 : 1; /*!< [17..17] GPIO49 interrupt. */ + } INT1CLR_b; + } ; + + union { + __IOM uint32_t INT1SET; /*!< (@ 0x0000021C) GPIO Interrupt Registers 49-32: Set */ + + struct { + __IOM uint32_t GPIO32 : 1; /*!< [0..0] GPIO32 interrupt. */ + __IOM uint32_t GPIO33 : 1; /*!< [1..1] GPIO33 interrupt. */ + __IOM uint32_t GPIO34 : 1; /*!< [2..2] GPIO34 interrupt. */ + __IOM uint32_t GPIO35 : 1; /*!< [3..3] GPIO35 interrupt. */ + __IOM uint32_t GPIO36 : 1; /*!< [4..4] GPIO36 interrupt. */ + __IOM uint32_t GPIO37 : 1; /*!< [5..5] GPIO37 interrupt. */ + __IOM uint32_t GPIO38 : 1; /*!< [6..6] GPIO38 interrupt. */ + __IOM uint32_t GPIO39 : 1; /*!< [7..7] GPIO39 interrupt. */ + __IOM uint32_t GPIO40 : 1; /*!< [8..8] GPIO40 interrupt. */ + __IOM uint32_t GPIO41 : 1; /*!< [9..9] GPIO41 interrupt. */ + __IOM uint32_t GPIO42 : 1; /*!< [10..10] GPIO42 interrupt. */ + __IOM uint32_t GPIO43 : 1; /*!< [11..11] GPIO43 interrupt. */ + __IOM uint32_t GPIO44 : 1; /*!< [12..12] GPIO44 interrupt. */ + __IOM uint32_t GPIO45 : 1; /*!< [13..13] GPIO45 interrupt. */ + __IOM uint32_t GPIO46 : 1; /*!< [14..14] GPIO46 interrupt. */ + __IOM uint32_t GPIO47 : 1; /*!< [15..15] GPIO47 interrupt. */ + __IOM uint32_t GPIO48 : 1; /*!< [16..16] GPIO48 interrupt. */ + __IOM uint32_t GPIO49 : 1; /*!< [17..17] GPIO49 interrupt. */ + } INT1SET_b; + } ; +} GPIO_Type; /*!< Size = 544 (0x220) */ + + + +/* =========================================================================================================================== */ +/* ================ IOM0 ================ */ +/* =========================================================================================================================== */ + + +/** + * @brief IO Peripheral Master (IOM0) + */ + +typedef struct { /*!< (@ 0x50004000) IOM0 Structure */ + + union { + __IOM uint32_t FIFO; /*!< (@ 0x00000000) FIFO Access Port */ + + struct { + __IOM uint32_t FIFO : 32; /*!< [31..0] FIFO direct access. Only locations 0 - 3F will return + valid information. */ + } FIFO_b; + } ; + __IM uint32_t RESERVED[63]; + + union { + __IOM uint32_t FIFOPTR; /*!< (@ 0x00000100) FIFO size and remaining slots open values */ + + struct { + __IOM uint32_t FIFO0SIZ : 8; /*!< [7..0] The number of valid data bytes currently in the FIFO + 0 (written by MCU, read by interface) */ + __IOM uint32_t FIFO0REM : 8; /*!< [15..8] The number of remaining data bytes slots currently in + FIFO 0 (written by MCU, read by interface) */ + __IOM uint32_t FIFO1SIZ : 8; /*!< [23..16] The number of valid data bytes currently in FIFO 1 + (written by interface, read by MCU) */ + __IOM uint32_t FIFO1REM : 8; /*!< [31..24] The number of remaining data bytes slots currently + in FIFO 1 (written by interface, read by MCU) */ + } FIFOPTR_b; + } ; + + union { + __IOM uint32_t FIFOTHR; /*!< (@ 0x00000104) FIFO Threshold Configuration */ + + struct { + __IOM uint32_t FIFORTHR : 6; /*!< [5..0] FIFO read threshold in bytes. A value of 0 will disable + the read FIFO level from activating the threshold interrupt. + If this field is non-zero, it will trigger a threshold + interrupt when the read fifo contains FIFORTHR valid bytes + of data, as indicated by the FIFO1SIZ field. This is intended + to signal when a data transfer of FIFORTHR bytes can be + done from the IOM module to the host via the read fifo + to support large IOM read operations. */ + __IM uint32_t : 2; + __IOM uint32_t FIFOWTHR : 6; /*!< [13..8] FIFO write threshold in bytes. A value of 0 will disable + the write FIFO level from activating the threshold interrupt. + If this field is non-zero, it will trigger a threshold + interrupt when the write fifo contains FIFOWTHR free bytes, + as indicated by the FIFO0REM field. This is intended to + signal when a transfer of FIFOWTHR bytes can be done from + the host to the IOM write fifo to support large IOM write + operations. */ + } FIFOTHR_b; + } ; + + union { + __IOM uint32_t FIFOPOP; /*!< (@ 0x00000108) FIFO POP register */ + + struct { + __IOM uint32_t FIFODOUT : 32; /*!< [31..0] This register will return the read data indicated by + the current read pointer on reads. If the POPWR control + bit in the FIFOCTRL register is reset (0), the fifo read + pointer will be advanced by one word as a result of the + read.If the POPWR bit is set (1), the fifo read pointer + will only be advanced after a write operation to this register. + The write data is ignored for this register.If less than + a even word multiple is available, and the command is completed, + the module will return the word containing */ + } FIFOPOP_b; + } ; + + union { + __IOM uint32_t FIFOPUSH; /*!< (@ 0x0000010C) FIFO PUSH register */ + + struct { + __IOM uint32_t FIFODIN : 32; /*!< [31..0] This register is used to write the FIFORAM in FIFO mode + and will cause a push event to occur to the next open slot + within the FIFORAM. Writing to this register will cause + the write point to increment by 1 word(4 bytes). */ + } FIFOPUSH_b; + } ; + + union { + __IOM uint32_t FIFOCTRL; /*!< (@ 0x00000110) FIFO Control Register */ + + struct { + __IOM uint32_t POPWR : 1; /*!< [0..0] Selects the mode in which 'pop' events are done for the + fifo read operations. A value of '1' will prevent a pop + event on a read operation, and will require a write to + the FIFOPOP register to create a pop event.A value of '0' + in this register will allow a pop event to occur on the + read of the FIFOPOP register, and may cause inadvertant + fifo pops when used in a debugging mode. */ + __IOM uint32_t FIFORSTN : 1; /*!< [1..1] Active low manual reset of the fifo. Write to 0 to reset + fifo, and then write to 1 to remove the reset. */ + } FIFOCTRL_b; + } ; + + union { + __IOM uint32_t FIFOLOC; /*!< (@ 0x00000114) FIFO Pointers */ + + struct { + __IOM uint32_t FIFOWPTR : 4; /*!< [3..0] Current FIFO write pointer. Value is the index into the + outgoing FIFO (FIFO0), which is used during write operations + to external devices. */ + __IM uint32_t : 4; + __IOM uint32_t FIFORPTR : 4; /*!< [11..8] Current FIFO read pointer. Used to index into the incoming + FIFO (FIFO1), which is used to store read data returned + from external devices during a read operation. */ + } FIFOLOC_b; + } ; + __IM uint32_t RESERVED1[58]; + + union { + __IOM uint32_t INTEN; /*!< (@ 0x00000200) IO Master Interrupts: Enable */ + + struct { + __IOM uint32_t CMDCMP : 1; /*!< [0..0] Command Complete interrupt. Asserted when the current + operation has completed. For repeated commands, this will + only be asserted when the final repeated command is completed. */ + __IOM uint32_t THR : 1; /*!< [1..1] FIFO Threshold interrupt. For write operations, asserted + when the number of free bytes in the write FIFO equals + or exceeds the WTHR field.For read operations, asserted + when the number of valid bytes in the read FIFO equals + of exceeds the value set in the RTHR field. */ + __IOM uint32_t FUNDFL : 1; /*!< [2..2] Read FIFO Underflow interrupt. This occurs when software + tries to pop from an empty fifo. */ + __IOM uint32_t FOVFL : 1; /*!< [3..3] Write FIFO Overflow interrupt. This occurs when software + tries to write to a full fifo. The current operation does + not stop. */ + __IOM uint32_t NAK : 1; /*!< [4..4] I2C NAK interrupt. Asserted when an unexpected NAK has + been received on the I2C bus. */ + __IOM uint32_t IACC : 1; /*!< [5..5] illegal FIFO access interrupt. Asserted when there is + a overflow or underflow event */ + __IOM uint32_t ICMD : 1; /*!< [6..6] illegal command interrupt. Asserted when a command is + written when an active command is in progress. */ + __IOM uint32_t START : 1; /*!< [7..7] START command interrupt. Asserted when another master + on the bus has signaled a START command. */ + __IOM uint32_t STOP : 1; /*!< [8..8] STOP command interrupt. Asserted when another master + on the bus has signaled a STOP command. */ + __IOM uint32_t ARB : 1; /*!< [9..9] Arbitration loss interrupt. Asserted when arbitration + is enabled and has been lost to another master on the bus. */ + __IOM uint32_t DCMP : 1; /*!< [10..10] DMA Complete. Processing of the DMA operation has completed + and the DMA submodule is returned into the idle state */ + __IOM uint32_t DERR : 1; /*!< [11..11] DMA Error encountered during the processing of the + DMA command. The DMA error could occur when the memory + access specified in the DMA operation is not available + or incorrectly specified. */ + __IOM uint32_t CQPAUSED : 1; /*!< [12..12] Command queue is paused due to an active event enabled + in the PAUSEEN register. The interrupt is posted when the + event is enabled within the PAUSEEN register, the mask + is active in the CQIRQMASK field and the event occurs. */ + __IOM uint32_t CQUPD : 1; /*!< [13..13] CQ write operation performed a register write with + the register address bit 0 set to 1. The low address bits + in the CQ address fields are unused and bit 0 can be used + to trigger an interrupt to indicate when this register + write is performed by the CQ operation. */ + __IOM uint32_t CQERR : 1; /*!< [14..14] Error during command queue operations */ + } INTEN_b; + } ; + + union { + __IOM uint32_t INTSTAT; /*!< (@ 0x00000204) IO Master Interrupts: Status */ + + struct { + __IOM uint32_t CMDCMP : 1; /*!< [0..0] Command Complete interrupt. Asserted when the current + operation has completed. For repeated commands, this will + only be asserted when the final repeated command is completed. */ + __IOM uint32_t THR : 1; /*!< [1..1] FIFO Threshold interrupt. For write operations, asserted + when the number of free bytes in the write FIFO equals + or exceeds the WTHR field.For read operations, asserted + when the number of valid bytes in the read FIFO equals + of exceeds the value set in the RTHR field. */ + __IOM uint32_t FUNDFL : 1; /*!< [2..2] Read FIFO Underflow interrupt. This occurs when software + tries to pop from an empty fifo. */ + __IOM uint32_t FOVFL : 1; /*!< [3..3] Write FIFO Overflow interrupt. This occurs when software + tries to write to a full fifo. The current operation does + not stop. */ + __IOM uint32_t NAK : 1; /*!< [4..4] I2C NAK interrupt. Asserted when an unexpected NAK has + been received on the I2C bus. */ + __IOM uint32_t IACC : 1; /*!< [5..5] illegal FIFO access interrupt. Asserted when there is + a overflow or underflow event */ + __IOM uint32_t ICMD : 1; /*!< [6..6] illegal command interrupt. Asserted when a command is + written when an active command is in progress. */ + __IOM uint32_t START : 1; /*!< [7..7] START command interrupt. Asserted when another master + on the bus has signaled a START command. */ + __IOM uint32_t STOP : 1; /*!< [8..8] STOP command interrupt. Asserted when another master + on the bus has signaled a STOP command. */ + __IOM uint32_t ARB : 1; /*!< [9..9] Arbitration loss interrupt. Asserted when arbitration + is enabled and has been lost to another master on the bus. */ + __IOM uint32_t DCMP : 1; /*!< [10..10] DMA Complete. Processing of the DMA operation has completed + and the DMA submodule is returned into the idle state */ + __IOM uint32_t DERR : 1; /*!< [11..11] DMA Error encountered during the processing of the + DMA command. The DMA error could occur when the memory + access specified in the DMA operation is not available + or incorrectly specified. */ + __IOM uint32_t CQPAUSED : 1; /*!< [12..12] Command queue is paused due to an active event enabled + in the PAUSEEN register. The interrupt is posted when the + event is enabled within the PAUSEEN register, the mask + is active in the CQIRQMASK field and the event occurs. */ + __IOM uint32_t CQUPD : 1; /*!< [13..13] CQ write operation performed a register write with + the register address bit 0 set to 1. The low address bits + in the CQ address fields are unused and bit 0 can be used + to trigger an interrupt to indicate when this register + write is performed by the CQ operation. */ + __IOM uint32_t CQERR : 1; /*!< [14..14] Error during command queue operations */ + } INTSTAT_b; + } ; + + union { + __IOM uint32_t INTCLR; /*!< (@ 0x00000208) IO Master Interrupts: Clear */ + + struct { + __IOM uint32_t CMDCMP : 1; /*!< [0..0] Command Complete interrupt. Asserted when the current + operation has completed. For repeated commands, this will + only be asserted when the final repeated command is completed. */ + __IOM uint32_t THR : 1; /*!< [1..1] FIFO Threshold interrupt. For write operations, asserted + when the number of free bytes in the write FIFO equals + or exceeds the WTHR field.For read operations, asserted + when the number of valid bytes in the read FIFO equals + of exceeds the value set in the RTHR field. */ + __IOM uint32_t FUNDFL : 1; /*!< [2..2] Read FIFO Underflow interrupt. This occurs when software + tries to pop from an empty fifo. */ + __IOM uint32_t FOVFL : 1; /*!< [3..3] Write FIFO Overflow interrupt. This occurs when software + tries to write to a full fifo. The current operation does + not stop. */ + __IOM uint32_t NAK : 1; /*!< [4..4] I2C NAK interrupt. Asserted when an unexpected NAK has + been received on the I2C bus. */ + __IOM uint32_t IACC : 1; /*!< [5..5] illegal FIFO access interrupt. Asserted when there is + a overflow or underflow event */ + __IOM uint32_t ICMD : 1; /*!< [6..6] illegal command interrupt. Asserted when a command is + written when an active command is in progress. */ + __IOM uint32_t START : 1; /*!< [7..7] START command interrupt. Asserted when another master + on the bus has signaled a START command. */ + __IOM uint32_t STOP : 1; /*!< [8..8] STOP command interrupt. Asserted when another master + on the bus has signaled a STOP command. */ + __IOM uint32_t ARB : 1; /*!< [9..9] Arbitration loss interrupt. Asserted when arbitration + is enabled and has been lost to another master on the bus. */ + __IOM uint32_t DCMP : 1; /*!< [10..10] DMA Complete. Processing of the DMA operation has completed + and the DMA submodule is returned into the idle state */ + __IOM uint32_t DERR : 1; /*!< [11..11] DMA Error encountered during the processing of the + DMA command. The DMA error could occur when the memory + access specified in the DMA operation is not available + or incorrectly specified. */ + __IOM uint32_t CQPAUSED : 1; /*!< [12..12] Command queue is paused due to an active event enabled + in the PAUSEEN register. The interrupt is posted when the + event is enabled within the PAUSEEN register, the mask + is active in the CQIRQMASK field and the event occurs. */ + __IOM uint32_t CQUPD : 1; /*!< [13..13] CQ write operation performed a register write with + the register address bit 0 set to 1. The low address bits + in the CQ address fields are unused and bit 0 can be used + to trigger an interrupt to indicate when this register + write is performed by the CQ operation. */ + __IOM uint32_t CQERR : 1; /*!< [14..14] Error during command queue operations */ + } INTCLR_b; + } ; + + union { + __IOM uint32_t INTSET; /*!< (@ 0x0000020C) IO Master Interrupts: Set */ + + struct { + __IOM uint32_t CMDCMP : 1; /*!< [0..0] Command Complete interrupt. Asserted when the current + operation has completed. For repeated commands, this will + only be asserted when the final repeated command is completed. */ + __IOM uint32_t THR : 1; /*!< [1..1] FIFO Threshold interrupt. For write operations, asserted + when the number of free bytes in the write FIFO equals + or exceeds the WTHR field.For read operations, asserted + when the number of valid bytes in the read FIFO equals + of exceeds the value set in the RTHR field. */ + __IOM uint32_t FUNDFL : 1; /*!< [2..2] Read FIFO Underflow interrupt. This occurs when software + tries to pop from an empty fifo. */ + __IOM uint32_t FOVFL : 1; /*!< [3..3] Write FIFO Overflow interrupt. This occurs when software + tries to write to a full fifo. The current operation does + not stop. */ + __IOM uint32_t NAK : 1; /*!< [4..4] I2C NAK interrupt. Asserted when an unexpected NAK has + been received on the I2C bus. */ + __IOM uint32_t IACC : 1; /*!< [5..5] illegal FIFO access interrupt. Asserted when there is + a overflow or underflow event */ + __IOM uint32_t ICMD : 1; /*!< [6..6] illegal command interrupt. Asserted when a command is + written when an active command is in progress. */ + __IOM uint32_t START : 1; /*!< [7..7] START command interrupt. Asserted when another master + on the bus has signaled a START command. */ + __IOM uint32_t STOP : 1; /*!< [8..8] STOP command interrupt. Asserted when another master + on the bus has signaled a STOP command. */ + __IOM uint32_t ARB : 1; /*!< [9..9] Arbitration loss interrupt. Asserted when arbitration + is enabled and has been lost to another master on the bus. */ + __IOM uint32_t DCMP : 1; /*!< [10..10] DMA Complete. Processing of the DMA operation has completed + and the DMA submodule is returned into the idle state */ + __IOM uint32_t DERR : 1; /*!< [11..11] DMA Error encountered during the processing of the + DMA command. The DMA error could occur when the memory + access specified in the DMA operation is not available + or incorrectly specified. */ + __IOM uint32_t CQPAUSED : 1; /*!< [12..12] Command queue is paused due to an active event enabled + in the PAUSEEN register. The interrupt is posted when the + event is enabled within the PAUSEEN register, the mask + is active in the CQIRQMASK field and the event occurs. */ + __IOM uint32_t CQUPD : 1; /*!< [13..13] CQ write operation performed a register write with + the register address bit 0 set to 1. The low address bits + in the CQ address fields are unused and bit 0 can be used + to trigger an interrupt to indicate when this register + write is performed by the CQ operation. */ + __IOM uint32_t CQERR : 1; /*!< [14..14] Error during command queue operations */ + } INTSET_b; + } ; + + union { + __IOM uint32_t CLKCFG; /*!< (@ 0x00000210) I/O Clock Configuration */ + + struct { + __IOM uint32_t IOCLKEN : 1; /*!< [0..0] Enable for the interface clock. Must be enabled prior + to executing any IO operations. */ + __IM uint32_t : 7; + __IOM uint32_t FSEL : 3; /*!< [10..8] Select the input clock frequency. */ + __IOM uint32_t DIV3 : 1; /*!< [11..11] Enable divide by 3 of the source IOCLK. Division by + 3 is done before the DIVEN programmable divider, and if + enabledwill provide the divided by 3 clock as the source + to the programmable divider. */ + __IOM uint32_t DIVEN : 1; /*!< [12..12] Enable clock division by TOTPER and LOWPER */ + __IM uint32_t : 3; + __IOM uint32_t LOWPER : 8; /*!< [23..16] Clock low clock count minus 1. This provides the number + of clocks the divided clock will be low when the DIVEN + = 1.Only applicable when DIVEN = 1. */ + __IOM uint32_t TOTPER : 8; /*!< [31..24] Clock total clock count minus 1. This provides the + total period of the divided clock -1 when the DIVEN is + active. Thesource clock is selected by FSEL. Only applicable + when DIVEN = 1. */ + } CLKCFG_b; + } ; + + union { + __IOM uint32_t SUBMODCTRL; /*!< (@ 0x00000214) Submodule control */ + + struct { + __IOM uint32_t SMOD0EN : 1; /*!< [0..0] Submodule 0 enable (1) or disable (0) */ + __IOM uint32_t SMOD0TYPE : 3; /*!< [3..1] Submodule 0 module type. This is the SPI Master interface. */ + __IOM uint32_t SMOD1EN : 1; /*!< [4..4] Submodule 1 enable (1) or disable (0) */ + __IOM uint32_t SMOD1TYPE : 3; /*!< [7..5] Submodule 0 module type. This is the I2C Master interface */ + } SUBMODCTRL_b; + } ; + + union { + __IOM uint32_t CMD; /*!< (@ 0x00000218) Command and offset Register */ + + struct { + __IOM uint32_t CMD : 5; /*!< [4..0] Command for submodule. */ + __IOM uint32_t OFFSETCNT : 2; /*!< [6..5] Number of offset bytes to use for the command - 0, 1, + 2, 3 are valid selections. The second (byte 1) and third + byte (byte 2) are read from the OFFSETHI register, and + the low order byte is pulled from this register in the + OFFSETLO field.Offset bytes are transmitted highest byte + first. EG if offsetcnt == 3, OFFSETHI[15:8] will be transmitted + first, then OFFSETHI[7:0] then OFFSETLO.If offsetcnt == + 2, OFFSETHI[7:0] will be transmitted, then OFFSETLO.If + offsetcnt == 1, only OFFSETLO will be transmitted. */ + __IOM uint32_t CONT : 1; /*!< [7..7] Contine to hold the bus after the current transaction + if set to a 1 with a new command issued. */ + __IOM uint32_t TSIZE : 12; /*!< [19..8] Defines the transaction size in bytes. The offset transfer + is not included in this size. */ + __IOM uint32_t CMDSEL : 2; /*!< [21..20] Command Specific selection information. Not used in + Master I2C. Used as CEn select for Master SPI transactions */ + __IM uint32_t : 2; + __IOM uint32_t OFFSETLO : 8; /*!< [31..24] This register holds the low order byte of offset to + be used in the transaction. The number of offset bytes + to use is set with bits 1:0 of the command. */ + } CMD_b; + } ; + + union { + __IOM uint32_t CMDRPT; /*!< (@ 0x0000021C) Command Repeat Register */ + + struct { + __IOM uint32_t CMDRPT : 5; /*!< [4..0] Count of number of times to repeat the next command. */ + } CMDRPT_b; + } ; + + union { + __IOM uint32_t OFFSETHI; /*!< (@ 0x00000220) High order 2 bytes of 3 byte offset for IO transaction */ + + struct { + __IOM uint32_t OFFSETHI : 16; /*!< [15..0] Holds the high order 2 bytes of the 3 byte addressing/offset + field to use with IO commands. The number of offset bytes + to use is specified in the command register */ + } OFFSETHI_b; + } ; + + union { + __IOM uint32_t CMDSTAT; /*!< (@ 0x00000224) Command status */ + + struct { + __IOM uint32_t CCMD : 5; /*!< [4..0] current command that is being executed */ + __IOM uint32_t CMDSTAT : 3; /*!< [7..5] The current status of the command execution. */ + __IOM uint32_t CTSIZE : 12; /*!< [19..8] The current number of bytes still to be transferred + with this command. This field will count down to zero. */ + } CMDSTAT_b; + } ; + __IM uint32_t RESERVED2[6]; + + union { + __IOM uint32_t DMATRIGEN; /*!< (@ 0x00000240) DMA Trigger Enable Register */ + + struct { + __IOM uint32_t DCMDCMPEN : 1; /*!< [0..0] Trigger DMA upon command complete. Enables the trigger + of the DMA when a command is completed. When this event + is triggered, the number of words transferred will be the + lesser of the remaining TOTCOUNT bytes, or */ + __IOM uint32_t DTHREN : 1; /*!< [1..1] Trigger DMA upon THR level reached. For M2P DMA operations + (IOM writes), the trigger will assert when the write FIFO + has (WTHR/4) number of words free in the write FIFO, and + will transfer (WTHR/4) number of wordsor, if the number + of words left to transfer is less than the WTHR value, + will transfer the remaining byte count.For P2M DMA operations, + the trigger will assert when the read FIFO has (RTHR/4) + words available in the read FIFO, and will transfer (RTHR/4) + words to SRAM. This trigger will NOT asser */ + } DMATRIGEN_b; + } ; + + union { + __IOM uint32_t DMATRIGSTAT; /*!< (@ 0x00000244) DMA Trigger Status Register */ + + struct { + __IOM uint32_t DCMDCMP : 1; /*!< [0..0] Triggered DMA from Command complete event. Bit is read + only and can be cleared by disabling the DCMDCMP trigger + enable or by disabling DMA. */ + __IOM uint32_t DTHR : 1; /*!< [1..1] Triggered DMA from THR event. Bit is read only and can + be cleared by disabling the DTHR trigger enable or by disabling + DMA. */ + __IOM uint32_t DTOTCMP : 1; /*!< [2..2] DMA triggered when DCMDCMP = 0, and the amount of data + in the FIFO was enough to complete the DMA operation (greater + than or equal to current TOTCOUNT) when the command completed. + This trigger is default active when the DCMDCMP trigger + isdisabled and there is enough data in the FIFO to complete + the DMA operation. */ + } DMATRIGSTAT_b; + } ; + __IM uint32_t RESERVED3[14]; + + union { + __IOM uint32_t DMACFG; /*!< (@ 0x00000280) DMA Configuration Register */ + + struct { + __IOM uint32_t DMAEN : 1; /*!< [0..0] DMA Enable. Setting this bit to EN will start the DMA + operation. This should be the last DMA related register + set prior to issuing the command */ + __IOM uint32_t DMADIR : 1; /*!< [1..1] Direction */ + __IM uint32_t : 6; + __IOM uint32_t DMAPRI : 1; /*!< [8..8] Sets the Priority of the DMA request */ + __IOM uint32_t DPWROFF : 1; /*!< [9..9] Power off module after DMA is complete. If this bit is + active, the module will request to power off the supply + it is attached to. If there are other units still requiring + power from the same domain, power down will not be performed. */ + } DMACFG_b; + } ; + __IM uint32_t RESERVED4; + + union { + __IOM uint32_t DMATOTCOUNT; /*!< (@ 0x00000288) DMA Total Transfer Count */ + + struct { + __IOM uint32_t TOTCOUNT : 12; /*!< [11..0] Triggered DMA from Command complete event occured. Bit + is read only and can be cleared by disabling the DTHR trigger + enable or by disabling DMA. */ + } DMATOTCOUNT_b; + } ; + + union { + __IOM uint32_t DMATARGADDR; /*!< (@ 0x0000028C) DMA Target Address Register */ + + struct { + __IOM uint32_t TARGADDR : 20; /*!< [19..0] Bits [19:0] of the target byte address for source of + DMA (either read or write). The address can be any byte + alignment, and does not have to be word aligned. In cases + of non-word aligned addresses, the DMA logic will take + care for ensuring only the target bytes are read/written. */ + __IM uint32_t : 8; + __IOM uint32_t TARGADDR28 : 1; /*!< [28..28] Bit 28 of the target byte address for source of DMA + (either read or write). In cases of non-word aligned addresses, + the DMA logic will take care for ensuring only the target + bytes are read/written.Setting to '1' will select the SRAM. + Setting to '0' will select the flash */ + } DMATARGADDR_b; + } ; + + union { + __IOM uint32_t DMASTAT; /*!< (@ 0x00000290) DMA Status Register */ + + struct { + __IOM uint32_t DMATIP : 1; /*!< [0..0] DMA Transfer In Progress indicator. 1 will indicate that + a DMA transfer is active. The DMA transfer may be waiting + on data, transferring data, or waiting for priority.All + of these will be indicated with a 1. A 0 will indicate + that the DMA is fully complete and no further transactions + will be done. This bit is read only. */ + __IOM uint32_t DMACPL : 1; /*!< [1..1] DMA Transfer Complete. This signals the end of the DMA + operation. This bit can be cleared by writing to 0, and + will also be cleared when a new DMA is started. */ + __IOM uint32_t DMAERR : 1; /*!< [2..2] DMA Error. This active high bit signals an error was + encountered during the DMA operation. The bit can be cleared + by writing to 0. Once set, this bit will remain set until + cleared by software. */ + } DMASTAT_b; + } ; + + union { + __IOM uint32_t CQCFG; /*!< (@ 0x00000294) Command Queue Configuration Register */ + + struct { + __IOM uint32_t CQEN : 1; /*!< [0..0] Command queue enable. When set, will enable the processing + of the command queue and fetches of address/data pairs + will proceed from the word address within the CQADDR register. + Can be disabledusing a CQ executed write to this bit as + well. */ + __IOM uint32_t CQPRI : 1; /*!< [1..1] Sets the Priority of the command queue dma request */ + } CQCFG_b; + } ; + + union { + __IOM uint32_t CQADDR; /*!< (@ 0x00000298) CQ Target Read Address Register */ + + struct { + __IM uint32_t : 2; + __IOM uint32_t CQADDR : 18; /*!< [19..2] Bits 19:2 of target byte address for source of CQ (read + only). The buffer must be aligned on a word boundary */ + __IM uint32_t : 8; + __IOM uint32_t CQADDR28 : 1; /*!< [28..28] Bit 28 of target byte address for source of CQ (read + only). Used to denote Flash (0) or SRAM (1) access */ + } CQADDR_b; + } ; + + union { + __IOM uint32_t CQSTAT; /*!< (@ 0x0000029C) Command Queue Status Register */ + + struct { + __IOM uint32_t CQTIP : 1; /*!< [0..0] Command queue Transfer In Progress indicator. 1 will + indicate that a CQ transfer is active and this will remain + active even when paused waiting for external event. */ + __IOM uint32_t CQPAUSED : 1; /*!< [1..1] Command queue operation is currently paused. */ + __IOM uint32_t CQERR : 1; /*!< [2..2] Command queue processing Error. This active high bit + signals that an error was encountered during the CQ operation. */ + } CQSTAT_b; + } ; + + union { + __IOM uint32_t CQFLAGS; /*!< (@ 0x000002A0) Command Queue Flag Register */ + + struct { + __IOM uint32_t CQFLAGS : 16; /*!< [15..0] Current flag status (read-only). Bits [7:0] are software + controllable and bits [15:8] are hardware status. */ + __IOM uint32_t CQIRQMASK : 16; /*!< [31..16] Mask the bits used to generate the command queue interrupt. + A '1' in the bit position will enable the pause event to + trigger the interrupt, if the CQWT_int interrupt is enabled. + Bits definitions are the same as CQPAUSE */ + } CQFLAGS_b; + } ; + + union { + __IOM uint32_t CQSETCLEAR; /*!< (@ 0x000002A4) Command Queue Flag Set/Clear Register */ + + struct { + __IOM uint32_t CQFSET : 8; /*!< [7..0] Set CQFlag status bits. Will set to 1 the value of any + SWFLAG with a '1' in the corresponding bit position of + this field */ + __IOM uint32_t CQFTGL : 8; /*!< [15..8] Toggle the indicated bit. Will toggle the value of any + SWFLAG with a '1' in the corresponding bit position of + this field */ + __IOM uint32_t CQFCLR : 8; /*!< [23..16] Clear CQFlag status bits. Will clear to 0 any SWFLAG + with a '1' in the corresponding bit position of this field */ + } CQSETCLEAR_b; + } ; + + union { + __IOM uint32_t CQPAUSEEN; /*!< (@ 0x000002A8) Command Queue Pause Enable Register */ + + struct { + __IOM uint32_t CQPEN : 16; /*!< [15..0] Enables the specified event to pause command processing + when active */ + } CQPAUSEEN_b; + } ; + + union { + __IOM uint32_t CQCURIDX; /*!< (@ 0x000002AC) IOM Command Queue current index value . Compared + to the CQENDIDX reg contents to generate + the IDXEQ Pause event for command queue */ + + struct { + __IOM uint32_t CQCURIDX : 8; /*!< [7..0] Holds 8 bits of data that will be compared with the CQENDIX + register field. If the values match, the IDXEQ pause event + will be activated, which will cause the pausing of command + quue operation if the IDXEQ bit is enabled in CQPAUSEEN. */ + } CQCURIDX_b; + } ; + + union { + __IOM uint32_t CQENDIDX; /*!< (@ 0x000002B0) IOM Command Queue current index value . Compared + to the CQCURIDX reg contents to generate + the IDXEQ Pause event for command queue */ + + struct { + __IOM uint32_t CQENDIDX : 8; /*!< [7..0] Holds 8 bits of data that will be compared with the CQCURIX + register field. If the values match, the IDXEQ pause event + will be activated, which will cause the pausing of command + quue operation if the IDXEQ bit is enabled in CQPAUSEEN. */ + } CQENDIDX_b; + } ; + + union { + __IOM uint32_t STATUS; /*!< (@ 0x000002B4) IOM Module Status Register */ + + struct { + __IOM uint32_t ERR : 1; /*!< [0..0] Bit has been deprecated. Please refer to the other error + indicators. This will always return 0. */ + __IOM uint32_t CMDACT : 1; /*!< [1..1] Indicates if the active I/O Command is currently processing + a transaction, or command is complete, but the FIFO pointers + are still syncronizing internally. This bit will go high + atthe start of the transaction, and will go low when the + command is complete, and the data and pointers within the + FIFO have been syncronized. */ + __IOM uint32_t IDLEST : 1; /*!< [2..2] indicates if the active I/O state machine is IDLE. Note + - The state machine could be in idle state due to holdoffs + from data availability, or as the command gets propagated + into the logic from the registers. */ + } STATUS_b; + } ; + __IM uint32_t RESERVED5[18]; + + union { + __IOM uint32_t MSPICFG; /*!< (@ 0x00000300) SPI module master configuration */ + + struct { + __IOM uint32_t SPOL : 1; /*!< [0..0] selects SPI polarity. */ + __IOM uint32_t SPHA : 1; /*!< [1..1] selects SPI phase. */ + __IOM uint32_t FULLDUP : 1; /*!< [2..2] Enables full duplex mode for Master SPI write operations. + Data will be captured simultaneously into the read fifo */ + __IM uint32_t : 13; + __IOM uint32_t WTFC : 1; /*!< [16..16] enables write mode flow control. */ + __IOM uint32_t RDFC : 1; /*!< [17..17] enables read mode flow control. */ + __IOM uint32_t MOSIINV : 1; /*!< [18..18] inverts MOSI when flow control is enabled. */ + __IM uint32_t : 1; + __IOM uint32_t WTFCIRQ : 1; /*!< [20..20] selects the write mode flow control signal. */ + __IOM uint32_t WTFCPOL : 1; /*!< [21..21] selects the write flow control signal polarity. The + transfers are halted when the selected flow control signal + is OPPOSITE polarity of bit. (For example: WTFCPOL = 0 + will allow a IRQ=1 to pause transfers). */ + __IOM uint32_t RDFCPOL : 1; /*!< [22..22] selects the read flow control signal polarity. */ + __IOM uint32_t SPILSB : 1; /*!< [23..23] Selects data transfer as MSB first (0) or LSB first + (1) for the data portion of the SPI transaction. The offset + bytes are always transmitted MSB first. */ + __IOM uint32_t DINDLY : 3; /*!< [26..24] Delay tap to use for the input signal (MISO). This + gives more hold time on the input data. */ + __IOM uint32_t DOUTDLY : 3; /*!< [29..27] Delay tap to use for the output signal (MOSI). This + give more hold time on the output data */ + __IOM uint32_t MSPIRST : 1; /*!< [30..30] Not used. To reset the module, toggle the SMOD_EN for + the module */ + } MSPICFG_b; + } ; + __IM uint32_t RESERVED6[63]; + + union { + __IOM uint32_t MI2CCFG; /*!< (@ 0x00000400) I2C Master configuration */ + + struct { + __IOM uint32_t ADDRSZ : 1; /*!< [0..0] Sets the I2C master device address size to either 7b + (0) or 10b (1). */ + __IOM uint32_t I2CLSB : 1; /*!< [1..1] Direction of data transmit and receive, MSB(0) or LSB(1) + first. Default per I2C specification is MSB first. This + applies to both read and write data, and read data will + be bit */ + __IOM uint32_t ARBEN : 1; /*!< [2..2] Enables multi-master arbitration for the I2C master. + If the bus is known to have only a single master, this + function can be disabled to save clock cycles on I2C transactions */ + __IM uint32_t : 1; + __IOM uint32_t SDADLY : 2; /*!< [5..4] Delay to enable on the SDA output. Values are 0x0-0x3. */ + __IOM uint32_t MI2CRST : 1; /*!< [6..6] Not used. To reset the module, toggle the SMOD_EN for + the module */ + __IM uint32_t : 1; + __IOM uint32_t SCLENDLY : 4; /*!< [11..8] Number of IOCLK cycles to delay the rising edge of the + SCL output en (clock will go low on this edge). Used to + allow clock shaping. */ + __IOM uint32_t SDAENDLY : 4; /*!< [15..12] Number of IOCLK cycles to delay the SDA output en (all + transitions affected). Used to delay data relative to clock */ + __IOM uint32_t SMPCNT : 8; /*!< [23..16] Number of Base clk cycles to wait before sampling the + SCL clock to determine if a clock stretch event has occured */ + __IOM uint32_t STRDIS : 1; /*!< [24..24] Disable detection of clock stretch events smaller than + 1 cycle */ + } MI2CCFG_b; + } ; + + union { + __IOM uint32_t DEVCFG; /*!< (@ 0x00000404) I2C Device Configuration register */ + + struct { + __IOM uint32_t DEVADDR : 10; /*!< [9..0] I2C address of the device that the Master will use to + target for read/write operations. This can be either a + 7b or 10b address. */ + } DEVCFG_b; + } ; + __IM uint32_t RESERVED7[2]; + + union { + __IOM uint32_t IOMDBG; /*!< (@ 0x00000410) IOM Debug Register */ + + struct { + __IOM uint32_t DBGEN : 1; /*!< [0..0] Debug Enable. Setting bit will enable the update of data + within this register, otherwise it is clock gated for power + savings */ + __IOM uint32_t IOCLKON : 1; /*!< [1..1] IOCLK debug clock control. Enable IO_CLK to be active + when this bit is '1'. Otherwise, the clock is controlled + with gating from the logic as needed. */ + __IOM uint32_t APBCLKON : 1; /*!< [2..2] APBCLK debug clock control. Enable APB_CLK to be active + when this bit is '1'. Otherwise, the clock is controlled + with gating from the logic as needed. */ + __IOM uint32_t DBGDATA : 29; /*!< [31..3] Debug control for various options. DBGDATA[1:0] is used + to select between different debug data available in the + DBG0 and DBG1 registers. */ + } IOMDBG_b; + } ; +} IOM0_Type; /*!< Size = 1044 (0x414) */ + + + +/* =========================================================================================================================== */ +/* ================ IOSLAVE ================ */ +/* =========================================================================================================================== */ + + +/** + * @brief I2C/SPI Slave (IOSLAVE) + */ + +typedef struct { /*!< (@ 0x50000000) IOSLAVE Structure */ + __IM uint32_t RESERVED[64]; + + union { + __IOM uint32_t FIFOPTR; /*!< (@ 0x00000100) Current FIFO Pointer */ + + struct { + __IOM uint32_t FIFOPTR : 8; /*!< [7..0] Current FIFO pointer. */ + __IOM uint32_t FIFOSIZ : 8; /*!< [15..8] The number of bytes currently in the hardware FIFO. */ + } FIFOPTR_b; + } ; + + union { + __IOM uint32_t FIFOCFG; /*!< (@ 0x00000104) FIFO Configuration */ + + struct { + __IOM uint32_t FIFOBASE : 5; /*!< [4..0] These bits hold the base address of the I/O FIFO in 8 + byte segments. The IO Slave FIFO is situated in LRAM at + (FIFOBASE*8) to (FIFOMAX*8-1). */ + __IM uint32_t : 3; + __IOM uint32_t FIFOMAX : 6; /*!< [13..8] These bits hold the maximum FIFO address in 8 byte segments. + It is also the beginning of the RAM area of the LRAM. Note + that no RAM area is configured if FIFOMAX is set to 0x1F. */ + __IM uint32_t : 10; + __IOM uint32_t ROBASE : 6; /*!< [29..24] Defines the read-only area. The IO Slave read-only + area is situated in LRAM at (ROBASE*8) to (FIFOBASE*8-1) */ + } FIFOCFG_b; + } ; + + union { + __IOM uint32_t FIFOTHR; /*!< (@ 0x00000108) FIFO Threshold Configuration */ + + struct { + __IOM uint32_t FIFOTHR : 8; /*!< [7..0] FIFO size interrupt threshold. */ + } FIFOTHR_b; + } ; + + union { + __IOM uint32_t FUPD; /*!< (@ 0x0000010C) FIFO Update Status */ + + struct { + __IOM uint32_t FIFOUPD : 1; /*!< [0..0] This bit indicates that a FIFO update is underway. */ + __IOM uint32_t IOREAD : 1; /*!< [1..1] This bitfield indicates an IO read is active. */ + } FUPD_b; + } ; + + union { + __IOM uint32_t FIFOCTR; /*!< (@ 0x00000110) Overall FIFO Counter */ + + struct { + __IOM uint32_t FIFOCTR : 10; /*!< [9..0] Virtual FIFO byte count */ + } FIFOCTR_b; + } ; + + union { + __IOM uint32_t FIFOINC; /*!< (@ 0x00000114) Overall FIFO Counter Increment */ + + struct { + __IOM uint32_t FIFOINC : 10; /*!< [9..0] Increment the Overall FIFO Counter by this value on a + write */ + } FIFOINC_b; + } ; + + union { + __IOM uint32_t CFG; /*!< (@ 0x00000118) I/O Slave Configuration */ + + struct { + __IOM uint32_t IFCSEL : 1; /*!< [0..0] This bit selects the I/O interface. */ + __IOM uint32_t SPOL : 1; /*!< [1..1] This bit selects SPI polarity. */ + __IOM uint32_t LSB : 1; /*!< [2..2] This bit selects the transfer bit ordering. */ + __IM uint32_t : 1; + __IOM uint32_t STARTRD : 1; /*!< [4..4] This bit holds the cycle to initiate an I/O RAM read. */ + __IM uint32_t : 3; + __IOM uint32_t I2CADDR : 12; /*!< [19..8] 7-bit or 10-bit I2C device address. */ + __IM uint32_t : 11; + __IOM uint32_t IFCEN : 1; /*!< [31..31] IOSLAVE interface enable. */ + } CFG_b; + } ; + + union { + __IOM uint32_t PRENC; /*!< (@ 0x0000011C) I/O Slave Interrupt Priority Encode */ + + struct { + __IOM uint32_t PRENC : 5; /*!< [4..0] These bits hold the priority encode of the REGACC interrupts. */ + } PRENC_b; + } ; + + union { + __IOM uint32_t IOINTCTL; /*!< (@ 0x00000120) I/O Interrupt Control */ + + struct { + __IOM uint32_t IOINTEN : 8; /*!< [7..0] These read-only bits indicate whether the IOINT interrupts + are enabled. */ + __IOM uint32_t IOINT : 8; /*!< [15..8] These bits read the IOINT interrupts. */ + __IOM uint32_t IOINTCLR : 1; /*!< [16..16] This bit clears all of the IOINT interrupts when written + with a 1. */ + __IM uint32_t : 7; + __IOM uint32_t IOINTSET : 8; /*!< [31..24] These bits set the IOINT interrupts when written with + a 1. */ + } IOINTCTL_b; + } ; + + union { + __IOM uint32_t GENADD; /*!< (@ 0x00000124) General Address Data */ + + struct { + __IOM uint32_t GADATA : 8; /*!< [7..0] The data supplied on the last General Address reference. */ + } GENADD_b; + } ; + __IM uint32_t RESERVED1[54]; + + union { + __IOM uint32_t INTEN; /*!< (@ 0x00000200) IO Slave Interrupts: Enable */ + + struct { + __IOM uint32_t FSIZE : 1; /*!< [0..0] FIFO Size interrupt. */ + __IOM uint32_t FOVFL : 1; /*!< [1..1] FIFO Overflow interrupt. */ + __IOM uint32_t FUNDFL : 1; /*!< [2..2] FIFO Underflow interrupt. */ + __IOM uint32_t FRDERR : 1; /*!< [3..3] FIFO Read Error interrupt. */ + __IOM uint32_t GENAD : 1; /*!< [4..4] I2C General Address interrupt. */ + __IOM uint32_t IOINTW : 1; /*!< [5..5] IO Write interrupt. */ + __IOM uint32_t XCMPRF : 1; /*!< [6..6] Transfer complete interrupt, read from FIFO space. */ + __IOM uint32_t XCMPRR : 1; /*!< [7..7] Transfer complete interrupt, read from register space. */ + __IOM uint32_t XCMPWF : 1; /*!< [8..8] Transfer complete interrupt, write to FIFO space. */ + __IOM uint32_t XCMPWR : 1; /*!< [9..9] Transfer complete interrupt, write to register space. */ + } INTEN_b; + } ; + + union { + __IOM uint32_t INTSTAT; /*!< (@ 0x00000204) IO Slave Interrupts: Status */ + + struct { + __IOM uint32_t FSIZE : 1; /*!< [0..0] FIFO Size interrupt. */ + __IOM uint32_t FOVFL : 1; /*!< [1..1] FIFO Overflow interrupt. */ + __IOM uint32_t FUNDFL : 1; /*!< [2..2] FIFO Underflow interrupt. */ + __IOM uint32_t FRDERR : 1; /*!< [3..3] FIFO Read Error interrupt. */ + __IOM uint32_t GENAD : 1; /*!< [4..4] I2C General Address interrupt. */ + __IOM uint32_t IOINTW : 1; /*!< [5..5] IO Write interrupt. */ + __IOM uint32_t XCMPRF : 1; /*!< [6..6] Transfer complete interrupt, read from FIFO space. */ + __IOM uint32_t XCMPRR : 1; /*!< [7..7] Transfer complete interrupt, read from register space. */ + __IOM uint32_t XCMPWF : 1; /*!< [8..8] Transfer complete interrupt, write to FIFO space. */ + __IOM uint32_t XCMPWR : 1; /*!< [9..9] Transfer complete interrupt, write to register space. */ + } INTSTAT_b; + } ; + + union { + __IOM uint32_t INTCLR; /*!< (@ 0x00000208) IO Slave Interrupts: Clear */ + + struct { + __IOM uint32_t FSIZE : 1; /*!< [0..0] FIFO Size interrupt. */ + __IOM uint32_t FOVFL : 1; /*!< [1..1] FIFO Overflow interrupt. */ + __IOM uint32_t FUNDFL : 1; /*!< [2..2] FIFO Underflow interrupt. */ + __IOM uint32_t FRDERR : 1; /*!< [3..3] FIFO Read Error interrupt. */ + __IOM uint32_t GENAD : 1; /*!< [4..4] I2C General Address interrupt. */ + __IOM uint32_t IOINTW : 1; /*!< [5..5] IO Write interrupt. */ + __IOM uint32_t XCMPRF : 1; /*!< [6..6] Transfer complete interrupt, read from FIFO space. */ + __IOM uint32_t XCMPRR : 1; /*!< [7..7] Transfer complete interrupt, read from register space. */ + __IOM uint32_t XCMPWF : 1; /*!< [8..8] Transfer complete interrupt, write to FIFO space. */ + __IOM uint32_t XCMPWR : 1; /*!< [9..9] Transfer complete interrupt, write to register space. */ + } INTCLR_b; + } ; + + union { + __IOM uint32_t INTSET; /*!< (@ 0x0000020C) IO Slave Interrupts: Set */ + + struct { + __IOM uint32_t FSIZE : 1; /*!< [0..0] FIFO Size interrupt. */ + __IOM uint32_t FOVFL : 1; /*!< [1..1] FIFO Overflow interrupt. */ + __IOM uint32_t FUNDFL : 1; /*!< [2..2] FIFO Underflow interrupt. */ + __IOM uint32_t FRDERR : 1; /*!< [3..3] FIFO Read Error interrupt. */ + __IOM uint32_t GENAD : 1; /*!< [4..4] I2C General Address interrupt. */ + __IOM uint32_t IOINTW : 1; /*!< [5..5] IO Write interrupt. */ + __IOM uint32_t XCMPRF : 1; /*!< [6..6] Transfer complete interrupt, read from FIFO space. */ + __IOM uint32_t XCMPRR : 1; /*!< [7..7] Transfer complete interrupt, read from register space. */ + __IOM uint32_t XCMPWF : 1; /*!< [8..8] Transfer complete interrupt, write to FIFO space. */ + __IOM uint32_t XCMPWR : 1; /*!< [9..9] Transfer complete interrupt, write to register space. */ + } INTSET_b; + } ; + + union { + __IOM uint32_t REGACCINTEN; /*!< (@ 0x00000210) Register Access Interrupts: Enable */ + + struct { + __IOM uint32_t REGACC : 32; /*!< [31..0] Register access interrupts. */ + } REGACCINTEN_b; + } ; + + union { + __IOM uint32_t REGACCINTSTAT; /*!< (@ 0x00000214) Register Access Interrupts: Status */ + + struct { + __IOM uint32_t REGACC : 32; /*!< [31..0] Register access interrupts. */ + } REGACCINTSTAT_b; + } ; + + union { + __IOM uint32_t REGACCINTCLR; /*!< (@ 0x00000218) Register Access Interrupts: Clear */ + + struct { + __IOM uint32_t REGACC : 32; /*!< [31..0] Register access interrupts. */ + } REGACCINTCLR_b; + } ; + + union { + __IOM uint32_t REGACCINTSET; /*!< (@ 0x0000021C) Register Access Interrupts: Set */ + + struct { + __IOM uint32_t REGACC : 32; /*!< [31..0] Register access interrupts. */ + } REGACCINTSET_b; + } ; +} IOSLAVE_Type; /*!< Size = 544 (0x220) */ + + + +/* =========================================================================================================================== */ +/* ================ MCUCTRL ================ */ +/* =========================================================================================================================== */ + + +/** + * @brief MCU Miscellaneous Control Logic (MCUCTRL) + */ + +typedef struct { /*!< (@ 0x40020000) MCUCTRL Structure */ + + union { + __IOM uint32_t CHIPPN; /*!< (@ 0x00000000) Chip Information Register */ + + struct { + __IOM uint32_t PARTNUM : 32; /*!< [31..0] BCD part number. */ + } CHIPPN_b; + } ; + + union { + __IOM uint32_t CHIPID0; /*!< (@ 0x00000004) Unique Chip ID 0 */ + + struct { + __IOM uint32_t CHIPID0 : 32; /*!< [31..0] Unique chip ID 0. */ + } CHIPID0_b; + } ; + + union { + __IOM uint32_t CHIPID1; /*!< (@ 0x00000008) Unique Chip ID 1 */ + + struct { + __IOM uint32_t CHIPID1 : 32; /*!< [31..0] Unique chip ID 1. */ + } CHIPID1_b; + } ; + + union { + __IOM uint32_t CHIPREV; /*!< (@ 0x0000000C) Chip Revision */ + + struct { + __IOM uint32_t REVMIN : 4; /*!< [3..0] Minor Revision ID. */ + __IOM uint32_t REVMAJ : 4; /*!< [7..4] Major Revision ID. */ + __IOM uint32_t SIPART : 12; /*!< [19..8] Silicon Part ID */ + } CHIPREV_b; + } ; + + union { + __IOM uint32_t VENDORID; /*!< (@ 0x00000010) Unique Vendor ID */ + + struct { + __IOM uint32_t VENDORID : 32; /*!< [31..0] Unique Vendor ID */ + } VENDORID_b; + } ; + + union { + __IOM uint32_t SKU; /*!< (@ 0x00000014) Unique Chip SKU */ + + struct { + __IOM uint32_t ALLOWBURST : 1; /*!< [0..0] Allow Burst feature */ + __IOM uint32_t ALLOWBLE : 1; /*!< [1..1] Allow BLE feature */ + __IOM uint32_t SECBOOT : 1; /*!< [2..2] Secure boot feature allowed */ + } SKU_b; + } ; + + union { + __IOM uint32_t FEATUREENABLE; /*!< (@ 0x00000018) Feature Enable on Burst and BLE */ + + struct { + __IOM uint32_t BLEREQ : 1; /*!< [0..0] Controls the BLE functionality */ + __IOM uint32_t BLEACK : 1; /*!< [1..1] ACK for BLEREQ */ + __IOM uint32_t BLEAVAIL : 1; /*!< [2..2] AVAILABILITY of the BLE functionality */ + __IM uint32_t : 1; + __IOM uint32_t BURSTREQ : 1; /*!< [4..4] Controls the Burst functionality */ + __IOM uint32_t BURSTACK : 1; /*!< [5..5] ACK for BURSTREQ */ + __IOM uint32_t BURSTAVAIL : 1; /*!< [6..6] Availability of Burst functionality */ + } FEATUREENABLE_b; + } ; + __IM uint32_t RESERVED; + + union { + __IOM uint32_t DEBUGGER; /*!< (@ 0x00000020) Debugger Control */ + + struct { + __IOM uint32_t LOCKOUT : 1; /*!< [0..0] Lockout of debugger (SWD). */ + } DEBUGGER_b; + } ; + __IM uint32_t RESERVED1[55]; + + union { + __IOM uint32_t BODCTRL; /*!< (@ 0x00000100) BOD control Register */ + + struct { + __IOM uint32_t BODLPWD : 1; /*!< [0..0] BODL Power Down. */ + __IOM uint32_t BODHPWD : 1; /*!< [1..1] BODH Power Down. */ + __IOM uint32_t BODCPWD : 1; /*!< [2..2] BODC Power Down. */ + __IOM uint32_t BODFPWD : 1; /*!< [3..3] BODF Power Down. */ + __IOM uint32_t BODLVREFSEL : 1; /*!< [4..4] BODL External Reference Select. Note: the SWE mux select + in PWRSEQ2SWE must be set for this to take effect. */ + __IOM uint32_t BODHVREFSEL : 1; /*!< [5..5] BODH External Reference Select. Note: the SWE mux select + in PWRSEQ2SWE must be set for this to take effect. */ + } BODCTRL_b; + } ; + + union { + __IOM uint32_t ADCPWRDLY; /*!< (@ 0x00000104) ADC Power Up Delay Control */ + + struct { + __IOM uint32_t ADCPWR0 : 8; /*!< [7..0] ADC Reference Buffer Power Enable delay in 64 ADC CLK + increments for ADC_CLKSEL = 0x1, 32 ADC CLOCK increments + for ADC_CLKSEL = 0x2. */ + __IOM uint32_t ADCPWR1 : 8; /*!< [15..8] ADC Reference Keeper enable delay in 16 ADC CLK increments + for ADC_CLKSEL = 0x1, 8 ADC CLOCK increments for ADC_CLKSEL + = 0x2. */ + } ADCPWRDLY_b; + } ; + __IM uint32_t RESERVED2; + + union { + __IOM uint32_t ADCCAL; /*!< (@ 0x0000010C) ADC Calibration Control */ + + struct { + __IOM uint32_t CALONPWRUP : 1; /*!< [0..0] Run ADC Calibration on initial power up sequence */ + __IOM uint32_t ADCCALIBRATED : 1; /*!< [1..1] Status for ADC Calibration */ + } ADCCAL_b; + } ; + + union { + __IOM uint32_t ADCBATTLOAD; /*!< (@ 0x00000110) ADC Battery Load Enable */ + + struct { + __IOM uint32_t BATTLOAD : 1; /*!< [0..0] Enable the ADC battery load resistor */ + } ADCBATTLOAD_b; + } ; + __IM uint32_t RESERVED3; + + union { + __IOM uint32_t ADCTRIM; /*!< (@ 0x00000118) ADC Trims */ + + struct { + __IOM uint32_t ADCREFKEEPIBTRIM : 2; /*!< [1..0] ADC Reference Ibias trim */ + __IM uint32_t : 4; + __IOM uint32_t ADCREFBUFTRIM : 5; /*!< [10..6] ADC Reference buffer trim */ + __IOM uint32_t ADCRFBUFIBTRIM : 2; /*!< [12..11] ADC reference buffer input bias trim */ + } ADCTRIM_b; + } ; + + union { + __IOM uint32_t ADCREFCOMP; /*!< (@ 0x0000011C) ADC Referece Keeper and Comparator Control */ + + struct { + __IOM uint32_t ADC_REFCOMP_OUT : 1; /*!< [0..0] Output of the ADC reference comparator */ + __IM uint32_t : 7; + __IOM uint32_t ADCREFKEEPTRIM : 5; /*!< [12..8] ADC Reference Keeper Trim */ + __IM uint32_t : 3; + __IOM uint32_t ADCRFCMPEN : 1; /*!< [16..16] ADC Reference comparator power down */ + } ADCREFCOMP_b; + } ; + + union { + __IOM uint32_t XTALCTRL; /*!< (@ 0x00000120) XTAL Oscillator Control */ + + struct { + __IOM uint32_t XTALSWE : 1; /*!< [0..0] XTAL Software Override Enable. */ + __IOM uint32_t FDBKDSBLXTAL : 1; /*!< [1..1] XTAL Oscillator Disable Feedback. */ + __IOM uint32_t BYPCMPRXTAL : 1; /*!< [2..2] XTAL Oscillator Bypass Comparator. */ + __IOM uint32_t PDNBCOREXTAL : 1; /*!< [3..3] XTAL Oscillator Power Down Core. */ + __IOM uint32_t PDNBCMPRXTAL : 1; /*!< [4..4] XTAL Oscillator Power Down Comparator. */ + __IOM uint32_t PWDBODXTAL : 1; /*!< [5..5] XTAL Power down on brown out. */ + __IOM uint32_t XTALIBUFTRIM : 2; /*!< [7..6] XTAL IBUFF trim */ + __IOM uint32_t XTALICOMPTRIM : 2; /*!< [9..8] XTAL ICOMP trim */ + } XTALCTRL_b; + } ; + + union { + __IOM uint32_t XTALGENCTRL; /*!< (@ 0x00000124) XTAL Oscillator General Control */ + + struct { + __IOM uint32_t ACWARMUP : 2; /*!< [1..0] Auto-calibration delay control */ + __IOM uint32_t XTALBIASTRIM : 6; /*!< [7..2] XTAL BIAS trim */ + __IOM uint32_t XTALKSBIASTRIM : 6; /*!< [13..8] XTAL IBIAS Kick start trim. This trim value is used + during the startup process to enable a faster lock. */ + } XTALGENCTRL_b; + } ; + __IM uint32_t RESERVED4[28]; + + union { + __IOM uint32_t MISCCTRL; /*!< (@ 0x00000198) Miscellaneous control register. */ + + struct { + __IOM uint32_t RESERVED_RW_0 : 5; /*!< [4..0] Reserved bits, always leave unchanged. The MISCCTRL register + must be modified via atomic RMW, leaving this bitfield + completely unmodified. Failure to do so will result in + unpredictable behavior. */ + __IOM uint32_t BLE_RESETN : 1; /*!< [5..5] BLE reset signal. */ + } MISCCTRL_b; + } ; + __IM uint32_t RESERVED5; + + union { + __IOM uint32_t BOOTLOADER; /*!< (@ 0x000001A0) Bootloader and secure boot functions */ + + struct { + __IOM uint32_t BOOTLOADERLOW : 1; /*!< [0..0] Determines whether the bootloader code is visible at + address 0x00000000 or not. Resets to 1, write 1 to clear. */ + __IOM uint32_t SBLOCK : 1; /*!< [1..1] Secure boot lock. Always resets to 1, write 1 to clear. + Enables system visibility to bootloader until set. */ + __IOM uint32_t PROTLOCK : 1; /*!< [2..2] Flash protection lock. Always resets to 1, write 1 to + clear. Enables writes to flash protection register set. */ + __IM uint32_t : 23; + __IOM uint32_t SECBOOTFEATURE : 2; /*!< [27..26] Indicates whether the secure boot feature is enabled. */ + __IOM uint32_t SECBOOT : 2; /*!< [29..28] Indicates whether the secure boot on cold reset is + enabled */ + __IOM uint32_t SECBOOTONRST : 2; /*!< [31..30] Indicates whether the secure boot on warm reset is + enabled */ + } BOOTLOADER_b; + } ; + + union { + __IOM uint32_t SHADOWVALID; /*!< (@ 0x000001A4) Register to indicate whether the shadow registers + have been successfully loaded from the Flash + Information Space. */ + + struct { + __IOM uint32_t VALID : 1; /*!< [0..0] Indicates whether the shadow registers contain valid + data from the Flash Information Space. */ + __IOM uint32_t BLDSLEEP : 1; /*!< [1..1] Indicates whether the bootloader should sleep or deep + sleep if no image loaded. */ + __IOM uint32_t INFO0_VALID : 1; /*!< [2..2] Indicates whether info0 contains valid data */ + } SHADOWVALID_b; + } ; + __IM uint32_t RESERVED6[2]; + + union { + __IOM uint32_t SCRATCH0; /*!< (@ 0x000001B0) Scratch register that is not reset by any reset */ + + struct { + __IOM uint32_t SCRATCH0 : 32; /*!< [31..0] Scratch register 0. */ + } SCRATCH0_b; + } ; + + union { + __IOM uint32_t SCRATCH1; /*!< (@ 0x000001B4) Scratch register that is not reset by any reset */ + + struct { + __IOM uint32_t SCRATCH1 : 32; /*!< [31..0] Scratch register 1. */ + } SCRATCH1_b; + } ; + __IM uint32_t RESERVED7[2]; + + union { + __IOM uint32_t ICODEFAULTADDR; /*!< (@ 0x000001C0) ICODE bus address which was present when a bus + fault occurred. */ + + struct { + __IOM uint32_t ICODEFAULTADDR : 32; /*!< [31..0] The ICODE bus address observed when a Bus Fault occurred. + Once an address is captured in this field, it is held until + the corresponding Fault Observed bit is cleared in the + FAULTSTATUS register. */ + } ICODEFAULTADDR_b; + } ; + + union { + __IOM uint32_t DCODEFAULTADDR; /*!< (@ 0x000001C4) DCODE bus address which was present when a bus + fault occurred. */ + + struct { + __IOM uint32_t DCODEFAULTADDR : 32; /*!< [31..0] The DCODE bus address observed when a Bus Fault occurred. + Once an address is captured in this field, it is held until + the corresponding Fault Observed bit is cleared in the + FAULTSTATUS register. */ + } DCODEFAULTADDR_b; + } ; + + union { + __IOM uint32_t SYSFAULTADDR; /*!< (@ 0x000001C8) System bus address which was present when a bus + fault occurred. */ + + struct { + __IOM uint32_t SYSFAULTADDR : 32; /*!< [31..0] SYS bus address observed when a Bus Fault occurred. + Once an address is captured in this field, it is held until + the corresponding Fault Observed bit is cleared in the + FAULTSTATUS register. */ + } SYSFAULTADDR_b; + } ; + + union { + __IOM uint32_t FAULTSTATUS; /*!< (@ 0x000001CC) Reflects the status of the bus decoders' fault + detection. Any write to this register will + clear all of the status bits within the + register. */ + + struct { + __IOM uint32_t ICODEFAULT : 1; /*!< [0..0] The ICODE Bus Decoder Fault Detected bit. When set, a + fault has been detected, and the ICODEFAULTADDR register + will contain the bus address which generated the fault. */ + __IOM uint32_t DCODEFAULT : 1; /*!< [1..1] DCODE Bus Decoder Fault Detected bit. When set, a fault + has been detected, and the DCODEFAULTADDR register will + contain the bus address which generated the fault. */ + __IOM uint32_t SYSFAULT : 1; /*!< [2..2] SYS Bus Decoder Fault Detected bit. When set, a fault + has been detected, and the SYSFAULTADDR register will contain + the bus address which generated the fault. */ + } FAULTSTATUS_b; + } ; + + union { + __IOM uint32_t FAULTCAPTUREEN; /*!< (@ 0x000001D0) Enable the fault capture registers */ + + struct { + __IOM uint32_t FAULTCAPTUREEN : 1; /*!< [0..0] Fault Capture Enable field. When set, the Fault Capture + monitors are enabled and addresses which generate a hard + fault are captured into the FAULTADDR registers. */ + } FAULTCAPTUREEN_b; + } ; + __IM uint32_t RESERVED8[11]; + + union { + __IOM uint32_t DBGR1; /*!< (@ 0x00000200) Read-only debug register 1 */ + + struct { + __IOM uint32_t ONETO8 : 32; /*!< [31..0] Read-only register for communication validation */ + } DBGR1_b; + } ; + + union { + __IOM uint32_t DBGR2; /*!< (@ 0x00000204) Read-only debug register 2 */ + + struct { + __IOM uint32_t COOLCODE : 32; /*!< [31..0] Read-only register for communication validation */ + } DBGR2_b; + } ; + __IM uint32_t RESERVED9[6]; + + union { + __IOM uint32_t PMUENABLE; /*!< (@ 0x00000220) Control bit to enable/disable the PMU */ + + struct { + __IOM uint32_t ENABLE : 1; /*!< [0..0] PMU Enable Control bit. When set, the MCU's PMU will + place the MCU into the lowest power consuming Deep Sleep + mode upon execution of a WFI instruction (dependent on + the setting of the SLEEPDEEP bit in the ARM SCR register). + When cleared, regardless of the requested sleep mode, the + PMU will not enter the lowest power Deep Sleep mode, instead + entering the Sleep mode. */ + } PMUENABLE_b; + } ; + __IM uint32_t RESERVED10[11]; + + union { + __IOM uint32_t TPIUCTRL; /*!< (@ 0x00000250) TPIU Control Register. Determines the clock enable + and frequency for the M4's TPIU interface. */ + + struct { + __IOM uint32_t ENABLE : 1; /*!< [0..0] TPIU Enable field. When set, the ARM M4 TPIU is enabled + and data can be streamed out of the MCU's SWO port using + the ARM ITM and TPIU modules. */ + __IM uint32_t : 7; + __IOM uint32_t CLKSEL : 3; /*!< [10..8] This field selects the frequency of the ARM M4 TPIU + port. */ + } TPIUCTRL_b; + } ; + __IM uint32_t RESERVED11[4]; + + union { + __IOM uint32_t OTAPOINTER; /*!< (@ 0x00000264) OTA (Over the Air) Update Pointer/Status. Reset + only by POA */ + + struct { + __IOM uint32_t OTAVALID : 1; /*!< [0..0] Indicates that an OTA update is valid */ + __IOM uint32_t OTASBLUPDATE : 1; /*!< [1..1] Indicates that the sbl_init has been updated */ + __IOM uint32_t OTAPOINTER : 30; /*!< [31..2] Flash page pointer with updated OTA image */ + } OTAPOINTER_b; + } ; + __IM uint32_t RESERVED12[6]; + + union { + __IOM uint32_t APBDMACTRL; /*!< (@ 0x00000280) DMA Control Register. Determines misc settings + for DMA operation */ + + struct { + __IOM uint32_t DMA_ENABLE : 1; /*!< [0..0] Enable the DMA controller. When disabled, DMA requests + will be ignored by the controller */ + __IOM uint32_t DECODEABORT : 1; /*!< [1..1] APB Decode Abort. When set, the APB bridge will issue + a data abort (bus fault) on transactions to peripherals + that are powered down. When set to 0, writes are quietly + discarded and reads return 0. */ + __IM uint32_t : 6; + __IOM uint32_t HYSTERESIS : 8; /*!< [15..8] This field determines how long the DMA will remain active + during deep sleep before shutting down and returning the + system to full deep sleep. Values are based on a 94KHz + clock and are roughly 10us increments for a range of ~10us + to 2.55ms */ + } APBDMACTRL_b; + } ; + + union { + __IOM uint32_t SRAMMODE; /*!< (@ 0x00000284) SRAM Controller mode bits */ + + struct { + __IOM uint32_t IPREFETCH : 1; /*!< [0..0] When set, instruction accesses to the SRAM banks will + be prefetched (normally 2 cycle read access). Generally, + this mode bit should be set for improved performance when + executing instructions from SRAM. */ + __IOM uint32_t IPREFETCH_CACHE : 1; /*!< [1..1] Secondary prefetch feature that will cache prefetched + data across bus waitstates (requires IPREFETCH to be set). */ + __IM uint32_t : 2; + __IOM uint32_t DPREFETCH : 1; /*!< [4..4] When set, data bus accesses to the SRAM banks will be + prefetched (normally 2 cycle read access). Use of this + mode bit is only recommended if the work flow has a large + number of sequential accesses. */ + __IOM uint32_t DPREFETCH_CACHE : 1; /*!< [5..5] Secondary prefetch feature that will cache prefetched + data across bus waitstates (requires DPREFETCH to be set). */ + } SRAMMODE_b; + } ; + __IM uint32_t RESERVED13[48]; + + union { + __IOM uint32_t KEXTCLKSEL; /*!< (@ 0x00000348) Key Register to enable the use of external clock + selects via the EXTCLKSEL reg */ + + struct { + __IOM uint32_t KEXTCLKSEL : 32; /*!< [31..0] Key register value. */ + } KEXTCLKSEL_b; + } ; + __IM uint32_t RESERVED14[4]; + + union { + __IOM uint32_t SIMOBUCK4; /*!< (@ 0x0000035C) SIMO Buck Control Reg1 */ + + struct { + __IOM uint32_t SIMOBUCKMEMLPLOWTONTRIM : 4;/*!< [3..0] simobuck_mem_lp_low_ton_trim */ + __IOM uint32_t SIMOBUCKMEMACTDRVSTRTRIM : 2;/*!< [5..4] simobuck_mem_act_drvstr_trim */ + __IOM uint32_t SIMOBUCKMEMLPDRVSTRTRIM : 2;/*!< [7..6] simobuck_mem_lp_drvstr_trim */ + __IOM uint32_t SIMOBUCKMEMLEAKAGETRIM : 2;/*!< [9..8] simobuck_mem_leakage_trim */ + __IOM uint32_t SIMOBUCKZXTRIM : 4; /*!< [13..10] simobuck_zx_trim */ + __IOM uint32_t SIMOBUCKUVLOCNTRTRIM : 3; /*!< [16..14] simobuck_uvlo_cntr_trim */ + __IOM uint32_t SIMOBUCKUVLODRVSTRTRIM : 3;/*!< [19..17] simobuck_uvlo_drvstr_trim */ + __IOM uint32_t SIMOBUCKEXTCLKSEL : 1; /*!< [20..20] simobuck_extclk_sel */ + __IOM uint32_t SIMOBUCKCLKDIVSEL : 2; /*!< [22..21] simobuck_clkdiv_sel */ + __IOM uint32_t SIMOBUCKCOMP2LPEN : 1; /*!< [23..23] simobuck_comp2_lp_en */ + __IOM uint32_t SIMOBUCKCOMP2TIMEOUTEN : 1;/*!< [24..24] simobuck_comp2_timeout_en */ + __IOM uint32_t SIMOBUCKPRIORITYSEL : 1; /*!< [25..25] simobuck_priority_sel */ + __IOM uint32_t SIMOBUCKUVLOMODE : 2; /*!< [27..26] simobuck_uvlo_mode */ + __IOM uint32_t SIMOBUCKIBIASTRIM : 4; /*!< [31..28] simobuck_bias_trim */ + } SIMOBUCK4_b; + } ; + __IM uint32_t RESERVED15[2]; + + union { + __IOM uint32_t BLEBUCK2; /*!< (@ 0x00000368) BLEBUCK2 Control Reg */ + + struct { + __IOM uint32_t BLEBUCKTONLOWTRIM : 6; /*!< [5..0] blebuck_ton_low_trim */ + __IOM uint32_t BLEBUCKTONHITRIM : 6; /*!< [11..6] blebuck_ton_hi_trim */ + __IOM uint32_t BLEBUCKTOND2ATRIM : 6; /*!< [17..12] blebuck_ton_trim */ + } BLEBUCK2_b; + } ; + __IM uint32_t RESERVED16[13]; + + union { + __IOM uint32_t FLASHWPROT0; /*!< (@ 0x000003A0) Flash Write Protection Bits */ + + struct { + __IOM uint32_t FW0BITS : 32; /*!< [31..0] Write protect flash 0x00000000 - 0x0007FFFF. Each bit + provides write protection for 16KB chunks of flash data + space. Bits are cleared by writing a 1 to the bit. When + read, 0 indicates the region is protected. Bits are sticky + (can be set when PROTLOCK is 1, but only cleared by reset) */ + } FLASHWPROT0_b; + } ; + + union { + __IOM uint32_t FLASHWPROT1; /*!< (@ 0x000003A4) Flash Write Protection Bits */ + + struct { + __IOM uint32_t FW1BITS : 32; /*!< [31..0] Write protect flash 0x00080000 - 0x000FFFFF. Each bit + provides write protection for 16KB chunks of flash data + space. Bits are cleared by writing a 1 to the bit. When + read, 0 indicates the region is protected. Bits are sticky + (can be set when PROTLOCK is 1, but only cleared by reset) */ + } FLASHWPROT1_b; + } ; + __IM uint32_t RESERVED17[2]; + + union { + __IOM uint32_t FLASHRPROT0; /*!< (@ 0x000003B0) Flash Read Protection Bits */ + + struct { + __IOM uint32_t FR0BITS : 32; /*!< [31..0] Copy (read) protect flash 0x00000000 - 0x0007FFFF. Each + bit provides read protection for 16KB chunks of flash. + Bits are cleared by writing a 1 to the bit. When read, + 0 indicates the region is protected. Bits are sticky (can + be set when PROTLOCK is 1, but only cleared by reset) */ + } FLASHRPROT0_b; + } ; + + union { + __IOM uint32_t FLASHRPROT1; /*!< (@ 0x000003B4) Flash Read Protection Bits */ + + struct { + __IOM uint32_t FR1BITS : 32; /*!< [31..0] Copy (read) protect flash 0x00080000 - 0x000FFFFF. Each + bit provides read protection for 16KB chunks of flash. + Bits are cleared by writing a 1 to the bit. When read, + 0 indicates the region is protected. Bits are sticky (can + be set when PROTLOCK is 1, but only cleared by reset) */ + } FLASHRPROT1_b; + } ; + __IM uint32_t RESERVED18[2]; + + union { + __IOM uint32_t DMASRAMWRITEPROTECT0; /*!< (@ 0x000003C0) SRAM write-protection bits. */ + + struct { + __IOM uint32_t DMA_WPROT0 : 32; /*!< [31..0] Write protect SRAM from DMA. Each bit provides write + protection for an 8KB region of memory. When set to 1, + the region will be protected from DMA writes, when set + to 0, DMA may write the region. */ + } DMASRAMWRITEPROTECT0_b; + } ; + + union { + __IOM uint32_t DMASRAMWRITEPROTECT1; /*!< (@ 0x000003C4) SRAM write-protection bits. */ + + struct { + __IOM uint32_t DMA_WPROT1 : 16; /*!< [15..0] Write protect SRAM from DMA. Each bit provides write + protection for an 8KB region of memory. When set to 1, + the region will be protected from DMA writes, when set + to 0, DMA may write the region. */ + } DMASRAMWRITEPROTECT1_b; + } ; + __IM uint32_t RESERVED19[2]; + + union { + __IOM uint32_t DMASRAMREADPROTECT0; /*!< (@ 0x000003D0) SRAM read-protection bits. */ + + struct { + __IOM uint32_t DMA_RPROT0 : 32; /*!< [31..0] Read protect SRAM from DMA. Each bit provides write + protection for an 8KB region of memory. When set to 1, + the region will be protected from DMA reads, when set to + 0, DMA may read the region. */ + } DMASRAMREADPROTECT0_b; + } ; + + union { + __IOM uint32_t DMASRAMREADPROTECT1; /*!< (@ 0x000003D4) SRAM read-protection bits. */ + + struct { + __IOM uint32_t DMA_RPROT1 : 16; /*!< [15..0] Read protect SRAM from DMA. Each bit provides write + protection for an 8KB region of memory. When set to 1, + the region will be protected from DMA reads, when set to + 0, DMA may read the region. */ + } DMASRAMREADPROTECT1_b; + } ; +} MCUCTRL_Type; /*!< Size = 984 (0x3d8) */ + + + +/* =========================================================================================================================== */ +/* ================ MSPI ================ */ +/* =========================================================================================================================== */ + + +/** + * @brief Multibit SPI Master (MSPI) + */ + +typedef struct { /*!< (@ 0x50014000) MSPI Structure */ + + union { + __IOM uint32_t CTRL; /*!< (@ 0x00000000) MSPI PIO Transfer Control/Status Register */ + + struct { + __IOM uint32_t START : 1; /*!< [0..0] Write to 1 to initiate a PIO transaction on the bus (typically + the entire register should be written at once with this + bit set). */ + __IOM uint32_t STATUS : 1; /*!< [1..1] Command status: 1 indicates command has completed. Cleared + by writing 1 to this bit or starting a new transfer. */ + __IOM uint32_t BUSY : 1; /*!< [2..2] Command status: 1 indicates controller is busy (command + in progress) */ + __IOM uint32_t QUADCMD : 1; /*!< [3..3] Flag indicating that the operation is a command that + should be replicated to both devices in paired QUAD mode. + This is typically only used when reading/writing configuration + registers in paired flash devices (do not set for memory + transfers). */ + __IM uint32_t : 1; + __IOM uint32_t CONT : 1; /*!< [5..5] Continuation transfer. When 1, indicates that the MSPI + will hold CE low after the transaction completes. This + is included for compatibility with IOM module since the + MSPI transfer module can handle most cases in a single + transfer. NOTE: CONT functionality only works with CLKDIV=2 + (24 MHz). */ + __IOM uint32_t BIGENDIAN : 1; /*!< [6..6] 1 indicates data in FIFO is in big endian format (MSB + first); 0 indicates little endian data (default, LSB first). */ + __IOM uint32_t ENTURN : 1; /*!< [7..7] Indicates whether TX->RX turnaround cycles should be + enabled for this operation (see TURNAROUND field in CFG + register). */ + __IOM uint32_t SENDA : 1; /*!< [8..8] Indicates whether an address phase should be sent (see + ADDR register and ASIZE field in CFG register) */ + __IOM uint32_t SENDI : 1; /*!< [9..9] Indicates whether an instruction phase should be sent + (see INSTR field and ISIZE field in CFG register) */ + __IOM uint32_t TXRX : 1; /*!< [10..10] 1 Indicates a TX operation, 0 indicates an RX operation + of XFERBYTES */ + __IOM uint32_t PIOSCRAMBLE : 1; /*!< [11..11] Enables data scrambling for PIO opertions. This should + only be used for data operations and never for commands + to a device. */ + __IM uint32_t : 4; + __IOM uint32_t XFERBYTES : 16; /*!< [31..16] Number of bytes to transmit or receive (based on TXRX + bit) */ + } CTRL_b; + } ; + + union { + __IOM uint32_t CFG; /*!< (@ 0x00000004) MSPI Transfer Configuration Register */ + + struct { + __IOM uint32_t DEVCFG : 4; /*!< [3..0] Flash configuration for XIP and AUTO DMA operations. + Controls value for SER (Slave Enable) for XIP operations + and address generation for DMA/XIP modes. Also used to + configure SPIFRF (frame format). */ + __IOM uint32_t ASIZE : 2; /*!< [5..4] Address Size. Address bytes to send from ADDR register + name = A1 value = 0x0 desc = Send one address byteenum + name = A2 value = 0x1 desc = Send two address bytesenum + name = A3 value = 0x2 desc = Send three address bytesenum + name = A4 value = 0x3 desc = Send four address bytes */ + __IOM uint32_t ISIZE : 1; /*!< [6..6] Instruction Sizeenum name = I8 value = 0x0 desc = Instruction + is 1 byteenum name = I16 value = 0x1 desc = Instruction + is 2 bytes */ + __IOM uint32_t SEPIO : 1; /*!< [7..7] Separate IO configuration. This bit should be set when + the target device has separate MOSI and MISO pins. Respective + IN/OUT bits below should be set to map pins. */ + __IOM uint32_t TURNAROUND : 6; /*!< [13..8] Number of turnaound cycles (for TX->RX transitions). + Qualified by ENTURN or XIPENTURN bit field. */ + __IM uint32_t : 2; + __IOM uint32_t CPHA : 1; /*!< [16..16] Serial clock phase. */ + __IOM uint32_t CPOL : 1; /*!< [17..17] Serial clock polarity. */ + } CFG_b; + } ; + + union { + __IOM uint32_t ADDR; /*!< (@ 0x00000008) MSPI Transfer Address Register */ + + struct { + __IOM uint32_t ADDR : 32; /*!< [31..0] Optional Address field to send (after optional instruction + field) - qualified by ASIZE in CMD register. NOTE: This + register is aliased to DMADEVADDR. */ + } ADDR_b; + } ; + + union { + __IOM uint32_t INSTR; /*!< (@ 0x0000000C) MSPI Transfer Instruction */ + + struct { + __IOM uint32_t INSTR : 16; /*!< [15..0] Optional Instruction field to send (1st byte) - qualified + by ISEND/ISIZE */ + } INSTR_b; + } ; + + union { + __IOM uint32_t TXFIFO; /*!< (@ 0x00000010) TX Data FIFO */ + + struct { + __IOM uint32_t TXFIFO : 32; /*!< [31..0] Data to be transmitted. Data should normall be aligned + to the LSB (pad the upper bits with zeros) unless BIGENDIAN + is set. */ + } TXFIFO_b; + } ; + + union { + __IOM uint32_t RXFIFO; /*!< (@ 0x00000014) RX Data FIFO */ + + struct { + __IOM uint32_t RXFIFO : 32; /*!< [31..0] Receive data. Data is aligned to the LSB (padded zeros + on upper bits) unless BIGENDIAN is set. */ + } RXFIFO_b; + } ; + + union { + __IOM uint32_t TXENTRIES; /*!< (@ 0x00000018) TX FIFO Entries */ + + struct { + __IOM uint32_t TXENTRIES : 5; /*!< [4..0] Number of 32-bit words/entries in TX FIFO */ + } TXENTRIES_b; + } ; + + union { + __IOM uint32_t RXENTRIES; /*!< (@ 0x0000001C) RX FIFO Entries */ + + struct { + __IOM uint32_t RXENTRIES : 5; /*!< [4..0] Number of 32-bit words/entries in RX FIFO */ + } RXENTRIES_b; + } ; + + union { + __IOM uint32_t THRESHOLD; /*!< (@ 0x00000020) TX/RX FIFO Threshhold Levels */ + + struct { + __IOM uint32_t TXTHRESH : 5; /*!< [4..0] Number of entries in TX FIFO that cause TXF interrupt */ + __IM uint32_t : 3; + __IOM uint32_t RXTHRESH : 5; /*!< [12..8] Number of entries in TX FIFO that cause RXE interrupt */ + } THRESHOLD_b; + } ; + __IM uint32_t RESERVED[55]; + + union { + __IOM uint32_t MSPICFG; /*!< (@ 0x00000100) MSPI Module Configuration */ + + struct { + __IOM uint32_t APBCLK : 1; /*!< [0..0] Enable continuous APB clock. For power-efficient operation, + APBCLK should be set to 0. */ + __IOM uint32_t RXCAP : 1; /*!< [1..1] Controls RX data capture phase. A setting of 0 (NORMAL) + captures read data at the normal capture point relative + to the internal clock launch point. However, to accomodate + chip/pad/board delays, a setting of RXCAP of 1 is expected + to be used to align the capture point with the return data + window. This bit is used in conjunction with RXNEG to provide + 4 unique capture points, all about 10ns apart. */ + __IOM uint32_t RXNEG : 1; /*!< [2..2] Adjusts the RX capture phase to the negedge of the 48MHz + internal clock (~10ns early). For normal operation, it + is expected that RXNEG will be set to 0. */ + __IOM uint32_t TXNEG : 1; /*!< [3..3] Launches TX data a half clock cycle (~10ns) early. This + should normally be programmed to zero (NORMAL). */ + __IOM uint32_t IOMSEL : 3; /*!< [6..4] Selects which IOM is selected for CQ handshake status. */ + __IM uint32_t : 1; + __IOM uint32_t CLKDIV : 6; /*!< [13..8] Clock Divider. Allows dividing 48 MHz base clock by + integer multiples. Enumerations are provided for common + frequency, but any integer divide from 48 MHz is allowed. + Odd divide ratios will result in a 33/66 percent duty cycle + with a long low clock pulse (to allow longer round-trip + for read data). */ + __IM uint32_t : 15; + __IOM uint32_t FIFORESET : 1; /*!< [29..29] Reset MSPI FIFO (active high). 1=reset FIFO, 0=normal + operation. May be used to manually flush the FIFO in error + handling. */ + __IOM uint32_t IPRSTN : 1; /*!< [30..30] IP block reset. Write to 0 to put the transfer module + in reset or 1 for normal operation. This may be required + after error conditions to clear the transfer on the bus. */ + __IOM uint32_t PRSTN : 1; /*!< [31..31] Peripheral reset. Master reset to the entire MSPI module + (DMA, XIP, and transfer state machines). 1=normal operation, + 0=in reset. */ + } MSPICFG_b; + } ; + + union { + __IOM uint32_t PADCFG; /*!< (@ 0x00000104) MSPI Output Pad Configuration */ + + struct { + __IOM uint32_t OUT3 : 1; /*!< [0..0] Output pad 3 configuration. 0=data[3] 1=CLK */ + __IOM uint32_t OUT4 : 1; /*!< [1..1] Output pad 4 configuration. 0=data[4] 1=data[0] */ + __IOM uint32_t OUT5 : 1; /*!< [2..2] Output pad 5 configuration. 0=data[5] 1=data[1] */ + __IOM uint32_t OUT6 : 1; /*!< [3..3] Output pad 6 configuration. 0=data[6] 1=data[2] */ + __IOM uint32_t OUT7 : 1; /*!< [4..4] Output pad 7 configuration. 0=data[7] 1=data[3] */ + __IM uint32_t : 11; + __IOM uint32_t IN0 : 2; /*!< [17..16] Data Input pad 0 pin muxing: 0=pad[0] 1=pad[4] 2=pad[1] + 3=pad[5] */ + __IOM uint32_t IN1 : 1; /*!< [18..18] Data Input pad 1 pin muxing: 0=pad[1] 1=pad[5] */ + __IOM uint32_t IN2 : 1; /*!< [19..19] Data Input pad 2 pin muxing: 0=pad[2] 1=pad[6] */ + __IOM uint32_t IN3 : 1; /*!< [20..20] Data Input pad 3 pin muxing: 0=pad[3] 1=pad[7] */ + __IOM uint32_t REVCS : 1; /*!< [21..21] Reverse CS connections. Allows CS1 to be associated + with lower data lanes and CS0 to be associated with upper + data lines */ + } PADCFG_b; + } ; + + union { + __IOM uint32_t PADOUTEN; /*!< (@ 0x00000108) MSPI Output Enable Pad Configuration */ + + struct { + __IOM uint32_t OUTEN : 9; /*!< [8..0] Output pad enable configuration. Indicates which pads + should be driven. Bits [3:0] are Quad0 data, [7:4] are + Quad1 data, and [8] is clock. */ + } PADOUTEN_b; + } ; + + union { + __IOM uint32_t FLASH; /*!< (@ 0x0000010C) Configuration for XIP/DMA support of SPI flash + modules. */ + + struct { + __IOM uint32_t XIPEN : 1; /*!< [0..0] Enable the XIP (eXecute In Place) function which effectively + enables the address decoding of the MSPI device in the + flash/cache address space at address 0x04000000-0x07FFFFFF. */ + __IM uint32_t : 1; + __IOM uint32_t XIPACK : 2; /*!< [3..2] Controls transmission of Micron XIP acknowledge cycles + (Micron Flash devices only) */ + __IOM uint32_t XIPBIGENDIAN : 1; /*!< [4..4] Indicates whether XIP/AUTO DMA data transfers are in + big or little endian format */ + __IOM uint32_t XIPENTURN : 1; /*!< [5..5] Indicates whether XIP/AUTO DMA operations should enable + TX->RX turnaround cycles */ + __IOM uint32_t XIPSENDA : 1; /*!< [6..6] Indicates whether XIP/AUTO DMA operations should send + an an address phase (see DMADEVADDR register and ASIZE + field in CFG) */ + __IOM uint32_t XIPSENDI : 1; /*!< [7..7] Indicates whether XIP/AUTO DMA operations should send + an instruction (see READINSTR field and ISIZE field in + CFG) */ + __IOM uint32_t XIPMIXED : 3; /*!< [10..8] Reserved. Set to 0x0 */ + __IM uint32_t : 5; + __IOM uint32_t WRITEINSTR : 8; /*!< [23..16] Write command sent for DMA operations */ + __IOM uint32_t READINSTR : 8; /*!< [31..24] Read command sent to flash for DMA/XIP operations */ + } FLASH_b; + } ; + __IM uint32_t RESERVED1[4]; + + union { + __IOM uint32_t SCRAMBLING; /*!< (@ 0x00000120) External Flash Scrambling Controls */ + + struct { + __IOM uint32_t SCRSTART : 10; /*!< [9..0] Scrambling region start address [25:16] (64K block granularity). + The START block is the FIRST block included in the scrambled + address range. */ + __IM uint32_t : 6; + __IOM uint32_t SCREND : 10; /*!< [25..16] Scrambling region end address [25:16] (64K block granularity). + The END block is the LAST block included in the scrambled + address range. */ + __IM uint32_t : 5; + __IOM uint32_t SCRENABLE : 1; /*!< [31..31] Enables Data Scrambling Region. When 1 reads and writes + to the range will be scrambled. When 0, data will be read/written + unmodified. Address range is specified in 64K granularity + and the START/END ranges are included within the range. */ + } SCRAMBLING_b; + } ; + __IM uint32_t RESERVED2[55]; + + union { + __IOM uint32_t INTEN; /*!< (@ 0x00000200) MSPI Master Interrupts: Enable */ + + struct { + __IOM uint32_t CMDCMP : 1; /*!< [0..0] Transfer complete. Note that DMA and CQ operations are + layered, so CMDCMP, DCMP, and CQ* can all be signalled + simultaneously */ + __IOM uint32_t TXE : 1; /*!< [1..1] Transmit FIFO empty. */ + __IOM uint32_t TXO : 1; /*!< [2..2] Transmit FIFO Overflow (only occurs when SW writes to + a full FIFO). */ + __IOM uint32_t RXU : 1; /*!< [3..3] Receive FIFO underflow (only occurs when SW reads from + an empty FIFO) */ + __IOM uint32_t RXO : 1; /*!< [4..4] Receive FIFO overflow (cannot happen in MSPI design -- + MSPI bus pins will stall) */ + __IOM uint32_t RXF : 1; /*!< [5..5] Receive FIFO full */ + __IOM uint32_t DCMP : 1; /*!< [6..6] DMA Complete Interrupt */ + __IOM uint32_t DERR : 1; /*!< [7..7] DMA Error Interrupt */ + __IOM uint32_t CQCMP : 1; /*!< [8..8] Command Queue Complete Interrupt */ + __IOM uint32_t CQUPD : 1; /*!< [9..9] Command Queue Update Interrupt. Issued whenever the CQ + performs an operation where address bit[0] is set. Useful + for triggering CURIDX interrupts. */ + __IOM uint32_t CQPAUSED : 1; /*!< [10..10] Command Queue is Paused. */ + __IOM uint32_t CQERR : 1; /*!< [11..11] Command Queue Error Interrupt */ + __IOM uint32_t SCRERR : 1; /*!< [12..12] Scrambling Alignment Error. Scrambling operations must + be aligned to word (4-byte) start address. */ + } INTEN_b; + } ; + + union { + __IOM uint32_t INTSTAT; /*!< (@ 0x00000204) MSPI Master Interrupts: Status */ + + struct { + __IOM uint32_t CMDCMP : 1; /*!< [0..0] Transfer complete. Note that DMA and CQ operations are + layered, so CMDCMP, DCMP, and CQ* can all be signalled + simultaneously */ + __IOM uint32_t TXE : 1; /*!< [1..1] Transmit FIFO empty. */ + __IOM uint32_t TXO : 1; /*!< [2..2] Transmit FIFO Overflow (only occurs when SW writes to + a full FIFO). */ + __IOM uint32_t RXU : 1; /*!< [3..3] Receive FIFO underflow (only occurs when SW reads from + an empty FIFO) */ + __IOM uint32_t RXO : 1; /*!< [4..4] Receive FIFO overflow (cannot happen in MSPI design -- + MSPI bus pins will stall) */ + __IOM uint32_t RXF : 1; /*!< [5..5] Receive FIFO full */ + __IOM uint32_t DCMP : 1; /*!< [6..6] DMA Complete Interrupt */ + __IOM uint32_t DERR : 1; /*!< [7..7] DMA Error Interrupt */ + __IOM uint32_t CQCMP : 1; /*!< [8..8] Command Queue Complete Interrupt */ + __IOM uint32_t CQUPD : 1; /*!< [9..9] Command Queue Update Interrupt. Issued whenever the CQ + performs an operation where address bit[0] is set. Useful + for triggering CURIDX interrupts. */ + __IOM uint32_t CQPAUSED : 1; /*!< [10..10] Command Queue is Paused. */ + __IOM uint32_t CQERR : 1; /*!< [11..11] Command Queue Error Interrupt */ + __IOM uint32_t SCRERR : 1; /*!< [12..12] Scrambling Alignment Error. Scrambling operations must + be aligned to word (4-byte) start address. */ + } INTSTAT_b; + } ; + + union { + __IOM uint32_t INTCLR; /*!< (@ 0x00000208) MSPI Master Interrupts: Clear */ + + struct { + __IOM uint32_t CMDCMP : 1; /*!< [0..0] Transfer complete. Note that DMA and CQ operations are + layered, so CMDCMP, DCMP, and CQ* can all be signalled + simultaneously */ + __IOM uint32_t TXE : 1; /*!< [1..1] Transmit FIFO empty. */ + __IOM uint32_t TXO : 1; /*!< [2..2] Transmit FIFO Overflow (only occurs when SW writes to + a full FIFO). */ + __IOM uint32_t RXU : 1; /*!< [3..3] Receive FIFO underflow (only occurs when SW reads from + an empty FIFO) */ + __IOM uint32_t RXO : 1; /*!< [4..4] Receive FIFO overflow (cannot happen in MSPI design -- + MSPI bus pins will stall) */ + __IOM uint32_t RXF : 1; /*!< [5..5] Receive FIFO full */ + __IOM uint32_t DCMP : 1; /*!< [6..6] DMA Complete Interrupt */ + __IOM uint32_t DERR : 1; /*!< [7..7] DMA Error Interrupt */ + __IOM uint32_t CQCMP : 1; /*!< [8..8] Command Queue Complete Interrupt */ + __IOM uint32_t CQUPD : 1; /*!< [9..9] Command Queue Update Interrupt. Issued whenever the CQ + performs an operation where address bit[0] is set. Useful + for triggering CURIDX interrupts. */ + __IOM uint32_t CQPAUSED : 1; /*!< [10..10] Command Queue is Paused. */ + __IOM uint32_t CQERR : 1; /*!< [11..11] Command Queue Error Interrupt */ + __IOM uint32_t SCRERR : 1; /*!< [12..12] Scrambling Alignment Error. Scrambling operations must + be aligned to word (4-byte) start address. */ + } INTCLR_b; + } ; + + union { + __IOM uint32_t INTSET; /*!< (@ 0x0000020C) MSPI Master Interrupts: Set */ + + struct { + __IOM uint32_t CMDCMP : 1; /*!< [0..0] Transfer complete. Note that DMA and CQ operations are + layered, so CMDCMP, DCMP, and CQ* can all be signalled + simultaneously */ + __IOM uint32_t TXE : 1; /*!< [1..1] Transmit FIFO empty. */ + __IOM uint32_t TXO : 1; /*!< [2..2] Transmit FIFO Overflow (only occurs when SW writes to + a full FIFO). */ + __IOM uint32_t RXU : 1; /*!< [3..3] Receive FIFO underflow (only occurs when SW reads from + an empty FIFO) */ + __IOM uint32_t RXO : 1; /*!< [4..4] Receive FIFO overflow (cannot happen in MSPI design -- + MSPI bus pins will stall) */ + __IOM uint32_t RXF : 1; /*!< [5..5] Receive FIFO full */ + __IOM uint32_t DCMP : 1; /*!< [6..6] DMA Complete Interrupt */ + __IOM uint32_t DERR : 1; /*!< [7..7] DMA Error Interrupt */ + __IOM uint32_t CQCMP : 1; /*!< [8..8] Command Queue Complete Interrupt */ + __IOM uint32_t CQUPD : 1; /*!< [9..9] Command Queue Update Interrupt. Issued whenever the CQ + performs an operation where address bit[0] is set. Useful + for triggering CURIDX interrupts. */ + __IOM uint32_t CQPAUSED : 1; /*!< [10..10] Command Queue is Paused. */ + __IOM uint32_t CQERR : 1; /*!< [11..11] Command Queue Error Interrupt */ + __IOM uint32_t SCRERR : 1; /*!< [12..12] Scrambling Alignment Error. Scrambling operations must + be aligned to word (4-byte) start address. */ + } INTSET_b; + } ; + __IM uint32_t RESERVED3[16]; + + union { + __IOM uint32_t DMACFG; /*!< (@ 0x00000250) DMA Configuration Register */ + + struct { + __IOM uint32_t DMAEN : 2; /*!< [1..0] DMA Enable. Setting this bit to EN will start the DMA + operation */ + __IOM uint32_t DMADIR : 1; /*!< [2..2] Direction */ + __IOM uint32_t DMAPRI : 2; /*!< [4..3] Sets the Priority of the DMA request */ + __IM uint32_t : 13; + __IOM uint32_t DMAPWROFF : 1; /*!< [18..18] Power off MSPI domain upon completion of DMA operation. */ + } DMACFG_b; + } ; + + union { + __IOM uint32_t DMASTAT; /*!< (@ 0x00000254) DMA Status Register */ + + struct { + __IOM uint32_t DMATIP : 1; /*!< [0..0] DMA Transfer In Progress indicator. 1 will indicate that + a DMA transfer is active. The DMA transfer may be waiting + on data, transferring data, or waiting for priority. All + of these will be indicated with a 1. A 0 will indicate + that the DMA is fully complete and no further transactions + will be done. */ + __IOM uint32_t DMACPL : 1; /*!< [1..1] DMA Transfer Complete. This signals the end of the DMA + operation. */ + __IOM uint32_t DMAERR : 1; /*!< [2..2] DMA Error. This active high bit signals that an error + was encountered during the DMA operation. */ + __IOM uint32_t SCRERR : 1; /*!< [3..3] Scrambling Access Alignment Error. This active high bit + signals that a scrambling operation was specified for a + non-word aligned DEVADDR. */ + } DMASTAT_b; + } ; + + union { + __IOM uint32_t DMATARGADDR; /*!< (@ 0x00000258) DMA Target Address Register */ + + struct { + __IOM uint32_t TARGADDR : 32; /*!< [31..0] Target byte address for source of DMA (either read or + write). In cases of non-word aligned addresses, the DMA + logic will take care for ensuring only the target bytes + are read/written. */ + } DMATARGADDR_b; + } ; + + union { + __IOM uint32_t DMADEVADDR; /*!< (@ 0x0000025C) DMA Device Address Register */ + + struct { + __IOM uint32_t DEVADDR : 32; /*!< [31..0] SPI Device address for automated DMA transactions (both + read and write). */ + } DMADEVADDR_b; + } ; + + union { + __IOM uint32_t DMATOTCOUNT; /*!< (@ 0x00000260) DMA Total Transfer Count */ + + struct { + __IOM uint32_t TOTCOUNT : 16; /*!< [15..0] Total Transfer Count in bytes. */ + } DMATOTCOUNT_b; + } ; + + union { + __IOM uint32_t DMABCOUNT; /*!< (@ 0x00000264) DMA BYTE Transfer Count */ + + struct { + __IOM uint32_t BCOUNT : 8; /*!< [7..0] Burst transfer size in bytes. This is the number of bytes + transferred when a FIFO trigger event occurs. Recommended + values are 16 or 32. */ + } DMABCOUNT_b; + } ; + __IM uint32_t RESERVED4[4]; + + union { + __IOM uint32_t DMATHRESH; /*!< (@ 0x00000278) DMA Transmit Trigger Threshhold */ + + struct { + __IOM uint32_t DMATHRESH : 4; /*!< [3..0] DMA transfer FIFO level trigger. For read operations, + DMA is triggered when the FIFO level is greater than this + value. For write operations, DMA is triggered when the + FIFO level is less than this level. Each DMA operation + will consist of BCOUNT bytes. */ + } DMATHRESH_b; + } ; + __IM uint32_t RESERVED5[9]; + + union { + __IOM uint32_t CQCFG; /*!< (@ 0x000002A0) Command Queue Configuration Register */ + + struct { + __IOM uint32_t CQEN : 1; /*!< [0..0] Command queue enable. When set, will enable the processing + of the command queue */ + __IOM uint32_t CQPRI : 1; /*!< [1..1] Sets the Priority of the command queue dma request */ + __IOM uint32_t CQPWROFF : 1; /*!< [2..2] Power off MSPI domain upon completion of DMA operation. */ + __IOM uint32_t CQAUTOCLEARMASK : 1; /*!< [3..3] Eanble clear of CQMASK after each pause operation. This + may be useful when using software flags to pause CQ. */ + } CQCFG_b; + } ; + __IM uint32_t RESERVED6; + + union { + __IOM uint32_t CQADDR; /*!< (@ 0x000002A8) CQ Target Read Address Register */ + + struct { + __IOM uint32_t CQADDR : 29; /*!< [28..0] Address of command queue buffer in SRAM or flash. The + buffer address must be aligned to a word boundary. */ + } CQADDR_b; + } ; + + union { + __IOM uint32_t CQSTAT; /*!< (@ 0x000002AC) Command Queue Status Register */ + + struct { + __IOM uint32_t CQTIP : 1; /*!< [0..0] Command queue Transfer In Progress indicator. 1 will + indicate that a CQ transfer is active and this will remain + active even when paused waiting for external event. */ + __IOM uint32_t CQCPL : 1; /*!< [1..1] Command queue operation Complete. This signals the end + of the command queue operation. */ + __IOM uint32_t CQERR : 1; /*!< [2..2] Command queue processing Error. This active high bit + signals that an error was encountered during the CQ operation. */ + __IOM uint32_t CQPAUSED : 1; /*!< [3..3] Command queue is currently paused status. */ + } CQSTAT_b; + } ; + + union { + __IOM uint32_t CQFLAGS; /*!< (@ 0x000002B0) Command Queue Flag Register */ + + struct { + __IOM uint32_t CQFLAGS : 16; /*!< [15..0] Current flag status (read-only). Bits [7:0] are software + controllable and bits [15:8] are hardware status. */ + } CQFLAGS_b; + } ; + + union { + __IOM uint32_t CQSETCLEAR; /*!< (@ 0x000002B4) Command Queue Flag Set/Clear Register */ + + struct { + __IOM uint32_t CQFSET : 8; /*!< [7..0] Set CQFlag status bits. Set has priority over clear if + both are high. */ + __IOM uint32_t CQFTOGGLE : 8; /*!< [15..8] Toggle CQFlag status bits */ + __IOM uint32_t CQFCLR : 8; /*!< [23..16] Clear CQFlag status bits. */ + } CQSETCLEAR_b; + } ; + + union { + __IOM uint32_t CQPAUSE; /*!< (@ 0x000002B8) Command Queue Pause Mask Register */ + + struct { + __IOM uint32_t CQMASK : 16; /*!< [15..0] CQ will pause processing until all specified events + are satisfied. */ + } CQPAUSE_b; + } ; + __IM uint32_t RESERVED7; + + union { + __IOM uint32_t CQCURIDX; /*!< (@ 0x000002C0) Command Queue Current Index */ + + struct { + __IOM uint32_t CQCURIDX : 8; /*!< [7..0] Can be used to indicate the current position of the command + queue by having CQ operations write this field. A CQ hardware + status flag indicates when CURIDX and ENDIDX are not equal, + allowing SW to pause the CQ processing until the end index + is updated. */ + } CQCURIDX_b; + } ; + + union { + __IOM uint32_t CQENDIDX; /*!< (@ 0x000002C4) Command Queue End Index */ + + struct { + __IOM uint32_t CQENDIDX : 8; /*!< [7..0] Can be used to indicate the end position of the command + queue. A CQ hardware status bit indices when CURIDX != + ENDIDX so that the CQ can be paused when it reaches the + end pointer. */ + } CQENDIDX_b; + } ; +} MSPI_Type; /*!< Size = 712 (0x2c8) */ + + + +/* =========================================================================================================================== */ +/* ================ PDM ================ */ +/* =========================================================================================================================== */ + + +/** + * @brief PDM Audio (PDM) + */ + +typedef struct { /*!< (@ 0x50011000) PDM Structure */ + + union { + __IOM uint32_t PCFG; /*!< (@ 0x00000000) PDM Configuration Register */ + + struct { + __IOM uint32_t PDMCOREEN : 1; /*!< [0..0] Data Streaming Control. */ + __IOM uint32_t SOFTMUTE : 1; /*!< [1..1] Soft mute control. */ + __IOM uint32_t CYCLES : 3; /*!< [4..2] Number of clocks during gain-setting changes. */ + __IOM uint32_t HPCUTOFF : 4; /*!< [8..5] High pass filter coefficients. */ + __IOM uint32_t ADCHPD : 1; /*!< [9..9] High pass filter control. */ + __IOM uint32_t SINCRATE : 7; /*!< [16..10] SINC decimation rate. */ + __IOM uint32_t MCLKDIV : 2; /*!< [18..17] PDM_CLK frequency divisor. */ + __IM uint32_t : 2; + __IOM uint32_t PGALEFT : 5; /*!< [25..21] Left channel PGA gain. */ + __IOM uint32_t PGARIGHT : 5; /*!< [30..26] Right channel PGA gain. */ + __IOM uint32_t LRSWAP : 1; /*!< [31..31] Left/right channel swap. */ + } PCFG_b; + } ; + + union { + __IOM uint32_t VCFG; /*!< (@ 0x00000004) Voice Configuration Register */ + + struct { + __IM uint32_t : 3; + __IOM uint32_t CHSET : 2; /*!< [4..3] Set PCM channels. */ + __IM uint32_t : 3; + __IOM uint32_t PCMPACK : 1; /*!< [8..8] PCM data packing enable. */ + __IM uint32_t : 7; + __IOM uint32_t SELAP : 1; /*!< [16..16] Select PDM input clock source. */ + __IOM uint32_t DMICKDEL : 1; /*!< [17..17] PDM clock sampling delay. */ + __IM uint32_t : 1; + __IOM uint32_t BCLKINV : 1; /*!< [19..19] I2S BCLK input inversion. */ + __IOM uint32_t I2SEN : 1; /*!< [20..20] I2S interface enable. */ + __IM uint32_t : 5; + __IOM uint32_t PDMCLKEN : 1; /*!< [26..26] Enable the serial clock. */ + __IOM uint32_t PDMCLKSEL : 3; /*!< [29..27] Select the PDM input clock. */ + __IOM uint32_t RSTB : 1; /*!< [30..30] Reset the IP core. */ + __IOM uint32_t IOCLKEN : 1; /*!< [31..31] Enable the IO clock. */ + } VCFG_b; + } ; + + union { + __IOM uint32_t VOICESTAT; /*!< (@ 0x00000008) Voice Status Register */ + + struct { + __IOM uint32_t FIFOCNT : 6; /*!< [5..0] Valid 32-bit entries currently in the FIFO. */ + } VOICESTAT_b; + } ; + + union { + __IOM uint32_t FIFOREAD; /*!< (@ 0x0000000C) FIFO Read */ + + struct { + __IOM uint32_t FIFOREAD : 32; /*!< [31..0] FIFO read data. */ + } FIFOREAD_b; + } ; + + union { + __IOM uint32_t FIFOFLUSH; /*!< (@ 0x00000010) FIFO Flush */ + + struct { + __IOM uint32_t FIFOFLUSH : 1; /*!< [0..0] FIFO FLUSH. */ + } FIFOFLUSH_b; + } ; + + union { + __IOM uint32_t FIFOTHR; /*!< (@ 0x00000014) FIFO Threshold */ + + struct { + __IOM uint32_t FIFOTHR : 5; /*!< [4..0] FIFO Threshold value. When the FIFO count is equal to, + or larger than this value (in words), a THR interrupt is + generated (if enabled) */ + } FIFOTHR_b; + } ; + __IM uint32_t RESERVED[122]; + + union { + __IOM uint32_t INTEN; /*!< (@ 0x00000200) IO Master Interrupts: Enable */ + + struct { + __IOM uint32_t THR : 1; /*!< [0..0] This is the FIFO threshold interrupt. */ + __IOM uint32_t OVF : 1; /*!< [1..1] This is the FIFO overflow interrupt. */ + __IOM uint32_t UNDFL : 1; /*!< [2..2] This is the FIFO underflow interrupt. */ + __IOM uint32_t DCMP : 1; /*!< [3..3] DMA completed a transfer */ + __IOM uint32_t DERR : 1; /*!< [4..4] DMA Error receieved */ + } INTEN_b; + } ; + + union { + __IOM uint32_t INTSTAT; /*!< (@ 0x00000204) IO Master Interrupts: Status */ + + struct { + __IOM uint32_t THR : 1; /*!< [0..0] This is the FIFO threshold interrupt. */ + __IOM uint32_t OVF : 1; /*!< [1..1] This is the FIFO overflow interrupt. */ + __IOM uint32_t UNDFL : 1; /*!< [2..2] This is the FIFO underflow interrupt. */ + __IOM uint32_t DCMP : 1; /*!< [3..3] DMA completed a transfer */ + __IOM uint32_t DERR : 1; /*!< [4..4] DMA Error receieved */ + } INTSTAT_b; + } ; + + union { + __IOM uint32_t INTCLR; /*!< (@ 0x00000208) IO Master Interrupts: Clear */ + + struct { + __IOM uint32_t THR : 1; /*!< [0..0] This is the FIFO threshold interrupt. */ + __IOM uint32_t OVF : 1; /*!< [1..1] This is the FIFO overflow interrupt. */ + __IOM uint32_t UNDFL : 1; /*!< [2..2] This is the FIFO underflow interrupt. */ + __IOM uint32_t DCMP : 1; /*!< [3..3] DMA completed a transfer */ + __IOM uint32_t DERR : 1; /*!< [4..4] DMA Error receieved */ + } INTCLR_b; + } ; + + union { + __IOM uint32_t INTSET; /*!< (@ 0x0000020C) IO Master Interrupts: Set */ + + struct { + __IOM uint32_t THR : 1; /*!< [0..0] This is the FIFO threshold interrupt. */ + __IOM uint32_t OVF : 1; /*!< [1..1] This is the FIFO overflow interrupt. */ + __IOM uint32_t UNDFL : 1; /*!< [2..2] This is the FIFO underflow interrupt. */ + __IOM uint32_t DCMP : 1; /*!< [3..3] DMA completed a transfer */ + __IOM uint32_t DERR : 1; /*!< [4..4] DMA Error receieved */ + } INTSET_b; + } ; + __IM uint32_t RESERVED1[12]; + + union { + __IOM uint32_t DMATRIGEN; /*!< (@ 0x00000240) DMA Trigger Enable Register */ + + struct { + __IOM uint32_t DTHR : 1; /*!< [0..0] Trigger DMA upon when FIFO iss filled to level indicated + by the FIFO THRESHOLD,at granularity of 16 bytes only */ + __IOM uint32_t DTHR90 : 1; /*!< [1..1] Trigger DMA at FIFO 90 percent full. This signal is also + used internally for AUTOHIP function */ + } DMATRIGEN_b; + } ; + + union { + __IOM uint32_t DMATRIGSTAT; /*!< (@ 0x00000244) DMA Trigger Status Register */ + + struct { + __IOM uint32_t DTHRSTAT : 1; /*!< [0..0] Triggered DMA from FIFO reaching threshold */ + __IOM uint32_t DTHR90STAT : 1; /*!< [1..1] Triggered DMA from FIFO reaching 90 percent full */ + } DMATRIGSTAT_b; + } ; + __IM uint32_t RESERVED2[14]; + + union { + __IOM uint32_t DMACFG; /*!< (@ 0x00000280) DMA Configuration Register */ + + struct { + __IOM uint32_t DMAEN : 1; /*!< [0..0] DMA Enable */ + __IM uint32_t : 1; + __IOM uint32_t DMADIR : 1; /*!< [2..2] Direction */ + __IM uint32_t : 5; + __IOM uint32_t DMAPRI : 1; /*!< [8..8] Sets the Priority of the DMA request */ + __IOM uint32_t DAUTOHIP : 1; /*!< [9..9] Raise priority to high on fifo full, and DMAPRI set to + low */ + __IOM uint32_t DPWROFF : 1; /*!< [10..10] Power Off the ADC System upon DMACPL. */ + } DMACFG_b; + } ; + __IM uint32_t RESERVED3; + + union { + __IOM uint32_t DMATOTCOUNT; /*!< (@ 0x00000288) DMA Total Transfer Count */ + + struct { + __IOM uint32_t TOTCOUNT : 20; /*!< [19..0] Total Transfer Count. The transfer count must be a multiple + of the THR setting to avoid DMA overruns. */ + } DMATOTCOUNT_b; + } ; + + union { + __IOM uint32_t DMATARGADDR; /*!< (@ 0x0000028C) DMA Target Address Register */ + + struct { + __IOM uint32_t LTARGADDR : 20; /*!< [19..0] DMA Target Address. This register is not updated with + the current address of the DMA, but will remain static + with the original address during the DMA transfer. */ + __IOM uint32_t UTARGADDR : 12; /*!< [31..20] SRAM Target */ + } DMATARGADDR_b; + } ; + + union { + __IOM uint32_t DMASTAT; /*!< (@ 0x00000290) DMA Status Register */ + + struct { + __IOM uint32_t DMATIP : 1; /*!< [0..0] DMA Transfer In Progress */ + __IOM uint32_t DMACPL : 1; /*!< [1..1] DMA Transfer Complete */ + __IOM uint32_t DMAERR : 1; /*!< [2..2] DMA Error */ + } DMASTAT_b; + } ; +} PDM_Type; /*!< Size = 660 (0x294) */ + + + +/* =========================================================================================================================== */ +/* ================ PWRCTRL ================ */ +/* =========================================================================================================================== */ + + +/** + * @brief PWR Controller Register Bank (PWRCTRL) + */ + +typedef struct { /*!< (@ 0x40021000) PWRCTRL Structure */ + + union { + __IOM uint32_t SUPPLYSRC; /*!< (@ 0x00000000) Voltage Regulator Select Register */ + + struct { + __IOM uint32_t BLEBUCKEN : 1; /*!< [0..0] Enables and Selects the BLE Buck as the supply for the + BLE power domain or for Burst LDO. It takes the initial + value from Customer INFO space. Buck will be powered up + only if there is an active request for BLEH domain or Burst + mode and appropriate feature is allowed. */ + } SUPPLYSRC_b; + } ; + + union { + __IOM uint32_t SUPPLYSTATUS; /*!< (@ 0x00000004) Voltage Regulators status */ + + struct { + __IOM uint32_t SIMOBUCKON : 1; /*!< [0..0] Indicates whether the Core/Mem low-voltage domains are + supplied from the LDO or the Buck. */ + __IOM uint32_t BLEBUCKON : 1; /*!< [1..1] Indicates whether the BLE (if supported) domain and burst + (if supported) domain is supplied from the LDO or the Buck. + Buck will be powered up only if there is an active request + for BLEH domain or Burst mode and appropriate reature is + allowed. */ + } SUPPLYSTATUS_b; + } ; + + union { + __IOM uint32_t DEVPWREN; /*!< (@ 0x00000008) Device Power Enables */ + + struct { + __IOM uint32_t PWRIOS : 1; /*!< [0..0] Power up IO Slave */ + __IOM uint32_t PWRIOM0 : 1; /*!< [1..1] Power up IO Master 0 */ + __IOM uint32_t PWRIOM1 : 1; /*!< [2..2] Power up IO Master 1 */ + __IOM uint32_t PWRIOM2 : 1; /*!< [3..3] Power up IO Master 2 */ + __IOM uint32_t PWRIOM3 : 1; /*!< [4..4] Power up IO Master 3 */ + __IOM uint32_t PWRIOM4 : 1; /*!< [5..5] Power up IO Master 4 */ + __IOM uint32_t PWRIOM5 : 1; /*!< [6..6] Power up IO Master 5 */ + __IOM uint32_t PWRUART0 : 1; /*!< [7..7] Power up UART Controller 0 */ + __IOM uint32_t PWRUART1 : 1; /*!< [8..8] Power up UART Controller 1 */ + __IOM uint32_t PWRADC : 1; /*!< [9..9] Power up ADC Digital Controller */ + __IOM uint32_t PWRSCARD : 1; /*!< [10..10] Power up SCARD Controller */ + __IOM uint32_t PWRMSPI : 1; /*!< [11..11] Power up MSPI Controller */ + __IOM uint32_t PWRPDM : 1; /*!< [12..12] Power up PDM block */ + __IOM uint32_t PWRBLEL : 1; /*!< [13..13] Power up BLE controller */ + } DEVPWREN_b; + } ; + + union { + __IOM uint32_t MEMPWDINSLEEP; /*!< (@ 0x0000000C) Powerdown SRAM banks in Deep Sleep mode */ + + struct { + __IOM uint32_t DTCMPWDSLP : 3; /*!< [2..0] power down DTCM in deep sleep */ + __IOM uint32_t SRAMPWDSLP : 10; /*!< [12..3] Selects which SRAM banks are powered down in deep sleep + mode, causing the contents of the bank to be lost. */ + __IOM uint32_t FLASH0PWDSLP : 1; /*!< [13..13] Powerdown flash0 in deep sleep */ + __IOM uint32_t FLASH1PWDSLP : 1; /*!< [14..14] Powerdown flash1 in deep sleep */ + __IM uint32_t : 16; + __IOM uint32_t CACHEPWDSLP : 1; /*!< [31..31] power down cache in deep sleep */ + } MEMPWDINSLEEP_b; + } ; + + union { + __IOM uint32_t MEMPWREN; /*!< (@ 0x00000010) Enables individual banks of the MEMORY array */ + + struct { + __IOM uint32_t DTCM : 3; /*!< [2..0] Power up DTCM */ + __IOM uint32_t SRAM : 10; /*!< [12..3] Power up SRAM groups */ + __IOM uint32_t FLASH0 : 1; /*!< [13..13] Power up Flash0 */ + __IOM uint32_t FLASH1 : 1; /*!< [14..14] Power up Flash1 */ + __IM uint32_t : 15; + __IOM uint32_t CACHEB0 : 1; /*!< [30..30] Power up Cache Bank 0. This works in conjunction with + Cache enable from flash_cache module. To power up cache + bank0, cache has to be enabled and this bit has to be set. */ + __IOM uint32_t CACHEB2 : 1; /*!< [31..31] Power up Cache Bank 2. This works in conjunction with + Cache enable from flash_cache module. To power up cache + bank2, cache has to be enabled and this bit has to be set. */ + } MEMPWREN_b; + } ; + + union { + __IOM uint32_t MEMPWRSTATUS; /*!< (@ 0x00000014) Mem Power ON Status */ + + struct { + __IOM uint32_t DTCM00 : 1; /*!< [0..0] This bit is 1 if power is supplied to DTCM GROUP0_0 */ + __IOM uint32_t DTCM01 : 1; /*!< [1..1] This bit is 1 if power is supplied to DTCM GROUP0_1 */ + __IOM uint32_t DTCM1 : 1; /*!< [2..2] This bit is 1 if power is supplied to DTCM GROUP1 */ + __IOM uint32_t SRAM0 : 1; /*!< [3..3] This bit is 1 if power is supplied to SRAM GROUP0 */ + __IOM uint32_t SRAM1 : 1; /*!< [4..4] This bit is 1 if power is supplied to SRAM GROUP1 */ + __IOM uint32_t SRAM2 : 1; /*!< [5..5] This bit is 1 if power is supplied to SRAM GROUP2 */ + __IOM uint32_t SRAM3 : 1; /*!< [6..6] This bit is 1 if power is supplied to SRAM GROUP3 */ + __IOM uint32_t SRAM4 : 1; /*!< [7..7] This bit is 1 if power is supplied to SRAM GROUP4 */ + __IOM uint32_t SRAM5 : 1; /*!< [8..8] This bit is 1 if power is supplied to SRAM GROUP5 */ + __IOM uint32_t SRAM6 : 1; /*!< [9..9] This bit is 1 if power is supplied to SRAM GROUP6 */ + __IOM uint32_t SRAM7 : 1; /*!< [10..10] This bit is 1 if power is supplied to SRAM GROUP7 */ + __IOM uint32_t SRAM8 : 1; /*!< [11..11] This bit is 1 if power is supplied to SRAM GROUP8 */ + __IOM uint32_t SRAM9 : 1; /*!< [12..12] This bit is 1 if power is supplied to SRAM GROUP9 */ + __IOM uint32_t FLASH0 : 1; /*!< [13..13] This bit is 1 if power is supplied to FLASH 0 */ + __IOM uint32_t FLASH1 : 1; /*!< [14..14] This bit is 1 if power is supplied to FLASH 1 */ + __IOM uint32_t CACHEB0 : 1; /*!< [15..15] This bit is 1 if power is supplied to Cache Bank 0 */ + __IOM uint32_t CACHEB2 : 1; /*!< [16..16] This bit is 1 if power is supplied to Cache Bank 2 */ + } MEMPWRSTATUS_b; + } ; + + union { + __IOM uint32_t DEVPWRSTATUS; /*!< (@ 0x00000018) Device Power ON Status */ + + struct { + __IOM uint32_t MCUL : 1; /*!< [0..0] This bit is 1 if power is supplied to MCUL */ + __IOM uint32_t MCUH : 1; /*!< [1..1] This bit is 1 if power is supplied to MCUH */ + __IOM uint32_t HCPA : 1; /*!< [2..2] This bit is 1 if power is supplied to HCPA domain (IO + SLAVE, UART0, UART1, SCARD) */ + __IOM uint32_t HCPB : 1; /*!< [3..3] This bit is 1 if power is supplied to HCPB domain (IO + MASTER 0, 1, 2) */ + __IOM uint32_t HCPC : 1; /*!< [4..4] This bit is 1 if power is supplied to HCPC domain (IO + MASTER4, 5, 6) */ + __IOM uint32_t PWRADC : 1; /*!< [5..5] This bit is 1 if power is supplied to ADC */ + __IOM uint32_t PWRMSPI : 1; /*!< [6..6] This bit is 1 if power is supplied to MSPI */ + __IOM uint32_t PWRPDM : 1; /*!< [7..7] This bit is 1 if power is supplied to PDM */ + __IOM uint32_t BLEL : 1; /*!< [8..8] This bit is 1 if power is supplied to BLEL */ + __IOM uint32_t BLEH : 1; /*!< [9..9] This bit is 1 if power is supplied to BLEH */ + __IM uint32_t : 19; + __IOM uint32_t CORESLEEP : 1; /*!< [29..29] This bit is 1 if CORE has been in SLEEP State. Write + '1' to this bit to clear it. */ + __IOM uint32_t COREDEEPSLEEP : 1; /*!< [30..30] This bit is 1 if CORE has been in Deep Sleep. Write + '1' to this bit to clear it. */ + __IOM uint32_t SYSDEEPSLEEP : 1; /*!< [31..31] This bit is 1 if SYSTEM has been in Deep Sleep. Write + '1' to this bit to clear it. */ + } DEVPWRSTATUS_b; + } ; + + union { + __IOM uint32_t SRAMCTRL; /*!< (@ 0x0000001C) SRAM Control register */ + + struct { + __IM uint32_t : 1; + __IOM uint32_t SRAMCLKGATE : 1; /*!< [1..1] This bit is 1 if clock gating is allowed for individual + system SRAMs */ + __IOM uint32_t SRAMMASTERCLKGATE : 1; /*!< [2..2] This bit is 1 when the master clock gate is enabled (top-level + clock gate for entire SRAM block) */ + __IM uint32_t : 5; + __IOM uint32_t SRAMLIGHTSLEEP : 12; /*!< [19..8] Light Sleep enable for each TCM/SRAM bank. When 1, corresponding + bank will be put into light sleep. For optimal power, banks + should be put into light sleep while the system is active + but the bank has minimal or no accesses. */ + } SRAMCTRL_b; + } ; + + union { + __IOM uint32_t ADCSTATUS; /*!< (@ 0x00000020) Power Status Register for ADC Block */ + + struct { + __IOM uint32_t ADCPWD : 1; /*!< [0..0] This bit indicates that the ADC is powered down */ + __IOM uint32_t BGTPWD : 1; /*!< [1..1] This bit indicates that the ADC Band Gap is powered down */ + __IOM uint32_t VPTATPWD : 1; /*!< [2..2] This bit indicates that the ADC temperature sensor input + buffer is powered down */ + __IOM uint32_t VBATPWD : 1; /*!< [3..3] This bit indicates that the ADC VBAT resistor divider + is powered down */ + __IOM uint32_t REFKEEPPWD : 1; /*!< [4..4] This bit indicates that the ADC REFKEEP is powered down */ + __IOM uint32_t REFBUFPWD : 1; /*!< [5..5] This bit indicates that the ADC REFBUF is powered down */ + } ADCSTATUS_b; + } ; + + union { + __IOM uint32_t MISC; /*!< (@ 0x00000024) Power Optimization Control Bits */ + + struct { + __IOM uint32_t SIMOBUCKEN : 1; /*!< [0..0] Enables and Selects the SIMO Buck as the supply for the + low-voltage power domain. It takes the initial value from + the bit set in Customer INFO space. */ + __IOM uint32_t FORCECOREVRLPPDM : 1; /*!< [1..1] Control bit to enable the core VR to go into LP mode + with HCPA/B/C/MSPI are powered off but PDM is powered on */ + __IOM uint32_t FORCECOREVRLPTIMERS : 1; /*!< [2..2] Control Bit to force Core VR to LP mode in deep sleep + even when hfrc based ctimer or stimer is running. */ + __IOM uint32_t FORCEMEMVRLPTIMERS : 1; /*!< [3..3] Control Bit to force Mem VR to LP mode in deep sleep + even when hfrc based ctimer or stimer is running. */ + __IOM uint32_t FORCEMEMVRADC : 2; /*!< [5..4] Control Bit to force mem VR to LP or ACT mode in deep + sleep when ADC is powered ON. 0x3 results in picking LP + mode. */ + __IOM uint32_t MEMVRLPBLE : 1; /*!< [6..6] Control Bit to let Mem VR go to lp mode in deep sleep + even when BLEL or BLEH is powered on given none of the + other domains require it. */ + __IOM uint32_t FORCEBLEBUCKACT : 1; /*!< [7..7] Control Bit to enable BLE Buck to be in active state + when BLE Buck is enabled. Default behavior is to be in + active only when Burst or BLEH power on are requested. */ + } MISC_b; + } ; + + union { + __IOM uint32_t DEVPWREVENTEN; /*!< (@ 0x00000028) Event enable register to control which DEVPWRSTATUS + bits are routed to event input of CPU. */ + + struct { + __IOM uint32_t MCULEVEN : 1; /*!< [0..0] Control MCUL power-on status event */ + __IOM uint32_t MCUHEVEN : 1; /*!< [1..1] Control MCUH power-on status event */ + __IOM uint32_t HCPAEVEN : 1; /*!< [2..2] Control HCPA power-on status event */ + __IOM uint32_t HCPBEVEN : 1; /*!< [3..3] Control HCPB power-on status event */ + __IOM uint32_t HCPCEVEN : 1; /*!< [4..4] Control HCPC power-on status event */ + __IOM uint32_t ADCEVEN : 1; /*!< [5..5] Control ADC power-on status event */ + __IOM uint32_t MSPIEVEN : 1; /*!< [6..6] Control MSPI power-on status event */ + __IOM uint32_t PDMEVEN : 1; /*!< [7..7] Control PDM power-on status event */ + __IOM uint32_t BLELEVEN : 1; /*!< [8..8] Control BLE power-on status event */ + __IM uint32_t : 20; + __IOM uint32_t BLEFEATUREEVEN : 1; /*!< [29..29] Control BLEFEATURE status event */ + __IOM uint32_t BURSTFEATUREEVEN : 1; /*!< [30..30] Control BURSTFEATURE status event */ + __IOM uint32_t BURSTEVEN : 1; /*!< [31..31] Control BURST status event */ + } DEVPWREVENTEN_b; + } ; + + union { + __IOM uint32_t MEMPWREVENTEN; /*!< (@ 0x0000002C) Event enable register to control which MEMPWRSTATUS + bits are routed to event input of CPU. */ + + struct { + __IOM uint32_t DTCMEN : 3; /*!< [2..0] Enable DTCM power-on status event */ + __IOM uint32_t SRAMEN : 10; /*!< [12..3] Control SRAM power-on status event */ + __IOM uint32_t FLASH0EN : 1; /*!< [13..13] Control Flash power-on status event */ + __IOM uint32_t FLASH1EN : 1; /*!< [14..14] Control Flash power-on status event */ + __IM uint32_t : 15; + __IOM uint32_t CACHEB0EN : 1; /*!< [30..30] Control CACHE BANK 0 power-on status event */ + __IOM uint32_t CACHEB2EN : 1; /*!< [31..31] Control CACHEB2 power-on status event */ + } MEMPWREVENTEN_b; + } ; +} PWRCTRL_Type; /*!< Size = 48 (0x30) */ + + + +/* =========================================================================================================================== */ +/* ================ RSTGEN ================ */ +/* =========================================================================================================================== */ + + +/** + * @brief MCU Reset Generator (RSTGEN) + */ + +typedef struct { /*!< (@ 0x40000000) RSTGEN Structure */ + + union { + __IOM uint32_t CFG; /*!< (@ 0x00000000) Configuration Register */ + + struct { + __IOM uint32_t BODHREN : 1; /*!< [0..0] Brown out high (2.1v) reset enable. */ + __IOM uint32_t WDREN : 1; /*!< [1..1] Watchdog Timer Reset Enable. NOTE: The WDT module must + also be configured for WDT reset. This includes enabling + the RESEN bit in WDTCFG register in Watch dog timer block. */ + } CFG_b; + } ; + + union { + __IOM uint32_t SWPOI; /*!< (@ 0x00000004) Software POI Reset */ + + struct { + __IOM uint32_t SWPOIKEY : 8; /*!< [7..0] 0x1B generates a software POI reset. This is a write-only + register. Reading from this register will yield only all + 0s. */ + } SWPOI_b; + } ; + + union { + __IOM uint32_t SWPOR; /*!< (@ 0x00000008) Software POR Reset */ + + struct { + __IOM uint32_t SWPORKEY : 8; /*!< [7..0] 0xD4 generates a software POR reset. */ + } SWPOR_b; + } ; + __IM uint32_t RESERVED[2]; + + union { + __IOM uint32_t TPIURST; /*!< (@ 0x00000014) TPIU reset */ + + struct { + __IOM uint32_t TPIURST : 1; /*!< [0..0] Static reset for the TPIU. Write to '1' to assert reset + to TPIU. Write to '0' to clear the reset. */ + } TPIURST_b; + } ; + __IM uint32_t RESERVED1[122]; + + union { + __IOM uint32_t INTEN; /*!< (@ 0x00000200) Reset Interrupt register: Enable */ + + struct { + __IOM uint32_t BODH : 1; /*!< [0..0] Enables an interrupt that triggers when VCC is below + BODH level. */ + } INTEN_b; + } ; + + union { + __IOM uint32_t INTSTAT; /*!< (@ 0x00000204) Reset Interrupt register: Status */ + + struct { + __IOM uint32_t BODH : 1; /*!< [0..0] Enables an interrupt that triggers when VCC is below + BODH level. */ + } INTSTAT_b; + } ; + + union { + __IOM uint32_t INTCLR; /*!< (@ 0x00000208) Reset Interrupt register: Clear */ + + struct { + __IOM uint32_t BODH : 1; /*!< [0..0] Enables an interrupt that triggers when VCC is below + BODH level. */ + } INTCLR_b; + } ; + + union { + __IOM uint32_t INTSET; /*!< (@ 0x0000020C) Reset Interrupt register: Set */ + + struct { + __IOM uint32_t BODH : 1; /*!< [0..0] Enables an interrupt that triggers when VCC is below + BODH level. */ + } INTSET_b; + } ; + __IM uint32_t RESERVED2[67107708]; + + union { + __IOM uint32_t STAT; /*!< (@ 0x0FFFF000) Status Register (SBL) */ + + struct { + __IOM uint32_t EXRSTAT : 1; /*!< [0..0] Reset was initiated by an External Reset (SBL). */ + __IOM uint32_t PORSTAT : 1; /*!< [1..1] Reset was initiated by a Power-On Reset (SBL). */ + __IOM uint32_t BORSTAT : 1; /*!< [2..2] Reset was initiated by a Brown-Out Reset (SBL). */ + __IOM uint32_t SWRSTAT : 1; /*!< [3..3] Reset was a initiated by SW POR or AIRCR Reset (SBL). */ + __IOM uint32_t POIRSTAT : 1; /*!< [4..4] Reset was a initiated by Software POI Reset (SBL). */ + __IOM uint32_t DBGRSTAT : 1; /*!< [5..5] Reset was a initiated by Debugger Reset (SBL). */ + __IOM uint32_t WDRSTAT : 1; /*!< [6..6] Reset was initiated by a Watchdog Timer Reset (SBL). */ + __IOM uint32_t BOUSTAT : 1; /*!< [7..7] An Unregulated Supply Brownout Event occurred (SBL). */ + __IOM uint32_t BOCSTAT : 1; /*!< [8..8] A Core Regulator Brownout Event occurred (SBL). */ + __IOM uint32_t BOFSTAT : 1; /*!< [9..9] A Memory Regulator Brownout Event occurred (SBL). */ + __IOM uint32_t BOBSTAT : 1; /*!< [10..10] A BLE/Burst Regulator Brownout Event occurred (SBL). */ + __IM uint32_t : 19; + __IOM uint32_t FBOOT : 1; /*!< [30..30] Set if current boot was initiated by soft reset and + resulted in Fast Boot (SBL). */ + __IOM uint32_t SBOOT : 1; /*!< [31..31] Set when booting securely (SBL). */ + } STAT_b; + } ; +} RSTGEN_Type; /*!< Size = 268431364 (0xffff004) */ + + + +/* =========================================================================================================================== */ +/* ================ RTC ================ */ +/* =========================================================================================================================== */ + + +/** + * @brief Real Time Clock (RTC) + */ + +typedef struct { /*!< (@ 0x40004200) RTC Structure */ + __IM uint32_t RESERVED[16]; + + union { + __IOM uint32_t CTRLOW; /*!< (@ 0x00000040) RTC Counters Lower */ + + struct { + __IOM uint32_t CTR100 : 8; /*!< [7..0] 100ths of a second Counter */ + __IOM uint32_t CTRSEC : 7; /*!< [14..8] Seconds Counter */ + __IM uint32_t : 1; + __IOM uint32_t CTRMIN : 7; /*!< [22..16] Minutes Counter */ + __IM uint32_t : 1; + __IOM uint32_t CTRHR : 6; /*!< [29..24] Hours Counter */ + } CTRLOW_b; + } ; + + union { + __IOM uint32_t CTRUP; /*!< (@ 0x00000044) RTC Counters Upper */ + + struct { + __IOM uint32_t CTRDATE : 6; /*!< [5..0] Date Counter */ + __IM uint32_t : 2; + __IOM uint32_t CTRMO : 5; /*!< [12..8] Months Counter */ + __IM uint32_t : 3; + __IOM uint32_t CTRYR : 8; /*!< [23..16] Years Counter */ + __IOM uint32_t CTRWKDY : 3; /*!< [26..24] Weekdays Counter */ + __IOM uint32_t CB : 1; /*!< [27..27] Century */ + __IOM uint32_t CEB : 1; /*!< [28..28] Century enable */ + __IM uint32_t : 2; + __IOM uint32_t CTERR : 1; /*!< [31..31] Counter read error status. Error is triggered when + software reads the lower word of the counters, and fails + to read the upper counter within 1/100 second. This is + because when the lower counter is read, the upper counter + is held off from incrementing until it is read so that + the full time stamp can be read. */ + } CTRUP_b; + } ; + + union { + __IOM uint32_t ALMLOW; /*!< (@ 0x00000048) RTC Alarms Lower */ + + struct { + __IOM uint32_t ALM100 : 8; /*!< [7..0] 100ths of a second Alarm */ + __IOM uint32_t ALMSEC : 7; /*!< [14..8] Seconds Alarm */ + __IM uint32_t : 1; + __IOM uint32_t ALMMIN : 7; /*!< [22..16] Minutes Alarm */ + __IM uint32_t : 1; + __IOM uint32_t ALMHR : 6; /*!< [29..24] Hours Alarm */ + } ALMLOW_b; + } ; + + union { + __IOM uint32_t ALMUP; /*!< (@ 0x0000004C) RTC Alarms Upper */ + + struct { + __IOM uint32_t ALMDATE : 6; /*!< [5..0] Date Alarm */ + __IM uint32_t : 2; + __IOM uint32_t ALMMO : 5; /*!< [12..8] Months Alarm */ + __IM uint32_t : 3; + __IOM uint32_t ALMWKDY : 3; /*!< [18..16] Weekdays Alarm */ + } ALMUP_b; + } ; + + union { + __IOM uint32_t RTCCTL; /*!< (@ 0x00000050) RTC Control Register */ + + struct { + __IOM uint32_t WRTC : 1; /*!< [0..0] Counter write control */ + __IOM uint32_t RPT : 3; /*!< [3..1] Alarm repeat interval */ + __IOM uint32_t RSTOP : 1; /*!< [4..4] RTC input clock control */ + __IOM uint32_t HR1224 : 1; /*!< [5..5] Hours Counter mode */ + } RTCCTL_b; + } ; + __IM uint32_t RESERVED1[43]; + + union { + __IOM uint32_t INTEN; /*!< (@ 0x00000100) RTC Interrupt Register: Enable */ + + struct { + __IOM uint32_t ALM : 1; /*!< [0..0] RTC Alarm interrupt */ + } INTEN_b; + } ; + + union { + __IOM uint32_t INTSTAT; /*!< (@ 0x00000104) RTC Interrupt Register: Status */ + + struct { + __IOM uint32_t ALM : 1; /*!< [0..0] RTC Alarm interrupt */ + } INTSTAT_b; + } ; + + union { + __IOM uint32_t INTCLR; /*!< (@ 0x00000108) RTC Interrupt Register: Clear */ + + struct { + __IOM uint32_t ALM : 1; /*!< [0..0] RTC Alarm interrupt */ + } INTCLR_b; + } ; + + union { + __IOM uint32_t INTSET; /*!< (@ 0x0000010C) RTC Interrupt Register: Set */ + + struct { + __IOM uint32_t ALM : 1; /*!< [0..0] RTC Alarm interrupt */ + } INTSET_b; + } ; +} RTC_Type; /*!< Size = 272 (0x110) */ + + + +/* =========================================================================================================================== */ +/* ================ SCARD ================ */ +/* =========================================================================================================================== */ + + +/** + * @brief Serial ISO7816 (SCARD) + */ + +typedef struct { /*!< (@ 0x40080000) SCARD Structure */ + + union { + __IOM uint32_t SR; /*!< (@ 0x00000000) ISO7816 interrupt status */ + + struct { + __IOM uint32_t FNE : 1; /*!< [0..0] RX FIFO not empty. */ + __IOM uint32_t TBERBF : 1; /*!< [1..1] FIFO empty (transmit) or full (receive). */ + __IOM uint32_t FER : 1; /*!< [2..2] Framing error. */ + __IOM uint32_t OVR : 1; /*!< [3..3] RX FIFO overflow. */ + __IOM uint32_t PE : 1; /*!< [4..4] Parity Error. */ + __IOM uint32_t FT2REND : 1; /*!< [5..5] TX to RX finished. */ + __IOM uint32_t FHF : 1; /*!< [6..6] FIFO Half Full. */ + } SR_b; + } ; + __IM uint32_t RESERVED[3]; + + union { + __IOM uint32_t DR; /*!< (@ 0x00000010) ISO7816 data */ + + struct { + __IOM uint32_t DR : 8; /*!< [7..0] Data register. */ + } DR_b; + } ; + __IM uint32_t RESERVED1[3]; + + union { + __IOM uint32_t SR1; /*!< (@ 0x00000020) ISO7816 interrupt status 1 */ + + struct { + __IOM uint32_t ECNTOVER : 1; /*!< [0..0] ETU counter overflow. */ + __IOM uint32_t PRL : 1; /*!< [1..1] Card insert/remove. */ + __IOM uint32_t SYNCEND : 1; /*!< [2..2] Write complete synchronization. */ + __IOM uint32_t IDLE : 1; /*!< [3..3] ISO7816 idle. */ + } SR1_b; + } ; + __IM uint32_t RESERVED2[5]; + + union { + __IOM uint32_t RETXCNTRMI; /*!< (@ 0x00000038) ISO7816 resent count inquiry */ + + struct { + __IOM uint32_t RETXCNTRMI : 4; /*!< [3..0] Resent count inquiry register. */ + } RETXCNTRMI_b; + } ; + __IM uint32_t RESERVED3[49]; + + union { + __IOM uint32_t CLKCTRL; /*!< (@ 0x00000100) Clock Control */ + + struct { + __IOM uint32_t CLKEN : 1; /*!< [0..0] Enable the serial source clock for SCARD. */ + __IOM uint32_t APBCLKEN : 1; /*!< [1..1] Enable the SCARD APB clock to run continuously. */ + } CLKCTRL_b; + } ; +} SCARD_Type; /*!< Size = 260 (0x104) */ + + + +/* =========================================================================================================================== */ +/* ================ SECURITY ================ */ +/* =========================================================================================================================== */ + + +/** + * @brief Security Interfaces (SECURITY) + */ + +typedef struct { /*!< (@ 0x40030000) SECURITY Structure */ + + union { + __IOM uint32_t CTRL; /*!< (@ 0x00000000) Control Register */ + + struct { + __IOM uint32_t ENABLE : 1; /*!< [0..0] Function Enable. Software should set the ENABLE bit to + initiate a CRC operation. Hardware will clear the ENABLE + bit upon completion. */ + __IM uint32_t : 3; + __IOM uint32_t FUNCTION : 4; /*!< [7..4] Function Select */ + __IM uint32_t : 23; + __IOM uint32_t CRCERROR : 1; /*!< [31..31] CRC Error Status - Set to 1 if an error occurs during + a CRC operation. Cleared when CTRL register is written + (with any value). Usually indicates an invalid address + range. */ + } CTRL_b; + } ; + __IM uint32_t RESERVED[3]; + + union { + __IOM uint32_t SRCADDR; /*!< (@ 0x00000010) Source Addresss */ + + struct { + __IOM uint32_t ADDR : 32; /*!< [31..0] Source Buffer Address. Address may be byte aligned, + but the length must be a multiple of 4 bits. */ + } SRCADDR_b; + } ; + __IM uint32_t RESERVED1[3]; + + union { + __IOM uint32_t LEN; /*!< (@ 0x00000020) Length */ + + struct { + __IM uint32_t : 2; + __IOM uint32_t LEN : 18; /*!< [19..2] Buffer size (bottom two bits assumed to be zero to ensure + a multiple of 4 bytes) */ + } LEN_b; + } ; + __IM uint32_t RESERVED2[3]; + + union { + __IOM uint32_t RESULT; /*!< (@ 0x00000030) CRC Seed/Result Register */ + + struct { + __IOM uint32_t CRC : 32; /*!< [31..0] CRC Seed/Result. Software must seed the CRC with 0xFFFFFFFF + before starting a CRC operation (unless the CRC is continued + from a previous operation). */ + } RESULT_b; + } ; + __IM uint32_t RESERVED3[17]; + + union { + __IOM uint32_t LOCKCTRL; /*!< (@ 0x00000078) LOCK Control Register */ + + struct { + __IOM uint32_t SELECT : 8; /*!< [7..0] LOCK Function Select register. */ + } LOCKCTRL_b; + } ; + + union { + __IOM uint32_t LOCKSTAT; /*!< (@ 0x0000007C) LOCK Status Register */ + + struct { + __IOM uint32_t STATUS : 32; /*!< [31..0] LOCK Status register. This register is a bitmask for + which resources are currently unlocked. These bits are + one-hot per resource. */ + } LOCKSTAT_b; + } ; + + union { + __IOM uint32_t KEY0; /*!< (@ 0x00000080) Key0 Register */ + + struct { + __IOM uint32_t KEY0 : 32; /*!< [31..0] Bits [31:0] of the 128-bit key should be written to + this register. To protect key values, the register always + returns 0x00000000. */ + } KEY0_b; + } ; + + union { + __IOM uint32_t KEY1; /*!< (@ 0x00000084) Key1 Register */ + + struct { + __IOM uint32_t KEY1 : 32; /*!< [31..0] Bits [63:32] of the 128-bit key should be written to + this register. To protect key values, the register always + returns 0x00000000. */ + } KEY1_b; + } ; + + union { + __IOM uint32_t KEY2; /*!< (@ 0x00000088) Key2 Register */ + + struct { + __IOM uint32_t KEY2 : 32; /*!< [31..0] Bits [95:64] of the 128-bit key should be written to + this register. To protect key values, the register always + returns 0x00000000. */ + } KEY2_b; + } ; + + union { + __IOM uint32_t KEY3; /*!< (@ 0x0000008C) Key3 Register */ + + struct { + __IOM uint32_t KEY3 : 32; /*!< [31..0] Bits [127:96] of the 128-bit key should be written to + this register. To protect key values, the register always + returns 0x00000000. */ + } KEY3_b; + } ; +} SECURITY_Type; /*!< Size = 144 (0x90) */ + + + +/* =========================================================================================================================== */ +/* ================ UART0 ================ */ +/* =========================================================================================================================== */ + + +/** + * @brief Serial UART (UART0) + */ + +typedef struct { /*!< (@ 0x4001C000) UART0 Structure */ + + union { + __IOM uint32_t DR; /*!< (@ 0x00000000) UART Data Register */ + + struct { + __IOM uint32_t DATA : 8; /*!< [7..0] This is the UART data port. */ + __IOM uint32_t FEDATA : 1; /*!< [8..8] This is the framing error indicator. */ + __IOM uint32_t PEDATA : 1; /*!< [9..9] This is the parity error indicator. */ + __IOM uint32_t BEDATA : 1; /*!< [10..10] This is the break error indicator. */ + __IOM uint32_t OEDATA : 1; /*!< [11..11] This is the overrun error indicator. */ + } DR_b; + } ; + + union { + __IOM uint32_t RSR; /*!< (@ 0x00000004) UART Status Register */ + + struct { + __IOM uint32_t FESTAT : 1; /*!< [0..0] This is the framing error indicator. */ + __IOM uint32_t PESTAT : 1; /*!< [1..1] This is the parity error indicator. */ + __IOM uint32_t BESTAT : 1; /*!< [2..2] This is the break error indicator. */ + __IOM uint32_t OESTAT : 1; /*!< [3..3] This is the overrun error indicator. */ + } RSR_b; + } ; + __IM uint32_t RESERVED[4]; + + union { + __IOM uint32_t FR; /*!< (@ 0x00000018) Flag Register */ + + struct { + __IOM uint32_t CTS : 1; /*!< [0..0] This bit holds the clear to send indicator. */ + __IOM uint32_t DSR : 1; /*!< [1..1] This bit holds the data set ready indicator. */ + __IOM uint32_t DCD : 1; /*!< [2..2] This bit holds the data carrier detect indicator. */ + __IOM uint32_t BUSY : 1; /*!< [3..3] This bit holds the busy indicator. */ + __IOM uint32_t RXFE : 1; /*!< [4..4] This bit holds the receive FIFO empty indicator. */ + __IOM uint32_t TXFF : 1; /*!< [5..5] This bit holds the transmit FIFO full indicator. */ + __IOM uint32_t RXFF : 1; /*!< [6..6] This bit holds the receive FIFO full indicator. */ + __IOM uint32_t TXFE : 1; /*!< [7..7] This bit holds the transmit FIFO empty indicator. */ + __IOM uint32_t TXBUSY : 1; /*!< [8..8] This bit holds the transmit BUSY indicator. */ + } FR_b; + } ; + __IM uint32_t RESERVED1; + + union { + __IOM uint32_t ILPR; /*!< (@ 0x00000020) IrDA Counter */ + + struct { + __IOM uint32_t ILPDVSR : 8; /*!< [7..0] These bits hold the IrDA counter divisor. */ + } ILPR_b; + } ; + + union { + __IOM uint32_t IBRD; /*!< (@ 0x00000024) Integer Baud Rate Divisor */ + + struct { + __IOM uint32_t DIVINT : 16; /*!< [15..0] These bits hold the baud integer divisor. */ + } IBRD_b; + } ; + + union { + __IOM uint32_t FBRD; /*!< (@ 0x00000028) Fractional Baud Rate Divisor */ + + struct { + __IOM uint32_t DIVFRAC : 6; /*!< [5..0] These bits hold the baud fractional divisor. */ + } FBRD_b; + } ; + + union { + __IOM uint32_t LCRH; /*!< (@ 0x0000002C) Line Control High */ + + struct { + __IOM uint32_t BRK : 1; /*!< [0..0] This bit holds the break set. */ + __IOM uint32_t PEN : 1; /*!< [1..1] This bit holds the parity enable. */ + __IOM uint32_t EPS : 1; /*!< [2..2] This bit holds the even parity select. */ + __IOM uint32_t STP2 : 1; /*!< [3..3] This bit holds the two stop bits select. */ + __IOM uint32_t FEN : 1; /*!< [4..4] This bit holds the FIFO enable. */ + __IOM uint32_t WLEN : 2; /*!< [6..5] These bits hold the write length. */ + __IOM uint32_t SPS : 1; /*!< [7..7] This bit holds the stick parity select. */ + } LCRH_b; + } ; + + union { + __IOM uint32_t CR; /*!< (@ 0x00000030) Control Register */ + + struct { + __IOM uint32_t UARTEN : 1; /*!< [0..0] This bit is the UART enable. */ + __IOM uint32_t SIREN : 1; /*!< [1..1] This bit is the SIR ENDEC enable. */ + __IOM uint32_t SIRLP : 1; /*!< [2..2] This bit is the SIR low power select. */ + __IOM uint32_t CLKEN : 1; /*!< [3..3] This bit is the UART clock enable. */ + __IOM uint32_t CLKSEL : 3; /*!< [6..4] This bitfield is the UART clock select. */ + __IOM uint32_t LBE : 1; /*!< [7..7] This bit is the loopback enable. */ + __IOM uint32_t TXE : 1; /*!< [8..8] This bit is the transmit enable. */ + __IOM uint32_t RXE : 1; /*!< [9..9] This bit is the receive enable. */ + __IOM uint32_t DTR : 1; /*!< [10..10] This bit enables data transmit ready. */ + __IOM uint32_t RTS : 1; /*!< [11..11] This bit enables request to send. */ + __IOM uint32_t OUT1 : 1; /*!< [12..12] This bit holds modem Out1. */ + __IOM uint32_t OUT2 : 1; /*!< [13..13] This bit holds modem Out2. */ + __IOM uint32_t RTSEN : 1; /*!< [14..14] This bit enables RTS hardware flow control. */ + __IOM uint32_t CTSEN : 1; /*!< [15..15] This bit enables CTS hardware flow control. */ + } CR_b; + } ; + + union { + __IOM uint32_t IFLS; /*!< (@ 0x00000034) FIFO Interrupt Level Select */ + + struct { + __IOM uint32_t TXIFLSEL : 3; /*!< [2..0] These bits hold the transmit FIFO interrupt level. */ + __IOM uint32_t RXIFLSEL : 3; /*!< [5..3] These bits hold the receive FIFO interrupt level. */ + } IFLS_b; + } ; + + union { + __IOM uint32_t IER; /*!< (@ 0x00000038) Interrupt Enable */ + + struct { + __IOM uint32_t TXCMPMIM : 1; /*!< [0..0] This bit holds the modem TXCMP interrupt enable. */ + __IOM uint32_t CTSMIM : 1; /*!< [1..1] This bit holds the modem CTS interrupt enable. */ + __IOM uint32_t DCDMIM : 1; /*!< [2..2] This bit holds the modem DCD interrupt enable. */ + __IOM uint32_t DSRMIM : 1; /*!< [3..3] This bit holds the modem DSR interrupt enable. */ + __IOM uint32_t RXIM : 1; /*!< [4..4] This bit holds the receive interrupt enable. */ + __IOM uint32_t TXIM : 1; /*!< [5..5] This bit holds the transmit interrupt enable. */ + __IOM uint32_t RTIM : 1; /*!< [6..6] This bit holds the receive timeout interrupt enable. */ + __IOM uint32_t FEIM : 1; /*!< [7..7] This bit holds the framing error interrupt enable. */ + __IOM uint32_t PEIM : 1; /*!< [8..8] This bit holds the parity error interrupt enable. */ + __IOM uint32_t BEIM : 1; /*!< [9..9] This bit holds the break error interrupt enable. */ + __IOM uint32_t OEIM : 1; /*!< [10..10] This bit holds the overflow interrupt enable. */ + } IER_b; + } ; + + union { + __IOM uint32_t IES; /*!< (@ 0x0000003C) Interrupt Status */ + + struct { + __IOM uint32_t TXCMPMRIS : 1; /*!< [0..0] This bit holds the modem TXCMP interrupt status. */ + __IOM uint32_t CTSMRIS : 1; /*!< [1..1] This bit holds the modem CTS interrupt status. */ + __IOM uint32_t DCDMRIS : 1; /*!< [2..2] This bit holds the modem DCD interrupt status. */ + __IOM uint32_t DSRMRIS : 1; /*!< [3..3] This bit holds the modem DSR interrupt status. */ + __IOM uint32_t RXRIS : 1; /*!< [4..4] This bit holds the receive interrupt status. */ + __IOM uint32_t TXRIS : 1; /*!< [5..5] This bit holds the transmit interrupt status. */ + __IOM uint32_t RTRIS : 1; /*!< [6..6] This bit holds the receive timeout interrupt status. */ + __IOM uint32_t FERIS : 1; /*!< [7..7] This bit holds the framing error interrupt status. */ + __IOM uint32_t PERIS : 1; /*!< [8..8] This bit holds the parity error interrupt status. */ + __IOM uint32_t BERIS : 1; /*!< [9..9] This bit holds the break error interrupt status. */ + __IOM uint32_t OERIS : 1; /*!< [10..10] This bit holds the overflow interrupt status. */ + } IES_b; + } ; + + union { + __IOM uint32_t MIS; /*!< (@ 0x00000040) Masked Interrupt Status */ + + struct { + __IOM uint32_t TXCMPMMIS : 1; /*!< [0..0] This bit holds the modem TXCMP interrupt status masked. */ + __IOM uint32_t CTSMMIS : 1; /*!< [1..1] This bit holds the modem CTS interrupt status masked. */ + __IOM uint32_t DCDMMIS : 1; /*!< [2..2] This bit holds the modem DCD interrupt status masked. */ + __IOM uint32_t DSRMMIS : 1; /*!< [3..3] This bit holds the modem DSR interrupt status masked. */ + __IOM uint32_t RXMIS : 1; /*!< [4..4] This bit holds the receive interrupt status masked. */ + __IOM uint32_t TXMIS : 1; /*!< [5..5] This bit holds the transmit interrupt status masked. */ + __IOM uint32_t RTMIS : 1; /*!< [6..6] This bit holds the receive timeout interrupt status masked. */ + __IOM uint32_t FEMIS : 1; /*!< [7..7] This bit holds the framing error interrupt status masked. */ + __IOM uint32_t PEMIS : 1; /*!< [8..8] This bit holds the parity error interrupt status masked. */ + __IOM uint32_t BEMIS : 1; /*!< [9..9] This bit holds the break error interrupt status masked. */ + __IOM uint32_t OEMIS : 1; /*!< [10..10] This bit holds the overflow interrupt status masked. */ + } MIS_b; + } ; + + union { + __IOM uint32_t IEC; /*!< (@ 0x00000044) Interrupt Clear */ + + struct { + __IOM uint32_t TXCMPMIC : 1; /*!< [0..0] This bit holds the modem TXCMP interrupt clear. */ + __IOM uint32_t CTSMIC : 1; /*!< [1..1] This bit holds the modem CTS interrupt clear. */ + __IOM uint32_t DCDMIC : 1; /*!< [2..2] This bit holds the modem DCD interrupt clear. */ + __IOM uint32_t DSRMIC : 1; /*!< [3..3] This bit holds the modem DSR interrupt clear. */ + __IOM uint32_t RXIC : 1; /*!< [4..4] This bit holds the receive interrupt clear. */ + __IOM uint32_t TXIC : 1; /*!< [5..5] This bit holds the transmit interrupt clear. */ + __IOM uint32_t RTIC : 1; /*!< [6..6] This bit holds the receive timeout interrupt clear. */ + __IOM uint32_t FEIC : 1; /*!< [7..7] This bit holds the framing error interrupt clear. */ + __IOM uint32_t PEIC : 1; /*!< [8..8] This bit holds the parity error interrupt clear. */ + __IOM uint32_t BEIC : 1; /*!< [9..9] This bit holds the break error interrupt clear. */ + __IOM uint32_t OEIC : 1; /*!< [10..10] This bit holds the overflow interrupt clear. */ + } IEC_b; + } ; +} UART0_Type; /*!< Size = 72 (0x48) */ + + + +/* =========================================================================================================================== */ +/* ================ VCOMP ================ */ +/* =========================================================================================================================== */ + + +/** + * @brief Voltage Comparator (VCOMP) + */ + +typedef struct { /*!< (@ 0x4000C000) VCOMP Structure */ + + union { + __IOM uint32_t CFG; /*!< (@ 0x00000000) Configuration Register */ + + struct { + __IOM uint32_t PSEL : 2; /*!< [1..0] This bitfield selects the positive input to the comparator. */ + __IM uint32_t : 6; + __IOM uint32_t NSEL : 2; /*!< [9..8] This bitfield selects the negative input to the comparator. */ + __IM uint32_t : 6; + __IOM uint32_t LVLSEL : 4; /*!< [19..16] When the reference input NSEL is set to NSEL_DAC, this + bitfield selects the voltage level for the negative input + to the comparator. */ + } CFG_b; + } ; + + union { + __IOM uint32_t STAT; /*!< (@ 0x00000004) Status Register */ + + struct { + __IOM uint32_t CMPOUT : 1; /*!< [0..0] This bit is 1 if the positive input of the comparator + is greater than the negative input. */ + __IOM uint32_t PWDSTAT : 1; /*!< [1..1] This bit indicates the power down state of the voltage + comparator. */ + } STAT_b; + } ; + + union { + __IOM uint32_t PWDKEY; /*!< (@ 0x00000008) Key Register for Powering Down the Voltage Comparator */ + + struct { + __IOM uint32_t PWDKEY : 32; /*!< [31..0] Key register value. */ + } PWDKEY_b; + } ; + __IM uint32_t RESERVED[125]; + + union { + __IOM uint32_t INTEN; /*!< (@ 0x00000200) Voltage Comparator Interrupt registers: Enable */ + + struct { + __IOM uint32_t OUTLOW : 1; /*!< [0..0] This bit is the vcompout low interrupt. */ + __IOM uint32_t OUTHI : 1; /*!< [1..1] This bit is the vcompout high interrupt. */ + } INTEN_b; + } ; + + union { + __IOM uint32_t INTSTAT; /*!< (@ 0x00000204) Voltage Comparator Interrupt registers: Status */ + + struct { + __IOM uint32_t OUTLOW : 1; /*!< [0..0] This bit is the vcompout low interrupt. */ + __IOM uint32_t OUTHI : 1; /*!< [1..1] This bit is the vcompout high interrupt. */ + } INTSTAT_b; + } ; + + union { + __IOM uint32_t INTCLR; /*!< (@ 0x00000208) Voltage Comparator Interrupt registers: Clear */ + + struct { + __IOM uint32_t OUTLOW : 1; /*!< [0..0] This bit is the vcompout low interrupt. */ + __IOM uint32_t OUTHI : 1; /*!< [1..1] This bit is the vcompout high interrupt. */ + } INTCLR_b; + } ; + + union { + __IOM uint32_t INTSET; /*!< (@ 0x0000020C) Voltage Comparator Interrupt registers: Set */ + + struct { + __IOM uint32_t OUTLOW : 1; /*!< [0..0] This bit is the vcompout low interrupt. */ + __IOM uint32_t OUTHI : 1; /*!< [1..1] This bit is the vcompout high interrupt. */ + } INTSET_b; + } ; +} VCOMP_Type; /*!< Size = 528 (0x210) */ + + + +/* =========================================================================================================================== */ +/* ================ WDT ================ */ +/* =========================================================================================================================== */ + + +/** + * @brief Watchdog Timer (WDT) + */ + +typedef struct { /*!< (@ 0x40024000) WDT Structure */ + + union { + __IOM uint32_t CFG; /*!< (@ 0x00000000) Configuration Register */ + + struct { + __IOM uint32_t WDTEN : 1; /*!< [0..0] This bitfield enables the WDT. */ + __IOM uint32_t INTEN : 1; /*!< [1..1] This bitfield enables the WDT interrupt. Note : This + bit must be set before the interrupt status bit will reflect + a watchdog timer expiration. The IER interrupt register + must also be enabled for a WDT interrupt to be sent to + the NVIC. */ + __IOM uint32_t RESEN : 1; /*!< [2..2] This bitfield enables the WDT reset. This needs to be + set together with the WDREN bit in REG_RSTGEN_CFG register + (in reset gen) to trigger the reset. */ + __IM uint32_t : 5; + __IOM uint32_t RESVAL : 8; /*!< [15..8] This bitfield is the compare value for counter bits + 7:0 to generate a watchdog reset. This will cause a software + reset. */ + __IOM uint32_t INTVAL : 8; /*!< [23..16] This bitfield is the compare value for counter bits + 7:0 to generate a watchdog interrupt. */ + __IOM uint32_t CLKSEL : 3; /*!< [26..24] Select the frequency for the WDT. All values not enumerated + below are undefined. */ + } CFG_b; + } ; + + union { + __IOM uint32_t RSTRT; /*!< (@ 0x00000004) Restart the watchdog timer. */ + + struct { + __IOM uint32_t RSTRT : 8; /*!< [7..0] Writing 0xB2 to WDTRSTRT restarts the watchdog timer. + This is a write only register. Reading this register will + only provide all 0. */ + } RSTRT_b; + } ; + + union { + __IOM uint32_t LOCK; /*!< (@ 0x00000008) Locks the WDT */ + + struct { + __IOM uint32_t LOCK : 8; /*!< [7..0] Writing 0x3A locks the watchdog timer. Once locked, the + WDTCFG reg cannot be written and WDTEN is set. */ + } LOCK_b; + } ; + + union { + __IOM uint32_t COUNT; /*!< (@ 0x0000000C) Current Counter Value for WDT */ + + struct { + __IOM uint32_t COUNT : 8; /*!< [7..0] Read-Only current value of the WDT counter */ + } COUNT_b; + } ; + __IM uint32_t RESERVED[124]; + + union { + __IOM uint32_t INTEN; /*!< (@ 0x00000200) WDT Interrupt register: Enable */ + + struct { + __IOM uint32_t WDTINT : 1; /*!< [0..0] Watchdog Timer Interrupt. */ + } INTEN_b; + } ; + + union { + __IOM uint32_t INTSTAT; /*!< (@ 0x00000204) WDT Interrupt register: Status */ + + struct { + __IOM uint32_t WDTINT : 1; /*!< [0..0] Watchdog Timer Interrupt. */ + } INTSTAT_b; + } ; + + union { + __IOM uint32_t INTCLR; /*!< (@ 0x00000208) WDT Interrupt register: Clear */ + + struct { + __IOM uint32_t WDTINT : 1; /*!< [0..0] Watchdog Timer Interrupt. */ + } INTCLR_b; + } ; + + union { + __IOM uint32_t INTSET; /*!< (@ 0x0000020C) WDT Interrupt register: Set */ + + struct { + __IOM uint32_t WDTINT : 1; /*!< [0..0] Watchdog Timer Interrupt. */ + } INTSET_b; + } ; +} WDT_Type; /*!< Size = 528 (0x210) */ + + +/** @} */ /* End of group Device_Peripheral_peripherals */ + + +/* =========================================================================================================================== */ +/* ================ Device Specific Peripheral Address Map ================ */ +/* =========================================================================================================================== */ + + +/** @addtogroup Device_Peripheral_peripheralAddr + * @{ + */ + +#define ADC_BASE 0x50010000UL +#define APBDMA_BASE 0x40011000UL +#define BLEIF_BASE 0x5000C000UL +#define CACHECTRL_BASE 0x40018000UL +#define CLKGEN_BASE 0x40004000UL +#define CTIMER_BASE 0x40008000UL +#define GPIO_BASE 0x40010000UL +#define IOM0_BASE 0x50004000UL +#define IOM1_BASE 0x50005000UL +#define IOM2_BASE 0x50006000UL +#define IOM3_BASE 0x50007000UL +#define IOM4_BASE 0x50008000UL +#define IOM5_BASE 0x50009000UL +#define IOSLAVE_BASE 0x50000000UL +#define MCUCTRL_BASE 0x40020000UL +#define MSPI_BASE 0x50014000UL +#define PDM_BASE 0x50011000UL +#define PWRCTRL_BASE 0x40021000UL +#define RSTGEN_BASE 0x40000000UL +#define RTC_BASE 0x40004200UL +#define SCARD_BASE 0x40080000UL +#define SECURITY_BASE 0x40030000UL +#define UART0_BASE 0x4001C000UL +#define UART1_BASE 0x4001D000UL +#define VCOMP_BASE 0x4000C000UL +#define WDT_BASE 0x40024000UL + +/** @} */ /* End of group Device_Peripheral_peripheralAddr */ + + +/* =========================================================================================================================== */ +/* ================ Peripheral declaration ================ */ +/* =========================================================================================================================== */ + + +/** @addtogroup Device_Peripheral_declaration + * @{ + */ + +#define ADC ((ADC_Type*) ADC_BASE) +#define APBDMA ((APBDMA_Type*) APBDMA_BASE) +#define BLEIF ((BLEIF_Type*) BLEIF_BASE) +#define CACHECTRL ((CACHECTRL_Type*) CACHECTRL_BASE) +#define CLKGEN ((CLKGEN_Type*) CLKGEN_BASE) +#define CTIMER ((CTIMER_Type*) CTIMER_BASE) +#define GPIO ((GPIO_Type*) GPIO_BASE) +#define IOM0 ((IOM0_Type*) IOM0_BASE) +#define IOM1 ((IOM0_Type*) IOM1_BASE) +#define IOM2 ((IOM0_Type*) IOM2_BASE) +#define IOM3 ((IOM0_Type*) IOM3_BASE) +#define IOM4 ((IOM0_Type*) IOM4_BASE) +#define IOM5 ((IOM0_Type*) IOM5_BASE) +#define IOSLAVE ((IOSLAVE_Type*) IOSLAVE_BASE) +#define MCUCTRL ((MCUCTRL_Type*) MCUCTRL_BASE) +#define MSPI ((MSPI_Type*) MSPI_BASE) +#define PDM ((PDM_Type*) PDM_BASE) +#define PWRCTRL ((PWRCTRL_Type*) PWRCTRL_BASE) +#define RSTGEN ((RSTGEN_Type*) RSTGEN_BASE) +#define RTC ((RTC_Type*) RTC_BASE) +#define SCARD ((SCARD_Type*) SCARD_BASE) +#define SECURITY ((SECURITY_Type*) SECURITY_BASE) +#define UART0 ((UART0_Type*) UART0_BASE) +#define UART1 ((UART0_Type*) UART1_BASE) +#define VCOMP ((VCOMP_Type*) VCOMP_BASE) +#define WDT ((WDT_Type*) WDT_BASE) + +/** @} */ /* End of group Device_Peripheral_declaration */ + + +/* ========================================= End of section using anonymous unions ========================================= */ +#if defined (__CC_ARM) + #pragma pop +#elif defined (__ICCARM__) + /* leave anonymous unions enabled */ +#elif (__ARMCC_VERSION >= 6010050) + #pragma clang diagnostic pop +#elif defined (__GNUC__) + /* anonymous unions are enabled by default */ +#elif defined (__TMS470__) + /* anonymous unions are enabled by default */ +#elif defined (__TASKING__) + #pragma warning restore +#elif defined (__CSMC__) + /* anonymous unions are enabled by default */ +#endif + + +/* =========================================================================================================================== */ +/* ================ Pos/Mask Peripheral Section ================ */ +/* =========================================================================================================================== */ + + +/** @addtogroup PosMask_peripherals + * @{ + */ + + + +/* =========================================================================================================================== */ +/* ================ ADC ================ */ +/* =========================================================================================================================== */ + +/* ========================================================== CFG ========================================================== */ +#define ADC_CFG_CLKSEL_Pos (24UL) /*!< ADC CFG: CLKSEL (Bit 24) */ +#define ADC_CFG_CLKSEL_Msk (0x3000000UL) /*!< ADC CFG: CLKSEL (Bitfield-Mask: 0x03) */ +#define ADC_CFG_TRIGPOL_Pos (19UL) /*!< ADC CFG: TRIGPOL (Bit 19) */ +#define ADC_CFG_TRIGPOL_Msk (0x80000UL) /*!< ADC CFG: TRIGPOL (Bitfield-Mask: 0x01) */ +#define ADC_CFG_TRIGSEL_Pos (16UL) /*!< ADC CFG: TRIGSEL (Bit 16) */ +#define ADC_CFG_TRIGSEL_Msk (0x70000UL) /*!< ADC CFG: TRIGSEL (Bitfield-Mask: 0x07) */ +#define ADC_CFG_DFIFORDEN_Pos (12UL) /*!< ADC CFG: DFIFORDEN (Bit 12) */ +#define ADC_CFG_DFIFORDEN_Msk (0x1000UL) /*!< ADC CFG: DFIFORDEN (Bitfield-Mask: 0x01) */ +#define ADC_CFG_REFSEL_Pos (8UL) /*!< ADC CFG: REFSEL (Bit 8) */ +#define ADC_CFG_REFSEL_Msk (0x300UL) /*!< ADC CFG: REFSEL (Bitfield-Mask: 0x03) */ +#define ADC_CFG_CKMODE_Pos (4UL) /*!< ADC CFG: CKMODE (Bit 4) */ +#define ADC_CFG_CKMODE_Msk (0x10UL) /*!< ADC CFG: CKMODE (Bitfield-Mask: 0x01) */ +#define ADC_CFG_LPMODE_Pos (3UL) /*!< ADC CFG: LPMODE (Bit 3) */ +#define ADC_CFG_LPMODE_Msk (0x8UL) /*!< ADC CFG: LPMODE (Bitfield-Mask: 0x01) */ +#define ADC_CFG_RPTEN_Pos (2UL) /*!< ADC CFG: RPTEN (Bit 2) */ +#define ADC_CFG_RPTEN_Msk (0x4UL) /*!< ADC CFG: RPTEN (Bitfield-Mask: 0x01) */ +#define ADC_CFG_ADCEN_Pos (0UL) /*!< ADC CFG: ADCEN (Bit 0) */ +#define ADC_CFG_ADCEN_Msk (0x1UL) /*!< ADC CFG: ADCEN (Bitfield-Mask: 0x01) */ +/* ========================================================= STAT ========================================================== */ +#define ADC_STAT_PWDSTAT_Pos (0UL) /*!< ADC STAT: PWDSTAT (Bit 0) */ +#define ADC_STAT_PWDSTAT_Msk (0x1UL) /*!< ADC STAT: PWDSTAT (Bitfield-Mask: 0x01) */ +/* ========================================================== SWT ========================================================== */ +#define ADC_SWT_SWT_Pos (0UL) /*!< ADC SWT: SWT (Bit 0) */ +#define ADC_SWT_SWT_Msk (0xffUL) /*!< ADC SWT: SWT (Bitfield-Mask: 0xff) */ +/* ======================================================== SL0CFG ========================================================= */ +#define ADC_SL0CFG_ADSEL0_Pos (24UL) /*!< ADC SL0CFG: ADSEL0 (Bit 24) */ +#define ADC_SL0CFG_ADSEL0_Msk (0x7000000UL) /*!< ADC SL0CFG: ADSEL0 (Bitfield-Mask: 0x07) */ +#define ADC_SL0CFG_PRMODE0_Pos (16UL) /*!< ADC SL0CFG: PRMODE0 (Bit 16) */ +#define ADC_SL0CFG_PRMODE0_Msk (0x30000UL) /*!< ADC SL0CFG: PRMODE0 (Bitfield-Mask: 0x03) */ +#define ADC_SL0CFG_CHSEL0_Pos (8UL) /*!< ADC SL0CFG: CHSEL0 (Bit 8) */ +#define ADC_SL0CFG_CHSEL0_Msk (0xf00UL) /*!< ADC SL0CFG: CHSEL0 (Bitfield-Mask: 0x0f) */ +#define ADC_SL0CFG_WCEN0_Pos (1UL) /*!< ADC SL0CFG: WCEN0 (Bit 1) */ +#define ADC_SL0CFG_WCEN0_Msk (0x2UL) /*!< ADC SL0CFG: WCEN0 (Bitfield-Mask: 0x01) */ +#define ADC_SL0CFG_SLEN0_Pos (0UL) /*!< ADC SL0CFG: SLEN0 (Bit 0) */ +#define ADC_SL0CFG_SLEN0_Msk (0x1UL) /*!< ADC SL0CFG: SLEN0 (Bitfield-Mask: 0x01) */ +/* ======================================================== SL1CFG ========================================================= */ +#define ADC_SL1CFG_ADSEL1_Pos (24UL) /*!< ADC SL1CFG: ADSEL1 (Bit 24) */ +#define ADC_SL1CFG_ADSEL1_Msk (0x7000000UL) /*!< ADC SL1CFG: ADSEL1 (Bitfield-Mask: 0x07) */ +#define ADC_SL1CFG_PRMODE1_Pos (16UL) /*!< ADC SL1CFG: PRMODE1 (Bit 16) */ +#define ADC_SL1CFG_PRMODE1_Msk (0x30000UL) /*!< ADC SL1CFG: PRMODE1 (Bitfield-Mask: 0x03) */ +#define ADC_SL1CFG_CHSEL1_Pos (8UL) /*!< ADC SL1CFG: CHSEL1 (Bit 8) */ +#define ADC_SL1CFG_CHSEL1_Msk (0xf00UL) /*!< ADC SL1CFG: CHSEL1 (Bitfield-Mask: 0x0f) */ +#define ADC_SL1CFG_WCEN1_Pos (1UL) /*!< ADC SL1CFG: WCEN1 (Bit 1) */ +#define ADC_SL1CFG_WCEN1_Msk (0x2UL) /*!< ADC SL1CFG: WCEN1 (Bitfield-Mask: 0x01) */ +#define ADC_SL1CFG_SLEN1_Pos (0UL) /*!< ADC SL1CFG: SLEN1 (Bit 0) */ +#define ADC_SL1CFG_SLEN1_Msk (0x1UL) /*!< ADC SL1CFG: SLEN1 (Bitfield-Mask: 0x01) */ +/* ======================================================== SL2CFG ========================================================= */ +#define ADC_SL2CFG_ADSEL2_Pos (24UL) /*!< ADC SL2CFG: ADSEL2 (Bit 24) */ +#define ADC_SL2CFG_ADSEL2_Msk (0x7000000UL) /*!< ADC SL2CFG: ADSEL2 (Bitfield-Mask: 0x07) */ +#define ADC_SL2CFG_PRMODE2_Pos (16UL) /*!< ADC SL2CFG: PRMODE2 (Bit 16) */ +#define ADC_SL2CFG_PRMODE2_Msk (0x30000UL) /*!< ADC SL2CFG: PRMODE2 (Bitfield-Mask: 0x03) */ +#define ADC_SL2CFG_CHSEL2_Pos (8UL) /*!< ADC SL2CFG: CHSEL2 (Bit 8) */ +#define ADC_SL2CFG_CHSEL2_Msk (0xf00UL) /*!< ADC SL2CFG: CHSEL2 (Bitfield-Mask: 0x0f) */ +#define ADC_SL2CFG_WCEN2_Pos (1UL) /*!< ADC SL2CFG: WCEN2 (Bit 1) */ +#define ADC_SL2CFG_WCEN2_Msk (0x2UL) /*!< ADC SL2CFG: WCEN2 (Bitfield-Mask: 0x01) */ +#define ADC_SL2CFG_SLEN2_Pos (0UL) /*!< ADC SL2CFG: SLEN2 (Bit 0) */ +#define ADC_SL2CFG_SLEN2_Msk (0x1UL) /*!< ADC SL2CFG: SLEN2 (Bitfield-Mask: 0x01) */ +/* ======================================================== SL3CFG ========================================================= */ +#define ADC_SL3CFG_ADSEL3_Pos (24UL) /*!< ADC SL3CFG: ADSEL3 (Bit 24) */ +#define ADC_SL3CFG_ADSEL3_Msk (0x7000000UL) /*!< ADC SL3CFG: ADSEL3 (Bitfield-Mask: 0x07) */ +#define ADC_SL3CFG_PRMODE3_Pos (16UL) /*!< ADC SL3CFG: PRMODE3 (Bit 16) */ +#define ADC_SL3CFG_PRMODE3_Msk (0x30000UL) /*!< ADC SL3CFG: PRMODE3 (Bitfield-Mask: 0x03) */ +#define ADC_SL3CFG_CHSEL3_Pos (8UL) /*!< ADC SL3CFG: CHSEL3 (Bit 8) */ +#define ADC_SL3CFG_CHSEL3_Msk (0xf00UL) /*!< ADC SL3CFG: CHSEL3 (Bitfield-Mask: 0x0f) */ +#define ADC_SL3CFG_WCEN3_Pos (1UL) /*!< ADC SL3CFG: WCEN3 (Bit 1) */ +#define ADC_SL3CFG_WCEN3_Msk (0x2UL) /*!< ADC SL3CFG: WCEN3 (Bitfield-Mask: 0x01) */ +#define ADC_SL3CFG_SLEN3_Pos (0UL) /*!< ADC SL3CFG: SLEN3 (Bit 0) */ +#define ADC_SL3CFG_SLEN3_Msk (0x1UL) /*!< ADC SL3CFG: SLEN3 (Bitfield-Mask: 0x01) */ +/* ======================================================== SL4CFG ========================================================= */ +#define ADC_SL4CFG_ADSEL4_Pos (24UL) /*!< ADC SL4CFG: ADSEL4 (Bit 24) */ +#define ADC_SL4CFG_ADSEL4_Msk (0x7000000UL) /*!< ADC SL4CFG: ADSEL4 (Bitfield-Mask: 0x07) */ +#define ADC_SL4CFG_PRMODE4_Pos (16UL) /*!< ADC SL4CFG: PRMODE4 (Bit 16) */ +#define ADC_SL4CFG_PRMODE4_Msk (0x30000UL) /*!< ADC SL4CFG: PRMODE4 (Bitfield-Mask: 0x03) */ +#define ADC_SL4CFG_CHSEL4_Pos (8UL) /*!< ADC SL4CFG: CHSEL4 (Bit 8) */ +#define ADC_SL4CFG_CHSEL4_Msk (0xf00UL) /*!< ADC SL4CFG: CHSEL4 (Bitfield-Mask: 0x0f) */ +#define ADC_SL4CFG_WCEN4_Pos (1UL) /*!< ADC SL4CFG: WCEN4 (Bit 1) */ +#define ADC_SL4CFG_WCEN4_Msk (0x2UL) /*!< ADC SL4CFG: WCEN4 (Bitfield-Mask: 0x01) */ +#define ADC_SL4CFG_SLEN4_Pos (0UL) /*!< ADC SL4CFG: SLEN4 (Bit 0) */ +#define ADC_SL4CFG_SLEN4_Msk (0x1UL) /*!< ADC SL4CFG: SLEN4 (Bitfield-Mask: 0x01) */ +/* ======================================================== SL5CFG ========================================================= */ +#define ADC_SL5CFG_ADSEL5_Pos (24UL) /*!< ADC SL5CFG: ADSEL5 (Bit 24) */ +#define ADC_SL5CFG_ADSEL5_Msk (0x7000000UL) /*!< ADC SL5CFG: ADSEL5 (Bitfield-Mask: 0x07) */ +#define ADC_SL5CFG_PRMODE5_Pos (16UL) /*!< ADC SL5CFG: PRMODE5 (Bit 16) */ +#define ADC_SL5CFG_PRMODE5_Msk (0x30000UL) /*!< ADC SL5CFG: PRMODE5 (Bitfield-Mask: 0x03) */ +#define ADC_SL5CFG_CHSEL5_Pos (8UL) /*!< ADC SL5CFG: CHSEL5 (Bit 8) */ +#define ADC_SL5CFG_CHSEL5_Msk (0xf00UL) /*!< ADC SL5CFG: CHSEL5 (Bitfield-Mask: 0x0f) */ +#define ADC_SL5CFG_WCEN5_Pos (1UL) /*!< ADC SL5CFG: WCEN5 (Bit 1) */ +#define ADC_SL5CFG_WCEN5_Msk (0x2UL) /*!< ADC SL5CFG: WCEN5 (Bitfield-Mask: 0x01) */ +#define ADC_SL5CFG_SLEN5_Pos (0UL) /*!< ADC SL5CFG: SLEN5 (Bit 0) */ +#define ADC_SL5CFG_SLEN5_Msk (0x1UL) /*!< ADC SL5CFG: SLEN5 (Bitfield-Mask: 0x01) */ +/* ======================================================== SL6CFG ========================================================= */ +#define ADC_SL6CFG_ADSEL6_Pos (24UL) /*!< ADC SL6CFG: ADSEL6 (Bit 24) */ +#define ADC_SL6CFG_ADSEL6_Msk (0x7000000UL) /*!< ADC SL6CFG: ADSEL6 (Bitfield-Mask: 0x07) */ +#define ADC_SL6CFG_PRMODE6_Pos (16UL) /*!< ADC SL6CFG: PRMODE6 (Bit 16) */ +#define ADC_SL6CFG_PRMODE6_Msk (0x30000UL) /*!< ADC SL6CFG: PRMODE6 (Bitfield-Mask: 0x03) */ +#define ADC_SL6CFG_CHSEL6_Pos (8UL) /*!< ADC SL6CFG: CHSEL6 (Bit 8) */ +#define ADC_SL6CFG_CHSEL6_Msk (0xf00UL) /*!< ADC SL6CFG: CHSEL6 (Bitfield-Mask: 0x0f) */ +#define ADC_SL6CFG_WCEN6_Pos (1UL) /*!< ADC SL6CFG: WCEN6 (Bit 1) */ +#define ADC_SL6CFG_WCEN6_Msk (0x2UL) /*!< ADC SL6CFG: WCEN6 (Bitfield-Mask: 0x01) */ +#define ADC_SL6CFG_SLEN6_Pos (0UL) /*!< ADC SL6CFG: SLEN6 (Bit 0) */ +#define ADC_SL6CFG_SLEN6_Msk (0x1UL) /*!< ADC SL6CFG: SLEN6 (Bitfield-Mask: 0x01) */ +/* ======================================================== SL7CFG ========================================================= */ +#define ADC_SL7CFG_ADSEL7_Pos (24UL) /*!< ADC SL7CFG: ADSEL7 (Bit 24) */ +#define ADC_SL7CFG_ADSEL7_Msk (0x7000000UL) /*!< ADC SL7CFG: ADSEL7 (Bitfield-Mask: 0x07) */ +#define ADC_SL7CFG_PRMODE7_Pos (16UL) /*!< ADC SL7CFG: PRMODE7 (Bit 16) */ +#define ADC_SL7CFG_PRMODE7_Msk (0x30000UL) /*!< ADC SL7CFG: PRMODE7 (Bitfield-Mask: 0x03) */ +#define ADC_SL7CFG_CHSEL7_Pos (8UL) /*!< ADC SL7CFG: CHSEL7 (Bit 8) */ +#define ADC_SL7CFG_CHSEL7_Msk (0xf00UL) /*!< ADC SL7CFG: CHSEL7 (Bitfield-Mask: 0x0f) */ +#define ADC_SL7CFG_WCEN7_Pos (1UL) /*!< ADC SL7CFG: WCEN7 (Bit 1) */ +#define ADC_SL7CFG_WCEN7_Msk (0x2UL) /*!< ADC SL7CFG: WCEN7 (Bitfield-Mask: 0x01) */ +#define ADC_SL7CFG_SLEN7_Pos (0UL) /*!< ADC SL7CFG: SLEN7 (Bit 0) */ +#define ADC_SL7CFG_SLEN7_Msk (0x1UL) /*!< ADC SL7CFG: SLEN7 (Bitfield-Mask: 0x01) */ +/* ========================================================= WULIM ========================================================= */ +#define ADC_WULIM_ULIM_Pos (0UL) /*!< ADC WULIM: ULIM (Bit 0) */ +#define ADC_WULIM_ULIM_Msk (0xfffffUL) /*!< ADC WULIM: ULIM (Bitfield-Mask: 0xfffff) */ +/* ========================================================= WLLIM ========================================================= */ +#define ADC_WLLIM_LLIM_Pos (0UL) /*!< ADC WLLIM: LLIM (Bit 0) */ +#define ADC_WLLIM_LLIM_Msk (0xfffffUL) /*!< ADC WLLIM: LLIM (Bitfield-Mask: 0xfffff) */ +/* ======================================================== SCWLIM ========================================================= */ +#define ADC_SCWLIM_SCWLIMEN_Pos (0UL) /*!< ADC SCWLIM: SCWLIMEN (Bit 0) */ +#define ADC_SCWLIM_SCWLIMEN_Msk (0x1UL) /*!< ADC SCWLIM: SCWLIMEN (Bitfield-Mask: 0x01) */ +/* ========================================================= FIFO ========================================================== */ +#define ADC_FIFO_RSVD_Pos (31UL) /*!< ADC FIFO: RSVD (Bit 31) */ +#define ADC_FIFO_RSVD_Msk (0x80000000UL) /*!< ADC FIFO: RSVD (Bitfield-Mask: 0x01) */ +#define ADC_FIFO_SLOTNUM_Pos (28UL) /*!< ADC FIFO: SLOTNUM (Bit 28) */ +#define ADC_FIFO_SLOTNUM_Msk (0x70000000UL) /*!< ADC FIFO: SLOTNUM (Bitfield-Mask: 0x07) */ +#define ADC_FIFO_COUNT_Pos (20UL) /*!< ADC FIFO: COUNT (Bit 20) */ +#define ADC_FIFO_COUNT_Msk (0xff00000UL) /*!< ADC FIFO: COUNT (Bitfield-Mask: 0xff) */ +#define ADC_FIFO_DATA_Pos (0UL) /*!< ADC FIFO: DATA (Bit 0) */ +#define ADC_FIFO_DATA_Msk (0xfffffUL) /*!< ADC FIFO: DATA (Bitfield-Mask: 0xfffff) */ +/* ======================================================== FIFOPR ========================================================= */ +#define ADC_FIFOPR_RSVDPR_Pos (31UL) /*!< ADC FIFOPR: RSVDPR (Bit 31) */ +#define ADC_FIFOPR_RSVDPR_Msk (0x80000000UL) /*!< ADC FIFOPR: RSVDPR (Bitfield-Mask: 0x01) */ +#define ADC_FIFOPR_SLOTNUMPR_Pos (28UL) /*!< ADC FIFOPR: SLOTNUMPR (Bit 28) */ +#define ADC_FIFOPR_SLOTNUMPR_Msk (0x70000000UL) /*!< ADC FIFOPR: SLOTNUMPR (Bitfield-Mask: 0x07) */ +#define ADC_FIFOPR_COUNT_Pos (20UL) /*!< ADC FIFOPR: COUNT (Bit 20) */ +#define ADC_FIFOPR_COUNT_Msk (0xff00000UL) /*!< ADC FIFOPR: COUNT (Bitfield-Mask: 0xff) */ +#define ADC_FIFOPR_DATA_Pos (0UL) /*!< ADC FIFOPR: DATA (Bit 0) */ +#define ADC_FIFOPR_DATA_Msk (0xfffffUL) /*!< ADC FIFOPR: DATA (Bitfield-Mask: 0xfffff) */ +/* ========================================================= INTEN ========================================================= */ +#define ADC_INTEN_DERR_Pos (7UL) /*!< ADC INTEN: DERR (Bit 7) */ +#define ADC_INTEN_DERR_Msk (0x80UL) /*!< ADC INTEN: DERR (Bitfield-Mask: 0x01) */ +#define ADC_INTEN_DCMP_Pos (6UL) /*!< ADC INTEN: DCMP (Bit 6) */ +#define ADC_INTEN_DCMP_Msk (0x40UL) /*!< ADC INTEN: DCMP (Bitfield-Mask: 0x01) */ +#define ADC_INTEN_WCINC_Pos (5UL) /*!< ADC INTEN: WCINC (Bit 5) */ +#define ADC_INTEN_WCINC_Msk (0x20UL) /*!< ADC INTEN: WCINC (Bitfield-Mask: 0x01) */ +#define ADC_INTEN_WCEXC_Pos (4UL) /*!< ADC INTEN: WCEXC (Bit 4) */ +#define ADC_INTEN_WCEXC_Msk (0x10UL) /*!< ADC INTEN: WCEXC (Bitfield-Mask: 0x01) */ +#define ADC_INTEN_FIFOOVR2_Pos (3UL) /*!< ADC INTEN: FIFOOVR2 (Bit 3) */ +#define ADC_INTEN_FIFOOVR2_Msk (0x8UL) /*!< ADC INTEN: FIFOOVR2 (Bitfield-Mask: 0x01) */ +#define ADC_INTEN_FIFOOVR1_Pos (2UL) /*!< ADC INTEN: FIFOOVR1 (Bit 2) */ +#define ADC_INTEN_FIFOOVR1_Msk (0x4UL) /*!< ADC INTEN: FIFOOVR1 (Bitfield-Mask: 0x01) */ +#define ADC_INTEN_SCNCMP_Pos (1UL) /*!< ADC INTEN: SCNCMP (Bit 1) */ +#define ADC_INTEN_SCNCMP_Msk (0x2UL) /*!< ADC INTEN: SCNCMP (Bitfield-Mask: 0x01) */ +#define ADC_INTEN_CNVCMP_Pos (0UL) /*!< ADC INTEN: CNVCMP (Bit 0) */ +#define ADC_INTEN_CNVCMP_Msk (0x1UL) /*!< ADC INTEN: CNVCMP (Bitfield-Mask: 0x01) */ +/* ======================================================== INTSTAT ======================================================== */ +#define ADC_INTSTAT_DERR_Pos (7UL) /*!< ADC INTSTAT: DERR (Bit 7) */ +#define ADC_INTSTAT_DERR_Msk (0x80UL) /*!< ADC INTSTAT: DERR (Bitfield-Mask: 0x01) */ +#define ADC_INTSTAT_DCMP_Pos (6UL) /*!< ADC INTSTAT: DCMP (Bit 6) */ +#define ADC_INTSTAT_DCMP_Msk (0x40UL) /*!< ADC INTSTAT: DCMP (Bitfield-Mask: 0x01) */ +#define ADC_INTSTAT_WCINC_Pos (5UL) /*!< ADC INTSTAT: WCINC (Bit 5) */ +#define ADC_INTSTAT_WCINC_Msk (0x20UL) /*!< ADC INTSTAT: WCINC (Bitfield-Mask: 0x01) */ +#define ADC_INTSTAT_WCEXC_Pos (4UL) /*!< ADC INTSTAT: WCEXC (Bit 4) */ +#define ADC_INTSTAT_WCEXC_Msk (0x10UL) /*!< ADC INTSTAT: WCEXC (Bitfield-Mask: 0x01) */ +#define ADC_INTSTAT_FIFOOVR2_Pos (3UL) /*!< ADC INTSTAT: FIFOOVR2 (Bit 3) */ +#define ADC_INTSTAT_FIFOOVR2_Msk (0x8UL) /*!< ADC INTSTAT: FIFOOVR2 (Bitfield-Mask: 0x01) */ +#define ADC_INTSTAT_FIFOOVR1_Pos (2UL) /*!< ADC INTSTAT: FIFOOVR1 (Bit 2) */ +#define ADC_INTSTAT_FIFOOVR1_Msk (0x4UL) /*!< ADC INTSTAT: FIFOOVR1 (Bitfield-Mask: 0x01) */ +#define ADC_INTSTAT_SCNCMP_Pos (1UL) /*!< ADC INTSTAT: SCNCMP (Bit 1) */ +#define ADC_INTSTAT_SCNCMP_Msk (0x2UL) /*!< ADC INTSTAT: SCNCMP (Bitfield-Mask: 0x01) */ +#define ADC_INTSTAT_CNVCMP_Pos (0UL) /*!< ADC INTSTAT: CNVCMP (Bit 0) */ +#define ADC_INTSTAT_CNVCMP_Msk (0x1UL) /*!< ADC INTSTAT: CNVCMP (Bitfield-Mask: 0x01) */ +/* ======================================================== INTCLR ========================================================= */ +#define ADC_INTCLR_DERR_Pos (7UL) /*!< ADC INTCLR: DERR (Bit 7) */ +#define ADC_INTCLR_DERR_Msk (0x80UL) /*!< ADC INTCLR: DERR (Bitfield-Mask: 0x01) */ +#define ADC_INTCLR_DCMP_Pos (6UL) /*!< ADC INTCLR: DCMP (Bit 6) */ +#define ADC_INTCLR_DCMP_Msk (0x40UL) /*!< ADC INTCLR: DCMP (Bitfield-Mask: 0x01) */ +#define ADC_INTCLR_WCINC_Pos (5UL) /*!< ADC INTCLR: WCINC (Bit 5) */ +#define ADC_INTCLR_WCINC_Msk (0x20UL) /*!< ADC INTCLR: WCINC (Bitfield-Mask: 0x01) */ +#define ADC_INTCLR_WCEXC_Pos (4UL) /*!< ADC INTCLR: WCEXC (Bit 4) */ +#define ADC_INTCLR_WCEXC_Msk (0x10UL) /*!< ADC INTCLR: WCEXC (Bitfield-Mask: 0x01) */ +#define ADC_INTCLR_FIFOOVR2_Pos (3UL) /*!< ADC INTCLR: FIFOOVR2 (Bit 3) */ +#define ADC_INTCLR_FIFOOVR2_Msk (0x8UL) /*!< ADC INTCLR: FIFOOVR2 (Bitfield-Mask: 0x01) */ +#define ADC_INTCLR_FIFOOVR1_Pos (2UL) /*!< ADC INTCLR: FIFOOVR1 (Bit 2) */ +#define ADC_INTCLR_FIFOOVR1_Msk (0x4UL) /*!< ADC INTCLR: FIFOOVR1 (Bitfield-Mask: 0x01) */ +#define ADC_INTCLR_SCNCMP_Pos (1UL) /*!< ADC INTCLR: SCNCMP (Bit 1) */ +#define ADC_INTCLR_SCNCMP_Msk (0x2UL) /*!< ADC INTCLR: SCNCMP (Bitfield-Mask: 0x01) */ +#define ADC_INTCLR_CNVCMP_Pos (0UL) /*!< ADC INTCLR: CNVCMP (Bit 0) */ +#define ADC_INTCLR_CNVCMP_Msk (0x1UL) /*!< ADC INTCLR: CNVCMP (Bitfield-Mask: 0x01) */ +/* ======================================================== INTSET ========================================================= */ +#define ADC_INTSET_DERR_Pos (7UL) /*!< ADC INTSET: DERR (Bit 7) */ +#define ADC_INTSET_DERR_Msk (0x80UL) /*!< ADC INTSET: DERR (Bitfield-Mask: 0x01) */ +#define ADC_INTSET_DCMP_Pos (6UL) /*!< ADC INTSET: DCMP (Bit 6) */ +#define ADC_INTSET_DCMP_Msk (0x40UL) /*!< ADC INTSET: DCMP (Bitfield-Mask: 0x01) */ +#define ADC_INTSET_WCINC_Pos (5UL) /*!< ADC INTSET: WCINC (Bit 5) */ +#define ADC_INTSET_WCINC_Msk (0x20UL) /*!< ADC INTSET: WCINC (Bitfield-Mask: 0x01) */ +#define ADC_INTSET_WCEXC_Pos (4UL) /*!< ADC INTSET: WCEXC (Bit 4) */ +#define ADC_INTSET_WCEXC_Msk (0x10UL) /*!< ADC INTSET: WCEXC (Bitfield-Mask: 0x01) */ +#define ADC_INTSET_FIFOOVR2_Pos (3UL) /*!< ADC INTSET: FIFOOVR2 (Bit 3) */ +#define ADC_INTSET_FIFOOVR2_Msk (0x8UL) /*!< ADC INTSET: FIFOOVR2 (Bitfield-Mask: 0x01) */ +#define ADC_INTSET_FIFOOVR1_Pos (2UL) /*!< ADC INTSET: FIFOOVR1 (Bit 2) */ +#define ADC_INTSET_FIFOOVR1_Msk (0x4UL) /*!< ADC INTSET: FIFOOVR1 (Bitfield-Mask: 0x01) */ +#define ADC_INTSET_SCNCMP_Pos (1UL) /*!< ADC INTSET: SCNCMP (Bit 1) */ +#define ADC_INTSET_SCNCMP_Msk (0x2UL) /*!< ADC INTSET: SCNCMP (Bitfield-Mask: 0x01) */ +#define ADC_INTSET_CNVCMP_Pos (0UL) /*!< ADC INTSET: CNVCMP (Bit 0) */ +#define ADC_INTSET_CNVCMP_Msk (0x1UL) /*!< ADC INTSET: CNVCMP (Bitfield-Mask: 0x01) */ +/* ======================================================= DMATRIGEN ======================================================= */ +#define ADC_DMATRIGEN_DFIFOFULL_Pos (1UL) /*!< ADC DMATRIGEN: DFIFOFULL (Bit 1) */ +#define ADC_DMATRIGEN_DFIFOFULL_Msk (0x2UL) /*!< ADC DMATRIGEN: DFIFOFULL (Bitfield-Mask: 0x01) */ +#define ADC_DMATRIGEN_DFIFO75_Pos (0UL) /*!< ADC DMATRIGEN: DFIFO75 (Bit 0) */ +#define ADC_DMATRIGEN_DFIFO75_Msk (0x1UL) /*!< ADC DMATRIGEN: DFIFO75 (Bitfield-Mask: 0x01) */ +/* ====================================================== DMATRIGSTAT ====================================================== */ +#define ADC_DMATRIGSTAT_DFULLSTAT_Pos (1UL) /*!< ADC DMATRIGSTAT: DFULLSTAT (Bit 1) */ +#define ADC_DMATRIGSTAT_DFULLSTAT_Msk (0x2UL) /*!< ADC DMATRIGSTAT: DFULLSTAT (Bitfield-Mask: 0x01) */ +#define ADC_DMATRIGSTAT_D75STAT_Pos (0UL) /*!< ADC DMATRIGSTAT: D75STAT (Bit 0) */ +#define ADC_DMATRIGSTAT_D75STAT_Msk (0x1UL) /*!< ADC DMATRIGSTAT: D75STAT (Bitfield-Mask: 0x01) */ +/* ======================================================== DMACFG ========================================================= */ +#define ADC_DMACFG_DPWROFF_Pos (18UL) /*!< ADC DMACFG: DPWROFF (Bit 18) */ +#define ADC_DMACFG_DPWROFF_Msk (0x40000UL) /*!< ADC DMACFG: DPWROFF (Bitfield-Mask: 0x01) */ +#define ADC_DMACFG_DMAMSK_Pos (17UL) /*!< ADC DMACFG: DMAMSK (Bit 17) */ +#define ADC_DMACFG_DMAMSK_Msk (0x20000UL) /*!< ADC DMACFG: DMAMSK (Bitfield-Mask: 0x01) */ +#define ADC_DMACFG_DMAHONSTAT_Pos (16UL) /*!< ADC DMACFG: DMAHONSTAT (Bit 16) */ +#define ADC_DMACFG_DMAHONSTAT_Msk (0x10000UL) /*!< ADC DMACFG: DMAHONSTAT (Bitfield-Mask: 0x01) */ +#define ADC_DMACFG_DMADYNPRI_Pos (9UL) /*!< ADC DMACFG: DMADYNPRI (Bit 9) */ +#define ADC_DMACFG_DMADYNPRI_Msk (0x200UL) /*!< ADC DMACFG: DMADYNPRI (Bitfield-Mask: 0x01) */ +#define ADC_DMACFG_DMAPRI_Pos (8UL) /*!< ADC DMACFG: DMAPRI (Bit 8) */ +#define ADC_DMACFG_DMAPRI_Msk (0x100UL) /*!< ADC DMACFG: DMAPRI (Bitfield-Mask: 0x01) */ +#define ADC_DMACFG_DMADIR_Pos (2UL) /*!< ADC DMACFG: DMADIR (Bit 2) */ +#define ADC_DMACFG_DMADIR_Msk (0x4UL) /*!< ADC DMACFG: DMADIR (Bitfield-Mask: 0x01) */ +#define ADC_DMACFG_DMAEN_Pos (0UL) /*!< ADC DMACFG: DMAEN (Bit 0) */ +#define ADC_DMACFG_DMAEN_Msk (0x1UL) /*!< ADC DMACFG: DMAEN (Bitfield-Mask: 0x01) */ +/* ====================================================== DMATOTCOUNT ====================================================== */ +#define ADC_DMATOTCOUNT_TOTCOUNT_Pos (2UL) /*!< ADC DMATOTCOUNT: TOTCOUNT (Bit 2) */ +#define ADC_DMATOTCOUNT_TOTCOUNT_Msk (0x3fffcUL) /*!< ADC DMATOTCOUNT: TOTCOUNT (Bitfield-Mask: 0xffff) */ +/* ====================================================== DMATARGADDR ====================================================== */ +#define ADC_DMATARGADDR_UTARGADDR_Pos (19UL) /*!< ADC DMATARGADDR: UTARGADDR (Bit 19) */ +#define ADC_DMATARGADDR_UTARGADDR_Msk (0xfff80000UL) /*!< ADC DMATARGADDR: UTARGADDR (Bitfield-Mask: 0x1fff) */ +#define ADC_DMATARGADDR_LTARGADDR_Pos (0UL) /*!< ADC DMATARGADDR: LTARGADDR (Bit 0) */ +#define ADC_DMATARGADDR_LTARGADDR_Msk (0x7ffffUL) /*!< ADC DMATARGADDR: LTARGADDR (Bitfield-Mask: 0x7ffff) */ +/* ======================================================== DMASTAT ======================================================== */ +#define ADC_DMASTAT_DMAERR_Pos (2UL) /*!< ADC DMASTAT: DMAERR (Bit 2) */ +#define ADC_DMASTAT_DMAERR_Msk (0x4UL) /*!< ADC DMASTAT: DMAERR (Bitfield-Mask: 0x01) */ +#define ADC_DMASTAT_DMACPL_Pos (1UL) /*!< ADC DMASTAT: DMACPL (Bit 1) */ +#define ADC_DMASTAT_DMACPL_Msk (0x2UL) /*!< ADC DMASTAT: DMACPL (Bitfield-Mask: 0x01) */ +#define ADC_DMASTAT_DMATIP_Pos (0UL) /*!< ADC DMASTAT: DMATIP (Bit 0) */ +#define ADC_DMASTAT_DMATIP_Msk (0x1UL) /*!< ADC DMASTAT: DMATIP (Bitfield-Mask: 0x01) */ + + +/* =========================================================================================================================== */ +/* ================ APBDMA ================ */ +/* =========================================================================================================================== */ + +/* ======================================================== BBVALUE ======================================================== */ +#define APBDMA_BBVALUE_PIN_Pos (16UL) /*!< APBDMA BBVALUE: PIN (Bit 16) */ +#define APBDMA_BBVALUE_PIN_Msk (0xff0000UL) /*!< APBDMA BBVALUE: PIN (Bitfield-Mask: 0xff) */ +#define APBDMA_BBVALUE_DATAOUT_Pos (0UL) /*!< APBDMA BBVALUE: DATAOUT (Bit 0) */ +#define APBDMA_BBVALUE_DATAOUT_Msk (0xffUL) /*!< APBDMA BBVALUE: DATAOUT (Bitfield-Mask: 0xff) */ +/* ====================================================== BBSETCLEAR ======================================================= */ +#define APBDMA_BBSETCLEAR_CLEAR_Pos (16UL) /*!< APBDMA BBSETCLEAR: CLEAR (Bit 16) */ +#define APBDMA_BBSETCLEAR_CLEAR_Msk (0xff0000UL) /*!< APBDMA BBSETCLEAR: CLEAR (Bitfield-Mask: 0xff) */ +#define APBDMA_BBSETCLEAR_SET_Pos (0UL) /*!< APBDMA BBSETCLEAR: SET (Bit 0) */ +#define APBDMA_BBSETCLEAR_SET_Msk (0xffUL) /*!< APBDMA BBSETCLEAR: SET (Bitfield-Mask: 0xff) */ +/* ======================================================== BBINPUT ======================================================== */ +#define APBDMA_BBINPUT_DATAIN_Pos (0UL) /*!< APBDMA BBINPUT: DATAIN (Bit 0) */ +#define APBDMA_BBINPUT_DATAIN_Msk (0xffUL) /*!< APBDMA BBINPUT: DATAIN (Bitfield-Mask: 0xff) */ +/* ======================================================= DEBUGDATA ======================================================= */ +#define APBDMA_DEBUGDATA_DEBUGDATA_Pos (0UL) /*!< APBDMA DEBUGDATA: DEBUGDATA (Bit 0) */ +#define APBDMA_DEBUGDATA_DEBUGDATA_Msk (0xffffffffUL) /*!< APBDMA DEBUGDATA: DEBUGDATA (Bitfield-Mask: 0xffffffff) */ +/* ========================================================= DEBUG ========================================================= */ +#define APBDMA_DEBUG_DEBUGEN_Pos (0UL) /*!< APBDMA DEBUG: DEBUGEN (Bit 0) */ +#define APBDMA_DEBUG_DEBUGEN_Msk (0xfUL) /*!< APBDMA DEBUG: DEBUGEN (Bitfield-Mask: 0x0f) */ + + +/* =========================================================================================================================== */ +/* ================ BLEIF ================ */ +/* =========================================================================================================================== */ + +/* ========================================================= FIFO ========================================================== */ +#define BLEIF_FIFO_FIFO_Pos (0UL) /*!< BLEIF FIFO: FIFO (Bit 0) */ +#define BLEIF_FIFO_FIFO_Msk (0xffffffffUL) /*!< BLEIF FIFO: FIFO (Bitfield-Mask: 0xffffffff) */ +/* ======================================================== FIFOPTR ======================================================== */ +#define BLEIF_FIFOPTR_FIFO1REM_Pos (24UL) /*!< BLEIF FIFOPTR: FIFO1REM (Bit 24) */ +#define BLEIF_FIFOPTR_FIFO1REM_Msk (0xff000000UL) /*!< BLEIF FIFOPTR: FIFO1REM (Bitfield-Mask: 0xff) */ +#define BLEIF_FIFOPTR_FIFO1SIZ_Pos (16UL) /*!< BLEIF FIFOPTR: FIFO1SIZ (Bit 16) */ +#define BLEIF_FIFOPTR_FIFO1SIZ_Msk (0xff0000UL) /*!< BLEIF FIFOPTR: FIFO1SIZ (Bitfield-Mask: 0xff) */ +#define BLEIF_FIFOPTR_FIFO0REM_Pos (8UL) /*!< BLEIF FIFOPTR: FIFO0REM (Bit 8) */ +#define BLEIF_FIFOPTR_FIFO0REM_Msk (0xff00UL) /*!< BLEIF FIFOPTR: FIFO0REM (Bitfield-Mask: 0xff) */ +#define BLEIF_FIFOPTR_FIFO0SIZ_Pos (0UL) /*!< BLEIF FIFOPTR: FIFO0SIZ (Bit 0) */ +#define BLEIF_FIFOPTR_FIFO0SIZ_Msk (0xffUL) /*!< BLEIF FIFOPTR: FIFO0SIZ (Bitfield-Mask: 0xff) */ +/* ======================================================== FIFOTHR ======================================================== */ +#define BLEIF_FIFOTHR_FIFOWTHR_Pos (8UL) /*!< BLEIF FIFOTHR: FIFOWTHR (Bit 8) */ +#define BLEIF_FIFOTHR_FIFOWTHR_Msk (0x3f00UL) /*!< BLEIF FIFOTHR: FIFOWTHR (Bitfield-Mask: 0x3f) */ +#define BLEIF_FIFOTHR_FIFORTHR_Pos (0UL) /*!< BLEIF FIFOTHR: FIFORTHR (Bit 0) */ +#define BLEIF_FIFOTHR_FIFORTHR_Msk (0x3fUL) /*!< BLEIF FIFOTHR: FIFORTHR (Bitfield-Mask: 0x3f) */ +/* ======================================================== FIFOPOP ======================================================== */ +#define BLEIF_FIFOPOP_FIFODOUT_Pos (0UL) /*!< BLEIF FIFOPOP: FIFODOUT (Bit 0) */ +#define BLEIF_FIFOPOP_FIFODOUT_Msk (0xffffffffUL) /*!< BLEIF FIFOPOP: FIFODOUT (Bitfield-Mask: 0xffffffff) */ +/* ======================================================= FIFOPUSH ======================================================== */ +#define BLEIF_FIFOPUSH_FIFODIN_Pos (0UL) /*!< BLEIF FIFOPUSH: FIFODIN (Bit 0) */ +#define BLEIF_FIFOPUSH_FIFODIN_Msk (0xffffffffUL) /*!< BLEIF FIFOPUSH: FIFODIN (Bitfield-Mask: 0xffffffff) */ +/* ======================================================= FIFOCTRL ======================================================== */ +#define BLEIF_FIFOCTRL_FIFORSTN_Pos (1UL) /*!< BLEIF FIFOCTRL: FIFORSTN (Bit 1) */ +#define BLEIF_FIFOCTRL_FIFORSTN_Msk (0x2UL) /*!< BLEIF FIFOCTRL: FIFORSTN (Bitfield-Mask: 0x01) */ +#define BLEIF_FIFOCTRL_POPWR_Pos (0UL) /*!< BLEIF FIFOCTRL: POPWR (Bit 0) */ +#define BLEIF_FIFOCTRL_POPWR_Msk (0x1UL) /*!< BLEIF FIFOCTRL: POPWR (Bitfield-Mask: 0x01) */ +/* ======================================================== FIFOLOC ======================================================== */ +#define BLEIF_FIFOLOC_FIFORPTR_Pos (8UL) /*!< BLEIF FIFOLOC: FIFORPTR (Bit 8) */ +#define BLEIF_FIFOLOC_FIFORPTR_Msk (0xf00UL) /*!< BLEIF FIFOLOC: FIFORPTR (Bitfield-Mask: 0x0f) */ +#define BLEIF_FIFOLOC_FIFOWPTR_Pos (0UL) /*!< BLEIF FIFOLOC: FIFOWPTR (Bit 0) */ +#define BLEIF_FIFOLOC_FIFOWPTR_Msk (0xfUL) /*!< BLEIF FIFOLOC: FIFOWPTR (Bitfield-Mask: 0x0f) */ +/* ======================================================== CLKCFG ========================================================= */ +#define BLEIF_CLKCFG_DIV3_Pos (12UL) /*!< BLEIF CLKCFG: DIV3 (Bit 12) */ +#define BLEIF_CLKCFG_DIV3_Msk (0x1000UL) /*!< BLEIF CLKCFG: DIV3 (Bitfield-Mask: 0x01) */ +#define BLEIF_CLKCFG_CLK32KEN_Pos (11UL) /*!< BLEIF CLKCFG: CLK32KEN (Bit 11) */ +#define BLEIF_CLKCFG_CLK32KEN_Msk (0x800UL) /*!< BLEIF CLKCFG: CLK32KEN (Bitfield-Mask: 0x01) */ +#define BLEIF_CLKCFG_FSEL_Pos (8UL) /*!< BLEIF CLKCFG: FSEL (Bit 8) */ +#define BLEIF_CLKCFG_FSEL_Msk (0x700UL) /*!< BLEIF CLKCFG: FSEL (Bitfield-Mask: 0x07) */ +#define BLEIF_CLKCFG_IOCLKEN_Pos (0UL) /*!< BLEIF CLKCFG: IOCLKEN (Bit 0) */ +#define BLEIF_CLKCFG_IOCLKEN_Msk (0x1UL) /*!< BLEIF CLKCFG: IOCLKEN (Bitfield-Mask: 0x01) */ +/* ========================================================== CMD ========================================================== */ +#define BLEIF_CMD_OFFSETLO_Pos (24UL) /*!< BLEIF CMD: OFFSETLO (Bit 24) */ +#define BLEIF_CMD_OFFSETLO_Msk (0xff000000UL) /*!< BLEIF CMD: OFFSETLO (Bitfield-Mask: 0xff) */ +#define BLEIF_CMD_CMDSEL_Pos (20UL) /*!< BLEIF CMD: CMDSEL (Bit 20) */ +#define BLEIF_CMD_CMDSEL_Msk (0x300000UL) /*!< BLEIF CMD: CMDSEL (Bitfield-Mask: 0x03) */ +#define BLEIF_CMD_TSIZE_Pos (8UL) /*!< BLEIF CMD: TSIZE (Bit 8) */ +#define BLEIF_CMD_TSIZE_Msk (0xfff00UL) /*!< BLEIF CMD: TSIZE (Bitfield-Mask: 0xfff) */ +#define BLEIF_CMD_CONT_Pos (7UL) /*!< BLEIF CMD: CONT (Bit 7) */ +#define BLEIF_CMD_CONT_Msk (0x80UL) /*!< BLEIF CMD: CONT (Bitfield-Mask: 0x01) */ +#define BLEIF_CMD_OFFSETCNT_Pos (5UL) /*!< BLEIF CMD: OFFSETCNT (Bit 5) */ +#define BLEIF_CMD_OFFSETCNT_Msk (0x60UL) /*!< BLEIF CMD: OFFSETCNT (Bitfield-Mask: 0x03) */ +#define BLEIF_CMD_CMD_Pos (0UL) /*!< BLEIF CMD: CMD (Bit 0) */ +#define BLEIF_CMD_CMD_Msk (0x1fUL) /*!< BLEIF CMD: CMD (Bitfield-Mask: 0x1f) */ +/* ======================================================== CMDRPT ========================================================= */ +#define BLEIF_CMDRPT_CMDRPT_Pos (0UL) /*!< BLEIF CMDRPT: CMDRPT (Bit 0) */ +#define BLEIF_CMDRPT_CMDRPT_Msk (0x1fUL) /*!< BLEIF CMDRPT: CMDRPT (Bitfield-Mask: 0x1f) */ +/* ======================================================= OFFSETHI ======================================================== */ +#define BLEIF_OFFSETHI_OFFSETHI_Pos (0UL) /*!< BLEIF OFFSETHI: OFFSETHI (Bit 0) */ +#define BLEIF_OFFSETHI_OFFSETHI_Msk (0xffffUL) /*!< BLEIF OFFSETHI: OFFSETHI (Bitfield-Mask: 0xffff) */ +/* ======================================================== CMDSTAT ======================================================== */ +#define BLEIF_CMDSTAT_CTSIZE_Pos (8UL) /*!< BLEIF CMDSTAT: CTSIZE (Bit 8) */ +#define BLEIF_CMDSTAT_CTSIZE_Msk (0xfff00UL) /*!< BLEIF CMDSTAT: CTSIZE (Bitfield-Mask: 0xfff) */ +#define BLEIF_CMDSTAT_CMDSTAT_Pos (5UL) /*!< BLEIF CMDSTAT: CMDSTAT (Bit 5) */ +#define BLEIF_CMDSTAT_CMDSTAT_Msk (0xe0UL) /*!< BLEIF CMDSTAT: CMDSTAT (Bitfield-Mask: 0x07) */ +#define BLEIF_CMDSTAT_CCMD_Pos (0UL) /*!< BLEIF CMDSTAT: CCMD (Bit 0) */ +#define BLEIF_CMDSTAT_CCMD_Msk (0x1fUL) /*!< BLEIF CMDSTAT: CCMD (Bitfield-Mask: 0x1f) */ +/* ========================================================= INTEN ========================================================= */ +#define BLEIF_INTEN_B2MSHUTDN_Pos (16UL) /*!< BLEIF INTEN: B2MSHUTDN (Bit 16) */ +#define BLEIF_INTEN_B2MSHUTDN_Msk (0x10000UL) /*!< BLEIF INTEN: B2MSHUTDN (Bitfield-Mask: 0x01) */ +#define BLEIF_INTEN_B2MACTIVE_Pos (15UL) /*!< BLEIF INTEN: B2MACTIVE (Bit 15) */ +#define BLEIF_INTEN_B2MACTIVE_Msk (0x8000UL) /*!< BLEIF INTEN: B2MACTIVE (Bitfield-Mask: 0x01) */ +#define BLEIF_INTEN_B2MSLEEP_Pos (14UL) /*!< BLEIF INTEN: B2MSLEEP (Bit 14) */ +#define BLEIF_INTEN_B2MSLEEP_Msk (0x4000UL) /*!< BLEIF INTEN: B2MSLEEP (Bitfield-Mask: 0x01) */ +#define BLEIF_INTEN_CQERR_Pos (13UL) /*!< BLEIF INTEN: CQERR (Bit 13) */ +#define BLEIF_INTEN_CQERR_Msk (0x2000UL) /*!< BLEIF INTEN: CQERR (Bitfield-Mask: 0x01) */ +#define BLEIF_INTEN_CQUPD_Pos (12UL) /*!< BLEIF INTEN: CQUPD (Bit 12) */ +#define BLEIF_INTEN_CQUPD_Msk (0x1000UL) /*!< BLEIF INTEN: CQUPD (Bitfield-Mask: 0x01) */ +#define BLEIF_INTEN_CQPAUSED_Pos (11UL) /*!< BLEIF INTEN: CQPAUSED (Bit 11) */ +#define BLEIF_INTEN_CQPAUSED_Msk (0x800UL) /*!< BLEIF INTEN: CQPAUSED (Bitfield-Mask: 0x01) */ +#define BLEIF_INTEN_DERR_Pos (10UL) /*!< BLEIF INTEN: DERR (Bit 10) */ +#define BLEIF_INTEN_DERR_Msk (0x400UL) /*!< BLEIF INTEN: DERR (Bitfield-Mask: 0x01) */ +#define BLEIF_INTEN_DCMP_Pos (9UL) /*!< BLEIF INTEN: DCMP (Bit 9) */ +#define BLEIF_INTEN_DCMP_Msk (0x200UL) /*!< BLEIF INTEN: DCMP (Bitfield-Mask: 0x01) */ +#define BLEIF_INTEN_BLECSSTAT_Pos (8UL) /*!< BLEIF INTEN: BLECSSTAT (Bit 8) */ +#define BLEIF_INTEN_BLECSSTAT_Msk (0x100UL) /*!< BLEIF INTEN: BLECSSTAT (Bitfield-Mask: 0x01) */ +#define BLEIF_INTEN_BLECIRQ_Pos (7UL) /*!< BLEIF INTEN: BLECIRQ (Bit 7) */ +#define BLEIF_INTEN_BLECIRQ_Msk (0x80UL) /*!< BLEIF INTEN: BLECIRQ (Bitfield-Mask: 0x01) */ +#define BLEIF_INTEN_ICMD_Pos (6UL) /*!< BLEIF INTEN: ICMD (Bit 6) */ +#define BLEIF_INTEN_ICMD_Msk (0x40UL) /*!< BLEIF INTEN: ICMD (Bitfield-Mask: 0x01) */ +#define BLEIF_INTEN_IACC_Pos (5UL) /*!< BLEIF INTEN: IACC (Bit 5) */ +#define BLEIF_INTEN_IACC_Msk (0x20UL) /*!< BLEIF INTEN: IACC (Bitfield-Mask: 0x01) */ +#define BLEIF_INTEN_B2MST_Pos (4UL) /*!< BLEIF INTEN: B2MST (Bit 4) */ +#define BLEIF_INTEN_B2MST_Msk (0x10UL) /*!< BLEIF INTEN: B2MST (Bitfield-Mask: 0x01) */ +#define BLEIF_INTEN_FOVFL_Pos (3UL) /*!< BLEIF INTEN: FOVFL (Bit 3) */ +#define BLEIF_INTEN_FOVFL_Msk (0x8UL) /*!< BLEIF INTEN: FOVFL (Bitfield-Mask: 0x01) */ +#define BLEIF_INTEN_FUNDFL_Pos (2UL) /*!< BLEIF INTEN: FUNDFL (Bit 2) */ +#define BLEIF_INTEN_FUNDFL_Msk (0x4UL) /*!< BLEIF INTEN: FUNDFL (Bitfield-Mask: 0x01) */ +#define BLEIF_INTEN_THR_Pos (1UL) /*!< BLEIF INTEN: THR (Bit 1) */ +#define BLEIF_INTEN_THR_Msk (0x2UL) /*!< BLEIF INTEN: THR (Bitfield-Mask: 0x01) */ +#define BLEIF_INTEN_CMDCMP_Pos (0UL) /*!< BLEIF INTEN: CMDCMP (Bit 0) */ +#define BLEIF_INTEN_CMDCMP_Msk (0x1UL) /*!< BLEIF INTEN: CMDCMP (Bitfield-Mask: 0x01) */ +/* ======================================================== INTSTAT ======================================================== */ +#define BLEIF_INTSTAT_B2MSHUTDN_Pos (16UL) /*!< BLEIF INTSTAT: B2MSHUTDN (Bit 16) */ +#define BLEIF_INTSTAT_B2MSHUTDN_Msk (0x10000UL) /*!< BLEIF INTSTAT: B2MSHUTDN (Bitfield-Mask: 0x01) */ +#define BLEIF_INTSTAT_B2MACTIVE_Pos (15UL) /*!< BLEIF INTSTAT: B2MACTIVE (Bit 15) */ +#define BLEIF_INTSTAT_B2MACTIVE_Msk (0x8000UL) /*!< BLEIF INTSTAT: B2MACTIVE (Bitfield-Mask: 0x01) */ +#define BLEIF_INTSTAT_B2MSLEEP_Pos (14UL) /*!< BLEIF INTSTAT: B2MSLEEP (Bit 14) */ +#define BLEIF_INTSTAT_B2MSLEEP_Msk (0x4000UL) /*!< BLEIF INTSTAT: B2MSLEEP (Bitfield-Mask: 0x01) */ +#define BLEIF_INTSTAT_CQERR_Pos (13UL) /*!< BLEIF INTSTAT: CQERR (Bit 13) */ +#define BLEIF_INTSTAT_CQERR_Msk (0x2000UL) /*!< BLEIF INTSTAT: CQERR (Bitfield-Mask: 0x01) */ +#define BLEIF_INTSTAT_CQUPD_Pos (12UL) /*!< BLEIF INTSTAT: CQUPD (Bit 12) */ +#define BLEIF_INTSTAT_CQUPD_Msk (0x1000UL) /*!< BLEIF INTSTAT: CQUPD (Bitfield-Mask: 0x01) */ +#define BLEIF_INTSTAT_CQPAUSED_Pos (11UL) /*!< BLEIF INTSTAT: CQPAUSED (Bit 11) */ +#define BLEIF_INTSTAT_CQPAUSED_Msk (0x800UL) /*!< BLEIF INTSTAT: CQPAUSED (Bitfield-Mask: 0x01) */ +#define BLEIF_INTSTAT_DERR_Pos (10UL) /*!< BLEIF INTSTAT: DERR (Bit 10) */ +#define BLEIF_INTSTAT_DERR_Msk (0x400UL) /*!< BLEIF INTSTAT: DERR (Bitfield-Mask: 0x01) */ +#define BLEIF_INTSTAT_DCMP_Pos (9UL) /*!< BLEIF INTSTAT: DCMP (Bit 9) */ +#define BLEIF_INTSTAT_DCMP_Msk (0x200UL) /*!< BLEIF INTSTAT: DCMP (Bitfield-Mask: 0x01) */ +#define BLEIF_INTSTAT_BLECSSTAT_Pos (8UL) /*!< BLEIF INTSTAT: BLECSSTAT (Bit 8) */ +#define BLEIF_INTSTAT_BLECSSTAT_Msk (0x100UL) /*!< BLEIF INTSTAT: BLECSSTAT (Bitfield-Mask: 0x01) */ +#define BLEIF_INTSTAT_BLECIRQ_Pos (7UL) /*!< BLEIF INTSTAT: BLECIRQ (Bit 7) */ +#define BLEIF_INTSTAT_BLECIRQ_Msk (0x80UL) /*!< BLEIF INTSTAT: BLECIRQ (Bitfield-Mask: 0x01) */ +#define BLEIF_INTSTAT_ICMD_Pos (6UL) /*!< BLEIF INTSTAT: ICMD (Bit 6) */ +#define BLEIF_INTSTAT_ICMD_Msk (0x40UL) /*!< BLEIF INTSTAT: ICMD (Bitfield-Mask: 0x01) */ +#define BLEIF_INTSTAT_IACC_Pos (5UL) /*!< BLEIF INTSTAT: IACC (Bit 5) */ +#define BLEIF_INTSTAT_IACC_Msk (0x20UL) /*!< BLEIF INTSTAT: IACC (Bitfield-Mask: 0x01) */ +#define BLEIF_INTSTAT_B2MST_Pos (4UL) /*!< BLEIF INTSTAT: B2MST (Bit 4) */ +#define BLEIF_INTSTAT_B2MST_Msk (0x10UL) /*!< BLEIF INTSTAT: B2MST (Bitfield-Mask: 0x01) */ +#define BLEIF_INTSTAT_FOVFL_Pos (3UL) /*!< BLEIF INTSTAT: FOVFL (Bit 3) */ +#define BLEIF_INTSTAT_FOVFL_Msk (0x8UL) /*!< BLEIF INTSTAT: FOVFL (Bitfield-Mask: 0x01) */ +#define BLEIF_INTSTAT_FUNDFL_Pos (2UL) /*!< BLEIF INTSTAT: FUNDFL (Bit 2) */ +#define BLEIF_INTSTAT_FUNDFL_Msk (0x4UL) /*!< BLEIF INTSTAT: FUNDFL (Bitfield-Mask: 0x01) */ +#define BLEIF_INTSTAT_THR_Pos (1UL) /*!< BLEIF INTSTAT: THR (Bit 1) */ +#define BLEIF_INTSTAT_THR_Msk (0x2UL) /*!< BLEIF INTSTAT: THR (Bitfield-Mask: 0x01) */ +#define BLEIF_INTSTAT_CMDCMP_Pos (0UL) /*!< BLEIF INTSTAT: CMDCMP (Bit 0) */ +#define BLEIF_INTSTAT_CMDCMP_Msk (0x1UL) /*!< BLEIF INTSTAT: CMDCMP (Bitfield-Mask: 0x01) */ +/* ======================================================== INTCLR ========================================================= */ +#define BLEIF_INTCLR_B2MSHUTDN_Pos (16UL) /*!< BLEIF INTCLR: B2MSHUTDN (Bit 16) */ +#define BLEIF_INTCLR_B2MSHUTDN_Msk (0x10000UL) /*!< BLEIF INTCLR: B2MSHUTDN (Bitfield-Mask: 0x01) */ +#define BLEIF_INTCLR_B2MACTIVE_Pos (15UL) /*!< BLEIF INTCLR: B2MACTIVE (Bit 15) */ +#define BLEIF_INTCLR_B2MACTIVE_Msk (0x8000UL) /*!< BLEIF INTCLR: B2MACTIVE (Bitfield-Mask: 0x01) */ +#define BLEIF_INTCLR_B2MSLEEP_Pos (14UL) /*!< BLEIF INTCLR: B2MSLEEP (Bit 14) */ +#define BLEIF_INTCLR_B2MSLEEP_Msk (0x4000UL) /*!< BLEIF INTCLR: B2MSLEEP (Bitfield-Mask: 0x01) */ +#define BLEIF_INTCLR_CQERR_Pos (13UL) /*!< BLEIF INTCLR: CQERR (Bit 13) */ +#define BLEIF_INTCLR_CQERR_Msk (0x2000UL) /*!< BLEIF INTCLR: CQERR (Bitfield-Mask: 0x01) */ +#define BLEIF_INTCLR_CQUPD_Pos (12UL) /*!< BLEIF INTCLR: CQUPD (Bit 12) */ +#define BLEIF_INTCLR_CQUPD_Msk (0x1000UL) /*!< BLEIF INTCLR: CQUPD (Bitfield-Mask: 0x01) */ +#define BLEIF_INTCLR_CQPAUSED_Pos (11UL) /*!< BLEIF INTCLR: CQPAUSED (Bit 11) */ +#define BLEIF_INTCLR_CQPAUSED_Msk (0x800UL) /*!< BLEIF INTCLR: CQPAUSED (Bitfield-Mask: 0x01) */ +#define BLEIF_INTCLR_DERR_Pos (10UL) /*!< BLEIF INTCLR: DERR (Bit 10) */ +#define BLEIF_INTCLR_DERR_Msk (0x400UL) /*!< BLEIF INTCLR: DERR (Bitfield-Mask: 0x01) */ +#define BLEIF_INTCLR_DCMP_Pos (9UL) /*!< BLEIF INTCLR: DCMP (Bit 9) */ +#define BLEIF_INTCLR_DCMP_Msk (0x200UL) /*!< BLEIF INTCLR: DCMP (Bitfield-Mask: 0x01) */ +#define BLEIF_INTCLR_BLECSSTAT_Pos (8UL) /*!< BLEIF INTCLR: BLECSSTAT (Bit 8) */ +#define BLEIF_INTCLR_BLECSSTAT_Msk (0x100UL) /*!< BLEIF INTCLR: BLECSSTAT (Bitfield-Mask: 0x01) */ +#define BLEIF_INTCLR_BLECIRQ_Pos (7UL) /*!< BLEIF INTCLR: BLECIRQ (Bit 7) */ +#define BLEIF_INTCLR_BLECIRQ_Msk (0x80UL) /*!< BLEIF INTCLR: BLECIRQ (Bitfield-Mask: 0x01) */ +#define BLEIF_INTCLR_ICMD_Pos (6UL) /*!< BLEIF INTCLR: ICMD (Bit 6) */ +#define BLEIF_INTCLR_ICMD_Msk (0x40UL) /*!< BLEIF INTCLR: ICMD (Bitfield-Mask: 0x01) */ +#define BLEIF_INTCLR_IACC_Pos (5UL) /*!< BLEIF INTCLR: IACC (Bit 5) */ +#define BLEIF_INTCLR_IACC_Msk (0x20UL) /*!< BLEIF INTCLR: IACC (Bitfield-Mask: 0x01) */ +#define BLEIF_INTCLR_B2MST_Pos (4UL) /*!< BLEIF INTCLR: B2MST (Bit 4) */ +#define BLEIF_INTCLR_B2MST_Msk (0x10UL) /*!< BLEIF INTCLR: B2MST (Bitfield-Mask: 0x01) */ +#define BLEIF_INTCLR_FOVFL_Pos (3UL) /*!< BLEIF INTCLR: FOVFL (Bit 3) */ +#define BLEIF_INTCLR_FOVFL_Msk (0x8UL) /*!< BLEIF INTCLR: FOVFL (Bitfield-Mask: 0x01) */ +#define BLEIF_INTCLR_FUNDFL_Pos (2UL) /*!< BLEIF INTCLR: FUNDFL (Bit 2) */ +#define BLEIF_INTCLR_FUNDFL_Msk (0x4UL) /*!< BLEIF INTCLR: FUNDFL (Bitfield-Mask: 0x01) */ +#define BLEIF_INTCLR_THR_Pos (1UL) /*!< BLEIF INTCLR: THR (Bit 1) */ +#define BLEIF_INTCLR_THR_Msk (0x2UL) /*!< BLEIF INTCLR: THR (Bitfield-Mask: 0x01) */ +#define BLEIF_INTCLR_CMDCMP_Pos (0UL) /*!< BLEIF INTCLR: CMDCMP (Bit 0) */ +#define BLEIF_INTCLR_CMDCMP_Msk (0x1UL) /*!< BLEIF INTCLR: CMDCMP (Bitfield-Mask: 0x01) */ +/* ======================================================== INTSET ========================================================= */ +#define BLEIF_INTSET_B2MSHUTDN_Pos (16UL) /*!< BLEIF INTSET: B2MSHUTDN (Bit 16) */ +#define BLEIF_INTSET_B2MSHUTDN_Msk (0x10000UL) /*!< BLEIF INTSET: B2MSHUTDN (Bitfield-Mask: 0x01) */ +#define BLEIF_INTSET_B2MACTIVE_Pos (15UL) /*!< BLEIF INTSET: B2MACTIVE (Bit 15) */ +#define BLEIF_INTSET_B2MACTIVE_Msk (0x8000UL) /*!< BLEIF INTSET: B2MACTIVE (Bitfield-Mask: 0x01) */ +#define BLEIF_INTSET_B2MSLEEP_Pos (14UL) /*!< BLEIF INTSET: B2MSLEEP (Bit 14) */ +#define BLEIF_INTSET_B2MSLEEP_Msk (0x4000UL) /*!< BLEIF INTSET: B2MSLEEP (Bitfield-Mask: 0x01) */ +#define BLEIF_INTSET_CQERR_Pos (13UL) /*!< BLEIF INTSET: CQERR (Bit 13) */ +#define BLEIF_INTSET_CQERR_Msk (0x2000UL) /*!< BLEIF INTSET: CQERR (Bitfield-Mask: 0x01) */ +#define BLEIF_INTSET_CQUPD_Pos (12UL) /*!< BLEIF INTSET: CQUPD (Bit 12) */ +#define BLEIF_INTSET_CQUPD_Msk (0x1000UL) /*!< BLEIF INTSET: CQUPD (Bitfield-Mask: 0x01) */ +#define BLEIF_INTSET_CQPAUSED_Pos (11UL) /*!< BLEIF INTSET: CQPAUSED (Bit 11) */ +#define BLEIF_INTSET_CQPAUSED_Msk (0x800UL) /*!< BLEIF INTSET: CQPAUSED (Bitfield-Mask: 0x01) */ +#define BLEIF_INTSET_DERR_Pos (10UL) /*!< BLEIF INTSET: DERR (Bit 10) */ +#define BLEIF_INTSET_DERR_Msk (0x400UL) /*!< BLEIF INTSET: DERR (Bitfield-Mask: 0x01) */ +#define BLEIF_INTSET_DCMP_Pos (9UL) /*!< BLEIF INTSET: DCMP (Bit 9) */ +#define BLEIF_INTSET_DCMP_Msk (0x200UL) /*!< BLEIF INTSET: DCMP (Bitfield-Mask: 0x01) */ +#define BLEIF_INTSET_BLECSSTAT_Pos (8UL) /*!< BLEIF INTSET: BLECSSTAT (Bit 8) */ +#define BLEIF_INTSET_BLECSSTAT_Msk (0x100UL) /*!< BLEIF INTSET: BLECSSTAT (Bitfield-Mask: 0x01) */ +#define BLEIF_INTSET_BLECIRQ_Pos (7UL) /*!< BLEIF INTSET: BLECIRQ (Bit 7) */ +#define BLEIF_INTSET_BLECIRQ_Msk (0x80UL) /*!< BLEIF INTSET: BLECIRQ (Bitfield-Mask: 0x01) */ +#define BLEIF_INTSET_ICMD_Pos (6UL) /*!< BLEIF INTSET: ICMD (Bit 6) */ +#define BLEIF_INTSET_ICMD_Msk (0x40UL) /*!< BLEIF INTSET: ICMD (Bitfield-Mask: 0x01) */ +#define BLEIF_INTSET_IACC_Pos (5UL) /*!< BLEIF INTSET: IACC (Bit 5) */ +#define BLEIF_INTSET_IACC_Msk (0x20UL) /*!< BLEIF INTSET: IACC (Bitfield-Mask: 0x01) */ +#define BLEIF_INTSET_B2MST_Pos (4UL) /*!< BLEIF INTSET: B2MST (Bit 4) */ +#define BLEIF_INTSET_B2MST_Msk (0x10UL) /*!< BLEIF INTSET: B2MST (Bitfield-Mask: 0x01) */ +#define BLEIF_INTSET_FOVFL_Pos (3UL) /*!< BLEIF INTSET: FOVFL (Bit 3) */ +#define BLEIF_INTSET_FOVFL_Msk (0x8UL) /*!< BLEIF INTSET: FOVFL (Bitfield-Mask: 0x01) */ +#define BLEIF_INTSET_FUNDFL_Pos (2UL) /*!< BLEIF INTSET: FUNDFL (Bit 2) */ +#define BLEIF_INTSET_FUNDFL_Msk (0x4UL) /*!< BLEIF INTSET: FUNDFL (Bitfield-Mask: 0x01) */ +#define BLEIF_INTSET_THR_Pos (1UL) /*!< BLEIF INTSET: THR (Bit 1) */ +#define BLEIF_INTSET_THR_Msk (0x2UL) /*!< BLEIF INTSET: THR (Bitfield-Mask: 0x01) */ +#define BLEIF_INTSET_CMDCMP_Pos (0UL) /*!< BLEIF INTSET: CMDCMP (Bit 0) */ +#define BLEIF_INTSET_CMDCMP_Msk (0x1UL) /*!< BLEIF INTSET: CMDCMP (Bitfield-Mask: 0x01) */ +/* ======================================================= DMATRIGEN ======================================================= */ +#define BLEIF_DMATRIGEN_DTHREN_Pos (1UL) /*!< BLEIF DMATRIGEN: DTHREN (Bit 1) */ +#define BLEIF_DMATRIGEN_DTHREN_Msk (0x2UL) /*!< BLEIF DMATRIGEN: DTHREN (Bitfield-Mask: 0x01) */ +#define BLEIF_DMATRIGEN_DCMDCMPEN_Pos (0UL) /*!< BLEIF DMATRIGEN: DCMDCMPEN (Bit 0) */ +#define BLEIF_DMATRIGEN_DCMDCMPEN_Msk (0x1UL) /*!< BLEIF DMATRIGEN: DCMDCMPEN (Bitfield-Mask: 0x01) */ +/* ====================================================== DMATRIGSTAT ====================================================== */ +#define BLEIF_DMATRIGSTAT_DTOTCMP_Pos (2UL) /*!< BLEIF DMATRIGSTAT: DTOTCMP (Bit 2) */ +#define BLEIF_DMATRIGSTAT_DTOTCMP_Msk (0x4UL) /*!< BLEIF DMATRIGSTAT: DTOTCMP (Bitfield-Mask: 0x01) */ +#define BLEIF_DMATRIGSTAT_DTHR_Pos (1UL) /*!< BLEIF DMATRIGSTAT: DTHR (Bit 1) */ +#define BLEIF_DMATRIGSTAT_DTHR_Msk (0x2UL) /*!< BLEIF DMATRIGSTAT: DTHR (Bitfield-Mask: 0x01) */ +#define BLEIF_DMATRIGSTAT_DCMDCMP_Pos (0UL) /*!< BLEIF DMATRIGSTAT: DCMDCMP (Bit 0) */ +#define BLEIF_DMATRIGSTAT_DCMDCMP_Msk (0x1UL) /*!< BLEIF DMATRIGSTAT: DCMDCMP (Bitfield-Mask: 0x01) */ +/* ======================================================== DMACFG ========================================================= */ +#define BLEIF_DMACFG_DPWROFF_Pos (9UL) /*!< BLEIF DMACFG: DPWROFF (Bit 9) */ +#define BLEIF_DMACFG_DPWROFF_Msk (0x200UL) /*!< BLEIF DMACFG: DPWROFF (Bitfield-Mask: 0x01) */ +#define BLEIF_DMACFG_DMAPRI_Pos (8UL) /*!< BLEIF DMACFG: DMAPRI (Bit 8) */ +#define BLEIF_DMACFG_DMAPRI_Msk (0x100UL) /*!< BLEIF DMACFG: DMAPRI (Bitfield-Mask: 0x01) */ +#define BLEIF_DMACFG_DMADIR_Pos (1UL) /*!< BLEIF DMACFG: DMADIR (Bit 1) */ +#define BLEIF_DMACFG_DMADIR_Msk (0x2UL) /*!< BLEIF DMACFG: DMADIR (Bitfield-Mask: 0x01) */ +#define BLEIF_DMACFG_DMAEN_Pos (0UL) /*!< BLEIF DMACFG: DMAEN (Bit 0) */ +#define BLEIF_DMACFG_DMAEN_Msk (0x1UL) /*!< BLEIF DMACFG: DMAEN (Bitfield-Mask: 0x01) */ +/* ====================================================== DMATOTCOUNT ====================================================== */ +#define BLEIF_DMATOTCOUNT_TOTCOUNT_Pos (0UL) /*!< BLEIF DMATOTCOUNT: TOTCOUNT (Bit 0) */ +#define BLEIF_DMATOTCOUNT_TOTCOUNT_Msk (0xfffUL) /*!< BLEIF DMATOTCOUNT: TOTCOUNT (Bitfield-Mask: 0xfff) */ +/* ====================================================== DMATARGADDR ====================================================== */ +#define BLEIF_DMATARGADDR_TARGADDR28_Pos (28UL) /*!< BLEIF DMATARGADDR: TARGADDR28 (Bit 28) */ +#define BLEIF_DMATARGADDR_TARGADDR28_Msk (0x10000000UL) /*!< BLEIF DMATARGADDR: TARGADDR28 (Bitfield-Mask: 0x01) */ +#define BLEIF_DMATARGADDR_TARGADDR_Pos (0UL) /*!< BLEIF DMATARGADDR: TARGADDR (Bit 0) */ +#define BLEIF_DMATARGADDR_TARGADDR_Msk (0xfffffUL) /*!< BLEIF DMATARGADDR: TARGADDR (Bitfield-Mask: 0xfffff) */ +/* ======================================================== DMASTAT ======================================================== */ +#define BLEIF_DMASTAT_DMAERR_Pos (2UL) /*!< BLEIF DMASTAT: DMAERR (Bit 2) */ +#define BLEIF_DMASTAT_DMAERR_Msk (0x4UL) /*!< BLEIF DMASTAT: DMAERR (Bitfield-Mask: 0x01) */ +#define BLEIF_DMASTAT_DMACPL_Pos (1UL) /*!< BLEIF DMASTAT: DMACPL (Bit 1) */ +#define BLEIF_DMASTAT_DMACPL_Msk (0x2UL) /*!< BLEIF DMASTAT: DMACPL (Bitfield-Mask: 0x01) */ +#define BLEIF_DMASTAT_DMATIP_Pos (0UL) /*!< BLEIF DMASTAT: DMATIP (Bit 0) */ +#define BLEIF_DMASTAT_DMATIP_Msk (0x1UL) /*!< BLEIF DMASTAT: DMATIP (Bitfield-Mask: 0x01) */ +/* ========================================================= CQCFG ========================================================= */ +#define BLEIF_CQCFG_CQPRI_Pos (1UL) /*!< BLEIF CQCFG: CQPRI (Bit 1) */ +#define BLEIF_CQCFG_CQPRI_Msk (0x2UL) /*!< BLEIF CQCFG: CQPRI (Bitfield-Mask: 0x01) */ +#define BLEIF_CQCFG_CQEN_Pos (0UL) /*!< BLEIF CQCFG: CQEN (Bit 0) */ +#define BLEIF_CQCFG_CQEN_Msk (0x1UL) /*!< BLEIF CQCFG: CQEN (Bitfield-Mask: 0x01) */ +/* ======================================================== CQADDR ========================================================= */ +#define BLEIF_CQADDR_CQADDR28_Pos (28UL) /*!< BLEIF CQADDR: CQADDR28 (Bit 28) */ +#define BLEIF_CQADDR_CQADDR28_Msk (0x10000000UL) /*!< BLEIF CQADDR: CQADDR28 (Bitfield-Mask: 0x01) */ +#define BLEIF_CQADDR_CQADDR_Pos (2UL) /*!< BLEIF CQADDR: CQADDR (Bit 2) */ +#define BLEIF_CQADDR_CQADDR_Msk (0xffffcUL) /*!< BLEIF CQADDR: CQADDR (Bitfield-Mask: 0x3ffff) */ +/* ======================================================== CQSTAT ========================================================= */ +#define BLEIF_CQSTAT_CQERR_Pos (2UL) /*!< BLEIF CQSTAT: CQERR (Bit 2) */ +#define BLEIF_CQSTAT_CQERR_Msk (0x4UL) /*!< BLEIF CQSTAT: CQERR (Bitfield-Mask: 0x01) */ +#define BLEIF_CQSTAT_CQPAUSED_Pos (1UL) /*!< BLEIF CQSTAT: CQPAUSED (Bit 1) */ +#define BLEIF_CQSTAT_CQPAUSED_Msk (0x2UL) /*!< BLEIF CQSTAT: CQPAUSED (Bitfield-Mask: 0x01) */ +#define BLEIF_CQSTAT_CQTIP_Pos (0UL) /*!< BLEIF CQSTAT: CQTIP (Bit 0) */ +#define BLEIF_CQSTAT_CQTIP_Msk (0x1UL) /*!< BLEIF CQSTAT: CQTIP (Bitfield-Mask: 0x01) */ +/* ======================================================== CQFLAGS ======================================================== */ +#define BLEIF_CQFLAGS_CQIRQMASK_Pos (16UL) /*!< BLEIF CQFLAGS: CQIRQMASK (Bit 16) */ +#define BLEIF_CQFLAGS_CQIRQMASK_Msk (0xffff0000UL) /*!< BLEIF CQFLAGS: CQIRQMASK (Bitfield-Mask: 0xffff) */ +#define BLEIF_CQFLAGS_CQFLAGS_Pos (0UL) /*!< BLEIF CQFLAGS: CQFLAGS (Bit 0) */ +#define BLEIF_CQFLAGS_CQFLAGS_Msk (0xffffUL) /*!< BLEIF CQFLAGS: CQFLAGS (Bitfield-Mask: 0xffff) */ +/* ====================================================== CQSETCLEAR ======================================================= */ +#define BLEIF_CQSETCLEAR_CQFCLR_Pos (16UL) /*!< BLEIF CQSETCLEAR: CQFCLR (Bit 16) */ +#define BLEIF_CQSETCLEAR_CQFCLR_Msk (0xff0000UL) /*!< BLEIF CQSETCLEAR: CQFCLR (Bitfield-Mask: 0xff) */ +#define BLEIF_CQSETCLEAR_CQFTGL_Pos (8UL) /*!< BLEIF CQSETCLEAR: CQFTGL (Bit 8) */ +#define BLEIF_CQSETCLEAR_CQFTGL_Msk (0xff00UL) /*!< BLEIF CQSETCLEAR: CQFTGL (Bitfield-Mask: 0xff) */ +#define BLEIF_CQSETCLEAR_CQFSET_Pos (0UL) /*!< BLEIF CQSETCLEAR: CQFSET (Bit 0) */ +#define BLEIF_CQSETCLEAR_CQFSET_Msk (0xffUL) /*!< BLEIF CQSETCLEAR: CQFSET (Bitfield-Mask: 0xff) */ +/* ======================================================= CQPAUSEEN ======================================================= */ +#define BLEIF_CQPAUSEEN_CQPEN_Pos (0UL) /*!< BLEIF CQPAUSEEN: CQPEN (Bit 0) */ +#define BLEIF_CQPAUSEEN_CQPEN_Msk (0xffffUL) /*!< BLEIF CQPAUSEEN: CQPEN (Bitfield-Mask: 0xffff) */ +/* ======================================================= CQCURIDX ======================================================== */ +#define BLEIF_CQCURIDX_CQCURIDX_Pos (0UL) /*!< BLEIF CQCURIDX: CQCURIDX (Bit 0) */ +#define BLEIF_CQCURIDX_CQCURIDX_Msk (0xffUL) /*!< BLEIF CQCURIDX: CQCURIDX (Bitfield-Mask: 0xff) */ +/* ======================================================= CQENDIDX ======================================================== */ +#define BLEIF_CQENDIDX_CQENDIDX_Pos (0UL) /*!< BLEIF CQENDIDX: CQENDIDX (Bit 0) */ +#define BLEIF_CQENDIDX_CQENDIDX_Msk (0xffUL) /*!< BLEIF CQENDIDX: CQENDIDX (Bitfield-Mask: 0xff) */ +/* ======================================================== STATUS ========================================================= */ +#define BLEIF_STATUS_IDLEST_Pos (2UL) /*!< BLEIF STATUS: IDLEST (Bit 2) */ +#define BLEIF_STATUS_IDLEST_Msk (0x4UL) /*!< BLEIF STATUS: IDLEST (Bitfield-Mask: 0x01) */ +#define BLEIF_STATUS_CMDACT_Pos (1UL) /*!< BLEIF STATUS: CMDACT (Bit 1) */ +#define BLEIF_STATUS_CMDACT_Msk (0x2UL) /*!< BLEIF STATUS: CMDACT (Bitfield-Mask: 0x01) */ +#define BLEIF_STATUS_ERR_Pos (0UL) /*!< BLEIF STATUS: ERR (Bit 0) */ +#define BLEIF_STATUS_ERR_Msk (0x1UL) /*!< BLEIF STATUS: ERR (Bitfield-Mask: 0x01) */ +/* ======================================================== MSPICFG ======================================================== */ +#define BLEIF_MSPICFG_MSPIRST_Pos (30UL) /*!< BLEIF MSPICFG: MSPIRST (Bit 30) */ +#define BLEIF_MSPICFG_MSPIRST_Msk (0x40000000UL) /*!< BLEIF MSPICFG: MSPIRST (Bitfield-Mask: 0x01) */ +#define BLEIF_MSPICFG_DOUTDLY_Pos (27UL) /*!< BLEIF MSPICFG: DOUTDLY (Bit 27) */ +#define BLEIF_MSPICFG_DOUTDLY_Msk (0x38000000UL) /*!< BLEIF MSPICFG: DOUTDLY (Bitfield-Mask: 0x07) */ +#define BLEIF_MSPICFG_DINDLY_Pos (24UL) /*!< BLEIF MSPICFG: DINDLY (Bit 24) */ +#define BLEIF_MSPICFG_DINDLY_Msk (0x7000000UL) /*!< BLEIF MSPICFG: DINDLY (Bitfield-Mask: 0x07) */ +#define BLEIF_MSPICFG_SPILSB_Pos (23UL) /*!< BLEIF MSPICFG: SPILSB (Bit 23) */ +#define BLEIF_MSPICFG_SPILSB_Msk (0x800000UL) /*!< BLEIF MSPICFG: SPILSB (Bitfield-Mask: 0x01) */ +#define BLEIF_MSPICFG_RDFCPOL_Pos (22UL) /*!< BLEIF MSPICFG: RDFCPOL (Bit 22) */ +#define BLEIF_MSPICFG_RDFCPOL_Msk (0x400000UL) /*!< BLEIF MSPICFG: RDFCPOL (Bitfield-Mask: 0x01) */ +#define BLEIF_MSPICFG_WTFCPOL_Pos (21UL) /*!< BLEIF MSPICFG: WTFCPOL (Bit 21) */ +#define BLEIF_MSPICFG_WTFCPOL_Msk (0x200000UL) /*!< BLEIF MSPICFG: WTFCPOL (Bitfield-Mask: 0x01) */ +#define BLEIF_MSPICFG_RDFC_Pos (17UL) /*!< BLEIF MSPICFG: RDFC (Bit 17) */ +#define BLEIF_MSPICFG_RDFC_Msk (0x20000UL) /*!< BLEIF MSPICFG: RDFC (Bitfield-Mask: 0x01) */ +#define BLEIF_MSPICFG_WTFC_Pos (16UL) /*!< BLEIF MSPICFG: WTFC (Bit 16) */ +#define BLEIF_MSPICFG_WTFC_Msk (0x10000UL) /*!< BLEIF MSPICFG: WTFC (Bitfield-Mask: 0x01) */ +#define BLEIF_MSPICFG_FULLDUP_Pos (2UL) /*!< BLEIF MSPICFG: FULLDUP (Bit 2) */ +#define BLEIF_MSPICFG_FULLDUP_Msk (0x4UL) /*!< BLEIF MSPICFG: FULLDUP (Bitfield-Mask: 0x01) */ +#define BLEIF_MSPICFG_SPHA_Pos (1UL) /*!< BLEIF MSPICFG: SPHA (Bit 1) */ +#define BLEIF_MSPICFG_SPHA_Msk (0x2UL) /*!< BLEIF MSPICFG: SPHA (Bitfield-Mask: 0x01) */ +#define BLEIF_MSPICFG_SPOL_Pos (0UL) /*!< BLEIF MSPICFG: SPOL (Bit 0) */ +#define BLEIF_MSPICFG_SPOL_Msk (0x1UL) /*!< BLEIF MSPICFG: SPOL (Bitfield-Mask: 0x01) */ +/* ======================================================== BLECFG ========================================================= */ +#define BLEIF_BLECFG_SPIISOCTL_Pos (14UL) /*!< BLEIF BLECFG: SPIISOCTL (Bit 14) */ +#define BLEIF_BLECFG_SPIISOCTL_Msk (0xc000UL) /*!< BLEIF BLECFG: SPIISOCTL (Bitfield-Mask: 0x03) */ +#define BLEIF_BLECFG_PWRISOCTL_Pos (12UL) /*!< BLEIF BLECFG: PWRISOCTL (Bit 12) */ +#define BLEIF_BLECFG_PWRISOCTL_Msk (0x3000UL) /*!< BLEIF BLECFG: PWRISOCTL (Bitfield-Mask: 0x03) */ +#define BLEIF_BLECFG_STAYASLEEP_Pos (11UL) /*!< BLEIF BLECFG: STAYASLEEP (Bit 11) */ +#define BLEIF_BLECFG_STAYASLEEP_Msk (0x800UL) /*!< BLEIF BLECFG: STAYASLEEP (Bitfield-Mask: 0x01) */ +#define BLEIF_BLECFG_FRCCLK_Pos (10UL) /*!< BLEIF BLECFG: FRCCLK (Bit 10) */ +#define BLEIF_BLECFG_FRCCLK_Msk (0x400UL) /*!< BLEIF BLECFG: FRCCLK (Bitfield-Mask: 0x01) */ +#define BLEIF_BLECFG_MCUFRCSLP_Pos (9UL) /*!< BLEIF BLECFG: MCUFRCSLP (Bit 9) */ +#define BLEIF_BLECFG_MCUFRCSLP_Msk (0x200UL) /*!< BLEIF BLECFG: MCUFRCSLP (Bitfield-Mask: 0x01) */ +#define BLEIF_BLECFG_WT4ACTOFF_Pos (8UL) /*!< BLEIF BLECFG: WT4ACTOFF (Bit 8) */ +#define BLEIF_BLECFG_WT4ACTOFF_Msk (0x100UL) /*!< BLEIF BLECFG: WT4ACTOFF (Bitfield-Mask: 0x01) */ +#define BLEIF_BLECFG_BLEHREQCTL_Pos (6UL) /*!< BLEIF BLECFG: BLEHREQCTL (Bit 6) */ +#define BLEIF_BLECFG_BLEHREQCTL_Msk (0xc0UL) /*!< BLEIF BLECFG: BLEHREQCTL (Bitfield-Mask: 0x03) */ +#define BLEIF_BLECFG_DCDCFLGCTL_Pos (4UL) /*!< BLEIF BLECFG: DCDCFLGCTL (Bit 4) */ +#define BLEIF_BLECFG_DCDCFLGCTL_Msk (0x30UL) /*!< BLEIF BLECFG: DCDCFLGCTL (Bitfield-Mask: 0x03) */ +#define BLEIF_BLECFG_WAKEUPCTL_Pos (2UL) /*!< BLEIF BLECFG: WAKEUPCTL (Bit 2) */ +#define BLEIF_BLECFG_WAKEUPCTL_Msk (0xcUL) /*!< BLEIF BLECFG: WAKEUPCTL (Bitfield-Mask: 0x03) */ +#define BLEIF_BLECFG_BLERSTN_Pos (1UL) /*!< BLEIF BLECFG: BLERSTN (Bit 1) */ +#define BLEIF_BLECFG_BLERSTN_Msk (0x2UL) /*!< BLEIF BLECFG: BLERSTN (Bitfield-Mask: 0x01) */ +#define BLEIF_BLECFG_PWRSMEN_Pos (0UL) /*!< BLEIF BLECFG: PWRSMEN (Bit 0) */ +#define BLEIF_BLECFG_PWRSMEN_Msk (0x1UL) /*!< BLEIF BLECFG: PWRSMEN (Bitfield-Mask: 0x01) */ +/* ======================================================== PWRCMD ========================================================= */ +#define BLEIF_PWRCMD_RESTART_Pos (1UL) /*!< BLEIF PWRCMD: RESTART (Bit 1) */ +#define BLEIF_PWRCMD_RESTART_Msk (0x2UL) /*!< BLEIF PWRCMD: RESTART (Bitfield-Mask: 0x01) */ +#define BLEIF_PWRCMD_WAKEREQ_Pos (0UL) /*!< BLEIF PWRCMD: WAKEREQ (Bit 0) */ +#define BLEIF_PWRCMD_WAKEREQ_Msk (0x1UL) /*!< BLEIF PWRCMD: WAKEREQ (Bitfield-Mask: 0x01) */ +/* ======================================================== BSTATUS ======================================================== */ +#define BLEIF_BSTATUS_BLEHREQ_Pos (12UL) /*!< BLEIF BSTATUS: BLEHREQ (Bit 12) */ +#define BLEIF_BSTATUS_BLEHREQ_Msk (0x1000UL) /*!< BLEIF BSTATUS: BLEHREQ (Bitfield-Mask: 0x01) */ +#define BLEIF_BSTATUS_BLEHACK_Pos (11UL) /*!< BLEIF BSTATUS: BLEHACK (Bit 11) */ +#define BLEIF_BSTATUS_BLEHACK_Msk (0x800UL) /*!< BLEIF BSTATUS: BLEHACK (Bitfield-Mask: 0x01) */ +#define BLEIF_BSTATUS_PWRST_Pos (8UL) /*!< BLEIF BSTATUS: PWRST (Bit 8) */ +#define BLEIF_BSTATUS_PWRST_Msk (0x700UL) /*!< BLEIF BSTATUS: PWRST (Bitfield-Mask: 0x07) */ +#define BLEIF_BSTATUS_BLEIRQ_Pos (7UL) /*!< BLEIF BSTATUS: BLEIRQ (Bit 7) */ +#define BLEIF_BSTATUS_BLEIRQ_Msk (0x80UL) /*!< BLEIF BSTATUS: BLEIRQ (Bitfield-Mask: 0x01) */ +#define BLEIF_BSTATUS_WAKEUP_Pos (6UL) /*!< BLEIF BSTATUS: WAKEUP (Bit 6) */ +#define BLEIF_BSTATUS_WAKEUP_Msk (0x40UL) /*!< BLEIF BSTATUS: WAKEUP (Bitfield-Mask: 0x01) */ +#define BLEIF_BSTATUS_DCDCFLAG_Pos (5UL) /*!< BLEIF BSTATUS: DCDCFLAG (Bit 5) */ +#define BLEIF_BSTATUS_DCDCFLAG_Msk (0x20UL) /*!< BLEIF BSTATUS: DCDCFLAG (Bitfield-Mask: 0x01) */ +#define BLEIF_BSTATUS_DCDCREQ_Pos (4UL) /*!< BLEIF BSTATUS: DCDCREQ (Bit 4) */ +#define BLEIF_BSTATUS_DCDCREQ_Msk (0x10UL) /*!< BLEIF BSTATUS: DCDCREQ (Bitfield-Mask: 0x01) */ +#define BLEIF_BSTATUS_SPISTATUS_Pos (3UL) /*!< BLEIF BSTATUS: SPISTATUS (Bit 3) */ +#define BLEIF_BSTATUS_SPISTATUS_Msk (0x8UL) /*!< BLEIF BSTATUS: SPISTATUS (Bitfield-Mask: 0x01) */ +#define BLEIF_BSTATUS_B2MSTATE_Pos (0UL) /*!< BLEIF BSTATUS: B2MSTATE (Bit 0) */ +#define BLEIF_BSTATUS_B2MSTATE_Msk (0x7UL) /*!< BLEIF BSTATUS: B2MSTATE (Bitfield-Mask: 0x07) */ +/* ======================================================== BLEDBG ========================================================= */ +#define BLEIF_BLEDBG_DBGDATA_Pos (3UL) /*!< BLEIF BLEDBG: DBGDATA (Bit 3) */ +#define BLEIF_BLEDBG_DBGDATA_Msk (0xfffffff8UL) /*!< BLEIF BLEDBG: DBGDATA (Bitfield-Mask: 0x1fffffff) */ +#define BLEIF_BLEDBG_APBCLKON_Pos (2UL) /*!< BLEIF BLEDBG: APBCLKON (Bit 2) */ +#define BLEIF_BLEDBG_APBCLKON_Msk (0x4UL) /*!< BLEIF BLEDBG: APBCLKON (Bitfield-Mask: 0x01) */ +#define BLEIF_BLEDBG_IOCLKON_Pos (1UL) /*!< BLEIF BLEDBG: IOCLKON (Bit 1) */ +#define BLEIF_BLEDBG_IOCLKON_Msk (0x2UL) /*!< BLEIF BLEDBG: IOCLKON (Bitfield-Mask: 0x01) */ +#define BLEIF_BLEDBG_DBGEN_Pos (0UL) /*!< BLEIF BLEDBG: DBGEN (Bit 0) */ +#define BLEIF_BLEDBG_DBGEN_Msk (0x1UL) /*!< BLEIF BLEDBG: DBGEN (Bitfield-Mask: 0x01) */ + + +/* =========================================================================================================================== */ +/* ================ CACHECTRL ================ */ +/* =========================================================================================================================== */ + +/* ======================================================= CACHECFG ======================================================== */ +#define CACHECTRL_CACHECFG_ENABLE_MONITOR_Pos (24UL) /*!< CACHECTRL CACHECFG: ENABLE_MONITOR (Bit 24) */ +#define CACHECTRL_CACHECFG_ENABLE_MONITOR_Msk (0x1000000UL) /*!< CACHECTRL CACHECFG: ENABLE_MONITOR (Bitfield-Mask: 0x01) */ +#define CACHECTRL_CACHECFG_DATA_CLKGATE_Pos (20UL) /*!< CACHECTRL CACHECFG: DATA_CLKGATE (Bit 20) */ +#define CACHECTRL_CACHECFG_DATA_CLKGATE_Msk (0x100000UL) /*!< CACHECTRL CACHECFG: DATA_CLKGATE (Bitfield-Mask: 0x01) */ +#define CACHECTRL_CACHECFG_CACHE_LS_Pos (11UL) /*!< CACHECTRL CACHECFG: CACHE_LS (Bit 11) */ +#define CACHECTRL_CACHECFG_CACHE_LS_Msk (0x800UL) /*!< CACHECTRL CACHECFG: CACHE_LS (Bitfield-Mask: 0x01) */ +#define CACHECTRL_CACHECFG_CACHE_CLKGATE_Pos (10UL) /*!< CACHECTRL CACHECFG: CACHE_CLKGATE (Bit 10) */ +#define CACHECTRL_CACHECFG_CACHE_CLKGATE_Msk (0x400UL) /*!< CACHECTRL CACHECFG: CACHE_CLKGATE (Bitfield-Mask: 0x01) */ +#define CACHECTRL_CACHECFG_DCACHE_ENABLE_Pos (9UL) /*!< CACHECTRL CACHECFG: DCACHE_ENABLE (Bit 9) */ +#define CACHECTRL_CACHECFG_DCACHE_ENABLE_Msk (0x200UL) /*!< CACHECTRL CACHECFG: DCACHE_ENABLE (Bitfield-Mask: 0x01) */ +#define CACHECTRL_CACHECFG_ICACHE_ENABLE_Pos (8UL) /*!< CACHECTRL CACHECFG: ICACHE_ENABLE (Bit 8) */ +#define CACHECTRL_CACHECFG_ICACHE_ENABLE_Msk (0x100UL) /*!< CACHECTRL CACHECFG: ICACHE_ENABLE (Bitfield-Mask: 0x01) */ +#define CACHECTRL_CACHECFG_CONFIG_Pos (4UL) /*!< CACHECTRL CACHECFG: CONFIG (Bit 4) */ +#define CACHECTRL_CACHECFG_CONFIG_Msk (0xf0UL) /*!< CACHECTRL CACHECFG: CONFIG (Bitfield-Mask: 0x0f) */ +#define CACHECTRL_CACHECFG_ENABLE_NC1_Pos (3UL) /*!< CACHECTRL CACHECFG: ENABLE_NC1 (Bit 3) */ +#define CACHECTRL_CACHECFG_ENABLE_NC1_Msk (0x8UL) /*!< CACHECTRL CACHECFG: ENABLE_NC1 (Bitfield-Mask: 0x01) */ +#define CACHECTRL_CACHECFG_ENABLE_NC0_Pos (2UL) /*!< CACHECTRL CACHECFG: ENABLE_NC0 (Bit 2) */ +#define CACHECTRL_CACHECFG_ENABLE_NC0_Msk (0x4UL) /*!< CACHECTRL CACHECFG: ENABLE_NC0 (Bitfield-Mask: 0x01) */ +#define CACHECTRL_CACHECFG_LRU_Pos (1UL) /*!< CACHECTRL CACHECFG: LRU (Bit 1) */ +#define CACHECTRL_CACHECFG_LRU_Msk (0x2UL) /*!< CACHECTRL CACHECFG: LRU (Bitfield-Mask: 0x01) */ +#define CACHECTRL_CACHECFG_ENABLE_Pos (0UL) /*!< CACHECTRL CACHECFG: ENABLE (Bit 0) */ +#define CACHECTRL_CACHECFG_ENABLE_Msk (0x1UL) /*!< CACHECTRL CACHECFG: ENABLE (Bitfield-Mask: 0x01) */ +/* ======================================================= FLASHCFG ======================================================== */ +#define CACHECTRL_FLASHCFG_LPMMODE_Pos (12UL) /*!< CACHECTRL FLASHCFG: LPMMODE (Bit 12) */ +#define CACHECTRL_FLASHCFG_LPMMODE_Msk (0x3000UL) /*!< CACHECTRL FLASHCFG: LPMMODE (Bitfield-Mask: 0x03) */ +#define CACHECTRL_FLASHCFG_LPM_RD_WAIT_Pos (8UL) /*!< CACHECTRL FLASHCFG: LPM_RD_WAIT (Bit 8) */ +#define CACHECTRL_FLASHCFG_LPM_RD_WAIT_Msk (0xf00UL) /*!< CACHECTRL FLASHCFG: LPM_RD_WAIT (Bitfield-Mask: 0x0f) */ +#define CACHECTRL_FLASHCFG_SEDELAY_Pos (4UL) /*!< CACHECTRL FLASHCFG: SEDELAY (Bit 4) */ +#define CACHECTRL_FLASHCFG_SEDELAY_Msk (0x70UL) /*!< CACHECTRL FLASHCFG: SEDELAY (Bitfield-Mask: 0x07) */ +#define CACHECTRL_FLASHCFG_RD_WAIT_Pos (0UL) /*!< CACHECTRL FLASHCFG: RD_WAIT (Bit 0) */ +#define CACHECTRL_FLASHCFG_RD_WAIT_Msk (0xfUL) /*!< CACHECTRL FLASHCFG: RD_WAIT (Bitfield-Mask: 0x0f) */ +/* ========================================================= CTRL ========================================================== */ +#define CACHECTRL_CTRL_FLASH1_SLM_ENABLE_Pos (10UL) /*!< CACHECTRL CTRL: FLASH1_SLM_ENABLE (Bit 10) */ +#define CACHECTRL_CTRL_FLASH1_SLM_ENABLE_Msk (0x400UL) /*!< CACHECTRL CTRL: FLASH1_SLM_ENABLE (Bitfield-Mask: 0x01) */ +#define CACHECTRL_CTRL_FLASH1_SLM_DISABLE_Pos (9UL) /*!< CACHECTRL CTRL: FLASH1_SLM_DISABLE (Bit 9) */ +#define CACHECTRL_CTRL_FLASH1_SLM_DISABLE_Msk (0x200UL) /*!< CACHECTRL CTRL: FLASH1_SLM_DISABLE (Bitfield-Mask: 0x01) */ +#define CACHECTRL_CTRL_FLASH1_SLM_STATUS_Pos (8UL) /*!< CACHECTRL CTRL: FLASH1_SLM_STATUS (Bit 8) */ +#define CACHECTRL_CTRL_FLASH1_SLM_STATUS_Msk (0x100UL) /*!< CACHECTRL CTRL: FLASH1_SLM_STATUS (Bitfield-Mask: 0x01) */ +#define CACHECTRL_CTRL_FLASH0_SLM_ENABLE_Pos (6UL) /*!< CACHECTRL CTRL: FLASH0_SLM_ENABLE (Bit 6) */ +#define CACHECTRL_CTRL_FLASH0_SLM_ENABLE_Msk (0x40UL) /*!< CACHECTRL CTRL: FLASH0_SLM_ENABLE (Bitfield-Mask: 0x01) */ +#define CACHECTRL_CTRL_FLASH0_SLM_DISABLE_Pos (5UL) /*!< CACHECTRL CTRL: FLASH0_SLM_DISABLE (Bit 5) */ +#define CACHECTRL_CTRL_FLASH0_SLM_DISABLE_Msk (0x20UL) /*!< CACHECTRL CTRL: FLASH0_SLM_DISABLE (Bitfield-Mask: 0x01) */ +#define CACHECTRL_CTRL_FLASH0_SLM_STATUS_Pos (4UL) /*!< CACHECTRL CTRL: FLASH0_SLM_STATUS (Bit 4) */ +#define CACHECTRL_CTRL_FLASH0_SLM_STATUS_Msk (0x10UL) /*!< CACHECTRL CTRL: FLASH0_SLM_STATUS (Bitfield-Mask: 0x01) */ +#define CACHECTRL_CTRL_CACHE_READY_Pos (2UL) /*!< CACHECTRL CTRL: CACHE_READY (Bit 2) */ +#define CACHECTRL_CTRL_CACHE_READY_Msk (0x4UL) /*!< CACHECTRL CTRL: CACHE_READY (Bitfield-Mask: 0x01) */ +#define CACHECTRL_CTRL_RESET_STAT_Pos (1UL) /*!< CACHECTRL CTRL: RESET_STAT (Bit 1) */ +#define CACHECTRL_CTRL_RESET_STAT_Msk (0x2UL) /*!< CACHECTRL CTRL: RESET_STAT (Bitfield-Mask: 0x01) */ +#define CACHECTRL_CTRL_INVALIDATE_Pos (0UL) /*!< CACHECTRL CTRL: INVALIDATE (Bit 0) */ +#define CACHECTRL_CTRL_INVALIDATE_Msk (0x1UL) /*!< CACHECTRL CTRL: INVALIDATE (Bitfield-Mask: 0x01) */ +/* ======================================================= NCR0START ======================================================= */ +#define CACHECTRL_NCR0START_ADDR_Pos (4UL) /*!< CACHECTRL NCR0START: ADDR (Bit 4) */ +#define CACHECTRL_NCR0START_ADDR_Msk (0x7fffff0UL) /*!< CACHECTRL NCR0START: ADDR (Bitfield-Mask: 0x7fffff) */ +/* ======================================================== NCR0END ======================================================== */ +#define CACHECTRL_NCR0END_ADDR_Pos (4UL) /*!< CACHECTRL NCR0END: ADDR (Bit 4) */ +#define CACHECTRL_NCR0END_ADDR_Msk (0x7fffff0UL) /*!< CACHECTRL NCR0END: ADDR (Bitfield-Mask: 0x7fffff) */ +/* ======================================================= NCR1START ======================================================= */ +#define CACHECTRL_NCR1START_ADDR_Pos (4UL) /*!< CACHECTRL NCR1START: ADDR (Bit 4) */ +#define CACHECTRL_NCR1START_ADDR_Msk (0x7fffff0UL) /*!< CACHECTRL NCR1START: ADDR (Bitfield-Mask: 0x7fffff) */ +/* ======================================================== NCR1END ======================================================== */ +#define CACHECTRL_NCR1END_ADDR_Pos (4UL) /*!< CACHECTRL NCR1END: ADDR (Bit 4) */ +#define CACHECTRL_NCR1END_ADDR_Msk (0x7fffff0UL) /*!< CACHECTRL NCR1END: ADDR (Bitfield-Mask: 0x7fffff) */ +/* ========================================================= DMON0 ========================================================= */ +#define CACHECTRL_DMON0_DACCESS_COUNT_Pos (0UL) /*!< CACHECTRL DMON0: DACCESS_COUNT (Bit 0) */ +#define CACHECTRL_DMON0_DACCESS_COUNT_Msk (0xffffffffUL) /*!< CACHECTRL DMON0: DACCESS_COUNT (Bitfield-Mask: 0xffffffff) */ +/* ========================================================= DMON1 ========================================================= */ +#define CACHECTRL_DMON1_DLOOKUP_COUNT_Pos (0UL) /*!< CACHECTRL DMON1: DLOOKUP_COUNT (Bit 0) */ +#define CACHECTRL_DMON1_DLOOKUP_COUNT_Msk (0xffffffffUL) /*!< CACHECTRL DMON1: DLOOKUP_COUNT (Bitfield-Mask: 0xffffffff) */ +/* ========================================================= DMON2 ========================================================= */ +#define CACHECTRL_DMON2_DHIT_COUNT_Pos (0UL) /*!< CACHECTRL DMON2: DHIT_COUNT (Bit 0) */ +#define CACHECTRL_DMON2_DHIT_COUNT_Msk (0xffffffffUL) /*!< CACHECTRL DMON2: DHIT_COUNT (Bitfield-Mask: 0xffffffff) */ +/* ========================================================= DMON3 ========================================================= */ +#define CACHECTRL_DMON3_DLINE_COUNT_Pos (0UL) /*!< CACHECTRL DMON3: DLINE_COUNT (Bit 0) */ +#define CACHECTRL_DMON3_DLINE_COUNT_Msk (0xffffffffUL) /*!< CACHECTRL DMON3: DLINE_COUNT (Bitfield-Mask: 0xffffffff) */ +/* ========================================================= IMON0 ========================================================= */ +#define CACHECTRL_IMON0_IACCESS_COUNT_Pos (0UL) /*!< CACHECTRL IMON0: IACCESS_COUNT (Bit 0) */ +#define CACHECTRL_IMON0_IACCESS_COUNT_Msk (0xffffffffUL) /*!< CACHECTRL IMON0: IACCESS_COUNT (Bitfield-Mask: 0xffffffff) */ +/* ========================================================= IMON1 ========================================================= */ +#define CACHECTRL_IMON1_ILOOKUP_COUNT_Pos (0UL) /*!< CACHECTRL IMON1: ILOOKUP_COUNT (Bit 0) */ +#define CACHECTRL_IMON1_ILOOKUP_COUNT_Msk (0xffffffffUL) /*!< CACHECTRL IMON1: ILOOKUP_COUNT (Bitfield-Mask: 0xffffffff) */ +/* ========================================================= IMON2 ========================================================= */ +#define CACHECTRL_IMON2_IHIT_COUNT_Pos (0UL) /*!< CACHECTRL IMON2: IHIT_COUNT (Bit 0) */ +#define CACHECTRL_IMON2_IHIT_COUNT_Msk (0xffffffffUL) /*!< CACHECTRL IMON2: IHIT_COUNT (Bitfield-Mask: 0xffffffff) */ +/* ========================================================= IMON3 ========================================================= */ +#define CACHECTRL_IMON3_ILINE_COUNT_Pos (0UL) /*!< CACHECTRL IMON3: ILINE_COUNT (Bit 0) */ +#define CACHECTRL_IMON3_ILINE_COUNT_Msk (0xffffffffUL) /*!< CACHECTRL IMON3: ILINE_COUNT (Bitfield-Mask: 0xffffffff) */ + + +/* =========================================================================================================================== */ +/* ================ CLKGEN ================ */ +/* =========================================================================================================================== */ + +/* ========================================================= CALXT ========================================================= */ +#define CLKGEN_CALXT_CALXT_Pos (0UL) /*!< CLKGEN CALXT: CALXT (Bit 0) */ +#define CLKGEN_CALXT_CALXT_Msk (0x7ffUL) /*!< CLKGEN CALXT: CALXT (Bitfield-Mask: 0x7ff) */ +/* ========================================================= CALRC ========================================================= */ +#define CLKGEN_CALRC_CALRC_Pos (0UL) /*!< CLKGEN CALRC: CALRC (Bit 0) */ +#define CLKGEN_CALRC_CALRC_Msk (0x3ffffUL) /*!< CLKGEN CALRC: CALRC (Bitfield-Mask: 0x3ffff) */ +/* ======================================================== ACALCTR ======================================================== */ +#define CLKGEN_ACALCTR_ACALCTR_Pos (0UL) /*!< CLKGEN ACALCTR: ACALCTR (Bit 0) */ +#define CLKGEN_ACALCTR_ACALCTR_Msk (0xffffffUL) /*!< CLKGEN ACALCTR: ACALCTR (Bitfield-Mask: 0xffffff) */ +/* ========================================================= OCTRL ========================================================= */ +#define CLKGEN_OCTRL_ACAL_Pos (8UL) /*!< CLKGEN OCTRL: ACAL (Bit 8) */ +#define CLKGEN_OCTRL_ACAL_Msk (0x700UL) /*!< CLKGEN OCTRL: ACAL (Bitfield-Mask: 0x07) */ +#define CLKGEN_OCTRL_OSEL_Pos (7UL) /*!< CLKGEN OCTRL: OSEL (Bit 7) */ +#define CLKGEN_OCTRL_OSEL_Msk (0x80UL) /*!< CLKGEN OCTRL: OSEL (Bitfield-Mask: 0x01) */ +#define CLKGEN_OCTRL_FOS_Pos (6UL) /*!< CLKGEN OCTRL: FOS (Bit 6) */ +#define CLKGEN_OCTRL_FOS_Msk (0x40UL) /*!< CLKGEN OCTRL: FOS (Bitfield-Mask: 0x01) */ +#define CLKGEN_OCTRL_STOPRC_Pos (1UL) /*!< CLKGEN OCTRL: STOPRC (Bit 1) */ +#define CLKGEN_OCTRL_STOPRC_Msk (0x2UL) /*!< CLKGEN OCTRL: STOPRC (Bitfield-Mask: 0x01) */ +#define CLKGEN_OCTRL_STOPXT_Pos (0UL) /*!< CLKGEN OCTRL: STOPXT (Bit 0) */ +#define CLKGEN_OCTRL_STOPXT_Msk (0x1UL) /*!< CLKGEN OCTRL: STOPXT (Bitfield-Mask: 0x01) */ +/* ======================================================== CLKOUT ========================================================= */ +#define CLKGEN_CLKOUT_CKEN_Pos (7UL) /*!< CLKGEN CLKOUT: CKEN (Bit 7) */ +#define CLKGEN_CLKOUT_CKEN_Msk (0x80UL) /*!< CLKGEN CLKOUT: CKEN (Bitfield-Mask: 0x01) */ +#define CLKGEN_CLKOUT_CKSEL_Pos (0UL) /*!< CLKGEN CLKOUT: CKSEL (Bit 0) */ +#define CLKGEN_CLKOUT_CKSEL_Msk (0x3fUL) /*!< CLKGEN CLKOUT: CKSEL (Bitfield-Mask: 0x3f) */ +/* ======================================================== CLKKEY ========================================================= */ +#define CLKGEN_CLKKEY_CLKKEY_Pos (0UL) /*!< CLKGEN CLKKEY: CLKKEY (Bit 0) */ +#define CLKGEN_CLKKEY_CLKKEY_Msk (0xffffffffUL) /*!< CLKGEN CLKKEY: CLKKEY (Bitfield-Mask: 0xffffffff) */ +/* ========================================================= CCTRL ========================================================= */ +#define CLKGEN_CCTRL_CORESEL_Pos (0UL) /*!< CLKGEN CCTRL: CORESEL (Bit 0) */ +#define CLKGEN_CCTRL_CORESEL_Msk (0x1UL) /*!< CLKGEN CCTRL: CORESEL (Bitfield-Mask: 0x01) */ +/* ======================================================== STATUS ========================================================= */ +#define CLKGEN_STATUS_OSCF_Pos (1UL) /*!< CLKGEN STATUS: OSCF (Bit 1) */ +#define CLKGEN_STATUS_OSCF_Msk (0x2UL) /*!< CLKGEN STATUS: OSCF (Bitfield-Mask: 0x01) */ +#define CLKGEN_STATUS_OMODE_Pos (0UL) /*!< CLKGEN STATUS: OMODE (Bit 0) */ +#define CLKGEN_STATUS_OMODE_Msk (0x1UL) /*!< CLKGEN STATUS: OMODE (Bitfield-Mask: 0x01) */ +/* ========================================================= HFADJ ========================================================= */ +#define CLKGEN_HFADJ_HFADJGAIN_Pos (21UL) /*!< CLKGEN HFADJ: HFADJGAIN (Bit 21) */ +#define CLKGEN_HFADJ_HFADJGAIN_Msk (0xe00000UL) /*!< CLKGEN HFADJ: HFADJGAIN (Bitfield-Mask: 0x07) */ +#define CLKGEN_HFADJ_HFWARMUP_Pos (20UL) /*!< CLKGEN HFADJ: HFWARMUP (Bit 20) */ +#define CLKGEN_HFADJ_HFWARMUP_Msk (0x100000UL) /*!< CLKGEN HFADJ: HFWARMUP (Bitfield-Mask: 0x01) */ +#define CLKGEN_HFADJ_HFXTADJ_Pos (8UL) /*!< CLKGEN HFADJ: HFXTADJ (Bit 8) */ +#define CLKGEN_HFADJ_HFXTADJ_Msk (0xfff00UL) /*!< CLKGEN HFADJ: HFXTADJ (Bitfield-Mask: 0xfff) */ +#define CLKGEN_HFADJ_HFADJCK_Pos (1UL) /*!< CLKGEN HFADJ: HFADJCK (Bit 1) */ +#define CLKGEN_HFADJ_HFADJCK_Msk (0xeUL) /*!< CLKGEN HFADJ: HFADJCK (Bitfield-Mask: 0x07) */ +#define CLKGEN_HFADJ_HFADJEN_Pos (0UL) /*!< CLKGEN HFADJ: HFADJEN (Bit 0) */ +#define CLKGEN_HFADJ_HFADJEN_Msk (0x1UL) /*!< CLKGEN HFADJ: HFADJEN (Bitfield-Mask: 0x01) */ +/* ====================================================== CLOCKENSTAT ====================================================== */ +#define CLKGEN_CLOCKENSTAT_CLOCKENSTAT_Pos (0UL) /*!< CLKGEN CLOCKENSTAT: CLOCKENSTAT (Bit 0) */ +#define CLKGEN_CLOCKENSTAT_CLOCKENSTAT_Msk (0xffffffffUL) /*!< CLKGEN CLOCKENSTAT: CLOCKENSTAT (Bitfield-Mask: 0xffffffff) */ +/* ===================================================== CLOCKEN2STAT ====================================================== */ +#define CLKGEN_CLOCKEN2STAT_CLOCKEN2STAT_Pos (0UL) /*!< CLKGEN CLOCKEN2STAT: CLOCKEN2STAT (Bit 0) */ +#define CLKGEN_CLOCKEN2STAT_CLOCKEN2STAT_Msk (0xffffffffUL) /*!< CLKGEN CLOCKEN2STAT: CLOCKEN2STAT (Bitfield-Mask: 0xffffffff) */ +/* ===================================================== CLOCKEN3STAT ====================================================== */ +#define CLKGEN_CLOCKEN3STAT_CLOCKEN3STAT_Pos (0UL) /*!< CLKGEN CLOCKEN3STAT: CLOCKEN3STAT (Bit 0) */ +#define CLKGEN_CLOCKEN3STAT_CLOCKEN3STAT_Msk (0xffffffffUL) /*!< CLKGEN CLOCKEN3STAT: CLOCKEN3STAT (Bitfield-Mask: 0xffffffff) */ +/* ======================================================= FREQCTRL ======================================================== */ +#define CLKGEN_FREQCTRL_BURSTSTATUS_Pos (2UL) /*!< CLKGEN FREQCTRL: BURSTSTATUS (Bit 2) */ +#define CLKGEN_FREQCTRL_BURSTSTATUS_Msk (0x4UL) /*!< CLKGEN FREQCTRL: BURSTSTATUS (Bitfield-Mask: 0x01) */ +#define CLKGEN_FREQCTRL_BURSTACK_Pos (1UL) /*!< CLKGEN FREQCTRL: BURSTACK (Bit 1) */ +#define CLKGEN_FREQCTRL_BURSTACK_Msk (0x2UL) /*!< CLKGEN FREQCTRL: BURSTACK (Bitfield-Mask: 0x01) */ +#define CLKGEN_FREQCTRL_BURSTREQ_Pos (0UL) /*!< CLKGEN FREQCTRL: BURSTREQ (Bit 0) */ +#define CLKGEN_FREQCTRL_BURSTREQ_Msk (0x1UL) /*!< CLKGEN FREQCTRL: BURSTREQ (Bitfield-Mask: 0x01) */ +/* ===================================================== BLEBUCKTONADJ ===================================================== */ +#define CLKGEN_BLEBUCKTONADJ_ZEROLENDETECTEN_Pos (27UL) /*!< CLKGEN BLEBUCKTONADJ: ZEROLENDETECTEN (Bit 27) */ +#define CLKGEN_BLEBUCKTONADJ_ZEROLENDETECTEN_Msk (0x8000000UL) /*!< CLKGEN BLEBUCKTONADJ: ZEROLENDETECTEN (Bitfield-Mask: 0x01) */ +#define CLKGEN_BLEBUCKTONADJ_ZEROLENDETECTTRIM_Pos (23UL) /*!< CLKGEN BLEBUCKTONADJ: ZEROLENDETECTTRIM (Bit 23) */ +#define CLKGEN_BLEBUCKTONADJ_ZEROLENDETECTTRIM_Msk (0x7800000UL) /*!< CLKGEN BLEBUCKTONADJ: ZEROLENDETECTTRIM (Bitfield-Mask: 0x0f) */ +#define CLKGEN_BLEBUCKTONADJ_TONADJUSTEN_Pos (22UL) /*!< CLKGEN BLEBUCKTONADJ: TONADJUSTEN (Bit 22) */ +#define CLKGEN_BLEBUCKTONADJ_TONADJUSTEN_Msk (0x400000UL) /*!< CLKGEN BLEBUCKTONADJ: TONADJUSTEN (Bitfield-Mask: 0x01) */ +#define CLKGEN_BLEBUCKTONADJ_TONADJUSTPERIOD_Pos (20UL) /*!< CLKGEN BLEBUCKTONADJ: TONADJUSTPERIOD (Bit 20) */ +#define CLKGEN_BLEBUCKTONADJ_TONADJUSTPERIOD_Msk (0x300000UL) /*!< CLKGEN BLEBUCKTONADJ: TONADJUSTPERIOD (Bitfield-Mask: 0x03) */ +#define CLKGEN_BLEBUCKTONADJ_TONHIGHTHRESHOLD_Pos (10UL) /*!< CLKGEN BLEBUCKTONADJ: TONHIGHTHRESHOLD (Bit 10) */ +#define CLKGEN_BLEBUCKTONADJ_TONHIGHTHRESHOLD_Msk (0xffc00UL) /*!< CLKGEN BLEBUCKTONADJ: TONHIGHTHRESHOLD (Bitfield-Mask: 0x3ff) */ +#define CLKGEN_BLEBUCKTONADJ_TONLOWTHRESHOLD_Pos (0UL) /*!< CLKGEN BLEBUCKTONADJ: TONLOWTHRESHOLD (Bit 0) */ +#define CLKGEN_BLEBUCKTONADJ_TONLOWTHRESHOLD_Msk (0x3ffUL) /*!< CLKGEN BLEBUCKTONADJ: TONLOWTHRESHOLD (Bitfield-Mask: 0x3ff) */ +/* ======================================================= INTRPTEN ======================================================== */ +#define CLKGEN_INTRPTEN_OF_Pos (2UL) /*!< CLKGEN INTRPTEN: OF (Bit 2) */ +#define CLKGEN_INTRPTEN_OF_Msk (0x4UL) /*!< CLKGEN INTRPTEN: OF (Bitfield-Mask: 0x01) */ +#define CLKGEN_INTRPTEN_ACC_Pos (1UL) /*!< CLKGEN INTRPTEN: ACC (Bit 1) */ +#define CLKGEN_INTRPTEN_ACC_Msk (0x2UL) /*!< CLKGEN INTRPTEN: ACC (Bitfield-Mask: 0x01) */ +#define CLKGEN_INTRPTEN_ACF_Pos (0UL) /*!< CLKGEN INTRPTEN: ACF (Bit 0) */ +#define CLKGEN_INTRPTEN_ACF_Msk (0x1UL) /*!< CLKGEN INTRPTEN: ACF (Bitfield-Mask: 0x01) */ +/* ====================================================== INTRPTSTAT ======================================================= */ +#define CLKGEN_INTRPTSTAT_OF_Pos (2UL) /*!< CLKGEN INTRPTSTAT: OF (Bit 2) */ +#define CLKGEN_INTRPTSTAT_OF_Msk (0x4UL) /*!< CLKGEN INTRPTSTAT: OF (Bitfield-Mask: 0x01) */ +#define CLKGEN_INTRPTSTAT_ACC_Pos (1UL) /*!< CLKGEN INTRPTSTAT: ACC (Bit 1) */ +#define CLKGEN_INTRPTSTAT_ACC_Msk (0x2UL) /*!< CLKGEN INTRPTSTAT: ACC (Bitfield-Mask: 0x01) */ +#define CLKGEN_INTRPTSTAT_ACF_Pos (0UL) /*!< CLKGEN INTRPTSTAT: ACF (Bit 0) */ +#define CLKGEN_INTRPTSTAT_ACF_Msk (0x1UL) /*!< CLKGEN INTRPTSTAT: ACF (Bitfield-Mask: 0x01) */ +/* ======================================================= INTRPTCLR ======================================================= */ +#define CLKGEN_INTRPTCLR_OF_Pos (2UL) /*!< CLKGEN INTRPTCLR: OF (Bit 2) */ +#define CLKGEN_INTRPTCLR_OF_Msk (0x4UL) /*!< CLKGEN INTRPTCLR: OF (Bitfield-Mask: 0x01) */ +#define CLKGEN_INTRPTCLR_ACC_Pos (1UL) /*!< CLKGEN INTRPTCLR: ACC (Bit 1) */ +#define CLKGEN_INTRPTCLR_ACC_Msk (0x2UL) /*!< CLKGEN INTRPTCLR: ACC (Bitfield-Mask: 0x01) */ +#define CLKGEN_INTRPTCLR_ACF_Pos (0UL) /*!< CLKGEN INTRPTCLR: ACF (Bit 0) */ +#define CLKGEN_INTRPTCLR_ACF_Msk (0x1UL) /*!< CLKGEN INTRPTCLR: ACF (Bitfield-Mask: 0x01) */ +/* ======================================================= INTRPTSET ======================================================= */ +#define CLKGEN_INTRPTSET_OF_Pos (2UL) /*!< CLKGEN INTRPTSET: OF (Bit 2) */ +#define CLKGEN_INTRPTSET_OF_Msk (0x4UL) /*!< CLKGEN INTRPTSET: OF (Bitfield-Mask: 0x01) */ +#define CLKGEN_INTRPTSET_ACC_Pos (1UL) /*!< CLKGEN INTRPTSET: ACC (Bit 1) */ +#define CLKGEN_INTRPTSET_ACC_Msk (0x2UL) /*!< CLKGEN INTRPTSET: ACC (Bitfield-Mask: 0x01) */ +#define CLKGEN_INTRPTSET_ACF_Pos (0UL) /*!< CLKGEN INTRPTSET: ACF (Bit 0) */ +#define CLKGEN_INTRPTSET_ACF_Msk (0x1UL) /*!< CLKGEN INTRPTSET: ACF (Bitfield-Mask: 0x01) */ + + +/* =========================================================================================================================== */ +/* ================ CTIMER ================ */ +/* =========================================================================================================================== */ + +/* ========================================================= TMR0 ========================================================== */ +#define CTIMER_TMR0_CTTMRB0_Pos (16UL) /*!< CTIMER TMR0: CTTMRB0 (Bit 16) */ +#define CTIMER_TMR0_CTTMRB0_Msk (0xffff0000UL) /*!< CTIMER TMR0: CTTMRB0 (Bitfield-Mask: 0xffff) */ +#define CTIMER_TMR0_CTTMRA0_Pos (0UL) /*!< CTIMER TMR0: CTTMRA0 (Bit 0) */ +#define CTIMER_TMR0_CTTMRA0_Msk (0xffffUL) /*!< CTIMER TMR0: CTTMRA0 (Bitfield-Mask: 0xffff) */ +/* ======================================================== CMPRA0 ========================================================= */ +#define CTIMER_CMPRA0_CMPR1A0_Pos (16UL) /*!< CTIMER CMPRA0: CMPR1A0 (Bit 16) */ +#define CTIMER_CMPRA0_CMPR1A0_Msk (0xffff0000UL) /*!< CTIMER CMPRA0: CMPR1A0 (Bitfield-Mask: 0xffff) */ +#define CTIMER_CMPRA0_CMPR0A0_Pos (0UL) /*!< CTIMER CMPRA0: CMPR0A0 (Bit 0) */ +#define CTIMER_CMPRA0_CMPR0A0_Msk (0xffffUL) /*!< CTIMER CMPRA0: CMPR0A0 (Bitfield-Mask: 0xffff) */ +/* ======================================================== CMPRB0 ========================================================= */ +#define CTIMER_CMPRB0_CMPR1B0_Pos (16UL) /*!< CTIMER CMPRB0: CMPR1B0 (Bit 16) */ +#define CTIMER_CMPRB0_CMPR1B0_Msk (0xffff0000UL) /*!< CTIMER CMPRB0: CMPR1B0 (Bitfield-Mask: 0xffff) */ +#define CTIMER_CMPRB0_CMPR0B0_Pos (0UL) /*!< CTIMER CMPRB0: CMPR0B0 (Bit 0) */ +#define CTIMER_CMPRB0_CMPR0B0_Msk (0xffffUL) /*!< CTIMER CMPRB0: CMPR0B0 (Bitfield-Mask: 0xffff) */ +/* ========================================================= CTRL0 ========================================================= */ +#define CTIMER_CTRL0_CTLINK0_Pos (31UL) /*!< CTIMER CTRL0: CTLINK0 (Bit 31) */ +#define CTIMER_CTRL0_CTLINK0_Msk (0x80000000UL) /*!< CTIMER CTRL0: CTLINK0 (Bitfield-Mask: 0x01) */ +#define CTIMER_CTRL0_TMRB0POL_Pos (28UL) /*!< CTIMER CTRL0: TMRB0POL (Bit 28) */ +#define CTIMER_CTRL0_TMRB0POL_Msk (0x10000000UL) /*!< CTIMER CTRL0: TMRB0POL (Bitfield-Mask: 0x01) */ +#define CTIMER_CTRL0_TMRB0CLR_Pos (27UL) /*!< CTIMER CTRL0: TMRB0CLR (Bit 27) */ +#define CTIMER_CTRL0_TMRB0CLR_Msk (0x8000000UL) /*!< CTIMER CTRL0: TMRB0CLR (Bitfield-Mask: 0x01) */ +#define CTIMER_CTRL0_TMRB0IE1_Pos (26UL) /*!< CTIMER CTRL0: TMRB0IE1 (Bit 26) */ +#define CTIMER_CTRL0_TMRB0IE1_Msk (0x4000000UL) /*!< CTIMER CTRL0: TMRB0IE1 (Bitfield-Mask: 0x01) */ +#define CTIMER_CTRL0_TMRB0IE0_Pos (25UL) /*!< CTIMER CTRL0: TMRB0IE0 (Bit 25) */ +#define CTIMER_CTRL0_TMRB0IE0_Msk (0x2000000UL) /*!< CTIMER CTRL0: TMRB0IE0 (Bitfield-Mask: 0x01) */ +#define CTIMER_CTRL0_TMRB0FN_Pos (22UL) /*!< CTIMER CTRL0: TMRB0FN (Bit 22) */ +#define CTIMER_CTRL0_TMRB0FN_Msk (0x1c00000UL) /*!< CTIMER CTRL0: TMRB0FN (Bitfield-Mask: 0x07) */ +#define CTIMER_CTRL0_TMRB0CLK_Pos (17UL) /*!< CTIMER CTRL0: TMRB0CLK (Bit 17) */ +#define CTIMER_CTRL0_TMRB0CLK_Msk (0x3e0000UL) /*!< CTIMER CTRL0: TMRB0CLK (Bitfield-Mask: 0x1f) */ +#define CTIMER_CTRL0_TMRB0EN_Pos (16UL) /*!< CTIMER CTRL0: TMRB0EN (Bit 16) */ +#define CTIMER_CTRL0_TMRB0EN_Msk (0x10000UL) /*!< CTIMER CTRL0: TMRB0EN (Bitfield-Mask: 0x01) */ +#define CTIMER_CTRL0_TMRA0POL_Pos (12UL) /*!< CTIMER CTRL0: TMRA0POL (Bit 12) */ +#define CTIMER_CTRL0_TMRA0POL_Msk (0x1000UL) /*!< CTIMER CTRL0: TMRA0POL (Bitfield-Mask: 0x01) */ +#define CTIMER_CTRL0_TMRA0CLR_Pos (11UL) /*!< CTIMER CTRL0: TMRA0CLR (Bit 11) */ +#define CTIMER_CTRL0_TMRA0CLR_Msk (0x800UL) /*!< CTIMER CTRL0: TMRA0CLR (Bitfield-Mask: 0x01) */ +#define CTIMER_CTRL0_TMRA0IE1_Pos (10UL) /*!< CTIMER CTRL0: TMRA0IE1 (Bit 10) */ +#define CTIMER_CTRL0_TMRA0IE1_Msk (0x400UL) /*!< CTIMER CTRL0: TMRA0IE1 (Bitfield-Mask: 0x01) */ +#define CTIMER_CTRL0_TMRA0IE0_Pos (9UL) /*!< CTIMER CTRL0: TMRA0IE0 (Bit 9) */ +#define CTIMER_CTRL0_TMRA0IE0_Msk (0x200UL) /*!< CTIMER CTRL0: TMRA0IE0 (Bitfield-Mask: 0x01) */ +#define CTIMER_CTRL0_TMRA0FN_Pos (6UL) /*!< CTIMER CTRL0: TMRA0FN (Bit 6) */ +#define CTIMER_CTRL0_TMRA0FN_Msk (0x1c0UL) /*!< CTIMER CTRL0: TMRA0FN (Bitfield-Mask: 0x07) */ +#define CTIMER_CTRL0_TMRA0CLK_Pos (1UL) /*!< CTIMER CTRL0: TMRA0CLK (Bit 1) */ +#define CTIMER_CTRL0_TMRA0CLK_Msk (0x3eUL) /*!< CTIMER CTRL0: TMRA0CLK (Bitfield-Mask: 0x1f) */ +#define CTIMER_CTRL0_TMRA0EN_Pos (0UL) /*!< CTIMER CTRL0: TMRA0EN (Bit 0) */ +#define CTIMER_CTRL0_TMRA0EN_Msk (0x1UL) /*!< CTIMER CTRL0: TMRA0EN (Bitfield-Mask: 0x01) */ +/* ======================================================= CMPRAUXA0 ======================================================= */ +#define CTIMER_CMPRAUXA0_CMPR3A0_Pos (16UL) /*!< CTIMER CMPRAUXA0: CMPR3A0 (Bit 16) */ +#define CTIMER_CMPRAUXA0_CMPR3A0_Msk (0xffff0000UL) /*!< CTIMER CMPRAUXA0: CMPR3A0 (Bitfield-Mask: 0xffff) */ +#define CTIMER_CMPRAUXA0_CMPR2A0_Pos (0UL) /*!< CTIMER CMPRAUXA0: CMPR2A0 (Bit 0) */ +#define CTIMER_CMPRAUXA0_CMPR2A0_Msk (0xffffUL) /*!< CTIMER CMPRAUXA0: CMPR2A0 (Bitfield-Mask: 0xffff) */ +/* ======================================================= CMPRAUXB0 ======================================================= */ +#define CTIMER_CMPRAUXB0_CMPR3B0_Pos (16UL) /*!< CTIMER CMPRAUXB0: CMPR3B0 (Bit 16) */ +#define CTIMER_CMPRAUXB0_CMPR3B0_Msk (0xffff0000UL) /*!< CTIMER CMPRAUXB0: CMPR3B0 (Bitfield-Mask: 0xffff) */ +#define CTIMER_CMPRAUXB0_CMPR2B0_Pos (0UL) /*!< CTIMER CMPRAUXB0: CMPR2B0 (Bit 0) */ +#define CTIMER_CMPRAUXB0_CMPR2B0_Msk (0xffffUL) /*!< CTIMER CMPRAUXB0: CMPR2B0 (Bitfield-Mask: 0xffff) */ +/* ========================================================= AUX0 ========================================================== */ +#define CTIMER_AUX0_TMRB0EN23_Pos (30UL) /*!< CTIMER AUX0: TMRB0EN23 (Bit 30) */ +#define CTIMER_AUX0_TMRB0EN23_Msk (0x40000000UL) /*!< CTIMER AUX0: TMRB0EN23 (Bitfield-Mask: 0x01) */ +#define CTIMER_AUX0_TMRB0POL23_Pos (29UL) /*!< CTIMER AUX0: TMRB0POL23 (Bit 29) */ +#define CTIMER_AUX0_TMRB0POL23_Msk (0x20000000UL) /*!< CTIMER AUX0: TMRB0POL23 (Bitfield-Mask: 0x01) */ +#define CTIMER_AUX0_TMRB0TINV_Pos (28UL) /*!< CTIMER AUX0: TMRB0TINV (Bit 28) */ +#define CTIMER_AUX0_TMRB0TINV_Msk (0x10000000UL) /*!< CTIMER AUX0: TMRB0TINV (Bitfield-Mask: 0x01) */ +#define CTIMER_AUX0_TMRB0NOSYNC_Pos (27UL) /*!< CTIMER AUX0: TMRB0NOSYNC (Bit 27) */ +#define CTIMER_AUX0_TMRB0NOSYNC_Msk (0x8000000UL) /*!< CTIMER AUX0: TMRB0NOSYNC (Bitfield-Mask: 0x01) */ +#define CTIMER_AUX0_TMRB0TRIG_Pos (23UL) /*!< CTIMER AUX0: TMRB0TRIG (Bit 23) */ +#define CTIMER_AUX0_TMRB0TRIG_Msk (0x7800000UL) /*!< CTIMER AUX0: TMRB0TRIG (Bitfield-Mask: 0x0f) */ +#define CTIMER_AUX0_TMRB0LMT_Pos (16UL) /*!< CTIMER AUX0: TMRB0LMT (Bit 16) */ +#define CTIMER_AUX0_TMRB0LMT_Msk (0x3f0000UL) /*!< CTIMER AUX0: TMRB0LMT (Bitfield-Mask: 0x3f) */ +#define CTIMER_AUX0_TMRA0EN23_Pos (14UL) /*!< CTIMER AUX0: TMRA0EN23 (Bit 14) */ +#define CTIMER_AUX0_TMRA0EN23_Msk (0x4000UL) /*!< CTIMER AUX0: TMRA0EN23 (Bitfield-Mask: 0x01) */ +#define CTIMER_AUX0_TMRA0POL23_Pos (13UL) /*!< CTIMER AUX0: TMRA0POL23 (Bit 13) */ +#define CTIMER_AUX0_TMRA0POL23_Msk (0x2000UL) /*!< CTIMER AUX0: TMRA0POL23 (Bitfield-Mask: 0x01) */ +#define CTIMER_AUX0_TMRA0TINV_Pos (12UL) /*!< CTIMER AUX0: TMRA0TINV (Bit 12) */ +#define CTIMER_AUX0_TMRA0TINV_Msk (0x1000UL) /*!< CTIMER AUX0: TMRA0TINV (Bitfield-Mask: 0x01) */ +#define CTIMER_AUX0_TMRA0NOSYNC_Pos (11UL) /*!< CTIMER AUX0: TMRA0NOSYNC (Bit 11) */ +#define CTIMER_AUX0_TMRA0NOSYNC_Msk (0x800UL) /*!< CTIMER AUX0: TMRA0NOSYNC (Bitfield-Mask: 0x01) */ +#define CTIMER_AUX0_TMRA0TRIG_Pos (7UL) /*!< CTIMER AUX0: TMRA0TRIG (Bit 7) */ +#define CTIMER_AUX0_TMRA0TRIG_Msk (0x780UL) /*!< CTIMER AUX0: TMRA0TRIG (Bitfield-Mask: 0x0f) */ +#define CTIMER_AUX0_TMRA0LMT_Pos (0UL) /*!< CTIMER AUX0: TMRA0LMT (Bit 0) */ +#define CTIMER_AUX0_TMRA0LMT_Msk (0x7fUL) /*!< CTIMER AUX0: TMRA0LMT (Bitfield-Mask: 0x7f) */ +/* ========================================================= TMR1 ========================================================== */ +#define CTIMER_TMR1_CTTMRB1_Pos (16UL) /*!< CTIMER TMR1: CTTMRB1 (Bit 16) */ +#define CTIMER_TMR1_CTTMRB1_Msk (0xffff0000UL) /*!< CTIMER TMR1: CTTMRB1 (Bitfield-Mask: 0xffff) */ +#define CTIMER_TMR1_CTTMRA1_Pos (0UL) /*!< CTIMER TMR1: CTTMRA1 (Bit 0) */ +#define CTIMER_TMR1_CTTMRA1_Msk (0xffffUL) /*!< CTIMER TMR1: CTTMRA1 (Bitfield-Mask: 0xffff) */ +/* ======================================================== CMPRA1 ========================================================= */ +#define CTIMER_CMPRA1_CMPR1A1_Pos (16UL) /*!< CTIMER CMPRA1: CMPR1A1 (Bit 16) */ +#define CTIMER_CMPRA1_CMPR1A1_Msk (0xffff0000UL) /*!< CTIMER CMPRA1: CMPR1A1 (Bitfield-Mask: 0xffff) */ +#define CTIMER_CMPRA1_CMPR0A1_Pos (0UL) /*!< CTIMER CMPRA1: CMPR0A1 (Bit 0) */ +#define CTIMER_CMPRA1_CMPR0A1_Msk (0xffffUL) /*!< CTIMER CMPRA1: CMPR0A1 (Bitfield-Mask: 0xffff) */ +/* ======================================================== CMPRB1 ========================================================= */ +#define CTIMER_CMPRB1_CMPR1B1_Pos (16UL) /*!< CTIMER CMPRB1: CMPR1B1 (Bit 16) */ +#define CTIMER_CMPRB1_CMPR1B1_Msk (0xffff0000UL) /*!< CTIMER CMPRB1: CMPR1B1 (Bitfield-Mask: 0xffff) */ +#define CTIMER_CMPRB1_CMPR0B1_Pos (0UL) /*!< CTIMER CMPRB1: CMPR0B1 (Bit 0) */ +#define CTIMER_CMPRB1_CMPR0B1_Msk (0xffffUL) /*!< CTIMER CMPRB1: CMPR0B1 (Bitfield-Mask: 0xffff) */ +/* ========================================================= CTRL1 ========================================================= */ +#define CTIMER_CTRL1_CTLINK1_Pos (31UL) /*!< CTIMER CTRL1: CTLINK1 (Bit 31) */ +#define CTIMER_CTRL1_CTLINK1_Msk (0x80000000UL) /*!< CTIMER CTRL1: CTLINK1 (Bitfield-Mask: 0x01) */ +#define CTIMER_CTRL1_TMRB1POL_Pos (28UL) /*!< CTIMER CTRL1: TMRB1POL (Bit 28) */ +#define CTIMER_CTRL1_TMRB1POL_Msk (0x10000000UL) /*!< CTIMER CTRL1: TMRB1POL (Bitfield-Mask: 0x01) */ +#define CTIMER_CTRL1_TMRB1CLR_Pos (27UL) /*!< CTIMER CTRL1: TMRB1CLR (Bit 27) */ +#define CTIMER_CTRL1_TMRB1CLR_Msk (0x8000000UL) /*!< CTIMER CTRL1: TMRB1CLR (Bitfield-Mask: 0x01) */ +#define CTIMER_CTRL1_TMRB1IE1_Pos (26UL) /*!< CTIMER CTRL1: TMRB1IE1 (Bit 26) */ +#define CTIMER_CTRL1_TMRB1IE1_Msk (0x4000000UL) /*!< CTIMER CTRL1: TMRB1IE1 (Bitfield-Mask: 0x01) */ +#define CTIMER_CTRL1_TMRB1IE0_Pos (25UL) /*!< CTIMER CTRL1: TMRB1IE0 (Bit 25) */ +#define CTIMER_CTRL1_TMRB1IE0_Msk (0x2000000UL) /*!< CTIMER CTRL1: TMRB1IE0 (Bitfield-Mask: 0x01) */ +#define CTIMER_CTRL1_TMRB1FN_Pos (22UL) /*!< CTIMER CTRL1: TMRB1FN (Bit 22) */ +#define CTIMER_CTRL1_TMRB1FN_Msk (0x1c00000UL) /*!< CTIMER CTRL1: TMRB1FN (Bitfield-Mask: 0x07) */ +#define CTIMER_CTRL1_TMRB1CLK_Pos (17UL) /*!< CTIMER CTRL1: TMRB1CLK (Bit 17) */ +#define CTIMER_CTRL1_TMRB1CLK_Msk (0x3e0000UL) /*!< CTIMER CTRL1: TMRB1CLK (Bitfield-Mask: 0x1f) */ +#define CTIMER_CTRL1_TMRB1EN_Pos (16UL) /*!< CTIMER CTRL1: TMRB1EN (Bit 16) */ +#define CTIMER_CTRL1_TMRB1EN_Msk (0x10000UL) /*!< CTIMER CTRL1: TMRB1EN (Bitfield-Mask: 0x01) */ +#define CTIMER_CTRL1_TMRA1POL_Pos (12UL) /*!< CTIMER CTRL1: TMRA1POL (Bit 12) */ +#define CTIMER_CTRL1_TMRA1POL_Msk (0x1000UL) /*!< CTIMER CTRL1: TMRA1POL (Bitfield-Mask: 0x01) */ +#define CTIMER_CTRL1_TMRA1CLR_Pos (11UL) /*!< CTIMER CTRL1: TMRA1CLR (Bit 11) */ +#define CTIMER_CTRL1_TMRA1CLR_Msk (0x800UL) /*!< CTIMER CTRL1: TMRA1CLR (Bitfield-Mask: 0x01) */ +#define CTIMER_CTRL1_TMRA1IE1_Pos (10UL) /*!< CTIMER CTRL1: TMRA1IE1 (Bit 10) */ +#define CTIMER_CTRL1_TMRA1IE1_Msk (0x400UL) /*!< CTIMER CTRL1: TMRA1IE1 (Bitfield-Mask: 0x01) */ +#define CTIMER_CTRL1_TMRA1IE0_Pos (9UL) /*!< CTIMER CTRL1: TMRA1IE0 (Bit 9) */ +#define CTIMER_CTRL1_TMRA1IE0_Msk (0x200UL) /*!< CTIMER CTRL1: TMRA1IE0 (Bitfield-Mask: 0x01) */ +#define CTIMER_CTRL1_TMRA1FN_Pos (6UL) /*!< CTIMER CTRL1: TMRA1FN (Bit 6) */ +#define CTIMER_CTRL1_TMRA1FN_Msk (0x1c0UL) /*!< CTIMER CTRL1: TMRA1FN (Bitfield-Mask: 0x07) */ +#define CTIMER_CTRL1_TMRA1CLK_Pos (1UL) /*!< CTIMER CTRL1: TMRA1CLK (Bit 1) */ +#define CTIMER_CTRL1_TMRA1CLK_Msk (0x3eUL) /*!< CTIMER CTRL1: TMRA1CLK (Bitfield-Mask: 0x1f) */ +#define CTIMER_CTRL1_TMRA1EN_Pos (0UL) /*!< CTIMER CTRL1: TMRA1EN (Bit 0) */ +#define CTIMER_CTRL1_TMRA1EN_Msk (0x1UL) /*!< CTIMER CTRL1: TMRA1EN (Bitfield-Mask: 0x01) */ +/* ======================================================= CMPRAUXA1 ======================================================= */ +#define CTIMER_CMPRAUXA1_CMPR3A1_Pos (16UL) /*!< CTIMER CMPRAUXA1: CMPR3A1 (Bit 16) */ +#define CTIMER_CMPRAUXA1_CMPR3A1_Msk (0xffff0000UL) /*!< CTIMER CMPRAUXA1: CMPR3A1 (Bitfield-Mask: 0xffff) */ +#define CTIMER_CMPRAUXA1_CMPR2A1_Pos (0UL) /*!< CTIMER CMPRAUXA1: CMPR2A1 (Bit 0) */ +#define CTIMER_CMPRAUXA1_CMPR2A1_Msk (0xffffUL) /*!< CTIMER CMPRAUXA1: CMPR2A1 (Bitfield-Mask: 0xffff) */ +/* ======================================================= CMPRAUXB1 ======================================================= */ +#define CTIMER_CMPRAUXB1_CMPR3B1_Pos (16UL) /*!< CTIMER CMPRAUXB1: CMPR3B1 (Bit 16) */ +#define CTIMER_CMPRAUXB1_CMPR3B1_Msk (0xffff0000UL) /*!< CTIMER CMPRAUXB1: CMPR3B1 (Bitfield-Mask: 0xffff) */ +#define CTIMER_CMPRAUXB1_CMPR2B1_Pos (0UL) /*!< CTIMER CMPRAUXB1: CMPR2B1 (Bit 0) */ +#define CTIMER_CMPRAUXB1_CMPR2B1_Msk (0xffffUL) /*!< CTIMER CMPRAUXB1: CMPR2B1 (Bitfield-Mask: 0xffff) */ +/* ========================================================= AUX1 ========================================================== */ +#define CTIMER_AUX1_TMRB1EN23_Pos (30UL) /*!< CTIMER AUX1: TMRB1EN23 (Bit 30) */ +#define CTIMER_AUX1_TMRB1EN23_Msk (0x40000000UL) /*!< CTIMER AUX1: TMRB1EN23 (Bitfield-Mask: 0x01) */ +#define CTIMER_AUX1_TMRB1POL23_Pos (29UL) /*!< CTIMER AUX1: TMRB1POL23 (Bit 29) */ +#define CTIMER_AUX1_TMRB1POL23_Msk (0x20000000UL) /*!< CTIMER AUX1: TMRB1POL23 (Bitfield-Mask: 0x01) */ +#define CTIMER_AUX1_TMRB1TINV_Pos (28UL) /*!< CTIMER AUX1: TMRB1TINV (Bit 28) */ +#define CTIMER_AUX1_TMRB1TINV_Msk (0x10000000UL) /*!< CTIMER AUX1: TMRB1TINV (Bitfield-Mask: 0x01) */ +#define CTIMER_AUX1_TMRB1NOSYNC_Pos (27UL) /*!< CTIMER AUX1: TMRB1NOSYNC (Bit 27) */ +#define CTIMER_AUX1_TMRB1NOSYNC_Msk (0x8000000UL) /*!< CTIMER AUX1: TMRB1NOSYNC (Bitfield-Mask: 0x01) */ +#define CTIMER_AUX1_TMRB1TRIG_Pos (23UL) /*!< CTIMER AUX1: TMRB1TRIG (Bit 23) */ +#define CTIMER_AUX1_TMRB1TRIG_Msk (0x7800000UL) /*!< CTIMER AUX1: TMRB1TRIG (Bitfield-Mask: 0x0f) */ +#define CTIMER_AUX1_TMRB1LMT_Pos (16UL) /*!< CTIMER AUX1: TMRB1LMT (Bit 16) */ +#define CTIMER_AUX1_TMRB1LMT_Msk (0x3f0000UL) /*!< CTIMER AUX1: TMRB1LMT (Bitfield-Mask: 0x3f) */ +#define CTIMER_AUX1_TMRA1EN23_Pos (14UL) /*!< CTIMER AUX1: TMRA1EN23 (Bit 14) */ +#define CTIMER_AUX1_TMRA1EN23_Msk (0x4000UL) /*!< CTIMER AUX1: TMRA1EN23 (Bitfield-Mask: 0x01) */ +#define CTIMER_AUX1_TMRA1POL23_Pos (13UL) /*!< CTIMER AUX1: TMRA1POL23 (Bit 13) */ +#define CTIMER_AUX1_TMRA1POL23_Msk (0x2000UL) /*!< CTIMER AUX1: TMRA1POL23 (Bitfield-Mask: 0x01) */ +#define CTIMER_AUX1_TMRA1TINV_Pos (12UL) /*!< CTIMER AUX1: TMRA1TINV (Bit 12) */ +#define CTIMER_AUX1_TMRA1TINV_Msk (0x1000UL) /*!< CTIMER AUX1: TMRA1TINV (Bitfield-Mask: 0x01) */ +#define CTIMER_AUX1_TMRA1NOSYNC_Pos (11UL) /*!< CTIMER AUX1: TMRA1NOSYNC (Bit 11) */ +#define CTIMER_AUX1_TMRA1NOSYNC_Msk (0x800UL) /*!< CTIMER AUX1: TMRA1NOSYNC (Bitfield-Mask: 0x01) */ +#define CTIMER_AUX1_TMRA1TRIG_Pos (7UL) /*!< CTIMER AUX1: TMRA1TRIG (Bit 7) */ +#define CTIMER_AUX1_TMRA1TRIG_Msk (0x780UL) /*!< CTIMER AUX1: TMRA1TRIG (Bitfield-Mask: 0x0f) */ +#define CTIMER_AUX1_TMRA1LMT_Pos (0UL) /*!< CTIMER AUX1: TMRA1LMT (Bit 0) */ +#define CTIMER_AUX1_TMRA1LMT_Msk (0x7fUL) /*!< CTIMER AUX1: TMRA1LMT (Bitfield-Mask: 0x7f) */ +/* ========================================================= TMR2 ========================================================== */ +#define CTIMER_TMR2_CTTMRB2_Pos (16UL) /*!< CTIMER TMR2: CTTMRB2 (Bit 16) */ +#define CTIMER_TMR2_CTTMRB2_Msk (0xffff0000UL) /*!< CTIMER TMR2: CTTMRB2 (Bitfield-Mask: 0xffff) */ +#define CTIMER_TMR2_CTTMRA2_Pos (0UL) /*!< CTIMER TMR2: CTTMRA2 (Bit 0) */ +#define CTIMER_TMR2_CTTMRA2_Msk (0xffffUL) /*!< CTIMER TMR2: CTTMRA2 (Bitfield-Mask: 0xffff) */ +/* ======================================================== CMPRA2 ========================================================= */ +#define CTIMER_CMPRA2_CMPR1A2_Pos (16UL) /*!< CTIMER CMPRA2: CMPR1A2 (Bit 16) */ +#define CTIMER_CMPRA2_CMPR1A2_Msk (0xffff0000UL) /*!< CTIMER CMPRA2: CMPR1A2 (Bitfield-Mask: 0xffff) */ +#define CTIMER_CMPRA2_CMPR0A2_Pos (0UL) /*!< CTIMER CMPRA2: CMPR0A2 (Bit 0) */ +#define CTIMER_CMPRA2_CMPR0A2_Msk (0xffffUL) /*!< CTIMER CMPRA2: CMPR0A2 (Bitfield-Mask: 0xffff) */ +/* ======================================================== CMPRB2 ========================================================= */ +#define CTIMER_CMPRB2_CMPR1B2_Pos (16UL) /*!< CTIMER CMPRB2: CMPR1B2 (Bit 16) */ +#define CTIMER_CMPRB2_CMPR1B2_Msk (0xffff0000UL) /*!< CTIMER CMPRB2: CMPR1B2 (Bitfield-Mask: 0xffff) */ +#define CTIMER_CMPRB2_CMPR0B2_Pos (0UL) /*!< CTIMER CMPRB2: CMPR0B2 (Bit 0) */ +#define CTIMER_CMPRB2_CMPR0B2_Msk (0xffffUL) /*!< CTIMER CMPRB2: CMPR0B2 (Bitfield-Mask: 0xffff) */ +/* ========================================================= CTRL2 ========================================================= */ +#define CTIMER_CTRL2_CTLINK2_Pos (31UL) /*!< CTIMER CTRL2: CTLINK2 (Bit 31) */ +#define CTIMER_CTRL2_CTLINK2_Msk (0x80000000UL) /*!< CTIMER CTRL2: CTLINK2 (Bitfield-Mask: 0x01) */ +#define CTIMER_CTRL2_TMRB2POL_Pos (28UL) /*!< CTIMER CTRL2: TMRB2POL (Bit 28) */ +#define CTIMER_CTRL2_TMRB2POL_Msk (0x10000000UL) /*!< CTIMER CTRL2: TMRB2POL (Bitfield-Mask: 0x01) */ +#define CTIMER_CTRL2_TMRB2CLR_Pos (27UL) /*!< CTIMER CTRL2: TMRB2CLR (Bit 27) */ +#define CTIMER_CTRL2_TMRB2CLR_Msk (0x8000000UL) /*!< CTIMER CTRL2: TMRB2CLR (Bitfield-Mask: 0x01) */ +#define CTIMER_CTRL2_TMRB2IE1_Pos (26UL) /*!< CTIMER CTRL2: TMRB2IE1 (Bit 26) */ +#define CTIMER_CTRL2_TMRB2IE1_Msk (0x4000000UL) /*!< CTIMER CTRL2: TMRB2IE1 (Bitfield-Mask: 0x01) */ +#define CTIMER_CTRL2_TMRB2IE0_Pos (25UL) /*!< CTIMER CTRL2: TMRB2IE0 (Bit 25) */ +#define CTIMER_CTRL2_TMRB2IE0_Msk (0x2000000UL) /*!< CTIMER CTRL2: TMRB2IE0 (Bitfield-Mask: 0x01) */ +#define CTIMER_CTRL2_TMRB2FN_Pos (22UL) /*!< CTIMER CTRL2: TMRB2FN (Bit 22) */ +#define CTIMER_CTRL2_TMRB2FN_Msk (0x1c00000UL) /*!< CTIMER CTRL2: TMRB2FN (Bitfield-Mask: 0x07) */ +#define CTIMER_CTRL2_TMRB2CLK_Pos (17UL) /*!< CTIMER CTRL2: TMRB2CLK (Bit 17) */ +#define CTIMER_CTRL2_TMRB2CLK_Msk (0x3e0000UL) /*!< CTIMER CTRL2: TMRB2CLK (Bitfield-Mask: 0x1f) */ +#define CTIMER_CTRL2_TMRB2EN_Pos (16UL) /*!< CTIMER CTRL2: TMRB2EN (Bit 16) */ +#define CTIMER_CTRL2_TMRB2EN_Msk (0x10000UL) /*!< CTIMER CTRL2: TMRB2EN (Bitfield-Mask: 0x01) */ +#define CTIMER_CTRL2_TMRA2POL_Pos (12UL) /*!< CTIMER CTRL2: TMRA2POL (Bit 12) */ +#define CTIMER_CTRL2_TMRA2POL_Msk (0x1000UL) /*!< CTIMER CTRL2: TMRA2POL (Bitfield-Mask: 0x01) */ +#define CTIMER_CTRL2_TMRA2CLR_Pos (11UL) /*!< CTIMER CTRL2: TMRA2CLR (Bit 11) */ +#define CTIMER_CTRL2_TMRA2CLR_Msk (0x800UL) /*!< CTIMER CTRL2: TMRA2CLR (Bitfield-Mask: 0x01) */ +#define CTIMER_CTRL2_TMRA2IE1_Pos (10UL) /*!< CTIMER CTRL2: TMRA2IE1 (Bit 10) */ +#define CTIMER_CTRL2_TMRA2IE1_Msk (0x400UL) /*!< CTIMER CTRL2: TMRA2IE1 (Bitfield-Mask: 0x01) */ +#define CTIMER_CTRL2_TMRA2IE0_Pos (9UL) /*!< CTIMER CTRL2: TMRA2IE0 (Bit 9) */ +#define CTIMER_CTRL2_TMRA2IE0_Msk (0x200UL) /*!< CTIMER CTRL2: TMRA2IE0 (Bitfield-Mask: 0x01) */ +#define CTIMER_CTRL2_TMRA2FN_Pos (6UL) /*!< CTIMER CTRL2: TMRA2FN (Bit 6) */ +#define CTIMER_CTRL2_TMRA2FN_Msk (0x1c0UL) /*!< CTIMER CTRL2: TMRA2FN (Bitfield-Mask: 0x07) */ +#define CTIMER_CTRL2_TMRA2CLK_Pos (1UL) /*!< CTIMER CTRL2: TMRA2CLK (Bit 1) */ +#define CTIMER_CTRL2_TMRA2CLK_Msk (0x3eUL) /*!< CTIMER CTRL2: TMRA2CLK (Bitfield-Mask: 0x1f) */ +#define CTIMER_CTRL2_TMRA2EN_Pos (0UL) /*!< CTIMER CTRL2: TMRA2EN (Bit 0) */ +#define CTIMER_CTRL2_TMRA2EN_Msk (0x1UL) /*!< CTIMER CTRL2: TMRA2EN (Bitfield-Mask: 0x01) */ +/* ======================================================= CMPRAUXA2 ======================================================= */ +#define CTIMER_CMPRAUXA2_CMPR3A2_Pos (16UL) /*!< CTIMER CMPRAUXA2: CMPR3A2 (Bit 16) */ +#define CTIMER_CMPRAUXA2_CMPR3A2_Msk (0xffff0000UL) /*!< CTIMER CMPRAUXA2: CMPR3A2 (Bitfield-Mask: 0xffff) */ +#define CTIMER_CMPRAUXA2_CMPR2A2_Pos (0UL) /*!< CTIMER CMPRAUXA2: CMPR2A2 (Bit 0) */ +#define CTIMER_CMPRAUXA2_CMPR2A2_Msk (0xffffUL) /*!< CTIMER CMPRAUXA2: CMPR2A2 (Bitfield-Mask: 0xffff) */ +/* ======================================================= CMPRAUXB2 ======================================================= */ +#define CTIMER_CMPRAUXB2_CMPR3B2_Pos (16UL) /*!< CTIMER CMPRAUXB2: CMPR3B2 (Bit 16) */ +#define CTIMER_CMPRAUXB2_CMPR3B2_Msk (0xffff0000UL) /*!< CTIMER CMPRAUXB2: CMPR3B2 (Bitfield-Mask: 0xffff) */ +#define CTIMER_CMPRAUXB2_CMPR2B2_Pos (0UL) /*!< CTIMER CMPRAUXB2: CMPR2B2 (Bit 0) */ +#define CTIMER_CMPRAUXB2_CMPR2B2_Msk (0xffffUL) /*!< CTIMER CMPRAUXB2: CMPR2B2 (Bitfield-Mask: 0xffff) */ +/* ========================================================= AUX2 ========================================================== */ +#define CTIMER_AUX2_TMRB2EN23_Pos (30UL) /*!< CTIMER AUX2: TMRB2EN23 (Bit 30) */ +#define CTIMER_AUX2_TMRB2EN23_Msk (0x40000000UL) /*!< CTIMER AUX2: TMRB2EN23 (Bitfield-Mask: 0x01) */ +#define CTIMER_AUX2_TMRB2POL23_Pos (29UL) /*!< CTIMER AUX2: TMRB2POL23 (Bit 29) */ +#define CTIMER_AUX2_TMRB2POL23_Msk (0x20000000UL) /*!< CTIMER AUX2: TMRB2POL23 (Bitfield-Mask: 0x01) */ +#define CTIMER_AUX2_TMRB2TINV_Pos (28UL) /*!< CTIMER AUX2: TMRB2TINV (Bit 28) */ +#define CTIMER_AUX2_TMRB2TINV_Msk (0x10000000UL) /*!< CTIMER AUX2: TMRB2TINV (Bitfield-Mask: 0x01) */ +#define CTIMER_AUX2_TMRB2NOSYNC_Pos (27UL) /*!< CTIMER AUX2: TMRB2NOSYNC (Bit 27) */ +#define CTIMER_AUX2_TMRB2NOSYNC_Msk (0x8000000UL) /*!< CTIMER AUX2: TMRB2NOSYNC (Bitfield-Mask: 0x01) */ +#define CTIMER_AUX2_TMRB2TRIG_Pos (23UL) /*!< CTIMER AUX2: TMRB2TRIG (Bit 23) */ +#define CTIMER_AUX2_TMRB2TRIG_Msk (0x7800000UL) /*!< CTIMER AUX2: TMRB2TRIG (Bitfield-Mask: 0x0f) */ +#define CTIMER_AUX2_TMRB2LMT_Pos (16UL) /*!< CTIMER AUX2: TMRB2LMT (Bit 16) */ +#define CTIMER_AUX2_TMRB2LMT_Msk (0x3f0000UL) /*!< CTIMER AUX2: TMRB2LMT (Bitfield-Mask: 0x3f) */ +#define CTIMER_AUX2_TMRA2EN23_Pos (14UL) /*!< CTIMER AUX2: TMRA2EN23 (Bit 14) */ +#define CTIMER_AUX2_TMRA2EN23_Msk (0x4000UL) /*!< CTIMER AUX2: TMRA2EN23 (Bitfield-Mask: 0x01) */ +#define CTIMER_AUX2_TMRA2POL23_Pos (13UL) /*!< CTIMER AUX2: TMRA2POL23 (Bit 13) */ +#define CTIMER_AUX2_TMRA2POL23_Msk (0x2000UL) /*!< CTIMER AUX2: TMRA2POL23 (Bitfield-Mask: 0x01) */ +#define CTIMER_AUX2_TMRA2TINV_Pos (12UL) /*!< CTIMER AUX2: TMRA2TINV (Bit 12) */ +#define CTIMER_AUX2_TMRA2TINV_Msk (0x1000UL) /*!< CTIMER AUX2: TMRA2TINV (Bitfield-Mask: 0x01) */ +#define CTIMER_AUX2_TMRA2NOSYNC_Pos (11UL) /*!< CTIMER AUX2: TMRA2NOSYNC (Bit 11) */ +#define CTIMER_AUX2_TMRA2NOSYNC_Msk (0x800UL) /*!< CTIMER AUX2: TMRA2NOSYNC (Bitfield-Mask: 0x01) */ +#define CTIMER_AUX2_TMRA2TRIG_Pos (7UL) /*!< CTIMER AUX2: TMRA2TRIG (Bit 7) */ +#define CTIMER_AUX2_TMRA2TRIG_Msk (0x780UL) /*!< CTIMER AUX2: TMRA2TRIG (Bitfield-Mask: 0x0f) */ +#define CTIMER_AUX2_TMRA2LMT_Pos (0UL) /*!< CTIMER AUX2: TMRA2LMT (Bit 0) */ +#define CTIMER_AUX2_TMRA2LMT_Msk (0x7fUL) /*!< CTIMER AUX2: TMRA2LMT (Bitfield-Mask: 0x7f) */ +/* ========================================================= TMR3 ========================================================== */ +#define CTIMER_TMR3_CTTMRB3_Pos (16UL) /*!< CTIMER TMR3: CTTMRB3 (Bit 16) */ +#define CTIMER_TMR3_CTTMRB3_Msk (0xffff0000UL) /*!< CTIMER TMR3: CTTMRB3 (Bitfield-Mask: 0xffff) */ +#define CTIMER_TMR3_CTTMRA3_Pos (0UL) /*!< CTIMER TMR3: CTTMRA3 (Bit 0) */ +#define CTIMER_TMR3_CTTMRA3_Msk (0xffffUL) /*!< CTIMER TMR3: CTTMRA3 (Bitfield-Mask: 0xffff) */ +/* ======================================================== CMPRA3 ========================================================= */ +#define CTIMER_CMPRA3_CMPR1A3_Pos (16UL) /*!< CTIMER CMPRA3: CMPR1A3 (Bit 16) */ +#define CTIMER_CMPRA3_CMPR1A3_Msk (0xffff0000UL) /*!< CTIMER CMPRA3: CMPR1A3 (Bitfield-Mask: 0xffff) */ +#define CTIMER_CMPRA3_CMPR0A3_Pos (0UL) /*!< CTIMER CMPRA3: CMPR0A3 (Bit 0) */ +#define CTIMER_CMPRA3_CMPR0A3_Msk (0xffffUL) /*!< CTIMER CMPRA3: CMPR0A3 (Bitfield-Mask: 0xffff) */ +/* ======================================================== CMPRB3 ========================================================= */ +#define CTIMER_CMPRB3_CMPR1B3_Pos (16UL) /*!< CTIMER CMPRB3: CMPR1B3 (Bit 16) */ +#define CTIMER_CMPRB3_CMPR1B3_Msk (0xffff0000UL) /*!< CTIMER CMPRB3: CMPR1B3 (Bitfield-Mask: 0xffff) */ +#define CTIMER_CMPRB3_CMPR0B3_Pos (0UL) /*!< CTIMER CMPRB3: CMPR0B3 (Bit 0) */ +#define CTIMER_CMPRB3_CMPR0B3_Msk (0xffffUL) /*!< CTIMER CMPRB3: CMPR0B3 (Bitfield-Mask: 0xffff) */ +/* ========================================================= CTRL3 ========================================================= */ +#define CTIMER_CTRL3_CTLINK3_Pos (31UL) /*!< CTIMER CTRL3: CTLINK3 (Bit 31) */ +#define CTIMER_CTRL3_CTLINK3_Msk (0x80000000UL) /*!< CTIMER CTRL3: CTLINK3 (Bitfield-Mask: 0x01) */ +#define CTIMER_CTRL3_TMRB3POL_Pos (28UL) /*!< CTIMER CTRL3: TMRB3POL (Bit 28) */ +#define CTIMER_CTRL3_TMRB3POL_Msk (0x10000000UL) /*!< CTIMER CTRL3: TMRB3POL (Bitfield-Mask: 0x01) */ +#define CTIMER_CTRL3_TMRB3CLR_Pos (27UL) /*!< CTIMER CTRL3: TMRB3CLR (Bit 27) */ +#define CTIMER_CTRL3_TMRB3CLR_Msk (0x8000000UL) /*!< CTIMER CTRL3: TMRB3CLR (Bitfield-Mask: 0x01) */ +#define CTIMER_CTRL3_TMRB3IE1_Pos (26UL) /*!< CTIMER CTRL3: TMRB3IE1 (Bit 26) */ +#define CTIMER_CTRL3_TMRB3IE1_Msk (0x4000000UL) /*!< CTIMER CTRL3: TMRB3IE1 (Bitfield-Mask: 0x01) */ +#define CTIMER_CTRL3_TMRB3IE0_Pos (25UL) /*!< CTIMER CTRL3: TMRB3IE0 (Bit 25) */ +#define CTIMER_CTRL3_TMRB3IE0_Msk (0x2000000UL) /*!< CTIMER CTRL3: TMRB3IE0 (Bitfield-Mask: 0x01) */ +#define CTIMER_CTRL3_TMRB3FN_Pos (22UL) /*!< CTIMER CTRL3: TMRB3FN (Bit 22) */ +#define CTIMER_CTRL3_TMRB3FN_Msk (0x1c00000UL) /*!< CTIMER CTRL3: TMRB3FN (Bitfield-Mask: 0x07) */ +#define CTIMER_CTRL3_TMRB3CLK_Pos (17UL) /*!< CTIMER CTRL3: TMRB3CLK (Bit 17) */ +#define CTIMER_CTRL3_TMRB3CLK_Msk (0x3e0000UL) /*!< CTIMER CTRL3: TMRB3CLK (Bitfield-Mask: 0x1f) */ +#define CTIMER_CTRL3_TMRB3EN_Pos (16UL) /*!< CTIMER CTRL3: TMRB3EN (Bit 16) */ +#define CTIMER_CTRL3_TMRB3EN_Msk (0x10000UL) /*!< CTIMER CTRL3: TMRB3EN (Bitfield-Mask: 0x01) */ +#define CTIMER_CTRL3_ADCEN_Pos (15UL) /*!< CTIMER CTRL3: ADCEN (Bit 15) */ +#define CTIMER_CTRL3_ADCEN_Msk (0x8000UL) /*!< CTIMER CTRL3: ADCEN (Bitfield-Mask: 0x01) */ +#define CTIMER_CTRL3_TMRA3POL_Pos (12UL) /*!< CTIMER CTRL3: TMRA3POL (Bit 12) */ +#define CTIMER_CTRL3_TMRA3POL_Msk (0x1000UL) /*!< CTIMER CTRL3: TMRA3POL (Bitfield-Mask: 0x01) */ +#define CTIMER_CTRL3_TMRA3CLR_Pos (11UL) /*!< CTIMER CTRL3: TMRA3CLR (Bit 11) */ +#define CTIMER_CTRL3_TMRA3CLR_Msk (0x800UL) /*!< CTIMER CTRL3: TMRA3CLR (Bitfield-Mask: 0x01) */ +#define CTIMER_CTRL3_TMRA3IE1_Pos (10UL) /*!< CTIMER CTRL3: TMRA3IE1 (Bit 10) */ +#define CTIMER_CTRL3_TMRA3IE1_Msk (0x400UL) /*!< CTIMER CTRL3: TMRA3IE1 (Bitfield-Mask: 0x01) */ +#define CTIMER_CTRL3_TMRA3IE0_Pos (9UL) /*!< CTIMER CTRL3: TMRA3IE0 (Bit 9) */ +#define CTIMER_CTRL3_TMRA3IE0_Msk (0x200UL) /*!< CTIMER CTRL3: TMRA3IE0 (Bitfield-Mask: 0x01) */ +#define CTIMER_CTRL3_TMRA3FN_Pos (6UL) /*!< CTIMER CTRL3: TMRA3FN (Bit 6) */ +#define CTIMER_CTRL3_TMRA3FN_Msk (0x1c0UL) /*!< CTIMER CTRL3: TMRA3FN (Bitfield-Mask: 0x07) */ +#define CTIMER_CTRL3_TMRA3CLK_Pos (1UL) /*!< CTIMER CTRL3: TMRA3CLK (Bit 1) */ +#define CTIMER_CTRL3_TMRA3CLK_Msk (0x3eUL) /*!< CTIMER CTRL3: TMRA3CLK (Bitfield-Mask: 0x1f) */ +#define CTIMER_CTRL3_TMRA3EN_Pos (0UL) /*!< CTIMER CTRL3: TMRA3EN (Bit 0) */ +#define CTIMER_CTRL3_TMRA3EN_Msk (0x1UL) /*!< CTIMER CTRL3: TMRA3EN (Bitfield-Mask: 0x01) */ +/* ======================================================= CMPRAUXA3 ======================================================= */ +#define CTIMER_CMPRAUXA3_CMPR3A3_Pos (16UL) /*!< CTIMER CMPRAUXA3: CMPR3A3 (Bit 16) */ +#define CTIMER_CMPRAUXA3_CMPR3A3_Msk (0xffff0000UL) /*!< CTIMER CMPRAUXA3: CMPR3A3 (Bitfield-Mask: 0xffff) */ +#define CTIMER_CMPRAUXA3_CMPR2A3_Pos (0UL) /*!< CTIMER CMPRAUXA3: CMPR2A3 (Bit 0) */ +#define CTIMER_CMPRAUXA3_CMPR2A3_Msk (0xffffUL) /*!< CTIMER CMPRAUXA3: CMPR2A3 (Bitfield-Mask: 0xffff) */ +/* ======================================================= CMPRAUXB3 ======================================================= */ +#define CTIMER_CMPRAUXB3_CMPR3B3_Pos (16UL) /*!< CTIMER CMPRAUXB3: CMPR3B3 (Bit 16) */ +#define CTIMER_CMPRAUXB3_CMPR3B3_Msk (0xffff0000UL) /*!< CTIMER CMPRAUXB3: CMPR3B3 (Bitfield-Mask: 0xffff) */ +#define CTIMER_CMPRAUXB3_CMPR2B3_Pos (0UL) /*!< CTIMER CMPRAUXB3: CMPR2B3 (Bit 0) */ +#define CTIMER_CMPRAUXB3_CMPR2B3_Msk (0xffffUL) /*!< CTIMER CMPRAUXB3: CMPR2B3 (Bitfield-Mask: 0xffff) */ +/* ========================================================= AUX3 ========================================================== */ +#define CTIMER_AUX3_TMRB3EN23_Pos (30UL) /*!< CTIMER AUX3: TMRB3EN23 (Bit 30) */ +#define CTIMER_AUX3_TMRB3EN23_Msk (0x40000000UL) /*!< CTIMER AUX3: TMRB3EN23 (Bitfield-Mask: 0x01) */ +#define CTIMER_AUX3_TMRB3POL23_Pos (29UL) /*!< CTIMER AUX3: TMRB3POL23 (Bit 29) */ +#define CTIMER_AUX3_TMRB3POL23_Msk (0x20000000UL) /*!< CTIMER AUX3: TMRB3POL23 (Bitfield-Mask: 0x01) */ +#define CTIMER_AUX3_TMRB3TINV_Pos (28UL) /*!< CTIMER AUX3: TMRB3TINV (Bit 28) */ +#define CTIMER_AUX3_TMRB3TINV_Msk (0x10000000UL) /*!< CTIMER AUX3: TMRB3TINV (Bitfield-Mask: 0x01) */ +#define CTIMER_AUX3_TMRB3NOSYNC_Pos (27UL) /*!< CTIMER AUX3: TMRB3NOSYNC (Bit 27) */ +#define CTIMER_AUX3_TMRB3NOSYNC_Msk (0x8000000UL) /*!< CTIMER AUX3: TMRB3NOSYNC (Bitfield-Mask: 0x01) */ +#define CTIMER_AUX3_TMRB3TRIG_Pos (23UL) /*!< CTIMER AUX3: TMRB3TRIG (Bit 23) */ +#define CTIMER_AUX3_TMRB3TRIG_Msk (0x7800000UL) /*!< CTIMER AUX3: TMRB3TRIG (Bitfield-Mask: 0x0f) */ +#define CTIMER_AUX3_TMRB3LMT_Pos (16UL) /*!< CTIMER AUX3: TMRB3LMT (Bit 16) */ +#define CTIMER_AUX3_TMRB3LMT_Msk (0x3f0000UL) /*!< CTIMER AUX3: TMRB3LMT (Bitfield-Mask: 0x3f) */ +#define CTIMER_AUX3_TMRA3EN23_Pos (14UL) /*!< CTIMER AUX3: TMRA3EN23 (Bit 14) */ +#define CTIMER_AUX3_TMRA3EN23_Msk (0x4000UL) /*!< CTIMER AUX3: TMRA3EN23 (Bitfield-Mask: 0x01) */ +#define CTIMER_AUX3_TMRA3POL23_Pos (13UL) /*!< CTIMER AUX3: TMRA3POL23 (Bit 13) */ +#define CTIMER_AUX3_TMRA3POL23_Msk (0x2000UL) /*!< CTIMER AUX3: TMRA3POL23 (Bitfield-Mask: 0x01) */ +#define CTIMER_AUX3_TMRA3TINV_Pos (12UL) /*!< CTIMER AUX3: TMRA3TINV (Bit 12) */ +#define CTIMER_AUX3_TMRA3TINV_Msk (0x1000UL) /*!< CTIMER AUX3: TMRA3TINV (Bitfield-Mask: 0x01) */ +#define CTIMER_AUX3_TMRA3NOSYNC_Pos (11UL) /*!< CTIMER AUX3: TMRA3NOSYNC (Bit 11) */ +#define CTIMER_AUX3_TMRA3NOSYNC_Msk (0x800UL) /*!< CTIMER AUX3: TMRA3NOSYNC (Bitfield-Mask: 0x01) */ +#define CTIMER_AUX3_TMRA3TRIG_Pos (7UL) /*!< CTIMER AUX3: TMRA3TRIG (Bit 7) */ +#define CTIMER_AUX3_TMRA3TRIG_Msk (0x780UL) /*!< CTIMER AUX3: TMRA3TRIG (Bitfield-Mask: 0x0f) */ +#define CTIMER_AUX3_TMRA3LMT_Pos (0UL) /*!< CTIMER AUX3: TMRA3LMT (Bit 0) */ +#define CTIMER_AUX3_TMRA3LMT_Msk (0x7fUL) /*!< CTIMER AUX3: TMRA3LMT (Bitfield-Mask: 0x7f) */ +/* ========================================================= TMR4 ========================================================== */ +#define CTIMER_TMR4_CTTMRB4_Pos (16UL) /*!< CTIMER TMR4: CTTMRB4 (Bit 16) */ +#define CTIMER_TMR4_CTTMRB4_Msk (0xffff0000UL) /*!< CTIMER TMR4: CTTMRB4 (Bitfield-Mask: 0xffff) */ +#define CTIMER_TMR4_CTTMRA4_Pos (0UL) /*!< CTIMER TMR4: CTTMRA4 (Bit 0) */ +#define CTIMER_TMR4_CTTMRA4_Msk (0xffffUL) /*!< CTIMER TMR4: CTTMRA4 (Bitfield-Mask: 0xffff) */ +/* ======================================================== CMPRA4 ========================================================= */ +#define CTIMER_CMPRA4_CMPR1A4_Pos (16UL) /*!< CTIMER CMPRA4: CMPR1A4 (Bit 16) */ +#define CTIMER_CMPRA4_CMPR1A4_Msk (0xffff0000UL) /*!< CTIMER CMPRA4: CMPR1A4 (Bitfield-Mask: 0xffff) */ +#define CTIMER_CMPRA4_CMPR0A4_Pos (0UL) /*!< CTIMER CMPRA4: CMPR0A4 (Bit 0) */ +#define CTIMER_CMPRA4_CMPR0A4_Msk (0xffffUL) /*!< CTIMER CMPRA4: CMPR0A4 (Bitfield-Mask: 0xffff) */ +/* ======================================================== CMPRB4 ========================================================= */ +#define CTIMER_CMPRB4_CMPR1B4_Pos (16UL) /*!< CTIMER CMPRB4: CMPR1B4 (Bit 16) */ +#define CTIMER_CMPRB4_CMPR1B4_Msk (0xffff0000UL) /*!< CTIMER CMPRB4: CMPR1B4 (Bitfield-Mask: 0xffff) */ +#define CTIMER_CMPRB4_CMPR0B4_Pos (0UL) /*!< CTIMER CMPRB4: CMPR0B4 (Bit 0) */ +#define CTIMER_CMPRB4_CMPR0B4_Msk (0xffffUL) /*!< CTIMER CMPRB4: CMPR0B4 (Bitfield-Mask: 0xffff) */ +/* ========================================================= CTRL4 ========================================================= */ +#define CTIMER_CTRL4_CTLINK4_Pos (31UL) /*!< CTIMER CTRL4: CTLINK4 (Bit 31) */ +#define CTIMER_CTRL4_CTLINK4_Msk (0x80000000UL) /*!< CTIMER CTRL4: CTLINK4 (Bitfield-Mask: 0x01) */ +#define CTIMER_CTRL4_TMRB4POL_Pos (28UL) /*!< CTIMER CTRL4: TMRB4POL (Bit 28) */ +#define CTIMER_CTRL4_TMRB4POL_Msk (0x10000000UL) /*!< CTIMER CTRL4: TMRB4POL (Bitfield-Mask: 0x01) */ +#define CTIMER_CTRL4_TMRB4CLR_Pos (27UL) /*!< CTIMER CTRL4: TMRB4CLR (Bit 27) */ +#define CTIMER_CTRL4_TMRB4CLR_Msk (0x8000000UL) /*!< CTIMER CTRL4: TMRB4CLR (Bitfield-Mask: 0x01) */ +#define CTIMER_CTRL4_TMRB4IE1_Pos (26UL) /*!< CTIMER CTRL4: TMRB4IE1 (Bit 26) */ +#define CTIMER_CTRL4_TMRB4IE1_Msk (0x4000000UL) /*!< CTIMER CTRL4: TMRB4IE1 (Bitfield-Mask: 0x01) */ +#define CTIMER_CTRL4_TMRB4IE0_Pos (25UL) /*!< CTIMER CTRL4: TMRB4IE0 (Bit 25) */ +#define CTIMER_CTRL4_TMRB4IE0_Msk (0x2000000UL) /*!< CTIMER CTRL4: TMRB4IE0 (Bitfield-Mask: 0x01) */ +#define CTIMER_CTRL4_TMRB4FN_Pos (22UL) /*!< CTIMER CTRL4: TMRB4FN (Bit 22) */ +#define CTIMER_CTRL4_TMRB4FN_Msk (0x1c00000UL) /*!< CTIMER CTRL4: TMRB4FN (Bitfield-Mask: 0x07) */ +#define CTIMER_CTRL4_TMRB4CLK_Pos (17UL) /*!< CTIMER CTRL4: TMRB4CLK (Bit 17) */ +#define CTIMER_CTRL4_TMRB4CLK_Msk (0x3e0000UL) /*!< CTIMER CTRL4: TMRB4CLK (Bitfield-Mask: 0x1f) */ +#define CTIMER_CTRL4_TMRB4EN_Pos (16UL) /*!< CTIMER CTRL4: TMRB4EN (Bit 16) */ +#define CTIMER_CTRL4_TMRB4EN_Msk (0x10000UL) /*!< CTIMER CTRL4: TMRB4EN (Bitfield-Mask: 0x01) */ +#define CTIMER_CTRL4_TMRA4POL_Pos (12UL) /*!< CTIMER CTRL4: TMRA4POL (Bit 12) */ +#define CTIMER_CTRL4_TMRA4POL_Msk (0x1000UL) /*!< CTIMER CTRL4: TMRA4POL (Bitfield-Mask: 0x01) */ +#define CTIMER_CTRL4_TMRA4CLR_Pos (11UL) /*!< CTIMER CTRL4: TMRA4CLR (Bit 11) */ +#define CTIMER_CTRL4_TMRA4CLR_Msk (0x800UL) /*!< CTIMER CTRL4: TMRA4CLR (Bitfield-Mask: 0x01) */ +#define CTIMER_CTRL4_TMRA4IE1_Pos (10UL) /*!< CTIMER CTRL4: TMRA4IE1 (Bit 10) */ +#define CTIMER_CTRL4_TMRA4IE1_Msk (0x400UL) /*!< CTIMER CTRL4: TMRA4IE1 (Bitfield-Mask: 0x01) */ +#define CTIMER_CTRL4_TMRA4IE0_Pos (9UL) /*!< CTIMER CTRL4: TMRA4IE0 (Bit 9) */ +#define CTIMER_CTRL4_TMRA4IE0_Msk (0x200UL) /*!< CTIMER CTRL4: TMRA4IE0 (Bitfield-Mask: 0x01) */ +#define CTIMER_CTRL4_TMRA4FN_Pos (6UL) /*!< CTIMER CTRL4: TMRA4FN (Bit 6) */ +#define CTIMER_CTRL4_TMRA4FN_Msk (0x1c0UL) /*!< CTIMER CTRL4: TMRA4FN (Bitfield-Mask: 0x07) */ +#define CTIMER_CTRL4_TMRA4CLK_Pos (1UL) /*!< CTIMER CTRL4: TMRA4CLK (Bit 1) */ +#define CTIMER_CTRL4_TMRA4CLK_Msk (0x3eUL) /*!< CTIMER CTRL4: TMRA4CLK (Bitfield-Mask: 0x1f) */ +#define CTIMER_CTRL4_TMRA4EN_Pos (0UL) /*!< CTIMER CTRL4: TMRA4EN (Bit 0) */ +#define CTIMER_CTRL4_TMRA4EN_Msk (0x1UL) /*!< CTIMER CTRL4: TMRA4EN (Bitfield-Mask: 0x01) */ +/* ======================================================= CMPRAUXA4 ======================================================= */ +#define CTIMER_CMPRAUXA4_CMPR3A4_Pos (16UL) /*!< CTIMER CMPRAUXA4: CMPR3A4 (Bit 16) */ +#define CTIMER_CMPRAUXA4_CMPR3A4_Msk (0xffff0000UL) /*!< CTIMER CMPRAUXA4: CMPR3A4 (Bitfield-Mask: 0xffff) */ +#define CTIMER_CMPRAUXA4_CMPR2A4_Pos (0UL) /*!< CTIMER CMPRAUXA4: CMPR2A4 (Bit 0) */ +#define CTIMER_CMPRAUXA4_CMPR2A4_Msk (0xffffUL) /*!< CTIMER CMPRAUXA4: CMPR2A4 (Bitfield-Mask: 0xffff) */ +/* ======================================================= CMPRAUXB4 ======================================================= */ +#define CTIMER_CMPRAUXB4_CMPR3B4_Pos (16UL) /*!< CTIMER CMPRAUXB4: CMPR3B4 (Bit 16) */ +#define CTIMER_CMPRAUXB4_CMPR3B4_Msk (0xffff0000UL) /*!< CTIMER CMPRAUXB4: CMPR3B4 (Bitfield-Mask: 0xffff) */ +#define CTIMER_CMPRAUXB4_CMPR2B4_Pos (0UL) /*!< CTIMER CMPRAUXB4: CMPR2B4 (Bit 0) */ +#define CTIMER_CMPRAUXB4_CMPR2B4_Msk (0xffffUL) /*!< CTIMER CMPRAUXB4: CMPR2B4 (Bitfield-Mask: 0xffff) */ +/* ========================================================= AUX4 ========================================================== */ +#define CTIMER_AUX4_TMRB4EN23_Pos (30UL) /*!< CTIMER AUX4: TMRB4EN23 (Bit 30) */ +#define CTIMER_AUX4_TMRB4EN23_Msk (0x40000000UL) /*!< CTIMER AUX4: TMRB4EN23 (Bitfield-Mask: 0x01) */ +#define CTIMER_AUX4_TMRB4POL23_Pos (29UL) /*!< CTIMER AUX4: TMRB4POL23 (Bit 29) */ +#define CTIMER_AUX4_TMRB4POL23_Msk (0x20000000UL) /*!< CTIMER AUX4: TMRB4POL23 (Bitfield-Mask: 0x01) */ +#define CTIMER_AUX4_TMRB4TINV_Pos (28UL) /*!< CTIMER AUX4: TMRB4TINV (Bit 28) */ +#define CTIMER_AUX4_TMRB4TINV_Msk (0x10000000UL) /*!< CTIMER AUX4: TMRB4TINV (Bitfield-Mask: 0x01) */ +#define CTIMER_AUX4_TMRB4NOSYNC_Pos (27UL) /*!< CTIMER AUX4: TMRB4NOSYNC (Bit 27) */ +#define CTIMER_AUX4_TMRB4NOSYNC_Msk (0x8000000UL) /*!< CTIMER AUX4: TMRB4NOSYNC (Bitfield-Mask: 0x01) */ +#define CTIMER_AUX4_TMRB4TRIG_Pos (23UL) /*!< CTIMER AUX4: TMRB4TRIG (Bit 23) */ +#define CTIMER_AUX4_TMRB4TRIG_Msk (0x7800000UL) /*!< CTIMER AUX4: TMRB4TRIG (Bitfield-Mask: 0x0f) */ +#define CTIMER_AUX4_TMRB4LMT_Pos (16UL) /*!< CTIMER AUX4: TMRB4LMT (Bit 16) */ +#define CTIMER_AUX4_TMRB4LMT_Msk (0x3f0000UL) /*!< CTIMER AUX4: TMRB4LMT (Bitfield-Mask: 0x3f) */ +#define CTIMER_AUX4_TMRA4EN23_Pos (14UL) /*!< CTIMER AUX4: TMRA4EN23 (Bit 14) */ +#define CTIMER_AUX4_TMRA4EN23_Msk (0x4000UL) /*!< CTIMER AUX4: TMRA4EN23 (Bitfield-Mask: 0x01) */ +#define CTIMER_AUX4_TMRA4POL23_Pos (13UL) /*!< CTIMER AUX4: TMRA4POL23 (Bit 13) */ +#define CTIMER_AUX4_TMRA4POL23_Msk (0x2000UL) /*!< CTIMER AUX4: TMRA4POL23 (Bitfield-Mask: 0x01) */ +#define CTIMER_AUX4_TMRA4TINV_Pos (12UL) /*!< CTIMER AUX4: TMRA4TINV (Bit 12) */ +#define CTIMER_AUX4_TMRA4TINV_Msk (0x1000UL) /*!< CTIMER AUX4: TMRA4TINV (Bitfield-Mask: 0x01) */ +#define CTIMER_AUX4_TMRA4NOSYNC_Pos (11UL) /*!< CTIMER AUX4: TMRA4NOSYNC (Bit 11) */ +#define CTIMER_AUX4_TMRA4NOSYNC_Msk (0x800UL) /*!< CTIMER AUX4: TMRA4NOSYNC (Bitfield-Mask: 0x01) */ +#define CTIMER_AUX4_TMRA4TRIG_Pos (7UL) /*!< CTIMER AUX4: TMRA4TRIG (Bit 7) */ +#define CTIMER_AUX4_TMRA4TRIG_Msk (0x780UL) /*!< CTIMER AUX4: TMRA4TRIG (Bitfield-Mask: 0x0f) */ +#define CTIMER_AUX4_TMRA4LMT_Pos (0UL) /*!< CTIMER AUX4: TMRA4LMT (Bit 0) */ +#define CTIMER_AUX4_TMRA4LMT_Msk (0x7fUL) /*!< CTIMER AUX4: TMRA4LMT (Bitfield-Mask: 0x7f) */ +/* ========================================================= TMR5 ========================================================== */ +#define CTIMER_TMR5_CTTMRB5_Pos (16UL) /*!< CTIMER TMR5: CTTMRB5 (Bit 16) */ +#define CTIMER_TMR5_CTTMRB5_Msk (0xffff0000UL) /*!< CTIMER TMR5: CTTMRB5 (Bitfield-Mask: 0xffff) */ +#define CTIMER_TMR5_CTTMRA5_Pos (0UL) /*!< CTIMER TMR5: CTTMRA5 (Bit 0) */ +#define CTIMER_TMR5_CTTMRA5_Msk (0xffffUL) /*!< CTIMER TMR5: CTTMRA5 (Bitfield-Mask: 0xffff) */ +/* ======================================================== CMPRA5 ========================================================= */ +#define CTIMER_CMPRA5_CMPR1A5_Pos (16UL) /*!< CTIMER CMPRA5: CMPR1A5 (Bit 16) */ +#define CTIMER_CMPRA5_CMPR1A5_Msk (0xffff0000UL) /*!< CTIMER CMPRA5: CMPR1A5 (Bitfield-Mask: 0xffff) */ +#define CTIMER_CMPRA5_CMPR0A5_Pos (0UL) /*!< CTIMER CMPRA5: CMPR0A5 (Bit 0) */ +#define CTIMER_CMPRA5_CMPR0A5_Msk (0xffffUL) /*!< CTIMER CMPRA5: CMPR0A5 (Bitfield-Mask: 0xffff) */ +/* ======================================================== CMPRB5 ========================================================= */ +#define CTIMER_CMPRB5_CMPR1B5_Pos (16UL) /*!< CTIMER CMPRB5: CMPR1B5 (Bit 16) */ +#define CTIMER_CMPRB5_CMPR1B5_Msk (0xffff0000UL) /*!< CTIMER CMPRB5: CMPR1B5 (Bitfield-Mask: 0xffff) */ +#define CTIMER_CMPRB5_CMPR0B5_Pos (0UL) /*!< CTIMER CMPRB5: CMPR0B5 (Bit 0) */ +#define CTIMER_CMPRB5_CMPR0B5_Msk (0xffffUL) /*!< CTIMER CMPRB5: CMPR0B5 (Bitfield-Mask: 0xffff) */ +/* ========================================================= CTRL5 ========================================================= */ +#define CTIMER_CTRL5_CTLINK5_Pos (31UL) /*!< CTIMER CTRL5: CTLINK5 (Bit 31) */ +#define CTIMER_CTRL5_CTLINK5_Msk (0x80000000UL) /*!< CTIMER CTRL5: CTLINK5 (Bitfield-Mask: 0x01) */ +#define CTIMER_CTRL5_TMRB5POL_Pos (28UL) /*!< CTIMER CTRL5: TMRB5POL (Bit 28) */ +#define CTIMER_CTRL5_TMRB5POL_Msk (0x10000000UL) /*!< CTIMER CTRL5: TMRB5POL (Bitfield-Mask: 0x01) */ +#define CTIMER_CTRL5_TMRB5CLR_Pos (27UL) /*!< CTIMER CTRL5: TMRB5CLR (Bit 27) */ +#define CTIMER_CTRL5_TMRB5CLR_Msk (0x8000000UL) /*!< CTIMER CTRL5: TMRB5CLR (Bitfield-Mask: 0x01) */ +#define CTIMER_CTRL5_TMRB5IE1_Pos (26UL) /*!< CTIMER CTRL5: TMRB5IE1 (Bit 26) */ +#define CTIMER_CTRL5_TMRB5IE1_Msk (0x4000000UL) /*!< CTIMER CTRL5: TMRB5IE1 (Bitfield-Mask: 0x01) */ +#define CTIMER_CTRL5_TMRB5IE0_Pos (25UL) /*!< CTIMER CTRL5: TMRB5IE0 (Bit 25) */ +#define CTIMER_CTRL5_TMRB5IE0_Msk (0x2000000UL) /*!< CTIMER CTRL5: TMRB5IE0 (Bitfield-Mask: 0x01) */ +#define CTIMER_CTRL5_TMRB5FN_Pos (22UL) /*!< CTIMER CTRL5: TMRB5FN (Bit 22) */ +#define CTIMER_CTRL5_TMRB5FN_Msk (0x1c00000UL) /*!< CTIMER CTRL5: TMRB5FN (Bitfield-Mask: 0x07) */ +#define CTIMER_CTRL5_TMRB5CLK_Pos (17UL) /*!< CTIMER CTRL5: TMRB5CLK (Bit 17) */ +#define CTIMER_CTRL5_TMRB5CLK_Msk (0x3e0000UL) /*!< CTIMER CTRL5: TMRB5CLK (Bitfield-Mask: 0x1f) */ +#define CTIMER_CTRL5_TMRB5EN_Pos (16UL) /*!< CTIMER CTRL5: TMRB5EN (Bit 16) */ +#define CTIMER_CTRL5_TMRB5EN_Msk (0x10000UL) /*!< CTIMER CTRL5: TMRB5EN (Bitfield-Mask: 0x01) */ +#define CTIMER_CTRL5_TMRA5POL_Pos (12UL) /*!< CTIMER CTRL5: TMRA5POL (Bit 12) */ +#define CTIMER_CTRL5_TMRA5POL_Msk (0x1000UL) /*!< CTIMER CTRL5: TMRA5POL (Bitfield-Mask: 0x01) */ +#define CTIMER_CTRL5_TMRA5CLR_Pos (11UL) /*!< CTIMER CTRL5: TMRA5CLR (Bit 11) */ +#define CTIMER_CTRL5_TMRA5CLR_Msk (0x800UL) /*!< CTIMER CTRL5: TMRA5CLR (Bitfield-Mask: 0x01) */ +#define CTIMER_CTRL5_TMRA5IE1_Pos (10UL) /*!< CTIMER CTRL5: TMRA5IE1 (Bit 10) */ +#define CTIMER_CTRL5_TMRA5IE1_Msk (0x400UL) /*!< CTIMER CTRL5: TMRA5IE1 (Bitfield-Mask: 0x01) */ +#define CTIMER_CTRL5_TMRA5IE0_Pos (9UL) /*!< CTIMER CTRL5: TMRA5IE0 (Bit 9) */ +#define CTIMER_CTRL5_TMRA5IE0_Msk (0x200UL) /*!< CTIMER CTRL5: TMRA5IE0 (Bitfield-Mask: 0x01) */ +#define CTIMER_CTRL5_TMRA5FN_Pos (6UL) /*!< CTIMER CTRL5: TMRA5FN (Bit 6) */ +#define CTIMER_CTRL5_TMRA5FN_Msk (0x1c0UL) /*!< CTIMER CTRL5: TMRA5FN (Bitfield-Mask: 0x07) */ +#define CTIMER_CTRL5_TMRA5CLK_Pos (1UL) /*!< CTIMER CTRL5: TMRA5CLK (Bit 1) */ +#define CTIMER_CTRL5_TMRA5CLK_Msk (0x3eUL) /*!< CTIMER CTRL5: TMRA5CLK (Bitfield-Mask: 0x1f) */ +#define CTIMER_CTRL5_TMRA5EN_Pos (0UL) /*!< CTIMER CTRL5: TMRA5EN (Bit 0) */ +#define CTIMER_CTRL5_TMRA5EN_Msk (0x1UL) /*!< CTIMER CTRL5: TMRA5EN (Bitfield-Mask: 0x01) */ +/* ======================================================= CMPRAUXA5 ======================================================= */ +#define CTIMER_CMPRAUXA5_CMPR3A5_Pos (16UL) /*!< CTIMER CMPRAUXA5: CMPR3A5 (Bit 16) */ +#define CTIMER_CMPRAUXA5_CMPR3A5_Msk (0xffff0000UL) /*!< CTIMER CMPRAUXA5: CMPR3A5 (Bitfield-Mask: 0xffff) */ +#define CTIMER_CMPRAUXA5_CMPR2A5_Pos (0UL) /*!< CTIMER CMPRAUXA5: CMPR2A5 (Bit 0) */ +#define CTIMER_CMPRAUXA5_CMPR2A5_Msk (0xffffUL) /*!< CTIMER CMPRAUXA5: CMPR2A5 (Bitfield-Mask: 0xffff) */ +/* ======================================================= CMPRAUXB5 ======================================================= */ +#define CTIMER_CMPRAUXB5_CMPR3B5_Pos (16UL) /*!< CTIMER CMPRAUXB5: CMPR3B5 (Bit 16) */ +#define CTIMER_CMPRAUXB5_CMPR3B5_Msk (0xffff0000UL) /*!< CTIMER CMPRAUXB5: CMPR3B5 (Bitfield-Mask: 0xffff) */ +#define CTIMER_CMPRAUXB5_CMPR2B5_Pos (0UL) /*!< CTIMER CMPRAUXB5: CMPR2B5 (Bit 0) */ +#define CTIMER_CMPRAUXB5_CMPR2B5_Msk (0xffffUL) /*!< CTIMER CMPRAUXB5: CMPR2B5 (Bitfield-Mask: 0xffff) */ +/* ========================================================= AUX5 ========================================================== */ +#define CTIMER_AUX5_TMRB5EN23_Pos (30UL) /*!< CTIMER AUX5: TMRB5EN23 (Bit 30) */ +#define CTIMER_AUX5_TMRB5EN23_Msk (0x40000000UL) /*!< CTIMER AUX5: TMRB5EN23 (Bitfield-Mask: 0x01) */ +#define CTIMER_AUX5_TMRB5POL23_Pos (29UL) /*!< CTIMER AUX5: TMRB5POL23 (Bit 29) */ +#define CTIMER_AUX5_TMRB5POL23_Msk (0x20000000UL) /*!< CTIMER AUX5: TMRB5POL23 (Bitfield-Mask: 0x01) */ +#define CTIMER_AUX5_TMRB5TINV_Pos (28UL) /*!< CTIMER AUX5: TMRB5TINV (Bit 28) */ +#define CTIMER_AUX5_TMRB5TINV_Msk (0x10000000UL) /*!< CTIMER AUX5: TMRB5TINV (Bitfield-Mask: 0x01) */ +#define CTIMER_AUX5_TMRB5NOSYNC_Pos (27UL) /*!< CTIMER AUX5: TMRB5NOSYNC (Bit 27) */ +#define CTIMER_AUX5_TMRB5NOSYNC_Msk (0x8000000UL) /*!< CTIMER AUX5: TMRB5NOSYNC (Bitfield-Mask: 0x01) */ +#define CTIMER_AUX5_TMRB5TRIG_Pos (23UL) /*!< CTIMER AUX5: TMRB5TRIG (Bit 23) */ +#define CTIMER_AUX5_TMRB5TRIG_Msk (0x7800000UL) /*!< CTIMER AUX5: TMRB5TRIG (Bitfield-Mask: 0x0f) */ +#define CTIMER_AUX5_TMRB5LMT_Pos (16UL) /*!< CTIMER AUX5: TMRB5LMT (Bit 16) */ +#define CTIMER_AUX5_TMRB5LMT_Msk (0x3f0000UL) /*!< CTIMER AUX5: TMRB5LMT (Bitfield-Mask: 0x3f) */ +#define CTIMER_AUX5_TMRA5EN23_Pos (14UL) /*!< CTIMER AUX5: TMRA5EN23 (Bit 14) */ +#define CTIMER_AUX5_TMRA5EN23_Msk (0x4000UL) /*!< CTIMER AUX5: TMRA5EN23 (Bitfield-Mask: 0x01) */ +#define CTIMER_AUX5_TMRA5POL23_Pos (13UL) /*!< CTIMER AUX5: TMRA5POL23 (Bit 13) */ +#define CTIMER_AUX5_TMRA5POL23_Msk (0x2000UL) /*!< CTIMER AUX5: TMRA5POL23 (Bitfield-Mask: 0x01) */ +#define CTIMER_AUX5_TMRA5TINV_Pos (12UL) /*!< CTIMER AUX5: TMRA5TINV (Bit 12) */ +#define CTIMER_AUX5_TMRA5TINV_Msk (0x1000UL) /*!< CTIMER AUX5: TMRA5TINV (Bitfield-Mask: 0x01) */ +#define CTIMER_AUX5_TMRA5NOSYNC_Pos (11UL) /*!< CTIMER AUX5: TMRA5NOSYNC (Bit 11) */ +#define CTIMER_AUX5_TMRA5NOSYNC_Msk (0x800UL) /*!< CTIMER AUX5: TMRA5NOSYNC (Bitfield-Mask: 0x01) */ +#define CTIMER_AUX5_TMRA5TRIG_Pos (7UL) /*!< CTIMER AUX5: TMRA5TRIG (Bit 7) */ +#define CTIMER_AUX5_TMRA5TRIG_Msk (0x780UL) /*!< CTIMER AUX5: TMRA5TRIG (Bitfield-Mask: 0x0f) */ +#define CTIMER_AUX5_TMRA5LMT_Pos (0UL) /*!< CTIMER AUX5: TMRA5LMT (Bit 0) */ +#define CTIMER_AUX5_TMRA5LMT_Msk (0x7fUL) /*!< CTIMER AUX5: TMRA5LMT (Bitfield-Mask: 0x7f) */ +/* ========================================================= TMR6 ========================================================== */ +#define CTIMER_TMR6_CTTMRB6_Pos (16UL) /*!< CTIMER TMR6: CTTMRB6 (Bit 16) */ +#define CTIMER_TMR6_CTTMRB6_Msk (0xffff0000UL) /*!< CTIMER TMR6: CTTMRB6 (Bitfield-Mask: 0xffff) */ +#define CTIMER_TMR6_CTTMRA6_Pos (0UL) /*!< CTIMER TMR6: CTTMRA6 (Bit 0) */ +#define CTIMER_TMR6_CTTMRA6_Msk (0xffffUL) /*!< CTIMER TMR6: CTTMRA6 (Bitfield-Mask: 0xffff) */ +/* ======================================================== CMPRA6 ========================================================= */ +#define CTIMER_CMPRA6_CMPR1A6_Pos (16UL) /*!< CTIMER CMPRA6: CMPR1A6 (Bit 16) */ +#define CTIMER_CMPRA6_CMPR1A6_Msk (0xffff0000UL) /*!< CTIMER CMPRA6: CMPR1A6 (Bitfield-Mask: 0xffff) */ +#define CTIMER_CMPRA6_CMPR0A6_Pos (0UL) /*!< CTIMER CMPRA6: CMPR0A6 (Bit 0) */ +#define CTIMER_CMPRA6_CMPR0A6_Msk (0xffffUL) /*!< CTIMER CMPRA6: CMPR0A6 (Bitfield-Mask: 0xffff) */ +/* ======================================================== CMPRB6 ========================================================= */ +#define CTIMER_CMPRB6_CMPR1B6_Pos (16UL) /*!< CTIMER CMPRB6: CMPR1B6 (Bit 16) */ +#define CTIMER_CMPRB6_CMPR1B6_Msk (0xffff0000UL) /*!< CTIMER CMPRB6: CMPR1B6 (Bitfield-Mask: 0xffff) */ +#define CTIMER_CMPRB6_CMPR0B6_Pos (0UL) /*!< CTIMER CMPRB6: CMPR0B6 (Bit 0) */ +#define CTIMER_CMPRB6_CMPR0B6_Msk (0xffffUL) /*!< CTIMER CMPRB6: CMPR0B6 (Bitfield-Mask: 0xffff) */ +/* ========================================================= CTRL6 ========================================================= */ +#define CTIMER_CTRL6_CTLINK6_Pos (31UL) /*!< CTIMER CTRL6: CTLINK6 (Bit 31) */ +#define CTIMER_CTRL6_CTLINK6_Msk (0x80000000UL) /*!< CTIMER CTRL6: CTLINK6 (Bitfield-Mask: 0x01) */ +#define CTIMER_CTRL6_TMRB6POL_Pos (28UL) /*!< CTIMER CTRL6: TMRB6POL (Bit 28) */ +#define CTIMER_CTRL6_TMRB6POL_Msk (0x10000000UL) /*!< CTIMER CTRL6: TMRB6POL (Bitfield-Mask: 0x01) */ +#define CTIMER_CTRL6_TMRB6CLR_Pos (27UL) /*!< CTIMER CTRL6: TMRB6CLR (Bit 27) */ +#define CTIMER_CTRL6_TMRB6CLR_Msk (0x8000000UL) /*!< CTIMER CTRL6: TMRB6CLR (Bitfield-Mask: 0x01) */ +#define CTIMER_CTRL6_TMRB6IE1_Pos (26UL) /*!< CTIMER CTRL6: TMRB6IE1 (Bit 26) */ +#define CTIMER_CTRL6_TMRB6IE1_Msk (0x4000000UL) /*!< CTIMER CTRL6: TMRB6IE1 (Bitfield-Mask: 0x01) */ +#define CTIMER_CTRL6_TMRB6IE0_Pos (25UL) /*!< CTIMER CTRL6: TMRB6IE0 (Bit 25) */ +#define CTIMER_CTRL6_TMRB6IE0_Msk (0x2000000UL) /*!< CTIMER CTRL6: TMRB6IE0 (Bitfield-Mask: 0x01) */ +#define CTIMER_CTRL6_TMRB6FN_Pos (22UL) /*!< CTIMER CTRL6: TMRB6FN (Bit 22) */ +#define CTIMER_CTRL6_TMRB6FN_Msk (0x1c00000UL) /*!< CTIMER CTRL6: TMRB6FN (Bitfield-Mask: 0x07) */ +#define CTIMER_CTRL6_TMRB6CLK_Pos (17UL) /*!< CTIMER CTRL6: TMRB6CLK (Bit 17) */ +#define CTIMER_CTRL6_TMRB6CLK_Msk (0x3e0000UL) /*!< CTIMER CTRL6: TMRB6CLK (Bitfield-Mask: 0x1f) */ +#define CTIMER_CTRL6_TMRB6EN_Pos (16UL) /*!< CTIMER CTRL6: TMRB6EN (Bit 16) */ +#define CTIMER_CTRL6_TMRB6EN_Msk (0x10000UL) /*!< CTIMER CTRL6: TMRB6EN (Bitfield-Mask: 0x01) */ +#define CTIMER_CTRL6_TMRA6POL_Pos (12UL) /*!< CTIMER CTRL6: TMRA6POL (Bit 12) */ +#define CTIMER_CTRL6_TMRA6POL_Msk (0x1000UL) /*!< CTIMER CTRL6: TMRA6POL (Bitfield-Mask: 0x01) */ +#define CTIMER_CTRL6_TMRA6CLR_Pos (11UL) /*!< CTIMER CTRL6: TMRA6CLR (Bit 11) */ +#define CTIMER_CTRL6_TMRA6CLR_Msk (0x800UL) /*!< CTIMER CTRL6: TMRA6CLR (Bitfield-Mask: 0x01) */ +#define CTIMER_CTRL6_TMRA6IE1_Pos (10UL) /*!< CTIMER CTRL6: TMRA6IE1 (Bit 10) */ +#define CTIMER_CTRL6_TMRA6IE1_Msk (0x400UL) /*!< CTIMER CTRL6: TMRA6IE1 (Bitfield-Mask: 0x01) */ +#define CTIMER_CTRL6_TMRA6IE0_Pos (9UL) /*!< CTIMER CTRL6: TMRA6IE0 (Bit 9) */ +#define CTIMER_CTRL6_TMRA6IE0_Msk (0x200UL) /*!< CTIMER CTRL6: TMRA6IE0 (Bitfield-Mask: 0x01) */ +#define CTIMER_CTRL6_TMRA6FN_Pos (6UL) /*!< CTIMER CTRL6: TMRA6FN (Bit 6) */ +#define CTIMER_CTRL6_TMRA6FN_Msk (0x1c0UL) /*!< CTIMER CTRL6: TMRA6FN (Bitfield-Mask: 0x07) */ +#define CTIMER_CTRL6_TMRA6CLK_Pos (1UL) /*!< CTIMER CTRL6: TMRA6CLK (Bit 1) */ +#define CTIMER_CTRL6_TMRA6CLK_Msk (0x3eUL) /*!< CTIMER CTRL6: TMRA6CLK (Bitfield-Mask: 0x1f) */ +#define CTIMER_CTRL6_TMRA6EN_Pos (0UL) /*!< CTIMER CTRL6: TMRA6EN (Bit 0) */ +#define CTIMER_CTRL6_TMRA6EN_Msk (0x1UL) /*!< CTIMER CTRL6: TMRA6EN (Bitfield-Mask: 0x01) */ +/* ======================================================= CMPRAUXA6 ======================================================= */ +#define CTIMER_CMPRAUXA6_CMPR3A6_Pos (16UL) /*!< CTIMER CMPRAUXA6: CMPR3A6 (Bit 16) */ +#define CTIMER_CMPRAUXA6_CMPR3A6_Msk (0xffff0000UL) /*!< CTIMER CMPRAUXA6: CMPR3A6 (Bitfield-Mask: 0xffff) */ +#define CTIMER_CMPRAUXA6_CMPR2A6_Pos (0UL) /*!< CTIMER CMPRAUXA6: CMPR2A6 (Bit 0) */ +#define CTIMER_CMPRAUXA6_CMPR2A6_Msk (0xffffUL) /*!< CTIMER CMPRAUXA6: CMPR2A6 (Bitfield-Mask: 0xffff) */ +/* ======================================================= CMPRAUXB6 ======================================================= */ +#define CTIMER_CMPRAUXB6_CMPR3B6_Pos (16UL) /*!< CTIMER CMPRAUXB6: CMPR3B6 (Bit 16) */ +#define CTIMER_CMPRAUXB6_CMPR3B6_Msk (0xffff0000UL) /*!< CTIMER CMPRAUXB6: CMPR3B6 (Bitfield-Mask: 0xffff) */ +#define CTIMER_CMPRAUXB6_CMPR2B6_Pos (0UL) /*!< CTIMER CMPRAUXB6: CMPR2B6 (Bit 0) */ +#define CTIMER_CMPRAUXB6_CMPR2B6_Msk (0xffffUL) /*!< CTIMER CMPRAUXB6: CMPR2B6 (Bitfield-Mask: 0xffff) */ +/* ========================================================= AUX6 ========================================================== */ +#define CTIMER_AUX6_TMRB6EN23_Pos (30UL) /*!< CTIMER AUX6: TMRB6EN23 (Bit 30) */ +#define CTIMER_AUX6_TMRB6EN23_Msk (0x40000000UL) /*!< CTIMER AUX6: TMRB6EN23 (Bitfield-Mask: 0x01) */ +#define CTIMER_AUX6_TMRB6POL23_Pos (29UL) /*!< CTIMER AUX6: TMRB6POL23 (Bit 29) */ +#define CTIMER_AUX6_TMRB6POL23_Msk (0x20000000UL) /*!< CTIMER AUX6: TMRB6POL23 (Bitfield-Mask: 0x01) */ +#define CTIMER_AUX6_TMRB6TINV_Pos (28UL) /*!< CTIMER AUX6: TMRB6TINV (Bit 28) */ +#define CTIMER_AUX6_TMRB6TINV_Msk (0x10000000UL) /*!< CTIMER AUX6: TMRB6TINV (Bitfield-Mask: 0x01) */ +#define CTIMER_AUX6_TMRB6NOSYNC_Pos (27UL) /*!< CTIMER AUX6: TMRB6NOSYNC (Bit 27) */ +#define CTIMER_AUX6_TMRB6NOSYNC_Msk (0x8000000UL) /*!< CTIMER AUX6: TMRB6NOSYNC (Bitfield-Mask: 0x01) */ +#define CTIMER_AUX6_TMRB6TRIG_Pos (23UL) /*!< CTIMER AUX6: TMRB6TRIG (Bit 23) */ +#define CTIMER_AUX6_TMRB6TRIG_Msk (0x7800000UL) /*!< CTIMER AUX6: TMRB6TRIG (Bitfield-Mask: 0x0f) */ +#define CTIMER_AUX6_TMRB6LMT_Pos (16UL) /*!< CTIMER AUX6: TMRB6LMT (Bit 16) */ +#define CTIMER_AUX6_TMRB6LMT_Msk (0x3f0000UL) /*!< CTIMER AUX6: TMRB6LMT (Bitfield-Mask: 0x3f) */ +#define CTIMER_AUX6_TMRA6EN23_Pos (14UL) /*!< CTIMER AUX6: TMRA6EN23 (Bit 14) */ +#define CTIMER_AUX6_TMRA6EN23_Msk (0x4000UL) /*!< CTIMER AUX6: TMRA6EN23 (Bitfield-Mask: 0x01) */ +#define CTIMER_AUX6_TMRA6POL23_Pos (13UL) /*!< CTIMER AUX6: TMRA6POL23 (Bit 13) */ +#define CTIMER_AUX6_TMRA6POL23_Msk (0x2000UL) /*!< CTIMER AUX6: TMRA6POL23 (Bitfield-Mask: 0x01) */ +#define CTIMER_AUX6_TMRA6TINV_Pos (12UL) /*!< CTIMER AUX6: TMRA6TINV (Bit 12) */ +#define CTIMER_AUX6_TMRA6TINV_Msk (0x1000UL) /*!< CTIMER AUX6: TMRA6TINV (Bitfield-Mask: 0x01) */ +#define CTIMER_AUX6_TMRA6NOSYNC_Pos (11UL) /*!< CTIMER AUX6: TMRA6NOSYNC (Bit 11) */ +#define CTIMER_AUX6_TMRA6NOSYNC_Msk (0x800UL) /*!< CTIMER AUX6: TMRA6NOSYNC (Bitfield-Mask: 0x01) */ +#define CTIMER_AUX6_TMRA6TRIG_Pos (7UL) /*!< CTIMER AUX6: TMRA6TRIG (Bit 7) */ +#define CTIMER_AUX6_TMRA6TRIG_Msk (0x780UL) /*!< CTIMER AUX6: TMRA6TRIG (Bitfield-Mask: 0x0f) */ +#define CTIMER_AUX6_TMRA6LMT_Pos (0UL) /*!< CTIMER AUX6: TMRA6LMT (Bit 0) */ +#define CTIMER_AUX6_TMRA6LMT_Msk (0x7fUL) /*!< CTIMER AUX6: TMRA6LMT (Bitfield-Mask: 0x7f) */ +/* ========================================================= TMR7 ========================================================== */ +#define CTIMER_TMR7_CTTMRB7_Pos (16UL) /*!< CTIMER TMR7: CTTMRB7 (Bit 16) */ +#define CTIMER_TMR7_CTTMRB7_Msk (0xffff0000UL) /*!< CTIMER TMR7: CTTMRB7 (Bitfield-Mask: 0xffff) */ +#define CTIMER_TMR7_CTTMRA7_Pos (0UL) /*!< CTIMER TMR7: CTTMRA7 (Bit 0) */ +#define CTIMER_TMR7_CTTMRA7_Msk (0xffffUL) /*!< CTIMER TMR7: CTTMRA7 (Bitfield-Mask: 0xffff) */ +/* ======================================================== CMPRA7 ========================================================= */ +#define CTIMER_CMPRA7_CMPR1A7_Pos (16UL) /*!< CTIMER CMPRA7: CMPR1A7 (Bit 16) */ +#define CTIMER_CMPRA7_CMPR1A7_Msk (0xffff0000UL) /*!< CTIMER CMPRA7: CMPR1A7 (Bitfield-Mask: 0xffff) */ +#define CTIMER_CMPRA7_CMPR0A7_Pos (0UL) /*!< CTIMER CMPRA7: CMPR0A7 (Bit 0) */ +#define CTIMER_CMPRA7_CMPR0A7_Msk (0xffffUL) /*!< CTIMER CMPRA7: CMPR0A7 (Bitfield-Mask: 0xffff) */ +/* ======================================================== CMPRB7 ========================================================= */ +#define CTIMER_CMPRB7_CMPR1B7_Pos (16UL) /*!< CTIMER CMPRB7: CMPR1B7 (Bit 16) */ +#define CTIMER_CMPRB7_CMPR1B7_Msk (0xffff0000UL) /*!< CTIMER CMPRB7: CMPR1B7 (Bitfield-Mask: 0xffff) */ +#define CTIMER_CMPRB7_CMPR0B7_Pos (0UL) /*!< CTIMER CMPRB7: CMPR0B7 (Bit 0) */ +#define CTIMER_CMPRB7_CMPR0B7_Msk (0xffffUL) /*!< CTIMER CMPRB7: CMPR0B7 (Bitfield-Mask: 0xffff) */ +/* ========================================================= CTRL7 ========================================================= */ +#define CTIMER_CTRL7_CTLINK7_Pos (31UL) /*!< CTIMER CTRL7: CTLINK7 (Bit 31) */ +#define CTIMER_CTRL7_CTLINK7_Msk (0x80000000UL) /*!< CTIMER CTRL7: CTLINK7 (Bitfield-Mask: 0x01) */ +#define CTIMER_CTRL7_TMRB7POL_Pos (28UL) /*!< CTIMER CTRL7: TMRB7POL (Bit 28) */ +#define CTIMER_CTRL7_TMRB7POL_Msk (0x10000000UL) /*!< CTIMER CTRL7: TMRB7POL (Bitfield-Mask: 0x01) */ +#define CTIMER_CTRL7_TMRB7CLR_Pos (27UL) /*!< CTIMER CTRL7: TMRB7CLR (Bit 27) */ +#define CTIMER_CTRL7_TMRB7CLR_Msk (0x8000000UL) /*!< CTIMER CTRL7: TMRB7CLR (Bitfield-Mask: 0x01) */ +#define CTIMER_CTRL7_TMRB7IE1_Pos (26UL) /*!< CTIMER CTRL7: TMRB7IE1 (Bit 26) */ +#define CTIMER_CTRL7_TMRB7IE1_Msk (0x4000000UL) /*!< CTIMER CTRL7: TMRB7IE1 (Bitfield-Mask: 0x01) */ +#define CTIMER_CTRL7_TMRB7IE0_Pos (25UL) /*!< CTIMER CTRL7: TMRB7IE0 (Bit 25) */ +#define CTIMER_CTRL7_TMRB7IE0_Msk (0x2000000UL) /*!< CTIMER CTRL7: TMRB7IE0 (Bitfield-Mask: 0x01) */ +#define CTIMER_CTRL7_TMRB7FN_Pos (22UL) /*!< CTIMER CTRL7: TMRB7FN (Bit 22) */ +#define CTIMER_CTRL7_TMRB7FN_Msk (0x1c00000UL) /*!< CTIMER CTRL7: TMRB7FN (Bitfield-Mask: 0x07) */ +#define CTIMER_CTRL7_TMRB7CLK_Pos (17UL) /*!< CTIMER CTRL7: TMRB7CLK (Bit 17) */ +#define CTIMER_CTRL7_TMRB7CLK_Msk (0x3e0000UL) /*!< CTIMER CTRL7: TMRB7CLK (Bitfield-Mask: 0x1f) */ +#define CTIMER_CTRL7_TMRB7EN_Pos (16UL) /*!< CTIMER CTRL7: TMRB7EN (Bit 16) */ +#define CTIMER_CTRL7_TMRB7EN_Msk (0x10000UL) /*!< CTIMER CTRL7: TMRB7EN (Bitfield-Mask: 0x01) */ +#define CTIMER_CTRL7_TMRA7POL_Pos (12UL) /*!< CTIMER CTRL7: TMRA7POL (Bit 12) */ +#define CTIMER_CTRL7_TMRA7POL_Msk (0x1000UL) /*!< CTIMER CTRL7: TMRA7POL (Bitfield-Mask: 0x01) */ +#define CTIMER_CTRL7_TMRA7CLR_Pos (11UL) /*!< CTIMER CTRL7: TMRA7CLR (Bit 11) */ +#define CTIMER_CTRL7_TMRA7CLR_Msk (0x800UL) /*!< CTIMER CTRL7: TMRA7CLR (Bitfield-Mask: 0x01) */ +#define CTIMER_CTRL7_TMRA7IE1_Pos (10UL) /*!< CTIMER CTRL7: TMRA7IE1 (Bit 10) */ +#define CTIMER_CTRL7_TMRA7IE1_Msk (0x400UL) /*!< CTIMER CTRL7: TMRA7IE1 (Bitfield-Mask: 0x01) */ +#define CTIMER_CTRL7_TMRA7IE0_Pos (9UL) /*!< CTIMER CTRL7: TMRA7IE0 (Bit 9) */ +#define CTIMER_CTRL7_TMRA7IE0_Msk (0x200UL) /*!< CTIMER CTRL7: TMRA7IE0 (Bitfield-Mask: 0x01) */ +#define CTIMER_CTRL7_TMRA7FN_Pos (6UL) /*!< CTIMER CTRL7: TMRA7FN (Bit 6) */ +#define CTIMER_CTRL7_TMRA7FN_Msk (0x1c0UL) /*!< CTIMER CTRL7: TMRA7FN (Bitfield-Mask: 0x07) */ +#define CTIMER_CTRL7_TMRA7CLK_Pos (1UL) /*!< CTIMER CTRL7: TMRA7CLK (Bit 1) */ +#define CTIMER_CTRL7_TMRA7CLK_Msk (0x3eUL) /*!< CTIMER CTRL7: TMRA7CLK (Bitfield-Mask: 0x1f) */ +#define CTIMER_CTRL7_TMRA7EN_Pos (0UL) /*!< CTIMER CTRL7: TMRA7EN (Bit 0) */ +#define CTIMER_CTRL7_TMRA7EN_Msk (0x1UL) /*!< CTIMER CTRL7: TMRA7EN (Bitfield-Mask: 0x01) */ +/* ======================================================= CMPRAUXA7 ======================================================= */ +#define CTIMER_CMPRAUXA7_CMPR3A7_Pos (16UL) /*!< CTIMER CMPRAUXA7: CMPR3A7 (Bit 16) */ +#define CTIMER_CMPRAUXA7_CMPR3A7_Msk (0xffff0000UL) /*!< CTIMER CMPRAUXA7: CMPR3A7 (Bitfield-Mask: 0xffff) */ +#define CTIMER_CMPRAUXA7_CMPR2A7_Pos (0UL) /*!< CTIMER CMPRAUXA7: CMPR2A7 (Bit 0) */ +#define CTIMER_CMPRAUXA7_CMPR2A7_Msk (0xffffUL) /*!< CTIMER CMPRAUXA7: CMPR2A7 (Bitfield-Mask: 0xffff) */ +/* ======================================================= CMPRAUXB7 ======================================================= */ +#define CTIMER_CMPRAUXB7_CMPR3B7_Pos (16UL) /*!< CTIMER CMPRAUXB7: CMPR3B7 (Bit 16) */ +#define CTIMER_CMPRAUXB7_CMPR3B7_Msk (0xffff0000UL) /*!< CTIMER CMPRAUXB7: CMPR3B7 (Bitfield-Mask: 0xffff) */ +#define CTIMER_CMPRAUXB7_CMPR2B7_Pos (0UL) /*!< CTIMER CMPRAUXB7: CMPR2B7 (Bit 0) */ +#define CTIMER_CMPRAUXB7_CMPR2B7_Msk (0xffffUL) /*!< CTIMER CMPRAUXB7: CMPR2B7 (Bitfield-Mask: 0xffff) */ +/* ========================================================= AUX7 ========================================================== */ +#define CTIMER_AUX7_TMRB7EN23_Pos (30UL) /*!< CTIMER AUX7: TMRB7EN23 (Bit 30) */ +#define CTIMER_AUX7_TMRB7EN23_Msk (0x40000000UL) /*!< CTIMER AUX7: TMRB7EN23 (Bitfield-Mask: 0x01) */ +#define CTIMER_AUX7_TMRB7POL23_Pos (29UL) /*!< CTIMER AUX7: TMRB7POL23 (Bit 29) */ +#define CTIMER_AUX7_TMRB7POL23_Msk (0x20000000UL) /*!< CTIMER AUX7: TMRB7POL23 (Bitfield-Mask: 0x01) */ +#define CTIMER_AUX7_TMRB7TINV_Pos (28UL) /*!< CTIMER AUX7: TMRB7TINV (Bit 28) */ +#define CTIMER_AUX7_TMRB7TINV_Msk (0x10000000UL) /*!< CTIMER AUX7: TMRB7TINV (Bitfield-Mask: 0x01) */ +#define CTIMER_AUX7_TMRB7NOSYNC_Pos (27UL) /*!< CTIMER AUX7: TMRB7NOSYNC (Bit 27) */ +#define CTIMER_AUX7_TMRB7NOSYNC_Msk (0x8000000UL) /*!< CTIMER AUX7: TMRB7NOSYNC (Bitfield-Mask: 0x01) */ +#define CTIMER_AUX7_TMRB7TRIG_Pos (23UL) /*!< CTIMER AUX7: TMRB7TRIG (Bit 23) */ +#define CTIMER_AUX7_TMRB7TRIG_Msk (0x7800000UL) /*!< CTIMER AUX7: TMRB7TRIG (Bitfield-Mask: 0x0f) */ +#define CTIMER_AUX7_TMRB7LMT_Pos (16UL) /*!< CTIMER AUX7: TMRB7LMT (Bit 16) */ +#define CTIMER_AUX7_TMRB7LMT_Msk (0x3f0000UL) /*!< CTIMER AUX7: TMRB7LMT (Bitfield-Mask: 0x3f) */ +#define CTIMER_AUX7_TMRA7EN23_Pos (14UL) /*!< CTIMER AUX7: TMRA7EN23 (Bit 14) */ +#define CTIMER_AUX7_TMRA7EN23_Msk (0x4000UL) /*!< CTIMER AUX7: TMRA7EN23 (Bitfield-Mask: 0x01) */ +#define CTIMER_AUX7_TMRA7POL23_Pos (13UL) /*!< CTIMER AUX7: TMRA7POL23 (Bit 13) */ +#define CTIMER_AUX7_TMRA7POL23_Msk (0x2000UL) /*!< CTIMER AUX7: TMRA7POL23 (Bitfield-Mask: 0x01) */ +#define CTIMER_AUX7_TMRA7TINV_Pos (12UL) /*!< CTIMER AUX7: TMRA7TINV (Bit 12) */ +#define CTIMER_AUX7_TMRA7TINV_Msk (0x1000UL) /*!< CTIMER AUX7: TMRA7TINV (Bitfield-Mask: 0x01) */ +#define CTIMER_AUX7_TMRA7NOSYNC_Pos (11UL) /*!< CTIMER AUX7: TMRA7NOSYNC (Bit 11) */ +#define CTIMER_AUX7_TMRA7NOSYNC_Msk (0x800UL) /*!< CTIMER AUX7: TMRA7NOSYNC (Bitfield-Mask: 0x01) */ +#define CTIMER_AUX7_TMRA7TRIG_Pos (7UL) /*!< CTIMER AUX7: TMRA7TRIG (Bit 7) */ +#define CTIMER_AUX7_TMRA7TRIG_Msk (0x780UL) /*!< CTIMER AUX7: TMRA7TRIG (Bitfield-Mask: 0x0f) */ +#define CTIMER_AUX7_TMRA7LMT_Pos (0UL) /*!< CTIMER AUX7: TMRA7LMT (Bit 0) */ +#define CTIMER_AUX7_TMRA7LMT_Msk (0x7fUL) /*!< CTIMER AUX7: TMRA7LMT (Bitfield-Mask: 0x7f) */ +/* ======================================================== GLOBEN ========================================================= */ +#define CTIMER_GLOBEN_ENB7_Pos (15UL) /*!< CTIMER GLOBEN: ENB7 (Bit 15) */ +#define CTIMER_GLOBEN_ENB7_Msk (0x8000UL) /*!< CTIMER GLOBEN: ENB7 (Bitfield-Mask: 0x01) */ +#define CTIMER_GLOBEN_ENA7_Pos (14UL) /*!< CTIMER GLOBEN: ENA7 (Bit 14) */ +#define CTIMER_GLOBEN_ENA7_Msk (0x4000UL) /*!< CTIMER GLOBEN: ENA7 (Bitfield-Mask: 0x01) */ +#define CTIMER_GLOBEN_ENB6_Pos (13UL) /*!< CTIMER GLOBEN: ENB6 (Bit 13) */ +#define CTIMER_GLOBEN_ENB6_Msk (0x2000UL) /*!< CTIMER GLOBEN: ENB6 (Bitfield-Mask: 0x01) */ +#define CTIMER_GLOBEN_ENA6_Pos (12UL) /*!< CTIMER GLOBEN: ENA6 (Bit 12) */ +#define CTIMER_GLOBEN_ENA6_Msk (0x1000UL) /*!< CTIMER GLOBEN: ENA6 (Bitfield-Mask: 0x01) */ +#define CTIMER_GLOBEN_ENB5_Pos (11UL) /*!< CTIMER GLOBEN: ENB5 (Bit 11) */ +#define CTIMER_GLOBEN_ENB5_Msk (0x800UL) /*!< CTIMER GLOBEN: ENB5 (Bitfield-Mask: 0x01) */ +#define CTIMER_GLOBEN_ENA5_Pos (10UL) /*!< CTIMER GLOBEN: ENA5 (Bit 10) */ +#define CTIMER_GLOBEN_ENA5_Msk (0x400UL) /*!< CTIMER GLOBEN: ENA5 (Bitfield-Mask: 0x01) */ +#define CTIMER_GLOBEN_ENB4_Pos (9UL) /*!< CTIMER GLOBEN: ENB4 (Bit 9) */ +#define CTIMER_GLOBEN_ENB4_Msk (0x200UL) /*!< CTIMER GLOBEN: ENB4 (Bitfield-Mask: 0x01) */ +#define CTIMER_GLOBEN_ENA4_Pos (8UL) /*!< CTIMER GLOBEN: ENA4 (Bit 8) */ +#define CTIMER_GLOBEN_ENA4_Msk (0x100UL) /*!< CTIMER GLOBEN: ENA4 (Bitfield-Mask: 0x01) */ +#define CTIMER_GLOBEN_ENB3_Pos (7UL) /*!< CTIMER GLOBEN: ENB3 (Bit 7) */ +#define CTIMER_GLOBEN_ENB3_Msk (0x80UL) /*!< CTIMER GLOBEN: ENB3 (Bitfield-Mask: 0x01) */ +#define CTIMER_GLOBEN_ENA3_Pos (6UL) /*!< CTIMER GLOBEN: ENA3 (Bit 6) */ +#define CTIMER_GLOBEN_ENA3_Msk (0x40UL) /*!< CTIMER GLOBEN: ENA3 (Bitfield-Mask: 0x01) */ +#define CTIMER_GLOBEN_ENB2_Pos (5UL) /*!< CTIMER GLOBEN: ENB2 (Bit 5) */ +#define CTIMER_GLOBEN_ENB2_Msk (0x20UL) /*!< CTIMER GLOBEN: ENB2 (Bitfield-Mask: 0x01) */ +#define CTIMER_GLOBEN_ENA2_Pos (4UL) /*!< CTIMER GLOBEN: ENA2 (Bit 4) */ +#define CTIMER_GLOBEN_ENA2_Msk (0x10UL) /*!< CTIMER GLOBEN: ENA2 (Bitfield-Mask: 0x01) */ +#define CTIMER_GLOBEN_ENB1_Pos (3UL) /*!< CTIMER GLOBEN: ENB1 (Bit 3) */ +#define CTIMER_GLOBEN_ENB1_Msk (0x8UL) /*!< CTIMER GLOBEN: ENB1 (Bitfield-Mask: 0x01) */ +#define CTIMER_GLOBEN_ENA1_Pos (2UL) /*!< CTIMER GLOBEN: ENA1 (Bit 2) */ +#define CTIMER_GLOBEN_ENA1_Msk (0x4UL) /*!< CTIMER GLOBEN: ENA1 (Bitfield-Mask: 0x01) */ +#define CTIMER_GLOBEN_ENB0_Pos (1UL) /*!< CTIMER GLOBEN: ENB0 (Bit 1) */ +#define CTIMER_GLOBEN_ENB0_Msk (0x2UL) /*!< CTIMER GLOBEN: ENB0 (Bitfield-Mask: 0x01) */ +#define CTIMER_GLOBEN_ENA0_Pos (0UL) /*!< CTIMER GLOBEN: ENA0 (Bit 0) */ +#define CTIMER_GLOBEN_ENA0_Msk (0x1UL) /*!< CTIMER GLOBEN: ENA0 (Bitfield-Mask: 0x01) */ +/* ======================================================== OUTCFG0 ======================================================== */ +#define CTIMER_OUTCFG0_CFG9_Pos (28UL) /*!< CTIMER OUTCFG0: CFG9 (Bit 28) */ +#define CTIMER_OUTCFG0_CFG9_Msk (0x70000000UL) /*!< CTIMER OUTCFG0: CFG9 (Bitfield-Mask: 0x07) */ +#define CTIMER_OUTCFG0_CFG8_Pos (25UL) /*!< CTIMER OUTCFG0: CFG8 (Bit 25) */ +#define CTIMER_OUTCFG0_CFG8_Msk (0xe000000UL) /*!< CTIMER OUTCFG0: CFG8 (Bitfield-Mask: 0x07) */ +#define CTIMER_OUTCFG0_CFG7_Pos (22UL) /*!< CTIMER OUTCFG0: CFG7 (Bit 22) */ +#define CTIMER_OUTCFG0_CFG7_Msk (0x1c00000UL) /*!< CTIMER OUTCFG0: CFG7 (Bitfield-Mask: 0x07) */ +#define CTIMER_OUTCFG0_CFG6_Pos (19UL) /*!< CTIMER OUTCFG0: CFG6 (Bit 19) */ +#define CTIMER_OUTCFG0_CFG6_Msk (0x380000UL) /*!< CTIMER OUTCFG0: CFG6 (Bitfield-Mask: 0x07) */ +#define CTIMER_OUTCFG0_CFG5_Pos (16UL) /*!< CTIMER OUTCFG0: CFG5 (Bit 16) */ +#define CTIMER_OUTCFG0_CFG5_Msk (0x70000UL) /*!< CTIMER OUTCFG0: CFG5 (Bitfield-Mask: 0x07) */ +#define CTIMER_OUTCFG0_CFG4_Pos (12UL) /*!< CTIMER OUTCFG0: CFG4 (Bit 12) */ +#define CTIMER_OUTCFG0_CFG4_Msk (0x7000UL) /*!< CTIMER OUTCFG0: CFG4 (Bitfield-Mask: 0x07) */ +#define CTIMER_OUTCFG0_CFG3_Pos (9UL) /*!< CTIMER OUTCFG0: CFG3 (Bit 9) */ +#define CTIMER_OUTCFG0_CFG3_Msk (0xe00UL) /*!< CTIMER OUTCFG0: CFG3 (Bitfield-Mask: 0x07) */ +#define CTIMER_OUTCFG0_CFG2_Pos (6UL) /*!< CTIMER OUTCFG0: CFG2 (Bit 6) */ +#define CTIMER_OUTCFG0_CFG2_Msk (0x1c0UL) /*!< CTIMER OUTCFG0: CFG2 (Bitfield-Mask: 0x07) */ +#define CTIMER_OUTCFG0_CFG1_Pos (3UL) /*!< CTIMER OUTCFG0: CFG1 (Bit 3) */ +#define CTIMER_OUTCFG0_CFG1_Msk (0x38UL) /*!< CTIMER OUTCFG0: CFG1 (Bitfield-Mask: 0x07) */ +#define CTIMER_OUTCFG0_CFG0_Pos (0UL) /*!< CTIMER OUTCFG0: CFG0 (Bit 0) */ +#define CTIMER_OUTCFG0_CFG0_Msk (0x7UL) /*!< CTIMER OUTCFG0: CFG0 (Bitfield-Mask: 0x07) */ +/* ======================================================== OUTCFG1 ======================================================== */ +#define CTIMER_OUTCFG1_CFG19_Pos (28UL) /*!< CTIMER OUTCFG1: CFG19 (Bit 28) */ +#define CTIMER_OUTCFG1_CFG19_Msk (0x70000000UL) /*!< CTIMER OUTCFG1: CFG19 (Bitfield-Mask: 0x07) */ +#define CTIMER_OUTCFG1_CFG18_Pos (25UL) /*!< CTIMER OUTCFG1: CFG18 (Bit 25) */ +#define CTIMER_OUTCFG1_CFG18_Msk (0xe000000UL) /*!< CTIMER OUTCFG1: CFG18 (Bitfield-Mask: 0x07) */ +#define CTIMER_OUTCFG1_CFG17_Pos (22UL) /*!< CTIMER OUTCFG1: CFG17 (Bit 22) */ +#define CTIMER_OUTCFG1_CFG17_Msk (0x1c00000UL) /*!< CTIMER OUTCFG1: CFG17 (Bitfield-Mask: 0x07) */ +#define CTIMER_OUTCFG1_CFG16_Pos (19UL) /*!< CTIMER OUTCFG1: CFG16 (Bit 19) */ +#define CTIMER_OUTCFG1_CFG16_Msk (0x380000UL) /*!< CTIMER OUTCFG1: CFG16 (Bitfield-Mask: 0x07) */ +#define CTIMER_OUTCFG1_CFG15_Pos (16UL) /*!< CTIMER OUTCFG1: CFG15 (Bit 16) */ +#define CTIMER_OUTCFG1_CFG15_Msk (0x70000UL) /*!< CTIMER OUTCFG1: CFG15 (Bitfield-Mask: 0x07) */ +#define CTIMER_OUTCFG1_CFG14_Pos (12UL) /*!< CTIMER OUTCFG1: CFG14 (Bit 12) */ +#define CTIMER_OUTCFG1_CFG14_Msk (0x7000UL) /*!< CTIMER OUTCFG1: CFG14 (Bitfield-Mask: 0x07) */ +#define CTIMER_OUTCFG1_CFG13_Pos (9UL) /*!< CTIMER OUTCFG1: CFG13 (Bit 9) */ +#define CTIMER_OUTCFG1_CFG13_Msk (0xe00UL) /*!< CTIMER OUTCFG1: CFG13 (Bitfield-Mask: 0x07) */ +#define CTIMER_OUTCFG1_CFG12_Pos (6UL) /*!< CTIMER OUTCFG1: CFG12 (Bit 6) */ +#define CTIMER_OUTCFG1_CFG12_Msk (0x1c0UL) /*!< CTIMER OUTCFG1: CFG12 (Bitfield-Mask: 0x07) */ +#define CTIMER_OUTCFG1_CFG11_Pos (3UL) /*!< CTIMER OUTCFG1: CFG11 (Bit 3) */ +#define CTIMER_OUTCFG1_CFG11_Msk (0x38UL) /*!< CTIMER OUTCFG1: CFG11 (Bitfield-Mask: 0x07) */ +#define CTIMER_OUTCFG1_CFG10_Pos (0UL) /*!< CTIMER OUTCFG1: CFG10 (Bit 0) */ +#define CTIMER_OUTCFG1_CFG10_Msk (0x7UL) /*!< CTIMER OUTCFG1: CFG10 (Bitfield-Mask: 0x07) */ +/* ======================================================== OUTCFG2 ======================================================== */ +#define CTIMER_OUTCFG2_CFG29_Pos (28UL) /*!< CTIMER OUTCFG2: CFG29 (Bit 28) */ +#define CTIMER_OUTCFG2_CFG29_Msk (0x70000000UL) /*!< CTIMER OUTCFG2: CFG29 (Bitfield-Mask: 0x07) */ +#define CTIMER_OUTCFG2_CFG28_Pos (25UL) /*!< CTIMER OUTCFG2: CFG28 (Bit 25) */ +#define CTIMER_OUTCFG2_CFG28_Msk (0xe000000UL) /*!< CTIMER OUTCFG2: CFG28 (Bitfield-Mask: 0x07) */ +#define CTIMER_OUTCFG2_CFG27_Pos (22UL) /*!< CTIMER OUTCFG2: CFG27 (Bit 22) */ +#define CTIMER_OUTCFG2_CFG27_Msk (0x1c00000UL) /*!< CTIMER OUTCFG2: CFG27 (Bitfield-Mask: 0x07) */ +#define CTIMER_OUTCFG2_CFG26_Pos (19UL) /*!< CTIMER OUTCFG2: CFG26 (Bit 19) */ +#define CTIMER_OUTCFG2_CFG26_Msk (0x380000UL) /*!< CTIMER OUTCFG2: CFG26 (Bitfield-Mask: 0x07) */ +#define CTIMER_OUTCFG2_CFG25_Pos (16UL) /*!< CTIMER OUTCFG2: CFG25 (Bit 16) */ +#define CTIMER_OUTCFG2_CFG25_Msk (0x70000UL) /*!< CTIMER OUTCFG2: CFG25 (Bitfield-Mask: 0x07) */ +#define CTIMER_OUTCFG2_CFG24_Pos (12UL) /*!< CTIMER OUTCFG2: CFG24 (Bit 12) */ +#define CTIMER_OUTCFG2_CFG24_Msk (0x7000UL) /*!< CTIMER OUTCFG2: CFG24 (Bitfield-Mask: 0x07) */ +#define CTIMER_OUTCFG2_CFG23_Pos (9UL) /*!< CTIMER OUTCFG2: CFG23 (Bit 9) */ +#define CTIMER_OUTCFG2_CFG23_Msk (0xe00UL) /*!< CTIMER OUTCFG2: CFG23 (Bitfield-Mask: 0x07) */ +#define CTIMER_OUTCFG2_CFG22_Pos (6UL) /*!< CTIMER OUTCFG2: CFG22 (Bit 6) */ +#define CTIMER_OUTCFG2_CFG22_Msk (0x1c0UL) /*!< CTIMER OUTCFG2: CFG22 (Bitfield-Mask: 0x07) */ +#define CTIMER_OUTCFG2_CFG21_Pos (3UL) /*!< CTIMER OUTCFG2: CFG21 (Bit 3) */ +#define CTIMER_OUTCFG2_CFG21_Msk (0x38UL) /*!< CTIMER OUTCFG2: CFG21 (Bitfield-Mask: 0x07) */ +#define CTIMER_OUTCFG2_CFG20_Pos (0UL) /*!< CTIMER OUTCFG2: CFG20 (Bit 0) */ +#define CTIMER_OUTCFG2_CFG20_Msk (0x7UL) /*!< CTIMER OUTCFG2: CFG20 (Bitfield-Mask: 0x07) */ +/* ======================================================== OUTCFG3 ======================================================== */ +#define CTIMER_OUTCFG3_CFG31_Pos (3UL) /*!< CTIMER OUTCFG3: CFG31 (Bit 3) */ +#define CTIMER_OUTCFG3_CFG31_Msk (0x38UL) /*!< CTIMER OUTCFG3: CFG31 (Bitfield-Mask: 0x07) */ +#define CTIMER_OUTCFG3_CFG30_Pos (0UL) /*!< CTIMER OUTCFG3: CFG30 (Bit 0) */ +#define CTIMER_OUTCFG3_CFG30_Msk (0x7UL) /*!< CTIMER OUTCFG3: CFG30 (Bitfield-Mask: 0x07) */ +/* ========================================================= INCFG ========================================================= */ +#define CTIMER_INCFG_CFGB7_Pos (15UL) /*!< CTIMER INCFG: CFGB7 (Bit 15) */ +#define CTIMER_INCFG_CFGB7_Msk (0x8000UL) /*!< CTIMER INCFG: CFGB7 (Bitfield-Mask: 0x01) */ +#define CTIMER_INCFG_CFGA7_Pos (14UL) /*!< CTIMER INCFG: CFGA7 (Bit 14) */ +#define CTIMER_INCFG_CFGA7_Msk (0x4000UL) /*!< CTIMER INCFG: CFGA7 (Bitfield-Mask: 0x01) */ +#define CTIMER_INCFG_CFGB6_Pos (13UL) /*!< CTIMER INCFG: CFGB6 (Bit 13) */ +#define CTIMER_INCFG_CFGB6_Msk (0x2000UL) /*!< CTIMER INCFG: CFGB6 (Bitfield-Mask: 0x01) */ +#define CTIMER_INCFG_CFGA6_Pos (12UL) /*!< CTIMER INCFG: CFGA6 (Bit 12) */ +#define CTIMER_INCFG_CFGA6_Msk (0x1000UL) /*!< CTIMER INCFG: CFGA6 (Bitfield-Mask: 0x01) */ +#define CTIMER_INCFG_CFGB5_Pos (11UL) /*!< CTIMER INCFG: CFGB5 (Bit 11) */ +#define CTIMER_INCFG_CFGB5_Msk (0x800UL) /*!< CTIMER INCFG: CFGB5 (Bitfield-Mask: 0x01) */ +#define CTIMER_INCFG_CFGA5_Pos (10UL) /*!< CTIMER INCFG: CFGA5 (Bit 10) */ +#define CTIMER_INCFG_CFGA5_Msk (0x400UL) /*!< CTIMER INCFG: CFGA5 (Bitfield-Mask: 0x01) */ +#define CTIMER_INCFG_CFGB4_Pos (9UL) /*!< CTIMER INCFG: CFGB4 (Bit 9) */ +#define CTIMER_INCFG_CFGB4_Msk (0x200UL) /*!< CTIMER INCFG: CFGB4 (Bitfield-Mask: 0x01) */ +#define CTIMER_INCFG_CFGA4_Pos (8UL) /*!< CTIMER INCFG: CFGA4 (Bit 8) */ +#define CTIMER_INCFG_CFGA4_Msk (0x100UL) /*!< CTIMER INCFG: CFGA4 (Bitfield-Mask: 0x01) */ +#define CTIMER_INCFG_CFGB3_Pos (7UL) /*!< CTIMER INCFG: CFGB3 (Bit 7) */ +#define CTIMER_INCFG_CFGB3_Msk (0x80UL) /*!< CTIMER INCFG: CFGB3 (Bitfield-Mask: 0x01) */ +#define CTIMER_INCFG_CFGA3_Pos (6UL) /*!< CTIMER INCFG: CFGA3 (Bit 6) */ +#define CTIMER_INCFG_CFGA3_Msk (0x40UL) /*!< CTIMER INCFG: CFGA3 (Bitfield-Mask: 0x01) */ +#define CTIMER_INCFG_CFGB2_Pos (5UL) /*!< CTIMER INCFG: CFGB2 (Bit 5) */ +#define CTIMER_INCFG_CFGB2_Msk (0x20UL) /*!< CTIMER INCFG: CFGB2 (Bitfield-Mask: 0x01) */ +#define CTIMER_INCFG_CFGA2_Pos (4UL) /*!< CTIMER INCFG: CFGA2 (Bit 4) */ +#define CTIMER_INCFG_CFGA2_Msk (0x10UL) /*!< CTIMER INCFG: CFGA2 (Bitfield-Mask: 0x01) */ +#define CTIMER_INCFG_CFGB1_Pos (3UL) /*!< CTIMER INCFG: CFGB1 (Bit 3) */ +#define CTIMER_INCFG_CFGB1_Msk (0x8UL) /*!< CTIMER INCFG: CFGB1 (Bitfield-Mask: 0x01) */ +#define CTIMER_INCFG_CFGA1_Pos (2UL) /*!< CTIMER INCFG: CFGA1 (Bit 2) */ +#define CTIMER_INCFG_CFGA1_Msk (0x4UL) /*!< CTIMER INCFG: CFGA1 (Bitfield-Mask: 0x01) */ +#define CTIMER_INCFG_CFGB0_Pos (1UL) /*!< CTIMER INCFG: CFGB0 (Bit 1) */ +#define CTIMER_INCFG_CFGB0_Msk (0x2UL) /*!< CTIMER INCFG: CFGB0 (Bitfield-Mask: 0x01) */ +#define CTIMER_INCFG_CFGA0_Pos (0UL) /*!< CTIMER INCFG: CFGA0 (Bit 0) */ +#define CTIMER_INCFG_CFGA0_Msk (0x1UL) /*!< CTIMER INCFG: CFGA0 (Bitfield-Mask: 0x01) */ +/* ========================================================= STCFG ========================================================= */ +#define CTIMER_STCFG_FREEZE_Pos (31UL) /*!< CTIMER STCFG: FREEZE (Bit 31) */ +#define CTIMER_STCFG_FREEZE_Msk (0x80000000UL) /*!< CTIMER STCFG: FREEZE (Bitfield-Mask: 0x01) */ +#define CTIMER_STCFG_CLEAR_Pos (30UL) /*!< CTIMER STCFG: CLEAR (Bit 30) */ +#define CTIMER_STCFG_CLEAR_Msk (0x40000000UL) /*!< CTIMER STCFG: CLEAR (Bitfield-Mask: 0x01) */ +#define CTIMER_STCFG_COMPARE_H_EN_Pos (15UL) /*!< CTIMER STCFG: COMPARE_H_EN (Bit 15) */ +#define CTIMER_STCFG_COMPARE_H_EN_Msk (0x8000UL) /*!< CTIMER STCFG: COMPARE_H_EN (Bitfield-Mask: 0x01) */ +#define CTIMER_STCFG_COMPARE_G_EN_Pos (14UL) /*!< CTIMER STCFG: COMPARE_G_EN (Bit 14) */ +#define CTIMER_STCFG_COMPARE_G_EN_Msk (0x4000UL) /*!< CTIMER STCFG: COMPARE_G_EN (Bitfield-Mask: 0x01) */ +#define CTIMER_STCFG_COMPARE_F_EN_Pos (13UL) /*!< CTIMER STCFG: COMPARE_F_EN (Bit 13) */ +#define CTIMER_STCFG_COMPARE_F_EN_Msk (0x2000UL) /*!< CTIMER STCFG: COMPARE_F_EN (Bitfield-Mask: 0x01) */ +#define CTIMER_STCFG_COMPARE_E_EN_Pos (12UL) /*!< CTIMER STCFG: COMPARE_E_EN (Bit 12) */ +#define CTIMER_STCFG_COMPARE_E_EN_Msk (0x1000UL) /*!< CTIMER STCFG: COMPARE_E_EN (Bitfield-Mask: 0x01) */ +#define CTIMER_STCFG_COMPARE_D_EN_Pos (11UL) /*!< CTIMER STCFG: COMPARE_D_EN (Bit 11) */ +#define CTIMER_STCFG_COMPARE_D_EN_Msk (0x800UL) /*!< CTIMER STCFG: COMPARE_D_EN (Bitfield-Mask: 0x01) */ +#define CTIMER_STCFG_COMPARE_C_EN_Pos (10UL) /*!< CTIMER STCFG: COMPARE_C_EN (Bit 10) */ +#define CTIMER_STCFG_COMPARE_C_EN_Msk (0x400UL) /*!< CTIMER STCFG: COMPARE_C_EN (Bitfield-Mask: 0x01) */ +#define CTIMER_STCFG_COMPARE_B_EN_Pos (9UL) /*!< CTIMER STCFG: COMPARE_B_EN (Bit 9) */ +#define CTIMER_STCFG_COMPARE_B_EN_Msk (0x200UL) /*!< CTIMER STCFG: COMPARE_B_EN (Bitfield-Mask: 0x01) */ +#define CTIMER_STCFG_COMPARE_A_EN_Pos (8UL) /*!< CTIMER STCFG: COMPARE_A_EN (Bit 8) */ +#define CTIMER_STCFG_COMPARE_A_EN_Msk (0x100UL) /*!< CTIMER STCFG: COMPARE_A_EN (Bitfield-Mask: 0x01) */ +#define CTIMER_STCFG_CLKSEL_Pos (0UL) /*!< CTIMER STCFG: CLKSEL (Bit 0) */ +#define CTIMER_STCFG_CLKSEL_Msk (0xfUL) /*!< CTIMER STCFG: CLKSEL (Bitfield-Mask: 0x0f) */ +/* ========================================================= STTMR ========================================================= */ +#define CTIMER_STTMR_STTMR_Pos (0UL) /*!< CTIMER STTMR: STTMR (Bit 0) */ +#define CTIMER_STTMR_STTMR_Msk (0xffffffffUL) /*!< CTIMER STTMR: STTMR (Bitfield-Mask: 0xffffffff) */ +/* ==================================================== CAPTURECONTROL ===================================================== */ +#define CTIMER_CAPTURECONTROL_CAPTURE3_Pos (3UL) /*!< CTIMER CAPTURECONTROL: CAPTURE3 (Bit 3) */ +#define CTIMER_CAPTURECONTROL_CAPTURE3_Msk (0x8UL) /*!< CTIMER CAPTURECONTROL: CAPTURE3 (Bitfield-Mask: 0x01) */ +#define CTIMER_CAPTURECONTROL_CAPTURE2_Pos (2UL) /*!< CTIMER CAPTURECONTROL: CAPTURE2 (Bit 2) */ +#define CTIMER_CAPTURECONTROL_CAPTURE2_Msk (0x4UL) /*!< CTIMER CAPTURECONTROL: CAPTURE2 (Bitfield-Mask: 0x01) */ +#define CTIMER_CAPTURECONTROL_CAPTURE1_Pos (1UL) /*!< CTIMER CAPTURECONTROL: CAPTURE1 (Bit 1) */ +#define CTIMER_CAPTURECONTROL_CAPTURE1_Msk (0x2UL) /*!< CTIMER CAPTURECONTROL: CAPTURE1 (Bitfield-Mask: 0x01) */ +#define CTIMER_CAPTURECONTROL_CAPTURE0_Pos (0UL) /*!< CTIMER CAPTURECONTROL: CAPTURE0 (Bit 0) */ +#define CTIMER_CAPTURECONTROL_CAPTURE0_Msk (0x1UL) /*!< CTIMER CAPTURECONTROL: CAPTURE0 (Bitfield-Mask: 0x01) */ +/* ======================================================== SCMPR0 ========================================================= */ +#define CTIMER_SCMPR0_SCMPR0_Pos (0UL) /*!< CTIMER SCMPR0: SCMPR0 (Bit 0) */ +#define CTIMER_SCMPR0_SCMPR0_Msk (0xffffffffUL) /*!< CTIMER SCMPR0: SCMPR0 (Bitfield-Mask: 0xffffffff) */ +/* ======================================================== SCMPR1 ========================================================= */ +#define CTIMER_SCMPR1_SCMPR1_Pos (0UL) /*!< CTIMER SCMPR1: SCMPR1 (Bit 0) */ +#define CTIMER_SCMPR1_SCMPR1_Msk (0xffffffffUL) /*!< CTIMER SCMPR1: SCMPR1 (Bitfield-Mask: 0xffffffff) */ +/* ======================================================== SCMPR2 ========================================================= */ +#define CTIMER_SCMPR2_SCMPR2_Pos (0UL) /*!< CTIMER SCMPR2: SCMPR2 (Bit 0) */ +#define CTIMER_SCMPR2_SCMPR2_Msk (0xffffffffUL) /*!< CTIMER SCMPR2: SCMPR2 (Bitfield-Mask: 0xffffffff) */ +/* ======================================================== SCMPR3 ========================================================= */ +#define CTIMER_SCMPR3_SCMPR3_Pos (0UL) /*!< CTIMER SCMPR3: SCMPR3 (Bit 0) */ +#define CTIMER_SCMPR3_SCMPR3_Msk (0xffffffffUL) /*!< CTIMER SCMPR3: SCMPR3 (Bitfield-Mask: 0xffffffff) */ +/* ======================================================== SCMPR4 ========================================================= */ +#define CTIMER_SCMPR4_SCMPR4_Pos (0UL) /*!< CTIMER SCMPR4: SCMPR4 (Bit 0) */ +#define CTIMER_SCMPR4_SCMPR4_Msk (0xffffffffUL) /*!< CTIMER SCMPR4: SCMPR4 (Bitfield-Mask: 0xffffffff) */ +/* ======================================================== SCMPR5 ========================================================= */ +#define CTIMER_SCMPR5_SCMPR5_Pos (0UL) /*!< CTIMER SCMPR5: SCMPR5 (Bit 0) */ +#define CTIMER_SCMPR5_SCMPR5_Msk (0xffffffffUL) /*!< CTIMER SCMPR5: SCMPR5 (Bitfield-Mask: 0xffffffff) */ +/* ======================================================== SCMPR6 ========================================================= */ +#define CTIMER_SCMPR6_SCMPR6_Pos (0UL) /*!< CTIMER SCMPR6: SCMPR6 (Bit 0) */ +#define CTIMER_SCMPR6_SCMPR6_Msk (0xffffffffUL) /*!< CTIMER SCMPR6: SCMPR6 (Bitfield-Mask: 0xffffffff) */ +/* ======================================================== SCMPR7 ========================================================= */ +#define CTIMER_SCMPR7_SCMPR7_Pos (0UL) /*!< CTIMER SCMPR7: SCMPR7 (Bit 0) */ +#define CTIMER_SCMPR7_SCMPR7_Msk (0xffffffffUL) /*!< CTIMER SCMPR7: SCMPR7 (Bitfield-Mask: 0xffffffff) */ +/* ======================================================== SCAPT0 ========================================================= */ +#define CTIMER_SCAPT0_SCAPT0_Pos (0UL) /*!< CTIMER SCAPT0: SCAPT0 (Bit 0) */ +#define CTIMER_SCAPT0_SCAPT0_Msk (0xffffffffUL) /*!< CTIMER SCAPT0: SCAPT0 (Bitfield-Mask: 0xffffffff) */ +/* ======================================================== SCAPT1 ========================================================= */ +#define CTIMER_SCAPT1_SCAPT1_Pos (0UL) /*!< CTIMER SCAPT1: SCAPT1 (Bit 0) */ +#define CTIMER_SCAPT1_SCAPT1_Msk (0xffffffffUL) /*!< CTIMER SCAPT1: SCAPT1 (Bitfield-Mask: 0xffffffff) */ +/* ======================================================== SCAPT2 ========================================================= */ +#define CTIMER_SCAPT2_SCAPT2_Pos (0UL) /*!< CTIMER SCAPT2: SCAPT2 (Bit 0) */ +#define CTIMER_SCAPT2_SCAPT2_Msk (0xffffffffUL) /*!< CTIMER SCAPT2: SCAPT2 (Bitfield-Mask: 0xffffffff) */ +/* ======================================================== SCAPT3 ========================================================= */ +#define CTIMER_SCAPT3_SCAPT3_Pos (0UL) /*!< CTIMER SCAPT3: SCAPT3 (Bit 0) */ +#define CTIMER_SCAPT3_SCAPT3_Msk (0xffffffffUL) /*!< CTIMER SCAPT3: SCAPT3 (Bitfield-Mask: 0xffffffff) */ +/* ========================================================= SNVR0 ========================================================= */ +#define CTIMER_SNVR0_SNVR0_Pos (0UL) /*!< CTIMER SNVR0: SNVR0 (Bit 0) */ +#define CTIMER_SNVR0_SNVR0_Msk (0xffffffffUL) /*!< CTIMER SNVR0: SNVR0 (Bitfield-Mask: 0xffffffff) */ +/* ========================================================= SNVR1 ========================================================= */ +#define CTIMER_SNVR1_SNVR1_Pos (0UL) /*!< CTIMER SNVR1: SNVR1 (Bit 0) */ +#define CTIMER_SNVR1_SNVR1_Msk (0xffffffffUL) /*!< CTIMER SNVR1: SNVR1 (Bitfield-Mask: 0xffffffff) */ +/* ========================================================= SNVR2 ========================================================= */ +#define CTIMER_SNVR2_SNVR2_Pos (0UL) /*!< CTIMER SNVR2: SNVR2 (Bit 0) */ +#define CTIMER_SNVR2_SNVR2_Msk (0xffffffffUL) /*!< CTIMER SNVR2: SNVR2 (Bitfield-Mask: 0xffffffff) */ +/* ========================================================= SNVR3 ========================================================= */ +#define CTIMER_SNVR3_SNVR3_Pos (0UL) /*!< CTIMER SNVR3: SNVR3 (Bit 0) */ +#define CTIMER_SNVR3_SNVR3_Msk (0xffffffffUL) /*!< CTIMER SNVR3: SNVR3 (Bitfield-Mask: 0xffffffff) */ +/* ========================================================= INTEN ========================================================= */ +#define CTIMER_INTEN_CTMRB7C1INT_Pos (31UL) /*!< CTIMER INTEN: CTMRB7C1INT (Bit 31) */ +#define CTIMER_INTEN_CTMRB7C1INT_Msk (0x80000000UL) /*!< CTIMER INTEN: CTMRB7C1INT (Bitfield-Mask: 0x01) */ +#define CTIMER_INTEN_CTMRA7C1INT_Pos (30UL) /*!< CTIMER INTEN: CTMRA7C1INT (Bit 30) */ +#define CTIMER_INTEN_CTMRA7C1INT_Msk (0x40000000UL) /*!< CTIMER INTEN: CTMRA7C1INT (Bitfield-Mask: 0x01) */ +#define CTIMER_INTEN_CTMRB6C1INT_Pos (29UL) /*!< CTIMER INTEN: CTMRB6C1INT (Bit 29) */ +#define CTIMER_INTEN_CTMRB6C1INT_Msk (0x20000000UL) /*!< CTIMER INTEN: CTMRB6C1INT (Bitfield-Mask: 0x01) */ +#define CTIMER_INTEN_CTMRA6C1INT_Pos (28UL) /*!< CTIMER INTEN: CTMRA6C1INT (Bit 28) */ +#define CTIMER_INTEN_CTMRA6C1INT_Msk (0x10000000UL) /*!< CTIMER INTEN: CTMRA6C1INT (Bitfield-Mask: 0x01) */ +#define CTIMER_INTEN_CTMRB5C1INT_Pos (27UL) /*!< CTIMER INTEN: CTMRB5C1INT (Bit 27) */ +#define CTIMER_INTEN_CTMRB5C1INT_Msk (0x8000000UL) /*!< CTIMER INTEN: CTMRB5C1INT (Bitfield-Mask: 0x01) */ +#define CTIMER_INTEN_CTMRA5C1INT_Pos (26UL) /*!< CTIMER INTEN: CTMRA5C1INT (Bit 26) */ +#define CTIMER_INTEN_CTMRA5C1INT_Msk (0x4000000UL) /*!< CTIMER INTEN: CTMRA5C1INT (Bitfield-Mask: 0x01) */ +#define CTIMER_INTEN_CTMRB4C1INT_Pos (25UL) /*!< CTIMER INTEN: CTMRB4C1INT (Bit 25) */ +#define CTIMER_INTEN_CTMRB4C1INT_Msk (0x2000000UL) /*!< CTIMER INTEN: CTMRB4C1INT (Bitfield-Mask: 0x01) */ +#define CTIMER_INTEN_CTMRA4C1INT_Pos (24UL) /*!< CTIMER INTEN: CTMRA4C1INT (Bit 24) */ +#define CTIMER_INTEN_CTMRA4C1INT_Msk (0x1000000UL) /*!< CTIMER INTEN: CTMRA4C1INT (Bitfield-Mask: 0x01) */ +#define CTIMER_INTEN_CTMRB3C1INT_Pos (23UL) /*!< CTIMER INTEN: CTMRB3C1INT (Bit 23) */ +#define CTIMER_INTEN_CTMRB3C1INT_Msk (0x800000UL) /*!< CTIMER INTEN: CTMRB3C1INT (Bitfield-Mask: 0x01) */ +#define CTIMER_INTEN_CTMRA3C1INT_Pos (22UL) /*!< CTIMER INTEN: CTMRA3C1INT (Bit 22) */ +#define CTIMER_INTEN_CTMRA3C1INT_Msk (0x400000UL) /*!< CTIMER INTEN: CTMRA3C1INT (Bitfield-Mask: 0x01) */ +#define CTIMER_INTEN_CTMRB2C1INT_Pos (21UL) /*!< CTIMER INTEN: CTMRB2C1INT (Bit 21) */ +#define CTIMER_INTEN_CTMRB2C1INT_Msk (0x200000UL) /*!< CTIMER INTEN: CTMRB2C1INT (Bitfield-Mask: 0x01) */ +#define CTIMER_INTEN_CTMRA2C1INT_Pos (20UL) /*!< CTIMER INTEN: CTMRA2C1INT (Bit 20) */ +#define CTIMER_INTEN_CTMRA2C1INT_Msk (0x100000UL) /*!< CTIMER INTEN: CTMRA2C1INT (Bitfield-Mask: 0x01) */ +#define CTIMER_INTEN_CTMRB1C1INT_Pos (19UL) /*!< CTIMER INTEN: CTMRB1C1INT (Bit 19) */ +#define CTIMER_INTEN_CTMRB1C1INT_Msk (0x80000UL) /*!< CTIMER INTEN: CTMRB1C1INT (Bitfield-Mask: 0x01) */ +#define CTIMER_INTEN_CTMRA1C1INT_Pos (18UL) /*!< CTIMER INTEN: CTMRA1C1INT (Bit 18) */ +#define CTIMER_INTEN_CTMRA1C1INT_Msk (0x40000UL) /*!< CTIMER INTEN: CTMRA1C1INT (Bitfield-Mask: 0x01) */ +#define CTIMER_INTEN_CTMRB0C1INT_Pos (17UL) /*!< CTIMER INTEN: CTMRB0C1INT (Bit 17) */ +#define CTIMER_INTEN_CTMRB0C1INT_Msk (0x20000UL) /*!< CTIMER INTEN: CTMRB0C1INT (Bitfield-Mask: 0x01) */ +#define CTIMER_INTEN_CTMRA0C1INT_Pos (16UL) /*!< CTIMER INTEN: CTMRA0C1INT (Bit 16) */ +#define CTIMER_INTEN_CTMRA0C1INT_Msk (0x10000UL) /*!< CTIMER INTEN: CTMRA0C1INT (Bitfield-Mask: 0x01) */ +#define CTIMER_INTEN_CTMRB7C0INT_Pos (15UL) /*!< CTIMER INTEN: CTMRB7C0INT (Bit 15) */ +#define CTIMER_INTEN_CTMRB7C0INT_Msk (0x8000UL) /*!< CTIMER INTEN: CTMRB7C0INT (Bitfield-Mask: 0x01) */ +#define CTIMER_INTEN_CTMRA7C0INT_Pos (14UL) /*!< CTIMER INTEN: CTMRA7C0INT (Bit 14) */ +#define CTIMER_INTEN_CTMRA7C0INT_Msk (0x4000UL) /*!< CTIMER INTEN: CTMRA7C0INT (Bitfield-Mask: 0x01) */ +#define CTIMER_INTEN_CTMRB6C0INT_Pos (13UL) /*!< CTIMER INTEN: CTMRB6C0INT (Bit 13) */ +#define CTIMER_INTEN_CTMRB6C0INT_Msk (0x2000UL) /*!< CTIMER INTEN: CTMRB6C0INT (Bitfield-Mask: 0x01) */ +#define CTIMER_INTEN_CTMRA6C0INT_Pos (12UL) /*!< CTIMER INTEN: CTMRA6C0INT (Bit 12) */ +#define CTIMER_INTEN_CTMRA6C0INT_Msk (0x1000UL) /*!< CTIMER INTEN: CTMRA6C0INT (Bitfield-Mask: 0x01) */ +#define CTIMER_INTEN_CTMRB5C0INT_Pos (11UL) /*!< CTIMER INTEN: CTMRB5C0INT (Bit 11) */ +#define CTIMER_INTEN_CTMRB5C0INT_Msk (0x800UL) /*!< CTIMER INTEN: CTMRB5C0INT (Bitfield-Mask: 0x01) */ +#define CTIMER_INTEN_CTMRA5C0INT_Pos (10UL) /*!< CTIMER INTEN: CTMRA5C0INT (Bit 10) */ +#define CTIMER_INTEN_CTMRA5C0INT_Msk (0x400UL) /*!< CTIMER INTEN: CTMRA5C0INT (Bitfield-Mask: 0x01) */ +#define CTIMER_INTEN_CTMRB4C0INT_Pos (9UL) /*!< CTIMER INTEN: CTMRB4C0INT (Bit 9) */ +#define CTIMER_INTEN_CTMRB4C0INT_Msk (0x200UL) /*!< CTIMER INTEN: CTMRB4C0INT (Bitfield-Mask: 0x01) */ +#define CTIMER_INTEN_CTMRA4C0INT_Pos (8UL) /*!< CTIMER INTEN: CTMRA4C0INT (Bit 8) */ +#define CTIMER_INTEN_CTMRA4C0INT_Msk (0x100UL) /*!< CTIMER INTEN: CTMRA4C0INT (Bitfield-Mask: 0x01) */ +#define CTIMER_INTEN_CTMRB3C0INT_Pos (7UL) /*!< CTIMER INTEN: CTMRB3C0INT (Bit 7) */ +#define CTIMER_INTEN_CTMRB3C0INT_Msk (0x80UL) /*!< CTIMER INTEN: CTMRB3C0INT (Bitfield-Mask: 0x01) */ +#define CTIMER_INTEN_CTMRA3C0INT_Pos (6UL) /*!< CTIMER INTEN: CTMRA3C0INT (Bit 6) */ +#define CTIMER_INTEN_CTMRA3C0INT_Msk (0x40UL) /*!< CTIMER INTEN: CTMRA3C0INT (Bitfield-Mask: 0x01) */ +#define CTIMER_INTEN_CTMRB2C0INT_Pos (5UL) /*!< CTIMER INTEN: CTMRB2C0INT (Bit 5) */ +#define CTIMER_INTEN_CTMRB2C0INT_Msk (0x20UL) /*!< CTIMER INTEN: CTMRB2C0INT (Bitfield-Mask: 0x01) */ +#define CTIMER_INTEN_CTMRA2C0INT_Pos (4UL) /*!< CTIMER INTEN: CTMRA2C0INT (Bit 4) */ +#define CTIMER_INTEN_CTMRA2C0INT_Msk (0x10UL) /*!< CTIMER INTEN: CTMRA2C0INT (Bitfield-Mask: 0x01) */ +#define CTIMER_INTEN_CTMRB1C0INT_Pos (3UL) /*!< CTIMER INTEN: CTMRB1C0INT (Bit 3) */ +#define CTIMER_INTEN_CTMRB1C0INT_Msk (0x8UL) /*!< CTIMER INTEN: CTMRB1C0INT (Bitfield-Mask: 0x01) */ +#define CTIMER_INTEN_CTMRA1C0INT_Pos (2UL) /*!< CTIMER INTEN: CTMRA1C0INT (Bit 2) */ +#define CTIMER_INTEN_CTMRA1C0INT_Msk (0x4UL) /*!< CTIMER INTEN: CTMRA1C0INT (Bitfield-Mask: 0x01) */ +#define CTIMER_INTEN_CTMRB0C0INT_Pos (1UL) /*!< CTIMER INTEN: CTMRB0C0INT (Bit 1) */ +#define CTIMER_INTEN_CTMRB0C0INT_Msk (0x2UL) /*!< CTIMER INTEN: CTMRB0C0INT (Bitfield-Mask: 0x01) */ +#define CTIMER_INTEN_CTMRA0C0INT_Pos (0UL) /*!< CTIMER INTEN: CTMRA0C0INT (Bit 0) */ +#define CTIMER_INTEN_CTMRA0C0INT_Msk (0x1UL) /*!< CTIMER INTEN: CTMRA0C0INT (Bitfield-Mask: 0x01) */ +/* ======================================================== INTSTAT ======================================================== */ +#define CTIMER_INTSTAT_CTMRB7C1INT_Pos (31UL) /*!< CTIMER INTSTAT: CTMRB7C1INT (Bit 31) */ +#define CTIMER_INTSTAT_CTMRB7C1INT_Msk (0x80000000UL) /*!< CTIMER INTSTAT: CTMRB7C1INT (Bitfield-Mask: 0x01) */ +#define CTIMER_INTSTAT_CTMRA7C1INT_Pos (30UL) /*!< CTIMER INTSTAT: CTMRA7C1INT (Bit 30) */ +#define CTIMER_INTSTAT_CTMRA7C1INT_Msk (0x40000000UL) /*!< CTIMER INTSTAT: CTMRA7C1INT (Bitfield-Mask: 0x01) */ +#define CTIMER_INTSTAT_CTMRB6C1INT_Pos (29UL) /*!< CTIMER INTSTAT: CTMRB6C1INT (Bit 29) */ +#define CTIMER_INTSTAT_CTMRB6C1INT_Msk (0x20000000UL) /*!< CTIMER INTSTAT: CTMRB6C1INT (Bitfield-Mask: 0x01) */ +#define CTIMER_INTSTAT_CTMRA6C1INT_Pos (28UL) /*!< CTIMER INTSTAT: CTMRA6C1INT (Bit 28) */ +#define CTIMER_INTSTAT_CTMRA6C1INT_Msk (0x10000000UL) /*!< CTIMER INTSTAT: CTMRA6C1INT (Bitfield-Mask: 0x01) */ +#define CTIMER_INTSTAT_CTMRB5C1INT_Pos (27UL) /*!< CTIMER INTSTAT: CTMRB5C1INT (Bit 27) */ +#define CTIMER_INTSTAT_CTMRB5C1INT_Msk (0x8000000UL) /*!< CTIMER INTSTAT: CTMRB5C1INT (Bitfield-Mask: 0x01) */ +#define CTIMER_INTSTAT_CTMRA5C1INT_Pos (26UL) /*!< CTIMER INTSTAT: CTMRA5C1INT (Bit 26) */ +#define CTIMER_INTSTAT_CTMRA5C1INT_Msk (0x4000000UL) /*!< CTIMER INTSTAT: CTMRA5C1INT (Bitfield-Mask: 0x01) */ +#define CTIMER_INTSTAT_CTMRB4C1INT_Pos (25UL) /*!< CTIMER INTSTAT: CTMRB4C1INT (Bit 25) */ +#define CTIMER_INTSTAT_CTMRB4C1INT_Msk (0x2000000UL) /*!< CTIMER INTSTAT: CTMRB4C1INT (Bitfield-Mask: 0x01) */ +#define CTIMER_INTSTAT_CTMRA4C1INT_Pos (24UL) /*!< CTIMER INTSTAT: CTMRA4C1INT (Bit 24) */ +#define CTIMER_INTSTAT_CTMRA4C1INT_Msk (0x1000000UL) /*!< CTIMER INTSTAT: CTMRA4C1INT (Bitfield-Mask: 0x01) */ +#define CTIMER_INTSTAT_CTMRB3C1INT_Pos (23UL) /*!< CTIMER INTSTAT: CTMRB3C1INT (Bit 23) */ +#define CTIMER_INTSTAT_CTMRB3C1INT_Msk (0x800000UL) /*!< CTIMER INTSTAT: CTMRB3C1INT (Bitfield-Mask: 0x01) */ +#define CTIMER_INTSTAT_CTMRA3C1INT_Pos (22UL) /*!< CTIMER INTSTAT: CTMRA3C1INT (Bit 22) */ +#define CTIMER_INTSTAT_CTMRA3C1INT_Msk (0x400000UL) /*!< CTIMER INTSTAT: CTMRA3C1INT (Bitfield-Mask: 0x01) */ +#define CTIMER_INTSTAT_CTMRB2C1INT_Pos (21UL) /*!< CTIMER INTSTAT: CTMRB2C1INT (Bit 21) */ +#define CTIMER_INTSTAT_CTMRB2C1INT_Msk (0x200000UL) /*!< CTIMER INTSTAT: CTMRB2C1INT (Bitfield-Mask: 0x01) */ +#define CTIMER_INTSTAT_CTMRA2C1INT_Pos (20UL) /*!< CTIMER INTSTAT: CTMRA2C1INT (Bit 20) */ +#define CTIMER_INTSTAT_CTMRA2C1INT_Msk (0x100000UL) /*!< CTIMER INTSTAT: CTMRA2C1INT (Bitfield-Mask: 0x01) */ +#define CTIMER_INTSTAT_CTMRB1C1INT_Pos (19UL) /*!< CTIMER INTSTAT: CTMRB1C1INT (Bit 19) */ +#define CTIMER_INTSTAT_CTMRB1C1INT_Msk (0x80000UL) /*!< CTIMER INTSTAT: CTMRB1C1INT (Bitfield-Mask: 0x01) */ +#define CTIMER_INTSTAT_CTMRA1C1INT_Pos (18UL) /*!< CTIMER INTSTAT: CTMRA1C1INT (Bit 18) */ +#define CTIMER_INTSTAT_CTMRA1C1INT_Msk (0x40000UL) /*!< CTIMER INTSTAT: CTMRA1C1INT (Bitfield-Mask: 0x01) */ +#define CTIMER_INTSTAT_CTMRB0C1INT_Pos (17UL) /*!< CTIMER INTSTAT: CTMRB0C1INT (Bit 17) */ +#define CTIMER_INTSTAT_CTMRB0C1INT_Msk (0x20000UL) /*!< CTIMER INTSTAT: CTMRB0C1INT (Bitfield-Mask: 0x01) */ +#define CTIMER_INTSTAT_CTMRA0C1INT_Pos (16UL) /*!< CTIMER INTSTAT: CTMRA0C1INT (Bit 16) */ +#define CTIMER_INTSTAT_CTMRA0C1INT_Msk (0x10000UL) /*!< CTIMER INTSTAT: CTMRA0C1INT (Bitfield-Mask: 0x01) */ +#define CTIMER_INTSTAT_CTMRB7C0INT_Pos (15UL) /*!< CTIMER INTSTAT: CTMRB7C0INT (Bit 15) */ +#define CTIMER_INTSTAT_CTMRB7C0INT_Msk (0x8000UL) /*!< CTIMER INTSTAT: CTMRB7C0INT (Bitfield-Mask: 0x01) */ +#define CTIMER_INTSTAT_CTMRA7C0INT_Pos (14UL) /*!< CTIMER INTSTAT: CTMRA7C0INT (Bit 14) */ +#define CTIMER_INTSTAT_CTMRA7C0INT_Msk (0x4000UL) /*!< CTIMER INTSTAT: CTMRA7C0INT (Bitfield-Mask: 0x01) */ +#define CTIMER_INTSTAT_CTMRB6C0INT_Pos (13UL) /*!< CTIMER INTSTAT: CTMRB6C0INT (Bit 13) */ +#define CTIMER_INTSTAT_CTMRB6C0INT_Msk (0x2000UL) /*!< CTIMER INTSTAT: CTMRB6C0INT (Bitfield-Mask: 0x01) */ +#define CTIMER_INTSTAT_CTMRA6C0INT_Pos (12UL) /*!< CTIMER INTSTAT: CTMRA6C0INT (Bit 12) */ +#define CTIMER_INTSTAT_CTMRA6C0INT_Msk (0x1000UL) /*!< CTIMER INTSTAT: CTMRA6C0INT (Bitfield-Mask: 0x01) */ +#define CTIMER_INTSTAT_CTMRB5C0INT_Pos (11UL) /*!< CTIMER INTSTAT: CTMRB5C0INT (Bit 11) */ +#define CTIMER_INTSTAT_CTMRB5C0INT_Msk (0x800UL) /*!< CTIMER INTSTAT: CTMRB5C0INT (Bitfield-Mask: 0x01) */ +#define CTIMER_INTSTAT_CTMRA5C0INT_Pos (10UL) /*!< CTIMER INTSTAT: CTMRA5C0INT (Bit 10) */ +#define CTIMER_INTSTAT_CTMRA5C0INT_Msk (0x400UL) /*!< CTIMER INTSTAT: CTMRA5C0INT (Bitfield-Mask: 0x01) */ +#define CTIMER_INTSTAT_CTMRB4C0INT_Pos (9UL) /*!< CTIMER INTSTAT: CTMRB4C0INT (Bit 9) */ +#define CTIMER_INTSTAT_CTMRB4C0INT_Msk (0x200UL) /*!< CTIMER INTSTAT: CTMRB4C0INT (Bitfield-Mask: 0x01) */ +#define CTIMER_INTSTAT_CTMRA4C0INT_Pos (8UL) /*!< CTIMER INTSTAT: CTMRA4C0INT (Bit 8) */ +#define CTIMER_INTSTAT_CTMRA4C0INT_Msk (0x100UL) /*!< CTIMER INTSTAT: CTMRA4C0INT (Bitfield-Mask: 0x01) */ +#define CTIMER_INTSTAT_CTMRB3C0INT_Pos (7UL) /*!< CTIMER INTSTAT: CTMRB3C0INT (Bit 7) */ +#define CTIMER_INTSTAT_CTMRB3C0INT_Msk (0x80UL) /*!< CTIMER INTSTAT: CTMRB3C0INT (Bitfield-Mask: 0x01) */ +#define CTIMER_INTSTAT_CTMRA3C0INT_Pos (6UL) /*!< CTIMER INTSTAT: CTMRA3C0INT (Bit 6) */ +#define CTIMER_INTSTAT_CTMRA3C0INT_Msk (0x40UL) /*!< CTIMER INTSTAT: CTMRA3C0INT (Bitfield-Mask: 0x01) */ +#define CTIMER_INTSTAT_CTMRB2C0INT_Pos (5UL) /*!< CTIMER INTSTAT: CTMRB2C0INT (Bit 5) */ +#define CTIMER_INTSTAT_CTMRB2C0INT_Msk (0x20UL) /*!< CTIMER INTSTAT: CTMRB2C0INT (Bitfield-Mask: 0x01) */ +#define CTIMER_INTSTAT_CTMRA2C0INT_Pos (4UL) /*!< CTIMER INTSTAT: CTMRA2C0INT (Bit 4) */ +#define CTIMER_INTSTAT_CTMRA2C0INT_Msk (0x10UL) /*!< CTIMER INTSTAT: CTMRA2C0INT (Bitfield-Mask: 0x01) */ +#define CTIMER_INTSTAT_CTMRB1C0INT_Pos (3UL) /*!< CTIMER INTSTAT: CTMRB1C0INT (Bit 3) */ +#define CTIMER_INTSTAT_CTMRB1C0INT_Msk (0x8UL) /*!< CTIMER INTSTAT: CTMRB1C0INT (Bitfield-Mask: 0x01) */ +#define CTIMER_INTSTAT_CTMRA1C0INT_Pos (2UL) /*!< CTIMER INTSTAT: CTMRA1C0INT (Bit 2) */ +#define CTIMER_INTSTAT_CTMRA1C0INT_Msk (0x4UL) /*!< CTIMER INTSTAT: CTMRA1C0INT (Bitfield-Mask: 0x01) */ +#define CTIMER_INTSTAT_CTMRB0C0INT_Pos (1UL) /*!< CTIMER INTSTAT: CTMRB0C0INT (Bit 1) */ +#define CTIMER_INTSTAT_CTMRB0C0INT_Msk (0x2UL) /*!< CTIMER INTSTAT: CTMRB0C0INT (Bitfield-Mask: 0x01) */ +#define CTIMER_INTSTAT_CTMRA0C0INT_Pos (0UL) /*!< CTIMER INTSTAT: CTMRA0C0INT (Bit 0) */ +#define CTIMER_INTSTAT_CTMRA0C0INT_Msk (0x1UL) /*!< CTIMER INTSTAT: CTMRA0C0INT (Bitfield-Mask: 0x01) */ +/* ======================================================== INTCLR ========================================================= */ +#define CTIMER_INTCLR_CTMRB7C1INT_Pos (31UL) /*!< CTIMER INTCLR: CTMRB7C1INT (Bit 31) */ +#define CTIMER_INTCLR_CTMRB7C1INT_Msk (0x80000000UL) /*!< CTIMER INTCLR: CTMRB7C1INT (Bitfield-Mask: 0x01) */ +#define CTIMER_INTCLR_CTMRA7C1INT_Pos (30UL) /*!< CTIMER INTCLR: CTMRA7C1INT (Bit 30) */ +#define CTIMER_INTCLR_CTMRA7C1INT_Msk (0x40000000UL) /*!< CTIMER INTCLR: CTMRA7C1INT (Bitfield-Mask: 0x01) */ +#define CTIMER_INTCLR_CTMRB6C1INT_Pos (29UL) /*!< CTIMER INTCLR: CTMRB6C1INT (Bit 29) */ +#define CTIMER_INTCLR_CTMRB6C1INT_Msk (0x20000000UL) /*!< CTIMER INTCLR: CTMRB6C1INT (Bitfield-Mask: 0x01) */ +#define CTIMER_INTCLR_CTMRA6C1INT_Pos (28UL) /*!< CTIMER INTCLR: CTMRA6C1INT (Bit 28) */ +#define CTIMER_INTCLR_CTMRA6C1INT_Msk (0x10000000UL) /*!< CTIMER INTCLR: CTMRA6C1INT (Bitfield-Mask: 0x01) */ +#define CTIMER_INTCLR_CTMRB5C1INT_Pos (27UL) /*!< CTIMER INTCLR: CTMRB5C1INT (Bit 27) */ +#define CTIMER_INTCLR_CTMRB5C1INT_Msk (0x8000000UL) /*!< CTIMER INTCLR: CTMRB5C1INT (Bitfield-Mask: 0x01) */ +#define CTIMER_INTCLR_CTMRA5C1INT_Pos (26UL) /*!< CTIMER INTCLR: CTMRA5C1INT (Bit 26) */ +#define CTIMER_INTCLR_CTMRA5C1INT_Msk (0x4000000UL) /*!< CTIMER INTCLR: CTMRA5C1INT (Bitfield-Mask: 0x01) */ +#define CTIMER_INTCLR_CTMRB4C1INT_Pos (25UL) /*!< CTIMER INTCLR: CTMRB4C1INT (Bit 25) */ +#define CTIMER_INTCLR_CTMRB4C1INT_Msk (0x2000000UL) /*!< CTIMER INTCLR: CTMRB4C1INT (Bitfield-Mask: 0x01) */ +#define CTIMER_INTCLR_CTMRA4C1INT_Pos (24UL) /*!< CTIMER INTCLR: CTMRA4C1INT (Bit 24) */ +#define CTIMER_INTCLR_CTMRA4C1INT_Msk (0x1000000UL) /*!< CTIMER INTCLR: CTMRA4C1INT (Bitfield-Mask: 0x01) */ +#define CTIMER_INTCLR_CTMRB3C1INT_Pos (23UL) /*!< CTIMER INTCLR: CTMRB3C1INT (Bit 23) */ +#define CTIMER_INTCLR_CTMRB3C1INT_Msk (0x800000UL) /*!< CTIMER INTCLR: CTMRB3C1INT (Bitfield-Mask: 0x01) */ +#define CTIMER_INTCLR_CTMRA3C1INT_Pos (22UL) /*!< CTIMER INTCLR: CTMRA3C1INT (Bit 22) */ +#define CTIMER_INTCLR_CTMRA3C1INT_Msk (0x400000UL) /*!< CTIMER INTCLR: CTMRA3C1INT (Bitfield-Mask: 0x01) */ +#define CTIMER_INTCLR_CTMRB2C1INT_Pos (21UL) /*!< CTIMER INTCLR: CTMRB2C1INT (Bit 21) */ +#define CTIMER_INTCLR_CTMRB2C1INT_Msk (0x200000UL) /*!< CTIMER INTCLR: CTMRB2C1INT (Bitfield-Mask: 0x01) */ +#define CTIMER_INTCLR_CTMRA2C1INT_Pos (20UL) /*!< CTIMER INTCLR: CTMRA2C1INT (Bit 20) */ +#define CTIMER_INTCLR_CTMRA2C1INT_Msk (0x100000UL) /*!< CTIMER INTCLR: CTMRA2C1INT (Bitfield-Mask: 0x01) */ +#define CTIMER_INTCLR_CTMRB1C1INT_Pos (19UL) /*!< CTIMER INTCLR: CTMRB1C1INT (Bit 19) */ +#define CTIMER_INTCLR_CTMRB1C1INT_Msk (0x80000UL) /*!< CTIMER INTCLR: CTMRB1C1INT (Bitfield-Mask: 0x01) */ +#define CTIMER_INTCLR_CTMRA1C1INT_Pos (18UL) /*!< CTIMER INTCLR: CTMRA1C1INT (Bit 18) */ +#define CTIMER_INTCLR_CTMRA1C1INT_Msk (0x40000UL) /*!< CTIMER INTCLR: CTMRA1C1INT (Bitfield-Mask: 0x01) */ +#define CTIMER_INTCLR_CTMRB0C1INT_Pos (17UL) /*!< CTIMER INTCLR: CTMRB0C1INT (Bit 17) */ +#define CTIMER_INTCLR_CTMRB0C1INT_Msk (0x20000UL) /*!< CTIMER INTCLR: CTMRB0C1INT (Bitfield-Mask: 0x01) */ +#define CTIMER_INTCLR_CTMRA0C1INT_Pos (16UL) /*!< CTIMER INTCLR: CTMRA0C1INT (Bit 16) */ +#define CTIMER_INTCLR_CTMRA0C1INT_Msk (0x10000UL) /*!< CTIMER INTCLR: CTMRA0C1INT (Bitfield-Mask: 0x01) */ +#define CTIMER_INTCLR_CTMRB7C0INT_Pos (15UL) /*!< CTIMER INTCLR: CTMRB7C0INT (Bit 15) */ +#define CTIMER_INTCLR_CTMRB7C0INT_Msk (0x8000UL) /*!< CTIMER INTCLR: CTMRB7C0INT (Bitfield-Mask: 0x01) */ +#define CTIMER_INTCLR_CTMRA7C0INT_Pos (14UL) /*!< CTIMER INTCLR: CTMRA7C0INT (Bit 14) */ +#define CTIMER_INTCLR_CTMRA7C0INT_Msk (0x4000UL) /*!< CTIMER INTCLR: CTMRA7C0INT (Bitfield-Mask: 0x01) */ +#define CTIMER_INTCLR_CTMRB6C0INT_Pos (13UL) /*!< CTIMER INTCLR: CTMRB6C0INT (Bit 13) */ +#define CTIMER_INTCLR_CTMRB6C0INT_Msk (0x2000UL) /*!< CTIMER INTCLR: CTMRB6C0INT (Bitfield-Mask: 0x01) */ +#define CTIMER_INTCLR_CTMRA6C0INT_Pos (12UL) /*!< CTIMER INTCLR: CTMRA6C0INT (Bit 12) */ +#define CTIMER_INTCLR_CTMRA6C0INT_Msk (0x1000UL) /*!< CTIMER INTCLR: CTMRA6C0INT (Bitfield-Mask: 0x01) */ +#define CTIMER_INTCLR_CTMRB5C0INT_Pos (11UL) /*!< CTIMER INTCLR: CTMRB5C0INT (Bit 11) */ +#define CTIMER_INTCLR_CTMRB5C0INT_Msk (0x800UL) /*!< CTIMER INTCLR: CTMRB5C0INT (Bitfield-Mask: 0x01) */ +#define CTIMER_INTCLR_CTMRA5C0INT_Pos (10UL) /*!< CTIMER INTCLR: CTMRA5C0INT (Bit 10) */ +#define CTIMER_INTCLR_CTMRA5C0INT_Msk (0x400UL) /*!< CTIMER INTCLR: CTMRA5C0INT (Bitfield-Mask: 0x01) */ +#define CTIMER_INTCLR_CTMRB4C0INT_Pos (9UL) /*!< CTIMER INTCLR: CTMRB4C0INT (Bit 9) */ +#define CTIMER_INTCLR_CTMRB4C0INT_Msk (0x200UL) /*!< CTIMER INTCLR: CTMRB4C0INT (Bitfield-Mask: 0x01) */ +#define CTIMER_INTCLR_CTMRA4C0INT_Pos (8UL) /*!< CTIMER INTCLR: CTMRA4C0INT (Bit 8) */ +#define CTIMER_INTCLR_CTMRA4C0INT_Msk (0x100UL) /*!< CTIMER INTCLR: CTMRA4C0INT (Bitfield-Mask: 0x01) */ +#define CTIMER_INTCLR_CTMRB3C0INT_Pos (7UL) /*!< CTIMER INTCLR: CTMRB3C0INT (Bit 7) */ +#define CTIMER_INTCLR_CTMRB3C0INT_Msk (0x80UL) /*!< CTIMER INTCLR: CTMRB3C0INT (Bitfield-Mask: 0x01) */ +#define CTIMER_INTCLR_CTMRA3C0INT_Pos (6UL) /*!< CTIMER INTCLR: CTMRA3C0INT (Bit 6) */ +#define CTIMER_INTCLR_CTMRA3C0INT_Msk (0x40UL) /*!< CTIMER INTCLR: CTMRA3C0INT (Bitfield-Mask: 0x01) */ +#define CTIMER_INTCLR_CTMRB2C0INT_Pos (5UL) /*!< CTIMER INTCLR: CTMRB2C0INT (Bit 5) */ +#define CTIMER_INTCLR_CTMRB2C0INT_Msk (0x20UL) /*!< CTIMER INTCLR: CTMRB2C0INT (Bitfield-Mask: 0x01) */ +#define CTIMER_INTCLR_CTMRA2C0INT_Pos (4UL) /*!< CTIMER INTCLR: CTMRA2C0INT (Bit 4) */ +#define CTIMER_INTCLR_CTMRA2C0INT_Msk (0x10UL) /*!< CTIMER INTCLR: CTMRA2C0INT (Bitfield-Mask: 0x01) */ +#define CTIMER_INTCLR_CTMRB1C0INT_Pos (3UL) /*!< CTIMER INTCLR: CTMRB1C0INT (Bit 3) */ +#define CTIMER_INTCLR_CTMRB1C0INT_Msk (0x8UL) /*!< CTIMER INTCLR: CTMRB1C0INT (Bitfield-Mask: 0x01) */ +#define CTIMER_INTCLR_CTMRA1C0INT_Pos (2UL) /*!< CTIMER INTCLR: CTMRA1C0INT (Bit 2) */ +#define CTIMER_INTCLR_CTMRA1C0INT_Msk (0x4UL) /*!< CTIMER INTCLR: CTMRA1C0INT (Bitfield-Mask: 0x01) */ +#define CTIMER_INTCLR_CTMRB0C0INT_Pos (1UL) /*!< CTIMER INTCLR: CTMRB0C0INT (Bit 1) */ +#define CTIMER_INTCLR_CTMRB0C0INT_Msk (0x2UL) /*!< CTIMER INTCLR: CTMRB0C0INT (Bitfield-Mask: 0x01) */ +#define CTIMER_INTCLR_CTMRA0C0INT_Pos (0UL) /*!< CTIMER INTCLR: CTMRA0C0INT (Bit 0) */ +#define CTIMER_INTCLR_CTMRA0C0INT_Msk (0x1UL) /*!< CTIMER INTCLR: CTMRA0C0INT (Bitfield-Mask: 0x01) */ +/* ======================================================== INTSET ========================================================= */ +#define CTIMER_INTSET_CTMRB7C1INT_Pos (31UL) /*!< CTIMER INTSET: CTMRB7C1INT (Bit 31) */ +#define CTIMER_INTSET_CTMRB7C1INT_Msk (0x80000000UL) /*!< CTIMER INTSET: CTMRB7C1INT (Bitfield-Mask: 0x01) */ +#define CTIMER_INTSET_CTMRA7C1INT_Pos (30UL) /*!< CTIMER INTSET: CTMRA7C1INT (Bit 30) */ +#define CTIMER_INTSET_CTMRA7C1INT_Msk (0x40000000UL) /*!< CTIMER INTSET: CTMRA7C1INT (Bitfield-Mask: 0x01) */ +#define CTIMER_INTSET_CTMRB6C1INT_Pos (29UL) /*!< CTIMER INTSET: CTMRB6C1INT (Bit 29) */ +#define CTIMER_INTSET_CTMRB6C1INT_Msk (0x20000000UL) /*!< CTIMER INTSET: CTMRB6C1INT (Bitfield-Mask: 0x01) */ +#define CTIMER_INTSET_CTMRA6C1INT_Pos (28UL) /*!< CTIMER INTSET: CTMRA6C1INT (Bit 28) */ +#define CTIMER_INTSET_CTMRA6C1INT_Msk (0x10000000UL) /*!< CTIMER INTSET: CTMRA6C1INT (Bitfield-Mask: 0x01) */ +#define CTIMER_INTSET_CTMRB5C1INT_Pos (27UL) /*!< CTIMER INTSET: CTMRB5C1INT (Bit 27) */ +#define CTIMER_INTSET_CTMRB5C1INT_Msk (0x8000000UL) /*!< CTIMER INTSET: CTMRB5C1INT (Bitfield-Mask: 0x01) */ +#define CTIMER_INTSET_CTMRA5C1INT_Pos (26UL) /*!< CTIMER INTSET: CTMRA5C1INT (Bit 26) */ +#define CTIMER_INTSET_CTMRA5C1INT_Msk (0x4000000UL) /*!< CTIMER INTSET: CTMRA5C1INT (Bitfield-Mask: 0x01) */ +#define CTIMER_INTSET_CTMRB4C1INT_Pos (25UL) /*!< CTIMER INTSET: CTMRB4C1INT (Bit 25) */ +#define CTIMER_INTSET_CTMRB4C1INT_Msk (0x2000000UL) /*!< CTIMER INTSET: CTMRB4C1INT (Bitfield-Mask: 0x01) */ +#define CTIMER_INTSET_CTMRA4C1INT_Pos (24UL) /*!< CTIMER INTSET: CTMRA4C1INT (Bit 24) */ +#define CTIMER_INTSET_CTMRA4C1INT_Msk (0x1000000UL) /*!< CTIMER INTSET: CTMRA4C1INT (Bitfield-Mask: 0x01) */ +#define CTIMER_INTSET_CTMRB3C1INT_Pos (23UL) /*!< CTIMER INTSET: CTMRB3C1INT (Bit 23) */ +#define CTIMER_INTSET_CTMRB3C1INT_Msk (0x800000UL) /*!< CTIMER INTSET: CTMRB3C1INT (Bitfield-Mask: 0x01) */ +#define CTIMER_INTSET_CTMRA3C1INT_Pos (22UL) /*!< CTIMER INTSET: CTMRA3C1INT (Bit 22) */ +#define CTIMER_INTSET_CTMRA3C1INT_Msk (0x400000UL) /*!< CTIMER INTSET: CTMRA3C1INT (Bitfield-Mask: 0x01) */ +#define CTIMER_INTSET_CTMRB2C1INT_Pos (21UL) /*!< CTIMER INTSET: CTMRB2C1INT (Bit 21) */ +#define CTIMER_INTSET_CTMRB2C1INT_Msk (0x200000UL) /*!< CTIMER INTSET: CTMRB2C1INT (Bitfield-Mask: 0x01) */ +#define CTIMER_INTSET_CTMRA2C1INT_Pos (20UL) /*!< CTIMER INTSET: CTMRA2C1INT (Bit 20) */ +#define CTIMER_INTSET_CTMRA2C1INT_Msk (0x100000UL) /*!< CTIMER INTSET: CTMRA2C1INT (Bitfield-Mask: 0x01) */ +#define CTIMER_INTSET_CTMRB1C1INT_Pos (19UL) /*!< CTIMER INTSET: CTMRB1C1INT (Bit 19) */ +#define CTIMER_INTSET_CTMRB1C1INT_Msk (0x80000UL) /*!< CTIMER INTSET: CTMRB1C1INT (Bitfield-Mask: 0x01) */ +#define CTIMER_INTSET_CTMRA1C1INT_Pos (18UL) /*!< CTIMER INTSET: CTMRA1C1INT (Bit 18) */ +#define CTIMER_INTSET_CTMRA1C1INT_Msk (0x40000UL) /*!< CTIMER INTSET: CTMRA1C1INT (Bitfield-Mask: 0x01) */ +#define CTIMER_INTSET_CTMRB0C1INT_Pos (17UL) /*!< CTIMER INTSET: CTMRB0C1INT (Bit 17) */ +#define CTIMER_INTSET_CTMRB0C1INT_Msk (0x20000UL) /*!< CTIMER INTSET: CTMRB0C1INT (Bitfield-Mask: 0x01) */ +#define CTIMER_INTSET_CTMRA0C1INT_Pos (16UL) /*!< CTIMER INTSET: CTMRA0C1INT (Bit 16) */ +#define CTIMER_INTSET_CTMRA0C1INT_Msk (0x10000UL) /*!< CTIMER INTSET: CTMRA0C1INT (Bitfield-Mask: 0x01) */ +#define CTIMER_INTSET_CTMRB7C0INT_Pos (15UL) /*!< CTIMER INTSET: CTMRB7C0INT (Bit 15) */ +#define CTIMER_INTSET_CTMRB7C0INT_Msk (0x8000UL) /*!< CTIMER INTSET: CTMRB7C0INT (Bitfield-Mask: 0x01) */ +#define CTIMER_INTSET_CTMRA7C0INT_Pos (14UL) /*!< CTIMER INTSET: CTMRA7C0INT (Bit 14) */ +#define CTIMER_INTSET_CTMRA7C0INT_Msk (0x4000UL) /*!< CTIMER INTSET: CTMRA7C0INT (Bitfield-Mask: 0x01) */ +#define CTIMER_INTSET_CTMRB6C0INT_Pos (13UL) /*!< CTIMER INTSET: CTMRB6C0INT (Bit 13) */ +#define CTIMER_INTSET_CTMRB6C0INT_Msk (0x2000UL) /*!< CTIMER INTSET: CTMRB6C0INT (Bitfield-Mask: 0x01) */ +#define CTIMER_INTSET_CTMRA6C0INT_Pos (12UL) /*!< CTIMER INTSET: CTMRA6C0INT (Bit 12) */ +#define CTIMER_INTSET_CTMRA6C0INT_Msk (0x1000UL) /*!< CTIMER INTSET: CTMRA6C0INT (Bitfield-Mask: 0x01) */ +#define CTIMER_INTSET_CTMRB5C0INT_Pos (11UL) /*!< CTIMER INTSET: CTMRB5C0INT (Bit 11) */ +#define CTIMER_INTSET_CTMRB5C0INT_Msk (0x800UL) /*!< CTIMER INTSET: CTMRB5C0INT (Bitfield-Mask: 0x01) */ +#define CTIMER_INTSET_CTMRA5C0INT_Pos (10UL) /*!< CTIMER INTSET: CTMRA5C0INT (Bit 10) */ +#define CTIMER_INTSET_CTMRA5C0INT_Msk (0x400UL) /*!< CTIMER INTSET: CTMRA5C0INT (Bitfield-Mask: 0x01) */ +#define CTIMER_INTSET_CTMRB4C0INT_Pos (9UL) /*!< CTIMER INTSET: CTMRB4C0INT (Bit 9) */ +#define CTIMER_INTSET_CTMRB4C0INT_Msk (0x200UL) /*!< CTIMER INTSET: CTMRB4C0INT (Bitfield-Mask: 0x01) */ +#define CTIMER_INTSET_CTMRA4C0INT_Pos (8UL) /*!< CTIMER INTSET: CTMRA4C0INT (Bit 8) */ +#define CTIMER_INTSET_CTMRA4C0INT_Msk (0x100UL) /*!< CTIMER INTSET: CTMRA4C0INT (Bitfield-Mask: 0x01) */ +#define CTIMER_INTSET_CTMRB3C0INT_Pos (7UL) /*!< CTIMER INTSET: CTMRB3C0INT (Bit 7) */ +#define CTIMER_INTSET_CTMRB3C0INT_Msk (0x80UL) /*!< CTIMER INTSET: CTMRB3C0INT (Bitfield-Mask: 0x01) */ +#define CTIMER_INTSET_CTMRA3C0INT_Pos (6UL) /*!< CTIMER INTSET: CTMRA3C0INT (Bit 6) */ +#define CTIMER_INTSET_CTMRA3C0INT_Msk (0x40UL) /*!< CTIMER INTSET: CTMRA3C0INT (Bitfield-Mask: 0x01) */ +#define CTIMER_INTSET_CTMRB2C0INT_Pos (5UL) /*!< CTIMER INTSET: CTMRB2C0INT (Bit 5) */ +#define CTIMER_INTSET_CTMRB2C0INT_Msk (0x20UL) /*!< CTIMER INTSET: CTMRB2C0INT (Bitfield-Mask: 0x01) */ +#define CTIMER_INTSET_CTMRA2C0INT_Pos (4UL) /*!< CTIMER INTSET: CTMRA2C0INT (Bit 4) */ +#define CTIMER_INTSET_CTMRA2C0INT_Msk (0x10UL) /*!< CTIMER INTSET: CTMRA2C0INT (Bitfield-Mask: 0x01) */ +#define CTIMER_INTSET_CTMRB1C0INT_Pos (3UL) /*!< CTIMER INTSET: CTMRB1C0INT (Bit 3) */ +#define CTIMER_INTSET_CTMRB1C0INT_Msk (0x8UL) /*!< CTIMER INTSET: CTMRB1C0INT (Bitfield-Mask: 0x01) */ +#define CTIMER_INTSET_CTMRA1C0INT_Pos (2UL) /*!< CTIMER INTSET: CTMRA1C0INT (Bit 2) */ +#define CTIMER_INTSET_CTMRA1C0INT_Msk (0x4UL) /*!< CTIMER INTSET: CTMRA1C0INT (Bitfield-Mask: 0x01) */ +#define CTIMER_INTSET_CTMRB0C0INT_Pos (1UL) /*!< CTIMER INTSET: CTMRB0C0INT (Bit 1) */ +#define CTIMER_INTSET_CTMRB0C0INT_Msk (0x2UL) /*!< CTIMER INTSET: CTMRB0C0INT (Bitfield-Mask: 0x01) */ +#define CTIMER_INTSET_CTMRA0C0INT_Pos (0UL) /*!< CTIMER INTSET: CTMRA0C0INT (Bit 0) */ +#define CTIMER_INTSET_CTMRA0C0INT_Msk (0x1UL) /*!< CTIMER INTSET: CTMRA0C0INT (Bitfield-Mask: 0x01) */ +/* ======================================================= STMINTEN ======================================================== */ +#define CTIMER_STMINTEN_CAPTURED_Pos (12UL) /*!< CTIMER STMINTEN: CAPTURED (Bit 12) */ +#define CTIMER_STMINTEN_CAPTURED_Msk (0x1000UL) /*!< CTIMER STMINTEN: CAPTURED (Bitfield-Mask: 0x01) */ +#define CTIMER_STMINTEN_CAPTUREC_Pos (11UL) /*!< CTIMER STMINTEN: CAPTUREC (Bit 11) */ +#define CTIMER_STMINTEN_CAPTUREC_Msk (0x800UL) /*!< CTIMER STMINTEN: CAPTUREC (Bitfield-Mask: 0x01) */ +#define CTIMER_STMINTEN_CAPTUREB_Pos (10UL) /*!< CTIMER STMINTEN: CAPTUREB (Bit 10) */ +#define CTIMER_STMINTEN_CAPTUREB_Msk (0x400UL) /*!< CTIMER STMINTEN: CAPTUREB (Bitfield-Mask: 0x01) */ +#define CTIMER_STMINTEN_CAPTUREA_Pos (9UL) /*!< CTIMER STMINTEN: CAPTUREA (Bit 9) */ +#define CTIMER_STMINTEN_CAPTUREA_Msk (0x200UL) /*!< CTIMER STMINTEN: CAPTUREA (Bitfield-Mask: 0x01) */ +#define CTIMER_STMINTEN_OVERFLOW_Pos (8UL) /*!< CTIMER STMINTEN: OVERFLOW (Bit 8) */ +#define CTIMER_STMINTEN_OVERFLOW_Msk (0x100UL) /*!< CTIMER STMINTEN: OVERFLOW (Bitfield-Mask: 0x01) */ +#define CTIMER_STMINTEN_COMPAREH_Pos (7UL) /*!< CTIMER STMINTEN: COMPAREH (Bit 7) */ +#define CTIMER_STMINTEN_COMPAREH_Msk (0x80UL) /*!< CTIMER STMINTEN: COMPAREH (Bitfield-Mask: 0x01) */ +#define CTIMER_STMINTEN_COMPAREG_Pos (6UL) /*!< CTIMER STMINTEN: COMPAREG (Bit 6) */ +#define CTIMER_STMINTEN_COMPAREG_Msk (0x40UL) /*!< CTIMER STMINTEN: COMPAREG (Bitfield-Mask: 0x01) */ +#define CTIMER_STMINTEN_COMPAREF_Pos (5UL) /*!< CTIMER STMINTEN: COMPAREF (Bit 5) */ +#define CTIMER_STMINTEN_COMPAREF_Msk (0x20UL) /*!< CTIMER STMINTEN: COMPAREF (Bitfield-Mask: 0x01) */ +#define CTIMER_STMINTEN_COMPAREE_Pos (4UL) /*!< CTIMER STMINTEN: COMPAREE (Bit 4) */ +#define CTIMER_STMINTEN_COMPAREE_Msk (0x10UL) /*!< CTIMER STMINTEN: COMPAREE (Bitfield-Mask: 0x01) */ +#define CTIMER_STMINTEN_COMPARED_Pos (3UL) /*!< CTIMER STMINTEN: COMPARED (Bit 3) */ +#define CTIMER_STMINTEN_COMPARED_Msk (0x8UL) /*!< CTIMER STMINTEN: COMPARED (Bitfield-Mask: 0x01) */ +#define CTIMER_STMINTEN_COMPAREC_Pos (2UL) /*!< CTIMER STMINTEN: COMPAREC (Bit 2) */ +#define CTIMER_STMINTEN_COMPAREC_Msk (0x4UL) /*!< CTIMER STMINTEN: COMPAREC (Bitfield-Mask: 0x01) */ +#define CTIMER_STMINTEN_COMPAREB_Pos (1UL) /*!< CTIMER STMINTEN: COMPAREB (Bit 1) */ +#define CTIMER_STMINTEN_COMPAREB_Msk (0x2UL) /*!< CTIMER STMINTEN: COMPAREB (Bitfield-Mask: 0x01) */ +#define CTIMER_STMINTEN_COMPAREA_Pos (0UL) /*!< CTIMER STMINTEN: COMPAREA (Bit 0) */ +#define CTIMER_STMINTEN_COMPAREA_Msk (0x1UL) /*!< CTIMER STMINTEN: COMPAREA (Bitfield-Mask: 0x01) */ +/* ====================================================== STMINTSTAT ======================================================= */ +#define CTIMER_STMINTSTAT_CAPTURED_Pos (12UL) /*!< CTIMER STMINTSTAT: CAPTURED (Bit 12) */ +#define CTIMER_STMINTSTAT_CAPTURED_Msk (0x1000UL) /*!< CTIMER STMINTSTAT: CAPTURED (Bitfield-Mask: 0x01) */ +#define CTIMER_STMINTSTAT_CAPTUREC_Pos (11UL) /*!< CTIMER STMINTSTAT: CAPTUREC (Bit 11) */ +#define CTIMER_STMINTSTAT_CAPTUREC_Msk (0x800UL) /*!< CTIMER STMINTSTAT: CAPTUREC (Bitfield-Mask: 0x01) */ +#define CTIMER_STMINTSTAT_CAPTUREB_Pos (10UL) /*!< CTIMER STMINTSTAT: CAPTUREB (Bit 10) */ +#define CTIMER_STMINTSTAT_CAPTUREB_Msk (0x400UL) /*!< CTIMER STMINTSTAT: CAPTUREB (Bitfield-Mask: 0x01) */ +#define CTIMER_STMINTSTAT_CAPTUREA_Pos (9UL) /*!< CTIMER STMINTSTAT: CAPTUREA (Bit 9) */ +#define CTIMER_STMINTSTAT_CAPTUREA_Msk (0x200UL) /*!< CTIMER STMINTSTAT: CAPTUREA (Bitfield-Mask: 0x01) */ +#define CTIMER_STMINTSTAT_OVERFLOW_Pos (8UL) /*!< CTIMER STMINTSTAT: OVERFLOW (Bit 8) */ +#define CTIMER_STMINTSTAT_OVERFLOW_Msk (0x100UL) /*!< CTIMER STMINTSTAT: OVERFLOW (Bitfield-Mask: 0x01) */ +#define CTIMER_STMINTSTAT_COMPAREH_Pos (7UL) /*!< CTIMER STMINTSTAT: COMPAREH (Bit 7) */ +#define CTIMER_STMINTSTAT_COMPAREH_Msk (0x80UL) /*!< CTIMER STMINTSTAT: COMPAREH (Bitfield-Mask: 0x01) */ +#define CTIMER_STMINTSTAT_COMPAREG_Pos (6UL) /*!< CTIMER STMINTSTAT: COMPAREG (Bit 6) */ +#define CTIMER_STMINTSTAT_COMPAREG_Msk (0x40UL) /*!< CTIMER STMINTSTAT: COMPAREG (Bitfield-Mask: 0x01) */ +#define CTIMER_STMINTSTAT_COMPAREF_Pos (5UL) /*!< CTIMER STMINTSTAT: COMPAREF (Bit 5) */ +#define CTIMER_STMINTSTAT_COMPAREF_Msk (0x20UL) /*!< CTIMER STMINTSTAT: COMPAREF (Bitfield-Mask: 0x01) */ +#define CTIMER_STMINTSTAT_COMPAREE_Pos (4UL) /*!< CTIMER STMINTSTAT: COMPAREE (Bit 4) */ +#define CTIMER_STMINTSTAT_COMPAREE_Msk (0x10UL) /*!< CTIMER STMINTSTAT: COMPAREE (Bitfield-Mask: 0x01) */ +#define CTIMER_STMINTSTAT_COMPARED_Pos (3UL) /*!< CTIMER STMINTSTAT: COMPARED (Bit 3) */ +#define CTIMER_STMINTSTAT_COMPARED_Msk (0x8UL) /*!< CTIMER STMINTSTAT: COMPARED (Bitfield-Mask: 0x01) */ +#define CTIMER_STMINTSTAT_COMPAREC_Pos (2UL) /*!< CTIMER STMINTSTAT: COMPAREC (Bit 2) */ +#define CTIMER_STMINTSTAT_COMPAREC_Msk (0x4UL) /*!< CTIMER STMINTSTAT: COMPAREC (Bitfield-Mask: 0x01) */ +#define CTIMER_STMINTSTAT_COMPAREB_Pos (1UL) /*!< CTIMER STMINTSTAT: COMPAREB (Bit 1) */ +#define CTIMER_STMINTSTAT_COMPAREB_Msk (0x2UL) /*!< CTIMER STMINTSTAT: COMPAREB (Bitfield-Mask: 0x01) */ +#define CTIMER_STMINTSTAT_COMPAREA_Pos (0UL) /*!< CTIMER STMINTSTAT: COMPAREA (Bit 0) */ +#define CTIMER_STMINTSTAT_COMPAREA_Msk (0x1UL) /*!< CTIMER STMINTSTAT: COMPAREA (Bitfield-Mask: 0x01) */ +/* ======================================================= STMINTCLR ======================================================= */ +#define CTIMER_STMINTCLR_CAPTURED_Pos (12UL) /*!< CTIMER STMINTCLR: CAPTURED (Bit 12) */ +#define CTIMER_STMINTCLR_CAPTURED_Msk (0x1000UL) /*!< CTIMER STMINTCLR: CAPTURED (Bitfield-Mask: 0x01) */ +#define CTIMER_STMINTCLR_CAPTUREC_Pos (11UL) /*!< CTIMER STMINTCLR: CAPTUREC (Bit 11) */ +#define CTIMER_STMINTCLR_CAPTUREC_Msk (0x800UL) /*!< CTIMER STMINTCLR: CAPTUREC (Bitfield-Mask: 0x01) */ +#define CTIMER_STMINTCLR_CAPTUREB_Pos (10UL) /*!< CTIMER STMINTCLR: CAPTUREB (Bit 10) */ +#define CTIMER_STMINTCLR_CAPTUREB_Msk (0x400UL) /*!< CTIMER STMINTCLR: CAPTUREB (Bitfield-Mask: 0x01) */ +#define CTIMER_STMINTCLR_CAPTUREA_Pos (9UL) /*!< CTIMER STMINTCLR: CAPTUREA (Bit 9) */ +#define CTIMER_STMINTCLR_CAPTUREA_Msk (0x200UL) /*!< CTIMER STMINTCLR: CAPTUREA (Bitfield-Mask: 0x01) */ +#define CTIMER_STMINTCLR_OVERFLOW_Pos (8UL) /*!< CTIMER STMINTCLR: OVERFLOW (Bit 8) */ +#define CTIMER_STMINTCLR_OVERFLOW_Msk (0x100UL) /*!< CTIMER STMINTCLR: OVERFLOW (Bitfield-Mask: 0x01) */ +#define CTIMER_STMINTCLR_COMPAREH_Pos (7UL) /*!< CTIMER STMINTCLR: COMPAREH (Bit 7) */ +#define CTIMER_STMINTCLR_COMPAREH_Msk (0x80UL) /*!< CTIMER STMINTCLR: COMPAREH (Bitfield-Mask: 0x01) */ +#define CTIMER_STMINTCLR_COMPAREG_Pos (6UL) /*!< CTIMER STMINTCLR: COMPAREG (Bit 6) */ +#define CTIMER_STMINTCLR_COMPAREG_Msk (0x40UL) /*!< CTIMER STMINTCLR: COMPAREG (Bitfield-Mask: 0x01) */ +#define CTIMER_STMINTCLR_COMPAREF_Pos (5UL) /*!< CTIMER STMINTCLR: COMPAREF (Bit 5) */ +#define CTIMER_STMINTCLR_COMPAREF_Msk (0x20UL) /*!< CTIMER STMINTCLR: COMPAREF (Bitfield-Mask: 0x01) */ +#define CTIMER_STMINTCLR_COMPAREE_Pos (4UL) /*!< CTIMER STMINTCLR: COMPAREE (Bit 4) */ +#define CTIMER_STMINTCLR_COMPAREE_Msk (0x10UL) /*!< CTIMER STMINTCLR: COMPAREE (Bitfield-Mask: 0x01) */ +#define CTIMER_STMINTCLR_COMPARED_Pos (3UL) /*!< CTIMER STMINTCLR: COMPARED (Bit 3) */ +#define CTIMER_STMINTCLR_COMPARED_Msk (0x8UL) /*!< CTIMER STMINTCLR: COMPARED (Bitfield-Mask: 0x01) */ +#define CTIMER_STMINTCLR_COMPAREC_Pos (2UL) /*!< CTIMER STMINTCLR: COMPAREC (Bit 2) */ +#define CTIMER_STMINTCLR_COMPAREC_Msk (0x4UL) /*!< CTIMER STMINTCLR: COMPAREC (Bitfield-Mask: 0x01) */ +#define CTIMER_STMINTCLR_COMPAREB_Pos (1UL) /*!< CTIMER STMINTCLR: COMPAREB (Bit 1) */ +#define CTIMER_STMINTCLR_COMPAREB_Msk (0x2UL) /*!< CTIMER STMINTCLR: COMPAREB (Bitfield-Mask: 0x01) */ +#define CTIMER_STMINTCLR_COMPAREA_Pos (0UL) /*!< CTIMER STMINTCLR: COMPAREA (Bit 0) */ +#define CTIMER_STMINTCLR_COMPAREA_Msk (0x1UL) /*!< CTIMER STMINTCLR: COMPAREA (Bitfield-Mask: 0x01) */ +/* ======================================================= STMINTSET ======================================================= */ +#define CTIMER_STMINTSET_CAPTURED_Pos (12UL) /*!< CTIMER STMINTSET: CAPTURED (Bit 12) */ +#define CTIMER_STMINTSET_CAPTURED_Msk (0x1000UL) /*!< CTIMER STMINTSET: CAPTURED (Bitfield-Mask: 0x01) */ +#define CTIMER_STMINTSET_CAPTUREC_Pos (11UL) /*!< CTIMER STMINTSET: CAPTUREC (Bit 11) */ +#define CTIMER_STMINTSET_CAPTUREC_Msk (0x800UL) /*!< CTIMER STMINTSET: CAPTUREC (Bitfield-Mask: 0x01) */ +#define CTIMER_STMINTSET_CAPTUREB_Pos (10UL) /*!< CTIMER STMINTSET: CAPTUREB (Bit 10) */ +#define CTIMER_STMINTSET_CAPTUREB_Msk (0x400UL) /*!< CTIMER STMINTSET: CAPTUREB (Bitfield-Mask: 0x01) */ +#define CTIMER_STMINTSET_CAPTUREA_Pos (9UL) /*!< CTIMER STMINTSET: CAPTUREA (Bit 9) */ +#define CTIMER_STMINTSET_CAPTUREA_Msk (0x200UL) /*!< CTIMER STMINTSET: CAPTUREA (Bitfield-Mask: 0x01) */ +#define CTIMER_STMINTSET_OVERFLOW_Pos (8UL) /*!< CTIMER STMINTSET: OVERFLOW (Bit 8) */ +#define CTIMER_STMINTSET_OVERFLOW_Msk (0x100UL) /*!< CTIMER STMINTSET: OVERFLOW (Bitfield-Mask: 0x01) */ +#define CTIMER_STMINTSET_COMPAREH_Pos (7UL) /*!< CTIMER STMINTSET: COMPAREH (Bit 7) */ +#define CTIMER_STMINTSET_COMPAREH_Msk (0x80UL) /*!< CTIMER STMINTSET: COMPAREH (Bitfield-Mask: 0x01) */ +#define CTIMER_STMINTSET_COMPAREG_Pos (6UL) /*!< CTIMER STMINTSET: COMPAREG (Bit 6) */ +#define CTIMER_STMINTSET_COMPAREG_Msk (0x40UL) /*!< CTIMER STMINTSET: COMPAREG (Bitfield-Mask: 0x01) */ +#define CTIMER_STMINTSET_COMPAREF_Pos (5UL) /*!< CTIMER STMINTSET: COMPAREF (Bit 5) */ +#define CTIMER_STMINTSET_COMPAREF_Msk (0x20UL) /*!< CTIMER STMINTSET: COMPAREF (Bitfield-Mask: 0x01) */ +#define CTIMER_STMINTSET_COMPAREE_Pos (4UL) /*!< CTIMER STMINTSET: COMPAREE (Bit 4) */ +#define CTIMER_STMINTSET_COMPAREE_Msk (0x10UL) /*!< CTIMER STMINTSET: COMPAREE (Bitfield-Mask: 0x01) */ +#define CTIMER_STMINTSET_COMPARED_Pos (3UL) /*!< CTIMER STMINTSET: COMPARED (Bit 3) */ +#define CTIMER_STMINTSET_COMPARED_Msk (0x8UL) /*!< CTIMER STMINTSET: COMPARED (Bitfield-Mask: 0x01) */ +#define CTIMER_STMINTSET_COMPAREC_Pos (2UL) /*!< CTIMER STMINTSET: COMPAREC (Bit 2) */ +#define CTIMER_STMINTSET_COMPAREC_Msk (0x4UL) /*!< CTIMER STMINTSET: COMPAREC (Bitfield-Mask: 0x01) */ +#define CTIMER_STMINTSET_COMPAREB_Pos (1UL) /*!< CTIMER STMINTSET: COMPAREB (Bit 1) */ +#define CTIMER_STMINTSET_COMPAREB_Msk (0x2UL) /*!< CTIMER STMINTSET: COMPAREB (Bitfield-Mask: 0x01) */ +#define CTIMER_STMINTSET_COMPAREA_Pos (0UL) /*!< CTIMER STMINTSET: COMPAREA (Bit 0) */ +#define CTIMER_STMINTSET_COMPAREA_Msk (0x1UL) /*!< CTIMER STMINTSET: COMPAREA (Bitfield-Mask: 0x01) */ + + +/* =========================================================================================================================== */ +/* ================ GPIO ================ */ +/* =========================================================================================================================== */ + +/* ======================================================== PADREGA ======================================================== */ +#define GPIO_PADREGA_PAD3PWRUP_Pos (30UL) /*!< GPIO PADREGA: PAD3PWRUP (Bit 30) */ +#define GPIO_PADREGA_PAD3PWRUP_Msk (0x40000000UL) /*!< GPIO PADREGA: PAD3PWRUP (Bitfield-Mask: 0x01) */ +#define GPIO_PADREGA_PAD3FNCSEL_Pos (27UL) /*!< GPIO PADREGA: PAD3FNCSEL (Bit 27) */ +#define GPIO_PADREGA_PAD3FNCSEL_Msk (0x38000000UL) /*!< GPIO PADREGA: PAD3FNCSEL (Bitfield-Mask: 0x07) */ +#define GPIO_PADREGA_PAD3STRNG_Pos (26UL) /*!< GPIO PADREGA: PAD3STRNG (Bit 26) */ +#define GPIO_PADREGA_PAD3STRNG_Msk (0x4000000UL) /*!< GPIO PADREGA: PAD3STRNG (Bitfield-Mask: 0x01) */ +#define GPIO_PADREGA_PAD3INPEN_Pos (25UL) /*!< GPIO PADREGA: PAD3INPEN (Bit 25) */ +#define GPIO_PADREGA_PAD3INPEN_Msk (0x2000000UL) /*!< GPIO PADREGA: PAD3INPEN (Bitfield-Mask: 0x01) */ +#define GPIO_PADREGA_PAD3PULL_Pos (24UL) /*!< GPIO PADREGA: PAD3PULL (Bit 24) */ +#define GPIO_PADREGA_PAD3PULL_Msk (0x1000000UL) /*!< GPIO PADREGA: PAD3PULL (Bitfield-Mask: 0x01) */ +#define GPIO_PADREGA_PAD2FNCSEL_Pos (19UL) /*!< GPIO PADREGA: PAD2FNCSEL (Bit 19) */ +#define GPIO_PADREGA_PAD2FNCSEL_Msk (0x380000UL) /*!< GPIO PADREGA: PAD2FNCSEL (Bitfield-Mask: 0x07) */ +#define GPIO_PADREGA_PAD2STRNG_Pos (18UL) /*!< GPIO PADREGA: PAD2STRNG (Bit 18) */ +#define GPIO_PADREGA_PAD2STRNG_Msk (0x40000UL) /*!< GPIO PADREGA: PAD2STRNG (Bitfield-Mask: 0x01) */ +#define GPIO_PADREGA_PAD2INPEN_Pos (17UL) /*!< GPIO PADREGA: PAD2INPEN (Bit 17) */ +#define GPIO_PADREGA_PAD2INPEN_Msk (0x20000UL) /*!< GPIO PADREGA: PAD2INPEN (Bitfield-Mask: 0x01) */ +#define GPIO_PADREGA_PAD2PULL_Pos (16UL) /*!< GPIO PADREGA: PAD2PULL (Bit 16) */ +#define GPIO_PADREGA_PAD2PULL_Msk (0x10000UL) /*!< GPIO PADREGA: PAD2PULL (Bitfield-Mask: 0x01) */ +#define GPIO_PADREGA_PAD1RSEL_Pos (14UL) /*!< GPIO PADREGA: PAD1RSEL (Bit 14) */ +#define GPIO_PADREGA_PAD1RSEL_Msk (0xc000UL) /*!< GPIO PADREGA: PAD1RSEL (Bitfield-Mask: 0x03) */ +#define GPIO_PADREGA_PAD1FNCSEL_Pos (11UL) /*!< GPIO PADREGA: PAD1FNCSEL (Bit 11) */ +#define GPIO_PADREGA_PAD1FNCSEL_Msk (0x3800UL) /*!< GPIO PADREGA: PAD1FNCSEL (Bitfield-Mask: 0x07) */ +#define GPIO_PADREGA_PAD1STRNG_Pos (10UL) /*!< GPIO PADREGA: PAD1STRNG (Bit 10) */ +#define GPIO_PADREGA_PAD1STRNG_Msk (0x400UL) /*!< GPIO PADREGA: PAD1STRNG (Bitfield-Mask: 0x01) */ +#define GPIO_PADREGA_PAD1INPEN_Pos (9UL) /*!< GPIO PADREGA: PAD1INPEN (Bit 9) */ +#define GPIO_PADREGA_PAD1INPEN_Msk (0x200UL) /*!< GPIO PADREGA: PAD1INPEN (Bitfield-Mask: 0x01) */ +#define GPIO_PADREGA_PAD1PULL_Pos (8UL) /*!< GPIO PADREGA: PAD1PULL (Bit 8) */ +#define GPIO_PADREGA_PAD1PULL_Msk (0x100UL) /*!< GPIO PADREGA: PAD1PULL (Bitfield-Mask: 0x01) */ +#define GPIO_PADREGA_PAD0RSEL_Pos (6UL) /*!< GPIO PADREGA: PAD0RSEL (Bit 6) */ +#define GPIO_PADREGA_PAD0RSEL_Msk (0xc0UL) /*!< GPIO PADREGA: PAD0RSEL (Bitfield-Mask: 0x03) */ +#define GPIO_PADREGA_PAD0FNCSEL_Pos (3UL) /*!< GPIO PADREGA: PAD0FNCSEL (Bit 3) */ +#define GPIO_PADREGA_PAD0FNCSEL_Msk (0x38UL) /*!< GPIO PADREGA: PAD0FNCSEL (Bitfield-Mask: 0x07) */ +#define GPIO_PADREGA_PAD0STRNG_Pos (2UL) /*!< GPIO PADREGA: PAD0STRNG (Bit 2) */ +#define GPIO_PADREGA_PAD0STRNG_Msk (0x4UL) /*!< GPIO PADREGA: PAD0STRNG (Bitfield-Mask: 0x01) */ +#define GPIO_PADREGA_PAD0INPEN_Pos (1UL) /*!< GPIO PADREGA: PAD0INPEN (Bit 1) */ +#define GPIO_PADREGA_PAD0INPEN_Msk (0x2UL) /*!< GPIO PADREGA: PAD0INPEN (Bitfield-Mask: 0x01) */ +#define GPIO_PADREGA_PAD0PULL_Pos (0UL) /*!< GPIO PADREGA: PAD0PULL (Bit 0) */ +#define GPIO_PADREGA_PAD0PULL_Msk (0x1UL) /*!< GPIO PADREGA: PAD0PULL (Bitfield-Mask: 0x01) */ +/* ======================================================== PADREGB ======================================================== */ +#define GPIO_PADREGB_PAD7FNCSEL_Pos (27UL) /*!< GPIO PADREGB: PAD7FNCSEL (Bit 27) */ +#define GPIO_PADREGB_PAD7FNCSEL_Msk (0x38000000UL) /*!< GPIO PADREGB: PAD7FNCSEL (Bitfield-Mask: 0x07) */ +#define GPIO_PADREGB_PAD7STRNG_Pos (26UL) /*!< GPIO PADREGB: PAD7STRNG (Bit 26) */ +#define GPIO_PADREGB_PAD7STRNG_Msk (0x4000000UL) /*!< GPIO PADREGB: PAD7STRNG (Bitfield-Mask: 0x01) */ +#define GPIO_PADREGB_PAD7INPEN_Pos (25UL) /*!< GPIO PADREGB: PAD7INPEN (Bit 25) */ +#define GPIO_PADREGB_PAD7INPEN_Msk (0x2000000UL) /*!< GPIO PADREGB: PAD7INPEN (Bitfield-Mask: 0x01) */ +#define GPIO_PADREGB_PAD7PULL_Pos (24UL) /*!< GPIO PADREGB: PAD7PULL (Bit 24) */ +#define GPIO_PADREGB_PAD7PULL_Msk (0x1000000UL) /*!< GPIO PADREGB: PAD7PULL (Bitfield-Mask: 0x01) */ +#define GPIO_PADREGB_PAD6RSEL_Pos (22UL) /*!< GPIO PADREGB: PAD6RSEL (Bit 22) */ +#define GPIO_PADREGB_PAD6RSEL_Msk (0xc00000UL) /*!< GPIO PADREGB: PAD6RSEL (Bitfield-Mask: 0x03) */ +#define GPIO_PADREGB_PAD6FNCSEL_Pos (19UL) /*!< GPIO PADREGB: PAD6FNCSEL (Bit 19) */ +#define GPIO_PADREGB_PAD6FNCSEL_Msk (0x380000UL) /*!< GPIO PADREGB: PAD6FNCSEL (Bitfield-Mask: 0x07) */ +#define GPIO_PADREGB_PAD6STRNG_Pos (18UL) /*!< GPIO PADREGB: PAD6STRNG (Bit 18) */ +#define GPIO_PADREGB_PAD6STRNG_Msk (0x40000UL) /*!< GPIO PADREGB: PAD6STRNG (Bitfield-Mask: 0x01) */ +#define GPIO_PADREGB_PAD6INPEN_Pos (17UL) /*!< GPIO PADREGB: PAD6INPEN (Bit 17) */ +#define GPIO_PADREGB_PAD6INPEN_Msk (0x20000UL) /*!< GPIO PADREGB: PAD6INPEN (Bitfield-Mask: 0x01) */ +#define GPIO_PADREGB_PAD6PULL_Pos (16UL) /*!< GPIO PADREGB: PAD6PULL (Bit 16) */ +#define GPIO_PADREGB_PAD6PULL_Msk (0x10000UL) /*!< GPIO PADREGB: PAD6PULL (Bitfield-Mask: 0x01) */ +#define GPIO_PADREGB_PAD5RSEL_Pos (14UL) /*!< GPIO PADREGB: PAD5RSEL (Bit 14) */ +#define GPIO_PADREGB_PAD5RSEL_Msk (0xc000UL) /*!< GPIO PADREGB: PAD5RSEL (Bitfield-Mask: 0x03) */ +#define GPIO_PADREGB_PAD5FNCSEL_Pos (11UL) /*!< GPIO PADREGB: PAD5FNCSEL (Bit 11) */ +#define GPIO_PADREGB_PAD5FNCSEL_Msk (0x3800UL) /*!< GPIO PADREGB: PAD5FNCSEL (Bitfield-Mask: 0x07) */ +#define GPIO_PADREGB_PAD5STRNG_Pos (10UL) /*!< GPIO PADREGB: PAD5STRNG (Bit 10) */ +#define GPIO_PADREGB_PAD5STRNG_Msk (0x400UL) /*!< GPIO PADREGB: PAD5STRNG (Bitfield-Mask: 0x01) */ +#define GPIO_PADREGB_PAD5INPEN_Pos (9UL) /*!< GPIO PADREGB: PAD5INPEN (Bit 9) */ +#define GPIO_PADREGB_PAD5INPEN_Msk (0x200UL) /*!< GPIO PADREGB: PAD5INPEN (Bitfield-Mask: 0x01) */ +#define GPIO_PADREGB_PAD5PULL_Pos (8UL) /*!< GPIO PADREGB: PAD5PULL (Bit 8) */ +#define GPIO_PADREGB_PAD5PULL_Msk (0x100UL) /*!< GPIO PADREGB: PAD5PULL (Bitfield-Mask: 0x01) */ +#define GPIO_PADREGB_PAD4FNCSEL_Pos (3UL) /*!< GPIO PADREGB: PAD4FNCSEL (Bit 3) */ +#define GPIO_PADREGB_PAD4FNCSEL_Msk (0x38UL) /*!< GPIO PADREGB: PAD4FNCSEL (Bitfield-Mask: 0x07) */ +#define GPIO_PADREGB_PAD4STRNG_Pos (2UL) /*!< GPIO PADREGB: PAD4STRNG (Bit 2) */ +#define GPIO_PADREGB_PAD4STRNG_Msk (0x4UL) /*!< GPIO PADREGB: PAD4STRNG (Bitfield-Mask: 0x01) */ +#define GPIO_PADREGB_PAD4INPEN_Pos (1UL) /*!< GPIO PADREGB: PAD4INPEN (Bit 1) */ +#define GPIO_PADREGB_PAD4INPEN_Msk (0x2UL) /*!< GPIO PADREGB: PAD4INPEN (Bitfield-Mask: 0x01) */ +#define GPIO_PADREGB_PAD4PULL_Pos (0UL) /*!< GPIO PADREGB: PAD4PULL (Bit 0) */ +#define GPIO_PADREGB_PAD4PULL_Msk (0x1UL) /*!< GPIO PADREGB: PAD4PULL (Bitfield-Mask: 0x01) */ +/* ======================================================== PADREGC ======================================================== */ +#define GPIO_PADREGC_PAD11FNCSEL_Pos (27UL) /*!< GPIO PADREGC: PAD11FNCSEL (Bit 27) */ +#define GPIO_PADREGC_PAD11FNCSEL_Msk (0x38000000UL) /*!< GPIO PADREGC: PAD11FNCSEL (Bitfield-Mask: 0x07) */ +#define GPIO_PADREGC_PAD11STRNG_Pos (26UL) /*!< GPIO PADREGC: PAD11STRNG (Bit 26) */ +#define GPIO_PADREGC_PAD11STRNG_Msk (0x4000000UL) /*!< GPIO PADREGC: PAD11STRNG (Bitfield-Mask: 0x01) */ +#define GPIO_PADREGC_PAD11INPEN_Pos (25UL) /*!< GPIO PADREGC: PAD11INPEN (Bit 25) */ +#define GPIO_PADREGC_PAD11INPEN_Msk (0x2000000UL) /*!< GPIO PADREGC: PAD11INPEN (Bitfield-Mask: 0x01) */ +#define GPIO_PADREGC_PAD11PULL_Pos (24UL) /*!< GPIO PADREGC: PAD11PULL (Bit 24) */ +#define GPIO_PADREGC_PAD11PULL_Msk (0x1000000UL) /*!< GPIO PADREGC: PAD11PULL (Bitfield-Mask: 0x01) */ +#define GPIO_PADREGC_PAD10FNCSEL_Pos (19UL) /*!< GPIO PADREGC: PAD10FNCSEL (Bit 19) */ +#define GPIO_PADREGC_PAD10FNCSEL_Msk (0x380000UL) /*!< GPIO PADREGC: PAD10FNCSEL (Bitfield-Mask: 0x07) */ +#define GPIO_PADREGC_PAD10STRNG_Pos (18UL) /*!< GPIO PADREGC: PAD10STRNG (Bit 18) */ +#define GPIO_PADREGC_PAD10STRNG_Msk (0x40000UL) /*!< GPIO PADREGC: PAD10STRNG (Bitfield-Mask: 0x01) */ +#define GPIO_PADREGC_PAD10INPEN_Pos (17UL) /*!< GPIO PADREGC: PAD10INPEN (Bit 17) */ +#define GPIO_PADREGC_PAD10INPEN_Msk (0x20000UL) /*!< GPIO PADREGC: PAD10INPEN (Bitfield-Mask: 0x01) */ +#define GPIO_PADREGC_PAD10PULL_Pos (16UL) /*!< GPIO PADREGC: PAD10PULL (Bit 16) */ +#define GPIO_PADREGC_PAD10PULL_Msk (0x10000UL) /*!< GPIO PADREGC: PAD10PULL (Bitfield-Mask: 0x01) */ +#define GPIO_PADREGC_PAD9RSEL_Pos (14UL) /*!< GPIO PADREGC: PAD9RSEL (Bit 14) */ +#define GPIO_PADREGC_PAD9RSEL_Msk (0xc000UL) /*!< GPIO PADREGC: PAD9RSEL (Bitfield-Mask: 0x03) */ +#define GPIO_PADREGC_PAD9FNCSEL_Pos (11UL) /*!< GPIO PADREGC: PAD9FNCSEL (Bit 11) */ +#define GPIO_PADREGC_PAD9FNCSEL_Msk (0x3800UL) /*!< GPIO PADREGC: PAD9FNCSEL (Bitfield-Mask: 0x07) */ +#define GPIO_PADREGC_PAD9STRNG_Pos (10UL) /*!< GPIO PADREGC: PAD9STRNG (Bit 10) */ +#define GPIO_PADREGC_PAD9STRNG_Msk (0x400UL) /*!< GPIO PADREGC: PAD9STRNG (Bitfield-Mask: 0x01) */ +#define GPIO_PADREGC_PAD9INPEN_Pos (9UL) /*!< GPIO PADREGC: PAD9INPEN (Bit 9) */ +#define GPIO_PADREGC_PAD9INPEN_Msk (0x200UL) /*!< GPIO PADREGC: PAD9INPEN (Bitfield-Mask: 0x01) */ +#define GPIO_PADREGC_PAD9PULL_Pos (8UL) /*!< GPIO PADREGC: PAD9PULL (Bit 8) */ +#define GPIO_PADREGC_PAD9PULL_Msk (0x100UL) /*!< GPIO PADREGC: PAD9PULL (Bitfield-Mask: 0x01) */ +#define GPIO_PADREGC_PAD8RSEL_Pos (6UL) /*!< GPIO PADREGC: PAD8RSEL (Bit 6) */ +#define GPIO_PADREGC_PAD8RSEL_Msk (0xc0UL) /*!< GPIO PADREGC: PAD8RSEL (Bitfield-Mask: 0x03) */ +#define GPIO_PADREGC_PAD8FNCSEL_Pos (3UL) /*!< GPIO PADREGC: PAD8FNCSEL (Bit 3) */ +#define GPIO_PADREGC_PAD8FNCSEL_Msk (0x38UL) /*!< GPIO PADREGC: PAD8FNCSEL (Bitfield-Mask: 0x07) */ +#define GPIO_PADREGC_PAD8STRNG_Pos (2UL) /*!< GPIO PADREGC: PAD8STRNG (Bit 2) */ +#define GPIO_PADREGC_PAD8STRNG_Msk (0x4UL) /*!< GPIO PADREGC: PAD8STRNG (Bitfield-Mask: 0x01) */ +#define GPIO_PADREGC_PAD8INPEN_Pos (1UL) /*!< GPIO PADREGC: PAD8INPEN (Bit 1) */ +#define GPIO_PADREGC_PAD8INPEN_Msk (0x2UL) /*!< GPIO PADREGC: PAD8INPEN (Bitfield-Mask: 0x01) */ +#define GPIO_PADREGC_PAD8PULL_Pos (0UL) /*!< GPIO PADREGC: PAD8PULL (Bit 0) */ +#define GPIO_PADREGC_PAD8PULL_Msk (0x1UL) /*!< GPIO PADREGC: PAD8PULL (Bitfield-Mask: 0x01) */ +/* ======================================================== PADREGD ======================================================== */ +#define GPIO_PADREGD_PAD15FNCSEL_Pos (27UL) /*!< GPIO PADREGD: PAD15FNCSEL (Bit 27) */ +#define GPIO_PADREGD_PAD15FNCSEL_Msk (0x38000000UL) /*!< GPIO PADREGD: PAD15FNCSEL (Bitfield-Mask: 0x07) */ +#define GPIO_PADREGD_PAD15STRNG_Pos (26UL) /*!< GPIO PADREGD: PAD15STRNG (Bit 26) */ +#define GPIO_PADREGD_PAD15STRNG_Msk (0x4000000UL) /*!< GPIO PADREGD: PAD15STRNG (Bitfield-Mask: 0x01) */ +#define GPIO_PADREGD_PAD15INPEN_Pos (25UL) /*!< GPIO PADREGD: PAD15INPEN (Bit 25) */ +#define GPIO_PADREGD_PAD15INPEN_Msk (0x2000000UL) /*!< GPIO PADREGD: PAD15INPEN (Bitfield-Mask: 0x01) */ +#define GPIO_PADREGD_PAD15PULL_Pos (24UL) /*!< GPIO PADREGD: PAD15PULL (Bit 24) */ +#define GPIO_PADREGD_PAD15PULL_Msk (0x1000000UL) /*!< GPIO PADREGD: PAD15PULL (Bitfield-Mask: 0x01) */ +#define GPIO_PADREGD_PAD14FNCSEL_Pos (19UL) /*!< GPIO PADREGD: PAD14FNCSEL (Bit 19) */ +#define GPIO_PADREGD_PAD14FNCSEL_Msk (0x380000UL) /*!< GPIO PADREGD: PAD14FNCSEL (Bitfield-Mask: 0x07) */ +#define GPIO_PADREGD_PAD14STRNG_Pos (18UL) /*!< GPIO PADREGD: PAD14STRNG (Bit 18) */ +#define GPIO_PADREGD_PAD14STRNG_Msk (0x40000UL) /*!< GPIO PADREGD: PAD14STRNG (Bitfield-Mask: 0x01) */ +#define GPIO_PADREGD_PAD14INPEN_Pos (17UL) /*!< GPIO PADREGD: PAD14INPEN (Bit 17) */ +#define GPIO_PADREGD_PAD14INPEN_Msk (0x20000UL) /*!< GPIO PADREGD: PAD14INPEN (Bitfield-Mask: 0x01) */ +#define GPIO_PADREGD_PAD14PULL_Pos (16UL) /*!< GPIO PADREGD: PAD14PULL (Bit 16) */ +#define GPIO_PADREGD_PAD14PULL_Msk (0x10000UL) /*!< GPIO PADREGD: PAD14PULL (Bitfield-Mask: 0x01) */ +#define GPIO_PADREGD_PAD13FNCSEL_Pos (11UL) /*!< GPIO PADREGD: PAD13FNCSEL (Bit 11) */ +#define GPIO_PADREGD_PAD13FNCSEL_Msk (0x3800UL) /*!< GPIO PADREGD: PAD13FNCSEL (Bitfield-Mask: 0x07) */ +#define GPIO_PADREGD_PAD13STRNG_Pos (10UL) /*!< GPIO PADREGD: PAD13STRNG (Bit 10) */ +#define GPIO_PADREGD_PAD13STRNG_Msk (0x400UL) /*!< GPIO PADREGD: PAD13STRNG (Bitfield-Mask: 0x01) */ +#define GPIO_PADREGD_PAD13INPEN_Pos (9UL) /*!< GPIO PADREGD: PAD13INPEN (Bit 9) */ +#define GPIO_PADREGD_PAD13INPEN_Msk (0x200UL) /*!< GPIO PADREGD: PAD13INPEN (Bitfield-Mask: 0x01) */ +#define GPIO_PADREGD_PAD13PULL_Pos (8UL) /*!< GPIO PADREGD: PAD13PULL (Bit 8) */ +#define GPIO_PADREGD_PAD13PULL_Msk (0x100UL) /*!< GPIO PADREGD: PAD13PULL (Bitfield-Mask: 0x01) */ +#define GPIO_PADREGD_PAD12FNCSEL_Pos (3UL) /*!< GPIO PADREGD: PAD12FNCSEL (Bit 3) */ +#define GPIO_PADREGD_PAD12FNCSEL_Msk (0x38UL) /*!< GPIO PADREGD: PAD12FNCSEL (Bitfield-Mask: 0x07) */ +#define GPIO_PADREGD_PAD12STRNG_Pos (2UL) /*!< GPIO PADREGD: PAD12STRNG (Bit 2) */ +#define GPIO_PADREGD_PAD12STRNG_Msk (0x4UL) /*!< GPIO PADREGD: PAD12STRNG (Bitfield-Mask: 0x01) */ +#define GPIO_PADREGD_PAD12INPEN_Pos (1UL) /*!< GPIO PADREGD: PAD12INPEN (Bit 1) */ +#define GPIO_PADREGD_PAD12INPEN_Msk (0x2UL) /*!< GPIO PADREGD: PAD12INPEN (Bitfield-Mask: 0x01) */ +#define GPIO_PADREGD_PAD12PULL_Pos (0UL) /*!< GPIO PADREGD: PAD12PULL (Bit 0) */ +#define GPIO_PADREGD_PAD12PULL_Msk (0x1UL) /*!< GPIO PADREGD: PAD12PULL (Bitfield-Mask: 0x01) */ +/* ======================================================== PADREGE ======================================================== */ +#define GPIO_PADREGE_PAD19FNCSEL_Pos (27UL) /*!< GPIO PADREGE: PAD19FNCSEL (Bit 27) */ +#define GPIO_PADREGE_PAD19FNCSEL_Msk (0x38000000UL) /*!< GPIO PADREGE: PAD19FNCSEL (Bitfield-Mask: 0x07) */ +#define GPIO_PADREGE_PAD19STRNG_Pos (26UL) /*!< GPIO PADREGE: PAD19STRNG (Bit 26) */ +#define GPIO_PADREGE_PAD19STRNG_Msk (0x4000000UL) /*!< GPIO PADREGE: PAD19STRNG (Bitfield-Mask: 0x01) */ +#define GPIO_PADREGE_PAD19INPEN_Pos (25UL) /*!< GPIO PADREGE: PAD19INPEN (Bit 25) */ +#define GPIO_PADREGE_PAD19INPEN_Msk (0x2000000UL) /*!< GPIO PADREGE: PAD19INPEN (Bitfield-Mask: 0x01) */ +#define GPIO_PADREGE_PAD19PULL_Pos (24UL) /*!< GPIO PADREGE: PAD19PULL (Bit 24) */ +#define GPIO_PADREGE_PAD19PULL_Msk (0x1000000UL) /*!< GPIO PADREGE: PAD19PULL (Bitfield-Mask: 0x01) */ +#define GPIO_PADREGE_PAD18FNCSEL_Pos (19UL) /*!< GPIO PADREGE: PAD18FNCSEL (Bit 19) */ +#define GPIO_PADREGE_PAD18FNCSEL_Msk (0x380000UL) /*!< GPIO PADREGE: PAD18FNCSEL (Bitfield-Mask: 0x07) */ +#define GPIO_PADREGE_PAD18STRNG_Pos (18UL) /*!< GPIO PADREGE: PAD18STRNG (Bit 18) */ +#define GPIO_PADREGE_PAD18STRNG_Msk (0x40000UL) /*!< GPIO PADREGE: PAD18STRNG (Bitfield-Mask: 0x01) */ +#define GPIO_PADREGE_PAD18INPEN_Pos (17UL) /*!< GPIO PADREGE: PAD18INPEN (Bit 17) */ +#define GPIO_PADREGE_PAD18INPEN_Msk (0x20000UL) /*!< GPIO PADREGE: PAD18INPEN (Bitfield-Mask: 0x01) */ +#define GPIO_PADREGE_PAD18PULL_Pos (16UL) /*!< GPIO PADREGE: PAD18PULL (Bit 16) */ +#define GPIO_PADREGE_PAD18PULL_Msk (0x10000UL) /*!< GPIO PADREGE: PAD18PULL (Bitfield-Mask: 0x01) */ +#define GPIO_PADREGE_PAD17FNCSEL_Pos (11UL) /*!< GPIO PADREGE: PAD17FNCSEL (Bit 11) */ +#define GPIO_PADREGE_PAD17FNCSEL_Msk (0x3800UL) /*!< GPIO PADREGE: PAD17FNCSEL (Bitfield-Mask: 0x07) */ +#define GPIO_PADREGE_PAD17STRNG_Pos (10UL) /*!< GPIO PADREGE: PAD17STRNG (Bit 10) */ +#define GPIO_PADREGE_PAD17STRNG_Msk (0x400UL) /*!< GPIO PADREGE: PAD17STRNG (Bitfield-Mask: 0x01) */ +#define GPIO_PADREGE_PAD17INPEN_Pos (9UL) /*!< GPIO PADREGE: PAD17INPEN (Bit 9) */ +#define GPIO_PADREGE_PAD17INPEN_Msk (0x200UL) /*!< GPIO PADREGE: PAD17INPEN (Bitfield-Mask: 0x01) */ +#define GPIO_PADREGE_PAD17PULL_Pos (8UL) /*!< GPIO PADREGE: PAD17PULL (Bit 8) */ +#define GPIO_PADREGE_PAD17PULL_Msk (0x100UL) /*!< GPIO PADREGE: PAD17PULL (Bitfield-Mask: 0x01) */ +#define GPIO_PADREGE_PAD16FNCSEL_Pos (3UL) /*!< GPIO PADREGE: PAD16FNCSEL (Bit 3) */ +#define GPIO_PADREGE_PAD16FNCSEL_Msk (0x38UL) /*!< GPIO PADREGE: PAD16FNCSEL (Bitfield-Mask: 0x07) */ +#define GPIO_PADREGE_PAD16STRNG_Pos (2UL) /*!< GPIO PADREGE: PAD16STRNG (Bit 2) */ +#define GPIO_PADREGE_PAD16STRNG_Msk (0x4UL) /*!< GPIO PADREGE: PAD16STRNG (Bitfield-Mask: 0x01) */ +#define GPIO_PADREGE_PAD16INPEN_Pos (1UL) /*!< GPIO PADREGE: PAD16INPEN (Bit 1) */ +#define GPIO_PADREGE_PAD16INPEN_Msk (0x2UL) /*!< GPIO PADREGE: PAD16INPEN (Bitfield-Mask: 0x01) */ +#define GPIO_PADREGE_PAD16PULL_Pos (0UL) /*!< GPIO PADREGE: PAD16PULL (Bit 0) */ +#define GPIO_PADREGE_PAD16PULL_Msk (0x1UL) /*!< GPIO PADREGE: PAD16PULL (Bitfield-Mask: 0x01) */ +/* ======================================================== PADREGF ======================================================== */ +#define GPIO_PADREGF_PAD23FNCSEL_Pos (27UL) /*!< GPIO PADREGF: PAD23FNCSEL (Bit 27) */ +#define GPIO_PADREGF_PAD23FNCSEL_Msk (0x38000000UL) /*!< GPIO PADREGF: PAD23FNCSEL (Bitfield-Mask: 0x07) */ +#define GPIO_PADREGF_PAD23STRNG_Pos (26UL) /*!< GPIO PADREGF: PAD23STRNG (Bit 26) */ +#define GPIO_PADREGF_PAD23STRNG_Msk (0x4000000UL) /*!< GPIO PADREGF: PAD23STRNG (Bitfield-Mask: 0x01) */ +#define GPIO_PADREGF_PAD23INPEN_Pos (25UL) /*!< GPIO PADREGF: PAD23INPEN (Bit 25) */ +#define GPIO_PADREGF_PAD23INPEN_Msk (0x2000000UL) /*!< GPIO PADREGF: PAD23INPEN (Bitfield-Mask: 0x01) */ +#define GPIO_PADREGF_PAD23PULL_Pos (24UL) /*!< GPIO PADREGF: PAD23PULL (Bit 24) */ +#define GPIO_PADREGF_PAD23PULL_Msk (0x1000000UL) /*!< GPIO PADREGF: PAD23PULL (Bitfield-Mask: 0x01) */ +#define GPIO_PADREGF_PAD22FNCSEL_Pos (19UL) /*!< GPIO PADREGF: PAD22FNCSEL (Bit 19) */ +#define GPIO_PADREGF_PAD22FNCSEL_Msk (0x380000UL) /*!< GPIO PADREGF: PAD22FNCSEL (Bitfield-Mask: 0x07) */ +#define GPIO_PADREGF_PAD22STRNG_Pos (18UL) /*!< GPIO PADREGF: PAD22STRNG (Bit 18) */ +#define GPIO_PADREGF_PAD22STRNG_Msk (0x40000UL) /*!< GPIO PADREGF: PAD22STRNG (Bitfield-Mask: 0x01) */ +#define GPIO_PADREGF_PAD22INPEN_Pos (17UL) /*!< GPIO PADREGF: PAD22INPEN (Bit 17) */ +#define GPIO_PADREGF_PAD22INPEN_Msk (0x20000UL) /*!< GPIO PADREGF: PAD22INPEN (Bitfield-Mask: 0x01) */ +#define GPIO_PADREGF_PAD22PULL_Pos (16UL) /*!< GPIO PADREGF: PAD22PULL (Bit 16) */ +#define GPIO_PADREGF_PAD22PULL_Msk (0x10000UL) /*!< GPIO PADREGF: PAD22PULL (Bitfield-Mask: 0x01) */ +#define GPIO_PADREGF_PAD21FNCSEL_Pos (11UL) /*!< GPIO PADREGF: PAD21FNCSEL (Bit 11) */ +#define GPIO_PADREGF_PAD21FNCSEL_Msk (0x3800UL) /*!< GPIO PADREGF: PAD21FNCSEL (Bitfield-Mask: 0x07) */ +#define GPIO_PADREGF_PAD21STRNG_Pos (10UL) /*!< GPIO PADREGF: PAD21STRNG (Bit 10) */ +#define GPIO_PADREGF_PAD21STRNG_Msk (0x400UL) /*!< GPIO PADREGF: PAD21STRNG (Bitfield-Mask: 0x01) */ +#define GPIO_PADREGF_PAD21INPEN_Pos (9UL) /*!< GPIO PADREGF: PAD21INPEN (Bit 9) */ +#define GPIO_PADREGF_PAD21INPEN_Msk (0x200UL) /*!< GPIO PADREGF: PAD21INPEN (Bitfield-Mask: 0x01) */ +#define GPIO_PADREGF_PAD21PULL_Pos (8UL) /*!< GPIO PADREGF: PAD21PULL (Bit 8) */ +#define GPIO_PADREGF_PAD21PULL_Msk (0x100UL) /*!< GPIO PADREGF: PAD21PULL (Bitfield-Mask: 0x01) */ +#define GPIO_PADREGF_PAD20FNCSEL_Pos (3UL) /*!< GPIO PADREGF: PAD20FNCSEL (Bit 3) */ +#define GPIO_PADREGF_PAD20FNCSEL_Msk (0x38UL) /*!< GPIO PADREGF: PAD20FNCSEL (Bitfield-Mask: 0x07) */ +#define GPIO_PADREGF_PAD20STRNG_Pos (2UL) /*!< GPIO PADREGF: PAD20STRNG (Bit 2) */ +#define GPIO_PADREGF_PAD20STRNG_Msk (0x4UL) /*!< GPIO PADREGF: PAD20STRNG (Bitfield-Mask: 0x01) */ +#define GPIO_PADREGF_PAD20INPEN_Pos (1UL) /*!< GPIO PADREGF: PAD20INPEN (Bit 1) */ +#define GPIO_PADREGF_PAD20INPEN_Msk (0x2UL) /*!< GPIO PADREGF: PAD20INPEN (Bitfield-Mask: 0x01) */ +#define GPIO_PADREGF_PAD20PULL_Pos (0UL) /*!< GPIO PADREGF: PAD20PULL (Bit 0) */ +#define GPIO_PADREGF_PAD20PULL_Msk (0x1UL) /*!< GPIO PADREGF: PAD20PULL (Bitfield-Mask: 0x01) */ +/* ======================================================== PADREGG ======================================================== */ +#define GPIO_PADREGG_PAD27RSEL_Pos (30UL) /*!< GPIO PADREGG: PAD27RSEL (Bit 30) */ +#define GPIO_PADREGG_PAD27RSEL_Msk (0xc0000000UL) /*!< GPIO PADREGG: PAD27RSEL (Bitfield-Mask: 0x03) */ +#define GPIO_PADREGG_PAD27FNCSEL_Pos (27UL) /*!< GPIO PADREGG: PAD27FNCSEL (Bit 27) */ +#define GPIO_PADREGG_PAD27FNCSEL_Msk (0x38000000UL) /*!< GPIO PADREGG: PAD27FNCSEL (Bitfield-Mask: 0x07) */ +#define GPIO_PADREGG_PAD27STRNG_Pos (26UL) /*!< GPIO PADREGG: PAD27STRNG (Bit 26) */ +#define GPIO_PADREGG_PAD27STRNG_Msk (0x4000000UL) /*!< GPIO PADREGG: PAD27STRNG (Bitfield-Mask: 0x01) */ +#define GPIO_PADREGG_PAD27INPEN_Pos (25UL) /*!< GPIO PADREGG: PAD27INPEN (Bit 25) */ +#define GPIO_PADREGG_PAD27INPEN_Msk (0x2000000UL) /*!< GPIO PADREGG: PAD27INPEN (Bitfield-Mask: 0x01) */ +#define GPIO_PADREGG_PAD27PULL_Pos (24UL) /*!< GPIO PADREGG: PAD27PULL (Bit 24) */ +#define GPIO_PADREGG_PAD27PULL_Msk (0x1000000UL) /*!< GPIO PADREGG: PAD27PULL (Bitfield-Mask: 0x01) */ +#define GPIO_PADREGG_PAD26FNCSEL_Pos (19UL) /*!< GPIO PADREGG: PAD26FNCSEL (Bit 19) */ +#define GPIO_PADREGG_PAD26FNCSEL_Msk (0x380000UL) /*!< GPIO PADREGG: PAD26FNCSEL (Bitfield-Mask: 0x07) */ +#define GPIO_PADREGG_PAD26STRNG_Pos (18UL) /*!< GPIO PADREGG: PAD26STRNG (Bit 18) */ +#define GPIO_PADREGG_PAD26STRNG_Msk (0x40000UL) /*!< GPIO PADREGG: PAD26STRNG (Bitfield-Mask: 0x01) */ +#define GPIO_PADREGG_PAD26INPEN_Pos (17UL) /*!< GPIO PADREGG: PAD26INPEN (Bit 17) */ +#define GPIO_PADREGG_PAD26INPEN_Msk (0x20000UL) /*!< GPIO PADREGG: PAD26INPEN (Bitfield-Mask: 0x01) */ +#define GPIO_PADREGG_PAD26PULL_Pos (16UL) /*!< GPIO PADREGG: PAD26PULL (Bit 16) */ +#define GPIO_PADREGG_PAD26PULL_Msk (0x10000UL) /*!< GPIO PADREGG: PAD26PULL (Bitfield-Mask: 0x01) */ +#define GPIO_PADREGG_PAD25RSEL_Pos (14UL) /*!< GPIO PADREGG: PAD25RSEL (Bit 14) */ +#define GPIO_PADREGG_PAD25RSEL_Msk (0xc000UL) /*!< GPIO PADREGG: PAD25RSEL (Bitfield-Mask: 0x03) */ +#define GPIO_PADREGG_PAD25FNCSEL_Pos (11UL) /*!< GPIO PADREGG: PAD25FNCSEL (Bit 11) */ +#define GPIO_PADREGG_PAD25FNCSEL_Msk (0x3800UL) /*!< GPIO PADREGG: PAD25FNCSEL (Bitfield-Mask: 0x07) */ +#define GPIO_PADREGG_PAD25STRNG_Pos (10UL) /*!< GPIO PADREGG: PAD25STRNG (Bit 10) */ +#define GPIO_PADREGG_PAD25STRNG_Msk (0x400UL) /*!< GPIO PADREGG: PAD25STRNG (Bitfield-Mask: 0x01) */ +#define GPIO_PADREGG_PAD25INPEN_Pos (9UL) /*!< GPIO PADREGG: PAD25INPEN (Bit 9) */ +#define GPIO_PADREGG_PAD25INPEN_Msk (0x200UL) /*!< GPIO PADREGG: PAD25INPEN (Bitfield-Mask: 0x01) */ +#define GPIO_PADREGG_PAD25PULL_Pos (8UL) /*!< GPIO PADREGG: PAD25PULL (Bit 8) */ +#define GPIO_PADREGG_PAD25PULL_Msk (0x100UL) /*!< GPIO PADREGG: PAD25PULL (Bitfield-Mask: 0x01) */ +#define GPIO_PADREGG_PAD24FNCSEL_Pos (3UL) /*!< GPIO PADREGG: PAD24FNCSEL (Bit 3) */ +#define GPIO_PADREGG_PAD24FNCSEL_Msk (0x38UL) /*!< GPIO PADREGG: PAD24FNCSEL (Bitfield-Mask: 0x07) */ +#define GPIO_PADREGG_PAD24STRNG_Pos (2UL) /*!< GPIO PADREGG: PAD24STRNG (Bit 2) */ +#define GPIO_PADREGG_PAD24STRNG_Msk (0x4UL) /*!< GPIO PADREGG: PAD24STRNG (Bitfield-Mask: 0x01) */ +#define GPIO_PADREGG_PAD24INPEN_Pos (1UL) /*!< GPIO PADREGG: PAD24INPEN (Bit 1) */ +#define GPIO_PADREGG_PAD24INPEN_Msk (0x2UL) /*!< GPIO PADREGG: PAD24INPEN (Bitfield-Mask: 0x01) */ +#define GPIO_PADREGG_PAD24PULL_Pos (0UL) /*!< GPIO PADREGG: PAD24PULL (Bit 0) */ +#define GPIO_PADREGG_PAD24PULL_Msk (0x1UL) /*!< GPIO PADREGG: PAD24PULL (Bitfield-Mask: 0x01) */ +/* ======================================================== PADREGH ======================================================== */ +#define GPIO_PADREGH_PAD31FNCSEL_Pos (27UL) /*!< GPIO PADREGH: PAD31FNCSEL (Bit 27) */ +#define GPIO_PADREGH_PAD31FNCSEL_Msk (0x38000000UL) /*!< GPIO PADREGH: PAD31FNCSEL (Bitfield-Mask: 0x07) */ +#define GPIO_PADREGH_PAD31STRNG_Pos (26UL) /*!< GPIO PADREGH: PAD31STRNG (Bit 26) */ +#define GPIO_PADREGH_PAD31STRNG_Msk (0x4000000UL) /*!< GPIO PADREGH: PAD31STRNG (Bitfield-Mask: 0x01) */ +#define GPIO_PADREGH_PAD31INPEN_Pos (25UL) /*!< GPIO PADREGH: PAD31INPEN (Bit 25) */ +#define GPIO_PADREGH_PAD31INPEN_Msk (0x2000000UL) /*!< GPIO PADREGH: PAD31INPEN (Bitfield-Mask: 0x01) */ +#define GPIO_PADREGH_PAD31PULL_Pos (24UL) /*!< GPIO PADREGH: PAD31PULL (Bit 24) */ +#define GPIO_PADREGH_PAD31PULL_Msk (0x1000000UL) /*!< GPIO PADREGH: PAD31PULL (Bitfield-Mask: 0x01) */ +#define GPIO_PADREGH_PAD30FNCSEL_Pos (19UL) /*!< GPIO PADREGH: PAD30FNCSEL (Bit 19) */ +#define GPIO_PADREGH_PAD30FNCSEL_Msk (0x380000UL) /*!< GPIO PADREGH: PAD30FNCSEL (Bitfield-Mask: 0x07) */ +#define GPIO_PADREGH_PAD30STRNG_Pos (18UL) /*!< GPIO PADREGH: PAD30STRNG (Bit 18) */ +#define GPIO_PADREGH_PAD30STRNG_Msk (0x40000UL) /*!< GPIO PADREGH: PAD30STRNG (Bitfield-Mask: 0x01) */ +#define GPIO_PADREGH_PAD30INPEN_Pos (17UL) /*!< GPIO PADREGH: PAD30INPEN (Bit 17) */ +#define GPIO_PADREGH_PAD30INPEN_Msk (0x20000UL) /*!< GPIO PADREGH: PAD30INPEN (Bitfield-Mask: 0x01) */ +#define GPIO_PADREGH_PAD30PULL_Pos (16UL) /*!< GPIO PADREGH: PAD30PULL (Bit 16) */ +#define GPIO_PADREGH_PAD30PULL_Msk (0x10000UL) /*!< GPIO PADREGH: PAD30PULL (Bitfield-Mask: 0x01) */ +#define GPIO_PADREGH_PAD29FNCSEL_Pos (11UL) /*!< GPIO PADREGH: PAD29FNCSEL (Bit 11) */ +#define GPIO_PADREGH_PAD29FNCSEL_Msk (0x3800UL) /*!< GPIO PADREGH: PAD29FNCSEL (Bitfield-Mask: 0x07) */ +#define GPIO_PADREGH_PAD29STRNG_Pos (10UL) /*!< GPIO PADREGH: PAD29STRNG (Bit 10) */ +#define GPIO_PADREGH_PAD29STRNG_Msk (0x400UL) /*!< GPIO PADREGH: PAD29STRNG (Bitfield-Mask: 0x01) */ +#define GPIO_PADREGH_PAD29INPEN_Pos (9UL) /*!< GPIO PADREGH: PAD29INPEN (Bit 9) */ +#define GPIO_PADREGH_PAD29INPEN_Msk (0x200UL) /*!< GPIO PADREGH: PAD29INPEN (Bitfield-Mask: 0x01) */ +#define GPIO_PADREGH_PAD29PULL_Pos (8UL) /*!< GPIO PADREGH: PAD29PULL (Bit 8) */ +#define GPIO_PADREGH_PAD29PULL_Msk (0x100UL) /*!< GPIO PADREGH: PAD29PULL (Bitfield-Mask: 0x01) */ +#define GPIO_PADREGH_PAD28FNCSEL_Pos (3UL) /*!< GPIO PADREGH: PAD28FNCSEL (Bit 3) */ +#define GPIO_PADREGH_PAD28FNCSEL_Msk (0x38UL) /*!< GPIO PADREGH: PAD28FNCSEL (Bitfield-Mask: 0x07) */ +#define GPIO_PADREGH_PAD28STRNG_Pos (2UL) /*!< GPIO PADREGH: PAD28STRNG (Bit 2) */ +#define GPIO_PADREGH_PAD28STRNG_Msk (0x4UL) /*!< GPIO PADREGH: PAD28STRNG (Bitfield-Mask: 0x01) */ +#define GPIO_PADREGH_PAD28INPEN_Pos (1UL) /*!< GPIO PADREGH: PAD28INPEN (Bit 1) */ +#define GPIO_PADREGH_PAD28INPEN_Msk (0x2UL) /*!< GPIO PADREGH: PAD28INPEN (Bitfield-Mask: 0x01) */ +#define GPIO_PADREGH_PAD28PULL_Pos (0UL) /*!< GPIO PADREGH: PAD28PULL (Bit 0) */ +#define GPIO_PADREGH_PAD28PULL_Msk (0x1UL) /*!< GPIO PADREGH: PAD28PULL (Bitfield-Mask: 0x01) */ +/* ======================================================== PADREGI ======================================================== */ +#define GPIO_PADREGI_PAD35FNCSEL_Pos (27UL) /*!< GPIO PADREGI: PAD35FNCSEL (Bit 27) */ +#define GPIO_PADREGI_PAD35FNCSEL_Msk (0x38000000UL) /*!< GPIO PADREGI: PAD35FNCSEL (Bitfield-Mask: 0x07) */ +#define GPIO_PADREGI_PAD35STRNG_Pos (26UL) /*!< GPIO PADREGI: PAD35STRNG (Bit 26) */ +#define GPIO_PADREGI_PAD35STRNG_Msk (0x4000000UL) /*!< GPIO PADREGI: PAD35STRNG (Bitfield-Mask: 0x01) */ +#define GPIO_PADREGI_PAD35INPEN_Pos (25UL) /*!< GPIO PADREGI: PAD35INPEN (Bit 25) */ +#define GPIO_PADREGI_PAD35INPEN_Msk (0x2000000UL) /*!< GPIO PADREGI: PAD35INPEN (Bitfield-Mask: 0x01) */ +#define GPIO_PADREGI_PAD35PULL_Pos (24UL) /*!< GPIO PADREGI: PAD35PULL (Bit 24) */ +#define GPIO_PADREGI_PAD35PULL_Msk (0x1000000UL) /*!< GPIO PADREGI: PAD35PULL (Bitfield-Mask: 0x01) */ +#define GPIO_PADREGI_PAD34FNCSEL_Pos (19UL) /*!< GPIO PADREGI: PAD34FNCSEL (Bit 19) */ +#define GPIO_PADREGI_PAD34FNCSEL_Msk (0x380000UL) /*!< GPIO PADREGI: PAD34FNCSEL (Bitfield-Mask: 0x07) */ +#define GPIO_PADREGI_PAD34STRNG_Pos (18UL) /*!< GPIO PADREGI: PAD34STRNG (Bit 18) */ +#define GPIO_PADREGI_PAD34STRNG_Msk (0x40000UL) /*!< GPIO PADREGI: PAD34STRNG (Bitfield-Mask: 0x01) */ +#define GPIO_PADREGI_PAD34INPEN_Pos (17UL) /*!< GPIO PADREGI: PAD34INPEN (Bit 17) */ +#define GPIO_PADREGI_PAD34INPEN_Msk (0x20000UL) /*!< GPIO PADREGI: PAD34INPEN (Bitfield-Mask: 0x01) */ +#define GPIO_PADREGI_PAD34PULL_Pos (16UL) /*!< GPIO PADREGI: PAD34PULL (Bit 16) */ +#define GPIO_PADREGI_PAD34PULL_Msk (0x10000UL) /*!< GPIO PADREGI: PAD34PULL (Bitfield-Mask: 0x01) */ +#define GPIO_PADREGI_PAD33FNCSEL_Pos (11UL) /*!< GPIO PADREGI: PAD33FNCSEL (Bit 11) */ +#define GPIO_PADREGI_PAD33FNCSEL_Msk (0x3800UL) /*!< GPIO PADREGI: PAD33FNCSEL (Bitfield-Mask: 0x07) */ +#define GPIO_PADREGI_PAD33STRNG_Pos (10UL) /*!< GPIO PADREGI: PAD33STRNG (Bit 10) */ +#define GPIO_PADREGI_PAD33STRNG_Msk (0x400UL) /*!< GPIO PADREGI: PAD33STRNG (Bitfield-Mask: 0x01) */ +#define GPIO_PADREGI_PAD33INPEN_Pos (9UL) /*!< GPIO PADREGI: PAD33INPEN (Bit 9) */ +#define GPIO_PADREGI_PAD33INPEN_Msk (0x200UL) /*!< GPIO PADREGI: PAD33INPEN (Bitfield-Mask: 0x01) */ +#define GPIO_PADREGI_PAD33PULL_Pos (8UL) /*!< GPIO PADREGI: PAD33PULL (Bit 8) */ +#define GPIO_PADREGI_PAD33PULL_Msk (0x100UL) /*!< GPIO PADREGI: PAD33PULL (Bitfield-Mask: 0x01) */ +#define GPIO_PADREGI_PAD32FNCSEL_Pos (3UL) /*!< GPIO PADREGI: PAD32FNCSEL (Bit 3) */ +#define GPIO_PADREGI_PAD32FNCSEL_Msk (0x38UL) /*!< GPIO PADREGI: PAD32FNCSEL (Bitfield-Mask: 0x07) */ +#define GPIO_PADREGI_PAD32STRNG_Pos (2UL) /*!< GPIO PADREGI: PAD32STRNG (Bit 2) */ +#define GPIO_PADREGI_PAD32STRNG_Msk (0x4UL) /*!< GPIO PADREGI: PAD32STRNG (Bitfield-Mask: 0x01) */ +#define GPIO_PADREGI_PAD32INPEN_Pos (1UL) /*!< GPIO PADREGI: PAD32INPEN (Bit 1) */ +#define GPIO_PADREGI_PAD32INPEN_Msk (0x2UL) /*!< GPIO PADREGI: PAD32INPEN (Bitfield-Mask: 0x01) */ +#define GPIO_PADREGI_PAD32PULL_Pos (0UL) /*!< GPIO PADREGI: PAD32PULL (Bit 0) */ +#define GPIO_PADREGI_PAD32PULL_Msk (0x1UL) /*!< GPIO PADREGI: PAD32PULL (Bitfield-Mask: 0x01) */ +/* ======================================================== PADREGJ ======================================================== */ +#define GPIO_PADREGJ_PAD39RSEL_Pos (30UL) /*!< GPIO PADREGJ: PAD39RSEL (Bit 30) */ +#define GPIO_PADREGJ_PAD39RSEL_Msk (0xc0000000UL) /*!< GPIO PADREGJ: PAD39RSEL (Bitfield-Mask: 0x03) */ +#define GPIO_PADREGJ_PAD39FNCSEL_Pos (27UL) /*!< GPIO PADREGJ: PAD39FNCSEL (Bit 27) */ +#define GPIO_PADREGJ_PAD39FNCSEL_Msk (0x38000000UL) /*!< GPIO PADREGJ: PAD39FNCSEL (Bitfield-Mask: 0x07) */ +#define GPIO_PADREGJ_PAD39STRNG_Pos (26UL) /*!< GPIO PADREGJ: PAD39STRNG (Bit 26) */ +#define GPIO_PADREGJ_PAD39STRNG_Msk (0x4000000UL) /*!< GPIO PADREGJ: PAD39STRNG (Bitfield-Mask: 0x01) */ +#define GPIO_PADREGJ_PAD39INPEN_Pos (25UL) /*!< GPIO PADREGJ: PAD39INPEN (Bit 25) */ +#define GPIO_PADREGJ_PAD39INPEN_Msk (0x2000000UL) /*!< GPIO PADREGJ: PAD39INPEN (Bitfield-Mask: 0x01) */ +#define GPIO_PADREGJ_PAD39PULL_Pos (24UL) /*!< GPIO PADREGJ: PAD39PULL (Bit 24) */ +#define GPIO_PADREGJ_PAD39PULL_Msk (0x1000000UL) /*!< GPIO PADREGJ: PAD39PULL (Bitfield-Mask: 0x01) */ +#define GPIO_PADREGJ_PAD38FNCSEL_Pos (19UL) /*!< GPIO PADREGJ: PAD38FNCSEL (Bit 19) */ +#define GPIO_PADREGJ_PAD38FNCSEL_Msk (0x380000UL) /*!< GPIO PADREGJ: PAD38FNCSEL (Bitfield-Mask: 0x07) */ +#define GPIO_PADREGJ_PAD38STRNG_Pos (18UL) /*!< GPIO PADREGJ: PAD38STRNG (Bit 18) */ +#define GPIO_PADREGJ_PAD38STRNG_Msk (0x40000UL) /*!< GPIO PADREGJ: PAD38STRNG (Bitfield-Mask: 0x01) */ +#define GPIO_PADREGJ_PAD38INPEN_Pos (17UL) /*!< GPIO PADREGJ: PAD38INPEN (Bit 17) */ +#define GPIO_PADREGJ_PAD38INPEN_Msk (0x20000UL) /*!< GPIO PADREGJ: PAD38INPEN (Bitfield-Mask: 0x01) */ +#define GPIO_PADREGJ_PAD38PULL_Pos (16UL) /*!< GPIO PADREGJ: PAD38PULL (Bit 16) */ +#define GPIO_PADREGJ_PAD38PULL_Msk (0x10000UL) /*!< GPIO PADREGJ: PAD38PULL (Bitfield-Mask: 0x01) */ +#define GPIO_PADREGJ_PAD37PWRDN_Pos (15UL) /*!< GPIO PADREGJ: PAD37PWRDN (Bit 15) */ +#define GPIO_PADREGJ_PAD37PWRDN_Msk (0x8000UL) /*!< GPIO PADREGJ: PAD37PWRDN (Bitfield-Mask: 0x01) */ +#define GPIO_PADREGJ_PAD37FNCSEL_Pos (11UL) /*!< GPIO PADREGJ: PAD37FNCSEL (Bit 11) */ +#define GPIO_PADREGJ_PAD37FNCSEL_Msk (0x3800UL) /*!< GPIO PADREGJ: PAD37FNCSEL (Bitfield-Mask: 0x07) */ +#define GPIO_PADREGJ_PAD37STRNG_Pos (10UL) /*!< GPIO PADREGJ: PAD37STRNG (Bit 10) */ +#define GPIO_PADREGJ_PAD37STRNG_Msk (0x400UL) /*!< GPIO PADREGJ: PAD37STRNG (Bitfield-Mask: 0x01) */ +#define GPIO_PADREGJ_PAD37INPEN_Pos (9UL) /*!< GPIO PADREGJ: PAD37INPEN (Bit 9) */ +#define GPIO_PADREGJ_PAD37INPEN_Msk (0x200UL) /*!< GPIO PADREGJ: PAD37INPEN (Bitfield-Mask: 0x01) */ +#define GPIO_PADREGJ_PAD37PULL_Pos (8UL) /*!< GPIO PADREGJ: PAD37PULL (Bit 8) */ +#define GPIO_PADREGJ_PAD37PULL_Msk (0x100UL) /*!< GPIO PADREGJ: PAD37PULL (Bitfield-Mask: 0x01) */ +#define GPIO_PADREGJ_PAD36PWRUP_Pos (6UL) /*!< GPIO PADREGJ: PAD36PWRUP (Bit 6) */ +#define GPIO_PADREGJ_PAD36PWRUP_Msk (0x40UL) /*!< GPIO PADREGJ: PAD36PWRUP (Bitfield-Mask: 0x01) */ +#define GPIO_PADREGJ_PAD36FNCSEL_Pos (3UL) /*!< GPIO PADREGJ: PAD36FNCSEL (Bit 3) */ +#define GPIO_PADREGJ_PAD36FNCSEL_Msk (0x38UL) /*!< GPIO PADREGJ: PAD36FNCSEL (Bitfield-Mask: 0x07) */ +#define GPIO_PADREGJ_PAD36STRNG_Pos (2UL) /*!< GPIO PADREGJ: PAD36STRNG (Bit 2) */ +#define GPIO_PADREGJ_PAD36STRNG_Msk (0x4UL) /*!< GPIO PADREGJ: PAD36STRNG (Bitfield-Mask: 0x01) */ +#define GPIO_PADREGJ_PAD36INPEN_Pos (1UL) /*!< GPIO PADREGJ: PAD36INPEN (Bit 1) */ +#define GPIO_PADREGJ_PAD36INPEN_Msk (0x2UL) /*!< GPIO PADREGJ: PAD36INPEN (Bitfield-Mask: 0x01) */ +#define GPIO_PADREGJ_PAD36PULL_Pos (0UL) /*!< GPIO PADREGJ: PAD36PULL (Bit 0) */ +#define GPIO_PADREGJ_PAD36PULL_Msk (0x1UL) /*!< GPIO PADREGJ: PAD36PULL (Bitfield-Mask: 0x01) */ +/* ======================================================== PADREGK ======================================================== */ +#define GPIO_PADREGK_PAD43RSEL_Pos (30UL) /*!< GPIO PADREGK: PAD43RSEL (Bit 30) */ +#define GPIO_PADREGK_PAD43RSEL_Msk (0xc0000000UL) /*!< GPIO PADREGK: PAD43RSEL (Bitfield-Mask: 0x03) */ +#define GPIO_PADREGK_PAD43FNCSEL_Pos (27UL) /*!< GPIO PADREGK: PAD43FNCSEL (Bit 27) */ +#define GPIO_PADREGK_PAD43FNCSEL_Msk (0x38000000UL) /*!< GPIO PADREGK: PAD43FNCSEL (Bitfield-Mask: 0x07) */ +#define GPIO_PADREGK_PAD43STRNG_Pos (26UL) /*!< GPIO PADREGK: PAD43STRNG (Bit 26) */ +#define GPIO_PADREGK_PAD43STRNG_Msk (0x4000000UL) /*!< GPIO PADREGK: PAD43STRNG (Bitfield-Mask: 0x01) */ +#define GPIO_PADREGK_PAD43INPEN_Pos (25UL) /*!< GPIO PADREGK: PAD43INPEN (Bit 25) */ +#define GPIO_PADREGK_PAD43INPEN_Msk (0x2000000UL) /*!< GPIO PADREGK: PAD43INPEN (Bitfield-Mask: 0x01) */ +#define GPIO_PADREGK_PAD43PULL_Pos (24UL) /*!< GPIO PADREGK: PAD43PULL (Bit 24) */ +#define GPIO_PADREGK_PAD43PULL_Msk (0x1000000UL) /*!< GPIO PADREGK: PAD43PULL (Bitfield-Mask: 0x01) */ +#define GPIO_PADREGK_PAD42RSEL_Pos (22UL) /*!< GPIO PADREGK: PAD42RSEL (Bit 22) */ +#define GPIO_PADREGK_PAD42RSEL_Msk (0xc00000UL) /*!< GPIO PADREGK: PAD42RSEL (Bitfield-Mask: 0x03) */ +#define GPIO_PADREGK_PAD42FNCSEL_Pos (19UL) /*!< GPIO PADREGK: PAD42FNCSEL (Bit 19) */ +#define GPIO_PADREGK_PAD42FNCSEL_Msk (0x380000UL) /*!< GPIO PADREGK: PAD42FNCSEL (Bitfield-Mask: 0x07) */ +#define GPIO_PADREGK_PAD42STRNG_Pos (18UL) /*!< GPIO PADREGK: PAD42STRNG (Bit 18) */ +#define GPIO_PADREGK_PAD42STRNG_Msk (0x40000UL) /*!< GPIO PADREGK: PAD42STRNG (Bitfield-Mask: 0x01) */ +#define GPIO_PADREGK_PAD42INPEN_Pos (17UL) /*!< GPIO PADREGK: PAD42INPEN (Bit 17) */ +#define GPIO_PADREGK_PAD42INPEN_Msk (0x20000UL) /*!< GPIO PADREGK: PAD42INPEN (Bitfield-Mask: 0x01) */ +#define GPIO_PADREGK_PAD42PULL_Pos (16UL) /*!< GPIO PADREGK: PAD42PULL (Bit 16) */ +#define GPIO_PADREGK_PAD42PULL_Msk (0x10000UL) /*!< GPIO PADREGK: PAD42PULL (Bitfield-Mask: 0x01) */ +#define GPIO_PADREGK_PAD41PWRDN_Pos (15UL) /*!< GPIO PADREGK: PAD41PWRDN (Bit 15) */ +#define GPIO_PADREGK_PAD41PWRDN_Msk (0x8000UL) /*!< GPIO PADREGK: PAD41PWRDN (Bitfield-Mask: 0x01) */ +#define GPIO_PADREGK_PAD41FNCSEL_Pos (11UL) /*!< GPIO PADREGK: PAD41FNCSEL (Bit 11) */ +#define GPIO_PADREGK_PAD41FNCSEL_Msk (0x3800UL) /*!< GPIO PADREGK: PAD41FNCSEL (Bitfield-Mask: 0x07) */ +#define GPIO_PADREGK_PAD41STRNG_Pos (10UL) /*!< GPIO PADREGK: PAD41STRNG (Bit 10) */ +#define GPIO_PADREGK_PAD41STRNG_Msk (0x400UL) /*!< GPIO PADREGK: PAD41STRNG (Bitfield-Mask: 0x01) */ +#define GPIO_PADREGK_PAD41INPEN_Pos (9UL) /*!< GPIO PADREGK: PAD41INPEN (Bit 9) */ +#define GPIO_PADREGK_PAD41INPEN_Msk (0x200UL) /*!< GPIO PADREGK: PAD41INPEN (Bitfield-Mask: 0x01) */ +#define GPIO_PADREGK_PAD41PULL_Pos (8UL) /*!< GPIO PADREGK: PAD41PULL (Bit 8) */ +#define GPIO_PADREGK_PAD41PULL_Msk (0x100UL) /*!< GPIO PADREGK: PAD41PULL (Bitfield-Mask: 0x01) */ +#define GPIO_PADREGK_PAD40RSEL_Pos (6UL) /*!< GPIO PADREGK: PAD40RSEL (Bit 6) */ +#define GPIO_PADREGK_PAD40RSEL_Msk (0xc0UL) /*!< GPIO PADREGK: PAD40RSEL (Bitfield-Mask: 0x03) */ +#define GPIO_PADREGK_PAD40FNCSEL_Pos (3UL) /*!< GPIO PADREGK: PAD40FNCSEL (Bit 3) */ +#define GPIO_PADREGK_PAD40FNCSEL_Msk (0x38UL) /*!< GPIO PADREGK: PAD40FNCSEL (Bitfield-Mask: 0x07) */ +#define GPIO_PADREGK_PAD40STRNG_Pos (2UL) /*!< GPIO PADREGK: PAD40STRNG (Bit 2) */ +#define GPIO_PADREGK_PAD40STRNG_Msk (0x4UL) /*!< GPIO PADREGK: PAD40STRNG (Bitfield-Mask: 0x01) */ +#define GPIO_PADREGK_PAD40INPEN_Pos (1UL) /*!< GPIO PADREGK: PAD40INPEN (Bit 1) */ +#define GPIO_PADREGK_PAD40INPEN_Msk (0x2UL) /*!< GPIO PADREGK: PAD40INPEN (Bitfield-Mask: 0x01) */ +#define GPIO_PADREGK_PAD40PULL_Pos (0UL) /*!< GPIO PADREGK: PAD40PULL (Bit 0) */ +#define GPIO_PADREGK_PAD40PULL_Msk (0x1UL) /*!< GPIO PADREGK: PAD40PULL (Bitfield-Mask: 0x01) */ +/* ======================================================== PADREGL ======================================================== */ +#define GPIO_PADREGL_PAD47FNCSEL_Pos (27UL) /*!< GPIO PADREGL: PAD47FNCSEL (Bit 27) */ +#define GPIO_PADREGL_PAD47FNCSEL_Msk (0x38000000UL) /*!< GPIO PADREGL: PAD47FNCSEL (Bitfield-Mask: 0x07) */ +#define GPIO_PADREGL_PAD47STRNG_Pos (26UL) /*!< GPIO PADREGL: PAD47STRNG (Bit 26) */ +#define GPIO_PADREGL_PAD47STRNG_Msk (0x4000000UL) /*!< GPIO PADREGL: PAD47STRNG (Bitfield-Mask: 0x01) */ +#define GPIO_PADREGL_PAD47INPEN_Pos (25UL) /*!< GPIO PADREGL: PAD47INPEN (Bit 25) */ +#define GPIO_PADREGL_PAD47INPEN_Msk (0x2000000UL) /*!< GPIO PADREGL: PAD47INPEN (Bitfield-Mask: 0x01) */ +#define GPIO_PADREGL_PAD47PULL_Pos (24UL) /*!< GPIO PADREGL: PAD47PULL (Bit 24) */ +#define GPIO_PADREGL_PAD47PULL_Msk (0x1000000UL) /*!< GPIO PADREGL: PAD47PULL (Bitfield-Mask: 0x01) */ +#define GPIO_PADREGL_PAD46FNCSEL_Pos (19UL) /*!< GPIO PADREGL: PAD46FNCSEL (Bit 19) */ +#define GPIO_PADREGL_PAD46FNCSEL_Msk (0x380000UL) /*!< GPIO PADREGL: PAD46FNCSEL (Bitfield-Mask: 0x07) */ +#define GPIO_PADREGL_PAD46STRNG_Pos (18UL) /*!< GPIO PADREGL: PAD46STRNG (Bit 18) */ +#define GPIO_PADREGL_PAD46STRNG_Msk (0x40000UL) /*!< GPIO PADREGL: PAD46STRNG (Bitfield-Mask: 0x01) */ +#define GPIO_PADREGL_PAD46INPEN_Pos (17UL) /*!< GPIO PADREGL: PAD46INPEN (Bit 17) */ +#define GPIO_PADREGL_PAD46INPEN_Msk (0x20000UL) /*!< GPIO PADREGL: PAD46INPEN (Bitfield-Mask: 0x01) */ +#define GPIO_PADREGL_PAD46PULL_Pos (16UL) /*!< GPIO PADREGL: PAD46PULL (Bit 16) */ +#define GPIO_PADREGL_PAD46PULL_Msk (0x10000UL) /*!< GPIO PADREGL: PAD46PULL (Bitfield-Mask: 0x01) */ +#define GPIO_PADREGL_PAD45FNCSEL_Pos (11UL) /*!< GPIO PADREGL: PAD45FNCSEL (Bit 11) */ +#define GPIO_PADREGL_PAD45FNCSEL_Msk (0x3800UL) /*!< GPIO PADREGL: PAD45FNCSEL (Bitfield-Mask: 0x07) */ +#define GPIO_PADREGL_PAD45STRNG_Pos (10UL) /*!< GPIO PADREGL: PAD45STRNG (Bit 10) */ +#define GPIO_PADREGL_PAD45STRNG_Msk (0x400UL) /*!< GPIO PADREGL: PAD45STRNG (Bitfield-Mask: 0x01) */ +#define GPIO_PADREGL_PAD45INPEN_Pos (9UL) /*!< GPIO PADREGL: PAD45INPEN (Bit 9) */ +#define GPIO_PADREGL_PAD45INPEN_Msk (0x200UL) /*!< GPIO PADREGL: PAD45INPEN (Bitfield-Mask: 0x01) */ +#define GPIO_PADREGL_PAD45PULL_Pos (8UL) /*!< GPIO PADREGL: PAD45PULL (Bit 8) */ +#define GPIO_PADREGL_PAD45PULL_Msk (0x100UL) /*!< GPIO PADREGL: PAD45PULL (Bitfield-Mask: 0x01) */ +#define GPIO_PADREGL_PAD44FNCSEL_Pos (3UL) /*!< GPIO PADREGL: PAD44FNCSEL (Bit 3) */ +#define GPIO_PADREGL_PAD44FNCSEL_Msk (0x38UL) /*!< GPIO PADREGL: PAD44FNCSEL (Bitfield-Mask: 0x07) */ +#define GPIO_PADREGL_PAD44STRNG_Pos (2UL) /*!< GPIO PADREGL: PAD44STRNG (Bit 2) */ +#define GPIO_PADREGL_PAD44STRNG_Msk (0x4UL) /*!< GPIO PADREGL: PAD44STRNG (Bitfield-Mask: 0x01) */ +#define GPIO_PADREGL_PAD44INPEN_Pos (1UL) /*!< GPIO PADREGL: PAD44INPEN (Bit 1) */ +#define GPIO_PADREGL_PAD44INPEN_Msk (0x2UL) /*!< GPIO PADREGL: PAD44INPEN (Bitfield-Mask: 0x01) */ +#define GPIO_PADREGL_PAD44PULL_Pos (0UL) /*!< GPIO PADREGL: PAD44PULL (Bit 0) */ +#define GPIO_PADREGL_PAD44PULL_Msk (0x1UL) /*!< GPIO PADREGL: PAD44PULL (Bitfield-Mask: 0x01) */ +/* ======================================================== PADREGM ======================================================== */ +#define GPIO_PADREGM_PAD49RSEL_Pos (14UL) /*!< GPIO PADREGM: PAD49RSEL (Bit 14) */ +#define GPIO_PADREGM_PAD49RSEL_Msk (0xc000UL) /*!< GPIO PADREGM: PAD49RSEL (Bitfield-Mask: 0x03) */ +#define GPIO_PADREGM_PAD49FNCSEL_Pos (11UL) /*!< GPIO PADREGM: PAD49FNCSEL (Bit 11) */ +#define GPIO_PADREGM_PAD49FNCSEL_Msk (0x3800UL) /*!< GPIO PADREGM: PAD49FNCSEL (Bitfield-Mask: 0x07) */ +#define GPIO_PADREGM_PAD49STRNG_Pos (10UL) /*!< GPIO PADREGM: PAD49STRNG (Bit 10) */ +#define GPIO_PADREGM_PAD49STRNG_Msk (0x400UL) /*!< GPIO PADREGM: PAD49STRNG (Bitfield-Mask: 0x01) */ +#define GPIO_PADREGM_PAD49INPEN_Pos (9UL) /*!< GPIO PADREGM: PAD49INPEN (Bit 9) */ +#define GPIO_PADREGM_PAD49INPEN_Msk (0x200UL) /*!< GPIO PADREGM: PAD49INPEN (Bitfield-Mask: 0x01) */ +#define GPIO_PADREGM_PAD49PULL_Pos (8UL) /*!< GPIO PADREGM: PAD49PULL (Bit 8) */ +#define GPIO_PADREGM_PAD49PULL_Msk (0x100UL) /*!< GPIO PADREGM: PAD49PULL (Bitfield-Mask: 0x01) */ +#define GPIO_PADREGM_PAD48RSEL_Pos (6UL) /*!< GPIO PADREGM: PAD48RSEL (Bit 6) */ +#define GPIO_PADREGM_PAD48RSEL_Msk (0xc0UL) /*!< GPIO PADREGM: PAD48RSEL (Bitfield-Mask: 0x03) */ +#define GPIO_PADREGM_PAD48FNCSEL_Pos (3UL) /*!< GPIO PADREGM: PAD48FNCSEL (Bit 3) */ +#define GPIO_PADREGM_PAD48FNCSEL_Msk (0x38UL) /*!< GPIO PADREGM: PAD48FNCSEL (Bitfield-Mask: 0x07) */ +#define GPIO_PADREGM_PAD48STRNG_Pos (2UL) /*!< GPIO PADREGM: PAD48STRNG (Bit 2) */ +#define GPIO_PADREGM_PAD48STRNG_Msk (0x4UL) /*!< GPIO PADREGM: PAD48STRNG (Bitfield-Mask: 0x01) */ +#define GPIO_PADREGM_PAD48INPEN_Pos (1UL) /*!< GPIO PADREGM: PAD48INPEN (Bit 1) */ +#define GPIO_PADREGM_PAD48INPEN_Msk (0x2UL) /*!< GPIO PADREGM: PAD48INPEN (Bitfield-Mask: 0x01) */ +#define GPIO_PADREGM_PAD48PULL_Pos (0UL) /*!< GPIO PADREGM: PAD48PULL (Bit 0) */ +#define GPIO_PADREGM_PAD48PULL_Msk (0x1UL) /*!< GPIO PADREGM: PAD48PULL (Bitfield-Mask: 0x01) */ +/* ========================================================= CFGA ========================================================== */ +#define GPIO_CFGA_GPIO7INTD_Pos (31UL) /*!< GPIO CFGA: GPIO7INTD (Bit 31) */ +#define GPIO_CFGA_GPIO7INTD_Msk (0x80000000UL) /*!< GPIO CFGA: GPIO7INTD (Bitfield-Mask: 0x01) */ +#define GPIO_CFGA_GPIO7OUTCFG_Pos (29UL) /*!< GPIO CFGA: GPIO7OUTCFG (Bit 29) */ +#define GPIO_CFGA_GPIO7OUTCFG_Msk (0x60000000UL) /*!< GPIO CFGA: GPIO7OUTCFG (Bitfield-Mask: 0x03) */ +#define GPIO_CFGA_GPIO7INCFG_Pos (28UL) /*!< GPIO CFGA: GPIO7INCFG (Bit 28) */ +#define GPIO_CFGA_GPIO7INCFG_Msk (0x10000000UL) /*!< GPIO CFGA: GPIO7INCFG (Bitfield-Mask: 0x01) */ +#define GPIO_CFGA_GPIO6INTD_Pos (27UL) /*!< GPIO CFGA: GPIO6INTD (Bit 27) */ +#define GPIO_CFGA_GPIO6INTD_Msk (0x8000000UL) /*!< GPIO CFGA: GPIO6INTD (Bitfield-Mask: 0x01) */ +#define GPIO_CFGA_GPIO6OUTCFG_Pos (25UL) /*!< GPIO CFGA: GPIO6OUTCFG (Bit 25) */ +#define GPIO_CFGA_GPIO6OUTCFG_Msk (0x6000000UL) /*!< GPIO CFGA: GPIO6OUTCFG (Bitfield-Mask: 0x03) */ +#define GPIO_CFGA_GPIO6INCFG_Pos (24UL) /*!< GPIO CFGA: GPIO6INCFG (Bit 24) */ +#define GPIO_CFGA_GPIO6INCFG_Msk (0x1000000UL) /*!< GPIO CFGA: GPIO6INCFG (Bitfield-Mask: 0x01) */ +#define GPIO_CFGA_GPIO5INTD_Pos (23UL) /*!< GPIO CFGA: GPIO5INTD (Bit 23) */ +#define GPIO_CFGA_GPIO5INTD_Msk (0x800000UL) /*!< GPIO CFGA: GPIO5INTD (Bitfield-Mask: 0x01) */ +#define GPIO_CFGA_GPIO5OUTCFG_Pos (21UL) /*!< GPIO CFGA: GPIO5OUTCFG (Bit 21) */ +#define GPIO_CFGA_GPIO5OUTCFG_Msk (0x600000UL) /*!< GPIO CFGA: GPIO5OUTCFG (Bitfield-Mask: 0x03) */ +#define GPIO_CFGA_GPIO5INCFG_Pos (20UL) /*!< GPIO CFGA: GPIO5INCFG (Bit 20) */ +#define GPIO_CFGA_GPIO5INCFG_Msk (0x100000UL) /*!< GPIO CFGA: GPIO5INCFG (Bitfield-Mask: 0x01) */ +#define GPIO_CFGA_GPIO4INTD_Pos (19UL) /*!< GPIO CFGA: GPIO4INTD (Bit 19) */ +#define GPIO_CFGA_GPIO4INTD_Msk (0x80000UL) /*!< GPIO CFGA: GPIO4INTD (Bitfield-Mask: 0x01) */ +#define GPIO_CFGA_GPIO4OUTCFG_Pos (17UL) /*!< GPIO CFGA: GPIO4OUTCFG (Bit 17) */ +#define GPIO_CFGA_GPIO4OUTCFG_Msk (0x60000UL) /*!< GPIO CFGA: GPIO4OUTCFG (Bitfield-Mask: 0x03) */ +#define GPIO_CFGA_GPIO4INCFG_Pos (16UL) /*!< GPIO CFGA: GPIO4INCFG (Bit 16) */ +#define GPIO_CFGA_GPIO4INCFG_Msk (0x10000UL) /*!< GPIO CFGA: GPIO4INCFG (Bitfield-Mask: 0x01) */ +#define GPIO_CFGA_GPIO3INTD_Pos (15UL) /*!< GPIO CFGA: GPIO3INTD (Bit 15) */ +#define GPIO_CFGA_GPIO3INTD_Msk (0x8000UL) /*!< GPIO CFGA: GPIO3INTD (Bitfield-Mask: 0x01) */ +#define GPIO_CFGA_GPIO3OUTCFG_Pos (13UL) /*!< GPIO CFGA: GPIO3OUTCFG (Bit 13) */ +#define GPIO_CFGA_GPIO3OUTCFG_Msk (0x6000UL) /*!< GPIO CFGA: GPIO3OUTCFG (Bitfield-Mask: 0x03) */ +#define GPIO_CFGA_GPIO3INCFG_Pos (12UL) /*!< GPIO CFGA: GPIO3INCFG (Bit 12) */ +#define GPIO_CFGA_GPIO3INCFG_Msk (0x1000UL) /*!< GPIO CFGA: GPIO3INCFG (Bitfield-Mask: 0x01) */ +#define GPIO_CFGA_GPIO2INTD_Pos (11UL) /*!< GPIO CFGA: GPIO2INTD (Bit 11) */ +#define GPIO_CFGA_GPIO2INTD_Msk (0x800UL) /*!< GPIO CFGA: GPIO2INTD (Bitfield-Mask: 0x01) */ +#define GPIO_CFGA_GPIO2OUTCFG_Pos (9UL) /*!< GPIO CFGA: GPIO2OUTCFG (Bit 9) */ +#define GPIO_CFGA_GPIO2OUTCFG_Msk (0x600UL) /*!< GPIO CFGA: GPIO2OUTCFG (Bitfield-Mask: 0x03) */ +#define GPIO_CFGA_GPIO2INCFG_Pos (8UL) /*!< GPIO CFGA: GPIO2INCFG (Bit 8) */ +#define GPIO_CFGA_GPIO2INCFG_Msk (0x100UL) /*!< GPIO CFGA: GPIO2INCFG (Bitfield-Mask: 0x01) */ +#define GPIO_CFGA_GPIO1INTD_Pos (7UL) /*!< GPIO CFGA: GPIO1INTD (Bit 7) */ +#define GPIO_CFGA_GPIO1INTD_Msk (0x80UL) /*!< GPIO CFGA: GPIO1INTD (Bitfield-Mask: 0x01) */ +#define GPIO_CFGA_GPIO1OUTCFG_Pos (5UL) /*!< GPIO CFGA: GPIO1OUTCFG (Bit 5) */ +#define GPIO_CFGA_GPIO1OUTCFG_Msk (0x60UL) /*!< GPIO CFGA: GPIO1OUTCFG (Bitfield-Mask: 0x03) */ +#define GPIO_CFGA_GPIO1INCFG_Pos (4UL) /*!< GPIO CFGA: GPIO1INCFG (Bit 4) */ +#define GPIO_CFGA_GPIO1INCFG_Msk (0x10UL) /*!< GPIO CFGA: GPIO1INCFG (Bitfield-Mask: 0x01) */ +#define GPIO_CFGA_GPIO0INTD_Pos (3UL) /*!< GPIO CFGA: GPIO0INTD (Bit 3) */ +#define GPIO_CFGA_GPIO0INTD_Msk (0x8UL) /*!< GPIO CFGA: GPIO0INTD (Bitfield-Mask: 0x01) */ +#define GPIO_CFGA_GPIO0OUTCFG_Pos (1UL) /*!< GPIO CFGA: GPIO0OUTCFG (Bit 1) */ +#define GPIO_CFGA_GPIO0OUTCFG_Msk (0x6UL) /*!< GPIO CFGA: GPIO0OUTCFG (Bitfield-Mask: 0x03) */ +#define GPIO_CFGA_GPIO0INCFG_Pos (0UL) /*!< GPIO CFGA: GPIO0INCFG (Bit 0) */ +#define GPIO_CFGA_GPIO0INCFG_Msk (0x1UL) /*!< GPIO CFGA: GPIO0INCFG (Bitfield-Mask: 0x01) */ +/* ========================================================= CFGB ========================================================== */ +#define GPIO_CFGB_GPIO15INTD_Pos (31UL) /*!< GPIO CFGB: GPIO15INTD (Bit 31) */ +#define GPIO_CFGB_GPIO15INTD_Msk (0x80000000UL) /*!< GPIO CFGB: GPIO15INTD (Bitfield-Mask: 0x01) */ +#define GPIO_CFGB_GPIO15OUTCFG_Pos (29UL) /*!< GPIO CFGB: GPIO15OUTCFG (Bit 29) */ +#define GPIO_CFGB_GPIO15OUTCFG_Msk (0x60000000UL) /*!< GPIO CFGB: GPIO15OUTCFG (Bitfield-Mask: 0x03) */ +#define GPIO_CFGB_GPIO15INCFG_Pos (28UL) /*!< GPIO CFGB: GPIO15INCFG (Bit 28) */ +#define GPIO_CFGB_GPIO15INCFG_Msk (0x10000000UL) /*!< GPIO CFGB: GPIO15INCFG (Bitfield-Mask: 0x01) */ +#define GPIO_CFGB_GPIO14INTD_Pos (27UL) /*!< GPIO CFGB: GPIO14INTD (Bit 27) */ +#define GPIO_CFGB_GPIO14INTD_Msk (0x8000000UL) /*!< GPIO CFGB: GPIO14INTD (Bitfield-Mask: 0x01) */ +#define GPIO_CFGB_GPIO14OUTCFG_Pos (25UL) /*!< GPIO CFGB: GPIO14OUTCFG (Bit 25) */ +#define GPIO_CFGB_GPIO14OUTCFG_Msk (0x6000000UL) /*!< GPIO CFGB: GPIO14OUTCFG (Bitfield-Mask: 0x03) */ +#define GPIO_CFGB_GPIO14INCFG_Pos (24UL) /*!< GPIO CFGB: GPIO14INCFG (Bit 24) */ +#define GPIO_CFGB_GPIO14INCFG_Msk (0x1000000UL) /*!< GPIO CFGB: GPIO14INCFG (Bitfield-Mask: 0x01) */ +#define GPIO_CFGB_GPIO13INTD_Pos (23UL) /*!< GPIO CFGB: GPIO13INTD (Bit 23) */ +#define GPIO_CFGB_GPIO13INTD_Msk (0x800000UL) /*!< GPIO CFGB: GPIO13INTD (Bitfield-Mask: 0x01) */ +#define GPIO_CFGB_GPIO13OUTCFG_Pos (21UL) /*!< GPIO CFGB: GPIO13OUTCFG (Bit 21) */ +#define GPIO_CFGB_GPIO13OUTCFG_Msk (0x600000UL) /*!< GPIO CFGB: GPIO13OUTCFG (Bitfield-Mask: 0x03) */ +#define GPIO_CFGB_GPIO13INCFG_Pos (20UL) /*!< GPIO CFGB: GPIO13INCFG (Bit 20) */ +#define GPIO_CFGB_GPIO13INCFG_Msk (0x100000UL) /*!< GPIO CFGB: GPIO13INCFG (Bitfield-Mask: 0x01) */ +#define GPIO_CFGB_GPIO12INTD_Pos (19UL) /*!< GPIO CFGB: GPIO12INTD (Bit 19) */ +#define GPIO_CFGB_GPIO12INTD_Msk (0x80000UL) /*!< GPIO CFGB: GPIO12INTD (Bitfield-Mask: 0x01) */ +#define GPIO_CFGB_GPIO12OUTCFG_Pos (17UL) /*!< GPIO CFGB: GPIO12OUTCFG (Bit 17) */ +#define GPIO_CFGB_GPIO12OUTCFG_Msk (0x60000UL) /*!< GPIO CFGB: GPIO12OUTCFG (Bitfield-Mask: 0x03) */ +#define GPIO_CFGB_GPIO12INCFG_Pos (16UL) /*!< GPIO CFGB: GPIO12INCFG (Bit 16) */ +#define GPIO_CFGB_GPIO12INCFG_Msk (0x10000UL) /*!< GPIO CFGB: GPIO12INCFG (Bitfield-Mask: 0x01) */ +#define GPIO_CFGB_GPIO11INTD_Pos (15UL) /*!< GPIO CFGB: GPIO11INTD (Bit 15) */ +#define GPIO_CFGB_GPIO11INTD_Msk (0x8000UL) /*!< GPIO CFGB: GPIO11INTD (Bitfield-Mask: 0x01) */ +#define GPIO_CFGB_GPIO11OUTCFG_Pos (13UL) /*!< GPIO CFGB: GPIO11OUTCFG (Bit 13) */ +#define GPIO_CFGB_GPIO11OUTCFG_Msk (0x6000UL) /*!< GPIO CFGB: GPIO11OUTCFG (Bitfield-Mask: 0x03) */ +#define GPIO_CFGB_GPIO11INCFG_Pos (12UL) /*!< GPIO CFGB: GPIO11INCFG (Bit 12) */ +#define GPIO_CFGB_GPIO11INCFG_Msk (0x1000UL) /*!< GPIO CFGB: GPIO11INCFG (Bitfield-Mask: 0x01) */ +#define GPIO_CFGB_GPIO10INTD_Pos (11UL) /*!< GPIO CFGB: GPIO10INTD (Bit 11) */ +#define GPIO_CFGB_GPIO10INTD_Msk (0x800UL) /*!< GPIO CFGB: GPIO10INTD (Bitfield-Mask: 0x01) */ +#define GPIO_CFGB_GPIO10OUTCFG_Pos (9UL) /*!< GPIO CFGB: GPIO10OUTCFG (Bit 9) */ +#define GPIO_CFGB_GPIO10OUTCFG_Msk (0x600UL) /*!< GPIO CFGB: GPIO10OUTCFG (Bitfield-Mask: 0x03) */ +#define GPIO_CFGB_GPIO10INCFG_Pos (8UL) /*!< GPIO CFGB: GPIO10INCFG (Bit 8) */ +#define GPIO_CFGB_GPIO10INCFG_Msk (0x100UL) /*!< GPIO CFGB: GPIO10INCFG (Bitfield-Mask: 0x01) */ +#define GPIO_CFGB_GPIO9INTD_Pos (7UL) /*!< GPIO CFGB: GPIO9INTD (Bit 7) */ +#define GPIO_CFGB_GPIO9INTD_Msk (0x80UL) /*!< GPIO CFGB: GPIO9INTD (Bitfield-Mask: 0x01) */ +#define GPIO_CFGB_GPIO9OUTCFG_Pos (5UL) /*!< GPIO CFGB: GPIO9OUTCFG (Bit 5) */ +#define GPIO_CFGB_GPIO9OUTCFG_Msk (0x60UL) /*!< GPIO CFGB: GPIO9OUTCFG (Bitfield-Mask: 0x03) */ +#define GPIO_CFGB_GPIO9INCFG_Pos (4UL) /*!< GPIO CFGB: GPIO9INCFG (Bit 4) */ +#define GPIO_CFGB_GPIO9INCFG_Msk (0x10UL) /*!< GPIO CFGB: GPIO9INCFG (Bitfield-Mask: 0x01) */ +#define GPIO_CFGB_GPIO8INTD_Pos (3UL) /*!< GPIO CFGB: GPIO8INTD (Bit 3) */ +#define GPIO_CFGB_GPIO8INTD_Msk (0x8UL) /*!< GPIO CFGB: GPIO8INTD (Bitfield-Mask: 0x01) */ +#define GPIO_CFGB_GPIO8OUTCFG_Pos (1UL) /*!< GPIO CFGB: GPIO8OUTCFG (Bit 1) */ +#define GPIO_CFGB_GPIO8OUTCFG_Msk (0x6UL) /*!< GPIO CFGB: GPIO8OUTCFG (Bitfield-Mask: 0x03) */ +#define GPIO_CFGB_GPIO8INCFG_Pos (0UL) /*!< GPIO CFGB: GPIO8INCFG (Bit 0) */ +#define GPIO_CFGB_GPIO8INCFG_Msk (0x1UL) /*!< GPIO CFGB: GPIO8INCFG (Bitfield-Mask: 0x01) */ +/* ========================================================= CFGC ========================================================== */ +#define GPIO_CFGC_GPIO23INTD_Pos (31UL) /*!< GPIO CFGC: GPIO23INTD (Bit 31) */ +#define GPIO_CFGC_GPIO23INTD_Msk (0x80000000UL) /*!< GPIO CFGC: GPIO23INTD (Bitfield-Mask: 0x01) */ +#define GPIO_CFGC_GPIO23OUTCFG_Pos (29UL) /*!< GPIO CFGC: GPIO23OUTCFG (Bit 29) */ +#define GPIO_CFGC_GPIO23OUTCFG_Msk (0x60000000UL) /*!< GPIO CFGC: GPIO23OUTCFG (Bitfield-Mask: 0x03) */ +#define GPIO_CFGC_GPIO23INCFG_Pos (28UL) /*!< GPIO CFGC: GPIO23INCFG (Bit 28) */ +#define GPIO_CFGC_GPIO23INCFG_Msk (0x10000000UL) /*!< GPIO CFGC: GPIO23INCFG (Bitfield-Mask: 0x01) */ +#define GPIO_CFGC_GPIO22INTD_Pos (27UL) /*!< GPIO CFGC: GPIO22INTD (Bit 27) */ +#define GPIO_CFGC_GPIO22INTD_Msk (0x8000000UL) /*!< GPIO CFGC: GPIO22INTD (Bitfield-Mask: 0x01) */ +#define GPIO_CFGC_GPIO22OUTCFG_Pos (25UL) /*!< GPIO CFGC: GPIO22OUTCFG (Bit 25) */ +#define GPIO_CFGC_GPIO22OUTCFG_Msk (0x6000000UL) /*!< GPIO CFGC: GPIO22OUTCFG (Bitfield-Mask: 0x03) */ +#define GPIO_CFGC_GPIO22INCFG_Pos (24UL) /*!< GPIO CFGC: GPIO22INCFG (Bit 24) */ +#define GPIO_CFGC_GPIO22INCFG_Msk (0x1000000UL) /*!< GPIO CFGC: GPIO22INCFG (Bitfield-Mask: 0x01) */ +#define GPIO_CFGC_GPIO21INTD_Pos (23UL) /*!< GPIO CFGC: GPIO21INTD (Bit 23) */ +#define GPIO_CFGC_GPIO21INTD_Msk (0x800000UL) /*!< GPIO CFGC: GPIO21INTD (Bitfield-Mask: 0x01) */ +#define GPIO_CFGC_GPIO21OUTCFG_Pos (21UL) /*!< GPIO CFGC: GPIO21OUTCFG (Bit 21) */ +#define GPIO_CFGC_GPIO21OUTCFG_Msk (0x600000UL) /*!< GPIO CFGC: GPIO21OUTCFG (Bitfield-Mask: 0x03) */ +#define GPIO_CFGC_GPIO21INCFG_Pos (20UL) /*!< GPIO CFGC: GPIO21INCFG (Bit 20) */ +#define GPIO_CFGC_GPIO21INCFG_Msk (0x100000UL) /*!< GPIO CFGC: GPIO21INCFG (Bitfield-Mask: 0x01) */ +#define GPIO_CFGC_GPIO20INTD_Pos (19UL) /*!< GPIO CFGC: GPIO20INTD (Bit 19) */ +#define GPIO_CFGC_GPIO20INTD_Msk (0x80000UL) /*!< GPIO CFGC: GPIO20INTD (Bitfield-Mask: 0x01) */ +#define GPIO_CFGC_GPIO20OUTCFG_Pos (17UL) /*!< GPIO CFGC: GPIO20OUTCFG (Bit 17) */ +#define GPIO_CFGC_GPIO20OUTCFG_Msk (0x60000UL) /*!< GPIO CFGC: GPIO20OUTCFG (Bitfield-Mask: 0x03) */ +#define GPIO_CFGC_GPIO20INCFG_Pos (16UL) /*!< GPIO CFGC: GPIO20INCFG (Bit 16) */ +#define GPIO_CFGC_GPIO20INCFG_Msk (0x10000UL) /*!< GPIO CFGC: GPIO20INCFG (Bitfield-Mask: 0x01) */ +#define GPIO_CFGC_GPIO19INTD_Pos (15UL) /*!< GPIO CFGC: GPIO19INTD (Bit 15) */ +#define GPIO_CFGC_GPIO19INTD_Msk (0x8000UL) /*!< GPIO CFGC: GPIO19INTD (Bitfield-Mask: 0x01) */ +#define GPIO_CFGC_GPIO19OUTCFG_Pos (13UL) /*!< GPIO CFGC: GPIO19OUTCFG (Bit 13) */ +#define GPIO_CFGC_GPIO19OUTCFG_Msk (0x6000UL) /*!< GPIO CFGC: GPIO19OUTCFG (Bitfield-Mask: 0x03) */ +#define GPIO_CFGC_GPIO19INCFG_Pos (12UL) /*!< GPIO CFGC: GPIO19INCFG (Bit 12) */ +#define GPIO_CFGC_GPIO19INCFG_Msk (0x1000UL) /*!< GPIO CFGC: GPIO19INCFG (Bitfield-Mask: 0x01) */ +#define GPIO_CFGC_GPIO18INTD_Pos (11UL) /*!< GPIO CFGC: GPIO18INTD (Bit 11) */ +#define GPIO_CFGC_GPIO18INTD_Msk (0x800UL) /*!< GPIO CFGC: GPIO18INTD (Bitfield-Mask: 0x01) */ +#define GPIO_CFGC_GPIO18OUTCFG_Pos (9UL) /*!< GPIO CFGC: GPIO18OUTCFG (Bit 9) */ +#define GPIO_CFGC_GPIO18OUTCFG_Msk (0x600UL) /*!< GPIO CFGC: GPIO18OUTCFG (Bitfield-Mask: 0x03) */ +#define GPIO_CFGC_GPIO18INCFG_Pos (8UL) /*!< GPIO CFGC: GPIO18INCFG (Bit 8) */ +#define GPIO_CFGC_GPIO18INCFG_Msk (0x100UL) /*!< GPIO CFGC: GPIO18INCFG (Bitfield-Mask: 0x01) */ +#define GPIO_CFGC_GPIO17INTD_Pos (7UL) /*!< GPIO CFGC: GPIO17INTD (Bit 7) */ +#define GPIO_CFGC_GPIO17INTD_Msk (0x80UL) /*!< GPIO CFGC: GPIO17INTD (Bitfield-Mask: 0x01) */ +#define GPIO_CFGC_GPIO17OUTCFG_Pos (5UL) /*!< GPIO CFGC: GPIO17OUTCFG (Bit 5) */ +#define GPIO_CFGC_GPIO17OUTCFG_Msk (0x60UL) /*!< GPIO CFGC: GPIO17OUTCFG (Bitfield-Mask: 0x03) */ +#define GPIO_CFGC_GPIO17INCFG_Pos (4UL) /*!< GPIO CFGC: GPIO17INCFG (Bit 4) */ +#define GPIO_CFGC_GPIO17INCFG_Msk (0x10UL) /*!< GPIO CFGC: GPIO17INCFG (Bitfield-Mask: 0x01) */ +#define GPIO_CFGC_GPIO16INTD_Pos (3UL) /*!< GPIO CFGC: GPIO16INTD (Bit 3) */ +#define GPIO_CFGC_GPIO16INTD_Msk (0x8UL) /*!< GPIO CFGC: GPIO16INTD (Bitfield-Mask: 0x01) */ +#define GPIO_CFGC_GPIO16OUTCFG_Pos (1UL) /*!< GPIO CFGC: GPIO16OUTCFG (Bit 1) */ +#define GPIO_CFGC_GPIO16OUTCFG_Msk (0x6UL) /*!< GPIO CFGC: GPIO16OUTCFG (Bitfield-Mask: 0x03) */ +#define GPIO_CFGC_GPIO16INCFG_Pos (0UL) /*!< GPIO CFGC: GPIO16INCFG (Bit 0) */ +#define GPIO_CFGC_GPIO16INCFG_Msk (0x1UL) /*!< GPIO CFGC: GPIO16INCFG (Bitfield-Mask: 0x01) */ +/* ========================================================= CFGD ========================================================== */ +#define GPIO_CFGD_GPIO31INTD_Pos (31UL) /*!< GPIO CFGD: GPIO31INTD (Bit 31) */ +#define GPIO_CFGD_GPIO31INTD_Msk (0x80000000UL) /*!< GPIO CFGD: GPIO31INTD (Bitfield-Mask: 0x01) */ +#define GPIO_CFGD_GPIO31OUTCFG_Pos (29UL) /*!< GPIO CFGD: GPIO31OUTCFG (Bit 29) */ +#define GPIO_CFGD_GPIO31OUTCFG_Msk (0x60000000UL) /*!< GPIO CFGD: GPIO31OUTCFG (Bitfield-Mask: 0x03) */ +#define GPIO_CFGD_GPIO31INCFG_Pos (28UL) /*!< GPIO CFGD: GPIO31INCFG (Bit 28) */ +#define GPIO_CFGD_GPIO31INCFG_Msk (0x10000000UL) /*!< GPIO CFGD: GPIO31INCFG (Bitfield-Mask: 0x01) */ +#define GPIO_CFGD_GPIO30INTD_Pos (27UL) /*!< GPIO CFGD: GPIO30INTD (Bit 27) */ +#define GPIO_CFGD_GPIO30INTD_Msk (0x8000000UL) /*!< GPIO CFGD: GPIO30INTD (Bitfield-Mask: 0x01) */ +#define GPIO_CFGD_GPIO30OUTCFG_Pos (25UL) /*!< GPIO CFGD: GPIO30OUTCFG (Bit 25) */ +#define GPIO_CFGD_GPIO30OUTCFG_Msk (0x6000000UL) /*!< GPIO CFGD: GPIO30OUTCFG (Bitfield-Mask: 0x03) */ +#define GPIO_CFGD_GPIO30INCFG_Pos (24UL) /*!< GPIO CFGD: GPIO30INCFG (Bit 24) */ +#define GPIO_CFGD_GPIO30INCFG_Msk (0x1000000UL) /*!< GPIO CFGD: GPIO30INCFG (Bitfield-Mask: 0x01) */ +#define GPIO_CFGD_GPIO29INTD_Pos (23UL) /*!< GPIO CFGD: GPIO29INTD (Bit 23) */ +#define GPIO_CFGD_GPIO29INTD_Msk (0x800000UL) /*!< GPIO CFGD: GPIO29INTD (Bitfield-Mask: 0x01) */ +#define GPIO_CFGD_GPIO29OUTCFG_Pos (21UL) /*!< GPIO CFGD: GPIO29OUTCFG (Bit 21) */ +#define GPIO_CFGD_GPIO29OUTCFG_Msk (0x600000UL) /*!< GPIO CFGD: GPIO29OUTCFG (Bitfield-Mask: 0x03) */ +#define GPIO_CFGD_GPIO29INCFG_Pos (20UL) /*!< GPIO CFGD: GPIO29INCFG (Bit 20) */ +#define GPIO_CFGD_GPIO29INCFG_Msk (0x100000UL) /*!< GPIO CFGD: GPIO29INCFG (Bitfield-Mask: 0x01) */ +#define GPIO_CFGD_GPIO28INTD_Pos (19UL) /*!< GPIO CFGD: GPIO28INTD (Bit 19) */ +#define GPIO_CFGD_GPIO28INTD_Msk (0x80000UL) /*!< GPIO CFGD: GPIO28INTD (Bitfield-Mask: 0x01) */ +#define GPIO_CFGD_GPIO28OUTCFG_Pos (17UL) /*!< GPIO CFGD: GPIO28OUTCFG (Bit 17) */ +#define GPIO_CFGD_GPIO28OUTCFG_Msk (0x60000UL) /*!< GPIO CFGD: GPIO28OUTCFG (Bitfield-Mask: 0x03) */ +#define GPIO_CFGD_GPIO28INCFG_Pos (16UL) /*!< GPIO CFGD: GPIO28INCFG (Bit 16) */ +#define GPIO_CFGD_GPIO28INCFG_Msk (0x10000UL) /*!< GPIO CFGD: GPIO28INCFG (Bitfield-Mask: 0x01) */ +#define GPIO_CFGD_GPIO27INTD_Pos (15UL) /*!< GPIO CFGD: GPIO27INTD (Bit 15) */ +#define GPIO_CFGD_GPIO27INTD_Msk (0x8000UL) /*!< GPIO CFGD: GPIO27INTD (Bitfield-Mask: 0x01) */ +#define GPIO_CFGD_GPIO27OUTCFG_Pos (13UL) /*!< GPIO CFGD: GPIO27OUTCFG (Bit 13) */ +#define GPIO_CFGD_GPIO27OUTCFG_Msk (0x6000UL) /*!< GPIO CFGD: GPIO27OUTCFG (Bitfield-Mask: 0x03) */ +#define GPIO_CFGD_GPIO27INCFG_Pos (12UL) /*!< GPIO CFGD: GPIO27INCFG (Bit 12) */ +#define GPIO_CFGD_GPIO27INCFG_Msk (0x1000UL) /*!< GPIO CFGD: GPIO27INCFG (Bitfield-Mask: 0x01) */ +#define GPIO_CFGD_GPIO26INTD_Pos (11UL) /*!< GPIO CFGD: GPIO26INTD (Bit 11) */ +#define GPIO_CFGD_GPIO26INTD_Msk (0x800UL) /*!< GPIO CFGD: GPIO26INTD (Bitfield-Mask: 0x01) */ +#define GPIO_CFGD_GPIO26OUTCFG_Pos (9UL) /*!< GPIO CFGD: GPIO26OUTCFG (Bit 9) */ +#define GPIO_CFGD_GPIO26OUTCFG_Msk (0x600UL) /*!< GPIO CFGD: GPIO26OUTCFG (Bitfield-Mask: 0x03) */ +#define GPIO_CFGD_GPIO26INCFG_Pos (8UL) /*!< GPIO CFGD: GPIO26INCFG (Bit 8) */ +#define GPIO_CFGD_GPIO26INCFG_Msk (0x100UL) /*!< GPIO CFGD: GPIO26INCFG (Bitfield-Mask: 0x01) */ +#define GPIO_CFGD_GPIO25INTD_Pos (7UL) /*!< GPIO CFGD: GPIO25INTD (Bit 7) */ +#define GPIO_CFGD_GPIO25INTD_Msk (0x80UL) /*!< GPIO CFGD: GPIO25INTD (Bitfield-Mask: 0x01) */ +#define GPIO_CFGD_GPIO25OUTCFG_Pos (5UL) /*!< GPIO CFGD: GPIO25OUTCFG (Bit 5) */ +#define GPIO_CFGD_GPIO25OUTCFG_Msk (0x60UL) /*!< GPIO CFGD: GPIO25OUTCFG (Bitfield-Mask: 0x03) */ +#define GPIO_CFGD_GPIO25INCFG_Pos (4UL) /*!< GPIO CFGD: GPIO25INCFG (Bit 4) */ +#define GPIO_CFGD_GPIO25INCFG_Msk (0x10UL) /*!< GPIO CFGD: GPIO25INCFG (Bitfield-Mask: 0x01) */ +#define GPIO_CFGD_GPIO24INTD_Pos (3UL) /*!< GPIO CFGD: GPIO24INTD (Bit 3) */ +#define GPIO_CFGD_GPIO24INTD_Msk (0x8UL) /*!< GPIO CFGD: GPIO24INTD (Bitfield-Mask: 0x01) */ +#define GPIO_CFGD_GPIO24OUTCFG_Pos (1UL) /*!< GPIO CFGD: GPIO24OUTCFG (Bit 1) */ +#define GPIO_CFGD_GPIO24OUTCFG_Msk (0x6UL) /*!< GPIO CFGD: GPIO24OUTCFG (Bitfield-Mask: 0x03) */ +#define GPIO_CFGD_GPIO24INCFG_Pos (0UL) /*!< GPIO CFGD: GPIO24INCFG (Bit 0) */ +#define GPIO_CFGD_GPIO24INCFG_Msk (0x1UL) /*!< GPIO CFGD: GPIO24INCFG (Bitfield-Mask: 0x01) */ +/* ========================================================= CFGE ========================================================== */ +#define GPIO_CFGE_GPIO39INTD_Pos (31UL) /*!< GPIO CFGE: GPIO39INTD (Bit 31) */ +#define GPIO_CFGE_GPIO39INTD_Msk (0x80000000UL) /*!< GPIO CFGE: GPIO39INTD (Bitfield-Mask: 0x01) */ +#define GPIO_CFGE_GPIO39OUTCFG_Pos (29UL) /*!< GPIO CFGE: GPIO39OUTCFG (Bit 29) */ +#define GPIO_CFGE_GPIO39OUTCFG_Msk (0x60000000UL) /*!< GPIO CFGE: GPIO39OUTCFG (Bitfield-Mask: 0x03) */ +#define GPIO_CFGE_GPIO39INCFG_Pos (28UL) /*!< GPIO CFGE: GPIO39INCFG (Bit 28) */ +#define GPIO_CFGE_GPIO39INCFG_Msk (0x10000000UL) /*!< GPIO CFGE: GPIO39INCFG (Bitfield-Mask: 0x01) */ +#define GPIO_CFGE_GPIO38INTD_Pos (27UL) /*!< GPIO CFGE: GPIO38INTD (Bit 27) */ +#define GPIO_CFGE_GPIO38INTD_Msk (0x8000000UL) /*!< GPIO CFGE: GPIO38INTD (Bitfield-Mask: 0x01) */ +#define GPIO_CFGE_GPIO38OUTCFG_Pos (25UL) /*!< GPIO CFGE: GPIO38OUTCFG (Bit 25) */ +#define GPIO_CFGE_GPIO38OUTCFG_Msk (0x6000000UL) /*!< GPIO CFGE: GPIO38OUTCFG (Bitfield-Mask: 0x03) */ +#define GPIO_CFGE_GPIO38INCFG_Pos (24UL) /*!< GPIO CFGE: GPIO38INCFG (Bit 24) */ +#define GPIO_CFGE_GPIO38INCFG_Msk (0x1000000UL) /*!< GPIO CFGE: GPIO38INCFG (Bitfield-Mask: 0x01) */ +#define GPIO_CFGE_GPIO37INTD_Pos (23UL) /*!< GPIO CFGE: GPIO37INTD (Bit 23) */ +#define GPIO_CFGE_GPIO37INTD_Msk (0x800000UL) /*!< GPIO CFGE: GPIO37INTD (Bitfield-Mask: 0x01) */ +#define GPIO_CFGE_GPIO37OUTCFG_Pos (21UL) /*!< GPIO CFGE: GPIO37OUTCFG (Bit 21) */ +#define GPIO_CFGE_GPIO37OUTCFG_Msk (0x600000UL) /*!< GPIO CFGE: GPIO37OUTCFG (Bitfield-Mask: 0x03) */ +#define GPIO_CFGE_GPIO37INCFG_Pos (20UL) /*!< GPIO CFGE: GPIO37INCFG (Bit 20) */ +#define GPIO_CFGE_GPIO37INCFG_Msk (0x100000UL) /*!< GPIO CFGE: GPIO37INCFG (Bitfield-Mask: 0x01) */ +#define GPIO_CFGE_GPIO36INTD_Pos (19UL) /*!< GPIO CFGE: GPIO36INTD (Bit 19) */ +#define GPIO_CFGE_GPIO36INTD_Msk (0x80000UL) /*!< GPIO CFGE: GPIO36INTD (Bitfield-Mask: 0x01) */ +#define GPIO_CFGE_GPIO36OUTCFG_Pos (17UL) /*!< GPIO CFGE: GPIO36OUTCFG (Bit 17) */ +#define GPIO_CFGE_GPIO36OUTCFG_Msk (0x60000UL) /*!< GPIO CFGE: GPIO36OUTCFG (Bitfield-Mask: 0x03) */ +#define GPIO_CFGE_GPIO36INCFG_Pos (16UL) /*!< GPIO CFGE: GPIO36INCFG (Bit 16) */ +#define GPIO_CFGE_GPIO36INCFG_Msk (0x10000UL) /*!< GPIO CFGE: GPIO36INCFG (Bitfield-Mask: 0x01) */ +#define GPIO_CFGE_GPIO35INTD_Pos (15UL) /*!< GPIO CFGE: GPIO35INTD (Bit 15) */ +#define GPIO_CFGE_GPIO35INTD_Msk (0x8000UL) /*!< GPIO CFGE: GPIO35INTD (Bitfield-Mask: 0x01) */ +#define GPIO_CFGE_GPIO35OUTCFG_Pos (13UL) /*!< GPIO CFGE: GPIO35OUTCFG (Bit 13) */ +#define GPIO_CFGE_GPIO35OUTCFG_Msk (0x6000UL) /*!< GPIO CFGE: GPIO35OUTCFG (Bitfield-Mask: 0x03) */ +#define GPIO_CFGE_GPIO35INCFG_Pos (12UL) /*!< GPIO CFGE: GPIO35INCFG (Bit 12) */ +#define GPIO_CFGE_GPIO35INCFG_Msk (0x1000UL) /*!< GPIO CFGE: GPIO35INCFG (Bitfield-Mask: 0x01) */ +#define GPIO_CFGE_GPIO34INTD_Pos (11UL) /*!< GPIO CFGE: GPIO34INTD (Bit 11) */ +#define GPIO_CFGE_GPIO34INTD_Msk (0x800UL) /*!< GPIO CFGE: GPIO34INTD (Bitfield-Mask: 0x01) */ +#define GPIO_CFGE_GPIO34OUTCFG_Pos (9UL) /*!< GPIO CFGE: GPIO34OUTCFG (Bit 9) */ +#define GPIO_CFGE_GPIO34OUTCFG_Msk (0x600UL) /*!< GPIO CFGE: GPIO34OUTCFG (Bitfield-Mask: 0x03) */ +#define GPIO_CFGE_GPIO34INCFG_Pos (8UL) /*!< GPIO CFGE: GPIO34INCFG (Bit 8) */ +#define GPIO_CFGE_GPIO34INCFG_Msk (0x100UL) /*!< GPIO CFGE: GPIO34INCFG (Bitfield-Mask: 0x01) */ +#define GPIO_CFGE_GPIO33INTD_Pos (7UL) /*!< GPIO CFGE: GPIO33INTD (Bit 7) */ +#define GPIO_CFGE_GPIO33INTD_Msk (0x80UL) /*!< GPIO CFGE: GPIO33INTD (Bitfield-Mask: 0x01) */ +#define GPIO_CFGE_GPIO33OUTCFG_Pos (5UL) /*!< GPIO CFGE: GPIO33OUTCFG (Bit 5) */ +#define GPIO_CFGE_GPIO33OUTCFG_Msk (0x60UL) /*!< GPIO CFGE: GPIO33OUTCFG (Bitfield-Mask: 0x03) */ +#define GPIO_CFGE_GPIO33INCFG_Pos (4UL) /*!< GPIO CFGE: GPIO33INCFG (Bit 4) */ +#define GPIO_CFGE_GPIO33INCFG_Msk (0x10UL) /*!< GPIO CFGE: GPIO33INCFG (Bitfield-Mask: 0x01) */ +#define GPIO_CFGE_GPIO32INTD_Pos (3UL) /*!< GPIO CFGE: GPIO32INTD (Bit 3) */ +#define GPIO_CFGE_GPIO32INTD_Msk (0x8UL) /*!< GPIO CFGE: GPIO32INTD (Bitfield-Mask: 0x01) */ +#define GPIO_CFGE_GPIO32OUTCFG_Pos (1UL) /*!< GPIO CFGE: GPIO32OUTCFG (Bit 1) */ +#define GPIO_CFGE_GPIO32OUTCFG_Msk (0x6UL) /*!< GPIO CFGE: GPIO32OUTCFG (Bitfield-Mask: 0x03) */ +#define GPIO_CFGE_GPIO32INCFG_Pos (0UL) /*!< GPIO CFGE: GPIO32INCFG (Bit 0) */ +#define GPIO_CFGE_GPIO32INCFG_Msk (0x1UL) /*!< GPIO CFGE: GPIO32INCFG (Bitfield-Mask: 0x01) */ +/* ========================================================= CFGF ========================================================== */ +#define GPIO_CFGF_GPIO47INTD_Pos (31UL) /*!< GPIO CFGF: GPIO47INTD (Bit 31) */ +#define GPIO_CFGF_GPIO47INTD_Msk (0x80000000UL) /*!< GPIO CFGF: GPIO47INTD (Bitfield-Mask: 0x01) */ +#define GPIO_CFGF_GPIO47OUTCFG_Pos (29UL) /*!< GPIO CFGF: GPIO47OUTCFG (Bit 29) */ +#define GPIO_CFGF_GPIO47OUTCFG_Msk (0x60000000UL) /*!< GPIO CFGF: GPIO47OUTCFG (Bitfield-Mask: 0x03) */ +#define GPIO_CFGF_GPIO47INCFG_Pos (28UL) /*!< GPIO CFGF: GPIO47INCFG (Bit 28) */ +#define GPIO_CFGF_GPIO47INCFG_Msk (0x10000000UL) /*!< GPIO CFGF: GPIO47INCFG (Bitfield-Mask: 0x01) */ +#define GPIO_CFGF_GPIO46INTD_Pos (27UL) /*!< GPIO CFGF: GPIO46INTD (Bit 27) */ +#define GPIO_CFGF_GPIO46INTD_Msk (0x8000000UL) /*!< GPIO CFGF: GPIO46INTD (Bitfield-Mask: 0x01) */ +#define GPIO_CFGF_GPIO46OUTCFG_Pos (25UL) /*!< GPIO CFGF: GPIO46OUTCFG (Bit 25) */ +#define GPIO_CFGF_GPIO46OUTCFG_Msk (0x6000000UL) /*!< GPIO CFGF: GPIO46OUTCFG (Bitfield-Mask: 0x03) */ +#define GPIO_CFGF_GPIO46INCFG_Pos (24UL) /*!< GPIO CFGF: GPIO46INCFG (Bit 24) */ +#define GPIO_CFGF_GPIO46INCFG_Msk (0x1000000UL) /*!< GPIO CFGF: GPIO46INCFG (Bitfield-Mask: 0x01) */ +#define GPIO_CFGF_GPIO45INTD_Pos (23UL) /*!< GPIO CFGF: GPIO45INTD (Bit 23) */ +#define GPIO_CFGF_GPIO45INTD_Msk (0x800000UL) /*!< GPIO CFGF: GPIO45INTD (Bitfield-Mask: 0x01) */ +#define GPIO_CFGF_GPIO45OUTCFG_Pos (21UL) /*!< GPIO CFGF: GPIO45OUTCFG (Bit 21) */ +#define GPIO_CFGF_GPIO45OUTCFG_Msk (0x600000UL) /*!< GPIO CFGF: GPIO45OUTCFG (Bitfield-Mask: 0x03) */ +#define GPIO_CFGF_GPIO45INCFG_Pos (20UL) /*!< GPIO CFGF: GPIO45INCFG (Bit 20) */ +#define GPIO_CFGF_GPIO45INCFG_Msk (0x100000UL) /*!< GPIO CFGF: GPIO45INCFG (Bitfield-Mask: 0x01) */ +#define GPIO_CFGF_GPIO44INTD_Pos (19UL) /*!< GPIO CFGF: GPIO44INTD (Bit 19) */ +#define GPIO_CFGF_GPIO44INTD_Msk (0x80000UL) /*!< GPIO CFGF: GPIO44INTD (Bitfield-Mask: 0x01) */ +#define GPIO_CFGF_GPIO44OUTCFG_Pos (17UL) /*!< GPIO CFGF: GPIO44OUTCFG (Bit 17) */ +#define GPIO_CFGF_GPIO44OUTCFG_Msk (0x60000UL) /*!< GPIO CFGF: GPIO44OUTCFG (Bitfield-Mask: 0x03) */ +#define GPIO_CFGF_GPIO44INCFG_Pos (16UL) /*!< GPIO CFGF: GPIO44INCFG (Bit 16) */ +#define GPIO_CFGF_GPIO44INCFG_Msk (0x10000UL) /*!< GPIO CFGF: GPIO44INCFG (Bitfield-Mask: 0x01) */ +#define GPIO_CFGF_GPIO43INTD_Pos (15UL) /*!< GPIO CFGF: GPIO43INTD (Bit 15) */ +#define GPIO_CFGF_GPIO43INTD_Msk (0x8000UL) /*!< GPIO CFGF: GPIO43INTD (Bitfield-Mask: 0x01) */ +#define GPIO_CFGF_GPIO43OUTCFG_Pos (13UL) /*!< GPIO CFGF: GPIO43OUTCFG (Bit 13) */ +#define GPIO_CFGF_GPIO43OUTCFG_Msk (0x6000UL) /*!< GPIO CFGF: GPIO43OUTCFG (Bitfield-Mask: 0x03) */ +#define GPIO_CFGF_GPIO43INCFG_Pos (12UL) /*!< GPIO CFGF: GPIO43INCFG (Bit 12) */ +#define GPIO_CFGF_GPIO43INCFG_Msk (0x1000UL) /*!< GPIO CFGF: GPIO43INCFG (Bitfield-Mask: 0x01) */ +#define GPIO_CFGF_GPIO42INTD_Pos (11UL) /*!< GPIO CFGF: GPIO42INTD (Bit 11) */ +#define GPIO_CFGF_GPIO42INTD_Msk (0x800UL) /*!< GPIO CFGF: GPIO42INTD (Bitfield-Mask: 0x01) */ +#define GPIO_CFGF_GPIO42OUTCFG_Pos (9UL) /*!< GPIO CFGF: GPIO42OUTCFG (Bit 9) */ +#define GPIO_CFGF_GPIO42OUTCFG_Msk (0x600UL) /*!< GPIO CFGF: GPIO42OUTCFG (Bitfield-Mask: 0x03) */ +#define GPIO_CFGF_GPIO42INCFG_Pos (8UL) /*!< GPIO CFGF: GPIO42INCFG (Bit 8) */ +#define GPIO_CFGF_GPIO42INCFG_Msk (0x100UL) /*!< GPIO CFGF: GPIO42INCFG (Bitfield-Mask: 0x01) */ +#define GPIO_CFGF_GPIO41INTD_Pos (7UL) /*!< GPIO CFGF: GPIO41INTD (Bit 7) */ +#define GPIO_CFGF_GPIO41INTD_Msk (0x80UL) /*!< GPIO CFGF: GPIO41INTD (Bitfield-Mask: 0x01) */ +#define GPIO_CFGF_GPIO41OUTCFG_Pos (5UL) /*!< GPIO CFGF: GPIO41OUTCFG (Bit 5) */ +#define GPIO_CFGF_GPIO41OUTCFG_Msk (0x60UL) /*!< GPIO CFGF: GPIO41OUTCFG (Bitfield-Mask: 0x03) */ +#define GPIO_CFGF_GPIO41INCFG_Pos (4UL) /*!< GPIO CFGF: GPIO41INCFG (Bit 4) */ +#define GPIO_CFGF_GPIO41INCFG_Msk (0x10UL) /*!< GPIO CFGF: GPIO41INCFG (Bitfield-Mask: 0x01) */ +#define GPIO_CFGF_GPIO40INTD_Pos (3UL) /*!< GPIO CFGF: GPIO40INTD (Bit 3) */ +#define GPIO_CFGF_GPIO40INTD_Msk (0x8UL) /*!< GPIO CFGF: GPIO40INTD (Bitfield-Mask: 0x01) */ +#define GPIO_CFGF_GPIO40OUTCFG_Pos (1UL) /*!< GPIO CFGF: GPIO40OUTCFG (Bit 1) */ +#define GPIO_CFGF_GPIO40OUTCFG_Msk (0x6UL) /*!< GPIO CFGF: GPIO40OUTCFG (Bitfield-Mask: 0x03) */ +#define GPIO_CFGF_GPIO40INCFG_Pos (0UL) /*!< GPIO CFGF: GPIO40INCFG (Bit 0) */ +#define GPIO_CFGF_GPIO40INCFG_Msk (0x1UL) /*!< GPIO CFGF: GPIO40INCFG (Bitfield-Mask: 0x01) */ +/* ========================================================= CFGG ========================================================== */ +#define GPIO_CFGG_GPIO49INTD_Pos (7UL) /*!< GPIO CFGG: GPIO49INTD (Bit 7) */ +#define GPIO_CFGG_GPIO49INTD_Msk (0x80UL) /*!< GPIO CFGG: GPIO49INTD (Bitfield-Mask: 0x01) */ +#define GPIO_CFGG_GPIO49OUTCFG_Pos (5UL) /*!< GPIO CFGG: GPIO49OUTCFG (Bit 5) */ +#define GPIO_CFGG_GPIO49OUTCFG_Msk (0x60UL) /*!< GPIO CFGG: GPIO49OUTCFG (Bitfield-Mask: 0x03) */ +#define GPIO_CFGG_GPIO49INCFG_Pos (4UL) /*!< GPIO CFGG: GPIO49INCFG (Bit 4) */ +#define GPIO_CFGG_GPIO49INCFG_Msk (0x10UL) /*!< GPIO CFGG: GPIO49INCFG (Bitfield-Mask: 0x01) */ +#define GPIO_CFGG_GPIO48INTD_Pos (3UL) /*!< GPIO CFGG: GPIO48INTD (Bit 3) */ +#define GPIO_CFGG_GPIO48INTD_Msk (0x8UL) /*!< GPIO CFGG: GPIO48INTD (Bitfield-Mask: 0x01) */ +#define GPIO_CFGG_GPIO48OUTCFG_Pos (1UL) /*!< GPIO CFGG: GPIO48OUTCFG (Bit 1) */ +#define GPIO_CFGG_GPIO48OUTCFG_Msk (0x6UL) /*!< GPIO CFGG: GPIO48OUTCFG (Bitfield-Mask: 0x03) */ +#define GPIO_CFGG_GPIO48INCFG_Pos (0UL) /*!< GPIO CFGG: GPIO48INCFG (Bit 0) */ +#define GPIO_CFGG_GPIO48INCFG_Msk (0x1UL) /*!< GPIO CFGG: GPIO48INCFG (Bitfield-Mask: 0x01) */ +/* ======================================================== PADKEY ========================================================= */ +#define GPIO_PADKEY_PADKEY_Pos (0UL) /*!< GPIO PADKEY: PADKEY (Bit 0) */ +#define GPIO_PADKEY_PADKEY_Msk (0xffffffffUL) /*!< GPIO PADKEY: PADKEY (Bitfield-Mask: 0xffffffff) */ +/* ========================================================== RDA ========================================================== */ +#define GPIO_RDA_RDA_Pos (0UL) /*!< GPIO RDA: RDA (Bit 0) */ +#define GPIO_RDA_RDA_Msk (0xffffffffUL) /*!< GPIO RDA: RDA (Bitfield-Mask: 0xffffffff) */ +/* ========================================================== RDB ========================================================== */ +#define GPIO_RDB_RDB_Pos (0UL) /*!< GPIO RDB: RDB (Bit 0) */ +#define GPIO_RDB_RDB_Msk (0x3ffffUL) /*!< GPIO RDB: RDB (Bitfield-Mask: 0x3ffff) */ +/* ========================================================== WTA ========================================================== */ +#define GPIO_WTA_WTA_Pos (0UL) /*!< GPIO WTA: WTA (Bit 0) */ +#define GPIO_WTA_WTA_Msk (0xffffffffUL) /*!< GPIO WTA: WTA (Bitfield-Mask: 0xffffffff) */ +/* ========================================================== WTB ========================================================== */ +#define GPIO_WTB_WTB_Pos (0UL) /*!< GPIO WTB: WTB (Bit 0) */ +#define GPIO_WTB_WTB_Msk (0x3ffffUL) /*!< GPIO WTB: WTB (Bitfield-Mask: 0x3ffff) */ +/* ========================================================= WTSA ========================================================== */ +#define GPIO_WTSA_WTSA_Pos (0UL) /*!< GPIO WTSA: WTSA (Bit 0) */ +#define GPIO_WTSA_WTSA_Msk (0xffffffffUL) /*!< GPIO WTSA: WTSA (Bitfield-Mask: 0xffffffff) */ +/* ========================================================= WTSB ========================================================== */ +#define GPIO_WTSB_WTSB_Pos (0UL) /*!< GPIO WTSB: WTSB (Bit 0) */ +#define GPIO_WTSB_WTSB_Msk (0x3ffffUL) /*!< GPIO WTSB: WTSB (Bitfield-Mask: 0x3ffff) */ +/* ========================================================= WTCA ========================================================== */ +#define GPIO_WTCA_WTCA_Pos (0UL) /*!< GPIO WTCA: WTCA (Bit 0) */ +#define GPIO_WTCA_WTCA_Msk (0xffffffffUL) /*!< GPIO WTCA: WTCA (Bitfield-Mask: 0xffffffff) */ +/* ========================================================= WTCB ========================================================== */ +#define GPIO_WTCB_WTCB_Pos (0UL) /*!< GPIO WTCB: WTCB (Bit 0) */ +#define GPIO_WTCB_WTCB_Msk (0x3ffffUL) /*!< GPIO WTCB: WTCB (Bitfield-Mask: 0x3ffff) */ +/* ========================================================== ENA ========================================================== */ +#define GPIO_ENA_ENA_Pos (0UL) /*!< GPIO ENA: ENA (Bit 0) */ +#define GPIO_ENA_ENA_Msk (0xffffffffUL) /*!< GPIO ENA: ENA (Bitfield-Mask: 0xffffffff) */ +/* ========================================================== ENB ========================================================== */ +#define GPIO_ENB_ENB_Pos (0UL) /*!< GPIO ENB: ENB (Bit 0) */ +#define GPIO_ENB_ENB_Msk (0x3ffffUL) /*!< GPIO ENB: ENB (Bitfield-Mask: 0x3ffff) */ +/* ========================================================= ENSA ========================================================== */ +#define GPIO_ENSA_ENSA_Pos (0UL) /*!< GPIO ENSA: ENSA (Bit 0) */ +#define GPIO_ENSA_ENSA_Msk (0xffffffffUL) /*!< GPIO ENSA: ENSA (Bitfield-Mask: 0xffffffff) */ +/* ========================================================= ENSB ========================================================== */ +#define GPIO_ENSB_ENSB_Pos (0UL) /*!< GPIO ENSB: ENSB (Bit 0) */ +#define GPIO_ENSB_ENSB_Msk (0x3ffffUL) /*!< GPIO ENSB: ENSB (Bitfield-Mask: 0x3ffff) */ +/* ========================================================= ENCA ========================================================== */ +#define GPIO_ENCA_ENCA_Pos (0UL) /*!< GPIO ENCA: ENCA (Bit 0) */ +#define GPIO_ENCA_ENCA_Msk (0xffffffffUL) /*!< GPIO ENCA: ENCA (Bitfield-Mask: 0xffffffff) */ +/* ========================================================= ENCB ========================================================== */ +#define GPIO_ENCB_ENCB_Pos (0UL) /*!< GPIO ENCB: ENCB (Bit 0) */ +#define GPIO_ENCB_ENCB_Msk (0x3ffffUL) /*!< GPIO ENCB: ENCB (Bitfield-Mask: 0x3ffff) */ +/* ======================================================== STMRCAP ======================================================== */ +#define GPIO_STMRCAP_STPOL3_Pos (30UL) /*!< GPIO STMRCAP: STPOL3 (Bit 30) */ +#define GPIO_STMRCAP_STPOL3_Msk (0x40000000UL) /*!< GPIO STMRCAP: STPOL3 (Bitfield-Mask: 0x01) */ +#define GPIO_STMRCAP_STSEL3_Pos (24UL) /*!< GPIO STMRCAP: STSEL3 (Bit 24) */ +#define GPIO_STMRCAP_STSEL3_Msk (0x3f000000UL) /*!< GPIO STMRCAP: STSEL3 (Bitfield-Mask: 0x3f) */ +#define GPIO_STMRCAP_STPOL2_Pos (22UL) /*!< GPIO STMRCAP: STPOL2 (Bit 22) */ +#define GPIO_STMRCAP_STPOL2_Msk (0x400000UL) /*!< GPIO STMRCAP: STPOL2 (Bitfield-Mask: 0x01) */ +#define GPIO_STMRCAP_STSEL2_Pos (16UL) /*!< GPIO STMRCAP: STSEL2 (Bit 16) */ +#define GPIO_STMRCAP_STSEL2_Msk (0x3f0000UL) /*!< GPIO STMRCAP: STSEL2 (Bitfield-Mask: 0x3f) */ +#define GPIO_STMRCAP_STPOL1_Pos (14UL) /*!< GPIO STMRCAP: STPOL1 (Bit 14) */ +#define GPIO_STMRCAP_STPOL1_Msk (0x4000UL) /*!< GPIO STMRCAP: STPOL1 (Bitfield-Mask: 0x01) */ +#define GPIO_STMRCAP_STSEL1_Pos (8UL) /*!< GPIO STMRCAP: STSEL1 (Bit 8) */ +#define GPIO_STMRCAP_STSEL1_Msk (0x3f00UL) /*!< GPIO STMRCAP: STSEL1 (Bitfield-Mask: 0x3f) */ +#define GPIO_STMRCAP_STPOL0_Pos (6UL) /*!< GPIO STMRCAP: STPOL0 (Bit 6) */ +#define GPIO_STMRCAP_STPOL0_Msk (0x40UL) /*!< GPIO STMRCAP: STPOL0 (Bitfield-Mask: 0x01) */ +#define GPIO_STMRCAP_STSEL0_Pos (0UL) /*!< GPIO STMRCAP: STSEL0 (Bit 0) */ +#define GPIO_STMRCAP_STSEL0_Msk (0x3fUL) /*!< GPIO STMRCAP: STSEL0 (Bitfield-Mask: 0x3f) */ +/* ======================================================== IOM0IRQ ======================================================== */ +#define GPIO_IOM0IRQ_IOM0IRQ_Pos (0UL) /*!< GPIO IOM0IRQ: IOM0IRQ (Bit 0) */ +#define GPIO_IOM0IRQ_IOM0IRQ_Msk (0x3fUL) /*!< GPIO IOM0IRQ: IOM0IRQ (Bitfield-Mask: 0x3f) */ +/* ======================================================== IOM1IRQ ======================================================== */ +#define GPIO_IOM1IRQ_IOM1IRQ_Pos (0UL) /*!< GPIO IOM1IRQ: IOM1IRQ (Bit 0) */ +#define GPIO_IOM1IRQ_IOM1IRQ_Msk (0x3fUL) /*!< GPIO IOM1IRQ: IOM1IRQ (Bitfield-Mask: 0x3f) */ +/* ======================================================== IOM2IRQ ======================================================== */ +#define GPIO_IOM2IRQ_IOM2IRQ_Pos (0UL) /*!< GPIO IOM2IRQ: IOM2IRQ (Bit 0) */ +#define GPIO_IOM2IRQ_IOM2IRQ_Msk (0x3fUL) /*!< GPIO IOM2IRQ: IOM2IRQ (Bitfield-Mask: 0x3f) */ +/* ======================================================== IOM3IRQ ======================================================== */ +#define GPIO_IOM3IRQ_IOM3IRQ_Pos (0UL) /*!< GPIO IOM3IRQ: IOM3IRQ (Bit 0) */ +#define GPIO_IOM3IRQ_IOM3IRQ_Msk (0x3fUL) /*!< GPIO IOM3IRQ: IOM3IRQ (Bitfield-Mask: 0x3f) */ +/* ======================================================== IOM4IRQ ======================================================== */ +#define GPIO_IOM4IRQ_IOM4IRQ_Pos (0UL) /*!< GPIO IOM4IRQ: IOM4IRQ (Bit 0) */ +#define GPIO_IOM4IRQ_IOM4IRQ_Msk (0x3fUL) /*!< GPIO IOM4IRQ: IOM4IRQ (Bitfield-Mask: 0x3f) */ +/* ======================================================== IOM5IRQ ======================================================== */ +#define GPIO_IOM5IRQ_IOM5IRQ_Pos (0UL) /*!< GPIO IOM5IRQ: IOM5IRQ (Bit 0) */ +#define GPIO_IOM5IRQ_IOM5IRQ_Msk (0x3fUL) /*!< GPIO IOM5IRQ: IOM5IRQ (Bitfield-Mask: 0x3f) */ +/* ======================================================= BLEIFIRQ ======================================================== */ +#define GPIO_BLEIFIRQ_BLEIFIRQ_Pos (0UL) /*!< GPIO BLEIFIRQ: BLEIFIRQ (Bit 0) */ +#define GPIO_BLEIFIRQ_BLEIFIRQ_Msk (0x3fUL) /*!< GPIO BLEIFIRQ: BLEIFIRQ (Bitfield-Mask: 0x3f) */ +/* ======================================================== GPIOOBS ======================================================== */ +#define GPIO_GPIOOBS_OBS_DATA_Pos (0UL) /*!< GPIO GPIOOBS: OBS_DATA (Bit 0) */ +#define GPIO_GPIOOBS_OBS_DATA_Msk (0xffffUL) /*!< GPIO GPIOOBS: OBS_DATA (Bitfield-Mask: 0xffff) */ +/* ====================================================== ALTPADCFGA ======================================================= */ +#define GPIO_ALTPADCFGA_PAD3_SR_Pos (28UL) /*!< GPIO ALTPADCFGA: PAD3_SR (Bit 28) */ +#define GPIO_ALTPADCFGA_PAD3_SR_Msk (0x10000000UL) /*!< GPIO ALTPADCFGA: PAD3_SR (Bitfield-Mask: 0x01) */ +#define GPIO_ALTPADCFGA_PAD3_DS1_Pos (24UL) /*!< GPIO ALTPADCFGA: PAD3_DS1 (Bit 24) */ +#define GPIO_ALTPADCFGA_PAD3_DS1_Msk (0x1000000UL) /*!< GPIO ALTPADCFGA: PAD3_DS1 (Bitfield-Mask: 0x01) */ +#define GPIO_ALTPADCFGA_PAD2_SR_Pos (20UL) /*!< GPIO ALTPADCFGA: PAD2_SR (Bit 20) */ +#define GPIO_ALTPADCFGA_PAD2_SR_Msk (0x100000UL) /*!< GPIO ALTPADCFGA: PAD2_SR (Bitfield-Mask: 0x01) */ +#define GPIO_ALTPADCFGA_PAD2_DS1_Pos (16UL) /*!< GPIO ALTPADCFGA: PAD2_DS1 (Bit 16) */ +#define GPIO_ALTPADCFGA_PAD2_DS1_Msk (0x10000UL) /*!< GPIO ALTPADCFGA: PAD2_DS1 (Bitfield-Mask: 0x01) */ +#define GPIO_ALTPADCFGA_PAD1_SR_Pos (12UL) /*!< GPIO ALTPADCFGA: PAD1_SR (Bit 12) */ +#define GPIO_ALTPADCFGA_PAD1_SR_Msk (0x1000UL) /*!< GPIO ALTPADCFGA: PAD1_SR (Bitfield-Mask: 0x01) */ +#define GPIO_ALTPADCFGA_PAD1_DS1_Pos (8UL) /*!< GPIO ALTPADCFGA: PAD1_DS1 (Bit 8) */ +#define GPIO_ALTPADCFGA_PAD1_DS1_Msk (0x100UL) /*!< GPIO ALTPADCFGA: PAD1_DS1 (Bitfield-Mask: 0x01) */ +#define GPIO_ALTPADCFGA_PAD0_SR_Pos (4UL) /*!< GPIO ALTPADCFGA: PAD0_SR (Bit 4) */ +#define GPIO_ALTPADCFGA_PAD0_SR_Msk (0x10UL) /*!< GPIO ALTPADCFGA: PAD0_SR (Bitfield-Mask: 0x01) */ +#define GPIO_ALTPADCFGA_PAD0_DS1_Pos (0UL) /*!< GPIO ALTPADCFGA: PAD0_DS1 (Bit 0) */ +#define GPIO_ALTPADCFGA_PAD0_DS1_Msk (0x1UL) /*!< GPIO ALTPADCFGA: PAD0_DS1 (Bitfield-Mask: 0x01) */ +/* ====================================================== ALTPADCFGB ======================================================= */ +#define GPIO_ALTPADCFGB_PAD7_SR_Pos (28UL) /*!< GPIO ALTPADCFGB: PAD7_SR (Bit 28) */ +#define GPIO_ALTPADCFGB_PAD7_SR_Msk (0x10000000UL) /*!< GPIO ALTPADCFGB: PAD7_SR (Bitfield-Mask: 0x01) */ +#define GPIO_ALTPADCFGB_PAD7_DS1_Pos (24UL) /*!< GPIO ALTPADCFGB: PAD7_DS1 (Bit 24) */ +#define GPIO_ALTPADCFGB_PAD7_DS1_Msk (0x1000000UL) /*!< GPIO ALTPADCFGB: PAD7_DS1 (Bitfield-Mask: 0x01) */ +#define GPIO_ALTPADCFGB_PAD6_SR_Pos (20UL) /*!< GPIO ALTPADCFGB: PAD6_SR (Bit 20) */ +#define GPIO_ALTPADCFGB_PAD6_SR_Msk (0x100000UL) /*!< GPIO ALTPADCFGB: PAD6_SR (Bitfield-Mask: 0x01) */ +#define GPIO_ALTPADCFGB_PAD6_DS1_Pos (16UL) /*!< GPIO ALTPADCFGB: PAD6_DS1 (Bit 16) */ +#define GPIO_ALTPADCFGB_PAD6_DS1_Msk (0x10000UL) /*!< GPIO ALTPADCFGB: PAD6_DS1 (Bitfield-Mask: 0x01) */ +#define GPIO_ALTPADCFGB_PAD5_SR_Pos (12UL) /*!< GPIO ALTPADCFGB: PAD5_SR (Bit 12) */ +#define GPIO_ALTPADCFGB_PAD5_SR_Msk (0x1000UL) /*!< GPIO ALTPADCFGB: PAD5_SR (Bitfield-Mask: 0x01) */ +#define GPIO_ALTPADCFGB_PAD5_DS1_Pos (8UL) /*!< GPIO ALTPADCFGB: PAD5_DS1 (Bit 8) */ +#define GPIO_ALTPADCFGB_PAD5_DS1_Msk (0x100UL) /*!< GPIO ALTPADCFGB: PAD5_DS1 (Bitfield-Mask: 0x01) */ +#define GPIO_ALTPADCFGB_PAD4_SR_Pos (4UL) /*!< GPIO ALTPADCFGB: PAD4_SR (Bit 4) */ +#define GPIO_ALTPADCFGB_PAD4_SR_Msk (0x10UL) /*!< GPIO ALTPADCFGB: PAD4_SR (Bitfield-Mask: 0x01) */ +#define GPIO_ALTPADCFGB_PAD4_DS1_Pos (0UL) /*!< GPIO ALTPADCFGB: PAD4_DS1 (Bit 0) */ +#define GPIO_ALTPADCFGB_PAD4_DS1_Msk (0x1UL) /*!< GPIO ALTPADCFGB: PAD4_DS1 (Bitfield-Mask: 0x01) */ +/* ====================================================== ALTPADCFGC ======================================================= */ +#define GPIO_ALTPADCFGC_PAD11_SR_Pos (28UL) /*!< GPIO ALTPADCFGC: PAD11_SR (Bit 28) */ +#define GPIO_ALTPADCFGC_PAD11_SR_Msk (0x10000000UL) /*!< GPIO ALTPADCFGC: PAD11_SR (Bitfield-Mask: 0x01) */ +#define GPIO_ALTPADCFGC_PAD11_DS1_Pos (24UL) /*!< GPIO ALTPADCFGC: PAD11_DS1 (Bit 24) */ +#define GPIO_ALTPADCFGC_PAD11_DS1_Msk (0x1000000UL) /*!< GPIO ALTPADCFGC: PAD11_DS1 (Bitfield-Mask: 0x01) */ +#define GPIO_ALTPADCFGC_PAD10_SR_Pos (20UL) /*!< GPIO ALTPADCFGC: PAD10_SR (Bit 20) */ +#define GPIO_ALTPADCFGC_PAD10_SR_Msk (0x100000UL) /*!< GPIO ALTPADCFGC: PAD10_SR (Bitfield-Mask: 0x01) */ +#define GPIO_ALTPADCFGC_PAD10_DS1_Pos (16UL) /*!< GPIO ALTPADCFGC: PAD10_DS1 (Bit 16) */ +#define GPIO_ALTPADCFGC_PAD10_DS1_Msk (0x10000UL) /*!< GPIO ALTPADCFGC: PAD10_DS1 (Bitfield-Mask: 0x01) */ +#define GPIO_ALTPADCFGC_PAD9_SR_Pos (12UL) /*!< GPIO ALTPADCFGC: PAD9_SR (Bit 12) */ +#define GPIO_ALTPADCFGC_PAD9_SR_Msk (0x1000UL) /*!< GPIO ALTPADCFGC: PAD9_SR (Bitfield-Mask: 0x01) */ +#define GPIO_ALTPADCFGC_PAD9_DS1_Pos (8UL) /*!< GPIO ALTPADCFGC: PAD9_DS1 (Bit 8) */ +#define GPIO_ALTPADCFGC_PAD9_DS1_Msk (0x100UL) /*!< GPIO ALTPADCFGC: PAD9_DS1 (Bitfield-Mask: 0x01) */ +#define GPIO_ALTPADCFGC_PAD8_SR_Pos (4UL) /*!< GPIO ALTPADCFGC: PAD8_SR (Bit 4) */ +#define GPIO_ALTPADCFGC_PAD8_SR_Msk (0x10UL) /*!< GPIO ALTPADCFGC: PAD8_SR (Bitfield-Mask: 0x01) */ +#define GPIO_ALTPADCFGC_PAD8_DS1_Pos (0UL) /*!< GPIO ALTPADCFGC: PAD8_DS1 (Bit 0) */ +#define GPIO_ALTPADCFGC_PAD8_DS1_Msk (0x1UL) /*!< GPIO ALTPADCFGC: PAD8_DS1 (Bitfield-Mask: 0x01) */ +/* ====================================================== ALTPADCFGD ======================================================= */ +#define GPIO_ALTPADCFGD_PAD15_SR_Pos (28UL) /*!< GPIO ALTPADCFGD: PAD15_SR (Bit 28) */ +#define GPIO_ALTPADCFGD_PAD15_SR_Msk (0x10000000UL) /*!< GPIO ALTPADCFGD: PAD15_SR (Bitfield-Mask: 0x01) */ +#define GPIO_ALTPADCFGD_PAD15_DS1_Pos (24UL) /*!< GPIO ALTPADCFGD: PAD15_DS1 (Bit 24) */ +#define GPIO_ALTPADCFGD_PAD15_DS1_Msk (0x1000000UL) /*!< GPIO ALTPADCFGD: PAD15_DS1 (Bitfield-Mask: 0x01) */ +#define GPIO_ALTPADCFGD_PAD14_SR_Pos (20UL) /*!< GPIO ALTPADCFGD: PAD14_SR (Bit 20) */ +#define GPIO_ALTPADCFGD_PAD14_SR_Msk (0x100000UL) /*!< GPIO ALTPADCFGD: PAD14_SR (Bitfield-Mask: 0x01) */ +#define GPIO_ALTPADCFGD_PAD14_DS1_Pos (16UL) /*!< GPIO ALTPADCFGD: PAD14_DS1 (Bit 16) */ +#define GPIO_ALTPADCFGD_PAD14_DS1_Msk (0x10000UL) /*!< GPIO ALTPADCFGD: PAD14_DS1 (Bitfield-Mask: 0x01) */ +#define GPIO_ALTPADCFGD_PAD13_SR_Pos (12UL) /*!< GPIO ALTPADCFGD: PAD13_SR (Bit 12) */ +#define GPIO_ALTPADCFGD_PAD13_SR_Msk (0x1000UL) /*!< GPIO ALTPADCFGD: PAD13_SR (Bitfield-Mask: 0x01) */ +#define GPIO_ALTPADCFGD_PAD13_DS1_Pos (8UL) /*!< GPIO ALTPADCFGD: PAD13_DS1 (Bit 8) */ +#define GPIO_ALTPADCFGD_PAD13_DS1_Msk (0x100UL) /*!< GPIO ALTPADCFGD: PAD13_DS1 (Bitfield-Mask: 0x01) */ +#define GPIO_ALTPADCFGD_PAD12_SR_Pos (4UL) /*!< GPIO ALTPADCFGD: PAD12_SR (Bit 4) */ +#define GPIO_ALTPADCFGD_PAD12_SR_Msk (0x10UL) /*!< GPIO ALTPADCFGD: PAD12_SR (Bitfield-Mask: 0x01) */ +#define GPIO_ALTPADCFGD_PAD12_DS1_Pos (0UL) /*!< GPIO ALTPADCFGD: PAD12_DS1 (Bit 0) */ +#define GPIO_ALTPADCFGD_PAD12_DS1_Msk (0x1UL) /*!< GPIO ALTPADCFGD: PAD12_DS1 (Bitfield-Mask: 0x01) */ +/* ====================================================== ALTPADCFGE ======================================================= */ +#define GPIO_ALTPADCFGE_PAD19_SR_Pos (28UL) /*!< GPIO ALTPADCFGE: PAD19_SR (Bit 28) */ +#define GPIO_ALTPADCFGE_PAD19_SR_Msk (0x10000000UL) /*!< GPIO ALTPADCFGE: PAD19_SR (Bitfield-Mask: 0x01) */ +#define GPIO_ALTPADCFGE_PAD19_DS1_Pos (24UL) /*!< GPIO ALTPADCFGE: PAD19_DS1 (Bit 24) */ +#define GPIO_ALTPADCFGE_PAD19_DS1_Msk (0x1000000UL) /*!< GPIO ALTPADCFGE: PAD19_DS1 (Bitfield-Mask: 0x01) */ +#define GPIO_ALTPADCFGE_PAD18_SR_Pos (20UL) /*!< GPIO ALTPADCFGE: PAD18_SR (Bit 20) */ +#define GPIO_ALTPADCFGE_PAD18_SR_Msk (0x100000UL) /*!< GPIO ALTPADCFGE: PAD18_SR (Bitfield-Mask: 0x01) */ +#define GPIO_ALTPADCFGE_PAD18_DS1_Pos (16UL) /*!< GPIO ALTPADCFGE: PAD18_DS1 (Bit 16) */ +#define GPIO_ALTPADCFGE_PAD18_DS1_Msk (0x10000UL) /*!< GPIO ALTPADCFGE: PAD18_DS1 (Bitfield-Mask: 0x01) */ +#define GPIO_ALTPADCFGE_PAD17_SR_Pos (12UL) /*!< GPIO ALTPADCFGE: PAD17_SR (Bit 12) */ +#define GPIO_ALTPADCFGE_PAD17_SR_Msk (0x1000UL) /*!< GPIO ALTPADCFGE: PAD17_SR (Bitfield-Mask: 0x01) */ +#define GPIO_ALTPADCFGE_PAD17_DS1_Pos (8UL) /*!< GPIO ALTPADCFGE: PAD17_DS1 (Bit 8) */ +#define GPIO_ALTPADCFGE_PAD17_DS1_Msk (0x100UL) /*!< GPIO ALTPADCFGE: PAD17_DS1 (Bitfield-Mask: 0x01) */ +#define GPIO_ALTPADCFGE_PAD16_SR_Pos (4UL) /*!< GPIO ALTPADCFGE: PAD16_SR (Bit 4) */ +#define GPIO_ALTPADCFGE_PAD16_SR_Msk (0x10UL) /*!< GPIO ALTPADCFGE: PAD16_SR (Bitfield-Mask: 0x01) */ +#define GPIO_ALTPADCFGE_PAD16_DS1_Pos (0UL) /*!< GPIO ALTPADCFGE: PAD16_DS1 (Bit 0) */ +#define GPIO_ALTPADCFGE_PAD16_DS1_Msk (0x1UL) /*!< GPIO ALTPADCFGE: PAD16_DS1 (Bitfield-Mask: 0x01) */ +/* ====================================================== ALTPADCFGF ======================================================= */ +#define GPIO_ALTPADCFGF_PAD23_SR_Pos (28UL) /*!< GPIO ALTPADCFGF: PAD23_SR (Bit 28) */ +#define GPIO_ALTPADCFGF_PAD23_SR_Msk (0x10000000UL) /*!< GPIO ALTPADCFGF: PAD23_SR (Bitfield-Mask: 0x01) */ +#define GPIO_ALTPADCFGF_PAD23_DS1_Pos (24UL) /*!< GPIO ALTPADCFGF: PAD23_DS1 (Bit 24) */ +#define GPIO_ALTPADCFGF_PAD23_DS1_Msk (0x1000000UL) /*!< GPIO ALTPADCFGF: PAD23_DS1 (Bitfield-Mask: 0x01) */ +#define GPIO_ALTPADCFGF_PAD22_SR_Pos (20UL) /*!< GPIO ALTPADCFGF: PAD22_SR (Bit 20) */ +#define GPIO_ALTPADCFGF_PAD22_SR_Msk (0x100000UL) /*!< GPIO ALTPADCFGF: PAD22_SR (Bitfield-Mask: 0x01) */ +#define GPIO_ALTPADCFGF_PAD22_DS1_Pos (16UL) /*!< GPIO ALTPADCFGF: PAD22_DS1 (Bit 16) */ +#define GPIO_ALTPADCFGF_PAD22_DS1_Msk (0x10000UL) /*!< GPIO ALTPADCFGF: PAD22_DS1 (Bitfield-Mask: 0x01) */ +#define GPIO_ALTPADCFGF_PAD21_SR_Pos (12UL) /*!< GPIO ALTPADCFGF: PAD21_SR (Bit 12) */ +#define GPIO_ALTPADCFGF_PAD21_SR_Msk (0x1000UL) /*!< GPIO ALTPADCFGF: PAD21_SR (Bitfield-Mask: 0x01) */ +#define GPIO_ALTPADCFGF_PAD21_DS1_Pos (8UL) /*!< GPIO ALTPADCFGF: PAD21_DS1 (Bit 8) */ +#define GPIO_ALTPADCFGF_PAD21_DS1_Msk (0x100UL) /*!< GPIO ALTPADCFGF: PAD21_DS1 (Bitfield-Mask: 0x01) */ +#define GPIO_ALTPADCFGF_PAD20_SR_Pos (4UL) /*!< GPIO ALTPADCFGF: PAD20_SR (Bit 4) */ +#define GPIO_ALTPADCFGF_PAD20_SR_Msk (0x10UL) /*!< GPIO ALTPADCFGF: PAD20_SR (Bitfield-Mask: 0x01) */ +#define GPIO_ALTPADCFGF_PAD20_DS1_Pos (0UL) /*!< GPIO ALTPADCFGF: PAD20_DS1 (Bit 0) */ +#define GPIO_ALTPADCFGF_PAD20_DS1_Msk (0x1UL) /*!< GPIO ALTPADCFGF: PAD20_DS1 (Bitfield-Mask: 0x01) */ +/* ====================================================== ALTPADCFGG ======================================================= */ +#define GPIO_ALTPADCFGG_PAD27_SR_Pos (28UL) /*!< GPIO ALTPADCFGG: PAD27_SR (Bit 28) */ +#define GPIO_ALTPADCFGG_PAD27_SR_Msk (0x10000000UL) /*!< GPIO ALTPADCFGG: PAD27_SR (Bitfield-Mask: 0x01) */ +#define GPIO_ALTPADCFGG_PAD27_DS1_Pos (24UL) /*!< GPIO ALTPADCFGG: PAD27_DS1 (Bit 24) */ +#define GPIO_ALTPADCFGG_PAD27_DS1_Msk (0x1000000UL) /*!< GPIO ALTPADCFGG: PAD27_DS1 (Bitfield-Mask: 0x01) */ +#define GPIO_ALTPADCFGG_PAD26_SR_Pos (20UL) /*!< GPIO ALTPADCFGG: PAD26_SR (Bit 20) */ +#define GPIO_ALTPADCFGG_PAD26_SR_Msk (0x100000UL) /*!< GPIO ALTPADCFGG: PAD26_SR (Bitfield-Mask: 0x01) */ +#define GPIO_ALTPADCFGG_PAD26_DS1_Pos (16UL) /*!< GPIO ALTPADCFGG: PAD26_DS1 (Bit 16) */ +#define GPIO_ALTPADCFGG_PAD26_DS1_Msk (0x10000UL) /*!< GPIO ALTPADCFGG: PAD26_DS1 (Bitfield-Mask: 0x01) */ +#define GPIO_ALTPADCFGG_PAD25_SR_Pos (12UL) /*!< GPIO ALTPADCFGG: PAD25_SR (Bit 12) */ +#define GPIO_ALTPADCFGG_PAD25_SR_Msk (0x1000UL) /*!< GPIO ALTPADCFGG: PAD25_SR (Bitfield-Mask: 0x01) */ +#define GPIO_ALTPADCFGG_PAD25_DS1_Pos (8UL) /*!< GPIO ALTPADCFGG: PAD25_DS1 (Bit 8) */ +#define GPIO_ALTPADCFGG_PAD25_DS1_Msk (0x100UL) /*!< GPIO ALTPADCFGG: PAD25_DS1 (Bitfield-Mask: 0x01) */ +#define GPIO_ALTPADCFGG_PAD24_SR_Pos (4UL) /*!< GPIO ALTPADCFGG: PAD24_SR (Bit 4) */ +#define GPIO_ALTPADCFGG_PAD24_SR_Msk (0x10UL) /*!< GPIO ALTPADCFGG: PAD24_SR (Bitfield-Mask: 0x01) */ +#define GPIO_ALTPADCFGG_PAD24_DS1_Pos (0UL) /*!< GPIO ALTPADCFGG: PAD24_DS1 (Bit 0) */ +#define GPIO_ALTPADCFGG_PAD24_DS1_Msk (0x1UL) /*!< GPIO ALTPADCFGG: PAD24_DS1 (Bitfield-Mask: 0x01) */ +/* ====================================================== ALTPADCFGH ======================================================= */ +#define GPIO_ALTPADCFGH_PAD31_SR_Pos (28UL) /*!< GPIO ALTPADCFGH: PAD31_SR (Bit 28) */ +#define GPIO_ALTPADCFGH_PAD31_SR_Msk (0x10000000UL) /*!< GPIO ALTPADCFGH: PAD31_SR (Bitfield-Mask: 0x01) */ +#define GPIO_ALTPADCFGH_PAD31_DS1_Pos (24UL) /*!< GPIO ALTPADCFGH: PAD31_DS1 (Bit 24) */ +#define GPIO_ALTPADCFGH_PAD31_DS1_Msk (0x1000000UL) /*!< GPIO ALTPADCFGH: PAD31_DS1 (Bitfield-Mask: 0x01) */ +#define GPIO_ALTPADCFGH_PAD30_SR_Pos (20UL) /*!< GPIO ALTPADCFGH: PAD30_SR (Bit 20) */ +#define GPIO_ALTPADCFGH_PAD30_SR_Msk (0x100000UL) /*!< GPIO ALTPADCFGH: PAD30_SR (Bitfield-Mask: 0x01) */ +#define GPIO_ALTPADCFGH_PAD30_DS1_Pos (16UL) /*!< GPIO ALTPADCFGH: PAD30_DS1 (Bit 16) */ +#define GPIO_ALTPADCFGH_PAD30_DS1_Msk (0x10000UL) /*!< GPIO ALTPADCFGH: PAD30_DS1 (Bitfield-Mask: 0x01) */ +#define GPIO_ALTPADCFGH_PAD29_SR_Pos (12UL) /*!< GPIO ALTPADCFGH: PAD29_SR (Bit 12) */ +#define GPIO_ALTPADCFGH_PAD29_SR_Msk (0x1000UL) /*!< GPIO ALTPADCFGH: PAD29_SR (Bitfield-Mask: 0x01) */ +#define GPIO_ALTPADCFGH_PAD29_DS1_Pos (8UL) /*!< GPIO ALTPADCFGH: PAD29_DS1 (Bit 8) */ +#define GPIO_ALTPADCFGH_PAD29_DS1_Msk (0x100UL) /*!< GPIO ALTPADCFGH: PAD29_DS1 (Bitfield-Mask: 0x01) */ +#define GPIO_ALTPADCFGH_PAD28_SR_Pos (4UL) /*!< GPIO ALTPADCFGH: PAD28_SR (Bit 4) */ +#define GPIO_ALTPADCFGH_PAD28_SR_Msk (0x10UL) /*!< GPIO ALTPADCFGH: PAD28_SR (Bitfield-Mask: 0x01) */ +#define GPIO_ALTPADCFGH_PAD28_DS1_Pos (0UL) /*!< GPIO ALTPADCFGH: PAD28_DS1 (Bit 0) */ +#define GPIO_ALTPADCFGH_PAD28_DS1_Msk (0x1UL) /*!< GPIO ALTPADCFGH: PAD28_DS1 (Bitfield-Mask: 0x01) */ +/* ====================================================== ALTPADCFGI ======================================================= */ +#define GPIO_ALTPADCFGI_PAD35_SR_Pos (28UL) /*!< GPIO ALTPADCFGI: PAD35_SR (Bit 28) */ +#define GPIO_ALTPADCFGI_PAD35_SR_Msk (0x10000000UL) /*!< GPIO ALTPADCFGI: PAD35_SR (Bitfield-Mask: 0x01) */ +#define GPIO_ALTPADCFGI_PAD35_DS1_Pos (24UL) /*!< GPIO ALTPADCFGI: PAD35_DS1 (Bit 24) */ +#define GPIO_ALTPADCFGI_PAD35_DS1_Msk (0x1000000UL) /*!< GPIO ALTPADCFGI: PAD35_DS1 (Bitfield-Mask: 0x01) */ +#define GPIO_ALTPADCFGI_PAD34_SR_Pos (20UL) /*!< GPIO ALTPADCFGI: PAD34_SR (Bit 20) */ +#define GPIO_ALTPADCFGI_PAD34_SR_Msk (0x100000UL) /*!< GPIO ALTPADCFGI: PAD34_SR (Bitfield-Mask: 0x01) */ +#define GPIO_ALTPADCFGI_PAD34_DS1_Pos (16UL) /*!< GPIO ALTPADCFGI: PAD34_DS1 (Bit 16) */ +#define GPIO_ALTPADCFGI_PAD34_DS1_Msk (0x10000UL) /*!< GPIO ALTPADCFGI: PAD34_DS1 (Bitfield-Mask: 0x01) */ +#define GPIO_ALTPADCFGI_PAD33_SR_Pos (12UL) /*!< GPIO ALTPADCFGI: PAD33_SR (Bit 12) */ +#define GPIO_ALTPADCFGI_PAD33_SR_Msk (0x1000UL) /*!< GPIO ALTPADCFGI: PAD33_SR (Bitfield-Mask: 0x01) */ +#define GPIO_ALTPADCFGI_PAD33_DS1_Pos (8UL) /*!< GPIO ALTPADCFGI: PAD33_DS1 (Bit 8) */ +#define GPIO_ALTPADCFGI_PAD33_DS1_Msk (0x100UL) /*!< GPIO ALTPADCFGI: PAD33_DS1 (Bitfield-Mask: 0x01) */ +#define GPIO_ALTPADCFGI_PAD32_SR_Pos (4UL) /*!< GPIO ALTPADCFGI: PAD32_SR (Bit 4) */ +#define GPIO_ALTPADCFGI_PAD32_SR_Msk (0x10UL) /*!< GPIO ALTPADCFGI: PAD32_SR (Bitfield-Mask: 0x01) */ +#define GPIO_ALTPADCFGI_PAD32_DS1_Pos (0UL) /*!< GPIO ALTPADCFGI: PAD32_DS1 (Bit 0) */ +#define GPIO_ALTPADCFGI_PAD32_DS1_Msk (0x1UL) /*!< GPIO ALTPADCFGI: PAD32_DS1 (Bitfield-Mask: 0x01) */ +/* ====================================================== ALTPADCFGJ ======================================================= */ +#define GPIO_ALTPADCFGJ_PAD39_SR_Pos (28UL) /*!< GPIO ALTPADCFGJ: PAD39_SR (Bit 28) */ +#define GPIO_ALTPADCFGJ_PAD39_SR_Msk (0x10000000UL) /*!< GPIO ALTPADCFGJ: PAD39_SR (Bitfield-Mask: 0x01) */ +#define GPIO_ALTPADCFGJ_PAD39_DS1_Pos (24UL) /*!< GPIO ALTPADCFGJ: PAD39_DS1 (Bit 24) */ +#define GPIO_ALTPADCFGJ_PAD39_DS1_Msk (0x1000000UL) /*!< GPIO ALTPADCFGJ: PAD39_DS1 (Bitfield-Mask: 0x01) */ +#define GPIO_ALTPADCFGJ_PAD38_SR_Pos (20UL) /*!< GPIO ALTPADCFGJ: PAD38_SR (Bit 20) */ +#define GPIO_ALTPADCFGJ_PAD38_SR_Msk (0x100000UL) /*!< GPIO ALTPADCFGJ: PAD38_SR (Bitfield-Mask: 0x01) */ +#define GPIO_ALTPADCFGJ_PAD38_DS1_Pos (16UL) /*!< GPIO ALTPADCFGJ: PAD38_DS1 (Bit 16) */ +#define GPIO_ALTPADCFGJ_PAD38_DS1_Msk (0x10000UL) /*!< GPIO ALTPADCFGJ: PAD38_DS1 (Bitfield-Mask: 0x01) */ +#define GPIO_ALTPADCFGJ_PAD37_SR_Pos (12UL) /*!< GPIO ALTPADCFGJ: PAD37_SR (Bit 12) */ +#define GPIO_ALTPADCFGJ_PAD37_SR_Msk (0x1000UL) /*!< GPIO ALTPADCFGJ: PAD37_SR (Bitfield-Mask: 0x01) */ +#define GPIO_ALTPADCFGJ_PAD37_DS1_Pos (8UL) /*!< GPIO ALTPADCFGJ: PAD37_DS1 (Bit 8) */ +#define GPIO_ALTPADCFGJ_PAD37_DS1_Msk (0x100UL) /*!< GPIO ALTPADCFGJ: PAD37_DS1 (Bitfield-Mask: 0x01) */ +#define GPIO_ALTPADCFGJ_PAD36_SR_Pos (4UL) /*!< GPIO ALTPADCFGJ: PAD36_SR (Bit 4) */ +#define GPIO_ALTPADCFGJ_PAD36_SR_Msk (0x10UL) /*!< GPIO ALTPADCFGJ: PAD36_SR (Bitfield-Mask: 0x01) */ +#define GPIO_ALTPADCFGJ_PAD36_DS1_Pos (0UL) /*!< GPIO ALTPADCFGJ: PAD36_DS1 (Bit 0) */ +#define GPIO_ALTPADCFGJ_PAD36_DS1_Msk (0x1UL) /*!< GPIO ALTPADCFGJ: PAD36_DS1 (Bitfield-Mask: 0x01) */ +/* ====================================================== ALTPADCFGK ======================================================= */ +#define GPIO_ALTPADCFGK_PAD43_SR_Pos (28UL) /*!< GPIO ALTPADCFGK: PAD43_SR (Bit 28) */ +#define GPIO_ALTPADCFGK_PAD43_SR_Msk (0x10000000UL) /*!< GPIO ALTPADCFGK: PAD43_SR (Bitfield-Mask: 0x01) */ +#define GPIO_ALTPADCFGK_PAD43_DS1_Pos (24UL) /*!< GPIO ALTPADCFGK: PAD43_DS1 (Bit 24) */ +#define GPIO_ALTPADCFGK_PAD43_DS1_Msk (0x1000000UL) /*!< GPIO ALTPADCFGK: PAD43_DS1 (Bitfield-Mask: 0x01) */ +#define GPIO_ALTPADCFGK_PAD42_SR_Pos (20UL) /*!< GPIO ALTPADCFGK: PAD42_SR (Bit 20) */ +#define GPIO_ALTPADCFGK_PAD42_SR_Msk (0x100000UL) /*!< GPIO ALTPADCFGK: PAD42_SR (Bitfield-Mask: 0x01) */ +#define GPIO_ALTPADCFGK_PAD42_DS1_Pos (16UL) /*!< GPIO ALTPADCFGK: PAD42_DS1 (Bit 16) */ +#define GPIO_ALTPADCFGK_PAD42_DS1_Msk (0x10000UL) /*!< GPIO ALTPADCFGK: PAD42_DS1 (Bitfield-Mask: 0x01) */ +#define GPIO_ALTPADCFGK_PAD41_SR_Pos (12UL) /*!< GPIO ALTPADCFGK: PAD41_SR (Bit 12) */ +#define GPIO_ALTPADCFGK_PAD41_SR_Msk (0x1000UL) /*!< GPIO ALTPADCFGK: PAD41_SR (Bitfield-Mask: 0x01) */ +#define GPIO_ALTPADCFGK_PAD41_DS1_Pos (8UL) /*!< GPIO ALTPADCFGK: PAD41_DS1 (Bit 8) */ +#define GPIO_ALTPADCFGK_PAD41_DS1_Msk (0x100UL) /*!< GPIO ALTPADCFGK: PAD41_DS1 (Bitfield-Mask: 0x01) */ +#define GPIO_ALTPADCFGK_PAD40_SR_Pos (4UL) /*!< GPIO ALTPADCFGK: PAD40_SR (Bit 4) */ +#define GPIO_ALTPADCFGK_PAD40_SR_Msk (0x10UL) /*!< GPIO ALTPADCFGK: PAD40_SR (Bitfield-Mask: 0x01) */ +#define GPIO_ALTPADCFGK_PAD40_DS1_Pos (0UL) /*!< GPIO ALTPADCFGK: PAD40_DS1 (Bit 0) */ +#define GPIO_ALTPADCFGK_PAD40_DS1_Msk (0x1UL) /*!< GPIO ALTPADCFGK: PAD40_DS1 (Bitfield-Mask: 0x01) */ +/* ====================================================== ALTPADCFGL ======================================================= */ +#define GPIO_ALTPADCFGL_PAD47_SR_Pos (28UL) /*!< GPIO ALTPADCFGL: PAD47_SR (Bit 28) */ +#define GPIO_ALTPADCFGL_PAD47_SR_Msk (0x10000000UL) /*!< GPIO ALTPADCFGL: PAD47_SR (Bitfield-Mask: 0x01) */ +#define GPIO_ALTPADCFGL_PAD47_DS1_Pos (24UL) /*!< GPIO ALTPADCFGL: PAD47_DS1 (Bit 24) */ +#define GPIO_ALTPADCFGL_PAD47_DS1_Msk (0x1000000UL) /*!< GPIO ALTPADCFGL: PAD47_DS1 (Bitfield-Mask: 0x01) */ +#define GPIO_ALTPADCFGL_PAD46_SR_Pos (20UL) /*!< GPIO ALTPADCFGL: PAD46_SR (Bit 20) */ +#define GPIO_ALTPADCFGL_PAD46_SR_Msk (0x100000UL) /*!< GPIO ALTPADCFGL: PAD46_SR (Bitfield-Mask: 0x01) */ +#define GPIO_ALTPADCFGL_PAD46_DS1_Pos (16UL) /*!< GPIO ALTPADCFGL: PAD46_DS1 (Bit 16) */ +#define GPIO_ALTPADCFGL_PAD46_DS1_Msk (0x10000UL) /*!< GPIO ALTPADCFGL: PAD46_DS1 (Bitfield-Mask: 0x01) */ +#define GPIO_ALTPADCFGL_PAD45_SR_Pos (12UL) /*!< GPIO ALTPADCFGL: PAD45_SR (Bit 12) */ +#define GPIO_ALTPADCFGL_PAD45_SR_Msk (0x1000UL) /*!< GPIO ALTPADCFGL: PAD45_SR (Bitfield-Mask: 0x01) */ +#define GPIO_ALTPADCFGL_PAD45_DS1_Pos (8UL) /*!< GPIO ALTPADCFGL: PAD45_DS1 (Bit 8) */ +#define GPIO_ALTPADCFGL_PAD45_DS1_Msk (0x100UL) /*!< GPIO ALTPADCFGL: PAD45_DS1 (Bitfield-Mask: 0x01) */ +#define GPIO_ALTPADCFGL_PAD44_SR_Pos (4UL) /*!< GPIO ALTPADCFGL: PAD44_SR (Bit 4) */ +#define GPIO_ALTPADCFGL_PAD44_SR_Msk (0x10UL) /*!< GPIO ALTPADCFGL: PAD44_SR (Bitfield-Mask: 0x01) */ +#define GPIO_ALTPADCFGL_PAD44_DS1_Pos (0UL) /*!< GPIO ALTPADCFGL: PAD44_DS1 (Bit 0) */ +#define GPIO_ALTPADCFGL_PAD44_DS1_Msk (0x1UL) /*!< GPIO ALTPADCFGL: PAD44_DS1 (Bitfield-Mask: 0x01) */ +/* ====================================================== ALTPADCFGM ======================================================= */ +#define GPIO_ALTPADCFGM_PAD49_SR_Pos (12UL) /*!< GPIO ALTPADCFGM: PAD49_SR (Bit 12) */ +#define GPIO_ALTPADCFGM_PAD49_SR_Msk (0x1000UL) /*!< GPIO ALTPADCFGM: PAD49_SR (Bitfield-Mask: 0x01) */ +#define GPIO_ALTPADCFGM_PAD49_DS1_Pos (8UL) /*!< GPIO ALTPADCFGM: PAD49_DS1 (Bit 8) */ +#define GPIO_ALTPADCFGM_PAD49_DS1_Msk (0x100UL) /*!< GPIO ALTPADCFGM: PAD49_DS1 (Bitfield-Mask: 0x01) */ +#define GPIO_ALTPADCFGM_PAD48_SR_Pos (4UL) /*!< GPIO ALTPADCFGM: PAD48_SR (Bit 4) */ +#define GPIO_ALTPADCFGM_PAD48_SR_Msk (0x10UL) /*!< GPIO ALTPADCFGM: PAD48_SR (Bitfield-Mask: 0x01) */ +#define GPIO_ALTPADCFGM_PAD48_DS1_Pos (0UL) /*!< GPIO ALTPADCFGM: PAD48_DS1 (Bit 0) */ +#define GPIO_ALTPADCFGM_PAD48_DS1_Msk (0x1UL) /*!< GPIO ALTPADCFGM: PAD48_DS1 (Bitfield-Mask: 0x01) */ +/* ========================================================= SCDET ========================================================= */ +#define GPIO_SCDET_SCDET_Pos (0UL) /*!< GPIO SCDET: SCDET (Bit 0) */ +#define GPIO_SCDET_SCDET_Msk (0x3fUL) /*!< GPIO SCDET: SCDET (Bitfield-Mask: 0x3f) */ +/* ======================================================== CTENCFG ======================================================== */ +#define GPIO_CTENCFG_EN31_Pos (31UL) /*!< GPIO CTENCFG: EN31 (Bit 31) */ +#define GPIO_CTENCFG_EN31_Msk (0x80000000UL) /*!< GPIO CTENCFG: EN31 (Bitfield-Mask: 0x01) */ +#define GPIO_CTENCFG_EN30_Pos (30UL) /*!< GPIO CTENCFG: EN30 (Bit 30) */ +#define GPIO_CTENCFG_EN30_Msk (0x40000000UL) /*!< GPIO CTENCFG: EN30 (Bitfield-Mask: 0x01) */ +#define GPIO_CTENCFG_EN29_Pos (29UL) /*!< GPIO CTENCFG: EN29 (Bit 29) */ +#define GPIO_CTENCFG_EN29_Msk (0x20000000UL) /*!< GPIO CTENCFG: EN29 (Bitfield-Mask: 0x01) */ +#define GPIO_CTENCFG_EN28_Pos (28UL) /*!< GPIO CTENCFG: EN28 (Bit 28) */ +#define GPIO_CTENCFG_EN28_Msk (0x10000000UL) /*!< GPIO CTENCFG: EN28 (Bitfield-Mask: 0x01) */ +#define GPIO_CTENCFG_EN27_Pos (27UL) /*!< GPIO CTENCFG: EN27 (Bit 27) */ +#define GPIO_CTENCFG_EN27_Msk (0x8000000UL) /*!< GPIO CTENCFG: EN27 (Bitfield-Mask: 0x01) */ +#define GPIO_CTENCFG_EN26_Pos (26UL) /*!< GPIO CTENCFG: EN26 (Bit 26) */ +#define GPIO_CTENCFG_EN26_Msk (0x4000000UL) /*!< GPIO CTENCFG: EN26 (Bitfield-Mask: 0x01) */ +#define GPIO_CTENCFG_EN25_Pos (25UL) /*!< GPIO CTENCFG: EN25 (Bit 25) */ +#define GPIO_CTENCFG_EN25_Msk (0x2000000UL) /*!< GPIO CTENCFG: EN25 (Bitfield-Mask: 0x01) */ +#define GPIO_CTENCFG_EN24_Pos (24UL) /*!< GPIO CTENCFG: EN24 (Bit 24) */ +#define GPIO_CTENCFG_EN24_Msk (0x1000000UL) /*!< GPIO CTENCFG: EN24 (Bitfield-Mask: 0x01) */ +#define GPIO_CTENCFG_EN23_Pos (23UL) /*!< GPIO CTENCFG: EN23 (Bit 23) */ +#define GPIO_CTENCFG_EN23_Msk (0x800000UL) /*!< GPIO CTENCFG: EN23 (Bitfield-Mask: 0x01) */ +#define GPIO_CTENCFG_EN22_Pos (22UL) /*!< GPIO CTENCFG: EN22 (Bit 22) */ +#define GPIO_CTENCFG_EN22_Msk (0x400000UL) /*!< GPIO CTENCFG: EN22 (Bitfield-Mask: 0x01) */ +#define GPIO_CTENCFG_EN21_Pos (21UL) /*!< GPIO CTENCFG: EN21 (Bit 21) */ +#define GPIO_CTENCFG_EN21_Msk (0x200000UL) /*!< GPIO CTENCFG: EN21 (Bitfield-Mask: 0x01) */ +#define GPIO_CTENCFG_EN20_Pos (20UL) /*!< GPIO CTENCFG: EN20 (Bit 20) */ +#define GPIO_CTENCFG_EN20_Msk (0x100000UL) /*!< GPIO CTENCFG: EN20 (Bitfield-Mask: 0x01) */ +#define GPIO_CTENCFG_EN19_Pos (19UL) /*!< GPIO CTENCFG: EN19 (Bit 19) */ +#define GPIO_CTENCFG_EN19_Msk (0x80000UL) /*!< GPIO CTENCFG: EN19 (Bitfield-Mask: 0x01) */ +#define GPIO_CTENCFG_EN18_Pos (18UL) /*!< GPIO CTENCFG: EN18 (Bit 18) */ +#define GPIO_CTENCFG_EN18_Msk (0x40000UL) /*!< GPIO CTENCFG: EN18 (Bitfield-Mask: 0x01) */ +#define GPIO_CTENCFG_EN17_Pos (17UL) /*!< GPIO CTENCFG: EN17 (Bit 17) */ +#define GPIO_CTENCFG_EN17_Msk (0x20000UL) /*!< GPIO CTENCFG: EN17 (Bitfield-Mask: 0x01) */ +#define GPIO_CTENCFG_EN16_Pos (16UL) /*!< GPIO CTENCFG: EN16 (Bit 16) */ +#define GPIO_CTENCFG_EN16_Msk (0x10000UL) /*!< GPIO CTENCFG: EN16 (Bitfield-Mask: 0x01) */ +#define GPIO_CTENCFG_EN15_Pos (15UL) /*!< GPIO CTENCFG: EN15 (Bit 15) */ +#define GPIO_CTENCFG_EN15_Msk (0x8000UL) /*!< GPIO CTENCFG: EN15 (Bitfield-Mask: 0x01) */ +#define GPIO_CTENCFG_EN14_Pos (14UL) /*!< GPIO CTENCFG: EN14 (Bit 14) */ +#define GPIO_CTENCFG_EN14_Msk (0x4000UL) /*!< GPIO CTENCFG: EN14 (Bitfield-Mask: 0x01) */ +#define GPIO_CTENCFG_EN13_Pos (13UL) /*!< GPIO CTENCFG: EN13 (Bit 13) */ +#define GPIO_CTENCFG_EN13_Msk (0x2000UL) /*!< GPIO CTENCFG: EN13 (Bitfield-Mask: 0x01) */ +#define GPIO_CTENCFG_EN12_Pos (12UL) /*!< GPIO CTENCFG: EN12 (Bit 12) */ +#define GPIO_CTENCFG_EN12_Msk (0x1000UL) /*!< GPIO CTENCFG: EN12 (Bitfield-Mask: 0x01) */ +#define GPIO_CTENCFG_EN11_Pos (11UL) /*!< GPIO CTENCFG: EN11 (Bit 11) */ +#define GPIO_CTENCFG_EN11_Msk (0x800UL) /*!< GPIO CTENCFG: EN11 (Bitfield-Mask: 0x01) */ +#define GPIO_CTENCFG_EN10_Pos (10UL) /*!< GPIO CTENCFG: EN10 (Bit 10) */ +#define GPIO_CTENCFG_EN10_Msk (0x400UL) /*!< GPIO CTENCFG: EN10 (Bitfield-Mask: 0x01) */ +#define GPIO_CTENCFG_EN9_Pos (9UL) /*!< GPIO CTENCFG: EN9 (Bit 9) */ +#define GPIO_CTENCFG_EN9_Msk (0x200UL) /*!< GPIO CTENCFG: EN9 (Bitfield-Mask: 0x01) */ +#define GPIO_CTENCFG_EN8_Pos (8UL) /*!< GPIO CTENCFG: EN8 (Bit 8) */ +#define GPIO_CTENCFG_EN8_Msk (0x100UL) /*!< GPIO CTENCFG: EN8 (Bitfield-Mask: 0x01) */ +#define GPIO_CTENCFG_EN7_Pos (7UL) /*!< GPIO CTENCFG: EN7 (Bit 7) */ +#define GPIO_CTENCFG_EN7_Msk (0x80UL) /*!< GPIO CTENCFG: EN7 (Bitfield-Mask: 0x01) */ +#define GPIO_CTENCFG_EN6_Pos (6UL) /*!< GPIO CTENCFG: EN6 (Bit 6) */ +#define GPIO_CTENCFG_EN6_Msk (0x40UL) /*!< GPIO CTENCFG: EN6 (Bitfield-Mask: 0x01) */ +#define GPIO_CTENCFG_EN5_Pos (5UL) /*!< GPIO CTENCFG: EN5 (Bit 5) */ +#define GPIO_CTENCFG_EN5_Msk (0x20UL) /*!< GPIO CTENCFG: EN5 (Bitfield-Mask: 0x01) */ +#define GPIO_CTENCFG_EN4_Pos (4UL) /*!< GPIO CTENCFG: EN4 (Bit 4) */ +#define GPIO_CTENCFG_EN4_Msk (0x10UL) /*!< GPIO CTENCFG: EN4 (Bitfield-Mask: 0x01) */ +#define GPIO_CTENCFG_EN3_Pos (3UL) /*!< GPIO CTENCFG: EN3 (Bit 3) */ +#define GPIO_CTENCFG_EN3_Msk (0x8UL) /*!< GPIO CTENCFG: EN3 (Bitfield-Mask: 0x01) */ +#define GPIO_CTENCFG_EN2_Pos (2UL) /*!< GPIO CTENCFG: EN2 (Bit 2) */ +#define GPIO_CTENCFG_EN2_Msk (0x4UL) /*!< GPIO CTENCFG: EN2 (Bitfield-Mask: 0x01) */ +#define GPIO_CTENCFG_EN1_Pos (1UL) /*!< GPIO CTENCFG: EN1 (Bit 1) */ +#define GPIO_CTENCFG_EN1_Msk (0x2UL) /*!< GPIO CTENCFG: EN1 (Bitfield-Mask: 0x01) */ +#define GPIO_CTENCFG_EN0_Pos (0UL) /*!< GPIO CTENCFG: EN0 (Bit 0) */ +#define GPIO_CTENCFG_EN0_Msk (0x1UL) /*!< GPIO CTENCFG: EN0 (Bitfield-Mask: 0x01) */ +/* ======================================================== INT0EN ========================================================= */ +#define GPIO_INT0EN_GPIO31_Pos (31UL) /*!< GPIO INT0EN: GPIO31 (Bit 31) */ +#define GPIO_INT0EN_GPIO31_Msk (0x80000000UL) /*!< GPIO INT0EN: GPIO31 (Bitfield-Mask: 0x01) */ +#define GPIO_INT0EN_GPIO30_Pos (30UL) /*!< GPIO INT0EN: GPIO30 (Bit 30) */ +#define GPIO_INT0EN_GPIO30_Msk (0x40000000UL) /*!< GPIO INT0EN: GPIO30 (Bitfield-Mask: 0x01) */ +#define GPIO_INT0EN_GPIO29_Pos (29UL) /*!< GPIO INT0EN: GPIO29 (Bit 29) */ +#define GPIO_INT0EN_GPIO29_Msk (0x20000000UL) /*!< GPIO INT0EN: GPIO29 (Bitfield-Mask: 0x01) */ +#define GPIO_INT0EN_GPIO28_Pos (28UL) /*!< GPIO INT0EN: GPIO28 (Bit 28) */ +#define GPIO_INT0EN_GPIO28_Msk (0x10000000UL) /*!< GPIO INT0EN: GPIO28 (Bitfield-Mask: 0x01) */ +#define GPIO_INT0EN_GPIO27_Pos (27UL) /*!< GPIO INT0EN: GPIO27 (Bit 27) */ +#define GPIO_INT0EN_GPIO27_Msk (0x8000000UL) /*!< GPIO INT0EN: GPIO27 (Bitfield-Mask: 0x01) */ +#define GPIO_INT0EN_GPIO26_Pos (26UL) /*!< GPIO INT0EN: GPIO26 (Bit 26) */ +#define GPIO_INT0EN_GPIO26_Msk (0x4000000UL) /*!< GPIO INT0EN: GPIO26 (Bitfield-Mask: 0x01) */ +#define GPIO_INT0EN_GPIO25_Pos (25UL) /*!< GPIO INT0EN: GPIO25 (Bit 25) */ +#define GPIO_INT0EN_GPIO25_Msk (0x2000000UL) /*!< GPIO INT0EN: GPIO25 (Bitfield-Mask: 0x01) */ +#define GPIO_INT0EN_GPIO24_Pos (24UL) /*!< GPIO INT0EN: GPIO24 (Bit 24) */ +#define GPIO_INT0EN_GPIO24_Msk (0x1000000UL) /*!< GPIO INT0EN: GPIO24 (Bitfield-Mask: 0x01) */ +#define GPIO_INT0EN_GPIO23_Pos (23UL) /*!< GPIO INT0EN: GPIO23 (Bit 23) */ +#define GPIO_INT0EN_GPIO23_Msk (0x800000UL) /*!< GPIO INT0EN: GPIO23 (Bitfield-Mask: 0x01) */ +#define GPIO_INT0EN_GPIO22_Pos (22UL) /*!< GPIO INT0EN: GPIO22 (Bit 22) */ +#define GPIO_INT0EN_GPIO22_Msk (0x400000UL) /*!< GPIO INT0EN: GPIO22 (Bitfield-Mask: 0x01) */ +#define GPIO_INT0EN_GPIO21_Pos (21UL) /*!< GPIO INT0EN: GPIO21 (Bit 21) */ +#define GPIO_INT0EN_GPIO21_Msk (0x200000UL) /*!< GPIO INT0EN: GPIO21 (Bitfield-Mask: 0x01) */ +#define GPIO_INT0EN_GPIO20_Pos (20UL) /*!< GPIO INT0EN: GPIO20 (Bit 20) */ +#define GPIO_INT0EN_GPIO20_Msk (0x100000UL) /*!< GPIO INT0EN: GPIO20 (Bitfield-Mask: 0x01) */ +#define GPIO_INT0EN_GPIO19_Pos (19UL) /*!< GPIO INT0EN: GPIO19 (Bit 19) */ +#define GPIO_INT0EN_GPIO19_Msk (0x80000UL) /*!< GPIO INT0EN: GPIO19 (Bitfield-Mask: 0x01) */ +#define GPIO_INT0EN_GPIO18_Pos (18UL) /*!< GPIO INT0EN: GPIO18 (Bit 18) */ +#define GPIO_INT0EN_GPIO18_Msk (0x40000UL) /*!< GPIO INT0EN: GPIO18 (Bitfield-Mask: 0x01) */ +#define GPIO_INT0EN_GPIO17_Pos (17UL) /*!< GPIO INT0EN: GPIO17 (Bit 17) */ +#define GPIO_INT0EN_GPIO17_Msk (0x20000UL) /*!< GPIO INT0EN: GPIO17 (Bitfield-Mask: 0x01) */ +#define GPIO_INT0EN_GPIO16_Pos (16UL) /*!< GPIO INT0EN: GPIO16 (Bit 16) */ +#define GPIO_INT0EN_GPIO16_Msk (0x10000UL) /*!< GPIO INT0EN: GPIO16 (Bitfield-Mask: 0x01) */ +#define GPIO_INT0EN_GPIO15_Pos (15UL) /*!< GPIO INT0EN: GPIO15 (Bit 15) */ +#define GPIO_INT0EN_GPIO15_Msk (0x8000UL) /*!< GPIO INT0EN: GPIO15 (Bitfield-Mask: 0x01) */ +#define GPIO_INT0EN_GPIO14_Pos (14UL) /*!< GPIO INT0EN: GPIO14 (Bit 14) */ +#define GPIO_INT0EN_GPIO14_Msk (0x4000UL) /*!< GPIO INT0EN: GPIO14 (Bitfield-Mask: 0x01) */ +#define GPIO_INT0EN_GPIO13_Pos (13UL) /*!< GPIO INT0EN: GPIO13 (Bit 13) */ +#define GPIO_INT0EN_GPIO13_Msk (0x2000UL) /*!< GPIO INT0EN: GPIO13 (Bitfield-Mask: 0x01) */ +#define GPIO_INT0EN_GPIO12_Pos (12UL) /*!< GPIO INT0EN: GPIO12 (Bit 12) */ +#define GPIO_INT0EN_GPIO12_Msk (0x1000UL) /*!< GPIO INT0EN: GPIO12 (Bitfield-Mask: 0x01) */ +#define GPIO_INT0EN_GPIO11_Pos (11UL) /*!< GPIO INT0EN: GPIO11 (Bit 11) */ +#define GPIO_INT0EN_GPIO11_Msk (0x800UL) /*!< GPIO INT0EN: GPIO11 (Bitfield-Mask: 0x01) */ +#define GPIO_INT0EN_GPIO10_Pos (10UL) /*!< GPIO INT0EN: GPIO10 (Bit 10) */ +#define GPIO_INT0EN_GPIO10_Msk (0x400UL) /*!< GPIO INT0EN: GPIO10 (Bitfield-Mask: 0x01) */ +#define GPIO_INT0EN_GPIO9_Pos (9UL) /*!< GPIO INT0EN: GPIO9 (Bit 9) */ +#define GPIO_INT0EN_GPIO9_Msk (0x200UL) /*!< GPIO INT0EN: GPIO9 (Bitfield-Mask: 0x01) */ +#define GPIO_INT0EN_GPIO8_Pos (8UL) /*!< GPIO INT0EN: GPIO8 (Bit 8) */ +#define GPIO_INT0EN_GPIO8_Msk (0x100UL) /*!< GPIO INT0EN: GPIO8 (Bitfield-Mask: 0x01) */ +#define GPIO_INT0EN_GPIO7_Pos (7UL) /*!< GPIO INT0EN: GPIO7 (Bit 7) */ +#define GPIO_INT0EN_GPIO7_Msk (0x80UL) /*!< GPIO INT0EN: GPIO7 (Bitfield-Mask: 0x01) */ +#define GPIO_INT0EN_GPIO6_Pos (6UL) /*!< GPIO INT0EN: GPIO6 (Bit 6) */ +#define GPIO_INT0EN_GPIO6_Msk (0x40UL) /*!< GPIO INT0EN: GPIO6 (Bitfield-Mask: 0x01) */ +#define GPIO_INT0EN_GPIO5_Pos (5UL) /*!< GPIO INT0EN: GPIO5 (Bit 5) */ +#define GPIO_INT0EN_GPIO5_Msk (0x20UL) /*!< GPIO INT0EN: GPIO5 (Bitfield-Mask: 0x01) */ +#define GPIO_INT0EN_GPIO4_Pos (4UL) /*!< GPIO INT0EN: GPIO4 (Bit 4) */ +#define GPIO_INT0EN_GPIO4_Msk (0x10UL) /*!< GPIO INT0EN: GPIO4 (Bitfield-Mask: 0x01) */ +#define GPIO_INT0EN_GPIO3_Pos (3UL) /*!< GPIO INT0EN: GPIO3 (Bit 3) */ +#define GPIO_INT0EN_GPIO3_Msk (0x8UL) /*!< GPIO INT0EN: GPIO3 (Bitfield-Mask: 0x01) */ +#define GPIO_INT0EN_GPIO2_Pos (2UL) /*!< GPIO INT0EN: GPIO2 (Bit 2) */ +#define GPIO_INT0EN_GPIO2_Msk (0x4UL) /*!< GPIO INT0EN: GPIO2 (Bitfield-Mask: 0x01) */ +#define GPIO_INT0EN_GPIO1_Pos (1UL) /*!< GPIO INT0EN: GPIO1 (Bit 1) */ +#define GPIO_INT0EN_GPIO1_Msk (0x2UL) /*!< GPIO INT0EN: GPIO1 (Bitfield-Mask: 0x01) */ +#define GPIO_INT0EN_GPIO0_Pos (0UL) /*!< GPIO INT0EN: GPIO0 (Bit 0) */ +#define GPIO_INT0EN_GPIO0_Msk (0x1UL) /*!< GPIO INT0EN: GPIO0 (Bitfield-Mask: 0x01) */ +/* ======================================================= INT0STAT ======================================================== */ +#define GPIO_INT0STAT_GPIO31_Pos (31UL) /*!< GPIO INT0STAT: GPIO31 (Bit 31) */ +#define GPIO_INT0STAT_GPIO31_Msk (0x80000000UL) /*!< GPIO INT0STAT: GPIO31 (Bitfield-Mask: 0x01) */ +#define GPIO_INT0STAT_GPIO30_Pos (30UL) /*!< GPIO INT0STAT: GPIO30 (Bit 30) */ +#define GPIO_INT0STAT_GPIO30_Msk (0x40000000UL) /*!< GPIO INT0STAT: GPIO30 (Bitfield-Mask: 0x01) */ +#define GPIO_INT0STAT_GPIO29_Pos (29UL) /*!< GPIO INT0STAT: GPIO29 (Bit 29) */ +#define GPIO_INT0STAT_GPIO29_Msk (0x20000000UL) /*!< GPIO INT0STAT: GPIO29 (Bitfield-Mask: 0x01) */ +#define GPIO_INT0STAT_GPIO28_Pos (28UL) /*!< GPIO INT0STAT: GPIO28 (Bit 28) */ +#define GPIO_INT0STAT_GPIO28_Msk (0x10000000UL) /*!< GPIO INT0STAT: GPIO28 (Bitfield-Mask: 0x01) */ +#define GPIO_INT0STAT_GPIO27_Pos (27UL) /*!< GPIO INT0STAT: GPIO27 (Bit 27) */ +#define GPIO_INT0STAT_GPIO27_Msk (0x8000000UL) /*!< GPIO INT0STAT: GPIO27 (Bitfield-Mask: 0x01) */ +#define GPIO_INT0STAT_GPIO26_Pos (26UL) /*!< GPIO INT0STAT: GPIO26 (Bit 26) */ +#define GPIO_INT0STAT_GPIO26_Msk (0x4000000UL) /*!< GPIO INT0STAT: GPIO26 (Bitfield-Mask: 0x01) */ +#define GPIO_INT0STAT_GPIO25_Pos (25UL) /*!< GPIO INT0STAT: GPIO25 (Bit 25) */ +#define GPIO_INT0STAT_GPIO25_Msk (0x2000000UL) /*!< GPIO INT0STAT: GPIO25 (Bitfield-Mask: 0x01) */ +#define GPIO_INT0STAT_GPIO24_Pos (24UL) /*!< GPIO INT0STAT: GPIO24 (Bit 24) */ +#define GPIO_INT0STAT_GPIO24_Msk (0x1000000UL) /*!< GPIO INT0STAT: GPIO24 (Bitfield-Mask: 0x01) */ +#define GPIO_INT0STAT_GPIO23_Pos (23UL) /*!< GPIO INT0STAT: GPIO23 (Bit 23) */ +#define GPIO_INT0STAT_GPIO23_Msk (0x800000UL) /*!< GPIO INT0STAT: GPIO23 (Bitfield-Mask: 0x01) */ +#define GPIO_INT0STAT_GPIO22_Pos (22UL) /*!< GPIO INT0STAT: GPIO22 (Bit 22) */ +#define GPIO_INT0STAT_GPIO22_Msk (0x400000UL) /*!< GPIO INT0STAT: GPIO22 (Bitfield-Mask: 0x01) */ +#define GPIO_INT0STAT_GPIO21_Pos (21UL) /*!< GPIO INT0STAT: GPIO21 (Bit 21) */ +#define GPIO_INT0STAT_GPIO21_Msk (0x200000UL) /*!< GPIO INT0STAT: GPIO21 (Bitfield-Mask: 0x01) */ +#define GPIO_INT0STAT_GPIO20_Pos (20UL) /*!< GPIO INT0STAT: GPIO20 (Bit 20) */ +#define GPIO_INT0STAT_GPIO20_Msk (0x100000UL) /*!< GPIO INT0STAT: GPIO20 (Bitfield-Mask: 0x01) */ +#define GPIO_INT0STAT_GPIO19_Pos (19UL) /*!< GPIO INT0STAT: GPIO19 (Bit 19) */ +#define GPIO_INT0STAT_GPIO19_Msk (0x80000UL) /*!< GPIO INT0STAT: GPIO19 (Bitfield-Mask: 0x01) */ +#define GPIO_INT0STAT_GPIO18_Pos (18UL) /*!< GPIO INT0STAT: GPIO18 (Bit 18) */ +#define GPIO_INT0STAT_GPIO18_Msk (0x40000UL) /*!< GPIO INT0STAT: GPIO18 (Bitfield-Mask: 0x01) */ +#define GPIO_INT0STAT_GPIO17_Pos (17UL) /*!< GPIO INT0STAT: GPIO17 (Bit 17) */ +#define GPIO_INT0STAT_GPIO17_Msk (0x20000UL) /*!< GPIO INT0STAT: GPIO17 (Bitfield-Mask: 0x01) */ +#define GPIO_INT0STAT_GPIO16_Pos (16UL) /*!< GPIO INT0STAT: GPIO16 (Bit 16) */ +#define GPIO_INT0STAT_GPIO16_Msk (0x10000UL) /*!< GPIO INT0STAT: GPIO16 (Bitfield-Mask: 0x01) */ +#define GPIO_INT0STAT_GPIO15_Pos (15UL) /*!< GPIO INT0STAT: GPIO15 (Bit 15) */ +#define GPIO_INT0STAT_GPIO15_Msk (0x8000UL) /*!< GPIO INT0STAT: GPIO15 (Bitfield-Mask: 0x01) */ +#define GPIO_INT0STAT_GPIO14_Pos (14UL) /*!< GPIO INT0STAT: GPIO14 (Bit 14) */ +#define GPIO_INT0STAT_GPIO14_Msk (0x4000UL) /*!< GPIO INT0STAT: GPIO14 (Bitfield-Mask: 0x01) */ +#define GPIO_INT0STAT_GPIO13_Pos (13UL) /*!< GPIO INT0STAT: GPIO13 (Bit 13) */ +#define GPIO_INT0STAT_GPIO13_Msk (0x2000UL) /*!< GPIO INT0STAT: GPIO13 (Bitfield-Mask: 0x01) */ +#define GPIO_INT0STAT_GPIO12_Pos (12UL) /*!< GPIO INT0STAT: GPIO12 (Bit 12) */ +#define GPIO_INT0STAT_GPIO12_Msk (0x1000UL) /*!< GPIO INT0STAT: GPIO12 (Bitfield-Mask: 0x01) */ +#define GPIO_INT0STAT_GPIO11_Pos (11UL) /*!< GPIO INT0STAT: GPIO11 (Bit 11) */ +#define GPIO_INT0STAT_GPIO11_Msk (0x800UL) /*!< GPIO INT0STAT: GPIO11 (Bitfield-Mask: 0x01) */ +#define GPIO_INT0STAT_GPIO10_Pos (10UL) /*!< GPIO INT0STAT: GPIO10 (Bit 10) */ +#define GPIO_INT0STAT_GPIO10_Msk (0x400UL) /*!< GPIO INT0STAT: GPIO10 (Bitfield-Mask: 0x01) */ +#define GPIO_INT0STAT_GPIO9_Pos (9UL) /*!< GPIO INT0STAT: GPIO9 (Bit 9) */ +#define GPIO_INT0STAT_GPIO9_Msk (0x200UL) /*!< GPIO INT0STAT: GPIO9 (Bitfield-Mask: 0x01) */ +#define GPIO_INT0STAT_GPIO8_Pos (8UL) /*!< GPIO INT0STAT: GPIO8 (Bit 8) */ +#define GPIO_INT0STAT_GPIO8_Msk (0x100UL) /*!< GPIO INT0STAT: GPIO8 (Bitfield-Mask: 0x01) */ +#define GPIO_INT0STAT_GPIO7_Pos (7UL) /*!< GPIO INT0STAT: GPIO7 (Bit 7) */ +#define GPIO_INT0STAT_GPIO7_Msk (0x80UL) /*!< GPIO INT0STAT: GPIO7 (Bitfield-Mask: 0x01) */ +#define GPIO_INT0STAT_GPIO6_Pos (6UL) /*!< GPIO INT0STAT: GPIO6 (Bit 6) */ +#define GPIO_INT0STAT_GPIO6_Msk (0x40UL) /*!< GPIO INT0STAT: GPIO6 (Bitfield-Mask: 0x01) */ +#define GPIO_INT0STAT_GPIO5_Pos (5UL) /*!< GPIO INT0STAT: GPIO5 (Bit 5) */ +#define GPIO_INT0STAT_GPIO5_Msk (0x20UL) /*!< GPIO INT0STAT: GPIO5 (Bitfield-Mask: 0x01) */ +#define GPIO_INT0STAT_GPIO4_Pos (4UL) /*!< GPIO INT0STAT: GPIO4 (Bit 4) */ +#define GPIO_INT0STAT_GPIO4_Msk (0x10UL) /*!< GPIO INT0STAT: GPIO4 (Bitfield-Mask: 0x01) */ +#define GPIO_INT0STAT_GPIO3_Pos (3UL) /*!< GPIO INT0STAT: GPIO3 (Bit 3) */ +#define GPIO_INT0STAT_GPIO3_Msk (0x8UL) /*!< GPIO INT0STAT: GPIO3 (Bitfield-Mask: 0x01) */ +#define GPIO_INT0STAT_GPIO2_Pos (2UL) /*!< GPIO INT0STAT: GPIO2 (Bit 2) */ +#define GPIO_INT0STAT_GPIO2_Msk (0x4UL) /*!< GPIO INT0STAT: GPIO2 (Bitfield-Mask: 0x01) */ +#define GPIO_INT0STAT_GPIO1_Pos (1UL) /*!< GPIO INT0STAT: GPIO1 (Bit 1) */ +#define GPIO_INT0STAT_GPIO1_Msk (0x2UL) /*!< GPIO INT0STAT: GPIO1 (Bitfield-Mask: 0x01) */ +#define GPIO_INT0STAT_GPIO0_Pos (0UL) /*!< GPIO INT0STAT: GPIO0 (Bit 0) */ +#define GPIO_INT0STAT_GPIO0_Msk (0x1UL) /*!< GPIO INT0STAT: GPIO0 (Bitfield-Mask: 0x01) */ +/* ======================================================== INT0CLR ======================================================== */ +#define GPIO_INT0CLR_GPIO31_Pos (31UL) /*!< GPIO INT0CLR: GPIO31 (Bit 31) */ +#define GPIO_INT0CLR_GPIO31_Msk (0x80000000UL) /*!< GPIO INT0CLR: GPIO31 (Bitfield-Mask: 0x01) */ +#define GPIO_INT0CLR_GPIO30_Pos (30UL) /*!< GPIO INT0CLR: GPIO30 (Bit 30) */ +#define GPIO_INT0CLR_GPIO30_Msk (0x40000000UL) /*!< GPIO INT0CLR: GPIO30 (Bitfield-Mask: 0x01) */ +#define GPIO_INT0CLR_GPIO29_Pos (29UL) /*!< GPIO INT0CLR: GPIO29 (Bit 29) */ +#define GPIO_INT0CLR_GPIO29_Msk (0x20000000UL) /*!< GPIO INT0CLR: GPIO29 (Bitfield-Mask: 0x01) */ +#define GPIO_INT0CLR_GPIO28_Pos (28UL) /*!< GPIO INT0CLR: GPIO28 (Bit 28) */ +#define GPIO_INT0CLR_GPIO28_Msk (0x10000000UL) /*!< GPIO INT0CLR: GPIO28 (Bitfield-Mask: 0x01) */ +#define GPIO_INT0CLR_GPIO27_Pos (27UL) /*!< GPIO INT0CLR: GPIO27 (Bit 27) */ +#define GPIO_INT0CLR_GPIO27_Msk (0x8000000UL) /*!< GPIO INT0CLR: GPIO27 (Bitfield-Mask: 0x01) */ +#define GPIO_INT0CLR_GPIO26_Pos (26UL) /*!< GPIO INT0CLR: GPIO26 (Bit 26) */ +#define GPIO_INT0CLR_GPIO26_Msk (0x4000000UL) /*!< GPIO INT0CLR: GPIO26 (Bitfield-Mask: 0x01) */ +#define GPIO_INT0CLR_GPIO25_Pos (25UL) /*!< GPIO INT0CLR: GPIO25 (Bit 25) */ +#define GPIO_INT0CLR_GPIO25_Msk (0x2000000UL) /*!< GPIO INT0CLR: GPIO25 (Bitfield-Mask: 0x01) */ +#define GPIO_INT0CLR_GPIO24_Pos (24UL) /*!< GPIO INT0CLR: GPIO24 (Bit 24) */ +#define GPIO_INT0CLR_GPIO24_Msk (0x1000000UL) /*!< GPIO INT0CLR: GPIO24 (Bitfield-Mask: 0x01) */ +#define GPIO_INT0CLR_GPIO23_Pos (23UL) /*!< GPIO INT0CLR: GPIO23 (Bit 23) */ +#define GPIO_INT0CLR_GPIO23_Msk (0x800000UL) /*!< GPIO INT0CLR: GPIO23 (Bitfield-Mask: 0x01) */ +#define GPIO_INT0CLR_GPIO22_Pos (22UL) /*!< GPIO INT0CLR: GPIO22 (Bit 22) */ +#define GPIO_INT0CLR_GPIO22_Msk (0x400000UL) /*!< GPIO INT0CLR: GPIO22 (Bitfield-Mask: 0x01) */ +#define GPIO_INT0CLR_GPIO21_Pos (21UL) /*!< GPIO INT0CLR: GPIO21 (Bit 21) */ +#define GPIO_INT0CLR_GPIO21_Msk (0x200000UL) /*!< GPIO INT0CLR: GPIO21 (Bitfield-Mask: 0x01) */ +#define GPIO_INT0CLR_GPIO20_Pos (20UL) /*!< GPIO INT0CLR: GPIO20 (Bit 20) */ +#define GPIO_INT0CLR_GPIO20_Msk (0x100000UL) /*!< GPIO INT0CLR: GPIO20 (Bitfield-Mask: 0x01) */ +#define GPIO_INT0CLR_GPIO19_Pos (19UL) /*!< GPIO INT0CLR: GPIO19 (Bit 19) */ +#define GPIO_INT0CLR_GPIO19_Msk (0x80000UL) /*!< GPIO INT0CLR: GPIO19 (Bitfield-Mask: 0x01) */ +#define GPIO_INT0CLR_GPIO18_Pos (18UL) /*!< GPIO INT0CLR: GPIO18 (Bit 18) */ +#define GPIO_INT0CLR_GPIO18_Msk (0x40000UL) /*!< GPIO INT0CLR: GPIO18 (Bitfield-Mask: 0x01) */ +#define GPIO_INT0CLR_GPIO17_Pos (17UL) /*!< GPIO INT0CLR: GPIO17 (Bit 17) */ +#define GPIO_INT0CLR_GPIO17_Msk (0x20000UL) /*!< GPIO INT0CLR: GPIO17 (Bitfield-Mask: 0x01) */ +#define GPIO_INT0CLR_GPIO16_Pos (16UL) /*!< GPIO INT0CLR: GPIO16 (Bit 16) */ +#define GPIO_INT0CLR_GPIO16_Msk (0x10000UL) /*!< GPIO INT0CLR: GPIO16 (Bitfield-Mask: 0x01) */ +#define GPIO_INT0CLR_GPIO15_Pos (15UL) /*!< GPIO INT0CLR: GPIO15 (Bit 15) */ +#define GPIO_INT0CLR_GPIO15_Msk (0x8000UL) /*!< GPIO INT0CLR: GPIO15 (Bitfield-Mask: 0x01) */ +#define GPIO_INT0CLR_GPIO14_Pos (14UL) /*!< GPIO INT0CLR: GPIO14 (Bit 14) */ +#define GPIO_INT0CLR_GPIO14_Msk (0x4000UL) /*!< GPIO INT0CLR: GPIO14 (Bitfield-Mask: 0x01) */ +#define GPIO_INT0CLR_GPIO13_Pos (13UL) /*!< GPIO INT0CLR: GPIO13 (Bit 13) */ +#define GPIO_INT0CLR_GPIO13_Msk (0x2000UL) /*!< GPIO INT0CLR: GPIO13 (Bitfield-Mask: 0x01) */ +#define GPIO_INT0CLR_GPIO12_Pos (12UL) /*!< GPIO INT0CLR: GPIO12 (Bit 12) */ +#define GPIO_INT0CLR_GPIO12_Msk (0x1000UL) /*!< GPIO INT0CLR: GPIO12 (Bitfield-Mask: 0x01) */ +#define GPIO_INT0CLR_GPIO11_Pos (11UL) /*!< GPIO INT0CLR: GPIO11 (Bit 11) */ +#define GPIO_INT0CLR_GPIO11_Msk (0x800UL) /*!< GPIO INT0CLR: GPIO11 (Bitfield-Mask: 0x01) */ +#define GPIO_INT0CLR_GPIO10_Pos (10UL) /*!< GPIO INT0CLR: GPIO10 (Bit 10) */ +#define GPIO_INT0CLR_GPIO10_Msk (0x400UL) /*!< GPIO INT0CLR: GPIO10 (Bitfield-Mask: 0x01) */ +#define GPIO_INT0CLR_GPIO9_Pos (9UL) /*!< GPIO INT0CLR: GPIO9 (Bit 9) */ +#define GPIO_INT0CLR_GPIO9_Msk (0x200UL) /*!< GPIO INT0CLR: GPIO9 (Bitfield-Mask: 0x01) */ +#define GPIO_INT0CLR_GPIO8_Pos (8UL) /*!< GPIO INT0CLR: GPIO8 (Bit 8) */ +#define GPIO_INT0CLR_GPIO8_Msk (0x100UL) /*!< GPIO INT0CLR: GPIO8 (Bitfield-Mask: 0x01) */ +#define GPIO_INT0CLR_GPIO7_Pos (7UL) /*!< GPIO INT0CLR: GPIO7 (Bit 7) */ +#define GPIO_INT0CLR_GPIO7_Msk (0x80UL) /*!< GPIO INT0CLR: GPIO7 (Bitfield-Mask: 0x01) */ +#define GPIO_INT0CLR_GPIO6_Pos (6UL) /*!< GPIO INT0CLR: GPIO6 (Bit 6) */ +#define GPIO_INT0CLR_GPIO6_Msk (0x40UL) /*!< GPIO INT0CLR: GPIO6 (Bitfield-Mask: 0x01) */ +#define GPIO_INT0CLR_GPIO5_Pos (5UL) /*!< GPIO INT0CLR: GPIO5 (Bit 5) */ +#define GPIO_INT0CLR_GPIO5_Msk (0x20UL) /*!< GPIO INT0CLR: GPIO5 (Bitfield-Mask: 0x01) */ +#define GPIO_INT0CLR_GPIO4_Pos (4UL) /*!< GPIO INT0CLR: GPIO4 (Bit 4) */ +#define GPIO_INT0CLR_GPIO4_Msk (0x10UL) /*!< GPIO INT0CLR: GPIO4 (Bitfield-Mask: 0x01) */ +#define GPIO_INT0CLR_GPIO3_Pos (3UL) /*!< GPIO INT0CLR: GPIO3 (Bit 3) */ +#define GPIO_INT0CLR_GPIO3_Msk (0x8UL) /*!< GPIO INT0CLR: GPIO3 (Bitfield-Mask: 0x01) */ +#define GPIO_INT0CLR_GPIO2_Pos (2UL) /*!< GPIO INT0CLR: GPIO2 (Bit 2) */ +#define GPIO_INT0CLR_GPIO2_Msk (0x4UL) /*!< GPIO INT0CLR: GPIO2 (Bitfield-Mask: 0x01) */ +#define GPIO_INT0CLR_GPIO1_Pos (1UL) /*!< GPIO INT0CLR: GPIO1 (Bit 1) */ +#define GPIO_INT0CLR_GPIO1_Msk (0x2UL) /*!< GPIO INT0CLR: GPIO1 (Bitfield-Mask: 0x01) */ +#define GPIO_INT0CLR_GPIO0_Pos (0UL) /*!< GPIO INT0CLR: GPIO0 (Bit 0) */ +#define GPIO_INT0CLR_GPIO0_Msk (0x1UL) /*!< GPIO INT0CLR: GPIO0 (Bitfield-Mask: 0x01) */ +/* ======================================================== INT0SET ======================================================== */ +#define GPIO_INT0SET_GPIO31_Pos (31UL) /*!< GPIO INT0SET: GPIO31 (Bit 31) */ +#define GPIO_INT0SET_GPIO31_Msk (0x80000000UL) /*!< GPIO INT0SET: GPIO31 (Bitfield-Mask: 0x01) */ +#define GPIO_INT0SET_GPIO30_Pos (30UL) /*!< GPIO INT0SET: GPIO30 (Bit 30) */ +#define GPIO_INT0SET_GPIO30_Msk (0x40000000UL) /*!< GPIO INT0SET: GPIO30 (Bitfield-Mask: 0x01) */ +#define GPIO_INT0SET_GPIO29_Pos (29UL) /*!< GPIO INT0SET: GPIO29 (Bit 29) */ +#define GPIO_INT0SET_GPIO29_Msk (0x20000000UL) /*!< GPIO INT0SET: GPIO29 (Bitfield-Mask: 0x01) */ +#define GPIO_INT0SET_GPIO28_Pos (28UL) /*!< GPIO INT0SET: GPIO28 (Bit 28) */ +#define GPIO_INT0SET_GPIO28_Msk (0x10000000UL) /*!< GPIO INT0SET: GPIO28 (Bitfield-Mask: 0x01) */ +#define GPIO_INT0SET_GPIO27_Pos (27UL) /*!< GPIO INT0SET: GPIO27 (Bit 27) */ +#define GPIO_INT0SET_GPIO27_Msk (0x8000000UL) /*!< GPIO INT0SET: GPIO27 (Bitfield-Mask: 0x01) */ +#define GPIO_INT0SET_GPIO26_Pos (26UL) /*!< GPIO INT0SET: GPIO26 (Bit 26) */ +#define GPIO_INT0SET_GPIO26_Msk (0x4000000UL) /*!< GPIO INT0SET: GPIO26 (Bitfield-Mask: 0x01) */ +#define GPIO_INT0SET_GPIO25_Pos (25UL) /*!< GPIO INT0SET: GPIO25 (Bit 25) */ +#define GPIO_INT0SET_GPIO25_Msk (0x2000000UL) /*!< GPIO INT0SET: GPIO25 (Bitfield-Mask: 0x01) */ +#define GPIO_INT0SET_GPIO24_Pos (24UL) /*!< GPIO INT0SET: GPIO24 (Bit 24) */ +#define GPIO_INT0SET_GPIO24_Msk (0x1000000UL) /*!< GPIO INT0SET: GPIO24 (Bitfield-Mask: 0x01) */ +#define GPIO_INT0SET_GPIO23_Pos (23UL) /*!< GPIO INT0SET: GPIO23 (Bit 23) */ +#define GPIO_INT0SET_GPIO23_Msk (0x800000UL) /*!< GPIO INT0SET: GPIO23 (Bitfield-Mask: 0x01) */ +#define GPIO_INT0SET_GPIO22_Pos (22UL) /*!< GPIO INT0SET: GPIO22 (Bit 22) */ +#define GPIO_INT0SET_GPIO22_Msk (0x400000UL) /*!< GPIO INT0SET: GPIO22 (Bitfield-Mask: 0x01) */ +#define GPIO_INT0SET_GPIO21_Pos (21UL) /*!< GPIO INT0SET: GPIO21 (Bit 21) */ +#define GPIO_INT0SET_GPIO21_Msk (0x200000UL) /*!< GPIO INT0SET: GPIO21 (Bitfield-Mask: 0x01) */ +#define GPIO_INT0SET_GPIO20_Pos (20UL) /*!< GPIO INT0SET: GPIO20 (Bit 20) */ +#define GPIO_INT0SET_GPIO20_Msk (0x100000UL) /*!< GPIO INT0SET: GPIO20 (Bitfield-Mask: 0x01) */ +#define GPIO_INT0SET_GPIO19_Pos (19UL) /*!< GPIO INT0SET: GPIO19 (Bit 19) */ +#define GPIO_INT0SET_GPIO19_Msk (0x80000UL) /*!< GPIO INT0SET: GPIO19 (Bitfield-Mask: 0x01) */ +#define GPIO_INT0SET_GPIO18_Pos (18UL) /*!< GPIO INT0SET: GPIO18 (Bit 18) */ +#define GPIO_INT0SET_GPIO18_Msk (0x40000UL) /*!< GPIO INT0SET: GPIO18 (Bitfield-Mask: 0x01) */ +#define GPIO_INT0SET_GPIO17_Pos (17UL) /*!< GPIO INT0SET: GPIO17 (Bit 17) */ +#define GPIO_INT0SET_GPIO17_Msk (0x20000UL) /*!< GPIO INT0SET: GPIO17 (Bitfield-Mask: 0x01) */ +#define GPIO_INT0SET_GPIO16_Pos (16UL) /*!< GPIO INT0SET: GPIO16 (Bit 16) */ +#define GPIO_INT0SET_GPIO16_Msk (0x10000UL) /*!< GPIO INT0SET: GPIO16 (Bitfield-Mask: 0x01) */ +#define GPIO_INT0SET_GPIO15_Pos (15UL) /*!< GPIO INT0SET: GPIO15 (Bit 15) */ +#define GPIO_INT0SET_GPIO15_Msk (0x8000UL) /*!< GPIO INT0SET: GPIO15 (Bitfield-Mask: 0x01) */ +#define GPIO_INT0SET_GPIO14_Pos (14UL) /*!< GPIO INT0SET: GPIO14 (Bit 14) */ +#define GPIO_INT0SET_GPIO14_Msk (0x4000UL) /*!< GPIO INT0SET: GPIO14 (Bitfield-Mask: 0x01) */ +#define GPIO_INT0SET_GPIO13_Pos (13UL) /*!< GPIO INT0SET: GPIO13 (Bit 13) */ +#define GPIO_INT0SET_GPIO13_Msk (0x2000UL) /*!< GPIO INT0SET: GPIO13 (Bitfield-Mask: 0x01) */ +#define GPIO_INT0SET_GPIO12_Pos (12UL) /*!< GPIO INT0SET: GPIO12 (Bit 12) */ +#define GPIO_INT0SET_GPIO12_Msk (0x1000UL) /*!< GPIO INT0SET: GPIO12 (Bitfield-Mask: 0x01) */ +#define GPIO_INT0SET_GPIO11_Pos (11UL) /*!< GPIO INT0SET: GPIO11 (Bit 11) */ +#define GPIO_INT0SET_GPIO11_Msk (0x800UL) /*!< GPIO INT0SET: GPIO11 (Bitfield-Mask: 0x01) */ +#define GPIO_INT0SET_GPIO10_Pos (10UL) /*!< GPIO INT0SET: GPIO10 (Bit 10) */ +#define GPIO_INT0SET_GPIO10_Msk (0x400UL) /*!< GPIO INT0SET: GPIO10 (Bitfield-Mask: 0x01) */ +#define GPIO_INT0SET_GPIO9_Pos (9UL) /*!< GPIO INT0SET: GPIO9 (Bit 9) */ +#define GPIO_INT0SET_GPIO9_Msk (0x200UL) /*!< GPIO INT0SET: GPIO9 (Bitfield-Mask: 0x01) */ +#define GPIO_INT0SET_GPIO8_Pos (8UL) /*!< GPIO INT0SET: GPIO8 (Bit 8) */ +#define GPIO_INT0SET_GPIO8_Msk (0x100UL) /*!< GPIO INT0SET: GPIO8 (Bitfield-Mask: 0x01) */ +#define GPIO_INT0SET_GPIO7_Pos (7UL) /*!< GPIO INT0SET: GPIO7 (Bit 7) */ +#define GPIO_INT0SET_GPIO7_Msk (0x80UL) /*!< GPIO INT0SET: GPIO7 (Bitfield-Mask: 0x01) */ +#define GPIO_INT0SET_GPIO6_Pos (6UL) /*!< GPIO INT0SET: GPIO6 (Bit 6) */ +#define GPIO_INT0SET_GPIO6_Msk (0x40UL) /*!< GPIO INT0SET: GPIO6 (Bitfield-Mask: 0x01) */ +#define GPIO_INT0SET_GPIO5_Pos (5UL) /*!< GPIO INT0SET: GPIO5 (Bit 5) */ +#define GPIO_INT0SET_GPIO5_Msk (0x20UL) /*!< GPIO INT0SET: GPIO5 (Bitfield-Mask: 0x01) */ +#define GPIO_INT0SET_GPIO4_Pos (4UL) /*!< GPIO INT0SET: GPIO4 (Bit 4) */ +#define GPIO_INT0SET_GPIO4_Msk (0x10UL) /*!< GPIO INT0SET: GPIO4 (Bitfield-Mask: 0x01) */ +#define GPIO_INT0SET_GPIO3_Pos (3UL) /*!< GPIO INT0SET: GPIO3 (Bit 3) */ +#define GPIO_INT0SET_GPIO3_Msk (0x8UL) /*!< GPIO INT0SET: GPIO3 (Bitfield-Mask: 0x01) */ +#define GPIO_INT0SET_GPIO2_Pos (2UL) /*!< GPIO INT0SET: GPIO2 (Bit 2) */ +#define GPIO_INT0SET_GPIO2_Msk (0x4UL) /*!< GPIO INT0SET: GPIO2 (Bitfield-Mask: 0x01) */ +#define GPIO_INT0SET_GPIO1_Pos (1UL) /*!< GPIO INT0SET: GPIO1 (Bit 1) */ +#define GPIO_INT0SET_GPIO1_Msk (0x2UL) /*!< GPIO INT0SET: GPIO1 (Bitfield-Mask: 0x01) */ +#define GPIO_INT0SET_GPIO0_Pos (0UL) /*!< GPIO INT0SET: GPIO0 (Bit 0) */ +#define GPIO_INT0SET_GPIO0_Msk (0x1UL) /*!< GPIO INT0SET: GPIO0 (Bitfield-Mask: 0x01) */ +/* ======================================================== INT1EN ========================================================= */ +#define GPIO_INT1EN_GPIO49_Pos (17UL) /*!< GPIO INT1EN: GPIO49 (Bit 17) */ +#define GPIO_INT1EN_GPIO49_Msk (0x20000UL) /*!< GPIO INT1EN: GPIO49 (Bitfield-Mask: 0x01) */ +#define GPIO_INT1EN_GPIO48_Pos (16UL) /*!< GPIO INT1EN: GPIO48 (Bit 16) */ +#define GPIO_INT1EN_GPIO48_Msk (0x10000UL) /*!< GPIO INT1EN: GPIO48 (Bitfield-Mask: 0x01) */ +#define GPIO_INT1EN_GPIO47_Pos (15UL) /*!< GPIO INT1EN: GPIO47 (Bit 15) */ +#define GPIO_INT1EN_GPIO47_Msk (0x8000UL) /*!< GPIO INT1EN: GPIO47 (Bitfield-Mask: 0x01) */ +#define GPIO_INT1EN_GPIO46_Pos (14UL) /*!< GPIO INT1EN: GPIO46 (Bit 14) */ +#define GPIO_INT1EN_GPIO46_Msk (0x4000UL) /*!< GPIO INT1EN: GPIO46 (Bitfield-Mask: 0x01) */ +#define GPIO_INT1EN_GPIO45_Pos (13UL) /*!< GPIO INT1EN: GPIO45 (Bit 13) */ +#define GPIO_INT1EN_GPIO45_Msk (0x2000UL) /*!< GPIO INT1EN: GPIO45 (Bitfield-Mask: 0x01) */ +#define GPIO_INT1EN_GPIO44_Pos (12UL) /*!< GPIO INT1EN: GPIO44 (Bit 12) */ +#define GPIO_INT1EN_GPIO44_Msk (0x1000UL) /*!< GPIO INT1EN: GPIO44 (Bitfield-Mask: 0x01) */ +#define GPIO_INT1EN_GPIO43_Pos (11UL) /*!< GPIO INT1EN: GPIO43 (Bit 11) */ +#define GPIO_INT1EN_GPIO43_Msk (0x800UL) /*!< GPIO INT1EN: GPIO43 (Bitfield-Mask: 0x01) */ +#define GPIO_INT1EN_GPIO42_Pos (10UL) /*!< GPIO INT1EN: GPIO42 (Bit 10) */ +#define GPIO_INT1EN_GPIO42_Msk (0x400UL) /*!< GPIO INT1EN: GPIO42 (Bitfield-Mask: 0x01) */ +#define GPIO_INT1EN_GPIO41_Pos (9UL) /*!< GPIO INT1EN: GPIO41 (Bit 9) */ +#define GPIO_INT1EN_GPIO41_Msk (0x200UL) /*!< GPIO INT1EN: GPIO41 (Bitfield-Mask: 0x01) */ +#define GPIO_INT1EN_GPIO40_Pos (8UL) /*!< GPIO INT1EN: GPIO40 (Bit 8) */ +#define GPIO_INT1EN_GPIO40_Msk (0x100UL) /*!< GPIO INT1EN: GPIO40 (Bitfield-Mask: 0x01) */ +#define GPIO_INT1EN_GPIO39_Pos (7UL) /*!< GPIO INT1EN: GPIO39 (Bit 7) */ +#define GPIO_INT1EN_GPIO39_Msk (0x80UL) /*!< GPIO INT1EN: GPIO39 (Bitfield-Mask: 0x01) */ +#define GPIO_INT1EN_GPIO38_Pos (6UL) /*!< GPIO INT1EN: GPIO38 (Bit 6) */ +#define GPIO_INT1EN_GPIO38_Msk (0x40UL) /*!< GPIO INT1EN: GPIO38 (Bitfield-Mask: 0x01) */ +#define GPIO_INT1EN_GPIO37_Pos (5UL) /*!< GPIO INT1EN: GPIO37 (Bit 5) */ +#define GPIO_INT1EN_GPIO37_Msk (0x20UL) /*!< GPIO INT1EN: GPIO37 (Bitfield-Mask: 0x01) */ +#define GPIO_INT1EN_GPIO36_Pos (4UL) /*!< GPIO INT1EN: GPIO36 (Bit 4) */ +#define GPIO_INT1EN_GPIO36_Msk (0x10UL) /*!< GPIO INT1EN: GPIO36 (Bitfield-Mask: 0x01) */ +#define GPIO_INT1EN_GPIO35_Pos (3UL) /*!< GPIO INT1EN: GPIO35 (Bit 3) */ +#define GPIO_INT1EN_GPIO35_Msk (0x8UL) /*!< GPIO INT1EN: GPIO35 (Bitfield-Mask: 0x01) */ +#define GPIO_INT1EN_GPIO34_Pos (2UL) /*!< GPIO INT1EN: GPIO34 (Bit 2) */ +#define GPIO_INT1EN_GPIO34_Msk (0x4UL) /*!< GPIO INT1EN: GPIO34 (Bitfield-Mask: 0x01) */ +#define GPIO_INT1EN_GPIO33_Pos (1UL) /*!< GPIO INT1EN: GPIO33 (Bit 1) */ +#define GPIO_INT1EN_GPIO33_Msk (0x2UL) /*!< GPIO INT1EN: GPIO33 (Bitfield-Mask: 0x01) */ +#define GPIO_INT1EN_GPIO32_Pos (0UL) /*!< GPIO INT1EN: GPIO32 (Bit 0) */ +#define GPIO_INT1EN_GPIO32_Msk (0x1UL) /*!< GPIO INT1EN: GPIO32 (Bitfield-Mask: 0x01) */ +/* ======================================================= INT1STAT ======================================================== */ +#define GPIO_INT1STAT_GPIO49_Pos (17UL) /*!< GPIO INT1STAT: GPIO49 (Bit 17) */ +#define GPIO_INT1STAT_GPIO49_Msk (0x20000UL) /*!< GPIO INT1STAT: GPIO49 (Bitfield-Mask: 0x01) */ +#define GPIO_INT1STAT_GPIO48_Pos (16UL) /*!< GPIO INT1STAT: GPIO48 (Bit 16) */ +#define GPIO_INT1STAT_GPIO48_Msk (0x10000UL) /*!< GPIO INT1STAT: GPIO48 (Bitfield-Mask: 0x01) */ +#define GPIO_INT1STAT_GPIO47_Pos (15UL) /*!< GPIO INT1STAT: GPIO47 (Bit 15) */ +#define GPIO_INT1STAT_GPIO47_Msk (0x8000UL) /*!< GPIO INT1STAT: GPIO47 (Bitfield-Mask: 0x01) */ +#define GPIO_INT1STAT_GPIO46_Pos (14UL) /*!< GPIO INT1STAT: GPIO46 (Bit 14) */ +#define GPIO_INT1STAT_GPIO46_Msk (0x4000UL) /*!< GPIO INT1STAT: GPIO46 (Bitfield-Mask: 0x01) */ +#define GPIO_INT1STAT_GPIO45_Pos (13UL) /*!< GPIO INT1STAT: GPIO45 (Bit 13) */ +#define GPIO_INT1STAT_GPIO45_Msk (0x2000UL) /*!< GPIO INT1STAT: GPIO45 (Bitfield-Mask: 0x01) */ +#define GPIO_INT1STAT_GPIO44_Pos (12UL) /*!< GPIO INT1STAT: GPIO44 (Bit 12) */ +#define GPIO_INT1STAT_GPIO44_Msk (0x1000UL) /*!< GPIO INT1STAT: GPIO44 (Bitfield-Mask: 0x01) */ +#define GPIO_INT1STAT_GPIO43_Pos (11UL) /*!< GPIO INT1STAT: GPIO43 (Bit 11) */ +#define GPIO_INT1STAT_GPIO43_Msk (0x800UL) /*!< GPIO INT1STAT: GPIO43 (Bitfield-Mask: 0x01) */ +#define GPIO_INT1STAT_GPIO42_Pos (10UL) /*!< GPIO INT1STAT: GPIO42 (Bit 10) */ +#define GPIO_INT1STAT_GPIO42_Msk (0x400UL) /*!< GPIO INT1STAT: GPIO42 (Bitfield-Mask: 0x01) */ +#define GPIO_INT1STAT_GPIO41_Pos (9UL) /*!< GPIO INT1STAT: GPIO41 (Bit 9) */ +#define GPIO_INT1STAT_GPIO41_Msk (0x200UL) /*!< GPIO INT1STAT: GPIO41 (Bitfield-Mask: 0x01) */ +#define GPIO_INT1STAT_GPIO40_Pos (8UL) /*!< GPIO INT1STAT: GPIO40 (Bit 8) */ +#define GPIO_INT1STAT_GPIO40_Msk (0x100UL) /*!< GPIO INT1STAT: GPIO40 (Bitfield-Mask: 0x01) */ +#define GPIO_INT1STAT_GPIO39_Pos (7UL) /*!< GPIO INT1STAT: GPIO39 (Bit 7) */ +#define GPIO_INT1STAT_GPIO39_Msk (0x80UL) /*!< GPIO INT1STAT: GPIO39 (Bitfield-Mask: 0x01) */ +#define GPIO_INT1STAT_GPIO38_Pos (6UL) /*!< GPIO INT1STAT: GPIO38 (Bit 6) */ +#define GPIO_INT1STAT_GPIO38_Msk (0x40UL) /*!< GPIO INT1STAT: GPIO38 (Bitfield-Mask: 0x01) */ +#define GPIO_INT1STAT_GPIO37_Pos (5UL) /*!< GPIO INT1STAT: GPIO37 (Bit 5) */ +#define GPIO_INT1STAT_GPIO37_Msk (0x20UL) /*!< GPIO INT1STAT: GPIO37 (Bitfield-Mask: 0x01) */ +#define GPIO_INT1STAT_GPIO36_Pos (4UL) /*!< GPIO INT1STAT: GPIO36 (Bit 4) */ +#define GPIO_INT1STAT_GPIO36_Msk (0x10UL) /*!< GPIO INT1STAT: GPIO36 (Bitfield-Mask: 0x01) */ +#define GPIO_INT1STAT_GPIO35_Pos (3UL) /*!< GPIO INT1STAT: GPIO35 (Bit 3) */ +#define GPIO_INT1STAT_GPIO35_Msk (0x8UL) /*!< GPIO INT1STAT: GPIO35 (Bitfield-Mask: 0x01) */ +#define GPIO_INT1STAT_GPIO34_Pos (2UL) /*!< GPIO INT1STAT: GPIO34 (Bit 2) */ +#define GPIO_INT1STAT_GPIO34_Msk (0x4UL) /*!< GPIO INT1STAT: GPIO34 (Bitfield-Mask: 0x01) */ +#define GPIO_INT1STAT_GPIO33_Pos (1UL) /*!< GPIO INT1STAT: GPIO33 (Bit 1) */ +#define GPIO_INT1STAT_GPIO33_Msk (0x2UL) /*!< GPIO INT1STAT: GPIO33 (Bitfield-Mask: 0x01) */ +#define GPIO_INT1STAT_GPIO32_Pos (0UL) /*!< GPIO INT1STAT: GPIO32 (Bit 0) */ +#define GPIO_INT1STAT_GPIO32_Msk (0x1UL) /*!< GPIO INT1STAT: GPIO32 (Bitfield-Mask: 0x01) */ +/* ======================================================== INT1CLR ======================================================== */ +#define GPIO_INT1CLR_GPIO49_Pos (17UL) /*!< GPIO INT1CLR: GPIO49 (Bit 17) */ +#define GPIO_INT1CLR_GPIO49_Msk (0x20000UL) /*!< GPIO INT1CLR: GPIO49 (Bitfield-Mask: 0x01) */ +#define GPIO_INT1CLR_GPIO48_Pos (16UL) /*!< GPIO INT1CLR: GPIO48 (Bit 16) */ +#define GPIO_INT1CLR_GPIO48_Msk (0x10000UL) /*!< GPIO INT1CLR: GPIO48 (Bitfield-Mask: 0x01) */ +#define GPIO_INT1CLR_GPIO47_Pos (15UL) /*!< GPIO INT1CLR: GPIO47 (Bit 15) */ +#define GPIO_INT1CLR_GPIO47_Msk (0x8000UL) /*!< GPIO INT1CLR: GPIO47 (Bitfield-Mask: 0x01) */ +#define GPIO_INT1CLR_GPIO46_Pos (14UL) /*!< GPIO INT1CLR: GPIO46 (Bit 14) */ +#define GPIO_INT1CLR_GPIO46_Msk (0x4000UL) /*!< GPIO INT1CLR: GPIO46 (Bitfield-Mask: 0x01) */ +#define GPIO_INT1CLR_GPIO45_Pos (13UL) /*!< GPIO INT1CLR: GPIO45 (Bit 13) */ +#define GPIO_INT1CLR_GPIO45_Msk (0x2000UL) /*!< GPIO INT1CLR: GPIO45 (Bitfield-Mask: 0x01) */ +#define GPIO_INT1CLR_GPIO44_Pos (12UL) /*!< GPIO INT1CLR: GPIO44 (Bit 12) */ +#define GPIO_INT1CLR_GPIO44_Msk (0x1000UL) /*!< GPIO INT1CLR: GPIO44 (Bitfield-Mask: 0x01) */ +#define GPIO_INT1CLR_GPIO43_Pos (11UL) /*!< GPIO INT1CLR: GPIO43 (Bit 11) */ +#define GPIO_INT1CLR_GPIO43_Msk (0x800UL) /*!< GPIO INT1CLR: GPIO43 (Bitfield-Mask: 0x01) */ +#define GPIO_INT1CLR_GPIO42_Pos (10UL) /*!< GPIO INT1CLR: GPIO42 (Bit 10) */ +#define GPIO_INT1CLR_GPIO42_Msk (0x400UL) /*!< GPIO INT1CLR: GPIO42 (Bitfield-Mask: 0x01) */ +#define GPIO_INT1CLR_GPIO41_Pos (9UL) /*!< GPIO INT1CLR: GPIO41 (Bit 9) */ +#define GPIO_INT1CLR_GPIO41_Msk (0x200UL) /*!< GPIO INT1CLR: GPIO41 (Bitfield-Mask: 0x01) */ +#define GPIO_INT1CLR_GPIO40_Pos (8UL) /*!< GPIO INT1CLR: GPIO40 (Bit 8) */ +#define GPIO_INT1CLR_GPIO40_Msk (0x100UL) /*!< GPIO INT1CLR: GPIO40 (Bitfield-Mask: 0x01) */ +#define GPIO_INT1CLR_GPIO39_Pos (7UL) /*!< GPIO INT1CLR: GPIO39 (Bit 7) */ +#define GPIO_INT1CLR_GPIO39_Msk (0x80UL) /*!< GPIO INT1CLR: GPIO39 (Bitfield-Mask: 0x01) */ +#define GPIO_INT1CLR_GPIO38_Pos (6UL) /*!< GPIO INT1CLR: GPIO38 (Bit 6) */ +#define GPIO_INT1CLR_GPIO38_Msk (0x40UL) /*!< GPIO INT1CLR: GPIO38 (Bitfield-Mask: 0x01) */ +#define GPIO_INT1CLR_GPIO37_Pos (5UL) /*!< GPIO INT1CLR: GPIO37 (Bit 5) */ +#define GPIO_INT1CLR_GPIO37_Msk (0x20UL) /*!< GPIO INT1CLR: GPIO37 (Bitfield-Mask: 0x01) */ +#define GPIO_INT1CLR_GPIO36_Pos (4UL) /*!< GPIO INT1CLR: GPIO36 (Bit 4) */ +#define GPIO_INT1CLR_GPIO36_Msk (0x10UL) /*!< GPIO INT1CLR: GPIO36 (Bitfield-Mask: 0x01) */ +#define GPIO_INT1CLR_GPIO35_Pos (3UL) /*!< GPIO INT1CLR: GPIO35 (Bit 3) */ +#define GPIO_INT1CLR_GPIO35_Msk (0x8UL) /*!< GPIO INT1CLR: GPIO35 (Bitfield-Mask: 0x01) */ +#define GPIO_INT1CLR_GPIO34_Pos (2UL) /*!< GPIO INT1CLR: GPIO34 (Bit 2) */ +#define GPIO_INT1CLR_GPIO34_Msk (0x4UL) /*!< GPIO INT1CLR: GPIO34 (Bitfield-Mask: 0x01) */ +#define GPIO_INT1CLR_GPIO33_Pos (1UL) /*!< GPIO INT1CLR: GPIO33 (Bit 1) */ +#define GPIO_INT1CLR_GPIO33_Msk (0x2UL) /*!< GPIO INT1CLR: GPIO33 (Bitfield-Mask: 0x01) */ +#define GPIO_INT1CLR_GPIO32_Pos (0UL) /*!< GPIO INT1CLR: GPIO32 (Bit 0) */ +#define GPIO_INT1CLR_GPIO32_Msk (0x1UL) /*!< GPIO INT1CLR: GPIO32 (Bitfield-Mask: 0x01) */ +/* ======================================================== INT1SET ======================================================== */ +#define GPIO_INT1SET_GPIO49_Pos (17UL) /*!< GPIO INT1SET: GPIO49 (Bit 17) */ +#define GPIO_INT1SET_GPIO49_Msk (0x20000UL) /*!< GPIO INT1SET: GPIO49 (Bitfield-Mask: 0x01) */ +#define GPIO_INT1SET_GPIO48_Pos (16UL) /*!< GPIO INT1SET: GPIO48 (Bit 16) */ +#define GPIO_INT1SET_GPIO48_Msk (0x10000UL) /*!< GPIO INT1SET: GPIO48 (Bitfield-Mask: 0x01) */ +#define GPIO_INT1SET_GPIO47_Pos (15UL) /*!< GPIO INT1SET: GPIO47 (Bit 15) */ +#define GPIO_INT1SET_GPIO47_Msk (0x8000UL) /*!< GPIO INT1SET: GPIO47 (Bitfield-Mask: 0x01) */ +#define GPIO_INT1SET_GPIO46_Pos (14UL) /*!< GPIO INT1SET: GPIO46 (Bit 14) */ +#define GPIO_INT1SET_GPIO46_Msk (0x4000UL) /*!< GPIO INT1SET: GPIO46 (Bitfield-Mask: 0x01) */ +#define GPIO_INT1SET_GPIO45_Pos (13UL) /*!< GPIO INT1SET: GPIO45 (Bit 13) */ +#define GPIO_INT1SET_GPIO45_Msk (0x2000UL) /*!< GPIO INT1SET: GPIO45 (Bitfield-Mask: 0x01) */ +#define GPIO_INT1SET_GPIO44_Pos (12UL) /*!< GPIO INT1SET: GPIO44 (Bit 12) */ +#define GPIO_INT1SET_GPIO44_Msk (0x1000UL) /*!< GPIO INT1SET: GPIO44 (Bitfield-Mask: 0x01) */ +#define GPIO_INT1SET_GPIO43_Pos (11UL) /*!< GPIO INT1SET: GPIO43 (Bit 11) */ +#define GPIO_INT1SET_GPIO43_Msk (0x800UL) /*!< GPIO INT1SET: GPIO43 (Bitfield-Mask: 0x01) */ +#define GPIO_INT1SET_GPIO42_Pos (10UL) /*!< GPIO INT1SET: GPIO42 (Bit 10) */ +#define GPIO_INT1SET_GPIO42_Msk (0x400UL) /*!< GPIO INT1SET: GPIO42 (Bitfield-Mask: 0x01) */ +#define GPIO_INT1SET_GPIO41_Pos (9UL) /*!< GPIO INT1SET: GPIO41 (Bit 9) */ +#define GPIO_INT1SET_GPIO41_Msk (0x200UL) /*!< GPIO INT1SET: GPIO41 (Bitfield-Mask: 0x01) */ +#define GPIO_INT1SET_GPIO40_Pos (8UL) /*!< GPIO INT1SET: GPIO40 (Bit 8) */ +#define GPIO_INT1SET_GPIO40_Msk (0x100UL) /*!< GPIO INT1SET: GPIO40 (Bitfield-Mask: 0x01) */ +#define GPIO_INT1SET_GPIO39_Pos (7UL) /*!< GPIO INT1SET: GPIO39 (Bit 7) */ +#define GPIO_INT1SET_GPIO39_Msk (0x80UL) /*!< GPIO INT1SET: GPIO39 (Bitfield-Mask: 0x01) */ +#define GPIO_INT1SET_GPIO38_Pos (6UL) /*!< GPIO INT1SET: GPIO38 (Bit 6) */ +#define GPIO_INT1SET_GPIO38_Msk (0x40UL) /*!< GPIO INT1SET: GPIO38 (Bitfield-Mask: 0x01) */ +#define GPIO_INT1SET_GPIO37_Pos (5UL) /*!< GPIO INT1SET: GPIO37 (Bit 5) */ +#define GPIO_INT1SET_GPIO37_Msk (0x20UL) /*!< GPIO INT1SET: GPIO37 (Bitfield-Mask: 0x01) */ +#define GPIO_INT1SET_GPIO36_Pos (4UL) /*!< GPIO INT1SET: GPIO36 (Bit 4) */ +#define GPIO_INT1SET_GPIO36_Msk (0x10UL) /*!< GPIO INT1SET: GPIO36 (Bitfield-Mask: 0x01) */ +#define GPIO_INT1SET_GPIO35_Pos (3UL) /*!< GPIO INT1SET: GPIO35 (Bit 3) */ +#define GPIO_INT1SET_GPIO35_Msk (0x8UL) /*!< GPIO INT1SET: GPIO35 (Bitfield-Mask: 0x01) */ +#define GPIO_INT1SET_GPIO34_Pos (2UL) /*!< GPIO INT1SET: GPIO34 (Bit 2) */ +#define GPIO_INT1SET_GPIO34_Msk (0x4UL) /*!< GPIO INT1SET: GPIO34 (Bitfield-Mask: 0x01) */ +#define GPIO_INT1SET_GPIO33_Pos (1UL) /*!< GPIO INT1SET: GPIO33 (Bit 1) */ +#define GPIO_INT1SET_GPIO33_Msk (0x2UL) /*!< GPIO INT1SET: GPIO33 (Bitfield-Mask: 0x01) */ +#define GPIO_INT1SET_GPIO32_Pos (0UL) /*!< GPIO INT1SET: GPIO32 (Bit 0) */ +#define GPIO_INT1SET_GPIO32_Msk (0x1UL) /*!< GPIO INT1SET: GPIO32 (Bitfield-Mask: 0x01) */ + + +/* =========================================================================================================================== */ +/* ================ IOM0 ================ */ +/* =========================================================================================================================== */ + +/* ========================================================= FIFO ========================================================== */ +#define IOM0_FIFO_FIFO_Pos (0UL) /*!< IOM0 FIFO: FIFO (Bit 0) */ +#define IOM0_FIFO_FIFO_Msk (0xffffffffUL) /*!< IOM0 FIFO: FIFO (Bitfield-Mask: 0xffffffff) */ +/* ======================================================== FIFOPTR ======================================================== */ +#define IOM0_FIFOPTR_FIFO1REM_Pos (24UL) /*!< IOM0 FIFOPTR: FIFO1REM (Bit 24) */ +#define IOM0_FIFOPTR_FIFO1REM_Msk (0xff000000UL) /*!< IOM0 FIFOPTR: FIFO1REM (Bitfield-Mask: 0xff) */ +#define IOM0_FIFOPTR_FIFO1SIZ_Pos (16UL) /*!< IOM0 FIFOPTR: FIFO1SIZ (Bit 16) */ +#define IOM0_FIFOPTR_FIFO1SIZ_Msk (0xff0000UL) /*!< IOM0 FIFOPTR: FIFO1SIZ (Bitfield-Mask: 0xff) */ +#define IOM0_FIFOPTR_FIFO0REM_Pos (8UL) /*!< IOM0 FIFOPTR: FIFO0REM (Bit 8) */ +#define IOM0_FIFOPTR_FIFO0REM_Msk (0xff00UL) /*!< IOM0 FIFOPTR: FIFO0REM (Bitfield-Mask: 0xff) */ +#define IOM0_FIFOPTR_FIFO0SIZ_Pos (0UL) /*!< IOM0 FIFOPTR: FIFO0SIZ (Bit 0) */ +#define IOM0_FIFOPTR_FIFO0SIZ_Msk (0xffUL) /*!< IOM0 FIFOPTR: FIFO0SIZ (Bitfield-Mask: 0xff) */ +/* ======================================================== FIFOTHR ======================================================== */ +#define IOM0_FIFOTHR_FIFOWTHR_Pos (8UL) /*!< IOM0 FIFOTHR: FIFOWTHR (Bit 8) */ +#define IOM0_FIFOTHR_FIFOWTHR_Msk (0x3f00UL) /*!< IOM0 FIFOTHR: FIFOWTHR (Bitfield-Mask: 0x3f) */ +#define IOM0_FIFOTHR_FIFORTHR_Pos (0UL) /*!< IOM0 FIFOTHR: FIFORTHR (Bit 0) */ +#define IOM0_FIFOTHR_FIFORTHR_Msk (0x3fUL) /*!< IOM0 FIFOTHR: FIFORTHR (Bitfield-Mask: 0x3f) */ +/* ======================================================== FIFOPOP ======================================================== */ +#define IOM0_FIFOPOP_FIFODOUT_Pos (0UL) /*!< IOM0 FIFOPOP: FIFODOUT (Bit 0) */ +#define IOM0_FIFOPOP_FIFODOUT_Msk (0xffffffffUL) /*!< IOM0 FIFOPOP: FIFODOUT (Bitfield-Mask: 0xffffffff) */ +/* ======================================================= FIFOPUSH ======================================================== */ +#define IOM0_FIFOPUSH_FIFODIN_Pos (0UL) /*!< IOM0 FIFOPUSH: FIFODIN (Bit 0) */ +#define IOM0_FIFOPUSH_FIFODIN_Msk (0xffffffffUL) /*!< IOM0 FIFOPUSH: FIFODIN (Bitfield-Mask: 0xffffffff) */ +/* ======================================================= FIFOCTRL ======================================================== */ +#define IOM0_FIFOCTRL_FIFORSTN_Pos (1UL) /*!< IOM0 FIFOCTRL: FIFORSTN (Bit 1) */ +#define IOM0_FIFOCTRL_FIFORSTN_Msk (0x2UL) /*!< IOM0 FIFOCTRL: FIFORSTN (Bitfield-Mask: 0x01) */ +#define IOM0_FIFOCTRL_POPWR_Pos (0UL) /*!< IOM0 FIFOCTRL: POPWR (Bit 0) */ +#define IOM0_FIFOCTRL_POPWR_Msk (0x1UL) /*!< IOM0 FIFOCTRL: POPWR (Bitfield-Mask: 0x01) */ +/* ======================================================== FIFOLOC ======================================================== */ +#define IOM0_FIFOLOC_FIFORPTR_Pos (8UL) /*!< IOM0 FIFOLOC: FIFORPTR (Bit 8) */ +#define IOM0_FIFOLOC_FIFORPTR_Msk (0xf00UL) /*!< IOM0 FIFOLOC: FIFORPTR (Bitfield-Mask: 0x0f) */ +#define IOM0_FIFOLOC_FIFOWPTR_Pos (0UL) /*!< IOM0 FIFOLOC: FIFOWPTR (Bit 0) */ +#define IOM0_FIFOLOC_FIFOWPTR_Msk (0xfUL) /*!< IOM0 FIFOLOC: FIFOWPTR (Bitfield-Mask: 0x0f) */ +/* ========================================================= INTEN ========================================================= */ +#define IOM0_INTEN_CQERR_Pos (14UL) /*!< IOM0 INTEN: CQERR (Bit 14) */ +#define IOM0_INTEN_CQERR_Msk (0x4000UL) /*!< IOM0 INTEN: CQERR (Bitfield-Mask: 0x01) */ +#define IOM0_INTEN_CQUPD_Pos (13UL) /*!< IOM0 INTEN: CQUPD (Bit 13) */ +#define IOM0_INTEN_CQUPD_Msk (0x2000UL) /*!< IOM0 INTEN: CQUPD (Bitfield-Mask: 0x01) */ +#define IOM0_INTEN_CQPAUSED_Pos (12UL) /*!< IOM0 INTEN: CQPAUSED (Bit 12) */ +#define IOM0_INTEN_CQPAUSED_Msk (0x1000UL) /*!< IOM0 INTEN: CQPAUSED (Bitfield-Mask: 0x01) */ +#define IOM0_INTEN_DERR_Pos (11UL) /*!< IOM0 INTEN: DERR (Bit 11) */ +#define IOM0_INTEN_DERR_Msk (0x800UL) /*!< IOM0 INTEN: DERR (Bitfield-Mask: 0x01) */ +#define IOM0_INTEN_DCMP_Pos (10UL) /*!< IOM0 INTEN: DCMP (Bit 10) */ +#define IOM0_INTEN_DCMP_Msk (0x400UL) /*!< IOM0 INTEN: DCMP (Bitfield-Mask: 0x01) */ +#define IOM0_INTEN_ARB_Pos (9UL) /*!< IOM0 INTEN: ARB (Bit 9) */ +#define IOM0_INTEN_ARB_Msk (0x200UL) /*!< IOM0 INTEN: ARB (Bitfield-Mask: 0x01) */ +#define IOM0_INTEN_STOP_Pos (8UL) /*!< IOM0 INTEN: STOP (Bit 8) */ +#define IOM0_INTEN_STOP_Msk (0x100UL) /*!< IOM0 INTEN: STOP (Bitfield-Mask: 0x01) */ +#define IOM0_INTEN_START_Pos (7UL) /*!< IOM0 INTEN: START (Bit 7) */ +#define IOM0_INTEN_START_Msk (0x80UL) /*!< IOM0 INTEN: START (Bitfield-Mask: 0x01) */ +#define IOM0_INTEN_ICMD_Pos (6UL) /*!< IOM0 INTEN: ICMD (Bit 6) */ +#define IOM0_INTEN_ICMD_Msk (0x40UL) /*!< IOM0 INTEN: ICMD (Bitfield-Mask: 0x01) */ +#define IOM0_INTEN_IACC_Pos (5UL) /*!< IOM0 INTEN: IACC (Bit 5) */ +#define IOM0_INTEN_IACC_Msk (0x20UL) /*!< IOM0 INTEN: IACC (Bitfield-Mask: 0x01) */ +#define IOM0_INTEN_NAK_Pos (4UL) /*!< IOM0 INTEN: NAK (Bit 4) */ +#define IOM0_INTEN_NAK_Msk (0x10UL) /*!< IOM0 INTEN: NAK (Bitfield-Mask: 0x01) */ +#define IOM0_INTEN_FOVFL_Pos (3UL) /*!< IOM0 INTEN: FOVFL (Bit 3) */ +#define IOM0_INTEN_FOVFL_Msk (0x8UL) /*!< IOM0 INTEN: FOVFL (Bitfield-Mask: 0x01) */ +#define IOM0_INTEN_FUNDFL_Pos (2UL) /*!< IOM0 INTEN: FUNDFL (Bit 2) */ +#define IOM0_INTEN_FUNDFL_Msk (0x4UL) /*!< IOM0 INTEN: FUNDFL (Bitfield-Mask: 0x01) */ +#define IOM0_INTEN_THR_Pos (1UL) /*!< IOM0 INTEN: THR (Bit 1) */ +#define IOM0_INTEN_THR_Msk (0x2UL) /*!< IOM0 INTEN: THR (Bitfield-Mask: 0x01) */ +#define IOM0_INTEN_CMDCMP_Pos (0UL) /*!< IOM0 INTEN: CMDCMP (Bit 0) */ +#define IOM0_INTEN_CMDCMP_Msk (0x1UL) /*!< IOM0 INTEN: CMDCMP (Bitfield-Mask: 0x01) */ +/* ======================================================== INTSTAT ======================================================== */ +#define IOM0_INTSTAT_CQERR_Pos (14UL) /*!< IOM0 INTSTAT: CQERR (Bit 14) */ +#define IOM0_INTSTAT_CQERR_Msk (0x4000UL) /*!< IOM0 INTSTAT: CQERR (Bitfield-Mask: 0x01) */ +#define IOM0_INTSTAT_CQUPD_Pos (13UL) /*!< IOM0 INTSTAT: CQUPD (Bit 13) */ +#define IOM0_INTSTAT_CQUPD_Msk (0x2000UL) /*!< IOM0 INTSTAT: CQUPD (Bitfield-Mask: 0x01) */ +#define IOM0_INTSTAT_CQPAUSED_Pos (12UL) /*!< IOM0 INTSTAT: CQPAUSED (Bit 12) */ +#define IOM0_INTSTAT_CQPAUSED_Msk (0x1000UL) /*!< IOM0 INTSTAT: CQPAUSED (Bitfield-Mask: 0x01) */ +#define IOM0_INTSTAT_DERR_Pos (11UL) /*!< IOM0 INTSTAT: DERR (Bit 11) */ +#define IOM0_INTSTAT_DERR_Msk (0x800UL) /*!< IOM0 INTSTAT: DERR (Bitfield-Mask: 0x01) */ +#define IOM0_INTSTAT_DCMP_Pos (10UL) /*!< IOM0 INTSTAT: DCMP (Bit 10) */ +#define IOM0_INTSTAT_DCMP_Msk (0x400UL) /*!< IOM0 INTSTAT: DCMP (Bitfield-Mask: 0x01) */ +#define IOM0_INTSTAT_ARB_Pos (9UL) /*!< IOM0 INTSTAT: ARB (Bit 9) */ +#define IOM0_INTSTAT_ARB_Msk (0x200UL) /*!< IOM0 INTSTAT: ARB (Bitfield-Mask: 0x01) */ +#define IOM0_INTSTAT_STOP_Pos (8UL) /*!< IOM0 INTSTAT: STOP (Bit 8) */ +#define IOM0_INTSTAT_STOP_Msk (0x100UL) /*!< IOM0 INTSTAT: STOP (Bitfield-Mask: 0x01) */ +#define IOM0_INTSTAT_START_Pos (7UL) /*!< IOM0 INTSTAT: START (Bit 7) */ +#define IOM0_INTSTAT_START_Msk (0x80UL) /*!< IOM0 INTSTAT: START (Bitfield-Mask: 0x01) */ +#define IOM0_INTSTAT_ICMD_Pos (6UL) /*!< IOM0 INTSTAT: ICMD (Bit 6) */ +#define IOM0_INTSTAT_ICMD_Msk (0x40UL) /*!< IOM0 INTSTAT: ICMD (Bitfield-Mask: 0x01) */ +#define IOM0_INTSTAT_IACC_Pos (5UL) /*!< IOM0 INTSTAT: IACC (Bit 5) */ +#define IOM0_INTSTAT_IACC_Msk (0x20UL) /*!< IOM0 INTSTAT: IACC (Bitfield-Mask: 0x01) */ +#define IOM0_INTSTAT_NAK_Pos (4UL) /*!< IOM0 INTSTAT: NAK (Bit 4) */ +#define IOM0_INTSTAT_NAK_Msk (0x10UL) /*!< IOM0 INTSTAT: NAK (Bitfield-Mask: 0x01) */ +#define IOM0_INTSTAT_FOVFL_Pos (3UL) /*!< IOM0 INTSTAT: FOVFL (Bit 3) */ +#define IOM0_INTSTAT_FOVFL_Msk (0x8UL) /*!< IOM0 INTSTAT: FOVFL (Bitfield-Mask: 0x01) */ +#define IOM0_INTSTAT_FUNDFL_Pos (2UL) /*!< IOM0 INTSTAT: FUNDFL (Bit 2) */ +#define IOM0_INTSTAT_FUNDFL_Msk (0x4UL) /*!< IOM0 INTSTAT: FUNDFL (Bitfield-Mask: 0x01) */ +#define IOM0_INTSTAT_THR_Pos (1UL) /*!< IOM0 INTSTAT: THR (Bit 1) */ +#define IOM0_INTSTAT_THR_Msk (0x2UL) /*!< IOM0 INTSTAT: THR (Bitfield-Mask: 0x01) */ +#define IOM0_INTSTAT_CMDCMP_Pos (0UL) /*!< IOM0 INTSTAT: CMDCMP (Bit 0) */ +#define IOM0_INTSTAT_CMDCMP_Msk (0x1UL) /*!< IOM0 INTSTAT: CMDCMP (Bitfield-Mask: 0x01) */ +/* ======================================================== INTCLR ========================================================= */ +#define IOM0_INTCLR_CQERR_Pos (14UL) /*!< IOM0 INTCLR: CQERR (Bit 14) */ +#define IOM0_INTCLR_CQERR_Msk (0x4000UL) /*!< IOM0 INTCLR: CQERR (Bitfield-Mask: 0x01) */ +#define IOM0_INTCLR_CQUPD_Pos (13UL) /*!< IOM0 INTCLR: CQUPD (Bit 13) */ +#define IOM0_INTCLR_CQUPD_Msk (0x2000UL) /*!< IOM0 INTCLR: CQUPD (Bitfield-Mask: 0x01) */ +#define IOM0_INTCLR_CQPAUSED_Pos (12UL) /*!< IOM0 INTCLR: CQPAUSED (Bit 12) */ +#define IOM0_INTCLR_CQPAUSED_Msk (0x1000UL) /*!< IOM0 INTCLR: CQPAUSED (Bitfield-Mask: 0x01) */ +#define IOM0_INTCLR_DERR_Pos (11UL) /*!< IOM0 INTCLR: DERR (Bit 11) */ +#define IOM0_INTCLR_DERR_Msk (0x800UL) /*!< IOM0 INTCLR: DERR (Bitfield-Mask: 0x01) */ +#define IOM0_INTCLR_DCMP_Pos (10UL) /*!< IOM0 INTCLR: DCMP (Bit 10) */ +#define IOM0_INTCLR_DCMP_Msk (0x400UL) /*!< IOM0 INTCLR: DCMP (Bitfield-Mask: 0x01) */ +#define IOM0_INTCLR_ARB_Pos (9UL) /*!< IOM0 INTCLR: ARB (Bit 9) */ +#define IOM0_INTCLR_ARB_Msk (0x200UL) /*!< IOM0 INTCLR: ARB (Bitfield-Mask: 0x01) */ +#define IOM0_INTCLR_STOP_Pos (8UL) /*!< IOM0 INTCLR: STOP (Bit 8) */ +#define IOM0_INTCLR_STOP_Msk (0x100UL) /*!< IOM0 INTCLR: STOP (Bitfield-Mask: 0x01) */ +#define IOM0_INTCLR_START_Pos (7UL) /*!< IOM0 INTCLR: START (Bit 7) */ +#define IOM0_INTCLR_START_Msk (0x80UL) /*!< IOM0 INTCLR: START (Bitfield-Mask: 0x01) */ +#define IOM0_INTCLR_ICMD_Pos (6UL) /*!< IOM0 INTCLR: ICMD (Bit 6) */ +#define IOM0_INTCLR_ICMD_Msk (0x40UL) /*!< IOM0 INTCLR: ICMD (Bitfield-Mask: 0x01) */ +#define IOM0_INTCLR_IACC_Pos (5UL) /*!< IOM0 INTCLR: IACC (Bit 5) */ +#define IOM0_INTCLR_IACC_Msk (0x20UL) /*!< IOM0 INTCLR: IACC (Bitfield-Mask: 0x01) */ +#define IOM0_INTCLR_NAK_Pos (4UL) /*!< IOM0 INTCLR: NAK (Bit 4) */ +#define IOM0_INTCLR_NAK_Msk (0x10UL) /*!< IOM0 INTCLR: NAK (Bitfield-Mask: 0x01) */ +#define IOM0_INTCLR_FOVFL_Pos (3UL) /*!< IOM0 INTCLR: FOVFL (Bit 3) */ +#define IOM0_INTCLR_FOVFL_Msk (0x8UL) /*!< IOM0 INTCLR: FOVFL (Bitfield-Mask: 0x01) */ +#define IOM0_INTCLR_FUNDFL_Pos (2UL) /*!< IOM0 INTCLR: FUNDFL (Bit 2) */ +#define IOM0_INTCLR_FUNDFL_Msk (0x4UL) /*!< IOM0 INTCLR: FUNDFL (Bitfield-Mask: 0x01) */ +#define IOM0_INTCLR_THR_Pos (1UL) /*!< IOM0 INTCLR: THR (Bit 1) */ +#define IOM0_INTCLR_THR_Msk (0x2UL) /*!< IOM0 INTCLR: THR (Bitfield-Mask: 0x01) */ +#define IOM0_INTCLR_CMDCMP_Pos (0UL) /*!< IOM0 INTCLR: CMDCMP (Bit 0) */ +#define IOM0_INTCLR_CMDCMP_Msk (0x1UL) /*!< IOM0 INTCLR: CMDCMP (Bitfield-Mask: 0x01) */ +/* ======================================================== INTSET ========================================================= */ +#define IOM0_INTSET_CQERR_Pos (14UL) /*!< IOM0 INTSET: CQERR (Bit 14) */ +#define IOM0_INTSET_CQERR_Msk (0x4000UL) /*!< IOM0 INTSET: CQERR (Bitfield-Mask: 0x01) */ +#define IOM0_INTSET_CQUPD_Pos (13UL) /*!< IOM0 INTSET: CQUPD (Bit 13) */ +#define IOM0_INTSET_CQUPD_Msk (0x2000UL) /*!< IOM0 INTSET: CQUPD (Bitfield-Mask: 0x01) */ +#define IOM0_INTSET_CQPAUSED_Pos (12UL) /*!< IOM0 INTSET: CQPAUSED (Bit 12) */ +#define IOM0_INTSET_CQPAUSED_Msk (0x1000UL) /*!< IOM0 INTSET: CQPAUSED (Bitfield-Mask: 0x01) */ +#define IOM0_INTSET_DERR_Pos (11UL) /*!< IOM0 INTSET: DERR (Bit 11) */ +#define IOM0_INTSET_DERR_Msk (0x800UL) /*!< IOM0 INTSET: DERR (Bitfield-Mask: 0x01) */ +#define IOM0_INTSET_DCMP_Pos (10UL) /*!< IOM0 INTSET: DCMP (Bit 10) */ +#define IOM0_INTSET_DCMP_Msk (0x400UL) /*!< IOM0 INTSET: DCMP (Bitfield-Mask: 0x01) */ +#define IOM0_INTSET_ARB_Pos (9UL) /*!< IOM0 INTSET: ARB (Bit 9) */ +#define IOM0_INTSET_ARB_Msk (0x200UL) /*!< IOM0 INTSET: ARB (Bitfield-Mask: 0x01) */ +#define IOM0_INTSET_STOP_Pos (8UL) /*!< IOM0 INTSET: STOP (Bit 8) */ +#define IOM0_INTSET_STOP_Msk (0x100UL) /*!< IOM0 INTSET: STOP (Bitfield-Mask: 0x01) */ +#define IOM0_INTSET_START_Pos (7UL) /*!< IOM0 INTSET: START (Bit 7) */ +#define IOM0_INTSET_START_Msk (0x80UL) /*!< IOM0 INTSET: START (Bitfield-Mask: 0x01) */ +#define IOM0_INTSET_ICMD_Pos (6UL) /*!< IOM0 INTSET: ICMD (Bit 6) */ +#define IOM0_INTSET_ICMD_Msk (0x40UL) /*!< IOM0 INTSET: ICMD (Bitfield-Mask: 0x01) */ +#define IOM0_INTSET_IACC_Pos (5UL) /*!< IOM0 INTSET: IACC (Bit 5) */ +#define IOM0_INTSET_IACC_Msk (0x20UL) /*!< IOM0 INTSET: IACC (Bitfield-Mask: 0x01) */ +#define IOM0_INTSET_NAK_Pos (4UL) /*!< IOM0 INTSET: NAK (Bit 4) */ +#define IOM0_INTSET_NAK_Msk (0x10UL) /*!< IOM0 INTSET: NAK (Bitfield-Mask: 0x01) */ +#define IOM0_INTSET_FOVFL_Pos (3UL) /*!< IOM0 INTSET: FOVFL (Bit 3) */ +#define IOM0_INTSET_FOVFL_Msk (0x8UL) /*!< IOM0 INTSET: FOVFL (Bitfield-Mask: 0x01) */ +#define IOM0_INTSET_FUNDFL_Pos (2UL) /*!< IOM0 INTSET: FUNDFL (Bit 2) */ +#define IOM0_INTSET_FUNDFL_Msk (0x4UL) /*!< IOM0 INTSET: FUNDFL (Bitfield-Mask: 0x01) */ +#define IOM0_INTSET_THR_Pos (1UL) /*!< IOM0 INTSET: THR (Bit 1) */ +#define IOM0_INTSET_THR_Msk (0x2UL) /*!< IOM0 INTSET: THR (Bitfield-Mask: 0x01) */ +#define IOM0_INTSET_CMDCMP_Pos (0UL) /*!< IOM0 INTSET: CMDCMP (Bit 0) */ +#define IOM0_INTSET_CMDCMP_Msk (0x1UL) /*!< IOM0 INTSET: CMDCMP (Bitfield-Mask: 0x01) */ +/* ======================================================== CLKCFG ========================================================= */ +#define IOM0_CLKCFG_TOTPER_Pos (24UL) /*!< IOM0 CLKCFG: TOTPER (Bit 24) */ +#define IOM0_CLKCFG_TOTPER_Msk (0xff000000UL) /*!< IOM0 CLKCFG: TOTPER (Bitfield-Mask: 0xff) */ +#define IOM0_CLKCFG_LOWPER_Pos (16UL) /*!< IOM0 CLKCFG: LOWPER (Bit 16) */ +#define IOM0_CLKCFG_LOWPER_Msk (0xff0000UL) /*!< IOM0 CLKCFG: LOWPER (Bitfield-Mask: 0xff) */ +#define IOM0_CLKCFG_DIVEN_Pos (12UL) /*!< IOM0 CLKCFG: DIVEN (Bit 12) */ +#define IOM0_CLKCFG_DIVEN_Msk (0x1000UL) /*!< IOM0 CLKCFG: DIVEN (Bitfield-Mask: 0x01) */ +#define IOM0_CLKCFG_DIV3_Pos (11UL) /*!< IOM0 CLKCFG: DIV3 (Bit 11) */ +#define IOM0_CLKCFG_DIV3_Msk (0x800UL) /*!< IOM0 CLKCFG: DIV3 (Bitfield-Mask: 0x01) */ +#define IOM0_CLKCFG_FSEL_Pos (8UL) /*!< IOM0 CLKCFG: FSEL (Bit 8) */ +#define IOM0_CLKCFG_FSEL_Msk (0x700UL) /*!< IOM0 CLKCFG: FSEL (Bitfield-Mask: 0x07) */ +#define IOM0_CLKCFG_IOCLKEN_Pos (0UL) /*!< IOM0 CLKCFG: IOCLKEN (Bit 0) */ +#define IOM0_CLKCFG_IOCLKEN_Msk (0x1UL) /*!< IOM0 CLKCFG: IOCLKEN (Bitfield-Mask: 0x01) */ +/* ====================================================== SUBMODCTRL ======================================================= */ +#define IOM0_SUBMODCTRL_SMOD1TYPE_Pos (5UL) /*!< IOM0 SUBMODCTRL: SMOD1TYPE (Bit 5) */ +#define IOM0_SUBMODCTRL_SMOD1TYPE_Msk (0xe0UL) /*!< IOM0 SUBMODCTRL: SMOD1TYPE (Bitfield-Mask: 0x07) */ +#define IOM0_SUBMODCTRL_SMOD1EN_Pos (4UL) /*!< IOM0 SUBMODCTRL: SMOD1EN (Bit 4) */ +#define IOM0_SUBMODCTRL_SMOD1EN_Msk (0x10UL) /*!< IOM0 SUBMODCTRL: SMOD1EN (Bitfield-Mask: 0x01) */ +#define IOM0_SUBMODCTRL_SMOD0TYPE_Pos (1UL) /*!< IOM0 SUBMODCTRL: SMOD0TYPE (Bit 1) */ +#define IOM0_SUBMODCTRL_SMOD0TYPE_Msk (0xeUL) /*!< IOM0 SUBMODCTRL: SMOD0TYPE (Bitfield-Mask: 0x07) */ +#define IOM0_SUBMODCTRL_SMOD0EN_Pos (0UL) /*!< IOM0 SUBMODCTRL: SMOD0EN (Bit 0) */ +#define IOM0_SUBMODCTRL_SMOD0EN_Msk (0x1UL) /*!< IOM0 SUBMODCTRL: SMOD0EN (Bitfield-Mask: 0x01) */ +/* ========================================================== CMD ========================================================== */ +#define IOM0_CMD_OFFSETLO_Pos (24UL) /*!< IOM0 CMD: OFFSETLO (Bit 24) */ +#define IOM0_CMD_OFFSETLO_Msk (0xff000000UL) /*!< IOM0 CMD: OFFSETLO (Bitfield-Mask: 0xff) */ +#define IOM0_CMD_CMDSEL_Pos (20UL) /*!< IOM0 CMD: CMDSEL (Bit 20) */ +#define IOM0_CMD_CMDSEL_Msk (0x300000UL) /*!< IOM0 CMD: CMDSEL (Bitfield-Mask: 0x03) */ +#define IOM0_CMD_TSIZE_Pos (8UL) /*!< IOM0 CMD: TSIZE (Bit 8) */ +#define IOM0_CMD_TSIZE_Msk (0xfff00UL) /*!< IOM0 CMD: TSIZE (Bitfield-Mask: 0xfff) */ +#define IOM0_CMD_CONT_Pos (7UL) /*!< IOM0 CMD: CONT (Bit 7) */ +#define IOM0_CMD_CONT_Msk (0x80UL) /*!< IOM0 CMD: CONT (Bitfield-Mask: 0x01) */ +#define IOM0_CMD_OFFSETCNT_Pos (5UL) /*!< IOM0 CMD: OFFSETCNT (Bit 5) */ +#define IOM0_CMD_OFFSETCNT_Msk (0x60UL) /*!< IOM0 CMD: OFFSETCNT (Bitfield-Mask: 0x03) */ +#define IOM0_CMD_CMD_Pos (0UL) /*!< IOM0 CMD: CMD (Bit 0) */ +#define IOM0_CMD_CMD_Msk (0x1fUL) /*!< IOM0 CMD: CMD (Bitfield-Mask: 0x1f) */ +/* ======================================================== CMDRPT ========================================================= */ +#define IOM0_CMDRPT_CMDRPT_Pos (0UL) /*!< IOM0 CMDRPT: CMDRPT (Bit 0) */ +#define IOM0_CMDRPT_CMDRPT_Msk (0x1fUL) /*!< IOM0 CMDRPT: CMDRPT (Bitfield-Mask: 0x1f) */ +/* ======================================================= OFFSETHI ======================================================== */ +#define IOM0_OFFSETHI_OFFSETHI_Pos (0UL) /*!< IOM0 OFFSETHI: OFFSETHI (Bit 0) */ +#define IOM0_OFFSETHI_OFFSETHI_Msk (0xffffUL) /*!< IOM0 OFFSETHI: OFFSETHI (Bitfield-Mask: 0xffff) */ +/* ======================================================== CMDSTAT ======================================================== */ +#define IOM0_CMDSTAT_CTSIZE_Pos (8UL) /*!< IOM0 CMDSTAT: CTSIZE (Bit 8) */ +#define IOM0_CMDSTAT_CTSIZE_Msk (0xfff00UL) /*!< IOM0 CMDSTAT: CTSIZE (Bitfield-Mask: 0xfff) */ +#define IOM0_CMDSTAT_CMDSTAT_Pos (5UL) /*!< IOM0 CMDSTAT: CMDSTAT (Bit 5) */ +#define IOM0_CMDSTAT_CMDSTAT_Msk (0xe0UL) /*!< IOM0 CMDSTAT: CMDSTAT (Bitfield-Mask: 0x07) */ +#define IOM0_CMDSTAT_CCMD_Pos (0UL) /*!< IOM0 CMDSTAT: CCMD (Bit 0) */ +#define IOM0_CMDSTAT_CCMD_Msk (0x1fUL) /*!< IOM0 CMDSTAT: CCMD (Bitfield-Mask: 0x1f) */ +/* ======================================================= DMATRIGEN ======================================================= */ +#define IOM0_DMATRIGEN_DTHREN_Pos (1UL) /*!< IOM0 DMATRIGEN: DTHREN (Bit 1) */ +#define IOM0_DMATRIGEN_DTHREN_Msk (0x2UL) /*!< IOM0 DMATRIGEN: DTHREN (Bitfield-Mask: 0x01) */ +#define IOM0_DMATRIGEN_DCMDCMPEN_Pos (0UL) /*!< IOM0 DMATRIGEN: DCMDCMPEN (Bit 0) */ +#define IOM0_DMATRIGEN_DCMDCMPEN_Msk (0x1UL) /*!< IOM0 DMATRIGEN: DCMDCMPEN (Bitfield-Mask: 0x01) */ +/* ====================================================== DMATRIGSTAT ====================================================== */ +#define IOM0_DMATRIGSTAT_DTOTCMP_Pos (2UL) /*!< IOM0 DMATRIGSTAT: DTOTCMP (Bit 2) */ +#define IOM0_DMATRIGSTAT_DTOTCMP_Msk (0x4UL) /*!< IOM0 DMATRIGSTAT: DTOTCMP (Bitfield-Mask: 0x01) */ +#define IOM0_DMATRIGSTAT_DTHR_Pos (1UL) /*!< IOM0 DMATRIGSTAT: DTHR (Bit 1) */ +#define IOM0_DMATRIGSTAT_DTHR_Msk (0x2UL) /*!< IOM0 DMATRIGSTAT: DTHR (Bitfield-Mask: 0x01) */ +#define IOM0_DMATRIGSTAT_DCMDCMP_Pos (0UL) /*!< IOM0 DMATRIGSTAT: DCMDCMP (Bit 0) */ +#define IOM0_DMATRIGSTAT_DCMDCMP_Msk (0x1UL) /*!< IOM0 DMATRIGSTAT: DCMDCMP (Bitfield-Mask: 0x01) */ +/* ======================================================== DMACFG ========================================================= */ +#define IOM0_DMACFG_DPWROFF_Pos (9UL) /*!< IOM0 DMACFG: DPWROFF (Bit 9) */ +#define IOM0_DMACFG_DPWROFF_Msk (0x200UL) /*!< IOM0 DMACFG: DPWROFF (Bitfield-Mask: 0x01) */ +#define IOM0_DMACFG_DMAPRI_Pos (8UL) /*!< IOM0 DMACFG: DMAPRI (Bit 8) */ +#define IOM0_DMACFG_DMAPRI_Msk (0x100UL) /*!< IOM0 DMACFG: DMAPRI (Bitfield-Mask: 0x01) */ +#define IOM0_DMACFG_DMADIR_Pos (1UL) /*!< IOM0 DMACFG: DMADIR (Bit 1) */ +#define IOM0_DMACFG_DMADIR_Msk (0x2UL) /*!< IOM0 DMACFG: DMADIR (Bitfield-Mask: 0x01) */ +#define IOM0_DMACFG_DMAEN_Pos (0UL) /*!< IOM0 DMACFG: DMAEN (Bit 0) */ +#define IOM0_DMACFG_DMAEN_Msk (0x1UL) /*!< IOM0 DMACFG: DMAEN (Bitfield-Mask: 0x01) */ +/* ====================================================== DMATOTCOUNT ====================================================== */ +#define IOM0_DMATOTCOUNT_TOTCOUNT_Pos (0UL) /*!< IOM0 DMATOTCOUNT: TOTCOUNT (Bit 0) */ +#define IOM0_DMATOTCOUNT_TOTCOUNT_Msk (0xfffUL) /*!< IOM0 DMATOTCOUNT: TOTCOUNT (Bitfield-Mask: 0xfff) */ +/* ====================================================== DMATARGADDR ====================================================== */ +#define IOM0_DMATARGADDR_TARGADDR28_Pos (28UL) /*!< IOM0 DMATARGADDR: TARGADDR28 (Bit 28) */ +#define IOM0_DMATARGADDR_TARGADDR28_Msk (0x10000000UL) /*!< IOM0 DMATARGADDR: TARGADDR28 (Bitfield-Mask: 0x01) */ +#define IOM0_DMATARGADDR_TARGADDR_Pos (0UL) /*!< IOM0 DMATARGADDR: TARGADDR (Bit 0) */ +#define IOM0_DMATARGADDR_TARGADDR_Msk (0xfffffUL) /*!< IOM0 DMATARGADDR: TARGADDR (Bitfield-Mask: 0xfffff) */ +/* ======================================================== DMASTAT ======================================================== */ +#define IOM0_DMASTAT_DMAERR_Pos (2UL) /*!< IOM0 DMASTAT: DMAERR (Bit 2) */ +#define IOM0_DMASTAT_DMAERR_Msk (0x4UL) /*!< IOM0 DMASTAT: DMAERR (Bitfield-Mask: 0x01) */ +#define IOM0_DMASTAT_DMACPL_Pos (1UL) /*!< IOM0 DMASTAT: DMACPL (Bit 1) */ +#define IOM0_DMASTAT_DMACPL_Msk (0x2UL) /*!< IOM0 DMASTAT: DMACPL (Bitfield-Mask: 0x01) */ +#define IOM0_DMASTAT_DMATIP_Pos (0UL) /*!< IOM0 DMASTAT: DMATIP (Bit 0) */ +#define IOM0_DMASTAT_DMATIP_Msk (0x1UL) /*!< IOM0 DMASTAT: DMATIP (Bitfield-Mask: 0x01) */ +/* ========================================================= CQCFG ========================================================= */ +#define IOM0_CQCFG_CQPRI_Pos (1UL) /*!< IOM0 CQCFG: CQPRI (Bit 1) */ +#define IOM0_CQCFG_CQPRI_Msk (0x2UL) /*!< IOM0 CQCFG: CQPRI (Bitfield-Mask: 0x01) */ +#define IOM0_CQCFG_CQEN_Pos (0UL) /*!< IOM0 CQCFG: CQEN (Bit 0) */ +#define IOM0_CQCFG_CQEN_Msk (0x1UL) /*!< IOM0 CQCFG: CQEN (Bitfield-Mask: 0x01) */ +/* ======================================================== CQADDR ========================================================= */ +#define IOM0_CQADDR_CQADDR28_Pos (28UL) /*!< IOM0 CQADDR: CQADDR28 (Bit 28) */ +#define IOM0_CQADDR_CQADDR28_Msk (0x10000000UL) /*!< IOM0 CQADDR: CQADDR28 (Bitfield-Mask: 0x01) */ +#define IOM0_CQADDR_CQADDR_Pos (2UL) /*!< IOM0 CQADDR: CQADDR (Bit 2) */ +#define IOM0_CQADDR_CQADDR_Msk (0xffffcUL) /*!< IOM0 CQADDR: CQADDR (Bitfield-Mask: 0x3ffff) */ +/* ======================================================== CQSTAT ========================================================= */ +#define IOM0_CQSTAT_CQERR_Pos (2UL) /*!< IOM0 CQSTAT: CQERR (Bit 2) */ +#define IOM0_CQSTAT_CQERR_Msk (0x4UL) /*!< IOM0 CQSTAT: CQERR (Bitfield-Mask: 0x01) */ +#define IOM0_CQSTAT_CQPAUSED_Pos (1UL) /*!< IOM0 CQSTAT: CQPAUSED (Bit 1) */ +#define IOM0_CQSTAT_CQPAUSED_Msk (0x2UL) /*!< IOM0 CQSTAT: CQPAUSED (Bitfield-Mask: 0x01) */ +#define IOM0_CQSTAT_CQTIP_Pos (0UL) /*!< IOM0 CQSTAT: CQTIP (Bit 0) */ +#define IOM0_CQSTAT_CQTIP_Msk (0x1UL) /*!< IOM0 CQSTAT: CQTIP (Bitfield-Mask: 0x01) */ +/* ======================================================== CQFLAGS ======================================================== */ +#define IOM0_CQFLAGS_CQIRQMASK_Pos (16UL) /*!< IOM0 CQFLAGS: CQIRQMASK (Bit 16) */ +#define IOM0_CQFLAGS_CQIRQMASK_Msk (0xffff0000UL) /*!< IOM0 CQFLAGS: CQIRQMASK (Bitfield-Mask: 0xffff) */ +#define IOM0_CQFLAGS_CQFLAGS_Pos (0UL) /*!< IOM0 CQFLAGS: CQFLAGS (Bit 0) */ +#define IOM0_CQFLAGS_CQFLAGS_Msk (0xffffUL) /*!< IOM0 CQFLAGS: CQFLAGS (Bitfield-Mask: 0xffff) */ +/* ====================================================== CQSETCLEAR ======================================================= */ +#define IOM0_CQSETCLEAR_CQFCLR_Pos (16UL) /*!< IOM0 CQSETCLEAR: CQFCLR (Bit 16) */ +#define IOM0_CQSETCLEAR_CQFCLR_Msk (0xff0000UL) /*!< IOM0 CQSETCLEAR: CQFCLR (Bitfield-Mask: 0xff) */ +#define IOM0_CQSETCLEAR_CQFTGL_Pos (8UL) /*!< IOM0 CQSETCLEAR: CQFTGL (Bit 8) */ +#define IOM0_CQSETCLEAR_CQFTGL_Msk (0xff00UL) /*!< IOM0 CQSETCLEAR: CQFTGL (Bitfield-Mask: 0xff) */ +#define IOM0_CQSETCLEAR_CQFSET_Pos (0UL) /*!< IOM0 CQSETCLEAR: CQFSET (Bit 0) */ +#define IOM0_CQSETCLEAR_CQFSET_Msk (0xffUL) /*!< IOM0 CQSETCLEAR: CQFSET (Bitfield-Mask: 0xff) */ +/* ======================================================= CQPAUSEEN ======================================================= */ +#define IOM0_CQPAUSEEN_CQPEN_Pos (0UL) /*!< IOM0 CQPAUSEEN: CQPEN (Bit 0) */ +#define IOM0_CQPAUSEEN_CQPEN_Msk (0xffffUL) /*!< IOM0 CQPAUSEEN: CQPEN (Bitfield-Mask: 0xffff) */ +/* ======================================================= CQCURIDX ======================================================== */ +#define IOM0_CQCURIDX_CQCURIDX_Pos (0UL) /*!< IOM0 CQCURIDX: CQCURIDX (Bit 0) */ +#define IOM0_CQCURIDX_CQCURIDX_Msk (0xffUL) /*!< IOM0 CQCURIDX: CQCURIDX (Bitfield-Mask: 0xff) */ +/* ======================================================= CQENDIDX ======================================================== */ +#define IOM0_CQENDIDX_CQENDIDX_Pos (0UL) /*!< IOM0 CQENDIDX: CQENDIDX (Bit 0) */ +#define IOM0_CQENDIDX_CQENDIDX_Msk (0xffUL) /*!< IOM0 CQENDIDX: CQENDIDX (Bitfield-Mask: 0xff) */ +/* ======================================================== STATUS ========================================================= */ +#define IOM0_STATUS_IDLEST_Pos (2UL) /*!< IOM0 STATUS: IDLEST (Bit 2) */ +#define IOM0_STATUS_IDLEST_Msk (0x4UL) /*!< IOM0 STATUS: IDLEST (Bitfield-Mask: 0x01) */ +#define IOM0_STATUS_CMDACT_Pos (1UL) /*!< IOM0 STATUS: CMDACT (Bit 1) */ +#define IOM0_STATUS_CMDACT_Msk (0x2UL) /*!< IOM0 STATUS: CMDACT (Bitfield-Mask: 0x01) */ +#define IOM0_STATUS_ERR_Pos (0UL) /*!< IOM0 STATUS: ERR (Bit 0) */ +#define IOM0_STATUS_ERR_Msk (0x1UL) /*!< IOM0 STATUS: ERR (Bitfield-Mask: 0x01) */ +/* ======================================================== MSPICFG ======================================================== */ +#define IOM0_MSPICFG_MSPIRST_Pos (30UL) /*!< IOM0 MSPICFG: MSPIRST (Bit 30) */ +#define IOM0_MSPICFG_MSPIRST_Msk (0x40000000UL) /*!< IOM0 MSPICFG: MSPIRST (Bitfield-Mask: 0x01) */ +#define IOM0_MSPICFG_DOUTDLY_Pos (27UL) /*!< IOM0 MSPICFG: DOUTDLY (Bit 27) */ +#define IOM0_MSPICFG_DOUTDLY_Msk (0x38000000UL) /*!< IOM0 MSPICFG: DOUTDLY (Bitfield-Mask: 0x07) */ +#define IOM0_MSPICFG_DINDLY_Pos (24UL) /*!< IOM0 MSPICFG: DINDLY (Bit 24) */ +#define IOM0_MSPICFG_DINDLY_Msk (0x7000000UL) /*!< IOM0 MSPICFG: DINDLY (Bitfield-Mask: 0x07) */ +#define IOM0_MSPICFG_SPILSB_Pos (23UL) /*!< IOM0 MSPICFG: SPILSB (Bit 23) */ +#define IOM0_MSPICFG_SPILSB_Msk (0x800000UL) /*!< IOM0 MSPICFG: SPILSB (Bitfield-Mask: 0x01) */ +#define IOM0_MSPICFG_RDFCPOL_Pos (22UL) /*!< IOM0 MSPICFG: RDFCPOL (Bit 22) */ +#define IOM0_MSPICFG_RDFCPOL_Msk (0x400000UL) /*!< IOM0 MSPICFG: RDFCPOL (Bitfield-Mask: 0x01) */ +#define IOM0_MSPICFG_WTFCPOL_Pos (21UL) /*!< IOM0 MSPICFG: WTFCPOL (Bit 21) */ +#define IOM0_MSPICFG_WTFCPOL_Msk (0x200000UL) /*!< IOM0 MSPICFG: WTFCPOL (Bitfield-Mask: 0x01) */ +#define IOM0_MSPICFG_WTFCIRQ_Pos (20UL) /*!< IOM0 MSPICFG: WTFCIRQ (Bit 20) */ +#define IOM0_MSPICFG_WTFCIRQ_Msk (0x100000UL) /*!< IOM0 MSPICFG: WTFCIRQ (Bitfield-Mask: 0x01) */ +#define IOM0_MSPICFG_MOSIINV_Pos (18UL) /*!< IOM0 MSPICFG: MOSIINV (Bit 18) */ +#define IOM0_MSPICFG_MOSIINV_Msk (0x40000UL) /*!< IOM0 MSPICFG: MOSIINV (Bitfield-Mask: 0x01) */ +#define IOM0_MSPICFG_RDFC_Pos (17UL) /*!< IOM0 MSPICFG: RDFC (Bit 17) */ +#define IOM0_MSPICFG_RDFC_Msk (0x20000UL) /*!< IOM0 MSPICFG: RDFC (Bitfield-Mask: 0x01) */ +#define IOM0_MSPICFG_WTFC_Pos (16UL) /*!< IOM0 MSPICFG: WTFC (Bit 16) */ +#define IOM0_MSPICFG_WTFC_Msk (0x10000UL) /*!< IOM0 MSPICFG: WTFC (Bitfield-Mask: 0x01) */ +#define IOM0_MSPICFG_FULLDUP_Pos (2UL) /*!< IOM0 MSPICFG: FULLDUP (Bit 2) */ +#define IOM0_MSPICFG_FULLDUP_Msk (0x4UL) /*!< IOM0 MSPICFG: FULLDUP (Bitfield-Mask: 0x01) */ +#define IOM0_MSPICFG_SPHA_Pos (1UL) /*!< IOM0 MSPICFG: SPHA (Bit 1) */ +#define IOM0_MSPICFG_SPHA_Msk (0x2UL) /*!< IOM0 MSPICFG: SPHA (Bitfield-Mask: 0x01) */ +#define IOM0_MSPICFG_SPOL_Pos (0UL) /*!< IOM0 MSPICFG: SPOL (Bit 0) */ +#define IOM0_MSPICFG_SPOL_Msk (0x1UL) /*!< IOM0 MSPICFG: SPOL (Bitfield-Mask: 0x01) */ +/* ======================================================== MI2CCFG ======================================================== */ +#define IOM0_MI2CCFG_STRDIS_Pos (24UL) /*!< IOM0 MI2CCFG: STRDIS (Bit 24) */ +#define IOM0_MI2CCFG_STRDIS_Msk (0x1000000UL) /*!< IOM0 MI2CCFG: STRDIS (Bitfield-Mask: 0x01) */ +#define IOM0_MI2CCFG_SMPCNT_Pos (16UL) /*!< IOM0 MI2CCFG: SMPCNT (Bit 16) */ +#define IOM0_MI2CCFG_SMPCNT_Msk (0xff0000UL) /*!< IOM0 MI2CCFG: SMPCNT (Bitfield-Mask: 0xff) */ +#define IOM0_MI2CCFG_SDAENDLY_Pos (12UL) /*!< IOM0 MI2CCFG: SDAENDLY (Bit 12) */ +#define IOM0_MI2CCFG_SDAENDLY_Msk (0xf000UL) /*!< IOM0 MI2CCFG: SDAENDLY (Bitfield-Mask: 0x0f) */ +#define IOM0_MI2CCFG_SCLENDLY_Pos (8UL) /*!< IOM0 MI2CCFG: SCLENDLY (Bit 8) */ +#define IOM0_MI2CCFG_SCLENDLY_Msk (0xf00UL) /*!< IOM0 MI2CCFG: SCLENDLY (Bitfield-Mask: 0x0f) */ +#define IOM0_MI2CCFG_MI2CRST_Pos (6UL) /*!< IOM0 MI2CCFG: MI2CRST (Bit 6) */ +#define IOM0_MI2CCFG_MI2CRST_Msk (0x40UL) /*!< IOM0 MI2CCFG: MI2CRST (Bitfield-Mask: 0x01) */ +#define IOM0_MI2CCFG_SDADLY_Pos (4UL) /*!< IOM0 MI2CCFG: SDADLY (Bit 4) */ +#define IOM0_MI2CCFG_SDADLY_Msk (0x30UL) /*!< IOM0 MI2CCFG: SDADLY (Bitfield-Mask: 0x03) */ +#define IOM0_MI2CCFG_ARBEN_Pos (2UL) /*!< IOM0 MI2CCFG: ARBEN (Bit 2) */ +#define IOM0_MI2CCFG_ARBEN_Msk (0x4UL) /*!< IOM0 MI2CCFG: ARBEN (Bitfield-Mask: 0x01) */ +#define IOM0_MI2CCFG_I2CLSB_Pos (1UL) /*!< IOM0 MI2CCFG: I2CLSB (Bit 1) */ +#define IOM0_MI2CCFG_I2CLSB_Msk (0x2UL) /*!< IOM0 MI2CCFG: I2CLSB (Bitfield-Mask: 0x01) */ +#define IOM0_MI2CCFG_ADDRSZ_Pos (0UL) /*!< IOM0 MI2CCFG: ADDRSZ (Bit 0) */ +#define IOM0_MI2CCFG_ADDRSZ_Msk (0x1UL) /*!< IOM0 MI2CCFG: ADDRSZ (Bitfield-Mask: 0x01) */ +/* ======================================================== DEVCFG ========================================================= */ +#define IOM0_DEVCFG_DEVADDR_Pos (0UL) /*!< IOM0 DEVCFG: DEVADDR (Bit 0) */ +#define IOM0_DEVCFG_DEVADDR_Msk (0x3ffUL) /*!< IOM0 DEVCFG: DEVADDR (Bitfield-Mask: 0x3ff) */ +/* ======================================================== IOMDBG ========================================================= */ +#define IOM0_IOMDBG_DBGDATA_Pos (3UL) /*!< IOM0 IOMDBG: DBGDATA (Bit 3) */ +#define IOM0_IOMDBG_DBGDATA_Msk (0xfffffff8UL) /*!< IOM0 IOMDBG: DBGDATA (Bitfield-Mask: 0x1fffffff) */ +#define IOM0_IOMDBG_APBCLKON_Pos (2UL) /*!< IOM0 IOMDBG: APBCLKON (Bit 2) */ +#define IOM0_IOMDBG_APBCLKON_Msk (0x4UL) /*!< IOM0 IOMDBG: APBCLKON (Bitfield-Mask: 0x01) */ +#define IOM0_IOMDBG_IOCLKON_Pos (1UL) /*!< IOM0 IOMDBG: IOCLKON (Bit 1) */ +#define IOM0_IOMDBG_IOCLKON_Msk (0x2UL) /*!< IOM0 IOMDBG: IOCLKON (Bitfield-Mask: 0x01) */ +#define IOM0_IOMDBG_DBGEN_Pos (0UL) /*!< IOM0 IOMDBG: DBGEN (Bit 0) */ +#define IOM0_IOMDBG_DBGEN_Msk (0x1UL) /*!< IOM0 IOMDBG: DBGEN (Bitfield-Mask: 0x01) */ + + +/* =========================================================================================================================== */ +/* ================ IOSLAVE ================ */ +/* =========================================================================================================================== */ + +/* ======================================================== FIFOPTR ======================================================== */ +#define IOSLAVE_FIFOPTR_FIFOSIZ_Pos (8UL) /*!< IOSLAVE FIFOPTR: FIFOSIZ (Bit 8) */ +#define IOSLAVE_FIFOPTR_FIFOSIZ_Msk (0xff00UL) /*!< IOSLAVE FIFOPTR: FIFOSIZ (Bitfield-Mask: 0xff) */ +#define IOSLAVE_FIFOPTR_FIFOPTR_Pos (0UL) /*!< IOSLAVE FIFOPTR: FIFOPTR (Bit 0) */ +#define IOSLAVE_FIFOPTR_FIFOPTR_Msk (0xffUL) /*!< IOSLAVE FIFOPTR: FIFOPTR (Bitfield-Mask: 0xff) */ +/* ======================================================== FIFOCFG ======================================================== */ +#define IOSLAVE_FIFOCFG_ROBASE_Pos (24UL) /*!< IOSLAVE FIFOCFG: ROBASE (Bit 24) */ +#define IOSLAVE_FIFOCFG_ROBASE_Msk (0x3f000000UL) /*!< IOSLAVE FIFOCFG: ROBASE (Bitfield-Mask: 0x3f) */ +#define IOSLAVE_FIFOCFG_FIFOMAX_Pos (8UL) /*!< IOSLAVE FIFOCFG: FIFOMAX (Bit 8) */ +#define IOSLAVE_FIFOCFG_FIFOMAX_Msk (0x3f00UL) /*!< IOSLAVE FIFOCFG: FIFOMAX (Bitfield-Mask: 0x3f) */ +#define IOSLAVE_FIFOCFG_FIFOBASE_Pos (0UL) /*!< IOSLAVE FIFOCFG: FIFOBASE (Bit 0) */ +#define IOSLAVE_FIFOCFG_FIFOBASE_Msk (0x1fUL) /*!< IOSLAVE FIFOCFG: FIFOBASE (Bitfield-Mask: 0x1f) */ +/* ======================================================== FIFOTHR ======================================================== */ +#define IOSLAVE_FIFOTHR_FIFOTHR_Pos (0UL) /*!< IOSLAVE FIFOTHR: FIFOTHR (Bit 0) */ +#define IOSLAVE_FIFOTHR_FIFOTHR_Msk (0xffUL) /*!< IOSLAVE FIFOTHR: FIFOTHR (Bitfield-Mask: 0xff) */ +/* ========================================================= FUPD ========================================================== */ +#define IOSLAVE_FUPD_IOREAD_Pos (1UL) /*!< IOSLAVE FUPD: IOREAD (Bit 1) */ +#define IOSLAVE_FUPD_IOREAD_Msk (0x2UL) /*!< IOSLAVE FUPD: IOREAD (Bitfield-Mask: 0x01) */ +#define IOSLAVE_FUPD_FIFOUPD_Pos (0UL) /*!< IOSLAVE FUPD: FIFOUPD (Bit 0) */ +#define IOSLAVE_FUPD_FIFOUPD_Msk (0x1UL) /*!< IOSLAVE FUPD: FIFOUPD (Bitfield-Mask: 0x01) */ +/* ======================================================== FIFOCTR ======================================================== */ +#define IOSLAVE_FIFOCTR_FIFOCTR_Pos (0UL) /*!< IOSLAVE FIFOCTR: FIFOCTR (Bit 0) */ +#define IOSLAVE_FIFOCTR_FIFOCTR_Msk (0x3ffUL) /*!< IOSLAVE FIFOCTR: FIFOCTR (Bitfield-Mask: 0x3ff) */ +/* ======================================================== FIFOINC ======================================================== */ +#define IOSLAVE_FIFOINC_FIFOINC_Pos (0UL) /*!< IOSLAVE FIFOINC: FIFOINC (Bit 0) */ +#define IOSLAVE_FIFOINC_FIFOINC_Msk (0x3ffUL) /*!< IOSLAVE FIFOINC: FIFOINC (Bitfield-Mask: 0x3ff) */ +/* ========================================================== CFG ========================================================== */ +#define IOSLAVE_CFG_IFCEN_Pos (31UL) /*!< IOSLAVE CFG: IFCEN (Bit 31) */ +#define IOSLAVE_CFG_IFCEN_Msk (0x80000000UL) /*!< IOSLAVE CFG: IFCEN (Bitfield-Mask: 0x01) */ +#define IOSLAVE_CFG_I2CADDR_Pos (8UL) /*!< IOSLAVE CFG: I2CADDR (Bit 8) */ +#define IOSLAVE_CFG_I2CADDR_Msk (0xfff00UL) /*!< IOSLAVE CFG: I2CADDR (Bitfield-Mask: 0xfff) */ +#define IOSLAVE_CFG_STARTRD_Pos (4UL) /*!< IOSLAVE CFG: STARTRD (Bit 4) */ +#define IOSLAVE_CFG_STARTRD_Msk (0x10UL) /*!< IOSLAVE CFG: STARTRD (Bitfield-Mask: 0x01) */ +#define IOSLAVE_CFG_LSB_Pos (2UL) /*!< IOSLAVE CFG: LSB (Bit 2) */ +#define IOSLAVE_CFG_LSB_Msk (0x4UL) /*!< IOSLAVE CFG: LSB (Bitfield-Mask: 0x01) */ +#define IOSLAVE_CFG_SPOL_Pos (1UL) /*!< IOSLAVE CFG: SPOL (Bit 1) */ +#define IOSLAVE_CFG_SPOL_Msk (0x2UL) /*!< IOSLAVE CFG: SPOL (Bitfield-Mask: 0x01) */ +#define IOSLAVE_CFG_IFCSEL_Pos (0UL) /*!< IOSLAVE CFG: IFCSEL (Bit 0) */ +#define IOSLAVE_CFG_IFCSEL_Msk (0x1UL) /*!< IOSLAVE CFG: IFCSEL (Bitfield-Mask: 0x01) */ +/* ========================================================= PRENC ========================================================= */ +#define IOSLAVE_PRENC_PRENC_Pos (0UL) /*!< IOSLAVE PRENC: PRENC (Bit 0) */ +#define IOSLAVE_PRENC_PRENC_Msk (0x1fUL) /*!< IOSLAVE PRENC: PRENC (Bitfield-Mask: 0x1f) */ +/* ======================================================= IOINTCTL ======================================================== */ +#define IOSLAVE_IOINTCTL_IOINTSET_Pos (24UL) /*!< IOSLAVE IOINTCTL: IOINTSET (Bit 24) */ +#define IOSLAVE_IOINTCTL_IOINTSET_Msk (0xff000000UL) /*!< IOSLAVE IOINTCTL: IOINTSET (Bitfield-Mask: 0xff) */ +#define IOSLAVE_IOINTCTL_IOINTCLR_Pos (16UL) /*!< IOSLAVE IOINTCTL: IOINTCLR (Bit 16) */ +#define IOSLAVE_IOINTCTL_IOINTCLR_Msk (0x10000UL) /*!< IOSLAVE IOINTCTL: IOINTCLR (Bitfield-Mask: 0x01) */ +#define IOSLAVE_IOINTCTL_IOINT_Pos (8UL) /*!< IOSLAVE IOINTCTL: IOINT (Bit 8) */ +#define IOSLAVE_IOINTCTL_IOINT_Msk (0xff00UL) /*!< IOSLAVE IOINTCTL: IOINT (Bitfield-Mask: 0xff) */ +#define IOSLAVE_IOINTCTL_IOINTEN_Pos (0UL) /*!< IOSLAVE IOINTCTL: IOINTEN (Bit 0) */ +#define IOSLAVE_IOINTCTL_IOINTEN_Msk (0xffUL) /*!< IOSLAVE IOINTCTL: IOINTEN (Bitfield-Mask: 0xff) */ +/* ======================================================== GENADD ========================================================= */ +#define IOSLAVE_GENADD_GADATA_Pos (0UL) /*!< IOSLAVE GENADD: GADATA (Bit 0) */ +#define IOSLAVE_GENADD_GADATA_Msk (0xffUL) /*!< IOSLAVE GENADD: GADATA (Bitfield-Mask: 0xff) */ +/* ========================================================= INTEN ========================================================= */ +#define IOSLAVE_INTEN_XCMPWR_Pos (9UL) /*!< IOSLAVE INTEN: XCMPWR (Bit 9) */ +#define IOSLAVE_INTEN_XCMPWR_Msk (0x200UL) /*!< IOSLAVE INTEN: XCMPWR (Bitfield-Mask: 0x01) */ +#define IOSLAVE_INTEN_XCMPWF_Pos (8UL) /*!< IOSLAVE INTEN: XCMPWF (Bit 8) */ +#define IOSLAVE_INTEN_XCMPWF_Msk (0x100UL) /*!< IOSLAVE INTEN: XCMPWF (Bitfield-Mask: 0x01) */ +#define IOSLAVE_INTEN_XCMPRR_Pos (7UL) /*!< IOSLAVE INTEN: XCMPRR (Bit 7) */ +#define IOSLAVE_INTEN_XCMPRR_Msk (0x80UL) /*!< IOSLAVE INTEN: XCMPRR (Bitfield-Mask: 0x01) */ +#define IOSLAVE_INTEN_XCMPRF_Pos (6UL) /*!< IOSLAVE INTEN: XCMPRF (Bit 6) */ +#define IOSLAVE_INTEN_XCMPRF_Msk (0x40UL) /*!< IOSLAVE INTEN: XCMPRF (Bitfield-Mask: 0x01) */ +#define IOSLAVE_INTEN_IOINTW_Pos (5UL) /*!< IOSLAVE INTEN: IOINTW (Bit 5) */ +#define IOSLAVE_INTEN_IOINTW_Msk (0x20UL) /*!< IOSLAVE INTEN: IOINTW (Bitfield-Mask: 0x01) */ +#define IOSLAVE_INTEN_GENAD_Pos (4UL) /*!< IOSLAVE INTEN: GENAD (Bit 4) */ +#define IOSLAVE_INTEN_GENAD_Msk (0x10UL) /*!< IOSLAVE INTEN: GENAD (Bitfield-Mask: 0x01) */ +#define IOSLAVE_INTEN_FRDERR_Pos (3UL) /*!< IOSLAVE INTEN: FRDERR (Bit 3) */ +#define IOSLAVE_INTEN_FRDERR_Msk (0x8UL) /*!< IOSLAVE INTEN: FRDERR (Bitfield-Mask: 0x01) */ +#define IOSLAVE_INTEN_FUNDFL_Pos (2UL) /*!< IOSLAVE INTEN: FUNDFL (Bit 2) */ +#define IOSLAVE_INTEN_FUNDFL_Msk (0x4UL) /*!< IOSLAVE INTEN: FUNDFL (Bitfield-Mask: 0x01) */ +#define IOSLAVE_INTEN_FOVFL_Pos (1UL) /*!< IOSLAVE INTEN: FOVFL (Bit 1) */ +#define IOSLAVE_INTEN_FOVFL_Msk (0x2UL) /*!< IOSLAVE INTEN: FOVFL (Bitfield-Mask: 0x01) */ +#define IOSLAVE_INTEN_FSIZE_Pos (0UL) /*!< IOSLAVE INTEN: FSIZE (Bit 0) */ +#define IOSLAVE_INTEN_FSIZE_Msk (0x1UL) /*!< IOSLAVE INTEN: FSIZE (Bitfield-Mask: 0x01) */ +/* ======================================================== INTSTAT ======================================================== */ +#define IOSLAVE_INTSTAT_XCMPWR_Pos (9UL) /*!< IOSLAVE INTSTAT: XCMPWR (Bit 9) */ +#define IOSLAVE_INTSTAT_XCMPWR_Msk (0x200UL) /*!< IOSLAVE INTSTAT: XCMPWR (Bitfield-Mask: 0x01) */ +#define IOSLAVE_INTSTAT_XCMPWF_Pos (8UL) /*!< IOSLAVE INTSTAT: XCMPWF (Bit 8) */ +#define IOSLAVE_INTSTAT_XCMPWF_Msk (0x100UL) /*!< IOSLAVE INTSTAT: XCMPWF (Bitfield-Mask: 0x01) */ +#define IOSLAVE_INTSTAT_XCMPRR_Pos (7UL) /*!< IOSLAVE INTSTAT: XCMPRR (Bit 7) */ +#define IOSLAVE_INTSTAT_XCMPRR_Msk (0x80UL) /*!< IOSLAVE INTSTAT: XCMPRR (Bitfield-Mask: 0x01) */ +#define IOSLAVE_INTSTAT_XCMPRF_Pos (6UL) /*!< IOSLAVE INTSTAT: XCMPRF (Bit 6) */ +#define IOSLAVE_INTSTAT_XCMPRF_Msk (0x40UL) /*!< IOSLAVE INTSTAT: XCMPRF (Bitfield-Mask: 0x01) */ +#define IOSLAVE_INTSTAT_IOINTW_Pos (5UL) /*!< IOSLAVE INTSTAT: IOINTW (Bit 5) */ +#define IOSLAVE_INTSTAT_IOINTW_Msk (0x20UL) /*!< IOSLAVE INTSTAT: IOINTW (Bitfield-Mask: 0x01) */ +#define IOSLAVE_INTSTAT_GENAD_Pos (4UL) /*!< IOSLAVE INTSTAT: GENAD (Bit 4) */ +#define IOSLAVE_INTSTAT_GENAD_Msk (0x10UL) /*!< IOSLAVE INTSTAT: GENAD (Bitfield-Mask: 0x01) */ +#define IOSLAVE_INTSTAT_FRDERR_Pos (3UL) /*!< IOSLAVE INTSTAT: FRDERR (Bit 3) */ +#define IOSLAVE_INTSTAT_FRDERR_Msk (0x8UL) /*!< IOSLAVE INTSTAT: FRDERR (Bitfield-Mask: 0x01) */ +#define IOSLAVE_INTSTAT_FUNDFL_Pos (2UL) /*!< IOSLAVE INTSTAT: FUNDFL (Bit 2) */ +#define IOSLAVE_INTSTAT_FUNDFL_Msk (0x4UL) /*!< IOSLAVE INTSTAT: FUNDFL (Bitfield-Mask: 0x01) */ +#define IOSLAVE_INTSTAT_FOVFL_Pos (1UL) /*!< IOSLAVE INTSTAT: FOVFL (Bit 1) */ +#define IOSLAVE_INTSTAT_FOVFL_Msk (0x2UL) /*!< IOSLAVE INTSTAT: FOVFL (Bitfield-Mask: 0x01) */ +#define IOSLAVE_INTSTAT_FSIZE_Pos (0UL) /*!< IOSLAVE INTSTAT: FSIZE (Bit 0) */ +#define IOSLAVE_INTSTAT_FSIZE_Msk (0x1UL) /*!< IOSLAVE INTSTAT: FSIZE (Bitfield-Mask: 0x01) */ +/* ======================================================== INTCLR ========================================================= */ +#define IOSLAVE_INTCLR_XCMPWR_Pos (9UL) /*!< IOSLAVE INTCLR: XCMPWR (Bit 9) */ +#define IOSLAVE_INTCLR_XCMPWR_Msk (0x200UL) /*!< IOSLAVE INTCLR: XCMPWR (Bitfield-Mask: 0x01) */ +#define IOSLAVE_INTCLR_XCMPWF_Pos (8UL) /*!< IOSLAVE INTCLR: XCMPWF (Bit 8) */ +#define IOSLAVE_INTCLR_XCMPWF_Msk (0x100UL) /*!< IOSLAVE INTCLR: XCMPWF (Bitfield-Mask: 0x01) */ +#define IOSLAVE_INTCLR_XCMPRR_Pos (7UL) /*!< IOSLAVE INTCLR: XCMPRR (Bit 7) */ +#define IOSLAVE_INTCLR_XCMPRR_Msk (0x80UL) /*!< IOSLAVE INTCLR: XCMPRR (Bitfield-Mask: 0x01) */ +#define IOSLAVE_INTCLR_XCMPRF_Pos (6UL) /*!< IOSLAVE INTCLR: XCMPRF (Bit 6) */ +#define IOSLAVE_INTCLR_XCMPRF_Msk (0x40UL) /*!< IOSLAVE INTCLR: XCMPRF (Bitfield-Mask: 0x01) */ +#define IOSLAVE_INTCLR_IOINTW_Pos (5UL) /*!< IOSLAVE INTCLR: IOINTW (Bit 5) */ +#define IOSLAVE_INTCLR_IOINTW_Msk (0x20UL) /*!< IOSLAVE INTCLR: IOINTW (Bitfield-Mask: 0x01) */ +#define IOSLAVE_INTCLR_GENAD_Pos (4UL) /*!< IOSLAVE INTCLR: GENAD (Bit 4) */ +#define IOSLAVE_INTCLR_GENAD_Msk (0x10UL) /*!< IOSLAVE INTCLR: GENAD (Bitfield-Mask: 0x01) */ +#define IOSLAVE_INTCLR_FRDERR_Pos (3UL) /*!< IOSLAVE INTCLR: FRDERR (Bit 3) */ +#define IOSLAVE_INTCLR_FRDERR_Msk (0x8UL) /*!< IOSLAVE INTCLR: FRDERR (Bitfield-Mask: 0x01) */ +#define IOSLAVE_INTCLR_FUNDFL_Pos (2UL) /*!< IOSLAVE INTCLR: FUNDFL (Bit 2) */ +#define IOSLAVE_INTCLR_FUNDFL_Msk (0x4UL) /*!< IOSLAVE INTCLR: FUNDFL (Bitfield-Mask: 0x01) */ +#define IOSLAVE_INTCLR_FOVFL_Pos (1UL) /*!< IOSLAVE INTCLR: FOVFL (Bit 1) */ +#define IOSLAVE_INTCLR_FOVFL_Msk (0x2UL) /*!< IOSLAVE INTCLR: FOVFL (Bitfield-Mask: 0x01) */ +#define IOSLAVE_INTCLR_FSIZE_Pos (0UL) /*!< IOSLAVE INTCLR: FSIZE (Bit 0) */ +#define IOSLAVE_INTCLR_FSIZE_Msk (0x1UL) /*!< IOSLAVE INTCLR: FSIZE (Bitfield-Mask: 0x01) */ +/* ======================================================== INTSET ========================================================= */ +#define IOSLAVE_INTSET_XCMPWR_Pos (9UL) /*!< IOSLAVE INTSET: XCMPWR (Bit 9) */ +#define IOSLAVE_INTSET_XCMPWR_Msk (0x200UL) /*!< IOSLAVE INTSET: XCMPWR (Bitfield-Mask: 0x01) */ +#define IOSLAVE_INTSET_XCMPWF_Pos (8UL) /*!< IOSLAVE INTSET: XCMPWF (Bit 8) */ +#define IOSLAVE_INTSET_XCMPWF_Msk (0x100UL) /*!< IOSLAVE INTSET: XCMPWF (Bitfield-Mask: 0x01) */ +#define IOSLAVE_INTSET_XCMPRR_Pos (7UL) /*!< IOSLAVE INTSET: XCMPRR (Bit 7) */ +#define IOSLAVE_INTSET_XCMPRR_Msk (0x80UL) /*!< IOSLAVE INTSET: XCMPRR (Bitfield-Mask: 0x01) */ +#define IOSLAVE_INTSET_XCMPRF_Pos (6UL) /*!< IOSLAVE INTSET: XCMPRF (Bit 6) */ +#define IOSLAVE_INTSET_XCMPRF_Msk (0x40UL) /*!< IOSLAVE INTSET: XCMPRF (Bitfield-Mask: 0x01) */ +#define IOSLAVE_INTSET_IOINTW_Pos (5UL) /*!< IOSLAVE INTSET: IOINTW (Bit 5) */ +#define IOSLAVE_INTSET_IOINTW_Msk (0x20UL) /*!< IOSLAVE INTSET: IOINTW (Bitfield-Mask: 0x01) */ +#define IOSLAVE_INTSET_GENAD_Pos (4UL) /*!< IOSLAVE INTSET: GENAD (Bit 4) */ +#define IOSLAVE_INTSET_GENAD_Msk (0x10UL) /*!< IOSLAVE INTSET: GENAD (Bitfield-Mask: 0x01) */ +#define IOSLAVE_INTSET_FRDERR_Pos (3UL) /*!< IOSLAVE INTSET: FRDERR (Bit 3) */ +#define IOSLAVE_INTSET_FRDERR_Msk (0x8UL) /*!< IOSLAVE INTSET: FRDERR (Bitfield-Mask: 0x01) */ +#define IOSLAVE_INTSET_FUNDFL_Pos (2UL) /*!< IOSLAVE INTSET: FUNDFL (Bit 2) */ +#define IOSLAVE_INTSET_FUNDFL_Msk (0x4UL) /*!< IOSLAVE INTSET: FUNDFL (Bitfield-Mask: 0x01) */ +#define IOSLAVE_INTSET_FOVFL_Pos (1UL) /*!< IOSLAVE INTSET: FOVFL (Bit 1) */ +#define IOSLAVE_INTSET_FOVFL_Msk (0x2UL) /*!< IOSLAVE INTSET: FOVFL (Bitfield-Mask: 0x01) */ +#define IOSLAVE_INTSET_FSIZE_Pos (0UL) /*!< IOSLAVE INTSET: FSIZE (Bit 0) */ +#define IOSLAVE_INTSET_FSIZE_Msk (0x1UL) /*!< IOSLAVE INTSET: FSIZE (Bitfield-Mask: 0x01) */ +/* ====================================================== REGACCINTEN ====================================================== */ +#define IOSLAVE_REGACCINTEN_REGACC_Pos (0UL) /*!< IOSLAVE REGACCINTEN: REGACC (Bit 0) */ +#define IOSLAVE_REGACCINTEN_REGACC_Msk (0xffffffffUL) /*!< IOSLAVE REGACCINTEN: REGACC (Bitfield-Mask: 0xffffffff) */ +/* ===================================================== REGACCINTSTAT ===================================================== */ +#define IOSLAVE_REGACCINTSTAT_REGACC_Pos (0UL) /*!< IOSLAVE REGACCINTSTAT: REGACC (Bit 0) */ +#define IOSLAVE_REGACCINTSTAT_REGACC_Msk (0xffffffffUL) /*!< IOSLAVE REGACCINTSTAT: REGACC (Bitfield-Mask: 0xffffffff) */ +/* ===================================================== REGACCINTCLR ====================================================== */ +#define IOSLAVE_REGACCINTCLR_REGACC_Pos (0UL) /*!< IOSLAVE REGACCINTCLR: REGACC (Bit 0) */ +#define IOSLAVE_REGACCINTCLR_REGACC_Msk (0xffffffffUL) /*!< IOSLAVE REGACCINTCLR: REGACC (Bitfield-Mask: 0xffffffff) */ +/* ===================================================== REGACCINTSET ====================================================== */ +#define IOSLAVE_REGACCINTSET_REGACC_Pos (0UL) /*!< IOSLAVE REGACCINTSET: REGACC (Bit 0) */ +#define IOSLAVE_REGACCINTSET_REGACC_Msk (0xffffffffUL) /*!< IOSLAVE REGACCINTSET: REGACC (Bitfield-Mask: 0xffffffff) */ + + +/* =========================================================================================================================== */ +/* ================ MCUCTRL ================ */ +/* =========================================================================================================================== */ + +/* ======================================================== CHIPPN ========================================================= */ +#define MCUCTRL_CHIPPN_PARTNUM_Pos (0UL) /*!< MCUCTRL CHIPPN: PARTNUM (Bit 0) */ +#define MCUCTRL_CHIPPN_PARTNUM_Msk (0xffffffffUL) /*!< MCUCTRL CHIPPN: PARTNUM (Bitfield-Mask: 0xffffffff) */ +/* ======================================================== CHIPID0 ======================================================== */ +#define MCUCTRL_CHIPID0_CHIPID0_Pos (0UL) /*!< MCUCTRL CHIPID0: CHIPID0 (Bit 0) */ +#define MCUCTRL_CHIPID0_CHIPID0_Msk (0xffffffffUL) /*!< MCUCTRL CHIPID0: CHIPID0 (Bitfield-Mask: 0xffffffff) */ +/* ======================================================== CHIPID1 ======================================================== */ +#define MCUCTRL_CHIPID1_CHIPID1_Pos (0UL) /*!< MCUCTRL CHIPID1: CHIPID1 (Bit 0) */ +#define MCUCTRL_CHIPID1_CHIPID1_Msk (0xffffffffUL) /*!< MCUCTRL CHIPID1: CHIPID1 (Bitfield-Mask: 0xffffffff) */ +/* ======================================================== CHIPREV ======================================================== */ +#define MCUCTRL_CHIPREV_SIPART_Pos (8UL) /*!< MCUCTRL CHIPREV: SIPART (Bit 8) */ +#define MCUCTRL_CHIPREV_SIPART_Msk (0xfff00UL) /*!< MCUCTRL CHIPREV: SIPART (Bitfield-Mask: 0xfff) */ +#define MCUCTRL_CHIPREV_REVMAJ_Pos (4UL) /*!< MCUCTRL CHIPREV: REVMAJ (Bit 4) */ +#define MCUCTRL_CHIPREV_REVMAJ_Msk (0xf0UL) /*!< MCUCTRL CHIPREV: REVMAJ (Bitfield-Mask: 0x0f) */ +#define MCUCTRL_CHIPREV_REVMIN_Pos (0UL) /*!< MCUCTRL CHIPREV: REVMIN (Bit 0) */ +#define MCUCTRL_CHIPREV_REVMIN_Msk (0xfUL) /*!< MCUCTRL CHIPREV: REVMIN (Bitfield-Mask: 0x0f) */ +/* ======================================================= VENDORID ======================================================== */ +#define MCUCTRL_VENDORID_VENDORID_Pos (0UL) /*!< MCUCTRL VENDORID: VENDORID (Bit 0) */ +#define MCUCTRL_VENDORID_VENDORID_Msk (0xffffffffUL) /*!< MCUCTRL VENDORID: VENDORID (Bitfield-Mask: 0xffffffff) */ +/* ========================================================== SKU ========================================================== */ +#define MCUCTRL_SKU_SECBOOT_Pos (2UL) /*!< MCUCTRL SKU: SECBOOT (Bit 2) */ +#define MCUCTRL_SKU_SECBOOT_Msk (0x4UL) /*!< MCUCTRL SKU: SECBOOT (Bitfield-Mask: 0x01) */ +#define MCUCTRL_SKU_ALLOWBLE_Pos (1UL) /*!< MCUCTRL SKU: ALLOWBLE (Bit 1) */ +#define MCUCTRL_SKU_ALLOWBLE_Msk (0x2UL) /*!< MCUCTRL SKU: ALLOWBLE (Bitfield-Mask: 0x01) */ +#define MCUCTRL_SKU_ALLOWBURST_Pos (0UL) /*!< MCUCTRL SKU: ALLOWBURST (Bit 0) */ +#define MCUCTRL_SKU_ALLOWBURST_Msk (0x1UL) /*!< MCUCTRL SKU: ALLOWBURST (Bitfield-Mask: 0x01) */ +/* ===================================================== FEATUREENABLE ===================================================== */ +#define MCUCTRL_FEATUREENABLE_BURSTAVAIL_Pos (6UL) /*!< MCUCTRL FEATUREENABLE: BURSTAVAIL (Bit 6) */ +#define MCUCTRL_FEATUREENABLE_BURSTAVAIL_Msk (0x40UL) /*!< MCUCTRL FEATUREENABLE: BURSTAVAIL (Bitfield-Mask: 0x01) */ +#define MCUCTRL_FEATUREENABLE_BURSTACK_Pos (5UL) /*!< MCUCTRL FEATUREENABLE: BURSTACK (Bit 5) */ +#define MCUCTRL_FEATUREENABLE_BURSTACK_Msk (0x20UL) /*!< MCUCTRL FEATUREENABLE: BURSTACK (Bitfield-Mask: 0x01) */ +#define MCUCTRL_FEATUREENABLE_BURSTREQ_Pos (4UL) /*!< MCUCTRL FEATUREENABLE: BURSTREQ (Bit 4) */ +#define MCUCTRL_FEATUREENABLE_BURSTREQ_Msk (0x10UL) /*!< MCUCTRL FEATUREENABLE: BURSTREQ (Bitfield-Mask: 0x01) */ +#define MCUCTRL_FEATUREENABLE_BLEAVAIL_Pos (2UL) /*!< MCUCTRL FEATUREENABLE: BLEAVAIL (Bit 2) */ +#define MCUCTRL_FEATUREENABLE_BLEAVAIL_Msk (0x4UL) /*!< MCUCTRL FEATUREENABLE: BLEAVAIL (Bitfield-Mask: 0x01) */ +#define MCUCTRL_FEATUREENABLE_BLEACK_Pos (1UL) /*!< MCUCTRL FEATUREENABLE: BLEACK (Bit 1) */ +#define MCUCTRL_FEATUREENABLE_BLEACK_Msk (0x2UL) /*!< MCUCTRL FEATUREENABLE: BLEACK (Bitfield-Mask: 0x01) */ +#define MCUCTRL_FEATUREENABLE_BLEREQ_Pos (0UL) /*!< MCUCTRL FEATUREENABLE: BLEREQ (Bit 0) */ +#define MCUCTRL_FEATUREENABLE_BLEREQ_Msk (0x1UL) /*!< MCUCTRL FEATUREENABLE: BLEREQ (Bitfield-Mask: 0x01) */ +/* ======================================================= DEBUGGER ======================================================== */ +#define MCUCTRL_DEBUGGER_LOCKOUT_Pos (0UL) /*!< MCUCTRL DEBUGGER: LOCKOUT (Bit 0) */ +#define MCUCTRL_DEBUGGER_LOCKOUT_Msk (0x1UL) /*!< MCUCTRL DEBUGGER: LOCKOUT (Bitfield-Mask: 0x01) */ +/* ======================================================== BODCTRL ======================================================== */ +#define MCUCTRL_BODCTRL_BODHVREFSEL_Pos (5UL) /*!< MCUCTRL BODCTRL: BODHVREFSEL (Bit 5) */ +#define MCUCTRL_BODCTRL_BODHVREFSEL_Msk (0x20UL) /*!< MCUCTRL BODCTRL: BODHVREFSEL (Bitfield-Mask: 0x01) */ +#define MCUCTRL_BODCTRL_BODLVREFSEL_Pos (4UL) /*!< MCUCTRL BODCTRL: BODLVREFSEL (Bit 4) */ +#define MCUCTRL_BODCTRL_BODLVREFSEL_Msk (0x10UL) /*!< MCUCTRL BODCTRL: BODLVREFSEL (Bitfield-Mask: 0x01) */ +#define MCUCTRL_BODCTRL_BODFPWD_Pos (3UL) /*!< MCUCTRL BODCTRL: BODFPWD (Bit 3) */ +#define MCUCTRL_BODCTRL_BODFPWD_Msk (0x8UL) /*!< MCUCTRL BODCTRL: BODFPWD (Bitfield-Mask: 0x01) */ +#define MCUCTRL_BODCTRL_BODCPWD_Pos (2UL) /*!< MCUCTRL BODCTRL: BODCPWD (Bit 2) */ +#define MCUCTRL_BODCTRL_BODCPWD_Msk (0x4UL) /*!< MCUCTRL BODCTRL: BODCPWD (Bitfield-Mask: 0x01) */ +#define MCUCTRL_BODCTRL_BODHPWD_Pos (1UL) /*!< MCUCTRL BODCTRL: BODHPWD (Bit 1) */ +#define MCUCTRL_BODCTRL_BODHPWD_Msk (0x2UL) /*!< MCUCTRL BODCTRL: BODHPWD (Bitfield-Mask: 0x01) */ +#define MCUCTRL_BODCTRL_BODLPWD_Pos (0UL) /*!< MCUCTRL BODCTRL: BODLPWD (Bit 0) */ +#define MCUCTRL_BODCTRL_BODLPWD_Msk (0x1UL) /*!< MCUCTRL BODCTRL: BODLPWD (Bitfield-Mask: 0x01) */ +/* ======================================================= ADCPWRDLY ======================================================= */ +#define MCUCTRL_ADCPWRDLY_ADCPWR1_Pos (8UL) /*!< MCUCTRL ADCPWRDLY: ADCPWR1 (Bit 8) */ +#define MCUCTRL_ADCPWRDLY_ADCPWR1_Msk (0xff00UL) /*!< MCUCTRL ADCPWRDLY: ADCPWR1 (Bitfield-Mask: 0xff) */ +#define MCUCTRL_ADCPWRDLY_ADCPWR0_Pos (0UL) /*!< MCUCTRL ADCPWRDLY: ADCPWR0 (Bit 0) */ +#define MCUCTRL_ADCPWRDLY_ADCPWR0_Msk (0xffUL) /*!< MCUCTRL ADCPWRDLY: ADCPWR0 (Bitfield-Mask: 0xff) */ +/* ======================================================== ADCCAL ========================================================= */ +#define MCUCTRL_ADCCAL_ADCCALIBRATED_Pos (1UL) /*!< MCUCTRL ADCCAL: ADCCALIBRATED (Bit 1) */ +#define MCUCTRL_ADCCAL_ADCCALIBRATED_Msk (0x2UL) /*!< MCUCTRL ADCCAL: ADCCALIBRATED (Bitfield-Mask: 0x01) */ +#define MCUCTRL_ADCCAL_CALONPWRUP_Pos (0UL) /*!< MCUCTRL ADCCAL: CALONPWRUP (Bit 0) */ +#define MCUCTRL_ADCCAL_CALONPWRUP_Msk (0x1UL) /*!< MCUCTRL ADCCAL: CALONPWRUP (Bitfield-Mask: 0x01) */ +/* ====================================================== ADCBATTLOAD ====================================================== */ +#define MCUCTRL_ADCBATTLOAD_BATTLOAD_Pos (0UL) /*!< MCUCTRL ADCBATTLOAD: BATTLOAD (Bit 0) */ +#define MCUCTRL_ADCBATTLOAD_BATTLOAD_Msk (0x1UL) /*!< MCUCTRL ADCBATTLOAD: BATTLOAD (Bitfield-Mask: 0x01) */ +/* ======================================================== ADCTRIM ======================================================== */ +#define MCUCTRL_ADCTRIM_ADCRFBUFIBTRIM_Pos (11UL) /*!< MCUCTRL ADCTRIM: ADCRFBUFIBTRIM (Bit 11) */ +#define MCUCTRL_ADCTRIM_ADCRFBUFIBTRIM_Msk (0x1800UL) /*!< MCUCTRL ADCTRIM: ADCRFBUFIBTRIM (Bitfield-Mask: 0x03) */ +#define MCUCTRL_ADCTRIM_ADCREFBUFTRIM_Pos (6UL) /*!< MCUCTRL ADCTRIM: ADCREFBUFTRIM (Bit 6) */ +#define MCUCTRL_ADCTRIM_ADCREFBUFTRIM_Msk (0x7c0UL) /*!< MCUCTRL ADCTRIM: ADCREFBUFTRIM (Bitfield-Mask: 0x1f) */ +#define MCUCTRL_ADCTRIM_ADCREFKEEPIBTRIM_Pos (0UL) /*!< MCUCTRL ADCTRIM: ADCREFKEEPIBTRIM (Bit 0) */ +#define MCUCTRL_ADCTRIM_ADCREFKEEPIBTRIM_Msk (0x3UL) /*!< MCUCTRL ADCTRIM: ADCREFKEEPIBTRIM (Bitfield-Mask: 0x03) */ +/* ====================================================== ADCREFCOMP ======================================================= */ +#define MCUCTRL_ADCREFCOMP_ADCRFCMPEN_Pos (16UL) /*!< MCUCTRL ADCREFCOMP: ADCRFCMPEN (Bit 16) */ +#define MCUCTRL_ADCREFCOMP_ADCRFCMPEN_Msk (0x10000UL) /*!< MCUCTRL ADCREFCOMP: ADCRFCMPEN (Bitfield-Mask: 0x01) */ +#define MCUCTRL_ADCREFCOMP_ADCREFKEEPTRIM_Pos (8UL) /*!< MCUCTRL ADCREFCOMP: ADCREFKEEPTRIM (Bit 8) */ +#define MCUCTRL_ADCREFCOMP_ADCREFKEEPTRIM_Msk (0x1f00UL) /*!< MCUCTRL ADCREFCOMP: ADCREFKEEPTRIM (Bitfield-Mask: 0x1f) */ +#define MCUCTRL_ADCREFCOMP_ADC_REFCOMP_OUT_Pos (0UL) /*!< MCUCTRL ADCREFCOMP: ADC_REFCOMP_OUT (Bit 0) */ +#define MCUCTRL_ADCREFCOMP_ADC_REFCOMP_OUT_Msk (0x1UL) /*!< MCUCTRL ADCREFCOMP: ADC_REFCOMP_OUT (Bitfield-Mask: 0x01) */ +/* ======================================================= XTALCTRL ======================================================== */ +#define MCUCTRL_XTALCTRL_XTALICOMPTRIM_Pos (8UL) /*!< MCUCTRL XTALCTRL: XTALICOMPTRIM (Bit 8) */ +#define MCUCTRL_XTALCTRL_XTALICOMPTRIM_Msk (0x300UL) /*!< MCUCTRL XTALCTRL: XTALICOMPTRIM (Bitfield-Mask: 0x03) */ +#define MCUCTRL_XTALCTRL_XTALIBUFTRIM_Pos (6UL) /*!< MCUCTRL XTALCTRL: XTALIBUFTRIM (Bit 6) */ +#define MCUCTRL_XTALCTRL_XTALIBUFTRIM_Msk (0xc0UL) /*!< MCUCTRL XTALCTRL: XTALIBUFTRIM (Bitfield-Mask: 0x03) */ +#define MCUCTRL_XTALCTRL_PWDBODXTAL_Pos (5UL) /*!< MCUCTRL XTALCTRL: PWDBODXTAL (Bit 5) */ +#define MCUCTRL_XTALCTRL_PWDBODXTAL_Msk (0x20UL) /*!< MCUCTRL XTALCTRL: PWDBODXTAL (Bitfield-Mask: 0x01) */ +#define MCUCTRL_XTALCTRL_PDNBCMPRXTAL_Pos (4UL) /*!< MCUCTRL XTALCTRL: PDNBCMPRXTAL (Bit 4) */ +#define MCUCTRL_XTALCTRL_PDNBCMPRXTAL_Msk (0x10UL) /*!< MCUCTRL XTALCTRL: PDNBCMPRXTAL (Bitfield-Mask: 0x01) */ +#define MCUCTRL_XTALCTRL_PDNBCOREXTAL_Pos (3UL) /*!< MCUCTRL XTALCTRL: PDNBCOREXTAL (Bit 3) */ +#define MCUCTRL_XTALCTRL_PDNBCOREXTAL_Msk (0x8UL) /*!< MCUCTRL XTALCTRL: PDNBCOREXTAL (Bitfield-Mask: 0x01) */ +#define MCUCTRL_XTALCTRL_BYPCMPRXTAL_Pos (2UL) /*!< MCUCTRL XTALCTRL: BYPCMPRXTAL (Bit 2) */ +#define MCUCTRL_XTALCTRL_BYPCMPRXTAL_Msk (0x4UL) /*!< MCUCTRL XTALCTRL: BYPCMPRXTAL (Bitfield-Mask: 0x01) */ +#define MCUCTRL_XTALCTRL_FDBKDSBLXTAL_Pos (1UL) /*!< MCUCTRL XTALCTRL: FDBKDSBLXTAL (Bit 1) */ +#define MCUCTRL_XTALCTRL_FDBKDSBLXTAL_Msk (0x2UL) /*!< MCUCTRL XTALCTRL: FDBKDSBLXTAL (Bitfield-Mask: 0x01) */ +#define MCUCTRL_XTALCTRL_XTALSWE_Pos (0UL) /*!< MCUCTRL XTALCTRL: XTALSWE (Bit 0) */ +#define MCUCTRL_XTALCTRL_XTALSWE_Msk (0x1UL) /*!< MCUCTRL XTALCTRL: XTALSWE (Bitfield-Mask: 0x01) */ +/* ====================================================== XTALGENCTRL ====================================================== */ +#define MCUCTRL_XTALGENCTRL_XTALKSBIASTRIM_Pos (8UL) /*!< MCUCTRL XTALGENCTRL: XTALKSBIASTRIM (Bit 8) */ +#define MCUCTRL_XTALGENCTRL_XTALKSBIASTRIM_Msk (0x3f00UL) /*!< MCUCTRL XTALGENCTRL: XTALKSBIASTRIM (Bitfield-Mask: 0x3f) */ +#define MCUCTRL_XTALGENCTRL_XTALBIASTRIM_Pos (2UL) /*!< MCUCTRL XTALGENCTRL: XTALBIASTRIM (Bit 2) */ +#define MCUCTRL_XTALGENCTRL_XTALBIASTRIM_Msk (0xfcUL) /*!< MCUCTRL XTALGENCTRL: XTALBIASTRIM (Bitfield-Mask: 0x3f) */ +#define MCUCTRL_XTALGENCTRL_ACWARMUP_Pos (0UL) /*!< MCUCTRL XTALGENCTRL: ACWARMUP (Bit 0) */ +#define MCUCTRL_XTALGENCTRL_ACWARMUP_Msk (0x3UL) /*!< MCUCTRL XTALGENCTRL: ACWARMUP (Bitfield-Mask: 0x03) */ +/* ======================================================= MISCCTRL ======================================================== */ +#define MCUCTRL_MISCCTRL_BLE_RESETN_Pos (5UL) /*!< MCUCTRL MISCCTRL: BLE_RESETN (Bit 5) */ +#define MCUCTRL_MISCCTRL_BLE_RESETN_Msk (0x20UL) /*!< MCUCTRL MISCCTRL: BLE_RESETN (Bitfield-Mask: 0x01) */ +#define MCUCTRL_MISCCTRL_RESERVED_RW_0_Pos (0UL) /*!< MCUCTRL MISCCTRL: RESERVED_RW_0 (Bit 0) */ +#define MCUCTRL_MISCCTRL_RESERVED_RW_0_Msk (0x1fUL) /*!< MCUCTRL MISCCTRL: RESERVED_RW_0 (Bitfield-Mask: 0x1f) */ +/* ====================================================== BOOTLOADER ======================================================= */ +#define MCUCTRL_BOOTLOADER_SECBOOTONRST_Pos (30UL) /*!< MCUCTRL BOOTLOADER: SECBOOTONRST (Bit 30) */ +#define MCUCTRL_BOOTLOADER_SECBOOTONRST_Msk (0xc0000000UL) /*!< MCUCTRL BOOTLOADER: SECBOOTONRST (Bitfield-Mask: 0x03) */ +#define MCUCTRL_BOOTLOADER_SECBOOT_Pos (28UL) /*!< MCUCTRL BOOTLOADER: SECBOOT (Bit 28) */ +#define MCUCTRL_BOOTLOADER_SECBOOT_Msk (0x30000000UL) /*!< MCUCTRL BOOTLOADER: SECBOOT (Bitfield-Mask: 0x03) */ +#define MCUCTRL_BOOTLOADER_SECBOOTFEATURE_Pos (26UL) /*!< MCUCTRL BOOTLOADER: SECBOOTFEATURE (Bit 26) */ +#define MCUCTRL_BOOTLOADER_SECBOOTFEATURE_Msk (0xc000000UL) /*!< MCUCTRL BOOTLOADER: SECBOOTFEATURE (Bitfield-Mask: 0x03) */ +#define MCUCTRL_BOOTLOADER_PROTLOCK_Pos (2UL) /*!< MCUCTRL BOOTLOADER: PROTLOCK (Bit 2) */ +#define MCUCTRL_BOOTLOADER_PROTLOCK_Msk (0x4UL) /*!< MCUCTRL BOOTLOADER: PROTLOCK (Bitfield-Mask: 0x01) */ +#define MCUCTRL_BOOTLOADER_SBLOCK_Pos (1UL) /*!< MCUCTRL BOOTLOADER: SBLOCK (Bit 1) */ +#define MCUCTRL_BOOTLOADER_SBLOCK_Msk (0x2UL) /*!< MCUCTRL BOOTLOADER: SBLOCK (Bitfield-Mask: 0x01) */ +#define MCUCTRL_BOOTLOADER_BOOTLOADERLOW_Pos (0UL) /*!< MCUCTRL BOOTLOADER: BOOTLOADERLOW (Bit 0) */ +#define MCUCTRL_BOOTLOADER_BOOTLOADERLOW_Msk (0x1UL) /*!< MCUCTRL BOOTLOADER: BOOTLOADERLOW (Bitfield-Mask: 0x01) */ +/* ====================================================== SHADOWVALID ====================================================== */ +#define MCUCTRL_SHADOWVALID_INFO0_VALID_Pos (2UL) /*!< MCUCTRL SHADOWVALID: INFO0_VALID (Bit 2) */ +#define MCUCTRL_SHADOWVALID_INFO0_VALID_Msk (0x4UL) /*!< MCUCTRL SHADOWVALID: INFO0_VALID (Bitfield-Mask: 0x01) */ +#define MCUCTRL_SHADOWVALID_BLDSLEEP_Pos (1UL) /*!< MCUCTRL SHADOWVALID: BLDSLEEP (Bit 1) */ +#define MCUCTRL_SHADOWVALID_BLDSLEEP_Msk (0x2UL) /*!< MCUCTRL SHADOWVALID: BLDSLEEP (Bitfield-Mask: 0x01) */ +#define MCUCTRL_SHADOWVALID_VALID_Pos (0UL) /*!< MCUCTRL SHADOWVALID: VALID (Bit 0) */ +#define MCUCTRL_SHADOWVALID_VALID_Msk (0x1UL) /*!< MCUCTRL SHADOWVALID: VALID (Bitfield-Mask: 0x01) */ +/* ======================================================= SCRATCH0 ======================================================== */ +#define MCUCTRL_SCRATCH0_SCRATCH0_Pos (0UL) /*!< MCUCTRL SCRATCH0: SCRATCH0 (Bit 0) */ +#define MCUCTRL_SCRATCH0_SCRATCH0_Msk (0xffffffffUL) /*!< MCUCTRL SCRATCH0: SCRATCH0 (Bitfield-Mask: 0xffffffff) */ +/* ======================================================= SCRATCH1 ======================================================== */ +#define MCUCTRL_SCRATCH1_SCRATCH1_Pos (0UL) /*!< MCUCTRL SCRATCH1: SCRATCH1 (Bit 0) */ +#define MCUCTRL_SCRATCH1_SCRATCH1_Msk (0xffffffffUL) /*!< MCUCTRL SCRATCH1: SCRATCH1 (Bitfield-Mask: 0xffffffff) */ +/* ==================================================== ICODEFAULTADDR ===================================================== */ +#define MCUCTRL_ICODEFAULTADDR_ICODEFAULTADDR_Pos (0UL) /*!< MCUCTRL ICODEFAULTADDR: ICODEFAULTADDR (Bit 0) */ +#define MCUCTRL_ICODEFAULTADDR_ICODEFAULTADDR_Msk (0xffffffffUL) /*!< MCUCTRL ICODEFAULTADDR: ICODEFAULTADDR (Bitfield-Mask: 0xffffffff) */ +/* ==================================================== DCODEFAULTADDR ===================================================== */ +#define MCUCTRL_DCODEFAULTADDR_DCODEFAULTADDR_Pos (0UL) /*!< MCUCTRL DCODEFAULTADDR: DCODEFAULTADDR (Bit 0) */ +#define MCUCTRL_DCODEFAULTADDR_DCODEFAULTADDR_Msk (0xffffffffUL) /*!< MCUCTRL DCODEFAULTADDR: DCODEFAULTADDR (Bitfield-Mask: 0xffffffff) */ +/* ===================================================== SYSFAULTADDR ====================================================== */ +#define MCUCTRL_SYSFAULTADDR_SYSFAULTADDR_Pos (0UL) /*!< MCUCTRL SYSFAULTADDR: SYSFAULTADDR (Bit 0) */ +#define MCUCTRL_SYSFAULTADDR_SYSFAULTADDR_Msk (0xffffffffUL) /*!< MCUCTRL SYSFAULTADDR: SYSFAULTADDR (Bitfield-Mask: 0xffffffff) */ +/* ====================================================== FAULTSTATUS ====================================================== */ +#define MCUCTRL_FAULTSTATUS_SYSFAULT_Pos (2UL) /*!< MCUCTRL FAULTSTATUS: SYSFAULT (Bit 2) */ +#define MCUCTRL_FAULTSTATUS_SYSFAULT_Msk (0x4UL) /*!< MCUCTRL FAULTSTATUS: SYSFAULT (Bitfield-Mask: 0x01) */ +#define MCUCTRL_FAULTSTATUS_DCODEFAULT_Pos (1UL) /*!< MCUCTRL FAULTSTATUS: DCODEFAULT (Bit 1) */ +#define MCUCTRL_FAULTSTATUS_DCODEFAULT_Msk (0x2UL) /*!< MCUCTRL FAULTSTATUS: DCODEFAULT (Bitfield-Mask: 0x01) */ +#define MCUCTRL_FAULTSTATUS_ICODEFAULT_Pos (0UL) /*!< MCUCTRL FAULTSTATUS: ICODEFAULT (Bit 0) */ +#define MCUCTRL_FAULTSTATUS_ICODEFAULT_Msk (0x1UL) /*!< MCUCTRL FAULTSTATUS: ICODEFAULT (Bitfield-Mask: 0x01) */ +/* ==================================================== FAULTCAPTUREEN ===================================================== */ +#define MCUCTRL_FAULTCAPTUREEN_FAULTCAPTUREEN_Pos (0UL) /*!< MCUCTRL FAULTCAPTUREEN: FAULTCAPTUREEN (Bit 0) */ +#define MCUCTRL_FAULTCAPTUREEN_FAULTCAPTUREEN_Msk (0x1UL) /*!< MCUCTRL FAULTCAPTUREEN: FAULTCAPTUREEN (Bitfield-Mask: 0x01) */ +/* ========================================================= DBGR1 ========================================================= */ +#define MCUCTRL_DBGR1_ONETO8_Pos (0UL) /*!< MCUCTRL DBGR1: ONETO8 (Bit 0) */ +#define MCUCTRL_DBGR1_ONETO8_Msk (0xffffffffUL) /*!< MCUCTRL DBGR1: ONETO8 (Bitfield-Mask: 0xffffffff) */ +/* ========================================================= DBGR2 ========================================================= */ +#define MCUCTRL_DBGR2_COOLCODE_Pos (0UL) /*!< MCUCTRL DBGR2: COOLCODE (Bit 0) */ +#define MCUCTRL_DBGR2_COOLCODE_Msk (0xffffffffUL) /*!< MCUCTRL DBGR2: COOLCODE (Bitfield-Mask: 0xffffffff) */ +/* ======================================================= PMUENABLE ======================================================= */ +#define MCUCTRL_PMUENABLE_ENABLE_Pos (0UL) /*!< MCUCTRL PMUENABLE: ENABLE (Bit 0) */ +#define MCUCTRL_PMUENABLE_ENABLE_Msk (0x1UL) /*!< MCUCTRL PMUENABLE: ENABLE (Bitfield-Mask: 0x01) */ +/* ======================================================= TPIUCTRL ======================================================== */ +#define MCUCTRL_TPIUCTRL_CLKSEL_Pos (8UL) /*!< MCUCTRL TPIUCTRL: CLKSEL (Bit 8) */ +#define MCUCTRL_TPIUCTRL_CLKSEL_Msk (0x700UL) /*!< MCUCTRL TPIUCTRL: CLKSEL (Bitfield-Mask: 0x07) */ +#define MCUCTRL_TPIUCTRL_ENABLE_Pos (0UL) /*!< MCUCTRL TPIUCTRL: ENABLE (Bit 0) */ +#define MCUCTRL_TPIUCTRL_ENABLE_Msk (0x1UL) /*!< MCUCTRL TPIUCTRL: ENABLE (Bitfield-Mask: 0x01) */ +/* ====================================================== OTAPOINTER ======================================================= */ +#define MCUCTRL_OTAPOINTER_OTAPOINTER_Pos (2UL) /*!< MCUCTRL OTAPOINTER: OTAPOINTER (Bit 2) */ +#define MCUCTRL_OTAPOINTER_OTAPOINTER_Msk (0xfffffffcUL) /*!< MCUCTRL OTAPOINTER: OTAPOINTER (Bitfield-Mask: 0x3fffffff) */ +#define MCUCTRL_OTAPOINTER_OTASBLUPDATE_Pos (1UL) /*!< MCUCTRL OTAPOINTER: OTASBLUPDATE (Bit 1) */ +#define MCUCTRL_OTAPOINTER_OTASBLUPDATE_Msk (0x2UL) /*!< MCUCTRL OTAPOINTER: OTASBLUPDATE (Bitfield-Mask: 0x01) */ +#define MCUCTRL_OTAPOINTER_OTAVALID_Pos (0UL) /*!< MCUCTRL OTAPOINTER: OTAVALID (Bit 0) */ +#define MCUCTRL_OTAPOINTER_OTAVALID_Msk (0x1UL) /*!< MCUCTRL OTAPOINTER: OTAVALID (Bitfield-Mask: 0x01) */ +/* ====================================================== APBDMACTRL ======================================================= */ +#define MCUCTRL_APBDMACTRL_HYSTERESIS_Pos (8UL) /*!< MCUCTRL APBDMACTRL: HYSTERESIS (Bit 8) */ +#define MCUCTRL_APBDMACTRL_HYSTERESIS_Msk (0xff00UL) /*!< MCUCTRL APBDMACTRL: HYSTERESIS (Bitfield-Mask: 0xff) */ +#define MCUCTRL_APBDMACTRL_DECODEABORT_Pos (1UL) /*!< MCUCTRL APBDMACTRL: DECODEABORT (Bit 1) */ +#define MCUCTRL_APBDMACTRL_DECODEABORT_Msk (0x2UL) /*!< MCUCTRL APBDMACTRL: DECODEABORT (Bitfield-Mask: 0x01) */ +#define MCUCTRL_APBDMACTRL_DMA_ENABLE_Pos (0UL) /*!< MCUCTRL APBDMACTRL: DMA_ENABLE (Bit 0) */ +#define MCUCTRL_APBDMACTRL_DMA_ENABLE_Msk (0x1UL) /*!< MCUCTRL APBDMACTRL: DMA_ENABLE (Bitfield-Mask: 0x01) */ +/* ======================================================= SRAMMODE ======================================================== */ +#define MCUCTRL_SRAMMODE_DPREFETCH_CACHE_Pos (5UL) /*!< MCUCTRL SRAMMODE: DPREFETCH_CACHE (Bit 5) */ +#define MCUCTRL_SRAMMODE_DPREFETCH_CACHE_Msk (0x20UL) /*!< MCUCTRL SRAMMODE: DPREFETCH_CACHE (Bitfield-Mask: 0x01) */ +#define MCUCTRL_SRAMMODE_DPREFETCH_Pos (4UL) /*!< MCUCTRL SRAMMODE: DPREFETCH (Bit 4) */ +#define MCUCTRL_SRAMMODE_DPREFETCH_Msk (0x10UL) /*!< MCUCTRL SRAMMODE: DPREFETCH (Bitfield-Mask: 0x01) */ +#define MCUCTRL_SRAMMODE_IPREFETCH_CACHE_Pos (1UL) /*!< MCUCTRL SRAMMODE: IPREFETCH_CACHE (Bit 1) */ +#define MCUCTRL_SRAMMODE_IPREFETCH_CACHE_Msk (0x2UL) /*!< MCUCTRL SRAMMODE: IPREFETCH_CACHE (Bitfield-Mask: 0x01) */ +#define MCUCTRL_SRAMMODE_IPREFETCH_Pos (0UL) /*!< MCUCTRL SRAMMODE: IPREFETCH (Bit 0) */ +#define MCUCTRL_SRAMMODE_IPREFETCH_Msk (0x1UL) /*!< MCUCTRL SRAMMODE: IPREFETCH (Bitfield-Mask: 0x01) */ +/* ====================================================== KEXTCLKSEL ======================================================= */ +#define MCUCTRL_KEXTCLKSEL_KEXTCLKSEL_Pos (0UL) /*!< MCUCTRL KEXTCLKSEL: KEXTCLKSEL (Bit 0) */ +#define MCUCTRL_KEXTCLKSEL_KEXTCLKSEL_Msk (0xffffffffUL) /*!< MCUCTRL KEXTCLKSEL: KEXTCLKSEL (Bitfield-Mask: 0xffffffff) */ +/* ======================================================= SIMOBUCK4 ======================================================= */ +#define MCUCTRL_SIMOBUCK4_SIMOBUCKIBIASTRIM_Pos (28UL) /*!< MCUCTRL SIMOBUCK4: SIMOBUCKIBIASTRIM (Bit 28) */ +#define MCUCTRL_SIMOBUCK4_SIMOBUCKIBIASTRIM_Msk (0xf0000000UL) /*!< MCUCTRL SIMOBUCK4: SIMOBUCKIBIASTRIM (Bitfield-Mask: 0x0f) */ +#define MCUCTRL_SIMOBUCK4_SIMOBUCKUVLOMODE_Pos (26UL) /*!< MCUCTRL SIMOBUCK4: SIMOBUCKUVLOMODE (Bit 26) */ +#define MCUCTRL_SIMOBUCK4_SIMOBUCKUVLOMODE_Msk (0xc000000UL) /*!< MCUCTRL SIMOBUCK4: SIMOBUCKUVLOMODE (Bitfield-Mask: 0x03) */ +#define MCUCTRL_SIMOBUCK4_SIMOBUCKPRIORITYSEL_Pos (25UL) /*!< MCUCTRL SIMOBUCK4: SIMOBUCKPRIORITYSEL (Bit 25) */ +#define MCUCTRL_SIMOBUCK4_SIMOBUCKPRIORITYSEL_Msk (0x2000000UL) /*!< MCUCTRL SIMOBUCK4: SIMOBUCKPRIORITYSEL (Bitfield-Mask: 0x01) */ +#define MCUCTRL_SIMOBUCK4_SIMOBUCKCOMP2TIMEOUTEN_Pos (24UL) /*!< MCUCTRL SIMOBUCK4: SIMOBUCKCOMP2TIMEOUTEN (Bit 24) */ +#define MCUCTRL_SIMOBUCK4_SIMOBUCKCOMP2TIMEOUTEN_Msk (0x1000000UL) /*!< MCUCTRL SIMOBUCK4: SIMOBUCKCOMP2TIMEOUTEN (Bitfield-Mask: 0x01) */ +#define MCUCTRL_SIMOBUCK4_SIMOBUCKCOMP2LPEN_Pos (23UL) /*!< MCUCTRL SIMOBUCK4: SIMOBUCKCOMP2LPEN (Bit 23) */ +#define MCUCTRL_SIMOBUCK4_SIMOBUCKCOMP2LPEN_Msk (0x800000UL) /*!< MCUCTRL SIMOBUCK4: SIMOBUCKCOMP2LPEN (Bitfield-Mask: 0x01) */ +#define MCUCTRL_SIMOBUCK4_SIMOBUCKCLKDIVSEL_Pos (21UL) /*!< MCUCTRL SIMOBUCK4: SIMOBUCKCLKDIVSEL (Bit 21) */ +#define MCUCTRL_SIMOBUCK4_SIMOBUCKCLKDIVSEL_Msk (0x600000UL) /*!< MCUCTRL SIMOBUCK4: SIMOBUCKCLKDIVSEL (Bitfield-Mask: 0x03) */ +#define MCUCTRL_SIMOBUCK4_SIMOBUCKEXTCLKSEL_Pos (20UL) /*!< MCUCTRL SIMOBUCK4: SIMOBUCKEXTCLKSEL (Bit 20) */ +#define MCUCTRL_SIMOBUCK4_SIMOBUCKEXTCLKSEL_Msk (0x100000UL) /*!< MCUCTRL SIMOBUCK4: SIMOBUCKEXTCLKSEL (Bitfield-Mask: 0x01) */ +#define MCUCTRL_SIMOBUCK4_SIMOBUCKUVLODRVSTRTRIM_Pos (17UL) /*!< MCUCTRL SIMOBUCK4: SIMOBUCKUVLODRVSTRTRIM (Bit 17) */ +#define MCUCTRL_SIMOBUCK4_SIMOBUCKUVLODRVSTRTRIM_Msk (0xe0000UL) /*!< MCUCTRL SIMOBUCK4: SIMOBUCKUVLODRVSTRTRIM (Bitfield-Mask: 0x07) */ +#define MCUCTRL_SIMOBUCK4_SIMOBUCKUVLOCNTRTRIM_Pos (14UL) /*!< MCUCTRL SIMOBUCK4: SIMOBUCKUVLOCNTRTRIM (Bit 14) */ +#define MCUCTRL_SIMOBUCK4_SIMOBUCKUVLOCNTRTRIM_Msk (0x1c000UL) /*!< MCUCTRL SIMOBUCK4: SIMOBUCKUVLOCNTRTRIM (Bitfield-Mask: 0x07) */ +#define MCUCTRL_SIMOBUCK4_SIMOBUCKZXTRIM_Pos (10UL) /*!< MCUCTRL SIMOBUCK4: SIMOBUCKZXTRIM (Bit 10) */ +#define MCUCTRL_SIMOBUCK4_SIMOBUCKZXTRIM_Msk (0x3c00UL) /*!< MCUCTRL SIMOBUCK4: SIMOBUCKZXTRIM (Bitfield-Mask: 0x0f) */ +#define MCUCTRL_SIMOBUCK4_SIMOBUCKMEMLEAKAGETRIM_Pos (8UL) /*!< MCUCTRL SIMOBUCK4: SIMOBUCKMEMLEAKAGETRIM (Bit 8) */ +#define MCUCTRL_SIMOBUCK4_SIMOBUCKMEMLEAKAGETRIM_Msk (0x300UL) /*!< MCUCTRL SIMOBUCK4: SIMOBUCKMEMLEAKAGETRIM (Bitfield-Mask: 0x03) */ +#define MCUCTRL_SIMOBUCK4_SIMOBUCKMEMLPDRVSTRTRIM_Pos (6UL) /*!< MCUCTRL SIMOBUCK4: SIMOBUCKMEMLPDRVSTRTRIM (Bit 6) */ +#define MCUCTRL_SIMOBUCK4_SIMOBUCKMEMLPDRVSTRTRIM_Msk (0xc0UL) /*!< MCUCTRL SIMOBUCK4: SIMOBUCKMEMLPDRVSTRTRIM (Bitfield-Mask: 0x03) */ +#define MCUCTRL_SIMOBUCK4_SIMOBUCKMEMACTDRVSTRTRIM_Pos (4UL) /*!< MCUCTRL SIMOBUCK4: SIMOBUCKMEMACTDRVSTRTRIM (Bit 4) */ +#define MCUCTRL_SIMOBUCK4_SIMOBUCKMEMACTDRVSTRTRIM_Msk (0x30UL) /*!< MCUCTRL SIMOBUCK4: SIMOBUCKMEMACTDRVSTRTRIM (Bitfield-Mask: 0x03) */ +#define MCUCTRL_SIMOBUCK4_SIMOBUCKMEMLPLOWTONTRIM_Pos (0UL) /*!< MCUCTRL SIMOBUCK4: SIMOBUCKMEMLPLOWTONTRIM (Bit 0) */ +#define MCUCTRL_SIMOBUCK4_SIMOBUCKMEMLPLOWTONTRIM_Msk (0xfUL) /*!< MCUCTRL SIMOBUCK4: SIMOBUCKMEMLPLOWTONTRIM (Bitfield-Mask: 0x0f) */ +/* ======================================================= BLEBUCK2 ======================================================== */ +#define MCUCTRL_BLEBUCK2_BLEBUCKTOND2ATRIM_Pos (12UL) /*!< MCUCTRL BLEBUCK2: BLEBUCKTOND2ATRIM (Bit 12) */ +#define MCUCTRL_BLEBUCK2_BLEBUCKTOND2ATRIM_Msk (0x3f000UL) /*!< MCUCTRL BLEBUCK2: BLEBUCKTOND2ATRIM (Bitfield-Mask: 0x3f) */ +#define MCUCTRL_BLEBUCK2_BLEBUCKTONHITRIM_Pos (6UL) /*!< MCUCTRL BLEBUCK2: BLEBUCKTONHITRIM (Bit 6) */ +#define MCUCTRL_BLEBUCK2_BLEBUCKTONHITRIM_Msk (0xfc0UL) /*!< MCUCTRL BLEBUCK2: BLEBUCKTONHITRIM (Bitfield-Mask: 0x3f) */ +#define MCUCTRL_BLEBUCK2_BLEBUCKTONLOWTRIM_Pos (0UL) /*!< MCUCTRL BLEBUCK2: BLEBUCKTONLOWTRIM (Bit 0) */ +#define MCUCTRL_BLEBUCK2_BLEBUCKTONLOWTRIM_Msk (0x3fUL) /*!< MCUCTRL BLEBUCK2: BLEBUCKTONLOWTRIM (Bitfield-Mask: 0x3f) */ +/* ====================================================== FLASHWPROT0 ====================================================== */ +#define MCUCTRL_FLASHWPROT0_FW0BITS_Pos (0UL) /*!< MCUCTRL FLASHWPROT0: FW0BITS (Bit 0) */ +#define MCUCTRL_FLASHWPROT0_FW0BITS_Msk (0xffffffffUL) /*!< MCUCTRL FLASHWPROT0: FW0BITS (Bitfield-Mask: 0xffffffff) */ +/* ====================================================== FLASHWPROT1 ====================================================== */ +#define MCUCTRL_FLASHWPROT1_FW1BITS_Pos (0UL) /*!< MCUCTRL FLASHWPROT1: FW1BITS (Bit 0) */ +#define MCUCTRL_FLASHWPROT1_FW1BITS_Msk (0xffffffffUL) /*!< MCUCTRL FLASHWPROT1: FW1BITS (Bitfield-Mask: 0xffffffff) */ +/* ====================================================== FLASHRPROT0 ====================================================== */ +#define MCUCTRL_FLASHRPROT0_FR0BITS_Pos (0UL) /*!< MCUCTRL FLASHRPROT0: FR0BITS (Bit 0) */ +#define MCUCTRL_FLASHRPROT0_FR0BITS_Msk (0xffffffffUL) /*!< MCUCTRL FLASHRPROT0: FR0BITS (Bitfield-Mask: 0xffffffff) */ +/* ====================================================== FLASHRPROT1 ====================================================== */ +#define MCUCTRL_FLASHRPROT1_FR1BITS_Pos (0UL) /*!< MCUCTRL FLASHRPROT1: FR1BITS (Bit 0) */ +#define MCUCTRL_FLASHRPROT1_FR1BITS_Msk (0xffffffffUL) /*!< MCUCTRL FLASHRPROT1: FR1BITS (Bitfield-Mask: 0xffffffff) */ +/* ================================================= DMASRAMWRITEPROTECT0 ================================================== */ +#define MCUCTRL_DMASRAMWRITEPROTECT0_DMA_WPROT0_Pos (0UL) /*!< MCUCTRL DMASRAMWRITEPROTECT0: DMA_WPROT0 (Bit 0) */ +#define MCUCTRL_DMASRAMWRITEPROTECT0_DMA_WPROT0_Msk (0xffffffffUL) /*!< MCUCTRL DMASRAMWRITEPROTECT0: DMA_WPROT0 (Bitfield-Mask: 0xffffffff) */ +/* ================================================= DMASRAMWRITEPROTECT1 ================================================== */ +#define MCUCTRL_DMASRAMWRITEPROTECT1_DMA_WPROT1_Pos (0UL) /*!< MCUCTRL DMASRAMWRITEPROTECT1: DMA_WPROT1 (Bit 0) */ +#define MCUCTRL_DMASRAMWRITEPROTECT1_DMA_WPROT1_Msk (0xffffUL) /*!< MCUCTRL DMASRAMWRITEPROTECT1: DMA_WPROT1 (Bitfield-Mask: 0xffff) */ +/* ================================================== DMASRAMREADPROTECT0 ================================================== */ +#define MCUCTRL_DMASRAMREADPROTECT0_DMA_RPROT0_Pos (0UL) /*!< MCUCTRL DMASRAMREADPROTECT0: DMA_RPROT0 (Bit 0) */ +#define MCUCTRL_DMASRAMREADPROTECT0_DMA_RPROT0_Msk (0xffffffffUL) /*!< MCUCTRL DMASRAMREADPROTECT0: DMA_RPROT0 (Bitfield-Mask: 0xffffffff) */ +/* ================================================== DMASRAMREADPROTECT1 ================================================== */ +#define MCUCTRL_DMASRAMREADPROTECT1_DMA_RPROT1_Pos (0UL) /*!< MCUCTRL DMASRAMREADPROTECT1: DMA_RPROT1 (Bit 0) */ +#define MCUCTRL_DMASRAMREADPROTECT1_DMA_RPROT1_Msk (0xffffUL) /*!< MCUCTRL DMASRAMREADPROTECT1: DMA_RPROT1 (Bitfield-Mask: 0xffff) */ + + +/* =========================================================================================================================== */ +/* ================ MSPI ================ */ +/* =========================================================================================================================== */ + +/* ========================================================= CTRL ========================================================== */ +#define MSPI_CTRL_XFERBYTES_Pos (16UL) /*!< MSPI CTRL: XFERBYTES (Bit 16) */ +#define MSPI_CTRL_XFERBYTES_Msk (0xffff0000UL) /*!< MSPI CTRL: XFERBYTES (Bitfield-Mask: 0xffff) */ +#define MSPI_CTRL_PIOSCRAMBLE_Pos (11UL) /*!< MSPI CTRL: PIOSCRAMBLE (Bit 11) */ +#define MSPI_CTRL_PIOSCRAMBLE_Msk (0x800UL) /*!< MSPI CTRL: PIOSCRAMBLE (Bitfield-Mask: 0x01) */ +#define MSPI_CTRL_TXRX_Pos (10UL) /*!< MSPI CTRL: TXRX (Bit 10) */ +#define MSPI_CTRL_TXRX_Msk (0x400UL) /*!< MSPI CTRL: TXRX (Bitfield-Mask: 0x01) */ +#define MSPI_CTRL_SENDI_Pos (9UL) /*!< MSPI CTRL: SENDI (Bit 9) */ +#define MSPI_CTRL_SENDI_Msk (0x200UL) /*!< MSPI CTRL: SENDI (Bitfield-Mask: 0x01) */ +#define MSPI_CTRL_SENDA_Pos (8UL) /*!< MSPI CTRL: SENDA (Bit 8) */ +#define MSPI_CTRL_SENDA_Msk (0x100UL) /*!< MSPI CTRL: SENDA (Bitfield-Mask: 0x01) */ +#define MSPI_CTRL_ENTURN_Pos (7UL) /*!< MSPI CTRL: ENTURN (Bit 7) */ +#define MSPI_CTRL_ENTURN_Msk (0x80UL) /*!< MSPI CTRL: ENTURN (Bitfield-Mask: 0x01) */ +#define MSPI_CTRL_BIGENDIAN_Pos (6UL) /*!< MSPI CTRL: BIGENDIAN (Bit 6) */ +#define MSPI_CTRL_BIGENDIAN_Msk (0x40UL) /*!< MSPI CTRL: BIGENDIAN (Bitfield-Mask: 0x01) */ +#define MSPI_CTRL_CONT_Pos (5UL) /*!< MSPI CTRL: CONT (Bit 5) */ +#define MSPI_CTRL_CONT_Msk (0x20UL) /*!< MSPI CTRL: CONT (Bitfield-Mask: 0x01) */ +#define MSPI_CTRL_QUADCMD_Pos (3UL) /*!< MSPI CTRL: QUADCMD (Bit 3) */ +#define MSPI_CTRL_QUADCMD_Msk (0x8UL) /*!< MSPI CTRL: QUADCMD (Bitfield-Mask: 0x01) */ +#define MSPI_CTRL_BUSY_Pos (2UL) /*!< MSPI CTRL: BUSY (Bit 2) */ +#define MSPI_CTRL_BUSY_Msk (0x4UL) /*!< MSPI CTRL: BUSY (Bitfield-Mask: 0x01) */ +#define MSPI_CTRL_STATUS_Pos (1UL) /*!< MSPI CTRL: STATUS (Bit 1) */ +#define MSPI_CTRL_STATUS_Msk (0x2UL) /*!< MSPI CTRL: STATUS (Bitfield-Mask: 0x01) */ +#define MSPI_CTRL_START_Pos (0UL) /*!< MSPI CTRL: START (Bit 0) */ +#define MSPI_CTRL_START_Msk (0x1UL) /*!< MSPI CTRL: START (Bitfield-Mask: 0x01) */ +/* ========================================================== CFG ========================================================== */ +#define MSPI_CFG_CPOL_Pos (17UL) /*!< MSPI CFG: CPOL (Bit 17) */ +#define MSPI_CFG_CPOL_Msk (0x20000UL) /*!< MSPI CFG: CPOL (Bitfield-Mask: 0x01) */ +#define MSPI_CFG_CPHA_Pos (16UL) /*!< MSPI CFG: CPHA (Bit 16) */ +#define MSPI_CFG_CPHA_Msk (0x10000UL) /*!< MSPI CFG: CPHA (Bitfield-Mask: 0x01) */ +#define MSPI_CFG_TURNAROUND_Pos (8UL) /*!< MSPI CFG: TURNAROUND (Bit 8) */ +#define MSPI_CFG_TURNAROUND_Msk (0x3f00UL) /*!< MSPI CFG: TURNAROUND (Bitfield-Mask: 0x3f) */ +#define MSPI_CFG_SEPIO_Pos (7UL) /*!< MSPI CFG: SEPIO (Bit 7) */ +#define MSPI_CFG_SEPIO_Msk (0x80UL) /*!< MSPI CFG: SEPIO (Bitfield-Mask: 0x01) */ +#define MSPI_CFG_ISIZE_Pos (6UL) /*!< MSPI CFG: ISIZE (Bit 6) */ +#define MSPI_CFG_ISIZE_Msk (0x40UL) /*!< MSPI CFG: ISIZE (Bitfield-Mask: 0x01) */ +#define MSPI_CFG_ASIZE_Pos (4UL) /*!< MSPI CFG: ASIZE (Bit 4) */ +#define MSPI_CFG_ASIZE_Msk (0x30UL) /*!< MSPI CFG: ASIZE (Bitfield-Mask: 0x03) */ +#define MSPI_CFG_DEVCFG_Pos (0UL) /*!< MSPI CFG: DEVCFG (Bit 0) */ +#define MSPI_CFG_DEVCFG_Msk (0xfUL) /*!< MSPI CFG: DEVCFG (Bitfield-Mask: 0x0f) */ +/* ========================================================= ADDR ========================================================== */ +#define MSPI_ADDR_ADDR_Pos (0UL) /*!< MSPI ADDR: ADDR (Bit 0) */ +#define MSPI_ADDR_ADDR_Msk (0xffffffffUL) /*!< MSPI ADDR: ADDR (Bitfield-Mask: 0xffffffff) */ +/* ========================================================= INSTR ========================================================= */ +#define MSPI_INSTR_INSTR_Pos (0UL) /*!< MSPI INSTR: INSTR (Bit 0) */ +#define MSPI_INSTR_INSTR_Msk (0xffffUL) /*!< MSPI INSTR: INSTR (Bitfield-Mask: 0xffff) */ +/* ======================================================== TXFIFO ========================================================= */ +#define MSPI_TXFIFO_TXFIFO_Pos (0UL) /*!< MSPI TXFIFO: TXFIFO (Bit 0) */ +#define MSPI_TXFIFO_TXFIFO_Msk (0xffffffffUL) /*!< MSPI TXFIFO: TXFIFO (Bitfield-Mask: 0xffffffff) */ +/* ======================================================== RXFIFO ========================================================= */ +#define MSPI_RXFIFO_RXFIFO_Pos (0UL) /*!< MSPI RXFIFO: RXFIFO (Bit 0) */ +#define MSPI_RXFIFO_RXFIFO_Msk (0xffffffffUL) /*!< MSPI RXFIFO: RXFIFO (Bitfield-Mask: 0xffffffff) */ +/* ======================================================= TXENTRIES ======================================================= */ +#define MSPI_TXENTRIES_TXENTRIES_Pos (0UL) /*!< MSPI TXENTRIES: TXENTRIES (Bit 0) */ +#define MSPI_TXENTRIES_TXENTRIES_Msk (0x1fUL) /*!< MSPI TXENTRIES: TXENTRIES (Bitfield-Mask: 0x1f) */ +/* ======================================================= RXENTRIES ======================================================= */ +#define MSPI_RXENTRIES_RXENTRIES_Pos (0UL) /*!< MSPI RXENTRIES: RXENTRIES (Bit 0) */ +#define MSPI_RXENTRIES_RXENTRIES_Msk (0x1fUL) /*!< MSPI RXENTRIES: RXENTRIES (Bitfield-Mask: 0x1f) */ +/* ======================================================= THRESHOLD ======================================================= */ +#define MSPI_THRESHOLD_RXTHRESH_Pos (8UL) /*!< MSPI THRESHOLD: RXTHRESH (Bit 8) */ +#define MSPI_THRESHOLD_RXTHRESH_Msk (0x1f00UL) /*!< MSPI THRESHOLD: RXTHRESH (Bitfield-Mask: 0x1f) */ +#define MSPI_THRESHOLD_TXTHRESH_Pos (0UL) /*!< MSPI THRESHOLD: TXTHRESH (Bit 0) */ +#define MSPI_THRESHOLD_TXTHRESH_Msk (0x1fUL) /*!< MSPI THRESHOLD: TXTHRESH (Bitfield-Mask: 0x1f) */ +/* ======================================================== MSPICFG ======================================================== */ +#define MSPI_MSPICFG_PRSTN_Pos (31UL) /*!< MSPI MSPICFG: PRSTN (Bit 31) */ +#define MSPI_MSPICFG_PRSTN_Msk (0x80000000UL) /*!< MSPI MSPICFG: PRSTN (Bitfield-Mask: 0x01) */ +#define MSPI_MSPICFG_IPRSTN_Pos (30UL) /*!< MSPI MSPICFG: IPRSTN (Bit 30) */ +#define MSPI_MSPICFG_IPRSTN_Msk (0x40000000UL) /*!< MSPI MSPICFG: IPRSTN (Bitfield-Mask: 0x01) */ +#define MSPI_MSPICFG_FIFORESET_Pos (29UL) /*!< MSPI MSPICFG: FIFORESET (Bit 29) */ +#define MSPI_MSPICFG_FIFORESET_Msk (0x20000000UL) /*!< MSPI MSPICFG: FIFORESET (Bitfield-Mask: 0x01) */ +#define MSPI_MSPICFG_CLKDIV_Pos (8UL) /*!< MSPI MSPICFG: CLKDIV (Bit 8) */ +#define MSPI_MSPICFG_CLKDIV_Msk (0x3f00UL) /*!< MSPI MSPICFG: CLKDIV (Bitfield-Mask: 0x3f) */ +#define MSPI_MSPICFG_IOMSEL_Pos (4UL) /*!< MSPI MSPICFG: IOMSEL (Bit 4) */ +#define MSPI_MSPICFG_IOMSEL_Msk (0x70UL) /*!< MSPI MSPICFG: IOMSEL (Bitfield-Mask: 0x07) */ +#define MSPI_MSPICFG_TXNEG_Pos (3UL) /*!< MSPI MSPICFG: TXNEG (Bit 3) */ +#define MSPI_MSPICFG_TXNEG_Msk (0x8UL) /*!< MSPI MSPICFG: TXNEG (Bitfield-Mask: 0x01) */ +#define MSPI_MSPICFG_RXNEG_Pos (2UL) /*!< MSPI MSPICFG: RXNEG (Bit 2) */ +#define MSPI_MSPICFG_RXNEG_Msk (0x4UL) /*!< MSPI MSPICFG: RXNEG (Bitfield-Mask: 0x01) */ +#define MSPI_MSPICFG_RXCAP_Pos (1UL) /*!< MSPI MSPICFG: RXCAP (Bit 1) */ +#define MSPI_MSPICFG_RXCAP_Msk (0x2UL) /*!< MSPI MSPICFG: RXCAP (Bitfield-Mask: 0x01) */ +#define MSPI_MSPICFG_APBCLK_Pos (0UL) /*!< MSPI MSPICFG: APBCLK (Bit 0) */ +#define MSPI_MSPICFG_APBCLK_Msk (0x1UL) /*!< MSPI MSPICFG: APBCLK (Bitfield-Mask: 0x01) */ +/* ======================================================== PADCFG ========================================================= */ +#define MSPI_PADCFG_REVCS_Pos (21UL) /*!< MSPI PADCFG: REVCS (Bit 21) */ +#define MSPI_PADCFG_REVCS_Msk (0x200000UL) /*!< MSPI PADCFG: REVCS (Bitfield-Mask: 0x01) */ +#define MSPI_PADCFG_IN3_Pos (20UL) /*!< MSPI PADCFG: IN3 (Bit 20) */ +#define MSPI_PADCFG_IN3_Msk (0x100000UL) /*!< MSPI PADCFG: IN3 (Bitfield-Mask: 0x01) */ +#define MSPI_PADCFG_IN2_Pos (19UL) /*!< MSPI PADCFG: IN2 (Bit 19) */ +#define MSPI_PADCFG_IN2_Msk (0x80000UL) /*!< MSPI PADCFG: IN2 (Bitfield-Mask: 0x01) */ +#define MSPI_PADCFG_IN1_Pos (18UL) /*!< MSPI PADCFG: IN1 (Bit 18) */ +#define MSPI_PADCFG_IN1_Msk (0x40000UL) /*!< MSPI PADCFG: IN1 (Bitfield-Mask: 0x01) */ +#define MSPI_PADCFG_IN0_Pos (16UL) /*!< MSPI PADCFG: IN0 (Bit 16) */ +#define MSPI_PADCFG_IN0_Msk (0x30000UL) /*!< MSPI PADCFG: IN0 (Bitfield-Mask: 0x03) */ +#define MSPI_PADCFG_OUT7_Pos (4UL) /*!< MSPI PADCFG: OUT7 (Bit 4) */ +#define MSPI_PADCFG_OUT7_Msk (0x10UL) /*!< MSPI PADCFG: OUT7 (Bitfield-Mask: 0x01) */ +#define MSPI_PADCFG_OUT6_Pos (3UL) /*!< MSPI PADCFG: OUT6 (Bit 3) */ +#define MSPI_PADCFG_OUT6_Msk (0x8UL) /*!< MSPI PADCFG: OUT6 (Bitfield-Mask: 0x01) */ +#define MSPI_PADCFG_OUT5_Pos (2UL) /*!< MSPI PADCFG: OUT5 (Bit 2) */ +#define MSPI_PADCFG_OUT5_Msk (0x4UL) /*!< MSPI PADCFG: OUT5 (Bitfield-Mask: 0x01) */ +#define MSPI_PADCFG_OUT4_Pos (1UL) /*!< MSPI PADCFG: OUT4 (Bit 1) */ +#define MSPI_PADCFG_OUT4_Msk (0x2UL) /*!< MSPI PADCFG: OUT4 (Bitfield-Mask: 0x01) */ +#define MSPI_PADCFG_OUT3_Pos (0UL) /*!< MSPI PADCFG: OUT3 (Bit 0) */ +#define MSPI_PADCFG_OUT3_Msk (0x1UL) /*!< MSPI PADCFG: OUT3 (Bitfield-Mask: 0x01) */ +/* ======================================================= PADOUTEN ======================================================== */ +#define MSPI_PADOUTEN_OUTEN_Pos (0UL) /*!< MSPI PADOUTEN: OUTEN (Bit 0) */ +#define MSPI_PADOUTEN_OUTEN_Msk (0x1ffUL) /*!< MSPI PADOUTEN: OUTEN (Bitfield-Mask: 0x1ff) */ +/* ========================================================= FLASH ========================================================= */ +#define MSPI_FLASH_READINSTR_Pos (24UL) /*!< MSPI FLASH: READINSTR (Bit 24) */ +#define MSPI_FLASH_READINSTR_Msk (0xff000000UL) /*!< MSPI FLASH: READINSTR (Bitfield-Mask: 0xff) */ +#define MSPI_FLASH_WRITEINSTR_Pos (16UL) /*!< MSPI FLASH: WRITEINSTR (Bit 16) */ +#define MSPI_FLASH_WRITEINSTR_Msk (0xff0000UL) /*!< MSPI FLASH: WRITEINSTR (Bitfield-Mask: 0xff) */ +#define MSPI_FLASH_XIPMIXED_Pos (8UL) /*!< MSPI FLASH: XIPMIXED (Bit 8) */ +#define MSPI_FLASH_XIPMIXED_Msk (0x700UL) /*!< MSPI FLASH: XIPMIXED (Bitfield-Mask: 0x07) */ +#define MSPI_FLASH_XIPSENDI_Pos (7UL) /*!< MSPI FLASH: XIPSENDI (Bit 7) */ +#define MSPI_FLASH_XIPSENDI_Msk (0x80UL) /*!< MSPI FLASH: XIPSENDI (Bitfield-Mask: 0x01) */ +#define MSPI_FLASH_XIPSENDA_Pos (6UL) /*!< MSPI FLASH: XIPSENDA (Bit 6) */ +#define MSPI_FLASH_XIPSENDA_Msk (0x40UL) /*!< MSPI FLASH: XIPSENDA (Bitfield-Mask: 0x01) */ +#define MSPI_FLASH_XIPENTURN_Pos (5UL) /*!< MSPI FLASH: XIPENTURN (Bit 5) */ +#define MSPI_FLASH_XIPENTURN_Msk (0x20UL) /*!< MSPI FLASH: XIPENTURN (Bitfield-Mask: 0x01) */ +#define MSPI_FLASH_XIPBIGENDIAN_Pos (4UL) /*!< MSPI FLASH: XIPBIGENDIAN (Bit 4) */ +#define MSPI_FLASH_XIPBIGENDIAN_Msk (0x10UL) /*!< MSPI FLASH: XIPBIGENDIAN (Bitfield-Mask: 0x01) */ +#define MSPI_FLASH_XIPACK_Pos (2UL) /*!< MSPI FLASH: XIPACK (Bit 2) */ +#define MSPI_FLASH_XIPACK_Msk (0xcUL) /*!< MSPI FLASH: XIPACK (Bitfield-Mask: 0x03) */ +#define MSPI_FLASH_XIPEN_Pos (0UL) /*!< MSPI FLASH: XIPEN (Bit 0) */ +#define MSPI_FLASH_XIPEN_Msk (0x1UL) /*!< MSPI FLASH: XIPEN (Bitfield-Mask: 0x01) */ +/* ====================================================== SCRAMBLING ======================================================= */ +#define MSPI_SCRAMBLING_SCRENABLE_Pos (31UL) /*!< MSPI SCRAMBLING: SCRENABLE (Bit 31) */ +#define MSPI_SCRAMBLING_SCRENABLE_Msk (0x80000000UL) /*!< MSPI SCRAMBLING: SCRENABLE (Bitfield-Mask: 0x01) */ +#define MSPI_SCRAMBLING_SCREND_Pos (16UL) /*!< MSPI SCRAMBLING: SCREND (Bit 16) */ +#define MSPI_SCRAMBLING_SCREND_Msk (0x3ff0000UL) /*!< MSPI SCRAMBLING: SCREND (Bitfield-Mask: 0x3ff) */ +#define MSPI_SCRAMBLING_SCRSTART_Pos (0UL) /*!< MSPI SCRAMBLING: SCRSTART (Bit 0) */ +#define MSPI_SCRAMBLING_SCRSTART_Msk (0x3ffUL) /*!< MSPI SCRAMBLING: SCRSTART (Bitfield-Mask: 0x3ff) */ +/* ========================================================= INTEN ========================================================= */ +#define MSPI_INTEN_SCRERR_Pos (12UL) /*!< MSPI INTEN: SCRERR (Bit 12) */ +#define MSPI_INTEN_SCRERR_Msk (0x1000UL) /*!< MSPI INTEN: SCRERR (Bitfield-Mask: 0x01) */ +#define MSPI_INTEN_CQERR_Pos (11UL) /*!< MSPI INTEN: CQERR (Bit 11) */ +#define MSPI_INTEN_CQERR_Msk (0x800UL) /*!< MSPI INTEN: CQERR (Bitfield-Mask: 0x01) */ +#define MSPI_INTEN_CQPAUSED_Pos (10UL) /*!< MSPI INTEN: CQPAUSED (Bit 10) */ +#define MSPI_INTEN_CQPAUSED_Msk (0x400UL) /*!< MSPI INTEN: CQPAUSED (Bitfield-Mask: 0x01) */ +#define MSPI_INTEN_CQUPD_Pos (9UL) /*!< MSPI INTEN: CQUPD (Bit 9) */ +#define MSPI_INTEN_CQUPD_Msk (0x200UL) /*!< MSPI INTEN: CQUPD (Bitfield-Mask: 0x01) */ +#define MSPI_INTEN_CQCMP_Pos (8UL) /*!< MSPI INTEN: CQCMP (Bit 8) */ +#define MSPI_INTEN_CQCMP_Msk (0x100UL) /*!< MSPI INTEN: CQCMP (Bitfield-Mask: 0x01) */ +#define MSPI_INTEN_DERR_Pos (7UL) /*!< MSPI INTEN: DERR (Bit 7) */ +#define MSPI_INTEN_DERR_Msk (0x80UL) /*!< MSPI INTEN: DERR (Bitfield-Mask: 0x01) */ +#define MSPI_INTEN_DCMP_Pos (6UL) /*!< MSPI INTEN: DCMP (Bit 6) */ +#define MSPI_INTEN_DCMP_Msk (0x40UL) /*!< MSPI INTEN: DCMP (Bitfield-Mask: 0x01) */ +#define MSPI_INTEN_RXF_Pos (5UL) /*!< MSPI INTEN: RXF (Bit 5) */ +#define MSPI_INTEN_RXF_Msk (0x20UL) /*!< MSPI INTEN: RXF (Bitfield-Mask: 0x01) */ +#define MSPI_INTEN_RXO_Pos (4UL) /*!< MSPI INTEN: RXO (Bit 4) */ +#define MSPI_INTEN_RXO_Msk (0x10UL) /*!< MSPI INTEN: RXO (Bitfield-Mask: 0x01) */ +#define MSPI_INTEN_RXU_Pos (3UL) /*!< MSPI INTEN: RXU (Bit 3) */ +#define MSPI_INTEN_RXU_Msk (0x8UL) /*!< MSPI INTEN: RXU (Bitfield-Mask: 0x01) */ +#define MSPI_INTEN_TXO_Pos (2UL) /*!< MSPI INTEN: TXO (Bit 2) */ +#define MSPI_INTEN_TXO_Msk (0x4UL) /*!< MSPI INTEN: TXO (Bitfield-Mask: 0x01) */ +#define MSPI_INTEN_TXE_Pos (1UL) /*!< MSPI INTEN: TXE (Bit 1) */ +#define MSPI_INTEN_TXE_Msk (0x2UL) /*!< MSPI INTEN: TXE (Bitfield-Mask: 0x01) */ +#define MSPI_INTEN_CMDCMP_Pos (0UL) /*!< MSPI INTEN: CMDCMP (Bit 0) */ +#define MSPI_INTEN_CMDCMP_Msk (0x1UL) /*!< MSPI INTEN: CMDCMP (Bitfield-Mask: 0x01) */ +/* ======================================================== INTSTAT ======================================================== */ +#define MSPI_INTSTAT_SCRERR_Pos (12UL) /*!< MSPI INTSTAT: SCRERR (Bit 12) */ +#define MSPI_INTSTAT_SCRERR_Msk (0x1000UL) /*!< MSPI INTSTAT: SCRERR (Bitfield-Mask: 0x01) */ +#define MSPI_INTSTAT_CQERR_Pos (11UL) /*!< MSPI INTSTAT: CQERR (Bit 11) */ +#define MSPI_INTSTAT_CQERR_Msk (0x800UL) /*!< MSPI INTSTAT: CQERR (Bitfield-Mask: 0x01) */ +#define MSPI_INTSTAT_CQPAUSED_Pos (10UL) /*!< MSPI INTSTAT: CQPAUSED (Bit 10) */ +#define MSPI_INTSTAT_CQPAUSED_Msk (0x400UL) /*!< MSPI INTSTAT: CQPAUSED (Bitfield-Mask: 0x01) */ +#define MSPI_INTSTAT_CQUPD_Pos (9UL) /*!< MSPI INTSTAT: CQUPD (Bit 9) */ +#define MSPI_INTSTAT_CQUPD_Msk (0x200UL) /*!< MSPI INTSTAT: CQUPD (Bitfield-Mask: 0x01) */ +#define MSPI_INTSTAT_CQCMP_Pos (8UL) /*!< MSPI INTSTAT: CQCMP (Bit 8) */ +#define MSPI_INTSTAT_CQCMP_Msk (0x100UL) /*!< MSPI INTSTAT: CQCMP (Bitfield-Mask: 0x01) */ +#define MSPI_INTSTAT_DERR_Pos (7UL) /*!< MSPI INTSTAT: DERR (Bit 7) */ +#define MSPI_INTSTAT_DERR_Msk (0x80UL) /*!< MSPI INTSTAT: DERR (Bitfield-Mask: 0x01) */ +#define MSPI_INTSTAT_DCMP_Pos (6UL) /*!< MSPI INTSTAT: DCMP (Bit 6) */ +#define MSPI_INTSTAT_DCMP_Msk (0x40UL) /*!< MSPI INTSTAT: DCMP (Bitfield-Mask: 0x01) */ +#define MSPI_INTSTAT_RXF_Pos (5UL) /*!< MSPI INTSTAT: RXF (Bit 5) */ +#define MSPI_INTSTAT_RXF_Msk (0x20UL) /*!< MSPI INTSTAT: RXF (Bitfield-Mask: 0x01) */ +#define MSPI_INTSTAT_RXO_Pos (4UL) /*!< MSPI INTSTAT: RXO (Bit 4) */ +#define MSPI_INTSTAT_RXO_Msk (0x10UL) /*!< MSPI INTSTAT: RXO (Bitfield-Mask: 0x01) */ +#define MSPI_INTSTAT_RXU_Pos (3UL) /*!< MSPI INTSTAT: RXU (Bit 3) */ +#define MSPI_INTSTAT_RXU_Msk (0x8UL) /*!< MSPI INTSTAT: RXU (Bitfield-Mask: 0x01) */ +#define MSPI_INTSTAT_TXO_Pos (2UL) /*!< MSPI INTSTAT: TXO (Bit 2) */ +#define MSPI_INTSTAT_TXO_Msk (0x4UL) /*!< MSPI INTSTAT: TXO (Bitfield-Mask: 0x01) */ +#define MSPI_INTSTAT_TXE_Pos (1UL) /*!< MSPI INTSTAT: TXE (Bit 1) */ +#define MSPI_INTSTAT_TXE_Msk (0x2UL) /*!< MSPI INTSTAT: TXE (Bitfield-Mask: 0x01) */ +#define MSPI_INTSTAT_CMDCMP_Pos (0UL) /*!< MSPI INTSTAT: CMDCMP (Bit 0) */ +#define MSPI_INTSTAT_CMDCMP_Msk (0x1UL) /*!< MSPI INTSTAT: CMDCMP (Bitfield-Mask: 0x01) */ +/* ======================================================== INTCLR ========================================================= */ +#define MSPI_INTCLR_SCRERR_Pos (12UL) /*!< MSPI INTCLR: SCRERR (Bit 12) */ +#define MSPI_INTCLR_SCRERR_Msk (0x1000UL) /*!< MSPI INTCLR: SCRERR (Bitfield-Mask: 0x01) */ +#define MSPI_INTCLR_CQERR_Pos (11UL) /*!< MSPI INTCLR: CQERR (Bit 11) */ +#define MSPI_INTCLR_CQERR_Msk (0x800UL) /*!< MSPI INTCLR: CQERR (Bitfield-Mask: 0x01) */ +#define MSPI_INTCLR_CQPAUSED_Pos (10UL) /*!< MSPI INTCLR: CQPAUSED (Bit 10) */ +#define MSPI_INTCLR_CQPAUSED_Msk (0x400UL) /*!< MSPI INTCLR: CQPAUSED (Bitfield-Mask: 0x01) */ +#define MSPI_INTCLR_CQUPD_Pos (9UL) /*!< MSPI INTCLR: CQUPD (Bit 9) */ +#define MSPI_INTCLR_CQUPD_Msk (0x200UL) /*!< MSPI INTCLR: CQUPD (Bitfield-Mask: 0x01) */ +#define MSPI_INTCLR_CQCMP_Pos (8UL) /*!< MSPI INTCLR: CQCMP (Bit 8) */ +#define MSPI_INTCLR_CQCMP_Msk (0x100UL) /*!< MSPI INTCLR: CQCMP (Bitfield-Mask: 0x01) */ +#define MSPI_INTCLR_DERR_Pos (7UL) /*!< MSPI INTCLR: DERR (Bit 7) */ +#define MSPI_INTCLR_DERR_Msk (0x80UL) /*!< MSPI INTCLR: DERR (Bitfield-Mask: 0x01) */ +#define MSPI_INTCLR_DCMP_Pos (6UL) /*!< MSPI INTCLR: DCMP (Bit 6) */ +#define MSPI_INTCLR_DCMP_Msk (0x40UL) /*!< MSPI INTCLR: DCMP (Bitfield-Mask: 0x01) */ +#define MSPI_INTCLR_RXF_Pos (5UL) /*!< MSPI INTCLR: RXF (Bit 5) */ +#define MSPI_INTCLR_RXF_Msk (0x20UL) /*!< MSPI INTCLR: RXF (Bitfield-Mask: 0x01) */ +#define MSPI_INTCLR_RXO_Pos (4UL) /*!< MSPI INTCLR: RXO (Bit 4) */ +#define MSPI_INTCLR_RXO_Msk (0x10UL) /*!< MSPI INTCLR: RXO (Bitfield-Mask: 0x01) */ +#define MSPI_INTCLR_RXU_Pos (3UL) /*!< MSPI INTCLR: RXU (Bit 3) */ +#define MSPI_INTCLR_RXU_Msk (0x8UL) /*!< MSPI INTCLR: RXU (Bitfield-Mask: 0x01) */ +#define MSPI_INTCLR_TXO_Pos (2UL) /*!< MSPI INTCLR: TXO (Bit 2) */ +#define MSPI_INTCLR_TXO_Msk (0x4UL) /*!< MSPI INTCLR: TXO (Bitfield-Mask: 0x01) */ +#define MSPI_INTCLR_TXE_Pos (1UL) /*!< MSPI INTCLR: TXE (Bit 1) */ +#define MSPI_INTCLR_TXE_Msk (0x2UL) /*!< MSPI INTCLR: TXE (Bitfield-Mask: 0x01) */ +#define MSPI_INTCLR_CMDCMP_Pos (0UL) /*!< MSPI INTCLR: CMDCMP (Bit 0) */ +#define MSPI_INTCLR_CMDCMP_Msk (0x1UL) /*!< MSPI INTCLR: CMDCMP (Bitfield-Mask: 0x01) */ +/* ======================================================== INTSET ========================================================= */ +#define MSPI_INTSET_SCRERR_Pos (12UL) /*!< MSPI INTSET: SCRERR (Bit 12) */ +#define MSPI_INTSET_SCRERR_Msk (0x1000UL) /*!< MSPI INTSET: SCRERR (Bitfield-Mask: 0x01) */ +#define MSPI_INTSET_CQERR_Pos (11UL) /*!< MSPI INTSET: CQERR (Bit 11) */ +#define MSPI_INTSET_CQERR_Msk (0x800UL) /*!< MSPI INTSET: CQERR (Bitfield-Mask: 0x01) */ +#define MSPI_INTSET_CQPAUSED_Pos (10UL) /*!< MSPI INTSET: CQPAUSED (Bit 10) */ +#define MSPI_INTSET_CQPAUSED_Msk (0x400UL) /*!< MSPI INTSET: CQPAUSED (Bitfield-Mask: 0x01) */ +#define MSPI_INTSET_CQUPD_Pos (9UL) /*!< MSPI INTSET: CQUPD (Bit 9) */ +#define MSPI_INTSET_CQUPD_Msk (0x200UL) /*!< MSPI INTSET: CQUPD (Bitfield-Mask: 0x01) */ +#define MSPI_INTSET_CQCMP_Pos (8UL) /*!< MSPI INTSET: CQCMP (Bit 8) */ +#define MSPI_INTSET_CQCMP_Msk (0x100UL) /*!< MSPI INTSET: CQCMP (Bitfield-Mask: 0x01) */ +#define MSPI_INTSET_DERR_Pos (7UL) /*!< MSPI INTSET: DERR (Bit 7) */ +#define MSPI_INTSET_DERR_Msk (0x80UL) /*!< MSPI INTSET: DERR (Bitfield-Mask: 0x01) */ +#define MSPI_INTSET_DCMP_Pos (6UL) /*!< MSPI INTSET: DCMP (Bit 6) */ +#define MSPI_INTSET_DCMP_Msk (0x40UL) /*!< MSPI INTSET: DCMP (Bitfield-Mask: 0x01) */ +#define MSPI_INTSET_RXF_Pos (5UL) /*!< MSPI INTSET: RXF (Bit 5) */ +#define MSPI_INTSET_RXF_Msk (0x20UL) /*!< MSPI INTSET: RXF (Bitfield-Mask: 0x01) */ +#define MSPI_INTSET_RXO_Pos (4UL) /*!< MSPI INTSET: RXO (Bit 4) */ +#define MSPI_INTSET_RXO_Msk (0x10UL) /*!< MSPI INTSET: RXO (Bitfield-Mask: 0x01) */ +#define MSPI_INTSET_RXU_Pos (3UL) /*!< MSPI INTSET: RXU (Bit 3) */ +#define MSPI_INTSET_RXU_Msk (0x8UL) /*!< MSPI INTSET: RXU (Bitfield-Mask: 0x01) */ +#define MSPI_INTSET_TXO_Pos (2UL) /*!< MSPI INTSET: TXO (Bit 2) */ +#define MSPI_INTSET_TXO_Msk (0x4UL) /*!< MSPI INTSET: TXO (Bitfield-Mask: 0x01) */ +#define MSPI_INTSET_TXE_Pos (1UL) /*!< MSPI INTSET: TXE (Bit 1) */ +#define MSPI_INTSET_TXE_Msk (0x2UL) /*!< MSPI INTSET: TXE (Bitfield-Mask: 0x01) */ +#define MSPI_INTSET_CMDCMP_Pos (0UL) /*!< MSPI INTSET: CMDCMP (Bit 0) */ +#define MSPI_INTSET_CMDCMP_Msk (0x1UL) /*!< MSPI INTSET: CMDCMP (Bitfield-Mask: 0x01) */ +/* ======================================================== DMACFG ========================================================= */ +#define MSPI_DMACFG_DMAPWROFF_Pos (18UL) /*!< MSPI DMACFG: DMAPWROFF (Bit 18) */ +#define MSPI_DMACFG_DMAPWROFF_Msk (0x40000UL) /*!< MSPI DMACFG: DMAPWROFF (Bitfield-Mask: 0x01) */ +#define MSPI_DMACFG_DMAPRI_Pos (3UL) /*!< MSPI DMACFG: DMAPRI (Bit 3) */ +#define MSPI_DMACFG_DMAPRI_Msk (0x18UL) /*!< MSPI DMACFG: DMAPRI (Bitfield-Mask: 0x03) */ +#define MSPI_DMACFG_DMADIR_Pos (2UL) /*!< MSPI DMACFG: DMADIR (Bit 2) */ +#define MSPI_DMACFG_DMADIR_Msk (0x4UL) /*!< MSPI DMACFG: DMADIR (Bitfield-Mask: 0x01) */ +#define MSPI_DMACFG_DMAEN_Pos (0UL) /*!< MSPI DMACFG: DMAEN (Bit 0) */ +#define MSPI_DMACFG_DMAEN_Msk (0x3UL) /*!< MSPI DMACFG: DMAEN (Bitfield-Mask: 0x03) */ +/* ======================================================== DMASTAT ======================================================== */ +#define MSPI_DMASTAT_SCRERR_Pos (3UL) /*!< MSPI DMASTAT: SCRERR (Bit 3) */ +#define MSPI_DMASTAT_SCRERR_Msk (0x8UL) /*!< MSPI DMASTAT: SCRERR (Bitfield-Mask: 0x01) */ +#define MSPI_DMASTAT_DMAERR_Pos (2UL) /*!< MSPI DMASTAT: DMAERR (Bit 2) */ +#define MSPI_DMASTAT_DMAERR_Msk (0x4UL) /*!< MSPI DMASTAT: DMAERR (Bitfield-Mask: 0x01) */ +#define MSPI_DMASTAT_DMACPL_Pos (1UL) /*!< MSPI DMASTAT: DMACPL (Bit 1) */ +#define MSPI_DMASTAT_DMACPL_Msk (0x2UL) /*!< MSPI DMASTAT: DMACPL (Bitfield-Mask: 0x01) */ +#define MSPI_DMASTAT_DMATIP_Pos (0UL) /*!< MSPI DMASTAT: DMATIP (Bit 0) */ +#define MSPI_DMASTAT_DMATIP_Msk (0x1UL) /*!< MSPI DMASTAT: DMATIP (Bitfield-Mask: 0x01) */ +/* ====================================================== DMATARGADDR ====================================================== */ +#define MSPI_DMATARGADDR_TARGADDR_Pos (0UL) /*!< MSPI DMATARGADDR: TARGADDR (Bit 0) */ +#define MSPI_DMATARGADDR_TARGADDR_Msk (0xffffffffUL) /*!< MSPI DMATARGADDR: TARGADDR (Bitfield-Mask: 0xffffffff) */ +/* ====================================================== DMADEVADDR ======================================================= */ +#define MSPI_DMADEVADDR_DEVADDR_Pos (0UL) /*!< MSPI DMADEVADDR: DEVADDR (Bit 0) */ +#define MSPI_DMADEVADDR_DEVADDR_Msk (0xffffffffUL) /*!< MSPI DMADEVADDR: DEVADDR (Bitfield-Mask: 0xffffffff) */ +/* ====================================================== DMATOTCOUNT ====================================================== */ +#define MSPI_DMATOTCOUNT_TOTCOUNT_Pos (0UL) /*!< MSPI DMATOTCOUNT: TOTCOUNT (Bit 0) */ +#define MSPI_DMATOTCOUNT_TOTCOUNT_Msk (0xffffUL) /*!< MSPI DMATOTCOUNT: TOTCOUNT (Bitfield-Mask: 0xffff) */ +/* ======================================================= DMABCOUNT ======================================================= */ +#define MSPI_DMABCOUNT_BCOUNT_Pos (0UL) /*!< MSPI DMABCOUNT: BCOUNT (Bit 0) */ +#define MSPI_DMABCOUNT_BCOUNT_Msk (0xffUL) /*!< MSPI DMABCOUNT: BCOUNT (Bitfield-Mask: 0xff) */ +/* ======================================================= DMATHRESH ======================================================= */ +#define MSPI_DMATHRESH_DMATHRESH_Pos (0UL) /*!< MSPI DMATHRESH: DMATHRESH (Bit 0) */ +#define MSPI_DMATHRESH_DMATHRESH_Msk (0xfUL) /*!< MSPI DMATHRESH: DMATHRESH (Bitfield-Mask: 0x0f) */ +/* ========================================================= CQCFG ========================================================= */ +#define MSPI_CQCFG_CQAUTOCLEARMASK_Pos (3UL) /*!< MSPI CQCFG: CQAUTOCLEARMASK (Bit 3) */ +#define MSPI_CQCFG_CQAUTOCLEARMASK_Msk (0x8UL) /*!< MSPI CQCFG: CQAUTOCLEARMASK (Bitfield-Mask: 0x01) */ +#define MSPI_CQCFG_CQPWROFF_Pos (2UL) /*!< MSPI CQCFG: CQPWROFF (Bit 2) */ +#define MSPI_CQCFG_CQPWROFF_Msk (0x4UL) /*!< MSPI CQCFG: CQPWROFF (Bitfield-Mask: 0x01) */ +#define MSPI_CQCFG_CQPRI_Pos (1UL) /*!< MSPI CQCFG: CQPRI (Bit 1) */ +#define MSPI_CQCFG_CQPRI_Msk (0x2UL) /*!< MSPI CQCFG: CQPRI (Bitfield-Mask: 0x01) */ +#define MSPI_CQCFG_CQEN_Pos (0UL) /*!< MSPI CQCFG: CQEN (Bit 0) */ +#define MSPI_CQCFG_CQEN_Msk (0x1UL) /*!< MSPI CQCFG: CQEN (Bitfield-Mask: 0x01) */ +/* ======================================================== CQADDR ========================================================= */ +#define MSPI_CQADDR_CQADDR_Pos (0UL) /*!< MSPI CQADDR: CQADDR (Bit 0) */ +#define MSPI_CQADDR_CQADDR_Msk (0x1fffffffUL) /*!< MSPI CQADDR: CQADDR (Bitfield-Mask: 0x1fffffff) */ +/* ======================================================== CQSTAT ========================================================= */ +#define MSPI_CQSTAT_CQPAUSED_Pos (3UL) /*!< MSPI CQSTAT: CQPAUSED (Bit 3) */ +#define MSPI_CQSTAT_CQPAUSED_Msk (0x8UL) /*!< MSPI CQSTAT: CQPAUSED (Bitfield-Mask: 0x01) */ +#define MSPI_CQSTAT_CQERR_Pos (2UL) /*!< MSPI CQSTAT: CQERR (Bit 2) */ +#define MSPI_CQSTAT_CQERR_Msk (0x4UL) /*!< MSPI CQSTAT: CQERR (Bitfield-Mask: 0x01) */ +#define MSPI_CQSTAT_CQCPL_Pos (1UL) /*!< MSPI CQSTAT: CQCPL (Bit 1) */ +#define MSPI_CQSTAT_CQCPL_Msk (0x2UL) /*!< MSPI CQSTAT: CQCPL (Bitfield-Mask: 0x01) */ +#define MSPI_CQSTAT_CQTIP_Pos (0UL) /*!< MSPI CQSTAT: CQTIP (Bit 0) */ +#define MSPI_CQSTAT_CQTIP_Msk (0x1UL) /*!< MSPI CQSTAT: CQTIP (Bitfield-Mask: 0x01) */ +/* ======================================================== CQFLAGS ======================================================== */ +#define MSPI_CQFLAGS_CQFLAGS_Pos (0UL) /*!< MSPI CQFLAGS: CQFLAGS (Bit 0) */ +#define MSPI_CQFLAGS_CQFLAGS_Msk (0xffffUL) /*!< MSPI CQFLAGS: CQFLAGS (Bitfield-Mask: 0xffff) */ +/* ====================================================== CQSETCLEAR ======================================================= */ +#define MSPI_CQSETCLEAR_CQFCLR_Pos (16UL) /*!< MSPI CQSETCLEAR: CQFCLR (Bit 16) */ +#define MSPI_CQSETCLEAR_CQFCLR_Msk (0xff0000UL) /*!< MSPI CQSETCLEAR: CQFCLR (Bitfield-Mask: 0xff) */ +#define MSPI_CQSETCLEAR_CQFTOGGLE_Pos (8UL) /*!< MSPI CQSETCLEAR: CQFTOGGLE (Bit 8) */ +#define MSPI_CQSETCLEAR_CQFTOGGLE_Msk (0xff00UL) /*!< MSPI CQSETCLEAR: CQFTOGGLE (Bitfield-Mask: 0xff) */ +#define MSPI_CQSETCLEAR_CQFSET_Pos (0UL) /*!< MSPI CQSETCLEAR: CQFSET (Bit 0) */ +#define MSPI_CQSETCLEAR_CQFSET_Msk (0xffUL) /*!< MSPI CQSETCLEAR: CQFSET (Bitfield-Mask: 0xff) */ +/* ======================================================== CQPAUSE ======================================================== */ +#define MSPI_CQPAUSE_CQMASK_Pos (0UL) /*!< MSPI CQPAUSE: CQMASK (Bit 0) */ +#define MSPI_CQPAUSE_CQMASK_Msk (0xffffUL) /*!< MSPI CQPAUSE: CQMASK (Bitfield-Mask: 0xffff) */ +/* ======================================================= CQCURIDX ======================================================== */ +#define MSPI_CQCURIDX_CQCURIDX_Pos (0UL) /*!< MSPI CQCURIDX: CQCURIDX (Bit 0) */ +#define MSPI_CQCURIDX_CQCURIDX_Msk (0xffUL) /*!< MSPI CQCURIDX: CQCURIDX (Bitfield-Mask: 0xff) */ +/* ======================================================= CQENDIDX ======================================================== */ +#define MSPI_CQENDIDX_CQENDIDX_Pos (0UL) /*!< MSPI CQENDIDX: CQENDIDX (Bit 0) */ +#define MSPI_CQENDIDX_CQENDIDX_Msk (0xffUL) /*!< MSPI CQENDIDX: CQENDIDX (Bitfield-Mask: 0xff) */ + + +/* =========================================================================================================================== */ +/* ================ PDM ================ */ +/* =========================================================================================================================== */ + +/* ========================================================= PCFG ========================================================== */ +#define PDM_PCFG_LRSWAP_Pos (31UL) /*!< PDM PCFG: LRSWAP (Bit 31) */ +#define PDM_PCFG_LRSWAP_Msk (0x80000000UL) /*!< PDM PCFG: LRSWAP (Bitfield-Mask: 0x01) */ +#define PDM_PCFG_PGARIGHT_Pos (26UL) /*!< PDM PCFG: PGARIGHT (Bit 26) */ +#define PDM_PCFG_PGARIGHT_Msk (0x7c000000UL) /*!< PDM PCFG: PGARIGHT (Bitfield-Mask: 0x1f) */ +#define PDM_PCFG_PGALEFT_Pos (21UL) /*!< PDM PCFG: PGALEFT (Bit 21) */ +#define PDM_PCFG_PGALEFT_Msk (0x3e00000UL) /*!< PDM PCFG: PGALEFT (Bitfield-Mask: 0x1f) */ +#define PDM_PCFG_MCLKDIV_Pos (17UL) /*!< PDM PCFG: MCLKDIV (Bit 17) */ +#define PDM_PCFG_MCLKDIV_Msk (0x60000UL) /*!< PDM PCFG: MCLKDIV (Bitfield-Mask: 0x03) */ +#define PDM_PCFG_SINCRATE_Pos (10UL) /*!< PDM PCFG: SINCRATE (Bit 10) */ +#define PDM_PCFG_SINCRATE_Msk (0x1fc00UL) /*!< PDM PCFG: SINCRATE (Bitfield-Mask: 0x7f) */ +#define PDM_PCFG_ADCHPD_Pos (9UL) /*!< PDM PCFG: ADCHPD (Bit 9) */ +#define PDM_PCFG_ADCHPD_Msk (0x200UL) /*!< PDM PCFG: ADCHPD (Bitfield-Mask: 0x01) */ +#define PDM_PCFG_HPCUTOFF_Pos (5UL) /*!< PDM PCFG: HPCUTOFF (Bit 5) */ +#define PDM_PCFG_HPCUTOFF_Msk (0x1e0UL) /*!< PDM PCFG: HPCUTOFF (Bitfield-Mask: 0x0f) */ +#define PDM_PCFG_CYCLES_Pos (2UL) /*!< PDM PCFG: CYCLES (Bit 2) */ +#define PDM_PCFG_CYCLES_Msk (0x1cUL) /*!< PDM PCFG: CYCLES (Bitfield-Mask: 0x07) */ +#define PDM_PCFG_SOFTMUTE_Pos (1UL) /*!< PDM PCFG: SOFTMUTE (Bit 1) */ +#define PDM_PCFG_SOFTMUTE_Msk (0x2UL) /*!< PDM PCFG: SOFTMUTE (Bitfield-Mask: 0x01) */ +#define PDM_PCFG_PDMCOREEN_Pos (0UL) /*!< PDM PCFG: PDMCOREEN (Bit 0) */ +#define PDM_PCFG_PDMCOREEN_Msk (0x1UL) /*!< PDM PCFG: PDMCOREEN (Bitfield-Mask: 0x01) */ +/* ========================================================= VCFG ========================================================== */ +#define PDM_VCFG_IOCLKEN_Pos (31UL) /*!< PDM VCFG: IOCLKEN (Bit 31) */ +#define PDM_VCFG_IOCLKEN_Msk (0x80000000UL) /*!< PDM VCFG: IOCLKEN (Bitfield-Mask: 0x01) */ +#define PDM_VCFG_RSTB_Pos (30UL) /*!< PDM VCFG: RSTB (Bit 30) */ +#define PDM_VCFG_RSTB_Msk (0x40000000UL) /*!< PDM VCFG: RSTB (Bitfield-Mask: 0x01) */ +#define PDM_VCFG_PDMCLKSEL_Pos (27UL) /*!< PDM VCFG: PDMCLKSEL (Bit 27) */ +#define PDM_VCFG_PDMCLKSEL_Msk (0x38000000UL) /*!< PDM VCFG: PDMCLKSEL (Bitfield-Mask: 0x07) */ +#define PDM_VCFG_PDMCLKEN_Pos (26UL) /*!< PDM VCFG: PDMCLKEN (Bit 26) */ +#define PDM_VCFG_PDMCLKEN_Msk (0x4000000UL) /*!< PDM VCFG: PDMCLKEN (Bitfield-Mask: 0x01) */ +#define PDM_VCFG_I2SEN_Pos (20UL) /*!< PDM VCFG: I2SEN (Bit 20) */ +#define PDM_VCFG_I2SEN_Msk (0x100000UL) /*!< PDM VCFG: I2SEN (Bitfield-Mask: 0x01) */ +#define PDM_VCFG_BCLKINV_Pos (19UL) /*!< PDM VCFG: BCLKINV (Bit 19) */ +#define PDM_VCFG_BCLKINV_Msk (0x80000UL) /*!< PDM VCFG: BCLKINV (Bitfield-Mask: 0x01) */ +#define PDM_VCFG_DMICKDEL_Pos (17UL) /*!< PDM VCFG: DMICKDEL (Bit 17) */ +#define PDM_VCFG_DMICKDEL_Msk (0x20000UL) /*!< PDM VCFG: DMICKDEL (Bitfield-Mask: 0x01) */ +#define PDM_VCFG_SELAP_Pos (16UL) /*!< PDM VCFG: SELAP (Bit 16) */ +#define PDM_VCFG_SELAP_Msk (0x10000UL) /*!< PDM VCFG: SELAP (Bitfield-Mask: 0x01) */ +#define PDM_VCFG_PCMPACK_Pos (8UL) /*!< PDM VCFG: PCMPACK (Bit 8) */ +#define PDM_VCFG_PCMPACK_Msk (0x100UL) /*!< PDM VCFG: PCMPACK (Bitfield-Mask: 0x01) */ +#define PDM_VCFG_CHSET_Pos (3UL) /*!< PDM VCFG: CHSET (Bit 3) */ +#define PDM_VCFG_CHSET_Msk (0x18UL) /*!< PDM VCFG: CHSET (Bitfield-Mask: 0x03) */ +/* ======================================================= VOICESTAT ======================================================= */ +#define PDM_VOICESTAT_FIFOCNT_Pos (0UL) /*!< PDM VOICESTAT: FIFOCNT (Bit 0) */ +#define PDM_VOICESTAT_FIFOCNT_Msk (0x3fUL) /*!< PDM VOICESTAT: FIFOCNT (Bitfield-Mask: 0x3f) */ +/* ======================================================= FIFOREAD ======================================================== */ +#define PDM_FIFOREAD_FIFOREAD_Pos (0UL) /*!< PDM FIFOREAD: FIFOREAD (Bit 0) */ +#define PDM_FIFOREAD_FIFOREAD_Msk (0xffffffffUL) /*!< PDM FIFOREAD: FIFOREAD (Bitfield-Mask: 0xffffffff) */ +/* ======================================================= FIFOFLUSH ======================================================= */ +#define PDM_FIFOFLUSH_FIFOFLUSH_Pos (0UL) /*!< PDM FIFOFLUSH: FIFOFLUSH (Bit 0) */ +#define PDM_FIFOFLUSH_FIFOFLUSH_Msk (0x1UL) /*!< PDM FIFOFLUSH: FIFOFLUSH (Bitfield-Mask: 0x01) */ +/* ======================================================== FIFOTHR ======================================================== */ +#define PDM_FIFOTHR_FIFOTHR_Pos (0UL) /*!< PDM FIFOTHR: FIFOTHR (Bit 0) */ +#define PDM_FIFOTHR_FIFOTHR_Msk (0x1fUL) /*!< PDM FIFOTHR: FIFOTHR (Bitfield-Mask: 0x1f) */ +/* ========================================================= INTEN ========================================================= */ +#define PDM_INTEN_DERR_Pos (4UL) /*!< PDM INTEN: DERR (Bit 4) */ +#define PDM_INTEN_DERR_Msk (0x10UL) /*!< PDM INTEN: DERR (Bitfield-Mask: 0x01) */ +#define PDM_INTEN_DCMP_Pos (3UL) /*!< PDM INTEN: DCMP (Bit 3) */ +#define PDM_INTEN_DCMP_Msk (0x8UL) /*!< PDM INTEN: DCMP (Bitfield-Mask: 0x01) */ +#define PDM_INTEN_UNDFL_Pos (2UL) /*!< PDM INTEN: UNDFL (Bit 2) */ +#define PDM_INTEN_UNDFL_Msk (0x4UL) /*!< PDM INTEN: UNDFL (Bitfield-Mask: 0x01) */ +#define PDM_INTEN_OVF_Pos (1UL) /*!< PDM INTEN: OVF (Bit 1) */ +#define PDM_INTEN_OVF_Msk (0x2UL) /*!< PDM INTEN: OVF (Bitfield-Mask: 0x01) */ +#define PDM_INTEN_THR_Pos (0UL) /*!< PDM INTEN: THR (Bit 0) */ +#define PDM_INTEN_THR_Msk (0x1UL) /*!< PDM INTEN: THR (Bitfield-Mask: 0x01) */ +/* ======================================================== INTSTAT ======================================================== */ +#define PDM_INTSTAT_DERR_Pos (4UL) /*!< PDM INTSTAT: DERR (Bit 4) */ +#define PDM_INTSTAT_DERR_Msk (0x10UL) /*!< PDM INTSTAT: DERR (Bitfield-Mask: 0x01) */ +#define PDM_INTSTAT_DCMP_Pos (3UL) /*!< PDM INTSTAT: DCMP (Bit 3) */ +#define PDM_INTSTAT_DCMP_Msk (0x8UL) /*!< PDM INTSTAT: DCMP (Bitfield-Mask: 0x01) */ +#define PDM_INTSTAT_UNDFL_Pos (2UL) /*!< PDM INTSTAT: UNDFL (Bit 2) */ +#define PDM_INTSTAT_UNDFL_Msk (0x4UL) /*!< PDM INTSTAT: UNDFL (Bitfield-Mask: 0x01) */ +#define PDM_INTSTAT_OVF_Pos (1UL) /*!< PDM INTSTAT: OVF (Bit 1) */ +#define PDM_INTSTAT_OVF_Msk (0x2UL) /*!< PDM INTSTAT: OVF (Bitfield-Mask: 0x01) */ +#define PDM_INTSTAT_THR_Pos (0UL) /*!< PDM INTSTAT: THR (Bit 0) */ +#define PDM_INTSTAT_THR_Msk (0x1UL) /*!< PDM INTSTAT: THR (Bitfield-Mask: 0x01) */ +/* ======================================================== INTCLR ========================================================= */ +#define PDM_INTCLR_DERR_Pos (4UL) /*!< PDM INTCLR: DERR (Bit 4) */ +#define PDM_INTCLR_DERR_Msk (0x10UL) /*!< PDM INTCLR: DERR (Bitfield-Mask: 0x01) */ +#define PDM_INTCLR_DCMP_Pos (3UL) /*!< PDM INTCLR: DCMP (Bit 3) */ +#define PDM_INTCLR_DCMP_Msk (0x8UL) /*!< PDM INTCLR: DCMP (Bitfield-Mask: 0x01) */ +#define PDM_INTCLR_UNDFL_Pos (2UL) /*!< PDM INTCLR: UNDFL (Bit 2) */ +#define PDM_INTCLR_UNDFL_Msk (0x4UL) /*!< PDM INTCLR: UNDFL (Bitfield-Mask: 0x01) */ +#define PDM_INTCLR_OVF_Pos (1UL) /*!< PDM INTCLR: OVF (Bit 1) */ +#define PDM_INTCLR_OVF_Msk (0x2UL) /*!< PDM INTCLR: OVF (Bitfield-Mask: 0x01) */ +#define PDM_INTCLR_THR_Pos (0UL) /*!< PDM INTCLR: THR (Bit 0) */ +#define PDM_INTCLR_THR_Msk (0x1UL) /*!< PDM INTCLR: THR (Bitfield-Mask: 0x01) */ +/* ======================================================== INTSET ========================================================= */ +#define PDM_INTSET_DERR_Pos (4UL) /*!< PDM INTSET: DERR (Bit 4) */ +#define PDM_INTSET_DERR_Msk (0x10UL) /*!< PDM INTSET: DERR (Bitfield-Mask: 0x01) */ +#define PDM_INTSET_DCMP_Pos (3UL) /*!< PDM INTSET: DCMP (Bit 3) */ +#define PDM_INTSET_DCMP_Msk (0x8UL) /*!< PDM INTSET: DCMP (Bitfield-Mask: 0x01) */ +#define PDM_INTSET_UNDFL_Pos (2UL) /*!< PDM INTSET: UNDFL (Bit 2) */ +#define PDM_INTSET_UNDFL_Msk (0x4UL) /*!< PDM INTSET: UNDFL (Bitfield-Mask: 0x01) */ +#define PDM_INTSET_OVF_Pos (1UL) /*!< PDM INTSET: OVF (Bit 1) */ +#define PDM_INTSET_OVF_Msk (0x2UL) /*!< PDM INTSET: OVF (Bitfield-Mask: 0x01) */ +#define PDM_INTSET_THR_Pos (0UL) /*!< PDM INTSET: THR (Bit 0) */ +#define PDM_INTSET_THR_Msk (0x1UL) /*!< PDM INTSET: THR (Bitfield-Mask: 0x01) */ +/* ======================================================= DMATRIGEN ======================================================= */ +#define PDM_DMATRIGEN_DTHR90_Pos (1UL) /*!< PDM DMATRIGEN: DTHR90 (Bit 1) */ +#define PDM_DMATRIGEN_DTHR90_Msk (0x2UL) /*!< PDM DMATRIGEN: DTHR90 (Bitfield-Mask: 0x01) */ +#define PDM_DMATRIGEN_DTHR_Pos (0UL) /*!< PDM DMATRIGEN: DTHR (Bit 0) */ +#define PDM_DMATRIGEN_DTHR_Msk (0x1UL) /*!< PDM DMATRIGEN: DTHR (Bitfield-Mask: 0x01) */ +/* ====================================================== DMATRIGSTAT ====================================================== */ +#define PDM_DMATRIGSTAT_DTHR90STAT_Pos (1UL) /*!< PDM DMATRIGSTAT: DTHR90STAT (Bit 1) */ +#define PDM_DMATRIGSTAT_DTHR90STAT_Msk (0x2UL) /*!< PDM DMATRIGSTAT: DTHR90STAT (Bitfield-Mask: 0x01) */ +#define PDM_DMATRIGSTAT_DTHRSTAT_Pos (0UL) /*!< PDM DMATRIGSTAT: DTHRSTAT (Bit 0) */ +#define PDM_DMATRIGSTAT_DTHRSTAT_Msk (0x1UL) /*!< PDM DMATRIGSTAT: DTHRSTAT (Bitfield-Mask: 0x01) */ +/* ======================================================== DMACFG ========================================================= */ +#define PDM_DMACFG_DPWROFF_Pos (10UL) /*!< PDM DMACFG: DPWROFF (Bit 10) */ +#define PDM_DMACFG_DPWROFF_Msk (0x400UL) /*!< PDM DMACFG: DPWROFF (Bitfield-Mask: 0x01) */ +#define PDM_DMACFG_DAUTOHIP_Pos (9UL) /*!< PDM DMACFG: DAUTOHIP (Bit 9) */ +#define PDM_DMACFG_DAUTOHIP_Msk (0x200UL) /*!< PDM DMACFG: DAUTOHIP (Bitfield-Mask: 0x01) */ +#define PDM_DMACFG_DMAPRI_Pos (8UL) /*!< PDM DMACFG: DMAPRI (Bit 8) */ +#define PDM_DMACFG_DMAPRI_Msk (0x100UL) /*!< PDM DMACFG: DMAPRI (Bitfield-Mask: 0x01) */ +#define PDM_DMACFG_DMADIR_Pos (2UL) /*!< PDM DMACFG: DMADIR (Bit 2) */ +#define PDM_DMACFG_DMADIR_Msk (0x4UL) /*!< PDM DMACFG: DMADIR (Bitfield-Mask: 0x01) */ +#define PDM_DMACFG_DMAEN_Pos (0UL) /*!< PDM DMACFG: DMAEN (Bit 0) */ +#define PDM_DMACFG_DMAEN_Msk (0x1UL) /*!< PDM DMACFG: DMAEN (Bitfield-Mask: 0x01) */ +/* ====================================================== DMATOTCOUNT ====================================================== */ +#define PDM_DMATOTCOUNT_TOTCOUNT_Pos (0UL) /*!< PDM DMATOTCOUNT: TOTCOUNT (Bit 0) */ +#define PDM_DMATOTCOUNT_TOTCOUNT_Msk (0xfffffUL) /*!< PDM DMATOTCOUNT: TOTCOUNT (Bitfield-Mask: 0xfffff) */ +/* ====================================================== DMATARGADDR ====================================================== */ +#define PDM_DMATARGADDR_UTARGADDR_Pos (20UL) /*!< PDM DMATARGADDR: UTARGADDR (Bit 20) */ +#define PDM_DMATARGADDR_UTARGADDR_Msk (0xfff00000UL) /*!< PDM DMATARGADDR: UTARGADDR (Bitfield-Mask: 0xfff) */ +#define PDM_DMATARGADDR_LTARGADDR_Pos (0UL) /*!< PDM DMATARGADDR: LTARGADDR (Bit 0) */ +#define PDM_DMATARGADDR_LTARGADDR_Msk (0xfffffUL) /*!< PDM DMATARGADDR: LTARGADDR (Bitfield-Mask: 0xfffff) */ +/* ======================================================== DMASTAT ======================================================== */ +#define PDM_DMASTAT_DMAERR_Pos (2UL) /*!< PDM DMASTAT: DMAERR (Bit 2) */ +#define PDM_DMASTAT_DMAERR_Msk (0x4UL) /*!< PDM DMASTAT: DMAERR (Bitfield-Mask: 0x01) */ +#define PDM_DMASTAT_DMACPL_Pos (1UL) /*!< PDM DMASTAT: DMACPL (Bit 1) */ +#define PDM_DMASTAT_DMACPL_Msk (0x2UL) /*!< PDM DMASTAT: DMACPL (Bitfield-Mask: 0x01) */ +#define PDM_DMASTAT_DMATIP_Pos (0UL) /*!< PDM DMASTAT: DMATIP (Bit 0) */ +#define PDM_DMASTAT_DMATIP_Msk (0x1UL) /*!< PDM DMASTAT: DMATIP (Bitfield-Mask: 0x01) */ + + +/* =========================================================================================================================== */ +/* ================ PWRCTRL ================ */ +/* =========================================================================================================================== */ + +/* ======================================================= SUPPLYSRC ======================================================= */ +#define PWRCTRL_SUPPLYSRC_BLEBUCKEN_Pos (0UL) /*!< PWRCTRL SUPPLYSRC: BLEBUCKEN (Bit 0) */ +#define PWRCTRL_SUPPLYSRC_BLEBUCKEN_Msk (0x1UL) /*!< PWRCTRL SUPPLYSRC: BLEBUCKEN (Bitfield-Mask: 0x01) */ +/* ===================================================== SUPPLYSTATUS ====================================================== */ +#define PWRCTRL_SUPPLYSTATUS_BLEBUCKON_Pos (1UL) /*!< PWRCTRL SUPPLYSTATUS: BLEBUCKON (Bit 1) */ +#define PWRCTRL_SUPPLYSTATUS_BLEBUCKON_Msk (0x2UL) /*!< PWRCTRL SUPPLYSTATUS: BLEBUCKON (Bitfield-Mask: 0x01) */ +#define PWRCTRL_SUPPLYSTATUS_SIMOBUCKON_Pos (0UL) /*!< PWRCTRL SUPPLYSTATUS: SIMOBUCKON (Bit 0) */ +#define PWRCTRL_SUPPLYSTATUS_SIMOBUCKON_Msk (0x1UL) /*!< PWRCTRL SUPPLYSTATUS: SIMOBUCKON (Bitfield-Mask: 0x01) */ +/* ======================================================= DEVPWREN ======================================================== */ +#define PWRCTRL_DEVPWREN_PWRBLEL_Pos (13UL) /*!< PWRCTRL DEVPWREN: PWRBLEL (Bit 13) */ +#define PWRCTRL_DEVPWREN_PWRBLEL_Msk (0x2000UL) /*!< PWRCTRL DEVPWREN: PWRBLEL (Bitfield-Mask: 0x01) */ +#define PWRCTRL_DEVPWREN_PWRPDM_Pos (12UL) /*!< PWRCTRL DEVPWREN: PWRPDM (Bit 12) */ +#define PWRCTRL_DEVPWREN_PWRPDM_Msk (0x1000UL) /*!< PWRCTRL DEVPWREN: PWRPDM (Bitfield-Mask: 0x01) */ +#define PWRCTRL_DEVPWREN_PWRMSPI_Pos (11UL) /*!< PWRCTRL DEVPWREN: PWRMSPI (Bit 11) */ +#define PWRCTRL_DEVPWREN_PWRMSPI_Msk (0x800UL) /*!< PWRCTRL DEVPWREN: PWRMSPI (Bitfield-Mask: 0x01) */ +#define PWRCTRL_DEVPWREN_PWRSCARD_Pos (10UL) /*!< PWRCTRL DEVPWREN: PWRSCARD (Bit 10) */ +#define PWRCTRL_DEVPWREN_PWRSCARD_Msk (0x400UL) /*!< PWRCTRL DEVPWREN: PWRSCARD (Bitfield-Mask: 0x01) */ +#define PWRCTRL_DEVPWREN_PWRADC_Pos (9UL) /*!< PWRCTRL DEVPWREN: PWRADC (Bit 9) */ +#define PWRCTRL_DEVPWREN_PWRADC_Msk (0x200UL) /*!< PWRCTRL DEVPWREN: PWRADC (Bitfield-Mask: 0x01) */ +#define PWRCTRL_DEVPWREN_PWRUART1_Pos (8UL) /*!< PWRCTRL DEVPWREN: PWRUART1 (Bit 8) */ +#define PWRCTRL_DEVPWREN_PWRUART1_Msk (0x100UL) /*!< PWRCTRL DEVPWREN: PWRUART1 (Bitfield-Mask: 0x01) */ +#define PWRCTRL_DEVPWREN_PWRUART0_Pos (7UL) /*!< PWRCTRL DEVPWREN: PWRUART0 (Bit 7) */ +#define PWRCTRL_DEVPWREN_PWRUART0_Msk (0x80UL) /*!< PWRCTRL DEVPWREN: PWRUART0 (Bitfield-Mask: 0x01) */ +#define PWRCTRL_DEVPWREN_PWRIOM5_Pos (6UL) /*!< PWRCTRL DEVPWREN: PWRIOM5 (Bit 6) */ +#define PWRCTRL_DEVPWREN_PWRIOM5_Msk (0x40UL) /*!< PWRCTRL DEVPWREN: PWRIOM5 (Bitfield-Mask: 0x01) */ +#define PWRCTRL_DEVPWREN_PWRIOM4_Pos (5UL) /*!< PWRCTRL DEVPWREN: PWRIOM4 (Bit 5) */ +#define PWRCTRL_DEVPWREN_PWRIOM4_Msk (0x20UL) /*!< PWRCTRL DEVPWREN: PWRIOM4 (Bitfield-Mask: 0x01) */ +#define PWRCTRL_DEVPWREN_PWRIOM3_Pos (4UL) /*!< PWRCTRL DEVPWREN: PWRIOM3 (Bit 4) */ +#define PWRCTRL_DEVPWREN_PWRIOM3_Msk (0x10UL) /*!< PWRCTRL DEVPWREN: PWRIOM3 (Bitfield-Mask: 0x01) */ +#define PWRCTRL_DEVPWREN_PWRIOM2_Pos (3UL) /*!< PWRCTRL DEVPWREN: PWRIOM2 (Bit 3) */ +#define PWRCTRL_DEVPWREN_PWRIOM2_Msk (0x8UL) /*!< PWRCTRL DEVPWREN: PWRIOM2 (Bitfield-Mask: 0x01) */ +#define PWRCTRL_DEVPWREN_PWRIOM1_Pos (2UL) /*!< PWRCTRL DEVPWREN: PWRIOM1 (Bit 2) */ +#define PWRCTRL_DEVPWREN_PWRIOM1_Msk (0x4UL) /*!< PWRCTRL DEVPWREN: PWRIOM1 (Bitfield-Mask: 0x01) */ +#define PWRCTRL_DEVPWREN_PWRIOM0_Pos (1UL) /*!< PWRCTRL DEVPWREN: PWRIOM0 (Bit 1) */ +#define PWRCTRL_DEVPWREN_PWRIOM0_Msk (0x2UL) /*!< PWRCTRL DEVPWREN: PWRIOM0 (Bitfield-Mask: 0x01) */ +#define PWRCTRL_DEVPWREN_PWRIOS_Pos (0UL) /*!< PWRCTRL DEVPWREN: PWRIOS (Bit 0) */ +#define PWRCTRL_DEVPWREN_PWRIOS_Msk (0x1UL) /*!< PWRCTRL DEVPWREN: PWRIOS (Bitfield-Mask: 0x01) */ +/* ===================================================== MEMPWDINSLEEP ===================================================== */ +#define PWRCTRL_MEMPWDINSLEEP_CACHEPWDSLP_Pos (31UL) /*!< PWRCTRL MEMPWDINSLEEP: CACHEPWDSLP (Bit 31) */ +#define PWRCTRL_MEMPWDINSLEEP_CACHEPWDSLP_Msk (0x80000000UL) /*!< PWRCTRL MEMPWDINSLEEP: CACHEPWDSLP (Bitfield-Mask: 0x01) */ +#define PWRCTRL_MEMPWDINSLEEP_FLASH1PWDSLP_Pos (14UL) /*!< PWRCTRL MEMPWDINSLEEP: FLASH1PWDSLP (Bit 14) */ +#define PWRCTRL_MEMPWDINSLEEP_FLASH1PWDSLP_Msk (0x4000UL) /*!< PWRCTRL MEMPWDINSLEEP: FLASH1PWDSLP (Bitfield-Mask: 0x01) */ +#define PWRCTRL_MEMPWDINSLEEP_FLASH0PWDSLP_Pos (13UL) /*!< PWRCTRL MEMPWDINSLEEP: FLASH0PWDSLP (Bit 13) */ +#define PWRCTRL_MEMPWDINSLEEP_FLASH0PWDSLP_Msk (0x2000UL) /*!< PWRCTRL MEMPWDINSLEEP: FLASH0PWDSLP (Bitfield-Mask: 0x01) */ +#define PWRCTRL_MEMPWDINSLEEP_SRAMPWDSLP_Pos (3UL) /*!< PWRCTRL MEMPWDINSLEEP: SRAMPWDSLP (Bit 3) */ +#define PWRCTRL_MEMPWDINSLEEP_SRAMPWDSLP_Msk (0x1ff8UL) /*!< PWRCTRL MEMPWDINSLEEP: SRAMPWDSLP (Bitfield-Mask: 0x3ff) */ +#define PWRCTRL_MEMPWDINSLEEP_DTCMPWDSLP_Pos (0UL) /*!< PWRCTRL MEMPWDINSLEEP: DTCMPWDSLP (Bit 0) */ +#define PWRCTRL_MEMPWDINSLEEP_DTCMPWDSLP_Msk (0x7UL) /*!< PWRCTRL MEMPWDINSLEEP: DTCMPWDSLP (Bitfield-Mask: 0x07) */ +/* ======================================================= MEMPWREN ======================================================== */ +#define PWRCTRL_MEMPWREN_CACHEB2_Pos (31UL) /*!< PWRCTRL MEMPWREN: CACHEB2 (Bit 31) */ +#define PWRCTRL_MEMPWREN_CACHEB2_Msk (0x80000000UL) /*!< PWRCTRL MEMPWREN: CACHEB2 (Bitfield-Mask: 0x01) */ +#define PWRCTRL_MEMPWREN_CACHEB0_Pos (30UL) /*!< PWRCTRL MEMPWREN: CACHEB0 (Bit 30) */ +#define PWRCTRL_MEMPWREN_CACHEB0_Msk (0x40000000UL) /*!< PWRCTRL MEMPWREN: CACHEB0 (Bitfield-Mask: 0x01) */ +#define PWRCTRL_MEMPWREN_FLASH1_Pos (14UL) /*!< PWRCTRL MEMPWREN: FLASH1 (Bit 14) */ +#define PWRCTRL_MEMPWREN_FLASH1_Msk (0x4000UL) /*!< PWRCTRL MEMPWREN: FLASH1 (Bitfield-Mask: 0x01) */ +#define PWRCTRL_MEMPWREN_FLASH0_Pos (13UL) /*!< PWRCTRL MEMPWREN: FLASH0 (Bit 13) */ +#define PWRCTRL_MEMPWREN_FLASH0_Msk (0x2000UL) /*!< PWRCTRL MEMPWREN: FLASH0 (Bitfield-Mask: 0x01) */ +#define PWRCTRL_MEMPWREN_SRAM_Pos (3UL) /*!< PWRCTRL MEMPWREN: SRAM (Bit 3) */ +#define PWRCTRL_MEMPWREN_SRAM_Msk (0x1ff8UL) /*!< PWRCTRL MEMPWREN: SRAM (Bitfield-Mask: 0x3ff) */ +#define PWRCTRL_MEMPWREN_DTCM_Pos (0UL) /*!< PWRCTRL MEMPWREN: DTCM (Bit 0) */ +#define PWRCTRL_MEMPWREN_DTCM_Msk (0x7UL) /*!< PWRCTRL MEMPWREN: DTCM (Bitfield-Mask: 0x07) */ +/* ===================================================== MEMPWRSTATUS ====================================================== */ +#define PWRCTRL_MEMPWRSTATUS_CACHEB2_Pos (16UL) /*!< PWRCTRL MEMPWRSTATUS: CACHEB2 (Bit 16) */ +#define PWRCTRL_MEMPWRSTATUS_CACHEB2_Msk (0x10000UL) /*!< PWRCTRL MEMPWRSTATUS: CACHEB2 (Bitfield-Mask: 0x01) */ +#define PWRCTRL_MEMPWRSTATUS_CACHEB0_Pos (15UL) /*!< PWRCTRL MEMPWRSTATUS: CACHEB0 (Bit 15) */ +#define PWRCTRL_MEMPWRSTATUS_CACHEB0_Msk (0x8000UL) /*!< PWRCTRL MEMPWRSTATUS: CACHEB0 (Bitfield-Mask: 0x01) */ +#define PWRCTRL_MEMPWRSTATUS_FLASH1_Pos (14UL) /*!< PWRCTRL MEMPWRSTATUS: FLASH1 (Bit 14) */ +#define PWRCTRL_MEMPWRSTATUS_FLASH1_Msk (0x4000UL) /*!< PWRCTRL MEMPWRSTATUS: FLASH1 (Bitfield-Mask: 0x01) */ +#define PWRCTRL_MEMPWRSTATUS_FLASH0_Pos (13UL) /*!< PWRCTRL MEMPWRSTATUS: FLASH0 (Bit 13) */ +#define PWRCTRL_MEMPWRSTATUS_FLASH0_Msk (0x2000UL) /*!< PWRCTRL MEMPWRSTATUS: FLASH0 (Bitfield-Mask: 0x01) */ +#define PWRCTRL_MEMPWRSTATUS_SRAM9_Pos (12UL) /*!< PWRCTRL MEMPWRSTATUS: SRAM9 (Bit 12) */ +#define PWRCTRL_MEMPWRSTATUS_SRAM9_Msk (0x1000UL) /*!< PWRCTRL MEMPWRSTATUS: SRAM9 (Bitfield-Mask: 0x01) */ +#define PWRCTRL_MEMPWRSTATUS_SRAM8_Pos (11UL) /*!< PWRCTRL MEMPWRSTATUS: SRAM8 (Bit 11) */ +#define PWRCTRL_MEMPWRSTATUS_SRAM8_Msk (0x800UL) /*!< PWRCTRL MEMPWRSTATUS: SRAM8 (Bitfield-Mask: 0x01) */ +#define PWRCTRL_MEMPWRSTATUS_SRAM7_Pos (10UL) /*!< PWRCTRL MEMPWRSTATUS: SRAM7 (Bit 10) */ +#define PWRCTRL_MEMPWRSTATUS_SRAM7_Msk (0x400UL) /*!< PWRCTRL MEMPWRSTATUS: SRAM7 (Bitfield-Mask: 0x01) */ +#define PWRCTRL_MEMPWRSTATUS_SRAM6_Pos (9UL) /*!< PWRCTRL MEMPWRSTATUS: SRAM6 (Bit 9) */ +#define PWRCTRL_MEMPWRSTATUS_SRAM6_Msk (0x200UL) /*!< PWRCTRL MEMPWRSTATUS: SRAM6 (Bitfield-Mask: 0x01) */ +#define PWRCTRL_MEMPWRSTATUS_SRAM5_Pos (8UL) /*!< PWRCTRL MEMPWRSTATUS: SRAM5 (Bit 8) */ +#define PWRCTRL_MEMPWRSTATUS_SRAM5_Msk (0x100UL) /*!< PWRCTRL MEMPWRSTATUS: SRAM5 (Bitfield-Mask: 0x01) */ +#define PWRCTRL_MEMPWRSTATUS_SRAM4_Pos (7UL) /*!< PWRCTRL MEMPWRSTATUS: SRAM4 (Bit 7) */ +#define PWRCTRL_MEMPWRSTATUS_SRAM4_Msk (0x80UL) /*!< PWRCTRL MEMPWRSTATUS: SRAM4 (Bitfield-Mask: 0x01) */ +#define PWRCTRL_MEMPWRSTATUS_SRAM3_Pos (6UL) /*!< PWRCTRL MEMPWRSTATUS: SRAM3 (Bit 6) */ +#define PWRCTRL_MEMPWRSTATUS_SRAM3_Msk (0x40UL) /*!< PWRCTRL MEMPWRSTATUS: SRAM3 (Bitfield-Mask: 0x01) */ +#define PWRCTRL_MEMPWRSTATUS_SRAM2_Pos (5UL) /*!< PWRCTRL MEMPWRSTATUS: SRAM2 (Bit 5) */ +#define PWRCTRL_MEMPWRSTATUS_SRAM2_Msk (0x20UL) /*!< PWRCTRL MEMPWRSTATUS: SRAM2 (Bitfield-Mask: 0x01) */ +#define PWRCTRL_MEMPWRSTATUS_SRAM1_Pos (4UL) /*!< PWRCTRL MEMPWRSTATUS: SRAM1 (Bit 4) */ +#define PWRCTRL_MEMPWRSTATUS_SRAM1_Msk (0x10UL) /*!< PWRCTRL MEMPWRSTATUS: SRAM1 (Bitfield-Mask: 0x01) */ +#define PWRCTRL_MEMPWRSTATUS_SRAM0_Pos (3UL) /*!< PWRCTRL MEMPWRSTATUS: SRAM0 (Bit 3) */ +#define PWRCTRL_MEMPWRSTATUS_SRAM0_Msk (0x8UL) /*!< PWRCTRL MEMPWRSTATUS: SRAM0 (Bitfield-Mask: 0x01) */ +#define PWRCTRL_MEMPWRSTATUS_DTCM1_Pos (2UL) /*!< PWRCTRL MEMPWRSTATUS: DTCM1 (Bit 2) */ +#define PWRCTRL_MEMPWRSTATUS_DTCM1_Msk (0x4UL) /*!< PWRCTRL MEMPWRSTATUS: DTCM1 (Bitfield-Mask: 0x01) */ +#define PWRCTRL_MEMPWRSTATUS_DTCM01_Pos (1UL) /*!< PWRCTRL MEMPWRSTATUS: DTCM01 (Bit 1) */ +#define PWRCTRL_MEMPWRSTATUS_DTCM01_Msk (0x2UL) /*!< PWRCTRL MEMPWRSTATUS: DTCM01 (Bitfield-Mask: 0x01) */ +#define PWRCTRL_MEMPWRSTATUS_DTCM00_Pos (0UL) /*!< PWRCTRL MEMPWRSTATUS: DTCM00 (Bit 0) */ +#define PWRCTRL_MEMPWRSTATUS_DTCM00_Msk (0x1UL) /*!< PWRCTRL MEMPWRSTATUS: DTCM00 (Bitfield-Mask: 0x01) */ +/* ===================================================== DEVPWRSTATUS ====================================================== */ +#define PWRCTRL_DEVPWRSTATUS_SYSDEEPSLEEP_Pos (31UL) /*!< PWRCTRL DEVPWRSTATUS: SYSDEEPSLEEP (Bit 31) */ +#define PWRCTRL_DEVPWRSTATUS_SYSDEEPSLEEP_Msk (0x80000000UL) /*!< PWRCTRL DEVPWRSTATUS: SYSDEEPSLEEP (Bitfield-Mask: 0x01) */ +#define PWRCTRL_DEVPWRSTATUS_COREDEEPSLEEP_Pos (30UL) /*!< PWRCTRL DEVPWRSTATUS: COREDEEPSLEEP (Bit 30) */ +#define PWRCTRL_DEVPWRSTATUS_COREDEEPSLEEP_Msk (0x40000000UL) /*!< PWRCTRL DEVPWRSTATUS: COREDEEPSLEEP (Bitfield-Mask: 0x01) */ +#define PWRCTRL_DEVPWRSTATUS_CORESLEEP_Pos (29UL) /*!< PWRCTRL DEVPWRSTATUS: CORESLEEP (Bit 29) */ +#define PWRCTRL_DEVPWRSTATUS_CORESLEEP_Msk (0x20000000UL) /*!< PWRCTRL DEVPWRSTATUS: CORESLEEP (Bitfield-Mask: 0x01) */ +#define PWRCTRL_DEVPWRSTATUS_BLEH_Pos (9UL) /*!< PWRCTRL DEVPWRSTATUS: BLEH (Bit 9) */ +#define PWRCTRL_DEVPWRSTATUS_BLEH_Msk (0x200UL) /*!< PWRCTRL DEVPWRSTATUS: BLEH (Bitfield-Mask: 0x01) */ +#define PWRCTRL_DEVPWRSTATUS_BLEL_Pos (8UL) /*!< PWRCTRL DEVPWRSTATUS: BLEL (Bit 8) */ +#define PWRCTRL_DEVPWRSTATUS_BLEL_Msk (0x100UL) /*!< PWRCTRL DEVPWRSTATUS: BLEL (Bitfield-Mask: 0x01) */ +#define PWRCTRL_DEVPWRSTATUS_PWRPDM_Pos (7UL) /*!< PWRCTRL DEVPWRSTATUS: PWRPDM (Bit 7) */ +#define PWRCTRL_DEVPWRSTATUS_PWRPDM_Msk (0x80UL) /*!< PWRCTRL DEVPWRSTATUS: PWRPDM (Bitfield-Mask: 0x01) */ +#define PWRCTRL_DEVPWRSTATUS_PWRMSPI_Pos (6UL) /*!< PWRCTRL DEVPWRSTATUS: PWRMSPI (Bit 6) */ +#define PWRCTRL_DEVPWRSTATUS_PWRMSPI_Msk (0x40UL) /*!< PWRCTRL DEVPWRSTATUS: PWRMSPI (Bitfield-Mask: 0x01) */ +#define PWRCTRL_DEVPWRSTATUS_PWRADC_Pos (5UL) /*!< PWRCTRL DEVPWRSTATUS: PWRADC (Bit 5) */ +#define PWRCTRL_DEVPWRSTATUS_PWRADC_Msk (0x20UL) /*!< PWRCTRL DEVPWRSTATUS: PWRADC (Bitfield-Mask: 0x01) */ +#define PWRCTRL_DEVPWRSTATUS_HCPC_Pos (4UL) /*!< PWRCTRL DEVPWRSTATUS: HCPC (Bit 4) */ +#define PWRCTRL_DEVPWRSTATUS_HCPC_Msk (0x10UL) /*!< PWRCTRL DEVPWRSTATUS: HCPC (Bitfield-Mask: 0x01) */ +#define PWRCTRL_DEVPWRSTATUS_HCPB_Pos (3UL) /*!< PWRCTRL DEVPWRSTATUS: HCPB (Bit 3) */ +#define PWRCTRL_DEVPWRSTATUS_HCPB_Msk (0x8UL) /*!< PWRCTRL DEVPWRSTATUS: HCPB (Bitfield-Mask: 0x01) */ +#define PWRCTRL_DEVPWRSTATUS_HCPA_Pos (2UL) /*!< PWRCTRL DEVPWRSTATUS: HCPA (Bit 2) */ +#define PWRCTRL_DEVPWRSTATUS_HCPA_Msk (0x4UL) /*!< PWRCTRL DEVPWRSTATUS: HCPA (Bitfield-Mask: 0x01) */ +#define PWRCTRL_DEVPWRSTATUS_MCUH_Pos (1UL) /*!< PWRCTRL DEVPWRSTATUS: MCUH (Bit 1) */ +#define PWRCTRL_DEVPWRSTATUS_MCUH_Msk (0x2UL) /*!< PWRCTRL DEVPWRSTATUS: MCUH (Bitfield-Mask: 0x01) */ +#define PWRCTRL_DEVPWRSTATUS_MCUL_Pos (0UL) /*!< PWRCTRL DEVPWRSTATUS: MCUL (Bit 0) */ +#define PWRCTRL_DEVPWRSTATUS_MCUL_Msk (0x1UL) /*!< PWRCTRL DEVPWRSTATUS: MCUL (Bitfield-Mask: 0x01) */ +/* ======================================================= SRAMCTRL ======================================================== */ +#define PWRCTRL_SRAMCTRL_SRAMLIGHTSLEEP_Pos (8UL) /*!< PWRCTRL SRAMCTRL: SRAMLIGHTSLEEP (Bit 8) */ +#define PWRCTRL_SRAMCTRL_SRAMLIGHTSLEEP_Msk (0xfff00UL) /*!< PWRCTRL SRAMCTRL: SRAMLIGHTSLEEP (Bitfield-Mask: 0xfff) */ +#define PWRCTRL_SRAMCTRL_SRAMMASTERCLKGATE_Pos (2UL) /*!< PWRCTRL SRAMCTRL: SRAMMASTERCLKGATE (Bit 2) */ +#define PWRCTRL_SRAMCTRL_SRAMMASTERCLKGATE_Msk (0x4UL) /*!< PWRCTRL SRAMCTRL: SRAMMASTERCLKGATE (Bitfield-Mask: 0x01) */ +#define PWRCTRL_SRAMCTRL_SRAMCLKGATE_Pos (1UL) /*!< PWRCTRL SRAMCTRL: SRAMCLKGATE (Bit 1) */ +#define PWRCTRL_SRAMCTRL_SRAMCLKGATE_Msk (0x2UL) /*!< PWRCTRL SRAMCTRL: SRAMCLKGATE (Bitfield-Mask: 0x01) */ +/* ======================================================= ADCSTATUS ======================================================= */ +#define PWRCTRL_ADCSTATUS_REFBUFPWD_Pos (5UL) /*!< PWRCTRL ADCSTATUS: REFBUFPWD (Bit 5) */ +#define PWRCTRL_ADCSTATUS_REFBUFPWD_Msk (0x20UL) /*!< PWRCTRL ADCSTATUS: REFBUFPWD (Bitfield-Mask: 0x01) */ +#define PWRCTRL_ADCSTATUS_REFKEEPPWD_Pos (4UL) /*!< PWRCTRL ADCSTATUS: REFKEEPPWD (Bit 4) */ +#define PWRCTRL_ADCSTATUS_REFKEEPPWD_Msk (0x10UL) /*!< PWRCTRL ADCSTATUS: REFKEEPPWD (Bitfield-Mask: 0x01) */ +#define PWRCTRL_ADCSTATUS_VBATPWD_Pos (3UL) /*!< PWRCTRL ADCSTATUS: VBATPWD (Bit 3) */ +#define PWRCTRL_ADCSTATUS_VBATPWD_Msk (0x8UL) /*!< PWRCTRL ADCSTATUS: VBATPWD (Bitfield-Mask: 0x01) */ +#define PWRCTRL_ADCSTATUS_VPTATPWD_Pos (2UL) /*!< PWRCTRL ADCSTATUS: VPTATPWD (Bit 2) */ +#define PWRCTRL_ADCSTATUS_VPTATPWD_Msk (0x4UL) /*!< PWRCTRL ADCSTATUS: VPTATPWD (Bitfield-Mask: 0x01) */ +#define PWRCTRL_ADCSTATUS_BGTPWD_Pos (1UL) /*!< PWRCTRL ADCSTATUS: BGTPWD (Bit 1) */ +#define PWRCTRL_ADCSTATUS_BGTPWD_Msk (0x2UL) /*!< PWRCTRL ADCSTATUS: BGTPWD (Bitfield-Mask: 0x01) */ +#define PWRCTRL_ADCSTATUS_ADCPWD_Pos (0UL) /*!< PWRCTRL ADCSTATUS: ADCPWD (Bit 0) */ +#define PWRCTRL_ADCSTATUS_ADCPWD_Msk (0x1UL) /*!< PWRCTRL ADCSTATUS: ADCPWD (Bitfield-Mask: 0x01) */ +/* ========================================================= MISC ========================================================== */ +#define PWRCTRL_MISC_FORCEBLEBUCKACT_Pos (7UL) /*!< PWRCTRL MISC: FORCEBLEBUCKACT (Bit 7) */ +#define PWRCTRL_MISC_FORCEBLEBUCKACT_Msk (0x80UL) /*!< PWRCTRL MISC: FORCEBLEBUCKACT (Bitfield-Mask: 0x01) */ +#define PWRCTRL_MISC_MEMVRLPBLE_Pos (6UL) /*!< PWRCTRL MISC: MEMVRLPBLE (Bit 6) */ +#define PWRCTRL_MISC_MEMVRLPBLE_Msk (0x40UL) /*!< PWRCTRL MISC: MEMVRLPBLE (Bitfield-Mask: 0x01) */ +#define PWRCTRL_MISC_FORCEMEMVRADC_Pos (4UL) /*!< PWRCTRL MISC: FORCEMEMVRADC (Bit 4) */ +#define PWRCTRL_MISC_FORCEMEMVRADC_Msk (0x30UL) /*!< PWRCTRL MISC: FORCEMEMVRADC (Bitfield-Mask: 0x03) */ +#define PWRCTRL_MISC_FORCEMEMVRLPTIMERS_Pos (3UL) /*!< PWRCTRL MISC: FORCEMEMVRLPTIMERS (Bit 3) */ +#define PWRCTRL_MISC_FORCEMEMVRLPTIMERS_Msk (0x8UL) /*!< PWRCTRL MISC: FORCEMEMVRLPTIMERS (Bitfield-Mask: 0x01) */ +#define PWRCTRL_MISC_FORCECOREVRLPTIMERS_Pos (2UL) /*!< PWRCTRL MISC: FORCECOREVRLPTIMERS (Bit 2) */ +#define PWRCTRL_MISC_FORCECOREVRLPTIMERS_Msk (0x4UL) /*!< PWRCTRL MISC: FORCECOREVRLPTIMERS (Bitfield-Mask: 0x01) */ +#define PWRCTRL_MISC_FORCECOREVRLPPDM_Pos (1UL) /*!< PWRCTRL MISC: FORCECOREVRLPPDM (Bit 1) */ +#define PWRCTRL_MISC_FORCECOREVRLPPDM_Msk (0x2UL) /*!< PWRCTRL MISC: FORCECOREVRLPPDM (Bitfield-Mask: 0x01) */ +#define PWRCTRL_MISC_SIMOBUCKEN_Pos (0UL) /*!< PWRCTRL MISC: SIMOBUCKEN (Bit 0) */ +#define PWRCTRL_MISC_SIMOBUCKEN_Msk (0x1UL) /*!< PWRCTRL MISC: SIMOBUCKEN (Bitfield-Mask: 0x01) */ +/* ===================================================== DEVPWREVENTEN ===================================================== */ +#define PWRCTRL_DEVPWREVENTEN_BURSTEVEN_Pos (31UL) /*!< PWRCTRL DEVPWREVENTEN: BURSTEVEN (Bit 31) */ +#define PWRCTRL_DEVPWREVENTEN_BURSTEVEN_Msk (0x80000000UL) /*!< PWRCTRL DEVPWREVENTEN: BURSTEVEN (Bitfield-Mask: 0x01) */ +#define PWRCTRL_DEVPWREVENTEN_BURSTFEATUREEVEN_Pos (30UL) /*!< PWRCTRL DEVPWREVENTEN: BURSTFEATUREEVEN (Bit 30) */ +#define PWRCTRL_DEVPWREVENTEN_BURSTFEATUREEVEN_Msk (0x40000000UL) /*!< PWRCTRL DEVPWREVENTEN: BURSTFEATUREEVEN (Bitfield-Mask: 0x01) */ +#define PWRCTRL_DEVPWREVENTEN_BLEFEATUREEVEN_Pos (29UL) /*!< PWRCTRL DEVPWREVENTEN: BLEFEATUREEVEN (Bit 29) */ +#define PWRCTRL_DEVPWREVENTEN_BLEFEATUREEVEN_Msk (0x20000000UL) /*!< PWRCTRL DEVPWREVENTEN: BLEFEATUREEVEN (Bitfield-Mask: 0x01) */ +#define PWRCTRL_DEVPWREVENTEN_BLELEVEN_Pos (8UL) /*!< PWRCTRL DEVPWREVENTEN: BLELEVEN (Bit 8) */ +#define PWRCTRL_DEVPWREVENTEN_BLELEVEN_Msk (0x100UL) /*!< PWRCTRL DEVPWREVENTEN: BLELEVEN (Bitfield-Mask: 0x01) */ +#define PWRCTRL_DEVPWREVENTEN_PDMEVEN_Pos (7UL) /*!< PWRCTRL DEVPWREVENTEN: PDMEVEN (Bit 7) */ +#define PWRCTRL_DEVPWREVENTEN_PDMEVEN_Msk (0x80UL) /*!< PWRCTRL DEVPWREVENTEN: PDMEVEN (Bitfield-Mask: 0x01) */ +#define PWRCTRL_DEVPWREVENTEN_MSPIEVEN_Pos (6UL) /*!< PWRCTRL DEVPWREVENTEN: MSPIEVEN (Bit 6) */ +#define PWRCTRL_DEVPWREVENTEN_MSPIEVEN_Msk (0x40UL) /*!< PWRCTRL DEVPWREVENTEN: MSPIEVEN (Bitfield-Mask: 0x01) */ +#define PWRCTRL_DEVPWREVENTEN_ADCEVEN_Pos (5UL) /*!< PWRCTRL DEVPWREVENTEN: ADCEVEN (Bit 5) */ +#define PWRCTRL_DEVPWREVENTEN_ADCEVEN_Msk (0x20UL) /*!< PWRCTRL DEVPWREVENTEN: ADCEVEN (Bitfield-Mask: 0x01) */ +#define PWRCTRL_DEVPWREVENTEN_HCPCEVEN_Pos (4UL) /*!< PWRCTRL DEVPWREVENTEN: HCPCEVEN (Bit 4) */ +#define PWRCTRL_DEVPWREVENTEN_HCPCEVEN_Msk (0x10UL) /*!< PWRCTRL DEVPWREVENTEN: HCPCEVEN (Bitfield-Mask: 0x01) */ +#define PWRCTRL_DEVPWREVENTEN_HCPBEVEN_Pos (3UL) /*!< PWRCTRL DEVPWREVENTEN: HCPBEVEN (Bit 3) */ +#define PWRCTRL_DEVPWREVENTEN_HCPBEVEN_Msk (0x8UL) /*!< PWRCTRL DEVPWREVENTEN: HCPBEVEN (Bitfield-Mask: 0x01) */ +#define PWRCTRL_DEVPWREVENTEN_HCPAEVEN_Pos (2UL) /*!< PWRCTRL DEVPWREVENTEN: HCPAEVEN (Bit 2) */ +#define PWRCTRL_DEVPWREVENTEN_HCPAEVEN_Msk (0x4UL) /*!< PWRCTRL DEVPWREVENTEN: HCPAEVEN (Bitfield-Mask: 0x01) */ +#define PWRCTRL_DEVPWREVENTEN_MCUHEVEN_Pos (1UL) /*!< PWRCTRL DEVPWREVENTEN: MCUHEVEN (Bit 1) */ +#define PWRCTRL_DEVPWREVENTEN_MCUHEVEN_Msk (0x2UL) /*!< PWRCTRL DEVPWREVENTEN: MCUHEVEN (Bitfield-Mask: 0x01) */ +#define PWRCTRL_DEVPWREVENTEN_MCULEVEN_Pos (0UL) /*!< PWRCTRL DEVPWREVENTEN: MCULEVEN (Bit 0) */ +#define PWRCTRL_DEVPWREVENTEN_MCULEVEN_Msk (0x1UL) /*!< PWRCTRL DEVPWREVENTEN: MCULEVEN (Bitfield-Mask: 0x01) */ +/* ===================================================== MEMPWREVENTEN ===================================================== */ +#define PWRCTRL_MEMPWREVENTEN_CACHEB2EN_Pos (31UL) /*!< PWRCTRL MEMPWREVENTEN: CACHEB2EN (Bit 31) */ +#define PWRCTRL_MEMPWREVENTEN_CACHEB2EN_Msk (0x80000000UL) /*!< PWRCTRL MEMPWREVENTEN: CACHEB2EN (Bitfield-Mask: 0x01) */ +#define PWRCTRL_MEMPWREVENTEN_CACHEB0EN_Pos (30UL) /*!< PWRCTRL MEMPWREVENTEN: CACHEB0EN (Bit 30) */ +#define PWRCTRL_MEMPWREVENTEN_CACHEB0EN_Msk (0x40000000UL) /*!< PWRCTRL MEMPWREVENTEN: CACHEB0EN (Bitfield-Mask: 0x01) */ +#define PWRCTRL_MEMPWREVENTEN_FLASH1EN_Pos (14UL) /*!< PWRCTRL MEMPWREVENTEN: FLASH1EN (Bit 14) */ +#define PWRCTRL_MEMPWREVENTEN_FLASH1EN_Msk (0x4000UL) /*!< PWRCTRL MEMPWREVENTEN: FLASH1EN (Bitfield-Mask: 0x01) */ +#define PWRCTRL_MEMPWREVENTEN_FLASH0EN_Pos (13UL) /*!< PWRCTRL MEMPWREVENTEN: FLASH0EN (Bit 13) */ +#define PWRCTRL_MEMPWREVENTEN_FLASH0EN_Msk (0x2000UL) /*!< PWRCTRL MEMPWREVENTEN: FLASH0EN (Bitfield-Mask: 0x01) */ +#define PWRCTRL_MEMPWREVENTEN_SRAMEN_Pos (3UL) /*!< PWRCTRL MEMPWREVENTEN: SRAMEN (Bit 3) */ +#define PWRCTRL_MEMPWREVENTEN_SRAMEN_Msk (0x1ff8UL) /*!< PWRCTRL MEMPWREVENTEN: SRAMEN (Bitfield-Mask: 0x3ff) */ +#define PWRCTRL_MEMPWREVENTEN_DTCMEN_Pos (0UL) /*!< PWRCTRL MEMPWREVENTEN: DTCMEN (Bit 0) */ +#define PWRCTRL_MEMPWREVENTEN_DTCMEN_Msk (0x7UL) /*!< PWRCTRL MEMPWREVENTEN: DTCMEN (Bitfield-Mask: 0x07) */ + + +/* =========================================================================================================================== */ +/* ================ RSTGEN ================ */ +/* =========================================================================================================================== */ + +/* ========================================================== CFG ========================================================== */ +#define RSTGEN_CFG_WDREN_Pos (1UL) /*!< RSTGEN CFG: WDREN (Bit 1) */ +#define RSTGEN_CFG_WDREN_Msk (0x2UL) /*!< RSTGEN CFG: WDREN (Bitfield-Mask: 0x01) */ +#define RSTGEN_CFG_BODHREN_Pos (0UL) /*!< RSTGEN CFG: BODHREN (Bit 0) */ +#define RSTGEN_CFG_BODHREN_Msk (0x1UL) /*!< RSTGEN CFG: BODHREN (Bitfield-Mask: 0x01) */ +/* ========================================================= SWPOI ========================================================= */ +#define RSTGEN_SWPOI_SWPOIKEY_Pos (0UL) /*!< RSTGEN SWPOI: SWPOIKEY (Bit 0) */ +#define RSTGEN_SWPOI_SWPOIKEY_Msk (0xffUL) /*!< RSTGEN SWPOI: SWPOIKEY (Bitfield-Mask: 0xff) */ +/* ========================================================= SWPOR ========================================================= */ +#define RSTGEN_SWPOR_SWPORKEY_Pos (0UL) /*!< RSTGEN SWPOR: SWPORKEY (Bit 0) */ +#define RSTGEN_SWPOR_SWPORKEY_Msk (0xffUL) /*!< RSTGEN SWPOR: SWPORKEY (Bitfield-Mask: 0xff) */ +/* ======================================================== TPIURST ======================================================== */ +#define RSTGEN_TPIURST_TPIURST_Pos (0UL) /*!< RSTGEN TPIURST: TPIURST (Bit 0) */ +#define RSTGEN_TPIURST_TPIURST_Msk (0x1UL) /*!< RSTGEN TPIURST: TPIURST (Bitfield-Mask: 0x01) */ +/* ========================================================= INTEN ========================================================= */ +#define RSTGEN_INTEN_BODH_Pos (0UL) /*!< RSTGEN INTEN: BODH (Bit 0) */ +#define RSTGEN_INTEN_BODH_Msk (0x1UL) /*!< RSTGEN INTEN: BODH (Bitfield-Mask: 0x01) */ +/* ======================================================== INTSTAT ======================================================== */ +#define RSTGEN_INTSTAT_BODH_Pos (0UL) /*!< RSTGEN INTSTAT: BODH (Bit 0) */ +#define RSTGEN_INTSTAT_BODH_Msk (0x1UL) /*!< RSTGEN INTSTAT: BODH (Bitfield-Mask: 0x01) */ +/* ======================================================== INTCLR ========================================================= */ +#define RSTGEN_INTCLR_BODH_Pos (0UL) /*!< RSTGEN INTCLR: BODH (Bit 0) */ +#define RSTGEN_INTCLR_BODH_Msk (0x1UL) /*!< RSTGEN INTCLR: BODH (Bitfield-Mask: 0x01) */ +/* ======================================================== INTSET ========================================================= */ +#define RSTGEN_INTSET_BODH_Pos (0UL) /*!< RSTGEN INTSET: BODH (Bit 0) */ +#define RSTGEN_INTSET_BODH_Msk (0x1UL) /*!< RSTGEN INTSET: BODH (Bitfield-Mask: 0x01) */ +/* ========================================================= STAT ========================================================== */ +#define RSTGEN_STAT_SBOOT_Pos (31UL) /*!< RSTGEN STAT: SBOOT (Bit 31) */ +#define RSTGEN_STAT_SBOOT_Msk (0x80000000UL) /*!< RSTGEN STAT: SBOOT (Bitfield-Mask: 0x01) */ +#define RSTGEN_STAT_FBOOT_Pos (30UL) /*!< RSTGEN STAT: FBOOT (Bit 30) */ +#define RSTGEN_STAT_FBOOT_Msk (0x40000000UL) /*!< RSTGEN STAT: FBOOT (Bitfield-Mask: 0x01) */ +#define RSTGEN_STAT_BOBSTAT_Pos (10UL) /*!< RSTGEN STAT: BOBSTAT (Bit 10) */ +#define RSTGEN_STAT_BOBSTAT_Msk (0x400UL) /*!< RSTGEN STAT: BOBSTAT (Bitfield-Mask: 0x01) */ +#define RSTGEN_STAT_BOFSTAT_Pos (9UL) /*!< RSTGEN STAT: BOFSTAT (Bit 9) */ +#define RSTGEN_STAT_BOFSTAT_Msk (0x200UL) /*!< RSTGEN STAT: BOFSTAT (Bitfield-Mask: 0x01) */ +#define RSTGEN_STAT_BOCSTAT_Pos (8UL) /*!< RSTGEN STAT: BOCSTAT (Bit 8) */ +#define RSTGEN_STAT_BOCSTAT_Msk (0x100UL) /*!< RSTGEN STAT: BOCSTAT (Bitfield-Mask: 0x01) */ +#define RSTGEN_STAT_BOUSTAT_Pos (7UL) /*!< RSTGEN STAT: BOUSTAT (Bit 7) */ +#define RSTGEN_STAT_BOUSTAT_Msk (0x80UL) /*!< RSTGEN STAT: BOUSTAT (Bitfield-Mask: 0x01) */ +#define RSTGEN_STAT_WDRSTAT_Pos (6UL) /*!< RSTGEN STAT: WDRSTAT (Bit 6) */ +#define RSTGEN_STAT_WDRSTAT_Msk (0x40UL) /*!< RSTGEN STAT: WDRSTAT (Bitfield-Mask: 0x01) */ +#define RSTGEN_STAT_DBGRSTAT_Pos (5UL) /*!< RSTGEN STAT: DBGRSTAT (Bit 5) */ +#define RSTGEN_STAT_DBGRSTAT_Msk (0x20UL) /*!< RSTGEN STAT: DBGRSTAT (Bitfield-Mask: 0x01) */ +#define RSTGEN_STAT_POIRSTAT_Pos (4UL) /*!< RSTGEN STAT: POIRSTAT (Bit 4) */ +#define RSTGEN_STAT_POIRSTAT_Msk (0x10UL) /*!< RSTGEN STAT: POIRSTAT (Bitfield-Mask: 0x01) */ +#define RSTGEN_STAT_SWRSTAT_Pos (3UL) /*!< RSTGEN STAT: SWRSTAT (Bit 3) */ +#define RSTGEN_STAT_SWRSTAT_Msk (0x8UL) /*!< RSTGEN STAT: SWRSTAT (Bitfield-Mask: 0x01) */ +#define RSTGEN_STAT_BORSTAT_Pos (2UL) /*!< RSTGEN STAT: BORSTAT (Bit 2) */ +#define RSTGEN_STAT_BORSTAT_Msk (0x4UL) /*!< RSTGEN STAT: BORSTAT (Bitfield-Mask: 0x01) */ +#define RSTGEN_STAT_PORSTAT_Pos (1UL) /*!< RSTGEN STAT: PORSTAT (Bit 1) */ +#define RSTGEN_STAT_PORSTAT_Msk (0x2UL) /*!< RSTGEN STAT: PORSTAT (Bitfield-Mask: 0x01) */ +#define RSTGEN_STAT_EXRSTAT_Pos (0UL) /*!< RSTGEN STAT: EXRSTAT (Bit 0) */ +#define RSTGEN_STAT_EXRSTAT_Msk (0x1UL) /*!< RSTGEN STAT: EXRSTAT (Bitfield-Mask: 0x01) */ + + +/* =========================================================================================================================== */ +/* ================ RTC ================ */ +/* =========================================================================================================================== */ + +/* ======================================================== CTRLOW ========================================================= */ +#define RTC_CTRLOW_CTRHR_Pos (24UL) /*!< RTC CTRLOW: CTRHR (Bit 24) */ +#define RTC_CTRLOW_CTRHR_Msk (0x3f000000UL) /*!< RTC CTRLOW: CTRHR (Bitfield-Mask: 0x3f) */ +#define RTC_CTRLOW_CTRMIN_Pos (16UL) /*!< RTC CTRLOW: CTRMIN (Bit 16) */ +#define RTC_CTRLOW_CTRMIN_Msk (0x7f0000UL) /*!< RTC CTRLOW: CTRMIN (Bitfield-Mask: 0x7f) */ +#define RTC_CTRLOW_CTRSEC_Pos (8UL) /*!< RTC CTRLOW: CTRSEC (Bit 8) */ +#define RTC_CTRLOW_CTRSEC_Msk (0x7f00UL) /*!< RTC CTRLOW: CTRSEC (Bitfield-Mask: 0x7f) */ +#define RTC_CTRLOW_CTR100_Pos (0UL) /*!< RTC CTRLOW: CTR100 (Bit 0) */ +#define RTC_CTRLOW_CTR100_Msk (0xffUL) /*!< RTC CTRLOW: CTR100 (Bitfield-Mask: 0xff) */ +/* ========================================================= CTRUP ========================================================= */ +#define RTC_CTRUP_CTERR_Pos (31UL) /*!< RTC CTRUP: CTERR (Bit 31) */ +#define RTC_CTRUP_CTERR_Msk (0x80000000UL) /*!< RTC CTRUP: CTERR (Bitfield-Mask: 0x01) */ +#define RTC_CTRUP_CEB_Pos (28UL) /*!< RTC CTRUP: CEB (Bit 28) */ +#define RTC_CTRUP_CEB_Msk (0x10000000UL) /*!< RTC CTRUP: CEB (Bitfield-Mask: 0x01) */ +#define RTC_CTRUP_CB_Pos (27UL) /*!< RTC CTRUP: CB (Bit 27) */ +#define RTC_CTRUP_CB_Msk (0x8000000UL) /*!< RTC CTRUP: CB (Bitfield-Mask: 0x01) */ +#define RTC_CTRUP_CTRWKDY_Pos (24UL) /*!< RTC CTRUP: CTRWKDY (Bit 24) */ +#define RTC_CTRUP_CTRWKDY_Msk (0x7000000UL) /*!< RTC CTRUP: CTRWKDY (Bitfield-Mask: 0x07) */ +#define RTC_CTRUP_CTRYR_Pos (16UL) /*!< RTC CTRUP: CTRYR (Bit 16) */ +#define RTC_CTRUP_CTRYR_Msk (0xff0000UL) /*!< RTC CTRUP: CTRYR (Bitfield-Mask: 0xff) */ +#define RTC_CTRUP_CTRMO_Pos (8UL) /*!< RTC CTRUP: CTRMO (Bit 8) */ +#define RTC_CTRUP_CTRMO_Msk (0x1f00UL) /*!< RTC CTRUP: CTRMO (Bitfield-Mask: 0x1f) */ +#define RTC_CTRUP_CTRDATE_Pos (0UL) /*!< RTC CTRUP: CTRDATE (Bit 0) */ +#define RTC_CTRUP_CTRDATE_Msk (0x3fUL) /*!< RTC CTRUP: CTRDATE (Bitfield-Mask: 0x3f) */ +/* ======================================================== ALMLOW ========================================================= */ +#define RTC_ALMLOW_ALMHR_Pos (24UL) /*!< RTC ALMLOW: ALMHR (Bit 24) */ +#define RTC_ALMLOW_ALMHR_Msk (0x3f000000UL) /*!< RTC ALMLOW: ALMHR (Bitfield-Mask: 0x3f) */ +#define RTC_ALMLOW_ALMMIN_Pos (16UL) /*!< RTC ALMLOW: ALMMIN (Bit 16) */ +#define RTC_ALMLOW_ALMMIN_Msk (0x7f0000UL) /*!< RTC ALMLOW: ALMMIN (Bitfield-Mask: 0x7f) */ +#define RTC_ALMLOW_ALMSEC_Pos (8UL) /*!< RTC ALMLOW: ALMSEC (Bit 8) */ +#define RTC_ALMLOW_ALMSEC_Msk (0x7f00UL) /*!< RTC ALMLOW: ALMSEC (Bitfield-Mask: 0x7f) */ +#define RTC_ALMLOW_ALM100_Pos (0UL) /*!< RTC ALMLOW: ALM100 (Bit 0) */ +#define RTC_ALMLOW_ALM100_Msk (0xffUL) /*!< RTC ALMLOW: ALM100 (Bitfield-Mask: 0xff) */ +/* ========================================================= ALMUP ========================================================= */ +#define RTC_ALMUP_ALMWKDY_Pos (16UL) /*!< RTC ALMUP: ALMWKDY (Bit 16) */ +#define RTC_ALMUP_ALMWKDY_Msk (0x70000UL) /*!< RTC ALMUP: ALMWKDY (Bitfield-Mask: 0x07) */ +#define RTC_ALMUP_ALMMO_Pos (8UL) /*!< RTC ALMUP: ALMMO (Bit 8) */ +#define RTC_ALMUP_ALMMO_Msk (0x1f00UL) /*!< RTC ALMUP: ALMMO (Bitfield-Mask: 0x1f) */ +#define RTC_ALMUP_ALMDATE_Pos (0UL) /*!< RTC ALMUP: ALMDATE (Bit 0) */ +#define RTC_ALMUP_ALMDATE_Msk (0x3fUL) /*!< RTC ALMUP: ALMDATE (Bitfield-Mask: 0x3f) */ +/* ======================================================== RTCCTL ========================================================= */ +#define RTC_RTCCTL_HR1224_Pos (5UL) /*!< RTC RTCCTL: HR1224 (Bit 5) */ +#define RTC_RTCCTL_HR1224_Msk (0x20UL) /*!< RTC RTCCTL: HR1224 (Bitfield-Mask: 0x01) */ +#define RTC_RTCCTL_RSTOP_Pos (4UL) /*!< RTC RTCCTL: RSTOP (Bit 4) */ +#define RTC_RTCCTL_RSTOP_Msk (0x10UL) /*!< RTC RTCCTL: RSTOP (Bitfield-Mask: 0x01) */ +#define RTC_RTCCTL_RPT_Pos (1UL) /*!< RTC RTCCTL: RPT (Bit 1) */ +#define RTC_RTCCTL_RPT_Msk (0xeUL) /*!< RTC RTCCTL: RPT (Bitfield-Mask: 0x07) */ +#define RTC_RTCCTL_WRTC_Pos (0UL) /*!< RTC RTCCTL: WRTC (Bit 0) */ +#define RTC_RTCCTL_WRTC_Msk (0x1UL) /*!< RTC RTCCTL: WRTC (Bitfield-Mask: 0x01) */ +/* ========================================================= INTEN ========================================================= */ +#define RTC_INTEN_ALM_Pos (0UL) /*!< RTC INTEN: ALM (Bit 0) */ +#define RTC_INTEN_ALM_Msk (0x1UL) /*!< RTC INTEN: ALM (Bitfield-Mask: 0x01) */ +/* ======================================================== INTSTAT ======================================================== */ +#define RTC_INTSTAT_ALM_Pos (0UL) /*!< RTC INTSTAT: ALM (Bit 0) */ +#define RTC_INTSTAT_ALM_Msk (0x1UL) /*!< RTC INTSTAT: ALM (Bitfield-Mask: 0x01) */ +/* ======================================================== INTCLR ========================================================= */ +#define RTC_INTCLR_ALM_Pos (0UL) /*!< RTC INTCLR: ALM (Bit 0) */ +#define RTC_INTCLR_ALM_Msk (0x1UL) /*!< RTC INTCLR: ALM (Bitfield-Mask: 0x01) */ +/* ======================================================== INTSET ========================================================= */ +#define RTC_INTSET_ALM_Pos (0UL) /*!< RTC INTSET: ALM (Bit 0) */ +#define RTC_INTSET_ALM_Msk (0x1UL) /*!< RTC INTSET: ALM (Bitfield-Mask: 0x01) */ + + +/* =========================================================================================================================== */ +/* ================ SCARD ================ */ +/* =========================================================================================================================== */ + +/* ========================================================== SR =========================================================== */ +#define SCARD_SR_FHF_Pos (6UL) /*!< SCARD SR: FHF (Bit 6) */ +#define SCARD_SR_FHF_Msk (0x40UL) /*!< SCARD SR: FHF (Bitfield-Mask: 0x01) */ +#define SCARD_SR_FT2REND_Pos (5UL) /*!< SCARD SR: FT2REND (Bit 5) */ +#define SCARD_SR_FT2REND_Msk (0x20UL) /*!< SCARD SR: FT2REND (Bitfield-Mask: 0x01) */ +#define SCARD_SR_PE_Pos (4UL) /*!< SCARD SR: PE (Bit 4) */ +#define SCARD_SR_PE_Msk (0x10UL) /*!< SCARD SR: PE (Bitfield-Mask: 0x01) */ +#define SCARD_SR_OVR_Pos (3UL) /*!< SCARD SR: OVR (Bit 3) */ +#define SCARD_SR_OVR_Msk (0x8UL) /*!< SCARD SR: OVR (Bitfield-Mask: 0x01) */ +#define SCARD_SR_FER_Pos (2UL) /*!< SCARD SR: FER (Bit 2) */ +#define SCARD_SR_FER_Msk (0x4UL) /*!< SCARD SR: FER (Bitfield-Mask: 0x01) */ +#define SCARD_SR_TBERBF_Pos (1UL) /*!< SCARD SR: TBERBF (Bit 1) */ +#define SCARD_SR_TBERBF_Msk (0x2UL) /*!< SCARD SR: TBERBF (Bitfield-Mask: 0x01) */ +#define SCARD_SR_FNE_Pos (0UL) /*!< SCARD SR: FNE (Bit 0) */ +#define SCARD_SR_FNE_Msk (0x1UL) /*!< SCARD SR: FNE (Bitfield-Mask: 0x01) */ +/* ========================================================== DR =========================================================== */ +#define SCARD_DR_DR_Pos (0UL) /*!< SCARD DR: DR (Bit 0) */ +#define SCARD_DR_DR_Msk (0xffUL) /*!< SCARD DR: DR (Bitfield-Mask: 0xff) */ +/* ========================================================== SR1 ========================================================== */ +#define SCARD_SR1_IDLE_Pos (3UL) /*!< SCARD SR1: IDLE (Bit 3) */ +#define SCARD_SR1_IDLE_Msk (0x8UL) /*!< SCARD SR1: IDLE (Bitfield-Mask: 0x01) */ +#define SCARD_SR1_SYNCEND_Pos (2UL) /*!< SCARD SR1: SYNCEND (Bit 2) */ +#define SCARD_SR1_SYNCEND_Msk (0x4UL) /*!< SCARD SR1: SYNCEND (Bitfield-Mask: 0x01) */ +#define SCARD_SR1_PRL_Pos (1UL) /*!< SCARD SR1: PRL (Bit 1) */ +#define SCARD_SR1_PRL_Msk (0x2UL) /*!< SCARD SR1: PRL (Bitfield-Mask: 0x01) */ +#define SCARD_SR1_ECNTOVER_Pos (0UL) /*!< SCARD SR1: ECNTOVER (Bit 0) */ +#define SCARD_SR1_ECNTOVER_Msk (0x1UL) /*!< SCARD SR1: ECNTOVER (Bitfield-Mask: 0x01) */ +/* ====================================================== RETXCNTRMI ======================================================= */ +#define SCARD_RETXCNTRMI_RETXCNTRMI_Pos (0UL) /*!< SCARD RETXCNTRMI: RETXCNTRMI (Bit 0) */ +#define SCARD_RETXCNTRMI_RETXCNTRMI_Msk (0xfUL) /*!< SCARD RETXCNTRMI: RETXCNTRMI (Bitfield-Mask: 0x0f) */ +/* ======================================================== CLKCTRL ======================================================== */ +#define SCARD_CLKCTRL_APBCLKEN_Pos (1UL) /*!< SCARD CLKCTRL: APBCLKEN (Bit 1) */ +#define SCARD_CLKCTRL_APBCLKEN_Msk (0x2UL) /*!< SCARD CLKCTRL: APBCLKEN (Bitfield-Mask: 0x01) */ +#define SCARD_CLKCTRL_CLKEN_Pos (0UL) /*!< SCARD CLKCTRL: CLKEN (Bit 0) */ +#define SCARD_CLKCTRL_CLKEN_Msk (0x1UL) /*!< SCARD CLKCTRL: CLKEN (Bitfield-Mask: 0x01) */ + + +/* =========================================================================================================================== */ +/* ================ SECURITY ================ */ +/* =========================================================================================================================== */ + +/* ========================================================= CTRL ========================================================== */ +#define SECURITY_CTRL_CRCERROR_Pos (31UL) /*!< SECURITY CTRL: CRCERROR (Bit 31) */ +#define SECURITY_CTRL_CRCERROR_Msk (0x80000000UL) /*!< SECURITY CTRL: CRCERROR (Bitfield-Mask: 0x01) */ +#define SECURITY_CTRL_FUNCTION_Pos (4UL) /*!< SECURITY CTRL: FUNCTION (Bit 4) */ +#define SECURITY_CTRL_FUNCTION_Msk (0xf0UL) /*!< SECURITY CTRL: FUNCTION (Bitfield-Mask: 0x0f) */ +#define SECURITY_CTRL_ENABLE_Pos (0UL) /*!< SECURITY CTRL: ENABLE (Bit 0) */ +#define SECURITY_CTRL_ENABLE_Msk (0x1UL) /*!< SECURITY CTRL: ENABLE (Bitfield-Mask: 0x01) */ +/* ======================================================== SRCADDR ======================================================== */ +#define SECURITY_SRCADDR_ADDR_Pos (0UL) /*!< SECURITY SRCADDR: ADDR (Bit 0) */ +#define SECURITY_SRCADDR_ADDR_Msk (0xffffffffUL) /*!< SECURITY SRCADDR: ADDR (Bitfield-Mask: 0xffffffff) */ +/* ========================================================== LEN ========================================================== */ +#define SECURITY_LEN_LEN_Pos (2UL) /*!< SECURITY LEN: LEN (Bit 2) */ +#define SECURITY_LEN_LEN_Msk (0xffffcUL) /*!< SECURITY LEN: LEN (Bitfield-Mask: 0x3ffff) */ +/* ======================================================== RESULT ========================================================= */ +#define SECURITY_RESULT_CRC_Pos (0UL) /*!< SECURITY RESULT: CRC (Bit 0) */ +#define SECURITY_RESULT_CRC_Msk (0xffffffffUL) /*!< SECURITY RESULT: CRC (Bitfield-Mask: 0xffffffff) */ +/* ======================================================= LOCKCTRL ======================================================== */ +#define SECURITY_LOCKCTRL_SELECT_Pos (0UL) /*!< SECURITY LOCKCTRL: SELECT (Bit 0) */ +#define SECURITY_LOCKCTRL_SELECT_Msk (0xffUL) /*!< SECURITY LOCKCTRL: SELECT (Bitfield-Mask: 0xff) */ +/* ======================================================= LOCKSTAT ======================================================== */ +#define SECURITY_LOCKSTAT_STATUS_Pos (0UL) /*!< SECURITY LOCKSTAT: STATUS (Bit 0) */ +#define SECURITY_LOCKSTAT_STATUS_Msk (0xffffffffUL) /*!< SECURITY LOCKSTAT: STATUS (Bitfield-Mask: 0xffffffff) */ +/* ========================================================= KEY0 ========================================================== */ +#define SECURITY_KEY0_KEY0_Pos (0UL) /*!< SECURITY KEY0: KEY0 (Bit 0) */ +#define SECURITY_KEY0_KEY0_Msk (0xffffffffUL) /*!< SECURITY KEY0: KEY0 (Bitfield-Mask: 0xffffffff) */ +/* ========================================================= KEY1 ========================================================== */ +#define SECURITY_KEY1_KEY1_Pos (0UL) /*!< SECURITY KEY1: KEY1 (Bit 0) */ +#define SECURITY_KEY1_KEY1_Msk (0xffffffffUL) /*!< SECURITY KEY1: KEY1 (Bitfield-Mask: 0xffffffff) */ +/* ========================================================= KEY2 ========================================================== */ +#define SECURITY_KEY2_KEY2_Pos (0UL) /*!< SECURITY KEY2: KEY2 (Bit 0) */ +#define SECURITY_KEY2_KEY2_Msk (0xffffffffUL) /*!< SECURITY KEY2: KEY2 (Bitfield-Mask: 0xffffffff) */ +/* ========================================================= KEY3 ========================================================== */ +#define SECURITY_KEY3_KEY3_Pos (0UL) /*!< SECURITY KEY3: KEY3 (Bit 0) */ +#define SECURITY_KEY3_KEY3_Msk (0xffffffffUL) /*!< SECURITY KEY3: KEY3 (Bitfield-Mask: 0xffffffff) */ + + +/* =========================================================================================================================== */ +/* ================ UART0 ================ */ +/* =========================================================================================================================== */ + +/* ========================================================== DR =========================================================== */ +#define UART0_DR_OEDATA_Pos (11UL) /*!< UART0 DR: OEDATA (Bit 11) */ +#define UART0_DR_OEDATA_Msk (0x800UL) /*!< UART0 DR: OEDATA (Bitfield-Mask: 0x01) */ +#define UART0_DR_BEDATA_Pos (10UL) /*!< UART0 DR: BEDATA (Bit 10) */ +#define UART0_DR_BEDATA_Msk (0x400UL) /*!< UART0 DR: BEDATA (Bitfield-Mask: 0x01) */ +#define UART0_DR_PEDATA_Pos (9UL) /*!< UART0 DR: PEDATA (Bit 9) */ +#define UART0_DR_PEDATA_Msk (0x200UL) /*!< UART0 DR: PEDATA (Bitfield-Mask: 0x01) */ +#define UART0_DR_FEDATA_Pos (8UL) /*!< UART0 DR: FEDATA (Bit 8) */ +#define UART0_DR_FEDATA_Msk (0x100UL) /*!< UART0 DR: FEDATA (Bitfield-Mask: 0x01) */ +#define UART0_DR_DATA_Pos (0UL) /*!< UART0 DR: DATA (Bit 0) */ +#define UART0_DR_DATA_Msk (0xffUL) /*!< UART0 DR: DATA (Bitfield-Mask: 0xff) */ +/* ========================================================== RSR ========================================================== */ +#define UART0_RSR_OESTAT_Pos (3UL) /*!< UART0 RSR: OESTAT (Bit 3) */ +#define UART0_RSR_OESTAT_Msk (0x8UL) /*!< UART0 RSR: OESTAT (Bitfield-Mask: 0x01) */ +#define UART0_RSR_BESTAT_Pos (2UL) /*!< UART0 RSR: BESTAT (Bit 2) */ +#define UART0_RSR_BESTAT_Msk (0x4UL) /*!< UART0 RSR: BESTAT (Bitfield-Mask: 0x01) */ +#define UART0_RSR_PESTAT_Pos (1UL) /*!< UART0 RSR: PESTAT (Bit 1) */ +#define UART0_RSR_PESTAT_Msk (0x2UL) /*!< UART0 RSR: PESTAT (Bitfield-Mask: 0x01) */ +#define UART0_RSR_FESTAT_Pos (0UL) /*!< UART0 RSR: FESTAT (Bit 0) */ +#define UART0_RSR_FESTAT_Msk (0x1UL) /*!< UART0 RSR: FESTAT (Bitfield-Mask: 0x01) */ +/* ========================================================== FR =========================================================== */ +#define UART0_FR_TXBUSY_Pos (8UL) /*!< UART0 FR: TXBUSY (Bit 8) */ +#define UART0_FR_TXBUSY_Msk (0x100UL) /*!< UART0 FR: TXBUSY (Bitfield-Mask: 0x01) */ +#define UART0_FR_TXFE_Pos (7UL) /*!< UART0 FR: TXFE (Bit 7) */ +#define UART0_FR_TXFE_Msk (0x80UL) /*!< UART0 FR: TXFE (Bitfield-Mask: 0x01) */ +#define UART0_FR_RXFF_Pos (6UL) /*!< UART0 FR: RXFF (Bit 6) */ +#define UART0_FR_RXFF_Msk (0x40UL) /*!< UART0 FR: RXFF (Bitfield-Mask: 0x01) */ +#define UART0_FR_TXFF_Pos (5UL) /*!< UART0 FR: TXFF (Bit 5) */ +#define UART0_FR_TXFF_Msk (0x20UL) /*!< UART0 FR: TXFF (Bitfield-Mask: 0x01) */ +#define UART0_FR_RXFE_Pos (4UL) /*!< UART0 FR: RXFE (Bit 4) */ +#define UART0_FR_RXFE_Msk (0x10UL) /*!< UART0 FR: RXFE (Bitfield-Mask: 0x01) */ +#define UART0_FR_BUSY_Pos (3UL) /*!< UART0 FR: BUSY (Bit 3) */ +#define UART0_FR_BUSY_Msk (0x8UL) /*!< UART0 FR: BUSY (Bitfield-Mask: 0x01) */ +#define UART0_FR_DCD_Pos (2UL) /*!< UART0 FR: DCD (Bit 2) */ +#define UART0_FR_DCD_Msk (0x4UL) /*!< UART0 FR: DCD (Bitfield-Mask: 0x01) */ +#define UART0_FR_DSR_Pos (1UL) /*!< UART0 FR: DSR (Bit 1) */ +#define UART0_FR_DSR_Msk (0x2UL) /*!< UART0 FR: DSR (Bitfield-Mask: 0x01) */ +#define UART0_FR_CTS_Pos (0UL) /*!< UART0 FR: CTS (Bit 0) */ +#define UART0_FR_CTS_Msk (0x1UL) /*!< UART0 FR: CTS (Bitfield-Mask: 0x01) */ +/* ========================================================= ILPR ========================================================== */ +#define UART0_ILPR_ILPDVSR_Pos (0UL) /*!< UART0 ILPR: ILPDVSR (Bit 0) */ +#define UART0_ILPR_ILPDVSR_Msk (0xffUL) /*!< UART0 ILPR: ILPDVSR (Bitfield-Mask: 0xff) */ +/* ========================================================= IBRD ========================================================== */ +#define UART0_IBRD_DIVINT_Pos (0UL) /*!< UART0 IBRD: DIVINT (Bit 0) */ +#define UART0_IBRD_DIVINT_Msk (0xffffUL) /*!< UART0 IBRD: DIVINT (Bitfield-Mask: 0xffff) */ +/* ========================================================= FBRD ========================================================== */ +#define UART0_FBRD_DIVFRAC_Pos (0UL) /*!< UART0 FBRD: DIVFRAC (Bit 0) */ +#define UART0_FBRD_DIVFRAC_Msk (0x3fUL) /*!< UART0 FBRD: DIVFRAC (Bitfield-Mask: 0x3f) */ +/* ========================================================= LCRH ========================================================== */ +#define UART0_LCRH_SPS_Pos (7UL) /*!< UART0 LCRH: SPS (Bit 7) */ +#define UART0_LCRH_SPS_Msk (0x80UL) /*!< UART0 LCRH: SPS (Bitfield-Mask: 0x01) */ +#define UART0_LCRH_WLEN_Pos (5UL) /*!< UART0 LCRH: WLEN (Bit 5) */ +#define UART0_LCRH_WLEN_Msk (0x60UL) /*!< UART0 LCRH: WLEN (Bitfield-Mask: 0x03) */ +#define UART0_LCRH_FEN_Pos (4UL) /*!< UART0 LCRH: FEN (Bit 4) */ +#define UART0_LCRH_FEN_Msk (0x10UL) /*!< UART0 LCRH: FEN (Bitfield-Mask: 0x01) */ +#define UART0_LCRH_STP2_Pos (3UL) /*!< UART0 LCRH: STP2 (Bit 3) */ +#define UART0_LCRH_STP2_Msk (0x8UL) /*!< UART0 LCRH: STP2 (Bitfield-Mask: 0x01) */ +#define UART0_LCRH_EPS_Pos (2UL) /*!< UART0 LCRH: EPS (Bit 2) */ +#define UART0_LCRH_EPS_Msk (0x4UL) /*!< UART0 LCRH: EPS (Bitfield-Mask: 0x01) */ +#define UART0_LCRH_PEN_Pos (1UL) /*!< UART0 LCRH: PEN (Bit 1) */ +#define UART0_LCRH_PEN_Msk (0x2UL) /*!< UART0 LCRH: PEN (Bitfield-Mask: 0x01) */ +#define UART0_LCRH_BRK_Pos (0UL) /*!< UART0 LCRH: BRK (Bit 0) */ +#define UART0_LCRH_BRK_Msk (0x1UL) /*!< UART0 LCRH: BRK (Bitfield-Mask: 0x01) */ +/* ========================================================== CR =========================================================== */ +#define UART0_CR_CTSEN_Pos (15UL) /*!< UART0 CR: CTSEN (Bit 15) */ +#define UART0_CR_CTSEN_Msk (0x8000UL) /*!< UART0 CR: CTSEN (Bitfield-Mask: 0x01) */ +#define UART0_CR_RTSEN_Pos (14UL) /*!< UART0 CR: RTSEN (Bit 14) */ +#define UART0_CR_RTSEN_Msk (0x4000UL) /*!< UART0 CR: RTSEN (Bitfield-Mask: 0x01) */ +#define UART0_CR_OUT2_Pos (13UL) /*!< UART0 CR: OUT2 (Bit 13) */ +#define UART0_CR_OUT2_Msk (0x2000UL) /*!< UART0 CR: OUT2 (Bitfield-Mask: 0x01) */ +#define UART0_CR_OUT1_Pos (12UL) /*!< UART0 CR: OUT1 (Bit 12) */ +#define UART0_CR_OUT1_Msk (0x1000UL) /*!< UART0 CR: OUT1 (Bitfield-Mask: 0x01) */ +#define UART0_CR_RTS_Pos (11UL) /*!< UART0 CR: RTS (Bit 11) */ +#define UART0_CR_RTS_Msk (0x800UL) /*!< UART0 CR: RTS (Bitfield-Mask: 0x01) */ +#define UART0_CR_DTR_Pos (10UL) /*!< UART0 CR: DTR (Bit 10) */ +#define UART0_CR_DTR_Msk (0x400UL) /*!< UART0 CR: DTR (Bitfield-Mask: 0x01) */ +#define UART0_CR_RXE_Pos (9UL) /*!< UART0 CR: RXE (Bit 9) */ +#define UART0_CR_RXE_Msk (0x200UL) /*!< UART0 CR: RXE (Bitfield-Mask: 0x01) */ +#define UART0_CR_TXE_Pos (8UL) /*!< UART0 CR: TXE (Bit 8) */ +#define UART0_CR_TXE_Msk (0x100UL) /*!< UART0 CR: TXE (Bitfield-Mask: 0x01) */ +#define UART0_CR_LBE_Pos (7UL) /*!< UART0 CR: LBE (Bit 7) */ +#define UART0_CR_LBE_Msk (0x80UL) /*!< UART0 CR: LBE (Bitfield-Mask: 0x01) */ +#define UART0_CR_CLKSEL_Pos (4UL) /*!< UART0 CR: CLKSEL (Bit 4) */ +#define UART0_CR_CLKSEL_Msk (0x70UL) /*!< UART0 CR: CLKSEL (Bitfield-Mask: 0x07) */ +#define UART0_CR_CLKEN_Pos (3UL) /*!< UART0 CR: CLKEN (Bit 3) */ +#define UART0_CR_CLKEN_Msk (0x8UL) /*!< UART0 CR: CLKEN (Bitfield-Mask: 0x01) */ +#define UART0_CR_SIRLP_Pos (2UL) /*!< UART0 CR: SIRLP (Bit 2) */ +#define UART0_CR_SIRLP_Msk (0x4UL) /*!< UART0 CR: SIRLP (Bitfield-Mask: 0x01) */ +#define UART0_CR_SIREN_Pos (1UL) /*!< UART0 CR: SIREN (Bit 1) */ +#define UART0_CR_SIREN_Msk (0x2UL) /*!< UART0 CR: SIREN (Bitfield-Mask: 0x01) */ +#define UART0_CR_UARTEN_Pos (0UL) /*!< UART0 CR: UARTEN (Bit 0) */ +#define UART0_CR_UARTEN_Msk (0x1UL) /*!< UART0 CR: UARTEN (Bitfield-Mask: 0x01) */ +/* ========================================================= IFLS ========================================================== */ +#define UART0_IFLS_RXIFLSEL_Pos (3UL) /*!< UART0 IFLS: RXIFLSEL (Bit 3) */ +#define UART0_IFLS_RXIFLSEL_Msk (0x38UL) /*!< UART0 IFLS: RXIFLSEL (Bitfield-Mask: 0x07) */ +#define UART0_IFLS_TXIFLSEL_Pos (0UL) /*!< UART0 IFLS: TXIFLSEL (Bit 0) */ +#define UART0_IFLS_TXIFLSEL_Msk (0x7UL) /*!< UART0 IFLS: TXIFLSEL (Bitfield-Mask: 0x07) */ +/* ========================================================== IER ========================================================== */ +#define UART0_IER_OEIM_Pos (10UL) /*!< UART0 IER: OEIM (Bit 10) */ +#define UART0_IER_OEIM_Msk (0x400UL) /*!< UART0 IER: OEIM (Bitfield-Mask: 0x01) */ +#define UART0_IER_BEIM_Pos (9UL) /*!< UART0 IER: BEIM (Bit 9) */ +#define UART0_IER_BEIM_Msk (0x200UL) /*!< UART0 IER: BEIM (Bitfield-Mask: 0x01) */ +#define UART0_IER_PEIM_Pos (8UL) /*!< UART0 IER: PEIM (Bit 8) */ +#define UART0_IER_PEIM_Msk (0x100UL) /*!< UART0 IER: PEIM (Bitfield-Mask: 0x01) */ +#define UART0_IER_FEIM_Pos (7UL) /*!< UART0 IER: FEIM (Bit 7) */ +#define UART0_IER_FEIM_Msk (0x80UL) /*!< UART0 IER: FEIM (Bitfield-Mask: 0x01) */ +#define UART0_IER_RTIM_Pos (6UL) /*!< UART0 IER: RTIM (Bit 6) */ +#define UART0_IER_RTIM_Msk (0x40UL) /*!< UART0 IER: RTIM (Bitfield-Mask: 0x01) */ +#define UART0_IER_TXIM_Pos (5UL) /*!< UART0 IER: TXIM (Bit 5) */ +#define UART0_IER_TXIM_Msk (0x20UL) /*!< UART0 IER: TXIM (Bitfield-Mask: 0x01) */ +#define UART0_IER_RXIM_Pos (4UL) /*!< UART0 IER: RXIM (Bit 4) */ +#define UART0_IER_RXIM_Msk (0x10UL) /*!< UART0 IER: RXIM (Bitfield-Mask: 0x01) */ +#define UART0_IER_DSRMIM_Pos (3UL) /*!< UART0 IER: DSRMIM (Bit 3) */ +#define UART0_IER_DSRMIM_Msk (0x8UL) /*!< UART0 IER: DSRMIM (Bitfield-Mask: 0x01) */ +#define UART0_IER_DCDMIM_Pos (2UL) /*!< UART0 IER: DCDMIM (Bit 2) */ +#define UART0_IER_DCDMIM_Msk (0x4UL) /*!< UART0 IER: DCDMIM (Bitfield-Mask: 0x01) */ +#define UART0_IER_CTSMIM_Pos (1UL) /*!< UART0 IER: CTSMIM (Bit 1) */ +#define UART0_IER_CTSMIM_Msk (0x2UL) /*!< UART0 IER: CTSMIM (Bitfield-Mask: 0x01) */ +#define UART0_IER_TXCMPMIM_Pos (0UL) /*!< UART0 IER: TXCMPMIM (Bit 0) */ +#define UART0_IER_TXCMPMIM_Msk (0x1UL) /*!< UART0 IER: TXCMPMIM (Bitfield-Mask: 0x01) */ +/* ========================================================== IES ========================================================== */ +#define UART0_IES_OERIS_Pos (10UL) /*!< UART0 IES: OERIS (Bit 10) */ +#define UART0_IES_OERIS_Msk (0x400UL) /*!< UART0 IES: OERIS (Bitfield-Mask: 0x01) */ +#define UART0_IES_BERIS_Pos (9UL) /*!< UART0 IES: BERIS (Bit 9) */ +#define UART0_IES_BERIS_Msk (0x200UL) /*!< UART0 IES: BERIS (Bitfield-Mask: 0x01) */ +#define UART0_IES_PERIS_Pos (8UL) /*!< UART0 IES: PERIS (Bit 8) */ +#define UART0_IES_PERIS_Msk (0x100UL) /*!< UART0 IES: PERIS (Bitfield-Mask: 0x01) */ +#define UART0_IES_FERIS_Pos (7UL) /*!< UART0 IES: FERIS (Bit 7) */ +#define UART0_IES_FERIS_Msk (0x80UL) /*!< UART0 IES: FERIS (Bitfield-Mask: 0x01) */ +#define UART0_IES_RTRIS_Pos (6UL) /*!< UART0 IES: RTRIS (Bit 6) */ +#define UART0_IES_RTRIS_Msk (0x40UL) /*!< UART0 IES: RTRIS (Bitfield-Mask: 0x01) */ +#define UART0_IES_TXRIS_Pos (5UL) /*!< UART0 IES: TXRIS (Bit 5) */ +#define UART0_IES_TXRIS_Msk (0x20UL) /*!< UART0 IES: TXRIS (Bitfield-Mask: 0x01) */ +#define UART0_IES_RXRIS_Pos (4UL) /*!< UART0 IES: RXRIS (Bit 4) */ +#define UART0_IES_RXRIS_Msk (0x10UL) /*!< UART0 IES: RXRIS (Bitfield-Mask: 0x01) */ +#define UART0_IES_DSRMRIS_Pos (3UL) /*!< UART0 IES: DSRMRIS (Bit 3) */ +#define UART0_IES_DSRMRIS_Msk (0x8UL) /*!< UART0 IES: DSRMRIS (Bitfield-Mask: 0x01) */ +#define UART0_IES_DCDMRIS_Pos (2UL) /*!< UART0 IES: DCDMRIS (Bit 2) */ +#define UART0_IES_DCDMRIS_Msk (0x4UL) /*!< UART0 IES: DCDMRIS (Bitfield-Mask: 0x01) */ +#define UART0_IES_CTSMRIS_Pos (1UL) /*!< UART0 IES: CTSMRIS (Bit 1) */ +#define UART0_IES_CTSMRIS_Msk (0x2UL) /*!< UART0 IES: CTSMRIS (Bitfield-Mask: 0x01) */ +#define UART0_IES_TXCMPMRIS_Pos (0UL) /*!< UART0 IES: TXCMPMRIS (Bit 0) */ +#define UART0_IES_TXCMPMRIS_Msk (0x1UL) /*!< UART0 IES: TXCMPMRIS (Bitfield-Mask: 0x01) */ +/* ========================================================== MIS ========================================================== */ +#define UART0_MIS_OEMIS_Pos (10UL) /*!< UART0 MIS: OEMIS (Bit 10) */ +#define UART0_MIS_OEMIS_Msk (0x400UL) /*!< UART0 MIS: OEMIS (Bitfield-Mask: 0x01) */ +#define UART0_MIS_BEMIS_Pos (9UL) /*!< UART0 MIS: BEMIS (Bit 9) */ +#define UART0_MIS_BEMIS_Msk (0x200UL) /*!< UART0 MIS: BEMIS (Bitfield-Mask: 0x01) */ +#define UART0_MIS_PEMIS_Pos (8UL) /*!< UART0 MIS: PEMIS (Bit 8) */ +#define UART0_MIS_PEMIS_Msk (0x100UL) /*!< UART0 MIS: PEMIS (Bitfield-Mask: 0x01) */ +#define UART0_MIS_FEMIS_Pos (7UL) /*!< UART0 MIS: FEMIS (Bit 7) */ +#define UART0_MIS_FEMIS_Msk (0x80UL) /*!< UART0 MIS: FEMIS (Bitfield-Mask: 0x01) */ +#define UART0_MIS_RTMIS_Pos (6UL) /*!< UART0 MIS: RTMIS (Bit 6) */ +#define UART0_MIS_RTMIS_Msk (0x40UL) /*!< UART0 MIS: RTMIS (Bitfield-Mask: 0x01) */ +#define UART0_MIS_TXMIS_Pos (5UL) /*!< UART0 MIS: TXMIS (Bit 5) */ +#define UART0_MIS_TXMIS_Msk (0x20UL) /*!< UART0 MIS: TXMIS (Bitfield-Mask: 0x01) */ +#define UART0_MIS_RXMIS_Pos (4UL) /*!< UART0 MIS: RXMIS (Bit 4) */ +#define UART0_MIS_RXMIS_Msk (0x10UL) /*!< UART0 MIS: RXMIS (Bitfield-Mask: 0x01) */ +#define UART0_MIS_DSRMMIS_Pos (3UL) /*!< UART0 MIS: DSRMMIS (Bit 3) */ +#define UART0_MIS_DSRMMIS_Msk (0x8UL) /*!< UART0 MIS: DSRMMIS (Bitfield-Mask: 0x01) */ +#define UART0_MIS_DCDMMIS_Pos (2UL) /*!< UART0 MIS: DCDMMIS (Bit 2) */ +#define UART0_MIS_DCDMMIS_Msk (0x4UL) /*!< UART0 MIS: DCDMMIS (Bitfield-Mask: 0x01) */ +#define UART0_MIS_CTSMMIS_Pos (1UL) /*!< UART0 MIS: CTSMMIS (Bit 1) */ +#define UART0_MIS_CTSMMIS_Msk (0x2UL) /*!< UART0 MIS: CTSMMIS (Bitfield-Mask: 0x01) */ +#define UART0_MIS_TXCMPMMIS_Pos (0UL) /*!< UART0 MIS: TXCMPMMIS (Bit 0) */ +#define UART0_MIS_TXCMPMMIS_Msk (0x1UL) /*!< UART0 MIS: TXCMPMMIS (Bitfield-Mask: 0x01) */ +/* ========================================================== IEC ========================================================== */ +#define UART0_IEC_OEIC_Pos (10UL) /*!< UART0 IEC: OEIC (Bit 10) */ +#define UART0_IEC_OEIC_Msk (0x400UL) /*!< UART0 IEC: OEIC (Bitfield-Mask: 0x01) */ +#define UART0_IEC_BEIC_Pos (9UL) /*!< UART0 IEC: BEIC (Bit 9) */ +#define UART0_IEC_BEIC_Msk (0x200UL) /*!< UART0 IEC: BEIC (Bitfield-Mask: 0x01) */ +#define UART0_IEC_PEIC_Pos (8UL) /*!< UART0 IEC: PEIC (Bit 8) */ +#define UART0_IEC_PEIC_Msk (0x100UL) /*!< UART0 IEC: PEIC (Bitfield-Mask: 0x01) */ +#define UART0_IEC_FEIC_Pos (7UL) /*!< UART0 IEC: FEIC (Bit 7) */ +#define UART0_IEC_FEIC_Msk (0x80UL) /*!< UART0 IEC: FEIC (Bitfield-Mask: 0x01) */ +#define UART0_IEC_RTIC_Pos (6UL) /*!< UART0 IEC: RTIC (Bit 6) */ +#define UART0_IEC_RTIC_Msk (0x40UL) /*!< UART0 IEC: RTIC (Bitfield-Mask: 0x01) */ +#define UART0_IEC_TXIC_Pos (5UL) /*!< UART0 IEC: TXIC (Bit 5) */ +#define UART0_IEC_TXIC_Msk (0x20UL) /*!< UART0 IEC: TXIC (Bitfield-Mask: 0x01) */ +#define UART0_IEC_RXIC_Pos (4UL) /*!< UART0 IEC: RXIC (Bit 4) */ +#define UART0_IEC_RXIC_Msk (0x10UL) /*!< UART0 IEC: RXIC (Bitfield-Mask: 0x01) */ +#define UART0_IEC_DSRMIC_Pos (3UL) /*!< UART0 IEC: DSRMIC (Bit 3) */ +#define UART0_IEC_DSRMIC_Msk (0x8UL) /*!< UART0 IEC: DSRMIC (Bitfield-Mask: 0x01) */ +#define UART0_IEC_DCDMIC_Pos (2UL) /*!< UART0 IEC: DCDMIC (Bit 2) */ +#define UART0_IEC_DCDMIC_Msk (0x4UL) /*!< UART0 IEC: DCDMIC (Bitfield-Mask: 0x01) */ +#define UART0_IEC_CTSMIC_Pos (1UL) /*!< UART0 IEC: CTSMIC (Bit 1) */ +#define UART0_IEC_CTSMIC_Msk (0x2UL) /*!< UART0 IEC: CTSMIC (Bitfield-Mask: 0x01) */ +#define UART0_IEC_TXCMPMIC_Pos (0UL) /*!< UART0 IEC: TXCMPMIC (Bit 0) */ +#define UART0_IEC_TXCMPMIC_Msk (0x1UL) /*!< UART0 IEC: TXCMPMIC (Bitfield-Mask: 0x01) */ + + +/* =========================================================================================================================== */ +/* ================ VCOMP ================ */ +/* =========================================================================================================================== */ + +/* ========================================================== CFG ========================================================== */ +#define VCOMP_CFG_LVLSEL_Pos (16UL) /*!< VCOMP CFG: LVLSEL (Bit 16) */ +#define VCOMP_CFG_LVLSEL_Msk (0xf0000UL) /*!< VCOMP CFG: LVLSEL (Bitfield-Mask: 0x0f) */ +#define VCOMP_CFG_NSEL_Pos (8UL) /*!< VCOMP CFG: NSEL (Bit 8) */ +#define VCOMP_CFG_NSEL_Msk (0x300UL) /*!< VCOMP CFG: NSEL (Bitfield-Mask: 0x03) */ +#define VCOMP_CFG_PSEL_Pos (0UL) /*!< VCOMP CFG: PSEL (Bit 0) */ +#define VCOMP_CFG_PSEL_Msk (0x3UL) /*!< VCOMP CFG: PSEL (Bitfield-Mask: 0x03) */ +/* ========================================================= STAT ========================================================== */ +#define VCOMP_STAT_PWDSTAT_Pos (1UL) /*!< VCOMP STAT: PWDSTAT (Bit 1) */ +#define VCOMP_STAT_PWDSTAT_Msk (0x2UL) /*!< VCOMP STAT: PWDSTAT (Bitfield-Mask: 0x01) */ +#define VCOMP_STAT_CMPOUT_Pos (0UL) /*!< VCOMP STAT: CMPOUT (Bit 0) */ +#define VCOMP_STAT_CMPOUT_Msk (0x1UL) /*!< VCOMP STAT: CMPOUT (Bitfield-Mask: 0x01) */ +/* ======================================================== PWDKEY ========================================================= */ +#define VCOMP_PWDKEY_PWDKEY_Pos (0UL) /*!< VCOMP PWDKEY: PWDKEY (Bit 0) */ +#define VCOMP_PWDKEY_PWDKEY_Msk (0xffffffffUL) /*!< VCOMP PWDKEY: PWDKEY (Bitfield-Mask: 0xffffffff) */ +/* ========================================================= INTEN ========================================================= */ +#define VCOMP_INTEN_OUTHI_Pos (1UL) /*!< VCOMP INTEN: OUTHI (Bit 1) */ +#define VCOMP_INTEN_OUTHI_Msk (0x2UL) /*!< VCOMP INTEN: OUTHI (Bitfield-Mask: 0x01) */ +#define VCOMP_INTEN_OUTLOW_Pos (0UL) /*!< VCOMP INTEN: OUTLOW (Bit 0) */ +#define VCOMP_INTEN_OUTLOW_Msk (0x1UL) /*!< VCOMP INTEN: OUTLOW (Bitfield-Mask: 0x01) */ +/* ======================================================== INTSTAT ======================================================== */ +#define VCOMP_INTSTAT_OUTHI_Pos (1UL) /*!< VCOMP INTSTAT: OUTHI (Bit 1) */ +#define VCOMP_INTSTAT_OUTHI_Msk (0x2UL) /*!< VCOMP INTSTAT: OUTHI (Bitfield-Mask: 0x01) */ +#define VCOMP_INTSTAT_OUTLOW_Pos (0UL) /*!< VCOMP INTSTAT: OUTLOW (Bit 0) */ +#define VCOMP_INTSTAT_OUTLOW_Msk (0x1UL) /*!< VCOMP INTSTAT: OUTLOW (Bitfield-Mask: 0x01) */ +/* ======================================================== INTCLR ========================================================= */ +#define VCOMP_INTCLR_OUTHI_Pos (1UL) /*!< VCOMP INTCLR: OUTHI (Bit 1) */ +#define VCOMP_INTCLR_OUTHI_Msk (0x2UL) /*!< VCOMP INTCLR: OUTHI (Bitfield-Mask: 0x01) */ +#define VCOMP_INTCLR_OUTLOW_Pos (0UL) /*!< VCOMP INTCLR: OUTLOW (Bit 0) */ +#define VCOMP_INTCLR_OUTLOW_Msk (0x1UL) /*!< VCOMP INTCLR: OUTLOW (Bitfield-Mask: 0x01) */ +/* ======================================================== INTSET ========================================================= */ +#define VCOMP_INTSET_OUTHI_Pos (1UL) /*!< VCOMP INTSET: OUTHI (Bit 1) */ +#define VCOMP_INTSET_OUTHI_Msk (0x2UL) /*!< VCOMP INTSET: OUTHI (Bitfield-Mask: 0x01) */ +#define VCOMP_INTSET_OUTLOW_Pos (0UL) /*!< VCOMP INTSET: OUTLOW (Bit 0) */ +#define VCOMP_INTSET_OUTLOW_Msk (0x1UL) /*!< VCOMP INTSET: OUTLOW (Bitfield-Mask: 0x01) */ + + +/* =========================================================================================================================== */ +/* ================ WDT ================ */ +/* =========================================================================================================================== */ + +/* ========================================================== CFG ========================================================== */ +#define WDT_CFG_CLKSEL_Pos (24UL) /*!< WDT CFG: CLKSEL (Bit 24) */ +#define WDT_CFG_CLKSEL_Msk (0x7000000UL) /*!< WDT CFG: CLKSEL (Bitfield-Mask: 0x07) */ +#define WDT_CFG_INTVAL_Pos (16UL) /*!< WDT CFG: INTVAL (Bit 16) */ +#define WDT_CFG_INTVAL_Msk (0xff0000UL) /*!< WDT CFG: INTVAL (Bitfield-Mask: 0xff) */ +#define WDT_CFG_RESVAL_Pos (8UL) /*!< WDT CFG: RESVAL (Bit 8) */ +#define WDT_CFG_RESVAL_Msk (0xff00UL) /*!< WDT CFG: RESVAL (Bitfield-Mask: 0xff) */ +#define WDT_CFG_RESEN_Pos (2UL) /*!< WDT CFG: RESEN (Bit 2) */ +#define WDT_CFG_RESEN_Msk (0x4UL) /*!< WDT CFG: RESEN (Bitfield-Mask: 0x01) */ +#define WDT_CFG_INTEN_Pos (1UL) /*!< WDT CFG: INTEN (Bit 1) */ +#define WDT_CFG_INTEN_Msk (0x2UL) /*!< WDT CFG: INTEN (Bitfield-Mask: 0x01) */ +#define WDT_CFG_WDTEN_Pos (0UL) /*!< WDT CFG: WDTEN (Bit 0) */ +#define WDT_CFG_WDTEN_Msk (0x1UL) /*!< WDT CFG: WDTEN (Bitfield-Mask: 0x01) */ +/* ========================================================= RSTRT ========================================================= */ +#define WDT_RSTRT_RSTRT_Pos (0UL) /*!< WDT RSTRT: RSTRT (Bit 0) */ +#define WDT_RSTRT_RSTRT_Msk (0xffUL) /*!< WDT RSTRT: RSTRT (Bitfield-Mask: 0xff) */ +/* ========================================================= LOCK ========================================================== */ +#define WDT_LOCK_LOCK_Pos (0UL) /*!< WDT LOCK: LOCK (Bit 0) */ +#define WDT_LOCK_LOCK_Msk (0xffUL) /*!< WDT LOCK: LOCK (Bitfield-Mask: 0xff) */ +/* ========================================================= COUNT ========================================================= */ +#define WDT_COUNT_COUNT_Pos (0UL) /*!< WDT COUNT: COUNT (Bit 0) */ +#define WDT_COUNT_COUNT_Msk (0xffUL) /*!< WDT COUNT: COUNT (Bitfield-Mask: 0xff) */ +/* ========================================================= INTEN ========================================================= */ +#define WDT_INTEN_WDTINT_Pos (0UL) /*!< WDT INTEN: WDTINT (Bit 0) */ +#define WDT_INTEN_WDTINT_Msk (0x1UL) /*!< WDT INTEN: WDTINT (Bitfield-Mask: 0x01) */ +/* ======================================================== INTSTAT ======================================================== */ +#define WDT_INTSTAT_WDTINT_Pos (0UL) /*!< WDT INTSTAT: WDTINT (Bit 0) */ +#define WDT_INTSTAT_WDTINT_Msk (0x1UL) /*!< WDT INTSTAT: WDTINT (Bitfield-Mask: 0x01) */ +/* ======================================================== INTCLR ========================================================= */ +#define WDT_INTCLR_WDTINT_Pos (0UL) /*!< WDT INTCLR: WDTINT (Bit 0) */ +#define WDT_INTCLR_WDTINT_Msk (0x1UL) /*!< WDT INTCLR: WDTINT (Bitfield-Mask: 0x01) */ +/* ======================================================== INTSET ========================================================= */ +#define WDT_INTSET_WDTINT_Pos (0UL) /*!< WDT INTSET: WDTINT (Bit 0) */ +#define WDT_INTSET_WDTINT_Msk (0x1UL) /*!< WDT INTSET: WDTINT (Bitfield-Mask: 0x01) */ + +/** @} */ /* End of group PosMask_peripherals */ + + +/* =========================================================================================================================== */ +/* ================ Enumerated Values Peripheral Section ================ */ +/* =========================================================================================================================== */ + + +/** @addtogroup EnumValue_peripherals + * @{ + */ + + + +/* =========================================================================================================================== */ +/* ================ ADC ================ */ +/* =========================================================================================================================== */ + +/* ========================================================== CFG ========================================================== */ +/* ================================================ ADC CFG CLKSEL [24..25] ================================================ */ +typedef enum { /*!< ADC_CFG_CLKSEL */ + ADC_CFG_CLKSEL_OFF = 0, /*!< OFF : Off mode. The HFRC or HFRC_DIV2 clock must be selected + for the ADC to function. The ADC controller + automatically shuts off the clock in it's + low power modes. When setting ADCEN to '0', + the CLKSEL should remain set to one of the + two clock selects for proper power down + sequencing. value. */ + ADC_CFG_CLKSEL_HFRC = 1, /*!< HFRC : HFRC Core Clock divided by (CORESEL+1) value. */ + ADC_CFG_CLKSEL_HFRC_DIV2 = 2, /*!< HFRC_DIV2 : HFRC Core Clock / 2 further divided by (CORESEL+1) + value. */ +} ADC_CFG_CLKSEL_Enum; + +/* =============================================== ADC CFG TRIGPOL [19..19] ================================================ */ +typedef enum { /*!< ADC_CFG_TRIGPOL */ + ADC_CFG_TRIGPOL_RISING_EDGE = 0, /*!< RISING_EDGE : Trigger on rising edge. value. */ + ADC_CFG_TRIGPOL_FALLING_EDGE = 1, /*!< FALLING_EDGE : Trigger on falling edge. value. */ +} ADC_CFG_TRIGPOL_Enum; + +/* =============================================== ADC CFG TRIGSEL [16..18] ================================================ */ +typedef enum { /*!< ADC_CFG_TRIGSEL */ + ADC_CFG_TRIGSEL_EXT0 = 0, /*!< EXT0 : Off chip External Trigger0 (ADC_ET0) value. */ + ADC_CFG_TRIGSEL_EXT1 = 1, /*!< EXT1 : Off chip External Trigger1 (ADC_ET1) value. */ + ADC_CFG_TRIGSEL_EXT2 = 2, /*!< EXT2 : Off chip External Trigger2 (ADC_ET2) value. */ + ADC_CFG_TRIGSEL_EXT3 = 3, /*!< EXT3 : Off chip External Trigger3 (ADC_ET3) value. */ + ADC_CFG_TRIGSEL_VCOMP = 4, /*!< VCOMP : Voltage Comparator Output value. */ + ADC_CFG_TRIGSEL_SWT = 7, /*!< SWT : Software Trigger value. */ +} ADC_CFG_TRIGSEL_Enum; + +/* ============================================== ADC CFG DFIFORDEN [12..12] =============================================== */ +typedef enum { /*!< ADC_CFG_DFIFORDEN */ + ADC_CFG_DFIFORDEN_DIS = 0, /*!< DIS : Destructive Reads are prevented. Reads to the FIFOPR register + will not POP an entry off the FIFO. value. */ + ADC_CFG_DFIFORDEN_EN = 1, /*!< EN : Reads to the FIFOPR registger will automatically pop an + entry off the FIFO. value. */ +} ADC_CFG_DFIFORDEN_Enum; + +/* ================================================= ADC CFG REFSEL [8..9] ================================================= */ +typedef enum { /*!< ADC_CFG_REFSEL */ + ADC_CFG_REFSEL_INT2P0 = 0, /*!< INT2P0 : Internal 2.0V Bandgap Reference Voltage value. */ + ADC_CFG_REFSEL_INT1P5 = 1, /*!< INT1P5 : Internal 1.5V Bandgap Reference Voltage value. */ + ADC_CFG_REFSEL_EXT2P0 = 2, /*!< EXT2P0 : Off Chip 2.0V Reference value. */ + ADC_CFG_REFSEL_EXT1P5 = 3, /*!< EXT1P5 : Off Chip 1.5V Reference value. */ +} ADC_CFG_REFSEL_Enum; + +/* ================================================= ADC CFG CKMODE [4..4] ================================================= */ +typedef enum { /*!< ADC_CFG_CKMODE */ + ADC_CFG_CKMODE_LPCKMODE = 0, /*!< LPCKMODE : Disable the clock between scans for LPMODE0. Set + LPCKMODE to 0x1 while configuring the ADC. value. */ + ADC_CFG_CKMODE_LLCKMODE = 1, /*!< LLCKMODE : Low Latency Clock Mode. When set, HFRC and the adc_clk + will remain on while in functioning in LPMODE0. value. */ +} ADC_CFG_CKMODE_Enum; + +/* ================================================= ADC CFG LPMODE [3..3] ================================================= */ +typedef enum { /*!< ADC_CFG_LPMODE */ + ADC_CFG_LPMODE_MODE0 = 0, /*!< MODE0 : Low Power Mode 0. Leaves the ADC fully powered between + scans with minimum latency between a trigger event and + sample data collection. value. */ + ADC_CFG_LPMODE_MODE1 = 1, /*!< MODE1 : Low Power Mode 1. Powers down all circuity and clocks + associated with the ADC until the next trigger event. Between + scans, the reference buffer requires up to 50us of delay + from a scan trigger event before the conversion will commence + while operating in this mode. value. */ +} ADC_CFG_LPMODE_Enum; + +/* ================================================= ADC CFG RPTEN [2..2] ================================================== */ +typedef enum { /*!< ADC_CFG_RPTEN */ + ADC_CFG_RPTEN_SINGLE_SCAN = 0, /*!< SINGLE_SCAN : In Single Scan Mode, the ADC will complete a single + scan upon each trigger event. value. */ + ADC_CFG_RPTEN_REPEATING_SCAN = 1, /*!< REPEATING_SCAN : In Repeating Scan Mode, the ADC will complete + it's first scan upon the initial trigger event and all + subsequent scans will occur at regular intervals defined + by the configuration programmed for the CTTMRA3 internal + timer until the timer is disabled or the ADC is disabled. + When disabling the ADC (setting ADCEN to '0'), the RPTEN + bit should be cleared. value. */ +} ADC_CFG_RPTEN_Enum; + +/* ================================================= ADC CFG ADCEN [0..0] ================================================== */ +typedef enum { /*!< ADC_CFG_ADCEN */ + ADC_CFG_ADCEN_DIS = 0, /*!< DIS : Disable the ADC module. value. */ + ADC_CFG_ADCEN_EN = 1, /*!< EN : Enable the ADC module. value. */ +} ADC_CFG_ADCEN_Enum; + +/* ========================================================= STAT ========================================================== */ +/* ================================================ ADC STAT PWDSTAT [0..0] ================================================ */ +typedef enum { /*!< ADC_STAT_PWDSTAT */ + ADC_STAT_PWDSTAT_ON = 0, /*!< ON : Powered on. value. */ + ADC_STAT_PWDSTAT_POWERED_DOWN = 1, /*!< POWERED_DOWN : ADC Low Power Mode 1. value. */ +} ADC_STAT_PWDSTAT_Enum; + +/* ========================================================== SWT ========================================================== */ +/* ================================================== ADC SWT SWT [0..7] =================================================== */ +typedef enum { /*!< ADC_SWT_SWT */ + ADC_SWT_SWT_GEN_SW_TRIGGER = 55, /*!< GEN_SW_TRIGGER : Writing this value generates a software trigger. + value. */ +} ADC_SWT_SWT_Enum; + +/* ======================================================== SL0CFG ========================================================= */ +/* ============================================== ADC SL0CFG ADSEL0 [24..26] =============================================== */ +typedef enum { /*!< ADC_SL0CFG_ADSEL0 */ + ADC_SL0CFG_ADSEL0_AVG_1_MSRMT = 0, /*!< AVG_1_MSRMT : Average in 1 measurement in the accumulate divide + module for this slot. value. */ + ADC_SL0CFG_ADSEL0_AVG_2_MSRMTS = 1, /*!< AVG_2_MSRMTS : Average in 2 measurements in the accumulate divide + module for this slot. value. */ + ADC_SL0CFG_ADSEL0_AVG_4_MSRMTS = 2, /*!< AVG_4_MSRMTS : Average in 4 measurements in the accumulate divide + module for this slot. value. */ + ADC_SL0CFG_ADSEL0_AVG_8_MSRMT = 3, /*!< AVG_8_MSRMT : Average in 8 measurements in the accumulate divide + module for this slot. value. */ + ADC_SL0CFG_ADSEL0_AVG_16_MSRMTS = 4, /*!< AVG_16_MSRMTS : Average in 16 measurements in the accumulate + divide module for this slot. value. */ + ADC_SL0CFG_ADSEL0_AVG_32_MSRMTS = 5, /*!< AVG_32_MSRMTS : Average in 32 measurements in the accumulate + divide module for this slot. value. */ + ADC_SL0CFG_ADSEL0_AVG_64_MSRMTS = 6, /*!< AVG_64_MSRMTS : Average in 64 measurements in the accumulate + divide module for this slot. value. */ + ADC_SL0CFG_ADSEL0_AVG_128_MSRMTS = 7, /*!< AVG_128_MSRMTS : Average in 128 measurements in the accumulate + divide module for this slot. value. */ +} ADC_SL0CFG_ADSEL0_Enum; + +/* ============================================== ADC SL0CFG PRMODE0 [16..17] ============================================== */ +typedef enum { /*!< ADC_SL0CFG_PRMODE0 */ + ADC_SL0CFG_PRMODE0_P14B = 0, /*!< P14B : 14-bit precision mode value. */ + ADC_SL0CFG_PRMODE0_P12B = 1, /*!< P12B : 12-bit precision mode value. */ + ADC_SL0CFG_PRMODE0_P10B = 2, /*!< P10B : 10-bit precision mode value. */ + ADC_SL0CFG_PRMODE0_P8B = 3, /*!< P8B : 8-bit precision mode value. */ +} ADC_SL0CFG_PRMODE0_Enum; + +/* =============================================== ADC SL0CFG CHSEL0 [8..11] =============================================== */ +typedef enum { /*!< ADC_SL0CFG_CHSEL0 */ + ADC_SL0CFG_CHSEL0_SE0 = 0, /*!< SE0 : single ended external GPIO connection to pad16. value. */ + ADC_SL0CFG_CHSEL0_SE1 = 1, /*!< SE1 : single ended external GPIO connection to pad29. value. */ + ADC_SL0CFG_CHSEL0_SE2 = 2, /*!< SE2 : single ended external GPIO connection to pad11. value. */ + ADC_SL0CFG_CHSEL0_SE3 = 3, /*!< SE3 : single ended external GPIO connection to pad31. value. */ + ADC_SL0CFG_CHSEL0_SE4 = 4, /*!< SE4 : single ended external GPIO connection to pad32. value. */ + ADC_SL0CFG_CHSEL0_SE5 = 5, /*!< SE5 : single ended external GPIO connection to pad33. value. */ + ADC_SL0CFG_CHSEL0_SE6 = 6, /*!< SE6 : single ended external GPIO connection to pad34. value. */ + ADC_SL0CFG_CHSEL0_SE7 = 7, /*!< SE7 : single ended external GPIO connection to pad35. value. */ + ADC_SL0CFG_CHSEL0_SE8 = 8, /*!< SE8 : single ended external GPIO connection to pad13. value. */ + ADC_SL0CFG_CHSEL0_SE9 = 9, /*!< SE9 : single ended external GPIO connection to pad12. value. */ + ADC_SL0CFG_CHSEL0_DF0 = 10, /*!< DF0 : differential external GPIO connections to pad12(N) and + pad13(P). value. */ + ADC_SL0CFG_CHSEL0_DF1 = 11, /*!< DF1 : differential external GPIO connections to pad15(N) and + pad14(P). value. */ + ADC_SL0CFG_CHSEL0_TEMP = 12, /*!< TEMP : internal temperature sensor. value. */ + ADC_SL0CFG_CHSEL0_BATT = 13, /*!< BATT : internal voltage divide-by-3 connection. value. */ + ADC_SL0CFG_CHSEL0_VSS = 14, /*!< VSS : Input VSS value. */ +} ADC_SL0CFG_CHSEL0_Enum; + +/* ================================================ ADC SL0CFG WCEN0 [1..1] ================================================ */ +typedef enum { /*!< ADC_SL0CFG_WCEN0 */ + ADC_SL0CFG_WCEN0_WCEN = 1, /*!< WCEN : Enable the window compare for slot 0. value. */ +} ADC_SL0CFG_WCEN0_Enum; + +/* ================================================ ADC SL0CFG SLEN0 [0..0] ================================================ */ +typedef enum { /*!< ADC_SL0CFG_SLEN0 */ + ADC_SL0CFG_SLEN0_SLEN = 1, /*!< SLEN : Enable slot 0 for ADC conversions. value. */ +} ADC_SL0CFG_SLEN0_Enum; + +/* ======================================================== SL1CFG ========================================================= */ +/* ============================================== ADC SL1CFG ADSEL1 [24..26] =============================================== */ +typedef enum { /*!< ADC_SL1CFG_ADSEL1 */ + ADC_SL1CFG_ADSEL1_AVG_1_MSRMT = 0, /*!< AVG_1_MSRMT : Average in 1 measurement in the accumulate divide + module for this slot. value. */ + ADC_SL1CFG_ADSEL1_AVG_2_MSRMTS = 1, /*!< AVG_2_MSRMTS : Average in 2 measurements in the accumulate divide + module for this slot. value. */ + ADC_SL1CFG_ADSEL1_AVG_4_MSRMTS = 2, /*!< AVG_4_MSRMTS : Average in 4 measurements in the accumulate divide + module for this slot. value. */ + ADC_SL1CFG_ADSEL1_AVG_8_MSRMT = 3, /*!< AVG_8_MSRMT : Average in 8 measurements in the accumulate divide + module for this slot. value. */ + ADC_SL1CFG_ADSEL1_AVG_16_MSRMTS = 4, /*!< AVG_16_MSRMTS : Average in 16 measurements in the accumulate + divide module for this slot. value. */ + ADC_SL1CFG_ADSEL1_AVG_32_MSRMTS = 5, /*!< AVG_32_MSRMTS : Average in 32 measurements in the accumulate + divide module for this slot. value. */ + ADC_SL1CFG_ADSEL1_AVG_64_MSRMTS = 6, /*!< AVG_64_MSRMTS : Average in 64 measurements in the accumulate + divide module for this slot. value. */ + ADC_SL1CFG_ADSEL1_AVG_128_MSRMTS = 7, /*!< AVG_128_MSRMTS : Average in 128 measurements in the accumulate + divide module for this slot. value. */ +} ADC_SL1CFG_ADSEL1_Enum; + +/* ============================================== ADC SL1CFG PRMODE1 [16..17] ============================================== */ +typedef enum { /*!< ADC_SL1CFG_PRMODE1 */ + ADC_SL1CFG_PRMODE1_P14B = 0, /*!< P14B : 14-bit precision mode value. */ + ADC_SL1CFG_PRMODE1_P12B = 1, /*!< P12B : 12-bit precision mode value. */ + ADC_SL1CFG_PRMODE1_P10B = 2, /*!< P10B : 10-bit precision mode value. */ + ADC_SL1CFG_PRMODE1_P8B = 3, /*!< P8B : 8-bit precision mode value. */ +} ADC_SL1CFG_PRMODE1_Enum; + +/* =============================================== ADC SL1CFG CHSEL1 [8..11] =============================================== */ +typedef enum { /*!< ADC_SL1CFG_CHSEL1 */ + ADC_SL1CFG_CHSEL1_SE0 = 0, /*!< SE0 : single ended external GPIO connection to pad16. value. */ + ADC_SL1CFG_CHSEL1_SE1 = 1, /*!< SE1 : single ended external GPIO connection to pad29. value. */ + ADC_SL1CFG_CHSEL1_SE2 = 2, /*!< SE2 : single ended external GPIO connection to pad11. value. */ + ADC_SL1CFG_CHSEL1_SE3 = 3, /*!< SE3 : single ended external GPIO connection to pad31. value. */ + ADC_SL1CFG_CHSEL1_SE4 = 4, /*!< SE4 : single ended external GPIO connection to pad32. value. */ + ADC_SL1CFG_CHSEL1_SE5 = 5, /*!< SE5 : single ended external GPIO connection to pad33. value. */ + ADC_SL1CFG_CHSEL1_SE6 = 6, /*!< SE6 : single ended external GPIO connection to pad34. value. */ + ADC_SL1CFG_CHSEL1_SE7 = 7, /*!< SE7 : single ended external GPIO connection to pad35. value. */ + ADC_SL1CFG_CHSEL1_SE8 = 8, /*!< SE8 : single ended external GPIO connection to pad13. value. */ + ADC_SL1CFG_CHSEL1_SE9 = 9, /*!< SE9 : single ended external GPIO connection to pad12. value. */ + ADC_SL1CFG_CHSEL1_DF0 = 10, /*!< DF0 : differential external GPIO connections to pad12(N) and + pad13(P). value. */ + ADC_SL1CFG_CHSEL1_DF1 = 11, /*!< DF1 : differential external GPIO connections to pad15(N) and + pad14(P). value. */ + ADC_SL1CFG_CHSEL1_TEMP = 12, /*!< TEMP : internal temperature sensor. value. */ + ADC_SL1CFG_CHSEL1_BATT = 13, /*!< BATT : internal voltage divide-by-3 connection. value. */ + ADC_SL1CFG_CHSEL1_VSS = 14, /*!< VSS : Input VSS value. */ +} ADC_SL1CFG_CHSEL1_Enum; + +/* ================================================ ADC SL1CFG WCEN1 [1..1] ================================================ */ +typedef enum { /*!< ADC_SL1CFG_WCEN1 */ + ADC_SL1CFG_WCEN1_WCEN = 1, /*!< WCEN : Enable the window compare for slot 1. value. */ +} ADC_SL1CFG_WCEN1_Enum; + +/* ================================================ ADC SL1CFG SLEN1 [0..0] ================================================ */ +typedef enum { /*!< ADC_SL1CFG_SLEN1 */ + ADC_SL1CFG_SLEN1_SLEN = 1, /*!< SLEN : Enable slot 1 for ADC conversions. value. */ +} ADC_SL1CFG_SLEN1_Enum; + +/* ======================================================== SL2CFG ========================================================= */ +/* ============================================== ADC SL2CFG ADSEL2 [24..26] =============================================== */ +typedef enum { /*!< ADC_SL2CFG_ADSEL2 */ + ADC_SL2CFG_ADSEL2_AVG_1_MSRMT = 0, /*!< AVG_1_MSRMT : Average in 1 measurement in the accumulate divide + module for this slot. value. */ + ADC_SL2CFG_ADSEL2_AVG_2_MSRMTS = 1, /*!< AVG_2_MSRMTS : Average in 2 measurements in the accumulate divide + module for this slot. value. */ + ADC_SL2CFG_ADSEL2_AVG_4_MSRMTS = 2, /*!< AVG_4_MSRMTS : Average in 4 measurements in the accumulate divide + module for this slot. value. */ + ADC_SL2CFG_ADSEL2_AVG_8_MSRMT = 3, /*!< AVG_8_MSRMT : Average in 8 measurements in the accumulate divide + module for this slot. value. */ + ADC_SL2CFG_ADSEL2_AVG_16_MSRMTS = 4, /*!< AVG_16_MSRMTS : Average in 16 measurements in the accumulate + divide module for this slot. value. */ + ADC_SL2CFG_ADSEL2_AVG_32_MSRMTS = 5, /*!< AVG_32_MSRMTS : Average in 32 measurements in the accumulate + divide module for this slot. value. */ + ADC_SL2CFG_ADSEL2_AVG_64_MSRMTS = 6, /*!< AVG_64_MSRMTS : Average in 64 measurements in the accumulate + divide module for this slot. value. */ + ADC_SL2CFG_ADSEL2_AVG_128_MSRMTS = 7, /*!< AVG_128_MSRMTS : Average in 128 measurements in the accumulate + divide module for this slot. value. */ +} ADC_SL2CFG_ADSEL2_Enum; + +/* ============================================== ADC SL2CFG PRMODE2 [16..17] ============================================== */ +typedef enum { /*!< ADC_SL2CFG_PRMODE2 */ + ADC_SL2CFG_PRMODE2_P14B = 0, /*!< P14B : 14-bit precision mode value. */ + ADC_SL2CFG_PRMODE2_P12B = 1, /*!< P12B : 12-bit precision mode value. */ + ADC_SL2CFG_PRMODE2_P10B = 2, /*!< P10B : 10-bit precision mode value. */ + ADC_SL2CFG_PRMODE2_P8B = 3, /*!< P8B : 8-bit precision mode value. */ +} ADC_SL2CFG_PRMODE2_Enum; + +/* =============================================== ADC SL2CFG CHSEL2 [8..11] =============================================== */ +typedef enum { /*!< ADC_SL2CFG_CHSEL2 */ + ADC_SL2CFG_CHSEL2_SE0 = 0, /*!< SE0 : single ended external GPIO connection to pad16. value. */ + ADC_SL2CFG_CHSEL2_SE1 = 1, /*!< SE1 : single ended external GPIO connection to pad29. value. */ + ADC_SL2CFG_CHSEL2_SE2 = 2, /*!< SE2 : single ended external GPIO connection to pad11. value. */ + ADC_SL2CFG_CHSEL2_SE3 = 3, /*!< SE3 : single ended external GPIO connection to pad31. value. */ + ADC_SL2CFG_CHSEL2_SE4 = 4, /*!< SE4 : single ended external GPIO connection to pad32. value. */ + ADC_SL2CFG_CHSEL2_SE5 = 5, /*!< SE5 : single ended external GPIO connection to pad33. value. */ + ADC_SL2CFG_CHSEL2_SE6 = 6, /*!< SE6 : single ended external GPIO connection to pad34. value. */ + ADC_SL2CFG_CHSEL2_SE7 = 7, /*!< SE7 : single ended external GPIO connection to pad35. value. */ + ADC_SL2CFG_CHSEL2_SE8 = 8, /*!< SE8 : single ended external GPIO connection to pad13. value. */ + ADC_SL2CFG_CHSEL2_SE9 = 9, /*!< SE9 : single ended external GPIO connection to pad12. value. */ + ADC_SL2CFG_CHSEL2_DF0 = 10, /*!< DF0 : differential external GPIO connections to pad12(N) and + pad13(P). value. */ + ADC_SL2CFG_CHSEL2_DF1 = 11, /*!< DF1 : differential external GPIO connections to pad15(N) and + pad14(P). value. */ + ADC_SL2CFG_CHSEL2_TEMP = 12, /*!< TEMP : internal temperature sensor. value. */ + ADC_SL2CFG_CHSEL2_BATT = 13, /*!< BATT : internal voltage divide-by-3 connection. value. */ + ADC_SL2CFG_CHSEL2_VSS = 14, /*!< VSS : Input VSS value. */ +} ADC_SL2CFG_CHSEL2_Enum; + +/* ================================================ ADC SL2CFG WCEN2 [1..1] ================================================ */ +typedef enum { /*!< ADC_SL2CFG_WCEN2 */ + ADC_SL2CFG_WCEN2_WCEN = 1, /*!< WCEN : Enable the window compare for slot 2. value. */ +} ADC_SL2CFG_WCEN2_Enum; + +/* ================================================ ADC SL2CFG SLEN2 [0..0] ================================================ */ +typedef enum { /*!< ADC_SL2CFG_SLEN2 */ + ADC_SL2CFG_SLEN2_SLEN = 1, /*!< SLEN : Enable slot 2 for ADC conversions. value. */ +} ADC_SL2CFG_SLEN2_Enum; + +/* ======================================================== SL3CFG ========================================================= */ +/* ============================================== ADC SL3CFG ADSEL3 [24..26] =============================================== */ +typedef enum { /*!< ADC_SL3CFG_ADSEL3 */ + ADC_SL3CFG_ADSEL3_AVG_1_MSRMT = 0, /*!< AVG_1_MSRMT : Average in 1 measurement in the accumulate divide + module for this slot. value. */ + ADC_SL3CFG_ADSEL3_AVG_2_MSRMTS = 1, /*!< AVG_2_MSRMTS : Average in 2 measurements in the accumulate divide + module for this slot. value. */ + ADC_SL3CFG_ADSEL3_AVG_4_MSRMTS = 2, /*!< AVG_4_MSRMTS : Average in 4 measurements in the accumulate divide + module for this slot. value. */ + ADC_SL3CFG_ADSEL3_AVG_8_MSRMT = 3, /*!< AVG_8_MSRMT : Average in 8 measurements in the accumulate divide + module for this slot. value. */ + ADC_SL3CFG_ADSEL3_AVG_16_MSRMTS = 4, /*!< AVG_16_MSRMTS : Average in 16 measurements in the accumulate + divide module for this slot. value. */ + ADC_SL3CFG_ADSEL3_AVG_32_MSRMTS = 5, /*!< AVG_32_MSRMTS : Average in 32 measurements in the accumulate + divide module for this slot. value. */ + ADC_SL3CFG_ADSEL3_AVG_64_MSRMTS = 6, /*!< AVG_64_MSRMTS : Average in 64 measurements in the accumulate + divide module for this slot. value. */ + ADC_SL3CFG_ADSEL3_AVG_128_MSRMTS = 7, /*!< AVG_128_MSRMTS : Average in 128 measurements in the accumulate + divide module for this slot. value. */ +} ADC_SL3CFG_ADSEL3_Enum; + +/* ============================================== ADC SL3CFG PRMODE3 [16..17] ============================================== */ +typedef enum { /*!< ADC_SL3CFG_PRMODE3 */ + ADC_SL3CFG_PRMODE3_P14B = 0, /*!< P14B : 14-bit precision mode value. */ + ADC_SL3CFG_PRMODE3_P12B = 1, /*!< P12B : 12-bit precision mode value. */ + ADC_SL3CFG_PRMODE3_P10B = 2, /*!< P10B : 10-bit precision mode value. */ + ADC_SL3CFG_PRMODE3_P8B = 3, /*!< P8B : 8-bit precision mode value. */ +} ADC_SL3CFG_PRMODE3_Enum; + +/* =============================================== ADC SL3CFG CHSEL3 [8..11] =============================================== */ +typedef enum { /*!< ADC_SL3CFG_CHSEL3 */ + ADC_SL3CFG_CHSEL3_SE0 = 0, /*!< SE0 : single ended external GPIO connection to pad16. value. */ + ADC_SL3CFG_CHSEL3_SE1 = 1, /*!< SE1 : single ended external GPIO connection to pad29. value. */ + ADC_SL3CFG_CHSEL3_SE2 = 2, /*!< SE2 : single ended external GPIO connection to pad11. value. */ + ADC_SL3CFG_CHSEL3_SE3 = 3, /*!< SE3 : single ended external GPIO connection to pad31. value. */ + ADC_SL3CFG_CHSEL3_SE4 = 4, /*!< SE4 : single ended external GPIO connection to pad32. value. */ + ADC_SL3CFG_CHSEL3_SE5 = 5, /*!< SE5 : single ended external GPIO connection to pad33. value. */ + ADC_SL3CFG_CHSEL3_SE6 = 6, /*!< SE6 : single ended external GPIO connection to pad34. value. */ + ADC_SL3CFG_CHSEL3_SE7 = 7, /*!< SE7 : single ended external GPIO connection to pad35. value. */ + ADC_SL3CFG_CHSEL3_SE8 = 8, /*!< SE8 : single ended external GPIO connection to pad13. value. */ + ADC_SL3CFG_CHSEL3_SE9 = 9, /*!< SE9 : single ended external GPIO connection to pad12. value. */ + ADC_SL3CFG_CHSEL3_DF0 = 10, /*!< DF0 : differential external GPIO connections to pad12(N) and + pad13(P). value. */ + ADC_SL3CFG_CHSEL3_DF1 = 11, /*!< DF1 : differential external GPIO connections to pad15(N) and + pad14(P). value. */ + ADC_SL3CFG_CHSEL3_TEMP = 12, /*!< TEMP : internal temperature sensor. value. */ + ADC_SL3CFG_CHSEL3_BATT = 13, /*!< BATT : internal voltage divide-by-3 connection. value. */ + ADC_SL3CFG_CHSEL3_VSS = 14, /*!< VSS : Input VSS value. */ +} ADC_SL3CFG_CHSEL3_Enum; + +/* ================================================ ADC SL3CFG WCEN3 [1..1] ================================================ */ +typedef enum { /*!< ADC_SL3CFG_WCEN3 */ + ADC_SL3CFG_WCEN3_WCEN = 1, /*!< WCEN : Enable the window compare for slot 3. value. */ +} ADC_SL3CFG_WCEN3_Enum; + +/* ================================================ ADC SL3CFG SLEN3 [0..0] ================================================ */ +typedef enum { /*!< ADC_SL3CFG_SLEN3 */ + ADC_SL3CFG_SLEN3_SLEN = 1, /*!< SLEN : Enable slot 3 for ADC conversions. value. */ +} ADC_SL3CFG_SLEN3_Enum; + +/* ======================================================== SL4CFG ========================================================= */ +/* ============================================== ADC SL4CFG ADSEL4 [24..26] =============================================== */ +typedef enum { /*!< ADC_SL4CFG_ADSEL4 */ + ADC_SL4CFG_ADSEL4_AVG_1_MSRMT = 0, /*!< AVG_1_MSRMT : Average in 1 measurement in the accumulate divide + module for this slot. value. */ + ADC_SL4CFG_ADSEL4_AVG_2_MSRMTS = 1, /*!< AVG_2_MSRMTS : Average in 2 measurements in the accumulate divide + module for this slot. value. */ + ADC_SL4CFG_ADSEL4_AVG_4_MSRMTS = 2, /*!< AVG_4_MSRMTS : Average in 4 measurements in the accumulate divide + module for this slot. value. */ + ADC_SL4CFG_ADSEL4_AVG_8_MSRMT = 3, /*!< AVG_8_MSRMT : Average in 8 measurements in the accumulate divide + module for this slot. value. */ + ADC_SL4CFG_ADSEL4_AVG_16_MSRMTS = 4, /*!< AVG_16_MSRMTS : Average in 16 measurements in the accumulate + divide module for this slot. value. */ + ADC_SL4CFG_ADSEL4_AVG_32_MSRMTS = 5, /*!< AVG_32_MSRMTS : Average in 32 measurements in the accumulate + divide module for this slot. value. */ + ADC_SL4CFG_ADSEL4_AVG_64_MSRMTS = 6, /*!< AVG_64_MSRMTS : Average in 64 measurements in the accumulate + divide module for this slot. value. */ + ADC_SL4CFG_ADSEL4_AVG_128_MSRMTS = 7, /*!< AVG_128_MSRMTS : Average in 128 measurements in the accumulate + divide module for this slot. value. */ +} ADC_SL4CFG_ADSEL4_Enum; + +/* ============================================== ADC SL4CFG PRMODE4 [16..17] ============================================== */ +typedef enum { /*!< ADC_SL4CFG_PRMODE4 */ + ADC_SL4CFG_PRMODE4_P14B = 0, /*!< P14B : 14-bit precision mode value. */ + ADC_SL4CFG_PRMODE4_P12B = 1, /*!< P12B : 12-bit precision mode value. */ + ADC_SL4CFG_PRMODE4_P10B = 2, /*!< P10B : 10-bit precision mode value. */ + ADC_SL4CFG_PRMODE4_P8B = 3, /*!< P8B : 8-bit precision mode value. */ +} ADC_SL4CFG_PRMODE4_Enum; + +/* =============================================== ADC SL4CFG CHSEL4 [8..11] =============================================== */ +typedef enum { /*!< ADC_SL4CFG_CHSEL4 */ + ADC_SL4CFG_CHSEL4_SE0 = 0, /*!< SE0 : single ended external GPIO connection to pad16. value. */ + ADC_SL4CFG_CHSEL4_SE1 = 1, /*!< SE1 : single ended external GPIO connection to pad29. value. */ + ADC_SL4CFG_CHSEL4_SE2 = 2, /*!< SE2 : single ended external GPIO connection to pad11. value. */ + ADC_SL4CFG_CHSEL4_SE3 = 3, /*!< SE3 : single ended external GPIO connection to pad31. value. */ + ADC_SL4CFG_CHSEL4_SE4 = 4, /*!< SE4 : single ended external GPIO connection to pad32. value. */ + ADC_SL4CFG_CHSEL4_SE5 = 5, /*!< SE5 : single ended external GPIO connection to pad33. value. */ + ADC_SL4CFG_CHSEL4_SE6 = 6, /*!< SE6 : single ended external GPIO connection to pad34. value. */ + ADC_SL4CFG_CHSEL4_SE7 = 7, /*!< SE7 : single ended external GPIO connection to pad35. value. */ + ADC_SL4CFG_CHSEL4_SE8 = 8, /*!< SE8 : single ended external GPIO connection to pad13. value. */ + ADC_SL4CFG_CHSEL4_SE9 = 9, /*!< SE9 : single ended external GPIO connection to pad12. value. */ + ADC_SL4CFG_CHSEL4_DF0 = 10, /*!< DF0 : differential external GPIO connections to pad12(N) and + pad13(P). value. */ + ADC_SL4CFG_CHSEL4_DF1 = 11, /*!< DF1 : differential external GPIO connections to pad15(N) and + pad14(P). value. */ + ADC_SL4CFG_CHSEL4_TEMP = 12, /*!< TEMP : internal temperature sensor. value. */ + ADC_SL4CFG_CHSEL4_BATT = 13, /*!< BATT : internal voltage divide-by-3 connection. value. */ + ADC_SL4CFG_CHSEL4_VSS = 14, /*!< VSS : Input VSS value. */ +} ADC_SL4CFG_CHSEL4_Enum; + +/* ================================================ ADC SL4CFG WCEN4 [1..1] ================================================ */ +typedef enum { /*!< ADC_SL4CFG_WCEN4 */ + ADC_SL4CFG_WCEN4_WCEN = 1, /*!< WCEN : Enable the window compare for slot 4. value. */ +} ADC_SL4CFG_WCEN4_Enum; + +/* ================================================ ADC SL4CFG SLEN4 [0..0] ================================================ */ +typedef enum { /*!< ADC_SL4CFG_SLEN4 */ + ADC_SL4CFG_SLEN4_SLEN = 1, /*!< SLEN : Enable slot 4 for ADC conversions. value. */ +} ADC_SL4CFG_SLEN4_Enum; + +/* ======================================================== SL5CFG ========================================================= */ +/* ============================================== ADC SL5CFG ADSEL5 [24..26] =============================================== */ +typedef enum { /*!< ADC_SL5CFG_ADSEL5 */ + ADC_SL5CFG_ADSEL5_AVG_1_MSRMT = 0, /*!< AVG_1_MSRMT : Average in 1 measurement in the accumulate divide + module for this slot. value. */ + ADC_SL5CFG_ADSEL5_AVG_2_MSRMTS = 1, /*!< AVG_2_MSRMTS : Average in 2 measurements in the accumulate divide + module for this slot. value. */ + ADC_SL5CFG_ADSEL5_AVG_4_MSRMTS = 2, /*!< AVG_4_MSRMTS : Average in 4 measurements in the accumulate divide + module for this slot. value. */ + ADC_SL5CFG_ADSEL5_AVG_8_MSRMT = 3, /*!< AVG_8_MSRMT : Average in 8 measurements in the accumulate divide + module for this slot. value. */ + ADC_SL5CFG_ADSEL5_AVG_16_MSRMTS = 4, /*!< AVG_16_MSRMTS : Average in 16 measurements in the accumulate + divide module for this slot. value. */ + ADC_SL5CFG_ADSEL5_AVG_32_MSRMTS = 5, /*!< AVG_32_MSRMTS : Average in 32 measurements in the accumulate + divide module for this slot. value. */ + ADC_SL5CFG_ADSEL5_AVG_64_MSRMTS = 6, /*!< AVG_64_MSRMTS : Average in 64 measurements in the accumulate + divide module for this slot. value. */ + ADC_SL5CFG_ADSEL5_AVG_128_MSRMTS = 7, /*!< AVG_128_MSRMTS : Average in 128 measurements in the accumulate + divide module for this slot. value. */ +} ADC_SL5CFG_ADSEL5_Enum; + +/* ============================================== ADC SL5CFG PRMODE5 [16..17] ============================================== */ +typedef enum { /*!< ADC_SL5CFG_PRMODE5 */ + ADC_SL5CFG_PRMODE5_P14B = 0, /*!< P14B : 14-bit precision mode value. */ + ADC_SL5CFG_PRMODE5_P12B = 1, /*!< P12B : 12-bit precision mode value. */ + ADC_SL5CFG_PRMODE5_P10B = 2, /*!< P10B : 10-bit precision mode value. */ + ADC_SL5CFG_PRMODE5_P8B = 3, /*!< P8B : 8-bit precision mode value. */ +} ADC_SL5CFG_PRMODE5_Enum; + +/* =============================================== ADC SL5CFG CHSEL5 [8..11] =============================================== */ +typedef enum { /*!< ADC_SL5CFG_CHSEL5 */ + ADC_SL5CFG_CHSEL5_SE0 = 0, /*!< SE0 : single ended external GPIO connection to pad16. value. */ + ADC_SL5CFG_CHSEL5_SE1 = 1, /*!< SE1 : single ended external GPIO connection to pad29. value. */ + ADC_SL5CFG_CHSEL5_SE2 = 2, /*!< SE2 : single ended external GPIO connection to pad11. value. */ + ADC_SL5CFG_CHSEL5_SE3 = 3, /*!< SE3 : single ended external GPIO connection to pad31. value. */ + ADC_SL5CFG_CHSEL5_SE4 = 4, /*!< SE4 : single ended external GPIO connection to pad32. value. */ + ADC_SL5CFG_CHSEL5_SE5 = 5, /*!< SE5 : single ended external GPIO connection to pad33. value. */ + ADC_SL5CFG_CHSEL5_SE6 = 6, /*!< SE6 : single ended external GPIO connection to pad34. value. */ + ADC_SL5CFG_CHSEL5_SE7 = 7, /*!< SE7 : single ended external GPIO connection to pad35. value. */ + ADC_SL5CFG_CHSEL5_SE8 = 8, /*!< SE8 : single ended external GPIO connection to pad13. value. */ + ADC_SL5CFG_CHSEL5_SE9 = 9, /*!< SE9 : single ended external GPIO connection to pad12. value. */ + ADC_SL5CFG_CHSEL5_DF0 = 10, /*!< DF0 : differential external GPIO connections to pad12(N) and + pad13(P). value. */ + ADC_SL5CFG_CHSEL5_DF1 = 11, /*!< DF1 : differential external GPIO connections to pad15(N) and + pad14(P). value. */ + ADC_SL5CFG_CHSEL5_TEMP = 12, /*!< TEMP : internal temperature sensor. value. */ + ADC_SL5CFG_CHSEL5_BATT = 13, /*!< BATT : internal voltage divide-by-3 connection. value. */ + ADC_SL5CFG_CHSEL5_VSS = 14, /*!< VSS : Input VSS value. */ +} ADC_SL5CFG_CHSEL5_Enum; + +/* ================================================ ADC SL5CFG WCEN5 [1..1] ================================================ */ +typedef enum { /*!< ADC_SL5CFG_WCEN5 */ + ADC_SL5CFG_WCEN5_WCEN = 1, /*!< WCEN : Enable the window compare for slot 5. value. */ +} ADC_SL5CFG_WCEN5_Enum; + +/* ================================================ ADC SL5CFG SLEN5 [0..0] ================================================ */ +typedef enum { /*!< ADC_SL5CFG_SLEN5 */ + ADC_SL5CFG_SLEN5_SLEN = 1, /*!< SLEN : Enable slot 5 for ADC conversions. value. */ +} ADC_SL5CFG_SLEN5_Enum; + +/* ======================================================== SL6CFG ========================================================= */ +/* ============================================== ADC SL6CFG ADSEL6 [24..26] =============================================== */ +typedef enum { /*!< ADC_SL6CFG_ADSEL6 */ + ADC_SL6CFG_ADSEL6_AVG_1_MSRMT = 0, /*!< AVG_1_MSRMT : Average in 1 measurement in the accumulate divide + module for this slot. value. */ + ADC_SL6CFG_ADSEL6_AVG_2_MSRMTS = 1, /*!< AVG_2_MSRMTS : Average in 2 measurements in the accumulate divide + module for this slot. value. */ + ADC_SL6CFG_ADSEL6_AVG_4_MSRMTS = 2, /*!< AVG_4_MSRMTS : Average in 4 measurements in the accumulate divide + module for this slot. value. */ + ADC_SL6CFG_ADSEL6_AVG_8_MSRMT = 3, /*!< AVG_8_MSRMT : Average in 8 measurements in the accumulate divide + module for this slot. value. */ + ADC_SL6CFG_ADSEL6_AVG_16_MSRMTS = 4, /*!< AVG_16_MSRMTS : Average in 16 measurements in the accumulate + divide module for this slot. value. */ + ADC_SL6CFG_ADSEL6_AVG_32_MSRMTS = 5, /*!< AVG_32_MSRMTS : Average in 32 measurements in the accumulate + divide module for this slot. value. */ + ADC_SL6CFG_ADSEL6_AVG_64_MSRMTS = 6, /*!< AVG_64_MSRMTS : Average in 64 measurements in the accumulate + divide module for this slot. value. */ + ADC_SL6CFG_ADSEL6_AVG_128_MSRMTS = 7, /*!< AVG_128_MSRMTS : Average in 128 measurements in the accumulate + divide module for this slot. value. */ +} ADC_SL6CFG_ADSEL6_Enum; + +/* ============================================== ADC SL6CFG PRMODE6 [16..17] ============================================== */ +typedef enum { /*!< ADC_SL6CFG_PRMODE6 */ + ADC_SL6CFG_PRMODE6_P14B = 0, /*!< P14B : 14-bit precision mode value. */ + ADC_SL6CFG_PRMODE6_P12B = 1, /*!< P12B : 12-bit precision mode value. */ + ADC_SL6CFG_PRMODE6_P10B = 2, /*!< P10B : 10-bit precision mode value. */ + ADC_SL6CFG_PRMODE6_P8B = 3, /*!< P8B : 8-bit precision mode value. */ +} ADC_SL6CFG_PRMODE6_Enum; + +/* =============================================== ADC SL6CFG CHSEL6 [8..11] =============================================== */ +typedef enum { /*!< ADC_SL6CFG_CHSEL6 */ + ADC_SL6CFG_CHSEL6_SE0 = 0, /*!< SE0 : single ended external GPIO connection to pad16. value. */ + ADC_SL6CFG_CHSEL6_SE1 = 1, /*!< SE1 : single ended external GPIO connection to pad29. value. */ + ADC_SL6CFG_CHSEL6_SE2 = 2, /*!< SE2 : single ended external GPIO connection to pad11. value. */ + ADC_SL6CFG_CHSEL6_SE3 = 3, /*!< SE3 : single ended external GPIO connection to pad31. value. */ + ADC_SL6CFG_CHSEL6_SE4 = 4, /*!< SE4 : single ended external GPIO connection to pad32. value. */ + ADC_SL6CFG_CHSEL6_SE5 = 5, /*!< SE5 : single ended external GPIO connection to pad33. value. */ + ADC_SL6CFG_CHSEL6_SE6 = 6, /*!< SE6 : single ended external GPIO connection to pad34. value. */ + ADC_SL6CFG_CHSEL6_SE7 = 7, /*!< SE7 : single ended external GPIO connection to pad35. value. */ + ADC_SL6CFG_CHSEL6_SE8 = 8, /*!< SE8 : single ended external GPIO connection to pad13. value. */ + ADC_SL6CFG_CHSEL6_SE9 = 9, /*!< SE9 : single ended external GPIO connection to pad12. value. */ + ADC_SL6CFG_CHSEL6_DF0 = 10, /*!< DF0 : differential external GPIO connections to pad12(N) and + pad13(P). value. */ + ADC_SL6CFG_CHSEL6_DF1 = 11, /*!< DF1 : differential external GPIO connections to pad15(N) and + pad14(P). value. */ + ADC_SL6CFG_CHSEL6_TEMP = 12, /*!< TEMP : internal temperature sensor. value. */ + ADC_SL6CFG_CHSEL6_BATT = 13, /*!< BATT : internal voltage divide-by-3 connection. value. */ + ADC_SL6CFG_CHSEL6_VSS = 14, /*!< VSS : Input VSS value. */ +} ADC_SL6CFG_CHSEL6_Enum; + +/* ================================================ ADC SL6CFG WCEN6 [1..1] ================================================ */ +typedef enum { /*!< ADC_SL6CFG_WCEN6 */ + ADC_SL6CFG_WCEN6_WCEN = 1, /*!< WCEN : Enable the window compare for slot 6. value. */ +} ADC_SL6CFG_WCEN6_Enum; + +/* ================================================ ADC SL6CFG SLEN6 [0..0] ================================================ */ +typedef enum { /*!< ADC_SL6CFG_SLEN6 */ + ADC_SL6CFG_SLEN6_SLEN = 1, /*!< SLEN : Enable slot 6 for ADC conversions. value. */ +} ADC_SL6CFG_SLEN6_Enum; + +/* ======================================================== SL7CFG ========================================================= */ +/* ============================================== ADC SL7CFG ADSEL7 [24..26] =============================================== */ +typedef enum { /*!< ADC_SL7CFG_ADSEL7 */ + ADC_SL7CFG_ADSEL7_AVG_1_MSRMT = 0, /*!< AVG_1_MSRMT : Average in 1 measurement in the accumulate divide + module for this slot. value. */ + ADC_SL7CFG_ADSEL7_AVG_2_MSRMTS = 1, /*!< AVG_2_MSRMTS : Average in 2 measurements in the accumulate divide + module for this slot. value. */ + ADC_SL7CFG_ADSEL7_AVG_4_MSRMTS = 2, /*!< AVG_4_MSRMTS : Average in 4 measurements in the accumulate divide + module for this slot. value. */ + ADC_SL7CFG_ADSEL7_AVG_8_MSRMT = 3, /*!< AVG_8_MSRMT : Average in 8 measurements in the accumulate divide + module for this slot. value. */ + ADC_SL7CFG_ADSEL7_AVG_16_MSRMTS = 4, /*!< AVG_16_MSRMTS : Average in 16 measurements in the accumulate + divide module for this slot. value. */ + ADC_SL7CFG_ADSEL7_AVG_32_MSRMTS = 5, /*!< AVG_32_MSRMTS : Average in 32 measurements in the accumulate + divide module for this slot. value. */ + ADC_SL7CFG_ADSEL7_AVG_64_MSRMTS = 6, /*!< AVG_64_MSRMTS : Average in 64 measurements in the accumulate + divide module for this slot. value. */ + ADC_SL7CFG_ADSEL7_AVG_128_MSRMTS = 7, /*!< AVG_128_MSRMTS : Average in 128 measurements in the accumulate + divide module for this slot. value. */ +} ADC_SL7CFG_ADSEL7_Enum; + +/* ============================================== ADC SL7CFG PRMODE7 [16..17] ============================================== */ +typedef enum { /*!< ADC_SL7CFG_PRMODE7 */ + ADC_SL7CFG_PRMODE7_P14B = 0, /*!< P14B : 14-bit precision mode value. */ + ADC_SL7CFG_PRMODE7_P12B = 1, /*!< P12B : 12-bit precision mode value. */ + ADC_SL7CFG_PRMODE7_P10B = 2, /*!< P10B : 10-bit precision mode value. */ + ADC_SL7CFG_PRMODE7_P8B = 3, /*!< P8B : 8-bit precision mode value. */ +} ADC_SL7CFG_PRMODE7_Enum; + +/* =============================================== ADC SL7CFG CHSEL7 [8..11] =============================================== */ +typedef enum { /*!< ADC_SL7CFG_CHSEL7 */ + ADC_SL7CFG_CHSEL7_SE0 = 0, /*!< SE0 : single ended external GPIO connection to pad16. value. */ + ADC_SL7CFG_CHSEL7_SE1 = 1, /*!< SE1 : single ended external GPIO connection to pad29. value. */ + ADC_SL7CFG_CHSEL7_SE2 = 2, /*!< SE2 : single ended external GPIO connection to pad11. value. */ + ADC_SL7CFG_CHSEL7_SE3 = 3, /*!< SE3 : single ended external GPIO connection to pad31. value. */ + ADC_SL7CFG_CHSEL7_SE4 = 4, /*!< SE4 : single ended external GPIO connection to pad32. value. */ + ADC_SL7CFG_CHSEL7_SE5 = 5, /*!< SE5 : single ended external GPIO connection to pad33. value. */ + ADC_SL7CFG_CHSEL7_SE6 = 6, /*!< SE6 : single ended external GPIO connection to pad34. value. */ + ADC_SL7CFG_CHSEL7_SE7 = 7, /*!< SE7 : single ended external GPIO connection to pad35. value. */ + ADC_SL7CFG_CHSEL7_SE8 = 8, /*!< SE8 : single ended external GPIO connection to pad13. value. */ + ADC_SL7CFG_CHSEL7_SE9 = 9, /*!< SE9 : single ended external GPIO connection to pad12. value. */ + ADC_SL7CFG_CHSEL7_DF0 = 10, /*!< DF0 : differential external GPIO connections to pad12(N) and + pad13(P). value. */ + ADC_SL7CFG_CHSEL7_DF1 = 11, /*!< DF1 : differential external GPIO connections to pad15(N) and + pad14(P). value. */ + ADC_SL7CFG_CHSEL7_TEMP = 12, /*!< TEMP : internal temperature sensor. value. */ + ADC_SL7CFG_CHSEL7_BATT = 13, /*!< BATT : internal voltage divide-by-3 connection. value. */ + ADC_SL7CFG_CHSEL7_VSS = 14, /*!< VSS : Input VSS value. */ +} ADC_SL7CFG_CHSEL7_Enum; + +/* ================================================ ADC SL7CFG WCEN7 [1..1] ================================================ */ +typedef enum { /*!< ADC_SL7CFG_WCEN7 */ + ADC_SL7CFG_WCEN7_WCEN = 1, /*!< WCEN : Enable the window compare for slot 7. value. */ +} ADC_SL7CFG_WCEN7_Enum; + +/* ================================================ ADC SL7CFG SLEN7 [0..0] ================================================ */ +typedef enum { /*!< ADC_SL7CFG_SLEN7 */ + ADC_SL7CFG_SLEN7_SLEN = 1, /*!< SLEN : Enable slot 7 for ADC conversions. value. */ +} ADC_SL7CFG_SLEN7_Enum; + +/* ========================================================= WULIM ========================================================= */ +/* ========================================================= WLLIM ========================================================= */ +/* ======================================================== SCWLIM ========================================================= */ +/* ========================================================= FIFO ========================================================== */ +/* ======================================================== FIFOPR ========================================================= */ +/* ========================================================= INTEN ========================================================= */ +/* ================================================= ADC INTEN DERR [7..7] ================================================= */ +typedef enum { /*!< ADC_INTEN_DERR */ + ADC_INTEN_DERR_DMAERROR = 1, /*!< DMAERROR : DMA Error Condition Occurred value. */ +} ADC_INTEN_DERR_Enum; + +/* ================================================= ADC INTEN DCMP [6..6] ================================================= */ +typedef enum { /*!< ADC_INTEN_DCMP */ + ADC_INTEN_DCMP_DMACOMPLETE = 1, /*!< DMACOMPLETE : DMA Completed a transfer value. */ +} ADC_INTEN_DCMP_Enum; + +/* ================================================ ADC INTEN WCINC [5..5] ================================================= */ +typedef enum { /*!< ADC_INTEN_WCINC */ + ADC_INTEN_WCINC_WCINCINT = 1, /*!< WCINCINT : Window comparitor voltage incursion interrupt. value. */ +} ADC_INTEN_WCINC_Enum; + +/* ================================================ ADC INTEN WCEXC [4..4] ================================================= */ +typedef enum { /*!< ADC_INTEN_WCEXC */ + ADC_INTEN_WCEXC_WCEXCINT = 1, /*!< WCEXCINT : Window comparitor voltage excursion interrupt. value. */ +} ADC_INTEN_WCEXC_Enum; + +/* =============================================== ADC INTEN FIFOOVR2 [3..3] =============================================== */ +typedef enum { /*!< ADC_INTEN_FIFOOVR2 */ + ADC_INTEN_FIFOOVR2_FIFOFULLINT = 1, /*!< FIFOFULLINT : FIFO 100 percent full interrupt. value. */ +} ADC_INTEN_FIFOOVR2_Enum; + +/* =============================================== ADC INTEN FIFOOVR1 [2..2] =============================================== */ +typedef enum { /*!< ADC_INTEN_FIFOOVR1 */ + ADC_INTEN_FIFOOVR1_FIFO75INT = 1, /*!< FIFO75INT : FIFO 75 percent full interrupt. value. */ +} ADC_INTEN_FIFOOVR1_Enum; + +/* ================================================ ADC INTEN SCNCMP [1..1] ================================================ */ +typedef enum { /*!< ADC_INTEN_SCNCMP */ + ADC_INTEN_SCNCMP_SCNCMPINT = 1, /*!< SCNCMPINT : ADC scan complete interrupt. value. */ +} ADC_INTEN_SCNCMP_Enum; + +/* ================================================ ADC INTEN CNVCMP [0..0] ================================================ */ +typedef enum { /*!< ADC_INTEN_CNVCMP */ + ADC_INTEN_CNVCMP_CNVCMPINT = 1, /*!< CNVCMPINT : ADC conversion complete interrupt. value. */ +} ADC_INTEN_CNVCMP_Enum; + +/* ======================================================== INTSTAT ======================================================== */ +/* ================================================ ADC INTSTAT DERR [7..7] ================================================ */ +typedef enum { /*!< ADC_INTSTAT_DERR */ + ADC_INTSTAT_DERR_DMAERROR = 1, /*!< DMAERROR : DMA Error Condition Occurred value. */ +} ADC_INTSTAT_DERR_Enum; + +/* ================================================ ADC INTSTAT DCMP [6..6] ================================================ */ +typedef enum { /*!< ADC_INTSTAT_DCMP */ + ADC_INTSTAT_DCMP_DMACOMPLETE = 1, /*!< DMACOMPLETE : DMA Completed a transfer value. */ +} ADC_INTSTAT_DCMP_Enum; + +/* =============================================== ADC INTSTAT WCINC [5..5] ================================================ */ +typedef enum { /*!< ADC_INTSTAT_WCINC */ + ADC_INTSTAT_WCINC_WCINCINT = 1, /*!< WCINCINT : Window comparitor voltage incursion interrupt. value. */ +} ADC_INTSTAT_WCINC_Enum; + +/* =============================================== ADC INTSTAT WCEXC [4..4] ================================================ */ +typedef enum { /*!< ADC_INTSTAT_WCEXC */ + ADC_INTSTAT_WCEXC_WCEXCINT = 1, /*!< WCEXCINT : Window comparitor voltage excursion interrupt. value. */ +} ADC_INTSTAT_WCEXC_Enum; + +/* ============================================== ADC INTSTAT FIFOOVR2 [3..3] ============================================== */ +typedef enum { /*!< ADC_INTSTAT_FIFOOVR2 */ + ADC_INTSTAT_FIFOOVR2_FIFOFULLINT = 1, /*!< FIFOFULLINT : FIFO 100 percent full interrupt. value. */ +} ADC_INTSTAT_FIFOOVR2_Enum; + +/* ============================================== ADC INTSTAT FIFOOVR1 [2..2] ============================================== */ +typedef enum { /*!< ADC_INTSTAT_FIFOOVR1 */ + ADC_INTSTAT_FIFOOVR1_FIFO75INT = 1, /*!< FIFO75INT : FIFO 75 percent full interrupt. value. */ +} ADC_INTSTAT_FIFOOVR1_Enum; + +/* =============================================== ADC INTSTAT SCNCMP [1..1] =============================================== */ +typedef enum { /*!< ADC_INTSTAT_SCNCMP */ + ADC_INTSTAT_SCNCMP_SCNCMPINT = 1, /*!< SCNCMPINT : ADC scan complete interrupt. value. */ +} ADC_INTSTAT_SCNCMP_Enum; + +/* =============================================== ADC INTSTAT CNVCMP [0..0] =============================================== */ +typedef enum { /*!< ADC_INTSTAT_CNVCMP */ + ADC_INTSTAT_CNVCMP_CNVCMPINT = 1, /*!< CNVCMPINT : ADC conversion complete interrupt. value. */ +} ADC_INTSTAT_CNVCMP_Enum; + +/* ======================================================== INTCLR ========================================================= */ +/* ================================================ ADC INTCLR DERR [7..7] ================================================= */ +typedef enum { /*!< ADC_INTCLR_DERR */ + ADC_INTCLR_DERR_DMAERROR = 1, /*!< DMAERROR : DMA Error Condition Occurred value. */ +} ADC_INTCLR_DERR_Enum; + +/* ================================================ ADC INTCLR DCMP [6..6] ================================================= */ +typedef enum { /*!< ADC_INTCLR_DCMP */ + ADC_INTCLR_DCMP_DMACOMPLETE = 1, /*!< DMACOMPLETE : DMA Completed a transfer value. */ +} ADC_INTCLR_DCMP_Enum; + +/* ================================================ ADC INTCLR WCINC [5..5] ================================================ */ +typedef enum { /*!< ADC_INTCLR_WCINC */ + ADC_INTCLR_WCINC_WCINCINT = 1, /*!< WCINCINT : Window comparitor voltage incursion interrupt. value. */ +} ADC_INTCLR_WCINC_Enum; + +/* ================================================ ADC INTCLR WCEXC [4..4] ================================================ */ +typedef enum { /*!< ADC_INTCLR_WCEXC */ + ADC_INTCLR_WCEXC_WCEXCINT = 1, /*!< WCEXCINT : Window comparitor voltage excursion interrupt. value. */ +} ADC_INTCLR_WCEXC_Enum; + +/* ============================================== ADC INTCLR FIFOOVR2 [3..3] =============================================== */ +typedef enum { /*!< ADC_INTCLR_FIFOOVR2 */ + ADC_INTCLR_FIFOOVR2_FIFOFULLINT = 1, /*!< FIFOFULLINT : FIFO 100 percent full interrupt. value. */ +} ADC_INTCLR_FIFOOVR2_Enum; + +/* ============================================== ADC INTCLR FIFOOVR1 [2..2] =============================================== */ +typedef enum { /*!< ADC_INTCLR_FIFOOVR1 */ + ADC_INTCLR_FIFOOVR1_FIFO75INT = 1, /*!< FIFO75INT : FIFO 75 percent full interrupt. value. */ +} ADC_INTCLR_FIFOOVR1_Enum; + +/* =============================================== ADC INTCLR SCNCMP [1..1] ================================================ */ +typedef enum { /*!< ADC_INTCLR_SCNCMP */ + ADC_INTCLR_SCNCMP_SCNCMPINT = 1, /*!< SCNCMPINT : ADC scan complete interrupt. value. */ +} ADC_INTCLR_SCNCMP_Enum; + +/* =============================================== ADC INTCLR CNVCMP [0..0] ================================================ */ +typedef enum { /*!< ADC_INTCLR_CNVCMP */ + ADC_INTCLR_CNVCMP_CNVCMPINT = 1, /*!< CNVCMPINT : ADC conversion complete interrupt. value. */ +} ADC_INTCLR_CNVCMP_Enum; + +/* ======================================================== INTSET ========================================================= */ +/* ================================================ ADC INTSET DERR [7..7] ================================================= */ +typedef enum { /*!< ADC_INTSET_DERR */ + ADC_INTSET_DERR_DMAERROR = 1, /*!< DMAERROR : DMA Error Condition Occurred value. */ +} ADC_INTSET_DERR_Enum; + +/* ================================================ ADC INTSET DCMP [6..6] ================================================= */ +typedef enum { /*!< ADC_INTSET_DCMP */ + ADC_INTSET_DCMP_DMACOMPLETE = 1, /*!< DMACOMPLETE : DMA Completed a transfer value. */ +} ADC_INTSET_DCMP_Enum; + +/* ================================================ ADC INTSET WCINC [5..5] ================================================ */ +typedef enum { /*!< ADC_INTSET_WCINC */ + ADC_INTSET_WCINC_WCINCINT = 1, /*!< WCINCINT : Window comparitor voltage incursion interrupt. value. */ +} ADC_INTSET_WCINC_Enum; + +/* ================================================ ADC INTSET WCEXC [4..4] ================================================ */ +typedef enum { /*!< ADC_INTSET_WCEXC */ + ADC_INTSET_WCEXC_WCEXCINT = 1, /*!< WCEXCINT : Window comparitor voltage excursion interrupt. value. */ +} ADC_INTSET_WCEXC_Enum; + +/* ============================================== ADC INTSET FIFOOVR2 [3..3] =============================================== */ +typedef enum { /*!< ADC_INTSET_FIFOOVR2 */ + ADC_INTSET_FIFOOVR2_FIFOFULLINT = 1, /*!< FIFOFULLINT : FIFO 100 percent full interrupt. value. */ +} ADC_INTSET_FIFOOVR2_Enum; + +/* ============================================== ADC INTSET FIFOOVR1 [2..2] =============================================== */ +typedef enum { /*!< ADC_INTSET_FIFOOVR1 */ + ADC_INTSET_FIFOOVR1_FIFO75INT = 1, /*!< FIFO75INT : FIFO 75 percent full interrupt. value. */ +} ADC_INTSET_FIFOOVR1_Enum; + +/* =============================================== ADC INTSET SCNCMP [1..1] ================================================ */ +typedef enum { /*!< ADC_INTSET_SCNCMP */ + ADC_INTSET_SCNCMP_SCNCMPINT = 1, /*!< SCNCMPINT : ADC scan complete interrupt. value. */ +} ADC_INTSET_SCNCMP_Enum; + +/* =============================================== ADC INTSET CNVCMP [0..0] ================================================ */ +typedef enum { /*!< ADC_INTSET_CNVCMP */ + ADC_INTSET_CNVCMP_CNVCMPINT = 1, /*!< CNVCMPINT : ADC conversion complete interrupt. value. */ +} ADC_INTSET_CNVCMP_Enum; + +/* ======================================================= DMATRIGEN ======================================================= */ +/* ====================================================== DMATRIGSTAT ====================================================== */ +/* ======================================================== DMACFG ========================================================= */ +/* ============================================== ADC DMACFG DMAMSK [17..17] =============================================== */ +typedef enum { /*!< ADC_DMACFG_DMAMSK */ + ADC_DMACFG_DMAMSK_DIS = 0, /*!< DIS : FIFO Contents are copied directly to memory without modification. + value. */ + ADC_DMACFG_DMAMSK_EN = 1, /*!< EN : Only the FIFODATA contents are copied to memory on DMA + transfers. The SLOTNUM and FIFOCNT contents are cleared + to zero. value. */ +} ADC_DMACFG_DMAMSK_Enum; + +/* ============================================ ADC DMACFG DMAHONSTAT [16..16] ============================================= */ +typedef enum { /*!< ADC_DMACFG_DMAHONSTAT */ + ADC_DMACFG_DMAHONSTAT_DIS = 0, /*!< DIS : ADC conversions will continue regardless of DMA status + register value. */ + ADC_DMACFG_DMAHONSTAT_EN = 1, /*!< EN : ADC conversions will not progress if DMAERR or DMACPL bits + in DMA status register are set. value. */ +} ADC_DMACFG_DMAHONSTAT_Enum; + +/* ============================================== ADC DMACFG DMADYNPRI [9..9] ============================================== */ +typedef enum { /*!< ADC_DMACFG_DMADYNPRI */ + ADC_DMACFG_DMADYNPRI_DIS = 0, /*!< DIS : Disable dynamic priority (use DMAPRI setting only) value. */ + ADC_DMACFG_DMADYNPRI_EN = 1, /*!< EN : Enable dynamic priority value. */ +} ADC_DMACFG_DMADYNPRI_Enum; + +/* =============================================== ADC DMACFG DMAPRI [8..8] ================================================ */ +typedef enum { /*!< ADC_DMACFG_DMAPRI */ + ADC_DMACFG_DMAPRI_LOW = 0, /*!< LOW : Low Priority (service as best effort) value. */ + ADC_DMACFG_DMAPRI_HIGH = 1, /*!< HIGH : High Priority (service immediately) value. */ +} ADC_DMACFG_DMAPRI_Enum; + +/* =============================================== ADC DMACFG DMADIR [2..2] ================================================ */ +typedef enum { /*!< ADC_DMACFG_DMADIR */ + ADC_DMACFG_DMADIR_P2M = 0, /*!< P2M : Peripheral to Memory (SRAM) transaction value. */ + ADC_DMACFG_DMADIR_M2P = 1, /*!< M2P : Memory to Peripheral transaction value. */ +} ADC_DMACFG_DMADIR_Enum; + +/* ================================================ ADC DMACFG DMAEN [0..0] ================================================ */ +typedef enum { /*!< ADC_DMACFG_DMAEN */ + ADC_DMACFG_DMAEN_DIS = 0, /*!< DIS : Disable DMA Function value. */ + ADC_DMACFG_DMAEN_EN = 1, /*!< EN : Enable DMA Function value. */ +} ADC_DMACFG_DMAEN_Enum; + +/* ====================================================== DMATOTCOUNT ====================================================== */ +/* ====================================================== DMATARGADDR ====================================================== */ +/* ======================================================== DMASTAT ======================================================== */ + + +/* =========================================================================================================================== */ +/* ================ APBDMA ================ */ +/* =========================================================================================================================== */ + +/* ======================================================== BBVALUE ======================================================== */ +/* ====================================================== BBSETCLEAR ======================================================= */ +/* ======================================================== BBINPUT ======================================================== */ +/* ======================================================= DEBUGDATA ======================================================= */ +/* ========================================================= DEBUG ========================================================= */ +/* ============================================== APBDMA DEBUG DEBUGEN [0..3] ============================================== */ +typedef enum { /*!< APBDMA_DEBUG_DEBUGEN */ + APBDMA_DEBUG_DEBUGEN_OFF = 0, /*!< OFF : Debug Disabled value. */ + APBDMA_DEBUG_DEBUGEN_ARB = 1, /*!< ARB : Debug Arb values value. */ +} APBDMA_DEBUG_DEBUGEN_Enum; + + + +/* =========================================================================================================================== */ +/* ================ BLEIF ================ */ +/* =========================================================================================================================== */ + +/* ========================================================= FIFO ========================================================== */ +/* ======================================================== FIFOPTR ======================================================== */ +/* ======================================================== FIFOTHR ======================================================== */ +/* ======================================================== FIFOPOP ======================================================== */ +/* ======================================================= FIFOPUSH ======================================================== */ +/* ======================================================= FIFOCTRL ======================================================== */ +/* ======================================================== FIFOLOC ======================================================== */ +/* ======================================================== CLKCFG ========================================================= */ +/* =============================================== BLEIF CLKCFG FSEL [8..10] =============================================== */ +typedef enum { /*!< BLEIF_CLKCFG_FSEL */ + BLEIF_CLKCFG_FSEL_MIN_PWR = 0, /*!< MIN_PWR : Selects the minimum power clock. This setting should + be used whenever the IOM is not active. value. */ + BLEIF_CLKCFG_FSEL_HFRC = 1, /*!< HFRC : Selects the HFRC as the input clock. value. */ + BLEIF_CLKCFG_FSEL_HFRC_DIV2 = 2, /*!< HFRC_DIV2 : Selects the HFRC / 2 as the input clock. value. */ + BLEIF_CLKCFG_FSEL_HFRC_DIV4 = 3, /*!< HFRC_DIV4 : Selects the HFRC / 4 as the input clock. value. */ + BLEIF_CLKCFG_FSEL_HFRC_DIV8 = 4, /*!< HFRC_DIV8 : Selects the HFRC / 8 as the input clock. value. */ + BLEIF_CLKCFG_FSEL_HFRC_DIV16 = 5, /*!< HFRC_DIV16 : Selects the HFRC / 16 as the input clock. value. */ + BLEIF_CLKCFG_FSEL_HFRC_DIV32 = 6, /*!< HFRC_DIV32 : Selects the HFRC / 32 as the input clock. value. */ + BLEIF_CLKCFG_FSEL_HFRC_DIV64 = 7, /*!< HFRC_DIV64 : Selects the HFRC / 64 as the input clock. value. */ +} BLEIF_CLKCFG_FSEL_Enum; + +/* ========================================================== CMD ========================================================== */ +/* ================================================= BLEIF CMD CMD [0..4] ================================================== */ +typedef enum { /*!< BLEIF_CMD_CMD */ + BLEIF_CMD_CMD_WRITE = 1, /*!< WRITE : Write command using count of offset bytes specified + in the OFFSETCNT field value. */ + BLEIF_CMD_CMD_READ = 2, /*!< READ : Read command using count of offset bytes specified in + the OFFSETCNT field value. */ +} BLEIF_CMD_CMD_Enum; + +/* ======================================================== CMDRPT ========================================================= */ +/* ======================================================= OFFSETHI ======================================================== */ +/* ======================================================== CMDSTAT ======================================================== */ +/* ============================================= BLEIF CMDSTAT CMDSTAT [5..7] ============================================== */ +typedef enum { /*!< BLEIF_CMDSTAT_CMDSTAT */ + BLEIF_CMDSTAT_CMDSTAT_ERR = 1, /*!< ERR : Error encountered with command value. */ + BLEIF_CMDSTAT_CMDSTAT_ACTIVE = 2, /*!< ACTIVE : Actively processing command value. */ + BLEIF_CMDSTAT_CMDSTAT_IDLE = 4, /*!< IDLE : Idle state, no active command, no error value. */ + BLEIF_CMDSTAT_CMDSTAT_WAIT = 6, /*!< WAIT : Command in progress, but waiting on data from host value. */ +} BLEIF_CMDSTAT_CMDSTAT_Enum; + +/* ========================================================= INTEN ========================================================= */ +/* ======================================================== INTSTAT ======================================================== */ +/* ======================================================== INTCLR ========================================================= */ +/* ======================================================== INTSET ========================================================= */ +/* ======================================================= DMATRIGEN ======================================================= */ +/* ====================================================== DMATRIGSTAT ====================================================== */ +/* ======================================================== DMACFG ========================================================= */ +/* ============================================== BLEIF DMACFG DPWROFF [9..9] ============================================== */ +typedef enum { /*!< BLEIF_DMACFG_DPWROFF */ + BLEIF_DMACFG_DPWROFF_DIS = 0, /*!< DIS : Power off disabled value. */ + BLEIF_DMACFG_DPWROFF_EN = 1, /*!< EN : Power off enabled value. */ +} BLEIF_DMACFG_DPWROFF_Enum; + +/* ============================================== BLEIF DMACFG DMAPRI [8..8] =============================================== */ +typedef enum { /*!< BLEIF_DMACFG_DMAPRI */ + BLEIF_DMACFG_DMAPRI_LOW = 0, /*!< LOW : Low Priority (service as best effort) value. */ + BLEIF_DMACFG_DMAPRI_HIGH = 1, /*!< HIGH : High Priority (service immediately) value. */ +} BLEIF_DMACFG_DMAPRI_Enum; + +/* ============================================== BLEIF DMACFG DMADIR [1..1] =============================================== */ +typedef enum { /*!< BLEIF_DMACFG_DMADIR */ + BLEIF_DMACFG_DMADIR_P2M = 0, /*!< P2M : Peripheral to Memory (SRAM) transaction. To be set when + doing IOM read operations, ie reading data from external + devices. value. */ + BLEIF_DMACFG_DMADIR_M2P = 1, /*!< M2P : Memory to Peripheral transaction. To be set when doing + IOM write operations, ie writing data to external devices. + value. */ +} BLEIF_DMACFG_DMADIR_Enum; + +/* =============================================== BLEIF DMACFG DMAEN [0..0] =============================================== */ +typedef enum { /*!< BLEIF_DMACFG_DMAEN */ + BLEIF_DMACFG_DMAEN_DIS = 0, /*!< DIS : Disable DMA Function value. */ + BLEIF_DMACFG_DMAEN_EN = 1, /*!< EN : Enable DMA Function value. */ +} BLEIF_DMACFG_DMAEN_Enum; + +/* ====================================================== DMATOTCOUNT ====================================================== */ +/* ====================================================== DMATARGADDR ====================================================== */ +/* ======================================================== DMASTAT ======================================================== */ +/* ========================================================= CQCFG ========================================================= */ +/* =============================================== BLEIF CQCFG CQPRI [1..1] ================================================ */ +typedef enum { /*!< BLEIF_CQCFG_CQPRI */ + BLEIF_CQCFG_CQPRI_LOW = 0, /*!< LOW : Low Priority (service as best effort) value. */ + BLEIF_CQCFG_CQPRI_HIGH = 1, /*!< HIGH : High Priority (service immediately) value. */ +} BLEIF_CQCFG_CQPRI_Enum; + +/* ================================================ BLEIF CQCFG CQEN [0..0] ================================================ */ +typedef enum { /*!< BLEIF_CQCFG_CQEN */ + BLEIF_CQCFG_CQEN_DIS = 0, /*!< DIS : Disable CQ Function value. */ + BLEIF_CQCFG_CQEN_EN = 1, /*!< EN : Enable CQ Function value. */ +} BLEIF_CQCFG_CQEN_Enum; + +/* ======================================================== CQADDR ========================================================= */ +/* ======================================================== CQSTAT ========================================================= */ +/* ======================================================== CQFLAGS ======================================================== */ +/* ====================================================== CQSETCLEAR ======================================================= */ +/* ======================================================= CQPAUSEEN ======================================================= */ +/* ============================================= BLEIF CQPAUSEEN CQPEN [0..15] ============================================= */ +typedef enum { /*!< BLEIF_CQPAUSEEN_CQPEN */ + BLEIF_CQPAUSEEN_CQPEN_CNTEQ = 32768, /*!< CNTEQ : Pauses command queue processing when HWCNT matches SWCNT + value. */ + BLEIF_CQPAUSEEN_CQPEN_BLEXOREN = 16384, /*!< BLEXOREN : Pause command queue when input BLE bit XORed with + SWFLAG4 is '1' value. */ + BLEIF_CQPAUSEEN_CQPEN_IOMXOREN = 8192, /*!< IOMXOREN : Pause command queue when input IOM bit XORed with + SWFLAG3 is '1' value. */ + BLEIF_CQPAUSEEN_CQPEN_GPIOXOREN = 4096, /*!< GPIOXOREN : Pause command queue when input GPIO irq_bit XORed + with SWFLAG2 is '1' value. */ + BLEIF_CQPAUSEEN_CQPEN_MSPI1XNOREN = 2048, /*!< MSPI1XNOREN : Pause command queue when input MSPI1 bit XNORed + with SWFLAG1 is '1' value. */ + BLEIF_CQPAUSEEN_CQPEN_MSPI0XNOREN = 1024, /*!< MSPI0XNOREN : Pause command queue when input MSPI0 bit XNORed + with SWFLAG0 is '1' value. */ + BLEIF_CQPAUSEEN_CQPEN_MSPI1XOREN = 512, /*!< MSPI1XOREN : Pause command queue when input MSPI1 bit XORed + with SWFLAG1 is '1' value. */ + BLEIF_CQPAUSEEN_CQPEN_MSPI0XOREN = 256, /*!< MSPI0XOREN : Pause command queue when input MSPI0 bit XORed + with SWFLAG0 is '1' value. */ + BLEIF_CQPAUSEEN_CQPEN_SWFLAGEN7 = 128, /*!< SWFLAGEN7 : Pause the command queue when software flag bit 7 + is '1'. value. */ + BLEIF_CQPAUSEEN_CQPEN_SWFLAGEN6 = 64, /*!< SWFLAGEN6 : Pause the command queue when software flag bit 7 + is '1' value. */ + BLEIF_CQPAUSEEN_CQPEN_SWFLAGEN5 = 32, /*!< SWFLAGEN5 : Pause the command queue when software flag bit 7 + is '1' value. */ + BLEIF_CQPAUSEEN_CQPEN_SWFLAGEN4 = 16, /*!< SWFLAGEN4 : Pause the command queue when software flag bit 7 + is '1' value. */ + BLEIF_CQPAUSEEN_CQPEN_SWFLAGEN3 = 8, /*!< SWFLAGEN3 : Pause the command queue when software flag bit 7 + is '1' value. */ + BLEIF_CQPAUSEEN_CQPEN_SWFLAGEN2 = 4, /*!< SWFLAGEN2 : Pause the command queue when software flag bit 7 + is '1' value. */ + BLEIF_CQPAUSEEN_CQPEN_SWFLAGEN1 = 2, /*!< SWFLAGEN1 : Pause the command queue when software flag bit 7 + is '1' value. */ + BLEIF_CQPAUSEEN_CQPEN_SWFLGEN0 = 1, /*!< SWFLGEN0 : Pause the command queue when software flag bit 7 + is '1' value. */ +} BLEIF_CQPAUSEEN_CQPEN_Enum; + +/* ======================================================= CQCURIDX ======================================================== */ +/* ======================================================= CQENDIDX ======================================================== */ +/* ======================================================== STATUS ========================================================= */ +/* ============================================== BLEIF STATUS IDLEST [2..2] =============================================== */ +typedef enum { /*!< BLEIF_STATUS_IDLEST */ + BLEIF_STATUS_IDLEST_IDLE = 1, /*!< IDLE : The I/O state machine is in the idle state. value. */ +} BLEIF_STATUS_IDLEST_Enum; + +/* ============================================== BLEIF STATUS CMDACT [1..1] =============================================== */ +typedef enum { /*!< BLEIF_STATUS_CMDACT */ + BLEIF_STATUS_CMDACT_ACTIVE = 1, /*!< ACTIVE : An I/O command is active. Indicates the active module + has an active command and is processing this. De-asserted + when the command is completed. value. */ +} BLEIF_STATUS_CMDACT_Enum; + +/* ================================================ BLEIF STATUS ERR [0..0] ================================================ */ +typedef enum { /*!< BLEIF_STATUS_ERR */ + BLEIF_STATUS_ERR_ERROR = 1, /*!< ERROR : Bit has been deprecated and will always return 0. value. */ +} BLEIF_STATUS_ERR_Enum; + +/* ======================================================== MSPICFG ======================================================== */ +/* ============================================= BLEIF MSPICFG SPILSB [23..23] ============================================= */ +typedef enum { /*!< BLEIF_MSPICFG_SPILSB */ + BLEIF_MSPICFG_SPILSB_MSB = 0, /*!< MSB : Send and receive MSB bit first value. */ + BLEIF_MSPICFG_SPILSB_LSB = 1, /*!< LSB : Send and receive LSB bit first value. */ +} BLEIF_MSPICFG_SPILSB_Enum; + +/* ============================================ BLEIF MSPICFG RDFCPOL [22..22] ============================================= */ +typedef enum { /*!< BLEIF_MSPICFG_RDFCPOL */ + BLEIF_MSPICFG_RDFCPOL_NORMAL = 0, /*!< NORMAL : SPI_STATUS signal from BLE Core high(1) creates flow + control and new read spi transactions will not be started + until the signal goes low.(default) value. */ + BLEIF_MSPICFG_RDFCPOL_INVERTED = 1, /*!< INVERTED : SPI_STATUS signal from BLE Core low(0) creates flow + control and new read spi transactions will not be started + until the signal goes high. value. */ +} BLEIF_MSPICFG_RDFCPOL_Enum; + +/* ============================================ BLEIF MSPICFG WTFCPOL [21..21] ============================================= */ +typedef enum { /*!< BLEIF_MSPICFG_WTFCPOL */ + BLEIF_MSPICFG_WTFCPOL_NORMAL = 0, /*!< NORMAL : SPI_STATUS signal from BLE Core high(1) creates flow + control and new write spi transactions will not be started + until the signal goes low.(default) value. */ + BLEIF_MSPICFG_WTFCPOL_INVERTED = 1, /*!< INVERTED : SPI_STATUS signal from BLE Core high(1) creates low(0) + control and new write spi transactions will not be started + until the signal goes high. value. */ +} BLEIF_MSPICFG_WTFCPOL_Enum; + +/* ============================================== BLEIF MSPICFG RDFC [17..17] ============================================== */ +typedef enum { /*!< BLEIF_MSPICFG_RDFC */ + BLEIF_MSPICFG_RDFC_DIS = 0, /*!< DIS : Read mode flow control disabled. value. */ + BLEIF_MSPICFG_RDFC_EN = 1, /*!< EN : Read mode flow control enabled. value. */ +} BLEIF_MSPICFG_RDFC_Enum; + +/* ============================================== BLEIF MSPICFG WTFC [16..16] ============================================== */ +typedef enum { /*!< BLEIF_MSPICFG_WTFC */ + BLEIF_MSPICFG_WTFC_DIS = 0, /*!< DIS : Write mode flow control disabled. value. */ + BLEIF_MSPICFG_WTFC_EN = 1, /*!< EN : Write mode flow control enabled. value. */ +} BLEIF_MSPICFG_WTFC_Enum; + +/* =============================================== BLEIF MSPICFG SPHA [1..1] =============================================== */ +typedef enum { /*!< BLEIF_MSPICFG_SPHA */ + BLEIF_MSPICFG_SPHA_SAMPLE_LEADING_EDGE = 0, /*!< SAMPLE_LEADING_EDGE : Sample on the leading (first) clock edge, + rising or falling dependant on the value of SPOL value. */ + BLEIF_MSPICFG_SPHA_SAMPLE_TRAILING_EDGE = 1, /*!< SAMPLE_TRAILING_EDGE : Sample on the trailing (second) clock + edge, rising of falling dependant on the value of SPOL + value. */ +} BLEIF_MSPICFG_SPHA_Enum; + +/* =============================================== BLEIF MSPICFG SPOL [0..0] =============================================== */ +typedef enum { /*!< BLEIF_MSPICFG_SPOL */ + BLEIF_MSPICFG_SPOL_CLK_BASE_0 = 0, /*!< CLK_BASE_0 : The initial value of the clock is 0. value. */ + BLEIF_MSPICFG_SPOL_CLK_BASE_1 = 1, /*!< CLK_BASE_1 : The initial value of the clock is 1. value. */ +} BLEIF_MSPICFG_SPOL_Enum; + +/* ======================================================== BLECFG ========================================================= */ +/* ============================================ BLEIF BLECFG SPIISOCTL [14..15] ============================================ */ +typedef enum { /*!< BLEIF_BLECFG_SPIISOCTL */ + BLEIF_BLECFG_SPIISOCTL_ON = 3, /*!< ON : SPI signals from BLE Core to/from MCU Core are isolated. + value. */ + BLEIF_BLECFG_SPIISOCTL_OFF = 2, /*!< OFF : SPI signals from BLE Core to/from MCU Core are not isolated. + value. */ + BLEIF_BLECFG_SPIISOCTL_AUTO = 0, /*!< AUTO : SPI signals from BLE Core to/from MCU Core are automatically + isolated by the logic value. */ +} BLEIF_BLECFG_SPIISOCTL_Enum; + +/* ============================================ BLEIF BLECFG PWRISOCTL [12..13] ============================================ */ +typedef enum { /*!< BLEIF_BLECFG_PWRISOCTL */ + BLEIF_BLECFG_PWRISOCTL_ON = 3, /*!< ON : BLEH power signal isolation to on (isolated). value. */ + BLEIF_BLECFG_PWRISOCTL_OFF = 2, /*!< OFF : BLEH power signal isolation to off (not isolated). value. */ + BLEIF_BLECFG_PWRISOCTL_AUTO = 0, /*!< AUTO : BLEH Power signal isolation is controlled automatically + through the interface logic value. */ +} BLEIF_BLECFG_PWRISOCTL_Enum; + +/* ============================================ BLEIF BLECFG BLEHREQCTL [6..7] ============================================= */ +typedef enum { /*!< BLEIF_BLECFG_BLEHREQCTL */ + BLEIF_BLECFG_BLEHREQCTL_ON = 3, /*!< ON : BLEH Power-on reg signal is set to on (1). value. */ + BLEIF_BLECFG_BLEHREQCTL_OFF = 2, /*!< OFF : BLEH Power-on signal is set to off (0). value. */ + BLEIF_BLECFG_BLEHREQCTL_AUTO = 0, /*!< AUTO : BLEH Power-on signal is controlled by the PWRSM logic + and automatically controlled value. */ +} BLEIF_BLECFG_BLEHREQCTL_Enum; + +/* ============================================ BLEIF BLECFG DCDCFLGCTL [4..5] ============================================= */ +typedef enum { /*!< BLEIF_BLECFG_DCDCFLGCTL */ + BLEIF_BLECFG_DCDCFLGCTL_ON = 3, /*!< ON : DCDC Flag signal is set to on (1). value. */ + BLEIF_BLECFG_DCDCFLGCTL_OFF = 2, /*!< OFF : DCDC Flag signal is set to off (0). value. */ + BLEIF_BLECFG_DCDCFLGCTL_AUTO = 0, /*!< AUTO : DCDC Flag signal is controlled by the PWRSM logic and + automatically controlled value. */ +} BLEIF_BLECFG_DCDCFLGCTL_Enum; + +/* ============================================= BLEIF BLECFG WAKEUPCTL [2..3] ============================================= */ +typedef enum { /*!< BLEIF_BLECFG_WAKEUPCTL */ + BLEIF_BLECFG_WAKEUPCTL_ON = 3, /*!< ON : Wake signal is set to on (1). value. */ + BLEIF_BLECFG_WAKEUPCTL_OFF = 2, /*!< OFF : Wake signal is set to off (0). value. */ + BLEIF_BLECFG_WAKEUPCTL_AUTO = 0, /*!< AUTO : Wake signal is controlled by the PWRSM logic and automatically + controlled value. */ +} BLEIF_BLECFG_WAKEUPCTL_Enum; + +/* ============================================== BLEIF BLECFG BLERSTN [1..1] ============================================== */ +typedef enum { /*!< BLEIF_BLECFG_BLERSTN */ + BLEIF_BLECFG_BLERSTN_ACTIVE = 1, /*!< ACTIVE : The reset signal is active (0) value. */ + BLEIF_BLECFG_BLERSTN_INACTIVE = 0, /*!< INACTIVE : The reset signal is inactive (1) value. */ +} BLEIF_BLECFG_BLERSTN_Enum; + +/* ============================================== BLEIF BLECFG PWRSMEN [0..0] ============================================== */ +typedef enum { /*!< BLEIF_BLECFG_PWRSMEN */ + BLEIF_BLECFG_PWRSMEN_ON = 1, /*!< ON : Internal power state machine is enabled and will sequence + the BLEH power domain as indicated in the design document. + Overrides for the power signals are not enabled. value. */ + BLEIF_BLECFG_PWRSMEN_OFF = 0, /*!< OFF : Internal power state machine is disabled and will not + sequence the BLEH power domain. The values of the overrides + will be used to drive the output sequencing signals value. */ +} BLEIF_BLECFG_PWRSMEN_Enum; + +/* ======================================================== PWRCMD ========================================================= */ +/* ======================================================== BSTATUS ======================================================== */ +/* ============================================== BLEIF BSTATUS PWRST [8..10] ============================================== */ +typedef enum { /*!< BLEIF_BSTATUS_PWRST */ + BLEIF_BSTATUS_PWRST_OFF = 0, /*!< OFF : Internal power state machine is disabled and will not + sequence the BLEH power domain. The values of the overrides + will be used to drive the output sequencing signals value. */ + BLEIF_BSTATUS_PWRST_INIT = 1, /*!< INIT : Initialization state. BLEH not powered value. */ + BLEIF_BSTATUS_PWRST_PWRON = 2, /*!< PWRON : Waiting for the powerup of the BLEH value. */ + BLEIF_BSTATUS_PWRST_ACTIVE = 3, /*!< ACTIVE : The BLE Core is powered and active value. */ + BLEIF_BSTATUS_PWRST_SLEEP = 6, /*!< SLEEP : The BLE Core has entered sleep mode and the power request + is inactive value. */ + BLEIF_BSTATUS_PWRST_SHUTDOWN = 4, /*!< SHUTDOWN : The BLE Core is in shutdown mode value. */ +} BLEIF_BSTATUS_PWRST_Enum; + +/* ============================================= BLEIF BSTATUS B2MSTATE [0..2] ============================================= */ +typedef enum { /*!< BLEIF_BSTATUS_B2MSTATE */ + BLEIF_BSTATUS_B2MSTATE_RESET = 0, /*!< RESET : Reset State value. */ + BLEIF_BSTATUS_B2MSTATE_Sleep = 1, /*!< Sleep : Sleep state. value. */ + BLEIF_BSTATUS_B2MSTATE_Standby = 2, /*!< Standby : Standby State value. */ + BLEIF_BSTATUS_B2MSTATE_Idle = 3, /*!< Idle : Idle state value. */ + BLEIF_BSTATUS_B2MSTATE_Active = 4, /*!< Active : Active state. value. */ +} BLEIF_BSTATUS_B2MSTATE_Enum; + +/* ======================================================== BLEDBG ========================================================= */ + + +/* =========================================================================================================================== */ +/* ================ CACHECTRL ================ */ +/* =========================================================================================================================== */ + +/* ======================================================= CACHECFG ======================================================== */ +/* =========================================== CACHECTRL CACHECFG CONFIG [4..7] ============================================ */ +typedef enum { /*!< CACHECTRL_CACHECFG_CONFIG */ + CACHECTRL_CACHECFG_CONFIG_W1_128B_512E = 4, /*!< W1_128B_512E : Direct mapped, 128-bit linesize, 512 entries + (4 SRAMs active) value. */ + CACHECTRL_CACHECFG_CONFIG_W2_128B_512E = 5, /*!< W2_128B_512E : Two-way set associative, 128-bit linesize, 512 + entries (8 SRAMs active) value. */ + CACHECTRL_CACHECFG_CONFIG_W1_128B_1024E = 8, /*!< W1_128B_1024E : Direct mapped, 128-bit linesize, 1024 entries + (8 SRAMs active) value. */ +} CACHECTRL_CACHECFG_CONFIG_Enum; + +/* ======================================================= FLASHCFG ======================================================== */ +/* ========================================== CACHECTRL FLASHCFG LPMMODE [12..13] ========================================== */ +typedef enum { /*!< CACHECTRL_FLASHCFG_LPMMODE */ + CACHECTRL_FLASHCFG_LPMMODE_NEVER = 0, /*!< NEVER : High power mode (LPM not used). value. */ + CACHECTRL_FLASHCFG_LPMMODE_STANDBY = 1, /*!< STANDBY : Fast Standby mode. LPM deasserted for read operations, + but asserted while flash IDLE. value. */ + CACHECTRL_FLASHCFG_LPMMODE_ALWAYS = 2, /*!< ALWAYS : Low Power mode. LPM always asserted for reads. LPM_RD_WAIT + must be programmed to accomodate longer read access times. + value. */ +} CACHECTRL_FLASHCFG_LPMMODE_Enum; + +/* ========================================================= CTRL ========================================================== */ +/* =========================================== CACHECTRL CTRL RESET_STAT [1..1] ============================================ */ +typedef enum { /*!< CACHECTRL_CTRL_RESET_STAT */ + CACHECTRL_CTRL_RESET_STAT_CLEAR = 1, /*!< CLEAR : Clear Cache Stats value. */ +} CACHECTRL_CTRL_RESET_STAT_Enum; + +/* ======================================================= NCR0START ======================================================= */ +/* ======================================================== NCR0END ======================================================== */ +/* ======================================================= NCR1START ======================================================= */ +/* ======================================================== NCR1END ======================================================== */ +/* ========================================================= DMON0 ========================================================= */ +/* ========================================================= DMON1 ========================================================= */ +/* ========================================================= DMON2 ========================================================= */ +/* ========================================================= DMON3 ========================================================= */ +/* ========================================================= IMON0 ========================================================= */ +/* ========================================================= IMON1 ========================================================= */ +/* ========================================================= IMON2 ========================================================= */ +/* ========================================================= IMON3 ========================================================= */ + + +/* =========================================================================================================================== */ +/* ================ CLKGEN ================ */ +/* =========================================================================================================================== */ + +/* ========================================================= CALXT ========================================================= */ +/* ========================================================= CALRC ========================================================= */ +/* ======================================================== ACALCTR ======================================================== */ +/* ========================================================= OCTRL ========================================================= */ +/* =============================================== CLKGEN OCTRL ACAL [8..10] =============================================== */ +typedef enum { /*!< CLKGEN_OCTRL_ACAL */ + CLKGEN_OCTRL_ACAL_DIS = 0, /*!< DIS : Disable Autocalibration value. */ + CLKGEN_OCTRL_ACAL_1024SEC = 2, /*!< 1024SEC : Autocalibrate every 1024 seconds. Once autocalibration + is done, an interrupt will be triggered at the end of 1024 + seconds. value. */ + CLKGEN_OCTRL_ACAL_512SEC = 3, /*!< 512SEC : Autocalibrate every 512 seconds. Once autocalibration + is done, an interrupt will be trigged at the end of 512 + seconds. value. */ + CLKGEN_OCTRL_ACAL_XTFREQ = 6, /*!< XTFREQ : Frequency measurement using XT. The XT clock is normally + considered much more accurate than the LFRC clock source. + value. */ + CLKGEN_OCTRL_ACAL_EXTFREQ = 7, /*!< EXTFREQ : Frequency measurement using external clock. value. */ +} CLKGEN_OCTRL_ACAL_Enum; + +/* =============================================== CLKGEN OCTRL OSEL [7..7] ================================================ */ +typedef enum { /*!< CLKGEN_OCTRL_OSEL */ + CLKGEN_OCTRL_OSEL_RTC_XT = 0, /*!< RTC_XT : RTC uses the XT value. */ + CLKGEN_OCTRL_OSEL_RTC_LFRC = 1, /*!< RTC_LFRC : RTC uses the LFRC value. */ +} CLKGEN_OCTRL_OSEL_Enum; + +/* ================================================ CLKGEN OCTRL FOS [6..6] ================================================ */ +typedef enum { /*!< CLKGEN_OCTRL_FOS */ + CLKGEN_OCTRL_FOS_DIS = 0, /*!< DIS : Disable the oscillator switch on failure function. value. */ + CLKGEN_OCTRL_FOS_EN = 1, /*!< EN : Enable the oscillator switch on failure function. value. */ +} CLKGEN_OCTRL_FOS_Enum; + +/* ============================================== CLKGEN OCTRL STOPRC [1..1] =============================================== */ +typedef enum { /*!< CLKGEN_OCTRL_STOPRC */ + CLKGEN_OCTRL_STOPRC_EN = 0, /*!< EN : Enable the LFRC Oscillator to drive the RTC value. */ + CLKGEN_OCTRL_STOPRC_STOP = 1, /*!< STOP : Stop the LFRC Oscillator when driving the RTC value. */ +} CLKGEN_OCTRL_STOPRC_Enum; + +/* ============================================== CLKGEN OCTRL STOPXT [0..0] =============================================== */ +typedef enum { /*!< CLKGEN_OCTRL_STOPXT */ + CLKGEN_OCTRL_STOPXT_EN = 0, /*!< EN : Enable the XT Oscillator to drive the RTC value. */ + CLKGEN_OCTRL_STOPXT_STOP = 1, /*!< STOP : Stop the XT Oscillator when driving the RTC value. */ +} CLKGEN_OCTRL_STOPXT_Enum; + +/* ======================================================== CLKOUT ========================================================= */ +/* =============================================== CLKGEN CLKOUT CKEN [7..7] =============================================== */ +typedef enum { /*!< CLKGEN_CLKOUT_CKEN */ + CLKGEN_CLKOUT_CKEN_DIS = 0, /*!< DIS : Disable CLKOUT value. */ + CLKGEN_CLKOUT_CKEN_EN = 1, /*!< EN : Enable CLKOUT value. */ +} CLKGEN_CLKOUT_CKEN_Enum; + +/* ============================================== CLKGEN CLKOUT CKSEL [0..5] =============================================== */ +typedef enum { /*!< CLKGEN_CLKOUT_CKSEL */ + CLKGEN_CLKOUT_CKSEL_LFRC = 0, /*!< LFRC : LFRC value. */ + CLKGEN_CLKOUT_CKSEL_XT_DIV2 = 1, /*!< XT_DIV2 : XT / 2 value. */ + CLKGEN_CLKOUT_CKSEL_XT_DIV4 = 2, /*!< XT_DIV4 : XT / 4 value. */ + CLKGEN_CLKOUT_CKSEL_XT_DIV8 = 3, /*!< XT_DIV8 : XT / 8 value. */ + CLKGEN_CLKOUT_CKSEL_XT_DIV16 = 4, /*!< XT_DIV16 : XT / 16 value. */ + CLKGEN_CLKOUT_CKSEL_XT_DIV32 = 5, /*!< XT_DIV32 : XT / 32 value. */ + CLKGEN_CLKOUT_CKSEL_RTC_1Hz = 16, /*!< RTC_1Hz : 1 Hz as selected in RTC value. */ + CLKGEN_CLKOUT_CKSEL_XT_DIV2M = 22, /*!< XT_DIV2M : XT / 2^21 value. */ + CLKGEN_CLKOUT_CKSEL_XT = 23, /*!< XT : XT value. */ + CLKGEN_CLKOUT_CKSEL_CG_100Hz = 24, /*!< CG_100Hz : 100 Hz as selected in CLKGEN value. */ + CLKGEN_CLKOUT_CKSEL_HFRC = 25, /*!< HFRC : HFRC value. */ + CLKGEN_CLKOUT_CKSEL_HFRC_DIV4 = 26, /*!< HFRC_DIV4 : HFRC / 4 value. */ + CLKGEN_CLKOUT_CKSEL_HFRC_DIV8 = 27, /*!< HFRC_DIV8 : HFRC / 8 value. */ + CLKGEN_CLKOUT_CKSEL_HFRC_DIV16 = 28, /*!< HFRC_DIV16 : HFRC / 16 value. */ + CLKGEN_CLKOUT_CKSEL_HFRC_DIV64 = 29, /*!< HFRC_DIV64 : HFRC / 64 value. */ + CLKGEN_CLKOUT_CKSEL_HFRC_DIV128 = 30, /*!< HFRC_DIV128 : HFRC / 128 value. */ + CLKGEN_CLKOUT_CKSEL_HFRC_DIV256 = 31, /*!< HFRC_DIV256 : HFRC / 256 value. */ + CLKGEN_CLKOUT_CKSEL_HFRC_DIV512 = 32, /*!< HFRC_DIV512 : HFRC / 512 value. */ + CLKGEN_CLKOUT_CKSEL_FLASH_CLK = 34, /*!< FLASH_CLK : Flash Clock value. */ + CLKGEN_CLKOUT_CKSEL_LFRC_DIV2 = 35, /*!< LFRC_DIV2 : LFRC / 2 value. */ + CLKGEN_CLKOUT_CKSEL_LFRC_DIV32 = 36, /*!< LFRC_DIV32 : LFRC / 32 value. */ + CLKGEN_CLKOUT_CKSEL_LFRC_DIV512 = 37, /*!< LFRC_DIV512 : LFRC / 512 value. */ + CLKGEN_CLKOUT_CKSEL_LFRC_DIV32K = 38, /*!< LFRC_DIV32K : LFRC / 32768 value. */ + CLKGEN_CLKOUT_CKSEL_XT_DIV256 = 39, /*!< XT_DIV256 : XT / 256 value. */ + CLKGEN_CLKOUT_CKSEL_XT_DIV8K = 40, /*!< XT_DIV8K : XT / 8192 value. */ + CLKGEN_CLKOUT_CKSEL_XT_DIV64K = 41, /*!< XT_DIV64K : XT / 2^16 value. */ + CLKGEN_CLKOUT_CKSEL_ULFRC_DIV16 = 42, /*!< ULFRC_DIV16 : Uncal LFRC / 16 value. */ + CLKGEN_CLKOUT_CKSEL_ULFRC_DIV128 = 43, /*!< ULFRC_DIV128 : Uncal LFRC / 128 value. */ + CLKGEN_CLKOUT_CKSEL_ULFRC_1Hz = 44, /*!< ULFRC_1Hz : Uncal LFRC / 1024 value. */ + CLKGEN_CLKOUT_CKSEL_ULFRC_DIV4K = 45, /*!< ULFRC_DIV4K : Uncal LFRC / 4096 value. */ + CLKGEN_CLKOUT_CKSEL_ULFRC_DIV1M = 46, /*!< ULFRC_DIV1M : Uncal LFRC / 2^20 value. */ + CLKGEN_CLKOUT_CKSEL_HFRC_DIV64K = 47, /*!< HFRC_DIV64K : HFRC / 2^16 value. */ + CLKGEN_CLKOUT_CKSEL_HFRC_DIV16M = 48, /*!< HFRC_DIV16M : HFRC / 2^24 value. */ + CLKGEN_CLKOUT_CKSEL_LFRC_DIV1M = 49, /*!< LFRC_DIV1M : LFRC / 2^20 value. */ + CLKGEN_CLKOUT_CKSEL_HFRCNE = 50, /*!< HFRCNE : HFRC (not autoenabled) value. */ + CLKGEN_CLKOUT_CKSEL_HFRCNE_DIV8 = 51, /*!< HFRCNE_DIV8 : HFRC / 8 (not autoenabled) value. */ + CLKGEN_CLKOUT_CKSEL_XTNE = 53, /*!< XTNE : XT (not autoenabled) value. */ + CLKGEN_CLKOUT_CKSEL_XTNE_DIV16 = 54, /*!< XTNE_DIV16 : XT / 16 (not autoenabled) value. */ + CLKGEN_CLKOUT_CKSEL_LFRCNE_DIV32 = 55, /*!< LFRCNE_DIV32 : LFRC / 32 (not autoenabled) value. */ + CLKGEN_CLKOUT_CKSEL_LFRCNE = 57, /*!< LFRCNE : LFRC (not autoenabled) - Default for undefined values + value. */ +} CLKGEN_CLKOUT_CKSEL_Enum; + +/* ======================================================== CLKKEY ========================================================= */ +/* ============================================= CLKGEN CLKKEY CLKKEY [0..31] ============================================== */ +typedef enum { /*!< CLKGEN_CLKKEY_CLKKEY */ + CLKGEN_CLKKEY_CLKKEY_Key = 71, /*!< Key : Key value. */ +} CLKGEN_CLKKEY_CLKKEY_Enum; + +/* ========================================================= CCTRL ========================================================= */ +/* ============================================== CLKGEN CCTRL CORESEL [0..0] ============================================== */ +typedef enum { /*!< CLKGEN_CCTRL_CORESEL */ + CLKGEN_CCTRL_CORESEL_HFRC = 0, /*!< HFRC : Core Clock is HFRC value. */ + CLKGEN_CCTRL_CORESEL_HFRC_DIV2 = 1, /*!< HFRC_DIV2 : Core Clock is HFRC / 2 value. */ +} CLKGEN_CCTRL_CORESEL_Enum; + +/* ======================================================== STATUS ========================================================= */ +/* ========================================================= HFADJ ========================================================= */ +/* ============================================ CLKGEN HFADJ HFADJGAIN [21..23] ============================================ */ +typedef enum { /*!< CLKGEN_HFADJ_HFADJGAIN */ + CLKGEN_HFADJ_HFADJGAIN_Gain_of_1 = 0, /*!< Gain_of_1 : HF Adjust with Gain of 1 value. */ + CLKGEN_HFADJ_HFADJGAIN_Gain_of_1_in_2 = 1, /*!< Gain_of_1_in_2 : HF Adjust with Gain of 0.5 value. */ + CLKGEN_HFADJ_HFADJGAIN_Gain_of_1_in_4 = 2, /*!< Gain_of_1_in_4 : HF Adjust with Gain of 0.25 value. */ + CLKGEN_HFADJ_HFADJGAIN_Gain_of_1_in_8 = 3, /*!< Gain_of_1_in_8 : HF Adjust with Gain of 0.125 value. */ + CLKGEN_HFADJ_HFADJGAIN_Gain_of_1_in_16 = 4, /*!< Gain_of_1_in_16 : HF Adjust with Gain of 0.0625 value. */ + CLKGEN_HFADJ_HFADJGAIN_Gain_of_1_in_32 = 5, /*!< Gain_of_1_in_32 : HF Adjust with Gain of 0.03125 value. */ +} CLKGEN_HFADJ_HFADJGAIN_Enum; + +/* ============================================ CLKGEN HFADJ HFWARMUP [20..20] ============================================= */ +typedef enum { /*!< CLKGEN_HFADJ_HFWARMUP */ + CLKGEN_HFADJ_HFWARMUP_1SEC = 0, /*!< 1SEC : Autoadjust XT warmup period = 1-2 seconds value. */ + CLKGEN_HFADJ_HFWARMUP_2SEC = 1, /*!< 2SEC : Autoadjust XT warmup period = 2-4 seconds value. */ +} CLKGEN_HFADJ_HFWARMUP_Enum; + +/* ============================================== CLKGEN HFADJ HFADJCK [1..3] ============================================== */ +typedef enum { /*!< CLKGEN_HFADJ_HFADJCK */ + CLKGEN_HFADJ_HFADJCK_4SEC = 0, /*!< 4SEC : Autoadjust repeat period = 4 seconds value. */ + CLKGEN_HFADJ_HFADJCK_16SEC = 1, /*!< 16SEC : Autoadjust repeat period = 16 seconds value. */ + CLKGEN_HFADJ_HFADJCK_32SEC = 2, /*!< 32SEC : Autoadjust repeat period = 32 seconds value. */ + CLKGEN_HFADJ_HFADJCK_64SEC = 3, /*!< 64SEC : Autoadjust repeat period = 64 seconds value. */ + CLKGEN_HFADJ_HFADJCK_128SEC = 4, /*!< 128SEC : Autoadjust repeat period = 128 seconds value. */ + CLKGEN_HFADJ_HFADJCK_256SEC = 5, /*!< 256SEC : Autoadjust repeat period = 256 seconds value. */ + CLKGEN_HFADJ_HFADJCK_512SEC = 6, /*!< 512SEC : Autoadjust repeat period = 512 seconds value. */ + CLKGEN_HFADJ_HFADJCK_1024SEC = 7, /*!< 1024SEC : Autoadjust repeat period = 1024 seconds value. */ +} CLKGEN_HFADJ_HFADJCK_Enum; + +/* ============================================== CLKGEN HFADJ HFADJEN [0..0] ============================================== */ +typedef enum { /*!< CLKGEN_HFADJ_HFADJEN */ + CLKGEN_HFADJ_HFADJEN_DIS = 0, /*!< DIS : Disable the HFRC adjustment value. */ + CLKGEN_HFADJ_HFADJEN_EN = 1, /*!< EN : Enable the HFRC adjustment value. */ +} CLKGEN_HFADJ_HFADJEN_Enum; + +/* ====================================================== CLOCKENSTAT ====================================================== */ +/* ======================================== CLKGEN CLOCKENSTAT CLOCKENSTAT [0..31] ========================================= */ +typedef enum { /*!< CLKGEN_CLOCKENSTAT_CLOCKENSTAT */ + CLKGEN_CLOCKENSTAT_CLOCKENSTAT_ADC_CLKEN = 1, /*!< ADC_CLKEN : Clock enable for the ADC. value. */ + CLKGEN_CLOCKENSTAT_CLOCKENSTAT_APBDMA_ACTIVITY_CLKEN = 2,/*!< APBDMA_ACTIVITY_CLKEN : Clock enable for the APBDMA ACTIVITY + value. */ + CLKGEN_CLOCKENSTAT_CLOCKENSTAT_APBDMA_AOH_CLKEN = 4,/*!< APBDMA_AOH_CLKEN : Clock enable for the APBDMA AOH DOMAIN value. */ + CLKGEN_CLOCKENSTAT_CLOCKENSTAT_APBDMA_AOL_CLKEN = 8,/*!< APBDMA_AOL_CLKEN : Clock enable for the APBDMA AOL DOMAIN value. */ + CLKGEN_CLOCKENSTAT_CLOCKENSTAT_APBDMA_APB_CLKEN = 16,/*!< APBDMA_APB_CLKEN : Clock enable for the APBDMA_APB value. */ + CLKGEN_CLOCKENSTAT_CLOCKENSTAT_APBDMA_BLEL_CLKEN = 32,/*!< APBDMA_BLEL_CLKEN : Clock enable for the APBDMA_BLEL value. */ + CLKGEN_CLOCKENSTAT_CLOCKENSTAT_APBDMA_HCPA_CLKEN = 64,/*!< APBDMA_HCPA_CLKEN : Clock enable for the APBDMA_HCPA value. */ + CLKGEN_CLOCKENSTAT_CLOCKENSTAT_APBDMA_HCPB_CLKEN = 128,/*!< APBDMA_HCPB_CLKEN : Clock enable for the APBDMA_HCPB value. */ + CLKGEN_CLOCKENSTAT_CLOCKENSTAT_APBDMA_HCPC_CLKEN = 256,/*!< APBDMA_HCPC_CLKEN : Clock enable for the APBDMA_HCPC value. */ + CLKGEN_CLOCKENSTAT_CLOCKENSTAT_APBDMA_MSPI_CLKEN = 512,/*!< APBDMA_MSPI_CLKEN : Clock enable for the APBDMA_MSPI value. */ + CLKGEN_CLOCKENSTAT_CLOCKENSTAT_APBDMA_PDM_CLKEN = 1024,/*!< APBDMA_PDM_CLKEN : Clock enable for the APBDMA_PDM value. */ + CLKGEN_CLOCKENSTAT_CLOCKENSTAT_BLEIF_CLK_CLKEN = 2048,/*!< BLEIF_CLK_CLKEN : Clock enable for the BLEIF value. */ + CLKGEN_CLOCKENSTAT_CLOCKENSTAT_BLEIF_CLK32K_CLKEN = 4096,/*!< BLEIF_CLK32K_CLKEN : Clock enable for the BLEIF 32khZ CLOCK + value. */ + CLKGEN_CLOCKENSTAT_CLOCKENSTAT_CTIMER_CLKEN = 8192,/*!< CTIMER_CLKEN : Clock enable for the CTIMER BLOCK value. */ + CLKGEN_CLOCKENSTAT_CLOCKENSTAT_CTIMER0A_CLKEN = 16384,/*!< CTIMER0A_CLKEN : Clock enable for the CTIMER0A value. */ + CLKGEN_CLOCKENSTAT_CLOCKENSTAT_CTIMER0B_CLKEN = 32768,/*!< CTIMER0B_CLKEN : Clock enable for the CTIMER0B value. */ + CLKGEN_CLOCKENSTAT_CLOCKENSTAT_CTIMER1A_CLKEN = 65536,/*!< CTIMER1A_CLKEN : Clock enable for the CTIMER1A value. */ + CLKGEN_CLOCKENSTAT_CLOCKENSTAT_CTIMER1B_CLKEN = 131072,/*!< CTIMER1B_CLKEN : Clock enable for the CTIMER1B value. */ + CLKGEN_CLOCKENSTAT_CLOCKENSTAT_CTIMER2A_CLKEN = 262144,/*!< CTIMER2A_CLKEN : Clock enable for the CTIMER2A value. */ + CLKGEN_CLOCKENSTAT_CLOCKENSTAT_CTIMER2B_CLKEN = 524288,/*!< CTIMER2B_CLKEN : Clock enable for the CTIMER2B value. */ + CLKGEN_CLOCKENSTAT_CLOCKENSTAT_CTIMER3A_CLKEN = 1048576,/*!< CTIMER3A_CLKEN : Clock enable for the CTIMER3A value. */ + CLKGEN_CLOCKENSTAT_CLOCKENSTAT_CTIMER3B_CLKEN = 2097152,/*!< CTIMER3B_CLKEN : Clock enable for the CTIMER3B value. */ + CLKGEN_CLOCKENSTAT_CLOCKENSTAT_CTIMER4A_CLKEN = 4194304,/*!< CTIMER4A_CLKEN : Clock enable for the CTIMER4A value. */ + CLKGEN_CLOCKENSTAT_CLOCKENSTAT_CTIMER4B_CLKEN = 8388608,/*!< CTIMER4B_CLKEN : Clock enable for the CTIMER4B value. */ + CLKGEN_CLOCKENSTAT_CLOCKENSTAT_CTIMER5A_CLKEN = 16777216,/*!< CTIMER5A_CLKEN : Clock enable for the CTIMER5A value. */ + CLKGEN_CLOCKENSTAT_CLOCKENSTAT_CTIMER5B_CLKEN = 33554432,/*!< CTIMER5B_CLKEN : Clock enable for the CTIMER5B value. */ + CLKGEN_CLOCKENSTAT_CLOCKENSTAT_CTIMER6A_CLKEN = 67108864,/*!< CTIMER6A_CLKEN : Clock enable for the CTIMER6A value. */ + CLKGEN_CLOCKENSTAT_CLOCKENSTAT_CTIMER6B_CLKEN = 134217728,/*!< CTIMER6B_CLKEN : Clock enable for the CTIMER6B value. */ + CLKGEN_CLOCKENSTAT_CLOCKENSTAT_CTIMER7A_CLKEN = 268435456,/*!< CTIMER7A_CLKEN : Clock enable for the CTIMER7A value. */ + CLKGEN_CLOCKENSTAT_CLOCKENSTAT_CTIMER7B_CLKEN = 536870912,/*!< CTIMER7B_CLKEN : Clock enable for the CTIMER7B value. */ + CLKGEN_CLOCKENSTAT_CLOCKENSTAT_DAP_CLKEN = 1073741824,/*!< DAP_CLKEN : Clock enable for the DAP value. */ + CLKGEN_CLOCKENSTAT_CLOCKENSTAT_IOMSTRIFC0_CLKEN = -2147483648,/*!< IOMSTRIFC0_CLKEN : Clock enable for the IOMSTRIFC0 value. */ +} CLKGEN_CLOCKENSTAT_CLOCKENSTAT_Enum; + +/* ===================================================== CLOCKEN2STAT ====================================================== */ +/* ======================================= CLKGEN CLOCKEN2STAT CLOCKEN2STAT [0..31] ======================================== */ +typedef enum { /*!< CLKGEN_CLOCKEN2STAT_CLOCKEN2STAT */ + CLKGEN_CLOCKEN2STAT_CLOCKEN2STAT_IOMSTRIFC1_CLKEN = 1,/*!< IOMSTRIFC1_CLKEN : Clock enable for the IO MASTER 1 IFC INTERFACE + value. */ + CLKGEN_CLOCKEN2STAT_CLOCKEN2STAT_IOMSTRIFC2_CLKEN = 2,/*!< IOMSTRIFC2_CLKEN : Clock enable for the IO MASTER 2 IFC INTERFACE + value. */ + CLKGEN_CLOCKEN2STAT_CLOCKEN2STAT_IOMSTRIFC3_CLKEN = 4,/*!< IOMSTRIFC3_CLKEN : Clock enable for the IO MASTER 3 IFC INTERFACE + value. */ + CLKGEN_CLOCKEN2STAT_CLOCKEN2STAT_IOMSTRIFC4_CLKEN = 8,/*!< IOMSTRIFC4_CLKEN : Clock enable for the IO MASTER 4 IFC INTERFACE + value. */ + CLKGEN_CLOCKEN2STAT_CLOCKEN2STAT_IOMSTRIFC5_CLKEN = 16,/*!< IOMSTRIFC5_CLKEN : Clock enable for the IO MASTER 5 IFC INTERFACE + value. */ + CLKGEN_CLOCKEN2STAT_CLOCKEN2STAT_PDM_CLKEN = 32,/*!< PDM_CLKEN : Clock enable for the PDM value. */ + CLKGEN_CLOCKEN2STAT_CLOCKEN2STAT_PDMIFC_CLKEN = 64,/*!< PDMIFC_CLKEN : Clock enable for the PDM INTERFACE value. */ + CLKGEN_CLOCKEN2STAT_CLOCKEN2STAT_PWRCTRL_CLKEN = 128,/*!< PWRCTRL_CLKEN : Clock enable for the PWRCTRL value. */ + CLKGEN_CLOCKEN2STAT_CLOCKEN2STAT_RSTGEN_CLKEN = 256,/*!< RSTGEN_CLKEN : Clock enable for the RSTGEN value. */ + CLKGEN_CLOCKEN2STAT_CLOCKEN2STAT_SCARD_CLKEN = 512,/*!< SCARD_CLKEN : Clock enable for the SCARD value. */ + CLKGEN_CLOCKEN2STAT_CLOCKEN2STAT_SCARD_ALTAPB_CLKEN = 1024,/*!< SCARD_ALTAPB_CLKEN : Clock enable for the SCARD ALTAPB value. */ + CLKGEN_CLOCKEN2STAT_CLOCKEN2STAT_STIMER_CNT_CLKEN = 2048,/*!< STIMER_CNT_CLKEN : Clock enable for the STIMER_CNT_CLKEN value. */ + CLKGEN_CLOCKEN2STAT_CLOCKEN2STAT_TPIU_CLKEN = 4096,/*!< TPIU_CLKEN : Clock enable for the TPIU_CLKEN value. */ + CLKGEN_CLOCKEN2STAT_CLOCKEN2STAT_UART0HF_CLKEN = 8192,/*!< UART0HF_CLKEN : Clock enable for the UART0 HF value. */ + CLKGEN_CLOCKEN2STAT_CLOCKEN2STAT_UART1HF_CLKEN = 16384,/*!< UART1HF_CLKEN : Clock enable for the UART1 HF value. */ + CLKGEN_CLOCKEN2STAT_CLOCKEN2STAT_XT_32KHZ_EN = 1073741824,/*!< XT_32KHZ_EN : Clock enable for the XT 32KHZ value. */ + CLKGEN_CLOCKEN2STAT_CLOCKEN2STAT_FORCEHFRC = -2147483648,/*!< FORCEHFRC : HFRC is forced on Status. value. */ +} CLKGEN_CLOCKEN2STAT_CLOCKEN2STAT_Enum; + +/* ===================================================== CLOCKEN3STAT ====================================================== */ +/* ======================================= CLKGEN CLOCKEN3STAT CLOCKEN3STAT [0..31] ======================================== */ +typedef enum { /*!< CLKGEN_CLOCKEN3STAT_CLOCKEN3STAT */ + CLKGEN_CLOCKEN3STAT_CLOCKEN3STAT_XTAL_enabled = 16777216,/*!< XTAL_enabled : XTAL is enabled value. */ + CLKGEN_CLOCKEN3STAT_CLOCKEN3STAT_HFRC_enabled = 33554432,/*!< HFRC_enabled : HFRC is enabled value. */ + CLKGEN_CLOCKEN3STAT_CLOCKEN3STAT_HFADJEN = 67108864,/*!< HFADJEN : HFRC Adjust enabled value. */ + CLKGEN_CLOCKEN3STAT_CLOCKEN3STAT_HFRC_en_out = 134217728,/*!< HFRC_en_out : HFRC Enabled out value. */ + CLKGEN_CLOCKEN3STAT_CLOCKEN3STAT_RTC_XT = 268435456,/*!< RTC_XT : RTC use XT value. */ + CLKGEN_CLOCKEN3STAT_CLOCKEN3STAT_clkout_xtal_en = 536870912,/*!< clkout_xtal_en : XTAL clkout enabled value. */ + CLKGEN_CLOCKEN3STAT_CLOCKEN3STAT_clkout_hfrc_en = 1073741824,/*!< clkout_hfrc_en : HFRC clkout enabled value. */ + CLKGEN_CLOCKEN3STAT_CLOCKEN3STAT_flashclk_en = -2147483648,/*!< flashclk_en : Flash clk is enabled value. */ +} CLKGEN_CLOCKEN3STAT_CLOCKEN3STAT_Enum; + +/* ======================================================= FREQCTRL ======================================================== */ +/* ============================================ CLKGEN FREQCTRL BURSTREQ [0..0] ============================================ */ +typedef enum { /*!< CLKGEN_FREQCTRL_BURSTREQ */ + CLKGEN_FREQCTRL_BURSTREQ_DIS = 0, /*!< DIS : Frequency for ARM core stays at 48MHz value. */ + CLKGEN_FREQCTRL_BURSTREQ_EN = 1, /*!< EN : Frequency for ARM core is increased to 96MHz value. */ +} CLKGEN_FREQCTRL_BURSTREQ_Enum; + +/* ===================================================== BLEBUCKTONADJ ===================================================== */ +/* ===================================== CLKGEN BLEBUCKTONADJ ZEROLENDETECTEN [27..27] ===================================== */ +typedef enum { /*!< CLKGEN_BLEBUCKTONADJ_ZEROLENDETECTEN */ + CLKGEN_BLEBUCKTONADJ_ZEROLENDETECTEN_DIS = 0, /*!< DIS : Disable Zero Length Detect value. */ + CLKGEN_BLEBUCKTONADJ_ZEROLENDETECTEN_EN = 1, /*!< EN : Enable Zero Length Detect value. */ +} CLKGEN_BLEBUCKTONADJ_ZEROLENDETECTEN_Enum; + +/* ==================================== CLKGEN BLEBUCKTONADJ ZEROLENDETECTTRIM [23..26] ==================================== */ +typedef enum { /*!< CLKGEN_BLEBUCKTONADJ_ZEROLENDETECTTRIM */ + CLKGEN_BLEBUCKTONADJ_ZEROLENDETECTTRIM_SetF = 15,/*!< SetF : Indicator send when the BLE BUCK asserts blebuck_comp1 + for about 81us (10 percent margin of error) or more value. */ + CLKGEN_BLEBUCKTONADJ_ZEROLENDETECTTRIM_SetE = 14,/*!< SetE : Indicator send when the BLE BUCK asserts blebuck_comp1 + for about 75.6us (10 percent margin of error) or more value. */ + CLKGEN_BLEBUCKTONADJ_ZEROLENDETECTTRIM_SetD = 13,/*!< SetD : Indicator send when the BLE BUCK asserts blebuck_comp1 + for about 70.2us (10 percent margin of error) or more value. */ + CLKGEN_BLEBUCKTONADJ_ZEROLENDETECTTRIM_SetC = 12,/*!< SetC : Indicator send when the BLE BUCK asserts blebuck_comp1 + for about 64.8us (10 percent margin of error) or more value. */ + CLKGEN_BLEBUCKTONADJ_ZEROLENDETECTTRIM_SetB = 11,/*!< SetB : Indicator send when the BLE BUCK asserts blebuck_comp1 + for about 59.4us (10 percent margin of error) or more value. */ + CLKGEN_BLEBUCKTONADJ_ZEROLENDETECTTRIM_SetA = 10,/*!< SetA : Indicator send when the BLE BUCK asserts blebuck_comp1 + for about 54.0us (10 percent margin of error) or more value. */ + CLKGEN_BLEBUCKTONADJ_ZEROLENDETECTTRIM_Set9 = 9,/*!< Set9 : Indicator send when the BLE BUCK asserts blebuck_comp1 + for about 48.6us (10 percent margin of error) or more value. */ + CLKGEN_BLEBUCKTONADJ_ZEROLENDETECTTRIM_Set8 = 8,/*!< Set8 : Indicator send when the BLE BUCK asserts blebuck_comp1 + for about 43.2us (10 percent margin of error) or more value. */ + CLKGEN_BLEBUCKTONADJ_ZEROLENDETECTTRIM_Set7 = 7,/*!< Set7 : Indicator send when the BLE BUCK asserts blebuck_comp1 + for about 37.8us (10 percent margin of error) or more value. */ + CLKGEN_BLEBUCKTONADJ_ZEROLENDETECTTRIM_Set6 = 6,/*!< Set6 : Indicator send when the BLE BUCK asserts blebuck_comp1 + for about 32.4us (10 percent margin of error) or more value. */ + CLKGEN_BLEBUCKTONADJ_ZEROLENDETECTTRIM_Set5 = 5,/*!< Set5 : Indicator send when the BLE BUCK asserts blebuck_comp1 + for about 27.0us (10 percent margin of error) or more value. */ + CLKGEN_BLEBUCKTONADJ_ZEROLENDETECTTRIM_Set4 = 4,/*!< Set4 : Indicator send when the BLE BUCK asserts blebuck_comp1 + for about 21.6us (10 percent margin of error) or more value. */ + CLKGEN_BLEBUCKTONADJ_ZEROLENDETECTTRIM_Set3 = 3,/*!< Set3 : Indicator send when the BLE BUCK asserts blebuck_comp1 + for about 16.2us (10 percent margin of error) or more value. */ + CLKGEN_BLEBUCKTONADJ_ZEROLENDETECTTRIM_Set2 = 2,/*!< Set2 : Indicator send when the BLE BUCK asserts blebuck_comp1 + for about 10.8us (10 percent margin of error) or more value. */ + CLKGEN_BLEBUCKTONADJ_ZEROLENDETECTTRIM_Set1 = 1,/*!< Set1 : Indicator send when the BLE BUCK asserts blebuck_comp1 + for about 5.4us (10 percent margin of error) or more value. */ + CLKGEN_BLEBUCKTONADJ_ZEROLENDETECTTRIM_Set0 = 0,/*!< Set0 : Indicator send when the BLE BUCK asserts blebuck_comp1 + for about 2.0us (10 percent margin of error) or more value. */ +} CLKGEN_BLEBUCKTONADJ_ZEROLENDETECTTRIM_Enum; + +/* ======================================= CLKGEN BLEBUCKTONADJ TONADJUSTEN [22..22] ======================================= */ +typedef enum { /*!< CLKGEN_BLEBUCKTONADJ_TONADJUSTEN */ + CLKGEN_BLEBUCKTONADJ_TONADJUSTEN_DIS = 0, /*!< DIS : Disable Adjust for BLE BUCK TON trim value. */ + CLKGEN_BLEBUCKTONADJ_TONADJUSTEN_EN = 1, /*!< EN : Enable Adjust for BLE BUCK TON trim value. */ +} CLKGEN_BLEBUCKTONADJ_TONADJUSTEN_Enum; + +/* ===================================== CLKGEN BLEBUCKTONADJ TONADJUSTPERIOD [20..21] ===================================== */ +typedef enum { /*!< CLKGEN_BLEBUCKTONADJ_TONADJUSTPERIOD */ + CLKGEN_BLEBUCKTONADJ_TONADJUSTPERIOD_HFRC_3KHz = 3,/*!< HFRC_3KHz : Adjust done for every 1 3KHz period value. */ + CLKGEN_BLEBUCKTONADJ_TONADJUSTPERIOD_HFRC_12KHz = 2,/*!< HFRC_12KHz : Adjust done for every 1 12KHz period value. */ + CLKGEN_BLEBUCKTONADJ_TONADJUSTPERIOD_HFRC_47KHz = 1,/*!< HFRC_47KHz : Adjust done for every 1 47KHz period value. */ + CLKGEN_BLEBUCKTONADJ_TONADJUSTPERIOD_HFRC_94KHz = 0,/*!< HFRC_94KHz : Adjust done for every 1 94KHz period value. */ +} CLKGEN_BLEBUCKTONADJ_TONADJUSTPERIOD_Enum; + +/* ======================================================= INTRPTEN ======================================================== */ +/* ====================================================== INTRPTSTAT ======================================================= */ +/* ======================================================= INTRPTCLR ======================================================= */ +/* ======================================================= INTRPTSET ======================================================= */ + + +/* =========================================================================================================================== */ +/* ================ CTIMER ================ */ +/* =========================================================================================================================== */ + +/* ========================================================= TMR0 ========================================================== */ +/* ======================================================== CMPRA0 ========================================================= */ +/* ======================================================== CMPRB0 ========================================================= */ +/* ========================================================= CTRL0 ========================================================= */ +/* ============================================= CTIMER CTRL0 CTLINK0 [31..31] ============================================= */ +typedef enum { /*!< CTIMER_CTRL0_CTLINK0 */ + CTIMER_CTRL0_CTLINK0_TWO_16BIT_TIMERS = 0, /*!< TWO_16BIT_TIMERS : Use A0/B0 timers as two independent 16-bit + timers (default). value. */ + CTIMER_CTRL0_CTLINK0_32BIT_TIMER = 1, /*!< 32BIT_TIMER : Link A0/B0 timers into a single 32-bit timer. + value. */ +} CTIMER_CTRL0_CTLINK0_Enum; + +/* ============================================ CTIMER CTRL0 TMRB0POL [28..28] ============================================= */ +typedef enum { /*!< CTIMER_CTRL0_TMRB0POL */ + CTIMER_CTRL0_TMRB0POL_NORMAL = 0, /*!< NORMAL : The polarity of the TMRPINB0 pin is the same as the + timer output. value. */ + CTIMER_CTRL0_TMRB0POL_INVERTED = 1, /*!< INVERTED : The polarity of the TMRPINB0 pin is the inverse of + the timer output. value. */ +} CTIMER_CTRL0_TMRB0POL_Enum; + +/* ============================================ CTIMER CTRL0 TMRB0CLR [27..27] ============================================= */ +typedef enum { /*!< CTIMER_CTRL0_TMRB0CLR */ + CTIMER_CTRL0_TMRB0CLR_RUN = 0, /*!< RUN : Allow counter/timer B0 to run value. */ + CTIMER_CTRL0_TMRB0CLR_CLEAR = 1, /*!< CLEAR : Holds counter/timer B0 at 0x0000. value. */ +} CTIMER_CTRL0_TMRB0CLR_Enum; + +/* ============================================ CTIMER CTRL0 TMRB0IE1 [26..26] ============================================= */ +typedef enum { /*!< CTIMER_CTRL0_TMRB0IE1 */ + CTIMER_CTRL0_TMRB0IE1_DIS = 0, /*!< DIS : Disable counter/timer B0 from generating an interrupt + based on COMPR1. value. */ + CTIMER_CTRL0_TMRB0IE1_EN = 1, /*!< EN : Enable counter/timer B0 to generate an interrupt based + on COMPR1. value. */ +} CTIMER_CTRL0_TMRB0IE1_Enum; + +/* ============================================ CTIMER CTRL0 TMRB0IE0 [25..25] ============================================= */ +typedef enum { /*!< CTIMER_CTRL0_TMRB0IE0 */ + CTIMER_CTRL0_TMRB0IE0_DIS = 0, /*!< DIS : Disable counter/timer B0 from generating an interrupt + based on COMPR0. value. */ + CTIMER_CTRL0_TMRB0IE0_EN = 1, /*!< EN : Enable counter/timer B0 to generate an interrupt based + on COMPR0 value. */ +} CTIMER_CTRL0_TMRB0IE0_Enum; + +/* ============================================= CTIMER CTRL0 TMRB0FN [22..24] ============================================= */ +typedef enum { /*!< CTIMER_CTRL0_TMRB0FN */ + CTIMER_CTRL0_TMRB0FN_SINGLECOUNT = 0, /*!< SINGLECOUNT : Single count (output toggles and sticks). Count + to CMPR0B0, stop. value. */ + CTIMER_CTRL0_TMRB0FN_REPEATEDCOUNT = 1, /*!< REPEATEDCOUNT : Repeated count (periodic 1-clock-cycle-wide + pulses). Count to CMPR0B0, restart. value. */ + CTIMER_CTRL0_TMRB0FN_PULSE_ONCE = 2, /*!< PULSE_ONCE : Pulse once (aka one-shot). Count to CMPR0B0, assert, + count to CMPR1B0, deassert, stop. value. */ + CTIMER_CTRL0_TMRB0FN_PULSE_CONT = 3, /*!< PULSE_CONT : Pulse continously. Count to CMPR0B0, assert, count + to CMPR1B0, deassert, restart. value. */ + CTIMER_CTRL0_TMRB0FN_SINGLEPATTERN = 4, /*!< SINGLEPATTERN : Single pattern. value. */ + CTIMER_CTRL0_TMRB0FN_REPEATPATTERN = 5, /*!< REPEATPATTERN : Repeated pattern. value. */ + CTIMER_CTRL0_TMRB0FN_CONTINUOUS = 6, /*!< CONTINUOUS : Continuous run (aka Free Run). Count continuously. + value. */ + CTIMER_CTRL0_TMRB0FN_ALTPWN = 7, /*!< ALTPWN : Alternate PWM value. */ +} CTIMER_CTRL0_TMRB0FN_Enum; + +/* ============================================ CTIMER CTRL0 TMRB0CLK [17..21] ============================================= */ +typedef enum { /*!< CTIMER_CTRL0_TMRB0CLK */ + CTIMER_CTRL0_TMRB0CLK_TMRPIN = 0, /*!< TMRPIN : Clock source is TMRPINB. value. */ + CTIMER_CTRL0_TMRB0CLK_HFRC_DIV4 = 1, /*!< HFRC_DIV4 : Clock source is the HFRC / 4 value. */ + CTIMER_CTRL0_TMRB0CLK_HFRC_DIV16 = 2, /*!< HFRC_DIV16 : Clock source is HFRC / 16 value. */ + CTIMER_CTRL0_TMRB0CLK_HFRC_DIV256 = 3, /*!< HFRC_DIV256 : Clock source is HFRC / 256 value. */ + CTIMER_CTRL0_TMRB0CLK_HFRC_DIV1024 = 4, /*!< HFRC_DIV1024 : Clock source is HFRC / 1024 value. */ + CTIMER_CTRL0_TMRB0CLK_HFRC_DIV4K = 5, /*!< HFRC_DIV4K : Clock source is HFRC / 4096 value. */ + CTIMER_CTRL0_TMRB0CLK_XT = 6, /*!< XT : Clock source is the XT (uncalibrated). value. */ + CTIMER_CTRL0_TMRB0CLK_XT_DIV2 = 7, /*!< XT_DIV2 : Clock source is XT / 2 value. */ + CTIMER_CTRL0_TMRB0CLK_XT_DIV16 = 8, /*!< XT_DIV16 : Clock source is XT / 16 value. */ + CTIMER_CTRL0_TMRB0CLK_XT_DIV128 = 9, /*!< XT_DIV128 : Clock source is XT / 128 value. */ + CTIMER_CTRL0_TMRB0CLK_LFRC_DIV2 = 10, /*!< LFRC_DIV2 : Clock source is LFRC / 2 value. */ + CTIMER_CTRL0_TMRB0CLK_LFRC_DIV32 = 11, /*!< LFRC_DIV32 : Clock source is LFRC / 32 value. */ + CTIMER_CTRL0_TMRB0CLK_LFRC_DIV1K = 12, /*!< LFRC_DIV1K : Clock source is LFRC / 1024 value. */ + CTIMER_CTRL0_TMRB0CLK_LFRC = 13, /*!< LFRC : Clock source is LFRC value. */ + CTIMER_CTRL0_TMRB0CLK_RTC_100HZ = 14, /*!< RTC_100HZ : Clock source is 100 Hz from the current RTC oscillator. + value. */ + CTIMER_CTRL0_TMRB0CLK_HCLK = 15, /*!< HCLK : Clock source is HCLK. value. */ + CTIMER_CTRL0_TMRB0CLK_XT_DIV4 = 16, /*!< XT_DIV4 : Clock source is XT / 4 value. */ + CTIMER_CTRL0_TMRB0CLK_XT_DIV8 = 17, /*!< XT_DIV8 : Clock source is XT / 8 value. */ + CTIMER_CTRL0_TMRB0CLK_XT_DIV32 = 18, /*!< XT_DIV32 : Clock source is XT / 32 value. */ + CTIMER_CTRL0_TMRB0CLK_CTMRA0 = 20, /*!< CTMRA0 : Clock source is CTIMERA0 OUT. value. */ + CTIMER_CTRL0_TMRB0CLK_CTMRB1 = 21, /*!< CTMRB1 : Clock source is CTIMERB1 OUT. value. */ + CTIMER_CTRL0_TMRB0CLK_CTMRA1 = 22, /*!< CTMRA1 : Clock source is CTIMERA1 OUT. value. */ + CTIMER_CTRL0_TMRB0CLK_CTMRA2 = 23, /*!< CTMRA2 : Clock source is CTIMERA2 OUT. value. */ + CTIMER_CTRL0_TMRB0CLK_CTMRB2 = 24, /*!< CTMRB2 : Clock source is CTIMERB2 OUT. value. */ + CTIMER_CTRL0_TMRB0CLK_CTMRB3 = 25, /*!< CTMRB3 : Clock source is CTIMERB3 OUT. value. */ + CTIMER_CTRL0_TMRB0CLK_CTMRB4 = 26, /*!< CTMRB4 : Clock source is CTIMERB4 OUT. value. */ + CTIMER_CTRL0_TMRB0CLK_CTMRB5 = 27, /*!< CTMRB5 : Clock source is CTIMERB5 OUT. value. */ + CTIMER_CTRL0_TMRB0CLK_CTMRB6 = 28, /*!< CTMRB6 : Clock source is CTIMERB6 OUT. value. */ + CTIMER_CTRL0_TMRB0CLK_BUCKBLE = 29, /*!< BUCKBLE : Clock source is BLE buck converter TON pulses. value. */ + CTIMER_CTRL0_TMRB0CLK_BUCKB = 30, /*!< BUCKB : Clock source is Memory buck converter TON pulses. value. */ + CTIMER_CTRL0_TMRB0CLK_BUCKA = 31, /*!< BUCKA : Clock source is CPU buck converter TON pulses. value. */ +} CTIMER_CTRL0_TMRB0CLK_Enum; + +/* ============================================= CTIMER CTRL0 TMRB0EN [16..16] ============================================= */ +typedef enum { /*!< CTIMER_CTRL0_TMRB0EN */ + CTIMER_CTRL0_TMRB0EN_DIS = 0, /*!< DIS : Counter/Timer B0 Disable. value. */ + CTIMER_CTRL0_TMRB0EN_EN = 1, /*!< EN : Counter/Timer B0 Enable. value. */ +} CTIMER_CTRL0_TMRB0EN_Enum; + +/* ============================================ CTIMER CTRL0 TMRA0POL [12..12] ============================================= */ +typedef enum { /*!< CTIMER_CTRL0_TMRA0POL */ + CTIMER_CTRL0_TMRA0POL_NORMAL = 0, /*!< NORMAL : The polarity of the TMRPINA0 pin is the same as the + timer output. value. */ + CTIMER_CTRL0_TMRA0POL_INVERTED = 1, /*!< INVERTED : The polarity of the TMRPINA0 pin is the inverse of + the timer output. value. */ +} CTIMER_CTRL0_TMRA0POL_Enum; + +/* ============================================ CTIMER CTRL0 TMRA0CLR [11..11] ============================================= */ +typedef enum { /*!< CTIMER_CTRL0_TMRA0CLR */ + CTIMER_CTRL0_TMRA0CLR_RUN = 0, /*!< RUN : Allow counter/timer A0 to run value. */ + CTIMER_CTRL0_TMRA0CLR_CLEAR = 1, /*!< CLEAR : Holds counter/timer A0 at 0x0000. value. */ +} CTIMER_CTRL0_TMRA0CLR_Enum; + +/* ============================================ CTIMER CTRL0 TMRA0IE1 [10..10] ============================================= */ +typedef enum { /*!< CTIMER_CTRL0_TMRA0IE1 */ + CTIMER_CTRL0_TMRA0IE1_DIS = 0, /*!< DIS : Disable counter/timer A0 from generating an interrupt + based on COMPR1. value. */ + CTIMER_CTRL0_TMRA0IE1_EN = 1, /*!< EN : Enable counter/timer A0 to generate an interrupt based + on COMPR1. value. */ +} CTIMER_CTRL0_TMRA0IE1_Enum; + +/* ============================================= CTIMER CTRL0 TMRA0IE0 [9..9] ============================================== */ +typedef enum { /*!< CTIMER_CTRL0_TMRA0IE0 */ + CTIMER_CTRL0_TMRA0IE0_DIS = 0, /*!< DIS : Disable counter/timer A0 from generating an interrupt + based on COMPR0. value. */ + CTIMER_CTRL0_TMRA0IE0_EN = 1, /*!< EN : Enable counter/timer A0 to generate an interrupt based + on COMPR0. value. */ +} CTIMER_CTRL0_TMRA0IE0_Enum; + +/* ============================================== CTIMER CTRL0 TMRA0FN [6..8] ============================================== */ +typedef enum { /*!< CTIMER_CTRL0_TMRA0FN */ + CTIMER_CTRL0_TMRA0FN_SINGLECOUNT = 0, /*!< SINGLECOUNT : Single count (output toggles and sticks). Count + to CMPR0A0, stop. value. */ + CTIMER_CTRL0_TMRA0FN_REPEATEDCOUNT = 1, /*!< REPEATEDCOUNT : Repeated count (periodic 1-clock-cycle-wide + pulses). Count to CMPR0A0, restart. value. */ + CTIMER_CTRL0_TMRA0FN_PULSE_ONCE = 2, /*!< PULSE_ONCE : Pulse once (aka one-shot). Count to CMPR0A0, assert, + count to CMPR1A0, deassert, stop. value. */ + CTIMER_CTRL0_TMRA0FN_PULSE_CONT = 3, /*!< PULSE_CONT : Pulse continously. Count to CMPR0A0, assert, count + to CMPR1A0, deassert, restart. value. */ + CTIMER_CTRL0_TMRA0FN_SINGLEPATTERN = 4, /*!< SINGLEPATTERN : Single pattern. value. */ + CTIMER_CTRL0_TMRA0FN_REPEATPATTERN = 5, /*!< REPEATPATTERN : Repeated pattern. value. */ + CTIMER_CTRL0_TMRA0FN_CONTINUOUS = 6, /*!< CONTINUOUS : Continuous run (aka Free Run). Count continuously. + value. */ + CTIMER_CTRL0_TMRA0FN_ALTPWN = 7, /*!< ALTPWN : Alternate PWM value. */ +} CTIMER_CTRL0_TMRA0FN_Enum; + +/* ============================================= CTIMER CTRL0 TMRA0CLK [1..5] ============================================== */ +typedef enum { /*!< CTIMER_CTRL0_TMRA0CLK */ + CTIMER_CTRL0_TMRA0CLK_TMRPIN = 0, /*!< TMRPIN : Clock source is TMRPINA. value. */ + CTIMER_CTRL0_TMRA0CLK_HFRC_DIV4 = 1, /*!< HFRC_DIV4 : Clock source is the HFRC / 4 value. */ + CTIMER_CTRL0_TMRA0CLK_HFRC_DIV16 = 2, /*!< HFRC_DIV16 : Clock source is HFRC / 16 value. */ + CTIMER_CTRL0_TMRA0CLK_HFRC_DIV256 = 3, /*!< HFRC_DIV256 : Clock source is HFRC / 256 value. */ + CTIMER_CTRL0_TMRA0CLK_HFRC_DIV1024 = 4, /*!< HFRC_DIV1024 : Clock source is HFRC / 1024 value. */ + CTIMER_CTRL0_TMRA0CLK_HFRC_DIV4K = 5, /*!< HFRC_DIV4K : Clock source is HFRC / 4096 value. */ + CTIMER_CTRL0_TMRA0CLK_XT = 6, /*!< XT : Clock source is the XT (uncalibrated). value. */ + CTIMER_CTRL0_TMRA0CLK_XT_DIV2 = 7, /*!< XT_DIV2 : Clock source is XT / 2 value. */ + CTIMER_CTRL0_TMRA0CLK_XT_DIV16 = 8, /*!< XT_DIV16 : Clock source is XT / 16 value. */ + CTIMER_CTRL0_TMRA0CLK_XT_DIV128 = 9, /*!< XT_DIV128 : Clock source is XT / 128 value. */ + CTIMER_CTRL0_TMRA0CLK_LFRC_DIV2 = 10, /*!< LFRC_DIV2 : Clock source is LFRC / 2 value. */ + CTIMER_CTRL0_TMRA0CLK_LFRC_DIV32 = 11, /*!< LFRC_DIV32 : Clock source is LFRC / 32 value. */ + CTIMER_CTRL0_TMRA0CLK_LFRC_DIV1K = 12, /*!< LFRC_DIV1K : Clock source is LFRC / 1024 value. */ + CTIMER_CTRL0_TMRA0CLK_LFRC = 13, /*!< LFRC : Clock source is LFRC value. */ + CTIMER_CTRL0_TMRA0CLK_RTC_100HZ = 14, /*!< RTC_100HZ : Clock source is 100 Hz from the current RTC oscillator. + value. */ + CTIMER_CTRL0_TMRA0CLK_HCLK_DIV4 = 15, /*!< HCLK_DIV4 : Clock source is HCLK / 4. value. */ + CTIMER_CTRL0_TMRA0CLK_XT_DIV4 = 16, /*!< XT_DIV4 : Clock source is XT / 4 value. */ + CTIMER_CTRL0_TMRA0CLK_XT_DIV8 = 17, /*!< XT_DIV8 : Clock source is XT / 8 value. */ + CTIMER_CTRL0_TMRA0CLK_XT_DIV32 = 18, /*!< XT_DIV32 : Clock source is XT / 32 value. */ + CTIMER_CTRL0_TMRA0CLK_CTMRB0 = 20, /*!< CTMRB0 : Clock source is CTIMERB0 OUT. value. */ + CTIMER_CTRL0_TMRA0CLK_CTMRA1 = 21, /*!< CTMRA1 : Clock source is CTIMERA1 OUT. value. */ + CTIMER_CTRL0_TMRA0CLK_CTMRB1 = 22, /*!< CTMRB1 : Clock source is CTIMERB1 OUT. value. */ + CTIMER_CTRL0_TMRA0CLK_CTMRA2 = 23, /*!< CTMRA2 : Clock source is CTIMERA2 OUT. value. */ + CTIMER_CTRL0_TMRA0CLK_CTMRB2 = 24, /*!< CTMRB2 : Clock source is CTIMERB2 OUT. value. */ + CTIMER_CTRL0_TMRA0CLK_CTMRB3 = 25, /*!< CTMRB3 : Clock source is CTIMERB3 OUT. value. */ + CTIMER_CTRL0_TMRA0CLK_CTMRB4 = 26, /*!< CTMRB4 : Clock source is CTIMERB4 OUT. value. */ + CTIMER_CTRL0_TMRA0CLK_CTMRB5 = 27, /*!< CTMRB5 : Clock source is CTIMERB5 OUT. value. */ + CTIMER_CTRL0_TMRA0CLK_CTMRB6 = 28, /*!< CTMRB6 : Clock source is CTIMERB6 OUT. value. */ + CTIMER_CTRL0_TMRA0CLK_BUCKBLE = 29, /*!< BUCKBLE : Clock source is BLE buck converter TON pulses. value. */ + CTIMER_CTRL0_TMRA0CLK_BUCKB = 30, /*!< BUCKB : Clock source is Memory buck converter TON pulses. value. */ + CTIMER_CTRL0_TMRA0CLK_BUCKA = 31, /*!< BUCKA : Clock source is CPU buck converter TON pulses. value. */ +} CTIMER_CTRL0_TMRA0CLK_Enum; + +/* ============================================== CTIMER CTRL0 TMRA0EN [0..0] ============================================== */ +typedef enum { /*!< CTIMER_CTRL0_TMRA0EN */ + CTIMER_CTRL0_TMRA0EN_DIS = 0, /*!< DIS : Counter/Timer A0 Disable. value. */ + CTIMER_CTRL0_TMRA0EN_EN = 1, /*!< EN : Counter/Timer A0 Enable. value. */ +} CTIMER_CTRL0_TMRA0EN_Enum; + +/* ======================================================= CMPRAUXA0 ======================================================= */ +/* ======================================================= CMPRAUXB0 ======================================================= */ +/* ========================================================= AUX0 ========================================================== */ +/* ============================================ CTIMER AUX0 TMRB0EN23 [30..30] ============================================= */ +typedef enum { /*!< CTIMER_AUX0_TMRB0EN23 */ + CTIMER_AUX0_TMRB0EN23_DIS = 1, /*!< DIS : Disable enhanced functions. value. */ + CTIMER_AUX0_TMRB0EN23_EN = 0, /*!< EN : Enable enhanced functions. value. */ +} CTIMER_AUX0_TMRB0EN23_Enum; + +/* ============================================ CTIMER AUX0 TMRB0POL23 [29..29] ============================================ */ +typedef enum { /*!< CTIMER_AUX0_TMRB0POL23 */ + CTIMER_AUX0_TMRB0POL23_NORM = 0, /*!< NORM : Upper output normal polarity value. */ + CTIMER_AUX0_TMRB0POL23_INV = 1, /*!< INV : Upper output inverted polarity. value. */ +} CTIMER_AUX0_TMRB0POL23_Enum; + +/* ============================================ CTIMER AUX0 TMRB0TINV [28..28] ============================================= */ +typedef enum { /*!< CTIMER_AUX0_TMRB0TINV */ + CTIMER_AUX0_TMRB0TINV_DIS = 0, /*!< DIS : Disable invert on trigger value. */ + CTIMER_AUX0_TMRB0TINV_EN = 1, /*!< EN : Enable invert on trigger value. */ +} CTIMER_AUX0_TMRB0TINV_Enum; + +/* =========================================== CTIMER AUX0 TMRB0NOSYNC [27..27] ============================================ */ +typedef enum { /*!< CTIMER_AUX0_TMRB0NOSYNC */ + CTIMER_AUX0_TMRB0NOSYNC_DIS = 0, /*!< DIS : Synchronization on source clock value. */ + CTIMER_AUX0_TMRB0NOSYNC_NOSYNC = 1, /*!< NOSYNC : No synchronization on source clock value. */ +} CTIMER_AUX0_TMRB0NOSYNC_Enum; + +/* ============================================ CTIMER AUX0 TMRB0TRIG [23..26] ============================================= */ +typedef enum { /*!< CTIMER_AUX0_TMRB0TRIG */ + CTIMER_AUX0_TMRB0TRIG_DIS = 0, /*!< DIS : Trigger source is disabled. value. */ + CTIMER_AUX0_TMRB0TRIG_A0OUT = 1, /*!< A0OUT : Trigger source is CTIMERA0 OUT. value. */ + CTIMER_AUX0_TMRB0TRIG_B3OUT = 2, /*!< B3OUT : Trigger source is CTIMERB3 OUT. value. */ + CTIMER_AUX0_TMRB0TRIG_A3OUT = 3, /*!< A3OUT : Trigger source is CTIMERA3 OUT. value. */ + CTIMER_AUX0_TMRB0TRIG_B2OUT = 4, /*!< B2OUT : Trigger source is CTIMERB2 OUT. value. */ + CTIMER_AUX0_TMRB0TRIG_B5OUT = 5, /*!< B5OUT : Trigger source is CTIMERB5 OUT. value. */ + CTIMER_AUX0_TMRB0TRIG_A4OUT = 6, /*!< A4OUT : Trigger source is CTIMERA4 OUT. value. */ + CTIMER_AUX0_TMRB0TRIG_B4OUT = 7, /*!< B4OUT : Trigger source is CTIMERB4 OUT. value. */ + CTIMER_AUX0_TMRB0TRIG_B3OUT2 = 8, /*!< B3OUT2 : Trigger source is CTIMERB3 OUT2. value. */ + CTIMER_AUX0_TMRB0TRIG_A3OUT2 = 9, /*!< A3OUT2 : Trigger source is CTIMERA3 OUT2. value. */ + CTIMER_AUX0_TMRB0TRIG_B7OUT2 = 10, /*!< B7OUT2 : Trigger source is CTIMERB7 OUT2. value. */ + CTIMER_AUX0_TMRB0TRIG_A2OUT2 = 11, /*!< A2OUT2 : Trigger source is CTIMERA2 OUT2. value. */ + CTIMER_AUX0_TMRB0TRIG_A6OUT2DUAL = 12, /*!< A6OUT2DUAL : Trigger source is CTIMERA6 OUT2, dual edge. value. */ + CTIMER_AUX0_TMRB0TRIG_A7OUT2DUAL = 13, /*!< A7OUT2DUAL : Trigger source is CTIMERA7 OUT2, dual edge. value. */ + CTIMER_AUX0_TMRB0TRIG_B5OUT2DUAL = 14, /*!< B5OUT2DUAL : Trigger source is CTIMERB5 OUT2, dual edge. value. */ + CTIMER_AUX0_TMRB0TRIG_A5OUT2DUAL = 15, /*!< A5OUT2DUAL : Trigger source is CTIMERA5 OUT2, dual edge. value. */ +} CTIMER_AUX0_TMRB0TRIG_Enum; + +/* ============================================ CTIMER AUX0 TMRA0EN23 [14..14] ============================================= */ +typedef enum { /*!< CTIMER_AUX0_TMRA0EN23 */ + CTIMER_AUX0_TMRA0EN23_DIS = 1, /*!< DIS : Disable enhanced functions. value. */ + CTIMER_AUX0_TMRA0EN23_EN = 0, /*!< EN : Enable enhanced functions. value. */ +} CTIMER_AUX0_TMRA0EN23_Enum; + +/* ============================================ CTIMER AUX0 TMRA0POL23 [13..13] ============================================ */ +typedef enum { /*!< CTIMER_AUX0_TMRA0POL23 */ + CTIMER_AUX0_TMRA0POL23_NORM = 0, /*!< NORM : Upper output normal polarity value. */ + CTIMER_AUX0_TMRA0POL23_INV = 1, /*!< INV : Upper output inverted polarity. value. */ +} CTIMER_AUX0_TMRA0POL23_Enum; + +/* ============================================ CTIMER AUX0 TMRA0TINV [12..12] ============================================= */ +typedef enum { /*!< CTIMER_AUX0_TMRA0TINV */ + CTIMER_AUX0_TMRA0TINV_DIS = 0, /*!< DIS : Disable invert on trigger value. */ + CTIMER_AUX0_TMRA0TINV_EN = 1, /*!< EN : Enable invert on trigger value. */ +} CTIMER_AUX0_TMRA0TINV_Enum; + +/* =========================================== CTIMER AUX0 TMRA0NOSYNC [11..11] ============================================ */ +typedef enum { /*!< CTIMER_AUX0_TMRA0NOSYNC */ + CTIMER_AUX0_TMRA0NOSYNC_DIS = 0, /*!< DIS : Synchronization on source clock value. */ + CTIMER_AUX0_TMRA0NOSYNC_NOSYNC = 1, /*!< NOSYNC : No synchronization on source clock value. */ +} CTIMER_AUX0_TMRA0NOSYNC_Enum; + +/* ============================================= CTIMER AUX0 TMRA0TRIG [7..10] ============================================= */ +typedef enum { /*!< CTIMER_AUX0_TMRA0TRIG */ + CTIMER_AUX0_TMRA0TRIG_DIS = 0, /*!< DIS : Trigger source is disabled. value. */ + CTIMER_AUX0_TMRA0TRIG_B0OUT = 1, /*!< B0OUT : Trigger source is CTIMERB0 OUT. value. */ + CTIMER_AUX0_TMRA0TRIG_B3OUT = 2, /*!< B3OUT : Trigger source is CTIMERB3 OUT. value. */ + CTIMER_AUX0_TMRA0TRIG_A3OUT = 3, /*!< A3OUT : Trigger source is CTIMERA3 OUT. value. */ + CTIMER_AUX0_TMRA0TRIG_A1OUT = 4, /*!< A1OUT : Trigger source is CTIMERA1 OUT. value. */ + CTIMER_AUX0_TMRA0TRIG_B1OUT = 5, /*!< B1OUT : Trigger source is CTIMERB1 OUT. value. */ + CTIMER_AUX0_TMRA0TRIG_A5OUT = 6, /*!< A5OUT : Trigger source is CTIMERA5 OUT. value. */ + CTIMER_AUX0_TMRA0TRIG_B5OUT = 7, /*!< B5OUT : Trigger source is CTIMERB5 OUT. value. */ + CTIMER_AUX0_TMRA0TRIG_B3OUT2 = 8, /*!< B3OUT2 : Trigger source is CTIMERB3 OUT2. value. */ + CTIMER_AUX0_TMRA0TRIG_A3OUT2 = 9, /*!< A3OUT2 : Trigger source is CTIMERA3 OUT2. value. */ + CTIMER_AUX0_TMRA0TRIG_B6OUT2 = 10, /*!< B6OUT2 : Trigger source is CTIMERB6 OUT2. value. */ + CTIMER_AUX0_TMRA0TRIG_A2OUT2 = 11, /*!< A2OUT2 : Trigger source is CTIMERA2 OUT2. value. */ + CTIMER_AUX0_TMRA0TRIG_A6OUT2DUAL = 12, /*!< A6OUT2DUAL : Trigger source is CTIMERA6 OUT2, dual edge. value. */ + CTIMER_AUX0_TMRA0TRIG_A7OUT2DUAL = 13, /*!< A7OUT2DUAL : Trigger source is CTIMERA7 OUT2, dual edge. value. */ + CTIMER_AUX0_TMRA0TRIG_B4OUT2DUAL = 14, /*!< B4OUT2DUAL : Trigger source is CTIMERB4 OUT2, dual edge. value. */ + CTIMER_AUX0_TMRA0TRIG_A4OUT2DUAL = 15, /*!< A4OUT2DUAL : Trigger source is CTIMERA4 OUT2, dual edge. value. */ +} CTIMER_AUX0_TMRA0TRIG_Enum; + +/* ========================================================= TMR1 ========================================================== */ +/* ======================================================== CMPRA1 ========================================================= */ +/* ======================================================== CMPRB1 ========================================================= */ +/* ========================================================= CTRL1 ========================================================= */ +/* ============================================= CTIMER CTRL1 CTLINK1 [31..31] ============================================= */ +typedef enum { /*!< CTIMER_CTRL1_CTLINK1 */ + CTIMER_CTRL1_CTLINK1_TWO_16BIT_TIMERS = 0, /*!< TWO_16BIT_TIMERS : Use A1/B1 timers as two independent 16-bit + timers (default). value. */ + CTIMER_CTRL1_CTLINK1_32BIT_TIMER = 1, /*!< 32BIT_TIMER : Link A1/B1 timers into a single 32-bit timer. + value. */ +} CTIMER_CTRL1_CTLINK1_Enum; + +/* ============================================ CTIMER CTRL1 TMRB1POL [28..28] ============================================= */ +typedef enum { /*!< CTIMER_CTRL1_TMRB1POL */ + CTIMER_CTRL1_TMRB1POL_NORMAL = 0, /*!< NORMAL : The polarity of the TMRPINB1 pin is the same as the + timer output. value. */ + CTIMER_CTRL1_TMRB1POL_INVERTED = 1, /*!< INVERTED : The polarity of the TMRPINB1 pin is the inverse of + the timer output. value. */ +} CTIMER_CTRL1_TMRB1POL_Enum; + +/* ============================================ CTIMER CTRL1 TMRB1CLR [27..27] ============================================= */ +typedef enum { /*!< CTIMER_CTRL1_TMRB1CLR */ + CTIMER_CTRL1_TMRB1CLR_RUN = 0, /*!< RUN : Allow counter/timer B1 to run value. */ + CTIMER_CTRL1_TMRB1CLR_CLEAR = 1, /*!< CLEAR : Holds counter/timer B1 at 0x0000. value. */ +} CTIMER_CTRL1_TMRB1CLR_Enum; + +/* ============================================ CTIMER CTRL1 TMRB1IE1 [26..26] ============================================= */ +typedef enum { /*!< CTIMER_CTRL1_TMRB1IE1 */ + CTIMER_CTRL1_TMRB1IE1_DIS = 0, /*!< DIS : Disable counter/timer B1 from generating an interrupt + based on COMPR1. value. */ + CTIMER_CTRL1_TMRB1IE1_EN = 1, /*!< EN : Enable counter/timer B1 to generate an interrupt based + on COMPR1. value. */ +} CTIMER_CTRL1_TMRB1IE1_Enum; + +/* ============================================ CTIMER CTRL1 TMRB1IE0 [25..25] ============================================= */ +typedef enum { /*!< CTIMER_CTRL1_TMRB1IE0 */ + CTIMER_CTRL1_TMRB1IE0_DIS = 0, /*!< DIS : Disable counter/timer B1 from generating an interrupt + based on COMPR0. value. */ + CTIMER_CTRL1_TMRB1IE0_EN = 1, /*!< EN : Enable counter/timer B1 to generate an interrupt based + on COMPR0 value. */ +} CTIMER_CTRL1_TMRB1IE0_Enum; + +/* ============================================= CTIMER CTRL1 TMRB1FN [22..24] ============================================= */ +typedef enum { /*!< CTIMER_CTRL1_TMRB1FN */ + CTIMER_CTRL1_TMRB1FN_SINGLECOUNT = 0, /*!< SINGLECOUNT : Single count (output toggles and sticks). Count + to CMPR0B1, stop. value. */ + CTIMER_CTRL1_TMRB1FN_REPEATEDCOUNT = 1, /*!< REPEATEDCOUNT : Repeated count (periodic 1-clock-cycle-wide + pulses). Count to CMPR0B1, restart. value. */ + CTIMER_CTRL1_TMRB1FN_PULSE_ONCE = 2, /*!< PULSE_ONCE : Pulse once (aka one-shot). Count to CMPR0B1, assert, + count to CMPR1B1, deassert, stop. value. */ + CTIMER_CTRL1_TMRB1FN_PULSE_CONT = 3, /*!< PULSE_CONT : Pulse continously. Count to CMPR0B1, assert, count + to CMPR1B1, deassert, restart. value. */ + CTIMER_CTRL1_TMRB1FN_SINGLEPATTERN = 4, /*!< SINGLEPATTERN : Single pattern. value. */ + CTIMER_CTRL1_TMRB1FN_REPEATPATTERN = 5, /*!< REPEATPATTERN : Repeated pattern. value. */ + CTIMER_CTRL1_TMRB1FN_CONTINUOUS = 6, /*!< CONTINUOUS : Continuous run (aka Free Run). Count continuously. + value. */ + CTIMER_CTRL1_TMRB1FN_ALTPWN = 7, /*!< ALTPWN : Alternate PWM value. */ +} CTIMER_CTRL1_TMRB1FN_Enum; + +/* ============================================ CTIMER CTRL1 TMRB1CLK [17..21] ============================================= */ +typedef enum { /*!< CTIMER_CTRL1_TMRB1CLK */ + CTIMER_CTRL1_TMRB1CLK_TMRPIN = 0, /*!< TMRPIN : Clock source is TMRPINB. value. */ + CTIMER_CTRL1_TMRB1CLK_HFRC_DIV4 = 1, /*!< HFRC_DIV4 : Clock source is the HFRC / 4 value. */ + CTIMER_CTRL1_TMRB1CLK_HFRC_DIV16 = 2, /*!< HFRC_DIV16 : Clock source is HFRC / 16 value. */ + CTIMER_CTRL1_TMRB1CLK_HFRC_DIV256 = 3, /*!< HFRC_DIV256 : Clock source is HFRC / 256 value. */ + CTIMER_CTRL1_TMRB1CLK_HFRC_DIV1024 = 4, /*!< HFRC_DIV1024 : Clock source is HFRC / 1024 value. */ + CTIMER_CTRL1_TMRB1CLK_HFRC_DIV4K = 5, /*!< HFRC_DIV4K : Clock source is HFRC / 4096 value. */ + CTIMER_CTRL1_TMRB1CLK_XT = 6, /*!< XT : Clock source is the XT (uncalibrated). value. */ + CTIMER_CTRL1_TMRB1CLK_XT_DIV2 = 7, /*!< XT_DIV2 : Clock source is XT / 2 value. */ + CTIMER_CTRL1_TMRB1CLK_XT_DIV16 = 8, /*!< XT_DIV16 : Clock source is XT / 16 value. */ + CTIMER_CTRL1_TMRB1CLK_XT_DIV128 = 9, /*!< XT_DIV128 : Clock source is XT / 128 value. */ + CTIMER_CTRL1_TMRB1CLK_LFRC_DIV2 = 10, /*!< LFRC_DIV2 : Clock source is LFRC / 2 value. */ + CTIMER_CTRL1_TMRB1CLK_LFRC_DIV32 = 11, /*!< LFRC_DIV32 : Clock source is LFRC / 32 value. */ + CTIMER_CTRL1_TMRB1CLK_LFRC_DIV1K = 12, /*!< LFRC_DIV1K : Clock source is LFRC / 1024 value. */ + CTIMER_CTRL1_TMRB1CLK_LFRC = 13, /*!< LFRC : Clock source is LFRC value. */ + CTIMER_CTRL1_TMRB1CLK_RTC_100HZ = 14, /*!< RTC_100HZ : Clock source is 100 Hz from the current RTC oscillator. + value. */ + CTIMER_CTRL1_TMRB1CLK_HCLK = 15, /*!< HCLK : Clock source is HCLK. value. */ + CTIMER_CTRL1_TMRB1CLK_XT_DIV4 = 16, /*!< XT_DIV4 : Clock source is XT / 4 value. */ + CTIMER_CTRL1_TMRB1CLK_XT_DIV8 = 17, /*!< XT_DIV8 : Clock source is XT / 8 value. */ + CTIMER_CTRL1_TMRB1CLK_XT_DIV32 = 18, /*!< XT_DIV32 : Clock source is XT / 32 value. */ + CTIMER_CTRL1_TMRB1CLK_CTMRA1 = 20, /*!< CTMRA1 : Clock source is CTIMERA1 OUT. value. */ + CTIMER_CTRL1_TMRB1CLK_CTMRA0 = 21, /*!< CTMRA0 : Clock source is CTIMERA0 OUT. value. */ + CTIMER_CTRL1_TMRB1CLK_CTMRB0 = 22, /*!< CTMRB0 : Clock source is CTIMERB0 OUT. value. */ + CTIMER_CTRL1_TMRB1CLK_CTMRA2 = 23, /*!< CTMRA2 : Clock source is CTIMERA2 OUT. value. */ + CTIMER_CTRL1_TMRB1CLK_CTMRB2 = 24, /*!< CTMRB2 : Clock source is CTIMERB2 OUT. value. */ + CTIMER_CTRL1_TMRB1CLK_CTMRB3 = 25, /*!< CTMRB3 : Clock source is CTIMERB3 OUT. value. */ + CTIMER_CTRL1_TMRB1CLK_CTMRB4 = 26, /*!< CTMRB4 : Clock source is CTIMERB4 OUT. value. */ + CTIMER_CTRL1_TMRB1CLK_CTMRB5 = 27, /*!< CTMRB5 : Clock source is CTIMERB5 OUT. value. */ + CTIMER_CTRL1_TMRB1CLK_CTMRB6 = 28, /*!< CTMRB6 : Clock source is CTIMERB6 OUT. value. */ + CTIMER_CTRL1_TMRB1CLK_BUCKBLE = 29, /*!< BUCKBLE : Clock source is BLE buck converter TON pulses. value. */ + CTIMER_CTRL1_TMRB1CLK_BUCKB = 30, /*!< BUCKB : Clock source is Memory buck converter TON pulses. value. */ + CTIMER_CTRL1_TMRB1CLK_BUCKA = 31, /*!< BUCKA : Clock source is CPU buck converter TON pulses. value. */ +} CTIMER_CTRL1_TMRB1CLK_Enum; + +/* ============================================= CTIMER CTRL1 TMRB1EN [16..16] ============================================= */ +typedef enum { /*!< CTIMER_CTRL1_TMRB1EN */ + CTIMER_CTRL1_TMRB1EN_DIS = 0, /*!< DIS : Counter/Timer B1 Disable. value. */ + CTIMER_CTRL1_TMRB1EN_EN = 1, /*!< EN : Counter/Timer B1 Enable. value. */ +} CTIMER_CTRL1_TMRB1EN_Enum; + +/* ============================================ CTIMER CTRL1 TMRA1POL [12..12] ============================================= */ +typedef enum { /*!< CTIMER_CTRL1_TMRA1POL */ + CTIMER_CTRL1_TMRA1POL_NORMAL = 0, /*!< NORMAL : The polarity of the TMRPINA1 pin is the same as the + timer output. value. */ + CTIMER_CTRL1_TMRA1POL_INVERTED = 1, /*!< INVERTED : The polarity of the TMRPINA1 pin is the inverse of + the timer output. value. */ +} CTIMER_CTRL1_TMRA1POL_Enum; + +/* ============================================ CTIMER CTRL1 TMRA1CLR [11..11] ============================================= */ +typedef enum { /*!< CTIMER_CTRL1_TMRA1CLR */ + CTIMER_CTRL1_TMRA1CLR_RUN = 0, /*!< RUN : Allow counter/timer A1 to run value. */ + CTIMER_CTRL1_TMRA1CLR_CLEAR = 1, /*!< CLEAR : Holds counter/timer A1 at 0x0000. value. */ +} CTIMER_CTRL1_TMRA1CLR_Enum; + +/* ============================================ CTIMER CTRL1 TMRA1IE1 [10..10] ============================================= */ +typedef enum { /*!< CTIMER_CTRL1_TMRA1IE1 */ + CTIMER_CTRL1_TMRA1IE1_DIS = 0, /*!< DIS : Disable counter/timer A1 from generating an interrupt + based on COMPR1. value. */ + CTIMER_CTRL1_TMRA1IE1_EN = 1, /*!< EN : Enable counter/timer A1 to generate an interrupt based + on COMPR1. value. */ +} CTIMER_CTRL1_TMRA1IE1_Enum; + +/* ============================================= CTIMER CTRL1 TMRA1IE0 [9..9] ============================================== */ +typedef enum { /*!< CTIMER_CTRL1_TMRA1IE0 */ + CTIMER_CTRL1_TMRA1IE0_DIS = 0, /*!< DIS : Disable counter/timer A1 from generating an interrupt + based on COMPR0. value. */ + CTIMER_CTRL1_TMRA1IE0_EN = 1, /*!< EN : Enable counter/timer A1 to generate an interrupt based + on COMPR0. value. */ +} CTIMER_CTRL1_TMRA1IE0_Enum; + +/* ============================================== CTIMER CTRL1 TMRA1FN [6..8] ============================================== */ +typedef enum { /*!< CTIMER_CTRL1_TMRA1FN */ + CTIMER_CTRL1_TMRA1FN_SINGLECOUNT = 0, /*!< SINGLECOUNT : Single count (output toggles and sticks). Count + to CMPR0A1, stop. value. */ + CTIMER_CTRL1_TMRA1FN_REPEATEDCOUNT = 1, /*!< REPEATEDCOUNT : Repeated count (periodic 1-clock-cycle-wide + pulses). Count to CMPR0A1, restart. value. */ + CTIMER_CTRL1_TMRA1FN_PULSE_ONCE = 2, /*!< PULSE_ONCE : Pulse once (aka one-shot). Count to CMPR0A1, assert, + count to CMPR1A1, deassert, stop. value. */ + CTIMER_CTRL1_TMRA1FN_PULSE_CONT = 3, /*!< PULSE_CONT : Pulse continously. Count to CMPR0A1, assert, count + to CMPR1A1, deassert, restart. value. */ + CTIMER_CTRL1_TMRA1FN_SINGLEPATTERN = 4, /*!< SINGLEPATTERN : Single pattern. value. */ + CTIMER_CTRL1_TMRA1FN_REPEATPATTERN = 5, /*!< REPEATPATTERN : Repeated pattern. value. */ + CTIMER_CTRL1_TMRA1FN_CONTINUOUS = 6, /*!< CONTINUOUS : Continuous run (aka Free Run). Count continuously. + value. */ + CTIMER_CTRL1_TMRA1FN_ALTPWN = 7, /*!< ALTPWN : Alternate PWM value. */ +} CTIMER_CTRL1_TMRA1FN_Enum; + +/* ============================================= CTIMER CTRL1 TMRA1CLK [1..5] ============================================== */ +typedef enum { /*!< CTIMER_CTRL1_TMRA1CLK */ + CTIMER_CTRL1_TMRA1CLK_TMRPIN = 0, /*!< TMRPIN : Clock source is TMRPINA. value. */ + CTIMER_CTRL1_TMRA1CLK_HFRC_DIV4 = 1, /*!< HFRC_DIV4 : Clock source is the HFRC / 4 value. */ + CTIMER_CTRL1_TMRA1CLK_HFRC_DIV16 = 2, /*!< HFRC_DIV16 : Clock source is HFRC / 16 value. */ + CTIMER_CTRL1_TMRA1CLK_HFRC_DIV256 = 3, /*!< HFRC_DIV256 : Clock source is HFRC / 256 value. */ + CTIMER_CTRL1_TMRA1CLK_HFRC_DIV1024 = 4, /*!< HFRC_DIV1024 : Clock source is HFRC / 1024 value. */ + CTIMER_CTRL1_TMRA1CLK_HFRC_DIV4K = 5, /*!< HFRC_DIV4K : Clock source is HFRC / 4096 value. */ + CTIMER_CTRL1_TMRA1CLK_XT = 6, /*!< XT : Clock source is the XT (uncalibrated). value. */ + CTIMER_CTRL1_TMRA1CLK_XT_DIV2 = 7, /*!< XT_DIV2 : Clock source is XT / 2 value. */ + CTIMER_CTRL1_TMRA1CLK_XT_DIV16 = 8, /*!< XT_DIV16 : Clock source is XT / 16 value. */ + CTIMER_CTRL1_TMRA1CLK_XT_DIV128 = 9, /*!< XT_DIV128 : Clock source is XT / 128 value. */ + CTIMER_CTRL1_TMRA1CLK_LFRC_DIV2 = 10, /*!< LFRC_DIV2 : Clock source is LFRC / 2 value. */ + CTIMER_CTRL1_TMRA1CLK_LFRC_DIV32 = 11, /*!< LFRC_DIV32 : Clock source is LFRC / 32 value. */ + CTIMER_CTRL1_TMRA1CLK_LFRC_DIV1K = 12, /*!< LFRC_DIV1K : Clock source is LFRC / 1024 value. */ + CTIMER_CTRL1_TMRA1CLK_LFRC = 13, /*!< LFRC : Clock source is LFRC value. */ + CTIMER_CTRL1_TMRA1CLK_RTC_100HZ = 14, /*!< RTC_100HZ : Clock source is 100 Hz from the current RTC oscillator. + value. */ + CTIMER_CTRL1_TMRA1CLK_HCLK = 15, /*!< HCLK : Clock source is HCLK. value. */ + CTIMER_CTRL1_TMRA1CLK_XT_DIV4 = 16, /*!< XT_DIV4 : Clock source is XT / 4 value. */ + CTIMER_CTRL1_TMRA1CLK_XT_DIV8 = 17, /*!< XT_DIV8 : Clock source is XT / 8 value. */ + CTIMER_CTRL1_TMRA1CLK_XT_DIV32 = 18, /*!< XT_DIV32 : Clock source is XT / 32 value. */ + CTIMER_CTRL1_TMRA1CLK_CTMRB1 = 20, /*!< CTMRB1 : Clock source is CTIMERB1 OUT. value. */ + CTIMER_CTRL1_TMRA1CLK_CTMRA0 = 21, /*!< CTMRA0 : Clock source is CTIMERA0 OUT. value. */ + CTIMER_CTRL1_TMRA1CLK_CTMRB0 = 22, /*!< CTMRB0 : Clock source is CTIMERB0 OUT. value. */ + CTIMER_CTRL1_TMRA1CLK_CTMRA2 = 23, /*!< CTMRA2 : Clock source is CTIMERA2 OUT. value. */ + CTIMER_CTRL1_TMRA1CLK_CTMRB2 = 24, /*!< CTMRB2 : Clock source is CTIMERB2 OUT. value. */ + CTIMER_CTRL1_TMRA1CLK_CTMRB3 = 25, /*!< CTMRB3 : Clock source is CTIMERB3 OUT. value. */ + CTIMER_CTRL1_TMRA1CLK_CTMRB4 = 26, /*!< CTMRB4 : Clock source is CTIMERB4 OUT. value. */ + CTIMER_CTRL1_TMRA1CLK_CTMRB5 = 27, /*!< CTMRB5 : Clock source is CTIMERB5 OUT. value. */ + CTIMER_CTRL1_TMRA1CLK_CTMRB6 = 28, /*!< CTMRB6 : Clock source is CTIMERB6 OUT. value. */ + CTIMER_CTRL1_TMRA1CLK_BUCKBLE = 29, /*!< BUCKBLE : Clock source is BLE buck converter TON pulses. value. */ + CTIMER_CTRL1_TMRA1CLK_BUCKB = 30, /*!< BUCKB : Clock source is Memory buck converter TON pulses. value. */ + CTIMER_CTRL1_TMRA1CLK_BUCKA = 31, /*!< BUCKA : Clock source is CPU buck converter TON pulses. value. */ +} CTIMER_CTRL1_TMRA1CLK_Enum; + +/* ============================================== CTIMER CTRL1 TMRA1EN [0..0] ============================================== */ +typedef enum { /*!< CTIMER_CTRL1_TMRA1EN */ + CTIMER_CTRL1_TMRA1EN_DIS = 0, /*!< DIS : Counter/Timer A1 Disable. value. */ + CTIMER_CTRL1_TMRA1EN_EN = 1, /*!< EN : Counter/Timer A1 Enable. value. */ +} CTIMER_CTRL1_TMRA1EN_Enum; + +/* ======================================================= CMPRAUXA1 ======================================================= */ +/* ======================================================= CMPRAUXB1 ======================================================= */ +/* ========================================================= AUX1 ========================================================== */ +/* ============================================ CTIMER AUX1 TMRB1EN23 [30..30] ============================================= */ +typedef enum { /*!< CTIMER_AUX1_TMRB1EN23 */ + CTIMER_AUX1_TMRB1EN23_DIS = 1, /*!< DIS : Disable enhanced functions. value. */ + CTIMER_AUX1_TMRB1EN23_EN = 0, /*!< EN : Enable enhanced functions. value. */ +} CTIMER_AUX1_TMRB1EN23_Enum; + +/* ============================================ CTIMER AUX1 TMRB1POL23 [29..29] ============================================ */ +typedef enum { /*!< CTIMER_AUX1_TMRB1POL23 */ + CTIMER_AUX1_TMRB1POL23_NORM = 0, /*!< NORM : Upper output normal polarity value. */ + CTIMER_AUX1_TMRB1POL23_INV = 1, /*!< INV : Upper output inverted polarity. value. */ +} CTIMER_AUX1_TMRB1POL23_Enum; + +/* ============================================ CTIMER AUX1 TMRB1TINV [28..28] ============================================= */ +typedef enum { /*!< CTIMER_AUX1_TMRB1TINV */ + CTIMER_AUX1_TMRB1TINV_DIS = 0, /*!< DIS : Disable invert on trigger value. */ + CTIMER_AUX1_TMRB1TINV_EN = 1, /*!< EN : Enable invert on trigger value. */ +} CTIMER_AUX1_TMRB1TINV_Enum; + +/* =========================================== CTIMER AUX1 TMRB1NOSYNC [27..27] ============================================ */ +typedef enum { /*!< CTIMER_AUX1_TMRB1NOSYNC */ + CTIMER_AUX1_TMRB1NOSYNC_DIS = 0, /*!< DIS : Synchronization on source clock value. */ + CTIMER_AUX1_TMRB1NOSYNC_NOSYNC = 1, /*!< NOSYNC : No synchronization on source clock value. */ +} CTIMER_AUX1_TMRB1NOSYNC_Enum; + +/* ============================================ CTIMER AUX1 TMRB1TRIG [23..26] ============================================= */ +typedef enum { /*!< CTIMER_AUX1_TMRB1TRIG */ + CTIMER_AUX1_TMRB1TRIG_DIS = 0, /*!< DIS : Trigger source is disabled. value. */ + CTIMER_AUX1_TMRB1TRIG_A1OUT = 1, /*!< A1OUT : Trigger source is CTIMERA1 OUT. value. */ + CTIMER_AUX1_TMRB1TRIG_B3OUT = 2, /*!< B3OUT : Trigger source is CTIMERB3 OUT. value. */ + CTIMER_AUX1_TMRB1TRIG_A3OUT = 3, /*!< A3OUT : Trigger source is CTIMERA3 OUT. value. */ + CTIMER_AUX1_TMRB1TRIG_A6OUT = 4, /*!< A6OUT : Trigger source is CTIMERA6 OUT. value. */ + CTIMER_AUX1_TMRB1TRIG_B6OUT = 5, /*!< B6OUT : Trigger source is CTIMERB6 OUT. value. */ + CTIMER_AUX1_TMRB1TRIG_A0OUT = 6, /*!< A0OUT : Trigger source is CTIMERA0 OUT. value. */ + CTIMER_AUX1_TMRB1TRIG_B0OUT = 7, /*!< B0OUT : Trigger source is CTIMERB0 OUT. value. */ + CTIMER_AUX1_TMRB1TRIG_B3OUT2 = 8, /*!< B3OUT2 : Trigger source is CTIMERB3 OUT2. value. */ + CTIMER_AUX1_TMRB1TRIG_A3OUT2 = 9, /*!< A3OUT2 : Trigger source is CTIMERA3 OUT2. value. */ + CTIMER_AUX1_TMRB1TRIG_A4OUT2 = 10, /*!< A4OUT2 : Trigger source is CTIMERA4 OUT2. value. */ + CTIMER_AUX1_TMRB1TRIG_B4OUT2 = 11, /*!< B4OUT2 : Trigger source is CTIMERB4 OUT2. value. */ + CTIMER_AUX1_TMRB1TRIG_A6OUT2DUAL = 12, /*!< A6OUT2DUAL : Trigger source is CTIMERA6 OUT2, dual edge. value. */ + CTIMER_AUX1_TMRB1TRIG_A7OUT2DUAL = 13, /*!< A7OUT2DUAL : Trigger source is CTIMERA7 OUT2, dual edge. value. */ + CTIMER_AUX1_TMRB1TRIG_B5OUT2DUAL = 14, /*!< B5OUT2DUAL : Trigger source is CTIMERB5 OUT2, dual edge. value. */ + CTIMER_AUX1_TMRB1TRIG_A5OUT2DUAL = 15, /*!< A5OUT2DUAL : Trigger source is CTIMERA5 OUT2, dual edge. value. */ +} CTIMER_AUX1_TMRB1TRIG_Enum; + +/* ============================================ CTIMER AUX1 TMRA1EN23 [14..14] ============================================= */ +typedef enum { /*!< CTIMER_AUX1_TMRA1EN23 */ + CTIMER_AUX1_TMRA1EN23_DIS = 1, /*!< DIS : Disable enhanced functions. value. */ + CTIMER_AUX1_TMRA1EN23_EN = 0, /*!< EN : Enable enhanced functions. value. */ +} CTIMER_AUX1_TMRA1EN23_Enum; + +/* ============================================ CTIMER AUX1 TMRA1POL23 [13..13] ============================================ */ +typedef enum { /*!< CTIMER_AUX1_TMRA1POL23 */ + CTIMER_AUX1_TMRA1POL23_NORMAL = 0, /*!< NORMAL : Upper output normal polarity value. */ + CTIMER_AUX1_TMRA1POL23_INV = 1, /*!< INV : Upper output inverted polarity. value. */ +} CTIMER_AUX1_TMRA1POL23_Enum; + +/* ============================================ CTIMER AUX1 TMRA1TINV [12..12] ============================================= */ +typedef enum { /*!< CTIMER_AUX1_TMRA1TINV */ + CTIMER_AUX1_TMRA1TINV_DIS = 0, /*!< DIS : Disable invert on trigger value. */ + CTIMER_AUX1_TMRA1TINV_EN = 1, /*!< EN : Enable invert on trigger value. */ +} CTIMER_AUX1_TMRA1TINV_Enum; + +/* =========================================== CTIMER AUX1 TMRA1NOSYNC [11..11] ============================================ */ +typedef enum { /*!< CTIMER_AUX1_TMRA1NOSYNC */ + CTIMER_AUX1_TMRA1NOSYNC_DIS = 0, /*!< DIS : Synchronization on source clock value. */ + CTIMER_AUX1_TMRA1NOSYNC_NOSYNC = 1, /*!< NOSYNC : No synchronization on source clock value. */ +} CTIMER_AUX1_TMRA1NOSYNC_Enum; + +/* ============================================= CTIMER AUX1 TMRA1TRIG [7..10] ============================================= */ +typedef enum { /*!< CTIMER_AUX1_TMRA1TRIG */ + CTIMER_AUX1_TMRA1TRIG_DIS = 0, /*!< DIS : Trigger source is disabled. value. */ + CTIMER_AUX1_TMRA1TRIG_B1OUT = 1, /*!< B1OUT : Trigger source is CTIMERB1 OUT. value. */ + CTIMER_AUX1_TMRA1TRIG_B3OUT = 2, /*!< B3OUT : Trigger source is CTIMERB3 OUT. value. */ + CTIMER_AUX1_TMRA1TRIG_A3OUT = 3, /*!< A3OUT : Trigger source is CTIMERA3 OUT. value. */ + CTIMER_AUX1_TMRA1TRIG_A0OUT = 4, /*!< A0OUT : Trigger source is CTIMERA0 OUT. value. */ + CTIMER_AUX1_TMRA1TRIG_B0OUT = 5, /*!< B0OUT : Trigger source is CTIMERB0 OUT. value. */ + CTIMER_AUX1_TMRA1TRIG_A5OUT = 6, /*!< A5OUT : Trigger source is CTIMERA5 OUT. value. */ + CTIMER_AUX1_TMRA1TRIG_B5OUT = 7, /*!< B5OUT : Trigger source is CTIMERB5 OUT. value. */ + CTIMER_AUX1_TMRA1TRIG_B3OUT2 = 8, /*!< B3OUT2 : Trigger source is CTIMERB3 OUT2. value. */ + CTIMER_AUX1_TMRA1TRIG_A3OUT2 = 9, /*!< A3OUT2 : Trigger source is CTIMERA3 OUT2. value. */ + CTIMER_AUX1_TMRA1TRIG_A4OUT2 = 10, /*!< A4OUT2 : Trigger source is CTIMERA4 OUT2. value. */ + CTIMER_AUX1_TMRA1TRIG_B4OUT2 = 11, /*!< B4OUT2 : Trigger source is CTIMERB4 OUT2. value. */ + CTIMER_AUX1_TMRA1TRIG_A6OUT2DUAL = 12, /*!< A6OUT2DUAL : Trigger source is CTIMERA6 OUT2, dual edge. value. */ + CTIMER_AUX1_TMRA1TRIG_A7OUT2DUAL = 13, /*!< A7OUT2DUAL : Trigger source is CTIMERA7 OUT2, dual edge. value. */ + CTIMER_AUX1_TMRA1TRIG_B5OUT2DUAL = 14, /*!< B5OUT2DUAL : Trigger source is CTIMERB5 OUT2, dual edge. value. */ + CTIMER_AUX1_TMRA1TRIG_A5OUT2DUAL = 15, /*!< A5OUT2DUAL : Trigger source is CTIMERA5 OUT2, dual edge. value. */ +} CTIMER_AUX1_TMRA1TRIG_Enum; + +/* ========================================================= TMR2 ========================================================== */ +/* ======================================================== CMPRA2 ========================================================= */ +/* ======================================================== CMPRB2 ========================================================= */ +/* ========================================================= CTRL2 ========================================================= */ +/* ============================================= CTIMER CTRL2 CTLINK2 [31..31] ============================================= */ +typedef enum { /*!< CTIMER_CTRL2_CTLINK2 */ + CTIMER_CTRL2_CTLINK2_TWO_16BIT_TIMERS = 0, /*!< TWO_16BIT_TIMERS : Use A2/B2 timers as two independent 16-bit + timers (default). value. */ + CTIMER_CTRL2_CTLINK2_32BIT_TIMER = 1, /*!< 32BIT_TIMER : Link A2/B2 timers into a single 32-bit timer. + value. */ +} CTIMER_CTRL2_CTLINK2_Enum; + +/* ============================================ CTIMER CTRL2 TMRB2POL [28..28] ============================================= */ +typedef enum { /*!< CTIMER_CTRL2_TMRB2POL */ + CTIMER_CTRL2_TMRB2POL_NORMAL = 0, /*!< NORMAL : The polarity of the TMRPINB2 pin is the same as the + timer output. value. */ + CTIMER_CTRL2_TMRB2POL_INVERTED = 1, /*!< INVERTED : The polarity of the TMRPINB2 pin is the inverse of + the timer output. value. */ +} CTIMER_CTRL2_TMRB2POL_Enum; + +/* ============================================ CTIMER CTRL2 TMRB2CLR [27..27] ============================================= */ +typedef enum { /*!< CTIMER_CTRL2_TMRB2CLR */ + CTIMER_CTRL2_TMRB2CLR_RUN = 0, /*!< RUN : Allow counter/timer B2 to run value. */ + CTIMER_CTRL2_TMRB2CLR_CLEAR = 1, /*!< CLEAR : Holds counter/timer B2 at 0x0000. value. */ +} CTIMER_CTRL2_TMRB2CLR_Enum; + +/* ============================================ CTIMER CTRL2 TMRB2IE1 [26..26] ============================================= */ +typedef enum { /*!< CTIMER_CTRL2_TMRB2IE1 */ + CTIMER_CTRL2_TMRB2IE1_DIS = 0, /*!< DIS : Disable counter/timer B2 from generating an interrupt + based on COMPR1. value. */ + CTIMER_CTRL2_TMRB2IE1_EN = 1, /*!< EN : Enable counter/timer B2 to generate an interrupt based + on COMPR1. value. */ +} CTIMER_CTRL2_TMRB2IE1_Enum; + +/* ============================================ CTIMER CTRL2 TMRB2IE0 [25..25] ============================================= */ +typedef enum { /*!< CTIMER_CTRL2_TMRB2IE0 */ + CTIMER_CTRL2_TMRB2IE0_DIS = 0, /*!< DIS : Disable counter/timer B2 from generating an interrupt + based on COMPR0. value. */ + CTIMER_CTRL2_TMRB2IE0_EN = 1, /*!< EN : Enable counter/timer B2 to generate an interrupt based + on COMPR0 value. */ +} CTIMER_CTRL2_TMRB2IE0_Enum; + +/* ============================================= CTIMER CTRL2 TMRB2FN [22..24] ============================================= */ +typedef enum { /*!< CTIMER_CTRL2_TMRB2FN */ + CTIMER_CTRL2_TMRB2FN_SINGLECOUNT = 0, /*!< SINGLECOUNT : Single count (output toggles and sticks). Count + to CMPR0B2, stop. value. */ + CTIMER_CTRL2_TMRB2FN_REPEATEDCOUNT = 1, /*!< REPEATEDCOUNT : Repeated count (periodic 1-clock-cycle-wide + pulses). Count to CMPR0B2, restart. value. */ + CTIMER_CTRL2_TMRB2FN_PULSE_ONCE = 2, /*!< PULSE_ONCE : Pulse once (aka one-shot). Count to CMPR0B2, assert, + count to CMPR1B2, deassert, stop. value. */ + CTIMER_CTRL2_TMRB2FN_PULSE_CONT = 3, /*!< PULSE_CONT : Pulse continously. Count to CMPR0B2, assert, count + to CMPR1B2, deassert, restart. value. */ + CTIMER_CTRL2_TMRB2FN_SINGLEPATTERN = 4, /*!< SINGLEPATTERN : Single pattern. value. */ + CTIMER_CTRL2_TMRB2FN_REPEATPATTERN = 5, /*!< REPEATPATTERN : Repeated pattern. value. */ + CTIMER_CTRL2_TMRB2FN_CONTINUOUS = 6, /*!< CONTINUOUS : Continuous run (aka Free Run). Count continuously. + value. */ + CTIMER_CTRL2_TMRB2FN_ALTPWN = 7, /*!< ALTPWN : Alternate PWM value. */ +} CTIMER_CTRL2_TMRB2FN_Enum; + +/* ============================================ CTIMER CTRL2 TMRB2CLK [17..21] ============================================= */ +typedef enum { /*!< CTIMER_CTRL2_TMRB2CLK */ + CTIMER_CTRL2_TMRB2CLK_TMRPIN = 0, /*!< TMRPIN : Clock source is TMRPINB. value. */ + CTIMER_CTRL2_TMRB2CLK_HFRC_DIV4 = 1, /*!< HFRC_DIV4 : Clock source is the HFRC / 4 value. */ + CTIMER_CTRL2_TMRB2CLK_HFRC_DIV16 = 2, /*!< HFRC_DIV16 : Clock source is HFRC / 16 value. */ + CTIMER_CTRL2_TMRB2CLK_HFRC_DIV256 = 3, /*!< HFRC_DIV256 : Clock source is HFRC / 256 value. */ + CTIMER_CTRL2_TMRB2CLK_HFRC_DIV1024 = 4, /*!< HFRC_DIV1024 : Clock source is HFRC / 1024 value. */ + CTIMER_CTRL2_TMRB2CLK_HFRC_DIV4K = 5, /*!< HFRC_DIV4K : Clock source is HFRC / 4096 value. */ + CTIMER_CTRL2_TMRB2CLK_XT = 6, /*!< XT : Clock source is the XT (uncalibrated). value. */ + CTIMER_CTRL2_TMRB2CLK_XT_DIV2 = 7, /*!< XT_DIV2 : Clock source is XT / 2 value. */ + CTIMER_CTRL2_TMRB2CLK_XT_DIV16 = 8, /*!< XT_DIV16 : Clock source is XT / 16 value. */ + CTIMER_CTRL2_TMRB2CLK_XT_DIV128 = 9, /*!< XT_DIV128 : Clock source is XT / 128 value. */ + CTIMER_CTRL2_TMRB2CLK_LFRC_DIV2 = 10, /*!< LFRC_DIV2 : Clock source is LFRC / 2 value. */ + CTIMER_CTRL2_TMRB2CLK_LFRC_DIV32 = 11, /*!< LFRC_DIV32 : Clock source is LFRC / 32 value. */ + CTIMER_CTRL2_TMRB2CLK_LFRC_DIV1K = 12, /*!< LFRC_DIV1K : Clock source is LFRC / 1024 value. */ + CTIMER_CTRL2_TMRB2CLK_LFRC = 13, /*!< LFRC : Clock source is LFRC value. */ + CTIMER_CTRL2_TMRB2CLK_RTC_100HZ = 14, /*!< RTC_100HZ : Clock source is 100 Hz from the current RTC oscillator. + value. */ + CTIMER_CTRL2_TMRB2CLK_HCLK = 15, /*!< HCLK : Clock source is HCLK. value. */ + CTIMER_CTRL2_TMRB2CLK_XT_DIV4 = 16, /*!< XT_DIV4 : Clock source is XT / 4 value. */ + CTIMER_CTRL2_TMRB2CLK_XT_DIV8 = 17, /*!< XT_DIV8 : Clock source is XT / 8 value. */ + CTIMER_CTRL2_TMRB2CLK_XT_DIV32 = 18, /*!< XT_DIV32 : Clock source is XT / 32 value. */ + CTIMER_CTRL2_TMRB2CLK_CTMRA2 = 20, /*!< CTMRA2 : Clock source is CTIMERA2 OUT. value. */ + CTIMER_CTRL2_TMRB2CLK_CTMRB3 = 21, /*!< CTMRB3 : Clock source is CTIMERA3 OUT. value. */ + CTIMER_CTRL2_TMRB2CLK_CTMRA3 = 22, /*!< CTMRA3 : Clock source is CTIMERB3 OUT. value. */ + CTIMER_CTRL2_TMRB2CLK_CTMRA4 = 23, /*!< CTMRA4 : Clock source is CTIMERA4 OUT. value. */ + CTIMER_CTRL2_TMRB2CLK_CTMRB4 = 24, /*!< CTMRB4 : Clock source is CTIMERB4 OUT. value. */ + CTIMER_CTRL2_TMRB2CLK_CTMRB0 = 25, /*!< CTMRB0 : Clock source is CTIMERB0 OUT. value. */ + CTIMER_CTRL2_TMRB2CLK_CTMRB1 = 26, /*!< CTMRB1 : Clock source is CTIMERB1 OUT. value. */ + CTIMER_CTRL2_TMRB2CLK_CTMRB5 = 27, /*!< CTMRB5 : Clock source is CTIMERB5 OUT. value. */ + CTIMER_CTRL2_TMRB2CLK_CTMRB6 = 28, /*!< CTMRB6 : Clock source is CTIMERB6 OUT. value. */ + CTIMER_CTRL2_TMRB2CLK_BUCKBLE = 29, /*!< BUCKBLE : Clock source is BLE buck converter TON pulses. value. */ + CTIMER_CTRL2_TMRB2CLK_BUCKB = 30, /*!< BUCKB : Clock source is Memory buck converter TON pulses. value. */ + CTIMER_CTRL2_TMRB2CLK_BUCKA = 31, /*!< BUCKA : Clock source is CPU buck converter TON pulses. value. */ +} CTIMER_CTRL2_TMRB2CLK_Enum; + +/* ============================================= CTIMER CTRL2 TMRB2EN [16..16] ============================================= */ +typedef enum { /*!< CTIMER_CTRL2_TMRB2EN */ + CTIMER_CTRL2_TMRB2EN_DIS = 0, /*!< DIS : Counter/Timer B2 Disable. value. */ + CTIMER_CTRL2_TMRB2EN_EN = 1, /*!< EN : Counter/Timer B2 Enable. value. */ +} CTIMER_CTRL2_TMRB2EN_Enum; + +/* ============================================ CTIMER CTRL2 TMRA2POL [12..12] ============================================= */ +typedef enum { /*!< CTIMER_CTRL2_TMRA2POL */ + CTIMER_CTRL2_TMRA2POL_NORMAL = 0, /*!< NORMAL : The polarity of the TMRPINA2 pin is the same as the + timer output. value. */ + CTIMER_CTRL2_TMRA2POL_INVERTED = 1, /*!< INVERTED : The polarity of the TMRPINA2 pin is the inverse of + the timer output. value. */ +} CTIMER_CTRL2_TMRA2POL_Enum; + +/* ============================================ CTIMER CTRL2 TMRA2CLR [11..11] ============================================= */ +typedef enum { /*!< CTIMER_CTRL2_TMRA2CLR */ + CTIMER_CTRL2_TMRA2CLR_RUN = 0, /*!< RUN : Allow counter/timer A2 to run value. */ + CTIMER_CTRL2_TMRA2CLR_CLEAR = 1, /*!< CLEAR : Holds counter/timer A2 at 0x0000. value. */ +} CTIMER_CTRL2_TMRA2CLR_Enum; + +/* ============================================ CTIMER CTRL2 TMRA2IE1 [10..10] ============================================= */ +typedef enum { /*!< CTIMER_CTRL2_TMRA2IE1 */ + CTIMER_CTRL2_TMRA2IE1_DIS = 0, /*!< DIS : Disable counter/timer A2 from generating an interrupt + based on COMPR1. value. */ + CTIMER_CTRL2_TMRA2IE1_EN = 1, /*!< EN : Enable counter/timer A2 to generate an interrupt based + on COMPR1. value. */ +} CTIMER_CTRL2_TMRA2IE1_Enum; + +/* ============================================= CTIMER CTRL2 TMRA2IE0 [9..9] ============================================== */ +typedef enum { /*!< CTIMER_CTRL2_TMRA2IE0 */ + CTIMER_CTRL2_TMRA2IE0_DIS = 0, /*!< DIS : Disable counter/timer A2 from generating an interrupt + based on COMPR0. value. */ + CTIMER_CTRL2_TMRA2IE0_EN = 1, /*!< EN : Enable counter/timer A2 to generate an interrupt based + on COMPR0. value. */ +} CTIMER_CTRL2_TMRA2IE0_Enum; + +/* ============================================== CTIMER CTRL2 TMRA2FN [6..8] ============================================== */ +typedef enum { /*!< CTIMER_CTRL2_TMRA2FN */ + CTIMER_CTRL2_TMRA2FN_SINGLECOUNT = 0, /*!< SINGLECOUNT : Single count (output toggles and sticks). Count + to CMPR0A2, stop. value. */ + CTIMER_CTRL2_TMRA2FN_REPEATEDCOUNT = 1, /*!< REPEATEDCOUNT : Repeated count (periodic 1-clock-cycle-wide + pulses). Count to CMPR0A2, restart. value. */ + CTIMER_CTRL2_TMRA2FN_PULSE_ONCE = 2, /*!< PULSE_ONCE : Pulse once (aka one-shot). Count to CMPR0A2, assert, + count to CMPR1A2, deassert, stop. value. */ + CTIMER_CTRL2_TMRA2FN_PULSE_CONT = 3, /*!< PULSE_CONT : Pulse continously. Count to CMPR0A2, assert, count + to CMPR1A2, deassert, restart. value. */ + CTIMER_CTRL2_TMRA2FN_SINGLEPATTERN = 4, /*!< SINGLEPATTERN : Single pattern. value. */ + CTIMER_CTRL2_TMRA2FN_REPEATPATTERN = 5, /*!< REPEATPATTERN : Repeated pattern. value. */ + CTIMER_CTRL2_TMRA2FN_CONTINUOUS = 6, /*!< CONTINUOUS : Continuous run (aka Free Run). Count continuously. + value. */ + CTIMER_CTRL2_TMRA2FN_ALTPWN = 7, /*!< ALTPWN : Alternate PWM value. */ +} CTIMER_CTRL2_TMRA2FN_Enum; + +/* ============================================= CTIMER CTRL2 TMRA2CLK [1..5] ============================================== */ +typedef enum { /*!< CTIMER_CTRL2_TMRA2CLK */ + CTIMER_CTRL2_TMRA2CLK_TMRPIN = 0, /*!< TMRPIN : Clock source is TMRPINA. value. */ + CTIMER_CTRL2_TMRA2CLK_HFRC_DIV4 = 1, /*!< HFRC_DIV4 : Clock source is the HFRC / 4 value. */ + CTIMER_CTRL2_TMRA2CLK_HFRC_DIV16 = 2, /*!< HFRC_DIV16 : Clock source is HFRC / 16 value. */ + CTIMER_CTRL2_TMRA2CLK_HFRC_DIV256 = 3, /*!< HFRC_DIV256 : Clock source is HFRC / 256 value. */ + CTIMER_CTRL2_TMRA2CLK_HFRC_DIV1024 = 4, /*!< HFRC_DIV1024 : Clock source is HFRC / 1024 value. */ + CTIMER_CTRL2_TMRA2CLK_HFRC_DIV4K = 5, /*!< HFRC_DIV4K : Clock source is HFRC / 4096 value. */ + CTIMER_CTRL2_TMRA2CLK_XT = 6, /*!< XT : Clock source is the XT (uncalibrated). value. */ + CTIMER_CTRL2_TMRA2CLK_XT_DIV2 = 7, /*!< XT_DIV2 : Clock source is XT / 2 value. */ + CTIMER_CTRL2_TMRA2CLK_XT_DIV16 = 8, /*!< XT_DIV16 : Clock source is XT / 16 value. */ + CTIMER_CTRL2_TMRA2CLK_XT_DIV128 = 9, /*!< XT_DIV128 : Clock source is XT / 128 value. */ + CTIMER_CTRL2_TMRA2CLK_LFRC_DIV2 = 10, /*!< LFRC_DIV2 : Clock source is LFRC / 2 value. */ + CTIMER_CTRL2_TMRA2CLK_LFRC_DIV32 = 11, /*!< LFRC_DIV32 : Clock source is LFRC / 32 value. */ + CTIMER_CTRL2_TMRA2CLK_LFRC_DIV1K = 12, /*!< LFRC_DIV1K : Clock source is LFRC / 1024 value. */ + CTIMER_CTRL2_TMRA2CLK_LFRC = 13, /*!< LFRC : Clock source is LFRC value. */ + CTIMER_CTRL2_TMRA2CLK_RTC_100HZ = 14, /*!< RTC_100HZ : Clock source is 100 Hz from the current RTC oscillator. + value. */ + CTIMER_CTRL2_TMRA2CLK_HCLK = 15, /*!< HCLK : Clock source is HCLK. value. */ + CTIMER_CTRL2_TMRA2CLK_XT_DIV4 = 16, /*!< XT_DIV4 : Clock source is XT / 4 value. */ + CTIMER_CTRL2_TMRA2CLK_XT_DIV8 = 17, /*!< XT_DIV8 : Clock source is XT / 8 value. */ + CTIMER_CTRL2_TMRA2CLK_XT_DIV32 = 18, /*!< XT_DIV32 : Clock source is XT / 32 value. */ + CTIMER_CTRL2_TMRA2CLK_CTMRB2 = 20, /*!< CTMRB2 : Clock source is CTIMERB2 OUT. value. */ + CTIMER_CTRL2_TMRA2CLK_CTMRB3 = 21, /*!< CTMRB3 : Clock source is CTIMERA3 OUT. value. */ + CTIMER_CTRL2_TMRA2CLK_CTMRA3 = 22, /*!< CTMRA3 : Clock source is CTIMERB3 OUT. value. */ + CTIMER_CTRL2_TMRA2CLK_CTMRA4 = 23, /*!< CTMRA4 : Clock source is CTIMERA4 OUT. value. */ + CTIMER_CTRL2_TMRA2CLK_CTMRB4 = 24, /*!< CTMRB4 : Clock source is CTIMERB4 OUT. value. */ + CTIMER_CTRL2_TMRA2CLK_CTMRB0 = 25, /*!< CTMRB0 : Clock source is CTIMERB0 OUT. value. */ + CTIMER_CTRL2_TMRA2CLK_CTMRB1 = 26, /*!< CTMRB1 : Clock source is CTIMERB1 OUT. value. */ + CTIMER_CTRL2_TMRA2CLK_CTMRB5 = 27, /*!< CTMRB5 : Clock source is CTIMERB5 OUT. value. */ + CTIMER_CTRL2_TMRA2CLK_CTMRB6 = 28, /*!< CTMRB6 : Clock source is CTIMERB6 OUT. value. */ + CTIMER_CTRL2_TMRA2CLK_BUCKBLE = 29, /*!< BUCKBLE : Clock source is BLE buck converter TON pulses. value. */ + CTIMER_CTRL2_TMRA2CLK_BUCKB = 30, /*!< BUCKB : Clock source is Memory buck converter TON pulses. value. */ + CTIMER_CTRL2_TMRA2CLK_BUCKA = 31, /*!< BUCKA : Clock source is CPU buck converter TON pulses. value. */ +} CTIMER_CTRL2_TMRA2CLK_Enum; + +/* ============================================== CTIMER CTRL2 TMRA2EN [0..0] ============================================== */ +typedef enum { /*!< CTIMER_CTRL2_TMRA2EN */ + CTIMER_CTRL2_TMRA2EN_DIS = 0, /*!< DIS : Counter/Timer A2 Disable. value. */ + CTIMER_CTRL2_TMRA2EN_EN = 1, /*!< EN : Counter/Timer A2 Enable. value. */ +} CTIMER_CTRL2_TMRA2EN_Enum; + +/* ======================================================= CMPRAUXA2 ======================================================= */ +/* ======================================================= CMPRAUXB2 ======================================================= */ +/* ========================================================= AUX2 ========================================================== */ +/* ============================================ CTIMER AUX2 TMRB2EN23 [30..30] ============================================= */ +typedef enum { /*!< CTIMER_AUX2_TMRB2EN23 */ + CTIMER_AUX2_TMRB2EN23_DIS = 1, /*!< DIS : Disable enhanced functions. value. */ + CTIMER_AUX2_TMRB2EN23_EN = 0, /*!< EN : Enable enhanced functions. value. */ +} CTIMER_AUX2_TMRB2EN23_Enum; + +/* ============================================ CTIMER AUX2 TMRB2POL23 [29..29] ============================================ */ +typedef enum { /*!< CTIMER_AUX2_TMRB2POL23 */ + CTIMER_AUX2_TMRB2POL23_NORM = 0, /*!< NORM : Upper output normal polarity value. */ + CTIMER_AUX2_TMRB2POL23_INV = 1, /*!< INV : Upper output inverted polarity. value. */ +} CTIMER_AUX2_TMRB2POL23_Enum; + +/* ============================================ CTIMER AUX2 TMRB2TINV [28..28] ============================================= */ +typedef enum { /*!< CTIMER_AUX2_TMRB2TINV */ + CTIMER_AUX2_TMRB2TINV_DIS = 0, /*!< DIS : Disable invert on trigger value. */ + CTIMER_AUX2_TMRB2TINV_EN = 1, /*!< EN : Enable invert on trigger value. */ +} CTIMER_AUX2_TMRB2TINV_Enum; + +/* =========================================== CTIMER AUX2 TMRB2NOSYNC [27..27] ============================================ */ +typedef enum { /*!< CTIMER_AUX2_TMRB2NOSYNC */ + CTIMER_AUX2_TMRB2NOSYNC_DIS = 0, /*!< DIS : Synchronization on source clock value. */ + CTIMER_AUX2_TMRB2NOSYNC_NOSYNC = 1, /*!< NOSYNC : No synchronization on source clock value. */ +} CTIMER_AUX2_TMRB2NOSYNC_Enum; + +/* ============================================ CTIMER AUX2 TMRB2TRIG [23..26] ============================================= */ +typedef enum { /*!< CTIMER_AUX2_TMRB2TRIG */ + CTIMER_AUX2_TMRB2TRIG_DIS = 0, /*!< DIS : Trigger source is disabled. value. */ + CTIMER_AUX2_TMRB2TRIG_A2OUT = 1, /*!< A2OUT : Trigger source is CTIMERA2 OUT. value. */ + CTIMER_AUX2_TMRB2TRIG_B3OUT = 2, /*!< B3OUT : Trigger source is CTIMERB3 OUT. value. */ + CTIMER_AUX2_TMRB2TRIG_A3OUT = 3, /*!< A3OUT : Trigger source is CTIMERA3 OUT. value. */ + CTIMER_AUX2_TMRB2TRIG_A1OUT = 4, /*!< A1OUT : Trigger source is CTIMERA1 OUT. value. */ + CTIMER_AUX2_TMRB2TRIG_B1OUT = 5, /*!< B1OUT : Trigger source is CTIMERB1 OUT. value. */ + CTIMER_AUX2_TMRB2TRIG_A4OUT = 6, /*!< A4OUT : Trigger source is CTIMERA4 OUT. value. */ + CTIMER_AUX2_TMRB2TRIG_B4OUT = 7, /*!< B4OUT : Trigger source is CTIMERB4 OUT. value. */ + CTIMER_AUX2_TMRB2TRIG_B3OUT2 = 8, /*!< B3OUT2 : Trigger source is CTIMERB3 OUT2. value. */ + CTIMER_AUX2_TMRB2TRIG_A3OUT2 = 9, /*!< A3OUT2 : Trigger source is CTIMERA3 OUT2. value. */ + CTIMER_AUX2_TMRB2TRIG_A5OUT2 = 10, /*!< A5OUT2 : Trigger source is CTIMERA5 OUT2. value. */ + CTIMER_AUX2_TMRB2TRIG_B5OUT2 = 11, /*!< B5OUT2 : Trigger source is CTIMERB5 OUT2. value. */ + CTIMER_AUX2_TMRB2TRIG_A6OUT2DUAL = 12, /*!< A6OUT2DUAL : Trigger source is CTIMERA6 OUT2, dual edge. value. */ + CTIMER_AUX2_TMRB2TRIG_A7OUT2DUAL = 13, /*!< A7OUT2DUAL : Trigger source is CTIMERA7 OUT2, dual edge. value. */ + CTIMER_AUX2_TMRB2TRIG_B4OUT2DUAL = 14, /*!< B4OUT2DUAL : Trigger source is CTIMERB4 OUT2, dual edge. value. */ + CTIMER_AUX2_TMRB2TRIG_A4OUT2DUAL = 15, /*!< A4OUT2DUAL : Trigger source is CTIMERA4 OUT2, dual edge. value. */ +} CTIMER_AUX2_TMRB2TRIG_Enum; + +/* ============================================ CTIMER AUX2 TMRA2EN23 [14..14] ============================================= */ +typedef enum { /*!< CTIMER_AUX2_TMRA2EN23 */ + CTIMER_AUX2_TMRA2EN23_DIS = 1, /*!< DIS : Disable enhanced functions. value. */ + CTIMER_AUX2_TMRA2EN23_EN = 0, /*!< EN : Enable enhanced functions. value. */ +} CTIMER_AUX2_TMRA2EN23_Enum; + +/* ============================================ CTIMER AUX2 TMRA2POL23 [13..13] ============================================ */ +typedef enum { /*!< CTIMER_AUX2_TMRA2POL23 */ + CTIMER_AUX2_TMRA2POL23_NORM = 0, /*!< NORM : Upper output normal polarity value. */ + CTIMER_AUX2_TMRA2POL23_INV = 1, /*!< INV : Upper output inverted polarity. value. */ +} CTIMER_AUX2_TMRA2POL23_Enum; + +/* ============================================ CTIMER AUX2 TMRA2TINV [12..12] ============================================= */ +typedef enum { /*!< CTIMER_AUX2_TMRA2TINV */ + CTIMER_AUX2_TMRA2TINV_DIS = 0, /*!< DIS : Disable invert on trigger value. */ + CTIMER_AUX2_TMRA2TINV_EN = 1, /*!< EN : Enable invert on trigger value. */ +} CTIMER_AUX2_TMRA2TINV_Enum; + +/* =========================================== CTIMER AUX2 TMRA2NOSYNC [11..11] ============================================ */ +typedef enum { /*!< CTIMER_AUX2_TMRA2NOSYNC */ + CTIMER_AUX2_TMRA2NOSYNC_DIS = 0, /*!< DIS : Synchronization on source clock value. */ + CTIMER_AUX2_TMRA2NOSYNC_NOSYNC = 1, /*!< NOSYNC : No synchronization on source clock value. */ +} CTIMER_AUX2_TMRA2NOSYNC_Enum; + +/* ============================================= CTIMER AUX2 TMRA2TRIG [7..10] ============================================= */ +typedef enum { /*!< CTIMER_AUX2_TMRA2TRIG */ + CTIMER_AUX2_TMRA2TRIG_DIS = 0, /*!< DIS : Trigger source is disabled. value. */ + CTIMER_AUX2_TMRA2TRIG_B2OUT = 1, /*!< B2OUT : Trigger source is CTIMERB2 OUT. value. */ + CTIMER_AUX2_TMRA2TRIG_B3OUT = 2, /*!< B3OUT : Trigger source is CTIMERB3 OUT. value. */ + CTIMER_AUX2_TMRA2TRIG_A3OUT = 3, /*!< A3OUT : Trigger source is CTIMERA3 OUT. value. */ + CTIMER_AUX2_TMRA2TRIG_A0OUT = 4, /*!< A0OUT : Trigger source is CTIMERA0 OUT. value. */ + CTIMER_AUX2_TMRA2TRIG_B0OUT = 5, /*!< B0OUT : Trigger source is CTIMERB0 OUT. value. */ + CTIMER_AUX2_TMRA2TRIG_A4OUT = 6, /*!< A4OUT : Trigger source is CTIMERA4 OUT. value. */ + CTIMER_AUX2_TMRA2TRIG_B4OUT = 7, /*!< B4OUT : Trigger source is CTIMERB4 OUT. value. */ + CTIMER_AUX2_TMRA2TRIG_B3OUT2 = 8, /*!< B3OUT2 : Trigger source is CTIMERB3 OUT2. value. */ + CTIMER_AUX2_TMRA2TRIG_A3OUT2 = 9, /*!< A3OUT2 : Trigger source is CTIMERA3 OUT2. value. */ + CTIMER_AUX2_TMRA2TRIG_A5OUT2 = 10, /*!< A5OUT2 : Trigger source is CTIMERA5 OUT2. value. */ + CTIMER_AUX2_TMRA2TRIG_B5OUT2 = 11, /*!< B5OUT2 : Trigger source is CTIMERB5 OUT2. value. */ + CTIMER_AUX2_TMRA2TRIG_A6OUT2DUAL = 12, /*!< A6OUT2DUAL : Trigger source is CTIMERA6 OUT2, dual edge. value. */ + CTIMER_AUX2_TMRA2TRIG_A7OUT2DUAL = 13, /*!< A7OUT2DUAL : Trigger source is CTIMERA7 OUT2, dual edge. value. */ + CTIMER_AUX2_TMRA2TRIG_B4OUT2DUAL = 14, /*!< B4OUT2DUAL : Trigger source is CTIMERB4 OUT2, dual edge. value. */ + CTIMER_AUX2_TMRA2TRIG_A4OUT2DUAL = 15, /*!< A4OUT2DUAL : Trigger source is CTIMERA4 OUT2, dual edge. value. */ +} CTIMER_AUX2_TMRA2TRIG_Enum; + +/* ========================================================= TMR3 ========================================================== */ +/* ======================================================== CMPRA3 ========================================================= */ +/* ======================================================== CMPRB3 ========================================================= */ +/* ========================================================= CTRL3 ========================================================= */ +/* ============================================= CTIMER CTRL3 CTLINK3 [31..31] ============================================= */ +typedef enum { /*!< CTIMER_CTRL3_CTLINK3 */ + CTIMER_CTRL3_CTLINK3_TWO_16BIT_TIMERS = 0, /*!< TWO_16BIT_TIMERS : Use A3/B3 timers as two independent 16-bit + timers (default). value. */ + CTIMER_CTRL3_CTLINK3_32BIT_TIMER = 1, /*!< 32BIT_TIMER : Link A3/B3 timers into a single 32-bit timer. + value. */ +} CTIMER_CTRL3_CTLINK3_Enum; + +/* ============================================ CTIMER CTRL3 TMRB3POL [28..28] ============================================= */ +typedef enum { /*!< CTIMER_CTRL3_TMRB3POL */ + CTIMER_CTRL3_TMRB3POL_NORMAL = 0, /*!< NORMAL : The polarity of the TMRPINB3 pin is the same as the + timer output. value. */ + CTIMER_CTRL3_TMRB3POL_INVERTED = 1, /*!< INVERTED : The polarity of the TMRPINB3 pin is the inverse of + the timer output. value. */ +} CTIMER_CTRL3_TMRB3POL_Enum; + +/* ============================================ CTIMER CTRL3 TMRB3CLR [27..27] ============================================= */ +typedef enum { /*!< CTIMER_CTRL3_TMRB3CLR */ + CTIMER_CTRL3_TMRB3CLR_RUN = 0, /*!< RUN : Allow counter/timer B3 to run value. */ + CTIMER_CTRL3_TMRB3CLR_CLEAR = 1, /*!< CLEAR : Holds counter/timer B3 at 0x0000. value. */ +} CTIMER_CTRL3_TMRB3CLR_Enum; + +/* ============================================ CTIMER CTRL3 TMRB3IE1 [26..26] ============================================= */ +typedef enum { /*!< CTIMER_CTRL3_TMRB3IE1 */ + CTIMER_CTRL3_TMRB3IE1_DIS = 0, /*!< DIS : Disable counter/timer B3 from generating an interrupt + based on COMPR1. value. */ + CTIMER_CTRL3_TMRB3IE1_EN = 1, /*!< EN : Enable counter/timer B3 to generate an interrupt based + on COMPR1. value. */ +} CTIMER_CTRL3_TMRB3IE1_Enum; + +/* ============================================ CTIMER CTRL3 TMRB3IE0 [25..25] ============================================= */ +typedef enum { /*!< CTIMER_CTRL3_TMRB3IE0 */ + CTIMER_CTRL3_TMRB3IE0_DIS = 0, /*!< DIS : Disable counter/timer B3 from generating an interrupt + based on COMPR0. value. */ + CTIMER_CTRL3_TMRB3IE0_EN = 1, /*!< EN : Enable counter/timer B3 to generate an interrupt based + on COMPR0 value. */ +} CTIMER_CTRL3_TMRB3IE0_Enum; + +/* ============================================= CTIMER CTRL3 TMRB3FN [22..24] ============================================= */ +typedef enum { /*!< CTIMER_CTRL3_TMRB3FN */ + CTIMER_CTRL3_TMRB3FN_SINGLECOUNT = 0, /*!< SINGLECOUNT : Single count (output toggles and sticks). Count + to CMPR0B3, stop. value. */ + CTIMER_CTRL3_TMRB3FN_REPEATEDCOUNT = 1, /*!< REPEATEDCOUNT : Repeated count (periodic 1-clock-cycle-wide + pulses). Count to CMPR0B3, restart. value. */ + CTIMER_CTRL3_TMRB3FN_PULSE_ONCE = 2, /*!< PULSE_ONCE : Pulse once (aka one-shot). Count to CMPR0B3, assert, + count to CMPR1B3, deassert, stop. value. */ + CTIMER_CTRL3_TMRB3FN_PULSE_CONT = 3, /*!< PULSE_CONT : Pulse continously. Count to CMPR0B3, assert, count + to CMPR1B3, deassert, restart. value. */ + CTIMER_CTRL3_TMRB3FN_SINGLEPATTERN = 4, /*!< SINGLEPATTERN : Single pattern. value. */ + CTIMER_CTRL3_TMRB3FN_REPEATPATTERN = 5, /*!< REPEATPATTERN : Repeated pattern. value. */ + CTIMER_CTRL3_TMRB3FN_CONTINUOUS = 6, /*!< CONTINUOUS : Continuous run (aka Free Run). Count continuously. + value. */ + CTIMER_CTRL3_TMRB3FN_ALTPWN = 7, /*!< ALTPWN : Alternate PWM value. */ +} CTIMER_CTRL3_TMRB3FN_Enum; + +/* ============================================ CTIMER CTRL3 TMRB3CLK [17..21] ============================================= */ +typedef enum { /*!< CTIMER_CTRL3_TMRB3CLK */ + CTIMER_CTRL3_TMRB3CLK_TMRPIN = 0, /*!< TMRPIN : Clock source is TMRPINB. value. */ + CTIMER_CTRL3_TMRB3CLK_HFRC_DIV4 = 1, /*!< HFRC_DIV4 : Clock source is the HFRC / 4 value. */ + CTIMER_CTRL3_TMRB3CLK_HFRC_DIV16 = 2, /*!< HFRC_DIV16 : Clock source is HFRC / 16 value. */ + CTIMER_CTRL3_TMRB3CLK_HFRC_DIV256 = 3, /*!< HFRC_DIV256 : Clock source is HFRC / 256 value. */ + CTIMER_CTRL3_TMRB3CLK_HFRC_DIV1024 = 4, /*!< HFRC_DIV1024 : Clock source is HFRC / 1024 value. */ + CTIMER_CTRL3_TMRB3CLK_HFRC_DIV4K = 5, /*!< HFRC_DIV4K : Clock source is HFRC / 4096 value. */ + CTIMER_CTRL3_TMRB3CLK_XT = 6, /*!< XT : Clock source is the XT (uncalibrated). value. */ + CTIMER_CTRL3_TMRB3CLK_XT_DIV2 = 7, /*!< XT_DIV2 : Clock source is XT / 2 value. */ + CTIMER_CTRL3_TMRB3CLK_XT_DIV16 = 8, /*!< XT_DIV16 : Clock source is XT / 16 value. */ + CTIMER_CTRL3_TMRB3CLK_XT_DIV128 = 9, /*!< XT_DIV128 : Clock source is XT / 128 value. */ + CTIMER_CTRL3_TMRB3CLK_LFRC_DIV2 = 10, /*!< LFRC_DIV2 : Clock source is LFRC / 2 value. */ + CTIMER_CTRL3_TMRB3CLK_LFRC_DIV32 = 11, /*!< LFRC_DIV32 : Clock source is LFRC / 32 value. */ + CTIMER_CTRL3_TMRB3CLK_LFRC_DIV1K = 12, /*!< LFRC_DIV1K : Clock source is LFRC / 1024 value. */ + CTIMER_CTRL3_TMRB3CLK_LFRC = 13, /*!< LFRC : Clock source is LFRC value. */ + CTIMER_CTRL3_TMRB3CLK_RTC_100HZ = 14, /*!< RTC_100HZ : Clock source is 100 Hz from the current RTC oscillator. + value. */ + CTIMER_CTRL3_TMRB3CLK_HCLK = 15, /*!< HCLK : Clock source is HCLK. value. */ + CTIMER_CTRL3_TMRB3CLK_XT_DIV4 = 16, /*!< XT_DIV4 : Clock source is XT / 4 value. */ + CTIMER_CTRL3_TMRB3CLK_XT_DIV8 = 17, /*!< XT_DIV8 : Clock source is XT / 8 value. */ + CTIMER_CTRL3_TMRB3CLK_XT_DIV32 = 18, /*!< XT_DIV32 : Clock source is XT / 32 value. */ + CTIMER_CTRL3_TMRB3CLK_CTMRA3 = 20, /*!< CTMRA3 : Clock source is CTIMERA3 OUT. value. */ + CTIMER_CTRL3_TMRB3CLK_CTMRA2 = 21, /*!< CTMRA2 : Clock source is CTIMERA2 OUT. value. */ + CTIMER_CTRL3_TMRB3CLK_CTMRB2 = 22, /*!< CTMRB2 : Clock source is CTIMERB2 OUT. value. */ + CTIMER_CTRL3_TMRB3CLK_CTMRA4 = 23, /*!< CTMRA4 : Clock source is CTIMERA4 OUT. value. */ + CTIMER_CTRL3_TMRB3CLK_CTMRB4 = 24, /*!< CTMRB4 : Clock source is CTIMERB4 OUT. value. */ + CTIMER_CTRL3_TMRB3CLK_CTMRB0 = 25, /*!< CTMRB0 : Clock source is CTIMERB0 OUT. value. */ + CTIMER_CTRL3_TMRB3CLK_CTMRB1 = 26, /*!< CTMRB1 : Clock source is CTIMERB1 OUT. value. */ + CTIMER_CTRL3_TMRB3CLK_CTMRB5 = 27, /*!< CTMRB5 : Clock source is CTIMERB5 OUT. value. */ + CTIMER_CTRL3_TMRB3CLK_CTMRB6 = 28, /*!< CTMRB6 : Clock source is CTIMERB6 OUT. value. */ + CTIMER_CTRL3_TMRB3CLK_BUCKBLE = 29, /*!< BUCKBLE : Clock source is BLE buck converter TON pulses. value. */ + CTIMER_CTRL3_TMRB3CLK_BUCKB = 30, /*!< BUCKB : Clock source is Memory buck converter TON pulses. value. */ + CTIMER_CTRL3_TMRB3CLK_BUCKA = 31, /*!< BUCKA : Clock source is CPU buck converter TON pulses. value. */ +} CTIMER_CTRL3_TMRB3CLK_Enum; + +/* ============================================= CTIMER CTRL3 TMRB3EN [16..16] ============================================= */ +typedef enum { /*!< CTIMER_CTRL3_TMRB3EN */ + CTIMER_CTRL3_TMRB3EN_DIS = 0, /*!< DIS : Counter/Timer B3 Disable. value. */ + CTIMER_CTRL3_TMRB3EN_EN = 1, /*!< EN : Counter/Timer B3 Enable. value. */ +} CTIMER_CTRL3_TMRB3EN_Enum; + +/* ============================================ CTIMER CTRL3 TMRA3POL [12..12] ============================================= */ +typedef enum { /*!< CTIMER_CTRL3_TMRA3POL */ + CTIMER_CTRL3_TMRA3POL_NORMAL = 0, /*!< NORMAL : The polarity of the TMRPINA3 pin is the same as the + timer output. value. */ + CTIMER_CTRL3_TMRA3POL_INVERTED = 1, /*!< INVERTED : The polarity of the TMRPINA3 pin is the inverse of + the timer output. value. */ +} CTIMER_CTRL3_TMRA3POL_Enum; + +/* ============================================ CTIMER CTRL3 TMRA3CLR [11..11] ============================================= */ +typedef enum { /*!< CTIMER_CTRL3_TMRA3CLR */ + CTIMER_CTRL3_TMRA3CLR_RUN = 0, /*!< RUN : Allow counter/timer A3 to run value. */ + CTIMER_CTRL3_TMRA3CLR_CLEAR = 1, /*!< CLEAR : Holds counter/timer A3 at 0x0000. value. */ +} CTIMER_CTRL3_TMRA3CLR_Enum; + +/* ============================================ CTIMER CTRL3 TMRA3IE1 [10..10] ============================================= */ +typedef enum { /*!< CTIMER_CTRL3_TMRA3IE1 */ + CTIMER_CTRL3_TMRA3IE1_DIS = 0, /*!< DIS : Disable counter/timer A3 from generating an interrupt + based on COMPR1. value. */ + CTIMER_CTRL3_TMRA3IE1_EN = 1, /*!< EN : Enable counter/timer A3 to generate an interrupt based + on COMPR1. value. */ +} CTIMER_CTRL3_TMRA3IE1_Enum; + +/* ============================================= CTIMER CTRL3 TMRA3IE0 [9..9] ============================================== */ +typedef enum { /*!< CTIMER_CTRL3_TMRA3IE0 */ + CTIMER_CTRL3_TMRA3IE0_DIS = 0, /*!< DIS : Disable counter/timer A3 from generating an interrupt + based on COMPR0. value. */ + CTIMER_CTRL3_TMRA3IE0_EN = 1, /*!< EN : Enable counter/timer A3 to generate an interrupt based + on COMPR0. value. */ +} CTIMER_CTRL3_TMRA3IE0_Enum; + +/* ============================================== CTIMER CTRL3 TMRA3FN [6..8] ============================================== */ +typedef enum { /*!< CTIMER_CTRL3_TMRA3FN */ + CTIMER_CTRL3_TMRA3FN_SINGLECOUNT = 0, /*!< SINGLECOUNT : Single count (output toggles and sticks). Count + to CMPR0A3, stop. value. */ + CTIMER_CTRL3_TMRA3FN_REPEATEDCOUNT = 1, /*!< REPEATEDCOUNT : Repeated count (periodic 1-clock-cycle-wide + pulses). Count to CMPR0A3, restart. value. */ + CTIMER_CTRL3_TMRA3FN_PULSE_ONCE = 2, /*!< PULSE_ONCE : Pulse once (aka one-shot). Count to CMPR0A3, assert, + count to CMPR1A3, deassert, stop. value. */ + CTIMER_CTRL3_TMRA3FN_PULSE_CONT = 3, /*!< PULSE_CONT : Pulse continously. Count to CMPR0A3, assert, count + to CMPR1A3, deassert, restart. value. */ + CTIMER_CTRL3_TMRA3FN_SINGLEPATTERN = 4, /*!< SINGLEPATTERN : Single pattern. value. */ + CTIMER_CTRL3_TMRA3FN_REPEATPATTERN = 5, /*!< REPEATPATTERN : Repeated pattern. value. */ + CTIMER_CTRL3_TMRA3FN_CONTINUOUS = 6, /*!< CONTINUOUS : Continuous run (aka Free Run). Count continuously. + value. */ + CTIMER_CTRL3_TMRA3FN_ALTPWN = 7, /*!< ALTPWN : Alternate PWM value. */ +} CTIMER_CTRL3_TMRA3FN_Enum; + +/* ============================================= CTIMER CTRL3 TMRA3CLK [1..5] ============================================== */ +typedef enum { /*!< CTIMER_CTRL3_TMRA3CLK */ + CTIMER_CTRL3_TMRA3CLK_TMRPIN = 0, /*!< TMRPIN : Clock source is TMRPINA. value. */ + CTIMER_CTRL3_TMRA3CLK_HFRC_DIV4 = 1, /*!< HFRC_DIV4 : Clock source is the HFRC / 4 value. */ + CTIMER_CTRL3_TMRA3CLK_HFRC_DIV16 = 2, /*!< HFRC_DIV16 : Clock source is HFRC / 16 value. */ + CTIMER_CTRL3_TMRA3CLK_HFRC_DIV256 = 3, /*!< HFRC_DIV256 : Clock source is HFRC / 256 value. */ + CTIMER_CTRL3_TMRA3CLK_HFRC_DIV1024 = 4, /*!< HFRC_DIV1024 : Clock source is HFRC / 1024 value. */ + CTIMER_CTRL3_TMRA3CLK_HFRC_DIV4K = 5, /*!< HFRC_DIV4K : Clock source is HFRC / 4096 value. */ + CTIMER_CTRL3_TMRA3CLK_XT = 6, /*!< XT : Clock source is the XT (uncalibrated). value. */ + CTIMER_CTRL3_TMRA3CLK_XT_DIV2 = 7, /*!< XT_DIV2 : Clock source is XT / 2 value. */ + CTIMER_CTRL3_TMRA3CLK_XT_DIV16 = 8, /*!< XT_DIV16 : Clock source is XT / 16 value. */ + CTIMER_CTRL3_TMRA3CLK_XT_DIV128 = 9, /*!< XT_DIV128 : Clock source is XT / 128 value. */ + CTIMER_CTRL3_TMRA3CLK_LFRC_DIV2 = 10, /*!< LFRC_DIV2 : Clock source is LFRC / 2 value. */ + CTIMER_CTRL3_TMRA3CLK_LFRC_DIV32 = 11, /*!< LFRC_DIV32 : Clock source is LFRC / 32 value. */ + CTIMER_CTRL3_TMRA3CLK_LFRC_DIV1K = 12, /*!< LFRC_DIV1K : Clock source is LFRC / 1024 value. */ + CTIMER_CTRL3_TMRA3CLK_LFRC = 13, /*!< LFRC : Clock source is LFRC value. */ + CTIMER_CTRL3_TMRA3CLK_RTC_100HZ = 14, /*!< RTC_100HZ : Clock source is 100 Hz from the current RTC oscillator. + value. */ + CTIMER_CTRL3_TMRA3CLK_HCLK = 15, /*!< HCLK : Clock source is HCLK. value. */ + CTIMER_CTRL3_TMRA3CLK_XT_DIV4 = 16, /*!< XT_DIV4 : Clock source is XT / 4 value. */ + CTIMER_CTRL3_TMRA3CLK_XT_DIV8 = 17, /*!< XT_DIV8 : Clock source is XT / 8 value. */ + CTIMER_CTRL3_TMRA3CLK_XT_DIV32 = 18, /*!< XT_DIV32 : Clock source is XT / 32 value. */ + CTIMER_CTRL3_TMRA3CLK_CTMRB3 = 20, /*!< CTMRB3 : Clock source is CTIMERB3 OUT. value. */ + CTIMER_CTRL3_TMRA3CLK_CTMRA2 = 21, /*!< CTMRA2 : Clock source is CTIMERA2 OUT. value. */ + CTIMER_CTRL3_TMRA3CLK_CTMRB2 = 22, /*!< CTMRB2 : Clock source is CTIMERB2 OUT. value. */ + CTIMER_CTRL3_TMRA3CLK_CTMRA4 = 23, /*!< CTMRA4 : Clock source is CTIMERA4 OUT. value. */ + CTIMER_CTRL3_TMRA3CLK_CTMRB4 = 24, /*!< CTMRB4 : Clock source is CTIMERB4 OUT. value. */ + CTIMER_CTRL3_TMRA3CLK_CTMRB0 = 25, /*!< CTMRB0 : Clock source is CTIMERB0 OUT. value. */ + CTIMER_CTRL3_TMRA3CLK_CTMRB1 = 26, /*!< CTMRB1 : Clock source is CTIMERB1 OUT. value. */ + CTIMER_CTRL3_TMRA3CLK_CTMRB5 = 27, /*!< CTMRB5 : Clock source is CTIMERB5 OUT. value. */ + CTIMER_CTRL3_TMRA3CLK_CTMRB6 = 28, /*!< CTMRB6 : Clock source is CTIMERB6 OUT. value. */ + CTIMER_CTRL3_TMRA3CLK_BUCKBLE = 29, /*!< BUCKBLE : Clock source is BLE buck converter TON pulses. value. */ + CTIMER_CTRL3_TMRA3CLK_BUCKB = 30, /*!< BUCKB : Clock source is Memory buck converter TON pulses. value. */ + CTIMER_CTRL3_TMRA3CLK_BUCKA = 31, /*!< BUCKA : Clock source is CPU buck converter TON pulses. value. */ +} CTIMER_CTRL3_TMRA3CLK_Enum; + +/* ============================================== CTIMER CTRL3 TMRA3EN [0..0] ============================================== */ +typedef enum { /*!< CTIMER_CTRL3_TMRA3EN */ + CTIMER_CTRL3_TMRA3EN_DIS = 0, /*!< DIS : Counter/Timer A3 Disable. value. */ + CTIMER_CTRL3_TMRA3EN_EN = 1, /*!< EN : Counter/Timer A3 Enable. value. */ +} CTIMER_CTRL3_TMRA3EN_Enum; + +/* ======================================================= CMPRAUXA3 ======================================================= */ +/* ======================================================= CMPRAUXB3 ======================================================= */ +/* ========================================================= AUX3 ========================================================== */ +/* ============================================ CTIMER AUX3 TMRB3EN23 [30..30] ============================================= */ +typedef enum { /*!< CTIMER_AUX3_TMRB3EN23 */ + CTIMER_AUX3_TMRB3EN23_DIS = 1, /*!< DIS : Disable enhanced functions. value. */ + CTIMER_AUX3_TMRB3EN23_EN = 0, /*!< EN : Enable enhanced functions. value. */ +} CTIMER_AUX3_TMRB3EN23_Enum; + +/* ============================================ CTIMER AUX3 TMRB3POL23 [29..29] ============================================ */ +typedef enum { /*!< CTIMER_AUX3_TMRB3POL23 */ + CTIMER_AUX3_TMRB3POL23_NORM = 0, /*!< NORM : Upper output normal polarity value. */ + CTIMER_AUX3_TMRB3POL23_INV = 1, /*!< INV : Upper output inverted polarity. value. */ +} CTIMER_AUX3_TMRB3POL23_Enum; + +/* ============================================ CTIMER AUX3 TMRB3TINV [28..28] ============================================= */ +typedef enum { /*!< CTIMER_AUX3_TMRB3TINV */ + CTIMER_AUX3_TMRB3TINV_DIS = 0, /*!< DIS : Disable invert on trigger value. */ + CTIMER_AUX3_TMRB3TINV_EN = 1, /*!< EN : Enable invert on trigger value. */ +} CTIMER_AUX3_TMRB3TINV_Enum; + +/* =========================================== CTIMER AUX3 TMRB3NOSYNC [27..27] ============================================ */ +typedef enum { /*!< CTIMER_AUX3_TMRB3NOSYNC */ + CTIMER_AUX3_TMRB3NOSYNC_DIS = 0, /*!< DIS : Synchronization on source clock value. */ + CTIMER_AUX3_TMRB3NOSYNC_NOSYNC = 1, /*!< NOSYNC : No synchronization on source clock value. */ +} CTIMER_AUX3_TMRB3NOSYNC_Enum; + +/* ============================================ CTIMER AUX3 TMRB3TRIG [23..26] ============================================= */ +typedef enum { /*!< CTIMER_AUX3_TMRB3TRIG */ + CTIMER_AUX3_TMRB3TRIG_DIS = 0, /*!< DIS : Trigger source is disabled. value. */ + CTIMER_AUX3_TMRB3TRIG_A3OUT = 1, /*!< A3OUT : Trigger source is CTIMERA3 OUT. value. */ + CTIMER_AUX3_TMRB3TRIG_B2OUT = 2, /*!< B2OUT : Trigger source is CTIMERB2 OUT. value. */ + CTIMER_AUX3_TMRB3TRIG_A2OUT = 3, /*!< A2OUT : Trigger source is CTIMERA2 OUT. value. */ + CTIMER_AUX3_TMRB3TRIG_A4OUT = 4, /*!< A4OUT : Trigger source is CTIMERA4 OUT. value. */ + CTIMER_AUX3_TMRB3TRIG_B4OUT = 5, /*!< B4OUT : Trigger source is CTIMERB4 OUT. value. */ + CTIMER_AUX3_TMRB3TRIG_A6OUT = 6, /*!< A6OUT : Trigger source is CTIMERA6 OUT. value. */ + CTIMER_AUX3_TMRB3TRIG_B6OUT = 7, /*!< B6OUT : Trigger source is CTIMERB6 OUT. value. */ + CTIMER_AUX3_TMRB3TRIG_B5OUT2 = 8, /*!< B5OUT2 : Trigger source is CTIMERB5 OUT2. value. */ + CTIMER_AUX3_TMRB3TRIG_A5OUT2 = 9, /*!< A5OUT2 : Trigger source is CTIMERA5 OUT2. value. */ + CTIMER_AUX3_TMRB3TRIG_A1OUT2 = 10, /*!< A1OUT2 : Trigger source is CTIMERA1 OUT2. value. */ + CTIMER_AUX3_TMRB3TRIG_B1OUT2 = 11, /*!< B1OUT2 : Trigger source is CTIMERB1 OUT2. value. */ + CTIMER_AUX3_TMRB3TRIG_A6OUT2DUAL = 12, /*!< A6OUT2DUAL : Trigger source is CTIMERA6 OUT2, dual edge. value. */ + CTIMER_AUX3_TMRB3TRIG_A7OUT2DUAL = 13, /*!< A7OUT2DUAL : Trigger source is CTIMERA7 OUT2, dual edge. value. */ + CTIMER_AUX3_TMRB3TRIG_B2OUT2DUAL = 14, /*!< B2OUT2DUAL : Trigger source is CTIMERB2 OUT2, dual edge. value. */ + CTIMER_AUX3_TMRB3TRIG_A2OUT2DUAL = 15, /*!< A2OUT2DUAL : Trigger source is CTIMERA2 OUT2, dual edge. value. */ +} CTIMER_AUX3_TMRB3TRIG_Enum; + +/* ============================================ CTIMER AUX3 TMRA3EN23 [14..14] ============================================= */ +typedef enum { /*!< CTIMER_AUX3_TMRA3EN23 */ + CTIMER_AUX3_TMRA3EN23_DIS = 1, /*!< DIS : Disable enhanced functions. value. */ + CTIMER_AUX3_TMRA3EN23_EN = 0, /*!< EN : Enable enhanced functions. value. */ +} CTIMER_AUX3_TMRA3EN23_Enum; + +/* ============================================ CTIMER AUX3 TMRA3POL23 [13..13] ============================================ */ +typedef enum { /*!< CTIMER_AUX3_TMRA3POL23 */ + CTIMER_AUX3_TMRA3POL23_NORM = 0, /*!< NORM : Upper output normal polarity value. */ + CTIMER_AUX3_TMRA3POL23_INV = 1, /*!< INV : Upper output inverted polarity. value. */ +} CTIMER_AUX3_TMRA3POL23_Enum; + +/* ============================================ CTIMER AUX3 TMRA3TINV [12..12] ============================================= */ +typedef enum { /*!< CTIMER_AUX3_TMRA3TINV */ + CTIMER_AUX3_TMRA3TINV_DIS = 0, /*!< DIS : Disable invert on trigger value. */ + CTIMER_AUX3_TMRA3TINV_EN = 1, /*!< EN : Enable invert on trigger value. */ +} CTIMER_AUX3_TMRA3TINV_Enum; + +/* =========================================== CTIMER AUX3 TMRA3NOSYNC [11..11] ============================================ */ +typedef enum { /*!< CTIMER_AUX3_TMRA3NOSYNC */ + CTIMER_AUX3_TMRA3NOSYNC_DIS = 0, /*!< DIS : Synchronization on source clock value. */ + CTIMER_AUX3_TMRA3NOSYNC_NOSYNC = 1, /*!< NOSYNC : No synchronization on source clock value. */ +} CTIMER_AUX3_TMRA3NOSYNC_Enum; + +/* ============================================= CTIMER AUX3 TMRA3TRIG [7..10] ============================================= */ +typedef enum { /*!< CTIMER_AUX3_TMRA3TRIG */ + CTIMER_AUX3_TMRA3TRIG_DIS = 0, /*!< DIS : Trigger source is disabled. value. */ + CTIMER_AUX3_TMRA3TRIG_B3OUT = 1, /*!< B3OUT : Trigger source is CTIMERB3 OUT. value. */ + CTIMER_AUX3_TMRA3TRIG_B2OUT = 2, /*!< B2OUT : Trigger source is CTIMERB2 OUT. value. */ + CTIMER_AUX3_TMRA3TRIG_A2OUT = 3, /*!< A2OUT : Trigger source is CTIMERA2 OUT. value. */ + CTIMER_AUX3_TMRA3TRIG_A4OUT = 4, /*!< A4OUT : Trigger source is CTIMERA4 OUT. value. */ + CTIMER_AUX3_TMRA3TRIG_B4OUT = 5, /*!< B4OUT : Trigger source is CTIMERB4 OUT. value. */ + CTIMER_AUX3_TMRA3TRIG_A7OUT = 6, /*!< A7OUT : Trigger source is CTIMERA7 OUT. value. */ + CTIMER_AUX3_TMRA3TRIG_B7OUT = 7, /*!< B7OUT : Trigger source is CTIMERB7 OUT. value. */ + CTIMER_AUX3_TMRA3TRIG_B5OUT2 = 8, /*!< B5OUT2 : Trigger source is CTIMERB5 OUT2. value. */ + CTIMER_AUX3_TMRA3TRIG_A5OUT2 = 9, /*!< A5OUT2 : Trigger source is CTIMERA5 OUT2. value. */ + CTIMER_AUX3_TMRA3TRIG_A1OUT2 = 10, /*!< A1OUT2 : Trigger source is CTIMERA1 OUT2. value. */ + CTIMER_AUX3_TMRA3TRIG_B1OUT2 = 11, /*!< B1OUT2 : Trigger source is CTIMERB1 OUT2. value. */ + CTIMER_AUX3_TMRA3TRIG_A6OUT2DUAL = 12, /*!< A6OUT2DUAL : Trigger source is CTIMERA6 OUT2, dual edge. value. */ + CTIMER_AUX3_TMRA3TRIG_A7OUT2DUAL = 13, /*!< A7OUT2DUAL : Trigger source is CTIMERA7 OUT2, dual edge. value. */ + CTIMER_AUX3_TMRA3TRIG_B2OUT2DUAL = 14, /*!< B2OUT2DUAL : Trigger source is CTIMERB2 OUT2, dual edge. value. */ + CTIMER_AUX3_TMRA3TRIG_A2OUT2DUAL = 15, /*!< A2OUT2DUAL : Trigger source is CTIMERA2 OUT2, dual edge. value. */ +} CTIMER_AUX3_TMRA3TRIG_Enum; + +/* ========================================================= TMR4 ========================================================== */ +/* ======================================================== CMPRA4 ========================================================= */ +/* ======================================================== CMPRB4 ========================================================= */ +/* ========================================================= CTRL4 ========================================================= */ +/* ============================================= CTIMER CTRL4 CTLINK4 [31..31] ============================================= */ +typedef enum { /*!< CTIMER_CTRL4_CTLINK4 */ + CTIMER_CTRL4_CTLINK4_TWO_16BIT_TIMERS = 0, /*!< TWO_16BIT_TIMERS : Use A4/B4 timers as two independent 16-bit + timers (default). value. */ + CTIMER_CTRL4_CTLINK4_32BIT_TIMER = 1, /*!< 32BIT_TIMER : Link A4/B4 timers into a single 32-bit timer. + value. */ +} CTIMER_CTRL4_CTLINK4_Enum; + +/* ============================================ CTIMER CTRL4 TMRB4POL [28..28] ============================================= */ +typedef enum { /*!< CTIMER_CTRL4_TMRB4POL */ + CTIMER_CTRL4_TMRB4POL_NORMAL = 0, /*!< NORMAL : The polarity of the TMRPINB4 pin is the same as the + timer output. value. */ + CTIMER_CTRL4_TMRB4POL_INVERTED = 1, /*!< INVERTED : The polarity of the TMRPINB4 pin is the inverse of + the timer output. value. */ +} CTIMER_CTRL4_TMRB4POL_Enum; + +/* ============================================ CTIMER CTRL4 TMRB4CLR [27..27] ============================================= */ +typedef enum { /*!< CTIMER_CTRL4_TMRB4CLR */ + CTIMER_CTRL4_TMRB4CLR_RUN = 0, /*!< RUN : Allow counter/timer B4 to run value. */ + CTIMER_CTRL4_TMRB4CLR_CLEAR = 1, /*!< CLEAR : Holds counter/timer B4 at 0x0000. value. */ +} CTIMER_CTRL4_TMRB4CLR_Enum; + +/* ============================================ CTIMER CTRL4 TMRB4IE1 [26..26] ============================================= */ +typedef enum { /*!< CTIMER_CTRL4_TMRB4IE1 */ + CTIMER_CTRL4_TMRB4IE1_DIS = 0, /*!< DIS : Disable counter/timer B4 from generating an interrupt + based on COMPR1. value. */ + CTIMER_CTRL4_TMRB4IE1_EN = 1, /*!< EN : Enable counter/timer B4 to generate an interrupt based + on COMPR1. value. */ +} CTIMER_CTRL4_TMRB4IE1_Enum; + +/* ============================================ CTIMER CTRL4 TMRB4IE0 [25..25] ============================================= */ +typedef enum { /*!< CTIMER_CTRL4_TMRB4IE0 */ + CTIMER_CTRL4_TMRB4IE0_DIS = 0, /*!< DIS : Disable counter/timer B4 from generating an interrupt + based on COMPR0. value. */ + CTIMER_CTRL4_TMRB4IE0_EN = 1, /*!< EN : Enable counter/timer B4 to generate an interrupt based + on COMPR0 value. */ +} CTIMER_CTRL4_TMRB4IE0_Enum; + +/* ============================================= CTIMER CTRL4 TMRB4FN [22..24] ============================================= */ +typedef enum { /*!< CTIMER_CTRL4_TMRB4FN */ + CTIMER_CTRL4_TMRB4FN_SINGLECOUNT = 0, /*!< SINGLECOUNT : Single count (output toggles and sticks). Count + to CMPR0B4, stop. value. */ + CTIMER_CTRL4_TMRB4FN_REPEATEDCOUNT = 1, /*!< REPEATEDCOUNT : Repeated count (periodic 1-clock-cycle-wide + pulses). Count to CMPR0B4, restart. value. */ + CTIMER_CTRL4_TMRB4FN_PULSE_ONCE = 2, /*!< PULSE_ONCE : Pulse once (aka one-shot). Count to CMPR0B4, assert, + count to CMPR1B4, deassert, stop. value. */ + CTIMER_CTRL4_TMRB4FN_PULSE_CONT = 3, /*!< PULSE_CONT : Pulse continously. Count to CMPR0B4, assert, count + to CMPR1B4, deassert, restart. value. */ + CTIMER_CTRL4_TMRB4FN_SINGLEPATTERN = 4, /*!< SINGLEPATTERN : Single pattern. value. */ + CTIMER_CTRL4_TMRB4FN_REPEATPATTERN = 5, /*!< REPEATPATTERN : Repeated pattern. value. */ + CTIMER_CTRL4_TMRB4FN_CONTINUOUS = 6, /*!< CONTINUOUS : Continuous run (aka Free Run). Count continuously. + value. */ + CTIMER_CTRL4_TMRB4FN_ALTPWN = 7, /*!< ALTPWN : Alternate PWM value. */ +} CTIMER_CTRL4_TMRB4FN_Enum; + +/* ============================================ CTIMER CTRL4 TMRB4CLK [17..21] ============================================= */ +typedef enum { /*!< CTIMER_CTRL4_TMRB4CLK */ + CTIMER_CTRL4_TMRB4CLK_TMRPIN = 0, /*!< TMRPIN : Clock source is TMRPINB. value. */ + CTIMER_CTRL4_TMRB4CLK_HFRC_DIV4 = 1, /*!< HFRC_DIV4 : Clock source is the HFRC / 4 value. */ + CTIMER_CTRL4_TMRB4CLK_HFRC_DIV16 = 2, /*!< HFRC_DIV16 : Clock source is HFRC / 16 value. */ + CTIMER_CTRL4_TMRB4CLK_HFRC_DIV256 = 3, /*!< HFRC_DIV256 : Clock source is HFRC / 256 value. */ + CTIMER_CTRL4_TMRB4CLK_HFRC_DIV1024 = 4, /*!< HFRC_DIV1024 : Clock source is HFRC / 1024 value. */ + CTIMER_CTRL4_TMRB4CLK_HFRC_DIV4K = 5, /*!< HFRC_DIV4K : Clock source is HFRC / 4096 value. */ + CTIMER_CTRL4_TMRB4CLK_XT = 6, /*!< XT : Clock source is the XT (uncalibrated). value. */ + CTIMER_CTRL4_TMRB4CLK_XT_DIV2 = 7, /*!< XT_DIV2 : Clock source is XT / 2 value. */ + CTIMER_CTRL4_TMRB4CLK_XT_DIV16 = 8, /*!< XT_DIV16 : Clock source is XT / 16 value. */ + CTIMER_CTRL4_TMRB4CLK_XT_DIV128 = 9, /*!< XT_DIV128 : Clock source is XT / 128 value. */ + CTIMER_CTRL4_TMRB4CLK_LFRC_DIV2 = 10, /*!< LFRC_DIV2 : Clock source is LFRC / 2 value. */ + CTIMER_CTRL4_TMRB4CLK_LFRC_DIV32 = 11, /*!< LFRC_DIV32 : Clock source is LFRC / 32 value. */ + CTIMER_CTRL4_TMRB4CLK_LFRC_DIV1K = 12, /*!< LFRC_DIV1K : Clock source is LFRC / 1024 value. */ + CTIMER_CTRL4_TMRB4CLK_LFRC = 13, /*!< LFRC : Clock source is LFRC value. */ + CTIMER_CTRL4_TMRB4CLK_RTC_100HZ = 14, /*!< RTC_100HZ : Clock source is 100 Hz from the current RTC oscillator. + value. */ + CTIMER_CTRL4_TMRB4CLK_HCLK = 15, /*!< HCLK : Clock source is HCLK. value. */ + CTIMER_CTRL4_TMRB4CLK_XT_DIV4 = 16, /*!< XT_DIV4 : Clock source is XT / 4 value. */ + CTIMER_CTRL4_TMRB4CLK_XT_DIV8 = 17, /*!< XT_DIV8 : Clock source is XT / 8 value. */ + CTIMER_CTRL4_TMRB4CLK_XT_DIV32 = 18, /*!< XT_DIV32 : Clock source is XT / 32 value. */ + CTIMER_CTRL4_TMRB4CLK_CTMRA4 = 20, /*!< CTMRA4 : Clock source is CTIMERA4 OUT. value. */ + CTIMER_CTRL4_TMRB4CLK_CTMRA1 = 21, /*!< CTMRA1 : Clock source is CTIMERA1 OUT. value. */ + CTIMER_CTRL4_TMRB4CLK_CTMRB1 = 22, /*!< CTMRB1 : Clock source is CTIMERB1 OUT. value. */ + CTIMER_CTRL4_TMRB4CLK_CTMRA5 = 23, /*!< CTMRA5 : Clock source is CTIMERA5 OUT. value. */ + CTIMER_CTRL4_TMRB4CLK_CTMRB5 = 24, /*!< CTMRB5 : Clock source is CTIMERB5 OUT. value. */ + CTIMER_CTRL4_TMRB4CLK_CTMRB0 = 25, /*!< CTMRB0 : Clock source is CTIMERB0 OUT. value. */ + CTIMER_CTRL4_TMRB4CLK_CTMRB2 = 26, /*!< CTMRB2 : Clock source is CTIMERB2 OUT. value. */ + CTIMER_CTRL4_TMRB4CLK_CTMRB3 = 27, /*!< CTMRB3 : Clock source is CTIMERB3 OUT. value. */ + CTIMER_CTRL4_TMRB4CLK_CTMRB6 = 28, /*!< CTMRB6 : Clock source is CTIMERB6 OUT. value. */ + CTIMER_CTRL4_TMRB4CLK_BUCKBLE = 29, /*!< BUCKBLE : Clock source is BLE buck converter TON pulses. value. */ + CTIMER_CTRL4_TMRB4CLK_BUCKB = 30, /*!< BUCKB : Clock source is Memory buck converter TON pulses. value. */ + CTIMER_CTRL4_TMRB4CLK_BUCKA = 31, /*!< BUCKA : Clock source is CPU buck converter TON pulses. value. */ +} CTIMER_CTRL4_TMRB4CLK_Enum; + +/* ============================================= CTIMER CTRL4 TMRB4EN [16..16] ============================================= */ +typedef enum { /*!< CTIMER_CTRL4_TMRB4EN */ + CTIMER_CTRL4_TMRB4EN_DIS = 0, /*!< DIS : Counter/Timer B4 Disable. value. */ + CTIMER_CTRL4_TMRB4EN_EN = 1, /*!< EN : Counter/Timer B4 Enable. value. */ +} CTIMER_CTRL4_TMRB4EN_Enum; + +/* ============================================ CTIMER CTRL4 TMRA4POL [12..12] ============================================= */ +typedef enum { /*!< CTIMER_CTRL4_TMRA4POL */ + CTIMER_CTRL4_TMRA4POL_NORMAL = 0, /*!< NORMAL : The polarity of the TMRPINA4 pin is the same as the + timer output. value. */ + CTIMER_CTRL4_TMRA4POL_INVERTED = 1, /*!< INVERTED : The polarity of the TMRPINA4 pin is the inverse of + the timer output. value. */ +} CTIMER_CTRL4_TMRA4POL_Enum; + +/* ============================================ CTIMER CTRL4 TMRA4CLR [11..11] ============================================= */ +typedef enum { /*!< CTIMER_CTRL4_TMRA4CLR */ + CTIMER_CTRL4_TMRA4CLR_RUN = 0, /*!< RUN : Allow counter/timer A4 to run value. */ + CTIMER_CTRL4_TMRA4CLR_CLEAR = 1, /*!< CLEAR : Holds counter/timer A4 at 0x0000. value. */ +} CTIMER_CTRL4_TMRA4CLR_Enum; + +/* ============================================ CTIMER CTRL4 TMRA4IE1 [10..10] ============================================= */ +typedef enum { /*!< CTIMER_CTRL4_TMRA4IE1 */ + CTIMER_CTRL4_TMRA4IE1_DIS = 0, /*!< DIS : Disable counter/timer A4 from generating an interrupt + based on COMPR1. value. */ + CTIMER_CTRL4_TMRA4IE1_EN = 1, /*!< EN : Enable counter/timer A4 to generate an interrupt based + on COMPR1. value. */ +} CTIMER_CTRL4_TMRA4IE1_Enum; + +/* ============================================= CTIMER CTRL4 TMRA4IE0 [9..9] ============================================== */ +typedef enum { /*!< CTIMER_CTRL4_TMRA4IE0 */ + CTIMER_CTRL4_TMRA4IE0_DIS = 0, /*!< DIS : Disable counter/timer A4 from generating an interrupt + based on COMPR0. value. */ + CTIMER_CTRL4_TMRA4IE0_EN = 1, /*!< EN : Enable counter/timer A4 to generate an interrupt based + on COMPR0. value. */ +} CTIMER_CTRL4_TMRA4IE0_Enum; + +/* ============================================== CTIMER CTRL4 TMRA4FN [6..8] ============================================== */ +typedef enum { /*!< CTIMER_CTRL4_TMRA4FN */ + CTIMER_CTRL4_TMRA4FN_SINGLECOUNT = 0, /*!< SINGLECOUNT : Single count (output toggles and sticks). Count + to CMPR0A4, stop. value. */ + CTIMER_CTRL4_TMRA4FN_REPEATEDCOUNT = 1, /*!< REPEATEDCOUNT : Repeated count (periodic 1-clock-cycle-wide + pulses). Count to CMPR0A4, restart. value. */ + CTIMER_CTRL4_TMRA4FN_PULSE_ONCE = 2, /*!< PULSE_ONCE : Pulse once (aka one-shot). Count to CMPR0A4, assert, + count to CMPR1A4, deassert, stop. value. */ + CTIMER_CTRL4_TMRA4FN_PULSE_CONT = 3, /*!< PULSE_CONT : Pulse continously. Count to CMPR0A4, assert, count + to CMPR1A4, deassert, restart. value. */ + CTIMER_CTRL4_TMRA4FN_SINGLEPATTERN = 4, /*!< SINGLEPATTERN : Single pattern. value. */ + CTIMER_CTRL4_TMRA4FN_REPEATPATTERN = 5, /*!< REPEATPATTERN : Repeated pattern. value. */ + CTIMER_CTRL4_TMRA4FN_CONTINUOUS = 6, /*!< CONTINUOUS : Continuous run (aka Free Run). Count continuously. + value. */ + CTIMER_CTRL4_TMRA4FN_ALTPWN = 7, /*!< ALTPWN : Alternate PWM value. */ +} CTIMER_CTRL4_TMRA4FN_Enum; + +/* ============================================= CTIMER CTRL4 TMRA4CLK [1..5] ============================================== */ +typedef enum { /*!< CTIMER_CTRL4_TMRA4CLK */ + CTIMER_CTRL4_TMRA4CLK_TMRPIN = 0, /*!< TMRPIN : Clock source is TMRPINA. value. */ + CTIMER_CTRL4_TMRA4CLK_HFRC_DIV4 = 1, /*!< HFRC_DIV4 : Clock source is the HFRC / 4 value. */ + CTIMER_CTRL4_TMRA4CLK_HFRC_DIV16 = 2, /*!< HFRC_DIV16 : Clock source is HFRC / 16 value. */ + CTIMER_CTRL4_TMRA4CLK_HFRC_DIV256 = 3, /*!< HFRC_DIV256 : Clock source is HFRC / 256 value. */ + CTIMER_CTRL4_TMRA4CLK_HFRC_DIV1024 = 4, /*!< HFRC_DIV1024 : Clock source is HFRC / 1024 value. */ + CTIMER_CTRL4_TMRA4CLK_HFRC_DIV4K = 5, /*!< HFRC_DIV4K : Clock source is HFRC / 4096 value. */ + CTIMER_CTRL4_TMRA4CLK_XT = 6, /*!< XT : Clock source is the XT (uncalibrated). value. */ + CTIMER_CTRL4_TMRA4CLK_XT_DIV2 = 7, /*!< XT_DIV2 : Clock source is XT / 2 value. */ + CTIMER_CTRL4_TMRA4CLK_XT_DIV16 = 8, /*!< XT_DIV16 : Clock source is XT / 16 value. */ + CTIMER_CTRL4_TMRA4CLK_XT_DIV128 = 9, /*!< XT_DIV128 : Clock source is XT / 128 value. */ + CTIMER_CTRL4_TMRA4CLK_LFRC_DIV2 = 10, /*!< LFRC_DIV2 : Clock source is LFRC / 2 value. */ + CTIMER_CTRL4_TMRA4CLK_LFRC_DIV32 = 11, /*!< LFRC_DIV32 : Clock source is LFRC / 32 value. */ + CTIMER_CTRL4_TMRA4CLK_LFRC_DIV1K = 12, /*!< LFRC_DIV1K : Clock source is LFRC / 1024 value. */ + CTIMER_CTRL4_TMRA4CLK_LFRC = 13, /*!< LFRC : Clock source is LFRC value. */ + CTIMER_CTRL4_TMRA4CLK_RTC_100HZ = 14, /*!< RTC_100HZ : Clock source is 100 Hz from the current RTC oscillator. + value. */ + CTIMER_CTRL4_TMRA4CLK_HCLK_DIV4 = 15, /*!< HCLK_DIV4 : Clock source is HCLK / 4. value. */ + CTIMER_CTRL4_TMRA4CLK_XT_DIV4 = 16, /*!< XT_DIV4 : Clock source is XT / 4 value. */ + CTIMER_CTRL4_TMRA4CLK_XT_DIV8 = 17, /*!< XT_DIV8 : Clock source is XT / 8 value. */ + CTIMER_CTRL4_TMRA4CLK_XT_DIV32 = 18, /*!< XT_DIV32 : Clock source is XT / 32 value. */ + CTIMER_CTRL4_TMRA4CLK_CTMRB4 = 20, /*!< CTMRB4 : Clock source is CTIMERB4 OUT. value. */ + CTIMER_CTRL4_TMRA4CLK_CTMRA1 = 21, /*!< CTMRA1 : Clock source is CTIMERA1 OUT. value. */ + CTIMER_CTRL4_TMRA4CLK_CTMRB1 = 22, /*!< CTMRB1 : Clock source is CTIMERB1 OUT. value. */ + CTIMER_CTRL4_TMRA4CLK_CTMRA5 = 23, /*!< CTMRA5 : Clock source is CTIMERA5 OUT. value. */ + CTIMER_CTRL4_TMRA4CLK_CTMRB5 = 24, /*!< CTMRB5 : Clock source is CTIMERB5 OUT. value. */ + CTIMER_CTRL4_TMRA4CLK_CTMRB0 = 25, /*!< CTMRB0 : Clock source is CTIMERB0 OUT. value. */ + CTIMER_CTRL4_TMRA4CLK_CTMRB2 = 26, /*!< CTMRB2 : Clock source is CTIMERB2 OUT. value. */ + CTIMER_CTRL4_TMRA4CLK_CTMRB3 = 27, /*!< CTMRB3 : Clock source is CTIMERB3 OUT. value. */ + CTIMER_CTRL4_TMRA4CLK_CTMRB6 = 28, /*!< CTMRB6 : Clock source is CTIMERB6 OUT. value. */ + CTIMER_CTRL4_TMRA4CLK_BUCKBLE = 29, /*!< BUCKBLE : Clock source is BLE buck converter TON pulses. value. */ + CTIMER_CTRL4_TMRA4CLK_BUCKB = 30, /*!< BUCKB : Clock source is Memory buck converter TON pulses. value. */ + CTIMER_CTRL4_TMRA4CLK_BUCKA = 31, /*!< BUCKA : Clock source is CPU buck converter TON pulses. value. */ +} CTIMER_CTRL4_TMRA4CLK_Enum; + +/* ============================================== CTIMER CTRL4 TMRA4EN [0..0] ============================================== */ +typedef enum { /*!< CTIMER_CTRL4_TMRA4EN */ + CTIMER_CTRL4_TMRA4EN_DIS = 0, /*!< DIS : Counter/Timer A4 Disable. value. */ + CTIMER_CTRL4_TMRA4EN_EN = 1, /*!< EN : Counter/Timer A4 Enable. value. */ +} CTIMER_CTRL4_TMRA4EN_Enum; + +/* ======================================================= CMPRAUXA4 ======================================================= */ +/* ======================================================= CMPRAUXB4 ======================================================= */ +/* ========================================================= AUX4 ========================================================== */ +/* ============================================ CTIMER AUX4 TMRB4EN23 [30..30] ============================================= */ +typedef enum { /*!< CTIMER_AUX4_TMRB4EN23 */ + CTIMER_AUX4_TMRB4EN23_DIS = 1, /*!< DIS : Disable enhanced functions. value. */ + CTIMER_AUX4_TMRB4EN23_EN = 0, /*!< EN : Enable enhanced functions. value. */ +} CTIMER_AUX4_TMRB4EN23_Enum; + +/* ============================================ CTIMER AUX4 TMRB4POL23 [29..29] ============================================ */ +typedef enum { /*!< CTIMER_AUX4_TMRB4POL23 */ + CTIMER_AUX4_TMRB4POL23_NORM = 0, /*!< NORM : Upper output normal polarity value. */ + CTIMER_AUX4_TMRB4POL23_INV = 1, /*!< INV : Upper output inverted polarity. value. */ +} CTIMER_AUX4_TMRB4POL23_Enum; + +/* ============================================ CTIMER AUX4 TMRB4TINV [28..28] ============================================= */ +typedef enum { /*!< CTIMER_AUX4_TMRB4TINV */ + CTIMER_AUX4_TMRB4TINV_DIS = 0, /*!< DIS : Disable invert on trigger value. */ + CTIMER_AUX4_TMRB4TINV_EN = 1, /*!< EN : Enable invert on trigger value. */ +} CTIMER_AUX4_TMRB4TINV_Enum; + +/* =========================================== CTIMER AUX4 TMRB4NOSYNC [27..27] ============================================ */ +typedef enum { /*!< CTIMER_AUX4_TMRB4NOSYNC */ + CTIMER_AUX4_TMRB4NOSYNC_DIS = 0, /*!< DIS : Synchronization on source clock value. */ + CTIMER_AUX4_TMRB4NOSYNC_NOSYNC = 1, /*!< NOSYNC : No synchronization on source clock value. */ +} CTIMER_AUX4_TMRB4NOSYNC_Enum; + +/* ============================================ CTIMER AUX4 TMRB4TRIG [23..26] ============================================= */ +typedef enum { /*!< CTIMER_AUX4_TMRB4TRIG */ + CTIMER_AUX4_TMRB4TRIG_DIS = 0, /*!< DIS : Trigger source is disabled. value. */ + CTIMER_AUX4_TMRB4TRIG_A4OUT = 1, /*!< A4OUT : Trigger source is CTIMERA4 OUT. value. */ + CTIMER_AUX4_TMRB4TRIG_B3OUT = 2, /*!< B3OUT : Trigger source is CTIMERB3 OUT. value. */ + CTIMER_AUX4_TMRB4TRIG_A3OUT = 3, /*!< A3OUT : Trigger source is CTIMERA3 OUT. value. */ + CTIMER_AUX4_TMRB4TRIG_A7OUT = 4, /*!< A7OUT : Trigger source is CTIMERA7 OUT. value. */ + CTIMER_AUX4_TMRB4TRIG_B7OUT = 5, /*!< B7OUT : Trigger source is CTIMERB7 OUT. value. */ + CTIMER_AUX4_TMRB4TRIG_A1OUT = 6, /*!< A1OUT : Trigger source is CTIMERA1 OUT. value. */ + CTIMER_AUX4_TMRB4TRIG_B1OUT = 7, /*!< B1OUT : Trigger source is CTIMERB1 OUT. value. */ + CTIMER_AUX4_TMRB4TRIG_B3OUT2 = 8, /*!< B3OUT2 : Trigger source is CTIMERB3 OUT2. value. */ + CTIMER_AUX4_TMRB4TRIG_A3OUT2 = 9, /*!< A3OUT2 : Trigger source is CTIMERA3 OUT2. value. */ + CTIMER_AUX4_TMRB4TRIG_A1OUT2 = 10, /*!< A1OUT2 : Trigger source is CTIMERA1 OUT2. value. */ + CTIMER_AUX4_TMRB4TRIG_B1OUT2 = 11, /*!< B1OUT2 : Trigger source is CTIMERB1 OUT2. value. */ + CTIMER_AUX4_TMRB4TRIG_A6OUT2DUAL = 12, /*!< A6OUT2DUAL : Trigger source is CTIMERA6 OUT2, dual edge. value. */ + CTIMER_AUX4_TMRB4TRIG_A7OUT2DUAL = 13, /*!< A7OUT2DUAL : Trigger source is CTIMERA7 OUT2, dual edge. value. */ + CTIMER_AUX4_TMRB4TRIG_B5OUT2DUAL = 14, /*!< B5OUT2DUAL : Trigger source is CTIMERB5 OUT2, dual edge. value. */ + CTIMER_AUX4_TMRB4TRIG_A5OUT2DUAL = 15, /*!< A5OUT2DUAL : Trigger source is CTIMERA5 OUT2, dual edge. value. */ +} CTIMER_AUX4_TMRB4TRIG_Enum; + +/* ============================================ CTIMER AUX4 TMRA4EN23 [14..14] ============================================= */ +typedef enum { /*!< CTIMER_AUX4_TMRA4EN23 */ + CTIMER_AUX4_TMRA4EN23_DIS = 1, /*!< DIS : Disable enhanced functions. value. */ + CTIMER_AUX4_TMRA4EN23_EN = 0, /*!< EN : Enable enhanced functions. value. */ +} CTIMER_AUX4_TMRA4EN23_Enum; + +/* ============================================ CTIMER AUX4 TMRA4POL23 [13..13] ============================================ */ +typedef enum { /*!< CTIMER_AUX4_TMRA4POL23 */ + CTIMER_AUX4_TMRA4POL23_NORM = 0, /*!< NORM : Upper output normal polarity value. */ + CTIMER_AUX4_TMRA4POL23_INV = 1, /*!< INV : Upper output inverted polarity. value. */ +} CTIMER_AUX4_TMRA4POL23_Enum; + +/* ============================================ CTIMER AUX4 TMRA4TINV [12..12] ============================================= */ +typedef enum { /*!< CTIMER_AUX4_TMRA4TINV */ + CTIMER_AUX4_TMRA4TINV_DIS = 0, /*!< DIS : Disable invert on trigger value. */ + CTIMER_AUX4_TMRA4TINV_EN = 1, /*!< EN : Enable invert on trigger value. */ +} CTIMER_AUX4_TMRA4TINV_Enum; + +/* =========================================== CTIMER AUX4 TMRA4NOSYNC [11..11] ============================================ */ +typedef enum { /*!< CTIMER_AUX4_TMRA4NOSYNC */ + CTIMER_AUX4_TMRA4NOSYNC_DIS = 0, /*!< DIS : Synchronization on source clock value. */ + CTIMER_AUX4_TMRA4NOSYNC_NOSYNC = 1, /*!< NOSYNC : No synchronization on source clock value. */ +} CTIMER_AUX4_TMRA4NOSYNC_Enum; + +/* ============================================= CTIMER AUX4 TMRA4TRIG [7..10] ============================================= */ +typedef enum { /*!< CTIMER_AUX4_TMRA4TRIG */ + CTIMER_AUX4_TMRA4TRIG_DIS = 0, /*!< DIS : Trigger source is disabled. value. */ + CTIMER_AUX4_TMRA4TRIG_B4OUT = 1, /*!< B4OUT : Trigger source is CTIMERB4 OUT. value. */ + CTIMER_AUX4_TMRA4TRIG_B3OUT = 2, /*!< B3OUT : Trigger source is CTIMERB3 OUT. value. */ + CTIMER_AUX4_TMRA4TRIG_A3OUT = 3, /*!< A3OUT : Trigger source is CTIMERA3 OUT. value. */ + CTIMER_AUX4_TMRA4TRIG_A6OUT = 4, /*!< A6OUT : Trigger source is CTIMERA6 OUT. value. */ + CTIMER_AUX4_TMRA4TRIG_B6OUT = 5, /*!< B6OUT : Trigger source is CTIMERB6 OUT. value. */ + CTIMER_AUX4_TMRA4TRIG_A2OUT = 6, /*!< A2OUT : Trigger source is CTIMERA2 OUT. value. */ + CTIMER_AUX4_TMRA4TRIG_B2OUT = 7, /*!< B2OUT : Trigger source is CTIMERB2 OUT. value. */ + CTIMER_AUX4_TMRA4TRIG_B3OUT2 = 8, /*!< B3OUT2 : Trigger source is CTIMERB3 OUT2. value. */ + CTIMER_AUX4_TMRA4TRIG_A3OUT2 = 9, /*!< A3OUT2 : Trigger source is CTIMERA3 OUT2. value. */ + CTIMER_AUX4_TMRA4TRIG_A1OUT2 = 10, /*!< A1OUT2 : Trigger source is CTIMERA1 OUT2. value. */ + CTIMER_AUX4_TMRA4TRIG_B1OUT2 = 11, /*!< B1OUT2 : Trigger source is CTIMERB1 OUT2. value. */ + CTIMER_AUX4_TMRA4TRIG_A6OUT2DUAL = 12, /*!< A6OUT2DUAL : Trigger source is CTIMERA6 OUT2, dual edge. value. */ + CTIMER_AUX4_TMRA4TRIG_A7OUT2DUAL = 13, /*!< A7OUT2DUAL : Trigger source is CTIMERA7 OUT2, dual edge. value. */ + CTIMER_AUX4_TMRA4TRIG_B5OUT2DUAL = 14, /*!< B5OUT2DUAL : Trigger source is CTIMERB5 OUT2, dual edge. value. */ + CTIMER_AUX4_TMRA4TRIG_A5OUT2DUAL = 15, /*!< A5OUT2DUAL : Trigger source is CTIMERA5 OUT2, dual edge. value. */ +} CTIMER_AUX4_TMRA4TRIG_Enum; + +/* ========================================================= TMR5 ========================================================== */ +/* ======================================================== CMPRA5 ========================================================= */ +/* ======================================================== CMPRB5 ========================================================= */ +/* ========================================================= CTRL5 ========================================================= */ +/* ============================================= CTIMER CTRL5 CTLINK5 [31..31] ============================================= */ +typedef enum { /*!< CTIMER_CTRL5_CTLINK5 */ + CTIMER_CTRL5_CTLINK5_TWO_16BIT_TIMERS = 0, /*!< TWO_16BIT_TIMERS : Use A5/B5 timers as two independent 16-bit + timers (default). value. */ + CTIMER_CTRL5_CTLINK5_32BIT_TIMER = 1, /*!< 32BIT_TIMER : Link A5/B5 timers into a single 32-bit timer. + value. */ +} CTIMER_CTRL5_CTLINK5_Enum; + +/* ============================================ CTIMER CTRL5 TMRB5POL [28..28] ============================================= */ +typedef enum { /*!< CTIMER_CTRL5_TMRB5POL */ + CTIMER_CTRL5_TMRB5POL_NORMAL = 0, /*!< NORMAL : The polarity of the TMRPINB5 pin is the same as the + timer output. value. */ + CTIMER_CTRL5_TMRB5POL_INVERTED = 1, /*!< INVERTED : The polarity of the TMRPINB5 pin is the inverse of + the timer output. value. */ +} CTIMER_CTRL5_TMRB5POL_Enum; + +/* ============================================ CTIMER CTRL5 TMRB5CLR [27..27] ============================================= */ +typedef enum { /*!< CTIMER_CTRL5_TMRB5CLR */ + CTIMER_CTRL5_TMRB5CLR_RUN = 0, /*!< RUN : Allow counter/timer B5 to run value. */ + CTIMER_CTRL5_TMRB5CLR_CLEAR = 1, /*!< CLEAR : Holds counter/timer B5 at 0x0000. value. */ +} CTIMER_CTRL5_TMRB5CLR_Enum; + +/* ============================================ CTIMER CTRL5 TMRB5IE1 [26..26] ============================================= */ +typedef enum { /*!< CTIMER_CTRL5_TMRB5IE1 */ + CTIMER_CTRL5_TMRB5IE1_DIS = 0, /*!< DIS : Disable counter/timer B5 from generating an interrupt + based on COMPR1. value. */ + CTIMER_CTRL5_TMRB5IE1_EN = 1, /*!< EN : Enable counter/timer B5 to generate an interrupt based + on COMPR1. value. */ +} CTIMER_CTRL5_TMRB5IE1_Enum; + +/* ============================================ CTIMER CTRL5 TMRB5IE0 [25..25] ============================================= */ +typedef enum { /*!< CTIMER_CTRL5_TMRB5IE0 */ + CTIMER_CTRL5_TMRB5IE0_DIS = 0, /*!< DIS : Disable counter/timer B5 from generating an interrupt + based on COMPR0. value. */ + CTIMER_CTRL5_TMRB5IE0_EN = 1, /*!< EN : Enable counter/timer B5 to generate an interrupt based + on COMPR0 value. */ +} CTIMER_CTRL5_TMRB5IE0_Enum; + +/* ============================================= CTIMER CTRL5 TMRB5FN [22..24] ============================================= */ +typedef enum { /*!< CTIMER_CTRL5_TMRB5FN */ + CTIMER_CTRL5_TMRB5FN_SINGLECOUNT = 0, /*!< SINGLECOUNT : Single count (output toggles and sticks). Count + to CMPR0B5, stop. value. */ + CTIMER_CTRL5_TMRB5FN_REPEATEDCOUNT = 1, /*!< REPEATEDCOUNT : Repeated count (periodic 1-clock-cycle-wide + pulses). Count to CMPR0B5, restart. value. */ + CTIMER_CTRL5_TMRB5FN_PULSE_ONCE = 2, /*!< PULSE_ONCE : Pulse once (aka one-shot). Count to CMPR0B5, assert, + count to CMPR1B5, deassert, stop. value. */ + CTIMER_CTRL5_TMRB5FN_PULSE_CONT = 3, /*!< PULSE_CONT : Pulse continously. Count to CMPR0B5, assert, count + to CMPR1B5, deassert, restart. value. */ + CTIMER_CTRL5_TMRB5FN_SINGLEPATTERN = 4, /*!< SINGLEPATTERN : Single pattern. value. */ + CTIMER_CTRL5_TMRB5FN_REPEATPATTERN = 5, /*!< REPEATPATTERN : Repeated pattern. value. */ + CTIMER_CTRL5_TMRB5FN_CONTINUOUS = 6, /*!< CONTINUOUS : Continuous run (aka Free Run). Count continuously. + value. */ + CTIMER_CTRL5_TMRB5FN_ALTPWN = 7, /*!< ALTPWN : Alternate PWM value. */ +} CTIMER_CTRL5_TMRB5FN_Enum; + +/* ============================================ CTIMER CTRL5 TMRB5CLK [17..21] ============================================= */ +typedef enum { /*!< CTIMER_CTRL5_TMRB5CLK */ + CTIMER_CTRL5_TMRB5CLK_TMRPIN = 0, /*!< TMRPIN : Clock source is TMRPINB. value. */ + CTIMER_CTRL5_TMRB5CLK_HFRC_DIV4 = 1, /*!< HFRC_DIV4 : Clock source is the HFRC / 4 value. */ + CTIMER_CTRL5_TMRB5CLK_HFRC_DIV16 = 2, /*!< HFRC_DIV16 : Clock source is HFRC / 16 value. */ + CTIMER_CTRL5_TMRB5CLK_HFRC_DIV256 = 3, /*!< HFRC_DIV256 : Clock source is HFRC / 256 value. */ + CTIMER_CTRL5_TMRB5CLK_HFRC_DIV1024 = 4, /*!< HFRC_DIV1024 : Clock source is HFRC / 1024 value. */ + CTIMER_CTRL5_TMRB5CLK_HFRC_DIV4K = 5, /*!< HFRC_DIV4K : Clock source is HFRC / 4096 value. */ + CTIMER_CTRL5_TMRB5CLK_XT = 6, /*!< XT : Clock source is the XT (uncalibrated). value. */ + CTIMER_CTRL5_TMRB5CLK_XT_DIV2 = 7, /*!< XT_DIV2 : Clock source is XT / 2 value. */ + CTIMER_CTRL5_TMRB5CLK_XT_DIV16 = 8, /*!< XT_DIV16 : Clock source is XT / 16 value. */ + CTIMER_CTRL5_TMRB5CLK_XT_DIV128 = 9, /*!< XT_DIV128 : Clock source is XT / 128 value. */ + CTIMER_CTRL5_TMRB5CLK_LFRC_DIV2 = 10, /*!< LFRC_DIV2 : Clock source is LFRC / 2 value. */ + CTIMER_CTRL5_TMRB5CLK_LFRC_DIV32 = 11, /*!< LFRC_DIV32 : Clock source is LFRC / 32 value. */ + CTIMER_CTRL5_TMRB5CLK_LFRC_DIV1K = 12, /*!< LFRC_DIV1K : Clock source is LFRC / 1024 value. */ + CTIMER_CTRL5_TMRB5CLK_LFRC = 13, /*!< LFRC : Clock source is LFRC value. */ + CTIMER_CTRL5_TMRB5CLK_RTC_100HZ = 14, /*!< RTC_100HZ : Clock source is 100 Hz from the current RTC oscillator. + value. */ + CTIMER_CTRL5_TMRB5CLK_HCLK = 15, /*!< HCLK : Clock source is HCLK. value. */ + CTIMER_CTRL5_TMRB5CLK_XT_DIV4 = 16, /*!< XT_DIV4 : Clock source is XT / 4 value. */ + CTIMER_CTRL5_TMRB5CLK_XT_DIV8 = 17, /*!< XT_DIV8 : Clock source is XT / 8 value. */ + CTIMER_CTRL5_TMRB5CLK_XT_DIV32 = 18, /*!< XT_DIV32 : Clock source is XT / 32 value. */ + CTIMER_CTRL5_TMRB5CLK_CTMRA5 = 20, /*!< CTMRA5 : Clock source is CTIMERA5 OUT. value. */ + CTIMER_CTRL5_TMRB5CLK_CTMRA0 = 21, /*!< CTMRA0 : Clock source is CTIMERA0 OUT. value. */ + CTIMER_CTRL5_TMRB5CLK_CTMRB0 = 22, /*!< CTMRB0 : Clock source is CTIMERB0 OUT. value. */ + CTIMER_CTRL5_TMRB5CLK_CTMRA6 = 23, /*!< CTMRA6 : Clock source is CTIMERA6 OUT. value. */ + CTIMER_CTRL5_TMRB5CLK_CTMRB6 = 24, /*!< CTMRB6 : Clock source is CTIMERB6 OUT. value. */ + CTIMER_CTRL5_TMRB5CLK_CTMRB1 = 25, /*!< CTMRB1 : Clock source is CTIMERB1 OUT. value. */ + CTIMER_CTRL5_TMRB5CLK_CTMRB2 = 26, /*!< CTMRB2 : Clock source is CTIMERB2 OUT. value. */ + CTIMER_CTRL5_TMRB5CLK_CTMRB3 = 27, /*!< CTMRB3 : Clock source is CTIMERB3 OUT. value. */ + CTIMER_CTRL5_TMRB5CLK_CTMRB4 = 28, /*!< CTMRB4 : Clock source is CTIMERB4 OUT. value. */ + CTIMER_CTRL5_TMRB5CLK_BUCKBLE = 29, /*!< BUCKBLE : Clock source is BLE buck converter TON pulses. value. */ + CTIMER_CTRL5_TMRB5CLK_BUCKB = 30, /*!< BUCKB : Clock source is Memory buck converter TON pulses. value. */ + CTIMER_CTRL5_TMRB5CLK_BUCKA = 31, /*!< BUCKA : Clock source is CPU buck converter TON pulses. value. */ +} CTIMER_CTRL5_TMRB5CLK_Enum; + +/* ============================================= CTIMER CTRL5 TMRB5EN [16..16] ============================================= */ +typedef enum { /*!< CTIMER_CTRL5_TMRB5EN */ + CTIMER_CTRL5_TMRB5EN_DIS = 0, /*!< DIS : Counter/Timer B5 Disable. value. */ + CTIMER_CTRL5_TMRB5EN_EN = 1, /*!< EN : Counter/Timer B5 Enable. value. */ +} CTIMER_CTRL5_TMRB5EN_Enum; + +/* ============================================ CTIMER CTRL5 TMRA5POL [12..12] ============================================= */ +typedef enum { /*!< CTIMER_CTRL5_TMRA5POL */ + CTIMER_CTRL5_TMRA5POL_NORMAL = 0, /*!< NORMAL : The polarity of the TMRPINA5 pin is the same as the + timer output. value. */ + CTIMER_CTRL5_TMRA5POL_INVERTED = 1, /*!< INVERTED : The polarity of the TMRPINA5 pin is the inverse of + the timer output. value. */ +} CTIMER_CTRL5_TMRA5POL_Enum; + +/* ============================================ CTIMER CTRL5 TMRA5CLR [11..11] ============================================= */ +typedef enum { /*!< CTIMER_CTRL5_TMRA5CLR */ + CTIMER_CTRL5_TMRA5CLR_RUN = 0, /*!< RUN : Allow counter/timer A5 to run value. */ + CTIMER_CTRL5_TMRA5CLR_CLEAR = 1, /*!< CLEAR : Holds counter/timer A5 at 0x0000. value. */ +} CTIMER_CTRL5_TMRA5CLR_Enum; + +/* ============================================ CTIMER CTRL5 TMRA5IE1 [10..10] ============================================= */ +typedef enum { /*!< CTIMER_CTRL5_TMRA5IE1 */ + CTIMER_CTRL5_TMRA5IE1_DIS = 0, /*!< DIS : Disable counter/timer A5 from generating an interrupt + based on COMPR1. value. */ + CTIMER_CTRL5_TMRA5IE1_EN = 1, /*!< EN : Enable counter/timer A5 to generate an interrupt based + on COMPR1. value. */ +} CTIMER_CTRL5_TMRA5IE1_Enum; + +/* ============================================= CTIMER CTRL5 TMRA5IE0 [9..9] ============================================== */ +typedef enum { /*!< CTIMER_CTRL5_TMRA5IE0 */ + CTIMER_CTRL5_TMRA5IE0_DIS = 0, /*!< DIS : Disable counter/timer A5 from generating an interrupt + based on COMPR0. value. */ + CTIMER_CTRL5_TMRA5IE0_EN = 1, /*!< EN : Enable counter/timer A5 to generate an interrupt based + on COMPR0. value. */ +} CTIMER_CTRL5_TMRA5IE0_Enum; + +/* ============================================== CTIMER CTRL5 TMRA5FN [6..8] ============================================== */ +typedef enum { /*!< CTIMER_CTRL5_TMRA5FN */ + CTIMER_CTRL5_TMRA5FN_SINGLECOUNT = 0, /*!< SINGLECOUNT : Single count (output toggles and sticks). Count + to CMPR0A5, stop. value. */ + CTIMER_CTRL5_TMRA5FN_REPEATEDCOUNT = 1, /*!< REPEATEDCOUNT : Repeated count (periodic 1-clock-cycle-wide + pulses). Count to CMPR0A5, restart. value. */ + CTIMER_CTRL5_TMRA5FN_PULSE_ONCE = 2, /*!< PULSE_ONCE : Pulse once (aka one-shot). Count to CMPR0A5, assert, + count to CMPR1A5, deassert, stop. value. */ + CTIMER_CTRL5_TMRA5FN_PULSE_CONT = 3, /*!< PULSE_CONT : Pulse continously. Count to CMPR0A5, assert, count + to CMPR1A5, deassert, restart. value. */ + CTIMER_CTRL5_TMRA5FN_SINGLEPATTERN = 4, /*!< SINGLEPATTERN : Single pattern. value. */ + CTIMER_CTRL5_TMRA5FN_REPEATPATTERN = 5, /*!< REPEATPATTERN : Repeated pattern. value. */ + CTIMER_CTRL5_TMRA5FN_CONTINUOUS = 6, /*!< CONTINUOUS : Continuous run (aka Free Run). Count continuously. + value. */ + CTIMER_CTRL5_TMRA5FN_ALTPWN = 7, /*!< ALTPWN : Alternate PWM value. */ +} CTIMER_CTRL5_TMRA5FN_Enum; + +/* ============================================= CTIMER CTRL5 TMRA5CLK [1..5] ============================================== */ +typedef enum { /*!< CTIMER_CTRL5_TMRA5CLK */ + CTIMER_CTRL5_TMRA5CLK_TMRPIN = 0, /*!< TMRPIN : Clock source is TMRPINA. value. */ + CTIMER_CTRL5_TMRA5CLK_HFRC_DIV4 = 1, /*!< HFRC_DIV4 : Clock source is the HFRC / 4 value. */ + CTIMER_CTRL5_TMRA5CLK_HFRC_DIV16 = 2, /*!< HFRC_DIV16 : Clock source is HFRC / 16 value. */ + CTIMER_CTRL5_TMRA5CLK_HFRC_DIV256 = 3, /*!< HFRC_DIV256 : Clock source is HFRC / 256 value. */ + CTIMER_CTRL5_TMRA5CLK_HFRC_DIV1024 = 4, /*!< HFRC_DIV1024 : Clock source is HFRC / 1024 value. */ + CTIMER_CTRL5_TMRA5CLK_HFRC_DIV4K = 5, /*!< HFRC_DIV4K : Clock source is HFRC / 4096 value. */ + CTIMER_CTRL5_TMRA5CLK_XT = 6, /*!< XT : Clock source is the XT (uncalibrated). value. */ + CTIMER_CTRL5_TMRA5CLK_XT_DIV2 = 7, /*!< XT_DIV2 : Clock source is XT / 2 value. */ + CTIMER_CTRL5_TMRA5CLK_XT_DIV16 = 8, /*!< XT_DIV16 : Clock source is XT / 16 value. */ + CTIMER_CTRL5_TMRA5CLK_XT_DIV128 = 9, /*!< XT_DIV128 : Clock source is XT / 128 value. */ + CTIMER_CTRL5_TMRA5CLK_LFRC_DIV2 = 10, /*!< LFRC_DIV2 : Clock source is LFRC / 2 value. */ + CTIMER_CTRL5_TMRA5CLK_LFRC_DIV32 = 11, /*!< LFRC_DIV32 : Clock source is LFRC / 32 value. */ + CTIMER_CTRL5_TMRA5CLK_LFRC_DIV1K = 12, /*!< LFRC_DIV1K : Clock source is LFRC / 1024 value. */ + CTIMER_CTRL5_TMRA5CLK_LFRC = 13, /*!< LFRC : Clock source is LFRC value. */ + CTIMER_CTRL5_TMRA5CLK_RTC_100HZ = 14, /*!< RTC_100HZ : Clock source is 100 Hz from the current RTC oscillator. + value. */ + CTIMER_CTRL5_TMRA5CLK_HCLK = 15, /*!< HCLK : Clock source is HCLK. value. */ + CTIMER_CTRL5_TMRA5CLK_XT_DIV4 = 16, /*!< XT_DIV4 : Clock source is XT / 4 value. */ + CTIMER_CTRL5_TMRA5CLK_XT_DIV8 = 17, /*!< XT_DIV8 : Clock source is XT / 8 value. */ + CTIMER_CTRL5_TMRA5CLK_XT_DIV32 = 18, /*!< XT_DIV32 : Clock source is XT / 32 value. */ + CTIMER_CTRL5_TMRA5CLK_CTMRB5 = 20, /*!< CTMRB5 : Clock source is CTIMERB5 OUT. value. */ + CTIMER_CTRL5_TMRA5CLK_CTMRA0 = 21, /*!< CTMRA0 : Clock source is CTIMERA0 OUT. value. */ + CTIMER_CTRL5_TMRA5CLK_CTMRB0 = 22, /*!< CTMRB0 : Clock source is CTIMERB0 OUT. value. */ + CTIMER_CTRL5_TMRA5CLK_CTMRA6 = 23, /*!< CTMRA6 : Clock source is CTIMERA6 OUT. value. */ + CTIMER_CTRL5_TMRA5CLK_CTMRB6 = 24, /*!< CTMRB6 : Clock source is CTIMERB6 OUT. value. */ + CTIMER_CTRL5_TMRA5CLK_CTMRB1 = 25, /*!< CTMRB1 : Clock source is CTIMERB1 OUT. value. */ + CTIMER_CTRL5_TMRA5CLK_CTMRB2 = 26, /*!< CTMRB2 : Clock source is CTIMERB2 OUT. value. */ + CTIMER_CTRL5_TMRA5CLK_CTMRB3 = 27, /*!< CTMRB3 : Clock source is CTIMERB3 OUT. value. */ + CTIMER_CTRL5_TMRA5CLK_CTMRB4 = 28, /*!< CTMRB4 : Clock source is CTIMERB4 OUT. value. */ + CTIMER_CTRL5_TMRA5CLK_BUCKBLE = 29, /*!< BUCKBLE : Clock source is BLE buck converter TON pulses. value. */ + CTIMER_CTRL5_TMRA5CLK_BUCKB = 30, /*!< BUCKB : Clock source is Memory buck converter TON pulses. value. */ + CTIMER_CTRL5_TMRA5CLK_BUCKA = 31, /*!< BUCKA : Clock source is CPU buck converter TON pulses. value. */ +} CTIMER_CTRL5_TMRA5CLK_Enum; + +/* ============================================== CTIMER CTRL5 TMRA5EN [0..0] ============================================== */ +typedef enum { /*!< CTIMER_CTRL5_TMRA5EN */ + CTIMER_CTRL5_TMRA5EN_DIS = 0, /*!< DIS : Counter/Timer A5 Disable. value. */ + CTIMER_CTRL5_TMRA5EN_EN = 1, /*!< EN : Counter/Timer A5 Enable. value. */ +} CTIMER_CTRL5_TMRA5EN_Enum; + +/* ======================================================= CMPRAUXA5 ======================================================= */ +/* ======================================================= CMPRAUXB5 ======================================================= */ +/* ========================================================= AUX5 ========================================================== */ +/* ============================================ CTIMER AUX5 TMRB5EN23 [30..30] ============================================= */ +typedef enum { /*!< CTIMER_AUX5_TMRB5EN23 */ + CTIMER_AUX5_TMRB5EN23_DIS = 1, /*!< DIS : Disable enhanced functions. value. */ + CTIMER_AUX5_TMRB5EN23_EN = 0, /*!< EN : Enable enhanced functions. value. */ +} CTIMER_AUX5_TMRB5EN23_Enum; + +/* ============================================ CTIMER AUX5 TMRB5POL23 [29..29] ============================================ */ +typedef enum { /*!< CTIMER_AUX5_TMRB5POL23 */ + CTIMER_AUX5_TMRB5POL23_NORM = 0, /*!< NORM : Upper output normal polarity value. */ + CTIMER_AUX5_TMRB5POL23_INV = 1, /*!< INV : Upper output inverted polarity. value. */ +} CTIMER_AUX5_TMRB5POL23_Enum; + +/* ============================================ CTIMER AUX5 TMRB5TINV [28..28] ============================================= */ +typedef enum { /*!< CTIMER_AUX5_TMRB5TINV */ + CTIMER_AUX5_TMRB5TINV_DIS = 0, /*!< DIS : Disable invert on trigger value. */ + CTIMER_AUX5_TMRB5TINV_EN = 1, /*!< EN : Enable invert on trigger value. */ +} CTIMER_AUX5_TMRB5TINV_Enum; + +/* =========================================== CTIMER AUX5 TMRB5NOSYNC [27..27] ============================================ */ +typedef enum { /*!< CTIMER_AUX5_TMRB5NOSYNC */ + CTIMER_AUX5_TMRB5NOSYNC_DIS = 0, /*!< DIS : Synchronization on source clock value. */ + CTIMER_AUX5_TMRB5NOSYNC_NOSYNC = 1, /*!< NOSYNC : No synchronization on source clock value. */ +} CTIMER_AUX5_TMRB5NOSYNC_Enum; + +/* ============================================ CTIMER AUX5 TMRB5TRIG [23..26] ============================================= */ +typedef enum { /*!< CTIMER_AUX5_TMRB5TRIG */ + CTIMER_AUX5_TMRB5TRIG_DIS = 0, /*!< DIS : Trigger source is disabled. value. */ + CTIMER_AUX5_TMRB5TRIG_A5OUT = 1, /*!< A5OUT : Trigger source is CTIMERA5 OUT. value. */ + CTIMER_AUX5_TMRB5TRIG_B3OUT = 2, /*!< B3OUT : Trigger source is CTIMERB3 OUT. value. */ + CTIMER_AUX5_TMRB5TRIG_A3OUT = 3, /*!< A3OUT : Trigger source is CTIMERA3 OUT. value. */ + CTIMER_AUX5_TMRB5TRIG_A6OUT = 4, /*!< A6OUT : Trigger source is CTIMERA6 OUT. value. */ + CTIMER_AUX5_TMRB5TRIG_B6OUT = 5, /*!< B6OUT : Trigger source is CTIMERB6 OUT. value. */ + CTIMER_AUX5_TMRB5TRIG_A1OUT = 6, /*!< A1OUT : Trigger source is CTIMERA1 OUT. value. */ + CTIMER_AUX5_TMRB5TRIG_B1OUT = 7, /*!< B1OUT : Trigger source is CTIMERB1 OUT. value. */ + CTIMER_AUX5_TMRB5TRIG_B3OUT2 = 8, /*!< B3OUT2 : Trigger source is CTIMERB3 OUT2. value. */ + CTIMER_AUX5_TMRB5TRIG_A3OUT2 = 9, /*!< A3OUT2 : Trigger source is CTIMERA3 OUT2. value. */ + CTIMER_AUX5_TMRB5TRIG_A0OUT2 = 10, /*!< A0OUT2 : Trigger source is CTIMERA0 OUT2. value. */ + CTIMER_AUX5_TMRB5TRIG_B0OUT2 = 11, /*!< B0OUT2 : Trigger source is CTIMERB0 OUT2. value. */ + CTIMER_AUX5_TMRB5TRIG_A6OUT2DUAL = 12, /*!< A6OUT2DUAL : Trigger source is CTIMERA6 OUT2, dual edge. value. */ + CTIMER_AUX5_TMRB5TRIG_A7OUT2DUAL = 13, /*!< A7OUT2DUAL : Trigger source is CTIMERA7 OUT2, dual edge. value. */ + CTIMER_AUX5_TMRB5TRIG_B4OUT2DUAL = 14, /*!< B4OUT2DUAL : Trigger source is CTIMERB4 OUT2, dual edge. value. */ + CTIMER_AUX5_TMRB5TRIG_A4OUT2DUAL = 15, /*!< A4OUT2DUAL : Trigger source is CTIMERA4 OUT2, dual edge. value. */ +} CTIMER_AUX5_TMRB5TRIG_Enum; + +/* ============================================ CTIMER AUX5 TMRA5EN23 [14..14] ============================================= */ +typedef enum { /*!< CTIMER_AUX5_TMRA5EN23 */ + CTIMER_AUX5_TMRA5EN23_DIS = 1, /*!< DIS : Disable enhanced functions. value. */ + CTIMER_AUX5_TMRA5EN23_EN = 0, /*!< EN : Enable enhanced functions. value. */ +} CTIMER_AUX5_TMRA5EN23_Enum; + +/* ============================================ CTIMER AUX5 TMRA5POL23 [13..13] ============================================ */ +typedef enum { /*!< CTIMER_AUX5_TMRA5POL23 */ + CTIMER_AUX5_TMRA5POL23_NORMAL = 0, /*!< NORMAL : Upper output normal polarity value. */ + CTIMER_AUX5_TMRA5POL23_INV = 1, /*!< INV : Upper output inverted polarity. value. */ +} CTIMER_AUX5_TMRA5POL23_Enum; + +/* ============================================ CTIMER AUX5 TMRA5TINV [12..12] ============================================= */ +typedef enum { /*!< CTIMER_AUX5_TMRA5TINV */ + CTIMER_AUX5_TMRA5TINV_DIS = 0, /*!< DIS : Disable invert on trigger value. */ + CTIMER_AUX5_TMRA5TINV_EN = 1, /*!< EN : Enable invert on trigger value. */ +} CTIMER_AUX5_TMRA5TINV_Enum; + +/* =========================================== CTIMER AUX5 TMRA5NOSYNC [11..11] ============================================ */ +typedef enum { /*!< CTIMER_AUX5_TMRA5NOSYNC */ + CTIMER_AUX5_TMRA5NOSYNC_DIS = 0, /*!< DIS : Synchronization on source clock value. */ + CTIMER_AUX5_TMRA5NOSYNC_NOSYNC = 1, /*!< NOSYNC : No synchronization on source clock value. */ +} CTIMER_AUX5_TMRA5NOSYNC_Enum; + +/* ============================================= CTIMER AUX5 TMRA5TRIG [7..10] ============================================= */ +typedef enum { /*!< CTIMER_AUX5_TMRA5TRIG */ + CTIMER_AUX5_TMRA5TRIG_DIS = 0, /*!< DIS : Trigger source is disabled. value. */ + CTIMER_AUX5_TMRA5TRIG_B5OUT = 1, /*!< B5OUT : Trigger source is CTIMERB5 OUT. value. */ + CTIMER_AUX5_TMRA5TRIG_B3OUT = 2, /*!< B3OUT : Trigger source is CTIMERB3 OUT. value. */ + CTIMER_AUX5_TMRA5TRIG_A3OUT = 3, /*!< A3OUT : Trigger source is CTIMERA3 OUT. value. */ + CTIMER_AUX5_TMRA5TRIG_A4OUT = 4, /*!< A4OUT : Trigger source is CTIMERA4 OUT. value. */ + CTIMER_AUX5_TMRA5TRIG_B4OUT = 5, /*!< B4OUT : Trigger source is CTIMERB4 OUT. value. */ + CTIMER_AUX5_TMRA5TRIG_A2OUT = 6, /*!< A2OUT : Trigger source is CTIMERA2 OUT. value. */ + CTIMER_AUX5_TMRA5TRIG_B2OUT = 7, /*!< B2OUT : Trigger source is CTIMERB2 OUT. value. */ + CTIMER_AUX5_TMRA5TRIG_B3OUT2 = 8, /*!< B3OUT2 : Trigger source is CTIMERB3 OUT2. value. */ + CTIMER_AUX5_TMRA5TRIG_A3OUT2 = 9, /*!< A3OUT2 : Trigger source is CTIMERA3 OUT2. value. */ + CTIMER_AUX5_TMRA5TRIG_A0OUT2 = 10, /*!< A0OUT2 : Trigger source is CTIMERA0 OUT2. value. */ + CTIMER_AUX5_TMRA5TRIG_B0OUT2 = 11, /*!< B0OUT2 : Trigger source is CTIMERB0 OUT2. value. */ + CTIMER_AUX5_TMRA5TRIG_A6OUT2DUAL = 12, /*!< A6OUT2DUAL : Trigger source is CTIMERA6 OUT2, dual edge. value. */ + CTIMER_AUX5_TMRA5TRIG_A7OUT2DUAL = 13, /*!< A7OUT2DUAL : Trigger source is CTIMERA7 OUT2, dual edge. value. */ + CTIMER_AUX5_TMRA5TRIG_B4OUT2DUAL = 14, /*!< B4OUT2DUAL : Trigger source is CTIMERB4 OUT2, dual edge. value. */ + CTIMER_AUX5_TMRA5TRIG_A4OUT2DUAL = 15, /*!< A4OUT2DUAL : Trigger source is CTIMERA4 OUT2, dual edge. value. */ +} CTIMER_AUX5_TMRA5TRIG_Enum; + +/* ========================================================= TMR6 ========================================================== */ +/* ======================================================== CMPRA6 ========================================================= */ +/* ======================================================== CMPRB6 ========================================================= */ +/* ========================================================= CTRL6 ========================================================= */ +/* ============================================= CTIMER CTRL6 CTLINK6 [31..31] ============================================= */ +typedef enum { /*!< CTIMER_CTRL6_CTLINK6 */ + CTIMER_CTRL6_CTLINK6_TWO_16BIT_TIMERS = 0, /*!< TWO_16BIT_TIMERS : Use A6/B6 timers as two independent 16-bit + timers (default). value. */ + CTIMER_CTRL6_CTLINK6_32BIT_TIMER = 1, /*!< 32BIT_TIMER : Link A6/B6 timers into a single 32-bit timer. + value. */ +} CTIMER_CTRL6_CTLINK6_Enum; + +/* ============================================ CTIMER CTRL6 TMRB6POL [28..28] ============================================= */ +typedef enum { /*!< CTIMER_CTRL6_TMRB6POL */ + CTIMER_CTRL6_TMRB6POL_NORMAL = 0, /*!< NORMAL : The polarity of the TMRPINB6 pin is the same as the + timer output. value. */ + CTIMER_CTRL6_TMRB6POL_INVERTED = 1, /*!< INVERTED : The polarity of the TMRPINB6 pin is the inverse of + the timer output. value. */ +} CTIMER_CTRL6_TMRB6POL_Enum; + +/* ============================================ CTIMER CTRL6 TMRB6CLR [27..27] ============================================= */ +typedef enum { /*!< CTIMER_CTRL6_TMRB6CLR */ + CTIMER_CTRL6_TMRB6CLR_RUN = 0, /*!< RUN : Allow counter/timer B6 to run value. */ + CTIMER_CTRL6_TMRB6CLR_CLEAR = 1, /*!< CLEAR : Holds counter/timer B6 at 0x0000. value. */ +} CTIMER_CTRL6_TMRB6CLR_Enum; + +/* ============================================ CTIMER CTRL6 TMRB6IE1 [26..26] ============================================= */ +typedef enum { /*!< CTIMER_CTRL6_TMRB6IE1 */ + CTIMER_CTRL6_TMRB6IE1_DIS = 0, /*!< DIS : Disable counter/timer B6 from generating an interrupt + based on COMPR1. value. */ + CTIMER_CTRL6_TMRB6IE1_EN = 1, /*!< EN : Enable counter/timer B6 to generate an interrupt based + on COMPR1. value. */ +} CTIMER_CTRL6_TMRB6IE1_Enum; + +/* ============================================ CTIMER CTRL6 TMRB6IE0 [25..25] ============================================= */ +typedef enum { /*!< CTIMER_CTRL6_TMRB6IE0 */ + CTIMER_CTRL6_TMRB6IE0_DIS = 0, /*!< DIS : Disable counter/timer B6 from generating an interrupt + based on COMPR0. value. */ + CTIMER_CTRL6_TMRB6IE0_EN = 1, /*!< EN : Enable counter/timer B6 to generate an interrupt based + on COMPR0 value. */ +} CTIMER_CTRL6_TMRB6IE0_Enum; + +/* ============================================= CTIMER CTRL6 TMRB6FN [22..24] ============================================= */ +typedef enum { /*!< CTIMER_CTRL6_TMRB6FN */ + CTIMER_CTRL6_TMRB6FN_SINGLECOUNT = 0, /*!< SINGLECOUNT : Single count (output toggles and sticks). Count + to CMPR0B6, stop. value. */ + CTIMER_CTRL6_TMRB6FN_REPEATEDCOUNT = 1, /*!< REPEATEDCOUNT : Repeated count (periodic 1-clock-cycle-wide + pulses). Count to CMPR0B6, restart. value. */ + CTIMER_CTRL6_TMRB6FN_PULSE_ONCE = 2, /*!< PULSE_ONCE : Pulse once (aka one-shot). Count to CMPR0B6, assert, + count to CMPR1B6, deassert, stop. value. */ + CTIMER_CTRL6_TMRB6FN_PULSE_CONT = 3, /*!< PULSE_CONT : Pulse continously. Count to CMPR0B6, assert, count + to CMPR1B6, deassert, restart. value. */ + CTIMER_CTRL6_TMRB6FN_SINGLEPATTERN = 4, /*!< SINGLEPATTERN : Single pattern. value. */ + CTIMER_CTRL6_TMRB6FN_REPEATPATTERN = 5, /*!< REPEATPATTERN : Repeated pattern. value. */ + CTIMER_CTRL6_TMRB6FN_CONTINUOUS = 6, /*!< CONTINUOUS : Continuous run (aka Free Run). Count continuously. + value. */ + CTIMER_CTRL6_TMRB6FN_ALTPWN = 7, /*!< ALTPWN : Alternate PWM value. */ +} CTIMER_CTRL6_TMRB6FN_Enum; + +/* ============================================ CTIMER CTRL6 TMRB6CLK [17..21] ============================================= */ +typedef enum { /*!< CTIMER_CTRL6_TMRB6CLK */ + CTIMER_CTRL6_TMRB6CLK_TMRPIN = 0, /*!< TMRPIN : Clock source is TMRPINB. value. */ + CTIMER_CTRL6_TMRB6CLK_HFRC_DIV4 = 1, /*!< HFRC_DIV4 : Clock source is the HFRC / 4 value. */ + CTIMER_CTRL6_TMRB6CLK_HFRC_DIV16 = 2, /*!< HFRC_DIV16 : Clock source is HFRC / 16 value. */ + CTIMER_CTRL6_TMRB6CLK_HFRC_DIV256 = 3, /*!< HFRC_DIV256 : Clock source is HFRC / 256 value. */ + CTIMER_CTRL6_TMRB6CLK_HFRC_DIV1024 = 4, /*!< HFRC_DIV1024 : Clock source is HFRC / 1024 value. */ + CTIMER_CTRL6_TMRB6CLK_HFRC_DIV4K = 5, /*!< HFRC_DIV4K : Clock source is HFRC / 4096 value. */ + CTIMER_CTRL6_TMRB6CLK_XT = 6, /*!< XT : Clock source is the XT (uncalibrated). value. */ + CTIMER_CTRL6_TMRB6CLK_XT_DIV2 = 7, /*!< XT_DIV2 : Clock source is XT / 2 value. */ + CTIMER_CTRL6_TMRB6CLK_XT_DIV16 = 8, /*!< XT_DIV16 : Clock source is XT / 16 value. */ + CTIMER_CTRL6_TMRB6CLK_XT_DIV128 = 9, /*!< XT_DIV128 : Clock source is XT / 128 value. */ + CTIMER_CTRL6_TMRB6CLK_LFRC_DIV2 = 10, /*!< LFRC_DIV2 : Clock source is LFRC / 2 value. */ + CTIMER_CTRL6_TMRB6CLK_LFRC_DIV32 = 11, /*!< LFRC_DIV32 : Clock source is LFRC / 32 value. */ + CTIMER_CTRL6_TMRB6CLK_LFRC_DIV1K = 12, /*!< LFRC_DIV1K : Clock source is LFRC / 1024 value. */ + CTIMER_CTRL6_TMRB6CLK_LFRC = 13, /*!< LFRC : Clock source is LFRC value. */ + CTIMER_CTRL6_TMRB6CLK_RTC_100HZ = 14, /*!< RTC_100HZ : Clock source is 100 Hz from the current RTC oscillator. + value. */ + CTIMER_CTRL6_TMRB6CLK_HCLK = 15, /*!< HCLK : Clock source is HCLK. value. */ + CTIMER_CTRL6_TMRB6CLK_XT_DIV4 = 16, /*!< XT_DIV4 : Clock source is XT / 4 value. */ + CTIMER_CTRL6_TMRB6CLK_XT_DIV8 = 17, /*!< XT_DIV8 : Clock source is XT / 8 value. */ + CTIMER_CTRL6_TMRB6CLK_XT_DIV32 = 18, /*!< XT_DIV32 : Clock source is XT / 32 value. */ + CTIMER_CTRL6_TMRB6CLK_CTMRA6 = 20, /*!< CTMRA6 : Clock source is CTIMERA6 OUT. value. */ + CTIMER_CTRL6_TMRB6CLK_CTMRA3 = 21, /*!< CTMRA3 : Clock source is CTIMERA3 OUT. value. */ + CTIMER_CTRL6_TMRB6CLK_CTMRB3 = 22, /*!< CTMRB3 : Clock source is CTIMERB3 OUT. value. */ + CTIMER_CTRL6_TMRB6CLK_CTMRA7 = 23, /*!< CTMRA7 : Clock source is CTIMERA7 OUT. value. */ + CTIMER_CTRL6_TMRB6CLK_CTMRB7 = 24, /*!< CTMRB7 : Clock source is CTIMERB7 OUT. value. */ + CTIMER_CTRL6_TMRB6CLK_CTMRB0 = 25, /*!< CTMRB0 : Clock source is CTIMERB0 OUT. value. */ + CTIMER_CTRL6_TMRB6CLK_CTMRB1 = 26, /*!< CTMRB1 : Clock source is CTIMERB1 OUT. value. */ + CTIMER_CTRL6_TMRB6CLK_CTMRB2 = 27, /*!< CTMRB2 : Clock source is CTIMERB2 OUT. value. */ + CTIMER_CTRL6_TMRB6CLK_CTMRB4 = 28, /*!< CTMRB4 : Clock source is CTIMERB4 OUT. value. */ + CTIMER_CTRL6_TMRB6CLK_BUCKBLE = 29, /*!< BUCKBLE : Clock source is BLE buck converter TON pulses. value. */ + CTIMER_CTRL6_TMRB6CLK_BUCKB = 30, /*!< BUCKB : Clock source is Memory buck converter TON pulses. value. */ + CTIMER_CTRL6_TMRB6CLK_BUCKA = 31, /*!< BUCKA : Clock source is CPU buck converter TON pulses. value. */ +} CTIMER_CTRL6_TMRB6CLK_Enum; + +/* ============================================= CTIMER CTRL6 TMRB6EN [16..16] ============================================= */ +typedef enum { /*!< CTIMER_CTRL6_TMRB6EN */ + CTIMER_CTRL6_TMRB6EN_DIS = 0, /*!< DIS : Counter/Timer B6 Disable. value. */ + CTIMER_CTRL6_TMRB6EN_EN = 1, /*!< EN : Counter/Timer B6 Enable. value. */ +} CTIMER_CTRL6_TMRB6EN_Enum; + +/* ============================================ CTIMER CTRL6 TMRA6POL [12..12] ============================================= */ +typedef enum { /*!< CTIMER_CTRL6_TMRA6POL */ + CTIMER_CTRL6_TMRA6POL_NORMAL = 0, /*!< NORMAL : The polarity of the TMRPINA6 pin is the same as the + timer output. value. */ + CTIMER_CTRL6_TMRA6POL_INVERTED = 1, /*!< INVERTED : The polarity of the TMRPINA6 pin is the inverse of + the timer output. value. */ +} CTIMER_CTRL6_TMRA6POL_Enum; + +/* ============================================ CTIMER CTRL6 TMRA6CLR [11..11] ============================================= */ +typedef enum { /*!< CTIMER_CTRL6_TMRA6CLR */ + CTIMER_CTRL6_TMRA6CLR_RUN = 0, /*!< RUN : Allow counter/timer A6 to run value. */ + CTIMER_CTRL6_TMRA6CLR_CLEAR = 1, /*!< CLEAR : Holds counter/timer A6 at 0x0000. value. */ +} CTIMER_CTRL6_TMRA6CLR_Enum; + +/* ============================================ CTIMER CTRL6 TMRA6IE1 [10..10] ============================================= */ +typedef enum { /*!< CTIMER_CTRL6_TMRA6IE1 */ + CTIMER_CTRL6_TMRA6IE1_DIS = 0, /*!< DIS : Disable counter/timer A6 from generating an interrupt + based on COMPR1. value. */ + CTIMER_CTRL6_TMRA6IE1_EN = 1, /*!< EN : Enable counter/timer A6 to generate an interrupt based + on COMPR1. value. */ +} CTIMER_CTRL6_TMRA6IE1_Enum; + +/* ============================================= CTIMER CTRL6 TMRA6IE0 [9..9] ============================================== */ +typedef enum { /*!< CTIMER_CTRL6_TMRA6IE0 */ + CTIMER_CTRL6_TMRA6IE0_DIS = 0, /*!< DIS : Disable counter/timer A6 from generating an interrupt + based on COMPR0. value. */ + CTIMER_CTRL6_TMRA6IE0_EN = 1, /*!< EN : Enable counter/timer A6 to generate an interrupt based + on COMPR0. value. */ +} CTIMER_CTRL6_TMRA6IE0_Enum; + +/* ============================================== CTIMER CTRL6 TMRA6FN [6..8] ============================================== */ +typedef enum { /*!< CTIMER_CTRL6_TMRA6FN */ + CTIMER_CTRL6_TMRA6FN_SINGLECOUNT = 0, /*!< SINGLECOUNT : Single count (output toggles and sticks). Count + to CMPR0A6, stop. value. */ + CTIMER_CTRL6_TMRA6FN_REPEATEDCOUNT = 1, /*!< REPEATEDCOUNT : Repeated count (periodic 1-clock-cycle-wide + pulses). Count to CMPR0A6, restart. value. */ + CTIMER_CTRL6_TMRA6FN_PULSE_ONCE = 2, /*!< PULSE_ONCE : Pulse once (aka one-shot). Count to CMPR0A6, assert, + count to CMPR1A6, deassert, stop. value. */ + CTIMER_CTRL6_TMRA6FN_PULSE_CONT = 3, /*!< PULSE_CONT : Pulse continously. Count to CMPR0A6, assert, count + to CMPR1A6, deassert, restart. value. */ + CTIMER_CTRL6_TMRA6FN_SINGLEPATTERN = 4, /*!< SINGLEPATTERN : Single pattern. value. */ + CTIMER_CTRL6_TMRA6FN_REPEATPATTERN = 5, /*!< REPEATPATTERN : Repeated pattern. value. */ + CTIMER_CTRL6_TMRA6FN_CONTINUOUS = 6, /*!< CONTINUOUS : Continuous run (aka Free Run). Count continuously. + value. */ + CTIMER_CTRL6_TMRA6FN_ALTPWN = 7, /*!< ALTPWN : Alternate PWM value. */ +} CTIMER_CTRL6_TMRA6FN_Enum; + +/* ============================================= CTIMER CTRL6 TMRA6CLK [1..5] ============================================== */ +typedef enum { /*!< CTIMER_CTRL6_TMRA6CLK */ + CTIMER_CTRL6_TMRA6CLK_TMRPIN = 0, /*!< TMRPIN : Clock source is TMRPINA. value. */ + CTIMER_CTRL6_TMRA6CLK_HFRC_DIV4 = 1, /*!< HFRC_DIV4 : Clock source is the HFRC / 4 value. */ + CTIMER_CTRL6_TMRA6CLK_HFRC_DIV16 = 2, /*!< HFRC_DIV16 : Clock source is HFRC / 16 value. */ + CTIMER_CTRL6_TMRA6CLK_HFRC_DIV256 = 3, /*!< HFRC_DIV256 : Clock source is HFRC / 256 value. */ + CTIMER_CTRL6_TMRA6CLK_HFRC_DIV1024 = 4, /*!< HFRC_DIV1024 : Clock source is HFRC / 1024 value. */ + CTIMER_CTRL6_TMRA6CLK_HFRC_DIV4K = 5, /*!< HFRC_DIV4K : Clock source is HFRC / 4096 value. */ + CTIMER_CTRL6_TMRA6CLK_XT = 6, /*!< XT : Clock source is the XT (uncalibrated). value. */ + CTIMER_CTRL6_TMRA6CLK_XT_DIV2 = 7, /*!< XT_DIV2 : Clock source is XT / 2 value. */ + CTIMER_CTRL6_TMRA6CLK_XT_DIV16 = 8, /*!< XT_DIV16 : Clock source is XT / 16 value. */ + CTIMER_CTRL6_TMRA6CLK_XT_DIV128 = 9, /*!< XT_DIV128 : Clock source is XT / 128 value. */ + CTIMER_CTRL6_TMRA6CLK_LFRC_DIV2 = 10, /*!< LFRC_DIV2 : Clock source is LFRC / 2 value. */ + CTIMER_CTRL6_TMRA6CLK_LFRC_DIV32 = 11, /*!< LFRC_DIV32 : Clock source is LFRC / 32 value. */ + CTIMER_CTRL6_TMRA6CLK_LFRC_DIV1K = 12, /*!< LFRC_DIV1K : Clock source is LFRC / 1024 value. */ + CTIMER_CTRL6_TMRA6CLK_LFRC = 13, /*!< LFRC : Clock source is LFRC value. */ + CTIMER_CTRL6_TMRA6CLK_RTC_100HZ = 14, /*!< RTC_100HZ : Clock source is 100 Hz from the current RTC oscillator. + value. */ + CTIMER_CTRL6_TMRA6CLK_HCLK = 15, /*!< HCLK : Clock source is HCLK. value. */ + CTIMER_CTRL6_TMRA6CLK_XT_DIV4 = 16, /*!< XT_DIV4 : Clock source is XT / 4 value. */ + CTIMER_CTRL6_TMRA6CLK_XT_DIV8 = 17, /*!< XT_DIV8 : Clock source is XT / 8 value. */ + CTIMER_CTRL6_TMRA6CLK_XT_DIV32 = 18, /*!< XT_DIV32 : Clock source is XT / 32 value. */ + CTIMER_CTRL6_TMRA6CLK_CTMRB6 = 20, /*!< CTMRB6 : Clock source is CTIMERB6 OUT. value. */ + CTIMER_CTRL6_TMRA6CLK_CTMRA3 = 21, /*!< CTMRA3 : Clock source is CTIMERA3 OUT. value. */ + CTIMER_CTRL6_TMRA6CLK_CTMRB3 = 22, /*!< CTMRB3 : Clock source is CTIMERB3 OUT. value. */ + CTIMER_CTRL6_TMRA6CLK_CTMRA7 = 23, /*!< CTMRA7 : Clock source is CTIMERA7 OUT. value. */ + CTIMER_CTRL6_TMRA6CLK_CTMRB7 = 24, /*!< CTMRB7 : Clock source is CTIMERB7 OUT. value. */ + CTIMER_CTRL6_TMRA6CLK_CTMRB0 = 25, /*!< CTMRB0 : Clock source is CTIMERB0 OUT. value. */ + CTIMER_CTRL6_TMRA6CLK_CTMRB1 = 26, /*!< CTMRB1 : Clock source is CTIMERB1 OUT. value. */ + CTIMER_CTRL6_TMRA6CLK_CTMRB2 = 27, /*!< CTMRB2 : Clock source is CTIMERB2 OUT. value. */ + CTIMER_CTRL6_TMRA6CLK_CTMRB4 = 28, /*!< CTMRB4 : Clock source is CTIMERB4 OUT. value. */ + CTIMER_CTRL6_TMRA6CLK_BUCKBLE = 29, /*!< BUCKBLE : Clock source is BLE buck converter TON pulses. value. */ + CTIMER_CTRL6_TMRA6CLK_BUCKB = 30, /*!< BUCKB : Clock source is Memory buck converter TON pulses. value. */ + CTIMER_CTRL6_TMRA6CLK_BUCKA = 31, /*!< BUCKA : Clock source is CPU buck converter TON pulses. value. */ +} CTIMER_CTRL6_TMRA6CLK_Enum; + +/* ============================================== CTIMER CTRL6 TMRA6EN [0..0] ============================================== */ +typedef enum { /*!< CTIMER_CTRL6_TMRA6EN */ + CTIMER_CTRL6_TMRA6EN_DIS = 0, /*!< DIS : Counter/Timer A6 Disable. value. */ + CTIMER_CTRL6_TMRA6EN_EN = 1, /*!< EN : Counter/Timer A6 Enable. value. */ +} CTIMER_CTRL6_TMRA6EN_Enum; + +/* ======================================================= CMPRAUXA6 ======================================================= */ +/* ======================================================= CMPRAUXB6 ======================================================= */ +/* ========================================================= AUX6 ========================================================== */ +/* ============================================ CTIMER AUX6 TMRB6EN23 [30..30] ============================================= */ +typedef enum { /*!< CTIMER_AUX6_TMRB6EN23 */ + CTIMER_AUX6_TMRB6EN23_DIS = 1, /*!< DIS : Disable enhanced functions. value. */ + CTIMER_AUX6_TMRB6EN23_EN = 0, /*!< EN : Enable enhanced functions. value. */ +} CTIMER_AUX6_TMRB6EN23_Enum; + +/* ============================================ CTIMER AUX6 TMRB6POL23 [29..29] ============================================ */ +typedef enum { /*!< CTIMER_AUX6_TMRB6POL23 */ + CTIMER_AUX6_TMRB6POL23_NORM = 0, /*!< NORM : Upper output normal polarity value. */ + CTIMER_AUX6_TMRB6POL23_INV = 1, /*!< INV : Upper output inverted polarity. value. */ +} CTIMER_AUX6_TMRB6POL23_Enum; + +/* ============================================ CTIMER AUX6 TMRB6TINV [28..28] ============================================= */ +typedef enum { /*!< CTIMER_AUX6_TMRB6TINV */ + CTIMER_AUX6_TMRB6TINV_DIS = 0, /*!< DIS : Disable invert on trigger value. */ + CTIMER_AUX6_TMRB6TINV_EN = 1, /*!< EN : Enable invert on trigger value. */ +} CTIMER_AUX6_TMRB6TINV_Enum; + +/* =========================================== CTIMER AUX6 TMRB6NOSYNC [27..27] ============================================ */ +typedef enum { /*!< CTIMER_AUX6_TMRB6NOSYNC */ + CTIMER_AUX6_TMRB6NOSYNC_DIS = 0, /*!< DIS : Synchronization on source clock value. */ + CTIMER_AUX6_TMRB6NOSYNC_NOSYNC = 1, /*!< NOSYNC : No synchronization on source clock value. */ +} CTIMER_AUX6_TMRB6NOSYNC_Enum; + +/* ============================================ CTIMER AUX6 TMRB6TRIG [23..26] ============================================= */ +typedef enum { /*!< CTIMER_AUX6_TMRB6TRIG */ + CTIMER_AUX6_TMRB6TRIG_DIS = 0, /*!< DIS : Trigger source is disabled. value. */ + CTIMER_AUX6_TMRB6TRIG_A6OUT = 1, /*!< A6OUT : Trigger source is CTIMERA6 OUT. value. */ + CTIMER_AUX6_TMRB6TRIG_B3OUT = 2, /*!< B3OUT : Trigger source is CTIMERB3 OUT. value. */ + CTIMER_AUX6_TMRB6TRIG_A3OUT = 3, /*!< A3OUT : Trigger source is CTIMERA3 OUT. value. */ + CTIMER_AUX6_TMRB6TRIG_A4OUT = 4, /*!< A4OUT : Trigger source is CTIMERA4 OUT. value. */ + CTIMER_AUX6_TMRB6TRIG_B4OUT = 5, /*!< B4OUT : Trigger source is CTIMERB4 OUT. value. */ + CTIMER_AUX6_TMRB6TRIG_A1OUT = 6, /*!< A1OUT : Trigger source is CTIMERA1 OUT. value. */ + CTIMER_AUX6_TMRB6TRIG_B1OUT = 7, /*!< B1OUT : Trigger source is CTIMERB1 OUT. value. */ + CTIMER_AUX6_TMRB6TRIG_B3OUT2 = 8, /*!< B3OUT2 : Trigger source is CTIMERB3 OUT2. value. */ + CTIMER_AUX6_TMRB6TRIG_A3OUT2 = 9, /*!< A3OUT2 : Trigger source is CTIMERA3 OUT2. value. */ + CTIMER_AUX6_TMRB6TRIG_A2OUT2 = 10, /*!< A2OUT2 : Trigger source is CTIMERA2 OUT2. value. */ + CTIMER_AUX6_TMRB6TRIG_B2OUT2 = 11, /*!< B2OUT2 : Trigger source is CTIMERB2 OUT2. value. */ + CTIMER_AUX6_TMRB6TRIG_A6OUT2DUAL = 12, /*!< A6OUT2DUAL : Trigger source is CTIMERA6 OUT2, dual edge. value. */ + CTIMER_AUX6_TMRB6TRIG_A7OUT2DUAL = 13, /*!< A7OUT2DUAL : Trigger source is CTIMERA7 OUT2, dual edge. value. */ + CTIMER_AUX6_TMRB6TRIG_B0OUT2DUAL = 14, /*!< B0OUT2DUAL : Trigger source is CTIMERB0 OUT2, dual edge. value. */ + CTIMER_AUX6_TMRB6TRIG_A0OUT2DUAL = 15, /*!< A0OUT2DUAL : Trigger source is CTIMERA0 OUT2, dual edge. value. */ +} CTIMER_AUX6_TMRB6TRIG_Enum; + +/* ============================================ CTIMER AUX6 TMRA6EN23 [14..14] ============================================= */ +typedef enum { /*!< CTIMER_AUX6_TMRA6EN23 */ + CTIMER_AUX6_TMRA6EN23_DIS = 1, /*!< DIS : Disable enhanced functions. value. */ + CTIMER_AUX6_TMRA6EN23_EN = 0, /*!< EN : Enable enhanced functions. value. */ +} CTIMER_AUX6_TMRA6EN23_Enum; + +/* ============================================ CTIMER AUX6 TMRA6POL23 [13..13] ============================================ */ +typedef enum { /*!< CTIMER_AUX6_TMRA6POL23 */ + CTIMER_AUX6_TMRA6POL23_NORM = 0, /*!< NORM : Upper output normal polarity value. */ + CTIMER_AUX6_TMRA6POL23_INV = 1, /*!< INV : Upper output inverted polarity. value. */ +} CTIMER_AUX6_TMRA6POL23_Enum; + +/* ============================================ CTIMER AUX6 TMRA6TINV [12..12] ============================================= */ +typedef enum { /*!< CTIMER_AUX6_TMRA6TINV */ + CTIMER_AUX6_TMRA6TINV_DIS = 0, /*!< DIS : Disable invert on trigger value. */ + CTIMER_AUX6_TMRA6TINV_EN = 1, /*!< EN : Enable invert on trigger value. */ +} CTIMER_AUX6_TMRA6TINV_Enum; + +/* =========================================== CTIMER AUX6 TMRA6NOSYNC [11..11] ============================================ */ +typedef enum { /*!< CTIMER_AUX6_TMRA6NOSYNC */ + CTIMER_AUX6_TMRA6NOSYNC_DIS = 0, /*!< DIS : Synchronization on source clock value. */ + CTIMER_AUX6_TMRA6NOSYNC_NOSYNC = 1, /*!< NOSYNC : No synchronization on source clock value. */ +} CTIMER_AUX6_TMRA6NOSYNC_Enum; + +/* ============================================= CTIMER AUX6 TMRA6TRIG [7..10] ============================================= */ +typedef enum { /*!< CTIMER_AUX6_TMRA6TRIG */ + CTIMER_AUX6_TMRA6TRIG_DIS = 0, /*!< DIS : Trigger source is disabled. value. */ + CTIMER_AUX6_TMRA6TRIG_B6OUT = 1, /*!< B6OUT : Trigger source is CTIMERB6 OUT. value. */ + CTIMER_AUX6_TMRA6TRIG_B3OUT = 2, /*!< B3OUT : Trigger source is CTIMERB3 OUT. value. */ + CTIMER_AUX6_TMRA6TRIG_A3OUT = 3, /*!< A3OUT : Trigger source is CTIMERA3 OUT. value. */ + CTIMER_AUX6_TMRA6TRIG_A5OUT = 4, /*!< A5OUT : Trigger source is CTIMERA5 OUT. value. */ + CTIMER_AUX6_TMRA6TRIG_B5OUT = 5, /*!< B5OUT : Trigger source is CTIMERB5 OUT. value. */ + CTIMER_AUX6_TMRA6TRIG_A1OUT = 6, /*!< A1OUT : Trigger source is CTIMERA1 OUT. value. */ + CTIMER_AUX6_TMRA6TRIG_B1OUT = 7, /*!< B1OUT : Trigger source is CTIMERB1 OUT. value. */ + CTIMER_AUX6_TMRA6TRIG_B3OUT2 = 8, /*!< B3OUT2 : Trigger source is CTIMERB3 OUT2. value. */ + CTIMER_AUX6_TMRA6TRIG_A3OUT2 = 9, /*!< A3OUT2 : Trigger source is CTIMERA3 OUT2. value. */ + CTIMER_AUX6_TMRA6TRIG_A2OUT2 = 10, /*!< A2OUT2 : Trigger source is CTIMERA2 OUT2. value. */ + CTIMER_AUX6_TMRA6TRIG_B2OUT2 = 11, /*!< B2OUT2 : Trigger source is CTIMERBb OUT2. value. */ + CTIMER_AUX6_TMRA6TRIG_A5OUT2DUAL = 12, /*!< A5OUT2DUAL : Trigger source is CTIMERA5 OUT2, dual edge. value. */ + CTIMER_AUX6_TMRA6TRIG_A7OUT2DUAL = 13, /*!< A7OUT2DUAL : Trigger source is CTIMERA7 OUT2, dual edge. value. */ + CTIMER_AUX6_TMRA6TRIG_B0OUT2DUAL = 14, /*!< B0OUT2DUAL : Trigger source is CTIMERB0 OUT2, dual edge. value. */ + CTIMER_AUX6_TMRA6TRIG_A0OUT2DUAL = 15, /*!< A0OUT2DUAL : Trigger source is CTIMERA0 OUT2, dual edge. value. */ +} CTIMER_AUX6_TMRA6TRIG_Enum; + +/* ========================================================= TMR7 ========================================================== */ +/* ======================================================== CMPRA7 ========================================================= */ +/* ======================================================== CMPRB7 ========================================================= */ +/* ========================================================= CTRL7 ========================================================= */ +/* ============================================= CTIMER CTRL7 CTLINK7 [31..31] ============================================= */ +typedef enum { /*!< CTIMER_CTRL7_CTLINK7 */ + CTIMER_CTRL7_CTLINK7_TWO_16BIT_TIMERS = 0, /*!< TWO_16BIT_TIMERS : Use A7/B7 timers as two independent 16-bit + timers (default). value. */ + CTIMER_CTRL7_CTLINK7_32BIT_TIMER = 1, /*!< 32BIT_TIMER : Link A7/B7 timers into a single 32-bit timer. + value. */ +} CTIMER_CTRL7_CTLINK7_Enum; + +/* ============================================ CTIMER CTRL7 TMRB7POL [28..28] ============================================= */ +typedef enum { /*!< CTIMER_CTRL7_TMRB7POL */ + CTIMER_CTRL7_TMRB7POL_NORMAL = 0, /*!< NORMAL : The polarity of the TMRPINB7 pin is the same as the + timer output. value. */ + CTIMER_CTRL7_TMRB7POL_INVERTED = 1, /*!< INVERTED : The polarity of the TMRPINB7 pin is the inverse of + the timer output. value. */ +} CTIMER_CTRL7_TMRB7POL_Enum; + +/* ============================================ CTIMER CTRL7 TMRB7CLR [27..27] ============================================= */ +typedef enum { /*!< CTIMER_CTRL7_TMRB7CLR */ + CTIMER_CTRL7_TMRB7CLR_RUN = 0, /*!< RUN : Allow counter/timer B7 to run value. */ + CTIMER_CTRL7_TMRB7CLR_CLEAR = 1, /*!< CLEAR : Holds counter/timer B7 at 0x0000. value. */ +} CTIMER_CTRL7_TMRB7CLR_Enum; + +/* ============================================ CTIMER CTRL7 TMRB7IE1 [26..26] ============================================= */ +typedef enum { /*!< CTIMER_CTRL7_TMRB7IE1 */ + CTIMER_CTRL7_TMRB7IE1_DIS = 0, /*!< DIS : Disable counter/timer B7 from generating an interrupt + based on COMPR1. value. */ + CTIMER_CTRL7_TMRB7IE1_EN = 1, /*!< EN : Enable counter/timer B7 to generate an interrupt based + on COMPR1. value. */ +} CTIMER_CTRL7_TMRB7IE1_Enum; + +/* ============================================ CTIMER CTRL7 TMRB7IE0 [25..25] ============================================= */ +typedef enum { /*!< CTIMER_CTRL7_TMRB7IE0 */ + CTIMER_CTRL7_TMRB7IE0_DIS = 0, /*!< DIS : Disable counter/timer B7 from generating an interrupt + based on COMPR0. value. */ + CTIMER_CTRL7_TMRB7IE0_EN = 1, /*!< EN : Enable counter/timer B7 to generate an interrupt based + on COMPR0 value. */ +} CTIMER_CTRL7_TMRB7IE0_Enum; + +/* ============================================= CTIMER CTRL7 TMRB7FN [22..24] ============================================= */ +typedef enum { /*!< CTIMER_CTRL7_TMRB7FN */ + CTIMER_CTRL7_TMRB7FN_SINGLECOUNT = 0, /*!< SINGLECOUNT : Single count (output toggles and sticks). Count + to CMPR0B7, stop. value. */ + CTIMER_CTRL7_TMRB7FN_REPEATEDCOUNT = 1, /*!< REPEATEDCOUNT : Repeated count (periodic 1-clock-cycle-wide + pulses). Count to CMPR0B7, restart. value. */ + CTIMER_CTRL7_TMRB7FN_PULSE_ONCE = 2, /*!< PULSE_ONCE : Pulse once (aka one-shot). Count to CMPR0B7, assert, + count to CMPR1B7, deassert, stop. value. */ + CTIMER_CTRL7_TMRB7FN_PULSE_CONT = 3, /*!< PULSE_CONT : Pulse continously. Count to CMPR0B7, assert, count + to CMPR1B7, deassert, restart. value. */ + CTIMER_CTRL7_TMRB7FN_SINGLEPATTERN = 4, /*!< SINGLEPATTERN : Single pattern. value. */ + CTIMER_CTRL7_TMRB7FN_REPEATPATTERN = 5, /*!< REPEATPATTERN : Repeated pattern. value. */ + CTIMER_CTRL7_TMRB7FN_CONTINUOUS = 6, /*!< CONTINUOUS : Continuous run (aka Free Run). Count continuously. + value. */ + CTIMER_CTRL7_TMRB7FN_ALTPWN = 7, /*!< ALTPWN : Alternate PWM value. */ +} CTIMER_CTRL7_TMRB7FN_Enum; + +/* ============================================ CTIMER CTRL7 TMRB7CLK [17..21] ============================================= */ +typedef enum { /*!< CTIMER_CTRL7_TMRB7CLK */ + CTIMER_CTRL7_TMRB7CLK_TMRPIN = 0, /*!< TMRPIN : Clock source is TMRPINB. value. */ + CTIMER_CTRL7_TMRB7CLK_HFRC_DIV4 = 1, /*!< HFRC_DIV4 : Clock source is the HFRC / 4 value. */ + CTIMER_CTRL7_TMRB7CLK_HFRC_DIV16 = 2, /*!< HFRC_DIV16 : Clock source is HFRC / 16 value. */ + CTIMER_CTRL7_TMRB7CLK_HFRC_DIV256 = 3, /*!< HFRC_DIV256 : Clock source is HFRC / 256 value. */ + CTIMER_CTRL7_TMRB7CLK_HFRC_DIV1024 = 4, /*!< HFRC_DIV1024 : Clock source is HFRC / 1024 value. */ + CTIMER_CTRL7_TMRB7CLK_HFRC_DIV4K = 5, /*!< HFRC_DIV4K : Clock source is HFRC / 4096 value. */ + CTIMER_CTRL7_TMRB7CLK_XT = 6, /*!< XT : Clock source is the XT (uncalibrated). value. */ + CTIMER_CTRL7_TMRB7CLK_XT_DIV2 = 7, /*!< XT_DIV2 : Clock source is XT / 2 value. */ + CTIMER_CTRL7_TMRB7CLK_XT_DIV16 = 8, /*!< XT_DIV16 : Clock source is XT / 16 value. */ + CTIMER_CTRL7_TMRB7CLK_XT_DIV128 = 9, /*!< XT_DIV128 : Clock source is XT / 128 value. */ + CTIMER_CTRL7_TMRB7CLK_LFRC_DIV2 = 10, /*!< LFRC_DIV2 : Clock source is LFRC / 2 value. */ + CTIMER_CTRL7_TMRB7CLK_LFRC_DIV32 = 11, /*!< LFRC_DIV32 : Clock source is LFRC / 32 value. */ + CTIMER_CTRL7_TMRB7CLK_LFRC_DIV1K = 12, /*!< LFRC_DIV1K : Clock source is LFRC / 1024 value. */ + CTIMER_CTRL7_TMRB7CLK_LFRC = 13, /*!< LFRC : Clock source is LFRC value. */ + CTIMER_CTRL7_TMRB7CLK_RTC_100HZ = 14, /*!< RTC_100HZ : Clock source is 100 Hz from the current RTC oscillator. + value. */ + CTIMER_CTRL7_TMRB7CLK_HCLK = 15, /*!< HCLK : Clock source is HCLK. value. */ + CTIMER_CTRL7_TMRB7CLK_XT_DIV4 = 16, /*!< XT_DIV4 : Clock source is XT / 4 value. */ + CTIMER_CTRL7_TMRB7CLK_XT_DIV8 = 17, /*!< XT_DIV8 : Clock source is XT / 8 value. */ + CTIMER_CTRL7_TMRB7CLK_XT_DIV32 = 18, /*!< XT_DIV32 : Clock source is XT / 32 value. */ + CTIMER_CTRL7_TMRB7CLK_CTMRA7 = 20, /*!< CTMRA7 : Clock source is CTIMERA7 OUT. value. */ + CTIMER_CTRL7_TMRB7CLK_CTMRA2 = 21, /*!< CTMRA2 : Clock source is CTIMERA2 OUT. value. */ + CTIMER_CTRL7_TMRB7CLK_CTMRB2 = 22, /*!< CTMRB2 : Clock source is CTIMERB2 OUT. value. */ + CTIMER_CTRL7_TMRB7CLK_CTMRA0 = 23, /*!< CTMRA0 : Clock source is CTIMERA0 OUT. value. */ + CTIMER_CTRL7_TMRB7CLK_CTMRB0 = 24, /*!< CTMRB0 : Clock source is CTIMERB0 OUT. value. */ + CTIMER_CTRL7_TMRB7CLK_CTMRB1 = 25, /*!< CTMRB1 : Clock source is CTIMERB1 OUT. value. */ + CTIMER_CTRL7_TMRB7CLK_CTMRB3 = 26, /*!< CTMRB3 : Clock source is CTIMERB3 OUT. value. */ + CTIMER_CTRL7_TMRB7CLK_CTMRB4 = 27, /*!< CTMRB4 : Clock source is CTIMERB4 OUT. value. */ + CTIMER_CTRL7_TMRB7CLK_CTMRB5 = 28, /*!< CTMRB5 : Clock source is CTIMERB5 OUT. value. */ + CTIMER_CTRL7_TMRB7CLK_BUCKBLE = 29, /*!< BUCKBLE : Clock source is BLE buck converter TON pulses. value. */ + CTIMER_CTRL7_TMRB7CLK_BUCKB = 30, /*!< BUCKB : Clock source is Memory buck converter TON pulses. value. */ + CTIMER_CTRL7_TMRB7CLK_BUCKA = 31, /*!< BUCKA : Clock source is CPU buck converter TON pulses. value. */ +} CTIMER_CTRL7_TMRB7CLK_Enum; + +/* ============================================= CTIMER CTRL7 TMRB7EN [16..16] ============================================= */ +typedef enum { /*!< CTIMER_CTRL7_TMRB7EN */ + CTIMER_CTRL7_TMRB7EN_DIS = 0, /*!< DIS : Counter/Timer B7 Disable. value. */ + CTIMER_CTRL7_TMRB7EN_EN = 1, /*!< EN : Counter/Timer B7 Enable. value. */ +} CTIMER_CTRL7_TMRB7EN_Enum; + +/* ============================================ CTIMER CTRL7 TMRA7POL [12..12] ============================================= */ +typedef enum { /*!< CTIMER_CTRL7_TMRA7POL */ + CTIMER_CTRL7_TMRA7POL_NORMAL = 0, /*!< NORMAL : The polarity of the TMRPINA7 pin is the same as the + timer output. value. */ + CTIMER_CTRL7_TMRA7POL_INVERTED = 1, /*!< INVERTED : The polarity of the TMRPINA7 pin is the inverse of + the timer output. value. */ +} CTIMER_CTRL7_TMRA7POL_Enum; + +/* ============================================ CTIMER CTRL7 TMRA7CLR [11..11] ============================================= */ +typedef enum { /*!< CTIMER_CTRL7_TMRA7CLR */ + CTIMER_CTRL7_TMRA7CLR_RUN = 0, /*!< RUN : Allow counter/timer A7 to run value. */ + CTIMER_CTRL7_TMRA7CLR_CLEAR = 1, /*!< CLEAR : Holds counter/timer A7 at 0x0000. value. */ +} CTIMER_CTRL7_TMRA7CLR_Enum; + +/* ============================================ CTIMER CTRL7 TMRA7IE1 [10..10] ============================================= */ +typedef enum { /*!< CTIMER_CTRL7_TMRA7IE1 */ + CTIMER_CTRL7_TMRA7IE1_DIS = 0, /*!< DIS : Disable counter/timer A7 from generating an interrupt + based on COMPR1. value. */ + CTIMER_CTRL7_TMRA7IE1_EN = 1, /*!< EN : Enable counter/timer A7 to generate an interrupt based + on COMPR1. value. */ +} CTIMER_CTRL7_TMRA7IE1_Enum; + +/* ============================================= CTIMER CTRL7 TMRA7IE0 [9..9] ============================================== */ +typedef enum { /*!< CTIMER_CTRL7_TMRA7IE0 */ + CTIMER_CTRL7_TMRA7IE0_DIS = 0, /*!< DIS : Disable counter/timer A7 from generating an interrupt + based on COMPR0. value. */ + CTIMER_CTRL7_TMRA7IE0_EN = 1, /*!< EN : Enable counter/timer A7 to generate an interrupt based + on COMPR0. value. */ +} CTIMER_CTRL7_TMRA7IE0_Enum; + +/* ============================================== CTIMER CTRL7 TMRA7FN [6..8] ============================================== */ +typedef enum { /*!< CTIMER_CTRL7_TMRA7FN */ + CTIMER_CTRL7_TMRA7FN_SINGLECOUNT = 0, /*!< SINGLECOUNT : Single count (output toggles and sticks). Count + to CMPR0A7, stop. value. */ + CTIMER_CTRL7_TMRA7FN_REPEATEDCOUNT = 1, /*!< REPEATEDCOUNT : Repeated count (periodic 1-clock-cycle-wide + pulses). Count to CMPR0A7, restart. value. */ + CTIMER_CTRL7_TMRA7FN_PULSE_ONCE = 2, /*!< PULSE_ONCE : Pulse once (aka one-shot). Count to CMPR0A7, assert, + count to CMPR1A7, deassert, stop. value. */ + CTIMER_CTRL7_TMRA7FN_PULSE_CONT = 3, /*!< PULSE_CONT : Pulse continously. Count to CMPR0A7, assert, count + to CMPR1A7, deassert, restart. value. */ + CTIMER_CTRL7_TMRA7FN_SINGLEPATTERN = 4, /*!< SINGLEPATTERN : Single pattern. value. */ + CTIMER_CTRL7_TMRA7FN_REPEATPATTERN = 5, /*!< REPEATPATTERN : Repeated pattern. value. */ + CTIMER_CTRL7_TMRA7FN_CONTINUOUS = 6, /*!< CONTINUOUS : Continuous run (aka Free Run). Count continuously. + value. */ + CTIMER_CTRL7_TMRA7FN_ALTPWN = 7, /*!< ALTPWN : Alternate PWM value. */ +} CTIMER_CTRL7_TMRA7FN_Enum; + +/* ============================================= CTIMER CTRL7 TMRA7CLK [1..5] ============================================== */ +typedef enum { /*!< CTIMER_CTRL7_TMRA7CLK */ + CTIMER_CTRL7_TMRA7CLK_TMRPIN = 0, /*!< TMRPIN : Clock source is TMRPINA. value. */ + CTIMER_CTRL7_TMRA7CLK_HFRC_DIV4 = 1, /*!< HFRC_DIV4 : Clock source is the HFRC / 4 value. */ + CTIMER_CTRL7_TMRA7CLK_HFRC_DIV16 = 2, /*!< HFRC_DIV16 : Clock source is HFRC / 16 value. */ + CTIMER_CTRL7_TMRA7CLK_HFRC_DIV256 = 3, /*!< HFRC_DIV256 : Clock source is HFRC / 256 value. */ + CTIMER_CTRL7_TMRA7CLK_HFRC_DIV1024 = 4, /*!< HFRC_DIV1024 : Clock source is HFRC / 1024 value. */ + CTIMER_CTRL7_TMRA7CLK_HFRC_DIV4K = 5, /*!< HFRC_DIV4K : Clock source is HFRC / 4096 value. */ + CTIMER_CTRL7_TMRA7CLK_XT = 6, /*!< XT : Clock source is the XT (uncalibrated). value. */ + CTIMER_CTRL7_TMRA7CLK_XT_DIV2 = 7, /*!< XT_DIV2 : Clock source is XT / 2 value. */ + CTIMER_CTRL7_TMRA7CLK_XT_DIV16 = 8, /*!< XT_DIV16 : Clock source is XT / 16 value. */ + CTIMER_CTRL7_TMRA7CLK_XT_DIV128 = 9, /*!< XT_DIV128 : Clock source is XT / 128 value. */ + CTIMER_CTRL7_TMRA7CLK_LFRC_DIV2 = 10, /*!< LFRC_DIV2 : Clock source is LFRC / 2 value. */ + CTIMER_CTRL7_TMRA7CLK_LFRC_DIV32 = 11, /*!< LFRC_DIV32 : Clock source is LFRC / 32 value. */ + CTIMER_CTRL7_TMRA7CLK_LFRC_DIV1K = 12, /*!< LFRC_DIV1K : Clock source is LFRC / 1024 value. */ + CTIMER_CTRL7_TMRA7CLK_LFRC = 13, /*!< LFRC : Clock source is LFRC value. */ + CTIMER_CTRL7_TMRA7CLK_RTC_100HZ = 14, /*!< RTC_100HZ : Clock source is 100 Hz from the current RTC oscillator. + value. */ + CTIMER_CTRL7_TMRA7CLK_HCLK = 15, /*!< HCLK : Clock source is HCLK. value. */ + CTIMER_CTRL7_TMRA7CLK_XT_DIV4 = 16, /*!< XT_DIV4 : Clock source is XT / 4 value. */ + CTIMER_CTRL7_TMRA7CLK_XT_DIV8 = 17, /*!< XT_DIV8 : Clock source is XT / 8 value. */ + CTIMER_CTRL7_TMRA7CLK_XT_DIV32 = 18, /*!< XT_DIV32 : Clock source is XT / 32 value. */ + CTIMER_CTRL7_TMRA7CLK_CTMRB7 = 20, /*!< CTMRB7 : Clock source is CTIMERB7 OUT. value. */ + CTIMER_CTRL7_TMRA7CLK_CTMRA2 = 21, /*!< CTMRA2 : Clock source is CTIMERA2 OUT. value. */ + CTIMER_CTRL7_TMRA7CLK_CTMRB2 = 22, /*!< CTMRB2 : Clock source is CTIMERB2 OUT. value. */ + CTIMER_CTRL7_TMRA7CLK_CTMRA0 = 23, /*!< CTMRA0 : Clock source is CTIMERA0 OUT. value. */ + CTIMER_CTRL7_TMRA7CLK_CTMRB0 = 24, /*!< CTMRB0 : Clock source is CTIMERB0 OUT. value. */ + CTIMER_CTRL7_TMRA7CLK_CTMRB1 = 25, /*!< CTMRB1 : Clock source is CTIMERB1 OUT. value. */ + CTIMER_CTRL7_TMRA7CLK_CTMRB3 = 26, /*!< CTMRB3 : Clock source is CTIMERB3 OUT. value. */ + CTIMER_CTRL7_TMRA7CLK_CTMRB4 = 27, /*!< CTMRB4 : Clock source is CTIMERB4 OUT. value. */ + CTIMER_CTRL7_TMRA7CLK_CTMRB5 = 28, /*!< CTMRB5 : Clock source is CTIMERB5 OUT. value. */ + CTIMER_CTRL7_TMRA7CLK_BUCKBLE = 29, /*!< BUCKBLE : Clock source is BLE buck converter TON pulses. value. */ + CTIMER_CTRL7_TMRA7CLK_BUCKB = 30, /*!< BUCKB : Clock source is Memory buck converter TON pulses. value. */ + CTIMER_CTRL7_TMRA7CLK_BUCKA = 31, /*!< BUCKA : Clock source is CPU buck converter TON pulses. value. */ +} CTIMER_CTRL7_TMRA7CLK_Enum; + +/* ============================================== CTIMER CTRL7 TMRA7EN [0..0] ============================================== */ +typedef enum { /*!< CTIMER_CTRL7_TMRA7EN */ + CTIMER_CTRL7_TMRA7EN_DIS = 0, /*!< DIS : Counter/Timer A7 Disable. value. */ + CTIMER_CTRL7_TMRA7EN_EN = 1, /*!< EN : Counter/Timer A7 Enable. value. */ +} CTIMER_CTRL7_TMRA7EN_Enum; + +/* ======================================================= CMPRAUXA7 ======================================================= */ +/* ======================================================= CMPRAUXB7 ======================================================= */ +/* ========================================================= AUX7 ========================================================== */ +/* ============================================ CTIMER AUX7 TMRB7EN23 [30..30] ============================================= */ +typedef enum { /*!< CTIMER_AUX7_TMRB7EN23 */ + CTIMER_AUX7_TMRB7EN23_DIS = 1, /*!< DIS : Disable enhanced functions. value. */ + CTIMER_AUX7_TMRB7EN23_EN = 0, /*!< EN : Enable enhanced functions. value. */ +} CTIMER_AUX7_TMRB7EN23_Enum; + +/* ============================================ CTIMER AUX7 TMRB7POL23 [29..29] ============================================ */ +typedef enum { /*!< CTIMER_AUX7_TMRB7POL23 */ + CTIMER_AUX7_TMRB7POL23_NORM = 0, /*!< NORM : Upper output normal polarity value. */ + CTIMER_AUX7_TMRB7POL23_INV = 1, /*!< INV : Upper output inverted polarity. value. */ +} CTIMER_AUX7_TMRB7POL23_Enum; + +/* ============================================ CTIMER AUX7 TMRB7TINV [28..28] ============================================= */ +typedef enum { /*!< CTIMER_AUX7_TMRB7TINV */ + CTIMER_AUX7_TMRB7TINV_DIS = 0, /*!< DIS : Disable invert on trigger value. */ + CTIMER_AUX7_TMRB7TINV_EN = 1, /*!< EN : Enable invert on trigger value. */ +} CTIMER_AUX7_TMRB7TINV_Enum; + +/* =========================================== CTIMER AUX7 TMRB7NOSYNC [27..27] ============================================ */ +typedef enum { /*!< CTIMER_AUX7_TMRB7NOSYNC */ + CTIMER_AUX7_TMRB7NOSYNC_DIS = 0, /*!< DIS : Synchronization on source clock value. */ + CTIMER_AUX7_TMRB7NOSYNC_NOSYNC = 1, /*!< NOSYNC : No synchronization on source clock value. */ +} CTIMER_AUX7_TMRB7NOSYNC_Enum; + +/* ============================================ CTIMER AUX7 TMRB7TRIG [23..26] ============================================= */ +typedef enum { /*!< CTIMER_AUX7_TMRB7TRIG */ + CTIMER_AUX7_TMRB7TRIG_DIS = 0, /*!< DIS : Trigger source is disabled. value. */ + CTIMER_AUX7_TMRB7TRIG_A7OUT = 1, /*!< A7OUT : Trigger source is CTIMERA7 OUT. value. */ + CTIMER_AUX7_TMRB7TRIG_B3OUT = 2, /*!< B3OUT : Trigger source is CTIMERB3 OUT. value. */ + CTIMER_AUX7_TMRB7TRIG_A3OUT = 3, /*!< A3OUT : Trigger source is CTIMERA3 OUT. value. */ + CTIMER_AUX7_TMRB7TRIG_A5OUT = 4, /*!< A5OUT : Trigger source is CTIMERA5 OUT. value. */ + CTIMER_AUX7_TMRB7TRIG_B5OUT = 5, /*!< B5OUT : Trigger source is CTIMERB5 OUT. value. */ + CTIMER_AUX7_TMRB7TRIG_A2OUT = 6, /*!< A2OUT : Trigger source is CTIMERA2 OUT. value. */ + CTIMER_AUX7_TMRB7TRIG_B2OUT = 7, /*!< B2OUT : Trigger source is CTIMERB2 OUT. value. */ + CTIMER_AUX7_TMRB7TRIG_B3OUT2 = 8, /*!< B3OUT2 : Trigger source is CTIMERB3 OUT2. value. */ + CTIMER_AUX7_TMRB7TRIG_A3OUT2 = 9, /*!< A3OUT2 : Trigger source is CTIMERA3 OUT2. value. */ + CTIMER_AUX7_TMRB7TRIG_A2OUT2 = 10, /*!< A2OUT2 : Trigger source is CTIMERA2 OUT2. value. */ + CTIMER_AUX7_TMRB7TRIG_B2OUT2 = 11, /*!< B2OUT2 : Trigger source is CTIMERB2 OUT2. value. */ + CTIMER_AUX7_TMRB7TRIG_A6OUT2DUAL = 12, /*!< A6OUT2DUAL : Trigger source is CTIMERA6 OUT2, dual edge. value. */ + CTIMER_AUX7_TMRB7TRIG_A7OUT2DUAL = 13, /*!< A7OUT2DUAL : Trigger source is CTIMERA7 OUT2, dual edge. value. */ + CTIMER_AUX7_TMRB7TRIG_B1OUT2DUAL = 14, /*!< B1OUT2DUAL : Trigger source is CTIMERB1 OUT2, dual edge. value. */ + CTIMER_AUX7_TMRB7TRIG_A1OUT2DUAL = 15, /*!< A1OUT2DUAL : Trigger source is CTIMERA1 OUT2, dual edge. value. */ +} CTIMER_AUX7_TMRB7TRIG_Enum; + +/* ============================================ CTIMER AUX7 TMRA7EN23 [14..14] ============================================= */ +typedef enum { /*!< CTIMER_AUX7_TMRA7EN23 */ + CTIMER_AUX7_TMRA7EN23_DIS = 1, /*!< DIS : Disable enhanced functions. value. */ + CTIMER_AUX7_TMRA7EN23_EN = 0, /*!< EN : Enable enhanced functions. value. */ +} CTIMER_AUX7_TMRA7EN23_Enum; + +/* ============================================ CTIMER AUX7 TMRA7POL23 [13..13] ============================================ */ +typedef enum { /*!< CTIMER_AUX7_TMRA7POL23 */ + CTIMER_AUX7_TMRA7POL23_NORM = 0, /*!< NORM : Upper output normal polarity value. */ + CTIMER_AUX7_TMRA7POL23_INV = 1, /*!< INV : Upper output inverted polarity. value. */ +} CTIMER_AUX7_TMRA7POL23_Enum; + +/* ============================================ CTIMER AUX7 TMRA7TINV [12..12] ============================================= */ +typedef enum { /*!< CTIMER_AUX7_TMRA7TINV */ + CTIMER_AUX7_TMRA7TINV_DIS = 0, /*!< DIS : Disable invert on trigger value. */ + CTIMER_AUX7_TMRA7TINV_EN = 1, /*!< EN : Enable invert on trigger value. */ +} CTIMER_AUX7_TMRA7TINV_Enum; + +/* =========================================== CTIMER AUX7 TMRA7NOSYNC [11..11] ============================================ */ +typedef enum { /*!< CTIMER_AUX7_TMRA7NOSYNC */ + CTIMER_AUX7_TMRA7NOSYNC_DIS = 0, /*!< DIS : Synchronization on source clock value. */ + CTIMER_AUX7_TMRA7NOSYNC_NOSYNC = 1, /*!< NOSYNC : No synchronization on source clock value. */ +} CTIMER_AUX7_TMRA7NOSYNC_Enum; + +/* ============================================= CTIMER AUX7 TMRA7TRIG [7..10] ============================================= */ +typedef enum { /*!< CTIMER_AUX7_TMRA7TRIG */ + CTIMER_AUX7_TMRA7TRIG_DIS = 0, /*!< DIS : Trigger source is disabled. value. */ + CTIMER_AUX7_TMRA7TRIG_B7OUT = 1, /*!< B7OUT : Trigger source is CTIMERB7 OUT. value. */ + CTIMER_AUX7_TMRA7TRIG_B3OUT = 2, /*!< B3OUT : Trigger source is CTIMERB3 OUT. value. */ + CTIMER_AUX7_TMRA7TRIG_A3OUT = 3, /*!< A3OUT : Trigger source is CTIMERA3 OUT. value. */ + CTIMER_AUX7_TMRA7TRIG_A1OUT = 4, /*!< A1OUT : Trigger source is CTIMERA1 OUT. value. */ + CTIMER_AUX7_TMRA7TRIG_B1OUT = 5, /*!< B1OUT : Trigger source is CTIMERB1 OUT. value. */ + CTIMER_AUX7_TMRA7TRIG_A4OUT = 6, /*!< A4OUT : Trigger source is CTIMERA4 OUT. value. */ + CTIMER_AUX7_TMRA7TRIG_B4OUT = 7, /*!< B4OUT : Trigger source is CTIMERB4 OUT. value. */ + CTIMER_AUX7_TMRA7TRIG_B3OUT2 = 8, /*!< B3OUT2 : Trigger source is CTIMERB3 OUT2. value. */ + CTIMER_AUX7_TMRA7TRIG_A3OUT2 = 9, /*!< A3OUT2 : Trigger source is CTIMERA3 OUT2. value. */ + CTIMER_AUX7_TMRA7TRIG_A2OUT2 = 10, /*!< A2OUT2 : Trigger source is CTIMERA2 OUT2. value. */ + CTIMER_AUX7_TMRA7TRIG_B2OUT2 = 11, /*!< B2OUT2 : Trigger source is CTIMERB2 OUT2. value. */ + CTIMER_AUX7_TMRA7TRIG_A6OUT2DUAL = 12, /*!< A6OUT2DUAL : Trigger source is CTIMERA6 OUT2, dual edge. value. */ + CTIMER_AUX7_TMRA7TRIG_A5OUT2DUAL = 13, /*!< A5OUT2DUAL : Trigger source is CTIMERA5 OUT2, dual edge. value. */ + CTIMER_AUX7_TMRA7TRIG_B4OUT2DUAL = 14, /*!< B4OUT2DUAL : Trigger source is CTIMERB4 OUT2, dual edge. value. */ + CTIMER_AUX7_TMRA7TRIG_A4OUT2DUAL = 15, /*!< A4OUT2DUAL : Trigger source is CTIMERA4 OUT2, dual edge. value. */ +} CTIMER_AUX7_TMRA7TRIG_Enum; + +/* ======================================================== GLOBEN ========================================================= */ +/* ============================================== CTIMER GLOBEN ENB7 [15..15] ============================================== */ +typedef enum { /*!< CTIMER_GLOBEN_ENB7 */ + CTIMER_GLOBEN_ENB7_LCO = 1, /*!< LCO : Use local enable. value. */ + CTIMER_GLOBEN_ENB7_DIS = 0, /*!< DIS : Disable CTIMER. value. */ +} CTIMER_GLOBEN_ENB7_Enum; + +/* ============================================== CTIMER GLOBEN ENA7 [14..14] ============================================== */ +typedef enum { /*!< CTIMER_GLOBEN_ENA7 */ + CTIMER_GLOBEN_ENA7_LCO = 1, /*!< LCO : Use local enable. value. */ + CTIMER_GLOBEN_ENA7_DIS = 0, /*!< DIS : Disable CTIMER. value. */ +} CTIMER_GLOBEN_ENA7_Enum; + +/* ============================================== CTIMER GLOBEN ENB6 [13..13] ============================================== */ +typedef enum { /*!< CTIMER_GLOBEN_ENB6 */ + CTIMER_GLOBEN_ENB6_LCO = 1, /*!< LCO : Use local enable. value. */ + CTIMER_GLOBEN_ENB6_DIS = 0, /*!< DIS : Disable CTIMER. value. */ +} CTIMER_GLOBEN_ENB6_Enum; + +/* ============================================== CTIMER GLOBEN ENA6 [12..12] ============================================== */ +typedef enum { /*!< CTIMER_GLOBEN_ENA6 */ + CTIMER_GLOBEN_ENA6_LCO = 1, /*!< LCO : Use local enable. value. */ + CTIMER_GLOBEN_ENA6_DIS = 0, /*!< DIS : Disable CTIMER. value. */ +} CTIMER_GLOBEN_ENA6_Enum; + +/* ============================================== CTIMER GLOBEN ENB5 [11..11] ============================================== */ +typedef enum { /*!< CTIMER_GLOBEN_ENB5 */ + CTIMER_GLOBEN_ENB5_LCO = 1, /*!< LCO : Use local enable. value. */ + CTIMER_GLOBEN_ENB5_DIS = 0, /*!< DIS : Disable CTIMER. value. */ +} CTIMER_GLOBEN_ENB5_Enum; + +/* ============================================== CTIMER GLOBEN ENA5 [10..10] ============================================== */ +typedef enum { /*!< CTIMER_GLOBEN_ENA5 */ + CTIMER_GLOBEN_ENA5_LCO = 1, /*!< LCO : Use local enable. value. */ + CTIMER_GLOBEN_ENA5_DIS = 0, /*!< DIS : Disable CTIMER. value. */ +} CTIMER_GLOBEN_ENA5_Enum; + +/* =============================================== CTIMER GLOBEN ENB4 [9..9] =============================================== */ +typedef enum { /*!< CTIMER_GLOBEN_ENB4 */ + CTIMER_GLOBEN_ENB4_LCO = 1, /*!< LCO : Use local enable. value. */ + CTIMER_GLOBEN_ENB4_DIS = 0, /*!< DIS : Disable CTIMER. value. */ +} CTIMER_GLOBEN_ENB4_Enum; + +/* =============================================== CTIMER GLOBEN ENA4 [8..8] =============================================== */ +typedef enum { /*!< CTIMER_GLOBEN_ENA4 */ + CTIMER_GLOBEN_ENA4_LCO = 1, /*!< LCO : Use local enable. value. */ + CTIMER_GLOBEN_ENA4_DIS = 0, /*!< DIS : Disable CTIMER. value. */ +} CTIMER_GLOBEN_ENA4_Enum; + +/* =============================================== CTIMER GLOBEN ENB3 [7..7] =============================================== */ +typedef enum { /*!< CTIMER_GLOBEN_ENB3 */ + CTIMER_GLOBEN_ENB3_LCO = 1, /*!< LCO : Use local enable. value. */ + CTIMER_GLOBEN_ENB3_DIS = 0, /*!< DIS : Disable CTIMER. value. */ +} CTIMER_GLOBEN_ENB3_Enum; + +/* =============================================== CTIMER GLOBEN ENA3 [6..6] =============================================== */ +typedef enum { /*!< CTIMER_GLOBEN_ENA3 */ + CTIMER_GLOBEN_ENA3_LCO = 1, /*!< LCO : Use local enable. value. */ + CTIMER_GLOBEN_ENA3_DIS = 0, /*!< DIS : Disable CTIMER. value. */ +} CTIMER_GLOBEN_ENA3_Enum; + +/* =============================================== CTIMER GLOBEN ENB2 [5..5] =============================================== */ +typedef enum { /*!< CTIMER_GLOBEN_ENB2 */ + CTIMER_GLOBEN_ENB2_LCO = 1, /*!< LCO : Use local enable. value. */ + CTIMER_GLOBEN_ENB2_DIS = 0, /*!< DIS : Disable CTIMER. value. */ +} CTIMER_GLOBEN_ENB2_Enum; + +/* =============================================== CTIMER GLOBEN ENA2 [4..4] =============================================== */ +typedef enum { /*!< CTIMER_GLOBEN_ENA2 */ + CTIMER_GLOBEN_ENA2_LCO = 1, /*!< LCO : Use local enable. value. */ + CTIMER_GLOBEN_ENA2_DIS = 0, /*!< DIS : Disable CTIMER. value. */ +} CTIMER_GLOBEN_ENA2_Enum; + +/* =============================================== CTIMER GLOBEN ENB1 [3..3] =============================================== */ +typedef enum { /*!< CTIMER_GLOBEN_ENB1 */ + CTIMER_GLOBEN_ENB1_LCO = 1, /*!< LCO : Use local enable. value. */ + CTIMER_GLOBEN_ENB1_DIS = 0, /*!< DIS : Disable CTIMER. value. */ +} CTIMER_GLOBEN_ENB1_Enum; + +/* =============================================== CTIMER GLOBEN ENA1 [2..2] =============================================== */ +typedef enum { /*!< CTIMER_GLOBEN_ENA1 */ + CTIMER_GLOBEN_ENA1_LCO = 1, /*!< LCO : Use local enable. value. */ + CTIMER_GLOBEN_ENA1_DIS = 0, /*!< DIS : Disable CTIMER. value. */ +} CTIMER_GLOBEN_ENA1_Enum; + +/* =============================================== CTIMER GLOBEN ENB0 [1..1] =============================================== */ +typedef enum { /*!< CTIMER_GLOBEN_ENB0 */ + CTIMER_GLOBEN_ENB0_LCO = 1, /*!< LCO : Use local enable. value. */ + CTIMER_GLOBEN_ENB0_DIS = 0, /*!< DIS : Disable CTIMER. value. */ +} CTIMER_GLOBEN_ENB0_Enum; + +/* =============================================== CTIMER GLOBEN ENA0 [0..0] =============================================== */ +typedef enum { /*!< CTIMER_GLOBEN_ENA0 */ + CTIMER_GLOBEN_ENA0_LCO = 1, /*!< LCO : Use local enable. value. */ + CTIMER_GLOBEN_ENA0_DIS = 0, /*!< DIS : Disable CTIMER. value. */ +} CTIMER_GLOBEN_ENA0_Enum; + +/* ======================================================== OUTCFG0 ======================================================== */ +/* ============================================= CTIMER OUTCFG0 CFG9 [28..30] ============================================== */ +typedef enum { /*!< CTIMER_OUTCFG0_CFG9 */ + CTIMER_OUTCFG0_CFG9_A7OUT2 = 7, /*!< A7OUT2 : Output is A7OUT2. value. */ + CTIMER_OUTCFG0_CFG9_A6OUT2 = 6, /*!< A6OUT2 : Output is A6OUT2. value. */ + CTIMER_OUTCFG0_CFG9_B0OUT = 5, /*!< B0OUT : Output is B0OUT. value. */ + CTIMER_OUTCFG0_CFG9_A4OUT = 4, /*!< A4OUT : Output is A4OUT. value. */ + CTIMER_OUTCFG0_CFG9_A2OUT = 3, /*!< A2OUT : Output is A2OUT. value. */ + CTIMER_OUTCFG0_CFG9_A2OUT2 = 2, /*!< A2OUT2 : Output is A2OUT2 value. */ + CTIMER_OUTCFG0_CFG9_ONE = 1, /*!< ONE : Force output to 1. value. */ + CTIMER_OUTCFG0_CFG9_ZERO = 0, /*!< ZERO : Force output to 0 value. */ +} CTIMER_OUTCFG0_CFG9_Enum; + +/* ============================================= CTIMER OUTCFG0 CFG8 [25..27] ============================================== */ +typedef enum { /*!< CTIMER_OUTCFG0_CFG8 */ + CTIMER_OUTCFG0_CFG8_A7OUT2 = 7, /*!< A7OUT2 : Output is A7OUT2. value. */ + CTIMER_OUTCFG0_CFG8_A6OUT2 = 6, /*!< A6OUT2 : Output is A6OUT2. value. */ + CTIMER_OUTCFG0_CFG8_B6OUT = 5, /*!< B6OUT : Output is B6OUT. value. */ + CTIMER_OUTCFG0_CFG8_A4OUT2 = 4, /*!< A4OUT2 : Output is A4OUT2. value. */ + CTIMER_OUTCFG0_CFG8_A3OUT2 = 3, /*!< A3OUT2 : Output is A3OUT. value. */ + CTIMER_OUTCFG0_CFG8_A2OUT = 2, /*!< A2OUT : Output is A2OUT value. */ + CTIMER_OUTCFG0_CFG8_ONE = 1, /*!< ONE : Force output to 1. value. */ + CTIMER_OUTCFG0_CFG8_ZERO = 0, /*!< ZERO : Force output to 0 value. */ +} CTIMER_OUTCFG0_CFG8_Enum; + +/* ============================================= CTIMER OUTCFG0 CFG7 [22..24] ============================================== */ +typedef enum { /*!< CTIMER_OUTCFG0_CFG7 */ + CTIMER_OUTCFG0_CFG7_A7OUT2 = 7, /*!< A7OUT2 : Output is A7OUT2. value. */ + CTIMER_OUTCFG0_CFG7_A6OUT2 = 6, /*!< A6OUT2 : Output is A6OUT2. value. */ + CTIMER_OUTCFG0_CFG7_A7OUT = 5, /*!< A7OUT : Output is A7OUT. value. */ + CTIMER_OUTCFG0_CFG7_B5OUT = 4, /*!< B5OUT : Output is B5OUT. value. */ + CTIMER_OUTCFG0_CFG7_B1OUT = 3, /*!< B1OUT : Output is B1OUT. value. */ + CTIMER_OUTCFG0_CFG7_B1OUT2 = 2, /*!< B1OUT2 : Output is B1OUT2 value. */ + CTIMER_OUTCFG0_CFG7_ONE = 1, /*!< ONE : Force output to 1. value. */ + CTIMER_OUTCFG0_CFG7_ZERO = 0, /*!< ZERO : Force output to 0 value. */ +} CTIMER_OUTCFG0_CFG7_Enum; + +/* ============================================= CTIMER OUTCFG0 CFG6 [19..21] ============================================== */ +typedef enum { /*!< CTIMER_OUTCFG0_CFG6 */ + CTIMER_OUTCFG0_CFG6_A7OUT2 = 7, /*!< A7OUT2 : Output is A7OUT2. value. */ + CTIMER_OUTCFG0_CFG6_A6OUT2 = 6, /*!< A6OUT2 : Output is A6OUT2. value. */ + CTIMER_OUTCFG0_CFG6_B7OUT = 5, /*!< B7OUT : Output is B7OUT. value. */ + CTIMER_OUTCFG0_CFG6_B5OUT2 = 4, /*!< B5OUT2 : Output is B5OUT2. value. */ + CTIMER_OUTCFG0_CFG6_A1OUT = 3, /*!< A1OUT : Output is A1OUT. value. */ + CTIMER_OUTCFG0_CFG6_B1OUT = 2, /*!< B1OUT : Output is B1OUT value. */ + CTIMER_OUTCFG0_CFG6_ONE = 1, /*!< ONE : Force output to 1. value. */ + CTIMER_OUTCFG0_CFG6_ZERO = 0, /*!< ZERO : Force output to 0 value. */ +} CTIMER_OUTCFG0_CFG6_Enum; + +/* ============================================= CTIMER OUTCFG0 CFG5 [16..18] ============================================== */ +typedef enum { /*!< CTIMER_OUTCFG0_CFG5 */ + CTIMER_OUTCFG0_CFG5_A7OUT2 = 7, /*!< A7OUT2 : Output is A7OUT2. value. */ + CTIMER_OUTCFG0_CFG5_A6OUT2 = 6, /*!< A6OUT2 : Output is A6OUT2. value. */ + CTIMER_OUTCFG0_CFG5_A7OUT = 5, /*!< A7OUT : Output is A7OUT. value. */ + CTIMER_OUTCFG0_CFG5_B6OUT = 4, /*!< B6OUT : Output is A5OUT. value. */ + CTIMER_OUTCFG0_CFG5_A1OUT = 3, /*!< A1OUT : Output is A1OUT. value. */ + CTIMER_OUTCFG0_CFG5_A1OUT2 = 2, /*!< A1OUT2 : Output is A1OUT2 value. */ + CTIMER_OUTCFG0_CFG5_ONE = 1, /*!< ONE : Force output to 1. value. */ + CTIMER_OUTCFG0_CFG5_ZERO = 0, /*!< ZERO : Force output to 0 value. */ +} CTIMER_OUTCFG0_CFG5_Enum; + +/* ============================================= CTIMER OUTCFG0 CFG4 [12..14] ============================================== */ +typedef enum { /*!< CTIMER_OUTCFG0_CFG4 */ + CTIMER_OUTCFG0_CFG4_A7OUT2 = 7, /*!< A7OUT2 : Output is A7OUT2. value. */ + CTIMER_OUTCFG0_CFG4_A6OUT2 = 6, /*!< A6OUT2 : Output is A6OUT2. value. */ + CTIMER_OUTCFG0_CFG4_B5OUT = 5, /*!< B5OUT : Output is B5OUT. value. */ + CTIMER_OUTCFG0_CFG4_A5OUT2 = 4, /*!< A5OUT2 : Output is A5OUT2. value. */ + CTIMER_OUTCFG0_CFG4_A2OUT2 = 3, /*!< A2OUT2 : Output is A2OUT2. value. */ + CTIMER_OUTCFG0_CFG4_A1OUT = 2, /*!< A1OUT : Output is A1OUT value. */ + CTIMER_OUTCFG0_CFG4_ONE = 1, /*!< ONE : Force output to 1. value. */ + CTIMER_OUTCFG0_CFG4_ZERO = 0, /*!< ZERO : Force output to 0 value. */ +} CTIMER_OUTCFG0_CFG4_Enum; + +/* ============================================== CTIMER OUTCFG0 CFG3 [9..11] ============================================== */ +typedef enum { /*!< CTIMER_OUTCFG0_CFG3 */ + CTIMER_OUTCFG0_CFG3_A7OUT2 = 7, /*!< A7OUT2 : Output is A7OUT2. value. */ + CTIMER_OUTCFG0_CFG3_A6OUT2 = 6, /*!< A6OUT2 : Output is A6OUT2. value. */ + CTIMER_OUTCFG0_CFG3_A6OUT = 5, /*!< A6OUT : Output is A6OUT. value. */ + CTIMER_OUTCFG0_CFG3_A1OUT = 4, /*!< A1OUT : Output is A1OUT. value. */ + CTIMER_OUTCFG0_CFG3_B0OUT = 3, /*!< B0OUT : Output is B0OUT. value. */ + CTIMER_OUTCFG0_CFG3_B0OUT2 = 2, /*!< B0OUT2 : Output is B0OUT2 value. */ + CTIMER_OUTCFG0_CFG3_ONE = 1, /*!< ONE : Force output to 1. value. */ + CTIMER_OUTCFG0_CFG3_ZERO = 0, /*!< ZERO : Force output to 0 value. */ +} CTIMER_OUTCFG0_CFG3_Enum; + +/* ============================================== CTIMER OUTCFG0 CFG2 [6..8] =============================================== */ +typedef enum { /*!< CTIMER_OUTCFG0_CFG2 */ + CTIMER_OUTCFG0_CFG2_A7OUT2 = 7, /*!< A7OUT2 : Output is A7OUT2. value. */ + CTIMER_OUTCFG0_CFG2_A6OUT2 = 6, /*!< A6OUT2 : Output is A6OUT2. value. */ + CTIMER_OUTCFG0_CFG2_A7OUT = 5, /*!< A7OUT : Output is A7OUT. value. */ + CTIMER_OUTCFG0_CFG2_B6OUT2 = 4, /*!< B6OUT2 : Output is B6OUT2. value. */ + CTIMER_OUTCFG0_CFG2_B1OUT2 = 3, /*!< B1OUT2 : Output is B1OUT2. value. */ + CTIMER_OUTCFG0_CFG2_B0OUT = 2, /*!< B0OUT : Output is B0OUT value. */ + CTIMER_OUTCFG0_CFG2_ONE = 1, /*!< ONE : Force output to 1. value. */ + CTIMER_OUTCFG0_CFG2_ZERO = 0, /*!< ZERO : Force output to 0 value. */ +} CTIMER_OUTCFG0_CFG2_Enum; + +/* ============================================== CTIMER OUTCFG0 CFG1 [3..5] =============================================== */ +typedef enum { /*!< CTIMER_OUTCFG0_CFG1 */ + CTIMER_OUTCFG0_CFG1_A7OUT2 = 7, /*!< A7OUT2 : Output is A7OUT2. value. */ + CTIMER_OUTCFG0_CFG1_A6OUT2 = 6, /*!< A6OUT2 : Output is A6OUT2. value. */ + CTIMER_OUTCFG0_CFG1_B7OUT2 = 5, /*!< B7OUT2 : Output is B7OUT2. value. */ + CTIMER_OUTCFG0_CFG1_A5OUT = 4, /*!< A5OUT : Output is A5OUT. value. */ + CTIMER_OUTCFG0_CFG1_A0OUT = 3, /*!< A0OUT : Output is A0OUT. value. */ + CTIMER_OUTCFG0_CFG1_A0OUT2 = 2, /*!< A0OUT2 : Output is A0OUT2 value. */ + CTIMER_OUTCFG0_CFG1_ONE = 1, /*!< ONE : Force output to 1. value. */ + CTIMER_OUTCFG0_CFG1_ZERO = 0, /*!< ZERO : Force output to 0 value. */ +} CTIMER_OUTCFG0_CFG1_Enum; + +/* ============================================== CTIMER OUTCFG0 CFG0 [0..2] =============================================== */ +typedef enum { /*!< CTIMER_OUTCFG0_CFG0 */ + CTIMER_OUTCFG0_CFG0_A7OUT2 = 7, /*!< A7OUT2 : Output is A7OUT2. value. */ + CTIMER_OUTCFG0_CFG0_A6OUT2 = 6, /*!< A6OUT2 : Output is A6OUT2. value. */ + CTIMER_OUTCFG0_CFG0_A6OUT = 5, /*!< A6OUT : Output is A6OUT. value. */ + CTIMER_OUTCFG0_CFG0_A5OUT2 = 4, /*!< A5OUT2 : Output is A5OUT2. value. */ + CTIMER_OUTCFG0_CFG0_B2OUT2 = 3, /*!< B2OUT2 : Output is B2OUT2. value. */ + CTIMER_OUTCFG0_CFG0_A0OUT = 2, /*!< A0OUT : Output is A0OUT value. */ + CTIMER_OUTCFG0_CFG0_ONE = 1, /*!< ONE : Force output to 1. value. */ + CTIMER_OUTCFG0_CFG0_ZERO = 0, /*!< ZERO : Force output to 0 value. */ +} CTIMER_OUTCFG0_CFG0_Enum; + +/* ======================================================== OUTCFG1 ======================================================== */ +/* ============================================= CTIMER OUTCFG1 CFG19 [28..30] ============================================= */ +typedef enum { /*!< CTIMER_OUTCFG1_CFG19 */ + CTIMER_OUTCFG1_CFG19_A7OUT2 = 7, /*!< A7OUT2 : Output is A7OUT2. value. */ + CTIMER_OUTCFG1_CFG19_A6OUT2 = 6, /*!< A6OUT2 : Output is A6OUT2. value. */ + CTIMER_OUTCFG1_CFG19_B1OUT2 = 5, /*!< B1OUT2 : Output is B1OUT2. value. */ + CTIMER_OUTCFG1_CFG19_B4OUT = 4, /*!< B4OUT : Output is B4OUT. value. */ + CTIMER_OUTCFG1_CFG19_A2OUT = 3, /*!< A2OUT : Output is A2OUT. value. */ + CTIMER_OUTCFG1_CFG19_B4OUT2 = 2, /*!< B4OUT2 : Output is B4OUT2 value. */ + CTIMER_OUTCFG1_CFG19_ONE = 1, /*!< ONE : Force output to 1. value. */ + CTIMER_OUTCFG1_CFG19_ZERO = 0, /*!< ZERO : Force output to 0 value. */ +} CTIMER_OUTCFG1_CFG19_Enum; + +/* ============================================= CTIMER OUTCFG1 CFG18 [25..27] ============================================= */ +typedef enum { /*!< CTIMER_OUTCFG1_CFG18 */ + CTIMER_OUTCFG1_CFG18_A7OUT2 = 7, /*!< A7OUT2 : Output is A7OUT2. value. */ + CTIMER_OUTCFG1_CFG18_A6OUT2 = 6, /*!< A6OUT2 : Output is A6OUT2. value. */ + CTIMER_OUTCFG1_CFG18_A3OUT2 = 5, /*!< A3OUT2 : Output is A3OUT2. value. */ + CTIMER_OUTCFG1_CFG18_A0OUT = 4, /*!< A0OUT : Output is A0OUT. value. */ + CTIMER_OUTCFG1_CFG18_B0OUT = 3, /*!< B0OUT : Output is B0OUT. value. */ + CTIMER_OUTCFG1_CFG18_B4OUT = 2, /*!< B4OUT : Output is B4OUT value. */ + CTIMER_OUTCFG1_CFG18_ONE = 1, /*!< ONE : Force output to 1. value. */ + CTIMER_OUTCFG1_CFG18_ZERO = 0, /*!< ZERO : Force output to 0 value. */ +} CTIMER_OUTCFG1_CFG18_Enum; + +/* ============================================= CTIMER OUTCFG1 CFG17 [22..24] ============================================= */ +typedef enum { /*!< CTIMER_OUTCFG1_CFG17 */ + CTIMER_OUTCFG1_CFG17_A7OUT2 = 7, /*!< A7OUT2 : Output is A7OUT2. value. */ + CTIMER_OUTCFG1_CFG17_A6OUT2 = 6, /*!< A6OUT2 : Output is A6OUT2. value. */ + CTIMER_OUTCFG1_CFG17_A1OUT2 = 5, /*!< A1OUT2 : Output is A1OUT2. value. */ + CTIMER_OUTCFG1_CFG17_A4OUT = 4, /*!< A4OUT : Output is A4OUT. value. */ + CTIMER_OUTCFG1_CFG17_B7OUT = 3, /*!< B7OUT : Output is B7OUT. value. */ + CTIMER_OUTCFG1_CFG17_A4OUT2 = 2, /*!< A4OUT2 : Output is A4OUT2 value. */ + CTIMER_OUTCFG1_CFG17_ONE = 1, /*!< ONE : Force output to 1. value. */ + CTIMER_OUTCFG1_CFG17_ZERO = 0, /*!< ZERO : Force output to 0 value. */ +} CTIMER_OUTCFG1_CFG17_Enum; + +/* ============================================= CTIMER OUTCFG1 CFG16 [19..21] ============================================= */ +typedef enum { /*!< CTIMER_OUTCFG1_CFG16 */ + CTIMER_OUTCFG1_CFG16_A7OUT2 = 7, /*!< A7OUT2 : Output is A7OUT2. value. */ + CTIMER_OUTCFG1_CFG16_A6OUT2 = 6, /*!< A6OUT2 : Output is A6OUT2. value. */ + CTIMER_OUTCFG1_CFG16_B3OUT2 = 5, /*!< B3OUT2 : Output is B3OUT2. value. */ + CTIMER_OUTCFG1_CFG16_A0OUT2 = 4, /*!< A0OUT2 : Output is A0OUT2. value. */ + CTIMER_OUTCFG1_CFG16_A0OUT = 3, /*!< A0OUT : Output is A0OUT. value. */ + CTIMER_OUTCFG1_CFG16_A4OUT = 2, /*!< A4OUT : Output is A4OUT value. */ + CTIMER_OUTCFG1_CFG16_ONE = 1, /*!< ONE : Force output to 1. value. */ + CTIMER_OUTCFG1_CFG16_ZERO = 0, /*!< ZERO : Force output to 0 value. */ +} CTIMER_OUTCFG1_CFG16_Enum; + +/* ============================================= CTIMER OUTCFG1 CFG15 [16..18] ============================================= */ +typedef enum { /*!< CTIMER_OUTCFG1_CFG15 */ + CTIMER_OUTCFG1_CFG15_A7OUT2 = 7, /*!< A7OUT2 : Output is A7OUT2. value. */ + CTIMER_OUTCFG1_CFG15_A6OUT2 = 6, /*!< A6OUT2 : Output is A6OUT2. value. */ + CTIMER_OUTCFG1_CFG15_A4OUT2 = 5, /*!< A4OUT2 : Output is A4OUT2. value. */ + CTIMER_OUTCFG1_CFG15_A7OUT = 4, /*!< A7OUT : Output is B1OUT. value. */ + CTIMER_OUTCFG1_CFG15_B3OUT = 3, /*!< B3OUT : Output is B3OUT. value. */ + CTIMER_OUTCFG1_CFG15_B3OUT2 = 2, /*!< B3OUT2 : Output is B3OUT2 value. */ + CTIMER_OUTCFG1_CFG15_ONE = 1, /*!< ONE : Force output to 1. value. */ + CTIMER_OUTCFG1_CFG15_ZERO = 0, /*!< ZERO : Force output to 0 value. */ +} CTIMER_OUTCFG1_CFG15_Enum; + +/* ============================================= CTIMER OUTCFG1 CFG14 [12..14] ============================================= */ +typedef enum { /*!< CTIMER_OUTCFG1_CFG14 */ + CTIMER_OUTCFG1_CFG14_A7OUT2 = 7, /*!< A7OUT2 : Output is A7OUT2. value. */ + CTIMER_OUTCFG1_CFG14_A6OUT2 = 6, /*!< A6OUT2 : Output is A6OUT2. value. */ + CTIMER_OUTCFG1_CFG14_A7OUT = 5, /*!< A7OUT : Output is A7OUT. value. */ + CTIMER_OUTCFG1_CFG14_B7OUT2 = 4, /*!< B7OUT2 : Output is B7OUT2. value. */ + CTIMER_OUTCFG1_CFG14_B1OUT = 3, /*!< B1OUT : Output is B1OUT. value. */ + CTIMER_OUTCFG1_CFG14_B3OUT = 2, /*!< B3OUT : Output is B3OUT value. */ + CTIMER_OUTCFG1_CFG14_ONE = 1, /*!< ONE : Force output to 1. value. */ + CTIMER_OUTCFG1_CFG14_ZERO = 0, /*!< ZERO : Force output to 0 value. */ +} CTIMER_OUTCFG1_CFG14_Enum; + +/* ============================================= CTIMER OUTCFG1 CFG13 [9..11] ============================================== */ +typedef enum { /*!< CTIMER_OUTCFG1_CFG13 */ + CTIMER_OUTCFG1_CFG13_A7OUT2 = 7, /*!< A7OUT2 : Output is A7OUT2. value. */ + CTIMER_OUTCFG1_CFG13_A6OUT2 = 6, /*!< A6OUT2 : Output is A6OUT2. value. */ + CTIMER_OUTCFG1_CFG13_B4OUT2 = 5, /*!< B4OUT2 : Output is B4OUT2. value. */ + CTIMER_OUTCFG1_CFG13_A6OUT = 4, /*!< A6OUT : Output is B1OUT. value. */ + CTIMER_OUTCFG1_CFG13_A3OUT = 3, /*!< A3OUT : Output is A3OUT. value. */ + CTIMER_OUTCFG1_CFG13_A3OUT2 = 2, /*!< A3OUT2 : Output is A3OUT2 value. */ + CTIMER_OUTCFG1_CFG13_ONE = 1, /*!< ONE : Force output to 1. value. */ + CTIMER_OUTCFG1_CFG13_ZERO = 0, /*!< ZERO : Force output to 0 value. */ +} CTIMER_OUTCFG1_CFG13_Enum; + +/* ============================================== CTIMER OUTCFG1 CFG12 [6..8] ============================================== */ +typedef enum { /*!< CTIMER_OUTCFG1_CFG12 */ + CTIMER_OUTCFG1_CFG12_A7OUT2 = 7, /*!< A7OUT2 : Output is A7OUT2. value. */ + CTIMER_OUTCFG1_CFG12_A6OUT2 = 6, /*!< A6OUT2 : Output is A6OUT2. value. */ + CTIMER_OUTCFG1_CFG12_B6OUT2 = 5, /*!< B6OUT2 : Output is B6OUT2. value. */ + CTIMER_OUTCFG1_CFG12_B0OUT2 = 4, /*!< B0OUT2 : Output is B0OUT2. value. */ + CTIMER_OUTCFG1_CFG12_B1OUT = 3, /*!< B1OUT : Output is B1OUT. value. */ + CTIMER_OUTCFG1_CFG12_A3OUT = 2, /*!< A3OUT : Output is A3OUT value. */ + CTIMER_OUTCFG1_CFG12_ONE = 1, /*!< ONE : Force output to 1. value. */ + CTIMER_OUTCFG1_CFG12_ZERO = 0, /*!< ZERO : Force output to 0 value. */ +} CTIMER_OUTCFG1_CFG12_Enum; + +/* ============================================== CTIMER OUTCFG1 CFG11 [3..5] ============================================== */ +typedef enum { /*!< CTIMER_OUTCFG1_CFG11 */ + CTIMER_OUTCFG1_CFG11_A7OUT2 = 7, /*!< A7OUT2 : Output is A7OUT2. value. */ + CTIMER_OUTCFG1_CFG11_A6OUT2 = 6, /*!< A6OUT2 : Output is A6OUT2. value. */ + CTIMER_OUTCFG1_CFG11_B5OUT2 = 5, /*!< B5OUT2 : Output is B5OUT2. value. */ + CTIMER_OUTCFG1_CFG11_B4OUT = 4, /*!< B4OUT : Output is B4OUT. value. */ + CTIMER_OUTCFG1_CFG11_B2OUT = 3, /*!< B2OUT : Output is B2OUT. value. */ + CTIMER_OUTCFG1_CFG11_B2OUT2 = 2, /*!< B2OUT2 : Output is B2OUT2 value. */ + CTIMER_OUTCFG1_CFG11_ONE = 1, /*!< ONE : Force output to 1. value. */ + CTIMER_OUTCFG1_CFG11_ZERO = 0, /*!< ZERO : Force output to 0 value. */ +} CTIMER_OUTCFG1_CFG11_Enum; + +/* ============================================== CTIMER OUTCFG1 CFG10 [0..2] ============================================== */ +typedef enum { /*!< CTIMER_OUTCFG1_CFG10 */ + CTIMER_OUTCFG1_CFG10_A7OUT2 = 7, /*!< A7OUT2 : Output is A7OUT2. value. */ + CTIMER_OUTCFG1_CFG10_A6OUT2 = 6, /*!< A6OUT2 : Output is A6OUT2. value. */ + CTIMER_OUTCFG1_CFG10_A6OUT = 5, /*!< A6OUT : Output is A6OUT. value. */ + CTIMER_OUTCFG1_CFG10_B4OUT2 = 4, /*!< B4OUT2 : Output is B4OUT2. value. */ + CTIMER_OUTCFG1_CFG10_B3OUT2 = 3, /*!< B3OUT2 : Output is B3OUT2. value. */ + CTIMER_OUTCFG1_CFG10_B2OUT = 2, /*!< B2OUT : Output is B2OUT value. */ + CTIMER_OUTCFG1_CFG10_ONE = 1, /*!< ONE : Force output to 1. value. */ + CTIMER_OUTCFG1_CFG10_ZERO = 0, /*!< ZERO : Force output to 0 value. */ +} CTIMER_OUTCFG1_CFG10_Enum; + +/* ======================================================== OUTCFG2 ======================================================== */ +/* ============================================= CTIMER OUTCFG2 CFG29 [28..30] ============================================= */ +typedef enum { /*!< CTIMER_OUTCFG2_CFG29 */ + CTIMER_OUTCFG2_CFG29_A7OUT2 = 7, /*!< A7OUT2 : Output is A7OUT2. value. */ + CTIMER_OUTCFG2_CFG29_A6OUT2 = 6, /*!< A6OUT2 : Output is A6OUT2. value. */ + CTIMER_OUTCFG2_CFG29_A3OUT2 = 5, /*!< A3OUT2 : Output is A3OUT2. value. */ + CTIMER_OUTCFG2_CFG29_A7OUT = 4, /*!< A7OUT : Output is A7OUT. value. */ + CTIMER_OUTCFG2_CFG29_A1OUT = 3, /*!< A1OUT : Output is A1OUT. value. */ + CTIMER_OUTCFG2_CFG29_B5OUT2 = 2, /*!< B5OUT2 : Output is B5OUT2 value. */ + CTIMER_OUTCFG2_CFG29_ONE = 1, /*!< ONE : Force output to 1. value. */ + CTIMER_OUTCFG2_CFG29_ZERO = 0, /*!< ZERO : Force output to 0 value. */ +} CTIMER_OUTCFG2_CFG29_Enum; + +/* ============================================= CTIMER OUTCFG2 CFG28 [25..27] ============================================= */ +typedef enum { /*!< CTIMER_OUTCFG2_CFG28 */ + CTIMER_OUTCFG2_CFG28_A7OUT2 = 7, /*!< A7OUT2 : Output is A7OUT2. value. */ + CTIMER_OUTCFG2_CFG28_A6OUT2 = 6, /*!< A6OUT2 : Output is A6OUT2. value. */ + CTIMER_OUTCFG2_CFG28_B0OUT2 = 5, /*!< B0OUT2 : Output is B0OUT2. value. */ + CTIMER_OUTCFG2_CFG28_A5OUT2 = 4, /*!< A5OUT2 : Output is A5OUT2. value. */ + CTIMER_OUTCFG2_CFG28_A3OUT = 3, /*!< A3OUT : Output is A3OUT. value. */ + CTIMER_OUTCFG2_CFG28_A7OUT = 2, /*!< A7OUT : Output is A7OUT value. */ + CTIMER_OUTCFG2_CFG28_ONE = 1, /*!< ONE : Force output to 1. value. */ + CTIMER_OUTCFG2_CFG28_ZERO = 0, /*!< ZERO : Force output to 0 value. */ +} CTIMER_OUTCFG2_CFG28_Enum; + +/* ============================================= CTIMER OUTCFG2 CFG27 [22..24] ============================================= */ +typedef enum { /*!< CTIMER_OUTCFG2_CFG27 */ + CTIMER_OUTCFG2_CFG27_A7OUT2 = 7, /*!< A7OUT2 : Output is A7OUT2. value. */ + CTIMER_OUTCFG2_CFG27_A6OUT2 = 6, /*!< A6OUT2 : Output is A6OUT2. value. */ + CTIMER_OUTCFG2_CFG27_B2OUT2 = 5, /*!< B2OUT2 : Output is B2OUT2. value. */ + CTIMER_OUTCFG2_CFG27_B6OUT = 4, /*!< B6OUT : Output is B6OUT. value. */ + CTIMER_OUTCFG2_CFG27_A1OUT = 3, /*!< A1OUT : Output is A1OUT. value. */ + CTIMER_OUTCFG2_CFG27_B6OUT2 = 2, /*!< B6OUT2 : Output is B6OUT2 value. */ + CTIMER_OUTCFG2_CFG27_ONE = 1, /*!< ONE : Force output to 1. value. */ + CTIMER_OUTCFG2_CFG27_ZERO = 0, /*!< ZERO : Force output to 0 value. */ +} CTIMER_OUTCFG2_CFG27_Enum; + +/* ============================================= CTIMER OUTCFG2 CFG26 [19..21] ============================================= */ +typedef enum { /*!< CTIMER_OUTCFG2_CFG26 */ + CTIMER_OUTCFG2_CFG26_A7OUT2 = 7, /*!< A7OUT2 : Output is A7OUT2. value. */ + CTIMER_OUTCFG2_CFG26_A6OUT2 = 6, /*!< A6OUT2 : Output is A6OUT2. value. */ + CTIMER_OUTCFG2_CFG26_A1OUT2 = 5, /*!< A1OUT2 : Output is A1OUT2. value. */ + CTIMER_OUTCFG2_CFG26_A5OUT = 4, /*!< A5OUT : Output is A5OUT. value. */ + CTIMER_OUTCFG2_CFG26_B2OUT = 3, /*!< B2OUT : Output is B2OUT. value. */ + CTIMER_OUTCFG2_CFG26_B6OUT = 2, /*!< B6OUT : Output is B6OUT value. */ + CTIMER_OUTCFG2_CFG26_ONE = 1, /*!< ONE : Force output to 1. value. */ + CTIMER_OUTCFG2_CFG26_ZERO = 0, /*!< ZERO : Force output to 0 value. */ +} CTIMER_OUTCFG2_CFG26_Enum; + +/* ============================================= CTIMER OUTCFG2 CFG25 [16..18] ============================================= */ +typedef enum { /*!< CTIMER_OUTCFG2_CFG25 */ + CTIMER_OUTCFG2_CFG25_A7OUT2 = 7, /*!< A7OUT2 : Output is A7OUT2. value. */ + CTIMER_OUTCFG2_CFG25_A6OUT2 = 6, /*!< A6OUT2 : Output is A6OUT2. value. */ + CTIMER_OUTCFG2_CFG25_A2OUT2 = 5, /*!< A2OUT2 : Output is A2OUT2. value. */ + CTIMER_OUTCFG2_CFG25_A6OUT = 4, /*!< A6OUT : Output is A6OUT. value. */ + CTIMER_OUTCFG2_CFG25_B2OUT = 3, /*!< B2OUT : Output is B2OUT. value. */ + CTIMER_OUTCFG2_CFG25_B4OUT2 = 2, /*!< B4OUT2 : Output is B4OUT2 value. */ + CTIMER_OUTCFG2_CFG25_ONE = 1, /*!< ONE : Force output to 1. value. */ + CTIMER_OUTCFG2_CFG25_ZERO = 0, /*!< ZERO : Force output to 0 value. */ +} CTIMER_OUTCFG2_CFG25_Enum; + +/* ============================================= CTIMER OUTCFG2 CFG24 [12..14] ============================================= */ +typedef enum { /*!< CTIMER_OUTCFG2_CFG24 */ + CTIMER_OUTCFG2_CFG24_A7OUT2 = 7, /*!< A7OUT2 : Output is A7OUT2. value. */ + CTIMER_OUTCFG2_CFG24_A6OUT2 = 6, /*!< A6OUT2 : Output is A6OUT2. value. */ + CTIMER_OUTCFG2_CFG24_B1OUT2 = 5, /*!< B1OUT2 : Output is B1OUT2. value. */ + CTIMER_OUTCFG2_CFG24_A1OUT = 4, /*!< A1OUT : Output is A1OUT. value. */ + CTIMER_OUTCFG2_CFG24_A2OUT = 3, /*!< A2OUT : Output is A2OUT. value. */ + CTIMER_OUTCFG2_CFG24_A6OUT = 2, /*!< A6OUT : Output is A6OUT value. */ + CTIMER_OUTCFG2_CFG24_ONE = 1, /*!< ONE : Force output to 1. value. */ + CTIMER_OUTCFG2_CFG24_ZERO = 0, /*!< ZERO : Force output to 0 value. */ +} CTIMER_OUTCFG2_CFG24_Enum; + +/* ============================================= CTIMER OUTCFG2 CFG23 [9..11] ============================================== */ +typedef enum { /*!< CTIMER_OUTCFG2_CFG23 */ + CTIMER_OUTCFG2_CFG23_A7OUT2 = 7, /*!< A7OUT2 : Output is A7OUT2. value. */ + CTIMER_OUTCFG2_CFG23_A6OUT2 = 6, /*!< A6OUT2 : Output is A6OUT2. value. */ + CTIMER_OUTCFG2_CFG23_B0OUT2 = 5, /*!< B0OUT2 : Output is B0OUT2. value. */ + CTIMER_OUTCFG2_CFG23_A5OUT = 4, /*!< A5OUT : Output is A5OUT. value. */ + CTIMER_OUTCFG2_CFG23_A7OUT = 3, /*!< A7OUT : Output is B1OUT. value. */ + CTIMER_OUTCFG2_CFG23_B5OUT2 = 2, /*!< B5OUT2 : Output is B5OUT2 value. */ + CTIMER_OUTCFG2_CFG23_ONE = 1, /*!< ONE : Force output to 1. value. */ + CTIMER_OUTCFG2_CFG23_ZERO = 0, /*!< ZERO : Force output to 0 value. */ +} CTIMER_OUTCFG2_CFG23_Enum; + +/* ============================================== CTIMER OUTCFG2 CFG22 [6..8] ============================================== */ +typedef enum { /*!< CTIMER_OUTCFG2_CFG22 */ + CTIMER_OUTCFG2_CFG22_A7OUT2 = 7, /*!< A7OUT2 : Output is A7OUT2. value. */ + CTIMER_OUTCFG2_CFG22_A6OUT2 = 6, /*!< A6OUT2 : Output is A6OUT2. value. */ + CTIMER_OUTCFG2_CFG22_A2OUT2 = 5, /*!< A2OUT2 : Output is A2OUT2. value. */ + CTIMER_OUTCFG2_CFG22_A1OUT = 4, /*!< A1OUT : Output is A1OUT. value. */ + CTIMER_OUTCFG2_CFG22_A6OUT = 3, /*!< A6OUT : Output is B1OUT. value. */ + CTIMER_OUTCFG2_CFG22_B5OUT = 2, /*!< B5OUT : Output is B5OUT value. */ + CTIMER_OUTCFG2_CFG22_ONE = 1, /*!< ONE : Force output to 1. value. */ + CTIMER_OUTCFG2_CFG22_ZERO = 0, /*!< ZERO : Force output to 0 value. */ +} CTIMER_OUTCFG2_CFG22_Enum; + +/* ============================================== CTIMER OUTCFG2 CFG21 [3..5] ============================================== */ +typedef enum { /*!< CTIMER_OUTCFG2_CFG21 */ + CTIMER_OUTCFG2_CFG21_A7OUT2 = 7, /*!< A7OUT2 : Output is A7OUT2. value. */ + CTIMER_OUTCFG2_CFG21_A6OUT2 = 6, /*!< A6OUT2 : Output is A6OUT2. value. */ + CTIMER_OUTCFG2_CFG21_A0OUT2 = 5, /*!< A0OUT2 : Output is A0OUT2. value. */ + CTIMER_OUTCFG2_CFG21_B5OUT = 4, /*!< B5OUT : Output is B5OUT. value. */ + CTIMER_OUTCFG2_CFG21_A1OUT = 3, /*!< A1OUT : Output is A1OUT. value. */ + CTIMER_OUTCFG2_CFG21_A5OUT2 = 2, /*!< A5OUT2 : Output is A5OUT2 value. */ + CTIMER_OUTCFG2_CFG21_ONE = 1, /*!< ONE : Force output to 1. value. */ + CTIMER_OUTCFG2_CFG21_ZERO = 0, /*!< ZERO : Force output to 0 value. */ +} CTIMER_OUTCFG2_CFG21_Enum; + +/* ============================================== CTIMER OUTCFG2 CFG20 [0..2] ============================================== */ +typedef enum { /*!< CTIMER_OUTCFG2_CFG20 */ + CTIMER_OUTCFG2_CFG20_A7OUT2 = 7, /*!< A7OUT2 : Output is A7OUT2. value. */ + CTIMER_OUTCFG2_CFG20_A6OUT2 = 6, /*!< A6OUT2 : Output is A6OUT2. value. */ + CTIMER_OUTCFG2_CFG20_B2OUT2 = 5, /*!< B2OUT2 : Output is B2OUT2. value. */ + CTIMER_OUTCFG2_CFG20_A1OUT2 = 4, /*!< A1OUT2 : Output is A1OUT2. value. */ + CTIMER_OUTCFG2_CFG20_A1OUT = 3, /*!< A1OUT : Output is A1OUT. value. */ + CTIMER_OUTCFG2_CFG20_A5OUT = 2, /*!< A5OUT : Output is A5OUT value. */ + CTIMER_OUTCFG2_CFG20_ONE = 1, /*!< ONE : Force output to 1. value. */ + CTIMER_OUTCFG2_CFG20_ZERO = 0, /*!< ZERO : Force output to 0 value. */ +} CTIMER_OUTCFG2_CFG20_Enum; + +/* ======================================================== OUTCFG3 ======================================================== */ +/* ============================================== CTIMER OUTCFG3 CFG31 [3..5] ============================================== */ +typedef enum { /*!< CTIMER_OUTCFG3_CFG31 */ + CTIMER_OUTCFG3_CFG31_A7OUT2 = 7, /*!< A7OUT2 : Output is A7OUT2. value. */ + CTIMER_OUTCFG3_CFG31_A6OUT2 = 6, /*!< A6OUT2 : Output is A6OUT2. value. */ + CTIMER_OUTCFG3_CFG31_B3OUT2 = 5, /*!< B3OUT2 : Output is B3OUT2. value. */ + CTIMER_OUTCFG3_CFG31_B7OUT = 4, /*!< B7OUT : Output is B7OUT. value. */ + CTIMER_OUTCFG3_CFG31_A6OUT = 3, /*!< A6OUT : Output is A6OUT. value. */ + CTIMER_OUTCFG3_CFG31_B7OUT2 = 2, /*!< B7OUT2 : Output is B7OUT2 value. */ + CTIMER_OUTCFG3_CFG31_ONE = 1, /*!< ONE : Force output to 1. value. */ + CTIMER_OUTCFG3_CFG31_ZERO = 0, /*!< ZERO : Force output to 0 value. */ +} CTIMER_OUTCFG3_CFG31_Enum; + +/* ============================================== CTIMER OUTCFG3 CFG30 [0..2] ============================================== */ +typedef enum { /*!< CTIMER_OUTCFG3_CFG30 */ + CTIMER_OUTCFG3_CFG30_A7OUT2 = 7, /*!< A7OUT2 : Output is A7OUT2. value. */ + CTIMER_OUTCFG3_CFG30_A6OUT2 = 6, /*!< A6OUT2 : Output is A6OUT2. value. */ + CTIMER_OUTCFG3_CFG30_A0OUT2 = 5, /*!< A0OUT2 : Output is A0OUT2. value. */ + CTIMER_OUTCFG3_CFG30_A4OUT2 = 4, /*!< A4OUT2 : Output is A4OUT2. value. */ + CTIMER_OUTCFG3_CFG30_B3OUT = 3, /*!< B3OUT : Output is B3OUT. value. */ + CTIMER_OUTCFG3_CFG30_B7OUT = 2, /*!< B7OUT : Output is B7OUT value. */ + CTIMER_OUTCFG3_CFG30_ONE = 1, /*!< ONE : Force output to 1. value. */ + CTIMER_OUTCFG3_CFG30_ZERO = 0, /*!< ZERO : Force output to 0 value. */ +} CTIMER_OUTCFG3_CFG30_Enum; + +/* ========================================================= INCFG ========================================================= */ +/* ============================================== CTIMER INCFG CFGB7 [15..15] ============================================== */ +typedef enum { /*!< CTIMER_INCFG_CFGB7 */ + CTIMER_INCFG_CFGB7_CT31 = 1, /*!< CT31 : Input is CT31 value. */ + CTIMER_INCFG_CFGB7_CT30 = 0, /*!< CT30 : Input is CT30 value. */ +} CTIMER_INCFG_CFGB7_Enum; + +/* ============================================== CTIMER INCFG CFGA7 [14..14] ============================================== */ +typedef enum { /*!< CTIMER_INCFG_CFGA7 */ + CTIMER_INCFG_CFGA7_CT29 = 1, /*!< CT29 : Input is CT29 value. */ + CTIMER_INCFG_CFGA7_CT28 = 0, /*!< CT28 : Input is CT28 value. */ +} CTIMER_INCFG_CFGA7_Enum; + +/* ============================================== CTIMER INCFG CFGB6 [13..13] ============================================== */ +typedef enum { /*!< CTIMER_INCFG_CFGB6 */ + CTIMER_INCFG_CFGB6_CT27 = 1, /*!< CT27 : Input is CT27 value. */ + CTIMER_INCFG_CFGB6_CT26 = 0, /*!< CT26 : Input is CT26 value. */ +} CTIMER_INCFG_CFGB6_Enum; + +/* ============================================== CTIMER INCFG CFGA6 [12..12] ============================================== */ +typedef enum { /*!< CTIMER_INCFG_CFGA6 */ + CTIMER_INCFG_CFGA6_CT25 = 1, /*!< CT25 : Input is CT25 value. */ + CTIMER_INCFG_CFGA6_CT24 = 0, /*!< CT24 : Input is CT24 value. */ +} CTIMER_INCFG_CFGA6_Enum; + +/* ============================================== CTIMER INCFG CFGB5 [11..11] ============================================== */ +typedef enum { /*!< CTIMER_INCFG_CFGB5 */ + CTIMER_INCFG_CFGB5_CT23 = 1, /*!< CT23 : Input is CT23 value. */ + CTIMER_INCFG_CFGB5_CT22 = 0, /*!< CT22 : Input is CT22 value. */ +} CTIMER_INCFG_CFGB5_Enum; + +/* ============================================== CTIMER INCFG CFGA5 [10..10] ============================================== */ +typedef enum { /*!< CTIMER_INCFG_CFGA5 */ + CTIMER_INCFG_CFGA5_CT21 = 1, /*!< CT21 : Input is CT21 value. */ + CTIMER_INCFG_CFGA5_CT20 = 0, /*!< CT20 : Input is CT20 value. */ +} CTIMER_INCFG_CFGA5_Enum; + +/* =============================================== CTIMER INCFG CFGB4 [9..9] =============================================== */ +typedef enum { /*!< CTIMER_INCFG_CFGB4 */ + CTIMER_INCFG_CFGB4_CT19 = 1, /*!< CT19 : Input is CT19 value. */ + CTIMER_INCFG_CFGB4_CT18 = 0, /*!< CT18 : Input is CT18 value. */ +} CTIMER_INCFG_CFGB4_Enum; + +/* =============================================== CTIMER INCFG CFGA4 [8..8] =============================================== */ +typedef enum { /*!< CTIMER_INCFG_CFGA4 */ + CTIMER_INCFG_CFGA4_CT17 = 1, /*!< CT17 : Input is CT17 value. */ + CTIMER_INCFG_CFGA4_CT16 = 0, /*!< CT16 : Input is CT16 value. */ +} CTIMER_INCFG_CFGA4_Enum; + +/* =============================================== CTIMER INCFG CFGB3 [7..7] =============================================== */ +typedef enum { /*!< CTIMER_INCFG_CFGB3 */ + CTIMER_INCFG_CFGB3_CT15 = 1, /*!< CT15 : Input is CT15 value. */ + CTIMER_INCFG_CFGB3_CT14 = 0, /*!< CT14 : Input is CT14 value. */ +} CTIMER_INCFG_CFGB3_Enum; + +/* =============================================== CTIMER INCFG CFGA3 [6..6] =============================================== */ +typedef enum { /*!< CTIMER_INCFG_CFGA3 */ + CTIMER_INCFG_CFGA3_CT13 = 1, /*!< CT13 : Input is CT13 value. */ + CTIMER_INCFG_CFGA3_CT12 = 0, /*!< CT12 : Input is CT12 value. */ +} CTIMER_INCFG_CFGA3_Enum; + +/* =============================================== CTIMER INCFG CFGB2 [5..5] =============================================== */ +typedef enum { /*!< CTIMER_INCFG_CFGB2 */ + CTIMER_INCFG_CFGB2_CT11 = 1, /*!< CT11 : Input is CT11 value. */ + CTIMER_INCFG_CFGB2_CT10 = 0, /*!< CT10 : Input is CT10 value. */ +} CTIMER_INCFG_CFGB2_Enum; + +/* =============================================== CTIMER INCFG CFGA2 [4..4] =============================================== */ +typedef enum { /*!< CTIMER_INCFG_CFGA2 */ + CTIMER_INCFG_CFGA2_CT9 = 1, /*!< CT9 : Input is CT9 value. */ + CTIMER_INCFG_CFGA2_CT8 = 0, /*!< CT8 : Input is CT8 value. */ +} CTIMER_INCFG_CFGA2_Enum; + +/* =============================================== CTIMER INCFG CFGB1 [3..3] =============================================== */ +typedef enum { /*!< CTIMER_INCFG_CFGB1 */ + CTIMER_INCFG_CFGB1_CT7 = 1, /*!< CT7 : Input is CT7 value. */ + CTIMER_INCFG_CFGB1_CT6 = 0, /*!< CT6 : Input is CT6 value. */ +} CTIMER_INCFG_CFGB1_Enum; + +/* =============================================== CTIMER INCFG CFGA1 [2..2] =============================================== */ +typedef enum { /*!< CTIMER_INCFG_CFGA1 */ + CTIMER_INCFG_CFGA1_CT5 = 1, /*!< CT5 : Input is CT5 value. */ + CTIMER_INCFG_CFGA1_CT4 = 0, /*!< CT4 : Input is CT4 value. */ +} CTIMER_INCFG_CFGA1_Enum; + +/* =============================================== CTIMER INCFG CFGB0 [1..1] =============================================== */ +typedef enum { /*!< CTIMER_INCFG_CFGB0 */ + CTIMER_INCFG_CFGB0_CT3 = 1, /*!< CT3 : Input is CT3 value. */ + CTIMER_INCFG_CFGB0_CT2 = 0, /*!< CT2 : Input is CT2 value. */ +} CTIMER_INCFG_CFGB0_Enum; + +/* =============================================== CTIMER INCFG CFGA0 [0..0] =============================================== */ +typedef enum { /*!< CTIMER_INCFG_CFGA0 */ + CTIMER_INCFG_CFGA0_CT1 = 1, /*!< CT1 : Input is CT1 value. */ + CTIMER_INCFG_CFGA0_CT0 = 0, /*!< CT0 : Input is CT0 value. */ +} CTIMER_INCFG_CFGA0_Enum; + +/* ========================================================= STCFG ========================================================= */ +/* ============================================= CTIMER STCFG FREEZE [31..31] ============================================== */ +typedef enum { /*!< CTIMER_STCFG_FREEZE */ + CTIMER_STCFG_FREEZE_THAW = 0, /*!< THAW : Let the COUNTER register run on its input clock. value. */ + CTIMER_STCFG_FREEZE_FREEZE = 1, /*!< FREEZE : Stop the COUNTER register for loading. value. */ +} CTIMER_STCFG_FREEZE_Enum; + +/* ============================================== CTIMER STCFG CLEAR [30..30] ============================================== */ +typedef enum { /*!< CTIMER_STCFG_CLEAR */ + CTIMER_STCFG_CLEAR_RUN = 0, /*!< RUN : Let the COUNTER register run on its input clock. value. */ + CTIMER_STCFG_CLEAR_CLEAR = 1, /*!< CLEAR : Stop the COUNTER register for loading. value. */ +} CTIMER_STCFG_CLEAR_Enum; + +/* ========================================== CTIMER STCFG COMPARE_H_EN [15..15] =========================================== */ +typedef enum { /*!< CTIMER_STCFG_COMPARE_H_EN */ + CTIMER_STCFG_COMPARE_H_EN_DISABLE = 0, /*!< DISABLE : Compare H disabled. value. */ + CTIMER_STCFG_COMPARE_H_EN_ENABLE = 1, /*!< ENABLE : Compare H enabled. value. */ +} CTIMER_STCFG_COMPARE_H_EN_Enum; + +/* ========================================== CTIMER STCFG COMPARE_G_EN [14..14] =========================================== */ +typedef enum { /*!< CTIMER_STCFG_COMPARE_G_EN */ + CTIMER_STCFG_COMPARE_G_EN_DISABLE = 0, /*!< DISABLE : Compare G disabled. value. */ + CTIMER_STCFG_COMPARE_G_EN_ENABLE = 1, /*!< ENABLE : Compare G enabled. value. */ +} CTIMER_STCFG_COMPARE_G_EN_Enum; + +/* ========================================== CTIMER STCFG COMPARE_F_EN [13..13] =========================================== */ +typedef enum { /*!< CTIMER_STCFG_COMPARE_F_EN */ + CTIMER_STCFG_COMPARE_F_EN_DISABLE = 0, /*!< DISABLE : Compare F disabled. value. */ + CTIMER_STCFG_COMPARE_F_EN_ENABLE = 1, /*!< ENABLE : Compare F enabled. value. */ +} CTIMER_STCFG_COMPARE_F_EN_Enum; + +/* ========================================== CTIMER STCFG COMPARE_E_EN [12..12] =========================================== */ +typedef enum { /*!< CTIMER_STCFG_COMPARE_E_EN */ + CTIMER_STCFG_COMPARE_E_EN_DISABLE = 0, /*!< DISABLE : Compare E disabled. value. */ + CTIMER_STCFG_COMPARE_E_EN_ENABLE = 1, /*!< ENABLE : Compare E enabled. value. */ +} CTIMER_STCFG_COMPARE_E_EN_Enum; + +/* ========================================== CTIMER STCFG COMPARE_D_EN [11..11] =========================================== */ +typedef enum { /*!< CTIMER_STCFG_COMPARE_D_EN */ + CTIMER_STCFG_COMPARE_D_EN_DISABLE = 0, /*!< DISABLE : Compare D disabled. value. */ + CTIMER_STCFG_COMPARE_D_EN_ENABLE = 1, /*!< ENABLE : Compare D enabled. value. */ +} CTIMER_STCFG_COMPARE_D_EN_Enum; + +/* ========================================== CTIMER STCFG COMPARE_C_EN [10..10] =========================================== */ +typedef enum { /*!< CTIMER_STCFG_COMPARE_C_EN */ + CTIMER_STCFG_COMPARE_C_EN_DISABLE = 0, /*!< DISABLE : Compare C disabled. value. */ + CTIMER_STCFG_COMPARE_C_EN_ENABLE = 1, /*!< ENABLE : Compare C enabled. value. */ +} CTIMER_STCFG_COMPARE_C_EN_Enum; + +/* =========================================== CTIMER STCFG COMPARE_B_EN [9..9] ============================================ */ +typedef enum { /*!< CTIMER_STCFG_COMPARE_B_EN */ + CTIMER_STCFG_COMPARE_B_EN_DISABLE = 0, /*!< DISABLE : Compare B disabled. value. */ + CTIMER_STCFG_COMPARE_B_EN_ENABLE = 1, /*!< ENABLE : Compare B enabled. value. */ +} CTIMER_STCFG_COMPARE_B_EN_Enum; + +/* =========================================== CTIMER STCFG COMPARE_A_EN [8..8] ============================================ */ +typedef enum { /*!< CTIMER_STCFG_COMPARE_A_EN */ + CTIMER_STCFG_COMPARE_A_EN_DISABLE = 0, /*!< DISABLE : Compare A disabled. value. */ + CTIMER_STCFG_COMPARE_A_EN_ENABLE = 1, /*!< ENABLE : Compare A enabled. value. */ +} CTIMER_STCFG_COMPARE_A_EN_Enum; + +/* ============================================== CTIMER STCFG CLKSEL [0..3] =============================================== */ +typedef enum { /*!< CTIMER_STCFG_CLKSEL */ + CTIMER_STCFG_CLKSEL_NOCLK = 0, /*!< NOCLK : No clock enabled. value. */ + CTIMER_STCFG_CLKSEL_HFRC_DIV16 = 1, /*!< HFRC_DIV16 : 3MHz from the HFRC clock divider. value. */ + CTIMER_STCFG_CLKSEL_HFRC_DIV256 = 2, /*!< HFRC_DIV256 : 187.5KHz from the HFRC clock divider. value. */ + CTIMER_STCFG_CLKSEL_XTAL_DIV1 = 3, /*!< XTAL_DIV1 : 32768Hz from the crystal oscillator. value. */ + CTIMER_STCFG_CLKSEL_XTAL_DIV2 = 4, /*!< XTAL_DIV2 : 16384Hz from the crystal oscillator. value. */ + CTIMER_STCFG_CLKSEL_XTAL_DIV32 = 5, /*!< XTAL_DIV32 : 1024Hz from the crystal oscillator. value. */ + CTIMER_STCFG_CLKSEL_LFRC_DIV1 = 6, /*!< LFRC_DIV1 : Approximately 1KHz from the LFRC oscillator (uncalibrated). + value. */ + CTIMER_STCFG_CLKSEL_CTIMER0A = 7, /*!< CTIMER0A : Use CTIMER 0 section A as a prescaler for the clock + source. value. */ + CTIMER_STCFG_CLKSEL_CTIMER0B = 8, /*!< CTIMER0B : Use CTIMER 0 section B (or A and B linked together) + as a prescaler for the clock source. value. */ +} CTIMER_STCFG_CLKSEL_Enum; + +/* ========================================================= STTMR ========================================================= */ +/* ==================================================== CAPTURECONTROL ===================================================== */ +/* ========================================= CTIMER CAPTURECONTROL CAPTURE3 [3..3] ========================================= */ +typedef enum { /*!< CTIMER_CAPTURECONTROL_CAPTURE3 */ + CTIMER_CAPTURECONTROL_CAPTURE3_DISABLE = 0, /*!< DISABLE : Capture function disabled. value. */ + CTIMER_CAPTURECONTROL_CAPTURE3_ENABLE = 1, /*!< ENABLE : Capture function enabled. value. */ +} CTIMER_CAPTURECONTROL_CAPTURE3_Enum; + +/* ========================================= CTIMER CAPTURECONTROL CAPTURE2 [2..2] ========================================= */ +typedef enum { /*!< CTIMER_CAPTURECONTROL_CAPTURE2 */ + CTIMER_CAPTURECONTROL_CAPTURE2_DISABLE = 0, /*!< DISABLE : Capture function disabled. value. */ + CTIMER_CAPTURECONTROL_CAPTURE2_ENABLE = 1, /*!< ENABLE : Capture function enabled. value. */ +} CTIMER_CAPTURECONTROL_CAPTURE2_Enum; + +/* ========================================= CTIMER CAPTURECONTROL CAPTURE1 [1..1] ========================================= */ +typedef enum { /*!< CTIMER_CAPTURECONTROL_CAPTURE1 */ + CTIMER_CAPTURECONTROL_CAPTURE1_DISABLE = 0, /*!< DISABLE : Capture function disabled. value. */ + CTIMER_CAPTURECONTROL_CAPTURE1_ENABLE = 1, /*!< ENABLE : Capture function enabled. value. */ +} CTIMER_CAPTURECONTROL_CAPTURE1_Enum; + +/* ========================================= CTIMER CAPTURECONTROL CAPTURE0 [0..0] ========================================= */ +typedef enum { /*!< CTIMER_CAPTURECONTROL_CAPTURE0 */ + CTIMER_CAPTURECONTROL_CAPTURE0_DISABLE = 0, /*!< DISABLE : Capture function disabled. value. */ + CTIMER_CAPTURECONTROL_CAPTURE0_ENABLE = 1, /*!< ENABLE : Capture function enabled. value. */ +} CTIMER_CAPTURECONTROL_CAPTURE0_Enum; + +/* ======================================================== SCMPR0 ========================================================= */ +/* ======================================================== SCMPR1 ========================================================= */ +/* ======================================================== SCMPR2 ========================================================= */ +/* ======================================================== SCMPR3 ========================================================= */ +/* ======================================================== SCMPR4 ========================================================= */ +/* ======================================================== SCMPR5 ========================================================= */ +/* ======================================================== SCMPR6 ========================================================= */ +/* ======================================================== SCMPR7 ========================================================= */ +/* ======================================================== SCAPT0 ========================================================= */ +/* ======================================================== SCAPT1 ========================================================= */ +/* ======================================================== SCAPT2 ========================================================= */ +/* ======================================================== SCAPT3 ========================================================= */ +/* ========================================================= SNVR0 ========================================================= */ +/* ========================================================= SNVR1 ========================================================= */ +/* ========================================================= SNVR2 ========================================================= */ +/* ========================================================= SNVR3 ========================================================= */ +/* ========================================================= INTEN ========================================================= */ +/* ======================================================== INTSTAT ======================================================== */ +/* ======================================================== INTCLR ========================================================= */ +/* ======================================================== INTSET ========================================================= */ +/* ======================================================= STMINTEN ======================================================== */ +/* =========================================== CTIMER STMINTEN CAPTURED [12..12] =========================================== */ +typedef enum { /*!< CTIMER_STMINTEN_CAPTURED */ + CTIMER_STMINTEN_CAPTURED_CAPD_INT = 1, /*!< CAPD_INT : Capture D interrupt status bit was set. value. */ +} CTIMER_STMINTEN_CAPTURED_Enum; + +/* =========================================== CTIMER STMINTEN CAPTUREC [11..11] =========================================== */ +typedef enum { /*!< CTIMER_STMINTEN_CAPTUREC */ + CTIMER_STMINTEN_CAPTUREC_CAPC_INT = 1, /*!< CAPC_INT : CAPTURE C interrupt status bit was set. value. */ +} CTIMER_STMINTEN_CAPTUREC_Enum; + +/* =========================================== CTIMER STMINTEN CAPTUREB [10..10] =========================================== */ +typedef enum { /*!< CTIMER_STMINTEN_CAPTUREB */ + CTIMER_STMINTEN_CAPTUREB_CAPB_INT = 1, /*!< CAPB_INT : CAPTURE B interrupt status bit was set. value. */ +} CTIMER_STMINTEN_CAPTUREB_Enum; + +/* ============================================ CTIMER STMINTEN CAPTUREA [9..9] ============================================ */ +typedef enum { /*!< CTIMER_STMINTEN_CAPTUREA */ + CTIMER_STMINTEN_CAPTUREA_CAPA_INT = 1, /*!< CAPA_INT : CAPTURE A interrupt status bit was set. value. */ +} CTIMER_STMINTEN_CAPTUREA_Enum; + +/* ============================================ CTIMER STMINTEN OVERFLOW [8..8] ============================================ */ +typedef enum { /*!< CTIMER_STMINTEN_OVERFLOW */ + CTIMER_STMINTEN_OVERFLOW_OFLOW_INT = 1, /*!< OFLOW_INT : Overflow interrupt status bit was set. value. */ +} CTIMER_STMINTEN_OVERFLOW_Enum; + +/* ============================================ CTIMER STMINTEN COMPAREH [7..7] ============================================ */ +typedef enum { /*!< CTIMER_STMINTEN_COMPAREH */ + CTIMER_STMINTEN_COMPAREH_COMPARED = 1, /*!< COMPARED : COUNTER greater than or equal to COMPARE register. + value. */ +} CTIMER_STMINTEN_COMPAREH_Enum; + +/* ============================================ CTIMER STMINTEN COMPAREG [6..6] ============================================ */ +typedef enum { /*!< CTIMER_STMINTEN_COMPAREG */ + CTIMER_STMINTEN_COMPAREG_COMPARED = 1, /*!< COMPARED : COUNTER greater than or equal to COMPARE register. + value. */ +} CTIMER_STMINTEN_COMPAREG_Enum; + +/* ============================================ CTIMER STMINTEN COMPAREF [5..5] ============================================ */ +typedef enum { /*!< CTIMER_STMINTEN_COMPAREF */ + CTIMER_STMINTEN_COMPAREF_COMPARED = 1, /*!< COMPARED : COUNTER greater than or equal to COMPARE register. + value. */ +} CTIMER_STMINTEN_COMPAREF_Enum; + +/* ============================================ CTIMER STMINTEN COMPAREE [4..4] ============================================ */ +typedef enum { /*!< CTIMER_STMINTEN_COMPAREE */ + CTIMER_STMINTEN_COMPAREE_COMPARED = 1, /*!< COMPARED : COUNTER greater than or equal to COMPARE register. + value. */ +} CTIMER_STMINTEN_COMPAREE_Enum; + +/* ============================================ CTIMER STMINTEN COMPARED [3..3] ============================================ */ +typedef enum { /*!< CTIMER_STMINTEN_COMPARED */ + CTIMER_STMINTEN_COMPARED_COMPARED = 1, /*!< COMPARED : COUNTER greater than or equal to COMPARE register. + value. */ +} CTIMER_STMINTEN_COMPARED_Enum; + +/* ============================================ CTIMER STMINTEN COMPAREC [2..2] ============================================ */ +typedef enum { /*!< CTIMER_STMINTEN_COMPAREC */ + CTIMER_STMINTEN_COMPAREC_COMPARED = 1, /*!< COMPARED : COUNTER greater than or equal to COMPARE register. + value. */ +} CTIMER_STMINTEN_COMPAREC_Enum; + +/* ============================================ CTIMER STMINTEN COMPAREB [1..1] ============================================ */ +typedef enum { /*!< CTIMER_STMINTEN_COMPAREB */ + CTIMER_STMINTEN_COMPAREB_COMPARED = 1, /*!< COMPARED : COUNTER greater than or equal to COMPARE register. + value. */ +} CTIMER_STMINTEN_COMPAREB_Enum; + +/* ============================================ CTIMER STMINTEN COMPAREA [0..0] ============================================ */ +typedef enum { /*!< CTIMER_STMINTEN_COMPAREA */ + CTIMER_STMINTEN_COMPAREA_COMPARED = 1, /*!< COMPARED : COUNTER greater than or equal to COMPARE register. + value. */ +} CTIMER_STMINTEN_COMPAREA_Enum; + +/* ====================================================== STMINTSTAT ======================================================= */ +/* ========================================== CTIMER STMINTSTAT CAPTURED [12..12] ========================================== */ +typedef enum { /*!< CTIMER_STMINTSTAT_CAPTURED */ + CTIMER_STMINTSTAT_CAPTURED_CAPD_INT = 1, /*!< CAPD_INT : Capture D interrupt status bit was set. value. */ +} CTIMER_STMINTSTAT_CAPTURED_Enum; + +/* ========================================== CTIMER STMINTSTAT CAPTUREC [11..11] ========================================== */ +typedef enum { /*!< CTIMER_STMINTSTAT_CAPTUREC */ + CTIMER_STMINTSTAT_CAPTUREC_CAPC_INT = 1, /*!< CAPC_INT : CAPTURE C interrupt status bit was set. value. */ +} CTIMER_STMINTSTAT_CAPTUREC_Enum; + +/* ========================================== CTIMER STMINTSTAT CAPTUREB [10..10] ========================================== */ +typedef enum { /*!< CTIMER_STMINTSTAT_CAPTUREB */ + CTIMER_STMINTSTAT_CAPTUREB_CAPB_INT = 1, /*!< CAPB_INT : CAPTURE B interrupt status bit was set. value. */ +} CTIMER_STMINTSTAT_CAPTUREB_Enum; + +/* =========================================== CTIMER STMINTSTAT CAPTUREA [9..9] =========================================== */ +typedef enum { /*!< CTIMER_STMINTSTAT_CAPTUREA */ + CTIMER_STMINTSTAT_CAPTUREA_CAPA_INT = 1, /*!< CAPA_INT : CAPTURE A interrupt status bit was set. value. */ +} CTIMER_STMINTSTAT_CAPTUREA_Enum; + +/* =========================================== CTIMER STMINTSTAT OVERFLOW [8..8] =========================================== */ +typedef enum { /*!< CTIMER_STMINTSTAT_OVERFLOW */ + CTIMER_STMINTSTAT_OVERFLOW_OFLOW_INT = 1, /*!< OFLOW_INT : Overflow interrupt status bit was set. value. */ +} CTIMER_STMINTSTAT_OVERFLOW_Enum; + +/* =========================================== CTIMER STMINTSTAT COMPAREH [7..7] =========================================== */ +typedef enum { /*!< CTIMER_STMINTSTAT_COMPAREH */ + CTIMER_STMINTSTAT_COMPAREH_COMPARED = 1, /*!< COMPARED : COUNTER greater than or equal to COMPARE register. + value. */ +} CTIMER_STMINTSTAT_COMPAREH_Enum; + +/* =========================================== CTIMER STMINTSTAT COMPAREG [6..6] =========================================== */ +typedef enum { /*!< CTIMER_STMINTSTAT_COMPAREG */ + CTIMER_STMINTSTAT_COMPAREG_COMPARED = 1, /*!< COMPARED : COUNTER greater than or equal to COMPARE register. + value. */ +} CTIMER_STMINTSTAT_COMPAREG_Enum; + +/* =========================================== CTIMER STMINTSTAT COMPAREF [5..5] =========================================== */ +typedef enum { /*!< CTIMER_STMINTSTAT_COMPAREF */ + CTIMER_STMINTSTAT_COMPAREF_COMPARED = 1, /*!< COMPARED : COUNTER greater than or equal to COMPARE register. + value. */ +} CTIMER_STMINTSTAT_COMPAREF_Enum; + +/* =========================================== CTIMER STMINTSTAT COMPAREE [4..4] =========================================== */ +typedef enum { /*!< CTIMER_STMINTSTAT_COMPAREE */ + CTIMER_STMINTSTAT_COMPAREE_COMPARED = 1, /*!< COMPARED : COUNTER greater than or equal to COMPARE register. + value. */ +} CTIMER_STMINTSTAT_COMPAREE_Enum; + +/* =========================================== CTIMER STMINTSTAT COMPARED [3..3] =========================================== */ +typedef enum { /*!< CTIMER_STMINTSTAT_COMPARED */ + CTIMER_STMINTSTAT_COMPARED_COMPARED = 1, /*!< COMPARED : COUNTER greater than or equal to COMPARE register. + value. */ +} CTIMER_STMINTSTAT_COMPARED_Enum; + +/* =========================================== CTIMER STMINTSTAT COMPAREC [2..2] =========================================== */ +typedef enum { /*!< CTIMER_STMINTSTAT_COMPAREC */ + CTIMER_STMINTSTAT_COMPAREC_COMPARED = 1, /*!< COMPARED : COUNTER greater than or equal to COMPARE register. + value. */ +} CTIMER_STMINTSTAT_COMPAREC_Enum; + +/* =========================================== CTIMER STMINTSTAT COMPAREB [1..1] =========================================== */ +typedef enum { /*!< CTIMER_STMINTSTAT_COMPAREB */ + CTIMER_STMINTSTAT_COMPAREB_COMPARED = 1, /*!< COMPARED : COUNTER greater than or equal to COMPARE register. + value. */ +} CTIMER_STMINTSTAT_COMPAREB_Enum; + +/* =========================================== CTIMER STMINTSTAT COMPAREA [0..0] =========================================== */ +typedef enum { /*!< CTIMER_STMINTSTAT_COMPAREA */ + CTIMER_STMINTSTAT_COMPAREA_COMPARED = 1, /*!< COMPARED : COUNTER greater than or equal to COMPARE register. + value. */ +} CTIMER_STMINTSTAT_COMPAREA_Enum; + +/* ======================================================= STMINTCLR ======================================================= */ +/* ========================================== CTIMER STMINTCLR CAPTURED [12..12] =========================================== */ +typedef enum { /*!< CTIMER_STMINTCLR_CAPTURED */ + CTIMER_STMINTCLR_CAPTURED_CAPD_INT = 1, /*!< CAPD_INT : Capture D interrupt status bit was set. value. */ +} CTIMER_STMINTCLR_CAPTURED_Enum; + +/* ========================================== CTIMER STMINTCLR CAPTUREC [11..11] =========================================== */ +typedef enum { /*!< CTIMER_STMINTCLR_CAPTUREC */ + CTIMER_STMINTCLR_CAPTUREC_CAPC_INT = 1, /*!< CAPC_INT : CAPTURE C interrupt status bit was set. value. */ +} CTIMER_STMINTCLR_CAPTUREC_Enum; + +/* ========================================== CTIMER STMINTCLR CAPTUREB [10..10] =========================================== */ +typedef enum { /*!< CTIMER_STMINTCLR_CAPTUREB */ + CTIMER_STMINTCLR_CAPTUREB_CAPB_INT = 1, /*!< CAPB_INT : CAPTURE B interrupt status bit was set. value. */ +} CTIMER_STMINTCLR_CAPTUREB_Enum; + +/* =========================================== CTIMER STMINTCLR CAPTUREA [9..9] ============================================ */ +typedef enum { /*!< CTIMER_STMINTCLR_CAPTUREA */ + CTIMER_STMINTCLR_CAPTUREA_CAPA_INT = 1, /*!< CAPA_INT : CAPTURE A interrupt status bit was set. value. */ +} CTIMER_STMINTCLR_CAPTUREA_Enum; + +/* =========================================== CTIMER STMINTCLR OVERFLOW [8..8] ============================================ */ +typedef enum { /*!< CTIMER_STMINTCLR_OVERFLOW */ + CTIMER_STMINTCLR_OVERFLOW_OFLOW_INT = 1, /*!< OFLOW_INT : Overflow interrupt status bit was set. value. */ +} CTIMER_STMINTCLR_OVERFLOW_Enum; + +/* =========================================== CTIMER STMINTCLR COMPAREH [7..7] ============================================ */ +typedef enum { /*!< CTIMER_STMINTCLR_COMPAREH */ + CTIMER_STMINTCLR_COMPAREH_COMPARED = 1, /*!< COMPARED : COUNTER greater than or equal to COMPARE register. + value. */ +} CTIMER_STMINTCLR_COMPAREH_Enum; + +/* =========================================== CTIMER STMINTCLR COMPAREG [6..6] ============================================ */ +typedef enum { /*!< CTIMER_STMINTCLR_COMPAREG */ + CTIMER_STMINTCLR_COMPAREG_COMPARED = 1, /*!< COMPARED : COUNTER greater than or equal to COMPARE register. + value. */ +} CTIMER_STMINTCLR_COMPAREG_Enum; + +/* =========================================== CTIMER STMINTCLR COMPAREF [5..5] ============================================ */ +typedef enum { /*!< CTIMER_STMINTCLR_COMPAREF */ + CTIMER_STMINTCLR_COMPAREF_COMPARED = 1, /*!< COMPARED : COUNTER greater than or equal to COMPARE register. + value. */ +} CTIMER_STMINTCLR_COMPAREF_Enum; + +/* =========================================== CTIMER STMINTCLR COMPAREE [4..4] ============================================ */ +typedef enum { /*!< CTIMER_STMINTCLR_COMPAREE */ + CTIMER_STMINTCLR_COMPAREE_COMPARED = 1, /*!< COMPARED : COUNTER greater than or equal to COMPARE register. + value. */ +} CTIMER_STMINTCLR_COMPAREE_Enum; + +/* =========================================== CTIMER STMINTCLR COMPARED [3..3] ============================================ */ +typedef enum { /*!< CTIMER_STMINTCLR_COMPARED */ + CTIMER_STMINTCLR_COMPARED_COMPARED = 1, /*!< COMPARED : COUNTER greater than or equal to COMPARE register. + value. */ +} CTIMER_STMINTCLR_COMPARED_Enum; + +/* =========================================== CTIMER STMINTCLR COMPAREC [2..2] ============================================ */ +typedef enum { /*!< CTIMER_STMINTCLR_COMPAREC */ + CTIMER_STMINTCLR_COMPAREC_COMPARED = 1, /*!< COMPARED : COUNTER greater than or equal to COMPARE register. + value. */ +} CTIMER_STMINTCLR_COMPAREC_Enum; + +/* =========================================== CTIMER STMINTCLR COMPAREB [1..1] ============================================ */ +typedef enum { /*!< CTIMER_STMINTCLR_COMPAREB */ + CTIMER_STMINTCLR_COMPAREB_COMPARED = 1, /*!< COMPARED : COUNTER greater than or equal to COMPARE register. + value. */ +} CTIMER_STMINTCLR_COMPAREB_Enum; + +/* =========================================== CTIMER STMINTCLR COMPAREA [0..0] ============================================ */ +typedef enum { /*!< CTIMER_STMINTCLR_COMPAREA */ + CTIMER_STMINTCLR_COMPAREA_COMPARED = 1, /*!< COMPARED : COUNTER greater than or equal to COMPARE register. + value. */ +} CTIMER_STMINTCLR_COMPAREA_Enum; + +/* ======================================================= STMINTSET ======================================================= */ +/* ========================================== CTIMER STMINTSET CAPTURED [12..12] =========================================== */ +typedef enum { /*!< CTIMER_STMINTSET_CAPTURED */ + CTIMER_STMINTSET_CAPTURED_CAPD_INT = 1, /*!< CAPD_INT : Capture D interrupt status bit was set. value. */ +} CTIMER_STMINTSET_CAPTURED_Enum; + +/* ========================================== CTIMER STMINTSET CAPTUREC [11..11] =========================================== */ +typedef enum { /*!< CTIMER_STMINTSET_CAPTUREC */ + CTIMER_STMINTSET_CAPTUREC_CAPC_INT = 1, /*!< CAPC_INT : CAPTURE C interrupt status bit was set. value. */ +} CTIMER_STMINTSET_CAPTUREC_Enum; + +/* ========================================== CTIMER STMINTSET CAPTUREB [10..10] =========================================== */ +typedef enum { /*!< CTIMER_STMINTSET_CAPTUREB */ + CTIMER_STMINTSET_CAPTUREB_CAPB_INT = 1, /*!< CAPB_INT : CAPTURE B interrupt status bit was set. value. */ +} CTIMER_STMINTSET_CAPTUREB_Enum; + +/* =========================================== CTIMER STMINTSET CAPTUREA [9..9] ============================================ */ +typedef enum { /*!< CTIMER_STMINTSET_CAPTUREA */ + CTIMER_STMINTSET_CAPTUREA_CAPA_INT = 1, /*!< CAPA_INT : CAPTURE A interrupt status bit was set. value. */ +} CTIMER_STMINTSET_CAPTUREA_Enum; + +/* =========================================== CTIMER STMINTSET OVERFLOW [8..8] ============================================ */ +typedef enum { /*!< CTIMER_STMINTSET_OVERFLOW */ + CTIMER_STMINTSET_OVERFLOW_OFLOW_INT = 1, /*!< OFLOW_INT : Overflow interrupt status bit was set. value. */ +} CTIMER_STMINTSET_OVERFLOW_Enum; + +/* =========================================== CTIMER STMINTSET COMPAREH [7..7] ============================================ */ +typedef enum { /*!< CTIMER_STMINTSET_COMPAREH */ + CTIMER_STMINTSET_COMPAREH_COMPARED = 1, /*!< COMPARED : COUNTER greater than or equal to COMPARE register. + value. */ +} CTIMER_STMINTSET_COMPAREH_Enum; + +/* =========================================== CTIMER STMINTSET COMPAREG [6..6] ============================================ */ +typedef enum { /*!< CTIMER_STMINTSET_COMPAREG */ + CTIMER_STMINTSET_COMPAREG_COMPARED = 1, /*!< COMPARED : COUNTER greater than or equal to COMPARE register. + value. */ +} CTIMER_STMINTSET_COMPAREG_Enum; + +/* =========================================== CTIMER STMINTSET COMPAREF [5..5] ============================================ */ +typedef enum { /*!< CTIMER_STMINTSET_COMPAREF */ + CTIMER_STMINTSET_COMPAREF_COMPARED = 1, /*!< COMPARED : COUNTER greater than or equal to COMPARE register. + value. */ +} CTIMER_STMINTSET_COMPAREF_Enum; + +/* =========================================== CTIMER STMINTSET COMPAREE [4..4] ============================================ */ +typedef enum { /*!< CTIMER_STMINTSET_COMPAREE */ + CTIMER_STMINTSET_COMPAREE_COMPARED = 1, /*!< COMPARED : COUNTER greater than or equal to COMPARE register. + value. */ +} CTIMER_STMINTSET_COMPAREE_Enum; + +/* =========================================== CTIMER STMINTSET COMPARED [3..3] ============================================ */ +typedef enum { /*!< CTIMER_STMINTSET_COMPARED */ + CTIMER_STMINTSET_COMPARED_COMPARED = 1, /*!< COMPARED : COUNTER greater than or equal to COMPARE register. + value. */ +} CTIMER_STMINTSET_COMPARED_Enum; + +/* =========================================== CTIMER STMINTSET COMPAREC [2..2] ============================================ */ +typedef enum { /*!< CTIMER_STMINTSET_COMPAREC */ + CTIMER_STMINTSET_COMPAREC_COMPARED = 1, /*!< COMPARED : COUNTER greater than or equal to COMPARE register. + value. */ +} CTIMER_STMINTSET_COMPAREC_Enum; + +/* =========================================== CTIMER STMINTSET COMPAREB [1..1] ============================================ */ +typedef enum { /*!< CTIMER_STMINTSET_COMPAREB */ + CTIMER_STMINTSET_COMPAREB_COMPARED = 1, /*!< COMPARED : COUNTER greater than or equal to COMPARE register. + value. */ +} CTIMER_STMINTSET_COMPAREB_Enum; + +/* =========================================== CTIMER STMINTSET COMPAREA [0..0] ============================================ */ +typedef enum { /*!< CTIMER_STMINTSET_COMPAREA */ + CTIMER_STMINTSET_COMPAREA_COMPARED = 1, /*!< COMPARED : COUNTER greater than or equal to COMPARE register. + value. */ +} CTIMER_STMINTSET_COMPAREA_Enum; + + + +/* =========================================================================================================================== */ +/* ================ GPIO ================ */ +/* =========================================================================================================================== */ + +/* ======================================================== PADREGA ======================================================== */ +/* ============================================ GPIO PADREGA PAD3PWRUP [30..30] ============================================ */ +typedef enum { /*!< GPIO_PADREGA_PAD3PWRUP */ + GPIO_PADREGA_PAD3PWRUP_DIS = 0, /*!< DIS : Power switch disabled value. */ + GPIO_PADREGA_PAD3PWRUP_EN = 1, /*!< EN : Power switch enabled (switched to VDD) value. */ +} GPIO_PADREGA_PAD3PWRUP_Enum; + +/* =========================================== GPIO PADREGA PAD3FNCSEL [27..29] ============================================ */ +typedef enum { /*!< GPIO_PADREGA_PAD3FNCSEL */ + GPIO_PADREGA_PAD3FNCSEL_UA0RTS = 0, /*!< UA0RTS : Configure as the UART0 RTS output value. */ + GPIO_PADREGA_PAD3FNCSEL_SLnCE = 1, /*!< SLnCE : Configure as the IOSLAVE SPI nCE signal value. */ + GPIO_PADREGA_PAD3FNCSEL_NCE3 = 2, /*!< NCE3 : IOM/MSPI nCE group 3 value. */ + GPIO_PADREGA_PAD3FNCSEL_GPIO3 = 3, /*!< GPIO3 : Configure as GPIO3 value. */ + GPIO_PADREGA_PAD3FNCSEL_MSPI7 = 5, /*!< MSPI7 : MSPI data connection 7 value. */ + GPIO_PADREGA_PAD3FNCSEL_TRIG1 = 6, /*!< TRIG1 : Configure as the ADC Trigger 1 signal value. */ + GPIO_PADREGA_PAD3FNCSEL_I2S_WCLK = 7, /*!< I2S_WCLK : Configure as the PDM I2S Word Clock input value. */ +} GPIO_PADREGA_PAD3FNCSEL_Enum; + +/* ============================================ GPIO PADREGA PAD3STRNG [26..26] ============================================ */ +typedef enum { /*!< GPIO_PADREGA_PAD3STRNG */ + GPIO_PADREGA_PAD3STRNG_LOW = 0, /*!< LOW : Low drive strength value. */ + GPIO_PADREGA_PAD3STRNG_HIGH = 1, /*!< HIGH : High drive strength value. */ +} GPIO_PADREGA_PAD3STRNG_Enum; + +/* ============================================ GPIO PADREGA PAD3INPEN [25..25] ============================================ */ +typedef enum { /*!< GPIO_PADREGA_PAD3INPEN */ + GPIO_PADREGA_PAD3INPEN_DIS = 0, /*!< DIS : Pad input disabled value. */ + GPIO_PADREGA_PAD3INPEN_EN = 1, /*!< EN : Pad input enabled value. */ +} GPIO_PADREGA_PAD3INPEN_Enum; + +/* ============================================ GPIO PADREGA PAD3PULL [24..24] ============================================= */ +typedef enum { /*!< GPIO_PADREGA_PAD3PULL */ + GPIO_PADREGA_PAD3PULL_DIS = 0, /*!< DIS : Pullup disabled value. */ + GPIO_PADREGA_PAD3PULL_EN = 1, /*!< EN : Pullup enabled value. */ +} GPIO_PADREGA_PAD3PULL_Enum; + +/* =========================================== GPIO PADREGA PAD2FNCSEL [19..21] ============================================ */ +typedef enum { /*!< GPIO_PADREGA_PAD2FNCSEL */ + GPIO_PADREGA_PAD2FNCSEL_SLMISO = 1, /*!< SLMISO : Configure as the IOSLAVE SPI MISO signal value. */ + GPIO_PADREGA_PAD2FNCSEL_UART0RX = 2, /*!< UART0RX : Configure as the UART0 RX input value. */ + GPIO_PADREGA_PAD2FNCSEL_GPIO2 = 3, /*!< GPIO2 : Configure as GPIO2 value. */ + GPIO_PADREGA_PAD2FNCSEL_MSPI6 = 5, /*!< MSPI6 : CMSPI data connection 6 value. */ + GPIO_PADREGA_PAD2FNCSEL_NCE2 = 7, /*!< NCE2 : IOM/MSPI nCE group 2 value. */ +} GPIO_PADREGA_PAD2FNCSEL_Enum; + +/* ============================================ GPIO PADREGA PAD2STRNG [18..18] ============================================ */ +typedef enum { /*!< GPIO_PADREGA_PAD2STRNG */ + GPIO_PADREGA_PAD2STRNG_LOW = 0, /*!< LOW : Low drive strength value. */ + GPIO_PADREGA_PAD2STRNG_HIGH = 1, /*!< HIGH : High drive strength value. */ +} GPIO_PADREGA_PAD2STRNG_Enum; + +/* ============================================ GPIO PADREGA PAD2INPEN [17..17] ============================================ */ +typedef enum { /*!< GPIO_PADREGA_PAD2INPEN */ + GPIO_PADREGA_PAD2INPEN_DIS = 0, /*!< DIS : Pad input disabled value. */ + GPIO_PADREGA_PAD2INPEN_EN = 1, /*!< EN : Pad input enabled value. */ +} GPIO_PADREGA_PAD2INPEN_Enum; + +/* ============================================ GPIO PADREGA PAD2PULL [16..16] ============================================= */ +typedef enum { /*!< GPIO_PADREGA_PAD2PULL */ + GPIO_PADREGA_PAD2PULL_DIS = 0, /*!< DIS : Pullup disabled value. */ + GPIO_PADREGA_PAD2PULL_EN = 1, /*!< EN : Pullup enabled value. */ +} GPIO_PADREGA_PAD2PULL_Enum; + +/* ============================================ GPIO PADREGA PAD1RSEL [14..15] ============================================= */ +typedef enum { /*!< GPIO_PADREGA_PAD1RSEL */ + GPIO_PADREGA_PAD1RSEL_PULL1_5K = 0, /*!< PULL1_5K : Pullup is ~1.5 KOhms value. */ + GPIO_PADREGA_PAD1RSEL_PULL6K = 1, /*!< PULL6K : Pullup is ~6 KOhms value. */ + GPIO_PADREGA_PAD1RSEL_PULL12K = 2, /*!< PULL12K : Pullup is ~12 KOhms value. */ + GPIO_PADREGA_PAD1RSEL_PULL24K = 3, /*!< PULL24K : Pullup is ~24 KOhms value. */ +} GPIO_PADREGA_PAD1RSEL_Enum; + +/* =========================================== GPIO PADREGA PAD1FNCSEL [11..13] ============================================ */ +typedef enum { /*!< GPIO_PADREGA_PAD1FNCSEL */ + GPIO_PADREGA_PAD1FNCSEL_SLSDAWIR3 = 0, /*!< SLSDAWIR3 : Configure as the IOSLAVE I2C SDA or SPI WIR3 signal + value. */ + GPIO_PADREGA_PAD1FNCSEL_SLMOSI = 1, /*!< SLMOSI : Configure as the IOSLAVE SPI MOSI signal value. */ + GPIO_PADREGA_PAD1FNCSEL_UART0TX = 2, /*!< UART0TX : Configure as the UART0 TX output signal value. */ + GPIO_PADREGA_PAD1FNCSEL_GPIO1 = 3, /*!< GPIO1 : Configure as GPIO1 value. */ + GPIO_PADREGA_PAD1FNCSEL_MSPI5 = 5, /*!< MSPI5 : MSPI data connection 5 value. */ + GPIO_PADREGA_PAD1FNCSEL_NCE1 = 7, /*!< NCE1 : IOM/MSPI nCE group 1 value. */ +} GPIO_PADREGA_PAD1FNCSEL_Enum; + +/* ============================================ GPIO PADREGA PAD1STRNG [10..10] ============================================ */ +typedef enum { /*!< GPIO_PADREGA_PAD1STRNG */ + GPIO_PADREGA_PAD1STRNG_LOW = 0, /*!< LOW : Low drive strength value. */ + GPIO_PADREGA_PAD1STRNG_HIGH = 1, /*!< HIGH : High drive strength value. */ +} GPIO_PADREGA_PAD1STRNG_Enum; + +/* ============================================= GPIO PADREGA PAD1INPEN [9..9] ============================================= */ +typedef enum { /*!< GPIO_PADREGA_PAD1INPEN */ + GPIO_PADREGA_PAD1INPEN_DIS = 0, /*!< DIS : Pad input disabled value. */ + GPIO_PADREGA_PAD1INPEN_EN = 1, /*!< EN : Pad input enabled value. */ +} GPIO_PADREGA_PAD1INPEN_Enum; + +/* ============================================= GPIO PADREGA PAD1PULL [8..8] ============================================== */ +typedef enum { /*!< GPIO_PADREGA_PAD1PULL */ + GPIO_PADREGA_PAD1PULL_DIS = 0, /*!< DIS : Pullup disabled value. */ + GPIO_PADREGA_PAD1PULL_EN = 1, /*!< EN : Pullup enabled value. */ +} GPIO_PADREGA_PAD1PULL_Enum; + +/* ============================================= GPIO PADREGA PAD0RSEL [6..7] ============================================== */ +typedef enum { /*!< GPIO_PADREGA_PAD0RSEL */ + GPIO_PADREGA_PAD0RSEL_PULL1_5K = 0, /*!< PULL1_5K : Pullup is ~1.5 KOhms value. */ + GPIO_PADREGA_PAD0RSEL_PULL6K = 1, /*!< PULL6K : Pullup is ~6 KOhms value. */ + GPIO_PADREGA_PAD0RSEL_PULL12K = 2, /*!< PULL12K : Pullup is ~12 KOhms value. */ + GPIO_PADREGA_PAD0RSEL_PULL24K = 3, /*!< PULL24K : Pullup is ~24 KOhms value. */ +} GPIO_PADREGA_PAD0RSEL_Enum; + +/* ============================================ GPIO PADREGA PAD0FNCSEL [3..5] ============================================= */ +typedef enum { /*!< GPIO_PADREGA_PAD0FNCSEL */ + GPIO_PADREGA_PAD0FNCSEL_SLSCL = 0, /*!< SLSCL : Configure as the IOSLAVE I2C SCL signal value. */ + GPIO_PADREGA_PAD0FNCSEL_SLSCK = 1, /*!< SLSCK : Configure as the IOSLAVE SPI SCK signal value. */ + GPIO_PADREGA_PAD0FNCSEL_CLKOUT = 2, /*!< CLKOUT : Configure as the CLKOUT signal value. */ + GPIO_PADREGA_PAD0FNCSEL_GPIO0 = 3, /*!< GPIO0 : Configure as GPIO0 value. */ + GPIO_PADREGA_PAD0FNCSEL_MSPI4 = 5, /*!< MSPI4 : MSPI data connection 4 value. */ + GPIO_PADREGA_PAD0FNCSEL_NCE0 = 7, /*!< NCE0 : IOM/MSPI nCE group 0 value. */ +} GPIO_PADREGA_PAD0FNCSEL_Enum; + +/* ============================================= GPIO PADREGA PAD0STRNG [2..2] ============================================= */ +typedef enum { /*!< GPIO_PADREGA_PAD0STRNG */ + GPIO_PADREGA_PAD0STRNG_LOW = 0, /*!< LOW : Low drive strength value. */ + GPIO_PADREGA_PAD0STRNG_HIGH = 1, /*!< HIGH : High drive strength value. */ +} GPIO_PADREGA_PAD0STRNG_Enum; + +/* ============================================= GPIO PADREGA PAD0INPEN [1..1] ============================================= */ +typedef enum { /*!< GPIO_PADREGA_PAD0INPEN */ + GPIO_PADREGA_PAD0INPEN_DIS = 0, /*!< DIS : Pad input disabled value. */ + GPIO_PADREGA_PAD0INPEN_EN = 1, /*!< EN : Pad input enabled value. */ +} GPIO_PADREGA_PAD0INPEN_Enum; + +/* ============================================= GPIO PADREGA PAD0PULL [0..0] ============================================== */ +typedef enum { /*!< GPIO_PADREGA_PAD0PULL */ + GPIO_PADREGA_PAD0PULL_DIS = 0, /*!< DIS : Pullup disabled value. */ + GPIO_PADREGA_PAD0PULL_EN = 1, /*!< EN : Pullup enabled value. */ +} GPIO_PADREGA_PAD0PULL_Enum; + +/* ======================================================== PADREGB ======================================================== */ +/* =========================================== GPIO PADREGB PAD7FNCSEL [27..29] ============================================ */ +typedef enum { /*!< GPIO_PADREGB_PAD7FNCSEL */ + GPIO_PADREGB_PAD7FNCSEL_NCE7 = 0, /*!< NCE7 : IOM/MSPI nCE group 7 value. */ + GPIO_PADREGB_PAD7FNCSEL_M0MOSI = 1, /*!< M0MOSI : Configure as the IOMSTR0 SPI MOSI signal value. */ + GPIO_PADREGB_PAD7FNCSEL_CLKOUT = 2, /*!< CLKOUT : Configure as the CLKOUT signal value. */ + GPIO_PADREGB_PAD7FNCSEL_GPIO7 = 3, /*!< GPIO7 : Configure as GPIO7 value. */ + GPIO_PADREGB_PAD7FNCSEL_TRIG0 = 4, /*!< TRIG0 : Configure as the ADC Trigger 0 signal value. */ + GPIO_PADREGB_PAD7FNCSEL_UART0TX = 5, /*!< UART0TX : Configure as the UART0 TX output signal value. */ + GPIO_PADREGB_PAD7FNCSEL_CT19 = 7, /*!< CT19 : CTIMER connection 19 value. */ +} GPIO_PADREGB_PAD7FNCSEL_Enum; + +/* ============================================ GPIO PADREGB PAD7STRNG [26..26] ============================================ */ +typedef enum { /*!< GPIO_PADREGB_PAD7STRNG */ + GPIO_PADREGB_PAD7STRNG_LOW = 0, /*!< LOW : Low drive strength value. */ + GPIO_PADREGB_PAD7STRNG_HIGH = 1, /*!< HIGH : High drive strength value. */ +} GPIO_PADREGB_PAD7STRNG_Enum; + +/* ============================================ GPIO PADREGB PAD7INPEN [25..25] ============================================ */ +typedef enum { /*!< GPIO_PADREGB_PAD7INPEN */ + GPIO_PADREGB_PAD7INPEN_DIS = 0, /*!< DIS : Pad input disabled value. */ + GPIO_PADREGB_PAD7INPEN_EN = 1, /*!< EN : Pad input enabled value. */ +} GPIO_PADREGB_PAD7INPEN_Enum; + +/* ============================================ GPIO PADREGB PAD7PULL [24..24] ============================================= */ +typedef enum { /*!< GPIO_PADREGB_PAD7PULL */ + GPIO_PADREGB_PAD7PULL_DIS = 0, /*!< DIS : Pullup disabled value. */ + GPIO_PADREGB_PAD7PULL_EN = 1, /*!< EN : Pullup enabled value. */ +} GPIO_PADREGB_PAD7PULL_Enum; + +/* ============================================ GPIO PADREGB PAD6RSEL [22..23] ============================================= */ +typedef enum { /*!< GPIO_PADREGB_PAD6RSEL */ + GPIO_PADREGB_PAD6RSEL_PULL1_5K = 0, /*!< PULL1_5K : Pullup is ~1.5 KOhms value. */ + GPIO_PADREGB_PAD6RSEL_PULL6K = 1, /*!< PULL6K : Pullup is ~6 KOhms value. */ + GPIO_PADREGB_PAD6RSEL_PULL12K = 2, /*!< PULL12K : Pullup is ~12 KOhms value. */ + GPIO_PADREGB_PAD6RSEL_PULL24K = 3, /*!< PULL24K : Pullup is ~24 KOhms value. */ +} GPIO_PADREGB_PAD6RSEL_Enum; + +/* =========================================== GPIO PADREGB PAD6FNCSEL [19..21] ============================================ */ +typedef enum { /*!< GPIO_PADREGB_PAD6FNCSEL */ + GPIO_PADREGB_PAD6FNCSEL_M0SDAWIR3 = 0, /*!< M0SDAWIR3 : Configure as the IOMSTR0 I2C SDA or SPI WIR3 signal + value. */ + GPIO_PADREGB_PAD6FNCSEL_M0MISO = 1, /*!< M0MISO : Configure as the IOMSTR0 SPI MISO signal value. */ + GPIO_PADREGB_PAD6FNCSEL_UA0CTS = 2, /*!< UA0CTS : Configure as the UART0 CTS input signal value. */ + GPIO_PADREGB_PAD6FNCSEL_GPIO6 = 3, /*!< GPIO6 : Configure as GPIO6 value. */ + GPIO_PADREGB_PAD6FNCSEL_CT10 = 5, /*!< CT10 : CTIMER connection 10 value. */ + GPIO_PADREGB_PAD6FNCSEL_I2S_DAT = 7, /*!< I2S_DAT : Configure as the PDM I2S Data output signal value. */ +} GPIO_PADREGB_PAD6FNCSEL_Enum; + +/* ============================================ GPIO PADREGB PAD6STRNG [18..18] ============================================ */ +typedef enum { /*!< GPIO_PADREGB_PAD6STRNG */ + GPIO_PADREGB_PAD6STRNG_LOW = 0, /*!< LOW : Low drive strength value. */ + GPIO_PADREGB_PAD6STRNG_HIGH = 1, /*!< HIGH : High drive strength value. */ +} GPIO_PADREGB_PAD6STRNG_Enum; + +/* ============================================ GPIO PADREGB PAD6INPEN [17..17] ============================================ */ +typedef enum { /*!< GPIO_PADREGB_PAD6INPEN */ + GPIO_PADREGB_PAD6INPEN_DIS = 0, /*!< DIS : Pad input disabled value. */ + GPIO_PADREGB_PAD6INPEN_EN = 1, /*!< EN : Pad input enabled value. */ +} GPIO_PADREGB_PAD6INPEN_Enum; + +/* ============================================ GPIO PADREGB PAD6PULL [16..16] ============================================= */ +typedef enum { /*!< GPIO_PADREGB_PAD6PULL */ + GPIO_PADREGB_PAD6PULL_DIS = 0, /*!< DIS : Pullup disabled value. */ + GPIO_PADREGB_PAD6PULL_EN = 1, /*!< EN : Pullup enabled value. */ +} GPIO_PADREGB_PAD6PULL_Enum; + +/* ============================================ GPIO PADREGB PAD5RSEL [14..15] ============================================= */ +typedef enum { /*!< GPIO_PADREGB_PAD5RSEL */ + GPIO_PADREGB_PAD5RSEL_PULL1_5K = 0, /*!< PULL1_5K : Pullup is ~1.5 KOhms value. */ + GPIO_PADREGB_PAD5RSEL_PULL6K = 1, /*!< PULL6K : Pullup is ~6 KOhms value. */ + GPIO_PADREGB_PAD5RSEL_PULL12K = 2, /*!< PULL12K : Pullup is ~12 KOhms value. */ + GPIO_PADREGB_PAD5RSEL_PULL24K = 3, /*!< PULL24K : Pullup is ~24 KOhms value. */ +} GPIO_PADREGB_PAD5RSEL_Enum; + +/* =========================================== GPIO PADREGB PAD5FNCSEL [11..13] ============================================ */ +typedef enum { /*!< GPIO_PADREGB_PAD5FNCSEL */ + GPIO_PADREGB_PAD5FNCSEL_M0SCL = 0, /*!< M0SCL : Configure as the IOMSTR0 I2C SCL signal value. */ + GPIO_PADREGB_PAD5FNCSEL_M0SCK = 1, /*!< M0SCK : Configure as the IOMSTR0 SPI SCK signal value. */ + GPIO_PADREGB_PAD5FNCSEL_UA0RTS = 2, /*!< UA0RTS : Configure as the UART0 RTS signal output value. */ + GPIO_PADREGB_PAD5FNCSEL_GPIO5 = 3, /*!< GPIO5 : Configure as GPIO5 value. */ + GPIO_PADREGB_PAD5FNCSEL_EXTHFA = 5, /*!< EXTHFA : Configure as the External HFA input clock value. */ + GPIO_PADREGB_PAD5FNCSEL_CT8 = 7, /*!< CT8 : CTIMER connection 8 value. */ +} GPIO_PADREGB_PAD5FNCSEL_Enum; + +/* ============================================ GPIO PADREGB PAD5STRNG [10..10] ============================================ */ +typedef enum { /*!< GPIO_PADREGB_PAD5STRNG */ + GPIO_PADREGB_PAD5STRNG_LOW = 0, /*!< LOW : Low drive strength value. */ + GPIO_PADREGB_PAD5STRNG_HIGH = 1, /*!< HIGH : High drive strength value. */ +} GPIO_PADREGB_PAD5STRNG_Enum; + +/* ============================================= GPIO PADREGB PAD5INPEN [9..9] ============================================= */ +typedef enum { /*!< GPIO_PADREGB_PAD5INPEN */ + GPIO_PADREGB_PAD5INPEN_DIS = 0, /*!< DIS : Pad input disabled value. */ + GPIO_PADREGB_PAD5INPEN_EN = 1, /*!< EN : Pad input enabled value. */ +} GPIO_PADREGB_PAD5INPEN_Enum; + +/* ============================================= GPIO PADREGB PAD5PULL [8..8] ============================================== */ +typedef enum { /*!< GPIO_PADREGB_PAD5PULL */ + GPIO_PADREGB_PAD5PULL_DIS = 0, /*!< DIS : Pullup disabled value. */ + GPIO_PADREGB_PAD5PULL_EN = 1, /*!< EN : Pullup enabled value. */ +} GPIO_PADREGB_PAD5PULL_Enum; + +/* ============================================ GPIO PADREGB PAD4FNCSEL [3..5] ============================================= */ +typedef enum { /*!< GPIO_PADREGB_PAD4FNCSEL */ + GPIO_PADREGB_PAD4FNCSEL_UA0CTS = 0, /*!< UA0CTS : Configure as the UART0 CTS input signal value. */ + GPIO_PADREGB_PAD4FNCSEL_SLINT = 1, /*!< SLINT : Configure as the IOSLAVE interrupt out signal value. */ + GPIO_PADREGB_PAD4FNCSEL_NCE4 = 2, /*!< NCE4 : IOM/SPI nCE group 4 value. */ + GPIO_PADREGB_PAD4FNCSEL_GPIO4 = 3, /*!< GPIO4 : Configure as GPIO4 value. */ + GPIO_PADREGB_PAD4FNCSEL_UART0RX = 5, /*!< UART0RX : Configure as the UART0 RX input value. */ + GPIO_PADREGB_PAD4FNCSEL_CT17 = 6, /*!< CT17 : CTIMER connection 17 value. */ + GPIO_PADREGB_PAD4FNCSEL_MSPI2 = 7, /*!< MSPI2 : MSPI data connection 2 value. */ +} GPIO_PADREGB_PAD4FNCSEL_Enum; + +/* ============================================= GPIO PADREGB PAD4STRNG [2..2] ============================================= */ +typedef enum { /*!< GPIO_PADREGB_PAD4STRNG */ + GPIO_PADREGB_PAD4STRNG_LOW = 0, /*!< LOW : Low drive strength value. */ + GPIO_PADREGB_PAD4STRNG_HIGH = 1, /*!< HIGH : High drive strength value. */ +} GPIO_PADREGB_PAD4STRNG_Enum; + +/* ============================================= GPIO PADREGB PAD4INPEN [1..1] ============================================= */ +typedef enum { /*!< GPIO_PADREGB_PAD4INPEN */ + GPIO_PADREGB_PAD4INPEN_DIS = 0, /*!< DIS : Pad input disabled value. */ + GPIO_PADREGB_PAD4INPEN_EN = 1, /*!< EN : Pad input enabled value. */ +} GPIO_PADREGB_PAD4INPEN_Enum; + +/* ============================================= GPIO PADREGB PAD4PULL [0..0] ============================================== */ +typedef enum { /*!< GPIO_PADREGB_PAD4PULL */ + GPIO_PADREGB_PAD4PULL_DIS = 0, /*!< DIS : Pullup disabled value. */ + GPIO_PADREGB_PAD4PULL_EN = 1, /*!< EN : Pullup enabled value. */ +} GPIO_PADREGB_PAD4PULL_Enum; + +/* ======================================================== PADREGC ======================================================== */ +/* =========================================== GPIO PADREGC PAD11FNCSEL [27..29] =========================================== */ +typedef enum { /*!< GPIO_PADREGC_PAD11FNCSEL */ + GPIO_PADREGC_PAD11FNCSEL_ADCSE2 = 0, /*!< ADCSE2 : Configure as the analog input for ADC single ended + input 2 value. */ + GPIO_PADREGC_PAD11FNCSEL_NCE11 = 1, /*!< NCE11 : IOM/MSPI nCE group 11 value. */ + GPIO_PADREGC_PAD11FNCSEL_CT31 = 2, /*!< CT31 : CTIMER connection 31 value. */ + GPIO_PADREGC_PAD11FNCSEL_GPIO11 = 3, /*!< GPIO11 : Configure as GPIO11 value. */ + GPIO_PADREGC_PAD11FNCSEL_SLINT = 4, /*!< SLINT : Configure as the IOSLAVE interrupt out signal value. */ + GPIO_PADREGC_PAD11FNCSEL_UA1CTS = 5, /*!< UA1CTS : Configure as the UART1 CTS input signal value. */ + GPIO_PADREGC_PAD11FNCSEL_UART0RX = 6, /*!< UART0RX : Configure as the UART0 RX input signal value. */ + GPIO_PADREGC_PAD11FNCSEL_PDM_DATA = 7, /*!< PDM_DATA : Configure as the PDM Data input signal value. */ +} GPIO_PADREGC_PAD11FNCSEL_Enum; + +/* =========================================== GPIO PADREGC PAD11STRNG [26..26] ============================================ */ +typedef enum { /*!< GPIO_PADREGC_PAD11STRNG */ + GPIO_PADREGC_PAD11STRNG_LOW = 0, /*!< LOW : Low drive strength value. */ + GPIO_PADREGC_PAD11STRNG_HIGH = 1, /*!< HIGH : High drive strength value. */ +} GPIO_PADREGC_PAD11STRNG_Enum; + +/* =========================================== GPIO PADREGC PAD11INPEN [25..25] ============================================ */ +typedef enum { /*!< GPIO_PADREGC_PAD11INPEN */ + GPIO_PADREGC_PAD11INPEN_DIS = 0, /*!< DIS : Pad input disabled value. */ + GPIO_PADREGC_PAD11INPEN_EN = 1, /*!< EN : Pad input enabled value. */ +} GPIO_PADREGC_PAD11INPEN_Enum; + +/* ============================================ GPIO PADREGC PAD11PULL [24..24] ============================================ */ +typedef enum { /*!< GPIO_PADREGC_PAD11PULL */ + GPIO_PADREGC_PAD11PULL_DIS = 0, /*!< DIS : Pullup disabled value. */ + GPIO_PADREGC_PAD11PULL_EN = 1, /*!< EN : Pullup enabled value. */ +} GPIO_PADREGC_PAD11PULL_Enum; + +/* =========================================== GPIO PADREGC PAD10FNCSEL [19..21] =========================================== */ +typedef enum { /*!< GPIO_PADREGC_PAD10FNCSEL */ + GPIO_PADREGC_PAD10FNCSEL_M1MOSI = 1, /*!< M1MOSI : Configure as the IOMSTR1 SPI MOSI signal value. */ + GPIO_PADREGC_PAD10FNCSEL_NCE10 = 2, /*!< NCE10 : IOM/MSPI nCE group 10 value. */ + GPIO_PADREGC_PAD10FNCSEL_GPIO10 = 3, /*!< GPIO10 : Configure as GPIO10 value. */ + GPIO_PADREGC_PAD10FNCSEL_PDMCLK = 4, /*!< PDMCLK : PDM serial clock out value. */ + GPIO_PADREGC_PAD10FNCSEL_UA1RTS = 5, /*!< UA1RTS : Configure as the UART1 RTS output signal value. */ +} GPIO_PADREGC_PAD10FNCSEL_Enum; + +/* =========================================== GPIO PADREGC PAD10STRNG [18..18] ============================================ */ +typedef enum { /*!< GPIO_PADREGC_PAD10STRNG */ + GPIO_PADREGC_PAD10STRNG_LOW = 0, /*!< LOW : Low drive strength value. */ + GPIO_PADREGC_PAD10STRNG_HIGH = 1, /*!< HIGH : High drive strength value. */ +} GPIO_PADREGC_PAD10STRNG_Enum; + +/* =========================================== GPIO PADREGC PAD10INPEN [17..17] ============================================ */ +typedef enum { /*!< GPIO_PADREGC_PAD10INPEN */ + GPIO_PADREGC_PAD10INPEN_DIS = 0, /*!< DIS : Pad input disabled value. */ + GPIO_PADREGC_PAD10INPEN_EN = 1, /*!< EN : Pad input enabled value. */ +} GPIO_PADREGC_PAD10INPEN_Enum; + +/* ============================================ GPIO PADREGC PAD10PULL [16..16] ============================================ */ +typedef enum { /*!< GPIO_PADREGC_PAD10PULL */ + GPIO_PADREGC_PAD10PULL_DIS = 0, /*!< DIS : Pullup disabled value. */ + GPIO_PADREGC_PAD10PULL_EN = 1, /*!< EN : Pullup enabled value. */ +} GPIO_PADREGC_PAD10PULL_Enum; + +/* ============================================ GPIO PADREGC PAD9RSEL [14..15] ============================================= */ +typedef enum { /*!< GPIO_PADREGC_PAD9RSEL */ + GPIO_PADREGC_PAD9RSEL_PULL1_5K = 0, /*!< PULL1_5K : Pullup is ~1.5 KOhms value. */ + GPIO_PADREGC_PAD9RSEL_PULL6K = 1, /*!< PULL6K : Pullup is ~6 KOhms value. */ + GPIO_PADREGC_PAD9RSEL_PULL12K = 2, /*!< PULL12K : Pullup is ~12 KOhms value. */ + GPIO_PADREGC_PAD9RSEL_PULL24K = 3, /*!< PULL24K : Pullup is ~24 KOhms value. */ +} GPIO_PADREGC_PAD9RSEL_Enum; + +/* =========================================== GPIO PADREGC PAD9FNCSEL [11..13] ============================================ */ +typedef enum { /*!< GPIO_PADREGC_PAD9FNCSEL */ + GPIO_PADREGC_PAD9FNCSEL_M1SDAWIR3 = 0, /*!< M1SDAWIR3 : Configure as the IOMSTR1 I2C SDA or SPI WIR3 signal + value. */ + GPIO_PADREGC_PAD9FNCSEL_M1MISO = 1, /*!< M1MISO : Configure as the IOMSTR1 SPI MISO signal value. */ + GPIO_PADREGC_PAD9FNCSEL_NCE9 = 2, /*!< NCE9 : IOM/MSPI nCE group 9 value. */ + GPIO_PADREGC_PAD9FNCSEL_GPIO9 = 3, /*!< GPIO9 : Configure as GPIO9 value. */ + GPIO_PADREGC_PAD9FNCSEL_SCCIO = 4, /*!< SCCIO : SCARD data I/O connection value. */ + GPIO_PADREGC_PAD9FNCSEL_UART1RX = 6, /*!< UART1RX : Configure as UART1 RX input signal value. */ +} GPIO_PADREGC_PAD9FNCSEL_Enum; + +/* ============================================ GPIO PADREGC PAD9STRNG [10..10] ============================================ */ +typedef enum { /*!< GPIO_PADREGC_PAD9STRNG */ + GPIO_PADREGC_PAD9STRNG_LOW = 0, /*!< LOW : Low drive strength value. */ + GPIO_PADREGC_PAD9STRNG_HIGH = 1, /*!< HIGH : High drive strength value. */ +} GPIO_PADREGC_PAD9STRNG_Enum; + +/* ============================================= GPIO PADREGC PAD9INPEN [9..9] ============================================= */ +typedef enum { /*!< GPIO_PADREGC_PAD9INPEN */ + GPIO_PADREGC_PAD9INPEN_DIS = 0, /*!< DIS : Pad input disabled value. */ + GPIO_PADREGC_PAD9INPEN_EN = 1, /*!< EN : Pad input enabled value. */ +} GPIO_PADREGC_PAD9INPEN_Enum; + +/* ============================================= GPIO PADREGC PAD9PULL [8..8] ============================================== */ +typedef enum { /*!< GPIO_PADREGC_PAD9PULL */ + GPIO_PADREGC_PAD9PULL_DIS = 0, /*!< DIS : Pullup disabled value. */ + GPIO_PADREGC_PAD9PULL_EN = 1, /*!< EN : Pullup enabled value. */ +} GPIO_PADREGC_PAD9PULL_Enum; + +/* ============================================= GPIO PADREGC PAD8RSEL [6..7] ============================================== */ +typedef enum { /*!< GPIO_PADREGC_PAD8RSEL */ + GPIO_PADREGC_PAD8RSEL_PULL1_5K = 0, /*!< PULL1_5K : Pullup is ~1.5 KOhms value. */ + GPIO_PADREGC_PAD8RSEL_PULL6K = 1, /*!< PULL6K : Pullup is ~6 KOhms value. */ + GPIO_PADREGC_PAD8RSEL_PULL12K = 2, /*!< PULL12K : Pullup is ~12 KOhms value. */ + GPIO_PADREGC_PAD8RSEL_PULL24K = 3, /*!< PULL24K : Pullup is ~24 KOhms value. */ +} GPIO_PADREGC_PAD8RSEL_Enum; + +/* ============================================ GPIO PADREGC PAD8FNCSEL [3..5] ============================================= */ +typedef enum { /*!< GPIO_PADREGC_PAD8FNCSEL */ + GPIO_PADREGC_PAD8FNCSEL_M1SCL = 0, /*!< M1SCL : Configure as the IOMSTR1 I2C SCL signal value. */ + GPIO_PADREGC_PAD8FNCSEL_M1SCK = 1, /*!< M1SCK : Configure as the IOMSTR1 SPI SCK signal value. */ + GPIO_PADREGC_PAD8FNCSEL_NCE8 = 2, /*!< NCE8 : IOM/MSPI nCE group 8 value. */ + GPIO_PADREGC_PAD8FNCSEL_GPIO8 = 3, /*!< GPIO8 : Configure as GPIO8 value. */ + GPIO_PADREGC_PAD8FNCSEL_SCCLK = 4, /*!< SCCLK : SCARD serial clock output value. */ + GPIO_PADREGC_PAD8FNCSEL_UART1TX = 6, /*!< UART1TX : Configure as the UART1 TX output signal value. */ +} GPIO_PADREGC_PAD8FNCSEL_Enum; + +/* ============================================= GPIO PADREGC PAD8STRNG [2..2] ============================================= */ +typedef enum { /*!< GPIO_PADREGC_PAD8STRNG */ + GPIO_PADREGC_PAD8STRNG_LOW = 0, /*!< LOW : Low drive strength value. */ + GPIO_PADREGC_PAD8STRNG_HIGH = 1, /*!< HIGH : High drive strength value. */ +} GPIO_PADREGC_PAD8STRNG_Enum; + +/* ============================================= GPIO PADREGC PAD8INPEN [1..1] ============================================= */ +typedef enum { /*!< GPIO_PADREGC_PAD8INPEN */ + GPIO_PADREGC_PAD8INPEN_DIS = 0, /*!< DIS : Pad input disabled value. */ + GPIO_PADREGC_PAD8INPEN_EN = 1, /*!< EN : Pad input enabled value. */ +} GPIO_PADREGC_PAD8INPEN_Enum; + +/* ============================================= GPIO PADREGC PAD8PULL [0..0] ============================================== */ +typedef enum { /*!< GPIO_PADREGC_PAD8PULL */ + GPIO_PADREGC_PAD8PULL_DIS = 0, /*!< DIS : Pullup disabled value. */ + GPIO_PADREGC_PAD8PULL_EN = 1, /*!< EN : Pullup enabled value. */ +} GPIO_PADREGC_PAD8PULL_Enum; + +/* ======================================================== PADREGD ======================================================== */ +/* =========================================== GPIO PADREGD PAD15FNCSEL [27..29] =========================================== */ +typedef enum { /*!< GPIO_PADREGD_PAD15FNCSEL */ + GPIO_PADREGD_PAD15FNCSEL_ADCD1N = 0, /*!< ADCD1N : Configure as the analog ADC differential pair 1 N input + signal value. */ + GPIO_PADREGD_PAD15FNCSEL_NCE15 = 1, /*!< NCE15 : IOM/MSPI nCE group 15 value. */ + GPIO_PADREGD_PAD15FNCSEL_UART1RX = 2, /*!< UART1RX : Configure as the UART1 RX signal value. */ + GPIO_PADREGD_PAD15FNCSEL_GPIO15 = 3, /*!< GPIO15 : Configure as GPIO15 value. */ + GPIO_PADREGD_PAD15FNCSEL_PDMDATA = 4, /*!< PDMDATA : PDM serial data input value. */ + GPIO_PADREGD_PAD15FNCSEL_EXTXT = 5, /*!< EXTXT : Configure as the external XTAL oscillator input value. */ + GPIO_PADREGD_PAD15FNCSEL_SWDIO = 6, /*!< SWDIO : Configure as an alternate port for the SWDIO I/O signal + value. */ + GPIO_PADREGD_PAD15FNCSEL_SWO = 7, /*!< SWO : Configure as an SWO (Serial Wire Trace output) value. */ +} GPIO_PADREGD_PAD15FNCSEL_Enum; + +/* =========================================== GPIO PADREGD PAD15STRNG [26..26] ============================================ */ +typedef enum { /*!< GPIO_PADREGD_PAD15STRNG */ + GPIO_PADREGD_PAD15STRNG_LOW = 0, /*!< LOW : Low drive strength value. */ + GPIO_PADREGD_PAD15STRNG_HIGH = 1, /*!< HIGH : High drive strength value. */ +} GPIO_PADREGD_PAD15STRNG_Enum; + +/* =========================================== GPIO PADREGD PAD15INPEN [25..25] ============================================ */ +typedef enum { /*!< GPIO_PADREGD_PAD15INPEN */ + GPIO_PADREGD_PAD15INPEN_DIS = 0, /*!< DIS : Pad input disabled value. */ + GPIO_PADREGD_PAD15INPEN_EN = 1, /*!< EN : Pad input enabled value. */ +} GPIO_PADREGD_PAD15INPEN_Enum; + +/* ============================================ GPIO PADREGD PAD15PULL [24..24] ============================================ */ +typedef enum { /*!< GPIO_PADREGD_PAD15PULL */ + GPIO_PADREGD_PAD15PULL_DIS = 0, /*!< DIS : Pullup disabled value. */ + GPIO_PADREGD_PAD15PULL_EN = 1, /*!< EN : Pullup enabled value. */ +} GPIO_PADREGD_PAD15PULL_Enum; + +/* =========================================== GPIO PADREGD PAD14FNCSEL [19..21] =========================================== */ +typedef enum { /*!< GPIO_PADREGD_PAD14FNCSEL */ + GPIO_PADREGD_PAD14FNCSEL_ADCD1P = 0, /*!< ADCD1P : Configure as the analog ADC differential pair 1 P input + signal value. */ + GPIO_PADREGD_PAD14FNCSEL_NCE14 = 1, /*!< NCE14 : IOM/MSPI nCE group 14 value. */ + GPIO_PADREGD_PAD14FNCSEL_UART1TX = 2, /*!< UART1TX : Configure as the UART1 TX output signal value. */ + GPIO_PADREGD_PAD14FNCSEL_GPIO14 = 3, /*!< GPIO14 : Configure as GPIO14 value. */ + GPIO_PADREGD_PAD14FNCSEL_PDMCLK = 4, /*!< PDMCLK : PDM serial clock output value. */ + GPIO_PADREGD_PAD14FNCSEL_EXTHFS = 5, /*!< EXTHFS : Configure as the External HFRC oscillator input select + value. */ + GPIO_PADREGD_PAD14FNCSEL_SWDCK = 6, /*!< SWDCK : Configure as the alternate input for the SWDCK input + signal value. */ + GPIO_PADREGD_PAD14FNCSEL_32kHzXT = 7, /*!< 32kHzXT : Configure as the 32kHz crystal output signal value. */ +} GPIO_PADREGD_PAD14FNCSEL_Enum; + +/* =========================================== GPIO PADREGD PAD14STRNG [18..18] ============================================ */ +typedef enum { /*!< GPIO_PADREGD_PAD14STRNG */ + GPIO_PADREGD_PAD14STRNG_LOW = 0, /*!< LOW : Low drive strength value. */ + GPIO_PADREGD_PAD14STRNG_HIGH = 1, /*!< HIGH : High drive strength value. */ +} GPIO_PADREGD_PAD14STRNG_Enum; + +/* =========================================== GPIO PADREGD PAD14INPEN [17..17] ============================================ */ +typedef enum { /*!< GPIO_PADREGD_PAD14INPEN */ + GPIO_PADREGD_PAD14INPEN_DIS = 0, /*!< DIS : Pad input disabled value. */ + GPIO_PADREGD_PAD14INPEN_EN = 1, /*!< EN : Pad input enabled value. */ +} GPIO_PADREGD_PAD14INPEN_Enum; + +/* ============================================ GPIO PADREGD PAD14PULL [16..16] ============================================ */ +typedef enum { /*!< GPIO_PADREGD_PAD14PULL */ + GPIO_PADREGD_PAD14PULL_DIS = 0, /*!< DIS : Pullup disabled value. */ + GPIO_PADREGD_PAD14PULL_EN = 1, /*!< EN : Pullup enabled value. */ +} GPIO_PADREGD_PAD14PULL_Enum; + +/* =========================================== GPIO PADREGD PAD13FNCSEL [11..13] =========================================== */ +typedef enum { /*!< GPIO_PADREGD_PAD13FNCSEL */ + GPIO_PADREGD_PAD13FNCSEL_ADCD0PSE8 = 0, /*!< ADCD0PSE8 : Configure as the ADC Differential pair 0 P, or Single + Ended input 8 analog input signal. Determination of the + D0P vs SE8 usage is done when the particular channel is + selected within the ADC module value. */ + GPIO_PADREGD_PAD13FNCSEL_NCE13 = 1, /*!< NCE13 : IOM/MSPI nCE group 13 value. */ + GPIO_PADREGD_PAD13FNCSEL_CT2 = 2, /*!< CT2 : CTIMER connection 2 value. */ + GPIO_PADREGD_PAD13FNCSEL_GPIO13 = 3, /*!< GPIO13 : Configure as GPIO13 value. */ + GPIO_PADREGD_PAD13FNCSEL_I2SBCLK = 4, /*!< I2SBCLK : I2C interface bit clock value. */ + GPIO_PADREGD_PAD13FNCSEL_EXTHFB = 5, /*!< EXTHFB : Configure as the external HFRC oscillator input value. */ + GPIO_PADREGD_PAD13FNCSEL_UA0RTS = 6, /*!< UA0RTS : Configure as the UART0 RTS signal output value. */ + GPIO_PADREGD_PAD13FNCSEL_UART1RX = 7, /*!< UART1RX : Configure as the UART1 RX input signal value. */ +} GPIO_PADREGD_PAD13FNCSEL_Enum; + +/* =========================================== GPIO PADREGD PAD13STRNG [10..10] ============================================ */ +typedef enum { /*!< GPIO_PADREGD_PAD13STRNG */ + GPIO_PADREGD_PAD13STRNG_LOW = 0, /*!< LOW : Low drive strength value. */ + GPIO_PADREGD_PAD13STRNG_HIGH = 1, /*!< HIGH : High drive strength value. */ +} GPIO_PADREGD_PAD13STRNG_Enum; + +/* ============================================ GPIO PADREGD PAD13INPEN [9..9] ============================================= */ +typedef enum { /*!< GPIO_PADREGD_PAD13INPEN */ + GPIO_PADREGD_PAD13INPEN_DIS = 0, /*!< DIS : Pad input disabled value. */ + GPIO_PADREGD_PAD13INPEN_EN = 1, /*!< EN : Pad input enabled value. */ +} GPIO_PADREGD_PAD13INPEN_Enum; + +/* ============================================= GPIO PADREGD PAD13PULL [8..8] ============================================= */ +typedef enum { /*!< GPIO_PADREGD_PAD13PULL */ + GPIO_PADREGD_PAD13PULL_DIS = 0, /*!< DIS : Pullup disabled value. */ + GPIO_PADREGD_PAD13PULL_EN = 1, /*!< EN : Pullup enabled value. */ +} GPIO_PADREGD_PAD13PULL_Enum; + +/* ============================================ GPIO PADREGD PAD12FNCSEL [3..5] ============================================ */ +typedef enum { /*!< GPIO_PADREGD_PAD12FNCSEL */ + GPIO_PADREGD_PAD12FNCSEL_ADCD0NSE9 = 0, /*!< ADCD0NSE9 : Configure as the ADC Differential pair 0 N, or Single + Ended input 9 analog input signal. Determination of the + D0N vs SE9 usage is done when the particular channel is + selected within the ADC module value. */ + GPIO_PADREGD_PAD12FNCSEL_NCE12 = 1, /*!< NCE12 : IOM/MSPI nCE group 12 value. */ + GPIO_PADREGD_PAD12FNCSEL_CT0 = 2, /*!< CT0 : CTIMER connection 0 value. */ + GPIO_PADREGD_PAD12FNCSEL_GPIO12 = 3, /*!< GPIO12 : Configure as GPIO12 value. */ + GPIO_PADREGD_PAD12FNCSEL_SLnCE = 4, /*!< SLnCE : Configure as the IOSLAVE SPI nCE signal value. */ + GPIO_PADREGD_PAD12FNCSEL_PDMCLK = 5, /*!< PDMCLK : PDM serial clock output value. */ + GPIO_PADREGD_PAD12FNCSEL_UA0CTS = 6, /*!< UA0CTS : Configure as the UART0 CTS input signal value. */ + GPIO_PADREGD_PAD12FNCSEL_UART1TX = 7, /*!< UART1TX : Configure as the UART1 TX output signal value. */ +} GPIO_PADREGD_PAD12FNCSEL_Enum; + +/* ============================================ GPIO PADREGD PAD12STRNG [2..2] ============================================= */ +typedef enum { /*!< GPIO_PADREGD_PAD12STRNG */ + GPIO_PADREGD_PAD12STRNG_LOW = 0, /*!< LOW : Low drive strength value. */ + GPIO_PADREGD_PAD12STRNG_HIGH = 1, /*!< HIGH : High drive strength value. */ +} GPIO_PADREGD_PAD12STRNG_Enum; + +/* ============================================ GPIO PADREGD PAD12INPEN [1..1] ============================================= */ +typedef enum { /*!< GPIO_PADREGD_PAD12INPEN */ + GPIO_PADREGD_PAD12INPEN_DIS = 0, /*!< DIS : Pad input disabled value. */ + GPIO_PADREGD_PAD12INPEN_EN = 1, /*!< EN : Pad input enabled value. */ +} GPIO_PADREGD_PAD12INPEN_Enum; + +/* ============================================= GPIO PADREGD PAD12PULL [0..0] ============================================= */ +typedef enum { /*!< GPIO_PADREGD_PAD12PULL */ + GPIO_PADREGD_PAD12PULL_DIS = 0, /*!< DIS : Pullup disabled value. */ + GPIO_PADREGD_PAD12PULL_EN = 1, /*!< EN : Pullup enabled value. */ +} GPIO_PADREGD_PAD12PULL_Enum; + +/* ======================================================== PADREGE ======================================================== */ +/* =========================================== GPIO PADREGE PAD19FNCSEL [27..29] =========================================== */ +typedef enum { /*!< GPIO_PADREGE_PAD19FNCSEL */ + GPIO_PADREGE_PAD19FNCSEL_CMPRF0 = 0, /*!< CMPRF0 : Configure as the analog comparator reference 0 signal + value. */ + GPIO_PADREGE_PAD19FNCSEL_NCE19 = 1, /*!< NCE19 : IOM/MSPI nCE group 19 value. */ + GPIO_PADREGE_PAD19FNCSEL_CT6 = 2, /*!< CT6 : CTIMER conenction 6 value. */ + GPIO_PADREGE_PAD19FNCSEL_GPIO19 = 3, /*!< GPIO19 : Configure as GPIO19 value. */ + GPIO_PADREGE_PAD19FNCSEL_SCCLK = 4, /*!< SCCLK : SCARD serial clock value. */ + GPIO_PADREGE_PAD19FNCSEL_ANATEST1 = 5, /*!< ANATEST1 : Configure as the ANATEST1 I/O signal value. */ + GPIO_PADREGE_PAD19FNCSEL_UART1RX = 6, /*!< UART1RX : Configure as the UART1 RX input signal value. */ + GPIO_PADREGE_PAD19FNCSEL_I2SBCLK = 7, /*!< I2SBCLK : Configure as the PDM I2S bit clock input signal value. */ +} GPIO_PADREGE_PAD19FNCSEL_Enum; + +/* =========================================== GPIO PADREGE PAD19STRNG [26..26] ============================================ */ +typedef enum { /*!< GPIO_PADREGE_PAD19STRNG */ + GPIO_PADREGE_PAD19STRNG_LOW = 0, /*!< LOW : Low drive strength value. */ + GPIO_PADREGE_PAD19STRNG_HIGH = 1, /*!< HIGH : High drive strength value. */ +} GPIO_PADREGE_PAD19STRNG_Enum; + +/* =========================================== GPIO PADREGE PAD19INPEN [25..25] ============================================ */ +typedef enum { /*!< GPIO_PADREGE_PAD19INPEN */ + GPIO_PADREGE_PAD19INPEN_DIS = 0, /*!< DIS : Pad input disabled value. */ + GPIO_PADREGE_PAD19INPEN_EN = 1, /*!< EN : Pad input enabled value. */ +} GPIO_PADREGE_PAD19INPEN_Enum; + +/* ============================================ GPIO PADREGE PAD19PULL [24..24] ============================================ */ +typedef enum { /*!< GPIO_PADREGE_PAD19PULL */ + GPIO_PADREGE_PAD19PULL_DIS = 0, /*!< DIS : Pullup disabled value. */ + GPIO_PADREGE_PAD19PULL_EN = 1, /*!< EN : Pullup enabled value. */ +} GPIO_PADREGE_PAD19PULL_Enum; + +/* =========================================== GPIO PADREGE PAD18FNCSEL [19..21] =========================================== */ +typedef enum { /*!< GPIO_PADREGE_PAD18FNCSEL */ + GPIO_PADREGE_PAD18FNCSEL_CMPIN1 = 0, /*!< CMPIN1 : Configure as the analog comparator input 1 signal value. */ + GPIO_PADREGE_PAD18FNCSEL_NCE18 = 1, /*!< NCE18 : IOM/MSPI nCE group 18 value. */ + GPIO_PADREGE_PAD18FNCSEL_CT4 = 2, /*!< CT4 : CTIMER connection 4 value. */ + GPIO_PADREGE_PAD18FNCSEL_GPIO18 = 3, /*!< GPIO18 : Configure as GPIO18 value. */ + GPIO_PADREGE_PAD18FNCSEL_UA0RTS = 4, /*!< UA0RTS : Configure as UART0 RTS output signal value. */ + GPIO_PADREGE_PAD18FNCSEL_ANATEST2 = 5, /*!< ANATEST2 : Configure as ANATEST2 I/O signal value. */ + GPIO_PADREGE_PAD18FNCSEL_UART1TX = 6, /*!< UART1TX : Configure as UART1 TX output signal value. */ + GPIO_PADREGE_PAD18FNCSEL_SCCIO = 7, /*!< SCCIO : SCARD data input/output connectin value. */ +} GPIO_PADREGE_PAD18FNCSEL_Enum; + +/* =========================================== GPIO PADREGE PAD18STRNG [18..18] ============================================ */ +typedef enum { /*!< GPIO_PADREGE_PAD18STRNG */ + GPIO_PADREGE_PAD18STRNG_LOW = 0, /*!< LOW : Low drive strength value. */ + GPIO_PADREGE_PAD18STRNG_HIGH = 1, /*!< HIGH : High drive strength value. */ +} GPIO_PADREGE_PAD18STRNG_Enum; + +/* =========================================== GPIO PADREGE PAD18INPEN [17..17] ============================================ */ +typedef enum { /*!< GPIO_PADREGE_PAD18INPEN */ + GPIO_PADREGE_PAD18INPEN_DIS = 0, /*!< DIS : Pad input disabled value. */ + GPIO_PADREGE_PAD18INPEN_EN = 1, /*!< EN : Pad input enabled value. */ +} GPIO_PADREGE_PAD18INPEN_Enum; + +/* ============================================ GPIO PADREGE PAD18PULL [16..16] ============================================ */ +typedef enum { /*!< GPIO_PADREGE_PAD18PULL */ + GPIO_PADREGE_PAD18PULL_DIS = 0, /*!< DIS : Pullup disabled value. */ + GPIO_PADREGE_PAD18PULL_EN = 1, /*!< EN : Pullup enabled value. */ +} GPIO_PADREGE_PAD18PULL_Enum; + +/* =========================================== GPIO PADREGE PAD17FNCSEL [11..13] =========================================== */ +typedef enum { /*!< GPIO_PADREGE_PAD17FNCSEL */ + GPIO_PADREGE_PAD17FNCSEL_CMPRF1 = 0, /*!< CMPRF1 : Configure as the analog comparator reference signal + 1 input signal value. */ + GPIO_PADREGE_PAD17FNCSEL_NCE17 = 1, /*!< NCE17 : IOM/MSPI nCE group 17 value. */ + GPIO_PADREGE_PAD17FNCSEL_TRIG1 = 2, /*!< TRIG1 : Configure as the ADC Trigger 1 signal value. */ + GPIO_PADREGE_PAD17FNCSEL_GPIO17 = 3, /*!< GPIO17 : Configure as GPIO17 value. */ + GPIO_PADREGE_PAD17FNCSEL_SCCCLK = 4, /*!< SCCCLK : SCARD serial clock output value. */ + GPIO_PADREGE_PAD17FNCSEL_UART0RX = 6, /*!< UART0RX : Configure as UART0 RX input signal value. */ + GPIO_PADREGE_PAD17FNCSEL_UA1CTS = 7, /*!< UA1CTS : Configure as UART1 CTS input signal value. */ +} GPIO_PADREGE_PAD17FNCSEL_Enum; + +/* =========================================== GPIO PADREGE PAD17STRNG [10..10] ============================================ */ +typedef enum { /*!< GPIO_PADREGE_PAD17STRNG */ + GPIO_PADREGE_PAD17STRNG_LOW = 0, /*!< LOW : Low drive strength value. */ + GPIO_PADREGE_PAD17STRNG_HIGH = 1, /*!< HIGH : High drive strength value. */ +} GPIO_PADREGE_PAD17STRNG_Enum; + +/* ============================================ GPIO PADREGE PAD17INPEN [9..9] ============================================= */ +typedef enum { /*!< GPIO_PADREGE_PAD17INPEN */ + GPIO_PADREGE_PAD17INPEN_DIS = 0, /*!< DIS : Pad input disabled value. */ + GPIO_PADREGE_PAD17INPEN_EN = 1, /*!< EN : Pad input enabled value. */ +} GPIO_PADREGE_PAD17INPEN_Enum; + +/* ============================================= GPIO PADREGE PAD17PULL [8..8] ============================================= */ +typedef enum { /*!< GPIO_PADREGE_PAD17PULL */ + GPIO_PADREGE_PAD17PULL_DIS = 0, /*!< DIS : Pullup disabled value. */ + GPIO_PADREGE_PAD17PULL_EN = 1, /*!< EN : Pullup enabled value. */ +} GPIO_PADREGE_PAD17PULL_Enum; + +/* ============================================ GPIO PADREGE PAD16FNCSEL [3..5] ============================================ */ +typedef enum { /*!< GPIO_PADREGE_PAD16FNCSEL */ + GPIO_PADREGE_PAD16FNCSEL_ADCSE0 = 0, /*!< ADCSE0 : Configure as the analog ADC single ended port 0 input + signal value. */ + GPIO_PADREGE_PAD16FNCSEL_NCE16 = 1, /*!< NCE16 : IOM/MSPI nCE group 16 value. */ + GPIO_PADREGE_PAD16FNCSEL_TRIG0 = 2, /*!< TRIG0 : Configure as the ADC Trigger 0 signal value. */ + GPIO_PADREGE_PAD16FNCSEL_GPIO16 = 3, /*!< GPIO16 : Configure as GPIO16 value. */ + GPIO_PADREGE_PAD16FNCSEL_SCCRST = 4, /*!< SCCRST : SCARD reset output value. */ + GPIO_PADREGE_PAD16FNCSEL_CMPIN0 = 5, /*!< CMPIN0 : Configure as comparator input 0 signal value. */ + GPIO_PADREGE_PAD16FNCSEL_UART0TX = 6, /*!< UART0TX : Configure as UART0 TX output signal value. */ + GPIO_PADREGE_PAD16FNCSEL_UA1RTS = 7, /*!< UA1RTS : Configure as UART1 RTS output signal value. */ +} GPIO_PADREGE_PAD16FNCSEL_Enum; + +/* ============================================ GPIO PADREGE PAD16STRNG [2..2] ============================================= */ +typedef enum { /*!< GPIO_PADREGE_PAD16STRNG */ + GPIO_PADREGE_PAD16STRNG_LOW = 0, /*!< LOW : Low drive strength value. */ + GPIO_PADREGE_PAD16STRNG_HIGH = 1, /*!< HIGH : High drive strength value. */ +} GPIO_PADREGE_PAD16STRNG_Enum; + +/* ============================================ GPIO PADREGE PAD16INPEN [1..1] ============================================= */ +typedef enum { /*!< GPIO_PADREGE_PAD16INPEN */ + GPIO_PADREGE_PAD16INPEN_DIS = 0, /*!< DIS : Pad input disabled value. */ + GPIO_PADREGE_PAD16INPEN_EN = 1, /*!< EN : Pad input enabled value. */ +} GPIO_PADREGE_PAD16INPEN_Enum; + +/* ============================================= GPIO PADREGE PAD16PULL [0..0] ============================================= */ +typedef enum { /*!< GPIO_PADREGE_PAD16PULL */ + GPIO_PADREGE_PAD16PULL_DIS = 0, /*!< DIS : Pullup disabled value. */ + GPIO_PADREGE_PAD16PULL_EN = 1, /*!< EN : Pullup enabled value. */ +} GPIO_PADREGE_PAD16PULL_Enum; + +/* ======================================================== PADREGF ======================================================== */ +/* =========================================== GPIO PADREGF PAD23FNCSEL [27..29] =========================================== */ +typedef enum { /*!< GPIO_PADREGF_PAD23FNCSEL */ + GPIO_PADREGF_PAD23FNCSEL_UART0RX = 0, /*!< UART0RX : Configure as the UART0 RX signal value. */ + GPIO_PADREGF_PAD23FNCSEL_NCE23 = 1, /*!< NCE23 : IOM/MSPI nCE group 23 value. */ + GPIO_PADREGF_PAD23FNCSEL_CT14 = 2, /*!< CT14 : CTIMER connection 14 value. */ + GPIO_PADREGF_PAD23FNCSEL_GPIO23 = 3, /*!< GPIO23 : Configure as GPIO23 value. */ + GPIO_PADREGF_PAD23FNCSEL_I2SWCLK = 4, /*!< I2SWCLK : I2S word clock input value. */ + GPIO_PADREGF_PAD23FNCSEL_CMPOUT = 5, /*!< CMPOUT : Configure as voltage comparitor output value. */ + GPIO_PADREGF_PAD23FNCSEL_MSPI3 = 6, /*!< MSPI3 : MSPI data connection 3 value. */ + GPIO_PADREGF_PAD23FNCSEL_EXTXT = 7, /*!< EXTXT : External XTAL osacillatgor input value. */ +} GPIO_PADREGF_PAD23FNCSEL_Enum; + +/* =========================================== GPIO PADREGF PAD23STRNG [26..26] ============================================ */ +typedef enum { /*!< GPIO_PADREGF_PAD23STRNG */ + GPIO_PADREGF_PAD23STRNG_LOW = 0, /*!< LOW : Low drive strength value. */ + GPIO_PADREGF_PAD23STRNG_HIGH = 1, /*!< HIGH : High drive strength value. */ +} GPIO_PADREGF_PAD23STRNG_Enum; + +/* =========================================== GPIO PADREGF PAD23INPEN [25..25] ============================================ */ +typedef enum { /*!< GPIO_PADREGF_PAD23INPEN */ + GPIO_PADREGF_PAD23INPEN_DIS = 0, /*!< DIS : Pad input disabled value. */ + GPIO_PADREGF_PAD23INPEN_EN = 1, /*!< EN : Pad input enabled value. */ +} GPIO_PADREGF_PAD23INPEN_Enum; + +/* ============================================ GPIO PADREGF PAD23PULL [24..24] ============================================ */ +typedef enum { /*!< GPIO_PADREGF_PAD23PULL */ + GPIO_PADREGF_PAD23PULL_DIS = 0, /*!< DIS : Pullup disabled value. */ + GPIO_PADREGF_PAD23PULL_EN = 1, /*!< EN : Pullup enabled value. */ +} GPIO_PADREGF_PAD23PULL_Enum; + +/* =========================================== GPIO PADREGF PAD22FNCSEL [19..21] =========================================== */ +typedef enum { /*!< GPIO_PADREGF_PAD22FNCSEL */ + GPIO_PADREGF_PAD22FNCSEL_UART0TX = 0, /*!< UART0TX : Configure as the UART0 TX signal value. */ + GPIO_PADREGF_PAD22FNCSEL_NCE22 = 1, /*!< NCE22 : IOM/MSPI nCE group 22 value. */ + GPIO_PADREGF_PAD22FNCSEL_CT12 = 2, /*!< CT12 : CTIMER connection 12 value. */ + GPIO_PADREGF_PAD22FNCSEL_GPIO22 = 3, /*!< GPIO22 : Configure as GPIO22 value. */ + GPIO_PADREGF_PAD22FNCSEL_PDM_CLK = 4, /*!< PDM_CLK : Configure as the PDM CLK output value. */ + GPIO_PADREGF_PAD22FNCSEL_EXTLF = 5, /*!< EXTLF : External LFRC input value. */ + GPIO_PADREGF_PAD22FNCSEL_MSPI0 = 6, /*!< MSPI0 : MSPI data connection 0 value. */ + GPIO_PADREGF_PAD22FNCSEL_SWO = 7, /*!< SWO : Configure as the serial trace data output signal value. */ +} GPIO_PADREGF_PAD22FNCSEL_Enum; + +/* =========================================== GPIO PADREGF PAD22STRNG [18..18] ============================================ */ +typedef enum { /*!< GPIO_PADREGF_PAD22STRNG */ + GPIO_PADREGF_PAD22STRNG_LOW = 0, /*!< LOW : Low drive strength value. */ + GPIO_PADREGF_PAD22STRNG_HIGH = 1, /*!< HIGH : High drive strength value. */ +} GPIO_PADREGF_PAD22STRNG_Enum; + +/* =========================================== GPIO PADREGF PAD22INPEN [17..17] ============================================ */ +typedef enum { /*!< GPIO_PADREGF_PAD22INPEN */ + GPIO_PADREGF_PAD22INPEN_DIS = 0, /*!< DIS : Pad input disabled value. */ + GPIO_PADREGF_PAD22INPEN_EN = 1, /*!< EN : Pad input enabled value. */ +} GPIO_PADREGF_PAD22INPEN_Enum; + +/* ============================================ GPIO PADREGF PAD22PULL [16..16] ============================================ */ +typedef enum { /*!< GPIO_PADREGF_PAD22PULL */ + GPIO_PADREGF_PAD22PULL_DIS = 0, /*!< DIS : Pullup disabled value. */ + GPIO_PADREGF_PAD22PULL_EN = 1, /*!< EN : Pullup enabled value. */ +} GPIO_PADREGF_PAD22PULL_Enum; + +/* =========================================== GPIO PADREGF PAD21FNCSEL [11..13] =========================================== */ +typedef enum { /*!< GPIO_PADREGF_PAD21FNCSEL */ + GPIO_PADREGF_PAD21FNCSEL_SWDIO = 0, /*!< SWDIO : Configure as the serial wire debug data signal value. */ + GPIO_PADREGF_PAD21FNCSEL_NCE21 = 1, /*!< NCE21 : IOM/MSPI nCE group 21 value. */ + GPIO_PADREGF_PAD21FNCSEL_GPIO21 = 3, /*!< GPIO21 : Configure as GPIO21 value. */ + GPIO_PADREGF_PAD21FNCSEL_UART0RX = 4, /*!< UART0RX : Configure as UART0 RX input signal value. */ + GPIO_PADREGF_PAD21FNCSEL_UART1RX = 5, /*!< UART1RX : Configure as UART1 RX input signal value. */ + GPIO_PADREGF_PAD21FNCSEL_I2SBCLK = 6, /*!< I2SBCLK : I2S byte clock input value. */ + GPIO_PADREGF_PAD21FNCSEL_UA1CTS = 7, /*!< UA1CTS : Configure as UART1 CTS input signal value. */ +} GPIO_PADREGF_PAD21FNCSEL_Enum; + +/* =========================================== GPIO PADREGF PAD21STRNG [10..10] ============================================ */ +typedef enum { /*!< GPIO_PADREGF_PAD21STRNG */ + GPIO_PADREGF_PAD21STRNG_LOW = 0, /*!< LOW : Low drive strength value. */ + GPIO_PADREGF_PAD21STRNG_HIGH = 1, /*!< HIGH : High drive strength value. */ +} GPIO_PADREGF_PAD21STRNG_Enum; + +/* ============================================ GPIO PADREGF PAD21INPEN [9..9] ============================================= */ +typedef enum { /*!< GPIO_PADREGF_PAD21INPEN */ + GPIO_PADREGF_PAD21INPEN_DIS = 0, /*!< DIS : Pad input disabled value. */ + GPIO_PADREGF_PAD21INPEN_EN = 1, /*!< EN : Pad input enabled value. */ +} GPIO_PADREGF_PAD21INPEN_Enum; + +/* ============================================= GPIO PADREGF PAD21PULL [8..8] ============================================= */ +typedef enum { /*!< GPIO_PADREGF_PAD21PULL */ + GPIO_PADREGF_PAD21PULL_DIS = 0, /*!< DIS : Pullup disabled value. */ + GPIO_PADREGF_PAD21PULL_EN = 1, /*!< EN : Pullup enabled value. */ +} GPIO_PADREGF_PAD21PULL_Enum; + +/* ============================================ GPIO PADREGF PAD20FNCSEL [3..5] ============================================ */ +typedef enum { /*!< GPIO_PADREGF_PAD20FNCSEL */ + GPIO_PADREGF_PAD20FNCSEL_SWDCK = 0, /*!< SWDCK : Configure as the serial wire debug clock signal value. */ + GPIO_PADREGF_PAD20FNCSEL_NCE20 = 1, /*!< NCE20 : IOM/MSPI nCE group 20 value. */ + GPIO_PADREGF_PAD20FNCSEL_GPIO20 = 3, /*!< GPIO20 : Configure as GPIO20 value. */ + GPIO_PADREGF_PAD20FNCSEL_UART0TX = 4, /*!< UART0TX : Configure as UART0 TX output signal value. */ + GPIO_PADREGF_PAD20FNCSEL_UART1TX = 5, /*!< UART1TX : Configure as UART1 TX output signal value. */ + GPIO_PADREGF_PAD20FNCSEL_I2SBCLK = 6, /*!< I2SBCLK : I2S byte clock input value. */ + GPIO_PADREGF_PAD20FNCSEL_UA1RTS = 7, /*!< UA1RTS : Configure as UART1 RTS output signal value. */ +} GPIO_PADREGF_PAD20FNCSEL_Enum; + +/* ============================================ GPIO PADREGF PAD20STRNG [2..2] ============================================= */ +typedef enum { /*!< GPIO_PADREGF_PAD20STRNG */ + GPIO_PADREGF_PAD20STRNG_LOW = 0, /*!< LOW : Low drive strength value. */ + GPIO_PADREGF_PAD20STRNG_HIGH = 1, /*!< HIGH : High drive strength value. */ +} GPIO_PADREGF_PAD20STRNG_Enum; + +/* ============================================ GPIO PADREGF PAD20INPEN [1..1] ============================================= */ +typedef enum { /*!< GPIO_PADREGF_PAD20INPEN */ + GPIO_PADREGF_PAD20INPEN_DIS = 0, /*!< DIS : Pad input disabled value. */ + GPIO_PADREGF_PAD20INPEN_EN = 1, /*!< EN : Pad input enabled value. */ +} GPIO_PADREGF_PAD20INPEN_Enum; + +/* ============================================= GPIO PADREGF PAD20PULL [0..0] ============================================= */ +typedef enum { /*!< GPIO_PADREGF_PAD20PULL */ + GPIO_PADREGF_PAD20PULL_DIS = 0, /*!< DIS : Pulldown disabled value. */ + GPIO_PADREGF_PAD20PULL_EN = 1, /*!< EN : Pulldown enabled value. */ +} GPIO_PADREGF_PAD20PULL_Enum; + +/* ======================================================== PADREGG ======================================================== */ +/* ============================================ GPIO PADREGG PAD27RSEL [30..31] ============================================ */ +typedef enum { /*!< GPIO_PADREGG_PAD27RSEL */ + GPIO_PADREGG_PAD27RSEL_PULL1_5K = 0, /*!< PULL1_5K : Pullup is ~1.5 KOhms value. */ + GPIO_PADREGG_PAD27RSEL_PULL6K = 1, /*!< PULL6K : Pullup is ~6 KOhms value. */ + GPIO_PADREGG_PAD27RSEL_PULL12K = 2, /*!< PULL12K : Pullup is ~12 KOhms value. */ + GPIO_PADREGG_PAD27RSEL_PULL24K = 3, /*!< PULL24K : Pullup is ~24 KOhms value. */ +} GPIO_PADREGG_PAD27RSEL_Enum; + +/* =========================================== GPIO PADREGG PAD27FNCSEL [27..29] =========================================== */ +typedef enum { /*!< GPIO_PADREGG_PAD27FNCSEL */ + GPIO_PADREGG_PAD27FNCSEL_UART0RX = 0, /*!< UART0RX : Configure as UART0 RX input signal value. */ + GPIO_PADREGG_PAD27FNCSEL_NCE27 = 1, /*!< NCE27 : IOM/MSPI nCE group 27 value. */ + GPIO_PADREGG_PAD27FNCSEL_CT5 = 2, /*!< CT5 : CTIMER connection 5 value. */ + GPIO_PADREGG_PAD27FNCSEL_GPIO27 = 3, /*!< GPIO27 : Configure as GPIO27 value. */ + GPIO_PADREGG_PAD27FNCSEL_M2SCL = 4, /*!< M2SCL : Configure as I2C clock I/O signal from IOMSTR2 value. */ + GPIO_PADREGG_PAD27FNCSEL_M2SCK = 5, /*!< M2SCK : Configure as SPI clock output signal from IOMSTR2 value. */ +} GPIO_PADREGG_PAD27FNCSEL_Enum; + +/* =========================================== GPIO PADREGG PAD27STRNG [26..26] ============================================ */ +typedef enum { /*!< GPIO_PADREGG_PAD27STRNG */ + GPIO_PADREGG_PAD27STRNG_LOW = 0, /*!< LOW : Low drive strength value. */ + GPIO_PADREGG_PAD27STRNG_HIGH = 1, /*!< HIGH : High drive strength value. */ +} GPIO_PADREGG_PAD27STRNG_Enum; + +/* =========================================== GPIO PADREGG PAD27INPEN [25..25] ============================================ */ +typedef enum { /*!< GPIO_PADREGG_PAD27INPEN */ + GPIO_PADREGG_PAD27INPEN_DIS = 0, /*!< DIS : Pad input disabled value. */ + GPIO_PADREGG_PAD27INPEN_EN = 1, /*!< EN : Pad input enabled value. */ +} GPIO_PADREGG_PAD27INPEN_Enum; + +/* ============================================ GPIO PADREGG PAD27PULL [24..24] ============================================ */ +typedef enum { /*!< GPIO_PADREGG_PAD27PULL */ + GPIO_PADREGG_PAD27PULL_DIS = 0, /*!< DIS : Pullup disabled value. */ + GPIO_PADREGG_PAD27PULL_EN = 1, /*!< EN : Pullup enabled value. */ +} GPIO_PADREGG_PAD27PULL_Enum; + +/* =========================================== GPIO PADREGG PAD26FNCSEL [19..21] =========================================== */ +typedef enum { /*!< GPIO_PADREGG_PAD26FNCSEL */ + GPIO_PADREGG_PAD26FNCSEL_EXTHF = 0, /*!< EXTHF : Configure as the external HFRC oscillator input value. */ + GPIO_PADREGG_PAD26FNCSEL_NCE26 = 1, /*!< NCE26 : IOM/MSPI nCE group 26 value. */ + GPIO_PADREGG_PAD26FNCSEL_CT3 = 2, /*!< CT3 : CTIMER connection 3 value. */ + GPIO_PADREGG_PAD26FNCSEL_GPIO26 = 3, /*!< GPIO26 : Configure as GPIO26 value. */ + GPIO_PADREGG_PAD26FNCSEL_SCCRST = 4, /*!< SCCRST : SCARD reset output value. */ + GPIO_PADREGG_PAD26FNCSEL_MSPI1 = 5, /*!< MSPI1 : MSPI data connection 1 value. */ + GPIO_PADREGG_PAD26FNCSEL_UART0TX = 6, /*!< UART0TX : Configure as UART0 TX output signal value. */ + GPIO_PADREGG_PAD26FNCSEL_UA1CTS = 7, /*!< UA1CTS : Configure as UART1 CTS input signal value. */ +} GPIO_PADREGG_PAD26FNCSEL_Enum; + +/* =========================================== GPIO PADREGG PAD26STRNG [18..18] ============================================ */ +typedef enum { /*!< GPIO_PADREGG_PAD26STRNG */ + GPIO_PADREGG_PAD26STRNG_LOW = 0, /*!< LOW : Low drive strength value. */ + GPIO_PADREGG_PAD26STRNG_HIGH = 1, /*!< HIGH : High drive strength value. */ +} GPIO_PADREGG_PAD26STRNG_Enum; + +/* =========================================== GPIO PADREGG PAD26INPEN [17..17] ============================================ */ +typedef enum { /*!< GPIO_PADREGG_PAD26INPEN */ + GPIO_PADREGG_PAD26INPEN_DIS = 0, /*!< DIS : Pad input disabled value. */ + GPIO_PADREGG_PAD26INPEN_EN = 1, /*!< EN : Pad input enabled value. */ +} GPIO_PADREGG_PAD26INPEN_Enum; + +/* ============================================ GPIO PADREGG PAD26PULL [16..16] ============================================ */ +typedef enum { /*!< GPIO_PADREGG_PAD26PULL */ + GPIO_PADREGG_PAD26PULL_DIS = 0, /*!< DIS : Pullup disabled value. */ + GPIO_PADREGG_PAD26PULL_EN = 1, /*!< EN : Pullup enabled value. */ +} GPIO_PADREGG_PAD26PULL_Enum; + +/* ============================================ GPIO PADREGG PAD25RSEL [14..15] ============================================ */ +typedef enum { /*!< GPIO_PADREGG_PAD25RSEL */ + GPIO_PADREGG_PAD25RSEL_PULL1_5K = 0, /*!< PULL1_5K : Pullup is ~1.5 KOhms value. */ + GPIO_PADREGG_PAD25RSEL_PULL6K = 1, /*!< PULL6K : Pullup is ~6 KOhms value. */ + GPIO_PADREGG_PAD25RSEL_PULL12K = 2, /*!< PULL12K : Pullup is ~12 KOhms value. */ + GPIO_PADREGG_PAD25RSEL_PULL24K = 3, /*!< PULL24K : Pullup is ~24 KOhms value. */ +} GPIO_PADREGG_PAD25RSEL_Enum; + +/* =========================================== GPIO PADREGG PAD25FNCSEL [11..13] =========================================== */ +typedef enum { /*!< GPIO_PADREGG_PAD25FNCSEL */ + GPIO_PADREGG_PAD25FNCSEL_UART1RX = 0, /*!< UART1RX : Configure as UART1 RX input signal value. */ + GPIO_PADREGG_PAD25FNCSEL_NCE25 = 1, /*!< NCE25 : IOM/MSPI nCE group 25 value. */ + GPIO_PADREGG_PAD25FNCSEL_CT1 = 2, /*!< CT1 : CTIMER connection 1 value. */ + GPIO_PADREGG_PAD25FNCSEL_GPIO25 = 3, /*!< GPIO25 : Configure as GPIO25 value. */ + GPIO_PADREGG_PAD25FNCSEL_M2SDAWIR3 = 4, /*!< M2SDAWIR3 : Configure as the IOMSTR2 I2C SDA or SPI WIR3 signal + value. */ + GPIO_PADREGG_PAD25FNCSEL_M2MISO = 5, /*!< M2MISO : Configure as the IOMSTR2 SPI MISO input signal value. */ +} GPIO_PADREGG_PAD25FNCSEL_Enum; + +/* =========================================== GPIO PADREGG PAD25STRNG [10..10] ============================================ */ +typedef enum { /*!< GPIO_PADREGG_PAD25STRNG */ + GPIO_PADREGG_PAD25STRNG_LOW = 0, /*!< LOW : Low drive strength value. */ + GPIO_PADREGG_PAD25STRNG_HIGH = 1, /*!< HIGH : High drive strength value. */ +} GPIO_PADREGG_PAD25STRNG_Enum; + +/* ============================================ GPIO PADREGG PAD25INPEN [9..9] ============================================= */ +typedef enum { /*!< GPIO_PADREGG_PAD25INPEN */ + GPIO_PADREGG_PAD25INPEN_DIS = 0, /*!< DIS : Pad input disabled value. */ + GPIO_PADREGG_PAD25INPEN_EN = 1, /*!< EN : Pad input enabled value. */ +} GPIO_PADREGG_PAD25INPEN_Enum; + +/* ============================================= GPIO PADREGG PAD25PULL [8..8] ============================================= */ +typedef enum { /*!< GPIO_PADREGG_PAD25PULL */ + GPIO_PADREGG_PAD25PULL_DIS = 0, /*!< DIS : Pullup disabled value. */ + GPIO_PADREGG_PAD25PULL_EN = 1, /*!< EN : Pullup enabled value. */ +} GPIO_PADREGG_PAD25PULL_Enum; + +/* ============================================ GPIO PADREGG PAD24FNCSEL [3..5] ============================================ */ +typedef enum { /*!< GPIO_PADREGG_PAD24FNCSEL */ + GPIO_PADREGG_PAD24FNCSEL_UART1TX = 0, /*!< UART1TX : Configure as UART1 TX output signal value. */ + GPIO_PADREGG_PAD24FNCSEL_NCE24 = 1, /*!< NCE24 : IOM/MSPI nCE group 24 value. */ + GPIO_PADREGG_PAD24FNCSEL_MSPI8 = 2, /*!< MSPI8 : MSPI data connection 8 value. */ + GPIO_PADREGG_PAD24FNCSEL_GPIO24 = 3, /*!< GPIO24 : Configure as GPIO24 value. */ + GPIO_PADREGG_PAD24FNCSEL_UA0CTS = 4, /*!< UA0CTS : Configure as UART0 CTS input signal value. */ + GPIO_PADREGG_PAD24FNCSEL_CT21 = 5, /*!< CT21 : CTIMER connection 21 value. */ + GPIO_PADREGG_PAD24FNCSEL_32kHzXT = 6, /*!< 32kHzXT : Configure as the 32kHz crystal output signal value. */ + GPIO_PADREGG_PAD24FNCSEL_SWO = 7, /*!< SWO : Configure as the serial trace data output signal value. */ +} GPIO_PADREGG_PAD24FNCSEL_Enum; + +/* ============================================ GPIO PADREGG PAD24STRNG [2..2] ============================================= */ +typedef enum { /*!< GPIO_PADREGG_PAD24STRNG */ + GPIO_PADREGG_PAD24STRNG_LOW = 0, /*!< LOW : Low drive strength value. */ + GPIO_PADREGG_PAD24STRNG_HIGH = 1, /*!< HIGH : High drive strength value. */ +} GPIO_PADREGG_PAD24STRNG_Enum; + +/* ============================================ GPIO PADREGG PAD24INPEN [1..1] ============================================= */ +typedef enum { /*!< GPIO_PADREGG_PAD24INPEN */ + GPIO_PADREGG_PAD24INPEN_DIS = 0, /*!< DIS : Pad input disabled value. */ + GPIO_PADREGG_PAD24INPEN_EN = 1, /*!< EN : Pad input enabled value. */ +} GPIO_PADREGG_PAD24INPEN_Enum; + +/* ============================================= GPIO PADREGG PAD24PULL [0..0] ============================================= */ +typedef enum { /*!< GPIO_PADREGG_PAD24PULL */ + GPIO_PADREGG_PAD24PULL_DIS = 0, /*!< DIS : Pullup disabled value. */ + GPIO_PADREGG_PAD24PULL_EN = 1, /*!< EN : Pullup enabled value. */ +} GPIO_PADREGG_PAD24PULL_Enum; + +/* ======================================================== PADREGH ======================================================== */ +/* =========================================== GPIO PADREGH PAD31FNCSEL [27..29] =========================================== */ +typedef enum { /*!< GPIO_PADREGH_PAD31FNCSEL */ + GPIO_PADREGH_PAD31FNCSEL_ADCSE3 = 0, /*!< ADCSE3 : Configure as the analog input for ADC single ended + input 3 value. */ + GPIO_PADREGH_PAD31FNCSEL_NCE31 = 1, /*!< NCE31 : IOM/MSPI nCE group 31 value. */ + GPIO_PADREGH_PAD31FNCSEL_CT13 = 2, /*!< CT13 : CTIMER connection 13 value. */ + GPIO_PADREGH_PAD31FNCSEL_GPIO31 = 3, /*!< GPIO31 : Configure as GPIO31 value. */ + GPIO_PADREGH_PAD31FNCSEL_UART0RX = 4, /*!< UART0RX : Configure as the UART0 RX input signal value. */ + GPIO_PADREGH_PAD31FNCSEL_SCCCLK = 5, /*!< SCCCLK : SCARD serial clock output value. */ + GPIO_PADREGH_PAD31FNCSEL_UA1RTS = 7, /*!< UA1RTS : Configure as UART1 RTS output signal value. */ +} GPIO_PADREGH_PAD31FNCSEL_Enum; + +/* =========================================== GPIO PADREGH PAD31STRNG [26..26] ============================================ */ +typedef enum { /*!< GPIO_PADREGH_PAD31STRNG */ + GPIO_PADREGH_PAD31STRNG_LOW = 0, /*!< LOW : Low drive strength value. */ + GPIO_PADREGH_PAD31STRNG_HIGH = 1, /*!< HIGH : High drive strength value. */ +} GPIO_PADREGH_PAD31STRNG_Enum; + +/* =========================================== GPIO PADREGH PAD31INPEN [25..25] ============================================ */ +typedef enum { /*!< GPIO_PADREGH_PAD31INPEN */ + GPIO_PADREGH_PAD31INPEN_DIS = 0, /*!< DIS : Pad input disabled value. */ + GPIO_PADREGH_PAD31INPEN_EN = 1, /*!< EN : Pad input enabled value. */ +} GPIO_PADREGH_PAD31INPEN_Enum; + +/* ============================================ GPIO PADREGH PAD31PULL [24..24] ============================================ */ +typedef enum { /*!< GPIO_PADREGH_PAD31PULL */ + GPIO_PADREGH_PAD31PULL_DIS = 0, /*!< DIS : Pullup disabled value. */ + GPIO_PADREGH_PAD31PULL_EN = 1, /*!< EN : Pullup enabled value. */ +} GPIO_PADREGH_PAD31PULL_Enum; + +/* =========================================== GPIO PADREGH PAD30FNCSEL [19..21] =========================================== */ +typedef enum { /*!< GPIO_PADREGH_PAD30FNCSEL */ + GPIO_PADREGH_PAD30FNCSEL_ANATEST1 = 0, /*!< ANATEST1 : Configure as the ANATEST1 I/O signal value. */ + GPIO_PADREGH_PAD30FNCSEL_NCE30 = 1, /*!< NCE30 : IOM/MSPI nCE group 30 value. */ + GPIO_PADREGH_PAD30FNCSEL_CT11 = 2, /*!< CT11 : CTIMER connection 11 value. */ + GPIO_PADREGH_PAD30FNCSEL_GPIO30 = 3, /*!< GPIO30 : Configure as GPIO30 value. */ + GPIO_PADREGH_PAD30FNCSEL_UART0TX = 4, /*!< UART0TX : Configure as UART0 TX output signal value. */ + GPIO_PADREGH_PAD30FNCSEL_UA1RTS = 5, /*!< UA1RTS : Configure as UART1 RTS output signal value. */ + GPIO_PADREGH_PAD30FNCSEL_I2S_DAT = 7, /*!< I2S_DAT : Configure as the PDM I2S Data output signal value. */ +} GPIO_PADREGH_PAD30FNCSEL_Enum; + +/* =========================================== GPIO PADREGH PAD30STRNG [18..18] ============================================ */ +typedef enum { /*!< GPIO_PADREGH_PAD30STRNG */ + GPIO_PADREGH_PAD30STRNG_LOW = 0, /*!< LOW : Low drive strength value. */ + GPIO_PADREGH_PAD30STRNG_HIGH = 1, /*!< HIGH : High drive strength value. */ +} GPIO_PADREGH_PAD30STRNG_Enum; + +/* =========================================== GPIO PADREGH PAD30INPEN [17..17] ============================================ */ +typedef enum { /*!< GPIO_PADREGH_PAD30INPEN */ + GPIO_PADREGH_PAD30INPEN_DIS = 0, /*!< DIS : Pad input disabled value. */ + GPIO_PADREGH_PAD30INPEN_EN = 1, /*!< EN : Pad input enabled value. */ +} GPIO_PADREGH_PAD30INPEN_Enum; + +/* ============================================ GPIO PADREGH PAD30PULL [16..16] ============================================ */ +typedef enum { /*!< GPIO_PADREGH_PAD30PULL */ + GPIO_PADREGH_PAD30PULL_DIS = 0, /*!< DIS : Pullup disabled value. */ + GPIO_PADREGH_PAD30PULL_EN = 1, /*!< EN : Pullup enabled value. */ +} GPIO_PADREGH_PAD30PULL_Enum; + +/* =========================================== GPIO PADREGH PAD29FNCSEL [11..13] =========================================== */ +typedef enum { /*!< GPIO_PADREGH_PAD29FNCSEL */ + GPIO_PADREGH_PAD29FNCSEL_ADCSE1 = 0, /*!< ADCSE1 : Configure as the analog input for ADC single ended + input 1 value. */ + GPIO_PADREGH_PAD29FNCSEL_NCE29 = 1, /*!< NCE29 : IOM/MSPI nCE group 29 value. */ + GPIO_PADREGH_PAD29FNCSEL_CT9 = 2, /*!< CT9 : CTIMER connection 9 value. */ + GPIO_PADREGH_PAD29FNCSEL_GPIO29 = 3, /*!< GPIO29 : Configure as GPIO29 value. */ + GPIO_PADREGH_PAD29FNCSEL_UA0CTS = 4, /*!< UA0CTS : Configure as the UART0 CTS input signal value. */ + GPIO_PADREGH_PAD29FNCSEL_UA1CTS = 5, /*!< UA1CTS : Configure as the UART1 CTS input signal value. */ + GPIO_PADREGH_PAD29FNCSEL_UART0RX = 6, /*!< UART0RX : Configure as the UART0 RX input signal value. */ + GPIO_PADREGH_PAD29FNCSEL_PDM_DATA = 7, /*!< PDM_DATA : Configure as PDM DATA input value. */ +} GPIO_PADREGH_PAD29FNCSEL_Enum; + +/* =========================================== GPIO PADREGH PAD29STRNG [10..10] ============================================ */ +typedef enum { /*!< GPIO_PADREGH_PAD29STRNG */ + GPIO_PADREGH_PAD29STRNG_LOW = 0, /*!< LOW : Low drive strength value. */ + GPIO_PADREGH_PAD29STRNG_HIGH = 1, /*!< HIGH : High drive strength value. */ +} GPIO_PADREGH_PAD29STRNG_Enum; + +/* ============================================ GPIO PADREGH PAD29INPEN [9..9] ============================================= */ +typedef enum { /*!< GPIO_PADREGH_PAD29INPEN */ + GPIO_PADREGH_PAD29INPEN_DIS = 0, /*!< DIS : Pad input disabled value. */ + GPIO_PADREGH_PAD29INPEN_EN = 1, /*!< EN : Pad input enabled value. */ +} GPIO_PADREGH_PAD29INPEN_Enum; + +/* ============================================= GPIO PADREGH PAD29PULL [8..8] ============================================= */ +typedef enum { /*!< GPIO_PADREGH_PAD29PULL */ + GPIO_PADREGH_PAD29PULL_DIS = 0, /*!< DIS : Pullup disabled value. */ + GPIO_PADREGH_PAD29PULL_EN = 1, /*!< EN : Pullup enabled value. */ +} GPIO_PADREGH_PAD29PULL_Enum; + +/* ============================================ GPIO PADREGH PAD28FNCSEL [3..5] ============================================ */ +typedef enum { /*!< GPIO_PADREGH_PAD28FNCSEL */ + GPIO_PADREGH_PAD28FNCSEL_I2S_WCLK = 0, /*!< I2S_WCLK : Configure as the PDM I2S Word Clock input value. */ + GPIO_PADREGH_PAD28FNCSEL_NCE28 = 1, /*!< NCE28 : IOM/MSPI nCE group 28 value. */ + GPIO_PADREGH_PAD28FNCSEL_CT7 = 2, /*!< CT7 : CTIMER connection 7 value. */ + GPIO_PADREGH_PAD28FNCSEL_GPIO28 = 3, /*!< GPIO28 : Configure as GPIO28 value. */ + GPIO_PADREGH_PAD28FNCSEL_M2MOSI = 5, /*!< M2MOSI : Configure as the IOMSTR2 SPI MOSI output signal value. */ + GPIO_PADREGH_PAD28FNCSEL_UART0TX = 6, /*!< UART0TX : Configure as the UART0 TX output signal value. */ +} GPIO_PADREGH_PAD28FNCSEL_Enum; + +/* ============================================ GPIO PADREGH PAD28STRNG [2..2] ============================================= */ +typedef enum { /*!< GPIO_PADREGH_PAD28STRNG */ + GPIO_PADREGH_PAD28STRNG_LOW = 0, /*!< LOW : Low drive strength value. */ + GPIO_PADREGH_PAD28STRNG_HIGH = 1, /*!< HIGH : High drive strength value. */ +} GPIO_PADREGH_PAD28STRNG_Enum; + +/* ============================================ GPIO PADREGH PAD28INPEN [1..1] ============================================= */ +typedef enum { /*!< GPIO_PADREGH_PAD28INPEN */ + GPIO_PADREGH_PAD28INPEN_DIS = 0, /*!< DIS : Pad input disabled value. */ + GPIO_PADREGH_PAD28INPEN_EN = 1, /*!< EN : Pad input enabled value. */ +} GPIO_PADREGH_PAD28INPEN_Enum; + +/* ============================================= GPIO PADREGH PAD28PULL [0..0] ============================================= */ +typedef enum { /*!< GPIO_PADREGH_PAD28PULL */ + GPIO_PADREGH_PAD28PULL_DIS = 0, /*!< DIS : Pullup disabled value. */ + GPIO_PADREGH_PAD28PULL_EN = 1, /*!< EN : Pullup enabled value. */ +} GPIO_PADREGH_PAD28PULL_Enum; + +/* ======================================================== PADREGI ======================================================== */ +/* =========================================== GPIO PADREGI PAD35FNCSEL [27..29] =========================================== */ +typedef enum { /*!< GPIO_PADREGI_PAD35FNCSEL */ + GPIO_PADREGI_PAD35FNCSEL_ADCSE7 = 0, /*!< ADCSE7 : Configure as the analog input for ADC single ended + input 7 value. */ + GPIO_PADREGI_PAD35FNCSEL_NCE35 = 1, /*!< NCE35 : IOM/MSPI nCE group 35 value. */ + GPIO_PADREGI_PAD35FNCSEL_UART1TX = 2, /*!< UART1TX : Configure as the UART1 TX signal value. */ + GPIO_PADREGI_PAD35FNCSEL_GPIO35 = 3, /*!< GPIO35 : Configure as GPIO35 value. */ + GPIO_PADREGI_PAD35FNCSEL_I2SDAT = 4, /*!< I2SDAT : I2S serial data output value. */ + GPIO_PADREGI_PAD35FNCSEL_CT27 = 5, /*!< CT27 : CTIMER connection 27 value. */ + GPIO_PADREGI_PAD35FNCSEL_UA0RTS = 6, /*!< UA0RTS : Configure as the UART0 RTS output value. */ +} GPIO_PADREGI_PAD35FNCSEL_Enum; + +/* =========================================== GPIO PADREGI PAD35STRNG [26..26] ============================================ */ +typedef enum { /*!< GPIO_PADREGI_PAD35STRNG */ + GPIO_PADREGI_PAD35STRNG_LOW = 0, /*!< LOW : Low drive strength value. */ + GPIO_PADREGI_PAD35STRNG_HIGH = 1, /*!< HIGH : High drive strength value. */ +} GPIO_PADREGI_PAD35STRNG_Enum; + +/* =========================================== GPIO PADREGI PAD35INPEN [25..25] ============================================ */ +typedef enum { /*!< GPIO_PADREGI_PAD35INPEN */ + GPIO_PADREGI_PAD35INPEN_DIS = 0, /*!< DIS : Pad input disabled value. */ + GPIO_PADREGI_PAD35INPEN_EN = 1, /*!< EN : Pad input enabled value. */ +} GPIO_PADREGI_PAD35INPEN_Enum; + +/* ============================================ GPIO PADREGI PAD35PULL [24..24] ============================================ */ +typedef enum { /*!< GPIO_PADREGI_PAD35PULL */ + GPIO_PADREGI_PAD35PULL_DIS = 0, /*!< DIS : Pullup disabled value. */ + GPIO_PADREGI_PAD35PULL_EN = 1, /*!< EN : Pullup enabled value. */ +} GPIO_PADREGI_PAD35PULL_Enum; + +/* =========================================== GPIO PADREGI PAD34FNCSEL [19..21] =========================================== */ +typedef enum { /*!< GPIO_PADREGI_PAD34FNCSEL */ + GPIO_PADREGI_PAD34FNCSEL_ADCSE6 = 0, /*!< ADCSE6 : Configure as the analog input for ADC single ended + input 6 value. */ + GPIO_PADREGI_PAD34FNCSEL_NCE34 = 1, /*!< NCE34 : IOM/MSPI nCE group 34 value. */ + GPIO_PADREGI_PAD34FNCSEL_UA1RTS = 2, /*!< UA1RTS : Configure as the UART1 RTS output value. */ + GPIO_PADREGI_PAD34FNCSEL_GPIO34 = 3, /*!< GPIO34 : Configure as GPIO34 value. */ + GPIO_PADREGI_PAD34FNCSEL_CMPRF2 = 4, /*!< CMPRF2 : Configure as the analog comparator reference 2 signal + value. */ + GPIO_PADREGI_PAD34FNCSEL_UA0RTS = 5, /*!< UA0RTS : Configure as the UART0 RTS output value. */ + GPIO_PADREGI_PAD34FNCSEL_UART0RX = 6, /*!< UART0RX : Configure as the UART0 RX input value. */ + GPIO_PADREGI_PAD34FNCSEL_PDMDATA = 7, /*!< PDMDATA : PDM serial data input value. */ +} GPIO_PADREGI_PAD34FNCSEL_Enum; + +/* =========================================== GPIO PADREGI PAD34STRNG [18..18] ============================================ */ +typedef enum { /*!< GPIO_PADREGI_PAD34STRNG */ + GPIO_PADREGI_PAD34STRNG_LOW = 0, /*!< LOW : Low drive strength value. */ + GPIO_PADREGI_PAD34STRNG_HIGH = 1, /*!< HIGH : High drive strength value. */ +} GPIO_PADREGI_PAD34STRNG_Enum; + +/* =========================================== GPIO PADREGI PAD34INPEN [17..17] ============================================ */ +typedef enum { /*!< GPIO_PADREGI_PAD34INPEN */ + GPIO_PADREGI_PAD34INPEN_DIS = 0, /*!< DIS : Pad input disabled value. */ + GPIO_PADREGI_PAD34INPEN_EN = 1, /*!< EN : Pad input enabled value. */ +} GPIO_PADREGI_PAD34INPEN_Enum; + +/* ============================================ GPIO PADREGI PAD34PULL [16..16] ============================================ */ +typedef enum { /*!< GPIO_PADREGI_PAD34PULL */ + GPIO_PADREGI_PAD34PULL_DIS = 0, /*!< DIS : Pullup disabled value. */ + GPIO_PADREGI_PAD34PULL_EN = 1, /*!< EN : Pullup enabled value. */ +} GPIO_PADREGI_PAD34PULL_Enum; + +/* =========================================== GPIO PADREGI PAD33FNCSEL [11..13] =========================================== */ +typedef enum { /*!< GPIO_PADREGI_PAD33FNCSEL */ + GPIO_PADREGI_PAD33FNCSEL_ADCSE5 = 0, /*!< ADCSE5 : Configure as the analog ADC single ended port 5 input + signal value. */ + GPIO_PADREGI_PAD33FNCSEL_NCE33 = 1, /*!< NCE33 : IOM/MSPI nCE group 33 value. */ + GPIO_PADREGI_PAD33FNCSEL_32kHzXT = 2, /*!< 32kHzXT : Configure as the 32kHz crystal output signal value. */ + GPIO_PADREGI_PAD33FNCSEL_GPIO33 = 3, /*!< GPIO33 : Configure as GPIO33 value. */ + GPIO_PADREGI_PAD33FNCSEL_UA0CTS = 5, /*!< UA0CTS : Configure as the UART0 CTS input value. */ + GPIO_PADREGI_PAD33FNCSEL_CT23 = 6, /*!< CT23 : CTIMER connection 23 value. */ + GPIO_PADREGI_PAD33FNCSEL_SWO = 7, /*!< SWO : Configure as the serial trace data output signal value. */ +} GPIO_PADREGI_PAD33FNCSEL_Enum; + +/* =========================================== GPIO PADREGI PAD33STRNG [10..10] ============================================ */ +typedef enum { /*!< GPIO_PADREGI_PAD33STRNG */ + GPIO_PADREGI_PAD33STRNG_LOW = 0, /*!< LOW : Low drive strength value. */ + GPIO_PADREGI_PAD33STRNG_HIGH = 1, /*!< HIGH : High drive strength value. */ +} GPIO_PADREGI_PAD33STRNG_Enum; + +/* ============================================ GPIO PADREGI PAD33INPEN [9..9] ============================================= */ +typedef enum { /*!< GPIO_PADREGI_PAD33INPEN */ + GPIO_PADREGI_PAD33INPEN_DIS = 0, /*!< DIS : Pad input disabled value. */ + GPIO_PADREGI_PAD33INPEN_EN = 1, /*!< EN : Pad input enabled value. */ +} GPIO_PADREGI_PAD33INPEN_Enum; + +/* ============================================= GPIO PADREGI PAD33PULL [8..8] ============================================= */ +typedef enum { /*!< GPIO_PADREGI_PAD33PULL */ + GPIO_PADREGI_PAD33PULL_DIS = 0, /*!< DIS : Pullup disabled value. */ + GPIO_PADREGI_PAD33PULL_EN = 1, /*!< EN : Pullup enabled value. */ +} GPIO_PADREGI_PAD33PULL_Enum; + +/* ============================================ GPIO PADREGI PAD32FNCSEL [3..5] ============================================ */ +typedef enum { /*!< GPIO_PADREGI_PAD32FNCSEL */ + GPIO_PADREGI_PAD32FNCSEL_ADCSE4 = 0, /*!< ADCSE4 : Configure as the analog input for ADC single ended + input 4 value. */ + GPIO_PADREGI_PAD32FNCSEL_NCE32 = 1, /*!< NCE32 : IOM/MSPI nCE group 32 value. */ + GPIO_PADREGI_PAD32FNCSEL_CT15 = 2, /*!< CT15 : CTIMER connection 15 value. */ + GPIO_PADREGI_PAD32FNCSEL_GPIO32 = 3, /*!< GPIO32 : Configure as GPIO32 value. */ + GPIO_PADREGI_PAD32FNCSEL_SCCIO = 4, /*!< SCCIO : SCARD serial data input/output value. */ + GPIO_PADREGI_PAD32FNCSEL_EXTLF = 5, /*!< EXTLF : External input to the LFRC oscillator value. */ + GPIO_PADREGI_PAD32FNCSEL_UA1CTS = 7, /*!< UA1CTS : Configure as the UART1 CTS input value. */ +} GPIO_PADREGI_PAD32FNCSEL_Enum; + +/* ============================================ GPIO PADREGI PAD32STRNG [2..2] ============================================= */ +typedef enum { /*!< GPIO_PADREGI_PAD32STRNG */ + GPIO_PADREGI_PAD32STRNG_LOW = 0, /*!< LOW : Low drive strength value. */ + GPIO_PADREGI_PAD32STRNG_HIGH = 1, /*!< HIGH : High drive strength value. */ +} GPIO_PADREGI_PAD32STRNG_Enum; + +/* ============================================ GPIO PADREGI PAD32INPEN [1..1] ============================================= */ +typedef enum { /*!< GPIO_PADREGI_PAD32INPEN */ + GPIO_PADREGI_PAD32INPEN_DIS = 0, /*!< DIS : Pad input disabled value. */ + GPIO_PADREGI_PAD32INPEN_EN = 1, /*!< EN : Pad input enabled value. */ +} GPIO_PADREGI_PAD32INPEN_Enum; + +/* ============================================= GPIO PADREGI PAD32PULL [0..0] ============================================= */ +typedef enum { /*!< GPIO_PADREGI_PAD32PULL */ + GPIO_PADREGI_PAD32PULL_DIS = 0, /*!< DIS : Pullup disabled value. */ + GPIO_PADREGI_PAD32PULL_EN = 1, /*!< EN : Pullup enabled value. */ +} GPIO_PADREGI_PAD32PULL_Enum; + +/* ======================================================== PADREGJ ======================================================== */ +/* ============================================ GPIO PADREGJ PAD39RSEL [30..31] ============================================ */ +typedef enum { /*!< GPIO_PADREGJ_PAD39RSEL */ + GPIO_PADREGJ_PAD39RSEL_PULL1_5K = 0, /*!< PULL1_5K : Pullup is ~1.5 KOhms value. */ + GPIO_PADREGJ_PAD39RSEL_PULL6K = 1, /*!< PULL6K : Pullup is ~6 KOhms value. */ + GPIO_PADREGJ_PAD39RSEL_PULL12K = 2, /*!< PULL12K : Pullup is ~12 KOhms value. */ + GPIO_PADREGJ_PAD39RSEL_PULL24K = 3, /*!< PULL24K : Pullup is ~24 KOhms value. */ +} GPIO_PADREGJ_PAD39RSEL_Enum; + +/* =========================================== GPIO PADREGJ PAD39FNCSEL [27..29] =========================================== */ +typedef enum { /*!< GPIO_PADREGJ_PAD39FNCSEL */ + GPIO_PADREGJ_PAD39FNCSEL_UART0TX = 0, /*!< UART0TX : Configure as the UART0 TX output signal value. */ + GPIO_PADREGJ_PAD39FNCSEL_UART1TX = 1, /*!< UART1TX : Configure as the UART1 TX output signal value. */ + GPIO_PADREGJ_PAD39FNCSEL_CT25 = 2, /*!< CT25 : CTIMER connection 25 value. */ + GPIO_PADREGJ_PAD39FNCSEL_GPIO39 = 3, /*!< GPIO39 : Configure as GPIO39 value. */ + GPIO_PADREGJ_PAD39FNCSEL_M4SCL = 4, /*!< M4SCL : Configure as the IOMSTR4 I2C SCL signal value. */ + GPIO_PADREGJ_PAD39FNCSEL_M4SCK = 5, /*!< M4SCK : Configure as the IOMSTR4 SPI SCK signal value. */ +} GPIO_PADREGJ_PAD39FNCSEL_Enum; + +/* =========================================== GPIO PADREGJ PAD39STRNG [26..26] ============================================ */ +typedef enum { /*!< GPIO_PADREGJ_PAD39STRNG */ + GPIO_PADREGJ_PAD39STRNG_LOW = 0, /*!< LOW : Low drive strength value. */ + GPIO_PADREGJ_PAD39STRNG_HIGH = 1, /*!< HIGH : High drive strength value. */ +} GPIO_PADREGJ_PAD39STRNG_Enum; + +/* =========================================== GPIO PADREGJ PAD39INPEN [25..25] ============================================ */ +typedef enum { /*!< GPIO_PADREGJ_PAD39INPEN */ + GPIO_PADREGJ_PAD39INPEN_DIS = 0, /*!< DIS : Pad input disabled value. */ + GPIO_PADREGJ_PAD39INPEN_EN = 1, /*!< EN : Pad input enabled value. */ +} GPIO_PADREGJ_PAD39INPEN_Enum; + +/* ============================================ GPIO PADREGJ PAD39PULL [24..24] ============================================ */ +typedef enum { /*!< GPIO_PADREGJ_PAD39PULL */ + GPIO_PADREGJ_PAD39PULL_DIS = 0, /*!< DIS : Pullup disabled value. */ + GPIO_PADREGJ_PAD39PULL_EN = 1, /*!< EN : Pullup enabled value. */ +} GPIO_PADREGJ_PAD39PULL_Enum; + +/* =========================================== GPIO PADREGJ PAD38FNCSEL [19..21] =========================================== */ +typedef enum { /*!< GPIO_PADREGJ_PAD38FNCSEL */ + GPIO_PADREGJ_PAD38FNCSEL_TRIG3 = 0, /*!< TRIG3 : Configure as the ADC Trigger 3 signal value. */ + GPIO_PADREGJ_PAD38FNCSEL_NCE38 = 1, /*!< NCE38 : IOM/MSPI nCE group 38 value. */ + GPIO_PADREGJ_PAD38FNCSEL_UA0CTS = 2, /*!< UA0CTS : Configure as the UART0 CTS signal value. */ + GPIO_PADREGJ_PAD38FNCSEL_GPIO38 = 3, /*!< GPIO38 : Configure as GPIO38 value. */ + GPIO_PADREGJ_PAD38FNCSEL_M3MOSI = 5, /*!< M3MOSI : Configure as the IOMSTR3 SPI MOSI output signal value. */ + GPIO_PADREGJ_PAD38FNCSEL_UART1RX = 6, /*!< UART1RX : Configure as the UART1 RX input signal value. */ +} GPIO_PADREGJ_PAD38FNCSEL_Enum; + +/* =========================================== GPIO PADREGJ PAD38STRNG [18..18] ============================================ */ +typedef enum { /*!< GPIO_PADREGJ_PAD38STRNG */ + GPIO_PADREGJ_PAD38STRNG_LOW = 0, /*!< LOW : Low drive strength value. */ + GPIO_PADREGJ_PAD38STRNG_HIGH = 1, /*!< HIGH : High drive strength value. */ +} GPIO_PADREGJ_PAD38STRNG_Enum; + +/* =========================================== GPIO PADREGJ PAD38INPEN [17..17] ============================================ */ +typedef enum { /*!< GPIO_PADREGJ_PAD38INPEN */ + GPIO_PADREGJ_PAD38INPEN_DIS = 0, /*!< DIS : Pad input disabled value. */ + GPIO_PADREGJ_PAD38INPEN_EN = 1, /*!< EN : Pad input enabled value. */ +} GPIO_PADREGJ_PAD38INPEN_Enum; + +/* ============================================ GPIO PADREGJ PAD38PULL [16..16] ============================================ */ +typedef enum { /*!< GPIO_PADREGJ_PAD38PULL */ + GPIO_PADREGJ_PAD38PULL_DIS = 0, /*!< DIS : Pullup disabled value. */ + GPIO_PADREGJ_PAD38PULL_EN = 1, /*!< EN : Pullup enabled value. */ +} GPIO_PADREGJ_PAD38PULL_Enum; + +/* =========================================== GPIO PADREGJ PAD37PWRDN [15..15] ============================================ */ +typedef enum { /*!< GPIO_PADREGJ_PAD37PWRDN */ + GPIO_PADREGJ_PAD37PWRDN_DIS = 0, /*!< DIS : Power switch disabled value. */ + GPIO_PADREGJ_PAD37PWRDN_EN = 1, /*!< EN : Power switch enabled (switch to GND) value. */ +} GPIO_PADREGJ_PAD37PWRDN_Enum; + +/* =========================================== GPIO PADREGJ PAD37FNCSEL [11..13] =========================================== */ +typedef enum { /*!< GPIO_PADREGJ_PAD37FNCSEL */ + GPIO_PADREGJ_PAD37FNCSEL_TRIG2 = 0, /*!< TRIG2 : Configure as the ADC Trigger 2 signal value. */ + GPIO_PADREGJ_PAD37FNCSEL_NCE37 = 1, /*!< NCE37 : IOM/MSPI nCE group 37 value. */ + GPIO_PADREGJ_PAD37FNCSEL_UA0RTS = 2, /*!< UA0RTS : Configure as the UART0 RTS output signal value. */ + GPIO_PADREGJ_PAD37FNCSEL_GPIO37 = 3, /*!< GPIO37 : Configure as GPIO37 value. */ + GPIO_PADREGJ_PAD37FNCSEL_SCCIO = 4, /*!< SCCIO : SCARD serial data input/output value. */ + GPIO_PADREGJ_PAD37FNCSEL_UART1TX = 5, /*!< UART1TX : Configure as the UART1 TX output signal value. */ + GPIO_PADREGJ_PAD37FNCSEL_PDMCLK = 6, /*!< PDMCLK : Configure as the PDM CLK output signal value. */ + GPIO_PADREGJ_PAD37FNCSEL_CT29 = 7, /*!< CT29 : CTIMER connection 29 value. */ +} GPIO_PADREGJ_PAD37FNCSEL_Enum; + +/* =========================================== GPIO PADREGJ PAD37STRNG [10..10] ============================================ */ +typedef enum { /*!< GPIO_PADREGJ_PAD37STRNG */ + GPIO_PADREGJ_PAD37STRNG_LOW = 0, /*!< LOW : Low drive strength value. */ + GPIO_PADREGJ_PAD37STRNG_HIGH = 1, /*!< HIGH : High drive strength value. */ +} GPIO_PADREGJ_PAD37STRNG_Enum; + +/* ============================================ GPIO PADREGJ PAD37INPEN [9..9] ============================================= */ +typedef enum { /*!< GPIO_PADREGJ_PAD37INPEN */ + GPIO_PADREGJ_PAD37INPEN_DIS = 0, /*!< DIS : Pad input disabled value. */ + GPIO_PADREGJ_PAD37INPEN_EN = 1, /*!< EN : Pad input enabled value. */ +} GPIO_PADREGJ_PAD37INPEN_Enum; + +/* ============================================= GPIO PADREGJ PAD37PULL [8..8] ============================================= */ +typedef enum { /*!< GPIO_PADREGJ_PAD37PULL */ + GPIO_PADREGJ_PAD37PULL_DIS = 0, /*!< DIS : Pullup disabled value. */ + GPIO_PADREGJ_PAD37PULL_EN = 1, /*!< EN : Pullup enabled value. */ +} GPIO_PADREGJ_PAD37PULL_Enum; + +/* ============================================ GPIO PADREGJ PAD36PWRUP [6..6] ============================================= */ +typedef enum { /*!< GPIO_PADREGJ_PAD36PWRUP */ + GPIO_PADREGJ_PAD36PWRUP_DIS = 0, /*!< DIS : Power switch disabled value. */ + GPIO_PADREGJ_PAD36PWRUP_EN = 1, /*!< EN : Power switch enabled (switched to VDD) value. */ +} GPIO_PADREGJ_PAD36PWRUP_Enum; + +/* ============================================ GPIO PADREGJ PAD36FNCSEL [3..5] ============================================ */ +typedef enum { /*!< GPIO_PADREGJ_PAD36FNCSEL */ + GPIO_PADREGJ_PAD36FNCSEL_TRIG1 = 0, /*!< TRIG1 : Configure as the ADC Trigger 1 signal value. */ + GPIO_PADREGJ_PAD36FNCSEL_NCE36 = 1, /*!< NCE36 : IOM/MSPI nCE group 36 value. */ + GPIO_PADREGJ_PAD36FNCSEL_UART1RX = 2, /*!< UART1RX : Configure as the UART1 RX input signal value. */ + GPIO_PADREGJ_PAD36FNCSEL_GPIO36 = 3, /*!< GPIO36 : Configure as GPIO36 value. */ + GPIO_PADREGJ_PAD36FNCSEL_32kHzXT = 4, /*!< 32kHzXT : Configure as the 32kHz output clock from the crystal + value. */ + GPIO_PADREGJ_PAD36FNCSEL_UA1CTS = 5, /*!< UA1CTS : Configure as the UART1 CTS input signal value. */ + GPIO_PADREGJ_PAD36FNCSEL_UA0CTS = 6, /*!< UA0CTS : Configure as the UART0 CTS input signal value. */ + GPIO_PADREGJ_PAD36FNCSEL_PDMDATA = 7, /*!< PDMDATA : PDM serial data input value. */ +} GPIO_PADREGJ_PAD36FNCSEL_Enum; + +/* ============================================ GPIO PADREGJ PAD36STRNG [2..2] ============================================= */ +typedef enum { /*!< GPIO_PADREGJ_PAD36STRNG */ + GPIO_PADREGJ_PAD36STRNG_LOW = 0, /*!< LOW : Low drive strength value. */ + GPIO_PADREGJ_PAD36STRNG_HIGH = 1, /*!< HIGH : High drive strength value. */ +} GPIO_PADREGJ_PAD36STRNG_Enum; + +/* ============================================ GPIO PADREGJ PAD36INPEN [1..1] ============================================= */ +typedef enum { /*!< GPIO_PADREGJ_PAD36INPEN */ + GPIO_PADREGJ_PAD36INPEN_DIS = 0, /*!< DIS : Pad input disabled value. */ + GPIO_PADREGJ_PAD36INPEN_EN = 1, /*!< EN : Pad input enabled value. */ +} GPIO_PADREGJ_PAD36INPEN_Enum; + +/* ============================================= GPIO PADREGJ PAD36PULL [0..0] ============================================= */ +typedef enum { /*!< GPIO_PADREGJ_PAD36PULL */ + GPIO_PADREGJ_PAD36PULL_DIS = 0, /*!< DIS : Pullup disabled value. */ + GPIO_PADREGJ_PAD36PULL_EN = 1, /*!< EN : Pullup enabled value. */ +} GPIO_PADREGJ_PAD36PULL_Enum; + +/* ======================================================== PADREGK ======================================================== */ +/* ============================================ GPIO PADREGK PAD43RSEL [30..31] ============================================ */ +typedef enum { /*!< GPIO_PADREGK_PAD43RSEL */ + GPIO_PADREGK_PAD43RSEL_PULL1_5K = 0, /*!< PULL1_5K : Pullup is ~1.5 KOhms value. */ + GPIO_PADREGK_PAD43RSEL_PULL6K = 1, /*!< PULL6K : Pullup is ~6 KOhms value. */ + GPIO_PADREGK_PAD43RSEL_PULL12K = 2, /*!< PULL12K : Pullup is ~12 KOhms value. */ + GPIO_PADREGK_PAD43RSEL_PULL24K = 3, /*!< PULL24K : Pullup is ~24 KOhms value. */ +} GPIO_PADREGK_PAD43RSEL_Enum; + +/* =========================================== GPIO PADREGK PAD43FNCSEL [27..29] =========================================== */ +typedef enum { /*!< GPIO_PADREGK_PAD43FNCSEL */ + GPIO_PADREGK_PAD43FNCSEL_UART1RX = 0, /*!< UART1RX : Configure as the UART1 RX input signal value. */ + GPIO_PADREGK_PAD43FNCSEL_NCE43 = 1, /*!< NCE43 : IOM/MSPI nCE group 43 value. */ + GPIO_PADREGK_PAD43FNCSEL_CT18 = 2, /*!< CT18 : CTIMER connection 18 value. */ + GPIO_PADREGK_PAD43FNCSEL_GPIO43 = 3, /*!< GPIO43 : Configure as GPIO43 value. */ + GPIO_PADREGK_PAD43FNCSEL_M3SDAWIR3 = 4, /*!< M3SDAWIR3 : Configure as the IOMSTR3 I2C SDA or SPI WIR3 signal + value. */ + GPIO_PADREGK_PAD43FNCSEL_M3MISO = 5, /*!< M3MISO : Configure as the IOMSTR3 SPI MISO signal value. */ +} GPIO_PADREGK_PAD43FNCSEL_Enum; + +/* =========================================== GPIO PADREGK PAD43STRNG [26..26] ============================================ */ +typedef enum { /*!< GPIO_PADREGK_PAD43STRNG */ + GPIO_PADREGK_PAD43STRNG_LOW = 0, /*!< LOW : Low drive strength value. */ + GPIO_PADREGK_PAD43STRNG_HIGH = 1, /*!< HIGH : High drive strength value. */ +} GPIO_PADREGK_PAD43STRNG_Enum; + +/* =========================================== GPIO PADREGK PAD43INPEN [25..25] ============================================ */ +typedef enum { /*!< GPIO_PADREGK_PAD43INPEN */ + GPIO_PADREGK_PAD43INPEN_DIS = 0, /*!< DIS : Pad input disabled value. */ + GPIO_PADREGK_PAD43INPEN_EN = 1, /*!< EN : Pad input enabled value. */ +} GPIO_PADREGK_PAD43INPEN_Enum; + +/* ============================================ GPIO PADREGK PAD43PULL [24..24] ============================================ */ +typedef enum { /*!< GPIO_PADREGK_PAD43PULL */ + GPIO_PADREGK_PAD43PULL_DIS = 0, /*!< DIS : Pullup disabled value. */ + GPIO_PADREGK_PAD43PULL_EN = 1, /*!< EN : Pullup enabled value. */ +} GPIO_PADREGK_PAD43PULL_Enum; + +/* ============================================ GPIO PADREGK PAD42RSEL [22..23] ============================================ */ +typedef enum { /*!< GPIO_PADREGK_PAD42RSEL */ + GPIO_PADREGK_PAD42RSEL_PULL1_5K = 0, /*!< PULL1_5K : Pullup is ~1.5 KOhms value. */ + GPIO_PADREGK_PAD42RSEL_PULL6K = 1, /*!< PULL6K : Pullup is ~6 KOhms value. */ + GPIO_PADREGK_PAD42RSEL_PULL12K = 2, /*!< PULL12K : Pullup is ~12 KOhms value. */ + GPIO_PADREGK_PAD42RSEL_PULL24K = 3, /*!< PULL24K : Pullup is ~24 KOhms value. */ +} GPIO_PADREGK_PAD42RSEL_Enum; + +/* =========================================== GPIO PADREGK PAD42FNCSEL [19..21] =========================================== */ +typedef enum { /*!< GPIO_PADREGK_PAD42FNCSEL */ + GPIO_PADREGK_PAD42FNCSEL_UART1TX = 0, /*!< UART1TX : Configure as the UART1 TX output signal value. */ + GPIO_PADREGK_PAD42FNCSEL_NCE42 = 1, /*!< NCE42 : IOM/MSPI nCE group 42 value. */ + GPIO_PADREGK_PAD42FNCSEL_CT16 = 2, /*!< CT16 : CTIMER connection 16 value. */ + GPIO_PADREGK_PAD42FNCSEL_GPIO42 = 3, /*!< GPIO42 : Configure as GPIO42 value. */ + GPIO_PADREGK_PAD42FNCSEL_M3SCL = 4, /*!< M3SCL : Configure as the IOMSTR3 I2C SCL clock I/O signal value. */ + GPIO_PADREGK_PAD42FNCSEL_M3SCK = 5, /*!< M3SCK : Configure as the IOMSTR3 SPI SCK output value. */ +} GPIO_PADREGK_PAD42FNCSEL_Enum; + +/* =========================================== GPIO PADREGK PAD42STRNG [18..18] ============================================ */ +typedef enum { /*!< GPIO_PADREGK_PAD42STRNG */ + GPIO_PADREGK_PAD42STRNG_LOW = 0, /*!< LOW : Low drive strength value. */ + GPIO_PADREGK_PAD42STRNG_HIGH = 1, /*!< HIGH : High drive strength value. */ +} GPIO_PADREGK_PAD42STRNG_Enum; + +/* =========================================== GPIO PADREGK PAD42INPEN [17..17] ============================================ */ +typedef enum { /*!< GPIO_PADREGK_PAD42INPEN */ + GPIO_PADREGK_PAD42INPEN_DIS = 0, /*!< DIS : Pad input disabled value. */ + GPIO_PADREGK_PAD42INPEN_EN = 1, /*!< EN : Pad input enabled value. */ +} GPIO_PADREGK_PAD42INPEN_Enum; + +/* ============================================ GPIO PADREGK PAD42PULL [16..16] ============================================ */ +typedef enum { /*!< GPIO_PADREGK_PAD42PULL */ + GPIO_PADREGK_PAD42PULL_DIS = 0, /*!< DIS : Pullup disabled value. */ + GPIO_PADREGK_PAD42PULL_EN = 1, /*!< EN : Pullup enabled value. */ +} GPIO_PADREGK_PAD42PULL_Enum; + +/* =========================================== GPIO PADREGK PAD41PWRDN [15..15] ============================================ */ +typedef enum { /*!< GPIO_PADREGK_PAD41PWRDN */ + GPIO_PADREGK_PAD41PWRDN_DIS = 0, /*!< DIS : Power switch disabled value. */ + GPIO_PADREGK_PAD41PWRDN_EN = 1, /*!< EN : Power switch enabled (Switch pad to VSS) value. */ +} GPIO_PADREGK_PAD41PWRDN_Enum; + +/* =========================================== GPIO PADREGK PAD41FNCSEL [11..13] =========================================== */ +typedef enum { /*!< GPIO_PADREGK_PAD41FNCSEL */ + GPIO_PADREGK_PAD41FNCSEL_NCE41 = 0, /*!< NCE41 : IOM/MSPI nCE group 41 value. */ + GPIO_PADREGK_PAD41FNCSEL_SWO = 2, /*!< SWO : Configure as the serial wire debug SWO signal value. */ + GPIO_PADREGK_PAD41FNCSEL_GPIO41 = 3, /*!< GPIO41 : Configure as GPIO41 value. */ + GPIO_PADREGK_PAD41FNCSEL_I2SWCLK = 4, /*!< I2SWCLK : I2S word clock input value. */ + GPIO_PADREGK_PAD41FNCSEL_UA1RTS = 5, /*!< UA1RTS : Configure as the UART1 RTS output signal value. */ + GPIO_PADREGK_PAD41FNCSEL_UART0TX = 6, /*!< UART0TX : Configure as the UART0 TX output signal value. */ + GPIO_PADREGK_PAD41FNCSEL_UA0RTS = 7, /*!< UA0RTS : Configure as the UART0 RTS output signal value. */ +} GPIO_PADREGK_PAD41FNCSEL_Enum; + +/* =========================================== GPIO PADREGK PAD41STRNG [10..10] ============================================ */ +typedef enum { /*!< GPIO_PADREGK_PAD41STRNG */ + GPIO_PADREGK_PAD41STRNG_LOW = 0, /*!< LOW : Low drive strength value. */ + GPIO_PADREGK_PAD41STRNG_HIGH = 1, /*!< HIGH : High drive strength value. */ +} GPIO_PADREGK_PAD41STRNG_Enum; + +/* ============================================ GPIO PADREGK PAD41INPEN [9..9] ============================================= */ +typedef enum { /*!< GPIO_PADREGK_PAD41INPEN */ + GPIO_PADREGK_PAD41INPEN_DIS = 0, /*!< DIS : Pad input disabled value. */ + GPIO_PADREGK_PAD41INPEN_EN = 1, /*!< EN : Pad input enabled value. */ +} GPIO_PADREGK_PAD41INPEN_Enum; + +/* ============================================= GPIO PADREGK PAD41PULL [8..8] ============================================= */ +typedef enum { /*!< GPIO_PADREGK_PAD41PULL */ + GPIO_PADREGK_PAD41PULL_DIS = 0, /*!< DIS : Pullup disabled value. */ + GPIO_PADREGK_PAD41PULL_EN = 1, /*!< EN : Pullup enabled value. */ +} GPIO_PADREGK_PAD41PULL_Enum; + +/* ============================================= GPIO PADREGK PAD40RSEL [6..7] ============================================= */ +typedef enum { /*!< GPIO_PADREGK_PAD40RSEL */ + GPIO_PADREGK_PAD40RSEL_PULL1_5K = 0, /*!< PULL1_5K : Pullup is ~1.5 KOhms value. */ + GPIO_PADREGK_PAD40RSEL_PULL6K = 1, /*!< PULL6K : Pullup is ~6 KOhms value. */ + GPIO_PADREGK_PAD40RSEL_PULL12K = 2, /*!< PULL12K : Pullup is ~12 KOhms value. */ + GPIO_PADREGK_PAD40RSEL_PULL24K = 3, /*!< PULL24K : Pullup is ~24 KOhms value. */ +} GPIO_PADREGK_PAD40RSEL_Enum; + +/* ============================================ GPIO PADREGK PAD40FNCSEL [3..5] ============================================ */ +typedef enum { /*!< GPIO_PADREGK_PAD40FNCSEL */ + GPIO_PADREGK_PAD40FNCSEL_UART0RX = 0, /*!< UART0RX : Configure as the UART0 RX input signal value. */ + GPIO_PADREGK_PAD40FNCSEL_UART1RX = 1, /*!< UART1RX : Configure as the UART1 RX input signal value. */ + GPIO_PADREGK_PAD40FNCSEL_TRIG0 = 2, /*!< TRIG0 : Configure as the ADC Trigger 0 signal value. */ + GPIO_PADREGK_PAD40FNCSEL_GPIO40 = 3, /*!< GPIO40 : Configure as GPIO40 value. */ + GPIO_PADREGK_PAD40FNCSEL_M4SDAWIR3 = 4, /*!< M4SDAWIR3 : Configure as the IOMSTR4 I2C SDA or SPI WIR3 signal + value. */ + GPIO_PADREGK_PAD40FNCSEL_M4MISO = 5, /*!< M4MISO : Configure as the IOMSTR4 SPI MISO input signal value. */ +} GPIO_PADREGK_PAD40FNCSEL_Enum; + +/* ============================================ GPIO PADREGK PAD40STRNG [2..2] ============================================= */ +typedef enum { /*!< GPIO_PADREGK_PAD40STRNG */ + GPIO_PADREGK_PAD40STRNG_LOW = 0, /*!< LOW : Low drive strength value. */ + GPIO_PADREGK_PAD40STRNG_HIGH = 1, /*!< HIGH : High drive strength value. */ +} GPIO_PADREGK_PAD40STRNG_Enum; + +/* ============================================ GPIO PADREGK PAD40INPEN [1..1] ============================================= */ +typedef enum { /*!< GPIO_PADREGK_PAD40INPEN */ + GPIO_PADREGK_PAD40INPEN_DIS = 0, /*!< DIS : Pad input disabled value. */ + GPIO_PADREGK_PAD40INPEN_EN = 1, /*!< EN : Pad input enabled value. */ +} GPIO_PADREGK_PAD40INPEN_Enum; + +/* ============================================= GPIO PADREGK PAD40PULL [0..0] ============================================= */ +typedef enum { /*!< GPIO_PADREGK_PAD40PULL */ + GPIO_PADREGK_PAD40PULL_DIS = 0, /*!< DIS : Pullup disabled value. */ + GPIO_PADREGK_PAD40PULL_EN = 1, /*!< EN : Pullup enabled value. */ +} GPIO_PADREGK_PAD40PULL_Enum; + +/* ======================================================== PADREGL ======================================================== */ +/* =========================================== GPIO PADREGL PAD47FNCSEL [27..29] =========================================== */ +typedef enum { /*!< GPIO_PADREGL_PAD47FNCSEL */ + GPIO_PADREGL_PAD47FNCSEL_32kHzXT = 0, /*!< 32kHzXT : Configure as the 32kHz output clock from the crystal + value. */ + GPIO_PADREGL_PAD47FNCSEL_NCE47 = 1, /*!< NCE47 : IOM/MSPI nCE group 47 value. */ + GPIO_PADREGL_PAD47FNCSEL_CT26 = 2, /*!< CT26 : CTIMER connection 26 value. */ + GPIO_PADREGL_PAD47FNCSEL_GPIO47 = 3, /*!< GPIO47 : Configure as GPIO47 value. */ + GPIO_PADREGL_PAD47FNCSEL_M5MOSI = 5, /*!< M5MOSI : Configure as the IOMSTR5 SPI MOSI output signal value. */ + GPIO_PADREGL_PAD47FNCSEL_UART1RX = 6, /*!< UART1RX : Configure as the UART1 RX input signal value. */ +} GPIO_PADREGL_PAD47FNCSEL_Enum; + +/* =========================================== GPIO PADREGL PAD47STRNG [26..26] ============================================ */ +typedef enum { /*!< GPIO_PADREGL_PAD47STRNG */ + GPIO_PADREGL_PAD47STRNG_LOW = 0, /*!< LOW : Low drive strength value. */ + GPIO_PADREGL_PAD47STRNG_HIGH = 1, /*!< HIGH : High drive strength value. */ +} GPIO_PADREGL_PAD47STRNG_Enum; + +/* =========================================== GPIO PADREGL PAD47INPEN [25..25] ============================================ */ +typedef enum { /*!< GPIO_PADREGL_PAD47INPEN */ + GPIO_PADREGL_PAD47INPEN_DIS = 0, /*!< DIS : Pad input disabled value. */ + GPIO_PADREGL_PAD47INPEN_EN = 1, /*!< EN : Pad input enabled value. */ +} GPIO_PADREGL_PAD47INPEN_Enum; + +/* ============================================ GPIO PADREGL PAD47PULL [24..24] ============================================ */ +typedef enum { /*!< GPIO_PADREGL_PAD47PULL */ + GPIO_PADREGL_PAD47PULL_DIS = 0, /*!< DIS : Pullup disabled value. */ + GPIO_PADREGL_PAD47PULL_EN = 1, /*!< EN : Pullup enabled value. */ +} GPIO_PADREGL_PAD47PULL_Enum; + +/* =========================================== GPIO PADREGL PAD46FNCSEL [19..21] =========================================== */ +typedef enum { /*!< GPIO_PADREGL_PAD46FNCSEL */ + GPIO_PADREGL_PAD46FNCSEL_32khz_XT = 0, /*!< 32khz_XT : Configure as the 32kHz output clock from the crystal + value. */ + GPIO_PADREGL_PAD46FNCSEL_NCE46 = 1, /*!< NCE46 : IOM/MSPI nCE group 46 value. */ + GPIO_PADREGL_PAD46FNCSEL_CT24 = 2, /*!< CT24 : CTIMER connection 24 value. */ + GPIO_PADREGL_PAD46FNCSEL_GPIO46 = 3, /*!< GPIO46 : Configure as GPIO46 value. */ + GPIO_PADREGL_PAD46FNCSEL_SCCRST = 4, /*!< SCCRST : SCARD reset output value. */ + GPIO_PADREGL_PAD46FNCSEL_PDMCLK = 5, /*!< PDMCLK : PDM serial clock output value. */ + GPIO_PADREGL_PAD46FNCSEL_UART1TX = 6, /*!< UART1TX : Configure as the UART1 TX output signal value. */ + GPIO_PADREGL_PAD46FNCSEL_SWO = 7, /*!< SWO : Configure as the serial wire debug SWO signal value. */ +} GPIO_PADREGL_PAD46FNCSEL_Enum; + +/* =========================================== GPIO PADREGL PAD46STRNG [18..18] ============================================ */ +typedef enum { /*!< GPIO_PADREGL_PAD46STRNG */ + GPIO_PADREGL_PAD46STRNG_LOW = 0, /*!< LOW : Low drive strength value. */ + GPIO_PADREGL_PAD46STRNG_HIGH = 1, /*!< HIGH : High drive strength value. */ +} GPIO_PADREGL_PAD46STRNG_Enum; + +/* =========================================== GPIO PADREGL PAD46INPEN [17..17] ============================================ */ +typedef enum { /*!< GPIO_PADREGL_PAD46INPEN */ + GPIO_PADREGL_PAD46INPEN_DIS = 0, /*!< DIS : Pad input disabled value. */ + GPIO_PADREGL_PAD46INPEN_EN = 1, /*!< EN : Pad input enabled value. */ +} GPIO_PADREGL_PAD46INPEN_Enum; + +/* ============================================ GPIO PADREGL PAD46PULL [16..16] ============================================ */ +typedef enum { /*!< GPIO_PADREGL_PAD46PULL */ + GPIO_PADREGL_PAD46PULL_DIS = 0, /*!< DIS : Pullup disabled value. */ + GPIO_PADREGL_PAD46PULL_EN = 1, /*!< EN : Pullup enabled value. */ +} GPIO_PADREGL_PAD46PULL_Enum; + +/* =========================================== GPIO PADREGL PAD45FNCSEL [11..13] =========================================== */ +typedef enum { /*!< GPIO_PADREGL_PAD45FNCSEL */ + GPIO_PADREGL_PAD45FNCSEL_UA1CTS = 0, /*!< UA1CTS : Configure as the UART1 CTS input signal value. */ + GPIO_PADREGL_PAD45FNCSEL_NCE45 = 1, /*!< NCE45 : IOM/MSPI nCE group 45 value. */ + GPIO_PADREGL_PAD45FNCSEL_CT22 = 2, /*!< CT22 : CTIMER connection 22 value. */ + GPIO_PADREGL_PAD45FNCSEL_GPIO45 = 3, /*!< GPIO45 : Configure as GPIO45 value. */ + GPIO_PADREGL_PAD45FNCSEL_I2SDAT = 4, /*!< I2SDAT : I2S serial data output value. */ + GPIO_PADREGL_PAD45FNCSEL_PDMDATA = 5, /*!< PDMDATA : PDM serial data input value. */ + GPIO_PADREGL_PAD45FNCSEL_UART0RX = 6, /*!< UART0RX : Configure as the SPI channel 5 nCE signal from IOMSTR5 + value. */ + GPIO_PADREGL_PAD45FNCSEL_SWO = 7, /*!< SWO : Configure as the serial wire debug SWO signal value. */ +} GPIO_PADREGL_PAD45FNCSEL_Enum; + +/* =========================================== GPIO PADREGL PAD45STRNG [10..10] ============================================ */ +typedef enum { /*!< GPIO_PADREGL_PAD45STRNG */ + GPIO_PADREGL_PAD45STRNG_LOW = 0, /*!< LOW : Low drive strength value. */ + GPIO_PADREGL_PAD45STRNG_HIGH = 1, /*!< HIGH : High drive strength value. */ +} GPIO_PADREGL_PAD45STRNG_Enum; + +/* ============================================ GPIO PADREGL PAD45INPEN [9..9] ============================================= */ +typedef enum { /*!< GPIO_PADREGL_PAD45INPEN */ + GPIO_PADREGL_PAD45INPEN_DIS = 0, /*!< DIS : Pad input disabled value. */ + GPIO_PADREGL_PAD45INPEN_EN = 1, /*!< EN : Pad input enabled value. */ +} GPIO_PADREGL_PAD45INPEN_Enum; + +/* ============================================= GPIO PADREGL PAD45PULL [8..8] ============================================= */ +typedef enum { /*!< GPIO_PADREGL_PAD45PULL */ + GPIO_PADREGL_PAD45PULL_DIS = 0, /*!< DIS : Pullup disabled value. */ + GPIO_PADREGL_PAD45PULL_EN = 1, /*!< EN : Pullup enabled value. */ +} GPIO_PADREGL_PAD45PULL_Enum; + +/* ============================================ GPIO PADREGL PAD44FNCSEL [3..5] ============================================ */ +typedef enum { /*!< GPIO_PADREGL_PAD44FNCSEL */ + GPIO_PADREGL_PAD44FNCSEL_UA1RTS = 0, /*!< UA1RTS : Configure as the UART1 RTS output signal value. */ + GPIO_PADREGL_PAD44FNCSEL_NCE44 = 1, /*!< NCE44 : IOM/MSPI nCE group 44 value. */ + GPIO_PADREGL_PAD44FNCSEL_CT20 = 2, /*!< CT20 : CTIMER connection 20 value. */ + GPIO_PADREGL_PAD44FNCSEL_GPIO44 = 3, /*!< GPIO44 : Configure as GPIO44 value. */ + GPIO_PADREGL_PAD44FNCSEL_M4MOSI = 5, /*!< M4MOSI : Configure as the IOMSTR4 SPI MOSI signal value. */ + GPIO_PADREGL_PAD44FNCSEL_M5nCE6 = 6, /*!< M5nCE6 : Configure as the SPI channel 6 nCE signal from IOMSTR5 + value. */ +} GPIO_PADREGL_PAD44FNCSEL_Enum; + +/* ============================================ GPIO PADREGL PAD44STRNG [2..2] ============================================= */ +typedef enum { /*!< GPIO_PADREGL_PAD44STRNG */ + GPIO_PADREGL_PAD44STRNG_LOW = 0, /*!< LOW : Low drive strength value. */ + GPIO_PADREGL_PAD44STRNG_HIGH = 1, /*!< HIGH : High drive strength value. */ +} GPIO_PADREGL_PAD44STRNG_Enum; + +/* ============================================ GPIO PADREGL PAD44INPEN [1..1] ============================================= */ +typedef enum { /*!< GPIO_PADREGL_PAD44INPEN */ + GPIO_PADREGL_PAD44INPEN_DIS = 0, /*!< DIS : Pad input disabled value. */ + GPIO_PADREGL_PAD44INPEN_EN = 1, /*!< EN : Pad input enabled value. */ +} GPIO_PADREGL_PAD44INPEN_Enum; + +/* ============================================= GPIO PADREGL PAD44PULL [0..0] ============================================= */ +typedef enum { /*!< GPIO_PADREGL_PAD44PULL */ + GPIO_PADREGL_PAD44PULL_DIS = 0, /*!< DIS : Pullup disabled value. */ + GPIO_PADREGL_PAD44PULL_EN = 1, /*!< EN : Pullup enabled value. */ +} GPIO_PADREGL_PAD44PULL_Enum; + +/* ======================================================== PADREGM ======================================================== */ +/* ============================================ GPIO PADREGM PAD49RSEL [14..15] ============================================ */ +typedef enum { /*!< GPIO_PADREGM_PAD49RSEL */ + GPIO_PADREGM_PAD49RSEL_PULL1_5K = 0, /*!< PULL1_5K : Pullup is ~1.5 KOhms value. */ + GPIO_PADREGM_PAD49RSEL_PULL6K = 1, /*!< PULL6K : Pullup is ~6 KOhms value. */ + GPIO_PADREGM_PAD49RSEL_PULL12K = 2, /*!< PULL12K : Pullup is ~12 KOhms value. */ + GPIO_PADREGM_PAD49RSEL_PULL24K = 3, /*!< PULL24K : Pullup is ~24 KOhms value. */ +} GPIO_PADREGM_PAD49RSEL_Enum; + +/* =========================================== GPIO PADREGM PAD49FNCSEL [11..13] =========================================== */ +typedef enum { /*!< GPIO_PADREGM_PAD49FNCSEL */ + GPIO_PADREGM_PAD49FNCSEL_UART0RX = 0, /*!< UART0RX : Configure as the UART0 RX input signal value. */ + GPIO_PADREGM_PAD49FNCSEL_NCE49 = 1, /*!< NCE49 : IOM/MSPPI nCE group 49 value. */ + GPIO_PADREGM_PAD49FNCSEL_CT30 = 2, /*!< CT30 : CTIMER connection 30 value. */ + GPIO_PADREGM_PAD49FNCSEL_GPIO49 = 3, /*!< GPIO49 : Configure as GPIO49 value. */ + GPIO_PADREGM_PAD49FNCSEL_M5SDAWIR3 = 4, /*!< M5SDAWIR3 : Configure as the IOMSTR5 I2C SDA or SPI WIR3 signal + value. */ + GPIO_PADREGM_PAD49FNCSEL_M5MISO = 5, /*!< M5MISO : Configure as the IOMSTR5 SPI MISO input signal value. */ +} GPIO_PADREGM_PAD49FNCSEL_Enum; + +/* =========================================== GPIO PADREGM PAD49STRNG [10..10] ============================================ */ +typedef enum { /*!< GPIO_PADREGM_PAD49STRNG */ + GPIO_PADREGM_PAD49STRNG_LOW = 0, /*!< LOW : Low drive strength value. */ + GPIO_PADREGM_PAD49STRNG_HIGH = 1, /*!< HIGH : High drive strength value. */ +} GPIO_PADREGM_PAD49STRNG_Enum; + +/* ============================================ GPIO PADREGM PAD49INPEN [9..9] ============================================= */ +typedef enum { /*!< GPIO_PADREGM_PAD49INPEN */ + GPIO_PADREGM_PAD49INPEN_DIS = 0, /*!< DIS : Pad input disabled value. */ + GPIO_PADREGM_PAD49INPEN_EN = 1, /*!< EN : Pad input enabled value. */ +} GPIO_PADREGM_PAD49INPEN_Enum; + +/* ============================================= GPIO PADREGM PAD49PULL [8..8] ============================================= */ +typedef enum { /*!< GPIO_PADREGM_PAD49PULL */ + GPIO_PADREGM_PAD49PULL_DIS = 0, /*!< DIS : Pullup disabled value. */ + GPIO_PADREGM_PAD49PULL_EN = 1, /*!< EN : Pullup enabled value. */ +} GPIO_PADREGM_PAD49PULL_Enum; + +/* ============================================= GPIO PADREGM PAD48RSEL [6..7] ============================================= */ +typedef enum { /*!< GPIO_PADREGM_PAD48RSEL */ + GPIO_PADREGM_PAD48RSEL_PULL1_5K = 0, /*!< PULL1_5K : Pullup is ~1.5 KOhms value. */ + GPIO_PADREGM_PAD48RSEL_PULL6K = 1, /*!< PULL6K : Pullup is ~6 KOhms value. */ + GPIO_PADREGM_PAD48RSEL_PULL12K = 2, /*!< PULL12K : Pullup is ~12 KOhms value. */ + GPIO_PADREGM_PAD48RSEL_PULL24K = 3, /*!< PULL24K : Pullup is ~24 KOhms value. */ +} GPIO_PADREGM_PAD48RSEL_Enum; + +/* ============================================ GPIO PADREGM PAD48FNCSEL [3..5] ============================================ */ +typedef enum { /*!< GPIO_PADREGM_PAD48FNCSEL */ + GPIO_PADREGM_PAD48FNCSEL_UART0TX = 0, /*!< UART0TX : Configure as the UART0 TX output signal value. */ + GPIO_PADREGM_PAD48FNCSEL_NCE48 = 1, /*!< NCE48 : IOM/MSPI nCE group 48 value. */ + GPIO_PADREGM_PAD48FNCSEL_CT28 = 2, /*!< CT28 : CTIMER conenction 28 value. */ + GPIO_PADREGM_PAD48FNCSEL_GPIO48 = 3, /*!< GPIO48 : Configure as GPIO48 value. */ + GPIO_PADREGM_PAD48FNCSEL_M5SCL = 4, /*!< M5SCL : Configure as the IOMSTR5 I2C SCL clock I/O signal value. */ + GPIO_PADREGM_PAD48FNCSEL_M5SCK = 5, /*!< M5SCK : Configure as the IOMSTR5 SPI SCK output value. */ +} GPIO_PADREGM_PAD48FNCSEL_Enum; + +/* ============================================ GPIO PADREGM PAD48STRNG [2..2] ============================================= */ +typedef enum { /*!< GPIO_PADREGM_PAD48STRNG */ + GPIO_PADREGM_PAD48STRNG_LOW = 0, /*!< LOW : Low drive strength value. */ + GPIO_PADREGM_PAD48STRNG_HIGH = 1, /*!< HIGH : High drive strength value. */ +} GPIO_PADREGM_PAD48STRNG_Enum; + +/* ============================================ GPIO PADREGM PAD48INPEN [1..1] ============================================= */ +typedef enum { /*!< GPIO_PADREGM_PAD48INPEN */ + GPIO_PADREGM_PAD48INPEN_DIS = 0, /*!< DIS : Pad input disabled value. */ + GPIO_PADREGM_PAD48INPEN_EN = 1, /*!< EN : Pad input enabled value. */ +} GPIO_PADREGM_PAD48INPEN_Enum; + +/* ============================================= GPIO PADREGM PAD48PULL [0..0] ============================================= */ +typedef enum { /*!< GPIO_PADREGM_PAD48PULL */ + GPIO_PADREGM_PAD48PULL_DIS = 0, /*!< DIS : Pullup disabled value. */ + GPIO_PADREGM_PAD48PULL_EN = 1, /*!< EN : Pullup enabled value. */ +} GPIO_PADREGM_PAD48PULL_Enum; + +/* ========================================================= CFGA ========================================================== */ +/* ============================================= GPIO CFGA GPIO7INTD [31..31] ============================================== */ +typedef enum { /*!< GPIO_CFGA_GPIO7INTD */ + GPIO_CFGA_GPIO7INTD_nCELOW = 0, /*!< nCELOW : FNCSEL = 0x0 - nCE polarity active low value. */ + GPIO_CFGA_GPIO7INTD_nCEHIGH = 1, /*!< nCEHIGH : FNCSEL = 0x0 - nCE polarity active high value. */ +} GPIO_CFGA_GPIO7INTD_Enum; + +/* ============================================ GPIO CFGA GPIO7OUTCFG [29..30] ============================================= */ +typedef enum { /*!< GPIO_CFGA_GPIO7OUTCFG */ + GPIO_CFGA_GPIO7OUTCFG_DIS = 0, /*!< DIS : FNCSEL = 0x3 - Output disabled value. */ + GPIO_CFGA_GPIO7OUTCFG_PUSHPULL = 1, /*!< PUSHPULL : FNCSEL = 0x3 - Output is push-pull value. */ + GPIO_CFGA_GPIO7OUTCFG_OD = 2, /*!< OD : FNCSEL = 0x3 - Output is open drain value. */ + GPIO_CFGA_GPIO7OUTCFG_TS = 3, /*!< TS : FNCSEL = 0x3 - Output is tri-state value. */ +} GPIO_CFGA_GPIO7OUTCFG_Enum; + +/* ============================================= GPIO CFGA GPIO7INCFG [28..28] ============================================= */ +typedef enum { /*!< GPIO_CFGA_GPIO7INCFG */ + GPIO_CFGA_GPIO7INCFG_READ = 0, /*!< READ : Read the GPIO pin data value. */ + GPIO_CFGA_GPIO7INCFG_RDZERO = 1, /*!< RDZERO : INTD = 0 - Readback will always be zero value. */ +} GPIO_CFGA_GPIO7INCFG_Enum; + +/* ============================================= GPIO CFGA GPIO6INTD [27..27] ============================================== */ +typedef enum { /*!< GPIO_CFGA_GPIO6INTD */ + GPIO_CFGA_GPIO6INTD_INTDIS = 0, /*!< INTDIS : INCFG = 1 - No interrupt on GPIO transition value. */ + GPIO_CFGA_GPIO6INTD_INTBOTH = 1, /*!< INTBOTH : INCFG = 1 - Interrupt on either low to high or high + to low GPIO transition value. */ +} GPIO_CFGA_GPIO6INTD_Enum; + +/* ============================================ GPIO CFGA GPIO6OUTCFG [25..26] ============================================= */ +typedef enum { /*!< GPIO_CFGA_GPIO6OUTCFG */ + GPIO_CFGA_GPIO6OUTCFG_DIS = 0, /*!< DIS : FNCSEL = 0x3 - Output disabled value. */ + GPIO_CFGA_GPIO6OUTCFG_PUSHPULL = 1, /*!< PUSHPULL : FNCSEL = 0x3 - Output is push-pull value. */ + GPIO_CFGA_GPIO6OUTCFG_OD = 2, /*!< OD : FNCSEL = 0x3 - Output is open drain value. */ + GPIO_CFGA_GPIO6OUTCFG_TS = 3, /*!< TS : FNCSEL = 0x3 - Output is tri-state value. */ +} GPIO_CFGA_GPIO6OUTCFG_Enum; + +/* ============================================= GPIO CFGA GPIO6INCFG [24..24] ============================================= */ +typedef enum { /*!< GPIO_CFGA_GPIO6INCFG */ + GPIO_CFGA_GPIO6INCFG_READ = 0, /*!< READ : Read the GPIO pin data value. */ + GPIO_CFGA_GPIO6INCFG_RDZERO = 1, /*!< RDZERO : INTD = 0 - Readback will always be zero value. */ +} GPIO_CFGA_GPIO6INCFG_Enum; + +/* ============================================= GPIO CFGA GPIO5INTD [23..23] ============================================== */ +typedef enum { /*!< GPIO_CFGA_GPIO5INTD */ + GPIO_CFGA_GPIO5INTD_INTDIS = 0, /*!< INTDIS : INCFG = 1 - No interrupt on GPIO transition value. */ + GPIO_CFGA_GPIO5INTD_INTBOTH = 1, /*!< INTBOTH : INCFG = 1 - Interrupt on either low to high or high + to low GPIO transition value. */ +} GPIO_CFGA_GPIO5INTD_Enum; + +/* ============================================ GPIO CFGA GPIO5OUTCFG [21..22] ============================================= */ +typedef enum { /*!< GPIO_CFGA_GPIO5OUTCFG */ + GPIO_CFGA_GPIO5OUTCFG_DIS = 0, /*!< DIS : FNCSEL = 0x3 - Output disabled value. */ + GPIO_CFGA_GPIO5OUTCFG_PUSHPULL = 1, /*!< PUSHPULL : FNCSEL = 0x3 - Output is push-pull value. */ + GPIO_CFGA_GPIO5OUTCFG_OD = 2, /*!< OD : FNCSEL = 0x3 - Output is open drain value. */ + GPIO_CFGA_GPIO5OUTCFG_TS = 3, /*!< TS : FNCSEL = 0x3 - Output is tri-state value. */ +} GPIO_CFGA_GPIO5OUTCFG_Enum; + +/* ============================================= GPIO CFGA GPIO5INCFG [20..20] ============================================= */ +typedef enum { /*!< GPIO_CFGA_GPIO5INCFG */ + GPIO_CFGA_GPIO5INCFG_READ = 0, /*!< READ : Read the GPIO pin data value. */ + GPIO_CFGA_GPIO5INCFG_RDZERO = 1, /*!< RDZERO : INTD = 0 - Readback will always be zero value. */ +} GPIO_CFGA_GPIO5INCFG_Enum; + +/* ============================================= GPIO CFGA GPIO4INTD [19..19] ============================================== */ +typedef enum { /*!< GPIO_CFGA_GPIO4INTD */ + GPIO_CFGA_GPIO4INTD_nCELOW = 0, /*!< nCELOW : FNCSEL = 0x2 - nCE polarity active low value. */ + GPIO_CFGA_GPIO4INTD_nCEHIGH = 1, /*!< nCEHIGH : FNCSEL = 0x2 - nCE polarity active high value. */ +} GPIO_CFGA_GPIO4INTD_Enum; + +/* ============================================ GPIO CFGA GPIO4OUTCFG [17..18] ============================================= */ +typedef enum { /*!< GPIO_CFGA_GPIO4OUTCFG */ + GPIO_CFGA_GPIO4OUTCFG_DIS = 0, /*!< DIS : FNCSEL = 0x3 - Output disabled value. */ + GPIO_CFGA_GPIO4OUTCFG_PUSHPULL = 1, /*!< PUSHPULL : FNCSEL = 0x3 - Output is push-pull value. */ + GPIO_CFGA_GPIO4OUTCFG_OD = 2, /*!< OD : FNCSEL = 0x3 - Output is open drain value. */ + GPIO_CFGA_GPIO4OUTCFG_TS = 3, /*!< TS : FNCSEL = 0x3 - Output is tri-state value. */ +} GPIO_CFGA_GPIO4OUTCFG_Enum; + +/* ============================================= GPIO CFGA GPIO4INCFG [16..16] ============================================= */ +typedef enum { /*!< GPIO_CFGA_GPIO4INCFG */ + GPIO_CFGA_GPIO4INCFG_READ = 0, /*!< READ : Read the GPIO pin data value. */ + GPIO_CFGA_GPIO4INCFG_RDZERO = 1, /*!< RDZERO : INTD = 0 - Readback will always be zero value. */ +} GPIO_CFGA_GPIO4INCFG_Enum; + +/* ============================================= GPIO CFGA GPIO3INTD [15..15] ============================================== */ +typedef enum { /*!< GPIO_CFGA_GPIO3INTD */ + GPIO_CFGA_GPIO3INTD_nCELOW = 0, /*!< nCELOW : FNCSEL = 0x2 - nCE polarity active low value. */ + GPIO_CFGA_GPIO3INTD_nCEHIGH = 1, /*!< nCEHIGH : FNCSEL = 0x2 - nCE polarity active high value. */ +} GPIO_CFGA_GPIO3INTD_Enum; + +/* ============================================ GPIO CFGA GPIO3OUTCFG [13..14] ============================================= */ +typedef enum { /*!< GPIO_CFGA_GPIO3OUTCFG */ + GPIO_CFGA_GPIO3OUTCFG_DIS = 0, /*!< DIS : FNCSEL = 0x3 - Output disabled value. */ + GPIO_CFGA_GPIO3OUTCFG_PUSHPULL = 1, /*!< PUSHPULL : FNCSEL = 0x3 - Output is push-pull value. */ + GPIO_CFGA_GPIO3OUTCFG_OD = 2, /*!< OD : FNCSEL = 0x3 - Output is open drain value. */ + GPIO_CFGA_GPIO3OUTCFG_TS = 3, /*!< TS : FNCSEL = 0x3 - Output is tri-state value. */ +} GPIO_CFGA_GPIO3OUTCFG_Enum; + +/* ============================================= GPIO CFGA GPIO3INCFG [12..12] ============================================= */ +typedef enum { /*!< GPIO_CFGA_GPIO3INCFG */ + GPIO_CFGA_GPIO3INCFG_READ = 0, /*!< READ : Read the GPIO pin data value. */ + GPIO_CFGA_GPIO3INCFG_RDZERO = 1, /*!< RDZERO : INTD = 0 - Readback will always be zero value. */ +} GPIO_CFGA_GPIO3INCFG_Enum; + +/* ============================================= GPIO CFGA GPIO2INTD [11..11] ============================================== */ +typedef enum { /*!< GPIO_CFGA_GPIO2INTD */ + GPIO_CFGA_GPIO2INTD_nCELOW = 0, /*!< nCELOW : FNCSEL = 0x7 - nCE polarity active low value. */ + GPIO_CFGA_GPIO2INTD_nCEHIGH = 1, /*!< nCEHIGH : FNCSEL = 0x7 - nCE polarity active high value. */ +} GPIO_CFGA_GPIO2INTD_Enum; + +/* ============================================= GPIO CFGA GPIO2OUTCFG [9..10] ============================================= */ +typedef enum { /*!< GPIO_CFGA_GPIO2OUTCFG */ + GPIO_CFGA_GPIO2OUTCFG_DIS = 0, /*!< DIS : FNCSEL = 0x3 - Output disabled value. */ + GPIO_CFGA_GPIO2OUTCFG_PUSHPULL = 1, /*!< PUSHPULL : FNCSEL = 0x3 - Output is push-pull value. */ + GPIO_CFGA_GPIO2OUTCFG_OD = 2, /*!< OD : FNCSEL = 0x3 - Output is open drain value. */ + GPIO_CFGA_GPIO2OUTCFG_TS = 3, /*!< TS : FNCSEL = 0x3 - Output is tri-state value. */ +} GPIO_CFGA_GPIO2OUTCFG_Enum; + +/* ============================================== GPIO CFGA GPIO2INCFG [8..8] ============================================== */ +typedef enum { /*!< GPIO_CFGA_GPIO2INCFG */ + GPIO_CFGA_GPIO2INCFG_READ = 0, /*!< READ : Read the GPIO pin data value. */ + GPIO_CFGA_GPIO2INCFG_RDZERO = 1, /*!< RDZERO : INTD = 0 - Readback will always be zero value. */ +} GPIO_CFGA_GPIO2INCFG_Enum; + +/* ============================================== GPIO CFGA GPIO1INTD [7..7] =============================================== */ +typedef enum { /*!< GPIO_CFGA_GPIO1INTD */ + GPIO_CFGA_GPIO1INTD_nCELOW = 0, /*!< nCELOW : FNCSEL = 0x7 - nCE polarity active low value. */ + GPIO_CFGA_GPIO1INTD_nCEHIGH = 1, /*!< nCEHIGH : FNCSEL = 0x7 - nCE polarity active high value. */ +} GPIO_CFGA_GPIO1INTD_Enum; + +/* ============================================= GPIO CFGA GPIO1OUTCFG [5..6] ============================================== */ +typedef enum { /*!< GPIO_CFGA_GPIO1OUTCFG */ + GPIO_CFGA_GPIO1OUTCFG_DIS = 0, /*!< DIS : FNCSEL = 0x3 - Output disabled value. */ + GPIO_CFGA_GPIO1OUTCFG_PUSHPULL = 1, /*!< PUSHPULL : FNCSEL = 0x3 - Output is push-pull value. */ + GPIO_CFGA_GPIO1OUTCFG_OD = 2, /*!< OD : FNCSEL = 0x3 - Output is open drain value. */ + GPIO_CFGA_GPIO1OUTCFG_TS = 3, /*!< TS : FNCSEL = 0x3 - Output is tri-state value. */ +} GPIO_CFGA_GPIO1OUTCFG_Enum; + +/* ============================================== GPIO CFGA GPIO1INCFG [4..4] ============================================== */ +typedef enum { /*!< GPIO_CFGA_GPIO1INCFG */ + GPIO_CFGA_GPIO1INCFG_READ = 0, /*!< READ : Read the GPIO pin data value. */ + GPIO_CFGA_GPIO1INCFG_RDZERO = 1, /*!< RDZERO : INTD = 0 - Readback will always be zero value. */ +} GPIO_CFGA_GPIO1INCFG_Enum; + +/* ============================================== GPIO CFGA GPIO0INTD [3..3] =============================================== */ +typedef enum { /*!< GPIO_CFGA_GPIO0INTD */ + GPIO_CFGA_GPIO0INTD_nCELOW = 0, /*!< nCELOW : FNCSEL = 0x7 - nCE polarity active low value. */ + GPIO_CFGA_GPIO0INTD_nCEHIGH = 1, /*!< nCEHIGH : FNCSEL = 0x7 - nCE polarity active high value. */ +} GPIO_CFGA_GPIO0INTD_Enum; + +/* ============================================= GPIO CFGA GPIO0OUTCFG [1..2] ============================================== */ +typedef enum { /*!< GPIO_CFGA_GPIO0OUTCFG */ + GPIO_CFGA_GPIO0OUTCFG_DIS = 0, /*!< DIS : FNCSEL = 0x3 - Output disabled value. */ + GPIO_CFGA_GPIO0OUTCFG_PUSHPULL = 1, /*!< PUSHPULL : FNCSEL = 0x3 - Output is push-pull value. */ + GPIO_CFGA_GPIO0OUTCFG_OD = 2, /*!< OD : FNCSEL = 0x3 - Output is open drain value. */ + GPIO_CFGA_GPIO0OUTCFG_TS = 3, /*!< TS : FNCSEL = 0x3 - Output is tri-state value. */ +} GPIO_CFGA_GPIO0OUTCFG_Enum; + +/* ============================================== GPIO CFGA GPIO0INCFG [0..0] ============================================== */ +typedef enum { /*!< GPIO_CFGA_GPIO0INCFG */ + GPIO_CFGA_GPIO0INCFG_READ = 0, /*!< READ : Read the GPIO pin data value. */ + GPIO_CFGA_GPIO0INCFG_RDZERO = 1, /*!< RDZERO : INTD = 0 - Readback will always be zero value. */ +} GPIO_CFGA_GPIO0INCFG_Enum; + +/* ========================================================= CFGB ========================================================== */ +/* ============================================= GPIO CFGB GPIO15INTD [31..31] ============================================= */ +typedef enum { /*!< GPIO_CFGB_GPIO15INTD */ + GPIO_CFGB_GPIO15INTD_nCELOW = 0, /*!< nCELOW : FNCSEL = 0x1 - nCE polarity active low value. */ + GPIO_CFGB_GPIO15INTD_nCEHIGH = 1, /*!< nCEHIGH : FNCSEL = 0x1 - nCE polarity active high value. */ +} GPIO_CFGB_GPIO15INTD_Enum; + +/* ============================================ GPIO CFGB GPIO15OUTCFG [29..30] ============================================ */ +typedef enum { /*!< GPIO_CFGB_GPIO15OUTCFG */ + GPIO_CFGB_GPIO15OUTCFG_DIS = 0, /*!< DIS : FNCSEL = 0x3 - Output disabled value. */ + GPIO_CFGB_GPIO15OUTCFG_PUSHPULL = 1, /*!< PUSHPULL : FNCSEL = 0x3 - Output is push-pull value. */ + GPIO_CFGB_GPIO15OUTCFG_OD = 2, /*!< OD : FNCSEL = 0x3 - Output is open drain value. */ + GPIO_CFGB_GPIO15OUTCFG_TS = 3, /*!< TS : FNCSEL = 0x3 - Output is tri-state value. */ +} GPIO_CFGB_GPIO15OUTCFG_Enum; + +/* ============================================ GPIO CFGB GPIO15INCFG [28..28] ============================================= */ +typedef enum { /*!< GPIO_CFGB_GPIO15INCFG */ + GPIO_CFGB_GPIO15INCFG_READ = 0, /*!< READ : Read the GPIO pin data value. */ + GPIO_CFGB_GPIO15INCFG_RDZERO = 1, /*!< RDZERO : INTD = 0 - Readback will always be zero value. */ +} GPIO_CFGB_GPIO15INCFG_Enum; + +/* ============================================= GPIO CFGB GPIO14INTD [27..27] ============================================= */ +typedef enum { /*!< GPIO_CFGB_GPIO14INTD */ + GPIO_CFGB_GPIO14INTD_nCELOW = 0, /*!< nCELOW : FNCSEL = 0x1 - nCE polarity active low value. */ + GPIO_CFGB_GPIO14INTD_nCEHIGH = 1, /*!< nCEHIGH : FNCSEL = 0x1 - nCE polarity active high value. */ +} GPIO_CFGB_GPIO14INTD_Enum; + +/* ============================================ GPIO CFGB GPIO14OUTCFG [25..26] ============================================ */ +typedef enum { /*!< GPIO_CFGB_GPIO14OUTCFG */ + GPIO_CFGB_GPIO14OUTCFG_DIS = 0, /*!< DIS : FNCSEL = 0x3 - Output disabled value. */ + GPIO_CFGB_GPIO14OUTCFG_PUSHPULL = 1, /*!< PUSHPULL : FNCSEL = 0x3 - Output is push-pull value. */ + GPIO_CFGB_GPIO14OUTCFG_OD = 2, /*!< OD : FNCSEL = 0x3 - Output is open drain value. */ + GPIO_CFGB_GPIO14OUTCFG_TS = 3, /*!< TS : FNCSEL = 0x3 - Output is tri-state value. */ +} GPIO_CFGB_GPIO14OUTCFG_Enum; + +/* ============================================ GPIO CFGB GPIO14INCFG [24..24] ============================================= */ +typedef enum { /*!< GPIO_CFGB_GPIO14INCFG */ + GPIO_CFGB_GPIO14INCFG_READ = 0, /*!< READ : Read the GPIO pin data value. */ + GPIO_CFGB_GPIO14INCFG_RDZERO = 1, /*!< RDZERO : INTD = 0 - Readback will always be zero value. */ +} GPIO_CFGB_GPIO14INCFG_Enum; + +/* ============================================= GPIO CFGB GPIO13INTD [23..23] ============================================= */ +typedef enum { /*!< GPIO_CFGB_GPIO13INTD */ + GPIO_CFGB_GPIO13INTD_nCELOW = 0, /*!< nCELOW : FNCSEL = 0x1 - nCE polarity active low value. */ + GPIO_CFGB_GPIO13INTD_nCEHIGH = 1, /*!< nCEHIGH : FNCSEL = 0x1 - nCE polarity active high value. */ +} GPIO_CFGB_GPIO13INTD_Enum; + +/* ============================================ GPIO CFGB GPIO13OUTCFG [21..22] ============================================ */ +typedef enum { /*!< GPIO_CFGB_GPIO13OUTCFG */ + GPIO_CFGB_GPIO13OUTCFG_DIS = 0, /*!< DIS : FNCSEL = 0x3 - Output disabled value. */ + GPIO_CFGB_GPIO13OUTCFG_PUSHPULL = 1, /*!< PUSHPULL : FNCSEL = 0x3 - Output is push-pull value. */ + GPIO_CFGB_GPIO13OUTCFG_OD = 2, /*!< OD : FNCSEL = 0x3 - Output is open drain value. */ + GPIO_CFGB_GPIO13OUTCFG_TS = 3, /*!< TS : FNCSEL = 0x3 - Output is tri-state value. */ +} GPIO_CFGB_GPIO13OUTCFG_Enum; + +/* ============================================ GPIO CFGB GPIO13INCFG [20..20] ============================================= */ +typedef enum { /*!< GPIO_CFGB_GPIO13INCFG */ + GPIO_CFGB_GPIO13INCFG_READ = 0, /*!< READ : Read the GPIO pin data value. */ + GPIO_CFGB_GPIO13INCFG_RDZERO = 1, /*!< RDZERO : INTD = 0 - Readback will always be zero value. */ +} GPIO_CFGB_GPIO13INCFG_Enum; + +/* ============================================= GPIO CFGB GPIO12INTD [19..19] ============================================= */ +typedef enum { /*!< GPIO_CFGB_GPIO12INTD */ + GPIO_CFGB_GPIO12INTD_nCELOW = 0, /*!< nCELOW : FNCSEL = 0x1 - nCE polarity active low value. */ + GPIO_CFGB_GPIO12INTD_nCEHIGH = 1, /*!< nCEHIGH : FNCSEL = 0x1 - nCE polarity active high value. */ +} GPIO_CFGB_GPIO12INTD_Enum; + +/* ============================================ GPIO CFGB GPIO12OUTCFG [17..18] ============================================ */ +typedef enum { /*!< GPIO_CFGB_GPIO12OUTCFG */ + GPIO_CFGB_GPIO12OUTCFG_DIS = 0, /*!< DIS : FNCSEL = 0x3 - Output disabled value. */ + GPIO_CFGB_GPIO12OUTCFG_PUSHPULL = 1, /*!< PUSHPULL : FNCSEL = 0x3 - Output is push-pull value. */ + GPIO_CFGB_GPIO12OUTCFG_OD = 2, /*!< OD : FNCSEL = 0x3 - Output is open drain value. */ + GPIO_CFGB_GPIO12OUTCFG_TS = 3, /*!< TS : FNCSEL = 0x3 - Output is tri-state value. */ +} GPIO_CFGB_GPIO12OUTCFG_Enum; + +/* ============================================ GPIO CFGB GPIO12INCFG [16..16] ============================================= */ +typedef enum { /*!< GPIO_CFGB_GPIO12INCFG */ + GPIO_CFGB_GPIO12INCFG_READ = 0, /*!< READ : Read the GPIO pin data value. */ + GPIO_CFGB_GPIO12INCFG_RDZERO = 1, /*!< RDZERO : INTD = 0 - Readback will always be zero value. */ +} GPIO_CFGB_GPIO12INCFG_Enum; + +/* ============================================= GPIO CFGB GPIO11INTD [15..15] ============================================= */ +typedef enum { /*!< GPIO_CFGB_GPIO11INTD */ + GPIO_CFGB_GPIO11INTD_nCELOW = 0, /*!< nCELOW : FNCSEL = 0x1 - nCE polarity active low value. */ + GPIO_CFGB_GPIO11INTD_nCEHIGH = 1, /*!< nCEHIGH : FNCSEL = 0x1 - nCE polarity active high value. */ +} GPIO_CFGB_GPIO11INTD_Enum; + +/* ============================================ GPIO CFGB GPIO11OUTCFG [13..14] ============================================ */ +typedef enum { /*!< GPIO_CFGB_GPIO11OUTCFG */ + GPIO_CFGB_GPIO11OUTCFG_DIS = 0, /*!< DIS : FNCSEL = 0x3 - Output disabled value. */ + GPIO_CFGB_GPIO11OUTCFG_PUSHPULL = 1, /*!< PUSHPULL : FNCSEL = 0x3 - Output is push-pull value. */ + GPIO_CFGB_GPIO11OUTCFG_OD = 2, /*!< OD : FNCSEL = 0x3 - Output is open drain value. */ + GPIO_CFGB_GPIO11OUTCFG_TS = 3, /*!< TS : FNCSEL = 0x3 - Output is tri-state value. */ +} GPIO_CFGB_GPIO11OUTCFG_Enum; + +/* ============================================ GPIO CFGB GPIO11INCFG [12..12] ============================================= */ +typedef enum { /*!< GPIO_CFGB_GPIO11INCFG */ + GPIO_CFGB_GPIO11INCFG_READ = 0, /*!< READ : Read the GPIO pin data value. */ + GPIO_CFGB_GPIO11INCFG_RDZERO = 1, /*!< RDZERO : INTD = 0 - Readback will always be zero value. */ +} GPIO_CFGB_GPIO11INCFG_Enum; + +/* ============================================= GPIO CFGB GPIO10INTD [11..11] ============================================= */ +typedef enum { /*!< GPIO_CFGB_GPIO10INTD */ + GPIO_CFGB_GPIO10INTD_nCELOW = 0, /*!< nCELOW : FNCSEL = 0x2 - nCE polarity active low value. */ + GPIO_CFGB_GPIO10INTD_nCEHIGH = 1, /*!< nCEHIGH : FNCSEL = 0x2 - nCE polarity active high value. */ +} GPIO_CFGB_GPIO10INTD_Enum; + +/* ============================================ GPIO CFGB GPIO10OUTCFG [9..10] ============================================= */ +typedef enum { /*!< GPIO_CFGB_GPIO10OUTCFG */ + GPIO_CFGB_GPIO10OUTCFG_DIS = 0, /*!< DIS : FNCSEL = 0x3 - Output disabled value. */ + GPIO_CFGB_GPIO10OUTCFG_PUSHPULL = 1, /*!< PUSHPULL : FNCSEL = 0x3 - Output is push-pull value. */ + GPIO_CFGB_GPIO10OUTCFG_OD = 2, /*!< OD : FNCSEL = 0x3 - Output is open drain value. */ + GPIO_CFGB_GPIO10OUTCFG_TS = 3, /*!< TS : FNCSEL = 0x3 - Output is tri-state value. */ +} GPIO_CFGB_GPIO10OUTCFG_Enum; + +/* ============================================= GPIO CFGB GPIO10INCFG [8..8] ============================================== */ +typedef enum { /*!< GPIO_CFGB_GPIO10INCFG */ + GPIO_CFGB_GPIO10INCFG_READ = 0, /*!< READ : Read the GPIO pin data value. */ + GPIO_CFGB_GPIO10INCFG_RDZERO = 1, /*!< RDZERO : INTD = 0 - Readback will always be zero value. */ +} GPIO_CFGB_GPIO10INCFG_Enum; + +/* ============================================== GPIO CFGB GPIO9INTD [7..7] =============================================== */ +typedef enum { /*!< GPIO_CFGB_GPIO9INTD */ + GPIO_CFGB_GPIO9INTD_nCELOW = 0, /*!< nCELOW : FNCSEL = 0x2 - nCE polarity active low value. */ + GPIO_CFGB_GPIO9INTD_nCEHIGH = 1, /*!< nCEHIGH : FNCSEL = 0x2 - nCE polarity active high value. */ +} GPIO_CFGB_GPIO9INTD_Enum; + +/* ============================================= GPIO CFGB GPIO9OUTCFG [5..6] ============================================== */ +typedef enum { /*!< GPIO_CFGB_GPIO9OUTCFG */ + GPIO_CFGB_GPIO9OUTCFG_DIS = 0, /*!< DIS : FNCSEL = 0x3 - Output disabled value. */ + GPIO_CFGB_GPIO9OUTCFG_PUSHPULL = 1, /*!< PUSHPULL : FNCSEL = 0x3 - Output is push-pull value. */ + GPIO_CFGB_GPIO9OUTCFG_OD = 2, /*!< OD : FNCSEL = 0x3 - Output is open drain value. */ + GPIO_CFGB_GPIO9OUTCFG_TS = 3, /*!< TS : FNCSEL = 0x3 - Output is tri-state value. */ +} GPIO_CFGB_GPIO9OUTCFG_Enum; + +/* ============================================== GPIO CFGB GPIO9INCFG [4..4] ============================================== */ +typedef enum { /*!< GPIO_CFGB_GPIO9INCFG */ + GPIO_CFGB_GPIO9INCFG_READ = 0, /*!< READ : Read the GPIO pin data value. */ + GPIO_CFGB_GPIO9INCFG_RDZERO = 1, /*!< RDZERO : INTD = 0 - Readback will always be zero value. */ +} GPIO_CFGB_GPIO9INCFG_Enum; + +/* ============================================== GPIO CFGB GPIO8INTD [3..3] =============================================== */ +typedef enum { /*!< GPIO_CFGB_GPIO8INTD */ + GPIO_CFGB_GPIO8INTD_nCELOW = 0, /*!< nCELOW : FNCSEL = 0x2 - nCE polarity active low value. */ + GPIO_CFGB_GPIO8INTD_nCEHIGH = 1, /*!< nCEHIGH : FNCSEL = 0x2 - nCE polarity active high value. */ +} GPIO_CFGB_GPIO8INTD_Enum; + +/* ============================================= GPIO CFGB GPIO8OUTCFG [1..2] ============================================== */ +typedef enum { /*!< GPIO_CFGB_GPIO8OUTCFG */ + GPIO_CFGB_GPIO8OUTCFG_DIS = 0, /*!< DIS : FNCSEL = 0x3 - Output disabled value. */ + GPIO_CFGB_GPIO8OUTCFG_PUSHPULL = 1, /*!< PUSHPULL : FNCSEL = 0x3 - Output is push-pull value. */ + GPIO_CFGB_GPIO8OUTCFG_OD = 2, /*!< OD : FNCSEL = 0x3 - Output is open drain value. */ + GPIO_CFGB_GPIO8OUTCFG_TS = 3, /*!< TS : FNCSEL = 0x3 - Output is tri-state value. */ +} GPIO_CFGB_GPIO8OUTCFG_Enum; + +/* ============================================== GPIO CFGB GPIO8INCFG [0..0] ============================================== */ +typedef enum { /*!< GPIO_CFGB_GPIO8INCFG */ + GPIO_CFGB_GPIO8INCFG_READ = 0, /*!< READ : Read the GPIO pin data value. */ + GPIO_CFGB_GPIO8INCFG_RDZERO = 1, /*!< RDZERO : INTD = 0 - Readback will always be zero value. */ +} GPIO_CFGB_GPIO8INCFG_Enum; + +/* ========================================================= CFGC ========================================================== */ +/* ============================================= GPIO CFGC GPIO23INTD [31..31] ============================================= */ +typedef enum { /*!< GPIO_CFGC_GPIO23INTD */ + GPIO_CFGC_GPIO23INTD_nCELOW = 0, /*!< nCELOW : FNCSEL = 0x1 - nCE polarity active low value. */ + GPIO_CFGC_GPIO23INTD_nCEHIGH = 1, /*!< nCEHIGH : FNCSEL = 0x1 - nCE polarity active high value. */ +} GPIO_CFGC_GPIO23INTD_Enum; + +/* ============================================ GPIO CFGC GPIO23OUTCFG [29..30] ============================================ */ +typedef enum { /*!< GPIO_CFGC_GPIO23OUTCFG */ + GPIO_CFGC_GPIO23OUTCFG_DIS = 0, /*!< DIS : FNCSEL = 0x3 - Output disabled value. */ + GPIO_CFGC_GPIO23OUTCFG_PUSHPULL = 1, /*!< PUSHPULL : FNCSEL = 0x3 - Output is push-pull value. */ + GPIO_CFGC_GPIO23OUTCFG_OD = 2, /*!< OD : FNCSEL = 0x3 - Output is open drain value. */ + GPIO_CFGC_GPIO23OUTCFG_TS = 3, /*!< TS : FNCSEL = 0x3 - Output is tri-state value. */ +} GPIO_CFGC_GPIO23OUTCFG_Enum; + +/* ============================================ GPIO CFGC GPIO23INCFG [28..28] ============================================= */ +typedef enum { /*!< GPIO_CFGC_GPIO23INCFG */ + GPIO_CFGC_GPIO23INCFG_READ = 0, /*!< READ : Read the GPIO pin data value. */ + GPIO_CFGC_GPIO23INCFG_RDZERO = 1, /*!< RDZERO : INTD = 0 - Readback will always be zero value. */ +} GPIO_CFGC_GPIO23INCFG_Enum; + +/* ============================================= GPIO CFGC GPIO22INTD [27..27] ============================================= */ +typedef enum { /*!< GPIO_CFGC_GPIO22INTD */ + GPIO_CFGC_GPIO22INTD_nCELOW = 0, /*!< nCELOW : FNCSEL = 0x1 - nCE polarity active low value. */ + GPIO_CFGC_GPIO22INTD_nCEHIGH = 1, /*!< nCEHIGH : FNCSEL = 0x1 - nCE polarity active high value. */ +} GPIO_CFGC_GPIO22INTD_Enum; + +/* ============================================ GPIO CFGC GPIO22OUTCFG [25..26] ============================================ */ +typedef enum { /*!< GPIO_CFGC_GPIO22OUTCFG */ + GPIO_CFGC_GPIO22OUTCFG_DIS = 0, /*!< DIS : FNCSEL = 0x3 - Output disabled value. */ + GPIO_CFGC_GPIO22OUTCFG_PUSHPULL = 1, /*!< PUSHPULL : FNCSEL = 0x3 - Output is push-pull value. */ + GPIO_CFGC_GPIO22OUTCFG_OD = 2, /*!< OD : FNCSEL = 0x3 - Output is open drain value. */ + GPIO_CFGC_GPIO22OUTCFG_TS = 3, /*!< TS : FNCSEL = 0x3 - Output is tri-state value. */ +} GPIO_CFGC_GPIO22OUTCFG_Enum; + +/* ============================================ GPIO CFGC GPIO22INCFG [24..24] ============================================= */ +typedef enum { /*!< GPIO_CFGC_GPIO22INCFG */ + GPIO_CFGC_GPIO22INCFG_READ = 0, /*!< READ : Read the GPIO pin data value. */ + GPIO_CFGC_GPIO22INCFG_RDZERO = 1, /*!< RDZERO : INTD = 0 - Readback will always be zero value. */ +} GPIO_CFGC_GPIO22INCFG_Enum; + +/* ============================================= GPIO CFGC GPIO21INTD [23..23] ============================================= */ +typedef enum { /*!< GPIO_CFGC_GPIO21INTD */ + GPIO_CFGC_GPIO21INTD_nCELOW = 0, /*!< nCELOW : FNCSEL = 0x1 - nCE polarity active low value. */ + GPIO_CFGC_GPIO21INTD_nCEHIGH = 1, /*!< nCEHIGH : FNCSEL = 0x1 - nCE polarity active high value. */ +} GPIO_CFGC_GPIO21INTD_Enum; + +/* ============================================ GPIO CFGC GPIO21OUTCFG [21..22] ============================================ */ +typedef enum { /*!< GPIO_CFGC_GPIO21OUTCFG */ + GPIO_CFGC_GPIO21OUTCFG_DIS = 0, /*!< DIS : FNCSEL = 0x3 - Output disabled value. */ + GPIO_CFGC_GPIO21OUTCFG_PUSHPULL = 1, /*!< PUSHPULL : FNCSEL = 0x3 - Output is push-pull value. */ + GPIO_CFGC_GPIO21OUTCFG_OD = 2, /*!< OD : FNCSEL = 0x3 - Output is open drain value. */ + GPIO_CFGC_GPIO21OUTCFG_TS = 3, /*!< TS : FNCSEL = 0x3 - Output is tri-state value. */ +} GPIO_CFGC_GPIO21OUTCFG_Enum; + +/* ============================================ GPIO CFGC GPIO21INCFG [20..20] ============================================= */ +typedef enum { /*!< GPIO_CFGC_GPIO21INCFG */ + GPIO_CFGC_GPIO21INCFG_READ = 0, /*!< READ : Read the GPIO pin data value. */ + GPIO_CFGC_GPIO21INCFG_RDZERO = 1, /*!< RDZERO : INTD = 0 - Readback will always be zero value. */ +} GPIO_CFGC_GPIO21INCFG_Enum; + +/* ============================================= GPIO CFGC GPIO20INTD [19..19] ============================================= */ +typedef enum { /*!< GPIO_CFGC_GPIO20INTD */ + GPIO_CFGC_GPIO20INTD_nCELOW = 0, /*!< nCELOW : FNCSEL = 0x1 - nCE polarity active low value. */ + GPIO_CFGC_GPIO20INTD_nCEHIGH = 1, /*!< nCEHIGH : FNCSEL = 0x1 - nCE polarity active high value. */ +} GPIO_CFGC_GPIO20INTD_Enum; + +/* ============================================ GPIO CFGC GPIO20OUTCFG [17..18] ============================================ */ +typedef enum { /*!< GPIO_CFGC_GPIO20OUTCFG */ + GPIO_CFGC_GPIO20OUTCFG_DIS = 0, /*!< DIS : FNCSEL = 0x3 - Output disabled value. */ + GPIO_CFGC_GPIO20OUTCFG_PUSHPULL = 1, /*!< PUSHPULL : FNCSEL = 0x3 - Output is push-pull value. */ + GPIO_CFGC_GPIO20OUTCFG_OD = 2, /*!< OD : FNCSEL = 0x3 - Output is open drain value. */ + GPIO_CFGC_GPIO20OUTCFG_TS = 3, /*!< TS : FNCSEL = 0x3 - Output is tri-state value. */ +} GPIO_CFGC_GPIO20OUTCFG_Enum; + +/* ============================================ GPIO CFGC GPIO20INCFG [16..16] ============================================= */ +typedef enum { /*!< GPIO_CFGC_GPIO20INCFG */ + GPIO_CFGC_GPIO20INCFG_READ = 0, /*!< READ : Read the GPIO pin data value. */ + GPIO_CFGC_GPIO20INCFG_RDZERO = 1, /*!< RDZERO : INTD = 0 - Readback will always be zero value. */ +} GPIO_CFGC_GPIO20INCFG_Enum; + +/* ============================================= GPIO CFGC GPIO19INTD [15..15] ============================================= */ +typedef enum { /*!< GPIO_CFGC_GPIO19INTD */ + GPIO_CFGC_GPIO19INTD_nCELOW = 0, /*!< nCELOW : FNCSEL = 0x1 - nCE polarity active low value. */ + GPIO_CFGC_GPIO19INTD_nCEHIGH = 1, /*!< nCEHIGH : FNCSEL = 0x1 - nCE polarity active high value. */ +} GPIO_CFGC_GPIO19INTD_Enum; + +/* ============================================ GPIO CFGC GPIO19OUTCFG [13..14] ============================================ */ +typedef enum { /*!< GPIO_CFGC_GPIO19OUTCFG */ + GPIO_CFGC_GPIO19OUTCFG_DIS = 0, /*!< DIS : FNCSEL = 0x3 - Output disabled value. */ + GPIO_CFGC_GPIO19OUTCFG_PUSHPULL = 1, /*!< PUSHPULL : FNCSEL = 0x3 - Output is push-pull value. */ + GPIO_CFGC_GPIO19OUTCFG_OD = 2, /*!< OD : FNCSEL = 0x3 - Output is open drain value. */ + GPIO_CFGC_GPIO19OUTCFG_TS = 3, /*!< TS : FNCSEL = 0x3 - Output is tri-state value. */ +} GPIO_CFGC_GPIO19OUTCFG_Enum; + +/* ============================================ GPIO CFGC GPIO19INCFG [12..12] ============================================= */ +typedef enum { /*!< GPIO_CFGC_GPIO19INCFG */ + GPIO_CFGC_GPIO19INCFG_READ = 0, /*!< READ : Read the GPIO pin data value. */ + GPIO_CFGC_GPIO19INCFG_RDZERO = 1, /*!< RDZERO : INTD = 0 - Readback will always be zero value. */ +} GPIO_CFGC_GPIO19INCFG_Enum; + +/* ============================================= GPIO CFGC GPIO18INTD [11..11] ============================================= */ +typedef enum { /*!< GPIO_CFGC_GPIO18INTD */ + GPIO_CFGC_GPIO18INTD_nCELOW = 0, /*!< nCELOW : FNCSEL = 0x1 - nCE polarity active low value. */ + GPIO_CFGC_GPIO18INTD_nCEHIGH = 1, /*!< nCEHIGH : FNCSEL = 0x1 - nCE polarity active high value. */ +} GPIO_CFGC_GPIO18INTD_Enum; + +/* ============================================ GPIO CFGC GPIO18OUTCFG [9..10] ============================================= */ +typedef enum { /*!< GPIO_CFGC_GPIO18OUTCFG */ + GPIO_CFGC_GPIO18OUTCFG_DIS = 0, /*!< DIS : FNCSEL = 0x3 - Output disabled value. */ + GPIO_CFGC_GPIO18OUTCFG_PUSHPULL = 1, /*!< PUSHPULL : FNCSEL = 0x3 - Output is push-pull value. */ + GPIO_CFGC_GPIO18OUTCFG_OD = 2, /*!< OD : FNCSEL = 0x3 - Output is open drain value. */ + GPIO_CFGC_GPIO18OUTCFG_TS = 3, /*!< TS : FNCSEL = 0x3 - Output is tri-state value. */ +} GPIO_CFGC_GPIO18OUTCFG_Enum; + +/* ============================================= GPIO CFGC GPIO18INCFG [8..8] ============================================== */ +typedef enum { /*!< GPIO_CFGC_GPIO18INCFG */ + GPIO_CFGC_GPIO18INCFG_READ = 0, /*!< READ : Read the GPIO pin data value. */ + GPIO_CFGC_GPIO18INCFG_RDZERO = 1, /*!< RDZERO : INTD = 0 - Readback will always be zero value. */ +} GPIO_CFGC_GPIO18INCFG_Enum; + +/* ============================================== GPIO CFGC GPIO17INTD [7..7] ============================================== */ +typedef enum { /*!< GPIO_CFGC_GPIO17INTD */ + GPIO_CFGC_GPIO17INTD_nCELOW = 0, /*!< nCELOW : FNCSEL = 0x1 - nCE polarity active low value. */ + GPIO_CFGC_GPIO17INTD_nCEHIGH = 1, /*!< nCEHIGH : FNCSEL = 0x1 - nCE polarity active high value. */ +} GPIO_CFGC_GPIO17INTD_Enum; + +/* ============================================= GPIO CFGC GPIO17OUTCFG [5..6] ============================================= */ +typedef enum { /*!< GPIO_CFGC_GPIO17OUTCFG */ + GPIO_CFGC_GPIO17OUTCFG_DIS = 0, /*!< DIS : FNCSEL = 0x3 - Output disabled value. */ + GPIO_CFGC_GPIO17OUTCFG_PUSHPULL = 1, /*!< PUSHPULL : FNCSEL = 0x3 - Output is push-pull value. */ + GPIO_CFGC_GPIO17OUTCFG_OD = 2, /*!< OD : FNCSEL = 0x3 - Output is open drain value. */ + GPIO_CFGC_GPIO17OUTCFG_TS = 3, /*!< TS : FNCSEL = 0x3 - Output is tri-state value. */ +} GPIO_CFGC_GPIO17OUTCFG_Enum; + +/* ============================================= GPIO CFGC GPIO17INCFG [4..4] ============================================== */ +typedef enum { /*!< GPIO_CFGC_GPIO17INCFG */ + GPIO_CFGC_GPIO17INCFG_READ = 0, /*!< READ : Read the GPIO pin data value. */ + GPIO_CFGC_GPIO17INCFG_RDZERO = 1, /*!< RDZERO : INTD = 0 - Readback will always be zero value. */ +} GPIO_CFGC_GPIO17INCFG_Enum; + +/* ============================================== GPIO CFGC GPIO16INTD [3..3] ============================================== */ +typedef enum { /*!< GPIO_CFGC_GPIO16INTD */ + GPIO_CFGC_GPIO16INTD_nCELOW = 0, /*!< nCELOW : FNCSEL = 0x1 - nCE polarity active low value. */ + GPIO_CFGC_GPIO16INTD_nCEHIGH = 1, /*!< nCEHIGH : FNCSEL = 0x1 - nCE polarity active high value. */ +} GPIO_CFGC_GPIO16INTD_Enum; + +/* ============================================= GPIO CFGC GPIO16OUTCFG [1..2] ============================================= */ +typedef enum { /*!< GPIO_CFGC_GPIO16OUTCFG */ + GPIO_CFGC_GPIO16OUTCFG_DIS = 0, /*!< DIS : FNCSEL = 0x3 - Output disabled value. */ + GPIO_CFGC_GPIO16OUTCFG_PUSHPULL = 1, /*!< PUSHPULL : FNCSEL = 0x3 - Output is push-pull value. */ + GPIO_CFGC_GPIO16OUTCFG_OD = 2, /*!< OD : FNCSEL = 0x3 - Output is open drain value. */ + GPIO_CFGC_GPIO16OUTCFG_TS = 3, /*!< TS : FNCSEL = 0x3 - Output is tri-state value. */ +} GPIO_CFGC_GPIO16OUTCFG_Enum; + +/* ============================================= GPIO CFGC GPIO16INCFG [0..0] ============================================== */ +typedef enum { /*!< GPIO_CFGC_GPIO16INCFG */ + GPIO_CFGC_GPIO16INCFG_READ = 0, /*!< READ : Read the GPIO pin data value. */ + GPIO_CFGC_GPIO16INCFG_RDZERO = 1, /*!< RDZERO : INTD = 0 - Readback will always be zero value. */ +} GPIO_CFGC_GPIO16INCFG_Enum; + +/* ========================================================= CFGD ========================================================== */ +/* ============================================= GPIO CFGD GPIO31INTD [31..31] ============================================= */ +typedef enum { /*!< GPIO_CFGD_GPIO31INTD */ + GPIO_CFGD_GPIO31INTD_nCELOW = 0, /*!< nCELOW : FNCSEL = 0x1 - nCE polarity active low value. */ + GPIO_CFGD_GPIO31INTD_nCEHIGH = 1, /*!< nCEHIGH : FNCSEL = 0x1 - nCE polarity active high value. */ +} GPIO_CFGD_GPIO31INTD_Enum; + +/* ============================================ GPIO CFGD GPIO31OUTCFG [29..30] ============================================ */ +typedef enum { /*!< GPIO_CFGD_GPIO31OUTCFG */ + GPIO_CFGD_GPIO31OUTCFG_DIS = 0, /*!< DIS : FNCSEL = 0x3 - Output disabled value. */ + GPIO_CFGD_GPIO31OUTCFG_PUSHPULL = 1, /*!< PUSHPULL : FNCSEL = 0x3 - Output is push-pull value. */ + GPIO_CFGD_GPIO31OUTCFG_OD = 2, /*!< OD : FNCSEL = 0x3 - Output is open drain value. */ + GPIO_CFGD_GPIO31OUTCFG_TS = 3, /*!< TS : FNCSEL = 0x3 - Output is tri-state value. */ +} GPIO_CFGD_GPIO31OUTCFG_Enum; + +/* ============================================ GPIO CFGD GPIO31INCFG [28..28] ============================================= */ +typedef enum { /*!< GPIO_CFGD_GPIO31INCFG */ + GPIO_CFGD_GPIO31INCFG_READ = 0, /*!< READ : Read the GPIO pin data value. */ + GPIO_CFGD_GPIO31INCFG_RDZERO = 1, /*!< RDZERO : INTD = 0 - Readback will always be zero value. */ +} GPIO_CFGD_GPIO31INCFG_Enum; + +/* ============================================= GPIO CFGD GPIO30INTD [27..27] ============================================= */ +typedef enum { /*!< GPIO_CFGD_GPIO30INTD */ + GPIO_CFGD_GPIO30INTD_nCELOW = 0, /*!< nCELOW : FNCSEL = 0x1 - nCE polarity active low value. */ + GPIO_CFGD_GPIO30INTD_nCEHIGH = 1, /*!< nCEHIGH : FNCSEL = 0x1 - nCE polarity active high value. */ +} GPIO_CFGD_GPIO30INTD_Enum; + +/* ============================================ GPIO CFGD GPIO30OUTCFG [25..26] ============================================ */ +typedef enum { /*!< GPIO_CFGD_GPIO30OUTCFG */ + GPIO_CFGD_GPIO30OUTCFG_DIS = 0, /*!< DIS : FNCSEL = 0x3 - Output disabled value. */ + GPIO_CFGD_GPIO30OUTCFG_PUSHPULL = 1, /*!< PUSHPULL : FNCSEL = 0x3 - Output is push-pull value. */ + GPIO_CFGD_GPIO30OUTCFG_OD = 2, /*!< OD : FNCSEL = 0x3 - Output is open drain value. */ + GPIO_CFGD_GPIO30OUTCFG_TS = 3, /*!< TS : FNCSEL = 0x3 - Output is tri-state value. */ +} GPIO_CFGD_GPIO30OUTCFG_Enum; + +/* ============================================ GPIO CFGD GPIO30INCFG [24..24] ============================================= */ +typedef enum { /*!< GPIO_CFGD_GPIO30INCFG */ + GPIO_CFGD_GPIO30INCFG_READ = 0, /*!< READ : Read the GPIO pin data value. */ + GPIO_CFGD_GPIO30INCFG_RDZERO = 1, /*!< RDZERO : INTD = 0 - Readback will always be zero value. */ +} GPIO_CFGD_GPIO30INCFG_Enum; + +/* ============================================= GPIO CFGD GPIO29INTD [23..23] ============================================= */ +typedef enum { /*!< GPIO_CFGD_GPIO29INTD */ + GPIO_CFGD_GPIO29INTD_nCELOW = 0, /*!< nCELOW : FNCSEL = 0x1 - nCE polarity active low value. */ + GPIO_CFGD_GPIO29INTD_nCEHIGH = 1, /*!< nCEHIGH : FNCSEL = 0x1 - nCE polarity active high value. */ +} GPIO_CFGD_GPIO29INTD_Enum; + +/* ============================================ GPIO CFGD GPIO29OUTCFG [21..22] ============================================ */ +typedef enum { /*!< GPIO_CFGD_GPIO29OUTCFG */ + GPIO_CFGD_GPIO29OUTCFG_DIS = 0, /*!< DIS : FNCSEL = 0x3 - Output disabled value. */ + GPIO_CFGD_GPIO29OUTCFG_PUSHPULL = 1, /*!< PUSHPULL : FNCSEL = 0x3 - Output is push-pull value. */ + GPIO_CFGD_GPIO29OUTCFG_OD = 2, /*!< OD : FNCSEL = 0x3 - Output is open drain value. */ + GPIO_CFGD_GPIO29OUTCFG_TS = 3, /*!< TS : FNCSEL = 0x3 - Output is tri-state value. */ +} GPIO_CFGD_GPIO29OUTCFG_Enum; + +/* ============================================ GPIO CFGD GPIO29INCFG [20..20] ============================================= */ +typedef enum { /*!< GPIO_CFGD_GPIO29INCFG */ + GPIO_CFGD_GPIO29INCFG_READ = 0, /*!< READ : Read the GPIO pin data value. */ + GPIO_CFGD_GPIO29INCFG_RDZERO = 1, /*!< RDZERO : INTD = 0 - Readback will always be zero value. */ +} GPIO_CFGD_GPIO29INCFG_Enum; + +/* ============================================= GPIO CFGD GPIO28INTD [19..19] ============================================= */ +typedef enum { /*!< GPIO_CFGD_GPIO28INTD */ + GPIO_CFGD_GPIO28INTD_nCELOW = 0, /*!< nCELOW : FNCSEL = 0x1 - nCE polarity active low value. */ + GPIO_CFGD_GPIO28INTD_nCEHIGH = 1, /*!< nCEHIGH : FNCSEL = 0x1 - nCE polarity active high value. */ +} GPIO_CFGD_GPIO28INTD_Enum; + +/* ============================================ GPIO CFGD GPIO28OUTCFG [17..18] ============================================ */ +typedef enum { /*!< GPIO_CFGD_GPIO28OUTCFG */ + GPIO_CFGD_GPIO28OUTCFG_DIS = 0, /*!< DIS : FNCSEL = 0x3 - Output disabled value. */ + GPIO_CFGD_GPIO28OUTCFG_PUSHPULL = 1, /*!< PUSHPULL : FNCSEL = 0x3 - Output is push-pull value. */ + GPIO_CFGD_GPIO28OUTCFG_OD = 2, /*!< OD : FNCSEL = 0x3 - Output is open drain value. */ + GPIO_CFGD_GPIO28OUTCFG_TS = 3, /*!< TS : FNCSEL = 0x3 - Output is tri-state value. */ +} GPIO_CFGD_GPIO28OUTCFG_Enum; + +/* ============================================ GPIO CFGD GPIO28INCFG [16..16] ============================================= */ +typedef enum { /*!< GPIO_CFGD_GPIO28INCFG */ + GPIO_CFGD_GPIO28INCFG_READ = 0, /*!< READ : Read the GPIO pin data value. */ + GPIO_CFGD_GPIO28INCFG_RDZERO = 1, /*!< RDZERO : INTD = 0 - Readback will always be zero value. */ +} GPIO_CFGD_GPIO28INCFG_Enum; + +/* ============================================= GPIO CFGD GPIO27INTD [15..15] ============================================= */ +typedef enum { /*!< GPIO_CFGD_GPIO27INTD */ + GPIO_CFGD_GPIO27INTD_nCELOW = 0, /*!< nCELOW : FNCSEL = 0x1 - nCE polarity active low value. */ + GPIO_CFGD_GPIO27INTD_nCEHIGH = 1, /*!< nCEHIGH : FNCSEL = 0x1 - nCE polarity active high value. */ +} GPIO_CFGD_GPIO27INTD_Enum; + +/* ============================================ GPIO CFGD GPIO27OUTCFG [13..14] ============================================ */ +typedef enum { /*!< GPIO_CFGD_GPIO27OUTCFG */ + GPIO_CFGD_GPIO27OUTCFG_DIS = 0, /*!< DIS : FNCSEL = 0x3 - Output disabled value. */ + GPIO_CFGD_GPIO27OUTCFG_PUSHPULL = 1, /*!< PUSHPULL : FNCSEL = 0x3 - Output is push-pull value. */ + GPIO_CFGD_GPIO27OUTCFG_OD = 2, /*!< OD : FNCSEL = 0x3 - Output is open drain value. */ + GPIO_CFGD_GPIO27OUTCFG_TS = 3, /*!< TS : FNCSEL = 0x3 - Output is tri-state value. */ +} GPIO_CFGD_GPIO27OUTCFG_Enum; + +/* ============================================ GPIO CFGD GPIO27INCFG [12..12] ============================================= */ +typedef enum { /*!< GPIO_CFGD_GPIO27INCFG */ + GPIO_CFGD_GPIO27INCFG_READ = 0, /*!< READ : Read the GPIO pin data value. */ + GPIO_CFGD_GPIO27INCFG_RDZERO = 1, /*!< RDZERO : INTD = 0 - Readback will always be zero value. */ +} GPIO_CFGD_GPIO27INCFG_Enum; + +/* ============================================= GPIO CFGD GPIO26INTD [11..11] ============================================= */ +typedef enum { /*!< GPIO_CFGD_GPIO26INTD */ + GPIO_CFGD_GPIO26INTD_nCELOW = 0, /*!< nCELOW : FNCSEL = 0x1 - nCE polarity active low value. */ + GPIO_CFGD_GPIO26INTD_nCEHIGH = 1, /*!< nCEHIGH : FNCSEL = 0x1 - nCE polarity active high value. */ +} GPIO_CFGD_GPIO26INTD_Enum; + +/* ============================================ GPIO CFGD GPIO26OUTCFG [9..10] ============================================= */ +typedef enum { /*!< GPIO_CFGD_GPIO26OUTCFG */ + GPIO_CFGD_GPIO26OUTCFG_DIS = 0, /*!< DIS : FNCSEL = 0x3 - Output disabled value. */ + GPIO_CFGD_GPIO26OUTCFG_PUSHPULL = 1, /*!< PUSHPULL : FNCSEL = 0x3 - Output is push-pull value. */ + GPIO_CFGD_GPIO26OUTCFG_OD = 2, /*!< OD : FNCSEL = 0x3 - Output is open drain value. */ + GPIO_CFGD_GPIO26OUTCFG_TS = 3, /*!< TS : FNCSEL = 0x3 - Output is tri-state value. */ +} GPIO_CFGD_GPIO26OUTCFG_Enum; + +/* ============================================= GPIO CFGD GPIO26INCFG [8..8] ============================================== */ +typedef enum { /*!< GPIO_CFGD_GPIO26INCFG */ + GPIO_CFGD_GPIO26INCFG_READ = 0, /*!< READ : Read the GPIO pin data value. */ + GPIO_CFGD_GPIO26INCFG_RDZERO = 1, /*!< RDZERO : INTD = 0 - Readback will always be zero value. */ +} GPIO_CFGD_GPIO26INCFG_Enum; + +/* ============================================== GPIO CFGD GPIO25INTD [7..7] ============================================== */ +typedef enum { /*!< GPIO_CFGD_GPIO25INTD */ + GPIO_CFGD_GPIO25INTD_nCELOW = 0, /*!< nCELOW : FNCSEL = 0x1 - nCE polarity active low value. */ + GPIO_CFGD_GPIO25INTD_nCEHIGH = 1, /*!< nCEHIGH : FNCSEL = 0x1 - nCE polarity active high value. */ +} GPIO_CFGD_GPIO25INTD_Enum; + +/* ============================================= GPIO CFGD GPIO25OUTCFG [5..6] ============================================= */ +typedef enum { /*!< GPIO_CFGD_GPIO25OUTCFG */ + GPIO_CFGD_GPIO25OUTCFG_DIS = 0, /*!< DIS : FNCSEL = 0x3 - Output disabled value. */ + GPIO_CFGD_GPIO25OUTCFG_PUSHPULL = 1, /*!< PUSHPULL : FNCSEL = 0x3 - Output is push-pull value. */ + GPIO_CFGD_GPIO25OUTCFG_OD = 2, /*!< OD : FNCSEL = 0x3 - Output is open drain value. */ + GPIO_CFGD_GPIO25OUTCFG_TS = 3, /*!< TS : FNCSEL = 0x3 - Output is tri-state value. */ +} GPIO_CFGD_GPIO25OUTCFG_Enum; + +/* ============================================= GPIO CFGD GPIO25INCFG [4..4] ============================================== */ +typedef enum { /*!< GPIO_CFGD_GPIO25INCFG */ + GPIO_CFGD_GPIO25INCFG_READ = 0, /*!< READ : Read the GPIO pin data value. */ + GPIO_CFGD_GPIO25INCFG_RDZERO = 1, /*!< RDZERO : INTD = 0 - Readback will always be zero value. */ +} GPIO_CFGD_GPIO25INCFG_Enum; + +/* ============================================== GPIO CFGD GPIO24INTD [3..3] ============================================== */ +typedef enum { /*!< GPIO_CFGD_GPIO24INTD */ + GPIO_CFGD_GPIO24INTD_nCELOW = 0, /*!< nCELOW : FNCSEL = 0x1 - nCE polarity active low value. */ + GPIO_CFGD_GPIO24INTD_nCEHIGH = 1, /*!< nCEHIGH : FNCSEL = 0x1 - nCE polarity active high value. */ +} GPIO_CFGD_GPIO24INTD_Enum; + +/* ============================================= GPIO CFGD GPIO24OUTCFG [1..2] ============================================= */ +typedef enum { /*!< GPIO_CFGD_GPIO24OUTCFG */ + GPIO_CFGD_GPIO24OUTCFG_DIS = 0, /*!< DIS : FNCSEL = 0x3 - Output disabled value. */ + GPIO_CFGD_GPIO24OUTCFG_PUSHPULL = 1, /*!< PUSHPULL : FNCSEL = 0x3 - Output is push-pull value. */ + GPIO_CFGD_GPIO24OUTCFG_OD = 2, /*!< OD : FNCSEL = 0x3 - Output is open drain value. */ + GPIO_CFGD_GPIO24OUTCFG_TS = 3, /*!< TS : FNCSEL = 0x3 - Output is tri-state value. */ +} GPIO_CFGD_GPIO24OUTCFG_Enum; + +/* ============================================= GPIO CFGD GPIO24INCFG [0..0] ============================================== */ +typedef enum { /*!< GPIO_CFGD_GPIO24INCFG */ + GPIO_CFGD_GPIO24INCFG_READ = 0, /*!< READ : Read the GPIO pin data value. */ + GPIO_CFGD_GPIO24INCFG_RDZERO = 1, /*!< RDZERO : INTD = 0 - Readback will always be zero value. */ +} GPIO_CFGD_GPIO24INCFG_Enum; + +/* ========================================================= CFGE ========================================================== */ +/* ============================================= GPIO CFGE GPIO39INTD [31..31] ============================================= */ +typedef enum { /*!< GPIO_CFGE_GPIO39INTD */ + GPIO_CFGE_GPIO39INTD_INTDIS = 0, /*!< INTDIS : INCFG = 1 - No interrupt on GPIO transition value. */ + GPIO_CFGE_GPIO39INTD_INTBOTH = 1, /*!< INTBOTH : INCFG = 1 - Interrupt on either low to high or high + to low GPIO transition value. */ +} GPIO_CFGE_GPIO39INTD_Enum; + +/* ============================================ GPIO CFGE GPIO39OUTCFG [29..30] ============================================ */ +typedef enum { /*!< GPIO_CFGE_GPIO39OUTCFG */ + GPIO_CFGE_GPIO39OUTCFG_DIS = 0, /*!< DIS : FNCSEL = 0x3 - Output disabled value. */ + GPIO_CFGE_GPIO39OUTCFG_PUSHPULL = 1, /*!< PUSHPULL : FNCSEL = 0x3 - Output is push-pull value. */ + GPIO_CFGE_GPIO39OUTCFG_OD = 2, /*!< OD : FNCSEL = 0x3 - Output is open drain value. */ + GPIO_CFGE_GPIO39OUTCFG_TS = 3, /*!< TS : FNCSEL = 0x3 - Output is tri-state value. */ +} GPIO_CFGE_GPIO39OUTCFG_Enum; + +/* ============================================ GPIO CFGE GPIO39INCFG [28..28] ============================================= */ +typedef enum { /*!< GPIO_CFGE_GPIO39INCFG */ + GPIO_CFGE_GPIO39INCFG_READ = 0, /*!< READ : Read the GPIO pin data value. */ + GPIO_CFGE_GPIO39INCFG_RDZERO = 1, /*!< RDZERO : INTD = 0 - Readback will always be zero value. */ +} GPIO_CFGE_GPIO39INCFG_Enum; + +/* ============================================= GPIO CFGE GPIO38INTD [27..27] ============================================= */ +typedef enum { /*!< GPIO_CFGE_GPIO38INTD */ + GPIO_CFGE_GPIO38INTD_nCELOW = 0, /*!< nCELOW : FNCSEL = 0x1 - nCE polarity active low value. */ + GPIO_CFGE_GPIO38INTD_nCEHIGH = 1, /*!< nCEHIGH : FNCSEL = 0x1 - nCE polarity active high value. */ +} GPIO_CFGE_GPIO38INTD_Enum; + +/* ============================================ GPIO CFGE GPIO38OUTCFG [25..26] ============================================ */ +typedef enum { /*!< GPIO_CFGE_GPIO38OUTCFG */ + GPIO_CFGE_GPIO38OUTCFG_DIS = 0, /*!< DIS : FNCSEL = 0x3 - Output disabled value. */ + GPIO_CFGE_GPIO38OUTCFG_PUSHPULL = 1, /*!< PUSHPULL : FNCSEL = 0x3 - Output is push-pull value. */ + GPIO_CFGE_GPIO38OUTCFG_OD = 2, /*!< OD : FNCSEL = 0x3 - Output is open drain value. */ + GPIO_CFGE_GPIO38OUTCFG_TS = 3, /*!< TS : FNCSEL = 0x3 - Output is tri-state value. */ +} GPIO_CFGE_GPIO38OUTCFG_Enum; + +/* ============================================ GPIO CFGE GPIO38INCFG [24..24] ============================================= */ +typedef enum { /*!< GPIO_CFGE_GPIO38INCFG */ + GPIO_CFGE_GPIO38INCFG_READ = 0, /*!< READ : Read the GPIO pin data value. */ + GPIO_CFGE_GPIO38INCFG_RDZERO = 1, /*!< RDZERO : INTD = 0 - Readback will always be zero value. */ +} GPIO_CFGE_GPIO38INCFG_Enum; + +/* ============================================= GPIO CFGE GPIO37INTD [23..23] ============================================= */ +typedef enum { /*!< GPIO_CFGE_GPIO37INTD */ + GPIO_CFGE_GPIO37INTD_nCELOW = 0, /*!< nCELOW : FNCSEL = 0x1 - nCE polarity active low value. */ + GPIO_CFGE_GPIO37INTD_nCEHIGH = 1, /*!< nCEHIGH : FNCSEL = 0x1 - nCE polarity active high value. */ +} GPIO_CFGE_GPIO37INTD_Enum; + +/* ============================================ GPIO CFGE GPIO37OUTCFG [21..22] ============================================ */ +typedef enum { /*!< GPIO_CFGE_GPIO37OUTCFG */ + GPIO_CFGE_GPIO37OUTCFG_DIS = 0, /*!< DIS : FNCSEL = 0x3 - Output disabled value. */ + GPIO_CFGE_GPIO37OUTCFG_PUSHPULL = 1, /*!< PUSHPULL : FNCSEL = 0x3 - Output is push-pull value. */ + GPIO_CFGE_GPIO37OUTCFG_OD = 2, /*!< OD : FNCSEL = 0x3 - Output is open drain value. */ + GPIO_CFGE_GPIO37OUTCFG_TS = 3, /*!< TS : FNCSEL = 0x3 - Output is tri-state value. */ +} GPIO_CFGE_GPIO37OUTCFG_Enum; + +/* ============================================ GPIO CFGE GPIO37INCFG [20..20] ============================================= */ +typedef enum { /*!< GPIO_CFGE_GPIO37INCFG */ + GPIO_CFGE_GPIO37INCFG_READ = 0, /*!< READ : Read the GPIO pin data value. */ + GPIO_CFGE_GPIO37INCFG_RDZERO = 1, /*!< RDZERO : INTD = 0 - Readback will always be zero value. */ +} GPIO_CFGE_GPIO37INCFG_Enum; + +/* ============================================= GPIO CFGE GPIO36INTD [19..19] ============================================= */ +typedef enum { /*!< GPIO_CFGE_GPIO36INTD */ + GPIO_CFGE_GPIO36INTD_nCELOW = 0, /*!< nCELOW : FNCSEL = 0x1 - nCE polarity active low value. */ + GPIO_CFGE_GPIO36INTD_nCEHIGH = 1, /*!< nCEHIGH : FNCSEL = 0x1 - nCE polarity active high value. */ +} GPIO_CFGE_GPIO36INTD_Enum; + +/* ============================================ GPIO CFGE GPIO36OUTCFG [17..18] ============================================ */ +typedef enum { /*!< GPIO_CFGE_GPIO36OUTCFG */ + GPIO_CFGE_GPIO36OUTCFG_DIS = 0, /*!< DIS : FNCSEL = 0x3 - Output disabled value. */ + GPIO_CFGE_GPIO36OUTCFG_PUSHPULL = 1, /*!< PUSHPULL : FNCSEL = 0x3 - Output is push-pull value. */ + GPIO_CFGE_GPIO36OUTCFG_OD = 2, /*!< OD : FNCSEL = 0x3 - Output is open drain value. */ + GPIO_CFGE_GPIO36OUTCFG_TS = 3, /*!< TS : FNCSEL = 0x3 - Output is tri-state value. */ +} GPIO_CFGE_GPIO36OUTCFG_Enum; + +/* ============================================ GPIO CFGE GPIO36INCFG [16..16] ============================================= */ +typedef enum { /*!< GPIO_CFGE_GPIO36INCFG */ + GPIO_CFGE_GPIO36INCFG_READ = 0, /*!< READ : Read the GPIO pin data value. */ + GPIO_CFGE_GPIO36INCFG_RDZERO = 1, /*!< RDZERO : INTD = 0 - Readback will always be zero value. */ +} GPIO_CFGE_GPIO36INCFG_Enum; + +/* ============================================= GPIO CFGE GPIO35INTD [15..15] ============================================= */ +typedef enum { /*!< GPIO_CFGE_GPIO35INTD */ + GPIO_CFGE_GPIO35INTD_nCELOW = 0, /*!< nCELOW : FNCSEL = 0x1 - nCE polarity active low value. */ + GPIO_CFGE_GPIO35INTD_nCEHIGH = 1, /*!< nCEHIGH : FNCSEL = 0x1 - nCE polarity active high value. */ +} GPIO_CFGE_GPIO35INTD_Enum; + +/* ============================================ GPIO CFGE GPIO35OUTCFG [13..14] ============================================ */ +typedef enum { /*!< GPIO_CFGE_GPIO35OUTCFG */ + GPIO_CFGE_GPIO35OUTCFG_DIS = 0, /*!< DIS : FNCSEL = 0x3 - Output disabled value. */ + GPIO_CFGE_GPIO35OUTCFG_PUSHPULL = 1, /*!< PUSHPULL : FNCSEL = 0x3 - Output is push-pull value. */ + GPIO_CFGE_GPIO35OUTCFG_OD = 2, /*!< OD : FNCSEL = 0x3 - Output is open drain value. */ + GPIO_CFGE_GPIO35OUTCFG_TS = 3, /*!< TS : FNCSEL = 0x3 - Output is tri-state value. */ +} GPIO_CFGE_GPIO35OUTCFG_Enum; + +/* ============================================ GPIO CFGE GPIO35INCFG [12..12] ============================================= */ +typedef enum { /*!< GPIO_CFGE_GPIO35INCFG */ + GPIO_CFGE_GPIO35INCFG_READ = 0, /*!< READ : Read the GPIO pin data value. */ + GPIO_CFGE_GPIO35INCFG_RDZERO = 1, /*!< RDZERO : INTD = 0 - Readback will always be zero value. */ +} GPIO_CFGE_GPIO35INCFG_Enum; + +/* ============================================= GPIO CFGE GPIO34INTD [11..11] ============================================= */ +typedef enum { /*!< GPIO_CFGE_GPIO34INTD */ + GPIO_CFGE_GPIO34INTD_nCELOW = 0, /*!< nCELOW : FNCSEL = 0x1 - nCE polarity active low value. */ + GPIO_CFGE_GPIO34INTD_nCEHIGH = 1, /*!< nCEHIGH : FNCSEL = 0x1 - nCE polarity active high value. */ +} GPIO_CFGE_GPIO34INTD_Enum; + +/* ============================================ GPIO CFGE GPIO34OUTCFG [9..10] ============================================= */ +typedef enum { /*!< GPIO_CFGE_GPIO34OUTCFG */ + GPIO_CFGE_GPIO34OUTCFG_DIS = 0, /*!< DIS : FNCSEL = 0x3 - Output disabled value. */ + GPIO_CFGE_GPIO34OUTCFG_PUSHPULL = 1, /*!< PUSHPULL : FNCSEL = 0x3 - Output is push-pull value. */ + GPIO_CFGE_GPIO34OUTCFG_OD = 2, /*!< OD : FNCSEL = 0x3 - Output is open drain value. */ + GPIO_CFGE_GPIO34OUTCFG_TS = 3, /*!< TS : FNCSEL = 0x3 - Output is tri-state value. */ +} GPIO_CFGE_GPIO34OUTCFG_Enum; + +/* ============================================= GPIO CFGE GPIO34INCFG [8..8] ============================================== */ +typedef enum { /*!< GPIO_CFGE_GPIO34INCFG */ + GPIO_CFGE_GPIO34INCFG_READ = 0, /*!< READ : Read the GPIO pin data value. */ + GPIO_CFGE_GPIO34INCFG_RDZERO = 1, /*!< RDZERO : INTD = 0 - Readback will always be zero value. */ +} GPIO_CFGE_GPIO34INCFG_Enum; + +/* ============================================== GPIO CFGE GPIO33INTD [7..7] ============================================== */ +typedef enum { /*!< GPIO_CFGE_GPIO33INTD */ + GPIO_CFGE_GPIO33INTD_nCELOW = 0, /*!< nCELOW : FNCSEL = 0x1 - nCE polarity active low value. */ + GPIO_CFGE_GPIO33INTD_nCEHIGH = 1, /*!< nCEHIGH : FNCSEL = 0x1 - nCE polarity active high value. */ +} GPIO_CFGE_GPIO33INTD_Enum; + +/* ============================================= GPIO CFGE GPIO33OUTCFG [5..6] ============================================= */ +typedef enum { /*!< GPIO_CFGE_GPIO33OUTCFG */ + GPIO_CFGE_GPIO33OUTCFG_DIS = 0, /*!< DIS : FNCSEL = 0x3 - Output disabled value. */ + GPIO_CFGE_GPIO33OUTCFG_PUSHPULL = 1, /*!< PUSHPULL : FNCSEL = 0x3 - Output is push-pull value. */ + GPIO_CFGE_GPIO33OUTCFG_OD = 2, /*!< OD : FNCSEL = 0x3 - Output is open drain value. */ + GPIO_CFGE_GPIO33OUTCFG_TS = 3, /*!< TS : FNCSEL = 0x3 - Output is tri-state value. */ +} GPIO_CFGE_GPIO33OUTCFG_Enum; + +/* ============================================= GPIO CFGE GPIO33INCFG [4..4] ============================================== */ +typedef enum { /*!< GPIO_CFGE_GPIO33INCFG */ + GPIO_CFGE_GPIO33INCFG_READ = 0, /*!< READ : Read the GPIO pin data value. */ + GPIO_CFGE_GPIO33INCFG_RDZERO = 1, /*!< RDZERO : INTD = 0 - Readback will always be zero value. */ +} GPIO_CFGE_GPIO33INCFG_Enum; + +/* ============================================== GPIO CFGE GPIO32INTD [3..3] ============================================== */ +typedef enum { /*!< GPIO_CFGE_GPIO32INTD */ + GPIO_CFGE_GPIO32INTD_nCELOW = 0, /*!< nCELOW : FNCSEL = 0x1 - nCE polarity active low value. */ + GPIO_CFGE_GPIO32INTD_nCEHIGH = 1, /*!< nCEHIGH : FNCSEL = 0x1 - nCE polarity active high value. */ +} GPIO_CFGE_GPIO32INTD_Enum; + +/* ============================================= GPIO CFGE GPIO32OUTCFG [1..2] ============================================= */ +typedef enum { /*!< GPIO_CFGE_GPIO32OUTCFG */ + GPIO_CFGE_GPIO32OUTCFG_DIS = 0, /*!< DIS : FNCSEL = 0x3 - Output disabled value. */ + GPIO_CFGE_GPIO32OUTCFG_PUSHPULL = 1, /*!< PUSHPULL : FNCSEL = 0x3 - Output is push-pull value. */ + GPIO_CFGE_GPIO32OUTCFG_OD = 2, /*!< OD : FNCSEL = 0x3 - Output is open drain value. */ + GPIO_CFGE_GPIO32OUTCFG_TS = 3, /*!< TS : FNCSEL = 0x3 - Output is tri-state value. */ +} GPIO_CFGE_GPIO32OUTCFG_Enum; + +/* ============================================= GPIO CFGE GPIO32INCFG [0..0] ============================================== */ +typedef enum { /*!< GPIO_CFGE_GPIO32INCFG */ + GPIO_CFGE_GPIO32INCFG_READ = 0, /*!< READ : Read the GPIO pin data value. */ + GPIO_CFGE_GPIO32INCFG_RDZERO = 1, /*!< RDZERO : INTD = 0 - Readback will always be zero value. */ +} GPIO_CFGE_GPIO32INCFG_Enum; + +/* ========================================================= CFGF ========================================================== */ +/* ============================================= GPIO CFGF GPIO47INTD [31..31] ============================================= */ +typedef enum { /*!< GPIO_CFGF_GPIO47INTD */ + GPIO_CFGF_GPIO47INTD_nCELOW = 0, /*!< nCELOW : FNCSEL = 0x1 - nCE polarity active low value. */ + GPIO_CFGF_GPIO47INTD_nCEHIGH = 1, /*!< nCEHIGH : FNCSEL = 0x1 - nCE polarity active high value. */ +} GPIO_CFGF_GPIO47INTD_Enum; + +/* ============================================ GPIO CFGF GPIO47OUTCFG [29..30] ============================================ */ +typedef enum { /*!< GPIO_CFGF_GPIO47OUTCFG */ + GPIO_CFGF_GPIO47OUTCFG_DIS = 0, /*!< DIS : FNCSEL = 0x3 - Output disabled value. */ + GPIO_CFGF_GPIO47OUTCFG_PUSHPULL = 1, /*!< PUSHPULL : FNCSEL = 0x3 - Output is push-pull value. */ + GPIO_CFGF_GPIO47OUTCFG_OD = 2, /*!< OD : FNCSEL = 0x3 - Output is open drain value. */ + GPIO_CFGF_GPIO47OUTCFG_TS = 3, /*!< TS : FNCSEL = 0x3 - Output is tri-state value. */ +} GPIO_CFGF_GPIO47OUTCFG_Enum; + +/* ============================================ GPIO CFGF GPIO47INCFG [28..28] ============================================= */ +typedef enum { /*!< GPIO_CFGF_GPIO47INCFG */ + GPIO_CFGF_GPIO47INCFG_READ = 0, /*!< READ : Read the GPIO pin data value. */ + GPIO_CFGF_GPIO47INCFG_RDZERO = 1, /*!< RDZERO : INTD = 0 - Readback will always be zero value. */ +} GPIO_CFGF_GPIO47INCFG_Enum; + +/* ============================================= GPIO CFGF GPIO46INTD [27..27] ============================================= */ +typedef enum { /*!< GPIO_CFGF_GPIO46INTD */ + GPIO_CFGF_GPIO46INTD_nCELOW = 0, /*!< nCELOW : FNCSEL = 0x1 - nCE polarity active low value. */ + GPIO_CFGF_GPIO46INTD_nCEHIGH = 1, /*!< nCEHIGH : FNCSEL = 0x1 - nCE polarity active high value. */ +} GPIO_CFGF_GPIO46INTD_Enum; + +/* ============================================ GPIO CFGF GPIO46OUTCFG [25..26] ============================================ */ +typedef enum { /*!< GPIO_CFGF_GPIO46OUTCFG */ + GPIO_CFGF_GPIO46OUTCFG_DIS = 0, /*!< DIS : FNCSEL = 0x3 - Output disabled value. */ + GPIO_CFGF_GPIO46OUTCFG_PUSHPULL = 1, /*!< PUSHPULL : FNCSEL = 0x3 - Output is push-pull value. */ + GPIO_CFGF_GPIO46OUTCFG_OD = 2, /*!< OD : FNCSEL = 0x3 - Output is open drain value. */ + GPIO_CFGF_GPIO46OUTCFG_TS = 3, /*!< TS : FNCSEL = 0x3 - Output is tri-state value. */ +} GPIO_CFGF_GPIO46OUTCFG_Enum; + +/* ============================================ GPIO CFGF GPIO46INCFG [24..24] ============================================= */ +typedef enum { /*!< GPIO_CFGF_GPIO46INCFG */ + GPIO_CFGF_GPIO46INCFG_READ = 0, /*!< READ : Read the GPIO pin data value. */ + GPIO_CFGF_GPIO46INCFG_RDZERO = 1, /*!< RDZERO : INTD = 0 - Readback will always be zero value. */ +} GPIO_CFGF_GPIO46INCFG_Enum; + +/* ============================================= GPIO CFGF GPIO45INTD [23..23] ============================================= */ +typedef enum { /*!< GPIO_CFGF_GPIO45INTD */ + GPIO_CFGF_GPIO45INTD_nCELOW = 0, /*!< nCELOW : FNCSEL = 0x1 - nCE polarity active low value. */ + GPIO_CFGF_GPIO45INTD_nCEHIGH = 1, /*!< nCEHIGH : FNCSEL = 0x1 - nCE polarity active high value. */ +} GPIO_CFGF_GPIO45INTD_Enum; + +/* ============================================ GPIO CFGF GPIO45OUTCFG [21..22] ============================================ */ +typedef enum { /*!< GPIO_CFGF_GPIO45OUTCFG */ + GPIO_CFGF_GPIO45OUTCFG_DIS = 0, /*!< DIS : FNCSEL = 0x3 - Output disabled value. */ + GPIO_CFGF_GPIO45OUTCFG_PUSHPULL = 1, /*!< PUSHPULL : FNCSEL = 0x3 - Output is push-pull value. */ + GPIO_CFGF_GPIO45OUTCFG_OD = 2, /*!< OD : FNCSEL = 0x3 - Output is open drain value. */ + GPIO_CFGF_GPIO45OUTCFG_TS = 3, /*!< TS : FNCSEL = 0x3 - Output is tri-state value. */ +} GPIO_CFGF_GPIO45OUTCFG_Enum; + +/* ============================================ GPIO CFGF GPIO45INCFG [20..20] ============================================= */ +typedef enum { /*!< GPIO_CFGF_GPIO45INCFG */ + GPIO_CFGF_GPIO45INCFG_READ = 0, /*!< READ : Read the GPIO pin data value. */ + GPIO_CFGF_GPIO45INCFG_RDZERO = 1, /*!< RDZERO : INTD = 0 - Readback will always be zero value. */ +} GPIO_CFGF_GPIO45INCFG_Enum; + +/* ============================================= GPIO CFGF GPIO44INTD [19..19] ============================================= */ +typedef enum { /*!< GPIO_CFGF_GPIO44INTD */ + GPIO_CFGF_GPIO44INTD_nCELOW = 0, /*!< nCELOW : FNCSEL = 0x1 - nCE polarity active low value. */ + GPIO_CFGF_GPIO44INTD_nCEHIGH = 1, /*!< nCEHIGH : FNCSEL = 0x1 - nCE polarity active high value. */ +} GPIO_CFGF_GPIO44INTD_Enum; + +/* ============================================ GPIO CFGF GPIO44OUTCFG [17..18] ============================================ */ +typedef enum { /*!< GPIO_CFGF_GPIO44OUTCFG */ + GPIO_CFGF_GPIO44OUTCFG_DIS = 0, /*!< DIS : FNCSEL = 0x3 - Output disabled value. */ + GPIO_CFGF_GPIO44OUTCFG_PUSHPULL = 1, /*!< PUSHPULL : FNCSEL = 0x3 - Output is push-pull value. */ + GPIO_CFGF_GPIO44OUTCFG_OD = 2, /*!< OD : FNCSEL = 0x3 - Output is open drain value. */ + GPIO_CFGF_GPIO44OUTCFG_TS = 3, /*!< TS : FNCSEL = 0x3 - Output is tri-state value. */ +} GPIO_CFGF_GPIO44OUTCFG_Enum; + +/* ============================================ GPIO CFGF GPIO44INCFG [16..16] ============================================= */ +typedef enum { /*!< GPIO_CFGF_GPIO44INCFG */ + GPIO_CFGF_GPIO44INCFG_READ = 0, /*!< READ : Read the GPIO pin data value. */ + GPIO_CFGF_GPIO44INCFG_RDZERO = 1, /*!< RDZERO : INTD = 0 - Readback will always be zero value. */ +} GPIO_CFGF_GPIO44INCFG_Enum; + +/* ============================================= GPIO CFGF GPIO43INTD [15..15] ============================================= */ +typedef enum { /*!< GPIO_CFGF_GPIO43INTD */ + GPIO_CFGF_GPIO43INTD_nCELOW = 0, /*!< nCELOW : FNCSEL = 0x1 - nCE polarity active low value. */ + GPIO_CFGF_GPIO43INTD_nCEHIGH = 1, /*!< nCEHIGH : FNCSEL = 0x1 - nCE polarity active high value. */ +} GPIO_CFGF_GPIO43INTD_Enum; + +/* ============================================ GPIO CFGF GPIO43OUTCFG [13..14] ============================================ */ +typedef enum { /*!< GPIO_CFGF_GPIO43OUTCFG */ + GPIO_CFGF_GPIO43OUTCFG_DIS = 0, /*!< DIS : FNCSEL = 0x3 - Output disabled value. */ + GPIO_CFGF_GPIO43OUTCFG_PUSHPULL = 1, /*!< PUSHPULL : FNCSEL = 0x3 - Output is push-pull value. */ + GPIO_CFGF_GPIO43OUTCFG_OD = 2, /*!< OD : FNCSEL = 0x3 - Output is open drain value. */ + GPIO_CFGF_GPIO43OUTCFG_TS = 3, /*!< TS : FNCSEL = 0x3 - Output is tri-state value. */ +} GPIO_CFGF_GPIO43OUTCFG_Enum; + +/* ============================================ GPIO CFGF GPIO43INCFG [12..12] ============================================= */ +typedef enum { /*!< GPIO_CFGF_GPIO43INCFG */ + GPIO_CFGF_GPIO43INCFG_READ = 0, /*!< READ : Read the GPIO pin data value. */ + GPIO_CFGF_GPIO43INCFG_RDZERO = 1, /*!< RDZERO : INTD = 0 - Readback will always be zero value. */ +} GPIO_CFGF_GPIO43INCFG_Enum; + +/* ============================================= GPIO CFGF GPIO42INTD [11..11] ============================================= */ +typedef enum { /*!< GPIO_CFGF_GPIO42INTD */ + GPIO_CFGF_GPIO42INTD_nCELOW = 0, /*!< nCELOW : FNCSEL = 0x1 - nCE polarity active low value. */ + GPIO_CFGF_GPIO42INTD_nCEHIGH = 1, /*!< nCEHIGH : FNCSEL = 0x1 - nCE polarity active high value. */ +} GPIO_CFGF_GPIO42INTD_Enum; + +/* ============================================ GPIO CFGF GPIO42OUTCFG [9..10] ============================================= */ +typedef enum { /*!< GPIO_CFGF_GPIO42OUTCFG */ + GPIO_CFGF_GPIO42OUTCFG_DIS = 0, /*!< DIS : FNCSEL = 0x3 - Output disabled value. */ + GPIO_CFGF_GPIO42OUTCFG_PUSHPULL = 1, /*!< PUSHPULL : FNCSEL = 0x3 - Output is push-pull value. */ + GPIO_CFGF_GPIO42OUTCFG_OD = 2, /*!< OD : FNCSEL = 0x3 - Output is open drain value. */ + GPIO_CFGF_GPIO42OUTCFG_TS = 3, /*!< TS : FNCSEL = 0x3 - Output is tri-state value. */ +} GPIO_CFGF_GPIO42OUTCFG_Enum; + +/* ============================================= GPIO CFGF GPIO42INCFG [8..8] ============================================== */ +typedef enum { /*!< GPIO_CFGF_GPIO42INCFG */ + GPIO_CFGF_GPIO42INCFG_READ = 0, /*!< READ : Read the GPIO pin data value. */ + GPIO_CFGF_GPIO42INCFG_RDZERO = 1, /*!< RDZERO : INTD = 0 - Readback will always be zero value. */ +} GPIO_CFGF_GPIO42INCFG_Enum; + +/* ============================================== GPIO CFGF GPIO41INTD [7..7] ============================================== */ +typedef enum { /*!< GPIO_CFGF_GPIO41INTD */ + GPIO_CFGF_GPIO41INTD_nCELOW = 0, /*!< nCELOW : FNCSEL = 0x0 - nCE polarity active low value. */ + GPIO_CFGF_GPIO41INTD_nCEHIGH = 1, /*!< nCEHIGH : FNCSEL = 0x0 - nCE polarity active high value. */ +} GPIO_CFGF_GPIO41INTD_Enum; + +/* ============================================= GPIO CFGF GPIO41OUTCFG [5..6] ============================================= */ +typedef enum { /*!< GPIO_CFGF_GPIO41OUTCFG */ + GPIO_CFGF_GPIO41OUTCFG_DIS = 0, /*!< DIS : FNCSEL = 0x3 - Output disabled value. */ + GPIO_CFGF_GPIO41OUTCFG_PUSHPULL = 1, /*!< PUSHPULL : FNCSEL = 0x3 - Output is push-pull value. */ + GPIO_CFGF_GPIO41OUTCFG_OD = 2, /*!< OD : FNCSEL = 0x3 - Output is open drain value. */ + GPIO_CFGF_GPIO41OUTCFG_TS = 3, /*!< TS : FNCSEL = 0x3 - Output is tri-state value. */ +} GPIO_CFGF_GPIO41OUTCFG_Enum; + +/* ============================================= GPIO CFGF GPIO41INCFG [4..4] ============================================== */ +typedef enum { /*!< GPIO_CFGF_GPIO41INCFG */ + GPIO_CFGF_GPIO41INCFG_READ = 0, /*!< READ : Read the GPIO pin data value. */ + GPIO_CFGF_GPIO41INCFG_RDZERO = 1, /*!< RDZERO : INTD = 0 - Readback will always be zero value. */ +} GPIO_CFGF_GPIO41INCFG_Enum; + +/* ============================================== GPIO CFGF GPIO40INTD [3..3] ============================================== */ +typedef enum { /*!< GPIO_CFGF_GPIO40INTD */ + GPIO_CFGF_GPIO40INTD_INTDIS = 0, /*!< INTDIS : INCFG = 1 - No interrupt on GPIO transition value. */ + GPIO_CFGF_GPIO40INTD_INTBOTH = 1, /*!< INTBOTH : INCFG = 1 - Interrupt on either low to high or high + to low GPIO transition value. */ +} GPIO_CFGF_GPIO40INTD_Enum; + +/* ============================================= GPIO CFGF GPIO40OUTCFG [1..2] ============================================= */ +typedef enum { /*!< GPIO_CFGF_GPIO40OUTCFG */ + GPIO_CFGF_GPIO40OUTCFG_DIS = 0, /*!< DIS : FNCSEL = 0x3 - Output disabled value. */ + GPIO_CFGF_GPIO40OUTCFG_PUSHPULL = 1, /*!< PUSHPULL : FNCSEL = 0x3 - Output is push-pull value. */ + GPIO_CFGF_GPIO40OUTCFG_OD = 2, /*!< OD : FNCSEL = 0x3 - Output is open drain value. */ + GPIO_CFGF_GPIO40OUTCFG_TS = 3, /*!< TS : FNCSEL = 0x3 - Output is tri-state value. */ +} GPIO_CFGF_GPIO40OUTCFG_Enum; + +/* ============================================= GPIO CFGF GPIO40INCFG [0..0] ============================================== */ +typedef enum { /*!< GPIO_CFGF_GPIO40INCFG */ + GPIO_CFGF_GPIO40INCFG_READ = 0, /*!< READ : Read the GPIO pin data value. */ + GPIO_CFGF_GPIO40INCFG_RDZERO = 1, /*!< RDZERO : INTD = 0 - Readback will always be zero value. */ +} GPIO_CFGF_GPIO40INCFG_Enum; + +/* ========================================================= CFGG ========================================================== */ +/* ============================================== GPIO CFGG GPIO49INTD [7..7] ============================================== */ +typedef enum { /*!< GPIO_CFGG_GPIO49INTD */ + GPIO_CFGG_GPIO49INTD_nCELOW = 0, /*!< nCELOW : FNCSEL = 0x1 - nCE polarity active low value. */ + GPIO_CFGG_GPIO49INTD_nCEHIGH = 1, /*!< nCEHIGH : FNCSEL = 0x1 - nCE polarity active high value. */ +} GPIO_CFGG_GPIO49INTD_Enum; + +/* ============================================= GPIO CFGG GPIO49OUTCFG [5..6] ============================================= */ +typedef enum { /*!< GPIO_CFGG_GPIO49OUTCFG */ + GPIO_CFGG_GPIO49OUTCFG_DIS = 0, /*!< DIS : FNCSEL = 0x3 - Output disabled value. */ + GPIO_CFGG_GPIO49OUTCFG_PUSHPULL = 1, /*!< PUSHPULL : FNCSEL = 0x3 - Output is push-pull value. */ + GPIO_CFGG_GPIO49OUTCFG_OD = 2, /*!< OD : FNCSEL = 0x3 - Output is open drain value. */ + GPIO_CFGG_GPIO49OUTCFG_TS = 3, /*!< TS : FNCSEL = 0x3 - Output is tri-state value. */ +} GPIO_CFGG_GPIO49OUTCFG_Enum; + +/* ============================================= GPIO CFGG GPIO49INCFG [4..4] ============================================== */ +typedef enum { /*!< GPIO_CFGG_GPIO49INCFG */ + GPIO_CFGG_GPIO49INCFG_READ = 0, /*!< READ : Read the GPIO pin data value. */ + GPIO_CFGG_GPIO49INCFG_RDZERO = 1, /*!< RDZERO : INTD = 0 - Readback will always be zero value. */ +} GPIO_CFGG_GPIO49INCFG_Enum; + +/* ============================================== GPIO CFGG GPIO48INTD [3..3] ============================================== */ +typedef enum { /*!< GPIO_CFGG_GPIO48INTD */ + GPIO_CFGG_GPIO48INTD_nCELOW = 0, /*!< nCELOW : FNCSEL = 0x1 - nCE polarity active low value. */ + GPIO_CFGG_GPIO48INTD_nCEHIGH = 1, /*!< nCEHIGH : FNCSEL = 0x1 - nCE polarity active high value. */ +} GPIO_CFGG_GPIO48INTD_Enum; + +/* ============================================= GPIO CFGG GPIO48OUTCFG [1..2] ============================================= */ +typedef enum { /*!< GPIO_CFGG_GPIO48OUTCFG */ + GPIO_CFGG_GPIO48OUTCFG_DIS = 0, /*!< DIS : FNCSEL = 0x3 - Output disabled value. */ + GPIO_CFGG_GPIO48OUTCFG_PUSHPULL = 1, /*!< PUSHPULL : FNCSEL = 0x3 - Output is push-pull value. */ + GPIO_CFGG_GPIO48OUTCFG_OD = 2, /*!< OD : FNCSEL = 0x3 - Output is open drain value. */ + GPIO_CFGG_GPIO48OUTCFG_TS = 3, /*!< TS : FNCSEL = 0x3 - Output is tri-state value. */ +} GPIO_CFGG_GPIO48OUTCFG_Enum; + +/* ============================================= GPIO CFGG GPIO48INCFG [0..0] ============================================== */ +typedef enum { /*!< GPIO_CFGG_GPIO48INCFG */ + GPIO_CFGG_GPIO48INCFG_READ = 0, /*!< READ : Read the GPIO pin data value. */ + GPIO_CFGG_GPIO48INCFG_RDZERO = 1, /*!< RDZERO : INTD = 0 - Readback will always be zero value. */ +} GPIO_CFGG_GPIO48INCFG_Enum; + +/* ======================================================== PADKEY ========================================================= */ +/* ============================================== GPIO PADKEY PADKEY [0..31] =============================================== */ +typedef enum { /*!< GPIO_PADKEY_PADKEY */ + GPIO_PADKEY_PADKEY_Key = 115, /*!< Key : Key value. */ +} GPIO_PADKEY_PADKEY_Enum; + +/* ========================================================== RDA ========================================================== */ +/* ========================================================== RDB ========================================================== */ +/* ========================================================== WTA ========================================================== */ +/* ========================================================== WTB ========================================================== */ +/* ========================================================= WTSA ========================================================== */ +/* ========================================================= WTSB ========================================================== */ +/* ========================================================= WTCA ========================================================== */ +/* ========================================================= WTCB ========================================================== */ +/* ========================================================== ENA ========================================================== */ +/* ========================================================== ENB ========================================================== */ +/* ========================================================= ENSA ========================================================== */ +/* ========================================================= ENSB ========================================================== */ +/* ========================================================= ENCA ========================================================== */ +/* ========================================================= ENCB ========================================================== */ +/* ======================================================== STMRCAP ======================================================== */ +/* ============================================= GPIO STMRCAP STPOL3 [30..30] ============================================== */ +typedef enum { /*!< GPIO_STMRCAP_STPOL3 */ + GPIO_STMRCAP_STPOL3_CAPLH = 0, /*!< CAPLH : Capture on low to high GPIO transition value. */ + GPIO_STMRCAP_STPOL3_CAPHL = 1, /*!< CAPHL : Capture on high to low GPIO transition value. */ +} GPIO_STMRCAP_STPOL3_Enum; + +/* ============================================= GPIO STMRCAP STPOL2 [22..22] ============================================== */ +typedef enum { /*!< GPIO_STMRCAP_STPOL2 */ + GPIO_STMRCAP_STPOL2_CAPLH = 0, /*!< CAPLH : Capture on low to high GPIO transition value. */ + GPIO_STMRCAP_STPOL2_CAPHL = 1, /*!< CAPHL : Capture on high to low GPIO transition value. */ +} GPIO_STMRCAP_STPOL2_Enum; + +/* ============================================= GPIO STMRCAP STPOL1 [14..14] ============================================== */ +typedef enum { /*!< GPIO_STMRCAP_STPOL1 */ + GPIO_STMRCAP_STPOL1_CAPLH = 0, /*!< CAPLH : Capture on low to high GPIO transition value. */ + GPIO_STMRCAP_STPOL1_CAPHL = 1, /*!< CAPHL : Capture on high to low GPIO transition value. */ +} GPIO_STMRCAP_STPOL1_Enum; + +/* ============================================== GPIO STMRCAP STPOL0 [6..6] =============================================== */ +typedef enum { /*!< GPIO_STMRCAP_STPOL0 */ + GPIO_STMRCAP_STPOL0_CAPLH = 0, /*!< CAPLH : Capture on low to high GPIO transition value. */ + GPIO_STMRCAP_STPOL0_CAPHL = 1, /*!< CAPHL : Capture on high to low GPIO transition value. */ +} GPIO_STMRCAP_STPOL0_Enum; + +/* ======================================================== IOM0IRQ ======================================================== */ +/* ======================================================== IOM1IRQ ======================================================== */ +/* ======================================================== IOM2IRQ ======================================================== */ +/* ======================================================== IOM3IRQ ======================================================== */ +/* ======================================================== IOM4IRQ ======================================================== */ +/* ======================================================== IOM5IRQ ======================================================== */ +/* ======================================================= BLEIFIRQ ======================================================== */ +/* ======================================================== GPIOOBS ======================================================== */ +/* ====================================================== ALTPADCFGA ======================================================= */ +/* =========================================== GPIO ALTPADCFGA PAD3_SR [28..28] ============================================ */ +typedef enum { /*!< GPIO_ALTPADCFGA_PAD3_SR */ + GPIO_ALTPADCFGA_PAD3_SR_SR_EN = 1, /*!< SR_EN : Enables Slew rate control on pad value. */ +} GPIO_ALTPADCFGA_PAD3_SR_Enum; + +/* =========================================== GPIO ALTPADCFGA PAD2_SR [20..20] ============================================ */ +typedef enum { /*!< GPIO_ALTPADCFGA_PAD2_SR */ + GPIO_ALTPADCFGA_PAD2_SR_SR_EN = 1, /*!< SR_EN : Enables Slew rate control on pad value. */ +} GPIO_ALTPADCFGA_PAD2_SR_Enum; + +/* =========================================== GPIO ALTPADCFGA PAD1_SR [12..12] ============================================ */ +typedef enum { /*!< GPIO_ALTPADCFGA_PAD1_SR */ + GPIO_ALTPADCFGA_PAD1_SR_SR_EN = 1, /*!< SR_EN : Enables Slew rate control on pad value. */ +} GPIO_ALTPADCFGA_PAD1_SR_Enum; + +/* ============================================ GPIO ALTPADCFGA PAD0_SR [4..4] ============================================= */ +typedef enum { /*!< GPIO_ALTPADCFGA_PAD0_SR */ + GPIO_ALTPADCFGA_PAD0_SR_SR_EN = 1, /*!< SR_EN : Enables Slew rate control on pad value. */ +} GPIO_ALTPADCFGA_PAD0_SR_Enum; + +/* ====================================================== ALTPADCFGB ======================================================= */ +/* =========================================== GPIO ALTPADCFGB PAD7_SR [28..28] ============================================ */ +typedef enum { /*!< GPIO_ALTPADCFGB_PAD7_SR */ + GPIO_ALTPADCFGB_PAD7_SR_SR_EN = 1, /*!< SR_EN : Enables Slew rate control on pad value. */ +} GPIO_ALTPADCFGB_PAD7_SR_Enum; + +/* =========================================== GPIO ALTPADCFGB PAD6_SR [20..20] ============================================ */ +typedef enum { /*!< GPIO_ALTPADCFGB_PAD6_SR */ + GPIO_ALTPADCFGB_PAD6_SR_SR_EN = 1, /*!< SR_EN : Enables Slew rate control on pad value. */ +} GPIO_ALTPADCFGB_PAD6_SR_Enum; + +/* =========================================== GPIO ALTPADCFGB PAD5_SR [12..12] ============================================ */ +typedef enum { /*!< GPIO_ALTPADCFGB_PAD5_SR */ + GPIO_ALTPADCFGB_PAD5_SR_SR_EN = 1, /*!< SR_EN : Enables Slew rate control on pad value. */ +} GPIO_ALTPADCFGB_PAD5_SR_Enum; + +/* ============================================ GPIO ALTPADCFGB PAD4_SR [4..4] ============================================= */ +typedef enum { /*!< GPIO_ALTPADCFGB_PAD4_SR */ + GPIO_ALTPADCFGB_PAD4_SR_SR_EN = 1, /*!< SR_EN : Enables Slew rate control on pad value. */ +} GPIO_ALTPADCFGB_PAD4_SR_Enum; + +/* ====================================================== ALTPADCFGC ======================================================= */ +/* =========================================== GPIO ALTPADCFGC PAD11_SR [28..28] =========================================== */ +typedef enum { /*!< GPIO_ALTPADCFGC_PAD11_SR */ + GPIO_ALTPADCFGC_PAD11_SR_SR_EN = 1, /*!< SR_EN : Enables Slew rate control on pad value. */ +} GPIO_ALTPADCFGC_PAD11_SR_Enum; + +/* =========================================== GPIO ALTPADCFGC PAD10_SR [20..20] =========================================== */ +typedef enum { /*!< GPIO_ALTPADCFGC_PAD10_SR */ + GPIO_ALTPADCFGC_PAD10_SR_SR_EN = 1, /*!< SR_EN : Enables Slew rate control on pad value. */ +} GPIO_ALTPADCFGC_PAD10_SR_Enum; + +/* =========================================== GPIO ALTPADCFGC PAD9_SR [12..12] ============================================ */ +typedef enum { /*!< GPIO_ALTPADCFGC_PAD9_SR */ + GPIO_ALTPADCFGC_PAD9_SR_SR_EN = 1, /*!< SR_EN : Enables Slew rate control on pad value. */ +} GPIO_ALTPADCFGC_PAD9_SR_Enum; + +/* ============================================ GPIO ALTPADCFGC PAD8_SR [4..4] ============================================= */ +typedef enum { /*!< GPIO_ALTPADCFGC_PAD8_SR */ + GPIO_ALTPADCFGC_PAD8_SR_SR_EN = 1, /*!< SR_EN : Enables Slew rate control on pad value. */ +} GPIO_ALTPADCFGC_PAD8_SR_Enum; + +/* ====================================================== ALTPADCFGD ======================================================= */ +/* =========================================== GPIO ALTPADCFGD PAD15_SR [28..28] =========================================== */ +typedef enum { /*!< GPIO_ALTPADCFGD_PAD15_SR */ + GPIO_ALTPADCFGD_PAD15_SR_SR_EN = 1, /*!< SR_EN : Enables Slew rate control on pad value. */ +} GPIO_ALTPADCFGD_PAD15_SR_Enum; + +/* =========================================== GPIO ALTPADCFGD PAD14_SR [20..20] =========================================== */ +typedef enum { /*!< GPIO_ALTPADCFGD_PAD14_SR */ + GPIO_ALTPADCFGD_PAD14_SR_SR_EN = 1, /*!< SR_EN : Enables Slew rate control on pad value. */ +} GPIO_ALTPADCFGD_PAD14_SR_Enum; + +/* =========================================== GPIO ALTPADCFGD PAD13_SR [12..12] =========================================== */ +typedef enum { /*!< GPIO_ALTPADCFGD_PAD13_SR */ + GPIO_ALTPADCFGD_PAD13_SR_SR_EN = 1, /*!< SR_EN : Enables Slew rate control on pad value. */ +} GPIO_ALTPADCFGD_PAD13_SR_Enum; + +/* ============================================ GPIO ALTPADCFGD PAD12_SR [4..4] ============================================ */ +typedef enum { /*!< GPIO_ALTPADCFGD_PAD12_SR */ + GPIO_ALTPADCFGD_PAD12_SR_SR_EN = 1, /*!< SR_EN : Enables Slew rate control on pad value. */ +} GPIO_ALTPADCFGD_PAD12_SR_Enum; + +/* ====================================================== ALTPADCFGE ======================================================= */ +/* =========================================== GPIO ALTPADCFGE PAD19_SR [28..28] =========================================== */ +typedef enum { /*!< GPIO_ALTPADCFGE_PAD19_SR */ + GPIO_ALTPADCFGE_PAD19_SR_SR_EN = 1, /*!< SR_EN : Enables Slew rate control on pad value. */ +} GPIO_ALTPADCFGE_PAD19_SR_Enum; + +/* =========================================== GPIO ALTPADCFGE PAD18_SR [20..20] =========================================== */ +typedef enum { /*!< GPIO_ALTPADCFGE_PAD18_SR */ + GPIO_ALTPADCFGE_PAD18_SR_SR_EN = 1, /*!< SR_EN : Enables Slew rate control on pad value. */ +} GPIO_ALTPADCFGE_PAD18_SR_Enum; + +/* =========================================== GPIO ALTPADCFGE PAD17_SR [12..12] =========================================== */ +typedef enum { /*!< GPIO_ALTPADCFGE_PAD17_SR */ + GPIO_ALTPADCFGE_PAD17_SR_SR_EN = 1, /*!< SR_EN : Enables Slew rate control on pad value. */ +} GPIO_ALTPADCFGE_PAD17_SR_Enum; + +/* ============================================ GPIO ALTPADCFGE PAD16_SR [4..4] ============================================ */ +typedef enum { /*!< GPIO_ALTPADCFGE_PAD16_SR */ + GPIO_ALTPADCFGE_PAD16_SR_SR_EN = 1, /*!< SR_EN : Enables Slew rate control on pad value. */ +} GPIO_ALTPADCFGE_PAD16_SR_Enum; + +/* ====================================================== ALTPADCFGF ======================================================= */ +/* =========================================== GPIO ALTPADCFGF PAD23_SR [28..28] =========================================== */ +typedef enum { /*!< GPIO_ALTPADCFGF_PAD23_SR */ + GPIO_ALTPADCFGF_PAD23_SR_SR_EN = 1, /*!< SR_EN : Enables Slew rate control on pad value. */ +} GPIO_ALTPADCFGF_PAD23_SR_Enum; + +/* =========================================== GPIO ALTPADCFGF PAD22_SR [20..20] =========================================== */ +typedef enum { /*!< GPIO_ALTPADCFGF_PAD22_SR */ + GPIO_ALTPADCFGF_PAD22_SR_SR_EN = 1, /*!< SR_EN : Enables Slew rate control on pad value. */ +} GPIO_ALTPADCFGF_PAD22_SR_Enum; + +/* =========================================== GPIO ALTPADCFGF PAD21_SR [12..12] =========================================== */ +typedef enum { /*!< GPIO_ALTPADCFGF_PAD21_SR */ + GPIO_ALTPADCFGF_PAD21_SR_SR_EN = 1, /*!< SR_EN : Enables Slew rate control on pad value. */ +} GPIO_ALTPADCFGF_PAD21_SR_Enum; + +/* ============================================ GPIO ALTPADCFGF PAD20_SR [4..4] ============================================ */ +typedef enum { /*!< GPIO_ALTPADCFGF_PAD20_SR */ + GPIO_ALTPADCFGF_PAD20_SR_SR_EN = 1, /*!< SR_EN : Enables Slew rate control on pad value. */ +} GPIO_ALTPADCFGF_PAD20_SR_Enum; + +/* ====================================================== ALTPADCFGG ======================================================= */ +/* =========================================== GPIO ALTPADCFGG PAD27_SR [28..28] =========================================== */ +typedef enum { /*!< GPIO_ALTPADCFGG_PAD27_SR */ + GPIO_ALTPADCFGG_PAD27_SR_SR_EN = 1, /*!< SR_EN : Enables Slew rate control on pad value. */ +} GPIO_ALTPADCFGG_PAD27_SR_Enum; + +/* =========================================== GPIO ALTPADCFGG PAD26_SR [20..20] =========================================== */ +typedef enum { /*!< GPIO_ALTPADCFGG_PAD26_SR */ + GPIO_ALTPADCFGG_PAD26_SR_SR_EN = 1, /*!< SR_EN : Enables Slew rate control on pad value. */ +} GPIO_ALTPADCFGG_PAD26_SR_Enum; + +/* =========================================== GPIO ALTPADCFGG PAD25_SR [12..12] =========================================== */ +typedef enum { /*!< GPIO_ALTPADCFGG_PAD25_SR */ + GPIO_ALTPADCFGG_PAD25_SR_SR_EN = 1, /*!< SR_EN : Enables Slew rate control on pad value. */ +} GPIO_ALTPADCFGG_PAD25_SR_Enum; + +/* ============================================ GPIO ALTPADCFGG PAD24_SR [4..4] ============================================ */ +typedef enum { /*!< GPIO_ALTPADCFGG_PAD24_SR */ + GPIO_ALTPADCFGG_PAD24_SR_SR_EN = 1, /*!< SR_EN : Enables Slew rate control on pad value. */ +} GPIO_ALTPADCFGG_PAD24_SR_Enum; + +/* ====================================================== ALTPADCFGH ======================================================= */ +/* =========================================== GPIO ALTPADCFGH PAD31_SR [28..28] =========================================== */ +typedef enum { /*!< GPIO_ALTPADCFGH_PAD31_SR */ + GPIO_ALTPADCFGH_PAD31_SR_SR_EN = 1, /*!< SR_EN : Enables Slew rate control on pad value. */ +} GPIO_ALTPADCFGH_PAD31_SR_Enum; + +/* =========================================== GPIO ALTPADCFGH PAD30_SR [20..20] =========================================== */ +typedef enum { /*!< GPIO_ALTPADCFGH_PAD30_SR */ + GPIO_ALTPADCFGH_PAD30_SR_SR_EN = 1, /*!< SR_EN : Enables Slew rate control on pad value. */ +} GPIO_ALTPADCFGH_PAD30_SR_Enum; + +/* =========================================== GPIO ALTPADCFGH PAD29_SR [12..12] =========================================== */ +typedef enum { /*!< GPIO_ALTPADCFGH_PAD29_SR */ + GPIO_ALTPADCFGH_PAD29_SR_SR_EN = 1, /*!< SR_EN : Enables Slew rate control on pad value. */ +} GPIO_ALTPADCFGH_PAD29_SR_Enum; + +/* ============================================ GPIO ALTPADCFGH PAD28_SR [4..4] ============================================ */ +typedef enum { /*!< GPIO_ALTPADCFGH_PAD28_SR */ + GPIO_ALTPADCFGH_PAD28_SR_SR_EN = 1, /*!< SR_EN : Enables Slew rate control on pad value. */ +} GPIO_ALTPADCFGH_PAD28_SR_Enum; + +/* ====================================================== ALTPADCFGI ======================================================= */ +/* =========================================== GPIO ALTPADCFGI PAD35_SR [28..28] =========================================== */ +typedef enum { /*!< GPIO_ALTPADCFGI_PAD35_SR */ + GPIO_ALTPADCFGI_PAD35_SR_SR_EN = 1, /*!< SR_EN : Enables Slew rate control on pad value. */ +} GPIO_ALTPADCFGI_PAD35_SR_Enum; + +/* =========================================== GPIO ALTPADCFGI PAD34_SR [20..20] =========================================== */ +typedef enum { /*!< GPIO_ALTPADCFGI_PAD34_SR */ + GPIO_ALTPADCFGI_PAD34_SR_SR_EN = 1, /*!< SR_EN : Enables Slew rate control on pad value. */ +} GPIO_ALTPADCFGI_PAD34_SR_Enum; + +/* =========================================== GPIO ALTPADCFGI PAD33_SR [12..12] =========================================== */ +typedef enum { /*!< GPIO_ALTPADCFGI_PAD33_SR */ + GPIO_ALTPADCFGI_PAD33_SR_SR_EN = 1, /*!< SR_EN : Enables Slew rate control on pad value. */ +} GPIO_ALTPADCFGI_PAD33_SR_Enum; + +/* ============================================ GPIO ALTPADCFGI PAD32_SR [4..4] ============================================ */ +typedef enum { /*!< GPIO_ALTPADCFGI_PAD32_SR */ + GPIO_ALTPADCFGI_PAD32_SR_SR_EN = 1, /*!< SR_EN : Enables Slew rate control on pad value. */ +} GPIO_ALTPADCFGI_PAD32_SR_Enum; + +/* ====================================================== ALTPADCFGJ ======================================================= */ +/* =========================================== GPIO ALTPADCFGJ PAD39_SR [28..28] =========================================== */ +typedef enum { /*!< GPIO_ALTPADCFGJ_PAD39_SR */ + GPIO_ALTPADCFGJ_PAD39_SR_SR_EN = 1, /*!< SR_EN : Enables Slew rate control on pad value. */ +} GPIO_ALTPADCFGJ_PAD39_SR_Enum; + +/* =========================================== GPIO ALTPADCFGJ PAD38_SR [20..20] =========================================== */ +typedef enum { /*!< GPIO_ALTPADCFGJ_PAD38_SR */ + GPIO_ALTPADCFGJ_PAD38_SR_SR_EN = 1, /*!< SR_EN : Enables Slew rate control on pad value. */ +} GPIO_ALTPADCFGJ_PAD38_SR_Enum; + +/* =========================================== GPIO ALTPADCFGJ PAD37_SR [12..12] =========================================== */ +typedef enum { /*!< GPIO_ALTPADCFGJ_PAD37_SR */ + GPIO_ALTPADCFGJ_PAD37_SR_SR_EN = 1, /*!< SR_EN : Enables Slew rate control on pad value. */ +} GPIO_ALTPADCFGJ_PAD37_SR_Enum; + +/* ============================================ GPIO ALTPADCFGJ PAD36_SR [4..4] ============================================ */ +typedef enum { /*!< GPIO_ALTPADCFGJ_PAD36_SR */ + GPIO_ALTPADCFGJ_PAD36_SR_SR_EN = 1, /*!< SR_EN : Enables Slew rate control on pad value. */ +} GPIO_ALTPADCFGJ_PAD36_SR_Enum; + +/* ====================================================== ALTPADCFGK ======================================================= */ +/* =========================================== GPIO ALTPADCFGK PAD43_SR [28..28] =========================================== */ +typedef enum { /*!< GPIO_ALTPADCFGK_PAD43_SR */ + GPIO_ALTPADCFGK_PAD43_SR_SR_EN = 1, /*!< SR_EN : Enables Slew rate control on pad value. */ +} GPIO_ALTPADCFGK_PAD43_SR_Enum; + +/* =========================================== GPIO ALTPADCFGK PAD42_SR [20..20] =========================================== */ +typedef enum { /*!< GPIO_ALTPADCFGK_PAD42_SR */ + GPIO_ALTPADCFGK_PAD42_SR_SR_EN = 1, /*!< SR_EN : Enables Slew rate control on pad value. */ +} GPIO_ALTPADCFGK_PAD42_SR_Enum; + +/* =========================================== GPIO ALTPADCFGK PAD41_SR [12..12] =========================================== */ +typedef enum { /*!< GPIO_ALTPADCFGK_PAD41_SR */ + GPIO_ALTPADCFGK_PAD41_SR_SR_EN = 1, /*!< SR_EN : Enables Slew rate control on pad value. */ +} GPIO_ALTPADCFGK_PAD41_SR_Enum; + +/* ============================================ GPIO ALTPADCFGK PAD40_SR [4..4] ============================================ */ +typedef enum { /*!< GPIO_ALTPADCFGK_PAD40_SR */ + GPIO_ALTPADCFGK_PAD40_SR_SR_EN = 1, /*!< SR_EN : Enables Slew rate control on pad value. */ +} GPIO_ALTPADCFGK_PAD40_SR_Enum; + +/* ====================================================== ALTPADCFGL ======================================================= */ +/* =========================================== GPIO ALTPADCFGL PAD47_SR [28..28] =========================================== */ +typedef enum { /*!< GPIO_ALTPADCFGL_PAD47_SR */ + GPIO_ALTPADCFGL_PAD47_SR_SR_EN = 1, /*!< SR_EN : Enables Slew rate control on pad value. */ +} GPIO_ALTPADCFGL_PAD47_SR_Enum; + +/* =========================================== GPIO ALTPADCFGL PAD46_SR [20..20] =========================================== */ +typedef enum { /*!< GPIO_ALTPADCFGL_PAD46_SR */ + GPIO_ALTPADCFGL_PAD46_SR_SR_EN = 1, /*!< SR_EN : Enables Slew rate control on pad value. */ +} GPIO_ALTPADCFGL_PAD46_SR_Enum; + +/* =========================================== GPIO ALTPADCFGL PAD45_SR [12..12] =========================================== */ +typedef enum { /*!< GPIO_ALTPADCFGL_PAD45_SR */ + GPIO_ALTPADCFGL_PAD45_SR_SR_EN = 1, /*!< SR_EN : Enables Slew rate control on pad value. */ +} GPIO_ALTPADCFGL_PAD45_SR_Enum; + +/* ============================================ GPIO ALTPADCFGL PAD44_SR [4..4] ============================================ */ +typedef enum { /*!< GPIO_ALTPADCFGL_PAD44_SR */ + GPIO_ALTPADCFGL_PAD44_SR_SR_EN = 1, /*!< SR_EN : Enables Slew rate control on pad value. */ +} GPIO_ALTPADCFGL_PAD44_SR_Enum; + +/* ====================================================== ALTPADCFGM ======================================================= */ +/* =========================================== GPIO ALTPADCFGM PAD49_SR [12..12] =========================================== */ +typedef enum { /*!< GPIO_ALTPADCFGM_PAD49_SR */ + GPIO_ALTPADCFGM_PAD49_SR_SR_EN = 1, /*!< SR_EN : Enables Slew rate control on pad value. */ +} GPIO_ALTPADCFGM_PAD49_SR_Enum; + +/* ============================================ GPIO ALTPADCFGM PAD48_SR [4..4] ============================================ */ +typedef enum { /*!< GPIO_ALTPADCFGM_PAD48_SR */ + GPIO_ALTPADCFGM_PAD48_SR_SR_EN = 1, /*!< SR_EN : Enables Slew rate control on pad value. */ +} GPIO_ALTPADCFGM_PAD48_SR_Enum; + +/* ========================================================= SCDET ========================================================= */ +/* ======================================================== CTENCFG ======================================================== */ +/* ============================================== GPIO CTENCFG EN31 [31..31] =============================================== */ +typedef enum { /*!< GPIO_CTENCFG_EN31 */ + GPIO_CTENCFG_EN31_DIS = 1, /*!< DIS : Disable CT31 for output value. */ + GPIO_CTENCFG_EN31_EN = 0, /*!< EN : Enable CT31 for output value. */ +} GPIO_CTENCFG_EN31_Enum; + +/* ============================================== GPIO CTENCFG EN30 [30..30] =============================================== */ +typedef enum { /*!< GPIO_CTENCFG_EN30 */ + GPIO_CTENCFG_EN30_DIS = 1, /*!< DIS : Disable CT30 for output value. */ + GPIO_CTENCFG_EN30_EN = 0, /*!< EN : Enable CT30 for output value. */ +} GPIO_CTENCFG_EN30_Enum; + +/* ============================================== GPIO CTENCFG EN29 [29..29] =============================================== */ +typedef enum { /*!< GPIO_CTENCFG_EN29 */ + GPIO_CTENCFG_EN29_DIS = 1, /*!< DIS : Disable CT29 for output value. */ + GPIO_CTENCFG_EN29_EN = 0, /*!< EN : Enable CT29 for output value. */ +} GPIO_CTENCFG_EN29_Enum; + +/* ============================================== GPIO CTENCFG EN28 [28..28] =============================================== */ +typedef enum { /*!< GPIO_CTENCFG_EN28 */ + GPIO_CTENCFG_EN28_DIS = 1, /*!< DIS : Disable CT28 for output value. */ + GPIO_CTENCFG_EN28_EN = 0, /*!< EN : Enable CT28 for output value. */ +} GPIO_CTENCFG_EN28_Enum; + +/* ============================================== GPIO CTENCFG EN27 [27..27] =============================================== */ +typedef enum { /*!< GPIO_CTENCFG_EN27 */ + GPIO_CTENCFG_EN27_DIS = 1, /*!< DIS : Disable CT27 for output value. */ + GPIO_CTENCFG_EN27_EN = 0, /*!< EN : Enable CT27 for output value. */ +} GPIO_CTENCFG_EN27_Enum; + +/* ============================================== GPIO CTENCFG EN26 [26..26] =============================================== */ +typedef enum { /*!< GPIO_CTENCFG_EN26 */ + GPIO_CTENCFG_EN26_DIS = 1, /*!< DIS : Disable CT26 for output value. */ + GPIO_CTENCFG_EN26_EN = 0, /*!< EN : Enable CT26 for output value. */ +} GPIO_CTENCFG_EN26_Enum; + +/* ============================================== GPIO CTENCFG EN25 [25..25] =============================================== */ +typedef enum { /*!< GPIO_CTENCFG_EN25 */ + GPIO_CTENCFG_EN25_DIS = 1, /*!< DIS : Disable CT25 for output value. */ + GPIO_CTENCFG_EN25_EN = 0, /*!< EN : Enable CT25 for output value. */ +} GPIO_CTENCFG_EN25_Enum; + +/* ============================================== GPIO CTENCFG EN24 [24..24] =============================================== */ +typedef enum { /*!< GPIO_CTENCFG_EN24 */ + GPIO_CTENCFG_EN24_DIS = 1, /*!< DIS : Disable CT24 for output value. */ + GPIO_CTENCFG_EN24_EN = 0, /*!< EN : Enable CT24 for output value. */ +} GPIO_CTENCFG_EN24_Enum; + +/* ============================================== GPIO CTENCFG EN23 [23..23] =============================================== */ +typedef enum { /*!< GPIO_CTENCFG_EN23 */ + GPIO_CTENCFG_EN23_DIS = 1, /*!< DIS : Disable CT23 for output value. */ + GPIO_CTENCFG_EN23_EN = 0, /*!< EN : Enable CT23 for output value. */ +} GPIO_CTENCFG_EN23_Enum; + +/* ============================================== GPIO CTENCFG EN22 [22..22] =============================================== */ +typedef enum { /*!< GPIO_CTENCFG_EN22 */ + GPIO_CTENCFG_EN22_DIS = 1, /*!< DIS : Disable CT22 for output value. */ + GPIO_CTENCFG_EN22_EN = 0, /*!< EN : Enable CT22 for output value. */ +} GPIO_CTENCFG_EN22_Enum; + +/* ============================================== GPIO CTENCFG EN21 [21..21] =============================================== */ +typedef enum { /*!< GPIO_CTENCFG_EN21 */ + GPIO_CTENCFG_EN21_DIS = 1, /*!< DIS : Disable CT21 for output value. */ + GPIO_CTENCFG_EN21_EN = 0, /*!< EN : Enable CT21 for output value. */ +} GPIO_CTENCFG_EN21_Enum; + +/* ============================================== GPIO CTENCFG EN20 [20..20] =============================================== */ +typedef enum { /*!< GPIO_CTENCFG_EN20 */ + GPIO_CTENCFG_EN20_DIS = 1, /*!< DIS : Disable CT20 for output value. */ + GPIO_CTENCFG_EN20_EN = 0, /*!< EN : Enable CT20 for output value. */ +} GPIO_CTENCFG_EN20_Enum; + +/* ============================================== GPIO CTENCFG EN19 [19..19] =============================================== */ +typedef enum { /*!< GPIO_CTENCFG_EN19 */ + GPIO_CTENCFG_EN19_DIS = 1, /*!< DIS : Disable CT19 for output value. */ + GPIO_CTENCFG_EN19_EN = 0, /*!< EN : Enable CT19 for output value. */ +} GPIO_CTENCFG_EN19_Enum; + +/* ============================================== GPIO CTENCFG EN18 [18..18] =============================================== */ +typedef enum { /*!< GPIO_CTENCFG_EN18 */ + GPIO_CTENCFG_EN18_DIS = 1, /*!< DIS : Disable CT18 for output value. */ + GPIO_CTENCFG_EN18_EN = 0, /*!< EN : Enable CT18 for output value. */ +} GPIO_CTENCFG_EN18_Enum; + +/* ============================================== GPIO CTENCFG EN17 [17..17] =============================================== */ +typedef enum { /*!< GPIO_CTENCFG_EN17 */ + GPIO_CTENCFG_EN17_DIS = 1, /*!< DIS : Disable CT17 for output value. */ + GPIO_CTENCFG_EN17_EN = 0, /*!< EN : Enable CT17 for output value. */ +} GPIO_CTENCFG_EN17_Enum; + +/* ============================================== GPIO CTENCFG EN16 [16..16] =============================================== */ +typedef enum { /*!< GPIO_CTENCFG_EN16 */ + GPIO_CTENCFG_EN16_DIS = 1, /*!< DIS : Disable CT16 for output value. */ + GPIO_CTENCFG_EN16_EN = 0, /*!< EN : Enable CT16 for output value. */ +} GPIO_CTENCFG_EN16_Enum; + +/* ============================================== GPIO CTENCFG EN15 [15..15] =============================================== */ +typedef enum { /*!< GPIO_CTENCFG_EN15 */ + GPIO_CTENCFG_EN15_DIS = 1, /*!< DIS : Disable CT15 for output value. */ + GPIO_CTENCFG_EN15_EN = 0, /*!< EN : Enable CT15 for output value. */ +} GPIO_CTENCFG_EN15_Enum; + +/* ============================================== GPIO CTENCFG EN14 [14..14] =============================================== */ +typedef enum { /*!< GPIO_CTENCFG_EN14 */ + GPIO_CTENCFG_EN14_DIS = 1, /*!< DIS : Disable CT14 for output value. */ + GPIO_CTENCFG_EN14_EN = 0, /*!< EN : Enable CT14 for output value. */ +} GPIO_CTENCFG_EN14_Enum; + +/* ============================================== GPIO CTENCFG EN13 [13..13] =============================================== */ +typedef enum { /*!< GPIO_CTENCFG_EN13 */ + GPIO_CTENCFG_EN13_DIS = 1, /*!< DIS : Disable CT13 for output value. */ + GPIO_CTENCFG_EN13_EN = 0, /*!< EN : Enable CT13 for output value. */ +} GPIO_CTENCFG_EN13_Enum; + +/* ============================================== GPIO CTENCFG EN12 [12..12] =============================================== */ +typedef enum { /*!< GPIO_CTENCFG_EN12 */ + GPIO_CTENCFG_EN12_DIS = 1, /*!< DIS : Disable CT12 for output value. */ + GPIO_CTENCFG_EN12_EN = 0, /*!< EN : Enable CT12 for output value. */ +} GPIO_CTENCFG_EN12_Enum; + +/* ============================================== GPIO CTENCFG EN11 [11..11] =============================================== */ +typedef enum { /*!< GPIO_CTENCFG_EN11 */ + GPIO_CTENCFG_EN11_DIS = 1, /*!< DIS : Disable CT11 for output value. */ + GPIO_CTENCFG_EN11_EN = 0, /*!< EN : Enable CT11 for output value. */ +} GPIO_CTENCFG_EN11_Enum; + +/* ============================================== GPIO CTENCFG EN10 [10..10] =============================================== */ +typedef enum { /*!< GPIO_CTENCFG_EN10 */ + GPIO_CTENCFG_EN10_DIS = 1, /*!< DIS : Disable CT10 for output value. */ + GPIO_CTENCFG_EN10_EN = 0, /*!< EN : Enable CT10 for output value. */ +} GPIO_CTENCFG_EN10_Enum; + +/* ================================================ GPIO CTENCFG EN9 [9..9] ================================================ */ +typedef enum { /*!< GPIO_CTENCFG_EN9 */ + GPIO_CTENCFG_EN9_DIS = 0, /*!< DIS : Disable CT9 for output value. */ +} GPIO_CTENCFG_EN9_Enum; + +/* ================================================ GPIO CTENCFG EN8 [8..8] ================================================ */ +typedef enum { /*!< GPIO_CTENCFG_EN8 */ + GPIO_CTENCFG_EN8_DIS = 1, /*!< DIS : Disable CT8 for output value. */ + GPIO_CTENCFG_EN8_EN = 0, /*!< EN : Enable CT8 for output value. */ +} GPIO_CTENCFG_EN8_Enum; + +/* ================================================ GPIO CTENCFG EN7 [7..7] ================================================ */ +typedef enum { /*!< GPIO_CTENCFG_EN7 */ + GPIO_CTENCFG_EN7_DIS = 1, /*!< DIS : Disable CT7 for output value. */ + GPIO_CTENCFG_EN7_EN = 0, /*!< EN : Enable CT7 for output value. */ +} GPIO_CTENCFG_EN7_Enum; + +/* ================================================ GPIO CTENCFG EN6 [6..6] ================================================ */ +typedef enum { /*!< GPIO_CTENCFG_EN6 */ + GPIO_CTENCFG_EN6_DIS = 1, /*!< DIS : Disable CT6 for output value. */ + GPIO_CTENCFG_EN6_EN = 0, /*!< EN : Enable CT6 for output value. */ +} GPIO_CTENCFG_EN6_Enum; + +/* ================================================ GPIO CTENCFG EN5 [5..5] ================================================ */ +typedef enum { /*!< GPIO_CTENCFG_EN5 */ + GPIO_CTENCFG_EN5_DIS = 1, /*!< DIS : Disable CT5 for output value. */ + GPIO_CTENCFG_EN5_EN = 0, /*!< EN : Enable CT5 for output value. */ +} GPIO_CTENCFG_EN5_Enum; + +/* ================================================ GPIO CTENCFG EN4 [4..4] ================================================ */ +typedef enum { /*!< GPIO_CTENCFG_EN4 */ + GPIO_CTENCFG_EN4_DIS = 1, /*!< DIS : Disable CT4 for output value. */ + GPIO_CTENCFG_EN4_EN = 0, /*!< EN : Enable CT4 for output value. */ +} GPIO_CTENCFG_EN4_Enum; + +/* ================================================ GPIO CTENCFG EN3 [3..3] ================================================ */ +typedef enum { /*!< GPIO_CTENCFG_EN3 */ + GPIO_CTENCFG_EN3_DIS = 1, /*!< DIS : Disable CT3 for output value. */ + GPIO_CTENCFG_EN3_EN = 0, /*!< EN : Enable CT3 for output value. */ +} GPIO_CTENCFG_EN3_Enum; + +/* ================================================ GPIO CTENCFG EN2 [2..2] ================================================ */ +typedef enum { /*!< GPIO_CTENCFG_EN2 */ + GPIO_CTENCFG_EN2_DIS = 1, /*!< DIS : Disable CT2 for output value. */ + GPIO_CTENCFG_EN2_EN = 0, /*!< EN : Enable CT2 for output value. */ +} GPIO_CTENCFG_EN2_Enum; + +/* ================================================ GPIO CTENCFG EN1 [1..1] ================================================ */ +typedef enum { /*!< GPIO_CTENCFG_EN1 */ + GPIO_CTENCFG_EN1_DIS = 1, /*!< DIS : Disable CT1 for output value. */ + GPIO_CTENCFG_EN1_EN = 0, /*!< EN : Enable CT1 for output value. */ +} GPIO_CTENCFG_EN1_Enum; + +/* ================================================ GPIO CTENCFG EN0 [0..0] ================================================ */ +typedef enum { /*!< GPIO_CTENCFG_EN0 */ + GPIO_CTENCFG_EN0_DIS = 1, /*!< DIS : Disable CT0 for output value. */ + GPIO_CTENCFG_EN0_EN = 0, /*!< EN : Enable CT0 for output value. */ +} GPIO_CTENCFG_EN0_Enum; + +/* ======================================================== INT0EN ========================================================= */ +/* ======================================================= INT0STAT ======================================================== */ +/* ======================================================== INT0CLR ======================================================== */ +/* ======================================================== INT0SET ======================================================== */ +/* ======================================================== INT1EN ========================================================= */ +/* ======================================================= INT1STAT ======================================================== */ +/* ======================================================== INT1CLR ======================================================== */ +/* ======================================================== INT1SET ======================================================== */ + + +/* =========================================================================================================================== */ +/* ================ IOM0 ================ */ +/* =========================================================================================================================== */ + +/* ========================================================= FIFO ========================================================== */ +/* ======================================================== FIFOPTR ======================================================== */ +/* ======================================================== FIFOTHR ======================================================== */ +/* ======================================================== FIFOPOP ======================================================== */ +/* ======================================================= FIFOPUSH ======================================================== */ +/* ======================================================= FIFOCTRL ======================================================== */ +/* ======================================================== FIFOLOC ======================================================== */ +/* ========================================================= INTEN ========================================================= */ +/* ======================================================== INTSTAT ======================================================== */ +/* ======================================================== INTCLR ========================================================= */ +/* ======================================================== INTSET ========================================================= */ +/* ======================================================== CLKCFG ========================================================= */ +/* ============================================== IOM0 CLKCFG DIVEN [12..12] =============================================== */ +typedef enum { /*!< IOM0_CLKCFG_DIVEN */ + IOM0_CLKCFG_DIVEN_DIS = 0, /*!< DIS : Disable TOTPER division. value. */ + IOM0_CLKCFG_DIVEN_EN = 1, /*!< EN : Enable TOTPER division. value. */ +} IOM0_CLKCFG_DIVEN_Enum; + +/* =============================================== IOM0 CLKCFG DIV3 [11..11] =============================================== */ +typedef enum { /*!< IOM0_CLKCFG_DIV3 */ + IOM0_CLKCFG_DIV3_DIS = 0, /*!< DIS : Select divide by 1. value. */ + IOM0_CLKCFG_DIV3_EN = 1, /*!< EN : Select divide by 3. value. */ +} IOM0_CLKCFG_DIV3_Enum; + +/* =============================================== IOM0 CLKCFG FSEL [8..10] ================================================ */ +typedef enum { /*!< IOM0_CLKCFG_FSEL */ + IOM0_CLKCFG_FSEL_MIN_PWR = 0, /*!< MIN_PWR : Selects the minimum power clock. This setting should + be used whenever the IOM is not active. value. */ + IOM0_CLKCFG_FSEL_HFRC = 1, /*!< HFRC : Selects the HFRC as the input clock. value. */ + IOM0_CLKCFG_FSEL_HFRC_DIV2 = 2, /*!< HFRC_DIV2 : Selects the HFRC / 2 as the input clock. value. */ + IOM0_CLKCFG_FSEL_HFRC_DIV4 = 3, /*!< HFRC_DIV4 : Selects the HFRC / 4 as the input clock. value. */ + IOM0_CLKCFG_FSEL_HFRC_DIV8 = 4, /*!< HFRC_DIV8 : Selects the HFRC / 8 as the input clock. value. */ + IOM0_CLKCFG_FSEL_HFRC_DIV16 = 5, /*!< HFRC_DIV16 : Selects the HFRC / 16 as the input clock. value. */ + IOM0_CLKCFG_FSEL_HFRC_DIV32 = 6, /*!< HFRC_DIV32 : Selects the HFRC / 32 as the input clock. value. */ + IOM0_CLKCFG_FSEL_HFRC_DIV64 = 7, /*!< HFRC_DIV64 : Selects the HFRC / 64 as the input clock. value. */ +} IOM0_CLKCFG_FSEL_Enum; + +/* ====================================================== SUBMODCTRL ======================================================= */ +/* =========================================== IOM0 SUBMODCTRL SMOD1TYPE [5..7] ============================================ */ +typedef enum { /*!< IOM0_SUBMODCTRL_SMOD1TYPE */ + IOM0_SUBMODCTRL_SMOD1TYPE_MSPI = 0, /*!< MSPI : SPI Master submodule value. */ + IOM0_SUBMODCTRL_SMOD1TYPE_I2C_MASTER = 1, /*!< I2C_MASTER : MI2C submodule value. */ + IOM0_SUBMODCTRL_SMOD1TYPE_SSPI = 2, /*!< SSPI : SPI Slave submodule value. */ + IOM0_SUBMODCTRL_SMOD1TYPE_SI2C = 3, /*!< SI2C : I2C Slave submodule value. */ + IOM0_SUBMODCTRL_SMOD1TYPE_NA = 7, /*!< NA : NOT INSTALLED value. */ +} IOM0_SUBMODCTRL_SMOD1TYPE_Enum; + +/* =========================================== IOM0 SUBMODCTRL SMOD0TYPE [1..3] ============================================ */ +typedef enum { /*!< IOM0_SUBMODCTRL_SMOD0TYPE */ + IOM0_SUBMODCTRL_SMOD0TYPE_SPI_MASTER = 0, /*!< SPI_MASTER : MSPI submodule value. */ + IOM0_SUBMODCTRL_SMOD0TYPE_I2C_MASTER = 1, /*!< I2C_MASTER : I2C Master submodule value. */ + IOM0_SUBMODCTRL_SMOD0TYPE_SSPI = 2, /*!< SSPI : SPI Slave submodule value. */ + IOM0_SUBMODCTRL_SMOD0TYPE_SI2C = 3, /*!< SI2C : I2C Slave submodule value. */ + IOM0_SUBMODCTRL_SMOD0TYPE_NA = 7, /*!< NA : NOT INSTALLED value. */ +} IOM0_SUBMODCTRL_SMOD0TYPE_Enum; + +/* ========================================================== CMD ========================================================== */ +/* ================================================== IOM0 CMD CMD [0..4] ================================================== */ +typedef enum { /*!< IOM0_CMD_CMD */ + IOM0_CMD_CMD_WRITE = 1, /*!< WRITE : Write command using count of offset bytes specified + in the OFFSETCNT field value. */ + IOM0_CMD_CMD_READ = 2, /*!< READ : Read command using count of offset bytes specified in + the OFFSETCNT field value. */ + IOM0_CMD_CMD_TMW = 3, /*!< TMW : SPI only. Test mode to do constant write operations. Useful + for debug and power measurements. Will continually send + data in OFFSET field value. */ + IOM0_CMD_CMD_TMR = 4, /*!< TMR : SPI Only. Test mode to do constant read operations. Useful + for debug and power measurements. Will continually read + data from external input value. */ +} IOM0_CMD_CMD_Enum; + +/* ======================================================== CMDRPT ========================================================= */ +/* ======================================================= OFFSETHI ======================================================== */ +/* ======================================================== CMDSTAT ======================================================== */ +/* ============================================== IOM0 CMDSTAT CMDSTAT [5..7] ============================================== */ +typedef enum { /*!< IOM0_CMDSTAT_CMDSTAT */ + IOM0_CMDSTAT_CMDSTAT_ERR = 1, /*!< ERR : Error encountered with command value. */ + IOM0_CMDSTAT_CMDSTAT_ACTIVE = 2, /*!< ACTIVE : Actively processing command value. */ + IOM0_CMDSTAT_CMDSTAT_IDLE = 4, /*!< IDLE : Idle state, no active command, no error value. */ + IOM0_CMDSTAT_CMDSTAT_WAIT = 6, /*!< WAIT : Command in progress, but waiting on data from host value. */ +} IOM0_CMDSTAT_CMDSTAT_Enum; + +/* ======================================================= DMATRIGEN ======================================================= */ +/* ====================================================== DMATRIGSTAT ====================================================== */ +/* ======================================================== DMACFG ========================================================= */ +/* ============================================== IOM0 DMACFG DPWROFF [9..9] =============================================== */ +typedef enum { /*!< IOM0_DMACFG_DPWROFF */ + IOM0_DMACFG_DPWROFF_DIS = 0, /*!< DIS : Power off disabled value. */ + IOM0_DMACFG_DPWROFF_EN = 1, /*!< EN : Power off enabled value. */ +} IOM0_DMACFG_DPWROFF_Enum; + +/* =============================================== IOM0 DMACFG DMAPRI [8..8] =============================================== */ +typedef enum { /*!< IOM0_DMACFG_DMAPRI */ + IOM0_DMACFG_DMAPRI_LOW = 0, /*!< LOW : Low Priority (service as best effort) value. */ + IOM0_DMACFG_DMAPRI_HIGH = 1, /*!< HIGH : High Priority (service immediately) value. */ +} IOM0_DMACFG_DMAPRI_Enum; + +/* =============================================== IOM0 DMACFG DMADIR [1..1] =============================================== */ +typedef enum { /*!< IOM0_DMACFG_DMADIR */ + IOM0_DMACFG_DMADIR_P2M = 0, /*!< P2M : Peripheral to Memory (SRAM) transaction. To be set when + doing IOM read operations, ie reading data from external + devices. value. */ + IOM0_DMACFG_DMADIR_M2P = 1, /*!< M2P : Memory to Peripheral transaction. To be set when doing + IOM write operations, ie writing data to external devices. + value. */ +} IOM0_DMACFG_DMADIR_Enum; + +/* =============================================== IOM0 DMACFG DMAEN [0..0] ================================================ */ +typedef enum { /*!< IOM0_DMACFG_DMAEN */ + IOM0_DMACFG_DMAEN_DIS = 0, /*!< DIS : Disable DMA Function value. */ + IOM0_DMACFG_DMAEN_EN = 1, /*!< EN : Enable DMA Function value. */ +} IOM0_DMACFG_DMAEN_Enum; + +/* ====================================================== DMATOTCOUNT ====================================================== */ +/* ====================================================== DMATARGADDR ====================================================== */ +/* ======================================================== DMASTAT ======================================================== */ +/* ========================================================= CQCFG ========================================================= */ +/* ================================================ IOM0 CQCFG CQPRI [1..1] ================================================ */ +typedef enum { /*!< IOM0_CQCFG_CQPRI */ + IOM0_CQCFG_CQPRI_LOW = 0, /*!< LOW : Low Priority (service as best effort) value. */ + IOM0_CQCFG_CQPRI_HIGH = 1, /*!< HIGH : High Priority (service immediately) value. */ +} IOM0_CQCFG_CQPRI_Enum; + +/* ================================================ IOM0 CQCFG CQEN [0..0] ================================================= */ +typedef enum { /*!< IOM0_CQCFG_CQEN */ + IOM0_CQCFG_CQEN_DIS = 0, /*!< DIS : Disable CQ Function value. */ + IOM0_CQCFG_CQEN_EN = 1, /*!< EN : Enable CQ Function value. */ +} IOM0_CQCFG_CQEN_Enum; + +/* ======================================================== CQADDR ========================================================= */ +/* ======================================================== CQSTAT ========================================================= */ +/* ======================================================== CQFLAGS ======================================================== */ +/* ====================================================== CQSETCLEAR ======================================================= */ +/* ======================================================= CQPAUSEEN ======================================================= */ +/* ============================================= IOM0 CQPAUSEEN CQPEN [0..15] ============================================== */ +typedef enum { /*!< IOM0_CQPAUSEEN_CQPEN */ + IOM0_CQPAUSEEN_CQPEN_IDXEQ = 32768, /*!< IDXEQ : Pauses the command queue when the current index matches + the last index value. */ + IOM0_CQPAUSEEN_CQPEN_BLEXOREN = 16384, /*!< BLEXOREN : Pause command queue when input BLE bit XORed with + SWFLAG4 is '1' value. */ + IOM0_CQPAUSEEN_CQPEN_IOMXOREN = 8192, /*!< IOMXOREN : Pause command queue when input IOM bit XORed with + SWFLAG3 is '1' value. */ + IOM0_CQPAUSEEN_CQPEN_GPIOXOREN = 4096, /*!< GPIOXOREN : Pause command queue when input GPIO irq_bit XORed + with SWFLAG2 is '1' value. */ + IOM0_CQPAUSEEN_CQPEN_MSPI1XNOREN = 2048, /*!< MSPI1XNOREN : Pause command queue when input MSPI1 bit XNORed + with SWFLAG1 is '1' value. */ + IOM0_CQPAUSEEN_CQPEN_MSPI0XNOREN = 1024, /*!< MSPI0XNOREN : Pause command queue when input MSPI0 bit XNORed + with SWFLAG0 is '1' value. */ + IOM0_CQPAUSEEN_CQPEN_MSPI1XOREN = 512, /*!< MSPI1XOREN : Pause command queue when input MSPI1 bit XORed + with SWFLAG1 is '1' value. */ + IOM0_CQPAUSEEN_CQPEN_MSPI0XOREN = 256, /*!< MSPI0XOREN : Pause command queue when input MSPI0 bit XORed + with SWFLAG0 is '1' value. */ + IOM0_CQPAUSEEN_CQPEN_SWFLAGEN7 = 128, /*!< SWFLAGEN7 : Pause the command queue when software flag bit 7 + is '1'. value. */ + IOM0_CQPAUSEEN_CQPEN_SWFLAGEN6 = 64, /*!< SWFLAGEN6 : Pause the command queue when software flag bit 6 + is '1' value. */ + IOM0_CQPAUSEEN_CQPEN_SWFLAGEN5 = 32, /*!< SWFLAGEN5 : Pause the command queue when software flag bit 5 + is '1' value. */ + IOM0_CQPAUSEEN_CQPEN_SWFLAGEN4 = 16, /*!< SWFLAGEN4 : Pause the command queue when software flag bit 4 + is '1' value. */ + IOM0_CQPAUSEEN_CQPEN_SWFLAGEN3 = 8, /*!< SWFLAGEN3 : Pause the command queue when software flag bit 3 + is '1' value. */ + IOM0_CQPAUSEEN_CQPEN_SWFLAGEN2 = 4, /*!< SWFLAGEN2 : Pause the command queue when software flag bit 2 + is '1' value. */ + IOM0_CQPAUSEEN_CQPEN_SWFLAGEN1 = 2, /*!< SWFLAGEN1 : Pause the command queue when software flag bit 1 + is '1' value. */ + IOM0_CQPAUSEEN_CQPEN_SWFLAGEN0 = 1, /*!< SWFLAGEN0 : Pause the command queue when software flag bit 0 + is '1' value. */ +} IOM0_CQPAUSEEN_CQPEN_Enum; + +/* ======================================================= CQCURIDX ======================================================== */ +/* ======================================================= CQENDIDX ======================================================== */ +/* ======================================================== STATUS ========================================================= */ +/* =============================================== IOM0 STATUS IDLEST [2..2] =============================================== */ +typedef enum { /*!< IOM0_STATUS_IDLEST */ + IOM0_STATUS_IDLEST_IDLE = 1, /*!< IDLE : The I/O state machine is in the idle state. value. */ +} IOM0_STATUS_IDLEST_Enum; + +/* =============================================== IOM0 STATUS CMDACT [1..1] =============================================== */ +typedef enum { /*!< IOM0_STATUS_CMDACT */ + IOM0_STATUS_CMDACT_ACTIVE = 1, /*!< ACTIVE : An I/O command is active. Indicates the active module + has an active command and is processing this. De-asserted + when the command is completed. value. */ +} IOM0_STATUS_CMDACT_Enum; + +/* ================================================ IOM0 STATUS ERR [0..0] ================================================= */ +typedef enum { /*!< IOM0_STATUS_ERR */ + IOM0_STATUS_ERR_ERROR = 1, /*!< ERROR : Bit has been deprecated and will always return 0. value. */ +} IOM0_STATUS_ERR_Enum; + +/* ======================================================== MSPICFG ======================================================== */ +/* ============================================= IOM0 MSPICFG SPILSB [23..23] ============================================== */ +typedef enum { /*!< IOM0_MSPICFG_SPILSB */ + IOM0_MSPICFG_SPILSB_MSB = 0, /*!< MSB : Send and receive MSB bit first value. */ + IOM0_MSPICFG_SPILSB_LSB = 1, /*!< LSB : Send and receive LSB bit first value. */ +} IOM0_MSPICFG_SPILSB_Enum; + +/* ============================================= IOM0 MSPICFG RDFCPOL [22..22] ============================================= */ +typedef enum { /*!< IOM0_MSPICFG_RDFCPOL */ + IOM0_MSPICFG_RDFCPOL_HIGH = 0, /*!< HIGH : Flow control signal high creates flow control. value. */ + IOM0_MSPICFG_RDFCPOL_LOW = 1, /*!< LOW : Flow control signal low creates flow control. value. */ +} IOM0_MSPICFG_RDFCPOL_Enum; + +/* ============================================= IOM0 MSPICFG WTFCPOL [21..21] ============================================= */ +typedef enum { /*!< IOM0_MSPICFG_WTFCPOL */ + IOM0_MSPICFG_WTFCPOL_HIGH = 0, /*!< HIGH : Flow control signal high(1) creates flow control and + byte transfers will stop until the flow control signal + goes low. value. */ + IOM0_MSPICFG_WTFCPOL_LOW = 1, /*!< LOW : Flow control signal low(0) creates flow control and byte + transfers will stop until the flow control signal goes + high(1). value. */ +} IOM0_MSPICFG_WTFCPOL_Enum; + +/* ============================================= IOM0 MSPICFG WTFCIRQ [20..20] ============================================= */ +typedef enum { /*!< IOM0_MSPICFG_WTFCIRQ */ + IOM0_MSPICFG_WTFCIRQ_MISO = 0, /*!< MISO : MISO is used as the write mode flow control signal. value. */ + IOM0_MSPICFG_WTFCIRQ_IRQ = 1, /*!< IRQ : IRQ is used as the write mode flow control signal. value. */ +} IOM0_MSPICFG_WTFCIRQ_Enum; + +/* ============================================= IOM0 MSPICFG MOSIINV [18..18] ============================================= */ +typedef enum { /*!< IOM0_MSPICFG_MOSIINV */ + IOM0_MSPICFG_MOSIINV_NORMAL = 0, /*!< NORMAL : MOSI is set to 0 in read mode and 1 in write mode. + value. */ + IOM0_MSPICFG_MOSIINV_INVERT = 1, /*!< INVERT : MOSI is set to 1 in read mode and 0 in write mode. + value. */ +} IOM0_MSPICFG_MOSIINV_Enum; + +/* ============================================== IOM0 MSPICFG RDFC [17..17] =============================================== */ +typedef enum { /*!< IOM0_MSPICFG_RDFC */ + IOM0_MSPICFG_RDFC_DIS = 0, /*!< DIS : Read mode flow control disabled. value. */ + IOM0_MSPICFG_RDFC_EN = 1, /*!< EN : Read mode flow control enabled. value. */ +} IOM0_MSPICFG_RDFC_Enum; + +/* ============================================== IOM0 MSPICFG WTFC [16..16] =============================================== */ +typedef enum { /*!< IOM0_MSPICFG_WTFC */ + IOM0_MSPICFG_WTFC_DIS = 0, /*!< DIS : Write mode flow control disabled. value. */ + IOM0_MSPICFG_WTFC_EN = 1, /*!< EN : Write mode flow control enabled. value. */ +} IOM0_MSPICFG_WTFC_Enum; + +/* =============================================== IOM0 MSPICFG SPHA [1..1] ================================================ */ +typedef enum { /*!< IOM0_MSPICFG_SPHA */ + IOM0_MSPICFG_SPHA_SAMPLE_LEADING_EDGE = 0, /*!< SAMPLE_LEADING_EDGE : Sample on the leading (first) clock edge. + value. */ + IOM0_MSPICFG_SPHA_SAMPLE_TRAILING_EDGE = 1, /*!< SAMPLE_TRAILING_EDGE : Sample on the trailing (second) clock + edge. value. */ +} IOM0_MSPICFG_SPHA_Enum; + +/* =============================================== IOM0 MSPICFG SPOL [0..0] ================================================ */ +typedef enum { /*!< IOM0_MSPICFG_SPOL */ + IOM0_MSPICFG_SPOL_CLK_BASE_0 = 0, /*!< CLK_BASE_0 : The base value of the clock is 0. value. */ + IOM0_MSPICFG_SPOL_CLK_BASE_1 = 1, /*!< CLK_BASE_1 : The base value of the clock is 1. value. */ +} IOM0_MSPICFG_SPOL_Enum; + +/* ======================================================== MI2CCFG ======================================================== */ +/* =============================================== IOM0 MI2CCFG ARBEN [2..2] =============================================== */ +typedef enum { /*!< IOM0_MI2CCFG_ARBEN */ + IOM0_MI2CCFG_ARBEN_ARBEN = 1, /*!< ARBEN : Enable multi-master bus arbitration support for this + i2c master value. */ + IOM0_MI2CCFG_ARBEN_ARBDIS = 0, /*!< ARBDIS : Disable multi-master bus arbitration support for this + i2c master value. */ +} IOM0_MI2CCFG_ARBEN_Enum; + +/* ============================================== IOM0 MI2CCFG I2CLSB [1..1] =============================================== */ +typedef enum { /*!< IOM0_MI2CCFG_I2CLSB */ + IOM0_MI2CCFG_I2CLSB_MSBFIRST = 0, /*!< MSBFIRST : Byte data is transmitted MSB first onto the bus/read + from the bus value. */ + IOM0_MI2CCFG_I2CLSB_LSBFIRST = 1, /*!< LSBFIRST : Byte data is transmitted LSB first onto the bus/read + from the bus value. */ +} IOM0_MI2CCFG_I2CLSB_Enum; + +/* ============================================== IOM0 MI2CCFG ADDRSZ [0..0] =============================================== */ +typedef enum { /*!< IOM0_MI2CCFG_ADDRSZ */ + IOM0_MI2CCFG_ADDRSZ_ADDRSZ7 = 0, /*!< ADDRSZ7 : Use 7b addressing for I2C master transactions value. */ + IOM0_MI2CCFG_ADDRSZ_ADDRSZ10 = 1, /*!< ADDRSZ10 : Use 10b addressing for I2C master transactions value. */ +} IOM0_MI2CCFG_ADDRSZ_Enum; + +/* ======================================================== DEVCFG ========================================================= */ +/* ======================================================== IOMDBG ========================================================= */ + + +/* =========================================================================================================================== */ +/* ================ IOSLAVE ================ */ +/* =========================================================================================================================== */ + +/* ======================================================== FIFOPTR ======================================================== */ +/* ======================================================== FIFOCFG ======================================================== */ +/* ======================================================== FIFOTHR ======================================================== */ +/* ========================================================= FUPD ========================================================== */ +/* ======================================================== FIFOCTR ======================================================== */ +/* ======================================================== FIFOINC ======================================================== */ +/* ========================================================== CFG ========================================================== */ +/* ============================================== IOSLAVE CFG IFCEN [31..31] =============================================== */ +typedef enum { /*!< IOSLAVE_CFG_IFCEN */ + IOSLAVE_CFG_IFCEN_DIS = 0, /*!< DIS : Disable the IOSLAVE value. */ + IOSLAVE_CFG_IFCEN_EN = 1, /*!< EN : Enable the IOSLAVE value. */ +} IOSLAVE_CFG_IFCEN_Enum; + +/* ============================================== IOSLAVE CFG STARTRD [4..4] =============================================== */ +typedef enum { /*!< IOSLAVE_CFG_STARTRD */ + IOSLAVE_CFG_STARTRD_LATE = 0, /*!< LATE : Initiate I/O RAM read late in each transferred byte. + value. */ + IOSLAVE_CFG_STARTRD_EARLY = 1, /*!< EARLY : Initiate I/O RAM read early in each transferred byte. + value. */ +} IOSLAVE_CFG_STARTRD_Enum; + +/* ================================================ IOSLAVE CFG LSB [2..2] ================================================= */ +typedef enum { /*!< IOSLAVE_CFG_LSB */ + IOSLAVE_CFG_LSB_MSB_FIRST = 0, /*!< MSB_FIRST : Data is assumed to be sent and received with MSB + first. value. */ + IOSLAVE_CFG_LSB_LSB_FIRST = 1, /*!< LSB_FIRST : Data is assumed to be sent and received with LSB + first. value. */ +} IOSLAVE_CFG_LSB_Enum; + +/* ================================================ IOSLAVE CFG SPOL [1..1] ================================================ */ +typedef enum { /*!< IOSLAVE_CFG_SPOL */ + IOSLAVE_CFG_SPOL_SPI_MODES_0_3 = 0, /*!< SPI_MODES_0_3 : Polarity 0, handles SPI modes 0 and 3. value. */ + IOSLAVE_CFG_SPOL_SPI_MODES_1_2 = 1, /*!< SPI_MODES_1_2 : Polarity 1, handles SPI modes 1 and 2. value. */ +} IOSLAVE_CFG_SPOL_Enum; + +/* =============================================== IOSLAVE CFG IFCSEL [0..0] =============================================== */ +typedef enum { /*!< IOSLAVE_CFG_IFCSEL */ + IOSLAVE_CFG_IFCSEL_I2C = 0, /*!< I2C : Selects I2C interface for the IO Slave. value. */ + IOSLAVE_CFG_IFCSEL_SPI = 1, /*!< SPI : Selects SPI interface for the IO Slave. value. */ +} IOSLAVE_CFG_IFCSEL_Enum; + +/* ========================================================= PRENC ========================================================= */ +/* ======================================================= IOINTCTL ======================================================== */ +/* ======================================================== GENADD ========================================================= */ +/* ========================================================= INTEN ========================================================= */ +/* ======================================================== INTSTAT ======================================================== */ +/* ======================================================== INTCLR ========================================================= */ +/* ======================================================== INTSET ========================================================= */ +/* ====================================================== REGACCINTEN ====================================================== */ +/* ===================================================== REGACCINTSTAT ===================================================== */ +/* ===================================================== REGACCINTCLR ====================================================== */ +/* ===================================================== REGACCINTSET ====================================================== */ + + +/* =========================================================================================================================== */ +/* ================ MCUCTRL ================ */ +/* =========================================================================================================================== */ + +/* ======================================================== CHIPPN ========================================================= */ +/* ============================================ MCUCTRL CHIPPN PARTNUM [0..31] ============================================= */ +typedef enum { /*!< MCUCTRL_CHIPPN_PARTNUM */ + MCUCTRL_CHIPPN_PARTNUM_APOLLO3 = 100663296,/*!< APOLLO3 : Apollo3 part number is 0x06xxxxxx. value. */ + MCUCTRL_CHIPPN_PARTNUM_APOLLO2 = 50331648,/*!< APOLLO2 : Apollo2 part number is 0x03xxxxxx. value. */ + MCUCTRL_CHIPPN_PARTNUM_APOLLO = 16777216,/*!< APOLLO : Apollo part number is 0x01xxxxxx. value. */ + MCUCTRL_CHIPPN_PARTNUM_PN_M = -16777216,/*!< PN_M : Mask for the part number field. value. */ + MCUCTRL_CHIPPN_PARTNUM_PN_S = 24, /*!< PN_S : Bit position for the part number field. value. */ + MCUCTRL_CHIPPN_PARTNUM_FLASHSIZE_M = 15728640,/*!< FLASHSIZE_M : Mask for the FLASH_SIZE field.Values:0: 16KB1: + 32KB2: 64KB3: 128KB4: 256KB5: 512KB6: 1MB7: 2MB value. */ + MCUCTRL_CHIPPN_PARTNUM_FLASHSIZE_S = 20, /*!< FLASHSIZE_S : Bit position for the FLASH_SIZE field. value. */ + MCUCTRL_CHIPPN_PARTNUM_SRAMSIZE_M = 983040,/*!< SRAMSIZE_M : Mask for the SRAM_SIZE field.Values:0: 16KB1: 32KB2: + 64KB3: 128KB4: 256KB5: 512KB6: 1MB7: 384KB value. */ + MCUCTRL_CHIPPN_PARTNUM_SRAMSIZE_S = 16, /*!< SRAMSIZE_S : Bit position for the SRAM_SIZE field. value. */ + MCUCTRL_CHIPPN_PARTNUM_REV_M = 65280, /*!< REV_M : Mask for the revision field. Bits [15:12] are major + rev, [11:8] are minor rev.Values:0: Major Rev A, Minor + Rev 01: Major Rev B, Minor Rev 1 value. */ + MCUCTRL_CHIPPN_PARTNUM_REV_S = 8, /*!< REV_S : Bit position for the revision field. value. */ + MCUCTRL_CHIPPN_PARTNUM_PKG_M = 192, /*!< PKG_M : Mask for the package field.Values:0: SIP1: QFN2: BGA3: + CSP value. */ + MCUCTRL_CHIPPN_PARTNUM_PKG_S = 6, /*!< PKG_S : Bit position for the package field. value. */ + MCUCTRL_CHIPPN_PARTNUM_PINS_M = 56, /*!< PINS_M : Mask for the pins field.Values:0: 25 pins1: 49 pins2: + 64 pins3: 81 pins value. */ + MCUCTRL_CHIPPN_PARTNUM_PINS_S = 3, /*!< PINS_S : Bit position for the pins field. value. */ + MCUCTRL_CHIPPN_PARTNUM_TEMP_S = 1, /*!< TEMP_S : Bit position for the temperature field. value. */ + MCUCTRL_CHIPPN_PARTNUM_QUAL_S = 0, /*!< QUAL_S : Bit position for the qualified field. value. */ +} MCUCTRL_CHIPPN_PARTNUM_Enum; + +/* ======================================================== CHIPID0 ======================================================== */ +/* ============================================ MCUCTRL CHIPID0 CHIPID0 [0..31] ============================================ */ +typedef enum { /*!< MCUCTRL_CHIPID0_CHIPID0 */ + MCUCTRL_CHIPID0_CHIPID0_APOLLO3 = 0, /*!< APOLLO3 : Apollo3 CHIPID0. value. */ +} MCUCTRL_CHIPID0_CHIPID0_Enum; + +/* ======================================================== CHIPID1 ======================================================== */ +/* ============================================ MCUCTRL CHIPID1 CHIPID1 [0..31] ============================================ */ +typedef enum { /*!< MCUCTRL_CHIPID1_CHIPID1 */ + MCUCTRL_CHIPID1_CHIPID1_APOLLO3 = 0, /*!< APOLLO3 : Apollo3 CHIPID1. value. */ +} MCUCTRL_CHIPID1_CHIPID1_Enum; + +/* ======================================================== CHIPREV ======================================================== */ +/* ============================================= MCUCTRL CHIPREV REVMAJ [4..7] ============================================= */ +typedef enum { /*!< MCUCTRL_CHIPREV_REVMAJ */ + MCUCTRL_CHIPREV_REVMAJ_A = 1, /*!< A : Apollo3 revision A value. */ +} MCUCTRL_CHIPREV_REVMAJ_Enum; + +/* ============================================= MCUCTRL CHIPREV REVMIN [0..3] ============================================= */ +typedef enum { /*!< MCUCTRL_CHIPREV_REVMIN */ + MCUCTRL_CHIPREV_REVMIN_REV1 = 2, /*!< REV1 : Apollo3 minor rev 1. value. */ + MCUCTRL_CHIPREV_REVMIN_REV0 = 1, /*!< REV0 : Apollo3 minor rev 0. Minor revision value, succeeding + minor revisions will increment from this value. value. */ +} MCUCTRL_CHIPREV_REVMIN_Enum; + +/* ======================================================= VENDORID ======================================================== */ +/* =========================================== MCUCTRL VENDORID VENDORID [0..31] =========================================== */ +typedef enum { /*!< MCUCTRL_VENDORID_VENDORID */ + MCUCTRL_VENDORID_VENDORID_AMBIQ = 1095582289,/*!< AMBIQ : Ambiq Vendor ID value. */ +} MCUCTRL_VENDORID_VENDORID_Enum; + +/* ========================================================== SKU ========================================================== */ +/* ===================================================== FEATUREENABLE ===================================================== */ +/* ======================================== MCUCTRL FEATUREENABLE BURSTAVAIL [6..6] ======================================== */ +typedef enum { /*!< MCUCTRL_FEATUREENABLE_BURSTAVAIL */ + MCUCTRL_FEATUREENABLE_BURSTAVAIL_AVAIL = 1, /*!< AVAIL : Burst functionality available value. */ + MCUCTRL_FEATUREENABLE_BURSTAVAIL_NOTAVAIL = 0,/*!< NOTAVAIL : Burst functionality not available value. */ +} MCUCTRL_FEATUREENABLE_BURSTAVAIL_Enum; + +/* ========================================= MCUCTRL FEATUREENABLE BURSTREQ [4..4] ========================================= */ +typedef enum { /*!< MCUCTRL_FEATUREENABLE_BURSTREQ */ + MCUCTRL_FEATUREENABLE_BURSTREQ_EN = 1, /*!< EN : Enable the Burst functionality value. */ + MCUCTRL_FEATUREENABLE_BURSTREQ_DIS = 0, /*!< DIS : Disable the Burst functionality value. */ +} MCUCTRL_FEATUREENABLE_BURSTREQ_Enum; + +/* ========================================= MCUCTRL FEATUREENABLE BLEAVAIL [2..2] ========================================= */ +typedef enum { /*!< MCUCTRL_FEATUREENABLE_BLEAVAIL */ + MCUCTRL_FEATUREENABLE_BLEAVAIL_AVAIL = 1, /*!< AVAIL : BLE functionality available value. */ + MCUCTRL_FEATUREENABLE_BLEAVAIL_NOTAVAIL = 0, /*!< NOTAVAIL : BLE functionality not available value. */ +} MCUCTRL_FEATUREENABLE_BLEAVAIL_Enum; + +/* ========================================== MCUCTRL FEATUREENABLE BLEREQ [0..0] ========================================== */ +typedef enum { /*!< MCUCTRL_FEATUREENABLE_BLEREQ */ + MCUCTRL_FEATUREENABLE_BLEREQ_EN = 1, /*!< EN : Enable the BLE functionality value. */ + MCUCTRL_FEATUREENABLE_BLEREQ_DIS = 0, /*!< DIS : Disable the BLE functionality value. */ +} MCUCTRL_FEATUREENABLE_BLEREQ_Enum; + +/* ======================================================= DEBUGGER ======================================================== */ +/* ======================================================== BODCTRL ======================================================== */ +/* ======================================================= ADCPWRDLY ======================================================= */ +/* ======================================================== ADCCAL ========================================================= */ +/* ========================================== MCUCTRL ADCCAL ADCCALIBRATED [1..1] ========================================== */ +typedef enum { /*!< MCUCTRL_ADCCAL_ADCCALIBRATED */ + MCUCTRL_ADCCAL_ADCCALIBRATED_FALSE = 0, /*!< FALSE : ADC is not calibrated value. */ + MCUCTRL_ADCCAL_ADCCALIBRATED_TRUE = 1, /*!< TRUE : ADC is calibrated value. */ +} MCUCTRL_ADCCAL_ADCCALIBRATED_Enum; + +/* =========================================== MCUCTRL ADCCAL CALONPWRUP [0..0] ============================================ */ +typedef enum { /*!< MCUCTRL_ADCCAL_CALONPWRUP */ + MCUCTRL_ADCCAL_CALONPWRUP_DIS = 0, /*!< DIS : Disable automatic calibration on initial power up value. */ + MCUCTRL_ADCCAL_CALONPWRUP_EN = 1, /*!< EN : Enable automatic calibration on initial power up value. */ +} MCUCTRL_ADCCAL_CALONPWRUP_Enum; + +/* ====================================================== ADCBATTLOAD ====================================================== */ +/* ========================================== MCUCTRL ADCBATTLOAD BATTLOAD [0..0] ========================================== */ +typedef enum { /*!< MCUCTRL_ADCBATTLOAD_BATTLOAD */ + MCUCTRL_ADCBATTLOAD_BATTLOAD_DIS = 0, /*!< DIS : Battery load is disconnected value. */ + MCUCTRL_ADCBATTLOAD_BATTLOAD_EN = 1, /*!< EN : Battery load is enabled value. */ +} MCUCTRL_ADCBATTLOAD_BATTLOAD_Enum; + +/* ======================================================== ADCTRIM ======================================================== */ +/* ====================================================== ADCREFCOMP ======================================================= */ +/* ======================================================= XTALCTRL ======================================================== */ +/* ========================================== MCUCTRL XTALCTRL PWDBODXTAL [5..5] =========================================== */ +typedef enum { /*!< MCUCTRL_XTALCTRL_PWDBODXTAL */ + MCUCTRL_XTALCTRL_PWDBODXTAL_PWRUPBOD = 0, /*!< PWRUPBOD : Power up xtal on BOD value. */ + MCUCTRL_XTALCTRL_PWDBODXTAL_PWRDNBOD = 1, /*!< PWRDNBOD : Power down XTAL on BOD. value. */ +} MCUCTRL_XTALCTRL_PWDBODXTAL_Enum; + +/* ========================================= MCUCTRL XTALCTRL PDNBCMPRXTAL [4..4] ========================================== */ +typedef enum { /*!< MCUCTRL_XTALCTRL_PDNBCMPRXTAL */ + MCUCTRL_XTALCTRL_PDNBCMPRXTAL_PWRUPCOMP = 1, /*!< PWRUPCOMP : Power up XTAL oscillator comparator. value. */ + MCUCTRL_XTALCTRL_PDNBCMPRXTAL_PWRDNCOMP = 0, /*!< PWRDNCOMP : Power down XTAL oscillator comparator. value. */ +} MCUCTRL_XTALCTRL_PDNBCMPRXTAL_Enum; + +/* ========================================= MCUCTRL XTALCTRL PDNBCOREXTAL [3..3] ========================================== */ +typedef enum { /*!< MCUCTRL_XTALCTRL_PDNBCOREXTAL */ + MCUCTRL_XTALCTRL_PDNBCOREXTAL_PWRUPCORE = 1, /*!< PWRUPCORE : Power up XTAL oscillator core. value. */ + MCUCTRL_XTALCTRL_PDNBCOREXTAL_PWRDNCORE = 0, /*!< PWRDNCORE : Power down XTAL oscillator core. value. */ +} MCUCTRL_XTALCTRL_PDNBCOREXTAL_Enum; + +/* ========================================== MCUCTRL XTALCTRL BYPCMPRXTAL [2..2] ========================================== */ +typedef enum { /*!< MCUCTRL_XTALCTRL_BYPCMPRXTAL */ + MCUCTRL_XTALCTRL_BYPCMPRXTAL_USECOMP = 0, /*!< USECOMP : Use the XTAL oscillator comparator. value. */ + MCUCTRL_XTALCTRL_BYPCMPRXTAL_BYPCOMP = 1, /*!< BYPCOMP : Bypass the XTAL oscillator comparator. value. */ +} MCUCTRL_XTALCTRL_BYPCMPRXTAL_Enum; + +/* ========================================= MCUCTRL XTALCTRL FDBKDSBLXTAL [1..1] ========================================== */ +typedef enum { /*!< MCUCTRL_XTALCTRL_FDBKDSBLXTAL */ + MCUCTRL_XTALCTRL_FDBKDSBLXTAL_EN = 0, /*!< EN : Enable XTAL oscillator comparator. value. */ + MCUCTRL_XTALCTRL_FDBKDSBLXTAL_DIS = 1, /*!< DIS : Disable XTAL oscillator comparator. value. */ +} MCUCTRL_XTALCTRL_FDBKDSBLXTAL_Enum; + +/* ============================================ MCUCTRL XTALCTRL XTALSWE [0..0] ============================================ */ +typedef enum { /*!< MCUCTRL_XTALCTRL_XTALSWE */ + MCUCTRL_XTALCTRL_XTALSWE_OVERRIDE_DIS = 0, /*!< OVERRIDE_DIS : XTAL Software Override Disable. value. */ + MCUCTRL_XTALCTRL_XTALSWE_OVERRIDE_EN = 1, /*!< OVERRIDE_EN : XTAL Software Override Enable. value. */ +} MCUCTRL_XTALCTRL_XTALSWE_Enum; + +/* ====================================================== XTALGENCTRL ====================================================== */ +/* ========================================== MCUCTRL XTALGENCTRL ACWARMUP [0..1] ========================================== */ +typedef enum { /*!< MCUCTRL_XTALGENCTRL_ACWARMUP */ + MCUCTRL_XTALGENCTRL_ACWARMUP_SEC1 = 0, /*!< SEC1 : Warmup period of 1-2 seconds value. */ + MCUCTRL_XTALGENCTRL_ACWARMUP_SEC2 = 1, /*!< SEC2 : Warmup period of 2-4 seconds value. */ + MCUCTRL_XTALGENCTRL_ACWARMUP_SEC4 = 2, /*!< SEC4 : Warmup period of 4-8 seconds value. */ + MCUCTRL_XTALGENCTRL_ACWARMUP_SEC8 = 3, /*!< SEC8 : Warmup period of 8-16 seconds value. */ +} MCUCTRL_XTALGENCTRL_ACWARMUP_Enum; + +/* ======================================================= MISCCTRL ======================================================== */ +/* ====================================================== BOOTLOADER ======================================================= */ +/* ======================================= MCUCTRL BOOTLOADER SECBOOTONRST [30..31] ======================================== */ +typedef enum { /*!< MCUCTRL_BOOTLOADER_SECBOOTONRST */ + MCUCTRL_BOOTLOADER_SECBOOTONRST_DISABLED = 0, /*!< DISABLED : Secure boot disabled value. */ + MCUCTRL_BOOTLOADER_SECBOOTONRST_ENABLED = 1, /*!< ENABLED : Secure boot enabled value. */ + MCUCTRL_BOOTLOADER_SECBOOTONRST_ERROR = 2, /*!< ERROR : Error in secure boot configuration value. */ +} MCUCTRL_BOOTLOADER_SECBOOTONRST_Enum; + +/* ========================================== MCUCTRL BOOTLOADER SECBOOT [28..29] ========================================== */ +typedef enum { /*!< MCUCTRL_BOOTLOADER_SECBOOT */ + MCUCTRL_BOOTLOADER_SECBOOT_DISABLED = 0, /*!< DISABLED : Secure boot disabled value. */ + MCUCTRL_BOOTLOADER_SECBOOT_ENABLED = 1, /*!< ENABLED : Secure boot enabled value. */ + MCUCTRL_BOOTLOADER_SECBOOT_ERROR = 2, /*!< ERROR : Error in secure boot configuration value. */ +} MCUCTRL_BOOTLOADER_SECBOOT_Enum; + +/* ====================================== MCUCTRL BOOTLOADER SECBOOTFEATURE [26..27] ======================================= */ +typedef enum { /*!< MCUCTRL_BOOTLOADER_SECBOOTFEATURE */ + MCUCTRL_BOOTLOADER_SECBOOTFEATURE_DISABLED = 0,/*!< DISABLED : Secure boot disabled value. */ + MCUCTRL_BOOTLOADER_SECBOOTFEATURE_ENABLED = 1,/*!< ENABLED : Secure boot enabled value. */ + MCUCTRL_BOOTLOADER_SECBOOTFEATURE_ERROR = 2, /*!< ERROR : Error in secure boot configuration value. */ +} MCUCTRL_BOOTLOADER_SECBOOTFEATURE_Enum; + +/* ========================================== MCUCTRL BOOTLOADER PROTLOCK [2..2] =========================================== */ +typedef enum { /*!< MCUCTRL_BOOTLOADER_PROTLOCK */ + MCUCTRL_BOOTLOADER_PROTLOCK_LOCK = 1, /*!< LOCK : Enable the secure boot lock value. */ +} MCUCTRL_BOOTLOADER_PROTLOCK_Enum; + +/* =========================================== MCUCTRL BOOTLOADER SBLOCK [1..1] ============================================ */ +typedef enum { /*!< MCUCTRL_BOOTLOADER_SBLOCK */ + MCUCTRL_BOOTLOADER_SBLOCK_LOCK = 1, /*!< LOCK : Enable the secure boot lock value. */ +} MCUCTRL_BOOTLOADER_SBLOCK_Enum; + +/* ======================================== MCUCTRL BOOTLOADER BOOTLOADERLOW [0..0] ======================================== */ +typedef enum { /*!< MCUCTRL_BOOTLOADER_BOOTLOADERLOW */ + MCUCTRL_BOOTLOADER_BOOTLOADERLOW_ADDR0 = 1, /*!< ADDR0 : Bootloader code at 0x00000000. value. */ +} MCUCTRL_BOOTLOADER_BOOTLOADERLOW_Enum; + +/* ====================================================== SHADOWVALID ====================================================== */ +/* ======================================== MCUCTRL SHADOWVALID INFO0_VALID [2..2] ========================================= */ +typedef enum { /*!< MCUCTRL_SHADOWVALID_INFO0_VALID */ + MCUCTRL_SHADOWVALID_INFO0_VALID_VALID = 1, /*!< VALID : Flash info0 (customer) space contains valid data. value. */ +} MCUCTRL_SHADOWVALID_INFO0_VALID_Enum; + +/* ========================================== MCUCTRL SHADOWVALID BLDSLEEP [1..1] ========================================== */ +typedef enum { /*!< MCUCTRL_SHADOWVALID_BLDSLEEP */ + MCUCTRL_SHADOWVALID_BLDSLEEP_DEEPSLEEP = 1, /*!< DEEPSLEEP : Bootloader will go to deep sleep if no flash image + loaded value. */ +} MCUCTRL_SHADOWVALID_BLDSLEEP_Enum; + +/* =========================================== MCUCTRL SHADOWVALID VALID [0..0] ============================================ */ +typedef enum { /*!< MCUCTRL_SHADOWVALID_VALID */ + MCUCTRL_SHADOWVALID_VALID_VALID = 1, /*!< VALID : Flash information space contains valid data. value. */ +} MCUCTRL_SHADOWVALID_VALID_Enum; + +/* ======================================================= SCRATCH0 ======================================================== */ +/* ======================================================= SCRATCH1 ======================================================== */ +/* ==================================================== ICODEFAULTADDR ===================================================== */ +/* ==================================================== DCODEFAULTADDR ===================================================== */ +/* ===================================================== SYSFAULTADDR ====================================================== */ +/* ====================================================== FAULTSTATUS ====================================================== */ +/* ========================================== MCUCTRL FAULTSTATUS SYSFAULT [2..2] ========================================== */ +typedef enum { /*!< MCUCTRL_FAULTSTATUS_SYSFAULT */ + MCUCTRL_FAULTSTATUS_SYSFAULT_NOFAULT = 0, /*!< NOFAULT : No bus fault has been detected. value. */ + MCUCTRL_FAULTSTATUS_SYSFAULT_FAULT = 1, /*!< FAULT : Bus fault detected. value. */ +} MCUCTRL_FAULTSTATUS_SYSFAULT_Enum; + +/* ========================================= MCUCTRL FAULTSTATUS DCODEFAULT [1..1] ========================================= */ +typedef enum { /*!< MCUCTRL_FAULTSTATUS_DCODEFAULT */ + MCUCTRL_FAULTSTATUS_DCODEFAULT_NOFAULT = 0, /*!< NOFAULT : No DCODE fault has been detected. value. */ + MCUCTRL_FAULTSTATUS_DCODEFAULT_FAULT = 1, /*!< FAULT : DCODE fault detected. value. */ +} MCUCTRL_FAULTSTATUS_DCODEFAULT_Enum; + +/* ========================================= MCUCTRL FAULTSTATUS ICODEFAULT [0..0] ========================================= */ +typedef enum { /*!< MCUCTRL_FAULTSTATUS_ICODEFAULT */ + MCUCTRL_FAULTSTATUS_ICODEFAULT_NOFAULT = 0, /*!< NOFAULT : No ICODE fault has been detected. value. */ + MCUCTRL_FAULTSTATUS_ICODEFAULT_FAULT = 1, /*!< FAULT : ICODE fault detected. value. */ +} MCUCTRL_FAULTSTATUS_ICODEFAULT_Enum; + +/* ==================================================== FAULTCAPTUREEN ===================================================== */ +/* ===================================== MCUCTRL FAULTCAPTUREEN FAULTCAPTUREEN [0..0] ====================================== */ +typedef enum { /*!< MCUCTRL_FAULTCAPTUREEN_FAULTCAPTUREEN */ + MCUCTRL_FAULTCAPTUREEN_FAULTCAPTUREEN_DIS = 0,/*!< DIS : Disable fault capture. value. */ + MCUCTRL_FAULTCAPTUREEN_FAULTCAPTUREEN_EN = 1, /*!< EN : Enable fault capture. value. */ +} MCUCTRL_FAULTCAPTUREEN_FAULTCAPTUREEN_Enum; + +/* ========================================================= DBGR1 ========================================================= */ +/* ========================================================= DBGR2 ========================================================= */ +/* ======================================================= PMUENABLE ======================================================= */ +/* ============================================ MCUCTRL PMUENABLE ENABLE [0..0] ============================================ */ +typedef enum { /*!< MCUCTRL_PMUENABLE_ENABLE */ + MCUCTRL_PMUENABLE_ENABLE_DIS = 0, /*!< DIS : Disable MCU power management. value. */ + MCUCTRL_PMUENABLE_ENABLE_EN = 1, /*!< EN : Enable MCU power management. value. */ +} MCUCTRL_PMUENABLE_ENABLE_Enum; + +/* ======================================================= TPIUCTRL ======================================================== */ +/* ============================================ MCUCTRL TPIUCTRL CLKSEL [8..10] ============================================ */ +typedef enum { /*!< MCUCTRL_TPIUCTRL_CLKSEL */ + MCUCTRL_TPIUCTRL_CLKSEL_LOWPWR = 0, /*!< LOWPWR : Low power state. value. */ + MCUCTRL_TPIUCTRL_CLKSEL_HFRCDIV2 = 1, /*!< HFRCDIV2 : Selects HFRC divided by 2 as the source TPIU clk + value. */ + MCUCTRL_TPIUCTRL_CLKSEL_HFRCDIV8 = 2, /*!< HFRCDIV8 : Selects HFRC divided by 8 as the source TPIU clk + value. */ + MCUCTRL_TPIUCTRL_CLKSEL_HFRCDIV16 = 3, /*!< HFRCDIV16 : Selects HFRC divided by 16 as the source TPIU clk + value. */ + MCUCTRL_TPIUCTRL_CLKSEL_HFRCDIV32 = 4, /*!< HFRCDIV32 : Selects HFRC divided by 32 as the source TPIU clk + value. */ +} MCUCTRL_TPIUCTRL_CLKSEL_Enum; + +/* ============================================ MCUCTRL TPIUCTRL ENABLE [0..0] ============================================= */ +typedef enum { /*!< MCUCTRL_TPIUCTRL_ENABLE */ + MCUCTRL_TPIUCTRL_ENABLE_DIS = 0, /*!< DIS : Disable the TPIU. value. */ + MCUCTRL_TPIUCTRL_ENABLE_EN = 1, /*!< EN : Enable the TPIU. value. */ +} MCUCTRL_TPIUCTRL_ENABLE_Enum; + +/* ====================================================== OTAPOINTER ======================================================= */ +/* ====================================================== APBDMACTRL ======================================================= */ +/* ========================================= MCUCTRL APBDMACTRL DECODEABORT [1..1] ========================================= */ +typedef enum { /*!< MCUCTRL_APBDMACTRL_DECODEABORT */ + MCUCTRL_APBDMACTRL_DECODEABORT_DISABLE = 0, /*!< DISABLE : Bus operations to powered down peripherals are quietly + discarded value. */ + MCUCTRL_APBDMACTRL_DECODEABORT_ENABLE = 1, /*!< ENABLE : Bus operations to powered down peripherals result in + a bus fault. value. */ +} MCUCTRL_APBDMACTRL_DECODEABORT_Enum; + +/* ========================================= MCUCTRL APBDMACTRL DMA_ENABLE [0..0] ========================================== */ +typedef enum { /*!< MCUCTRL_APBDMACTRL_DMA_ENABLE */ + MCUCTRL_APBDMACTRL_DMA_ENABLE_DISABLE = 0, /*!< DISABLE : DMA operations disabled value. */ + MCUCTRL_APBDMACTRL_DMA_ENABLE_ENABLE = 1, /*!< ENABLE : DMA operations enabled value. */ +} MCUCTRL_APBDMACTRL_DMA_ENABLE_Enum; + +/* ======================================================= SRAMMODE ======================================================== */ +/* ====================================================== KEXTCLKSEL ======================================================= */ +/* ========================================= MCUCTRL KEXTCLKSEL KEXTCLKSEL [0..31] ========================================= */ +typedef enum { /*!< MCUCTRL_KEXTCLKSEL_KEXTCLKSEL */ + MCUCTRL_KEXTCLKSEL_KEXTCLKSEL_Key = 83, /*!< Key : Key value. */ +} MCUCTRL_KEXTCLKSEL_KEXTCLKSEL_Enum; + +/* ======================================================= SIMOBUCK4 ======================================================= */ +/* ======================================================= BLEBUCK2 ======================================================== */ +/* ====================================================== FLASHWPROT0 ====================================================== */ +/* ====================================================== FLASHWPROT1 ====================================================== */ +/* ====================================================== FLASHRPROT0 ====================================================== */ +/* ====================================================== FLASHRPROT1 ====================================================== */ +/* ================================================= DMASRAMWRITEPROTECT0 ================================================== */ +/* ================================================= DMASRAMWRITEPROTECT1 ================================================== */ +/* ================================================== DMASRAMREADPROTECT0 ================================================== */ +/* ================================================== DMASRAMREADPROTECT1 ================================================== */ + + +/* =========================================================================================================================== */ +/* ================ MSPI ================ */ +/* =========================================================================================================================== */ + +/* ========================================================= CTRL ========================================================== */ +/* ========================================================== CFG ========================================================== */ +/* ================================================ MSPI CFG CPOL [17..17] ================================================= */ +typedef enum { /*!< MSPI_CFG_CPOL */ + MSPI_CFG_CPOL_LOW = 0, /*!< LOW : Clock inactive state is low. value. */ + MSPI_CFG_CPOL_HIGH = 1, /*!< HIGH : Clock inactive state is high. value. */ +} MSPI_CFG_CPOL_Enum; + +/* ================================================ MSPI CFG CPHA [16..16] ================================================= */ +typedef enum { /*!< MSPI_CFG_CPHA */ + MSPI_CFG_CPHA_MIDDLE = 0, /*!< MIDDLE : Clock toggles in middle of data bit. value. */ + MSPI_CFG_CPHA_START = 1, /*!< START : Clock toggles at start of data bit. value. */ +} MSPI_CFG_CPHA_Enum; + +/* ================================================ MSPI CFG DEVCFG [0..3] ================================================= */ +typedef enum { /*!< MSPI_CFG_DEVCFG */ + MSPI_CFG_DEVCFG_SERIAL0 = 1, /*!< SERIAL0 : Single bit SPI flash on chip select 0 value. */ + MSPI_CFG_DEVCFG_SERIAL1 = 2, /*!< SERIAL1 : Single bit SPI flash on chip select 1 value. */ + MSPI_CFG_DEVCFG_DUAL0 = 5, /*!< DUAL0 : Dual SPI flash on chip select 0 value. */ + MSPI_CFG_DEVCFG_DUAL1 = 6, /*!< DUAL1 : Dual bit SPI flash on chip select 1 value. */ + MSPI_CFG_DEVCFG_QUAD0 = 9, /*!< QUAD0 : Quad SPI flash on chip select 0 value. */ + MSPI_CFG_DEVCFG_QUAD1 = 10, /*!< QUAD1 : Quad SPI flash on chip select 1 value. */ + MSPI_CFG_DEVCFG_OCTAL0 = 13, /*!< OCTAL0 : Octal SPI flash on chip select 0 value. */ + MSPI_CFG_DEVCFG_OCTAL1 = 14, /*!< OCTAL1 : Octal SPI flash on chip select 1 value. */ + MSPI_CFG_DEVCFG_QUADPAIRED = 15, /*!< QUADPAIRED : Dual Quad SPI flash on chip selects 0/1. value. */ + MSPI_CFG_DEVCFG_QUADPAIRED_SERIAL = 3, /*!< QUADPAIRED_SERIAL : Dual Quad SPI flash on chip selects 0/1, + but transmit in serial mode for initialization operations + value. */ +} MSPI_CFG_DEVCFG_Enum; + +/* ========================================================= ADDR ========================================================== */ +/* ========================================================= INSTR ========================================================= */ +/* ======================================================== TXFIFO ========================================================= */ +/* ======================================================== RXFIFO ========================================================= */ +/* ======================================================= TXENTRIES ======================================================= */ +/* ======================================================= RXENTRIES ======================================================= */ +/* ======================================================= THRESHOLD ======================================================= */ +/* ======================================================== MSPICFG ======================================================== */ +/* ============================================== MSPI MSPICFG CLKDIV [8..13] ============================================== */ +typedef enum { /*!< MSPI_MSPICFG_CLKDIV */ + MSPI_MSPICFG_CLKDIV_CLK24 = 2, /*!< CLK24 : 24 MHz MSPI clock value. */ + MSPI_MSPICFG_CLKDIV_CLK12 = 4, /*!< CLK12 : 12 MHz MSPI clock value. */ + MSPI_MSPICFG_CLKDIV_CLK6 = 8, /*!< CLK6 : 6 MHz MSPI clock value. */ + MSPI_MSPICFG_CLKDIV_CLK3 = 16, /*!< CLK3 : 3 MHz MSPI clock value. */ + MSPI_MSPICFG_CLKDIV_CLK1_5 = 32, /*!< CLK1_5 : 1.5 MHz MSPI clock value. */ +} MSPI_MSPICFG_CLKDIV_Enum; + +/* ============================================== MSPI MSPICFG IOMSEL [4..6] =============================================== */ +typedef enum { /*!< MSPI_MSPICFG_IOMSEL */ + MSPI_MSPICFG_IOMSEL_IOM0 = 0, /*!< IOM0 : ERROR: desc VALUE MISSING value. */ + MSPI_MSPICFG_IOMSEL_IOM1 = 1, /*!< IOM1 : ERROR: desc VALUE MISSING value. */ + MSPI_MSPICFG_IOMSEL_IOM2 = 2, /*!< IOM2 : ERROR: desc VALUE MISSING value. */ + MSPI_MSPICFG_IOMSEL_IOM3 = 3, /*!< IOM3 : ERROR: desc VALUE MISSING value. */ + MSPI_MSPICFG_IOMSEL_IOM4 = 4, /*!< IOM4 : ERROR: desc VALUE MISSING value. */ + MSPI_MSPICFG_IOMSEL_IOM5 = 5, /*!< IOM5 : ERROR: desc VALUE MISSING value. */ + MSPI_MSPICFG_IOMSEL_DISABLED = 7, /*!< DISABLED : No IOM selected. Signals always zero. value. */ +} MSPI_MSPICFG_IOMSEL_Enum; + +/* =============================================== MSPI MSPICFG TXNEG [3..3] =============================================== */ +typedef enum { /*!< MSPI_MSPICFG_TXNEG */ + MSPI_MSPICFG_TXNEG_NORMAL = 0, /*!< NORMAL : TX launched from posedge internal clock value. */ + MSPI_MSPICFG_TXNEG_NEGEDGE = 1, /*!< NEGEDGE : TX data launched from negedge of internal clock value. */ +} MSPI_MSPICFG_TXNEG_Enum; + +/* =============================================== MSPI MSPICFG RXNEG [2..2] =============================================== */ +typedef enum { /*!< MSPI_MSPICFG_RXNEG */ + MSPI_MSPICFG_RXNEG_NORMAL = 0, /*!< NORMAL : RX data sampled on posedge of internal clock value. */ + MSPI_MSPICFG_RXNEG_NEGEDGE = 1, /*!< NEGEDGE : RX data sampled on negedge of internal clock value. */ +} MSPI_MSPICFG_RXNEG_Enum; + +/* =============================================== MSPI MSPICFG RXCAP [1..1] =============================================== */ +typedef enum { /*!< MSPI_MSPICFG_RXCAP */ + MSPI_MSPICFG_RXCAP_NORMAL = 0, /*!< NORMAL : RX Capture phase aligns with CPHA setting value. */ + MSPI_MSPICFG_RXCAP_DELAY = 1, /*!< DELAY : RX Capture phase is delayed from CPHA setting by one + clock edge value. */ +} MSPI_MSPICFG_RXCAP_Enum; + +/* ============================================== MSPI MSPICFG APBCLK [0..0] =============================================== */ +typedef enum { /*!< MSPI_MSPICFG_APBCLK */ + MSPI_MSPICFG_APBCLK_DIS = 0, /*!< DIS : Disable continuous clock. value. */ + MSPI_MSPICFG_APBCLK_EN = 1, /*!< EN : Enable continuous clock. value. */ +} MSPI_MSPICFG_APBCLK_Enum; + +/* ======================================================== PADCFG ========================================================= */ +/* ======================================================= PADOUTEN ======================================================== */ +/* ============================================== MSPI PADOUTEN OUTEN [0..8] =============================================== */ +typedef enum { /*!< MSPI_PADOUTEN_OUTEN */ + MSPI_PADOUTEN_OUTEN_QUAD0 = 271, /*!< QUAD0 : Quad0 (4 data + 1 clock) value. */ + MSPI_PADOUTEN_OUTEN_QUAD1 = 496, /*!< QUAD1 : Quad1 (4 data + 1 clock) value. */ + MSPI_PADOUTEN_OUTEN_OCTAL = 511, /*!< OCTAL : Octal (8 data + 1 clock) value. */ + MSPI_PADOUTEN_OUTEN_SERIAL0 = 259, /*!< SERIAL0 : Serial (2 data + 1 clock) value. */ +} MSPI_PADOUTEN_OUTEN_Enum; + +/* ========================================================= FLASH ========================================================= */ +/* =============================================== MSPI FLASH XIPACK [2..3] ================================================ */ +typedef enum { /*!< MSPI_FLASH_XIPACK */ + MSPI_FLASH_XIPACK_NOACK = 0, /*!< NOACK : No acknowledege sent. Data IOs are tristated the first + turnaround cycle value. */ + MSPI_FLASH_XIPACK_ACK = 2, /*!< ACK : Positive acknowledege sent. Data IOs are driven to 0 the + first turnaround cycle to acknowledge XIP mode value. */ + MSPI_FLASH_XIPACK_TERMINATE = 3, /*!< TERMINATE : Negative acknowledege sent. Data IOs are driven + to 1 the first turnaround cycle to terminate XIP mode. + XIPSENDI should be reenabled for the next transfer value. */ +} MSPI_FLASH_XIPACK_Enum; + +/* ====================================================== SCRAMBLING ======================================================= */ +/* ========================================================= INTEN ========================================================= */ +/* ======================================================== INTSTAT ======================================================== */ +/* ======================================================== INTCLR ========================================================= */ +/* ======================================================== INTSET ========================================================= */ +/* ======================================================== DMACFG ========================================================= */ +/* =============================================== MSPI DMACFG DMAPRI [3..4] =============================================== */ +typedef enum { /*!< MSPI_DMACFG_DMAPRI */ + MSPI_DMACFG_DMAPRI_LOW = 0, /*!< LOW : Low Priority (service as best effort) value. */ + MSPI_DMACFG_DMAPRI_HIGH = 1, /*!< HIGH : High Priority (service immediately) value. */ + MSPI_DMACFG_DMAPRI_AUTO = 2, /*!< AUTO : Auto Priority (priority raised once TX FIFO empties or + RX FIFO fills) value. */ +} MSPI_DMACFG_DMAPRI_Enum; + +/* =============================================== MSPI DMACFG DMADIR [2..2] =============================================== */ +typedef enum { /*!< MSPI_DMACFG_DMADIR */ + MSPI_DMACFG_DMADIR_P2M = 0, /*!< P2M : Peripheral to Memory (SRAM) transaction value. */ + MSPI_DMACFG_DMADIR_M2P = 1, /*!< M2P : Memory to Peripheral transaction value. */ +} MSPI_DMACFG_DMADIR_Enum; + +/* =============================================== MSPI DMACFG DMAEN [0..1] ================================================ */ +typedef enum { /*!< MSPI_DMACFG_DMAEN */ + MSPI_DMACFG_DMAEN_DIS = 0, /*!< DIS : Disable DMA Function value. */ + MSPI_DMACFG_DMAEN_EN = 3, /*!< EN : Enable HW controlled DMA Function to manage DMA to flash + devices. HW will automatically handle issuance of instruction/address + bytes based on settings in the FLASH register. value. */ +} MSPI_DMACFG_DMAEN_Enum; + +/* ======================================================== DMASTAT ======================================================== */ +/* ====================================================== DMATARGADDR ====================================================== */ +/* ====================================================== DMADEVADDR ======================================================= */ +/* ====================================================== DMATOTCOUNT ====================================================== */ +/* ======================================================= DMABCOUNT ======================================================= */ +/* ======================================================= DMATHRESH ======================================================= */ +/* ========================================================= CQCFG ========================================================= */ +/* ================================================ MSPI CQCFG CQPRI [1..1] ================================================ */ +typedef enum { /*!< MSPI_CQCFG_CQPRI */ + MSPI_CQCFG_CQPRI_LOW = 0, /*!< LOW : Low Priority (service as best effort) value. */ + MSPI_CQCFG_CQPRI_HIGH = 1, /*!< HIGH : High Priority (service immediately) value. */ +} MSPI_CQCFG_CQPRI_Enum; + +/* ================================================ MSPI CQCFG CQEN [0..0] ================================================= */ +typedef enum { /*!< MSPI_CQCFG_CQEN */ + MSPI_CQCFG_CQEN_DIS = 0, /*!< DIS : Disable CQ Function value. */ + MSPI_CQCFG_CQEN_EN = 1, /*!< EN : Enable CQ Function value. */ +} MSPI_CQCFG_CQEN_Enum; + +/* ======================================================== CQADDR ========================================================= */ +/* ======================================================== CQSTAT ========================================================= */ +/* ======================================================== CQFLAGS ======================================================== */ +/* ============================================= MSPI CQFLAGS CQFLAGS [0..15] ============================================== */ +typedef enum { /*!< MSPI_CQFLAGS_CQFLAGS */ + MSPI_CQFLAGS_CQFLAGS_STOP = 32768, /*!< STOP : CQ Stop Flag. When set, CQ processing will complete. + value. */ + MSPI_CQFLAGS_CQFLAGS_CQIDX = 16384, /*!< CQIDX : CQ Index Pointers (CURIDX/ENDIDX) match. value. */ + MSPI_CQFLAGS_CQFLAGS_DMACPL = 2048, /*!< DMACPL : DMA Complete Status (hardwired DMACPL bit in DMASTAT) + value. */ + MSPI_CQFLAGS_CQFLAGS_CMDCPL = 1024, /*!< CMDCPL : PIO Operation completed (STATUS bit in CTRL register) + value. */ + MSPI_CQFLAGS_CQFLAGS_IOM1READY = 512, /*!< IOM1READY : IOM Buffer 1 Ready Status (from selected IOM). This + status is the result of XOR'ing the IOM0START with the + incoming status from the IOM. When high, MSPI can send + to the buffer. value. */ + MSPI_CQFLAGS_CQFLAGS_IOM0READY = 256, /*!< IOM0READY : IOM Buffer 0 Ready Status (from selected IOM). This + status is the result of XOR'ing the IOM0START with the + incoming status from the IOM. When high, MSPI can send + to the buffer. value. */ + MSPI_CQFLAGS_CQFLAGS_SWFLAG7 = 128, /*!< SWFLAG7 : Software flag 7. Can be used by software to start/pause + operations value. */ + MSPI_CQFLAGS_CQFLAGS_SWFLAG6 = 64, /*!< SWFLAG6 : Software flag 6. Can be used by software to start/pause + operatoins value. */ + MSPI_CQFLAGS_CQFLAGS_SWFLAG5 = 32, /*!< SWFLAG5 : Software flag 5. Can be used by software to start/pause + operations value. */ + MSPI_CQFLAGS_CQFLAGS_SWFLAG4 = 16, /*!< SWFLAG4 : Software flag 4. Can be used by software to start/pause + operatoins value. */ + MSPI_CQFLAGS_CQFLAGS_SWFLAG3 = 8, /*!< SWFLAG3 : Software flag 3. Can be used by software to start/pause + operations value. */ + MSPI_CQFLAGS_CQFLAGS_SWFLAG2 = 4, /*!< SWFLAG2 : Software flag 2. Can be used by software to start/pause + operatoins value. */ + MSPI_CQFLAGS_CQFLAGS_SWFLAG1 = 2, /*!< SWFLAG1 : Software flag 1. Can be used by software to start/pause + operations value. */ + MSPI_CQFLAGS_CQFLAGS_SWFLAG0 = 1, /*!< SWFLAG0 : Software flag 0. Can be used by software to start/pause + operatoins value. */ +} MSPI_CQFLAGS_CQFLAGS_Enum; + +/* ====================================================== CQSETCLEAR ======================================================= */ +/* ======================================================== CQPAUSE ======================================================== */ +/* ============================================== MSPI CQPAUSE CQMASK [0..15] ============================================== */ +typedef enum { /*!< MSPI_CQPAUSE_CQMASK */ + MSPI_CQPAUSE_CQMASK_STOP = 32768, /*!< STOP : CQ Stop Flag. When set, CQ processing will complete. + value. */ + MSPI_CQPAUSE_CQMASK_CQIDX = 16384, /*!< CQIDX : CQ Index Pointers (CURIDX/ENDIDX) match. value. */ + MSPI_CQPAUSE_CQMASK_DMACPL = 2048, /*!< DMACPL : DMA Complete Status (hardwired DMACPL bit in DMASTAT) + value. */ + MSPI_CQPAUSE_CQMASK_CMDCPL = 1024, /*!< CMDCPL : PIO Operation completed (STATUS bit in CTRL register) + value. */ + MSPI_CQPAUSE_CQMASK_IOM1READY = 512, /*!< IOM1READY : IOM Buffer 1 Ready Status (from selected IOM). This + status is the result of XOR'ing the IOM0START with the + incoming status from the IOM. When high, MSPI can send + to the buffer. value. */ + MSPI_CQPAUSE_CQMASK_IOM0READY = 256, /*!< IOM0READY : IOM Buffer 0 Ready Status (from selected IOM). This + status is the result of XOR'ing the IOM0START with the + incoming status from the IOM. When high, MSPI can send + to the buffer. value. */ + MSPI_CQPAUSE_CQMASK_SWFLAG7 = 128, /*!< SWFLAG7 : Software flag 7. Can be used by software to start/pause + operations value. */ + MSPI_CQPAUSE_CQMASK_SWFLAG6 = 64, /*!< SWFLAG6 : Software flag 6. Can be used by software to start/pause + operatoins value. */ + MSPI_CQPAUSE_CQMASK_SWFLAG5 = 32, /*!< SWFLAG5 : Software flag 5. Can be used by software to start/pause + operations value. */ + MSPI_CQPAUSE_CQMASK_SWFLAG4 = 16, /*!< SWFLAG4 : Software flag 4. Can be used by software to start/pause + operatoins value. */ + MSPI_CQPAUSE_CQMASK_SWFLAG3 = 8, /*!< SWFLAG3 : Software flag 3. Can be used by software to start/pause + operations value. */ + MSPI_CQPAUSE_CQMASK_SWFLAG2 = 4, /*!< SWFLAG2 : Software flag 2. Can be used by software to start/pause + operatoins value. */ + MSPI_CQPAUSE_CQMASK_SWFLAG1 = 2, /*!< SWFLAG1 : Software flag 1. Can be used by software to start/pause + operations value. */ + MSPI_CQPAUSE_CQMASK_SWFLAG0 = 1, /*!< SWFLAG0 : Software flag 0. Can be used by software to start/pause + operatoins value. */ +} MSPI_CQPAUSE_CQMASK_Enum; + +/* ======================================================= CQCURIDX ======================================================== */ +/* ======================================================= CQENDIDX ======================================================== */ + + +/* =========================================================================================================================== */ +/* ================ PDM ================ */ +/* =========================================================================================================================== */ + +/* ========================================================= PCFG ========================================================== */ +/* =============================================== PDM PCFG LRSWAP [31..31] ================================================ */ +typedef enum { /*!< PDM_PCFG_LRSWAP */ + PDM_PCFG_LRSWAP_EN = 1, /*!< EN : Swap left and right channels (FIFO Read RIGHT_LEFT). value. */ + PDM_PCFG_LRSWAP_NOSWAP = 0, /*!< NOSWAP : No channel swapping (IFO Read LEFT_RIGHT). value. */ +} PDM_PCFG_LRSWAP_Enum; + +/* ============================================== PDM PCFG PGARIGHT [26..30] =============================================== */ +typedef enum { /*!< PDM_PCFG_PGARIGHT */ + PDM_PCFG_PGARIGHT_P405DB = 31, /*!< P405DB : 40.5 db gain. value. */ + PDM_PCFG_PGARIGHT_P390DB = 30, /*!< P390DB : 39.0 db gain. value. */ + PDM_PCFG_PGARIGHT_P375DB = 29, /*!< P375DB : 37.5 db gain. value. */ + PDM_PCFG_PGARIGHT_P360DB = 28, /*!< P360DB : 36.0 db gain. value. */ + PDM_PCFG_PGARIGHT_P345DB = 27, /*!< P345DB : 34.5 db gain. value. */ + PDM_PCFG_PGARIGHT_P330DB = 26, /*!< P330DB : 33.0 db gain. value. */ + PDM_PCFG_PGARIGHT_P315DB = 25, /*!< P315DB : 31.5 db gain. value. */ + PDM_PCFG_PGARIGHT_P300DB = 24, /*!< P300DB : 30.0 db gain. value. */ + PDM_PCFG_PGARIGHT_P285DB = 23, /*!< P285DB : 28.5 db gain. value. */ + PDM_PCFG_PGARIGHT_P270DB = 22, /*!< P270DB : 27.0 db gain. value. */ + PDM_PCFG_PGARIGHT_P255DB = 21, /*!< P255DB : 25.5 db gain. value. */ + PDM_PCFG_PGARIGHT_P240DB = 20, /*!< P240DB : 24.0 db gain. value. */ + PDM_PCFG_PGARIGHT_P225DB = 19, /*!< P225DB : 22.5 db gain. value. */ + PDM_PCFG_PGARIGHT_P210DB = 18, /*!< P210DB : 21.0 db gain. value. */ + PDM_PCFG_PGARIGHT_P195DB = 17, /*!< P195DB : 19.5 db gain. value. */ + PDM_PCFG_PGARIGHT_P180DB = 16, /*!< P180DB : 18.0 db gain. value. */ + PDM_PCFG_PGARIGHT_P165DB = 15, /*!< P165DB : 16.5 db gain. value. */ + PDM_PCFG_PGARIGHT_P150DB = 14, /*!< P150DB : 15.0 db gain. value. */ + PDM_PCFG_PGARIGHT_P135DB = 13, /*!< P135DB : 13.5 db gain. value. */ + PDM_PCFG_PGARIGHT_P120DB = 12, /*!< P120DB : 12.0 db gain. value. */ + PDM_PCFG_PGARIGHT_P105DB = 11, /*!< P105DB : 10.5 db gain. value. */ + PDM_PCFG_PGARIGHT_P90DB = 10, /*!< P90DB : 9.0 db gain. value. */ + PDM_PCFG_PGARIGHT_P75DB = 9, /*!< P75DB : 7.5 db gain. value. */ + PDM_PCFG_PGARIGHT_P60DB = 8, /*!< P60DB : 6.0 db gain. value. */ + PDM_PCFG_PGARIGHT_P45DB = 7, /*!< P45DB : 4.5 db gain. value. */ + PDM_PCFG_PGARIGHT_P30DB = 6, /*!< P30DB : 3.0 db gain. value. */ + PDM_PCFG_PGARIGHT_P15DB = 5, /*!< P15DB : 1.5 db gain. value. */ + PDM_PCFG_PGARIGHT_0DB = 4, /*!< 0DB : 0.0 db gain. value. */ + PDM_PCFG_PGARIGHT_M15DB = 3, /*!< M15DB : -1.5 db gain. value. */ + PDM_PCFG_PGARIGHT_M300DB = 2, /*!< M300DB : -3.0 db gain. value. */ + PDM_PCFG_PGARIGHT_M45DB = 1, /*!< M45DB : -4.5 db gain. value. */ + PDM_PCFG_PGARIGHT_M60DB = 0, /*!< M60DB : -6.0 db gain. value. */ +} PDM_PCFG_PGARIGHT_Enum; + +/* =============================================== PDM PCFG PGALEFT [21..25] =============================================== */ +typedef enum { /*!< PDM_PCFG_PGALEFT */ + PDM_PCFG_PGALEFT_P405DB = 31, /*!< P405DB : 40.5 db gain. value. */ + PDM_PCFG_PGALEFT_P390DB = 30, /*!< P390DB : 39.0 db gain. value. */ + PDM_PCFG_PGALEFT_P375DB = 29, /*!< P375DB : 37.5 db gain. value. */ + PDM_PCFG_PGALEFT_P360DB = 28, /*!< P360DB : 36.0 db gain. value. */ + PDM_PCFG_PGALEFT_P345DB = 27, /*!< P345DB : 34.5 db gain. value. */ + PDM_PCFG_PGALEFT_P330DB = 26, /*!< P330DB : 33.0 db gain. value. */ + PDM_PCFG_PGALEFT_P315DB = 25, /*!< P315DB : 31.5 db gain. value. */ + PDM_PCFG_PGALEFT_P300DB = 24, /*!< P300DB : 30.0 db gain. value. */ + PDM_PCFG_PGALEFT_P285DB = 23, /*!< P285DB : 28.5 db gain. value. */ + PDM_PCFG_PGALEFT_P270DB = 22, /*!< P270DB : 27.0 db gain. value. */ + PDM_PCFG_PGALEFT_P255DB = 21, /*!< P255DB : 25.5 db gain. value. */ + PDM_PCFG_PGALEFT_P240DB = 20, /*!< P240DB : 24.0 db gain. value. */ + PDM_PCFG_PGALEFT_P225DB = 19, /*!< P225DB : 22.5 db gain. value. */ + PDM_PCFG_PGALEFT_P210DB = 18, /*!< P210DB : 21.0 db gain. value. */ + PDM_PCFG_PGALEFT_P195DB = 17, /*!< P195DB : 19.5 db gain. value. */ + PDM_PCFG_PGALEFT_P180DB = 16, /*!< P180DB : 18.0 db gain. value. */ + PDM_PCFG_PGALEFT_P165DB = 15, /*!< P165DB : 16.5 db gain. value. */ + PDM_PCFG_PGALEFT_P150DB = 14, /*!< P150DB : 15.0 db gain. value. */ + PDM_PCFG_PGALEFT_P135DB = 13, /*!< P135DB : 13.5 db gain. value. */ + PDM_PCFG_PGALEFT_P120DB = 12, /*!< P120DB : 12.0 db gain. value. */ + PDM_PCFG_PGALEFT_P105DB = 11, /*!< P105DB : 10.5 db gain. value. */ + PDM_PCFG_PGALEFT_P90DB = 10, /*!< P90DB : 9.0 db gain. value. */ + PDM_PCFG_PGALEFT_P75DB = 9, /*!< P75DB : 7.5 db gain. value. */ + PDM_PCFG_PGALEFT_P60DB = 8, /*!< P60DB : 6.0 db gain. value. */ + PDM_PCFG_PGALEFT_P45DB = 7, /*!< P45DB : 4.5 db gain. value. */ + PDM_PCFG_PGALEFT_P30DB = 6, /*!< P30DB : 3.0 db gain. value. */ + PDM_PCFG_PGALEFT_P15DB = 5, /*!< P15DB : 1.5 db gain. value. */ + PDM_PCFG_PGALEFT_0DB = 4, /*!< 0DB : 0.0 db gain. value. */ + PDM_PCFG_PGALEFT_M15DB = 3, /*!< M15DB : -1.5 db gain. value. */ + PDM_PCFG_PGALEFT_M300DB = 2, /*!< M300DB : -3.0 db gain. value. */ + PDM_PCFG_PGALEFT_M45DB = 1, /*!< M45DB : -4.5 db gain. value. */ + PDM_PCFG_PGALEFT_M60DB = 0, /*!< M60DB : -6.0 db gain. value. */ +} PDM_PCFG_PGALEFT_Enum; + +/* =============================================== PDM PCFG MCLKDIV [17..18] =============================================== */ +typedef enum { /*!< PDM_PCFG_MCLKDIV */ + PDM_PCFG_MCLKDIV_MCKDIV4 = 3, /*!< MCKDIV4 : Divide input clock by 4 value. */ + PDM_PCFG_MCLKDIV_MCKDIV3 = 2, /*!< MCKDIV3 : Divide input clock by 3 value. */ + PDM_PCFG_MCLKDIV_MCKDIV2 = 1, /*!< MCKDIV2 : Divide input clock by 2 value. */ + PDM_PCFG_MCLKDIV_MCKDIV1 = 0, /*!< MCKDIV1 : Divide input clock by 1 value. */ +} PDM_PCFG_MCLKDIV_Enum; + +/* ================================================ PDM PCFG ADCHPD [9..9] ================================================= */ +typedef enum { /*!< PDM_PCFG_ADCHPD */ + PDM_PCFG_ADCHPD_EN = 1, /*!< EN : Enable high pass filter. value. */ + PDM_PCFG_ADCHPD_DIS = 0, /*!< DIS : Disable high pass filter. value. */ +} PDM_PCFG_ADCHPD_Enum; + +/* =============================================== PDM PCFG SOFTMUTE [1..1] ================================================ */ +typedef enum { /*!< PDM_PCFG_SOFTMUTE */ + PDM_PCFG_SOFTMUTE_EN = 1, /*!< EN : Enable Soft Mute. value. */ + PDM_PCFG_SOFTMUTE_DIS = 0, /*!< DIS : Disable Soft Mute. value. */ +} PDM_PCFG_SOFTMUTE_Enum; + +/* =============================================== PDM PCFG PDMCOREEN [0..0] =============================================== */ +typedef enum { /*!< PDM_PCFG_PDMCOREEN */ + PDM_PCFG_PDMCOREEN_EN = 1, /*!< EN : Enable Data Streaming. value. */ + PDM_PCFG_PDMCOREEN_DIS = 0, /*!< DIS : Disable Data Streaming. value. */ +} PDM_PCFG_PDMCOREEN_Enum; + +/* ========================================================= VCFG ========================================================== */ +/* =============================================== PDM VCFG IOCLKEN [31..31] =============================================== */ +typedef enum { /*!< PDM_VCFG_IOCLKEN */ + PDM_VCFG_IOCLKEN_DIS = 0, /*!< DIS : Disable FIFO read. value. */ + PDM_VCFG_IOCLKEN_EN = 1, /*!< EN : Enable FIFO read. value. */ +} PDM_VCFG_IOCLKEN_Enum; + +/* ================================================ PDM VCFG RSTB [30..30] ================================================= */ +typedef enum { /*!< PDM_VCFG_RSTB */ + PDM_VCFG_RSTB_RESET = 0, /*!< RESET : Reset the core. value. */ + PDM_VCFG_RSTB_NORM = 1, /*!< NORM : Enable the core. value. */ +} PDM_VCFG_RSTB_Enum; + +/* ============================================== PDM VCFG PDMCLKSEL [27..29] ============================================== */ +typedef enum { /*!< PDM_VCFG_PDMCLKSEL */ + PDM_VCFG_PDMCLKSEL_DISABLE = 0, /*!< DISABLE : Static value. value. */ + PDM_VCFG_PDMCLKSEL_12MHz = 1, /*!< 12MHz : PDM clock is 12 MHz. value. */ + PDM_VCFG_PDMCLKSEL_6MHz = 2, /*!< 6MHz : PDM clock is 6 MHz. value. */ + PDM_VCFG_PDMCLKSEL_3MHz = 3, /*!< 3MHz : PDM clock is 3 MHz. value. */ + PDM_VCFG_PDMCLKSEL_1_5MHz = 4, /*!< 1_5MHz : PDM clock is 1.5 MHz. value. */ + PDM_VCFG_PDMCLKSEL_750KHz = 5, /*!< 750KHz : PDM clock is 750 KHz. value. */ + PDM_VCFG_PDMCLKSEL_375KHz = 6, /*!< 375KHz : PDM clock is 375 KHz. value. */ + PDM_VCFG_PDMCLKSEL_187KHz = 7, /*!< 187KHz : PDM clock is 187.5 KHz. value. */ +} PDM_VCFG_PDMCLKSEL_Enum; + +/* ============================================== PDM VCFG PDMCLKEN [26..26] =============================================== */ +typedef enum { /*!< PDM_VCFG_PDMCLKEN */ + PDM_VCFG_PDMCLKEN_DIS = 0, /*!< DIS : Disable serial clock. value. */ + PDM_VCFG_PDMCLKEN_EN = 1, /*!< EN : Enable serial clock. value. */ +} PDM_VCFG_PDMCLKEN_Enum; + +/* ================================================ PDM VCFG I2SEN [20..20] ================================================ */ +typedef enum { /*!< PDM_VCFG_I2SEN */ + PDM_VCFG_I2SEN_DIS = 0, /*!< DIS : Disable I2S interface. value. */ + PDM_VCFG_I2SEN_EN = 1, /*!< EN : Enable I2S interface. value. */ +} PDM_VCFG_I2SEN_Enum; + +/* =============================================== PDM VCFG BCLKINV [19..19] =============================================== */ +typedef enum { /*!< PDM_VCFG_BCLKINV */ + PDM_VCFG_BCLKINV_INV = 0, /*!< INV : BCLK inverted. value. */ + PDM_VCFG_BCLKINV_NORM = 1, /*!< NORM : BCLK not inverted. value. */ +} PDM_VCFG_BCLKINV_Enum; + +/* ============================================== PDM VCFG DMICKDEL [17..17] =============================================== */ +typedef enum { /*!< PDM_VCFG_DMICKDEL */ + PDM_VCFG_DMICKDEL_0CYC = 0, /*!< 0CYC : No delay. value. */ + PDM_VCFG_DMICKDEL_1CYC = 1, /*!< 1CYC : 1 cycle delay. value. */ +} PDM_VCFG_DMICKDEL_Enum; + +/* ================================================ PDM VCFG SELAP [16..16] ================================================ */ +typedef enum { /*!< PDM_VCFG_SELAP */ + PDM_VCFG_SELAP_I2S = 1, /*!< I2S : Clock source from I2S BCLK. value. */ + PDM_VCFG_SELAP_INTERNAL = 0, /*!< INTERNAL : Clock source from internal clock generator. value. */ +} PDM_VCFG_SELAP_Enum; + +/* ================================================ PDM VCFG PCMPACK [8..8] ================================================ */ +typedef enum { /*!< PDM_VCFG_PCMPACK */ + PDM_VCFG_PCMPACK_DIS = 0, /*!< DIS : Disable PCM packing. value. */ + PDM_VCFG_PCMPACK_EN = 1, /*!< EN : Enable PCM packing. value. */ +} PDM_VCFG_PCMPACK_Enum; + +/* ================================================= PDM VCFG CHSET [3..4] ================================================= */ +typedef enum { /*!< PDM_VCFG_CHSET */ + PDM_VCFG_CHSET_DIS = 0, /*!< DIS : Channel disabled. value. */ + PDM_VCFG_CHSET_LEFT = 1, /*!< LEFT : Mono left channel. value. */ + PDM_VCFG_CHSET_RIGHT = 2, /*!< RIGHT : Mono right channel. value. */ + PDM_VCFG_CHSET_STEREO = 3, /*!< STEREO : Stereo channels. value. */ +} PDM_VCFG_CHSET_Enum; + +/* ======================================================= VOICESTAT ======================================================= */ +/* ======================================================= FIFOREAD ======================================================== */ +/* ======================================================= FIFOFLUSH ======================================================= */ +/* ======================================================== FIFOTHR ======================================================== */ +/* ========================================================= INTEN ========================================================= */ +/* ======================================================== INTSTAT ======================================================== */ +/* ======================================================== INTCLR ========================================================= */ +/* ======================================================== INTSET ========================================================= */ +/* ======================================================= DMATRIGEN ======================================================= */ +/* ====================================================== DMATRIGSTAT ====================================================== */ +/* ======================================================== DMACFG ========================================================= */ +/* =============================================== PDM DMACFG DMAPRI [8..8] ================================================ */ +typedef enum { /*!< PDM_DMACFG_DMAPRI */ + PDM_DMACFG_DMAPRI_LOW = 0, /*!< LOW : Low Priority (service as best effort) value. */ + PDM_DMACFG_DMAPRI_HIGH = 1, /*!< HIGH : High Priority (service immediately) value. */ +} PDM_DMACFG_DMAPRI_Enum; + +/* =============================================== PDM DMACFG DMADIR [2..2] ================================================ */ +typedef enum { /*!< PDM_DMACFG_DMADIR */ + PDM_DMACFG_DMADIR_P2M = 0, /*!< P2M : Peripheral to Memory (SRAM) transaction. THe PDM module + will only DMA to memory. value. */ + PDM_DMACFG_DMADIR_M2P = 1, /*!< M2P : Memory to Peripheral transaction. Not available for PDM + module value. */ +} PDM_DMACFG_DMADIR_Enum; + +/* ================================================ PDM DMACFG DMAEN [0..0] ================================================ */ +typedef enum { /*!< PDM_DMACFG_DMAEN */ + PDM_DMACFG_DMAEN_DIS = 0, /*!< DIS : Disable DMA Function value. */ + PDM_DMACFG_DMAEN_EN = 1, /*!< EN : Enable DMA Function value. */ +} PDM_DMACFG_DMAEN_Enum; + +/* ====================================================== DMATOTCOUNT ====================================================== */ +/* ====================================================== DMATARGADDR ====================================================== */ +/* ======================================================== DMASTAT ======================================================== */ + + +/* =========================================================================================================================== */ +/* ================ PWRCTRL ================ */ +/* =========================================================================================================================== */ + +/* ======================================================= SUPPLYSRC ======================================================= */ +/* ========================================== PWRCTRL SUPPLYSRC BLEBUCKEN [0..0] =========================================== */ +typedef enum { /*!< PWRCTRL_SUPPLYSRC_BLEBUCKEN */ + PWRCTRL_SUPPLYSRC_BLEBUCKEN_EN = 1, /*!< EN : Enable the BLE Buck. value. */ + PWRCTRL_SUPPLYSRC_BLEBUCKEN_DIS = 0, /*!< DIS : Disable the BLE Buck. value. */ +} PWRCTRL_SUPPLYSRC_BLEBUCKEN_Enum; + +/* ===================================================== SUPPLYSTATUS ====================================================== */ +/* ========================================= PWRCTRL SUPPLYSTATUS BLEBUCKON [1..1] ========================================= */ +typedef enum { /*!< PWRCTRL_SUPPLYSTATUS_BLEBUCKON */ + PWRCTRL_SUPPLYSTATUS_BLEBUCKON_LDO = 0, /*!< LDO : Indicates the the LDO is supplying the BLE/Burst power + domain value. */ + PWRCTRL_SUPPLYSTATUS_BLEBUCKON_BUCK = 1, /*!< BUCK : Indicates the the Buck is supplying the BLE/Burst power + domain value. */ +} PWRCTRL_SUPPLYSTATUS_BLEBUCKON_Enum; + +/* ======================================== PWRCTRL SUPPLYSTATUS SIMOBUCKON [0..0] ========================================= */ +typedef enum { /*!< PWRCTRL_SUPPLYSTATUS_SIMOBUCKON */ + PWRCTRL_SUPPLYSTATUS_SIMOBUCKON_OFF = 0, /*!< OFF : Indicates the the SIMO Buck is OFF. value. */ + PWRCTRL_SUPPLYSTATUS_SIMOBUCKON_ON = 1, /*!< ON : Indicates the the SIMO Buck is ON. value. */ +} PWRCTRL_SUPPLYSTATUS_SIMOBUCKON_Enum; + +/* ======================================================= DEVPWREN ======================================================== */ +/* =========================================== PWRCTRL DEVPWREN PWRBLEL [13..13] =========================================== */ +typedef enum { /*!< PWRCTRL_DEVPWREN_PWRBLEL */ + PWRCTRL_DEVPWREN_PWRBLEL_EN = 1, /*!< EN : Power up BLE controller value. */ + PWRCTRL_DEVPWREN_PWRBLEL_DIS = 0, /*!< DIS : Power down BLE controller value. */ +} PWRCTRL_DEVPWREN_PWRBLEL_Enum; + +/* =========================================== PWRCTRL DEVPWREN PWRPDM [12..12] ============================================ */ +typedef enum { /*!< PWRCTRL_DEVPWREN_PWRPDM */ + PWRCTRL_DEVPWREN_PWRPDM_EN = 1, /*!< EN : Power up PDM value. */ + PWRCTRL_DEVPWREN_PWRPDM_DIS = 0, /*!< DIS : Power down PDM value. */ +} PWRCTRL_DEVPWREN_PWRPDM_Enum; + +/* =========================================== PWRCTRL DEVPWREN PWRMSPI [11..11] =========================================== */ +typedef enum { /*!< PWRCTRL_DEVPWREN_PWRMSPI */ + PWRCTRL_DEVPWREN_PWRMSPI_EN = 1, /*!< EN : Power up MSPI value. */ + PWRCTRL_DEVPWREN_PWRMSPI_DIS = 0, /*!< DIS : Power down MSPI value. */ +} PWRCTRL_DEVPWREN_PWRMSPI_Enum; + +/* ========================================== PWRCTRL DEVPWREN PWRSCARD [10..10] =========================================== */ +typedef enum { /*!< PWRCTRL_DEVPWREN_PWRSCARD */ + PWRCTRL_DEVPWREN_PWRSCARD_EN = 1, /*!< EN : Power up SCARD value. */ + PWRCTRL_DEVPWREN_PWRSCARD_DIS = 0, /*!< DIS : Power down SCARD value. */ +} PWRCTRL_DEVPWREN_PWRSCARD_Enum; + +/* ============================================ PWRCTRL DEVPWREN PWRADC [9..9] ============================================= */ +typedef enum { /*!< PWRCTRL_DEVPWREN_PWRADC */ + PWRCTRL_DEVPWREN_PWRADC_EN = 1, /*!< EN : Power up ADC value. */ + PWRCTRL_DEVPWREN_PWRADC_DIS = 0, /*!< DIS : Power Down ADC value. */ +} PWRCTRL_DEVPWREN_PWRADC_Enum; + +/* =========================================== PWRCTRL DEVPWREN PWRUART1 [8..8] ============================================ */ +typedef enum { /*!< PWRCTRL_DEVPWREN_PWRUART1 */ + PWRCTRL_DEVPWREN_PWRUART1_EN = 1, /*!< EN : Power up UART 1 value. */ + PWRCTRL_DEVPWREN_PWRUART1_DIS = 0, /*!< DIS : Power down UART 1 value. */ +} PWRCTRL_DEVPWREN_PWRUART1_Enum; + +/* =========================================== PWRCTRL DEVPWREN PWRUART0 [7..7] ============================================ */ +typedef enum { /*!< PWRCTRL_DEVPWREN_PWRUART0 */ + PWRCTRL_DEVPWREN_PWRUART0_EN = 1, /*!< EN : Power up UART 0 value. */ + PWRCTRL_DEVPWREN_PWRUART0_DIS = 0, /*!< DIS : Power down UART 0 value. */ +} PWRCTRL_DEVPWREN_PWRUART0_Enum; + +/* ============================================ PWRCTRL DEVPWREN PWRIOM5 [6..6] ============================================ */ +typedef enum { /*!< PWRCTRL_DEVPWREN_PWRIOM5 */ + PWRCTRL_DEVPWREN_PWRIOM5_EN = 1, /*!< EN : Power up IO Master 5 value. */ + PWRCTRL_DEVPWREN_PWRIOM5_DIS = 0, /*!< DIS : Power down IO Master 5 value. */ +} PWRCTRL_DEVPWREN_PWRIOM5_Enum; + +/* ============================================ PWRCTRL DEVPWREN PWRIOM4 [5..5] ============================================ */ +typedef enum { /*!< PWRCTRL_DEVPWREN_PWRIOM4 */ + PWRCTRL_DEVPWREN_PWRIOM4_EN = 1, /*!< EN : Power up IO Master 4 value. */ + PWRCTRL_DEVPWREN_PWRIOM4_DIS = 0, /*!< DIS : Power down IO Master 4 value. */ +} PWRCTRL_DEVPWREN_PWRIOM4_Enum; + +/* ============================================ PWRCTRL DEVPWREN PWRIOM3 [4..4] ============================================ */ +typedef enum { /*!< PWRCTRL_DEVPWREN_PWRIOM3 */ + PWRCTRL_DEVPWREN_PWRIOM3_EN = 1, /*!< EN : Power up IO Master 3 value. */ + PWRCTRL_DEVPWREN_PWRIOM3_DIS = 0, /*!< DIS : Power down IO Master 3 value. */ +} PWRCTRL_DEVPWREN_PWRIOM3_Enum; + +/* ============================================ PWRCTRL DEVPWREN PWRIOM2 [3..3] ============================================ */ +typedef enum { /*!< PWRCTRL_DEVPWREN_PWRIOM2 */ + PWRCTRL_DEVPWREN_PWRIOM2_EN = 1, /*!< EN : Power up IO Master 2 value. */ + PWRCTRL_DEVPWREN_PWRIOM2_DIS = 0, /*!< DIS : Power down IO Master 2 value. */ +} PWRCTRL_DEVPWREN_PWRIOM2_Enum; + +/* ============================================ PWRCTRL DEVPWREN PWRIOM1 [2..2] ============================================ */ +typedef enum { /*!< PWRCTRL_DEVPWREN_PWRIOM1 */ + PWRCTRL_DEVPWREN_PWRIOM1_EN = 1, /*!< EN : Power up IO Master 1 value. */ + PWRCTRL_DEVPWREN_PWRIOM1_DIS = 0, /*!< DIS : Power down IO Master 1 value. */ +} PWRCTRL_DEVPWREN_PWRIOM1_Enum; + +/* ============================================ PWRCTRL DEVPWREN PWRIOM0 [1..1] ============================================ */ +typedef enum { /*!< PWRCTRL_DEVPWREN_PWRIOM0 */ + PWRCTRL_DEVPWREN_PWRIOM0_EN = 1, /*!< EN : Power up IO Master 0 value. */ + PWRCTRL_DEVPWREN_PWRIOM0_DIS = 0, /*!< DIS : Power down IO Master 0 value. */ +} PWRCTRL_DEVPWREN_PWRIOM0_Enum; + +/* ============================================ PWRCTRL DEVPWREN PWRIOS [0..0] ============================================= */ +typedef enum { /*!< PWRCTRL_DEVPWREN_PWRIOS */ + PWRCTRL_DEVPWREN_PWRIOS_EN = 1, /*!< EN : Power up IO slave value. */ + PWRCTRL_DEVPWREN_PWRIOS_DIS = 0, /*!< DIS : Power down IO slave value. */ +} PWRCTRL_DEVPWREN_PWRIOS_Enum; + +/* ===================================================== MEMPWDINSLEEP ===================================================== */ +/* ====================================== PWRCTRL MEMPWDINSLEEP CACHEPWDSLP [31..31] ======================================= */ +typedef enum { /*!< PWRCTRL_MEMPWDINSLEEP_CACHEPWDSLP */ + PWRCTRL_MEMPWDINSLEEP_CACHEPWDSLP_EN = 1, /*!< EN : Power down cache in deep sleep value. */ + PWRCTRL_MEMPWDINSLEEP_CACHEPWDSLP_DIS = 0, /*!< DIS : Retain cache in deep sleep value. */ +} PWRCTRL_MEMPWDINSLEEP_CACHEPWDSLP_Enum; + +/* ====================================== PWRCTRL MEMPWDINSLEEP FLASH1PWDSLP [14..14] ====================================== */ +typedef enum { /*!< PWRCTRL_MEMPWDINSLEEP_FLASH1PWDSLP */ + PWRCTRL_MEMPWDINSLEEP_FLASH1PWDSLP_EN = 1, /*!< EN : Flash1 is powered down during deepsleep value. */ + PWRCTRL_MEMPWDINSLEEP_FLASH1PWDSLP_DIS = 0, /*!< DIS : Flash1 is kept powered on during deepsleep value. */ +} PWRCTRL_MEMPWDINSLEEP_FLASH1PWDSLP_Enum; + +/* ====================================== PWRCTRL MEMPWDINSLEEP FLASH0PWDSLP [13..13] ====================================== */ +typedef enum { /*!< PWRCTRL_MEMPWDINSLEEP_FLASH0PWDSLP */ + PWRCTRL_MEMPWDINSLEEP_FLASH0PWDSLP_EN = 1, /*!< EN : Flash0 is powered down during deepsleep value. */ + PWRCTRL_MEMPWDINSLEEP_FLASH0PWDSLP_DIS = 0, /*!< DIS : Flash0 is kept powered on during deepsleep value. */ +} PWRCTRL_MEMPWDINSLEEP_FLASH0PWDSLP_Enum; + +/* ======================================= PWRCTRL MEMPWDINSLEEP SRAMPWDSLP [3..12] ======================================== */ +typedef enum { /*!< PWRCTRL_MEMPWDINSLEEP_SRAMPWDSLP */ + PWRCTRL_MEMPWDINSLEEP_SRAMPWDSLP_NONE = 0, /*!< NONE : All banks retained value. */ + PWRCTRL_MEMPWDINSLEEP_SRAMPWDSLP_GROUP0 = 1, /*!< GROUP0 : SRAM GROUP0 powered down (64KB-96KB) value. */ + PWRCTRL_MEMPWDINSLEEP_SRAMPWDSLP_GROUP1 = 2, /*!< GROUP1 : SRAM GROUP1 powered down (96KB-128KB) value. */ + PWRCTRL_MEMPWDINSLEEP_SRAMPWDSLP_GROUP2 = 4, /*!< GROUP2 : SRAM GROUP2 powered down (128KB-160KB) value. */ + PWRCTRL_MEMPWDINSLEEP_SRAMPWDSLP_GROUP3 = 8, /*!< GROUP3 : SRAM GROUP3 powered down (160KB-192KB) value. */ + PWRCTRL_MEMPWDINSLEEP_SRAMPWDSLP_GROUP4 = 16, /*!< GROUP4 : SRAM GROUP4 powered down (192KB-224KB) value. */ + PWRCTRL_MEMPWDINSLEEP_SRAMPWDSLP_GROUP5 = 32, /*!< GROUP5 : SRAM GROUP5 powered down (224KB-256KB) value. */ + PWRCTRL_MEMPWDINSLEEP_SRAMPWDSLP_GROUP6 = 64, /*!< GROUP6 : SRAM GROUP6 powered down (256KB-288KB) value. */ + PWRCTRL_MEMPWDINSLEEP_SRAMPWDSLP_GROUP7 = 128,/*!< GROUP7 : SRAM GROUP7 powered down (288KB-320KB) value. */ + PWRCTRL_MEMPWDINSLEEP_SRAMPWDSLP_GROUP8 = 256,/*!< GROUP8 : SRAM GROUP8 powered down (320KB-352KB) value. */ + PWRCTRL_MEMPWDINSLEEP_SRAMPWDSLP_GROUP9 = 512,/*!< GROUP9 : SRAM GROUP9 powered down (352KB-384KB) value. */ + PWRCTRL_MEMPWDINSLEEP_SRAMPWDSLP_SRAM64K = 3, /*!< SRAM64K : Powerdown lower 64k SRAM (64KB-128KB) value. */ + PWRCTRL_MEMPWDINSLEEP_SRAMPWDSLP_SRAM128K = 15,/*!< SRAM128K : Powerdown lower 128k SRAM (64KB-192KB) value. */ + PWRCTRL_MEMPWDINSLEEP_SRAMPWDSLP_ALLBUTLOWER32K = 1022,/*!< ALLBUTLOWER32K : All SRAM banks but lower 32k powered down (96KB-384KB). + value. */ + PWRCTRL_MEMPWDINSLEEP_SRAMPWDSLP_ALLBUTLOWER64K = 1020,/*!< ALLBUTLOWER64K : All banks but lower 64k powered down. value. */ + PWRCTRL_MEMPWDINSLEEP_SRAMPWDSLP_ALLBUTLOWER128K = 1008,/*!< ALLBUTLOWER128K : All banks but lower 128k powered down. value. */ + PWRCTRL_MEMPWDINSLEEP_SRAMPWDSLP_ALL = 1023, /*!< ALL : All banks powered down. value. */ +} PWRCTRL_MEMPWDINSLEEP_SRAMPWDSLP_Enum; + +/* ======================================== PWRCTRL MEMPWDINSLEEP DTCMPWDSLP [0..2] ======================================== */ +typedef enum { /*!< PWRCTRL_MEMPWDINSLEEP_DTCMPWDSLP */ + PWRCTRL_MEMPWDINSLEEP_DTCMPWDSLP_NONE = 0, /*!< NONE : All DTCM retained value. */ + PWRCTRL_MEMPWDINSLEEP_DTCMPWDSLP_GROUP0DTCM0 = 1,/*!< GROUP0DTCM0 : Group0_DTCM0 powered down in deep sleep (0KB-8KB) + value. */ + PWRCTRL_MEMPWDINSLEEP_DTCMPWDSLP_GROUP0DTCM1 = 2,/*!< GROUP0DTCM1 : Group0_DTCM1 powered down in deep sleep (8KB-32KB) + value. */ + PWRCTRL_MEMPWDINSLEEP_DTCMPWDSLP_GROUP0 = 3, /*!< GROUP0 : Both DTCMs in group0 are powered down in deep sleep + (0KB-32KB) value. */ + PWRCTRL_MEMPWDINSLEEP_DTCMPWDSLP_ALLBUTGROUP0DTCM0 = 6,/*!< ALLBUTGROUP0DTCM0 : Group1 and Group0_DTCM1 are powered down + in deep sleep (8KB-64KB) value. */ + PWRCTRL_MEMPWDINSLEEP_DTCMPWDSLP_GROUP1 = 4, /*!< GROUP1 : Group1 DTCM powered down in deep sleep (32KB-64KB) + value. */ + PWRCTRL_MEMPWDINSLEEP_DTCMPWDSLP_ALL = 7, /*!< ALL : All DTCMs powered down in deep sleep (0KB-64KB) value. */ +} PWRCTRL_MEMPWDINSLEEP_DTCMPWDSLP_Enum; + +/* ======================================================= MEMPWREN ======================================================== */ +/* =========================================== PWRCTRL MEMPWREN CACHEB2 [31..31] =========================================== */ +typedef enum { /*!< PWRCTRL_MEMPWREN_CACHEB2 */ + PWRCTRL_MEMPWREN_CACHEB2_EN = 1, /*!< EN : Power up Cache Bank 2 value. */ + PWRCTRL_MEMPWREN_CACHEB2_DIS = 0, /*!< DIS : Power down Cache Bank 2 value. */ +} PWRCTRL_MEMPWREN_CACHEB2_Enum; + +/* =========================================== PWRCTRL MEMPWREN CACHEB0 [30..30] =========================================== */ +typedef enum { /*!< PWRCTRL_MEMPWREN_CACHEB0 */ + PWRCTRL_MEMPWREN_CACHEB0_EN = 1, /*!< EN : Power up Cache Bank 0 value. */ + PWRCTRL_MEMPWREN_CACHEB0_DIS = 0, /*!< DIS : Power down Cache Bank 0 value. */ +} PWRCTRL_MEMPWREN_CACHEB0_Enum; + +/* =========================================== PWRCTRL MEMPWREN FLASH1 [14..14] ============================================ */ +typedef enum { /*!< PWRCTRL_MEMPWREN_FLASH1 */ + PWRCTRL_MEMPWREN_FLASH1_EN = 1, /*!< EN : Power up Flash1 value. */ + PWRCTRL_MEMPWREN_FLASH1_DIS = 0, /*!< DIS : Power down Flash1 value. */ +} PWRCTRL_MEMPWREN_FLASH1_Enum; + +/* =========================================== PWRCTRL MEMPWREN FLASH0 [13..13] ============================================ */ +typedef enum { /*!< PWRCTRL_MEMPWREN_FLASH0 */ + PWRCTRL_MEMPWREN_FLASH0_EN = 1, /*!< EN : Power up Flash0 value. */ + PWRCTRL_MEMPWREN_FLASH0_DIS = 0, /*!< DIS : Power down Flash0 value. */ +} PWRCTRL_MEMPWREN_FLASH0_Enum; + +/* ============================================= PWRCTRL MEMPWREN SRAM [3..12] ============================================= */ +typedef enum { /*!< PWRCTRL_MEMPWREN_SRAM */ + PWRCTRL_MEMPWREN_SRAM_NONE = 0, /*!< NONE : Do not power ON any of the SRAM banks value. */ + PWRCTRL_MEMPWREN_SRAM_GROUP0 = 1, /*!< GROUP0 : Power ON only SRAM group0 (0KB-32KB) value. */ + PWRCTRL_MEMPWREN_SRAM_GROUP1 = 2, /*!< GROUP1 : Power ON only SRAM group1 (32KB-64KB) value. */ + PWRCTRL_MEMPWREN_SRAM_GROUP2 = 4, /*!< GROUP2 : Power ON only SRAM group2 (64KB-96KB) value. */ + PWRCTRL_MEMPWREN_SRAM_GROUP3 = 8, /*!< GROUP3 : Power ON only SRAM group3 (96KB-128KB) value. */ + PWRCTRL_MEMPWREN_SRAM_GROUP4 = 16, /*!< GROUP4 : Power ON only SRAM group4 (128KB-160KB) value. */ + PWRCTRL_MEMPWREN_SRAM_GROUP5 = 32, /*!< GROUP5 : Power ON only SRAM group5 (160KB-192KB) value. */ + PWRCTRL_MEMPWREN_SRAM_GROUP6 = 64, /*!< GROUP6 : Power ON only SRAM group6 (192KB-224KB) value. */ + PWRCTRL_MEMPWREN_SRAM_GROUP7 = 128, /*!< GROUP7 : Power ON only SRAM group7 (224KB-256KB) value. */ + PWRCTRL_MEMPWREN_SRAM_GROUP8 = 256, /*!< GROUP8 : Power ON only SRAM group8 (256KB-288KB) value. */ + PWRCTRL_MEMPWREN_SRAM_GROUP9 = 512, /*!< GROUP9 : Power ON only SRAM group9 (288KB-320KB) value. */ + PWRCTRL_MEMPWREN_SRAM_SRAM64K = 3, /*!< SRAM64K : Power ON only lower 64k value. */ + PWRCTRL_MEMPWREN_SRAM_SRAM128K = 15, /*!< SRAM128K : Power ON only lower 128k value. */ + PWRCTRL_MEMPWREN_SRAM_SRAM256K = 255, /*!< SRAM256K : Power ON only lower 256k value. */ + PWRCTRL_MEMPWREN_SRAM_ALL = 1023, /*!< ALL : All SRAM banks (320K) powered ON value. */ +} PWRCTRL_MEMPWREN_SRAM_Enum; + +/* ============================================= PWRCTRL MEMPWREN DTCM [0..2] ============================================== */ +typedef enum { /*!< PWRCTRL_MEMPWREN_DTCM */ + PWRCTRL_MEMPWREN_DTCM_NONE = 0, /*!< NONE : Do not enable power to any DTCMs value. */ + PWRCTRL_MEMPWREN_DTCM_GROUP0DTCM0 = 1, /*!< GROUP0DTCM0 : Power ON only GROUP0_DTCM0 value. */ + PWRCTRL_MEMPWREN_DTCM_GROUP0DTCM1 = 2, /*!< GROUP0DTCM1 : Power ON only GROUP0_DTCM1 value. */ + PWRCTRL_MEMPWREN_DTCM_GROUP0 = 3, /*!< GROUP0 : Power ON only DTCMs in group0 value. */ + PWRCTRL_MEMPWREN_DTCM_GROUP1 = 4, /*!< GROUP1 : Power ON only DTCMs in group1 value. */ + PWRCTRL_MEMPWREN_DTCM_ALL = 7, /*!< ALL : Power ON all DTCMs value. */ +} PWRCTRL_MEMPWREN_DTCM_Enum; + +/* ===================================================== MEMPWRSTATUS ====================================================== */ +/* ===================================================== DEVPWRSTATUS ====================================================== */ +/* ======================================================= SRAMCTRL ======================================================== */ +/* ======================================== PWRCTRL SRAMCTRL SRAMLIGHTSLEEP [8..19] ======================================== */ +typedef enum { /*!< PWRCTRL_SRAMCTRL_SRAMLIGHTSLEEP */ + PWRCTRL_SRAMCTRL_SRAMLIGHTSLEEP_ALL = 255, /*!< ALL : Enable LIGHT SLEEP for ALL SRAMs value. */ + PWRCTRL_SRAMCTRL_SRAMLIGHTSLEEP_DIS = 0, /*!< DIS : Disables LIGHT SLEEP for ALL SRAMs value. */ +} PWRCTRL_SRAMCTRL_SRAMLIGHTSLEEP_Enum; + +/* ======================================= PWRCTRL SRAMCTRL SRAMMASTERCLKGATE [2..2] ======================================= */ +typedef enum { /*!< PWRCTRL_SRAMCTRL_SRAMMASTERCLKGATE */ + PWRCTRL_SRAMCTRL_SRAMMASTERCLKGATE_EN = 1, /*!< EN : Enable Master SRAM Clock Gate value. */ + PWRCTRL_SRAMCTRL_SRAMMASTERCLKGATE_DIS = 0, /*!< DIS : Disables Master SRAM Clock Gating value. */ +} PWRCTRL_SRAMCTRL_SRAMMASTERCLKGATE_Enum; + +/* ========================================== PWRCTRL SRAMCTRL SRAMCLKGATE [1..1] ========================================== */ +typedef enum { /*!< PWRCTRL_SRAMCTRL_SRAMCLKGATE */ + PWRCTRL_SRAMCTRL_SRAMCLKGATE_EN = 1, /*!< EN : Enable Individual SRAM Clock Gating value. */ + PWRCTRL_SRAMCTRL_SRAMCLKGATE_DIS = 0, /*!< DIS : Disables Individual SRAM Clock Gating value. */ +} PWRCTRL_SRAMCTRL_SRAMCLKGATE_Enum; + +/* ======================================================= ADCSTATUS ======================================================= */ +/* ========================================================= MISC ========================================================== */ +/* ============================================ PWRCTRL MISC MEMVRLPBLE [6..6] ============================================= */ +typedef enum { /*!< PWRCTRL_MISC_MEMVRLPBLE */ + PWRCTRL_MISC_MEMVRLPBLE_EN = 1, /*!< EN : Mem VR can go to lp mode even when BLE is powered on. value. */ + PWRCTRL_MISC_MEMVRLPBLE_DIS = 0, /*!< DIS : Mem VR will stay in active mode when BLE is powered on. + value. */ +} PWRCTRL_MISC_MEMVRLPBLE_Enum; + +/* =========================================== PWRCTRL MISC FORCEMEMVRADC [4..5] =========================================== */ +typedef enum { /*!< PWRCTRL_MISC_FORCEMEMVRADC */ + PWRCTRL_MISC_FORCEMEMVRADC_ACT = 2, /*!< ACT : In this mode if all the other domains but ADC are powered + down, mem VR will stay in ACT mode. value. */ + PWRCTRL_MISC_FORCEMEMVRADC_LP = 1, /*!< LP : In this mode if all the other domains but ADC are powered + down, mem VR will stay in LP mode. value. */ + PWRCTRL_MISC_FORCEMEMVRADC_DIS = 0, /*!< DIS : In this mode if all the other domains but ADC are powered + down, mem VR will duty cycle between active and LP modes + depending on ADC sampling. value. */ +} PWRCTRL_MISC_FORCEMEMVRADC_Enum; + +/* ============================================ PWRCTRL MISC SIMOBUCKEN [0..0] ============================================= */ +typedef enum { /*!< PWRCTRL_MISC_SIMOBUCKEN */ + PWRCTRL_MISC_SIMOBUCKEN_EN = 1, /*!< EN : Enable the SIMO Buck value. */ + PWRCTRL_MISC_SIMOBUCKEN_DIS = 0, /*!< DIS : Disable the SIMO Buck value. */ +} PWRCTRL_MISC_SIMOBUCKEN_Enum; + +/* ===================================================== DEVPWREVENTEN ===================================================== */ +/* ======================================= PWRCTRL DEVPWREVENTEN BURSTEVEN [31..31] ======================================== */ +typedef enum { /*!< PWRCTRL_DEVPWREVENTEN_BURSTEVEN */ + PWRCTRL_DEVPWREVENTEN_BURSTEVEN_EN = 1, /*!< EN : Enable BURST status event value. */ + PWRCTRL_DEVPWREVENTEN_BURSTEVEN_DIS = 0, /*!< DIS : Disable BURST status event value. */ +} PWRCTRL_DEVPWREVENTEN_BURSTEVEN_Enum; + +/* ==================================== PWRCTRL DEVPWREVENTEN BURSTFEATUREEVEN [30..30] ==================================== */ +typedef enum { /*!< PWRCTRL_DEVPWREVENTEN_BURSTFEATUREEVEN */ + PWRCTRL_DEVPWREVENTEN_BURSTFEATUREEVEN_EN = 1,/*!< EN : Enable BURSTFEATURE status event value. */ + PWRCTRL_DEVPWREVENTEN_BURSTFEATUREEVEN_DIS = 0,/*!< DIS : Disable BURSTFEATURE status event value. */ +} PWRCTRL_DEVPWREVENTEN_BURSTFEATUREEVEN_Enum; + +/* ===================================== PWRCTRL DEVPWREVENTEN BLEFEATUREEVEN [29..29] ===================================== */ +typedef enum { /*!< PWRCTRL_DEVPWREVENTEN_BLEFEATUREEVEN */ + PWRCTRL_DEVPWREVENTEN_BLEFEATUREEVEN_EN = 1, /*!< EN : Enable BLEFEATURE status event value. */ + PWRCTRL_DEVPWREVENTEN_BLEFEATUREEVEN_DIS = 0, /*!< DIS : Disable BLEFEATURE status event value. */ +} PWRCTRL_DEVPWREVENTEN_BLEFEATUREEVEN_Enum; + +/* ========================================= PWRCTRL DEVPWREVENTEN BLELEVEN [8..8] ========================================= */ +typedef enum { /*!< PWRCTRL_DEVPWREVENTEN_BLELEVEN */ + PWRCTRL_DEVPWREVENTEN_BLELEVEN_EN = 1, /*!< EN : Enable BLE power-on status event value. */ + PWRCTRL_DEVPWREVENTEN_BLELEVEN_DIS = 0, /*!< DIS : Disable BLE power-on status event value. */ +} PWRCTRL_DEVPWREVENTEN_BLELEVEN_Enum; + +/* ========================================= PWRCTRL DEVPWREVENTEN PDMEVEN [7..7] ========================================== */ +typedef enum { /*!< PWRCTRL_DEVPWREVENTEN_PDMEVEN */ + PWRCTRL_DEVPWREVENTEN_PDMEVEN_EN = 1, /*!< EN : Enable PDM power-on status event value. */ + PWRCTRL_DEVPWREVENTEN_PDMEVEN_DIS = 0, /*!< DIS : Disable PDM power-on status event value. */ +} PWRCTRL_DEVPWREVENTEN_PDMEVEN_Enum; + +/* ========================================= PWRCTRL DEVPWREVENTEN MSPIEVEN [6..6] ========================================= */ +typedef enum { /*!< PWRCTRL_DEVPWREVENTEN_MSPIEVEN */ + PWRCTRL_DEVPWREVENTEN_MSPIEVEN_EN = 1, /*!< EN : Enable MSPI power-on status event value. */ + PWRCTRL_DEVPWREVENTEN_MSPIEVEN_DIS = 0, /*!< DIS : Disable MSPI power-on status event value. */ +} PWRCTRL_DEVPWREVENTEN_MSPIEVEN_Enum; + +/* ========================================= PWRCTRL DEVPWREVENTEN ADCEVEN [5..5] ========================================== */ +typedef enum { /*!< PWRCTRL_DEVPWREVENTEN_ADCEVEN */ + PWRCTRL_DEVPWREVENTEN_ADCEVEN_EN = 1, /*!< EN : Enable ADC power-on status event value. */ + PWRCTRL_DEVPWREVENTEN_ADCEVEN_DIS = 0, /*!< DIS : Disable ADC power-on status event value. */ +} PWRCTRL_DEVPWREVENTEN_ADCEVEN_Enum; + +/* ========================================= PWRCTRL DEVPWREVENTEN HCPCEVEN [4..4] ========================================= */ +typedef enum { /*!< PWRCTRL_DEVPWREVENTEN_HCPCEVEN */ + PWRCTRL_DEVPWREVENTEN_HCPCEVEN_EN = 1, /*!< EN : Enable HCPC power-on status event value. */ + PWRCTRL_DEVPWREVENTEN_HCPCEVEN_DIS = 0, /*!< DIS : Disable HCPC power-on status event value. */ +} PWRCTRL_DEVPWREVENTEN_HCPCEVEN_Enum; + +/* ========================================= PWRCTRL DEVPWREVENTEN HCPBEVEN [3..3] ========================================= */ +typedef enum { /*!< PWRCTRL_DEVPWREVENTEN_HCPBEVEN */ + PWRCTRL_DEVPWREVENTEN_HCPBEVEN_EN = 1, /*!< EN : Enable HCPB power-on status event value. */ + PWRCTRL_DEVPWREVENTEN_HCPBEVEN_DIS = 0, /*!< DIS : Disable HCPB power-on status event value. */ +} PWRCTRL_DEVPWREVENTEN_HCPBEVEN_Enum; + +/* ========================================= PWRCTRL DEVPWREVENTEN HCPAEVEN [2..2] ========================================= */ +typedef enum { /*!< PWRCTRL_DEVPWREVENTEN_HCPAEVEN */ + PWRCTRL_DEVPWREVENTEN_HCPAEVEN_EN = 1, /*!< EN : Enable HCPA power-on status event value. */ + PWRCTRL_DEVPWREVENTEN_HCPAEVEN_DIS = 0, /*!< DIS : Disable HCPA power-on status event value. */ +} PWRCTRL_DEVPWREVENTEN_HCPAEVEN_Enum; + +/* ========================================= PWRCTRL DEVPWREVENTEN MCUHEVEN [1..1] ========================================= */ +typedef enum { /*!< PWRCTRL_DEVPWREVENTEN_MCUHEVEN */ + PWRCTRL_DEVPWREVENTEN_MCUHEVEN_EN = 1, /*!< EN : Enable MCHU power-on status event value. */ + PWRCTRL_DEVPWREVENTEN_MCUHEVEN_DIS = 0, /*!< DIS : Disable MCUH power-on status event value. */ +} PWRCTRL_DEVPWREVENTEN_MCUHEVEN_Enum; + +/* ========================================= PWRCTRL DEVPWREVENTEN MCULEVEN [0..0] ========================================= */ +typedef enum { /*!< PWRCTRL_DEVPWREVENTEN_MCULEVEN */ + PWRCTRL_DEVPWREVENTEN_MCULEVEN_EN = 1, /*!< EN : Enable MCUL power-on status event value. */ + PWRCTRL_DEVPWREVENTEN_MCULEVEN_DIS = 0, /*!< DIS : Disable MCUL power-on status event value. */ +} PWRCTRL_DEVPWREVENTEN_MCULEVEN_Enum; + +/* ===================================================== MEMPWREVENTEN ===================================================== */ +/* ======================================= PWRCTRL MEMPWREVENTEN CACHEB2EN [31..31] ======================================== */ +typedef enum { /*!< PWRCTRL_MEMPWREVENTEN_CACHEB2EN */ + PWRCTRL_MEMPWREVENTEN_CACHEB2EN_EN = 1, /*!< EN : Enable CACHE BANK 2 status event value. */ + PWRCTRL_MEMPWREVENTEN_CACHEB2EN_DIS = 0, /*!< DIS : Disable CACHE BANK 2 status event value. */ +} PWRCTRL_MEMPWREVENTEN_CACHEB2EN_Enum; + +/* ======================================= PWRCTRL MEMPWREVENTEN CACHEB0EN [30..30] ======================================== */ +typedef enum { /*!< PWRCTRL_MEMPWREVENTEN_CACHEB0EN */ + PWRCTRL_MEMPWREVENTEN_CACHEB0EN_EN = 1, /*!< EN : Enable CACHE BANK 0 status event value. */ + PWRCTRL_MEMPWREVENTEN_CACHEB0EN_DIS = 0, /*!< DIS : Disable CACHE BANK 0 status event value. */ +} PWRCTRL_MEMPWREVENTEN_CACHEB0EN_Enum; + +/* ======================================== PWRCTRL MEMPWREVENTEN FLASH1EN [14..14] ======================================== */ +typedef enum { /*!< PWRCTRL_MEMPWREVENTEN_FLASH1EN */ + PWRCTRL_MEMPWREVENTEN_FLASH1EN_EN = 1, /*!< EN : Enable FLASH status event value. */ + PWRCTRL_MEMPWREVENTEN_FLASH1EN_DIS = 0, /*!< DIS : Disables FLASH status event value. */ +} PWRCTRL_MEMPWREVENTEN_FLASH1EN_Enum; + +/* ======================================== PWRCTRL MEMPWREVENTEN FLASH0EN [13..13] ======================================== */ +typedef enum { /*!< PWRCTRL_MEMPWREVENTEN_FLASH0EN */ + PWRCTRL_MEMPWREVENTEN_FLASH0EN_EN = 1, /*!< EN : Enable FLASH status event value. */ + PWRCTRL_MEMPWREVENTEN_FLASH0EN_DIS = 0, /*!< DIS : Disables FLASH status event value. */ +} PWRCTRL_MEMPWREVENTEN_FLASH0EN_Enum; + +/* ========================================= PWRCTRL MEMPWREVENTEN SRAMEN [3..12] ========================================== */ +typedef enum { /*!< PWRCTRL_MEMPWREVENTEN_SRAMEN */ + PWRCTRL_MEMPWREVENTEN_SRAMEN_NONE = 0, /*!< NONE : Disable SRAM power-on status event value. */ + PWRCTRL_MEMPWREVENTEN_SRAMEN_GROUP0EN = 1, /*!< GROUP0EN : Enable SRAM group0 (0KB-32KB) power on status event + value. */ + PWRCTRL_MEMPWREVENTEN_SRAMEN_GROUP1EN = 2, /*!< GROUP1EN : Enable SRAM group1 (32KB-64KB) power on status event + value. */ + PWRCTRL_MEMPWREVENTEN_SRAMEN_GROUP2EN = 4, /*!< GROUP2EN : Enable SRAM group2 (64KB-96KB) power on status event + value. */ + PWRCTRL_MEMPWREVENTEN_SRAMEN_GROUP3EN = 8, /*!< GROUP3EN : Enable SRAM group3 (96KB-128KB) power on status event + value. */ + PWRCTRL_MEMPWREVENTEN_SRAMEN_GROUP4EN = 16, /*!< GROUP4EN : Enable SRAM group4 (128KB-160KB) power on status + event value. */ + PWRCTRL_MEMPWREVENTEN_SRAMEN_GROUP5EN = 32, /*!< GROUP5EN : Enable SRAM group5 (160KB-192KB) power on status + event value. */ + PWRCTRL_MEMPWREVENTEN_SRAMEN_GROUP6EN = 64, /*!< GROUP6EN : Enable SRAM group6 (192KB-224KB) power on status + event value. */ + PWRCTRL_MEMPWREVENTEN_SRAMEN_GROUP7EN = 128, /*!< GROUP7EN : Enable SRAM group7 (224KB-256KB) power on status + event value. */ + PWRCTRL_MEMPWREVENTEN_SRAMEN_GROUP8EN = 256, /*!< GROUP8EN : Enable SRAM group8 (256KB-288KB) power on status + event value. */ + PWRCTRL_MEMPWREVENTEN_SRAMEN_GROUP9EN = 512, /*!< GROUP9EN : Enable SRAM group9 (288KB-320KB) power on status + event value. */ +} PWRCTRL_MEMPWREVENTEN_SRAMEN_Enum; + +/* ========================================== PWRCTRL MEMPWREVENTEN DTCMEN [0..2] ========================================== */ +typedef enum { /*!< PWRCTRL_MEMPWREVENTEN_DTCMEN */ + PWRCTRL_MEMPWREVENTEN_DTCMEN_NONE = 0, /*!< NONE : Do not enable DTCM power-on status event value. */ + PWRCTRL_MEMPWREVENTEN_DTCMEN_GROUP0DTCM0EN = 1,/*!< GROUP0DTCM0EN : Enable GROUP0_DTCM0 power on status event value. */ + PWRCTRL_MEMPWREVENTEN_DTCMEN_GROUP0DTCM1EN = 2,/*!< GROUP0DTCM1EN : Enable GROUP0_DTCM1 power on status event value. */ + PWRCTRL_MEMPWREVENTEN_DTCMEN_GROUP0EN = 3, /*!< GROUP0EN : Enable DTCMs in group0 power on status event value. */ + PWRCTRL_MEMPWREVENTEN_DTCMEN_GROUP1EN = 4, /*!< GROUP1EN : Enable DTCMs in group1 power on status event value. */ + PWRCTRL_MEMPWREVENTEN_DTCMEN_ALL = 7, /*!< ALL : Enable all DTCM power on status event value. */ +} PWRCTRL_MEMPWREVENTEN_DTCMEN_Enum; + + + +/* =========================================================================================================================== */ +/* ================ RSTGEN ================ */ +/* =========================================================================================================================== */ + +/* ========================================================== CFG ========================================================== */ +/* ========================================================= SWPOI ========================================================= */ +/* ============================================= RSTGEN SWPOI SWPOIKEY [0..7] ============================================== */ +typedef enum { /*!< RSTGEN_SWPOI_SWPOIKEY */ + RSTGEN_SWPOI_SWPOIKEY_KEYVALUE = 27, /*!< KEYVALUE : Writing 0x1B key value generates a software POI reset. + value. */ +} RSTGEN_SWPOI_SWPOIKEY_Enum; + +/* ========================================================= SWPOR ========================================================= */ +/* ============================================= RSTGEN SWPOR SWPORKEY [0..7] ============================================== */ +typedef enum { /*!< RSTGEN_SWPOR_SWPORKEY */ + RSTGEN_SWPOR_SWPORKEY_KEYVALUE = 212, /*!< KEYVALUE : Writing 0xD4 key value generates a software POR reset. + value. */ +} RSTGEN_SWPOR_SWPORKEY_Enum; + +/* ======================================================== TPIURST ======================================================== */ +/* ========================================================= INTEN ========================================================= */ +/* ======================================================== INTSTAT ======================================================== */ +/* ======================================================== INTCLR ========================================================= */ +/* ======================================================== INTSET ========================================================= */ +/* ========================================================= STAT ========================================================== */ + + +/* =========================================================================================================================== */ +/* ================ RTC ================ */ +/* =========================================================================================================================== */ + +/* ======================================================== CTRLOW ========================================================= */ +/* ========================================================= CTRUP ========================================================= */ +/* =============================================== RTC CTRUP CTERR [31..31] ================================================ */ +typedef enum { /*!< RTC_CTRUP_CTERR */ + RTC_CTRUP_CTERR_NOERR = 0, /*!< NOERR : No read error occurred value. */ + RTC_CTRUP_CTERR_RDERR = 1, /*!< RDERR : Read error occurred value. */ +} RTC_CTRUP_CTERR_Enum; + +/* ================================================ RTC CTRUP CEB [28..28] ================================================= */ +typedef enum { /*!< RTC_CTRUP_CEB */ + RTC_CTRUP_CEB_DIS = 0, /*!< DIS : Disable the Century bit from changing value. */ + RTC_CTRUP_CEB_EN = 1, /*!< EN : Enable the Century bit to change value. */ +} RTC_CTRUP_CEB_Enum; + +/* ================================================= RTC CTRUP CB [27..27] ================================================= */ +typedef enum { /*!< RTC_CTRUP_CB */ + RTC_CTRUP_CB_2000 = 0, /*!< 2000 : Century is 2000s value. */ + RTC_CTRUP_CB_1900_2100 = 1, /*!< 1900_2100 : Century is 1900s/2100s value. */ +} RTC_CTRUP_CB_Enum; + +/* ======================================================== ALMLOW ========================================================= */ +/* ========================================================= ALMUP ========================================================= */ +/* ======================================================== RTCCTL ========================================================= */ +/* =============================================== RTC RTCCTL HR1224 [5..5] ================================================ */ +typedef enum { /*!< RTC_RTCCTL_HR1224 */ + RTC_RTCCTL_HR1224_24HR = 0, /*!< 24HR : Hours in 24 hour mode value. */ + RTC_RTCCTL_HR1224_12HR = 1, /*!< 12HR : Hours in 12 hour mode value. */ +} RTC_RTCCTL_HR1224_Enum; + +/* ================================================ RTC RTCCTL RSTOP [4..4] ================================================ */ +typedef enum { /*!< RTC_RTCCTL_RSTOP */ + RTC_RTCCTL_RSTOP_RUN = 0, /*!< RUN : Allow the RTC input clock to run value. */ + RTC_RTCCTL_RSTOP_STOP = 1, /*!< STOP : Stop the RTC input clock value. */ +} RTC_RTCCTL_RSTOP_Enum; + +/* ================================================= RTC RTCCTL RPT [1..3] ================================================= */ +typedef enum { /*!< RTC_RTCCTL_RPT */ + RTC_RTCCTL_RPT_DIS = 0, /*!< DIS : Alarm interrupt disabled value. */ + RTC_RTCCTL_RPT_YEAR = 1, /*!< YEAR : Interrupt every year value. */ + RTC_RTCCTL_RPT_MONTH = 2, /*!< MONTH : Interrupt every month value. */ + RTC_RTCCTL_RPT_WEEK = 3, /*!< WEEK : Interrupt every week value. */ + RTC_RTCCTL_RPT_DAY = 4, /*!< DAY : Interrupt every day value. */ + RTC_RTCCTL_RPT_HR = 5, /*!< HR : Interrupt every hour value. */ + RTC_RTCCTL_RPT_MIN = 6, /*!< MIN : Interrupt every minute value. */ + RTC_RTCCTL_RPT_SEC = 7, /*!< SEC : Interrupt every second/10th/100th value. */ +} RTC_RTCCTL_RPT_Enum; + +/* ================================================ RTC RTCCTL WRTC [0..0] ================================================= */ +typedef enum { /*!< RTC_RTCCTL_WRTC */ + RTC_RTCCTL_WRTC_DIS = 0, /*!< DIS : Counter writes are disabled value. */ + RTC_RTCCTL_WRTC_EN = 1, /*!< EN : Counter writes are enabled value. */ +} RTC_RTCCTL_WRTC_Enum; + +/* ========================================================= INTEN ========================================================= */ +/* ======================================================== INTSTAT ======================================================== */ +/* ======================================================== INTCLR ========================================================= */ +/* ======================================================== INTSET ========================================================= */ + + +/* =========================================================================================================================== */ +/* ================ SCARD ================ */ +/* =========================================================================================================================== */ + +/* ========================================================== SR =========================================================== */ +/* ========================================================== DR =========================================================== */ +/* ========================================================== SR1 ========================================================== */ +/* ====================================================== RETXCNTRMI ======================================================= */ +/* ======================================================== CLKCTRL ======================================================== */ + + +/* =========================================================================================================================== */ +/* ================ SECURITY ================ */ +/* =========================================================================================================================== */ + +/* ========================================================= CTRL ========================================================== */ +/* ============================================= SECURITY CTRL FUNCTION [4..7] ============================================= */ +typedef enum { /*!< SECURITY_CTRL_FUNCTION */ + SECURITY_CTRL_FUNCTION_CRC32 = 0, /*!< CRC32 : Perform CRC32 operation value. */ +} SECURITY_CTRL_FUNCTION_Enum; + +/* ======================================================== SRCADDR ======================================================== */ +/* ========================================================== LEN ========================================================== */ +/* ======================================================== RESULT ========================================================= */ +/* ======================================================= LOCKCTRL ======================================================== */ +/* ============================================ SECURITY LOCKCTRL SELECT [0..7] ============================================ */ +typedef enum { /*!< SECURITY_LOCKCTRL_SELECT */ + SECURITY_LOCKCTRL_SELECT_CUSTOMER_KEY = 1, /*!< CUSTOMER_KEY : Unlock Customer Key (access to top half of info0) + value. */ + SECURITY_LOCKCTRL_SELECT_NONE = 0, /*!< NONE : Lock Control should be set to NONE when not in use. value. */ +} SECURITY_LOCKCTRL_SELECT_Enum; + +/* ======================================================= LOCKSTAT ======================================================== */ +/* =========================================== SECURITY LOCKSTAT STATUS [0..31] ============================================ */ +typedef enum { /*!< SECURITY_LOCKSTAT_STATUS */ + SECURITY_LOCKSTAT_STATUS_CUSTOMER_KEY = 1, /*!< CUSTOMER_KEY : Customer Key is unlocked (access is granted to + top half of info0) value. */ + SECURITY_LOCKSTAT_STATUS_NONE = 0, /*!< NONE : No resources are unlocked value. */ +} SECURITY_LOCKSTAT_STATUS_Enum; + +/* ========================================================= KEY0 ========================================================== */ +/* ========================================================= KEY1 ========================================================== */ +/* ========================================================= KEY2 ========================================================== */ +/* ========================================================= KEY3 ========================================================== */ + + +/* =========================================================================================================================== */ +/* ================ UART0 ================ */ +/* =========================================================================================================================== */ + +/* ========================================================== DR =========================================================== */ +/* =============================================== UART0 DR OEDATA [11..11] ================================================ */ +typedef enum { /*!< UART0_DR_OEDATA */ + UART0_DR_OEDATA_NOERR = 0, /*!< NOERR : No error on UART OEDATA, overrun error indicator. value. */ + UART0_DR_OEDATA_ERR = 1, /*!< ERR : Error on UART OEDATA, overrun error indicator. value. */ +} UART0_DR_OEDATA_Enum; + +/* =============================================== UART0 DR BEDATA [10..10] ================================================ */ +typedef enum { /*!< UART0_DR_BEDATA */ + UART0_DR_BEDATA_NOERR = 0, /*!< NOERR : No error on UART BEDATA, break error indicator. value. */ + UART0_DR_BEDATA_ERR = 1, /*!< ERR : Error on UART BEDATA, break error indicator. value. */ +} UART0_DR_BEDATA_Enum; + +/* ================================================ UART0 DR PEDATA [9..9] ================================================= */ +typedef enum { /*!< UART0_DR_PEDATA */ + UART0_DR_PEDATA_NOERR = 0, /*!< NOERR : No error on UART PEDATA, parity error indicator. value. */ + UART0_DR_PEDATA_ERR = 1, /*!< ERR : Error on UART PEDATA, parity error indicator. value. */ +} UART0_DR_PEDATA_Enum; + +/* ================================================ UART0 DR FEDATA [8..8] ================================================= */ +typedef enum { /*!< UART0_DR_FEDATA */ + UART0_DR_FEDATA_NOERR = 0, /*!< NOERR : No error on UART FEDATA, framing error indicator. value. */ + UART0_DR_FEDATA_ERR = 1, /*!< ERR : Error on UART FEDATA, framing error indicator. value. */ +} UART0_DR_FEDATA_Enum; + +/* ========================================================== RSR ========================================================== */ +/* ================================================ UART0 RSR OESTAT [3..3] ================================================ */ +typedef enum { /*!< UART0_RSR_OESTAT */ + UART0_RSR_OESTAT_NOERR = 0, /*!< NOERR : No error on UART OESTAT, overrun error indicator. value. */ + UART0_RSR_OESTAT_ERR = 1, /*!< ERR : Error on UART OESTAT, overrun error indicator. value. */ +} UART0_RSR_OESTAT_Enum; + +/* ================================================ UART0 RSR BESTAT [2..2] ================================================ */ +typedef enum { /*!< UART0_RSR_BESTAT */ + UART0_RSR_BESTAT_NOERR = 0, /*!< NOERR : No error on UART BESTAT, break error indicator. value. */ + UART0_RSR_BESTAT_ERR = 1, /*!< ERR : Error on UART BESTAT, break error indicator. value. */ +} UART0_RSR_BESTAT_Enum; + +/* ================================================ UART0 RSR PESTAT [1..1] ================================================ */ +typedef enum { /*!< UART0_RSR_PESTAT */ + UART0_RSR_PESTAT_NOERR = 0, /*!< NOERR : No error on UART PESTAT, parity error indicator. value. */ + UART0_RSR_PESTAT_ERR = 1, /*!< ERR : Error on UART PESTAT, parity error indicator. value. */ +} UART0_RSR_PESTAT_Enum; + +/* ================================================ UART0 RSR FESTAT [0..0] ================================================ */ +typedef enum { /*!< UART0_RSR_FESTAT */ + UART0_RSR_FESTAT_NOERR = 0, /*!< NOERR : No error on UART FESTAT, framing error indicator. value. */ + UART0_RSR_FESTAT_ERR = 1, /*!< ERR : Error on UART FESTAT, framing error indicator. value. */ +} UART0_RSR_FESTAT_Enum; + +/* ========================================================== FR =========================================================== */ +/* ================================================= UART0 FR TXFE [7..7] ================================================== */ +typedef enum { /*!< UART0_FR_TXFE */ + UART0_FR_TXFE_XMTFIFO_EMPTY = 1, /*!< XMTFIFO_EMPTY : Transmit fifo is empty. value. */ +} UART0_FR_TXFE_Enum; + +/* ================================================= UART0 FR RXFF [6..6] ================================================== */ +typedef enum { /*!< UART0_FR_RXFF */ + UART0_FR_RXFF_RCVFIFO_FULL = 1, /*!< RCVFIFO_FULL : Receive fifo is full. value. */ +} UART0_FR_RXFF_Enum; + +/* ================================================= UART0 FR TXFF [5..5] ================================================== */ +typedef enum { /*!< UART0_FR_TXFF */ + UART0_FR_TXFF_XMTFIFO_FULL = 1, /*!< XMTFIFO_FULL : Transmit fifo is full. value. */ +} UART0_FR_TXFF_Enum; + +/* ================================================= UART0 FR RXFE [4..4] ================================================== */ +typedef enum { /*!< UART0_FR_RXFE */ + UART0_FR_RXFE_RCVFIFO_EMPTY = 1, /*!< RCVFIFO_EMPTY : Receive fifo is empty. value. */ +} UART0_FR_RXFE_Enum; + +/* ================================================= UART0 FR BUSY [3..3] ================================================== */ +typedef enum { /*!< UART0_FR_BUSY */ + UART0_FR_BUSY_BUSY = 1, /*!< BUSY : UART busy indicator. value. */ +} UART0_FR_BUSY_Enum; + +/* ================================================== UART0 FR DCD [2..2] ================================================== */ +typedef enum { /*!< UART0_FR_DCD */ + UART0_FR_DCD_DETECTED = 1, /*!< DETECTED : Data carrier detect detected. value. */ +} UART0_FR_DCD_Enum; + +/* ================================================== UART0 FR DSR [1..1] ================================================== */ +typedef enum { /*!< UART0_FR_DSR */ + UART0_FR_DSR_READY = 1, /*!< READY : Data set ready. value. */ +} UART0_FR_DSR_Enum; + +/* ================================================== UART0 FR CTS [0..0] ================================================== */ +typedef enum { /*!< UART0_FR_CTS */ + UART0_FR_CTS_CLEARTOSEND = 1, /*!< CLEARTOSEND : Clear to send is indicated. value. */ +} UART0_FR_CTS_Enum; + +/* ========================================================= ILPR ========================================================== */ +/* ========================================================= IBRD ========================================================== */ +/* ========================================================= FBRD ========================================================== */ +/* ========================================================= LCRH ========================================================== */ +/* ========================================================== CR =========================================================== */ +/* ================================================ UART0 CR CLKSEL [4..6] ================================================= */ +typedef enum { /*!< UART0_CR_CLKSEL */ + UART0_CR_CLKSEL_NOCLK = 0, /*!< NOCLK : No UART clock. This is the low power default. value. */ + UART0_CR_CLKSEL_24MHZ = 1, /*!< 24MHZ : 24 MHz clock. value. */ + UART0_CR_CLKSEL_12MHZ = 2, /*!< 12MHZ : 12 MHz clock. value. */ + UART0_CR_CLKSEL_6MHZ = 3, /*!< 6MHZ : 6 MHz clock. value. */ + UART0_CR_CLKSEL_3MHZ = 4, /*!< 3MHZ : 3 MHz clock. value. */ +} UART0_CR_CLKSEL_Enum; + +/* ========================================================= IFLS ========================================================== */ +/* ========================================================== IER ========================================================== */ +/* ========================================================== IES ========================================================== */ +/* ========================================================== MIS ========================================================== */ +/* ========================================================== IEC ========================================================== */ + + +/* =========================================================================================================================== */ +/* ================ VCOMP ================ */ +/* =========================================================================================================================== */ + +/* ========================================================== CFG ========================================================== */ +/* =============================================== VCOMP CFG LVLSEL [16..19] =============================================== */ +typedef enum { /*!< VCOMP_CFG_LVLSEL */ + VCOMP_CFG_LVLSEL_0P58V = 0, /*!< 0P58V : Set Reference input to 0.58 Volts. value. */ + VCOMP_CFG_LVLSEL_0P77V = 1, /*!< 0P77V : Set Reference input to 0.77 Volts. value. */ + VCOMP_CFG_LVLSEL_0P97V = 2, /*!< 0P97V : Set Reference input to 0.97 Volts. value. */ + VCOMP_CFG_LVLSEL_1P16V = 3, /*!< 1P16V : Set Reference input to 1.16 Volts. value. */ + VCOMP_CFG_LVLSEL_1P35V = 4, /*!< 1P35V : Set Reference input to 1.35 Volts. value. */ + VCOMP_CFG_LVLSEL_1P55V = 5, /*!< 1P55V : Set Reference input to 1.55 Volts. value. */ + VCOMP_CFG_LVLSEL_1P74V = 6, /*!< 1P74V : Set Reference input to 1.74 Volts. value. */ + VCOMP_CFG_LVLSEL_1P93V = 7, /*!< 1P93V : Set Reference input to 1.93 Volts. value. */ + VCOMP_CFG_LVLSEL_2P13V = 8, /*!< 2P13V : Set Reference input to 2.13 Volts. value. */ + VCOMP_CFG_LVLSEL_2P32V = 9, /*!< 2P32V : Set Reference input to 2.32 Volts. value. */ + VCOMP_CFG_LVLSEL_2P51V = 10, /*!< 2P51V : Set Reference input to 2.51 Volts. value. */ + VCOMP_CFG_LVLSEL_2P71V = 11, /*!< 2P71V : Set Reference input to 2.71 Volts. value. */ + VCOMP_CFG_LVLSEL_2P90V = 12, /*!< 2P90V : Set Reference input to 2.90 Volts. value. */ + VCOMP_CFG_LVLSEL_3P09V = 13, /*!< 3P09V : Set Reference input to 3.09 Volts. value. */ + VCOMP_CFG_LVLSEL_3P29V = 14, /*!< 3P29V : Set Reference input to 3.29 Volts. value. */ + VCOMP_CFG_LVLSEL_3P48V = 15, /*!< 3P48V : Set Reference input to 3.48 Volts. value. */ +} VCOMP_CFG_LVLSEL_Enum; + +/* ================================================= VCOMP CFG NSEL [8..9] ================================================= */ +typedef enum { /*!< VCOMP_CFG_NSEL */ + VCOMP_CFG_NSEL_VREFEXT1 = 0, /*!< VREFEXT1 : Use external reference 1 for reference input. value. */ + VCOMP_CFG_NSEL_VREFEXT2 = 1, /*!< VREFEXT2 : Use external reference 2 for reference input. value. */ + VCOMP_CFG_NSEL_VREFEXT3 = 2, /*!< VREFEXT3 : Use external reference 3 for reference input. value. */ + VCOMP_CFG_NSEL_DAC = 3, /*!< DAC : Use DAC output selected by LVLSEL for reference input. + value. */ +} VCOMP_CFG_NSEL_Enum; + +/* ================================================= VCOMP CFG PSEL [0..1] ================================================= */ +typedef enum { /*!< VCOMP_CFG_PSEL */ + VCOMP_CFG_PSEL_VDDADJ = 0, /*!< VDDADJ : Use VDDADJ for the positive input. value. */ + VCOMP_CFG_PSEL_VTEMP = 1, /*!< VTEMP : Use the temperature sensor output for the positive input. + Note: If this channel is selected for PSEL, the bandap + circuit required for temperature comparisons will automatically + turn on. The bandgap circuit requires 11us to stabalize. + value. */ + VCOMP_CFG_PSEL_VEXT1 = 2, /*!< VEXT1 : Use external voltage 0 for positive input. value. */ + VCOMP_CFG_PSEL_VEXT2 = 3, /*!< VEXT2 : Use external voltage 1 for positive input. value. */ +} VCOMP_CFG_PSEL_Enum; + +/* ========================================================= STAT ========================================================== */ +/* =============================================== VCOMP STAT PWDSTAT [1..1] =============================================== */ +typedef enum { /*!< VCOMP_STAT_PWDSTAT */ + VCOMP_STAT_PWDSTAT_POWERED_DOWN = 1, /*!< POWERED_DOWN : The voltage comparator is powered down. value. */ +} VCOMP_STAT_PWDSTAT_Enum; + +/* =============================================== VCOMP STAT CMPOUT [0..0] ================================================ */ +typedef enum { /*!< VCOMP_STAT_CMPOUT */ + VCOMP_STAT_CMPOUT_VOUT_LOW = 0, /*!< VOUT_LOW : The negative input of the comparator is greater than + the positive input. value. */ + VCOMP_STAT_CMPOUT_VOUT_HIGH = 1, /*!< VOUT_HIGH : The positive input of the comparator is greater + than the negative input. value. */ +} VCOMP_STAT_CMPOUT_Enum; + +/* ======================================================== PWDKEY ========================================================= */ +/* ============================================== VCOMP PWDKEY PWDKEY [0..31] ============================================== */ +typedef enum { /*!< VCOMP_PWDKEY_PWDKEY */ + VCOMP_PWDKEY_PWDKEY_Key = 55, /*!< Key : Key value. */ +} VCOMP_PWDKEY_PWDKEY_Enum; + +/* ========================================================= INTEN ========================================================= */ +/* ======================================================== INTSTAT ======================================================== */ +/* ======================================================== INTCLR ========================================================= */ +/* ======================================================== INTSET ========================================================= */ + + +/* =========================================================================================================================== */ +/* ================ WDT ================ */ +/* =========================================================================================================================== */ + +/* ========================================================== CFG ========================================================== */ +/* ================================================ WDT CFG CLKSEL [24..26] ================================================ */ +typedef enum { /*!< WDT_CFG_CLKSEL */ + WDT_CFG_CLKSEL_OFF = 0, /*!< OFF : Low Power Mode. This setting disables the watch dog timer. + value. */ + WDT_CFG_CLKSEL_128HZ = 1, /*!< 128HZ : 128 Hz LFRC clock. value. */ + WDT_CFG_CLKSEL_16HZ = 2, /*!< 16HZ : 16 Hz LFRC clock. value. */ + WDT_CFG_CLKSEL_1HZ = 3, /*!< 1HZ : 1 Hz LFRC clock. value. */ + WDT_CFG_CLKSEL_1_16HZ = 4, /*!< 1_16HZ : 1/16th Hz LFRC clock. value. */ +} WDT_CFG_CLKSEL_Enum; + +/* ========================================================= RSTRT ========================================================= */ +/* ================================================ WDT RSTRT RSTRT [0..7] ================================================= */ +typedef enum { /*!< WDT_RSTRT_RSTRT */ + WDT_RSTRT_RSTRT_KEYVALUE = 178, /*!< KEYVALUE : This is the key value to write to WDTRSTRT to restart + the WDT. This is a write only register. value. */ +} WDT_RSTRT_RSTRT_Enum; + +/* ========================================================= LOCK ========================================================== */ +/* ================================================= WDT LOCK LOCK [0..7] ================================================== */ +typedef enum { /*!< WDT_LOCK_LOCK */ + WDT_LOCK_LOCK_KEYVALUE = 58, /*!< KEYVALUE : This is the key value to write to WDTLOCK to lock + the WDT. value. */ +} WDT_LOCK_LOCK_Enum; + +/* ========================================================= COUNT ========================================================= */ +/* ========================================================= INTEN ========================================================= */ +/* ======================================================== INTSTAT ======================================================== */ +/* ======================================================== INTCLR ========================================================= */ +/* ======================================================== INTSET ========================================================= */ + +/** @} */ /* End of group EnumValue_peripherals */ + + +#ifdef __cplusplus +} +#endif + +#endif /* APOLLO3_H */ + + +/** @} */ /* End of group apollo3 */ + +/** @} */ /* End of group Ambiq Micro */ diff --git a/tensorflow/lite/experimental/micro/examples/micro_speech/apollo3/captured_data_to_wav.py b/tensorflow/lite/experimental/micro/examples/micro_speech/apollo3/captured_data_to_wav.py new file mode 100644 index 00000000000..84b62594524 --- /dev/null +++ b/tensorflow/lite/experimental/micro/examples/micro_speech/apollo3/captured_data_to_wav.py @@ -0,0 +1,40 @@ +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the License); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an AS IS BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import numpy as np +import re +import struct +import matplotlib.pyplot as plt +import soundfile as sf + +def new_data_to_array(fn): + vals = [] + with open(fn) as f: + for n, line in enumerate(f): + if n is not 0: + vals.extend([int(v, 16) for v in line.split()]) + b = ''.join(map(chr, vals)) + y = struct.unpack('<'+'h'*int(len(b)/2), b) + + return y + + +data = 'captured_data.txt' +vals = np.array(new_data_to_array(data)).astype(float) + +#plt.plot(vals, 'o-') +#plt.show(block=False) + +wav = vals/np.max(np.abs(vals)) +sf.write('captured_data.wav', wav, 16000) diff --git a/tensorflow/lite/experimental/micro/examples/micro_speech/apollo3/compare_1k.py b/tensorflow/lite/experimental/micro/examples/micro_speech/apollo3/compare_1k.py new file mode 100644 index 00000000000..9c91560d505 --- /dev/null +++ b/tensorflow/lite/experimental/micro/examples/micro_speech/apollo3/compare_1k.py @@ -0,0 +1,153 @@ +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the License); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an AS IS BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import numpy as np +import re +import struct +import matplotlib.pyplot as plt +import soundfile as sf + +def new_data_to_array(fn, datatype='int16'): + vals = [] + with open(fn) as f: + for n, line in enumerate(f): + if n is not 0: + vals.extend([int(v, 16) for v in line.split()]) + b = ''.join(map(chr, vals)) + + if datatype == 'int8': + typestr = 'b' + arraylen = int(len(b)) + elif datatype == 'int16': + typestr = 'h' + arraylen = int(len(b)//2) + elif datatype == 'int32': + typestr = 'i' + arraylen = int(len(b)//4) + if datatype == 'uint8': + typestr = 'B' + arraylen = int(len(b)) + elif datatype == 'uint16': + typestr = 'H' + arraylen = int(len(b)//2) + elif datatype == 'uint32': + typestr = 'I' + arraylen = int(len(b)//4) + + y = np.array(struct.unpack('<'+typestr*arraylen, b)) + + return y + +# x is the fixed-point input in Qm.n format +def to_float(x, n): + return x.astype(float)*2**(-n) + +micro_windowed_input = new_data_to_array('micro_windowed_input.txt', datatype='int32') +cmsis_windowed_input = new_data_to_array('cmsis_windowed_input.txt', datatype='int16') + +micro_dft = new_data_to_array('micro_dft.txt', datatype='int32') +cmsis_dft = new_data_to_array('cmsis_dft.txt', datatype='int16') +py_dft = np.fft.rfft(to_float(cmsis_windowed_input,15), n=512) +py_result = np.empty((2*py_dft.size), dtype=np.float) +py_result[0::2] = np.real(py_dft) +py_result[1::2] = np.imag(py_dft) + +micro_power = new_data_to_array('micro_power.txt', datatype='int32') +cmsis_power = new_data_to_array('cmsis_power.txt', datatype='int16') +py_power = np.square(np.abs(py_dft)) + +micro_power_avg = new_data_to_array('micro_power_avg.txt', datatype='uint8') +cmsis_power_avg = new_data_to_array('cmsis_power_avg.txt', datatype='uint8') + +plt.figure(1) +plt.subplot(311) +plt.plot(micro_windowed_input, label='Micro fixed') +plt.legend() +plt.subplot(312) +plt.plot(cmsis_windowed_input, label='CMSIS fixed') +plt.legend() +plt.subplot(313) +plt.plot(to_float(micro_windowed_input, 30), label='Micro to float') +plt.plot(to_float(cmsis_windowed_input, 15), label='CMSIS to float') +plt.legend() + +plt.figure(2) +plt.subplot(311) +plt.plot(micro_dft, label='Micro fixed') +plt.legend() +plt.subplot(312) +plt.plot(cmsis_dft, label='CMSIS fixed') +plt.legend() +plt.subplot(313) +plt.plot(to_float(micro_dft, 22), label='Micro to float') +# CMSIS result has 6 fractionanl bits (not 7) due to documentation error (see README.md) +plt.plot(to_float(cmsis_dft, 6), label='CMSIS to float') +plt.plot(py_result, label='Python result') +plt.legend() + +plt.figure(3) +plt.subplot(311) +plt.plot(micro_power, label='Micro fixed') +plt.legend() +plt.subplot(312) +plt.plot(cmsis_power[0:256], label='CMSIS fixed') +plt.legend() +plt.subplot(313) +plt.plot(to_float(micro_power, 22), label='Micro to float') +plt.plot(to_float(cmsis_power[0:256], 6), label='CMSIS to float') +plt.plot(py_power, label='Python result') +plt.legend() + +plt.figure(4) +plt.plot(micro_power_avg, label='Micro fixed') +plt.plot(cmsis_power_avg, label='CMSIS fixed') +plt.legend() +plt.show() + +#t = np.arange(16000.*0.03)/16000. +#sin1k = 0.1*np.sin(2*np.pi*1000*t) # Factor of 10 because micro preprocessing overflows otherwise +# +#plt.figure(1) +#plt.subplot(511) +#plt.plot(sin1k) +#plt.title('Input sine') +# +#plt.subplot(512) +#plt.plot(to_float(micro_windowed_input, 30), label='Micro-Lite') +#plt.plot(to_float(cmsis_windowed_input, 15), label='CMSIS') +#plt.title('Windowed sine') +#plt.legend(loc='center right') +# +#plt.subplot(513) +#plt.plot(to_float(micro_dft, 22), label='Micro-Lite') +#plt.plot(to_float(cmsis_dft, 6), label='CMSIS') +#plt.title('FFT') +#plt.legend(loc='center') +# +#plt.subplot(514) +#plt.plot(to_float(micro_power, 22), label='Micro-Lite') +#plt.plot(to_float(cmsis_power[0:256], 6), label='CMSIS') +#plt.title('|FFT|^2') +#plt.legend(loc='center right') +# +#plt.subplot(515) +#plt.plot(micro_power_avg, label='Micro-Lite') +#plt.plot(cmsis_power_avg, label='CMSIS') +#plt.title('Averaged |FFT|^2') +#plt.legend(loc='center right') +# +#plt.tight_layout(pad=0, w_pad=0.2, h_pad=0.2) +# +#plt.show() +# diff --git a/tensorflow/lite/experimental/micro/examples/micro_speech/apollo3/preprocessor_1k.cc b/tensorflow/lite/experimental/micro/examples/micro_speech/apollo3/preprocessor_1k.cc new file mode 100644 index 00000000000..31fae1f2dc2 --- /dev/null +++ b/tensorflow/lite/experimental/micro/examples/micro_speech/apollo3/preprocessor_1k.cc @@ -0,0 +1,57 @@ +/* This file is a modification of the Tensorflow Micro Lite file preprocessor.cc + * We have retained the original copyright and header information, in + * accordance with the Apache 2.0 license terms. + */ + +/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/lite/experimental/micro/examples/micro_speech/CMSIS/sin_1k.h" +#include "tensorflow/lite/c/c_api_internal.h" +#include "tensorflow/lite/experimental/micro/micro_error_reporter.h" +#include "tensorflow/lite/experimental/micro/testing/micro_test.h" + +extern "C" { +#include "tensorflow/lite/experimental/micro/examples/micro_speech/apollo3/system_apollo3.h" +#include "tensorflow/lite/experimental/micro/examples/micro_speech/apollo3/apollo3.h" +} + +#define output_data_size 43 +int count; + +extern TfLiteStatus Preprocess(tflite::ErrorReporter* error_reporter, + const int16_t* input, int input_size, int output_size, + uint8_t* output); + +TF_LITE_MICRO_TESTS_BEGIN +CoreDebug->DEMCR |= CoreDebug_DEMCR_TRCENA_Msk; +//DWT->LAR = 0xC5ACCE55; +DWT->CYCCNT = 0; +DWT->CTRL |= DWT_CTRL_CYCCNTENA_Msk; + +TF_LITE_MICRO_TEST(TestPreprocessor) { + tflite::MicroErrorReporter micro_error_reporter; + tflite::ErrorReporter* error_reporter = µ_error_reporter; + + uint8_t calculated_data[output_data_size]; + TfLiteStatus yes_status = Preprocess( + error_reporter, g_sin_1k, g_sin_1k_size, + output_data_size, calculated_data); + count = DWT->CYCCNT; + TF_LITE_MICRO_EXPECT_EQ(kTfLiteOk, yes_status); + +} + +TF_LITE_MICRO_TESTS_END diff --git a/tensorflow/lite/experimental/micro/examples/micro_speech/apollo3/preprocessor_1k_cmsis_test.cmd b/tensorflow/lite/experimental/micro/examples/micro_speech/apollo3/preprocessor_1k_cmsis_test.cmd new file mode 100644 index 00000000000..0f701660a87 --- /dev/null +++ b/tensorflow/lite/experimental/micro/examples/micro_speech/apollo3/preprocessor_1k_cmsis_test.cmd @@ -0,0 +1,44 @@ +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the License); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an AS IS BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Needs to be compiled with -O0 +file ../../../tools/make/gen/apollo3evb_cortex-m4/bin/preprocessor_1k_cmsis_test +target remote localhost:2331 +load ../../../tools/make/gen/apollo3evb_cortex-m4/bin/preprocessor_1k_cmsis_test +monitor reset +break preprocessor.cc:70 +commands +dump verilog value cmsis_windowed_input.txt bufB +c +end +break preprocessor.cc:77 +commands +dump verilog value cmsis_dft.txt bufA +c +end +break preprocessor.cc:82 +commands +dump verilog value cmsis_power.txt bufB +c +end +break preprocessor.cc:84 +commands +dump verilog memory cmsis_power_avg.txt output output+42 +c +end +break preprocessor_1k.cc:53 +commands +print count +end +c diff --git a/tensorflow/lite/experimental/micro/examples/micro_speech/apollo3/preprocessor_1k_micro_test.cmd b/tensorflow/lite/experimental/micro/examples/micro_speech/apollo3/preprocessor_1k_micro_test.cmd new file mode 100644 index 00000000000..35c602d78ce --- /dev/null +++ b/tensorflow/lite/experimental/micro/examples/micro_speech/apollo3/preprocessor_1k_micro_test.cmd @@ -0,0 +1,32 @@ +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the License); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an AS IS BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Needs to be run when compiled with -O0 +file ../../../tools/make/gen/apollo3evb_cortex-m4/bin/preprocessor_1k_micro_test +target remote localhost:2331 +load ../../../tools/make/gen/apollo3evb_cortex-m4/bin/preprocessor_1k_micro_test +monitor reset +break preprocessor.cc:211 +commands +dump verilog value micro_windowed_input.txt fixed_input +dump verilog value micro_dft.txt fourier_values +dump verilog value micro_power.txt power_spectrum +dump verilog memory micro_power_avg.txt output output+42 +c +end +break preprocessor_1k.cc:53 +commands +print count +end +c diff --git a/tensorflow/lite/experimental/micro/examples/micro_speech/apollo3/preprocessor_test.cmd b/tensorflow/lite/experimental/micro/examples/micro_speech/apollo3/preprocessor_test.cmd new file mode 100644 index 00000000000..4458af17d67 --- /dev/null +++ b/tensorflow/lite/experimental/micro/examples/micro_speech/apollo3/preprocessor_test.cmd @@ -0,0 +1,18 @@ +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the License); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an AS IS BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +file ../../gen/apollo3evb_cortex-m4/bin/preprocessor_test +target remote localhost:2331 +load ../../gen/apollo3evb_cortex-m4/bin/preprocessor_test +monitor reset diff --git a/tensorflow/lite/experimental/micro/examples/micro_speech/apollo3/pushbutton_cmsis_scores.cmd b/tensorflow/lite/experimental/micro/examples/micro_speech/apollo3/pushbutton_cmsis_scores.cmd new file mode 100644 index 00000000000..db299f72771 --- /dev/null +++ b/tensorflow/lite/experimental/micro/examples/micro_speech/apollo3/pushbutton_cmsis_scores.cmd @@ -0,0 +1,33 @@ +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the License); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an AS IS BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +file ../../../tools/make/gen/apollo3evb_cortex-m4/bin/pushbutton_cmsis_speech_test +target remote localhost:2331 +load ../../../tools/make/gen/apollo3evb_cortex-m4/bin/pushbutton_cmsis_speech_test +monitor reset +break pushbutton_main.c:325 +commands +printf "Silence score: %d\n", g_silence_score +printf "Unknown score: %d\n", g_unknown_score +printf "Yes score: %d\n", g_yes_score +printf "No score: %d\n", g_no_score +printf "g_scores[0]: %d\n", g_scores[0] +printf "g_scores[1]: %d\n", g_scores[1] +printf "g_scores[2]: %d\n", g_scores[2] +printf "g_scores[3]: %d\n", g_scores[3] +printf "max_score: %d\n", max_score +printf "max_score_index: %d\n", max_score_index +c +end +c diff --git a/tensorflow/lite/experimental/micro/examples/micro_speech/apollo3/pushbutton_cmsis_voice.cmd b/tensorflow/lite/experimental/micro/examples/micro_speech/apollo3/pushbutton_cmsis_voice.cmd new file mode 100644 index 00000000000..8cd1a7e90eb --- /dev/null +++ b/tensorflow/lite/experimental/micro/examples/micro_speech/apollo3/pushbutton_cmsis_voice.cmd @@ -0,0 +1,24 @@ +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the License); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an AS IS BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +file ../../../tools/make/gen/apollo3evb_cortex-m4/bin/pushbutton_cmsis_speech_test +target remote localhost:2331 +load ../../../tools/make/gen/apollo3evb_cortex-m4/bin/pushbutton_cmsis_speech_test +monitor reset +break pushbutton_main.c:316 +commands +dump verilog value captured_data.txt captured_data +c +end +c diff --git a/tensorflow/lite/experimental/micro/examples/micro_speech/apollo3/pushbutton_main.c b/tensorflow/lite/experimental/micro/examples/micro_speech/apollo3/pushbutton_main.c new file mode 100644 index 00000000000..82337e63da1 --- /dev/null +++ b/tensorflow/lite/experimental/micro/examples/micro_speech/apollo3/pushbutton_main.c @@ -0,0 +1,339 @@ +/* This file is a modification of the Tensorflow Micro Lite file _main.c + * We have retained the original copyright and header information, in + * accordance with the Apache 2.0 license terms. + */ + +/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include +#include "am_mcu_apollo.h" // Defines AM_CMSIS_REGS +#include "am_bsp.h" +#include "am_util.h" + +#define ARM_MATH_CM4 +#include + +//***************************************************************************** +// Parameters +// +// Total number of bytes transferred = 320*50*2 = 32000 +//***************************************************************************** + +#define FRAME_SIZE 320 // Capture one 320-sample (20-ms) frame at a time +#define NUM_FRAMES 50 // Number of frames in 1 second + +//***************************************************************************** +// GLOBALS +//***************************************************************************** + +volatile int16_t g_numFramesCaptured = 0; +volatile bool g_bPDMDataReady = false; +int16_t captured_data[FRAME_SIZE*NUM_FRAMES]; // Location of 1-second data buffer +extern uint8_t g_silence_score; +extern uint8_t g_unknown_score; +extern uint8_t g_yes_score; +extern uint8_t g_no_score; +q7_t g_scores[4] = {0}; + + +//***************************************************************************** +// The entry point for the application. +//***************************************************************************** +extern int main(int argc, char**argv); + +void DebugLog(const char* s) { am_util_stdio_printf( "%s", s); } +void DebugLogInt32(int32_t i) { am_util_stdio_printf( "%d", i); } +void DebugLogUInt32(uint32_t i) { am_util_stdio_printf( "%d", i); } +void DebugLogHex(uint32_t i) { am_util_stdio_printf( "0x%8x", i); } +void DebugLogFloat(float i) { am_util_stdio_printf( "%f", i); } + +//***************************************************************************** +// PDM configuration information. +//***************************************************************************** +void *PDMHandle; + +am_hal_pdm_config_t g_sPdmConfig = +{ + .eClkDivider = AM_HAL_PDM_MCLKDIV_1, + .eLeftGain = AM_HAL_PDM_GAIN_P225DB, + .eRightGain = AM_HAL_PDM_GAIN_P225DB, + .ui32DecimationRate = 48, // OSR = 1500/16 = 96 = 2*SINCRATE --> SINC_RATE = 48 + .bHighPassEnable = 0, + .ui32HighPassCutoff = 0xB, + .ePDMClkSpeed = AM_HAL_PDM_CLK_1_5MHZ, + .bInvertI2SBCLK = 0, + .ePDMClkSource = AM_HAL_PDM_INTERNAL_CLK, + .bPDMSampleDelay = 0, + .bDataPacking = 1, + .ePCMChannels = AM_HAL_PDM_CHANNEL_RIGHT, + .bLRSwap = 0, +}; + + +//***************************************************************************** +// BUTTON0 pin configuration settings. +//***************************************************************************** +const am_hal_gpio_pincfg_t g_deepsleep_button0 = +{ + .uFuncSel = 3, + .eIntDir = AM_HAL_GPIO_PIN_INTDIR_LO2HI, + .eGPInput = AM_HAL_GPIO_PIN_INPUT_ENABLE, +}; + +//***************************************************************************** +// PDM initialization. +//***************************************************************************** +void pdm_init(void) +{ + // + // Initialize, power-up, and configure the PDM. + // + am_hal_pdm_initialize(0, &PDMHandle); + am_hal_pdm_power_control(PDMHandle, AM_HAL_PDM_POWER_ON, false); + am_hal_pdm_configure(PDMHandle, &g_sPdmConfig); + am_hal_pdm_enable(PDMHandle); + + // + // Configure the necessary pins. + // + am_hal_gpio_pincfg_t sPinCfg = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}; + +// ARPIT 181019 + //sPinCfg.uFuncSel = AM_HAL_PIN_10_PDMCLK; + //am_hal_gpio_pinconfig(10, sPinCfg); + sPinCfg.uFuncSel = AM_HAL_PIN_12_PDMCLK; + am_hal_gpio_pinconfig(12, sPinCfg); + + sPinCfg.uFuncSel = AM_HAL_PIN_11_PDMDATA; + am_hal_gpio_pinconfig(11, sPinCfg); + + //am_hal_gpio_state_write(14, AM_HAL_GPIO_OUTPUT_CLEAR); + //am_hal_gpio_pinconfig(14, g_AM_HAL_GPIO_OUTPUT); + + // + // Configure and enable PDM interrupts (set up to trigger on DMA + // completion). + // + am_hal_pdm_interrupt_enable(PDMHandle, (AM_HAL_PDM_INT_DERR + | AM_HAL_PDM_INT_DCMP + | AM_HAL_PDM_INT_UNDFL + | AM_HAL_PDM_INT_OVF)); + +#if AM_CMSIS_REGS + NVIC_EnableIRQ(PDM_IRQn); +#else + am_hal_interrupt_enable(AM_HAL_INTERRUPT_PDM); +#endif +} + +//***************************************************************************** +// +// Start a transaction to get some number of bytes from the PDM interface. +// +//***************************************************************************** +void pdm_data_get(void) +{ + // + // Configure DMA and target address. + // + am_hal_pdm_transfer_t sTransfer; + sTransfer.ui32TargetAddr = (uint32_t ) (&captured_data[FRAME_SIZE*g_numFramesCaptured]); + sTransfer.ui32TotalCount = 2*FRAME_SIZE; // Each sample is 2 bytes + + // + // Start the data transfer. + // + am_hal_pdm_dma_start(PDMHandle, &sTransfer); +} + + +//***************************************************************************** +// +// PDM interrupt handler. +// +//***************************************************************************** +void am_pdm_isr(void) +{ + uint32_t ui32Status; + // + // Read the interrupt status. + // + am_hal_pdm_interrupt_status_get(PDMHandle, &ui32Status, true); + am_hal_pdm_interrupt_clear(PDMHandle, ui32Status); + + // + // Once our DMA transaction completes, send a flag to the main routine + // + if (ui32Status & AM_HAL_PDM_INT_DCMP) + g_bPDMDataReady = true; +} + + +//***************************************************************************** +// GPIO ISR +// Will enable the PDM, set number of frames transferred to 0, and turn on LED +//***************************************************************************** +void +am_gpio_isr(void) +{ + // + // Delay for debounce. + // + am_util_delay_ms(200); + + // + // Clear the GPIO Interrupt (write to clear). + // + am_hal_gpio_interrupt_clear(AM_HAL_GPIO_BIT(AM_BSP_GPIO_BUTTON0)); + + // Start audio transfer + am_hal_pdm_fifo_flush(PDMHandle); + pdm_data_get(); + am_hal_pdm_enable(PDMHandle); + + // + // Turn on LED 0 + // + am_devices_led_on(am_bsp_psLEDs, 0); +} + +int _main(void) +{ + am_util_id_t sIdDevice; + uint32_t ui32StrBuf; + + // + // Set the clock frequency. + // + am_hal_clkgen_control(AM_HAL_CLKGEN_CONTROL_SYSCLK_MAX, 0); + + // + // Set the default cache configuration + // + am_hal_cachectrl_config(&am_hal_cachectrl_defaults); + am_hal_cachectrl_enable(); + + // + // Configure the board for low power operation. + // + am_bsp_low_power_init(); + + +#if defined(AM_BSP_NUM_BUTTONS) && defined(AM_BSP_NUM_LEDS) + // + // Configure the button pin. + // + am_hal_gpio_pinconfig(AM_BSP_GPIO_BUTTON0, g_deepsleep_button0); + + // + // Clear the GPIO Interrupt (write to clear). + // + am_hal_gpio_interrupt_clear(AM_HAL_GPIO_BIT(AM_BSP_GPIO_BUTTON0)); + + // + // Enable the GPIO/button interrupt. + // + am_hal_gpio_interrupt_enable(AM_HAL_GPIO_BIT(AM_BSP_GPIO_BUTTON0)); + + // + // Configure the LEDs. + // + am_devices_led_array_init(am_bsp_psLEDs, AM_BSP_NUM_LEDS); + + // + // Turn the LEDs off + // + for (int ix = 0; ix < AM_BSP_NUM_LEDS; ix++) + { + am_devices_led_off(am_bsp_psLEDs, ix); + } + +// am_devices_led_on(am_bsp_psLEDs, 1); +#endif // defined(AM_BSP_NUM_BUTTONS) && defined(AM_BSP_NUM_LEDS) + +#if AM_CMSIS_REGS + NVIC_EnableIRQ(GPIO_IRQn); +#else // AM_CMSIS_REGS + am_hal_interrupt_enable(AM_HAL_INTERRUPT_GPIO); +#endif // AM_CMSIS_REGS + + // + // Enable interrupts to the core. + // + am_hal_interrupt_master_enable(); + + // Turn on PDM + pdm_init(); + + // + // Initialize the printf interface for UART output + // + am_bsp_uart_printf_enable(); + + // + // Print the banner. + // + am_util_stdio_terminal_clear(); + am_util_stdio_printf("Starting streaming test\n\n"); + + // Score variables + q7_t max_score = 0; + uint32_t max_score_index = 0; + + while(1) + { + + am_hal_interrupt_master_disable(); + + if (g_bPDMDataReady) + { + g_bPDMDataReady = false; + g_numFramesCaptured++; + + if (g_numFramesCaptured < NUM_FRAMES) { + pdm_data_get(); // Start converting the next set of PCM samples. + } + + else + { + g_numFramesCaptured = 0; + //am_hal_pdm_disable(PDMHandle); + am_devices_led_off(am_bsp_psLEDs, 0); + + main(0, NULL); + + g_scores[0] = (q7_t) g_silence_score - 128; + g_scores[1] = (q7_t) g_unknown_score - 128; + g_scores[2] = (q7_t) g_yes_score - 128; + g_scores[3] = (q7_t) g_no_score - 128; + + am_devices_led_off(am_bsp_psLEDs, max_score_index+1); // Turn off LED for previous max score + arm_max_q7(g_scores, 4, &max_score, &max_score_index); + am_devices_led_on(am_bsp_psLEDs, max_score_index+1); // Turn on LED for new max score + } + } + + // + // Go to Deep Sleep. + // + am_hal_sysctrl_sleep(AM_HAL_SYSCTRL_SLEEP_DEEP); + + am_hal_interrupt_master_enable(); + + } + + //main(0, NULL); +} diff --git a/tensorflow/lite/experimental/micro/examples/micro_speech/apollo3/pushbutton_test.cc b/tensorflow/lite/experimental/micro/examples/micro_speech/apollo3/pushbutton_test.cc new file mode 100644 index 00000000000..ce4de4dbd8b --- /dev/null +++ b/tensorflow/lite/experimental/micro/examples/micro_speech/apollo3/pushbutton_test.cc @@ -0,0 +1,120 @@ +/* This file is a modification of the Tensorflow Micro Lite file micro_speech_test.cc + * We have retained the original copyright and header information, in + * accordance with the Apache 2.0 license terms. + */ + +/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/lite/experimental/micro/examples/micro_speech/preprocessor.h" +#include "tensorflow/lite/c/c_api_internal.h" +#include "tensorflow/lite/experimental/micro/micro_error_reporter.h" +#include "tensorflow/lite/experimental/micro/testing/micro_test.h" +#include "tensorflow/lite/experimental/micro/examples/micro_speech/tiny_conv_model_data.h" +#include "tensorflow/lite/experimental/micro/kernels/all_ops_resolver.h" +#include "tensorflow/lite/experimental/micro/micro_interpreter.h" +#include "tensorflow/lite/schema/schema_generated.h" +#include "tensorflow/lite/version.h" + +extern int16_t captured_data[16000]; +uint8_t g_silence_score = 0; +uint8_t g_unknown_score = 0; +uint8_t g_yes_score = 0; +uint8_t g_no_score = 0; + +TF_LITE_MICRO_TESTS_BEGIN + +TF_LITE_MICRO_TEST(TestPreprocessor) { + tflite::MicroErrorReporter micro_error_reporter; + tflite::ErrorReporter* error_reporter = µ_error_reporter; + + uint8_t preprocessed_data[43*49]; + TfLiteStatus preprocess_1sec_status = Preprocess_1sec( + error_reporter, captured_data, preprocessed_data); + TF_LITE_MICRO_EXPECT_EQ(kTfLiteOk, preprocess_1sec_status); + + + // Map the model into a usable data structure. This doesn't involve any + // copying or parsing, it's a very lightweight operation. + const tflite::Model* model = ::tflite::GetModel(g_tiny_conv_model_data); + if (model->version() != TFLITE_SCHEMA_VERSION) { + error_reporter->Report( + "Model provided is schema version %d not equal " + "to supported version %d.\n", + model->version(), TFLITE_SCHEMA_VERSION); + } + + // This pulls in all the operation implementations we need. + tflite::ops::micro::AllOpsResolver resolver; + + // Create an area of memory to use for input, output, and intermediate arrays. + const int tensor_arena_size = 10 * 1024; + uint8_t tensor_arena[tensor_arena_size]; + tflite::SimpleTensorAllocator tensor_allocator(tensor_arena, + tensor_arena_size); + + // Build an interpreter to run the model with. + tflite::MicroInterpreter interpreter(model, resolver, &tensor_allocator, + error_reporter); + + // Get information about the memory area to use for the model's input. + TfLiteTensor* input = interpreter.input(0); + + // Make sure the input has the properties we expect. + TF_LITE_MICRO_EXPECT_NE(nullptr, input); + TF_LITE_MICRO_EXPECT_EQ(4, input->dims->size); + TF_LITE_MICRO_EXPECT_EQ(1, input->dims->data[0]); + TF_LITE_MICRO_EXPECT_EQ(49, input->dims->data[1]); + TF_LITE_MICRO_EXPECT_EQ(43, input->dims->data[2]); + TF_LITE_MICRO_EXPECT_EQ(kTfLiteUInt8, input->type); + + // Copy a spectrogram created from a .wav audio file of someone saying "Yes", + // into the memory area used for the input. + for (int i = 0; i < input->bytes; ++i) { + input->data.uint8[i] = preprocessed_data[i]; + } + + // Run the model on this input and make sure it succeeds. + TfLiteStatus invoke_status = interpreter.Invoke(); + if (invoke_status != kTfLiteOk) { + error_reporter->Report("Invoke failed\n"); + } + TF_LITE_MICRO_EXPECT_EQ(kTfLiteOk, invoke_status); + + // Get the output from the model, and make sure it's the expected size and + // type. + TfLiteTensor* output = interpreter.output(0); + TF_LITE_MICRO_EXPECT_EQ(2, output->dims->size); + TF_LITE_MICRO_EXPECT_EQ(1, output->dims->data[0]); + TF_LITE_MICRO_EXPECT_EQ(4, output->dims->data[1]); + TF_LITE_MICRO_EXPECT_EQ(kTfLiteUInt8, output->type); + + // There are four possible classes in the output, each with a score. + const int kSilenceIndex = 0; + const int kUnknownIndex = 1; + const int kYesIndex = 2; + const int kNoIndex = 3; + + // Make sure that the expected "Yes" score is higher than the other classes. + g_silence_score = output->data.uint8[kSilenceIndex]; + g_unknown_score = output->data.uint8[kUnknownIndex]; + g_yes_score = output->data.uint8[kYesIndex]; + g_no_score = output->data.uint8[kNoIndex]; + + error_reporter->Report("Ran successfully\n"); + +} + +TF_LITE_MICRO_TESTS_END diff --git a/tensorflow/lite/experimental/micro/examples/micro_speech/apollo3/system_apollo3.c b/tensorflow/lite/experimental/micro/examples/micro_speech/apollo3/system_apollo3.c new file mode 100755 index 00000000000..c7f15735633 --- /dev/null +++ b/tensorflow/lite/experimental/micro/examples/micro_speech/apollo3/system_apollo3.c @@ -0,0 +1,116 @@ +//***************************************************************************** +// +//! @file system_apollo3.c +//! +//! @brief Ambiq Micro Apollo3 MCU specific functions. +// +//***************************************************************************** + +/* + * Copyright (C) 2015-2017, Ambiq Micro + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * 3. Neither the name of the copyright holder nor the names of itscontributors may be used to endorse + * or promote products derived from thissoftware without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * + * @file apollo3.h + * @brief CMSIS HeaderFile + * @version 1.0 + * @date 10. August 2018 + * @note Generated by SVDConv V3.3.18 on Friday, 10.08.2018 16:52:09 + * from File 'apollo3.svd', + * last modified on Friday, 10.08.2018 20:01:31 + */ + + +#include +#include "system_apollo3.h" +#include "apollo3.h" + +//***************************************************************************** +// +// Defines +// +//***************************************************************************** + +// +// Clocks +// +#define __HSI (6000000UL) +#define __XTAL (32768UL) // Crystal Oscillator frequency +#define __SYS_OSC_CLK (48000000) // Main oscillator frequency +#define __SYSTEM_CLOCK (1*__SYS_OSC_CLK) + +// +// Initialize SystemCoreClock with the system core clock frequency value +// achieved after system intitialization. +// This means system core clock frequency after call to SystemInit() +// +uint32_t SystemCoreClock = __SYSTEM_CLOCK; // System Clock Frequency (Core Clock) + +//***************************************************************************** +// +//! @brief Set the global clock frequncy. +//! +//! This function sets the global clock frequency. +//! +//! @return None. +// +//***************************************************************************** +void +SystemCoreClockUpdate(void) +{ + // + // Calculate the system frequency based upon the current register settings. + // This function can be used to retrieve the system core clock frequeny + // after user changed register sittings. + // + SystemCoreClock = __SYS_OSC_CLK / (CLKGEN->CCTRL_b.CORESEL + 1); +} + +//***************************************************************************** +// +//! @brief Initialize the system. +//! +//! This function sets up the microcontroller system. +//! +//! @return None. +// +//***************************************************************************** +void +SystemInit(void) +{ + // + // Initialize the system + // Do not use global variables because this function is called before + // reaching pre-main. RW section maybe overwritten afterwards. + // + SystemCoreClock = __SYSTEM_CLOCK; + + CLKGEN->CLKKEY = 0x47; // Enable write to CCTRL + CLKGEN->CCTRL_b.CORESEL = 0; // Div by 1 for 48MHz + CLKGEN->CLKKEY = 0; // Disable write to CCTRL +} + diff --git a/tensorflow/lite/experimental/micro/examples/micro_speech/apollo3/system_apollo3.h b/tensorflow/lite/experimental/micro/examples/micro_speech/apollo3/system_apollo3.h new file mode 100755 index 00000000000..7fd9b51d5a8 --- /dev/null +++ b/tensorflow/lite/experimental/micro/examples/micro_speech/apollo3/system_apollo3.h @@ -0,0 +1,72 @@ +//***************************************************************************** +// +//! @file system_Apollo3.h +//! +//! @brief Ambiq Micro Apollo3 MCU specific functions. +// +//***************************************************************************** + +/* + * Copyright (C) 2015-2017, Ambiq Micro + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * 3. Neither the name of the copyright holder nor the names of itscontributors may be used to endorse + * or promote products derived from thissoftware without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * + * @file apollo3.h + * @brief CMSIS HeaderFile + * @version 1.0 + * @date 10. August 2018 + * @note Generated by SVDConv V3.3.18 on Friday, 10.08.2018 16:52:09 + * from File 'apollo3.svd', + * last modified on Friday, 10.08.2018 20:01:31 + */ + + +#ifndef SYSTEM_APOLLO3_H +#define SYSTEM_APOLLO3_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include + +extern uint32_t SystemCoreClock; // System Clock Frequency (Core Clock) + +//***************************************************************************** +// +// External function definitions +// +//***************************************************************************** +extern void SystemInit (void); +extern void SystemCoreClockUpdate (void); + +#ifdef __cplusplus +} +#endif + +#endif // SYSTEM_APOLLO3_H + diff --git a/tensorflow/lite/experimental/micro/examples/micro_speech/audio_provider.cc b/tensorflow/lite/experimental/micro/examples/micro_speech/audio_provider.cc new file mode 100644 index 00000000000..c0365d56901 --- /dev/null +++ b/tensorflow/lite/experimental/micro/examples/micro_speech/audio_provider.cc @@ -0,0 +1,33 @@ +/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/lite/experimental/micro/examples/micro_speech/audio_provider.h" + +#include "tensorflow/lite/experimental/micro/examples/micro_speech/model_settings.h" + +namespace { +int16_t g_dummy_audio_data[kMaxAudioSampleSize]; +} // namespace + +TfLiteStatus GetAudioSamples(tflite::ErrorReporter* error_reporter, + int start_ms, int duration_ms, + int* audio_samples_size, int16_t** audio_samples) { + for (int i = 0; i < kMaxAudioSampleSize; ++i) { + g_dummy_audio_data[i] = 0; + } + *audio_samples_size = kMaxAudioSampleSize; + *audio_samples = g_dummy_audio_data; + return kTfLiteOk; +} diff --git a/tensorflow/lite/experimental/micro/examples/micro_speech/audio_provider.h b/tensorflow/lite/experimental/micro/examples/micro_speech/audio_provider.h new file mode 100644 index 00000000000..7e2442a5e83 --- /dev/null +++ b/tensorflow/lite/experimental/micro/examples/micro_speech/audio_provider.h @@ -0,0 +1,36 @@ +/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_LITE_EXPERIMENTAL_MICRO_EXAMPLES_MICRO_SPEECH_AUDIO_PROVIDER_H_ +#define TENSORFLOW_LITE_EXPERIMENTAL_MICRO_EXAMPLES_MICRO_SPEECH_AUDIO_PROVIDER_H_ + +#include "tensorflow/lite/c/c_api_internal.h" +#include "tensorflow/lite/experimental/micro/micro_error_reporter.h" + +// This is an abstraction around an audio source like a microphone, and is +// expected to return 16-bit PCM sample data for a given point in time. The +// sample data itself should be used as quickly as possible by the caller, since +// to allow memory optimizations there are no guarantees that the samples won't +// be overwritten by new data in the future. In practice, implementations should +// ensure that there's a reasonable time allowed for clients to access the data +// before any reuse. +// The reference implementation can have no platform-specific dependencies, so +// it just returns an array filled with zeros. For real applications, you should +// ensure there's a specialized implementation that accesses hardware APIs. +TfLiteStatus GetAudioSamples(tflite::ErrorReporter* error_reporter, + int start_ms, int duration_ms, + int* audio_samples_size, int16_t** audio_samples); + +#endif // TENSORFLOW_LITE_EXPERIMENTAL_MICRO_EXAMPLES_MICRO_SPEECH_AUDIO_PROVIDER_H_ diff --git a/tensorflow/lite/experimental/micro/examples/micro_speech/audio_provider_test.cc b/tensorflow/lite/experimental/micro/examples/micro_speech/audio_provider_test.cc new file mode 100644 index 00000000000..5f7c7605f0f --- /dev/null +++ b/tensorflow/lite/experimental/micro/examples/micro_speech/audio_provider_test.cc @@ -0,0 +1,44 @@ +/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/lite/experimental/micro/examples/micro_speech/audio_provider.h" +#include "tensorflow/lite/c/c_api_internal.h" +#include "tensorflow/lite/experimental/micro/examples/micro_speech/model_settings.h" +#include "tensorflow/lite/experimental/micro/micro_error_reporter.h" +#include "tensorflow/lite/experimental/micro/testing/micro_test.h" + +TF_LITE_MICRO_TESTS_BEGIN + +TF_LITE_MICRO_TEST(TestAudioProvider) { + tflite::MicroErrorReporter micro_error_reporter; + tflite::ErrorReporter* error_reporter = µ_error_reporter; + + int audio_samples_size = 0; + int16_t* audio_samples = nullptr; + TfLiteStatus get_status = + GetAudioSamples(error_reporter, 0, kFeatureSliceDurationMs, + &audio_samples_size, &audio_samples); + TF_LITE_MICRO_EXPECT_EQ(kTfLiteOk, get_status); + TF_LITE_MICRO_EXPECT_LE(audio_samples_size, kMaxAudioSampleSize); + TF_LITE_MICRO_EXPECT_NE(audio_samples, nullptr); + + // Make sure we can read all of the returned memory locations. + int total = 0; + for (int i = 0; i < audio_samples_size; ++i) { + total += audio_samples[i]; + } +} + +TF_LITE_MICRO_TESTS_END diff --git a/tensorflow/lite/experimental/micro/examples/micro_speech/feature_provider.cc b/tensorflow/lite/experimental/micro/examples/micro_speech/feature_provider.cc new file mode 100644 index 00000000000..c4c52ac0ff3 --- /dev/null +++ b/tensorflow/lite/experimental/micro/examples/micro_speech/feature_provider.cc @@ -0,0 +1,121 @@ +/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/lite/experimental/micro/examples/micro_speech/feature_provider.h" + +#include "tensorflow/lite/experimental/micro/examples/micro_speech/audio_provider.h" +#include "tensorflow/lite/experimental/micro/examples/micro_speech/model_settings.h" +#include "tensorflow/lite/experimental/micro/examples/micro_speech/preprocessor.h" +#include "tensorflow/lite/experimental/micro/examples/micro_speech/timer.h" + +namespace { +// Stores the timestamp for the previous fetch of audio data, so that we can +// avoid recalculating all the features from scratch if some earlier timeslices +// are still present. +int32_t g_last_time_in_ms = 0; +// Make sure we don't try to use cached information if this is the first call +// into the provider. +bool g_is_first_run = true; +} // namespace + +FeatureProvider::FeatureProvider(int feature_size, uint8_t* feature_data) + : feature_size_(feature_size), feature_data_(feature_data) { + // Initialize the feature data to default values. + for (int n = 0; n < feature_size_; ++n) { + feature_data_[n] = 0; + } +} + +FeatureProvider::~FeatureProvider() {} + +TfLiteStatus FeatureProvider::PopulateFeatureData( + tflite::ErrorReporter* error_reporter, int* how_many_new_slices) { + if (feature_size_ != kFeatureElementCount) { + error_reporter->Report("Requested feature_data_ size %d doesn't match %d", + feature_size_, kFeatureElementCount); + return kTfLiteError; + } + + const int32_t time_in_ms = TimeInMilliseconds(); + // Quantize the time into steps as long as each window stride, so we can + // figure out which audio data we need to fetch. + const int last_step = (g_last_time_in_ms / kFeatureSliceStrideMs); + const int current_step = (time_in_ms / kFeatureSliceStrideMs); + g_last_time_in_ms = time_in_ms; + + int slices_needed = current_step - last_step; + // If this is the first call, make sure we don't use any cached information. + if (g_is_first_run) { + g_is_first_run = false; + slices_needed = kFeatureSliceCount; + } + if (slices_needed > kFeatureSliceCount) { + slices_needed = kFeatureSliceCount; + } + *how_many_new_slices = slices_needed; + + const int slices_to_keep = kFeatureSliceCount - slices_needed; + const int slices_to_drop = kFeatureSliceCount - slices_to_keep; + // If we can avoid recalculating some slices, just move the existing data + // up in the spectrogram, to perform something like this: + // last time = 80ms current time = 120ms + // +-----------+ +-----------+ + // | data@20ms | --> | data@60ms | + // +-----------+ -- +-----------+ + // | data@40ms | -- --> | data@80ms | + // +-----------+ -- -- +-----------+ + // | data@60ms | -- -- | | + // +-----------+ -- +-----------+ + // | data@80ms | -- | | + // +-----------+ +-----------+ + if (slices_to_keep > 0) { + for (int dest_slice = 0; dest_slice < slices_to_keep; ++dest_slice) { + uint8_t* dest_slice_data = + feature_data_ + (dest_slice * kFeatureSliceSize); + const int src_slice = dest_slice + slices_to_drop; + const uint8_t* src_slice_data = + feature_data_ + (src_slice * kFeatureSliceSize); + for (int i = 0; i < kFeatureSliceSize; ++i) { + dest_slice_data[i] = src_slice_data[i]; + } + } + } + // Any slices that need to be filled in with feature data have their + // appropriate audio data pulled, and features calculated for that slice. + if (slices_needed > 0) { + for (int new_slice = slices_to_keep; new_slice < kFeatureSliceCount; + ++new_slice) { + const int new_step = (current_step - kFeatureSliceCount + 1) + new_slice; + const int32_t slice_start_ms = (new_step * kFeatureSliceStrideMs); + int16_t* audio_samples = nullptr; + int audio_samples_size = 0; + GetAudioSamples(error_reporter, slice_start_ms, kFeatureSliceDurationMs, + &audio_samples_size, &audio_samples); + if (audio_samples_size < kMaxAudioSampleSize) { + error_reporter->Report("Audio data size %d too small, want %d", + audio_samples_size, kMaxAudioSampleSize); + return kTfLiteError; + } + uint8_t* new_slice_data = feature_data_ + (new_slice * kFeatureSliceSize); + TfLiteStatus preprocess_status = + Preprocess(error_reporter, audio_samples, audio_samples_size, + kFeatureSliceSize, new_slice_data); + if (preprocess_status != kTfLiteOk) { + return preprocess_status; + } + } + } + return kTfLiteOk; +} diff --git a/tensorflow/lite/experimental/micro/examples/micro_speech/feature_provider.h b/tensorflow/lite/experimental/micro/examples/micro_speech/feature_provider.h new file mode 100644 index 00000000000..a86c56ebf05 --- /dev/null +++ b/tensorflow/lite/experimental/micro/examples/micro_speech/feature_provider.h @@ -0,0 +1,48 @@ +/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_LITE_EXPERIMENTAL_MICRO_EXAMPLES_MICRO_SPEECH_FEATURE_PROVIDER_H_ +#define TENSORFLOW_LITE_EXPERIMENTAL_MICRO_EXAMPLES_MICRO_SPEECH_FEATURE_PROVIDER_H_ + +#include "tensorflow/lite/c/c_api_internal.h" +#include "tensorflow/lite/experimental/micro/micro_error_reporter.h" + +// Binds itself to an area of memory intended to hold the input features for an +// audio-recognition neural network model, and fills that data area with the +// features representing the current audio input, for example from a microphone. +// The audio features themselves are a two-dimensional array, made up of +// horizontal slices representing the frequencies at one point in time, stacked +// on top of each other to form a spectrogram showing how those frequencies +// changed over time. +class FeatureProvider { + public: + // Create the provider, and bind it to an area of memory. This memory should + // remain accessible for the lifetime of the provider object, since subsequent + // calls will fill it with feature data. The provider does no memory + // management of this data. + FeatureProvider(int feature_size, uint8_t* feature_data); + ~FeatureProvider(); + + // Fills the feature data with information from audio inputs, and returns how + // many feature slices were updated. + TfLiteStatus PopulateFeatureData(tflite::ErrorReporter* error_reporter, + int* how_many_new_slices); + + private: + int feature_size_; + uint8_t* feature_data_; +}; + +#endif // TENSORFLOW_LITE_EXPERIMENTAL_MICRO_EXAMPLES_MICRO_SPEECH_FEATURE_PROVIDER_H_ diff --git a/tensorflow/lite/experimental/micro/examples/micro_speech/feature_provider_test.cc b/tensorflow/lite/experimental/micro/examples/micro_speech/feature_provider_test.cc new file mode 100644 index 00000000000..1e52aec8d27 --- /dev/null +++ b/tensorflow/lite/experimental/micro/examples/micro_speech/feature_provider_test.cc @@ -0,0 +1,38 @@ +/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/lite/experimental/micro/examples/micro_speech/feature_provider.h" +#include "tensorflow/lite/c/c_api_internal.h" +#include "tensorflow/lite/experimental/micro/examples/micro_speech/model_settings.h" +#include "tensorflow/lite/experimental/micro/micro_error_reporter.h" +#include "tensorflow/lite/experimental/micro/testing/micro_test.h" + +TF_LITE_MICRO_TESTS_BEGIN + +TF_LITE_MICRO_TEST(TestFeatureProvider) { + tflite::MicroErrorReporter micro_error_reporter; + tflite::ErrorReporter* error_reporter = µ_error_reporter; + + uint8_t feature_data[kFeatureElementCount]; + FeatureProvider feature_provider(kFeatureElementCount, feature_data); + + int how_many_new_slices = 0; + TfLiteStatus populate_status = feature_provider.PopulateFeatureData( + error_reporter, &how_many_new_slices); + TF_LITE_MICRO_EXPECT_EQ(kTfLiteOk, populate_status); + TF_LITE_MICRO_EXPECT_EQ(kFeatureSliceCount, how_many_new_slices); +} + +TF_LITE_MICRO_TESTS_END diff --git a/tensorflow/lite/experimental/micro/examples/micro_speech/fixed_point/preprocessor.cc b/tensorflow/lite/experimental/micro/examples/micro_speech/fixed_point/preprocessor.cc index de60c982f3a..b623d8d11b7 100644 --- a/tensorflow/lite/experimental/micro/examples/micro_speech/fixed_point/preprocessor.cc +++ b/tensorflow/lite/experimental/micro/examples/micro_speech/fixed_point/preprocessor.cc @@ -31,6 +31,8 @@ limitations under the License. #include +#include "tensorflow/lite/experimental/micro/examples/micro_speech/model_settings.h" + namespace { // q format notation: qx.y => 1 sign bit, x-1 integer bits, y fraction bits. @@ -66,13 +68,6 @@ inline int32_t FloatToFixed_Q2_30(float input) { return static_cast(roundf(input * (1 << 30))); } -// These constants allow us to allocate fixed-sized arrays on the stack for our -// working memory. -constexpr int kInputSize = 512; -constexpr int kAverageWindowSize = 6; -constexpr int kOutputSize = - ((kInputSize / 2) + (kAverageWindowSize - 1)) / kAverageWindowSize; - // Performs a discrete Fourier transform on the real inputs. This corresponds to // rdft() in the FFT package at http://www.kurims.kyoto-u.ac.jp/~ooura/fft.html, // and to kiss_fftr() in KISSFFT at https://github.com/mborgerding/kissfft. @@ -127,14 +122,14 @@ TfLiteStatus Preprocess(tflite::ErrorReporter* error_reporter, const int16_t* input, int input_size, int output_size, uint8_t* output) { // Ensure our input and output data arrays are valid. - if (input_size > kInputSize) { + if (input_size > kMaxAudioSampleSize) { error_reporter->Report("Input size %d larger than %d", input_size, - kInputSize); + kMaxAudioSampleSize); return kTfLiteError; } - if (output_size != kOutputSize) { + if (output_size != kFeatureSliceSize) { error_reporter->Report("Requested output size %d doesn't match %d", - output_size, kOutputSize); + output_size, kFeatureSliceSize); return kTfLiteError; } @@ -142,18 +137,17 @@ TfLiteStatus Preprocess(tflite::ErrorReporter* error_reporter, // In a real application, we'd calculate this table once in an initialization // function and store it for repeated reuse. // q1.15 format. - int16_t window_function[kInputSize]; + int16_t window_function[kMaxAudioSampleSize]; CalculatePeriodicHann(input_size, window_function); // Apply the window function to our time series input, and pad it with zeroes // to the next power of two. - int32_t fixed_input[kInputSize]; - for (int i = 0; i < kInputSize; ++i) { + int32_t fixed_input[kMaxAudioSampleSize]; + for (int i = 0; i < kMaxAudioSampleSize; ++i) { if (i < input_size) { // input is int16_t. Treat as q1.15 fixed point value in range [-1,1) // window_function is also q1.15 fixed point number - fixed_input[i] = - Q1_15_FixedMultiply_Q2_30(input[i], window_function[i]); + fixed_input[i] = Q1_15_FixedMultiply_Q2_30(input[i], window_function[i]); } else { fixed_input[i] = 0; } @@ -161,31 +155,31 @@ TfLiteStatus Preprocess(tflite::ErrorReporter* error_reporter, // Pull the frequency data from the time series sample. // Calculated in q10.22 format from q2.30 inputs. - int32_t fourier_values[kInputSize]; - CalculateDiscreteFourierTransform(fixed_input, kInputSize, fourier_values); + int32_t fourier_values[kMaxAudioSampleSize]; + CalculateDiscreteFourierTransform(fixed_input, kMaxAudioSampleSize, + fourier_values); // We have the complex numbers giving us information about each frequency // band, but all we want to know is how strong each frequency is, so calculate // the squared magnitude by adding together the squares of each component. - int32_t power_spectrum[kInputSize / 2]; - for (int i = 0; i < (kInputSize / 2); ++i) { + int32_t power_spectrum[kMaxAudioSampleSize / 2]; + for (int i = 0; i < (kMaxAudioSampleSize / 2); ++i) { const int32_t real = fourier_values[(i * 2) + 0]; const int32_t imaginary = fourier_values[(i * 2) + 1]; // q10.22 results - power_spectrum[i] = - Q10_22_FixedMultiply_Q10_22(real, real) + - Q10_22_FixedMultiply_Q10_22(imaginary, imaginary); + power_spectrum[i] = Q10_22_FixedMultiply_Q10_22(real, real) + + Q10_22_FixedMultiply_Q10_22(imaginary, imaginary); } // Finally, reduce the size of the output by averaging together six adjacent // frequencies into each slot, producing an array of 43 values. // Power_spectrum numbers are q10.22. Divide by kAverageWindowSize inside // loop to prevent overflow. - for (int i = 0; i < kOutputSize; ++i) { + for (int i = 0; i < kFeatureSliceSize; ++i) { int32_t average = 0; for (int j = 0; j < kAverageWindowSize; ++j) { const int index = (i * kAverageWindowSize) + j; - if (index < (kInputSize / 2)) { + if (index < (kMaxAudioSampleSize / 2)) { average += power_spectrum[index] / kAverageWindowSize; } } diff --git a/tensorflow/lite/experimental/micro/examples/micro_speech/main.cc b/tensorflow/lite/experimental/micro/examples/micro_speech/main.cc new file mode 100644 index 00000000000..1890c25cf2b --- /dev/null +++ b/tensorflow/lite/experimental/micro/examples/micro_speech/main.cc @@ -0,0 +1,112 @@ +/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/lite/experimental/micro/examples/micro_speech/feature_provider.h" +#include "tensorflow/lite/experimental/micro/examples/micro_speech/model_settings.h" +#include "tensorflow/lite/experimental/micro/examples/micro_speech/tiny_conv_model_data.h" +#include "tensorflow/lite/experimental/micro/kernels/all_ops_resolver.h" +#include "tensorflow/lite/experimental/micro/micro_error_reporter.h" +#include "tensorflow/lite/experimental/micro/micro_interpreter.h" +#include "tensorflow/lite/schema/schema_generated.h" +#include "tensorflow/lite/version.h" + +int main(int argc, char* argv[]) { + // Set up logging. + tflite::MicroErrorReporter micro_error_reporter; + tflite::ErrorReporter* error_reporter = µ_error_reporter; + + // Map the model into a usable data structure. This doesn't involve any + // copying or parsing, it's a very lightweight operation. + const tflite::Model* model = ::tflite::GetModel(g_tiny_conv_model_data); + if (model->version() != TFLITE_SCHEMA_VERSION) { + error_reporter->Report( + "Model provided is schema version %d not equal " + "to supported version %d.\n", + model->version(), TFLITE_SCHEMA_VERSION); + return 1; + } + + // This pulls in all the operation implementations we need. + tflite::ops::micro::AllOpsResolver resolver; + + // Create an area of memory to use for input, output, and intermediate arrays. + // The size of this will depend on the model you're using, and may need to be + // determined by experimentation. + const int tensor_arena_size = 10 * 1024; + uint8_t tensor_arena[tensor_arena_size]; + tflite::SimpleTensorAllocator tensor_allocator(tensor_arena, + tensor_arena_size); + + // Build an interpreter to run the model with. + tflite::MicroInterpreter interpreter(model, resolver, &tensor_allocator, + error_reporter); + + // Get information about the memory area to use for the model's input. + TfLiteTensor* model_input = interpreter.input(0); + if ((model_input->dims->size != 4) || (model_input->dims->data[0] != 1) || + (model_input->dims->data[1] != kFeatureSliceCount) || + (model_input->dims->data[2] != kFeatureSliceSize) || + (model_input->type != kTfLiteUInt8)) { + error_reporter->Report("Bad input tensor parameters in model"); + return 1; + } + + // Prepare to access the audio spectrograms from a microphone or other source + // that will provide the inputs to the neural network. + FeatureProvider feature_provider(kFeatureElementCount, + model_input->data.uint8); + + // Keep reading and analysing audio data in an infinite loop. + while (true) { + // Fetch the spectrogram for the current time. + int how_many_new_slices = 0; + TfLiteStatus feature_status = feature_provider.PopulateFeatureData( + error_reporter, &how_many_new_slices); + if (feature_status != kTfLiteOk) { + error_reporter->Report("Feature generation failed"); + return 1; + } + // If no new audio samples have been received since last time, don't bother + // running the network model. + if (how_many_new_slices == 0) { + continue; + } + + // Run the model on the spectrogram input and make sure it succeeds. + TfLiteStatus invoke_status = interpreter.Invoke(); + if (invoke_status != kTfLiteOk) { + error_reporter->Report("Invoke failed"); + return 1; + } + + // The output from the model is a vector containing the scores for each + // kind of prediction, so figure out what the highest scoring category was. + TfLiteTensor* output = interpreter.output(0); + uint8_t top_category_score = 0; + int top_category_index = 0; + for (int category_index = 0; category_index < kCategoryCount; + ++category_index) { + const uint8_t category_score = output->data.uint8[category_index]; + if (category_score > top_category_score) { + top_category_score = category_score; + top_category_index = category_index; + } + } + + error_reporter->Report("Heard %s", kCategoryLabels[top_category_index]); + } + + return 0; +} diff --git a/tensorflow/lite/experimental/micro/examples/micro_speech/CMSIS/no_power_spectrum_data.cc b/tensorflow/lite/experimental/micro/examples/micro_speech/model_settings.cc similarity index 73% rename from tensorflow/lite/experimental/micro/examples/micro_speech/CMSIS/no_power_spectrum_data.cc rename to tensorflow/lite/experimental/micro/examples/micro_speech/model_settings.cc index a3e561a8b51..b9b8fb37b19 100644 --- a/tensorflow/lite/experimental/micro/examples/micro_speech/CMSIS/no_power_spectrum_data.cc +++ b/tensorflow/lite/experimental/micro/examples/micro_speech/model_settings.cc @@ -13,8 +13,11 @@ See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ -// See the header for documentation on the meaning of this data. +#include "tensorflow/lite/experimental/micro/examples/micro_speech/model_settings.h" -#include "tensorflow/lite/experimental/micro/examples/micro_speech/CMSIS/no_power_spectrum_data.h" - -const uint8_t g_no_power_spectrum_data[g_no_power_spectrum_data_size] = {233,6,5,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,4,40}; +const char* kCategoryLabels[kCategoryCount] = { + "silence", + "unknown", + "yes", + "no", +}; diff --git a/tensorflow/lite/experimental/micro/examples/micro_speech/model_settings.h b/tensorflow/lite/experimental/micro/examples/micro_speech/model_settings.h new file mode 100644 index 00000000000..1d8f3123a57 --- /dev/null +++ b/tensorflow/lite/experimental/micro/examples/micro_speech/model_settings.h @@ -0,0 +1,42 @@ +/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_LITE_EXPERIMENTAL_MICRO_EXAMPLES_MICRO_SPEECH_MODEL_SETTINGS_H_ +#define TENSORFLOW_LITE_EXPERIMENTAL_MICRO_EXAMPLES_MICRO_SPEECH_MODEL_SETTINGS_H_ + +// Keeping these as constant expressions allow us to allocate fixed-sized arrays +// on the stack for our working memory. + +// The size of the input time series data we pass to the FFT to produce the +// frequency information. This has to be a power of two, and since we're dealing +// with 30ms of 16KHz inputs, which means 480 samples, this is the next value. +constexpr int kMaxAudioSampleSize = 512; + +// All of these values are derived from the values used during model training, +// if you change your model you'll need to update these constants. +constexpr int kAverageWindowSize = 6; +constexpr int kFeatureSliceSize = + ((kMaxAudioSampleSize / 2) + (kAverageWindowSize - 1)) / kAverageWindowSize; +constexpr int kFeatureSliceCount = 49; +constexpr int kFeatureElementCount = (kFeatureSliceSize * kFeatureSliceCount); +constexpr int kFeatureSliceStrideMs = 20; +constexpr int kFeatureSliceDurationMs = 30; + +constexpr int kCategoryCount = 4; +constexpr int kSilenceIndex = 0; +constexpr int kUnknownIndex = 1; +extern const char* kCategoryLabels[kCategoryCount]; + +#endif // TENSORFLOW_LITE_EXPERIMENTAL_MICRO_EXAMPLES_MICRO_SPEECH_MODEL_SETTINGS_H_ diff --git a/tensorflow/lite/experimental/micro/examples/micro_speech/preprocessor.cc b/tensorflow/lite/experimental/micro/examples/micro_speech/preprocessor.cc index 12f9e22038b..743d2292247 100644 --- a/tensorflow/lite/experimental/micro/examples/micro_speech/preprocessor.cc +++ b/tensorflow/lite/experimental/micro/examples/micro_speech/preprocessor.cc @@ -28,14 +28,9 @@ limitations under the License. #include -namespace { +#include "tensorflow/lite/experimental/micro/examples/micro_speech/model_settings.h" -// These constants allow us to allocate fixed-sized arrays on the stack for our -// working memory. -constexpr int kInputSize = 512; -constexpr int kAverageWindowSize = 6; -constexpr int kOutputSize = - ((kInputSize / 2) + (kAverageWindowSize - 1)) / kAverageWindowSize; +namespace { // Performs a discrete Fourier transform on the real inputs. This corresponds to // rdft() in the FFT package at http://www.kurims.kyoto-u.ac.jp/~ooura/fft.html, @@ -78,27 +73,27 @@ TfLiteStatus Preprocess(tflite::ErrorReporter* error_reporter, const int16_t* input, int input_size, int output_size, uint8_t* output) { // Ensure our input and output data arrays are valid. - if (input_size > kInputSize) { + if (input_size > kMaxAudioSampleSize) { error_reporter->Report("Input size %d larger than %d", input_size, - kInputSize); + kMaxAudioSampleSize); return kTfLiteError; } - if (output_size != kOutputSize) { + if (output_size != kFeatureSliceSize) { error_reporter->Report("Requested output size %d doesn't match %d", - output_size, kOutputSize); + output_size, kFeatureSliceSize); return kTfLiteError; } // Pre-calculate the window function we'll be applying to the input data. // In a real application, we'd calculate this table once in an initialization // function and store it for repeated reuse. - float window_function[kInputSize]; + float window_function[kMaxAudioSampleSize]; CalculatePeriodicHann(input_size, window_function); // Apply the window function to our time series input, and pad it with zeroes // to the next power of two. - float float_input[kInputSize]; - for (int i = 0; i < kInputSize; ++i) { + float float_input[kMaxAudioSampleSize]; + for (int i = 0; i < kMaxAudioSampleSize; ++i) { if (i < input_size) { float_input[i] = (input[i] * window_function[i]) / static_cast(1 << 15); @@ -108,14 +103,15 @@ TfLiteStatus Preprocess(tflite::ErrorReporter* error_reporter, } // Pull the frequency data from the time series sample. - float fourier_values[kInputSize]; - CalculateDiscreteFourierTransform(float_input, kInputSize, fourier_values); + float fourier_values[kMaxAudioSampleSize]; + CalculateDiscreteFourierTransform(float_input, kMaxAudioSampleSize, + fourier_values); // We have the complex numbers giving us information about each frequency // band, but all we want to know is how strong each frequency is, so calculate // the squared magnitude by adding together the squares of each component. - float power_spectrum[kInputSize / 2]; - for (int i = 0; i < (kInputSize / 2); ++i) { + float power_spectrum[kMaxAudioSampleSize / 2]; + for (int i = 0; i < (kMaxAudioSampleSize / 2); ++i) { const float real = fourier_values[(i * 2) + 0]; const float imaginary = fourier_values[(i * 2) + 1]; power_spectrum[i] = (real * real) + (imaginary * imaginary); @@ -123,11 +119,11 @@ TfLiteStatus Preprocess(tflite::ErrorReporter* error_reporter, // Finally, reduce the size of the output by averaging together six adjacent // frequencies into each slot, producing an array of 43 values. - for (int i = 0; i < kOutputSize; ++i) { + for (int i = 0; i < kFeatureSliceSize; ++i) { float total = 0.0f; for (int j = 0; j < kAverageWindowSize; ++j) { const int index = (i * kAverageWindowSize) + j; - if (index < (kInputSize / 2)) { + if (index < (kMaxAudioSampleSize / 2)) { total += power_spectrum[index]; } } @@ -147,3 +143,13 @@ TfLiteStatus Preprocess(tflite::ErrorReporter* error_reporter, } return kTfLiteOk; } + +TfLiteStatus Preprocess_1sec(tflite::ErrorReporter* error_reporter, + const int16_t* input, uint8_t* output) { + int i; + for(i=0; i<49; i++) { + Preprocess(error_reporter, input+i*320, 480, 43, output+i*43); + } + return kTfLiteOk; +} + diff --git a/tensorflow/lite/experimental/micro/examples/micro_speech/preprocessor.h b/tensorflow/lite/experimental/micro/examples/micro_speech/preprocessor.h index dede2a86421..0057b4505f0 100644 --- a/tensorflow/lite/experimental/micro/examples/micro_speech/preprocessor.h +++ b/tensorflow/lite/experimental/micro/examples/micro_speech/preprocessor.h @@ -19,8 +19,16 @@ limitations under the License. #include "tensorflow/lite/c/c_api_internal.h" #include "tensorflow/lite/experimental/micro/micro_error_reporter.h" +// Converts audio sample data into a more compact form that's appropriate for +// feeding into a neural network. There are reference implementations that use +// both floating point and fixed point available, but because the calculations +// involved can be time-consuming, it's recommended that you use or write +// specialized versions for your platform. TfLiteStatus Preprocess(tflite::ErrorReporter* error_reporter, const int16_t* input, int input_size, int output_size, uint8_t* output); +TfLiteStatus Preprocess_1sec(tflite::ErrorReporter* error_reporter, + const int16_t* input, uint8_t* output); + #endif // TENSORFLOW_LITE_EXPERIMENTAL_MICRO_EXAMPLES_MICRO_SPEECH_PREPROCESSOR_H_ diff --git a/tensorflow/lite/experimental/micro/examples/micro_speech/preprocessor_test.cc b/tensorflow/lite/experimental/micro/examples/micro_speech/preprocessor_test.cc index e8b49f67e3d..d9b0c48ba3b 100644 --- a/tensorflow/lite/experimental/micro/examples/micro_speech/preprocessor_test.cc +++ b/tensorflow/lite/experimental/micro/examples/micro_speech/preprocessor_test.cc @@ -22,6 +22,7 @@ limitations under the License. #include "tensorflow/lite/experimental/micro/micro_error_reporter.h" #include "tensorflow/lite/experimental/micro/testing/micro_test.h" + TF_LITE_MICRO_TESTS_BEGIN TF_LITE_MICRO_TEST(TestPreprocessor) { diff --git a/tensorflow/lite/experimental/micro/examples/micro_speech/CMSIS/yes_power_spectrum_data.cc b/tensorflow/lite/experimental/micro/examples/micro_speech/timer.cc similarity index 73% rename from tensorflow/lite/experimental/micro/examples/micro_speech/CMSIS/yes_power_spectrum_data.cc rename to tensorflow/lite/experimental/micro/examples/micro_speech/timer.cc index 89edb510401..6c96a61ab51 100644 --- a/tensorflow/lite/experimental/micro/examples/micro_speech/CMSIS/yes_power_spectrum_data.cc +++ b/tensorflow/lite/experimental/micro/examples/micro_speech/timer.cc @@ -13,8 +13,10 @@ See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ -// See the header for documentation on the meaning of this data. +#include "tensorflow/lite/experimental/micro/examples/micro_speech/timer.h" -#include "tensorflow/lite/experimental/micro/examples/micro_speech/CMSIS/yes_power_spectrum_data.h" - -const uint8_t g_yes_power_spectrum_data[g_yes_power_spectrum_data_size] = {8,88,8,0,0,0,0,0,0,0,0,3,12,0,5,22,19,5,4,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,9,1}; +int32_t TimeInMilliseconds() { + static int current_time = 0; + current_time += 100; + return current_time; +} diff --git a/tensorflow/lite/experimental/micro/examples/micro_speech/timer.h b/tensorflow/lite/experimental/micro/examples/micro_speech/timer.h new file mode 100644 index 00000000000..162952844a8 --- /dev/null +++ b/tensorflow/lite/experimental/micro/examples/micro_speech/timer.h @@ -0,0 +1,31 @@ +/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_LITE_EXPERIMENTAL_MICRO_EXAMPLES_MICRO_SPEECH_TIMER_H_ +#define TENSORFLOW_LITE_EXPERIMENTAL_MICRO_EXAMPLES_MICRO_SPEECH_TIMER_H_ + +#include + +// Returns the time in milliseconds. There's no contract about what time zero +// represents, the accuracy, or the granularity of the result. Subsequent calls +// will generally not return a lower value, but even that's not guaranteed if +// there's an overflow wraparound. +// The reference implementation of this function just returns a constantly +// incrementing value for each call, since it would need a non-portable platform +// call to access time information. For real applications, you'll need to write +// your own platform-specific implementation. +int32_t TimeInMilliseconds(); + +#endif // TENSORFLOW_LITE_EXPERIMENTAL_MICRO_EXAMPLES_MICRO_SPEECH_TIMER_H_ diff --git a/tensorflow/lite/experimental/micro/examples/micro_speech/timer_test.cc b/tensorflow/lite/experimental/micro/examples/micro_speech/timer_test.cc new file mode 100644 index 00000000000..0487a12b25f --- /dev/null +++ b/tensorflow/lite/experimental/micro/examples/micro_speech/timer_test.cc @@ -0,0 +1,49 @@ +/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/lite/experimental/micro/examples/micro_speech/timer.h" + +#include + +#include "tensorflow/lite/c/c_api_internal.h" +#include "tensorflow/lite/experimental/micro/micro_error_reporter.h" +#include "tensorflow/lite/experimental/micro/testing/micro_test.h" + +TF_LITE_MICRO_TESTS_BEGIN + +TF_LITE_MICRO_TEST(TestTimer) { + // Make sure that the technically-undefined overflow behavior we rely on below + // works on this platform. It's still not guaranteed, but at least this is a + // sanity check. Turn off when running with ASan, as it will complain about + // the following undefined behavior. +#ifndef ADDRESS_SANITIZER + int32_t overflow_value = std::numeric_limits::max(); + overflow_value += 1; + TF_LITE_MICRO_EXPECT_EQ(std::numeric_limits::min(), overflow_value); +#endif + + const int32_t first_time = TimeInMilliseconds(); + const int32_t second_time = TimeInMilliseconds(); + + // It's possible that the timer may have wrapped around from +BIG_NUM to + // -BIG_NUM between the first and second calls, since we're storing + // milliseconds in a 32-bit integer. It's not reasonable that the call itself + // would have taken more than 2^31 milliseconds though, so look at the + // difference and rely on integer overflow to ensure it's accurate. + const int32_t time_delta = (second_time - first_time); + TF_LITE_MICRO_EXPECT_LE(0, time_delta); +} + +TF_LITE_MICRO_TESTS_END diff --git a/tensorflow/lite/experimental/micro/testing/micro_test.h b/tensorflow/lite/experimental/micro/testing/micro_test.h index 10bab05faec..2f20dd5ac77 100644 --- a/tensorflow/lite/experimental/micro/testing/micro_test.h +++ b/tensorflow/lite/experimental/micro/testing/micro_test.h @@ -153,4 +153,22 @@ extern tflite::ErrorReporter* reporter; } \ } while (false) +#define TF_LITE_MICRO_EXPECT_GE(x, y) \ + do { \ + if ((x) < (y)) { \ + micro_test::reporter->Report(#x " >= " #y " failed at %s:%d", __FILE__, \ + __LINE__); \ + micro_test::did_test_fail = true; \ + } \ + } while (false) + +#define TF_LITE_MICRO_EXPECT_LE(x, y) \ + do { \ + if ((x) > (y)) { \ + micro_test::reporter->Report(#x " <= " #y " failed at %s:%d", __FILE__, \ + __LINE__); \ + micro_test::did_test_fail = true; \ + } \ + } while (false) + #endif // TENSORFLOW_LITE_EXPERIMENTAL_MICRO_TESTING_MICRO_TEST_H_ diff --git a/tensorflow/lite/experimental/micro/tools/make/Makefile b/tensorflow/lite/experimental/micro/tools/make/Makefile index 463709a6e5c..3f6f9273589 100644 --- a/tensorflow/lite/experimental/micro/tools/make/Makefile +++ b/tensorflow/lite/experimental/micro/tools/make/Makefile @@ -59,13 +59,27 @@ tensorflow/lite/experimental/micro/examples/micro_speech/tiny_conv_model_data.cc tensorflow/lite/experimental/micro/examples/micro_speech/no_features_data.cc \ tensorflow/lite/experimental/micro/examples/micro_speech/yes_features_data.cc +# Test binary for the streaming microcontroller speech model. +PUSHBUTTON_MICRO_SPEECH_TEST_SRCS := \ +tensorflow/lite/experimental/micro/examples/micro_speech/micro_speech_test.cc \ +tensorflow/lite/experimental/micro/examples/micro_speech/tiny_conv_model_data.cc \ +tensorflow/lite/experimental/micro/examples/micro_speech/no_features_data.cc \ +tensorflow/lite/experimental/micro/examples/micro_speech/yes_features_data.cc + +# Test binary for the streaming microcontroller speech model. +PUSHBUTTON_CMSIS_SPEECH_TEST_SRCS := \ +tensorflow/lite/experimental/micro/examples/micro_speech/micro_speech_test.cc \ +tensorflow/lite/experimental/micro/examples/micro_speech/tiny_conv_model_data.cc + # Test binary for the microcontroller speech model. PREPROCESSOR_TEST_SRCS := \ tensorflow/lite/experimental/micro/examples/micro_speech/preprocessor_test.cc \ tensorflow/lite/experimental/micro/examples/micro_speech/no_30ms_sample_data.cc \ tensorflow/lite/experimental/micro/examples/micro_speech/yes_30ms_sample_data.cc \ tensorflow/lite/experimental/micro/examples/micro_speech/no_power_spectrum_data.cc \ -tensorflow/lite/experimental/micro/examples/micro_speech/yes_power_spectrum_data.cc +tensorflow/lite/experimental/micro/examples/micro_speech/yes_power_spectrum_data.cc \ +tensorflow/lite/experimental/micro/examples/micro_speech/yes_waveform.cc \ +tensorflow/lite/experimental/micro/examples/micro_speech/yes_features_data.cc PREPROCESSOR_REFERENCE_TEST_SRCS = \ $(PREPROCESSOR_TEST_SRCS) \ @@ -75,6 +89,11 @@ PREPROCESSOR_FIXED_TEST_SRCS += \ $(PREPROCESSOR_TEST_SRCS) \ tensorflow/lite/experimental/micro/examples/micro_speech/fixed_point/preprocessor.cc +PREPROCESSOR_1K_SRCS := \ +tensorflow/lite/experimental/micro/examples/micro_speech/apollo3/preprocessor_1k.cc \ +tensorflow/lite/experimental/micro/examples/micro_speech/CMSIS/sin_1k.cc + + MICROLITE_TEST_SRCS := \ $(wildcard tensorflow/lite/experimental/micro/*test.cc) \ $(wildcard tensorflow/lite/experimental/micro/kernels/*test.cc) @@ -101,6 +120,8 @@ ALL_SRCS := \ $(PREPROCESSOR_REFERENCE_TEST_SRCS) \ $(PREPROCESSOR_FIXED_TEST_SRCS) \ $(MICROLITE_CC_SRCS) \ + $(PREPROCESSOR_1K_MICRO_TEST_SRCS) \ + $(PREPROCESSOR_1K_CMSIS_TEST_SRCS) \ $(MICROLITE_TEST_SRCS) # Where compiled objects are stored. @@ -112,8 +133,12 @@ LIBDIR := $(GENDIR)lib/ MICROLITE_LIB_PATH := $(LIBDIR)$(MICROLITE_LIB_NAME) MICRO_SPEECH_TEST_BINARY := $(BINDIR)micro_speech_test +PUSHBUTTON_MICRO_SPEECH_TEST_BINARY := $(BINDIR)pushbutton_micro_speech_test +PUSHBUTTON_CMSIS_SPEECH_TEST_BINARY := $(BINDIR)pushbutton_cmsis_speech_test PREPROCESSOR_REFERENCE_TEST_BINARY := $(BINDIR)preprocessor_reference_test PREPROCESSOR_FIXED_TEST_BINARY := $(BINDIR)preprocessor_fixed_test +PREPROCESSOR_1K_MICRO_TEST_BINARY := $(BINDIR)preprocessor_1k_micro_test +PREPROCESSOR_1K_CMSIS_TEST_BINARY := $(BINDIR)preprocessor_1k_cmsis_test CXX := $(CC_PREFIX)${TARGET_TOOLCHAIN_PREFIX}g++ CC := $(CC_PREFIX)${TARGET_TOOLCHAIN_PREFIX}gcc @@ -128,6 +153,13 @@ $(patsubst %.cc,%.o,$(patsubst %.c,%.o,$(PREPROCESSOR_REFERENCE_TEST_SRCS)))) PREPROCESSOR_FIXED_TEST_OBJS := $(addprefix $(OBJDIR), \ $(patsubst %.cc,%.o,$(patsubst %.c,%.o,$(PREPROCESSOR_FIXED_TEST_SRCS)))) +PREPROCESSOR_1K_MICRO_TEST_OBJS := $(addprefix $(OBJDIR), \ +$(patsubst %.cc,%.o,$(patsubst %.c,%.o,$(PREPROCESSOR_1K_MICRO_TEST_SRCS)))) + +PREPROCESSOR_1K_CMSIS_TEST_OBJS := $(addprefix $(OBJDIR), \ +$(patsubst %.cc,%.o,$(patsubst %.c,%.o,$(PREPROCESSOR_1K_CMSIS_TEST_SRCS))) \ +arm_bitreversal2.o) + MICROLITE_LIB_OBJS := $(addprefix $(OBJDIR), \ $(patsubst %.cc,%.o,$(patsubst %.c,%.o,$(MICROLITE_CC_SRCS)))) @@ -194,6 +226,35 @@ preprocessor_fixed_test_bin: $(PREPROCESSOR_FIXED_TEST_BINARY).bin test_preprocessor_fixed: $(PREPROCESSOR_FIXED_TEST_BINARY) $(TEST_SCRIPT) $(PREPROCESSOR_FIXED_TEST_BINARY) '~~~ALL TESTS PASSED~~~' +$(PREPROCESSOR_1K_MICRO_TEST_BINARY): $(PREPROCESSOR_1K_MICRO_TEST_OBJS) $(MICROLITE_LIB_PATH) + @mkdir -p $(dir $@) + $(CXX) $(CXXFLAGS) $(INCLUDES) \ + -o $(PREPROCESSOR_1K_MICRO_TEST_BINARY) $(PREPROCESSOR_1K_MICRO_TEST_OBJS) \ + $(LIBFLAGS) $(MICROLITE_LIB_PATH) $(LDFLAGS) $(MICROLITE_LIBS) + +preprocessor_1k_micro_test: $(PREPROCESSOR_1K_MICRO_TEST_BINARY) +preprocessor_1k_micro_test_bin: $(PREPROCESSOR_1K_MICRO_TEST_BINARY).bin + +test_preprocessor_1k_micro: $(PREPROCESSOR_1K_MICRO_TEST_BINARY) + $(TEST_SCRIPT) $(PREPROCESSOR_1K_MICRO_TEST_BINARY) '~~~ALL TESTS PASSED~~~' + + +$(PREPROCESSOR_1K_CMSIS_TEST_BINARY): $(PREPROCESSOR_1K_CMSIS_TEST_OBJS) $(MICROLITE_LIB_PATH) + @mkdir -p $(dir $@) + $(CXX) $(CXXFLAGS) $(INCLUDES) \ + -o $(PREPROCESSOR_1K_CMSIS_TEST_BINARY) $(PREPROCESSOR_1K_CMSIS_TEST_OBJS) \ + $(LIBFLAGS) $(MICROLITE_LIB_PATH) $(LDFLAGS) $(MICROLITE_LIBS) + +preprocessor_1k_cmsis_test: $(PREPROCESSOR_1K_CMSIS_TEST_BINARY) +preprocessor_1k_cmsis_test_bin: $(PREPROCESSOR_1K_CMSIS_TEST_BINARY).bin + +test_preprocessor_1k_cmsis: $(PREPROCESSOR_1K_CMSIS_TEST_BINARY) + $(TEST_SCRIPT) $(PREPROCESSOR_1K_CMSIS_TEST_BINARY) '~~~ALL TESTS PASSED~~~' + + +$(OBJDIR)arm_bitreversal2.o: + $(CXX) $(CXXFLAGS) $(INCLUDES) -c $(CMSIS_SRC_DIR)/TransformFunctions/arm_bitreversal2.S -o $(OBJDIR)arm_bitreversal2.o + $(BINDIR)%_test : $(OBJDIR)%_test.o $(MICROLITE_LIB_PATH) @mkdir -p $(dir $@) $(CXX) $(CXXFLAGS) $(INCLUDES) \ diff --git a/tensorflow/lite/experimental/micro/tools/make/targets/apollo3evb/README.md b/tensorflow/lite/experimental/micro/tools/make/targets/apollo3evb/README.md deleted file mode 100644 index fec4923e0ec..00000000000 --- a/tensorflow/lite/experimental/micro/tools/make/targets/apollo3evb/README.md +++ /dev/null @@ -1,13 +0,0 @@ -Follow these steps to get the preprocessor test working on Apollo 3: - -1. Download the SDK to the be at the same level as tensorflow.git -2. Copy and prepare files by running tensorflow/lite/experimental/micro/tools/make/targets/apollo3evb/prep_apollo3_files.sh -3. Recompile libarm_cortexM4lf_math.a with the softfp option, and call it libarm_cartexM4lf_math_softfp.a. The original version was compiled with the hard option, and this caused conflicts with existing software. We might be able to fix this in the future -4. Install Segger JLink tools from https://www.segger.com/downloads/jlink/ -5. Compile the preprocessor_test_bin project with the following command: make -f tensorflow/lite/experimental/micro/tools/make/Makefile TARGET=apollo3evb VENDORLIB=cmsis-dsp preprocessor_test_bin -6. Download to the target with JFlashLiteExe with the following settings: - 1. Device = AMA3B1KK-KBR - 2. Interface = SWD at 1000 kHz - 3. Data file = tensorflow/lite/experimental/micro/tools/make/gen/apollo3evb_cortex-m4/bin/preprocessor_test.bin - 4. Prog Addr = 0x0000C000 -7. Connect to device via serial port (115200 baud) and press reset button. Should see all tests passed. Seeing a discrepance between Windows and Linux testing --> need to debug diff --git a/tensorflow/lite/experimental/micro/tools/make/targets/apollo3evb/get_yesno_data.cmd b/tensorflow/lite/experimental/micro/tools/make/targets/apollo3evb/get_yesno_data.cmd deleted file mode 100644 index 71d5389ee30..00000000000 --- a/tensorflow/lite/experimental/micro/tools/make/targets/apollo3evb/get_yesno_data.cmd +++ /dev/null @@ -1,10 +0,0 @@ -file ../../gen/apollo3evb_cortex-m4/bin/preprocessor_test -target remote localhost:2331 -load ../../gen/apollo3evb_cortex-m4/bin/preprocessor_test -monitor reset -break preprocessor_test.cc:35 -break preprocessor_test.cc:51 -c -dump verilog value yes_calculated_data.txt yes_calculated_data -c -dump verilog value no_calculated_data.txt no_calculated_data diff --git a/tensorflow/lite/experimental/micro/tools/make/targets/apollo3evb/prep_apollo3_files.sh b/tensorflow/lite/experimental/micro/tools/make/targets/apollo3evb/prep_apollo3_files.sh index 79ce18f11f5..7ef23095022 100755 --- a/tensorflow/lite/experimental/micro/tools/make/targets/apollo3evb/prep_apollo3_files.sh +++ b/tensorflow/lite/experimental/micro/tools/make/targets/apollo3evb/prep_apollo3_files.sh @@ -14,19 +14,22 @@ # limitations under the License. # ============================================================================== -if [ ! -d "../Apollo3-SDK-2018.08.13" ]; then +AP3_DIR="tensorflow/lite/experimental/micro/tools/make/downloads/Apollo3-SDK-2018.08.13" +if [ ! -d $AP3_DIR ]; then echo "Apollo 3 SDK does not exist" - echo "Either the SDK has not been downloaded, or this script is not being done from the root of the repository" + echo "Either the SDK has not been downloaded, or this script is not being run from the root of the repository" else DEST_DIR="tensorflow/lite/experimental/micro/tools/make/targets/apollo3evb" - AP3_DIR="../Apollo3-SDK-2018.08.13" cp "$AP3_DIR/boards/apollo3_evb/examples/hello_world/gcc/startup_gcc.c" "$DEST_DIR" - cp "$AP3_DIR/utils/am_util_delay.c" "$DEST_DIR" - cp "$AP3_DIR/utils/am_util_faultisr.c" "$DEST_DIR" - cp "$AP3_DIR/utils/am_util_id.c" "$DEST_DIR" - cp "$AP3_DIR/utils/am_util_stdio.c" "$DEST_DIR" - cp "$AP3_DIR/boards/apollo3_evb/bsp/gcc/bin/libam_bsp.a" "$DEST_DIR" - cp "$AP3_DIR/mcu/apollo3/hal/gcc/bin/libam_hal.a" "$DEST_DIR" + cp "$AP3_DIR/boards/apollo3_evb/examples/hello_world/gcc/hello_world.ld" "$DEST_DIR/apollo3evb.ld" sed -i -e '131s/1024/1024\*20/g' "$DEST_DIR/startup_gcc.c" sed -i -e 's/main/_main/g' "$DEST_DIR/startup_gcc.c" + sed -i -e '3s/hello_world.ld/apollo3evb.ld/g' "$DEST_DIR/apollo3evb.ld" + sed -i -e '3s/startup_gnu/startup_gcc/g' "$DEST_DIR/apollo3evb.ld" + sed -i -e '6s/am_reset_isr/Reset_Handler/g' "$DEST_DIR/apollo3evb.ld" + sed -i -e '22s/\*(.text\*)/\*(.text\*)\n\n\t\/\* These are the C++ global constructors. Stick them all here and\n\t \* then walk through the array in main() calling them all.\n\t \*\/\n\t_init_array_start = .;\n\tKEEP (\*(SORT(.init_array\*)))\n\t_init_array_end = .;\n\n\t\/\* XXX Currently not doing anything for global destructors. \*\/\n/g' "$DEST_DIR/apollo3evb.ld" + sed -i -e "70s/} > SRAM/} > SRAM\n \/\* Add this to satisfy reference to symbol 'end' from libnosys.a(sbrk.o)\n \* to denote the HEAP start.\n \*\/\n end = .;/g" "$DEST_DIR/apollo3evb.ld" + echo "Finished preparing Apollo3 files" + + fi diff --git a/tensorflow/lite/experimental/micro/tools/make/targets/apollo3evb/preprocessor_test.cmd b/tensorflow/lite/experimental/micro/tools/make/targets/apollo3evb/preprocessor_test.cmd deleted file mode 100644 index 1b1db457fea..00000000000 --- a/tensorflow/lite/experimental/micro/tools/make/targets/apollo3evb/preprocessor_test.cmd +++ /dev/null @@ -1,4 +0,0 @@ -file ../../gen/apollo3evb_cortex-m4/bin/preprocessor_test -target remote localhost:2331 -load ../../gen/apollo3evb_cortex-m4/bin/preprocessor_test -monitor reset diff --git a/tensorflow/lite/experimental/micro/tools/make/targets/apollo3evb/replace_calculated_data.py b/tensorflow/lite/experimental/micro/tools/make/targets/apollo3evb/replace_calculated_data.py deleted file mode 100644 index 3c1b0110fcd..00000000000 --- a/tensorflow/lite/experimental/micro/tools/make/targets/apollo3evb/replace_calculated_data.py +++ /dev/null @@ -1,33 +0,0 @@ -import numpy as np -import re - -# This should be run from make/targets/apollo3evb - -def new_data_to_array(fn): - vals = [] - with open(fn) as f: - for n, line in enumerate(f): - if n is not 0: - vals.extend([str(int(v, 16)) for v in line.split()]) - - return ','.join(vals) - -def replace_data(fn_old, new_data): - patt = '(?<=\{).+?(?=\})' - with open(fn_old,'r') as f: - str_old = f.read() - str_new = re.sub(patt, new_data, str_old, flags=re.DOTALL) - with open(fn_old,'w') as f: - f.write(str_new) - - -yes_old = '../../../../examples/micro_speech/CMSIS/yes_power_spectrum_data.cc' -no_old = '../../../../examples/micro_speech/CMSIS/no_power_spectrum_data.cc' -yes_new = 'yes_calculated_data.txt' -no_new = 'no_calculated_data.txt' - -yes_new_vals = new_data_to_array(yes_new) -no_new_vals = new_data_to_array(no_new) - -replace_data(yes_old, yes_new_vals) -replace_data(no_old, no_new_vals) diff --git a/tensorflow/lite/experimental/micro/tools/make/targets/apollo3evb_makefile.inc b/tensorflow/lite/experimental/micro/tools/make/targets/apollo3evb_makefile.inc index 153a19fcd4d..3db4367a17c 100644 --- a/tensorflow/lite/experimental/micro/tools/make/targets/apollo3evb_makefile.inc +++ b/tensorflow/lite/experimental/micro/tools/make/targets/apollo3evb_makefile.inc @@ -4,10 +4,10 @@ ifeq ($(TARGET), apollo3evb) TARGET_TOOLCHAIN_PREFIX := arm-none-eabi- # Download the Ambiq Apollo3 SDK and set this variable to find the header # files: - APOLLO3_SDK := ../Apollo3-SDK-2018.08.13 + APOLLO3_SDK := $(MAKEFILE_DIR)/downloads/Apollo3-SDK-2018.08.13 # Need a pointer to the GNU ARM toolchain for crtbegin.o for the fp functions # with the softfp interfaces. - GCC_ARM := ../gcc-arm-none-eabi-7-2018-q2-update/ + GCC_ARM := $(MAKEFILE_DIR)/downloads/gcc-arm-none-eabi-7-2018-q2-update/ PLATFORM_FLAGS = \ -DPART_apollo3 \ @@ -17,6 +17,7 @@ ifeq ($(TARGET), apollo3evb) -DTF_LITE_STATIC_MEMORY \ -DTF_LITE_MCU_DEBUG_LOG \ -D __FPU_PRESENT=1 \ + -DARM_MATH_CM4 \ -fno-rtti \ -fmessage-length=0 \ -fno-exceptions \ @@ -42,7 +43,7 @@ ifeq ($(TARGET), apollo3evb) -fomit-frame-pointer \ -fpermissive \ -nostdlib \ - -g \ + -ggdb \ -O3 CXXFLAGS += $(PLATFORM_FLAGS) CCFLAGS += $(PLATFORM_FLAGS) @@ -56,11 +57,9 @@ ifeq ($(TARGET), apollo3evb) -Wl,-T,$(MAKEFILE_DIR)/targets/apollo3evb/apollo3evb.ld \ -Wl,-Map=$(MAKEFILE_DIR)/gen/$(TARGET).map,--cref BUILD_TYPE := micro - # The apollo3evb libs should be copied from the SDK after building them. MICROLITE_LIBS := \ - $(MAKEFILE_DIR)/targets/apollo3evb/libam_bsp.a \ - $(MAKEFILE_DIR)/targets/apollo3evb/libam_hal.a \ - $(MAKEFILE_DIR)/downloads/cmsis/CMSIS/Lib/GCC/libarm_cortexM4lf_math_softfp.a \ + $(APOLLO3_SDK)/boards/apollo3_evb/bsp/gcc/bin/libam_bsp.a \ + $(APOLLO3_SDK)/mcu/apollo3/hal/gcc/bin/libam_hal.a \ $(GCC_ARM)/lib/gcc/arm-none-eabi/7.3.1/thumb/v7e-m/fpv4-sp/softfp/crtbegin.o \ -lm INCLUDES += \ @@ -83,11 +82,63 @@ ifeq ($(TARGET), apollo3evb) # of the DebugLog interfaces. MICROLITE_CC_SRCS += \ $(MAKEFILE_DIR)/targets/apollo3evb/startup_gcc.c \ - $(MAKEFILE_DIR)/targets/apollo3evb/_main.c \ - $(MAKEFILE_DIR)/targets/apollo3evb/am_util_delay.c \ - $(MAKEFILE_DIR)/targets/apollo3evb/am_util_faultisr.c \ - $(MAKEFILE_DIR)/targets/apollo3evb/am_util_id.c \ - $(MAKEFILE_DIR)/targets/apollo3evb/am_util_stdio.c + $(APOLLO3_SDK)/utils/am_util_delay.c \ + $(APOLLO3_SDK)/utils/am_util_faultisr.c \ + $(APOLLO3_SDK)/utils/am_util_id.c \ + $(APOLLO3_SDK)/utils/am_util_stdio.c + + CMSIS_SRC_DIR := tensorflow/lite/experimental/micro/tools/make/downloads/cmsis/CMSIS/DSP/Source + CMSIS_SRCS := \ + $(CMSIS_SRC_DIR)/BasicMathFunctions/arm_mult_q15.c \ + $(CMSIS_SRC_DIR)/TransformFunctions/arm_rfft_init_q15.c \ + $(CMSIS_SRC_DIR)/TransformFunctions/arm_rfft_q15.c \ + $(CMSIS_SRC_DIR)/TransformFunctions/arm_cfft_q15.c \ + $(CMSIS_SRC_DIR)/TransformFunctions/arm_cfft_radix4_q15.c \ + $(CMSIS_SRC_DIR)/CommonTables/arm_const_structs.c \ + $(CMSIS_SRC_DIR)/CommonTables/arm_common_tables.c \ + $(CMSIS_SRC_DIR)/StatisticsFunctions/arm_mean_q15.c \ + $(CMSIS_SRC_DIR)/StatisticsFunctions/arm_max_q7.c + + AP3_MICRO_DIR := tensorflow/lite/experimental/micro/examples/micro_speech/apollo3 + CMSIS_DIR := tensorflow/lite/experimental/micro/examples/micro_speech/CMSIS + + MICRO_SPEECH_TEST_SRCS += \ + $(AP3_MICRO_DIR)/_main.c + + PREPROCESSOR_1K_CMSIS_TEST_SRCS := \ + $(PREPROCESSOR_1K_SRCS) \ + $(CMSIS_DIR)/preprocessor.cc \ + $(CMSIS_DIR)/arm_cmplx_mag_squared_q10p6.c \ + $(CMSIS_DIR)/hanning.c \ + $(AP3_MICRO_DIR)/system_apollo3.c \ + $(AP3_MICRO_DIR)/_main.c \ + $(CMSIS_SRCS) + + PREPROCESSOR_1K_MICRO_TEST_SRCS := \ + $(PREPROCESSOR_1K_SRCS) \ + tensorflow/lite/experimental/micro/examples/micro_speech/fixed_point/preprocessor.cc \ + tensorflow/lite/experimental/micro/examples/micro_speech/apollo3/system_apollo3.c \ + $(AP3_MICRO_DIR)/_main.c + + PUSHBUTTON_MICRO_SPEECH_TEST_SRCS := \ + $(AP3_MICRO_DIR)/../preprocessor.cc \ + $(AP3_MICRO_DIR)/pushbutton_main.c \ + $(AP3_MICRO_DIR)/pushbutton_test.cc \ + $(AP3_MICRO_DIR)/../tiny_conv_model_data.cc \ + $(APOLLO3_SDK)/devices/am_devices_led.c + + PUSHBUTTON_CMSIS_SPEECH_TEST_SRCS := \ + $(AP3_MICRO_DIR)/pushbutton_main.c \ + $(AP3_MICRO_DIR)/pushbutton_test.cc \ + $(AP3_MICRO_DIR)/../tiny_conv_model_data.cc \ + $(CMSIS_DIR)/preprocessor.cc \ + $(CMSIS_DIR)/arm_cmplx_mag_squared_q10p6.c \ + $(CMSIS_DIR)/hanning.c \ + $(APOLLO3_SDK)/devices/am_devices_led.c \ + $(CMSIS_SRCS) + + PREPROCESSOR_TEST_SRCS += \ + $(AP3_MICRO_DIR)/_main.c TEST_SCRIPT := tensorflow/lite/experimental/log_test/test_apollo3evb_binary.sh # These are tests that don't currently work on the blue pill. @@ -96,4 +147,5 @@ ifeq ($(TARGET), apollo3evb) tensorflow/lite/experimental/micro/simple_tensor_allocator_test.cc MICROLITE_TEST_SRCS := $(filter-out $(EXCLUDED_TESTS), $(MICROLITE_TEST_SRCS)) + endif diff --git a/tensorflow/lite/experimental/micro/tools/make/targets/cmsis-dsp_makefile.inc b/tensorflow/lite/experimental/micro/tools/make/targets/cmsis-dsp_makefile.inc deleted file mode 100644 index 54b33932292..00000000000 --- a/tensorflow/lite/experimental/micro/tools/make/targets/cmsis-dsp_makefile.inc +++ /dev/null @@ -1,9 +0,0 @@ -ifeq ($(VENDORLIB), cmsis-dsp) -PREPROCESSOR_TEST_SRCS := \ -tensorflow/lite/experimental/micro/examples/micro_speech/no_30ms_sample_data.cc \ -tensorflow/lite/experimental/micro/examples/micro_speech/yes_30ms_sample_data.cc \ -tensorflow/lite/experimental/micro/examples/micro_speech/CMSIS/preprocessor.cc \ -tensorflow/lite/experimental/micro/examples/micro_speech/CMSIS/preprocessor_test.cc \ -tensorflow/lite/experimental/micro/examples/micro_speech/CMSIS/no_power_spectrum_data.cc \ -tensorflow/lite/experimental/micro/examples/micro_speech/CMSIS/yes_power_spectrum_data.cc -endif diff --git a/tensorflow/lite/experimental/microfrontend/python/kernel_tests/audio_microfrontend_op_test.py b/tensorflow/lite/experimental/microfrontend/python/kernel_tests/audio_microfrontend_op_test.py index 020d40bc13c..561f5f7a50e 100644 --- a/tensorflow/lite/experimental/microfrontend/python/kernel_tests/audio_microfrontend_op_test.py +++ b/tensorflow/lite/experimental/microfrontend/python/kernel_tests/audio_microfrontend_op_test.py @@ -110,7 +110,7 @@ class AudioFeatureGenerationTest(tf.test.TestCase): left_context=1, right_context=1) self.assertAllEqual( - filterbanks.eval(), + self.evaluate(filterbanks), [[479, 425, 479, 425, 436, 378], [479, 425, 436, 378, 410, 350], [436, 378, 410, 350, 391, 325], [410, 350, 391, 325, 391, 325]]) @@ -153,7 +153,7 @@ class AudioFeatureGenerationTest(tf.test.TestCase): frame_stride=3, zero_padding=True) self.assertAllEqual( - filterbanks.eval(), + self.evaluate(filterbanks), [[0, 0, 0, 0, 479, 425], [436, 378, 410, 350, 391, 325], [374, 308, 362, 292, 352, 275]]) diff --git a/tensorflow/lite/experimental/writer/BUILD b/tensorflow/lite/experimental/writer/BUILD index 506c668cf2c..57ce6363671 100644 --- a/tensorflow/lite/experimental/writer/BUILD +++ b/tensorflow/lite/experimental/writer/BUILD @@ -1,6 +1,9 @@ -package(default_visibility = [ - "//visibility:public", -]) +package( + default_visibility = [ + "//visibility:public", + ], + features = ["-parse_headers"], +) licenses(["notice"]) # Apache 2.0 diff --git a/tensorflow/lite/experimental/writer/option_writer_generator.cc b/tensorflow/lite/experimental/writer/option_writer_generator.cc index 036809e94ab..fa360a2f47e 100644 --- a/tensorflow/lite/experimental/writer/option_writer_generator.cc +++ b/tensorflow/lite/experimental/writer/option_writer_generator.cc @@ -56,6 +56,7 @@ static const char* param_structs[] = {"TfLiteConvParams", "TfLiteTransposeParams", "TfLiteReducerParams", "TfLiteSplitParams", + "TfLiteSplitVParams", "TfLiteSqueezeParams", "TfLiteStridedSliceParams", "TfLiteArgMaxParams", @@ -66,6 +67,8 @@ static const char* param_structs[] = {"TfLiteConvParams", "TfLiteFakeQuantParams", "TfLitePackParams", "TfLiteOneHotParams", + "TfLiteLeakyReluParams", + "TfLiteMirrorPaddingParams", nullptr}; } // namespace @@ -152,6 +155,7 @@ class OpOptionData { op_to_option_["BIDIRECTIONAL_SEQUENCE_RNN"] = "SequenceRNNOptions"; op_to_option_["UNIDIRECTIONAL_SEQUENCE_RNN"] = "SequenceRNNOptions"; op_to_option_["UNIDIRECTIONAL_SEQUENCE_RNN"] = "SequenceRNNOptions"; + op_to_option_["MIRROR_PAD"] = ""; // TODO(karimnosseir): MirrorPadOptions. // Manually specified mappings between ops and options (none) op_to_option_["EMBEDDING_LOOKUP"] = ""; // TODO(aselle): maybe something else. diff --git a/tensorflow/lite/g3doc/_book.yaml b/tensorflow/lite/g3doc/_book.yaml index ab0d186848f..a51c7a667f3 100644 --- a/tensorflow/lite/g3doc/_book.yaml +++ b/tensorflow/lite/g3doc/_book.yaml @@ -3,11 +3,11 @@ upper_tabs: - include: /_upper_tabs_left.yaml - include: /api_docs/_upper_tabs_api.yaml # Dropdown menu -- name: Ecosystem - path: /ecosystem +- name: Resources + path: /resources is_default: true menu: - - include: /ecosystem/_menu_toc.yaml + - include: /resources/_menu_toc.yaml lower_tabs: # Subsite tabs other: diff --git a/tensorflow/lite/g3doc/_index.yaml b/tensorflow/lite/g3doc/_index.yaml index 093f86b5420..1b3f1d616ae 100644 --- a/tensorflow/lite/g3doc/_index.yaml +++ b/tensorflow/lite/g3doc/_index.yaml @@ -189,7 +189,7 @@ landing_page: - label: Read more path: https://cloud.google.com/blog/products/ai-machine-learning/ai-motion-designing-simple-system-see-understand-and-react-real-world-part-ii - heading: "Introducing the Model Optimization Toolkit" - image_path: /ecosystem/images/tf-logo-card-16x9.png + image_path: /resources/images/tf-logo-card-16x9.png path: https://medium.com/tensorflow/introducing-the-model-optimization-toolkit-for-tensorflow-254aca1ba0a3 buttons: - label: Read on TensorFlow blog @@ -205,7 +205,7 @@ landing_page: background: grey items: - heading: "Using TensorFlow Lite on Android" - image_path: /ecosystem/images/tf-logo-card-16x9.png + image_path: /resources/images/tf-logo-card-16x9.png path: https://medium.com/tensorflow/using-tensorflow-lite-on-android-9bbc9cb7d69d buttons: - label: Read on TensorFlow blog @@ -216,7 +216,7 @@ landing_page: - label: Watch the video path: https://www.youtube.com/watch?v=FAMfy7izB6A - heading: "TensorFlow Lite on GitHub" - image_path: /ecosystem/images/github-card-16x9.png + image_path: /resources/images/github-card-16x9.png path: https://github.com/tensorflow/tensorflow/tree/master/tensorflow/lite buttons: - label: View on GitHub diff --git a/tensorflow/lite/g3doc/convert/index.md b/tensorflow/lite/g3doc/convert/index.md index bc92a1c1a11..60fa265c295 100644 --- a/tensorflow/lite/g3doc/convert/index.md +++ b/tensorflow/lite/g3doc/convert/index.md @@ -6,14 +6,20 @@ file used by the TensorFlow Lite interpreter. ## From model training to device deployment After a TensorFlow model is trained, the TensorFlow Lite converter uses that -model to generate a TensorFlow Lite [FlatBuffer](https://google.github.io/flatbuffers/) -file (`.tflite`). The converter supports as input: +model to generate a TensorFlow Lite +[FlatBuffer](https://google.github.io/flatbuffers/) file (`.tflite`). The +converter supports as input: [SavedModels](https://www.tensorflow.org/guide/saved_model#using_savedmodel_with_estimators), frozen graphs (models generated by [freeze_graph.py](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/python/tools/freeze_graph.py)), -and `tf.keras` models. The TensorFlow Lite `FlatBuffer` file is deployed to a -client device (generally a mobile or embedded device), and the TensorFlow Lite +and `tf.keras` HDF5 models. The TensorFlow Lite `FlatBuffer` file is deployed to +a client device (generally a mobile or embedded device), and the TensorFlow Lite interpreter uses the compressed model for on-device inference. This conversion process is shown in the diagram below: ![TFLite converter workflow](../images/convert/workflow.svg) + +The TensorFlow Lite Converter can be used either from [Python](python_api.md) or +from the [command line](cmdline_examples.md). This allows you to integrate the +conversion step into the model design workflow, ensuring the model is easy to +convert to a mobile inference graph. diff --git a/tensorflow/lite/g3doc/convert/python_api.md b/tensorflow/lite/g3doc/convert/python_api.md index 4bdf0d8cbe8..b914a34fa87 100644 --- a/tensorflow/lite/g3doc/convert/python_api.md +++ b/tensorflow/lite/g3doc/convert/python_api.md @@ -3,10 +3,9 @@ This page provides examples on how to use the TensorFlow Lite Converter and the TensorFlow Lite interpreter using the Python API. -Note: TFLite recently moved from `tf.contrib.lite` to `tf.lite`. If you are -using tensorflow `r1.12` or earlier you will need to add `.contrib` to the -commands below. `tf.lite` works with newer builds, like the nightly build, -which can be installed with: `pip install tf-nightly` +Note: These docs describe the converter in the TensorFlow nightly release, +installed using `pip install tf-nightly`. For docs describing older versions +reference ["Converting models from TensorFlow 1.12"](#pre_tensorflow_1.12). [TOC] @@ -24,11 +23,6 @@ The API for converting TensorFlow models to TensorFlow Lite as of TensorFlow 1.9 is `tf.lite.TFLiteConverter`. The API for calling the Python intepreter is `tf.lite.Interpreter`. -Note: Reference "Additional Instructions" sections for converting TensorFlow -models to TensorFlow Lite -[in TensorFlow 1.9 to TensorFlow 1.11](#pre_tensorflow_1.11) and -[prior to TensorFlow 1.9](#pre_tensorflow_1.9) - `TFLiteConverter` provides class methods based on the original format of the model. `TFLiteConverter.from_session()` is available for GraphDefs. `TFLiteConverter.from_saved_model()` is available for SavedModels. @@ -250,14 +244,13 @@ either install the nightly build with [Docker](https://www.tensorflow.org/install/docker), or [build the pip package from source](https://www.tensorflow.org/install/source). -### Converting models in TensorFlow 1.9 to TensorFlow 1.11 +### Converting models from TensorFlow 1.12 -To convert TensorFlow models to TensorFlow Lite in TensorFlow 1.9 through -TensorFlow 1.11, use `TocoConverter`. `TocoConverter` is semantically -identically to `TFLiteConverter`. +Reference the following table to convert TensorFlow models to TensorFlow Lite in +and before TensorFlow 1.12. Run `help()` to get details of each API. -### Converting models prior to TensorFlow 1.9 - -To convert TensorFlow models to TensorFlow Lite in TensorFlow 1.7 and TensorFlow -1.8, use the `toco_convert` function. Run `help(tf.lite.toco_convert)` -to get details about accepted parameters. +TensorFlow Version | Python API +------------------ | --------------------------------- +1.12 | `tf.contrib.lite.TFLiteConverter` +1.9-1.11 | `tf.contrib.lite.TocoConverter` +1.7-1.8 | `tf.contrib.lite.toco_convert` diff --git a/tensorflow/lite/g3doc/devguide.md b/tensorflow/lite/g3doc/devguide.md index 270cb8ce378..fdd02638f9b 100644 --- a/tensorflow/lite/g3doc/devguide.md +++ b/tensorflow/lite/g3doc/devguide.md @@ -35,7 +35,7 @@ by suggesting contextually relevant messages. The model is built specifically fo memory constrained devices, such as watches and phones, and has been successfully used in Smart Replies on Android Wear. Currently, this model is Android-specific. -These pre-trained models are [available for download](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/lite/g3doc/models.md) +These pre-trained models are [available for download](models.md). ### Re-train Inception-V3 or MobileNet for a custom data set @@ -57,51 +57,59 @@ A developer may choose to train a custom model using Tensorflow (see the [TensorFlow tutorials](../tutorials/) for examples of building and training models). If you have already written a model, the first step is to export this to a `tf.GraphDef` file. This is required because some formats do not store the -model structure outside the code, and we must communicate with other parts of the -framework. See -[Exporting the Inference Graph](https://github.com/tensorflow/models/blob/master/research/slim/README.md) -to create .pb file for the custom model. +model structure outside the code, and we must communicate with other parts of +the framework. See +[Exporting the Inference Graph](https://www.tensorflow.org/tutorials/keras/save_and_restore_models#save_the_entire_model) +to create file for the custom model. -TensorFlow Lite currently supports a subset of TensorFlow operators. Refer to the -[TensorFlow Lite & TensorFlow Compatibility Guide](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/lite/g3doc/tf_ops_compatibility.md) +TensorFlow Lite currently supports a subset of TensorFlow operators. Refer to +the [TensorFlow Lite & TensorFlow Compatibility Guide](tf_ops_compatibility.md) for supported operators and their usage. This set of operators will continue to grow in future Tensorflow Lite releases. - ## 2. Convert the model format -The model generated (or downloaded) in the previous step is a *standard* -Tensorflow model and you should now have a .pb or .pbtxt `tf.GraphDef` file. -Models generated with transfer learning (re-training) or custom models must be -converted—but, we must first freeze the graph to convert the model to the -Tensorflow Lite format. This process uses several model formats: +The [TensorFlow Lite Converter](convert/index.md) accepts the following file +formats: -* `tf.GraphDef` (.pb) —A protobuf that represents the TensorFlow training or - computation graph. It contains operators, tensors, and variables definitions. -* *CheckPoint* (.ckpt) —Serialized variables from a TensorFlow graph. Since this - does not contain a graph structure, it cannot be interpreted by itself. -* `FrozenGraphDef` —A subclass of `GraphDef` that does not contain - variables. A `GraphDef` can be converted to a `FrozenGraphDef` by taking a - CheckPoint and a `GraphDef`, and converting each variable into a constant - using the value retrieved from the CheckPoint. -* `SavedModel` —A `GraphDef` and CheckPoint with a signature that labels - input and output arguments to a model. A `GraphDef` and CheckPoint can be - extracted from a `SavedModel`. -* *TensorFlow Lite model* (.tflite) —A serialized - [FlatBuffer](https://google.github.io/flatbuffers/) that contains TensorFlow - Lite operators and tensors for the TensorFlow Lite interpreter, similar to a - `FrozenGraphDef`. +* `SavedModel` — A `GraphDef` and checkpoint with a signature that labels + input and output arguments to a model. See the documentation for converting + SavedModels using [Python](convert/python_api.md#basic_savedmodel) or using + the [command line](convert/cmdline_examples.md#savedmodel). +* `tf.keras` - A HDF5 file containing a model with weights and input and + output arguments generated by `tf.Keras`. See the documentation for + converting HDF5 models using + [Python](convert/python_api.md#basic_keras_file) or using the + [command line](convert/cmdline_examples.md#keras). +* `frozen tf.GraphDef` — A subclass of `tf.GraphDef` that does not contain + variables. A `GraphDef` can be converted to a `frozen GraphDef` by taking a + checkpoint and a `GraphDef`, and converting each variable into a constant + using the value retrieved from the checkpoint. Instructions on converting a + `tf.GraphDef` to a TensorFlow Lite model are described in the next + subsection. -### Freeze Graph +### Converting a tf.GraphDef -To use the `GraphDef` .pb file with TensorFlow Lite, you must have checkpoints -that contain trained weight parameters. The .pb file only contains the structure -of the graph. The process of merging the checkpoint values with the graph -structure is called *freezing the graph*. +TensorFlow models may be saved as a .pb or .pbtxt `tf.GraphDef` file. In order +to convert the `tf.GraphDef` file to TensorFlow Lite, the model must first be +frozen. This process invovles several file formats including the `frozen +GraphDef`: -You should have a checkpoints folder or download them for a pre-trained model -(for example, -[MobileNets](https://github.com/tensorflow/models/blob/master/research/slim/nets/mobilenet_v1.md)). +* `tf.GraphDef` (.pb or .pbtxt) — A protobuf that represents the TensorFlow + training or computation graph. It contains operators, tensors, and variables + definitions. +* *checkpoint* (.ckpt) — Serialized variables from a TensorFlow graph. Since + this does not contain a graph structure, it cannot be interpreted by itself. +* *TensorFlow Lite model* (.tflite) — A serialized + [FlatBuffer](https://google.github.io/flatbuffers/) that contains TensorFlow + Lite operators and tensors for the TensorFlow Lite interpreter. + +You must have checkpoints that contain trained weights. The `tf.GraphDef` file +only contains the structure of the graph. The process of merging the checkpoint +values with the graph structure is called *freezing the graph*. + +`tf.GraphDef` and checkpoint files for MobileNet models are available +[here](https://github.com/tensorflow/models/blob/master/research/slim/nets/mobilenet_v1.md). To freeze the graph, use the following command (changing the arguments): @@ -113,69 +121,53 @@ freeze_graph --input_graph=/tmp/mobilenet_v1_224.pb \ --output_node_names=MobileNetV1/Predictions/Reshape_1 ``` -The `input_binary` flag must be enabled so the protobuf is read and written in -a binary format. Set the `input_graph` and `input_checkpoint` files. +Set the `input_binary` flag to `True` when reading a binary protobuf, a `.pb` +file. Set to `False` for a `.pbtxt` file. -The `output_node_names` may not be obvious outside of the code that built the -model. The easiest way to find them is to visualize the graph, either with -[TensorBoard](https://codelabs.developers.google.com/codelabs/tensorflow-for-poets-2/#3) -or `graphviz`. +Set `input_graph` and `input_checkpoint` to the respective filenames. The +`output_node_names` may not be obvious outside of the code that built the model. +The easiest way to find them is to visualize the graph, either with +[TensorBoard](https://www.tensorflow.org/guide/summaries_and_tensorboard) or +`graphviz`. The frozen `GraphDef` is now ready for conversion to the `FlatBuffer` format -(.tflite) for use on Android or iOS devices. For Android, the Tensorflow -Optimizing Converter tool supports both float and quantized models. To convert -the frozen `GraphDef` to the .tflite format: +(.tflite) for use on Android or iOS devices. For Android, the TensorFlow Lite +Converter tool supports both float and quantized models. To convert the frozen +`GraphDef` to the .tflite format use a command similar to the following: ``` -toco --input_file=$(pwd)/mobilenet_v1_1.0_224/frozen_graph.pb \ - --input_format=TENSORFLOW_GRAPHDEF \ - --output_format=TFLITE \ +tflite_convert \ --output_file=/tmp/mobilenet_v1_1.0_224.tflite \ - --inference_type=FLOAT \ - --input_type=FLOAT \ + --graph_def_file=/tmp/mobilenet_v1_0.50_128/frozen_graph.pb \ --input_arrays=input \ - --output_arrays=MobilenetV1/Predictions/Reshape_1 \ - --input_shapes=1,224,224,3 + --output_arrays=MobilenetV1/Predictions/Reshape_1 ``` -The `input_file` argument should reference the frozen `GraphDef` file -containing the model architecture. The [frozen_graph.pb](https://storage.googleapis.com/download.tensorflow.org/models/mobilenet_v1_1.0_224_frozen.tgz) -file used here is available for download. `output_file` is where the TensorFlow -Lite model will get generated. The `input_type` and `inference_type` -arguments should be set to `FLOAT`, unless converting a -quantized model. -Setting the `input_array`, `output_array`, and `input_shape` arguments are not as -straightforward. The easiest way to find these values is to explore the graph -using Tensorboard. Reuse the arguments for specifying the output nodes for -inference in the `freeze_graph` step. +The +[frozen_graph.pb](https://storage.googleapis.com/download.tensorflow.org/models/mobilenet_v1_1.0_224_frozen.tgz) +file used here is available for download. Setting the `input_array` and +`output_array` arguments is not straightforward. The easiest way to find these +values is to explore the graph using +[TensorBoard](https://www.tensorflow.org/guide/summaries_and_tensorboard). Reuse +the arguments for specifying the output nodes for inference in the +`freeze_graph` step. -It is also possible to use the Tensorflow Optimizing Converter with protobufs -from either Python or from the command line (see the -[toco_from_protos.py](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/lite/toco/python/toco_from_protos.py) -example). This allows you to integrate the conversion step into the model design -workflow, ensuring the model is easily convertible to a mobile inference graph. -For example: +### Full converter reference -```python -import tensorflow as tf +The [TensorFlow Lite Converter](convert/index.md) can be +[Python](convert/python_api.md) or from the +[command line](convert/cmdline_examples.md). This allows you to integrate the +conversion step into the model design workflow, ensuring the model is easy to +convert to a mobile inference graph. -img = tf.placeholder(name="img", dtype=tf.float32, shape=(1, 64, 64, 3)) -val = img + tf.constant([1., 2., 3.]) + tf.constant([1., 4., 4.]) -out = tf.identity(val, name="out") +### Ops compatibility -with tf.Session() as sess: - tflite_model = tf.lite.toco_convert(sess.graph_def, [img], [out]) - open("converteds_model.tflite", "wb").write(tflite_model) -``` - -For usage, see the Tensorflow Optimizing Converter -[command-line examples](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/lite/toco/g3doc/cmdline_examples.md). - -Refer to the -[Ops compatibility guide](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/lite/g3doc/tf_ops_compatibility.md) -for troubleshooting help, and if that doesn't help, please +Refer to the [ops compatibility guide](tf_ops_compatibility.md) for +troubleshooting help, and if that doesn't help, please [file an issue](https://github.com/tensorflow/tensorflow/issues). +### Graph vizualization tool + The [development repo](https://github.com/tensorflow/tensorflow) contains a tool to visualize TensorFlow Lite models after conversion. To build the [visualize.py](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/lite/tools/visualize.py) @@ -212,8 +204,8 @@ installing TensorFlow on Android and setting up `bazel` and Android Studio. ### iOS To integrate a TensorFlow model in an iOS app, see the -[TensorFlow Lite for iOS](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/lite/g3doc/ios.md) -guide and iOS demo guide. +[TensorFlow Lite for iOS](ios.md) guide and iOS demo +guide. #### Core ML support @@ -227,6 +219,5 @@ devices. To use the converter, refer to the ### Raspberry Pi Compile Tensorflow Lite for a Raspberry Pi by following the -[RPi build instructions](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/lite/g3doc/rpi.md) -This compiles a static library file (`.a`) used to build your app. There are -plans for Python bindings and a demo app. +[RPi build instructions](rpi.md) This compiles a static library file (`.a`) used +to build your app. There are plans for Python bindings and a demo app. diff --git a/tensorflow/lite/g3doc/models.md b/tensorflow/lite/g3doc/models.md index 537e285490f..62b3f17c79a 100644 --- a/tensorflow/lite/g3doc/models.md +++ b/tensorflow/lite/g3doc/models.md @@ -76,8 +76,11 @@ Mobilenet_V1_1.0_128_quant | [paper](https://arxiv.org/pdf/1712.05877.pdf), [tf Mobilenet_V1_1.0_160_quant | [paper](https://arxiv.org/pdf/1712.05877.pdf), [tflite&pb](http://download.tensorflow.org/models/mobilenet_v1_2018_08_02/mobilenet_v1_1.0_160_quant.tgz) | 4.3 Mb | 66.9% | 86.7% | 37.4 ms Mobilenet_V1_1.0_192_quant | [paper](https://arxiv.org/pdf/1712.05877.pdf), [tflite&pb](http://download.tensorflow.org/models/mobilenet_v1_2018_08_02/mobilenet_v1_1.0_192_quant.tgz) | 4.3 Mb | 69.1% | 88.1% | 51.9 ms Mobilenet_V1_1.0_224_quant | [paper](https://arxiv.org/pdf/1712.05877.pdf), [tflite&pb](http://download.tensorflow.org/models/mobilenet_v1_2018_08_02/mobilenet_v1_1.0_224_quant.tgz) | 4.3 Mb | 70.0% | 89.0% | 70.2 ms -Mobilenet_v2_1.0_224_quant | [paper](https://arxiv.org/abs/1806.08342), [tflite&pb](http://download.tensorflow.org/models/tflite_11_05_08/mobilenet_v2_1.0_224_quant.tgz) | 3.4 Mb | 70.8% | 89.9% | 80.3 ms -Inception_v3_quant | [paper](https://arxiv.org/abs/1806.08342),[tflite&pb](http://download.tensorflow.org/models/tflite_11_05_08/inception_v3_quant.tgz) | 23 Mb | 77.5% | 93.7% | 637 ms +Mobilenet_V2_1.0_224_quant | [paper](https://arxiv.org/abs/1806.08342), [tflite&pb](http://download.tensorflow.org/models/tflite_11_05_08/mobilenet_v2_1.0_224_quant.tgz) | 3.4 Mb | 70.8% | 89.9% | 80.3 ms +Inception_V1_quant | [paper](https://arxiv.org/abs/1409.4842), [tflite&pb](http://download.tensorflow.org/models/inception_v1_224_quant_20181026.tgz) | 6.4 Mb | 70.1% | 89.8% | 154.5 ms +Inception_V2_quant | [paper](https://arxiv.org/abs/1512.00567), [tflite&pb](http://download.tensorflow.org/models/inception_v2_224_quant_20181026.tgz) | 11 Mb | 73.5% | 91.4% | 235.0 ms +Inception_V3_quant | [paper](https://arxiv.org/abs/1806.08342),[tflite&pb](http://download.tensorflow.org/models/tflite_11_05_08/inception_v3_quant.tgz) | 23 Mb | 77.5% | 93.7% | 637 ms +Inception_V4_quant | [paper](https://arxiv.org/abs/1602.07261), [tflite&pb](http://download.tensorflow.org/models/inception_v4_299_quant_20181026.tgz) | 41 Mb | 79.5% | 93.9% | 1250.8 ms ## Other models diff --git a/tensorflow/lite/g3doc/tf_ops_compatibility.md b/tensorflow/lite/g3doc/tf_ops_compatibility.md index b0dfb0fed1f..dcfda72137c 100644 --- a/tensorflow/lite/g3doc/tf_ops_compatibility.md +++ b/tensorflow/lite/g3doc/tf_ops_compatibility.md @@ -1,4 +1,3 @@ - # TensorFlow Lite & TensorFlow Compatibility Guide TensorFlow Lite supports a number of TensorFlow operations used in common @@ -75,6 +74,7 @@ counterparts: 0D tensor* * [tf.squeeze](https://www.tensorflow.org/api_docs/python/tf/squeeze) - *as long as axis is not provided* +* [tf.squared_difference](https://www.tensorflow.org/versions/master/api_docs/python/tf/squared_difference) * [tf.strided_slice](https://www.tensorflow.org/api_docs/python/tf/strided_slice) - *as long as ellipsis_mask and new_axis_mask are not used* * [tf.transpose](https://www.tensorflow.org/versions/master/api_docs/python/tf/transpose) - @@ -139,6 +139,17 @@ following common ops are not supported at the moment: The following TensorFlow Lite operations are fully supported and used in place of the TensorFlow operations listed above: +**ABS** + +``` +Inputs { + 0: a tensor +} +Outputs { + 0: elementwise abs of the input +} +``` + **ADD** ``` @@ -154,6 +165,30 @@ Options { } ``` +**ARG_MAX** + +``` +Inputs { + 0: a tensor + 1: a tensor +} +Outputs { + 0: A tensor of indices of maximum values. +} +``` + +**ARG_MIN** + +``` +Inputs { + 0: a tensor + 1: a tensor +} +Outputs { + 0: A tensor of indices of minium values. +} +``` + **AVERAGE_POOL_2D** ``` @@ -280,6 +315,18 @@ Outputs { } ``` +**FILL** + +``` +Inputs { + 0: a 1D tensor + 1: a 0D (scalar) tensor +} +Outputs { + 0: A tensor of shape `tensor 0` filled with the value in `tensor 1`. +} +``` + **FLOOR** ``` @@ -291,6 +338,30 @@ outputs: { } ``` +**FLOOR_DIV** + +``` +Inputs { + 0: a tensor + 1: a tensor +} +Outputs { + 0: result of computing element-wise floor of `tensor 0` divided by `tensor 1`. +} +``` + +**FLOOR_MOD** + +``` +Inputs { + 0: a tensor + 1: a tensor +} +Outputs { + 0: result of computing element-wise floor of `tensor 0` modulo `tensor 1`. +} +``` + **FULLY_CONNECTED** ``` @@ -378,6 +449,34 @@ Options { } ``` +**LEAKY_RELU** + +``` +Inputs { + 0: a tensor +} +Outputs { + 0: a tensor equivalent to max(input, input * alpha) +} +Options { + alpha: slope of the activation at x < 0 (provided alpha <= 1) +} +``` + +**LEAKY_RELU** + +``` +Inputs { + 0: a tensor +} +Outputs { + 0: a tensor equivalent to max(input, input * alpha) +} +Options { + alpha +} +``` + **LESS** ``` @@ -421,6 +520,18 @@ Options { } ``` +**LOGICAL_OR** + +``` +Inputs { + 0: a list of tensors. + 1: a list of tensors. +} +Outputs { + 0: A tensor of logical_or output tensors. +} +``` + **LOGISTIC** ``` @@ -498,6 +609,18 @@ Outputs { } ``` +**PACK** + +``` +Inputs { + 0: a list of tensors. + 1: an integer. +} +Outputs { + 0: A tensor of stacked tensors. +} +``` + **PAD** ``` @@ -539,6 +662,35 @@ Outputs { } ``` +**POW** + +``` +Inputs { + 0: a tensor + 1: a tensor +} +Outputs { + 0: elementwise pow of the input tensors +} +``` + +**RANGE** + +``` +Inputs { + 0: a 0D (scalar) tensor + 1: a 0D (scalar) tensor + 2: a 0D (scalar) tensor +} +Outputs { + 0: A 1D tensor of type `dtype` defined by a sequence where `tensor 0` is the + start, `tensor 1` is the limit, and `tensor 2` is the delta. +} +Options { + dtype +} +``` + **RELU** ``` @@ -587,6 +739,22 @@ Options { } ``` +**RESIZE_NEAREST_NEIGHBOR** + +``` +Inputs { + 0: a 4D tensor + 1: a 1D tensor with 2 elements +} +Outputs { + 0: A tensor of type `tensor 0` resized according to `tensor 1` heigh/width values + using nearest neighbors interpolation. +} +Options { + align_corners +} +``` + **RSQRT** ``` @@ -698,6 +866,22 @@ Options { } ``` +**SPLIT_V** + +``` +Inputs { + 0: tensor (input) + 1: 1-D tensor (size_splits) + 2: 0-D tensor (axis) +} +Outputs { + 0-N: subtensors built from the input tensors +} +Options { + num_splits: Specifies number of outputs +} +``` + **SQRT** ``` @@ -781,66 +965,6 @@ Outputs { } ``` -**POW** - -``` -Inputs { - 0: a tensor - 1: a tensor -} -Outputs { - 0: elementwise pow of the input tensors -} -``` - -**ARG_MAX** - -``` -Inputs { - 0: a tensor - 1: a tensor -} -Outputs { - 0: A tensor of indices of maximum values. -} -``` - -**ARG_MIN** - -``` -Inputs { - 0: a tensor - 1: a tensor -} -Outputs { - 0: A tensor of indices of minium values. -} -``` - -**PACK** - -``` -Inputs { - 0: a list of tensors. - 1: an integer. -} -Outputs { - 0: A tensor of stacked tensors. -} -``` - -**LOGICAL_OR** - -``` -Inputs { - 0: a list of tensors. - 1: a list of tensors. -} -Outputs { - 0: A tensor of logical_or output tensors. -} -``` - **UNPACK** ``` @@ -854,18 +978,6 @@ Outputs { } ``` -**FLOOR_DIV** - -``` -Inputs { - 0: a list of tensors. - 1: a list of tensors. -} -Outputs { - 0: A tensor of floor_div output tensors. -} -``` - **ZEROS_LIKE** ``` @@ -877,6 +989,18 @@ Outputs { } ``` +**FILL** + +``` +Inputs { + 0: A Tensor. Must be one of the following types: int32, int64. 1-D. Represents the shape of the output tensor. + 1: A Tensor. 0-D (scalar). Value to fill the returned tensor. +} +Outputs { + 0: A tensor of the same type as value (input1). +} +``` + And these are TensorFlow Lite operations that are present but not ready for custom models yet: diff --git a/tensorflow/lite/g3doc/using_select_tf_ops.md b/tensorflow/lite/g3doc/using_select_tf_ops.md new file mode 100644 index 00000000000..aa51f58baa4 --- /dev/null +++ b/tensorflow/lite/g3doc/using_select_tf_ops.md @@ -0,0 +1,249 @@ +# [Experimental] Using TensorFlow Lite with select TensorFlow ops + +The TensorFlow Lite builtin op library has grown rapidly, and will continue to +grow, but there remains a long tail of TensorFlow ops that are not yet natively +supported by TensorFlow Lite . These unsupported ops can be a point of friction +in the TensorFlow Lite model conversion process. To that end, the team has +recently been working on an experimental mechanism for reducing this friction. + +This document outlines how to use TensorFlow Lite with select TensorFlow ops. +*Note that this feature is experimental and is under active development.* As you +use this feature, keep in mind the [known limitations](#known-limitations), and +please send feedback about models that work and issues you are facing to +tflite@tensorflow.org. + +TensorFlow Lite will continue to have +[TensorFlow Lite builtin ops](tf_ops_compatibility.md) optimized for mobile and +embedded devices. However, TensorFlow Lite models can now use a subset of +TensorFlow ops when TFLite builtin ops are not sufficient. + +Models converted with TensorFlow ops will require a TensorFlow Lite interpreter +that has a larger binary size than the interpreter with only TFLite builtin ops. +Additionally, performance optimizations will not be available for any TensorFlow +ops in the TensorFlow Lite model. + +This document outlines how to [convert](#converting-the-model) and +[run](#running-the-model) a TFLite model with TensorFlow ops on your platform of +choice. It also discusses some [known limitations](#known-limitations), the +[future plans](#future-plans) for this feature, and basic +[performance and size metrics](#metrics). + +## Converting the model + +To convert a TensorFlow model to a TensorFlow Lite model with TensorFlow ops, +use the `target_ops` argument in the +[TensorFlow Lite converter](https://www.tensorflow.org/lite/convert/). The +following values are valid options for `target_ops`: + +* `TFLITE_BUILTINS` - Converts models using TensorFlow Lite builtin ops. +* `SELECT_TF_OPS` - Converts models using TensorFlow ops. The exact subset of + supported ops can be found in the whitelist at + `lite/toco/tflite/whitelisted_flex_ops.cc`. + +The recommended approach is to convert the model with `TFLITE_BUILTINS`, then +with both `TFLITE_BUILTINS,SELECT_TF_OPS`, and finally with only +`SELECT_TF_OPS`. Using both options (i.e. `TFLITE_BUILTINS,SELECT_TF_OPS`) +creates models with TensorFlow Lite ops where possible. Using only +`SELECT_TF_OPS` is useful when the model contains TensorFlow ops that are only +partially supported by TensorFlow Lite, and one would like to avoid those +limitations. + +The following example shows how to use `target_ops` in the +[`TFLiteConverter`](https://www.tensorflow.org/lite/convert/python_api) Python +API. + +``` +import tensorflow as tf + +converter = tf.lite.TFLiteConverter.from_saved_model(saved_model_dir) +converter.target_ops = [tf.lite.OpsSet.TFLITE_BUILTINS, + tf.lite.OpsSet.SELECT_TF_OPS] +tflite_model = converter.convert() +open("converted_model.tflite", "wb").write(tflite_model) +``` + +The following example shows how to use `target_ops` in the +[`tflite_convert`](https://www.tensorflow.org/lite/convert/cmdline_examples) +command line tool. + +``` +tflite_convert \ + --output_file=/tmp/foo.tflite \ + --graph_def_file=/tmp/foo.pb \ + --input_arrays=input \ + --output_arrays=MobilenetV1/Predictions/Reshape_1 \ + --target_ops=TFLITE_BUILTINS,SELECT_TF_OPS +``` + +When building and running `tflite_convert` directly with `bazel`, please pass +`--define=with_select_tf_ops=true` as an additional argument. + +``` +bazel run --define=with_select_tf_ops=true tflite_convert -- \ + --output_file=/tmp/foo.tflite \ + --graph_def_file=/tmp/foo.pb \ + --input_arrays=input \ + --output_arrays=MobilenetV1/Predictions/Reshape_1 \ + --target_ops=TFLITE_BUILTINS,SELECT_TF_OPS +``` + +## Running the model + +When using a TensorFlow Lite model that has been converted with support for +select TensorFlow ops, the client must also use a TensorFlow Lite runtime that +includes the necessary library of TensorFlow ops. + +### Android AAR + +A new Android AAR target with select TensorFlow ops has been added for +convenience. Assuming a working TensorFlow Lite +build environment, build the Android AAR with select TensorFlow ops as +follows: + +```sh +bazel build --cxxopt='--std=c++11' -c opt \ + --config=android_arm --config=monolithic \ + //tensorflow/lite/java:tensorflow-lite-with-select-tf-ops +``` + +This will generate an AAR file in `bazel-genfiles/tensorflow/lite/java/`. From +there, you can either import the AAR directly into your project, or publish the +custom AAR to your local Maven repository: + +```sh +mvn install:install-file \ + -Dfile=bazel-genfiles/tensorflow/lite/java/tensorflow-lite-with-select-tf-ops.aar \ + -DgroupId=org.tensorflow \ + -DartifactId=tensorflow-lite-with-select-tf-ops -Dversion=0.1.100 -Dpackaging=aar +``` + +Finally, in your app's `build.gradle`, ensure you have the `mavenLocal()` +dependency and replace the standard TensorFlow Lite dependency with the one that +has support for select TensorFlow ops: + +``` +allprojects { + repositories { + jcenter() + mavenLocal() + } +} + +dependencies { + compile 'org.tensorflow:tensorflow-lite-with-select-tf-ops:0.1.100' +} +``` + +### iOS + +With XCode Command Line Tools installed, TensorFlow Lite with select TensorFlow +ops support can be built with the following command: + +```sh +tensorflow/contrib/makefile/build_all_ios_with_tflite.sh +``` + +This will generate the required static linking libraries in the +`tensorflow/contrib/makefile/gen/lib/` directory. + +The TensorFlow Lite camera example app can be used to test this. A new +TensorFlow Lite XCode project with support for select TensorFlow ops has been +added to +`tensorflow/lite/examples/ios/camera/tflite_camera_example_with_select_tf_ops.xcodeproj`. + +To use this feature in a your own project, either clone the example project or +set the project settings for a new or existing project to the following: + +* In Build Phases -> Link Binary With Libraries, add the static libraries + under `tensorflow/contrib/makefile/gen/lib/` directory: + * `libtensorflow-lite.a` + * `libprotobuf.a` + * `nsync.a` +* In Build Settings -> Header Search Paths, add the following directories: + * `tensorflow/lite/` + * `tensorflow/contrib/makefile/downloads/flatbuffer/include` + * `tensorflow/contrib/makefile/downloads/eigen` +* In Build Settings -> Other Linker Flags, add `-force_load + tensorflow/contrib/makefile/gen/lib/libtensorflow-lite.a`. + +A CocoaPod with support for select TensorFlow ops will also be released in the +future. + +### C++ + +When building TensorFlow Lite libraries using the bazel pipeline, the additional +TensorFlow ops library can be included and enabled as follows: + +* Enable monolithic builds if necessary by adding the `--config=monolithic` + build flag. +* Do one of the following: + * Include the `--define=with_select_tf_ops=true` build flag in the `bazel + build` invocation when building TensorFlow Lite. + * Add the TensorFlow ops delegate library dependency to the build + dependencies: `tensorflow/lite/delegates/flex:delegate`. + +Note that the necessary `TfLiteDelegate` will be installed automatically when +creating the interpreter at runtime as long as the delegate is linked into the +client library. It is not necessary to explicitly install the delegate instance +as is typically required with other delegate types. + +### Python pip Package + +Python support is actively under development. + +## Metrics + +### Performance + +When using a mixture of both builtin and select TensorFlow ops, all of the same +TensorFlow Lite optimizations and optimized builtin kernels will be be available +and usable with the converted model. For the TensorFlow ops, performance should +generally be comparable to that of +[TensorFlow Mobile](https://www.tensorflow.org/lite/tfmobile/). + +The following table describes the average time taken to run inference on +MobileNet on a Pixel 2. The listed times are an average of 100 runs. These +targets were built for Android using the flags: `--config=android_arm64 -c opt`. + +Build | Time (milliseconds) +------------------------------------ | ------------------- +Only built-in ops (`TFLITE_BUILTIN`) | 260.7 +Using only TF ops (`SELECT_TF_OPS`) | 264.5 + +### Binary Size + +The following table describes the binary size of TensorFlow Lite for each build. +These targets were built for Android using `--config=android_arm -c opt`. + +Build | C++ Binary Size | Android APK Size +--------------------- | --------------- | ---------------- +Only built-in ops | 796 KB | 561 KB +Built-in ops + TF ops | 23.0 MB | 8.0 MB + +## Known Limitations + +The following is a list of some of the known limitations: + +* Control flow ops are not yet supported. +* The + [`post_training_quantization`](https://www.tensorflow.org/performance/post_training_quantization) + flag is currently not supported for TensorFlow ops so it will not quantize + weights for any TensorFlow ops. In models with both TensorFlow Lite builtin + ops and TensorFlow ops, the weights for the builtin ops will be quantized. +* Ops that require explicit initialization from resources, like HashTableV2, + are not yet supported. +* Certain TensorFlow ops may not support the full set of input/output types + that are typically available on stock TensorFlow. + +## Future Plans + +The following is a list of improvements to this pipeline that are in progress: + +* *Selective registration* - There is work being done to make it simple to + generate TFLite interpreter binaries that only contain the TensorFlow ops + required for a particular set of models. +* *Improved usability* - The conversion process will be simplified to only + require a single pass through the converter. Additionally, pre-built Android + AAR and iOS CocoaPod binaries will be provided. +* *Improved performance* - There is work being done to ensure TensorFlow Lite + with TensorFlow ops has performance parity to TensorFlow Mobile. diff --git a/tensorflow/lite/interpreter.cc b/tensorflow/lite/interpreter.cc index c90fc3be87e..e2129ed46d9 100644 --- a/tensorflow/lite/interpreter.cc +++ b/tensorflow/lite/interpreter.cc @@ -20,7 +20,6 @@ limitations under the License. #include #include -#include "tensorflow/lite/arena_planner.h" #include "tensorflow/lite/c/c_api_internal.h" #include "tensorflow/lite/context_util.h" #include "tensorflow/lite/core/api/error_reporter.h" @@ -32,110 +31,15 @@ limitations under the License. #include "tensorflow/lite/util.h" namespace tflite { -namespace { - -TfLiteStatus ReportOpError(TfLiteContext* context, const TfLiteNode& node, - const TfLiteRegistration& registration, - int node_index, const char* message) { - context->ReportError( - context, "Node number %d (%s) %s.\n", node_index, - registration.custom_name - ? registration.custom_name - : EnumNameBuiltinOperator( - static_cast(registration.builtin_code)), - message); - return kTfLiteError; -} - -// Stub method which returns kTfLiteError when the function is forbidden. -// We're registrating this function to several different function to save -// compiled binary size. Please note the restrictions: -// * The type of first parameter have to be `TfLiteContext*`. -// * All paramteters must be trivailly destructible. (E.g. No C++ class) -TfLiteStatus ForbiddenContextFunction(TfLiteContext* context, ...) { - context->ReportError(context, - "The function is forbidden if not calling in delegate."); - return kTfLiteError; -} - -// Set the ForbiddenContextFunction to a compatible function pointer. -template -void SetForbiddenContextFunction(FunctionType* func) { - *func = reinterpret_cast(ForbiddenContextFunction); -} - -// Returns true if at least one tensor in the given list is kTfLiteDynamic. -template -bool HasDynamicTensorImpl(const TfLiteContext& context, - const TensorIntArray& int_array) { - for (int i : int_array) { - const TfLiteTensor& tensor = context.tensors[i]; - if (tensor.allocation_type == kTfLiteDynamic) { - return true; - } - } - return false; -} - -} // namespace - -// A trivial implementation of GraphInfo around the Interpreter. -// NOTE: this interpreter info represents the subset of the -// graph that is executed according to execution plan. Thus, -// the indices are execution plan indices rather than raw node -// indices. -class InterpreterInfo : public GraphInfo { - public: - explicit InterpreterInfo(Interpreter* interpreter) - : interpreter_(interpreter) {} - - size_t num_tensors() const override { return interpreter_->tensors_size(); } - TfLiteTensor* tensor(size_t index) override { - return interpreter_->tensor(index); - } - size_t num_nodes() const override { - return interpreter_->execution_plan().size(); - } - const TfLiteNode& node(size_t index) const override { - int node_index = interpreter_->execution_plan()[index]; - return interpreter_->node_and_registration(node_index)->first; - } - const std::vector& inputs() const override { - return interpreter_->inputs(); - } - const std::vector& outputs() const override { - return interpreter_->outputs(); - } - const std::vector& variables() const override { - return interpreter_->variables(); - } - - public: - Interpreter* interpreter_; -}; Interpreter::Interpreter(ErrorReporter* error_reporter) : error_reporter_(error_reporter ? error_reporter : DefaultErrorReporter()) { - context_.impl_ = static_cast(this); - context_.ResizeTensor = ResizeTensor; - context_.ReportError = ReportError; - context_.AddTensors = AddTensors; - context_.tensors = nullptr; - context_.tensors_size = 0; - context_.allow_fp32_relax_to_fp16 = false; - context_.recommended_num_threads = -1; - context_.GetExternalContext = GetExternalContext; - context_.SetExternalContext = SetExternalContext; - - // Invalid to call these these except from TfLiteDelegate - SwitchToKernelContext(); + // There's always at least 1 subgraph which is the primary subgraph. + AddSubgraphs(1); + context_ = primary_subgraph().context(); // Reserve some space for the tensors to avoid excessive resizing. - tensors_.reserve(kTensorsReservedCapacity); - nodes_and_registration_.reserve(kTensorsReservedCapacity); - next_execution_plan_index_to_prepare_ = 0; - for (int i = 0; i < kTfLiteMaxExternalContexts; ++i) { external_contexts_[i] = nullptr; } @@ -143,670 +47,88 @@ Interpreter::Interpreter(ErrorReporter* error_reporter) UseNNAPI(false); } -Interpreter::~Interpreter() { - for (auto& nodeAndReg : nodes_and_registration_) { - TfLiteNode& node = nodeAndReg.first; - TfLiteIntArrayFree(node.inputs); - TfLiteIntArrayFree(node.outputs); - TfLiteIntArrayFree(node.temporaries); - if (node.builtin_data) free(node.builtin_data); - OpFree(nodeAndReg.second, node.user_data); - node.builtin_data = nullptr; - } - - for (size_t i = 0; i < context_.tensors_size; i++) { - TfLiteTensor* tensor = &context_.tensors[i]; - if (tensor->buffer_handle != kTfLiteNullBufferHandle && - tensor->delegate->FreeBufferHandle != nullptr) { - tensor->delegate->FreeBufferHandle(&context_, tensor->delegate, - &tensor->buffer_handle); - } - TfLiteTensorFree(tensor); - } -} - -TfLiteStatus Interpreter::ReplaceNodeSubsetsWithDelegateKernels( - TfLiteContext* context, TfLiteRegistration registration, - const TfLiteIntArray* nodes_to_replace, TfLiteDelegate* delegate) { - return static_cast(context->impl_) - ->ReplaceNodeSubsetsWithDelegateKernels(registration, nodes_to_replace, - delegate); -} - -namespace { - -// Copy a std::vector to an existing TfLiteIntArray. -// This is a low-level data manipulation function, and it's caller's -// responsibility to ensure TfLiteIntArray has enough size. -void CopyVectorToTfLiteIntArray(const std::vector& vec, - TfLiteIntArray* arr) { - arr->size = vec.size(); - memcpy(arr->data, vec.data(), sizeof(int) * arr->size); -} - -// This function allocates a continuous memory space that contains a -// TfLiteDelegateParams followed by a several TfLiteIntArray. -// When calling `free` at TfLiteDelegateParams*, all the allocated space -// will be freed together. -// -// +-----------------------------------+ -// | TfLiteDelegateParams | -// | TfLiteDelegate* delegate; | -// | TfLiteIntArray* nodes_to_replace; |--\ -// | TfLiteIntArray* input_tensors; |--+--\ -// | TfLiteIntArray* output_tensors; |--+--+--\ -// +-----------------------------------+ | | | -// | TfLiteIntArray (variable size) |<-/ | | -// +-----------------------------------+ | | -// | TfLiteIntArray (variable size) |<----/ | -// +-----------------------------------+ | -// | TfLiteIntArray (variable size) |<-------/ -// +-----------------------------------+ -TfLiteDelegateParams* CreateDelegateParams(TfLiteDelegate* delegate, - const NodeSubset& node_subset) { - // Step 1: Calculate the allocation size. - int allocation_size = sizeof(TfLiteDelegateParams); - - int nodes_to_replace_size = - TfLiteIntArrayGetSizeInBytes(node_subset.nodes.size()); - allocation_size += nodes_to_replace_size; - - int input_tensors_size = - TfLiteIntArrayGetSizeInBytes(node_subset.input_tensors.size()); - allocation_size += input_tensors_size; - - int output_tensors_size = - TfLiteIntArrayGetSizeInBytes(node_subset.output_tensors.size()); - allocation_size += output_tensors_size; - - // Step 2: Allocate the memory. - // Use `char*` for conveniently step through the allocated space by bytes. - char* allocation = reinterpret_cast(malloc(allocation_size)); - - // Step 3: Fill all data structures structures. - TfLiteDelegateParams* params = - reinterpret_cast(allocation); - params->delegate = delegate; - allocation += sizeof(TfLiteDelegateParams); - - params->nodes_to_replace = reinterpret_cast(allocation); - CopyVectorToTfLiteIntArray(node_subset.nodes, params->nodes_to_replace); - allocation += nodes_to_replace_size; - - params->input_tensors = reinterpret_cast(allocation); - CopyVectorToTfLiteIntArray(node_subset.input_tensors, params->input_tensors); - allocation += input_tensors_size; - - params->output_tensors = reinterpret_cast(allocation); - CopyVectorToTfLiteIntArray(node_subset.output_tensors, - params->output_tensors); - allocation += output_tensors_size; - - return params; -} - -} // namespace - -TfLiteStatus Interpreter::ReplaceNodeSubsetsWithDelegateKernels( - TfLiteRegistration registration, const TfLiteIntArray* nodes_to_replace, - TfLiteDelegate* delegate) { - // Annotate the registration as DELEGATE op. - registration.builtin_code = BuiltinOperator_DELEGATE; - - // Analyze the graph to find all independent node_subsets that are either - // fully not-this-delegate or this-delegate computation. - InterpreterInfo info(this); - std::vector node_subsets; - PartitionGraphIntoIndependentNodeSubsets(&info, nodes_to_replace, - &node_subsets); - - execution_plan_.clear(); - for (auto& node_subset : node_subsets) { - // Subsets calimed by the delegate should have a "macro" op created, the - // other node_subsets (kTfNonPartition) just have their nodes added back to - // the execution plan. - switch (node_subset.type) { - case NodeSubset::kTfNonPartition: - for (auto it = node_subset.nodes.begin(); it != node_subset.nodes.end(); - ++it) { - execution_plan_.push_back(*it); - } - break; - case NodeSubset::kTfPartition: { - int node_index; - - TfLiteDelegateParams* params = - CreateDelegateParams(delegate, node_subset); - TF_LITE_ENSURE_STATUS(AddNodeWithParameters( - node_subset.input_tensors, node_subset.output_tensors, nullptr, 0, - params, ®istration, &node_index)); - - // Initialize the output tensors's delegate-related fields. - for (int tensor_index : node_subset.output_tensors) { - TfLiteTensor* tensor = &tensors_[tensor_index]; - TF_LITE_ENSURE(&context_, tensor->delegate == nullptr || - tensor->delegate == delegate); - tensor->delegate = delegate; - } - - // Associate the node with the delegate. - TfLiteNode* node = &nodes_and_registration_[node_index].first; - node->delegate = delegate; - } break; - case NodeSubset::kTfUnexplored: - return kTfLiteError; - break; - } - } - return kTfLiteOk; -} - -TfLiteExternalContext* Interpreter::GetExternalContext( - TfLiteExternalContextType type) { - if (type >= 0 && type < kTfLiteMaxExternalContexts) { - return external_contexts_[type]; - } - return nullptr; -} - -TfLiteExternalContext* Interpreter::GetExternalContext( - struct TfLiteContext* context, TfLiteExternalContextType type) { - return static_cast(context->impl_)->GetExternalContext(type); -} +Interpreter::~Interpreter() {} void Interpreter::SetExternalContext(TfLiteExternalContextType type, TfLiteExternalContext* ctx) { - if (type >= 0 && type < kTfLiteMaxExternalContexts) { - external_contexts_[type] = ctx; - } -} - -void Interpreter::SetExternalContext(struct TfLiteContext* context, - TfLiteExternalContextType type, - TfLiteExternalContext* ctx) { - return static_cast(context->impl_) - ->SetExternalContext(type, ctx); -} - -// Gets an TfLiteIntArray* representing the execution plan. The interpreter owns -// this memory and it is only guaranteed to exist during the invocation of the -// delegate prepare. -TfLiteStatus Interpreter::GetExecutionPlan(TfLiteIntArray** execution_plan) { - // TODO(aselle): Do not make a copy here - plan_cache_.reset(TfLiteIntArrayCreate(execution_plan_.size())); - *execution_plan = plan_cache_.get(); - static_assert(sizeof(plan_cache_->data[0]) == sizeof(execution_plan_[0]), - "TfLiteIntArray and execution_plan do not contain same type."); - std::memcpy(plan_cache_->data, execution_plan_.data(), - sizeof(plan_cache_->data[0]) * execution_plan_.size()); - return kTfLiteOk; -} - -// WARNING: This is an experimental interface that is subject to change. -// Entry point for C node plugin API to get the execution plan -TfLiteStatus Interpreter::GetExecutionPlan(struct TfLiteContext* context, - TfLiteIntArray** execution_plan) { - return static_cast(context->impl_) - ->GetExecutionPlan(execution_plan); + primary_subgraph().SetExternalContext(type, ctx); } TfLiteStatus Interpreter::SetInputs(std::vector inputs) { - TF_LITE_ENSURE_OK(&context_, - CheckTensorIndices("inputs", inputs.data(), inputs.size())); - inputs_ = std::move(inputs); - return kTfLiteOk; + return primary_subgraph().SetInputs(inputs); } TfLiteStatus Interpreter::SetOutputs(std::vector outputs) { - TF_LITE_ENSURE_OK( - &context_, CheckTensorIndices("outputs", outputs.data(), outputs.size())); - outputs_ = std::move(outputs); - return kTfLiteOk; + return primary_subgraph().SetOutputs(outputs); } TfLiteStatus Interpreter::SetVariables(std::vector variables) { - TF_LITE_ENSURE_OK(&context_, CheckTensorIndices("variables", variables.data(), - variables.size())); - variables_ = std::move(variables); - return kTfLiteOk; -} - -TfLiteStatus Interpreter::CheckTensorIndices(const char* label, - const int* indices, int length) { - // Making sure kOptionalTensor is not re-defined to something other than -1. - static_assert(kOptionalTensor == -1, "kOptionalTensor should be defined -1"); - - for (int i = 0; i < length; i++) { - int index = indices[i]; - // Continue if index == kOptionalTensor before additional comparisons below, - // size_t(-1) is always >= context_tensors_size. - if (index == kOptionalTensor) { - continue; - } - if (index < 0 || static_cast(index) >= context_.tensors_size) { - ReportError(&context_, "Invalid tensor index %d in %s\n", index, label); - consistent_ = false; - return kTfLiteError; - } - } - return kTfLiteOk; -} - -TfLiteStatus Interpreter::BytesRequired(TfLiteType type, const int* dims, - size_t dims_size, size_t* bytes) { - // TODO(aselle): Check for overflow here using overflow.h in TensorFlow - // MultiplyWithoutOverflow. - TF_LITE_ENSURE(&context_, bytes != nullptr); - size_t count = 1; - for (int k = 0; k < dims_size; k++) count *= dims[k]; - switch (type) { - case kTfLiteFloat32: - *bytes = sizeof(float) * count; - break; - case kTfLiteInt16: - *bytes = sizeof(int16_t) * count; - break; - case kTfLiteInt32: - *bytes = sizeof(int32_t) * count; - break; - case kTfLiteUInt8: - *bytes = sizeof(uint8_t) * count; - break; - case kTfLiteInt64: - *bytes = sizeof(int64_t) * count; - break; - case kTfLiteBool: - *bytes = sizeof(bool) * count; - break; - case kTfLiteComplex64: - *bytes = sizeof(std::complex) * count; - break; - default: - ReportError(&context_, - "Only float32, int16, int32, int64, uint8, bool, complex64 " - "supported currently."); - return kTfLiteError; - } - return kTfLiteOk; + return primary_subgraph().SetVariables(variables); } TfLiteStatus Interpreter::AllocateTensors() { - if (!consistent_) { - ReportError(&context_, "AllocateTensors() called on inconsistent model."); - return kTfLiteError; - } - - // Explicit (re)allocation is necessary if nodes have been changed or tensors - // have been resized. For inputs marked as dynamic, we can't short-circuit the - // allocation as the client may have done the resize manually. - if (state_ != kStateUninvokable && !HasDynamicTensorImpl(context_, inputs_)) { - return kTfLiteOk; - } - - next_execution_plan_index_to_prepare_ = 0; - if (memory_planner_) { - TF_LITE_ENSURE_STATUS(memory_planner_->ResetAllocations()); - } - - TF_LITE_ENSURE_STATUS(PrepareOpsAndTensors()); - - state_ = kStateInvokable; - - // Reset the variable tensors to zero after (re)allocating the tensors. - // Developers shouldn't rely on the side effect of this function to reset - // variable tesnsors. They should call `ResetVariableTensors` directly - // instead. - ResetVariableTensors(); - - return kTfLiteOk; -} - -// TODO(ycling): Support non-zero default values. -TfLiteStatus Interpreter::ResetVariableTensors() { - for (auto& tensor : tensors_) { - if (!tensor.is_variable) { - continue; - } - - // Variable tensors have to be `kTfLiteArenaRwPersistent`, and must be - // allocated after the initial `PrepareOpsAndTensors()` is called. - TF_LITE_ENSURE_EQ(&context_, tensor.allocation_type, - kTfLiteArenaRwPersistent); - TF_LITE_ENSURE(&context_, tensor.data.raw != nullptr); - - memset(tensor.data.raw, 0, tensor.bytes); - } - return kTfLiteOk; + return primary_subgraph().AllocateTensors(); } void Interpreter::ReserveNodes(int count) { - nodes_and_registration_.reserve(count); + primary_subgraph().nodes_and_registration().reserve(count); +} + +void Interpreter::AddSubgraphs(int subgraphs_to_add, + int* first_new_subgraph_index) { + const size_t base_index = subgraphs_.size(); + if (first_new_subgraph_index) *first_new_subgraph_index = base_index; + + subgraphs_.reserve(base_index + subgraphs_to_add); + for (int i = 0; i < subgraphs_to_add; ++i) { + Subgraph* subgraph = + new Subgraph(error_reporter_, external_contexts_, &subgraphs_); + subgraphs_.emplace_back(subgraph); + } } TfLiteStatus Interpreter::AddNodeWithParameters( const std::vector& inputs, const std::vector& outputs, const char* init_data, size_t init_data_size, void* builtin_data, const TfLiteRegistration* registration, int* node_index) { - if (state_ == kStateInvokableAndImmutable) { - ReportError(&context_, - "AddNodeWithParameters is disallowed when graph is immutable."); - return kTfLiteError; - } - state_ = kStateUninvokable; - - std::unique_ptr builtin_data_deleter(builtin_data, - free); - - TF_LITE_ENSURE_OK(&context_, CheckTensorIndices("node inputs", inputs.data(), - inputs.size())); - TF_LITE_ENSURE_OK( - &context_, - CheckTensorIndices("node outputs", outputs.data(), outputs.size())); - - int new_node_index = nodes_and_registration_.size(); - if (node_index) *node_index = new_node_index; - nodes_and_registration_.resize(nodes_and_registration_.size() + 1); - auto& node_and_reg = nodes_and_registration_.back(); - TfLiteNode& node = node_and_reg.first; - if (node.inputs) TfLiteIntArrayFree(node.inputs); - if (node.outputs) TfLiteIntArrayFree(node.outputs); - if (node.temporaries) TfLiteIntArrayFree(node.temporaries); - - // NOTE, here we are not using move semantics yet, since our internal - // representation isn't std::vector, but in the future we would like to avoid - // copies, so we want the interface to take r-value references now. - node.inputs = ConvertVectorToTfLiteIntArray(inputs); - node.outputs = ConvertVectorToTfLiteIntArray(outputs); - node.temporaries = TfLiteIntArrayCreate(0); - if (init_data) { - node.user_data = OpInit(*registration, init_data, init_data_size); - } else { - node.user_data = - OpInit(*registration, - reinterpret_cast(builtin_data_deleter.get()), 0); - } - - node.builtin_data = builtin_data_deleter.release(); - // TODO(ycling): Filling `custom_initial_data` and `custom_initial_data_size` - // properly for nodes generated by ReplaceNodeSubsetsWithDelegateKernels. - - if (registration->builtin_code == BuiltinOperator_CUSTOM) { - // When it's a CUSTOM op, the `custom_options` field in the Flatbuffer - // `Operator` table is passed in. - node.custom_initial_data = init_data; - node.custom_initial_data_size = init_data_size; - } else { - node.custom_initial_data = nullptr; - node.custom_initial_data_size = 0; - } - - node.delegate = nullptr; - node_and_reg.second = *registration; - execution_plan_.push_back(new_node_index); - return kTfLiteOk; + return primary_subgraph().AddNodeWithParameters(inputs, outputs, init_data, + init_data_size, builtin_data, + registration, node_index); } TfLiteStatus Interpreter::ResizeInputTensor(int tensor_index, const std::vector& dims) { - if (state_ == kStateInvokableAndImmutable) { - ReportError(&context_, - "ResizeInputTensor is disallowed when graph is immutable."); - return kTfLiteError; - } - - // TODO(aselle): All bounds checks can be implemented as one-sided bounds - // checks by casting to unsigned for efficiency. Profile before doing this. - TF_LITE_ENSURE(&context_, - tensor_index < context_.tensors_size && tensor_index >= 0); - TfLiteTensor* tensor = &context_.tensors[tensor_index]; - - // Short-circuit the state change if the dimensions don't change, avoiding - // unnecessary (re)allocations. - if (EqualArrayAndTfLiteIntArray(tensor->dims, dims.size(), dims.data())) { - return kTfLiteOk; - } - - state_ = kStateUninvokable; - return ResizeTensorImpl(tensor, ConvertVectorToTfLiteIntArray(dims)); -} - -bool HasDynamicTensor(const TfLiteContext& context, - const TfLiteIntArray* int_array) { - return HasDynamicTensorImpl(context, TfLiteIntArrayView{int_array}); -} - -TfLiteStatus Interpreter::PrepareOpsStartingAt( - int first_execution_plan_index, int* last_execution_plan_index_prepared) { - for (int execution_plan_index = first_execution_plan_index; - execution_plan_index < execution_plan_.size(); execution_plan_index++) { - int node_index = execution_plan_[execution_plan_index]; - TfLiteNode& node = nodes_and_registration_[node_index].first; - const TfLiteRegistration& registration = - nodes_and_registration_[node_index].second; - EnsureTensorsVectorCapacity(); - if (OpPrepare(registration, &node) == kTfLiteError) { - return ReportOpError(&context_, node, registration, node_index, - "failed to prepare"); - } - - *last_execution_plan_index_prepared = execution_plan_index; - - // Discontinue if the node has dynamic outputs. Note that we don't - // stop for dynamic temporary tensors since they won't affect the - // sizes of other tensors in the graph. - if (HasDynamicTensor(context_, node.outputs)) { - break; - } - } - return kTfLiteOk; -} - -TfLiteStatus Interpreter::PrepareOpsAndTensors() { - if (!memory_planner_) { - memory_planner_.reset(new ArenaPlanner( - &context_, std::unique_ptr(new InterpreterInfo(this)), - /*preserve_inputs=*/true, /*preserve_intermediates*/ false)); - memory_planner_->PlanAllocations(); - } - - int last_exec_plan_index_prepared = 0; - - TF_LITE_ENSURE_STATUS(PrepareOpsStartingAt( - next_execution_plan_index_to_prepare_, &last_exec_plan_index_prepared)); - TF_LITE_ENSURE_STATUS(memory_planner_->ExecuteAllocations( - next_execution_plan_index_to_prepare_, last_exec_plan_index_prepared)); - - next_execution_plan_index_to_prepare_ = last_exec_plan_index_prepared + 1; - return kTfLiteOk; + return primary_subgraph().ResizeInputTensor(tensor_index, dims); } TfLiteStatus Interpreter::Invoke() { - if (!consistent_) { - ReportError(&context_, "Invoke called on model that is not consistent."); - return kTfLiteError; - } - if (state_ == kStateUninvokable) { - ReportError(&context_, "Invoke called on model that is not ready."); - return kTfLiteError; - } - - TfLiteStatus status = kTfLiteOk; - if (nnapi_delegate_) { - if (next_execution_plan_index_to_prepare_ == execution_plan_.size()) { - TF_LITE_ENSURE_OK(&context_, nnapi_delegate_->Invoke(this)); - return kTfLiteOk; - } else { - // TODO(aselle): In the future, we would like this to be an - // automatic tflite CPU fallback. - ReportError(&context_, - "NNAPI was requested, but dependent sized tensors " - "being used.\n"); - return kTfLiteError; - } - } - - // Invocations are always done in node order. - // Note that calling Invoke repeatedly will cause the original memory plan to - // be reused, unless either ResizeInputTensor() or AllocateTensors() has been - // called. - for (int execution_plan_index = 0; - execution_plan_index < execution_plan_.size(); execution_plan_index++) { - if (execution_plan_index == next_execution_plan_index_to_prepare_) { - TF_LITE_ENSURE_STATUS(PrepareOpsAndTensors()); - TF_LITE_ENSURE(&context_, next_execution_plan_index_to_prepare_ >= - execution_plan_index); - } - int node_index = execution_plan_[execution_plan_index]; - TfLiteNode& node = nodes_and_registration_[node_index].first; - const TfLiteRegistration& registration = - nodes_and_registration_[node_index].second; - SCOPED_OPERATOR_PROFILE(profiler_, node_index); - - // TODO(ycling): This is an extra loop through inputs to check if the data - // need to be copied from Delegate buffer to raw memory, which is often not - // needed. We may want to cache this in prepare to know if this needs to be - // done for a node or not. - for (int i = 0; i < node.inputs->size; ++i) { - int tensor_index = node.inputs->data[i]; - if (tensor_index == kOptionalTensor) { - continue; - } - TfLiteTensor* tensor = &tensors_[tensor_index]; - if (tensor->delegate && tensor->delegate != node.delegate && - tensor->data_is_stale) { - EnsureTensorDataIsReadable(tensor_index); - } - } - - EnsureTensorsVectorCapacity(); - tensor_resized_since_op_invoke_ = false; - if (OpInvoke(registration, &node) == kTfLiteError) { - status = ReportOpError(&context_, node, registration, node_index, - "failed to invoke"); - } - - // Force execution prep for downstream ops if the latest op triggered the - // resize of a dynamic tensor. - if (tensor_resized_since_op_invoke_ && - HasDynamicTensor(context_, node.outputs)) { - next_execution_plan_index_to_prepare_ = execution_plan_index + 1; - } - } + TfLiteStatus status = primary_subgraph().Invoke(); if (!allow_buffer_handle_output_) { - for (int tensor_index : outputs_) { - EnsureTensorDataIsReadable(tensor_index); + for (int tensor_index : outputs()) { + primary_subgraph().EnsureTensorDataIsReadable(tensor_index); } } return status; } -TfLiteStatus Interpreter::ResizeTensor(TfLiteContext* context, - TfLiteTensor* tensor, - TfLiteIntArray* new_size) { - // Note here that context->impl_ is recovering the this pointer for an - // instance of Interpreter to call into the member function ResizeTensorImpl - // (this function is static). - return static_cast(context->impl_) - ->ResizeTensorImpl(tensor, new_size); -} - -void Interpreter::ReportErrorImpl(const char* format, va_list args) { - error_reporter_->Report(format, args); -} - -void Interpreter::ReportError(TfLiteContext* context, const char* format, ...) { - va_list args; - va_start(args, format); - auto* f = static_cast(context->impl_); - // Note here that context->impl_ is recovering the this pointer for an - // instance of Interpreter to call into the member function ReportErrorImpl - // (this function is static). - f->ReportErrorImpl(format, args); - va_end(args); -} - TfLiteStatus Interpreter::AddTensors(int tensors_to_add, int* first_new_tensor_index) { - const size_t base_index = tensors_.size(); - if (first_new_tensor_index) *first_new_tensor_index = base_index; - tensors_.resize(tensors_.size() + tensors_to_add); - for (size_t i = base_index; i < tensors_.size(); i++) { - memset(&tensors_[i], 0, sizeof(tensors_[i])); - tensors_[i].buffer_handle = kTfLiteNullBufferHandle; - } - context_.tensors = tensors_.data(); - context_.tensors_size = tensors_.size(); - return kTfLiteOk; + return primary_subgraph().AddTensors(tensors_to_add, first_new_tensor_index); } -TfLiteStatus Interpreter::AddTensors(TfLiteContext* context, int tensors_to_add, - int* first_new_tensor_index) { - // Note here that context->impl_ is recovering the this pointer for an - // instance of Interpreter to call into the member function AddTensors - // (this function is static). - return static_cast(context->impl_) - ->AddTensors(tensors_to_add, first_new_tensor_index); -} - -TfLiteStatus Interpreter::GetNodeAndRegistration( - int node_index, TfLiteNode** node, TfLiteRegistration** registration) { - TF_LITE_ENSURE(&context_, node_index >= 0); - TF_LITE_ENSURE(&context_, static_cast(node_index) < nodes_size()); - TF_LITE_ENSURE(&context_, node != nullptr && registration != nullptr); - *node = &nodes_and_registration_[node_index].first; - *registration = &nodes_and_registration_[node_index].second; - return kTfLiteOk; -} - -TfLiteStatus Interpreter::GetNodeAndRegistration( - struct TfLiteContext* context, int node_index, TfLiteNode** node, - TfLiteRegistration** registration) { - return static_cast(context->impl_) - ->GetNodeAndRegistration(node_index, node, registration); +TfLiteStatus Interpreter::ResetVariableTensors() { + return primary_subgraph().ResetVariableTensors(); } TfLiteStatus Interpreter::SetTensorParametersReadOnly( int tensor_index, TfLiteType type, const char* name, const size_t rank, const int* dims, TfLiteQuantizationParams quantization, const char* buffer, size_t bytes, const Allocation* allocation) { - if (state_ == kStateInvokableAndImmutable) { - ReportError( - &context_, - "SetTensorParametersReadOnly is disallowed when graph is immutable."); - return kTfLiteError; - } - - TF_LITE_ENSURE(&context_, - tensor_index < context_.tensors_size && tensor_index >= 0); - // For most tensors we know exactly how much memory is necessary so we can - // ensure the buffer is large enough. However, we need to skip string tensors - // because their sizes change with the contents of the individual strings. - if (type != kTfLiteString) { - size_t required_bytes; - TF_LITE_ENSURE_OK(&context_, - BytesRequired(type, dims, rank, &required_bytes)); - TF_LITE_ENSURE_EQ(&context_, required_bytes, bytes); - } - - TfLiteTensor& tensor = context_.tensors[tensor_index]; - if (type == tensor.type && - EqualArrayAndTfLiteIntArray(tensor.dims, rank, dims)) { - // Fast path which does not invalidate the invokable property. - TfLiteTensorDataFree(&tensor); - tensor.data.raw = const_cast(buffer); - if (!tensor.dims) tensor.dims = ConvertArrayToTfLiteIntArray(rank, dims); - tensor.params = quantization; - tensor.allocation_type = kTfLiteMmapRo; - tensor.allocation = allocation; - } else { - state_ = kStateUninvokable; - TfLiteTensorReset(type, name, ConvertArrayToTfLiteIntArray(rank, dims), - quantization, const_cast(buffer), bytes, - kTfLiteMmapRo, allocation, false, &tensor); - } - return kTfLiteOk; + return primary_subgraph().SetTensorParametersReadOnly( + tensor_index, type, name, rank, dims, quantization, buffer, bytes, + allocation); } // Set description of inputs/outputs/data/fptrs for node `node_index`. @@ -816,186 +138,52 @@ TfLiteStatus Interpreter::SetTensorParametersReadOnly( TfLiteStatus Interpreter::SetTensorParametersReadWrite( int tensor_index, TfLiteType type, const char* name, const size_t rank, const int* dims, TfLiteQuantizationParams quantization, bool is_variable) { - if (state_ == kStateInvokableAndImmutable) { - ReportError( - &context_, - "SetTensorParametersReadWrite is disallowed when graph is immutable."); - return kTfLiteError; - } - TF_LITE_ENSURE(&context_, - tensor_index < context_.tensors_size && tensor_index >= 0); - size_t required_bytes = 0; - if (type != kTfLiteString) { - // These types will be allocated in our arena so we need to record how - // many bytes we will need based on the dimensions. String tensors are - // allocated dynamically and we can't know ahead of time how much space - // they will require. - TF_LITE_ENSURE_OK(&context_, - BytesRequired(type, dims, rank, &required_bytes)); - } - - TfLiteAllocationType allocation_type = kTfLiteArenaRw; - if (type == kTfLiteString) { - if (is_variable) { - // We don't have a real use case for string variable tensor. - ReportError(&context_, "String variable tensor isn't supported."); - return kTfLiteError; - } - allocation_type = kTfLiteDynamic; - } else if (is_variable) { - allocation_type = kTfLiteArenaRwPersistent; - } - - TfLiteTensorReset(type, name, ConvertArrayToTfLiteIntArray(rank, dims), - quantization, - /*buffer=*/nullptr, required_bytes, allocation_type, - nullptr, is_variable, &context_.tensors[tensor_index]); - return kTfLiteOk; + return primary_subgraph().SetTensorParametersReadWrite( + tensor_index, type, name, rank, dims, quantization, is_variable); } TfLiteStatus Interpreter::SetExecutionPlan(const std::vector& new_plan) { - for (int node_index : new_plan) { - TF_LITE_ENSURE(&context_, node_index >= 0 && node_index < nodes_size()); - } - execution_plan_ = new_plan; - return kTfLiteOk; + return primary_subgraph().SetExecutionPlan(new_plan); } -TfLiteStatus Interpreter::ResizeTensorImpl(TfLiteTensor* tensor, - TfLiteIntArray* new_size) { - // Note that in theory we could resize kTfLiteArenaRwPersistent tensors too. - if (tensor->allocation_type == kTfLiteArenaRw || - tensor->allocation_type == kTfLiteDynamic || - tensor->allocation_type == kTfLiteArenaRwPersistent) { - tensor_resized_since_op_invoke_ |= - TfLiteIntArrayEqual(tensor->dims, new_size) == 0; - if (tensor->type != kTfLiteString) { - size_t bytesRequired; - TfLiteStatus status = BytesRequired(tensor->type, new_size->data, - new_size->size, &bytesRequired); - if (status != kTfLiteOk) { - TfLiteIntArrayFree(new_size); - return kTfLiteError; - } - - // Realloc space for kTfLiteDynamic tensors. - TfLiteTensorRealloc(bytesRequired, tensor); - tensor->bytes = bytesRequired; - } - if (tensor->dims) TfLiteIntArrayFree(tensor->dims); - tensor->dims = new_size; - - if (tensor->allocation_type != kTfLiteDynamic) { - tensor->data.raw = nullptr; - } - } else { - // kTfLiteMmapRo tensors are stored in the flatbuffer and are therefore - // of fixed size. - TfLiteIntArrayFree(new_size); - ReportError(&context_, "Attempting to resize a fixed-size tensor."); - return kTfLiteError; - } - return kTfLiteOk; -} - -void Interpreter::UseNNAPI(bool enable) { - // TODO(aselle): This is a workaround for finding if NNAPI exists. - // We also need to make sure getLibraryHandle() is renamed to be NNAPI - // prefixed. - if (!NNAPIDelegate::IsSupported()) enable = false; - if (!enable) { - nnapi_delegate_.reset(); - } else if (!nnapi_delegate_) { - nnapi_delegate_.reset(new NNAPIDelegate); - } -} +void Interpreter::UseNNAPI(bool enable) { primary_subgraph().UseNNAPI(enable); } void Interpreter::SetNumThreads(int num_threads) { - context_.recommended_num_threads = num_threads; + for (auto& subgraph : subgraphs_) { + subgraph->context()->recommended_num_threads = num_threads; + } for (int i = 0; i < kTfLiteMaxExternalContexts; ++i) { auto* c = external_contexts_[i]; if (c && c->Refresh) { - c->Refresh(&context_); + c->Refresh(context_); } } } -void Interpreter::SwitchToDelegateContext() { - context_.GetNodeAndRegistration = GetNodeAndRegistration; - context_.ReplaceNodeSubsetsWithDelegateKernels = - ReplaceNodeSubsetsWithDelegateKernels; - context_.GetExecutionPlan = GetExecutionPlan; -} - -void Interpreter::SwitchToKernelContext() { - SetForbiddenContextFunction(&context_.GetNodeAndRegistration); - SetForbiddenContextFunction(&context_.ReplaceNodeSubsetsWithDelegateKernels); - SetForbiddenContextFunction(&context_.GetExecutionPlan); +void Interpreter::SetAllowFp16PrecisionForFp32(bool allow) { + for (auto& subgraph : subgraphs_) { + subgraph->context()->allow_fp32_relax_to_fp16 = allow; + } } TfLiteStatus Interpreter::ModifyGraphWithDelegate(TfLiteDelegate* delegate) { - if (!(delegate->flags & kTfLiteDelegateFlagsAllowDynamicTensors)) { - int last_execution_plan_index_prepared; - TF_LITE_ENSURE_OK(&context_, PrepareOpsStartingAt( - 0, &last_execution_plan_index_prepared)); - - bool has_dynamic_tensors = true; - // Dynamic tensors exist if not all nodes can be prepared. - if (last_execution_plan_index_prepared + 1 == execution_plan_.size()) { - // If all the nodes can be prepared, check if the last node has dynamic - // tensors. - int node_index = execution_plan_[last_execution_plan_index_prepared]; - TfLiteNode& node = nodes_and_registration_[node_index].first; - if (!HasDynamicTensor(context_, node.outputs)) { - has_dynamic_tensors = false; - } - } - if (has_dynamic_tensors) { - ReportError( - &context_, - "Attempting to use a delegate that only supports static-sized " - "tensors with a graph that has dynamic-sized tensors."); - return kTfLiteError; - } - } - - // TODO(aselle): Consider if it is worth storing pointers to delegates. - // Setup additional context interface. - SwitchToDelegateContext(); - - TfLiteStatus status = delegate->Prepare(&context_, delegate); - - // Remove additional context info. - SwitchToKernelContext(); - - TF_LITE_ENSURE_OK(&context_, status); - - if (!(delegate->flags & kTfLiteDelegateFlagsAllowDynamicTensors)) { - // Reset the state to force tensor/op reallocation. - state_ = kStateUninvokable; - TF_LITE_ENSURE_OK(&context_, AllocateTensors()); - TF_LITE_ENSURE_EQ(&context_, state_, kStateInvokable); - // After using a delegate which doesn't support dynamic tensors, make the - // entire graph immutable. - state_ = kStateInvokableAndImmutable; - } - - return status; + return primary_subgraph().ModifyGraphWithDelegate(delegate); } TfLiteStatus Interpreter::SetBufferHandle(int tensor_index, TfLiteBufferHandle buffer_handle, TfLiteDelegate* delegate) { - TF_LITE_ENSURE(&context_, tensor_index < tensors_size()); - TfLiteTensor* tensor = &tensors_[tensor_index]; + TF_LITE_ENSURE(context_, tensor_index < tensors_size()); + std::vector& tensors = primary_subgraph().tensors(); + TfLiteTensor* tensor = &tensors[tensor_index]; - TF_LITE_ENSURE(&context_, + TF_LITE_ENSURE(context_, tensor->delegate == nullptr || tensor->delegate == delegate); tensor->delegate = delegate; if (tensor->buffer_handle != kTfLiteNullBufferHandle) { - TF_LITE_ENSURE(&context_, tensor->delegate->FreeBufferHandle != nullptr); - tensor->delegate->FreeBufferHandle(&context_, tensor->delegate, + TF_LITE_ENSURE(context_, tensor->delegate->FreeBufferHandle != nullptr); + tensor->delegate->FreeBufferHandle(context_, tensor->delegate, &tensor->buffer_handle); } tensor->buffer_handle = buffer_handle; @@ -1006,8 +194,9 @@ TfLiteStatus Interpreter::SetBufferHandle(int tensor_index, TfLiteStatus Interpreter::GetBufferHandle(int tensor_index, TfLiteBufferHandle* buffer_handle, TfLiteDelegate** delegate) { - TF_LITE_ENSURE(&context_, tensor_index < tensors_size()); - TfLiteTensor* tensor = &tensors_[tensor_index]; + TF_LITE_ENSURE(context_, tensor_index < tensors_size()); + std::vector& tensors = primary_subgraph().tensors(); + TfLiteTensor* tensor = &tensors[tensor_index]; *delegate = tensor->delegate; *buffer_handle = tensor->buffer_handle; @@ -1015,4 +204,12 @@ TfLiteStatus Interpreter::GetBufferHandle(int tensor_index, return kTfLiteOk; } +void Interpreter::SetProfiler(profiling::Profiler* profiler) { + for (auto& subgraph : subgraphs_) subgraph->SetProfiler(profiler); +} + +profiling::Profiler* Interpreter::GetProfiler() { + return primary_subgraph().GetProfiler(); +} + } // namespace tflite diff --git a/tensorflow/lite/interpreter.h b/tensorflow/lite/interpreter.h index 415c5f0979c..6192d56ca2b 100644 --- a/tensorflow/lite/interpreter.h +++ b/tensorflow/lite/interpreter.h @@ -25,6 +25,7 @@ limitations under the License. #include "tensorflow/lite/allocation.h" #include "tensorflow/lite/c/c_api_internal.h" #include "tensorflow/lite/core/api/error_reporter.h" +#include "tensorflow/lite/core/subgraph.h" #include "tensorflow/lite/memory_planner.h" #include "tensorflow/lite/profiling/profiler.h" #include "tensorflow/lite/stderr_reporter.h" @@ -57,6 +58,10 @@ constexpr TfLiteType typeToTfLiteType() { return kTfLiteUInt8; } template <> +constexpr TfLiteType typeToTfLiteType() { + return kTfLiteInt8; +} +template <> constexpr TfLiteType typeToTfLiteType() { return kTfLiteBool; } @@ -69,9 +74,6 @@ constexpr TfLiteType typeToTfLiteType() { return kTfLiteString; } -// Forward declare since NNAPIDelegate uses Interpreter. -class NNAPIDelegate; - // An interpreter for a graph of nodes that input and output from tensors. // Each node of the graph processes a set of input tensors and produces a // set of output Tensors. All inputs/output tensors are referenced by index. @@ -100,12 +102,6 @@ class NNAPIDelegate; // foo.Invoke(); // -struct TfLiteIntArrayDeleter { - void operator()(TfLiteIntArray* a) { - if (a) TfLiteIntArrayFree(a); - } -}; - class Interpreter { public: // Instantiate an interpreter. All errors associated with reading and @@ -117,6 +113,7 @@ class Interpreter { ~Interpreter(); + // Interpreters are not copyable as they have non-trivial memory semantics. Interpreter(const Interpreter&) = delete; Interpreter& operator=(const Interpreter&) = delete; @@ -197,34 +194,40 @@ class Interpreter { // Functions to access tensor data // Read only access to list of inputs. - const std::vector& inputs() const { return inputs_; } + const std::vector& inputs() const { return primary_subgraph().inputs(); } // Return the name of a given input. The given index must be between 0 and // inputs().size(). const char* GetInputName(int index) const { - return context_.tensors[inputs_[index]].name; + return context_->tensors[inputs()[index]].name; } // Read only access to list of outputs. - const std::vector& outputs() const { return outputs_; } + const std::vector& outputs() const { + return primary_subgraph().outputs(); + } // Read only access to list of variable tensors. - const std::vector& variables() const { return variables_; } + const std::vector& variables() const { + return primary_subgraph().variables(); + } // Return the name of a given output. The given index must be between 0 and // outputs().size(). const char* GetOutputName(int index) const { - return context_.tensors[outputs_[index]].name; + return context_->tensors[outputs()[index]].name; } // Return the number of tensors in the model. - size_t tensors_size() const { return context_.tensors_size; } + size_t tensors_size() const { return context_->tensors_size; } // Return the number of ops in the model. - size_t nodes_size() const { return nodes_and_registration_.size(); } + size_t nodes_size() const { return primary_subgraph().nodes_size(); } // WARNING: Experimental interface, subject to change - const std::vector& execution_plan() const { return execution_plan_; } + const std::vector& execution_plan() const { + return primary_subgraph().execution_plan(); + } // WARNING: Experimental interface, subject to change // Overrides execution plan. This bounds checks indices sent in. @@ -234,27 +237,18 @@ class Interpreter { // TODO(aselle): Create a safe ArrayHandle interface to avoid exposing this // read/write access to structure TfLiteTensor* tensor(int tensor_index) { - if (tensor_index < 0 || - static_cast(tensor_index) >= context_.tensors_size) - return nullptr; - return &context_.tensors[tensor_index]; + return primary_subgraph().tensor(tensor_index); } // Get an immutable tensor data structure. const TfLiteTensor* tensor(int tensor_index) const { - if (tensor_index < 0 || - static_cast(tensor_index) >= context_.tensors_size) - return nullptr; - return &context_.tensors[tensor_index]; + return primary_subgraph().tensor(tensor_index); } // Get a pointer to an operation and registration data structure if in bounds. const std::pair* node_and_registration( int node_index) const { - if (node_index < 0 || - static_cast(node_index) >= nodes_and_registration_.size()) - return nullptr; - return &nodes_and_registration_[node_index]; + return primary_subgraph().node_and_registration(node_index); } // Perform a checked cast to the appropriate tensor type (mutable pointer @@ -285,28 +279,28 @@ class Interpreter { // index must be between 0 and inputs().size(). template T* typed_input_tensor(int index) { - return typed_tensor(inputs_[index]); + return typed_tensor(inputs()[index]); } // Return an immutable pointer into the data of a given input tensor. The // given index must be between 0 and inputs().size(). template const T* typed_input_tensor(int index) const { - return typed_tensor(inputs_[index]); + return typed_tensor(inputs()[index]); } // Return a mutable pointer into the data of a given output tensor. The given // index must be between 0 and outputs().size(). template T* typed_output_tensor(int index) { - return typed_tensor(outputs_[index]); + return typed_tensor(outputs()[index]); } // Return an immutable pointer into the data of a given output tensor. The // given index must be between 0 and outputs().size(). template const T* typed_output_tensor(int index) const { - return typed_tensor(outputs_[index]); + return typed_tensor(outputs()[index]); } // Change the dimensionality of a given tensor. Note, this is only acceptable @@ -321,7 +315,6 @@ class Interpreter { // Update allocations for all tensors. This will redim dependent tensors using // the input tensor dimensionality as given. This is relatively expensive. // If you know that your sizes are not changing, you need not call this. - // Returns status of success or failure. TfLiteStatus AllocateTensors(); @@ -342,14 +335,12 @@ class Interpreter { // Allow float16 precision for FP32 calculation when possible. // default: not allow. // WARNING: This is an experimental API and subject to change. - void SetAllowFp16PrecisionForFp32(bool allow) { - context_.allow_fp32_relax_to_fp16 = allow; - } + void SetAllowFp16PrecisionForFp32(bool allow); // Get the half precision flag. // WARNING: This is an experimental API and subject to change. bool GetAllowFp16PrecisionForFp32() const { - return context_.allow_fp32_relax_to_fp16; + return context_->allow_fp32_relax_to_fp16; } // Owning handle to a TfLiteDelegate instance. @@ -366,18 +357,7 @@ class Interpreter { // it might require to copy the data from delegate buffer to raw memory. // WARNING: This is an experimental API and subject to change. TfLiteStatus EnsureTensorDataIsReadable(int tensor_index) { - TfLiteTensor* t = tensor(tensor_index); - TF_LITE_ENSURE(&context_, t != nullptr); - if (t->data_is_stale) { - TF_LITE_ENSURE(&context_, t->delegate != nullptr); - TF_LITE_ENSURE(&context_, t->buffer_handle != kTfLiteNullBufferHandle); - // This can be null if the delegate doesn't use its own buffer. - TF_LITE_ENSURE(&context_, t->delegate->CopyFromBufferHandle != nullptr); - t->delegate->CopyFromBufferHandle( - &context_, t->delegate, t->buffer_handle, t->data.raw, t->bytes); - t->data_is_stale = false; - } - return kTfLiteOk; + return primary_subgraph().EnsureTensorDataIsReadable(tensor_index); } // Set the delegate buffer handle to a tensor. It can be called in the @@ -400,9 +380,9 @@ class Interpreter { TfLiteBufferHandle* buffer_handle, TfLiteDelegate** delegate); - void SetProfiler(profiling::Profiler* profiler) { profiler_ = profiler; } + void SetProfiler(profiling::Profiler* profiler); - profiling::Profiler* GetProfiler() { return profiler_; } + profiling::Profiler* GetProfiler(); // The default capacity of `tensors_` vector. static constexpr int kTensorsReservedCapacity = 128; @@ -435,143 +415,48 @@ class Interpreter { const char* OpProfilingString(const TfLiteRegistration& op_reg, const TfLiteNode* node) const { if (op_reg.profiling_string == nullptr) return nullptr; - return op_reg.profiling_string(&context_, node); + return op_reg.profiling_string(context_, node); } // Set the value of an external context. void SetExternalContext(TfLiteExternalContextType type, TfLiteExternalContext* ctx); + // Adds `subgraphs_to_add` subgraphs, preserving pre-existing Subgraph + // entries. The value pointed to by `first_new_subgraph_index` will be set to + // the index of the first new subgraph if `first_new_subgraph_index` is + // non-null. + // WARNING: This is an experimental API and subject to change. + void AddSubgraphs(int subgraphs_to_add, + int* first_new_subgraph_index = nullptr); + + // Return the number of subgraphs in the model. + // WARNING: This is an experimental API and subject to change. + size_t subgraphs_size() const { return subgraphs_.size(); } + + // Get a pointer to a subgraph if in bounds. + // WARNING: This is an experimental API and subject to change. + Subgraph* subgraph(int subgraph_index) { + if (subgraph_index < 0 || + static_cast(subgraph_index) >= subgraphs_size()) + return nullptr; + return &*subgraphs_[subgraph_index]; + } + + // WARNING: Experimental interface, subject to change + Subgraph& primary_subgraph() { + return *subgraphs_.front(); // Safe as subgraphs_ always has 1 entry. + } + + // WARNING: Experimental interface, subject to change + const Subgraph& primary_subgraph() const { + return *subgraphs_.front(); // Safe as subgraphs_ always has 1 entry. + } + private: friend class InterpreterBuilder; friend class InterpreterTest; - // Prevent 'context_' from accessing functions that are only available to - // delegated kernels. - void SwitchToKernelContext(); - - // Add delegate-only functions to 'context_'. - void SwitchToDelegateContext(); - - // Give 'op_reg' a chance to initialize itself using the contents of - // 'buffer'. - void* OpInit(const TfLiteRegistration& op_reg, const char* buffer, - size_t length) { - if (op_reg.init == nullptr) return nullptr; - return op_reg.init(&context_, buffer, length); - } - - // Let 'op_reg' release any memory it might have allocated via 'OpInit'. - void OpFree(const TfLiteRegistration& op_reg, void* buffer) { - if (op_reg.free == nullptr) return; - if (buffer) { - op_reg.free(&context_, buffer); - } - } - - // Prepare the given 'node' for execution. - TfLiteStatus OpPrepare(const TfLiteRegistration& op_reg, TfLiteNode* node) { - if (op_reg.prepare == nullptr) return kTfLiteOk; - return op_reg.prepare(&context_, node); - } - - // Invoke the operator represented by 'node'. - TfLiteStatus OpInvoke(const TfLiteRegistration& op_reg, TfLiteNode* node) { - if (op_reg.invoke == nullptr) return kTfLiteError; - return op_reg.invoke(&context_, node); - } - - // Call OpPrepare() for as many ops as possible, allocating memory for their - // tensors. If an op containing dynamic tensors is found, preparation will be - // postponed until this function is called again. This allows the interpreter - // to wait until Invoke() to resolve the sizes of dynamic tensors. - TfLiteStatus PrepareOpsAndTensors(); - - // Call OpPrepare() for all ops starting at 'first_node'. Stop when a - // dynamic tensors is found or all ops have been prepared. Fill - // 'last_node_prepared' with the id of the op containing dynamic tensors, or - // the last in the graph. - TfLiteStatus PrepareOpsStartingAt(int first_execution_plan_index, - int* last_execution_plan_index_prepared); - - // Tensors needed by the interpreter. Use `AddTensors` to add more blank - // tensor entries. Note, `tensors_.data()` needs to be synchronized to the - // `context_` whenever this std::vector is reallocated. Currently this - // only happens in `AddTensors()`. - std::vector tensors_; - - // Check if an array of tensor indices are valid with respect to the Tensor - // array. - // NOTE: this changes consistent_ to be false if indices are out of bounds. - TfLiteStatus CheckTensorIndices(const char* label, const int* indices, - int length); - - // Compute the number of bytes required to represent a tensor with dimensions - // specified by the array dims (of length dims_size). Returns the status code - // and bytes. - TfLiteStatus BytesRequired(TfLiteType type, const int* dims, size_t dims_size, - size_t* bytes); - - // Request an tensor be resized implementation. If the given tensor is of - // type kTfLiteDynamic it will also be allocated new memory. - TfLiteStatus ResizeTensorImpl(TfLiteTensor* tensor, TfLiteIntArray* new_size); - - // Report a detailed error string (will be printed to stderr). - // TODO(aselle): allow user of class to provide alternative destinations. - void ReportErrorImpl(const char* format, va_list args); - - // Entry point for C node plugin API to request an tensor be resized. - static TfLiteStatus ResizeTensor(TfLiteContext* context, TfLiteTensor* tensor, - TfLiteIntArray* new_size); - // Entry point for C node plugin API to report an error. - static void ReportError(TfLiteContext* context, const char* format, ...); - - // Entry point for C node plugin API to add new tensors. - static TfLiteStatus AddTensors(TfLiteContext* context, int tensors_to_add, - int* first_new_tensor_index); - - // WARNING: This is an experimental API and subject to change. - // Entry point for C API ReplaceNodeSubsetsWithDelegateKernels - static TfLiteStatus ReplaceNodeSubsetsWithDelegateKernels( - TfLiteContext* context, TfLiteRegistration registration, - const TfLiteIntArray* nodes_to_replace, TfLiteDelegate* delegate); - - // Update the execution graph to replace some of the nodes with stub - // nodes. Specifically any node index that has `nodes[index]==1` will be - // slated for replacement with a delegate kernel specified by registration. - // Ownership of 'nodes_to_replace' and 'delegate' remains with the caller. - // WARNING: This is an experimental interface that is subject to change. - TfLiteStatus ReplaceNodeSubsetsWithDelegateKernels( - TfLiteRegistration registration, const TfLiteIntArray* nodes_to_replace, - TfLiteDelegate* delegate); - - // WARNING: This is an experimental interface that is subject to change. - // Gets the internal pointer to a TensorFlow lite node by node_index. - TfLiteStatus GetNodeAndRegistration(int node_index, TfLiteNode** node, - TfLiteRegistration** registration); - - // WARNING: This is an experimental interface that is subject to change. - // Entry point for C node plugin API to get a node by index. - static TfLiteStatus GetNodeAndRegistration(struct TfLiteContext*, - int node_index, TfLiteNode** node, - TfLiteRegistration** registration); - - // WARNING: This is an experimental interface that is subject to change. - // Gets an TfLiteIntArray* representing the execution plan. The interpreter - // owns this memory and it is only guaranteed to exist during the invocation - // of the delegate prepare. - TfLiteStatus GetExecutionPlan(TfLiteIntArray** execution_plan); - - // WARNING: This is an experimental interface that is subject to change. - // Entry point for C node plugin API to get the execution plan. - static TfLiteStatus GetExecutionPlan(struct TfLiteContext* context, - TfLiteIntArray** execution_plan); - - // Retrieve an existing external context by type. - TfLiteExternalContext* GetExternalContext(TfLiteExternalContextType type); - static TfLiteExternalContext* GetExternalContext( - struct TfLiteContext* context, TfLiteExternalContextType type); - // Set the value of an external context. static void SetExternalContext(struct TfLiteContext* context, TfLiteExternalContextType type, @@ -587,105 +472,28 @@ class Interpreter { return ModifyGraphWithDelegate(owned_delegates_.back().get()); } - // Ensures that `tensors_` has at least `kTensorsCapacityHeadroom` extra - // capacity. Calling this function may invalidate existing pointers to - // tensors. After calling this function, adding `kTensorsCapacityHeadroom` - // more tensors won't invalidate the pointer to existing tensors. - void EnsureTensorsVectorCapacity() { - const size_t required_capacity = tensors_size() + kTensorsCapacityHeadroom; - if (required_capacity > tensors_.capacity()) { - tensors_.reserve(required_capacity); - context_.tensors = tensors_.data(); - } - } - - // The state of the Interpreter. - enum State { - // The interpreter isn't ready to be invoked. - // `AllocateTensor` need to be called to enter an invokable state. - kStateUninvokable = 0, - // The interpreter is ready to be invoked. - kStateInvokable, - // The interpreter is ready to be invoked, and graph can't be further - // modified. The interpreter will enter this state when calling - // `ModifyGraphWithDelegate` with `allow_dynamic_tensors=false`. - kStateInvokableAndImmutable, - }; - State state_ = kStateUninvokable; - // A pure C data structure used to communicate with the pure C plugin // interface. To avoid copying tensor metadata, this is also the definitive // structure to store tensors. - TfLiteContext context_; - - // Node inputs/outputs are stored in TfLiteNode and TfLiteRegistration stores - // function pointers to actual implementation. - std::vector> - nodes_and_registration_; - - // Whether the model is consistent. That is to say if the inputs and outputs - // of every node and the global inputs and outputs are valid indexes into - // the tensor array. - bool consistent_ = true; - - // Array of indices representing the tensors that are inputs to the - // interpreter. - std::vector inputs_; - - // Array of indices representing the tensors that are outputs to the - // interpreter. - std::vector outputs_; - - // Array of indices representing the tensors that are variable tensors. - std::vector variables_; + // This is the primary subgraph context. + TfLiteContext* context_; // The error reporter delegate that tflite will forward queries errors to. ErrorReporter* error_reporter_; - // Index of the next node to prepare. - // During Invoke(), Interpreter will allocate input tensors first, which are - // known to be fixed size. Then it will allocate outputs from nodes as many - // as possible. When there is a node that produces dynamic sized tensor. - // Interpreter will stop allocating tensors, set the value of next allocate - // node id, and execute the node to generate the output tensor before continue - // to allocate successors. This process repeats until all nodes are executed. - // NOTE: this relies on the order of nodes that is in topological order. - int next_execution_plan_index_to_prepare_; - - // WARNING: This is an experimental interface that is subject to change. - // This is a list of node indices (to index into nodes_and_registration). - // This represents a valid topological sort (dependency ordered) execution - // plan. In particular, it is valid for this ordering to contain only a - // subset of the node indices. - std::vector execution_plan_; - - // In the future, we'd like a TfLiteIntArray compatible representation. - // TODO(aselle): replace execution_plan_ with this. - std::unique_ptr plan_cache_; - - // Whether to delegate to NN API - std::unique_ptr nnapi_delegate_; - // List of delegates that have been installed and are owned by this // interpreter instance. Useful if client delegate ownership is burdensome. // WARNING: This is an experimental API and subject to change. // TODO(b/116667551): Use TfLiteExternalContext for storing state. std::vector owned_delegates_; - std::unique_ptr memory_planner_; - bool allow_buffer_handle_output_ = false; - // Tracking bit for whether a tensor was resized in the course of an op - // invocation. This is a useful hint to ensure that dynamic tensor outputs - // trigger downstream reallocation after op invocation. - bool tensor_resized_since_op_invoke_ = false; - - // Profiler for this interpreter instance. - profiling::Profiler* profiler_ = nullptr; - // List of active external contexts. TfLiteExternalContext* external_contexts_[kTfLiteMaxExternalContexts]; + + // Subgraphs + std::vector> subgraphs_; }; } // namespace tflite diff --git a/tensorflow/lite/interpreter_test.cc b/tensorflow/lite/interpreter_test.cc index 7f03c3ceba1..78b5d1b8873 100644 --- a/tensorflow/lite/interpreter_test.cc +++ b/tensorflow/lite/interpreter_test.cc @@ -38,7 +38,7 @@ class InterpreterTest : public ::testing::Test { } protected: - TfLiteContext* GetInterpreterContext() { return &interpreter_.context_; } + TfLiteContext* GetInterpreterContext() { return interpreter_.context_; } Interpreter interpreter_; }; @@ -566,7 +566,7 @@ TEST(BasicInterpreter, ThreeStepAllocate) { DynamicBuffer buf; StringRef str_ref = GetString(input, 0); buf.AddString(str_ref); - buf.WriteToTensor(output); + buf.WriteToTensorAsVector(output); return kTfLiteOk; }; @@ -1090,17 +1090,17 @@ class TestDelegate : public ::testing::Test { TfLiteIntArrayFree(nodes_to_separate); return kTfLiteOk; }; - delegate_.CopyToBufferHandle = - [](TfLiteContext* context, TfLiteDelegate* delegate, - TfLiteBufferHandle buffer_handle, void* data, - size_t size) -> TfLiteStatus { + delegate_.CopyToBufferHandle = [](TfLiteContext* context, + TfLiteDelegate* delegate, + TfLiteBufferHandle buffer_handle, + TfLiteTensor* tensor) -> TfLiteStatus { // TODO(ycling): Implement tests to test buffer copying logic. return kTfLiteOk; }; delegate_.CopyFromBufferHandle = [](TfLiteContext* context, TfLiteDelegate* delegate, - TfLiteBufferHandle buffer_handle, void* data, - size_t size) -> TfLiteStatus { + TfLiteBufferHandle buffer_handle, + TfLiteTensor* output) -> TfLiteStatus { // TODO(ycling): Implement tests to test buffer copying logic. return kTfLiteOk; }; diff --git a/tensorflow/lite/java/src/main/java/org/tensorflow/lite/DataType.java b/tensorflow/lite/java/src/main/java/org/tensorflow/lite/DataType.java index 41093e8ffe6..bd47574f71b 100644 --- a/tensorflow/lite/java/src/main/java/org/tensorflow/lite/DataType.java +++ b/tensorflow/lite/java/src/main/java/org/tensorflow/lite/DataType.java @@ -27,7 +27,10 @@ public enum DataType { UINT8(3), /** 64-bit signed integer. */ - INT64(4); + INT64(4), + + /** Strings. */ + STRING(5); private final int value; @@ -46,6 +49,8 @@ public enum DataType { return 1; case INT64: return 8; + case STRING: + return -1; } throw new IllegalArgumentException( "DataType error: DataType " + this + " is not supported yet"); @@ -82,6 +87,8 @@ public enum DataType { return "byte"; case INT64: return "long"; + case STRING: + return "string"; } throw new IllegalArgumentException( "DataType error: DataType " + this + " is not supported yet"); diff --git a/tensorflow/lite/java/src/main/java/org/tensorflow/lite/Tensor.java b/tensorflow/lite/java/src/main/java/org/tensorflow/lite/Tensor.java index 6ca47aa3edf..7aa24b4198a 100644 --- a/tensorflow/lite/java/src/main/java/org/tensorflow/lite/Tensor.java +++ b/tensorflow/lite/java/src/main/java/org/tensorflow/lite/Tensor.java @@ -162,6 +162,8 @@ public final class Tensor { return DataType.UINT8; } else if (long.class.equals(c)) { return DataType.INT64; + } else if (String.class.equals(c)) { + return DataType.STRING; } } throw new IllegalArgumentException( diff --git a/tensorflow/lite/java/src/main/native/BUILD b/tensorflow/lite/java/src/main/native/BUILD index 2abba243458..8f95f14518a 100644 --- a/tensorflow/lite/java/src/main/native/BUILD +++ b/tensorflow/lite/java/src/main/native/BUILD @@ -43,6 +43,7 @@ cc_library( "//tensorflow/lite:context", "//tensorflow/lite:framework", "//tensorflow/lite:schema_fbs_version", + "//tensorflow/lite:string_util", ], alwayslink = 1, ) diff --git a/tensorflow/lite/java/src/main/native/nativeinterpreterwrapper_jni.cc b/tensorflow/lite/java/src/main/native/nativeinterpreterwrapper_jni.cc index c7389c58110..1e98f942504 100644 --- a/tensorflow/lite/java/src/main/native/nativeinterpreterwrapper_jni.cc +++ b/tensorflow/lite/java/src/main/native/nativeinterpreterwrapper_jni.cc @@ -78,6 +78,8 @@ int getDataType(TfLiteType data_type) { return 3; case kTfLiteInt64: return 4; + case kTfLiteString: + return 5; default: return -1; } diff --git a/tensorflow/lite/java/src/main/native/tensor_jni.cc b/tensorflow/lite/java/src/main/native/tensor_jni.cc index 1d813d50da4..82d2679de9c 100644 --- a/tensorflow/lite/java/src/main/native/tensor_jni.cc +++ b/tensorflow/lite/java/src/main/native/tensor_jni.cc @@ -16,8 +16,10 @@ limitations under the License. #include "tensorflow/lite/java/src/main/native/tensor_jni.h" #include #include +#include "tensorflow/lite/c/c_api_internal.h" #include "tensorflow/lite/interpreter.h" #include "tensorflow/lite/java/src/main/native/exception_jni.h" +#include "tensorflow/lite/string_util.h" namespace { @@ -48,7 +50,7 @@ TfLiteTensor* GetTensorFromHandle(JNIEnv* env, jlong handle) { return reinterpret_cast(handle)->tensor(); } -size_t elementByteSize(TfLiteType data_type) { +size_t ElementByteSize(TfLiteType data_type) { // The code in this file makes the assumption that the // TensorFlow TF_DataTypes and the Java primitive types // have the same byte sizes. Validate that: @@ -77,11 +79,11 @@ size_t elementByteSize(TfLiteType data_type) { } } -size_t writeOneDimensionalArray(JNIEnv* env, jobject object, TfLiteType type, +size_t WriteOneDimensionalArray(JNIEnv* env, jobject object, TfLiteType type, void* dst, size_t dst_size) { jarray array = static_cast(object); const int num_elements = env->GetArrayLength(array); - size_t to_copy = num_elements * elementByteSize(type); + size_t to_copy = num_elements * ElementByteSize(type); if (to_copy > dst_size) { throwException(env, kIllegalStateException, "Internal error: cannot write Java array of %d bytes to " @@ -126,10 +128,10 @@ size_t writeOneDimensionalArray(JNIEnv* env, jobject object, TfLiteType type, } } -size_t readOneDimensionalArray(JNIEnv* env, TfLiteType data_type, +size_t ReadOneDimensionalArray(JNIEnv* env, TfLiteType data_type, const void* src, size_t src_size, jarray dst) { const int len = env->GetArrayLength(dst); - const size_t size = len * elementByteSize(data_type); + const size_t size = len * ElementByteSize(data_type); if (size > src_size) { throwException( env, kIllegalStateException, @@ -170,17 +172,17 @@ size_t readOneDimensionalArray(JNIEnv* env, TfLiteType data_type, return 0; } -size_t readMultiDimensionalArray(JNIEnv* env, TfLiteType data_type, char* src, +size_t ReadMultiDimensionalArray(JNIEnv* env, TfLiteType data_type, char* src, size_t src_size, int dims_left, jarray dst) { if (dims_left == 1) { - return readOneDimensionalArray(env, data_type, src, src_size, dst); + return ReadOneDimensionalArray(env, data_type, src, src_size, dst); } else { jobjectArray ndarray = static_cast(dst); int len = env->GetArrayLength(ndarray); size_t size = 0; for (int i = 0; i < len; ++i) { jarray row = static_cast(env->GetObjectArrayElement(ndarray, i)); - size += readMultiDimensionalArray(env, data_type, src + size, + size += ReadMultiDimensionalArray(env, data_type, src + size, src_size - size, dims_left - 1, row); env->DeleteLocalRef(row); if (env->ExceptionCheck()) return size; @@ -189,10 +191,43 @@ size_t readMultiDimensionalArray(JNIEnv* env, TfLiteType data_type, char* src, } } -size_t writeMultiDimensionalArray(JNIEnv* env, jobject src, TfLiteType type, +// Returns the total number of strings read. +int ReadMultiDimensionalStringArray(JNIEnv* env, TfLiteTensor* tensor, + int dims_left, int start_str_index, + jarray dst) { + jobjectArray object_array = static_cast(dst); + int len = env->GetArrayLength(object_array); + int num_strings_read = 0; + + // If dst is a 1-dimensional array, copy the strings into it. Else + // recursively call ReadMultiDimensionalStringArray over sub-dimensions. + if (dims_left == 1) { + for (int i = 0; i < len; ++i) { + const tflite::StringRef strref = + tflite::GetString(tensor, start_str_index + num_strings_read); + jstring string_dest = env->NewStringUTF(strref.str); + env->SetObjectArrayElement(object_array, i, string_dest); + env->DeleteLocalRef(string_dest); + ++num_strings_read; + } + } else { + for (int i = 0; i < len; ++i) { + jarray row = + static_cast(env->GetObjectArrayElement(object_array, i)); + num_strings_read += ReadMultiDimensionalStringArray( + env, tensor, dims_left - 1, start_str_index + num_strings_read, row); + env->DeleteLocalRef(row); + if (env->ExceptionCheck()) return num_strings_read; + } + } + + return num_strings_read; +} + +size_t WriteMultiDimensionalArray(JNIEnv* env, jobject src, TfLiteType type, int dims_left, char** dst, int dst_size) { if (dims_left <= 1) { - return writeOneDimensionalArray(env, src, type, *dst, dst_size); + return WriteOneDimensionalArray(env, src, type, *dst, dst_size); } else { jobjectArray ndarray = static_cast(src); int len = env->GetArrayLength(ndarray); @@ -200,7 +235,7 @@ size_t writeMultiDimensionalArray(JNIEnv* env, jobject src, TfLiteType type, for (int i = 0; i < len; ++i) { jobject row = env->GetObjectArrayElement(ndarray, i); char* next_dst = *dst + sz; - sz += writeMultiDimensionalArray(env, row, type, dims_left - 1, &next_dst, + sz += WriteMultiDimensionalArray(env, row, type, dims_left - 1, &next_dst, dst_size - sz); env->DeleteLocalRef(row); if (env->ExceptionCheck()) return sz; @@ -209,6 +244,44 @@ size_t writeMultiDimensionalArray(JNIEnv* env, jobject src, TfLiteType type, } } +void PopulateStringDynamicBuffer(JNIEnv* env, jobject src, + tflite::DynamicBuffer* dst_buffer, + int dims_left) { + jobjectArray object_array = static_cast(src); + const int num_elements = env->GetArrayLength(object_array); + + // If src is a 1-dimensional array, add the strings into dst_buffer. Else + // recursively call populateStringDynamicBuffer over sub-dimensions. + if (dims_left <= 1) { + for (int i = 0; i < num_elements; ++i) { + jstring string_obj = + static_cast(env->GetObjectArrayElement(object_array, i)); + const char* chars = env->GetStringUTFChars(string_obj, nullptr); + // + 1 for terminating character. + const int byte_len = env->GetStringUTFLength(string_obj) + 1; + dst_buffer->AddString(chars, byte_len); + env->ReleaseStringUTFChars(string_obj, chars); + env->DeleteLocalRef(string_obj); + } + } else { + for (int i = 0; i < num_elements; ++i) { + jobject row = env->GetObjectArrayElement(object_array, i); + PopulateStringDynamicBuffer(env, row, dst_buffer, dims_left - 1); + env->DeleteLocalRef(row); + if (env->ExceptionCheck()) return; + } + } +} + +void WriteMultiDimensionalStringArray(JNIEnv* env, jobject src, + TfLiteTensor* tensor) { + tflite::DynamicBuffer dst_buffer; + PopulateStringDynamicBuffer(env, src, &dst_buffer, tensor->dims->size); + if (!env->ExceptionCheck()) { + dst_buffer.WriteToTensor(tensor, /*new_shape=*/nullptr); + } +} + } // namespace JNIEXPORT jlong JNICALL Java_org_tensorflow_lite_Tensor_create( @@ -266,8 +339,14 @@ Java_org_tensorflow_lite_Tensor_readMultiDimensionalArray(JNIEnv* env, "Internal error: Cannot copy empty/scalar Tensors."); return; } - readMultiDimensionalArray(env, tensor->type, tensor->data.raw, tensor->bytes, - num_dims, static_cast(value)); + if (tensor->type == kTfLiteString) { + ReadMultiDimensionalStringArray(env, tensor, num_dims, 0, + static_cast(value)); + } else { + ReadMultiDimensionalArray(env, tensor->type, tensor->data.raw, + tensor->bytes, num_dims, + static_cast(value)); + } } JNIEXPORT void JNICALL @@ -277,7 +356,7 @@ Java_org_tensorflow_lite_Tensor_writeMultiDimensionalArray(JNIEnv* env, jobject src) { TfLiteTensor* tensor = GetTensorFromHandle(env, handle); if (tensor == nullptr) return; - if (tensor->data.raw == nullptr) { + if (tensor->type != kTfLiteString && tensor->data.raw == nullptr) { throwException(env, kIllegalArgumentException, "Internal error: Target Tensor hasn't been allocated."); return; @@ -287,8 +366,12 @@ Java_org_tensorflow_lite_Tensor_writeMultiDimensionalArray(JNIEnv* env, "Internal error: Cannot copy empty/scalar Tensors."); return; } - writeMultiDimensionalArray(env, src, tensor->type, tensor->dims->size, - &tensor->data.raw, tensor->bytes); + if (tensor->type == kTfLiteString) { + WriteMultiDimensionalStringArray(env, src, tensor); + } else { + WriteMultiDimensionalArray(env, src, tensor->type, tensor->dims->size, + &tensor->data.raw, tensor->bytes); + } } JNIEXPORT jint JNICALL Java_org_tensorflow_lite_Tensor_dtype(JNIEnv* env, diff --git a/tensorflow/lite/java/src/test/java/org/tensorflow/lite/DataTypeTest.java b/tensorflow/lite/java/src/test/java/org/tensorflow/lite/DataTypeTest.java index 6d6417f895e..8412ec0e9da 100644 --- a/tensorflow/lite/java/src/test/java/org/tensorflow/lite/DataTypeTest.java +++ b/tensorflow/lite/java/src/test/java/org/tensorflow/lite/DataTypeTest.java @@ -30,6 +30,7 @@ public final class DataTypeTest { assertThat(DataType.INT32.byteSize()).isEqualTo(4); assertThat(DataType.UINT8.byteSize()).isEqualTo(1); assertThat(DataType.INT64.byteSize()).isEqualTo(8); + assertThat(DataType.STRING.byteSize()).isEqualTo(-1); } @Test diff --git a/tensorflow/lite/java/src/test/java/org/tensorflow/lite/NativeInterpreterWrapperTest.java b/tensorflow/lite/java/src/test/java/org/tensorflow/lite/NativeInterpreterWrapperTest.java index 07d334c33b2..b00efa77cbf 100644 --- a/tensorflow/lite/java/src/test/java/org/tensorflow/lite/NativeInterpreterWrapperTest.java +++ b/tensorflow/lite/java/src/test/java/org/tensorflow/lite/NativeInterpreterWrapperTest.java @@ -43,6 +43,9 @@ public final class NativeInterpreterWrapperTest { private static final String BYTE_MODEL_PATH = "tensorflow/lite/java/src/testdata/uint8.bin"; + private static final String STRING_MODEL_PATH = + "tensorflow/lite/java/src/testdata/string.bin"; + private static final String QUANTIZED_MODEL_PATH = "tensorflow/lite/java/src/testdata/quantized.bin"; @@ -224,6 +227,50 @@ public final class NativeInterpreterWrapperTest { wrapper.close(); } + @Test + public void testRunWithString() { + NativeInterpreterWrapper wrapper = new NativeInterpreterWrapper(STRING_MODEL_PATH); + String[] oneD = {"s1", "s22", "s333"}; + String[][] twoD = {oneD, oneD, oneD, oneD, oneD, oneD, oneD, oneD}; + String[][][] threeD = {twoD, twoD, twoD, twoD, twoD, twoD, twoD, twoD}; + String[][][][] fourD = {threeD, threeD}; + Object[] inputs = {fourD}; + String[][][][] parsedOutputs = new String[2][4][4][12]; + Map outputs = new HashMap<>(); + outputs.put(0, parsedOutputs); + wrapper.run(inputs, outputs); + String[] outputOneD = parsedOutputs[0][0][0]; + String[] expected = { + "s1", "s22", "s333", "s1", "s22", "s333", "s1", "s22", "s333", "s1", "s22", "s333" + }; + assertThat(outputOneD).isEqualTo(expected); + wrapper.close(); + } + + @Test + public void testRunWithString_wrongShapeError() { + NativeInterpreterWrapper wrapper = new NativeInterpreterWrapper(STRING_MODEL_PATH); + String[] oneD = {"s1", "s22", "s333"}; + String[][] twoD = {oneD, oneD, oneD, oneD, oneD, oneD, oneD, oneD}; + String[][][] threeD = {twoD, twoD, twoD, twoD, twoD, twoD, twoD, twoD}; + String[][][][] fourD = {threeD, threeD}; + Object[] inputs = {fourD}; + String[][][][] parsedOutputs = new String[2][4][4][10]; + Map outputs = new HashMap<>(); + outputs.put(0, parsedOutputs); + try { + wrapper.run(inputs, outputs); + fail(); + } catch (IllegalArgumentException e) { + assertThat(e) + .hasMessageThat() + .contains( + "Cannot copy between a TensorFlowLite tensor with shape [2, 4, 4, 12] and " + + "a Java object with shape [2, 4, 4, 10]"); + } + wrapper.close(); + } + @Test public void testRunWithByteBufferHavingBytes() { NativeInterpreterWrapper wrapper = new NativeInterpreterWrapper(BYTE_MODEL_PATH); diff --git a/tensorflow/lite/java/src/testdata/string.bin b/tensorflow/lite/java/src/testdata/string.bin new file mode 100644 index 0000000000000000000000000000000000000000..36a2509acdfa17841d0c128674e7b4e382ad00fc GIT binary patch literal 584 zcmZXRu};H442De!O@;zRimD79GB7Y8K!P`j#Dr7{Rg5i2M9?LPl9ZL#;Srdb7(buffer); } @@ -65,6 +74,10 @@ void LogSoftmaxFree(TfLiteContext* context, void* buffer) { delete reinterpret_cast(buffer); } +void PreluFree(TfLiteContext* context, void* buffer) { + delete reinterpret_cast(buffer); +} + TfLiteStatus GenericPrepare(TfLiteContext* context, TfLiteNode* node) { TF_LITE_ENSURE_EQ(context, NumInputs(node), 1); TF_LITE_ENSURE_EQ(context, NumOutputs(node), 1); @@ -253,13 +266,18 @@ TfLiteStatus PreluPrepare(TfLiteContext* context, TfLiteNode* node) { const TfLiteTensor* input = GetInput(context, node, 0); TfLiteTensor* output = GetOutput(context, node, 0); const TfLiteTensor* alpha = GetInput(context, node, 1); + PreluOpData* data = reinterpret_cast(node->user_data); - // Currently only Float32 is supported - // TODO(ycling): Support other data types. - TF_LITE_ENSURE_EQ(context, input->type, kTfLiteFloat32); - TF_LITE_ENSURE_EQ(context, alpha->type, kTfLiteFloat32); + TF_LITE_ENSURE_EQ(context, input->type, alpha->type); output->type = input->type; + if (output->type == kTfLiteUInt8 || output->type == kTfLiteInt16) { + double real_multiplier = + input->params.scale * alpha->params.scale / output->params.scale; + QuantizeMultiplierSmallerThanOneExp( + real_multiplier, &data->output_multiplier, &data->output_shift); + } + // PRelu (parameteric Relu) shares the same alpha value on "shared axis". // This means it's always required to "broadcast" alpha values in PRelu. TfLiteIntArray* output_size = nullptr; @@ -288,8 +306,8 @@ TfLiteStatus ReluEval(TfLiteContext* context, TfLiteNode* node) { return kTfLiteOk; } break; default: - context->ReportError(context, "Only float32 supported currently, got %d.", - input->type); + context->ReportError(context, "Only float32 supported currently, got %s.", + TfLiteTypeGetName(input->type)); return kTfLiteError; } } @@ -309,8 +327,8 @@ TfLiteStatus Relu1Eval(TfLiteContext* context, TfLiteNode* node) { return kTfLiteOk; } break; default: - context->ReportError(context, "Only float32 supported currently, got %d.", - input->type); + context->ReportError(context, "Only float32 supported currently, got %s.", + TfLiteTypeGetName(input->type)); return kTfLiteError; } } @@ -327,9 +345,24 @@ TfLiteStatus Relu6Eval(TfLiteContext* context, TfLiteNode* node) { for (; in < in_end; in++, out++) *out = std::min(std::max(0.f, *in), 6.f); return kTfLiteOk; } break; + case kTfLiteUInt8: { + ActivationParams params; + params.activation_type = FusedActivationFunctionType::kRelu6; + params.quantized_activation_min = std::max( + 0, output->params.zero_point + + static_cast(roundf(0.f / output->params.scale))); + params.quantized_activation_max = std::min( + 255, output->params.zero_point + + static_cast(roundf(6.f / output->params.scale))); + optimized_ops::ReluX(params, GetTensorShape(input), + GetTensorData(input), GetTensorShape(output), + GetTensorData(output)); + return kTfLiteOk; + } break; default: - context->ReportError(context, "Only float32 supported currently, got %d.", - input->type); + context->ReportError( + context, "Only float32 and uint8 supported currently, got %s.", + TfLiteTypeGetName(input->type)); return kTfLiteError; } } @@ -367,8 +400,8 @@ TfLiteStatus TanhEval(TfLiteContext* context, TfLiteNode* node) { return kTfLiteOk; } break; default: - context->ReportError(context, "Only float32 supported currently, got %d.", - input->type); + context->ReportError(context, "Only float32 supported currently, got %s.", + TfLiteTypeGetName(input->type)); return kTfLiteError; } } @@ -407,9 +440,8 @@ TfLiteStatus SigmoidEval(TfLiteContext* context, TfLiteNode* node) { break; } default: - context->ReportError(context, "Only float32 supported currently, got %d.", - input->type); - return kTfLiteError; + context->ReportError(context, "Only float32 supported currently, got %s.", + TfLiteTypeGetName(input->type)); } return kTfLiteOk; } @@ -604,8 +636,8 @@ TfLiteStatus SoftmaxEval(TfLiteContext* context, TfLiteNode* node) { } default: context->ReportError( - context, "Only float32 and uint8_t supported currently, got %d.", - input->type); + context, "Only float32 and uint8_t supported currently, got %s.", + TfLiteTypeGetName(input->type)); return kTfLiteError; } } @@ -636,8 +668,8 @@ TfLiteStatus LogSoftmaxEval(TfLiteContext* context, TfLiteNode* node) { return kTfLiteOk; } default: - context->ReportError(context, "Only float32 supported currently., got %d", - input->type); + context->ReportError(context, "Only float32 supported currently., got %s", + TfLiteTypeGetName(input->type)); return kTfLiteError; } } @@ -651,16 +683,57 @@ TfLiteStatus PreluEval(TfLiteContext* context, TfLiteNode* node) { const TfLiteTensor* input = GetInput(context, node, 0); const TfLiteTensor* alpha = GetInput(context, node, 1); TfLiteTensor* output = GetOutput(context, node, 0); - if (input->type != kTfLiteFloat32) { - context->ReportError(context, "Only float32 supported currently, got %d.", - input->type); - return kTfLiteError; + const PreluOpData* data = reinterpret_cast(node->user_data); + switch (input->type) { + case kTfLiteFloat32: { + reference_ops::BroadcastBinaryFunction4DSlow( + GetTensorShape(input), GetTensorData(input), + GetTensorShape(alpha), GetTensorData(alpha), + GetTensorShape(output), GetTensorData(output), + ApplyPrelu); + return kTfLiteOk; + } break; + case kTfLiteUInt8: { + PreluParams op_params; + op_params.input_offset = -input->params.zero_point; + op_params.alpha_offset = -alpha->params.zero_point; + op_params.output_offset = output->params.zero_point; + op_params.output_multiplier = data->output_multiplier; + op_params.output_shift = data->output_shift; + reference_ops::BroadcastPrelu4DSlow( + op_params, GetTensorShape(input), GetTensorData(input), + GetTensorShape(alpha), GetTensorData(alpha), + GetTensorShape(output), GetTensorData(output)); + return kTfLiteOk; + } break; + default: + context->ReportError(context, + "Only float32, uint8 supported currently, got %d.", + TfLiteTypeGetName(input->type)); + return kTfLiteError; + } +} + +TfLiteStatus LeakyReluEval(TfLiteContext* context, TfLiteNode* node) { + const TfLiteTensor* input = GetInput(context, node, 0); + TfLiteTensor* output = GetOutput(context, node, 0); + const auto* params = + reinterpret_cast(node->builtin_data); + + LeakyReluParams op_params; + op_params.alpha = params->alpha; + switch (input->type) { + case kTfLiteFloat32: { + optimized_ops::LeakyRelu( + op_params, GetTensorShape(input), GetTensorData(input), + GetTensorShape(output), GetTensorData(output)); + return kTfLiteOk; + } break; + default: + context->ReportError(context, "Only float32 supported currently, got %s.", + TfLiteTypeGetName(input->type)); + return kTfLiteError; } - reference_ops::BroadcastBinaryFunction4DSlow( - GetTensorShape(input), GetTensorData(input), GetTensorShape(alpha), - GetTensorData(alpha), GetTensorShape(output), - GetTensorData(output), ApplyPrelu); - return kTfLiteOk; } } // namespace activations @@ -715,12 +788,19 @@ TfLiteRegistration* Register_LOG_SOFTMAX() { } TfLiteRegistration* Register_PRELU() { - static TfLiteRegistration r = {/*init=*/nullptr, /*free=*/nullptr, + static TfLiteRegistration r = {activations::PreluInit, activations::PreluFree, activations::PreluPrepare, activations::PreluEval}; return &r; } +TfLiteRegistration* Register_LEAKY_RELU() { + static TfLiteRegistration r = {/*init=*/nullptr, /*free=*/nullptr, + activations::GenericPrepare, + activations::LeakyReluEval}; + return &r; +} + } // namespace builtin } // namespace ops } // namespace tflite diff --git a/tensorflow/lite/kernels/activations_test.cc b/tensorflow/lite/kernels/activations_test.cc index fff4121dc0c..67f137baff2 100644 --- a/tensorflow/lite/kernels/activations_test.cc +++ b/tensorflow/lite/kernels/activations_test.cc @@ -170,6 +170,29 @@ TEST(FloatActivationsOpTest, Tanh) { }))); } +TEST(QuantizedActivationsOpTest, Relu6) { + const float kMin = -1; + const float kMax = 127.f / 128.f; + QuantizedActivationsOpModel m( + BuiltinOperator_RELU6, + /*input=*/{TensorType_UINT8, {1, 2, 4, 1}, 8 * kMin, 8 * kMax}, + /*output=*/{TensorType_UINT8, {1, 2, 4, 1}, 8 * kMin, 8 * kMax}); + m.SetInput({ + 0, -6, 2, 4, // + 3, -2, 10, 1, // + }); + m.Invoke(); + EXPECT_THAT(m.GetDequantizedOutput(), + ElementsAreArray(ArrayFloatNear( + { + 0, 0, 2, 4, // + 3, 0, 6, 1, // + }, + kQuantizedTolerance))); + EXPECT_THAT(m.GetOutput(), + ElementsAreArray({128, 128, 160, 192, 176, 128, 224, 144})); +} + TEST(QuantizedActivationsOpTest, Tanh) { const float kMin = -1; const float kMax = 127.f / 128.f; @@ -563,22 +586,17 @@ TEST(QuantizedActivationsOpTest, LogSoftmax) { ElementsAreArray({189, 93, 221, 253, 142, 63, 255, 111})); } -class PReluOpModel : public SingleOpModel { +// A base class of PRelu op model. It provides the constructor for +// FloatPReluOpModel and QuantizedPReluOpModel. +class BasePReluOpModel : public SingleOpModel { public: - PReluOpModel(const TensorData& input, const TensorData& alpha) { + BasePReluOpModel(const TensorData& input, const TensorData& alpha) { input_ = AddInput(input); alpha_ = AddInput(alpha); - output_ = AddOutput(input); + output_ = AddOutput({input.type, input.shape, input.min, input.max}); SetBuiltinOp(BuiltinOperator_PRELU, BuiltinOptions_NONE, 0); BuildInterpreter({GetShape(input_), GetShape(alpha_)}); } - void SetInput(std::initializer_list data) { - PopulateTensor(input_, data); - } - void SetAlpha(std::initializer_list data) { - PopulateTensor(alpha_, data); - } - std::vector GetOutput() { return ExtractVector(output_); } protected: int input_; @@ -586,9 +604,47 @@ class PReluOpModel : public SingleOpModel { int output_; }; +// The FloatPReluOpModel class handles float input and output. +class FloatPReluOpModel : public BasePReluOpModel { + public: + using BasePReluOpModel::BasePReluOpModel; + + void SetInput(std::initializer_list data) { + PopulateTensor(input_, data); + } + void SetAlpha(std::initializer_list data) { + PopulateTensor(alpha_, data); + } + std::vector GetOutput() { return ExtractVector(output_); } +}; + +// The QuantizedPReluOpModel class handles quantized input and output. +class QuantizedPReluOpModel : public BasePReluOpModel { + public: + using BasePReluOpModel::BasePReluOpModel; + + template + void SetInput(std::initializer_list data) { + QuantizeAndPopulate(input_, data); + } + template + void SetAlpha(std::initializer_list data) { + QuantizeAndPopulate(alpha_, data); + } + template + std::vector GetOutput() { + return ExtractVector(output_); + } + template + std::vector GetDequantizedOutput() { + return Dequantize(ExtractVector(output_), GetScale(output_), + GetZeroPoint(output_)); + } +}; + TEST(FloatActivationsOpTest, PRelu) { - PReluOpModel m({TensorType_FLOAT32, {1, 2, 2, 3}}, - {TensorType_FLOAT32, {1, 1, 3}}); + FloatPReluOpModel m({TensorType_FLOAT32, {1, 2, 2, 3}}, + {TensorType_FLOAT32, {1, 1, 3}}); m.SetInput({ 0.0f, 0.0f, 0.0f, // Row 1, Column 1 @@ -606,6 +662,69 @@ TEST(FloatActivationsOpTest, PRelu) { })); } +TEST(QuantizedActivationsOpTest, PRelu) { + const float kMin = -1; + const float kMax = 127.f / 128.f; + QuantizedPReluOpModel m({TensorType_UINT8, {1, 2, 2, 3}, kMin, kMax}, + {TensorType_UINT8, {1, 1, 3}, kMin, kMax}); + m.SetInput({ + 0.0f, 0.0f, 0.0f, // Row 1, Column 1 + 0.5f, 0.5f, 0.5f, // Row 1, Column 2 + -1.0f, -1.0f, -1.0f, // Row 2, Column 1 + -0.25f, -0.25f, -0.25f, // Row 1, Column 2 + }); + m.SetAlpha({0.0f, 0.5f, -0.5f}); + m.Invoke(); + EXPECT_THAT(m.GetDequantizedOutput(), + ElementsAreArray(ArrayFloatNear( + { + 0.0f, 0.0f, 0.0f, // Row 1, Column 1 + 0.5f, 0.5f, 0.5f, // Row 1, Column 2 + 0.0f, -0.5f, 0.5f, // Row 2, Column 1 + 0.0f, -0.125f, 0.125f, // Row 1, Column 2 + }, + kQuantizedTolerance))); + EXPECT_THAT(m.GetOutput(), ElementsAreArray({ + 128, 128, 128, // Row 1, Column 1 + 192, 192, 192, // Row 1, Column 2 + 128, 64, 192, // Row 2, Column 1 + 128, 112, 144, // Row 1, Column 2 + })); +} + +class LeakyReluOpModel : public SingleOpModel { + public: + LeakyReluOpModel(const TensorData& input, float alpha) { + input_ = AddInput(input); + output_ = AddOutput(input); + SetBuiltinOp(BuiltinOperator_LEAKY_RELU, BuiltinOptions_LeakyReluOptions, + CreateLeakyReluOptions(builder_, alpha).Union()); + BuildInterpreter({GetShape(input_)}); + } + void SetInput(std::initializer_list data) { + PopulateTensor(input_, data); + } + std::vector GetOutput() { return ExtractVector(output_); } + + protected: + int input_; + int output_; +}; + +TEST(FloatActivationsOpTest, LeakyRelu) { + LeakyReluOpModel m({TensorType_FLOAT32, {2, 3}}, 0.5f); + + m.SetInput({ + 0.0f, 1.0f, 3.0f, // Row 1 + 1.0f, -1.0f, -2.0f, // Row 2 + }); + m.Invoke(); + EXPECT_THAT(m.GetOutput(), ElementsAreArray({ + 0.0f, 1.0f, 3.0f, // Row 1 + 1.0f, -0.5f, -1.0f, // Row 2 + })); +} + } // namespace } // namespace tflite diff --git a/tensorflow/lite/kernels/conv.cc b/tensorflow/lite/kernels/conv.cc index 0c14b9eb656..1fd870be93e 100644 --- a/tensorflow/lite/kernels/conv.cc +++ b/tensorflow/lite/kernels/conv.cc @@ -24,7 +24,6 @@ limitations under the License. #include "tensorflow/lite/c/c_api_internal.h" #include "tensorflow/lite/kernels/eigen_support.h" #include "tensorflow/lite/kernels/gemm_support.h" -#include "tensorflow/lite/kernels/internal/optimized/cblas_conv.h" #include "tensorflow/lite/kernels/internal/optimized/multithreaded_conv.h" #include "tensorflow/lite/kernels/internal/optimized/optimized_ops.h" #include "tensorflow/lite/kernels/internal/quantization_util.h" @@ -491,11 +490,10 @@ void EvalFloat(TfLiteContext* context, TfLiteNode* node, CalculateActivationRange(params->activation, &output_activation_min, &output_activation_max); KernelType effective_kernel_type; - if ((kernel_type == kMultithreadOptimized || - kernel_type == kCblasOptimized) && + if ((kernel_type == kMultithreadOptimized) && (params->dilation_width_factor != 1 || params->dilation_height_factor != 1)) { - // kMultithreadOptimized and kCblasOptimized do not support dilation. + // kMultithreadOptimized does not support dilation. // Therefore, fallback to optimized. effective_kernel_type = kGenericOptimized; } else { @@ -521,6 +519,7 @@ void EvalFloat(TfLiteContext* context, TfLiteNode* node, GetTensorData(im2col)); break; } + case kCblasOptimized: case kGenericOptimized: { optimized_ops::Conv(op_params, GetTensorShape(input), GetTensorData(input), GetTensorShape(filter), @@ -546,15 +545,6 @@ void EvalFloat(TfLiteContext* context, TfLiteNode* node, GetTensorData(im2col)); break; } - case kCblasOptimized: { - cblas_ops::Conv(op_params, GetTensorShape(input), - GetTensorData(input), GetTensorShape(filter), - GetTensorData(filter), GetTensorShape(bias), - GetTensorData(bias), GetTensorShape(output), - GetTensorData(output), GetTensorShape(im2col), - GetTensorData(im2col)); - break; - } } } diff --git a/tensorflow/lite/kernels/eigen_support.cc b/tensorflow/lite/kernels/eigen_support.cc index 44e0086ad88..bad5975a7c1 100644 --- a/tensorflow/lite/kernels/eigen_support.cc +++ b/tensorflow/lite/kernels/eigen_support.cc @@ -34,6 +34,15 @@ static_assert( "kDefaultArenaAlignment doesn't comply with Eigen alignment requirement."); #endif // EIGEN_DONT_ALIGN +// Helper routine for updating the global Eigen thread count used for OpenMP. +void SetEigenNbThreads(int threads) { +#if defined(EIGEN_HAS_OPENMP) + // The global Eigen thread count is only used when OpenMP is enabled. As this + // call causes problems with tsan, make it only when OpenMP is available. + Eigen::setNbThreads(context->recommended_num_threads); +#endif // defined(EIGEN_HAS_OPENMP) +} + // We have a single global threadpool for all convolution operations. This means // that inferences started from different threads may block each other, but // since the underlying resource of CPU cores should be consumed by the @@ -78,7 +87,7 @@ void InitDevice(TfLiteContext* context, RefCountedEigenContext* ptr) { } TfLiteStatus Refresh(TfLiteContext* context) { - Eigen::setNbThreads(context->recommended_num_threads); + SetEigenNbThreads(context->recommended_num_threads); auto* ptr = GetEigenContext(context); if (ptr != nullptr) { @@ -94,7 +103,7 @@ void IncrementUsageCounter(TfLiteContext* context) { auto* ptr = GetEigenContext(context); if (ptr == nullptr) { if (context->recommended_num_threads != -1) { - Eigen::setNbThreads(context->recommended_num_threads); + SetEigenNbThreads(context->recommended_num_threads); } ptr = new RefCountedEigenContext; ptr->type = kTfLiteEigenContext; diff --git a/tensorflow/lite/kernels/elementwise.cc b/tensorflow/lite/kernels/elementwise.cc index 416a69eb0ed..a79388b900e 100644 --- a/tensorflow/lite/kernels/elementwise.cc +++ b/tensorflow/lite/kernels/elementwise.cc @@ -15,6 +15,7 @@ limitations under the License. #include #include "tensorflow/lite/c/c_api_internal.h" +#include "tensorflow/lite/kernels/internal/reference/reference_ops.h" #include "tensorflow/lite/kernels/internal/tensor.h" #include "tensorflow/lite/kernels/kernel_util.h" @@ -74,6 +75,10 @@ inline TfLiteStatus EvalLogical(TfLiteContext* context, TfLiteNode* node, return EvalImpl(context, node, bool_func, kTfLiteBool); } +TfLiteStatus AbsEval(TfLiteContext* context, TfLiteNode* node) { + return EvalNumeric(context, node, std::abs); +} + TfLiteStatus SinEval(TfLiteContext* context, TfLiteNode* node) { return EvalNumeric(context, node, std::sin); } @@ -101,6 +106,14 @@ TfLiteStatus LogicalNotEval(TfLiteContext* context, TfLiteNode* node) { } // namespace } // namespace elementwise +TfLiteRegistration* Register_ABS() { + static TfLiteRegistration r = { + /*init=*/nullptr, /*free=*/nullptr, + elementwise::GenericPrepare, + elementwise::AbsEval}; + return &r; +} + TfLiteRegistration* Register_SIN() { static TfLiteRegistration r = { /*init=*/nullptr, /*free=*/nullptr, diff --git a/tensorflow/lite/kernels/elementwise_test.cc b/tensorflow/lite/kernels/elementwise_test.cc index 52df8dc3cca..7d243200812 100644 --- a/tensorflow/lite/kernels/elementwise_test.cc +++ b/tensorflow/lite/kernels/elementwise_test.cc @@ -74,6 +74,19 @@ TEST(ElementWise, Log) { EXPECT_THAT(m.GetTensorShape(m.output()), ElementsAreArray({1, 1, 4, 1})); } +TEST(FloatActivationsOpTest, Abs) { + ElementWiseOpFloatModel m(BuiltinOperator_ABS, {1, 2, 4, 1}); + m.PopulateTensor(m.input(), { + 0.f, -6.2f, 2.f, 4.f, // + 3.f, -2.f, 10.f, 1.f, // + }); + m.Invoke(); + EXPECT_THAT(m.ExtractVector(m.output()), ElementsAreArray({ + 0.f, 6.2f, 2.f, 4.f, // + 3.f, 2.f, 10.f, 1.f, // + })); +} + TEST(ElementWise, Sqrt) { ElementWiseOpFloatModel m(BuiltinOperator_SQRT, {1, 1, 4, 1}); m.PopulateTensor(m.input(), {0, 1, 2, 4}); diff --git a/tensorflow/lite/kernels/fill.cc b/tensorflow/lite/kernels/fill.cc new file mode 100644 index 00000000000..079ee44f371 --- /dev/null +++ b/tensorflow/lite/kernels/fill.cc @@ -0,0 +1,141 @@ +/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/lite/c/builtin_op_data.h" +#include "tensorflow/lite/c/c_api_internal.h" +#include "tensorflow/lite/kernels/internal/reference/reference_ops.h" +#include "tensorflow/lite/kernels/internal/tensor.h" +#include "tensorflow/lite/kernels/internal/tensor_ctypes.h" +#include "tensorflow/lite/kernels/kernel_util.h" + +namespace tflite { +namespace ops { +namespace builtin { +namespace fill { + +namespace { + +constexpr int kDimsTensor = 0; +constexpr int kValueTensor = 1; +constexpr int kOutputTensor = 0; + +template +TfLiteStatus ResizeOutputImpl(TfLiteContext* context, const TfLiteTensor* dims, + TfLiteTensor* output) { + TfLiteIntArray* output_shape = TfLiteIntArrayCreate(dims->dims->data[0]); + for (int i = 0; i < output_shape->size; ++i) { + T data = GetTensorData(dims)[i]; + if (data < 0) { + context->ReportError(context, "Fill dimensions must be >= 0", dims->type); + return kTfLiteError; + } + output_shape->data[i] = data; + } + return context->ResizeTensor(context, output, output_shape); +} + +TfLiteStatus ResizeOutput(TfLiteContext* context, const TfLiteTensor* dims, + TfLiteTensor* output) { + switch (dims->type) { + case kTfLiteInt32: + return ResizeOutputImpl(context, dims, output); + case kTfLiteInt64: + return ResizeOutputImpl(context, dims, output); + default: + context->ReportError( + context, + "Fill only currently supports int32, int64 for input 0, " + "got %d.", + dims->type); + return kTfLiteError; + } +} + +} // namespace + +TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node) { + TF_LITE_ENSURE_EQ(context, NumInputs(node), 2); + TF_LITE_ENSURE_EQ(context, NumOutputs(node), 1); + + const TfLiteTensor* dims = GetInput(context, node, kDimsTensor); + const TfLiteTensor* value = GetInput(context, node, kValueTensor); + + // Make sure the 1st input tensor is 1-D. + TF_LITE_ENSURE_EQ(context, NumDimensions(dims), 1); + + // Make sure the 1st input tensor is int32 or int64. + const auto dtype = dims->type; + TF_LITE_ENSURE(context, dtype == kTfLiteInt32 || dtype == kTfLiteInt64); + + // Make sure the 2nd input tensor is a scalar. + TF_LITE_ENSURE_EQ(context, NumDimensions(value), 0); + + TfLiteTensor* output = GetOutput(context, node, kOutputTensor); + output->type = value->type; + + if (IsConstantTensor(dims)) { + TF_LITE_ENSURE_OK(context, ResizeOutput(context, dims, output)); + } else { + SetTensorToDynamic(output); + } + return kTfLiteOk; +} + +TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) { + const TfLiteTensor* value = GetInput(context, node, kValueTensor); + + TfLiteTensor* output = GetOutput(context, node, kOutputTensor); + + if (IsDynamicTensor(output)) { + const TfLiteTensor* dims = GetInput(context, node, kDimsTensor); + TF_LITE_ENSURE_OK(context, ResizeOutput(context, dims, output)); + } +#define TF_LITE_FILL(data_type) \ + reference_ops::Fill(GetTensorShape(value), GetTensorData(value), \ + GetTensorShape(output), \ + GetTensorData(output)) + switch (output->type) { + case kTfLiteInt32: + TF_LITE_FILL(int32_t); + break; + case kTfLiteInt64: + TF_LITE_FILL(int64_t); + break; + case kTfLiteFloat32: + TF_LITE_FILL(float); + break; + default: + context->ReportError( + context, + "Fill only currently supports int32, int64, float32 for input 1," + "got %d.", + value->type); + return kTfLiteError; + } +#undef TF_LITE_FILL + return kTfLiteOk; +} + +} // namespace fill + +TfLiteRegistration* Register_FILL() { + static TfLiteRegistration r = {/*init=*/nullptr, /*free=*/nullptr, + fill::Prepare, fill::Eval}; + return &r; +} + +} // namespace builtin +} // namespace ops +} // namespace tflite diff --git a/tensorflow/lite/kernels/fill_test.cc b/tensorflow/lite/kernels/fill_test.cc new file mode 100644 index 00000000000..08044d76f9d --- /dev/null +++ b/tensorflow/lite/kernels/fill_test.cc @@ -0,0 +1,94 @@ +/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#include +#include "tensorflow/lite/interpreter.h" +#include "tensorflow/lite/kernels/register.h" +#include "tensorflow/lite/kernels/test_util.h" +#include "tensorflow/lite/model.h" + +namespace tflite { +namespace { + +using ::testing::ElementsAreArray; +using ::testing::IsEmpty; + +class FillOpModel : public SingleOpModel { + public: + explicit FillOpModel(const TensorData& input1, const TensorData& input2) { + input1_ = AddInput(input1); + input2_ = AddInput(input2); + output_ = AddOutput(input1); + SetBuiltinOp(BuiltinOperator_FILL, BuiltinOptions_FillOptions, + CreateFillOptions(builder_).Union()); + BuildInterpreter({GetShape(input1_), GetShape(input2_)}); + } + + int input1() { return input1_; } + int input2() { return input2_; } + int output() { return output_; } + + protected: + int input1_; + int input2_; + int output_; +}; + +TEST(FillOpModel, FillInt32) { + FillOpModel m({TensorType_INT32, {2}}, {TensorType_INT32}); + m.PopulateTensor(m.input1(), {2, 3}); + m.PopulateTensor(m.input2(), {-11}); + m.Invoke(); + EXPECT_THAT(m.ExtractVector(m.output()), + ElementsAreArray({-11, -11, -11, -11, -11, -11})); + EXPECT_THAT(m.GetTensorShape(m.output()), ElementsAreArray({2, 3})); +} + +TEST(FillOpModel, FillInt64) { + FillOpModel m({TensorType_INT32, {2}}, {TensorType_INT64}); + m.PopulateTensor(m.input1(), {2, 4}); + m.PopulateTensor(m.input2(), {2 ^ 45}); + m.Invoke(); + EXPECT_THAT(m.ExtractVector(m.output()), + ElementsAreArray({2 ^ 45, 2 ^ 45, 2 ^ 45, 2 ^ 45, 2 ^ 45, 2 ^ 45, + 2 ^ 45, 2 ^ 45})); + EXPECT_THAT(m.GetTensorShape(m.output()), ElementsAreArray({2, 4})); +} + +TEST(FillOpModel, FillFloat) { + FillOpModel m({TensorType_INT64, {3}}, {TensorType_FLOAT32}); + m.PopulateTensor(m.input1(), {2, 2, 2}); + m.PopulateTensor(m.input2(), {4.0}); + m.Invoke(); + EXPECT_THAT(m.ExtractVector(m.output()), + ElementsAreArray({4.0, 4.0, 4.0, 4.0, 4.0, 4.0, 4.0, 4.0})); + EXPECT_THAT(m.GetTensorShape(m.output()), ElementsAreArray({2, 2, 2})); +} + +TEST(FillOpModel, FillOutputScalar) { + FillOpModel m({TensorType_INT64, {0}}, {TensorType_FLOAT32}); + m.PopulateTensor(m.input2(), {4.0}); + m.Invoke(); + EXPECT_THAT(m.ExtractVector(m.output()), ElementsAreArray({4.0})); + EXPECT_THAT(m.GetTensorShape(m.output()), IsEmpty()); +} + +} // namespace +} // namespace tflite + +int main(int argc, char** argv) { + ::tflite::LogToStderr(); + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/tensorflow/lite/kernels/fully_connected.cc b/tensorflow/lite/kernels/fully_connected.cc index 63cca1cf542..a1eecb284ab 100644 --- a/tensorflow/lite/kernels/fully_connected.cc +++ b/tensorflow/lite/kernels/fully_connected.cc @@ -117,7 +117,7 @@ TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node) { // Note that quantized inference requires that all tensors have their // parameters set. This is usually done during quantized training. TfLiteType data_type = input->type; - if (data_type != kTfLiteFloat32) { + if (data_type != kTfLiteFloat32 && data_type != kTfLiteInt32) { double real_multiplier = 0.0; TF_LITE_ENSURE_STATUS(GetQuantizedConvolutionMultipler( context, input, filter, bias, output, &real_multiplier)); diff --git a/tensorflow/lite/kernels/gather.cc b/tensorflow/lite/kernels/gather.cc index 61884d6a12c..f205daae134 100644 --- a/tensorflow/lite/kernels/gather.cc +++ b/tensorflow/lite/kernels/gather.cc @@ -118,7 +118,7 @@ TfLiteStatus GatherStrings(TfLiteContext* context, const TfLiteTensor* input, const auto string_ref = GetString(input, pos); buffer.AddString(string_ref.str, string_ref.len); } - buffer.WriteToTensor(output); + buffer.WriteToTensorAsVector(output); return kTfLiteOk; } diff --git a/tensorflow/lite/kernels/hashtable_lookup.cc b/tensorflow/lite/kernels/hashtable_lookup.cc index b6ae7a3d1a5..da1116cf858 100644 --- a/tensorflow/lite/kernels/hashtable_lookup.cc +++ b/tensorflow/lite/kernels/hashtable_lookup.cc @@ -137,7 +137,7 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) { } } if (output->type == kTfLiteString) { - buf.WriteToTensor(output); + buf.WriteToTensorAsVector(output); } return kTfLiteOk; diff --git a/tensorflow/lite/kernels/internal/BUILD b/tensorflow/lite/kernels/internal/BUILD index 6d9690ea460..7d2653f0a1d 100644 --- a/tensorflow/lite/kernels/internal/BUILD +++ b/tensorflow/lite/kernels/internal/BUILD @@ -234,8 +234,6 @@ cc_library( cc_library( name = "optimized", hdrs = [ - "optimized/cblas_conv.h", - "optimized/cblas_reference.h", "optimized/eigen_spatial_convolutions.h", "optimized/eigen_tensor_reduced_instantiations_oss.h", "optimized/multithreaded_conv.h", diff --git a/tensorflow/lite/kernels/internal/optimized/cblas_conv.h b/tensorflow/lite/kernels/internal/optimized/cblas_conv.h deleted file mode 100644 index 53772050503..00000000000 --- a/tensorflow/lite/kernels/internal/optimized/cblas_conv.h +++ /dev/null @@ -1,109 +0,0 @@ -/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -#ifndef TENSORFLOW_LITE_KERNELS_INTERNAL_OPTIMIZED_CBLAS_CONV_H_ -#define TENSORFLOW_LITE_KERNELS_INTERNAL_OPTIMIZED_CBLAS_CONV_H_ - -// The Conv implementation based on CBLAS interface. This is only used on iOS -// for now, utilizing Apple's Accelerate framework. - -#if TFLITE_USE_APPLE_ACCELERATE_FOR_CONV -#include -#else -#include "tensorflow/lite/kernels/internal/optimized/cblas_reference.h" -#endif - -#include "tensorflow/lite/kernels/internal/optimized/multithreaded_conv.h" -#include "tensorflow/lite/kernels/internal/optimized/optimized_ops.h" - -namespace tflite { -namespace cblas_ops { - -inline void Conv(const ConvParams& params, const RuntimeShape& input_shape, - const float* input_data, const RuntimeShape& filter_shape, - const float* filter_data, const RuntimeShape& bias_shape, - const float* bias_data, const RuntimeShape& output_shape, - float* output_data, const RuntimeShape& im2col_shape, - float* im2col_data) { - const int stride_width = params.stride_width; - const int stride_height = params.stride_height; - const int pad_width = params.padding_values.width; - const int pad_height = params.padding_values.height; - const int dilation_width_factor = params.dilation_width_factor; - const int dilation_height_factor = params.dilation_height_factor; - const float output_activation_min = params.float_activation_min; - const float output_activation_max = params.float_activation_max; - TFLITE_DCHECK_EQ(input_shape.DimensionsCount(), 4); - TFLITE_DCHECK_EQ(filter_shape.DimensionsCount(), 4); - TFLITE_DCHECK_EQ(output_shape.DimensionsCount(), 4); - gemmlowp::ScopedProfilingLabel label("Conv/cblas"); - - const float* gemm_input_data = nullptr; - const RuntimeShape* gemm_input_shape = nullptr; - const int filter_width = filter_shape.Dims(2); - const int filter_height = filter_shape.Dims(1); - const bool need_im2col = stride_width != 1 || stride_height != 1 || - filter_width != 1 || filter_height != 1; - if (need_im2col) { - TFLITE_DCHECK(im2col_data); - ConvParams op_params; - op_params.padding_type = PaddingType::kSame; - op_params.padding_values.width = pad_width; - op_params.padding_values.height = pad_height; - op_params.stride_width = stride_width; - op_params.stride_height = stride_height; - op_params.dilation_width_factor = dilation_width_factor; - op_params.dilation_height_factor = dilation_height_factor; - optimized_ops::Im2col(op_params, filter_height, filter_width, 0, - input_shape, input_data, im2col_shape, im2col_data); - - gemm_input_data = im2col_data; - gemm_input_shape = &im2col_shape; - } else { - TFLITE_DCHECK(!im2col_data); - gemm_input_data = input_data; - gemm_input_shape = &input_shape; - } - - // The following code computes matrix multiplication c = a * transponse(b) - // with CBLAS, where: - // * `a` is a matrix with dimensions (m, k). - // * `b` is a matrix with dimensions (n, k), so transpose(b) is (k, n). - // * `c` is a matrix with dimensions (m, n). - // The naming of variables are aligned with CBLAS specification here. - const float* a = gemm_input_data; - const float* b = filter_data; - float* c = output_data; - const int gemm_input_dims = gemm_input_shape->DimensionsCount(); - int m = FlatSizeSkipDim(*gemm_input_shape, gemm_input_dims - 1); - int n = output_shape.Dims(3); - int k = gemm_input_shape->Dims(gemm_input_dims - 1); - // The stride of matrix a, b and c respectively. - int stride_a = k; - int stride_b = k; - int stride_c = n; - - cblas_sgemm(CblasRowMajor, CblasNoTrans, CblasTrans, m, n, k, 1.0f, a, - stride_a, b, stride_b, 0.0f, c, stride_c); - - optimized_ops::AddBiasAndEvalActivationFunction( - output_activation_min, output_activation_max, bias_shape, bias_data, - output_shape, output_data); -} - -} // namespace cblas_ops -} // namespace tflite - -#endif // TENSORFLOW_LITE_KERNELS_INTERNAL_OPTIMIZED_CBLAS_CONV_H_ diff --git a/tensorflow/lite/kernels/internal/optimized/cblas_reference.h b/tensorflow/lite/kernels/internal/optimized/cblas_reference.h deleted file mode 100644 index fa07578612a..00000000000 --- a/tensorflow/lite/kernels/internal/optimized/cblas_reference.h +++ /dev/null @@ -1,69 +0,0 @@ -/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -#ifndef TENSORFLOW_LITE_KERNELS_INTERNAL_OPTIMIZED_CBLAS_REFERENCE_H_ -#define TENSORFLOW_LITE_KERNELS_INTERNAL_OPTIMIZED_CBLAS_REFERENCE_H_ - -#include "tensorflow/lite/kernels/internal/compatibility.h" - -// The reference implementation for a small subset of CBLAS interface. -// This is only used for testing CBLAS implementation, and should never be used -// in production code. - -namespace tflite { -namespace cblas_ops { - -// The following code follows the original CBLAS specification, and it might -// conflict with the TensorFlow naming convention. -// TODO(ycling): Find another way to test CBLAS with bazel, without writing -// a reference implementation by ourselves. -enum CBLAS_ORDER { CblasRowMajor = 0, CblasColMajor = 1 }; - -enum CBLAS_TRANSPOSE { CblasNoTrans = 0, CblasTrans = 1, CblasConjTrans = 2 }; - -// A reference implementation for matrix multiplication. -// The following code computes, c = a * transponse(b) matrix multiplication -// with CBLAS, where: -// * `a` is a matrix with dimensions (m, k). -// * `b` is a matrix with dimensions (n, k), so transpose(b) is (k, n). -// * `c` is a matrix with dimensions (m, n). -// The naming of variables is aligned with CBLAS specification here. -void cblas_sgemm(const enum CBLAS_ORDER order, - const enum CBLAS_TRANSPOSE trans_a, - const enum CBLAS_TRANSPOSE trans_b, const int m, const int n, - const int k, const float alpha, const float *a, - const int stride_a, const float *b, const int stride_b, - const float beta, float *c, const int stride_c) { - TFLITE_DCHECK(order == CblasRowMajor); - TFLITE_DCHECK(trans_a == CblasNoTrans); - TFLITE_DCHECK(trans_b == CblasTrans); - TFLITE_DCHECK(beta == 0.0f); - for (int row = 0; row < m; ++row) { - for (int col = 0; col < n; ++col) { - // If `beta` non-zero, multiple it with the original values in output. - // Otherwise, ignore the original value in output completely. - float value = 0.0f; - for (int idx = 0; idx < k; ++idx) { - value += alpha * a[stride_a * row + idx] * b[stride_b * col + idx]; - } - c[stride_c * row + col] = value; - } - } -} - -} // namespace cblas_ops -} // namespace tflite - -#endif // TENSORFLOW_LITE_KERNELS_INTERNAL_OPTIMIZED_CBLAS_REFERENCE_H_ diff --git a/tensorflow/lite/kernels/internal/optimized/depthwiseconv_float.h b/tensorflow/lite/kernels/internal/optimized/depthwiseconv_float.h index 25b66d4b553..c77715de579 100644 --- a/tensorflow/lite/kernels/internal/optimized/depthwiseconv_float.h +++ b/tensorflow/lite/kernels/internal/optimized/depthwiseconv_float.h @@ -793,22 +793,26 @@ void FloatDepthwiseConvAccumRow(int stride, int dilation_factor, int out_x_loop_end_unclampled = 0; if (kAllowStrided) { if (stride == 2) { - out_x_loop_start_unclampled = (pad_width - filter_x + 1) / 2; + out_x_loop_start_unclampled = + (pad_width - dilation_factor * filter_x + 1) / 2; out_x_loop_end_unclampled = - (pad_width + input_width - filter_x + 1) / 2; + (pad_width + input_width - dilation_factor * filter_x + 1) / 2; } else if (stride == 4) { - out_x_loop_start_unclampled = (pad_width - filter_x + 3) / 4; + out_x_loop_start_unclampled = + (pad_width - dilation_factor * filter_x + 3) / 4; out_x_loop_end_unclampled = - (pad_width + input_width - filter_x + 3) / 4; + (pad_width + input_width - dilation_factor * filter_x + 3) / 4; } else { out_x_loop_start_unclampled = - (pad_width - filter_x + stride - 1) / stride; - out_x_loop_end_unclampled = - (pad_width + input_width - filter_x + stride - 1) / stride; + (pad_width - dilation_factor * filter_x + stride - 1) / stride; + out_x_loop_end_unclampled = (pad_width + input_width - + dilation_factor * filter_x + stride - 1) / + stride; } } else { - out_x_loop_start_unclampled = pad_width - filter_x; - out_x_loop_end_unclampled = pad_width + input_width - filter_x; + out_x_loop_start_unclampled = pad_width - dilation_factor * filter_x; + out_x_loop_end_unclampled = + pad_width + input_width - dilation_factor * filter_x; } // The kernel will have to iterate on the segment of the // output row that starts at out_x_loop_start and out_x_loop_end. @@ -819,7 +823,8 @@ void FloatDepthwiseConvAccumRow(int stride, int dilation_factor, float* acc_buffer_ptr = acc_buffer + (out_x_loop_start - out_x_buffer_start) * output_depth; - const int in_x_origin = (out_x_loop_start * stride) - pad_width + filter_x; + const int in_x_origin = + (out_x_loop_start * stride) - pad_width + dilation_factor * filter_x; const float* input_ptr = input_data + in_x_origin * input_depth; const int num_output_pixels = out_x_loop_end - out_x_loop_start; FloatDepthwiseConvKernel; \ diff --git a/tensorflow/lite/kernels/internal/optimized/depthwiseconv_uint8.h b/tensorflow/lite/kernels/internal/optimized/depthwiseconv_uint8.h index 5317cea8843..d3dca799a7c 100644 --- a/tensorflow/lite/kernels/internal/optimized/depthwiseconv_uint8.h +++ b/tensorflow/lite/kernels/internal/optimized/depthwiseconv_uint8.h @@ -1499,22 +1499,26 @@ void QuantizedDepthwiseConvAccumRow(int stride, int dilation_factor, int out_x_loop_end_unclampled = 0; if (kAllowStrided) { if (stride == 2) { - out_x_loop_start_unclampled = (pad_width - filter_x + 1) / 2; + out_x_loop_start_unclampled = + (pad_width - dilation_factor * filter_x + 1) / 2; out_x_loop_end_unclampled = - (pad_width + input_width - filter_x + 1) / 2; + (pad_width + input_width - dilation_factor * filter_x + 1) / 2; } else if (stride == 4) { - out_x_loop_start_unclampled = (pad_width - filter_x + 3) / 4; + out_x_loop_start_unclampled = + (pad_width - dilation_factor * filter_x + 3) / 4; out_x_loop_end_unclampled = - (pad_width + input_width - filter_x + 3) / 4; + (pad_width + input_width - dilation_factor * filter_x + 3) / 4; } else { out_x_loop_start_unclampled = - (pad_width - filter_x + stride - 1) / stride; - out_x_loop_end_unclampled = - (pad_width + input_width - filter_x + stride - 1) / stride; + (pad_width - dilation_factor * filter_x + stride - 1) / stride; + out_x_loop_end_unclampled = (pad_width + input_width - + dilation_factor * filter_x + stride - 1) / + stride; } } else { - out_x_loop_start_unclampled = pad_width - filter_x; - out_x_loop_end_unclampled = pad_width + input_width - filter_x; + out_x_loop_start_unclampled = pad_width - dilation_factor * filter_x; + out_x_loop_end_unclampled = + pad_width + input_width - dilation_factor * filter_x; } // The kernel will have to iterate on the segment of the // output row that starts at out_x_loop_start and out_x_loop_end. @@ -1525,7 +1529,8 @@ void QuantizedDepthwiseConvAccumRow(int stride, int dilation_factor, int32* acc_buffer_ptr = acc_buffer + (out_x_loop_start - out_x_buffer_start) * output_depth; - const int in_x_origin = (out_x_loop_start * stride) - pad_width + filter_x; + const int in_x_origin = + (out_x_loop_start * stride) - pad_width + dilation_factor * filter_x; const uint8* input_ptr = input_data + in_x_origin * input_depth; const int num_output_pixels = out_x_loop_end - out_x_loop_start; QuantizedDepthwiseConvKernel< @@ -1703,8 +1708,7 @@ inline void DepthwiseConvGeneral( FIXED_DEPTH_MULTIPLIER) \ if (!row_accum_func && (stride_width == 1 || ALLOW_STRIDED) && \ (input_depth == FIXED_INPUT_DEPTH || FIXED_INPUT_DEPTH == 0) && \ - depth_multiplier == FIXED_DEPTH_MULTIPLIER && \ - dilation_width_factor == 1 && dilation_height_factor == 1) { \ + depth_multiplier == FIXED_DEPTH_MULTIPLIER) { \ row_accum_func = \ QuantizedDepthwiseConvAccumRow; \ diff --git a/tensorflow/lite/kernels/internal/optimized/depthwiseconv_uint8_3x3_filter.h b/tensorflow/lite/kernels/internal/optimized/depthwiseconv_uint8_3x3_filter.h index 3f2ed0b1f0e..5859bcaed4a 100644 --- a/tensorflow/lite/kernels/internal/optimized/depthwiseconv_uint8_3x3_filter.h +++ b/tensorflow/lite/kernels/internal/optimized/depthwiseconv_uint8_3x3_filter.h @@ -23,11 +23,6 @@ limitations under the License. namespace tflite { namespace optimized_ops { -// clang-format gets confused with this file and ends up formatting lines to -// be larger than 80 characters. Turn off here and back on at the end of the -// file. -// clang-format off - // See CategorizeDotProductKernel for definitive taxonomy. enum class DotProduct3x3KernelType { kNone = 0, // Parameter combination is not supported for dot product kernels. @@ -120,42 +115,58 @@ struct DepthwiseConvParams { #define OFFSET_OUTPUT_WIDTH 84 #define OFFSET_OUTPUT_HEIGHT 88 -static_assert(offsetof(DepthwiseConvParams, input_depth) == - OFFSET_INPUT_DEPTH, ""); +static_assert(offsetof(DepthwiseConvParams, input_depth) == OFFSET_INPUT_DEPTH, + ""); static_assert(offsetof(DepthwiseConvParams, input_row_size) == - OFFSET_INPUT_ROW_SIZE, ""); + OFFSET_INPUT_ROW_SIZE, + ""); static_assert(offsetof(DepthwiseConvParams, output_depth) == - OFFSET_OUTPUT_DEPTH, ""); + OFFSET_OUTPUT_DEPTH, + ""); static_assert(offsetof(DepthwiseConvParams, output_row_size) == - OFFSET_OUTPUT_ROW_SIZE, ""); + OFFSET_OUTPUT_ROW_SIZE, + ""); static_assert(offsetof(DepthwiseConvParams, filter_row_size) == - OFFSET_FILTER_ROW_SIZE, ""); + OFFSET_FILTER_ROW_SIZE, + ""); static_assert(offsetof(DepthwiseConvParams, input_offset) == - OFFSET_INPUT_OFFSET, ""); + OFFSET_INPUT_OFFSET, + ""); static_assert(offsetof(DepthwiseConvParams, output_offset) == - OFFSET_OUTPUT_OFFSET, ""); + OFFSET_OUTPUT_OFFSET, + ""); static_assert(offsetof(DepthwiseConvParams, filter_offset) == - OFFSET_FILTER_OFFSET, ""); + OFFSET_FILTER_OFFSET, + ""); static_assert(offsetof(DepthwiseConvParams, output_multiplier) == - OFFSET_OUTPUT_MULTIPLIER, ""); + OFFSET_OUTPUT_MULTIPLIER, + ""); static_assert(offsetof(DepthwiseConvParams, output_activation_min) == - OFFSET_OUTPUT_ACTIVATION_MIN, ""); + OFFSET_OUTPUT_ACTIVATION_MIN, + ""); static_assert(offsetof(DepthwiseConvParams, output_activation_max) == - OFFSET_OUTPUT_ACTIVATION_MAX, ""); + OFFSET_OUTPUT_ACTIVATION_MAX, + ""); static_assert(offsetof(DepthwiseConvParams, output_right_shift) == - OFFSET_OUTPUT_RIGHT_SHIFT, ""); -static_assert(offsetof(DepthwiseConvParams, input_width) == - OFFSET_INPUT_WIDTH, ""); + OFFSET_OUTPUT_RIGHT_SHIFT, + ""); +static_assert(offsetof(DepthwiseConvParams, input_width) == OFFSET_INPUT_WIDTH, + ""); static_assert(offsetof(DepthwiseConvParams, input_height) == - OFFSET_INPUT_HEIGHT, ""); + OFFSET_INPUT_HEIGHT, + ""); static_assert(offsetof(DepthwiseConvParams, stride_width) == - OFFSET_STRIDE_WIDTH, ""); + OFFSET_STRIDE_WIDTH, + ""); static_assert(offsetof(DepthwiseConvParams, stride_height) == - OFFSET_STRIDE_HEIGHT, ""); + OFFSET_STRIDE_HEIGHT, + ""); static_assert(offsetof(DepthwiseConvParams, output_width) == - OFFSET_OUTPUT_WIDTH, ""); + OFFSET_OUTPUT_WIDTH, + ""); static_assert(offsetof(DepthwiseConvParams, output_height) == - OFFSET_OUTPUT_HEIGHT, ""); + OFFSET_OUTPUT_HEIGHT, + ""); template struct DepthwiseConvWindow {}; @@ -164,10 +175,10 @@ template <> struct DepthwiseConvWindow<8, 1, 1> { public: static inline void Run(const uint8* input_ptr, const uint8* filter_ptr, - const int32* bias_ptr, uint8* output_ptr, int64_t input_depth, - int64_t input_row_size, int32 output_window_height, - int32 output_window_width, - const DepthwiseConvParams* params_ptr) { + const int32* bias_ptr, uint8* output_ptr, + int64_t input_depth, int64_t input_row_size, + int32 output_window_height, int32 output_window_width, + const DepthwiseConvParams* params_ptr) { const int64_t input_width_increment = 2 * input_depth; const int64_t input_height_increment = 2 * input_row_size; const int64_t output_height_increment = 2 * params_ptr->output_row_size; @@ -1147,10 +1158,10 @@ struct DepthwiseConvWindow<8, 1, 1> { template <> struct DepthwiseConvWindow<8, 2, 2> { static inline void Run(const uint8* input_ptr, const uint8* filter_ptr, - const int32* bias_ptr, uint8* output_ptr, int64_t input_depth, - int64_t input_row_size, int32 output_window_height, - int32 output_window_width, - const DepthwiseConvParams* params_ptr) { + const int32* bias_ptr, uint8* output_ptr, + int64_t input_depth, int64_t input_row_size, + int32 output_window_height, int32 output_window_width, + const DepthwiseConvParams* params_ptr) { const int64_t input_width_increment = 4 * input_depth; const int64_t input_height_increment = 4 * input_row_size; const int64_t output_height_increment = 2 * params_ptr->output_row_size; @@ -2990,11 +3001,10 @@ struct ShuffleParams { ShuffleParams() = default; ShuffleParams(int32 output_width, int32 output_height, int32 stride_width, int32 stride_height) - : output_width(output_width) - , output_height(output_height) - , input_width(get_shuffle_input_size(stride_width, output_width)) - , input_height(get_shuffle_input_size(stride_height, output_height)) { - } + : output_width(output_width), + output_height(output_height), + input_width(get_shuffle_input_size(stride_width, output_width)), + input_height(get_shuffle_input_size(stride_height, output_height)) {} }; template @@ -3003,10 +3013,10 @@ struct DepthwiseConvThroughDepth { // |start_depth| to |end_depth|. Keep this not inlined to maintain a small // binary size. We use a DepthwiseConvParams struct for read only params // to minimize call overhead. - static __attribute__((noinline)) void Run(const uint8* input_ptr, - const uint8* filter_ptr, const int32* bias_ptr, uint8* output_ptr, - int64_t start_depth, int64_t end_depth, int64_t input_depth, - int64_t input_row_size, int32 output_window_height, + static __attribute__((noinline)) void Run( + const uint8* input_ptr, const uint8* filter_ptr, const int32* bias_ptr, + uint8* output_ptr, int64_t start_depth, int64_t end_depth, + int64_t input_depth, int64_t input_row_size, int32 output_window_height, int32 output_window_width, const DepthwiseConvParams& params) { for (; start_depth <= end_depth - 8; start_depth += 8) { DepthwiseConvWindow<8, kStrideWidth, kStrideHeight>::Run( @@ -3029,12 +3039,15 @@ struct DepthwiseConvMultiRow { uint8* output_data, const DepthwiseConvParams& params, const ShuffleParams& shuffle_params, uint8* shuffle_workspace) { - TFLITE_DCHECK(shuffle_params.input_height == + TFLITE_DCHECK( + shuffle_params.input_height == get_shuffle_input_size(kStrideHeight, shuffle_params.output_height)); - TFLITE_DCHECK(shuffle_params.input_width == + TFLITE_DCHECK( + shuffle_params.input_width == get_shuffle_input_size(kStrideWidth, shuffle_params.output_width)); - TFLITE_DCHECK(64 * shuffle_params.input_width * shuffle_params.input_height - <= DEPTHWISECONV_SHUFFLE_WORKSPACE_SIZE); + TFLITE_DCHECK(64 * shuffle_params.input_width * + shuffle_params.input_height <= + DEPTHWISECONV_SHUFFLE_WORKSPACE_SIZE); int32 out_x = start_x; @@ -3045,7 +3058,7 @@ struct DepthwiseConvMultiRow { if (params.output_depth > 64 || (params.output_depth <= 64 && params.input_width > 150)) { for (; out_x <= (end_x - shuffle_params.output_width); - out_x += shuffle_params.output_width) { + out_x += shuffle_params.output_width) { const uint8* input_ptr = input_data; const int32* bias_ptr = bias_data; const uint8* filter_ptr = filter_data; @@ -3091,8 +3104,8 @@ struct DepthwiseConvMultiRow { } // Handle leftover depth. - ConvKernel::Run(input_ptr, filter_ptr, bias_ptr, output_ptr, - depth, params.output_depth, params.input_depth, + ConvKernel::Run(input_ptr, filter_ptr, bias_ptr, output_ptr, depth, + params.output_depth, params.input_depth, params.input_row_size, shuffle_params.output_height, shuffle_params.output_width, params); @@ -3119,13 +3132,15 @@ struct DepthwiseConvMultiRow { // * Horizontal edges. // * Vertical edges. inline void DepthwiseConvHandlePadding(const uint8* input_data, - const uint8* filter_data, const int32* bias_data, uint8* output_data, - const DepthwiseConvParams& params) { + const uint8* filter_data, + const int32* bias_data, + uint8* output_data, + const DepthwiseConvParams& params) { if (params.input_width == 1 && params.input_height == 1) { - const uint8* filter_ptr = filter_data + params.filter_row_size - + params.output_depth; - DepthwiseConvPartial::Run(input_data, filter_ptr, - bias_data, output_data, ¶ms); + const uint8* filter_ptr = + filter_data + params.filter_row_size + params.output_depth; + DepthwiseConvPartial::Run( + input_data, filter_ptr, bias_data, output_data, ¶ms); return; } @@ -3136,27 +3151,27 @@ inline void DepthwiseConvHandlePadding(const uint8* input_data, // Handle top row. const uint8* input_ptr = input_data; - const uint8* filter_ptr = filter_data + params.filter_row_size - + params.output_depth; + const uint8* filter_ptr = + filter_data + params.filter_row_size + params.output_depth; uint8* output_ptr = output_data; - DepthwiseConvPartial::Run(input_ptr, filter_ptr, - bias_data, output_ptr, ¶ms); + DepthwiseConvPartial::Run( + input_ptr, filter_ptr, bias_data, output_ptr, ¶ms); input_ptr += (params.stride_width - 1) * params.input_depth; filter_ptr = filter_data + params.filter_row_size; output_ptr += params.output_depth; for (int32 out_x = out_x_start_corner + 1; out_x < out_x_end_corner; - out_x++) { + out_x++) { DepthwiseConvPartial::Run( input_ptr, filter_ptr, bias_data, output_ptr, ¶ms); input_ptr += params.stride_width * params.input_depth; output_ptr += params.output_depth; } - DepthwiseConvPartial::Run(input_ptr, filter_ptr, - bias_data, output_ptr, ¶ms); + DepthwiseConvPartial::Run( + input_ptr, filter_ptr, bias_data, output_ptr, ¶ms); // Handle left side. input_ptr = input_data + (params.stride_width - 1) * params.input_row_size; @@ -3164,7 +3179,7 @@ inline void DepthwiseConvHandlePadding(const uint8* input_data, output_ptr = output_data + params.output_row_size; for (int32 out_y = out_y_start_corner + 1; out_y < out_y_end_corner; - out_y++) { + out_y++) { DepthwiseConvPartial::Run( input_ptr, filter_ptr, bias_data, output_ptr, ¶ms); input_ptr += params.stride_width * params.input_row_size; @@ -3172,14 +3187,14 @@ inline void DepthwiseConvHandlePadding(const uint8* input_data, } // Handle right side. - input_ptr = input_data + (params.input_width - 2) * params.input_depth - + (params.stride_width - 1) * params.input_row_size; + input_ptr = input_data + (params.input_width - 2) * params.input_depth + + (params.stride_width - 1) * params.input_row_size; filter_ptr = filter_data; output_ptr = output_data + params.output_row_size + - (params.output_width - 1) * params.output_depth; + (params.output_width - 1) * params.output_depth; for (int32 out_y = out_y_start_corner + 1; out_y < out_y_end_corner; - out_y++) { + out_y++) { DepthwiseConvPartial::Run( input_ptr, filter_ptr, bias_data, output_ptr, ¶ms); input_ptr += params.stride_width * params.input_row_size; @@ -3189,26 +3204,26 @@ inline void DepthwiseConvHandlePadding(const uint8* input_data, // Handle bottom row. input_ptr = input_data + (params.input_height - 2) * params.input_row_size; filter_ptr = filter_data + params.output_depth; - output_ptr = output_data + - (params.output_height - 1) * params.output_row_size; + output_ptr = + output_data + (params.output_height - 1) * params.output_row_size; - DepthwiseConvPartial::Run(input_ptr, filter_ptr, - bias_data, output_ptr, ¶ms); + DepthwiseConvPartial::Run( + input_ptr, filter_ptr, bias_data, output_ptr, ¶ms); input_ptr += (params.stride_width == 1) ? 0 : params.input_depth; filter_ptr = filter_data; output_ptr += params.output_depth; for (int32 out_x = out_x_start_corner + 1; out_x < out_x_end_corner; - out_x++) { + out_x++) { DepthwiseConvPartial::Run( input_ptr, filter_ptr, bias_data, output_ptr, ¶ms); input_ptr += params.stride_width * params.input_depth; output_ptr += params.output_depth; } - DepthwiseConvPartial::Run(input_ptr, filter_ptr, - bias_data, output_ptr, ¶ms); + DepthwiseConvPartial::Run( + input_ptr, filter_ptr, bias_data, output_ptr, ¶ms); } inline bool Fast3x3FilterKernelSupported( @@ -3383,8 +3398,8 @@ inline void DepthwiseConv3x3Filter( const int in_x = (out_x * stride_width) - pad_width; const int in_y = (out_y * stride_height) - pad_height; input_ptr += in_y * params.input_row_size + in_x * params.input_depth; - output_ptr += out_y * params.output_row_size - + out_x * params.output_depth; + output_ptr += + out_y * params.output_row_size + out_x * params.output_depth; } // Shuffling shapes that maximize width over the shuffle workspace size @@ -3439,7 +3454,6 @@ inline void DepthwiseConv3x3Filter( } } } -// clang-format on #endif // __aarch64__ diff --git a/tensorflow/lite/kernels/internal/optimized/optimized_ops.h b/tensorflow/lite/kernels/internal/optimized/optimized_ops.h index 6f2cd4faab2..c7691e27632 100644 --- a/tensorflow/lite/kernels/internal/optimized/optimized_ops.h +++ b/tensorflow/lite/kernels/internal/optimized/optimized_ops.h @@ -25,6 +25,10 @@ limitations under the License. #include #include +#if defined(TF_LITE_USE_CBLAS) && defined(__APPLE__) +#include +#endif + #include "third_party/eigen3/Eigen/Core" #include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor" #include "fixedpoint/fixedpoint.h" @@ -60,11 +64,13 @@ using reference_ops::DepthConcatenation; using reference_ops::Dequantize; using reference_ops::Div; using reference_ops::FakeQuant; +using reference_ops::Fill; using reference_ops::Gather; using reference_ops::Greater; using reference_ops::GreaterEqual; using reference_ops::GreaterEqualWithScaling; using reference_ops::GreaterWithScaling; +using reference_ops::LeakyRelu; using reference_ops::Less; using reference_ops::LessEqual; using reference_ops::LessEqualWithScaling; @@ -1867,18 +1873,45 @@ inline void Conv(const ConvParams& params, const RuntimeShape& input_shape, gemm_input_shape = &input_shape; } - const auto im2col_matrix_map = - MapAsMatrixWithLastDimAsRows(gemm_input_data, *gemm_input_shape); - const auto filter_matrix_map = - MapAsMatrixWithFirstDimAsCols(filter_data, filter_shape); - auto output_matrix_map = - MapAsMatrixWithLastDimAsRows(output_data, output_shape); + // The following code computes matrix multiplication c = a * transponse(b) + // with CBLAS, where: + // * `a` is a matrix with dimensions (m, k). + // * `b` is a matrix with dimensions (n, k), so transpose(b) is (k, n). + // * `c` is a matrix with dimensions (m, n). + // The naming of variables are aligned with CBLAS specification here. + const float* a = gemm_input_data; + const float* b = filter_data; + float* c = output_data; + const int gemm_input_dims = gemm_input_shape->DimensionsCount(); + int m = FlatSizeSkipDim(*gemm_input_shape, gemm_input_dims - 1); + int n = output_shape.Dims(3); + int k = gemm_input_shape->Dims(gemm_input_dims - 1); - Gemm(filter_matrix_map.transpose(), im2col_matrix_map, &output_matrix_map); +#if defined(TF_LITE_USE_CBLAS) && defined(__APPLE__) + // The stride of matrix a, b and c respectively. + int stride_a = k; + int stride_b = k; + int stride_c = n; - AddBiasAndEvalActivationFunction(output_activation_min, output_activation_max, - bias_shape, bias_data, output_shape, - output_data); + cblas_sgemm(CblasRowMajor, CblasNoTrans, CblasTrans, m, n, k, 1.0f, a, + stride_a, b, stride_b, 0.0f, c, stride_c); +#else + // When an optimized CBLAS implementation is not available, fall back + // to using Eigen. + typedef Eigen::Matrix + Matrix; + typedef Eigen::Map MatrixRef; + typedef Eigen::Map ConstMatrixRef; + + MatrixRef matrix_c(c, m, n); + ConstMatrixRef matrix_a(a, m, k); + ConstMatrixRef matrix_b(b, n, k); + matrix_c.noalias() = matrix_a * matrix_b.transpose(); +#endif // defined(TF_LITE_USE_CBLAS) && defined(__APPLE__) + + optimized_ops::AddBiasAndEvalActivationFunction( + output_activation_min, output_activation_max, bias_shape, bias_data, + output_shape, output_data); } inline void HybridConv(const ConvParams& params, float* scaling_factors_ptr, @@ -4292,7 +4325,6 @@ inline void LogSoftmax(const SoftmaxParams& params, using FixedPointScaledDiff = gemmlowp::FixedPoint; using FixedPointAccum = gemmlowp::FixedPoint; - using FixedPoint0 = gemmlowp::FixedPoint; const int trailing_dim = input_shape.DimensionsCount() - 1; const int outer_size = diff --git a/tensorflow/lite/kernels/internal/reference/reference_ops.h b/tensorflow/lite/kernels/internal/reference/reference_ops.h index 1bd9129488a..ea3ab06da1f 100644 --- a/tensorflow/lite/kernels/internal/reference/reference_ops.h +++ b/tensorflow/lite/kernels/internal/reference/reference_ops.h @@ -558,6 +558,19 @@ inline void ReluX(const tflite::ActivationParams& params, } } +inline void LeakyRelu(const tflite::LeakyReluParams& params, + const RuntimeShape& input_shape, const float* input_data, + const RuntimeShape& output_shape, float* output_data) { + gemmlowp::ScopedProfilingLabel label("LeakyRelu (not fused)"); + const int flat_size = MatchingFlatSize(input_shape, output_shape); + for (int i = 0; i < flat_size; ++i) { + const float val = input_data[i]; + // Note that this implementation matches that of TensorFlow, and corresponds + // to the traditional LeakyRelu equation only for alpha <= 1. + output_data[i] = std::max(val, val * params.alpha); + } +} + inline void L2Normalization(const tflite::L2NormalizationParams& op_params, const RuntimeShape& input_shape, const float* input_data, @@ -2723,7 +2736,6 @@ inline void LogSoftmax(const SoftmaxParams& params, using FixedPointScaledDiff = gemmlowp::FixedPoint; using FixedPointAccum = gemmlowp::FixedPoint; - using FixedPoint0 = gemmlowp::FixedPoint; const int trailing_dim = input_shape.DimensionsCount() - 1; const int outer_size = @@ -3651,8 +3663,10 @@ inline void Mean(const tflite::MeanParams& op_params, const RuntimeShape& unextended_output_shape, T* output_data) { gemmlowp::ScopedProfilingLabel label("Mean"); - TFLITE_DCHECK_LE(unextended_input_shape.DimensionsCount(), 4); - TFLITE_DCHECK_LE(unextended_output_shape.DimensionsCount(), 4); + // Current implementation only supports dimension equals 4 and simultaneous + // reduction over width and height. + TFLITE_CHECK_EQ(unextended_input_shape.DimensionsCount(), 4); + TFLITE_CHECK_LE(unextended_output_shape.DimensionsCount(), 4); const RuntimeShape input_shape = RuntimeShape::ExtendedShape(4, unextended_input_shape); const RuntimeShape output_shape = @@ -3666,8 +3680,6 @@ inline void Mean(const tflite::MeanParams& op_params, const int input_height = input_shape.Dims(1); const int input_width = input_shape.Dims(2); - // The current implementation only supports simultaneous reduction over - // width and height. TFLITE_DCHECK_EQ(op_params.axis_count, 2); TFLITE_DCHECK((op_params.axis[0] == 1 && op_params.axis[1] == 2) || (op_params.axis[0] == 2 && op_params.axis[1] == 1)); @@ -4554,6 +4566,63 @@ inline void ResizeNearestNeighbor( } } +inline void BroadcastPrelu4DSlow(const PreluParams& params, + const RuntimeShape& input_shape, + const uint8* input_data, + const RuntimeShape& alpha_shape, + const uint8* alpha_data, + const RuntimeShape& output_shape, + uint8* output_data) { + TFLITE_DCHECK_LE(input_shape.DimensionsCount(), 4); + TFLITE_DCHECK_LE(alpha_shape.DimensionsCount(), 4); + TFLITE_DCHECK_LE(output_shape.DimensionsCount(), 4); + const RuntimeShape extended_output_shape = + RuntimeShape::ExtendedShape(4, output_shape); + NdArrayDesc<4> desc1; + NdArrayDesc<4> desc2; + NdArrayDescsForElementwiseBroadcast(input_shape, alpha_shape, &desc1, &desc2); + + for (int b = 0; b < extended_output_shape.Dims(0); ++b) { + for (int y = 0; y < extended_output_shape.Dims(1); ++y) { + for (int x = 0; x < extended_output_shape.Dims(2); ++x) { + for (int c = 0; c < extended_output_shape.Dims(3); ++c) { + int output_index = Offset(extended_output_shape, b, y, x, c); + int input_index = SubscriptToIndex(desc1, b, y, x, c); + const int32 input_value = + params.input_offset + input_data[input_index]; + if (input_value >= 0) { + output_data[output_index] = input_data[input_index]; + } else { + auto alpha_index = SubscriptToIndex(desc2, b, y, x, c); + const int32 alpha_value = + params.alpha_offset + alpha_data[alpha_index]; + const int32 unclamped_output = + params.output_offset + + MultiplyByQuantizedMultiplierSmallerThanOneExp( + input_value * alpha_value, params.output_multiplier, + params.output_shift); + const int32 quantized_min = std::numeric_limits::min(); + const int32 quantized_max = std::numeric_limits::max(); + const int32 clamped_output = std::min( + quantized_max, std::max(quantized_min, unclamped_output)); + output_data[output_index] = static_cast(clamped_output); + } + } + } + } + } +} + +template +void Fill(const RuntimeShape& value_shape, const T* value_data, + const RuntimeShape& output_shape, T* output_data) { + TFLITE_DCHECK_EQ(value_shape.DimensionsCount(), 0); + const int flat_size = output_shape.FlatSize(); + for (int i = 0; i < flat_size; ++i) { + output_data[i] = *value_data; + } +} + } // namespace reference_ops } // namespace tflite diff --git a/tensorflow/lite/kernels/internal/tensor_ctypes.h b/tensorflow/lite/kernels/internal/tensor_ctypes.h index d24dca9bfbb..4a94b703f8b 100644 --- a/tensorflow/lite/kernels/internal/tensor_ctypes.h +++ b/tensorflow/lite/kernels/internal/tensor_ctypes.h @@ -53,6 +53,11 @@ inline bool* GetTensorData(TfLiteTensor* tensor) { return tensor != nullptr ? tensor->data.b : nullptr; } +template <> +inline int8_t* GetTensorData(TfLiteTensor* tensor) { + return tensor != nullptr ? tensor->data.int8 : nullptr; +} + template inline const T* GetTensorData(const TfLiteTensor* tensor); @@ -66,6 +71,11 @@ inline const uint8_t* GetTensorData(const TfLiteTensor* tensor) { return tensor != nullptr ? tensor->data.uint8 : nullptr; } +template <> +inline const int8_t* GetTensorData(const TfLiteTensor* tensor) { + return tensor != nullptr ? tensor->data.int8 : nullptr; +} + template <> inline const int16_t* GetTensorData(const TfLiteTensor* tensor) { return tensor != nullptr ? tensor->data.i16 : nullptr; diff --git a/tensorflow/lite/kernels/internal/types.h b/tensorflow/lite/kernels/internal/types.h index a05bd5e0033..859ec8c6825 100644 --- a/tensorflow/lite/kernels/internal/types.h +++ b/tensorflow/lite/kernels/internal/types.h @@ -904,6 +904,14 @@ struct PadParams { ResizingCategory resizing_category; }; +struct PreluParams { + int32 input_offset; + int32 alpha_offset; + int32 output_offset; + int32 output_multiplier; + int output_shift; +}; + struct PoolParams { FusedActivationFunctionType activation; PaddingType padding_type; @@ -1006,6 +1014,10 @@ struct UnpackParams { int16 axis; }; +struct LeakyReluParams { + float alpha; +}; + template inline void SetActivationParams(float min, float max, P* params) { params->float_activation_min = min; diff --git a/tensorflow/lite/kernels/mirror_pad.cc b/tensorflow/lite/kernels/mirror_pad.cc new file mode 100644 index 00000000000..e74e47f7a37 --- /dev/null +++ b/tensorflow/lite/kernels/mirror_pad.cc @@ -0,0 +1,374 @@ +/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include + +#include "tensorflow/lite/c/builtin_op_data.h" +#include "tensorflow/lite/c/c_api_internal.h" +#include "tensorflow/lite/kernels/internal/optimized/optimized_ops.h" +#include "tensorflow/lite/kernels/internal/quantization_util.h" +#include "tensorflow/lite/kernels/internal/reference/reference_ops.h" +#include "tensorflow/lite/kernels/internal/tensor.h" +#include "tensorflow/lite/kernels/kernel_util.h" +#include "tensorflow/lite/kernels/op_macros.h" + +namespace tflite { +namespace ops { +namespace builtin { +namespace mirror_pad { +namespace { + +// Simple class that represents a mirror padded tensor - which is the output +// from the Op. +struct PaddedTensor { + // If not null that means this is a scalar value. + // Note: This is not owned by default. It will point to the value + // in the input tensor. + const void* value = nullptr; + // If this tensor is not one value, then this vector will have + // all the tensors that belongs to this tensor. + // Pointers are owned. + std::vector> values; + // Pointers to PaddedTensors that are padded on the left of the current + // tensor. + std::vector left_pad_ptrs; + // Pointers to PaddedTensors that are padded on the right of the current + // tensor. + std::vector right_pad_ptrs; + + // Returns mutable pointer to the tensor identified by 'indices'. + PaddedTensor* GetMutable(const std::vector& indices) { + auto* result = this; + for (int i = 0; i < indices.size(); ++i) { + if (indices[i] >= result->values.size()) { + return nullptr; + } + result = result->values[indices[i]].get(); + if (result == nullptr) break; + } + return result; + } +}; + +// Util method to initialize the memory of the padded tensor. +void InitializeTensorMemory(const TfLiteIntArray* const dims, int dim_index, + int dims_size, PaddedTensor* padded_tensor) { + if (dim_index >= dims_size) { + return; + } + padded_tensor->values.reserve(dims->data[dim_index]); + for (int i = 0; i < dims->data[dim_index]; ++i) { + padded_tensor->values.emplace_back(new PaddedTensor()); + InitializeTensorMemory(dims, dim_index + 1, dims_size, + padded_tensor->values.back().get()); + } +} + +// Returns pointer to the value at the specified index in 'data'. +inline const void* GetValuePointerAtIndex(const void* data, int index, + const TfLiteType data_type) { + switch (data_type) { + case kTfLiteFloat32: + return static_cast(data) + index; + case kTfLiteInt32: + return static_cast(data) + index; + case kTfLiteUInt8: + return static_cast(data) + index; + case kTfLiteInt64: + return static_cast(data) + index; + case kTfLiteBool: + return static_cast(data) + index; + case kTfLiteInt16: + return static_cast(data) + index; + case kTfLiteInt8: + return static_cast(data) + index; + // Unsupported types ? + default: + return nullptr; + } + return nullptr; +} + +// Util method that increment index in the N-d array. +void IncrementTensorIndex(const TfLiteIntArray* dims, + std::vector* tensor_index_ptr) { + int dimension_index = dims->size - 1; + auto& tensor_index = *tensor_index_ptr; + tensor_index[dimension_index]++; + while (dimension_index >= 0 && + tensor_index[dimension_index] == dims->data[dimension_index]) { + tensor_index[dimension_index] = 0; + dimension_index--; + if (dimension_index >= 0) tensor_index[dimension_index]++; + } +} + +// Fills the 'padded_tensor' with data from 'input_tensor'. +TfLiteStatus InitFromInputTensor(const TfLiteTensor* input_tensor, + PaddedTensor* padded_tensor) { + const auto* dims = input_tensor->dims; + const auto data_type = input_tensor->type; + const void* data = static_cast(input_tensor->data.raw_const); + // Either invalid input or unsupported type.+ + if (data == nullptr) { + return kTfLiteError; + } + // Index of current processing tensor. + std::vector tensor_index(dims->size, 0); + int flat_index = 0; + const int num_elements = NumElements(input_tensor); + while (flat_index < num_elements) { + auto* tensor = padded_tensor->GetMutable(tensor_index); + if (tensor == nullptr) { + return kTfLiteError; + } + tensor->value = GetValuePointerAtIndex(data, flat_index, data_type); + IncrementTensorIndex(dims, &tensor_index); + ++flat_index; + } + + return kTfLiteOk; +} + +template +inline void GetPadding(const T* data, int offset, int64_t* left_pad, + int64_t* right_pad) { + *left_pad = static_cast(*(data + offset * 2)); + *right_pad = static_cast(*(data + offset * 2 + 1)); +} + +inline TfLiteStatus GetPadding(const TfLiteTensor* padding_matrix, + int dimension, int64_t* left_pad, + int64_t* right_pad) { + switch (padding_matrix->type) { + case kTfLiteInt32: + GetPadding(padding_matrix->data.i32, dimension, left_pad, right_pad); + break; + case kTfLiteInt64: + GetPadding(padding_matrix->data.i64, dimension, left_pad, right_pad); + break; + default: + return kTfLiteError; + } + return kTfLiteOk; +} + +TfLiteStatus ValidateTensor(const TfLiteTensor* padding_matrix, int offset, + int dimension_index, PaddedTensor* padded_tensor, + TfLiteContext* context) { + if (dimension_index >= padding_matrix->dims->data[0]) { + return kTfLiteOk; + } + + int64_t left_pad = 0, right_pad = 0; + TF_LITE_ENSURE_STATUS( + GetPadding(padding_matrix, dimension_index, &left_pad, &right_pad)); + // If we are not going to include border we must have enough values + // to use. + if (left_pad + offset > padded_tensor->values.size()) { + context->ReportError( + context, "Not enough values for Mirror Pad, required %d, available %d.", + left_pad + offset, padded_tensor->values.size()); + return kTfLiteError; + } + if (right_pad + offset > padded_tensor->values.size()) { + context->ReportError( + context, "Not enough values for Mirror Pad, required %d, available %d.", + right_pad + offset, padded_tensor->values.size()); + return kTfLiteError; + } + if (!padded_tensor->values.empty()) { + ValidateTensor(padding_matrix, offset, dimension_index + 1, + padded_tensor->values[0].get(), context); + } + return kTfLiteOk; +} + +// Fills 'padded_tensor' with the padding information based on +// 'padding_matrix'. +// 'dimension_index' represents which dimension the function is operating on. +TfLiteStatus PadTensor(const TfLiteTensor* padding_matrix, int offset, + int dimension_index, PaddedTensor* padded_tensor, + TfLiteContext* context) { + if (dimension_index >= padding_matrix->dims->data[0]) return kTfLiteOk; + + int64_t left_pad = 0, right_pad = 0; + TF_LITE_ENSURE_STATUS( + GetPadding(padding_matrix, dimension_index, &left_pad, &right_pad)); + + for (int i = left_pad + offset - 1; i >= offset && left_pad > 0; + --i, --left_pad) { + padded_tensor->left_pad_ptrs.push_back(padded_tensor->values[i].get()); + } + for (int i = padded_tensor->values.size() - (1 + offset); + i >= 0 && right_pad > 0; --i, --right_pad) { + padded_tensor->right_pad_ptrs.push_back(padded_tensor->values[i].get()); + } + + for (auto& tensor : padded_tensor->values) { + TF_LITE_ENSURE_STATUS(PadTensor(padding_matrix, offset, dimension_index + 1, + tensor.get(), context)); + } + return kTfLiteOk; +} + +// Fills 'output_data' with data from 'padded_tensor'. +// The function does this recursively by setting left padding first then +// original data, followed by the right padding. +template +int FillOutput(const PaddedTensor* padded_tensor, T* output_data, + int index_in_output) { + if (padded_tensor == nullptr || output_data == nullptr) { + return -1; + } + if (padded_tensor->value != nullptr) { + output_data[index_in_output] = *static_cast(padded_tensor->value); + return index_in_output + 1; + } + for (const auto* tensor : padded_tensor->left_pad_ptrs) { + index_in_output = FillOutput(tensor, output_data, index_in_output); + } + for (const auto& tensor : padded_tensor->values) { + index_in_output = FillOutput(tensor.get(), output_data, index_in_output); + } + for (const auto* tensor : padded_tensor->right_pad_ptrs) { + index_in_output = FillOutput(tensor, output_data, index_in_output); + } + return index_in_output; +} + +// Returns the shape of the final output after padding. +std::unique_ptr GetPaddedOutputShape( + const TfLiteTensor* input, const TfLiteTensor* padding_matrix) { + const int input_dims = NumDimensions(input); + std::unique_ptr shape( + TfLiteIntArrayCreate(input_dims), TfLiteIntArrayFree); + + int64_t left_pad = 0, right_pad = 0; + for (int i = 0; i < input_dims; ++i) { + GetPadding(padding_matrix, i, &left_pad, &right_pad); + shape->data[i] = SizeOfDimension(input, i) + left_pad + right_pad; + } + return shape; +} + +} // namespace + +TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) { + const TfLiteTensor* input_tensor = GetInput(context, node, 0); + const TfLiteTensor* padding_matrix = GetInput(context, node, 1); + auto* params = + reinterpret_cast(node->builtin_data); + + if (params == nullptr) { + return kTfLiteError; + } + const int input_dims = NumDimensions(input_tensor); + + TfLiteTensor* output_tensor = GetOutput(context, node, 0); + if (IsDynamicTensor(output_tensor)) { + auto output_size = GetPaddedOutputShape(input_tensor, padding_matrix); + if (output_size == nullptr) { + return kTfLiteError; + } + TF_LITE_ENSURE_STATUS( + context->ResizeTensor(context, output_tensor, output_size.release())); + } + + PaddedTensor padded_tensor; + // Initialize memory. + InitializeTensorMemory(input_tensor->dims, 0, input_dims, &padded_tensor); + // Set the values from the input_tensor. + TF_LITE_ENSURE_STATUS(InitFromInputTensor(input_tensor, &padded_tensor)); + + const int offset = + params->mode != TfLiteMirrorPaddingMode::kTfLiteMirrorPaddingReflect ? 0 + : 1; + // Make sure padding values are sufficient and valid to use. + TF_LITE_ENSURE_STATUS( + ValidateTensor(padding_matrix, offset, 0, &padded_tensor, context)); + // Apply padding. + TF_LITE_ENSURE_STATUS( + PadTensor(padding_matrix, offset, 0, &padded_tensor, context)); + + // Fill the output tensor from the padded tensor. + TfLiteStatus status = kTfLiteOk; + +#define TF_LITE_MIRROR_PAD(type) \ + FillOutput(&padded_tensor, GetTensorData(output_tensor), 0); + + switch (output_tensor->type) { + case kTfLiteFloat32: { + TF_LITE_MIRROR_PAD(float); + break; + } + case kTfLiteInt32: { + TF_LITE_MIRROR_PAD(int32_t); + break; + } + case kTfLiteUInt8: { + TF_LITE_MIRROR_PAD(uint8_t); + break; + } + case kTfLiteInt64: { + TF_LITE_MIRROR_PAD(int64_t); + break; + } + default: + status = kTfLiteError; + break; + } +#undef TF_LITE_MIRROR_PAD + return status; +} + +void* Init(TfLiteContext* context, const char* buffer, size_t length) { + return nullptr; +} + +void Free(TfLiteContext* context, void* buffer) {} + +TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node) { + const TfLiteTensor* input_tensor = GetInput(context, node, 0); + const TfLiteTensor* padding_matrix = GetInput(context, node, 1); + TfLiteTensor* output_tensor = GetOutput(context, node, 0); + + TF_LITE_ENSURE_EQ(context, NumDimensions(padding_matrix), 2); + TF_LITE_ENSURE_EQ(context, SizeOfDimension(padding_matrix, 0), + NumDimensions(input_tensor)); + + if (!IsConstantTensor(padding_matrix)) { + SetTensorToDynamic(output_tensor); + return kTfLiteOk; + } + // We have constant padding, so we can infer output size. + + auto output_size = GetPaddedOutputShape(input_tensor, padding_matrix); + if (output_size == nullptr) { + return kTfLiteError; + } + return context->ResizeTensor(context, output_tensor, output_size.release()); +} + +} // namespace mirror_pad +TfLiteRegistration* Register_MIRROR_PAD() { + static TfLiteRegistration r = {mirror_pad::Init, mirror_pad::Free, + mirror_pad::Prepare, mirror_pad::Eval}; + return &r; +} + +} // namespace builtin +} // namespace ops +} // namespace tflite diff --git a/tensorflow/lite/kernels/mirror_pad_test.cc b/tensorflow/lite/kernels/mirror_pad_test.cc new file mode 100644 index 00000000000..fd09e6e4493 --- /dev/null +++ b/tensorflow/lite/kernels/mirror_pad_test.cc @@ -0,0 +1,189 @@ +/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#include +#include "tensorflow/lite/interpreter.h" +#include "tensorflow/lite/kernels/register.h" +#include "tensorflow/lite/kernels/test_util.h" +#include "tensorflow/lite/model.h" + +namespace tflite { +namespace { + +using ::testing::ElementsAreArray; + +template +class BaseMirrorPadOpModel : public SingleOpModel { + public: + BaseMirrorPadOpModel(const TensorData& input, + const TensorData& padding_matrix, + const TensorData& output, + const tflite::MirrorPadMode mode) { + input_id_ = AddInput(input); + padding_matrix_id_ = AddInput(padding_matrix); + output_id_ = AddOutput(output); + SetBuiltinOp(BuiltinOperator_MIRROR_PAD, BuiltinOptions_MirrorPadOptions, + CreateMirrorPadOptions(builder_, mode).Union()); + BuildInterpreter({GetShape(input_id_), GetShape(padding_matrix_id_)}); + } + + int input_tensor_id() { return input_id_; } + int padding_matrix_tensor_id() { return padding_matrix_id_; } + + std::vector GetOutput() { return ExtractVector(output_id_); } + + protected: + int input_id_; + int padding_matrix_id_; + int output_id_; +}; + +TEST(MirrorPadTest, EmptyPad) { + BaseMirrorPadOpModel model( + {TensorType_INT32, {2, 3}}, {TensorType_INT32, {2, 2}}, + {TensorType_INT32, {}}, tflite::MirrorPadMode_REFLECT); + model.PopulateTensor(model.input_tensor_id(), {1, 2, 3, 4, 5, 6}); + model.PopulateTensor(model.padding_matrix_tensor_id(), {0, 0, 0, 0}); + model.Invoke(); + EXPECT_THAT(model.GetOutput(), ElementsAreArray({1, 2, 3, 4, 5, 6})); +} + +TEST(MirrorPadTest, PadOneSide_right_Reflect) { + BaseMirrorPadOpModel model( + {TensorType_INT32, {2, 3}}, {TensorType_INT32, {2, 2}}, + {TensorType_INT32, {}}, tflite::MirrorPadMode_REFLECT); + model.PopulateTensor(model.input_tensor_id(), {1, 2, 3, 4, 5, 6}); + model.PopulateTensor(model.padding_matrix_tensor_id(), {0, 1, 0, 1}); + model.Invoke(); + EXPECT_THAT(model.GetOutput(), + ElementsAreArray({1, 2, 3, 2, 4, 5, 6, 5, 1, 2, 3, 2})); +} + +TEST(MirrorPadTest, PadOneSide_left_Reflect) { + BaseMirrorPadOpModel model( + {TensorType_INT32, {2, 3}}, {TensorType_INT32, {2, 2}}, + {TensorType_INT32, {}}, tflite::MirrorPadMode_REFLECT); + model.PopulateTensor(model.input_tensor_id(), {1, 2, 3, 4, 5, 6}); + model.PopulateTensor(model.padding_matrix_tensor_id(), {1, 0, 1, 0}); + model.Invoke(); + EXPECT_THAT(model.GetOutput(), + ElementsAreArray({5, 4, 5, 6, 2, 1, 2, 3, 5, 4, 5, 6})); +} + +TEST(MirrorPadTest, PadOneSide_right_Symmetric) { + BaseMirrorPadOpModel model( + {TensorType_INT32, {2, 3}}, {TensorType_INT32, {2, 2}}, + {TensorType_INT32, {}}, tflite::MirrorPadMode_SYMMETRIC); + model.PopulateTensor(model.input_tensor_id(), {1, 2, 3, 4, 5, 6}); + model.PopulateTensor(model.padding_matrix_tensor_id(), {0, 1, 0, 1}); + model.Invoke(); + EXPECT_THAT(model.GetOutput(), + ElementsAreArray({1, 2, 3, 3, 4, 5, 6, 6, 4, 5, 6, 6})); +} + +TEST(MirrorPadTest, PadOneSide_left_Symmetric) { + BaseMirrorPadOpModel model( + {TensorType_INT32, {2, 3}}, {TensorType_INT32, {2, 2}}, + {TensorType_INT32, {}}, tflite::MirrorPadMode_SYMMETRIC); + model.PopulateTensor(model.input_tensor_id(), {1, 2, 3, 4, 5, 6}); + model.PopulateTensor(model.padding_matrix_tensor_id(), {1, 0, 1, 0}); + model.Invoke(); + EXPECT_THAT(model.GetOutput(), + ElementsAreArray({1, 1, 2, 3, 1, 1, 2, 3, 4, 4, 5, 6})); +} + +TEST(MirrorPadTest, PadBothSides_Symmetric) { + BaseMirrorPadOpModel model( + {TensorType_INT32, {2, 3}}, {TensorType_INT32, {2, 2}}, + {TensorType_INT32, {}}, tflite::MirrorPadMode_SYMMETRIC); + model.PopulateTensor(model.input_tensor_id(), {1, 2, 3, 4, 5, 6}); + model.PopulateTensor(model.padding_matrix_tensor_id(), {1, 1, 1, 1}); + model.Invoke(); + EXPECT_THAT(model.GetOutput(), + ElementsAreArray({1, 1, 2, 3, 3, 1, 1, 2, 3, 3, + 4, 4, 5, 6, 6, 4, 4, 5, 6, 6})); +} + +TEST(MirrorPadTest, PadBothSides_Reflect) { + BaseMirrorPadOpModel model( + {TensorType_INT32, {2, 3}}, {TensorType_INT32, {2, 2}}, + {TensorType_INT32, {}}, tflite::MirrorPadMode_REFLECT); + model.PopulateTensor(model.input_tensor_id(), {1, 2, 3, 4, 5, 6}); + model.PopulateTensor(model.padding_matrix_tensor_id(), {1, 1, 1, 1}); + model.Invoke(); + EXPECT_THAT(model.GetOutput(), + ElementsAreArray({5, 4, 5, 6, 5, 2, 1, 2, 3, 2, + 5, 4, 5, 6, 5, 2, 1, 2, 3, 2})); +} + +TEST(MirrorPadTest, PadBothSides_Symmetric_Whole) { + BaseMirrorPadOpModel model( + {TensorType_INT32, {2, 3}}, {TensorType_INT32, {2, 2}}, + {TensorType_INT32, {}}, tflite::MirrorPadMode_SYMMETRIC); + model.PopulateTensor(model.input_tensor_id(), {1, 2, 3, 4, 5, 6}); + model.PopulateTensor(model.padding_matrix_tensor_id(), {2, 2, 3, 3}); + model.Invoke(); + EXPECT_THAT( + model.GetOutput(), + ElementsAreArray({6, 5, 4, 4, 5, 6, 6, 5, 4, 3, 2, 1, 1, 2, 3, 3, 2, 1, + 3, 2, 1, 1, 2, 3, 3, 2, 1, 6, 5, 4, 4, 5, 6, 6, 5, 4, + 6, 5, 4, 4, 5, 6, 6, 5, 4, 3, 2, 1, 1, 2, 3, 3, 2, 1})); +} + +TEST(MirrorPadTest, PadBothSides_Reflect_Whole) { + BaseMirrorPadOpModel model( + {TensorType_INT32, {2, 3}}, {TensorType_INT32, {2, 2}}, + {TensorType_INT32, {}}, tflite::MirrorPadMode_REFLECT); + model.PopulateTensor(model.input_tensor_id(), {1, 2, 3, 4, 5, 6}); + model.PopulateTensor(model.padding_matrix_tensor_id(), {1, 1, 2, 2}); + model.Invoke(); + EXPECT_THAT(model.GetOutput(), + ElementsAreArray({6, 5, 4, 5, 6, 5, 4, 3, 2, 1, 2, 3, 2, 1, + 6, 5, 4, 5, 6, 5, 4, 3, 2, 1, 2, 3, 2, 1})); +} + +TEST(MirrorPadTest, Pad_Symmetric) { + BaseMirrorPadOpModel model( + {TensorType_INT32, {2, 3}}, {TensorType_INT32, {2, 2}}, + {TensorType_INT32, {}}, tflite::MirrorPadMode_SYMMETRIC); + model.PopulateTensor(model.input_tensor_id(), {1, 2, 3, 4, 5, 6}); + model.PopulateTensor(model.padding_matrix_tensor_id(), {1, 1, 2, 2}); + model.Invoke(); + EXPECT_THAT(model.GetOutput(), + ElementsAreArray({2, 1, 1, 2, 3, 3, 2, 2, 1, 1, 2, 3, 3, 2, + 5, 4, 4, 5, 6, 6, 5, 5, 4, 4, 5, 6, 6, 5})); +} + +TEST(MirrorPadTest, Pad_1D_Reflect) { + BaseMirrorPadOpModel model( + {TensorType_INT32, {3}}, {TensorType_INT32, {1, 2}}, + {TensorType_INT32, {}}, tflite::MirrorPadMode_REFLECT); + model.PopulateTensor(model.input_tensor_id(), {1, 2, 3}); + model.PopulateTensor(model.padding_matrix_tensor_id(), {0, 2}); + model.Invoke(); + EXPECT_THAT(model.GetOutput(), ElementsAreArray({1, 2, 3, 2, 1})); +} + +TEST(MirrorPadTest, Pad_1D_Symmetric) { + BaseMirrorPadOpModel model( + {TensorType_INT32, {3}}, {TensorType_INT32, {1, 2}}, + {TensorType_INT32, {}}, tflite::MirrorPadMode_SYMMETRIC); + model.PopulateTensor(model.input_tensor_id(), {1, 2, 3}); + model.PopulateTensor(model.padding_matrix_tensor_id(), {0, 2}); + model.Invoke(); + EXPECT_THAT(model.GetOutput(), ElementsAreArray({1, 2, 3, 3, 2})); +} + +} // namespace +} // namespace tflite diff --git a/tensorflow/lite/kernels/pooling_test.cc b/tensorflow/lite/kernels/pooling_test.cc index 80eef025090..98777f1c13f 100644 --- a/tensorflow/lite/kernels/pooling_test.cc +++ b/tensorflow/lite/kernels/pooling_test.cc @@ -67,6 +67,10 @@ class QuantizedPoolingOpModel : public BasePoolingOpModel { QuantizeAndPopulate(input_, data); } + void SetInput(const std::vector& data) { + QuantizeAndPopulate(input_, data); + } + std::vector GetOutput() { return ExtractVector(output_); } std::vector GetDequantizedOutput() { return Dequantize(ExtractVector(output_), @@ -106,6 +110,45 @@ TEST(QuantizedPoolingOpTest, AveragePool) { EXPECT_THAT(m.GetOutput(), ElementsAreArray({44, 92})); } +// Send in a white image, expect a white pixel. +TEST(QuantizedPoolingOpTest, AveragePoolImageSize16) { + int image_size = 16; + QuantizedPoolingOpModel m( + BuiltinOperator_AVERAGE_POOL_2D, + /*input=*/{TensorType_UINT8, {1, image_size, image_size, 1}, 0, 16}, + /*filter_width=*/image_size, + /*filter_height=*/image_size, + /*output=*/{TensorType_UINT8, {}, 0, 16}); + + std::vector input(image_size * image_size, 16.f); + m.SetInput(input); + m.Invoke(); + + EXPECT_THAT(m.GetOutput(), ::testing::ElementsAre(255)); + EXPECT_THAT(m.GetDequantizedOutput(), ElementsAreArray(ArrayFloatNear({16}))); +} + +// Send in a white image, expect something other than a white pixel, due to +// overflow. +TEST(QuantizedPoolingOpTest, AveragePoolImageSize17) { + int image_size = 17; + QuantizedPoolingOpModel m( + BuiltinOperator_AVERAGE_POOL_2D, + /*input=*/{TensorType_UINT8, {1, image_size, image_size, 1}, 0, 16}, + /*filter_width=*/image_size, + /*filter_height=*/image_size, + /*output=*/{TensorType_UINT8, {}, 0, 16}); + + std::vector input(image_size * image_size, 16.f); + m.SetInput(input); + m.Invoke(); + + // Ordinarily we would see '255' here. However, the optimized version of + // AveragePool uses a uint16 accumulator which causes it to overflow for + // images this large. + EXPECT_THAT(m.GetOutput(), ::testing::ElementsAre(28)); +} + TEST(FloatPoolingOpTest, MaxPool) { FloatPoolingOpModel m(BuiltinOperator_MAX_POOL_2D, /*input=*/{TensorType_FLOAT32, {1, 2, 4, 1}}, diff --git a/tensorflow/lite/kernels/reduce.cc b/tensorflow/lite/kernels/reduce.cc index ed2d475f6d7..336e827ca4c 100644 --- a/tensorflow/lite/kernels/reduce.cc +++ b/tensorflow/lite/kernels/reduce.cc @@ -20,6 +20,8 @@ limitations under the License. #include "tensorflow/lite/kernels/internal/quantization_util.h" #include "tensorflow/lite/kernels/internal/reference/reference_ops.h" #include "tensorflow/lite/kernels/internal/tensor.h" +#include "tensorflow/lite/kernels/internal/tensor_ctypes.h" +#include "tensorflow/lite/kernels/internal/types.h" #include "tensorflow/lite/kernels/kernel_util.h" #include "tensorflow/lite/kernels/op_macros.h" @@ -229,6 +231,17 @@ TfLiteStatus PrepareMeanOrSum(TfLiteContext* context, TfLiteNode* node) { return ResizeTempSum(context, &op_context, temp_sum); } +void ResolveAxis(const int* axis_data, int axis_count, + tflite::MeanParams* op_params) { + int i = 0; + for (; i < axis_count; ++i) { + op_params->axis[i] = static_cast(axis_data[i]); + } + for (; i < 4; ++i) { + op_params->axis[i] = 1; + } +} + template TfLiteStatus EvalMean(TfLiteContext* context, TfLiteNode* node) { OpContext op_context(context, node); @@ -257,9 +270,23 @@ TfLiteStatus EvalMean(TfLiteContext* context, TfLiteNode* node) { if (kernel_type == kReference) { switch (op_context.input->type) { - case kTfLiteFloat32: - TF_LITE_ENSURE(context, TF_LITE_MEAN(reference_ops, float, float)); - break; + case kTfLiteFloat32: { + tflite::MeanParams op_params; + op_params.axis_count = num_axis; + ResolveAxis(GetTensorData(op_context.axis), num_axis, &op_params); + const TfLiteTensor* input = op_context.input; + if (op_context.params->keep_dims && NumDimensions(input) == 4 && + op_params.axis_count == 2 && + ((op_params.axis[0] == 1 && op_params.axis[1] == 2) || + (op_params.axis[0] == 2 && op_params.axis[1] == 1))) { + reference_ops::Mean(op_params, GetTensorShape(input), + GetTensorData(input), + GetTensorShape(op_context.output), + GetTensorData(op_context.output)); + } else { + TF_LITE_ENSURE(context, TF_LITE_MEAN(reference_ops, float, float)); + } + } break; case kTfLiteInt32: TF_LITE_ENSURE(context, TF_LITE_MEAN(reference_ops, int, int64_t)); break; @@ -286,7 +313,8 @@ TfLiteStatus EvalMean(TfLiteContext* context, TfLiteNode* node) { GetTensorData(op_context.axis), num_axis, op_context.params->keep_dims, GetTensorData(temp_index), GetTensorData(resolved_axis), - GetTensorData(temp_sum), /*compute_sum=*/false)); + GetTensorData(temp_sum), + /*compute_sum=*/false)); } break; default: diff --git a/tensorflow/lite/kernels/register.cc b/tensorflow/lite/kernels/register.cc index c6834537671..3c60d281b39 100644 --- a/tensorflow/lite/kernels/register.cc +++ b/tensorflow/lite/kernels/register.cc @@ -31,6 +31,7 @@ TfLiteRegistration* Register_RELU_1(); namespace builtin { +TfLiteRegistration* Register_ABS(); TfLiteRegistration* Register_RELU(); TfLiteRegistration* Register_RELU_N1_TO_1(); TfLiteRegistration* Register_RELU6(); @@ -74,6 +75,7 @@ TfLiteRegistration* Register_GATHER(); TfLiteRegistration* Register_TRANSPOSE(); TfLiteRegistration* Register_MEAN(); TfLiteRegistration* Register_SPLIT(); +TfLiteRegistration* Register_SPLIT_V(); TfLiteRegistration* Register_SQUEEZE(); TfLiteRegistration* Register_STRIDED_SLICE(); TfLiteRegistration* Register_EXP(); @@ -123,6 +125,10 @@ TfLiteRegistration* Register_SQUARE(); TfLiteRegistration* Register_ZEROS_LIKE(); TfLiteRegistration* Register_FLOOR_MOD(); TfLiteRegistration* Register_RANGE(); +TfLiteRegistration* Register_LEAKY_RELU(); +TfLiteRegistration* Register_SQUARED_DIFFERENCE(); +TfLiteRegistration* Register_FILL(); +TfLiteRegistration* Register_MIRROR_PAD(); TfLiteStatus UnsupportedTensorFlowOp(TfLiteContext* context, TfLiteNode* node) { context->ReportError( @@ -152,6 +158,7 @@ const TfLiteRegistration* BuiltinOpResolver::FindOp(const char* op, } BuiltinOpResolver::BuiltinOpResolver() { + AddBuiltin(BuiltinOperator_ABS, Register_ABS()); AddBuiltin(BuiltinOperator_RELU, Register_RELU()); AddBuiltin(BuiltinOperator_RELU_N1_TO_1, Register_RELU_N1_TO_1()); AddBuiltin(BuiltinOperator_RELU6, Register_RELU6()); @@ -207,6 +214,7 @@ BuiltinOpResolver::BuiltinOpResolver() { AddBuiltin(BuiltinOperator_DIV, Register_DIV()); AddBuiltin(BuiltinOperator_SUB, Register_SUB()); AddBuiltin(BuiltinOperator_SPLIT, Register_SPLIT()); + AddBuiltin(BuiltinOperator_SPLIT_V, Register_SPLIT_V()); AddBuiltin(BuiltinOperator_SQUEEZE, Register_SQUEEZE()); AddBuiltin(BuiltinOperator_STRIDED_SLICE, Register_STRIDED_SLICE()); AddBuiltin(BuiltinOperator_EXP, Register_EXP()); @@ -256,6 +264,10 @@ BuiltinOpResolver::BuiltinOpResolver() { AddBuiltin(BuiltinOperator_ZEROS_LIKE, Register_ZEROS_LIKE()); AddBuiltin(BuiltinOperator_FLOOR_MOD, Register_FLOOR_MOD()); AddBuiltin(BuiltinOperator_RANGE, Register_RANGE()); + AddBuiltin(BuiltinOperator_LEAKY_RELU, Register_LEAKY_RELU()); + AddBuiltin(BuiltinOperator_SQUARED_DIFFERENCE, Register_SQUARED_DIFFERENCE()); + AddBuiltin(BuiltinOperator_FILL, Register_FILL()); + AddBuiltin(BuiltinOperator_MIRROR_PAD, Register_MIRROR_PAD()); // TODO(andrewharp, ahentz): Move these somewhere more appropriate so that // custom ops aren't always included by default. diff --git a/tensorflow/lite/kernels/register.h b/tensorflow/lite/kernels/register.h index eb5ce667d4c..059c9d165ee 100644 --- a/tensorflow/lite/kernels/register.h +++ b/tensorflow/lite/kernels/register.h @@ -15,7 +15,6 @@ limitations under the License. #ifndef TENSORFLOW_LITE_KERNELS_REGISTER_H_ #define TENSORFLOW_LITE_KERNELS_REGISTER_H_ -#include #include "tensorflow/lite/c/c_api_internal.h" #include "tensorflow/lite/model.h" #include "tensorflow/lite/mutable_op_resolver.h" diff --git a/tensorflow/lite/kernels/skip_gram.cc b/tensorflow/lite/kernels/skip_gram.cc index f20719ecaf6..265ba18a3e3 100644 --- a/tensorflow/lite/kernels/skip_gram.cc +++ b/tensorflow/lite/kernels/skip_gram.cc @@ -107,7 +107,7 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) { // Generate n-grams recursively. tflite::DynamicBuffer buf; if (words.size() < params->ngram_size) { - buf.WriteToTensor(GetOutput(context, node, 0)); + buf.WriteToTensorAsVector(GetOutput(context, node, 0)); return kTfLiteOk; } @@ -145,7 +145,7 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) { } } - buf.WriteToTensor(GetOutput(context, node, 0)); + buf.WriteToTensorAsVector(GetOutput(context, node, 0)); return kTfLiteOk; } } // namespace diff --git a/tensorflow/lite/kernels/split_v.cc b/tensorflow/lite/kernels/split_v.cc new file mode 100644 index 00000000000..060e3c5f79c --- /dev/null +++ b/tensorflow/lite/kernels/split_v.cc @@ -0,0 +1,207 @@ +/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#include +#include "tensorflow/lite/c/builtin_op_data.h" +#include "tensorflow/lite/c/c_api_internal.h" +#include "tensorflow/lite/kernels/internal/optimized/optimized_ops.h" +#include "tensorflow/lite/kernels/internal/reference/reference_ops.h" +#include "tensorflow/lite/kernels/internal/tensor.h" +#include "tensorflow/lite/kernels/kernel_util.h" +#include "tensorflow/lite/kernels/op_macros.h" + +namespace tflite { +namespace ops { +namespace builtin { +namespace split_v { + +struct OpContext { + OpContext(TfLiteContext* context, TfLiteNode* node) { + params = reinterpret_cast(node->builtin_data); + input = GetInput(context, node, 0); + size_splits = GetInput(context, node, 1); + axis = GetInput(context, node, 2); + } + TfLiteSplitVParams* params; + const TfLiteTensor* input; + const TfLiteTensor* size_splits; + const TfLiteTensor* axis; +}; + +TfLiteStatus UseDynamicOutputTensors(TfLiteContext* context, TfLiteNode* node) { + for (int i = 0; i < NumOutputs(node); ++i) { + SetTensorToDynamic(GetOutput(context, node, i)); + } + return kTfLiteOk; +} + +template +void GetSizeSplitsVector(const TfLiteTensor* size_splits, + std::vector* size_splits_vector) { + const auto num_elements = NumElements(size_splits); + for (int i = 0; i < num_elements; ++i) { + size_splits_vector->push_back(GetTensorData(size_splits)[i]); + } +} + +TfLiteStatus ResizeOutputTensors(TfLiteContext* context, TfLiteNode* node, + const TfLiteTensor* input, + const TfLiteTensor* size_splits, + const TfLiteTensor* axis) { + int axis_value = GetTensorData(axis)[0]; + if (axis_value < 0) { + axis_value += NumDimensions(input); + } + + std::vector size_splits_vector; + if (size_splits->type == kTfLiteInt32) { + GetSizeSplitsVector(size_splits, &size_splits_vector); + } else if (size_splits->type == kTfLiteInt64) { + GetSizeSplitsVector(size_splits, &size_splits_vector); + } else { + context->ReportError(context, "size_splits only support type int32|int64."); + return kTfLiteError; + } + + int minus_one_index = -1; + int64_t size_splits_sum = 0; + + for (int i = 0; i < size_splits_vector.size(); ++i) { + if (size_splits_vector.at(i) == -1) { + if (minus_one_index == -1) { + minus_one_index = i; + } else { + context->ReportError(context, + "The size_splits contains more than one -1."); + } + } else { + size_splits_sum += size_splits_vector.at(i); + } + } + + const int input_size = SizeOfDimension(input, axis_value); + + if (minus_one_index != -1) { + if (size_splits_sum > input_size) { + context->ReportError( + context, + "The sum of size_splits must be less than the dimension of value."); + } else { + size_splits_vector[minus_one_index] = input_size - size_splits_sum; + } + } else if (size_splits_sum != input_size) { + context->ReportError( + context, + "The size_splits must sum to the dimension of value along axis."); + } + + for (int i = 0; i < NumOutputs(node); ++i) { + TfLiteIntArray* output_dims = TfLiteIntArrayCopy(input->dims); + output_dims->data[axis_value] = size_splits_vector.at(i); + TfLiteTensor* output = GetOutput(context, node, i); + TF_LITE_ENSURE_STATUS(context->ResizeTensor(context, output, output_dims)); + } + + return kTfLiteOk; +} + +TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node) { + TF_LITE_ENSURE_EQ(context, NumInputs(node), 3); + + OpContext op_context(context, node); + + TF_LITE_ENSURE_EQ(context, NumOutputs(node), op_context.params->num_splits); + + auto input_type = op_context.input->type; + TF_LITE_ENSURE(context, input_type == kTfLiteFloat32 || + input_type == kTfLiteUInt8 || + input_type == kTfLiteInt16); + for (int i = 0; i < NumOutputs(node); ++i) { + GetOutput(context, node, i)->type = input_type; + } + + auto size_splits = op_context.size_splits; + TF_LITE_ENSURE_EQ(context, NumDimensions(size_splits), 1); + TF_LITE_ENSURE_EQ(context, NumOutputs(node), NumElements(size_splits)); + + // If we know the contents of the 'size_splits' tensor and the 'axis' tensor, + // resize all outputs. Otherwise, wait until Eval(). + if (IsConstantTensor(op_context.size_splits) && + IsConstantTensor(op_context.axis)) { + return ResizeOutputTensors(context, node, op_context.input, + op_context.size_splits, op_context.axis); + } else { + return UseDynamicOutputTensors(context, node); + } +} + +TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) { + OpContext op_context(context, node); + + // When the 'size_splits' and the 'axis' tensor is non-const we can't resize + // output tensors in Prepare(), and we have to do it now. + if (!IsConstantTensor(op_context.axis) || + !IsConstantTensor(op_context.size_splits)) { + TF_LITE_ENSURE_OK( + context, ResizeOutputTensors(context, node, op_context.input, + op_context.size_splits, op_context.axis)); + } + + int axis_value = GetTensorData(op_context.axis)[0]; + + // Use split function to build the outputs since they share the same logic. +#define TF_LITE_SPLIT_V(scalar) \ + VectorOfTensors all_outputs(*context, *node->outputs); \ + tflite::SplitParams op_params; \ + op_params.num_split = NumOutputs(node); \ + op_params.axis = axis_value; \ + reference_ops::Split(op_params, GetTensorShape(op_context.input), \ + GetTensorData(op_context.input), \ + all_outputs.shapes(), all_outputs.data()); + switch (op_context.input->type) { + case kTfLiteFloat32: { + TF_LITE_SPLIT_V(float); + break; + } + case kTfLiteUInt8: { + TF_LITE_SPLIT_V(uint8_t); + break; + } + case kTfLiteInt16: { + TF_LITE_SPLIT_V(int16_t); + break; + } + default: + context->ReportError( + context, + "Only float32, uint8 and int16 are currently supported, got %d.", + op_context.input->type); + return kTfLiteError; + } +#undef TF_LITE_SPLIT_V + + return kTfLiteOk; +} + +} // namespace split_v + +TfLiteRegistration* Register_SPLIT_V() { + static TfLiteRegistration r = {nullptr, nullptr, split_v::Prepare, + split_v::Eval}; + return &r; +} + +} // namespace builtin +} // namespace ops +} // namespace tflite diff --git a/tensorflow/lite/kernels/split_v_test.cc b/tensorflow/lite/kernels/split_v_test.cc new file mode 100644 index 00000000000..2d1d36d6851 --- /dev/null +++ b/tensorflow/lite/kernels/split_v_test.cc @@ -0,0 +1,175 @@ +/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#include +#include +#include "tensorflow/lite/interpreter.h" +#include "tensorflow/lite/kernels/register.h" +#include "tensorflow/lite/kernels/test_util.h" +#include "tensorflow/lite/model.h" + +namespace tflite { +namespace { + +using ::testing::ElementsAreArray; + +constexpr int kAxisIsATensor = -1000; + +class SplitVOpModel : public SingleOpModel { + public: + SplitVOpModel(const TensorData& input, const TensorData& size_splits, + int num_splits, int axis) { + input_ = AddInput(input); + size_splits_ = AddInput(size_splits); + if (axis == kAxisIsATensor) { + axis_ = AddInput({TensorType_INT32, {1}}); + } else { + axis_ = AddConstInput(TensorType_INT32, {axis}, {1}); + } + for (int i = 0; i < num_splits; ++i) { + outputs_.push_back(AddOutput(input.type)); + } + SetBuiltinOp(BuiltinOperator_SPLIT_V, BuiltinOptions_SplitVOptions, + CreateSplitVOptions(builder_, num_splits).Union()); + if (axis == kAxisIsATensor) { + BuildInterpreter( + {GetShape(input_), GetShape(size_splits_), GetShape(axis_)}); + } else { + BuildInterpreter({GetShape(input_), GetShape(size_splits_), {}}); + } + } + + void SetInput(std::initializer_list data) { + PopulateTensor(input_, data); + } + void SetSizeSplits(std::initializer_list data) { + PopulateTensor(size_splits_, data); + } + void SetAxis(int axis) { PopulateTensor(axis_, {axis}); } + + std::vector GetOutput(int i) { + return ExtractVector(outputs_[i]); + } + std::vector GetOutputShape(int i) { return GetTensorShape(outputs_[i]); } + + private: + int input_; + int size_splits_; + int axis_; + std::vector outputs_; +}; + +// TODO(ruic): Add tests to test quantized values. b/119638735 +using TensorValues = std::initializer_list; + +void Check(int axis, std::initializer_list input_shape, + std::initializer_list size_splits_shape, + std::vector> output_shapes, + const TensorValues& input_data, + const std::initializer_list& size_splits_data, + const std::vector& output_data) { + int num_splits = size_splits_data.size(); + SplitVOpModel m({TensorType_FLOAT32, input_shape}, + {TensorType_INT32, size_splits_shape}, num_splits, + kAxisIsATensor); + m.SetInput(input_data); + m.SetSizeSplits(size_splits_data); + m.SetAxis(axis); + m.Invoke(); + for (int i = 0; i < num_splits; ++i) { + EXPECT_THAT(m.GetOutput(i), ElementsAreArray(output_data[i])); + EXPECT_THAT(m.GetOutputShape(i), ElementsAreArray(output_shapes[i])); + } + + SplitVOpModel const_m({TensorType_FLOAT32, input_shape}, + {TensorType_INT32, size_splits_shape}, num_splits, + axis); + const_m.SetInput(input_data); + const_m.SetSizeSplits(size_splits_data); + const_m.Invoke(); + for (int i = 0; i < num_splits; ++i) { + EXPECT_THAT(const_m.GetOutput(i), ElementsAreArray(output_data[i])); + EXPECT_THAT(const_m.GetOutputShape(i), ElementsAreArray(output_shapes[i])); + } +} + +TEST(SplitVOpTest, TwoDimensional) { + // Input shape: {4, 3} + // size_splits: {1, 1, 3} + // axis: 0 + // We should have 3 outpus with shapes respectively: + // output 0 : {1, 3} + // output 1 : {1, 3} + // output 1 : {2, 3} + Check(/*axis=*/0, {4, 3}, {3}, {{1, 3}, {1, 3}, {2, 3}}, + {1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12}, {1, 1, 2}, + {{1, 2, 3}, {4, 5, 6}, {7, 8, 9, 10, 11, 12}}); +} + +TEST(SplitVOpTest, FourDimensional) { + Check(/*axis=*/0, {2, 2, 2, 2}, {2}, {{1, 2, 2, 2}, {1, 2, 2, 2}}, + {1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16}, {1, 1}, + { + {1, 2, 3, 4, 5, 6, 7, 8}, + {9, 10, 11, 12, 13, 14, 15, 16}, + }); + Check(/*axis=*/1, {2, 2, 2, 2}, {2}, {{2, 1, 2, 2}, {2, 1, 2, 2}}, + {1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16}, {1, -1}, + { + {1, 2, 3, 4, 9, 10, 11, 12}, + {5, 6, 7, 8, 13, 14, 15, 16}, + }); + Check(/*axis=*/2, {2, 2, 2, 2}, {2}, {{2, 2, 1, 2}, {2, 2, 1, 2}}, + {1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16}, {1, 1}, + { + {1, 2, 5, 6, 9, 10, 13, 14}, + {3, 4, 7, 8, 11, 12, 15, 16}, + }); + Check(/*axis=*/3, {2, 2, 2, 2}, {2}, {{2, 2, 2, 1}, {2, 2, 2, 1}}, + {1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16}, {1, 1}, + { + {1, 3, 5, 7, 9, 11, 13, 15}, + {2, 4, 6, 8, 10, 12, 14, 16}, + }); +} + +TEST(SplitVOpTest, OneDimensional) { + Check(/*axis=*/0, {8}, {8}, {{1}, {1}, {1}, {1}, {1}, {1}, {1}, {1}}, + {1, 2, 3, 4, 5, 6, 7, 8}, {1, 1, 1, 1, 1, 1, 1, 1}, + {{1}, {2}, {3}, {4}, {5}, {6}, {7}, {8}}); +} + +TEST(SplitVOpTest, OneDimensional2) { + Check(/*axis=*/0, {8}, {8}, {{1}, {1}, {1}, {1}, {1}, {1}, {2}, {0}}, + {1, 2, 3, 4, 5, 6, 7, 8}, {1, 1, 1, 1, 1, 1, 2, -1}, + {{1}, {2}, {3}, {4}, {5}, {6}, {7, 8}, {}}); +} + +TEST(SplitVOpTest, NegativeAxis) { + Check(/*axis=*/-4, {2, 2, 2, 2}, {2}, {{1, 2, 2, 2}, {1, 2, 2, 2}}, + {1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16}, {1, 1}, + { + {1, 2, 3, 4, 5, 6, 7, 8}, + {9, 10, 11, 12, 13, 14, 15, 16}, + }); +} + +} // namespace +} // namespace tflite + +int main(int argc, char** argv) { + ::tflite::LogToStderr(); + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/tensorflow/lite/kernels/squared_difference.cc b/tensorflow/lite/kernels/squared_difference.cc new file mode 100644 index 00000000000..59b53a6287d --- /dev/null +++ b/tensorflow/lite/kernels/squared_difference.cc @@ -0,0 +1,129 @@ +/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#include "tensorflow/lite/c/builtin_op_data.h" +#include "tensorflow/lite/c/c_api_internal.h" +#include "tensorflow/lite/kernels/internal/optimized/optimized_ops.h" +#include "tensorflow/lite/kernels/internal/quantization_util.h" +#include "tensorflow/lite/kernels/internal/reference/reference_ops.h" +#include "tensorflow/lite/kernels/internal/tensor.h" +#include "tensorflow/lite/kernels/kernel_util.h" +#include "tensorflow/lite/kernels/op_macros.h" + +namespace tflite { +namespace ops { +namespace builtin { +namespace squared_difference { + +constexpr int kInputTensor1 = 0; +constexpr int kInputTensor2 = 1; +constexpr int kOutputTensor = 0; + +struct OpData { + bool requires_broadcast; +}; + +template +T SquaredDifference(T input1, T input2) { + const T difference = input1 - input2; + return difference * difference; +} + +void* Init(TfLiteContext* context, const char* buffer, size_t length) { + auto* data = new OpData; + data->requires_broadcast = false; + return data; +} + +void Free(TfLiteContext* context, void* buffer) { + delete reinterpret_cast(buffer); +} + +TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node) { + OpData* data = reinterpret_cast(node->user_data); + + TF_LITE_ENSURE_EQ(context, NumInputs(node), 2); + TF_LITE_ENSURE_EQ(context, NumOutputs(node), 1); + + const TfLiteTensor* input1 = GetInput(context, node, kInputTensor1); + const TfLiteTensor* input2 = GetInput(context, node, kInputTensor2); + TfLiteTensor* output = GetOutput(context, node, kOutputTensor); + + TF_LITE_ENSURE_EQ(context, input1->type, input2->type); + output->type = input2->type; + + data->requires_broadcast = !HaveSameShapes(input1, input2); + + TfLiteIntArray* output_size = nullptr; + if (data->requires_broadcast) { + TF_LITE_ENSURE_OK(context, CalculateShapeForBroadcast( + context, input1, input2, &output_size)); + } else { + output_size = TfLiteIntArrayCopy(input1->dims); + } + + return context->ResizeTensor(context, output, output_size); +} + +template +void EvalSquaredDifference(TfLiteContext* context, TfLiteNode* node, + const OpData* data, const TfLiteTensor* input1, + const TfLiteTensor* input2, TfLiteTensor* output) { + if (data->requires_broadcast) { + reference_ops::BroadcastBinaryFunction4DSlow( + GetTensorShape(input1), GetTensorData(input1), + GetTensorShape(input2), GetTensorData(input2), + GetTensorShape(output), GetTensorData(output), SquaredDifference); + } else { + reference_ops::BinaryFunction( + GetTensorShape(input1), GetTensorData(input1), + GetTensorShape(input2), GetTensorData(input2), + GetTensorShape(output), GetTensorData(output), SquaredDifference); + } +} + +TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) { + OpData* data = reinterpret_cast(node->user_data); + + const TfLiteTensor* input1 = GetInput(context, node, kInputTensor1); + const TfLiteTensor* input2 = GetInput(context, node, kInputTensor2); + TfLiteTensor* output = GetOutput(context, node, kOutputTensor); + + if (output->type == kTfLiteFloat32) { + EvalSquaredDifference(context, node, data, input1, input2, output); + } else if (output->type == kTfLiteInt32) { + EvalSquaredDifference(context, node, data, input1, input2, output); + } else { + context->ReportError(context, + "SquaredDifference only supports FLOAT32, INT32 and " + "quantized UINT8 now, got %d.", + output->type); + return kTfLiteError; + } + + return kTfLiteOk; +} + +} // namespace squared_difference + +TfLiteRegistration* Register_SQUARED_DIFFERENCE() { + static TfLiteRegistration r = { + squared_difference::Init, squared_difference::Free, + squared_difference::Prepare, squared_difference::Eval}; + return &r; +} + +} // namespace builtin +} // namespace ops +} // namespace tflite diff --git a/tensorflow/lite/kernels/squared_difference_test.cc b/tensorflow/lite/kernels/squared_difference_test.cc new file mode 100644 index 00000000000..32bcab3b87f --- /dev/null +++ b/tensorflow/lite/kernels/squared_difference_test.cc @@ -0,0 +1,157 @@ +/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#include +#include "tensorflow/lite/interpreter.h" +#include "tensorflow/lite/kernels/register.h" +#include "tensorflow/lite/kernels/test_util.h" +#include "tensorflow/lite/model.h" + +namespace tflite { +namespace { + +using ::testing::ElementsAreArray; + +class BaseSquaredDifferenceOpModel : public SingleOpModel { + public: + BaseSquaredDifferenceOpModel(const TensorData& input1, + const TensorData& input2, + const TensorData& output) { + input1_ = AddInput(input1); + input2_ = AddInput(input2); + output_ = AddOutput(output); + SetBuiltinOp(BuiltinOperator_SQUARED_DIFFERENCE, + BuiltinOptions_SquaredDifferenceOptions, + CreateSquaredDifferenceOptions(builder_).Union()); + BuildInterpreter({GetShape(input1_), GetShape(input2_)}); + } + + int input1() { return input1_; } + int input2() { return input2_; } + + protected: + int input1_; + int input2_; + int output_; +}; + +class FloatSquaredDifferenceOpModel : public BaseSquaredDifferenceOpModel { + public: + using BaseSquaredDifferenceOpModel::BaseSquaredDifferenceOpModel; + + std::vector GetOutput() { return ExtractVector(output_); } +}; + +class IntegerSquaredDifferenceOpModel : public BaseSquaredDifferenceOpModel { + public: + using BaseSquaredDifferenceOpModel::BaseSquaredDifferenceOpModel; + + std::vector GetOutput() { return ExtractVector(output_); } +}; + +TEST(FloatSquaredDifferenceOpTest, FloatType_SameShape) { + FloatSquaredDifferenceOpModel m({TensorType_FLOAT32, {1, 2, 2, 1}}, + {TensorType_FLOAT32, {1, 2, 2, 1}}, + {TensorType_FLOAT32, {}}); + m.PopulateTensor(m.input1(), {-0.2, 0.2, -1.2, 0.8}); + m.PopulateTensor(m.input2(), {0.5, 0.2, -1.5, 0.5}); + m.Invoke(); + EXPECT_THAT(m.GetOutput(), + ElementsAreArray(ArrayFloatNear({0.49, 0.0, 0.09, 0.09}))); +} + +TEST(FloatSquaredDifferenceOpTest, FloatType_VariousInputShapes) { + std::vector> test_shapes = { + {6}, {2, 3}, {2, 1, 3}, {1, 3, 1, 2}}; + for (int i = 0; i < test_shapes.size(); ++i) { + FloatSquaredDifferenceOpModel m({TensorType_FLOAT32, test_shapes[i]}, + {TensorType_FLOAT32, test_shapes[i]}, + {TensorType_FLOAT32, {}}); + m.PopulateTensor(m.input1(), {-2.0, 0.2, 0.3, 0.8, 1.1, -2.0}); + m.PopulateTensor(m.input2(), {1.0, 0.2, 0.6, 0.4, -1.0, -0.0}); + m.Invoke(); + EXPECT_THAT( + m.GetOutput(), + ElementsAreArray(ArrayFloatNear({9.0, 0.0, 0.09, 0.16, 4.41, 4.0}))) + << "With shape number " << i; + } +} + +TEST(FloatSquaredDifferenceOpTest, FloatType_WithBroadcast) { + std::vector> test_shapes = { + {6}, {2, 3}, {2, 1, 3}, {1, 3, 1, 2}}; + for (int i = 0; i < test_shapes.size(); ++i) { + FloatSquaredDifferenceOpModel m( + {TensorType_FLOAT32, test_shapes[i]}, + {TensorType_FLOAT32, {}}, // always a scalar + {TensorType_FLOAT32, {}}); + m.PopulateTensor(m.input1(), {-0.2, 0.2, 0.5, 0.8, 0.11, 1.1}); + m.PopulateTensor(m.input2(), {0.1}); + m.Invoke(); + EXPECT_THAT( + m.GetOutput(), + ElementsAreArray(ArrayFloatNear({0.09, 0.01, 0.16, 0.49, 0.0001, 1.0}))) + << "With shape number " << i; + } +} + +TEST(IntegerSquaredDifferenceOpTest, IntegerType_SameShape) { + IntegerSquaredDifferenceOpModel m({TensorType_INT32, {1, 2, 2, 1}}, + {TensorType_INT32, {1, 2, 2, 1}}, + {TensorType_INT32, {}}); + m.PopulateTensor(m.input1(), {-2, 2, -15, 8}); + m.PopulateTensor(m.input2(), {5, -2, -3, 5}); + m.Invoke(); + EXPECT_THAT(m.GetOutput(), ElementsAreArray({49, 16, 144, 9})); +} + +TEST(IntegerSquaredDifferenceOpTest, IntegerType_VariousInputShapes) { + std::vector> test_shapes = { + {6}, {2, 3}, {2, 1, 3}, {1, 3, 1, 2}}; + for (int i = 0; i < test_shapes.size(); ++i) { + IntegerSquaredDifferenceOpModel m({TensorType_INT32, test_shapes[i]}, + {TensorType_INT32, test_shapes[i]}, + {TensorType_INT32, {}}); + m.PopulateTensor(m.input1(), {-20, 2, 3, 8, 11, -20}); + m.PopulateTensor(m.input2(), {1, 2, 6, 5, -5, -20}); + m.Invoke(); + EXPECT_THAT(m.GetOutput(), ElementsAreArray({441, 0, 9, 9, 256, 0})) + << "With shape number " << i; + } +} + +TEST(IntegerSquaredDifferenceOpTest, IntegerType_WithBroadcast) { + std::vector> test_shapes = { + {6}, {2, 3}, {2, 1, 3}, {1, 3, 1, 2}}; + for (int i = 0; i < test_shapes.size(); ++i) { + IntegerSquaredDifferenceOpModel m( + {TensorType_INT32, test_shapes[i]}, + {TensorType_INT32, {}}, // always a scalar + {TensorType_INT32, {}}); + m.PopulateTensor(m.input1(), {-20, 10, 7, 3, 1, 13}); + m.PopulateTensor(m.input2(), {3}); + m.Invoke(); + EXPECT_THAT(m.GetOutput(), ElementsAreArray({529, 49, 16, 0, 4, 100})) + << "With shape number " << i; + } +} + +} // namespace +} // namespace tflite + +int main(int argc, char** argv) { + ::tflite::LogToStderr(); + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/tensorflow/lite/kernels/test_util.h b/tensorflow/lite/kernels/test_util.h index 43a5137a941..dadabb86abb 100644 --- a/tensorflow/lite/kernels/test_util.h +++ b/tensorflow/lite/kernels/test_util.h @@ -199,7 +199,7 @@ class SingleOpModel { for (const string& s : content) { buf.AddString(s.data(), s.length()); } - buf.WriteToTensor(tensor); + buf.WriteToTensor(tensor, /*new_shape=*/nullptr); } // Populate the tensor given its index. @@ -307,6 +307,7 @@ class SingleOpModel { if (is_quantized) { if (t.min != 0 || t.max != 0) { + // TODO(b/119422369): Handle signed int8 here. if (t.type == TensorType_UINT8) { std::tie(t.scale, t.zero_point) = QuantizationParams(t.min, t.max); diff --git a/tensorflow/lite/lib_package/create_ios_frameworks.sh b/tensorflow/lite/lib_package/create_ios_frameworks.sh index fa466ed5bc7..7901655b7c6 100755 --- a/tensorflow/lite/lib_package/create_ios_frameworks.sh +++ b/tensorflow/lite/lib_package/create_ios_frameworks.sh @@ -30,7 +30,7 @@ echo "Creating target Headers directories" mkdir -p $FW_DIR_TFLITE_HDRS echo "Headers, populating: TensorFlow Lite" -cd $TFLITE_DIR/../../.. +cd $TFLITE_DIR/../.. find tensorflow/lite -name '*.h' \ -not -path 'tensorflow/lite/tools/*' \ @@ -51,10 +51,10 @@ cd $FW_DIR_TFLITE_HDRS tar xf tmp.tar rm -f tmp.tar -cd $TFLITE_DIR/../../.. +cd $TFLITE_DIR/../.. echo "Generate master LICENSE file and copy to target" bazel build //tensorflow/tools/lib_package:clicenses_generate -cp $TFLITE_DIR/../../../bazel-genfiles/tensorflow/tools/lib_package/include/tensorflow/c/LICENSE \ +cp $TFLITE_DIR/../../bazel-genfiles/tensorflow/tools/lib_package/include/tensorflow/c/LICENSE \ $FW_DIR_TFLITE echo "Copying static libraries" diff --git a/tensorflow/lite/models/smartreply/demo/app/src/main/java/com/example/android/smartreply/SmartReplyClient.java b/tensorflow/lite/models/smartreply/demo/app/src/main/java/com/example/android/smartreply/SmartReplyClient.java index d5b1ac0ffbc..fbd75051e71 100644 --- a/tensorflow/lite/models/smartreply/demo/app/src/main/java/com/example/android/smartreply/SmartReplyClient.java +++ b/tensorflow/lite/models/smartreply/demo/app/src/main/java/com/example/android/smartreply/SmartReplyClient.java @@ -90,29 +90,26 @@ public class SmartReplyClient implements AutoCloseable { } private MappedByteBuffer loadModelFile() throws IOException { - AssetFileDescriptor fileDescriptor = context.getAssets().openFd(MODEL_PATH); - FileInputStream inputStream = new FileInputStream(fileDescriptor.getFileDescriptor()); - try { + try (AssetFileDescriptor fileDescriptor = context.getAssets().openFd(MODEL_PATH); + FileInputStream inputStream = new FileInputStream(fileDescriptor.getFileDescriptor())) { FileChannel fileChannel = inputStream.getChannel(); long startOffset = fileDescriptor.getStartOffset(); long declaredLength = fileDescriptor.getDeclaredLength(); return fileChannel.map(FileChannel.MapMode.READ_ONLY, startOffset, declaredLength); - } finally { - inputStream.close(); } } private String[] loadBackoffList() throws IOException { List labelList = new ArrayList(); - BufferedReader reader = - new BufferedReader(new InputStreamReader(context.getAssets().open(BACKOFF_PATH))); - String line; - while ((line = reader.readLine()) != null) { - if (!line.isEmpty()) { - labelList.add(line); + try (BufferedReader reader = + new BufferedReader(new InputStreamReader(context.getAssets().open(BACKOFF_PATH)))) { + String line; + while ((line = reader.readLine()) != null) { + if (!line.isEmpty()) { + labelList.add(line); + } } } - reader.close(); String[] ans = new String[labelList.size()]; labelList.toArray(ans); return ans; diff --git a/tensorflow/lite/models/smartreply/ops/normalize.cc b/tensorflow/lite/models/smartreply/ops/normalize.cc index 8480260f279..3cb11cc055b 100644 --- a/tensorflow/lite/models/smartreply/ops/normalize.cc +++ b/tensorflow/lite/models/smartreply/ops/normalize.cc @@ -92,7 +92,7 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) { tflite::DynamicBuffer buf; buf.AddString(result.data(), result.length()); - buf.WriteToTensor(GetOutput(context, node, 0)); + buf.WriteToTensorAsVector(GetOutput(context, node, 0)); return kTfLiteOk; } diff --git a/tensorflow/lite/models/smartreply/predictor.cc b/tensorflow/lite/models/smartreply/predictor.cc index 7db25029777..59bf4a3cf1e 100644 --- a/tensorflow/lite/models/smartreply/predictor.cc +++ b/tensorflow/lite/models/smartreply/predictor.cc @@ -49,7 +49,7 @@ void ExecuteTfLite(const std::string& sentence, TfLiteTensor* input = interpreter->tensor(interpreter->inputs()[0]); tflite::DynamicBuffer buf; buf.AddString(sentence.data(), sentence.length()); - buf.WriteToTensor(input); + buf.WriteToTensorAsVector(input); interpreter->AllocateTensors(); interpreter->Invoke(); diff --git a/tensorflow/lite/nnapi_delegate.cc b/tensorflow/lite/nnapi_delegate.cc index 950bdb39425..26d75696a1c 100644 --- a/tensorflow/lite/nnapi_delegate.cc +++ b/tensorflow/lite/nnapi_delegate.cc @@ -140,13 +140,13 @@ NNAPIDelegate::~NNAPIDelegate() { // ANeuralNetworksShutdown(); } -// Adds the tensors of the interpreter to the NN API model. -TfLiteStatus addTensorOperands(tflite::Interpreter* interpreter, +// Adds the tensors of the subgraph to the NN API model. +TfLiteStatus addTensorOperands(tflite::Subgraph* subgraph, ANeuralNetworksModel* nn_model, uint32_t* no_of_operands_added, std::vector* nnapi_ids) { uint32_t next_id = 0; - for (size_t i = 0; i < interpreter->tensors_size(); i++) { + for (size_t i = 0; i < subgraph->tensors_size(); i++) { // Skip temporaries and RNN back-edges. if ((*nnapi_ids)[i] == kOperandNotNeeded) continue; @@ -156,7 +156,7 @@ TfLiteStatus addTensorOperands(tflite::Interpreter* interpreter, // NNAPI requires 32-bit float scale to be zero, tflite doesn't care float scale = 0.0f; int32_t zeroPoint = 0; - TfLiteTensor* tensor = interpreter->tensor(i); + TfLiteTensor* tensor = subgraph->tensor(i); switch (tensor->type) { case kTfLiteNoType: // Tensors added during initialization of Ops don't have a type yet and @@ -240,12 +240,12 @@ void MapAndAddTensorIds(const int* from_ids_buf, size_t from_ids_count, // Adds the operations and their parameters to the NN API model. // 'next-id' is the operand ID of the next operand of the model. TfLiteStatus AddOpsAndParams( - tflite::Interpreter* interpreter, ANeuralNetworksModel* nn_model, + tflite::Subgraph* subgraph, ANeuralNetworksModel* nn_model, uint32_t next_id, std::vector* model_state_inputs, std::vector* model_state_outputs, const std::vector& tensor_id_to_nnapi_id) { - for (size_t i = 0; i < interpreter->nodes_size(); i++) { - const auto* node_and_registration = interpreter->node_and_registration(i); + for (size_t i = 0; i < subgraph->nodes_size(); i++) { + const auto* node_and_registration = subgraph->node_and_registration(i); const TfLiteNode& node = node_and_registration->first; const TfLiteRegistration& registration = node_and_registration->second; tflite::BuiltinOperator builtin = @@ -291,9 +291,9 @@ TfLiteStatus AddOpsAndParams( // For each state_out tensor, a corresponding state_in operand needs to be // created for NNAPI. auto duplicate_state_tensor_float32 = - [interpreter, &nn_model, &next_id, &augmented_inputs, - &model_state_inputs, &model_state_outputs](int tensor_id) { - const TfLiteTensor* tensor = interpreter->tensor(tensor_id); + [subgraph, &nn_model, &next_id, &augmented_inputs, &model_state_inputs, + &model_state_outputs](int tensor_id) { + const TfLiteTensor* tensor = subgraph->tensor(tensor_id); ANeuralNetworksOperandType operand_type{ ANEURALNETWORKS_TENSOR_FLOAT32, static_cast(tensor->dims->size), @@ -388,11 +388,11 @@ TfLiteStatus AddOpsAndParams( }; // LSTM in NNAPI requires scratch tensor as an output operand. - auto add_lstm_scratch_tensor_float32 = [interpreter, &node, &nn_model, + auto add_lstm_scratch_tensor_float32 = [subgraph, &node, &nn_model, &next_id, &augmented_outputs]() { if (node.temporaries->size == 0) return; int scratch_buffer_index = node.temporaries->data[0]; - const TfLiteTensor* tensor = interpreter->tensor(scratch_buffer_index); + const TfLiteTensor* tensor = subgraph->tensor(scratch_buffer_index); ANeuralNetworksOperandType operand_type{ ANEURALNETWORKS_TENSOR_FLOAT32, static_cast(tensor->dims->size), @@ -584,7 +584,7 @@ TfLiteStatus AddOpsAndParams( // The permutation input tensor value dictates the output dimensions. // TODO(b/110888333): Support dynamically-sized tensors in delegates. if ((node.inputs->size > 1) && - (interpreter->tensor(node.inputs->data[1])->allocation_type != + (subgraph->tensor(node.inputs->data[1])->allocation_type != kTfLiteMmapRo)) { logError("NNAPI does not yet support dynamic tensors."); return kTfLiteError; @@ -601,14 +601,13 @@ TfLiteStatus AddOpsAndParams( return kTfLiteError; } if ((node.inputs->size > 0) && - (interpreter->tensor(node.inputs->data[0])->dims->size != 4)) { + (subgraph->tensor(node.inputs->data[0])->dims->size != 4)) { logError("NNAPI only supports input rank 4 for L2Normalization"); return kTfLiteError; } break; case tflite::BuiltinOperator_HASHTABLE_LOOKUP: - if (interpreter->tensor(node.outputs->data[0])->type != - kTfLiteFloat32) { + if (subgraph->tensor(node.outputs->data[0])->type != kTfLiteFloat32) { logError("NNAPI only support HASHTABLE_LOOKUP with float32 output", builtin); return kTfLiteError; @@ -682,6 +681,11 @@ TfLiteStatus AddOpsAndParams( case tflite::BuiltinOperator_FILL: case tflite::BuiltinOperator_FLOOR_MOD: case tflite::BuiltinOperator_RANGE: + case tflite::BuiltinOperator_LEAKY_RELU: + case tflite::BuiltinOperator_SQUARED_DIFFERENCE: + case tflite::BuiltinOperator_MIRROR_PAD: + case tflite::BuiltinOperator_ABS: + case tflite::BuiltinOperator_SPLIT_V: logError("Op code %d is currently not delegated to NNAPI", builtin); return kTfLiteError; break; @@ -706,7 +710,7 @@ TfLiteStatus AddOpsAndParams( return kTfLiteOk; } -TfLiteStatus NNAPIDelegate::BuildGraph(Interpreter* interpreter) { +TfLiteStatus NNAPIDelegate::BuildGraph(Subgraph* subgraph) { if (nn_model_ && nn_compiled_model_) return model_status_; // TODO(aselle): This is not correct. need to handle resize invalidation. @@ -718,7 +722,7 @@ TfLiteStatus NNAPIDelegate::BuildGraph(Interpreter* interpreter) { // inputs and outputs and mark the mapping in tensor_id_to_nnapi_id with // kOperandIdNotSet. addTensorOperands will replace those with the // corresponding NNAPI operand ids and skip kOperandNotNeeded entries. - std::vector tensor_id_to_nnapi_id(interpreter->tensors_size(), + std::vector tensor_id_to_nnapi_id(subgraph->tensors_size(), kOperandNotNeeded); auto set_ids_to_not_set = [&tensor_id_to_nnapi_id](const int* buf, size_t count) { @@ -729,35 +733,31 @@ TfLiteStatus NNAPIDelegate::BuildGraph(Interpreter* interpreter) { } } }; - for (size_t i = 0; i < interpreter->nodes_size(); i++) { - const auto* node_and_registration = interpreter->node_and_registration(i); + for (size_t i = 0; i < subgraph->nodes_size(); i++) { + const auto* node_and_registration = subgraph->node_and_registration(i); const TfLiteNode& node = node_and_registration->first; set_ids_to_not_set(node.inputs->data, node.inputs->size); set_ids_to_not_set(node.outputs->data, node.outputs->size); } - set_ids_to_not_set(interpreter->inputs().data(), - interpreter->inputs().size()); - set_ids_to_not_set(interpreter->outputs().data(), - interpreter->outputs().size()); + set_ids_to_not_set(subgraph->inputs().data(), subgraph->inputs().size()); + set_ids_to_not_set(subgraph->outputs().data(), subgraph->outputs().size()); uint32_t next_id = 0; RETURN_ERROR_IF_TFLITE_FAILED(addTensorOperands( - interpreter, nn_model_, &next_id, &tensor_id_to_nnapi_id)); + subgraph, nn_model_, &next_id, &tensor_id_to_nnapi_id)); RETURN_ERROR_IF_TFLITE_FAILED( - AddOpsAndParams(interpreter, nn_model_, next_id, &model_states_inputs_, + AddOpsAndParams(subgraph, nn_model_, next_id, &model_states_inputs_, &model_states_outputs_, tensor_id_to_nnapi_id)); std::vector augmented_inputs; - MapAndAddTensorIds(interpreter->inputs().data(), - interpreter->inputs().size(), &augmented_inputs, - tensor_id_to_nnapi_id); + MapAndAddTensorIds(subgraph->inputs().data(), subgraph->inputs().size(), + &augmented_inputs, tensor_id_to_nnapi_id); augmented_inputs.insert(augmented_inputs.end(), model_states_inputs_.begin(), model_states_inputs_.end()); std::vector augmented_outputs; - MapAndAddTensorIds(interpreter->outputs().data(), - interpreter->outputs().size(), &augmented_outputs, - tensor_id_to_nnapi_id); + MapAndAddTensorIds(subgraph->outputs().data(), subgraph->outputs().size(), + &augmented_outputs, tensor_id_to_nnapi_id); MapAndAddTensorIds(model_states_outputs_.data(), model_states_outputs_.size(), &augmented_outputs, tensor_id_to_nnapi_id); @@ -770,7 +770,7 @@ TfLiteStatus NNAPIDelegate::BuildGraph(Interpreter* interpreter) { if (GetAndroidSdkVersionCached() >= 28) { CHECK_NN(ANeuralNetworksModel_relaxComputationFloat32toFloat16( - nn_model_, interpreter->GetAllowFp16PrecisionForFp32())); + nn_model_, subgraph->GetAllowFp16PrecisionForFp32())); } CHECK_NN(ANeuralNetworksModel_finish(nn_model_)); } @@ -781,9 +781,9 @@ TfLiteStatus NNAPIDelegate::BuildGraph(Interpreter* interpreter) { return kTfLiteOk; } -TfLiteStatus NNAPIDelegate::Invoke(Interpreter* interpreter) { +TfLiteStatus NNAPIDelegate::Invoke(Subgraph* subgraph) { if (!nn_model_) { - model_status_ = BuildGraph(interpreter); + model_status_ = BuildGraph(subgraph); if (model_status_ != kTfLiteOk) { logError("Failed to build graph for NNAPI"); } @@ -796,19 +796,19 @@ TfLiteStatus NNAPIDelegate::Invoke(Interpreter* interpreter) { CHECK_NN(ANeuralNetworksExecution_create(nn_compiled_model_, &execution)); // Currently perform deep copy of input buffer - for (size_t i = 0; i < interpreter->inputs().size(); i++) { - int input = interpreter->inputs()[i]; + for (size_t i = 0; i < subgraph->inputs().size(); i++) { + int input = subgraph->inputs()[i]; // TODO(aselle): Is this what we want or do we want input instead? // TODO(aselle): This should be called setInputValue maybe to be cons. - TfLiteTensor* tensor = interpreter->tensor(input); + TfLiteTensor* tensor = subgraph->tensor(input); CHECK_NN(ANeuralNetworksExecution_setInput( execution, i, nullptr, tensor->data.raw, tensor->bytes)); } // Tell nn api where to place final data. - for (size_t i = 0; i < interpreter->outputs().size(); i++) { - int output = interpreter->outputs()[i]; - TfLiteTensor* tensor = interpreter->tensor(output); + for (size_t i = 0; i < subgraph->outputs().size(); i++) { + int output = subgraph->outputs()[i]; + TfLiteTensor* tensor = subgraph->tensor(output); CHECK_NN(ANeuralNetworksExecution_setOutput( execution, i, nullptr, tensor->data.raw, tensor->bytes)); } @@ -817,16 +817,16 @@ TfLiteStatus NNAPIDelegate::Invoke(Interpreter* interpreter) { // current invocation. for (size_t i = 0; i < model_states_outputs_.size(); i++) { int state_tensor_idx = model_states_outputs_[i]; - TfLiteTensor* tensor = interpreter->tensor(state_tensor_idx); + TfLiteTensor* tensor = subgraph->tensor(state_tensor_idx); // Here we are using a deep copy for state_in tensors so that we are not // reading and writing into the same buffer during a invocation. // TODO(miaowang): using double shared buffer to minimize the copies. CHECK_NN(ANeuralNetworksExecution_setInput( - execution, i + interpreter->inputs().size(), nullptr, tensor->data.raw, + execution, i + subgraph->inputs().size(), nullptr, tensor->data.raw, tensor->bytes)); // Tell NNAPI where to output the state_out. CHECK_NN(ANeuralNetworksExecution_setOutput( - execution, i + interpreter->outputs().size(), nullptr, tensor->data.raw, + execution, i + subgraph->outputs().size(), nullptr, tensor->data.raw, tensor->bytes)); } @@ -839,9 +839,9 @@ TfLiteStatus NNAPIDelegate::Invoke(Interpreter* interpreter) { #if 0 printf("From the NN API:\n"); - TfLiteTensor* tensor = interpreter->tensor(interpreter->outputs()[0]); + TfLiteTensor* tensor = subgraph->tensor(subgraph->outputs()[0]); if (float* data = - interpreter->typed_tensor(interpreter->outputs()[0])) { + subgraph->typed_tensor(subgraph->outputs()[0])) { size_t num = tensor->bytes / sizeof(float); for (float* p = data; p < data + num; p++) { printf(" %f", *p); diff --git a/tensorflow/lite/nnapi_delegate.h b/tensorflow/lite/nnapi_delegate.h index 63b408c1416..b4f8e4ecf39 100644 --- a/tensorflow/lite/nnapi_delegate.h +++ b/tensorflow/lite/nnapi_delegate.h @@ -18,6 +18,7 @@ limitations under the License. #include "tensorflow/lite/allocation.h" #include "tensorflow/lite/c/c_api_internal.h" #include "tensorflow/lite/core/api/error_reporter.h" +#include "tensorflow/lite/core/subgraph.h" #include "tensorflow/lite/interpreter.h" class ANeuralNetworksModel; @@ -50,10 +51,10 @@ class NNAPIDelegate { ~NNAPIDelegate(); // Convert a tflite graph to NNAPI - TfLiteStatus BuildGraph(Interpreter* interpreter); + TfLiteStatus BuildGraph(Subgraph* subgraph); // Run - TfLiteStatus Invoke(Interpreter* interpreter); + TfLiteStatus Invoke(Subgraph* subgraph); // Whether the current platform supports NNAPI delegation. static bool IsSupported(); diff --git a/tensorflow/lite/nnapi_delegate_disabled.cc b/tensorflow/lite/nnapi_delegate_disabled.cc index 44dc21f1b6c..a8f2c0bfe38 100644 --- a/tensorflow/lite/nnapi_delegate_disabled.cc +++ b/tensorflow/lite/nnapi_delegate_disabled.cc @@ -35,13 +35,11 @@ NNAPIDelegate::~NNAPIDelegate() { #undef UNUSED_MEMBER } -TfLiteStatus NNAPIDelegate::BuildGraph(Interpreter* interpreter) { +TfLiteStatus NNAPIDelegate::BuildGraph(Subgraph* subgraph) { return kTfLiteError; } -TfLiteStatus NNAPIDelegate::Invoke(Interpreter* interpreter) { - return kTfLiteError; -} +TfLiteStatus NNAPIDelegate::Invoke(Subgraph* subgraph) { return kTfLiteError; } bool NNAPIDelegate::IsSupported() { return false; } diff --git a/tensorflow/lite/optional_debug_tools.cc b/tensorflow/lite/optional_debug_tools.cc index 5ee1cf6d33d..1113bf01b17 100644 --- a/tensorflow/lite/optional_debug_tools.cc +++ b/tensorflow/lite/optional_debug_tools.cc @@ -44,6 +44,8 @@ const char* TensorTypeName(TfLiteType type) { return "kTfLiteInt32"; case kTfLiteUInt8: return "kTfLiteUInt8"; + case kTfLiteInt8: + return "kTfLiteInt8"; case kTfLiteInt64: return "kTfLiteInt64"; case kTfLiteString: diff --git a/tensorflow/lite/python/BUILD b/tensorflow/lite/python/BUILD index 017dd72f781..acf827892bf 100644 --- a/tensorflow/lite/python/BUILD +++ b/tensorflow/lite/python/BUILD @@ -89,6 +89,7 @@ py_library( srcs_version = "PY2AND3", deps = [ "//tensorflow/lite/toco:toco_flags_proto_py", + "//tensorflow/python:dtypes", ], ) @@ -103,6 +104,7 @@ py_library( "//tensorflow/lite/toco:toco_flags_proto_py", "//tensorflow/lite/toco/python:tensorflow_wrap_toco", "//tensorflow/lite/toco/python:toco_from_protos", + "//tensorflow/python:dtypes", "//tensorflow/python:platform", ], ) diff --git a/tensorflow/lite/python/convert.py b/tensorflow/lite/python/convert.py index 9991fb2a733..563312e0278 100644 --- a/tensorflow/lite/python/convert.py +++ b/tensorflow/lite/python/convert.py @@ -28,6 +28,8 @@ import tempfile as _tempfile from tensorflow.lite.python import lite_constants from tensorflow.lite.toco import model_flags_pb2 as _model_flags_pb2 from tensorflow.lite.toco import toco_flags_pb2 as _toco_flags_pb2 +from tensorflow.lite.toco import types_pb2 as _types_pb2 +from tensorflow.python.framework import dtypes from tensorflow.python.platform import resource_loader as _resource_loader from tensorflow.python.util import deprecation from tensorflow.python.util.lazy_loader import LazyLoader @@ -53,6 +55,18 @@ else: if _toco_from_proto_bin and not _os.path.exists(_toco_from_proto_bin): _toco_from_proto_bin = "toco_from_protos" + +# Map of tf.dtypes to TFLite types_flag_pb2. +_MAP_TF_TO_TFLITE_TYPES = { + dtypes.float32: _types_pb2.FLOAT, + dtypes.int32: _types_pb2.INT32, + dtypes.int64: _types_pb2.INT64, + dtypes.string: _types_pb2.STRING, + dtypes.uint8: _types_pb2.QUANTIZED_UINT8, + dtypes.complex64: _types_pb2.COMPLEX64 +} + + def _try_convert_to_unicode(output): if output is None: return u"" @@ -65,6 +79,24 @@ def _try_convert_to_unicode(output): return output +def convert_dtype_to_tflite_type(tf_dtype): + """Converts tf.dtype to TFLite proto type. + + Args: + tf_dtype: tf.dtype + + Raises: + ValueError: Unsupported tf.dtype. + + Returns: + types_flag_pb2. + """ + result = _MAP_TF_TO_TFLITE_TYPES.get(tf_dtype) + if result is None: + raise ValueError("Unsupported tf.dtype {0}".format(tf_dtype)) + return result + + class OpsSet(enum.Enum): """Enum class defining the sets of ops available to generate TFLite models. @@ -214,10 +246,10 @@ def build_toco_convert_protos(input_tensors, `foo.get_shape()` and `foo.dtype`. output_tensors: List of output tensors (only .name is used from this). inference_type: Target data type of real-number arrays in the output file. - Must be `{FLOAT, QUANTIZED_UINT8}`. (default FLOAT) + Must be `{tf.float32, tf.uint8}`. (default tf.float32) inference_input_type: Target data type of real-number input arrays. Allows for a different type for input arrays in the case of quantization. - Must be `{FLOAT, QUANTIZED_UINT8}`. (default `inference_type`) + Must be `{tf.float32, tf.uint8}`. (default `inference_type`) input_format: Type of data to read Currently must be `{TENSORFLOW_GRAPHDEF}`. (default TENSORFLOW_GRAPHDEF) input_shapes: Input array shape. It needs to be a list of the same length @@ -269,16 +301,19 @@ def build_toco_convert_protos(input_tensors, process. Raises: - ValueError: If the input tensor type is unknown + ValueError: + If the input tensor type is unknown + Missing mean_values or std_dev_values RuntimeError: If TOCO fails to convert (in which case the runtime error's error text will contain the TOCO error log) """ toco = _toco_flags_pb2.TocoFlags() toco.input_format = input_format toco.output_format = output_format - toco.inference_type = inference_type + toco.inference_type = convert_dtype_to_tflite_type(inference_type) if inference_input_type: - toco.inference_input_type = inference_input_type + toco.inference_input_type = convert_dtype_to_tflite_type( + inference_input_type) else: toco.inference_input_type = toco.inference_type toco.drop_control_dependency = drop_control_dependency @@ -302,9 +337,14 @@ def build_toco_convert_protos(input_tensors, model.change_concat_input_ranges = change_concat_input_ranges for idx, input_tensor in enumerate(input_tensors): input_array = model.input_arrays.add() - if toco.inference_input_type == lite_constants.QUANTIZED_UINT8: - input_array.mean_value, input_array.std_value = quantized_input_stats[idx] input_array.name = tensor_name(input_tensor) + input_array.data_type = convert_dtype_to_tflite_type(input_tensor.dtype) + + if toco.inference_input_type == _types_pb2.QUANTIZED_UINT8: + if not quantized_input_stats: + raise ValueError("std_dev and mean must be defined when " + "inference_input_type is QUANTIZED_UINT8.") + input_array.mean_value, input_array.std_value = quantized_input_stats[idx] if input_shapes is None: shape = input_tensor.get_shape() else: @@ -352,7 +392,11 @@ def toco_convert_graph_def(input_data, input_arrays_with_shape, output_arrays, for idx, (name, shape) in enumerate(input_arrays_with_shape): input_array = model_flags.input_arrays.add() - if kwargs["inference_type"] == lite_constants.QUANTIZED_UINT8: + if toco_flags.inference_input_type == _types_pb2.QUANTIZED_UINT8: + if (("quantized_input_stats" not in kwargs) or + (not kwargs["quantized_input_stats"])): + raise ValueError("std_dev and mean must be defined when " + "inference_input_type is QUANTIZED_UINT8.") input_array.mean_value, input_array.std_value = kwargs[ "quantized_input_stats"][idx] input_array.name = name diff --git a/tensorflow/lite/python/convert_saved_model.py b/tensorflow/lite/python/convert_saved_model.py index 3f54d2559c4..f8d986b7469 100644 --- a/tensorflow/lite/python/convert_saved_model.py +++ b/tensorflow/lite/python/convert_saved_model.py @@ -197,12 +197,27 @@ def set_tensor_shapes(tensors, shapes): tensors: TensorFlow ops.Tensor. shapes: Dict of strings representing input tensor names to list of integers representing input shapes (e.g., {"foo": : [1, 16, 16, 3]}). + + Raises: + ValueError: + `shapes` contains an invalid tensor. + `shapes` contains an invalid shape for a valid tensor. """ if shapes: - for tensor in tensors: - shape = shapes.get(tensor_name(tensor)) + tensor_names_to_tensor = {tensor_name(tensor): tensor for tensor in tensors} + for name, shape in shapes.items(): + if name not in tensor_names_to_tensor: + raise ValueError("Invalid tensor \'{}\' found in tensor shapes " + "map.".format(name)) if shape is not None: - tensor.set_shape(shape) + tensor = tensor_names_to_tensor[name] + try: + tensor.set_shape(shape) + except ValueError as error: + message = ("The shape of tensor '{0}' cannot be changed from {1} to " + "{2}. {3}".format(name, tensor.get_shape(), shape, + str(error))) + raise ValueError(message) def freeze_saved_model(saved_model_dir, input_arrays, input_shapes, diff --git a/tensorflow/lite/python/convert_saved_model_test.py b/tensorflow/lite/python/convert_saved_model_test.py index dff582f1a16..76113853ca9 100644 --- a/tensorflow/lite/python/convert_saved_model_test.py +++ b/tensorflow/lite/python/convert_saved_model_test.py @@ -75,12 +75,30 @@ class TensorFunctionsTest(test_util.TensorFlowTestCase): convert_saved_model.set_tensor_shapes([tensor], {"Placeholder": [1, 3, 5]}) self.assertEqual([1, 3, 5], tensor.shape.as_list()) - def testSetTensorShapeInvalid(self): + def testSetTensorShapeArrayInvalid(self): + # Tests set_tensor_shape where the tensor name passed in doesn't exist. tensor = array_ops.placeholder(shape=[None, 3, 5], dtype=dtypes.float32) self.assertEqual([None, 3, 5], tensor.shape.as_list()) - convert_saved_model.set_tensor_shapes([tensor], - {"invalid-input": [5, 3, 5]}) + with self.assertRaises(ValueError) as error: + convert_saved_model.set_tensor_shapes([tensor], + {"invalid-input": [5, 3, 5]}) + self.assertEqual( + "Invalid tensor 'invalid-input' found in tensor shapes map.", + str(error.exception)) + self.assertEqual([None, 3, 5], tensor.shape.as_list()) + + def testSetTensorShapeDimensionInvalid(self): + # Tests set_tensor_shape where the shape passed in is incompatiable. + tensor = array_ops.placeholder(shape=[None, 3, 5], dtype=dtypes.float32) + self.assertEqual([None, 3, 5], tensor.shape.as_list()) + + with self.assertRaises(ValueError) as error: + convert_saved_model.set_tensor_shapes([tensor], + {"Placeholder": [1, 5, 5]}) + self.assertIn( + "The shape of tensor 'Placeholder' cannot be changed from " + "(?, 3, 5) to [1, 5, 5].", str(error.exception)) self.assertEqual([None, 3, 5], tensor.shape.as_list()) def testSetTensorShapeEmpty(self): diff --git a/tensorflow/lite/python/convert_test.py b/tensorflow/lite/python/convert_test.py index 7a0bce921b5..2a6f1f634f8 100644 --- a/tensorflow/lite/python/convert_test.py +++ b/tensorflow/lite/python/convert_test.py @@ -23,6 +23,7 @@ from tensorflow.lite.python import convert from tensorflow.lite.python import lite_constants from tensorflow.lite.python import op_hint from tensorflow.lite.python.interpreter import Interpreter +from tensorflow.lite.toco import types_pb2 as _types_pb2 from tensorflow.python.client import session from tensorflow.python.framework import dtypes from tensorflow.python.framework import test_util @@ -65,6 +66,21 @@ class ConvertTest(test_util.TensorFlowTestCase): quantized_input_stats=[(0., 1.)]) self.assertTrue(tflite_model) + def testQuantizationInvalid(self): + in_tensor = array_ops.placeholder( + shape=[1, 16, 16, 3], dtype=dtypes.float32) + out_tensor = array_ops.fake_quant_with_min_max_args( + in_tensor + in_tensor, min=0., max=1.) + sess = session.Session() + + with self.assertRaises(ValueError) as error: + convert.toco_convert( + sess.graph_def, [in_tensor], [out_tensor], + inference_type=lite_constants.QUANTIZED_UINT8) + self.assertEqual( + "std_dev and mean must be defined when inference_input_type is " + "QUANTIZED_UINT8.", str(error.exception)) + def testGraphDefBasic(self): in_tensor = array_ops.placeholder( shape=[1, 16, 16, 3], dtype=dtypes.float32, name="input") @@ -138,6 +154,27 @@ class ConvertTest(test_util.TensorFlowTestCase): self.assertTrue(([1, 16, 16, 3] == output_details[0]["shape"]).all()) self.assertTrue(output_details[0]["quantization"][0] > 0) # scale + def testGraphDefQuantizationInvalid(self): + in_tensor_1 = array_ops.placeholder( + shape=[1, 16, 16, 3], dtype=dtypes.float32, name="inputA") + in_tensor_2 = array_ops.placeholder( + shape=[1, 16, 16, 3], dtype=dtypes.float32, name="inputB") + _ = array_ops.fake_quant_with_min_max_args( + in_tensor_1 + in_tensor_2, min=0., max=1., name="output") + sess = session.Session() + + input_arrays_map = [("inputA", [1, 16, 16, 3]), ("inputB", [1, 16, 16, 3])] + output_arrays = ["output"] + with self.assertRaises(ValueError) as error: + convert.toco_convert_graph_def( + sess.graph_def, + input_arrays_map, + output_arrays, + inference_type=lite_constants.QUANTIZED_UINT8) + self.assertEqual( + "std_dev and mean must be defined when inference_input_type is " + "QUANTIZED_UINT8.", str(error.exception)) + class ConvertTestOpHint(test_util.TensorFlowTestCase): """Test the hint to stub functionality.""" @@ -329,6 +366,27 @@ class ConvertTestOpHint(test_util.TensorFlowTestCase): output_nodes=[op_hint._tensor_name_base(output.name)]), set(["agg", "Const", "Identity"])) + def testConvertDtype(self): + self.assertEqual( + convert.convert_dtype_to_tflite_type(lite_constants.FLOAT), + _types_pb2.FLOAT) + self.assertEqual( + convert.convert_dtype_to_tflite_type(dtypes.float32), _types_pb2.FLOAT) + self.assertEqual( + convert.convert_dtype_to_tflite_type(dtypes.int32), _types_pb2.INT32) + self.assertEqual( + convert.convert_dtype_to_tflite_type(dtypes.int64), _types_pb2.INT64) + self.assertEqual( + convert.convert_dtype_to_tflite_type(dtypes.string), _types_pb2.STRING) + self.assertEqual( + convert.convert_dtype_to_tflite_type(dtypes.uint8), + _types_pb2.QUANTIZED_UINT8) + self.assertEqual( + convert.convert_dtype_to_tflite_type(dtypes.complex64), + _types_pb2.COMPLEX64) + with self.assertRaises(ValueError): + convert.convert_dtype_to_tflite_type(dtypes.bool) + if __name__ == "__main__": test.main() diff --git a/tensorflow/lite/python/interpreter_wrapper/interpreter_wrapper.cc b/tensorflow/lite/python/interpreter_wrapper/interpreter_wrapper.cc index e71752fe631..d14af439ec0 100644 --- a/tensorflow/lite/python/interpreter_wrapper/interpreter_wrapper.cc +++ b/tensorflow/lite/python/interpreter_wrapper/interpreter_wrapper.cc @@ -124,6 +124,8 @@ int TfLiteTypeToPyArrayType(TfLiteType tf_lite_type) { return NPY_INT16; case kTfLiteUInt8: return NPY_UINT8; + case kTfLiteInt8: + return NPY_INT8; case kTfLiteInt64: return NPY_INT64; case kTfLiteString: @@ -150,6 +152,8 @@ TfLiteType TfLiteTypeFromPyArray(PyArrayObject* array) { return kTfLiteInt16; case NPY_UINT8: return kTfLiteUInt8; + case NPY_INT8: + return kTfLiteInt8; case NPY_INT64: return kTfLiteInt64; case NPY_BOOL: diff --git a/tensorflow/lite/python/lite.py b/tensorflow/lite/python/lite.py index 5810553da2c..1b20ff2f92b 100644 --- a/tensorflow/lite/python/lite.py +++ b/tensorflow/lite/python/lite.py @@ -25,8 +25,6 @@ EXPERIMENTAL: APIs here are unstable and likely to change without notice. @@convert_op_hints_to_stubs @@build_toco_convert_protos -@@FLOAT -@@QUANTIZED_UINT8 @@TFLITE @@GRAPHVIZ_DOT @@ -78,10 +76,10 @@ class TFLiteConverter(object): Attributes: inference_type: Target data type of real-number arrays in the output file. - Must be `{FLOAT, QUANTIZED_UINT8}`. (default FLOAT) + Must be `{tf.float32, tf.uint8}`. (default tf.float32) inference_input_type: Target data type of real-number input arrays. Allows for a different type for input arrays in the case of quantization. - Must be `{FLOAT, QUANTIZED_UINT8}`. (default `inference_type`) + Must be `{tf.float32, tf.uint8}`. (default `inference_type`) output_format: Output file format. Currently must be `{TFLITE, GRAPHVIZ_DOT}`. (default TFLITE) quantized_input_stats: Dict of strings representing input tensor names @@ -402,15 +400,16 @@ class TFLiteConverter(object): # Checks dimensions in input tensor. if self._has_valid_tensors(): for tensor in self._input_tensors: - if not tensor.get_shape(): + shape = tensor.get_shape() + if not shape or not shape.as_list(): raise ValueError("Provide an input shape for input array " "'{0}'.".format(_tensor_name(tensor))) - shape = tensor.get_shape().as_list() - if None in shape[1:]: + shape_list = shape.as_list() + if None in shape_list[1:]: raise ValueError( "None is only supported in the 1st dimension. Tensor '{0}' has " - "invalid shape '{1}'.".format(_tensor_name(tensor), shape)) - elif shape[0] is None: + "invalid shape '{1}'.".format(_tensor_name(tensor), shape_list)) + elif shape_list[0] is None: self._set_batch_size(batch_size=1) # Get quantization stats. Ensures there is one stat per name if the stats diff --git a/tensorflow/lite/python/lite_constants.py b/tensorflow/lite/python/lite_constants.py index fdefc5e6cf0..f5d6d103795 100644 --- a/tensorflow/lite/python/lite_constants.py +++ b/tensorflow/lite/python/lite_constants.py @@ -19,26 +19,25 @@ from __future__ import division from __future__ import print_function from tensorflow.lite.toco import toco_flags_pb2 as _toco_flags_pb2 -from tensorflow.lite.toco import types_pb2 as _types_pb2 +from tensorflow.python.framework import dtypes from tensorflow.python.util.all_util import remove_undocumented from tensorflow.python.util.tf_export import tf_export as _tf_export -# Enum types from the protobuf promoted to the API -FLOAT = _types_pb2.FLOAT -INT32 = _types_pb2.INT32 -INT64 = _types_pb2.INT64 -STRING = _types_pb2.STRING -QUANTIZED_UINT8 = _types_pb2.QUANTIZED_UINT8 -COMPLEX64 = _types_pb2.COMPLEX64 +FLOAT = dtypes.float32 +INT32 = dtypes.int32 +INT64 = dtypes.int64 +STRING = dtypes.string +QUANTIZED_UINT8 = dtypes.uint8 +COMPLEX64 = dtypes.complex64 TENSORFLOW_GRAPHDEF = _toco_flags_pb2.TENSORFLOW_GRAPHDEF TFLITE = _toco_flags_pb2.TFLITE GRAPHVIZ_DOT = _toco_flags_pb2.GRAPHVIZ_DOT -_tf_export("lite.constants.FLOAT").export_constant(__name__, "FLOAT") -_tf_export("lite.constants.INT32").export_constant(__name__, "INT32") -_tf_export("lite.constants.INT64").export_constant(__name__, "INT64") -_tf_export("lite.constants.STRING").export_constant(__name__, "STRING") -_tf_export("lite.constants.QUANTIZED_UINT8").export_constant( +_tf_export(v1=["lite.constants.FLOAT"]).export_constant(__name__, "FLOAT") +_tf_export(v1=["lite.constants.INT32"]).export_constant(__name__, "INT32") +_tf_export(v1=["lite.constants.INT64"]).export_constant(__name__, "INT64") +_tf_export(v1=["lite.constants.STRING"]).export_constant(__name__, "STRING") +_tf_export(v1=["lite.constants.QUANTIZED_UINT8"]).export_constant( __name__, "QUANTIZED_UINT8") _tf_export("lite.constants.TFLITE").export_constant(__name__, "TFLITE") _tf_export("lite.constants.GRAPHVIZ_DOT").export_constant( diff --git a/tensorflow/lite/python/lite_test.py b/tensorflow/lite/python/lite_test.py index 5a5697db92b..1ae0d3c3ed0 100644 --- a/tensorflow/lite/python/lite_test.py +++ b/tensorflow/lite/python/lite_test.py @@ -182,7 +182,7 @@ class FromSessionTest(test_util.TensorFlowTestCase): out_tensor = in_tensor + in_tensor sess = session.Session() - # Test invalid shape. None after 1st dimension. + # Test None as shape. converter = lite.TFLiteConverter.from_session(sess, [in_tensor], [out_tensor]) with self.assertRaises(ValueError) as error: @@ -190,7 +190,20 @@ class FromSessionTest(test_util.TensorFlowTestCase): self.assertEqual('Provide an input shape for input array \'Placeholder\'.', str(error.exception)) - def testBatchSizeInvalid(self): + def testSizeEmptyInvalid(self): + in_tensor = array_ops.placeholder(dtype=dtypes.float32, shape=[]) + out_tensor = in_tensor + in_tensor + sess = session.Session() + + # Test empty shape. + converter = lite.TFLiteConverter.from_session(sess, [in_tensor], + [out_tensor]) + with self.assertRaises(ValueError) as error: + converter.convert() + self.assertEqual('Provide an input shape for input array \'Placeholder\'.', + str(error.exception)) + + def testSizeInvalid(self): in_tensor = array_ops.placeholder( shape=[1, None, 16, 3], dtype=dtypes.float32) out_tensor = in_tensor + in_tensor @@ -931,12 +944,13 @@ class FromKerasFile(test_util.TensorFlowTestCase): """Test a Sequential tf.keras model testing input shapes argument.""" keras_file = self._getSequentialModel() - # Passing in shape of invalid input array has no impact as long as all input - # arrays have a shape. - converter = lite.TFLiteConverter.from_keras_model_file( - keras_file, input_shapes={'invalid-input': [2, 3]}) - tflite_model = converter.convert() - self.assertTrue(tflite_model) + # Passing in shape of invalid input array raises error. + with self.assertRaises(ValueError) as error: + converter = lite.TFLiteConverter.from_keras_model_file( + keras_file, input_shapes={'invalid-input': [2, 3]}) + self.assertEqual( + "Invalid tensor 'invalid-input' found in tensor shapes map.", + str(error.exception)) # Passing in shape of valid input array. converter = lite.TFLiteConverter.from_keras_model_file( diff --git a/tensorflow/lite/python/op_hint.py b/tensorflow/lite/python/op_hint.py index 3afce1baf2e..8d7f9316bfe 100644 --- a/tensorflow/lite/python/op_hint.py +++ b/tensorflow/lite/python/op_hint.py @@ -104,9 +104,9 @@ class OpHint(object): that make up the pseudo op. A similar process is done to any output that is to be exported from the current op. - TODO(aselle): When TensorFlow functions functionality works for arbitrary - constructs, this mechanism can be retired and changed to use python defun's. """ + # TODO(aselle): When TensorFlow functions functionality works for arbitrary + # constructs, this mechanism can be retired and changed to use python defun's. # Attr constants that are used for representation in the GraphDef. These # will be used on every Identity op that is involved in a total OpHint. @@ -403,7 +403,7 @@ class _LiteOperand(object): out_graphdef: A graphdef that is ready to have this input added. Returns: - The the output that the stub should use as an input for this operand. + The output that the stub should use as an input for this operand. Raises: RuntimeError: if the method is not implemented. diff --git a/tensorflow/lite/python/tflite_convert.py b/tensorflow/lite/python/tflite_convert.py index 00ea6d722e2..341b539bead 100644 --- a/tensorflow/lite/python/tflite_convert.py +++ b/tensorflow/lite/python/tflite_convert.py @@ -25,7 +25,6 @@ import sys from tensorflow.lite.python import lite from tensorflow.lite.python import lite_constants from tensorflow.lite.toco import toco_flags_pb2 as _toco_flags_pb2 -from tensorflow.lite.toco import types_pb2 as _types_pb2 from tensorflow.python.platform import app @@ -41,6 +40,27 @@ def _parse_set(values): return None +def _parse_inference_type(value, flag): + """Converts the inference type to the value of the constant. + + Args: + value: str representing the inference type. + flag: str representing the flag name. + + Returns: + tf.dtype. + + Raises: + ValueError: Unsupported value. + """ + if value == "FLOAT": + return lite_constants.FLOAT + if value == "QUANTIZED_UINT8": + return lite_constants.QUANTIZED_UINT8 + raise ValueError("Unsupported value for --{0}. Only FLOAT and " + "QUANTIZED_UINT8 are supported.".format(flag)) + + def _get_toco_converter(flags): """Makes a TFLiteConverter object based on the flags provided. @@ -101,10 +121,11 @@ def _convert_model(flags): # Create converter. converter = _get_toco_converter(flags) if flags.inference_type: - converter.inference_type = _types_pb2.IODataType.Value(flags.inference_type) + converter.inference_type = _parse_inference_type(flags.inference_type, + "inference_type") if flags.inference_input_type: - converter.inference_input_type = _types_pb2.IODataType.Value( - flags.inference_input_type) + converter.inference_input_type = _parse_inference_type( + flags.inference_input_type, "inference_input_type") if flags.output_format: converter.output_format = _toco_flags_pb2.FileFormat.Value( flags.output_format) @@ -115,7 +136,7 @@ def _convert_model(flags): # In quantized inference, mean_value has to be integer so that the real # value 0.0 is exactly representable. - if flags.inference_type == lite_constants.QUANTIZED_UINT8: + if converter.inference_type == lite_constants.QUANTIZED_UINT8: mean_values = _parse_array(flags.mean_values, type_fn=int) else: mean_values = _parse_array(flags.mean_values, type_fn=float) @@ -156,7 +177,7 @@ def _convert_model(flags): if flags.post_training_quantize: converter.post_training_quantize = flags.post_training_quantize - if flags.inference_type == lite_constants.QUANTIZED_UINT8: + if converter.inference_type == lite_constants.QUANTIZED_UINT8: print("--post_training_quantize quantizes a graph of inference_type " "FLOAT. Overriding inference type QUANTIZED_UINT8 to FLOAT.") converter.inference_type = lite_constants.FLOAT diff --git a/tensorflow/lite/schema/schema.fbs b/tensorflow/lite/schema/schema.fbs index 9b0eae74c3b..6436167303b 100644 --- a/tensorflow/lite/schema/schema.fbs +++ b/tensorflow/lite/schema/schema.fbs @@ -200,6 +200,11 @@ enum BuiltinOperator : byte { FLOOR_MOD = 95, RANGE = 96, RESIZE_NEAREST_NEIGHBOR = 97, + LEAKY_RELU = 98, + SQUARED_DIFFERENCE = 99, + MIRROR_PAD = 100, + ABS = 101, + SPLIT_V = 102, } // Options for the builtin operators. @@ -278,6 +283,11 @@ union BuiltinOptions { FloorModOptions, RangeOptions, ResizeNearestNeighborOptions, + LeakyReluOptions, + SquaredDifferenceOptions, + MirrorPadOptions, + AbsOptions, + SplitVOptions, } enum Padding : byte { SAME, VALID } @@ -526,6 +536,10 @@ table SplitOptions { num_splits: int; } +table SplitVOptions { + num_splits: int; +} + table StridedSliceOptions { begin_mask: int; end_mask: int; @@ -629,6 +643,10 @@ table OneHotOptions { axis:int; } +table AbsOptions { +} + + table LogicalAndOptions { } @@ -658,6 +676,24 @@ table FloorModOptions { table RangeOptions { } +table LeakyReluOptions { + alpha:float; +} + +table SquaredDifferenceOptions { +} + +enum MirrorPadMode : byte { + // Doesn't include borders. + REFLECT = 0, + // Includes borders. + SYMMETRIC = 1, +} + +table MirrorPadOptions { + mode:MirrorPadMode; +} + // An OperatorCode can be an enum value (BuiltinOperator) if the operator is a // builtin, or a string if the operator is custom. table OperatorCode { diff --git a/tensorflow/lite/schema/schema_generated.h b/tensorflow/lite/schema/schema_generated.h index b7885cfcc50..af8b143364e 100755 --- a/tensorflow/lite/schema/schema_generated.h +++ b/tensorflow/lite/schema/schema_generated.h @@ -148,6 +148,9 @@ struct SqueezeOptionsT; struct SplitOptions; struct SplitOptionsT; +struct SplitVOptions; +struct SplitVOptionsT; + struct StridedSliceOptions; struct StridedSliceOptionsT; @@ -226,6 +229,9 @@ struct LogicalOrOptionsT; struct OneHotOptions; struct OneHotOptionsT; +struct AbsOptions; +struct AbsOptionsT; + struct LogicalAndOptions; struct LogicalAndOptionsT; @@ -253,6 +259,15 @@ struct FloorModOptionsT; struct RangeOptions; struct RangeOptionsT; +struct LeakyReluOptions; +struct LeakyReluOptionsT; + +struct SquaredDifferenceOptions; +struct SquaredDifferenceOptionsT; + +struct MirrorPadOptions; +struct MirrorPadOptionsT; + struct OperatorCode; struct OperatorCodeT; @@ -500,11 +515,16 @@ enum BuiltinOperator { BuiltinOperator_FLOOR_MOD = 95, BuiltinOperator_RANGE = 96, BuiltinOperator_RESIZE_NEAREST_NEIGHBOR = 97, + BuiltinOperator_LEAKY_RELU = 98, + BuiltinOperator_SQUARED_DIFFERENCE = 99, + BuiltinOperator_MIRROR_PAD = 100, + BuiltinOperator_ABS = 101, + BuiltinOperator_SPLIT_V = 102, BuiltinOperator_MIN = BuiltinOperator_ADD, - BuiltinOperator_MAX = BuiltinOperator_RESIZE_NEAREST_NEIGHBOR + BuiltinOperator_MAX = BuiltinOperator_SPLIT_V }; -inline const BuiltinOperator (&EnumValuesBuiltinOperator())[97] { +inline const BuiltinOperator (&EnumValuesBuiltinOperator())[102] { static const BuiltinOperator values[] = { BuiltinOperator_ADD, BuiltinOperator_AVERAGE_POOL_2D, @@ -602,7 +622,12 @@ inline const BuiltinOperator (&EnumValuesBuiltinOperator())[97] { BuiltinOperator_FILL, BuiltinOperator_FLOOR_MOD, BuiltinOperator_RANGE, - BuiltinOperator_RESIZE_NEAREST_NEIGHBOR + BuiltinOperator_RESIZE_NEAREST_NEIGHBOR, + BuiltinOperator_LEAKY_RELU, + BuiltinOperator_SQUARED_DIFFERENCE, + BuiltinOperator_MIRROR_PAD, + BuiltinOperator_ABS, + BuiltinOperator_SPLIT_V }; return values; } @@ -707,6 +732,11 @@ inline const char * const *EnumNamesBuiltinOperator() { "FLOOR_MOD", "RANGE", "RESIZE_NEAREST_NEIGHBOR", + "LEAKY_RELU", + "SQUARED_DIFFERENCE", + "MIRROR_PAD", + "ABS", + "SPLIT_V", nullptr }; return names; @@ -793,11 +823,16 @@ enum BuiltinOptions { BuiltinOptions_FloorModOptions = 72, BuiltinOptions_RangeOptions = 73, BuiltinOptions_ResizeNearestNeighborOptions = 74, + BuiltinOptions_LeakyReluOptions = 75, + BuiltinOptions_SquaredDifferenceOptions = 76, + BuiltinOptions_MirrorPadOptions = 77, + BuiltinOptions_AbsOptions = 78, + BuiltinOptions_SplitVOptions = 79, BuiltinOptions_MIN = BuiltinOptions_NONE, - BuiltinOptions_MAX = BuiltinOptions_ResizeNearestNeighborOptions + BuiltinOptions_MAX = BuiltinOptions_SplitVOptions }; -inline const BuiltinOptions (&EnumValuesBuiltinOptions())[75] { +inline const BuiltinOptions (&EnumValuesBuiltinOptions())[80] { static const BuiltinOptions values[] = { BuiltinOptions_NONE, BuiltinOptions_Conv2DOptions, @@ -873,7 +908,12 @@ inline const BuiltinOptions (&EnumValuesBuiltinOptions())[75] { BuiltinOptions_UnidirectionalSequenceLSTMOptions, BuiltinOptions_FloorModOptions, BuiltinOptions_RangeOptions, - BuiltinOptions_ResizeNearestNeighborOptions + BuiltinOptions_ResizeNearestNeighborOptions, + BuiltinOptions_LeakyReluOptions, + BuiltinOptions_SquaredDifferenceOptions, + BuiltinOptions_MirrorPadOptions, + BuiltinOptions_AbsOptions, + BuiltinOptions_SplitVOptions }; return values; } @@ -955,6 +995,11 @@ inline const char * const *EnumNamesBuiltinOptions() { "FloorModOptions", "RangeOptions", "ResizeNearestNeighborOptions", + "LeakyReluOptions", + "SquaredDifferenceOptions", + "MirrorPadOptions", + "AbsOptions", + "SplitVOptions", nullptr }; return names; @@ -1265,6 +1310,26 @@ template<> struct BuiltinOptionsTraits { static const BuiltinOptions enum_value = BuiltinOptions_ResizeNearestNeighborOptions; }; +template<> struct BuiltinOptionsTraits { + static const BuiltinOptions enum_value = BuiltinOptions_LeakyReluOptions; +}; + +template<> struct BuiltinOptionsTraits { + static const BuiltinOptions enum_value = BuiltinOptions_SquaredDifferenceOptions; +}; + +template<> struct BuiltinOptionsTraits { + static const BuiltinOptions enum_value = BuiltinOptions_MirrorPadOptions; +}; + +template<> struct BuiltinOptionsTraits { + static const BuiltinOptions enum_value = BuiltinOptions_AbsOptions; +}; + +template<> struct BuiltinOptionsTraits { + static const BuiltinOptions enum_value = BuiltinOptions_SplitVOptions; +}; + struct BuiltinOptionsUnion { BuiltinOptions type; void *value; @@ -1888,6 +1953,46 @@ struct BuiltinOptionsUnion { return type == BuiltinOptions_ResizeNearestNeighborOptions ? reinterpret_cast(value) : nullptr; } + LeakyReluOptionsT *AsLeakyReluOptions() { + return type == BuiltinOptions_LeakyReluOptions ? + reinterpret_cast(value) : nullptr; + } + const LeakyReluOptionsT *AsLeakyReluOptions() const { + return type == BuiltinOptions_LeakyReluOptions ? + reinterpret_cast(value) : nullptr; + } + SquaredDifferenceOptionsT *AsSquaredDifferenceOptions() { + return type == BuiltinOptions_SquaredDifferenceOptions ? + reinterpret_cast(value) : nullptr; + } + const SquaredDifferenceOptionsT *AsSquaredDifferenceOptions() const { + return type == BuiltinOptions_SquaredDifferenceOptions ? + reinterpret_cast(value) : nullptr; + } + MirrorPadOptionsT *AsMirrorPadOptions() { + return type == BuiltinOptions_MirrorPadOptions ? + reinterpret_cast(value) : nullptr; + } + const MirrorPadOptionsT *AsMirrorPadOptions() const { + return type == BuiltinOptions_MirrorPadOptions ? + reinterpret_cast(value) : nullptr; + } + AbsOptionsT *AsAbsOptions() { + return type == BuiltinOptions_AbsOptions ? + reinterpret_cast(value) : nullptr; + } + const AbsOptionsT *AsAbsOptions() const { + return type == BuiltinOptions_AbsOptions ? + reinterpret_cast(value) : nullptr; + } + SplitVOptionsT *AsSplitVOptions() { + return type == BuiltinOptions_SplitVOptions ? + reinterpret_cast(value) : nullptr; + } + const SplitVOptionsT *AsSplitVOptions() const { + return type == BuiltinOptions_SplitVOptions ? + reinterpret_cast(value) : nullptr; + } }; bool VerifyBuiltinOptions(flatbuffers::Verifier &verifier, const void *obj, BuiltinOptions type); @@ -2085,6 +2190,35 @@ inline const char *EnumNameCombinerType(CombinerType e) { return EnumNamesCombinerType()[index]; } +enum MirrorPadMode { + MirrorPadMode_REFLECT = 0, + MirrorPadMode_SYMMETRIC = 1, + MirrorPadMode_MIN = MirrorPadMode_REFLECT, + MirrorPadMode_MAX = MirrorPadMode_SYMMETRIC +}; + +inline const MirrorPadMode (&EnumValuesMirrorPadMode())[2] { + static const MirrorPadMode values[] = { + MirrorPadMode_REFLECT, + MirrorPadMode_SYMMETRIC + }; + return values; +} + +inline const char * const *EnumNamesMirrorPadMode() { + static const char * const names[] = { + "REFLECT", + "SYMMETRIC", + nullptr + }; + return names; +} + +inline const char *EnumNameMirrorPadMode(MirrorPadMode e) { + const size_t index = static_cast(e); + return EnumNamesMirrorPadMode()[index]; +} + enum CustomOptionsFormat { CustomOptionsFormat_FLEXBUFFERS = 0, CustomOptionsFormat_MIN = CustomOptionsFormat_FLEXBUFFERS, @@ -4935,6 +5069,60 @@ inline flatbuffers::Offset CreateSplitOptions( flatbuffers::Offset CreateSplitOptions(flatbuffers::FlatBufferBuilder &_fbb, const SplitOptionsT *_o, const flatbuffers::rehasher_function_t *_rehasher = nullptr); +struct SplitVOptionsT : public flatbuffers::NativeTable { + typedef SplitVOptions TableType; + int32_t num_splits; + SplitVOptionsT() + : num_splits(0) { + } +}; + +struct SplitVOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { + typedef SplitVOptionsT NativeTableType; + enum { + VT_NUM_SPLITS = 4 + }; + int32_t num_splits() const { + return GetField(VT_NUM_SPLITS, 0); + } + bool Verify(flatbuffers::Verifier &verifier) const { + return VerifyTableStart(verifier) && + VerifyField(verifier, VT_NUM_SPLITS) && + verifier.EndTable(); + } + SplitVOptionsT *UnPack(const flatbuffers::resolver_function_t *_resolver = nullptr) const; + void UnPackTo(SplitVOptionsT *_o, const flatbuffers::resolver_function_t *_resolver = nullptr) const; + static flatbuffers::Offset Pack(flatbuffers::FlatBufferBuilder &_fbb, const SplitVOptionsT* _o, const flatbuffers::rehasher_function_t *_rehasher = nullptr); +}; + +struct SplitVOptionsBuilder { + flatbuffers::FlatBufferBuilder &fbb_; + flatbuffers::uoffset_t start_; + void add_num_splits(int32_t num_splits) { + fbb_.AddElement(SplitVOptions::VT_NUM_SPLITS, num_splits, 0); + } + explicit SplitVOptionsBuilder(flatbuffers::FlatBufferBuilder &_fbb) + : fbb_(_fbb) { + start_ = fbb_.StartTable(); + } + SplitVOptionsBuilder &operator=(const SplitVOptionsBuilder &); + flatbuffers::Offset Finish() { + const auto end = fbb_.EndTable(start_); + auto o = flatbuffers::Offset(end); + return o; + } +}; + +inline flatbuffers::Offset CreateSplitVOptions( + flatbuffers::FlatBufferBuilder &_fbb, + int32_t num_splits = 0) { + SplitVOptionsBuilder builder_(_fbb); + builder_.add_num_splits(num_splits); + return builder_.Finish(); +} + +flatbuffers::Offset CreateSplitVOptions(flatbuffers::FlatBufferBuilder &_fbb, const SplitVOptionsT *_o, const flatbuffers::rehasher_function_t *_rehasher = nullptr); + struct StridedSliceOptionsT : public flatbuffers::NativeTable { typedef StridedSliceOptions TableType; int32_t begin_mask; @@ -6247,6 +6435,46 @@ inline flatbuffers::Offset CreateOneHotOptions( flatbuffers::Offset CreateOneHotOptions(flatbuffers::FlatBufferBuilder &_fbb, const OneHotOptionsT *_o, const flatbuffers::rehasher_function_t *_rehasher = nullptr); +struct AbsOptionsT : public flatbuffers::NativeTable { + typedef AbsOptions TableType; + AbsOptionsT() { + } +}; + +struct AbsOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { + typedef AbsOptionsT NativeTableType; + bool Verify(flatbuffers::Verifier &verifier) const { + return VerifyTableStart(verifier) && + verifier.EndTable(); + } + AbsOptionsT *UnPack(const flatbuffers::resolver_function_t *_resolver = nullptr) const; + void UnPackTo(AbsOptionsT *_o, const flatbuffers::resolver_function_t *_resolver = nullptr) const; + static flatbuffers::Offset Pack(flatbuffers::FlatBufferBuilder &_fbb, const AbsOptionsT* _o, const flatbuffers::rehasher_function_t *_rehasher = nullptr); +}; + +struct AbsOptionsBuilder { + flatbuffers::FlatBufferBuilder &fbb_; + flatbuffers::uoffset_t start_; + explicit AbsOptionsBuilder(flatbuffers::FlatBufferBuilder &_fbb) + : fbb_(_fbb) { + start_ = fbb_.StartTable(); + } + AbsOptionsBuilder &operator=(const AbsOptionsBuilder &); + flatbuffers::Offset Finish() { + const auto end = fbb_.EndTable(start_); + auto o = flatbuffers::Offset(end); + return o; + } +}; + +inline flatbuffers::Offset CreateAbsOptions( + flatbuffers::FlatBufferBuilder &_fbb) { + AbsOptionsBuilder builder_(_fbb); + return builder_.Finish(); +} + +flatbuffers::Offset CreateAbsOptions(flatbuffers::FlatBufferBuilder &_fbb, const AbsOptionsT *_o, const flatbuffers::rehasher_function_t *_rehasher = nullptr); + struct LogicalAndOptionsT : public flatbuffers::NativeTable { typedef LogicalAndOptions TableType; LogicalAndOptionsT() { @@ -6633,6 +6861,154 @@ inline flatbuffers::Offset CreateRangeOptions( flatbuffers::Offset CreateRangeOptions(flatbuffers::FlatBufferBuilder &_fbb, const RangeOptionsT *_o, const flatbuffers::rehasher_function_t *_rehasher = nullptr); +struct LeakyReluOptionsT : public flatbuffers::NativeTable { + typedef LeakyReluOptions TableType; + float alpha; + LeakyReluOptionsT() + : alpha(0.0f) { + } +}; + +struct LeakyReluOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { + typedef LeakyReluOptionsT NativeTableType; + enum { + VT_ALPHA = 4 + }; + float alpha() const { + return GetField(VT_ALPHA, 0.0f); + } + bool Verify(flatbuffers::Verifier &verifier) const { + return VerifyTableStart(verifier) && + VerifyField(verifier, VT_ALPHA) && + verifier.EndTable(); + } + LeakyReluOptionsT *UnPack(const flatbuffers::resolver_function_t *_resolver = nullptr) const; + void UnPackTo(LeakyReluOptionsT *_o, const flatbuffers::resolver_function_t *_resolver = nullptr) const; + static flatbuffers::Offset Pack(flatbuffers::FlatBufferBuilder &_fbb, const LeakyReluOptionsT* _o, const flatbuffers::rehasher_function_t *_rehasher = nullptr); +}; + +struct LeakyReluOptionsBuilder { + flatbuffers::FlatBufferBuilder &fbb_; + flatbuffers::uoffset_t start_; + void add_alpha(float alpha) { + fbb_.AddElement(LeakyReluOptions::VT_ALPHA, alpha, 0.0f); + } + explicit LeakyReluOptionsBuilder(flatbuffers::FlatBufferBuilder &_fbb) + : fbb_(_fbb) { + start_ = fbb_.StartTable(); + } + LeakyReluOptionsBuilder &operator=(const LeakyReluOptionsBuilder &); + flatbuffers::Offset Finish() { + const auto end = fbb_.EndTable(start_); + auto o = flatbuffers::Offset(end); + return o; + } +}; + +inline flatbuffers::Offset CreateLeakyReluOptions( + flatbuffers::FlatBufferBuilder &_fbb, + float alpha = 0.0f) { + LeakyReluOptionsBuilder builder_(_fbb); + builder_.add_alpha(alpha); + return builder_.Finish(); +} + +flatbuffers::Offset CreateLeakyReluOptions(flatbuffers::FlatBufferBuilder &_fbb, const LeakyReluOptionsT *_o, const flatbuffers::rehasher_function_t *_rehasher = nullptr); + +struct SquaredDifferenceOptionsT : public flatbuffers::NativeTable { + typedef SquaredDifferenceOptions TableType; + SquaredDifferenceOptionsT() { + } +}; + +struct SquaredDifferenceOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { + typedef SquaredDifferenceOptionsT NativeTableType; + bool Verify(flatbuffers::Verifier &verifier) const { + return VerifyTableStart(verifier) && + verifier.EndTable(); + } + SquaredDifferenceOptionsT *UnPack(const flatbuffers::resolver_function_t *_resolver = nullptr) const; + void UnPackTo(SquaredDifferenceOptionsT *_o, const flatbuffers::resolver_function_t *_resolver = nullptr) const; + static flatbuffers::Offset Pack(flatbuffers::FlatBufferBuilder &_fbb, const SquaredDifferenceOptionsT* _o, const flatbuffers::rehasher_function_t *_rehasher = nullptr); +}; + +struct SquaredDifferenceOptionsBuilder { + flatbuffers::FlatBufferBuilder &fbb_; + flatbuffers::uoffset_t start_; + explicit SquaredDifferenceOptionsBuilder(flatbuffers::FlatBufferBuilder &_fbb) + : fbb_(_fbb) { + start_ = fbb_.StartTable(); + } + SquaredDifferenceOptionsBuilder &operator=(const SquaredDifferenceOptionsBuilder &); + flatbuffers::Offset Finish() { + const auto end = fbb_.EndTable(start_); + auto o = flatbuffers::Offset(end); + return o; + } +}; + +inline flatbuffers::Offset CreateSquaredDifferenceOptions( + flatbuffers::FlatBufferBuilder &_fbb) { + SquaredDifferenceOptionsBuilder builder_(_fbb); + return builder_.Finish(); +} + +flatbuffers::Offset CreateSquaredDifferenceOptions(flatbuffers::FlatBufferBuilder &_fbb, const SquaredDifferenceOptionsT *_o, const flatbuffers::rehasher_function_t *_rehasher = nullptr); + +struct MirrorPadOptionsT : public flatbuffers::NativeTable { + typedef MirrorPadOptions TableType; + MirrorPadMode mode; + MirrorPadOptionsT() + : mode(MirrorPadMode_REFLECT) { + } +}; + +struct MirrorPadOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { + typedef MirrorPadOptionsT NativeTableType; + enum { + VT_MODE = 4 + }; + MirrorPadMode mode() const { + return static_cast(GetField(VT_MODE, 0)); + } + bool Verify(flatbuffers::Verifier &verifier) const { + return VerifyTableStart(verifier) && + VerifyField(verifier, VT_MODE) && + verifier.EndTable(); + } + MirrorPadOptionsT *UnPack(const flatbuffers::resolver_function_t *_resolver = nullptr) const; + void UnPackTo(MirrorPadOptionsT *_o, const flatbuffers::resolver_function_t *_resolver = nullptr) const; + static flatbuffers::Offset Pack(flatbuffers::FlatBufferBuilder &_fbb, const MirrorPadOptionsT* _o, const flatbuffers::rehasher_function_t *_rehasher = nullptr); +}; + +struct MirrorPadOptionsBuilder { + flatbuffers::FlatBufferBuilder &fbb_; + flatbuffers::uoffset_t start_; + void add_mode(MirrorPadMode mode) { + fbb_.AddElement(MirrorPadOptions::VT_MODE, static_cast(mode), 0); + } + explicit MirrorPadOptionsBuilder(flatbuffers::FlatBufferBuilder &_fbb) + : fbb_(_fbb) { + start_ = fbb_.StartTable(); + } + MirrorPadOptionsBuilder &operator=(const MirrorPadOptionsBuilder &); + flatbuffers::Offset Finish() { + const auto end = fbb_.EndTable(start_); + auto o = flatbuffers::Offset(end); + return o; + } +}; + +inline flatbuffers::Offset CreateMirrorPadOptions( + flatbuffers::FlatBufferBuilder &_fbb, + MirrorPadMode mode = MirrorPadMode_REFLECT) { + MirrorPadOptionsBuilder builder_(_fbb); + builder_.add_mode(mode); + return builder_.Finish(); +} + +flatbuffers::Offset CreateMirrorPadOptions(flatbuffers::FlatBufferBuilder &_fbb, const MirrorPadOptionsT *_o, const flatbuffers::rehasher_function_t *_rehasher = nullptr); + struct OperatorCodeT : public flatbuffers::NativeTable { typedef OperatorCode TableType; BuiltinOperator builtin_code; @@ -6988,6 +7364,21 @@ struct Operator FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { const ResizeNearestNeighborOptions *builtin_options_as_ResizeNearestNeighborOptions() const { return builtin_options_type() == BuiltinOptions_ResizeNearestNeighborOptions ? static_cast(builtin_options()) : nullptr; } + const LeakyReluOptions *builtin_options_as_LeakyReluOptions() const { + return builtin_options_type() == BuiltinOptions_LeakyReluOptions ? static_cast(builtin_options()) : nullptr; + } + const SquaredDifferenceOptions *builtin_options_as_SquaredDifferenceOptions() const { + return builtin_options_type() == BuiltinOptions_SquaredDifferenceOptions ? static_cast(builtin_options()) : nullptr; + } + const MirrorPadOptions *builtin_options_as_MirrorPadOptions() const { + return builtin_options_type() == BuiltinOptions_MirrorPadOptions ? static_cast(builtin_options()) : nullptr; + } + const AbsOptions *builtin_options_as_AbsOptions() const { + return builtin_options_type() == BuiltinOptions_AbsOptions ? static_cast(builtin_options()) : nullptr; + } + const SplitVOptions *builtin_options_as_SplitVOptions() const { + return builtin_options_type() == BuiltinOptions_SplitVOptions ? static_cast(builtin_options()) : nullptr; + } const flatbuffers::Vector *custom_options() const { return GetPointer *>(VT_CUSTOM_OPTIONS); } @@ -7315,6 +7706,26 @@ template<> inline const ResizeNearestNeighborOptions *Operator::builtin_options_ return builtin_options_as_ResizeNearestNeighborOptions(); } +template<> inline const LeakyReluOptions *Operator::builtin_options_as() const { + return builtin_options_as_LeakyReluOptions(); +} + +template<> inline const SquaredDifferenceOptions *Operator::builtin_options_as() const { + return builtin_options_as_SquaredDifferenceOptions(); +} + +template<> inline const MirrorPadOptions *Operator::builtin_options_as() const { + return builtin_options_as_MirrorPadOptions(); +} + +template<> inline const AbsOptions *Operator::builtin_options_as() const { + return builtin_options_as_AbsOptions(); +} + +template<> inline const SplitVOptions *Operator::builtin_options_as() const { + return builtin_options_as_SplitVOptions(); +} + struct OperatorBuilder { flatbuffers::FlatBufferBuilder &fbb_; flatbuffers::uoffset_t start_; @@ -8932,6 +9343,32 @@ inline flatbuffers::Offset CreateSplitOptions(flatbuffers::FlatBuf _num_splits); } +inline SplitVOptionsT *SplitVOptions::UnPack(const flatbuffers::resolver_function_t *_resolver) const { + auto _o = new SplitVOptionsT(); + UnPackTo(_o, _resolver); + return _o; +} + +inline void SplitVOptions::UnPackTo(SplitVOptionsT *_o, const flatbuffers::resolver_function_t *_resolver) const { + (void)_o; + (void)_resolver; + { auto _e = num_splits(); _o->num_splits = _e; }; +} + +inline flatbuffers::Offset SplitVOptions::Pack(flatbuffers::FlatBufferBuilder &_fbb, const SplitVOptionsT* _o, const flatbuffers::rehasher_function_t *_rehasher) { + return CreateSplitVOptions(_fbb, _o, _rehasher); +} + +inline flatbuffers::Offset CreateSplitVOptions(flatbuffers::FlatBufferBuilder &_fbb, const SplitVOptionsT *_o, const flatbuffers::rehasher_function_t *_rehasher) { + (void)_rehasher; + (void)_o; + struct _VectorArgs { flatbuffers::FlatBufferBuilder *__fbb; const SplitVOptionsT* __o; const flatbuffers::rehasher_function_t *__rehasher; } _va = { &_fbb, _o, _rehasher}; (void)_va; + auto _num_splits = _o->num_splits; + return tflite::CreateSplitVOptions( + _fbb, + _num_splits); +} + inline StridedSliceOptionsT *StridedSliceOptions::UnPack(const flatbuffers::resolver_function_t *_resolver) const { auto _o = new StridedSliceOptionsT(); UnPackTo(_o, _resolver); @@ -9593,6 +10030,29 @@ inline flatbuffers::Offset CreateOneHotOptions(flatbuffers::FlatB _axis); } +inline AbsOptionsT *AbsOptions::UnPack(const flatbuffers::resolver_function_t *_resolver) const { + auto _o = new AbsOptionsT(); + UnPackTo(_o, _resolver); + return _o; +} + +inline void AbsOptions::UnPackTo(AbsOptionsT *_o, const flatbuffers::resolver_function_t *_resolver) const { + (void)_o; + (void)_resolver; +} + +inline flatbuffers::Offset AbsOptions::Pack(flatbuffers::FlatBufferBuilder &_fbb, const AbsOptionsT* _o, const flatbuffers::rehasher_function_t *_rehasher) { + return CreateAbsOptions(_fbb, _o, _rehasher); +} + +inline flatbuffers::Offset CreateAbsOptions(flatbuffers::FlatBufferBuilder &_fbb, const AbsOptionsT *_o, const flatbuffers::rehasher_function_t *_rehasher) { + (void)_rehasher; + (void)_o; + struct _VectorArgs { flatbuffers::FlatBufferBuilder *__fbb; const AbsOptionsT* __o; const flatbuffers::rehasher_function_t *__rehasher; } _va = { &_fbb, _o, _rehasher}; (void)_va; + return tflite::CreateAbsOptions( + _fbb); +} + inline LogicalAndOptionsT *LogicalAndOptions::UnPack(const flatbuffers::resolver_function_t *_resolver) const { auto _o = new LogicalAndOptionsT(); UnPackTo(_o, _resolver); @@ -9806,6 +10266,81 @@ inline flatbuffers::Offset CreateRangeOptions(flatbuffers::FlatBuf _fbb); } +inline LeakyReluOptionsT *LeakyReluOptions::UnPack(const flatbuffers::resolver_function_t *_resolver) const { + auto _o = new LeakyReluOptionsT(); + UnPackTo(_o, _resolver); + return _o; +} + +inline void LeakyReluOptions::UnPackTo(LeakyReluOptionsT *_o, const flatbuffers::resolver_function_t *_resolver) const { + (void)_o; + (void)_resolver; + { auto _e = alpha(); _o->alpha = _e; }; +} + +inline flatbuffers::Offset LeakyReluOptions::Pack(flatbuffers::FlatBufferBuilder &_fbb, const LeakyReluOptionsT* _o, const flatbuffers::rehasher_function_t *_rehasher) { + return CreateLeakyReluOptions(_fbb, _o, _rehasher); +} + +inline flatbuffers::Offset CreateLeakyReluOptions(flatbuffers::FlatBufferBuilder &_fbb, const LeakyReluOptionsT *_o, const flatbuffers::rehasher_function_t *_rehasher) { + (void)_rehasher; + (void)_o; + struct _VectorArgs { flatbuffers::FlatBufferBuilder *__fbb; const LeakyReluOptionsT* __o; const flatbuffers::rehasher_function_t *__rehasher; } _va = { &_fbb, _o, _rehasher}; (void)_va; + auto _alpha = _o->alpha; + return tflite::CreateLeakyReluOptions( + _fbb, + _alpha); +} + +inline SquaredDifferenceOptionsT *SquaredDifferenceOptions::UnPack(const flatbuffers::resolver_function_t *_resolver) const { + auto _o = new SquaredDifferenceOptionsT(); + UnPackTo(_o, _resolver); + return _o; +} + +inline void SquaredDifferenceOptions::UnPackTo(SquaredDifferenceOptionsT *_o, const flatbuffers::resolver_function_t *_resolver) const { + (void)_o; + (void)_resolver; +} + +inline flatbuffers::Offset SquaredDifferenceOptions::Pack(flatbuffers::FlatBufferBuilder &_fbb, const SquaredDifferenceOptionsT* _o, const flatbuffers::rehasher_function_t *_rehasher) { + return CreateSquaredDifferenceOptions(_fbb, _o, _rehasher); +} + +inline flatbuffers::Offset CreateSquaredDifferenceOptions(flatbuffers::FlatBufferBuilder &_fbb, const SquaredDifferenceOptionsT *_o, const flatbuffers::rehasher_function_t *_rehasher) { + (void)_rehasher; + (void)_o; + struct _VectorArgs { flatbuffers::FlatBufferBuilder *__fbb; const SquaredDifferenceOptionsT* __o; const flatbuffers::rehasher_function_t *__rehasher; } _va = { &_fbb, _o, _rehasher}; (void)_va; + return tflite::CreateSquaredDifferenceOptions( + _fbb); +} + +inline MirrorPadOptionsT *MirrorPadOptions::UnPack(const flatbuffers::resolver_function_t *_resolver) const { + auto _o = new MirrorPadOptionsT(); + UnPackTo(_o, _resolver); + return _o; +} + +inline void MirrorPadOptions::UnPackTo(MirrorPadOptionsT *_o, const flatbuffers::resolver_function_t *_resolver) const { + (void)_o; + (void)_resolver; + { auto _e = mode(); _o->mode = _e; }; +} + +inline flatbuffers::Offset MirrorPadOptions::Pack(flatbuffers::FlatBufferBuilder &_fbb, const MirrorPadOptionsT* _o, const flatbuffers::rehasher_function_t *_rehasher) { + return CreateMirrorPadOptions(_fbb, _o, _rehasher); +} + +inline flatbuffers::Offset CreateMirrorPadOptions(flatbuffers::FlatBufferBuilder &_fbb, const MirrorPadOptionsT *_o, const flatbuffers::rehasher_function_t *_rehasher) { + (void)_rehasher; + (void)_o; + struct _VectorArgs { flatbuffers::FlatBufferBuilder *__fbb; const MirrorPadOptionsT* __o; const flatbuffers::rehasher_function_t *__rehasher; } _va = { &_fbb, _o, _rehasher}; (void)_va; + auto _mode = _o->mode; + return tflite::CreateMirrorPadOptions( + _fbb, + _mode); +} + inline OperatorCodeT *OperatorCode::UnPack(const flatbuffers::resolver_function_t *_resolver) const { auto _o = new OperatorCodeT(); UnPackTo(_o, _resolver); @@ -10360,6 +10895,26 @@ inline bool VerifyBuiltinOptions(flatbuffers::Verifier &verifier, const void *ob auto ptr = reinterpret_cast(obj); return verifier.VerifyTable(ptr); } + case BuiltinOptions_LeakyReluOptions: { + auto ptr = reinterpret_cast(obj); + return verifier.VerifyTable(ptr); + } + case BuiltinOptions_SquaredDifferenceOptions: { + auto ptr = reinterpret_cast(obj); + return verifier.VerifyTable(ptr); + } + case BuiltinOptions_MirrorPadOptions: { + auto ptr = reinterpret_cast(obj); + return verifier.VerifyTable(ptr); + } + case BuiltinOptions_AbsOptions: { + auto ptr = reinterpret_cast(obj); + return verifier.VerifyTable(ptr); + } + case BuiltinOptions_SplitVOptions: { + auto ptr = reinterpret_cast(obj); + return verifier.VerifyTable(ptr); + } default: return false; } } @@ -10674,6 +11229,26 @@ inline void *BuiltinOptionsUnion::UnPack(const void *obj, BuiltinOptions type, c auto ptr = reinterpret_cast(obj); return ptr->UnPack(resolver); } + case BuiltinOptions_LeakyReluOptions: { + auto ptr = reinterpret_cast(obj); + return ptr->UnPack(resolver); + } + case BuiltinOptions_SquaredDifferenceOptions: { + auto ptr = reinterpret_cast(obj); + return ptr->UnPack(resolver); + } + case BuiltinOptions_MirrorPadOptions: { + auto ptr = reinterpret_cast(obj); + return ptr->UnPack(resolver); + } + case BuiltinOptions_AbsOptions: { + auto ptr = reinterpret_cast(obj); + return ptr->UnPack(resolver); + } + case BuiltinOptions_SplitVOptions: { + auto ptr = reinterpret_cast(obj); + return ptr->UnPack(resolver); + } default: return nullptr; } } @@ -10976,6 +11551,26 @@ inline flatbuffers::Offset BuiltinOptionsUnion::Pack(flatbuffers::FlatBuff auto ptr = reinterpret_cast(value); return CreateResizeNearestNeighborOptions(_fbb, ptr, _rehasher).Union(); } + case BuiltinOptions_LeakyReluOptions: { + auto ptr = reinterpret_cast(value); + return CreateLeakyReluOptions(_fbb, ptr, _rehasher).Union(); + } + case BuiltinOptions_SquaredDifferenceOptions: { + auto ptr = reinterpret_cast(value); + return CreateSquaredDifferenceOptions(_fbb, ptr, _rehasher).Union(); + } + case BuiltinOptions_MirrorPadOptions: { + auto ptr = reinterpret_cast(value); + return CreateMirrorPadOptions(_fbb, ptr, _rehasher).Union(); + } + case BuiltinOptions_AbsOptions: { + auto ptr = reinterpret_cast(value); + return CreateAbsOptions(_fbb, ptr, _rehasher).Union(); + } + case BuiltinOptions_SplitVOptions: { + auto ptr = reinterpret_cast(value); + return CreateSplitVOptions(_fbb, ptr, _rehasher).Union(); + } default: return 0; } } @@ -11278,6 +11873,26 @@ inline BuiltinOptionsUnion::BuiltinOptionsUnion(const BuiltinOptionsUnion &u) FL value = new ResizeNearestNeighborOptionsT(*reinterpret_cast(u.value)); break; } + case BuiltinOptions_LeakyReluOptions: { + value = new LeakyReluOptionsT(*reinterpret_cast(u.value)); + break; + } + case BuiltinOptions_SquaredDifferenceOptions: { + value = new SquaredDifferenceOptionsT(*reinterpret_cast(u.value)); + break; + } + case BuiltinOptions_MirrorPadOptions: { + value = new MirrorPadOptionsT(*reinterpret_cast(u.value)); + break; + } + case BuiltinOptions_AbsOptions: { + value = new AbsOptionsT(*reinterpret_cast(u.value)); + break; + } + case BuiltinOptions_SplitVOptions: { + value = new SplitVOptionsT(*reinterpret_cast(u.value)); + break; + } default: break; } @@ -11655,6 +12270,31 @@ inline void BuiltinOptionsUnion::Reset() { delete ptr; break; } + case BuiltinOptions_LeakyReluOptions: { + auto ptr = reinterpret_cast(value); + delete ptr; + break; + } + case BuiltinOptions_SquaredDifferenceOptions: { + auto ptr = reinterpret_cast(value); + delete ptr; + break; + } + case BuiltinOptions_MirrorPadOptions: { + auto ptr = reinterpret_cast(value); + delete ptr; + break; + } + case BuiltinOptions_AbsOptions: { + auto ptr = reinterpret_cast(value); + delete ptr; + break; + } + case BuiltinOptions_SplitVOptions: { + auto ptr = reinterpret_cast(value); + delete ptr; + break; + } default: break; } value = nullptr; diff --git a/tensorflow/lite/string_util.cc b/tensorflow/lite/string_util.cc index 1b33f5bcba0..6efa11d60c5 100644 --- a/tensorflow/lite/string_util.cc +++ b/tensorflow/lite/string_util.cc @@ -96,8 +96,7 @@ int DynamicBuffer::WriteToBuffer(char** buffer) { return bytes; } -void DynamicBuffer::WriteToTensor(TfLiteTensor* tensor) { - // Set tensor content pointer to tensor_buffer, and release original data. +void DynamicBuffer::WriteToTensorAsVector(TfLiteTensor* tensor) { auto dims = TfLiteIntArrayCreate(1); dims->data[0] = offset_.size() - 1; // Store number of strings. WriteToTensor(tensor, dims); @@ -108,6 +107,10 @@ void DynamicBuffer::WriteToTensor(TfLiteTensor* tensor, char* tensor_buffer; int bytes = WriteToBuffer(&tensor_buffer); + if (new_shape == nullptr) { + new_shape = TfLiteIntArrayCopy(tensor->dims); + } + // Set tensor content pointer to tensor_buffer, and release original data. TfLiteTensorReset(tensor->type, tensor->name, new_shape, tensor->params, tensor_buffer, bytes, kTfLiteDynamic, tensor->allocation, diff --git a/tensorflow/lite/string_util.h b/tensorflow/lite/string_util.h index c9b74482f7d..f076db76f2d 100644 --- a/tensorflow/lite/string_util.h +++ b/tensorflow/lite/string_util.h @@ -74,12 +74,18 @@ class DynamicBuffer { // The function allocates space for the buffer but does NOT take ownership. int WriteToBuffer(char** buffer); - // Fill content into a string tensor, with the given new_shape. The new - // shape must match the number of strings in this object. + // Fill content into a string tensor, with the given new_shape. The new shape + // must match the number of strings in this object. Caller relinquishes + // ownership of new_shape. If 'new_shape' is nullptr, keep the tensor's + // existing shape. void WriteToTensor(TfLiteTensor* tensor, TfLiteIntArray* new_shape); // Fill content into a string tensor. Set shape to {num_strings}. - void WriteToTensor(TfLiteTensor* tensor); + void WriteToTensorAsVector(TfLiteTensor* tensor); + + // Deprecated. Use WriteToTensorAsVector() or pass in the new shpe. + // TODO(b/120230709): remove when people migrate away. + void WriteToTensor(TfLiteTensor* tensor) { WriteToTensorAsVector(tensor); } private: // Data buffer to store contents of strings, not including headers. diff --git a/tensorflow/lite/string_util_test.cc b/tensorflow/lite/string_util_test.cc index 377cdd77eb4..cbf1d7b226a 100644 --- a/tensorflow/lite/string_util_test.cc +++ b/tensorflow/lite/string_util_test.cc @@ -55,7 +55,7 @@ TEST(StringUtil, TestStringUtil) { new_shape->data[0] = 2; new_shape->data[1] = 1; buf0.WriteToTensor(t0, new_shape); - buf1.WriteToTensor(t1); + buf1.WriteToTensorAsVector(t1); // Check tensor shapes. EXPECT_EQ(t0->dims->size, 2); @@ -99,7 +99,7 @@ TEST(StringUtil, TestAddJoinedString) { DynamicBuffer buf; buf.AddJoinedString({{s0, 3}, {s1, 4}, {s2, 0}, {s3, 3}}, ' '); - buf.WriteToTensor(t0); + buf.WriteToTensorAsVector(t0); ASSERT_EQ(GetStringCount(t0), 1); StringRef str_ref; @@ -115,12 +115,43 @@ TEST(StringUtil, TestEmptyList) { t0->type = kTfLiteString; t0->allocation_type = kTfLiteDynamic; DynamicBuffer buf; - buf.WriteToTensor(t0); + buf.WriteToTensorAsVector(t0); ASSERT_EQ(GetStringCount(t0), 0); ASSERT_EQ(t0->bytes, 8); } +TEST(StringUtil, TestShapes) { + Interpreter interpreter; + interpreter.AddTensors(1); + TfLiteTensor* t0 = interpreter.tensor(0); + t0->type = kTfLiteString; + t0->allocation_type = kTfLiteDynamic; + t0->dims = TfLiteIntArrayCreate(2); + t0->dims->data[0] = 2; + t0->dims->data[1] = 1; + + // Not setting a new shape: number of strings must match + DynamicBuffer buf; + buf.AddString("ABC", 3); + buf.AddString("X", 1); + buf.WriteToTensor(t0, nullptr); + + ASSERT_EQ(t0->dims->size, 2); + EXPECT_EQ(t0->dims->data[0], 2); + EXPECT_EQ(t0->dims->data[1], 1); + + auto new_shape = TfLiteIntArrayCreate(2); + new_shape->data[0] = 1; + new_shape->data[1] = 2; + + buf.WriteToTensor(t0, new_shape); + + ASSERT_EQ(t0->dims->size, 2); + EXPECT_EQ(t0->dims->data[0], 1); + EXPECT_EQ(t0->dims->data[1], 2); +} + } // namespace tflite int main(int argc, char** argv) { diff --git a/tensorflow/lite/testing/generate_examples.py b/tensorflow/lite/testing/generate_examples.py index 2b129df766a..4c731a7d18e 100644 --- a/tensorflow/lite/testing/generate_examples.py +++ b/tensorflow/lite/testing/generate_examples.py @@ -103,8 +103,8 @@ KNOWN_BUGS = { r"batch_to_space_nd.*input_shape=\[8,2,2,2,1,1\]": "70594733", # Div will use floordiv. r"div.*int32": "72051395", - # No support for SplitV - r"split.*num_or_size_splits=\[2,2\]": "73377559", + # Constant 1D gather crashes toco. + r"gather_buggy.*input_shape=\[3\].*": "120029508", } @@ -370,7 +370,8 @@ def make_zip_of_tests(zip_path, make_graph, make_test_inputs, extra_toco_options=ExtraTocoOptions(), - use_frozen_graph=False): + use_frozen_graph=False, + expected_tf_success=None): """Helper to make a zip file of a bunch of TensorFlow models. This does a cartestian product of the dictionary of test_parameters and @@ -390,6 +391,8 @@ def make_zip_of_tests(zip_path, `output_tensors` and returns tuple `(input_values, output_values)`. extra_toco_options: Additional toco options. use_frozen_graph: Whether or not freeze graph before toco converter. + expected_tf_success: Number of times tensorflow is supposed to succeed in + executing the input graphs. `None` means "unknown". Raises: RuntimeError: if there are toco errors that can't be ignored. @@ -550,6 +553,11 @@ def make_zip_of_tests(zip_path, " and %d TOCO converted graphs (%.1f%%"), zip_path, total_conversions, tf_success, toco_success, percent) + if expected_tf_success is not None and tf_success != expected_tf_success: + raise RuntimeError( + "Expected TF to succeed %d times, but that happened %d times" % + (expected_tf_success, tf_success)) + if not FLAGS.ignore_toco_errors and toco_errors > 0: raise RuntimeError( "Found %d errors while generating toco models" % toco_errors) @@ -616,6 +624,30 @@ def make_max_pool_tests(zip_path): make_pool_tests(tf.nn.max_pool)(zip_path) +def make_abs_tests(zip_path): + """Make a set of tests to do relu.""" + + # Chose a set of parameters + test_parameters = [{ + "input_shape": [[], [1], [2, 3], [1, 1, 1, 1], [1, 3, 4, 3], + [3, 15, 14, 3], [3, 1, 2, 4, 6], [2, 2, 3, 4, 5, 6]], + }] + + def build_graph(parameters): + input_tensor = tf.placeholder( + dtype=tf.float32, name="input", shape=parameters["input_shape"]) + out = tf.abs(input_tensor) + return [input_tensor], [out] + + def build_inputs(parameters, sess, inputs, outputs): + input_values = create_tensor_data( + np.float32, parameters["input_shape"], min_value=-10, max_value=10) + return [input_values], sess.run( + outputs, feed_dict=dict(zip(inputs, [input_values]))) + + make_zip_of_tests(zip_path, test_parameters, build_graph, build_inputs) + + def make_relu_tests(zip_path): """Make a set of tests to do relu.""" @@ -747,6 +779,34 @@ def make_prelu_tests(zip_path): use_frozen_graph=True) +def make_leaky_relu_tests(zip_path): + """Make a set of tests to do LeakyRelu.""" + + test_parameters = [ + { + "input_shape": [[], [1], [5], [1, 10, 10, 3], [3, 3, 3, 3]], + "alpha": [0.1, 1.0, 2.0, -0.1, -1.0, -2.0], + }, + ] + + def build_graph(parameters): + """Build the graph for the test case.""" + + input_tensor = tf.placeholder( + dtype=tf.float32, name="input", shape=parameters["input_shape"]) + out = tf.nn.leaky_relu(input_tensor, alpha=parameters["alpha"]) + return [input_tensor], [out] + + def build_inputs(parameters, sess, inputs, outputs): + """Build the inputs for the test case.""" + input_values = create_tensor_data( + np.float32, parameters["input_shape"], min_value=-3, max_value=10) + return [input_values], sess.run( + outputs, feed_dict=dict(zip(inputs, [input_values]))) + + make_zip_of_tests(zip_path, test_parameters, build_graph, build_inputs) + + # This function tests various TensorFLow functions that generates Const op, # including `tf.ones`, `tf.zeros` and random functions. def make_constant_tests(zip_path): @@ -755,6 +815,7 @@ def make_constant_tests(zip_path): test_parameters = [{ "dtype": [tf.float32, tf.int32], "input_shape": [[], [1], [2], [1, 1, 1, 1], [2, 2, 2, 2]], + "constant_is_also_output": [True, False], }] def build_graph(parameters): @@ -764,17 +825,19 @@ def make_constant_tests(zip_path): shape=parameters["input_shape"]) constant = tf.constant( create_tensor_data(parameters["dtype"], parameters["input_shape"])) - # This maximum node is here to avoid the situation where a graph output is - # a constant, which is an error in toco. - out = tf.maximum(dummy_input, constant) - return [dummy_input], [out] + out = [tf.maximum(dummy_input, constant)] + if parameters["constant_is_also_output"]: + out.append(constant) + + return [dummy_input], out def build_inputs(parameters, sess, inputs, outputs): dummy_input = np.zeros( parameters["input_shape"], dtype=_TF_TYPE_INFO[parameters["dtype"]][0]) return [dummy_input], sess.run(outputs, feed_dict={inputs[0]: dummy_input}) - make_zip_of_tests(zip_path, test_parameters, build_graph, build_inputs) + make_zip_of_tests(zip_path, test_parameters, build_graph, build_inputs, + expected_tf_success=20) def make_binary_op_tests(zip_path, binary_operator): @@ -869,34 +932,46 @@ def make_reduce_tests(reduce_op, def f(zip_path): """Actual function that generates examples.""" - test_parameters = [{ - "input_dtype": [tf.float32, tf.int32, tf.int64], - "input_shape": [[3, 2, 4]], - "axis": [ - 0, 1, 2, [0, 1], [0, 2], [1, 2], [0, 1, 2], [1, 0], [2, 0], - [2, 1], [2, 1, 0], [2, 0, 1], -1, -2, -3, [1, -1], [0, -1], [-1, 0], - [-1, -2, -3], [0, 0, 0], [2, 2, 0], [1, 0, -3, -3] - ], - "const_axis": [True, False], - "keepdims": [True, False], - }, { - "input_dtype": [tf.float32], - "input_shape": [[1, 8, 8, 3]], - "axis": [ - 0, 1, 2, 3, [1, 2], [0, 3], [1, 2, 3], [0, 1, 2, 3], - [3, 2, 1, 0], [3, 1, 0, 2], [2, 0], [3, 0], [3, 1], [1, 0], -1, -2, - -3, -4, [0, -2], [2, 3, -1, 0], [3, 1, 2, -3], [3, -4], [2, 2, 2], - [2, 2, 3], [-3, -3, -4], [-3, 2, 1] - ], - "const_axis": [True, False], - "keepdims": [True, False], - }, { - "input_dtype": [tf.float32], - "input_shape": [[], [1, 8, 8, 3], [3, 2, 4]], - "axis": [None], - "const_axis": [True], - "keepdims": [True, False], - }] + test_parameters = [ + { + "input_dtype": [tf.float32, tf.int32, tf.int64], + "input_shape": [[3, 3, 2, 4]], + "axis": [ + 0, 1, 2, [0, 1], [0, 2], [1, 2], [0, 1, 2], [1, 0], [2, 0], + [2, 1], [2, 1, 0], [2, 0, 1], -1, -2, -3, [1, -1], [0, -1], + [-1, 0], [-1, -2, -3], [0, 0, 0], [2, 2, 0], [1, 0, -3, -3] + ], + "const_axis": [True, False], + "keepdims": [True, False], + }, + { + "input_dtype": [tf.float32], + "input_shape": [[1, 8, 8, 3]], + "axis": [ + 0, 1, 2, 3, [1, 2], [0, 3], [1, 2, 3], [0, 1, 2, + 3], [3, 2, 1, 0], + [3, 1, 0, 2], [2, 0], [3, 0], [3, 1], [1, 0], -1, -2, -3, -4, + [0, -2], [2, 3, -1, 0], [3, 1, 2, -3], [3, -4], [2, 2, 2], + [2, 2, 3], [-3, -3, -4], [-3, 2, 1] + ], + "const_axis": [True, False], + "keepdims": [True, False], + }, + { + "input_dtype": [tf.float32], + "input_shape": [[], [1, 8, 8, 3], [3, 2, 4]], + "axis": [[]], # shape is: [0] + "const_axis": [False], + "keepdims": [True, False], + }, + { + "input_dtype": [tf.float32], + "input_shape": [[], [1, 8, 8, 3], [3, 2, 4]], + "axis": [None], # shape is: [] + "const_axis": [True], + "keepdims": [True, False], + } + ] def build_graph(parameters): """Build the mean op testing graph.""" @@ -1135,6 +1210,10 @@ def make_floor_mod_tests(zip_path): make_binary_op_tests(zip_path, tf.floormod) +def make_squared_difference_tests(zip_path): + make_binary_op_tests(zip_path, tf.squared_difference) + + def make_gather_tests(zip_path): """Make a set of tests to do gather.""" @@ -1142,9 +1221,9 @@ def make_gather_tests(zip_path): # TODO(mgubin): add string tests when they are supported by Toco. # TODO(mgubin): add tests for Nd indices when they are supported by # TfLite. - "params_dtype": [tf.float32, tf.int32], + "params_dtype": [tf.float32, tf.int32, tf.int64], "params_shape": [[10], [1, 2, 20]], - "indices_dtype": [tf.int32], + "indices_dtype": [tf.int32, tf.int64], "indices_shape": [[3], [5]], "axis": [-1, 0, 1], }] @@ -1172,7 +1251,43 @@ def make_gather_tests(zip_path): return [params, indices], sess.run( outputs, feed_dict=dict(zip(inputs, [params, indices]))) - make_zip_of_tests(zip_path, test_parameters, build_graph, build_inputs) + # Note that TF can't execute with index=1 and params_shape=[10]. + make_zip_of_tests( + zip_path, + test_parameters, + build_graph, + build_inputs, + expected_tf_success=60) + + +def make_gather_buggy_tests(zip_path): + """Make a set of tests to show gather crashes toco.""" + + test_parameters = [{ + "input_shape": [[3]], + "reference_shape": [[2]], + }, { + "input_shape": [[2, 3]], + "reference_shape": [[2, 3]], + }] + + def build_graph(parameters): + """Build a graph where the inputs to Gather are constants.""" + reference = tf.placeholder( + dtype=tf.int32, shape=parameters["reference_shape"]) + gather_input = tf.constant( + create_tensor_data(tf.int32, parameters["input_shape"])) + gather_indices = tf.constant([0, 1], tf.int32) + out = tf.equal(reference, tf.gather(gather_input, gather_indices)) + return [reference], [out] + + def build_inputs(parameters, sess, inputs, outputs): + reference_values = np.zeros(parameters["reference_shape"], dtype=np.int32) + return [reference_values], sess.run( + outputs, feed_dict={inputs[0]: reference_values}) + + make_zip_of_tests(zip_path, test_parameters, build_graph, build_inputs, + expected_tf_success=2) def make_global_batch_norm_tests(zip_path): @@ -1340,23 +1455,27 @@ def make_conv_with_shared_weights_tests(zip_path): input_shape, filter_shape = get_tensor_shapes(parameters) input_tensor = tf.placeholder( dtype=tf.float32, name="input", shape=input_shape) + input_tensors = [input_tensor] # Construct a constant weights tensor which will be used by both Conv2D. filter_tensor = tf.constant( create_tensor_data(np.float32, filter_shape), dtype=tf.float32) - input_tensors = [input_tensor] + + # Ensure that FuseBinaryIntoFollowingAffine works with an input which + # is shared by multiple affine ops. + conv_input = input_tensor + 0.1 # Construct 2 Conv2D operations which use exactly the same input and # weights. result1 = tf.nn.conv2d( - input_tensor, + conv_input, filter_tensor, strides=parameters["strides"], dilations=parameters["dilations"], padding=parameters["padding"], data_format=parameters["data_format"]) result2 = tf.nn.conv2d( - input_tensor, + conv_input, filter_tensor, strides=parameters["strides"], dilations=parameters["dilations"], @@ -1524,7 +1643,7 @@ def make_split_tests(zip_path): test_parameters = [{ "input_shape": [[1, 3, 4, 6], [2, 4, 1], [6, 4], [8]], - "num_or_size_splits": [1, 2, 3, 4, 5, [2, 2]], + "num_or_size_splits": [1, 2, 3, 4, 5], "axis": [0, 1, 2, 3, -4, -3, -2, -1], }] @@ -1542,6 +1661,29 @@ def make_split_tests(zip_path): make_zip_of_tests(zip_path, test_parameters, build_graph, build_inputs) +def make_splitv_tests(zip_path): + """Make a set of tests to do tf.split_v.""" + + test_parameters = [{ + "input_shape": [[1, 3, 4, 6], [2, 4, 1], [6, 4], [8]], + "size_splits": [[2, 2], [1, 3], [4, 2], [5, 3], + [-1, 1], [-1, 2], [-1, 4]], + "axis": [0, 1, 2, 3, -4, -3, -2, -1], + }] + + def build_graph(parameters): + input_tensor = tf.placeholder( + dtype=tf.float32, name="input", shape=parameters["input_shape"]) + out = tf.split(input_tensor, parameters["size_splits"], parameters["axis"]) + return [input_tensor], [out[0]] + + def build_inputs(parameters, sess, inputs, outputs): + values = [create_tensor_data(np.float32, parameters["input_shape"])] + return values, sess.run(outputs, feed_dict=dict(zip(inputs, values))) + + make_zip_of_tests(zip_path, test_parameters, build_graph, build_inputs) + + def make_concat_tests(zip_path): """Make a set of tests to do concatenation.""" @@ -2468,6 +2610,32 @@ def make_strided_slice_1d_exhaustive_tests(zip_path): _make_strided_slice_tests(zip_path, test_parameters) +def make_strided_slice_buggy_tests(zip_path): + """Make a set of tests to show strided_slice yields incorrect results.""" + + test_parameters = [{ + "unused_iteration_counter": [1], + }] + + def build_graph(parameters): + """Build the strided_slice op testing graph.""" + del parameters + input_values = tf.placeholder(dtype=tf.float32, shape=[4, 2]) + data = tf.constant([[0, 1, 2, 3], + [4, 5, 6, 7], + [8, 9, 10, 11], + [12, 13, 14, 15]], tf.float32) + return [input_values], [input_values + data[:, :2]] + + def build_inputs(parameters, sess, inputs, outputs): + del parameters + input_values = np.zeros([4, 2], dtype=np.float32) + return [input_values], sess.run( + outputs, feed_dict={inputs[0]: input_values}) + + make_zip_of_tests(zip_path, test_parameters, build_graph, build_inputs) + + def make_lstm_tests(zip_path): """Make a set of tests to do basic Lstm cell.""" @@ -3121,7 +3289,7 @@ def make_transpose_conv_tests(zip_path): def make_tile_tests(zip_path): """Make a set of tests to do tile.""" test_parameters = [{ - "input_dtype": [tf.float32, tf.int32], + "input_dtype": [tf.float32, tf.int32, tf.bool], "input_shape": [[3, 2, 1], [2, 2, 2]], "multiplier_dtype": [tf.int32, tf.int64], "multiplier_shape": [[3]] @@ -3143,8 +3311,10 @@ def make_tile_tests(zip_path): def build_inputs(parameters, sess, inputs, outputs): input_value = create_tensor_data(parameters["input_dtype"], parameters["input_shape"]) - multipliers_value = create_tensor_data(parameters["multiplier_dtype"], - parameters["multiplier_shape"]) + multipliers_value = create_tensor_data( + parameters["multiplier_dtype"], + parameters["multiplier_shape"], + min_value=0) return [input_value, multipliers_value], sess.run( outputs, feed_dict={ @@ -3365,6 +3535,36 @@ def make_range_tests(zip_path): make_zip_of_tests(zip_path, test_parameters, build_graph, build_inputs) +def make_fill_tests(zip_path): + """Make a set of tests to do fill.""" + + test_parameters = [{ + "dims_dtype": [tf.int32, tf.int64], + "dims_shape": [[], [1], [3], [3, 3]], + "value_dtype": [tf.int32, tf.int64, tf.float32], + }] + + def build_graph(parameters): + """Build the fill op testing graph.""" + input1 = tf.placeholder( + dtype=parameters["dims_dtype"], + name="dims", + shape=parameters["dims_shape"]) + input2 = tf.placeholder( + dtype=parameters["value_dtype"], name="value", shape=[]) + out = tf.fill(input1, input2) + return [input1, input2], [out] + + def build_inputs(parameters, sess, inputs, outputs): + input1 = create_tensor_data(parameters["dims_dtype"], + parameters["dims_shape"], 1) + input2 = create_scalar_data(parameters["value_dtype"]) + return [input1, input2], sess.run( + outputs, feed_dict=dict(zip(inputs, [input1, input2]))) + + make_zip_of_tests(zip_path, test_parameters, build_graph, build_inputs) + + def _make_logical_tests(op): """Make a set of tests to do logical operations.""" @@ -3416,6 +3616,141 @@ def make_logical_xor_tests(zip_path): return _make_logical_tests(tf.logical_xor)(zip_path) +def make_mirror_pad_tests(zip_path): + """Make a set of tests to do mirror_pad.""" + + test_parameters = [ + { + "input_shape": [[2, 3]], + "padding_matrix": [[[1, 1], [2, 1]]], + "mode": ["REFLECT"], + "type": ["const"] + }, + { + "input_shape": [[2, 3]], + "padding_matrix": [[[1, 1], [1, 1]]], + "mode": ["REFLECT"], + "type": ["const"] + }, + { + "input_shape": [[2, 3]], + "padding_matrix": [[[1, 1], [2, 1]]], + "mode": ["SYMMETRIC"], + "type": ["placeholder"] + }, + { + "input_shape": [[2, 3]], + "padding_matrix": [[[1, 1], [2, 1]]], + "mode": ["REFLECT"], + "type": ["placeholder"] + }, + { + "input_shape": [[3]], + "padding_matrix": [[[0, 2]]], + "mode": ["SYMMETRIC"], + "type": ["placeholder"] + }, + { + "input_shape": [[3]], + "padding_matrix": [[[0, 2]]], + "mode": ["SYMMETRIC"], + "type": ["const"] + }, + { + "input_shape": [[3]], + "padding_matrix": [[[0, 2]]], + "mode": ["REFLECT"], + "type": ["const"] + }, + ] + + def build_graph(parameters): + """Build the graph for the test case.""" + + input_tensor = tf.placeholder( + dtype=tf.int32, name="input", shape=parameters["input_shape"]) + if parameters["type"] != "const": + padding_matrix = tf.placeholder( + dtype=tf.int32, + name="padding", + shape=[len(parameters["input_shape"]), 2]) + input_tensors = [input_tensor, padding_matrix] + else: + padding_matrix = tf.constant(np.array(parameters["padding_matrix"])) + input_tensors = [input_tensor] + output = tf.pad( + input_tensor, paddings=padding_matrix, mode=parameters["mode"]) + + return input_tensors, [output] + + def build_inputs(parameters, sess, inputs, outputs): + input_values = [create_tensor_data(tf.int32, parameters["input_shape"])] + if parameters["type"] != "const": + input_values.append(np.array(parameters["padding_matrix"])) + return input_values, sess.run( + outputs, feed_dict=dict(zip(inputs, input_values))) + + make_zip_of_tests( + zip_path, + test_parameters, + build_graph, + build_inputs, + expected_tf_success=7) + + +def make_unroll_batch_matmul_tests(zip_path): + """Make a set of tests to test unroll_batch_matmul.""" + + test_parameters = [{"dtype": [tf.float32], "shape": [[(2, 2, 3), (2, 3, 2)]]}] + + def build_graph(parameters): + """Build the batch_matmul op testing graph.""" + input_tensor1 = tf.placeholder( + dtype=parameters["dtype"], shape=parameters["shape"][0]) + input_tensor2 = tf.placeholder( + dtype=parameters["dtype"], shape=parameters["shape"][1]) + # Should be unrolled and replaced with fully_connected ops in the end. + out = tf.matmul(input_tensor1, input_tensor2) + return [input_tensor1, input_tensor2], [out] + + def build_inputs(parameters, sess, inputs, outputs): + input_value1 = create_tensor_data( + parameters["dtype"], shape=parameters["shape"][0]) + input_value2 = create_tensor_data( + parameters["dtype"], shape=parameters["shape"][1]) + return [input_value1, input_value2], sess.run( + outputs, feed_dict=dict(zip(inputs, [input_value1, input_value2]))) + + make_zip_of_tests(zip_path, test_parameters, build_graph, build_inputs) + + +def make_placeholder_with_default_tests(zip_path): + """Make a set of tests to test placeholder_with_default.""" + + test_parameters = [{ + "dtype": [tf.float32, tf.int32, tf.int64], + }] + + def build_graph(parameters): + """Build the placeholder_with_default testing graph.""" + const_node = tf.constant( + [1, 2, 2, 0], shape=[2, 2], dtype=parameters["dtype"]) + input_tensor = tf.placeholder_with_default( + const_node, shape=[2, 2], name="input") + out = tf.equal(input_tensor, const_node, name="output") + + return [input_tensor], [out] + + def build_inputs(parameters, sess, inputs, outputs): + numpy_type = _TF_TYPE_INFO[parameters["dtype"]][0] + input_value = np.array([[1, 0], [2, 1]], numpy_type) + return [input_value], sess.run( + outputs, feed_dict=dict(zip(inputs, [input_value]))) + + make_zip_of_tests(zip_path, test_parameters, build_graph, build_inputs, + expected_tf_success=3) + + # Toco binary path provided by the generate rule. bin_path = None diff --git a/tensorflow/lite/testing/generated_examples_zip_test.cc b/tensorflow/lite/testing/generated_examples_zip_test.cc index aedea52065f..91a4851fb02 100644 --- a/tensorflow/lite/testing/generated_examples_zip_test.cc +++ b/tensorflow/lite/testing/generated_examples_zip_test.cc @@ -101,6 +101,10 @@ std::map kBrokenTests = { {R"(^\/mul.*dtype=tf\.int64)", "119126484"}, {R"(^\/add.*dtype=tf\.int64)", "119126484"}, {R"(^\/floor_div.*dtype=tf\.int64)", "119126484"}, + {R"(^\/squared_difference.*dtype=tf\.int64)", "119126484"}, + + // Strided Slice chooses the wrong dimension. + {R"(^\/strided_slice_buggy)", "119786029"}, }; // Allows test data to be unarchived into a temporary directory and makes diff --git a/tensorflow/lite/testing/tflite_driver.cc b/tensorflow/lite/testing/tflite_driver.cc index 3a0febb780c..27e3a3770bb 100644 --- a/tensorflow/lite/testing/tflite_driver.cc +++ b/tensorflow/lite/testing/tflite_driver.cc @@ -147,9 +147,10 @@ TfLiteDriver::TfLiteDriver(bool use_nnapi, const string& delegate_name) } TfLiteDriver::~TfLiteDriver() { - for (TfLiteTensor* t : tensors_to_deallocate_) { - free(t->data.raw); + for (auto t : tensors_to_deallocate_) { + DeallocateStringTensor(t.second); } + interpreter_.reset(); } void TfLiteDriver::AllocateTensors() { @@ -242,12 +243,10 @@ void TfLiteDriver::SetInput(int id, const string& csv_values) { case kTfLiteString: { string s = absl::HexStringToBytes(csv_values); - tensor->data.raw = reinterpret_cast(malloc(s.size())); - tensor->bytes = s.size(); + DeallocateStringTensor(tensors_to_deallocate_[id]); + AllocateStringTensor(id, s.size(), tensor); memcpy(tensor->data.raw, s.data(), s.size()); - // We must remember to free the memory we allocated above. - tensors_to_deallocate_.push_back(tensor); break; } default: diff --git a/tensorflow/lite/testing/tflite_driver.h b/tensorflow/lite/testing/tflite_driver.h index d8b40565bac..1da0533c57c 100644 --- a/tensorflow/lite/testing/tflite_driver.h +++ b/tensorflow/lite/testing/tflite_driver.h @@ -49,6 +49,18 @@ class TfLiteDriver : public TestRunner { string ReadOutput(int id) override { return "no-op"; } private: + void DeallocateStringTensor(TfLiteTensor* t) { + if (t) { + free(t->data.raw); + t->data.raw = nullptr; + } + } + void AllocateStringTensor(int id, size_t num_bytes, TfLiteTensor* t) { + t->data.raw = reinterpret_cast(malloc(num_bytes)); + t->bytes = num_bytes; + tensors_to_deallocate_[id] = t; + } + void ResetLSTMStateTensors(); class Expectation; @@ -59,7 +71,7 @@ class TfLiteDriver : public TestRunner { std::unique_ptr interpreter_; std::map> expected_output_; bool must_allocate_tensors_ = true; - std::vector tensors_to_deallocate_; + std::map tensors_to_deallocate_; }; } // namespace testing diff --git a/tensorflow/lite/toco/BUILD b/tensorflow/lite/toco/BUILD index 14302874441..82aa1f557ef 100644 --- a/tensorflow/lite/toco/BUILD +++ b/tensorflow/lite/toco/BUILD @@ -395,9 +395,10 @@ tf_cc_test( # :toco is the main public command-line tool exposing the functionality # of the :toco_tooling library. -tf_cc_binary( - name = "toco", - srcs = ["toco.cc"], +cc_library( + name = "toco_convert", + srcs = ["toco_convert.cc"], + hdrs = ["toco_convert.h"], visibility = ["//visibility:public"], deps = [ ":model", @@ -416,6 +417,51 @@ tf_cc_binary( ], ) +tf_cc_binary( + name = "toco", + srcs = ["toco.cc"], + visibility = ["//visibility:public"], + deps = [ + ":model", + ":model_cmdline_flags", + ":model_flags_proto_cc", + ":toco_cmdline_flags", + ":toco_convert", + ":toco_flags_proto_cc", + ":toco_port", + ":toco_tooling", + ":types_proto_cc", + "@com_google_absl//absl/strings", + "//tensorflow/core:lib", + # We cannot embed the core:ops dependency directly into :toco_tooling as + # it can conflict with downstream deps when toco is used as a library. + "//tensorflow/core:ops", + ], +) + +tf_cc_test( + name = "toco_convert_test", + srcs = ["toco_convert_test.cc"], + visibility = ["//visibility:public"], + deps = [ + ":model", + ":model_cmdline_flags", + ":model_flags_proto_cc", + ":toco_cmdline_flags", + ":toco_convert", + ":toco_flags_proto_cc", + ":toco_port", + ":toco_tooling", + ":types_proto_cc", + "@com_google_googletest//:gtest_main", + "@com_google_absl//absl/strings", + "//tensorflow/core:lib", + # We cannot embed the core:ops dependency directly into :toco_tooling as + # it can conflict with downstream deps when toco is used as a library. + "//tensorflow/core:ops", + ], +) + tf_cc_test( name = "toco_port_test", srcs = ["toco_port_test.cc"], diff --git a/tensorflow/lite/toco/README.md b/tensorflow/lite/toco/README.md index 91f6f618a37..fe98a90b385 100644 --- a/tensorflow/lite/toco/README.md +++ b/tensorflow/lite/toco/README.md @@ -8,9 +8,9 @@ the usage documentation. Usage information is given in these documents: -* [Command-line glossary](g3doc/cmdline_reference.md) -* [Command-line examples](g3doc/cmdline_examples.md) -* [Python API examples](g3doc/python_api.md) +* [Command-line glossary](../g3doc/convert/cmdline_reference.md) +* [Command-line examples](../g3doc/convert/cmdline_examples.md) +* [Python API examples](../g3doc/convert/python_api.md) ## Where the converter fits in the TensorFlow landscape @@ -26,4 +26,4 @@ to client devices, generally mobile devices, where the TensorFlow Lite interpreter handles them on-device. This flow is represented in the diagram below. -![drawing](g3doc/toco_landscape.svg) +![drawing](../g3doc/images/convert/workflow.svg) diff --git a/tensorflow/lite/toco/export_tensorflow.cc b/tensorflow/lite/toco/export_tensorflow.cc index 1752745aaee..9fff0015527 100644 --- a/tensorflow/lite/toco/export_tensorflow.cc +++ b/tensorflow/lite/toco/export_tensorflow.cc @@ -48,7 +48,8 @@ using tensorflow::TensorProto; namespace toco { namespace { -tensorflow::DataType GetTensorFlowDataType(ArrayDataType data_type) { +tensorflow::DataType GetTensorFlowDataType(ArrayDataType data_type, + const string& error_location) { switch (data_type) { case ArrayDataType::kBool: return tensorflow::DT_BOOL; @@ -66,14 +67,21 @@ tensorflow::DataType GetTensorFlowDataType(ArrayDataType data_type) { return tensorflow::DT_COMPLEX64; default: case ArrayDataType::kNone: - LOG(FATAL) << "Unsupported data type: " << static_cast(data_type); + LOG(FATAL) << "Unsupported data type '" << ArrayDataTypeName(data_type) + << "' in " << error_location; return tensorflow::DT_INVALID; } } +tensorflow::DataType GetTensorFlowDataTypeForOp(ArrayDataType data_type, + const string& op_name) { + return GetTensorFlowDataType(data_type, "op '" + op_name + "'"); +} + tensorflow::DataType GetTensorFlowDataType(const Model& model, const string& array_name) { - return GetTensorFlowDataType(model.GetArray(array_name).data_type); + return GetTensorFlowDataType(model.GetArray(array_name).data_type, + "array '" + array_name + "'"); } // TensorFlow sometimes forbids what it calls "legacy scalars", @@ -1150,6 +1158,29 @@ void ConvertSplitOperator(const Model& model, tensorflow_graph); } +void ConvertSplitVOperator(const Model& model, + const TensorFlowSplitVOperator& src_op, + GraphDef* tensorflow_graph) { + tensorflow::NodeDef* split_v_op = tensorflow_graph->add_node(); + split_v_op->set_op("SplitV"); + split_v_op->set_name(src_op.outputs[0]); + for (const auto& input : src_op.inputs) { + *split_v_op->add_input() = input; + } + (*split_v_op->mutable_attr())["T"].set_type( + GetTensorFlowDataType(model, src_op.inputs[0])); + (*split_v_op->mutable_attr())["num_split"].set_i(src_op.num_split); + const auto& split_dim_array = model.GetArray(src_op.inputs[1]); + CHECK(split_dim_array.buffer); + CHECK(split_dim_array.data_type == ArrayDataType::kInt32); + const auto& split_dim_data = + split_dim_array.GetBuffer().data; + CHECK_EQ(split_dim_data.size(), 1); + const int split_dim = split_dim_data[0]; + CreateDummyConcatDimTensorConst(src_op.inputs[0], split_dim, + tensorflow_graph); +} + void ConvertCastOperator(const Model& model, const CastOperator& src_op, GraphDef* tensorflow_graph) { tensorflow::NodeDef* cast_op = tensorflow_graph->add_node(); @@ -1285,7 +1316,7 @@ void ConvertRangeOperator(const Model& model, const RangeOperator& src_op, *range_op->add_input() = src_op.inputs[1]; *range_op->add_input() = src_op.inputs[2]; (*range_op->mutable_attr())["Tidx"].set_type( - GetTensorFlowDataType(src_op.dtype)); + GetTensorFlowDataTypeForOp(src_op.dtype, /*op_name=*/src_op.outputs[0])); } void ConvertPackOperator(const Model& model, const PackOperator& src_op, @@ -1298,7 +1329,8 @@ void ConvertPackOperator(const Model& model, const PackOperator& src_op, } (*pack_op->mutable_attr())["axis"].set_i(src_op.axis); (*pack_op->mutable_attr())["N"].set_i(src_op.inputs.size()); - (*pack_op->mutable_attr())["T"].set_type(GetTensorFlowDataType(src_op.dtype)); + (*pack_op->mutable_attr())["T"].set_type( + GetTensorFlowDataTypeForOp(src_op.dtype, src_op.outputs[0])); } void ConvertFillOperator(const Model& model, const FillOperator& src_op, @@ -1887,7 +1919,7 @@ void ConvertRandomUniformOperator(const Model& model, GetTensorFlowDataType(model, src_op.inputs[0]); (*new_op->mutable_attr())["T"].set_type(shape_type); (*new_op->mutable_attr())["dtype"].set_type( - GetTensorFlowDataType(src_op.dtype)); + GetTensorFlowDataTypeForOp(src_op.dtype, src_op.outputs[0])); (*new_op->mutable_attr())["seed"].set_i(src_op.seed); (*new_op->mutable_attr())["seed2"].set_i(src_op.seed2); } @@ -2124,6 +2156,10 @@ void ConvertOperator(const Model& model, const Operator& src_op, ConvertSplitOperator(model, static_cast(src_op), tensorflow_graph); + } else if (src_op.type == OperatorType::kSplitV) { + ConvertSplitVOperator(model, + static_cast(src_op), + tensorflow_graph); } else if (src_op.type == OperatorType::kFakeQuant) { ConvertFakeQuantOperator(static_cast(src_op), tensorflow_graph); diff --git a/tensorflow/lite/toco/graph_transformations/fuse_binary_into_following_affine.cc b/tensorflow/lite/toco/graph_transformations/fuse_binary_into_following_affine.cc index 6b4765b23c4..436b639253f 100644 --- a/tensorflow/lite/toco/graph_transformations/fuse_binary_into_following_affine.cc +++ b/tensorflow/lite/toco/graph_transformations/fuse_binary_into_following_affine.cc @@ -221,9 +221,8 @@ void FuseMulOrDivParamsIntoFollowingAffine(Model* model, Operator* following_op, Operator* following_op = GetOpWithInput(*model, binary_op->outputs[0]); if (!following_op) { - AddMessageF( - "Not fusing %s because it is not consumed by exactly one other op", - LogName(*binary_op)); + AddMessageF("Not fusing %s because it is not consumed by any op", + LogName(*binary_op)); return ::tensorflow::Status::OK(); } @@ -288,7 +287,10 @@ void FuseMulOrDivParamsIntoFollowingAffine(Model* model, Operator* following_op, AddMessageF("Fusing %s into the following %s", LogName(*binary_op), LogName(*following_op)); - model->EraseArray(binary_op->outputs[0]); + if (CountOpsWithInput(*model, binary_op->outputs[0]) == 1) { + model->EraseArray(binary_op->outputs[0]); + } + following_op->inputs[0] = binary_op->inputs[index_of_variable_input]; const auto& old_constant_param_name = binary_op->inputs[index_of_constant_input]; diff --git a/tensorflow/lite/toco/graph_transformations/propagate_array_data_types.cc b/tensorflow/lite/toco/graph_transformations/propagate_array_data_types.cc index 9a458dccb9c..cbae6610d7f 100644 --- a/tensorflow/lite/toco/graph_transformations/propagate_array_data_types.cc +++ b/tensorflow/lite/toco/graph_transformations/propagate_array_data_types.cc @@ -86,6 +86,13 @@ void SetDataTypeForAllOutputs(Model* model, Operator* op, SetDataTypeForAllOutputs(model, op, data_type); break; } + case OperatorType::kSplitV: { + // These operators produce output with the same type as its 1st input + CHECK_GE(op->inputs.size(), 3); + const ArrayDataType data_type = model->GetArray(op->inputs[0]).data_type; + SetDataTypeForAllOutputs(model, op, data_type); + break; + } case OperatorType::kTransposeConv: { // These operators produce an output with the same type as their 3rd input CHECK_GE(op->inputs.size(), 3); diff --git a/tensorflow/lite/toco/graph_transformations/propagate_fixed_sizes.cc b/tensorflow/lite/toco/graph_transformations/propagate_fixed_sizes.cc index 78ea54e452b..0e653f08a04 100644 --- a/tensorflow/lite/toco/graph_transformations/propagate_fixed_sizes.cc +++ b/tensorflow/lite/toco/graph_transformations/propagate_fixed_sizes.cc @@ -15,6 +15,7 @@ limitations under the License. #include #include #include +#include #include #include #include @@ -786,6 +787,97 @@ void ProcessTensorFlowSplitOperator(Model* model, TensorFlowSplitOperator* op) { } } +void ProcessTensorFlowSplitVOperator(Model* model, + TensorFlowSplitVOperator* op) { + CHECK_EQ(op->inputs.size(), 3); + + const auto& input_array = model->GetArray(op->inputs[0]); + // Yield until input dims have been resolved. + if (!input_array.has_shape()) { + return; + } + const Shape& input_shape = input_array.shape(); + + // Yield until size_splits is constant. + if (!IsConstantParameterArray(*model, op->inputs[1])) { + return; + } + const auto& size_array = model->GetArray(op->inputs[1]); + // Yield until size_splits dims have been resolved. + if (!size_array.has_shape()) { + return; + } + const Shape& size_shape = size_array.shape(); + + CHECK(size_array.data_type == ArrayDataType::kInt32 || + size_array.data_type == ArrayDataType::kInt64) + << "size_splits must be int32, int64"; + CHECK_EQ(size_shape.dimensions_count(), 1) << "size_splits must be 1-D"; + + std::vector size_splits_vector; + if (size_array.data_type == ArrayDataType::kInt32) { + for (const auto each_size : + size_array.GetBuffer().data) { + size_splits_vector.push_back(each_size); + } + } else { + size_splits_vector = size_array.GetBuffer().data; + } + + // Yield until axis is constant. + if (!IsConstantParameterArray(*model, op->inputs[2])) { + return; + } + const auto& axis_array = model->GetArray(op->inputs[2]); + // Yield until axis dims have been resolved. + if (!axis_array.has_shape()) { + return; + } + + CHECK(axis_array.data_type == ArrayDataType::kInt32) + << "Axis array must be int32."; + CHECK_EQ(RequiredBufferSizeForShape(axis_array.shape()), 1) + << "Axis array must be scalar."; + + int axis = axis_array.GetBuffer().data[0]; + if (axis < 0) { + axis += input_shape.dimensions_count(); + } + + CHECK_EQ(op->num_split, size_splits_vector.size()); + + int64_t minus_one_count = 0, size_splits_sum = 0; + for (auto size : size_splits_vector) { + if (size == -1) { + ++minus_one_count; + } else { + size_splits_sum += size; + } + } + + const int input_size = input_shape.dims(axis); + + CHECK_LE(minus_one_count, 1) << "size_splits can contain at most one -1."; + + if (minus_one_count == 1) { + CHECK_LE(size_splits_sum, input_size); + auto iter = + std::find(size_splits_vector.begin(), size_splits_vector.end(), -1); + *iter = input_size - size_splits_sum; + } else { + CHECK_EQ(size_splits_sum, input_size); + } + + CHECK_EQ(op->outputs.size(), op->num_split); + + for (int i = 0; i < op->outputs.size(); ++i) { + const auto& output = op->outputs[i]; + Shape output_shape = input_shape; + (*output_shape.mutable_dims())[axis] = size_splits_vector.at(i); + model->GetArray(output).copy_shape(output_shape); + } +} + void ProcessAveragePoolOperator(Model* model, AveragePoolOperator* op) { const string& input_name = op->inputs[0]; const auto& input_array = model->GetArray(input_name); @@ -1691,6 +1783,51 @@ void ProcessUnpackOperator(Model* model, UnpackOperator* op) { } } +void ProcessMirrorPadOperator(Model* model, MirrorPadOperator* op) { + CHECK_EQ(op->inputs.size(), 2); + const auto& input_array = model->GetArray(op->inputs[0]); + const auto& padding_matrix = model->GetArray(op->inputs[1]); + + // Yield until input dims have been resolved. + if (!input_array.has_shape()) { + return; + } + + auto& output_array = model->GetArray(op->outputs[0]); + // If output already computed or padding matrix is non + // const then return. + if (output_array.has_shape() || + !IsConstantParameterArray(*model, op->inputs[1])) { + return; + } + Shape output_shape = input_array.shape(); + std::vector& dims = *output_shape.mutable_dims(); + + std::vector padding; + if (padding_matrix.data_type == ArrayDataType::kInt32) { + const auto& data = padding_matrix.GetBuffer().data; + for (auto elem : data) { + padding.push_back(static_cast(elem)); + } + } else if (padding_matrix.data_type == ArrayDataType::kInt64) { + const auto& data = padding_matrix.GetBuffer().data; + for (auto elem : data) { + padding.push_back(elem); + } + } else { + CHECK(padding_matrix.data_type == ArrayDataType::kInt64 || + padding_matrix.data_type == ArrayDataType::kInt32); + } + CHECK_EQ(padding_matrix.shape().dimensions_count(), 2); + CHECK_EQ(input_array.shape().dimensions_count(), + padding_matrix.shape().dims(0)); + for (int i = 0; i < input_array.shape().dimensions_count(); ++i) { + dims[i] += padding[i * 2] + padding[i * 2 + 1]; + } + + output_array.copy_shape(output_shape); +} + } // namespace ::tensorflow::Status PropagateFixedSizes::Run(Model* model, @@ -1707,6 +1844,7 @@ void ProcessUnpackOperator(Model* model, UnpackOperator* op) { } switch (op->type) { + case OperatorType::kAbs: case OperatorType::kBatchNormalization: case OperatorType::kL2Normalization: case OperatorType::kDequantize: @@ -1714,6 +1852,7 @@ void ProcessUnpackOperator(Model* model, UnpackOperator* op) { case OperatorType::kRelu1: case OperatorType::kRelu6: case OperatorType::kPRelu: + case OperatorType::kLeakyRelu: case OperatorType::kSoftmax: case OperatorType::kLogSoftmax: case OperatorType::kLog: @@ -1759,6 +1898,7 @@ void ProcessUnpackOperator(Model* model, UnpackOperator* op) { case OperatorType::kEqual: case OperatorType::kNotEqual: case OperatorType::kPow: + case OperatorType::kSquaredDifference: ProcessSimpleBinaryOperator(model, op); break; case OperatorType::kAddN: @@ -1834,6 +1974,10 @@ void ProcessUnpackOperator(Model* model, UnpackOperator* op) { ProcessTensorFlowSplitOperator(model, static_cast(op)); break; + case OperatorType::kSplitV: + ProcessTensorFlowSplitVOperator( + model, static_cast(op)); + break; case OperatorType::kSqueeze: ProcessSqueezeOperator(model, static_cast(op)); break; @@ -1956,6 +2100,9 @@ void ProcessUnpackOperator(Model* model, UnpackOperator* op) { case OperatorType::kUnpack: ProcessUnpackOperator(model, static_cast(op)); break; + case OperatorType::kMirrorPad: + ProcessMirrorPadOperator(model, static_cast(op)); + break; default: // Unimplemented, another graph transformation should drop it. LOG(FATAL) << "Unhandled operator type " << OperatorTypeName(op->type); diff --git a/tensorflow/lite/toco/graph_transformations/quantize.cc b/tensorflow/lite/toco/graph_transformations/quantize.cc index e28b7288f01..1146078c301 100644 --- a/tensorflow/lite/toco/graph_transformations/quantize.cc +++ b/tensorflow/lite/toco/graph_transformations/quantize.cc @@ -64,7 +64,8 @@ bool SupportsQuantization(const Operator& op) { type == OperatorType::kRelu1 || type == OperatorType::kRelu6 || type == OperatorType::kShape || type == OperatorType::kExpandDims || type == OperatorType::kPack || type == OperatorType::kTopK_V2 || - type == OperatorType::kResizeNearestNeighbor; + type == OperatorType::kResizeNearestNeighbor || + type == OperatorType::kPRelu; } // The quantized op allows output arrays of type float using @@ -360,7 +361,7 @@ bool ChooseQuantizationForOperatorOutput( op.type == OperatorType::kSpaceToDepth || op.type == OperatorType::kReshape || op.type == OperatorType::kSplit || op.type == OperatorType::kRelu || op.type == OperatorType::kRelu1 || - op.type == OperatorType::kRelu6) { + op.type == OperatorType::kRelu6 || op.type == OperatorType::kPRelu) { int data_input_index = 0; if (op.type == OperatorType::kSplit) { data_input_index = 1; diff --git a/tensorflow/lite/toco/graph_transformations/resolve_constant_gather.cc b/tensorflow/lite/toco/graph_transformations/resolve_constant_gather.cc index 1149930131e..c72135923e5 100644 --- a/tensorflow/lite/toco/graph_transformations/resolve_constant_gather.cc +++ b/tensorflow/lite/toco/graph_transformations/resolve_constant_gather.cc @@ -47,6 +47,10 @@ inline void Gather(const Array& input_array, int input_rank, stride *= input_shape.dims(i); } + // Let's make sure we have enough space for all element in the memcpy() + // below, which writes 'stride' elements startng at 'i * stride'. + CHECK_EQ(stride * coords_shape.dims(0), output_data.size()); + for (int i = 0; i < coords_shape.dims(0); ++i) { DCHECK_GE(coords_data[i], 0); DCHECK_LT(coords_data[i], input_shape.dims(rev_input_rank)); diff --git a/tensorflow/lite/toco/graph_transformations/resolve_reduce_attributes.cc b/tensorflow/lite/toco/graph_transformations/resolve_reduce_attributes.cc index ea5d33009b4..9ceba45e93f 100644 --- a/tensorflow/lite/toco/graph_transformations/resolve_reduce_attributes.cc +++ b/tensorflow/lite/toco/graph_transformations/resolve_reduce_attributes.cc @@ -35,6 +35,11 @@ bool ResolveAttributes(Model* model, T* op) { const Array& indices_array = model->GetArray(op->inputs[1]); if (!indices_array.has_shape()) return false; + + // It is ok for indices_array to have a shape for an empty tensor. In that + // case, we don't bother setting 'axis'. + if (indices_array.buffer->Length() == 0) return false; + op->axis = indices_array.GetBuffer().data; return true; } diff --git a/tensorflow/lite/toco/graph_transformations/unroll_batch_matmul.cc b/tensorflow/lite/toco/graph_transformations/unroll_batch_matmul.cc index d59954fc740..41a735394d7 100644 --- a/tensorflow/lite/toco/graph_transformations/unroll_batch_matmul.cc +++ b/tensorflow/lite/toco/graph_transformations/unroll_batch_matmul.cc @@ -117,7 +117,8 @@ namespace toco { auto* slice_b_op = new SliceOperator; slice_b_op->inputs = { batch_op->inputs[1], - CreateInt32Array(model, batch_name + "/slice_b/slice/begin", {0, 0, 0}), + CreateInt32Array(model, batch_name + "/slice_b/slice/begin", + {batch, 0, 0}), CreateInt32Array( model, batch_name + "/slice_b/slice/size", {1, input_array_b.shape().dims(1), input_array_b.shape().dims(2)}), diff --git a/tensorflow/lite/toco/import_tensorflow.cc b/tensorflow/lite/toco/import_tensorflow.cc index 96f3c6a6ab9..0b2f8103943 100644 --- a/tensorflow/lite/toco/import_tensorflow.cc +++ b/tensorflow/lite/toco/import_tensorflow.cc @@ -935,6 +935,25 @@ tensorflow::Status ConvertSplitOperator( return tensorflow::Status::OK(); } +tensorflow::Status ConvertSplitVOperator( + const NodeDef& node, const TensorFlowImportFlags& tf_import_flags, + Model* model) { + CHECK_EQ(node.op(), "SplitV"); + TF_QCHECK_OK(CheckInputsCount(node, tf_import_flags, 3)); + auto* op = new TensorFlowSplitVOperator; + op->inputs.push_back(node.input(0)); + op->inputs.push_back(node.input(1)); + op->inputs.push_back(node.input(2)); + const int num_split = GetIntAttr(node, "num_split"); + op->outputs.push_back(node.name()); + for (int i = 1; i < num_split; i++) { + op->outputs.push_back(absl::StrCat(node.name(), ":", i)); + } + op->num_split = num_split; + model->operators.emplace_back(op); + return tensorflow::Status::OK(); +} + tensorflow::Status ConvertSwitchOperator( const NodeDef& node, const TensorFlowImportFlags& tf_import_flags, Model* model) { @@ -1134,6 +1153,31 @@ tensorflow::Status ConvertConcatOperator( return tensorflow::Status::OK(); } +tensorflow::Status ConvertMirrorPadOperator( + const NodeDef& node, const TensorFlowImportFlags& tf_import_flags, + Model* model) { + if (node.op() != "MirrorPad") { + LOG(FATAL) << "Expected MirrorPad."; + } + const int num_inputs = GetInputsCount(node, tf_import_flags); + CHECK_EQ(num_inputs, 2); + auto* op = new MirrorPadOperator; + for (int i = 0; i < num_inputs; ++i) { + op->inputs.push_back(node.input(i)); + } + op->outputs.push_back(node.name()); + const auto mode = GetStringAttr(node, "mode"); + if (mode == "REFLECT") { + op->mode = toco::MirrorPadMode::kReflect; + } else if (mode == "SYMMETRIC") { + op->mode = toco::MirrorPadMode::kSymmetric; + } + + model->operators.emplace_back(op); + + return tensorflow::Status::OK(); +} + static constexpr int kAnyNumInputs = -1; enum FlexSupport { kFlexOk, kFlexNotOk }; @@ -1221,7 +1265,7 @@ void GetOutputNamesFromNodeDef(const NodeDef& node, void GetOutputTypesFromNodeDef(const NodeDef& node, const tensorflow::OpDef& op_def, TensorFlowUnsupportedOperator* op) { - // The the given type to the op, or clear the types if invalid. + // The given type to the op, or clear the types if invalid. auto add_type = [&node, op](tensorflow::DataType type) { if (type == tensorflow::DT_INVALID) { LOG(WARNING) << "Op node missing output type attribute: " << node.name(); @@ -2012,13 +2056,13 @@ bool InlineAllFunctions(GraphDef* graphdef) { tensorflow::SessionOptions options; auto* device_count = options.config.mutable_device_count(); device_count->insert({"CPU", 1}); - std::vector devices; + std::vector> devices; TF_CHECK_OK(tensorflow::DeviceFactory::AddDevices( options, "/job:localhost/replica:0/task:0", &devices)); tensorflow::FunctionLibraryDefinition fld(tensorflow::OpRegistry::Global(), graphdef_copy.library()); - tensorflow::DeviceMgr device_mgr(devices); + tensorflow::DeviceMgr device_mgr(std::move(devices)); tensorflow::OptimizerOptions o_opts; tensorflow::ProcessFunctionLibraryRuntime pflr( &device_mgr, tensorflow::Env::Default(), TF_GRAPH_DEF_VERSION, &fld, @@ -2220,6 +2264,21 @@ tensorflow::Status ConvertUnidirectionalSequenceLstm( return tensorflow::Status::OK(); } +tensorflow::Status ConvertLeakyReluOperator( + const NodeDef& node, const TensorFlowImportFlags& tf_import_flags, + Model* model) { + CHECK_EQ(node.op(), "LeakyRelu"); + TF_QCHECK_OK(CheckInputsCount(node, tf_import_flags, 1)); + CHECK_EQ(GetDataTypeAttr(node, "T"), DT_FLOAT); + const auto& input_name = node.input(0); + auto* op = new LeakyReluOperator; + op->inputs.push_back(input_name); + op->outputs.push_back(node.name()); + op->alpha = GetFloatAttr(node, "alpha"); + model->operators.emplace_back(op); + return tensorflow::Status::OK(); +} + } // namespace namespace internal { @@ -2233,12 +2292,14 @@ ConverterMapType GetTensorFlowNodeConverterMapForFlex() { return std::unordered_map({ // We need to let TCO convert Placeholder information into // array data, so that the data types are correct. + {"LegacyFedInput", ConvertPlaceholderOperator}, {"Placeholder", ConvertPlaceholderOperator}, }); } ConverterMapType GetTensorFlowNodeConverterMap() { return std::unordered_map({ + {"Abs", ConvertSimpleOperator}, {"Add", ConvertSimpleOperator}, {"AddN", ConvertSimpleOperatorFlexOk}, {"All", ConvertSimpleOperator}, @@ -2282,6 +2343,7 @@ ConverterMapType GetTensorFlowNodeConverterMap() { ConvertSimpleOperator}, {"Identity", ConvertIdentityOperator}, {"LRN", ConvertLRNOperator}, + {"LeakyRelu", ConvertLeakyReluOperator}, {"LegacyFedInput", ConvertPlaceholderOperator}, {"Less", ConvertSimpleOperator}, {"LessEqual", ConvertSimpleOperator}, @@ -2332,8 +2394,11 @@ ConverterMapType GetTensorFlowNodeConverterMap() { {"SpaceToDepth", ConvertSpaceToDepthOperator}, {"SparseToDense", ConvertSparseToDenseOperator}, {"Split", ConvertSplitOperator}, + {"SplitV", ConvertSplitVOperator}, {"Sqrt", ConvertSimpleOperator}, {"Square", ConvertSimpleOperator}, + {"SquaredDifference", + ConvertSimpleOperator}, {"Squeeze", ConvertSqueezeOperator}, {"StopGradient", ConvertIdentityOperator}, {"StridedSlice", ConvertStridedSliceOperator}, @@ -2349,6 +2414,7 @@ ConverterMapType GetTensorFlowNodeConverterMap() { {"Unpack", ConvertUnpackOperator}, {"ZerosLike", ConvertSimpleOperator}, {"UnidirectionalSequenceLstm", ConvertUnidirectionalSequenceLstm}, + {"MirrorPad", ConvertMirrorPadOperator}, }); } diff --git a/tensorflow/lite/toco/model.h b/tensorflow/lite/toco/model.h index f85e1c28787..f19355968fa 100644 --- a/tensorflow/lite/toco/model.h +++ b/tensorflow/lite/toco/model.h @@ -121,8 +121,10 @@ enum class OperatorType : uint8 { kRsqrt, kShape, kSplit, + kSplitV, kSqrt, kSquare, + kSquaredDifference, kSum, kSwitch, kTile, @@ -152,7 +154,10 @@ enum class OperatorType : uint8 { kCTCBeamSearchDecoder, kUnpack, kZerosLike, - kResizeNearestNeighbor + kResizeNearestNeighbor, + kLeakyRelu, + kAbs, + kMirrorPad }; // Helper to deal with TensorFlow arrays using a different ordering of @@ -653,6 +658,17 @@ struct MulOperator : Operator { MulOperator() : Operator(OperatorType::kMul) {} }; +// Element-wise Abs operator: +// x -> abs(x) +// +// Inputs: +// inputs[0]: required: the input array +// +// TensorFlow equivalent: Relu +struct AbsOperator : Operator { + AbsOperator() : Operator(OperatorType::kAbs) {} +}; + // Element-wise Relu operator: // x -> max(0, x) // @@ -699,6 +715,19 @@ struct PReluOperator : Operator { PReluOperator() : Operator(OperatorType::kPRelu) {} }; +// LeakyRelu +// x -> max(x, alpha * x) +// +// Inputs: +// inputs[0]: required: the input array +// +// TensorFlow equivalent: LeakyRelu +struct LeakyReluOperator : Operator { + LeakyReluOperator() : Operator(OperatorType::kLeakyRelu) {} + + float alpha = 0.2f; // 0.2 matches the default value for the TF op attribute. +}; + // Element-wise Logistic operator: // x -> Logistic(x) = 1 / (1 + exp(-x)) // @@ -1289,6 +1318,17 @@ struct TensorFlowSquareOperator : Operator { TensorFlowSquareOperator() : Operator(OperatorType::kSquare) {} }; +// Element-wise squared difference ((x-y)*(x-y)) operator. +// +// Inputs: +// inputs[0]: required: the left-hand side array +// inputs[1]: required: the right-hand side array +// +// TensorFlow equivalent: SquaredDifference +struct SquaredDifferenceOperator : Operator { + SquaredDifferenceOperator() : Operator(OperatorType::kSquaredDifference) {} +}; + // Transposes a tensor. // // By default, this operation performs a regular matrix transpose on 2-D input @@ -1363,6 +1403,12 @@ struct TensorFlowSplitOperator : Operator { int num_split = 0; }; +// TensorFlow SplitV equivalent. Refer to TensorFlow documentation for details. +struct TensorFlowSplitVOperator : Operator { + TensorFlowSplitVOperator() : Operator(OperatorType::kSplitV) {} + int num_split = 0; +}; + // TensorFlow Concat equivalent. Refer to TensorFlow documentation for details. // Not fully supported, just a placeholder to handle TensorFlow graphs and // support graph transformations to other operator types by matching sub-graphs. @@ -1887,6 +1933,23 @@ struct TensorFlowZerosLikeOperator : Operator { TensorFlowZerosLikeOperator() : Operator(OperatorType::kZerosLike) {} }; +enum class MirrorPadMode { kNone, kSymmetric, kReflect }; + +// MirrorPad Operator: +// +// Inputs: +// Inputs[0]: required: input tensor to be padded. +// Inputs[1]: required: 2 Column matrix specifying padding sizes. The number of +// rows must be the same as the rank of the input. +// Inputs[2]: required: REFLECT or SYMMETRIC. +// +// TensorFlow equivalent: MirrorPad. +struct MirrorPadOperator : Operator { + MirrorPadOperator() : Operator(OperatorType::kMirrorPad) {} + // mode is either SYMMETRIC or REFLECT. + MirrorPadMode mode; +}; + // Alloc's are used for transient arrays only. An Alloc specifies which interval // of the "transient_data" workspace buffer passed to inference functions, is to // be used for the transient array at hand. The 'start' and 'end' values are diff --git a/tensorflow/lite/toco/tflite/export.cc b/tensorflow/lite/toco/tflite/export.cc index 489c21295ef..8b9448486df 100644 --- a/tensorflow/lite/toco/tflite/export.cc +++ b/tensorflow/lite/toco/tflite/export.cc @@ -126,7 +126,6 @@ OperatorKey::OperatorKey( type_ = builtin_ops.at(name); return; } - // The logic below is all for custom ops or Flex ops. is_custom_op_ = true; type_ = BuiltinOperator_CUSTOM; @@ -332,6 +331,11 @@ Offset>> ExportOperators( std::set* variable_tensor_indices, const ExportParams& params) { variable_tensor_indices->clear(); + auto is_tflite_builtin = [](const BaseOperator* op) { + const auto& tflite_builtins = GetBuiltinOpsMap(); + return (op && tflite_builtins.find(op->name()) != tflite_builtins.end()); + }; + // The operators are in execution order, so we just follow tf.mini order. std::vector> op_vector; for (const auto& op : model.operators) { @@ -360,7 +364,19 @@ Offset>> ExportOperators( auto options = Options::Custom(0); std::vector mutating_input_variables; - if (tflite_op) { + + // It is conceivable that an op is exportable via Serialize() but does not + // have a corresponding TFLITE builtin. In that case, when flex mode is + // enabled we should export it as a flex op, not as a native. + bool export_as_flex_op = !is_tflite_builtin(tflite_op) && + key.is_flex_op() && + !op->tensorflow_node_def.empty(); + if (export_as_flex_op) { + auto fbb = WriteFlexOpOptions(op->tensorflow_node_def); + if (fbb) { + options = Options::Custom(builder->CreateVector(fbb->GetBuffer())); + } + } else if (tflite_op) { options = tflite_op->Serialize(*op, builder); mutating_input_variables = tflite_op->GetMutatingInputVariables(*op); @@ -373,12 +389,13 @@ Offset>> ExportOperators( variable_tensor_indices->insert(variable_tensor_index); } } - } else if (key.is_flex_op() && !op->tensorflow_node_def.empty()) { - auto fbb = WriteFlexOpOptions(op->tensorflow_node_def); - if (fbb) { - options = Options::Custom(builder->CreateVector(fbb->GetBuffer())); - } + } else { + // We don't know much about this op. It doesn't have a serializer and + // it is not supposed to be exported as a flex op. We will treat it as + // a regular custom op: we will still create an operator for it, but it + // will not have any 'options'. } + // The only supported CustomOptionFormat is FLEXBUFFERS now. op_vector.push_back(CreateOperator( *builder, op_index, builder->CreateVector(inputs), diff --git a/tensorflow/lite/toco/tflite/export_test.cc b/tensorflow/lite/toco/tflite/export_test.cc index b6c67772aca..b371296784a 100644 --- a/tensorflow/lite/toco/tflite/export_test.cc +++ b/tensorflow/lite/toco/tflite/export_test.cc @@ -46,6 +46,18 @@ class ExportTest : public ::testing::Test { input_model_.operators.emplace_back(new AddOperator); } else if (name == "Sub") { input_model_.operators.emplace_back(new SubOperator); + } else if (name == "Assert") { + auto* op = new TensorFlowAssertOperator; + + // Even though assert is known to TOCO, it doesn't have a tflite + // serializer, so it has to be exported as a custom op. If we attach a + // NodeDef to it, however, it will be exported as a flex op instead. + ::tensorflow::NodeDef node_def; + node_def.set_name("Assert"); + node_def.set_op("Assert"); + node_def.SerializeToString(&op->tensorflow_node_def); + + input_model_.operators.emplace_back(op); } else { auto* op = new TensorFlowUnsupportedOperator; op->tensorflow_op = name; @@ -232,37 +244,38 @@ class OpSetsTest : public ExportTest { TEST_F(OpSetsTest, BuiltinsOnly) { // --target_op_set=TFLITE_BUILTINS SetAllowedOpSets({kTfLiteBuiltins}); - EXPECT_THAT(ImportExport({"Add", "AdjustHue", "UnrollAndFold"}), + EXPECT_THAT(ImportExport({"Add", "AdjustHue", "UnrollAndFold", "Assert"}), ElementsAre()); EXPECT_THAT(ImportExport({"Add"}), ElementsAre("builtin:ADD")); // --target_op_set=TFLITE_BUILTINS --allow_custom_ops SetAllowedOpSets({kTfLiteBuiltins, kCustomOps}); - EXPECT_THAT( - ImportExport({"Add", "AdjustHue", "UnrollAndFold"}), - ElementsAre("builtin:ADD", "custom:AdjustHue", "custom:UnrollAndFold")); + EXPECT_THAT(ImportExport({"Add", "AdjustHue", "UnrollAndFold", "Assert"}), + ElementsAre("builtin:ADD", "custom:AdjustHue", "custom:Assert", + "custom:UnrollAndFold")); } TEST_F(OpSetsTest, TfSelectOnly) { // --target_op_set=SELECT_TF_OPS SetAllowedOpSets({kSelectTfOps}); - EXPECT_THAT( - ImportExport({"Add", "AdjustHue", "RandomUniform", "UnrollAndFold"}), - ElementsAre()); + EXPECT_THAT(ImportExport({"Add", "AdjustHue", "RandomUniform", + "UnrollAndFold", "Assert"}), + ElementsAre()); EXPECT_THAT(ImportExport({"Add"}), ElementsAre("custom:FlexAdd")); // --target_op_set=SELECT_TF_OPS --allow_custom_ops SetAllowedOpSets({kSelectTfOps, kCustomOps}); EXPECT_THAT( - ImportExport({"Add", "AdjustHue", "RandomUniform", "UnrollAndFold"}), - ElementsAre("custom:AdjustHue", "custom:FlexAdd", + ImportExport( + {"Add", "AdjustHue", "RandomUniform", "UnrollAndFold", "Assert"}), + ElementsAre("custom:AdjustHue", "custom:FlexAdd", "custom:FlexAssert", "custom:FlexRandomUniform", "custom:UnrollAndFold")); } TEST_F(OpSetsTest, BuiltinsAndTfSelect) { // --target_op_set=TFLITE_BUILTINS,SELECT_TF_OPS SetAllowedOpSets({kTfLiteBuiltins, kSelectTfOps}); - EXPECT_THAT(ImportExport({"Add", "AdjustHue", "UnrollAndFold"}), + EXPECT_THAT(ImportExport({"Add", "AdjustHue", "UnrollAndFold", "Assert"}), ElementsAre()); EXPECT_THAT(ImportExport({"Add", "RandomUniform"}), ElementsAre("builtin:ADD", "custom:FlexRandomUniform")); @@ -270,9 +283,10 @@ TEST_F(OpSetsTest, BuiltinsAndTfSelect) { // --target_op_set=TFLITE_BUILTINS,SELECT_TF_OPS --allow_custom_ops SetAllowedOpSets({kTfLiteBuiltins, kSelectTfOps, kCustomOps}); EXPECT_THAT( - ImportExport({"Add", "AdjustHue", "RandomUniform", "UnrollAndFold"}), - ElementsAre("builtin:ADD", "custom:AdjustHue", "custom:FlexRandomUniform", - "custom:UnrollAndFold")); + ImportExport( + {"Add", "AdjustHue", "RandomUniform", "UnrollAndFold", "Assert"}), + ElementsAre("builtin:ADD", "custom:AdjustHue", "custom:FlexAssert", + "custom:FlexRandomUniform", "custom:UnrollAndFold")); } // This test is based on a hypothetical scenario that dilation is supported diff --git a/tensorflow/lite/toco/tflite/operator.cc b/tensorflow/lite/toco/tflite/operator.cc index 84ae4482469..e0faed49271 100644 --- a/tensorflow/lite/toco/tflite/operator.cc +++ b/tensorflow/lite/toco/tflite/operator.cc @@ -978,6 +978,26 @@ class Split int GetVersion(const Operator& op) const override { return 1; } }; +class SplitV + : public BuiltinOperator { + public: + using BuiltinOperator::BuiltinOperator; + + flatbuffers::Offset WriteOptions( + const TocoOperator& op, + flatbuffers::FlatBufferBuilder* builder) const override { + return ::tflite::CreateSplitVOptions(*builder, op.num_split); + } + + void ReadOptions(const TfLiteOptions& options, + TocoOperator* op) const override { + op->num_split = options.num_splits(); + } + + int GetVersion(const Operator& op) const override { return 1; } +}; + class StridedSlice : public BuiltinOperator { + public: + using BuiltinOperator::BuiltinOperator; + flatbuffers::Offset WriteOptions( + const TocoOperator& op, + flatbuffers::FlatBufferBuilder* builder) const override { + return ::tflite::CreateLeakyReluOptions(*builder, op.alpha); + } + void ReadOptions(const TfLiteOptions& options, + TocoOperator* op) const override { + op->alpha = options.alpha(); + } + + int GetVersion(const Operator& op) const override { return 1; } +}; + +class SquaredDifference + : public BuiltinOperator< + SquaredDifferenceOperator, ::tflite::SquaredDifferenceOptions, + ::tflite::BuiltinOptions_SquaredDifferenceOptions> { + public: + using BuiltinOperator::BuiltinOperator; + + flatbuffers::Offset WriteOptions( + const TocoOperator& op, + flatbuffers::FlatBufferBuilder* builder) const override { + return ::tflite::CreateSquaredDifferenceOptions(*builder); + } + + void ReadOptions(const TfLiteOptions& options, + TocoOperator* op) const override {} + + int GetVersion(const Operator& op) const override { return 1; } +}; + +class MirrorPad + : public BuiltinOperator { + public: + using BuiltinOperator::BuiltinOperator; + flatbuffers::Offset WriteOptions( + const TocoOperator& op, + flatbuffers::FlatBufferBuilder* builder) const override { + return ::tflite::CreateMirrorPadOptions( + *builder, op.mode == MirrorPadMode::kReflect + ? ::tflite::MirrorPadMode::MirrorPadMode_REFLECT + : ::tflite::MirrorPadMode::MirrorPadMode_SYMMETRIC); + } + void ReadOptions(const TfLiteOptions& options, + TocoOperator* op) const override { + op->mode = options.mode() == ::tflite::MirrorPadMode::MirrorPadMode_REFLECT + ? MirrorPadMode::kReflect + : MirrorPadMode::kSymmetric; + } + + int GetVersion(const Operator& op) const override { return 1; } +}; + std::unique_ptr WriteFlexOpOptions( const string& tensorflow_node_def) { auto fbb = absl::make_unique(); @@ -1447,6 +1527,7 @@ std::vector> BuildOperatorList( OperatorType::kMaxPool)); ops.push_back( MakeUnique(::tflite::BuiltinOperator_MUL, OperatorType::kMul)); + ops.push_back( MakeUnique(::tflite::BuiltinOperator_PAD, OperatorType::kPad)); ops.push_back( @@ -1483,6 +1564,8 @@ std::vector> BuildOperatorList( OperatorType::kSqueeze)); ops.push_back( MakeUnique(::tflite::BuiltinOperator_SPLIT, OperatorType::kSplit)); + ops.push_back(MakeUnique(::tflite::BuiltinOperator_SPLIT_V, + OperatorType::kSplitV)); ops.push_back(MakeUnique( ::tflite::BuiltinOperator_STRIDED_SLICE, OperatorType::kStridedSlice)); ops.push_back(MakeUnique(::tflite::BuiltinOperator_TOPK_V2, @@ -1516,6 +1599,13 @@ std::vector> BuildOperatorList( OperatorType::kOneHot)); ops.push_back(MakeUnique(::tflite::BuiltinOperator_UNPACK, OperatorType::kUnpack)); + ops.push_back(MakeUnique(::tflite::BuiltinOperator_LEAKY_RELU, + OperatorType::kLeakyRelu)); + ops.push_back(MakeUnique( + ::tflite::BuiltinOperator_SQUARED_DIFFERENCE, + OperatorType::kSquaredDifference)); + ops.push_back(MakeUnique(::tflite::BuiltinOperator_MIRROR_PAD, + OperatorType::kMirrorPad)); // Custom Operators. ops.push_back( @@ -1600,7 +1690,10 @@ std::vector> BuildOperatorList( "SQUARE", OperatorType::kSquare)); ops.push_back(MakeUnique>( "ZEROS_LIKE", OperatorType::kZerosLike)); - + ops.push_back( + MakeUnique>("ABS", OperatorType::kAbs)); + ops.push_back( + MakeUnique>("FILL", OperatorType::kFill)); return ops; } } // namespace diff --git a/tensorflow/lite/toco/tflite/operator_test.cc b/tensorflow/lite/toco/tflite/operator_test.cc index 8a776cbf0be..14ec89cd73f 100644 --- a/tensorflow/lite/toco/tflite/operator_test.cc +++ b/tensorflow/lite/toco/tflite/operator_test.cc @@ -151,6 +151,7 @@ TEST_F(OperatorTest, SimpleOperators) { OperatorType::kZerosLike); CheckSimpleOperator("FLOOR_MOD", OperatorType::kFloorMod); CheckSimpleOperator("RANGE", OperatorType::kRange); + CheckSimpleOperator("FILL", OperatorType::kFill); } TEST_F(OperatorTest, BuiltinAdd) { @@ -310,6 +311,14 @@ TEST_F(OperatorTest, CustomSplit) { EXPECT_EQ(op.num_split, output_toco_op->num_split); } +TEST_F(OperatorTest, CustomSplitV) { + TensorFlowSplitVOperator op; + op.num_split = 123; + auto output_toco_op = SerializeAndDeserialize( + GetOperator("SPLIT_V", OperatorType::kSplitV), op); + EXPECT_EQ(op.num_split, output_toco_op->num_split); +} + TEST_F(OperatorTest, BuiltinAveragePool) { AveragePoolOperator op; op.fused_activation_function = FusedActivationFunctionType::kRelu6; @@ -517,6 +526,21 @@ TEST_F(OperatorTest, BuiltinUnpack) { EXPECT_EQ(op.axis, output_toco_op->axis); } +TEST_F(OperatorTest, BuiltinLeakyRelu) { + LeakyReluOperator op; + op.alpha = 3; + auto output_toco_op = SerializeAndDeserialize( + GetOperator("LEAKY_RELU", OperatorType::kLeakyRelu), op); + EXPECT_EQ(op.alpha, output_toco_op->alpha); +} + +TEST_F(OperatorTest, BuiltinSquaredDifference) { + SquaredDifferenceOperator op; + auto output_toco_op = SerializeAndDeserialize( + GetOperator("SQUARED_DIFFERENCE", OperatorType::kSquaredDifference), op); + ASSERT_NE(nullptr, output_toco_op.get()); +} + TEST_F(OperatorTest, CustomCTCBeamSearchDecoder) { CTCBeamSearchDecoderOperator op; op.beam_width = 3; @@ -592,6 +616,14 @@ TEST_F(OperatorTest, TestShouldExportAsFlexOp) { EXPECT_FALSE(ShouldExportAsFlexOp(true, "RFFT")); } +TEST_F(OperatorTest, BuiltinMirrorPad) { + MirrorPadOperator op; + op.mode = MirrorPadMode::kReflect; + auto output_toco_op = SerializeAndDeserialize( + GetOperator("MIRROR_PAD", OperatorType::kMirrorPad), op); + EXPECT_EQ(op.mode, output_toco_op->mode); +} + } // namespace } // namespace tflite diff --git a/tensorflow/lite/toco/tflite/whitelisted_flex_ops.cc b/tensorflow/lite/toco/tflite/whitelisted_flex_ops.cc index d251589b483..039a918af16 100644 --- a/tensorflow/lite/toco/tflite/whitelisted_flex_ops.cc +++ b/tensorflow/lite/toco/tflite/whitelisted_flex_ops.cc @@ -187,6 +187,7 @@ bool IsWhitelistedFlexOp(const std::string& tensorflow_op_name) { "MirrorPad", "MirrorPadGrad", "Mul", + "Multinomial", "Neg", "NextIteration", "NonMaxSuppression", diff --git a/tensorflow/lite/toco/toco.cc b/tensorflow/lite/toco/toco.cc index 9740015850a..4a3d6a58487 100644 --- a/tensorflow/lite/toco/toco.cc +++ b/tensorflow/lite/toco/toco.cc @@ -16,87 +16,9 @@ limitations under the License. #include #include -#include "absl/strings/string_view.h" -#include "tensorflow/lite/toco/model.h" #include "tensorflow/lite/toco/model_cmdline_flags.h" -#include "tensorflow/lite/toco/model_flags.pb.h" #include "tensorflow/lite/toco/toco_cmdline_flags.h" -#include "tensorflow/lite/toco/toco_flags.pb.h" -#include "tensorflow/lite/toco/toco_port.h" -#include "tensorflow/lite/toco/toco_tooling.h" -#include "tensorflow/lite/toco/toco_types.h" -#include "tensorflow/core/lib/core/errors.h" -#include "tensorflow/core/platform/logging.h" - -namespace toco { -namespace { - -// Checks the permissions of the output file to ensure it is writeable. -void CheckOutputFilePermissions(const Arg& output_file) { - QCHECK(output_file.specified()) << "Missing required flag --output_file.\n"; - QCHECK(port::file::Writable(output_file.value()).ok()) - << "Specified output_file is not writable: " << output_file.value() - << ".\n"; -} - -// Checks the permissions of the frozen model file. -void CheckFrozenModelPermissions(const Arg& input_file) { - QCHECK(input_file.specified()) << "Missing required flag --input_file.\n"; - QCHECK(port::file::Exists(input_file.value(), port::file::Defaults()).ok()) - << "Specified input_file does not exist: " << input_file.value() << ".\n"; - QCHECK(port::file::Readable(input_file.value(), port::file::Defaults()).ok()) - << "Specified input_file exists, but is not readable: " - << input_file.value() << ".\n"; -} - -// Reads the contents of the GraphDef from either the frozen graph file or the -// SavedModel directory. If it reads the SavedModel directory, it updates the -// ModelFlags and TocoFlags accordingly. -void ReadInputData(const ParsedTocoFlags& parsed_toco_flags, - const ParsedModelFlags& parsed_model_flags, - TocoFlags* toco_flags, ModelFlags* model_flags, - string* graph_def_contents) { - port::CheckInitGoogleIsDone("InitGoogle is not done yet.\n"); - - // Ensure savedmodel_directory is not set. - QCHECK(!parsed_toco_flags.savedmodel_directory.specified()) - << "Use `tensorflow/lite/python/tflite_convert` script with " - << "SavedModel directories.\n"; - - // Checks the input file permissions and reads the contents. - CheckFrozenModelPermissions(parsed_toco_flags.input_file); - CHECK(port::file::GetContents(parsed_toco_flags.input_file.value(), - graph_def_contents, port::file::Defaults()) - .ok()); -} - -tensorflow::Status ToolMain(const ParsedTocoFlags& parsed_toco_flags, - const ParsedModelFlags& parsed_model_flags) { - ModelFlags model_flags; - ReadModelFlagsFromCommandLineFlags(parsed_model_flags, &model_flags); - - TocoFlags toco_flags; - ReadTocoFlagsFromCommandLineFlags(parsed_toco_flags, &toco_flags); - - string graph_def_contents; - ReadInputData(parsed_toco_flags, parsed_model_flags, &toco_flags, - &model_flags, &graph_def_contents); - CheckOutputFilePermissions(parsed_toco_flags.output_file); - - std::unique_ptr model = - Import(toco_flags, model_flags, graph_def_contents); - Transform(toco_flags, model.get()); - string output_file_contents; - TF_RETURN_IF_ERROR(Export(toco_flags, *model, toco_flags.allow_custom_ops(), - &output_file_contents)); - TF_RETURN_IF_ERROR( - port::file::SetContents(parsed_toco_flags.output_file.value(), - output_file_contents, port::file::Defaults())); - return tensorflow::Status(); -} - -} // namespace -} // namespace toco +#include "tensorflow/lite/toco/toco_convert.h" int main(int argc, char** argv) { toco::string msg; @@ -126,6 +48,6 @@ int main(int argc, char** argv) { return 1; } toco::port::InitGoogle(argv[0], effective_argc, &effective_argv, true); - auto status = toco::ToolMain(parsed_toco_flags, parsed_model_flags); + auto status = toco::Convert(parsed_toco_flags, parsed_model_flags); return status.ok() ? 0 : -1; } diff --git a/tensorflow/lite/toco/toco_convert.cc b/tensorflow/lite/toco/toco_convert.cc new file mode 100644 index 00000000000..28e7b10ecd0 --- /dev/null +++ b/tensorflow/lite/toco/toco_convert.cc @@ -0,0 +1,108 @@ +/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#include +#include +#include + +#include "absl/strings/string_view.h" +#include "tensorflow/lite/toco/model.h" +#include "tensorflow/lite/toco/model_cmdline_flags.h" +#include "tensorflow/lite/toco/model_flags.pb.h" +#include "tensorflow/lite/toco/toco_cmdline_flags.h" +#include "tensorflow/lite/toco/toco_flags.pb.h" +#include "tensorflow/lite/toco/toco_port.h" +#include "tensorflow/lite/toco/toco_tooling.h" +#include "tensorflow/lite/toco/toco_types.h" +#include "tensorflow/core/lib/core/errors.h" +#include "tensorflow/core/platform/logging.h" + +namespace toco { +namespace { + +// Checks the permissions of the output file to ensure it is writeable. +void CheckOutputFilePermissions(const Arg& output_file) { + QCHECK(output_file.specified()) << "Missing required flag --output_file.\n"; + QCHECK(port::file::Writable(output_file.value()).ok()) + << "Specified output_file is not writable: " << output_file.value() + << ".\n"; +} + +// Checks the permissions of the frozen model file. +void CheckFrozenModelPermissions(const Arg& input_file) { + QCHECK(input_file.specified()) << "Missing required flag --input_file.\n"; + QCHECK(port::file::Exists(input_file.value(), port::file::Defaults()).ok()) + << "Specified input_file does not exist: " << input_file.value() << ".\n"; + QCHECK(port::file::Readable(input_file.value(), port::file::Defaults()).ok()) + << "Specified input_file exists, but is not readable: " + << input_file.value() << ".\n"; +} + +// Reads the contents of the GraphDef from either the frozen graph file or the +// SavedModel directory. If it reads the SavedModel directory, it updates the +// ModelFlags and TocoFlags accordingly. +void ReadInputData(const ParsedTocoFlags& parsed_toco_flags, + const ParsedModelFlags& parsed_model_flags, + TocoFlags* toco_flags, ModelFlags* model_flags, + string* graph_def_contents) { + port::CheckInitGoogleIsDone("InitGoogle is not done yet.\n"); + + // Ensure savedmodel_directory is not set. + QCHECK(!parsed_toco_flags.savedmodel_directory.specified()) + << "Use `tensorflow/lite/python/tflite_convert` script with " + << "SavedModel directories.\n"; + + // Checks the input file permissions and reads the contents. + CheckFrozenModelPermissions(parsed_toco_flags.input_file); + CHECK(port::file::GetContents(parsed_toco_flags.input_file.value(), + graph_def_contents, port::file::Defaults()) + .ok()); +} +} // namespace + +tensorflow::Status Convert(const string& graph_def_contents, + const TocoFlags& toco_flags, + const ModelFlags& model_flags, + string* output_file_contents) { + std::unique_ptr model = + Import(toco_flags, model_flags, graph_def_contents); + Transform(toco_flags, model.get()); + return Export(toco_flags, *model, toco_flags.allow_custom_ops(), + output_file_contents); +} + +tensorflow::Status Convert(const ParsedTocoFlags& parsed_toco_flags, + const ParsedModelFlags& parsed_model_flags) { + ModelFlags model_flags; + ReadModelFlagsFromCommandLineFlags(parsed_model_flags, &model_flags); + + TocoFlags toco_flags; + ReadTocoFlagsFromCommandLineFlags(parsed_toco_flags, &toco_flags); + + string graph_def_contents; + ReadInputData(parsed_toco_flags, parsed_model_flags, &toco_flags, + &model_flags, &graph_def_contents); + CheckOutputFilePermissions(parsed_toco_flags.output_file); + + string output_file_contents; + TF_RETURN_IF_ERROR(Convert(graph_def_contents, toco_flags, model_flags, + &output_file_contents)); + + TF_RETURN_IF_ERROR( + port::file::SetContents(parsed_toco_flags.output_file.value(), + output_file_contents, port::file::Defaults())); + return tensorflow::Status(); +} + +} // namespace toco diff --git a/tensorflow/lite/toco/toco_convert.h b/tensorflow/lite/toco/toco_convert.h new file mode 100644 index 00000000000..ebbd336d3f5 --- /dev/null +++ b/tensorflow/lite/toco/toco_convert.h @@ -0,0 +1,34 @@ +/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#ifndef TENSORFLOW_LITE_TOCO_TOCO_CONVERT_H_ +#define TENSORFLOW_LITE_TOCO_TOCO_CONVERT_H_ + +#include "tensorflow/core/lib/core/status.h" +#include "tensorflow/lite/toco/args.h" +#include "tensorflow/lite/toco/model_flags.pb.h" +#include "tensorflow/lite/toco/toco_flags.pb.h" + +namespace toco { + +tensorflow::Status Convert(const string& graph_def_contents, + const TocoFlags& toco_flags, + const ModelFlags& model_flags, + string* output_file_contents); + +tensorflow::Status Convert(const ParsedTocoFlags& parsed_toco_flags, + const ParsedModelFlags& parsed_model_flags); +} // namespace toco + +#endif // TENSORFLOW_LITE_TOCO_TOCO_CONVERT_H_ diff --git a/tensorflow/lite/toco/toco_convert_test.cc b/tensorflow/lite/toco/toco_convert_test.cc new file mode 100644 index 00000000000..c3c440db943 --- /dev/null +++ b/tensorflow/lite/toco/toco_convert_test.cc @@ -0,0 +1,173 @@ +/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#include "tensorflow/lite/toco/toco_convert.h" +#include +#include + +namespace toco { +namespace { + +TEST(TocoTest, MissingInputFile) { + ParsedTocoFlags toco_flags; + ParsedModelFlags model_flags; + EXPECT_DEATH(Convert(toco_flags, model_flags).ok(), + "Missing required flag --input_file"); +} + +TEST(TocoTest, BadInputFormat) { + TocoFlags toco_flags; + ModelFlags model_flags; + + string input; + string output; + + EXPECT_DEATH(Convert(input, toco_flags, model_flags, &output).ok(), + "Unhandled input_format='FILE_FORMAT_UNKNOWN'"); +} + +TEST(TocoTest, MissingOuputArrays) { + TocoFlags toco_flags; + ModelFlags model_flags; + + toco_flags.set_input_format(TENSORFLOW_GRAPHDEF); + string input; + string output; + + EXPECT_DEATH(Convert(input, toco_flags, model_flags, &output).ok(), + "This model does not define output arrays, so a --output_arrays " + "flag must be given on the command-line"); +} + +TEST(TocoTest, BadOutputArray) { + TocoFlags toco_flags; + ModelFlags model_flags; + + toco_flags.set_input_format(TENSORFLOW_GRAPHDEF); + model_flags.add_output_arrays("output1"); + string input; + string output; + + EXPECT_DEATH(Convert(input, toco_flags, model_flags, &output).ok(), + "Specified output array .output1. is not produced by any op " + "in this graph. Is it a typo. To silence this message, pass " + "this flag: allow_nonexistent_arrays"); +} + +TEST(TocoTest, BadOutputFormat) { + TocoFlags toco_flags; + ModelFlags model_flags; + + toco_flags.set_input_format(TENSORFLOW_GRAPHDEF); + model_flags.add_output_arrays("output1"); + string input = R"GraphDef( + node { + name: "output1" + input: "input1" + input: "input2" + op: "Sub" + attr { key: "T" value { type: DT_FLOAT } } + } + )GraphDef"; + + string output; + + EXPECT_DEATH(Convert(input, toco_flags, model_flags, &output).ok(), + "Unhandled output_format='FILE_FORMAT_UNKNOWN'"); +} + +TEST(TocoTest, SimpleFloatModel) { + TocoFlags toco_flags; + ModelFlags model_flags; + + toco_flags.set_input_format(TENSORFLOW_GRAPHDEF); + toco_flags.set_output_format(TENSORFLOW_GRAPHDEF); + + // Inputs are automatically selected (but that might not be a good idea). + model_flags.add_output_arrays("output1"); + string input = R"GraphDef( + node { + name: "input1" + op: "Placeholder" + attr { key: "dtype" value { type: DT_INT64 } } + } + node { + name: "input2" + op: "Placeholder" + attr { key: "dtype" value { type: DT_INT64 } } + } + node { + name: "output1" + input: "input1" + input: "input2" + op: "Sub" + attr { key: "T" value { type: DT_FLOAT } } + } + )GraphDef"; + + string output; + EXPECT_TRUE(Convert(input, toco_flags, model_flags, &output).ok()); + EXPECT_TRUE(!output.empty()); +} + +TEST(TocoTest, TransientStringTensors) { + TocoFlags toco_flags; + ModelFlags model_flags; + + toco_flags.set_input_format(TENSORFLOW_GRAPHDEF); + + // We need to do a couple of things to trigger the transient array + // initialization code: output format must support memory planning, and the + // input array must have a shape. + toco_flags.set_output_format(TFLITE); + + model_flags.add_output_arrays("output1"); + string input = R"GraphDef( + node { + name: "input1" + op: "Placeholder" + attr { key: "dtype" value { type: DT_STRING } } + attr { key: "shape" value { shape { dim { size:1 }}}} + } + node { + name: "indices1" + op: "Placeholder" + attr { key: "dtype" value { type: DT_INT64 } } + } + node { + name: "intermediate1" + op: "Gather" + input: "input1" + input: "indices1" + attr { key: "Tparams" value { type: DT_STRING } } + attr { key: "Tindices" value { type: DT_INT64 } } + } + node { + name: "output1" + op: "Gather" + input: "intermediate1" + input: "indices2" + attr { key: "Tparams" value { type: DT_STRING } } + attr { key: "Tindices" value { type: DT_INT64 } } + } + )GraphDef"; + + string output; + + EXPECT_TRUE(Convert(input, toco_flags, model_flags, &output).ok()); + EXPECT_TRUE(!output.empty()); +} + +} // namespace +} // namespace toco diff --git a/tensorflow/lite/toco/toco_tooling.cc b/tensorflow/lite/toco/toco_tooling.cc index 5f96e833fbf..55a454e66de 100644 --- a/tensorflow/lite/toco/toco_tooling.cc +++ b/tensorflow/lite/toco/toco_tooling.cc @@ -210,7 +210,8 @@ std::unique_ptr Import(const TocoFlags& toco_flags, CheckInvariants(*model); break; default: - LOG(FATAL) << "Unhandled input_format"; + LOG(FATAL) << "Unhandled input_format='" + << FileFormat_Name(toco_flags.input_format()) << "'"; } LogDump(kLogLevelModelChanged, "AT IMPORT", *model); @@ -308,6 +309,7 @@ void Transform(const TocoFlags& toco_flags, Model* model) { // Fix any issues with IO edges. This must happen after any transform that // may modify the structure of the edges. FixEdgeArrays(model); + FixOperatorOrdering(model); if (quantize_output) { // If the user specified default min/max ranges we need to set all arrays @@ -424,7 +426,8 @@ tensorflow::Status Export(const TocoFlags& toco_flags, const Model& model, DumpGraphviz(model, output_file_contents); break; default: - LOG(FATAL) << "Unhandled output_format"; + LOG(FATAL) << "Unhandled output_format='" + << FileFormat_Name(toco_flags.output_format()) << "'"; } return tensorflow::Status(); } diff --git a/tensorflow/lite/toco/tooling_util.cc b/tensorflow/lite/toco/tooling_util.cc index e33f7c8452f..af4cd386a20 100644 --- a/tensorflow/lite/toco/tooling_util.cc +++ b/tensorflow/lite/toco/tooling_util.cc @@ -308,6 +308,7 @@ const char* OperatorTypeName(OperatorType type) { #define HANDLE_OPERATORTYPENAME_CASE(c) \ case OperatorType::k##c: \ return #c; + HANDLE_OPERATORTYPENAME_CASE(Abs) HANDLE_OPERATORTYPENAME_CASE(Add) HANDLE_OPERATORTYPENAME_CASE(AddN) HANDLE_OPERATORTYPENAME_CASE(AveragePool) @@ -371,6 +372,7 @@ const char* OperatorTypeName(OperatorType type) { HANDLE_OPERATORTYPENAME_CASE(Shape) HANDLE_OPERATORTYPENAME_CASE(Slice) HANDLE_OPERATORTYPENAME_CASE(Split) + HANDLE_OPERATORTYPENAME_CASE(SplitV) HANDLE_OPERATORTYPENAME_CASE(Sqrt) HANDLE_OPERATORTYPENAME_CASE(Square) HANDLE_OPERATORTYPENAME_CASE(Switch) @@ -411,6 +413,9 @@ const char* OperatorTypeName(OperatorType type) { HANDLE_OPERATORTYPENAME_CASE(ZerosLike) HANDLE_OPERATORTYPENAME_CASE(UnidirectionalSequenceLstm) HANDLE_OPERATORTYPENAME_CASE(ResizeNearestNeighbor) + HANDLE_OPERATORTYPENAME_CASE(LeakyRelu) + HANDLE_OPERATORTYPENAME_CASE(SquaredDifference) + HANDLE_OPERATORTYPENAME_CASE(MirrorPad) default: LOG(FATAL) << "Unhandled op type"; #undef HANDLE_OPERATORTYPENAME_CASE @@ -439,6 +444,7 @@ bool OperatorSupportsFusedActivation(OperatorType type) { case OperatorType::kMaxPool: case OperatorType::kMul: case OperatorType::kSub: + case OperatorType::kSquaredDifference: return true; default: return false; @@ -531,12 +537,12 @@ void DumpGraphvizVideoFrame(const Model& model) { if (!dump_hashes.count(hash)) { LOG(INFO) << "DUMPING GRAPHVIZ VIDEO FRAME: " << dump_id; dump_hashes.insert(hash); - CHECK(port::file::SetContents( - port::file::JoinPath( - dump_options.dump_graphviz, - toco::port::StringF("toco_video_%05d.dot", dump_id)), - graphviz_dump, port::file::Defaults()) - .ok()); + const auto result = port::file::SetContents( + port::file::JoinPath( + dump_options.dump_graphviz, + toco::port::StringF("toco_video_%05d.dot", dump_id)), + graphviz_dump, port::file::Defaults()); + QCHECK(result.ok()) << result.error_message(); dump_id++; } } @@ -550,14 +556,13 @@ void LogDump(int log_level, const string& message, const Model& model) { string graphviz_dump; DumpGraphviz(model, &graphviz_dump); - CHECK(port::file::SetContents( - port::file::JoinPath( - dump_options.dump_graphviz, - absl::StrCat("toco_", - absl::StrReplaceAll(message, {{" ", "_"}}), - ".dot")), - graphviz_dump, port::file::Defaults()) - .ok()); + const auto result = port::file::SetContents( + port::file::JoinPath( + dump_options.dump_graphviz, + absl::StrCat("toco_", absl::StrReplaceAll(message, {{" ", "_"}}), + ".dot")), + graphviz_dump, port::file::Defaults()); + QCHECK(result.ok()) << result.error_message(); } if (!VLOG_IS_ON(log_level)) { @@ -894,6 +899,9 @@ void CheckNonExistentIOArrays(const Model& model) { << "\" is not consumed by any op in this graph. " << general_comment; } for (const string& output_array : model.flags.output_arrays()) { + if (IsConstantParameterArray(model, output_array)) { + continue; // It is OK to request that a constant be an output. + } QCHECK(GetOpWithOutput(model, output_array)) << "Specified output array \"" << output_array << "\" is not produced by any op in this graph. " << general_comment; @@ -1032,10 +1040,10 @@ void CheckEachArray(const Model& model) { if (colon_pos != string::npos) { CHECK_EQ(name.substr(colon_pos + 1).find_first_not_of("0123456789"), string::npos) - << "Array name must only have digits after colon"; + << "Array '" << name << "' has non-digit characters after colon."; } - CHECK_GT(colon_pos, 0) - << "First character of array name must not be a colon."; + CHECK_GT(colon_pos, 0) << "Array '" << name + << "' must not start with a colon."; } } @@ -1767,6 +1775,14 @@ bool IsAllocatableTransientArray(const Model& model, const string& array_name) { if (!array->has_shape()) { return false; } + + // The size of string tensors is rarely known ahead of time, so all transient + // tensors of this type will need to be dynamically allocated. + if (array->final_data_type == ArrayDataType::kString || + array->data_type == ArrayDataType::kString) { + return false; + } + return true; } @@ -2207,6 +2223,8 @@ ArrayDataType ConvertIODataTypeToArrayDataType(IODataType type) { return ArrayDataType::kFloat; case QUANTIZED_UINT8: return ArrayDataType::kUint8; + case INT8: + return ArrayDataType::kInt8; case QUANTIZED_INT16: return ArrayDataType::kInt16; case INT32: diff --git a/tensorflow/lite/toco/types.proto b/tensorflow/lite/toco/types.proto index 12f711fd8a3..fa911b8a4c8 100644 --- a/tensorflow/lite/toco/types.proto +++ b/tensorflow/lite/toco/types.proto @@ -43,4 +43,7 @@ enum IODataType { // Complex64, not quantized COMPLEX64 = 8; + + // Int8, quantized based on QuantizationParameters in schema. + INT8 = 9; } diff --git a/tensorflow/lite/tools/benchmark/benchmark_model.cc b/tensorflow/lite/tools/benchmark/benchmark_model.cc index 05148aea65b..e9b485efcaa 100644 --- a/tensorflow/lite/tools/benchmark/benchmark_model.cc +++ b/tensorflow/lite/tools/benchmark/benchmark_model.cc @@ -51,11 +51,13 @@ using tensorflow::Stat; BenchmarkParams BenchmarkModel::DefaultParams() { BenchmarkParams params; params.AddParam("num_runs", BenchmarkParam::Create(50)); + params.AddParam("min_secs", BenchmarkParam::Create(1.0f)); params.AddParam("run_delay", BenchmarkParam::Create(-1.0f)); params.AddParam("num_threads", BenchmarkParam::Create(1)); params.AddParam("benchmark_name", BenchmarkParam::Create("")); params.AddParam("output_prefix", BenchmarkParam::Create("")); params.AddParam("warmup_runs", BenchmarkParam::Create(1)); + params.AddParam("warmup_min_secs", BenchmarkParam::Create(0.5f)); return params; } @@ -73,19 +75,34 @@ void BenchmarkLoggingListener::OnBenchmarkEnd(const BenchmarkResults &results) { std::vector BenchmarkModel::GetFlags() { return { - CreateFlag("num_runs", ¶ms_, "number of runs"), + CreateFlag("num_runs", ¶ms_, + "minimum number of runs, see also min_secs"), + CreateFlag( + "min_secs", ¶ms_, + "minimum number of seconds to rerun for, potentially making the " + "actual number of runs to be greater than num_runs"), CreateFlag("run_delay", ¶ms_, "delay between runs in seconds"), CreateFlag("num_threads", ¶ms_, "number of threads"), CreateFlag("benchmark_name", ¶ms_, "benchmark name"), CreateFlag("output_prefix", ¶ms_, "benchmark output prefix"), - CreateFlag("warmup_runs", ¶ms_, - "how many runs to initialize model"), + CreateFlag( + "warmup_runs", ¶ms_, + "minimum number of runs performed on initialization, to " + "allow performance characteristics to settle, see also " + "warmup_min_secs"), + CreateFlag( + "warmup_min_secs", ¶ms_, + "minimum number of seconds to rerun for, potentially making the " + "actual number of warm-up runs to be greater than warmup_runs"), }; } void BenchmarkModel::LogParams() { - TFLITE_LOG(INFO) << "Num runs: [" << params_.Get("num_runs") << "]"; + TFLITE_LOG(INFO) << "Min num runs: [" << params_.Get("num_runs") + << "]"; + TFLITE_LOG(INFO) << "Min runs duration (seconds): [" + << params_.Get("min_secs") << "]"; TFLITE_LOG(INFO) << "Inter-run delay (seconds): [" << params_.Get("run_delay") << "]"; TFLITE_LOG(INFO) << "Num threads: [" << params_.Get("num_threads") @@ -94,16 +111,24 @@ void BenchmarkModel::LogParams() { << params_.Get("benchmark_name") << "]"; TFLITE_LOG(INFO) << "Output prefix: [" << params_.Get("output_prefix") << "]"; - TFLITE_LOG(INFO) << "Warmup runs: [" << params_.Get("warmup_runs") - << "]"; + TFLITE_LOG(INFO) << "Min warmup runs: [" + << params_.Get("warmup_runs") << "]"; + TFLITE_LOG(INFO) << "Min warmup runs duration (seconds): [" + << params_.Get("warmup_min_secs") << "]"; } void BenchmarkModel::PrepareInputsAndOutputs() {} -Stat BenchmarkModel::Run(int num_times, RunType run_type) { +Stat BenchmarkModel::Run(int min_num_times, float min_secs, + RunType run_type) { Stat run_stats; - TFLITE_LOG(INFO) << "Running benchmark for " << num_times << " iterations "; - for (int run = 0; run < num_times; run++) { + TFLITE_LOG(INFO) << "Running benchmark for at least " << min_num_times + << " iterations and at least " << min_secs << " seconds"; + int64_t min_finish_us = + profiling::time::NowMicros() + static_cast(min_secs * 1.e6f); + for (int run = 0; + run < min_num_times || profiling::time::NowMicros() < min_finish_us; + run++) { PrepareInputsAndOutputs(); listeners_.OnSingleRunStart(run_type); int64_t start_us = profiling::time::NowMicros(); @@ -145,9 +170,11 @@ void BenchmarkModel::Run() { uint64_t input_bytes = ComputeInputBytes(); Stat warmup_time_us = - Run(params_.Get("warmup_runs"), WARMUP); + Run(params_.Get("warmup_runs"), + params_.Get("warmup_min_secs"), WARMUP); Stat inference_time_us = - Run(params_.Get("num_runs"), REGULAR); + Run(params_.Get("num_runs"), params_.Get("min_secs"), + REGULAR); listeners_.OnBenchmarkEnd( {startup_latency_us, input_bytes, warmup_time_us, inference_time_us}); } diff --git a/tensorflow/lite/tools/benchmark/benchmark_model.h b/tensorflow/lite/tools/benchmark/benchmark_model.h index d8a9b05010a..31ee5c92aa3 100644 --- a/tensorflow/lite/tools/benchmark/benchmark_model.h +++ b/tensorflow/lite/tools/benchmark/benchmark_model.h @@ -150,7 +150,8 @@ class BenchmarkModel { bool ParseFlags(int argc, char** argv); virtual std::vector GetFlags(); virtual uint64_t ComputeInputBytes() = 0; - virtual tensorflow::Stat Run(int num_times, RunType run_type); + virtual tensorflow::Stat Run(int min_num_times, float min_secs, + RunType run_type); virtual void PrepareInputsAndOutputs(); virtual void RunImpl() = 0; BenchmarkParams params_; diff --git a/tensorflow/lite/tools/benchmark/benchmark_test.cc b/tensorflow/lite/tools/benchmark/benchmark_test.cc index 59d23d90086..8191fbcd735 100644 --- a/tensorflow/lite/tools/benchmark/benchmark_test.cc +++ b/tensorflow/lite/tools/benchmark/benchmark_test.cc @@ -33,6 +33,7 @@ namespace { BenchmarkParams CreateParams() { BenchmarkParams params; params.AddParam("num_runs", BenchmarkParam::Create(2)); + params.AddParam("min_secs", BenchmarkParam::Create(1.0f)); params.AddParam("run_delay", BenchmarkParam::Create(-1.0f)); params.AddParam("num_threads", BenchmarkParam::Create(1)); params.AddParam("benchmark_name", BenchmarkParam::Create("")); @@ -42,6 +43,7 @@ BenchmarkParams CreateParams() { params.AddParam("input_layer", BenchmarkParam::Create("")); params.AddParam("input_layer_shape", BenchmarkParam::Create("")); params.AddParam("use_nnapi", BenchmarkParam::Create(false)); + params.AddParam("warmup_min_secs", BenchmarkParam::Create(0.5f)); return params; } diff --git a/tensorflow/lite/tools/benchmark/benchmark_tflite_model.cc b/tensorflow/lite/tools/benchmark/benchmark_tflite_model.cc index 777d9dde7dd..16f70870b69 100644 --- a/tensorflow/lite/tools/benchmark/benchmark_tflite_model.cc +++ b/tensorflow/lite/tools/benchmark/benchmark_tflite_model.cc @@ -181,7 +181,9 @@ bool PopulateInputLayerInfo( return true; } -BenchmarkParams GetDefaultParams() { +} // namespace + +BenchmarkParams BenchmarkTfLiteModel::DefaultParams() { BenchmarkParams default_params = BenchmarkModel::DefaultParams(); default_params.AddParam("graph", BenchmarkParam::Create("")); default_params.AddParam("input_layer", @@ -192,10 +194,8 @@ BenchmarkParams GetDefaultParams() { return default_params; } -} // namespace - BenchmarkTfLiteModel::BenchmarkTfLiteModel() - : BenchmarkTfLiteModel(GetDefaultParams()) {} + : BenchmarkTfLiteModel(DefaultParams()) {} BenchmarkTfLiteModel::BenchmarkTfLiteModel(BenchmarkParams params) : BenchmarkModel(std::move(params)) { @@ -279,7 +279,7 @@ void BenchmarkTfLiteModel::PrepareInputsAndOutputs() { FillRandomString(&buffer, sizes, []() { return "we're have some friends over saturday to hang out in the yard"; }); - buffer.WriteToTensor(interpreter->tensor(i)); + buffer.WriteToTensor(interpreter->tensor(i), /*new_shape=*/nullptr); } else { TFLITE_LOG(FATAL) << "Don't know how to populate tensor " << t->name << " of type " << t->type; @@ -319,6 +319,7 @@ void BenchmarkTfLiteModel::Init() { bool use_nnapi = params_.Get("use_nnapi"); interpreter->UseNNAPI(use_nnapi); + ApplyDelegates(); auto interpreter_inputs = interpreter->inputs(); diff --git a/tensorflow/lite/tools/benchmark/benchmark_tflite_model.h b/tensorflow/lite/tools/benchmark/benchmark_tflite_model.h index 401ab5427d3..83599e644d1 100644 --- a/tensorflow/lite/tools/benchmark/benchmark_tflite_model.h +++ b/tensorflow/lite/tools/benchmark/benchmark_tflite_model.h @@ -77,11 +77,16 @@ class BenchmarkTfLiteModel : public BenchmarkModel { }; protected: + static BenchmarkParams DefaultParams(); void PrepareInputsAndOutputs() override; - private: + // Allows installation of custom delegates during initialization + virtual void ApplyDelegates() {} + std::unique_ptr model; std::unique_ptr interpreter; + + private: std::vector inputs; ProfilingListener profiling_listener_; GemmlowpProfilingListener gemmlowp_profiling_listener_; diff --git a/tensorflow/lite/tools/benchmark/ios/TFLiteBenchmark/TFLiteBenchmark.xcodeproj/project.pbxproj b/tensorflow/lite/tools/benchmark/ios/TFLiteBenchmark/TFLiteBenchmark.xcodeproj/project.pbxproj index 958936a6607..a5f5bfbbdaf 100644 --- a/tensorflow/lite/tools/benchmark/ios/TFLiteBenchmark/TFLiteBenchmark.xcodeproj/project.pbxproj +++ b/tensorflow/lite/tools/benchmark/ios/TFLiteBenchmark/TFLiteBenchmark.xcodeproj/project.pbxproj @@ -20,7 +20,7 @@ /* Begin PBXFileReference section */ 6FE7579920D59CE500F01636 /* benchmark_params.json */ = {isa = PBXFileReference; lastKnownFileType = text.json; path = benchmark_params.json; sourceTree = ""; }; - 6FE7579C20D5A5E000F01636 /* benchmark-lib.a */ = {isa = PBXFileReference; lastKnownFileType = archive.ar; name = "benchmark-lib.a"; path = "$SRCROOT/../../../../../../../tensorflow/lite/tools/make/gen/lib/benchmark-lib.a"; sourceTree = ""; }; + 6FE7579C20D5A5E000F01636 /* benchmark-lib.a */ = {isa = PBXFileReference; lastKnownFileType = archive.ar; name = "benchmark-lib.a"; path = "$SRCROOT/../../../../../../tensorflow/lite/tools/make/gen/lib/benchmark-lib.a"; sourceTree = ""; }; 6FE7579E20D5A6A700F01636 /* Accelerate.framework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.framework; name = Accelerate.framework; path = System/Library/Frameworks/Accelerate.framework; sourceTree = SDKROOT; }; 6FE757A020D5AB8000F01636 /* mobilenet_v1_1.0_224.tflite */ = {isa = PBXFileReference; lastKnownFileType = file; path = mobilenet_v1_1.0_224.tflite; sourceTree = ""; }; 6FE93FF820D592D8008C9FE4 /* TFLiteBenchmark.app */ = {isa = PBXFileReference; explicitFileType = wrapper.application; includeInIndex = 0; path = TFLiteBenchmark.app; sourceTree = BUILT_PRODUCTS_DIR; }; @@ -309,19 +309,19 @@ ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon; CODE_SIGN_STYLE = Automatic; "HEADER_SEARCH_PATHS[arch=*]" = ( - $SRCROOT/../../../../../../../, - $SRCROOT/../../../../../../../tensorflow/lite/tools/make/downloads/eigen, - $SRCROOT/../../../../../../../tensorflow/lite/tools/make/downloads/gemmlowp, - $SRCROOT/../../../../../../../tensorflow/lite/tools/make/downloads/neon_2_sse, - $SRCROOT/../../../../../../../tensorflow/lite/tools/make/downloads/farmhash/src, - $SRCROOT/../../../../../../../tensorflow/lite/tools/make/downloads/flatbuffers/include, + $SRCROOT/../../../../../../, + $SRCROOT/../../../../../../tensorflow/lite/tools/make/downloads/eigen, + $SRCROOT/../../../../../../tensorflow/lite/tools/make/downloads/gemmlowp, + $SRCROOT/../../../../../../tensorflow/lite/tools/make/downloads/neon_2_sse, + $SRCROOT/../../../../../../tensorflow/lite/tools/make/downloads/farmhash/src, + $SRCROOT/../../../../../../tensorflow/lite/tools/make/downloads/flatbuffers/include, ); INFOPLIST_FILE = TFLiteBenchmark/Info.plist; LD_RUNPATH_SEARCH_PATHS = ( "$(inherited)", "@executable_path/Frameworks", ); - "LIBRARY_SEARCH_PATHS[arch=*]" = $SRCROOT/../../../../../../../tensorflow/lite/tools/make/gen/lib; + "LIBRARY_SEARCH_PATHS[arch=*]" = $SRCROOT/../../../../../../tensorflow/lite/tools/make/gen/lib; PRODUCT_BUNDLE_IDENTIFIER = example.TFLiteBenchmark; PRODUCT_NAME = "$(TARGET_NAME)"; TARGETED_DEVICE_FAMILY = "1,2"; @@ -335,19 +335,19 @@ ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon; CODE_SIGN_STYLE = Automatic; "HEADER_SEARCH_PATHS[arch=*]" = ( - $SRCROOT/../../../../../../../, - $SRCROOT/../../../../../../../tensorflow/lite/tools/make/downloads/eigen, - $SRCROOT/../../../../../../../tensorflow/lite/tools/make/downloads/gemmlowp, - $SRCROOT/../../../../../../../tensorflow/lite/tools/make/downloads/neon_2_sse, - $SRCROOT/../../../../../../../tensorflow/lite/tools/make/downloads/farmhash/src, - $SRCROOT/../../../../../../../tensorflow/lite/tools/make/downloads/flatbuffers/include, + $SRCROOT/../../../../../../, + $SRCROOT/../../../../../../tensorflow/lite/tools/make/downloads/eigen, + $SRCROOT/../../../../../../tensorflow/lite/tools/make/downloads/gemmlowp, + $SRCROOT/../../../../../../tensorflow/lite/tools/make/downloads/neon_2_sse, + $SRCROOT/../../../../../../tensorflow/lite/tools/make/downloads/farmhash/src, + $SRCROOT/../../../../../../tensorflow/lite/tools/make/downloads/flatbuffers/include, ); INFOPLIST_FILE = TFLiteBenchmark/Info.plist; LD_RUNPATH_SEARCH_PATHS = ( "$(inherited)", "@executable_path/Frameworks", ); - "LIBRARY_SEARCH_PATHS[arch=*]" = $SRCROOT/../../../../../../../tensorflow/lite/tools/make/gen/lib; + "LIBRARY_SEARCH_PATHS[arch=*]" = $SRCROOT/../../../../../../tensorflow/lite/tools/make/gen/lib; PRODUCT_BUNDLE_IDENTIFIER = example.TFLiteBenchmark; PRODUCT_NAME = "$(TARGET_NAME)"; TARGETED_DEVICE_FAMILY = "1,2"; diff --git a/tensorflow/lite/tools/make/Makefile b/tensorflow/lite/tools/make/Makefile index 8f123558545..363a069d5e2 100644 --- a/tensorflow/lite/tools/make/Makefile +++ b/tensorflow/lite/tools/make/Makefile @@ -85,6 +85,7 @@ CORE_CC_ALL_SRCS := \ $(wildcard tensorflow/lite/*.cc) \ $(wildcard tensorflow/lite/*.c) \ $(wildcard tensorflow/lite/c/*.c) \ +$(wildcard tensorflow/lite/core/*.cc) \ $(wildcard tensorflow/lite/core/api/*.cc) ifneq ($(BUILD_TYPE),micro) CORE_CC_ALL_SRCS += \ @@ -113,6 +114,10 @@ ifeq ($(BUILD_TYPE),micro) CORE_CC_EXCLUDE_SRCS += \ tensorflow/lite/mmap_allocation.cc \ tensorflow/lite/nnapi_delegate.cc +else +CORE_CC_EXCLUDE_SRCS += \ +tensorflow/contrib/lite/mmap_allocation_disabled.cc \ +tensorflow/contrib/lite/nnapi_delegate_disabled.cc endif # Filter out all the excluded files. TF_LITE_CC_SRCS := $(filter-out $(CORE_CC_EXCLUDE_SRCS), $(CORE_CC_ALL_SRCS)) @@ -208,6 +213,9 @@ $(BENCHMARK_BINARY) : $(BENCHMARK_LIB) benchmark: $(BENCHMARK_BINARY) +libdir: + @echo $(LIBDIR) + # Gets rid of all generated files. clean: rm -rf $(MAKEFILE_DIR)/gen diff --git a/tensorflow/lite/tools/make/targets/ios_makefile.inc b/tensorflow/lite/tools/make/targets/ios_makefile.inc index 7f36b8ecef4..ae9276f9a63 100644 --- a/tensorflow/lite/tools/make/targets/ios_makefile.inc +++ b/tensorflow/lite/tools/make/targets/ios_makefile.inc @@ -22,7 +22,7 @@ ifeq ($(TARGET), ios) TARGET_ARCH := x86_64 CXXFLAGS += -miphoneos-version-min=$(MIN_SDK_VERSION) \ -DGEMMLOWP_ALLOW_SLOW_SCALAR_FALLBACK \ - -DTFLITE_USE_APPLE_ACCELERATE_FOR_CONV \ + -DTF_LITE_USE_CBLAS \ -fembed-bitcode \ -Wno-c++11-narrowing \ -mno-thumb \ diff --git a/tensorflow/lite/tools/optimize/g3doc/quantize_weights.md b/tensorflow/lite/tools/optimize/g3doc/quantize_weights.md index 2517882c84c..cea164c38f0 100644 --- a/tensorflow/lite/tools/optimize/g3doc/quantize_weights.md +++ b/tensorflow/lite/tools/optimize/g3doc/quantize_weights.md @@ -3,7 +3,7 @@ ## Recommended usage The Quantize Weights transformation is integrated with -[tflite_convert](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/lite/toco/g3doc/cmdline_reference.md#transformation-flags). +[tflite_convert](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/lite/g3doc/convert/cmdline_reference.md#transformation-flags). The recommended way of invoking this tool is by simply adding the `--post_training_quantize` flag to your original tflite_convert invocation. For diff --git a/tensorflow/lite/tools/pip_package/MANIFEST.in b/tensorflow/lite/tools/pip_package/MANIFEST.in new file mode 100644 index 00000000000..bb574e63a37 --- /dev/null +++ b/tensorflow/lite/tools/pip_package/MANIFEST.in @@ -0,0 +1 @@ +recursive-include * *.py diff --git a/tensorflow/lite/tools/pip_package/README.md b/tensorflow/lite/tools/pip_package/README.md new file mode 100644 index 00000000000..8190782c39f --- /dev/null +++ b/tensorflow/lite/tools/pip_package/README.md @@ -0,0 +1,33 @@ +# Building TensorFlow Lite Standalone Pip + +Many users would like to deploy TensorFlow lite interpreter and use it from +Python without requiring the rest of TensorFlow. + +## Steps + +To build a binary wheel run this script: +``` +sudo apt install swig libjpeg-dev zlib1g-dev python3-dev python3-numpy +sh tensorflow/lite/tools/pip_package/build_pip_package.sh +``` +That will print out some output and a .whl file. You can then install that +``` +pip install --upgrade +``` + +Note, unlike tensorflow this will be installed to a tflite_runtime namespace. +You can then use the Tensorflow Lite interpreter as. +``` +import tflite_runtime as tflr +interpreter = tflr.lite.Interpreter(model_path="foo.tflite") +``` + +This currently works to build on Linux machines including Raspberry Pi. In +the future, cross compilation to smaller SOCs like Raspberry Pi from +bigger host will be supported. + +## Caveats + +* You cannot use TensorFlow Select ops, only TensorFlow Lite builtins. +* Currently custom ops and delegates cannot be registered. + diff --git a/tensorflow/lite/tools/pip_package/build_pip_package.sh b/tensorflow/lite/tools/pip_package/build_pip_package.sh new file mode 100644 index 00000000000..2887ce84712 --- /dev/null +++ b/tensorflow/lite/tools/pip_package/build_pip_package.sh @@ -0,0 +1,54 @@ +#!/usr/bin/env bash +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +set -e + +# Find where this script lives and then the Tensorflow root. +MY_DIRECTORY=`dirname $0` +export TENSORFLOW_SRC_ROOT=`realpath $MY_DIRECTORY/../../../..` + +export TENSORFLOW_VERSION=`grep "_VERSION = " $TENSORFLOW_SRC_ROOT/tensorflow/tools/pip_package/setup.py | cut -d'=' -f 2 | sed "s/[ '-]//g"`; + + +# Build a pip build tree. +BUILD_ROOT=/tmp/tflite_pip +rm -rf $BUILD_ROOT +mkdir -p $BUILD_ROOT/tflite_runtime/lite +mkdir -p $BUILD_ROOT/tflite_runtime/lite/python + +# Build an importable module tree +cat > $BUILD_ROOT/tflite_runtime/__init__.py < $BUILD_ROOT/tflite_runtime/lite/__init__.py < $BUILD_ROOT/tflite_runtime/lite/python/__init__.py < hashes); +struct TfLiteIntArrayDeleter { + void operator()(TfLiteIntArray* a) { + if (a) TfLiteIntArrayFree(a); + } +}; + } // namespace tflite #endif // TENSORFLOW_LITE_UTIL_H_ diff --git a/tensorflow/opensource_only.files b/tensorflow/opensource_only.files new file mode 100644 index 00000000000..8ff1645b989 --- /dev/null +++ b/tensorflow/opensource_only.files @@ -0,0 +1,17 @@ +tensorflow/third_party/toolchains/preconfig/ubuntu14.04/py3/BUILD +tensorflow/third_party/toolchains/preconfig/ubuntu14.04/gcc-nvcc-cuda9.0/BUILD +tensorflow/third_party/toolchains/preconfig/ubuntu14.04/gcc-nvcc/BUILD +tensorflow/third_party/toolchains/preconfig/ubuntu14.04/cuda10.0-cudnn7/cuda/build_defs.bzl +tensorflow/third_party/toolchains/preconfig/ubuntu14.04/cuda10.0-cudnn7/cuda/BUILD +tensorflow/third_party/toolchains/preconfig/ubuntu14.04/gcc-nvcc-cuda10.0/BUILD +tensorflow/third_party/toolchains/preconfig/ubuntu14.04/cuda9.0-cudnn7/cuda/build_defs.bzl +tensorflow/third_party/toolchains/preconfig/ubuntu14.04/cuda9.0-cudnn7/cuda/BUILD +tensorflow/third_party/toolchains/preconfig/ubuntu14.04/nccl2/BUILD +tensorflow/third_party/toolchains/preconfig/generate/workspace.bzl +tensorflow/third_party/toolchains/preconfig/generate/containers.bzl +tensorflow/third_party/toolchains/preconfig/generate/generate.bzl +tensorflow/third_party/toolchains/preconfig/generate/BUILD +tensorflow/third_party/toolchains/preconfig/win_1803/bazel_018/BUILD +tensorflow/third_party/toolchains/preconfig/win_1803/bazel_018/dummy_toolchain.bzl +tensorflow/third_party/toolchains/preconfig/win_1803/py36/BUILD +tensorflow/third_party/toolchains/preconfig/win_1803/BUILD \ No newline at end of file diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD index 4fe92262ba6..a558045e4af 100644 --- a/tensorflow/python/BUILD +++ b/tensorflow/python/BUILD @@ -102,6 +102,7 @@ py_library( ":framework_for_generated_wrappers", ":functional_ops", ":gradient_checker", + ":gradient_checker_v2", ":graph_util", ":histogram_ops", ":image_ops", @@ -124,7 +125,6 @@ py_library( ":session_ops", ":sets", ":sparse_ops", - ":spectral_ops", ":spectral_ops_test_util", ":standard_ops", ":state_ops", @@ -132,6 +132,7 @@ py_library( ":subscribe", ":summary", ":tensor_array_ops", + ":tensor_forest_ops", ":test_ops", # TODO: Break testing code out into separate rule. ":tf_cluster", ":tf_item", @@ -524,6 +525,17 @@ py_test( ], ) +py_test( + name = "dispatch_test", + srcs = ["util/dispatch_test.py"], + srcs_version = "PY2AND3", + deps = [ + ":client_testlib", + ":platform", + ":util", + ], +) + py_test( name = "keyword_args_test", srcs = ["util/keyword_args_test.py"], @@ -854,7 +866,6 @@ py_library( deps = [ ":c_api_util", ":control_flow_util", - ":cpp_shape_inference_proto_py", ":device", ":dtypes", ":error_interpolation", @@ -862,6 +873,7 @@ py_library( ":platform", ":registry", ":tensor_shape", + ":tf2", ":traceable_stack", ":util", ":versions", @@ -880,6 +892,8 @@ py_library( deps = [ ":auto_control_deps", ":framework_ops", + ":sparse_tensor", + ":tensor_array_ops", "//tensorflow/python/autograph", "//tensorflow/python/eager:context", "//tensorflow/python/eager:graph_only_ops", @@ -894,6 +908,8 @@ py_library( deps = [ ":control_flow_ops", ":framework_ops", + ":sparse_tensor", + ":tensor_array_ops", ":util", ], ) @@ -981,6 +997,7 @@ py_library( srcs_version = "PY2AND3", deps = [ ":dtypes", + ":tf2", ":util", "//tensorflow/core:protos_all_py", ], @@ -994,6 +1011,7 @@ py_library( ":common_shapes", ":dtypes", ":tensor_shape", + ":util", "//third_party/py/numpy", ], ) @@ -1052,6 +1070,7 @@ py_library( ":random_seed", ":resource_variable_ops", ":session", + ":tensor_array_ops", ":training", ":util", ":variables", @@ -1076,10 +1095,13 @@ py_library( srcs_version = "PY2AND3", deps = [ ":client", + ":cond_v2", ":framework_test_lib", ":gradient_checker", + ":gradient_checker_v2", ":platform_test", ":util", + ":while_v2", ], ) @@ -1384,6 +1406,7 @@ py_test( srcs_version = "PY2AND3", tags = ["no_pip"], # test_ops_2 is not available in pip. deps = [ + ":cond_v2", ":control_flow_ops", ":errors", ":framework", @@ -1398,6 +1421,7 @@ py_test( ":util", ":variable_scope", ":variables", + ":while_v2", "//tensorflow/core:protos_all_py", "//tensorflow/python/eager:context", "//tensorflow/python/eager:function", @@ -1618,6 +1642,14 @@ tf_gen_op_wrapper_private_py( ], ) +tf_gen_op_wrapper_private_py( + name = "tensor_forest_ops_gen", + visibility = ["//tensorflow:internal"], + deps = [ + "//tensorflow/core:tensor_forest_ops_op_lib", + ], +) + tf_gen_op_wrapper_private_py( name = "summary_ops_gen", visibility = ["//tensorflow:__subpackages__"], @@ -1837,6 +1869,7 @@ tf_gen_op_wrapper_private_py( tf_gen_op_wrapper_private_py( name = "spectral_ops_gen", + visibility = ["//tensorflow/python/ops/signal:__pkg__"], ) tf_gen_op_wrapper_private_py( @@ -1941,6 +1974,28 @@ py_library( ], ) +py_library( + name = "tensor_forest_ops", + srcs = ["ops/tensor_forest_ops.py"], + srcs_version = "PY2AND3", + deps = [ + ":framework", + ":ops", + ":tensor_forest_ops_gen", + ":training", + "//tensorflow/core/kernels/boosted_trees:boosted_trees_proto_py", + ], +) + +py_library( + name = "optional_grad", + srcs = ["ops/optional_grad.py"], + srcs_version = "PY2AND3", + deps = [ + ":framework_ops", + ], +) + py_library( name = "sets", srcs = [ @@ -2056,7 +2111,6 @@ py_library( srcs = ["ops/control_flow_ops.py"], srcs_version = "PY2AND3", deps = [ - "tensor_shape", ":array_ops", ":array_ops_gen", ":constant_op", @@ -2071,6 +2125,7 @@ py_library( ":resource_variable_ops_gen", ":sparse_tensor", ":tensor_array_ops", + ":tensor_shape", ":tf2", ":tf_should_use", ":util", @@ -2093,7 +2148,9 @@ py_library( srcs = ["ops/control_flow_util_v2.py"], srcs_version = "PY2AND3", deps = [ - "framework_ops", + ":control_flow_util", + ":framework_ops", + "//tensorflow/core:protos_all_py", "//tensorflow/python/eager:context", "//tensorflow/python/eager:function", ], @@ -2118,7 +2175,7 @@ py_library( ":graph_to_function_def", ":pywrap_tensorflow", ":util", - "//tensorflow/core:protos_all_py", + "//tensorflow/python/data/ops:dataset_ops", "//tensorflow/python/eager:function", ], ) @@ -2145,7 +2202,6 @@ py_library( ":tensor_shape", ":tensor_util", ":util", - "//tensorflow/core:protos_all_py", "//tensorflow/python/eager:function", ], ) @@ -2263,10 +2319,10 @@ py_library( ":manip_ops", ":math_grad", ":math_ops", + ":optional_grad", ":platform", ":random_grad", ":resource_variable_ops", - ":spectral_grad", ":tensor_array_ops", ":tensor_util", ":unconnected_gradients", @@ -2508,7 +2564,6 @@ py_library( ":nn_ops_gen", ":sparse_ops_gen", ":sparse_tensor", - ":spectral_ops_gen", ":state_ops", ":state_ops_gen", ":tensor_shape", @@ -2814,33 +2869,34 @@ py_test( ":framework_test_lib", ":sparse_ops", ":sparse_tensor", + "@absl_py//absl/testing:parameterized", ], ) py_library( - name = "spectral_grad", - srcs = ["ops/spectral_grad.py"], + name = "sort_ops", + srcs = ["ops/sort_ops.py"], srcs_version = "PY2AND3", deps = [ ":array_ops", ":framework", - ":framework_for_generated_wrappers", ":math_ops", - ":spectral_ops", + ":nn_ops", "//third_party/py/numpy", ], ) -py_library( - name = "spectral_ops", - srcs = ["ops/spectral_ops.py"], +py_test( + name = "sort_ops_test", + srcs = ["ops/sort_ops_test.py"], srcs_version = "PY2AND3", deps = [ ":array_ops", - ":dtypes", - ":framework_ops", - ":math_ops", - ":spectral_ops_gen", + ":client_testlib", + ":framework", + ":random_ops", + ":sort_ops", + "//third_party/py/numpy", ], ) @@ -2958,10 +3014,10 @@ py_library( ":random_ops", ":script_ops", ":session_ops", + ":sort_ops", ":sparse_grad", ":sparse_ops", ":special_math_ops", - ":spectral_grad", ":state_grad", ":state_ops", ":stateless_random_ops", @@ -2972,6 +3028,7 @@ py_library( ":util", ":variable_scope", ":variables", + "//tensorflow/python/eager:wrap_function", "//tensorflow/python/ops/distributions", "//tensorflow/python/ops/linalg", ], @@ -3066,13 +3123,16 @@ py_library( deps = [ ":array_ops", ":constant_op", + ":control_flow_ops_gen", ":data_flow_ops_gen", ":dtypes", ":errors", ":framework_ops", + ":list_ops", ":math_ops", ":tensor_shape", ":tensor_util", + ":tf2", ":tf_should_use", "//tensorflow/python/eager:context", ], @@ -3131,6 +3191,19 @@ py_library( ], ) +py_library( + name = "gradient_checker_v2", + srcs = ["ops/gradient_checker_v2.py"], + srcs_version = "PY2AND3", + deps = [ + ":array_ops", + ":framework_for_generated_wrappers", + ":gradients", + ":platform", + "//third_party/py/numpy", + ], +) + # This target is deprecated. py_library( name = "ops", @@ -3196,6 +3269,22 @@ cuda_py_test( ], ) +cuda_py_test( + name = "gradient_checker_v2_test", + size = "medium", + srcs = ["ops/gradient_checker_v2_test.py"], + additional_deps = [ + ":array_ops", + ":client_testlib", + ":framework_for_generated_wrappers", + ":math_ops", + ":nn_grad", + ":nn_ops", + ":platform", + "//third_party/py/numpy", + ], +) + cuda_py_test( name = "gradients_test", size = "medium", @@ -3304,6 +3393,9 @@ cuda_py_test( ":client_testlib", ":framework_for_generated_wrappers", ":math_ops", + "//tensorflow/python/eager:backprop", + "//tensorflow/python/eager:context", + "//tensorflow/python/eager:execution_callbacks", "//third_party/py/numpy", ], tags = ["no_windows_gpu"], @@ -3475,6 +3567,7 @@ py_library( "@six_archive//:six", "//tensorflow/core:protos_all_py", "//tensorflow/python/data/ops:dataset_ops", + "//tensorflow/python/distribute:reduce_util", "//tensorflow/python/distribute:distribute_coordinator_context", "//tensorflow/python/eager:backprop", "//tensorflow/python/eager:context", @@ -3486,6 +3579,19 @@ py_library( ], ) +# Dependency added and used by ClusterResolvers to avoid circular dependency between keras, distribute, and training. +py_library( + name = "training_server_lib", + srcs = ["training/server_lib.py"], + srcs_version = "PY2AND3", + deps = [ + ":framework", + ":pywrap_tensorflow", + ":util", + "//tensorflow/core:protos_all_py", + ], +) + py_library( name = "saveable_object", srcs = ["training/saveable_object.py"], @@ -3560,17 +3666,6 @@ py_library( ], ) -py_library( - name = "device_util", - srcs = ["training/device_util.py"], - srcs_version = "PY2AND3", - deps = [ - ":device", - ":framework_ops", - "//tensorflow/python/eager:context", - ], -) - py_library( name = "distribute", srcs = [ @@ -3579,29 +3674,7 @@ py_library( ], srcs_version = "PY2AND3", deps = [ - ":array_ops", - ":control_flow_ops", - ":device_util", - ":framework_ops", - ":platform", - ":resource_variable_ops", - ":state_ops", - ":util", - ":variable_scope", - "//tensorflow/python/data", - "//tensorflow/python/ops/losses", - ], -) - -py_test( - name = "distribute_test", - size = "small", - srcs = ["training/distribute_test.py"], - srcs_version = "PY2AND3", - deps = [ - ":client_testlib", - ":distribute", - ":variable_scope", + "//tensorflow/python/distribute:distribute_lib", ], ) @@ -4108,11 +4181,24 @@ genrule( # Get the import library of _pywrap_tensorflow_internal.dll filegroup( - name = "pywrap_tensorflow_import_lib_file", + name = "get_pywrap_tensorflow_import_lib_file", srcs = [":_pywrap_tensorflow_internal.so"], output_group = "interface_library", ) +# Rename the import library for _pywrap_tensorflow_internal.pyd to _pywrap_tensorflow_internal.lib +# (It was _pywrap_tensorflow_internal.so.if.lib). +genrule( + name = "pywrap_tensorflow_import_lib_file", + srcs = [":get_pywrap_tensorflow_import_lib_file"], + outs = ["_pywrap_tensorflow_internal.lib"], + cmd = select({ + "//tensorflow:windows": "cp -f $< $@", + "//conditions:default": "touch $@", # Just a placeholder for Unix platforms + }), + visibility = ["//visibility:public"], +) + # Create a cc_import rule for the import library of _pywrap_tensorflow_internal.dll # so that custom ops' dynamic libraries can link against it. cc_import( @@ -4590,7 +4676,6 @@ cuda_py_tests( "training/basic_loops_test.py", "training/coordinator_test.py", "training/device_setter_test.py", - "training/device_util_test.py", "training/ftrl_test.py", "training/gradient_descent_test.py", "training/learning_rate_decay_test.py", @@ -4901,7 +4986,7 @@ py_test( ":training", ":variable_scope", ":variables", - "//tensorflow/python/feature_column", + "//tensorflow/python/feature_column:feature_column_py", "//third_party/py/numpy", ], ) diff --git a/tensorflow/python/__init__.py b/tensorflow/python/__init__.py index 5da304e38cc..547043030b1 100644 --- a/tensorflow/python/__init__.py +++ b/tensorflow/python/__init__.py @@ -86,12 +86,12 @@ from tensorflow.python.ops import image_ops as image from tensorflow.python.ops import manip_ops as manip from tensorflow.python.ops import metrics from tensorflow.python.ops import nn +from tensorflow.python.ops import ragged from tensorflow.python.ops import sets -from tensorflow.python.ops import spectral_ops as spectral from tensorflow.python.ops.distributions import distributions from tensorflow.python.ops.linalg import linalg from tensorflow.python.ops.losses import losses -from tensorflow.python.ops import signal +from tensorflow.python.ops.signal import signal from tensorflow.python.profiler import profiler from tensorflow.python.saved_model import saved_model from tensorflow.python.summary import summary @@ -163,7 +163,7 @@ tf_export('Summary', 'summary.Summary')(Summary) tf_export('summary.SummaryDescription')(SummaryDescription) tf_export('SummaryMetadata')(SummaryMetadata) tf_export('summary.TaggedRunMetadata')(TaggedRunMetadata) -tf_export('TensorInfo')(TensorInfo) +tf_export(v1=['TensorInfo'])(TensorInfo) # pylint: enable=undefined-variable # Special dunders that we choose to export: diff --git a/tensorflow/python/autograph/converters/BUILD b/tensorflow/python/autograph/converters/BUILD index ced2e4796b1..3ac446db02c 100644 --- a/tensorflow/python/autograph/converters/BUILD +++ b/tensorflow/python/autograph/converters/BUILD @@ -63,7 +63,6 @@ py_test( name = "asserts_test", srcs = ["asserts_test.py"], srcs_version = "PY2AND3", - tags = ["no_windows"], deps = [ ":converters", "//tensorflow/python:client_testlib", @@ -239,7 +238,6 @@ py_test( name = "error_handlers_test", srcs = ["error_handlers_test.py"], srcs_version = "PY2AND3", - tags = ["no_windows"], deps = [ ":converters", "//tensorflow/python:client_testlib", diff --git a/tensorflow/python/autograph/converters/asserts_test.py b/tensorflow/python/autograph/converters/asserts_test.py index eef628aeb6f..9ae448892a0 100644 --- a/tensorflow/python/autograph/converters/asserts_test.py +++ b/tensorflow/python/autograph/converters/asserts_test.py @@ -23,12 +23,14 @@ from tensorflow.python.autograph.converters import side_effect_guards from tensorflow.python.autograph.core import converter_testing from tensorflow.python.framework import constant_op from tensorflow.python.framework import errors_impl +from tensorflow.python.framework import test_util from tensorflow.python.ops import gen_control_flow_ops from tensorflow.python.platform import test class AssertsTest(converter_testing.TestCase): + @test_util.run_deprecated_v1 def test_basic(self): def test_fn(a): @@ -41,7 +43,7 @@ class AssertsTest(converter_testing.TestCase): op = result.test_fn(constant_op.constant(False)) with self.assertRaisesRegexp(errors_impl.InvalidArgumentError, 'test message'): - sess.run(op) + self.evaluate(op) if __name__ == '__main__': diff --git a/tensorflow/python/autograph/converters/builtin_functions_test.py b/tensorflow/python/autograph/converters/builtin_functions_test.py index 30cfb13233a..2683be16ec7 100644 --- a/tensorflow/python/autograph/converters/builtin_functions_test.py +++ b/tensorflow/python/autograph/converters/builtin_functions_test.py @@ -24,12 +24,14 @@ from tensorflow.python.autograph.converters import builtin_functions from tensorflow.python.autograph.core import converter_testing from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes +from tensorflow.python.framework import test_util from tensorflow.python.ops import array_ops from tensorflow.python.platform import test class BuiltinFunctionsTest(converter_testing.TestCase): + @test_util.run_deprecated_v1 def test_len(self): def test_fn(a): @@ -41,6 +43,7 @@ class BuiltinFunctionsTest(converter_testing.TestCase): ops = result.test_fn(p) self.assertEqual(sess.run(ops, {p: [0, 0, 0]}), 3) + @test_util.run_deprecated_v1 def test_print(self): if six.PY2: @@ -54,6 +57,7 @@ class BuiltinFunctionsTest(converter_testing.TestCase): with self.assertPrints('a\n'): sess.run(result.test_fn('a')) + @test_util.run_deprecated_v1 def test_print_multiple_values(self): if six.PY2: diff --git a/tensorflow/python/autograph/converters/call_trees.py b/tensorflow/python/autograph/converters/call_trees.py index 55cea89126a..9b85fc8367c 100644 --- a/tensorflow/python/autograph/converters/call_trees.py +++ b/tensorflow/python/autograph/converters/call_trees.py @@ -22,7 +22,7 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function -from collections import namedtuple +import collections import gast @@ -35,7 +35,7 @@ from tensorflow.python.autograph.pyct import templates from tensorflow.python.util import tf_inspect -class FunctionInfo(namedtuple('FunctionInfo', ('dtype',))): +class FunctionInfo(collections.namedtuple('FunctionInfo', ('dtype',))): pass @@ -116,12 +116,19 @@ class CallTreeTransformer(converter.Base): def _function_is_compilable(self, target_entity): """Determines whether an entity can be compiled at all.""" # TODO(mdan): Expand. + if target_entity.__module__ is None: # Functions like builtins and NumPy don't expose a module. # Those in general should not be compiled. return False + if inspect_utils.isbuiltin(target_entity): return False + + if inspect_utils.isnamedtuple(target_entity): + # namedtuple doesn't expose its source code, making it uncompilable. + return False + return True def _should_compile(self, node, fqn): @@ -140,6 +147,11 @@ class CallTreeTransformer(converter.Base): if target_entity is not None: + # Currently, lambdas are always converted. + # TODO(mdan): Allow markers of the kind f = ag.do_not_convert(lambda: ...) + if inspect_utils.islambda(target_entity): + return True + # This may be reached when "calling" a callable attribute of an object. # For example: # @@ -296,7 +308,13 @@ class CallTreeTransformer(converter.Base): # safe for graph mode. return node + elif inspect_utils.isnamedtuple(target_entity): + # Although not compilable, we assume they are safe for graph mode. + node = self.generic_visit(node) + return node + else: + # TODO(mdan): Instert dynamic conversion here instead. raise NotImplementedError( 'py_func with return values (unknown function)') else: diff --git a/tensorflow/python/autograph/converters/call_trees_test.py b/tensorflow/python/autograph/converters/call_trees_test.py index 916c736fb4b..454d75d755c 100644 --- a/tensorflow/python/autograph/converters/call_trees_test.py +++ b/tensorflow/python/autograph/converters/call_trees_test.py @@ -18,6 +18,8 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function +import collections + import numpy as np from tensorflow.python.autograph.converters import call_trees @@ -85,6 +87,34 @@ class CallTreesTest(converter_testing.TestCase): tc = TestClass() self.assertEquals(3, result.test_fn_2(tc, 1)) + def test_known_called_lambda(self): + + l = lambda x: x + + def test_fn(a): + return l(a) + + ns = {'l': l} + node, ctx = self.prepare(test_fn, ns) + node = call_trees.transform(node, ctx) + + with self.compiled(node, ns) as result: + self.assertEquals(1, result.test_fn(1)) + + def test_known_called_namedtuple(self): + + nt = collections.namedtuple('TestNamedTuple', ['a']) + + def test_fn(a): + return nt(a) + + ns = {'nt': nt} + node, ctx = self.prepare(test_fn, ns) + node = call_trees.transform(node, ctx) + + with self.compiled(node, ns) as result: + self.assertEquals(nt(1), result.test_fn(1)) + def test_py_func_known_function(self): def test_fn(): @@ -94,7 +124,7 @@ class CallTreesTest(converter_testing.TestCase): dtypes.int64) as result: with self.cached_session() as sess: self.assertTrue(isinstance(result.test_fn(), ops.Tensor)) - self.assertIn(sess.run(result.test_fn()), (0, 1, 2)) + self.assertIn(self.evaluate(result.test_fn()), (0, 1, 2)) def test_uncompiled_modules(self): @@ -113,7 +143,7 @@ class CallTreesTest(converter_testing.TestCase): with self.compiled(node, ns) as result: with self.cached_session() as sess: result_tensor = result.test_fn(constant_op.constant(1)) - self.assertEquals(sess.run(result_tensor), 3) + self.assertEquals(self.evaluate(result_tensor), 3) def test_call_to_decorated_function(self): diff --git a/tensorflow/python/autograph/converters/continue_statements.py b/tensorflow/python/autograph/converters/continue_statements.py index 584cdc1efd4..05e19e59fc6 100644 --- a/tensorflow/python/autograph/converters/continue_statements.py +++ b/tensorflow/python/autograph/converters/continue_statements.py @@ -24,94 +24,93 @@ from tensorflow.python.autograph.pyct import templates from tensorflow.python.autograph.pyct.static_analysis.annos import NodeAnno -# Tags for local state. -CONTROL_VAR_NAME = 'control_var_name' -CONTINUE_USED = 'continue_used' -GUARD_CREATED = 'guard_created' -CREATE_GUARD_NEXT = 'create_guard_next' +class _Continue(object): + + def __init__(self): + self.used = False + self.control_var_name = None + self.create_guard = False + self.guard_created = False + + def __repr__(self): + return 'used: %s, var: %s' % (self.used, self.control_var_name) class ContinueCanonicalizationTransformer(converter.Base): """Canonicalizes continue statements into additional conditionals.""" def visit_Continue(self, node): - self.set_local(CONTINUE_USED, True) + self.state[_Continue].used = True template = """ - var_name = tf.constant(True) + var_name = True """ return templates.replace( - template, var_name=self.get_local(CONTROL_VAR_NAME)) + template, var_name=self.state[_Continue].control_var_name) def _postprocess_statement(self, node): # Example of how the state machine below works: # - # 1| stmt # State: CONTINUE_USED = False + # 1| stmt # State: Continue_.used = False # | # Action: none # 2| if cond: - # 3| continue # State: CONTINUE_USED = True, - # | # GUARD_CREATED = False, - # | # CREATE_GUARD_NEXT = False - # | # Action: set CREATE_GUARD_NEXT = True - # 4| stmt # State: CONTINUE_USED = True, - # | # GUARD_CREATED = False, - # | # CREATE_GUARD_NEXT = True + # 3| continue # State: Continue_.used = True, + # | # Continue_.guard_created = False, + # | # Continue_.create_guard = False + # | # Action: Continue_.create_guard = True + # 4| stmt # State: Continue_.used = True, + # | # Continue_.guard_created = False, + # | # Continue_.create_guard = True # | # Action: create `if not continue_used`, - # | # set GUARD_CREATED = True - # 5| stmt # State: CONTINUE_USED = True, GUARD_CREATED = True + # | # set Continue_.guard_created = True + # 5| stmt # State: Continue_.used = True, + # | # Continue_.guard_created = True # | # Action: none (will be wrapped under previously # | # created if node) - if self.get_local(CONTINUE_USED, False): - if self.get_local(GUARD_CREATED, False): + if self.state[_Continue].used: + if self.state[_Continue].guard_created: return node, None - elif not self.get_local(CREATE_GUARD_NEXT, False): - self.set_local(CREATE_GUARD_NEXT, True) + elif not self.state[_Continue].create_guard: + self.state[_Continue].create_guard = True return node, None else: - self.set_local(GUARD_CREATED, True) + self.state[_Continue].guard_created = True template = """ if not var_name: original_node """ cond, = templates.replace( template, - var_name=self.get_local(CONTROL_VAR_NAME), + var_name=self.state[_Continue].control_var_name, original_node=node) return cond, cond.body return node, None def _visit_loop_body(self, node, nodes): - self.enter_local_scope() + self.state[_Continue].enter() scope = anno.getanno(node, NodeAnno.BODY_SCOPE) continue_var = self.ctx.namer.new_symbol('continue_', scope.referenced) - self.set_local(CONTROL_VAR_NAME, continue_var) + self.state[_Continue].control_var_name = continue_var nodes = self.visit_block(nodes, after_visit=self._postprocess_statement) - if self.get_local(CONTINUE_USED, False): + if self.state[_Continue].used: template = """ - var_name = tf.constant(False) + var_name = False """ control_var_init = templates.replace(template, var_name=continue_var) nodes = control_var_init + nodes - self.exit_local_scope() + self.state[_Continue].exit() return nodes - def _visit_non_loop_body(self, nodes): - self.enter_local_scope(inherit=(CONTROL_VAR_NAME,)) - nodes = self.visit_block(nodes, after_visit=self._postprocess_statement) - continue_used = self.get_local(CONTINUE_USED, False) - self.exit_local_scope(keep=(CONTINUE_USED,)) - return nodes, continue_used - def visit_While(self, node): node.test = self.visit(node.test) node.body = self._visit_loop_body(node, node.body) # A continue in the else clause applies to the containing scope. - node.orelse, _ = self._visit_non_loop_body(node.orelse) + node.orelse = self.visit_block(node.orelse) return node def visit_For(self, node): @@ -119,21 +118,11 @@ class ContinueCanonicalizationTransformer(converter.Base): node.iter = self.generic_visit(node.iter) node.body = self._visit_loop_body(node, node.body) # A continue in the else clause applies to the containing scope. - node.orelse, _ = self._visit_non_loop_body(node.orelse) - return node - - def visit_If(self, node): - node.test = self.generic_visit(node.test) - node.body, continue_used_body = self._visit_non_loop_body(node.body) - node.orelse, continue_used_orelse = self._visit_non_loop_body(node.orelse) - self.set_local(CONTINUE_USED, continue_used_body or continue_used_orelse) - return node - - def visit_With(self, node): - node.items = self.visit_block(node.items) - node.body, _ = self._visit_non_loop_body(node.body) + node.orelse = self.visit_block(node.orelse) return node def transform(node, ctx): - return ContinueCanonicalizationTransformer(ctx).visit(node) + transformer = ContinueCanonicalizationTransformer(ctx) + node = transformer.visit(node) + return node diff --git a/tensorflow/python/autograph/converters/control_flow.py b/tensorflow/python/autograph/converters/control_flow.py index 5853e044c53..bef6cae1bb8 100644 --- a/tensorflow/python/autograph/converters/control_flow.py +++ b/tensorflow/python/autograph/converters/control_flow.py @@ -106,14 +106,49 @@ class ControlFlowTransformer(converter.Base): return 'no variables' return ', '.join(map(str, symbol_set)) - def visit_If(self, node): - node = self.generic_visit(node) + def _determine_aliased_symbols(self, scope, node_defined_in, block): + if block: + block_live_in = set(anno.getanno(block[0], anno.Static.LIVE_VARS_IN)) + else: + block_live_in = set() + # For the purpose of aliasing, composite symbols with live owners are live + # as well. Otherwise this would leak tensors from the conditional's body. + # + # For example: + # + # obj = some_obj + # if cond: + # obj.a = val + # + # Thanslating to the code below would be incorrect: + # + # def true_fn(): + # obj.a = val() # Wrong! leaks ops owned by true_fn + # return obj.a + for s in scope.modified: + if s.is_composite(): + live_parents = block_live_in & s.owner_set + if live_parents: + block_live_in.add(s) + return scope.modified & node_defined_in & block_live_in + + def visit_If(self, node): body_scope = anno.getanno(node, annos.NodeAnno.BODY_SCOPE) orelse_scope = anno.getanno(node, annos.NodeAnno.ORELSE_SCOPE) defined_in = anno.getanno(node, anno.Static.DEFINED_VARS_IN) live_out = anno.getanno(node, anno.Static.LIVE_VARS_OUT) + # Note: this information needs to be extracted before the body conversion + # that happens in the call to generic_visit below, because the conversion + # generates nodes that lack static analysis annotations. + need_alias_in_body = self._determine_aliased_symbols( + body_scope, defined_in, node.body) + need_alias_in_orelse = self._determine_aliased_symbols( + orelse_scope, defined_in, node.orelse) + + node = self.generic_visit(node) + modified_in_cond = body_scope.modified | orelse_scope.modified returned_from_cond = set() for s in modified_in_cond: @@ -125,9 +160,6 @@ class ControlFlowTransformer(converter.Base): if live_out & s.owner_set: returned_from_cond.add(s) - need_alias_in_body = body_scope.modified & defined_in - need_alias_in_orelse = orelse_scope.modified & defined_in - created_in_body = body_scope.modified & returned_from_cond - defined_in created_in_orelse = orelse_scope.modified & returned_from_cond - defined_in diff --git a/tensorflow/python/autograph/converters/control_flow_test.py b/tensorflow/python/autograph/converters/control_flow_test.py index 03fdfc804e4..034fcbe3865 100644 --- a/tensorflow/python/autograph/converters/control_flow_test.py +++ b/tensorflow/python/autograph/converters/control_flow_test.py @@ -23,6 +23,7 @@ from tensorflow.python.autograph.core import converter_testing from tensorflow.python.autograph.pyct import transformer from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes +from tensorflow.python.framework import test_util from tensorflow.python.platform import test @@ -36,6 +37,7 @@ class ControlFlowTest(converter_testing.TestCase): with self.cached_session() as sess: self.assertEqual(sess.run(result.test_fn(*inputs)), expected) + @test_util.run_deprecated_v1 def test_while_basic(self): def test_fn(n): @@ -48,6 +50,7 @@ class ControlFlowTest(converter_testing.TestCase): self.assertTransformedResult(test_fn, constant_op.constant(5), (10, 5, 5)) + @test_util.run_deprecated_v1 def test_while_nested(self): def test_fn(n): @@ -66,6 +69,7 @@ class ControlFlowTest(converter_testing.TestCase): self.assertTransformedResult(test_fn, constant_op.constant(5), (25, 5, 0, 5)) + @test_util.run_deprecated_v1 def test_while_single_output(self): def test_fn(n): @@ -86,6 +90,7 @@ class ControlFlowTest(converter_testing.TestCase): with self.assertRaises(NameError): control_flow.transform(node, ctx) + @test_util.run_deprecated_v1 def test_if_basic(self): def test_fn(n): @@ -100,6 +105,7 @@ class ControlFlowTest(converter_testing.TestCase): self.assertTransformedResult(test_fn, constant_op.constant(1), (-1, 0)) self.assertTransformedResult(test_fn, constant_op.constant(-1), (0, -2)) + @test_util.run_deprecated_v1 def test_if_complex_outputs(self): class TestClass(object): @@ -124,6 +130,7 @@ class ControlFlowTest(converter_testing.TestCase): res_obj = result.test_fn(constant_op.constant(-1), TestClass(0, 0)) self.assertEqual(sess.run((res_obj.a, res_obj.b)), (0, -2)) + @test_util.run_deprecated_v1 def test_if_single_output(self): def test_fn(n): @@ -133,6 +140,7 @@ class ControlFlowTest(converter_testing.TestCase): self.assertTransformedResult(test_fn, constant_op.constant(1), -1) + @test_util.run_deprecated_v1 def test_if_semi(self): def test_fn(n): @@ -143,6 +151,7 @@ class ControlFlowTest(converter_testing.TestCase): self.assertTransformedResult(test_fn, constant_op.constant(2), 3) self.assertTransformedResult(test_fn, constant_op.constant(-3), -3) + @test_util.run_deprecated_v1 def test_if_local_var(self): def test_fn(n): @@ -154,6 +163,7 @@ class ControlFlowTest(converter_testing.TestCase): self.assertTransformedResult(test_fn, constant_op.constant(1), 5) self.assertTransformedResult(test_fn, constant_op.constant(-1), -1) + @test_util.run_deprecated_v1 def test_if_no_outputs(self): def test_fn(n): @@ -177,6 +187,7 @@ class ControlFlowTest(converter_testing.TestCase): with self.assertRaises(transformer.AutographParseError): control_flow.transform(node, ctx) + @test_util.run_deprecated_v1 def test_simple_for(self): def test_fn(l): @@ -191,6 +202,7 @@ class ControlFlowTest(converter_testing.TestCase): empty_vector = constant_op.constant([], shape=(0,), dtype=dtypes.int32) self.assertTransformedResult(test_fn, empty_vector, (0, 0)) + @test_util.run_deprecated_v1 def test_for_single_output(self): def test_fn(l): @@ -235,6 +247,7 @@ class ControlFlowTest(converter_testing.TestCase): with self.assertRaises(NameError): control_flow.transform(node, ctx) + @test_util.run_deprecated_v1 def test_for_tuple_unpacking(self): def test_fn(x_list): z = tf.constant(0) # pylint:disable=undefined-variable diff --git a/tensorflow/python/autograph/converters/function_scopes_test.py b/tensorflow/python/autograph/converters/function_scopes_test.py index e5ce03a1090..5a1248c8015 100644 --- a/tensorflow/python/autograph/converters/function_scopes_test.py +++ b/tensorflow/python/autograph/converters/function_scopes_test.py @@ -22,11 +22,13 @@ from tensorflow.python.autograph.converters import function_scopes from tensorflow.python.autograph.core import converter_testing from tensorflow.python.framework import constant_op from tensorflow.python.framework import ops +from tensorflow.python.framework import test_util from tensorflow.python.platform import test class FunctionBodyTransformerTest(converter_testing.TestCase): + @test_util.run_deprecated_v1 def test_basic(self): def test_fn(l): @@ -40,6 +42,7 @@ class FunctionBodyTransformerTest(converter_testing.TestCase): self.assertIn('test_fn/', result_op.op.name) self.assertEqual('Docstring.', result.test_fn.__doc__) + @test_util.run_deprecated_v1 def test_multiline_docstring(self): tf = None @@ -58,6 +61,7 @@ class FunctionBodyTransformerTest(converter_testing.TestCase): self.assertIn('First sentence.', result.test_fn.__doc__) self.assertIn('Second sentence.', result.test_fn.__doc__) + @test_util.run_deprecated_v1 def test_nested_functions(self): def test_fn(l): @@ -74,6 +78,7 @@ class FunctionBodyTransformerTest(converter_testing.TestCase): self.assertNotIn('inner_fn', first.op.name) self.assertIn('test_fn/inner_fn/', second.op.name) + @test_util.run_deprecated_v1 def test_method(self): class TestClass(object): diff --git a/tensorflow/python/autograph/converters/lists_test.py b/tensorflow/python/autograph/converters/lists_test.py index f6da845fcc3..39843c7d74f 100644 --- a/tensorflow/python/autograph/converters/lists_test.py +++ b/tensorflow/python/autograph/converters/lists_test.py @@ -68,7 +68,7 @@ class ListTest(converter_testing.TestCase): with self.cached_session() as sess: tl = result.test_fn() r = list_ops.tensor_list_stack(tl, dtypes.int32) - self.assertAllEqual(sess.run(r), [1, 2, 3]) + self.assertAllEqual(self.evaluate(r), [1, 2, 3]) def test_list_pop(self): @@ -91,8 +91,8 @@ class ListTest(converter_testing.TestCase): with self.cached_session() as sess: ts, tl = result.test_fn() r = list_ops.tensor_list_stack(tl, dtypes.int32) - self.assertAllEqual(sess.run(r), [1, 2]) - self.assertAllEqual(sess.run(ts), 3) + self.assertAllEqual(self.evaluate(r), [1, 2]) + self.assertAllEqual(self.evaluate(ts), 3) def test_double_list_pop(self): @@ -123,7 +123,7 @@ class ListTest(converter_testing.TestCase): with self.compiled(node, {}, array_ops.stack, dtypes.int32) as result: with self.cached_session() as sess: - self.assertAllEqual(sess.run(result.test_fn()), [1, 2, 3]) + self.assertAllEqual(self.evaluate(result.test_fn()), [1, 2, 3]) # TODO(mdan): Add a test with tf.stack with axis kwarg. diff --git a/tensorflow/python/autograph/converters/logical_expressions_test.py b/tensorflow/python/autograph/converters/logical_expressions_test.py index 99db04a7751..687412750e0 100644 --- a/tensorflow/python/autograph/converters/logical_expressions_test.py +++ b/tensorflow/python/autograph/converters/logical_expressions_test.py @@ -21,11 +21,13 @@ from __future__ import print_function from tensorflow.python.autograph.converters import logical_expressions from tensorflow.python.autograph.core import converter_testing from tensorflow.python.framework import constant_op +from tensorflow.python.framework import test_util from tensorflow.python.platform import test class LogicalExpressionTest(converter_testing.TestCase): + @test_util.run_deprecated_v1 def test_equals(self): def test_fn(a, b): @@ -36,6 +38,7 @@ class LogicalExpressionTest(converter_testing.TestCase): self.assertTrue(sess.run(result.test_fn(constant_op.constant(1), 1))) self.assertFalse(sess.run(result.test_fn(constant_op.constant(1), 2))) + @test_util.run_deprecated_v1 def test_bool_ops(self): def test_fn(a, b, c): @@ -48,6 +51,7 @@ class LogicalExpressionTest(converter_testing.TestCase): self.assertFalse( sess.run(result.test_fn(constant_op.constant(True), False, True))) + @test_util.run_deprecated_v1 def test_comparison(self): def test_fn(a, b, c, d): diff --git a/tensorflow/python/autograph/converters/side_effect_guards_test.py b/tensorflow/python/autograph/converters/side_effect_guards_test.py index cef3199169c..645267e5600 100644 --- a/tensorflow/python/autograph/converters/side_effect_guards_test.py +++ b/tensorflow/python/autograph/converters/side_effect_guards_test.py @@ -23,6 +23,7 @@ from tensorflow.python.autograph.core import converter_testing from tensorflow.python.framework import constant_op from tensorflow.python.framework import errors_impl from tensorflow.python.framework import ops +from tensorflow.python.framework import test_util from tensorflow.python.ops import control_flow_ops from tensorflow.python.ops import state_ops from tensorflow.python.ops import variable_scope @@ -34,6 +35,7 @@ tf = None # Will be replaced by a mock. class SideEffectGuardsTest(converter_testing.TestCase): + @test_util.run_deprecated_v1 def test_side_effect_on_return_only_variable(self): def test_fn(a): @@ -48,12 +50,12 @@ class SideEffectGuardsTest(converter_testing.TestCase): with self.compiled(node, {}, state_ops.assign) as result: with self.cached_session() as sess: v = variable_scope.get_variable('test', initializer=2) - sess.run(v.initializer) - sess.run(result.test_fn(v)) + self.evaluate(v.initializer) + self.evaluate(result.test_fn(v)) # TODO(mdan): Add support for this use case. # Right now the variable `a` is not conditioned on the `assign` because # there's no way to add control dependencies to a variable object. - self.assertEqual(2, sess.run(v)) + self.assertEqual(2, self.evaluate(v)) def test_side_effect_on_used_variable(self): @@ -69,12 +71,13 @@ class SideEffectGuardsTest(converter_testing.TestCase): with self.compiled(node, {}, state_ops.assign) as result: with self.cached_session() as sess: v = variable_scope.get_variable('test', initializer=2) - sess.run(v.initializer) - sess.run(result.test_fn(v)) + self.evaluate(v.initializer) + self.evaluate(result.test_fn(v)) # TODO(mdan): Ensure the result of test_fn(v) is also deterministic. # Right now it's 3 or 4 based on whether the read is synchronized. - self.assertEqual(3, sess.run(v)) + self.assertEqual(3, self.evaluate(v)) + @test_util.run_deprecated_v1 def test_side_effect_on_tensor(self): def test_fn(a): @@ -109,10 +112,10 @@ class SideEffectGuardsTest(converter_testing.TestCase): with self.compiled(node, {}, state_ops.assign_add) as result: with self.cached_session() as sess: v = variable_scope.get_variable('test', initializer=2) - sess.run(v.initializer) - sess.run(result.test_fn(v)) + self.evaluate(v.initializer) + self.evaluate(result.test_fn(v)) # TODO(mdan): Ensure the result of test_fn(v) is also deterministic. - self.assertEqual(4, sess.run(v)) + self.assertEqual(4, self.evaluate(v)) def test_multiline_nested_block(self): @@ -130,10 +133,10 @@ class SideEffectGuardsTest(converter_testing.TestCase): with self.compiled(node, {}, state_ops.assign, ops.name_scope) as result: with self.cached_session() as sess: v = variable_scope.get_variable('test', initializer=2) - sess.run(v.initializer) - sess.run(result.test_fn(v)) + self.evaluate(v.initializer) + self.evaluate(result.test_fn(v)) # TODO(mdan): Ensure the result of test_fn(v) is also deterministic. - self.assertEqual(3, sess.run(v)) + self.assertEqual(3, self.evaluate(v)) def test_multiline_block_unsafe(self): @@ -153,10 +156,10 @@ class SideEffectGuardsTest(converter_testing.TestCase): state_ops.assign_add) as result: with self.cached_session() as sess: v = variable_scope.get_variable('test', initializer=2) - sess.run(v.initializer) - sess.run(result.test_fn(v)) + self.evaluate(v.initializer) + self.evaluate(result.test_fn(v)) # TODO(mdan): Ensure the result of test_fn(v) is also deterministic. - self.assertEqual(4, sess.run(v)) + self.assertEqual(4, self.evaluate(v)) if __name__ == '__main__': diff --git a/tensorflow/python/autograph/converters/slices_test.py b/tensorflow/python/autograph/converters/slices_test.py index e190a7cfe84..bd049afdfce 100644 --- a/tensorflow/python/autograph/converters/slices_test.py +++ b/tensorflow/python/autograph/converters/slices_test.py @@ -49,7 +49,7 @@ class SliceTest(converter_testing.TestCase): tl = list_ops.tensor_list_from_tensor( [1, 2], element_shape=constant_op.constant([], dtype=dtypes.int32)) y = result.test_fn(tl) - self.assertEqual(2, sess.run(y)) + self.assertEqual(2, self.evaluate(y)) def test_index_access_multiple_definitions(self): diff --git a/tensorflow/python/autograph/core/converter.py b/tensorflow/python/autograph/core/converter.py index 49e24895a2b..e88c4674ee2 100644 --- a/tensorflow/python/autograph/core/converter.py +++ b/tensorflow/python/autograph/core/converter.py @@ -82,6 +82,7 @@ from tensorflow.python.autograph.pyct.static_analysis import live_values from tensorflow.python.autograph.pyct.static_analysis import liveness from tensorflow.python.autograph.pyct.static_analysis import reaching_definitions from tensorflow.python.autograph.pyct.static_analysis import type_info +from tensorflow.python.eager import function # TODO(mdan): These contexts can be refactored into first class objects. # For example, we could define Program and Entity abstractions that hold on @@ -96,7 +97,7 @@ class Verbosity(IntEnum): Attributes: * BRIEF: No logging, minimal error messages. * VERBOSE: Detailed logging of generated code, detailed error messages. - """ + """ BRIEF = 0 VERBOSE = 1 @@ -151,7 +152,7 @@ class ConversionOptions(object): optional_features=Feature.ALL): self.recursive = recursive self.verbose = verbose - self.strip_decorators = strip_decorators or () + self._strip_decorators = strip_decorators or () self.force_conversion = force_conversion # TODO(mdan): Rename to conversion_recursion_depth? self.internal_convert_user_code = internal_convert_user_code @@ -161,6 +162,12 @@ class ConversionOptions(object): optional_features = frozenset(optional_features) self.optional_features = optional_features + @property + def strip_decorators(self): + # A few decorators are included by default. + # TODO(mdan): Revert if function.defun becomes a public symbol. + return self._strip_decorators + (function.defun,) + def uses(self, feature): return (Feature.ALL in self.optional_features or feature in self.optional_features) @@ -216,7 +223,7 @@ class ConversionOptions(object): as_qualified_name(ConversionOptions)), recursive_val=parser.parse_expression(str(self.recursive)), verbose_val=parser.parse_expression(str(int(self.verbose))), - strip_decorators_val=list_of_names(self.strip_decorators), + strip_decorators_val=list_of_names(self._strip_decorators), force_conversion_val=parser.parse_expression( str(self.force_conversion)), internal_convert_user_code_val=parser.parse_expression( diff --git a/tensorflow/python/autograph/core/converter_testing.py b/tensorflow/python/autograph/core/converter_testing.py index 7b0608d03fc..f1374081d3c 100644 --- a/tensorflow/python/autograph/core/converter_testing.py +++ b/tensorflow/python/autograph/core/converter_testing.py @@ -32,6 +32,7 @@ from tensorflow.python.autograph.core import errors from tensorflow.python.autograph.core import function_wrapping from tensorflow.python.autograph.lang import special_functions from tensorflow.python.autograph.pyct import compiler +from tensorflow.python.autograph.pyct import inspect_utils from tensorflow.python.autograph.pyct import origin_info from tensorflow.python.autograph.pyct import parser from tensorflow.python.autograph.pyct import pretty_printer @@ -43,7 +44,7 @@ def imported_decorator(f): return lambda a: f(a) + 1 -# TODO(mdan): We might be able to use the real namer here. +# TODO(mdan): We should use the real namer here. class FakeNamer(object): """A fake namer that uses a global counter to generate unique names.""" @@ -61,7 +62,8 @@ class FakeNamer(object): original_fqn, live_entity=None, owner_type=None): - del live_entity + if inspect_utils.islambda(live_entity): + return None, False if owner_type is not None: return None, False return ('renamed_%s' % '_'.join(original_fqn)), True diff --git a/tensorflow/python/autograph/core/errors_test.py b/tensorflow/python/autograph/core/errors_test.py index aa6c293268c..845a28a5222 100644 --- a/tensorflow/python/autograph/core/errors_test.py +++ b/tensorflow/python/autograph/core/errors_test.py @@ -22,6 +22,7 @@ from tensorflow.python.autograph.core import errors from tensorflow.python.autograph.pyct import origin_info from tensorflow.python.framework import dtypes from tensorflow.python.framework import errors as tf_errors +from tensorflow.python.framework import test_util from tensorflow.python.ops import array_ops from tensorflow.python.platform import test from tensorflow.python.util import tf_inspect @@ -47,6 +48,7 @@ class RuntimeErrorsTest(test.TestCase): 'test_comment') return loc, origin + @test_util.run_deprecated_v1 def test_improved_errors_basic(self): loc, origin = self.fake_origin(zero_div, 2) zero_div_caller.ag_source_map = {loc: origin} @@ -55,13 +57,14 @@ class RuntimeErrorsTest(test.TestCase): with self.assertRaises(errors.TfRuntimeError) as cm: with errors.improved_errors(zero_div_caller): with self.cached_session() as sess: - sess.run(ops) + self.evaluate(ops) for frame in cm.exception.custom_traceback: _, _, function_name, _ = frame self.assertNotEqual('zero_div', function_name) self.assertIn(origin.as_frame(), set(cm.exception.custom_traceback)) + @test_util.run_deprecated_v1 def test_improved_errors_no_matching_lineno(self): loc, origin = self.fake_origin(zero_div, -1) zero_div_caller.ag_source_map = {loc: origin} @@ -70,7 +73,7 @@ class RuntimeErrorsTest(test.TestCase): with self.assertRaises(errors.TfRuntimeError) as cm: with errors.improved_errors(zero_div_caller): with self.cached_session() as sess: - sess.run(ops) + self.evaluate(ops) all_function_names = set() for frame in cm.exception.custom_traceback: @@ -79,6 +82,7 @@ class RuntimeErrorsTest(test.TestCase): self.assertNotEqual('test_function_name', function_name) self.assertIn('zero_div', all_function_names) + @test_util.run_deprecated_v1 def test_improved_errors_failures(self): loc, _ = self.fake_origin(zero_div, 2) zero_div_caller.ag_source_map = {loc: 'bogus object'} @@ -87,7 +91,7 @@ class RuntimeErrorsTest(test.TestCase): with self.assertRaises(tf_errors.InvalidArgumentError): with errors.improved_errors(zero_div_caller): with self.cached_session() as sess: - sess.run(ops) + self.evaluate(ops) def test_improved_errors_validation(self): with self.assertRaisesRegexp( diff --git a/tensorflow/python/autograph/core/function_wrapping_test.py b/tensorflow/python/autograph/core/function_wrapping_test.py index 5e217055c71..7e21b979dbc 100644 --- a/tensorflow/python/autograph/core/function_wrapping_test.py +++ b/tensorflow/python/autograph/core/function_wrapping_test.py @@ -20,11 +20,13 @@ from __future__ import print_function from tensorflow.python.autograph.core import function_wrapping from tensorflow.python.framework import constant_op +from tensorflow.python.framework import test_util from tensorflow.python.platform import test class FunctionWrappingTest(test.TestCase): + @test_util.run_deprecated_v1 def test_function_scope_name(self): with function_wrapping.function_scope('test_name'): t = constant_op.constant(1) diff --git a/tensorflow/python/autograph/core/naming.py b/tensorflow/python/autograph/core/naming.py index 43fcbcfc030..b8d79daebaa 100644 --- a/tensorflow/python/autograph/core/naming.py +++ b/tensorflow/python/autograph/core/naming.py @@ -18,8 +18,8 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function +from tensorflow.python.autograph.pyct import inspect_utils from tensorflow.python.autograph.pyct import qual_names -from tensorflow.python.util import tf_inspect class Namer(object): @@ -77,8 +77,7 @@ class Namer(object): if not self.recursive: return None, False - if (live_entity is not None and tf_inspect.isfunction(live_entity) and - live_entity.__name__ == ''): + if (live_entity is not None and inspect_utils.islambda(live_entity)): return None, False if owner_type is not None and owner_type not in self.partial_types: diff --git a/tensorflow/python/autograph/impl/BUILD b/tensorflow/python/autograph/impl/BUILD index 2f9037c43b6..201a8887541 100644 --- a/tensorflow/python/autograph/impl/BUILD +++ b/tensorflow/python/autograph/impl/BUILD @@ -41,7 +41,6 @@ py_test( name = "api_test", srcs = ["api_test.py"], srcs_version = "PY2AND3", - tags = ["no_windows"], deps = [ ":impl", "//tensorflow/python:client_testlib", @@ -54,7 +53,6 @@ py_test( name = "conversion_test", srcs = ["conversion_test.py"], srcs_version = "PY2AND3", - tags = ["no_windows"], deps = [ ":impl", "//tensorflow/python:client_testlib", diff --git a/tensorflow/python/autograph/impl/api.py b/tensorflow/python/autograph/impl/api.py index 69674b2be3c..19a472064ae 100644 --- a/tensorflow/python/autograph/impl/api.py +++ b/tensorflow/python/autograph/impl/api.py @@ -195,6 +195,17 @@ def converted_call(f, owner, options, *args, **kwargs): if not options.internal_convert_user_code: return f(*args, **kwargs) + # Unwrap functools.partial objects + # TODO(allenl, mdan): Consider sharing unwrapping logic with tf_inspect. + while isinstance(f, functools.partial): + args = f.args + args + new_kwargs = {} + if f.keywords is not None: + new_kwargs.update(f.keywords) + new_kwargs.update(kwargs) + kwargs = new_kwargs + f = f.func + if tf_inspect.isfunction(f) or tf_inspect.ismethod(f): # Regular functions target_entity = f diff --git a/tensorflow/python/autograph/impl/api_test.py b/tensorflow/python/autograph/impl/api_test.py index ef577568c4e..66edda51193 100644 --- a/tensorflow/python/autograph/impl/api_test.py +++ b/tensorflow/python/autograph/impl/api_test.py @@ -18,6 +18,7 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function +import functools import gc import numpy as np @@ -28,6 +29,7 @@ from tensorflow.python.autograph.impl import api from tensorflow.python.autograph.pyct import parser from tensorflow.python.autograph.utils import py_func from tensorflow.python.framework import constant_op +from tensorflow.python.framework import test_util from tensorflow.python.keras.engine import sequential from tensorflow.python.keras.layers import core from tensorflow.python.ops import variables @@ -43,6 +45,7 @@ class TestResource(str): class ApiTest(test.TestCase): + @test_util.run_deprecated_v1 def test_decorator_recurses(self): class TestClass(object): @@ -63,8 +66,9 @@ class ApiTest(test.TestCase): x = tc.test_method( constant_op.constant([2, 4]), constant_op.constant(1), constant_op.constant(-2)) - self.assertListEqual([0, 1], sess.run(x).tolist()) + self.assertListEqual([0, 1], self.evaluate(x).tolist()) + @test_util.run_deprecated_v1 def test_decorator_does_not_recurse(self): class TestClass(object): @@ -83,8 +87,9 @@ class ApiTest(test.TestCase): x = tc.test_method( constant_op.constant([2, 4]), constant_op.constant(1), constant_op.constant(-2)) - self.assertListEqual([0, 1], sess.run(x).tolist()) + self.assertListEqual([0, 1], self.evaluate(x).tolist()) + @test_util.run_deprecated_v1 def test_decorator_calls_unconverted_graph(self): class TestClass(object): @@ -104,8 +109,9 @@ class ApiTest(test.TestCase): x = tc.test_method( constant_op.constant([2, 4]), constant_op.constant(1), constant_op.constant(-2)) - self.assertListEqual([0, 1], sess.run(x).tolist()) + self.assertListEqual([0, 1], self.evaluate(x).tolist()) + @test_util.run_deprecated_v1 def test_decorator_calls_unconverted_py_func(self): class TestClass(object): @@ -130,8 +136,9 @@ class ApiTest(test.TestCase): x = tc.test_method( constant_op.constant([2, 4]), constant_op.constant(1), constant_op.constant(-2)) - self.assertListEqual([0, 1], sess.run(x).tolist()) + self.assertListEqual([0, 1], self.evaluate(x).tolist()) + @test_util.run_deprecated_v1 def test_decorator_calls_decorated(self): class TestClass(object): @@ -153,7 +160,7 @@ class ApiTest(test.TestCase): x = tc.test_method( constant_op.constant([2, 4]), constant_op.constant(1), constant_op.constant(-2)) - self.assertListEqual([0, 1], sess.run(x).tolist()) + self.assertListEqual([0, 1], self.evaluate(x).tolist()) def test_decorator_preserves_argspec(self): @@ -171,6 +178,7 @@ class ApiTest(test.TestCase): list(tf_inspect.getfullargspec(tc.called_member)), list(tf_inspect.getfullargspec(tc.called_member_converted))) + @test_util.run_deprecated_v1 def test_convert_call_site_decorator(self): class TestClass(object): @@ -192,7 +200,7 @@ class ApiTest(test.TestCase): x = tc.test_method( constant_op.constant([2, 4]), constant_op.constant(1), constant_op.constant(-2)) - self.assertListEqual([0, 1], sess.run(x).tolist()) + self.assertListEqual([0, 1], self.evaluate(x).tolist()) def test_converted_call_builtin(self): x = api.converted_call(range, None, converter.ConversionOptions(), 3) @@ -208,7 +216,27 @@ class ApiTest(test.TestCase): with self.cached_session() as sess: x = api.converted_call(test_fn, None, converter.ConversionOptions(), constant_op.constant(-1)) - self.assertEqual(1, sess.run(x)) + self.assertEqual(1, self.evaluate(x)) + + def test_converted_call_functools_partial(self): + + def test_fn(x, y, z): + if x < 0: + return -x, -y, -z + return x, y, z + + x = api.converted_call( + functools.partial(test_fn, constant_op.constant(-1), z=-3), + None, converter.ConversionOptions(), + constant_op.constant(-2)) + self.assertEqual((1, 2, 3), self.evaluate(x)) + + x = api.converted_call( + functools.partial( + functools.partial(test_fn, constant_op.constant(-1)), z=-3), + None, converter.ConversionOptions(), + constant_op.constant(-2)) + self.assertEqual((1, 2, 3), self.evaluate(x)) def test_converted_call_method_explicit_owner(self): # TODO(mdan): Implement. @@ -234,7 +262,7 @@ class ApiTest(test.TestCase): tc = TestClass(constant_op.constant(-1)) x = api.converted_call(tc.test_method, None, converter.ConversionOptions(), tc) - self.assertEqual(1, sess.run(x)) + self.assertEqual(1, self.evaluate(x)) def test_converted_call_method_by_class(self): @@ -252,7 +280,7 @@ class ApiTest(test.TestCase): tc = TestClass(constant_op.constant(-1)) x = api.converted_call(TestClass.test_method, None, converter.ConversionOptions(), tc) - self.assertEqual(1, sess.run(x)) + self.assertEqual(1, self.evaluate(x)) def test_converted_call_callable_object(self): @@ -269,7 +297,7 @@ class ApiTest(test.TestCase): with self.cached_session() as sess: tc = TestClass(constant_op.constant(-1)) x = api.converted_call(tc, None, converter.ConversionOptions()) - self.assertEqual(1, sess.run(x)) + self.assertEqual(1, self.evaluate(x)) def test_converted_call_constructor(self): @@ -288,7 +316,7 @@ class ApiTest(test.TestCase): constant_op.constant(-1)) # tc is now a converted object. x = tc.test_method() - self.assertEqual(1, sess.run(x)) + self.assertEqual(1, self.evaluate(x)) def test_converted_call_already_converted(self): @@ -298,13 +326,14 @@ class ApiTest(test.TestCase): with self.cached_session() as sess: x = api.converted_call(f, None, converter.ConversionOptions(), constant_op.constant(0)) - self.assertTrue(sess.run(x)) + self.assertTrue(self.evaluate(x)) converted_f = api.to_graph(f) x = api.converted_call(converted_f, None, converter.ConversionOptions(), constant_op.constant(0)) - self.assertTrue(sess.run(x)) + self.assertTrue(self.evaluate(x)) + @test_util.run_deprecated_v1 def test_converted_call_no_user_code(self): def f(x): @@ -334,8 +363,8 @@ class ApiTest(test.TestCase): constant_op.constant([[0.0]]), training=True) with self.cached_session() as sess: - sess.run(variables.global_variables_initializer()) - self.assertAllEqual([[0.0, 0.0]], sess.run(x)) + self.evaluate(variables.global_variables_initializer()) + self.assertAllEqual([[0.0, 0.0]], self.evaluate(x)) def test_converted_call_whitelisted_method_extra_self(self): @@ -349,8 +378,8 @@ class ApiTest(test.TestCase): model, constant_op.constant([[0.0]]), training=True) with self.cached_session() as sess: - sess.run(variables.global_variables_initializer()) - self.assertAllEqual([[0.0, 0.0]], sess.run(x)) + self.evaluate(variables.global_variables_initializer()) + self.assertAllEqual([[0.0, 0.0]], self.evaluate(x)) def test_converted_call_whitelisted_method_via_owner(self): @@ -364,8 +393,8 @@ class ApiTest(test.TestCase): constant_op.constant([[0.0]]), training=True) with self.cached_session() as sess: - sess.run(variables.global_variables_initializer()) - self.assertAllEqual([[0.0, 0.0]], sess.run(x)) + self.evaluate(variables.global_variables_initializer()) + self.assertAllEqual([[0.0, 0.0]], self.evaluate(x)) def test_converted_call_lambda(self): @@ -376,9 +405,10 @@ class ApiTest(test.TestCase): x = api.converted_call(l, None, opts, constant_op.constant(0)) with self.cached_session() as sess: - sess.run(variables.global_variables_initializer()) - self.assertAllEqual(True, sess.run(x)) + self.evaluate(variables.global_variables_initializer()) + self.assertAllEqual(True, self.evaluate(x)) + @test_util.run_deprecated_v1 def test_to_graph_basic(self): def test_fn(x, s): @@ -390,8 +420,9 @@ class ApiTest(test.TestCase): with self.cached_session() as sess: x = compiled_fn(constant_op.constant([4, 8]), 4) - self.assertListEqual([1, 2], sess.run(x).tolist()) + self.assertListEqual([1, 2], self.evaluate(x).tolist()) + @test_util.run_deprecated_v1 def test_to_graph_with_defaults(self): foo = 4 @@ -405,7 +436,7 @@ class ApiTest(test.TestCase): with self.cached_session() as sess: x = compiled_fn(constant_op.constant([4, 8])) - self.assertListEqual([1, 2], sess.run(x).tolist()) + self.assertListEqual([1, 2], self.evaluate(x).tolist()) def test_to_code_basic(self): diff --git a/tensorflow/python/autograph/impl/conversion.py b/tensorflow/python/autograph/impl/conversion.py index 328a4b5fe48..f8decd24e8e 100644 --- a/tensorflow/python/autograph/impl/conversion.py +++ b/tensorflow/python/autograph/impl/conversion.py @@ -18,6 +18,7 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function +import functools import imp import gast @@ -72,12 +73,31 @@ def is_whitelisted_for_graph(o): Returns: Boolean """ - m = tf_inspect.getmodule(o) + # TODO(b/120224672): Fix this. + if isinstance(o, functools.partial): + # tf_inspect.getmodule(functools.partial(...)) otherwise returns None since + # functools.partial objects do not have a __module__ attribute. + m = functools + else: + m = tf_inspect.getmodule(o) for prefix, in config.DEFAULT_UNCOMPILED_MODULES: if m.__name__.startswith(prefix): return True + if hasattr(o, 'autograph_info__'): return True + + if inspect_utils.isnamedtuple(o): + # Due to the way they're constructed, namedtuple types cannot be converted + # because they don't expose source code. But we assume they are safe for + # graph mode since they are just containers. + if tf_inspect.isclass(o) and len(o.__bases__) > 1: + logging.log_first_n( + logging.level_warning(), + 'Entity {} looks like a namedtuple subclass. If it has any custom' + ' methods, they will not be converted by AutoGraph.'.format(o), 1) + return True + return False @@ -281,11 +301,10 @@ def function_to_graph(f, node, source = parser.parse_entity(f) node = node.body[0] - # In general, the output of inspect.getsource is inexact because it uses crude - # regex matching methods to search the source file. This is particularly - # problematic for lambda functions, where the entire containing lines are - # returned. Certain distributions of CPython may also return the enclosing - # function for local functions. + # In general, the output of inspect.getsource is inexact because it uses + # regex matching to adjust the exact location around the line number that + # CPython records. This is particularly problematic for lambda functions, + # where the entire containing lines are returned. nodes = ast_util.find_matching_definitions(node, f) if len(nodes) != 1: if f.__name__ == '': @@ -298,8 +317,8 @@ def function_to_graph(f, raise ValueError( 'Unable to identify source code of function {}. The source code' ' reported by Python did not include exactly one matching signature:' - '\n{}\nTo avoid ambiguity, use a unique name for each' - ' function.'.format(f, source)) + '\n{}\n. This is an extremely rare occurrence. Please report it to' + ' the TensorFlow team.'.format(f, source)) node, = nodes # TODO(znado): Place inside standard_analysis. diff --git a/tensorflow/python/autograph/lang/special_functions_test.py b/tensorflow/python/autograph/lang/special_functions_test.py index 123ee65b326..8d40f4036c5 100644 --- a/tensorflow/python/autograph/lang/special_functions_test.py +++ b/tensorflow/python/autograph/lang/special_functions_test.py @@ -36,7 +36,7 @@ class SpecialFunctionsTest(test.TestCase): python_one = special_functions.match_staging_level(1, 1) with self.cached_session() as sess: self.assertTrue(tensor_util.is_tensor(tensor_one)) - self.assertAllEqual(sess.run(tensor_one), 1) + self.assertAllEqual(self.evaluate(tensor_one), 1) self.assertEqual(python_one, 1) def test_tensor_list_empty_list(self): @@ -45,21 +45,21 @@ class SpecialFunctionsTest(test.TestCase): element_shape=()) sl = list_ops.tensor_list_stack(l, element_dtype=dtypes.int32) with self.cached_session() as sess: - self.assertAllEqual(sess.run(sl), []) + self.assertAllEqual(self.evaluate(sl), []) l = special_functions.tensor_list((), element_dtype=dtypes.int32, element_shape=()) sl = list_ops.tensor_list_stack(l, element_dtype=dtypes.int32) with self.cached_session() as sess: - self.assertAllEqual(sess.run(sl), []) + self.assertAllEqual(self.evaluate(sl), []) def test_tensor_list_tensor(self): l = special_functions.tensor_list( constant_op.constant([], dtype=dtypes.int32)) sl = list_ops.tensor_list_stack(l, element_dtype=dtypes.int32) with self.cached_session() as sess: - self.assertAllEqual(sess.run(sl), []) + self.assertAllEqual(self.evaluate(sl), []) def test_tensor_list_unsupported_initializer(self): with self.assertRaisesRegexp(ValueError, 'unknown type'): @@ -76,7 +76,7 @@ class SpecialFunctionsTest(test.TestCase): l = special_functions.tensor_list(elements) sl = list_ops.tensor_list_stack(l, element_dtype=dtypes.int32) with self.cached_session() as sess: - self.assertAllEqual(sess.run(sl), [[1, 2], [3, 4]]) + self.assertAllEqual(self.evaluate(sl), [[1, 2], [3, 4]]) def test_tensor_list_array_from_elements(self): elements = [constant_op.constant([1, 2]), constant_op.constant([3, 4])] @@ -84,7 +84,7 @@ class SpecialFunctionsTest(test.TestCase): l = special_functions.tensor_list(elements, use_tensor_array=True) sl = l.stack() with self.cached_session() as sess: - self.assertAllEqual(sess.run(sl), [[1, 2], [3, 4]]) + self.assertAllEqual(self.evaluate(sl), [[1, 2], [3, 4]]) def test_stack(self): self.assertEqual(special_functions.stack(1, strict=False), 1) diff --git a/tensorflow/python/autograph/operators/control_flow.py b/tensorflow/python/autograph/operators/control_flow.py index 6eedd695a74..670897744ae 100644 --- a/tensorflow/python/autograph/operators/control_flow.py +++ b/tensorflow/python/autograph/operators/control_flow.py @@ -61,7 +61,7 @@ def for_stmt(iter_, extra_test, body, init_state): """ if tensor_util.is_tensor(iter_): return _known_len_for_stmt(iter_, extra_test, body, init_state) - elif isinstance(iter_, dataset_ops.Dataset): + elif isinstance(iter_, dataset_ops.DatasetV2): return _dataset_for_stmt(iter_, extra_test, body, init_state) else: return _py_for_stmt(iter_, extra_test, body, init_state) @@ -123,7 +123,7 @@ def _dataset_for_stmt(ds, extra_test, body, init_state): (dataset_ops.Dataset.from_tensors(tag).repeat(), ds)) ds_with_epoch = epoch_numbers.flat_map(lambda i: tag_with(ds, i)) - iterator = ds_with_epoch.make_initializable_iterator() + iterator = dataset_ops.make_initializable_iterator(ds_with_epoch) with ops.control_dependencies((iterator.initializer,)): epoch_number, iterate = iterator.get_next() diff --git a/tensorflow/python/autograph/operators/control_flow_test.py b/tensorflow/python/autograph/operators/control_flow_test.py index 2dea18dc5fa..0a7d4b64022 100644 --- a/tensorflow/python/autograph/operators/control_flow_test.py +++ b/tensorflow/python/autograph/operators/control_flow_test.py @@ -22,12 +22,14 @@ from tensorflow.python.autograph.operators import control_flow from tensorflow.python.data.ops import dataset_ops from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes +from tensorflow.python.framework import test_util from tensorflow.python.ops import math_ops from tensorflow.python.platform import test class ForLoopTest(test.TestCase): + @test_util.run_deprecated_v1 def test_tensor(self): s = control_flow.for_stmt( constant_op.constant([1, 2, 3, 4]), @@ -35,7 +37,7 @@ class ForLoopTest(test.TestCase): body=lambda i, s: (s + i,), init_state=(0,)) with self.cached_session() as sess: - self.assertEqual((10,), sess.run(s)) + self.assertEqual((10,), self.evaluate(s)) def test_python(self): s = control_flow.for_stmt( @@ -45,6 +47,7 @@ class ForLoopTest(test.TestCase): init_state=(0,)) self.assertEqual(10, s) + @test_util.run_deprecated_v1 def test_dataset(self): to_int32 = lambda i: math_ops.cast(i, dtypes.int32) s = control_flow.for_stmt( @@ -53,11 +56,12 @@ class ForLoopTest(test.TestCase): body=lambda i, s: (s + i,), init_state=(0,)) with self.cached_session() as sess: - self.assertEqual((10,), sess.run(s)) + self.assertEqual((10,), self.evaluate(s)) class WhileLoopTest(test.TestCase): + @test_util.run_deprecated_v1 def test_tensor(self): n = constant_op.constant(5) results = control_flow.while_stmt( @@ -66,7 +70,7 @@ class WhileLoopTest(test.TestCase): init_state=(0, 0), extra_deps=(n,)) with self.cached_session() as sess: - self.assertEqual((5, 10), sess.run(results)) + self.assertEqual((5, 10), self.evaluate(results)) def test_python(self): n = 5 @@ -87,23 +91,25 @@ class IfStmtTest(test.TestCase): return control_flow.if_stmt( cond=cond, body=lambda: (1, 2), orelse=lambda: (-1, -2)) + @test_util.run_deprecated_v1 def test_tensor(self): with self.cached_session() as sess: t = self.single_return_if_stmt(constant_op.constant(True)) - self.assertEqual(1, sess.run(t)) + self.assertEqual(1, self.evaluate(t)) t = self.single_return_if_stmt(constant_op.constant(False)) - self.assertEqual(-1, sess.run(t)) + self.assertEqual(-1, self.evaluate(t)) def test_python(self): self.assertEqual(1, self.single_return_if_stmt(True)) self.assertEqual(-1, self.single_return_if_stmt(False)) + @test_util.run_deprecated_v1 def test_tensor_multiple_returns(self): with self.cached_session() as sess: t = self.multi_return_if_stmt(constant_op.constant(True)) - self.assertAllEqual([1, 2], sess.run(t)) + self.assertAllEqual([1, 2], self.evaluate(t)) t = self.multi_return_if_stmt(constant_op.constant(False)) - self.assertAllEqual([-1, -2], sess.run(t)) + self.assertAllEqual([-1, -2], self.evaluate(t)) def test_python_multiple_returns(self): self.assertEqual((1, 2), self.multi_return_if_stmt(True)) diff --git a/tensorflow/python/autograph/operators/data_structures_test.py b/tensorflow/python/autograph/operators/data_structures_test.py index 6039b07982c..c5a3a3d1cac 100644 --- a/tensorflow/python/autograph/operators/data_structures_test.py +++ b/tensorflow/python/autograph/operators/data_structures_test.py @@ -22,6 +22,7 @@ from tensorflow.python.autograph.operators import data_structures from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops +from tensorflow.python.framework import test_util from tensorflow.python.ops import list_ops from tensorflow.python.ops import tensor_array_ops from tensorflow.python.platform import test @@ -43,7 +44,7 @@ class ListTest(test.TestCase): l = data_structures.tf_tensor_list_new([3, 4, 5]) t = list_ops.tensor_list_stack(l, element_dtype=dtypes.int32) with self.cached_session() as sess: - self.assertAllEqual(sess.run(t), [3, 4, 5]) + self.assertAllEqual(self.evaluate(t), [3, 4, 5]) def test_tf_tensor_list_new_empty(self): l = data_structures.tf_tensor_list_new([], @@ -51,14 +52,15 @@ class ListTest(test.TestCase): element_shape=()) t = list_ops.tensor_list_stack(l, element_dtype=dtypes.int32) with self.cached_session() as sess: - self.assertAllEqual(sess.run(t), []) + self.assertAllEqual(self.evaluate(t), []) def test_tf_tensor_list_new_from_tensor(self): l = data_structures.tf_tensor_list_new(constant_op.constant([3, 4, 5])) t = list_ops.tensor_list_stack(l, element_dtype=dtypes.int32) with self.cached_session() as sess: - self.assertAllEqual(sess.run(t), [3, 4, 5]) + self.assertAllEqual(self.evaluate(t), [3, 4, 5]) + @test_util.run_deprecated_v1 def test_tf_tensor_list_new_illegal_input(self): with self.assertRaises(ValueError): data_structures.tf_tensor_list_new([3, 4.0]) @@ -77,7 +79,7 @@ class ListTest(test.TestCase): l = data_structures.tf_tensor_array_new([3, 4, 5]) t = l.stack() with self.cached_session() as sess: - self.assertAllEqual(sess.run(t), [3, 4, 5]) + self.assertAllEqual(self.evaluate(t), [3, 4, 5]) def test_tf_tensor_array_new_illegal_input(self): with self.assertRaises(ValueError): @@ -102,15 +104,16 @@ class ListTest(test.TestCase): t = list_ops.tensor_list_stack(l, element_dtype=x.dtype) with self.cached_session() as sess: - self.assertAllEqual(sess.run(t), [[1, 2, 3]]) + self.assertAllEqual(self.evaluate(t), [[1, 2, 3]]) + @test_util.run_v1_only("b/117943489") def test_append_tensorarray(self): l = tensor_array_ops.TensorArray(dtypes.int32, size=0, dynamic_size=True) l1 = data_structures.list_append(l, 1) l2 = data_structures.list_append(l1, 2) with self.cached_session() as sess: - self.assertAllEqual(sess.run(l1.stack()), [1]) - self.assertAllEqual(sess.run(l2.stack()), [1, 2]) + self.assertAllEqual(self.evaluate(l1.stack()), [1]) + self.assertAllEqual(self.evaluate(l2.stack()), [1, 2]) def test_append_python(self): l = [] @@ -131,10 +134,10 @@ class ListTest(test.TestCase): with self.cached_session() as sess: l, x = data_structures.list_pop(l, None, opts) - self.assertAllEqual(sess.run(x), [3, 4]) + self.assertAllEqual(self.evaluate(x), [3, 4]) t = list_ops.tensor_list_stack(l, element_dtype=initial_list.dtype) - self.assertAllEqual(sess.run(t), [[1, 2]]) + self.assertAllEqual(self.evaluate(t), [[1, 2]]) def test_pop_python(self): l = [1, 2, 3] @@ -152,12 +155,12 @@ class ListTest(test.TestCase): with self.cached_session() as sess: t = data_structures.list_stack(l, opts) - self.assertAllEqual(sess.run(t), sess.run(initial_list)) + self.assertAllEqual(self.evaluate(t), self.evaluate(initial_list)) + @test_util.run_deprecated_v1 def test_stack_tensor_list_empty(self): l = list_ops.empty_tensor_list( - element_shape=-1, - element_dtype=dtypes.variant) + element_shape=None, element_dtype=dtypes.variant) opts = data_structures.ListStackOpts( element_dtype=dtypes.int32, original_call=None) diff --git a/tensorflow/python/autograph/operators/exceptions_test.py b/tensorflow/python/autograph/operators/exceptions_test.py index 186535d05b5..21ba76bb952 100644 --- a/tensorflow/python/autograph/operators/exceptions_test.py +++ b/tensorflow/python/autograph/operators/exceptions_test.py @@ -21,6 +21,7 @@ from __future__ import print_function from tensorflow.python.autograph.operators import exceptions from tensorflow.python.framework import constant_op from tensorflow.python.framework import errors_impl +from tensorflow.python.framework import test_util from tensorflow.python.platform import test @@ -30,8 +31,9 @@ class ExceptionsTest(test.TestCase): with self.cached_session() as sess: t = exceptions.assert_stmt( constant_op.constant(True), lambda: constant_op.constant('ignored')) - sess.run(t) + self.evaluate(t) + @test_util.run_deprecated_v1 def test_assert_tf_triggered(self): with self.cached_session() as sess: t = exceptions.assert_stmt( @@ -40,8 +42,9 @@ class ExceptionsTest(test.TestCase): with self.assertRaisesRegexp(errors_impl.InvalidArgumentError, 'test message'): - sess.run(t) + self.evaluate(t) + @test_util.run_deprecated_v1 def test_assert_tf_multiple_printed_values(self): two_tensors = [ constant_op.constant('test message'), @@ -53,7 +56,7 @@ class ExceptionsTest(test.TestCase): with self.assertRaisesRegexp(errors_impl.InvalidArgumentError, 'test message.*another message'): - sess.run(t) + self.evaluate(t) def test_assert_python_untriggered(self): side_effect_trace = [] diff --git a/tensorflow/python/autograph/operators/logical_test.py b/tensorflow/python/autograph/operators/logical_test.py index d6649f7b2bf..e22f39932d1 100644 --- a/tensorflow/python/autograph/operators/logical_test.py +++ b/tensorflow/python/autograph/operators/logical_test.py @@ -20,6 +20,7 @@ from __future__ import print_function from tensorflow.python.autograph.operators import logical from tensorflow.python.framework import constant_op +from tensorflow.python.framework import test_util from tensorflow.python.platform import test @@ -42,14 +43,15 @@ class LogicalOperatorsTest(test.TestCase): self.assertFalse(logical.and_(lambda: False, lambda: True)) self.assertFalse(logical.and_(lambda: False, self.assertNotCalled)) + @test_util.run_deprecated_v1 def test_and_tf(self): with self.cached_session() as sess: t = logical.and_(self._tf_true, self._tf_true) - self.assertEqual(sess.run(t), True) + self.assertEqual(self.evaluate(t), True) t = logical.and_(self._tf_true, lambda: True) - self.assertEqual(sess.run(t), True) + self.assertEqual(self.evaluate(t), True) t = logical.and_(self._tf_false, lambda: True) - self.assertEqual(sess.run(t), False) + self.assertEqual(self.evaluate(t), False) # TODO(mdan): Add a test for ops with side effects. def test_or_python(self): @@ -60,14 +62,15 @@ class LogicalOperatorsTest(test.TestCase): self.assertTrue(logical.or_(lambda: False, lambda: True)) self.assertTrue(logical.or_(lambda: True, self.assertNotCalled)) + @test_util.run_deprecated_v1 def test_or_tf(self): with self.cached_session() as sess: t = logical.or_(self._tf_false, self._tf_true) - self.assertEqual(sess.run(t), True) + self.assertEqual(self.evaluate(t), True) t = logical.or_(self._tf_false, lambda: True) - self.assertEqual(sess.run(t), True) + self.assertEqual(self.evaluate(t), True) t = logical.or_(self._tf_true, lambda: True) - self.assertEqual(sess.run(t), True) + self.assertEqual(self.evaluate(t), True) # TODO(mdan): Add a test for ops with side effects. def test_not_python(self): @@ -78,7 +81,7 @@ class LogicalOperatorsTest(test.TestCase): def test_not_tf(self): with self.cached_session() as sess: t = logical.not_(self._tf_false()) - self.assertEqual(sess.run(t), True) + self.assertEqual(self.evaluate(t), True) if __name__ == '__main__': diff --git a/tensorflow/python/autograph/operators/py_builtins_test.py b/tensorflow/python/autograph/operators/py_builtins_test.py index 443e30a475d..c856e39d141 100644 --- a/tensorflow/python/autograph/operators/py_builtins_test.py +++ b/tensorflow/python/autograph/operators/py_builtins_test.py @@ -27,6 +27,7 @@ from tensorflow.python.autograph.operators import py_builtins from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes from tensorflow.python.framework import errors_impl +from tensorflow.python.framework import test_util from tensorflow.python.ops import array_ops from tensorflow.python.ops import tensor_array_ops from tensorflow.python.platform import test @@ -38,29 +39,29 @@ class PyBuiltinsTest(test.TestCase): self.assertEqual(py_builtins.abs_(-1), 1) with self.cached_session() as sess: t = py_builtins.abs_(constant_op.constant(-1)) - self.assertEqual(sess.run(t), 1) + self.assertEqual(self.evaluate(t), 1) t = py_builtins.abs_(constant_op.constant([-1, 2, -3])) - self.assertAllEqual(sess.run(t), [1, 2, 3]) + self.assertAllEqual(self.evaluate(t), [1, 2, 3]) def test_float(self): self.assertEqual(py_builtins.float_(10), 10.0) self.assertEqual(py_builtins.float_('10.0'), 10.0) with self.cached_session() as sess: t = py_builtins.float_(constant_op.constant(1, dtype=dtypes.int64)) - self.assertEqual(sess.run(t), 1.0) + self.assertEqual(self.evaluate(t), 1.0) st = py_builtins.float_(constant_op.constant('1.0')) - self.assertEqual(sess.run(st), 1.0) + self.assertEqual(self.evaluate(st), 1.0) def test_int(self): self.assertEqual(py_builtins.int_(10.0), 10) self.assertEqual(py_builtins.int_('11', 2), 3) with self.cached_session() as sess: t = py_builtins.int_(constant_op.constant(1, dtype=dtypes.float64)) - self.assertEqual(sess.run(t), 1) + self.assertEqual(self.evaluate(t), 1) st = py_builtins.int_(constant_op.constant('1')) - self.assertEqual(sess.run(st), 1) + self.assertEqual(self.evaluate(st), 1) st = py_builtins.int_(constant_op.constant('1'), 10) - self.assertEqual(sess.run(st), 1) + self.assertEqual(self.evaluate(st), 1) def test_int_unsupported_base(self): t = constant_op.constant(1, dtype=dtypes.float64) @@ -73,14 +74,15 @@ class PyBuiltinsTest(test.TestCase): t = py_builtins.len_(constant_op.constant([[1], [2], [3]])) self.assertEqual(t, 3) ta = py_builtins.len_(tensor_array_ops.TensorArray(dtypes.int32, size=5)) - self.assertEqual(sess.run(ta), 5) + self.assertEqual(self.evaluate(ta), 5) tl = py_builtins.len_(data_structures.tf_tensor_list_new([3, 4, 5])) - self.assertEqual(sess.run(tl), 3) + self.assertEqual(self.evaluate(tl), 3) def test_len_scalar(self): with self.assertRaises(ValueError): py_builtins.len_(constant_op.constant(1)) + @test_util.run_deprecated_v1 def test_len_dynamic_shape(self): with self.cached_session() as sess: p = array_ops.placeholder(dtype=dtypes.int32, shape=None) @@ -91,6 +93,7 @@ class PyBuiltinsTest(test.TestCase): t = py_builtins.len_(p) sess.run(t, {p: 1}) + @test_util.run_deprecated_v1 def test_print_tensors(self): try: out_capturer = six.StringIO() @@ -101,6 +104,7 @@ class PyBuiltinsTest(test.TestCase): finally: sys.stdout = sys.__stdout__ + @test_util.run_deprecated_v1 def test_print_complex(self): try: out_capturer = six.StringIO() @@ -120,18 +124,18 @@ class PyBuiltinsTest(test.TestCase): def test_range_tensor(self): with self.cached_session() as sess: r = py_builtins.range_(constant_op.constant(3)) - self.assertAllEqual(sess.run(r), [0, 1, 2]) + self.assertAllEqual(self.evaluate(r), [0, 1, 2]) r = py_builtins.range_(1, constant_op.constant(3)) - self.assertAllEqual(sess.run(r), [1, 2]) + self.assertAllEqual(self.evaluate(r), [1, 2]) r = py_builtins.range_(2, 0, constant_op.constant(-1)) - self.assertAllEqual(sess.run(r), [2, 1]) + self.assertAllEqual(self.evaluate(r), [2, 1]) def test_range_tensor_empty_range(self): with self.session() as sess: r = py_builtins.range_(constant_op.constant(-3)) - self.assertAllEqual(sess.run(r), []) + self.assertAllEqual(self.evaluate(r), []) r = py_builtins.range_(5, constant_op.constant(2)) - self.assertAllEqual(sess.run(r), []) + self.assertAllEqual(self.evaluate(r), []) if __name__ == '__main__': diff --git a/tensorflow/python/autograph/operators/slices_test.py b/tensorflow/python/autograph/operators/slices_test.py index 9e4865b3c66..d444054fd77 100644 --- a/tensorflow/python/autograph/operators/slices_test.py +++ b/tensorflow/python/autograph/operators/slices_test.py @@ -34,7 +34,7 @@ class SlicesTest(test.TestCase): with self.cached_session() as sess: t = list_ops.tensor_list_stack(l, element_dtype=initial_list.dtype) - self.assertAllEqual(sess.run(t), [[5, 6], [3, 4]]) + self.assertAllEqual(self.evaluate(t), [[5, 6], [3, 4]]) def test_get_item_tensor_list(self): initial_list = constant_op.constant([[1, 2], [3, 4]]) @@ -44,7 +44,7 @@ class SlicesTest(test.TestCase): l, 1, slices.GetItemOpts(element_dtype=initial_list.dtype)) with self.cached_session() as sess: - self.assertAllEqual(sess.run(t), [3, 4]) + self.assertAllEqual(self.evaluate(t), [3, 4]) def test_get_item_tensor_string(self): initial_str = constant_op.constant('abcd') @@ -52,14 +52,14 @@ class SlicesTest(test.TestCase): slices.GetItemOpts(element_dtype=initial_str.dtype)) with self.cached_session() as sess: - self.assertEqual(sess.run(t), b'b') + self.assertEqual(self.evaluate(t), b'b') initial_list_str = constant_op.constant(['abcd', 'bcde']) t = slices.get_item(initial_list_str, 1, slices.GetItemOpts(element_dtype=initial_str.dtype)) with self.cached_session() as sess: - self.assertEqual(sess.run(t), b'bcde') + self.assertEqual(self.evaluate(t), b'bcde') if __name__ == '__main__': diff --git a/tensorflow/python/autograph/pyct/BUILD b/tensorflow/python/autograph/pyct/BUILD index ddadc6b96e8..ba8ec271394 100644 --- a/tensorflow/python/autograph/pyct/BUILD +++ b/tensorflow/python/autograph/pyct/BUILD @@ -80,7 +80,6 @@ py_test( name = "compiler_test", srcs = ["compiler_test.py"], srcs_version = "PY2AND3", - tags = ["no_windows"], deps = [ ":pyct", "//tensorflow/python:client_testlib", @@ -154,7 +153,6 @@ py_test( name = "transformer_test", srcs = ["transformer_test.py"], srcs_version = "PY2AND3", - tags = ["no_windows"], deps = [ ":pyct", "//tensorflow/python:client_testlib", diff --git a/tensorflow/python/autograph/pyct/inspect_utils.py b/tensorflow/python/autograph/pyct/inspect_utils.py index 4d56b93671e..7c819f364fa 100644 --- a/tensorflow/python/autograph/pyct/inspect_utils.py +++ b/tensorflow/python/autograph/pyct/inspect_utils.py @@ -46,6 +46,28 @@ if six.PY2: SPECIAL_BUILTINS['xrange'] = xrange +def islambda(f): + if not tf_inspect.isfunction(f): + return False + if not hasattr(f, '__name__'): + return False + return f.__name__ == '' + + +def isnamedtuple(f): + """Returns True if the argument is a namedtuple-like.""" + if not (tf_inspect.isclass(f) and issubclass(f, tuple)): + return False + if not hasattr(f, '_fields'): + return False + fields = getattr(f, '_fields') + if not isinstance(fields, tuple): + return False + if not all(isinstance(f, str) for f in fields): + return False + return True + + def isbuiltin(f): """Returns True if the argument is a built-in function.""" if f in SPECIAL_BUILTINS.values(): diff --git a/tensorflow/python/autograph/pyct/inspect_utils_test.py b/tensorflow/python/autograph/pyct/inspect_utils_test.py index 622e3bafc0a..a2c39056d1b 100644 --- a/tensorflow/python/autograph/pyct/inspect_utils_test.py +++ b/tensorflow/python/autograph/pyct/inspect_utils_test.py @@ -18,7 +18,8 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function -from functools import wraps +import collections +import functools import imp import types import weakref @@ -46,7 +47,7 @@ def wrapping_decorator(): def replacement(*_): return None - @wraps(f) + @functools.wraps(f) def wrapper(*args, **kwargs): return replacement(*args, **kwargs) return wrapper @@ -95,6 +96,38 @@ def free_factory(): class InspectUtilsTest(test.TestCase): + def test_islambda(self): + def test_fn(): + pass + + self.assertTrue(inspect_utils.islambda(lambda x: x)) + self.assertFalse(inspect_utils.islambda(test_fn)) + + def test_isnamedtuple(self): + nt = collections.namedtuple('TestNamedTuple', ['a', 'b']) + + class NotANamedTuple(tuple): + pass + + self.assertTrue(inspect_utils.isnamedtuple(nt)) + self.assertFalse(inspect_utils.isnamedtuple(NotANamedTuple)) + + def test_isnamedtuple_confounder(self): + """This test highlights false positives when detecting named tuples.""" + + class NamedTupleLike(tuple): + _fields = ('a', 'b') + + self.assertTrue(inspect_utils.isnamedtuple(NamedTupleLike)) + + def test_isnamedtuple_subclass(self): + """This test highlights false positives when detecting named tuples.""" + + class NamedTupleSubclass(collections.namedtuple('Test', ['a', 'b'])): + pass + + self.assertTrue(inspect_utils.isnamedtuple(NamedTupleSubclass)) + def test_getnamespace_globals(self): ns = inspect_utils.getnamespace(factory) self.assertEqual(ns['free_function'], free_function) diff --git a/tensorflow/python/autograph/pyct/static_analysis/BUILD b/tensorflow/python/autograph/pyct/static_analysis/BUILD index 4a4ccdcbd15..5e260c5730a 100644 --- a/tensorflow/python/autograph/pyct/static_analysis/BUILD +++ b/tensorflow/python/autograph/pyct/static_analysis/BUILD @@ -38,7 +38,6 @@ py_test( name = "activity_test", srcs = ["activity_test.py"], srcs_version = "PY2AND3", - tags = ["no_windows"], deps = [ ":static_analysis", "//tensorflow/python:client_testlib", @@ -51,7 +50,6 @@ py_test( name = "live_values_test", srcs = ["live_values_test.py"], srcs_version = "PY2AND3", - tags = ["no_windows"], deps = [ ":static_analysis", "//tensorflow/python:client_testlib", diff --git a/tensorflow/python/autograph/pyct/static_analysis/liveness.py b/tensorflow/python/autograph/pyct/static_analysis/liveness.py index 451398f1b70..f8b8d7fa77c 100644 --- a/tensorflow/python/autograph/pyct/static_analysis/liveness.py +++ b/tensorflow/python/autograph/pyct/static_analysis/liveness.py @@ -161,6 +161,16 @@ class Annotator(transformer.Base): self.cross_function_analyzer = cross_function_analyzer self.current_analyzer = None + def visit(self, node): + node = super(Annotator, self).visit(node) + if (self.current_analyzer is not None and + isinstance(node, gast.stmt) and + node in self.current_analyzer.graph.index): + cfg_node = self.current_analyzer.graph.index[node] + anno.setanno(node, anno.Static.LIVE_VARS_IN, + frozenset(self.current_analyzer.in_[cfg_node])) + return node + def visit_FunctionDef(self, node): parent_analyzer = self.current_analyzer self.current_analyzer = self.cross_function_analyzer.analyzers[node] @@ -198,6 +208,10 @@ class Annotator(transformer.Base): node = self._block_statement_live_out(node) return self._block_statement_live_in(node, node.test) + def visit_With(self, node): + node = self.generic_visit(node) + return self._block_statement_live_in(node, node.items[0]) + def visit_Expr(self, node): node = self.generic_visit(node) cfg_node = self.current_analyzer.graph.index[node] diff --git a/tensorflow/python/autograph/utils/misc_test.py b/tensorflow/python/autograph/utils/misc_test.py index 8d2b0d6e138..c78df48d626 100644 --- a/tensorflow/python/autograph/utils/misc_test.py +++ b/tensorflow/python/autograph/utils/misc_test.py @@ -19,6 +19,7 @@ from __future__ import division from __future__ import print_function from tensorflow.python.autograph.utils.misc import alias_tensors +from tensorflow.python.framework import test_util from tensorflow.python.framework.constant_op import constant from tensorflow.python.ops.variables import Variable from tensorflow.python.platform import test @@ -26,14 +27,16 @@ from tensorflow.python.platform import test class MiscTest(test.TestCase): + @test_util.run_deprecated_v1 def test_alias_single_tensor(self): a = constant(1) new_a = alias_tensors(a) self.assertFalse(new_a is a) with self.cached_session() as sess: - self.assertEqual(1, sess.run(new_a)) + self.assertEqual(1, self.evaluate(new_a)) + @test_util.run_deprecated_v1 def test_alias_tensors(self): a = constant(1) v = Variable(2) @@ -47,7 +50,7 @@ class MiscTest(test.TestCase): self.assertTrue(new_s is s) self.assertTrue(new_l is l) with self.cached_session() as sess: - self.assertEqual(1, sess.run(new_a)) + self.assertEqual(1, self.evaluate(new_a)) if __name__ == '__main__': diff --git a/tensorflow/python/autograph/utils/py_func_test.py b/tensorflow/python/autograph/utils/py_func_test.py index 1c220d94922..28cefd8c3ed 100644 --- a/tensorflow/python/autograph/utils/py_func_test.py +++ b/tensorflow/python/autograph/utils/py_func_test.py @@ -34,13 +34,13 @@ class PyFuncTest(test.TestCase): with self.cached_session() as sess: result = py_func.wrap_py_func(test_fn, dtypes.int64, (1, constant_op.constant(1), 1)) - self.assertEqual(3, sess.run(result)) + self.assertEqual(3, self.evaluate(result)) result = py_func.wrap_py_func(test_fn, dtypes.int64, (1, 1, 1)) - self.assertEqual(3, sess.run(result)) + self.assertEqual(3, self.evaluate(result)) result = py_func.wrap_py_func( test_fn, dtypes.int64, (constant_op.constant(1), 1, constant_op.constant(1))) - self.assertEqual(3, sess.run(result)) + self.assertEqual(3, self.evaluate(result)) def test_wrap_py_func_complex_args(self): @@ -54,10 +54,10 @@ class PyFuncTest(test.TestCase): with self.cached_session() as sess: result = py_func.wrap_py_func(test_fn, dtypes.int64, (7, TestClass())) - self.assertEqual(35, sess.run(result)) + self.assertEqual(35, self.evaluate(result)) result = py_func.wrap_py_func(test_fn, dtypes.int64, (constant_op.constant(7), TestClass())) - self.assertEqual(35, sess.run(result)) + self.assertEqual(35, self.evaluate(result)) def test_wrap_py_func_kwargs(self): @@ -74,13 +74,13 @@ class PyFuncTest(test.TestCase): 'c': 11, 'd': TestClass(13) }) - self.assertEqual(178, sess.run(result)) + self.assertEqual(178, self.evaluate(result)) result = py_func.wrap_py_func(test_fn, dtypes.int64, (constant_op.constant(7), TestClass(5)), { 'c': constant_op.constant(11), 'd': TestClass(13) }) - self.assertEqual(178, sess.run(result)) + self.assertEqual(178, self.evaluate(result)) def test_wrap_py_func_dummy_return(self): @@ -91,11 +91,11 @@ class PyFuncTest(test.TestCase): with self.cached_session() as sess: result = py_func.wrap_py_func(test_fn, None, (5,), use_dummy_return=True) - self.assertEqual(1, sess.run(result)) + self.assertEqual(1, self.evaluate(result)) self.assertEqual([1], side_counter) result = py_func.wrap_py_func( test_fn, None, (constant_op.constant(5),), use_dummy_return=True) - self.assertEqual(1, sess.run(result)) + self.assertEqual(1, self.evaluate(result)) self.assertEqual([2], side_counter) diff --git a/tensorflow/python/autograph/utils/tensor_list_test.py b/tensorflow/python/autograph/utils/tensor_list_test.py index 697c166eb12..bbbc3bf6918 100644 --- a/tensorflow/python/autograph/utils/tensor_list_test.py +++ b/tensorflow/python/autograph/utils/tensor_list_test.py @@ -19,10 +19,10 @@ from __future__ import division from __future__ import print_function from tensorflow.python.autograph.utils import tensor_list as tl -from tensorflow.python.client.session import Session from tensorflow.python.eager import context from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops +from tensorflow.python.framework import test_util from tensorflow.python.framework.constant_op import constant from tensorflow.python.ops import list_ops from tensorflow.python.ops import tensor_array_ops @@ -34,6 +34,7 @@ class TensorListTest(test.TestCase): def _shape(self, shape_tuple): return constant(shape_tuple, dtypes.int32) + @test_util.run_v1_only("b/117943489") def test_dynamic_list_append(self): l = [] l = tl.dynamic_list_append(l, 1) @@ -42,19 +43,16 @@ class TensorListTest(test.TestCase): l = list_ops.empty_tensor_list(self._shape(()), dtypes.int32) l = tl.dynamic_list_append(l, 1) s = list_ops.tensor_list_stack(l, element_dtype=dtypes.int32) - with self.cached_session() as sess: - self.assertAllEqual(sess.run(s), [1]) + self.assertAllEqual(s, [1]) l = tensor_array_ops.TensorArray(dtypes.int32, size=0, dynamic_size=True) l = tl.dynamic_list_append(l, 1) s = l.stack() - with self.cached_session() as sess: - self.assertAllEqual(sess.run(s), [1]) + self.assertAllEqual(s, [1]) l = tl.TensorList(self._shape(()), dtypes.int32) l = tl.dynamic_list_append(l, 1) - with self.cached_session() as sess: - self.assertAllEqual(sess.run(l[0]), 1) + self.assertAllEqual(l[0], 1) def test_list_append_python(self): with context.eager_mode(): @@ -80,6 +78,7 @@ class TensorListTest(test.TestCase): l[0] = ops.convert_to_tensor(b) self.assertEqual(l[0].numpy(), b.numpy()) + @test_util.run_deprecated_v1 def test_list_append_tf(self): a = constant(3.0) l = tl.TensorList(a.shape, a.dtype) @@ -91,13 +90,12 @@ class TensorListTest(test.TestCase): c3 = l.count() a2 = l.pop() c4 = l.count() - with Session() as sess: - c1, c2, c3, c4, a, a2 = sess.run([c1, c2, c3, c4, a, a2]) - self.assertEqual(c1, 1) - self.assertEqual(c2, 2) - self.assertEqual(c3, 1) - self.assertEqual(c4, 0) - self.assertEqual(a, a2) + c1, c2, c3, c4, a, a2 = self.evaluate([c1, c2, c3, c4, a, a2]) + self.assertEqual(c1, 1) + self.assertEqual(c2, 2) + self.assertEqual(c3, 1) + self.assertEqual(c4, 0) + self.assertEqual(a, a2) def test_list_index_tf(self): a = constant(3.0) @@ -107,10 +105,9 @@ class TensorListTest(test.TestCase): l0 = l[0] l[0] = b l1 = l[0] - with self.cached_session() as sess: - l0, l1, a, b = sess.run([l0, l1, a, b]) - self.assertEqual(l0, a) - self.assertEqual(l1, b) + l0, l1, a, b = self.evaluate([l0, l1, a, b]) + self.assertEqual(l0, a) + self.assertEqual(l1, b) if __name__ == '__main__': diff --git a/tensorflow/python/autograph/utils/type_check.py b/tensorflow/python/autograph/utils/type_check.py index 8748abc47bc..ccef7dee039 100644 --- a/tensorflow/python/autograph/utils/type_check.py +++ b/tensorflow/python/autograph/utils/type_check.py @@ -30,4 +30,4 @@ def is_tensor(*args): Returns: True if any *args are TensorFlow types, False if none are. """ - return any([tensor_util.is_tensor(a) for a in args]) + return any(tensor_util.is_tensor(a) for a in args) diff --git a/tensorflow/python/autograph/utils/type_check_test.py b/tensorflow/python/autograph/utils/type_check_test.py index b3d1304e16f..2521dc9f925 100644 --- a/tensorflow/python/autograph/utils/type_check_test.py +++ b/tensorflow/python/autograph/utils/type_check_test.py @@ -28,6 +28,7 @@ from tensorflow.python.platform import test class TypeCheckTest(test.TestCase): + @test_util.run_deprecated_v1 def test_checks(self): self.assertTrue(type_check.is_tensor(constant_op.constant([1, 2, 3]))) self.assertTrue( diff --git a/tensorflow/python/client/device_lib.i b/tensorflow/python/client/device_lib.i index 944e855cee2..3e579152d51 100644 --- a/tensorflow/python/client/device_lib.i +++ b/tensorflow/python/client/device_lib.i @@ -48,17 +48,14 @@ static std::vector ListDevicesWithSessionConfig( std::vector output; SessionOptions options; options.config = config; - std::vector devices; + std::vector> devices; Status status = DeviceFactory::AddDevices( options, "" /* name_prefix */, &devices); if (!status.ok()) { Set_TF_Status_from_Status(out_status, status); } - std::vector> device_holder(devices.begin(), - devices.end()); - - for (const Device* device : devices) { + for (const std::unique_ptr& device : devices) { const DeviceAttributes& attr = device->attributes(); string attr_serialized; if (!attr.SerializeToString(&attr_serialized)) { diff --git a/tensorflow/python/client/session.py b/tensorflow/python/client/session.py index 06c66dda9fb..87a200ed336 100644 --- a/tensorflow/python/client/session.py +++ b/tensorflow/python/client/session.py @@ -828,7 +828,7 @@ class BaseSession(SessionInterface): nested list, tuple, namedtuple, dict, or OrderedDict containing graph elements at its leaves. A graph element can be one of the following types: - * An `tf.Operation`. + * A `tf.Operation`. The corresponding fetched value will be `None`. * A `tf.Tensor`. The corresponding fetched value will be a numpy ndarray containing the @@ -1097,7 +1097,7 @@ class BaseSession(SessionInterface): if isinstance(subfeed_val, ops.Tensor): raise TypeError('The value of a feed cannot be a tf.Tensor object. ' 'Acceptable feed values include Python scalars, ' - 'strings, lists, numpy ndarrays, or TensorHandles.' + 'strings, lists, numpy ndarrays, or TensorHandles. ' 'For reference, the tensor object was ' + str(feed_val) + ' which was passed to the ' 'feed with key ' + str(feed) + '.') diff --git a/tensorflow/python/client/session_clusterspec_prop_test.py b/tensorflow/python/client/session_clusterspec_prop_test.py index df020f88a88..224f880ed15 100644 --- a/tensorflow/python/client/session_clusterspec_prop_test.py +++ b/tensorflow/python/client/session_clusterspec_prop_test.py @@ -62,7 +62,7 @@ class SessionClusterSpecPropagationTest(test_util.TensorFlowTestCase): const = constant_op.constant(17) sess = session.Session(server1.target, config=config) - output = sess.run(const) + output = self.evaluate(const) self.assertEqual(17, output) def testClusterSpecPropagationWorker2Placement(self): @@ -106,7 +106,7 @@ class SessionClusterSpecPropagationTest(test_util.TensorFlowTestCase): with ops.Graph().as_default() as g, ops.device('/job:worker/task:0'): const = constant_op.constant(17) sess = session.Session(server1.target, config=config, graph=g) - output = sess.run(const) + output = self.evaluate(const) self.assertEqual(17, output) def testCanonicalDeviceNames(self): @@ -208,7 +208,7 @@ class SessionClusterSpecPropagationTest(test_util.TensorFlowTestCase): with ops.device('/job:worker/task:0/cpu:0'): sum3 = sum1 + sum2 sess = session.Session(server1.target, config=config, graph=g) - output = sess.run(sum3) + output = self.evaluate(sum3) self.assertEqual(40, output) def testLegacyDeviceNames(self): diff --git a/tensorflow/python/client/session_partial_run_test.py b/tensorflow/python/client/session_partial_run_test.py index 92ca47efa93..a97930635af 100644 --- a/tensorflow/python/client/session_partial_run_test.py +++ b/tensorflow/python/client/session_partial_run_test.py @@ -117,7 +117,7 @@ class PartialRunTest(test_util.TensorFlowTestCase): a = constant_op.constant(2.0, dtypes.float32) b = a * 2 c = b * 3 - r1 = sess.run([b, c]) + r1 = self.evaluate([b, c]) h = sess.partial_run_setup([b, c], []) r2 = sess.partial_run(h, [b, c]) self.assertEqual(r1, r2) @@ -188,6 +188,7 @@ class PartialRunTest(test_util.TensorFlowTestCase): r = sess.partial_run(h, [b], {}) self.assertEqual([6.0], r) + @test_util.run_deprecated_v1 def testInvalidPartialRunSetup(self): sess = session.Session() x = array_ops.placeholder(dtypes.float32, shape=[]) @@ -196,6 +197,7 @@ class PartialRunTest(test_util.TensorFlowTestCase): 'specify at least one target to fetch or execute.'): sess.partial_run_setup(fetches=[], feeds=[x]) + @test_util.run_deprecated_v1 def testPartialRunSetupNoFeedsPassed(self): sess = session.Session() r1 = constant_op.constant([6.0]) @@ -204,80 +206,102 @@ class PartialRunTest(test_util.TensorFlowTestCase): result1 = sess.partial_run(h, r1) self.assertEqual([6.0], result1) + @test_util.run_deprecated_v1 def testPartialRunDirect(self): self.RunTestPartialRun(session.Session()) + @test_util.run_deprecated_v1 def testPartialRunIncompleteDirect(self): self.RunTestPartialRunIncomplete(session.Session()) + @test_util.run_deprecated_v1 def testConcurrentPartialRunDirect(self): self.RunTestConcurrentPartialRun(session.Session()) + @test_util.run_deprecated_v1 def testManyPartialRunDirect(self): self.RunTestManyPartialRun(session.Session()) + @test_util.run_deprecated_v1 def testRunAndPartialRunDirect(self): self.RunTestRunAndPartialRun(session.Session()) + @test_util.run_deprecated_v1 def testPartialRunMissingPlaceholderFeedExceptionDirect(self): self.RunTestPartialRunMissingPlaceholderFeedException(session.Session()) + @test_util.run_deprecated_v1 def testPartialRunUnspecifiedFeedDirect(self): self.RunTestPartialRunUnspecifiedFeed(session.Session()) + @test_util.run_deprecated_v1 def testPartialRunUnspecifiedFetchDirect(self): self.RunTestPartialRunUnspecifiedFetch(session.Session()) + @test_util.run_deprecated_v1 def testPartialRunAlreadyFedDirect(self): self.RunTestPartialRunAlreadyFed(session.Session()) + @test_util.run_deprecated_v1 def testPartialRunAlreadyFetchedDirect(self): self.RunTestPartialRunAlreadyFetched(session.Session()) + @test_util.run_deprecated_v1 def testPartialRunEmptyFetchesDirect(self): self.RunTestPartialRunEmptyFetches(session.Session()) + @test_util.run_deprecated_v1 def testPartialRunDist(self): server = server_lib.Server.create_local_server() self.RunTestPartialRun(session.Session(server.target)) + @test_util.run_deprecated_v1 def testPartialRunIncompleteDist(self): server = server_lib.Server.create_local_server() self.RunTestPartialRunIncomplete(session.Session(server.target)) + @test_util.run_deprecated_v1 def testConcurrentPartialRunDist(self): server = server_lib.Server.create_local_server() self.RunTestConcurrentPartialRun(session.Session(server.target)) + @test_util.run_deprecated_v1 def testManyPartialRunDist(self): server = server_lib.Server.create_local_server() self.RunTestManyPartialRun(session.Session(server.target)) + @test_util.run_deprecated_v1 def testRunAndPartialRunDist(self): server = server_lib.Server.create_local_server() self.RunTestRunAndPartialRun(session.Session(server.target)) + @test_util.run_deprecated_v1 def testPartialRunMissingPlaceholderFeedExceptionDist(self): server = server_lib.Server.create_local_server() self.RunTestPartialRunMissingPlaceholderFeedException( session.Session(server.target)) + @test_util.run_deprecated_v1 def testPartialRunUnspecifiedFeedDist(self): server = server_lib.Server.create_local_server() self.RunTestPartialRunUnspecifiedFeed(session.Session(server.target)) + @test_util.run_deprecated_v1 def testPartialRunUnspecifiedFetchDist(self): server = server_lib.Server.create_local_server() self.RunTestPartialRunUnspecifiedFetch(session.Session(server.target)) + @test_util.run_deprecated_v1 def testPartialRunAlreadyFedDist(self): server = server_lib.Server.create_local_server() self.RunTestPartialRunAlreadyFed(session.Session(server.target)) + @test_util.run_deprecated_v1 def testPartialRunAlreadyFetchedDist(self): server = server_lib.Server.create_local_server() self.RunTestPartialRunAlreadyFetched(session.Session(server.target)) + @test_util.run_deprecated_v1 def testPartialRunEmptyFetchesDist(self): server = server_lib.Server.create_local_server() self.RunTestPartialRunEmptyFetches(session.Session(server.target)) diff --git a/tensorflow/python/client/timeline_test.py b/tensorflow/python/client/timeline_test.py index dfd01476430..61c0da01b83 100644 --- a/tensorflow/python/client/timeline_test.py +++ b/tensorflow/python/client/timeline_test.py @@ -57,6 +57,7 @@ class TimelineTest(test.TestCase): ctf = tl.generate_chrome_trace_format() self._validateTrace(ctf) + @test_util.run_deprecated_v1 def testTimelineCpu(self): run_options = config_pb2.RunOptions( trace_level=config_pb2.RunOptions.FULL_TRACE) @@ -147,7 +148,7 @@ class TimelineTest(test.TestCase): num2 = variables.Variable(2.0, name='num2') with ops.device('/cpu:2'): result = num1 + num2 + num1 * num2 - sess.run(variables.global_variables_initializer()) + self.evaluate(variables.global_variables_initializer()) sess.run(result, options=run_options, run_metadata=run_metadata) self.assertTrue(run_metadata.HasField('step_stats')) @@ -176,7 +177,7 @@ class TimelineTest(test.TestCase): num2 = variables.Variable(2.0, name='num2') with ops.device('/cpu:2'): result = num1 + num2 + num1 * num2 - sess.run(variables.global_variables_initializer()) + self.evaluate(variables.global_variables_initializer()) sess.run(result, options=run_options, run_metadata=run_metadata) self.assertTrue(run_metadata.HasField('step_stats')) step_stats = run_metadata.step_stats diff --git a/tensorflow/python/client/virtual_gpu_test.py b/tensorflow/python/client/virtual_gpu_test.py index 5892e0fc845..e82ee0666c3 100644 --- a/tensorflow/python/client/virtual_gpu_test.py +++ b/tensorflow/python/client/virtual_gpu_test.py @@ -216,7 +216,7 @@ class VirtualGpuTest(test_util.TensorFlowTestCase): for d in self._util.devices: with ops.device(d): var = variables.Variable(random_ops.random_uniform(mat_shape)) - sess.run(var.initializer) + self.evaluate(var.initializer) data.append(var) s = data[0] for i in range(1, len(data)): diff --git a/tensorflow/python/compat/BUILD b/tensorflow/python/compat/BUILD index e0a1c8e0571..9f2ce8c676e 100644 --- a/tensorflow/python/compat/BUILD +++ b/tensorflow/python/compat/BUILD @@ -9,7 +9,10 @@ py_library( srcs = ["compat.py"], srcs_version = "PY2AND3", visibility = ["//tensorflow:internal"], - deps = ["//tensorflow/python:util"], + deps = [ + "//tensorflow/python:tf2", + "//tensorflow/python:util", + ], ) tf_py_test( diff --git a/tensorflow/python/compat/compat.py b/tensorflow/python/compat/compat.py index 385fd431f4c..0b6ff30488e 100644 --- a/tensorflow/python/compat/compat.py +++ b/tensorflow/python/compat/compat.py @@ -23,10 +23,16 @@ from __future__ import division from __future__ import print_function import datetime + +from tensorflow.python import tf2 +from tensorflow.python.framework import ops +from tensorflow.python.framework import tensor_shape +from tensorflow.python.ops import variable_scope + from tensorflow.python.util import tf_contextlib from tensorflow.python.util.tf_export import tf_export -_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2018, 11, 13) +_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2018, 12, 4) @tf_export("compat.forward_compatible") @@ -132,3 +138,40 @@ def forward_compatibility_horizon(year, month, day): yield finally: _FORWARD_COMPATIBILITY_HORIZON = old_compat_date + + +@tf_export(v1=["enable_v2_behavior"]) +def enable_v2_behavior(): + """Enables TensorFlow 2.x behaviors. + + This function can be called at the beginning of the program (before `Tensors`, + `Graphs` or other structures have been created, and before devices have been + initialized. It switches all global behaviors that are different between + TensorFlow 1.x and 2.x to behave as intended for 2.x. + + This function is called in the main TensorFlow `__init__.py` file, user should + not need to call it, except during complex migrations. + """ + tf2.enable() # Switches TensorArrayV2 and control flow V2 + ops.enable_eager_execution() + tensor_shape.enable_v2_tensorshape() # Also switched by tf2 + variable_scope.enable_resource_variables() + + +@tf_export(v1=["disable_v2_behavior"]) +def disable_v2_behavior(): + """Enables TensorFlow 2.x behaviors. + + This function can be called at the beginning of the program (before `Tensors`, + `Graphs` or other structures have been created, and before devices have been + initialized. It switches all global behaviors that are different between + TensorFlow 1.x and 2.x to behave as intended for 1.x. + + User can call this function to disable 2.x behavior during complex migrations. + """ + tf2.disable() # Switches TensorArrayV2 and control flow V2 + ops.disable_eager_execution() + tensor_shape.disable_v2_tensorshape() # Also switched by tf2 + variable_scope.disable_resource_variables() + + diff --git a/tensorflow/python/data/__init__.py b/tensorflow/python/data/__init__.py index 7536ba668ab..75ba88f3034 100644 --- a/tensorflow/python/data/__init__.py +++ b/tensorflow/python/data/__init__.py @@ -24,6 +24,8 @@ from __future__ import print_function # pylint: disable=unused-import from tensorflow.python.data import experimental from tensorflow.python.data.ops.dataset_ops import Dataset +from tensorflow.python.data.ops.dataset_ops import make_initializable_iterator +from tensorflow.python.data.ops.dataset_ops import make_one_shot_iterator from tensorflow.python.data.ops.iterator_ops import Iterator from tensorflow.python.data.ops.readers import FixedLengthRecordDataset from tensorflow.python.data.ops.readers import TextLineDataset diff --git a/tensorflow/python/data/benchmarks/BUILD b/tensorflow/python/data/benchmarks/BUILD index fd723e0d712..5b0500eae19 100644 --- a/tensorflow/python/data/benchmarks/BUILD +++ b/tensorflow/python/data/benchmarks/BUILD @@ -6,6 +6,61 @@ exports_files(["LICENSE"]) load("//tensorflow:tensorflow.bzl", "py_test") +py_test( + name = "batch_benchmark", + srcs = ["batch_benchmark.py"], + srcs_version = "PY2AND3", + deps = [ + "//tensorflow/python:array_ops", + "//tensorflow/python:client_testlib", + "//tensorflow/python:dtypes", + "//tensorflow/python:session", + "//tensorflow/python:sparse_tensor", + "//tensorflow/python/data/ops:dataset_ops", + "//third_party/py/numpy", + ], +) + +py_test( + name = "filter_benchmark", + srcs = ["filter_benchmark.py"], + srcs_version = "PY2AND3", + deps = [ + "//tensorflow/python:array_ops", + "//tensorflow/python:client_testlib", + "//tensorflow/python:framework_ops", + "//tensorflow/python:session", + "//tensorflow/python/data/ops:dataset_ops", + "//third_party/py/numpy", + ], +) + +py_test( + name = "from_tensor_slices_benchmark", + srcs = ["from_tensor_slices_benchmark.py"], + srcs_version = "PY2AND3", + deps = [ + "//tensorflow/python:client_testlib", + "//tensorflow/python:errors", + "//tensorflow/python:session", + "//tensorflow/python/data/ops:dataset_ops", + "//third_party/py/numpy", + ], +) + +py_test( + name = "map_benchmark", + srcs = ["map_benchmark.py"], + srcs_version = "PY2AND3", + deps = [ + "//tensorflow/python:client_testlib", + "//tensorflow/python:framework_ops", + "//tensorflow/python:session", + "//tensorflow/python/data/ops:dataset_ops", + "//third_party/py/numpy", + ], +) + py_test( name = "range_benchmark", srcs = ["range_benchmark.py"], diff --git a/tensorflow/python/data/benchmarks/batch_benchmark.py b/tensorflow/python/data/benchmarks/batch_benchmark.py new file mode 100644 index 00000000000..e063849f703 --- /dev/null +++ b/tensorflow/python/data/benchmarks/batch_benchmark.py @@ -0,0 +1,85 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Benchmarks for `tf.data.Dataset.batch()`.""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import time + +import numpy as np + +from tensorflow.python.client import session +from tensorflow.python.data.ops import dataset_ops +from tensorflow.python.framework import dtypes +from tensorflow.python.framework import sparse_tensor +from tensorflow.python.ops import array_ops +from tensorflow.python.platform import test + + +# TODO(b/119837791): Add eager benchmarks. +class BatchBenchmark(test.Benchmark): + """Benchmarks for `tf.data.Dataset.batch()`.""" + + def benchmarkBatchSparse(self): + non_zeros_per_row_values = [0, 1, 5, 10, 100] + batch_size_values = [1, 32, 64, 128, 1024] + + sparse_placeholder = array_ops.sparse_placeholder(dtype=dtypes.int64) + batch_size_placeholder = array_ops.placeholder(dtype=dtypes.int64, shape=[]) + + dataset = dataset_ops.Dataset.from_tensors(sparse_placeholder).repeat( + ).batch(batch_size_placeholder) + iterator = dataset_ops.make_initializable_iterator(dataset) + next_element = iterator.get_next() + + for non_zeros_per_row in non_zeros_per_row_values: + + sparse_value = sparse_tensor.SparseTensorValue( + indices=np.arange(non_zeros_per_row, dtype=np.int64)[:, np.newaxis], + values=np.arange(non_zeros_per_row, dtype=np.int64), + dense_shape=[1000]) + + for batch_size in batch_size_values: + + with session.Session() as sess: + sess.run(iterator.initializer, feed_dict={ + sparse_placeholder: sparse_value, + batch_size_placeholder: batch_size}) + # Run five steps to warm up the session caches before taking the + # first measurement. + for _ in range(5): + sess.run(next_element.indices.op) + deltas = [] + for _ in range(100): + start = time.time() + for _ in range(100): + sess.run(next_element.indices.op) + end = time.time() + deltas.append(end - start) + + median_wall_time = np.median(deltas) / 100.0 + + print("Batch sparse dataset non-zeros per row: %d batch_size: %d " + "wall time: %f" + % (non_zeros_per_row, batch_size, median_wall_time)) + self.report_benchmark( + iters=10000, wall_time=median_wall_time, + name="batch_sparse_dataset_nnz_%d_batch_size_%d" % ( + non_zeros_per_row, batch_size)) + + +if __name__ == "__main__": + test.main() diff --git a/tensorflow/python/data/benchmarks/filter_benchmark.py b/tensorflow/python/data/benchmarks/filter_benchmark.py new file mode 100644 index 00000000000..a6d86fe2218 --- /dev/null +++ b/tensorflow/python/data/benchmarks/filter_benchmark.py @@ -0,0 +1,69 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Benchmarks for `tf.data.Dataset.filter()`.""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import time + +import numpy as np + +from tensorflow.python.client import session +from tensorflow.python.data.ops import dataset_ops +from tensorflow.python.framework import ops +from tensorflow.python.ops import array_ops +from tensorflow.python.platform import test + + +# TODO(b/119837791): Add eager benchmarks. +class FilterBenchmark(test.Benchmark): + """Benchmarks for `tf.data.Dataset.filter()`.""" + + def _benchmark(self, predicate, name): + with ops.Graph().as_default(): + dataset = ( + dataset_ops.Dataset.from_tensors(True).repeat(None).filter(predicate)) + iterator = dataset_ops.make_one_shot_iterator(dataset) + next_element = iterator.get_next() + + with session.Session() as sess: + for _ in range(5): + sess.run(next_element.op) + deltas = [] + for _ in range(100): + start = time.time() + for _ in range(100): + sess.run(next_element.op) + end = time.time() + deltas.append(end - start) + + median_wall_time = np.median(deltas) / 100 + print("Filter dataset using %s. Median wall time: %f" % + (name, median_wall_time)) + self.report_benchmark( + iters=100, + wall_time=median_wall_time, + name=name) + + def benchmarkSimpleFunction(self): + self._benchmark(array_ops.identity, "simple_function") + + def benchmarkReturnComponentOptimization(self): + self._benchmark(lambda x: x, "return_component") + + +if __name__ == "__main__": + test.main() diff --git a/tensorflow/python/data/benchmarks/from_tensor_slices_benchmark.py b/tensorflow/python/data/benchmarks/from_tensor_slices_benchmark.py new file mode 100644 index 00000000000..d7f1a4e7af5 --- /dev/null +++ b/tensorflow/python/data/benchmarks/from_tensor_slices_benchmark.py @@ -0,0 +1,188 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Benchmarks for `tf.data.Dataset.from_tensor_slices()`.""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import time + +import numpy as np + +from tensorflow.python.client import session +from tensorflow.python.data.ops import dataset_ops +from tensorflow.python.framework import errors +from tensorflow.python.platform import test + + +# TODO(b/119837791): Add eager benchmarks. +class FromTensorSlicesBenchmark(test.Benchmark): + """Benchmarks for `tf.data.Dataset.from_tensor_slices()`.""" + + def benchmarkSliceRepeatBatch(self): + input_size = 10000 + batch_size = 100 + num_epochs = 100 + + input_data = np.random.randn(input_size) + + dataset = ( + dataset_ops.Dataset.from_tensor_slices(input_data) + .repeat(num_epochs + 1).batch(batch_size)) + iterator = dataset_ops.make_initializable_iterator(dataset) + next_element = iterator.get_next() + + with session.Session() as sess: + sess.run(iterator.initializer) + # Run one whole epoch to burn in the computation. + for _ in range(input_size // batch_size): + sess.run(next_element) + deltas = [] + try: + while True: + start = time.time() + sess.run(next_element) + deltas.append(time.time() - start) + except errors.OutOfRangeError: + pass + + median_wall_time = np.median(deltas) + print("Slice/repeat/batch with sess.run() input size: %d batch size: %d " + "Median wall time per element: %f" % (input_size, batch_size, + median_wall_time)) + self.report_benchmark( + iters=len(deltas), + wall_time=median_wall_time, + name="slice_repeat_batch_input_%d_batch_%d" % (input_size, batch_size)) + + def benchmarkSliceRepeatBatchCallable(self): + input_size = 10000 + batch_size = 100 + num_epochs = 100 + + input_data = np.random.randn(input_size) + + dataset = ( + dataset_ops.Dataset.from_tensor_slices(input_data) + .repeat(num_epochs + 1).batch(batch_size)) + iterator = dataset_ops.make_initializable_iterator(dataset) + next_element = iterator.get_next() + + with session.Session() as sess: + sess.run(iterator.initializer) + get_next_element = sess.make_callable(next_element) + # Run one whole epoch to burn in the computation. + for _ in range(input_size // batch_size): + get_next_element() + deltas = [] + try: + while True: + start = time.time() + get_next_element() + deltas.append(time.time() - start) + except errors.OutOfRangeError: + pass + + median_wall_time = np.median(deltas) + print( + "Slice/repeat/batch with callable input size: %d batch size: %d Median" + " wall time per element: %f" % (input_size, batch_size, + median_wall_time)) + self.report_benchmark( + iters=len(deltas), + wall_time=median_wall_time, + name="slice_repeat_batch_callable_input_%d_batch_%d" % + (input_size, batch_size)) + + def benchmarkReshapeSliceRepeatCallable(self): + input_size = 10000 + batch_size = 100 + num_epochs = 100 + + input_data = np.random.randn(input_size) + + dataset = ( + dataset_ops.Dataset.from_tensor_slices(input_data.reshape(100, 100)) + .repeat(num_epochs + 1)) + iterator = dataset_ops.make_initializable_iterator(dataset) + next_element = iterator.get_next() + + with session.Session() as sess: + sess.run(iterator.initializer) + get_next_element = sess.make_callable(next_element) + # Run one whole epoch to burn in the computation. + for _ in range(input_size // batch_size): + get_next_element() + deltas = [] + try: + while True: + start = time.time() + get_next_element() + deltas.append(time.time() - start) + except errors.OutOfRangeError: + pass + + median_wall_time = np.median(deltas) + print("Reshape/slice/repeat with callable input size: %d batch size: %d " + "Median wall time per element: %f" % (input_size, batch_size, + median_wall_time)) + self.report_benchmark( + iters=len(deltas), + wall_time=median_wall_time, + name="reshape_slice_repeat_callable_input_%d_batch_%d" % + (input_size, batch_size)) + + def benchmarkSliceBatchCacheRepeatCallable(self): + input_size = 10000 + batch_size = 100 + num_epochs = 100 + + input_data = np.random.randn(input_size) + + dataset = ( + dataset_ops.Dataset.from_tensor_slices(input_data).batch(batch_size) + .cache().repeat(num_epochs + 1)) + iterator = dataset_ops.make_initializable_iterator(dataset) + next_element = iterator.get_next() + + with session.Session() as sess: + sess.run(iterator.initializer) + get_next_element = sess.make_callable(next_element) + # Run one whole epoch to burn in the computation. + for _ in range(input_size // batch_size): + get_next_element() + deltas = [] + try: + while True: + start = time.time() + get_next_element() + deltas.append(time.time() - start) + except errors.OutOfRangeError: + pass + + median_wall_time = np.median(deltas) + print( + "Slice/batch/cache/repeat with callable input size: %d batch size: %d " + "Median wall time per element: %f" + % (input_size, batch_size, median_wall_time)) + self.report_benchmark( + iters=len(deltas), + wall_time=median_wall_time, + name="slice_batch_cache_repeat_callable_input_%d_batch_%d" % + (input_size, batch_size)) + + +if __name__ == "__main__": + test.main() diff --git a/tensorflow/python/data/benchmarks/map_benchmark.py b/tensorflow/python/data/benchmarks/map_benchmark.py new file mode 100644 index 00000000000..65d945cdae8 --- /dev/null +++ b/tensorflow/python/data/benchmarks/map_benchmark.py @@ -0,0 +1,135 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Bechmarks for `tf.data.Dataset.map()`.""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import time + +import numpy as np + +from tensorflow.python.client import session +from tensorflow.python.data.ops import dataset_ops +from tensorflow.python.framework import ops +from tensorflow.python.platform import test + + +# TODO(b/119837791): Add eager benchmarks. +class MapBenchmark(test.Benchmark): + """Bechmarks for `tf.data.Dataset.map()`.""" + + def benchmarkChainOfMaps(self): + chain_lengths = [0, 1, 2, 5, 10, 20, 50] + for chain_length in chain_lengths: + for mode in ["general", "single-threaded", "short-circuit"]: + if mode == "general": + map_fn = lambda x: x + 1 + use_inter_op_parallelism = True + print_label = "" + benchmark_label = "" + if mode == "single-threaded": + map_fn = lambda x: x + 1 + use_inter_op_parallelism = False + print_label = " (single threaded mode)" + benchmark_label = "_single_threaded" + if mode == "short-circuit": + map_fn = lambda x: x + use_inter_op_parallelism = True # should not have any significance + print_label = " (short circuit mode)" + benchmark_label = "_short_circuit" + + with ops.Graph().as_default(): + dataset = dataset_ops.Dataset.from_tensors(0).repeat(None) + for _ in range(chain_length): + dataset = dataset_ops.MapDataset( + dataset, + map_fn, + use_inter_op_parallelism=use_inter_op_parallelism) + iterator = dataset_ops.make_one_shot_iterator(dataset) + next_element = iterator.get_next() + + with session.Session() as sess: + for _ in range(5): + sess.run(next_element.op) + deltas = [] + for _ in range(100): + start = time.time() + for _ in range(100): + sess.run(next_element.op) + end = time.time() + deltas.append(end - start) + + median_wall_time = np.median(deltas) / 100 + print("Map dataset chain length%s: %d Median wall time: %f" % + (print_label, chain_length, median_wall_time)) + self.report_benchmark( + iters=1000, + wall_time=median_wall_time, + name="map_dataset_chain_length_%d%s" % (chain_length, + benchmark_label)) + + def benchmarkMapFanOut(self): + fan_outs = [1, 2, 5, 10, 20, 50, 100] + for fan_out in fan_outs: + for mode in ["general", "single-threaded", "short-circuit"]: + if mode == "general": + map_fn = lambda *xs: [x + 1 for x in xs] + use_inter_op_parallelism = True + print_label = "" + benchmark_label = "" + if mode == "single-threaded": + map_fn = lambda *xs: [x + 1 for x in xs] + use_inter_op_parallelism = False + print_label = " (single threaded mode)" + benchmark_label = "_single_threaded" + if mode == "short-circuit": + map_fn = lambda *xs: xs + use_inter_op_parallelism = True # should not have any significance + print_label = " (short circuit mode)" + benchmark_label = "_short_circuit" + + with ops.Graph().as_default(): + dataset = dataset_ops.Dataset.from_tensors( + tuple(0 for _ in range(fan_out))).repeat(None) + dataset = dataset_ops.MapDataset( + dataset, + map_fn, + use_inter_op_parallelism=use_inter_op_parallelism) + iterator = dataset_ops.make_one_shot_iterator(dataset) + next_element = iterator.get_next() + + with session.Session() as sess: + for _ in range(5): + sess.run(next_element[0].op) + deltas = [] + for _ in range(100): + start = time.time() + for _ in range(100): + sess.run(next_element[0].op) + end = time.time() + deltas.append(end - start) + + median_wall_time = np.median(deltas) / 100 + print("Map dataset fan out%s: %d Median wall time: %f" % + (print_label, fan_out, median_wall_time)) + self.report_benchmark( + iters=1000, + wall_time=median_wall_time, + name="map_dataset_fan_out_%d%s" % (fan_out, benchmark_label)) + + +if __name__ == "__main__": + test.main() diff --git a/tensorflow/python/data/benchmarks/range_benchmark.py b/tensorflow/python/data/benchmarks/range_benchmark.py index 25f63b79a26..a5020e28730 100644 --- a/tensorflow/python/data/benchmarks/range_benchmark.py +++ b/tensorflow/python/data/benchmarks/range_benchmark.py @@ -39,7 +39,7 @@ class RangeBenchmark(test.Benchmark): # costs). dataset = dataset_ops.Dataset.range(num_elements).skip( num_elements - 1).take(1).with_options(options) - iterator = dataset.make_initializable_iterator() + iterator = dataset_ops.make_initializable_iterator(dataset) next_element = iterator.get_next() with session.Session() as sess: diff --git a/tensorflow/python/data/experimental/__init__.py b/tensorflow/python/data/experimental/__init__.py index 126c2be4420..14dfec37cd0 100644 --- a/tensorflow/python/data/experimental/__init__.py +++ b/tensorflow/python/data/experimental/__init__.py @@ -25,6 +25,7 @@ See [Importing Data](https://tensorflow.org/guide/datasets) for an overview. @@Counter @@CheckpointInputPipelineHook @@CsvDataset +@@OptimizationOptions @@Optional @@RandomDataset @@Reducer @@ -32,12 +33,15 @@ See [Importing Data](https://tensorflow.org/guide/datasets) for an overview. @@StatsAggregator @@StatsOptions @@TFRecordWriter +@@ThreadingOptions @@bucket_by_sequence_length +@@cardinality @@choose_from_datasets @@copy_to_device @@dense_to_sparse_batch @@enumerate_dataset +@@filter_for_shard @@get_next_as_optional @@get_single_element @@group_by_reducer @@ -59,6 +63,8 @@ See [Importing Data](https://tensorflow.org/guide/datasets) for an overview. @@unique @@AUTOTUNE +@@INFINITE_CARDINALITY +@@UNKNOWN_CARDINALITY """ from __future__ import absolute_import @@ -70,9 +76,13 @@ from __future__ import print_function from tensorflow.python.data.experimental.ops.batching import dense_to_sparse_batch from tensorflow.python.data.experimental.ops.batching import map_and_batch from tensorflow.python.data.experimental.ops.batching import unbatch +from tensorflow.python.data.experimental.ops.cardinality import cardinality +from tensorflow.python.data.experimental.ops.cardinality import INFINITE as INFINITE_CARDINALITY +from tensorflow.python.data.experimental.ops.cardinality import UNKNOWN as UNKNOWN_CARDINALITY from tensorflow.python.data.experimental.ops.counter import Counter from tensorflow.python.data.experimental.ops.enumerate_ops import enumerate_dataset from tensorflow.python.data.experimental.ops.error_ops import ignore_errors +from tensorflow.python.data.experimental.ops.filter_for_shard_ops import filter_for_shard from tensorflow.python.data.experimental.ops.get_single_element import get_single_element from tensorflow.python.data.experimental.ops.grouping import bucket_by_sequence_length from tensorflow.python.data.experimental.ops.grouping import group_by_reducer @@ -83,10 +93,8 @@ from tensorflow.python.data.experimental.ops.interleave_ops import parallel_inte from tensorflow.python.data.experimental.ops.interleave_ops import sample_from_datasets from tensorflow.python.data.experimental.ops.iterator_ops import CheckpointInputPipelineHook from tensorflow.python.data.experimental.ops.iterator_ops import make_saveable_from_iterator - -# Optimization constant that can be used to enable auto-tuning. from tensorflow.python.data.experimental.ops.optimization import AUTOTUNE - +from tensorflow.python.data.experimental.ops.optimization_options import OptimizationOptions from tensorflow.python.data.experimental.ops.parsing_ops import parse_example_dataset from tensorflow.python.data.experimental.ops.prefetching_ops import copy_to_device from tensorflow.python.data.experimental.ops.prefetching_ops import prefetch_to_device @@ -101,6 +109,7 @@ from tensorflow.python.data.experimental.ops.shuffle_ops import shuffle_and_repe from tensorflow.python.data.experimental.ops.stats_aggregator import StatsAggregator from tensorflow.python.data.experimental.ops.stats_ops import latency_stats from tensorflow.python.data.experimental.ops.stats_options import StatsOptions +from tensorflow.python.data.experimental.ops.threading_options import ThreadingOptions from tensorflow.python.data.experimental.ops.unique import unique from tensorflow.python.data.experimental.ops.writers import TFRecordWriter from tensorflow.python.data.ops.iterator_ops import get_next_as_optional diff --git a/tensorflow/python/data/experimental/benchmarks/BUILD b/tensorflow/python/data/experimental/benchmarks/BUILD index b89fbe7757b..8175116c6ed 100644 --- a/tensorflow/python/data/experimental/benchmarks/BUILD +++ b/tensorflow/python/data/experimental/benchmarks/BUILD @@ -8,15 +8,12 @@ load("//tensorflow:tensorflow.bzl", "cuda_py_test") load("//tensorflow:tensorflow.bzl", "py_test") py_test( - name = "map_and_batch_benchmark", - size = "medium", - srcs = ["map_and_batch_benchmark.py"], + name = "autotune_benchmark", + srcs = ["autotune_benchmark.py"], srcs_version = "PY2AND3", deps = [ - "//tensorflow/python:array_ops", "//tensorflow/python:client_testlib", - "//tensorflow/python:framework_ops", - "//tensorflow/python:random_ops", + "//tensorflow/python:math_ops", "//tensorflow/python:session", "//tensorflow/python/data/experimental/ops:batching", "//tensorflow/python/data/experimental/ops:optimization", @@ -26,17 +23,102 @@ py_test( ) py_test( - name = "map_benchmark", - size = "medium", - srcs = ["map_benchmark.py"], + name = "csv_dataset_benchmark", + srcs = ["csv_dataset_benchmark.py"], + srcs_version = "PY2AND3", + tags = ["no_pip"], + deps = [ + "//tensorflow/python:client_testlib", + "//tensorflow/python:parsing_ops", + "//tensorflow/python:platform", + "//tensorflow/python:platform_test", + "//tensorflow/python:session", + "//tensorflow/python/data/experimental/ops:readers", + "//tensorflow/python/data/ops:readers", + "//third_party/py/numpy", + ], +) + +py_test( + name = "map_and_batch_benchmark", + srcs = ["map_and_batch_benchmark.py"], + srcs_version = "PY2AND3", + deps = [ + "//tensorflow/core:protos_all_py", + "//tensorflow/python:array_ops", + "//tensorflow/python:client_testlib", + "//tensorflow/python:constant_op", + "//tensorflow/python:dtypes", + "//tensorflow/python:math_ops", + "//tensorflow/python:random_ops", + "//tensorflow/python:session", + "//tensorflow/python/data/experimental/ops:batching", + "//tensorflow/python/data/ops:dataset_ops", + "//third_party/py/numpy", + ], +) + +py_test( + name = "map_vectorization_benchmark", + srcs = ["map_vectorization_benchmark.py"], + srcs_version = "PY2AND3", + deps = [ + "//tensorflow/core:protos_all_py", + "//tensorflow/python:array_ops", + "//tensorflow/python:client_testlib", + "//tensorflow/python:constant_op", + "//tensorflow/python:dtypes", + "//tensorflow/python:math_ops", + "//tensorflow/python:parsing_ops", + "//tensorflow/python:session", + "//tensorflow/python/data/ops:dataset_ops", + "//tensorflow/python/data/util:nest", + "//third_party/py/numpy", + ], +) + +py_test( + name = "matching_files_benchmark", + size = "small", + srcs = ["matching_files_benchmark.py"], + srcs_version = "PY2AND3", + deps = [ + "//tensorflow/python:array_ops", + "//tensorflow/python:client_testlib", + "//tensorflow/python:dtypes", + "//tensorflow/python:errors", + "//tensorflow/python:util", + "//tensorflow/python/data/experimental/ops:matching_files", + "//tensorflow/python/data/ops:dataset_ops", + "//third_party/py/numpy", + ], +) + +py_test( + name = "optimize_benchmark", + srcs = ["optimize_benchmark.py"], srcs_version = "PY2AND3", deps = [ "//tensorflow/python:client_testlib", "//tensorflow/python:framework_ops", "//tensorflow/python:math_ops", "//tensorflow/python:session", - "//tensorflow/python/data/experimental/ops:batching", - "//tensorflow/python/data/experimental/ops:optimization", + "//tensorflow/python/data/ops:dataset_ops", + "//third_party/py/numpy", + ], +) + +py_test( + name = "unbatch_benchmark", + srcs = ["unbatch_benchmark.py"], + srcs_version = "PY2AND3", + deps = [ + "//tensorflow/python:array_ops", + "//tensorflow/python:client_testlib", + "//tensorflow/python:dtypes", + "//tensorflow/python:framework_ops", + "//tensorflow/python:session", + "//tensorflow/python/data/experimental/ops:batching", "//tensorflow/python/data/ops:dataset_ops", "//third_party/py/numpy", ], diff --git a/tensorflow/python/data/experimental/benchmarks/autotune_benchmark.py b/tensorflow/python/data/experimental/benchmarks/autotune_benchmark.py new file mode 100644 index 00000000000..b48ef95666e --- /dev/null +++ b/tensorflow/python/data/experimental/benchmarks/autotune_benchmark.py @@ -0,0 +1,187 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Benchmarks for autotuning performance knobs.""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import time + +import numpy as np + +from tensorflow.python.client import session +from tensorflow.python.data.experimental.ops import batching +from tensorflow.python.data.experimental.ops import optimization +from tensorflow.python.data.ops import dataset_ops +from tensorflow.python.ops import math_ops +from tensorflow.python.platform import test + + +class AutotuneBenchmark(test.Benchmark): + """Benchmarks for autotuning performance knobs.""" + + def benchmarkMap(self): + k = 1024 * 1024 + dataset = dataset_ops.Dataset.from_tensors((np.random.rand(1, 4 * k), + np.random.rand(4 * k, + 1))).repeat() + dataset = dataset.map( + math_ops.matmul, num_parallel_calls=optimization.AUTOTUNE) + iterator = dataset_ops.make_one_shot_iterator(dataset) + get_next = iterator.get_next() + + deltas = [] + with session.Session() as sess: + for _ in range(5): + sess.run(get_next.op) + for _ in range(1000): + start = time.time() + sess.run(get_next.op) + end = time.time() + deltas.append(end - start) + + print("%f (median), %f (mean), %f (stddev), %f (min), %f (max)\n" % + (np.median(deltas), np.mean(deltas), np.std(deltas), np.min(deltas), + np.max(deltas))) + self.report_benchmark( + iters=1000, wall_time=np.median(deltas), name="map_autotune") + + def benchmarkMapAndBatch(self): + self._benchmarkMapAndBatch(numa_aware=False) + self._benchmarkMapAndBatch(numa_aware=True) + + def _benchmarkMapAndBatch(self, numa_aware): + batch_size = 16 + k = 1024 * 1024 + dataset = dataset_ops.Dataset.from_tensors((np.random.rand(1, 4 * k), + np.random.rand(4 * k, + 1))).repeat() + dataset = dataset.apply( + batching.map_and_batch( + math_ops.matmul, + num_parallel_calls=optimization.AUTOTUNE, + batch_size=batch_size)) + options = dataset_ops.Options() + options.experimental_numa_aware = numa_aware + dataset = dataset.with_options(options) + iterator = dataset_ops.make_one_shot_iterator(dataset) + get_next = iterator.get_next() + + deltas = [] + with session.Session() as sess: + for _ in range(5): + sess.run(get_next.op) + for _ in range(100): + start = time.time() + sess.run(get_next.op) + end = time.time() + deltas.append(end - start) + + print("%f (median), %f (mean), %f (stddev), %f (min), %f (max)\n" % + (np.median(deltas), np.mean(deltas), np.std(deltas), np.min(deltas), + np.max(deltas))) + + self.report_benchmark( + iters=100, + wall_time=np.median(deltas), + name=("numa_" if numa_aware else "") + "map_and_batch_autotune") + + def benchmarkInterleave(self): + k = 1024 * 1024 + dataset = dataset_ops.Dataset.from_tensors((np.random.rand(1, 4 * k), + np.random.rand(4 * k, + 1))).repeat() + dataset = dataset.map(math_ops.matmul) + dataset = dataset_ops.Dataset.range(1).repeat().interleave( + lambda _: dataset, + cycle_length=10, + num_parallel_calls=optimization.AUTOTUNE) + iterator = dataset_ops.make_one_shot_iterator(dataset) + get_next = iterator.get_next() + + deltas = [] + with session.Session() as sess: + for _ in range(5): + sess.run(get_next.op) + for _ in range(1000): + start = time.time() + sess.run(get_next.op) + end = time.time() + deltas.append(end - start) + + print("%f (median), %f (mean), %f (stddev), %f (min), %f (max)\n" % + (np.median(deltas), np.mean(deltas), np.std(deltas), np.min(deltas), + np.max(deltas))) + self.report_benchmark( + iters=1000, + wall_time=np.median(deltas), + name="interleave_autotune") + + def benchmarkMapAndInterleave(self): + k = 1024 * 1024 + a = (np.random.rand(1, 8 * k), np.random.rand(8 * k, 1)) + b = (np.random.rand(1, 4 * k), np.random.rand(4 * k, 1)) + c = (np.random.rand(1, 2 * k), np.random.rand(2 * k, 1)) + dataset = dataset_ops.Dataset.from_tensors((a, b, c)).repeat() + + def f1(a, b, c): + x, y = a + return math_ops.matmul(x, y), b, c + + def f2(a, b, c): + x, y = b + return a, math_ops.matmul(x, y), c + + def f3(a, b, c): + x, y = c + return a, b, math_ops.matmul(x, y) + + dataset = dataset.map(f1, num_parallel_calls=optimization.AUTOTUNE) + dataset = dataset_ops.Dataset.range(1).repeat().interleave( + lambda _: dataset, + num_parallel_calls=optimization.AUTOTUNE, + cycle_length=2) + + dataset = dataset.map(f2, num_parallel_calls=optimization.AUTOTUNE) + dataset = dataset_ops.Dataset.range(1).repeat().interleave( + lambda _: dataset, + num_parallel_calls=optimization.AUTOTUNE, + cycle_length=2) + + dataset = dataset.map(f3, num_parallel_calls=optimization.AUTOTUNE) + iterator = dataset_ops.make_one_shot_iterator(dataset) + get_next = iterator.get_next() + + deltas = [] + with session.Session() as sess: + for _ in range(5): + sess.run(get_next) + for _ in range(100): + start = time.time() + sess.run(get_next) + end = time.time() + deltas.append(end - start) + + print("%f (median), %f (mean), %f (stddev), %f (min), %f (max)\n" % + (np.median(deltas), np.mean(deltas), np.std(deltas), np.min(deltas), + np.max(deltas))) + self.report_benchmark( + iters=100, + wall_time=np.median(deltas), + name="map_and_interleave_autotune") + + +if __name__ == "__main__": + test.main() diff --git a/tensorflow/python/data/experimental/benchmarks/csv_dataset_benchmark.py b/tensorflow/python/data/experimental/benchmarks/csv_dataset_benchmark.py new file mode 100644 index 00000000000..03345ce4e66 --- /dev/null +++ b/tensorflow/python/data/experimental/benchmarks/csv_dataset_benchmark.py @@ -0,0 +1,130 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Benchmarks for `tf.data.experimental.CsvDataset`.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import os +import string +import tempfile +import time + +import numpy as np + +from tensorflow.python.client import session +from tensorflow.python.data.experimental.ops import readers +from tensorflow.python.data.ops import dataset_ops +from tensorflow.python.data.ops import readers as core_readers +from tensorflow.python.ops import parsing_ops +from tensorflow.python.platform import gfile +from tensorflow.python.platform import googletest +from tensorflow.python.platform import test + + +class CsvDatasetBenchmark(test.Benchmark): + """Benchmarks for `tf.data.experimental.CsvDataset`.""" + + FLOAT_VAL = '1.23456E12' + STR_VAL = string.ascii_letters * 10 + + def _setUp(self, str_val): + # Since this isn't test.TestCase, have to manually create a test dir + gfile.MakeDirs(googletest.GetTempDir()) + self._temp_dir = tempfile.mkdtemp(dir=googletest.GetTempDir()) + + self._num_cols = [4, 64, 256] + self._num_per_iter = 5000 + self._filenames = [] + for n in self._num_cols: + fn = os.path.join(self._temp_dir, 'file%d.csv' % n) + with open(fn, 'wb') as f: + # Just write 100 rows and use `repeat`... Assumes the cost + # of creating an iterator is not significant + row = ','.join([str_val for _ in range(n)]) + f.write('\n'.join([row for _ in range(100)])) + self._filenames.append(fn) + + def _tearDown(self): + gfile.DeleteRecursively(self._temp_dir) + + def _runBenchmark(self, dataset, num_cols, prefix): + dataset = dataset.skip(self._num_per_iter - 1) + deltas = [] + for _ in range(10): + next_element = dataset_ops.make_one_shot_iterator(dataset).get_next() + with session.Session() as sess: + start = time.time() + # NOTE: This depends on the underlying implementation of skip, to have + # the net effect of calling `GetNext` num_per_iter times on the + # input dataset. We do it this way (instead of a python for loop, or + # batching N inputs in one iter) so that the overhead from session.run + # or batch doesn't dominate. If we eventually optimize skip, this has + # to change. + sess.run(next_element) + end = time.time() + deltas.append(end - start) + # Median wall time per CSV record read and decoded + median_wall_time = np.median(deltas) / self._num_per_iter + print('%s num_cols: %d Median wall time: %f' % (prefix, num_cols, + median_wall_time)) + self.report_benchmark( + iters=self._num_per_iter, + wall_time=median_wall_time, + name='%s_with_cols_%d' % (prefix, num_cols)) + + def benchmarkMapWithFloats(self): + self._setUp(self.FLOAT_VAL) + for i in range(len(self._filenames)): + num_cols = self._num_cols[i] + kwargs = {'record_defaults': [[0.0]] * num_cols} + dataset = core_readers.TextLineDataset(self._filenames[i]).repeat() + dataset = dataset.map(lambda l: parsing_ops.decode_csv(l, **kwargs)) # pylint: disable=cell-var-from-loop + self._runBenchmark(dataset, num_cols, 'csv_float_map_decode_csv') + self._tearDown() + + def benchmarkMapWithStrings(self): + self._setUp(self.STR_VAL) + for i in range(len(self._filenames)): + num_cols = self._num_cols[i] + kwargs = {'record_defaults': [['']] * num_cols} + dataset = core_readers.TextLineDataset(self._filenames[i]).repeat() + dataset = dataset.map(lambda l: parsing_ops.decode_csv(l, **kwargs)) # pylint: disable=cell-var-from-loop + self._runBenchmark(dataset, num_cols, 'csv_strings_map_decode_csv') + self._tearDown() + + def benchmarkCsvDatasetWithFloats(self): + self._setUp(self.FLOAT_VAL) + for i in range(len(self._filenames)): + num_cols = self._num_cols[i] + kwargs = {'record_defaults': [[0.0]] * num_cols} + dataset = core_readers.TextLineDataset(self._filenames[i]).repeat() + dataset = readers.CsvDataset(self._filenames[i], **kwargs).repeat() # pylint: disable=cell-var-from-loop + self._runBenchmark(dataset, num_cols, 'csv_float_fused_dataset') + self._tearDown() + + def benchmarkCsvDatasetWithStrings(self): + self._setUp(self.STR_VAL) + for i in range(len(self._filenames)): + num_cols = self._num_cols[i] + kwargs = {'record_defaults': [['']] * num_cols} + dataset = core_readers.TextLineDataset(self._filenames[i]).repeat() + dataset = readers.CsvDataset(self._filenames[i], **kwargs).repeat() # pylint: disable=cell-var-from-loop + self._runBenchmark(dataset, num_cols, 'csv_strings_fused_dataset') + self._tearDown() + +if __name__ == '__main__': + test.main() diff --git a/tensorflow/python/data/experimental/benchmarks/map_and_batch_benchmark.py b/tensorflow/python/data/experimental/benchmarks/map_and_batch_benchmark.py index a90156cd33e..fbd06a5a78e 100644 --- a/tensorflow/python/data/experimental/benchmarks/map_and_batch_benchmark.py +++ b/tensorflow/python/data/experimental/benchmarks/map_and_batch_benchmark.py @@ -17,6 +17,8 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function +import hashlib +import itertools import time import numpy as np @@ -25,11 +27,15 @@ from tensorflow.core.protobuf import config_pb2 from tensorflow.python.client import session from tensorflow.python.data.experimental.ops import batching from tensorflow.python.data.ops import dataset_ops +from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes from tensorflow.python.ops import array_ops +from tensorflow.python.ops import math_ops from tensorflow.python.ops import random_ops from tensorflow.python.platform import test +_NUMPY_RANDOM_SEED = 42 + class MapAndBatchBenchmark(test.Benchmark): """Benchmarks for `tf.data.experimental.map_and_batch()`.""" @@ -48,7 +54,7 @@ class MapAndBatchBenchmark(test.Benchmark): dataset = dataset.apply(batching.map_and_batch( lambda _: dense_value, batch_size_placeholder)) - iterator = dataset.make_initializable_iterator() + iterator = dataset_ops.make_initializable_iterator(dataset) next_element = iterator.get_next() for shape in shapes: @@ -89,6 +95,129 @@ class MapAndBatchBenchmark(test.Benchmark): name="benchmark_batch_dense_dataset_nnz_%d_batch_size_%d" % ( np.prod(shape), batch_size)) + def benchmarkMapAndBatchChainingVersusFusing(self): + """Compares the performance of chaining and fusing map and batch. + + NOTE: It is recommended to build the benchmark with + `-c opt --copt=-mavx --copt=-mavx2 --copt=-mfma --copt=-gmlt` + and execute it on a machine with at least 32 CPU cores. + """ + + # Sequential pipeline configurations. + seq_elem_size_series = itertools.product([1], [1], [1, 2, 4, 8], [16]) + seq_batch_size_series = itertools.product([1], [1], [1], [8, 16, 32, 64]) + + # Parallel pipeline configuration. + par_elem_size_series = itertools.product([32], [32], [1, 2, 4, 8], [256]) + par_batch_size_series = itertools.product([32], [32], [1], + [128, 256, 512, 1024]) + par_num_calls_series = itertools.product([8, 16, 32, 64], [32], [1], [512]) + par_inter_op_series = itertools.product([32], [8, 16, 32, 64], [1], [512]) + + def name(method, label, num_calls, inter_op, element_size, batch_size): + return ("%s_id_%s_num_calls_%d_inter_op_%d_elem_size_%d_batch_size_%d" % ( + method, + hashlib.sha1(label).hexdigest()[:8], + num_calls, + inter_op, + element_size, + batch_size, + )) + + def benchmark(label, series): + """Runs benchmark the given series.""" + + print("%s:" % label) + + def make_base_dataset(element_size): + k = 1024 * 1024 + x = constant_op.constant(np.random.rand(element_size, 4 * k)) + y = constant_op.constant(np.random.rand(4 * k, 1)) + return dataset_ops.Dataset.range(1000000000000).map(lambda _: (x, y)) + + for num_calls, inter_op, element_size, batch_size in series: + + num_iters = 1024 // ( + (element_size * batch_size) // min(num_calls, inter_op)) + dataset = make_base_dataset(element_size) + chained_dataset = dataset.map( + math_ops.matmul, + num_parallel_calls=num_calls).batch(batch_size=batch_size) + chained_iterator = dataset_ops.make_one_shot_iterator(chained_dataset) + chained_get_next = chained_iterator.get_next() + + chained_deltas = [] + with session.Session( + config=config_pb2.ConfigProto( + inter_op_parallelism_threads=inter_op, + use_per_session_threads=True)) as sess: + for _ in range(5): + sess.run(chained_get_next.op) + for _ in range(num_iters): + start = time.time() + sess.run(chained_get_next.op) + end = time.time() + chained_deltas.append(end - start) + + fused_dataset = dataset.apply( + batching.map_and_batch( + math_ops.matmul, + num_parallel_calls=num_calls, + batch_size=batch_size)) + fused_iterator = dataset_ops.make_one_shot_iterator(fused_dataset) + fused_get_next = fused_iterator.get_next() + + fused_deltas = [] + with session.Session( + config=config_pb2.ConfigProto( + inter_op_parallelism_threads=inter_op, + use_per_session_threads=True)) as sess: + + for _ in range(5): + sess.run(fused_get_next.op) + for _ in range(num_iters): + start = time.time() + sess.run(fused_get_next.op) + end = time.time() + fused_deltas.append(end - start) + + print( + "batch size: %d, num parallel calls: %d, inter-op parallelism: %d, " + "element size: %d, num iters: %d\nchained wall time: %f (median), " + "%f (mean), %f (stddev), %f (min), %f (max)\n fused wall time: " + "%f (median), %f (mean), %f (stddev), %f (min), %f (max)\n " + "chained/fused: %.2fx (median), %.2fx (mean)" % + (batch_size, num_calls, inter_op, element_size, num_iters, + np.median(chained_deltas), np.mean(chained_deltas), + np.std(chained_deltas), np.min(chained_deltas), + np.max(chained_deltas), np.median(fused_deltas), + np.mean(fused_deltas), np.std(fused_deltas), np.min(fused_deltas), + np.max(fused_deltas), + np.median(chained_deltas) / np.median(fused_deltas), + np.mean(chained_deltas) / np.mean(fused_deltas))) + + self.report_benchmark( + iters=num_iters, + wall_time=np.median(chained_deltas), + name=name("chained", label, num_calls, inter_op, element_size, + batch_size)) + + self.report_benchmark( + iters=num_iters, + wall_time=np.median(fused_deltas), + name=name("fused", label, num_calls, inter_op, element_size, + batch_size)) + + print() + + np.random.seed(_NUMPY_RANDOM_SEED) + benchmark("Sequential element size evaluation", seq_elem_size_series) + benchmark("Sequential batch size evaluation", seq_batch_size_series) + benchmark("Parallel element size evaluation", par_elem_size_series) + benchmark("Parallel batch size evaluation", par_batch_size_series) + benchmark("Transformation parallelism evaluation", par_num_calls_series) + benchmark("Threadpool size evaluation", par_inter_op_series) + if __name__ == "__main__": test.main() diff --git a/tensorflow/python/data/experimental/benchmarks/map_benchmark.py b/tensorflow/python/data/experimental/benchmarks/map_benchmark.py deleted file mode 100644 index ad253cffa56..00000000000 --- a/tensorflow/python/data/experimental/benchmarks/map_benchmark.py +++ /dev/null @@ -1,245 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for the experimental input pipeline ops.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import hashlib -import itertools -import time - -import numpy as np - -from tensorflow.core.protobuf import config_pb2 -from tensorflow.python.client import session -from tensorflow.python.data.experimental.ops import batching -from tensorflow.python.data.experimental.ops import optimization -from tensorflow.python.data.ops import dataset_ops -from tensorflow.python.framework import ops -from tensorflow.python.ops import math_ops -from tensorflow.python.platform import test - -_NUMPY_RANDOM_SEED = 42 - - -class MapDatasetBenchmark(test.Benchmark): - - # The purpose of this benchmark is to compare the performance of chaining vs - # fusing of the map and batch transformations across various configurations. - # - # NOTE: It is recommended to build the benchmark with - # `-c opt --copt=-mavx --copt=-mavx2 --copt=-mfma --copt=-gmlt` - # and execute it on a machine with at least 32 CPU cores. - def benchmarkMapAndBatch(self): - - # Sequential pipeline configurations. - seq_elem_size_series = itertools.product([1], [1], [1, 2, 4, 8], [16]) - seq_batch_size_series = itertools.product([1], [1], [1], [8, 16, 32, 64]) - - # Parallel pipeline configuration. - par_elem_size_series = itertools.product([32], [32], [1, 2, 4, 8], [256]) - par_batch_size_series = itertools.product([32], [32], [1], - [128, 256, 512, 1024]) - par_num_calls_series = itertools.product([8, 16, 32, 64], [32], [1], [512]) - par_inter_op_series = itertools.product([32], [8, 16, 32, 64], [1], [512]) - - def name(method, label, num_calls, inter_op, element_size, batch_size): - return ("%s_id_%s_num_calls_%d_inter_op_%d_elem_size_%d_batch_size_%d" % ( - method, - hashlib.sha1(label).hexdigest(), - num_calls, - inter_op, - element_size, - batch_size, - )) - - def benchmark(label, series): - - print("%s:" % label) - for num_calls, inter_op, element_size, batch_size in series: - - num_iters = 1024 // ( - (element_size * batch_size) // min(num_calls, inter_op)) - k = 1024 * 1024 - dataset = dataset_ops.Dataset.from_tensors((np.random.rand( - element_size, 4 * k), np.random.rand(4 * k, 1))).repeat() - - chained_dataset = dataset.map( - math_ops.matmul, - num_parallel_calls=num_calls).batch(batch_size=batch_size) - chained_iterator = chained_dataset.make_one_shot_iterator() - chained_get_next = chained_iterator.get_next() - - chained_deltas = [] - with session.Session( - config=config_pb2.ConfigProto( - inter_op_parallelism_threads=inter_op, - use_per_session_threads=True)) as sess: - for _ in range(5): - sess.run(chained_get_next.op) - for _ in range(num_iters): - start = time.time() - sess.run(chained_get_next.op) - end = time.time() - chained_deltas.append(end - start) - - fused_dataset = dataset.apply( - batching.map_and_batch( - math_ops.matmul, - num_parallel_calls=num_calls, - batch_size=batch_size)) - fused_iterator = fused_dataset.make_one_shot_iterator() - fused_get_next = fused_iterator.get_next() - - fused_deltas = [] - with session.Session( - config=config_pb2.ConfigProto( - inter_op_parallelism_threads=inter_op, - use_per_session_threads=True)) as sess: - - for _ in range(5): - sess.run(fused_get_next.op) - for _ in range(num_iters): - start = time.time() - sess.run(fused_get_next.op) - end = time.time() - fused_deltas.append(end - start) - - print( - "batch size: %d, num parallel calls: %d, inter-op parallelism: %d, " - "element size: %d, num iters: %d\nchained wall time: %f (median), " - "%f (mean), %f (stddev), %f (min), %f (max)\n fused wall time: " - "%f (median), %f (mean), %f (stddev), %f (min), %f (max)\n " - "chained/fused: %.2fx (median), %.2fx (mean)" % - (batch_size, num_calls, inter_op, element_size, num_iters, - np.median(chained_deltas), np.mean(chained_deltas), - np.std(chained_deltas), np.min(chained_deltas), - np.max(chained_deltas), np.median(fused_deltas), - np.mean(fused_deltas), np.std(fused_deltas), np.min(fused_deltas), - np.max(fused_deltas), - np.median(chained_deltas) / np.median(fused_deltas), - np.mean(chained_deltas) / np.mean(fused_deltas))) - - self.report_benchmark( - iters=num_iters, - wall_time=np.median(chained_deltas), - name=name("chained", label, num_calls, inter_op, element_size, - batch_size)) - - self.report_benchmark( - iters=num_iters, - wall_time=np.median(fused_deltas), - name=name("fused", label, num_calls, inter_op, element_size, - batch_size)) - - print("") - - np.random.seed(_NUMPY_RANDOM_SEED) - benchmark("Sequential element size evaluation", seq_elem_size_series) - benchmark("Sequential batch size evaluation", seq_batch_size_series) - benchmark("Parallel element size evaluation", par_elem_size_series) - benchmark("Parallel batch size evaluation", par_batch_size_series) - benchmark("Transformation parallelism evaluation", par_num_calls_series) - benchmark("Threadpool size evaluation", par_inter_op_series) - - # This benchmark compares the performance of pipeline with multiple chained - # maps with and without map fusion. - def benchmarkChainOfMaps(self): - chain_lengths = [0, 1, 2, 5, 10, 20, 50] - for chain_length in chain_lengths: - self._benchmarkChainOfMaps(chain_length, False) - self._benchmarkChainOfMaps(chain_length, True) - - def _benchmarkChainOfMaps(self, chain_length, optimize_dataset): - with ops.Graph().as_default(): - dataset = dataset_ops.Dataset.from_tensors(0).repeat(None) - for _ in range(chain_length): - dataset = dataset.map(lambda x: x) - if optimize_dataset: - dataset = dataset.apply(optimization.optimize(["map_fusion"])) - - iterator = dataset.make_one_shot_iterator() - next_element = iterator.get_next() - - with session.Session() as sess: - for _ in range(5): - sess.run(next_element.op) - deltas = [] - for _ in range(100): - start = time.time() - for _ in range(100): - sess.run(next_element.op) - end = time.time() - deltas.append(end - start) - - median_wall_time = np.median(deltas) / 100 - opt_mark = "opt" if optimize_dataset else "no-opt" - print("Map dataset {} chain length: {} Median wall time: {}".format( - opt_mark, chain_length, median_wall_time)) - self.report_benchmark( - iters=1000, - wall_time=median_wall_time, - name="benchmark_map_dataset_chain_latency_{}_{}".format( - opt_mark, chain_length)) - - -class MapAndFilterBenchmark(test.Benchmark): - - # This benchmark compares the performance of pipeline with multiple chained - # map + filter with and without map fusion. - def benchmarkMapAndFilter(self): - chain_lengths = [0, 1, 2, 5, 10, 20, 50] - for chain_length in chain_lengths: - self._benchmarkMapAndFilter(chain_length, False) - self._benchmarkMapAndFilter(chain_length, True) - - def _benchmarkMapAndFilter(self, chain_length, optimize_dataset): - with ops.Graph().as_default(): - dataset = dataset_ops.Dataset.from_tensors(0).repeat(None) - for _ in range(chain_length): - dataset = dataset.map(lambda x: x + 5).filter( - lambda x: math_ops.greater_equal(x - 5, 0)) - if optimize_dataset: - dataset = dataset.apply( - optimization.optimize(["map_and_filter_fusion"])) - - iterator = dataset.make_one_shot_iterator() - next_element = iterator.get_next() - - with session.Session() as sess: - for _ in range(10): - sess.run(next_element.op) - deltas = [] - for _ in range(100): - start = time.time() - for _ in range(100): - sess.run(next_element.op) - end = time.time() - deltas.append(end - start) - - median_wall_time = np.median(deltas) / 100 - opt_mark = "opt" if optimize_dataset else "no-opt" - print("Map and filter dataset {} chain length: {} Median wall time: {}". - format(opt_mark, chain_length, median_wall_time)) - self.report_benchmark( - iters=1000, - wall_time=median_wall_time, - name="benchmark_map_and_filter_dataset_chain_latency_{}_{}".format( - opt_mark, chain_length)) - - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/python/data/experimental/benchmarks/map_vectorization_benchmark.py b/tensorflow/python/data/experimental/benchmarks/map_vectorization_benchmark.py new file mode 100644 index 00000000000..47ec6391f78 --- /dev/null +++ b/tensorflow/python/data/experimental/benchmarks/map_vectorization_benchmark.py @@ -0,0 +1,194 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Benchmarks for the `MapVectorization` optimization.""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import time + +import numpy as np + +from tensorflow.core.example import example_pb2 +from tensorflow.core.example import feature_pb2 +from tensorflow.python.client import session +from tensorflow.python.data.ops import dataset_ops +from tensorflow.python.data.util import nest +from tensorflow.python.framework import constant_op +from tensorflow.python.framework import dtypes +from tensorflow.python.ops import array_ops +from tensorflow.python.ops import math_ops +from tensorflow.python.ops import parsing_ops +from tensorflow.python.platform import test + + +def _generate_csv_test_case(): + """Generates a `decode_csv()` test case.""" + + def csv_factory(): + return dataset_ops.Dataset.from_tensor_slices(["1.0:2:a", + "2.4:5:c"]).repeat(5) + + def decode_csv_fn(x): + return parsing_ops.decode_csv( + x, + record_defaults=[ + constant_op.constant([], dtypes.float32), + constant_op.constant([], dtypes.int32), + constant_op.constant([], dtypes.string) + ], + field_delim=":") + + return decode_csv_fn, csv_factory + + +def _generate_parse_single_example_test_case(): + """Generates a `parse_single_example()` test case.""" + + def parse_example_factory(): + """Parse example factory.""" + + def _int64_feature(*values): + return feature_pb2.Feature(int64_list=feature_pb2.Int64List(value=values)) + + def _bytes_feature(*values): + return feature_pb2.Feature( + bytes_list=feature_pb2.BytesList( + value=[v.encode("utf-8") for v in values])) + + return dataset_ops.Dataset.from_tensor_slices( + constant_op.constant([ + example_pb2.Example( + features=feature_pb2.Features( + feature={ + "dense_int": _int64_feature(i), + "dense_str": _bytes_feature(str(i)), + "sparse_int": _int64_feature(i, i * 2, i * 4, i * 8), + "sparse_str": _bytes_feature(*["abc"] * i) + })).SerializeToString() for i in range(10) + ])) + + def parse_single_example_fn(x): + features = { + "dense_int": parsing_ops.FixedLenFeature((), dtypes.int64, 0), + "dense_str": parsing_ops.FixedLenFeature((), dtypes.string, ""), + "sparse_int": parsing_ops.VarLenFeature(dtypes.int64), + "sparse_str": parsing_ops.VarLenFeature(dtypes.string), + } + return parsing_ops.parse_single_example(x, features) + + return parse_single_example_fn, parse_example_factory + + +# TODO(rachelim): Add a benchmark for more expensive transformations, such as +# vgg_preprocessing. +class MapVectorizationBenchmark(test.Benchmark): + """Benchmarks for the `MapVectorization` optimization.""" + + def _run(self, x, num_iters=100, name=None): + deltas = [] + with session.Session() as sess: + for _ in range(5): + # Warm up session... + sess.run(x) + for _ in range(num_iters): + start = time.time() + sess.run(x) + end = time.time() + deltas.append(end - start) + median_time = np.median(deltas) + self.report_benchmark(iters=num_iters, wall_time=median_time, name=name) + return median_time + + def _compare(self, input_dataset, map_fn, batch_size, input_size, str_id): + num_elems = int(np.sum([np.prod(x) for x in input_size])) + name_template = "{}__batch_size_{}_input_element_size_{}_{}" + unoptimized = input_dataset.map(map_fn).batch(batch_size) + unoptimized_op = dataset_ops.make_one_shot_iterator(unoptimized).get_next() + + optimized = input_dataset.map(map_fn).batch(batch_size) + options = dataset_ops.Options() + options.experimental_map_vectorization = True + optimized = optimized.with_options(options) + optimized_op = dataset_ops.make_one_shot_iterator(optimized).get_next() + + unoptimized_time = self._run( + unoptimized_op, + name=name_template.format(str_id, batch_size, num_elems, "unoptimized")) + optimized_time = self._run( + optimized_op, + name=name_template.format(str_id, batch_size, num_elems, "optimized")) + + print("Batch size: {}\n" + "Input element size: {}\n" + "Transformation: {}\n" + "Speedup: {}\n".format(batch_size, input_size, str_id, + (unoptimized_time / optimized_time))) + + # Known cheap functions + def benchmarkIdentity(self): + self._benchmark_helper(lambda *args: [array_ops.identity(x) for x in args], + "identity") + + def benchmarkAddConst(self): + self._benchmark_helper(lambda *args: [x + 1 for x in args], "add_const") + + def benchmarkReturnConst(self): + self._benchmark_helper(lambda *args: [constant_op.constant(2)], "ret_const") + + def benchmarkSelect(self): + self._benchmark_helper(lambda *args: args[0], "select") + + def benchmarkCast(self): + self._benchmark_helper( + lambda *args: [math_ops.cast(x, dtypes.float64) for x in args], "cast") + + def benchmarkReshape(self): + self._benchmark_helper( + lambda *args: [array_ops.reshape(x, (-1, 30)) for x in args], "reshape") + + def benchmarkDecodeCSV(self): + csv_fn, csv_factory = _generate_csv_test_case() + self._benchmark_helper(csv_fn, "decode_csv", lambda: [csv_factory()]) + + def benchmarkParseSingleExample(self): + # NOTE: Since we haven't implemented a vectorizer for "SerializeSparse", + # this function is only naively vectorized. + parse_fn, parse_factory = _generate_parse_single_example_test_case() + + self._benchmark_helper(parse_fn, "parse_single_example", + lambda: [parse_factory()]) + + def _default_dataset_factory(self): + input_sizes = [(10, 10, 3), (10, 100, 300)] + for sz in input_sizes: + yield dataset_ops.Dataset.from_tensor_slices(np.random.rand(*sz)) + + def _benchmark_helper(self, map_fn, str_id, base_dataset_factory=None): + if base_dataset_factory is None: + base_dataset_factory = self._default_dataset_factory + + batch_size = 1000 + for base_dataset in base_dataset_factory(): + base_dataset = base_dataset.repeat() + input_size = [ + tuple(shape.as_list()) + for shape in nest.flatten(base_dataset.output_shapes) + ] + self._compare(base_dataset, map_fn, batch_size, input_size, str_id) + + +if __name__ == "__main__": + test.main() diff --git a/tensorflow/python/data/experimental/benchmarks/matching_files_benchmark.py b/tensorflow/python/data/experimental/benchmarks/matching_files_benchmark.py new file mode 100644 index 00000000000..c53f8dd7c53 --- /dev/null +++ b/tensorflow/python/data/experimental/benchmarks/matching_files_benchmark.py @@ -0,0 +1,102 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Benchmark for the experimental `MatchingFilesDataset`.""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import os +import shutil +import tempfile +import time + +import numpy as np + +from tensorflow.python.client import session +from tensorflow.python.data.experimental.ops import matching_files +from tensorflow.python.data.ops import dataset_ops +from tensorflow.python.framework import errors +from tensorflow.python.framework import ops +from tensorflow.python.platform import test + + +class MatchingFilesBenchmark(test.Benchmark): + """Benchmark for the experimental `MatchingFilesDataset`.""" + + def benchmarkNestedDirectories(self): + tmp_dir = tempfile.mkdtemp() + width = 500 + depth = 10 + for i in range(width): + for j in range(depth): + new_base = os.path.join(tmp_dir, str(i), + *[str(dir_name) for dir_name in range(j)]) + os.makedirs(new_base) + child_files = ['a.py', 'b.pyc'] if j < depth - 1 else ['c.txt', 'd.log'] + for f in child_files: + filename = os.path.join(new_base, f) + open(filename, 'w').close() + + patterns = [ + os.path.join(tmp_dir, os.path.join(*['**' + for _ in range(depth)]), suffix) + for suffix in ['*.txt', '*.log'] + ] + + deltas = [] + iters = 3 + for _ in range(iters): + with ops.Graph().as_default(): + dataset = matching_files.MatchingFilesDataset(patterns) + next_element = dataset_ops.make_one_shot_iterator(dataset).get_next() + + with session.Session() as sess: + sub_deltas = [] + while True: + try: + start = time.time() + sess.run(next_element) + end = time.time() + sub_deltas.append(end - start) + except errors.OutOfRangeError: + break + deltas.append(sub_deltas) + + median_deltas = np.median(deltas, axis=0) + print('Nested directory size (width*depth): %d*%d Median wall time: ' + '%fs (read first filename), %fs (read second filename), avg %fs' + ' (read %d more filenames)' % + (width, depth, median_deltas[0], median_deltas[1], + np.average(median_deltas[2:]), len(median_deltas) - 2)) + self.report_benchmark( + iters=iters, + wall_time=np.sum(median_deltas), + extras={ + 'read first file:': + median_deltas[0], + 'read second file:': + median_deltas[1], + 'avg time for reading %d more filenames:' % + (len(median_deltas) - 2): + np.average(median_deltas[2:]) + }, + name='dataset_nested_directory(%d*%d)' % + (width, depth)) + + shutil.rmtree(tmp_dir, ignore_errors=True) + + +if __name__ == '__main__': + test.main() diff --git a/tensorflow/python/data/experimental/benchmarks/optimize_benchmark.py b/tensorflow/python/data/experimental/benchmarks/optimize_benchmark.py new file mode 100644 index 00000000000..2f9b89111fc --- /dev/null +++ b/tensorflow/python/data/experimental/benchmarks/optimize_benchmark.py @@ -0,0 +1,120 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Benchmarks for static optimizations.""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import time + +import numpy as np + +from tensorflow.python.client import session +from tensorflow.python.data.ops import dataset_ops +from tensorflow.python.framework import ops +from tensorflow.python.ops import math_ops +from tensorflow.python.platform import test + + +class OptimizationBenchmark(test.Benchmark): + """Benchmarks for static optimizations.""" + + def benchmarkMapFusion(self): + """Evaluates performance map of fusion.""" + + chain_lengths = [0, 1, 2, 5, 10, 20, 50] + for chain_length in chain_lengths: + self._benchmarkMapFusion(chain_length, False) + self._benchmarkMapFusion(chain_length, True) + + def _benchmarkMapFusion(self, chain_length, optimize_dataset): + with ops.Graph().as_default(): + dataset = dataset_ops.Dataset.from_tensors(0).repeat(None) + for _ in range(chain_length): + dataset = dataset.map(lambda x: x) + if optimize_dataset: + options = dataset_ops.Options() + options.experimental_map_fusion = True + dataset = dataset.with_options(options) + + iterator = dataset_ops.make_one_shot_iterator(dataset) + next_element = iterator.get_next() + + with session.Session() as sess: + for _ in range(5): + sess.run(next_element.op) + deltas = [] + for _ in range(100): + start = time.time() + for _ in range(100): + sess.run(next_element.op) + end = time.time() + deltas.append(end - start) + + median_wall_time = np.median(deltas) / 100 + opt_mark = "opt" if optimize_dataset else "noopt" + print("Map dataset {} chain length: {} Median wall time: {}".format( + opt_mark, chain_length, median_wall_time)) + self.report_benchmark( + iters=100, + wall_time=median_wall_time, + name="map_fusion_{}_chain_length_{}".format( + opt_mark, chain_length)) + + def benchmarkMapAndFilterFusion(self): + """Evaluates performance map of fusion.""" + + chain_lengths = [0, 1, 2, 5, 10, 20, 50] + for chain_length in chain_lengths: + self._benchmarkMapAndFilterFusion(chain_length, False) + self._benchmarkMapAndFilterFusion(chain_length, True) + + def _benchmarkMapAndFilterFusion(self, chain_length, optimize_dataset): + with ops.Graph().as_default(): + dataset = dataset_ops.Dataset.from_tensors(0).repeat(None) + for _ in range(chain_length): + dataset = dataset.map(lambda x: x + 5).filter( + lambda x: math_ops.greater_equal(x - 5, 0)) + if optimize_dataset: + options = dataset_ops.Options() + options.experimental_map_and_filter_fusion = True + dataset = dataset.with_options(options) + iterator = dataset_ops.make_one_shot_iterator(dataset) + next_element = iterator.get_next() + + with session.Session() as sess: + for _ in range(10): + sess.run(next_element.op) + deltas = [] + for _ in range(100): + start = time.time() + for _ in range(100): + sess.run(next_element.op) + end = time.time() + deltas.append(end - start) + + median_wall_time = np.median(deltas) / 100 + opt_mark = "opt" if optimize_dataset else "noopt" + print("Map and filter dataset {} chain length: {} Median wall time: {}" + .format(opt_mark, chain_length, median_wall_time)) + self.report_benchmark( + iters=100, + wall_time=median_wall_time, + name="map_and_filter_fusion_{}_chain_length_{}".format( + opt_mark, chain_length)) + + +if __name__ == "__main__": + test.main() diff --git a/tensorflow/python/data/experimental/benchmarks/unbatch_benchmark.py b/tensorflow/python/data/experimental/benchmarks/unbatch_benchmark.py new file mode 100644 index 00000000000..c36a32534dd --- /dev/null +++ b/tensorflow/python/data/experimental/benchmarks/unbatch_benchmark.py @@ -0,0 +1,107 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Tests for `tf.data.experimental.unbatch()`.""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import time + +import numpy as np + +from tensorflow.python.client import session +from tensorflow.python.data.experimental.ops import batching +from tensorflow.python.data.ops import dataset_ops +from tensorflow.python.framework import dtypes +from tensorflow.python.framework import ops +from tensorflow.python.ops import array_ops +from tensorflow.python.platform import test + + +class UnbatchBenchmark(test.Benchmark): + """Benchmarks for `tf.data.experimental.unbatch()`.""" + + def benchmarkNativeUnbatch(self): + batch_sizes = [1, 2, 5, 10, 20, 50] + elems_per_trial = 10000 + with ops.Graph().as_default(): + dataset = dataset_ops.Dataset.from_tensors("element").repeat(None) + batch_size_placeholder = array_ops.placeholder(dtypes.int64, shape=[]) + dataset = dataset.batch(batch_size_placeholder) + dataset = dataset.apply(batching.unbatch()) + dataset = dataset.skip(elems_per_trial) + iterator = dataset_ops.make_initializable_iterator(dataset) + next_element = iterator.get_next() + + with session.Session() as sess: + for batch_size in batch_sizes: + deltas = [] + for _ in range(5): + sess.run( + iterator.initializer, + feed_dict={batch_size_placeholder: batch_size}) + start = time.time() + sess.run(next_element.op) + end = time.time() + deltas.append((end - start) / elems_per_trial) + + median_wall_time = np.median(deltas) + print("Unbatch (native) batch size: %d Median wall time per element:" + " %f microseconds" % (batch_size, median_wall_time * 1e6)) + self.report_benchmark( + iters=10000, + wall_time=median_wall_time, + name="native_batch_size_%d" % + batch_size) + + # Include a benchmark of the previous `unbatch()` implementation that uses + # a composition of more primitive ops. Eventually we'd hope to generate code + # that is as good in both cases. + def benchmarkOldUnbatchImplementation(self): + batch_sizes = [1, 2, 5, 10, 20, 50] + elems_per_trial = 10000 + with ops.Graph().as_default(): + dataset = dataset_ops.Dataset.from_tensors("element").repeat(None) + batch_size_placeholder = array_ops.placeholder(dtypes.int64, shape=[]) + dataset = dataset.batch(batch_size_placeholder) + dataset = dataset.flat_map(dataset_ops.Dataset.from_tensor_slices) + dataset = dataset.skip(elems_per_trial) + iterator = dataset_ops.make_initializable_iterator(dataset) + next_element = iterator.get_next() + + with session.Session() as sess: + for batch_size in batch_sizes: + deltas = [] + for _ in range(5): + sess.run( + iterator.initializer, + feed_dict={batch_size_placeholder: batch_size}) + start = time.time() + sess.run(next_element.op) + end = time.time() + deltas.append((end - start) / elems_per_trial) + + median_wall_time = np.median(deltas) + print("Unbatch (unfused) batch size: %d Median wall time per element:" + " %f microseconds" % (batch_size, median_wall_time * 1e6)) + self.report_benchmark( + iters=10000, + wall_time=median_wall_time, + name="unfused_batch_size_%d" % + batch_size) + + +if __name__ == "__main__": + test.main() diff --git a/tensorflow/python/data/experimental/kernel_tests/BUILD b/tensorflow/python/data/experimental/kernel_tests/BUILD index c9b11a2c381..c76e576b5b4 100644 --- a/tensorflow/python/data/experimental/kernel_tests/BUILD +++ b/tensorflow/python/data/experimental/kernel_tests/BUILD @@ -72,15 +72,11 @@ py_test( "//tensorflow/python:errors", "//tensorflow/python:framework_test_lib", "//tensorflow/python:parsing_ops", - "//tensorflow/python:platform", - "//tensorflow/python:platform_test", - "//tensorflow/python:session", "//tensorflow/python/data/experimental/ops:error_ops", "//tensorflow/python/data/experimental/ops:readers", "//tensorflow/python/data/kernel_tests:test_base", "//tensorflow/python/data/ops:readers", "//tensorflow/python/eager:context", - "//third_party/py/numpy", ], ) @@ -153,27 +149,6 @@ py_test( ], ) -cuda_py_test( - name = "function_buffering_resource_test", - size = "small", - srcs = ["function_buffering_resource_test.py"], - additional_deps = [ - "//tensorflow/python/data/experimental/ops:prefetching_ops", - "//tensorflow/core:protos_all_py", - "//tensorflow/python:client_testlib", - "//tensorflow/python:constant_op", - "//tensorflow/python/data/kernel_tests:test_base", - "//tensorflow/python/eager:function", - "//tensorflow/python:dtypes", - "//tensorflow/python:framework_ops", - "//tensorflow/python:framework_test_lib", - "//tensorflow/python:resource_variable_ops", - "//tensorflow/python/data/ops:dataset_ops", - "//tensorflow/python/data/ops:iterator_ops", - ], - tags = ["no_windows_gpu"], -) - py_test( name = "get_single_element_test", size = "small", @@ -371,6 +346,37 @@ py_test( ], ) +py_test( + name = "matching_files_test", + size = "small", + srcs = ["matching_files_test.py"], + srcs_version = "PY2AND3", + tags = ["no_pip"], + deps = [ + "//tensorflow/python:array_ops", + "//tensorflow/python:client_testlib", + "//tensorflow/python:dtypes", + "//tensorflow/python:errors", + "//tensorflow/python:util", + "//tensorflow/python/data/experimental/ops:matching_files", + "//tensorflow/python/data/kernel_tests:test_base", + "//tensorflow/python/data/ops:dataset_ops", + "//third_party/py/numpy", + ], +) + +py_test( + name = "cardinality_test", + srcs = ["cardinality_test.py"], + srcs_version = "PY2AND3", + deps = [ + "//tensorflow/python/data/experimental/ops:cardinality", + "//tensorflow/python/data/kernel_tests:test_base", + "//tensorflow/python/data/ops:dataset_ops", + "@absl_py//absl/testing:parameterized", + ], +) + py_test( name = "override_threadpool_test", size = "small", @@ -618,7 +624,9 @@ py_test( size = "medium", srcs = ["stats_dataset_ops_test.py"], srcs_version = "PY2AND3", - tags = ["no_pip"], + tags = [ + "no_pip", + ], deps = [ ":reader_dataset_ops_test_base", ":stats_dataset_test_base", diff --git a/tensorflow/python/data/experimental/kernel_tests/batch_dataset_op_test.py b/tensorflow/python/data/experimental/kernel_tests/batch_dataset_op_test.py deleted file mode 100644 index e896752a269..00000000000 --- a/tensorflow/python/data/experimental/kernel_tests/batch_dataset_op_test.py +++ /dev/null @@ -1,688 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for the experimental input pipeline ops.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import math -import time - -from absl.testing import parameterized -import numpy as np - -from tensorflow.python.client import session -from tensorflow.python.data.experimental.ops import batching -from tensorflow.python.data.kernel_tests import test_base -from tensorflow.python.data.ops import dataset_ops -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import errors -from tensorflow.python.framework import ops -from tensorflow.python.framework import sparse_tensor -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import script_ops -from tensorflow.python.ops import string_ops -from tensorflow.python.platform import test -from tensorflow.python.util import compat - - -class BatchDatasetTest(test_base.DatasetTestBase, parameterized.TestCase): - - def testDenseToSparseBatchDataset(self): - components = np.random.randint(12, size=(100,)).astype(np.int32) - iterator = ( - dataset_ops.Dataset.from_tensor_slices(components) - .map(lambda x: array_ops.fill([x], x)).apply( - batching.dense_to_sparse_batch(4, - [12])).make_initializable_iterator()) - init_op = iterator.initializer - get_next = iterator.get_next() - - with self.cached_session() as sess: - sess.run(init_op) - - for start in range(0, len(components), 4): - results = sess.run(get_next) - self.assertAllEqual([[i, j] - for i, c in enumerate(components[start:start + 4]) - for j in range(c)], results.indices) - self.assertAllEqual( - [c for c in components[start:start + 4] for _ in range(c)], - results.values) - self.assertAllEqual([min(4, - len(components) - start), 12], - results.dense_shape) - - with self.assertRaises(errors.OutOfRangeError): - sess.run(get_next) - - def testDenseToSparseBatchDatasetWithUnknownShape(self): - components = np.random.randint(5, size=(40,)).astype(np.int32) - iterator = ( - dataset_ops.Dataset.from_tensor_slices(components).map( - lambda x: array_ops.fill([x, x], x)).apply( - batching.dense_to_sparse_batch( - 4, [5, None])).make_initializable_iterator()) - init_op = iterator.initializer - get_next = iterator.get_next() - - with self.cached_session() as sess: - sess.run(init_op) - - for start in range(0, len(components), 4): - results = sess.run(get_next) - self.assertAllEqual([[i, j, z] - for i, c in enumerate(components[start:start + 4]) - for j in range(c) - for z in range(c)], results.indices) - self.assertAllEqual([ - c for c in components[start:start + 4] for _ in range(c) - for _ in range(c) - ], results.values) - self.assertAllEqual([ - min(4, - len(components) - start), 5, - np.max(components[start:start + 4]) - ], results.dense_shape) - - with self.assertRaises(errors.OutOfRangeError): - sess.run(get_next) - - def testDenseToSparseBatchDatasetWithInvalidShape(self): - input_tensor = array_ops.constant([[1]]) - with self.assertRaisesRegexp(ValueError, "Dimension -2 must be >= 0"): - dataset_ops.Dataset.from_tensors(input_tensor).apply( - batching.dense_to_sparse_batch(4, - [-2])).make_initializable_iterator() - - def testDenseToSparseBatchDatasetShapeErrors(self): - input_tensor = array_ops.placeholder(dtypes.int32) - iterator = ( - dataset_ops.Dataset.from_tensors(input_tensor).apply( - batching.dense_to_sparse_batch(4, - [12])).make_initializable_iterator()) - init_op = iterator.initializer - get_next = iterator.get_next() - - with self.cached_session() as sess: - # Initialize with an input tensor of incompatible rank. - sess.run(init_op, feed_dict={input_tensor: [[1]]}) - with self.assertRaisesRegexp(errors.InvalidArgumentError, - "incompatible with the row shape"): - sess.run(get_next) - - # Initialize with an input tensor that is larger than `row_shape`. - sess.run(init_op, feed_dict={input_tensor: range(13)}) - with self.assertRaisesRegexp(errors.DataLossError, - "larger than the row shape"): - sess.run(get_next) - - def testUnbatchWithUnknownRankInput(self): - placeholder = array_ops.placeholder(dtypes.int32) - dataset = dataset_ops.Dataset.from_tensors(placeholder).apply( - batching.unbatch()) - iterator = dataset.make_initializable_iterator() - next_elem = iterator.get_next() - - with self.cached_session() as sess: - sess.run(iterator.initializer, feed_dict={placeholder: [0, 1, 2, 3]}) - for i in range(4): - self.assertEqual(i, sess.run(next_elem)) - with self.assertRaises(errors.OutOfRangeError): - sess.run(next_elem) - - def testUnbatchScalarDataset(self): - data = tuple([math_ops.range(10) for _ in range(3)]) - data = dataset_ops.Dataset.from_tensor_slices(data) - expected_types = (dtypes.int32,) * 3 - data = data.batch(2) - self.assertEqual(expected_types, data.output_types) - data = data.apply(batching.unbatch()) - self.assertEqual(expected_types, data.output_types) - - iterator = data.make_one_shot_iterator() - op = iterator.get_next() - - with self.cached_session() as sess: - for i in range(10): - self.assertEqual((i,) * 3, sess.run(op)) - - with self.assertRaises(errors.OutOfRangeError): - sess.run(op) - - def testUnbatchDatasetWithStrings(self): - data = tuple([math_ops.range(10) for _ in range(3)]) - data = dataset_ops.Dataset.from_tensor_slices(data) - data = data.map(lambda x, y, z: (x, string_ops.as_string(y), z)) - expected_types = (dtypes.int32, dtypes.string, dtypes.int32) - data = data.batch(2) - self.assertEqual(expected_types, data.output_types) - data = data.apply(batching.unbatch()) - self.assertEqual(expected_types, data.output_types) - - iterator = data.make_one_shot_iterator() - op = iterator.get_next() - - with self.cached_session() as sess: - for i in range(10): - self.assertEqual((i, compat.as_bytes(str(i)), i), sess.run(op)) - - with self.assertRaises(errors.OutOfRangeError): - sess.run(op) - - def testUnbatchDatasetWithSparseTensor(self): - st = sparse_tensor.SparseTensorValue( - indices=[[i, i] for i in range(10)], - values=list(range(10)), - dense_shape=[10, 10]) - data = dataset_ops.Dataset.from_tensors(st) - data = data.apply(batching.unbatch()) - data = data.batch(5) - data = data.apply(batching.unbatch()) - iterator = data.make_one_shot_iterator() - next_element = iterator.get_next() - - with self.cached_session() as sess: - for i in range(10): - st_row = sess.run(next_element) - self.assertEqual([i], st_row.indices) - self.assertEqual([i], st_row.values) - self.assertEqual([10], st_row.dense_shape) - with self.assertRaises(errors.OutOfRangeError): - sess.run(next_element) - - def testUnbatchDatasetWithDenseAndSparseTensor(self): - st = sparse_tensor.SparseTensorValue( - indices=[[i, i] for i in range(10)], - values=list(range(10)), - dense_shape=[10, 10]) - data = dataset_ops.Dataset.from_tensors((list(range(10)), st)) - data = data.apply(batching.unbatch()) - data = data.batch(5) - data = data.apply(batching.unbatch()) - iterator = data.make_one_shot_iterator() - next_element = iterator.get_next() - - with self.cached_session() as sess: - for i in range(10): - dense_elem, st_row = sess.run(next_element) - self.assertEqual(i, dense_elem) - self.assertEqual([i], st_row.indices) - self.assertEqual([i], st_row.values) - self.assertEqual([10], st_row.dense_shape) - with self.assertRaises(errors.OutOfRangeError): - sess.run(next_element) - - def testUnbatchSingleElementTupleDataset(self): - data = tuple([(math_ops.range(10),) for _ in range(3)]) - data = dataset_ops.Dataset.from_tensor_slices(data) - expected_types = ((dtypes.int32,),) * 3 - data = data.batch(2) - self.assertEqual(expected_types, data.output_types) - data = data.apply(batching.unbatch()) - self.assertEqual(expected_types, data.output_types) - - iterator = data.make_one_shot_iterator() - op = iterator.get_next() - - with self.cached_session() as sess: - for i in range(10): - self.assertEqual(((i,),) * 3, sess.run(op)) - - with self.assertRaises(errors.OutOfRangeError): - sess.run(op) - - def testUnbatchMultiElementTupleDataset(self): - data = tuple([(math_ops.range(10 * i, 10 * i + 10), - array_ops.fill([10], "hi")) for i in range(3)]) - data = dataset_ops.Dataset.from_tensor_slices(data) - expected_types = ((dtypes.int32, dtypes.string),) * 3 - data = data.batch(2) - self.assertAllEqual(expected_types, data.output_types) - data = data.apply(batching.unbatch()) - self.assertAllEqual(expected_types, data.output_types) - - iterator = data.make_one_shot_iterator() - op = iterator.get_next() - - with self.cached_session() as sess: - for i in range(10): - self.assertEqual(((i, b"hi"), (10 + i, b"hi"), (20 + i, b"hi")), - sess.run(op)) - - with self.assertRaises(errors.OutOfRangeError): - sess.run(op) - - def testUnbatchEmpty(self): - data = dataset_ops.Dataset.from_tensors( - (constant_op.constant([]), constant_op.constant([], shape=[0, 4]), - constant_op.constant([], shape=[0, 4, 0]))) - data = data.apply(batching.unbatch()) - iterator = data.make_one_shot_iterator() - next_element = iterator.get_next() - - with self.cached_session() as sess: - with self.assertRaises(errors.OutOfRangeError): - sess.run(next_element) - - def testUnbatchStaticShapeMismatch(self): - data = dataset_ops.Dataset.from_tensors((np.arange(7), np.arange(8), - np.arange(9))) - with self.assertRaises(ValueError): - data.apply(batching.unbatch()) - - def testUnbatchDynamicShapeMismatch(self): - ph1 = array_ops.placeholder(dtypes.int32, shape=[None]) - ph2 = array_ops.placeholder(dtypes.int32, shape=None) - data = dataset_ops.Dataset.from_tensors((ph1, ph2)) - data = data.apply(batching.unbatch()) - iterator = data.make_initializable_iterator() - next_element = iterator.get_next() - - with self.cached_session() as sess: - # Mismatch in the 0th dimension. - sess.run( - iterator.initializer, - feed_dict={ - ph1: np.arange(7).astype(np.int32), - ph2: np.arange(8).astype(np.int32) - }) - with self.assertRaises(errors.InvalidArgumentError): - sess.run(next_element) - - # No 0th dimension (i.e. scalar value) for one component. - sess.run( - iterator.initializer, - feed_dict={ - ph1: np.arange(7).astype(np.int32), - ph2: 7 - }) - with self.assertRaises(errors.InvalidArgumentError): - sess.run(next_element) - - @parameterized.named_parameters( - ("Default", None, None), - ("SequentialCalls", 1, None), - ("ParallelCalls", 2, None), - ("ParallelBatches", None, 10), - ) - def testMapAndBatch(self, num_parallel_calls, num_parallel_batches): - """Test a dataset that maps a TF function across its input elements.""" - # The pipeline is TensorSliceDataset -> - # RepeatDataset(count) -> MapAndBatchDataset(square_3, batch_size). - components = (np.arange(7), - np.array([[1, 2, 3]]) * np.arange(7)[:, np.newaxis], - np.array(37.0) * np.arange(7)) - - count = array_ops.placeholder(dtypes.int64, shape=[]) - batch_size = array_ops.placeholder(dtypes.int64, shape=[]) - - def _map_fn(x, y, z): - return math_ops.square(x), math_ops.square(y), math_ops.square(z) - - iterator = ( - dataset_ops.Dataset.from_tensor_slices(components).repeat(count).apply( - batching.map_and_batch( - map_func=_map_fn, - batch_size=batch_size, - num_parallel_calls=num_parallel_calls, - num_parallel_batches=num_parallel_batches)) - .make_initializable_iterator()) - init_op = iterator.initializer - get_next = iterator.get_next() - - self.assertEqual([[None] + list(c.shape[1:]) for c in components], - [t.shape.as_list() for t in get_next]) - - with self.cached_session() as sess: - # Batch of a finite input, where the batch_size divides the - # total number of elements. - sess.run(init_op, feed_dict={count: 28, batch_size: 14}) - num_batches = (28 * 7) // 14 - for i in range(num_batches): - result = sess.run(get_next) - for component, result_component in zip(components, result): - for j in range(14): - self.assertAllEqual(component[(i * 14 + j) % 7]**2, - result_component[j]) - with self.assertRaises(errors.OutOfRangeError): - sess.run(get_next) - - # Batch of a finite input, where the batch_size does not - # divide the total number of elements. - sess.run(init_op, feed_dict={count: 14, batch_size: 8}) - - # We expect (num_batches - 1) full-sized batches. - num_batches = int(math.ceil((14 * 7) / 8)) - for i in range(num_batches - 1): - result = sess.run(get_next) - for component, result_component in zip(components, result): - for j in range(8): - self.assertAllEqual(component[(i * 8 + j) % 7]**2, - result_component[j]) - result = sess.run(get_next) - for component, result_component in zip(components, result): - for j in range((14 * 7) % 8): - self.assertAllEqual(component[((num_batches - 1) * 8 + j) % 7]**2, - result_component[j]) - with self.assertRaises(errors.OutOfRangeError): - sess.run(get_next) - - # Batch of an empty input should fail straight away. - sess.run(init_op, feed_dict={count: 0, batch_size: 8}) - with self.assertRaises(errors.OutOfRangeError): - sess.run(get_next) - - # Empty batch should be an initialization time error. - with self.assertRaises(errors.InvalidArgumentError): - sess.run(init_op, feed_dict={count: 14, batch_size: 0}) - - @parameterized.named_parameters( - ("Even", False), - ("Uneven", True), - ) - def testMapAndBatchPartialBatch(self, drop_remainder): - iterator = ( - dataset_ops.Dataset.range(10).apply( - batching.map_and_batch( - lambda x: array_ops.reshape(x * x, [1]), - batch_size=4, - drop_remainder=drop_remainder)).make_one_shot_iterator()) - if drop_remainder: - self.assertEqual([4, 1], iterator.output_shapes.as_list()) - else: - self.assertEqual([None, 1], iterator.output_shapes.as_list()) - next_element = iterator.get_next() - with self.cached_session() as sess: - self.assertAllEqual([[0], [1], [4], [9]], sess.run(next_element)) - self.assertAllEqual([[16], [25], [36], [49]], sess.run(next_element)) - if not drop_remainder: - self.assertAllEqual([[64], [81]], sess.run(next_element)) - with self.assertRaises(errors.OutOfRangeError): - sess.run(next_element) - - def testMapAndBatchYieldsPartialBatch(self): - iterator = ( - dataset_ops.Dataset.range(10).apply( - batching.map_and_batch(lambda x: array_ops.reshape(x * x, [1]), - 4)).make_one_shot_iterator()) - self.assertEqual([None, 1], iterator.output_shapes.as_list()) - next_element = iterator.get_next() - with self.cached_session() as sess: - self.assertAllEqual([[0], [1], [4], [9]], sess.run(next_element)) - self.assertAllEqual([[16], [25], [36], [49]], sess.run(next_element)) - self.assertAllEqual([[64], [81]], sess.run(next_element)) - with self.assertRaises(errors.OutOfRangeError): - sess.run(next_element) - - def testMapAndBatchParallelGetNext(self): - iterator = ( - dataset_ops.Dataset.range(50000).apply( - batching.map_and_batch(lambda x: x, - batch_size=100)).make_one_shot_iterator()) - elements = [] - for _ in range(100): - elements.append(iterator.get_next()) - with self.cached_session() as sess: - for i in range(5): - got = sess.run(elements) - got.sort(key=lambda x: x[0]) - expected = [] - for j in range(100): - expected.append(range(i * 10000 + j * 100, i * 10000 + (j + 1) * 100)) - self.assertAllEqual(got, expected) - with self.assertRaises(errors.OutOfRangeError): - sess.run(elements) - - def testMapAndBatchParallelGetNextDropRemainder(self): - iterator = ( - dataset_ops.Dataset.range(49999).apply( - batching.map_and_batch( - lambda x: x, batch_size=100, - drop_remainder=True)).make_one_shot_iterator()) - elements = [] - for _ in range(100): - elements.append(iterator.get_next()) - with self.cached_session() as sess: - for i in range(4): - got = sess.run(elements) - got.sort(key=lambda x: x[0]) - expected = [] - for j in range(100): - expected.append(range(i * 10000 + j * 100, i * 10000 + (j + 1) * 100)) - self.assertAllEqual(got, expected) - with self.assertRaises(errors.OutOfRangeError): - sess.run(elements) - - def testMapAndBatchSparse(self): - - def _sparse(i): - return sparse_tensor.SparseTensorValue( - indices=[[0]], values=(i * [1]), dense_shape=[1]) - - iterator = dataset_ops.Dataset.range(10).apply( - batching.map_and_batch(_sparse, 5)).make_initializable_iterator() - init_op = iterator.initializer - get_next = iterator.get_next() - - with self.cached_session() as sess: - sess.run(init_op) - for i in range(2): - actual = sess.run(get_next) - expected = sparse_tensor.SparseTensorValue( - indices=[[0, 0], [1, 0], [2, 0], [3, 0], [4, 0]], - values=[i * 5, i * 5 + 1, i * 5 + 2, i * 5 + 3, i * 5 + 4], - dense_shape=[5, 1]) - self.assertTrue(sparse_tensor.is_sparse(actual)) - self.assertSparseValuesEqual(actual, expected) - with self.assertRaises(errors.OutOfRangeError): - sess.run(get_next) - - def testMapAndBatchFails(self): - """Test a dataset that maps a TF function across its input elements.""" - dataset = dataset_ops.Dataset.from_tensors( - array_ops.check_numerics( - constant_op.constant(1.0) / constant_op.constant(0.0), "oops")) - batch_size = array_ops.placeholder(dtypes.int64, shape=[]) - iterator = ( - dataset.apply(batching.map_and_batch( - lambda x: x, batch_size)).make_initializable_iterator()) - init_op = iterator.initializer - with self.cached_session() as sess: - with self.assertRaisesRegexp(errors.InvalidArgumentError, "oops"): - sess.run(init_op, feed_dict={batch_size: 14}) - - def testMapAndBatchShapeMismatch(self): - """Test a dataset that maps a TF function across its input elements.""" - - def generator(): - yield [1] - yield [2] - yield [3] - yield [[4, 5, 6]] - - dataset = dataset_ops.Dataset.from_generator( - generator, output_types=dtypes.int32) - batch_size = 4 - iterator = ( - dataset.apply(batching.map_and_batch( - lambda x: x, batch_size)).make_initializable_iterator()) - init_op = iterator.initializer - get_next = iterator.get_next() - with self.cached_session() as sess: - sess.run(init_op) - with self.assertRaisesRegexp(errors.InvalidArgumentError, - "number of elements does not match"): - sess.run(get_next) - - def testMapAndBatchImplicitDispose(self): - # Tests whether a map and batch dataset will be cleaned up correctly when - # the pipeline does not run it until exhaustion. - # The pipeline is TensorSliceDataset -> RepeatDataset(1000) -> - # MapAndBatchDataset(f=square_3, batch_size=100). - components = (np.arange(1000), - np.array([[1, 2, 3]]) * np.arange(1000)[:, np.newaxis], - np.array(37.0) * np.arange(1000)) - - def _map_fn(x, y, z): - return math_ops.square(x), math_ops.square(y), math_ops.square(z) - - dataset = dataset_ops.Dataset.from_tensor_slices(components).repeat( - 1000).apply(batching.map_and_batch(_map_fn, batch_size=100)) - dataset = dataset.prefetch(5) - iterator = dataset.make_one_shot_iterator() - get_next = iterator.get_next() - - with self.cached_session() as sess: - for _ in range(3): - sess.run(get_next) - - @parameterized.named_parameters( - ("1", 0), - ("2", 5), - ("3", 10), - ("4", 90), - ("5", 95), - ("6", 99), - ) - def testMapAndBatchOutOfRangeError(self, threshold): - - def raising_py_fn(i): - if i >= threshold: - raise StopIteration() - else: - return i - - iterator = ( - dataset_ops.Dataset.range(100).apply( - batching.map_and_batch( - lambda x: script_ops.py_func(raising_py_fn, [x], dtypes.int64), - batch_size=10)).make_one_shot_iterator()) - get_next = iterator.get_next() - - with self.cached_session() as sess: - for i in range(threshold // 10): - self.assertAllEqual([i * 10 + j for j in range(10)], sess.run(get_next)) - if threshold % 10 != 0: - self.assertAllEqual( - [threshold // 10 * 10 + j for j in range(threshold % 10)], - sess.run(get_next)) - with self.assertRaises(errors.OutOfRangeError): - sess.run(get_next) - - @parameterized.named_parameters( - ("1", False, dtypes.bool), - ("2", -42, dtypes.int8), - ("3", -42, dtypes.int16), - ("4", -42, dtypes.int32), - ("5", -42, dtypes.int64), - ("6", 42, dtypes.uint8), - ("7", 42, dtypes.uint16), - ("8", 42.0, dtypes.float16), - ("9", 42.0, dtypes.float32), - ("10", 42.0, dtypes.float64), - ("11", b"hello", dtypes.string), - ) - def testMapAndBatchTypes(self, element, dtype): - - def gen(): - yield element - - dataset = dataset_ops.Dataset.from_generator(gen, dtype).repeat(100).apply( - batching.map_and_batch(lambda x: x, batch_size=10)) - - get_next = dataset.make_one_shot_iterator().get_next() - - with self.cached_session() as sess: - for _ in range(10): - self.assertAllEqual([element for _ in range(10)], sess.run(get_next)) - - -class UnbatchDatasetBenchmark(test.Benchmark): - - def benchmarkNativeUnbatch(self): - batch_sizes = [1, 2, 5, 10, 20, 50] - elems_per_trial = 10000 - with ops.Graph().as_default(): - dataset = dataset_ops.Dataset.from_tensors("element").repeat(None) - batch_size_placeholder = array_ops.placeholder(dtypes.int64, shape=[]) - dataset = dataset.batch(batch_size_placeholder) - dataset = dataset.apply(batching.unbatch()) - dataset = dataset.skip(elems_per_trial) - iterator = dataset.make_initializable_iterator() - next_element = iterator.get_next() - - with session.Session() as sess: - for batch_size in batch_sizes: - deltas = [] - for _ in range(5): - sess.run( - iterator.initializer, - feed_dict={batch_size_placeholder: batch_size}) - start = time.time() - sess.run(next_element.op) - end = time.time() - deltas.append((end - start) / elems_per_trial) - - median_wall_time = np.median(deltas) - print("Unbatch (native) batch size: %d Median wall time per element:" - " %f microseconds" % (batch_size, median_wall_time * 1e6)) - self.report_benchmark( - iters=10000, - wall_time=median_wall_time, - name="benchmark_unbatch_dataset_native_batch_size_%d" % - batch_size) - - # Include a benchmark of the previous `unbatch()` implementation that uses - # a composition of more primitive ops. Eventually we'd hope to generate code - # that is as good in both cases. - def benchmarkOldUnbatchImplementation(self): - batch_sizes = [1, 2, 5, 10, 20, 50] - elems_per_trial = 10000 - with ops.Graph().as_default(): - dataset = dataset_ops.Dataset.from_tensors("element").repeat(None) - batch_size_placeholder = array_ops.placeholder(dtypes.int64, shape=[]) - dataset = dataset.batch(batch_size_placeholder) - dataset = dataset.flat_map(dataset_ops.Dataset.from_tensor_slices) - dataset = dataset.skip(elems_per_trial) - iterator = dataset.make_initializable_iterator() - next_element = iterator.get_next() - - with session.Session() as sess: - for batch_size in batch_sizes: - deltas = [] - for _ in range(5): - sess.run( - iterator.initializer, - feed_dict={batch_size_placeholder: batch_size}) - start = time.time() - sess.run(next_element.op) - end = time.time() - deltas.append((end - start) / elems_per_trial) - - median_wall_time = np.median(deltas) - print("Unbatch (unfused) batch size: %d Median wall time per element:" - " %f microseconds" % (batch_size, median_wall_time * 1e6)) - self.report_benchmark( - iters=10000, - wall_time=median_wall_time, - name="benchmark_unbatch_dataset_unfused_batch_size_%d" % - batch_size) - - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/python/data/experimental/kernel_tests/bucket_by_sequence_length_test.py b/tensorflow/python/data/experimental/kernel_tests/bucket_by_sequence_length_test.py index 3903ec49b98..8264dee3c15 100644 --- a/tensorflow/python/data/experimental/kernel_tests/bucket_by_sequence_length_test.py +++ b/tensorflow/python/data/experimental/kernel_tests/bucket_by_sequence_length_test.py @@ -105,14 +105,14 @@ class BucketBySequenceLengthTest(test_base.DatasetTestBase): boundaries, batch_sizes, no_padding=no_padding)) - batch, = dataset.make_one_shot_iterator().get_next() + batch, = dataset_ops.make_one_shot_iterator(dataset).get_next() with self.cached_session() as sess: batches = [] for _ in range(4): - batches.append(sess.run(batch)) + batches.append(self.evaluate(batch)) with self.assertRaises(errors.OutOfRangeError): - sess.run(batch) + self.evaluate(batch) batch_sizes_val = [] lengths_val = [] for batch in batches: @@ -155,14 +155,14 @@ class BucketBySequenceLengthTest(test_base.DatasetTestBase): grouping.bucket_by_sequence_length( element_len, boundaries, batch_sizes, pad_to_bucket_boundary=True)) - batch, = dataset.make_one_shot_iterator().get_next() + batch, = dataset_ops.make_one_shot_iterator(dataset).get_next() with self.cached_session() as sess: batches = [] for _ in range(3): - batches.append(sess.run(batch)) + batches.append(self.evaluate(batch)) with self.assertRaisesOpError("bucket_boundaries"): - sess.run(batch) + self.evaluate(batch) batch_sizes_val = [] lengths_val = [] for batch in batches: @@ -192,14 +192,14 @@ class BucketBySequenceLengthTest(test_base.DatasetTestBase): grouping.bucket_by_sequence_length( element_len, boundaries, batch_sizes, pad_to_bucket_boundary=True)) - batch, = dataset.make_one_shot_iterator().get_next() + batch, = dataset_ops.make_one_shot_iterator(dataset).get_next() with self.cached_session() as sess: batches = [] for _ in range(5): - batches.append(sess.run(batch)) + batches.append(self.evaluate(batch)) with self.assertRaises(errors.OutOfRangeError): - sess.run(batch) + self.evaluate(batch) self.assertAllEqual(batches[0], [[1, 0], [1, 1]]) @@ -295,12 +295,12 @@ class BucketBySequenceLengthTest(test_base.DatasetTestBase): def _compute_batches(dataset): """Computes actual batch outputs of dataset and stores in a set.""" - batch = dataset.make_one_shot_iterator().get_next() + batch = dataset_ops.make_one_shot_iterator(dataset).get_next() all_sparse_tensors = set() with self.cached_session() as sess: with self.assertRaises(errors.OutOfRangeError): while True: - output = sess.run(batch) + output = self.evaluate(batch) sprs_tensor = (tuple([tuple(idx) for idx in output.indices]), tuple(output.values)) all_sparse_tensors.add(sprs_tensor) diff --git a/tensorflow/python/data/experimental/kernel_tests/cardinality_test.py b/tensorflow/python/data/experimental/kernel_tests/cardinality_test.py new file mode 100644 index 00000000000..943f0f1f812 --- /dev/null +++ b/tensorflow/python/data/experimental/kernel_tests/cardinality_test.py @@ -0,0 +1,158 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Tests for `tf.data.experimental.cardinality()`.""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from absl.testing import parameterized + +from tensorflow.python.data.experimental.ops import cardinality +from tensorflow.python.data.kernel_tests import test_base +from tensorflow.python.data.ops import dataset_ops +from tensorflow.python.platform import test + + +class NumElementsTest(test_base.DatasetTestBase, parameterized.TestCase): + """Tests for `tf.data.experimental.cardinality()`.""" + + @parameterized.named_parameters( + # pylint: disable=g-long-lambda + ("Batch1", + lambda: dataset_ops.Dataset.range(5).batch(2, drop_remainder=True), 2), + ("Batch2", + lambda: dataset_ops.Dataset.range(5).batch(2, drop_remainder=False), 3), + ("Batch3", + lambda: dataset_ops.Dataset.range(5).filter(lambda _: True).batch(2), + cardinality.UNKNOWN), + ("Batch4", lambda: dataset_ops.Dataset.range(5).repeat().batch(2), + cardinality.INFINITE), + ("Cache1", lambda: dataset_ops.Dataset.range(5).cache(), 5), + ("Cache2", lambda: dataset_ops.Dataset.range(5).cache("foo"), 5), + ("Concatenate1", lambda: dataset_ops.Dataset.range(5).concatenate( + dataset_ops.Dataset.range(5)), 10), + ("Concatenate2", + lambda: dataset_ops.Dataset.range(5).filter(lambda _: True).concatenate( + dataset_ops.Dataset.range(5)), cardinality.UNKNOWN), + ("Concatenate3", lambda: dataset_ops.Dataset.range(5).repeat(). + concatenate(dataset_ops.Dataset.range(5)), + cardinality.INFINITE), + ("Concatenate4", lambda: dataset_ops.Dataset.range(5).concatenate( + dataset_ops.Dataset.range(5).filter(lambda _: True)), + cardinality.UNKNOWN), + ("Concatenate5", + lambda: dataset_ops.Dataset.range(5).filter(lambda _: True).concatenate( + dataset_ops.Dataset.range(5).filter(lambda _: True)), + cardinality.UNKNOWN), + ("Concatenate6", lambda: dataset_ops.Dataset.range(5).repeat(). + concatenate(dataset_ops.Dataset.range(5).filter(lambda _: True)), + cardinality.INFINITE), + ("Concatenate7", lambda: dataset_ops.Dataset.range(5).concatenate( + dataset_ops.Dataset.range(5).repeat()), cardinality.INFINITE), + ("Concatenate8", + lambda: dataset_ops.Dataset.range(5).filter(lambda _: True).concatenate( + dataset_ops.Dataset.range(5).repeat()), cardinality.INFINITE), + ("Concatenate9", + lambda: dataset_ops.Dataset.range(5).repeat().concatenate( + dataset_ops.Dataset.range(5).repeat()), cardinality.INFINITE), + ("FlatMap", lambda: dataset_ops.Dataset.range(5).flat_map( + lambda _: dataset_ops.Dataset.from_tensors(0)), + cardinality.UNKNOWN), + ("Filter", lambda: dataset_ops.Dataset.range(5).filter(lambda _: True), + cardinality.UNKNOWN), + ("FromTensors1", lambda: dataset_ops.Dataset.from_tensors(0), 1), + ("FromTensors2", lambda: dataset_ops.Dataset.from_tensors((0, 1)), 1), + ("FromTensorSlices1", + lambda: dataset_ops.Dataset.from_tensor_slices([0, 0, 0]), 3), + ("FromTensorSlices2", + lambda: dataset_ops.Dataset.from_tensor_slices(([0, 0, 0], [1, 1, 1])), + 3), + ("Interleave1", lambda: dataset_ops.Dataset.range(5).interleave( + lambda _: dataset_ops.Dataset.from_tensors(0), cycle_length=1), + cardinality.UNKNOWN), + ("Interleave2", lambda: dataset_ops.Dataset.range(5).interleave( + lambda _: dataset_ops.Dataset.from_tensors(0), + cycle_length=1, + num_parallel_calls=1), cardinality.UNKNOWN), + ("Map1", lambda: dataset_ops.Dataset.range(5).map(lambda x: x), 5), + ("Map2", lambda: dataset_ops.Dataset.range(5).map( + lambda x: x, num_parallel_calls=1), 5), + ("PaddedBatch1", lambda: dataset_ops.Dataset.range(5).padded_batch( + 2, [], drop_remainder=True), 2), + ("PaddedBatch2", lambda: dataset_ops.Dataset.range(5).padded_batch( + 2, [], drop_remainder=False), 3), + ("PaddedBatch3", lambda: dataset_ops.Dataset.range(5).filter( + lambda _: True).padded_batch(2, []), cardinality.UNKNOWN), + ("PaddedBatch4", + lambda: dataset_ops.Dataset.range(5).repeat().padded_batch(2, []), + cardinality.INFINITE), + ("Prefetch", lambda: dataset_ops.Dataset.range(5).prefetch(buffer_size=1), + 5), + ("Range1", lambda: dataset_ops.Dataset.range(0), 0), + ("Range2", lambda: dataset_ops.Dataset.range(5), 5), + ("Range3", lambda: dataset_ops.Dataset.range(5, 10), 5), + ("Range4", lambda: dataset_ops.Dataset.range(10, 5), 0), + ("Range5", lambda: dataset_ops.Dataset.range(5, 10, 2), 3), + ("Range6", lambda: dataset_ops.Dataset.range(10, 5, -2), 3), + ("Repeat1", lambda: dataset_ops.Dataset.range(0).repeat(0), 0), + ("Repeat2", lambda: dataset_ops.Dataset.range(1).repeat(0), 0), + ("Repeat3", lambda: dataset_ops.Dataset.range(0).repeat(5), 0), + ("Repeat4", lambda: dataset_ops.Dataset.range(1).repeat(5), 5), + ("Repeat5", lambda: dataset_ops.Dataset.range(0).repeat(), 0), + ("Repeat6", lambda: dataset_ops.Dataset.range(1).repeat(), + cardinality.INFINITE), + ("Shuffle", lambda: dataset_ops.Dataset.range(5).shuffle(buffer_size=1), + 5), + ("Skip1", lambda: dataset_ops.Dataset.range(5).skip(2), 3), + ("Skip2", lambda: dataset_ops.Dataset.range(5).skip(8), 0), + ("Skip3", + lambda: dataset_ops.Dataset.range(5).filter(lambda _: True).skip(2), + cardinality.UNKNOWN), + ("Skip4", lambda: dataset_ops.Dataset.range(5).repeat().skip(2), + cardinality.INFINITE), + ("Take1", lambda: dataset_ops.Dataset.range(5).take(2), 2), + ("Take2", lambda: dataset_ops.Dataset.range(5).take(8), 5), + ("Take3", + lambda: dataset_ops.Dataset.range(5).filter(lambda _: True).take(2), + cardinality.UNKNOWN), + ("Take4", lambda: dataset_ops.Dataset.range(5).repeat().take(2), 2), + ("Window1", lambda: dataset_ops.Dataset.range(5).window( + size=2, shift=2, drop_remainder=True), 2), + ("Window2", lambda: dataset_ops.Dataset.range(5).window( + size=2, shift=2, drop_remainder=False), 3), + ("Zip1", lambda: dataset_ops.Dataset.zip(dataset_ops.Dataset.range(5)), + 5), + ("Zip2", lambda: dataset_ops.Dataset.zip( + (dataset_ops.Dataset.range(5), dataset_ops.Dataset.range(3))), 3), + ("Zip3", lambda: dataset_ops.Dataset.zip( + (dataset_ops.Dataset.range(5), + dataset_ops.Dataset.range(3).repeat())), 5), + ("Zip4", lambda: dataset_ops.Dataset.zip( + (dataset_ops.Dataset.range(5).repeat(), + dataset_ops.Dataset.range(3).repeat())), cardinality.INFINITE), + ("Zip5", lambda: dataset_ops.Dataset.zip( + (dataset_ops.Dataset.range(5), + dataset_ops.Dataset.range(3).filter(lambda _: True))), + cardinality.UNKNOWN), + # pylint: enable=g-long-lambda + ) + def testNumElements(self, dataset_fn, expected_result): + with self.cached_session() as sess: + self.assertEqual( + sess.run(cardinality.cardinality(dataset_fn())), expected_result) + + +if __name__ == "__main__": + test.main() diff --git a/tensorflow/python/data/experimental/kernel_tests/copy_to_device_test.py b/tensorflow/python/data/experimental/kernel_tests/copy_to_device_test.py index cea8bd6f0b7..b8166fe8334 100644 --- a/tensorflow/python/data/experimental/kernel_tests/copy_to_device_test.py +++ b/tensorflow/python/data/experimental/kernel_tests/copy_to_device_test.py @@ -35,13 +35,14 @@ from tensorflow.python.util import compat as util_compat class CopyToDeviceTest(test_base.DatasetTestBase): + @test_util.run_deprecated_v1 def testCopyToDevice(self): host_dataset = dataset_ops.Dataset.range(10) device_dataset = host_dataset.apply( prefetching_ops.copy_to_device("/cpu:1")) with ops.device("/cpu:1"): - iterator = device_dataset.make_one_shot_iterator() + iterator = dataset_ops.make_one_shot_iterator(device_dataset) next_element = iterator.get_next() self.assertEqual(host_dataset.output_types, device_dataset.output_types) @@ -55,19 +56,20 @@ class CopyToDeviceTest(test_base.DatasetTestBase): self.assertEqual([], next_element.shape) worker_config = config_pb2.ConfigProto(device_count={"CPU": 2}) - with self.test_session(config=worker_config) as sess: + with self.test_session(config=worker_config): for i in range(10): - self.assertEqual(i, sess.run(next_element)) + self.assertEqual(i, self.evaluate(next_element)) with self.assertRaises(errors.OutOfRangeError): - sess.run(next_element) + self.evaluate(next_element) + @test_util.run_deprecated_v1 def testCopyToDeviceInt32(self): host_dataset = dataset_ops.Dataset.from_tensors([0, 1, 2, 3]) device_dataset = host_dataset.apply( prefetching_ops.copy_to_device("/cpu:1")) with ops.device("/cpu:1"): - iterator = device_dataset.make_one_shot_iterator() + iterator = dataset_ops.make_one_shot_iterator(device_dataset) next_element = iterator.get_next() self.assertEqual(host_dataset.output_types, device_dataset.output_types) @@ -81,18 +83,19 @@ class CopyToDeviceTest(test_base.DatasetTestBase): self.assertEqual((4,), next_element.shape) worker_config = config_pb2.ConfigProto(device_count={"CPU": 2}) - with self.test_session(config=worker_config) as sess: - self.assertAllEqual([0, 1, 2, 3], sess.run(next_element)) + with self.test_session(config=worker_config): + self.assertAllEqual([0, 1, 2, 3], self.evaluate(next_element)) with self.assertRaises(errors.OutOfRangeError): - sess.run(next_element) + self.evaluate(next_element) + @test_util.run_deprecated_v1 def testCopyToSameDevice(self): host_dataset = dataset_ops.Dataset.range(10) device_dataset = host_dataset.apply( prefetching_ops.copy_to_device("/cpu:0")) with ops.device("/cpu:0"): - iterator = device_dataset.make_one_shot_iterator() + iterator = dataset_ops.make_one_shot_iterator(device_dataset) next_element = iterator.get_next() self.assertEqual(host_dataset.output_types, device_dataset.output_types) @@ -106,19 +109,20 @@ class CopyToDeviceTest(test_base.DatasetTestBase): self.assertEqual([], next_element.shape) worker_config = config_pb2.ConfigProto(device_count={"CPU": 2}) - with self.test_session(config=worker_config) as sess: + with self.test_session(config=worker_config): for i in range(10): - self.assertEqual(i, sess.run(next_element)) + self.assertEqual(i, self.evaluate(next_element)) with self.assertRaises(errors.OutOfRangeError): - sess.run(next_element) + self.evaluate(next_element) + @test_util.run_deprecated_v1 def testCopyToDeviceWithPrefetch(self): host_dataset = dataset_ops.Dataset.range(10) device_dataset = host_dataset.apply( prefetching_ops.copy_to_device("/cpu:1")).prefetch(1) with ops.device("/cpu:1"): - iterator = device_dataset.make_one_shot_iterator() + iterator = dataset_ops.make_one_shot_iterator(device_dataset) next_element = iterator.get_next() self.assertEqual(host_dataset.output_types, device_dataset.output_types) @@ -132,19 +136,20 @@ class CopyToDeviceTest(test_base.DatasetTestBase): self.assertEqual([], next_element.shape) worker_config = config_pb2.ConfigProto(device_count={"CPU": 2}) - with self.test_session(config=worker_config) as sess: + with self.test_session(config=worker_config): for i in range(10): - self.assertEqual(i, sess.run(next_element)) + self.assertEqual(i, self.evaluate(next_element)) with self.assertRaises(errors.OutOfRangeError): - sess.run(next_element) + self.evaluate(next_element) + @test_util.run_deprecated_v1 def testCopyDictToDevice(self): host_dataset = dataset_ops.Dataset.range(10).map(lambda x: {"a": x}) device_dataset = host_dataset.apply( prefetching_ops.copy_to_device("/cpu:1")) with ops.device("/cpu:1"): - iterator = device_dataset.make_one_shot_iterator() + iterator = dataset_ops.make_one_shot_iterator(device_dataset) next_element = iterator.get_next() self.assertEqual(host_dataset.output_types, device_dataset.output_types) @@ -158,19 +163,20 @@ class CopyToDeviceTest(test_base.DatasetTestBase): self.assertEqual([], next_element["a"].shape) worker_config = config_pb2.ConfigProto(device_count={"CPU": 2}) - with self.test_session(config=worker_config) as sess: + with self.test_session(config=worker_config): for i in range(10): - self.assertEqual({"a": i}, sess.run(next_element)) + self.assertEqual({"a": i}, self.evaluate(next_element)) with self.assertRaises(errors.OutOfRangeError): - sess.run(next_element) + self.evaluate(next_element) + @test_util.run_deprecated_v1 def testCopyDictToDeviceWithPrefetch(self): host_dataset = dataset_ops.Dataset.range(10).map(lambda x: {"a": x}) device_dataset = host_dataset.apply( prefetching_ops.copy_to_device("/cpu:1")).prefetch(1) with ops.device("/cpu:1"): - iterator = device_dataset.make_one_shot_iterator() + iterator = dataset_ops.make_one_shot_iterator(device_dataset) next_element = iterator.get_next() self.assertEqual(host_dataset.output_types, device_dataset.output_types) @@ -184,12 +190,13 @@ class CopyToDeviceTest(test_base.DatasetTestBase): self.assertEqual([], next_element["a"].shape) worker_config = config_pb2.ConfigProto(device_count={"CPU": 2}) - with self.test_session(config=worker_config) as sess: + with self.test_session(config=worker_config): for i in range(10): - self.assertEqual({"a": i}, sess.run(next_element)) + self.assertEqual({"a": i}, self.evaluate(next_element)) with self.assertRaises(errors.OutOfRangeError): - sess.run(next_element) + self.evaluate(next_element) + @test_util.run_deprecated_v1 def testCopySparseTensorsToDevice(self): def make_tensor(i): @@ -202,7 +209,7 @@ class CopyToDeviceTest(test_base.DatasetTestBase): prefetching_ops.copy_to_device("/cpu:1")) with ops.device("/cpu:1"): - iterator = device_dataset.make_one_shot_iterator() + iterator = dataset_ops.make_one_shot_iterator(device_dataset) next_element = iterator.get_next() self.assertEqual(host_dataset.output_types, device_dataset.output_types) @@ -215,15 +222,16 @@ class CopyToDeviceTest(test_base.DatasetTestBase): self.assertEqual(dtypes.int64, next_element.dtype) worker_config = config_pb2.ConfigProto(device_count={"CPU": 2}) - with self.test_session(config=worker_config) as sess: + with self.test_session(config=worker_config): for i in range(10): - actual = sess.run(next_element) + actual = self.evaluate(next_element) self.assertAllEqual([i], actual.values) self.assertAllEqual([[0, 0]], actual.indices) self.assertAllEqual([2, 2], actual.dense_shape) with self.assertRaises(errors.OutOfRangeError): - sess.run(next_element) + self.evaluate(next_element) + @test_util.run_deprecated_v1 def testCopySparseTensorsToDeviceWithPrefetch(self): def make_tensor(i): @@ -236,7 +244,7 @@ class CopyToDeviceTest(test_base.DatasetTestBase): prefetching_ops.copy_to_device("/cpu:1")).prefetch(1) with ops.device("/cpu:1"): - iterator = device_dataset.make_one_shot_iterator() + iterator = dataset_ops.make_one_shot_iterator(device_dataset) next_element = iterator.get_next() self.assertEqual(host_dataset.output_types, device_dataset.output_types) @@ -249,14 +257,14 @@ class CopyToDeviceTest(test_base.DatasetTestBase): self.assertEqual(dtypes.int64, next_element.dtype) worker_config = config_pb2.ConfigProto(device_count={"CPU": 2}) - with self.test_session(config=worker_config) as sess: + with self.test_session(config=worker_config): for i in range(10): - actual = sess.run(next_element) + actual = self.evaluate(next_element) self.assertAllEqual([i], actual.values) self.assertAllEqual([[0, 0]], actual.indices) self.assertAllEqual([2, 2], actual.dense_shape) with self.assertRaises(errors.OutOfRangeError): - sess.run(next_element) + self.evaluate(next_element) def testCopyToDeviceGpu(self): if not test_util.is_gpu_available(): @@ -267,15 +275,16 @@ class CopyToDeviceTest(test_base.DatasetTestBase): prefetching_ops.copy_to_device("/gpu:0")) with ops.device("/gpu:0"): - iterator = device_dataset.make_initializable_iterator() + iterator = dataset_ops.make_initializable_iterator(device_dataset) next_element = iterator.get_next() - with self.cached_session() as sess: - sess.run(iterator.initializer) + with self.cached_session( + config=config_pb2.ConfigProto(allow_soft_placement=False)): + self.evaluate(iterator.initializer) for i in range(10): - self.assertEqual(i, sess.run(next_element)) + self.assertEqual(i, self.evaluate(next_element)) with self.assertRaises(errors.OutOfRangeError): - sess.run(next_element) + self.evaluate(next_element) def testCopyToDeviceGpuWithPrefetch(self): if not test_util.is_gpu_available(): @@ -286,15 +295,16 @@ class CopyToDeviceTest(test_base.DatasetTestBase): prefetching_ops.copy_to_device("/gpu:0")).prefetch(1) with ops.device("/gpu:0"): - iterator = device_dataset.make_initializable_iterator() + iterator = dataset_ops.make_initializable_iterator(device_dataset) next_element = iterator.get_next() - with self.cached_session() as sess: - sess.run(iterator.initializer) + with self.cached_session( + config=config_pb2.ConfigProto(allow_soft_placement=False)): + self.evaluate(iterator.initializer) for i in range(10): - self.assertEqual(i, sess.run(next_element)) + self.assertEqual(i, self.evaluate(next_element)) with self.assertRaises(errors.OutOfRangeError): - sess.run(next_element) + self.evaluate(next_element) def testCopyToDeviceGpuWithMap(self): if not test_util.is_gpu_available(): @@ -319,18 +329,19 @@ class CopyToDeviceTest(test_base.DatasetTestBase): device_dataset = device_dataset.with_options(options) with ops.device("/gpu:0"): - iterator = device_dataset.make_initializable_iterator() + iterator = dataset_ops.make_initializable_iterator(device_dataset) next_element = iterator.get_next() - with self.cached_session() as sess: - sess.run(iterator.initializer) + with self.cached_session( + config=config_pb2.ConfigProto(allow_soft_placement=False)): + self.evaluate(iterator.initializer) for i in range(10): - x, y, z = sess.run(next_element) + x, y, z = self.evaluate(next_element) self.assertEqual(i**2, x) self.assertEqual(float(i**2), y) self.assertEqual(util_compat.as_bytes(str(i)), z) with self.assertRaises(errors.OutOfRangeError): - sess.run(next_element) + self.evaluate(next_element) def testCopyToDeviceGpuInt32(self): if not test_util.is_gpu_available(): @@ -341,14 +352,15 @@ class CopyToDeviceTest(test_base.DatasetTestBase): prefetching_ops.copy_to_device("/gpu:0")) with ops.device("/gpu:0"): - iterator = device_dataset.make_initializable_iterator() + iterator = dataset_ops.make_initializable_iterator(device_dataset) next_element = iterator.get_next() - with self.cached_session() as sess: - sess.run(iterator.initializer) - self.assertAllEqual([0, 1, 2, 3], sess.run(next_element)) + with self.cached_session( + config=config_pb2.ConfigProto(allow_soft_placement=False)): + self.evaluate(iterator.initializer) + self.assertAllEqual([0, 1, 2, 3], self.evaluate(next_element)) with self.assertRaises(errors.OutOfRangeError): - sess.run(next_element) + self.evaluate(next_element) def testCopyToDeviceGpuInt32AndPrefetch(self): if not test_util.is_gpu_available(): @@ -359,14 +371,15 @@ class CopyToDeviceTest(test_base.DatasetTestBase): prefetching_ops.copy_to_device("/gpu:0")).prefetch(1) with ops.device("/gpu:0"): - iterator = device_dataset.make_initializable_iterator() + iterator = dataset_ops.make_initializable_iterator(device_dataset) next_element = iterator.get_next() - with self.cached_session() as sess: - sess.run(iterator.initializer) - self.assertAllEqual([0, 1, 2, 3], sess.run(next_element)) + with self.cached_session( + config=config_pb2.ConfigProto(allow_soft_placement=False)): + self.evaluate(iterator.initializer) + self.assertAllEqual([0, 1, 2, 3], self.evaluate(next_element)) with self.assertRaises(errors.OutOfRangeError): - sess.run(next_element) + self.evaluate(next_element) def testCopyToDeviceGpuStrings(self): if not test_util.is_gpu_available(): @@ -377,14 +390,15 @@ class CopyToDeviceTest(test_base.DatasetTestBase): prefetching_ops.copy_to_device("/gpu:0")) with ops.device("/gpu:0"): - iterator = device_dataset.make_initializable_iterator() + iterator = dataset_ops.make_initializable_iterator(device_dataset) next_element = iterator.get_next() - with self.cached_session() as sess: - sess.run(iterator.initializer) - self.assertAllEqual([b"a", b"b", b"c"], sess.run(next_element)) + with self.cached_session( + config=config_pb2.ConfigProto(allow_soft_placement=False)): + self.evaluate(iterator.initializer) + self.assertAllEqual([b"a", b"b", b"c"], self.evaluate(next_element)) with self.assertRaises(errors.OutOfRangeError): - sess.run(next_element) + self.evaluate(next_element) def testCopyToDeviceGpuStringsAndPrefetch(self): if not test_util.is_gpu_available(): @@ -395,14 +409,15 @@ class CopyToDeviceTest(test_base.DatasetTestBase): prefetching_ops.copy_to_device("/gpu:0")) with ops.device("/gpu:0"): - iterator = device_dataset.make_initializable_iterator() + iterator = dataset_ops.make_initializable_iterator(device_dataset) next_element = iterator.get_next() - with self.cached_session() as sess: - sess.run(iterator.initializer) - self.assertAllEqual([b"a", b"b", b"c"], sess.run(next_element)) + with self.cached_session( + config=config_pb2.ConfigProto(allow_soft_placement=False)): + self.evaluate(iterator.initializer) + self.assertAllEqual([b"a", b"b", b"c"], self.evaluate(next_element)) with self.assertRaises(errors.OutOfRangeError): - sess.run(next_element) + self.evaluate(next_element) def testCopyToDevicePingPongCPUGPU(self): if not test_util.is_gpu_available(): @@ -416,23 +431,25 @@ class CopyToDeviceTest(test_base.DatasetTestBase): prefetching_ops.copy_to_device("/cpu:0", source_device="/gpu:0")) with ops.device("/cpu:0"): - iterator = back_to_cpu_dataset.make_initializable_iterator() + iterator = dataset_ops.make_initializable_iterator(back_to_cpu_dataset) next_element = iterator.get_next() - with self.cached_session() as sess: - sess.run(iterator.initializer) + with self.cached_session( + config=config_pb2.ConfigProto(allow_soft_placement=False)): + self.evaluate(iterator.initializer) for i in range(10): - self.assertEqual(i, sess.run(next_element)) + self.assertEqual(i, self.evaluate(next_element)) with self.assertRaises(errors.OutOfRangeError): - sess.run(next_element) + self.evaluate(next_element) + @test_util.run_deprecated_v1 def testCopyToDeviceWithReInit(self): host_dataset = dataset_ops.Dataset.range(10) device_dataset = host_dataset.apply( prefetching_ops.copy_to_device("/cpu:1")) with ops.device("/cpu:1"): - iterator = device_dataset.make_initializable_iterator() + iterator = dataset_ops.make_initializable_iterator(device_dataset) next_element = iterator.get_next() self.assertEqual(host_dataset.output_types, device_dataset.output_types) @@ -446,23 +463,24 @@ class CopyToDeviceTest(test_base.DatasetTestBase): self.assertEqual([], next_element.shape) worker_config = config_pb2.ConfigProto(device_count={"CPU": 2}) - with self.test_session(config=worker_config) as sess: - sess.run(iterator.initializer) + with self.test_session(config=worker_config): + self.evaluate(iterator.initializer) for i in range(5): - self.assertEqual(i, sess.run(next_element)) - sess.run(iterator.initializer) + self.assertEqual(i, self.evaluate(next_element)) + self.evaluate(iterator.initializer) for i in range(10): - self.assertEqual(i, sess.run(next_element)) + self.assertEqual(i, self.evaluate(next_element)) with self.assertRaises(errors.OutOfRangeError): - sess.run(next_element) + self.evaluate(next_element) + @test_util.run_deprecated_v1 def testCopyToDeviceWithReInitAndPrefetch(self): host_dataset = dataset_ops.Dataset.range(10) device_dataset = host_dataset.apply( prefetching_ops.copy_to_device("/cpu:1")).prefetch(1) with ops.device("/cpu:1"): - iterator = device_dataset.make_initializable_iterator() + iterator = dataset_ops.make_initializable_iterator(device_dataset) next_element = iterator.get_next() self.assertEqual(host_dataset.output_types, device_dataset.output_types) @@ -476,15 +494,15 @@ class CopyToDeviceTest(test_base.DatasetTestBase): self.assertEqual([], next_element.shape) worker_config = config_pb2.ConfigProto(device_count={"CPU": 2}) - with self.test_session(config=worker_config) as sess: - sess.run(iterator.initializer) + with self.test_session(config=worker_config): + self.evaluate(iterator.initializer) for i in range(5): - self.assertEqual(i, sess.run(next_element)) - sess.run(iterator.initializer) + self.assertEqual(i, self.evaluate(next_element)) + self.evaluate(iterator.initializer) for i in range(10): - self.assertEqual(i, sess.run(next_element)) + self.assertEqual(i, self.evaluate(next_element)) with self.assertRaises(errors.OutOfRangeError): - sess.run(next_element) + self.evaluate(next_element) def testCopyToDeviceGpuWithReInit(self): if not test_util.is_gpu_available(): @@ -495,18 +513,19 @@ class CopyToDeviceTest(test_base.DatasetTestBase): prefetching_ops.copy_to_device("/gpu:0")) with ops.device("/gpu:0"): - iterator = device_dataset.make_initializable_iterator() + iterator = dataset_ops.make_initializable_iterator(device_dataset) next_element = iterator.get_next() - with self.cached_session() as sess: - sess.run(iterator.initializer) + with self.cached_session( + config=config_pb2.ConfigProto(allow_soft_placement=False)): + self.evaluate(iterator.initializer) for i in range(5): - self.assertEqual(i, sess.run(next_element)) - sess.run(iterator.initializer) + self.assertEqual(i, self.evaluate(next_element)) + self.evaluate(iterator.initializer) for i in range(10): - self.assertEqual(i, sess.run(next_element)) + self.assertEqual(i, self.evaluate(next_element)) with self.assertRaises(errors.OutOfRangeError): - sess.run(next_element) + self.evaluate(next_element) def testCopyToDeviceGpuWithReInitAndPrefetch(self): if not test_util.is_gpu_available(): @@ -517,18 +536,19 @@ class CopyToDeviceTest(test_base.DatasetTestBase): prefetching_ops.copy_to_device("/gpu:0")).prefetch(1) with ops.device("/gpu:0"): - iterator = device_dataset.make_initializable_iterator() + iterator = dataset_ops.make_initializable_iterator(device_dataset) next_element = iterator.get_next() - with self.cached_session() as sess: - sess.run(iterator.initializer) + with self.cached_session( + config=config_pb2.ConfigProto(allow_soft_placement=False)): + self.evaluate(iterator.initializer) for i in range(5): - self.assertEqual(i, sess.run(next_element)) - sess.run(iterator.initializer) + self.assertEqual(i, self.evaluate(next_element)) + self.evaluate(iterator.initializer) for i in range(10): - self.assertEqual(i, sess.run(next_element)) + self.assertEqual(i, self.evaluate(next_element)) with self.assertRaises(errors.OutOfRangeError): - sess.run(next_element) + self.evaluate(next_element) def testIteratorGetNextAsOptionalOnGPU(self): if not test_util.is_gpu_available(): @@ -538,33 +558,35 @@ class CopyToDeviceTest(test_base.DatasetTestBase): device_dataset = host_dataset.apply( prefetching_ops.copy_to_device("/gpu:0")) with ops.device("/gpu:0"): - iterator = device_dataset.make_initializable_iterator() + iterator = dataset_ops.make_initializable_iterator(device_dataset) next_elem = iterator_ops.get_next_as_optional(iterator) elem_has_value_t = next_elem.has_value() elem_value_t = next_elem.get_value() - with self.cached_session() as sess: + with self.cached_session( + config=config_pb2.ConfigProto(allow_soft_placement=False)): # Before initializing the iterator, evaluating the optional fails with # a FailedPreconditionError. with self.assertRaises(errors.FailedPreconditionError): - sess.run(elem_has_value_t) + self.evaluate(elem_has_value_t) with self.assertRaises(errors.FailedPreconditionError): - sess.run(elem_value_t) + self.evaluate(elem_value_t) # For each element of the dataset, assert that the optional evaluates to # the expected value. - sess.run(iterator.initializer) + self.evaluate(iterator.initializer) for i in range(3): - elem_has_value, elem_value = sess.run([elem_has_value_t, elem_value_t]) + elem_has_value, elem_value = self.evaluate( + [elem_has_value_t, elem_value_t]) self.assertTrue(elem_has_value) self.assertEqual(i, elem_value) # After exhausting the iterator, `next_elem.has_value()` will evaluate to # false, and attempting to get the value will fail. for _ in range(2): - self.assertFalse(sess.run(elem_has_value_t)) + self.assertFalse(self.evaluate(elem_has_value_t)) with self.assertRaises(errors.InvalidArgumentError): - sess.run(elem_value_t) + self.evaluate(elem_value_t) if __name__ == "__main__": diff --git a/tensorflow/python/data/experimental/kernel_tests/counter_test.py b/tensorflow/python/data/experimental/kernel_tests/counter_test.py index 4e114ac4791..49e1f2272b7 100644 --- a/tensorflow/python/data/experimental/kernel_tests/counter_test.py +++ b/tensorflow/python/data/experimental/kernel_tests/counter_test.py @@ -19,32 +19,35 @@ from __future__ import print_function from tensorflow.python.data.experimental.ops import counter from tensorflow.python.data.kernel_tests import test_base +from tensorflow.python.data.ops import dataset_ops from tensorflow.python.framework import dtypes +from tensorflow.python.framework import test_util from tensorflow.python.platform import test class CounterTest(test_base.DatasetTestBase): + @test_util.run_deprecated_v1 def testCounter(self): """Test dataset construction using `count`.""" - iterator = (counter.Counter(start=3, step=4) - .make_one_shot_iterator()) + iterator = dataset_ops.make_one_shot_iterator( + counter.Counter(start=3, step=4)) get_next = iterator.get_next() self.assertEqual([], get_next.shape.as_list()) self.assertEqual(dtypes.int64, get_next.dtype) - negative_iterator = (counter.Counter(start=0, step=-1) - .make_one_shot_iterator()) + negative_iterator = dataset_ops.make_one_shot_iterator( + counter.Counter(start=0, step=-1)) negative_get_next = negative_iterator.get_next() with self.cached_session() as sess: - self.assertEqual(3, sess.run(get_next)) - self.assertEqual(3 + 4, sess.run(get_next)) - self.assertEqual(3 + 2 * 4, sess.run(get_next)) + self.assertEqual(3, self.evaluate(get_next)) + self.assertEqual(3 + 4, self.evaluate(get_next)) + self.assertEqual(3 + 2 * 4, self.evaluate(get_next)) - self.assertEqual(0, sess.run(negative_get_next)) - self.assertEqual(-1, sess.run(negative_get_next)) - self.assertEqual(-2, sess.run(negative_get_next)) + self.assertEqual(0, self.evaluate(negative_get_next)) + self.assertEqual(-1, self.evaluate(negative_get_next)) + self.assertEqual(-2, self.evaluate(negative_get_next)) if __name__ == "__main__": diff --git a/tensorflow/python/data/experimental/kernel_tests/csv_dataset_test.py b/tensorflow/python/data/experimental/kernel_tests/csv_dataset_test.py index fb75be1fbcf..b2f1b43ecf6 100644 --- a/tensorflow/python/data/experimental/kernel_tests/csv_dataset_test.py +++ b/tensorflow/python/data/experimental/kernel_tests/csv_dataset_test.py @@ -20,14 +20,8 @@ from __future__ import print_function import gzip import os -import string -import tempfile -import time import zlib -import numpy as np - -from tensorflow.python.client import session from tensorflow.python.data.experimental.ops import error_ops from tensorflow.python.data.experimental.ops import readers from tensorflow.python.data.kernel_tests import test_base @@ -38,8 +32,6 @@ from tensorflow.python.framework import dtypes from tensorflow.python.framework import errors from tensorflow.python.framework import test_util from tensorflow.python.ops import parsing_ops -from tensorflow.python.platform import gfile -from tensorflow.python.platform import googletest from tensorflow.python.platform import test @@ -537,96 +529,5 @@ class CsvDatasetTest(test_base.DatasetTestBase): record_defaults=record_defaults) -class CsvDatasetBenchmark(test.Benchmark): - """Benchmarks for the various ways of creating a dataset from CSV files. - """ - FLOAT_VAL = '1.23456E12' - STR_VAL = string.ascii_letters * 10 - - def _setUp(self, str_val): - # Since this isn't test.TestCase, have to manually create a test dir - gfile.MakeDirs(googletest.GetTempDir()) - self._temp_dir = tempfile.mkdtemp(dir=googletest.GetTempDir()) - - self._num_cols = [4, 64, 256] - self._num_per_iter = 5000 - self._filenames = [] - for n in self._num_cols: - fn = os.path.join(self._temp_dir, 'file%d.csv' % n) - with open(fn, 'wb') as f: - # Just write 100 rows and use `repeat`... Assumes the cost - # of creating an iterator is not significant - row = ','.join([str_val for _ in range(n)]) - f.write('\n'.join([row for _ in range(100)])) - self._filenames.append(fn) - - def _tearDown(self): - gfile.DeleteRecursively(self._temp_dir) - - def _runBenchmark(self, dataset, num_cols, prefix): - dataset = dataset.skip(self._num_per_iter - 1) - deltas = [] - for _ in range(10): - next_element = dataset.make_one_shot_iterator().get_next() - with session.Session() as sess: - start = time.time() - # NOTE: This depends on the underlying implementation of skip, to have - # the net effect of calling `GetNext` num_per_iter times on the - # input dataset. We do it this way (instead of a python for loop, or - # batching N inputs in one iter) so that the overhead from session.run - # or batch doesn't dominate. If we eventually optimize skip, this has - # to change. - sess.run(next_element) - end = time.time() - deltas.append(end - start) - # Median wall time per CSV record read and decoded - median_wall_time = np.median(deltas) / self._num_per_iter - print('%s num_cols: %d Median wall time: %f' % (prefix, num_cols, - median_wall_time)) - self.report_benchmark( - iters=self._num_per_iter, - wall_time=median_wall_time, - name='%s_with_cols_%d' % (prefix, num_cols)) - - def benchmarkMapWithFloats(self): - self._setUp(self.FLOAT_VAL) - for i in range(len(self._filenames)): - num_cols = self._num_cols[i] - kwargs = {'record_defaults': [[0.0]] * num_cols} - dataset = core_readers.TextLineDataset(self._filenames[i]).repeat() - dataset = dataset.map(lambda l: parsing_ops.decode_csv(l, **kwargs)) # pylint: disable=cell-var-from-loop - self._runBenchmark(dataset, num_cols, 'csv_float_map_decode_csv') - self._tearDown() - - def benchmarkMapWithStrings(self): - self._setUp(self.STR_VAL) - for i in range(len(self._filenames)): - num_cols = self._num_cols[i] - kwargs = {'record_defaults': [['']] * num_cols} - dataset = core_readers.TextLineDataset(self._filenames[i]).repeat() - dataset = dataset.map(lambda l: parsing_ops.decode_csv(l, **kwargs)) # pylint: disable=cell-var-from-loop - self._runBenchmark(dataset, num_cols, 'csv_strings_map_decode_csv') - self._tearDown() - - def benchmarkCsvDatasetWithFloats(self): - self._setUp(self.FLOAT_VAL) - for i in range(len(self._filenames)): - num_cols = self._num_cols[i] - kwargs = {'record_defaults': [[0.0]] * num_cols} - dataset = core_readers.TextLineDataset(self._filenames[i]).repeat() - dataset = readers.CsvDataset(self._filenames[i], **kwargs).repeat() # pylint: disable=cell-var-from-loop - self._runBenchmark(dataset, num_cols, 'csv_float_fused_dataset') - self._tearDown() - - def benchmarkCsvDatasetWithStrings(self): - self._setUp(self.STR_VAL) - for i in range(len(self._filenames)): - num_cols = self._num_cols[i] - kwargs = {'record_defaults': [['']] * num_cols} - dataset = core_readers.TextLineDataset(self._filenames[i]).repeat() - dataset = readers.CsvDataset(self._filenames[i], **kwargs).repeat() # pylint: disable=cell-var-from-loop - self._runBenchmark(dataset, num_cols, 'csv_strings_fused_dataset') - self._tearDown() - if __name__ == '__main__': test.main() diff --git a/tensorflow/python/data/experimental/kernel_tests/dense_to_sparse_batch_test.py b/tensorflow/python/data/experimental/kernel_tests/dense_to_sparse_batch_test.py index 73be6cbcca8..22e057a2848 100644 --- a/tensorflow/python/data/experimental/kernel_tests/dense_to_sparse_batch_test.py +++ b/tensorflow/python/data/experimental/kernel_tests/dense_to_sparse_batch_test.py @@ -24,27 +24,28 @@ from tensorflow.python.data.kernel_tests import test_base from tensorflow.python.data.ops import dataset_ops from tensorflow.python.framework import dtypes from tensorflow.python.framework import errors +from tensorflow.python.framework import test_util from tensorflow.python.ops import array_ops from tensorflow.python.platform import test class DenseToSparseBatchTest(test_base.DatasetTestBase): + @test_util.run_deprecated_v1 def testDenseToSparseBatchDataset(self): components = np.random.randint(12, size=(100,)).astype(np.int32) - iterator = ( + iterator = dataset_ops.make_initializable_iterator( dataset_ops.Dataset.from_tensor_slices(components) .map(lambda x: array_ops.fill([x], x)).apply( - batching.dense_to_sparse_batch(4, [12])) - .make_initializable_iterator()) + batching.dense_to_sparse_batch(4, [12]))) init_op = iterator.initializer get_next = iterator.get_next() with self.cached_session() as sess: - sess.run(init_op) + self.evaluate(init_op) for start in range(0, len(components), 4): - results = sess.run(get_next) + results = self.evaluate(get_next) self.assertAllEqual([[i, j] for i, c in enumerate(components[start:start + 4]) for j in range(c)], results.indices) @@ -56,23 +57,23 @@ class DenseToSparseBatchTest(test_base.DatasetTestBase): results.dense_shape) with self.assertRaises(errors.OutOfRangeError): - sess.run(get_next) + self.evaluate(get_next) + @test_util.run_deprecated_v1 def testDenseToSparseBatchDatasetWithUnknownShape(self): components = np.random.randint(5, size=(40,)).astype(np.int32) - iterator = ( + iterator = dataset_ops.make_initializable_iterator( dataset_ops.Dataset.from_tensor_slices(components) .map(lambda x: array_ops.fill([x, x], x)).apply( - batching.dense_to_sparse_batch( - 4, [5, None])).make_initializable_iterator()) + batching.dense_to_sparse_batch(4, [5, None]))) init_op = iterator.initializer get_next = iterator.get_next() with self.cached_session() as sess: - sess.run(init_op) + self.evaluate(init_op) for start in range(0, len(components), 4): - results = sess.run(get_next) + results = self.evaluate(get_next) self.assertAllEqual([[i, j, z] for i, c in enumerate(components[start:start + 4]) for j in range(c) @@ -89,20 +90,22 @@ class DenseToSparseBatchTest(test_base.DatasetTestBase): ], results.dense_shape) with self.assertRaises(errors.OutOfRangeError): - sess.run(get_next) + self.evaluate(get_next) + @test_util.run_deprecated_v1 def testDenseToSparseBatchDatasetWithInvalidShape(self): input_tensor = array_ops.constant([[1]]) with self.assertRaisesRegexp(ValueError, "Dimension -2 must be >= 0"): - dataset_ops.Dataset.from_tensors(input_tensor).apply( - batching.dense_to_sparse_batch(4, [-2])).make_initializable_iterator() + dataset_ops.make_initializable_iterator( + dataset_ops.Dataset.from_tensors(input_tensor).apply( + batching.dense_to_sparse_batch(4, [-2]))) + @test_util.run_deprecated_v1 def testDenseToSparseBatchDatasetShapeErrors(self): input_tensor = array_ops.placeholder(dtypes.int32) - iterator = ( + iterator = dataset_ops.make_initializable_iterator( dataset_ops.Dataset.from_tensors(input_tensor).apply( - batching.dense_to_sparse_batch(4, [12])) - .make_initializable_iterator()) + batching.dense_to_sparse_batch(4, [12]))) init_op = iterator.initializer get_next = iterator.get_next() @@ -111,13 +114,13 @@ class DenseToSparseBatchTest(test_base.DatasetTestBase): sess.run(init_op, feed_dict={input_tensor: [[1]]}) with self.assertRaisesRegexp(errors.InvalidArgumentError, "incompatible with the row shape"): - sess.run(get_next) + self.evaluate(get_next) # Initialize with an input tensor that is larger than `row_shape`. sess.run(init_op, feed_dict={input_tensor: range(13)}) with self.assertRaisesRegexp(errors.DataLossError, "larger than the row shape"): - sess.run(get_next) + self.evaluate(get_next) if __name__ == "__main__": diff --git a/tensorflow/python/data/experimental/kernel_tests/directed_interleave_dataset_test.py b/tensorflow/python/data/experimental/kernel_tests/directed_interleave_dataset_test.py index 796a692c56f..21443420666 100644 --- a/tensorflow/python/data/experimental/kernel_tests/directed_interleave_dataset_test.py +++ b/tensorflow/python/data/experimental/kernel_tests/directed_interleave_dataset_test.py @@ -24,11 +24,13 @@ from tensorflow.python.data.kernel_tests import test_base from tensorflow.python.data.ops import dataset_ops from tensorflow.python.framework import errors from tensorflow.python.framework import random_seed +from tensorflow.python.framework import test_util from tensorflow.python.platform import test class DirectedInterleaveDatasetTest(test_base.DatasetTestBase): + @test_util.run_deprecated_v1 def testBasic(self): selector_dataset = dataset_ops.Dataset.range(10).repeat(100) input_datasets = [ @@ -36,16 +38,16 @@ class DirectedInterleaveDatasetTest(test_base.DatasetTestBase): ] dataset = interleave_ops._DirectedInterleaveDataset(selector_dataset, input_datasets) - iterator = dataset.make_initializable_iterator() + iterator = dataset_ops.make_initializable_iterator(dataset) next_element = iterator.get_next() with self.cached_session() as sess: - sess.run(iterator.initializer) + self.evaluate(iterator.initializer) for _ in range(100): for i in range(10): - self.assertEqual(i, sess.run(next_element)) + self.assertEqual(i, self.evaluate(next_element)) with self.assertRaises(errors.OutOfRangeError): - sess.run(next_element) + self.evaluate(next_element) def _normalize(self, vec): return vec / vec.sum() @@ -65,18 +67,19 @@ class DirectedInterleaveDatasetTest(test_base.DatasetTestBase): for i in range(num_datasets) ], weights) dataset = dataset.take(num_samples) - iterator = dataset.make_one_shot_iterator() + iterator = dataset_ops.make_one_shot_iterator(dataset) next_element = iterator.get_next() with self.cached_session() as sess: freqs = np.zeros([num_datasets]) for _ in range(num_samples): - freqs[sess.run(next_element)] += 1 + freqs[self.evaluate(next_element)] += 1 with self.assertRaises(errors.OutOfRangeError): - sess.run(next_element) + self.evaluate(next_element) return freqs + @test_util.run_deprecated_v1 def testSampleFromDatasets(self): random_seed.set_random_seed(1619) num_samples = 5000 @@ -96,20 +99,21 @@ class DirectedInterleaveDatasetTest(test_base.DatasetTestBase): freqs = self._testSampleFromDatasetsHelper(probs_ds, classes, num_samples) self.assertLess(self._chi2(probs, freqs / num_samples), 1e-2) + @test_util.run_deprecated_v1 def testSelectFromDatasets(self): words = [b"foo", b"bar", b"baz"] datasets = [dataset_ops.Dataset.from_tensors(w).repeat() for w in words] choice_array = np.random.randint(3, size=(15,), dtype=np.int64) choice_dataset = dataset_ops.Dataset.from_tensor_slices(choice_array) dataset = interleave_ops.choose_from_datasets(datasets, choice_dataset) - iterator = dataset.make_one_shot_iterator() + iterator = dataset_ops.make_one_shot_iterator(dataset) next_element = iterator.get_next() with self.cached_session() as sess: for i in choice_array: - self.assertEqual(words[i], sess.run(next_element)) + self.assertEqual(words[i], self.evaluate(next_element)) with self.assertRaises(errors.OutOfRangeError): - sess.run(next_element) + self.evaluate(next_element) def testErrors(self): with self.assertRaisesRegexp(ValueError, diff --git a/tensorflow/python/data/experimental/kernel_tests/enumerate_dataset_test.py b/tensorflow/python/data/experimental/kernel_tests/enumerate_dataset_test.py index e54235d9f80..25742098f18 100644 --- a/tensorflow/python/data/experimental/kernel_tests/enumerate_dataset_test.py +++ b/tensorflow/python/data/experimental/kernel_tests/enumerate_dataset_test.py @@ -24,17 +24,20 @@ from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes from tensorflow.python.framework import errors from tensorflow.python.framework import tensor_shape +from tensorflow.python.framework import test_util from tensorflow.python.platform import test class EnumerateDatasetTest(test_base.DatasetTestBase): + @test_util.run_deprecated_v1 def testEnumerateDataset(self): components = (["a", "b"], [1, 2], [37.0, 38]) start = constant_op.constant(20, dtype=dtypes.int64) - iterator = (dataset_ops.Dataset.from_tensor_slices(components).apply( - enumerate_ops.enumerate_dataset(start)).make_initializable_iterator()) + iterator = dataset_ops.make_initializable_iterator( + dataset_ops.Dataset.from_tensor_slices(components).apply( + enumerate_ops.enumerate_dataset(start))) init_op = iterator.initializer get_next = iterator.get_next() @@ -44,12 +47,12 @@ class EnumerateDatasetTest(test_base.DatasetTestBase): [t.shape for t in get_next[1]]) with self.cached_session() as sess: - sess.run(init_op) - self.assertEqual((20, (b"a", 1, 37.0)), sess.run(get_next)) - self.assertEqual((21, (b"b", 2, 38.0)), sess.run(get_next)) + self.evaluate(init_op) + self.assertEqual((20, (b"a", 1, 37.0)), self.evaluate(get_next)) + self.assertEqual((21, (b"b", 2, 38.0)), self.evaluate(get_next)) with self.assertRaises(errors.OutOfRangeError): - sess.run(get_next) + self.evaluate(get_next) if __name__ == "__main__": diff --git a/tensorflow/python/data/experimental/kernel_tests/filter_dataset_op_test.py b/tensorflow/python/data/experimental/kernel_tests/filter_dataset_op_test.py index c6ee88c676d..357b5f1b49b 100644 --- a/tensorflow/python/data/experimental/kernel_tests/filter_dataset_op_test.py +++ b/tensorflow/python/data/experimental/kernel_tests/filter_dataset_op_test.py @@ -47,17 +47,17 @@ class FilterBenchmark(test.Benchmark): if optimize_dataset: dataset = dataset.apply(optimization.optimize(["filter_fusion"])) - iterator = dataset.make_one_shot_iterator() + iterator = dataset_ops.make_one_shot_iterator(dataset) next_element = iterator.get_next() with session.Session() as sess: for _ in range(10): - sess.run(next_element.op) + self.evaluate(next_element.op) deltas = [] for _ in range(100): start = time.time() for _ in range(100): - sess.run(next_element.op) + self.evaluate(next_element.op) end = time.time() deltas.append(end - start) diff --git a/tensorflow/python/data/experimental/kernel_tests/function_buffering_resource_test.py b/tensorflow/python/data/experimental/kernel_tests/function_buffering_resource_test.py deleted file mode 100644 index d38452e265a..00000000000 --- a/tensorflow/python/data/experimental/kernel_tests/function_buffering_resource_test.py +++ /dev/null @@ -1,248 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for the private `FunctionBufferingResource` used in prefetching.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import threading - -from tensorflow.core.protobuf import config_pb2 -from tensorflow.python.data.experimental.ops import prefetching_ops -from tensorflow.python.data.kernel_tests import test_base -from tensorflow.python.data.ops import dataset_ops -from tensorflow.python.data.ops import iterator_ops -from tensorflow.python.eager import function -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import errors -from tensorflow.python.framework import ops -from tensorflow.python.framework import tensor_spec -from tensorflow.python.framework import test_util -from tensorflow.python.ops import resource_variable_ops -from tensorflow.python.platform import test - - -class FunctionBufferingResourceTest(test_base.DatasetTestBase): - - def setUp(self): - self._event = threading.Event() - - def _create_ds_and_iterator(self, device0, initializable=False): - - def gen(): - for i in range(1, 10): - yield [float(i)] - if i == 6: - self._event.set() - - with ops.device(device0): - ds = dataset_ops.Dataset.from_generator(gen, (dtypes.float32)) - if initializable: - ds_iterator = ds.make_initializable_iterator() - else: - ds_iterator = ds.make_one_shot_iterator() - return (ds, ds_iterator) - - def _create_ops(self, ds, ds_iterator, buffer_name, device0, device1): - ds_iterator_handle = ds_iterator.string_handle() - - @function.defun(input_signature=[tensor_spec.TensorSpec([], dtypes.string)]) - def _remote_fn(h): - remote_iterator = iterator_ops.Iterator.from_string_handle( - h, ds.output_types, ds.output_shapes) - return remote_iterator.get_next() - - target = constant_op.constant(device0) - with ops.device(device1): - buffer_resource_handle = prefetching_ops.function_buffering_resource( - f=_remote_fn.get_concrete_function(), - output_types=[dtypes.float32], - target_device=target, - string_arg=ds_iterator_handle, - buffer_size=3, - shared_name=buffer_name) - - with ops.device(device1): - prefetch_op = prefetching_ops.function_buffering_resource_get_next( - function_buffer_resource=buffer_resource_handle, - output_types=[dtypes.float32]) - reset_op = prefetching_ops.function_buffering_resource_reset( - function_buffer_resource=buffer_resource_handle) - destroy_op = resource_variable_ops.destroy_resource_op( - buffer_resource_handle, ignore_lookup_error=True) - - return (prefetch_op, reset_op, destroy_op) - - def _prefetch_fn_helper_one_shot(self, buffer_name, device0, device1): - worker_config = config_pb2.ConfigProto(device_count={"CPU": 2}) - - ds, ds_iterator = self._create_ds_and_iterator(device0, initializable=False) - prefetch_op, _, destroy_op = self._create_ops(ds, ds_iterator, buffer_name, - device0, device1) - - with self.test_session(config=worker_config) as sess: - elem = sess.run(prefetch_op) - self.assertEqual(elem, [1.0]) - elem = sess.run(prefetch_op) - self.assertEqual(elem, [2.0]) - elem = sess.run(prefetch_op) - self.assertEqual(elem, [3.0]) - elem = sess.run(prefetch_op) - self.assertEqual(elem, [4.0]) - self._event.wait() - elem = sess.run(prefetch_op) - self.assertEqual(elem, [5.0]) - sess.run(destroy_op) - - def testSameDeviceCPU(self): - self._prefetch_fn_helper_one_shot("same_device_cpu", - "/job:localhost/replica:0/task:0/cpu:0", - "/job:localhost/replica:0/task:0/cpu:0") - - def testDifferentDeviceCPU(self): - self._prefetch_fn_helper_one_shot("diff_device_cpu", - "/job:localhost/replica:0/task:0/cpu:0", - "/job:localhost/replica:0/task:0/cpu:1") - - def testDifferentDeviceCPUGPU(self): - if not test_util.is_gpu_available(): - self.skipTest("No GPU available") - - self._prefetch_fn_helper_one_shot("cpu_gpu", - "/job:localhost/replica:0/task:0/cpu:0", - "/job:localhost/replica:0/task:0/gpu:0") - - def testReinitialization(self): - worker_config = config_pb2.ConfigProto(device_count={"CPU": 2}) - - device0 = "/job:localhost/replica:0/task:0/cpu:0" - device1 = "/job:localhost/replica:0/task:0/cpu:1" - ds, ds_iterator = self._create_ds_and_iterator(device0, initializable=True) - prefetch_op, reset_op, destroy_op = self._create_ops( - ds, ds_iterator, "reinit", device0, device1) - - with self.test_session(config=worker_config) as sess: - sess.run(ds_iterator.initializer) - elem = sess.run(prefetch_op) - self.assertEqual(elem, [1.0]) - elem = sess.run(prefetch_op) - self.assertEqual(elem, [2.0]) - elem = sess.run(prefetch_op) - self.assertEqual(elem, [3.0]) - elem = sess.run(prefetch_op) - self.assertEqual(elem, [4.0]) - self._event.wait() - elem = sess.run(prefetch_op) - self.assertEqual(elem, [5.0]) - # Lets reset the function buffering resource and reinitialize the - # iterator. Should be able to go through this again. - self._event.clear() - sess.run(reset_op) - sess.run(ds_iterator.initializer) - elem = sess.run(prefetch_op) - self.assertEqual(elem, [1.0]) - elem = sess.run(prefetch_op) - self.assertEqual(elem, [2.0]) - elem = sess.run(prefetch_op) - self.assertEqual(elem, [3.0]) - elem = sess.run(prefetch_op) - self.assertEqual(elem, [4.0]) - self._event.wait() - elem = sess.run(prefetch_op) - self.assertEqual(elem, [5.0]) - sess.run(destroy_op) - - def testReinitializationOutOfRange(self): - worker_config = config_pb2.ConfigProto(device_count={"CPU": 2}) - - device0 = "/job:localhost/replica:0/task:0/cpu:0" - device1 = "/job:localhost/replica:0/task:0/cpu:1" - ds, ds_iterator = self._create_ds_and_iterator(device0, initializable=True) - prefetch_op, reset_op, destroy_op = self._create_ops( - ds, ds_iterator, "reinit", device0, device1) - - with self.test_session(config=worker_config) as sess: - sess.run(ds_iterator.initializer) - for i in range(1, 10): - elem = sess.run(prefetch_op) - self.assertEqual(elem, [float(i)]) - # Try fetching after its over twice to test out end of sequence. - with self.assertRaises(errors.OutOfRangeError): - sess.run(prefetch_op) - with self.assertRaises(errors.OutOfRangeError): - sess.run(prefetch_op) - - # Now reset everything and try it out again. - self._event.clear() - sess.run(reset_op) - sess.run(ds_iterator.initializer) - for i in range(1, 10): - elem = sess.run(prefetch_op) - self.assertEqual(elem, [float(i)]) - # Try fetching after its over twice to test out end of sequence. - with self.assertRaises(errors.OutOfRangeError): - sess.run(prefetch_op) - with self.assertRaises(errors.OutOfRangeError): - sess.run(prefetch_op) - - sess.run(destroy_op) - - def testStringsGPU(self): - if not test_util.is_gpu_available(): - self.skipTest("No GPU available") - - device0 = "/job:localhost/replica:0/task:0/cpu:0" - device1 = "/job:localhost/replica:0/task:0/gpu:0" - - ds = dataset_ops.Dataset.from_tensor_slices(["a", "b", "c"]) - ds_iterator = ds.make_one_shot_iterator() - ds_iterator_handle = ds_iterator.string_handle() - - @function.defun(input_signature=[tensor_spec.TensorSpec([], dtypes.string)]) - def _remote_fn(h): - remote_iterator = iterator_ops.Iterator.from_string_handle( - h, ds.output_types, ds.output_shapes) - return remote_iterator.get_next() - - target = constant_op.constant(device0) - with ops.device(device1): - buffer_resource_handle = prefetching_ops.function_buffering_resource( - f=_remote_fn.get_concrete_function(), - output_types=[dtypes.string], - target_device=target, - string_arg=ds_iterator_handle, - buffer_size=3, - shared_name="strings") - - with ops.device(device1): - prefetch_op = prefetching_ops.function_buffering_resource_get_next( - function_buffer_resource=buffer_resource_handle, - output_types=[dtypes.string]) - destroy_op = resource_variable_ops.destroy_resource_op( - buffer_resource_handle, ignore_lookup_error=True) - - with self.cached_session() as sess: - self.assertEqual([b"a"], sess.run(prefetch_op)) - self.assertEqual([b"b"], sess.run(prefetch_op)) - self.assertEqual([b"c"], sess.run(prefetch_op)) - with self.assertRaises(errors.OutOfRangeError): - sess.run(prefetch_op) - - sess.run(destroy_op) - - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/python/data/experimental/kernel_tests/get_single_element_test.py b/tensorflow/python/data/experimental/kernel_tests/get_single_element_test.py index 8c07afbac57..ef576563a15 100644 --- a/tensorflow/python/data/experimental/kernel_tests/get_single_element_test.py +++ b/tensorflow/python/data/experimental/kernel_tests/get_single_element_test.py @@ -25,6 +25,7 @@ from tensorflow.python.data.ops import dataset_ops from tensorflow.python.framework import dtypes from tensorflow.python.framework import errors from tensorflow.python.framework import sparse_tensor +from tensorflow.python.framework import test_util from tensorflow.python.ops import array_ops from tensorflow.python.platform import test @@ -39,6 +40,7 @@ class GetSingleElementTest(test_base.DatasetTestBase, parameterized.TestCase): ("MoreThanOne", 0, 2, errors.InvalidArgumentError, "Dataset had more than one element."), ) + @test_util.run_deprecated_v1 def testGetSingleElement(self, skip, take, error=None, error_msg=None): skip_t = array_ops.placeholder(dtypes.int64, shape=[]) take_t = array_ops.placeholder(dtypes.int64, shape=[]) @@ -67,6 +69,17 @@ class GetSingleElementTest(test_base.DatasetTestBase, parameterized.TestCase): with self.assertRaisesRegexp(error, error_msg): sess.run(element, feed_dict={skip_t: skip, take_t: take}) + def testWindow(self): + """Test that `get_single_element()` can consume a nested dataset.""" + def flat_map_func(ds): + batched = ds.batch(2) + element = get_single_element.get_single_element(batched) + return dataset_ops.Dataset.from_tensors(element) + + dataset = dataset_ops.Dataset.range(10).window(2).flat_map(flat_map_func) + self.assertDatasetProduces( + dataset, [[0, 1], [2, 3], [4, 5], [6, 7], [8, 9]]) + if __name__ == "__main__": test.main() diff --git a/tensorflow/python/data/experimental/kernel_tests/group_by_reducer_test.py b/tensorflow/python/data/experimental/kernel_tests/group_by_reducer_test.py index 90303285931..8507df3d3a2 100644 --- a/tensorflow/python/data/experimental/kernel_tests/group_by_reducer_test.py +++ b/tensorflow/python/data/experimental/kernel_tests/group_by_reducer_test.py @@ -27,6 +27,7 @@ from tensorflow.python.framework import dtypes from tensorflow.python.framework import errors from tensorflow.python.framework import sparse_tensor from tensorflow.python.framework import tensor_shape +from tensorflow.python.framework import test_util from tensorflow.python.ops import array_ops from tensorflow.python.ops import math_ops from tensorflow.python.platform import test @@ -36,14 +37,15 @@ class GroupByReducerTest(test_base.DatasetTestBase): def checkResults(self, dataset, shapes, values): self.assertEqual(shapes, dataset.output_shapes) - get_next = dataset.make_one_shot_iterator().get_next() + get_next = dataset_ops.make_one_shot_iterator(dataset).get_next() with self.cached_session() as sess: for expected in values: - got = sess.run(get_next) + got = self.evaluate(get_next) self.assertEqual(got, expected) with self.assertRaises(errors.OutOfRangeError): - sess.run(get_next) + self.evaluate(get_next) + @test_util.run_deprecated_v1 def testSum(self): reducer = grouping.Reducer( init_func=lambda _: np.int64(0), @@ -55,6 +57,7 @@ class GroupByReducerTest(test_base.DatasetTestBase): self.checkResults( dataset, shapes=tensor_shape.scalar(), values=[(i - 1) * i, i * i]) + @test_util.run_deprecated_v1 def testAverage(self): def reduce_fn(x, y): @@ -72,6 +75,7 @@ class GroupByReducerTest(test_base.DatasetTestBase): self.checkResults( dataset, shapes=tensor_shape.scalar(), values=[i - 1, i]) + @test_util.run_deprecated_v1 def testConcat(self): components = np.array(list("abcdefghijklmnopqrst")).view(np.chararray) reducer = grouping.Reducer( @@ -88,6 +92,7 @@ class GroupByReducerTest(test_base.DatasetTestBase): shapes=tensor_shape.scalar(), values=[b"acegikmoqs" [:i], b"bdfhjlnprt" [:i]]) + @test_util.run_deprecated_v1 def testSparseSum(self): def _sparse(i): return sparse_tensor.SparseTensorValue( @@ -105,6 +110,7 @@ class GroupByReducerTest(test_base.DatasetTestBase): self.checkResults( dataset, shapes=tensor_shape.scalar(), values=[(i - 1) * i, i * i]) + @test_util.run_deprecated_v1 def testChangingStateShape(self): def reduce_fn(x, _): @@ -124,14 +130,14 @@ class GroupByReducerTest(test_base.DatasetTestBase): grouping.group_by_reducer(lambda x: x, reducer)) self.assertEqual([None], dataset.output_shapes[0].as_list()) self.assertIs(None, dataset.output_shapes[1].ndims) - iterator = dataset.make_one_shot_iterator() + iterator = dataset_ops.make_one_shot_iterator(dataset) get_next = iterator.get_next() with self.cached_session() as sess: - x, y = sess.run(get_next) + x, y = self.evaluate(get_next) self.assertAllEqual([0] * (2**i), x) self.assertAllEqual(np.array(1, ndmin=i), y) with self.assertRaises(errors.OutOfRangeError): - sess.run(get_next) + self.evaluate(get_next) def testTypeMismatch(self): reducer = grouping.Reducer( @@ -188,9 +194,9 @@ class GroupByReducerTest(test_base.DatasetTestBase): dataset = dataset_ops.Dataset.zip( (dataset_ops.Dataset.range(10), dataset_ops.Dataset.range(10))).apply( grouping.group_by_reducer(lambda x, y: np.int64(0), reducer)) - get_next = dataset.make_one_shot_iterator().get_next() + get_next = dataset_ops.make_one_shot_iterator(dataset).get_next() with self.cached_session() as sess: - x, y = sess.run(get_next) + x, y = self.evaluate(get_next) self.assertAllEqual(x, np.asarray([x for x in range(10)])) self.assertEqual(y, 45) diff --git a/tensorflow/python/data/experimental/kernel_tests/group_by_window_test.py b/tensorflow/python/data/experimental/kernel_tests/group_by_window_test.py index 557d56e8b9a..cbb79e55f50 100644 --- a/tensorflow/python/data/experimental/kernel_tests/group_by_window_test.py +++ b/tensorflow/python/data/experimental/kernel_tests/group_by_window_test.py @@ -27,6 +27,7 @@ from tensorflow.python.framework import dtypes from tensorflow.python.framework import errors from tensorflow.python.framework import ops from tensorflow.python.framework import tensor_shape +from tensorflow.python.framework import test_util from tensorflow.python.ops import array_ops from tensorflow.python.ops import math_ops from tensorflow.python.ops import string_ops @@ -49,6 +50,7 @@ class GroupByWindowTest(test_base.DatasetTestBase): 32, (tensor_shape.TensorShape([]), tensor_shape.TensorShape( [None]), tensor_shape.TensorShape([3]))))) + @test_util.run_deprecated_v1 def testSingleBucket(self): def _map_fn(v): @@ -63,14 +65,14 @@ class GroupByWindowTest(test_base.DatasetTestBase): lambda x, y, z: 0, lambda k, bucket: self._dynamicPad(k, bucket, 32), 32)) - iterator = bucketed_dataset.make_initializable_iterator() + iterator = dataset_ops.make_initializable_iterator(bucketed_dataset) init_op = iterator.initializer get_next = iterator.get_next() with self.cached_session() as sess: - sess.run(init_op) + self.evaluate(init_op) - which_bucket, bucketed_values = sess.run(get_next) + which_bucket, bucketed_values = self.evaluate(get_next) self.assertEqual(0, which_bucket) @@ -84,6 +86,7 @@ class GroupByWindowTest(test_base.DatasetTestBase): self.assertAllEqual(expected_unk_int64, bucketed_values[1]) self.assertAllEqual(expected_vec3_str, bucketed_values[2]) + @test_util.run_deprecated_v1 def testEvenOddBuckets(self): def _map_fn(v): @@ -98,16 +101,16 @@ class GroupByWindowTest(test_base.DatasetTestBase): lambda x, y, z: math_ops.cast(x % 2, dtypes.int64), lambda k, bucket: self._dynamicPad(k, bucket, 32), 32)) - iterator = bucketed_dataset.make_initializable_iterator() + iterator = dataset_ops.make_initializable_iterator(bucketed_dataset) init_op = iterator.initializer get_next = iterator.get_next() with self.cached_session() as sess: - sess.run(init_op) + self.evaluate(init_op) # Get two minibatches (one containing even values, one containing odds) - which_bucket_even, bucketed_values_even = sess.run(get_next) - which_bucket_odd, bucketed_values_odd = sess.run(get_next) + which_bucket_even, bucketed_values_even = self.evaluate(get_next) + which_bucket_odd, bucketed_values_odd = self.evaluate(get_next) # Count number of bucket_tensors. self.assertEqual(3, len(bucketed_values_even)) @@ -141,6 +144,7 @@ class GroupByWindowTest(test_base.DatasetTestBase): self.assertAllEqual(expected_unk_int64, bucketed_values_odd[1]) self.assertAllEqual(expected_vec3_str, bucketed_values_odd[2]) + @test_util.run_deprecated_v1 def testEvenOddBucketsFilterOutAllOdd(self): def _map_fn(v): @@ -169,16 +173,16 @@ class GroupByWindowTest(test_base.DatasetTestBase): lambda d: math_ops.cast(d["x"] % 2, dtypes.int64), lambda k, bucket: _dynamic_pad_fn(k, bucket, 32), 32)) - iterator = bucketed_dataset.make_initializable_iterator() + iterator = dataset_ops.make_initializable_iterator(bucketed_dataset) init_op = iterator.initializer get_next = iterator.get_next() with self.cached_session() as sess: - sess.run(init_op) + self.evaluate(init_op) # Get two minibatches ([0, 2, ...] and [64, 66, ...]) - which_bucket0, bucketed_values_even0 = sess.run(get_next) - which_bucket1, bucketed_values_even1 = sess.run(get_next) + which_bucket0, bucketed_values_even0 = self.evaluate(get_next) + which_bucket1, bucketed_values_even1 = self.evaluate(get_next) # Ensure that bucket 1 was completely filtered out self.assertAllEqual(0, which_bucket0) @@ -188,6 +192,7 @@ class GroupByWindowTest(test_base.DatasetTestBase): self.assertAllEqual( np.arange(64, 128, 2, dtype=np.int64), bucketed_values_even1["x"]) + @test_util.run_deprecated_v1 def testDynamicWindowSize(self): components = np.arange(100).astype(np.int64) @@ -202,16 +207,16 @@ class GroupByWindowTest(test_base.DatasetTestBase): dataset = dataset_ops.Dataset.from_tensor_slices(components).apply( grouping.group_by_window(lambda x: x % 2, lambda _, xs: xs.batch(20), None, window_size_func)) - iterator = dataset.make_initializable_iterator() + iterator = dataset_ops.make_initializable_iterator(dataset) init_op = iterator.initializer get_next = iterator.get_next() with self.cached_session() as sess: - sess.run(init_op) + self.evaluate(init_op) with self.assertRaises(errors.OutOfRangeError): batches = 0 while True: - result = sess.run(get_next) + result = self.evaluate(get_next) is_even = all(x % 2 == 0 for x in result) is_odd = all(x % 2 == 1 for x in result) self.assertTrue(is_even or is_odd) @@ -221,22 +226,23 @@ class GroupByWindowTest(test_base.DatasetTestBase): self.assertEqual(batches, 15) + @test_util.run_deprecated_v1 def testSimple(self): components = np.random.randint(100, size=(200,)).astype(np.int64) - iterator = ( + iterator = dataset_ops.make_initializable_iterator( dataset_ops.Dataset.from_tensor_slices(components).map(lambda x: x * x) .apply( grouping.group_by_window(lambda x: x % 2, lambda _, xs: xs.batch(4), - 4)).make_initializable_iterator()) + 4))) init_op = iterator.initializer get_next = iterator.get_next() with self.cached_session() as sess: - sess.run(init_op) + self.evaluate(init_op) counts = [] with self.assertRaises(errors.OutOfRangeError): while True: - result = sess.run(get_next) + result = self.evaluate(get_next) self.assertTrue( all(x % 2 == 0 for x in result) or all(x % 2 == 1) @@ -248,61 +254,64 @@ class GroupByWindowTest(test_base.DatasetTestBase): self.assertGreaterEqual(num_full_batches, 24) self.assertTrue(all(c == 4 for c in counts[:num_full_batches])) + @test_util.run_deprecated_v1 def testImmediateOutput(self): components = np.array( [0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 0, 0, 2, 2, 0, 0], dtype=np.int64) - iterator = ( + iterator = dataset_ops.make_initializable_iterator( dataset_ops.Dataset.from_tensor_slices(components).repeat(-1).apply( grouping.group_by_window(lambda x: x % 3, lambda _, xs: xs.batch(4), - 4)).make_initializable_iterator()) + 4))) init_op = iterator.initializer get_next = iterator.get_next() with self.cached_session() as sess: - sess.run(init_op) + self.evaluate(init_op) # The input is infinite, so this test demonstrates that: # 1. We produce output without having to consume the entire input, # 2. Different buckets can produce output at different rates, and # 3. For deterministic input, the output is deterministic. for _ in range(3): - self.assertAllEqual([0, 0, 0, 0], sess.run(get_next)) - self.assertAllEqual([1, 1, 1, 1], sess.run(get_next)) - self.assertAllEqual([2, 2, 2, 2], sess.run(get_next)) - self.assertAllEqual([0, 0, 0, 0], sess.run(get_next)) + self.assertAllEqual([0, 0, 0, 0], self.evaluate(get_next)) + self.assertAllEqual([1, 1, 1, 1], self.evaluate(get_next)) + self.assertAllEqual([2, 2, 2, 2], self.evaluate(get_next)) + self.assertAllEqual([0, 0, 0, 0], self.evaluate(get_next)) + @test_util.run_deprecated_v1 def testSmallGroups(self): components = np.array([0, 0, 0, 0, 1, 1, 1, 1, 1, 0, 0, 0], dtype=np.int64) - iterator = ( + iterator = dataset_ops.make_initializable_iterator( dataset_ops.Dataset.from_tensor_slices(components).apply( grouping.group_by_window(lambda x: x % 2, lambda _, xs: xs.batch(4), - 4)).make_initializable_iterator()) + 4))) init_op = iterator.initializer get_next = iterator.get_next() with self.cached_session() as sess: - sess.run(init_op) - self.assertAllEqual([0, 0, 0, 0], sess.run(get_next)) - self.assertAllEqual([1, 1, 1, 1], sess.run(get_next)) + self.evaluate(init_op) + self.assertAllEqual([0, 0, 0, 0], self.evaluate(get_next)) + self.assertAllEqual([1, 1, 1, 1], self.evaluate(get_next)) # The small outputs at the end are deterministically produced in key # order. - self.assertAllEqual([0, 0, 0], sess.run(get_next)) - self.assertAllEqual([1], sess.run(get_next)) + self.assertAllEqual([0, 0, 0], self.evaluate(get_next)) + self.assertAllEqual([1], self.evaluate(get_next)) + @test_util.run_deprecated_v1 def testEmpty(self): - iterator = ( + iterator = dataset_ops.make_initializable_iterator( dataset_ops.Dataset.range(4).apply( - grouping.group_by_window(lambda _: 0, lambda _, xs: xs, 0)) - .make_initializable_iterator()) + grouping.group_by_window(lambda _: 0, lambda _, xs: xs, 0))) init_op = iterator.initializer get_next = iterator.get_next() with self.cached_session() as sess: - sess.run(init_op) + self.evaluate(init_op) with self.assertRaisesRegexp( errors.InvalidArgumentError, "Window size must be greater than zero, but got 0."): - print(sess.run(get_next)) + print(self.evaluate(get_next)) + @test_util.run_deprecated_v1 def testReduceFuncError(self): components = np.random.randint(100, size=(200,)).astype(np.int64) @@ -314,19 +323,19 @@ class GroupByWindowTest(test_base.DatasetTestBase): padded_shapes=(tensor_shape.TensorShape([]), constant_op.constant([5], dtype=dtypes.int64) * -1)) - iterator = ( + iterator = dataset_ops.make_initializable_iterator( dataset_ops.Dataset.from_tensor_slices(components) .map(lambda x: (x, ops.convert_to_tensor([x * x]))).apply( - grouping.group_by_window(lambda x, _: x % 2, reduce_func, - 32)).make_initializable_iterator()) + grouping.group_by_window(lambda x, _: x % 2, reduce_func, 32))) init_op = iterator.initializer get_next = iterator.get_next() with self.cached_session() as sess: - sess.run(init_op) + self.evaluate(init_op) with self.assertRaises(errors.InvalidArgumentError): - sess.run(get_next) + self.evaluate(get_next) + @test_util.run_deprecated_v1 def testConsumeWindowDatasetMoreThanOnce(self): components = np.random.randint(50, size=(200,)).astype(np.int64) @@ -340,22 +349,21 @@ class GroupByWindowTest(test_base.DatasetTestBase): 4, padded_shapes=ops.convert_to_tensor([(key + 1) * 10])), )) - iterator = ( + iterator = dataset_ops.make_initializable_iterator( dataset_ops.Dataset.from_tensor_slices(components) .map(lambda x: array_ops.fill([math_ops.cast(x, dtypes.int32)], x)) .apply(grouping.group_by_window( lambda x: math_ops.cast(array_ops.shape(x)[0] // 10, dtypes.int64), - reduce_func, 4)) - .make_initializable_iterator()) + reduce_func, 4))) init_op = iterator.initializer get_next = iterator.get_next() with self.cached_session() as sess: - sess.run(init_op) + self.evaluate(init_op) counts = [] with self.assertRaises(errors.OutOfRangeError): while True: - tight_result, multiple_of_10_result = sess.run(get_next) + tight_result, multiple_of_10_result = self.evaluate(get_next) self.assertEqual(0, multiple_of_10_result.shape[1] % 10) self.assertAllEqual(tight_result, multiple_of_10_result[:, :tight_result.shape[1]]) diff --git a/tensorflow/python/data/experimental/kernel_tests/ignore_errors_test.py b/tensorflow/python/data/experimental/kernel_tests/ignore_errors_test.py index c0ec1486ab8..81f580fccbd 100644 --- a/tensorflow/python/data/experimental/kernel_tests/ignore_errors_test.py +++ b/tensorflow/python/data/experimental/kernel_tests/ignore_errors_test.py @@ -25,6 +25,7 @@ from tensorflow.python.data.experimental.ops import error_ops from tensorflow.python.data.kernel_tests import test_base from tensorflow.python.data.ops import dataset_ops from tensorflow.python.framework import errors +from tensorflow.python.framework import test_util from tensorflow.python.ops import array_ops from tensorflow.python.ops import io_ops from tensorflow.python.platform import test @@ -35,6 +36,7 @@ _NUMPY_RANDOM_SEED = 42 class IgnoreErrorsTest(test_base.DatasetTestBase): + @test_util.run_deprecated_v1 def testMapIgnoreError(self): components = np.array([1., 2., 3., np.nan, 5.]).astype(np.float32) @@ -42,17 +44,18 @@ class IgnoreErrorsTest(test_base.DatasetTestBase): dataset_ops.Dataset.from_tensor_slices(components) .map(lambda x: array_ops.check_numerics(x, "message")).apply( error_ops.ignore_errors())) - iterator = dataset.make_initializable_iterator() + iterator = dataset_ops.make_initializable_iterator(dataset) init_op = iterator.initializer get_next = iterator.get_next() with self.cached_session() as sess: - sess.run(init_op) + self.evaluate(init_op) for x in [1., 2., 3., 5.]: - self.assertEqual(x, sess.run(get_next)) + self.assertEqual(x, self.evaluate(get_next)) with self.assertRaises(errors.OutOfRangeError): - sess.run(get_next) + self.evaluate(get_next) + @test_util.run_deprecated_v1 def testParallelMapIgnoreError(self): components = np.array([1., 2., 3., np.nan, 5.]).astype(np.float32) @@ -60,17 +63,18 @@ class IgnoreErrorsTest(test_base.DatasetTestBase): dataset_ops.Dataset.from_tensor_slices(components).map( lambda x: array_ops.check_numerics(x, "message"), num_parallel_calls=2).prefetch(2).apply(error_ops.ignore_errors())) - iterator = dataset.make_initializable_iterator() + iterator = dataset_ops.make_initializable_iterator(dataset) init_op = iterator.initializer get_next = iterator.get_next() with self.cached_session() as sess: - sess.run(init_op) + self.evaluate(init_op) for x in [1., 2., 3., 5.]: - self.assertEqual(x, sess.run(get_next)) + self.assertEqual(x, self.evaluate(get_next)) with self.assertRaises(errors.OutOfRangeError): - sess.run(get_next) + self.evaluate(get_next) + @test_util.run_deprecated_v1 def testReadFileIgnoreError(self): def write_string_to_file(value, filename): @@ -87,28 +91,28 @@ class IgnoreErrorsTest(test_base.DatasetTestBase): dataset_ops.Dataset.from_tensor_slices(filenames).map( io_ops.read_file, num_parallel_calls=2).prefetch(2).apply(error_ops.ignore_errors())) - iterator = dataset.make_initializable_iterator() + iterator = dataset_ops.make_initializable_iterator(dataset) init_op = iterator.initializer get_next = iterator.get_next() with self.cached_session() as sess: # All of the files are present. - sess.run(init_op) + self.evaluate(init_op) for filename in filenames: - self.assertEqual(compat.as_bytes(filename), sess.run(get_next)) + self.assertEqual(compat.as_bytes(filename), self.evaluate(get_next)) with self.assertRaises(errors.OutOfRangeError): - sess.run(get_next) + self.evaluate(get_next) # Delete one of the files. os.remove(filenames[0]) # Attempting to read filenames[0] will fail, but ignore_errors() # will catch the error. - sess.run(init_op) + self.evaluate(init_op) for filename in filenames[1:]: - self.assertEqual(compat.as_bytes(filename), sess.run(get_next)) + self.assertEqual(compat.as_bytes(filename), self.evaluate(get_next)) with self.assertRaises(errors.OutOfRangeError): - sess.run(get_next) + self.evaluate(get_next) if __name__ == "__main__": diff --git a/tensorflow/python/data/experimental/kernel_tests/indexed_dataset_ops_test.py b/tensorflow/python/data/experimental/kernel_tests/indexed_dataset_ops_test.py index c93a8353ce0..c3c4ccd0770 100644 --- a/tensorflow/python/data/experimental/kernel_tests/indexed_dataset_ops_test.py +++ b/tensorflow/python/data/experimental/kernel_tests/indexed_dataset_ops_test.py @@ -24,6 +24,7 @@ from tensorflow.python.data.kernel_tests import test_base from tensorflow.python.framework import dtypes from tensorflow.python.framework import errors from tensorflow.python.framework import ops +from tensorflow.python.framework import test_util from tensorflow.python.ops import array_ops from tensorflow.python.ops import gen_experimental_dataset_ops as ged_ops from tensorflow.python.platform import test @@ -31,6 +32,7 @@ from tensorflow.python.platform import test class IndexedDatasetOpsTest(test_base.DatasetTestBase): + @test_util.run_deprecated_v1 def testLowLevelIndexedDatasetOps(self): identity = ged_ops.experimental_identity_indexed_dataset( ops.convert_to_tensor(16, dtype=dtypes.uint64)) @@ -46,14 +48,15 @@ class IndexedDatasetOpsTest(test_base.DatasetTestBase): handle, index, output_types=[dtypes.uint64], output_shapes=[[]]) with self.cached_session() as sess: - sess.run(materialize) + self.evaluate(materialize) self.assertEqual([3], sess.run(get_op, feed_dict={index: 3})) + @test_util.run_deprecated_v1 def testIdentityIndexedDataset(self): ds = indexed_dataset_ops.IdentityIndexedDataset(16) materialized = ds.materialize() with self.cached_session() as sess: - sess.run(materialized.initializer) + self.evaluate(materialized.initializer) placeholder = array_ops.placeholder(dtypes.uint64, shape=[]) for i in range(16): output = sess.run( @@ -68,12 +71,13 @@ class IndexedDatasetOpsTest(test_base.DatasetTestBase): itr = ds.make_initializable_iterator() n = itr.get_next() with self.cached_session() as sess: - sess.run(itr.initializer) + self.evaluate(itr.initializer) for i in range(16): - output = sess.run(n) + output = self.evaluate(n) self.assertEqual(i, output) with self.assertRaises(errors.OutOfRangeError): - sess.run(n) + self.evaluate(n) + if __name__ == "__main__": test.main() diff --git a/tensorflow/python/data/experimental/kernel_tests/make_batched_features_dataset_test.py b/tensorflow/python/data/experimental/kernel_tests/make_batched_features_dataset_test.py index 91ae8cb1bd2..7c788104948 100644 --- a/tensorflow/python/data/experimental/kernel_tests/make_batched_features_dataset_test.py +++ b/tensorflow/python/data/experimental/kernel_tests/make_batched_features_dataset_test.py @@ -21,11 +21,13 @@ import numpy as np from tensorflow.python.data.experimental.kernel_tests import reader_dataset_ops_test_base from tensorflow.python.data.experimental.ops import readers +from tensorflow.python.data.ops import dataset_ops from tensorflow.python.data.ops import readers as core_readers from tensorflow.python.data.util import nest from tensorflow.python.framework import dtypes from tensorflow.python.framework import errors from tensorflow.python.framework import ops +from tensorflow.python.framework import test_util from tensorflow.python.ops import io_ops from tensorflow.python.ops import parsing_ops from tensorflow.python.platform import test @@ -40,11 +42,12 @@ class MakeBatchedFeaturesDatasetTest( with ops.Graph().as_default() as g: with self.session(graph=g) as sess: # Basic test: read from file 0. - self.outputs = self.make_batch_feature( - filenames=self.test_filenames[0], - label_key="label", - num_epochs=num_epochs, - batch_size=batch_size).make_one_shot_iterator().get_next() + self.outputs = dataset_ops.make_one_shot_iterator( + self.make_batch_feature( + filenames=self.test_filenames[0], + label_key="label", + num_epochs=num_epochs, + batch_size=batch_size)).get_next() self.verify_records( sess, batch_size, @@ -57,11 +60,12 @@ class MakeBatchedFeaturesDatasetTest( with ops.Graph().as_default() as g: with self.session(graph=g) as sess: # Basic test: read from file 1. - self.outputs = self.make_batch_feature( - filenames=self.test_filenames[1], - label_key="label", - num_epochs=num_epochs, - batch_size=batch_size).make_one_shot_iterator().get_next() + self.outputs = dataset_ops.make_one_shot_iterator( + self.make_batch_feature( + filenames=self.test_filenames[1], + label_key="label", + num_epochs=num_epochs, + batch_size=batch_size)).get_next() self.verify_records( sess, batch_size, @@ -74,11 +78,12 @@ class MakeBatchedFeaturesDatasetTest( with ops.Graph().as_default() as g: with self.session(graph=g) as sess: # Basic test: read from both files. - self.outputs = self.make_batch_feature( - filenames=self.test_filenames, - label_key="label", - num_epochs=num_epochs, - batch_size=batch_size).make_one_shot_iterator().get_next() + self.outputs = dataset_ops.make_one_shot_iterator( + self.make_batch_feature( + filenames=self.test_filenames, + label_key="label", + num_epochs=num_epochs, + batch_size=batch_size)).get_next() self.verify_records( sess, batch_size, @@ -90,14 +95,16 @@ class MakeBatchedFeaturesDatasetTest( with ops.Graph().as_default() as g: with self.session(graph=g) as sess: # Basic test: read from both files. - self.outputs = self.make_batch_feature( - filenames=self.test_filenames, - num_epochs=num_epochs, - batch_size=batch_size).make_one_shot_iterator().get_next() + self.outputs = dataset_ops.make_one_shot_iterator( + self.make_batch_feature( + filenames=self.test_filenames, + num_epochs=num_epochs, + batch_size=batch_size)).get_next() self.verify_records(sess, batch_size, num_epochs=num_epochs) with self.assertRaises(errors.OutOfRangeError): self._next_actual_batch(sess) + @test_util.run_deprecated_v1 def testReadWithEquivalentDataset(self): features = { "file": parsing_ops.FixedLenFeature([], dtypes.int64), @@ -107,19 +114,19 @@ class MakeBatchedFeaturesDatasetTest( core_readers.TFRecordDataset(self.test_filenames) .map(lambda x: parsing_ops.parse_single_example(x, features)) .repeat(10).batch(2)) - iterator = dataset.make_initializable_iterator() + iterator = dataset_ops.make_initializable_iterator(dataset) init_op = iterator.initializer next_element = iterator.get_next() with self.cached_session() as sess: - sess.run(init_op) + self.evaluate(init_op) for file_batch, _, _, _, record_batch, _ in self._next_expected_batch( range(self._num_files), 2, 10): - actual_batch = sess.run(next_element) + actual_batch = self.evaluate(next_element) self.assertAllEqual(file_batch, actual_batch["file"]) self.assertAllEqual(record_batch, actual_batch["record"]) with self.assertRaises(errors.OutOfRangeError): - sess.run(next_element) + self.evaluate(next_element) def testReadWithFusedShuffleRepeatDataset(self): num_epochs = 5 @@ -128,18 +135,18 @@ class MakeBatchedFeaturesDatasetTest( # Test that shuffling with same seed produces the same result. with ops.Graph().as_default() as g: with self.session(graph=g) as sess: - outputs1 = self.make_batch_feature( + outputs1 = dataset_ops.make_one_shot_iterator(self.make_batch_feature( filenames=self.test_filenames[0], num_epochs=num_epochs, batch_size=batch_size, shuffle=True, - shuffle_seed=5).make_one_shot_iterator().get_next() - outputs2 = self.make_batch_feature( + shuffle_seed=5)).get_next() + outputs2 = dataset_ops.make_one_shot_iterator(self.make_batch_feature( filenames=self.test_filenames[0], num_epochs=num_epochs, batch_size=batch_size, shuffle=True, - shuffle_seed=5).make_one_shot_iterator().get_next() + shuffle_seed=5)).get_next() for _ in range(total_records // batch_size): batch1 = self._run_actual_batch(outputs1, sess) batch2 = self._run_actual_batch(outputs2, sess) @@ -149,18 +156,18 @@ class MakeBatchedFeaturesDatasetTest( # Test that shuffling with different seeds produces a different order. with ops.Graph().as_default() as g: with self.session(graph=g) as sess: - outputs1 = self.make_batch_feature( + outputs1 = dataset_ops.make_one_shot_iterator(self.make_batch_feature( filenames=self.test_filenames[0], num_epochs=num_epochs, batch_size=batch_size, shuffle=True, - shuffle_seed=5).make_one_shot_iterator().get_next() - outputs2 = self.make_batch_feature( + shuffle_seed=5)).get_next() + outputs2 = dataset_ops.make_one_shot_iterator(self.make_batch_feature( filenames=self.test_filenames[0], num_epochs=num_epochs, batch_size=batch_size, shuffle=True, - shuffle_seed=15).make_one_shot_iterator().get_next() + shuffle_seed=15)).get_next() all_equal = True for _ in range(total_records // batch_size): batch1 = self._run_actual_batch(outputs1, sess) @@ -176,14 +183,14 @@ class MakeBatchedFeaturesDatasetTest( for parser_num_threads in [2, 4]: with ops.Graph().as_default() as g: with self.session(graph=g) as sess: - self.outputs = self.make_batch_feature( - filenames=self.test_filenames, - label_key="label", - num_epochs=num_epochs, - batch_size=batch_size, - reader_num_threads=reader_num_threads, - parser_num_threads=parser_num_threads).make_one_shot_iterator( - ).get_next() + self.outputs = dataset_ops.make_one_shot_iterator( + self.make_batch_feature( + filenames=self.test_filenames, + label_key="label", + num_epochs=num_epochs, + batch_size=batch_size, + reader_num_threads=reader_num_threads, + parser_num_threads=parser_num_threads)).get_next() self.verify_records( sess, batch_size, @@ -195,13 +202,13 @@ class MakeBatchedFeaturesDatasetTest( with ops.Graph().as_default() as g: with self.session(graph=g) as sess: - self.outputs = self.make_batch_feature( - filenames=self.test_filenames, - num_epochs=num_epochs, - batch_size=batch_size, - reader_num_threads=reader_num_threads, - parser_num_threads=parser_num_threads).make_one_shot_iterator( - ).get_next() + self.outputs = dataset_ops.make_one_shot_iterator( + self.make_batch_feature( + filenames=self.test_filenames, + num_epochs=num_epochs, + batch_size=batch_size, + reader_num_threads=reader_num_threads, + parser_num_threads=parser_num_threads)).get_next() self.verify_records( sess, batch_size, @@ -215,12 +222,12 @@ class MakeBatchedFeaturesDatasetTest( for num_epochs in [1, 10]: with ops.Graph().as_default(): # Basic test: read from file 0. - outputs = self.make_batch_feature( + outputs = dataset_ops.make_one_shot_iterator(self.make_batch_feature( filenames=self.test_filenames[0], label_key="label", num_epochs=num_epochs, batch_size=batch_size, - drop_final_batch=True).make_one_shot_iterator().get_next() + drop_final_batch=True)).get_next() for tensor in nest.flatten(outputs): if isinstance(tensor, ops.Tensor): # Guard against SparseTensor. self.assertEqual(tensor.shape[0], batch_size) diff --git a/tensorflow/python/data/experimental/kernel_tests/make_csv_dataset_test.py b/tensorflow/python/data/experimental/kernel_tests/make_csv_dataset_test.py index e4bf0891842..e80accee330 100644 --- a/tensorflow/python/data/experimental/kernel_tests/make_csv_dataset_test.py +++ b/tensorflow/python/data/experimental/kernel_tests/make_csv_dataset_test.py @@ -25,11 +25,13 @@ import numpy as np from tensorflow.python.data.experimental.ops import readers from tensorflow.python.data.kernel_tests import test_base +from tensorflow.python.data.ops import dataset_ops from tensorflow.python.data.util import nest from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes from tensorflow.python.framework import errors from tensorflow.python.framework import ops +from tensorflow.python.framework import test_util from tensorflow.python.platform import test @@ -82,7 +84,7 @@ class MakeCsvDatasetTest(test_base.DatasetTestBase): expected_output, expected_keys, ): - nxt = dataset.make_one_shot_iterator().get_next() + nxt = dataset_ops.make_one_shot_iterator(dataset).get_next() for expected_features in self._next_expected_batch( expected_output, @@ -90,7 +92,7 @@ class MakeCsvDatasetTest(test_base.DatasetTestBase): batch_size, num_epochs, ): - actual_features = sess.run(nxt) + actual_features = self.evaluate(nxt) if label_name is not None: expected_labels = expected_features.pop(label_name) @@ -102,7 +104,7 @@ class MakeCsvDatasetTest(test_base.DatasetTestBase): self.assertAllEqual(expected_features[k], actual_features[k]) with self.assertRaises(errors.OutOfRangeError): - sess.run(nxt) + self.evaluate(nxt) def _test_dataset(self, inputs, @@ -127,6 +129,7 @@ class MakeCsvDatasetTest(test_base.DatasetTestBase): self._verify_output(sess, dataset, batch_size, num_epochs, label_name, expected_output, expected_keys) + @test_util.run_deprecated_v1 def testMakeCSVDataset(self): """Tests making a CSV dataset with keys and defaults provided.""" record_defaults = [ @@ -158,6 +161,7 @@ class MakeCsvDatasetTest(test_base.DatasetTestBase): column_defaults=record_defaults, ) + @test_util.run_deprecated_v1 def testMakeCSVDataset_withBatchSizeAndEpochs(self): """Tests making a CSV dataset with keys and defaults provided.""" record_defaults = [ @@ -189,6 +193,7 @@ class MakeCsvDatasetTest(test_base.DatasetTestBase): column_defaults=record_defaults, ) + @test_util.run_deprecated_v1 def testMakeCSVDataset_withCompressionType(self): """Tests `compression_type` argument.""" record_defaults = [ @@ -257,6 +262,7 @@ class MakeCsvDatasetTest(test_base.DatasetTestBase): label_name="not_a_real_label", column_names=column_names) + @test_util.run_deprecated_v1 def testMakeCSVDataset_withNoLabel(self): """Tests making a CSV dataset with no label provided.""" record_defaults = [ @@ -286,6 +292,7 @@ class MakeCsvDatasetTest(test_base.DatasetTestBase): column_defaults=record_defaults, ) + @test_util.run_deprecated_v1 def testMakeCSVDataset_withNoHeader(self): """Tests that datasets can be created from CSV files with no header line. """ @@ -347,6 +354,7 @@ class MakeCsvDatasetTest(test_base.DatasetTestBase): column_defaults=record_defaults, ) + @test_util.run_deprecated_v1 def testMakeCSVDataset_withNoColNames(self): """Tests that datasets can be created when column names are not specified. @@ -451,6 +459,7 @@ class MakeCsvDatasetTest(test_base.DatasetTestBase): header=True, ) + @test_util.run_deprecated_v1 def testMakeCSVDataset_withSelectCols(self): record_defaults = [ constant_op.constant([], dtypes.int32), @@ -557,6 +566,7 @@ class MakeCsvDatasetTest(test_base.DatasetTestBase): label_name=None, select_columns=["invalid_col_name"]) + @test_util.run_deprecated_v1 def testMakeCSVDataset_withShuffle(self): record_defaults = [ constant_op.constant([], dtypes.int32), @@ -604,11 +614,11 @@ class MakeCsvDatasetTest(test_base.DatasetTestBase): shuffle_seed=5, num_epochs=2, ) - outputs1 = dataset1.make_one_shot_iterator().get_next() - outputs2 = dataset2.make_one_shot_iterator().get_next() + outputs1 = dataset_ops.make_one_shot_iterator(dataset1).get_next() + outputs2 = dataset_ops.make_one_shot_iterator(dataset2).get_next() for _ in range(total_records // batch_size): - batch1 = nest.flatten(sess.run(outputs1)) - batch2 = nest.flatten(sess.run(outputs2)) + batch1 = nest.flatten(self.evaluate(outputs1)) + batch2 = nest.flatten(self.evaluate(outputs2)) for i in range(len(batch1)): self.assertAllEqual(batch1[i], batch2[i]) @@ -635,12 +645,12 @@ class MakeCsvDatasetTest(test_base.DatasetTestBase): shuffle_seed=6, num_epochs=2, ) - outputs1 = dataset1.make_one_shot_iterator().get_next() - outputs2 = dataset2.make_one_shot_iterator().get_next() + outputs1 = dataset_ops.make_one_shot_iterator(dataset1).get_next() + outputs2 = dataset_ops.make_one_shot_iterator(dataset2).get_next() all_equal = False for _ in range(total_records // batch_size): - batch1 = nest.flatten(sess.run(outputs1)) - batch2 = nest.flatten(sess.run(outputs2)) + batch1 = nest.flatten(self.evaluate(outputs1)) + batch2 = nest.flatten(self.evaluate(outputs2)) for i in range(len(batch1)): all_equal = all_equal and np.array_equal(batch1[i], batch2[i]) self.assertFalse(all_equal) diff --git a/tensorflow/python/data/experimental/kernel_tests/make_tf_record_dataset_test.py b/tensorflow/python/data/experimental/kernel_tests/make_tf_record_dataset_test.py index 657cf3c00ee..ab2feb64262 100644 --- a/tensorflow/python/data/experimental/kernel_tests/make_tf_record_dataset_test.py +++ b/tensorflow/python/data/experimental/kernel_tests/make_tf_record_dataset_test.py @@ -19,6 +19,7 @@ from __future__ import print_function from tensorflow.python.data.experimental.kernel_tests import reader_dataset_ops_test_base from tensorflow.python.data.experimental.ops import readers +from tensorflow.python.data.ops import dataset_ops from tensorflow.python.data.util import nest from tensorflow.python.framework import errors from tensorflow.python.framework import ops @@ -105,7 +106,7 @@ class MakeTFRecordDatasetTest( for expected_batch in self._next_expected_batch( file_indices, batch_size, num_epochs, interleave_cycle_length, drop_final_batch, use_parser_fn): - actual_batch = sess.run(outputs) + actual_batch = self.evaluate(outputs) self.assertAllEqual(expected_batch, actual_batch) def _read_test(self, batch_size, num_epochs, file_index=None, @@ -122,20 +123,21 @@ class MakeTFRecordDatasetTest( with ops.Graph().as_default() as g: with self.session(graph=g) as sess: - outputs = readers.make_tf_record_dataset( - file_pattern=file_pattern, - num_epochs=num_epochs, - batch_size=batch_size, - parser_fn=fn, - num_parallel_reads=num_parallel_reads, - drop_final_batch=drop_final_batch, - shuffle=False).make_one_shot_iterator().get_next() + outputs = dataset_ops.make_one_shot_iterator( + readers.make_tf_record_dataset( + file_pattern=file_pattern, + num_epochs=num_epochs, + batch_size=batch_size, + parser_fn=fn, + num_parallel_reads=num_parallel_reads, + drop_final_batch=drop_final_batch, + shuffle=False)).get_next() self._verify_records( sess, outputs, batch_size, file_index, num_epochs=num_epochs, interleave_cycle_length=num_parallel_reads, drop_final_batch=drop_final_batch, use_parser_fn=parser_fn) with self.assertRaises(errors.OutOfRangeError): - sess.run(outputs) + self.evaluate(outputs) def testRead(self): for batch_size in [1, 2]: @@ -185,22 +187,22 @@ class MakeTFRecordDatasetTest( num_parallel_reads=num_parallel_reads, shuffle=True, shuffle_seed=seed) - iterator = dataset.make_initializable_iterator() + iterator = dataset_ops.make_initializable_iterator(dataset) next_element = iterator.get_next() - sess.run(iterator.initializer) + self.evaluate(iterator.initializer) first_batches = [] try: while True: - first_batches.append(sess.run(next_element)) + first_batches.append(self.evaluate(next_element)) except errors.OutOfRangeError: pass - sess.run(iterator.initializer) + self.evaluate(iterator.initializer) second_batches = [] try: while True: - second_batches.append(sess.run(next_element)) + second_batches.append(self.evaluate(next_element)) except errors.OutOfRangeError: pass diff --git a/tensorflow/python/data/experimental/kernel_tests/map_and_batch_test.py b/tensorflow/python/data/experimental/kernel_tests/map_and_batch_test.py index 5ead6d1c754..e6e24c3db1f 100644 --- a/tensorflow/python/data/experimental/kernel_tests/map_and_batch_test.py +++ b/tensorflow/python/data/experimental/kernel_tests/map_and_batch_test.py @@ -29,6 +29,7 @@ from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes from tensorflow.python.framework import errors from tensorflow.python.framework import sparse_tensor +from tensorflow.python.framework import test_util from tensorflow.python.ops import array_ops from tensorflow.python.ops import control_flow_ops from tensorflow.python.ops import math_ops @@ -48,6 +49,7 @@ class MapAndBatchTest(test_base.DatasetTestBase, parameterized.TestCase): ("ParallelCallsNUMA", 2, None, True), ("ParallelBatchesNUMA", None, 10, True), ) + @test_util.run_deprecated_v1 def testMapAndBatch(self, num_parallel_calls, num_parallel_batches, numa_aware): """Test a dataset that maps a TF function across its input elements.""" @@ -76,7 +78,7 @@ class MapAndBatchTest(test_base.DatasetTestBase, parameterized.TestCase): options.experimental_numa_aware = True dataset = dataset.with_options(options) - iterator = dataset.make_initializable_iterator() + iterator = dataset_ops.make_initializable_iterator(dataset) init_op = iterator.initializer get_next = iterator.get_next() @@ -89,13 +91,13 @@ class MapAndBatchTest(test_base.DatasetTestBase, parameterized.TestCase): sess.run(init_op, feed_dict={count: 28, batch_size: 14}) num_batches = (28 * 7) // 14 for i in range(num_batches): - result = sess.run(get_next) + result = self.evaluate(get_next) for component, result_component in zip(components, result): for j in range(14): self.assertAllEqual(component[(i * 14 + j) % 7]**2, result_component[j]) with self.assertRaises(errors.OutOfRangeError): - sess.run(get_next) + self.evaluate(get_next) # Batch of a finite input, where the batch_size does not # divide the total number of elements. @@ -104,23 +106,23 @@ class MapAndBatchTest(test_base.DatasetTestBase, parameterized.TestCase): # We expect (num_batches - 1) full-sized batches. num_batches = int(math.ceil((14 * 7) / 8)) for i in range(num_batches - 1): - result = sess.run(get_next) + result = self.evaluate(get_next) for component, result_component in zip(components, result): for j in range(8): self.assertAllEqual(component[(i * 8 + j) % 7]**2, result_component[j]) - result = sess.run(get_next) + result = self.evaluate(get_next) for component, result_component in zip(components, result): for j in range((14 * 7) % 8): self.assertAllEqual(component[((num_batches - 1) * 8 + j) % 7]**2, result_component[j]) with self.assertRaises(errors.OutOfRangeError): - sess.run(get_next) + self.evaluate(get_next) # Batch of an empty input should fail straight away. sess.run(init_op, feed_dict={count: 0, batch_size: 8}) with self.assertRaises(errors.OutOfRangeError): - sess.run(get_next) + self.evaluate(get_next) # Empty batch should be an initialization time error. with self.assertRaises(errors.InvalidArgumentError): @@ -132,6 +134,7 @@ class MapAndBatchTest(test_base.DatasetTestBase, parameterized.TestCase): ("EvenNUMA", False, True), ("UnevenNUMA", True, True), ) + @test_util.run_deprecated_v1 def testMapAndBatchPartialBatch(self, drop_remainder, numa_aware): dataset = ( dataset_ops.Dataset.range(10).apply( @@ -144,7 +147,7 @@ class MapAndBatchTest(test_base.DatasetTestBase, parameterized.TestCase): options = dataset_ops.Options() options.experimental_numa_aware = True dataset = dataset.with_options(options) - iterator = dataset.make_one_shot_iterator() + iterator = dataset_ops.make_one_shot_iterator(dataset) if drop_remainder: self.assertEqual([4, 1], iterator.output_shapes.as_list()) @@ -152,17 +155,18 @@ class MapAndBatchTest(test_base.DatasetTestBase, parameterized.TestCase): self.assertEqual([None, 1], iterator.output_shapes.as_list()) next_element = iterator.get_next() with self.cached_session() as sess: - self.assertAllEqual([[0], [1], [4], [9]], sess.run(next_element)) - self.assertAllEqual([[16], [25], [36], [49]], sess.run(next_element)) + self.assertAllEqual([[0], [1], [4], [9]], self.evaluate(next_element)) + self.assertAllEqual([[16], [25], [36], [49]], self.evaluate(next_element)) if not drop_remainder: - self.assertAllEqual([[64], [81]], sess.run(next_element)) + self.assertAllEqual([[64], [81]], self.evaluate(next_element)) with self.assertRaises(errors.OutOfRangeError): - sess.run(next_element) + self.evaluate(next_element) @parameterized.named_parameters( ("Normal", False), ("NUMA", True), ) + @test_util.run_deprecated_v1 def testMapAndBatchYieldsPartialBatch(self, numa_aware): dataset = ( dataset_ops.Dataset.range(10).apply( @@ -173,20 +177,21 @@ class MapAndBatchTest(test_base.DatasetTestBase, parameterized.TestCase): options.experimental_numa_aware = True dataset = dataset.with_options(options) - iterator = dataset.make_one_shot_iterator() + iterator = dataset_ops.make_one_shot_iterator(dataset) self.assertEqual([None, 1], iterator.output_shapes.as_list()) next_element = iterator.get_next() with self.cached_session() as sess: - self.assertAllEqual([[0], [1], [4], [9]], sess.run(next_element)) - self.assertAllEqual([[16], [25], [36], [49]], sess.run(next_element)) - self.assertAllEqual([[64], [81]], sess.run(next_element)) + self.assertAllEqual([[0], [1], [4], [9]], self.evaluate(next_element)) + self.assertAllEqual([[16], [25], [36], [49]], self.evaluate(next_element)) + self.assertAllEqual([[64], [81]], self.evaluate(next_element)) with self.assertRaises(errors.OutOfRangeError): - sess.run(next_element) + self.evaluate(next_element) @parameterized.named_parameters( ("Normal", False), ("NUMA", True), ) + @test_util.run_deprecated_v1 def testMapAndBatchParallelGetNext(self, numa_aware): dataset = dataset_ops.Dataset.range(50000).apply( batching.map_and_batch(lambda x: x, batch_size=100)) @@ -194,26 +199,27 @@ class MapAndBatchTest(test_base.DatasetTestBase, parameterized.TestCase): options = dataset_ops.Options() options.experimental_numa_aware = True dataset = dataset.with_options(options) - iterator = dataset.make_one_shot_iterator() + iterator = dataset_ops.make_one_shot_iterator(dataset) elements = [] for _ in range(100): elements.append(iterator.get_next()) with self.cached_session() as sess: for i in range(5): - got = sess.run(elements) + got = self.evaluate(elements) got.sort(key=lambda x: x[0]) expected = [] for j in range(100): expected.append(range(i * 10000 + j * 100, i * 10000 + (j + 1) * 100)) self.assertAllEqual(got, expected) with self.assertRaises(errors.OutOfRangeError): - sess.run(elements) + self.evaluate(elements) @parameterized.named_parameters( ("Normal", False), ("NUMA", True), ) + @test_util.run_deprecated_v1 def testMapAndBatchParallelGetNextDropRemainder(self, numa_aware): dataset = dataset_ops.Dataset.range(49999).apply( batching.map_and_batch( @@ -223,26 +229,27 @@ class MapAndBatchTest(test_base.DatasetTestBase, parameterized.TestCase): options = dataset_ops.Options() options.experimental_numa_aware = True dataset = dataset.with_options(options) - iterator = dataset.make_one_shot_iterator() + iterator = dataset_ops.make_one_shot_iterator(dataset) elements = [] for _ in range(100): elements.append(iterator.get_next()) with self.cached_session() as sess: for i in range(4): - got = sess.run(elements) + got = self.evaluate(elements) got.sort(key=lambda x: x[0]) expected = [] for j in range(100): expected.append(range(i * 10000 + j * 100, i * 10000 + (j + 1) * 100)) self.assertAllEqual(got, expected) with self.assertRaises(errors.OutOfRangeError): - sess.run(elements) + self.evaluate(elements) @parameterized.named_parameters( ("Normal", False), ("NUMA", True), ) + @test_util.run_deprecated_v1 def testMapAndBatchSparse(self, numa_aware): def _sparse(i): @@ -255,15 +262,15 @@ class MapAndBatchTest(test_base.DatasetTestBase, parameterized.TestCase): options = dataset_ops.Options() options.experimental_numa_aware = True dataset = dataset.with_options(options) - iterator = dataset.make_initializable_iterator() + iterator = dataset_ops.make_initializable_iterator(dataset) init_op = iterator.initializer get_next = iterator.get_next() with self.cached_session() as sess: - sess.run(init_op) + self.evaluate(init_op) for i in range(2): - actual = sess.run(get_next) + actual = self.evaluate(get_next) expected = sparse_tensor.SparseTensorValue( indices=[[0, 0], [1, 0], [2, 0], [3, 0], [4, 0]], values=[i * 5, i * 5 + 1, i * 5 + 2, i * 5 + 3, i * 5 + 4], @@ -271,12 +278,13 @@ class MapAndBatchTest(test_base.DatasetTestBase, parameterized.TestCase): self.assertTrue(sparse_tensor.is_sparse(actual)) self.assertSparseValuesEqual(actual, expected) with self.assertRaises(errors.OutOfRangeError): - sess.run(get_next) + self.evaluate(get_next) @parameterized.named_parameters( ("Normal", False), ("NUMA", True), ) + @test_util.run_deprecated_v1 def testMapAndBatchFails(self, numa_aware): """Test a dataset that maps a TF function across its input elements.""" dataset = dataset_ops.Dataset.from_tensors( @@ -288,7 +296,7 @@ class MapAndBatchTest(test_base.DatasetTestBase, parameterized.TestCase): options = dataset_ops.Options() options.experimental_numa_aware = True dataset = dataset.with_options(options) - iterator = dataset.make_initializable_iterator() + iterator = dataset_ops.make_initializable_iterator(dataset) init_op = iterator.initializer with self.cached_session() as sess: @@ -299,6 +307,7 @@ class MapAndBatchTest(test_base.DatasetTestBase, parameterized.TestCase): ("Normal", False), ("NUMA", True), ) + @test_util.run_deprecated_v1 def testMapAndBatchShapeMismatch(self, numa_aware): """Test a dataset that maps a TF function across its input elements.""" @@ -316,15 +325,15 @@ class MapAndBatchTest(test_base.DatasetTestBase, parameterized.TestCase): options = dataset_ops.Options() options.experimental_numa_aware = True dataset = dataset.with_options(options) - iterator = dataset.make_initializable_iterator() + iterator = dataset_ops.make_initializable_iterator(dataset) init_op = iterator.initializer get_next = iterator.get_next() with self.cached_session() as sess: - sess.run(init_op) + self.evaluate(init_op) with self.assertRaisesRegexp(errors.InvalidArgumentError, "number of elements does not match"): - sess.run(get_next) + self.evaluate(get_next) @parameterized.named_parameters( ("Normal", False), @@ -349,12 +358,12 @@ class MapAndBatchTest(test_base.DatasetTestBase, parameterized.TestCase): options = dataset_ops.Options() options.experimental_numa_aware = True dataset = dataset.with_options(options) - iterator = dataset.make_one_shot_iterator() + iterator = dataset_ops.make_one_shot_iterator(dataset) get_next = iterator.get_next() with self.cached_session() as sess: for _ in range(3): - sess.run(get_next) + self.evaluate(get_next) @parameterized.named_parameters( ("1", 0, False), @@ -370,6 +379,7 @@ class MapAndBatchTest(test_base.DatasetTestBase, parameterized.TestCase): ("5NUMA", 95, True), ("6NUMA", 99, True), ) + @test_util.run_deprecated_v1 def testMapAndBatchOutOfRangeError(self, threshold, numa_aware): def raising_py_fn(i): @@ -388,18 +398,19 @@ class MapAndBatchTest(test_base.DatasetTestBase, parameterized.TestCase): options = dataset_ops.Options() options.experimental_numa_aware = True dataset = dataset.with_options(options) - iterator = dataset.make_one_shot_iterator() + iterator = dataset_ops.make_one_shot_iterator(dataset) get_next = iterator.get_next() with self.cached_session() as sess: for i in range(threshold // 10): - self.assertAllEqual([i * 10 + j for j in range(10)], sess.run(get_next)) + self.assertAllEqual([i * 10 + j for j in range(10)], + self.evaluate(get_next)) if threshold % 10 != 0: self.assertAllEqual( [threshold // 10 * 10 + j for j in range(threshold % 10)], - sess.run(get_next)) + self.evaluate(get_next)) with self.assertRaises(errors.OutOfRangeError): - sess.run(get_next) + self.evaluate(get_next) @parameterized.named_parameters( ("1", False, dtypes.bool, False), @@ -438,11 +449,12 @@ class MapAndBatchTest(test_base.DatasetTestBase, parameterized.TestCase): options.experimental_numa_aware = True dataset = dataset.with_options(options) - get_next = dataset.make_one_shot_iterator().get_next() + get_next = dataset_ops.make_one_shot_iterator(dataset).get_next() with self.cached_session() as sess: for _ in range(10): - self.assertAllEqual([element for _ in range(10)], sess.run(get_next)) + self.assertAllEqual([element for _ in range(10)], + self.evaluate(get_next)) @parameterized.named_parameters( ("Identity", None, lambda x: x, None), @@ -450,10 +462,11 @@ class MapAndBatchTest(test_base.DatasetTestBase, parameterized.TestCase): ("Swap", (None, None), lambda x, y: (y, x), None), ("Project", (None, None), lambda x, y: x, None), ) + @test_util.run_deprecated_v1 def testShortCircuit(self, structure, map_fn, num_parallel_calls): dataset = self.structuredDataset(structure).repeat().apply( batching.map_and_batch(map_fn, batch_size=10)) - get_next = dataset.make_one_shot_iterator().get_next() + get_next = dataset_ops.make_one_shot_iterator(dataset).get_next() with self.cached_session() as sess: if isinstance(structure, tuple): @@ -462,23 +475,25 @@ class MapAndBatchTest(test_base.DatasetTestBase, parameterized.TestCase): else: expected = map_fn( sess.run(self.structuredElement(structure, shape=[10]))) - self.assertAllEqual(expected, sess.run(get_next)) + self.assertAllEqual(expected, self.evaluate(get_next)) + @test_util.run_deprecated_v1 def testShortCircuitCapturedInput(self): captured_t = array_ops.placeholder(dtypes.int64, shape=[]) dataset = self.structuredDataset(None).repeat().apply( batching.map_and_batch(lambda x: captured_t, batch_size=10)) - iterator = dataset.make_initializable_iterator() + iterator = dataset_ops.make_initializable_iterator(dataset) get_next = iterator.get_next() with self.cached_session() as sess: sess.run(iterator.initializer, feed_dict={captured_t: 42}) - self.assertAllEqual([42] * 10, sess.run(get_next)) + self.assertAllEqual([42] * 10, self.evaluate(get_next)) @parameterized.named_parameters( ("Normal", False), ("NUMA", True), ) + @test_util.run_deprecated_v1 def testMapAndBatchControlFlow(self, numa_aware): def map_fn(x): @@ -494,20 +509,20 @@ class MapAndBatchTest(test_base.DatasetTestBase, parameterized.TestCase): options = dataset_ops.Options() options.experimental_numa_aware = True dataset = dataset.with_options(options) - iterator = dataset.make_one_shot_iterator() + iterator = dataset_ops.make_one_shot_iterator(dataset) get_next = iterator.get_next() with self.cached_session() as sess: for i in range(10): print("Case %d" % i) if i < 5: self.assertAllEqual([i * 10 + j + 1 for j in range(10)], - sess.run(get_next)) + self.evaluate(get_next)) else: self.assertAllEqual( [((i * 10) + j) * ((i * 10) + j) for j in range(10)], - sess.run(get_next)) + self.evaluate(get_next)) with self.assertRaises(errors.OutOfRangeError): - sess.run(get_next) + self.evaluate(get_next) if __name__ == "__main__": diff --git a/tensorflow/python/data/experimental/kernel_tests/map_defun_op_test.py b/tensorflow/python/data/experimental/kernel_tests/map_defun_op_test.py index 11694540fae..6042ca1c63f 100644 --- a/tensorflow/python/data/experimental/kernel_tests/map_defun_op_test.py +++ b/tensorflow/python/data/experimental/kernel_tests/map_defun_op_test.py @@ -218,7 +218,7 @@ class MapDefunTest(test_base.DatasetTestBase): def _assert_op_cancelled(self, sess, map_defun_op): with self.assertRaisesRegexp(errors.CancelledError, "was cancelled"): - sess.run(map_defun_op) + self.evaluate(map_defun_op) def testMapDefunWithParentCancellation(self): # Checks that a cancellation of the parent graph is threaded through to @@ -260,10 +260,10 @@ class MapDefunBenchmark(test.Benchmark): with session.Session() as sess: # Warm up the session for _ in range(5): - sess.run(op) + self.evaluate(op) start = time.time() for _ in range(num_iters): - sess.run(op) + self.evaluate(op) end = time.time() mean_us = (end - start) * 1e6 / num_iters self.report_benchmark( diff --git a/tensorflow/python/data/kernel_tests/matching_files_dataset_op_test.py b/tensorflow/python/data/experimental/kernel_tests/matching_files_test.py similarity index 57% rename from tensorflow/python/data/kernel_tests/matching_files_dataset_op_test.py rename to tensorflow/python/data/experimental/kernel_tests/matching_files_test.py index 4d86ec4228a..0ee7616d35e 100644 --- a/tensorflow/python/data/kernel_tests/matching_files_dataset_op_test.py +++ b/tensorflow/python/data/experimental/kernel_tests/matching_files_test.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================== -"""Tests for the experimental input pipeline ops.""" +"""Tests for the private `MatchingFilesDataset`.""" from __future__ import absolute_import from __future__ import division from __future__ import print_function @@ -20,20 +20,17 @@ from __future__ import print_function import os import shutil import tempfile -import time -import numpy as np - -from tensorflow.python.client import session +from tensorflow.python.data.experimental.ops import matching_files from tensorflow.python.data.kernel_tests import test_base -from tensorflow.python.data.ops.dataset_ops import MatchingFilesDataset +from tensorflow.python.data.ops import dataset_ops from tensorflow.python.framework import errors -from tensorflow.python.framework import ops +from tensorflow.python.framework import test_util from tensorflow.python.platform import test from tensorflow.python.util import compat -class MatchingFilesDatasetTest(test_base.DatasetTestBase): +class MatchingFilesTest(test_base.DatasetTestBase): def setUp(self): self.tmp_dir = tempfile.mkdtemp() @@ -45,34 +42,40 @@ class MatchingFilesDatasetTest(test_base.DatasetTestBase): for filename in filenames: open(os.path.join(self.tmp_dir, filename), 'a').close() + @test_util.run_deprecated_v1 def testNonExistingDirectory(self): - """Test the MatchingFiles dataset with a non-existing directory""" + """Test the MatchingFiles dataset with a non-existing directory.""" self.tmp_dir = os.path.join(self.tmp_dir, 'nonexistingdir') - dataset = MatchingFilesDataset(os.path.join(self.tmp_dir, '*')) + dataset = matching_files.MatchingFilesDataset( + os.path.join(self.tmp_dir, '*')) with self.cached_session() as sess: - next_element = dataset.make_one_shot_iterator().get_next() + next_element = dataset_ops.make_one_shot_iterator(dataset).get_next() with self.assertRaises(errors.NotFoundError): sess.run(next_element) + @test_util.run_deprecated_v1 def testEmptyDirectory(self): - """Test the MatchingFiles dataset with an empty directory""" + """Test the MatchingFiles dataset with an empty directory.""" - dataset = MatchingFilesDataset(os.path.join(self.tmp_dir, '*')) + dataset = matching_files.MatchingFilesDataset( + os.path.join(self.tmp_dir, '*')) with self.cached_session() as sess: - next_element = dataset.make_one_shot_iterator().get_next() + next_element = dataset_ops.make_one_shot_iterator(dataset).get_next() with self.assertRaises(errors.NotFoundError): sess.run(next_element) + @test_util.run_deprecated_v1 def testSimpleDirectory(self): - """Test the MatchingFiles dataset with a simple directory""" + """Test the MatchingFiles dataset with a simple directory.""" filenames = ['a', 'b', 'c'] self._touchTempFiles(filenames) - dataset = MatchingFilesDataset(os.path.join(self.tmp_dir, '*')) + dataset = matching_files.MatchingFilesDataset( + os.path.join(self.tmp_dir, '*')) with self.cached_session() as sess: - next_element = dataset.make_one_shot_iterator().get_next() + next_element = dataset_ops.make_one_shot_iterator(dataset).get_next() expected_filenames = [] actual_filenames = [] @@ -85,15 +88,17 @@ class MatchingFilesDatasetTest(test_base.DatasetTestBase): with self.assertRaises(errors.OutOfRangeError): sess.run(next_element) + @test_util.run_deprecated_v1 def testFileSuffixes(self): - """Test the MatchingFiles dataset using the suffixes of filename""" + """Test the MatchingFiles dataset using the suffixes of filename.""" filenames = ['a.txt', 'b.py', 'c.py', 'd.pyc'] self._touchTempFiles(filenames) - dataset = MatchingFilesDataset(os.path.join(self.tmp_dir, '*.py')) + dataset = matching_files.MatchingFilesDataset( + os.path.join(self.tmp_dir, '*.py')) with self.cached_session() as sess: - next_element = dataset.make_one_shot_iterator().get_next() + next_element = dataset_ops.make_one_shot_iterator(dataset).get_next() expected_filenames = [] actual_filenames = [] for filename in filenames[1:-1]: @@ -105,15 +110,17 @@ class MatchingFilesDatasetTest(test_base.DatasetTestBase): with self.assertRaises(errors.OutOfRangeError): sess.run(next_element) + @test_util.run_deprecated_v1 def testFileMiddles(self): - """Test the MatchingFiles dataset using the middles of filename""" + """Test the MatchingFiles dataset using the middles of filename.""" filenames = ['aa.txt', 'bb.py', 'bbc.pyc', 'cc.pyc'] self._touchTempFiles(filenames) - dataset = MatchingFilesDataset(os.path.join(self.tmp_dir, 'b*.py*')) + dataset = matching_files.MatchingFilesDataset( + os.path.join(self.tmp_dir, 'b*.py*')) with self.cached_session() as sess: - next_element = dataset.make_one_shot_iterator().get_next() + next_element = dataset_ops.make_one_shot_iterator(dataset).get_next() expected_filenames = [] actual_filenames = [] for filename in filenames[1:3]: @@ -125,8 +132,9 @@ class MatchingFilesDatasetTest(test_base.DatasetTestBase): with self.assertRaises(errors.OutOfRangeError): sess.run(next_element) + @test_util.run_deprecated_v1 def testNestedDirectories(self): - """Test the MatchingFiles dataset with nested directories""" + """Test the MatchingFiles dataset with nested directories.""" filenames = [] width = 8 @@ -147,9 +155,9 @@ class MatchingFilesDatasetTest(test_base.DatasetTestBase): suffix) for suffix in ['*.txt', '*.log'] ] - dataset = MatchingFilesDataset(patterns) + dataset = matching_files.MatchingFilesDataset(patterns) with self.cached_session() as sess: - next_element = dataset.make_one_shot_iterator().get_next() + next_element = dataset_ops.make_one_shot_iterator(dataset).get_next() expected_filenames = [ compat.as_bytes(filename) for filename in filenames @@ -165,70 +173,5 @@ class MatchingFilesDatasetTest(test_base.DatasetTestBase): self.assertItemsEqual(expected_filenames, actual_filenames) -class MatchingFilesDatasetBenchmark(test.Benchmark): - - def benchmarkNestedDirectories(self): - tmp_dir = tempfile.mkdtemp() - width = 500 - depth = 10 - for i in range(width): - for j in range(depth): - new_base = os.path.join(tmp_dir, str(i), - *[str(dir_name) for dir_name in range(j)]) - os.makedirs(new_base) - child_files = ['a.py', 'b.pyc'] if j < depth - 1 else ['c.txt', 'd.log'] - for f in child_files: - filename = os.path.join(new_base, f) - open(filename, 'w').close() - - patterns = [ - os.path.join(tmp_dir, os.path.join(*['**' - for _ in range(depth)]), suffix) - for suffix in ['*.txt', '*.log'] - ] - - deltas = [] - iters = 3 - for _ in range(iters): - with ops.Graph().as_default(): - dataset = MatchingFilesDataset(patterns) - next_element = dataset.make_one_shot_iterator().get_next() - - with session.Session() as sess: - sub_deltas = [] - while True: - try: - start = time.time() - sess.run(next_element) - end = time.time() - sub_deltas.append(end - start) - except errors.OutOfRangeError: - break - deltas.append(sub_deltas) - - median_deltas = np.median(deltas, axis=0) - print('Nested directory size (width*depth): %d*%d Median wall time: ' - '%fs (read first filename), %fs (read second filename), avg %fs' - ' (read %d more filenames)' % - (width, depth, median_deltas[0], median_deltas[1], - np.average(median_deltas[2:]), len(median_deltas) - 2)) - self.report_benchmark( - iters=iters, - wall_time=np.sum(median_deltas), - extras={ - 'read first file:': - median_deltas[0], - 'read second file:': - median_deltas[1], - 'avg time for reading %d more filenames:' % - (len(median_deltas) - 2): - np.average(median_deltas[2:]) - }, - name='benchmark_matching_files_dataset_nesteddirectory(%d*%d)' % - (width, depth)) - - shutil.rmtree(tmp_dir, ignore_errors=True) - - if __name__ == '__main__': test.main() diff --git a/tensorflow/python/data/experimental/kernel_tests/optimization/BUILD b/tensorflow/python/data/experimental/kernel_tests/optimization/BUILD index 9946ef5a42f..f214944254c 100644 --- a/tensorflow/python/data/experimental/kernel_tests/optimization/BUILD +++ b/tensorflow/python/data/experimental/kernel_tests/optimization/BUILD @@ -42,6 +42,7 @@ py_test( "//tensorflow/python:errors", "//tensorflow/python:math_ops", "//tensorflow/python/data/experimental/ops:optimization", + "//tensorflow/python/data/experimental/ops:optimization_options", "//tensorflow/python/data/kernel_tests:test_base", "//tensorflow/python/data/ops:dataset_ops", "@absl_py//absl/testing:parameterized", @@ -68,6 +69,7 @@ py_test( "//tensorflow/python:math_ops", "//tensorflow/python:random_ops", "//tensorflow/python/data/experimental/ops:optimization", + "//tensorflow/python/data/experimental/ops:optimization_options", "//tensorflow/python/data/kernel_tests:test_base", "//tensorflow/python/data/ops:dataset_ops", "@absl_py//absl/testing:parameterized", @@ -127,6 +129,7 @@ py_test( "//tensorflow/python:client_testlib", "//tensorflow/python:errors", "//tensorflow/python/data/experimental/ops:optimization", + "//tensorflow/python/data/experimental/ops:optimization_options", "//tensorflow/python/data/kernel_tests:test_base", "//tensorflow/python/data/ops:dataset_ops", ], @@ -148,6 +151,7 @@ py_test( "//tensorflow/python:errors", "//tensorflow/python:math_ops", "//tensorflow/python/data/experimental/ops:optimization", + "//tensorflow/python/data/experimental/ops:optimization_options", "//tensorflow/python/data/kernel_tests:test_base", "//tensorflow/python/data/ops:dataset_ops", "@absl_py//absl/testing:parameterized", @@ -167,6 +171,7 @@ py_test( "//tensorflow/python:client_testlib", "//tensorflow/python:errors", "//tensorflow/python/data/experimental/ops:optimization", + "//tensorflow/python/data/experimental/ops:optimization_options", "//tensorflow/python/data/kernel_tests:test_base", "//tensorflow/python/data/ops:dataset_ops", "@absl_py//absl/testing:parameterized", @@ -192,6 +197,7 @@ py_test( "//tensorflow/python:math_ops", "//tensorflow/python:random_ops", "//tensorflow/python/data/experimental/ops:optimization", + "//tensorflow/python/data/experimental/ops:optimization_options", "//tensorflow/python/data/kernel_tests:test_base", "//tensorflow/python/data/ops:dataset_ops", "@absl_py//absl/testing:parameterized", @@ -202,6 +208,7 @@ py_test( name = "map_vectorization_test", size = "medium", srcs = ["map_vectorization_test.py"], + shard_count = 8, srcs_version = "PY2AND3", tags = [ "no_oss", @@ -220,15 +227,15 @@ py_test( "//tensorflow/python:dtypes", "//tensorflow/python:errors", "//tensorflow/python:framework_ops", + "//tensorflow/python:framework_test_lib", "//tensorflow/python:math_ops", - "//tensorflow/python:nn_ops", + "//tensorflow/python:nn", "//tensorflow/python:parsing_ops", - "//tensorflow/python:session", "//tensorflow/python:sparse_tensor", "//tensorflow/python/data/experimental/ops:optimization", + "//tensorflow/python/data/experimental/ops:optimization_options", "//tensorflow/python/data/kernel_tests:test_base", "//tensorflow/python/data/ops:dataset_ops", - "//tensorflow/python/data/util:nest", "//third_party/py/numpy", "@absl_py//absl/testing:parameterized", ], @@ -248,12 +255,9 @@ py_test( deps = [ "//tensorflow/python:client_testlib", "//tensorflow/python:errors", - "//tensorflow/python:math_ops", - "//tensorflow/python/data/experimental/ops:batching", "//tensorflow/python/data/experimental/ops:optimization", "//tensorflow/python/data/kernel_tests:test_base", "//tensorflow/python/data/ops:dataset_ops", - "//third_party/py/numpy", "@absl_py//absl/testing:parameterized", ], ) @@ -275,6 +279,7 @@ py_test( "//tensorflow/python:errors", "//tensorflow/python:math_ops", "//tensorflow/python/data/experimental/ops:optimization", + "//tensorflow/python/data/experimental/ops:optimization_options", "//tensorflow/python/data/kernel_tests:test_base", "//tensorflow/python/data/ops:dataset_ops", ], @@ -296,10 +301,17 @@ py_test( "//tensorflow/python:dtypes", "//tensorflow/python:errors", "//tensorflow/python:random_ops", + "//tensorflow/python:variable_scope", + "//tensorflow/python/data/experimental/ops:batching", + "//tensorflow/python/data/experimental/ops:grouping", "//tensorflow/python/data/experimental/ops:optimization", + "//tensorflow/python/data/experimental/ops:optimization_options", + "//tensorflow/python/data/experimental/ops:scan_ops", "//tensorflow/python/data/kernel_tests:test_base", "//tensorflow/python/data/ops:dataset_ops", + "//tensorflow/python/eager:context", "//third_party/py/numpy", + "@absl_py//absl/testing:parameterized", ], ) @@ -316,6 +328,7 @@ py_test( "//tensorflow/python:client_testlib", "//tensorflow/python:errors", "//tensorflow/python/data/experimental/ops:optimization", + "//tensorflow/python/data/experimental/ops:optimization_options", "//tensorflow/python/data/kernel_tests:test_base", "//tensorflow/python/data/ops:dataset_ops", ], diff --git a/tensorflow/python/data/experimental/kernel_tests/optimization/assert_next_dataset_test.py b/tensorflow/python/data/experimental/kernel_tests/optimization/assert_next_dataset_test.py index ed719a0ce9b..9b8248a78da 100644 --- a/tensorflow/python/data/experimental/kernel_tests/optimization/assert_next_dataset_test.py +++ b/tensorflow/python/data/experimental/kernel_tests/optimization/assert_next_dataset_test.py @@ -21,32 +21,27 @@ from tensorflow.python.data.experimental.ops import optimization from tensorflow.python.data.kernel_tests import test_base from tensorflow.python.data.ops import dataset_ops from tensorflow.python.framework import errors +from tensorflow.python.framework import test_util from tensorflow.python.platform import test +@test_util.run_all_in_graph_and_eager_modes class AssertNextDatasetTest(test_base.DatasetTestBase): def testAssertNext(self): dataset = dataset_ops.Dataset.from_tensors(0).apply( optimization.assert_next(["Map"])).map(lambda x: x) - iterator = dataset.make_one_shot_iterator() - get_next = iterator.get_next() - - with self.cached_session() as sess: - self.assertEqual(0, sess.run(get_next)) + self.assertDatasetProduces(dataset, expected_output=[0]) def testAssertNextInvalid(self): dataset = dataset_ops.Dataset.from_tensors(0).apply( optimization.assert_next(["Whoops"])).map(lambda x: x) - iterator = dataset.make_one_shot_iterator() - get_next = iterator.get_next() - - with self.cached_session() as sess: - with self.assertRaisesRegexp( - errors.InvalidArgumentError, - "Asserted Whoops transformation at offset 0 but encountered " - "Map transformation instead."): - sess.run(get_next) + self.assertDatasetProduces( + dataset, + expected_error=( + errors.InvalidArgumentError, + "Asserted Whoops transformation at offset 0 but encountered " + "Map transformation instead.")) def testAssertNextShort(self): dataset = dataset_ops.Dataset.from_tensors(0).apply( @@ -54,14 +49,11 @@ class AssertNextDatasetTest(test_base.DatasetTestBase): options = dataset_ops.Options() options.experimental_autotune = False dataset = dataset.with_options(options) - iterator = dataset.make_one_shot_iterator() - get_next = iterator.get_next() - - with self.cached_session() as sess: - with self.assertRaisesRegexp( - errors.InvalidArgumentError, - "Asserted next 2 transformations but encountered only 1."): - sess.run(get_next) + self.assertDatasetProduces( + dataset, + expected_error=( + errors.InvalidArgumentError, + "Asserted next 2 transformations but encountered only 1.")) if __name__ == "__main__": diff --git a/tensorflow/python/data/experimental/kernel_tests/optimization/filter_fusion_test.py b/tensorflow/python/data/experimental/kernel_tests/optimization/filter_fusion_test.py index 80a0d879dc2..7371cf31dff 100644 --- a/tensorflow/python/data/experimental/kernel_tests/optimization/filter_fusion_test.py +++ b/tensorflow/python/data/experimental/kernel_tests/optimization/filter_fusion_test.py @@ -20,11 +20,12 @@ from __future__ import print_function from absl.testing import parameterized from tensorflow.python.data.experimental.ops import optimization +from tensorflow.python.data.experimental.ops.optimization_options import OptimizationOptions from tensorflow.python.data.kernel_tests import test_base from tensorflow.python.data.ops import dataset_ops from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes -from tensorflow.python.framework import errors +from tensorflow.python.framework import test_util from tensorflow.python.ops import math_ops from tensorflow.python.platform import test @@ -58,6 +59,7 @@ def _filter_fusion_test_cases(): return tuple(tests) +@test_util.run_all_in_graph_and_eager_modes class FilterFusionTest(test_base.DatasetTestBase, parameterized.TestCase): @parameterized.named_parameters(*_filter_fusion_test_cases()) @@ -70,28 +72,25 @@ class FilterFusionTest(test_base.DatasetTestBase, parameterized.TestCase): dataset = dataset.cache() options = dataset_ops.Options() - options.experimental_filter_fusion = True + options.experimental_optimization = OptimizationOptions() + options.experimental_optimization.filter_fusion = True dataset = dataset.with_options(options) - iterator = dataset.make_one_shot_iterator() - get_next = iterator.get_next() - with self.cached_session() as sess: - for x in range(5): - r = map_function(x) - filtered = False - for predicate in predicates: - if isinstance(r, tuple): - b = predicate(*r) # Pass tuple as multiple arguments. - else: - b = predicate(r) - if not sess.run(b): - filtered = True - break + expected_output = [] + for x in range(5): + r = map_function(x) + filtered = False + for predicate in predicates: + if isinstance(r, tuple): + b = predicate(*r) # Pass tuple as multiple arguments. + else: + b = predicate(r) + if not self.evaluate(b): + filtered = True + break - if not filtered: - result = sess.run(get_next) - self.assertAllEqual(r, result) - with self.assertRaises(errors.OutOfRangeError): - sess.run(get_next) + if not filtered: + expected_output.append(r) + self.assertDatasetProduces(dataset, expected_output=expected_output) if __name__ == "__main__": diff --git a/tensorflow/python/data/experimental/kernel_tests/optimization/hoist_random_uniform_test.py b/tensorflow/python/data/experimental/kernel_tests/optimization/hoist_random_uniform_test.py index 9f7fbfeba0d..5f3a8683fbb 100644 --- a/tensorflow/python/data/experimental/kernel_tests/optimization/hoist_random_uniform_test.py +++ b/tensorflow/python/data/experimental/kernel_tests/optimization/hoist_random_uniform_test.py @@ -20,12 +20,15 @@ from __future__ import print_function from absl.testing import parameterized from tensorflow.python.data.experimental.ops import optimization +from tensorflow.python.data.experimental.ops.optimization_options import OptimizationOptions from tensorflow.python.data.kernel_tests import test_base from tensorflow.python.data.ops import dataset_ops +from tensorflow.python.eager import context from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes from tensorflow.python.framework import errors from tensorflow.python.framework import ops +from tensorflow.python.framework import test_util from tensorflow.python.ops import control_flow_ops from tensorflow.python.ops import math_ops from tensorflow.python.ops import random_ops @@ -58,23 +61,29 @@ def _hoist_random_uniform_test_cases(): return tuple(tests) +@test_util.run_all_in_graph_and_eager_modes class HoistRandomUniformTest(test_base.DatasetTestBase, parameterized.TestCase): def _testDataset(self, dataset): - iterator = dataset.make_one_shot_iterator() - get_next = iterator.get_next() previous_result = 0 - with self.cached_session() as sess: - for _ in range(5): - result = sess.run(get_next) - self.assertLessEqual(1, result) - self.assertLessEqual(result, 10) - # This checks if the result is somehow random by checking if we are not - # generating the same values. - self.assertNotEqual(previous_result, result) - previous_result = result - with self.assertRaises(errors.OutOfRangeError): - sess.run(get_next) + if context.executing_eagerly(): + iterator = dataset.__iter__() + get_next = iterator._next_internal # pylint: disable=protected-access + else: + iterator = dataset_ops.make_one_shot_iterator(dataset) + get_next = iterator.get_next + for _ in range(5): + result = self.evaluate(get_next()) + self.assertLessEqual(1, result) + self.assertLessEqual(result, 10) + # This checks if the result is somehow random by checking if we are not + # generating the same values. + self.assertNotEqual(previous_result, result) + previous_result = result + with self.assertRaises(errors.OutOfRangeError): + self.evaluate(get_next()) + with self.assertRaises(errors.OutOfRangeError): + self.evaluate(get_next()) @parameterized.named_parameters(*_hoist_random_uniform_test_cases()) def testHoisting(self, function, will_optimize): @@ -83,7 +92,8 @@ class HoistRandomUniformTest(test_base.DatasetTestBase, parameterized.TestCase): ["Zip[0]", "Map"] if will_optimize else ["Map"])).map(function) options = dataset_ops.Options() - options.experimental_hoist_random_uniform = True + options.experimental_optimization = OptimizationOptions() + options.experimental_optimization.hoist_random_uniform = True dataset = dataset.with_options(options) self._testDataset(dataset) @@ -99,7 +109,8 @@ class HoistRandomUniformTest(test_base.DatasetTestBase, parameterized.TestCase): dataset = dataset_ops.Dataset.range(5).apply( optimization.assert_next(["Zip[0]", "Map"])).map(random_with_capture) options = dataset_ops.Options() - options.experimental_hoist_random_uniform = True + options.experimental_optimization = OptimizationOptions() + options.experimental_optimization.hoist_random_uniform = True dataset = dataset.with_options(options) self._testDataset(dataset) diff --git a/tensorflow/python/data/experimental/kernel_tests/optimization/latency_all_edges_test.py b/tensorflow/python/data/experimental/kernel_tests/optimization/latency_all_edges_test.py index 7144d834f9f..fc65f52704c 100644 --- a/tensorflow/python/data/experimental/kernel_tests/optimization/latency_all_edges_test.py +++ b/tensorflow/python/data/experimental/kernel_tests/optimization/latency_all_edges_test.py @@ -22,10 +22,11 @@ from tensorflow.python.data.experimental.ops import optimization from tensorflow.python.data.experimental.ops import stats_aggregator from tensorflow.python.data.experimental.ops import stats_options from tensorflow.python.data.ops import dataset_ops -from tensorflow.python.framework import errors +from tensorflow.python.framework import test_util from tensorflow.python.platform import test +@test_util.run_all_in_graph_and_eager_modes class LatencyAllEdgesTest(stats_dataset_test_base.StatsDatasetTestBase): def testLatencyStatsOptimization(self): @@ -39,22 +40,18 @@ class LatencyAllEdgesTest(stats_dataset_test_base.StatsDatasetTestBase): options.experimental_stats.latency_all_edges = True options.experimental_stats.aggregator = aggregator dataset = dataset.with_options(options) - iterator = dataset.make_initializable_iterator() - get_next = iterator.get_next() + self.assertDatasetProduces( + dataset, + expected_output=[1], + requires_initialization=True, + num_test_iterations=1) summary_t = aggregator.get_summary() - - with self.cached_session() as sess: - sess.run(iterator.initializer) - self.assertEqual(1 * 1, sess.run(get_next)) - with self.assertRaises(errors.OutOfRangeError): - sess.run(get_next) - summary_str = sess.run(summary_t) - self._assertSummaryHasCount(summary_str, - "record_latency_TensorDataset/_1", 1) - self._assertSummaryHasCount(summary_str, "record_latency_MapDataset/_4", - 1) - self._assertSummaryHasCount(summary_str, - "record_latency_PrefetchDataset/_6", 1) + summary_str = self.evaluate(summary_t) + self._assertSummaryHasCount(summary_str, "record_latency_TensorDataset/_1", + 1) + self._assertSummaryHasCount(summary_str, "record_latency_MapDataset/_4", 1) + self._assertSummaryHasCount(summary_str, + "record_latency_PrefetchDataset/_6", 1) def testLatencyStatsOptimizationV2(self): aggregator = stats_aggregator.StatsAggregator() @@ -63,24 +60,21 @@ class LatencyAllEdgesTest(stats_dataset_test_base.StatsDatasetTestBase): ["LatencyStats", "Map", "LatencyStats", "Prefetch", "LatencyStats"])).map(lambda x: x * x).prefetch(1) options = dataset_ops.Options() - options.experimental_stats = stats_options.StatsOptions(aggregator) + options.experimental_stats = stats_options.StatsOptions() + options.experimental_stats.aggregator = aggregator dataset = dataset.with_options(options) - iterator = dataset.make_initializable_iterator() - get_next = iterator.get_next() + self.assertDatasetProduces( + dataset, + expected_output=[1], + requires_initialization=True, + num_test_iterations=1) summary_t = aggregator.get_summary() - - with self.cached_session() as sess: - sess.run(iterator.initializer) - self.assertEqual(1, sess.run(get_next)) - with self.assertRaises(errors.OutOfRangeError): - sess.run(get_next) - summary_str = sess.run(summary_t) - self._assertSummaryHasCount(summary_str, - "record_latency_TensorDataset/_1", 1) - self._assertSummaryHasCount(summary_str, "record_latency_MapDataset/_4", - 1) - self._assertSummaryHasCount(summary_str, - "record_latency_PrefetchDataset/_6", 1) + summary_str = self.evaluate(summary_t) + self._assertSummaryHasCount(summary_str, "record_latency_TensorDataset/_1", + 1) + self._assertSummaryHasCount(summary_str, "record_latency_MapDataset/_4", 1) + self._assertSummaryHasCount(summary_str, + "record_latency_PrefetchDataset/_6", 1) if __name__ == "__main__": diff --git a/tensorflow/python/data/experimental/kernel_tests/optimization/make_numa_aware_test.py b/tensorflow/python/data/experimental/kernel_tests/optimization/make_numa_aware_test.py index 6191a7db084..2386dd5f116 100644 --- a/tensorflow/python/data/experimental/kernel_tests/optimization/make_numa_aware_test.py +++ b/tensorflow/python/data/experimental/kernel_tests/optimization/make_numa_aware_test.py @@ -21,10 +21,11 @@ from tensorflow.python.data.experimental.ops import batching from tensorflow.python.data.experimental.ops import optimization from tensorflow.python.data.kernel_tests import test_base from tensorflow.python.data.ops import dataset_ops -from tensorflow.python.framework import errors +from tensorflow.python.framework import test_util from tensorflow.python.platform import test +@test_util.run_all_in_graph_and_eager_modes class MakeNumaAwareTest(test_base.DatasetTestBase): def testMakeNumaAware(self): @@ -34,13 +35,8 @@ class MakeNumaAwareTest(test_base.DatasetTestBase): options = dataset_ops.Options() options.experimental_numa_aware = True dataset = dataset.with_options(options) - iterator = dataset.make_one_shot_iterator() - get_next = iterator.get_next() - - with self.cached_session() as sess: - self.assertAllEqual([x * x for x in range(10)], sess.run(get_next)) - with self.assertRaises(errors.OutOfRangeError): - sess.run(get_next) + self.assertDatasetProduces( + dataset, expected_output=[[x * x for x in range(10)]]) if __name__ == "__main__": diff --git a/tensorflow/python/data/experimental/kernel_tests/optimization/map_and_batch_fusion_test.py b/tensorflow/python/data/experimental/kernel_tests/optimization/map_and_batch_fusion_test.py index ddf3cbbcc35..801f664f09c 100644 --- a/tensorflow/python/data/experimental/kernel_tests/optimization/map_and_batch_fusion_test.py +++ b/tensorflow/python/data/experimental/kernel_tests/optimization/map_and_batch_fusion_test.py @@ -18,12 +18,14 @@ from __future__ import division from __future__ import print_function from tensorflow.python.data.experimental.ops import optimization +from tensorflow.python.data.experimental.ops.optimization_options import OptimizationOptions from tensorflow.python.data.kernel_tests import test_base from tensorflow.python.data.ops import dataset_ops -from tensorflow.python.framework import errors +from tensorflow.python.framework import test_util from tensorflow.python.platform import test +@test_util.run_all_in_graph_and_eager_modes class MapAndBatchFusionTest(test_base.DatasetTestBase): def testMapAndBatchFusion(self): @@ -31,15 +33,11 @@ class MapAndBatchFusionTest(test_base.DatasetTestBase): optimization.assert_next( ["MapAndBatch"])).map(lambda x: x * x).batch(10) options = dataset_ops.Options() - options.experimental_map_and_batch_fusion = True + options.experimental_optimization = OptimizationOptions() + options.experimental_optimization.map_and_batch_fusion = True dataset = dataset.with_options(options) - iterator = dataset.make_one_shot_iterator() - get_next = iterator.get_next() - - with self.cached_session() as sess: - self.assertAllEqual([x * x for x in range(10)], sess.run(get_next)) - with self.assertRaises(errors.OutOfRangeError): - sess.run(get_next) + self.assertDatasetProduces( + dataset, expected_output=[[x * x for x in range(10)]]) if __name__ == "__main__": diff --git a/tensorflow/python/data/experimental/kernel_tests/optimization/map_and_filter_fusion_test.py b/tensorflow/python/data/experimental/kernel_tests/optimization/map_and_filter_fusion_test.py index 3b4ca623409..db8f214fbfc 100644 --- a/tensorflow/python/data/experimental/kernel_tests/optimization/map_and_filter_fusion_test.py +++ b/tensorflow/python/data/experimental/kernel_tests/optimization/map_and_filter_fusion_test.py @@ -20,11 +20,12 @@ from __future__ import print_function from absl.testing import parameterized from tensorflow.python.data.experimental.ops import optimization +from tensorflow.python.data.experimental.ops.optimization_options import OptimizationOptions from tensorflow.python.data.kernel_tests import test_base from tensorflow.python.data.ops import dataset_ops from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes -from tensorflow.python.framework import errors +from tensorflow.python.framework import test_util from tensorflow.python.ops import math_ops from tensorflow.python.platform import test @@ -62,23 +63,20 @@ def _map_and_filter_fusion_test_cases(): return tuple(tests) +@test_util.run_all_in_graph_and_eager_modes class MapAndFilterFusionTest(test_base.DatasetTestBase, parameterized.TestCase): def _testMapAndFilter(self, dataset, function, predicate): - iterator = dataset.make_one_shot_iterator() - get_next = iterator.get_next() - with self.cached_session() as sess: - for x in range(10): - r = function(x) - if isinstance(r, tuple): - b = predicate(*r) # Pass tuple as multiple arguments. - else: - b = predicate(r) - if sess.run(b): - result = sess.run(get_next) - self.assertAllEqual(r, result) - with self.assertRaises(errors.OutOfRangeError): - sess.run(get_next) + expected_output = [] + for x in range(10): + r = function(x) + if isinstance(r, tuple): + b = predicate(*r) # Pass tuple as multiple arguments. + else: + b = predicate(r) + if self.evaluate(b): + expected_output.append(r) + self.assertDatasetProduces(dataset, expected_output=expected_output) @parameterized.named_parameters(*_map_and_filter_fusion_test_cases()) def testMapFilterFusion(self, function, predicate): @@ -86,7 +84,8 @@ class MapAndFilterFusionTest(test_base.DatasetTestBase, parameterized.TestCase): optimization.assert_next( ["Map", "FilterByLastComponent"])).map(function).filter(predicate) options = dataset_ops.Options() - options.experimental_map_and_filter_fusion = True + options.experimental_optimization = OptimizationOptions() + options.experimental_optimization.map_and_filter_fusion = True dataset = dataset.with_options(options) self._testMapAndFilter(dataset, function, predicate) @@ -104,7 +103,8 @@ class MapAndFilterFusionTest(test_base.DatasetTestBase, parameterized.TestCase): optimization.assert_next(["Map", "Filter"])).map(function).filter(predicate) options = dataset_ops.Options() - options.experimental_map_and_filter_fusion = True + options.experimental_optimization = OptimizationOptions() + options.experimental_optimization.map_and_filter_fusion = True dataset = dataset.with_options(options) self._testMapAndFilter(dataset, function, predicate) diff --git a/tensorflow/python/data/experimental/kernel_tests/optimization/map_fusion_test.py b/tensorflow/python/data/experimental/kernel_tests/optimization/map_fusion_test.py index ec63ad72006..d8d63903749 100644 --- a/tensorflow/python/data/experimental/kernel_tests/optimization/map_fusion_test.py +++ b/tensorflow/python/data/experimental/kernel_tests/optimization/map_fusion_test.py @@ -20,9 +20,10 @@ from __future__ import print_function from absl.testing import parameterized from tensorflow.python.data.experimental.ops import optimization +from tensorflow.python.data.experimental.ops.optimization_options import OptimizationOptions from tensorflow.python.data.kernel_tests import test_base from tensorflow.python.data.ops import dataset_ops -from tensorflow.python.framework import errors +from tensorflow.python.framework import test_util from tensorflow.python.platform import test @@ -62,6 +63,7 @@ def _map_fusion_test_cases(): return tuple(tests) +@test_util.run_all_in_graph_and_eager_modes class MapFusionTest(test_base.DatasetTestBase, parameterized.TestCase): @parameterized.named_parameters(*_map_fusion_test_cases()) @@ -73,23 +75,19 @@ class MapFusionTest(test_base.DatasetTestBase, parameterized.TestCase): dataset = dataset.cache() options = dataset_ops.Options() - options.experimental_map_fusion = True + options.experimental_optimization = OptimizationOptions() + options.experimental_optimization.map_fusion = True dataset = dataset.with_options(options) - iterator = dataset.make_one_shot_iterator() - get_next = iterator.get_next() - with self.cached_session() as sess: - for x in range(5): - result = sess.run(get_next) - r = x - for function in functions: - if isinstance(r, tuple): - r = function(*r) # Pass tuple as multiple arguments. - else: - r = function(r) - self.assertAllEqual(r, result) - - with self.assertRaises(errors.OutOfRangeError): - sess.run(get_next) + expected_output = [] + for x in range(5): + r = x + for function in functions: + if isinstance(r, tuple): + r = function(*r) # Pass tuple as multiple arguments. + else: + r = function(r) + expected_output.append(r) + self.assertDatasetProduces(dataset, expected_output=expected_output) if __name__ == "__main__": diff --git a/tensorflow/python/data/experimental/kernel_tests/optimization/map_parallelization_test.py b/tensorflow/python/data/experimental/kernel_tests/optimization/map_parallelization_test.py index c95f7b2eb19..0ff3fff4f85 100644 --- a/tensorflow/python/data/experimental/kernel_tests/optimization/map_parallelization_test.py +++ b/tensorflow/python/data/experimental/kernel_tests/optimization/map_parallelization_test.py @@ -20,11 +20,12 @@ from __future__ import print_function from absl.testing import parameterized from tensorflow.python.data.experimental.ops import optimization +from tensorflow.python.data.experimental.ops.optimization_options import OptimizationOptions from tensorflow.python.data.kernel_tests import test_base from tensorflow.python.data.ops import dataset_ops from tensorflow.python.framework import dtypes -from tensorflow.python.framework import errors from tensorflow.python.framework import ops +from tensorflow.python.framework import test_util from tensorflow.python.ops import control_flow_ops from tensorflow.python.ops import math_ops from tensorflow.python.ops import random_ops @@ -58,6 +59,7 @@ def _map_parallelization_test_cases(): ("AssertWithRandom", assert_with_random, False)) +@test_util.run_all_in_graph_and_eager_modes class MapParallelizationTest(test_base.DatasetTestBase, parameterized.TestCase): @parameterized.named_parameters(*_map_parallelization_test_cases()) @@ -66,23 +68,12 @@ class MapParallelizationTest(test_base.DatasetTestBase, parameterized.TestCase): dataset = dataset_ops.Dataset.range(5).apply( optimization.assert_next(next_nodes)).map(function) options = dataset_ops.Options() - options.experimental_map_parallelization = True + options.experimental_optimization = OptimizationOptions() + options.experimental_optimization.map_parallelization = True dataset = dataset.with_options(options) - iterator = dataset.make_one_shot_iterator() - get_next = iterator.get_next() - - with self.test_session() as sess: - for x in range(5): - result = sess.run(get_next) - # No need to run the pipeline if it was not optimized. Also the results - # might be hard to check because of random. - if not should_optimize: - return - r = function(x) - self.assertAllEqual(r, result) - - with self.assertRaises(errors.OutOfRangeError): - sess.run(get_next) + if should_optimize: + self.assertDatasetProduces( + dataset, expected_output=[function(x) for x in range(5)]) if __name__ == "__main__": diff --git a/tensorflow/python/data/experimental/kernel_tests/optimization/map_vectorization_test.py b/tensorflow/python/data/experimental/kernel_tests/optimization/map_vectorization_test.py index f10b66ff691..c2e08e2cd8c 100644 --- a/tensorflow/python/data/experimental/kernel_tests/optimization/map_vectorization_test.py +++ b/tensorflow/python/data/experimental/kernel_tests/optimization/map_vectorization_test.py @@ -17,23 +17,21 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function -import time - from absl.testing import parameterized import numpy as np from tensorflow.core.example import example_pb2 from tensorflow.core.example import feature_pb2 -from tensorflow.python.client import session from tensorflow.python.data.experimental.ops import optimization +from tensorflow.python.data.experimental.ops.optimization_options import OptimizationOptions from tensorflow.python.data.kernel_tests import test_base from tensorflow.python.data.ops import dataset_ops -from tensorflow.python.data.util import nest from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes from tensorflow.python.framework import errors from tensorflow.python.framework import ops from tensorflow.python.framework import sparse_tensor +from tensorflow.python.framework import test_util from tensorflow.python.ops import array_ops from tensorflow.python.ops import bitwise_ops from tensorflow.python.ops import check_ops @@ -319,6 +317,7 @@ def _generate_optimization_test_cases(): } for x in test_cases for num_parallel_calls in (None, 12)] +@test_util.run_all_in_graph_and_eager_modes class MapVectorizationTest(test_base.DatasetTestBase, parameterized.TestCase): def _get_test_datasets(self, @@ -355,7 +354,8 @@ class MapVectorizationTest(test_base.DatasetTestBase, parameterized.TestCase): optimized = _make_dataset(["Batch", map_node_name] if expect_optimized else [map_node_name, "Batch"]) options = dataset_ops.Options() - options.experimental_map_vectorization = True + options.experimental_optimization = OptimizationOptions() + options.experimental_optimization.map_vectorization = True optimized = optimized.with_options(options) return unoptimized, optimized @@ -366,7 +366,8 @@ class MapVectorizationTest(test_base.DatasetTestBase, parameterized.TestCase): num_parallel_calls) self.assertDatasetsEqual(unoptimized, optimized) - def testOptimizationBadMapFn(self): + # TODO(b/117581999): Add eager coverage for the following tests. + def testSkipEagerOptimizationBadMapFn(self): # Test map functions that give an error def map_fn(x): # x has leading dimension 5, this will raise an error @@ -375,7 +376,7 @@ class MapVectorizationTest(test_base.DatasetTestBase, parameterized.TestCase): base_dataset = dataset_ops.Dataset.range(5).repeat(5).batch( 5, drop_remainder=True) _, optimized = self._get_test_datasets(base_dataset, map_fn) - nxt = optimized.make_one_shot_iterator().get_next() + nxt = dataset_ops.make_one_shot_iterator(optimized).get_next() with self.assertRaisesRegexp(errors.InvalidArgumentError, r"indices = 10 is not in \[0, 5\)"): self.evaluate(nxt) @@ -394,7 +395,8 @@ class MapVectorizationTest(test_base.DatasetTestBase, parameterized.TestCase): base_dataset, map_fn, expect_optimized=True) self.assertDatasetsEqual(optimized, unoptimized) - def testOptimizationIgnoreStateful(self): + # TODO(b/117581999): Add eager coverage for the following tests. + def testSkipEagerOptimizationIgnoreStateful(self): def map_fn(x): with ops.control_dependencies([check_ops.assert_equal(x, 0)]): @@ -420,7 +422,8 @@ class MapVectorizationTest(test_base.DatasetTestBase, parameterized.TestCase): base_dataset, map_fn, expect_optimized=False) self.assertDatasetsEqual(unoptimized, optimized) - def testOptimizationIgnoreRaggedMap(self): + # TODO(b/117581999): Add eager coverage for the following tests. + def testSkipEagerOptimizationIgnoreRaggedMap(self): # Don't optimize when the output of the map fn shapes are unknown. def map_fn(x): return array_ops.tile(x, x) @@ -434,102 +437,5 @@ class MapVectorizationTest(test_base.DatasetTestBase, parameterized.TestCase): ("IteratorGetNext", "IteratorGetNext_1", 1)]) -class MapVectorizationBenchmark(test.Benchmark): - # TODO(rachelim): Add a benchmark for more expensive transformations, such as - # vgg_preprocessing. - - def _run(self, x, num_iters=100, name=None): - deltas = [] - with session.Session() as sess: - for _ in range(5): - # Warm up session... - sess.run(x) - for _ in range(num_iters): - start = time.time() - sess.run(x) - end = time.time() - deltas.append(end - start) - median_time = np.median(deltas) - self.report_benchmark(iters=num_iters, wall_time=median_time, name=name) - return median_time - - def _compare(self, input_dataset, map_fn, batch_size, input_size, str_id): - num_elems = int(np.sum([np.prod(x) for x in input_size])) - name_template = "{}__batch_size_{}_input_element_size_{}_{}" - unoptimized = input_dataset.map(map_fn).batch(batch_size) - unoptimized_op = unoptimized.make_one_shot_iterator().get_next() - - optimized = input_dataset.map(map_fn).batch(batch_size) - options = dataset_ops.Options() - options.experimental_map_vectorization = True - optimized = optimized.with_options(options) - optimized_op = optimized.make_one_shot_iterator().get_next() - - unoptimized_time = self._run( - unoptimized_op, - name=name_template.format(str_id, batch_size, num_elems, "unoptimized")) - optimized_time = self._run( - optimized_op, - name=name_template.format(str_id, batch_size, num_elems, "optimized")) - - print("Batch size: {}\n" - "Input element size: {}\n" - "Transformation: {}\n" - "Speedup: {}\n".format(batch_size, input_size, str_id, - (unoptimized_time / optimized_time))) - - # Known cheap functions - def benchmarkIdentity(self): - self._benchmark_helper(lambda *args: [array_ops.identity(x) for x in args], - "identity") - - def benchmarkAddConst(self): - self._benchmark_helper(lambda *args: [x + 1 for x in args], "add_const") - - def benchmarkReturnConst(self): - self._benchmark_helper(lambda *args: [constant_op.constant(2)], "ret_const") - - def benchmarkSelect(self): - self._benchmark_helper(lambda *args: args[0], "select") - - def benchmarkCast(self): - self._benchmark_helper( - lambda *args: [math_ops.cast(x, dtypes.float64) for x in args], "cast") - - def benchmarkReshape(self): - self._benchmark_helper( - lambda *args: [array_ops.reshape(x, (-1, 30)) for x in args], "reshape") - - def benchmarkDecodeCSV(self): - csv_fn, csv_factory = _generate_csv_test_case() - self._benchmark_helper(csv_fn, "decode_csv", lambda: [csv_factory()]) - - def benchmarkParseSingleExample(self): - # NOTE: Since we haven't implemented a vectorizer for "SerializeSparse", - # this function is only naively vectorized. - parse_fn, parse_factory = _generate_parse_single_example_test_case() - - self._benchmark_helper(parse_fn, "parse_single_example", - lambda: [parse_factory()]) - - def _default_dataset_factory(self): - input_sizes = [(10, 10, 3), (10, 100, 300)] - for sz in input_sizes: - yield dataset_ops.Dataset.from_tensor_slices(np.random.rand(*sz)) - - def _benchmark_helper(self, map_fn, str_id, base_dataset_factory=None): - if base_dataset_factory is None: - base_dataset_factory = self._default_dataset_factory - - batch_size = 1000 - for base_dataset in base_dataset_factory(): - base_dataset = base_dataset.repeat() - input_size = [ - tuple(shape.as_list()) - for shape in nest.flatten(base_dataset.output_shapes) - ] - self._compare(base_dataset, map_fn, batch_size, input_size, str_id) - - if __name__ == "__main__": test.main() diff --git a/tensorflow/python/data/experimental/kernel_tests/optimization/model_dataset_test.py b/tensorflow/python/data/experimental/kernel_tests/optimization/model_dataset_test.py index 5b49bdf4532..0f0274b41f2 100644 --- a/tensorflow/python/data/experimental/kernel_tests/optimization/model_dataset_test.py +++ b/tensorflow/python/data/experimental/kernel_tests/optimization/model_dataset_test.py @@ -17,181 +17,18 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function -import time - from absl.testing import parameterized -import numpy as np -from tensorflow.python.data.experimental.ops import batching from tensorflow.python.data.experimental.ops import optimization from tensorflow.python.data.kernel_tests import test_base from tensorflow.python.data.ops import dataset_ops from tensorflow.python.framework import errors -from tensorflow.python.ops import math_ops from tensorflow.python.platform import test +# TODO(b/117581999): Add eager coverage for the following tests. class ModelDatasetTest(test_base.DatasetTestBase, parameterized.TestCase): - def testModelMap(self): - k = 1024 * 1024 - dataset = dataset_ops.Dataset.from_tensors((np.random.rand(1, 4 * k), - np.random.rand(4 * k, - 1))).repeat() - dataset = dataset.map(math_ops.matmul) - dataset = dataset_ops._ModelDataset(dataset) - iterator = dataset.make_one_shot_iterator() - get_next = iterator.get_next() - - deltas = [] - with self.cached_session() as sess: - for _ in range(5): - sess.run(get_next.op) - for _ in range(100): - start = time.time() - sess.run(get_next.op) - end = time.time() - deltas.append(end - start) - - print("%f (median), %f (mean), %f (stddev), %f (min), %f (max)\n" % - (np.median(deltas), np.mean(deltas), np.std(deltas), np.min(deltas), - np.max(deltas))) - - def testModelParallelMap(self): - k = 1024 * 1024 - dataset = dataset_ops.Dataset.from_tensors((np.random.rand(1, 4 * k), - np.random.rand(4 * k, - 1))).repeat() - dataset = dataset.map( - math_ops.matmul, num_parallel_calls=optimization.AUTOTUNE) - dataset = dataset_ops._ModelDataset(dataset) - iterator = dataset.make_one_shot_iterator() - get_next = iterator.get_next() - - deltas = [] - with self.cached_session() as sess: - for _ in range(5): - sess.run(get_next.op) - for _ in range(1000): - start = time.time() - sess.run(get_next.op) - end = time.time() - deltas.append(end - start) - - print("%f (median), %f (mean), %f (stddev), %f (min), %f (max)\n" % - (np.median(deltas), np.mean(deltas), np.std(deltas), np.min(deltas), - np.max(deltas))) - - @parameterized.named_parameters( - ("Default", False), - ("NUMA", True), - ) - def testModelMapAndBatch(self, numa_aware): - batch_size = 16 - k = 1024 * 1024 - dataset = dataset_ops.Dataset.from_tensors((np.random.rand(1, 4 * k), - np.random.rand(4 * k, - 1))).repeat() - dataset = dataset.apply( - batching.map_and_batch( - math_ops.matmul, - num_parallel_calls=optimization.AUTOTUNE, - batch_size=batch_size)) - dataset = dataset_ops._ModelDataset(dataset) - options = dataset_ops.Options() - options.experimental_numa_aware = numa_aware - dataset = dataset.with_options(options) - iterator = dataset.make_one_shot_iterator() - get_next = iterator.get_next() - - deltas = [] - with self.cached_session() as sess: - for _ in range(5): - sess.run(get_next.op) - for _ in range(10): - start = time.time() - sess.run(get_next.op) - end = time.time() - deltas.append(end - start) - - print("%f (median), %f (mean), %f (stddev), %f (min), %f (max)\n" % - (np.median(deltas), np.mean(deltas), np.std(deltas), np.min(deltas), - np.max(deltas))) - - def testModelParallelInterleave(self): - k = 1024 * 1024 - dataset = dataset_ops.Dataset.from_tensors((np.random.rand(1, 4 * k), - np.random.rand(4 * k, - 1))).repeat() - dataset = dataset.map(math_ops.matmul) - dataset = dataset_ops.Dataset.range(1).repeat().interleave( - lambda _: dataset, - cycle_length=10, - num_parallel_calls=optimization.AUTOTUNE) - dataset = dataset_ops._ModelDataset(dataset) - iterator = dataset.make_one_shot_iterator() - get_next = iterator.get_next() - - deltas = [] - with self.cached_session() as sess: - for _ in range(5): - sess.run(get_next.op) - for _ in range(1000): - start = time.time() - sess.run(get_next.op) - end = time.time() - deltas.append(end - start) - - print("%f (median), %f (mean), %f (stddev), %f (min), %f (max)\n" % - (np.median(deltas), np.mean(deltas), np.std(deltas), np.min(deltas), - np.max(deltas))) - - def testModelNested(self): - k = 1024 * 1024 - a = (np.random.rand(1, 8 * k), np.random.rand(8 * k, 1)) - b = (np.random.rand(1, 4 * k), np.random.rand(4 * k, 1)) - c = (np.random.rand(1, 2 * k), np.random.rand(2 * k, 1)) - dataset = dataset_ops.Dataset.from_tensors((a, b, c)).repeat() - - def f1(a, b, c): - x, y = a - return math_ops.matmul(x, y), b, c - - def f2(a, b, c): - x, y = b - return a, math_ops.matmul(x, y), c - - def f3(a, b, c): - x, y = c - return a, b, math_ops.matmul(x, y) - - dataset = dataset.map(f1, num_parallel_calls=optimization.AUTOTUNE) - dataset = dataset_ops.Dataset.range(1).repeat().interleave( - lambda _: dataset, cycle_length=2) - - dataset = dataset.map(f2, num_parallel_calls=optimization.AUTOTUNE) - dataset = dataset_ops.Dataset.range(1).repeat().interleave( - lambda _: dataset, cycle_length=2) - - dataset = dataset.map(f3, num_parallel_calls=optimization.AUTOTUNE) - dataset = dataset_ops._ModelDataset(dataset) - iterator = dataset.make_one_shot_iterator() - get_next = iterator.get_next() - - deltas = [] - with self.cached_session() as sess: - for _ in range(5): - sess.run(get_next) - for _ in range(100): - start = time.time() - sess.run(get_next) - end = time.time() - deltas.append(end - start) - - print("%f (median), %f (mean), %f (stddev), %f (min), %f (max)\n" % - (np.median(deltas), np.mean(deltas), np.std(deltas), np.min(deltas), - np.max(deltas))) - def testAutotuneOption(self): dataset = dataset_ops.Dataset.from_tensors(0) dataset = dataset.map(lambda x: x).apply( @@ -200,13 +37,13 @@ class ModelDatasetTest(test_base.DatasetTestBase, parameterized.TestCase): options.experimental_autotune = True dataset = dataset.with_options(options) - iterator = dataset.make_one_shot_iterator() + iterator = dataset_ops.make_one_shot_iterator(dataset) get_next = iterator.get_next() with self.cached_session() as sess: - self.assertEqual(0, sess.run(get_next)) + self.assertEqual(0, self.evaluate(get_next)) with self.assertRaises(errors.OutOfRangeError): - sess.run(get_next) + self.evaluate(get_next) if __name__ == "__main__": diff --git a/tensorflow/python/data/experimental/kernel_tests/optimization/noop_elimination_test.py b/tensorflow/python/data/experimental/kernel_tests/optimization/noop_elimination_test.py index ddcd7f4da4b..ce86bfa4e0f 100644 --- a/tensorflow/python/data/experimental/kernel_tests/optimization/noop_elimination_test.py +++ b/tensorflow/python/data/experimental/kernel_tests/optimization/noop_elimination_test.py @@ -18,15 +18,17 @@ from __future__ import division from __future__ import print_function from tensorflow.python.data.experimental.ops import optimization +from tensorflow.python.data.experimental.ops.optimization_options import OptimizationOptions from tensorflow.python.data.kernel_tests import test_base from tensorflow.python.data.ops import dataset_ops from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes -from tensorflow.python.framework import errors +from tensorflow.python.framework import test_util from tensorflow.python.ops import math_ops from tensorflow.python.platform import test +@test_util.run_all_in_graph_and_eager_modes class NoopEliminationTest(test_base.DatasetTestBase): def testNoopElimination(self): @@ -41,18 +43,10 @@ class NoopEliminationTest(test_base.DatasetTestBase): dataset = dataset.repeat(some_tensor).skip(5).take(-1).skip(0).repeat( 1).prefetch(0).prefetch(1).cache() options = dataset_ops.Options() - options.experimental_noop_elimination = True + options.experimental_optimization = OptimizationOptions() + options.experimental_optimization.noop_elimination = True dataset = dataset.with_options(options) - iterator = dataset.make_one_shot_iterator() - get_next = iterator.get_next() - - with self.test_session() as sess: - for x in range(5): - result = sess.run(get_next) - self.assertAllEqual(result, x) - - with self.assertRaises(errors.OutOfRangeError): - sess.run(get_next) + self.assertDatasetProduces(dataset, expected_output=range(5)) if __name__ == "__main__": diff --git a/tensorflow/python/data/experimental/kernel_tests/optimization/optimize_dataset_test.py b/tensorflow/python/data/experimental/kernel_tests/optimization/optimize_dataset_test.py index 739b6a9bf4c..751be833267 100644 --- a/tensorflow/python/data/experimental/kernel_tests/optimization/optimize_dataset_test.py +++ b/tensorflow/python/data/experimental/kernel_tests/optimization/optimize_dataset_test.py @@ -17,54 +17,125 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function +import warnings + +from absl.testing import parameterized import numpy as np +from tensorflow.python.data.experimental.ops import batching +from tensorflow.python.data.experimental.ops import grouping from tensorflow.python.data.experimental.ops import optimization +from tensorflow.python.data.experimental.ops import optimization_options +from tensorflow.python.data.experimental.ops import scan_ops from tensorflow.python.data.experimental.ops import threadpool from tensorflow.python.data.kernel_tests import test_base from tensorflow.python.data.ops import dataset_ops from tensorflow.python.framework import dtypes from tensorflow.python.framework import errors +from tensorflow.python.framework import ops +from tensorflow.python.framework import test_util from tensorflow.python.ops import array_ops from tensorflow.python.ops import random_ops +from tensorflow.python.ops import variable_scope from tensorflow.python.platform import test -class OptimizeDatasetTest(test_base.DatasetTestBase): +def _generate_captured_refvar_test_cases(): + """Generates testcases. + + Returns: + A list of tuples of (testcase_name, make_dataset_fn). make_dataset_fn takes + a tf.Variable as input and creates a test dataset that uses that variable. + """ + + def make_map_dataset(var): + return dataset_ops.Dataset.from_tensors(0).map(lambda x: x + var) + + def make_flat_map_dataset(var): + return dataset_ops.Dataset.from_tensors( + 0).flat_map(lambda _: dataset_ops.Dataset.from_tensors(var)) + + def make_filter_dataset(var): + return dataset_ops.Dataset.from_tensors(0).filter(lambda x: x < var) + + def make_map_and_batch_dataset(var): + + def map_fn(x): + return x + var + + return dataset_ops.Dataset.from_tensors(0).apply( + batching.map_and_batch(map_fn, 1)) + + def make_group_by_reducer_dataset(var): + reducer = grouping.Reducer( + init_func=lambda _: 0, + reduce_func=lambda x, y: x, + finalize_func=lambda _: var) + return dataset_ops.Dataset.range(5).apply( + grouping.group_by_reducer(lambda x: x % 2, reducer)) + + def make_group_by_window_dataset(var): + + def reduce_fn(key, bucket): + del key, bucket + return dataset_ops.Dataset.from_tensors(var) + + return dataset_ops.Dataset.from_tensors(0).repeat(10).apply( + grouping.group_by_window(lambda _: 0, reduce_fn, 10)) + + def make_scan_dataset(var): + return dataset_ops.Dataset.from_tensors(0).apply( + scan_ops.scan( + 0, lambda old_state, elem: (old_state + 1, elem + old_state + var))) + + return [ + # Core datasets + ("Map", make_map_dataset), + ("FlatMap", make_flat_map_dataset), + ("Filter", make_filter_dataset), + # Experimental datasets + ("MapAndBatch", make_map_and_batch_dataset), + ("GroupByReducer", make_group_by_reducer_dataset), + ("GroupByWindow", make_group_by_window_dataset), + ("Scan", make_scan_dataset) + ] + + +@test_util.run_all_in_graph_and_eager_modes +class OptimizeDatasetTest(test_base.DatasetTestBase, parameterized.TestCase): def testOptimizationStatefulFunction(self): - dataset = dataset_ops.Dataset.range(10).map( - lambda _: random_ops.random_uniform([])).batch(10) + dataset = dataset_ops.Dataset.range( + 10).map(lambda _: random_ops.random_uniform([])).batch(10) dataset = dataset_ops._OptimizeDataset(dataset, []) - iterator = dataset.make_one_shot_iterator() - get_next = iterator.get_next() + get_next = self.getNext(dataset) + self.evaluate(get_next()) - with self.cached_session() as sess: - sess.run(get_next) - - def testOptimizationLargeInputFromTensor(self): + # TODO(b/117581999): Add eager coverage for the following tests. + def testSkipEagerOptimizationLargeInputFromTensor(self): input_t = array_ops.placeholder(dtypes.int32, (None, None, None)) dataset = dataset_ops.Dataset.from_tensors(input_t) dataset = dataset_ops._OptimizeDataset(dataset, []) - iterator = dataset.make_initializable_iterator() + iterator = dataset_ops.make_initializable_iterator(dataset) init_op = iterator.initializer get_next = iterator.get_next() with self.cached_session() as sess: sess.run(init_op, {input_t: np.ones([512, 1024, 1025], np.int32)}) - sess.run(get_next) + self.evaluate(get_next) - def testOptimizationLargeInputFromTensorSlices(self): + # TODO(b/117581999): Add eager coverage for the following tests. + def testSkipEagerOptimizationLargeInputFromTensorSlices(self): input_t = array_ops.placeholder(dtypes.int32, (None, None, None, None)) dataset = dataset_ops.Dataset.from_tensor_slices(input_t) dataset = dataset_ops._OptimizeDataset(dataset, []) - iterator = dataset.make_initializable_iterator() + iterator = dataset_ops.make_initializable_iterator(dataset) init_op = iterator.initializer get_next = iterator.get_next() with self.cached_session() as sess: sess.run(init_op, {input_t: np.ones([1, 512, 1024, 1025], np.int32)}) - sess.run(get_next) + self.evaluate(get_next) def testOptimizationNestedDataset(self): @@ -78,13 +149,22 @@ class OptimizeDatasetTest(test_base.DatasetTestBase): dataset = dataset_ops.Dataset.range(1) dataset = dataset.flat_map(flat_map_fn) dataset = dataset_ops._OptimizeDataset(dataset, ["noop_elimination"]) - iterator = dataset.make_one_shot_iterator() - get_next = iterator.get_next() + self.assertDatasetProduces(dataset, expected_output=[0]) - with self.cached_session() as sess: - self.assertEquals(0, sess.run(get_next)) - with self.assertRaises(errors.OutOfRangeError): - sess.run(get_next) + def testOptimizationNestedDatasetWithModifiedRetval(self): + + def flat_map_fn(_): + dataset = dataset_ops.Dataset.from_tensors(0) + dataset = dataset.apply(optimization.assert_next(["MapAndBatch"])) + # Should be fused by map and batch fusion + dataset = dataset.map(lambda x: x) + dataset = dataset.batch(1) + return dataset + + dataset = dataset_ops.Dataset.range(1) + dataset = dataset.flat_map(flat_map_fn) + dataset = dataset_ops._OptimizeDataset(dataset, ["map_and_batch_fusion"]) + self.assertDatasetProduces(dataset, expected_output=[[0]]) def testOptimizationThreadPoolDataset(self): dataset = dataset_ops.Dataset.range(10).batch(10) @@ -95,14 +175,10 @@ class OptimizeDatasetTest(test_base.DatasetTestBase): 2, display_name="private_thread_pool_%d" % 2)) dataset = dataset_ops._OptimizeDataset(dataset, []) - iterator = dataset.make_initializable_iterator() - get_next = iterator.get_next() - - with self.cached_session() as sess: - sess.run(iterator.initializer) - self.assertAllEqual(list(range(10)), sess.run(get_next)) - with self.assertRaises(errors.OutOfRangeError): - sess.run(get_next) + self.assertDatasetProduces( + dataset, + expected_output=[list(range(10))], + requires_initialization=True) def testOptimizationNonSerializable(self): dataset = dataset_ops.Dataset.from_tensors(0) @@ -113,26 +189,61 @@ class OptimizeDatasetTest(test_base.DatasetTestBase): dataset = dataset.skip(0) # Should be removed by noop elimination dataset = dataset.cache() dataset = dataset_ops._OptimizeDataset(dataset, ["noop_elimination"]) - iterator = dataset.make_one_shot_iterator() - get_next = iterator.get_next() - - with self.cached_session() as sess: - self.assertEquals(0, sess.run(get_next)) - with self.assertRaises(errors.OutOfRangeError): - sess.run(get_next) + self.assertDatasetProduces(dataset, expected_output=[0]) def testOptimizationNonSerializableAsDirectInput(self): - """Tests that non-serializable dataset can be OptimizeDataset's input. - """ + """Tests that non-serializable dataset can be OptimizeDataset's input.""" dataset = dataset_ops.Dataset.from_tensors(0) dataset = dataset.apply(optimization.non_serializable()) dataset = dataset_ops._OptimizeDataset(dataset, ["noop_elimination"]) - iterator = dataset.make_one_shot_iterator() - get_next = iterator.get_next() - with self.cached_session() as sess: - self.assertEquals(0, sess.run(get_next)) - with self.assertRaises(errors.OutOfRangeError): - sess.run(get_next) + self.assertDatasetProduces(dataset, expected_output=[0]) + + @parameterized.named_parameters(_generate_captured_refvar_test_cases()) + # Skip eager because RefVariables are not supported in eager mode. + def testSkipEagerOptimizationWithCapturedRefVar(self, dataset_fn): + """Tests that default optimizations are disabled with ref variables.""" + variable = variable_scope.get_variable( + "v", initializer=0, use_resource=False) + assign_op = variable.assign_add(1) + + unoptimized_dataset = dataset_fn(variable) + + options = dataset_ops.Options() + opt_options = optimization_options.OptimizationOptions() + opt_options.noop_elimination = True + opt_options.map_and_batch_fusion = True + options.experimental_optimization = opt_options + optimized_dataset = unoptimized_dataset.with_options(options) + + # Check that warning is logged. + warnings.simplefilter("always") + with warnings.catch_warnings(record=True) as w: + optimized_it = optimized_dataset.make_initializable_iterator() + + self.assertGreaterEqual(len(w), 1) + expected = ("tf.data static optimizations are not compatible with " + "tf.Variable. The following optimizations will be disabled: " + "map_and_batch_fusion, noop_elimination. To enable " + "optimizations, use resource variables instead by calling " + "`tf.enable_resource_variables()` at the start of the program.") + self.assertTrue(any([expected in str(warning) for warning in w])) + + # Check that outputs are the same in the optimized and unoptimized cases, + # when the variable value is changing. + unoptimized_it = unoptimized_dataset.make_initializable_iterator() + with ops.control_dependencies([assign_op]): + unoptimized_output = unoptimized_it.get_next() + optimized_output = optimized_it.get_next() + + self.evaluate(variable.initializer) + self.evaluate((unoptimized_it.initializer, optimized_it.initializer)) + while True: + try: + unoptimized, optimized = self.evaluate((unoptimized_output, + optimized_output)) + self.assertEqual(unoptimized, optimized) + except errors.OutOfRangeError: + break if __name__ == "__main__": diff --git a/tensorflow/python/data/experimental/kernel_tests/optimization/shuffle_and_repeat_fusion_test.py b/tensorflow/python/data/experimental/kernel_tests/optimization/shuffle_and_repeat_fusion_test.py index 36582f449f3..5f746ec63ac 100644 --- a/tensorflow/python/data/experimental/kernel_tests/optimization/shuffle_and_repeat_fusion_test.py +++ b/tensorflow/python/data/experimental/kernel_tests/optimization/shuffle_and_repeat_fusion_test.py @@ -18,31 +18,35 @@ from __future__ import division from __future__ import print_function from tensorflow.python.data.experimental.ops import optimization +from tensorflow.python.data.experimental.ops.optimization_options import OptimizationOptions from tensorflow.python.data.kernel_tests import test_base from tensorflow.python.data.ops import dataset_ops from tensorflow.python.framework import errors +from tensorflow.python.framework import test_util from tensorflow.python.platform import test +@test_util.run_all_in_graph_and_eager_modes class ShuffleAndRepeatFusionTest(test_base.DatasetTestBase): def testShuffleAndRepeatFusion(self): dataset = dataset_ops.Dataset.range(10).apply( optimization.assert_next(["ShuffleAndRepeat"])).shuffle(10).repeat(2) options = dataset_ops.Options() - options.experimental_shuffle_and_repeat_fusion = True + options.experimental_optimization = OptimizationOptions() + options.experimental_optimization.shuffle_and_repeat_fusion = True dataset = dataset.with_options(options) - iterator = dataset.make_one_shot_iterator() - get_next = iterator.get_next() + get_next = self.getNext(dataset) - with self.cached_session() as sess: - for _ in range(2): - results = [] - for _ in range(10): - results.append(sess.run(get_next)) - self.assertAllEqual([x for x in range(10)], sorted(results)) - with self.assertRaises(errors.OutOfRangeError): - sess.run(get_next) + for _ in range(2): + results = [] + for _ in range(10): + results.append(self.evaluate(get_next())) + self.assertAllEqual([x for x in range(10)], sorted(results)) + with self.assertRaises(errors.OutOfRangeError): + self.evaluate(get_next()) + with self.assertRaises(errors.OutOfRangeError): + self.evaluate(get_next()) if __name__ == "__main__": diff --git a/tensorflow/python/data/experimental/kernel_tests/override_threadpool_test.py b/tensorflow/python/data/experimental/kernel_tests/override_threadpool_test.py index 5e419a9b2f9..aa81663a188 100644 --- a/tensorflow/python/data/experimental/kernel_tests/override_threadpool_test.py +++ b/tensorflow/python/data/experimental/kernel_tests/override_threadpool_test.py @@ -22,12 +22,15 @@ import threading from absl.testing import parameterized import numpy as np +from tensorflow.core.framework import graph_pb2 +from tensorflow.python.data.experimental.ops import threading_options from tensorflow.python.data.experimental.ops import threadpool from tensorflow.python.data.experimental.ops import unique from tensorflow.python.data.kernel_tests import test_base from tensorflow.python.data.ops import dataset_ops from tensorflow.python.framework import dtypes from tensorflow.python.framework import errors +from tensorflow.python.framework import test_util from tensorflow.python.ops import script_ops from tensorflow.python.platform import test @@ -35,18 +38,7 @@ from tensorflow.python.platform import test class OverrideThreadpoolTest(test_base.DatasetTestBase, parameterized.TestCase): - @parameterized.named_parameters( - ("1", 1, None), - ("2", 2, None), - ("3", 4, None), - ("4", 8, None), - ("5", 16, None), - ("6", 4, -1), - ("7", 4, 0), - ("8", 4, 1), - ("9", 4, 4), - ) - def testNumThreads(self, num_threads, max_intra_op_parallelism): + def _testNumThreadsHelper(self, num_threads, override_threadpool_fn): def get_thread_id(_): # Python creates a dummy thread object to represent the current @@ -60,32 +52,86 @@ class OverrideThreadpoolTest(test_base.DatasetTestBase, dataset_ops.Dataset.range(1000).map( lambda x: script_ops.py_func(get_thread_id, [x], dtypes.int64), num_parallel_calls=32).apply(unique.unique())) - - dataset = threadpool.override_threadpool( - dataset, - threadpool.PrivateThreadPool( - num_threads, - max_intra_op_parallelism=max_intra_op_parallelism, - display_name="private_thread_pool_%d" % num_threads)) - - iterator = dataset.make_initializable_iterator() + dataset = override_threadpool_fn(dataset) + iterator = dataset_ops.make_initializable_iterator(dataset) next_element = iterator.get_next() - with self.cached_session() as sess: - sess.run(iterator.initializer) - thread_ids = [] - try: - while True: - thread_ids.append(sess.run(next_element)) - except errors.OutOfRangeError: - pass - self.assertEqual(len(thread_ids), len(set(thread_ids))) - self.assertGreater(len(thread_ids), 0) + self.evaluate(iterator.initializer) + thread_ids = [] + try: + while True: + thread_ids.append(self.evaluate(next_element)) + except errors.OutOfRangeError: + pass + self.assertLen(thread_ids, len(set(thread_ids))) + self.assertNotEmpty(thread_ids) + if num_threads: # NOTE(mrry): We don't control the thread pool scheduling, and # so cannot guarantee that all of the threads in the pool will # perform work. self.assertLessEqual(len(thread_ids), num_threads) + @parameterized.named_parameters( + ("1", 1, None), + ("2", 2, None), + ("3", 4, None), + ("4", 8, None), + ("5", 16, None), + ("6", 4, -1), + ("7", 4, 0), + ("8", 4, 1), + ("9", 4, 4), + ) + @test_util.run_deprecated_v1 + def testNumThreadsDeprecated(self, num_threads, max_intra_op_parallelism): + + def override_threadpool_fn(dataset): + return threadpool.override_threadpool( + dataset, + threadpool.PrivateThreadPool( + num_threads, + max_intra_op_parallelism=max_intra_op_parallelism, + display_name="private_thread_pool_%d" % num_threads)) + + self._testNumThreadsHelper(num_threads, override_threadpool_fn) + + @parameterized.named_parameters( + ("1", 1, None), + ("2", 2, None), + ("3", 4, None), + ("4", 8, None), + ("5", 16, None), + ("6", None, 0), + ("7", None, 1), + ("8", None, 4), + ("9", 4, 0), + ("10", 4, 1), + ("11", 4, 4), + ("12", None, None), + ) + @test_util.run_deprecated_v1 + def testNumThreads(self, num_threads, max_intra_op_parallelism): + + def override_threadpool_fn(dataset): + t_options = threading_options.ThreadingOptions() + if max_intra_op_parallelism is not None: + t_options.max_intra_op_parallelism = max_intra_op_parallelism + if num_threads is not None: + t_options.private_threadpool_size = num_threads + options = dataset_ops.Options() + options.experimental_threading = t_options + return dataset.with_options(options) + + self._testNumThreadsHelper(num_threads, override_threadpool_fn) + + def testMaxIntraOpParallelismAsGraphDefInternal(self): + dataset = dataset_ops.Dataset.from_tensors(0) + dataset = dataset_ops._MaxIntraOpParallelismDataset(dataset, 1) + graph = graph_pb2.GraphDef().FromString( + self.evaluate(dataset._as_serialized_graph())) + self.assertTrue( + any([node.op != "MaxIntraOpParallelismDataset" for node in graph.node])) + if __name__ == "__main__": test.main() diff --git a/tensorflow/python/data/experimental/kernel_tests/parallel_interleave_test.py b/tensorflow/python/data/experimental/kernel_tests/parallel_interleave_test.py index 90ac250df70..113326c028a 100644 --- a/tensorflow/python/data/experimental/kernel_tests/parallel_interleave_test.py +++ b/tensorflow/python/data/experimental/kernel_tests/parallel_interleave_test.py @@ -86,7 +86,7 @@ class ParallelInterleaveTest(test_base.DatasetTestBase): self.block_length, self.sloppy, self.buffer_output_elements, self.prefetch_input_elements))) - self.iterator = self.dataset.make_initializable_iterator() + self.iterator = dataset_ops.make_initializable_iterator(self.dataset) self.init_op = self.iterator.initializer self.next_element = self.iterator.get_next() @@ -195,9 +195,9 @@ class ParallelInterleaveTest(test_base.DatasetTestBase): [[4] * 4, [5] * 5, [6] * 6] * self.repeat_count, 1, 1): self.write_coordination_events[expected_element].set() self.assertEqual(expected_element * expected_element, - sess.run(self.next_element)) + self.evaluate(self.next_element)) with self.assertRaises(errors.OutOfRangeError): - sess.run(self.next_element) + self.evaluate(self.next_element) def testSingleThreaded(self): self._testSingleThreaded() @@ -235,10 +235,10 @@ class ParallelInterleaveTest(test_base.DatasetTestBase): for expected_element in self._interleave( [[3] * 3, [7] * 7, [4] * 4] * self.repeat_count, 2, 1): self.write_coordination_events[expected_element].set() - output = sess.run(self.next_element) + output = self.evaluate(self.next_element) self.assertEqual(expected_element * expected_element, output) with self.assertRaises(errors.OutOfRangeError): - sess.run(self.next_element) + self.evaluate(self.next_element) def _testTwoThreadsNoContention(self, sloppy=False): # num_threads > 1. @@ -262,7 +262,7 @@ class ParallelInterleaveTest(test_base.DatasetTestBase): self.write_coordination_events[expected_element].set() if done_first_event: # First event starts the worker threads. self.read_coordination_events[expected_element].acquire() - actual_element = sess.run(self.next_element) + actual_element = self.evaluate(self.next_element) if not done_first_event: self.read_coordination_events[expected_element].acquire() done_first_event = True @@ -270,7 +270,7 @@ class ParallelInterleaveTest(test_base.DatasetTestBase): "At index %s: %s expected, got: %s" % (i, expected_element, actual_element)) with self.assertRaises(errors.OutOfRangeError): - sess.run(self.next_element) + self.evaluate(self.next_element) def testTwoThreadsNoContention(self): self._testTwoThreadsNoContention() @@ -309,7 +309,7 @@ class ParallelInterleaveTest(test_base.DatasetTestBase): else: self.write_coordination_events[expected_element].set() time.sleep(0.5) # Sleep to consistently "avoid" the race condition. - actual_element = sess.run(self.next_element) + actual_element = self.evaluate(self.next_element) if not done_first_event: done_first_event = True self.assertTrue( @@ -318,7 +318,7 @@ class ParallelInterleaveTest(test_base.DatasetTestBase): "At index %s: %s expected, got: %s" % (i, expected_element, actual_element)) with self.assertRaises(errors.OutOfRangeError): - sess.run(self.next_element) + self.evaluate(self.next_element) def testTwoThreadsNoContentionWithRaces(self): self._testTwoThreadsNoContentionWithRaces() @@ -348,7 +348,7 @@ class ParallelInterleaveTest(test_base.DatasetTestBase): self.write_coordination_events[expected_element].set() if done_first_event: # First event starts the worker threads. self.read_coordination_events[expected_element].acquire() - actual_element = sess.run(self.next_element) + actual_element = self.evaluate(self.next_element) if not done_first_event: done_first_event = True self.read_coordination_events[expected_element].acquire() @@ -356,7 +356,7 @@ class ParallelInterleaveTest(test_base.DatasetTestBase): "At index %s: %s expected, got: %s" % (i, expected_element, actual_element)) with self.assertRaises(errors.OutOfRangeError): - sess.run(self.next_element) + self.evaluate(self.next_element) def testTwoThreadsNoContentionBlockLength(self): self._testTwoThreadsNoContentionBlockLength() @@ -396,7 +396,7 @@ class ParallelInterleaveTest(test_base.DatasetTestBase): else: self.write_coordination_events[expected_element].set() time.sleep(0.5) # Sleep to consistently "avoid" the race condition. - actual_element = sess.run(self.next_element) + actual_element = self.evaluate(self.next_element) if not done_first_event: done_first_event = True self.assertTrue( @@ -405,7 +405,7 @@ class ParallelInterleaveTest(test_base.DatasetTestBase): "At index %s: %s expected, got: %s" % (i, expected_element, actual_element)) with self.assertRaises(errors.OutOfRangeError): - sess.run(self.next_element) + self.evaluate(self.next_element) def testTwoThreadsNoContentionWithRacesAndBlocking(self): self._testTwoThreadsNoContentionWithRacesAndBlocking() @@ -428,7 +428,7 @@ class ParallelInterleaveTest(test_base.DatasetTestBase): self.prefetch_input_elements: 0, }) with self.assertRaises(errors.OutOfRangeError): - sess.run(self.next_element) + self.evaluate(self.next_element) def testEmptyInput(self): self._testEmptyInput() @@ -451,7 +451,7 @@ class ParallelInterleaveTest(test_base.DatasetTestBase): self.prefetch_input_elements: 0, }) with self.assertRaises(errors.OutOfRangeError): - sess.run(self.next_element) + self.evaluate(self.next_element) def testNonEmptyInputIntoEmptyOutputs(self): self._testNonEmptyInputIntoEmptyOutputs() @@ -484,7 +484,7 @@ class ParallelInterleaveTest(test_base.DatasetTestBase): # presence of finishing iterators. if done_first_event and not (sloppy and (i in race_indices)): self.read_coordination_events[expected_element].acquire() - actual_element = sess.run(self.next_element) + actual_element = self.evaluate(self.next_element) if not done_first_event or (sloppy and (i in race_indices)): done_first_event = True self.read_coordination_events[expected_element].acquire() @@ -520,10 +520,10 @@ class ParallelInterleaveTest(test_base.DatasetTestBase): ] for element in mis_ordering: self.write_coordination_events[element].set() - self.assertEqual(element * element, sess.run(self.next_element)) + self.assertEqual(element * element, self.evaluate(self.next_element)) self.assertTrue(self.read_coordination_events[element].acquire(False)) with self.assertRaises(errors.OutOfRangeError): - sess.run(self.next_element) + self.evaluate(self.next_element) def testBlockLengthWithContentionSloppy(self): with self.cached_session() as sess: @@ -549,7 +549,7 @@ class ParallelInterleaveTest(test_base.DatasetTestBase): self.write_coordination_events[expected_element].set() if done_first_event: # First event starts the worker threads. self.read_coordination_events[expected_element].acquire() - actual_element = sess.run(self.next_element) + actual_element = self.evaluate(self.next_element) if not done_first_event: self.read_coordination_events[expected_element].acquire() done_first_event = True @@ -557,7 +557,7 @@ class ParallelInterleaveTest(test_base.DatasetTestBase): "At index %s: %s expected, got: %s" % (i, expected_element, actual_element)) with self.assertRaises(errors.OutOfRangeError): - sess.run(self.next_element) + self.evaluate(self.next_element) def _testEarlyExit(self, sloppy=False): # Exiting without consuming all input should not block @@ -575,7 +575,7 @@ class ParallelInterleaveTest(test_base.DatasetTestBase): }) for i in range(4, 7): self.write_coordination_events[i].set() - elem = sess.run(self.next_element) # Start all workers + elem = self.evaluate(self.next_element) # Start all workers # Allow the one successful worker to progress beyond the py_func again. elem = int(math.sqrt(elem)) self.write_coordination_events[elem].set() @@ -603,12 +603,12 @@ class ParallelInterleaveTest(test_base.DatasetTestBase): dataset = dataset.apply( interleave_ops.parallel_interleave( interleave_fn, cycle_length=16, block_length=2, sloppy=sloppy)) - iterator = dataset.make_one_shot_iterator() + iterator = dataset_ops.make_one_shot_iterator(dataset) with self.cached_session() as sess: output_values = [] for _ in range(30): - output_values.append(sess.run(iterator.get_next())) + output_values.append(self.evaluate(iterator.get_next())) expected_values = self._interleave( [[4] * 4, [5] * 5, [6] * 6] * self.repeat_count, 1, 2) @@ -630,20 +630,19 @@ class ParallelInterleaveTest(test_base.DatasetTestBase): sparse_ops.sparse_to_dense(x.indices, x.dense_shape, x.values)) dataset = dataset_ops.Dataset.range(10).map(_map_fn) - iterator = dataset.apply( - interleave_ops.parallel_interleave( - _interleave_fn, cycle_length=1)).make_initializable_iterator() + iterator = dataset_ops.make_initializable_iterator(dataset.apply( + interleave_ops.parallel_interleave(_interleave_fn, cycle_length=1))) init_op = iterator.initializer get_next = iterator.get_next() with self.cached_session() as sess: - sess.run(init_op) + self.evaluate(init_op) for i in range(10): for j in range(2): expected = [i, 0] if j % 2 == 0 else [0, -i] - self.assertAllEqual(expected, sess.run(get_next)) + self.assertAllEqual(expected, self.evaluate(get_next)) with self.assertRaises(errors.OutOfRangeError): - sess.run(get_next) + self.evaluate(get_next) def testErrorsInOutputFn(self): with self.cached_session() as sess: @@ -668,15 +667,15 @@ class ParallelInterleaveTest(test_base.DatasetTestBase): self.error = ValueError() self.write_coordination_events[expected_element].set() with self.assertRaises(errors.InvalidArgumentError): - sess.run(self.next_element) + self.evaluate(self.next_element) else: self.write_coordination_events[expected_element].set() - actual_element = sess.run(self.next_element) + actual_element = self.evaluate(self.next_element) self.assertEqual(expected_element * expected_element, actual_element, "At index %s: %s expected, got: %s" % (i, expected_element, actual_element)) with self.assertRaises(errors.OutOfRangeError): - sess.run(self.next_element) + self.evaluate(self.next_element) def testErrorsInInputFn(self): @@ -701,7 +700,7 @@ class ParallelInterleaveTest(test_base.DatasetTestBase): self.buffer_output_elements, self.prefetch_input_elements))) - self.iterator = self.dataset.make_initializable_iterator() + self.iterator = dataset_ops.make_initializable_iterator(self.dataset) self.init_op = self.iterator.initializer self.next_element = self.iterator.get_next() @@ -720,14 +719,14 @@ class ParallelInterleaveTest(test_base.DatasetTestBase): self._interleave([[4] * 4, [5], [6] * 6] * self.repeat_count, 2, 1)): if expected_element == 5: with self.assertRaises(errors.InvalidArgumentError): - sess.run(self.next_element) + self.evaluate(self.next_element) else: - actual_element = sess.run(self.next_element) + actual_element = self.evaluate(self.next_element) self.assertEqual(expected_element, actual_element, "At index %s: %s expected, got: %s" % (i, expected_element, actual_element)) with self.assertRaises(errors.OutOfRangeError): - sess.run(self.next_element) + self.evaluate(self.next_element) def testErrorsInInterleaveFn(self): @@ -750,7 +749,7 @@ class ParallelInterleaveTest(test_base.DatasetTestBase): self.buffer_output_elements, self.prefetch_input_elements))) - self.iterator = self.dataset.make_initializable_iterator() + self.iterator = dataset_ops.make_initializable_iterator(self.dataset) self.init_op = self.iterator.initializer self.next_element = self.iterator.get_next() @@ -769,14 +768,14 @@ class ParallelInterleaveTest(test_base.DatasetTestBase): self._interleave([[4] * 4, [5], [6] * 6] * self.repeat_count, 2, 1)): if expected_element == 5: with self.assertRaises(errors.InvalidArgumentError): - sess.run(self.next_element) + self.evaluate(self.next_element) else: - actual_element = sess.run(self.next_element) + actual_element = self.evaluate(self.next_element) self.assertEqual(expected_element, actual_element, "At index %s: %s expected, got: %s" % (i, expected_element, actual_element)) with self.assertRaises(errors.OutOfRangeError): - sess.run(self.next_element) + self.evaluate(self.next_element) def testShutdownRace(self): dataset = dataset_ops.Dataset.range(20) @@ -789,17 +788,17 @@ class ParallelInterleaveTest(test_base.DatasetTestBase): buffer_output_elements=1, prefetch_input_elements=0)) dataset = dataset.batch(32) - iterator = dataset.make_initializable_iterator() + iterator = dataset_ops.make_initializable_iterator(dataset) next_element = iterator.get_next() results = [] with self.cached_session() as sess: for _ in range(2): elements = [] - sess.run(iterator.initializer) + self.evaluate(iterator.initializer) try: while True: - elements.extend(sess.run(next_element)) + elements.extend(self.evaluate(next_element)) except errors.OutOfRangeError: pass results.append(elements) diff --git a/tensorflow/python/data/experimental/kernel_tests/parse_example_dataset_test.py b/tensorflow/python/data/experimental/kernel_tests/parse_example_dataset_test.py index c74f754fefb..76e0d4d72a6 100644 --- a/tensorflow/python/data/experimental/kernel_tests/parse_example_dataset_test.py +++ b/tensorflow/python/data/experimental/kernel_tests/parse_example_dataset_test.py @@ -144,6 +144,7 @@ class ParseExampleDatasetTest(test_base.DatasetTestBase): expected_values=expected_output, create_iterator_twice=True) + @test_util.run_deprecated_v1 def testEmptySerializedWithoutDefaultsShouldFail(self): input_features = { "st_a": @@ -177,6 +178,7 @@ class ParseExampleDatasetTest(test_base.DatasetTestBase): expected_err=(errors_impl.InvalidArgumentError, "Feature: c \\(data type: float\\) is required")) + @test_util.run_deprecated_v1 def testDenseNotMatchingShapeShouldFail(self): original = [ example(features=features({ @@ -669,6 +671,7 @@ class ParseExampleDatasetTest(test_base.DatasetTestBase): for batch_size in (1, 10, 20, 100, 256): self._testSerializedContainingVarLenDenseLargerBatch(batch_size) + @test_util.run_deprecated_v1 def testSkipEagerSerializedShapeMismatch(self): aname = "a" bname = "b" @@ -706,6 +709,7 @@ class ParseExampleDatasetTest(test_base.DatasetTestBase): expected_err=(ValueError, "Cannot reshape a tensor with 0 elements to shape")) + @test_util.run_deprecated_v1 def testSerializedContainingVarLenDense(self): aname = "a" bname = "b" diff --git a/tensorflow/python/data/experimental/kernel_tests/prefetch_to_device_test.py b/tensorflow/python/data/experimental/kernel_tests/prefetch_to_device_test.py index f73725366c4..80bd43e9ade 100644 --- a/tensorflow/python/data/experimental/kernel_tests/prefetch_to_device_test.py +++ b/tensorflow/python/data/experimental/kernel_tests/prefetch_to_device_test.py @@ -31,17 +31,15 @@ from tensorflow.python.platform import test class PrefetchToDeviceTest(test_base.DatasetTestBase): + @test_util.run_deprecated_v1 def testPrefetchToDevice(self): host_dataset = dataset_ops.Dataset.range(10) device_dataset = host_dataset.apply( prefetching_ops.prefetch_to_device("/cpu:1")) - # NOTE(mrry): This device block creates the "host" dataset and iterator on - # /cpu:0, and ensures that the prefetching is across devices. In typical use - # this would not be necessary, because the GPU device would not support any - # of the dataset-related ops. - with ops.device("/cpu:0"): - iterator = device_dataset.make_one_shot_iterator() + with ops.device("/cpu:1"): + iterator = dataset_ops.make_one_shot_iterator(device_dataset) + next_element = iterator.get_next() self.assertEqual(host_dataset.output_types, device_dataset.output_types) self.assertEqual(host_dataset.output_types, iterator.output_types) @@ -50,29 +48,26 @@ class PrefetchToDeviceTest(test_base.DatasetTestBase): self.assertEqual(host_dataset.output_classes, device_dataset.output_classes) self.assertEqual(host_dataset.output_classes, iterator.output_classes) - next_element = iterator.get_next() self.assertEqual(dtypes.int64, next_element.dtype) self.assertEqual([], next_element.shape) worker_config = config_pb2.ConfigProto(device_count={"CPU": 2}) - with self.test_session(config=worker_config) as sess: + with self.test_session(config=worker_config): for i in range(10): - self.assertEqual(i, sess.run(next_element)) + self.assertEqual(i, self.evaluate(next_element)) with self.assertRaises(errors.OutOfRangeError): - sess.run(next_element) + self.evaluate(next_element) + @test_util.run_deprecated_v1 def testPrefetchToSameDevice(self): host_dataset = dataset_ops.Dataset.range(10) device_dataset = host_dataset.apply( prefetching_ops.prefetch_to_device( "/job:localhost/replica:0/task:0/device:CPU:0")) - # NOTE(mrry): This device block creates the "host" dataset and iterator on - # /cpu:0, and ensures that the prefetching is across devices. In typical use - # this would not be necessary, because the GPU device would not support any - # of the dataset-related ops. - with ops.device("/cpu:0"): - iterator = device_dataset.make_one_shot_iterator() + with ops.device("/cpu:1"): + iterator = dataset_ops.make_one_shot_iterator(device_dataset) + next_element = iterator.get_next() self.assertEqual(host_dataset.output_types, device_dataset.output_types) self.assertEqual(host_dataset.output_types, iterator.output_types) @@ -81,27 +76,24 @@ class PrefetchToDeviceTest(test_base.DatasetTestBase): self.assertEqual(host_dataset.output_classes, device_dataset.output_classes) self.assertEqual(host_dataset.output_classes, iterator.output_classes) - next_element = iterator.get_next() self.assertEqual(dtypes.int64, next_element.dtype) self.assertEqual([], next_element.shape) - with self.cached_session() as sess: + with self.cached_session(): for i in range(10): - self.assertEqual(i, sess.run(next_element)) + self.assertEqual(i, self.evaluate(next_element)) with self.assertRaises(errors.OutOfRangeError): - sess.run(next_element) + self.evaluate(next_element) + @test_util.run_deprecated_v1 def testPrefetchDictToDevice(self): host_dataset = dataset_ops.Dataset.range(10).map(lambda x: {"a": x}) device_dataset = host_dataset.apply( prefetching_ops.prefetch_to_device("/cpu:1")) - # NOTE(mrry): This device block creates the "host" dataset and iterator on - # /cpu:0, and ensures that the prefetching is across devices. In typical use - # this would not be necessary, because the GPU device would not support any - # of the dataset-related ops. - with ops.device("/cpu:0"): - iterator = device_dataset.make_one_shot_iterator() + with ops.device("/cpu:1"): + iterator = dataset_ops.make_one_shot_iterator(device_dataset) + next_element = iterator.get_next() self.assertEqual(host_dataset.output_types, device_dataset.output_types) self.assertEqual(host_dataset.output_types, iterator.output_types) @@ -110,17 +102,17 @@ class PrefetchToDeviceTest(test_base.DatasetTestBase): self.assertEqual(host_dataset.output_classes, device_dataset.output_classes) self.assertEqual(host_dataset.output_classes, iterator.output_classes) - next_element = iterator.get_next() self.assertEqual(dtypes.int64, next_element["a"].dtype) self.assertEqual([], next_element["a"].shape) worker_config = config_pb2.ConfigProto(device_count={"CPU": 2}) - with self.test_session(config=worker_config) as sess: + with self.test_session(config=worker_config): for i in range(10): - self.assertEqual({"a": i}, sess.run(next_element)) + self.assertEqual({"a": i}, self.evaluate(next_element)) with self.assertRaises(errors.OutOfRangeError): - sess.run(next_element) + self.evaluate(next_element) + @test_util.run_deprecated_v1 def testPrefetchSparseTensorsToDevice(self): def make_tensor(i): return sparse_tensor.SparseTensorValue( @@ -130,12 +122,9 @@ class PrefetchToDeviceTest(test_base.DatasetTestBase): device_dataset = host_dataset.apply( prefetching_ops.prefetch_to_device("/cpu:1")) - # NOTE(mrry): This device block creates the "host" dataset and iterator on - # /cpu:0, and ensures that the prefetching is across devices. In typical use - # this would not be necessary, because the GPU device would not support any - # of the dataset-related ops. - with ops.device("/cpu:0"): - iterator = device_dataset.make_one_shot_iterator() + with ops.device("/cpu:1"): + iterator = dataset_ops.make_one_shot_iterator(device_dataset) + next_element = iterator.get_next() self.assertEqual(host_dataset.output_types, device_dataset.output_types) self.assertEqual(host_dataset.output_types, iterator.output_types) @@ -144,18 +133,17 @@ class PrefetchToDeviceTest(test_base.DatasetTestBase): self.assertEqual(host_dataset.output_classes, device_dataset.output_classes) self.assertEqual(host_dataset.output_classes, iterator.output_classes) - next_element = iterator.get_next() self.assertEqual(dtypes.int64, next_element.dtype) worker_config = config_pb2.ConfigProto(device_count={"CPU": 2}) - with self.test_session(config=worker_config) as sess: + with self.test_session(config=worker_config): for i in range(10): - actual = sess.run(next_element) + actual = self.evaluate(next_element) self.assertAllEqual([i], actual.values) self.assertAllEqual([[0, 0]], actual.indices) self.assertAllEqual([2, 2], actual.dense_shape) with self.assertRaises(errors.OutOfRangeError): - sess.run(next_element) + self.evaluate(next_element) def testPrefetchToDeviceGpu(self): if not test_util.is_gpu_available(): @@ -165,26 +153,26 @@ class PrefetchToDeviceTest(test_base.DatasetTestBase): device_dataset = host_dataset.apply( prefetching_ops.prefetch_to_device("/gpu:0")) - iterator = device_dataset.make_one_shot_iterator() + iterator = dataset_ops.make_initializable_iterator(device_dataset) next_element = iterator.get_next() - with self.cached_session() as sess: + with self.cached_session( + config=config_pb2.ConfigProto(allow_soft_placement=False)): + self.evaluate(iterator.initializer) for i in range(10): - self.assertEqual(i, sess.run(next_element)) + self.assertEqual(i, self.evaluate(next_element)) with self.assertRaises(errors.OutOfRangeError): - sess.run(next_element) + self.evaluate(next_element) + @test_util.run_deprecated_v1 def testPrefetchToDeviceWithReInit(self): host_dataset = dataset_ops.Dataset.range(10) device_dataset = host_dataset.apply( prefetching_ops.prefetch_to_device("/cpu:1")) - # NOTE(mrry): This device block creates the "host" dataset and iterator on - # /cpu:0, and ensures that the prefetching is across devices. In typical use - # this would not be necessary, because the GPU device would not support any - # of the dataset-related ops. - with ops.device("/cpu:0"): - iterator = device_dataset.make_initializable_iterator() + with ops.device("/cpu:1"): + iterator = dataset_ops.make_initializable_iterator(device_dataset) + next_element = iterator.get_next() self.assertEqual(host_dataset.output_types, device_dataset.output_types) self.assertEqual(host_dataset.output_types, iterator.output_types) @@ -193,20 +181,19 @@ class PrefetchToDeviceTest(test_base.DatasetTestBase): self.assertEqual(host_dataset.output_classes, device_dataset.output_classes) self.assertEqual(host_dataset.output_classes, iterator.output_classes) - next_element = iterator.get_next() self.assertEqual(dtypes.int64, next_element.dtype) self.assertEqual([], next_element.shape) worker_config = config_pb2.ConfigProto(device_count={"CPU": 2}) - with self.test_session(config=worker_config) as sess: - sess.run(iterator.initializer) + with self.test_session(config=worker_config): + self.evaluate(iterator.initializer) for i in range(5): - self.assertEqual(i, sess.run(next_element)) - sess.run(iterator.initializer) + self.assertEqual(i, self.evaluate(next_element)) + self.evaluate(iterator.initializer) for i in range(10): - self.assertEqual(i, sess.run(next_element)) + self.assertEqual(i, self.evaluate(next_element)) with self.assertRaises(errors.OutOfRangeError): - sess.run(next_element) + self.evaluate(next_element) def testPrefetchToDeviceGpuWithReInit(self): if not test_util.is_gpu_available(): @@ -216,18 +203,19 @@ class PrefetchToDeviceTest(test_base.DatasetTestBase): device_dataset = host_dataset.apply( prefetching_ops.prefetch_to_device("/gpu:0")) - iterator = device_dataset.make_initializable_iterator() + iterator = dataset_ops.make_initializable_iterator(device_dataset) next_element = iterator.get_next() - with self.cached_session() as sess: - sess.run(iterator.initializer) + with self.cached_session( + config=config_pb2.ConfigProto(allow_soft_placement=False)): + self.evaluate(iterator.initializer) for i in range(5): - self.assertEqual(i, sess.run(next_element)) - sess.run(iterator.initializer) + self.assertEqual(i, self.evaluate(next_element)) + self.evaluate(iterator.initializer) for i in range(10): - self.assertEqual(i, sess.run(next_element)) + self.assertEqual(i, self.evaluate(next_element)) with self.assertRaises(errors.OutOfRangeError): - sess.run(next_element) + self.evaluate(next_element) if __name__ == "__main__": diff --git a/tensorflow/python/data/experimental/kernel_tests/rejection_resample_test.py b/tensorflow/python/data/experimental/kernel_tests/rejection_resample_test.py index 4c879dbae68..76f68f50c81 100644 --- a/tensorflow/python/data/experimental/kernel_tests/rejection_resample_test.py +++ b/tensorflow/python/data/experimental/kernel_tests/rejection_resample_test.py @@ -28,6 +28,7 @@ from tensorflow.python.data.kernel_tests import test_base from tensorflow.python.data.ops import dataset_ops from tensorflow.python.framework import dtypes from tensorflow.python.framework import errors +from tensorflow.python.framework import test_util from tensorflow.python.ops import math_ops from tensorflow.python.ops import random_ops from tensorflow.python.ops import string_ops @@ -47,7 +48,7 @@ def _time_resampling( initial_dist=init_dist, seed=142)) - get_next = dataset.make_one_shot_iterator().get_next() + get_next = dataset_ops.make_one_shot_iterator(dataset).get_next() with test_obj.test_session() as sess: start_time = time.time() @@ -63,6 +64,7 @@ class RejectionResampleTest(test_base.DatasetTestBase, parameterized.TestCase): @parameterized.named_parameters( ("InitialDistributionKnown", True), ("InitialDistributionUnknown", False)) + @test_util.run_deprecated_v1 def testDistribution(self, initial_known): classes = np.random.randint(5, size=(20000,)) # Uniformly sampled target_dist = [0.9, 0.05, 0.05, 0.0, 0.0] @@ -71,12 +73,12 @@ class RejectionResampleTest(test_base.DatasetTestBase, parameterized.TestCase): dataset = dataset_ops.Dataset.from_tensor_slices(classes).shuffle( 200, seed=21).map(lambda c: (c, string_ops.as_string(c))).repeat() - get_next = dataset.apply( + get_next = dataset_ops.make_one_shot_iterator(dataset.apply( resampling.rejection_resample( target_dist=target_dist, initial_dist=initial_dist, class_func=lambda c, _: c, - seed=27)).make_one_shot_iterator().get_next() + seed=27))).get_next() with self.cached_session() as sess: returned = [] @@ -97,6 +99,7 @@ class RejectionResampleTest(test_base.DatasetTestBase, parameterized.TestCase): @parameterized.named_parameters( ("OnlyInitial", True), ("NotInitial", False)) + @test_util.run_deprecated_v1 def testEdgeCasesSampleFromInitialDataset(self, only_initial_dist): init_dist = [0.5, 0.5] target_dist = [0.5, 0.5] if only_initial_dist else [0.0, 1.0] @@ -114,7 +117,7 @@ class RejectionResampleTest(test_base.DatasetTestBase, parameterized.TestCase): target_dist=target_dist, initial_dist=init_dist)) - get_next = dataset.make_one_shot_iterator().get_next() + get_next = dataset_ops.make_one_shot_iterator(dataset).get_next() with self.cached_session() as sess: returned = [] @@ -122,6 +125,7 @@ class RejectionResampleTest(test_base.DatasetTestBase, parameterized.TestCase): while True: returned.append(sess.run(get_next)) + @test_util.run_deprecated_v1 def testRandomClasses(self): init_dist = [0.25, 0.25, 0.25, 0.25] target_dist = [0.0, 0.0, 0.0, 1.0] @@ -145,7 +149,7 @@ class RejectionResampleTest(test_base.DatasetTestBase, parameterized.TestCase): target_dist=target_dist, initial_dist=init_dist)) - get_next = dataset.make_one_shot_iterator().get_next() + get_next = dataset_ops.make_one_shot_iterator(dataset).get_next() with self.cached_session() as sess: returned = [] diff --git a/tensorflow/python/data/experimental/kernel_tests/restructured_dataset_test.py b/tensorflow/python/data/experimental/kernel_tests/restructured_dataset_test.py index 516e489d043..658e6120cf9 100644 --- a/tensorflow/python/data/experimental/kernel_tests/restructured_dataset_test.py +++ b/tensorflow/python/data/experimental/kernel_tests/restructured_dataset_test.py @@ -22,12 +22,14 @@ from tensorflow.python.data.kernel_tests import test_base from tensorflow.python.data.ops import dataset_ops from tensorflow.python.data.util import nest from tensorflow.python.framework import dtypes +from tensorflow.python.framework import test_util from tensorflow.python.ops import array_ops from tensorflow.python.platform import test class RestructuredDatasetTest(test_base.DatasetTestBase): + @test_util.run_deprecated_v1 def testRestructureDataset(self): components = (array_ops.placeholder(dtypes.int32), (array_ops.placeholder(dtypes.int32, shape=[None]), diff --git a/tensorflow/python/data/experimental/kernel_tests/scan_test.py b/tensorflow/python/data/experimental/kernel_tests/scan_test.py index 0730455431f..03af7ecd2fa 100644 --- a/tensorflow/python/data/experimental/kernel_tests/scan_test.py +++ b/tensorflow/python/data/experimental/kernel_tests/scan_test.py @@ -40,6 +40,7 @@ class ScanTest(test_base.DatasetTestBase): return dataset_ops.Dataset.from_tensors(0).repeat().apply( scan_ops.scan(start, scan_fn)) + @test_util.run_deprecated_v1 def testCount(self): def make_scan_fn(step): return lambda state, _: (state + step, state) @@ -47,8 +48,8 @@ class ScanTest(test_base.DatasetTestBase): start = array_ops.placeholder(dtypes.int32, shape=[]) step = array_ops.placeholder(dtypes.int32, shape=[]) take = array_ops.placeholder(dtypes.int64, shape=[]) - iterator = self._counting_dataset( - start, make_scan_fn(step)).take(take).make_initializable_iterator() + iterator = dataset_ops.make_initializable_iterator(self._counting_dataset( + start, make_scan_fn(step)).take(take)) next_element = iterator.get_next() with self.cached_session() as sess: @@ -60,15 +61,15 @@ class ScanTest(test_base.DatasetTestBase): feed_dict={start: start_val, step: step_val, take: take_val}) for expected, _ in zip( itertools.count(start_val, step_val), range(take_val)): - self.assertEqual(expected, sess.run(next_element)) + self.assertEqual(expected, self.evaluate(next_element)) with self.assertRaises(errors.OutOfRangeError): - sess.run(next_element) + self.evaluate(next_element) @test_util.run_in_graph_and_eager_modes def testFibonacci(self): - iterator = dataset_ops.Dataset.from_tensors(1).repeat(None).apply( - scan_ops.scan([0, 1], lambda a, _: ([a[1], a[0] + a[1]], a[1])) - ).make_one_shot_iterator() + iterator = dataset_ops.make_one_shot_iterator( + dataset_ops.Dataset.from_tensors(1).repeat(None).apply( + scan_ops.scan([0, 1], lambda a, _: ([a[1], a[0] + a[1]], a[1])))) if context.executing_eagerly(): next_element = iterator.get_next @@ -83,6 +84,7 @@ class ScanTest(test_base.DatasetTestBase): self.assertEqual(5, self.evaluate(next_element())) self.assertEqual(8, self.evaluate(next_element())) + @test_util.run_deprecated_v1 def testSparseCount(self): def _sparse(i): return sparse_tensor.SparseTensorValue( @@ -96,9 +98,8 @@ class ScanTest(test_base.DatasetTestBase): start = array_ops.placeholder(dtypes.int32, shape=[]) step = array_ops.placeholder(dtypes.int32, shape=[]) take = array_ops.placeholder(dtypes.int64, shape=[]) - iterator = self._counting_dataset( - _sparse(start), - make_scan_fn(step)).take(take).make_initializable_iterator() + iterator = dataset_ops.make_initializable_iterator(self._counting_dataset( + _sparse(start), make_scan_fn(step)).take(take)) next_element = iterator.get_next() with self.cached_session() as sess: @@ -110,10 +111,11 @@ class ScanTest(test_base.DatasetTestBase): feed_dict={start: start_val, step: step_val, take: take_val}) for expected, _ in zip( itertools.count(start_val, step_val), range(take_val)): - self.assertEqual(expected, sess.run(next_element).values[0]) + self.assertEqual(expected, self.evaluate(next_element).values[0]) with self.assertRaises(errors.OutOfRangeError): - sess.run(next_element) + self.evaluate(next_element) + @test_util.run_deprecated_v1 def testChangingStateShape(self): # Test the fixed-point shape invariant calculations: start with # initial values with known shapes, and use a scan function that @@ -131,16 +133,16 @@ class ScanTest(test_base.DatasetTestBase): self.assertIs(None, dataset.output_shapes[0][1].ndims) self.assertEqual([], dataset.output_shapes[1].as_list()) - iterator = dataset.make_one_shot_iterator() + iterator = dataset_ops.make_one_shot_iterator(dataset) next_element = iterator.get_next() with self.cached_session() as sess: for i in range(5): - (longer_vector_val, larger_rank_val), _ = sess.run(next_element) + (longer_vector_val, larger_rank_val), _ = self.evaluate(next_element) self.assertAllEqual([0] * (2**i), longer_vector_val) self.assertAllEqual(np.array(1, ndmin=i), larger_rank_val) with self.assertRaises(errors.OutOfRangeError): - sess.run(next_element) + self.evaluate(next_element) def testIncorrectStateType(self): diff --git a/tensorflow/python/data/experimental/kernel_tests/serialization/BUILD b/tensorflow/python/data/experimental/kernel_tests/serialization/BUILD index 2cfb5759036..4a2e28f4964 100644 --- a/tensorflow/python/data/experimental/kernel_tests/serialization/BUILD +++ b/tensorflow/python/data/experimental/kernel_tests/serialization/BUILD @@ -74,7 +74,11 @@ py_test( size = "small", srcs = ["checkpoint_input_pipeline_hook_test.py"], srcs_version = "PY2AND3", - tags = ["no_pip"], + tags = [ + "no_pip", + "no_windows", + "notsan", + ], deps = [ "//tensorflow/python:client_testlib", "//tensorflow/python:constant_op", @@ -313,6 +317,7 @@ py_test( srcs_version = "PY2AND3", tags = [ "no_pip", + "no_windows", "notap", ], deps = [ @@ -355,9 +360,13 @@ py_test( size = "small", srcs = ["matching_files_dataset_serialization_test.py"], srcs_version = "PY2AND3", + tags = [ + "no_windows", + ], deps = [ ":dataset_serialization_test_base", "//tensorflow/python:client_testlib", + "//tensorflow/python/data/experimental/ops:matching_files", "//tensorflow/python/data/ops:dataset_ops", "//third_party/py/numpy", ], diff --git a/tensorflow/python/data/experimental/kernel_tests/serialization/checkpoint_input_pipeline_hook_test.py b/tensorflow/python/data/experimental/kernel_tests/serialization/checkpoint_input_pipeline_hook_test.py index 94393d6d4ba..8cc66d0c293 100644 --- a/tensorflow/python/data/experimental/kernel_tests/serialization/checkpoint_input_pipeline_hook_test.py +++ b/tensorflow/python/data/experimental/kernel_tests/serialization/checkpoint_input_pipeline_hook_test.py @@ -21,17 +21,18 @@ from __future__ import print_function from tensorflow.python.data.experimental.ops import iterator_ops from tensorflow.python.data.kernel_tests import test_base from tensorflow.python.data.ops import dataset_ops -from tensorflow.python.estimator import estimator -from tensorflow.python.estimator import model_fn from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops +from tensorflow.python.framework import test_util from tensorflow.python.ops import control_flow_ops from tensorflow.python.ops import variables from tensorflow.python.platform import test from tensorflow.python.training import checkpoint_management from tensorflow.python.training import saver as saver_lib from tensorflow.python.training import training_util +from tensorflow_estimator.python.estimator import estimator +from tensorflow_estimator.python.estimator import model_fn class CheckpointInputPipelineHookTest(test_base.DatasetTestBase): @@ -68,6 +69,7 @@ class CheckpointInputPipelineHookTest(test_base.DatasetTestBase): def _build_iterator_saver_hook(self, est): return iterator_ops.CheckpointInputPipelineHook(est) + @test_util.run_deprecated_v1 def testReturnDatasetFromInputFn(self): def _input_fn(): @@ -80,6 +82,7 @@ class CheckpointInputPipelineHookTest(test_base.DatasetTestBase): est.train(_input_fn, steps=2, hooks=[self._build_iterator_saver_hook(est)]) self.assertSequenceEqual(self._read_vars(est.model_dir), (4, 3)) + @test_util.run_deprecated_v1 def testBuildIteratorInInputFn(self): def _input_fn(): @@ -94,6 +97,7 @@ class CheckpointInputPipelineHookTest(test_base.DatasetTestBase): est.train(_input_fn, steps=2, hooks=[self._build_iterator_saver_hook(est)]) self.assertSequenceEqual(self._read_vars(est.model_dir), (4, 3)) + @test_util.run_deprecated_v1 def testDoNotRestore(self): def _input_fn(): diff --git a/tensorflow/python/data/experimental/kernel_tests/serialization/dataset_serialization_test_base.py b/tensorflow/python/data/experimental/kernel_tests/serialization/dataset_serialization_test_base.py index 7f435b82397..e65aa44d060 100644 --- a/tensorflow/python/data/experimental/kernel_tests/serialization/dataset_serialization_test_base.py +++ b/tensorflow/python/data/experimental/kernel_tests/serialization/dataset_serialization_test_base.py @@ -23,6 +23,7 @@ import os import numpy as np from tensorflow.python.data.experimental.ops import iterator_ops as contrib_iterator_ops +from tensorflow.python.data.ops import dataset_ops from tensorflow.python.data.ops import iterator_ops from tensorflow.python.framework import dtypes from tensorflow.python.framework import errors @@ -578,7 +579,7 @@ class DatasetSerializationTestBase(test.TestCase): return np.linspace(0, num_outputs, num_samples, dtype=int) def _build_graph(self, ds_fn, sparse_tensors=False): - iterator = ds_fn().make_initializable_iterator() + iterator = dataset_ops.make_initializable_iterator(ds_fn()) saveable = contrib_iterator_ops.make_saveable_from_iterator(iterator) ops.add_to_collection(ops.GraphKeys.SAVEABLE_OBJECTS, saveable) diff --git a/tensorflow/python/data/experimental/kernel_tests/serialization/filter_dataset_serialization_test.py b/tensorflow/python/data/experimental/kernel_tests/serialization/filter_dataset_serialization_test.py index 225f6cbac01..e3ba8ad231b 100644 --- a/tensorflow/python/data/experimental/kernel_tests/serialization/filter_dataset_serialization_test.py +++ b/tensorflow/python/data/experimental/kernel_tests/serialization/filter_dataset_serialization_test.py @@ -17,8 +17,6 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function -import numpy as np - from tensorflow.python.data.experimental.kernel_tests.serialization import dataset_serialization_test_base from tensorflow.python.data.ops import dataset_ops from tensorflow.python.framework import sparse_tensor @@ -35,7 +33,7 @@ class FilterDatasetSerializationTest( def testFilterCore(self): div = 3 - num_outputs = np.sum([x % 3 != 2 for x in range(100)]) + num_outputs = sum(x % 3 != 2 for x in range(100)) self.run_core_tests(lambda: self._build_filter_range_graph(div), lambda: self._build_filter_range_graph(div * 2), num_outputs) @@ -47,7 +45,7 @@ class FilterDatasetSerializationTest( lambda d: d["foo"] + d["bar"]) def testFilterDictCore(self): - num_outputs = np.sum([(x**2) % 2 == 0 for x in range(10)]) + num_outputs = sum((x**2) % 2 == 0 for x in range(10)) self.run_core_tests(self._build_filter_dict_graph, None, num_outputs) def _build_sparse_filter(self): diff --git a/tensorflow/python/data/experimental/kernel_tests/serialization/matching_files_dataset_serialization_test.py b/tensorflow/python/data/experimental/kernel_tests/serialization/matching_files_dataset_serialization_test.py index 7edb200d2ec..c026e97835c 100644 --- a/tensorflow/python/data/experimental/kernel_tests/serialization/matching_files_dataset_serialization_test.py +++ b/tensorflow/python/data/experimental/kernel_tests/serialization/matching_files_dataset_serialization_test.py @@ -22,7 +22,7 @@ import shutil import tempfile from tensorflow.python.data.experimental.kernel_tests.serialization import dataset_serialization_test_base -from tensorflow.python.data.ops.dataset_ops import MatchingFilesDataset +from tensorflow.python.data.experimental.ops import matching_files from tensorflow.python.platform import test @@ -30,7 +30,7 @@ class MatchingFilesDatasetSerializationTest( dataset_serialization_test_base.DatasetSerializationTestBase): def _build_iterator_graph(self, test_patterns): - return MatchingFilesDataset(test_patterns) + return matching_files.MatchingFilesDataset(test_patterns) def testMatchingFilesCore(self): tmp_dir = tempfile.mkdtemp() diff --git a/tensorflow/python/data/experimental/kernel_tests/serialization/range_dataset_serialization_test.py b/tensorflow/python/data/experimental/kernel_tests/serialization/range_dataset_serialization_test.py index ef99d01c73c..34419a31493 100644 --- a/tensorflow/python/data/experimental/kernel_tests/serialization/range_dataset_serialization_test.py +++ b/tensorflow/python/data/experimental/kernel_tests/serialization/range_dataset_serialization_test.py @@ -56,8 +56,8 @@ class RangeDatasetSerializationTest( def testSaveRestore(self): def _build_graph(start, stop): - iterator = dataset_ops.Dataset.range(start, - stop).make_initializable_iterator() + iterator = dataset_ops.make_initializable_iterator( + dataset_ops.Dataset.range(start, stop)) init_op = iterator.initializer get_next = iterator.get_next() save_op = self._save_op(iterator._iterator_resource) @@ -71,36 +71,36 @@ class RangeDatasetSerializationTest( with ops.Graph().as_default() as g: init_op, get_next, save_op, _ = _build_graph(start, stop) with self.session(graph=g) as sess: - sess.run(variables.global_variables_initializer()) - sess.run(init_op) + self.evaluate(variables.global_variables_initializer()) + self.evaluate(init_op) for i in range(start, break_point): - self.assertEqual(i, sess.run(get_next)) - sess.run(save_op) + self.assertEqual(i, self.evaluate(get_next)) + self.evaluate(save_op) with ops.Graph().as_default() as g: init_op, get_next, _, restore_op = _build_graph(start, stop) with self.session(graph=g) as sess: - sess.run(init_op) - sess.run(restore_op) + self.evaluate(init_op) + self.evaluate(restore_op) for i in range(break_point, stop): - self.assertEqual(i, sess.run(get_next)) + self.assertEqual(i, self.evaluate(get_next)) with self.assertRaises(errors.OutOfRangeError): - sess.run(get_next) + self.evaluate(get_next) # Saving and restoring in same session. with ops.Graph().as_default() as g: init_op, get_next, save_op, restore_op = _build_graph(start, stop) with self.session(graph=g) as sess: - sess.run(variables.global_variables_initializer()) - sess.run(init_op) + self.evaluate(variables.global_variables_initializer()) + self.evaluate(init_op) for i in range(start, break_point): - self.assertEqual(i, sess.run(get_next)) - sess.run(save_op) - sess.run(restore_op) + self.assertEqual(i, self.evaluate(get_next)) + self.evaluate(save_op) + self.evaluate(restore_op) for i in range(break_point, stop): - self.assertEqual(i, sess.run(get_next)) + self.assertEqual(i, self.evaluate(get_next)) with self.assertRaises(errors.OutOfRangeError): - sess.run(get_next) + self.evaluate(get_next) def _build_range_dataset(self, start, stop): return dataset_ops.Dataset.range(start, stop) diff --git a/tensorflow/python/data/experimental/kernel_tests/serialization/serialization_integration_test.py b/tensorflow/python/data/experimental/kernel_tests/serialization/serialization_integration_test.py index 88d5c896c9f..12fa0989d07 100644 --- a/tensorflow/python/data/experimental/kernel_tests/serialization/serialization_integration_test.py +++ b/tensorflow/python/data/experimental/kernel_tests/serialization/serialization_integration_test.py @@ -60,9 +60,9 @@ class SerializationIntegrationTest(test.TestCase): init_ops, get_next_ops, saver = self._build_graph(num_pipelines, num_outputs) with self.session(graph=g) as sess: - sess.run(init_ops) + self.evaluate(init_ops) for _ in range(break_point): - output = sess.run(get_next_ops) + output = self.evaluate(get_next_ops) for i in range(num_pipelines): all_outputs[i].append(output[i]) saver.save(sess, self._ckpt_path()) @@ -73,7 +73,7 @@ class SerializationIntegrationTest(test.TestCase): with self.session(graph=g) as sess: saver.restore(sess, self._ckpt_path()) for _ in range(num_outputs - break_point): - output = sess.run(get_next_ops) + output = self.evaluate(get_next_ops) for i in range(num_pipelines): all_outputs[i].append(output[i]) diff --git a/tensorflow/python/data/experimental/kernel_tests/serialization/shuffle_dataset_serialization_test.py b/tensorflow/python/data/experimental/kernel_tests/serialization/shuffle_dataset_serialization_test.py index a04f1ddafce..e753a7a15be 100644 --- a/tensorflow/python/data/experimental/kernel_tests/serialization/shuffle_dataset_serialization_test.py +++ b/tensorflow/python/data/experimental/kernel_tests/serialization/shuffle_dataset_serialization_test.py @@ -138,9 +138,9 @@ class ShuffleDatasetSerializationTest( saver = saver_lib.Saver(allow_empty=True) with self.session(graph=g) as sess: self._save(sess, saver) - expected = [sess.run(get_next_ops) for _ in range(num_outputs)] + expected = [self.evaluate(get_next_ops) for _ in range(num_outputs)] self._restore(saver, sess) - actual = [sess.run(get_next_ops) for _ in range(num_outputs)] + actual = [self.evaluate(get_next_ops) for _ in range(num_outputs)] self.match(expected, actual) diff --git a/tensorflow/python/data/experimental/kernel_tests/shuffle_and_repeat_test.py b/tensorflow/python/data/experimental/kernel_tests/shuffle_and_repeat_test.py index c208963a861..9528f83291f 100644 --- a/tensorflow/python/data/experimental/kernel_tests/shuffle_and_repeat_test.py +++ b/tensorflow/python/data/experimental/kernel_tests/shuffle_and_repeat_test.py @@ -24,6 +24,7 @@ from tensorflow.python.data.kernel_tests import test_base from tensorflow.python.data.ops import dataset_ops from tensorflow.python.framework import errors from tensorflow.python.framework import ops +from tensorflow.python.framework import test_util from tensorflow.python.platform import test @@ -34,16 +35,17 @@ class ShuffleAndRepeatTest(test_base.DatasetTestBase): shuffle_ops.shuffle_and_repeat(buffer_size=5, count=count, seed=seed)) def _gen_outputs(self, ds_fn, num_outputs, verify_exhausted=True): - get_next = ds_fn().make_one_shot_iterator().get_next() + get_next = dataset_ops.make_one_shot_iterator(ds_fn()).get_next() outputs = [] with self.cached_session() as sess: for _ in range(num_outputs): - outputs.append(sess.run(get_next)) + outputs.append(self.evaluate(get_next)) if verify_exhausted: with self.assertRaises(errors.OutOfRangeError): - sess.run(get_next) + self.evaluate(get_next) return outputs + @test_util.run_deprecated_v1 def testCorrectOutput(self): output = self._gen_outputs(lambda: self._build_ds(10), 100) self.assertSequenceEqual( @@ -52,6 +54,7 @@ class ShuffleAndRepeatTest(test_base.DatasetTestBase): for i in range(5): self.assertSequenceEqual(sorted(output[i * 20:(i + 1) * 20]), range(20)) + @test_util.run_deprecated_v1 def testReshuffling(self): # Check that the output orders of different epochs are indeed different. output = self._gen_outputs(lambda: self._build_ds(10), 100) @@ -60,17 +63,20 @@ class ShuffleAndRepeatTest(test_base.DatasetTestBase): epoch2 = output[(i + 1) * 20:(i + 2) * 20] self.assertNotEqual(epoch1, epoch2) + @test_util.run_deprecated_v1 def testSameOrderForSameSeeds(self): output1 = self._gen_outputs(lambda: self._build_ds(10), 100) output2 = self._gen_outputs(lambda: self._build_ds(10), 100) self.assertEqual(output1, output2) + @test_util.run_deprecated_v1 def testDifferentOrderForDifferentSeeds(self): output1 = self._gen_outputs(lambda: self._build_ds(10), 100) output2 = self._gen_outputs(lambda: self._build_ds(20), 100) self.assertNotEqual(output1, output2) self.assertEqual(sorted(output1), sorted(output2)) + @test_util.run_deprecated_v1 def testCountNone(self): output1 = self._gen_outputs( lambda: self._build_ds(10, count=None), 100, verify_exhausted=False) @@ -79,6 +85,7 @@ class ShuffleAndRepeatTest(test_base.DatasetTestBase): self.assertNotEqual(output1, output2) self.assertEqual(sorted(output1), sorted(output2)) + @test_util.run_deprecated_v1 def testCountMinusOne(self): output1 = self._gen_outputs( lambda: self._build_ds(10, count=-1), 100, verify_exhausted=False) @@ -108,7 +115,7 @@ class ShuffleAndRepeatTest(test_base.DatasetTestBase): shuffle_ops.shuffle_and_repeat(buffer_size=21)) get_next_op = ds.make_one_shot_iterator().get_next() with self.session(graph=g) as sess: - sess.run(get_next_op) + self.evaluate(get_next_op) if __name__ == "__main__": diff --git a/tensorflow/python/data/experimental/kernel_tests/sleep_test.py b/tensorflow/python/data/experimental/kernel_tests/sleep_test.py index bf53acc82a8..46b22f80b6d 100644 --- a/tensorflow/python/data/experimental/kernel_tests/sleep_test.py +++ b/tensorflow/python/data/experimental/kernel_tests/sleep_test.py @@ -23,6 +23,7 @@ from tensorflow.python.data.experimental.ops import sleep from tensorflow.python.data.kernel_tests import test_base from tensorflow.python.data.ops import dataset_ops from tensorflow.python.framework import errors +from tensorflow.python.framework import test_util from tensorflow.python.platform import test _NUMPY_RANDOM_SEED = 42 @@ -30,22 +31,23 @@ _NUMPY_RANDOM_SEED = 42 class SleepTest(test_base.DatasetTestBase): + @test_util.run_deprecated_v1 def testSleep(self): sleep_microseconds = 100 dataset = dataset_ops.Dataset.range(10).apply( sleep.sleep(sleep_microseconds)) - iterator = dataset.make_initializable_iterator() + iterator = dataset_ops.make_initializable_iterator(dataset) next_element = iterator.get_next() with self.cached_session() as sess: - sess.run(iterator.initializer) + self.evaluate(iterator.initializer) start_time = time.time() for i in range(10): - self.assertEqual(i, sess.run(next_element)) + self.assertEqual(i, self.evaluate(next_element)) end_time = time.time() self.assertGreater(end_time - start_time, (10 * sleep_microseconds) / 1e6) with self.assertRaises(errors.OutOfRangeError): - sess.run(next_element) + self.evaluate(next_element) if __name__ == "__main__": diff --git a/tensorflow/python/data/experimental/kernel_tests/sql_dataset_test.py b/tensorflow/python/data/experimental/kernel_tests/sql_dataset_test.py index a2c11696387..eb66927ee5c 100644 --- a/tensorflow/python/data/experimental/kernel_tests/sql_dataset_test.py +++ b/tensorflow/python/data/experimental/kernel_tests/sql_dataset_test.py @@ -39,10 +39,11 @@ class SqlDatasetTest(sql_dataset_test_base.SqlDatasetTestBase): "ORDER BY first_name DESC" }) for _ in range(2): # Dataset is repeated. See setUp. - self.assertEqual((b"John", b"Doe", b"Hi!"), sess.run(get_next)) - self.assertEqual((b"Jane", b"Moe", b"Hi again!"), sess.run(get_next)) + self.assertEqual((b"John", b"Doe", b"Hi!"), self.evaluate(get_next)) + self.assertEqual((b"Jane", b"Moe", b"Hi again!"), + self.evaluate(get_next)) with self.assertRaises(errors.OutOfRangeError): - sess.run(get_next) + self.evaluate(get_next) # Test that SqlDataset works on a join query. def testReadResultSetJoinQuery(self): @@ -58,9 +59,10 @@ class SqlDatasetTest(sql_dataset_test_base.SqlDatasetTestBase): "ON students.first_name = people.first_name " "AND students.last_name = people.last_name" }) - self.assertEqual((b"John", b"California", b"Hi!"), sess.run(get_next)) + self.assertEqual((b"John", b"California", b"Hi!"), + self.evaluate(get_next)) with self.assertRaises(errors.OutOfRangeError): - sess.run(get_next) + self.evaluate(get_next) # Test that SqlDataset can read a database entry with a null-terminator # in the middle of the text and place the entry in a `string` tensor. @@ -75,10 +77,11 @@ class SqlDatasetTest(sql_dataset_test_base.SqlDatasetTestBase): "SELECT first_name, last_name, favorite_nonsense_word " "FROM students ORDER BY first_name DESC" }) - self.assertEqual((b"John", b"Doe", b"n\0nsense"), sess.run(get_next)) - self.assertEqual((b"Jane", b"Moe", b"nonsense\0"), sess.run(get_next)) + self.assertEqual((b"John", b"Doe", b"n\0nsense"), self.evaluate(get_next)) + self.assertEqual((b"Jane", b"Moe", b"nonsense\0"), + self.evaluate(get_next)) with self.assertRaises(errors.OutOfRangeError): - sess.run(get_next) + self.evaluate(get_next) # Test that SqlDataset works when used on two different queries. # Because the output types of the dataset must be determined at graph-creation @@ -93,21 +96,22 @@ class SqlDatasetTest(sql_dataset_test_base.SqlDatasetTestBase): self.query: "SELECT first_name, last_name, motto FROM students " "ORDER BY first_name DESC" }) - self.assertEqual((b"John", b"Doe", b"Hi!"), sess.run(get_next)) - self.assertEqual((b"Jane", b"Moe", b"Hi again!"), sess.run(get_next)) + self.assertEqual((b"John", b"Doe", b"Hi!"), self.evaluate(get_next)) + self.assertEqual((b"Jane", b"Moe", b"Hi again!"), self.evaluate(get_next)) with self.assertRaises(errors.OutOfRangeError): - sess.run(get_next) + self.evaluate(get_next) sess.run( init_op, feed_dict={ self.query: "SELECT first_name, last_name, state FROM people " "ORDER BY first_name DESC" }) - self.assertEqual((b"John", b"Doe", b"California"), sess.run(get_next)) + self.assertEqual((b"John", b"Doe", b"California"), + self.evaluate(get_next)) self.assertEqual((b"Benjamin", b"Franklin", b"Pennsylvania"), - sess.run(get_next)) + self.evaluate(get_next)) with self.assertRaises(errors.OutOfRangeError): - sess.run(get_next) + self.evaluate(get_next) # Test that an `OutOfRangeError` is raised on the first call to # `get_next_str_only` if result set is empty. @@ -122,7 +126,7 @@ class SqlDatasetTest(sql_dataset_test_base.SqlDatasetTestBase): "WHERE first_name = 'Nonexistent'" }) with self.assertRaises(errors.OutOfRangeError): - sess.run(get_next) + self.evaluate(get_next) # Test that an error is raised when `driver_name` is invalid. def testReadResultSetWithInvalidDriverName(self): @@ -151,7 +155,7 @@ class SqlDatasetTest(sql_dataset_test_base.SqlDatasetTestBase): "ORDER BY first_name DESC" }) with self.assertRaises(errors.UnknownError): - sess.run(get_next) + self.evaluate(get_next) # Test that an error is raised when there is a syntax error in `query`. def testReadResultSetOfQueryWithSyntaxError(self): @@ -166,7 +170,7 @@ class SqlDatasetTest(sql_dataset_test_base.SqlDatasetTestBase): "ORDER BY first_name DESC" }) with self.assertRaises(errors.UnknownError): - sess.run(get_next) + self.evaluate(get_next) # Test that an error is raised when the number of columns in `query` # does not match the length of `output_types`. @@ -181,7 +185,7 @@ class SqlDatasetTest(sql_dataset_test_base.SqlDatasetTestBase): "ORDER BY first_name DESC" }) with self.assertRaises(errors.InvalidArgumentError): - sess.run(get_next) + self.evaluate(get_next) # Test that no results are returned when `query` is an insert query rather # than a select query. In particular, the error refers to the number of @@ -199,7 +203,7 @@ class SqlDatasetTest(sql_dataset_test_base.SqlDatasetTestBase): "VALUES ('Foo', 'Bar', 'Baz'), ('Fizz', 'Buzz', 'Fizzbuzz')" }) with self.assertRaises(errors.InvalidArgumentError): - sess.run(get_next) + self.evaluate(get_next) # Test that `SqlDataset` can read an integer from a SQLite database table and # place it in an `int8` tensor. @@ -212,10 +216,10 @@ class SqlDatasetTest(sql_dataset_test_base.SqlDatasetTestBase): self.query: "SELECT first_name, desk_number FROM students " "ORDER BY first_name DESC" }) - self.assertEqual((b"John", 9), sess.run(get_next)) - self.assertEqual((b"Jane", 127), sess.run(get_next)) + self.assertEqual((b"John", 9), self.evaluate(get_next)) + self.assertEqual((b"Jane", 127), self.evaluate(get_next)) with self.assertRaises(errors.OutOfRangeError): - sess.run(get_next) + self.evaluate(get_next) # Test that `SqlDataset` can read a negative or 0-valued integer from a # SQLite database table and place it in an `int8` tensor. @@ -230,9 +234,9 @@ class SqlDatasetTest(sql_dataset_test_base.SqlDatasetTestBase): "FROM students " "WHERE first_name = 'John' ORDER BY first_name DESC" }) - self.assertEqual((b"John", 0, -2), sess.run(get_next)) + self.assertEqual((b"John", 0, -2), self.evaluate(get_next)) with self.assertRaises(errors.OutOfRangeError): - sess.run(get_next) + self.evaluate(get_next) # Test that `SqlDataset` can read a large (positive or negative) integer from # a SQLite database table and place it in an `int8` tensor. @@ -246,11 +250,11 @@ class SqlDatasetTest(sql_dataset_test_base.SqlDatasetTestBase): "SELECT desk_number, favorite_negative_number FROM students " "ORDER BY first_name DESC" }) - self.assertEqual((9, -2), sess.run(get_next)) + self.assertEqual((9, -2), self.evaluate(get_next)) # Max and min values of int8 - self.assertEqual((127, -128), sess.run(get_next)) + self.assertEqual((127, -128), self.evaluate(get_next)) with self.assertRaises(errors.OutOfRangeError): - sess.run(get_next) + self.evaluate(get_next) # Test that `SqlDataset` can read an integer from a SQLite database table and # place it in an `int16` tensor. @@ -263,10 +267,10 @@ class SqlDatasetTest(sql_dataset_test_base.SqlDatasetTestBase): self.query: "SELECT first_name, desk_number FROM students " "ORDER BY first_name DESC" }) - self.assertEqual((b"John", 9), sess.run(get_next)) - self.assertEqual((b"Jane", 127), sess.run(get_next)) + self.assertEqual((b"John", 9), self.evaluate(get_next)) + self.assertEqual((b"Jane", 127), self.evaluate(get_next)) with self.assertRaises(errors.OutOfRangeError): - sess.run(get_next) + self.evaluate(get_next) # Test that `SqlDataset` can read a negative or 0-valued integer from a # SQLite database table and place it in an `int16` tensor. @@ -281,9 +285,9 @@ class SqlDatasetTest(sql_dataset_test_base.SqlDatasetTestBase): "FROM students " "WHERE first_name = 'John' ORDER BY first_name DESC" }) - self.assertEqual((b"John", 0, -2), sess.run(get_next)) + self.assertEqual((b"John", 0, -2), self.evaluate(get_next)) with self.assertRaises(errors.OutOfRangeError): - sess.run(get_next) + self.evaluate(get_next) # Test that `SqlDataset` can read a large (positive or negative) integer from # a SQLite database table and place it in an `int16` tensor. @@ -297,11 +301,11 @@ class SqlDatasetTest(sql_dataset_test_base.SqlDatasetTestBase): "FROM students ORDER BY first_name DESC" }) # Max value of int16 - self.assertEqual((b"John", 32767), sess.run(get_next)) + self.assertEqual((b"John", 32767), self.evaluate(get_next)) # Min value of int16 - self.assertEqual((b"Jane", -32768), sess.run(get_next)) + self.assertEqual((b"Jane", -32768), self.evaluate(get_next)) with self.assertRaises(errors.OutOfRangeError): - sess.run(get_next) + self.evaluate(get_next) # Test that `SqlDataset` can read an integer from a SQLite database table and # place it in an `int32` tensor. @@ -314,8 +318,8 @@ class SqlDatasetTest(sql_dataset_test_base.SqlDatasetTestBase): self.query: "SELECT first_name, desk_number FROM students " "ORDER BY first_name DESC" }) - self.assertEqual((b"John", 9), sess.run(get_next)) - self.assertEqual((b"Jane", 127), sess.run(get_next)) + self.assertEqual((b"John", 9), self.evaluate(get_next)) + self.assertEqual((b"Jane", 127), self.evaluate(get_next)) # Test that `SqlDataset` can read a negative or 0-valued integer from a # SQLite database table and place it in an `int32` tensor. @@ -328,10 +332,10 @@ class SqlDatasetTest(sql_dataset_test_base.SqlDatasetTestBase): self.query: "SELECT first_name, income FROM students " "ORDER BY first_name DESC" }) - self.assertEqual((b"John", 0), sess.run(get_next)) - self.assertEqual((b"Jane", -20000), sess.run(get_next)) + self.assertEqual((b"John", 0), self.evaluate(get_next)) + self.assertEqual((b"Jane", -20000), self.evaluate(get_next)) with self.assertRaises(errors.OutOfRangeError): - sess.run(get_next) + self.evaluate(get_next) # Test that `SqlDataset` can read a large (positive or negative) integer from # a SQLite database table and place it in an `int32` tensor. @@ -345,11 +349,11 @@ class SqlDatasetTest(sql_dataset_test_base.SqlDatasetTestBase): "ORDER BY first_name DESC" }) # Max value of int32 - self.assertEqual((b"John", 2147483647), sess.run(get_next)) + self.assertEqual((b"John", 2147483647), self.evaluate(get_next)) # Min value of int32 - self.assertEqual((b"Jane", -2147483648), sess.run(get_next)) + self.assertEqual((b"Jane", -2147483648), self.evaluate(get_next)) with self.assertRaises(errors.OutOfRangeError): - sess.run(get_next) + self.evaluate(get_next) # Test that `SqlDataset` can read a numeric `varchar` from a SQLite database # table and place it in an `int32` tensor. @@ -362,10 +366,10 @@ class SqlDatasetTest(sql_dataset_test_base.SqlDatasetTestBase): self.query: "SELECT first_name, school_id FROM students " "ORDER BY first_name DESC" }) - self.assertEqual((b"John", 123), sess.run(get_next)) - self.assertEqual((b"Jane", 1000), sess.run(get_next)) + self.assertEqual((b"John", 123), self.evaluate(get_next)) + self.assertEqual((b"Jane", 1000), self.evaluate(get_next)) with self.assertRaises(errors.OutOfRangeError): - sess.run(get_next) + self.evaluate(get_next) # Test that `SqlDataset` can read an integer from a SQLite database table # and place it in an `int64` tensor. @@ -378,10 +382,10 @@ class SqlDatasetTest(sql_dataset_test_base.SqlDatasetTestBase): self.query: "SELECT first_name, desk_number FROM students " "ORDER BY first_name DESC" }) - self.assertEqual((b"John", 9), sess.run(get_next)) - self.assertEqual((b"Jane", 127), sess.run(get_next)) + self.assertEqual((b"John", 9), self.evaluate(get_next)) + self.assertEqual((b"Jane", 127), self.evaluate(get_next)) with self.assertRaises(errors.OutOfRangeError): - sess.run(get_next) + self.evaluate(get_next) # Test that `SqlDataset` can read a negative or 0-valued integer from a # SQLite database table and place it in an `int64` tensor. @@ -394,10 +398,10 @@ class SqlDatasetTest(sql_dataset_test_base.SqlDatasetTestBase): self.query: "SELECT first_name, income FROM students " "ORDER BY first_name DESC" }) - self.assertEqual((b"John", 0), sess.run(get_next)) - self.assertEqual((b"Jane", -20000), sess.run(get_next)) + self.assertEqual((b"John", 0), self.evaluate(get_next)) + self.assertEqual((b"Jane", -20000), self.evaluate(get_next)) with self.assertRaises(errors.OutOfRangeError): - sess.run(get_next) + self.evaluate(get_next) # Test that `SqlDataset` can read a large (positive or negative) integer from # a SQLite database table and place it in an `int64` tensor. @@ -412,11 +416,11 @@ class SqlDatasetTest(sql_dataset_test_base.SqlDatasetTestBase): "ORDER BY first_name DESC" }) # Max value of int64 - self.assertEqual((b"John", 9223372036854775807), sess.run(get_next)) + self.assertEqual((b"John", 9223372036854775807), self.evaluate(get_next)) # Min value of int64 - self.assertEqual((b"Jane", -9223372036854775808), sess.run(get_next)) + self.assertEqual((b"Jane", -9223372036854775808), self.evaluate(get_next)) with self.assertRaises(errors.OutOfRangeError): - sess.run(get_next) + self.evaluate(get_next) # Test that `SqlDataset` can read an integer from a SQLite database table and # place it in a `uint8` tensor. @@ -429,10 +433,10 @@ class SqlDatasetTest(sql_dataset_test_base.SqlDatasetTestBase): self.query: "SELECT first_name, desk_number FROM students " "ORDER BY first_name DESC" }) - self.assertEqual((b"John", 9), sess.run(get_next)) - self.assertEqual((b"Jane", 127), sess.run(get_next)) + self.assertEqual((b"John", 9), self.evaluate(get_next)) + self.assertEqual((b"Jane", 127), self.evaluate(get_next)) with self.assertRaises(errors.OutOfRangeError): - sess.run(get_next) + self.evaluate(get_next) # Test that `SqlDataset` can read the minimum and maximum uint8 values from a # SQLite database table and place them in `uint8` tensors. @@ -446,11 +450,11 @@ class SqlDatasetTest(sql_dataset_test_base.SqlDatasetTestBase): "ORDER BY first_name DESC" }) # Min value of uint8 - self.assertEqual((b"John", 0), sess.run(get_next)) + self.assertEqual((b"John", 0), self.evaluate(get_next)) # Max value of uint8 - self.assertEqual((b"Jane", 255), sess.run(get_next)) + self.assertEqual((b"Jane", 255), self.evaluate(get_next)) with self.assertRaises(errors.OutOfRangeError): - sess.run(get_next) + self.evaluate(get_next) # Test that `SqlDataset` can read an integer from a SQLite database table # and place it in a `uint16` tensor. @@ -463,10 +467,10 @@ class SqlDatasetTest(sql_dataset_test_base.SqlDatasetTestBase): self.query: "SELECT first_name, desk_number FROM students " "ORDER BY first_name DESC" }) - self.assertEqual((b"John", 9), sess.run(get_next)) - self.assertEqual((b"Jane", 127), sess.run(get_next)) + self.assertEqual((b"John", 9), self.evaluate(get_next)) + self.assertEqual((b"Jane", 127), self.evaluate(get_next)) with self.assertRaises(errors.OutOfRangeError): - sess.run(get_next) + self.evaluate(get_next) # Test that `SqlDataset` can read the minimum and maximum uint16 values from a # SQLite database table and place them in `uint16` tensors. @@ -480,11 +484,11 @@ class SqlDatasetTest(sql_dataset_test_base.SqlDatasetTestBase): "ORDER BY first_name DESC" }) # Min value of uint16 - self.assertEqual((b"John", 0), sess.run(get_next)) + self.assertEqual((b"John", 0), self.evaluate(get_next)) # Max value of uint16 - self.assertEqual((b"Jane", 65535), sess.run(get_next)) + self.assertEqual((b"Jane", 65535), self.evaluate(get_next)) with self.assertRaises(errors.OutOfRangeError): - sess.run(get_next) + self.evaluate(get_next) # Test that `SqlDataset` can read a 0-valued and 1-valued integer from a # SQLite database table and place them as `True` and `False` respectively @@ -499,10 +503,10 @@ class SqlDatasetTest(sql_dataset_test_base.SqlDatasetTestBase): "SELECT first_name, registration_complete FROM students " "ORDER BY first_name DESC" }) - self.assertEqual((b"John", True), sess.run(get_next)) - self.assertEqual((b"Jane", False), sess.run(get_next)) + self.assertEqual((b"John", True), self.evaluate(get_next)) + self.assertEqual((b"Jane", False), self.evaluate(get_next)) with self.assertRaises(errors.OutOfRangeError): - sess.run(get_next) + self.evaluate(get_next) # Test that `SqlDataset` can read an integer that is not 0-valued or 1-valued # from a SQLite database table and place it as `True` in a `bool` tensor. @@ -515,10 +519,10 @@ class SqlDatasetTest(sql_dataset_test_base.SqlDatasetTestBase): self.query: "SELECT first_name, favorite_medium_sized_number " "FROM students ORDER BY first_name DESC" }) - self.assertEqual((b"John", True), sess.run(get_next)) - self.assertEqual((b"Jane", True), sess.run(get_next)) + self.assertEqual((b"John", True), self.evaluate(get_next)) + self.assertEqual((b"Jane", True), self.evaluate(get_next)) with self.assertRaises(errors.OutOfRangeError): - sess.run(get_next) + self.evaluate(get_next) # Test that `SqlDataset` can read a float from a SQLite database table # and place it in a `float64` tensor. @@ -533,10 +537,11 @@ class SqlDatasetTest(sql_dataset_test_base.SqlDatasetTestBase): "SELECT first_name, last_name, victories FROM townspeople " "ORDER BY first_name" }) - self.assertEqual((b"George", b"Washington", 20.0), sess.run(get_next)) - self.assertEqual((b"John", b"Adams", -19.95), sess.run(get_next)) + self.assertEqual((b"George", b"Washington", 20.0), + self.evaluate(get_next)) + self.assertEqual((b"John", b"Adams", -19.95), self.evaluate(get_next)) with self.assertRaises(errors.OutOfRangeError): - sess.run(get_next) + self.evaluate(get_next) # Test that `SqlDataset` can read a float from a SQLite database table beyond # the precision of 64-bit IEEE, without throwing an error. Test that @@ -555,13 +560,13 @@ class SqlDatasetTest(sql_dataset_test_base.SqlDatasetTestBase): self.assertEqual( (b"George", b"Washington", 1331241.321342132321324589798264627463827647382647382643874), - sess.run(get_next)) + self.evaluate(get_next)) self.assertEqual( (b"John", b"Adams", 1331241321342132321324589798264627463827647382647382643874.0), - sess.run(get_next)) + self.evaluate(get_next)) with self.assertRaises(errors.OutOfRangeError): - sess.run(get_next) + self.evaluate(get_next) # Test that `SqlDataset` can read a float from a SQLite database table, # representing the largest integer representable as a 64-bit IEEE float @@ -579,11 +584,11 @@ class SqlDatasetTest(sql_dataset_test_base.SqlDatasetTestBase): "ORDER BY first_name" }) self.assertNotEqual((b"George", b"Washington", 9007199254740992.0), - sess.run(get_next)) + self.evaluate(get_next)) self.assertNotEqual((b"John", b"Adams", 9007199254740991.0), - sess.run(get_next)) + self.evaluate(get_next)) with self.assertRaises(errors.OutOfRangeError): - sess.run(get_next) + self.evaluate(get_next) if __name__ == "__main__": diff --git a/tensorflow/python/data/experimental/kernel_tests/sql_dataset_test_base.py b/tensorflow/python/data/experimental/kernel_tests/sql_dataset_test_base.py index 6aaaa90c651..809e09c8042 100644 --- a/tensorflow/python/data/experimental/kernel_tests/sql_dataset_test_base.py +++ b/tensorflow/python/data/experimental/kernel_tests/sql_dataset_test_base.py @@ -24,6 +24,7 @@ import sqlite3 from tensorflow.python.data.experimental.ops import readers from tensorflow.python.data.kernel_tests import test_base +from tensorflow.python.data.ops import dataset_ops from tensorflow.python.framework import dtypes from tensorflow.python.ops import array_ops from tensorflow.python.platform import test @@ -35,7 +36,7 @@ class SqlDatasetTestBase(test_base.DatasetTestBase): def _createSqlDataset(self, output_types, num_repeats=1): dataset = readers.SqlDataset(self.driver_name, self.data_source_name, self.query, output_types).repeat(num_repeats) - iterator = dataset.make_initializable_iterator() + iterator = dataset_ops.make_initializable_iterator(dataset) init_op = iterator.initializer get_next = iterator.get_next() return init_op, get_next diff --git a/tensorflow/python/data/experimental/kernel_tests/stats_dataset_ops_test.py b/tensorflow/python/data/experimental/kernel_tests/stats_dataset_ops_test.py index 83028937d36..8a300364f95 100644 --- a/tensorflow/python/data/experimental/kernel_tests/stats_dataset_ops_test.py +++ b/tensorflow/python/data/experimental/kernel_tests/stats_dataset_ops_test.py @@ -30,6 +30,7 @@ from tensorflow.python.data.experimental.ops import stats_options from tensorflow.python.data.ops import dataset_ops from tensorflow.python.framework import errors from tensorflow.python.framework import ops +from tensorflow.python.framework import test_util from tensorflow.python.ops import array_ops from tensorflow.python.ops import math_ops from tensorflow.python.platform import test @@ -45,84 +46,84 @@ def function_set_stats_aggregator(dataset, def function_apply_options(dataset, aggregator, prefix="", counter_prefix=""): options = dataset_ops.Options() - options.experimental_stats = stats_options.StatsOptions(aggregator) + options.experimental_stats = stats_options.StatsOptions() + options.experimental_stats.aggregator = aggregator + options.experimental_stats.prefix = prefix + options.experimental_stats.counter_prefix = counter_prefix options.experimental_stats.latency_all_edges = False - if prefix: - options.experimental_stats.prefix = prefix - if counter_prefix: - options.experimental_stats.counter_prefix = counter_prefix return dataset.with_options(options) @parameterized.named_parameters( - dict( - testcase_name="SetStatsAggregator", - dataset_transformation=function_set_stats_aggregator), - dict( - testcase_name="StatsOptions", - dataset_transformation=function_apply_options)) + ("SetStatsAggregator", function_set_stats_aggregator), + ("StatsOptions", function_apply_options), +) class StatsDatasetTest(stats_dataset_test_base.StatsDatasetTestBase): + @test_util.run_deprecated_v1 def testBytesProduced(self, dataset_transformation): aggregator = stats_aggregator.StatsAggregator() dataset = dataset_ops.Dataset.range(100).map( lambda x: array_ops.tile([x], ops.convert_to_tensor([x]))).apply( stats_ops.bytes_produced_stats("bytes_produced")) dataset = dataset_transformation(dataset, aggregator) - iterator = dataset.make_initializable_iterator() + iterator = dataset_ops.make_initializable_iterator(dataset) next_element = iterator.get_next() summary_t = aggregator.get_summary() with self.cached_session() as sess: - sess.run(iterator.initializer) + self.evaluate(iterator.initializer) expected_sum = 0.0 for i in range(100): self.assertAllEqual( - np.array([i] * i, dtype=np.int64), sess.run(next_element)) - summary_str = sess.run(summary_t) + np.array([i] * i, dtype=np.int64), self.evaluate(next_element)) + summary_str = self.evaluate(summary_t) self._assertSummaryHasCount(summary_str, "bytes_produced", float(i + 1)) expected_sum += i * 8.0 self._assertSummaryHasSum(summary_str, "bytes_produced", expected_sum) with self.assertRaises(errors.OutOfRangeError): - sess.run(next_element) - summary_str = sess.run(summary_t) + self.evaluate(next_element) + summary_str = self.evaluate(summary_t) self._assertSummaryHasCount(summary_str, "bytes_produced", 100.0) self._assertSummaryHasSum(summary_str, "bytes_produced", expected_sum) + @test_util.run_deprecated_v1 def testLatencyStats(self, dataset_transformation): aggregator = stats_aggregator.StatsAggregator() dataset = dataset_ops.Dataset.range(100).apply( stats_ops.latency_stats("record_latency")) dataset = dataset_transformation(dataset, aggregator) - iterator = dataset.make_initializable_iterator() + iterator = dataset_ops.make_initializable_iterator(dataset) next_element = iterator.get_next() summary_t = aggregator.get_summary() with self.cached_session() as sess: - sess.run(iterator.initializer) + self.evaluate(iterator.initializer) for i in range(100): - self.assertEqual(i, sess.run(next_element)) + self.assertEqual(i, self.evaluate(next_element)) self._assertSummaryHasCount( - sess.run(summary_t), "record_latency", float(i + 1)) + self.evaluate(summary_t), "record_latency", float(i + 1)) with self.assertRaises(errors.OutOfRangeError): - sess.run(next_element) - self._assertSummaryHasCount(sess.run(summary_t), "record_latency", 100.0) + self.evaluate(next_element) + self._assertSummaryHasCount( + self.evaluate(summary_t), "record_latency", 100.0) + @test_util.run_deprecated_v1 def testPrefetchBufferUtilization(self, dataset_transformation): aggregator = stats_aggregator.StatsAggregator() dataset = dataset_ops.Dataset.range(100).map( lambda x: array_ops.tile([x], ops.convert_to_tensor([x]))).prefetch(-1) dataset = dataset_transformation(dataset, aggregator) - iterator = dataset.make_initializable_iterator() + iterator = dataset_ops.make_initializable_iterator(dataset) next_element = iterator.get_next() summary_t = aggregator.get_summary() with self.cached_session() as sess: - sess.run(iterator.initializer) + self.evaluate(iterator.initializer) for i in range(100): self.assertAllEqual( - np.array([i] * i, dtype=np.int64), sess.run(next_element)) - summary_str = sess.run(summary_t) + np.array([i] * i, dtype=np.int64), self.evaluate(next_element)) + summary_str = self.evaluate(summary_t) self._assertSummaryHasCount(summary_str, "Prefetch::buffer_utilization", float(i + 1)) self._assertSummaryContains(summary_str, "Prefetch::buffer_capacity") @@ -130,58 +131,62 @@ class StatsDatasetTest(stats_dataset_test_base.StatsDatasetTestBase): self._assertSummaryHasRange(summary_str, "Prefetch::buffer_utilization", 0, 1) with self.assertRaises(errors.OutOfRangeError): - sess.run(next_element) - summary_str = sess.run(summary_t) + self.evaluate(next_element) + summary_str = self.evaluate(summary_t) self._assertSummaryHasCount(summary_str, "Prefetch::buffer_utilization", 100) + @test_util.run_deprecated_v1 def testPrefetchBufferScalars(self, dataset_transformation): aggregator = stats_aggregator.StatsAggregator() dataset = dataset_ops.Dataset.range(10).map( lambda x: array_ops.tile([x], ops.convert_to_tensor([x]))).prefetch(0) dataset = dataset_transformation(dataset, aggregator) - iterator = dataset.make_initializable_iterator() + iterator = dataset_ops.make_initializable_iterator(dataset) next_element = iterator.get_next() summary_t = aggregator.get_summary() with self.cached_session() as sess: - sess.run(iterator.initializer) + self.evaluate(iterator.initializer) for i in range(10): self.assertAllEqual( - np.array([i] * i, dtype=np.int64), sess.run(next_element)) - summary_str = sess.run(summary_t) + np.array([i] * i, dtype=np.int64), self.evaluate(next_element)) + summary_str = self.evaluate(summary_t) self._assertSummaryHasScalarValue(summary_str, "Prefetch::buffer_capacity", 0) self._assertSummaryHasScalarValue(summary_str, "Prefetch::buffer_size", 0) with self.assertRaises(errors.OutOfRangeError): - sess.run(next_element) + self.evaluate(next_element) + @test_util.run_deprecated_v1 def testFilteredElementsStats(self, dataset_transformation): aggregator = stats_aggregator.StatsAggregator() dataset = dataset_ops.Dataset.range(101).filter( lambda x: math_ops.equal(math_ops.mod(x, 3), 0)) dataset = dataset_transformation(dataset, aggregator) - iterator = dataset.make_initializable_iterator() + iterator = dataset_ops.make_initializable_iterator(dataset) next_element = iterator.get_next() summary_t = aggregator.get_summary() with self.test_session() as sess: - sess.run(iterator.initializer) + self.evaluate(iterator.initializer) for i in range(34): - self.assertEqual(i * 3, sess.run(next_element)) + self.assertEqual(i * 3, self.evaluate(next_element)) if i is not 0: self._assertSummaryHasScalarValue( - sess.run(summary_t), "Filter::dropped_elements", float(i * 2)) + self.evaluate(summary_t), "Filter::dropped_elements", + float(i * 2)) self._assertSummaryHasScalarValue( - sess.run(summary_t), "Filter::filtered_elements", float(i + 1)) + self.evaluate(summary_t), "Filter::filtered_elements", float(i + 1)) with self.assertRaises(errors.OutOfRangeError): - sess.run(next_element) + self.evaluate(next_element) self._assertSummaryHasScalarValue( - sess.run(summary_t), "Filter::dropped_elements", 67.0) + self.evaluate(summary_t), "Filter::dropped_elements", 67.0) self._assertSummaryHasScalarValue( - sess.run(summary_t), "Filter::filtered_elements", 34.0) + self.evaluate(summary_t), "Filter::filtered_elements", 34.0) + @test_util.run_deprecated_v1 def testMapBufferUtilization(self, dataset_transformation): def dataset_fn(): @@ -196,6 +201,7 @@ class StatsDatasetTest(stats_dataset_test_base.StatsDatasetTestBase): dataset_transformation, function_processing_time=True) + @test_util.run_deprecated_v1 def testMapAutoTuneBufferUtilization(self, dataset_transformation): def dataset_fn(): @@ -213,6 +219,7 @@ class StatsDatasetTest(stats_dataset_test_base.StatsDatasetTestBase): dataset_transformation, function_processing_time=True) + @test_util.run_deprecated_v1 def testInterleaveAutoTuneBufferUtilization(self, dataset_transformation): def dataset_fn(): @@ -229,6 +236,7 @@ class StatsDatasetTest(stats_dataset_test_base.StatsDatasetTestBase): self._testParallelCallsStats(dataset_fn, "ParallelInterleaveV2", 10, dataset_transformation) + @test_util.run_deprecated_v1 def testMapAndBatchAutoTuneBufferUtilization(self, dataset_transformation): def dataset_fn(): @@ -250,104 +258,114 @@ class StatsDatasetTest(stats_dataset_test_base.StatsDatasetTestBase): check_elements=False, function_processing_time=True) + @test_util.run_deprecated_v1 def testReinitialize(self, dataset_transformation): aggregator = stats_aggregator.StatsAggregator() dataset = dataset_ops.Dataset.range(100).apply( stats_ops.latency_stats("record_latency")) dataset = dataset_transformation(dataset, aggregator) - iterator = dataset.make_initializable_iterator() + iterator = dataset_ops.make_initializable_iterator(dataset) next_element = iterator.get_next() summary_t = aggregator.get_summary() with self.cached_session() as sess: for j in range(5): - sess.run(iterator.initializer) + self.evaluate(iterator.initializer) for i in range(100): - self.assertEqual(i, sess.run(next_element)) + self.assertEqual(i, self.evaluate(next_element)) self._assertSummaryHasCount( - sess.run(summary_t), "record_latency", float((j * 100) + i + 1)) + self.evaluate(summary_t), "record_latency", + float((j * 100) + i + 1)) with self.assertRaises(errors.OutOfRangeError): - sess.run(next_element) + self.evaluate(next_element) self._assertSummaryHasCount( - sess.run(summary_t), "record_latency", (j + 1) * 100.0) + self.evaluate(summary_t), "record_latency", (j + 1) * 100.0) + @test_util.run_deprecated_v1 def testNoAggregatorRegistered(self, dataset_transformation): dataset = dataset_ops.Dataset.range(100).apply( stats_ops.latency_stats("record_latency")) - iterator = dataset.make_initializable_iterator() + iterator = dataset_ops.make_initializable_iterator(dataset) next_element = iterator.get_next() with self.cached_session() as sess: - sess.run(iterator.initializer) + self.evaluate(iterator.initializer) for i in range(100): - self.assertEqual(i, sess.run(next_element)) + self.assertEqual(i, self.evaluate(next_element)) with self.assertRaises(errors.OutOfRangeError): - sess.run(next_element) + self.evaluate(next_element) + @test_util.run_deprecated_v1 def testMultipleTags(self, dataset_transformation): aggregator = stats_aggregator.StatsAggregator() dataset = dataset_ops.Dataset.range(100).apply( stats_ops.latency_stats("record_latency")).apply( stats_ops.latency_stats("record_latency_2")) dataset = dataset_transformation(dataset, aggregator) - iterator = dataset.make_initializable_iterator() + iterator = dataset_ops.make_initializable_iterator(dataset) next_element = iterator.get_next() summary_t = aggregator.get_summary() with self.cached_session() as sess: - sess.run(iterator.initializer) + self.evaluate(iterator.initializer) for i in range(100): - self.assertEqual(i, sess.run(next_element)) + self.assertEqual(i, self.evaluate(next_element)) self._assertSummaryHasCount( - sess.run(summary_t), "record_latency", float(i + 1)) + self.evaluate(summary_t), "record_latency", float(i + 1)) self._assertSummaryHasCount( - sess.run(summary_t), "record_latency_2", float(i + 1)) + self.evaluate(summary_t), "record_latency_2", float(i + 1)) with self.assertRaises(errors.OutOfRangeError): - sess.run(next_element) - self._assertSummaryHasCount(sess.run(summary_t), "record_latency", 100.0) + self.evaluate(next_element) self._assertSummaryHasCount( - sess.run(summary_t), "record_latency_2", 100.0) + self.evaluate(summary_t), "record_latency", 100.0) + self._assertSummaryHasCount( + self.evaluate(summary_t), "record_latency_2", 100.0) + @test_util.run_deprecated_v1 def testRepeatedTags(self, dataset_transformation): aggregator = stats_aggregator.StatsAggregator() dataset = dataset_ops.Dataset.range(100).apply( stats_ops.latency_stats("record_latency")).apply( stats_ops.latency_stats("record_latency")) dataset = dataset_transformation(dataset, aggregator) - iterator = dataset.make_initializable_iterator() + iterator = dataset_ops.make_initializable_iterator(dataset) next_element = iterator.get_next() summary_t = aggregator.get_summary() with self.cached_session() as sess: - sess.run(iterator.initializer) + self.evaluate(iterator.initializer) for i in range(100): - self.assertEqual(i, sess.run(next_element)) + self.assertEqual(i, self.evaluate(next_element)) self._assertSummaryHasCount( - sess.run(summary_t), "record_latency", float(2 * (i + 1))) + self.evaluate(summary_t), "record_latency", float(2 * (i + 1))) with self.assertRaises(errors.OutOfRangeError): - sess.run(next_element) - self._assertSummaryHasCount(sess.run(summary_t), "record_latency", 200.0) + self.evaluate(next_element) + self._assertSummaryHasCount( + self.evaluate(summary_t), "record_latency", 200.0) + @test_util.run_deprecated_v1 def testMultipleIteratorsSameAggregator(self, dataset_transformation): aggregator = stats_aggregator.StatsAggregator() dataset = dataset_ops.Dataset.range(100).apply( stats_ops.latency_stats("record_latency")) dataset = dataset_transformation(dataset, aggregator) - iterator_0 = dataset.make_initializable_iterator() - iterator_1 = dataset.make_initializable_iterator() + iterator_0 = dataset_ops.make_initializable_iterator(dataset) + iterator_1 = dataset_ops.make_initializable_iterator(dataset) next_element = iterator_0.get_next() + iterator_1.get_next() summary_t = aggregator.get_summary() with self.cached_session() as sess: - sess.run([iterator_0.initializer, iterator_1.initializer]) + self.evaluate([iterator_0.initializer, iterator_1.initializer]) for i in range(100): - self.assertEqual(i * 2, sess.run(next_element)) + self.assertEqual(i * 2, self.evaluate(next_element)) self._assertSummaryHasCount( - sess.run(summary_t), "record_latency", float(2 * (i + 1))) + self.evaluate(summary_t), "record_latency", float(2 * (i + 1))) with self.assertRaises(errors.OutOfRangeError): - sess.run(next_element) - self._assertSummaryHasCount(sess.run(summary_t), "record_latency", 200.0) + self.evaluate(next_element) + self._assertSummaryHasCount( + self.evaluate(summary_t), "record_latency", 200.0) + @test_util.run_deprecated_v1 def testMultipleDatasetWithPrefixes(self, dataset_transformation): aggregator = stats_aggregator.StatsAggregator() dataset = dataset_ops.Dataset.range(100).apply( @@ -356,25 +374,25 @@ class StatsDatasetTest(stats_dataset_test_base.StatsDatasetTestBase): dataset2 = dataset_ops.Dataset.range(100).apply( stats_ops.latency_stats("record_latency")) dataset2 = dataset_transformation(dataset2, aggregator, prefix="dataset2") - iterator_0 = dataset.make_initializable_iterator() - iterator_1 = dataset2.make_initializable_iterator() + iterator_0 = dataset_ops.make_initializable_iterator(dataset) + iterator_1 = dataset_ops.make_initializable_iterator(dataset2) next_element = iterator_0.get_next() + iterator_1.get_next() summary_t = aggregator.get_summary() with self.test_session() as sess: - sess.run([iterator_0.initializer, iterator_1.initializer]) + self.evaluate([iterator_0.initializer, iterator_1.initializer]) for i in range(100): - self.assertEqual(i * 2, sess.run(next_element)) + self.assertEqual(i * 2, self.evaluate(next_element)) self._assertSummaryHasCount( - sess.run(summary_t), "dataset1_record_latency", float(i + 1)) + self.evaluate(summary_t), "dataset1_record_latency", float(i + 1)) self._assertSummaryHasCount( - sess.run(summary_t), "dataset2_record_latency", float(i + 1)) + self.evaluate(summary_t), "dataset2_record_latency", float(i + 1)) with self.assertRaises(errors.OutOfRangeError): - sess.run(next_element) + self.evaluate(next_element) self._assertSummaryHasCount( - sess.run(summary_t), "dataset1_record_latency", 100.0) + self.evaluate(summary_t), "dataset1_record_latency", 100.0) self._assertSummaryHasCount( - sess.run(summary_t), "dataset2_record_latency", 100.0) + self.evaluate(summary_t), "dataset2_record_latency", 100.0) @parameterized.named_parameters( @@ -388,6 +406,7 @@ class FeatureStatsDatasetTest( stats_dataset_test_base.StatsDatasetTestBase, reader_dataset_ops_test_base.MakeBatchedFeaturesDatasetTestBase): + @test_util.run_deprecated_v1 def testFeaturesStats(self, dataset_transformation): num_epochs = 5 total_records = num_epochs * self._num_records @@ -416,25 +435,26 @@ class FeatureStatsDatasetTest( dataset = dataset_transformation( dataset_fn(), aggregator, prefix="record_stats") - iterator = dataset.make_initializable_iterator() + iterator = dataset_ops.make_initializable_iterator(dataset) next_element = iterator.get_next() summary_t = aggregator.get_summary() with self.test_session() as sess: - sess.run(iterator.initializer) + self.evaluate(iterator.initializer) for _ in range(num_output): - sess.run(next_element) + self.evaluate(next_element) with self.assertRaises(errors.OutOfRangeError): - sess.run(next_element) + self.evaluate(next_element) self._assertSummaryHasCount( - sess.run(summary_t), "record_stats_features", total_records) + self.evaluate(summary_t), "record_stats_features", total_records) self._assertSummaryHasCount( - sess.run(summary_t), "record_stats_feature-values", total_records) + self.evaluate(summary_t), "record_stats_feature-values", + total_records) self._assertSummaryHasSum( - sess.run(summary_t), "record_stats_features", total_records * 4) + self.evaluate(summary_t), "record_stats_features", total_records * 4) self._assertSummaryHasSum( - sess.run(summary_t), "record_stats_feature-values", + self.evaluate(summary_t), "record_stats_feature-values", self._sum_keywords(1) * num_epochs + 3 * total_records) diff --git a/tensorflow/python/data/experimental/kernel_tests/stats_dataset_test_base.py b/tensorflow/python/data/experimental/kernel_tests/stats_dataset_test_base.py index c5bf9267590..ab1d1c3028a 100644 --- a/tensorflow/python/data/experimental/kernel_tests/stats_dataset_test_base.py +++ b/tensorflow/python/data/experimental/kernel_tests/stats_dataset_test_base.py @@ -22,6 +22,7 @@ import numpy as np from tensorflow.core.framework import summary_pb2 from tensorflow.python.data.experimental.ops import stats_aggregator from tensorflow.python.data.kernel_tests import test_base +from tensorflow.python.data.ops import dataset_ops from tensorflow.python.framework import errors @@ -93,7 +94,7 @@ class StatsDatasetTestBase(test_base.DatasetTestBase): aggregator = stats_aggregator.StatsAggregator() dataset = dataset_fn() dataset = dataset_transformation(dataset, aggregator) - iterator = dataset.make_initializable_iterator() + iterator = dataset_ops.make_initializable_iterator(dataset) next_element = iterator.get_next() summary_t = aggregator.get_summary() diff --git a/tensorflow/python/data/experimental/kernel_tests/unbatch_test.py b/tensorflow/python/data/experimental/kernel_tests/unbatch_test.py index 0278a208cbb..cef5e8d269c 100644 --- a/tensorflow/python/data/experimental/kernel_tests/unbatch_test.py +++ b/tensorflow/python/data/experimental/kernel_tests/unbatch_test.py @@ -17,20 +17,18 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function -import time from absl.testing import parameterized import numpy as np -from tensorflow.python.client import session from tensorflow.python.data.experimental.ops import batching from tensorflow.python.data.kernel_tests import test_base from tensorflow.python.data.ops import dataset_ops from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes from tensorflow.python.framework import errors -from tensorflow.python.framework import ops from tensorflow.python.framework import sparse_tensor +from tensorflow.python.framework import test_util from tensorflow.python.ops import array_ops from tensorflow.python.ops import math_ops from tensorflow.python.ops import string_ops @@ -40,20 +38,22 @@ from tensorflow.python.util import compat class UnbatchTest(test_base.DatasetTestBase, parameterized.TestCase): + @test_util.run_deprecated_v1 def testUnbatchWithUnknownRankInput(self): placeholder = array_ops.placeholder(dtypes.int32) dataset = dataset_ops.Dataset.from_tensors(placeholder).apply( batching.unbatch()) - iterator = dataset.make_initializable_iterator() + iterator = dataset_ops.make_initializable_iterator(dataset) next_elem = iterator.get_next() with self.cached_session() as sess: sess.run(iterator.initializer, feed_dict={placeholder: [0, 1, 2, 3]}) for i in range(4): - self.assertEqual(i, sess.run(next_elem)) + self.assertEqual(i, self.evaluate(next_elem)) with self.assertRaises(errors.OutOfRangeError): - sess.run(next_elem) + self.evaluate(next_elem) + @test_util.run_deprecated_v1 def testUnbatchScalarDataset(self): data = tuple([math_ops.range(10) for _ in range(3)]) data = dataset_ops.Dataset.from_tensor_slices(data) @@ -63,16 +63,17 @@ class UnbatchTest(test_base.DatasetTestBase, parameterized.TestCase): data = data.apply(batching.unbatch()) self.assertEqual(expected_types, data.output_types) - iterator = data.make_one_shot_iterator() + iterator = dataset_ops.make_one_shot_iterator(data) op = iterator.get_next() with self.cached_session() as sess: for i in range(10): - self.assertEqual((i,) * 3, sess.run(op)) + self.assertEqual((i,) * 3, self.evaluate(op)) with self.assertRaises(errors.OutOfRangeError): - sess.run(op) + self.evaluate(op) + @test_util.run_deprecated_v1 def testUnbatchDatasetWithStrings(self): data = tuple([math_ops.range(10) for _ in range(3)]) data = dataset_ops.Dataset.from_tensor_slices(data) @@ -83,16 +84,17 @@ class UnbatchTest(test_base.DatasetTestBase, parameterized.TestCase): data = data.apply(batching.unbatch()) self.assertEqual(expected_types, data.output_types) - iterator = data.make_one_shot_iterator() + iterator = dataset_ops.make_one_shot_iterator(data) op = iterator.get_next() with self.cached_session() as sess: for i in range(10): - self.assertEqual((i, compat.as_bytes(str(i)), i), sess.run(op)) + self.assertEqual((i, compat.as_bytes(str(i)), i), self.evaluate(op)) with self.assertRaises(errors.OutOfRangeError): - sess.run(op) + self.evaluate(op) + @test_util.run_deprecated_v1 def testUnbatchDatasetWithSparseTensor(self): st = sparse_tensor.SparseTensorValue( indices=[[i, i] for i in range(10)], @@ -102,18 +104,19 @@ class UnbatchTest(test_base.DatasetTestBase, parameterized.TestCase): data = data.apply(batching.unbatch()) data = data.batch(5) data = data.apply(batching.unbatch()) - iterator = data.make_one_shot_iterator() + iterator = dataset_ops.make_one_shot_iterator(data) next_element = iterator.get_next() with self.cached_session() as sess: for i in range(10): - st_row = sess.run(next_element) + st_row = self.evaluate(next_element) self.assertEqual([i], st_row.indices) self.assertEqual([i], st_row.values) self.assertEqual([10], st_row.dense_shape) with self.assertRaises(errors.OutOfRangeError): - sess.run(next_element) + self.evaluate(next_element) + @test_util.run_deprecated_v1 def testUnbatchDatasetWithDenseAndSparseTensor(self): st = sparse_tensor.SparseTensorValue( indices=[[i, i] for i in range(10)], @@ -123,19 +126,20 @@ class UnbatchTest(test_base.DatasetTestBase, parameterized.TestCase): data = data.apply(batching.unbatch()) data = data.batch(5) data = data.apply(batching.unbatch()) - iterator = data.make_one_shot_iterator() + iterator = dataset_ops.make_one_shot_iterator(data) next_element = iterator.get_next() with self.cached_session() as sess: for i in range(10): - dense_elem, st_row = sess.run(next_element) + dense_elem, st_row = self.evaluate(next_element) self.assertEqual(i, dense_elem) self.assertEqual([i], st_row.indices) self.assertEqual([i], st_row.values) self.assertEqual([10], st_row.dense_shape) with self.assertRaises(errors.OutOfRangeError): - sess.run(next_element) + self.evaluate(next_element) + @test_util.run_deprecated_v1 def testUnbatchSingleElementTupleDataset(self): data = tuple([(math_ops.range(10),) for _ in range(3)]) data = dataset_ops.Dataset.from_tensor_slices(data) @@ -145,16 +149,17 @@ class UnbatchTest(test_base.DatasetTestBase, parameterized.TestCase): data = data.apply(batching.unbatch()) self.assertEqual(expected_types, data.output_types) - iterator = data.make_one_shot_iterator() + iterator = dataset_ops.make_one_shot_iterator(data) op = iterator.get_next() with self.cached_session() as sess: for i in range(10): - self.assertEqual(((i,),) * 3, sess.run(op)) + self.assertEqual(((i,),) * 3, self.evaluate(op)) with self.assertRaises(errors.OutOfRangeError): - sess.run(op) + self.evaluate(op) + @test_util.run_deprecated_v1 def testUnbatchMultiElementTupleDataset(self): data = tuple([(math_ops.range(10 * i, 10 * i + 10), array_ops.fill([10], "hi")) for i in range(3)]) @@ -165,28 +170,29 @@ class UnbatchTest(test_base.DatasetTestBase, parameterized.TestCase): data = data.apply(batching.unbatch()) self.assertAllEqual(expected_types, data.output_types) - iterator = data.make_one_shot_iterator() + iterator = dataset_ops.make_one_shot_iterator(data) op = iterator.get_next() with self.cached_session() as sess: for i in range(10): self.assertEqual(((i, b"hi"), (10 + i, b"hi"), (20 + i, b"hi")), - sess.run(op)) + self.evaluate(op)) with self.assertRaises(errors.OutOfRangeError): - sess.run(op) + self.evaluate(op) + @test_util.run_deprecated_v1 def testUnbatchEmpty(self): data = dataset_ops.Dataset.from_tensors( (constant_op.constant([]), constant_op.constant([], shape=[0, 4]), constant_op.constant([], shape=[0, 4, 0]))) data = data.apply(batching.unbatch()) - iterator = data.make_one_shot_iterator() + iterator = dataset_ops.make_one_shot_iterator(data) next_element = iterator.get_next() with self.cached_session() as sess: with self.assertRaises(errors.OutOfRangeError): - sess.run(next_element) + self.evaluate(next_element) def testUnbatchStaticShapeMismatch(self): data = dataset_ops.Dataset.from_tensors((np.arange(7), np.arange(8), @@ -194,12 +200,13 @@ class UnbatchTest(test_base.DatasetTestBase, parameterized.TestCase): with self.assertRaises(ValueError): data.apply(batching.unbatch()) + @test_util.run_deprecated_v1 def testUnbatchDynamicShapeMismatch(self): ph1 = array_ops.placeholder(dtypes.int32, shape=[None]) ph2 = array_ops.placeholder(dtypes.int32, shape=None) data = dataset_ops.Dataset.from_tensors((ph1, ph2)) data = data.apply(batching.unbatch()) - iterator = data.make_initializable_iterator() + iterator = dataset_ops.make_initializable_iterator(data) next_element = iterator.get_next() with self.cached_session() as sess: @@ -211,7 +218,7 @@ class UnbatchTest(test_base.DatasetTestBase, parameterized.TestCase): ph2: np.arange(8).astype(np.int32) }) with self.assertRaises(errors.InvalidArgumentError): - sess.run(next_element) + self.evaluate(next_element) # No 0th dimension (i.e. scalar value) for one component. sess.run( @@ -221,79 +228,7 @@ class UnbatchTest(test_base.DatasetTestBase, parameterized.TestCase): ph2: 7 }) with self.assertRaises(errors.InvalidArgumentError): - sess.run(next_element) - - -class UnbatchBenchmark(test.Benchmark): - - def benchmarkNativeUnbatch(self): - batch_sizes = [1, 2, 5, 10, 20, 50] - elems_per_trial = 10000 - with ops.Graph().as_default(): - dataset = dataset_ops.Dataset.from_tensors("element").repeat(None) - batch_size_placeholder = array_ops.placeholder(dtypes.int64, shape=[]) - dataset = dataset.batch(batch_size_placeholder) - dataset = dataset.apply(batching.unbatch()) - dataset = dataset.skip(elems_per_trial) - iterator = dataset.make_initializable_iterator() - next_element = iterator.get_next() - - with session.Session() as sess: - for batch_size in batch_sizes: - deltas = [] - for _ in range(5): - sess.run( - iterator.initializer, - feed_dict={batch_size_placeholder: batch_size}) - start = time.time() - sess.run(next_element.op) - end = time.time() - deltas.append((end - start) / elems_per_trial) - - median_wall_time = np.median(deltas) - print("Unbatch (native) batch size: %d Median wall time per element:" - " %f microseconds" % (batch_size, median_wall_time * 1e6)) - self.report_benchmark( - iters=10000, - wall_time=median_wall_time, - name="benchmark_unbatch_dataset_native_batch_size_%d" % - batch_size) - - # Include a benchmark of the previous `unbatch()` implementation that uses - # a composition of more primitive ops. Eventually we'd hope to generate code - # that is as good in both cases. - def benchmarkOldUnbatchImplementation(self): - batch_sizes = [1, 2, 5, 10, 20, 50] - elems_per_trial = 10000 - with ops.Graph().as_default(): - dataset = dataset_ops.Dataset.from_tensors("element").repeat(None) - batch_size_placeholder = array_ops.placeholder(dtypes.int64, shape=[]) - dataset = dataset.batch(batch_size_placeholder) - dataset = dataset.flat_map(dataset_ops.Dataset.from_tensor_slices) - dataset = dataset.skip(elems_per_trial) - iterator = dataset.make_initializable_iterator() - next_element = iterator.get_next() - - with session.Session() as sess: - for batch_size in batch_sizes: - deltas = [] - for _ in range(5): - sess.run( - iterator.initializer, - feed_dict={batch_size_placeholder: batch_size}) - start = time.time() - sess.run(next_element.op) - end = time.time() - deltas.append((end - start) / elems_per_trial) - - median_wall_time = np.median(deltas) - print("Unbatch (unfused) batch size: %d Median wall time per element:" - " %f microseconds" % (batch_size, median_wall_time * 1e6)) - self.report_benchmark( - iters=10000, - wall_time=median_wall_time, - name="benchmark_unbatch_dataset_unfused_batch_size_%d" % - batch_size) + self.evaluate(next_element) if __name__ == "__main__": diff --git a/tensorflow/python/data/experimental/kernel_tests/unique_test.py b/tensorflow/python/data/experimental/kernel_tests/unique_test.py index 847cff26b0d..1d9941d7f4d 100644 --- a/tensorflow/python/data/experimental/kernel_tests/unique_test.py +++ b/tensorflow/python/data/experimental/kernel_tests/unique_test.py @@ -22,6 +22,7 @@ from tensorflow.python.data.kernel_tests import test_base from tensorflow.python.data.ops import dataset_ops from tensorflow.python.framework import dtypes from tensorflow.python.framework import errors +from tensorflow.python.framework import test_util from tensorflow.python.platform import test from tensorflow.python.util import compat @@ -43,20 +44,21 @@ class UniqueTest(test_base.DatasetTestBase): current_test_case = [] dataset = dataset_ops.Dataset.from_generator(lambda: current_test_case, dtype).apply(unique.unique()) - iterator = dataset.make_initializable_iterator() + iterator = dataset_ops.make_initializable_iterator(dataset) next_element = iterator.get_next() with self.cached_session() as sess: for test_case, expected in test_cases: current_test_case = test_case - sess.run(iterator.initializer) + self.evaluate(iterator.initializer) for element in expected: if dtype == dtypes.string: element = compat.as_bytes(element) - self.assertAllEqual(element, sess.run(next_element)) + self.assertAllEqual(element, self.evaluate(next_element)) with self.assertRaises(errors.OutOfRangeError): - sess.run(next_element) + self.evaluate(next_element) + @test_util.run_deprecated_v1 def testSimpleInt(self): for dtype in [dtypes.int32, dtypes.int64]: self._testSimpleHelper(dtype, [ @@ -69,6 +71,7 @@ class UniqueTest(test_base.DatasetTestBase): ([[1, 1], [1, 1], [2, 2], [3, 3], [1, 1]], [[1, 1], [2, 2], [3, 3]]), ]) + @test_util.run_deprecated_v1 def testSimpleString(self): self._testSimpleHelper(dtypes.string, [ ([], []), diff --git a/tensorflow/python/data/experimental/ops/BUILD b/tensorflow/python/data/experimental/ops/BUILD index 170fda90b68..43f43182f60 100644 --- a/tensorflow/python/data/experimental/ops/BUILD +++ b/tensorflow/python/data/experimental/ops/BUILD @@ -4,6 +4,16 @@ licenses(["notice"]) # Apache 2.0 exports_files(["LICENSE"]) +py_library( + name = "cardinality", + srcs = ["cardinality.py"], + srcs_version = "PY2AND3", + deps = [ + "//tensorflow/python:experimental_dataset_ops_gen", + "//tensorflow/python:tensor_util", + ], +) + py_library( name = "counter", srcs = ["counter.py"], @@ -54,8 +64,8 @@ py_library( srcs_version = "PY2AND3", deps = [ "//tensorflow/python:constant_op", - "//tensorflow/python:dataset_ops_gen", "//tensorflow/python:dtypes", + "//tensorflow/python:experimental_dataset_ops_gen", "//tensorflow/python:framework_ops", "//tensorflow/python:random_seed", "//tensorflow/python:tensor_shape", @@ -125,6 +135,7 @@ py_library( "//tensorflow/python/data/util:convert", "//tensorflow/python/data/util:nest", "//tensorflow/python/data/util:sparse", + "//tensorflow/python/data/util:structure", "//third_party/py/numpy", ], ) @@ -139,6 +150,18 @@ py_library( ], ) +py_library( + name = "filter_for_shard_ops", + srcs = ["filter_for_shard_ops.py"], + srcs_version = "PY2AND3", + deps = [ + "//tensorflow/python:dtypes", + "//tensorflow/python:math_ops", + "//tensorflow/python:ops", + "//tensorflow/python:tensor_util", + ], +) + py_library( name = "error_ops", srcs = ["error_ops.py"], @@ -165,7 +188,7 @@ py_library( "//tensorflow/python:tensor_shape", "//tensorflow/python/data/ops:dataset_ops", "//tensorflow/python/data/util:nest", - "//tensorflow/python/data/util:sparse", + "//tensorflow/python/data/util:structure", ], ) @@ -188,6 +211,28 @@ py_library( ], ) +py_library( + name = "map_defun", + srcs = ["map_defun.py"], + srcs_version = "PY2AND3", + deps = [ + "//tensorflow/python:dataset_ops_gen", + "//tensorflow/python:framework_ops", + "//tensorflow/python:tensor_shape", + ], +) + +py_library( + name = "matching_files", + srcs = ["matching_files.py"], + srcs_version = "PY2AND3", + deps = [ + "//tensorflow/python:dataset_ops_gen", + "//tensorflow/python:framework_ops", + "//tensorflow/python:tensor_shape", + ], +) + py_library( name = "optimization", srcs = ["optimization.py"], @@ -201,6 +246,16 @@ py_library( ], ) +py_library( + name = "optimization_options", + srcs = ["optimization_options.py"], + srcs_version = "PY2AND3", + deps = [ + "//tensorflow/python:util", + "//tensorflow/python/data/util:options", + ], +) + py_library( name = "parsing_ops", srcs = ["parsing_ops.py"], @@ -217,17 +272,6 @@ py_library( ], ) -py_library( - name = "map_defun", - srcs = ["map_defun.py"], - srcs_version = "PY2AND3", - deps = [ - "//tensorflow/python:dataset_ops_gen", - "//tensorflow/python:framework_ops", - "//tensorflow/python:tensor_shape", - ], -) - py_library( name = "resampling", srcs = ["resampling.py"], @@ -253,7 +297,7 @@ py_library( srcs = ["scan_ops.py"], srcs_version = "PY2AND3", deps = [ - "//tensorflow/python:dataset_ops_gen", + "//tensorflow/python:experimental_dataset_ops_gen", "//tensorflow/python:framework_ops", "//tensorflow/python:function", "//tensorflow/python/data/ops:dataset_ops", @@ -303,6 +347,18 @@ py_library( srcs_version = "PY2AND3", deps = [ ":stats_aggregator", + "//tensorflow/python:util", + "//tensorflow/python/data/util:options", + ], +) + +py_library( + name = "threading_options", + srcs = ["threading_options.py"], + srcs_version = "PY2AND3", + deps = [ + "//tensorflow/python:util", + "//tensorflow/python/data/util:options", ], ) @@ -313,9 +369,8 @@ py_library( deps = [ "//tensorflow/python:experimental_dataset_ops_gen", "//tensorflow/python:resource_variable_ops", + "//tensorflow/python:util", "//tensorflow/python/data/ops:dataset_ops", - "//tensorflow/python/data/util:nest", - "//tensorflow/python/data/util:sparse", "//tensorflow/python/eager:context", ], ) @@ -378,14 +433,17 @@ py_library( name = "dataset_ops", deps = [ ":batching", + ":cardinality", ":counter", ":enumerate_ops", ":error_ops", + ":filter_for_shard_ops", ":get_single_element", ":grouping", ":indexed_dataset_ops", ":interleave_ops", ":map_defun", + ":matching_files", ":optimization", ":prefetching_ops", ":readers", diff --git a/tensorflow/python/data/experimental/ops/batching.py b/tensorflow/python/data/experimental/ops/batching.py index d8985fd13bf..668bf3e8006 100644 --- a/tensorflow/python/data/experimental/ops/batching.py +++ b/tensorflow/python/data/experimental/ops/batching.py @@ -30,11 +30,12 @@ from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops from tensorflow.python.framework import sparse_tensor from tensorflow.python.framework import tensor_shape +from tensorflow.python.framework import tensor_util from tensorflow.python.ops import array_ops from tensorflow.python.ops import check_ops from tensorflow.python.ops import control_flow_ops from tensorflow.python.ops import gen_array_ops -from tensorflow.python.ops import gen_dataset_ops +from tensorflow.python.ops import gen_experimental_dataset_ops as ged_ops from tensorflow.python.ops import math_ops from tensorflow.python.ops import sparse_ops from tensorflow.python.util.tf_export import tf_export @@ -365,7 +366,7 @@ class _UnbatchDataset(dataset_ops.UnaryDataset): self._input_dataset = input_dataset def _as_variant_tensor(self): - return gen_dataset_ops.unbatch_dataset( + return ged_ops.experimental_unbatch_dataset( self._input_dataset._as_variant_tensor(), # pylint: disable=protected-access **dataset_ops.flat_structure(self)) @@ -455,7 +456,7 @@ class _DenseToSparseBatchDataset(dataset_ops.UnaryDataset): self._row_shape = row_shape def _as_variant_tensor(self): - return gen_dataset_ops.dense_to_sparse_batch_dataset( + return ged_ops.experimental_dense_to_sparse_batch_dataset( self._input_dataset._as_variant_tensor(), # pylint: disable=protected-access self._batch_size, row_shape=convert.partial_shape_to_tensor(self._row_shape), @@ -569,13 +570,16 @@ class _RestructuredDataset(dataset_ops.UnaryDataset): return self._output_shapes -class _MapAndBatchDataset(dataset_ops.MapDataset): +class _MapAndBatchDataset(dataset_ops.UnaryDataset): """A `Dataset` that maps a function over a batch of elements.""" def __init__(self, input_dataset, map_func, batch_size, num_parallel_calls, drop_remainder): """See `Dataset.map()` for details.""" - super(_MapAndBatchDataset, self).__init__(input_dataset, map_func) + super(_MapAndBatchDataset, self).__init__(input_dataset) + self._input_dataset = input_dataset + self._map_func = dataset_ops.StructuredFunctionWrapper( + map_func, "tf.data.experimental.map_and_batch()", dataset=input_dataset) self._batch_size_t = ops.convert_to_tensor( batch_size, dtype=dtypes.int64, name="batch_size") self._num_parallel_calls_t = ops.convert_to_tensor( @@ -583,36 +587,40 @@ class _MapAndBatchDataset(dataset_ops.MapDataset): self._drop_remainder_t = ops.convert_to_tensor( drop_remainder, dtype=dtypes.bool, name="drop_remainder") - self._batch_size = batch_size - self._drop_remainder = drop_remainder + constant_drop_remainder = tensor_util.constant_value(self._drop_remainder_t) + if constant_drop_remainder: + # NOTE(mrry): `constant_drop_remainder` may be `None` (unknown statically) + # or `False` (explicitly retaining the remainder). + self._output_structure = self._map_func.output_structure._batch( # pylint: disable=protected-access + tensor_util.constant_value(self._batch_size_t)) + else: + self._output_structure = self._map_func.output_structure._batch(None) # pylint: disable=protected-access + + def _functions(self): + return [self._map_func] def _as_variant_tensor(self): # pylint: disable=protected-access - input_resource = self._input_dataset._as_variant_tensor() - return gen_dataset_ops.map_and_batch_dataset_v2( - input_resource, - self._map_func.captured_inputs, - f=self._map_func, + return ged_ops.experimental_map_and_batch_dataset( + self._input_dataset._as_variant_tensor(), + self._map_func.function.captured_inputs, + f=self._map_func.function, batch_size=self._batch_size_t, num_parallel_calls=self._num_parallel_calls_t, drop_remainder=self._drop_remainder_t, - **dataset_ops.flat_structure(self)) - # pylint: enable=protected-access + **dataset_ops.flat_structure(structure=self._output_structure)) + + @property + def output_classes(self): + return self._output_structure._to_legacy_output_classes() # pylint: disable=protected-access @property def output_shapes(self): - dim = self._batch_size if self._drop_remainder else None - return nest.pack_sequence_as(self._output_shapes, [ - tensor_shape.vector(dim).concatenate(s) - for s in nest.flatten(self._output_shapes) - ]) + return self._output_structure._to_legacy_output_shapes() # pylint: disable=protected-access @property def output_types(self): - return self._output_types - - def _transformation_name(self): - return "tf.data.experimental.map_and_batch()" + return self._output_structure._to_legacy_output_types() # pylint: disable=protected-access @tf_export("data.experimental.map_and_batch") diff --git a/tensorflow/python/data/experimental/ops/cardinality.py b/tensorflow/python/data/experimental/ops/cardinality.py new file mode 100644 index 00000000000..9cf0a8801e8 --- /dev/null +++ b/tensorflow/python/data/experimental/ops/cardinality.py @@ -0,0 +1,50 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Cardinality analysis of `Dataset` objects.""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from tensorflow.python.ops import gen_experimental_dataset_ops as ged_ops +from tensorflow.python.util.tf_export import tf_export + + +INFINITE = -1 +UNKNOWN = -2 +tf_export("data.experimental.INFINITE_CARDINALITY").export_constant( + __name__, "INFINITE") +tf_export("data.experimental.UNKNOWN_CARDINALITY").export_constant( + __name__, "UNKNOWN") + + +@tf_export("data.experimental.cardinality") +def cardinality(dataset): + """Returns the cardinality of `dataset`, if known. + + The operation returns the cardinality of `dataset`. The operation may return + `tf.data.experimental.INFINITE_CARDINALITY` if `dataset` contains an infinite + number of elements or `tf.data.experimental.UNKNOWN_CARDINALITY` if the + analysis fails to determine the number of elements in `dataset` (e.g. when the + dataset source is a file). + + Args: + dataset: A `tf.data.Dataset` for which to determine cardinality. + + Returns: + A scalar `tf.int64` `Tensor` representing the cardinality of `dataset`. If + the cardinality is infinite or unknown, the operation returns the named + constant `INFINITE_CARDINALITY` and `UNKNOWN_CARDINALITY` respectively. + """ + return ged_ops.experimental_dataset_cardinality(dataset._as_variant_tensor()) # pylint: disable=protected-access diff --git a/tensorflow/python/data/experimental/ops/counter.py b/tensorflow/python/data/experimental/ops/counter.py index 42200eaef9c..652eb9d0029 100644 --- a/tensorflow/python/data/experimental/ops/counter.py +++ b/tensorflow/python/data/experimental/ops/counter.py @@ -25,8 +25,8 @@ from tensorflow.python.framework import ops from tensorflow.python.util.tf_export import tf_export -@tf_export("data.experimental.Counter") -def Counter(start=0, step=1, dtype=dtypes.int64): +@tf_export("data.experimental.Counter", v1=[]) +def CounterV2(start=0, step=1, dtype=dtypes.int64): """Creates a `Dataset` that counts from `start` in steps of size `step`. For example: @@ -53,3 +53,13 @@ def Counter(start=0, step=1, dtype=dtypes.int64): step = ops.convert_to_tensor(step, dtype=dtype, name="step") return dataset_ops.Dataset.from_tensors(0).repeat(None).apply( scan_ops.scan(start, lambda state, _: (state + step, state))) + + +@tf_export(v1=["data.experimental.Counter"]) +def CounterV1(start=0, step=1, dtype=dtypes.int64): + return dataset_ops.DatasetV1Adapter(CounterV2(start, step, dtype)) +CounterV1.__doc__ = CounterV2.__doc__ + +# TODO(b/119044825): Until all `tf.data` unit tests are converted to V2, keep +# this alias in place. +Counter = CounterV1 # pylint: disable=invalid-name diff --git a/tensorflow/python/data/experimental/ops/enumerate_ops.py b/tensorflow/python/data/experimental/ops/enumerate_ops.py index a1af98f552c..f38cab12a76 100644 --- a/tensorflow/python/data/experimental/ops/enumerate_ops.py +++ b/tensorflow/python/data/experimental/ops/enumerate_ops.py @@ -26,9 +26,9 @@ from tensorflow.python.util.tf_export import tf_export @tf_export("data.experimental.enumerate_dataset") def enumerate_dataset(start=0): - """A transformation that enumerate the elements of a dataset. + """A transformation that enumerates the elements of a dataset. - It is Similar to python's `enumerate`. + It is similar to python's `enumerate`. For example: ```python diff --git a/tensorflow/python/data/experimental/ops/error_ops.py b/tensorflow/python/data/experimental/ops/error_ops.py index 82e274b70c5..879b13ce092 100644 --- a/tensorflow/python/data/experimental/ops/error_ops.py +++ b/tensorflow/python/data/experimental/ops/error_ops.py @@ -52,7 +52,7 @@ def ignore_errors(): return _apply_fn -class _IgnoreErrorsDataset(dataset_ops.UnaryDataset): +class _IgnoreErrorsDataset(dataset_ops.UnaryUnchangedStructureDataset): """A `Dataset` that silently ignores errors when computing its input.""" def __init__(self, input_dataset): @@ -64,15 +64,3 @@ class _IgnoreErrorsDataset(dataset_ops.UnaryDataset): return gen_experimental_dataset_ops.experimental_ignore_errors_dataset( self._input_dataset._as_variant_tensor(), # pylint: disable=protected-access **dataset_ops.flat_structure(self)) - - @property - def output_classes(self): - return self._input_dataset.output_classes - - @property - def output_shapes(self): - return self._input_dataset.output_shapes - - @property - def output_types(self): - return self._input_dataset.output_types diff --git a/tensorflow/python/data/experimental/ops/filter_for_shard_ops.py b/tensorflow/python/data/experimental/ops/filter_for_shard_ops.py new file mode 100644 index 00000000000..91d3dca3e9a --- /dev/null +++ b/tensorflow/python/data/experimental/ops/filter_for_shard_ops.py @@ -0,0 +1,106 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Naive shard dataset transformation.""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from tensorflow.python.framework import dtypes +from tensorflow.python.framework import ops +from tensorflow.python.framework import tensor_util +from tensorflow.python.ops import math_ops +from tensorflow.python.util.tf_export import tf_export + + +@tf_export("data.experimental.filter_for_shard") +def filter_for_shard(num_shards, shard_index): + """Creates a `Dataset` that includes only 1/`num_shards` of this dataset. + + This dataset operator is very useful when running distributed training, as + it allows each worker to read a unique subset. + + When reading a single input file, you can skip elements as follows: + + ```python + d = tf.data.TFRecordDataset(FLAGS.input_file) + d = d.apply(tf.data.experimental.naive_shard(FLAGS.num_workers, + FLAGS.worker_index)) + d = d.repeat(FLAGS.num_epochs) + d = d.shuffle(FLAGS.shuffle_buffer_size) + d = d.map(parser_fn, num_parallel_calls=FLAGS.num_map_threads) + ``` + + Important caveats: + + - Be sure to shard before you use any randomizing operator (such as + shuffle). + - Generally it is best if the shard operator is used early in the dataset + pipeline. For example, when reading from a set of TFRecord files, shard + before converting the dataset to input samples. This avoids reading every + file on every worker. The following is an example of an efficient + sharding strategy within a complete pipeline: + + ```python + d = Dataset.list_files(FLAGS.pattern) + d = d.apply(tf.data.experimental.naive_shard(FLAGS.num_workers, + FLAGS.worker_index)) + d = d.repeat(FLAGS.num_epochs) + d = d.shuffle(FLAGS.shuffle_buffer_size) + d = d.interleave(tf.data.TFRecordDataset, + cycle_length=FLAGS.num_readers, block_length=1) + d = d.map(parser_fn, num_parallel_calls=FLAGS.num_map_threads) + ``` + + Args: + num_shards: A `tf.int64` scalar `tf.Tensor`, representing the number of + shards operating in parallel. + shard_index: A `tf.int64` scalar `tf.Tensor`, representing the worker index. + + Returns: + A `Dataset` transformation function, which can be passed to + `tf.data.Dataset.apply`. + + Raises: + ValueError: if `num_shards` or `shard_index` are illegal values. Note: error + checking is done on a best-effort basis, and errors aren't guaranteed to + be caught upon dataset creation. (e.g. providing in a placeholder tensor + bypasses the early checking, and will instead result in an error during + a session.run call.) + """ + num_shards = ops.convert_to_tensor( + num_shards, name="num_shards", dtype=dtypes.int64) + num_shards_static = tensor_util.constant_value(num_shards) + shard_index = ops.convert_to_tensor(shard_index, name="shard_index", + dtype=dtypes.int64) + shard_index_static = tensor_util.constant_value(shard_index) + + if num_shards_static is not None and num_shards_static < 1: + raise ValueError("num_shards must be >= 1; got: %s" % num_shards_static) + if shard_index_static is not None and shard_index_static < 0: + raise ValueError("shard_index must be >= 0; got: %s" % shard_index_static) + if (shard_index_static is not None and num_shards_static is not None and + shard_index_static >= num_shards_static): + raise ValueError("shard_index must be < num_shards; %s is not < %s" % + (shard_index_static, num_shards_static)) + + def filter_fn(elem_index, _): + mod_result = math_ops.mod(elem_index, num_shards) + return math_ops.equal(mod_result, shard_index) + + def _apply_fn(dataset): + # pylint: disable=protected-access + return dataset._enumerate().filter(filter_fn).map(lambda _, elem: elem) + + return _apply_fn diff --git a/tensorflow/python/data/experimental/ops/get_single_element.py b/tensorflow/python/data/experimental/ops/get_single_element.py index 132526166cf..73116edf128 100644 --- a/tensorflow/python/data/experimental/ops/get_single_element.py +++ b/tensorflow/python/data/experimental/ops/get_single_element.py @@ -60,7 +60,7 @@ def get_single_element(dataset): InvalidArgumentError (at runtime): if `dataset` does not contain exactly one element. """ - if not isinstance(dataset, dataset_ops.Dataset): + if not isinstance(dataset, dataset_ops.DatasetV2): raise TypeError("`dataset` must be a `tf.data.Dataset` object.") nested_ret = nest.pack_sequence_as( diff --git a/tensorflow/python/data/experimental/ops/grouping.py b/tensorflow/python/data/experimental/ops/grouping.py index 026867d405f..ad9fe9a4e89 100644 --- a/tensorflow/python/data/experimental/ops/grouping.py +++ b/tensorflow/python/data/experimental/ops/grouping.py @@ -21,13 +21,14 @@ import numpy as np from tensorflow.python.data.ops import dataset_ops from tensorflow.python.data.util import nest +from tensorflow.python.data.util import structure from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops from tensorflow.python.framework import tensor_shape from tensorflow.python.ops import array_ops from tensorflow.python.ops import check_ops -from tensorflow.python.ops import gen_dataset_ops +from tensorflow.python.ops import gen_experimental_dataset_ops as ged_ops from tensorflow.python.ops import math_ops from tensorflow.python.util.tf_export import tf_export @@ -236,29 +237,6 @@ def bucket_by_sequence_length(element_length_func, return _apply_fn -def _map_x_dataset(map_func): - """A transformation that maps `map_func` across its input. - - This transformation is similar to `tf.data.Dataset.map`, but in addition to - supporting dense and sparse tensor inputs, it also supports dataset inputs. - - Args: - map_func: A function mapping a nested structure of tensors and/or datasets - (having shapes and types defined by `self.output_shapes` and - `self.output_types`) to another nested structure of tensors and/or - datasets. - - Returns: - Dataset: A `Dataset`. - """ - - def _apply_fn(dataset): - """Function from `Dataset` to `Dataset` that applies the transformation.""" - return _MapXDataset(dataset, map_func) - - return _apply_fn - - class _GroupByReducerDataset(dataset_ops.UnaryDataset): """A `Dataset` that groups its input and performs a reduction.""" @@ -284,7 +262,7 @@ class _GroupByReducerDataset(dataset_ops.UnaryDataset): "`key_func` must return a single tf.int64 tensor. " "Got type=%s and shape=%s" % (wrapped_func.output_types, wrapped_func.output_shapes)) - self._key_func = wrapped_func.function + self._key_func = wrapped_func def _make_init_func(self, init_func): """Make wrapping defun for init_func.""" @@ -294,7 +272,7 @@ class _GroupByReducerDataset(dataset_ops.UnaryDataset): input_classes=ops.Tensor, input_shapes=tensor_shape.scalar(), input_types=dtypes.int64) - self._init_func = wrapped_func.function + self._init_func = wrapped_func self._state_classes = wrapped_func.output_classes self._state_shapes = wrapped_func.output_shapes self._state_types = wrapped_func.output_types @@ -356,8 +334,8 @@ class _GroupByReducerDataset(dataset_ops.UnaryDataset): self._state_shapes = nest.pack_sequence_as(self._state_shapes, weakened_state_shapes) - self._reduce_func = wrapped_func.function - self._reduce_func.add_to_graph(ops.get_default_graph()) + self._reduce_func = wrapped_func + self._reduce_func.function.add_to_graph(ops.get_default_graph()) def _make_finalize_func(self, finalize_func): """Make wrapping defun for finalize_func.""" @@ -367,7 +345,7 @@ class _GroupByReducerDataset(dataset_ops.UnaryDataset): input_classes=self._state_classes, input_shapes=self._state_shapes, input_types=self._state_types) - self._finalize_func = wrapped_func.function + self._finalize_func = wrapped_func self._output_classes = wrapped_func.output_classes self._output_shapes = wrapped_func.output_shapes self._output_types = wrapped_func.output_types @@ -384,17 +362,22 @@ class _GroupByReducerDataset(dataset_ops.UnaryDataset): def output_types(self): return self._output_types + def _functions(self): + return [ + self._key_func, self._init_func, self._reduce_func, self._finalize_func + ] + def _as_variant_tensor(self): - return gen_dataset_ops.group_by_reducer_dataset( + return ged_ops.experimental_group_by_reducer_dataset( self._input_dataset._as_variant_tensor(), # pylint: disable=protected-access - self._key_func.captured_inputs, - self._init_func.captured_inputs, - self._reduce_func.captured_inputs, - self._finalize_func.captured_inputs, - key_func=self._key_func, - init_func=self._init_func, - reduce_func=self._reduce_func, - finalize_func=self._finalize_func, + self._key_func.function.captured_inputs, + self._init_func.function.captured_inputs, + self._reduce_func.function.captured_inputs, + self._finalize_func.function.captured_inputs, + key_func=self._key_func.function, + init_func=self._init_func.function, + reduce_func=self._reduce_func.function, + finalize_func=self._finalize_func.function, **dataset_ops.flat_structure(self)) def _transformation_name(self): @@ -430,7 +413,7 @@ class _GroupByWindowDataset(dataset_ops.UnaryDataset): wrapped_func.output_shapes.is_compatible_with(tensor_shape.scalar())): raise ValueError( "`window_size_func` must return a single tf.int64 scalar tensor.") - self._window_size_func = wrapped_func.function + self._window_size_func = wrapped_func def _make_key_func(self, key_func, input_dataset): """Make wrapping defun for key_func.""" @@ -444,25 +427,29 @@ class _GroupByWindowDataset(dataset_ops.UnaryDataset): wrapped_func.output_shapes.is_compatible_with(tensor_shape.scalar())): raise ValueError( "`key_func` must return a single tf.int64 scalar tensor.") - self._key_func = wrapped_func.function + self._key_func = wrapped_func def _make_reduce_func(self, reduce_func, input_dataset): """Make wrapping defun for reduce_func.""" - nested_dataset = dataset_ops._NestedDatasetComponent(input_dataset) # pylint: disable=protected-access + nested_dataset = dataset_ops.DatasetStructure( + structure.Structure._from_legacy_structure( # pylint: disable=protected-access + input_dataset.output_types, input_dataset.output_shapes, + input_dataset.output_classes)) wrapped_func = dataset_ops.StructuredFunctionWrapper( reduce_func, self._transformation_name(), input_classes=(ops.Tensor, nested_dataset), input_shapes=(tensor_shape.scalar(), nested_dataset), - input_types=(dtypes.int64, nested_dataset), - experimental_nested_dataset_support=True) + input_types=(dtypes.int64, nested_dataset)) if not isinstance( - wrapped_func.output_classes, dataset_ops._NestedDatasetComponent): # pylint: disable=protected-access + wrapped_func.output_structure, dataset_ops.DatasetStructure): raise TypeError("`reduce_func` must return a `Dataset` object.") - self._output_classes = wrapped_func.output_classes.output_classes - self._output_types = wrapped_func.output_types.output_types - self._output_shapes = wrapped_func.output_shapes.output_shapes - self._reduce_func = wrapped_func.function + # pylint: disable=protected-access + element_structure = wrapped_func.output_structure._element_structure + self._output_classes = element_structure._to_legacy_output_classes() + self._output_types = element_structure._to_legacy_output_types() + self._output_shapes = element_structure._to_legacy_output_shapes() + self._reduce_func = wrapped_func @property def output_classes(self): @@ -476,15 +463,18 @@ class _GroupByWindowDataset(dataset_ops.UnaryDataset): def output_types(self): return self._output_types + def _functions(self): + return [self._key_func, self._reduce_func, self._window_size_func] + def _as_variant_tensor(self): - return gen_dataset_ops.group_by_window_dataset( + return ged_ops.experimental_group_by_window_dataset( self._input_dataset._as_variant_tensor(), # pylint: disable=protected-access - self._key_func.captured_inputs, - self._reduce_func.captured_inputs, - self._window_size_func.captured_inputs, - key_func=self._key_func, - reduce_func=self._reduce_func, - window_size_func=self._window_size_func, + self._key_func.function.captured_inputs, + self._reduce_func.function.captured_inputs, + self._window_size_func.function.captured_inputs, + key_func=self._key_func.function, + reduce_func=self._reduce_func.function, + window_size_func=self._window_size_func.function, **dataset_ops.flat_structure(self)) def _transformation_name(self): @@ -517,45 +507,3 @@ class Reducer(object): @property def finalize_func(self): return self._finalize_func - - -class _MapXDataset(dataset_ops.UnaryDataset): - """A `Dataset` that maps a function over elements in its input.""" - - def __init__(self, input_dataset, map_func): - """See `map_x_dataset()` for details.""" - super(_MapXDataset, self).__init__(input_dataset) - self._input_dataset = input_dataset - - wrapped_func = dataset_ops.StructuredFunctionWrapper( - map_func, - self._transformation_name(), - dataset=input_dataset, - experimental_nested_dataset_support=True) - self._output_classes = wrapped_func.output_classes - self._output_shapes = wrapped_func.output_shapes - self._output_types = wrapped_func.output_types - self._map_func = wrapped_func.function - - def _as_variant_tensor(self): - input_t = self._input_dataset._as_variant_tensor() # pylint: disable=protected-access - return gen_dataset_ops.map_dataset( - input_t, - self._map_func.captured_inputs, - f=self._map_func, - **dataset_ops.flat_structure(self)) - - @property - def output_classes(self): - return self._output_classes - - @property - def output_shapes(self): - return self._output_shapes - - @property - def output_types(self): - return self._output_types - - def _transformation_name(self): - return "tf.data.experimental.map_x_dataset()" diff --git a/tensorflow/python/data/experimental/ops/indexed_dataset_ops.py b/tensorflow/python/data/experimental/ops/indexed_dataset_ops.py index 9c06474a2f8..570f0116f76 100644 --- a/tensorflow/python/data/experimental/ops/indexed_dataset_ops.py +++ b/tensorflow/python/data/experimental/ops/indexed_dataset_ops.py @@ -65,6 +65,7 @@ class MaterializedIndexedDataset(object): sparse.as_dense_types(self._output_shapes, self._output_classes))) +# TODO(saeta): Add a `DatasetV1` wrapper if this is exposed via the public API. class IndexedDataset(dataset_ops.Dataset): """IndexedDataset is highly experimental! """ @@ -149,6 +150,7 @@ class IndexedDataset(dataset_ops.Dataset): raise NotImplementedError("IndexedDataset._as_variant_tensor") +# TODO(saeta): Add a `DatasetV1` wrapper if this is exposed via the public API. class IdentityIndexedDataset(IndexedDataset): """IdentityIndexedDataset is a trivial indexed dataset used for testing. """ diff --git a/tensorflow/python/data/experimental/ops/interleave_ops.py b/tensorflow/python/data/experimental/ops/interleave_ops.py index a3c094859ef..8b0fdfce11d 100644 --- a/tensorflow/python/data/experimental/ops/interleave_ops.py +++ b/tensorflow/python/data/experimental/ops/interleave_ops.py @@ -133,8 +133,8 @@ class _DirectedInterleaveDataset(dataset_ops.Dataset): return self._data_inputs[0].output_types -@tf_export("data.experimental.sample_from_datasets") -def sample_from_datasets(datasets, weights=None, seed=None): +@tf_export("data.experimental.sample_from_datasets", v1=[]) +def sample_from_datasets_v2(datasets, weights=None, seed=None): """Samples elements at random from the datasets in `datasets`. Args: @@ -158,7 +158,7 @@ def sample_from_datasets(datasets, weights=None, seed=None): length of the `datasets` element. """ num_datasets = len(datasets) - if not isinstance(weights, dataset_ops.Dataset): + if not isinstance(weights, dataset_ops.DatasetV2): if weights is None: # Select inputs with uniform probability. logits = [[1.0] * num_datasets] @@ -217,8 +217,15 @@ def sample_from_datasets(datasets, weights=None, seed=None): return _DirectedInterleaveDataset(selector_input, datasets) -@tf_export("data.experimental.choose_from_datasets") -def choose_from_datasets(datasets, choice_dataset): +@tf_export(v1=["data.experimental.sample_from_datasets"]) +def sample_from_datasets_v1(datasets, weights=None, seed=None): + return dataset_ops.DatasetV1Adapter( + sample_from_datasets_v2(datasets, weights, seed)) +sample_from_datasets_v1.__doc__ = sample_from_datasets_v2.__doc__ + + +@tf_export("data.experimental.choose_from_datasets", v1=[]) +def choose_from_datasets_v2(datasets, choice_dataset): """Creates a dataset that deterministically chooses elements from `datasets`. For example, given the following datasets: @@ -260,3 +267,16 @@ def choose_from_datasets(datasets, choice_dataset): raise TypeError("`choice_dataset` must be a dataset of scalar " "`tf.int64` tensors.") return _DirectedInterleaveDataset(choice_dataset, datasets) + + +@tf_export(v1=["data.experimental.choose_from_datasets"]) +def choose_from_datasets_v1(datasets, choice_dataset): + return dataset_ops.DatasetV1Adapter( + choose_from_datasets_v2(datasets, choice_dataset)) +choose_from_datasets_v1.__doc__ = choose_from_datasets_v2.__doc__ + + +# TODO(b/119044825): Until all `tf.data` unit tests are converted to V2, keep +# these aliases in place. +choose_from_datasets = choose_from_datasets_v1 +sample_from_datasets = sample_from_datasets_v1 diff --git a/tensorflow/python/data/experimental/ops/matching_files.py b/tensorflow/python/data/experimental/ops/matching_files.py new file mode 100644 index 00000000000..8398f86e31c --- /dev/null +++ b/tensorflow/python/data/experimental/ops/matching_files.py @@ -0,0 +1,51 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Experimental API for matching input filenames.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from tensorflow.python.data.ops import dataset_ops +from tensorflow.python.framework import dtypes +from tensorflow.python.framework import ops +from tensorflow.python.framework import tensor_shape +from tensorflow.python.ops import gen_experimental_dataset_ops as ged_ops + + +class MatchingFilesDataset(dataset_ops.DatasetSource): + """A `Dataset` that list the files according to the input patterns.""" + + def __init__(self, patterns): + super(MatchingFilesDataset, self).__init__() + self._patterns = ops.convert_to_tensor( + patterns, dtype=dtypes.string, name="patterns") + + def _as_variant_tensor(self): + return ged_ops.experimental_matching_files_dataset(self._patterns) + + @property + def output_classes(self): + return ops.Tensor + + @property + def output_shapes(self): + return tensor_shape.scalar() + + @property + def output_types(self): + return dtypes.string + + diff --git a/tensorflow/python/data/experimental/ops/optimization.py b/tensorflow/python/data/experimental/ops/optimization.py index b744db7f1e5..c6c7de9265c 100644 --- a/tensorflow/python/data/experimental/ops/optimization.py +++ b/tensorflow/python/data/experimental/ops/optimization.py @@ -100,7 +100,7 @@ def optimize(optimizations=None): return _apply_fn -class _AssertNextDataset(dataset_ops.UnaryDataset): +class _AssertNextDataset(dataset_ops.UnaryUnchangedStructureDataset): """A `Dataset` that asserts which transformations happen next.""" def __init__(self, input_dataset, transformations): @@ -118,20 +118,8 @@ class _AssertNextDataset(dataset_ops.UnaryDataset): self._transformations, **dataset_ops.flat_structure(self)) - @property - def output_classes(self): - return self._input_dataset.output_classes - @property - def output_shapes(self): - return self._input_dataset.output_shapes - - @property - def output_types(self): - return self._input_dataset.output_types - - -class _NonSerializableDataset(dataset_ops.UnaryDataset): +class _NonSerializableDataset(dataset_ops.UnaryUnchangedStructureDataset): """A `Dataset` that performs non-serializable identity transformation.""" def __init__(self, input_dataset): @@ -143,15 +131,3 @@ class _NonSerializableDataset(dataset_ops.UnaryDataset): return gen_experimental_dataset_ops.experimental_non_serializable_dataset( self._input_dataset._as_variant_tensor(), # pylint: disable=protected-access **dataset_ops.flat_structure(self)) - - @property - def output_classes(self): - return self._input_dataset.output_classes - - @property - def output_shapes(self): - return self._input_dataset.output_shapes - - @property - def output_types(self): - return self._input_dataset.output_types diff --git a/tensorflow/python/data/experimental/ops/optimization_options.py b/tensorflow/python/data/experimental/ops/optimization_options.py new file mode 100644 index 00000000000..dc9d3193748 --- /dev/null +++ b/tensorflow/python/data/experimental/ops/optimization_options.py @@ -0,0 +1,83 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Experimental API for controlling optimizations in `tf.data` pipelines.""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + + +from tensorflow.python.data.util import options +from tensorflow.python.util.tf_export import tf_export + + +@tf_export("data.experimental.OptimizationOptions") +class OptimizationOptions(options.OptionsBase): + """Represents options for dataset optimizations. + + You can apply `OptimizationOptions` to a `dataset` object, as follows: + + ```python + options = tf.data.Options() + options.optimization = tf.data.experimental.OptimizationOptions() + options.optimization.map_and_batch_fusion = True + dataset = dataset.with_options(options) + ``` + """ + + filter_fusion = options.create_option( + name="filter_fusion", + ty=bool, + docstring="Whether to fuse filter transformations.") + + hoist_random_uniform = options.create_option( + name="hoist_random_uniform", + ty=bool, + docstring= + "Whether to hoist `tf.random_uniform()` ops out of map transformations.") + + map_and_batch_fusion = options.create_option( + name="map_and_batch_fusion", + ty=bool, + docstring="Whether to fuse map and batch transformations.") + + map_and_filter_fusion = options.create_option( + name="map_and_filter_fusion", + ty=bool, + docstring="Whether to fuse map and filter transformations.") + + map_fusion = options.create_option( + name="map_and_filter_fusion", + ty=bool, + docstring="Whether to fuse map transformations.") + + map_parallelization = options.create_option( + name="map_parallelization", + ty=bool, + docstring="Whether to parallelize stateless map transformations.") + + map_vectorization = options.create_option( + name="map_vectorization", + ty=bool, + docstring="Whether to vectorize map transformations.") + + noop_elimination = options.create_option( + name="noop_elimination", + ty=bool, + docstring="Whether to eliminate no-op transformations.") + + shuffle_and_repeat_fusion = options.create_option( + name="shuffle_and_repeat_fusion", + ty=bool, + docstring="Whether to fuse shuffle and repeat transformations.") diff --git a/tensorflow/python/data/experimental/ops/parsing_ops.py b/tensorflow/python/data/experimental/ops/parsing_ops.py index 6615b9022a2..44cb0b8a2c2 100644 --- a/tensorflow/python/data/experimental/ops/parsing_ops.py +++ b/tensorflow/python/data/experimental/ops/parsing_ops.py @@ -22,7 +22,7 @@ from tensorflow.python.data.util import nest from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops from tensorflow.python.framework import sparse_tensor -from tensorflow.python.ops import gen_dataset_ops +from tensorflow.python.ops import gen_experimental_dataset_ops from tensorflow.python.ops import parsing_ops from tensorflow.python.util.tf_export import tf_export @@ -80,7 +80,7 @@ class _ParseExampleDataset(dataset_ops.UnaryDataset): ])) def _as_variant_tensor(self): - return gen_dataset_ops.parse_example_dataset( + return gen_experimental_dataset_ops.experimental_parse_example_dataset( self._input_dataset._as_variant_tensor(), # pylint: disable=protected-access self._num_parallel_calls, self._dense_defaults, @@ -138,10 +138,10 @@ def parse_example_dataset(features, num_parallel_calls=1): def _apply_fn(dataset): """Function from `Dataset` to `Dataset` that applies the transformation.""" out_dataset = _ParseExampleDataset(dataset, features, num_parallel_calls) - if any([ + if any( isinstance(feature, parsing_ops.SparseFeature) for _, feature in features.items() - ]): + ): # pylint: disable=protected-access # pylint: disable=g-long-lambda out_dataset = out_dataset.map( diff --git a/tensorflow/python/data/experimental/ops/prefetching_ops.py b/tensorflow/python/data/experimental/ops/prefetching_ops.py index d34f9f25bda..0894dd5f701 100644 --- a/tensorflow/python/data/experimental/ops/prefetching_ops.py +++ b/tensorflow/python/data/experimental/ops/prefetching_ops.py @@ -17,13 +17,10 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function -import warnings - from tensorflow.python.data.ops import dataset_ops from tensorflow.python.data.ops import iterator_ops from tensorflow.python.data.util import nest from tensorflow.python.data.util import sparse -from tensorflow.python.eager import context from tensorflow.python.eager import function from tensorflow.python.framework import device as framework_device from tensorflow.python.framework import dtypes @@ -37,304 +34,6 @@ from tensorflow.python.ops import resource_variable_ops from tensorflow.python.util.tf_export import tf_export -def function_buffering_resource(string_arg, - target_device, - f, - buffer_size, - output_types, - container="", - shared_name=None, - name=None): - """Creates a FunctionBufferingResource. - - A FunctionBufferingResource fills up a buffer by calling a function `f` on - `target_device`. `f` should take in only a single string argument as input. - - Args: - string_arg: The single string argument to the function. - target_device: The device to run `f` on. - f: The function to be executed. - buffer_size: Size of the buffer to be populated. - output_types: The output types generated by the function. - container: (Optional) string. Defaults to "". - shared_name: (Optional) string. - name: (Optional) string to name the op. - - Returns: - Handle to a FunctionBufferingResource. - """ - if shared_name is None: - shared_name = "" - return ged_ops.experimental_function_buffering_resource( - string_arg=string_arg, - target_device=target_device, - shared_name=shared_name, - f=f, - buffer_size=buffer_size, - container=container, - name=name, - output_types=output_types) - - -def function_buffering_resource_get_next(function_buffer_resource, - output_types, - name=None): - return ged_ops.experimental_function_buffering_resource_get_next( - function_buffer_resource=function_buffer_resource, - output_types=output_types, - name=name) - - -def function_buffering_resource_reset(function_buffer_resource, name=None): - return ged_ops.experimental_function_buffering_resource_reset( - function_buffer_resource=function_buffer_resource, name=name) - - -# pylint: disable=protected-access -class _PrefetchToDeviceIterator(object): - """A replacement for `tf.data.Iterator` that prefetches to another device. - - Args: - input_dataset: The input dataset - one_shot: If true, we make a one shot iterator that's already initialized. - device: A fully specified device string where we want to prefetch to - buffer_size: Size of the prefetching buffer. - shared_name: (Optional.) If non-empty, the returned iterator will be - shared under the given name across multiple sessions that share the - same devices (e.g. when using a remote server). - - Returns: - An Iterator type object. - """ - - def __init__(self, - input_dataset, - one_shot, - device, - buffer_size, - shared_name=None): - self._input_dataset = input_dataset - self._get_next_call_count = 0 - self._one_shot = one_shot - if shared_name is None: - shared_name = "" - - if self._one_shot: - self._input_iterator = input_dataset.make_one_shot_iterator() - else: - self._input_iterator = iterator_ops.Iterator.from_structure( - self._input_dataset.output_types, self._input_dataset.output_shapes, - shared_name, self._input_dataset.output_classes) - input_iterator_handle = self._input_iterator.string_handle() - - @function.defun(input_signature=[tensor_spec.TensorSpec([], dtypes.string)]) - # handle is a scalar `tf.Tensor` of type `tf.string` - def _prefetch_fn(handle): - """Prefetches one element from `input_iterator`.""" - remote_iterator = iterator_ops.Iterator.from_string_handle( - handle, self._input_iterator.output_types, - self._input_iterator.output_shapes, - self._input_iterator.output_classes) - ret = remote_iterator.get_next() - return nest.flatten(sparse.serialize_sparse_tensors(ret)) - - self._prefetch_fn = _prefetch_fn._get_concrete_function_internal() # pylint: disable=protected-access - - iterator_device = ged_ops.experimental_iterator_get_device( - self._input_iterator._iterator_resource) - - with ops.device(device): - self._buffering_resource = function_buffering_resource( - f=self._prefetch_fn, - target_device=iterator_device, - string_arg=input_iterator_handle, - buffer_size=buffer_size, - shared_name=shared_name, - output_types=nest.flatten( - sparse.as_dense_types(self._input_dataset.output_types, - self._input_dataset.output_classes))) - - if not self._one_shot: - reset_op = function_buffering_resource_reset(self._buffering_resource) - with ops.control_dependencies([reset_op]): - self._initializer = self._input_iterator.make_initializer( - self._input_dataset) - - def get_next(self, name=None): - """See `tf.data.Iterator.get_next`.""" - self._get_next_call_count += 1 - if self._get_next_call_count > iterator_ops.GET_NEXT_CALL_WARNING_THRESHOLD: - warnings.warn(iterator_ops.GET_NEXT_CALL_WARNING_MESSAGE) - - flat_ret = ged_ops.experimental_function_buffering_resource_get_next( - self._buffering_resource, - output_types=nest.flatten( - sparse.as_dense_types(self.output_types, self.output_classes)), - name=name) - - ret = sparse.deserialize_sparse_tensors( - nest.pack_sequence_as(self.output_types, flat_ret), - self.output_types, self.output_shapes, self.output_classes) - - for tensor, shape in zip( - nest.flatten(ret), nest.flatten(self.output_shapes)): - if isinstance(tensor, ops.Tensor): - tensor.set_shape(shape) - - return ret - - @property - def initializer(self): - if self._one_shot: - raise NotImplementedError("Can't initialize a one_shot_iterator") - return self._initializer - - @property - def output_classes(self): - return self._input_dataset.output_classes - - @property - def output_shapes(self): - return self._input_dataset.output_shapes - - @property - def output_types(self): - return self._input_dataset.output_types - - -class _PrefetchToDeviceEagerIterator(iterator_ops.EagerIterator): - """A replacement for `tf.data.Iterator` that prefetches to another device. - - Args: - input_dataset: The input dataset - one_shot: If true, we make a one shot iterator that's already initialized. - device: A fully specified device string where we want to prefetch to - buffer_size: Size of the prefetching buffer. - shared_name: (Optional.) If non-empty, the returned iterator will be - shared under the given name across multiple sessions that share the - same devices (e.g. when using a remote server). - - Returns: - An Iterator type object. - """ - - def __init__(self, - input_dataset, - device, - buffer_size): - with ops.device("/device:CPU:0"): - super(_PrefetchToDeviceEagerIterator, self).__init__(input_dataset) - input_iterator_handle = gen_dataset_ops.iterator_to_string_handle( - self._resource) - - self._device = device - - @function.defun(input_signature=[tensor_spec.TensorSpec([], dtypes.string)]) - def _prefetch_fn(handle): - """Prefetches one element from `input_iterator`.""" - remote_iterator = iterator_ops.Iterator.from_string_handle( - handle, self.output_types, self.output_shapes, self.output_classes) - ret = remote_iterator.get_next() - return nest.flatten(sparse.serialize_sparse_tensors(ret)) - - self._prefetch_fn = _prefetch_fn._get_concrete_function_internal() # pylint: disable=protected-access - - with ops.device(device): - self._buffering_resource = function_buffering_resource( - f=self._prefetch_fn, - output_types=self._flat_output_types, - target_device=ged_ops.experimental_iterator_get_device( - self._resource), - string_arg=input_iterator_handle, - buffer_size=buffer_size, - shared_name=iterator_ops._generate_shared_name( - "function_buffer_resource")) - - def _next_internal(self): - """Returns a nested structure of `tf.Tensor`s containing the next element. - """ - # This runs in sync mode as iterators use an error status to communicate - # that there is no more data to iterate over. - # TODO(b/77291417): Fix - with context.execution_mode(context.SYNC): - with ops.device(self._device): - flat_ret = ged_ops.experimental_function_buffering_resource_get_next( - function_buffer_resource=self._buffering_resource, - output_types=self._flat_output_types) - return self._element_structure._from_tensor_list(flat_ret) -# pylint: enable=protected-access - - -class _PrefetchToDeviceDataset(dataset_ops.UnaryDataset): - """A `Dataset` whose iterator prefetches elements to another device.""" - - def __init__(self, input_dataset, device, buffer_size): - super(_PrefetchToDeviceDataset, self).__init__(input_dataset) - self._input_dataset = input_dataset - self._device = device - self._buffer_size = buffer_size if buffer_size is not None else 1 - - # The static analysis cannot tell that the eager iterator's superclass has - # a `next()` method. - # pylint: disable=non-iterator-returned - def __iter__(self): - """Creates an `Iterator` for enumerating the elements of this dataset. - - The returned iterator implements the Python iterator protocol and therefore - can only be used in eager mode. - - Returns: - An `Iterator` over the elements of this dataset. - - Raises: - RuntimeError: If eager execution is enabled. - """ - if context.executing_eagerly(): - return _PrefetchToDeviceEagerIterator(self._input_dataset, self._device, - self._buffer_size) - else: - raise RuntimeError("dataset.__iter__() is only supported when eager " - "execution is enabled.") - # pylint: enable=non-iterator-returned - - def make_one_shot_iterator(self): - if context.executing_eagerly(): - return _PrefetchToDeviceEagerIterator(self._input_dataset, self._device, - self._buffer_size) - else: - return _PrefetchToDeviceIterator(self._input_dataset, one_shot=True, - device=self._device, - buffer_size=self._buffer_size) - - def make_initializable_iterator(self, shared_name=None): - return _PrefetchToDeviceIterator( - self._input_dataset, - one_shot=False, - device=self._device, - buffer_size=self._buffer_size, - shared_name=shared_name) - - def _as_variant_tensor(self): - # TODO(mrry): Raise this error earlier (e.g. when one of the Dataset - # transformation methods is called. - # TODO(mrry): Investigate support for chaining further transformations after - # the prefetch, including GPU support. - raise NotImplementedError("`prefetch_to_device()` must be the last " - "transformation in a dataset pipeline.") - - @property - def output_types(self): - return self._input_dataset.output_types - - @property - def output_shapes(self): - return self._input_dataset.output_shapes - - @property - def output_classes(self): - return self._input_dataset.output_classes - - @tf_export("data.experimental.prefetch_to_device") def prefetch_to_device(device, buffer_size=None): """A transformation that prefetches dataset values to the given `device`. @@ -352,7 +51,8 @@ def prefetch_to_device(device, buffer_size=None): `tf.data.Dataset.apply`. """ def _apply_fn(dataset): - return _PrefetchToDeviceDataset(dataset, device, buffer_size) + return dataset.apply( + copy_to_device(target_device=device)).prefetch(buffer_size) return _apply_fn @@ -371,8 +71,11 @@ def copy_to_device(target_device, source_device="/cpu:0"): """ def _apply_fn(dataset): + options = dataset_ops.Options() + options.experimental_autotune = False return _CopyToDeviceDataset( - dataset, target_device=target_device, source_device=source_device) + dataset, target_device=target_device, + source_device=source_device).with_options(options) return _apply_fn @@ -380,7 +83,7 @@ def copy_to_device(target_device, source_device="/cpu:0"): # TODO(rohanj): Use the _input_hostmem attr on the RemoteCall ops to indicate # all inputs to the Op are in host memory, thereby avoiding some unnecessary # Sends and Recvs. -class _CopyToDeviceDataset(dataset_ops.UnaryDataset): +class _CopyToDeviceDataset(dataset_ops.UnaryUnchangedStructureDataset): """A `Dataset` that copies elements to another device.""" def __init__(self, input_dataset, target_device, source_device="/cpu:0"): @@ -529,18 +232,6 @@ class _CopyToDeviceDataset(dataset_ops.UnaryDataset): output_types=self._flat_output_types, output_shapes=self._flat_output_shapes) - @property - def output_types(self): - return self._input_dataset.output_types - - @property - def output_shapes(self): - return self._input_dataset.output_shapes - - @property - def output_classes(self): - return self._input_dataset.output_classes - class _MapOnGpuDataset(dataset_ops.UnaryDataset): """A `Dataset` that maps a function over elements in its using a GPU.""" @@ -551,36 +242,35 @@ class _MapOnGpuDataset(dataset_ops.UnaryDataset): self._input_dataset = input_dataset self._use_inter_op_parallelism = use_inter_op_parallelism - wrapped_func = dataset_ops.StructuredFunctionWrapper( + self._map_func = dataset_ops.StructuredFunctionWrapper( map_func, self._transformation_name(), dataset=input_dataset, defun_kwargs={"experimental_ints_on_device": True}) - self._output_classes = wrapped_func.output_classes - self._output_shapes = wrapped_func.output_shapes - self._output_types = wrapped_func.output_types - self._map_func = wrapped_func.function + + def _functions(self): + return [self._map_func] def _as_variant_tensor(self): input_t = self._input_dataset._as_variant_tensor() # pylint: disable=protected-access return ged_ops.experimental_map_dataset( input_t, - self._map_func.captured_inputs, - f=self._map_func, + self._map_func.function.captured_inputs, + f=self._map_func.function, use_inter_op_parallelism=self._use_inter_op_parallelism, **dataset_ops.flat_structure(self)) @property def output_classes(self): - return self._output_classes + return self._map_func.output_classes @property def output_shapes(self): - return self._output_shapes + return self._map_func.output_shapes @property def output_types(self): - return self._output_types + return self._map_func.output_types def _transformation_name(self): return "map_on_gpu()" diff --git a/tensorflow/python/data/experimental/ops/random_ops.py b/tensorflow/python/data/experimental/ops/random_ops.py index e3a2aeab31e..a4359f356b1 100644 --- a/tensorflow/python/data/experimental/ops/random_ops.py +++ b/tensorflow/python/data/experimental/ops/random_ops.py @@ -17,26 +17,28 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function +import functools + from tensorflow.python.data.ops import dataset_ops from tensorflow.python.data.util import random_seed from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops from tensorflow.python.framework import tensor_shape -from tensorflow.python.ops import gen_dataset_ops +from tensorflow.python.ops import gen_experimental_dataset_ops from tensorflow.python.util.tf_export import tf_export -@tf_export("data.experimental.RandomDataset") -class RandomDataset(dataset_ops.DatasetSource): +@tf_export("data.experimental.RandomDataset", v1=[]) +class RandomDatasetV2(dataset_ops.DatasetSource): """A `Dataset` of pseudorandom values.""" def __init__(self, seed=None): """A `Dataset` of pseudorandom values.""" - super(RandomDataset, self).__init__() + super(RandomDatasetV2, self).__init__() self._seed, self._seed2 = random_seed.get_seed(seed) def _as_variant_tensor(self): - return gen_dataset_ops.random_dataset( + return gen_experimental_dataset_ops.experimental_random_dataset( seed=self._seed, seed2=self._seed2, **dataset_ops.flat_structure(self)) @@ -52,3 +54,18 @@ class RandomDataset(dataset_ops.DatasetSource): @property def output_types(self): return dtypes.int64 + + +@tf_export(v1=["data.experimental.RandomDataset"]) +class RandomDatasetV1(dataset_ops.DatasetV1Adapter): + """A `Dataset` of pseudorandom values.""" + + @functools.wraps(RandomDatasetV2.__init__) + def __init__(self, seed=None): + wrapped = RandomDatasetV2(seed) + super(RandomDatasetV1, self).__init__(wrapped) + + +# TODO(b/119044825): Until all `tf.data` unit tests are converted to V2, keep +# this alias in place. +RandomDataset = RandomDatasetV1 diff --git a/tensorflow/python/data/experimental/ops/readers.py b/tensorflow/python/data/experimental/ops/readers.py index fe601925860..a4c260dde78 100644 --- a/tensorflow/python/data/experimental/ops/readers.py +++ b/tensorflow/python/data/experimental/ops/readers.py @@ -19,6 +19,7 @@ from __future__ import print_function import collections import csv +import functools import numpy as np @@ -36,7 +37,6 @@ from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops from tensorflow.python.framework import tensor_shape from tensorflow.python.lib.io import file_io -from tensorflow.python.ops import gen_dataset_ops from tensorflow.python.ops import gen_experimental_dataset_ops from tensorflow.python.ops import io_ops from tensorflow.python.platform import gfile @@ -307,8 +307,8 @@ def make_tf_record_dataset(file_pattern, return dataset.prefetch(buffer_size=prefetch_buffer_size) -@tf_export("data.experimental.make_csv_dataset") -def make_csv_dataset( +@tf_export("data.experimental.make_csv_dataset", v1=[]) +def make_csv_dataset_v2( file_pattern, batch_size, column_names=None, @@ -507,11 +507,42 @@ def make_csv_dataset( return dataset +@tf_export(v1=["data.experimental.make_csv_dataset"]) +def make_csv_dataset_v1( + file_pattern, + batch_size, + column_names=None, + column_defaults=None, + label_name=None, + select_columns=None, + field_delim=",", + use_quote_delim=True, + na_value="", + header=True, + num_epochs=None, + shuffle=True, + shuffle_buffer_size=10000, + shuffle_seed=None, + prefetch_buffer_size=optimization.AUTOTUNE, + num_parallel_reads=1, + sloppy=False, + num_rows_for_inference=100, + compression_type=None, +): # pylint: disable=missing-docstring + return dataset_ops.DatasetV1Adapter(make_csv_dataset_v2( + file_pattern, batch_size, column_names, column_defaults, label_name, + select_columns, field_delim, use_quote_delim, na_value, header, + num_epochs, shuffle, shuffle_buffer_size, shuffle_seed, + prefetch_buffer_size, num_parallel_reads, sloppy, num_rows_for_inference, + compression_type)) +make_csv_dataset_v1.__doc__ = make_csv_dataset_v2.__doc__ + + _DEFAULT_READER_BUFFER_SIZE_BYTES = 4 * 1024 * 1024 # 4 MB -@tf_export("data.experimental.CsvDataset") -class CsvDataset(dataset_ops.DatasetSource): +@tf_export("data.experimental.CsvDataset", v1=[]) +class CsvDatasetV2(dataset_ops.DatasetSource): """A Dataset comprising lines from one or more CSV files.""" def __init__(self, @@ -541,7 +572,9 @@ class CsvDataset(dataset_ops.DatasetSource): We can construct a CsvDataset from it as follows: ```python - dataset = tf.data.experimental.CsvDataset( + tf.enable_eager_execution() + + dataset = tf.data.experimental.CsvDataset( "my_file*.csv", [tf.float32, # Required field, use dtype or empty tensor tf.constant([0.0], dtype=tf.float32), # Optional field, default to 0.0 @@ -553,13 +586,8 @@ class CsvDataset(dataset_ops.DatasetSource): The expected output of its iterations is: ```python - next_element = dataset.make_one_shot_iterator().get_next() - with tf.Session() as sess: - while True: - try: - print(sess.run(next_element)) - except tf.errors.OutOfRangeError: - break + for element in dataset: + print(element) >> (4.28e10, 5.55e6, 12) >> (-5.3e14, 0.0, 2) @@ -594,7 +622,7 @@ class CsvDataset(dataset_ops.DatasetSource): the input data. If specified, only this subset of columns will be parsed. Defaults to parsing all columns. """ - super(CsvDataset, self).__init__() + super(CsvDatasetV2, self).__init__() self._filenames = ops.convert_to_tensor( filenames, dtype=dtypes.string, name="filenames") self._compression_type = convert.optional_param_to_tensor( @@ -658,22 +686,43 @@ class CsvDataset(dataset_ops.DatasetSource): return self._output_classes -@tf_export("data.experimental.make_batched_features_dataset") -def make_batched_features_dataset(file_pattern, - batch_size, - features, - reader=core_readers.TFRecordDataset, - label_key=None, - reader_args=None, - num_epochs=None, - shuffle=True, - shuffle_buffer_size=10000, - shuffle_seed=None, - prefetch_buffer_size=optimization.AUTOTUNE, - reader_num_threads=1, - parser_num_threads=2, - sloppy_ordering=False, - drop_final_batch=False): +@tf_export(v1=["data.experimental.CsvDataset"]) +class CsvDatasetV1(dataset_ops.DatasetV1Adapter): + """A Dataset comprising lines from one or more CSV files.""" + + @functools.wraps(CsvDatasetV2.__init__) + def __init__(self, + filenames, + record_defaults, + compression_type=None, + buffer_size=None, + header=False, + field_delim=",", + use_quote_delim=True, + na_value="", + select_cols=None): + wrapped = CsvDatasetV2(filenames, record_defaults, compression_type, + buffer_size, header, field_delim, use_quote_delim, + na_value, select_cols) + super(CsvDatasetV1, self).__init__(wrapped) + + +@tf_export("data.experimental.make_batched_features_dataset", v1=[]) +def make_batched_features_dataset_v2(file_pattern, + batch_size, + features, + reader=core_readers.TFRecordDataset, + label_key=None, + reader_args=None, + num_epochs=None, + shuffle=True, + shuffle_buffer_size=10000, + shuffle_seed=None, + prefetch_buffer_size=optimization.AUTOTUNE, + reader_num_threads=1, + parser_num_threads=2, + sloppy_ordering=False, + drop_final_batch=False): """Returns a `Dataset` of feature dictionaries from `Example` protos. If label_key argument is provided, returns a `Dataset` of tuple @@ -819,6 +868,31 @@ def make_batched_features_dataset(file_pattern, return dataset +@tf_export(v1=["data.experimental.make_batched_features_dataset"]) +def make_batched_features_dataset_v1(file_pattern, # pylint: disable=missing-docstring + batch_size, + features, + reader=core_readers.TFRecordDataset, + label_key=None, + reader_args=None, + num_epochs=None, + shuffle=True, + shuffle_buffer_size=10000, + shuffle_seed=None, + prefetch_buffer_size=optimization.AUTOTUNE, + reader_num_threads=1, + parser_num_threads=2, + sloppy_ordering=False, + drop_final_batch=False): + return dataset_ops.DatasetV1Adapter(make_batched_features_dataset_v2( + file_pattern, batch_size, features, reader, label_key, reader_args, + num_epochs, shuffle, shuffle_buffer_size, shuffle_seed, + prefetch_buffer_size, reader_num_threads, parser_num_threads, + sloppy_ordering, drop_final_batch)) +make_batched_features_dataset_v2.__doc__ = ( + make_batched_features_dataset_v1.__doc__) + + def _get_file_names(file_pattern, shuffle): """Parse list of file names from pattern, optionally shuffled. @@ -850,8 +924,8 @@ def _get_file_names(file_pattern, shuffle): return file_names -@tf_export("data.experimental.SqlDataset") -class SqlDataset(dataset_ops.DatasetSource): +@tf_export("data.experimental.SqlDataset", v1=[]) +class SqlDatasetV2(dataset_ops.DatasetSource): """A `Dataset` consisting of the results from a SQL query.""" def __init__(self, driver_name, data_source_name, query, output_types): @@ -861,17 +935,14 @@ class SqlDataset(dataset_ops.DatasetSource): For example: ```python + tf.enable_eager_execution() + dataset = tf.data.experimental.SqlDataset("sqlite", "/foo/bar.sqlite3", "SELECT name, age FROM people", (tf.string, tf.int32)) - iterator = dataset.make_one_shot_iterator() - next_element = iterator.get_next() # Prints the rows of the result set of the above query. - while True: - try: - print(sess.run(next_element)) - except tf.errors.OutOfRangeError: - break + for element in dataset: + print(element) ``` Args: @@ -883,7 +954,7 @@ class SqlDataset(dataset_ops.DatasetSource): output_types: A tuple of `tf.DType` objects representing the types of the columns returned by `query`. """ - super(SqlDataset, self).__init__() + super(SqlDatasetV2, self).__init__() self._driver_name = ops.convert_to_tensor( driver_name, dtype=dtypes.string, name="driver_name") self._data_source_name = ops.convert_to_tensor( @@ -893,10 +964,9 @@ class SqlDataset(dataset_ops.DatasetSource): self._output_types = output_types def _as_variant_tensor(self): - return gen_dataset_ops.sql_dataset(self._driver_name, - self._data_source_name, self._query, - nest.flatten(self.output_types), - nest.flatten(self.output_shapes)) + return gen_experimental_dataset_ops.experimental_sql_dataset( + self._driver_name, self._data_source_name, self._query, + nest.flatten(self.output_types), nest.flatten(self.output_shapes)) @property def output_classes(self): @@ -910,3 +980,21 @@ class SqlDataset(dataset_ops.DatasetSource): @property def output_types(self): return self._output_types + + +@tf_export(v1=["data.experimental.SqlDataset"]) +class SqlDatasetV1(dataset_ops.DatasetV1Adapter): + """A `Dataset` consisting of the results from a SQL query.""" + + @functools.wraps(SqlDatasetV2.__init__) + def __init__(self, driver_name, data_source_name, query, output_types): + wrapped = SqlDatasetV2(driver_name, data_source_name, query, output_types) + super(SqlDatasetV1, self).__init__(wrapped) + + +# TODO(b/119044825): Until all `tf.data` unit tests are converted to V2, keep +# these aliases in place. +CsvDataset = CsvDatasetV1 +SqlDataset = SqlDatasetV1 +make_batched_features_dataset = make_batched_features_dataset_v1 +make_csv_dataset = make_csv_dataset_v1 diff --git a/tensorflow/python/data/experimental/ops/scan_ops.py b/tensorflow/python/data/experimental/ops/scan_ops.py index 1194238e2f9..c768373cf49 100644 --- a/tensorflow/python/data/experimental/ops/scan_ops.py +++ b/tensorflow/python/data/experimental/ops/scan_ops.py @@ -24,7 +24,7 @@ from tensorflow.python.data.util import nest from tensorflow.python.data.util import sparse from tensorflow.python.framework import ops from tensorflow.python.framework import sparse_tensor -from tensorflow.python.ops import gen_dataset_ops +from tensorflow.python.ops import gen_experimental_dataset_ops from tensorflow.python.util.tf_export import tf_export @@ -125,16 +125,19 @@ class _ScanDataset(dataset_ops.UnaryDataset): self._state_shapes = nest.pack_sequence_as(self._state_shapes, weakened_state_shapes) - self._scan_func = wrapped_func.function - self._scan_func.add_to_graph(ops.get_default_graph()) + self._scan_func = wrapped_func + self._scan_func.function.add_to_graph(ops.get_default_graph()) + + def _functions(self): + return [self._scan_func] def _as_variant_tensor(self): input_t = self._input_dataset._as_variant_tensor() # pylint: disable=protected-access - return gen_dataset_ops.scan_dataset( + return gen_experimental_dataset_ops.experimental_scan_dataset( input_t, nest.flatten(sparse.serialize_sparse_tensors(self._initial_state)), - self._scan_func.captured_inputs, - f=self._scan_func, + self._scan_func.function.captured_inputs, + f=self._scan_func.function, **dataset_ops.flat_structure(self)) @property diff --git a/tensorflow/python/data/experimental/ops/shuffle_ops.py b/tensorflow/python/data/experimental/ops/shuffle_ops.py index a4307212daf..d12328a7145 100644 --- a/tensorflow/python/data/experimental/ops/shuffle_ops.py +++ b/tensorflow/python/data/experimental/ops/shuffle_ops.py @@ -26,7 +26,7 @@ from tensorflow.python.ops import gen_dataset_ops from tensorflow.python.util.tf_export import tf_export -class _ShuffleAndRepeatDataset(dataset_ops.UnaryDataset): +class _ShuffleAndRepeatDataset(dataset_ops.UnaryUnchangedStructureDataset): """A `Dataset` that fuses `shuffle` and `repeat`.""" def __init__(self, input_dataset, buffer_size, count=None, seed=None): @@ -53,18 +53,6 @@ class _ShuffleAndRepeatDataset(dataset_ops.UnaryDataset): **dataset_ops.flat_structure(self)) # pylint: enable=protected-access - @property - def output_classes(self): - return self._input_dataset.output_classes - - @property - def output_shapes(self): - return self._input_dataset.output_shapes - - @property - def output_types(self): - return self._input_dataset.output_types - @tf_export("data.experimental.shuffle_and_repeat") def shuffle_and_repeat(buffer_size, count=None, seed=None): diff --git a/tensorflow/python/data/experimental/ops/sleep.py b/tensorflow/python/data/experimental/ops/sleep.py index 7e7d370f702..5e9d021ada9 100644 --- a/tensorflow/python/data/experimental/ops/sleep.py +++ b/tensorflow/python/data/experimental/ops/sleep.py @@ -21,7 +21,7 @@ from tensorflow.python.data.ops import dataset_ops from tensorflow.python.ops import gen_experimental_dataset_ops -class _SleepDataset(dataset_ops.UnaryDataset): +class _SleepDataset(dataset_ops.UnaryUnchangedStructureDataset): """A `Dataset` that sleeps before producing each upstream element.""" def __init__(self, input_dataset, sleep_microseconds): diff --git a/tensorflow/python/data/experimental/ops/stats_aggregator.py b/tensorflow/python/data/experimental/ops/stats_aggregator.py index 5274c816a49..d5fcc033ab7 100644 --- a/tensorflow/python/data/experimental/ops/stats_aggregator.py +++ b/tensorflow/python/data/experimental/ops/stats_aggregator.py @@ -17,7 +17,7 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function -from tensorflow.python.ops import gen_dataset_ops +from tensorflow.python.ops import gen_experimental_dataset_ops as ged_ops from tensorflow.python.util.tf_export import tf_export @@ -47,7 +47,6 @@ class StatsAggregator(object): options = dataset_ops.Options() options.experimental_stats = tf.data.experimental.StatsOptions(aggregator) dataset = dataset.with_options(options) - iterator = dataset.make_one_shot_iterator() ``` To get a protocol buffer summary of the currently aggregated statistics, @@ -69,7 +68,7 @@ class StatsAggregator(object): def __init__(self): """Creates a `StatsAggregator`.""" - self._resource = gen_dataset_ops.stats_aggregator_handle() + self._resource = ged_ops.experimental_stats_aggregator_handle() # TODO(b/116314787): Update this/add support for V2 summary API. def get_summary(self): @@ -81,4 +80,4 @@ class StatsAggregator(object): Returns: A scalar string `tf.Tensor` that summarizes the aggregated statistics. """ - return gen_dataset_ops.stats_aggregator_summary(self._resource) + return ged_ops.experimental_stats_aggregator_summary(self._resource) diff --git a/tensorflow/python/data/experimental/ops/stats_ops.py b/tensorflow/python/data/experimental/ops/stats_ops.py index ca2f5f2a887..15a9d24546e 100644 --- a/tensorflow/python/data/experimental/ops/stats_ops.py +++ b/tensorflow/python/data/experimental/ops/stats_ops.py @@ -20,7 +20,7 @@ from __future__ import print_function from tensorflow.python.data.ops import dataset_ops from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops -from tensorflow.python.ops import gen_dataset_ops +from tensorflow.python.ops import gen_experimental_dataset_ops from tensorflow.python.util import deprecation from tensorflow.python.util.tf_export import tf_export @@ -66,8 +66,10 @@ def bytes_produced_stats(tag): """ def _apply_fn(dataset): - return _StatsDataset(dataset, gen_dataset_ops.bytes_produced_stats_dataset, - tag) + return _StatsDataset( + dataset, + gen_experimental_dataset_ops.experimental_bytes_produced_stats_dataset, + tag) return _apply_fn @@ -89,12 +91,14 @@ def latency_stats(tag): """ def _apply_fn(dataset): - return _StatsDataset(dataset, gen_dataset_ops.latency_stats_dataset, tag) + return _StatsDataset( + dataset, + gen_experimental_dataset_ops.experimental_latency_stats_dataset, tag) return _apply_fn -class _StatsDataset(dataset_ops.UnaryDataset): +class _StatsDataset(dataset_ops.UnaryUnchangedStructureDataset): """A `Dataset` that acts as an identity, and also records statistics.""" def __init__(self, input_dataset, op_function, tag): @@ -108,15 +112,3 @@ class _StatsDataset(dataset_ops.UnaryDataset): self._input_dataset._as_variant_tensor(), # pylint: disable=protected-access self._tag, **dataset_ops.flat_structure(self)) - - @property - def output_shapes(self): - return self._input_dataset.output_shapes - - @property - def output_types(self): - return self._input_dataset.output_types - - @property - def output_classes(self): - return self._input_dataset.output_classes diff --git a/tensorflow/python/data/experimental/ops/stats_options.py b/tensorflow/python/data/experimental/ops/stats_options.py index c088d3d8881..6e884aa08ae 100644 --- a/tensorflow/python/data/experimental/ops/stats_options.py +++ b/tensorflow/python/data/experimental/ops/stats_options.py @@ -20,25 +20,24 @@ from __future__ import division from __future__ import print_function from tensorflow.python.data.experimental.ops import stats_aggregator +from tensorflow.python.data.util import options from tensorflow.python.util.tf_export import tf_export @tf_export("data.experimental.StatsOptions") -class StatsOptions(object): +class StatsOptions(options.OptionsBase): """Represents options for collecting dataset stats using `StatsAggregator`. To apply `StatsOptions` with a `tf.data.Dataset` object, use the following pattern: ```python - aggretator = tf.data.experimental.StatsAggregator() + aggregator = tf.data.experimental.StatsAggregator() - options = dataset_ops.Options() + options = tf.data.Options() options.experimental_stats = tf.data.experimental.StatsOptions() options.experimental_stats.aggregator = aggregator dataset = dataset.with_options(options) - - iterator = dataset.make_one_shot_iterator() ``` Note: a `StatsAggregator` object can be attached either duing construction or @@ -52,52 +51,29 @@ class StatsOptions(object): ``` """ - for _name, _ty, _default, _docstring in [ - ("aggregator", stats_aggregator.StatsAggregator, None, - "Associate the given statistics options with the dataset pipeline."), - ("prefix", str, "", - "Prefix to prepend all statistics recorded for the input `dataset` with." - ), - ("counter_prefix", str, "", - "Prefix for the statistics recorded as counter."), - ("latency_all_edges", bool, True, - "Whether to add latency measurements on all edges."), - ]: + aggregator = options.create_option( + name="aggregator", + ty=stats_aggregator.StatsAggregator, + docstring= + "Associates the given statistics aggregator with the dataset pipeline.") - def _make_getter(name): # pylint: disable=no-self-argument + prefix = options.create_option( + name="prefix", + ty=str, + docstring= + "Prefix to prepend all statistics recorded for the input `dataset` with.", + default="") - def getter(self): - return getattr(self, "_" + name) + counter_prefix = options.create_option( + name="counter_prefix", + ty=str, + docstring= + "Prefix for the statistics recorded as counter.", + default="") - return getter - - def _make_setter(name, ty): # pylint: disable=no-self-argument - - def setter(self, value): - if not isinstance(value, ty): - raise TypeError( - "Attempting to set the option %s to incompatible value: %r when " - "it expects %r" % (name, value, ty)) - setattr(self, "_" + name, value) - - return setter - - vars()["_" + _name] = _default - vars()[_name] = property( - _make_getter(_name), _make_setter(_name, _ty), _default, _docstring) - - def __init__(self, aggregator=None): - if aggregator: - self.aggregator = aggregator - - def __eq__(self, other): - if isinstance(other, self.__class__): - return self.__dict__ == other.__dict__ - else: - return False - - def __ne__(self, other): - return not self.__eq__(other) - - def __str__(self): - return str(self.__dict__) + latency_all_edges = options.create_option( + name="latency_all_edges", + ty=bool, + docstring= + "Whether to add latency measurements on all edges.", + default=True) diff --git a/tensorflow/python/data/experimental/ops/threading_options.py b/tensorflow/python/data/experimental/ops/threading_options.py new file mode 100644 index 00000000000..dbf662186f8 --- /dev/null +++ b/tensorflow/python/data/experimental/ops/threading_options.py @@ -0,0 +1,50 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Experimental API for controlling threading in `tf.data` pipelines.""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + + +from tensorflow.python.data.util import options +from tensorflow.python.util.tf_export import tf_export + + +@tf_export("data.experimental.ThreadingOptions") +class ThreadingOptions(options.OptionsBase): + """Represents options for dataset threading. + + To apply `ThreadingOptions` to a `dataset` object, use the following pattern: + + ```python + options = tf.data.Options() + options.experimental_threading = tf.data.experimental.ThreadingOptions() + options.experimental_threading.private_threadpool_size = 10 + dataset = dataset.with_options(options) + ``` + """ + + max_intra_op_parallelism = options.create_option( + name="max_intra_op_parallelism", + ty=int, + docstring= + "If set, it overrides the maximum degree of intra-op parallelism.") + + private_threadpool_size = options.create_option( + name="private_threadpool_size", + ty=int, + docstring= + "If set, the dataset will use a private threadpool of the given size.", + default=None) diff --git a/tensorflow/python/data/experimental/ops/threadpool.py b/tensorflow/python/data/experimental/ops/threadpool.py index 3ea017c6e80..69e8829d687 100644 --- a/tensorflow/python/data/experimental/ops/threadpool.py +++ b/tensorflow/python/data/experimental/ops/threadpool.py @@ -60,7 +60,7 @@ class PrivateThreadPool(object): display_name=display_name) -class _ThreadPoolDataset(dataset_ops.UnaryDataset): +class _ThreadPoolDataset(dataset_ops.UnaryUnchangedStructureDataset): """A `Dataset` that acts as an identity, and sets a custom threadpool.""" def __init__(self, input_dataset, thread_pool): @@ -74,18 +74,6 @@ class _ThreadPoolDataset(dataset_ops.UnaryDataset): self._thread_pool._resource, # pylint: disable=protected-access **dataset_ops.flat_structure(self)) - @property - def output_shapes(self): - return self._input_dataset.output_shapes - - @property - def output_types(self): - return self._input_dataset.output_types - - @property - def output_classes(self): - return self._input_dataset.output_classes - # TODO(b/73383364): Properly export in the `tf.data.experimental` API when # stable or make private / remove. diff --git a/tensorflow/python/data/experimental/ops/unique.py b/tensorflow/python/data/experimental/ops/unique.py index 2a7775c456e..55ed98d8542 100644 --- a/tensorflow/python/data/experimental/ops/unique.py +++ b/tensorflow/python/data/experimental/ops/unique.py @@ -48,7 +48,7 @@ def unique(): return _apply_fn -class _UniqueDataset(dataset_ops.UnaryDataset): +class _UniqueDataset(dataset_ops.UnaryUnchangedStructureDataset): """A `Dataset` contains the unique elements from its input.""" def __init__(self, input_dataset): @@ -65,15 +65,3 @@ class _UniqueDataset(dataset_ops.UnaryDataset): return gen_experimental_dataset_ops.experimental_unique_dataset( self._input_dataset._as_variant_tensor(), # pylint: disable=protected-access **dataset_ops.flat_structure(self)) - - @property - def output_classes(self): - return self._input_dataset.output_classes - - @property - def output_shapes(self): - return self._input_dataset.output_shapes - - @property - def output_types(self): - return self._input_dataset.output_types diff --git a/tensorflow/python/data/experimental/ops/writers.py b/tensorflow/python/data/experimental/ops/writers.py index 994447cb4db..aef6da51409 100644 --- a/tensorflow/python/data/experimental/ops/writers.py +++ b/tensorflow/python/data/experimental/ops/writers.py @@ -22,7 +22,7 @@ from tensorflow.python.data.util import convert from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops from tensorflow.python.framework import tensor_shape -from tensorflow.python.ops import gen_dataset_ops +from tensorflow.python.ops import gen_experimental_dataset_ops from tensorflow.python.util.tf_export import tf_export @@ -48,7 +48,7 @@ class TFRecordWriter(object): Returns: A `tf.Operation` that, when run, writes contents of `dataset` to a file. """ - if not isinstance(dataset, dataset_ops.Dataset): + if not isinstance(dataset, dataset_ops.DatasetV2): raise TypeError("`dataset` must be a `tf.data.Dataset` object.") if (dataset.output_types != dtypes.string or dataset.output_shapes != tensor_shape.scalar()): @@ -56,5 +56,5 @@ class TFRecordWriter(object): "`dataset` must produce scalar `DT_STRING` tensors whereas it " "produces shape {0} and types {1}".format(dataset.output_shapes, dataset.output_types)) - return gen_dataset_ops.dataset_to_tf_record( + return gen_experimental_dataset_ops.experimental_dataset_to_tf_record( dataset._as_variant_tensor(), self._filename, self._compression_type) # pylint: disable=protected-access diff --git a/tensorflow/python/data/kernel_tests/BUILD b/tensorflow/python/data/kernel_tests/BUILD index 21eed2b070a..9f7ce99cbc6 100644 --- a/tensorflow/python/data/kernel_tests/BUILD +++ b/tensorflow/python/data/kernel_tests/BUILD @@ -10,122 +10,101 @@ load("//tensorflow:tensorflow.bzl", "tf_py_test") load("//tensorflow:tensorflow.bzl", "cuda_py_test") tf_py_test( - name = "batch_dataset_op_test", + name = "batch_test", size = "small", - srcs = ["batch_dataset_op_test.py"], + srcs = ["batch_test.py"], additional_deps = [ ":test_base", "@absl_py//absl/testing:parameterized", "//third_party/py/numpy", + "//tensorflow/python/data/ops:dataset_ops", "//tensorflow/python:array_ops", "//tensorflow/python:client_testlib", - "//tensorflow/python:constant_op", "//tensorflow/python:dtypes", "//tensorflow/python:errors", "//tensorflow/python:math_ops", - "//tensorflow/python:string_ops", - "//tensorflow/python:tensor_shape", - "//tensorflow/python:util", - "//tensorflow/python/data/ops:dataset_ops", + "//tensorflow/python:sparse_tensor", ], ) tf_py_test( - name = "cache_dataset_op_test", + name = "cache_test", size = "small", - srcs = ["cache_dataset_op_test.py"], + srcs = ["cache_test.py"], additional_deps = [ ":test_base", "//third_party/py/numpy", + "//tensorflow/python/data/ops:dataset_ops", + "//tensorflow/python/data/ops:iterator_ops", "//tensorflow/python:array_ops", "//tensorflow/python:client_testlib", "//tensorflow/python:constant_op", "//tensorflow/python:dtypes", "//tensorflow/python:errors", - "//tensorflow/python:variables", - "//tensorflow/python/data/ops:dataset_ops", - "//tensorflow/python/data/ops:iterator_ops", - ], -) - -tf_py_test( - name = "concatenate_dataset_op_test", - size = "small", - srcs = ["concatenate_dataset_op_test.py"], - additional_deps = [ - ":test_base", - "//third_party/py/numpy", - "//tensorflow/python:client_testlib", - "//tensorflow/python:errors", - "//tensorflow/python:tensor_shape", - "//tensorflow/python/data/ops:dataset_ops", - "//tensorflow/python/data/util:nest", - ], -) - -tf_py_test( - name = "dataset_constructor_op_test", - size = "small", - srcs = ["dataset_constructor_op_test.py"], - additional_deps = [ - ":test_base", - "//third_party/py/numpy", - "//tensorflow/core:protos_all_py", - "//tensorflow/python:array_ops", - "//tensorflow/python:client_testlib", - "//tensorflow/python:dtypes", - "//tensorflow/python:errors", "//tensorflow/python:framework_ops", - "//tensorflow/python:math_ops", - "//tensorflow/python:random_ops", - "//tensorflow/python:resource_variable_ops", - "//tensorflow/python:session", - "//tensorflow/python:sparse_tensor", - "//tensorflow/python:tensor_shape", - "//tensorflow/python/data/ops:dataset_ops", - "//tensorflow/python/data/util:nest", - "//tensorflow/python/data/util:sparse", - ], - tags = [ - "manual", - "nomac", # b/62040583 + "//tensorflow/python:variables", ], ) tf_py_test( - name = "dataset_from_generator_op_test", - size = "medium", - srcs = ["dataset_from_generator_op_test.py"], + name = "concatenate_test", + size = "small", + srcs = ["concatenate_test.py"], additional_deps = [ ":test_base", "//third_party/py/numpy", "//tensorflow/python:client_testlib", - "//tensorflow/python:dtypes", "//tensorflow/python:errors", "//tensorflow/python:tensor_shape", "//tensorflow/python/data/ops:dataset_ops", - "//tensorflow/python/data/util:sparse", + "//tensorflow/python/data/util:nest", ], ) tf_py_test( - name = "dataset_ops_test", + name = "dataset_checkpoint_test", size = "small", - srcs = ["dataset_ops_test.py"], + srcs = ["dataset_checkpoint_test.py"], + additional_deps = [ + ":test_base", + "//tensorflow/python/data/ops:dataset_ops", + "//tensorflow/python/data/ops:iterator_ops", + "//tensorflow/python:client_testlib", + "//tensorflow/python:dataset_ops_gen", + "//tensorflow/python:dtypes", + "//tensorflow/python:errors", + "//tensorflow/python:framework_ops", + "//tensorflow/python:framework_test_lib", + "//tensorflow/python:io_ops", + "//tensorflow/python:parsing_ops", + "//tensorflow/python:platform", + "//tensorflow/python:tensor_shape", + "//tensorflow/python:variables", + ], +) + +tf_py_test( + name = "dataset_test", + size = "small", + srcs = ["dataset_test.py"], additional_deps = [ ":test_base", "@absl_py//absl/testing:parameterized", "//third_party/py/numpy", - "//tensorflow/python:client_testlib", - "//tensorflow/python:sparse_tensor", + "//tensorflow/core:protos_all_py", "//tensorflow/python/data/ops:dataset_ops", + "//tensorflow/python/data/ops:readers", + "//tensorflow/python/data/util:nest", + "//tensorflow/python:client_testlib", + "//tensorflow/python:dtypes", + "//tensorflow/python:sparse_tensor", ], ) tf_py_test( - name = "filter_dataset_op_test", + name = "filter_test", size = "small", - srcs = ["filter_dataset_op_test.py"], + srcs = ["filter_test.py"], additional_deps = [ ":test_base", "//third_party/py/numpy", @@ -141,12 +120,36 @@ tf_py_test( ) tf_py_test( - name = "flat_map_dataset_op_test", + name = "fixed_length_record_dataset_test", size = "small", - srcs = ["flat_map_dataset_op_test.py"], + srcs = ["fixed_length_record_dataset_test.py"], + additional_deps = [ + ":test_base", + "//tensorflow/python:array_ops", + "//tensorflow/python:client_testlib", + "//tensorflow/python:dataset_ops_gen", + "//tensorflow/python:dtypes", + "//tensorflow/python:errors", + "//tensorflow/python:framework_ops", + "//tensorflow/python:io_ops", + "//tensorflow/python:parsing_ops", + "//tensorflow/python:tensor_shape", + "//tensorflow/python:util", + "//tensorflow/python/data/ops:iterator_ops", + "//tensorflow/python/data/ops:readers", + ], +) + +tf_py_test( + name = "flat_map_test", + size = "medium", + srcs = ["flat_map_test.py"], additional_deps = [ ":test_base", "//third_party/py/numpy", + "//tensorflow/core:protos_all_py", + "//tensorflow/python/data/ops:readers", + "//tensorflow/python/data/util:nest", "//tensorflow/python:client_testlib", "//tensorflow/python:errors", "//tensorflow/python:session", @@ -159,58 +162,157 @@ tf_py_test( ) tf_py_test( - name = "list_files_dataset_op_test", - size = "small", - srcs = ["list_files_dataset_op_test.py"], + name = "from_generator_test", + size = "medium", + srcs = ["from_generator_test.py"], additional_deps = [ ":test_base", - "//tensorflow/python:array_ops", - "//tensorflow/python:client_testlib", - "//tensorflow/python:dtypes", - "//tensorflow/python:errors", - "//tensorflow/python:util", - "//tensorflow/python/data/ops:dataset_ops", - ], -) - -tf_py_test( - name = "inputs_test", - size = "small", - srcs = ["inputs_test.py"], - additional_deps = [ - ":test_base", - "@absl_py//absl/testing:parameterized", - "//third_party/py/numpy", - "//tensorflow/python:client_testlib", - "//tensorflow/python:sparse_tensor", - "//tensorflow/python/data/ops:dataset_ops", - ], -) - -tf_py_test( - name = "interleave_dataset_op_test", - size = "small", - srcs = ["interleave_dataset_op_test.py"], - additional_deps = [ - ":test_base", - "@absl_py//absl/testing:parameterized", - "//third_party/py/numpy", - "//tensorflow/python:array_ops", + "//third_party/py/numpy", + "//tensorflow/python/data/ops:dataset_ops", "//tensorflow/python:client_testlib", + "//tensorflow/python:constant_op", "//tensorflow/python:dtypes", "//tensorflow/python:errors", + "//tensorflow/python:script_ops", "//tensorflow/python:session", + ], +) + +tf_py_test( + name = "from_sparse_tensor_slices_test", + size = "small", + srcs = ["from_sparse_tensor_slices_test.py"], + additional_deps = [ + ":test_base", + "//third_party/py/numpy", + "//tensorflow/core:protos_all_py", + "//tensorflow/python:array_ops", + "//tensorflow/python:client_testlib", + "//tensorflow/python:dtypes", + "//tensorflow/python:errors", + "//tensorflow/python:framework_ops", + "//tensorflow/python:math_ops", + "//tensorflow/python:resource_variable_ops", + "//tensorflow/python:session", + "//tensorflow/python:sparse_tensor", + "//tensorflow/python:tensor_shape", + "//tensorflow/python/data/ops:dataset_ops", + "//tensorflow/python/data/util:nest", + ], +) + +tf_py_test( + name = "from_tensors_test", + size = "small", + srcs = ["from_tensors_test.py"], + additional_deps = [ + ":test_base", + "//third_party/py/numpy", + "//tensorflow/core:protos_all_py", + "//tensorflow/python:array_ops", + "//tensorflow/python:client_testlib", + "//tensorflow/python:dtypes", + "//tensorflow/python:errors", + "//tensorflow/python:framework_ops", + "//tensorflow/python:math_ops", + "//tensorflow/python:resource_variable_ops", + "//tensorflow/python:session", + "//tensorflow/python:sparse_tensor", + "//tensorflow/python:tensor_shape", + "//tensorflow/python/data/ops:dataset_ops", + "//tensorflow/python/data/util:nest", + ], + tags = [ + "nomac", # b/62040583 + ], +) + +tf_py_test( + name = "from_tensor_slices_test", + size = "small", + srcs = ["from_tensor_slices_test.py"], + additional_deps = [ + ":test_base", + "//third_party/py/numpy", + "//tensorflow/python:client_testlib", + "//tensorflow/python:dtypes", + "//tensorflow/python:errors", + "//tensorflow/python:sparse_tensor", + "//tensorflow/python:tensor_shape", + "//tensorflow/python/data/ops:dataset_ops", + ], +) + +tf_py_test( + name = "interleave_test", + size = "medium", + srcs = ["interleave_test.py"], + additional_deps = [ + ":test_base", + "@absl_py//absl/testing:parameterized", + "//third_party/py/numpy", + "//tensorflow/core:protos_all_py", + "//tensorflow/python/data/ops:dataset_ops", + "//tensorflow/python:array_ops", + "//tensorflow/python:client_testlib", + "//tensorflow/python:errors", + "//tensorflow/python:script_ops", "//tensorflow/python:sparse_ops", "//tensorflow/python:sparse_tensor", - "//tensorflow/python:training", + ], +) + +tf_py_test( + name = "iterator_checkpoint_test", + size = "medium", + srcs = ["iterator_checkpoint_test.py"], + additional_deps = [ + ":test_base", "//tensorflow/python/data/ops:dataset_ops", + "//tensorflow/python/eager:context", + "//tensorflow/python/training/checkpointable:util", + "//tensorflow/python:checkpoint_management", + "//tensorflow/python:client_testlib", + "//tensorflow/python:errors", + "//tensorflow/python:framework_test_lib", + "//tensorflow/python:math_ops", + ], + grpc_enabled = True, +) + +tf_py_test( + name = "iterator_cluster_test", + size = "small", + srcs = ["iterator_cluster_test.py"], + additional_deps = [ + "//tensorflow/core:protos_all_py", + "//tensorflow/python/data/ops:dataset_ops", + "//tensorflow/python/data/ops:iterator_ops", + "//tensorflow/python:array_ops", + "//tensorflow/python:client_testlib", + "//tensorflow/python:constant_op", + "//tensorflow/python:dtypes", + "//tensorflow/python:errors", + "//tensorflow/python:framework_ops", + "//tensorflow/python:framework_test_lib", + "//tensorflow/python:function", + "//tensorflow/python:functional_ops", + "//tensorflow/python:lookup_ops", + "//tensorflow/python:math_ops", + "//tensorflow/python:session", + "//tensorflow/python:string_ops", + ], + grpc_enabled = True, + tags = [ + "no_oss", # Test flaky due to port collisions. + "no_windows", ], ) cuda_py_test( - name = "iterator_ops_test", - size = "small", - srcs = ["iterator_ops_test.py"], + name = "iterator_test", + size = "medium", + srcs = ["iterator_test.py"], additional_deps = [ "@absl_py//absl/testing:parameterized", "//third_party/py/numpy", @@ -249,41 +351,30 @@ cuda_py_test( ) tf_py_test( - name = "iterator_ops_cluster_test", + name = "list_files_test", size = "small", - srcs = ["iterator_ops_cluster_test.py"], + srcs = ["list_files_test.py"], additional_deps = [ - "//tensorflow/core:protos_all_py", + ":test_base", "//tensorflow/python:array_ops", "//tensorflow/python:client_testlib", "//tensorflow/python:dtypes", "//tensorflow/python:errors", - "//tensorflow/python:framework_ops", - "//tensorflow/python:framework_test_lib", - "//tensorflow/python:function", - "//tensorflow/python:functional_ops", - "//tensorflow/python:session", + "//tensorflow/python:util", "//tensorflow/python/data/ops:dataset_ops", - "//tensorflow/python/data/ops:iterator_ops", - "//tensorflow/python:constant_op", - "//tensorflow/python:string_ops", - "//tensorflow/python:lookup_ops", - ], - grpc_enabled = True, - tags = [ - "no_oss", # Test flaky due to port collisions. - "no_windows", ], ) tf_py_test( - name = "map_dataset_op_test", - size = "small", - srcs = ["map_dataset_op_test.py"], + name = "map_test", + size = "medium", + srcs = ["map_test.py"], additional_deps = [ ":test_base", "@absl_py//absl/testing:parameterized", "//third_party/py/numpy", + "//tensorflow/core:protos_all_py", + "//tensorflow/python/data/ops:dataset_ops", "//tensorflow/python:array_ops", "//tensorflow/python:client_testlib", "//tensorflow/python:constant_op", @@ -297,27 +388,12 @@ tf_py_test( "//tensorflow/python:math_ops", "//tensorflow/python:random_ops", "//tensorflow/python:script_ops", + "//tensorflow/python:session", "//tensorflow/python:sparse_ops", "//tensorflow/python:sparse_tensor", "//tensorflow/python:string_ops", + "//tensorflow/python:tensor_util", "//tensorflow/python:variable_scope", - "//tensorflow/python/data/ops:dataset_ops", - ], -) - -tf_py_test( - name = "matching_files_dataset_op_test", - size = "small", - srcs = ["matching_files_dataset_op_test.py"], - additional_deps = [ - ":test_base", - "//third_party/py/numpy", - "//tensorflow/python:array_ops", - "//tensorflow/python:client_testlib", - "//tensorflow/python:dtypes", - "//tensorflow/python:errors", - "//tensorflow/python:util", - "//tensorflow/python/data/ops:dataset_ops", ], ) @@ -332,6 +408,7 @@ cuda_py_test( "//tensorflow/python/data/ops:multi_device_iterator_ops", "//tensorflow/python/data/ops:iterator_ops", "//tensorflow/python/data/experimental/ops:optimization", + "//tensorflow/python/data/experimental/ops:optimization_options", "//tensorflow/python:array_ops", "//tensorflow/python:client_testlib", "//tensorflow/python:dtypes", @@ -345,9 +422,9 @@ cuda_py_test( ) cuda_py_test( - name = "optional_ops_test", + name = "optional_test", size = "small", - srcs = ["optional_ops_test.py"], + srcs = ["optional_test.py"], additional_deps = [ ":test_base", "@absl_py//absl/testing:parameterized", @@ -366,9 +443,30 @@ cuda_py_test( ) tf_py_test( - name = "prefetch_dataset_op_test", + name = "padded_batch_test", size = "small", - srcs = ["prefetch_dataset_op_test.py"], + srcs = ["padded_batch_test.py"], + additional_deps = [ + ":test_base", + "@absl_py//absl/testing:parameterized", + "//third_party/py/numpy", + "//tensorflow/python/data/ops:dataset_ops", + "//tensorflow/python:array_ops", + "//tensorflow/python:client_testlib", + "//tensorflow/python:constant_op", + "//tensorflow/python:dtypes", + "//tensorflow/python:errors", + "//tensorflow/python:sparse_tensor", + "//tensorflow/python:string_ops", + "//tensorflow/python:tensor_shape", + "//tensorflow/python:util", + ], +) + +tf_py_test( + name = "prefetch_test", + size = "small", + srcs = ["prefetch_test.py"], additional_deps = [ ":test_base", "@absl_py//absl/testing:parameterized", @@ -377,59 +475,26 @@ tf_py_test( "//tensorflow/python:dataset_ops_gen", "//tensorflow/python:dtypes", "//tensorflow/python:errors", - "//tensorflow/python/data/ops:dataset_ops", ], ) tf_py_test( - name = "range_dataset_op_test", + name = "range_test", size = "small", - srcs = ["range_dataset_op_test.py"], + srcs = ["range_test.py"], additional_deps = [ ":test_base", - "//tensorflow/python:array_ops", - "//tensorflow/python:client_testlib", - "//tensorflow/python:dataset_ops_gen", - "//tensorflow/python:dtypes", - "//tensorflow/python:errors", - "//tensorflow/python:io_ops", - "//tensorflow/python:framework_ops", - "//tensorflow/python:parsing_ops", - "//tensorflow/python:platform", - "//tensorflow/python:tensor_shape", - "//tensorflow/python:variables", "//tensorflow/python/data/ops:dataset_ops", - "//tensorflow/python/data/ops:iterator_ops", - ], -) - -tf_py_test( - name = "reader_dataset_ops_test", - size = "small", - srcs = ["reader_dataset_ops_test.py"], - additional_deps = [ - ":test_base", - "//tensorflow/python:array_ops", "//tensorflow/python:client_testlib", - "//tensorflow/python:constant_op", - "//tensorflow/python:dataset_ops_gen", - "//tensorflow/python:dtypes", "//tensorflow/python:errors", - "//tensorflow/python:framework_ops", - "//tensorflow/python:io_ops", - "//tensorflow/python:lib", - "//tensorflow/python:parsing_ops", - "//tensorflow/python:tensor_shape", - "//tensorflow/python:util", - "//tensorflow/python/data/ops:iterator_ops", - "//tensorflow/python/data/ops:readers", + "//tensorflow/python:framework_test_lib", ], ) tf_py_test( - name = "reduce_dataset_op_test", + name = "reduce_test", size = "small", - srcs = ["reduce_dataset_op_test.py"], + srcs = ["reduce_test.py"], additional_deps = [ ":test_base", "@absl_py//absl/testing:parameterized", @@ -437,7 +502,6 @@ tf_py_test( "//tensorflow/python:array_ops", "//tensorflow/python:client_testlib", "//tensorflow/python:dtypes", - "//tensorflow/python:errors", "//tensorflow/python:math_ops", "//tensorflow/python:sparse_tensor", "//tensorflow/python/data/ops:dataset_ops", @@ -445,9 +509,9 @@ tf_py_test( ) tf_py_test( - name = "sequence_dataset_op_test", + name = "repeat_test", size = "small", - srcs = ["sequence_dataset_op_test.py"], + srcs = ["repeat_test.py"], additional_deps = [ ":test_base", "//third_party/py/numpy", @@ -460,9 +524,9 @@ tf_py_test( ) tf_py_test( - name = "shard_dataset_op_test", + name = "shard_test", size = "small", - srcs = ["shard_dataset_op_test.py"], + srcs = ["shard_test.py"], additional_deps = [ ":test_base", "//tensorflow/python:client_testlib", @@ -472,9 +536,9 @@ tf_py_test( ) tf_py_test( - name = "shuffle_dataset_op_test", + name = "shuffle_test", size = "small", - srcs = ["shuffle_dataset_op_test.py"], + srcs = ["shuffle_test.py"], additional_deps = [ ":test_base", "@absl_py//absl/testing:parameterized", @@ -491,21 +555,91 @@ tf_py_test( ], ) -py_library( - name = "test_base", - srcs = ["test_base.py"], - deps = [ +tf_py_test( + name = "skip_test", + size = "small", + srcs = ["skip_test.py"], + additional_deps = [ + ":test_base", + "//third_party/py/numpy", + "//tensorflow/python:array_ops", "//tensorflow/python:client_testlib", + "//tensorflow/python:dtypes", "//tensorflow/python:errors", - "//tensorflow/python:sparse_tensor", - "//tensorflow/python/data/util:nest", + "//tensorflow/python/data/ops:dataset_ops", ], ) tf_py_test( - name = "window_dataset_op_test", + name = "take_test", size = "small", - srcs = ["window_dataset_op_test.py"], + srcs = ["take_test.py"], + additional_deps = [ + ":test_base", + "//third_party/py/numpy", + "//tensorflow/python:array_ops", + "//tensorflow/python:client_testlib", + "//tensorflow/python:dtypes", + "//tensorflow/python:errors", + "//tensorflow/python/data/ops:dataset_ops", + ], +) + +tf_py_test( + name = "text_line_dataset_test", + size = "small", + srcs = ["text_line_dataset_test.py"], + additional_deps = [ + ":test_base", + "//tensorflow/python/data/ops:iterator_ops", + "//tensorflow/python/data/ops:readers", + "//tensorflow/python/eager:context", + "//tensorflow/python:array_ops", + "//tensorflow/python:client_testlib", + "//tensorflow/python:dtypes", + "//tensorflow/python:errors", + "//tensorflow/python:util", + ], +) + +tf_py_test( + name = "tf_record_dataset_test", + size = "small", + srcs = ["tf_record_dataset_test.py"], + additional_deps = [ + ":test_base", + "//tensorflow/python/data/ops:dataset_ops", + "//tensorflow/python/data/ops:iterator_ops", + "//tensorflow/python/data/ops:readers", + "//tensorflow/python:array_ops", + "//tensorflow/python:client_testlib", + "//tensorflow/python:constant_op", + "//tensorflow/python:dtypes", + "//tensorflow/python:errors", + "//tensorflow/python:lib", + "//tensorflow/python:util", + ], +) + +py_library( + name = "test_base", + srcs = ["test_base.py"], + deps = [ + "//tensorflow/python:array_ops", + "//tensorflow/python:client_testlib", + "//tensorflow/python:dtypes", + "//tensorflow/python:errors", + "//tensorflow/python:sparse_tensor", + "//tensorflow/python/data/ops:dataset_ops", + "//tensorflow/python/data/util:nest", + "//tensorflow/python/eager:context", + ], +) + +tf_py_test( + name = "window_test", + size = "medium", + srcs = ["window_test.py"], additional_deps = [ ":test_base", "@absl_py//absl/testing:parameterized", @@ -521,9 +655,9 @@ tf_py_test( ) tf_py_test( - name = "zip_dataset_op_test", + name = "zip_test", size = "small", - srcs = ["zip_dataset_op_test.py"], + srcs = ["zip_test.py"], additional_deps = [ ":test_base", "//third_party/py/numpy", diff --git a/tensorflow/python/data/kernel_tests/batch_dataset_op_test.py b/tensorflow/python/data/kernel_tests/batch_dataset_op_test.py deleted file mode 100644 index e8decb9ad0e..00000000000 --- a/tensorflow/python/data/kernel_tests/batch_dataset_op_test.py +++ /dev/null @@ -1,515 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for the experimental input pipeline ops.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import time - -from absl.testing import parameterized -import numpy as np - -from tensorflow.python.client import session -from tensorflow.python.data.kernel_tests import test_base -from tensorflow.python.data.ops import dataset_ops -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import errors -from tensorflow.python.framework import sparse_tensor -from tensorflow.python.framework import tensor_shape -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import string_ops -from tensorflow.python.platform import test -from tensorflow.python.util import compat - - -class BatchDatasetTest(test_base.DatasetTestBase, parameterized.TestCase): - - @parameterized.named_parameters( - ('even', 28, 14, False), - ('uneven_with_remainder', 28, 15, False), - ('uneven_without_remainder', 28, 15, True), - ('empty', 0, 14, False), - ) - def testBatchDataset(self, count, batch_size, drop_remainder): - """Tests the batch dataset logic for various input configurations. - - Args: - count: the number of input elements - batch_size: the batch size - drop_remainder: whether a smaller batch size should be produced if batch - size does not divide number of inputs evenly - """ - - # The pipeline is TensorSliceDataset -> MapDataset(square_3) -> - # RepeatDataset(count) -> BatchDataset(batch_size). - components = (np.arange(7), - np.array([[1, 2, 3]]) * np.arange(7)[:, np.newaxis], - np.array(37.0) * np.arange(7)) - - count_t = array_ops.placeholder(dtypes.int64, shape=[]) - batch_size_t = array_ops.placeholder(dtypes.int64, shape=[]) - drop_remainder_t = array_ops.placeholder(dtypes.bool, shape=[]) - - def _map_fn(x, y, z): - return math_ops.square(x), math_ops.square(y), math_ops.square(z) - - iterator = ( - dataset_ops.Dataset.from_tensor_slices(components).map(_map_fn) - .repeat(count).batch(batch_size, - drop_remainder).make_initializable_iterator()) - init_op = iterator.initializer - get_next = iterator.get_next() - - if drop_remainder: - dim0 = batch_size - else: - dim0 = None - self.assertEqual([[dim0] + list(c.shape[1:]) for c in components], - [t.shape.as_list() for t in get_next]) - - with self.cached_session() as sess: - sess.run( - init_op, - feed_dict={ - count_t: count, - batch_size_t: batch_size, - drop_remainder_t: drop_remainder - }) - num_full_batches = (count * 7) // batch_size - for i in range(num_full_batches): - result = sess.run(get_next) - for component, result_component in zip(components, result): - for j in range(batch_size): - self.assertAllEqual(component[(i * batch_size + j) % 7]**2, - result_component[j]) - if not drop_remainder and (count * 7) % batch_size > 0: - result = sess.run(get_next) - for component, result_component in zip(components, result): - for j in range((count * 7) % batch_size): - self.assertAllEqual( - component[(num_full_batches * batch_size + j) % 7]**2, - result_component[j]) - with self.assertRaises(errors.OutOfRangeError): - sess.run(get_next) - - def testBatchDatasetInvalidBatchSize(self): - iterator = (dataset_ops.Dataset.range(10).batch(0).make_one_shot_iterator()) - get_next = iterator.get_next() - - with self.cached_session() as sess: - with self.assertRaises(errors.InvalidArgumentError): - sess.run(get_next) - - def testBatchSparse(self): - - def _sparse(i): - return sparse_tensor.SparseTensorValue( - indices=[[0]], values=(i * [1]), dense_shape=[1]) - - iterator = dataset_ops.Dataset.range(10).map(_sparse).batch( - 5).make_initializable_iterator() - init_op = iterator.initializer - get_next = iterator.get_next() - - with self.cached_session() as sess: - sess.run(init_op) - for i in range(2): - actual = sess.run(get_next) - expected = sparse_tensor.SparseTensorValue( - indices=[[0, 0], [1, 0], [2, 0], [3, 0], [4, 0]], - values=[i * 5, i * 5 + 1, i * 5 + 2, i * 5 + 3, i * 5 + 4], - dense_shape=[5, 1]) - self.assertTrue(sparse_tensor.is_sparse(actual)) - self.assertSparseValuesEqual(actual, expected) - with self.assertRaises(errors.OutOfRangeError): - sess.run(get_next) - - def testBatchSparseWithDifferentDenseShapes(self): - - def _sparse(i): - return sparse_tensor.SparseTensorValue( - indices=array_ops.expand_dims( - math_ops.range(i, dtype=dtypes.int64), 1), - values=array_ops.fill([math_ops.to_int32(i)], i), - dense_shape=[i]) - - iterator = dataset_ops.Dataset.range(10).map(_sparse).batch( - 5).make_initializable_iterator() - init_op = iterator.initializer - get_next = iterator.get_next() - - with self.cached_session() as sess: - sess.run(init_op) - for i in range(2): - actual = sess.run(get_next) - expected_indices = [] - expected_values = [] - for j in range(5): - for k in range(i * 5 + j): - expected_indices.append([j, k]) - expected_values.append(i * 5 + j) - expected = sparse_tensor.SparseTensorValue( - indices=expected_indices, - values=expected_values, - dense_shape=[5, (i + 1) * 5 - 1]) - self.assertTrue(sparse_tensor.is_sparse(actual)) - self.assertSparseValuesEqual(actual, expected) - with self.assertRaises(errors.OutOfRangeError): - sess.run(get_next) - - def testNestedBatchSparse(self): - - def _sparse(i): - return sparse_tensor.SparseTensorValue( - indices=[[0]], values=(i * [1]), dense_shape=[1]) - - iterator = dataset_ops.Dataset.range(10).map(_sparse).batch(5).batch( - 2).make_initializable_iterator() - init_op = iterator.initializer - get_next = iterator.get_next() - - with self.cached_session() as sess: - sess.run(init_op) - actual = sess.run(get_next) - expected = sparse_tensor.SparseTensorValue( - indices=[[0, 0, 0], [0, 1, 0], [0, 2, 0], [0, 3, 0], [0, 4, 0], - [1, 0, 0], [1, 1, 0], [1, 2, 0], [1, 3, 0], [1, 4, 0]], - values=[0, 1, 2, 3, 4, 5, 6, 7, 8, 9], - dense_shape=[2, 5, 1]) - self.assertTrue(sparse_tensor.is_sparse(actual)) - self.assertSparseValuesEqual(actual, expected) - with self.assertRaises(errors.OutOfRangeError): - sess.run(get_next) - - def testBatchShapeError(self): - - def generator(): - yield [1.0, 2.0, 3.0] - yield [4.0, 5.0, 6.0] - yield [7.0, 8.0, 9.0, 10.0] - - iterator = ( - dataset_ops.Dataset.from_generator( - generator, dtypes.float32, output_shapes=[None]).batch(3) - .make_initializable_iterator()) - next_element = iterator.get_next() - - with self.cached_session() as sess: - sess.run(iterator.initializer) - with self.assertRaisesRegexp( - errors.InvalidArgumentError, - r'Cannot batch tensors with different shapes in component 0. ' - r'First element had shape \[3\] and element 2 had shape \[4\].'): - sess.run(next_element) - - -def _random_seq_lens(count): - return np.random.randint(20, size=(count,)).astype(np.int32) - - -class PaddedBatchDatasetTest(test_base.DatasetTestBase, parameterized.TestCase): - - @parameterized.named_parameters( - ('default_padding', _random_seq_lens(32), 4, [-1], False), - ('constant_padding', _random_seq_lens(32), 4, [25], False), - ('uneven_with_remainder', _random_seq_lens(34), 4, [-1], False), - ('uneven_without_remainder', _random_seq_lens(34), 4, [-1], True), - ) - def testPaddedBatchDataset(self, seq_lens, batch_size, padded_shapes, - drop_remainder): - """Tests the padded batch dataset logic for various input configurations. - - Args: - seq_lens: the input sequence lengths - batch_size: the batch size - padded_shapes: the padded shapes to use - drop_remainder: whether a smaller batch size should be produced if batch - size does not divide number of inputs evenly - """ - - seq_lens_t = array_ops.placeholder(dtypes.int32, shape=[None]) - batch_size_t = array_ops.placeholder(dtypes.int64, shape=[]) - padded_shapes_t = array_ops.placeholder(dtypes.int64, shape=[1]) - drop_remainder_t = array_ops.placeholder(dtypes.bool, shape=[]) - - iterator = ( - dataset_ops.Dataset.from_tensor_slices(seq_lens_t) - .map(lambda x: array_ops.fill([x], x)).padded_batch( - batch_size=batch_size_t, - drop_remainder=drop_remainder_t, - padded_shapes=padded_shapes_t).make_initializable_iterator()) - - init_op = iterator.initializer - get_next = iterator.get_next() - - with self.cached_session() as sess: - sess.run( - init_op, - feed_dict={ - seq_lens_t: seq_lens, - batch_size_t: batch_size, - padded_shapes_t: padded_shapes, - drop_remainder_t: drop_remainder, - }) - - num_full_batches = len(seq_lens) // batch_size - - for i in range(num_full_batches): - result = sess.run(get_next) - padded_len = padded_shapes[0] - if padded_len is None or padded_len == -1: - padded_len = np.max(result) if result.size > 0 else 0 - self.assertEqual((batch_size, padded_len), result.shape) - for j in range(batch_size): - seq_len = seq_lens[(i * batch_size) + j] - self.assertAllEqual(result[j, :seq_len], [seq_len] * seq_len) - self.assertAllEqual(result[j, seq_len:], - [0] * (padded_len - seq_len)) - - if not drop_remainder and len(seq_lens) % batch_size > 0: - result = sess.run(get_next) - padded_len = np.max(result) if result.size > 0 else 0 - self.assertEqual((len(seq_lens) % batch_size, padded_len), - result.shape) - for j in range(len(seq_lens) % batch_size): - seq_len = seq_lens[num_full_batches * batch_size + j] - self.assertAllEqual(result[j, :seq_len], [seq_len] * seq_len) - self.assertAllEqual(result[j, seq_len:], - [0] * (padded_len - seq_len)) - - with self.assertRaises(errors.OutOfRangeError): - sess.run(get_next) - - def testPaddedBatchShortPadding(self): - iterator = ( - dataset_ops.Dataset.from_tensor_slices([6, 5, 5, 5, 5]) - .map(lambda x: array_ops.fill([x], x)).padded_batch( - batch_size=4, padded_shapes=[5]).make_one_shot_iterator()) - get_next = iterator.get_next() - - with self.cached_session() as sess: - with self.assertRaises(errors.DataLossError): - sess.run(get_next) - - def testPaddedBatchEmptyTensors(self): - iterator = ( - dataset_ops.Dataset.from_tensor_slices([0, 0, 0, 0]) - .map(lambda x: array_ops.fill([x], x)).padded_batch( - batch_size=4, padded_shapes=[-1]).make_one_shot_iterator()) - get_next = iterator.get_next() - - with self.cached_session() as sess: - result = sess.run(get_next) - self.assertAllEqual([[], [], [], []], result) - with self.assertRaises(errors.OutOfRangeError): - sess.run(get_next) - - def testPaddedBatchDatasetNonDefaultPadding(self): - seq_lens = array_ops.placeholder(dtypes.int32, shape=[None]) - padded_shape = array_ops.placeholder(dtypes.int64, shape=[1]) - - def fill_tuple(x): - filled = array_ops.fill([x], x) - return (filled, string_ops.as_string(filled)) - - iterator = ( - dataset_ops.Dataset.from_tensor_slices(seq_lens).map(fill_tuple) - .padded_batch( - 4, - padded_shapes=(padded_shape, padded_shape), - padding_values=(-1, '')).make_initializable_iterator()) - - init_op = iterator.initializer - get_next = iterator.get_next() - - with self.cached_session() as sess: - # Test with random sequence lengths, and max padding. - random_seq_lens = np.random.randint(20, size=(32,)).astype(np.int32) - sess.run( - init_op, feed_dict={ - padded_shape: [-1], - seq_lens: random_seq_lens - }) - for i in range(8): - result = sess.run(get_next) - padded_len = np.max(result[0]) - self.assertEqual((4, padded_len), result[0].shape) - self.assertEqual((4, padded_len), result[1].shape) - for j in range(4): - seq_len = random_seq_lens[(i * 4) + j] - self.assertAllEqual(result[0][j, :seq_len], [seq_len] * seq_len) - self.assertAllEqual(result[0][j, seq_len:], - [-1] * (padded_len - seq_len)) - self.assertAllEqual(result[1][j, :seq_len], - [compat.as_bytes(str(seq_len))] * seq_len) - self.assertAllEqual(result[1][j, seq_len:], - [b''] * (padded_len - seq_len)) - with self.assertRaises(errors.OutOfRangeError): - sess.run(get_next) - - def testPaddedBatchDatasetUnicode(self): - # See GitHub issue 16149 - def generator(): - data = [[u'Простой', u'тест', u'юникода'], - [u'никогда', u'не', u'бывает', u'простым']] - - for seq in data: - yield seq, [0, 1, 2, 3] - - dataset = dataset_ops.Dataset.from_generator( - generator, (dtypes.string, dtypes.int32), - (tensor_shape.TensorShape([None]), tensor_shape.TensorShape([None]))) - padded_dataset = dataset.padded_batch( - 2, padded_shapes=([None], [None]), padding_values=('', 0)) - with self.cached_session() as sess: - next_element = padded_dataset.make_one_shot_iterator().get_next() - sess.run(next_element) - - def testPaddedBatchDatasetShapeSpecifications(self): - int_placeholder = array_ops.placeholder(dtypes.int32) - float_placeholder = array_ops.placeholder(dtypes.float32) - string_placeholder = array_ops.placeholder(dtypes.string) - input_dataset = dataset_ops.Dataset.from_tensors( - (int_placeholder, float_placeholder, string_placeholder)) - - # Test different ways of specifying the `padded_shapes` argument. - dynamic_padding_from_tensor_shapes = input_dataset.padded_batch( - 32, - padded_shapes=(tensor_shape.TensorShape([None]), - tensor_shape.TensorShape([None, None]), - tensor_shape.TensorShape([37]))) - dynamic_padding_from_lists = input_dataset.padded_batch( - 32, padded_shapes=([None], [None, None], [37])) - dynamic_padding_from_lists_with_minus_one = input_dataset.padded_batch( - 32, padded_shapes=([-1], [-1, -1], [37])) - dynamic_padding_from_tensors = input_dataset.padded_batch( - 32, - padded_shapes=(constant_op.constant([-1], dtype=dtypes.int64), - constant_op.constant([-1, -1], dtype=dtypes.int64), - constant_op.constant([37], dtype=dtypes.int64))) - - for dataset in [ - dynamic_padding_from_tensor_shapes, dynamic_padding_from_lists, - dynamic_padding_from_lists_with_minus_one, dynamic_padding_from_tensors - ]: - self.assertEqual([None, None], dataset.output_shapes[0].as_list()) - self.assertEqual([None, None, None], dataset.output_shapes[1].as_list()) - self.assertEqual([None, 37], dataset.output_shapes[2].as_list()) - - def testPaddedBatchSparseError(self): - - def _map_fn(i): - return sparse_tensor.SparseTensorValue( - indices=[[0, 0]], values=(i * [1]), dense_shape=[1, 1]), i - - with self.assertRaises(TypeError): - _ = dataset_ops.Dataset.range(10).map(_map_fn).padded_batch(10) - - def testPaddedBatchShapeError(self): - with self.assertRaisesRegexp( - ValueError, r'The padded shape \(1,\) is not compatible with the ' - r'corresponding input component shape \(\).'): - _ = dataset_ops.Dataset.range(10).padded_batch(5, padded_shapes=[1]) - - with self.assertRaisesRegexp( - ValueError, r'The padded shape \(1,\) is not compatible with the ' - r'corresponding input component shape \(3,\).'): - _ = dataset_ops.Dataset.from_tensors([1, 2, 3]).padded_batch( - 5, padded_shapes=[1]) - - with self.assertRaisesRegexp( - ValueError, r'Padded shape .* must be a 1-D tensor ' - r'of tf.int64 values, but its shape was \(2, 2\).'): - _ = dataset_ops.Dataset.from_tensors([1, 2, 3]).padded_batch( - 5, padded_shapes=[[1, 1], [1, 1]]) - - with self.assertRaisesRegexp( - TypeError, r'Padded shape .* must be a 1-D tensor ' - r'of tf.int64 values, but its element type was float32.'): - _ = dataset_ops.Dataset.from_tensors([1, 2, 3]).padded_batch( - 5, padded_shapes=constant_op.constant([1., 2., 3.])) - - with self.assertRaisesRegexp( - ValueError, r'The padded shape \(1,\) is not compatible with the ' - r'corresponding input component shape \(\).'): - shape_as_tensor = constant_op.constant([1], dtype=dtypes.int64) - _ = dataset_ops.Dataset.range(10).padded_batch( - 5, padded_shapes=shape_as_tensor) - - with self.assertRaisesRegexp( - ValueError, - r'The padded shape \((\?|None), (\?|None)\) is not compatible with the ' - r'corresponding input component shape \(\).'): - shape_as_tensor = array_ops.placeholder(dtypes.int64, shape=[2]) - _ = dataset_ops.Dataset.range(10).padded_batch( - 5, padded_shapes=shape_as_tensor) - - -class BatchDatasetBenchmark(test.Benchmark): - - def benchmarkBatchSparse(self): - non_zeros_per_row_values = [0, 1, 5, 10, 100] - batch_size_values = [1, 32, 64, 128, 1024] - - sparse_placeholder = array_ops.sparse_placeholder(dtype=dtypes.int64) - batch_size_placeholder = array_ops.placeholder(dtype=dtypes.int64, shape=[]) - - dataset = dataset_ops.Dataset.from_tensors(sparse_placeholder).repeat( - ).batch(batch_size_placeholder) - iterator = dataset.make_initializable_iterator() - next_element = iterator.get_next() - - for non_zeros_per_row in non_zeros_per_row_values: - - sparse_value = sparse_tensor.SparseTensorValue( - indices=np.arange(non_zeros_per_row, dtype=np.int64)[:, np.newaxis], - values=np.arange(non_zeros_per_row, dtype=np.int64), - dense_shape=[1000]) - - for batch_size in batch_size_values: - - with session.Session() as sess: - sess.run(iterator.initializer, feed_dict={ - sparse_placeholder: sparse_value, - batch_size_placeholder: batch_size}) - # Run five steps to warm up the session caches before taking the - # first measurement. - for _ in range(5): - sess.run(next_element.indices.op) - deltas = [] - for _ in range(100): - start = time.time() - for _ in range(100): - sess.run(next_element.indices.op) - end = time.time() - deltas.append(end - start) - - median_wall_time = np.median(deltas) / 100.0 - - print('Batch sparse dataset non-zeros per row: %d batch_size: %d ' - 'wall time: %f' - % (non_zeros_per_row, batch_size, median_wall_time)) - self.report_benchmark( - iters=10000, wall_time=median_wall_time, - name='benchmark_batch_sparse_dataset_nnz_%d_batch_size_%d' % ( - non_zeros_per_row, batch_size)) - - -if __name__ == '__main__': - test.main() diff --git a/tensorflow/python/data/kernel_tests/batch_test.py b/tensorflow/python/data/kernel_tests/batch_test.py new file mode 100644 index 00000000000..5b035e59173 --- /dev/null +++ b/tensorflow/python/data/kernel_tests/batch_test.py @@ -0,0 +1,173 @@ +# -*- coding: utf-8 -*- +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Tests for `tf.data.Dataset.batch()`.""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from absl.testing import parameterized +import numpy as np + +from tensorflow.python.data.kernel_tests import test_base +from tensorflow.python.data.ops import dataset_ops +from tensorflow.python.data.util import nest +from tensorflow.python.framework import dtypes +from tensorflow.python.framework import errors +from tensorflow.python.framework import sparse_tensor +from tensorflow.python.framework import test_util +from tensorflow.python.ops import array_ops +from tensorflow.python.ops import math_ops +from tensorflow.python.platform import test + + +@test_util.run_all_in_graph_and_eager_modes +class BatchTest(test_base.DatasetTestBase, parameterized.TestCase): + + @parameterized.named_parameters( + ('even', 28, 14, False), + ('uneven_with_remainder', 28, 15, False), + ('uneven_without_remainder', 28, 15, True), + ('empty', 0, 14, False), + ) + def testBatchDataset(self, count, batch_size, drop_remainder): + """Tests the batch dataset logic for various input configurations. + + Args: + count: the number of input elements + batch_size: the batch size + drop_remainder: whether a smaller batch size should be produced if batch + size does not divide number of inputs evenly + """ + + # The pipeline is TensorSliceDataset -> MapDataset(square_3) -> + # RepeatDataset(count) -> BatchDataset(batch_size). + components = (np.arange(7), + np.array([[1, 2, 3]]) * np.arange(7)[:, np.newaxis], + np.array(37.0) * np.arange(7)) + + def _map_fn(x, y, z): + return math_ops.square(x), math_ops.square(y), math_ops.square(z) + + dataset = dataset_ops.Dataset.from_tensor_slices(components).map( + _map_fn).repeat(count).batch(batch_size, drop_remainder) + get_next = self.getNext(dataset) + + if drop_remainder: + dim0 = batch_size + else: + dim0 = None + self.assertEqual( + [ts.as_list() for ts in nest.flatten(dataset.output_shapes)], + [[dim0] + list(c.shape[1:]) for c in components]) + + num_full_batches = (count * 7) // batch_size + for i in range(num_full_batches): + result = self.evaluate(get_next()) + for component, result_component in zip(components, result): + for j in range(batch_size): + self.assertAllEqual(component[(i * batch_size + j) % 7]**2, + result_component[j]) + if not drop_remainder and (count * 7) % batch_size > 0: + result = self.evaluate(get_next()) + for component, result_component in zip(components, result): + for j in range((count * 7) % batch_size): + self.assertAllEqual( + component[(num_full_batches * batch_size + j) % 7]**2, + result_component[j]) + with self.assertRaises(errors.OutOfRangeError): + result = self.evaluate(get_next()) + + def testBatchDatasetInvalidBatchSize(self): + dataset = (dataset_ops.Dataset.range(10).batch(0)) + self.assertDatasetProduces( + dataset, expected_error=(errors.InvalidArgumentError, '')) + + def testBatchSparse(self): + + def _sparse(i): + return sparse_tensor.SparseTensorValue( + indices=[[0]], values=(i * [1]), dense_shape=[1]) + + dataset = dataset_ops.Dataset.range(10).map(_sparse).batch(5) + expected_output = [ + sparse_tensor.SparseTensorValue( + indices=[[0, 0], [1, 0], [2, 0], [3, 0], [4, 0]], + values=[i * 5, i * 5 + 1, i * 5 + 2, i * 5 + 3, i * 5 + 4], + dense_shape=[5, 1]) for i in range(2) + ] + self.assertDatasetProduces(dataset, expected_output=expected_output) + + def testBatchSparseWithDifferentDenseShapes(self): + + def _sparse(i): + return sparse_tensor.SparseTensorValue( + indices=array_ops.expand_dims( + math_ops.range(i, dtype=dtypes.int64), 1), + values=array_ops.fill([math_ops.to_int32(i)], i), + dense_shape=[i]) + + dataset = dataset_ops.Dataset.range(10).map(_sparse).batch(5) + expected_output = [] + for i in range(2): + expected_indices = [] + expected_outputs = [] + for j in range(5): + for k in range(i * 5 + j): + expected_indices.append([j, k]) + expected_outputs.append(i * 5 + j) + expected_output.append( + sparse_tensor.SparseTensorValue( + indices=expected_indices, + values=expected_outputs, + dense_shape=[5, (i + 1) * 5 - 1])) + self.assertDatasetProduces(dataset, expected_output=expected_output) + + def testNestedBatchSparse(self): + + def _sparse(i): + return sparse_tensor.SparseTensorValue( + indices=[[0]], values=(i * [1]), dense_shape=[1]) + + dataset = dataset_ops.Dataset.range(10).map(_sparse).batch(5).batch(2) + expected_output = [ + sparse_tensor.SparseTensorValue( + indices=[[0, 0, 0], [0, 1, 0], [0, 2, 0], [0, 3, 0], [0, 4, 0], + [1, 0, 0], [1, 1, 0], [1, 2, 0], [1, 3, 0], [1, 4, 0]], + values=[0, 1, 2, 3, 4, 5, 6, 7, 8, 9], + dense_shape=[2, 5, 1]) + ] + self.assertDatasetProduces(dataset, expected_output=expected_output) + + def testBatchShapeError(self): + + def generator(): + yield [1.0, 2.0, 3.0] + yield [4.0, 5.0, 6.0] + yield [7.0, 8.0, 9.0, 10.0] + + dataset = ( + dataset_ops.Dataset.from_generator( + generator, dtypes.float32, output_shapes=[None]).batch(3)) + self.assertDatasetProduces( + dataset, + expected_error=( + errors.InvalidArgumentError, + r'Cannot batch tensors with different shapes in component 0. First ' + r'element had shape \[3\] and element 2 had shape \[4\].')) + + +if __name__ == '__main__': + test.main() diff --git a/tensorflow/python/data/kernel_tests/cache_dataset_op_test.py b/tensorflow/python/data/kernel_tests/cache_dataset_op_test.py deleted file mode 100644 index 63625fac03b..00000000000 --- a/tensorflow/python/data/kernel_tests/cache_dataset_op_test.py +++ /dev/null @@ -1,318 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for the experimental input pipeline ops.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from os import path -import shutil -import tempfile - -import numpy as np - -from tensorflow.python.data.kernel_tests import test_base -from tensorflow.python.data.ops import dataset_ops -from tensorflow.python.data.ops import iterator_ops -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import errors -from tensorflow.python.framework import ops -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import variables -from tensorflow.python.platform import test - - -class FileCacheDatasetTest(test_base.DatasetTestBase): - - def setUp(self): - self.tmp_dir = tempfile.mkdtemp() - self.cache_prefix = path.join(self.tmp_dir, "cache") - - def tearDown(self): - if self.tmp_dir: - shutil.rmtree(self.tmp_dir, ignore_errors=True) - - def testCacheDatasetPassthrough(self): - components = (np.array([1, 2, 3, 4]), np.array([5, 6, 7, 8]), - np.array([9.0, 10.0, 11.0, 12.0])) - count_placeholder = array_ops.placeholder_with_default( - constant_op.constant(5, dtypes.int64), shape=[]) - filename_placeholder = array_ops.placeholder(dtypes.string, shape=[]) - - repeat_dataset = (dataset_ops.Dataset.from_tensor_slices(components) - .repeat(count_placeholder)) - - cache_dataset = repeat_dataset.cache(filename_placeholder) - - self.assertEqual( - tuple([c.shape[1:] for c in components]), cache_dataset.output_shapes) - - # Create initialization ops for iterators without and with - # caching, respectively. - iterator = iterator_ops.Iterator.from_structure(cache_dataset.output_types, - cache_dataset.output_shapes) - init_fifo_op = iterator.make_initializer(repeat_dataset) - init_cache_op = iterator.make_initializer(cache_dataset) - - get_next = iterator.get_next() - - with self.cached_session() as sess: - # First run without caching to collect the "ground truth". - sess.run(init_fifo_op) - elements = [] - for _ in range(20): - elements.append(sess.run(get_next)) - with self.assertRaises(errors.OutOfRangeError): - sess.run(get_next) - - # Assert that the cached dataset has the same elements as the - # "ground truth". - sess.run( - init_cache_op, feed_dict={filename_placeholder: self.cache_prefix}) - cached_elements = [] - for _ in range(20): - cached_elements.append(sess.run(get_next)) - with self.assertRaises(errors.OutOfRangeError): - sess.run(get_next) - self.assertAllEqual(elements, cached_elements) - - # Re-initialize with an empty upstream (to throw errors.OutOfRangeError - # if we didn't use the cache). - sess.run( - init_cache_op, - feed_dict={ - count_placeholder: 0, - filename_placeholder: self.cache_prefix - }) - replayed_elements = [] - for _ in range(20): - replayed_elements.append(sess.run(get_next)) - with self.assertRaises(errors.OutOfRangeError): - sess.run(get_next) - self.assertEqual(cached_elements, replayed_elements) - - # Re-initialize with an empty upstream and a missing cache file (should - # throw errors.OutOfRangeError immediately). - sess.run( - init_cache_op, - feed_dict={ - count_placeholder: 0, - filename_placeholder: self.cache_prefix + "nonsense" - }) - with self.assertRaises(errors.OutOfRangeError): - sess.run(get_next) - - def testConcurrentWriters(self): - components = (np.array([1, 2, 3, 4]), np.array([5, 6, 7, 8]), - np.array([9.0, 10.0, 11.0, 12.0])) - filename_placeholder = array_ops.placeholder(dtypes.string, shape=[]) - - cache_dataset1 = (dataset_ops.Dataset.from_tensor_slices(components) - .cache(filename_placeholder)) - cache_dataset2 = (dataset_ops.Dataset.from_tensor_slices(components) - .cache(filename_placeholder)) - - iterator1 = cache_dataset1.make_initializable_iterator() - iterator2 = cache_dataset2.make_initializable_iterator() - init_cache_op1 = iterator1.initializer - init_cache_op2 = iterator2.initializer - - get_next1 = iterator1.get_next() - get_next2 = iterator2.get_next() - - with self.cached_session() as sess: - sess.run( - init_cache_op1, feed_dict={filename_placeholder: self.cache_prefix}) - sess.run(get_next1) # this should succeed - - sess.run( - init_cache_op2, feed_dict={filename_placeholder: self.cache_prefix}) - with self.assertRaises(errors.AlreadyExistsError): - sess.run(get_next2) - - sess.run(get_next1) # this should continue to succeed - - def testConcurrentReaders(self): - components = (np.array([1, 2, 3, 4]), np.array([5, 6, 7, 8]), - np.array([9.0, 10.0, 11.0, 12.0])) - filename_placeholder = array_ops.placeholder(dtypes.string, shape=[]) - - cache_dataset1 = (dataset_ops.Dataset.from_tensor_slices(components) - .cache(filename_placeholder)) - cache_dataset2 = (dataset_ops.Dataset.from_tensor_slices(components) - .cache(filename_placeholder)) - - iterator1 = cache_dataset1.make_initializable_iterator() - iterator2 = cache_dataset2.make_initializable_iterator() - init_cache_op1 = iterator1.initializer - init_cache_op2 = iterator2.initializer - - get_next1 = iterator1.get_next() - get_next2 = iterator2.get_next() - - with self.cached_session() as sess: - sess.run( - init_cache_op1, feed_dict={filename_placeholder: self.cache_prefix}) - elements = [] - for _ in range(4): - elements.append(sess.run(get_next1)) - with self.assertRaises(errors.OutOfRangeError): - sess.run(get_next1) - - # Re-initialize - sess.run( - init_cache_op1, feed_dict={filename_placeholder: self.cache_prefix}) - sess.run( - init_cache_op2, feed_dict={filename_placeholder: self.cache_prefix}) - - # Reading concurrently should succeed. - elements_itr1 = [] - elements_itr2 = [] - elements_itr2.append(sess.run(get_next2)) - elements_itr1.append(sess.run(get_next1)) - elements_itr2.append(sess.run(get_next2)) - elements_itr1.append(sess.run(get_next1)) - # Intentionally reversing the order - elements_itr1.append(sess.run(get_next1)) - elements_itr2.append(sess.run(get_next2)) - elements_itr1.append(sess.run(get_next1)) - elements_itr2.append(sess.run(get_next2)) - - with self.assertRaises(errors.OutOfRangeError): - sess.run(get_next2) - - with self.assertRaises(errors.OutOfRangeError): - sess.run(get_next1) - - self.assertAllEqual(elements, elements_itr1) - self.assertAllEqual(elements, elements_itr2) - - -class MemoryCacheDatasetTest(test_base.DatasetTestBase): - - def testCacheDatasetPassthrough(self): - with ops.device("cpu:0"): - repeat_count = variables.Variable(constant_op.constant(10, dtypes.int64)) - dataset = dataset_ops.Dataset.range(3).flat_map( - lambda x: dataset_ops.Dataset.from_tensors(x).repeat(repeat_count)) - - cached_dataset = dataset.cache().repeat(2) - uncached_dataset = dataset.repeat(2) - - # Needs to be initializable to capture the variable. - cached_iterator = cached_dataset.make_initializable_iterator() - cached_next = cached_iterator.get_next() - uncached_iterator = uncached_dataset.make_initializable_iterator() - uncached_next = uncached_iterator.get_next() - - with self.cached_session() as sess: - - sess.run(repeat_count.initializer) - sess.run(cached_iterator.initializer) - sess.run(uncached_iterator.initializer) - - for i in range(3): - for _ in range(10): - self.assertEqual(sess.run(cached_next), i) - self.assertEqual(sess.run(uncached_next), i) - - sess.run(repeat_count.assign(0)) - - # The uncached iterator should now be empty. - with self.assertRaises(errors.OutOfRangeError): - sess.run(uncached_next) - - # The cached iterator replays from cache. - for i in range(3): - for _ in range(10): - self.assertEqual(sess.run(cached_next), i) - - # The cached iterator should now be empty. - with self.assertRaises(errors.OutOfRangeError): - sess.run(cached_next) - - def testEmptyCacheReading(self): - components = (np.array([1, 2, 3, 4]), np.array([5, 6, 7, 8]), - np.array([9.0, 10.0, 11.0, 12.0])) - count_placeholder = array_ops.placeholder_with_default( - constant_op.constant(5, dtypes.int64), shape=[]) - - repeat_dataset = (dataset_ops.Dataset.from_tensor_slices(components) - .repeat(count_placeholder)) - - cache_dataset = repeat_dataset.cache() - - # Create initialization ops for iterators without and with - # caching, respectively. - iterator = cache_dataset.make_initializable_iterator() - init_cache_op = iterator.initializer - - get_next = iterator.get_next() - - with self.cached_session() as sess: - # Initialize with an empty upstream and a missing cache file (should - # throw errors.OutOfRangeError immediately). - sess.run(init_cache_op, feed_dict={count_placeholder: 0}) - with self.assertRaises(errors.OutOfRangeError): - sess.run(get_next) - - def testConcurrentReaders(self): - count_placeholder = array_ops.placeholder_with_default( - constant_op.constant(5, dtypes.int64), shape=[]) - dataset = dataset_ops.Dataset.range(count_placeholder).cache() - d1 = dataset.map(lambda x: x + 1) - d2 = dataset.map(lambda x: x + 6) - - i1 = d1.make_initializable_iterator() - i2 = d2.make_initializable_iterator() - - with self.cached_session() as sess: - sess.run(i1.initializer) - - self.assertEqual(1, sess.run(i1.get_next())) - self.assertEqual(2, sess.run(i1.get_next())) - self.assertEqual(3, sess.run(i1.get_next())) - - sess.run(i2.initializer, feed_dict={count_placeholder: 3}) - - self.assertEqual(6, sess.run(i2.get_next())) - self.assertEqual(7, sess.run(i2.get_next())) - self.assertEqual(4, sess.run(i1.get_next())) # interleave execution - self.assertEqual([8, 5], sess.run([i2.get_next(), i1.get_next()])) - - with self.assertRaises(errors.OutOfRangeError): - sess.run(i1.get_next()) - with self.assertRaises(errors.OutOfRangeError): - sess.run(i2.get_next()) - - def testCacheTakeRepeat(self): - dataset = dataset_ops.Dataset.range(10).cache().take(5).repeat(2) - itr = dataset.make_one_shot_iterator() - n = itr.get_next() - - expected_values = [0, 1, 2, 3, 4, 0, 1, 2, 3, 4] - - with self.cached_session() as sess: - for i, expected in enumerate(expected_values): - self.assertEqual(expected, sess.run(n), - "Unexpected value at index %s" % i) - - with self.assertRaises(errors.OutOfRangeError): - sess.run(itr.get_next()) - - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/python/data/kernel_tests/cache_test.py b/tensorflow/python/data/kernel_tests/cache_test.py new file mode 100644 index 00000000000..b561cd58baf --- /dev/null +++ b/tensorflow/python/data/kernel_tests/cache_test.py @@ -0,0 +1,253 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Tests for `tf.data.Dataset.cache()`.""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from os import path +import shutil +import tempfile + +import numpy as np + +from tensorflow.python.data.kernel_tests import test_base +from tensorflow.python.data.ops import dataset_ops +from tensorflow.python.framework import constant_op +from tensorflow.python.framework import dtypes +from tensorflow.python.framework import errors +from tensorflow.python.framework import ops +from tensorflow.python.framework import test_util +from tensorflow.python.ops import variables +from tensorflow.python.platform import test + + +@test_util.run_all_in_graph_and_eager_modes +class FileCacheTest(test_base.DatasetTestBase): + + def setUp(self): + self.tmp_dir = tempfile.mkdtemp() + self.cache_prefix = path.join(self.tmp_dir, "cache") + + def tearDown(self): + if self.tmp_dir: + shutil.rmtree(self.tmp_dir, ignore_errors=True) + + def testCacheDatasetPassthrough(self): + components = (np.array([1, 2, 3, 4]), np.array([5, 6, 7, 8]), + np.array([9.0, 10.0, 11.0, 12.0])) + + def dataset_fn(count=5, filename=None): + repeat_dataset = ( + dataset_ops.Dataset.from_tensor_slices(components).repeat(count)) + if filename: + return repeat_dataset.cache(filename) + else: + return repeat_dataset + + self.assertEqual( + tuple([c.shape[1:] for c in components]), + dataset_fn().output_shapes) + + get_next = self.getNext(dataset_fn()) + + # First run without caching to collect the "ground truth". + elements = [] + for _ in range(20): + elements.append(self.evaluate(get_next())) + with self.assertRaises(errors.OutOfRangeError): + self.evaluate(get_next()) + + # Assert that the cached dataset has the same elements as the + # "ground truth". + get_next = self.getNext(dataset_fn(filename=self.cache_prefix)) + cached_elements = [] + for _ in range(20): + cached_elements.append(self.evaluate(get_next())) + with self.assertRaises(errors.OutOfRangeError): + self.evaluate(get_next()) + self.assertAllEqual(elements, cached_elements) + + # Re-initialize with an empty upstream (to throw errors.OutOfRangeError + # if we didn't use the cache). + get_next = self.getNext(dataset_fn(count=0, filename=self.cache_prefix)) + replayed_elements = [] + for _ in range(20): + replayed_elements.append(self.evaluate(get_next())) + with self.assertRaises(errors.OutOfRangeError): + self.evaluate(get_next()) + self.assertEqual(cached_elements, replayed_elements) + + # Re-initialize with an empty upstream and a missing cache file (should + # throw errors.OutOfRangeError immediately). + get_next = self.getNext( + dataset_fn(count=0, filename=self.cache_prefix + "nonsense")) + with self.assertRaises(errors.OutOfRangeError): + self.evaluate(get_next()) + + def testConcurrentWriters(self): + components = (np.array([1, 2, 3, 4]), np.array([5, 6, 7, 8]), + np.array([9.0, 10.0, 11.0, 12.0])) + + cache_dataset1 = ( + dataset_ops.Dataset.from_tensor_slices(components).cache( + self.cache_prefix)) + cache_dataset2 = ( + dataset_ops.Dataset.from_tensor_slices(components).cache( + self.cache_prefix)) + + get_next1 = self.getNext(cache_dataset1) + get_next2 = self.getNext(cache_dataset2) + + self.evaluate(get_next1()) # this should succeed + + with self.assertRaises(errors.AlreadyExistsError): + self.evaluate(get_next2()) + + self.evaluate(get_next1()) # this should continue to succeed + + def testConcurrentReaders(self): + components = (np.array([1, 2, 3, 4]), np.array([5, 6, 7, 8]), + np.array([9.0, 10.0, 11.0, 12.0])) + + cache_dataset1 = ( + dataset_ops.Dataset.from_tensor_slices(components).cache( + self.cache_prefix)) + cache_dataset2 = ( + dataset_ops.Dataset.from_tensor_slices(components).cache( + self.cache_prefix)) + + get_next1 = self.getNext(cache_dataset1) + get_next2 = self.getNext(cache_dataset2) + + elements = [] + for _ in range(4): + elements.append(self.evaluate(get_next1())) + with self.assertRaises(errors.OutOfRangeError): + self.evaluate(get_next1()) + + # Re-initialize + get_next1 = self.getNext(cache_dataset1) + get_next2 = self.getNext(cache_dataset2) + + # Reading concurrently should succeed. + elements_itr1 = [] + elements_itr2 = [] + elements_itr2.append(self.evaluate(get_next2())) + elements_itr1.append(self.evaluate(get_next1())) + elements_itr2.append(self.evaluate(get_next2())) + elements_itr1.append(self.evaluate(get_next1())) + # Intentionally reversing the order + elements_itr1.append(self.evaluate(get_next1())) + elements_itr2.append(self.evaluate(get_next2())) + elements_itr1.append(self.evaluate(get_next1())) + elements_itr2.append(self.evaluate(get_next2())) + + with self.assertRaises(errors.OutOfRangeError): + self.evaluate(get_next2()) + + with self.assertRaises(errors.OutOfRangeError): + self.evaluate(get_next1()) + + self.assertAllEqual(elements, elements_itr1) + self.assertAllEqual(elements, elements_itr2) + + +@test_util.run_all_in_graph_and_eager_modes +class MemoryCacheTest(test_base.DatasetTestBase): + + def testCacheDatasetPassthrough(self): + with ops.device("cpu:0"): + repeat_count = variables.Variable(constant_op.constant(10, dtypes.int64)) + dataset = dataset_ops.Dataset.range(3).flat_map( + lambda x: dataset_ops.Dataset.from_tensors(x).repeat(repeat_count)) + + cached_dataset = dataset.cache().repeat(2) + uncached_dataset = dataset.repeat(2) + + self.evaluate(repeat_count.initializer) + # Needs to be initializable to capture the variable. + cached_next = self.getNext(cached_dataset, requires_initialization=True) + uncached_next = self.getNext( + uncached_dataset, requires_initialization=True) + for i in range(3): + for _ in range(10): + self.assertEqual(self.evaluate(cached_next()), i) + self.assertEqual(self.evaluate(uncached_next()), i) + + self.evaluate(repeat_count.assign(0)) + + # The uncached iterator should now be empty. + with self.assertRaises(errors.OutOfRangeError): + self.evaluate(uncached_next()) + + # The cached iterator replays from cache. + for i in range(3): + for _ in range(10): + self.assertEqual(self.evaluate(cached_next()), i) + + # The cached iterator should now be empty. + with self.assertRaises(errors.OutOfRangeError): + self.evaluate(cached_next()) + + def testEmptyCacheReading(self): + components = (np.array([1, 2, 3, 4]), np.array([5, 6, 7, 8]), + np.array([9.0, 10.0, 11.0, 12.0])) + + repeat_dataset = ( + dataset_ops.Dataset.from_tensor_slices(components).repeat(0)) + cache_dataset = repeat_dataset.cache() + + # Create initialization ops for iterators without and with + # caching, respectively. + self.assertDatasetProduces(cache_dataset, expected_output=[]) + + def testConcurrentReaders(self): + + dataset = dataset_ops.Dataset.range(5).cache() + d1 = dataset.map(lambda x: x + 1) + d2 = dataset.map(lambda x: x + 6) + + get_next1 = self.getNext(d1) + + self.assertEqual(1, self.evaluate(get_next1())) + self.assertEqual(2, self.evaluate(get_next1())) + self.assertEqual(3, self.evaluate(get_next1())) + + get_next2 = self.getNext(d2) + + self.assertEqual(6, self.evaluate(get_next2())) + self.assertEqual(7, self.evaluate(get_next2())) + self.assertEqual(4, self.evaluate(get_next1())) # interleave execution + self.assertEqual([8, 5], + [self.evaluate(get_next2()), + self.evaluate(get_next1())]) + self.assertEqual(9, self.evaluate(get_next2())) + self.assertEqual(10, self.evaluate(get_next2())) + + with self.assertRaises(errors.OutOfRangeError): + self.evaluate(get_next2()) + with self.assertRaises(errors.OutOfRangeError): + self.evaluate(get_next1()) + + def testCacheTakeRepeat(self): + dataset = dataset_ops.Dataset.range(10).cache().take(5).repeat(2) + + expected_output = [0, 1, 2, 3, 4, 0, 1, 2, 3, 4] + self.assertDatasetProduces(dataset, expected_output=expected_output) + + +if __name__ == "__main__": + test.main() diff --git a/tensorflow/python/data/kernel_tests/concatenate_dataset_op_test.py b/tensorflow/python/data/kernel_tests/concatenate_test.py similarity index 75% rename from tensorflow/python/data/kernel_tests/concatenate_dataset_op_test.py rename to tensorflow/python/data/kernel_tests/concatenate_test.py index 83af31f380e..5d8bfdc8f3a 100644 --- a/tensorflow/python/data/kernel_tests/concatenate_dataset_op_test.py +++ b/tensorflow/python/data/kernel_tests/concatenate_test.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================== -"""Tests for the experimental input pipeline ops.""" +"""Tests for `tf.data.Dataset.concatenate().""" from __future__ import absolute_import from __future__ import division from __future__ import print_function @@ -24,10 +24,12 @@ from tensorflow.python.data.ops import dataset_ops from tensorflow.python.data.util import nest from tensorflow.python.framework import errors from tensorflow.python.framework import tensor_shape +from tensorflow.python.framework import test_util from tensorflow.python.platform import test -class ConcatenateDatasetTest(test_base.DatasetTestBase): +@test_util.run_all_in_graph_and_eager_modes +class ConcatenateTest(test_base.DatasetTestBase): def testConcatenateDataset(self): input_components = ( @@ -46,23 +48,19 @@ class ConcatenateDatasetTest(test_base.DatasetTestBase): self.assertEqual(concatenated.output_shapes, (tensor_shape.TensorShape( [20]), tensor_shape.TensorShape([15]), tensor_shape.TensorShape([]))) - iterator = concatenated.make_initializable_iterator() - init_op = iterator.initializer - get_next = iterator.get_next() + get_next = self.getNext(concatenated) - with self.cached_session() as sess: - sess.run(init_op) - for i in range(9): - result = sess.run(get_next) - if i < 4: - for component, result_component in zip(input_components, result): - self.assertAllEqual(component[i], result_component) - else: - for component, result_component in zip(to_concatenate_components, - result): - self.assertAllEqual(component[i - 4], result_component) - with self.assertRaises(errors.OutOfRangeError): - sess.run(get_next) + for i in range(9): + result = self.evaluate(get_next()) + if i < 4: + for component, result_component in zip(input_components, result): + self.assertAllEqual(component[i], result_component) + else: + for component, result_component in zip(to_concatenate_components, + result): + self.assertAllEqual(component[i - 4], result_component) + with self.assertRaises(errors.OutOfRangeError): + self.evaluate(get_next()) def testConcatenateDatasetDifferentShape(self): input_components = ( @@ -79,24 +77,18 @@ class ConcatenateDatasetTest(test_base.DatasetTestBase): self.assertEqual( [ts.as_list() for ts in nest.flatten(concatenated.output_shapes)], [[20], [None]]) - - iterator = concatenated.make_initializable_iterator() - init_op = iterator.initializer - get_next = iterator.get_next() - - with self.cached_session() as sess: - sess.run(init_op) - for i in range(9): - result = sess.run(get_next) - if i < 4: - for component, result_component in zip(input_components, result): - self.assertAllEqual(component[i], result_component) - else: - for component, result_component in zip(to_concatenate_components, - result): - self.assertAllEqual(component[i - 4], result_component) - with self.assertRaises(errors.OutOfRangeError): - sess.run(get_next) + get_next = self.getNext(concatenated) + for i in range(9): + result = self.evaluate(get_next()) + if i < 4: + for component, result_component in zip(input_components, result): + self.assertAllEqual(component[i], result_component) + else: + for component, result_component in zip(to_concatenate_components, + result): + self.assertAllEqual(component[i - 4], result_component) + with self.assertRaises(errors.OutOfRangeError): + self.evaluate(get_next()) def testConcatenateDatasetDifferentStructure(self): input_components = ( diff --git a/tensorflow/python/data/kernel_tests/range_dataset_op_test.py b/tensorflow/python/data/kernel_tests/dataset_checkpoint_test.py similarity index 84% rename from tensorflow/python/data/kernel_tests/range_dataset_op_test.py rename to tensorflow/python/data/kernel_tests/dataset_checkpoint_test.py index b71e6b2ea43..6dcd94ea020 100644 --- a/tensorflow/python/data/kernel_tests/range_dataset_op_test.py +++ b/tensorflow/python/data/kernel_tests/dataset_checkpoint_test.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================== -"""Test RangeDataset.""" +"""Checkpoint tests for `tf.data.Dataset`.""" from __future__ import absolute_import from __future__ import division from __future__ import print_function @@ -26,7 +26,6 @@ from tensorflow.python.framework import dtypes from tensorflow.python.framework import errors from tensorflow.python.framework import ops from tensorflow.python.framework import tensor_shape -from tensorflow.python.framework import test_util from tensorflow.python.ops import gen_dataset_ops from tensorflow.python.ops import io_ops from tensorflow.python.ops import parsing_ops @@ -35,51 +34,7 @@ from tensorflow.python.platform import gfile from tensorflow.python.platform import test -@test_util.run_all_in_graph_and_eager_modes -class RangeDatasetTest(test_base.DatasetTestBase): - - def testStop(self): - dataset = dataset_ops.Dataset.range(5) - self.assertDatasetProduces(dataset, expected_output=range(5)) - - def testStartStop(self): - start, stop = 2, 5 - dataset = dataset_ops.Dataset.range(start, stop) - self.assertDatasetProduces(dataset, expected_output=range(2, 5)) - - def testStartStopStep(self): - start, stop, step = 2, 10, 2 - dataset = dataset_ops.Dataset.range(start, stop, step) - self.assertDatasetProduces(dataset, expected_output=range(2, 10, 2)) - - def testZeroStep(self): - start, stop, step = 2, 10, 0 - dataset = dataset_ops.Dataset.range(start, stop, step) - self.assertDatasetProduces( - dataset, expected_err=(errors.InvalidArgumentError, "")) - - def testNegativeStep(self): - start, stop, step = 2, 10, -1 - dataset = dataset_ops.Dataset.range(start, stop, step) - self.assertDatasetProduces(dataset, expected_output=range(2, 10, -1)) - - def testStopLessThanStart(self): - start, stop = 10, 2 - dataset = dataset_ops.Dataset.range(start, stop) - self.assertDatasetProduces(dataset, expected_output=range(10, 2)) - - def testStopLessThanStartWithPositiveStep(self): - start, stop, step = 10, 2, 2 - dataset = dataset_ops.Dataset.range(start, stop, step) - self.assertDatasetProduces(dataset, expected_output=range(10, 2, 2)) - - def testStopLessThanStartWithNegativeStep(self): - start, stop, step = 10, 2, -1 - dataset = dataset_ops.Dataset.range(start, stop, step) - self.assertDatasetProduces(dataset, expected_output=range(10, 2, -1)) - - -class ExperimentalCheckpointDatasetTest(test_base.DatasetTestBase): +class DatasetCheckpointTest(test_base.DatasetTestBase): def tearDown(self): # Remove all checkpoint files. @@ -109,8 +64,8 @@ class ExperimentalCheckpointDatasetTest(test_base.DatasetTestBase): def testSaveRestore(self): def _build_graph(start, stop): - iterator = dataset_ops.Dataset.range(start, - stop).make_initializable_iterator() + iterator = dataset_ops.make_initializable_iterator( + dataset_ops.Dataset.range(start, stop)) init_op = iterator.initializer get_next = iterator.get_next() save_op = self._save_op(iterator._iterator_resource) @@ -159,7 +114,7 @@ class ExperimentalCheckpointDatasetTest(test_base.DatasetTestBase): def _build_graph(start, stop, num_epochs): dataset = dataset_ops.Dataset.range(start, stop).repeat(num_epochs) - iterator = dataset.make_initializable_iterator() + iterator = dataset_ops.make_initializable_iterator(dataset) init_op = iterator.initializer get_next = iterator.get_next() save_op = self._save_op(iterator._iterator_resource) @@ -206,7 +161,7 @@ class ExperimentalCheckpointDatasetTest(test_base.DatasetTestBase): def _build_graph(start, stop): dataset = dataset_ops.Dataset.range(start, stop) - iterator = dataset.make_initializable_iterator() + iterator = dataset_ops.make_initializable_iterator(dataset) init_op = iterator.initializer get_next = iterator.get_next() save_op = self._save_op(iterator._iterator_resource) @@ -245,7 +200,7 @@ class ExperimentalCheckpointDatasetTest(test_base.DatasetTestBase): def _build_graph(start, stop): dataset = dataset_ops.Dataset.range(start, stop) - iterator = dataset.make_initializable_iterator() + iterator = dataset_ops.make_initializable_iterator(dataset) init_op = iterator.initializer get_next = iterator.get_next() save_op = self._save_op(iterator._iterator_resource) @@ -278,8 +233,8 @@ class ExperimentalCheckpointDatasetTest(test_base.DatasetTestBase): def testMultipleSaves(self): def _build_graph(start, stop): - iterator = dataset_ops.Dataset.range(start, - stop).make_initializable_iterator() + iterator = dataset_ops.make_initializable_iterator( + dataset_ops.Dataset.range(start, stop)) init_op = iterator.initializer get_next = iterator.get_next() save_op = self._save_op(iterator._iterator_resource) @@ -321,8 +276,8 @@ class ExperimentalCheckpointDatasetTest(test_base.DatasetTestBase): def testSaveRestoreWithRepeat(self): def _build_graph(start, stop, num_epochs): - iterator = dataset_ops.Dataset.range( - start, stop).repeat(num_epochs).make_initializable_iterator() + iterator = dataset_ops.make_initializable_iterator( + dataset_ops.Dataset.range(start, stop).repeat(num_epochs)) init_op = iterator.initializer get_next = iterator.get_next() save_op = self._save_op(iterator._iterator_resource) @@ -366,8 +321,8 @@ class ExperimentalCheckpointDatasetTest(test_base.DatasetTestBase): def testSaveRestoreExhaustedIterator(self): def _build_graph(start, stop, num_epochs): - iterator = dataset_ops.Dataset.range( - start, stop).repeat(num_epochs).make_initializable_iterator() + iterator = dataset_ops.make_initializable_iterator( + dataset_ops.Dataset.range(start, stop).repeat(num_epochs)) init_op = iterator.initializer get_next = iterator.get_next() save_op = self._save_op(iterator._iterator_resource) diff --git a/tensorflow/python/data/kernel_tests/dataset_constructor_op_test.py b/tensorflow/python/data/kernel_tests/dataset_constructor_op_test.py deleted file mode 100644 index bc6b36285aa..00000000000 --- a/tensorflow/python/data/kernel_tests/dataset_constructor_op_test.py +++ /dev/null @@ -1,650 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for the experimental input pipeline ops.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import time - -import numpy as np - -from tensorflow.core.protobuf import config_pb2 -from tensorflow.python.client import session -from tensorflow.python.data.kernel_tests import test_base -from tensorflow.python.data.ops import dataset_ops -from tensorflow.python.data.util import nest -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import errors -from tensorflow.python.framework import ops -from tensorflow.python.framework import sparse_tensor -from tensorflow.python.framework import tensor_shape -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import resource_variable_ops -from tensorflow.python.platform import test - - -class DatasetConstructorTest(test_base.DatasetTestBase): - - def testFromTensors(self): - """Test a dataset that represents a single tuple of tensors.""" - components = (np.array(1), np.array([1, 2, 3]), np.array(37.0)) - - iterator = (dataset_ops.Dataset.from_tensors(components) - .make_initializable_iterator()) - init_op = iterator.initializer - get_next = iterator.get_next() - - self.assertEqual([c.shape for c in components], - [t.shape for t in get_next]) - - with self.cached_session() as sess: - sess.run(init_op) - results = sess.run(get_next) - for component, result_component in zip(components, results): - self.assertAllEqual(component, result_component) - with self.assertRaises(errors.OutOfRangeError): - sess.run(get_next) - - def testFromTensorsSparse(self): - """Test a dataset that represents a single tuple of tensors.""" - components = (sparse_tensor.SparseTensorValue( - indices=np.array([[0]]), - values=np.array([0]), - dense_shape=np.array([1])), - sparse_tensor.SparseTensorValue( - indices=np.array([[0, 0], [1, 1]]), - values=np.array([-1, 1]), - dense_shape=np.array([2, 2]))) - - iterator = ( - dataset_ops.Dataset.from_tensors(components) - .make_initializable_iterator()) - init_op = iterator.initializer - get_next = iterator.get_next() - - self.assertEqual( - [tensor_shape.TensorShape(c.dense_shape) for c in components], - [shape for shape in iterator.output_shapes]) - - with self.cached_session() as sess: - sess.run(init_op) - results = sess.run(get_next) - for component, result_component in zip(components, results): - self.assertSparseValuesEqual(component, result_component) - with self.assertRaises(errors.OutOfRangeError): - sess.run(get_next) - - def testFromTensorsMixed(self): - """Test an dataset that represents a single tuple of tensors.""" - components = (np.array(1), np.array([1, 2, 3]), np.array(37.0), - sparse_tensor.SparseTensorValue( - indices=np.array([[0]]), - values=np.array([0]), - dense_shape=np.array([1])), - sparse_tensor.SparseTensorValue( - indices=np.array([[0, 0], [1, 1]]), - values=np.array([-1, 1]), - dense_shape=np.array([2, 2]))) - - iterator = ( - dataset_ops.Dataset.from_tensors(components) - .make_initializable_iterator()) - init_op = iterator.initializer - get_next = iterator.get_next() - - self.assertEqual([ - tensor_shape.TensorShape(c.dense_shape) - if sparse_tensor.is_sparse(c) else c.shape for c in components - ], [shape for shape in iterator.output_shapes]) - - with self.cached_session() as sess: - sess.run(init_op) - results = sess.run(get_next) - for component, result_component in zip(components, results): - if sparse_tensor.is_sparse(component): - self.assertSparseValuesEqual(component, result_component) - else: - self.assertAllEqual(component, result_component) - with self.assertRaises(errors.OutOfRangeError): - sess.run(get_next) - - def testFromTensorSlices(self): - """Test a dataset that represents the slices from a tuple of tensors.""" - components = ( - np.tile(np.array([[1], [2], [3], [4]]), 20), np.tile( - np.array([[12], [13], [14], [15]]), 22), - np.array([37.0, 38.0, 39.0, 40.0]) - ) - - iterator = (dataset_ops.Dataset.from_tensor_slices(components) - .make_initializable_iterator()) - init_op = iterator.initializer - get_next = iterator.get_next() - - self.assertEqual([c.shape[1:] for c in components], - [t.shape for t in get_next]) - - with self.cached_session() as sess: - sess.run(init_op) - for i in range(4): - results = sess.run(get_next) - for component, result_component in zip(components, results): - self.assertAllEqual(component[i], result_component) - with self.assertRaises(errors.OutOfRangeError): - sess.run(get_next) - - def testFromTensorSlicesSparse(self): - """Test a dataset that represents the slices from a tuple of tensors.""" - components = (sparse_tensor.SparseTensorValue( - indices=np.array([[0, 0], [1, 0], [2, 0]]), - values=np.array([0, 0, 0]), - dense_shape=np.array([3, 1])), - sparse_tensor.SparseTensorValue( - indices=np.array([[0, 0], [1, 1], [2, 2]]), - values=np.array([1, 2, 3]), - dense_shape=np.array([3, 3]))) - - iterator = ( - dataset_ops.Dataset.from_tensor_slices(components) - .make_initializable_iterator()) - init_op = iterator.initializer - get_next = iterator.get_next() - - self.assertEqual( - [tensor_shape.TensorShape(c.dense_shape[1:]) for c in components], - [shape for shape in iterator.output_shapes]) - - with self.cached_session() as sess: - sess.run(init_op) - expected = [ - (sparse_tensor.SparseTensorValue( - indices=np.array([[0]]), - values=np.array([0]), - dense_shape=np.array([1])), - sparse_tensor.SparseTensorValue( - indices=np.array([[0]]), - values=np.array([1]), - dense_shape=np.array([3]))), - (sparse_tensor.SparseTensorValue( - indices=np.array([[0]]), - values=np.array([0]), - dense_shape=np.array([1])), - sparse_tensor.SparseTensorValue( - indices=np.array([[1]]), - values=np.array([2]), - dense_shape=np.array([3]))), - (sparse_tensor.SparseTensorValue( - indices=np.array([[0]]), - values=np.array([0]), - dense_shape=np.array([1])), - sparse_tensor.SparseTensorValue( - indices=np.array([[2]]), - values=np.array([3]), - dense_shape=np.array([3]))), - ] - for i in range(3): - results = sess.run(get_next) - for component, result_component in zip(expected[i], results): - self.assertSparseValuesEqual(component, result_component) - with self.assertRaises(errors.OutOfRangeError): - sess.run(get_next) - - def testFromTensorSlicesMixed(self): - """Test a dataset that represents the slices from a tuple of tensors.""" - components = (np.tile(np.array([[1], [2], [3]]), 20), - np.tile(np.array([[12], [13], [14]]), 22), - np.array([37.0, 38.0, 39.0]), - sparse_tensor.SparseTensorValue( - indices=np.array([[0, 0], [1, 0], [2, 0]]), - values=np.array([0, 0, 0]), - dense_shape=np.array([3, 1])), - sparse_tensor.SparseTensorValue( - indices=np.array([[0, 0], [1, 1], [2, 2]]), - values=np.array([1, 2, 3]), - dense_shape=np.array([3, 3]))) - - iterator = ( - dataset_ops.Dataset.from_tensor_slices(components) - .make_initializable_iterator()) - init_op = iterator.initializer - get_next = iterator.get_next() - - self.assertEqual([ - tensor_shape.TensorShape(c.dense_shape[1:]) - if sparse_tensor.is_sparse(c) else c.shape[1:] for c in components - ], [shape for shape in iterator.output_shapes]) - - with self.cached_session() as sess: - sess.run(init_op) - expected = [ - (sparse_tensor.SparseTensorValue( - indices=np.array([[0]]), - values=np.array([0]), - dense_shape=np.array([1])), - sparse_tensor.SparseTensorValue( - indices=np.array([[0]]), - values=np.array([1]), - dense_shape=np.array([3]))), - (sparse_tensor.SparseTensorValue( - indices=np.array([[0]]), - values=np.array([0]), - dense_shape=np.array([1])), - sparse_tensor.SparseTensorValue( - indices=np.array([[1]]), - values=np.array([2]), - dense_shape=np.array([3]))), - (sparse_tensor.SparseTensorValue( - indices=np.array([[0]]), - values=np.array([0]), - dense_shape=np.array([1])), - sparse_tensor.SparseTensorValue( - indices=np.array([[2]]), - values=np.array([3]), - dense_shape=np.array([3]))), - ] - for i in range(3): - results = sess.run(get_next) - for component, result_component in zip( - (list(zip(*components[:3]))[i] + expected[i]), results): - if sparse_tensor.is_sparse(component): - self.assertSparseValuesEqual(component, result_component) - else: - self.assertAllEqual(component, result_component) - with self.assertRaises(errors.OutOfRangeError): - sess.run(get_next) - - def testFromTensorSlicesWithDict(self): - components = {"foo": [1, 2, 3], "bar": [[4.0], [5.0], [6.0]]} - iterator = (dataset_ops.Dataset.from_tensor_slices(components) - .make_initializable_iterator()) - init_op = iterator.initializer - get_next = iterator.get_next() - - self.assertEqual(dtypes.int32, iterator.output_types["foo"]) - self.assertEqual(dtypes.float32, iterator.output_types["bar"]) - self.assertEqual((), iterator.output_shapes["foo"]) - self.assertEqual((1,), iterator.output_shapes["bar"]) - - with self.cached_session() as sess: - sess.run(init_op) - for i in range(3): - results = sess.run(get_next) - self.assertEqual(components["foo"][i], results["foo"]) - self.assertEqual(components["bar"][i], results["bar"]) - with self.assertRaises(errors.OutOfRangeError): - sess.run(get_next) - - def testFromSparseTensorSlices(self): - """Test a dataset based on slices of a `tf.SparseTensor`.""" - st = array_ops.sparse_placeholder(dtypes.float64) - iterator = (dataset_ops.Dataset.from_sparse_tensor_slices(st) - .make_initializable_iterator()) - init_op = iterator.initializer - get_next = sparse_tensor.SparseTensor(*iterator.get_next()) - - with self.cached_session() as sess: - slices = [[1., 2., 3.], [1.], [1.], [1., 2.], [], [1., 2.], [], [], []] - - # Test with sparse tensor in the appropriate order. - indices = np.array( - [[i, j] for i in range(len(slices)) for j in range(len(slices[i]))]) - values = np.array([val for s in slices for val in s]) - dense_shape = np.array([len(slices), max(len(s) for s in slices) + 1]) - sparse_feed = sparse_tensor.SparseTensorValue(indices, values, - dense_shape) - sess.run(init_op, feed_dict={st: sparse_feed}) - for i, s in enumerate(slices): - results = sess.run(get_next) - self.assertAllEqual(s, results.values) - expected_indices = np.array( - [[j] for j in range(len(slices[i]))]).reshape([-1, 1]) - self.assertAllEqual(expected_indices, results.indices) - self.assertAllEqual(dense_shape[1:], results.dense_shape) - with self.assertRaises(errors.OutOfRangeError): - sess.run(get_next) - - # Test with sparse tensor in the reverse order, which is not - # currently supported. - reverse_order_indices = indices[::-1, :] - reverse_order_values = values[::-1] - sparse_feed = sparse_tensor.SparseTensorValue( - reverse_order_indices, reverse_order_values, dense_shape) - with self.assertRaises(errors.UnimplementedError): - sess.run(init_op, feed_dict={st: sparse_feed}) - - # Test with an empty sparse tensor. - empty_indices = np.empty((0, 4), dtype=np.int64) - empty_values = np.empty((0,), dtype=np.float64) - empty_dense_shape = [0, 4, 37, 9] - sparse_feed = sparse_tensor.SparseTensorValue(empty_indices, empty_values, - empty_dense_shape) - sess.run(init_op, feed_dict={st: sparse_feed}) - with self.assertRaises(errors.OutOfRangeError): - sess.run(get_next) - - # pylint: disable=g-long-lambda,unnecessary-lambda - def testNestedStructure(self): - components = (np.array([1, 2, 3], dtype=np.int64), - (np.array([4., 5.]), np.array([6., 7.])), - np.array([8, 9, 10], dtype=np.int64)) - - dataset = dataset_ops.Dataset.from_tensors(components) - self.assertEquals((dtypes.int64, (dtypes.float64, dtypes.float64), - dtypes.int64), dataset.output_types) - self.assertEquals(([3], ([2], [2]), [3]), dataset.output_shapes) - - dataset = dataset.shuffle(10, 10) - self.assertEquals((dtypes.int64, (dtypes.float64, dtypes.float64), - dtypes.int64), dataset.output_types) - self.assertEquals(([3], ([2], [2]), [3]), dataset.output_shapes) - - dataset = dataset.repeat(-1) - self.assertEquals((dtypes.int64, (dtypes.float64, dtypes.float64), - dtypes.int64), dataset.output_types) - self.assertEquals(([3], ([2], [2]), [3]), dataset.output_shapes) - - dataset = dataset.filter(lambda x, y, z: True) - self.assertEquals((dtypes.int64, (dtypes.float64, dtypes.float64), - dtypes.int64), dataset.output_types) - self.assertEquals(([3], ([2], [2]), [3]), dataset.output_shapes) - - dataset = dataset.take(5) - self.assertEquals((dtypes.int64, (dtypes.float64, dtypes.float64), - dtypes.int64), dataset.output_types) - self.assertEquals(([3], ([2], [2]), [3]), dataset.output_shapes) - - dataset = dataset.map(lambda x, y, z: ((x, z), (y[0], y[1]))) - self.assertEquals(((dtypes.int64, dtypes.int64), - (dtypes.float64, dtypes.float64)), dataset.output_types) - self.assertEquals((([3], [3]), ([2], [2])), dataset.output_shapes) - - dataset = dataset.flat_map( - lambda x, y: dataset_ops.Dataset.from_tensors(((x[0], x[1]), - (y[0], y[1]))) - ) - self.assertEquals(((dtypes.int64, dtypes.int64), - (dtypes.float64, dtypes.float64)), dataset.output_types) - self.assertEquals((([3], [3]), ([2], [2])), dataset.output_shapes) - - dataset = dataset.batch(32) - self.assertEquals(((dtypes.int64, dtypes.int64), - (dtypes.float64, dtypes.float64)), dataset.output_types) - self.assertEquals((([None, 3], [None, 3]), ([None, 2], [None, 2])), - nest.pack_sequence_as(dataset.output_shapes, [ - s.as_list() - for s in nest.flatten(dataset.output_shapes) - ])) - - iterator = dataset.make_one_shot_iterator() - (w, x), (y, z) = iterator.get_next() - self.assertEquals(dtypes.int64, w.dtype) - self.assertEquals(dtypes.int64, x.dtype) - self.assertEquals(dtypes.float64, y.dtype) - self.assertEquals(dtypes.float64, z.dtype) - self.assertEquals([None, 3], w.shape.as_list()) - self.assertEquals([None, 3], x.shape.as_list()) - self.assertEquals([None, 2], y.shape.as_list()) - self.assertEquals([None, 2], z.shape.as_list()) - - iterator = dataset.make_initializable_iterator() - (w, x), (y, z) = iterator.get_next() - self.assertEquals(dtypes.int64, w.dtype) - self.assertEquals(dtypes.int64, x.dtype) - self.assertEquals(dtypes.float64, y.dtype) - self.assertEquals(dtypes.float64, z.dtype) - self.assertEquals([None, 3], w.shape.as_list()) - self.assertEquals([None, 3], x.shape.as_list()) - self.assertEquals([None, 2], y.shape.as_list()) - self.assertEquals([None, 2], z.shape.as_list()) - - # Define a separate set of components with matching leading - # dimension for the from-slices constructor. - components_for_slices = (np.array([1, 2, 3], dtype=np.int64), - (np.array([4., 5., 6.]), - np.array([7., 8., 9.])), - np.array([10, 11, 12], dtype=np.int64)) - - dataset = dataset_ops.Dataset.from_tensor_slices(components_for_slices) - self.assertEquals((dtypes.int64, (dtypes.float64, dtypes.float64), - dtypes.int64), dataset.output_types) - self.assertEquals(([], ([], []), []), dataset.output_shapes) - - def testNestedDict(self): - components = {"a": {"aa": 1, "ab": [2.0, 2.0]}, "b": [3, 3, 3]} - dataset = dataset_ops.Dataset.from_tensors(components) - self.assertEquals(dtypes.int32, dataset.output_types["a"]["aa"]) - self.assertEquals(dtypes.float32, dataset.output_types["a"]["ab"]) - self.assertEquals(dtypes.int32, dataset.output_types["b"]) - self.assertEquals([], dataset.output_shapes["a"]["aa"]) - self.assertEquals([2], dataset.output_shapes["a"]["ab"]) - self.assertEquals([3], dataset.output_shapes["b"]) - - def testNonSequenceNestedStructure(self): - components = np.array([1, 2, 3], dtype=np.int64) - - dataset = dataset_ops.Dataset.from_tensors(components) - self.assertEquals(dtypes.int64, dataset.output_types) - self.assertEquals([3], dataset.output_shapes) - - dataset = dataset.filter( - lambda x: math_ops.reduce_all(math_ops.equal(x, components))) - self.assertEquals(dtypes.int64, dataset.output_types) - self.assertEquals([3], dataset.output_shapes) - - dataset = dataset.map(lambda x: array_ops.stack([x, x])) - self.assertEquals(dtypes.int64, dataset.output_types) - self.assertEquals([2, 3], dataset.output_shapes) - - dataset = dataset.flat_map( - lambda x: dataset_ops.Dataset.from_tensor_slices(x)) - self.assertEquals(dtypes.int64, dataset.output_types) - self.assertEquals([3], dataset.output_shapes) - - iterator = dataset.make_one_shot_iterator() - get_next = iterator.get_next() - self.assertEquals(dtypes.int64, get_next.dtype) - self.assertEquals([3], get_next.shape) - - def testSplitPipelineFailsWithPlacementError(self): - with session.Session( - target="", - config=config_pb2.ConfigProto(device_count={"CPU": 2})) as sess: - - dataset = dataset_ops.Dataset.from_tensors(0) - - # Define a pipeline that attempts to use variables on two - # different devices. - # - # Initialize the variables before creating to iterator, to avoid the - # placement algorithm overriding the DT_RESOURCE colocation constraints. - with ops.device("/cpu:0"): - var_0 = resource_variable_ops.ResourceVariable(initial_value=0) - dataset = dataset.map(lambda x: x + var_0.read_value()) - sess.run(var_0.initializer) - - with ops.device("/cpu:1"): - var_1 = resource_variable_ops.ResourceVariable(initial_value=0) - dataset = dataset.map(lambda x: x + var_1.read_value()) - sess.run(var_1.initializer) - - iterator = dataset.make_initializable_iterator() - sess.run(iterator.initializer) - - with self.assertRaisesRegexp( - errors.FailedPreconditionError, - "Error while reading resource variable Variable"): - sess.run(iterator.get_next()) - - -class DatasetConstructorBenchmark(test.Benchmark): - - def benchmarkSliceRepeatBatch(self): - input_size = 10000 - batch_size = 100 - num_epochs = 100 - - input_data = np.random.randn(input_size) - - dataset = ( - dataset_ops.Dataset.from_tensor_slices(input_data) - .repeat(num_epochs + 1).batch(batch_size)) - iterator = dataset.make_initializable_iterator() - next_element = iterator.get_next() - - with session.Session() as sess: - sess.run(iterator.initializer) - # Run one whole epoch to burn in the computation. - for _ in range(input_size // batch_size): - sess.run(next_element) - deltas = [] - try: - while True: - start = time.time() - sess.run(next_element) - deltas.append(time.time() - start) - except errors.OutOfRangeError: - pass - - median_wall_time = np.median(deltas) - print("Slice/repeat/batch with sess.run() input size: %d batch size: %d " - "Median wall time per element: %f" % (input_size, batch_size, - median_wall_time)) - self.report_benchmark( - iters=len(deltas), - wall_time=median_wall_time, - name="benchmark_slice_repeat_batch_input_%d_batch_%d" % (input_size, - batch_size)) - - def benchmarkSliceRepeatBatchCallable(self): - input_size = 10000 - batch_size = 100 - num_epochs = 100 - - input_data = np.random.randn(input_size) - - dataset = ( - dataset_ops.Dataset.from_tensor_slices(input_data) - .repeat(num_epochs + 1).batch(batch_size)) - iterator = dataset.make_initializable_iterator() - next_element = iterator.get_next() - - with session.Session() as sess: - sess.run(iterator.initializer) - get_next_element = sess.make_callable(next_element) - # Run one whole epoch to burn in the computation. - for _ in range(input_size // batch_size): - get_next_element() - deltas = [] - try: - while True: - start = time.time() - get_next_element() - deltas.append(time.time() - start) - except errors.OutOfRangeError: - pass - - median_wall_time = np.median(deltas) - print( - "Slice/repeat/batch with callable input size: %d batch size: %d Median" - " wall time per element: %f" % (input_size, batch_size, - median_wall_time)) - self.report_benchmark( - iters=len(deltas), - wall_time=median_wall_time, - name="benchmark_slice_repeat_batch_callable_input_%d_batch_%d" % - (input_size, batch_size)) - - def benchmarkReshapeSliceRepeatCallable(self): - input_size = 10000 - batch_size = 100 - num_epochs = 100 - - input_data = np.random.randn(input_size) - - dataset = ( - dataset_ops.Dataset.from_tensor_slices(input_data.reshape(100, 100)) - .repeat(num_epochs + 1)) - iterator = dataset.make_initializable_iterator() - next_element = iterator.get_next() - - with session.Session() as sess: - sess.run(iterator.initializer) - get_next_element = sess.make_callable(next_element) - # Run one whole epoch to burn in the computation. - for _ in range(input_size // batch_size): - get_next_element() - deltas = [] - try: - while True: - start = time.time() - get_next_element() - deltas.append(time.time() - start) - except errors.OutOfRangeError: - pass - - median_wall_time = np.median(deltas) - print("Reshape/slice/repeat with callable input size: %d batch size: %d " - "Median wall time per element: %f" % (input_size, batch_size, - median_wall_time)) - self.report_benchmark( - iters=len(deltas), - wall_time=median_wall_time, - name="benchmark_reshape_slice_repeat_callable_input_%d_batch_%d" % - (input_size, batch_size)) - - def benchmarkSliceBatchCacheRepeatCallable(self): - input_size = 10000 - batch_size = 100 - num_epochs = 100 - - input_data = np.random.randn(input_size) - - dataset = ( - dataset_ops.Dataset.from_tensor_slices(input_data).batch(batch_size) - .cache().repeat(num_epochs + 1)) - iterator = dataset.make_initializable_iterator() - next_element = iterator.get_next() - - with session.Session() as sess: - sess.run(iterator.initializer) - get_next_element = sess.make_callable(next_element) - # Run one whole epoch to burn in the computation. - for _ in range(input_size // batch_size): - get_next_element() - deltas = [] - try: - while True: - start = time.time() - get_next_element() - deltas.append(time.time() - start) - except errors.OutOfRangeError: - pass - - median_wall_time = np.median(deltas) - print( - "Slice/batch/cache/repeat with callable input size: %d batch size: %d " - "Median wall time per element: %f" - % (input_size, batch_size, median_wall_time)) - self.report_benchmark( - iters=len(deltas), - wall_time=median_wall_time, - name="benchmark_slice_batch_cache_repeat_callable_input_%d_batch_%d" % - (input_size, batch_size)) - - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/python/data/kernel_tests/dataset_ops_test.py b/tensorflow/python/data/kernel_tests/dataset_test.py similarity index 69% rename from tensorflow/python/data/kernel_tests/dataset_ops_test.py rename to tensorflow/python/data/kernel_tests/dataset_test.py index a5324af4d0c..2952c08be02 100644 --- a/tensorflow/python/data/kernel_tests/dataset_ops_test.py +++ b/tensorflow/python/data/kernel_tests/dataset_test.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================== -"""Tests for the input pipeline ops.""" +"""Tests for `tf.data.Dataset`.""" from __future__ import absolute_import from __future__ import division @@ -24,21 +24,26 @@ import numpy as np from tensorflow.core.framework import graph_pb2 from tensorflow.python.data.kernel_tests import test_base from tensorflow.python.data.ops import dataset_ops +from tensorflow.python.data.ops import optional_ops from tensorflow.python.data.ops import readers from tensorflow.python.data.util import nest +from tensorflow.python.data.util import structure +from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes from tensorflow.python.framework import sparse_tensor +from tensorflow.python.framework import tensor_shape +from tensorflow.python.framework import test_util from tensorflow.python.platform import test -class DatasetOpsTest(test_base.DatasetTestBase, parameterized.TestCase): +@test_util.run_all_in_graph_and_eager_modes +class DatasetTest(test_base.DatasetTestBase, parameterized.TestCase): def testAsSerializedGraph(self): dataset = dataset_ops.Dataset.range(10) - with self.cached_session() as sess: - graph = graph_pb2.GraphDef().FromString( - sess.run(dataset._as_serialized_graph())) - self.assertTrue(any([node.op != "RangeDataset" for node in graph.node])) + graph = graph_pb2.GraphDef().FromString( + self.evaluate(dataset._as_serialized_graph())) + self.assertTrue(any([node.op != "RangeDataset" for node in graph.node])) @staticmethod def make_apply_fn(dataset): @@ -76,7 +81,7 @@ class DatasetOpsTest(test_base.DatasetTestBase, parameterized.TestCase): lambda: readers.FixedLengthRecordDataset("", 42)), ("FromGenerator", lambda: dataset_ops.Dataset.from_generator( - DatasetOpsTest.make_gen(), dtypes.int32), + DatasetTest.make_gen(), dtypes.int32), 1), ("FromTensors", lambda: dataset_ops.Dataset.from_tensors([42])), ("FromTensorSlices", lambda: dataset_ops.Dataset.from_tensors([42])), @@ -222,12 +227,12 @@ class DatasetOpsTest(test_base.DatasetTestBase, parameterized.TestCase): options1 = dataset_ops.Options() options1.experimental_autotune = True options2 = dataset_ops.Options() - options2.experimental_filter_fusion = False + options2.experimental_deterministic = False ds = dataset_ops.Dataset.range(0).with_options(options1).with_options( options2) self.assertTrue(ds.options().experimental_autotune) # Explicitly check that flag is False since assertFalse allows None - self.assertIs(ds.options().experimental_filter_fusion, False) + self.assertIs(ds.options().experimental_deterministic, False) def testOptionsTwiceDifferentError(self): options1 = dataset_ops.Options() @@ -235,20 +240,78 @@ class DatasetOpsTest(test_base.DatasetTestBase, parameterized.TestCase): options2 = dataset_ops.Options() options2.experimental_autotune = False with self.assertRaisesRegexp(ValueError, - "Cannot merge incompatible values of option"): + "Cannot merge incompatible values"): dataset_ops.Dataset.range(0).with_options(options1).with_options(options2) def testOptionsMergeOptionsFromMultipleInputs(self): options1 = dataset_ops.Options() options1.experimental_autotune = True options2 = dataset_ops.Options() - options2.experimental_filter_fusion = True + options2.experimental_deterministic = True ds = dataset_ops.Dataset.zip( (dataset_ops.Dataset.range(0).with_options(options1), dataset_ops.Dataset.range(0).with_options(options2))) self.assertTrue(ds.options().experimental_autotune) - self.assertTrue(ds.options().experimental_filter_fusion) + self.assertTrue(ds.options().experimental_deterministic) + # TODO(b/119882922): use-after-free bug in eager mode. + # pylint: disable=g-long-lambda + @parameterized.named_parameters( + ("Tensor", lambda: constant_op.constant(37.0), + structure.TensorStructure(dtypes.float32, [])), + ("SparseTensor", lambda: sparse_tensor.SparseTensor( + indices=[[0]], values=constant_op.constant([0], dtype=dtypes.int32), + dense_shape=[1]), + structure.SparseTensorStructure(dtypes.int32, [1])), + ("Nest", lambda: { + "a": constant_op.constant(37.0), + "b": (constant_op.constant(["Foo"]), constant_op.constant("Bar"))}, + structure.NestedStructure({ + "a": structure.TensorStructure(dtypes.float32, []), + "b": (structure.TensorStructure(dtypes.string, [1]), + structure.TensorStructure(dtypes.string, []))})), + ("Dataset", lambda: dataset_ops.Dataset.from_tensor_slices( + constant_op.constant([1, 2, 3])), + dataset_ops.DatasetStructure( + structure.TensorStructure(dtypes.int32, []))), + ("Optional", lambda: optional_ops.Optional.from_value(37.0), + optional_ops.OptionalStructure( + structure.TensorStructure(dtypes.float32, []))), + ) + def testSkipEagerDatasetStructure(self, tf_value_fn, + expected_element_structure): + dataset = dataset_ops.Dataset.from_tensors(0).map(lambda _: tf_value_fn()) + dataset_structure = structure.Structure.from_value(dataset) + self.assertIsInstance(dataset_structure, dataset_ops.DatasetStructure) + + # TODO(b/110122868): Add a public API to `tf.data.Dataset` for accessing + # the element structure. + self.assertTrue(expected_element_structure.is_compatible_with( + dataset_structure._element_structure)) + self.assertTrue(dataset_structure._element_structure.is_compatible_with( + expected_element_structure)) + + self.assertEqual([dtypes.variant], dataset_structure._flat_types) + self.assertEqual([tensor_shape.scalar()], dataset_structure._flat_shapes) + + # Assert that the `Dataset` survives a round-trip via _from_tensor_list() + # and _to_tensor_list(). + round_trip_dataset = dataset_structure._from_tensor_list( + dataset_structure._to_tensor_list(dataset)) + + value = tf_value_fn() + + if isinstance(value, dataset_ops.Dataset): + self.assertDatasetsEqual(value, dataset.flat_map(lambda x: x)) + elif isinstance(value, optional_ops.Optional): + self.assertDatasetProduces( + round_trip_dataset.map(lambda opt: opt.get_value()), + [self.evaluate(value.get_value())], + requires_initialization=True) + else: + self.assertDatasetProduces( + round_trip_dataset, [self.evaluate(tf_value_fn())], + requires_initialization=True) if __name__ == "__main__": test.main() diff --git a/tensorflow/python/data/kernel_tests/filter_dataset_op_test.py b/tensorflow/python/data/kernel_tests/filter_dataset_op_test.py deleted file mode 100644 index a0c6b37a6dc..00000000000 --- a/tensorflow/python/data/kernel_tests/filter_dataset_op_test.py +++ /dev/null @@ -1,220 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for the experimental input pipeline ops.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import time - -import numpy as np - -from tensorflow.python.client import session -from tensorflow.python.data.kernel_tests import test_base -from tensorflow.python.data.ops import dataset_ops -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import errors -from tensorflow.python.framework import ops -from tensorflow.python.framework import sparse_tensor -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import functional_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.platform import test - - -class FilterDatasetTest(test_base.DatasetTestBase): - - def testFilterDataset(self): - components = ( - np.arange(7, dtype=np.int64), - np.array([[1, 2, 3]], dtype=np.int64) * np.arange( - 7, dtype=np.int64)[:, np.newaxis], - np.array(37.0, dtype=np.float64) * np.arange(7) - ) - count = array_ops.placeholder(dtypes.int64, shape=[]) - modulus = array_ops.placeholder(dtypes.int64) - - def _map_fn(x, y, z): - return math_ops.square(x), math_ops.square(y), math_ops.square(z) - - iterator = ( - dataset_ops.Dataset.from_tensor_slices(components).map(_map_fn) - .repeat(count) - .filter(lambda x, _y, _z: math_ops.equal(math_ops.mod(x, modulus), 0)) - .make_initializable_iterator()) - init_op = iterator.initializer - get_next = iterator.get_next() - - self.assertEqual([c.shape[1:] for c in components], - [t.shape for t in get_next]) - - with self.cached_session() as sess: - # Test that we can dynamically feed a different modulus value for each - # iterator. - def do_test(count_val, modulus_val): - sess.run(init_op, feed_dict={count: count_val, modulus: modulus_val}) - for _ in range(count_val): - for i in [x for x in range(7) if x**2 % modulus_val == 0]: - result = sess.run(get_next) - for component, result_component in zip(components, result): - self.assertAllEqual(component[i]**2, result_component) - with self.assertRaises(errors.OutOfRangeError): - sess.run(get_next) - - do_test(14, 2) - do_test(4, 18) - - # Test an empty dataset. - do_test(0, 1) - - def testFilterRange(self): - dataset = dataset_ops.Dataset.range(100).filter( - lambda x: math_ops.not_equal(math_ops.mod(x, 3), 2)) - iterator = dataset.make_one_shot_iterator() - get_next = iterator.get_next() - - with self.cached_session() as sess: - self.assertEqual(0, sess.run(get_next)) - self.assertEqual(1, sess.run(get_next)) - self.assertEqual(3, sess.run(get_next)) - - def testFilterDict(self): - iterator = (dataset_ops.Dataset.range(10) - .map(lambda x: {"foo": x * 2, "bar": x ** 2}) - .filter(lambda d: math_ops.equal(d["bar"] % 2, 0)) - .map(lambda d: d["foo"] + d["bar"]) - .make_initializable_iterator()) - init_op = iterator.initializer - get_next = iterator.get_next() - - with self.cached_session() as sess: - sess.run(init_op) - for i in range(10): - if (i ** 2) % 2 == 0: - self.assertEqual(i * 2 + i ** 2, sess.run(get_next)) - with self.assertRaises(errors.OutOfRangeError): - sess.run(get_next) - - def testUseStepContainerInFilter(self): - input_data = np.array([[1, 2, 3], [4, 5, 6]], dtype=np.int64) - - # Define a predicate that returns true for the first element of - # the sequence and not the second, and uses `tf.map_fn()`. - def _predicate(xs): - squared_xs = functional_ops.map_fn(lambda x: x * x, xs) - summed = math_ops.reduce_sum(squared_xs) - return math_ops.equal(summed, 1 + 4 + 9) - - iterator = ( - dataset_ops.Dataset.from_tensor_slices([[1, 2, 3], [4, 5, 6]]) - .filter(_predicate) - .make_initializable_iterator()) - init_op = iterator.initializer - get_next = iterator.get_next() - - with self.cached_session() as sess: - sess.run(init_op) - self.assertAllEqual(input_data[0], sess.run(get_next)) - with self.assertRaises(errors.OutOfRangeError): - sess.run(get_next) - - def testSparse(self): - - def _map_fn(i): - return sparse_tensor.SparseTensorValue( - indices=np.array([[0, 0]]), - values=(i * np.array([1])), - dense_shape=np.array([1, 1])), i - - def _filter_fn(_, i): - return math_ops.equal(i % 2, 0) - - iterator = ( - dataset_ops.Dataset.range(10).map(_map_fn).filter(_filter_fn).map( - lambda x, i: x).make_initializable_iterator()) - init_op = iterator.initializer - get_next = iterator.get_next() - - with self.cached_session() as sess: - sess.run(init_op) - for i in range(5): - actual = sess.run(get_next) - self.assertTrue(isinstance(actual, sparse_tensor.SparseTensorValue)) - self.assertSparseValuesEqual(actual, _map_fn(i * 2)[0]) - with self.assertRaises(errors.OutOfRangeError): - sess.run(get_next) - - def testShortCircuit(self): - iterator = ( - dataset_ops.Dataset.zip( - (dataset_ops.Dataset.range(10), - dataset_ops.Dataset.from_tensors(True).repeat(None))) - .filter(lambda x, y: y).make_initializable_iterator()) - init_op = iterator.initializer - get_next = iterator.get_next() - - with self.cached_session() as sess: - sess.run(init_op) - for i in range(10): - self.assertEqual((i, True), sess.run(get_next)) - with self.assertRaises(errors.OutOfRangeError): - sess.run(get_next) - - def testParallelFilters(self): - dataset = dataset_ops.Dataset.range(10).filter( - lambda x: math_ops.equal(x % 2, 0)) - iterators = [dataset.make_one_shot_iterator() for _ in range(10)] - next_elements = [iterator.get_next() for iterator in iterators] - with self.cached_session() as sess: - self.assertEqual([0 for _ in range(10)], sess.run(next_elements)) - - -class FilterDatasetBenchmark(test.Benchmark): - - def _benchmark(self, predicate, name): - with ops.Graph().as_default(): - dataset = ( - dataset_ops.Dataset.from_tensors(True).repeat(None).filter(predicate)) - iterator = dataset.make_one_shot_iterator() - next_element = iterator.get_next() - - with session.Session() as sess: - for _ in range(5): - sess.run(next_element.op) - deltas = [] - for _ in range(100): - start = time.time() - for _ in range(100): - sess.run(next_element.op) - end = time.time() - deltas.append(end - start) - - median_wall_time = np.median(deltas) / 100 - print("Filter dataset using %s. Median wall time: %f" % - (name, median_wall_time)) - self.report_benchmark( - iters=100, - wall_time=median_wall_time, - name="benchmark_filter_dataset_%s" % name) - - def benchmarkSimpleFunction(self): - self._benchmark(array_ops.identity, "simple_function") - - def benchmarkReturnComponentOptimization(self): - self._benchmark(lambda x: x, "return_component") - - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/python/data/kernel_tests/filter_test.py b/tensorflow/python/data/kernel_tests/filter_test.py new file mode 100644 index 00000000000..afaf954cbc6 --- /dev/null +++ b/tensorflow/python/data/kernel_tests/filter_test.py @@ -0,0 +1,128 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Tests for `tf.data.Dataset.filter()`.""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import numpy as np + +from tensorflow.python.data.kernel_tests import test_base +from tensorflow.python.data.ops import dataset_ops +from tensorflow.python.framework import errors +from tensorflow.python.framework import sparse_tensor +from tensorflow.python.framework import test_util +from tensorflow.python.ops import functional_ops +from tensorflow.python.ops import math_ops +from tensorflow.python.platform import test + + +@test_util.run_all_in_graph_and_eager_modes +class FilterTest(test_base.DatasetTestBase): + + def testFilterDataset(self): + components = ( + np.arange(7, dtype=np.int64), + np.array([[1, 2, 3]], dtype=np.int64) * np.arange( + 7, dtype=np.int64)[:, np.newaxis], + np.array(37.0, dtype=np.float64) * np.arange(7) + ) + def _map_fn(x, y, z): + return math_ops.square(x), math_ops.square(y), math_ops.square(z) + + def do_test(count, modulus): + dataset = dataset_ops.Dataset.from_tensor_slices(components).map( + _map_fn).repeat(count).filter( + lambda x, _y, _z: math_ops.equal(math_ops.mod(x, modulus), 0)) + self.assertEqual([c.shape[1:] for c in components], + [shape for shape in dataset.output_shapes]) + get_next = self.getNext(dataset) + for _ in range(count): + for i in [x for x in range(7) if x**2 % modulus == 0]: + result = self.evaluate(get_next()) + for component, result_component in zip(components, result): + self.assertAllEqual(component[i]**2, result_component) + with self.assertRaises(errors.OutOfRangeError): + self.evaluate(get_next()) + + do_test(14, 2) + do_test(4, 18) + + # Test an empty dataset. + do_test(0, 1) + + def testFilterRange(self): + dataset = dataset_ops.Dataset.range(4).filter( + lambda x: math_ops.not_equal(math_ops.mod(x, 3), 2)) + self.assertDatasetProduces(dataset, expected_output=[0, 1, 3]) + + def testFilterDict(self): + dataset = dataset_ops.Dataset.range(10).map( + lambda x: {"foo": x * 2, "bar": x ** 2}).filter( + lambda d: math_ops.equal(d["bar"] % 2, 0)).map( + lambda d: d["foo"] + d["bar"]) + self.assertDatasetProduces( + dataset, + expected_output=[(i * 2 + i**2) for i in range(10) if not (i**2) % 2]) + + def testUseStepContainerInFilter(self): + input_data = np.array([[1, 2, 3], [4, 5, 6]], dtype=np.int64) + + # Define a predicate that returns true for the first element of + # the sequence and not the second, and uses `tf.map_fn()`. + def _predicate(xs): + squared_xs = functional_ops.map_fn(lambda x: x * x, xs) + summed = math_ops.reduce_sum(squared_xs) + return math_ops.equal(summed, 1 + 4 + 9) + + dataset = dataset_ops.Dataset.from_tensor_slices( + [[1, 2, 3], [4, 5, 6]]).filter(_predicate) + self.assertDatasetProduces(dataset, expected_output=[input_data[0]]) + + def testSparse(self): + + def _map_fn(i): + return sparse_tensor.SparseTensorValue( + indices=np.array([[0, 0]]), + values=(i * np.array([1])), + dense_shape=np.array([1, 1])), i + + def _filter_fn(_, i): + return math_ops.equal(i % 2, 0) + + dataset = dataset_ops.Dataset.range(10).map(_map_fn).filter(_filter_fn).map( + lambda x, i: x) + self.assertDatasetProduces( + dataset, expected_output=[_map_fn(i * 2)[0] for i in range(5)]) + + def testShortCircuit(self): + dataset = dataset_ops.Dataset.zip( + (dataset_ops.Dataset.range(10), + dataset_ops.Dataset.from_tensors(True).repeat(None) + )).filter(lambda x, y: y) + self.assertDatasetProduces( + dataset, expected_output=[(i, True) for i in range(10)]) + + def testParallelFilters(self): + dataset = dataset_ops.Dataset.range(10).filter( + lambda x: math_ops.equal(x % 2, 0)) + next_elements = [self.getNext(dataset) for _ in range(10)] + self.assertEqual([0 for _ in range(10)], + self.evaluate( + [next_element() for next_element in next_elements])) + + +if __name__ == "__main__": + test.main() diff --git a/tensorflow/python/data/kernel_tests/fixed_length_record_dataset_test.py b/tensorflow/python/data/kernel_tests/fixed_length_record_dataset_test.py new file mode 100644 index 00000000000..9503e57ca7c --- /dev/null +++ b/tensorflow/python/data/kernel_tests/fixed_length_record_dataset_test.py @@ -0,0 +1,171 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Tests for `tf.data.FixedLengthRecordDataset`.""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import gzip +import os +import zlib + +from tensorflow.python.data.kernel_tests import test_base +from tensorflow.python.data.ops import readers +from tensorflow.python.framework import errors +from tensorflow.python.framework import test_util +from tensorflow.python.platform import test +from tensorflow.python.util import compat + + +@test_util.run_all_in_graph_and_eager_modes +class FixedLengthRecordDatasetTest(test_base.DatasetTestBase): + + def setUp(self): + super(FixedLengthRecordDatasetTest, self).setUp() + self._num_files = 2 + self._num_records = 7 + self._header_bytes = 5 + self._record_bytes = 3 + self._footer_bytes = 2 + + def _record(self, f, r): + return compat.as_bytes(str(f * 2 + r) * self._record_bytes) + + def _createFiles(self, compression_type=None): + filenames = [] + for i in range(self._num_files): + fn = os.path.join(self.get_temp_dir(), "fixed_length_record.%d.txt" % i) + filenames.append(fn) + + contents = [] + contents.append(b"H" * self._header_bytes) + for j in range(self._num_records): + contents.append(self._record(i, j)) + contents.append(b"F" * self._footer_bytes) + contents = b"".join(contents) + + if not compression_type: + with open(fn, "wb") as f: + f.write(contents) + elif compression_type == "GZIP": + with gzip.GzipFile(fn, "wb") as f: + f.write(contents) + elif compression_type == "ZLIB": + contents = zlib.compress(contents) + with open(fn, "wb") as f: + f.write(contents) + else: + raise ValueError("Unsupported compression_type", compression_type) + + return filenames + + def _testFixedLengthRecordDataset(self, compression_type=None): + test_filenames = self._createFiles(compression_type=compression_type) + + def dataset_fn(filenames, num_epochs, batch_size=None): + repeat_dataset = readers.FixedLengthRecordDataset( + filenames, + self._record_bytes, + self._header_bytes, + self._footer_bytes, + compression_type=compression_type).repeat(num_epochs) + if batch_size: + return repeat_dataset.batch(batch_size) + return repeat_dataset + + # Basic test: read from file 0. + self.assertDatasetProduces( + dataset_fn([test_filenames[0]], 1), + expected_output=[ + self._record(0, i) for i in range(self._num_records) + ]) + + # Basic test: read from file 1. + self.assertDatasetProduces( + dataset_fn([test_filenames[1]], 1), + expected_output=[ + self._record(1, i) for i in range(self._num_records) + ]) + + # Basic test: read from both files. + expected_output = [] + for j in range(self._num_files): + expected_output.extend( + [self._record(j, i) for i in range(self._num_records)]) + self.assertDatasetProduces( + dataset_fn(test_filenames, 1), expected_output=expected_output) + + # Test repeated iteration through both files. + get_next = self.getNext(dataset_fn(test_filenames, 10)) + for _ in range(10): + for j in range(self._num_files): + for i in range(self._num_records): + self.assertEqual(self._record(j, i), self.evaluate(get_next())) + with self.assertRaises(errors.OutOfRangeError): + self.evaluate(get_next()) + + # Test batched and repeated iteration through both files. + get_next = self.getNext(dataset_fn(test_filenames, 10, self._num_records)) + for _ in range(10): + for j in range(self._num_files): + self.assertAllEqual( + [self._record(j, i) for i in range(self._num_records)], + self.evaluate(get_next())) + with self.assertRaises(errors.OutOfRangeError): + self.evaluate(get_next()) + + def testFixedLengthRecordDatasetNoCompression(self): + self._testFixedLengthRecordDataset() + + def testFixedLengthRecordDatasetGzipCompression(self): + self._testFixedLengthRecordDataset(compression_type="GZIP") + + def testFixedLengthRecordDatasetZlibCompression(self): + self._testFixedLengthRecordDataset(compression_type="ZLIB") + + def testFixedLengthRecordDatasetBuffering(self): + test_filenames = self._createFiles() + dataset = readers.FixedLengthRecordDataset( + test_filenames, + self._record_bytes, + self._header_bytes, + self._footer_bytes, + buffer_size=10) + expected_output = [] + for j in range(self._num_files): + expected_output.extend( + [self._record(j, i) for i in range(self._num_records)]) + self.assertDatasetProduces(dataset, expected_output=expected_output) + + def testFixedLengthRecordDatasetWrongSize(self): + test_filenames = self._createFiles() + dataset = readers.FixedLengthRecordDataset( + test_filenames, + self._record_bytes + 1, # Incorrect record length. + self._header_bytes, + self._footer_bytes, + buffer_size=10) + self.assertDatasetProduces( + dataset, + expected_error=( + errors.InvalidArgumentError, + r"Excluding the header \(5 bytes\) and footer \(2 bytes\), input " + r"file \".*fixed_length_record.0.txt\" has body length 21 bytes, " + r"which is not an exact multiple of the record length \(4 bytes\).") + ) + + +if __name__ == "__main__": + test.main() diff --git a/tensorflow/python/data/kernel_tests/flat_map_dataset_op_test.py b/tensorflow/python/data/kernel_tests/flat_map_test.py similarity index 57% rename from tensorflow/python/data/kernel_tests/flat_map_dataset_op_test.py rename to tensorflow/python/data/kernel_tests/flat_map_test.py index 68038f9cfc0..ff52821b107 100644 --- a/tensorflow/python/data/kernel_tests/flat_map_dataset_op_test.py +++ b/tensorflow/python/data/kernel_tests/flat_map_test.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================== -"""Tests for the experimental input pipeline ops.""" +"""Tests for `tf.data.Dataset.flat_map()`.""" from __future__ import absolute_import from __future__ import division from __future__ import print_function @@ -26,54 +26,42 @@ from tensorflow.python.data.kernel_tests import test_base from tensorflow.python.data.ops import dataset_ops from tensorflow.python.framework import errors from tensorflow.python.framework import sparse_tensor +from tensorflow.python.framework import test_util from tensorflow.python.ops import sparse_ops from tensorflow.python.platform import test from tensorflow.python.training import server_lib -class FlatMapDatasetTest(test_base.DatasetTestBase): +@test_util.run_all_in_graph_and_eager_modes +class FlatMapTest(test_base.DatasetTestBase): # pylint: disable=g-long-lambda def testFlatMapDataset(self): repeats = [1, 2, 3, 4, 5, 0, 1] components = np.array(repeats, dtype=np.int64) - iterator = ( - dataset_ops.Dataset.from_tensor_slices(components) - .flat_map(lambda x: dataset_ops.Dataset.from_tensors([x]).repeat(x)) - .make_initializable_iterator()) - init_op = iterator.initializer - get_next = iterator.get_next() - - with self.cached_session() as sess: - sess.run(init_op) - for i in repeats: - for _ in range(i): - self.assertEqual(i, sess.run(get_next)) - with self.assertRaises(errors.OutOfRangeError): - sess.run(get_next) + dataset = dataset_ops.Dataset.from_tensor_slices(components).flat_map( + lambda x: dataset_ops.Dataset.from_tensors([x]).repeat(x)) + expected_output = [] + for i in repeats: + expected_output.extend([[i]] * i) + self.assertDatasetProduces(dataset, expected_output=expected_output) def testNestedFlatMapDataset(self): repeats = [[1, 2], [3, 4], [5, 0], [1, 7]] components = np.array(repeats, dtype=np.int64) - iterator = ( - dataset_ops.Dataset.from_tensor_slices(components) - .flat_map(lambda x: dataset_ops.Dataset.from_tensor_slices(x) - .flat_map(lambda y: dataset_ops.Dataset.from_tensors(y) - .repeat(y))).make_initializable_iterator()) - init_op = iterator.initializer - get_next = iterator.get_next() + dataset = dataset_ops.Dataset.from_tensor_slices(components).flat_map( + lambda x: dataset_ops.Dataset.from_tensor_slices(x).flat_map( + lambda y: dataset_ops.Dataset.from_tensors(y).repeat(y)) + ) + expected_output = [] + for row in repeats: + for i in row: + expected_output.extend([i] * i) + self.assertDatasetProduces(dataset, expected_output=expected_output) - with self.cached_session() as sess: - sess.run(init_op) - for row in repeats: - for i in row: - for _ in range(i): - self.assertEqual(i, sess.run(get_next)) - - with self.assertRaises(errors.OutOfRangeError): - sess.run(get_next) - - def testSharedResourceNestedFlatMapDataset(self): + # Note: no eager mode coverage, session specific test. + @test_util.run_deprecated_v1 + def testSkipEagerSharedResourceNestedFlatMapDataset(self): repeats = [[1, 2], [3, 4], [5, 0], [1, 7]] components = np.array(repeats, dtype=np.int64) iterator = ( @@ -106,22 +94,16 @@ class FlatMapDatasetTest(test_base.DatasetTestBase): sess.run(get_next) def testMapDict(self): - iterator = (dataset_ops.Dataset.range(10) - .map(lambda x: {"foo": x * 2, "bar": x ** 2}) - .flat_map(lambda d: dataset_ops.Dataset.from_tensors(d["foo"]) - .repeat(d["bar"])) - .make_initializable_iterator()) - init_op = iterator.initializer - get_next = iterator.get_next() - - with self.cached_session() as sess: - sess.run(init_op) - for i in range(10): - for _ in range(i ** 2): - self.assertEqual(i * 2, sess.run(get_next)) - with self.assertRaises(errors.OutOfRangeError): - sess.run(get_next) - # pylint: enable=g-long-lambda + dataset = dataset_ops.Dataset.range(10).map( + lambda x: {"foo": x * 2, "bar": x ** 2}).flat_map( + lambda d: dataset_ops.Dataset.from_tensors( + d["foo"]).repeat(d["bar"])) + get_next = self.getNext(dataset) + for i in range(10): + for _ in range(i**2): + self.assertEqual(i * 2, self.evaluate(get_next())) + with self.assertRaises(errors.OutOfRangeError): + self.evaluate(get_next()) def testSparse(self): def _map_fn(i): @@ -132,20 +114,12 @@ class FlatMapDatasetTest(test_base.DatasetTestBase): return dataset_ops.Dataset.from_tensor_slices( sparse_ops.sparse_to_dense(x.indices, x.dense_shape, x.values)) - iterator = ( - dataset_ops.Dataset.range(10).map(_map_fn).flat_map(_flat_map_fn) - .make_initializable_iterator()) - init_op = iterator.initializer - get_next = iterator.get_next() - - with self.cached_session() as sess: - sess.run(init_op) - for i in range(10): - for j in range(2): - expected = [i, 0] if j % 2 == 0 else [0, -i] - self.assertAllEqual(expected, sess.run(get_next)) - with self.assertRaises(errors.OutOfRangeError): - sess.run(get_next) + dataset = dataset_ops.Dataset.range(10).map(_map_fn).flat_map(_flat_map_fn) + expected_output = [] + for i in range(10): + for j in range(2): + expected_output.append([i, 0] if j % 2 == 0 else [0, -i]) + self.assertDatasetProduces(dataset, expected_output=expected_output) if __name__ == "__main__": diff --git a/tensorflow/python/data/kernel_tests/dataset_from_generator_op_test.py b/tensorflow/python/data/kernel_tests/from_generator_test.py similarity index 85% rename from tensorflow/python/data/kernel_tests/dataset_from_generator_op_test.py rename to tensorflow/python/data/kernel_tests/from_generator_test.py index cb8cb9a77df..a6625534e7a 100644 --- a/tensorflow/python/data/kernel_tests/dataset_from_generator_op_test.py +++ b/tensorflow/python/data/kernel_tests/from_generator_test.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================== -"""Tests for the experimental input pipeline ops.""" +"""Tests for tf.data.Dataset.from_generator().""" from __future__ import absolute_import from __future__ import division from __future__ import print_function @@ -27,21 +27,21 @@ from tensorflow.python.data.ops import dataset_ops from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes from tensorflow.python.framework import errors +from tensorflow.python.framework import test_util from tensorflow.python.ops import script_ops from tensorflow.python.platform import test -class DatasetConstructorTest(test_base.DatasetTestBase): +class FromGeneratorTest(test_base.DatasetTestBase): def _testFromGenerator(self, generator, elem_sequence, num_repeats, output_types=None): if output_types is None: output_types = dtypes.int64 - iterator = ( + iterator = dataset_ops.make_initializable_iterator( dataset_ops.Dataset.from_generator(generator, output_types=output_types) .repeat(num_repeats) - .prefetch(5) - .make_initializable_iterator()) + .prefetch(5)) init_op = iterator.initializer get_next = iterator.get_next() @@ -55,11 +55,10 @@ class DatasetConstructorTest(test_base.DatasetTestBase): sess.run(get_next) def _testFromGeneratorOneShot(self, generator, elem_sequence, num_repeats): - iterator = ( + iterator = dataset_ops.make_one_shot_iterator( dataset_ops.Dataset.from_generator(generator, output_types=dtypes.int64) .repeat(num_repeats) - .prefetch(5) - .make_one_shot_iterator()) + .prefetch(5)) get_next = iterator.get_next() with self.cached_session() as sess: @@ -69,6 +68,7 @@ class DatasetConstructorTest(test_base.DatasetTestBase): with self.assertRaises(errors.OutOfRangeError): sess.run(get_next) + @test_util.run_deprecated_v1 def testFromGeneratorUsingFunction(self): def generator(): for i in range(1, 100): @@ -79,18 +79,21 @@ class DatasetConstructorTest(test_base.DatasetTestBase): self._testFromGeneratorOneShot(generator, elem_sequence, 1) self._testFromGeneratorOneShot(generator, elem_sequence, 5) + @test_util.run_deprecated_v1 def testFromGeneratorUsingList(self): generator = lambda: [[i] * i for i in range(1, 100)] elem_sequence = list(generator()) self._testFromGenerator(generator, elem_sequence, 1) self._testFromGenerator(generator, elem_sequence, 5) + @test_util.run_deprecated_v1 def testFromGeneratorUsingNdarray(self): generator = lambda: np.arange(100, dtype=np.int64) elem_sequence = list(generator()) self._testFromGenerator(generator, elem_sequence, 1, output_types=np.int64) self._testFromGenerator(generator, elem_sequence, 5, output_types=np.int64) + @test_util.run_deprecated_v1 def testFromGeneratorUsingGeneratorExpression(self): # NOTE(mrry): Generator *expressions* are not repeatable (or in # general reusable), because they eagerly evaluate the `for` @@ -102,6 +105,7 @@ class DatasetConstructorTest(test_base.DatasetTestBase): self._testFromGenerator(generator, elem_sequence, 1) self._testFromGenerator(generator, elem_sequence, 5) + @test_util.run_deprecated_v1 def testFromMultipleConcurrentGenerators(self): num_inner_repeats = 5 num_outer_repeats = 100 @@ -124,11 +128,10 @@ class DatasetConstructorTest(test_base.DatasetTestBase): output_shapes=([None], [3])) .repeat(num_inner_repeats).prefetch(5)) - iterator = ( + iterator = dataset_ops.make_initializable_iterator( dataset_ops.Dataset.range(num_outer_repeats) .interleave(interleave_fn, cycle_length=10, - block_length=len(input_list)) - .make_initializable_iterator()) + block_length=len(input_list))) init_op = iterator.initializer get_next = iterator.get_next() @@ -183,11 +186,10 @@ class DatasetConstructorTest(test_base.DatasetTestBase): return dataset_ops.Dataset.from_generator( generator, output_types=dtypes.int64, output_shapes=[]).prefetch(2) - iterator = ( + iterator = dataset_ops.make_initializable_iterator( dataset_ops.Dataset.range(num_parallel_iterators) .interleave( - interleave_fn, cycle_length=num_parallel_iterators, block_length=1) - .make_initializable_iterator()) + interleave_fn, cycle_length=num_parallel_iterators, block_length=1)) init_op = iterator.initializer get_next = iterator.get_next() @@ -199,6 +201,7 @@ class DatasetConstructorTest(test_base.DatasetTestBase): with self.assertRaises(errors.OutOfRangeError): sess.run(get_next) + @test_util.run_deprecated_v1 def testFromGeneratorImplicitConversion(self): def generator(): yield [1] @@ -206,9 +209,9 @@ class DatasetConstructorTest(test_base.DatasetTestBase): yield [3] for dtype in [dtypes.int8, dtypes.int32, dtypes.int64]: - iterator = (dataset_ops.Dataset.from_generator( - generator, output_types=dtype, output_shapes=[1]) - .make_initializable_iterator()) + iterator = dataset_ops.make_initializable_iterator( + dataset_ops.Dataset.from_generator( + generator, output_types=dtype, output_shapes=[1])) init_op = iterator.initializer get_next = iterator.get_next() @@ -223,15 +226,16 @@ class DatasetConstructorTest(test_base.DatasetTestBase): with self.assertRaises(errors.OutOfRangeError): sess.run(get_next) + @test_util.run_deprecated_v1 def testFromGeneratorString(self): def generator(): yield "foo" yield b"bar" yield u"baz" - iterator = (dataset_ops.Dataset.from_generator( - generator, output_types=dtypes.string, output_shapes=[]) - .make_initializable_iterator()) + iterator = dataset_ops.make_initializable_iterator( + dataset_ops.Dataset.from_generator( + generator, output_types=dtypes.string, output_shapes=[])) init_op = iterator.initializer get_next = iterator.get_next() @@ -243,6 +247,7 @@ class DatasetConstructorTest(test_base.DatasetTestBase): with self.assertRaises(errors.OutOfRangeError): sess.run(get_next) + @test_util.run_deprecated_v1 def testFromGeneratorTypeError(self): def generator(): yield np.array([1, 2, 3], dtype=np.int64) @@ -250,9 +255,9 @@ class DatasetConstructorTest(test_base.DatasetTestBase): yield "ERROR" yield np.array([7, 8, 9], dtype=np.int64) - iterator = (dataset_ops.Dataset.from_generator( - generator, output_types=dtypes.int64, output_shapes=[3]) - .make_initializable_iterator()) + iterator = dataset_ops.make_initializable_iterator( + dataset_ops.Dataset.from_generator( + generator, output_types=dtypes.int64, output_shapes=[3])) init_op = iterator.initializer get_next = iterator.get_next() @@ -266,6 +271,7 @@ class DatasetConstructorTest(test_base.DatasetTestBase): with self.assertRaises(errors.OutOfRangeError): sess.run(get_next) + @test_util.run_deprecated_v1 def testFromGeneratorShapeError(self): def generator(): yield np.array([1, 2, 3], dtype=np.int64) @@ -273,9 +279,9 @@ class DatasetConstructorTest(test_base.DatasetTestBase): yield np.array([7, 8, 9, 10], dtype=np.int64) yield np.array([11, 12, 13], dtype=np.int64) - iterator = (dataset_ops.Dataset.from_generator( - generator, output_types=dtypes.int64, output_shapes=[3]) - .make_initializable_iterator()) + iterator = dataset_ops.make_initializable_iterator( + dataset_ops.Dataset.from_generator( + generator, output_types=dtypes.int64, output_shapes=[3])) init_op = iterator.initializer get_next = iterator.get_next() @@ -289,6 +295,7 @@ class DatasetConstructorTest(test_base.DatasetTestBase): with self.assertRaises(errors.OutOfRangeError): sess.run(get_next) + @test_util.run_deprecated_v1 def testFromGeneratorStructureError(self): def generator(): yield 1, 2 @@ -297,9 +304,9 @@ class DatasetConstructorTest(test_base.DatasetTestBase): yield 6, 7, 8 yield 9, 10 - iterator = (dataset_ops.Dataset.from_generator( - generator, output_types=(dtypes.int64, dtypes.int64)) - .make_initializable_iterator()) + iterator = dataset_ops.make_initializable_iterator( + dataset_ops.Dataset.from_generator( + generator, output_types=(dtypes.int64, dtypes.int64))) init_op = iterator.initializer get_next = iterator.get_next() @@ -317,14 +324,15 @@ class DatasetConstructorTest(test_base.DatasetTestBase): with self.assertRaises(errors.OutOfRangeError): sess.run(get_next) + @test_util.run_deprecated_v1 def testFromGeneratorHeterogeneous(self): def generator(): yield 1 yield [2, 3] - iterator = ( + iterator = dataset_ops.make_initializable_iterator( dataset_ops.Dataset.from_generator( - generator, output_types=dtypes.int64).make_initializable_iterator()) + generator, output_types=dtypes.int64)) init_op = iterator.initializer get_next = iterator.get_next() @@ -335,6 +343,7 @@ class DatasetConstructorTest(test_base.DatasetTestBase): with self.assertRaises(errors.OutOfRangeError): sess.run(get_next) + @test_util.run_deprecated_v1 def testFromGeneratorStopShort(self): def generator(): @@ -342,9 +351,9 @@ class DatasetConstructorTest(test_base.DatasetTestBase): yield 1 yield 2 - iterator = ( + iterator = dataset_ops.make_initializable_iterator( dataset_ops.Dataset.from_generator( - generator, output_types=dtypes.int64).make_initializable_iterator()) + generator, output_types=dtypes.int64)) init_op = iterator.initializer get_next = iterator.get_next() @@ -353,6 +362,7 @@ class DatasetConstructorTest(test_base.DatasetTestBase): self.assertAllEqual(0, sess.run(get_next)) self.assertAllEqual(1, sess.run(get_next)) + @test_util.run_deprecated_v1 def testFromGeneratorDestructorCalled(self): # Use an `Event` to signal that the generator has been deleted. event = threading.Event() @@ -371,9 +381,9 @@ class DatasetConstructorTest(test_base.DatasetTestBase): def __del__(self): event.set() - iterator = dataset_ops.Dataset.from_generator( - GeneratorWrapper, - output_types=dtypes.int64).take(2).make_initializable_iterator() + iterator = dataset_ops.make_initializable_iterator( + dataset_ops.Dataset.from_generator( + GeneratorWrapper, output_types=dtypes.int64).take(2)) init_op = iterator.initializer get_next = iterator.get_next() @@ -387,6 +397,7 @@ class DatasetConstructorTest(test_base.DatasetTestBase): # iterator terminates (and the generator iterator is deleted). self.assertTrue(event.is_set()) + @test_util.run_deprecated_v1 def testFromGeneratorWithArgs(self): def flat_map_fn(elem): @@ -399,10 +410,8 @@ class DatasetConstructorTest(test_base.DatasetTestBase): generator_with_arg, output_types=dtypes.int64, output_shapes=(), args=(elem,)) - iterator = (dataset_ops.Dataset - .range(5) - .flat_map(flat_map_fn) - .make_initializable_iterator()) + iterator = dataset_ops.make_initializable_iterator( + dataset_ops.Dataset.range(5).flat_map(flat_map_fn)) init_op = iterator.initializer get_next = iterator.get_next() @@ -414,6 +423,7 @@ class DatasetConstructorTest(test_base.DatasetTestBase): with self.assertRaises(errors.OutOfRangeError): sess.run(get_next) + @test_util.run_deprecated_v1 def testFromGeneratorWithTwoArgs(self): def flat_map_fn(elem, message): @@ -426,12 +436,11 @@ class DatasetConstructorTest(test_base.DatasetTestBase): generator_with_arg, output_types=(dtypes.int64, dtypes.string), output_shapes=((), ()), args=(elem, message)) - iterator = ( + iterator = dataset_ops.make_initializable_iterator( dataset_ops.Dataset.zip( (dataset_ops.Dataset.range(5), dataset_ops.Dataset.from_tensors("Hi!").repeat(None))) - .flat_map(flat_map_fn) - .make_initializable_iterator()) + .flat_map(flat_map_fn)) init_op = iterator.initializer get_next = iterator.get_next() @@ -446,6 +455,7 @@ class DatasetConstructorTest(test_base.DatasetTestBase): with self.assertRaises(errors.OutOfRangeError): sess.run(get_next) + @test_util.run_deprecated_v1 def testGeneratorDatasetFinalizeFunctionCalled(self): # NOTE(mrry): This test tests the internal `_GeneratorDataset`, # which affords more control over what the finalize function can do than @@ -462,10 +472,9 @@ class DatasetConstructorTest(test_base.DatasetTestBase): stateful=True) dummy = constant_op.constant(37) - iterator = (dataset_ops._GeneratorDataset(dummy, lambda x: x, - lambda x: x, finalize_fn) - .take(2) - .make_initializable_iterator()) + iterator = dataset_ops.make_initializable_iterator( + dataset_ops._GeneratorDataset( + dummy, lambda x: x, lambda x: x, finalize_fn).take(2)) init_op = iterator.initializer get_next = iterator.get_next() diff --git a/tensorflow/python/data/kernel_tests/from_sparse_tensor_slices_test.py b/tensorflow/python/data/kernel_tests/from_sparse_tensor_slices_test.py new file mode 100644 index 00000000000..ef608ebb670 --- /dev/null +++ b/tensorflow/python/data/kernel_tests/from_sparse_tensor_slices_test.py @@ -0,0 +1,86 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Tests for `tf.data.Dataset.from_sparse_tensor_slices()`.""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import numpy as np + +from tensorflow.python.data.kernel_tests import test_base +from tensorflow.python.data.ops import dataset_ops +from tensorflow.python.framework import dtypes +from tensorflow.python.framework import errors +from tensorflow.python.framework import sparse_tensor +from tensorflow.python.framework import test_util +from tensorflow.python.ops import array_ops +from tensorflow.python.platform import test + + +@test_util.run_all_in_graph_and_eager_modes +class FromSparseTensorSlicesTest(test_base.DatasetTestBase): + + @test_util.run_deprecated_v1 + def testSkipEagerFromSparseTensorSlices(self): + """Test a dataset based on slices of a `tf.SparseTensor`.""" + st = array_ops.sparse_placeholder(dtypes.float64) + iterator = dataset_ops.make_initializable_iterator( + dataset_ops.Dataset.from_sparse_tensor_slices(st)) + init_op = iterator.initializer + get_next = sparse_tensor.SparseTensor(*iterator.get_next()) + + with self.cached_session() as sess: + slices = [[1., 2., 3.], [1.], [1.], [1., 2.], [], [1., 2.], [], [], []] + + # Test with sparse tensor in the appropriate order. + indices = np.array( + [[i, j] for i in range(len(slices)) for j in range(len(slices[i]))]) + values = np.array([val for s in slices for val in s]) + dense_shape = np.array([len(slices), max(len(s) for s in slices) + 1]) + sparse_feed = sparse_tensor.SparseTensorValue(indices, values, + dense_shape) + sess.run(init_op, feed_dict={st: sparse_feed}) + for i, s in enumerate(slices): + results = sess.run(get_next) + self.assertAllEqual(s, results.values) + expected_indices = np.array( + [[j] for j in range(len(slices[i]))]).reshape([-1, 1]) + self.assertAllEqual(expected_indices, results.indices) + self.assertAllEqual(dense_shape[1:], results.dense_shape) + with self.assertRaises(errors.OutOfRangeError): + sess.run(get_next) + + # Test with sparse tensor in the reverse order, which is not + # currently supported. + reverse_order_indices = indices[::-1, :] + reverse_order_values = values[::-1] + sparse_feed = sparse_tensor.SparseTensorValue( + reverse_order_indices, reverse_order_values, dense_shape) + with self.assertRaises(errors.UnimplementedError): + sess.run(init_op, feed_dict={st: sparse_feed}) + + # Test with an empty sparse tensor. + empty_indices = np.empty((0, 4), dtype=np.int64) + empty_values = np.empty((0,), dtype=np.float64) + empty_dense_shape = [0, 4, 37, 9] + sparse_feed = sparse_tensor.SparseTensorValue(empty_indices, empty_values, + empty_dense_shape) + sess.run(init_op, feed_dict={st: sparse_feed}) + with self.assertRaises(errors.OutOfRangeError): + sess.run(get_next) + + +if __name__ == "__main__": + test.main() diff --git a/tensorflow/python/data/kernel_tests/from_tensor_slices_test.py b/tensorflow/python/data/kernel_tests/from_tensor_slices_test.py new file mode 100644 index 00000000000..9a480e56789 --- /dev/null +++ b/tensorflow/python/data/kernel_tests/from_tensor_slices_test.py @@ -0,0 +1,177 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Tests for `tf.data.Dataset.from_tensor_slices().""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import numpy as np + +from tensorflow.python.data.kernel_tests import test_base +from tensorflow.python.data.ops import dataset_ops +from tensorflow.python.framework import dtypes +from tensorflow.python.framework import errors +from tensorflow.python.framework import sparse_tensor +from tensorflow.python.framework import tensor_shape +from tensorflow.python.framework import test_util +from tensorflow.python.platform import test + + +@test_util.run_all_in_graph_and_eager_modes +class FromTensorSlicesTest(test_base.DatasetTestBase): + + def testFromTensorSlices(self): + """Test a dataset that represents the slices from a tuple of tensors.""" + components = ( + np.tile(np.array([[1], [2], [3], [4]]), 20), np.tile( + np.array([[12], [13], [14], [15]]), 22), + np.array([37.0, 38.0, 39.0, 40.0]) + ) + + dataset = dataset_ops.Dataset.from_tensor_slices(components) + get_next = self.getNext(dataset) + + self.assertEqual([c.shape[1:] for c in components], + [shape for shape in dataset.output_shapes]) + + for i in range(4): + results = self.evaluate(get_next()) + for component, result_component in zip(components, results): + self.assertAllEqual(component[i], result_component) + with self.assertRaises(errors.OutOfRangeError): + results = self.evaluate(get_next()) + + def testSkipEagerFromTensorSlicesSparse(self): + """Test a dataset that represents the slices from a tuple of tensors.""" + components = (sparse_tensor.SparseTensorValue( + indices=np.array([[0, 0], [1, 0], [2, 0]]), + values=np.array([0, 0, 0]), + dense_shape=np.array([3, 1])), + sparse_tensor.SparseTensorValue( + indices=np.array([[0, 0], [1, 1], [2, 2]]), + values=np.array([1, 2, 3]), + dense_shape=np.array([3, 3]))) + + dataset = dataset_ops.Dataset.from_tensor_slices(components) + + self.assertEqual( + [tensor_shape.TensorShape(c.dense_shape[1:]) for c in components], + [shape for shape in dataset.output_shapes]) + + expected = [ + (sparse_tensor.SparseTensorValue( + indices=np.array([[0]]), + values=np.array([0]), + dense_shape=np.array([1])), + sparse_tensor.SparseTensorValue( + indices=np.array([[0]]), + values=np.array([1]), + dense_shape=np.array([3]))), + (sparse_tensor.SparseTensorValue( + indices=np.array([[0]]), + values=np.array([0]), + dense_shape=np.array([1])), + sparse_tensor.SparseTensorValue( + indices=np.array([[1]]), + values=np.array([2]), + dense_shape=np.array([3]))), + (sparse_tensor.SparseTensorValue( + indices=np.array([[0]]), + values=np.array([0]), + dense_shape=np.array([1])), + sparse_tensor.SparseTensorValue( + indices=np.array([[2]]), + values=np.array([3]), + dense_shape=np.array([3]))), + ] + self.assertDatasetProduces(dataset, expected_output=expected) + + def testFromTensorSlicesMixed(self): + """Test a dataset that represents the slices from a tuple of tensors.""" + components = (np.tile(np.array([[1], [2], [3]]), 20), + np.tile(np.array([[12], [13], [14]]), 22), + np.array([37.0, 38.0, 39.0]), + sparse_tensor.SparseTensorValue( + indices=np.array([[0, 0], [1, 0], [2, 0]]), + values=np.array([0, 0, 0]), + dense_shape=np.array([3, 1])), + sparse_tensor.SparseTensorValue( + indices=np.array([[0, 0], [1, 1], [2, 2]]), + values=np.array([1, 2, 3]), + dense_shape=np.array([3, 3]))) + + dataset = dataset_ops.Dataset.from_tensor_slices(components) + get_next = self.getNext(dataset) + self.assertEqual([ + tensor_shape.TensorShape(c.dense_shape[1:]) + if sparse_tensor.is_sparse(c) else c.shape[1:] for c in components + ], [shape for shape in dataset.output_shapes]) + + expected = [ + (sparse_tensor.SparseTensorValue( + indices=np.array([[0]]), + values=np.array([0]), + dense_shape=np.array([1])), + sparse_tensor.SparseTensorValue( + indices=np.array([[0]]), + values=np.array([1]), + dense_shape=np.array([3]))), + (sparse_tensor.SparseTensorValue( + indices=np.array([[0]]), + values=np.array([0]), + dense_shape=np.array([1])), + sparse_tensor.SparseTensorValue( + indices=np.array([[1]]), + values=np.array([2]), + dense_shape=np.array([3]))), + (sparse_tensor.SparseTensorValue( + indices=np.array([[0]]), + values=np.array([0]), + dense_shape=np.array([1])), + sparse_tensor.SparseTensorValue( + indices=np.array([[2]]), + values=np.array([3]), + dense_shape=np.array([3]))), + ] + for i in range(3): + results = self.evaluate(get_next()) + for component, result_component in zip( + (list(zip(*components[:3]))[i] + expected[i]), results): + if sparse_tensor.is_sparse(component): + self.assertSparseValuesEqual(component, result_component) + else: + self.assertAllEqual(component, result_component) + with self.assertRaises(errors.OutOfRangeError): + self.evaluate(get_next()) + + def testFromTensorSlicesWithDict(self): + components = {"foo": [1, 2, 3], "bar": [[4.0], [5.0], [6.0]]} + dataset = dataset_ops.Dataset.from_tensor_slices(components) + get_next = self.getNext(dataset) + + self.assertEqual(dtypes.int32, dataset.output_types["foo"]) + self.assertEqual(dtypes.float32, dataset.output_types["bar"]) + self.assertEqual((), dataset.output_shapes["foo"]) + self.assertEqual((1,), dataset.output_shapes["bar"]) + + for i in range(3): + results = self.evaluate(get_next()) + self.assertEqual(components["foo"][i], results["foo"]) + self.assertEqual(components["bar"][i], results["bar"]) + with self.assertRaises(errors.OutOfRangeError): + self.evaluate(get_next()) + +if __name__ == "__main__": + test.main() diff --git a/tensorflow/python/data/kernel_tests/from_tensors_test.py b/tensorflow/python/data/kernel_tests/from_tensors_test.py new file mode 100644 index 00000000000..ab3c15263fd --- /dev/null +++ b/tensorflow/python/data/kernel_tests/from_tensors_test.py @@ -0,0 +1,259 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Tests for `tf.data.Dataset.from_tensors().""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import numpy as np + +from tensorflow.core.protobuf import config_pb2 +from tensorflow.python.client import session +from tensorflow.python.data.kernel_tests import test_base +from tensorflow.python.data.ops import dataset_ops +from tensorflow.python.data.util import nest +from tensorflow.python.framework import dtypes +from tensorflow.python.framework import errors +from tensorflow.python.framework import ops +from tensorflow.python.framework import sparse_tensor +from tensorflow.python.framework import tensor_shape +from tensorflow.python.framework import test_util +from tensorflow.python.ops import array_ops +from tensorflow.python.ops import math_ops +from tensorflow.python.ops import resource_variable_ops +from tensorflow.python.platform import test + + +@test_util.run_all_in_graph_and_eager_modes +class FromTensorsTest(test_base.DatasetTestBase): + + def testFromTensors(self): + """Test a dataset that represents a single tuple of tensors.""" + components = (np.array(1), np.array([1, 2, 3]), np.array(37.0)) + + dataset = dataset_ops.Dataset.from_tensors(components) + + self.assertEqual([c.shape for c in components], + nest.flatten(dataset.output_shapes)) + + self.assertDatasetProduces(dataset, expected_output=[components]) + + def testSkipEagerFromTensorsSparse(self): + """Test a dataset that represents a single tuple of tensors.""" + components = (sparse_tensor.SparseTensorValue( + indices=np.array([[0]]), + values=np.array([0]), + dense_shape=np.array([1])), + sparse_tensor.SparseTensorValue( + indices=np.array([[0, 0], [1, 1]]), + values=np.array([-1, 1]), + dense_shape=np.array([2, 2]))) + + dataset = dataset_ops.Dataset.from_tensors(components) + + self.assertEqual( + [tensor_shape.TensorShape(c.dense_shape) for c in components], + [shape for shape in dataset.output_shapes]) + self.assertDatasetProduces(dataset, expected_output=[components]) + + def testFromTensorsMixed(self): + """Test an dataset that represents a single tuple of tensors.""" + components = (np.array(1), np.array([1, 2, 3]), np.array(37.0), + sparse_tensor.SparseTensorValue( + indices=np.array([[0]]), + values=np.array([0]), + dense_shape=np.array([1])), + sparse_tensor.SparseTensorValue( + indices=np.array([[0, 0], [1, 1]]), + values=np.array([-1, 1]), + dense_shape=np.array([2, 2]))) + + dataset = dataset_ops.Dataset.from_tensors(components) + self.assertEqual([ + tensor_shape.TensorShape(c.dense_shape) + if sparse_tensor.is_sparse(c) else c.shape for c in components + ], [shape for shape in dataset.output_shapes]) + + self.assertDatasetProduces(dataset, expected_output=[components]) + + # pylint: disable=g-long-lambda,unnecessary-lambda + def testNestedStructure(self): + components = (np.array([1, 2, 3], dtype=np.int64), + (np.array([4., 5.]), np.array([6., 7.])), + np.array([8, 9, 10], dtype=np.int64)) + + dataset = dataset_ops.Dataset.from_tensors(components) + self.assertEquals((dtypes.int64, (dtypes.float64, dtypes.float64), + dtypes.int64), dataset.output_types) + self.assertEquals(([3], ([2], [2]), [3]), dataset.output_shapes) + + dataset = dataset.shuffle(10, 10) + self.assertEquals((dtypes.int64, (dtypes.float64, dtypes.float64), + dtypes.int64), dataset.output_types) + self.assertEquals(([3], ([2], [2]), [3]), dataset.output_shapes) + + dataset = dataset.repeat(-1) + self.assertEquals((dtypes.int64, (dtypes.float64, dtypes.float64), + dtypes.int64), dataset.output_types) + self.assertEquals(([3], ([2], [2]), [3]), dataset.output_shapes) + + dataset = dataset.filter(lambda x, y, z: True) + self.assertEquals((dtypes.int64, (dtypes.float64, dtypes.float64), + dtypes.int64), dataset.output_types) + self.assertEquals(([3], ([2], [2]), [3]), dataset.output_shapes) + + dataset = dataset.take(5) + self.assertEquals((dtypes.int64, (dtypes.float64, dtypes.float64), + dtypes.int64), dataset.output_types) + self.assertEquals(([3], ([2], [2]), [3]), dataset.output_shapes) + + dataset = dataset.map(lambda x, y, z: ((x, z), (y[0], y[1]))) + self.assertEquals(((dtypes.int64, dtypes.int64), + (dtypes.float64, dtypes.float64)), dataset.output_types) + self.assertEquals((([3], [3]), ([2], [2])), dataset.output_shapes) + + dataset = dataset.flat_map( + lambda x, y: dataset_ops.Dataset.from_tensors(((x[0], x[1]), + (y[0], y[1]))) + ) + self.assertEquals(((dtypes.int64, dtypes.int64), + (dtypes.float64, dtypes.float64)), dataset.output_types) + self.assertEquals((([3], [3]), ([2], [2])), dataset.output_shapes) + + dataset = dataset.batch(32) + self.assertEquals(((dtypes.int64, dtypes.int64), + (dtypes.float64, dtypes.float64)), dataset.output_types) + self.assertEquals((([None, 3], [None, 3]), ([None, 2], [None, 2])), + nest.pack_sequence_as(dataset.output_shapes, [ + s.as_list() + for s in nest.flatten(dataset.output_shapes) + ])) + + # Define a separate set of components with matching leading + # dimension for the from-slices constructor. + components_for_slices = (np.array([1, 2, 3], dtype=np.int64), + (np.array([4., 5., 6.]), np.array([7., 8., 9.])), + np.array([10, 11, 12], dtype=np.int64)) + + dataset = dataset_ops.Dataset.from_tensor_slices(components_for_slices) + self.assertEquals((dtypes.int64, + (dtypes.float64, dtypes.float64), dtypes.int64), + dataset.output_types) + self.assertEquals(([], ([], []), []), dataset.output_shapes) + + # TODO(b/117581999): more specific shapes in eager mode. + @test_util.run_deprecated_v1 + def testSkipEagerNestedStructure(self): + components = (np.array([1, 2, 3], dtype=np.int64), (np.array([4., 5.]), + np.array([6., 7.])), + np.array([8, 9, 10], dtype=np.int64)) + + dataset = dataset_ops.Dataset.from_tensors(components) + dataset = dataset.map(lambda x, y, z: ((x, z), (y[0], y[1]))) + + dataset = dataset.flat_map( + lambda x, y: dataset_ops.Dataset.from_tensors( + ((x[0], x[1]), (y[0], y[1])))).batch(32) + + get_next = self.getNext(dataset) + (w, x), (y, z) = get_next() + self.assertEquals(dtypes.int64, w.dtype) + self.assertEquals(dtypes.int64, x.dtype) + self.assertEquals(dtypes.float64, y.dtype) + self.assertEquals(dtypes.float64, z.dtype) + self.assertEquals([None, 3], w.shape.as_list()) + self.assertEquals([None, 3], x.shape.as_list()) + self.assertEquals([None, 2], y.shape.as_list()) + self.assertEquals([None, 2], z.shape.as_list()) + + get_next = self.getNext(dataset) + (w, x), (y, z) = get_next() + self.assertEquals(dtypes.int64, w.dtype) + self.assertEquals(dtypes.int64, x.dtype) + self.assertEquals(dtypes.float64, y.dtype) + self.assertEquals(dtypes.float64, z.dtype) + self.assertEquals([None, 3], w.shape.as_list()) + self.assertEquals([None, 3], x.shape.as_list()) + self.assertEquals([None, 2], y.shape.as_list()) + self.assertEquals([None, 2], z.shape.as_list()) + + def testNestedDict(self): + components = {"a": {"aa": 1, "ab": [2.0, 2.0]}, "b": [3, 3, 3]} + dataset = dataset_ops.Dataset.from_tensors(components) + self.assertEquals(dtypes.int32, dataset.output_types["a"]["aa"]) + self.assertEquals(dtypes.float32, dataset.output_types["a"]["ab"]) + self.assertEquals(dtypes.int32, dataset.output_types["b"]) + self.assertEquals([], dataset.output_shapes["a"]["aa"]) + self.assertEquals([2], dataset.output_shapes["a"]["ab"]) + self.assertEquals([3], dataset.output_shapes["b"]) + + def testNonSequenceNestedStructure(self): + components = np.array([1, 2, 3], dtype=np.int64) + + dataset = dataset_ops.Dataset.from_tensors(components) + self.assertEquals(dtypes.int64, dataset.output_types) + self.assertEquals([3], dataset.output_shapes) + + dataset = dataset.filter( + lambda x: math_ops.reduce_all(math_ops.equal(x, components))) + self.assertEquals(dtypes.int64, dataset.output_types) + self.assertEquals([3], dataset.output_shapes) + + dataset = dataset.map(lambda x: array_ops.stack([x, x])) + self.assertEquals(dtypes.int64, dataset.output_types) + self.assertEquals([2, 3], dataset.output_shapes) + + dataset = dataset.flat_map( + lambda x: dataset_ops.Dataset.from_tensor_slices(x)) + self.assertEquals(dtypes.int64, dataset.output_types) + self.assertEquals([3], dataset.output_shapes) + + get_next = self.getNext(dataset) + self.assertEquals(dtypes.int64, get_next().dtype) + self.assertEquals([3], get_next().shape) + + def testSkipEagerSplitPipelineFailsWithPlacementError(self): + with session.Session( + target="", + config=config_pb2.ConfigProto(device_count={"CPU": 2})) as sess: + + dataset = dataset_ops.Dataset.from_tensors(0) + + # Define a pipeline that attempts to use variables on two + # different devices. + # + # Initialize the variables before creating to iterator, to avoid the + # placement algorithm overriding the DT_RESOURCE colocation constraints. + with ops.device("/cpu:0"): + var_0 = resource_variable_ops.ResourceVariable(initial_value=0) + dataset = dataset.map(lambda x: x + var_0.read_value()) + sess.run(var_0.initializer) + + with ops.device("/cpu:1"): + var_1 = resource_variable_ops.ResourceVariable(initial_value=0) + dataset = dataset.map(lambda x: x + var_1.read_value()) + sess.run(var_1.initializer) + + iterator = dataset_ops.make_initializable_iterator(dataset) + sess.run(iterator.initializer) + + with self.assertRaisesRegexp( + errors.FailedPreconditionError, + "Error while reading resource variable Variable"): + sess.run(iterator.get_next()) + + +if __name__ == "__main__": + test.main() diff --git a/tensorflow/python/data/kernel_tests/inputs_test.py b/tensorflow/python/data/kernel_tests/inputs_test.py deleted file mode 100644 index d089b49bcc6..00000000000 --- a/tensorflow/python/data/kernel_tests/inputs_test.py +++ /dev/null @@ -1,149 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from absl.testing import parameterized -import numpy as np - -from tensorflow.python.data.kernel_tests import test_base -from tensorflow.python.data.ops import dataset_ops -from tensorflow.python.data.ops import readers -from tensorflow.python.data.util import nest -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import sparse_tensor -from tensorflow.python.platform import test - - -class InputsTest(test_base.DatasetTestBase, parameterized.TestCase): - - @staticmethod - def make_apply_fn(dataset): - - def apply_fn(dataset): - - def _apply_fn(dataset): - return dataset.cache() - - return dataset.apply(_apply_fn) - - return apply_fn - - @staticmethod - def make_gen(): - - def gen(): - yield 42 - - return gen - - @staticmethod - def make_interleave_fn(dataset, num_parallel_calls=None): - - def interleave_fn(dataset): - return dataset.interleave( - lambda x: dataset_ops.Dataset.range(0), - cycle_length=2, - num_parallel_calls=num_parallel_calls) - - return interleave_fn - - @parameterized.named_parameters( - ("FixedLengthRecord", readers.FixedLengthRecordDataset("", 42)), - ("FromGenerator", - dataset_ops.Dataset.from_generator(make_gen.__func__(), dtypes.int32), - 1), - ("FromSparseTensorSlices", - dataset_ops.Dataset.from_sparse_tensor_slices( - sparse_tensor.SparseTensor( - indices=np.array([[0, 0], [1, 0], [2, 0]]), - values=np.array([0, 0, 0]), - dense_shape=np.array([3, 1])))), - ("FromTensors", dataset_ops.Dataset.from_tensors([42])), - ("FromTensorSlices", dataset_ops.Dataset.from_tensors([42])), - ("Range", dataset_ops.Dataset.range(10)), - ("TextLine", readers.TextLineDataset("")), - ("TFRecord", readers.TFRecordDataset(""), 1), - ) - def testDatasetSourceInputs(self, dataset, num_inputs=0): - self.assertEqual(num_inputs, len(dataset._inputs())) - - @parameterized.named_parameters( - ("Apply", make_apply_fn.__func__(dataset_ops.Dataset.range(0)), - dataset_ops.Dataset.range(0)), - ("Batch", lambda x: x.batch(10), dataset_ops.Dataset.range(0)), - ("Cache", lambda x: x.cache(), dataset_ops.Dataset.range(0)), - ("Filter", lambda x: x.filter(lambda x: True), - dataset_ops.Dataset.range(0)), - ("FlatMap", lambda x: x.flat_map(lambda x: dataset_ops.Dataset.range(0)), - dataset_ops.Dataset.range(0)), - ("Interleave", make_interleave_fn.__func__(dataset_ops.Dataset.range(0)), - dataset_ops.Dataset.range(0)), - ("Map", lambda x: x.map(lambda x: x), dataset_ops.Dataset.range(0)), - ("PaddedBatch", lambda x: x.padded_batch(10, []), - dataset_ops.Dataset.range(0)), - ("ParallelInterleave", - make_interleave_fn.__func__(dataset_ops.Dataset.range(0), 2), - dataset_ops.Dataset.range(0)), - ("ParallelMap", lambda x: x.map(lambda x: x, num_parallel_calls=2), - dataset_ops.Dataset.range(0)), - ("Repeat", lambda x: x.repeat(), dataset_ops.Dataset.range(0)), - ("Shuffle", lambda x: x.shuffle(10), dataset_ops.Dataset.range(0)), - ("Skip", lambda x: x.skip(1), dataset_ops.Dataset.range(0)), - ("Take", lambda x: x.take(1), dataset_ops.Dataset.range(0)), - ("Window", lambda x: x.window(10), dataset_ops.Dataset.range(0)), - ) - def testUnaryTransformationInputs(self, dataset_fn, input_dataset): - self.assertEqual([input_dataset], dataset_fn(input_dataset)._inputs()) - - @parameterized.named_parameters( - ("Concatenate", lambda x, y: x.concatenate(y), - dataset_ops.Dataset.range(0), dataset_ops.Dataset.range(1))) - def testBinaryTransformationInputs(self, dataset_fn, input1, input2): - self.assertEqual([input1, input2], dataset_fn(input1, input2)._inputs()) - - @parameterized.named_parameters( - ("ZipOne", dataset_ops.Dataset.zip, (dataset_ops.Dataset.range(0))), - ("ZipNest", dataset_ops.Dataset.zip, - (dataset_ops.Dataset.range(0), - (dataset_ops.Dataset.range(1), dataset_ops.Dataset.range(2)))), - ("ZipTuple", dataset_ops.Dataset.zip, - (dataset_ops.Dataset.range(0), dataset_ops.Dataset.range(1)))) - def testVariadicTransformationInputs(self, dataset_fn, input_datasets): - self.assertEqual( - nest.flatten(input_datasets), - dataset_fn(input_datasets)._inputs()) - - def testCollectInputs(self): - ds1 = dataset_ops.Dataset.range(0) - ds2 = ds1.concatenate(ds1) - ds3 = dataset_ops.Dataset.zip((ds2, ds1, ds2)) - - inputs = [] - queue = [ds3] - while queue: - ds = queue[0] - queue = queue[1:] - queue.extend(ds._inputs()) - inputs.append(ds) - - self.assertEqual(5, inputs.count(ds1)) - self.assertEqual(2, inputs.count(ds2)) - self.assertEqual(1, inputs.count(ds3)) - - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/python/data/kernel_tests/interleave_dataset_op_test.py b/tensorflow/python/data/kernel_tests/interleave_test.py similarity index 85% rename from tensorflow/python/data/kernel_tests/interleave_dataset_op_test.py rename to tensorflow/python/data/kernel_tests/interleave_test.py index b911c249ced..c3450e65251 100644 --- a/tensorflow/python/data/kernel_tests/interleave_dataset_op_test.py +++ b/tensorflow/python/data/kernel_tests/interleave_test.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================== -"""Tests for the experimental input pipeline ops.""" +"""Tests for `tf.data.Dataset.interleave()`.""" from __future__ import absolute_import from __future__ import division from __future__ import print_function @@ -27,6 +27,7 @@ from tensorflow.python.data.kernel_tests import test_base from tensorflow.python.data.ops import dataset_ops from tensorflow.python.framework import errors from tensorflow.python.framework import sparse_tensor +from tensorflow.python.framework import test_util from tensorflow.python.ops import array_ops from tensorflow.python.ops import script_ops from tensorflow.python.ops import sparse_ops @@ -115,7 +116,7 @@ def _make_coordinated_sloppy_dataset(input_values, cycle_length, block_length, dataset = dataset_ops.Dataset.from_tensor_slices(input_values).repeat( 2).interleave(interleave_fn, cycle_length, block_length, num_parallel_calls).with_options(options) - iterator = dataset.make_one_shot_iterator() + iterator = dataset_ops.make_one_shot_iterator(dataset) get_next = iterator.get_next() return get_next, coordination_events @@ -133,7 +134,8 @@ def _repeat(values, count): return [[value] * value for value in np.tile(values, count)] -class InterleaveDatasetTest(test_base.DatasetTestBase, parameterized.TestCase): +@test_util.run_all_in_graph_and_eager_modes +class InterleaveTest(test_base.DatasetTestBase, parameterized.TestCase): @parameterized.named_parameters( ("1", [4, 5, 6], 1, 1, [ @@ -191,16 +193,11 @@ class InterleaveDatasetTest(test_base.DatasetTestBase, parameterized.TestCase): count).interleave( lambda x: dataset_ops.Dataset.from_tensors(x).repeat(x), cycle_length, block_length, num_parallel_calls) - get_next = dataset.make_one_shot_iterator().get_next() - - with self.cached_session() as sess: - for expected_element in _interleave( - _repeat(input_values, count), cycle_length, block_length): - self.assertEqual(expected_element, sess.run(get_next)) - - for _ in range(2): - with self.assertRaises(errors.OutOfRangeError): - sess.run(get_next) + expected_output = [ + element for element in _interleave( + _repeat(input_values, count), cycle_length, block_length) + ] + self.assertDatasetProduces(dataset, expected_output) @parameterized.named_parameters( ("1", np.float32([1., np.nan, 2., np.nan, 3.]), 1, 3, None), @@ -223,17 +220,16 @@ class InterleaveDatasetTest(test_base.DatasetTestBase, parameterized.TestCase): lambda x: array_ops.check_numerics(x, "message")).interleave( dataset_ops.Dataset.from_tensors, cycle_length, block_length, num_parallel_calls) - get_next = dataset.make_one_shot_iterator().get_next() + get_next = self.getNext(dataset) - with self.cached_session() as sess: - for value in input_values: - if np.isnan(value): - with self.assertRaises(errors.InvalidArgumentError): - sess.run(get_next) - else: - self.assertEqual(value, sess.run(get_next)) - with self.assertRaises(errors.OutOfRangeError): - sess.run(get_next) + for value in input_values: + if np.isnan(value): + with self.assertRaises(errors.InvalidArgumentError): + self.evaluate(get_next()) + else: + self.assertEqual(value, self.evaluate(get_next())) + with self.assertRaises(errors.OutOfRangeError): + self.evaluate(get_next()) def testInterleaveSparse(self): @@ -245,18 +241,17 @@ class InterleaveDatasetTest(test_base.DatasetTestBase, parameterized.TestCase): return dataset_ops.Dataset.from_tensor_slices( sparse_ops.sparse_to_dense(x.indices, x.dense_shape, x.values)) - iterator = ( - dataset_ops.Dataset.range(10).map(_map_fn).interleave( - _interleave_fn, cycle_length=1).make_one_shot_iterator()) - get_next = iterator.get_next() - - with self.cached_session() as sess: - for i in range(10): - for j in range(2): - expected = [i, 0] if j % 2 == 0 else [0, -i] - self.assertAllEqual(expected, sess.run(get_next)) - with self.assertRaises(errors.OutOfRangeError): - sess.run(get_next) + dataset = dataset_ops.Dataset.range(10).map(_map_fn).interleave( + _interleave_fn, cycle_length=1) + get_next = self.getNext(dataset) + for i in range(10): + for j in range(2): + expected = [i, 0] if j % 2 == 0 else [0, -i] + self.assertAllEqual(expected, self.evaluate(get_next())) + with self.assertRaises(errors.OutOfRangeError): + self.evaluate(get_next()) + with self.assertRaises(errors.OutOfRangeError): + self.evaluate(get_next()) @parameterized.named_parameters( ("1", np.int64([4, 5, 6]), 2, 1, 1), @@ -269,8 +264,8 @@ class InterleaveDatasetTest(test_base.DatasetTestBase, parameterized.TestCase): ("8", np.int64([4, 0, 6]), 2, 3, 1), ("9", np.int64([4, 0, 6]), 2, 3, 2), ) - def testSloppyInterleaveInOrder(self, input_values, cycle_length, - block_length, num_parallel_calls): + def testSkipEagerSloppyInterleaveInOrder(self, input_values, cycle_length, + block_length, num_parallel_calls): get_next, coordination_events = _make_coordinated_sloppy_dataset( input_values, cycle_length, block_length, num_parallel_calls) config = config_pb2.ConfigProto( @@ -281,7 +276,7 @@ class InterleaveDatasetTest(test_base.DatasetTestBase, parameterized.TestCase): _repeat(input_values, 2), cycle_length, block_length): coordination_events[expected_element].set() self.assertEqual(expected_element * expected_element, - sess.run(get_next)) + self.evaluate(get_next)) with self.assertRaises(errors.OutOfRangeError): sess.run(get_next) @@ -291,8 +286,8 @@ class InterleaveDatasetTest(test_base.DatasetTestBase, parameterized.TestCase): ("3", np.int64([4, 5, 6]), 3, 2, 3), ("4", np.int64([4, 0, 6]), 2, 3, 2), ) - def testSloppyInterleaveOutOfOrder(self, input_values, cycle_length, - block_length, num_parallel_calls): + def testSkipEagerSloppyInterleaveOutOfOrder(self, input_values, cycle_length, + block_length, num_parallel_calls): get_next, coordination_events = _make_coordinated_sloppy_dataset( input_values, cycle_length, block_length, num_parallel_calls) config = config_pb2.ConfigProto( @@ -308,7 +303,7 @@ class InterleaveDatasetTest(test_base.DatasetTestBase, parameterized.TestCase): for element in elements: coordination_events[element].set() - self.assertEqual(element * element, sess.run(get_next)) + self.assertEqual(element * element, self.evaluate(get_next)) with self.assertRaises(errors.OutOfRangeError): sess.run(get_next) diff --git a/tensorflow/python/data/kernel_tests/iterator_checkpoint_test.py b/tensorflow/python/data/kernel_tests/iterator_checkpoint_test.py new file mode 100644 index 00000000000..91b356691b7 --- /dev/null +++ b/tensorflow/python/data/kernel_tests/iterator_checkpoint_test.py @@ -0,0 +1,129 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Checkpoint tests for `tf.data.Iterator`.""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import functools +import os + +from tensorflow.python.data.kernel_tests import test_base +from tensorflow.python.data.ops import dataset_ops +from tensorflow.python.eager import context +from tensorflow.python.framework import errors +from tensorflow.python.framework import test_util +from tensorflow.python.ops import math_ops +from tensorflow.python.platform import test +from tensorflow.python.training import checkpoint_management +from tensorflow.python.training.checkpointable import util as checkpointable_utils + + +@test_util.run_all_in_graph_and_eager_modes +class IteratorCheckpointingTest(test_base.DatasetTestBase): + + def testSaveRestoreOneShotIterator(self): + checkpoint_directory = self.get_temp_dir() + checkpoint_prefix = os.path.join(checkpoint_directory, "ckpt") + dataset = dataset_ops.Dataset.from_tensor_slices([1, 2, 3, 4, 5, 6]).map( + math_ops.square).batch(2) + iterator = iter(dataset) if context.executing_eagerly( + ) else dataset_ops.make_one_shot_iterator(dataset) + get_next = iterator.get_next if context.executing_eagerly( + ) else functools.partial(self.evaluate, iterator.get_next()) + checkpoint = checkpointable_utils.Checkpoint(iterator=iterator) + self.assertAllEqual([1, 4], get_next()) + save_path = checkpoint.save(checkpoint_prefix) + self.assertAllEqual([9, 16], get_next()) + self.assertAllEqual([25, 36], get_next()) + checkpoint.restore(save_path).run_restore_ops() + self.assertAllEqual([9, 16], get_next()) + self.assertAllEqual([25, 36], get_next()) + with self.assertRaises(errors.OutOfRangeError): + get_next() + + def testSaveRestoreMultipleIterator(self): + checkpoint_directory = self.get_temp_dir() + checkpoint_prefix = os.path.join(checkpoint_directory, "ckpt") + dataset = dataset_ops.Dataset.from_tensor_slices( + [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]) + dataset = dataset.map(math_ops.square).batch(2) + iterator_1 = iter(dataset) if context.executing_eagerly( + ) else dataset_ops.make_one_shot_iterator(dataset) + get_next_1 = iterator_1.get_next if context.executing_eagerly( + ) else functools.partial(self.evaluate, iterator_1.get_next()) + iterator_2 = iter(dataset) if context.executing_eagerly( + ) else dataset_ops.make_one_shot_iterator(dataset) + get_next_2 = iterator_2.get_next if context.executing_eagerly( + ) else functools.partial(self.evaluate, iterator_2.get_next()) + dataset_2 = dataset_ops.Dataset.range(10) + iterator_3 = iter(dataset_2) if context.executing_eagerly( + ) else dataset_ops.make_one_shot_iterator(dataset_2) + get_next_3 = iterator_3.get_next if context.executing_eagerly( + ) else functools.partial(self.evaluate, iterator_3.get_next()) + checkpoint = checkpointable_utils.Checkpoint( + iterator_1=iterator_1, iterator_2=iterator_2, iterator_3=iterator_3) + self.assertAllEqual([1, 4], get_next_1()) + self.assertAllEqual(0, get_next_3()) + self.assertAllEqual(1, get_next_3()) + self.assertAllEqual(2, get_next_3()) + save_path = checkpoint.save(checkpoint_prefix) + self.assertAllEqual([1, 4], get_next_2()) + self.assertAllEqual([9, 16], get_next_2()) + self.assertAllEqual(3, get_next_3()) + checkpoint.restore(save_path).run_restore_ops() + self.assertAllEqual([9, 16], get_next_1()) + self.assertAllEqual([1, 4], get_next_2()) + self.assertAllEqual(3, get_next_3()) + + def testRestoreExhaustedIterator(self): + checkpoint_directory = self.get_temp_dir() + checkpoint_prefix = os.path.join(checkpoint_directory, "ckpt") + dataset = dataset_ops.Dataset.range(3) + iterator = iter(dataset) if context.executing_eagerly( + ) else dataset_ops.make_one_shot_iterator(dataset) + get_next = iterator.get_next if context.executing_eagerly( + ) else functools.partial(self.evaluate, iterator.get_next()) + checkpoint = checkpointable_utils.Checkpoint(iterator=iterator) + self.assertAllEqual(0, get_next()) + self.assertAllEqual(1, get_next()) + save_path = checkpoint.save(checkpoint_prefix) + self.assertAllEqual(2, get_next()) + checkpoint.restore(save_path).run_restore_ops() + self.assertAllEqual(2, get_next()) + save_path = checkpoint.save(checkpoint_prefix) + checkpoint.restore(save_path).run_restore_ops() + with self.assertRaises(errors.OutOfRangeError): + get_next() + + def testRestoreInReconstructedIteratorInitializable(self): + checkpoint_directory = self.get_temp_dir() + checkpoint_prefix = os.path.join(checkpoint_directory, "ckpt") + dataset = dataset_ops.Dataset.range(10) + iterator = iter(dataset) if context.executing_eagerly( + ) else dataset_ops.make_initializable_iterator(dataset) + get_next = iterator.get_next + checkpoint = checkpointable_utils.Checkpoint(iterator=iterator) + for i in range(5): + checkpoint.restore( + checkpoint_management.latest_checkpoint( + checkpoint_directory)).initialize_or_restore() + for j in range(2): + self.assertEqual(i * 2 + j, self.evaluate(get_next())) + checkpoint.save(file_prefix=checkpoint_prefix) + + +if __name__ == "__main__": + test.main() diff --git a/tensorflow/python/data/kernel_tests/iterator_ops_cluster_test.py b/tensorflow/python/data/kernel_tests/iterator_cluster_test.py similarity index 96% rename from tensorflow/python/data/kernel_tests/iterator_ops_cluster_test.py rename to tensorflow/python/data/kernel_tests/iterator_cluster_test.py index bf5fd781d65..728bed20a12 100644 --- a/tensorflow/python/data/kernel_tests/iterator_ops_cluster_test.py +++ b/tensorflow/python/data/kernel_tests/iterator_cluster_test.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================== -"""Tests for the experimental input pipeline ops that need test_util.""" +"""Tests for `tf.data.Iterator` using distributed sessions.""" from __future__ import absolute_import from __future__ import division from __future__ import print_function @@ -47,7 +47,7 @@ class IteratorClusterTest(test.TestCase): with ops.device("/job:worker/replica:0/task:0/cpu:1"): dataset_3 = dataset_ops.Dataset.from_tensor_slices([1, 2, 3]) - iterator_3 = dataset_3.make_one_shot_iterator() + iterator_3 = dataset_ops.make_one_shot_iterator(dataset_3) iterator_3_handle = iterator_3.string_handle() with ops.device("/job:worker/replica:0/task:0/cpu:0"): @@ -62,7 +62,7 @@ class IteratorClusterTest(test.TestCase): def _testRemoteIteratorHelper(self, device0, device1, target): with ops.device(device1): dataset_3 = dataset_ops.Dataset.from_tensor_slices([1, 2, 3]) - iterator_3 = dataset_3.make_one_shot_iterator() + iterator_3 = dataset_ops.make_one_shot_iterator(dataset_3) iterator_3_handle = iterator_3.string_handle() @function.Defun(dtypes.string) @@ -161,7 +161,7 @@ class IteratorClusterTest(test.TestCase): dataset_ops.Dataset.from_tensor_slices(components).map(_map_fn) .repeat(None).prefetch(10000)) - iterator = dataset.make_initializable_iterator() + iterator = dataset_ops.make_initializable_iterator(dataset) init_op = iterator.initializer get_next = iterator.get_next() diff --git a/tensorflow/python/data/kernel_tests/iterator_ops_test.py b/tensorflow/python/data/kernel_tests/iterator_test.py similarity index 83% rename from tensorflow/python/data/kernel_tests/iterator_ops_test.py rename to tensorflow/python/data/kernel_tests/iterator_test.py index a2a3528cc62..916cf8bb45c 100644 --- a/tensorflow/python/data/kernel_tests/iterator_ops_test.py +++ b/tensorflow/python/data/kernel_tests/iterator_test.py @@ -12,12 +12,11 @@ # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================== -"""Tests for the experimental input pipeline ops.""" +"""Tests for `tf.data.Iterator`.""" from __future__ import absolute_import from __future__ import division from __future__ import print_function -import functools import os import warnings @@ -50,24 +49,24 @@ from tensorflow.python.ops import parsing_ops from tensorflow.python.ops import script_ops from tensorflow.python.ops import variables from tensorflow.python.platform import test -from tensorflow.python.training import checkpoint_management from tensorflow.python.training import server_lib -from tensorflow.python.training.checkpointable import util as checkpointable_utils from tensorflow.python.util import compat class IteratorTest(test.TestCase, parameterized.TestCase): + @test_util.run_deprecated_v1 def testNoGradients(self): component = constant_op.constant([1.]) side = constant_op.constant(0.) add = lambda x: x + side dataset = dataset_ops.Dataset.from_tensor_slices(component).map(add) - value = dataset.make_one_shot_iterator().get_next() + value = dataset_ops.make_one_shot_iterator(dataset).get_next() self.assertIsNone(gradients_impl.gradients(value, component)[0]) self.assertIsNone(gradients_impl.gradients(value, side)[0]) self.assertIsNone(gradients_impl.gradients(value, [component, side])[0]) + @test_util.run_deprecated_v1 def testCapturingStateInOneShotRaisesException(self): var = variables.Variable(37.0, name="myvar") dataset = ( @@ -76,8 +75,9 @@ class IteratorTest(test.TestCase, parameterized.TestCase): with self.assertRaisesRegexp( ValueError, r"`Dataset.make_one_shot_iterator\(\)` does not support " "datasets that capture stateful objects.+myvar"): - dataset.make_one_shot_iterator() + dataset_ops.make_one_shot_iterator(dataset) + @test_util.run_deprecated_v1 def testOneShotIterator(self): components = (np.arange(7), np.array([[1, 2, 3]]) * np.arange(7)[:, np.newaxis], @@ -86,9 +86,9 @@ class IteratorTest(test.TestCase, parameterized.TestCase): def _map_fn(x, y, z): return math_ops.square(x), math_ops.square(y), math_ops.square(z) - iterator = ( + iterator = dataset_ops.make_one_shot_iterator( dataset_ops.Dataset.from_tensor_slices(components).map(_map_fn) - .repeat(14).make_one_shot_iterator()) + .repeat(14)) get_next = iterator.get_next() self.assertEqual([c.shape[1:] for c in components], @@ -103,6 +103,7 @@ class IteratorTest(test.TestCase, parameterized.TestCase): with self.assertRaises(errors.OutOfRangeError): sess.run(get_next) + @test_util.run_deprecated_v1 def testOneShotIteratorCaptureByValue(self): components = (np.arange(7), np.array([[1, 2, 3]]) * np.arange(7)[:, np.newaxis], @@ -112,9 +113,9 @@ class IteratorTest(test.TestCase, parameterized.TestCase): def _map_fn(x, y, z): return math_ops.square(x), math_ops.square(y), math_ops.square(z) - iterator = ( + iterator = dataset_ops.make_one_shot_iterator( dataset_ops.Dataset.from_tensor_slices(tensor_components) - .map(_map_fn).repeat(14).make_one_shot_iterator()) + .map(_map_fn).repeat(14)) get_next = iterator.get_next() self.assertEqual([c.shape[1:] for c in components], @@ -139,9 +140,9 @@ class IteratorTest(test.TestCase, parameterized.TestCase): def _map_fn(x, y, z): return math_ops.square(x), math_ops.square(y), math_ops.square(z) - iterator = ( + iterator = dataset_ops.make_one_shot_iterator( dataset_ops.Dataset.from_tensor_slices(components) - .map(_map_fn).repeat(14).make_one_shot_iterator()) + .map(_map_fn).repeat(14)) return iterator.get_next() server = server_lib.Server.create_local_server() @@ -165,9 +166,10 @@ class IteratorTest(test.TestCase, parameterized.TestCase): with self.assertRaises(errors.OutOfRangeError): sess.run(get_next) + @test_util.run_deprecated_v1 def testOneShotIteratorNonBlocking(self): dataset = dataset_ops.Dataset.from_tensors([1, 2, 3]).map(lambda x: x * x) - iterator = dataset.make_one_shot_iterator() + iterator = dataset_ops.make_one_shot_iterator(dataset) next_element = iterator.get_next() # Create a session with a single thread to ensure that the @@ -203,12 +205,13 @@ class IteratorTest(test.TestCase, parameterized.TestCase): len([None for r in results if r is None])) self.assertAllEqual([[1, 4, 9]], [r for r in results if r is not None]) + @test_util.run_deprecated_v1 def testOneShotIteratorInitializerFails(self): # Define a dataset whose initialization will always fail. dataset = dataset_ops.Dataset.from_tensors( array_ops.check_numerics( constant_op.constant(1.0) / constant_op.constant(0.0), "oops")) - iterator = dataset.make_one_shot_iterator() + iterator = dataset_ops.make_one_shot_iterator(dataset) next_element = iterator.get_next() with self.cached_session() as sess: @@ -283,11 +286,11 @@ class IteratorTest(test.TestCase, parameterized.TestCase): with self.assertRaises(errors.OutOfRangeError): sess.run(get_next) + @test_util.run_deprecated_v1 def testNotInitializedError(self): components = (np.array(1), np.array([1, 2, 3]), np.array(37.0)) - iterator = ( - dataset_ops.Dataset.from_tensors(components) - .make_initializable_iterator()) + iterator = dataset_ops.make_initializable_iterator( + dataset_ops.Dataset.from_tensors(components)) get_next = iterator.get_next() with self.cached_session() as sess: @@ -295,6 +298,7 @@ class IteratorTest(test.TestCase, parameterized.TestCase): "iterator has not been initialized"): sess.run(get_next) + @test_util.run_deprecated_v1 def testReinitializableIterator(self): dataset_3 = dataset_ops.Dataset.from_tensors( constant_op.constant([1, 2, 3])) @@ -334,6 +338,33 @@ class IteratorTest(test.TestCase, parameterized.TestCase): with self.assertRaises(errors.OutOfRangeError): sess.run(get_next) + @test_util.run_deprecated_v1 + def testReinitializableIteratorWithFunctions(self): + + def g(): + for i in range(10): + yield i + + iterator = iterator_ops.Iterator.from_structure(dtypes.int64, []) + next_element = iterator.get_next() + + with self.cached_session() as sess: + dataset_1 = dataset_ops.Dataset.from_generator( + g, output_types=dtypes.int64) + sess.run(iterator.make_initializer(dataset_1)) + for expected in range(10): + self.assertEqual(expected, sess.run(next_element)) + with self.assertRaises(errors.OutOfRangeError): + sess.run(next_element) + + dataset_2 = dataset_ops.Dataset.from_generator( + g, output_types=dtypes.int64) + sess.run(iterator.make_initializer(dataset_2)) + for expected in range(10): + self.assertEqual(expected, sess.run(next_element)) + with self.assertRaises(errors.OutOfRangeError): + sess.run(next_element) + def testReinitializableIteratorStaticErrors(self): # Non-matching structure for types and shapes. with self.assertRaises(TypeError): @@ -367,12 +398,13 @@ class IteratorTest(test.TestCase, parameterized.TestCase): (constant_op.constant([1, 2, 3], dtype=dtypes.int64), constant_op.constant([4., 5., 6., 7.], dtype=dtypes.float64)))) + @test_util.run_deprecated_v1 def testIteratorStringHandle(self): dataset_3 = dataset_ops.Dataset.from_tensor_slices([1, 2, 3]) dataset_4 = dataset_ops.Dataset.from_tensor_slices([10, 20, 30, 40]) - iterator_3 = dataset_3.make_one_shot_iterator() - iterator_4 = dataset_4.make_one_shot_iterator() + iterator_3 = dataset_ops.make_one_shot_iterator(dataset_3) + iterator_4 = dataset_ops.make_one_shot_iterator(dataset_4) handle_placeholder = array_ops.placeholder(dtypes.string, shape=[]) feedable_iterator = iterator_ops.Iterator.from_string_handle( @@ -422,13 +454,14 @@ class IteratorTest(test.TestCase, parameterized.TestCase): sess.run( next_element, feed_dict={handle_placeholder: iterator_4_handle}) + @test_util.run_deprecated_v1 def testIteratorStringHandleFuture(self): with forward_compat.forward_compatibility_horizon(2018, 8, 4): dataset_3 = dataset_ops.Dataset.from_tensor_slices([1, 2, 3]) dataset_4 = dataset_ops.Dataset.from_tensor_slices([10, 20, 30, 40]) - iterator_3 = dataset_3.make_one_shot_iterator() - iterator_4 = dataset_4.make_one_shot_iterator() + iterator_3 = dataset_ops.make_one_shot_iterator(dataset_3) + iterator_4 = dataset_ops.make_one_shot_iterator(dataset_4) handle_placeholder = array_ops.placeholder(dtypes.string, shape=[]) feedable_iterator = iterator_ops.Iterator.from_string_handle( @@ -485,10 +518,11 @@ class IteratorTest(test.TestCase, parameterized.TestCase): sess.run( next_element, feed_dict={handle_placeholder: iterator_4_handle}) + @test_util.run_deprecated_v1 def testIteratorStringHandleReuseTensorObject(self): dataset = dataset_ops.Dataset.from_tensor_slices([1, 2, 3]) - one_shot_iterator = dataset.make_one_shot_iterator() - initializable_iterator = dataset.make_initializable_iterator() + one_shot_iterator = dataset_ops.make_one_shot_iterator(dataset) + initializable_iterator = dataset_ops.make_initializable_iterator(dataset) structure_iterator = iterator_ops.Iterator.from_structure( dataset.output_types) @@ -513,6 +547,7 @@ class IteratorTest(test.TestCase, parameterized.TestCase): self.assertEqual("foo_1", handle_with_same_name.op.name) self.assertIsNot(handle_with_name, handle_with_same_name) + @test_util.run_deprecated_v1 def testIteratorStringHandleError(self): dataset_int_scalar = ( dataset_ops.Dataset.from_tensor_slices([1, 2, 3]).repeat()) @@ -528,10 +563,10 @@ class IteratorTest(test.TestCase, parameterized.TestCase): handle_placeholder, dtypes.int32) with self.cached_session() as sess: - handle_int_scalar = sess.run( - dataset_int_scalar.make_one_shot_iterator().string_handle()) - handle_float_vector = sess.run( - dataset_float_vector.make_one_shot_iterator().string_handle()) + handle_int_scalar = sess.run(dataset_ops.make_one_shot_iterator( + dataset_int_scalar).string_handle()) + handle_float_vector = sess.run(dataset_ops.make_one_shot_iterator( + dataset_float_vector).string_handle()) self.assertEqual(1, sess.run( @@ -553,13 +588,14 @@ class IteratorTest(test.TestCase, parameterized.TestCase): feedable_int_vector.get_next(), feed_dict={handle_placeholder: handle_float_vector})) + @test_util.run_deprecated_v1 def testRemoteIteratorUsingRemoteCallOpDirectSession(self): worker_config = config_pb2.ConfigProto() worker_config.device_count["CPU"] = 3 with ops.device("/job:localhost/replica:0/task:0/cpu:1"): dataset_3 = dataset_ops.Dataset.from_tensor_slices([1, 2, 3]) - iterator_3 = dataset_3.make_one_shot_iterator() + iterator_3 = dataset_ops.make_one_shot_iterator(dataset_3) iterator_3_handle = iterator_3.string_handle() @function.Defun(dtypes.string) @@ -609,6 +645,7 @@ class IteratorTest(test.TestCase, parameterized.TestCase): target_placeholder: "/job:localhost/replica:0/task:0/cpu:1" }) + @test_util.run_deprecated_v1 def testRemoteIteratorUsingRemoteCallOpMultiWorkers(self): s1 = server_lib.Server.create_local_server() s2 = server_lib.Server.create_local_server() @@ -631,7 +668,7 @@ class IteratorTest(test.TestCase, parameterized.TestCase): for device in worker_devices: with ops.device(device): src = dataset_ops.Dataset.from_tensor_slices([device]) - itr = src.make_one_shot_iterator() + itr = dataset_ops.make_one_shot_iterator(src) itr_handles.append(itr.string_handle()) targets = dataset_ops.Dataset.from_tensor_slices(worker_devices) @@ -649,7 +686,7 @@ class IteratorTest(test.TestCase, parameterized.TestCase): with ops.device("/job:client"): client_dataset = dataset_ops.Dataset.zip((targets, handles)).map(map_fn) - itr = client_dataset.make_initializable_iterator() + itr = dataset_ops.make_initializable_iterator(client_dataset) n = itr.get_next() with session.Session(s3.target, config=config) as sess: @@ -667,7 +704,7 @@ class IteratorTest(test.TestCase, parameterized.TestCase): with ops.device("/job:localhost/replica:0/task:0/cpu:0"): dataset_3 = dataset_ops.Dataset.from_tensor_slices([1, 2, 3]) - iterator_3 = dataset_3.make_one_shot_iterator() + iterator_3 = dataset_ops.make_one_shot_iterator(dataset_3) iterator_3_handle = iterator_3.string_handle() def _encode_raw(byte_array): @@ -716,6 +753,7 @@ class IteratorTest(test.TestCase, parameterized.TestCase): target_placeholder: "/job:localhost/replica:0/task:0/cpu:0" }) + @test_util.run_deprecated_v1 def testIncorrectIteratorRestore(self): def _path(): @@ -738,8 +776,8 @@ class IteratorTest(test.TestCase, parameterized.TestCase): def _build_range_dataset_graph(): start = 1 stop = 10 - iterator = dataset_ops.Dataset.range(start, - stop).make_initializable_iterator() + iterator = dataset_ops.make_initializable_iterator( + dataset_ops.Dataset.range(start, stop)) init_op = iterator.initializer get_next = iterator.get_next() save_op = _save_op(iterator._iterator_resource) @@ -748,8 +786,8 @@ class IteratorTest(test.TestCase, parameterized.TestCase): def _build_reader_dataset_graph(): filenames = ["test"] # Does not exist but we don't care in this test. - iterator = readers.FixedLengthRecordDataset( - filenames, 1, 0, 0).make_initializable_iterator() + iterator = dataset_ops.make_initializable_iterator( + readers.FixedLengthRecordDataset(filenames, 1, 0, 0)) init_op = iterator.initializer get_next_op = iterator.get_next() save_op = _save_op(iterator._iterator_resource) @@ -774,8 +812,9 @@ class IteratorTest(test.TestCase, parameterized.TestCase): with self.assertRaises(errors.InvalidArgumentError): sess.run(restore_op) + @test_util.run_deprecated_v1 def testRepeatedGetNextWarning(self): - iterator = dataset_ops.Dataset.range(10).make_one_shot_iterator() + iterator = dataset_ops.make_one_shot_iterator(dataset_ops.Dataset.range(10)) warnings.simplefilter("always") with warnings.catch_warnings(record=True) as w: for _ in range(100): @@ -818,8 +857,8 @@ class IteratorTest(test.TestCase, parameterized.TestCase): expected_output_classes, expected_output_types, expected_output_shapes): tf_value = tf_value_fn() - iterator = dataset_ops.Dataset.from_tensors( - tf_value).make_one_shot_iterator() + iterator = dataset_ops.make_one_shot_iterator( + dataset_ops.Dataset.from_tensors(tf_value)) self.assertTrue(expected_element_structure.is_compatible_with( iterator._element_structure)) @@ -832,100 +871,11 @@ class IteratorTest(test.TestCase, parameterized.TestCase): def testIteratorGetNextName(self): with ops.Graph().as_default(): - iterator = dataset_ops.Dataset.from_tensors(37.0).make_one_shot_iterator() + iterator = dataset_ops.make_one_shot_iterator( + dataset_ops.Dataset.from_tensors(37.0)) next_element = iterator.get_next(name="overridden_name") self.assertEqual("overridden_name", next_element.op.name) -class IteratorCheckpointingTest(test.TestCase): - - @test_util.run_in_graph_and_eager_modes - def testSaveRestoreOneShotIterator(self): - checkpoint_directory = self.get_temp_dir() - checkpoint_prefix = os.path.join(checkpoint_directory, "ckpt") - dataset = dataset_ops.Dataset.from_tensor_slices([1, 2, 3, 4, 5, 6]).map( - math_ops.square).batch(2) - iterator = dataset.make_one_shot_iterator() - get_next = iterator.get_next if context.executing_eagerly( - ) else functools.partial(self.evaluate, iterator.get_next()) - checkpoint = checkpointable_utils.Checkpoint(iterator=iterator) - self.assertAllEqual([1, 4], get_next()) - save_path = checkpoint.save(checkpoint_prefix) - self.assertAllEqual([9, 16], get_next()) - self.assertAllEqual([25, 36], get_next()) - checkpoint.restore(save_path).run_restore_ops() - self.assertAllEqual([9, 16], get_next()) - self.assertAllEqual([25, 36], get_next()) - with self.assertRaises(errors.OutOfRangeError): - get_next() - - @test_util.run_in_graph_and_eager_modes - def testSaveRestoreMultipleIterator(self): - checkpoint_directory = self.get_temp_dir() - checkpoint_prefix = os.path.join(checkpoint_directory, "ckpt") - dataset = dataset_ops.Dataset.from_tensor_slices( - [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]) - dataset = dataset.map(math_ops.square).batch(2) - iterator_1 = dataset.make_one_shot_iterator() - get_next_1 = iterator_1.get_next if context.executing_eagerly( - ) else functools.partial(self.evaluate, iterator_1.get_next()) - iterator_2 = dataset.make_one_shot_iterator() - get_next_2 = iterator_2.get_next if context.executing_eagerly( - ) else functools.partial(self.evaluate, iterator_2.get_next()) - dataset_2 = dataset_ops.Dataset.range(10) - iterator_3 = dataset_2.make_one_shot_iterator() - get_next_3 = iterator_3.get_next if context.executing_eagerly( - ) else functools.partial(self.evaluate, iterator_3.get_next()) - checkpoint = checkpointable_utils.Checkpoint( - iterator_1=iterator_1, iterator_2=iterator_2, iterator_3=iterator_3) - self.assertAllEqual([1, 4], get_next_1()) - self.assertAllEqual(0, get_next_3()) - self.assertAllEqual(1, get_next_3()) - self.assertAllEqual(2, get_next_3()) - save_path = checkpoint.save(checkpoint_prefix) - self.assertAllEqual([1, 4], get_next_2()) - self.assertAllEqual([9, 16], get_next_2()) - self.assertAllEqual(3, get_next_3()) - checkpoint.restore(save_path).run_restore_ops() - self.assertAllEqual([9, 16], get_next_1()) - self.assertAllEqual([1, 4], get_next_2()) - self.assertAllEqual(3, get_next_3()) - - @test_util.run_in_graph_and_eager_modes - def testRestoreExhaustedIterator(self): - checkpoint_directory = self.get_temp_dir() - checkpoint_prefix = os.path.join(checkpoint_directory, "ckpt") - dataset = dataset_ops.Dataset.range(3) - iterator = dataset.make_one_shot_iterator() - get_next = iterator.get_next if context.executing_eagerly( - ) else functools.partial(self.evaluate, iterator.get_next()) - checkpoint = checkpointable_utils.Checkpoint(iterator=iterator) - self.assertAllEqual(0, get_next()) - self.assertAllEqual(1, get_next()) - save_path = checkpoint.save(checkpoint_prefix) - self.assertAllEqual(2, get_next()) - checkpoint.restore(save_path).run_restore_ops() - self.assertAllEqual(2, get_next()) - save_path = checkpoint.save(checkpoint_prefix) - checkpoint.restore(save_path).run_restore_ops() - with self.assertRaises(errors.OutOfRangeError): - get_next() - - def testRestoreInReconstructedIteratorInitializable(self): - checkpoint_directory = self.get_temp_dir() - checkpoint_prefix = os.path.join(checkpoint_directory, "ckpt") - dataset = dataset_ops.Dataset.range(10) - iterator = dataset.make_initializable_iterator() - get_next = iterator.get_next() - checkpoint = checkpointable_utils.Checkpoint(iterator=iterator) - for i in range(5): - with self.cached_session() as sess: - checkpoint.restore(checkpoint_management.latest_checkpoint( - checkpoint_directory)).initialize_or_restore(sess) - for j in range(2): - self.assertEqual(i * 2 + j, sess.run(get_next)) - checkpoint.save(file_prefix=checkpoint_prefix) - - if __name__ == "__main__": test.main() diff --git a/tensorflow/python/data/kernel_tests/list_files_dataset_op_test.py b/tensorflow/python/data/kernel_tests/list_files_dataset_op_test.py deleted file mode 100644 index b58c1444dae..00000000000 --- a/tensorflow/python/data/kernel_tests/list_files_dataset_op_test.py +++ /dev/null @@ -1,291 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for the experimental input pipeline ops.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from os import path -import shutil -import tempfile - -from tensorflow.python.data.kernel_tests import test_base -from tensorflow.python.data.ops import dataset_ops -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import errors -from tensorflow.python.ops import array_ops -from tensorflow.python.platform import test -from tensorflow.python.util import compat - - -class ListFilesDatasetOpTest(test_base.DatasetTestBase): - - def setUp(self): - self.tmp_dir = tempfile.mkdtemp() - - def tearDown(self): - shutil.rmtree(self.tmp_dir, ignore_errors=True) - - def _touchTempFiles(self, filenames): - for filename in filenames: - open(path.join(self.tmp_dir, filename), 'a').close() - - def testEmptyDirectory(self): - dataset = dataset_ops.Dataset.list_files(path.join(self.tmp_dir, '*')) - with self.cached_session() as sess: - itr = dataset.make_one_shot_iterator() - next_element = itr.get_next() - with self.assertRaises(errors.OutOfRangeError): - sess.run(next_element) - - def testSimpleDirectory(self): - filenames = ['a', 'b', 'c'] - self._touchTempFiles(filenames) - - dataset = dataset_ops.Dataset.list_files(path.join(self.tmp_dir, '*')) - with self.cached_session() as sess: - itr = dataset.make_one_shot_iterator() - next_element = itr.get_next() - - full_filenames = [] - produced_filenames = [] - for filename in filenames: - full_filenames.append( - compat.as_bytes(path.join(self.tmp_dir, filename))) - produced_filenames.append(compat.as_bytes(sess.run(next_element))) - self.assertItemsEqual(full_filenames, produced_filenames) - with self.assertRaises(errors.OutOfRangeError): - sess.run(itr.get_next()) - - def testSimpleDirectoryNotShuffled(self): - filenames = ['b', 'c', 'a'] - self._touchTempFiles(filenames) - - dataset = dataset_ops.Dataset.list_files( - path.join(self.tmp_dir, '*'), shuffle=False) - with self.cached_session() as sess: - itr = dataset.make_one_shot_iterator() - next_element = itr.get_next() - - for filename in sorted(filenames): - self.assertEqual(compat.as_bytes(path.join(self.tmp_dir, filename)), - sess.run(next_element)) - with self.assertRaises(errors.OutOfRangeError): - sess.run(itr.get_next()) - - def testFixedSeedResultsInRepeatableOrder(self): - filenames = ['a', 'b', 'c'] - self._touchTempFiles(filenames) - - dataset = dataset_ops.Dataset.list_files( - path.join(self.tmp_dir, '*'), shuffle=True, seed=37) - with self.cached_session() as sess: - itr = dataset.make_initializable_iterator() - next_element = itr.get_next() - - full_filenames = [compat.as_bytes(path.join(self.tmp_dir, filename)) - for filename in filenames] - - all_produced_filenames = [] - for _ in range(3): - produced_filenames = [] - sess.run(itr.initializer) - try: - while True: - produced_filenames.append(sess.run(next_element)) - except errors.OutOfRangeError: - pass - all_produced_filenames.append(produced_filenames) - - # Each run should produce the same set of filenames, which may be - # different from the order of `full_filenames`. - self.assertItemsEqual(full_filenames, all_produced_filenames[0]) - # However, the different runs should produce filenames in the same order - # as each other. - self.assertEqual(all_produced_filenames[0], all_produced_filenames[1]) - self.assertEqual(all_produced_filenames[0], all_produced_filenames[2]) - - def testEmptyDirectoryInitializer(self): - filename_placeholder = array_ops.placeholder(dtypes.string, shape=[]) - dataset = dataset_ops.Dataset.list_files(filename_placeholder) - - with self.cached_session() as sess: - itr = dataset.make_initializable_iterator() - with self.assertRaisesRegexp( - errors.InvalidArgumentError, 'No files matched pattern: '): - sess.run( - itr.initializer, - feed_dict={filename_placeholder: path.join(self.tmp_dir, '*')}) - - def testSimpleDirectoryInitializer(self): - filenames = ['a', 'b', 'c'] - self._touchTempFiles(filenames) - - filename_placeholder = array_ops.placeholder(dtypes.string, shape=[]) - dataset = dataset_ops.Dataset.list_files(filename_placeholder) - - with self.cached_session() as sess: - itr = dataset.make_initializable_iterator() - next_element = itr.get_next() - sess.run( - itr.initializer, - feed_dict={filename_placeholder: path.join(self.tmp_dir, '*')}) - - full_filenames = [] - produced_filenames = [] - for filename in filenames: - full_filenames.append( - compat.as_bytes(path.join(self.tmp_dir, filename))) - produced_filenames.append(compat.as_bytes(sess.run(next_element))) - - self.assertItemsEqual(full_filenames, produced_filenames) - - with self.assertRaises(errors.OutOfRangeError): - sess.run(itr.get_next()) - - def testFileSuffixes(self): - filenames = ['a.txt', 'b.py', 'c.py', 'd.pyc'] - self._touchTempFiles(filenames) - - filename_placeholder = array_ops.placeholder(dtypes.string, shape=[]) - dataset = dataset_ops.Dataset.list_files(filename_placeholder) - - with self.cached_session() as sess: - itr = dataset.make_initializable_iterator() - next_element = itr.get_next() - sess.run( - itr.initializer, - feed_dict={filename_placeholder: path.join(self.tmp_dir, '*.py')}) - - full_filenames = [] - produced_filenames = [] - for filename in filenames[1:-1]: - full_filenames.append( - compat.as_bytes(path.join(self.tmp_dir, filename))) - produced_filenames.append(compat.as_bytes(sess.run(next_element))) - self.assertItemsEqual(full_filenames, produced_filenames) - - with self.assertRaises(errors.OutOfRangeError): - sess.run(itr.get_next()) - - def testFileMiddles(self): - filenames = ['a.txt', 'b.py', 'c.pyc'] - self._touchTempFiles(filenames) - - filename_placeholder = array_ops.placeholder(dtypes.string, shape=[]) - dataset = dataset_ops.Dataset.list_files(filename_placeholder) - - with self.cached_session() as sess: - itr = dataset.make_initializable_iterator() - next_element = itr.get_next() - sess.run( - itr.initializer, - feed_dict={filename_placeholder: path.join(self.tmp_dir, '*.py*')}) - - full_filenames = [] - produced_filenames = [] - for filename in filenames[1:]: - full_filenames.append( - compat.as_bytes(path.join(self.tmp_dir, filename))) - produced_filenames.append(compat.as_bytes(sess.run(next_element))) - - self.assertItemsEqual(full_filenames, produced_filenames) - - with self.assertRaises(errors.OutOfRangeError): - sess.run(itr.get_next()) - - def testNoShuffle(self): - filenames = ['a', 'b', 'c'] - self._touchTempFiles(filenames) - - # Repeat the list twice and ensure that the order is the same each time. - # NOTE(mrry): This depends on an implementation detail of `list_files()`, - # which is that the list of files is captured when the iterator is - # initialized. Otherwise, or if e.g. the iterator were initialized more than - # once, it's possible that the non-determinism of `tf.matching_files()` - # would cause this test to fail. However, it serves as a useful confirmation - # that the `shuffle=False` argument is working as intended. - # TODO(b/73959787): Provide some ordering guarantees so that this test is - # more meaningful. - dataset = dataset_ops.Dataset.list_files( - path.join(self.tmp_dir, '*'), shuffle=False).repeat(2) - with self.cached_session() as sess: - itr = dataset.make_one_shot_iterator() - next_element = itr.get_next() - - full_filenames = [] - produced_filenames = [] - for filename in filenames * 2: - full_filenames.append( - compat.as_bytes(path.join(self.tmp_dir, filename))) - produced_filenames.append(compat.as_bytes(sess.run(next_element))) - with self.assertRaises(errors.OutOfRangeError): - sess.run(itr.get_next()) - self.assertItemsEqual(full_filenames, produced_filenames) - self.assertEqual(produced_filenames[:len(filenames)], - produced_filenames[len(filenames):]) - - def testMultiplePatternsAsList(self): - filenames = ['a.txt', 'b.py', 'c.py', 'd.pyc'] - self._touchTempFiles(filenames) - - patterns = [path.join(self.tmp_dir, pat) for pat in ['*.py', '*.txt']] - dataset = dataset_ops.Dataset.list_files(patterns) - with self.cached_session() as sess: - itr = dataset.make_one_shot_iterator() - next_element = itr.get_next() - - full_filenames = [] - produced_filenames = [] - for filename in filenames[:-1]: - full_filenames.append( - compat.as_bytes(path.join(self.tmp_dir, filename))) - produced_filenames.append(compat.as_bytes(sess.run(next_element))) - self.assertItemsEqual(full_filenames, produced_filenames) - - with self.assertRaises(errors.OutOfRangeError): - sess.run(itr.get_next()) - - def testMultiplePatternsAsTensor(self): - filenames = ['a.txt', 'b.py', 'c.py', 'd.pyc'] - self._touchTempFiles(filenames) - - filename_placeholder = array_ops.placeholder( - dtypes.string, shape=[ - 2, - ]) - dataset = dataset_ops.Dataset.list_files(filename_placeholder) - - with self.cached_session() as sess: - itr = dataset.make_initializable_iterator() - next_element = itr.get_next() - patterns = [path.join(self.tmp_dir, pat) for pat in ['*.py', '*.txt']] - sess.run(itr.initializer, feed_dict={filename_placeholder: patterns}) - - full_filenames = [] - produced_filenames = [] - for filename in filenames[:-1]: - full_filenames.append( - compat.as_bytes(path.join(self.tmp_dir, filename))) - produced_filenames.append(compat.as_bytes(sess.run(next_element))) - self.assertItemsEqual(full_filenames, produced_filenames) - - with self.assertRaises(errors.OutOfRangeError): - sess.run(itr.get_next()) - - -if __name__ == '__main__': - test.main() diff --git a/tensorflow/python/data/kernel_tests/list_files_test.py b/tensorflow/python/data/kernel_tests/list_files_test.py new file mode 100644 index 00000000000..789f1ab6de7 --- /dev/null +++ b/tensorflow/python/data/kernel_tests/list_files_test.py @@ -0,0 +1,214 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Tests for `tf.data.Dataset.list_files()`.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from os import path +import shutil +import tempfile + +from tensorflow.python.data.kernel_tests import test_base +from tensorflow.python.data.ops import dataset_ops +from tensorflow.python.framework import errors +from tensorflow.python.framework import test_util +from tensorflow.python.platform import test +from tensorflow.python.util import compat + + +@test_util.run_all_in_graph_and_eager_modes +class ListFilesTest(test_base.DatasetTestBase): + + def setUp(self): + self.tmp_dir = tempfile.mkdtemp() + + def tearDown(self): + shutil.rmtree(self.tmp_dir, ignore_errors=True) + + def _touchTempFiles(self, filenames): + for filename in filenames: + open(path.join(self.tmp_dir, filename), 'a').close() + + # Note: eager mode fails in assertion error same as initializer in graph mode. + @test_util.run_deprecated_v1 + def testSkipEagerEmptyDirectory(self): + dataset = dataset_ops.Dataset.list_files(path.join(self.tmp_dir, '*')) + self.assertDatasetProduces(dataset, expected_output=[]) + + def testSimpleDirectory(self): + filenames = ['a', 'b', 'c'] + self._touchTempFiles(filenames) + + dataset = dataset_ops.Dataset.list_files(path.join(self.tmp_dir, '*')) + self.assertDatasetProduces( + dataset, + expected_output=[ + compat.as_bytes(path.join(self.tmp_dir, filename)) + for filename in filenames + ], + assert_items_equal=True) + + def testSimpleDirectoryNotShuffled(self): + filenames = ['b', 'c', 'a'] + self._touchTempFiles(filenames) + + dataset = dataset_ops.Dataset.list_files( + path.join(self.tmp_dir, '*'), shuffle=False) + self.assertDatasetProduces( + dataset, + expected_output=[ + compat.as_bytes(path.join(self.tmp_dir, filename)) + for filename in sorted(filenames) + ]) + + def testFixedSeedResultsInRepeatableOrder(self): + filenames = ['a', 'b', 'c'] + self._touchTempFiles(filenames) + + dataset = dataset_ops.Dataset.list_files( + path.join(self.tmp_dir, '*'), shuffle=True, seed=37) + + full_filenames = [compat.as_bytes(path.join(self.tmp_dir, filename)) + for filename in filenames] + + all_produced_filenames = [] + for _ in range(3): + produced_filenames = [] + next_element = self.getNext(dataset, requires_initialization=True) + try: + while True: + produced_filenames.append(self.evaluate(next_element())) + except errors.OutOfRangeError: + pass + all_produced_filenames.append(produced_filenames) + + # Each run should produce the same set of filenames, which may be + # different from the order of `full_filenames`. + self.assertItemsEqual(full_filenames, all_produced_filenames[0]) + # However, the different runs should produce filenames in the same order + # as each other. + self.assertEqual(all_produced_filenames[0], all_produced_filenames[1]) + self.assertEqual(all_produced_filenames[0], all_produced_filenames[2]) + + # TODO(b/117581999): eager mode assertion fail wrapped, debug. + def tesSkipEagerEmptyDirectoryInitializer(self): + dataset = dataset_ops.Dataset.list_files(path.join(self.tmp_dir, '*')) + self.assertDatasetProduces( + dataset, + expected_error=(errors.InvalidArgumentError, + 'No files matched pattern'), + requires_initialization=True) + + def testSimpleDirectoryInitializer(self): + filenames = ['a', 'b', 'c'] + self._touchTempFiles(filenames) + + dataset = dataset_ops.Dataset.list_files(path.join(self.tmp_dir, '*')) + self.assertDatasetProduces( + dataset, + expected_output=[ + compat.as_bytes(path.join(self.tmp_dir, filename)) + for filename in filenames + ], + assert_items_equal=True) + + def testFileSuffixes(self): + filenames = ['a.txt', 'b.py', 'c.py', 'd.pyc'] + self._touchTempFiles(filenames) + + dataset = dataset_ops.Dataset.list_files(path.join(self.tmp_dir, '*.py')) + self.assertDatasetProduces( + dataset, + expected_output=[ + compat.as_bytes(path.join(self.tmp_dir, filename)) + for filename in filenames[1:-1] + ], + assert_items_equal=True) + + def testFileMiddles(self): + filenames = ['a.txt', 'b.py', 'c.pyc'] + self._touchTempFiles(filenames) + + dataset = dataset_ops.Dataset.list_files(path.join(self.tmp_dir, '*.py*')) + self.assertDatasetProduces( + dataset, + expected_output=[ + compat.as_bytes(path.join(self.tmp_dir, filename)) + for filename in filenames[1:] + ], + assert_items_equal=True) + + def testNoShuffle(self): + filenames = ['a', 'b', 'c'] + self._touchTempFiles(filenames) + + # Repeat the list twice and ensure that the order is the same each time. + # NOTE(mrry): This depends on an implementation detail of `list_files()`, + # which is that the list of files is captured when the iterator is + # initialized. Otherwise, or if e.g. the iterator were initialized more than + # once, it's possible that the non-determinism of `tf.matching_files()` + # would cause this test to fail. However, it serves as a useful confirmation + # that the `shuffle=False` argument is working as intended. + # TODO(b/73959787): Provide some ordering guarantees so that this test is + # more meaningful. + dataset = dataset_ops.Dataset.list_files( + path.join(self.tmp_dir, '*'), shuffle=False).repeat(2) + next_element = self.getNext(dataset) + + full_filenames = [] + produced_filenames = [] + for filename in filenames * 2: + full_filenames.append(compat.as_bytes(path.join(self.tmp_dir, filename))) + produced_filenames.append(compat.as_bytes(self.evaluate(next_element()))) + with self.assertRaises(errors.OutOfRangeError): + self.evaluate(next_element()) + self.assertItemsEqual(full_filenames, produced_filenames) + self.assertEqual(produced_filenames[:len(filenames)], + produced_filenames[len(filenames):]) + + def testMultiplePatternsAsList(self): + filenames = ['a.txt', 'b.py', 'c.py', 'd.pyc'] + self._touchTempFiles(filenames) + + patterns = [path.join(self.tmp_dir, pat) for pat in ['*.py', '*.txt']] + dataset = dataset_ops.Dataset.list_files(patterns) + self.assertDatasetProduces( + dataset, + expected_output=[ + compat.as_bytes(path.join(self.tmp_dir, filename)) + for filename in filenames[:-1] + ], + assert_items_equal=True) + + def testMultiplePatternsAsTensor(self): + filenames = ['a.txt', 'b.py', 'c.py', 'd.pyc'] + self._touchTempFiles(filenames) + + dataset = dataset_ops.Dataset.list_files( + [path.join(self.tmp_dir, pat) for pat in ['*.py', '*.txt']]) + self.assertDatasetProduces( + dataset, + expected_output=[ + compat.as_bytes(path.join(self.tmp_dir, filename)) + for filename in filenames[:-1] + ], + assert_items_equal=True) + + + +if __name__ == '__main__': + test.main() diff --git a/tensorflow/python/data/kernel_tests/map_dataset_op_test.py b/tensorflow/python/data/kernel_tests/map_test.py similarity index 80% rename from tensorflow/python/data/kernel_tests/map_dataset_op_test.py rename to tensorflow/python/data/kernel_tests/map_test.py index 81ef7d16be2..fdce7914471 100644 --- a/tensorflow/python/data/kernel_tests/map_dataset_op_test.py +++ b/tensorflow/python/data/kernel_tests/map_test.py @@ -19,7 +19,6 @@ from __future__ import print_function from collections import namedtuple import threading -import time import warnings from absl.testing import parameterized @@ -27,7 +26,6 @@ import numpy as np from tensorflow.core.framework import attr_value_pb2 from tensorflow.core.protobuf import config_pb2 -from tensorflow.python.client import session from tensorflow.python.data.kernel_tests import test_base from tensorflow.python.data.ops import dataset_ops from tensorflow.python.framework import constant_op @@ -79,7 +77,7 @@ def _make_coordinated_sloppy_dataset(num_elements, num_parallel_calls): options.experimental_deterministic = False dataset = dataset_ops.Dataset.range(num_elements).map( map_fn, num_parallel_calls).with_options(options) - iterator = dataset.make_one_shot_iterator() + iterator = dataset_ops.make_one_shot_iterator(dataset) next_element = iterator.get_next() return next_element, coordination_events @@ -102,7 +100,7 @@ class MapDatasetTest(test_base.DatasetTestBase, parameterized.TestCase): count = array_ops.placeholder(dtypes.int64, shape=[]) dataset = self._buildMapDataset(components, count) - iterator = dataset.make_initializable_iterator() + iterator = dataset_ops.make_initializable_iterator(dataset) init_op = iterator.initializer get_next = iterator.get_next() @@ -168,7 +166,7 @@ class MapDatasetTest(test_base.DatasetTestBase, parameterized.TestCase): dataset = self._buildParallelMapDataset( components, count, num_parallel_calls, output_buffer_size) - iterator = dataset.make_initializable_iterator() + iterator = dataset_ops.make_initializable_iterator(dataset) init_op = iterator.initializer get_next = iterator.get_next() @@ -237,7 +235,7 @@ class MapDatasetTest(test_base.DatasetTestBase, parameterized.TestCase): dataset = self._buildParallelMapDataset(components, 1000, 100, 100) # NOTE(mrry): Also test that the prefetching thread is cancelled correctly. dataset = dataset.prefetch(100) - iterator = dataset.make_initializable_iterator() + iterator = dataset_ops.make_initializable_iterator(dataset) init_op = iterator.initializer get_next = iterator.get_next() @@ -252,7 +250,7 @@ class MapDatasetTest(test_base.DatasetTestBase, parameterized.TestCase): dataset = (dataset_ops.Dataset.from_tensor_slices(components) .map(lambda x: array_ops.check_numerics(x, "message"), num_parallel_calls=2)) - iterator = dataset.make_initializable_iterator() + iterator = dataset_ops.make_initializable_iterator(dataset) init_op = iterator.initializer get_next = iterator.get_next() @@ -267,7 +265,7 @@ class MapDatasetTest(test_base.DatasetTestBase, parameterized.TestCase): dataset = (dataset_ops.Dataset.from_tensor_slices(components) .map(lambda x: array_ops.check_numerics(x, "message"), num_parallel_calls=2)) - iterator = dataset.make_initializable_iterator() + iterator = dataset_ops.make_initializable_iterator(dataset) init_op = iterator.initializer get_next = iterator.get_next() @@ -288,7 +286,7 @@ class MapDatasetTest(test_base.DatasetTestBase, parameterized.TestCase): dataset = (dataset_ops.Dataset.from_tensor_slices(components) .map(lambda x: array_ops.check_numerics(x, "message")) .prefetch(2)) - iterator = dataset.make_initializable_iterator() + iterator = dataset_ops.make_initializable_iterator(dataset) init_op = iterator.initializer get_next = iterator.get_next() @@ -314,8 +312,8 @@ class MapDatasetTest(test_base.DatasetTestBase, parameterized.TestCase): return dataset_ops.Dataset.range(10).map(_map_fn) def _build_graph(): - captured_iterator = dataset_ops.Dataset.range( - 10).make_initializable_iterator() + captured_iterator = dataset_ops.make_initializable_iterator( + dataset_ops.Dataset.range(10)) ds = _build_ds(captured_iterator) iterator = ds.make_initializable_iterator() init_op = iterator.initializer @@ -345,10 +343,9 @@ class MapDatasetTest(test_base.DatasetTestBase, parameterized.TestCase): input_sentences = dataset_ops.Dataset.from_tensor_slices( ["brain brain tank salad surgery", "surgery brain"]) - iterator = (input_sentences - .map(lambda x: string_ops.string_split([x]).values) - .map(table.lookup) - .make_initializable_iterator()) + iterator = dataset_ops.make_initializable_iterator( + input_sentences + .map(lambda x: string_ops.string_split([x]).values).map(table.lookup)) init_op = iterator.initializer get_next = iterator.get_next() @@ -365,8 +362,9 @@ class MapDatasetTest(test_base.DatasetTestBase, parameterized.TestCase): queue = data_flow_ops.FIFOQueue(200, dtypes.int64, shapes=[]) enqueue_op = queue.enqueue_many(elements) close_op = queue.close() - iterator = (dataset_ops.Dataset.from_tensors(0).repeat(-1) - .map(lambda _: queue.dequeue()).make_initializable_iterator()) + iterator = dataset_ops.make_initializable_iterator( + dataset_ops.Dataset.from_tensors(0).repeat(-1) + .map(lambda _: queue.dequeue())) init_op = iterator.initializer get_next = iterator.get_next() @@ -389,9 +387,9 @@ class MapDatasetTest(test_base.DatasetTestBase, parameterized.TestCase): enqueue_op = queue.enqueue_many(elements) close_op = queue.close() - iterator = (dataset_ops.Dataset.from_tensors(0).repeat(-1) - .map(lambda _: (queue.dequeue(), queue_2.dequeue())) - .make_initializable_iterator()) + iterator = dataset_ops.make_initializable_iterator( + dataset_ops.Dataset.from_tensors(0).repeat(-1) + .map(lambda _: (queue.dequeue(), queue_2.dequeue()))) init_op = iterator.initializer get_next = iterator.get_next() @@ -408,9 +406,9 @@ class MapDatasetTest(test_base.DatasetTestBase, parameterized.TestCase): def testCaptureVariable(self): counter_var = variable_scope.get_variable( "counter", (), dtypes.int32, use_resource=True) - iterator = (dataset_ops.Dataset.from_tensors(0).repeat(10) - .map(lambda _: counter_var.assign_add(1)) - .make_initializable_iterator()) + iterator = dataset_ops.make_initializable_iterator( + dataset_ops.Dataset.from_tensors(0).repeat(10) + .map(lambda _: counter_var.assign_add(1))) init_op = iterator.initializer get_next = iterator.get_next() @@ -428,9 +426,9 @@ class MapDatasetTest(test_base.DatasetTestBase, parameterized.TestCase): def testCaptureUninitializedVariableError(self): counter_var = variable_scope.get_variable( "counter", (), dtypes.int32, use_resource=True) - iterator = (dataset_ops.Dataset.from_tensors(0).repeat(10) - .map(lambda _: counter_var.assign_add(1)) - .make_initializable_iterator()) + iterator = dataset_ops.make_initializable_iterator( + dataset_ops.Dataset.from_tensors(0).repeat(10) + .map(lambda _: counter_var.assign_add(1))) init_op = iterator.initializer get_next = iterator.get_next() @@ -440,9 +438,9 @@ class MapDatasetTest(test_base.DatasetTestBase, parameterized.TestCase): sess.run(get_next) def testSeededStatefulOperatorIsProperlyStateful(self): - iterator = (dataset_ops.Dataset.from_tensors(0).repeat(10) - .map(lambda _: random_ops.random_uniform((), seed=11)).batch(2) - .make_initializable_iterator()) + iterator = dataset_ops.make_initializable_iterator( + dataset_ops.Dataset.from_tensors(0).repeat(10) + .map(lambda _: random_ops.random_uniform((), seed=11)).batch(2)) init_op = iterator.initializer get_next = iterator.get_next() @@ -464,11 +462,11 @@ class MapDatasetTest(test_base.DatasetTestBase, parameterized.TestCase): self.assertAllClose(random_values, random_values_2) def testStatefulMapKeepsStateAcrossIterators(self): - iterator = (dataset_ops.Dataset.from_tensors(0).repeat(10) - .map(lambda _: random_ops.random_uniform((), seed=11)) - .repeat(1000) - .batch(10) - .make_initializable_iterator()) + iterator = dataset_ops.make_initializable_iterator( + dataset_ops.Dataset.from_tensors(0).repeat(10) + .map(lambda _: random_ops.random_uniform((), seed=11)) + .repeat(1000) + .batch(10)) init_op = iterator.initializer get_next = iterator.get_next() @@ -493,9 +491,8 @@ class MapDatasetTest(test_base.DatasetTestBase, parameterized.TestCase): counter_var.assign_add(1) return x - iterator = (dataset_ops.Dataset.range(10) - .map(increment_fn) - .make_initializable_iterator()) + iterator = dataset_ops.make_initializable_iterator( + dataset_ops.Dataset.range(10).map(increment_fn)) init_op = iterator.initializer get_next = iterator.get_next() @@ -511,17 +508,17 @@ class MapDatasetTest(test_base.DatasetTestBase, parameterized.TestCase): self.assertEqual(10, sess.run(counter_var)) def testMapDict(self): - iterator = (dataset_ops.Dataset.range(10) - .map(lambda x: {"foo": x * 2, "bar": x ** 2}) - .map(lambda d: d["foo"] + d["bar"]) - .make_initializable_iterator()) + iterator = dataset_ops.make_initializable_iterator( + dataset_ops.Dataset.range(10) + .map(lambda x: {"foo": x * 2, "bar": x ** 2}) + .map(lambda d: d["foo"] + d["bar"])) init_op = iterator.initializer get_next = iterator.get_next() with self.cached_session() as sess: sess.run(init_op) for i in range(10): - self.assertEqual(i * 2 + i ** 2, sess.run(get_next)) + self.assertEqual(i * 2 + i**2, sess.run(get_next)) with self.assertRaises(errors.OutOfRangeError): sess.run(get_next) @@ -546,8 +543,9 @@ class MapDatasetTest(test_base.DatasetTestBase, parameterized.TestCase): dataset_tuple = dataset_tuple.map(preprocess_tuple) dataset_namedtuple = dataset_namedtuple.map(preprocess_namedtuple) - next_tuple = dataset_tuple.make_one_shot_iterator().get_next() - next_namedtuple = dataset_namedtuple.make_one_shot_iterator().get_next() + next_tuple = dataset_ops.make_one_shot_iterator(dataset_tuple).get_next() + next_namedtuple = dataset_ops.make_one_shot_iterator( + dataset_namedtuple).get_next() # make sure both datasets contain the same data with self.cached_session() as sess: @@ -561,16 +559,15 @@ class MapDatasetTest(test_base.DatasetTestBase, parameterized.TestCase): def testUseStepContainerInMap(self): row = np.arange(6) - iterator = ( + iterator = dataset_ops.make_initializable_iterator( dataset_ops.Dataset.from_tensors(row) - .map(lambda elems: functional_ops.map_fn(lambda x: x * x, elems)) - .make_initializable_iterator()) + .map(lambda elems: functional_ops.map_fn(lambda x: x * x, elems))) init_op = iterator.initializer get_next = iterator.get_next() with self.cached_session() as sess: sess.run(init_op) - self.assertAllEqual(row ** 2, sess.run(get_next)) + self.assertAllEqual(row**2, sess.run(get_next)) with self.assertRaises(errors.OutOfRangeError): sess.run(get_next) @@ -600,9 +597,9 @@ class MapDatasetTest(test_base.DatasetTestBase, parameterized.TestCase): pred_fn_pairs, default=multiply, exclusive=True) def build_dataset(row, num): - iterator = ( + iterator = dataset_ops.make_initializable_iterator( dataset_ops.Dataset.from_tensor_slices(row).map( - lambda x: control_map_fn(x, num)).make_initializable_iterator()) + lambda x: control_map_fn(x, num))) init_op = iterator.initializer get_next = iterator.get_next() return init_op, get_next @@ -639,11 +636,10 @@ class MapDatasetTest(test_base.DatasetTestBase, parameterized.TestCase): def build_dataset(row, num): # pylint: disable=g-long-lambda - iterator = ( + iterator = dataset_ops.make_initializable_iterator( dataset_ops.Dataset.from_tensors(row).map( - lambda elems: functional_ops.map_fn(lambda x: - control_map_fn(x, num), elems) - ).make_initializable_iterator()) + lambda elems: functional_ops.map_fn( + lambda x: control_map_fn(x, num), elems))) init_op = iterator.initializer get_next = iterator.get_next() return init_op, get_next @@ -687,11 +683,10 @@ class MapDatasetTest(test_base.DatasetTestBase, parameterized.TestCase): row = np.arange(6) num = 2 # pylint: disable=g-long-lambda - iterator = ( + iterator = dataset_ops.make_initializable_iterator( dataset_ops.Dataset.from_tensors(row).map( - lambda elems: functional_ops.map_fn(lambda x: - control_map_fn(x, num), elems) - ).make_initializable_iterator()) + lambda elems: functional_ops.map_fn( + lambda x: control_map_fn(x, num), elems))) # pylint: enable=g-long-lambda init_op = iterator.initializer get_next = iterator.get_next() @@ -721,11 +716,10 @@ class MapDatasetTest(test_base.DatasetTestBase, parameterized.TestCase): return script_ops.py_func(_map_py_func, [x], x.dtype) buffer_size_placeholder = array_ops.placeholder(dtypes.int64, shape=[]) - iterator = ( + iterator = dataset_ops.make_initializable_iterator( dataset_ops.Dataset.range(100) .map(_map_fn) - .prefetch(buffer_size_placeholder) - .make_initializable_iterator()) + .prefetch(buffer_size_placeholder)) init_op = iterator.initializer get_next = iterator.get_next() @@ -761,9 +755,9 @@ class MapDatasetTest(test_base.DatasetTestBase, parameterized.TestCase): sess.run(get_next) def testReturnList(self): - iterator = (dataset_ops.Dataset.range(10) - .map(lambda x: [x, constant_op.constant(37.0)]) - .make_initializable_iterator()) + iterator = dataset_ops.make_initializable_iterator( + dataset_ops.Dataset.range(10) + .map(lambda x: [x, constant_op.constant(37.0)])) init_op = iterator.initializer get_next = iterator.get_next() @@ -782,9 +776,8 @@ class MapDatasetTest(test_base.DatasetTestBase, parameterized.TestCase): return script_ops.py_func( _map_py_func, [x_tensor], [dtypes.int64, dtypes.float64]) - iterator = (dataset_ops.Dataset.range(10) - .map(_map_fn) - .make_initializable_iterator()) + iterator = dataset_ops.make_initializable_iterator( + dataset_ops.Dataset.range(10).map(_map_fn)) init_op = iterator.initializer get_next = iterator.get_next() @@ -803,9 +796,8 @@ class MapDatasetTest(test_base.DatasetTestBase, parameterized.TestCase): values=(i * np.array([1])), dense_shape=np.array([1, 1])) - iterator = (dataset_ops.Dataset.range(10) - .map(_sparse) - .make_initializable_iterator()) + iterator = dataset_ops.make_initializable_iterator( + dataset_ops.Dataset.range(10).map(_sparse)) init_op = iterator.initializer get_next = iterator.get_next() @@ -830,9 +822,8 @@ class MapDatasetTest(test_base.DatasetTestBase, parameterized.TestCase): self.assertTrue(sparse_tensor.is_sparse(i)) return sparse_ops.sparse_concat(0, [i, i]) - iterator = ( - dataset_ops.Dataset.range(10).map(_sparse).map(_check) - .make_initializable_iterator()) + iterator = dataset_ops.make_initializable_iterator( + dataset_ops.Dataset.range(10).map(_sparse).map(_check)) init_op = iterator.initializer get_next = iterator.get_next() @@ -852,11 +843,10 @@ class MapDatasetTest(test_base.DatasetTestBase, parameterized.TestCase): else: return i - iterator = ( + iterator = dataset_ops.make_initializable_iterator( dataset_ops.Dataset.range(105) .map(lambda x: script_ops.py_func(raising_py_func, [x], dtypes.int64), - num_parallel_calls=2) - .make_initializable_iterator()) + num_parallel_calls=2)) init_op = iterator.initializer get_next = iterator.get_next() @@ -868,9 +858,8 @@ class MapDatasetTest(test_base.DatasetTestBase, parameterized.TestCase): sess.run(get_next) def testConstantOutput(self): - iterator = ( - dataset_ops.Dataset.range(10).map(lambda x: [x, "hello", 10]) - .make_initializable_iterator()) + iterator = dataset_ops.make_initializable_iterator( + dataset_ops.Dataset.range(10).map(lambda x: [x, "hello", 10])) init_op = iterator.initializer get_next = iterator.get_next() @@ -901,12 +890,14 @@ class MapDatasetTest(test_base.DatasetTestBase, parameterized.TestCase): break self.assertTrue(found_warning) - def testNestedDatasetError(self): - dataset = dataset_ops.Dataset.from_tensors([1.0, 2.0, 3.0]) - with self.assertRaisesRegexp( - NotImplementedError, r"The Dataset.map\(\) transformation does not " - "currently support nested datasets as outputs."): - _ = dataset.map(dataset_ops.Dataset.from_tensor_slices) + def testNestedDatasetMap(self): + # TODO(b/110122868): When iterators can yield a `tf.data.Dataset`, remove + # the `get_single_element()` call. + dataset = dataset_ops.Dataset.from_tensors([1.0, 2.0, 3.0]).map( + dataset_ops.Dataset.from_tensor_slices).map( + lambda ds: ds.batch(3)).flat_map(lambda x: x) + + self.assertDatasetProduces(dataset, [[1.0, 2.0, 3.0]]) def testReturnValueError(self): dataset = dataset_ops.Dataset.from_tensors([1.0, 2.0, 3.0]) @@ -939,7 +930,7 @@ class MapDatasetTest(test_base.DatasetTestBase, parameterized.TestCase): return const_tensor dataset = dataset.map(broken_function) - iterator = dataset.make_initializable_iterator() + iterator = dataset_ops.make_initializable_iterator(dataset) with self.cached_session() as sess: with self.assertRaisesRegexp(errors.InvalidArgumentError, "BrokenConst"): @@ -966,7 +957,7 @@ class MapDatasetTest(test_base.DatasetTestBase, parameterized.TestCase): return tids dataset = make_dataset_fn(dataset, _map_fn) - iterator = dataset.make_one_shot_iterator() + iterator = dataset_ops.make_one_shot_iterator(dataset) get_next = iterator.get_next() with self.cached_session() as sess: @@ -987,7 +978,7 @@ class MapDatasetTest(test_base.DatasetTestBase, parameterized.TestCase): def testShortCircuit(self, structure, map_fn, num_parallel_calls): dataset = self.structuredDataset(structure).repeat().map( map_fn, num_parallel_calls=num_parallel_calls) - get_next = dataset.make_one_shot_iterator().get_next() + get_next = dataset_ops.make_one_shot_iterator(dataset).get_next() with self.cached_session() as sess: if isinstance(structure, tuple): @@ -1004,7 +995,7 @@ class MapDatasetTest(test_base.DatasetTestBase, parameterized.TestCase): captured_t = array_ops.placeholder(dtypes.int64, shape=[]) dataset = self.structuredDataset(None).repeat().map( lambda x: captured_t, num_parallel_calls=num_parallel_calls) - iterator = dataset.make_initializable_iterator() + iterator = dataset_ops.make_initializable_iterator(dataset) get_next = iterator.get_next() with self.cached_session() as sess: @@ -1055,108 +1046,5 @@ class MapDatasetTest(test_base.DatasetTestBase, parameterized.TestCase): sess.run(get_next) -class MapDatasetBenchmark(test.Benchmark): - - def benchmarkChainOfMaps(self): - chain_lengths = [0, 1, 2, 5, 10, 20, 50] - for chain_length in chain_lengths: - for mode in ["general", "single-threaded", "short-circuit"]: - if mode == "general": - map_fn = lambda x: x + 1 - use_inter_op_parallelism = True - print_label = "" - benchmark_label = "" - if mode == "single-threaded": - map_fn = lambda x: x + 1 - use_inter_op_parallelism = False - print_label = " (single threaded mode)" - benchmark_label = "_single_threaded" - if mode == "short-circuit": - map_fn = lambda x: x - use_inter_op_parallelism = True # should not have any significance - print_label = " (short circuit mode)" - benchmark_label = "_short_circuit" - - with ops.Graph().as_default(): - dataset = dataset_ops.Dataset.from_tensors(0).repeat(None) - for _ in range(chain_length): - dataset = dataset_ops.MapDataset( - dataset, - map_fn, - use_inter_op_parallelism=use_inter_op_parallelism) - iterator = dataset.make_one_shot_iterator() - next_element = iterator.get_next() - - with session.Session() as sess: - for _ in range(5): - sess.run(next_element.op) - deltas = [] - for _ in range(100): - start = time.time() - for _ in range(100): - sess.run(next_element.op) - end = time.time() - deltas.append(end - start) - - median_wall_time = np.median(deltas) / 100 - print("Map dataset chain length%s: %d Median wall time: %f" % - (print_label, chain_length, median_wall_time)) - self.report_benchmark( - iters=1000, - wall_time=median_wall_time, - name="benchmark_map_dataset_chain_latency_%d%s" % - (chain_length, benchmark_label)) - - def benchmarkMapFanOut(self): - fan_outs = [1, 2, 5, 10, 20, 50, 100] - for fan_out in fan_outs: - for mode in ["general", "single-threaded", "short-circuit"]: - if mode == "general": - map_fn = lambda *xs: [x + 1 for x in xs] - use_inter_op_parallelism = True - print_label = "" - benchmark_label = "" - if mode == "single-threaded": - map_fn = lambda *xs: [x + 1 for x in xs] - use_inter_op_parallelism = False - print_label = " (single threaded mode)" - benchmark_label = "_single_threaded" - if mode == "short-circuit": - map_fn = lambda *xs: xs - use_inter_op_parallelism = True # should not have any significance - print_label = " (short circuit mode)" - benchmark_label = "_short_circuit" - - with ops.Graph().as_default(): - dataset = dataset_ops.Dataset.from_tensors( - tuple(0 for _ in range(fan_out))).repeat(None) - dataset = dataset_ops.MapDataset( - dataset, - map_fn, - use_inter_op_parallelism=use_inter_op_parallelism) - iterator = dataset.make_one_shot_iterator() - next_element = iterator.get_next() - - with session.Session() as sess: - for _ in range(5): - sess.run(next_element[0].op) - deltas = [] - for _ in range(100): - start = time.time() - for _ in range(100): - sess.run(next_element[0].op) - end = time.time() - deltas.append(end - start) - - median_wall_time = np.median(deltas) / 100 - print("Map dataset fan out%s: %d Median wall time: %f" % - (print_label, fan_out, median_wall_time)) - self.report_benchmark( - iters=1000, - wall_time=median_wall_time, - name="benchmark_map_dataset_fan_out_%d%s" % (fan_out, - benchmark_label)) - - if __name__ == "__main__": test.main() diff --git a/tensorflow/python/data/kernel_tests/multi_device_iterator_test.py b/tensorflow/python/data/kernel_tests/multi_device_iterator_test.py index 42ee1e21864..622ebb55dec 100644 --- a/tensorflow/python/data/kernel_tests/multi_device_iterator_test.py +++ b/tensorflow/python/data/kernel_tests/multi_device_iterator_test.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================== -"""MultiDeviceIterator tests.""" +"""Tests for `tf.data.MultiDeviceIterator`.""" from __future__ import absolute_import from __future__ import division @@ -20,6 +20,7 @@ from __future__ import print_function from tensorflow.core.protobuf import config_pb2 from tensorflow.python.data.experimental.ops import optimization +from tensorflow.python.data.experimental.ops.optimization_options import OptimizationOptions from tensorflow.python.data.kernel_tests import test_base from tensorflow.python.data.ops import dataset_ops from tensorflow.python.data.ops import multi_device_iterator_ops @@ -31,6 +32,7 @@ from tensorflow.python.ops import array_ops from tensorflow.python.platform import test +# TODO(b/117581999): Add eager coverage. class MultiDeviceIteratorTest(test_base.DatasetTestBase): def testNoGetNext(self): @@ -40,7 +42,7 @@ class MultiDeviceIteratorTest(test_base.DatasetTestBase): config = config_pb2.ConfigProto(device_count={"CPU": 3}) with self.test_session(config=config) as sess: - sess.run(multi_device_iterator.initializer) + self.evaluate(multi_device_iterator.initializer) def testBasic(self): dataset = dataset_ops.Dataset.range(10) @@ -50,13 +52,13 @@ class MultiDeviceIteratorTest(test_base.DatasetTestBase): config = config_pb2.ConfigProto(device_count={"CPU": 3}) with self.test_session(config=config) as sess: - sess.run(multi_device_iterator.initializer) + self.evaluate(multi_device_iterator.initializer) for i in range(0, 10, 2): - self.assertEqual(i, sess.run(elem_on_1)) - self.assertEqual(i + 1, sess.run(elem_on_2)) + self.assertEqual(i, self.evaluate(elem_on_1)) + self.assertEqual(i + 1, self.evaluate(elem_on_2)) with self.assertRaises(errors.OutOfRangeError): - sess.run(elem_on_1) - sess.run(elem_on_2) + self.evaluate(elem_on_1) + self.evaluate(elem_on_2) def testOneOnSameDevice(self): with ops.device("/cpu:0"): @@ -67,13 +69,13 @@ class MultiDeviceIteratorTest(test_base.DatasetTestBase): config = config_pb2.ConfigProto(device_count={"CPU": 2}) with self.test_session(config=config) as sess: - sess.run(multi_device_iterator.initializer) + self.evaluate(multi_device_iterator.initializer) for i in range(0, 10, 2): - self.assertEqual(i, sess.run(elem_on_1)) - self.assertEqual(i + 1, sess.run(elem_on_2)) + self.assertEqual(i, self.evaluate(elem_on_1)) + self.assertEqual(i + 1, self.evaluate(elem_on_2)) with self.assertRaises(errors.OutOfRangeError): - sess.run(elem_on_1) - sess.run(elem_on_2) + self.evaluate(elem_on_1) + self.evaluate(elem_on_2) def testRepeatDevices(self): with ops.device("/cpu:0"): @@ -85,17 +87,17 @@ class MultiDeviceIteratorTest(test_base.DatasetTestBase): config = config_pb2.ConfigProto(device_count={"CPU": 3}) with self.test_session(config=config) as sess: - sess.run(multi_device_iterator.initializer) + self.evaluate(multi_device_iterator.initializer) for i in range(0, 20, 4): - self.assertEqual(i, sess.run(elem_on_1)) - self.assertEqual(i + 1, sess.run(elem_on_2)) - self.assertEqual(i + 2, sess.run(elem_on_3)) - self.assertEqual(i + 3, sess.run(elem_on_4)) + self.assertEqual(i, self.evaluate(elem_on_1)) + self.assertEqual(i + 1, self.evaluate(elem_on_2)) + self.assertEqual(i + 2, self.evaluate(elem_on_3)) + self.assertEqual(i + 3, self.evaluate(elem_on_4)) with self.assertRaises(errors.OutOfRangeError): - sess.run(elem_on_1) - sess.run(elem_on_2) - sess.run(elem_on_3) - sess.run(elem_on_4) + self.evaluate(elem_on_1) + self.evaluate(elem_on_2) + self.evaluate(elem_on_3) + self.evaluate(elem_on_4) def testNotFullyDivisible(self): dataset = dataset_ops.Dataset.range(9) @@ -105,14 +107,14 @@ class MultiDeviceIteratorTest(test_base.DatasetTestBase): config = config_pb2.ConfigProto(device_count={"CPU": 3}) with self.test_session(config=config) as sess: - sess.run(multi_device_iterator.initializer) + self.evaluate(multi_device_iterator.initializer) for i in range(0, 8, 2): - self.assertEqual(i, sess.run(elem_on_1)) - self.assertEqual(i + 1, sess.run(elem_on_2)) - self.assertEqual(8, sess.run(elem_on_1)) + self.assertEqual(i, self.evaluate(elem_on_1)) + self.assertEqual(i + 1, self.evaluate(elem_on_2)) + self.assertEqual(8, self.evaluate(elem_on_1)) with self.assertRaises(errors.OutOfRangeError): - sess.run(elem_on_1) - sess.run(elem_on_2) + self.evaluate(elem_on_1) + self.evaluate(elem_on_2) def testGetNextAsOptional(self): dataset = dataset_ops.Dataset.range(9) @@ -126,7 +128,7 @@ class MultiDeviceIteratorTest(test_base.DatasetTestBase): config = config_pb2.ConfigProto(device_count={"CPU": 3}) with self.test_session(config=config) as sess: - sess.run(multi_device_iterator.initializer) + self.evaluate(multi_device_iterator.initializer) for i in range(0, 8, 2): elem_on_1_has_value, elem_on_1_value = sess.run( [elem_on_1_has_value_t, elem_on_1_t]) @@ -140,12 +142,12 @@ class MultiDeviceIteratorTest(test_base.DatasetTestBase): [elem_on_1_has_value_t, elem_on_1_t]) self.assertTrue(elem_on_1_has_value) self.assertEqual(8, elem_on_1_value) - self.assertFalse(sess.run(elem_on_1_has_value_t)) - self.assertFalse(sess.run(elem_on_2_has_value_t)) + self.assertFalse(self.evaluate(elem_on_1_has_value_t)) + self.assertFalse(self.evaluate(elem_on_2_has_value_t)) with self.assertRaises(errors.InvalidArgumentError): - sess.run(elem_on_1_t) + self.evaluate(elem_on_1_t) with self.assertRaises(errors.InvalidArgumentError): - sess.run(elem_on_2_t) + self.evaluate(elem_on_2_t) def testUneven(self): dataset = dataset_ops.Dataset.range(10) @@ -155,14 +157,14 @@ class MultiDeviceIteratorTest(test_base.DatasetTestBase): config = config_pb2.ConfigProto(device_count={"CPU": 3}) with self.test_session(config=config) as sess: - sess.run(multi_device_iterator.initializer) + self.evaluate(multi_device_iterator.initializer) for i in range(0, 10, 2): - self.assertEqual(i, sess.run(elem_on_1)) + self.assertEqual(i, self.evaluate(elem_on_1)) for i in range(0, 10, 2): - self.assertEqual(i + 1, sess.run(elem_on_2)) + self.assertEqual(i + 1, self.evaluate(elem_on_2)) with self.assertRaises(errors.OutOfRangeError): - sess.run(elem_on_1) - sess.run(elem_on_2) + self.evaluate(elem_on_1) + self.evaluate(elem_on_2) def testMultipleInitializations(self): with ops.device("/cpu:0"): @@ -179,7 +181,8 @@ class MultiDeviceIteratorTest(test_base.DatasetTestBase): with self.test_session(config=config) as sess: for i in range(1000): sess.run(init_op, feed_dict={epoch: i}) - self.assertEqual([(i, 0), (i, 1)], sess.run([elem_on_1, elem_on_2])) + self.assertEqual([(i, 0), (i, 1)], self.evaluate([elem_on_1, + elem_on_2])) def testBasicGpu(self): if not test_util.is_gpu_available(): @@ -192,13 +195,13 @@ class MultiDeviceIteratorTest(test_base.DatasetTestBase): config = config_pb2.ConfigProto(device_count={"CPU": 2, "GPU": 1}) with self.test_session(config=config) as sess: - sess.run(multi_device_iterator.initializer) + self.evaluate(multi_device_iterator.initializer) for i in range(0, 10, 2): - self.assertEqual(i, sess.run(elem_on_1)) - self.assertEqual(i + 1, sess.run(elem_on_2)) + self.assertEqual(i, self.evaluate(elem_on_1)) + self.assertEqual(i + 1, self.evaluate(elem_on_2)) with self.assertRaises(errors.OutOfRangeError): - sess.run(elem_on_1) - sess.run(elem_on_2) + self.evaluate(elem_on_1) + self.evaluate(elem_on_2) def testUnevenGpu(self): if not test_util.is_gpu_available(): @@ -211,14 +214,14 @@ class MultiDeviceIteratorTest(test_base.DatasetTestBase): config = config_pb2.ConfigProto(device_count={"CPU": 2, "GPU": 1}) with self.test_session(config=config) as sess: - sess.run(multi_device_iterator.initializer) + self.evaluate(multi_device_iterator.initializer) for i in range(0, 10, 2): - self.assertEqual(i, sess.run(elem_on_1)) + self.assertEqual(i, self.evaluate(elem_on_1)) for i in range(0, 10, 2): - self.assertEqual(i + 1, sess.run(elem_on_2)) + self.assertEqual(i + 1, self.evaluate(elem_on_2)) with self.assertRaises(errors.OutOfRangeError): - sess.run(elem_on_1) - sess.run(elem_on_2) + self.evaluate(elem_on_1) + self.evaluate(elem_on_2) def testGetNextAsOptionalGpu(self): if not test_util.is_gpu_available(): @@ -235,7 +238,7 @@ class MultiDeviceIteratorTest(test_base.DatasetTestBase): config = config_pb2.ConfigProto(device_count={"CPU": 2, "GPU": 1}) with self.test_session(config=config) as sess: - sess.run(multi_device_iterator.initializer) + self.evaluate(multi_device_iterator.initializer) for i in range(0, 8, 2): elem_on_1_has_value, elem_on_1_value = sess.run( [elem_on_1_has_value_t, elem_on_1_t]) @@ -249,12 +252,12 @@ class MultiDeviceIteratorTest(test_base.DatasetTestBase): [elem_on_1_has_value_t, elem_on_1_t]) self.assertTrue(elem_on_1_has_value) self.assertEqual(8, elem_on_1_value) - self.assertFalse(sess.run(elem_on_1_has_value_t)) - self.assertFalse(sess.run(elem_on_2_has_value_t)) + self.assertFalse(self.evaluate(elem_on_1_has_value_t)) + self.assertFalse(self.evaluate(elem_on_2_has_value_t)) with self.assertRaises(errors.InvalidArgumentError): - sess.run(elem_on_1_t) + self.evaluate(elem_on_1_t) with self.assertRaises(errors.InvalidArgumentError): - sess.run(elem_on_2_t) + self.evaluate(elem_on_2_t) def testOptimization(self): dataset = dataset_ops.Dataset.range(10) @@ -263,7 +266,8 @@ class MultiDeviceIteratorTest(test_base.DatasetTestBase): dataset = dataset.cache() options = dataset_ops.Options() - options.experimental_noop_elimination = True + options.experimental_optimization = OptimizationOptions() + options.experimental_optimization.noop_elimination = True dataset = dataset.with_options(options) multi_device_iterator = multi_device_iterator_ops.MultiDeviceIterator( @@ -272,13 +276,13 @@ class MultiDeviceIteratorTest(test_base.DatasetTestBase): config = config_pb2.ConfigProto(device_count={"CPU": 3}) with self.test_session(config=config) as sess: - sess.run(multi_device_iterator.initializer) + self.evaluate(multi_device_iterator.initializer) for i in range(0, 10, 2): - self.assertEqual(i, sess.run(elem_on_1)) - self.assertEqual(i + 1, sess.run(elem_on_2)) + self.assertEqual(i, self.evaluate(elem_on_1)) + self.assertEqual(i + 1, self.evaluate(elem_on_2)) with self.assertRaises(errors.OutOfRangeError): - sess.run(elem_on_1) - sess.run(elem_on_2) + self.evaluate(elem_on_1) + self.evaluate(elem_on_2) if __name__ == "__main__": diff --git a/tensorflow/python/data/kernel_tests/optional_ops_test.py b/tensorflow/python/data/kernel_tests/optional_test.py similarity index 64% rename from tensorflow/python/data/kernel_tests/optional_ops_test.py rename to tensorflow/python/data/kernel_tests/optional_test.py index 604e3ad88ec..c2c62e9423e 100644 --- a/tensorflow/python/data/kernel_tests/optional_ops_test.py +++ b/tensorflow/python/data/kernel_tests/optional_test.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================== -"""Tests for the Optional data type wrapper.""" +"""Tests for `tf.data.Optional`.""" from __future__ import absolute_import from __future__ import division from __future__ import print_function @@ -33,18 +33,18 @@ from tensorflow.python.framework import sparse_tensor from tensorflow.python.framework import tensor_shape from tensorflow.python.framework import test_util from tensorflow.python.ops import array_ops +from tensorflow.python.ops import math_ops from tensorflow.python.platform import test +@test_util.run_all_in_graph_and_eager_modes class OptionalTest(test_base.DatasetTestBase, parameterized.TestCase): - @test_util.run_in_graph_and_eager_modes def testFromValue(self): opt = optional_ops.Optional.from_value(constant_op.constant(37.0)) self.assertTrue(self.evaluate(opt.has_value())) self.assertEqual(37.0, self.evaluate(opt.get_value())) - @test_util.run_in_graph_and_eager_modes def testFromStructuredValue(self): opt = optional_ops.Optional.from_value({ "a": constant_op.constant(37.0), @@ -56,7 +56,6 @@ class OptionalTest(test_base.DatasetTestBase, parameterized.TestCase): "b": ([b"Foo"], b"Bar") }, self.evaluate(opt.get_value())) - @test_util.run_in_graph_and_eager_modes def testFromSparseTensor(self): st_0 = sparse_tensor.SparseTensorValue( indices=np.array([[0]]), @@ -75,7 +74,7 @@ class OptionalTest(test_base.DatasetTestBase, parameterized.TestCase): self.assertAllEqual(expected.dense_shape, self.evaluate(actual.dense_shape)) - @test_util.run_in_graph_and_eager_modes + @test_util.run_deprecated_v1 def testFromNone(self): value_structure = structure.TensorStructure(dtypes.float32, []) opt = optional_ops.Optional.none_from_structure(value_structure) @@ -90,7 +89,90 @@ class OptionalTest(test_base.DatasetTestBase, parameterized.TestCase): with self.assertRaises(errors.InvalidArgumentError): self.evaluate(opt.get_value()) - @test_util.run_in_graph_and_eager_modes + def testAddN(self): + devices = ["/cpu:0"] + if test_util.is_gpu_available(): + devices.append("/gpu:0") + for device in devices: + with ops.device(device): + # With value + opt1 = optional_ops.Optional.from_value((1.0, 2.0)) + opt2 = optional_ops.Optional.from_value((3.0, 4.0)) + + add_tensor = math_ops.add_n([opt1._variant_tensor, + opt2._variant_tensor]) + add_opt = optional_ops._OptionalImpl(add_tensor, opt1.value_structure) + self.assertAllEqual(self.evaluate(add_opt.get_value()), (4.0, 6.0)) + + # Without value + opt_none1 = optional_ops.Optional.none_from_structure( + opt1.value_structure) + opt_none2 = optional_ops.Optional.none_from_structure( + opt2.value_structure) + add_tensor = math_ops.add_n([opt_none1._variant_tensor, + opt_none2._variant_tensor]) + add_opt = optional_ops._OptionalImpl(add_tensor, + opt_none1.value_structure) + self.assertFalse(self.evaluate(add_opt.has_value())) + + def testNestedAddN(self): + devices = ["/cpu:0"] + if test_util.is_gpu_available(): + devices.append("/gpu:0") + for device in devices: + with ops.device(device): + opt1 = optional_ops.Optional.from_value([1, 2.0]) + opt2 = optional_ops.Optional.from_value([3, 4.0]) + opt3 = optional_ops.Optional.from_value((5.0, opt1._variant_tensor)) + opt4 = optional_ops.Optional.from_value((6.0, opt2._variant_tensor)) + + add_tensor = math_ops.add_n([opt3._variant_tensor, + opt4._variant_tensor]) + add_opt = optional_ops._OptionalImpl(add_tensor, opt3.value_structure) + self.assertEqual(self.evaluate(add_opt.get_value()[0]), 11.0) + + inner_add_opt = optional_ops._OptionalImpl(add_opt.get_value()[1], + opt1.value_structure) + self.assertAllEqual(inner_add_opt.get_value(), [4, 6.0]) + + def testZerosLike(self): + devices = ["/cpu:0"] + if test_util.is_gpu_available(): + devices.append("/gpu:0") + for device in devices: + with ops.device(device): + # With value + opt = optional_ops.Optional.from_value((1.0, 2.0)) + zeros_tensor = array_ops.zeros_like(opt._variant_tensor) + zeros_opt = optional_ops._OptionalImpl(zeros_tensor, + opt.value_structure) + self.assertAllEqual(self.evaluate(zeros_opt.get_value()), + (0.0, 0.0)) + + # Without value + opt_none = optional_ops.Optional.none_from_structure( + opt.value_structure) + zeros_tensor = array_ops.zeros_like(opt_none._variant_tensor) + zeros_opt = optional_ops._OptionalImpl(zeros_tensor, + opt_none.value_structure) + self.assertFalse(self.evaluate(zeros_opt.has_value())) + + def testNestedZerosLike(self): + devices = ["/cpu:0"] + if test_util.is_gpu_available(): + devices.append("/gpu:0") + for device in devices: + with ops.device(device): + opt1 = optional_ops.Optional.from_value(1.0) + opt2 = optional_ops.Optional.from_value(opt1._variant_tensor) + + zeros_tensor = array_ops.zeros_like(opt2._variant_tensor) + zeros_opt = optional_ops._OptionalImpl(zeros_tensor, + opt2.value_structure) + inner_zeros_opt = optional_ops._OptionalImpl(zeros_opt.get_value(), + opt1.value_structure) + self.assertEqual(self.evaluate(inner_zeros_opt.get_value()), 0.0) + def testCopyToGPU(self): if not test_util.is_gpu_available(): self.skipTest("No GPU available") @@ -120,6 +202,41 @@ class OptionalTest(test_base.DatasetTestBase, parameterized.TestCase): self.evaluate(gpu_optional_with_value_values)) self.assertFalse(self.evaluate(gpu_optional_none_has_value)) + def testNestedCopyToGPU(self): + if not test_util.is_gpu_available(): + self.skipTest("No GPU available") + + with ops.device("/cpu:0"): + optional_with_value = optional_ops.Optional.from_value( + (constant_op.constant(37.0), constant_op.constant("Foo"), + constant_op.constant(42))) + optional_none = optional_ops.Optional.none_from_structure( + structure.TensorStructure(dtypes.float32, [])) + nested_optional = optional_ops.Optional.from_value( + (optional_with_value._variant_tensor, optional_none._variant_tensor, + 1.0)) + + with ops.device("/gpu:0"): + gpu_nested_optional = optional_ops._OptionalImpl( + array_ops.identity(nested_optional._variant_tensor), + nested_optional.value_structure) + + gpu_nested_optional_has_value = gpu_nested_optional.has_value() + gpu_nested_optional_values = gpu_nested_optional.get_value() + + self.assertTrue(self.evaluate(gpu_nested_optional_has_value)) + + inner_with_value = optional_ops._OptionalImpl( + gpu_nested_optional_values[0], optional_with_value.value_structure) + + inner_none = optional_ops._OptionalImpl( + gpu_nested_optional_values[1], optional_none.value_structure) + + self.assertEqual((37.0, b"Foo", 42), + self.evaluate(inner_with_value.get_value())) + self.assertFalse(self.evaluate(inner_none.has_value())) + self.assertEqual(1.0, self.evaluate(gpu_nested_optional_values[2])) + def _assertElementValueEqual(self, expected, actual): if isinstance(expected, dict): self.assertItemsEqual(list(expected.keys()), list(actual.keys())) @@ -151,7 +268,9 @@ class OptionalTest(test_base.DatasetTestBase, parameterized.TestCase): optional_ops.OptionalStructure( structure.TensorStructure(dtypes.float32, []))), ) - def testOptionalStructure(self, tf_value_fn, expected_value_structure): + @test_util.run_deprecated_v1 + def testSkipEagerOptionalStructure(self, tf_value_fn, + expected_value_structure): tf_value = tf_value_fn() opt = optional_ops.Optional.from_value(tf_value) @@ -205,7 +324,9 @@ class OptionalTest(test_base.DatasetTestBase, parameterized.TestCase): indices=[[0, 1], [1, 0]], values=[37.0, 42.0], dense_shape=[2, 2])}, False), ) - def testIteratorGetNextAsOptional(self, np_value, tf_value_fn, works_on_gpu): + @test_util.run_deprecated_v1 + def testSkipEagerIteratorGetNextAsOptional(self, np_value, tf_value_fn, + works_on_gpu): if not works_on_gpu and test.is_gpu_available(): self.skipTest("Test case not yet supported on GPU.") ds = dataset_ops.Dataset.from_tensors(np_value).repeat(3) diff --git a/tensorflow/python/data/kernel_tests/padded_batch_test.py b/tensorflow/python/data/kernel_tests/padded_batch_test.py new file mode 100644 index 00000000000..dcfb2f507bf --- /dev/null +++ b/tensorflow/python/data/kernel_tests/padded_batch_test.py @@ -0,0 +1,243 @@ +# -*- coding: utf-8 -*- +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Tests for `tf.data.Dataset.padded_batch()`.""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from absl.testing import parameterized +import numpy as np + +from tensorflow.python.data.kernel_tests import test_base +from tensorflow.python.data.ops import dataset_ops +from tensorflow.python.framework import constant_op +from tensorflow.python.framework import dtypes +from tensorflow.python.framework import errors +from tensorflow.python.framework import sparse_tensor +from tensorflow.python.framework import tensor_shape +from tensorflow.python.framework import test_util +from tensorflow.python.ops import array_ops +from tensorflow.python.ops import string_ops +from tensorflow.python.platform import test +from tensorflow.python.util import compat + + +def _random_seq_lens(count): + return np.random.randint(20, size=(count,)).astype(np.int32) + + +@test_util.run_all_in_graph_and_eager_modes +class PaddedBatchTest(test_base.DatasetTestBase, parameterized.TestCase): + + @parameterized.named_parameters( + ('default_padding', _random_seq_lens(32), 4, [-1], False), + ('constant_padding', _random_seq_lens(32), 4, [25], False), + ('uneven_with_remainder', _random_seq_lens(34), 4, [-1], False), + ('uneven_without_remainder', _random_seq_lens(34), 4, [-1], True), + ) + def testPaddedBatchDataset(self, seq_lens, batch_size, padded_shapes, + drop_remainder): + """Tests the padded batch dataset logic for various input configurations. + + Args: + seq_lens: the input sequence lengths + batch_size: the batch size + padded_shapes: the padded shapes to use + drop_remainder: whether a smaller batch size should be produced if batch + size does not divide number of inputs evenly + """ + + dataset = dataset_ops.Dataset.from_tensor_slices(seq_lens).map( + lambda x: array_ops.fill([x], x)).padded_batch( + batch_size=batch_size, + drop_remainder=drop_remainder, + padded_shapes=padded_shapes) + + num_full_batches = len(seq_lens) // batch_size + get_next = self.getNext(dataset) + for i in range(num_full_batches): + result = self.evaluate(get_next()) + padded_len = padded_shapes[0] + if padded_len is None or padded_len == -1: + padded_len = np.max(result) if result.size > 0 else 0 + self.assertEqual((batch_size, padded_len), result.shape) + for j in range(batch_size): + seq_len = seq_lens[(i * batch_size) + j] + self.assertAllEqual(result[j, :seq_len], [seq_len] * seq_len) + self.assertAllEqual(result[j, seq_len:], [0] * (padded_len - seq_len)) + + if not drop_remainder and len(seq_lens) % batch_size > 0: + result = self.evaluate(get_next()) + padded_len = np.max(result) if result.size > 0 else 0 + self.assertEqual((len(seq_lens) % batch_size, padded_len), result.shape) + for j in range(len(seq_lens) % batch_size): + seq_len = seq_lens[num_full_batches * batch_size + j] + self.assertAllEqual(result[j, :seq_len], [seq_len] * seq_len) + self.assertAllEqual(result[j, seq_len:], [0] * (padded_len - seq_len)) + + with self.assertRaises(errors.OutOfRangeError): + self.evaluate(get_next()) + with self.assertRaises(errors.OutOfRangeError): + self.evaluate(get_next()) + + @test_util.run_deprecated_v1 + def testPaddedBatchShortPadding(self): + dataset = ( + dataset_ops.Dataset.from_tensor_slices( + [6, 5, 5, 5, 5]).map(lambda x: array_ops.fill([x], x)).padded_batch( + batch_size=4, padded_shapes=[5])) + self.assertDatasetProduces( + dataset, expected_error=(errors.DataLossError, '')) + + def testPaddedBatchEmptyTensors(self): + dataset = ( + dataset_ops.Dataset.from_tensor_slices( + [0, 0, 0, 0]).map(lambda x: array_ops.fill([x], x)).padded_batch( + batch_size=4, padded_shapes=[-1])) + self.assertDatasetProduces(dataset, expected_output=[[[], [], [], []]]) + + def testPaddedBatchDatasetNonDefaultPadding(self): + + def fill_tuple(x): + filled = array_ops.fill([x], x) + return (filled, string_ops.as_string(filled)) + + random_seq_lens = np.random.randint(20, size=(32,)).astype(np.int32) + dataset = ( + dataset_ops.Dataset.from_tensor_slices(random_seq_lens).map(fill_tuple) + .padded_batch( + 4, padded_shapes=([-1], [-1]), padding_values=(-1, ''))) + + get_next = self.getNext(dataset) + for i in range(8): + result = self.evaluate(get_next()) + padded_len = np.max(result[0]) + self.assertEqual((4, padded_len), result[0].shape) + self.assertEqual((4, padded_len), result[1].shape) + for j in range(4): + seq_len = random_seq_lens[(i * 4) + j] + self.assertAllEqual(result[0][j, :seq_len], [seq_len] * seq_len) + self.assertAllEqual(result[0][j, seq_len:], + [-1] * (padded_len - seq_len)) + self.assertAllEqual(result[1][j, :seq_len], + [compat.as_bytes(str(seq_len))] * seq_len) + self.assertAllEqual(result[1][j, seq_len:], + [b''] * (padded_len - seq_len)) + with self.assertRaises(errors.OutOfRangeError): + self.evaluate(get_next()) + + def testPaddedBatchDatasetUnicode(self): + # See GitHub issue 16149 + def generator(): + data = [[u'Простой', u'тест', u'юникода'], + [u'никогда', u'не', u'бывает', u'простым']] + + for seq in data: + yield seq, [0, 1, 2, 3] + + dataset = dataset_ops.Dataset.from_generator( + generator, (dtypes.string, dtypes.int32), + (tensor_shape.TensorShape([None]), tensor_shape.TensorShape([None]))) + padded_dataset = dataset.padded_batch( + 2, padded_shapes=([None], [None]), padding_values=('', 0)) + next_element = self.getNext(padded_dataset) + self.evaluate(next_element()) + + @test_util.run_deprecated_v1 + def testSkipEagerPaddedBatchDatasetShapeSpecifications(self): + int_placeholder = array_ops.placeholder(dtypes.int32) + float_placeholder = array_ops.placeholder(dtypes.float32) + string_placeholder = array_ops.placeholder(dtypes.string) + input_dataset = dataset_ops.Dataset.from_tensors( + (int_placeholder, float_placeholder, string_placeholder)) + + # Test different ways of specifying the `padded_shapes` argument. + dynamic_padding_from_tensor_shapes = input_dataset.padded_batch( + 32, + padded_shapes=(tensor_shape.TensorShape([None]), + tensor_shape.TensorShape([None, None]), + tensor_shape.TensorShape([37]))) + dynamic_padding_from_lists = input_dataset.padded_batch( + 32, padded_shapes=([None], [None, None], [37])) + dynamic_padding_from_lists_with_minus_one = input_dataset.padded_batch( + 32, padded_shapes=([-1], [-1, -1], [37])) + dynamic_padding_from_tensors = input_dataset.padded_batch( + 32, + padded_shapes=(constant_op.constant([-1], dtype=dtypes.int64), + constant_op.constant([-1, -1], dtype=dtypes.int64), + constant_op.constant([37], dtype=dtypes.int64))) + + for dataset in [ + dynamic_padding_from_tensor_shapes, dynamic_padding_from_lists, + dynamic_padding_from_lists_with_minus_one, dynamic_padding_from_tensors + ]: + self.assertEqual([None, None], dataset.output_shapes[0].as_list()) + self.assertEqual([None, None, None], dataset.output_shapes[1].as_list()) + self.assertEqual([None, 37], dataset.output_shapes[2].as_list()) + + def testPaddedBatchSparseError(self): + + def _map_fn(i): + return sparse_tensor.SparseTensorValue( + indices=[[0, 0]], values=(i * [1]), dense_shape=[1, 1]), i + + with self.assertRaises(TypeError): + _ = dataset_ops.Dataset.range(10).map(_map_fn).padded_batch(10) + + def testPaddedBatchShapeError(self): + with self.assertRaisesRegexp( + ValueError, r'The padded shape \(1,\) is not compatible with the ' + r'corresponding input component shape \(\).'): + _ = dataset_ops.Dataset.range(10).padded_batch(5, padded_shapes=[1]) + + with self.assertRaisesRegexp( + ValueError, r'The padded shape \(1,\) is not compatible with the ' + r'corresponding input component shape \(3,\).'): + _ = dataset_ops.Dataset.from_tensors([1, 2, 3]).padded_batch( + 5, padded_shapes=[1]) + + with self.assertRaisesRegexp( + ValueError, r'Padded shape .* must be a 1-D tensor ' + r'of tf.int64 values, but its shape was \(2, 2\).'): + _ = dataset_ops.Dataset.from_tensors([1, 2, 3]).padded_batch( + 5, padded_shapes=[[1, 1], [1, 1]]) + + with self.assertRaisesRegexp( + TypeError, r'Padded shape .* must be a 1-D tensor ' + r'of tf.int64 values, but its element type was float32.'): + _ = dataset_ops.Dataset.from_tensors([1, 2, 3]).padded_batch( + 5, padded_shapes=constant_op.constant([1., 2., 3.])) + + with self.assertRaisesRegexp( + ValueError, r'The padded shape \(1,\) is not compatible with the ' + r'corresponding input component shape \(\).'): + shape_as_tensor = constant_op.constant([1], dtype=dtypes.int64) + _ = dataset_ops.Dataset.range(10).padded_batch( + 5, padded_shapes=shape_as_tensor) + + @test_util.run_deprecated_v1 + def testSkipEagerPaddedBatchShapeError(self): + with self.assertRaisesRegexp( + ValueError, + r'The padded shape \((\?|None), (\?|None)\) is not compatible with the ' + r'corresponding input component shape \(\).'): + shape_as_tensor = array_ops.placeholder(dtypes.int64, shape=[2]) + _ = dataset_ops.Dataset.range(10).padded_batch( + 5, padded_shapes=shape_as_tensor) + + +if __name__ == '__main__': + test.main() diff --git a/tensorflow/python/data/kernel_tests/prefetch_dataset_op_test.py b/tensorflow/python/data/kernel_tests/prefetch_test.py similarity index 52% rename from tensorflow/python/data/kernel_tests/prefetch_dataset_op_test.py rename to tensorflow/python/data/kernel_tests/prefetch_test.py index 76e2697b29d..a143ba0ac63 100644 --- a/tensorflow/python/data/kernel_tests/prefetch_dataset_op_test.py +++ b/tensorflow/python/data/kernel_tests/prefetch_test.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================== -"""Test PrefetchDataset.""" +"""Tests for `tf.data.Dataset.prefetch()`.""" from __future__ import absolute_import from __future__ import division from __future__ import print_function @@ -21,40 +21,24 @@ from absl.testing import parameterized from tensorflow.python.data.kernel_tests import test_base from tensorflow.python.data.ops import dataset_ops -from tensorflow.python.framework import dtypes from tensorflow.python.framework import errors -from tensorflow.python.ops import array_ops +from tensorflow.python.framework import test_util from tensorflow.python.platform import test -class PrefetchDatasetTest(test_base.DatasetTestBase, parameterized.TestCase): +@test_util.run_all_in_graph_and_eager_modes +class PrefetchTest(test_base.DatasetTestBase, parameterized.TestCase): @parameterized.parameters((-1), (0), (5)) def testBufferSize(self, buffer_size): - buffer_size_t = array_ops.placeholder(dtypes.int64, shape=[]) - iterator = dataset_ops.Dataset.range(10).prefetch( - buffer_size=buffer_size_t).make_initializable_iterator() - init_op = iterator.initializer - get_next = iterator.get_next() - - with self.cached_session() as sess: - sess.run(init_op, feed_dict={buffer_size_t: buffer_size}) - for m in range(10): - self.assertEqual(m, sess.run(get_next)) - with self.assertRaises(errors.OutOfRangeError): - sess.run(get_next) + dataset = dataset_ops.Dataset.range(10).prefetch(buffer_size=buffer_size) + self.assertDatasetProduces(dataset, expected_output=range(10)) @parameterized.parameters((-2), (-42)) def testInvalidBufferSize(self, buffer_size): - buffer_size_t = array_ops.placeholder(dtypes.int64, shape=[]) - iterator = dataset_ops.Dataset.range(10).prefetch( - buffer_size=buffer_size_t).make_initializable_iterator() - init_op = iterator.initializer - - with self.assertRaisesRegexp(errors.InvalidArgumentError, "buffer_size"): - with self.cached_session() as sess: - sess.run(init_op, feed_dict={buffer_size_t: buffer_size}) - + dataset = dataset_ops.Dataset.range(10).prefetch(buffer_size=buffer_size) + self.assertDatasetProduces( + dataset, expected_error=(errors.InvalidArgumentError, "buffer_size")) if __name__ == "__main__": test.main() diff --git a/tensorflow/python/data/kernel_tests/range_test.py b/tensorflow/python/data/kernel_tests/range_test.py new file mode 100644 index 00000000000..3f5d25e7f39 --- /dev/null +++ b/tensorflow/python/data/kernel_tests/range_test.py @@ -0,0 +1,72 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Tests for `tf.data.Dataset.range()`.""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from tensorflow.python.data.kernel_tests import test_base +from tensorflow.python.data.ops import dataset_ops +from tensorflow.python.framework import errors +from tensorflow.python.framework import test_util +from tensorflow.python.platform import test + + +@test_util.run_all_in_graph_and_eager_modes +class RangeTest(test_base.DatasetTestBase): + + def testStop(self): + dataset = dataset_ops.Dataset.range(5) + self.assertDatasetProduces(dataset, expected_output=range(5)) + + def testStartStop(self): + start, stop = 2, 5 + dataset = dataset_ops.Dataset.range(start, stop) + self.assertDatasetProduces(dataset, expected_output=range(2, 5)) + + def testStartStopStep(self): + start, stop, step = 2, 10, 2 + dataset = dataset_ops.Dataset.range(start, stop, step) + self.assertDatasetProduces(dataset, expected_output=range(2, 10, 2)) + + def testZeroStep(self): + start, stop, step = 2, 10, 0 + dataset = dataset_ops.Dataset.range(start, stop, step) + self.assertDatasetProduces( + dataset, expected_error=(errors.InvalidArgumentError, "")) + + def testNegativeStep(self): + start, stop, step = 2, 10, -1 + dataset = dataset_ops.Dataset.range(start, stop, step) + self.assertDatasetProduces(dataset, expected_output=range(2, 10, -1)) + + def testStopLessThanStart(self): + start, stop = 10, 2 + dataset = dataset_ops.Dataset.range(start, stop) + self.assertDatasetProduces(dataset, expected_output=range(10, 2)) + + def testStopLessThanStartWithPositiveStep(self): + start, stop, step = 10, 2, 2 + dataset = dataset_ops.Dataset.range(start, stop, step) + self.assertDatasetProduces(dataset, expected_output=range(10, 2, 2)) + + def testStopLessThanStartWithNegativeStep(self): + start, stop, step = 10, 2, -1 + dataset = dataset_ops.Dataset.range(start, stop, step) + self.assertDatasetProduces(dataset, expected_output=range(10, 2, -1)) + + +if __name__ == "__main__": + test.main() diff --git a/tensorflow/python/data/kernel_tests/reader_dataset_ops_test.py b/tensorflow/python/data/kernel_tests/reader_dataset_ops_test.py deleted file mode 100644 index 4fef4f30bf9..00000000000 --- a/tensorflow/python/data/kernel_tests/reader_dataset_ops_test.py +++ /dev/null @@ -1,846 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for the experimental input pipeline ops.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import gzip -import os -import zlib - -from tensorflow.python.data.kernel_tests import test_base -from tensorflow.python.data.ops import dataset_ops -from tensorflow.python.data.ops import iterator_ops -from tensorflow.python.data.ops import readers -from tensorflow.python.eager import context -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import errors -from tensorflow.python.framework import ops -from tensorflow.python.framework import tensor_shape -from tensorflow.python.lib.io import python_io -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import gen_dataset_ops -from tensorflow.python.ops import io_ops -from tensorflow.python.ops import parsing_ops -from tensorflow.python.platform import test -from tensorflow.python.util import compat - - -try: - import psutil # pylint: disable=g-import-not-at-top - psutil_import_succeeded = True -except ImportError: - psutil_import_succeeded = False - - -class TextLineDatasetTest(test_base.DatasetTestBase): - - def _lineText(self, f, l): - return compat.as_bytes("%d: %d" % (f, l)) - - def _createFiles(self, - num_files, - num_lines, - crlf=False, - compression_type=None): - filenames = [] - for i in range(num_files): - fn = os.path.join(self.get_temp_dir(), "text_line.%d.txt" % i) - filenames.append(fn) - contents = [] - for j in range(num_lines): - contents.append(self._lineText(i, j)) - # Always include a newline after the record unless it is - # at the end of the file, in which case we include it - if j + 1 != num_lines or i == 0: - contents.append(b"\r\n" if crlf else b"\n") - contents = b"".join(contents) - - if not compression_type: - with open(fn, "wb") as f: - f.write(contents) - elif compression_type == "GZIP": - with gzip.GzipFile(fn, "wb") as f: - f.write(contents) - elif compression_type == "ZLIB": - contents = zlib.compress(contents) - with open(fn, "wb") as f: - f.write(contents) - else: - raise ValueError("Unsupported compression_type", compression_type) - - return filenames - - def _testTextLineDataset(self, compression_type=None): - test_filenames = self._createFiles( - 2, 5, crlf=True, compression_type=compression_type) - filenames = array_ops.placeholder(dtypes.string, shape=[None]) - num_epochs = array_ops.placeholder(dtypes.int64, shape=[]) - batch_size = array_ops.placeholder(dtypes.int64, shape=[]) - - repeat_dataset = readers.TextLineDataset( - filenames, compression_type=compression_type).repeat(num_epochs) - batch_dataset = repeat_dataset.batch(batch_size) - - iterator = iterator_ops.Iterator.from_structure(batch_dataset.output_types) - init_op = iterator.make_initializer(repeat_dataset) - init_batch_op = iterator.make_initializer(batch_dataset) - get_next = iterator.get_next() - - with self.cached_session() as sess: - # Basic test: read from file 0. - sess.run( - init_op, feed_dict={filenames: [test_filenames[0]], - num_epochs: 1}) - for i in range(5): - self.assertEqual(self._lineText(0, i), sess.run(get_next)) - with self.assertRaises(errors.OutOfRangeError): - sess.run(get_next) - - # Basic test: read from file 1. - sess.run( - init_op, feed_dict={filenames: [test_filenames[1]], - num_epochs: 1}) - for i in range(5): - self.assertEqual(self._lineText(1, i), sess.run(get_next)) - with self.assertRaises(errors.OutOfRangeError): - sess.run(get_next) - - # Basic test: read from both files. - sess.run(init_op, feed_dict={filenames: test_filenames, num_epochs: 1}) - for j in range(2): - for i in range(5): - self.assertEqual(self._lineText(j, i), sess.run(get_next)) - with self.assertRaises(errors.OutOfRangeError): - sess.run(get_next) - - # Test repeated iteration through both files. - sess.run(init_op, feed_dict={filenames: test_filenames, num_epochs: 10}) - for _ in range(10): - for j in range(2): - for i in range(5): - self.assertEqual(self._lineText(j, i), sess.run(get_next)) - with self.assertRaises(errors.OutOfRangeError): - sess.run(get_next) - - # Test batched and repeated iteration through both files. - sess.run( - init_batch_op, - feed_dict={filenames: test_filenames, - num_epochs: 10, - batch_size: 5}) - for _ in range(10): - self.assertAllEqual([self._lineText(0, i) for i in range(5)], - sess.run(get_next)) - self.assertAllEqual([self._lineText(1, i) for i in range(5)], - sess.run(get_next)) - - def testTextLineDatasetNoCompression(self): - self._testTextLineDataset() - - def testTextLineDatasetGzipCompression(self): - self._testTextLineDataset(compression_type="GZIP") - - def testTextLineDatasetZlibCompression(self): - self._testTextLineDataset(compression_type="ZLIB") - - def testTextLineDatasetBuffering(self): - test_filenames = self._createFiles(2, 5, crlf=True) - - repeat_dataset = readers.TextLineDataset(test_filenames, buffer_size=10) - iterator = repeat_dataset.make_one_shot_iterator() - - with self.cached_session() as sess: - for j in range(2): - for i in range(5): - self.assertEqual(self._lineText(j, i), sess.run(iterator.get_next())) - with self.assertRaises(errors.OutOfRangeError): - sess.run(iterator.get_next()) - - def testIteratorResourceCleanup(self): - filename = os.path.join(self.get_temp_dir(), "text.txt") - with open(filename, "wt") as f: - for i in range(3): - f.write("%d\n" % (i,)) - with context.eager_mode(): - first_iterator = iter(readers.TextLineDataset(filename)) - self.assertEqual(b"0", next(first_iterator).numpy()) - second_iterator = iter(readers.TextLineDataset(filename)) - self.assertEqual(b"0", next(second_iterator).numpy()) - # Eager kernel caching is based on op attributes, which includes the - # Dataset's output shape. Create a different kernel to test that they - # don't create resources with the same names. - different_kernel_iterator = iter( - readers.TextLineDataset(filename).repeat().batch(16)) - self.assertEqual([16], next(different_kernel_iterator).shape) - # Remove our references to the Python Iterator objects, which (assuming no - # reference cycles) is enough to trigger DestroyResourceOp and close the - # partially-read files. - del first_iterator - del second_iterator - del different_kernel_iterator - if not psutil_import_succeeded: - self.skipTest( - "psutil is required to check that we've closed our files.") - open_files = psutil.Process().open_files() - self.assertNotIn(filename, [open_file.path for open_file in open_files]) - - -class FixedLengthRecordReaderTest(test_base.DatasetTestBase): - - def setUp(self): - super(FixedLengthRecordReaderTest, self).setUp() - self._num_files = 2 - self._num_records = 7 - self._header_bytes = 5 - self._record_bytes = 3 - self._footer_bytes = 2 - - def _record(self, f, r): - return compat.as_bytes(str(f * 2 + r) * self._record_bytes) - - def _createFiles(self, compression_type=None): - filenames = [] - for i in range(self._num_files): - fn = os.path.join(self.get_temp_dir(), "fixed_length_record.%d.txt" % i) - filenames.append(fn) - - contents = [] - contents.append(b"H" * self._header_bytes) - for j in range(self._num_records): - contents.append(self._record(i, j)) - contents.append(b"F" * self._footer_bytes) - contents = b"".join(contents) - - if not compression_type: - with open(fn, "wb") as f: - f.write(contents) - elif compression_type == "GZIP": - with gzip.GzipFile(fn, "wb") as f: - f.write(contents) - elif compression_type == "ZLIB": - contents = zlib.compress(contents) - with open(fn, "wb") as f: - f.write(contents) - else: - raise ValueError("Unsupported compression_type", compression_type) - - return filenames - - def _testFixedLengthRecordDataset(self, compression_type=None): - test_filenames = self._createFiles(compression_type=compression_type) - filenames = array_ops.placeholder(dtypes.string, shape=[None]) - num_epochs = array_ops.placeholder(dtypes.int64, shape=[]) - batch_size = array_ops.placeholder(dtypes.int64, shape=[]) - - repeat_dataset = ( - readers.FixedLengthRecordDataset( - filenames, - self._record_bytes, - self._header_bytes, - self._footer_bytes, - compression_type=compression_type).repeat(num_epochs)) - batch_dataset = repeat_dataset.batch(batch_size) - - iterator = iterator_ops.Iterator.from_structure(batch_dataset.output_types) - init_op = iterator.make_initializer(repeat_dataset) - init_batch_op = iterator.make_initializer(batch_dataset) - get_next = iterator.get_next() - - with self.cached_session() as sess: - # Basic test: read from file 0. - sess.run( - init_op, feed_dict={filenames: [test_filenames[0]], - num_epochs: 1}) - for i in range(self._num_records): - self.assertEqual(self._record(0, i), sess.run(get_next)) - with self.assertRaises(errors.OutOfRangeError): - sess.run(get_next) - - # Basic test: read from file 1. - sess.run( - init_op, feed_dict={filenames: [test_filenames[1]], - num_epochs: 1}) - for i in range(self._num_records): - self.assertEqual(self._record(1, i), sess.run(get_next)) - with self.assertRaises(errors.OutOfRangeError): - sess.run(get_next) - - # Basic test: read from both files. - sess.run(init_op, feed_dict={filenames: test_filenames, num_epochs: 1}) - for j in range(self._num_files): - for i in range(self._num_records): - self.assertEqual(self._record(j, i), sess.run(get_next)) - with self.assertRaises(errors.OutOfRangeError): - sess.run(get_next) - - # Test repeated iteration through both files. - sess.run(init_op, feed_dict={filenames: test_filenames, num_epochs: 10}) - for _ in range(10): - for j in range(self._num_files): - for i in range(self._num_records): - self.assertEqual(self._record(j, i), sess.run(get_next)) - with self.assertRaises(errors.OutOfRangeError): - sess.run(get_next) - - # Test batched and repeated iteration through both files. - sess.run( - init_batch_op, - feed_dict={ - filenames: test_filenames, - num_epochs: 10, - batch_size: self._num_records - }) - for _ in range(10): - for j in range(self._num_files): - self.assertAllEqual( - [self._record(j, i) for i in range(self._num_records)], - sess.run(get_next)) - with self.assertRaises(errors.OutOfRangeError): - sess.run(get_next) - - def testFixedLengthRecordDatasetNoCompression(self): - self._testFixedLengthRecordDataset() - - def testFixedLengthRecordDatasetGzipCompression(self): - self._testFixedLengthRecordDataset(compression_type="GZIP") - - def testFixedLengthRecordDatasetZlibCompression(self): - self._testFixedLengthRecordDataset(compression_type="ZLIB") - - def testFixedLengthRecordDatasetBuffering(self): - test_filenames = self._createFiles() - dataset = readers.FixedLengthRecordDataset( - test_filenames, - self._record_bytes, - self._header_bytes, - self._footer_bytes, - buffer_size=10) - iterator = dataset.make_one_shot_iterator() - - with self.cached_session() as sess: - for j in range(self._num_files): - for i in range(self._num_records): - self.assertEqual(self._record(j, i), sess.run(iterator.get_next())) - with self.assertRaises(errors.OutOfRangeError): - sess.run(iterator.get_next()) - - def testFixedLengthRecordDatasetWrongSize(self): - test_filenames = self._createFiles() - dataset = readers.FixedLengthRecordDataset( - test_filenames, - self._record_bytes + 1, # Incorrect record length. - self._header_bytes, - self._footer_bytes, - buffer_size=10) - iterator = dataset.make_one_shot_iterator() - - with self.cached_session() as sess: - with self.assertRaisesRegexp( - errors.InvalidArgumentError, - r"Excluding the header \(5 bytes\) and footer \(2 bytes\), input " - r"file \".*fixed_length_record.0.txt\" has body length 21 bytes, " - r"which is not an exact multiple of the record length \(4 bytes\)."): - sess.run(iterator.get_next()) - - def _iterator_checkpoint_path(self): - return os.path.join(self.get_temp_dir(), "iterator") - - def _save_op(self, iterator_resource): - iterator_state_variant = gen_dataset_ops.serialize_iterator( - iterator_resource) - save_op = io_ops.write_file( - self._iterator_checkpoint_path(), - parsing_ops.serialize_tensor(iterator_state_variant)) - return save_op - - def _restore_op(self, iterator_resource): - iterator_state_variant = parsing_ops.parse_tensor( - io_ops.read_file(self._iterator_checkpoint_path()), dtypes.variant) - restore_op = gen_dataset_ops.deserialize_iterator(iterator_resource, - iterator_state_variant) - return restore_op - - def _build_iterator_graph(self, num_epochs): - filenames = self._createFiles() - dataset = (readers.FixedLengthRecordDataset( - filenames, self._record_bytes, self._header_bytes, self._footer_bytes) - .repeat(num_epochs)) - iterator = dataset.make_initializable_iterator() - init_op = iterator.initializer - get_next_op = iterator.get_next() - save_op = self._save_op(iterator._iterator_resource) - restore_op = self._restore_op(iterator._iterator_resource) - return init_op, get_next_op, save_op, restore_op - - def _restore_iterator(self): - output_types = dtypes.string - output_shapes = tensor_shape.scalar() - iterator = iterator_ops.Iterator.from_structure(output_types, output_shapes) - get_next = iterator.get_next() - restore_op = self._restore_op(iterator._iterator_resource) - return restore_op, get_next - - def testSaveRestore(self): - num_epochs = 10 - epoch_break = 5 - file_break = self._num_files // 2 - record_break = self._num_records // 2 - - with ops.Graph().as_default() as g: - init_op, get_next_op, save_op, restore_op = self._build_iterator_graph( - num_epochs=num_epochs) - with self.session(graph=g) as sess: - sess.run(init_op) - # Note: There is no checkpoint saved currently so a NotFoundError is - # raised. - with self.assertRaises(errors.NotFoundError): - sess.run(restore_op) - for epoch in range(num_epochs): - for f in range(self._num_files): - for r in range(self._num_records): - if (epoch == epoch_break and f == file_break and - r == record_break): - sess.run(save_op) - break - self.assertEqual(self._record(f, r), sess.run(get_next_op)) - else: - continue - break - else: - continue - break - else: - with self.assertRaises(errors.OutOfRangeError): - sess.run(get_next_op) - - with ops.Graph().as_default() as g: - init_op, get_next_op, save_op, restore_op = self._build_iterator_graph( - num_epochs=num_epochs) - with self.session(graph=g) as sess: - sess.run(restore_op) - for epoch in range(num_epochs): - for f in range(self._num_files): - for r in range(self._num_records): - if (epoch < epoch_break or - (epoch == epoch_break and f < file_break) or - (epoch == epoch_break and f == file_break and - r < record_break)): - continue - self.assertEqual(self._record(f, r), sess.run(get_next_op)) - with self.assertRaises(errors.OutOfRangeError): - sess.run(get_next_op) - - def testInitThenRestore(self): - # Note: Calling init_op before restore_op is redundant. This test just makes - # sure we do not fail if restore is called on an already initialized - # iterator resource. - num_epochs = 10 - epoch_break = 5 - file_break = self._num_files // 2 - record_break = self._num_records // 2 - - with ops.Graph().as_default() as g: - init_op, get_next_op, save_op, restore_op = self._build_iterator_graph( - num_epochs=num_epochs) - with self.session(graph=g) as sess: - sess.run(init_op) - # Note: There is no checkpoint saved currently so a NotFoundError is - # raised. - with self.assertRaises(errors.NotFoundError): - sess.run(restore_op) - for epoch in range(num_epochs): - for f in range(self._num_files): - for r in range(self._num_records): - if (epoch == epoch_break and f == file_break and - r == record_break): - sess.run(save_op) - break - self.assertEqual(self._record(f, r), sess.run(get_next_op)) - else: - continue - break - else: - continue - break - else: - with self.assertRaises(errors.OutOfRangeError): - sess.run(get_next_op) - - with ops.Graph().as_default() as g: - init_op, get_next_op, save_op, restore_op = self._build_iterator_graph( - num_epochs=num_epochs) - with self.session(graph=g) as sess: - sess.run(init_op) - sess.run(restore_op) - for epoch in range(num_epochs): - for f in range(self._num_files): - for r in range(self._num_records): - if (epoch < epoch_break or - (epoch == epoch_break and f < file_break) or - (epoch == epoch_break and f == file_break and - r < record_break)): - continue - self.assertEqual(self._record(f, r), sess.run(get_next_op)) - with self.assertRaises(errors.OutOfRangeError): - sess.run(get_next_op) - - def testRestoreInModifiedGraph(self): - num_epochs = 10 - num_epochs_1 = 20 - epoch_break = 5 - file_break = self._num_files // 2 - record_break = self._num_records // 2 - - with ops.Graph().as_default() as g: - init_op, get_next_op, save_op, restore_op = self._build_iterator_graph( - num_epochs=num_epochs) - with self.session(graph=g) as sess: - sess.run(init_op) - # Note: There is no checkpoint saved currently so a NotFoundError is - # raised. - with self.assertRaises(errors.NotFoundError): - sess.run(restore_op) - for epoch in range(num_epochs): - for f in range(self._num_files): - for r in range(self._num_records): - if (epoch == epoch_break and f == file_break and - r == record_break): - sess.run(save_op) - break - self.assertEqual(self._record(f, r), sess.run(get_next_op)) - else: - continue - break - else: - continue - break - else: - with self.assertRaises(errors.OutOfRangeError): - sess.run(get_next_op) - - with ops.Graph().as_default() as g: - init_op, get_next_op, save_op, restore_op = self._build_iterator_graph( - num_epochs=num_epochs_1) - with self.session(graph=g) as sess: - sess.run(restore_op) - for epoch in range(num_epochs): - for f in range(self._num_files): - for r in range(self._num_records): - if (epoch < epoch_break or - (epoch == epoch_break and f < file_break) or - (epoch == epoch_break and f == file_break and - r < record_break)): - continue - self.assertEqual(self._record(f, r), sess.run(get_next_op)) - with self.assertRaises(errors.OutOfRangeError): - sess.run(get_next_op) - - def testRestoreWithoutBuildingDatasetGraph(self): - num_epochs = 10 - epoch_break = 5 - file_break = self._num_files // 2 - record_break = self._num_records // 2 - - with ops.Graph().as_default() as g: - init_op, get_next_op, save_op, restore_op = self._build_iterator_graph( - num_epochs=num_epochs) - with self.session(graph=g) as sess: - sess.run(init_op) - # Note: There is no checkpoint saved currently so a NotFoundError is - # raised. - with self.assertRaises(errors.NotFoundError): - sess.run(restore_op) - for epoch in range(num_epochs): - for f in range(self._num_files): - for r in range(self._num_records): - if (epoch == epoch_break and f == file_break and - r == record_break): - sess.run(save_op) - break - self.assertEqual(self._record(f, r), sess.run(get_next_op)) - else: - continue - break - else: - continue - break - else: - with self.assertRaises(errors.OutOfRangeError): - sess.run(get_next_op) - - with ops.Graph().as_default() as g: - restore_op, get_next_op = self._restore_iterator() - with self.session(graph=g) as sess: - sess.run(restore_op) - for epoch in range(num_epochs): - for f in range(self._num_files): - for r in range(self._num_records): - if (epoch < epoch_break or - (epoch == epoch_break and f < file_break) or - (epoch == epoch_break and f == file_break and - r < record_break)): - continue - self.assertEqual(self._record(f, r), sess.run(get_next_op)) - with self.assertRaises(errors.OutOfRangeError): - sess.run(get_next_op) - - def testRestoreUnusedIterator(self): - num_epochs = 10 - with ops.Graph().as_default() as g: - init_op, get_next_op, save_op, restore_op = self._build_iterator_graph( - num_epochs=num_epochs) - with self.session(graph=g) as sess: - sess.run(init_op) - # Note: There is no checkpoint saved currently so a NotFoundError is - # raised. - with self.assertRaises(errors.NotFoundError): - sess.run(restore_op) - # Save unused iterator. - sess.run(save_op) - with ops.Graph().as_default() as g: - init_op, get_next_op, save_op, restore_op = self._build_iterator_graph( - num_epochs=num_epochs) - with self.session(graph=g) as sess: - sess.run(restore_op) - for _ in range(num_epochs * self._num_files * self._num_records): - sess.run(get_next_op) - with self.assertRaises(errors.OutOfRangeError): - sess.run(get_next_op) - - def testRestoreExhaustedIterator(self): - num_epochs = 10 - - with ops.Graph().as_default() as g: - init_op, get_next_op, save_op, restore_op = self._build_iterator_graph( - num_epochs=num_epochs) - with self.session(graph=g) as sess: - sess.run(init_op) - # Note: There is no checkpoint saved currently so a NotFoundError is - # raised. - with self.assertRaises(errors.NotFoundError): - sess.run(restore_op) - for _ in range(num_epochs): - for f in range(self._num_files): - for r in range(self._num_records): - self.assertEqual(self._record(f, r), sess.run(get_next_op)) - with self.assertRaises(errors.OutOfRangeError): - sess.run(get_next_op) - sess.run(save_op) - - with ops.Graph().as_default() as g: - init_op, get_next_op, save_op, restore_op = self._build_iterator_graph( - num_epochs=num_epochs) - with self.session(graph=g) as sess: - sess.run(restore_op) - with self.assertRaises(errors.OutOfRangeError): - sess.run(get_next_op) - - -class TFRecordDatasetTest(test_base.DatasetTestBase): - - def setUp(self): - super(TFRecordDatasetTest, self).setUp() - self._num_files = 2 - self._num_records = 7 - - self.test_filenames = self._createFiles() - - self.filenames = array_ops.placeholder(dtypes.string, shape=[None]) - self.num_epochs = array_ops.placeholder_with_default( - constant_op.constant(1, dtypes.int64), shape=[]) - self.compression_type = array_ops.placeholder_with_default("", shape=[]) - self.batch_size = array_ops.placeholder(dtypes.int64, shape=[]) - - repeat_dataset = readers.TFRecordDataset(self.filenames, - self.compression_type).repeat( - self.num_epochs) - batch_dataset = repeat_dataset.batch(self.batch_size) - - iterator = iterator_ops.Iterator.from_structure(batch_dataset.output_types) - self.init_op = iterator.make_initializer(repeat_dataset) - self.init_batch_op = iterator.make_initializer(batch_dataset) - self.get_next = iterator.get_next() - - def _record(self, f, r): - return compat.as_bytes("Record %d of file %d" % (r, f)) - - def _createFiles(self): - filenames = [] - for i in range(self._num_files): - fn = os.path.join(self.get_temp_dir(), "tf_record.%d.txt" % i) - filenames.append(fn) - writer = python_io.TFRecordWriter(fn) - for j in range(self._num_records): - writer.write(self._record(i, j)) - writer.close() - return filenames - - def testReadOneEpoch(self): - with self.cached_session() as sess: - # Basic test: read from file 0. - sess.run( - self.init_op, - feed_dict={ - self.filenames: [self.test_filenames[0]], - self.num_epochs: 1 - }) - for i in range(self._num_records): - self.assertAllEqual(self._record(0, i), sess.run(self.get_next)) - with self.assertRaises(errors.OutOfRangeError): - sess.run(self.get_next) - - # Basic test: read from file 1. - sess.run( - self.init_op, - feed_dict={ - self.filenames: [self.test_filenames[1]], - self.num_epochs: 1 - }) - for i in range(self._num_records): - self.assertAllEqual(self._record(1, i), sess.run(self.get_next)) - with self.assertRaises(errors.OutOfRangeError): - sess.run(self.get_next) - - # Basic test: read from both files. - sess.run( - self.init_op, - feed_dict={self.filenames: self.test_filenames, - self.num_epochs: 1}) - for j in range(self._num_files): - for i in range(self._num_records): - self.assertAllEqual(self._record(j, i), sess.run(self.get_next)) - with self.assertRaises(errors.OutOfRangeError): - sess.run(self.get_next) - - def testReadTenEpochs(self): - with self.cached_session() as sess: - sess.run( - self.init_op, - feed_dict={self.filenames: self.test_filenames, - self.num_epochs: 10}) - for _ in range(10): - for j in range(self._num_files): - for i in range(self._num_records): - self.assertAllEqual(self._record(j, i), sess.run(self.get_next)) - with self.assertRaises(errors.OutOfRangeError): - sess.run(self.get_next) - - def testReadTenEpochsOfBatches(self): - with self.cached_session() as sess: - sess.run( - self.init_batch_op, - feed_dict={ - self.filenames: self.test_filenames, - self.num_epochs: 10, - self.batch_size: self._num_records - }) - for _ in range(10): - for j in range(self._num_files): - values = sess.run(self.get_next) - self.assertAllEqual( - [self._record(j, i) for i in range(self._num_records)], values) - with self.assertRaises(errors.OutOfRangeError): - sess.run(self.get_next) - - def testReadZlibFiles(self): - zlib_files = [] - for i, fn in enumerate(self.test_filenames): - with open(fn, "rb") as f: - cdata = zlib.compress(f.read()) - - zfn = os.path.join(self.get_temp_dir(), "tfrecord_%s.z" % i) - with open(zfn, "wb") as f: - f.write(cdata) - zlib_files.append(zfn) - - with self.cached_session() as sess: - sess.run( - self.init_op, - feed_dict={self.filenames: zlib_files, - self.compression_type: "ZLIB"}) - for j in range(self._num_files): - for i in range(self._num_records): - self.assertAllEqual(self._record(j, i), sess.run(self.get_next)) - with self.assertRaises(errors.OutOfRangeError): - sess.run(self.get_next) - - def testReadGzipFiles(self): - gzip_files = [] - for i, fn in enumerate(self.test_filenames): - with open(fn, "rb") as f: - gzfn = os.path.join(self.get_temp_dir(), "tfrecord_%s.gz" % i) - with gzip.GzipFile(gzfn, "wb") as gzf: - gzf.write(f.read()) - gzip_files.append(gzfn) - - with self.cached_session() as sess: - sess.run( - self.init_op, - feed_dict={self.filenames: gzip_files, - self.compression_type: "GZIP"}) - for j in range(self._num_files): - for i in range(self._num_records): - self.assertAllEqual(self._record(j, i), sess.run(self.get_next)) - with self.assertRaises(errors.OutOfRangeError): - sess.run(self.get_next) - - def testReadWithBuffer(self): - one_mebibyte = 2**20 - d = readers.TFRecordDataset(self.test_filenames, buffer_size=one_mebibyte) - iterator = d.make_one_shot_iterator() - next_element = iterator.get_next() - with self.cached_session() as sess: - for j in range(self._num_files): - for i in range(self._num_records): - self.assertAllEqual(self._record(j, i), sess.run(next_element)) - with self.assertRaises(errors.OutOfRangeError): - sess.run(next_element) - - def testReadFromDatasetOfFiles(self): - files = dataset_ops.Dataset.from_tensor_slices(self.test_filenames) - d = readers.TFRecordDataset(files) - iterator = d.make_one_shot_iterator() - next_element = iterator.get_next() - with self.cached_session() as sess: - for j in range(self._num_files): - for i in range(self._num_records): - self.assertAllEqual(self._record(j, i), sess.run(next_element)) - with self.assertRaises(errors.OutOfRangeError): - sess.run(next_element) - - def testReadTenEpochsFromDatasetOfFilesInParallel(self): - files = dataset_ops.Dataset.from_tensor_slices( - self.test_filenames).repeat(10) - d = readers.TFRecordDataset(files, num_parallel_reads=4) - iterator = d.make_one_shot_iterator() - next_element = iterator.get_next() - expected = [] - actual = [] - with self.cached_session() as sess: - for _ in range(10): - for j in range(self._num_files): - for i in range(self._num_records): - expected.append(self._record(j, i)) - actual.append(sess.run(next_element)) - with self.assertRaises(errors.OutOfRangeError): - sess.run(next_element) - self.assertEqual(sorted(expected), sorted(actual)) - - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/python/data/kernel_tests/reduce_dataset_op_test.py b/tensorflow/python/data/kernel_tests/reduce_test.py similarity index 72% rename from tensorflow/python/data/kernel_tests/reduce_dataset_op_test.py rename to tensorflow/python/data/kernel_tests/reduce_test.py index 11e07300b97..14bbc0bf72c 100644 --- a/tensorflow/python/data/kernel_tests/reduce_dataset_op_test.py +++ b/tensorflow/python/data/kernel_tests/reduce_test.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================== -"""Tests for the experimental input pipeline ops.""" +"""Tests for `tf.data.Dataset.reduce()`.""" from __future__ import absolute_import from __future__ import division from __future__ import print_function @@ -22,21 +22,24 @@ import numpy as np from tensorflow.python.data.kernel_tests import test_base from tensorflow.python.data.ops import dataset_ops +from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes from tensorflow.python.framework import sparse_tensor +from tensorflow.python.framework import test_util from tensorflow.python.ops import array_ops from tensorflow.python.ops import math_ops from tensorflow.python.platform import test -class ReduceDatasetTest(test_base.DatasetTestBase, parameterized.TestCase): +@test_util.run_all_in_graph_and_eager_modes +class ReduceTest(test_base.DatasetTestBase, parameterized.TestCase): def testSum(self): for i in range(10): ds = dataset_ops.Dataset.range(1, i + 1) - result = ds.reduce(np.int64(0), lambda x, y: x + y) - with self.cached_session() as sess: - self.assertEqual(((i + 1) * i) // 2, sess.run(result)) + result = ds.reduce( + constant_op.constant(0, dtype=dtypes.int64), lambda x, y: x + y) + self.assertEqual(((i + 1) * i) // 2, self.evaluate(result)) def testSumTuple(self): @@ -47,9 +50,8 @@ class ReduceDatasetTest(test_base.DatasetTestBase, parameterized.TestCase): for i in range(10): ds = dataset_ops.Dataset.range(1, i + 1) ds = dataset_ops.Dataset.zip((ds, ds)) - result = ds.reduce(np.int64(0), reduce_fn) - with self.cached_session() as sess: - self.assertEqual(((i + 1) * i), sess.run(result)) + result = ds.reduce(constant_op.constant(0, dtype=dtypes.int64), reduce_fn) + self.assertEqual(((i + 1) * i), self.evaluate(result)) def testSumAndCount(self): @@ -59,13 +61,15 @@ class ReduceDatasetTest(test_base.DatasetTestBase, parameterized.TestCase): for i in range(10): ds = dataset_ops.Dataset.range(1, i + 1) - result = ds.reduce((np.int64(0), np.int64(0)), reduce_fn) - with self.cached_session() as sess: - s, c = sess.run(result) - self.assertEqual(((i + 1) * i) // 2, s) - self.assertEqual(i, c) + result = ds.reduce((constant_op.constant(0, dtype=dtypes.int64), + constant_op.constant(0, dtype=dtypes.int64)), + reduce_fn) + s, c = self.evaluate(result) + self.assertEqual(((i + 1) * i) // 2, s) + self.assertEqual(i, c) - def testSquareUsingPlaceholder(self): + @test_util.run_deprecated_v1 + def testSkipEagerSquareUsingPlaceholder(self): delta = array_ops.placeholder(dtype=dtypes.int64) def reduce_fn(state, _): @@ -92,8 +96,7 @@ class ReduceDatasetTest(test_base.DatasetTestBase, parameterized.TestCase): for i in range(10): ds = dataset_ops.Dataset.from_tensors(make_sparse_fn(i+1)) result = ds.reduce(make_sparse_fn(0), reduce_fn) - with self.cached_session() as sess: - self.assertSparseValuesEqual(make_sparse_fn(i+1), sess.run(result)) + self.assertSparseValuesEqual(make_sparse_fn(i + 1), self.evaluate(result)) def testNested(self): @@ -115,10 +118,10 @@ class ReduceDatasetTest(test_base.DatasetTestBase, parameterized.TestCase): for i in range(10): ds = dataset_ops.Dataset.range(1, i + 1).map(map_fn) result = ds.reduce(map_fn(0), reduce_fn) - with self.cached_session() as sess: - result = sess.run(result) - self.assertEqual(((i + 1) * i) // 2, result["dense"]) - self.assertSparseValuesEqual(make_sparse_fn(i), result["sparse"]) + result = self.evaluate(result) + self.assertEqual(((i + 1) * i) // 2, result["dense"]) + self.assertSparseValuesEqual(make_sparse_fn(i), result["sparse"]) + if __name__ == "__main__": test.main() diff --git a/tensorflow/python/data/kernel_tests/repeat_test.py b/tensorflow/python/data/kernel_tests/repeat_test.py new file mode 100644 index 00000000000..4ef2fc1bfc8 --- /dev/null +++ b/tensorflow/python/data/kernel_tests/repeat_test.py @@ -0,0 +1,84 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Tests for `tf.data.Dataset.repeat()`.""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import numpy as np + +from tensorflow.python.data.kernel_tests import test_base +from tensorflow.python.data.ops import dataset_ops +from tensorflow.python.framework import test_util +from tensorflow.python.platform import test + + +@test_util.run_all_in_graph_and_eager_modes +class RepeatTest(test_base.DatasetTestBase): + + def testRepeatTensorDataset(self): + """Test a dataset that repeats its input multiple times.""" + components = (np.array(1), np.array([1, 2, 3]), np.array(37.0)) + # This placeholder can be fed when dataset-definition subgraph + # runs (i.e. `init_op` below) to configure the number of + # repetitions used in a particular iterator. + + def do_test(count): + dataset = dataset_ops.Dataset.from_tensors(components).repeat(count) + self.assertEqual([c.shape for c in components], + [shape for shape in dataset.output_shapes]) + self.assertDatasetProduces(dataset, [components] * count) + + # Test a finite repetition. + do_test(3) + + # test a different finite repetition. + do_test(7) + + # Test an empty repetition. + do_test(0) + + # Test an infinite repetition. + # NOTE(mrry): There's not a good way to test that the sequence + # actually is infinite. + dataset = dataset_ops.Dataset.from_tensors(components).repeat(-1) + self.assertEqual([c.shape for c in components], + [shape for shape in dataset.output_shapes]) + get_next = self.getNext(dataset) + for _ in range(17): + results = self.evaluate(get_next()) + for component, result_component in zip(components, results): + self.assertAllEqual(component, result_component) + + def testRepeatRepeatTensorDataset(self): + """Test the composition of repeat datasets.""" + components = (np.array(1), np.array([1, 2, 3]), np.array(37.0)) + inner_count, outer_count = 7, 14 + + dataset = dataset_ops.Dataset.from_tensors(components).repeat( + inner_count).repeat(outer_count) + self.assertEqual([c.shape for c in components], + [shape for shape in dataset.output_shapes]) + self.assertDatasetProduces(dataset, + [components] * (inner_count * outer_count)) + + def testRepeatEmptyDataset(self): + """Test that repeating an empty dataset does not hang.""" + dataset = dataset_ops.Dataset.from_tensors(0).repeat(10).skip(10).repeat(-1) + self.assertDatasetProduces(dataset, []) + + +if __name__ == "__main__": + test.main() diff --git a/tensorflow/python/data/kernel_tests/sequence_dataset_op_test.py b/tensorflow/python/data/kernel_tests/sequence_dataset_op_test.py deleted file mode 100644 index e86356dee7c..00000000000 --- a/tensorflow/python/data/kernel_tests/sequence_dataset_op_test.py +++ /dev/null @@ -1,210 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for the experimental input pipeline ops.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np - -from tensorflow.python.data.kernel_tests import test_base -from tensorflow.python.data.ops import dataset_ops -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import errors -from tensorflow.python.ops import array_ops -from tensorflow.python.platform import test - - -class SequenceDatasetTest(test_base.DatasetTestBase): - - def testRepeatTensorDataset(self): - """Test a dataset that repeats its input multiple times.""" - components = (np.array(1), np.array([1, 2, 3]), np.array(37.0)) - # This placeholder can be fed when dataset-definition subgraph - # runs (i.e. `init_op` below) to configure the number of - # repetitions used in a particular iterator. - count_placeholder = array_ops.placeholder(dtypes.int64, shape=[]) - - iterator = (dataset_ops.Dataset.from_tensors(components) - .repeat(count_placeholder).make_initializable_iterator()) - init_op = iterator.initializer - get_next = iterator.get_next() - - self.assertEqual([c.shape for c in components], - [t.shape for t in get_next]) - - with self.cached_session() as sess: - # Test a finite repetition. - sess.run(init_op, feed_dict={count_placeholder: 3}) - for _ in range(3): - results = sess.run(get_next) - for component, result_component in zip(components, results): - self.assertAllEqual(component, result_component) - - with self.assertRaises(errors.OutOfRangeError): - sess.run(get_next) - - # Test a different finite repetition. - sess.run(init_op, feed_dict={count_placeholder: 7}) - for _ in range(7): - results = sess.run(get_next) - for component, result_component in zip(components, results): - self.assertAllEqual(component, result_component) - with self.assertRaises(errors.OutOfRangeError): - sess.run(get_next) - - # Test an empty repetition. - sess.run(init_op, feed_dict={count_placeholder: 0}) - with self.assertRaises(errors.OutOfRangeError): - sess.run(get_next) - - # Test an infinite repetition. - # NOTE(mrry): There's not a good way to test that the sequence - # actually is infinite. - sess.run(init_op, feed_dict={count_placeholder: -1}) - for _ in range(17): - results = sess.run(get_next) - for component, result_component in zip(components, results): - self.assertAllEqual(component, result_component) - - def testTakeTensorDataset(self): - components = (np.arange(10),) - count_placeholder = array_ops.placeholder(dtypes.int64, shape=[]) - - iterator = (dataset_ops.Dataset.from_tensor_slices(components) - .take(count_placeholder).make_initializable_iterator()) - init_op = iterator.initializer - get_next = iterator.get_next() - - self.assertEqual([c.shape[1:] for c in components], - [t.shape for t in get_next]) - - with self.cached_session() as sess: - # Take fewer than input size - sess.run(init_op, feed_dict={count_placeholder: 4}) - for i in range(4): - results = sess.run(get_next) - self.assertAllEqual(results, components[0][i:i+1]) - - with self.assertRaises(errors.OutOfRangeError): - sess.run(get_next) - - # Take more than input size - sess.run(init_op, feed_dict={count_placeholder: 25}) - for i in range(10): - results = sess.run(get_next) - self.assertAllEqual(results, components[0][i:i+1]) - - with self.assertRaises(errors.OutOfRangeError): - sess.run(get_next) - - # Take all of input - sess.run(init_op, feed_dict={count_placeholder: -1}) - for i in range(10): - results = sess.run(get_next) - self.assertAllEqual(results, components[0][i:i+1]) - - with self.assertRaises(errors.OutOfRangeError): - sess.run(get_next) - - # Take nothing - sess.run(init_op, feed_dict={count_placeholder: 0}) - - with self.assertRaises(errors.OutOfRangeError): - sess.run(get_next) - - def testSkipTensorDataset(self): - components = (np.arange(10),) - count_placeholder = array_ops.placeholder(dtypes.int64, shape=[]) - - iterator = (dataset_ops.Dataset.from_tensor_slices(components) - .skip(count_placeholder).make_initializable_iterator()) - init_op = iterator.initializer - get_next = iterator.get_next() - - self.assertEqual([c.shape[1:] for c in components], - [t.shape for t in get_next]) - - with self.cached_session() as sess: - # Skip fewer than input size, we should skip - # the first 4 elements and then read the rest. - sess.run(init_op, feed_dict={count_placeholder: 4}) - for i in range(4, 10): - results = sess.run(get_next) - self.assertAllEqual(results, components[0][i:i+1]) - with self.assertRaises(errors.OutOfRangeError): - sess.run(get_next) - - # Skip more than input size: get nothing. - sess.run(init_op, feed_dict={count_placeholder: 25}) - with self.assertRaises(errors.OutOfRangeError): - sess.run(get_next) - - # Skip exactly input size. - sess.run(init_op, feed_dict={count_placeholder: 10}) - with self.assertRaises(errors.OutOfRangeError): - sess.run(get_next) - - # Set -1 for 'count': skip the entire dataset. - sess.run(init_op, feed_dict={count_placeholder: -1}) - with self.assertRaises(errors.OutOfRangeError): - sess.run(get_next) - - # Skip nothing - sess.run(init_op, feed_dict={count_placeholder: 0}) - for i in range(0, 10): - results = sess.run(get_next) - self.assertAllEqual(results, components[0][i:i+1]) - with self.assertRaises(errors.OutOfRangeError): - sess.run(get_next) - - def testRepeatRepeatTensorDataset(self): - """Test the composition of repeat datasets.""" - components = (np.array(1), np.array([1, 2, 3]), np.array(37.0)) - inner_count = array_ops.placeholder(dtypes.int64, shape=[]) - outer_count = array_ops.placeholder(dtypes.int64, shape=[]) - - iterator = (dataset_ops.Dataset.from_tensors(components).repeat(inner_count) - .repeat(outer_count).make_initializable_iterator()) - init_op = iterator.initializer - get_next = iterator.get_next() - - self.assertEqual([c.shape for c in components], - [t.shape for t in get_next]) - - with self.cached_session() as sess: - sess.run(init_op, feed_dict={inner_count: 7, outer_count: 14}) - for _ in range(7 * 14): - results = sess.run(get_next) - for component, result_component in zip(components, results): - self.assertAllEqual(component, result_component) - with self.assertRaises(errors.OutOfRangeError): - sess.run(get_next) - - def testRepeatEmptyDataset(self): - """Test that repeating an empty dataset does not hang.""" - iterator = (dataset_ops.Dataset.from_tensors(0).repeat(10).skip(10) - .repeat(-1).make_initializable_iterator()) - init_op = iterator.initializer - get_next = iterator.get_next() - - with self.cached_session() as sess: - sess.run(init_op) - with self.assertRaises(errors.OutOfRangeError): - sess.run(get_next) - - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/python/data/kernel_tests/shard_dataset_op_test.py b/tensorflow/python/data/kernel_tests/shard_test.py similarity index 52% rename from tensorflow/python/data/kernel_tests/shard_dataset_op_test.py rename to tensorflow/python/data/kernel_tests/shard_test.py index b9f3c79da56..928550676d5 100644 --- a/tensorflow/python/data/kernel_tests/shard_dataset_op_test.py +++ b/tensorflow/python/data/kernel_tests/shard_test.py @@ -12,50 +12,33 @@ # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================== -"""Tests for the experimental input pipeline ops.""" +"""Tests for `tf.data.Dataset.shard()`.""" from __future__ import absolute_import from __future__ import division from __future__ import print_function from tensorflow.python.data.kernel_tests import test_base from tensorflow.python.data.ops import dataset_ops -from tensorflow.python.framework import errors +from tensorflow.python.framework import test_util from tensorflow.python.platform import test -class ShardDatasetOpTest(test_base.DatasetTestBase): +@test_util.run_all_in_graph_and_eager_modes +class ShardTest(test_base.DatasetTestBase): def testSimpleCase(self): dataset = dataset_ops.Dataset.range(10).shard(5, 2) - iterator = dataset.make_one_shot_iterator() - - with self.cached_session() as sess: - self.assertEqual(2, sess.run(iterator.get_next())) - self.assertEqual(7, sess.run(iterator.get_next())) - with self.assertRaises(errors.OutOfRangeError): - sess.run(iterator.get_next()) + self.assertDatasetProduces(dataset, expected_output=[2, 7]) def testNestedData(self): dataset_a = dataset_ops.Dataset.range(10) dataset_b = dataset_ops.Dataset.range(10, 0, -1) dataset = dataset_ops.Dataset.zip((dataset_a, dataset_b)).shard(5, 2) - iterator = dataset.make_one_shot_iterator() - - with self.cached_session() as sess: - self.assertEqual((2, 8), sess.run(iterator.get_next())) - self.assertEqual((7, 3), sess.run(iterator.get_next())) - with self.assertRaises(errors.OutOfRangeError): - sess.run(iterator.get_next()) + self.assertDatasetProduces(dataset, expected_output=[(2, 8), (7, 3)]) def testOffsetZero(self): dataset = dataset_ops.Dataset.range(10).shard(5, 0) - iterator = dataset.make_one_shot_iterator() - - with self.cached_session() as sess: - self.assertEqual(0, sess.run(iterator.get_next())) - self.assertEqual(5, sess.run(iterator.get_next())) - with self.assertRaises(errors.OutOfRangeError): - sess.run(iterator.get_next()) + self.assertDatasetProduces(dataset, expected_output=[0, 5]) def testOffsetGreaterNumShards(self): with self.assertRaises(ValueError): @@ -75,38 +58,19 @@ class ShardDatasetOpTest(test_base.DatasetTestBase): def testIteratorEndsBeforeFirstElem(self): dataset = dataset_ops.Dataset.range(1).shard(5, 2) - iterator = dataset.make_one_shot_iterator() - - with self.cached_session() as sess: - with self.assertRaises(errors.OutOfRangeError): - sess.run(iterator.get_next()) + self.assertDatasetProduces(dataset, expected_output=[]) def testLargerWorkerPool(self): dataset = dataset_ops.Dataset.range(10).shard(7, 5) - iterator = dataset.make_one_shot_iterator() - with self.cached_session() as sess: - self.assertEqual(5, sess.run(iterator.get_next())) - with self.assertRaises(errors.OutOfRangeError): - sess.run(iterator.get_next()) + self.assertDatasetProduces(dataset, expected_output=[5]) def testIndexEqualsNumShards(self): dataset = dataset_ops.Dataset.range(10).shard(5, 4) - iterator = dataset.make_one_shot_iterator() - with self.cached_session() as sess: - self.assertEqual(4, sess.run(iterator.get_next())) - self.assertEqual(9, sess.run(iterator.get_next())) - with self.assertRaises(errors.OutOfRangeError): - sess.run(iterator.get_next()) + self.assertDatasetProduces(dataset, expected_output=[4, 9]) def testIndexEqualsNumShards2(self): dataset = dataset_ops.Dataset.range(10).shard(4, 3) - iterator = dataset.make_one_shot_iterator() - with self.cached_session() as sess: - self.assertEqual(3, sess.run(iterator.get_next())) - self.assertEqual(7, sess.run(iterator.get_next())) - with self.assertRaises(errors.OutOfRangeError): - sess.run(iterator.get_next()) - + self.assertDatasetProduces(dataset, expected_output=[3, 7]) if __name__ == "__main__": test.main() diff --git a/tensorflow/python/data/kernel_tests/shuffle_dataset_op_test.py b/tensorflow/python/data/kernel_tests/shuffle_dataset_op_test.py deleted file mode 100644 index cad28f860e9..00000000000 --- a/tensorflow/python/data/kernel_tests/shuffle_dataset_op_test.py +++ /dev/null @@ -1,278 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for the experimental input pipeline ops.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import collections - -from absl.testing import parameterized -import numpy as np - -from tensorflow.python.data.kernel_tests import test_base -from tensorflow.python.data.ops import dataset_ops -from tensorflow.python.data.ops import iterator_ops -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import errors -from tensorflow.python.framework import ops -from tensorflow.python.framework import random_seed -from tensorflow.python.ops import array_ops -from tensorflow.python.platform import test - - -class ShuffleDatasetTest(test_base.DatasetTestBase, parameterized.TestCase): - - def testShuffleDataset(self): - components = ( - np.array([1, 2, 3, 4]), np.array([5, 6, 7, 8]), - np.array([9.0, 10.0, 11.0, 12.0]) - ) - count_placeholder = array_ops.placeholder_with_default( - constant_op.constant(5, dtypes.int64), shape=[]) - buffer_size_placeholder = array_ops.placeholder(dtypes.int64, shape=[]) - seed_placeholder = array_ops.placeholder(dtypes.int64, shape=[]) - - repeat_dataset = (dataset_ops.Dataset.from_tensor_slices(components) - .repeat(count_placeholder)) - - shuffle_dataset = repeat_dataset.shuffle(buffer_size_placeholder, - seed_placeholder) - - self.assertEqual(tuple([c.shape[1:] for c in components]), - shuffle_dataset.output_shapes) - - # Create initialization ops for iterators without and with - # shuffling, respectively. - iterator = iterator_ops.Iterator.from_structure( - shuffle_dataset.output_types, shuffle_dataset.output_shapes) - init_fifo_op = iterator.make_initializer(repeat_dataset) - init_shuffle_op = iterator.make_initializer(shuffle_dataset) - - get_next = iterator.get_next() - - with self.cached_session() as sess: - # First run without shuffling to collect the "ground truth". - sess.run(init_fifo_op) - unshuffled_elements = [] - for _ in range(20): - unshuffled_elements.append(sess.run(get_next)) - with self.assertRaises(errors.OutOfRangeError): - sess.run(get_next) - - # Assert that the shuffled dataset has the same elements as the - # "ground truth". - sess.run( - init_shuffle_op, - feed_dict={buffer_size_placeholder: 100, - seed_placeholder: 37}) - shuffled_elements = [] - for _ in range(20): - shuffled_elements.append(sess.run(get_next)) - with self.assertRaises(errors.OutOfRangeError): - sess.run(get_next) - self.assertAllEqual( - sorted(unshuffled_elements), sorted(shuffled_elements)) - - # Assert that shuffling twice with the same seeds gives the same sequence. - sess.run( - init_shuffle_op, - feed_dict={buffer_size_placeholder: 100, - seed_placeholder: 37}) - reshuffled_elements_same_seed = [] - for _ in range(20): - reshuffled_elements_same_seed.append(sess.run(get_next)) - with self.assertRaises(errors.OutOfRangeError): - sess.run(get_next) - self.assertEqual(shuffled_elements, reshuffled_elements_same_seed) - - # Assert that shuffling twice with a different seed gives a different - # permutation of the same elements. - sess.run( - init_shuffle_op, - feed_dict={buffer_size_placeholder: 100, - seed_placeholder: 1037}) - reshuffled_elements_different_seed = [] - for _ in range(20): - reshuffled_elements_different_seed.append(sess.run(get_next)) - with self.assertRaises(errors.OutOfRangeError): - sess.run(get_next) - self.assertNotEqual(shuffled_elements, reshuffled_elements_different_seed) - self.assertAllEqual( - sorted(shuffled_elements), sorted(reshuffled_elements_different_seed)) - - # Assert that the shuffled dataset has the same elements as the - # "ground truth" when the buffer size is smaller than the input - # dataset. - sess.run( - init_shuffle_op, - feed_dict={buffer_size_placeholder: 2, - seed_placeholder: 37}) - reshuffled_elements_small_buffer = [] - for _ in range(20): - reshuffled_elements_small_buffer.append(sess.run(get_next)) - with self.assertRaises(errors.OutOfRangeError): - sess.run(get_next) - self.assertAllEqual( - sorted(unshuffled_elements), sorted(reshuffled_elements_small_buffer)) - - # Test the case of shuffling an empty dataset. - sess.run(init_shuffle_op, feed_dict={buffer_size_placeholder: 2, - seed_placeholder: 37, - count_placeholder: 0}) - with self.assertRaises(errors.OutOfRangeError): - sess.run(get_next) - - def testSeedZero(self): - """Test for same behavior when the seed is a Python or Tensor zero.""" - iterator = ( - dataset_ops.Dataset.range(10).shuffle(10, seed=0) - .make_one_shot_iterator()) - get_next = iterator.get_next() - - elems = [] - with self.cached_session() as sess: - for _ in range(10): - elems.append(sess.run(get_next)) - with self.assertRaises(errors.OutOfRangeError): - sess.run(get_next) - - seed_placeholder = array_ops.placeholder(dtypes.int64, shape=[]) - iterator = ( - dataset_ops.Dataset.range(10).shuffle(10, seed=seed_placeholder) - .make_initializable_iterator()) - get_next = iterator.get_next() - - with self.cached_session() as sess: - sess.run(iterator.initializer, feed_dict={seed_placeholder: 0}) - for elem in elems: - self.assertEqual(elem, sess.run(get_next)) - with self.assertRaises(errors.OutOfRangeError): - sess.run(get_next) - - def testDefaultArguments(self): - components = [0, 1, 2, 3, 4] - iterator = (dataset_ops.Dataset.from_tensor_slices(components).shuffle(5) - .repeat().make_one_shot_iterator()) - - get_next = iterator.get_next() - - with self.cached_session() as sess: - counts = collections.defaultdict(lambda: 0) - for _ in range(10): - for _ in range(5): - counts[sess.run(get_next)] += 1 - - for i in range(5): - self.assertEqual(10, counts[i]) - - def testShuffleNoReshuffleEachIteration(self): - iterator = (dataset_ops.Dataset.range(10) - .shuffle(10, reshuffle_each_iteration=False) - .batch(10) - .repeat(3) - .make_one_shot_iterator()) - next_element = iterator.get_next() - - with self.cached_session() as sess: - initial_permutation = sess.run(next_element) - self.assertAllEqual(initial_permutation, sess.run(next_element)) - self.assertAllEqual(initial_permutation, sess.run(next_element)) - with self.assertRaises(errors.OutOfRangeError): - sess.run(next_element) - - def testShuffleReshuffleEachIteration(self): - iterator = (dataset_ops.Dataset.range(10) - .shuffle(10, seed=3, reshuffle_each_iteration=True) - .batch(10) - .repeat(3) - .make_one_shot_iterator()) - next_element = iterator.get_next() - - with self.cached_session() as sess: - initial_permutation = list(sess.run(next_element)) - for _ in range(2): - next_permutation = list(sess.run(next_element)) - self.assertNotEqual(initial_permutation, next_permutation) - self.assertAllEqual( - sorted(initial_permutation), sorted(next_permutation)) - with self.assertRaises(errors.OutOfRangeError): - sess.run(next_element) - - @parameterized.named_parameters( - ("ReshuffleGraphLevelSeed", True, 38, None), - ("ReshuffleOpLevelSeed", True, None, 42), - ("ReshuffleGraphAndOpLevelSeed", True, 38, 42), - ("NoReshuffleGraphLevelSeed", False, 38, None), - ("NoReshuffleOpLevelSeed", False, None, 42), - ("NoReshuffleGraphAndOpLevelSeed", False, 38, 42), - ) - def testShuffleSeed(self, reshuffle, graph_level_seed, op_level_seed): - results = [] - for _ in range(2): - with ops.Graph().as_default() as g: - random_seed.set_random_seed(graph_level_seed) - dataset = dataset_ops.Dataset.range(10).shuffle( - 10, seed=op_level_seed, reshuffle_each_iteration=reshuffle).repeat( - 3) - iterator = dataset.make_one_shot_iterator() - next_element = iterator.get_next() - - run_results = [] - with self.session(graph=g) as sess: - for _ in range(30): - run_results.append(sess.run(next_element)) - with self.assertRaises(errors.OutOfRangeError): - sess.run(next_element) - results.append(run_results) - - self.assertAllEqual(results[0], results[1]) - - @parameterized.named_parameters( - ("ReshuffleOneShot", True, False), - ("ReshuffleInitializable", True, True), - ("NoReshuffleOneShot", False, False), - ("NoReshuffleInitializable", False, True), - ) - def testMultipleIterators(self, reshuffle, initializable): - with ops.Graph().as_default() as g: - dataset = dataset_ops.Dataset.range(100).shuffle( - 10, reshuffle_each_iteration=reshuffle).repeat(3) - - if initializable: - iterators = [dataset.make_initializable_iterator() for _ in range(2)] - else: - iterators = [dataset.make_one_shot_iterator() for _ in range(2)] - - results = [] - with self.session(graph=g) as sess: - for iterator in iterators: - if initializable: - sess.run(iterator.initializer) - next_element = iterator.get_next() - run_results = [] - for _ in range(300): - run_results.append(sess.run(next_element)) - with self.assertRaises(errors.OutOfRangeError): - sess.run(next_element) - - results.append(run_results) - - self.assertNotEqual(results[0], results[1]) - - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/python/data/kernel_tests/shuffle_test.py b/tensorflow/python/data/kernel_tests/shuffle_test.py new file mode 100644 index 00000000000..13df870938d --- /dev/null +++ b/tensorflow/python/data/kernel_tests/shuffle_test.py @@ -0,0 +1,249 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Tests for `tf.data.Dataset.shuffle()`.""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import collections + +from absl.testing import parameterized +import numpy as np + +from tensorflow.python.data.kernel_tests import test_base +from tensorflow.python.data.ops import dataset_ops + +from tensorflow.python.framework import dtypes +from tensorflow.python.framework import errors +from tensorflow.python.framework import ops +from tensorflow.python.framework import random_seed +from tensorflow.python.framework import test_util +from tensorflow.python.ops import array_ops +from tensorflow.python.platform import test + + +@test_util.run_all_in_graph_and_eager_modes +class ShuffleTest(test_base.DatasetTestBase, parameterized.TestCase): + + def testShuffleDataset(self): + components = ( + np.array([1, 2, 3, 4]), np.array([5, 6, 7, 8]), + np.array([9.0, 10.0, 11.0, 12.0]) + ) + + def dataset_fn(count=5, buffer_size=None, seed=0): + repeat_dataset = ( + dataset_ops.Dataset.from_tensor_slices(components).repeat(count)) + if buffer_size: + shuffle_dataset = repeat_dataset.shuffle(buffer_size, seed) + + self.assertEqual( + tuple([c.shape[1:] for c in components]), + shuffle_dataset.output_shapes) + return shuffle_dataset + else: + return repeat_dataset + + # First run without shuffling to collect the "ground truth". + get_next = self.getNext(dataset_fn()) + unshuffled_elements = [] + for _ in range(20): + unshuffled_elements.append(self.evaluate(get_next())) + with self.assertRaises(errors.OutOfRangeError): + self.evaluate(get_next()) + + # Assert that the shuffled dataset has the same elements as the + # "ground truth". + get_next = self.getNext(dataset_fn(buffer_size=100, seed=37)) + shuffled_elements = [] + for _ in range(20): + shuffled_elements.append(self.evaluate(get_next())) + with self.assertRaises(errors.OutOfRangeError): + self.evaluate(get_next()) + with self.assertRaises(errors.OutOfRangeError): + self.evaluate(get_next()) + self.assertAllEqual(sorted(unshuffled_elements), sorted(shuffled_elements)) + + # Assert that shuffling twice with the same seeds gives the same sequence. + get_next = self.getNext(dataset_fn(buffer_size=100, seed=37)) + reshuffled_elements_same_seed = [] + for _ in range(20): + reshuffled_elements_same_seed.append(self.evaluate(get_next())) + with self.assertRaises(errors.OutOfRangeError): + self.evaluate(get_next()) + self.assertEqual(shuffled_elements, reshuffled_elements_same_seed) + + # Assert that shuffling twice with a different seed gives a different + # permutation of the same elements. + get_next = self.getNext(dataset_fn(buffer_size=100, seed=137)) + reshuffled_elements_different_seed = [] + for _ in range(20): + reshuffled_elements_different_seed.append(self.evaluate(get_next())) + with self.assertRaises(errors.OutOfRangeError): + self.evaluate(get_next()) + self.assertNotEqual(shuffled_elements, reshuffled_elements_different_seed) + self.assertAllEqual( + sorted(shuffled_elements), sorted(reshuffled_elements_different_seed)) + + # Assert that the shuffled dataset has the same elements as the + # "ground truth" when the buffer size is smaller than the input + # dataset. + get_next = self.getNext(dataset_fn(buffer_size=2, seed=37)) + reshuffled_elements_small_buffer = [] + for _ in range(20): + reshuffled_elements_small_buffer.append(self.evaluate(get_next())) + with self.assertRaises(errors.OutOfRangeError): + self.evaluate(get_next()) + self.assertAllEqual( + sorted(unshuffled_elements), sorted(reshuffled_elements_small_buffer)) + + # Test the case of shuffling an empty dataset. + get_next = self.getNext(dataset_fn(count=0, buffer_size=100, seed=37)) + + with self.assertRaises(errors.OutOfRangeError): + self.evaluate(get_next()) + + @test_util.run_deprecated_v1 + def testSkipEagerSeedZero(self): + """Test for same behavior when the seed is a Python or Tensor zero.""" + iterator = dataset_ops.make_one_shot_iterator( + dataset_ops.Dataset.range(10).shuffle(10, seed=0)) + get_next = iterator.get_next() + + elems = [] + with self.cached_session() as sess: + for _ in range(10): + elems.append(sess.run(get_next)) + with self.assertRaises(errors.OutOfRangeError): + sess.run(get_next) + + seed_placeholder = array_ops.placeholder(dtypes.int64, shape=[]) + iterator = dataset_ops.make_initializable_iterator( + dataset_ops.Dataset.range(10).shuffle(10, seed=seed_placeholder)) + get_next = iterator.get_next() + + with self.cached_session() as sess: + sess.run(iterator.initializer, feed_dict={seed_placeholder: 0}) + for elem in elems: + self.assertEqual(elem, sess.run(get_next)) + with self.assertRaises(errors.OutOfRangeError): + sess.run(get_next) + + def testDefaultArguments(self): + components = [0, 1, 2, 3, 4] + dataset = dataset_ops.Dataset.from_tensor_slices(components).shuffle( + 5).repeat() + get_next = self.getNext(dataset) + counts = collections.defaultdict(lambda: 0) + for _ in range(10): + for _ in range(5): + counts[self.evaluate(get_next())] += 1 + + for i in range(5): + self.assertEqual(10, counts[i]) + + def testShuffleNoReshuffleEachIteration(self): + dataset = dataset_ops.Dataset.range(10).shuffle( + 10, reshuffle_each_iteration=False).batch(10).repeat(3) + next_element = self.getNext(dataset) + + initial_permutation = self.evaluate(next_element()) + self.assertAllEqual(initial_permutation, self.evaluate(next_element())) + self.assertAllEqual(initial_permutation, self.evaluate(next_element())) + with self.assertRaises(errors.OutOfRangeError): + self.evaluate(next_element()) + + def testShuffleReshuffleEachIteration(self): + dataset = dataset_ops.Dataset.range(10).shuffle( + 10, seed=3, reshuffle_each_iteration=True).batch(10).repeat(3) + next_element = self.getNext(dataset) + + initial_permutation = list(self.evaluate(next_element())) + for _ in range(2): + next_permutation = list(self.evaluate(next_element())) + self.assertNotEqual(initial_permutation, next_permutation) + self.assertAllEqual(sorted(initial_permutation), sorted(next_permutation)) + with self.assertRaises(errors.OutOfRangeError): + self.evaluate(next_element()) + + @parameterized.named_parameters( + ("ReshuffleGraphLevelSeed", True, 38, None), + ("ReshuffleOpLevelSeed", True, None, 42), + ("ReshuffleGraphAndOpLevelSeed", True, 38, 42), + ("NoReshuffleGraphLevelSeed", False, 38, None), + ("NoReshuffleOpLevelSeed", False, None, 42), + ("NoReshuffleGraphAndOpLevelSeed", False, 38, 42), + ) + def testSkipEagerShuffleSeed(self, reshuffle, graph_level_seed, + op_level_seed): + results = [] + for _ in range(2): + with ops.Graph().as_default() as g: + random_seed.set_random_seed(graph_level_seed) + dataset = dataset_ops.Dataset.range(10).shuffle( + 10, seed=op_level_seed, reshuffle_each_iteration=reshuffle).repeat( + 3) + iterator = dataset_ops.make_one_shot_iterator(dataset) + next_element = iterator.get_next() + + run_results = [] + with self.session(graph=g) as sess: + for _ in range(30): + run_results.append(sess.run(next_element)) + with self.assertRaises(errors.OutOfRangeError): + sess.run(next_element) + results.append(run_results) + + self.assertAllEqual(results[0], results[1]) + + # TODO(b/117581999): fails for eager mode with result[0] equal to result[1], + # debug. + @parameterized.named_parameters( + ("ReshuffleOneShot", True, False), + ("ReshuffleInitializable", True, True), + ("NoReshuffleOneShot", False, False), + ("NoReshuffleInitializable", False, True), + ) + def testSkipEagerMultipleIterators(self, reshuffle, initializable): + with ops.Graph().as_default() as g: + dataset = dataset_ops.Dataset.range(100).shuffle( + 10, reshuffle_each_iteration=reshuffle).repeat(3) + + if initializable: + iterators = [dataset_ops.make_initializable_iterator(dataset) + for _ in range(2)] + else: + iterators = [dataset_ops.make_one_shot_iterator(dataset) + for _ in range(2)] + + results = [] + with self.session(graph=g) as sess: + for iterator in iterators: + if initializable: + sess.run(iterator.initializer) + next_element = iterator.get_next() + run_results = [] + for _ in range(300): + run_results.append(sess.run(next_element)) + with self.assertRaises(errors.OutOfRangeError): + sess.run(next_element) + + results.append(run_results) + + self.assertNotEqual(results[0], results[1]) + + +if __name__ == "__main__": + test.main() diff --git a/tensorflow/python/data/kernel_tests/skip_test.py b/tensorflow/python/data/kernel_tests/skip_test.py new file mode 100644 index 00000000000..c22be576921 --- /dev/null +++ b/tensorflow/python/data/kernel_tests/skip_test.py @@ -0,0 +1,62 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Tests for `tf.data.Dataset.skip()`.""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import numpy as np + +from tensorflow.python.data.kernel_tests import test_base +from tensorflow.python.data.ops import dataset_ops +from tensorflow.python.framework import test_util +from tensorflow.python.platform import test + + +@test_util.run_all_in_graph_and_eager_modes +class SkipTest(test_base.DatasetTestBase): + + def testSkipTensorDataset(self): + components = (np.arange(10),) + + def do_test(count): + dataset = dataset_ops.Dataset.from_tensor_slices(components).skip(count) + self.assertEqual([c.shape[1:] for c in components], + [shape for shape in dataset.output_shapes]) + start_range = min(count, 10) if count != -1 else 10 + self.assertDatasetProduces( + dataset, + [tuple(components[0][i:i + 1]) for i in range(start_range, 10)]) + + # Skip fewer than input size, we should skip + # the first 4 elements and then read the rest. + do_test(4) + + # Skip more than input size: get nothing. + do_test(25) + + # Skip exactly input size. + do_test(10) + + # Set -1 for 'count': skip the entire dataset. + do_test(-1) + + # Skip nothing + do_test(0) + + + +if __name__ == "__main__": + test.main() diff --git a/tensorflow/python/data/kernel_tests/take_test.py b/tensorflow/python/data/kernel_tests/take_test.py new file mode 100644 index 00000000000..03a7ece2d8c --- /dev/null +++ b/tensorflow/python/data/kernel_tests/take_test.py @@ -0,0 +1,55 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Tests for `tf.data.Dataset.take()`.""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import numpy as np + +from tensorflow.python.data.kernel_tests import test_base +from tensorflow.python.data.ops import dataset_ops +from tensorflow.python.framework import test_util +from tensorflow.python.platform import test + + +@test_util.run_all_in_graph_and_eager_modes +class TakeTest(test_base.DatasetTestBase): + + def testTakeTensorDataset(self): + components = (np.arange(10),) + + def do_test(count): + dataset = dataset_ops.Dataset.from_tensor_slices(components).take(count) + self.assertEqual([c.shape[1:] for c in components], + [shape for shape in dataset.output_shapes]) + num_output = min(count, 10) if count != -1 else 10 + self.assertDatasetProduces( + dataset, [tuple(components[0][i:i + 1]) for i in range(num_output)]) + + # Take fewer than input size + do_test(4) + + # Take more than input size + do_test(25) + + # Take all of input + do_test(-1) + + # Take nothing + do_test(0) + +if __name__ == "__main__": + test.main() diff --git a/tensorflow/python/data/kernel_tests/test_base.py b/tensorflow/python/data/kernel_tests/test_base.py index edb3eff3c17..85f6c9de231 100644 --- a/tensorflow/python/data/kernel_tests/test_base.py +++ b/tensorflow/python/data/kernel_tests/test_base.py @@ -38,56 +38,102 @@ class DatasetTestBase(test.TestCase): self.assertAllEqual(a.values, b.values) self.assertAllEqual(a.dense_shape, b.dense_shape) - def getNext(self, dataset): + def getNext(self, dataset, requires_initialization=False): """Returns a callable that returns the next element of the dataset. Example use: ```python # In both graph and eager modes dataset = ... - nxt = self.getNext(dataset) - result = self.evaluate(nxt()) + get_next = self.getNext(dataset) + result = self.evaluate(get_next()) ``` Args: - dataset: A dataset whose next element is returned - + dataset: A dataset whose elements will be returned. + requires_initialization: Indicates that when the test is executed in graph + mode, it should use an initializable iterator to iterate through the + dataset (e.g. when it contains stateful nodes). Defaults to False. Returns: - A callable that returns the next element of `dataset` + A callable that returns the next element of `dataset`. """ - it = dataset.make_one_shot_iterator() if context.executing_eagerly(): - return it.get_next + iterator = dataset.__iter__() + return iterator._next_internal # pylint: disable=protected-access else: - nxt = it.get_next() - return lambda: nxt - - def _compare_output_to_expected(self, result_values, expected_values): - for i in range(len(result_values)): - if sparse_tensor.is_sparse(result_values[i]): - self.assertSparseValuesEqual(result_values[i], expected_values[i]) + if requires_initialization: + iterator = dataset_ops.make_initializable_iterator(dataset) + self.evaluate(iterator.initializer) else: - self.assertAllEqual(result_values[i], expected_values[i]) + iterator = dataset_ops.make_one_shot_iterator(dataset) + get_next = iterator.get_next() + return lambda: get_next + + def _compareOutputToExpected(self, result_values, expected_values, + assert_items_equal): + if assert_items_equal: + # TODO(shivaniagrawal): add support for nested elements containing sparse + # tensors when needed. + self.assertItemsEqual(result_values, expected_values) + return + for i in range(len(result_values)): + nest.assert_same_structure(result_values[i], expected_values[i]) + for result_value, expected_value in zip( + nest.flatten(result_values[i]), nest.flatten(expected_values[i])): + if sparse_tensor.is_sparse(result_value): + self.assertSparseValuesEqual(result_value, expected_value) + else: + self.assertAllEqual(result_value, expected_value) def assertDatasetProduces(self, - input_dataset, + dataset, expected_output=None, - expected_err=None, - create_iterator_twice=True): + expected_error=None, + requires_initialization=False, + num_test_iterations=1, + assert_items_equal=False): + """Asserts that a dataset produces the expected output / error. - if expected_err: - with self.assertRaisesWithPredicateMatch(expected_err[0], - expected_err[1]): - get_next = self.getNext(input_dataset) + Args: + dataset: A dataset to check for the expected output / error. + expected_output: A list of elements that the dataset is expected to + produce. + expected_error: A tuple `(type, predicate)` identifying the expected error + `dataset` should raise. The `type` should match the expected exception + type, while `predicate` should either be 1) a unary function that inputs + the raised exception and returns a boolean indicator of success or 2) a + regular expression that is expected to match the error message + partially. + requires_initialization: Indicates that when the test is executed in graph + mode, it should use an initializable iterator to iterate through the + dataset (e.g. when it contains stateful nodes). Defaults to False. + num_test_iterations: Number of times `dataset` will be iterated. Defaults + to 2. + assert_items_equal: Tests expected_output has (only) the same elements + regardless of order. + """ + self.assertTrue( + expected_error is not None or expected_output is not None, + "Exactly one of expected_output or expected error should be provided.") + if expected_error: + self.assertTrue( + expected_output is None, + "Exactly one of expected_output or expected error should be provided." + ) + with self.assertRaisesWithPredicateMatch(expected_error[0], + expected_error[1]): + get_next = self.getNext( + dataset, requires_initialization=requires_initialization) self.evaluate(get_next()) return - repeated = 2 if create_iterator_twice else 1 - for _ in range(repeated): - get_next = self.getNext(input_dataset) + self.assertGreater(num_test_iterations, 0) + for _ in range(num_test_iterations): + get_next = self.getNext( + dataset, requires_initialization=requires_initialization) result = [] for _ in range(len(expected_output)): result.append(self.evaluate(get_next())) - self._compare_output_to_expected(result, expected_output) + self._compareOutputToExpected(result, expected_output, assert_items_equal) with self.assertRaises(errors.OutOfRangeError): self.evaluate(get_next()) with self.assertRaises(errors.OutOfRangeError): @@ -132,7 +178,7 @@ class DatasetTestBase(test.TestCase): try: self.evaluate(next1()) raise ValueError( - 'Expected dataset to raise an error of type %s, but it did not.' % + "Expected dataset to raise an error of type %s, but it did not." % repr(exception_class)) except exception_class as e: expected_message = e.message diff --git a/tensorflow/python/data/kernel_tests/text_line_dataset_test.py b/tensorflow/python/data/kernel_tests/text_line_dataset_test.py new file mode 100644 index 00000000000..4db09a98084 --- /dev/null +++ b/tensorflow/python/data/kernel_tests/text_line_dataset_test.py @@ -0,0 +1,165 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Tests for `tf.data.TextLineDataset`.""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import gzip +import os +import zlib + +from tensorflow.python.data.kernel_tests import test_base +from tensorflow.python.data.ops import readers +from tensorflow.python.eager import context +from tensorflow.python.framework import test_util +from tensorflow.python.platform import test +from tensorflow.python.util import compat + + +try: + import psutil # pylint: disable=g-import-not-at-top + psutil_import_succeeded = True +except ImportError: + psutil_import_succeeded = False + + +@test_util.run_all_in_graph_and_eager_modes +class TextLineDatasetTest(test_base.DatasetTestBase): + + def _lineText(self, f, l): + return compat.as_bytes("%d: %d" % (f, l)) + + def _createFiles(self, + num_files, + num_lines, + crlf=False, + compression_type=None): + filenames = [] + for i in range(num_files): + fn = os.path.join(self.get_temp_dir(), "text_line.%d.txt" % i) + filenames.append(fn) + contents = [] + for j in range(num_lines): + contents.append(self._lineText(i, j)) + # Always include a newline after the record unless it is + # at the end of the file, in which case we include it + if j + 1 != num_lines or i == 0: + contents.append(b"\r\n" if crlf else b"\n") + contents = b"".join(contents) + + if not compression_type: + with open(fn, "wb") as f: + f.write(contents) + elif compression_type == "GZIP": + with gzip.GzipFile(fn, "wb") as f: + f.write(contents) + elif compression_type == "ZLIB": + contents = zlib.compress(contents) + with open(fn, "wb") as f: + f.write(contents) + else: + raise ValueError("Unsupported compression_type", compression_type) + + return filenames + + def _testTextLineDataset(self, compression_type=None): + test_filenames = self._createFiles( + 2, 5, crlf=True, compression_type=compression_type) + + def dataset_fn(filenames, num_epochs, batch_size=None): + repeat_dataset = readers.TextLineDataset( + filenames, compression_type=compression_type).repeat(num_epochs) + if batch_size: + return repeat_dataset.batch(batch_size) + return repeat_dataset + + # Basic test: read from file 0. + expected_output = [self._lineText(0, i) for i in range(5)] + self.assertDatasetProduces( + dataset_fn([test_filenames[0]], 1), expected_output=expected_output) + + # Basic test: read from file 1. + self.assertDatasetProduces( + dataset_fn([test_filenames[1]], 1), + expected_output=[self._lineText(1, i) for i in range(5)]) + + # Basic test: read from both files. + expected_output = [self._lineText(0, i) for i in range(5)] + expected_output.extend([self._lineText(1, i) for i in range(5)]) + self.assertDatasetProduces( + dataset_fn(test_filenames, 1), expected_output=expected_output) + + # Test repeated iteration through both files. + expected_output = [self._lineText(0, i) for i in range(5)] + expected_output.extend([self._lineText(1, i) for i in range(5)]) + self.assertDatasetProduces( + dataset_fn(test_filenames, 10), expected_output=expected_output * 10) + + # Test batched and repeated iteration through both files. + self.assertDatasetProduces( + dataset_fn(test_filenames, 10, 5), + expected_output=[[self._lineText(0, i) for i in range(5)], + [self._lineText(1, i) for i in range(5)]] * 10) + + def testTextLineDatasetNoCompression(self): + self._testTextLineDataset() + + def testTextLineDatasetGzipCompression(self): + self._testTextLineDataset(compression_type="GZIP") + + def testTextLineDatasetZlibCompression(self): + self._testTextLineDataset(compression_type="ZLIB") + + def testTextLineDatasetBuffering(self): + test_filenames = self._createFiles(2, 5, crlf=True) + + repeat_dataset = readers.TextLineDataset(test_filenames, buffer_size=10) + expected_output = [] + for j in range(2): + expected_output.extend([self._lineText(j, i) for i in range(5)]) + self.assertDatasetProduces(repeat_dataset, expected_output=expected_output) + + def testIteratorResourceCleanup(self): + filename = os.path.join(self.get_temp_dir(), "text.txt") + with open(filename, "wt") as f: + for i in range(3): + f.write("%d\n" % (i,)) + with context.eager_mode(): + first_iterator = iter(readers.TextLineDataset(filename)) + self.assertEqual(b"0", next(first_iterator).numpy()) + second_iterator = iter(readers.TextLineDataset(filename)) + self.assertEqual(b"0", next(second_iterator).numpy()) + # Eager kernel caching is based on op attributes, which includes the + # Dataset's output shape. Create a different kernel to test that they + # don't create resources with the same names. + different_kernel_iterator = iter( + readers.TextLineDataset(filename).repeat().batch(16)) + self.assertEqual([16], next(different_kernel_iterator).shape) + # Remove our references to the Python Iterator objects, which (assuming no + # reference cycles) is enough to trigger DestroyResourceOp and close the + # partially-read files. + del first_iterator + del second_iterator + del different_kernel_iterator + if not psutil_import_succeeded: + self.skipTest( + "psutil is required to check that we've closed our files.") + open_files = psutil.Process().open_files() + self.assertNotIn(filename, [open_file.path for open_file in open_files]) + + +if __name__ == "__main__": + test.main() diff --git a/tensorflow/python/data/kernel_tests/tf_record_dataset_test.py b/tensorflow/python/data/kernel_tests/tf_record_dataset_test.py new file mode 100644 index 00000000000..13a70aa88d0 --- /dev/null +++ b/tensorflow/python/data/kernel_tests/tf_record_dataset_test.py @@ -0,0 +1,170 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Tests for `tf.data.TFRecordDataset`.""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import gzip +import os +import zlib + +from tensorflow.python.data.kernel_tests import test_base +from tensorflow.python.data.ops import dataset_ops +from tensorflow.python.data.ops import readers +from tensorflow.python.framework import test_util +from tensorflow.python.lib.io import python_io +from tensorflow.python.platform import test +from tensorflow.python.util import compat + + +@test_util.run_all_in_graph_and_eager_modes +class TFRecordDatasetTest(test_base.DatasetTestBase): + + def setUp(self): + super(TFRecordDatasetTest, self).setUp() + self._num_files = 2 + self._num_records = 7 + + self.test_filenames = self._createFiles() + + def dataset_fn(self, + filenames, + compression_type="", + num_epochs=1, + batch_size=None): + + repeat_dataset = readers.TFRecordDataset( + filenames, compression_type).repeat(num_epochs) + if batch_size: + return repeat_dataset.batch(batch_size) + return repeat_dataset + + def _record(self, f, r): + return compat.as_bytes("Record %d of file %d" % (r, f)) + + def _createFiles(self): + filenames = [] + for i in range(self._num_files): + fn = os.path.join(self.get_temp_dir(), "tf_record.%d.txt" % i) + filenames.append(fn) + writer = python_io.TFRecordWriter(fn) + for j in range(self._num_records): + writer.write(self._record(i, j)) + writer.close() + return filenames + + def testReadOneEpoch(self): + # Basic test: read from file 0. + dataset = self.dataset_fn(self.test_filenames[0]) + self.assertDatasetProduces( + dataset, + expected_output=[self._record(0, i) for i in range(self._num_records)]) + + # Basic test: read from file 1. + dataset = self.dataset_fn(self.test_filenames[1]) + self.assertDatasetProduces( + dataset, + expected_output=[self._record(1, i) for i in range(self._num_records)]) + + # Basic test: read from both files. + dataset = self.dataset_fn(self.test_filenames) + expected_output = [] + for j in range(self._num_files): + expected_output.extend( + [self._record(j, i) for i in range(self._num_records)]) + self.assertDatasetProduces(dataset, expected_output=expected_output) + + def testReadTenEpochs(self): + dataset = self.dataset_fn(self.test_filenames, num_epochs=10) + expected_output = [] + for j in range(self._num_files): + expected_output.extend( + [self._record(j, i) for i in range(self._num_records)]) + self.assertDatasetProduces(dataset, expected_output=expected_output * 10) + + def testReadTenEpochsOfBatches(self): + dataset = self.dataset_fn( + self.test_filenames, num_epochs=10, batch_size=self._num_records) + expected_output = [] + for j in range(self._num_files): + expected_output.append( + [self._record(j, i) for i in range(self._num_records)]) + self.assertDatasetProduces(dataset, expected_output=expected_output * 10) + + def testReadZlibFiles(self): + zlib_files = [] + for i, fn in enumerate(self.test_filenames): + with open(fn, "rb") as f: + cdata = zlib.compress(f.read()) + + zfn = os.path.join(self.get_temp_dir(), "tfrecord_%s.z" % i) + with open(zfn, "wb") as f: + f.write(cdata) + zlib_files.append(zfn) + expected_output = [] + for j in range(self._num_files): + expected_output.extend( + [self._record(j, i) for i in range(self._num_records)]) + dataset = self.dataset_fn(zlib_files, compression_type="ZLIB") + self.assertDatasetProduces(dataset, expected_output=expected_output) + + def testReadGzipFiles(self): + gzip_files = [] + for i, fn in enumerate(self.test_filenames): + with open(fn, "rb") as f: + gzfn = os.path.join(self.get_temp_dir(), "tfrecord_%s.gz" % i) + with gzip.GzipFile(gzfn, "wb") as gzf: + gzf.write(f.read()) + gzip_files.append(gzfn) + expected_output = [] + for j in range(self._num_files): + expected_output.extend( + [self._record(j, i) for i in range(self._num_records)]) + dataset = self.dataset_fn(gzip_files, compression_type="GZIP") + self.assertDatasetProduces(dataset, expected_output=expected_output) + + def testReadWithBuffer(self): + one_mebibyte = 2**20 + dataset = readers.TFRecordDataset( + self.test_filenames, buffer_size=one_mebibyte) + expected_output = [] + for j in range(self._num_files): + expected_output.extend( + [self._record(j, i) for i in range(self._num_records)]) + self.assertDatasetProduces(dataset, expected_output=expected_output) + + def testReadFromDatasetOfFiles(self): + files = dataset_ops.Dataset.from_tensor_slices(self.test_filenames) + expected_output = [] + for j in range(self._num_files): + expected_output.extend( + [self._record(j, i) for i in range(self._num_records)]) + dataset = readers.TFRecordDataset(files) + self.assertDatasetProduces(dataset, expected_output=expected_output) + + def testReadTenEpochsFromDatasetOfFilesInParallel(self): + files = dataset_ops.Dataset.from_tensor_slices( + self.test_filenames).repeat(10) + expected_output = [] + for j in range(self._num_files): + expected_output.extend( + [self._record(j, i) for i in range(self._num_records)]) + dataset = readers.TFRecordDataset(files, num_parallel_reads=4) + self.assertDatasetProduces( + dataset, expected_output=expected_output * 10, assert_items_equal=True) + +if __name__ == "__main__": + test.main() diff --git a/tensorflow/python/data/kernel_tests/window_dataset_op_test.py b/tensorflow/python/data/kernel_tests/window_dataset_op_test.py deleted file mode 100644 index 9d067810944..00000000000 --- a/tensorflow/python/data/kernel_tests/window_dataset_op_test.py +++ /dev/null @@ -1,291 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for the experimental input pipeline ops.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from absl.testing import parameterized -import numpy as np - -from tensorflow.python.data.kernel_tests import test_base -from tensorflow.python.data.ops import dataset_ops -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import errors -from tensorflow.python.framework import sparse_tensor -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.platform import test - - -class WindowDatasetTest(test_base.DatasetTestBase, parameterized.TestCase): - - @parameterized.named_parameters( - ("1", 20, 14, 7, 1), - ("2", 20, 17, 9, 1), - ("3", 20, 14, 14, 1), - ("4", 20, 10, 14, 1), - ("5", 20, 14, 19, 1), - ("6", 20, 4, 1, 2), - ("7", 20, 2, 1, 6), - ("8", 20, 4, 7, 2), - ("9", 20, 2, 7, 6), - ("10", 1, 10, 4, 1), - ("11", 0, 10, 4, 1), - ("12", 20, 14, 7, 1, False), - ("13", 20, 17, 9, 1, False), - ("14", 20, 14, 14, 1, False), - ("15", 20, 10, 14, 1, False), - ("16", 20, 14, 19, 1, False), - ("17", 20, 4, 1, 2, False), - ("18", 20, 2, 1, 6, False), - ("19", 20, 4, 7, 2, False), - ("20", 20, 2, 7, 6, False), - ("21", 1, 10, 4, 1, False), - ("22", 0, 10, 4, 1, False), - ) - def testWindowDataset(self, count, size, shift, stride, drop_remainder=True): - """Tests a dataset that slides a window its input elements.""" - components = (np.arange(7), - np.array([[1, 2, 3]]) * np.arange(7)[:, np.newaxis], - np.array(37.0) * np.arange(7)) - - count_t = array_ops.placeholder(dtypes.int64, shape=[]) - size_t = array_ops.placeholder(dtypes.int64, shape=[]) - shift_t = array_ops.placeholder(dtypes.int64, shape=[]) - stride_t = array_ops.placeholder(dtypes.int64, shape=[]) - drop_remainder_t = array_ops.placeholder(dtypes.bool, shape=[]) - - def _map_fn(x, y, z): - return math_ops.square(x), math_ops.square(y), math_ops.square(z) - - def _flat_map_fn(x, y, z): - return dataset_ops.Dataset.zip((x.batch(batch_size=size_t), - y.batch(batch_size=size_t), - z.batch(batch_size=size_t))) - - iterator = dataset_ops.Dataset.from_tensor_slices(components).map( - _map_fn).repeat(count).window( - size=size_t, - shift=shift_t, - stride=stride_t, - drop_remainder=drop_remainder_t).flat_map( - _flat_map_fn).make_initializable_iterator() - init_op = iterator.initializer - get_next = iterator.get_next() - - self.assertEqual([[None] + list(c.shape[1:]) for c in components], - [t.shape.as_list() for t in get_next]) - - with self.cached_session() as sess: - sess.run( - init_op, - feed_dict={ - count_t: count, - size_t: size, - shift_t: shift, - stride_t: stride, - drop_remainder_t: drop_remainder - }) - num_full_batches = max( - 0, (count * 7 - ((size - 1) * stride + 1)) // shift + 1) - for i in range(num_full_batches): - result = sess.run(get_next) - for component, result_component in zip(components, result): - for j in range(size): - self.assertAllEqual(component[(i * shift + j * stride) % 7]**2, - result_component[j]) - if not drop_remainder: - num_partial_batches = (count * 7) // shift + ( - (count * 7) % shift > 0) - num_full_batches - for i in range(num_partial_batches): - result = sess.run(get_next) - for component, result_component in zip(components, result): - remaining = (count * 7) - ((num_full_batches + i) * shift) - num_elements = remaining // stride + ((remaining % stride) > 0) - for j in range(num_elements): - self.assertAllEqual( - component[((num_full_batches + i) * shift + j * stride) % 7] - **2, result_component[j]) - with self.assertRaises(errors.OutOfRangeError): - sess.run(get_next) - - @parameterized.named_parameters( - ("1", 14, 0, 3, 1), - ("2", 14, 3, 0, 1), - ("3", 14, 3, 3, 0), - ) - def testWindowDatasetInvalid(self, count, size, shift, stride): - count_t = array_ops.placeholder(dtypes.int64, shape=[]) - size_t = array_ops.placeholder(dtypes.int64, shape=[]) - shift_t = array_ops.placeholder(dtypes.int64, shape=[]) - stride_t = array_ops.placeholder(dtypes.int64, shape=[]) - - iterator = dataset_ops.Dataset.range(10).map(lambda x: x).repeat( - count_t).window( - size=size_t, shift=shift_t, - stride=stride_t).flat_map(lambda x: x.batch(batch_size=size_t) - ).make_initializable_iterator() - init_op = iterator.initializer - - with self.cached_session() as sess: - with self.assertRaises(errors.InvalidArgumentError): - sess.run( - init_op, - feed_dict={ - count_t: count, - size_t: size, - shift_t: shift, - stride_t: stride - }) - - def testWindowSparse(self): - - def _sparse(i): - return sparse_tensor.SparseTensorValue( - indices=[[0]], values=(i * [1]), dense_shape=[1]) - - iterator = dataset_ops.Dataset.range(10).map(_sparse).window( - size=5, shift=3, drop_remainder=True).flat_map( - lambda x: x.batch(batch_size=5)).make_initializable_iterator() - init_op = iterator.initializer - get_next = iterator.get_next() - - with self.cached_session() as sess: - sess.run(init_op) - num_batches = (10 - 5) // 3 + 1 - for i in range(num_batches): - actual = sess.run(get_next) - expected = sparse_tensor.SparseTensorValue( - indices=[[0, 0], [1, 0], [2, 0], [3, 0], [4, 0]], - values=[i * 3, i * 3 + 1, i * 3 + 2, i * 3 + 3, i * 3 + 4], - dense_shape=[5, 1]) - self.assertTrue(sparse_tensor.is_sparse(actual)) - self.assertSparseValuesEqual(actual, expected) - with self.assertRaises(errors.OutOfRangeError): - sess.run(get_next) - - def testWindowSparseWithDifferentDenseShapes(self): - - def _sparse(i): - return sparse_tensor.SparseTensorValue( - indices=array_ops.expand_dims( - math_ops.range(i, dtype=dtypes.int64), 1), - values=array_ops.fill([math_ops.to_int32(i)], i), - dense_shape=[i]) - - iterator = dataset_ops.Dataset.range(10).map(_sparse).window( - size=5, shift=3, drop_remainder=True).flat_map( - lambda x: x.batch(batch_size=5)).make_initializable_iterator() - init_op = iterator.initializer - get_next = iterator.get_next() - - with self.cached_session() as sess: - sess.run(init_op) - num_batches = (10 - 5) // 3 + 1 - for i in range(num_batches): - actual = sess.run(get_next) - expected_indices = [] - expected_values = [] - for j in range(5): - for k in range(i * 3 + j): - expected_indices.append([j, k]) - expected_values.append(i * 3 + j) - expected = sparse_tensor.SparseTensorValue( - indices=expected_indices, - values=expected_values, - dense_shape=[5, i * 3 + 5 - 1]) - self.assertTrue(sparse_tensor.is_sparse(actual)) - self.assertSparseValuesEqual(actual, expected) - with self.assertRaises(errors.OutOfRangeError): - sess.run(get_next) - - def testNestedWindowSparse(self): - - def _sparse(i): - return sparse_tensor.SparseTensorValue( - indices=[[0]], values=(i * [1]), dense_shape=[1]) - - iterator = dataset_ops.Dataset.range(10).map(_sparse).window( - size=4, shift=2, - drop_remainder=True).flat_map(lambda x: x.batch(batch_size=4)).window( - size=3, shift=1, drop_remainder=True).flat_map( - lambda x: x.batch(batch_size=3)).make_initializable_iterator() - init_op = iterator.initializer - get_next = iterator.get_next() - - with self.cached_session() as sess: - sess.run(init_op) - # Slide: 1st batch. - actual = sess.run(get_next) - expected = sparse_tensor.SparseTensorValue( - indices=[[0, 0, 0], [0, 1, 0], [0, 2, 0], [0, 3, 0], [1, 0, 0], - [1, 1, 0], [1, 2, 0], [1, 3, 0], [2, 0, 0], [2, 1, 0], - [2, 2, 0], [2, 3, 0]], - values=[0, 1, 2, 3, 2, 3, 4, 5, 4, 5, 6, 7], - dense_shape=[3, 4, 1]) - self.assertTrue(sparse_tensor.is_sparse(actual)) - self.assertSparseValuesEqual(actual, expected) - # Slide: 2nd batch. - actual = sess.run(get_next) - expected = sparse_tensor.SparseTensorValue( - indices=[[0, 0, 0], [0, 1, 0], [0, 2, 0], [0, 3, 0], [1, 0, 0], - [1, 1, 0], [1, 2, 0], [1, 3, 0], [2, 0, 0], [2, 1, 0], - [2, 2, 0], [2, 3, 0]], - values=[2, 3, 4, 5, 4, 5, 6, 7, 6, 7, 8, 9], - dense_shape=[3, 4, 1]) - self.assertTrue(sparse_tensor.is_sparse(actual)) - self.assertSparseValuesEqual(actual, expected) - with self.assertRaises(errors.OutOfRangeError): - sess.run(get_next) - - def testWindowShapeError(self): - - def generator(): - yield [1.0, 2.0, 3.0] - yield [4.0, 5.0, 6.0] - yield [7.0, 8.0, 9.0, 10.0] - - iterator = dataset_ops.Dataset.from_generator( - generator, dtypes.float32, output_shapes=[None]).window( - size=3, shift=1).flat_map( - lambda x: x.batch(batch_size=3)).make_initializable_iterator() - next_element = iterator.get_next() - - with self.cached_session() as sess: - sess.run(iterator.initializer) - with self.assertRaisesRegexp( - errors.InvalidArgumentError, - r"Cannot batch tensors with different shapes in component 0. " - r"First element had shape \[3\] and element 2 had shape \[4\]."): - sess.run(next_element) - - def testWindowIgnoreErrors(self): - input_values = np.float32([1., np.nan, 2., np.nan, 3.]) - dataset = dataset_ops.Dataset.from_tensor_slices(input_values).map( - lambda x: array_ops.check_numerics(x, "message")).window( - size=2, shift=2, stride=2, - drop_remainder=True).flat_map(lambda x: x.batch(batch_size=2)) - get_next = dataset.make_one_shot_iterator().get_next() - - with self.cached_session() as sess: - self.assertAllEqual(np.float32([1., 2.]), sess.run(get_next)) - self.assertAllEqual(np.float32([2., 3.]), sess.run(get_next)) - with self.assertRaises(errors.OutOfRangeError): - sess.run(get_next) - - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/python/data/kernel_tests/window_test.py b/tensorflow/python/data/kernel_tests/window_test.py new file mode 100644 index 00000000000..d083142ab6a --- /dev/null +++ b/tensorflow/python/data/kernel_tests/window_test.py @@ -0,0 +1,231 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Tests for `tf.data.Dataset.window()`.""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from absl.testing import parameterized +import numpy as np + +from tensorflow.python.data.kernel_tests import test_base +from tensorflow.python.data.ops import dataset_ops +from tensorflow.python.data.util import nest +from tensorflow.python.framework import dtypes +from tensorflow.python.framework import errors +from tensorflow.python.framework import sparse_tensor +from tensorflow.python.framework import test_util +from tensorflow.python.ops import array_ops +from tensorflow.python.ops import math_ops +from tensorflow.python.platform import test + + +@test_util.run_all_in_graph_and_eager_modes +class WindowTest(test_base.DatasetTestBase, parameterized.TestCase): + + @parameterized.named_parameters( + ("1", 20, 14, 7, 1), + ("2", 20, 17, 9, 1), + ("3", 20, 14, 14, 1), + ("4", 20, 10, 14, 1), + ("5", 20, 14, 19, 1), + ("6", 20, 4, 1, 2), + ("7", 20, 2, 1, 6), + ("8", 20, 4, 7, 2), + ("9", 20, 2, 7, 6), + ("10", 1, 10, 4, 1), + ("11", 0, 10, 4, 1), + ("12", 20, 14, 7, 1, False), + ("13", 20, 17, 9, 1, False), + ("14", 20, 14, 14, 1, False), + ("15", 20, 10, 14, 1, False), + ("16", 20, 14, 19, 1, False), + ("17", 20, 4, 1, 2, False), + ("18", 20, 2, 1, 6, False), + ("19", 20, 4, 7, 2, False), + ("20", 20, 2, 7, 6, False), + ("21", 1, 10, 4, 1, False), + ("22", 0, 10, 4, 1, False), + ) + def testWindowDataset(self, count, size, shift, stride, drop_remainder=True): + """Tests a dataset that slides a window its input elements.""" + components = (np.arange(7), + np.array([[1, 2, 3]]) * np.arange(7)[:, np.newaxis], + np.array(37.0) * np.arange(7)) + + def _map_fn(x, y, z): + return math_ops.square(x), math_ops.square(y), math_ops.square(z) + + def _flat_map_fn(x, y, z): + return dataset_ops.Dataset.zip((x.batch(batch_size=size), + y.batch(batch_size=size), + z.batch(batch_size=size))) + + dataset = dataset_ops.Dataset.from_tensor_slices(components).map( + _map_fn).repeat(count).window( + size=size, + shift=shift, + stride=stride, + drop_remainder=drop_remainder).flat_map(_flat_map_fn) + get_next = self.getNext(dataset) + + self.assertEqual( + [[None] + list(c.shape[1:]) for c in components], + [ts.as_list() for ts in nest.flatten(dataset.output_shapes)]) + + num_full_batches = max(0, + (count * 7 - ((size - 1) * stride + 1)) // shift + 1) + for i in range(num_full_batches): + result = self.evaluate(get_next()) + for component, result_component in zip(components, result): + for j in range(size): + self.assertAllEqual(component[(i * shift + j * stride) % 7]**2, + result_component[j]) + if not drop_remainder: + num_partial_batches = (count * 7) // shift + ( + (count * 7) % shift > 0) - num_full_batches + for i in range(num_partial_batches): + result = self.evaluate(get_next()) + for component, result_component in zip(components, result): + remaining = (count * 7) - ((num_full_batches + i) * shift) + num_elements = remaining // stride + ((remaining % stride) > 0) + for j in range(num_elements): + self.assertAllEqual( + component[((num_full_batches + i) * shift + j * stride) % 7]**2, + result_component[j]) + with self.assertRaises(errors.OutOfRangeError): + self.evaluate(get_next()) + with self.assertRaises(errors.OutOfRangeError): + self.evaluate(get_next()) + + @parameterized.named_parameters( + ("1", 14, 0, 3, 1), + ("2", 14, 3, 0, 1), + ("3", 14, 3, 3, 0), + ) + def testWindowDatasetInvalid(self, count, size, shift, stride): + dataset = dataset_ops.Dataset.range(10).map(lambda x: x).repeat( + count).window( + size=size, shift=shift, + stride=stride).flat_map(lambda x: x.batch(batch_size=size)) + self.assertDatasetProduces( + dataset, expected_error=(errors.InvalidArgumentError, "")) + + def testWindowSparse(self): + + def _sparse(i): + return sparse_tensor.SparseTensorValue( + indices=[[0]], values=(i * [1]), dense_shape=[1]) + + dataset = dataset_ops.Dataset.range(10).map(_sparse).window( + size=5, shift=3, + drop_remainder=True).flat_map(lambda x: x.batch(batch_size=5)) + + num_batches = (10 - 5) // 3 + 1 + expected_output = [ + sparse_tensor.SparseTensorValue( + indices=[[0, 0], [1, 0], [2, 0], [3, 0], [4, 0]], + values=[i * 3, i * 3 + 1, i * 3 + 2, i * 3 + 3, i * 3 + 4], + dense_shape=[5, 1]) for i in range(num_batches) + ] + self.assertDatasetProduces(dataset, expected_output=expected_output) + + def testWindowSparseWithDifferentDenseShapes(self): + + def _sparse(i): + return sparse_tensor.SparseTensorValue( + indices=array_ops.expand_dims( + math_ops.range(i, dtype=dtypes.int64), 1), + values=array_ops.fill([math_ops.to_int32(i)], i), + dense_shape=[i]) + + dataset = dataset_ops.Dataset.range(10).map(_sparse).window( + size=5, shift=3, + drop_remainder=True).flat_map(lambda x: x.batch(batch_size=5)) + + expected_output = [] + num_batches = (10 - 5) // 3 + 1 + for i in range(num_batches): + expected_indices = [] + expected_values = [] + for j in range(5): + for k in range(i * 3 + j): + expected_indices.append([j, k]) + expected_values.append(i * 3 + j) + expected_output.append( + sparse_tensor.SparseTensorValue( + indices=expected_indices, + values=expected_values, + dense_shape=[5, i * 3 + 5 - 1])) + self.assertDatasetProduces(dataset, expected_output=expected_output) + + def testNestedWindowSparse(self): + + def _sparse(i): + return sparse_tensor.SparseTensorValue( + indices=[[0]], values=(i * [1]), dense_shape=[1]) + + dataset = dataset_ops.Dataset.range(10).map(_sparse).window( + size=4, shift=2, + drop_remainder=True).flat_map(lambda x: x.batch(batch_size=4)).window( + size=3, shift=1, + drop_remainder=True).flat_map(lambda x: x.batch(batch_size=3)) + + expected_output = [ + sparse_tensor.SparseTensorValue( + indices=[[0, 0, 0], [0, 1, 0], [0, 2, 0], [0, 3, 0], [1, 0, 0], + [1, 1, 0], [1, 2, 0], [1, 3, 0], [2, 0, 0], [2, 1, 0], + [2, 2, 0], [2, 3, 0]], + values=[0, 1, 2, 3, 2, 3, 4, 5, 4, 5, 6, 7], + dense_shape=[3, 4, 1]), + sparse_tensor.SparseTensorValue( + indices=[[0, 0, 0], [0, 1, 0], [0, 2, 0], [0, 3, 0], [1, 0, 0], + [1, 1, 0], [1, 2, 0], [1, 3, 0], [2, 0, 0], [2, 1, 0], + [2, 2, 0], [2, 3, 0]], + values=[2, 3, 4, 5, 4, 5, 6, 7, 6, 7, 8, 9], + dense_shape=[3, 4, 1]) + ] + self.assertDatasetProduces(dataset, expected_output=expected_output) + + def testWindowShapeError(self): + + def generator(): + yield [1.0, 2.0, 3.0] + yield [4.0, 5.0, 6.0] + yield [7.0, 8.0, 9.0, 10.0] + + dataset = dataset_ops.Dataset.from_generator( + generator, dtypes.float32, output_shapes=[None]).window( + size=3, shift=1).flat_map(lambda x: x.batch(batch_size=3)) + self.assertDatasetProduces( + dataset, + expected_error=( + errors.InvalidArgumentError, + r"Cannot batch tensors with different shapes in component 0. " + r"First element had shape \[3\] and element 2 had shape \[4\].")) + + def testWindowIgnoreErrors(self): + input_values = np.float32([1., np.nan, 2., np.nan, 3.]) + dataset = dataset_ops.Dataset.from_tensor_slices(input_values).map( + lambda x: array_ops.check_numerics(x, "message")).window( + size=2, shift=2, stride=2, + drop_remainder=True).flat_map(lambda x: x.batch(batch_size=2)) + self.assertDatasetProduces( + dataset, expected_output=[np.float32([1., 2.]), + np.float32([2., 3.])]) + + +if __name__ == "__main__": + test.main() diff --git a/tensorflow/python/data/kernel_tests/zip_dataset_op_test.py b/tensorflow/python/data/kernel_tests/zip_dataset_op_test.py deleted file mode 100644 index 9d76387a343..00000000000 --- a/tensorflow/python/data/kernel_tests/zip_dataset_op_test.py +++ /dev/null @@ -1,115 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for the experimental input pipeline ops.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np - -from tensorflow.python.data.kernel_tests import test_base -from tensorflow.python.data.ops import dataset_ops -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import errors -from tensorflow.python.ops import array_ops -from tensorflow.python.platform import test - - -class ZipDatasetTest(test_base.DatasetTestBase): - - def testZipDataset(self): - component_placeholders = [ - array_ops.placeholder(dtypes.int64), - array_ops.placeholder(dtypes.int64), - array_ops.placeholder(dtypes.float64) - ] - - datasets = tuple([ - dataset_ops.Dataset.from_tensor_slices(component_placeholder) - for component_placeholder in component_placeholders - ]) - zipped = dataset_ops.Dataset.zip(datasets) - - iterator = zipped.make_initializable_iterator() - init_op = iterator.initializer - get_next = iterator.get_next() - - with self.cached_session() as sess: - equal_length_components = [ - np.tile(np.array([[1], [2], [3], [4]]), 20), - np.tile(np.array([[12], [13], [14], [15]]), 22), - np.array([37.0, 38.0, 39.0, 40.0]) - ] - sess.run(init_op, feed_dict={ph: value for ph, value in zip( - component_placeholders, equal_length_components)}) - for i in range(4): - results = sess.run(get_next) - for component, result_component in zip( - equal_length_components, results): - self.assertAllEqual(component[i], result_component) - with self.assertRaises(errors.OutOfRangeError): - sess.run(get_next) - - variable_length_components = [[1, 2, 3, 4], [1, 2, 3, 4, 5], [1.0, 2.0]] - sess.run(init_op, feed_dict={ph: value for ph, value in zip( - component_placeholders, variable_length_components)}) - for i in range(2): - results = sess.run(get_next) - for component, result_component in zip( - variable_length_components, results): - self.assertAllEqual(component[i], result_component) - with self.assertRaises(errors.OutOfRangeError): - sess.run(get_next) - - def testNestedZipDataset(self): - component_placeholders = [ - array_ops.placeholder(dtypes.int64, shape=[4, 20]), - array_ops.placeholder(dtypes.int64, shape=[4, 22]), - array_ops.placeholder(dtypes.float64, shape=[4]) - ] - - datasets = [ - dataset_ops.Dataset.from_tensor_slices(component_placeholder) - for component_placeholder in component_placeholders - ] - zipped = dataset_ops.Dataset.zip((datasets[0], (datasets[1], datasets[2]))) - - iterator = zipped.make_initializable_iterator() - init_op = iterator.initializer - get_next = iterator.get_next() - - self.assertEqual([20], get_next[0].shape) - self.assertEqual([22], get_next[1][0].shape) - self.assertEqual([], get_next[1][1].shape) - - with self.cached_session() as sess: - equal_length_components = [ - np.tile(np.array([[1], [2], [3], [4]]), 20), - np.tile(np.array([[12], [13], [14], [15]]), 22), - np.array([37.0, 38.0, 39.0, 40.0]) - ] - sess.run(init_op, feed_dict={ph: value for ph, value in zip( - component_placeholders, equal_length_components)}) - for i in range(4): - result1, (result2, result3) = sess.run(get_next) - self.assertAllEqual(equal_length_components[0][i], result1) - self.assertAllEqual(equal_length_components[1][i], result2) - self.assertAllEqual(equal_length_components[2][i], result3) - with self.assertRaises(errors.OutOfRangeError): - sess.run(get_next) - - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/python/data/kernel_tests/zip_test.py b/tensorflow/python/data/kernel_tests/zip_test.py new file mode 100644 index 00000000000..477c9fa7da1 --- /dev/null +++ b/tensorflow/python/data/kernel_tests/zip_test.py @@ -0,0 +1,101 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Tests for `tf.data.Dataset.zip()`.""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import numpy as np + +from tensorflow.python.data.kernel_tests import test_base +from tensorflow.python.data.ops import dataset_ops +from tensorflow.python.framework import errors +from tensorflow.python.framework import tensor_shape +from tensorflow.python.framework import test_util +from tensorflow.python.platform import test + + +@test_util.run_all_in_graph_and_eager_modes +class ZipTest(test_base.DatasetTestBase): + + def testZipDataset(self): + + def dataset_fn(components): + datasets = tuple([ + dataset_ops.Dataset.from_tensor_slices(component) + for component in components + ]) + return dataset_ops.Dataset.zip(datasets) + + equal_length_components = [ + np.tile(np.array([[1], [2], [3], [4]]), 20), + np.tile(np.array([[12], [13], [14], [15]]), 22), + np.array([37.0, 38.0, 39.0, 40.0]) + ] + + get_next = self.getNext(dataset_fn(equal_length_components)) + for i in range(4): + results = self.evaluate(get_next()) + for component, result_component in zip(equal_length_components, results): + self.assertAllEqual(component[i], result_component) + with self.assertRaises(errors.OutOfRangeError): + self.evaluate(get_next()) + with self.assertRaises(errors.OutOfRangeError): + self.evaluate(get_next()) + + variable_length_components = [[1, 2, 3, 4], [1, 2, 3, 4, 5], [1.0, 2.0]] + get_next = self.getNext(dataset_fn(variable_length_components)) + for i in range(2): + results = self.evaluate(get_next()) + for component, result_component in zip(variable_length_components, + results): + self.assertAllEqual(component[i], result_component) + with self.assertRaises(errors.OutOfRangeError): + self.evaluate(get_next()) + with self.assertRaises(errors.OutOfRangeError): + self.evaluate(get_next()) + + def testNestedZipDataset(self): + + equal_length_components = [ + np.tile(np.array([[1], [2], [3], [4]]), 20), + np.tile(np.array([[12], [13], [14], [15]]), 22), + np.array([37.0, 38.0, 39.0, 40.0]) + ] + datasets = [ + dataset_ops.Dataset.from_tensor_slices(component) + for component in equal_length_components + ] + dataset = dataset_ops.Dataset.zip((datasets[0], (datasets[1], datasets[2]))) + + self.assertEqual( + dataset.output_shapes, + (tensor_shape.TensorShape([20]), + (tensor_shape.TensorShape([22]), tensor_shape.TensorShape([])))) + + get_next = self.getNext(dataset) + for i in range(4): + result1, (result2, result3) = self.evaluate(get_next()) + self.assertAllEqual(equal_length_components[0][i], result1) + self.assertAllEqual(equal_length_components[1][i], result2) + self.assertAllEqual(equal_length_components[2][i], result3) + with self.assertRaises(errors.OutOfRangeError): + self.evaluate(get_next()) + with self.assertRaises(errors.OutOfRangeError): + self.evaluate(get_next()) + + +if __name__ == "__main__": + test.main() diff --git a/tensorflow/python/data/ops/BUILD b/tensorflow/python/data/ops/BUILD index 18edc0872d7..0c5acda180f 100644 --- a/tensorflow/python/data/ops/BUILD +++ b/tensorflow/python/data/ops/BUILD @@ -14,6 +14,7 @@ py_library( "//tensorflow/python:control_flow_ops", "//tensorflow/python:dataset_ops_gen", "//tensorflow/python:dtypes", + "//tensorflow/python:experimental_dataset_ops_gen", "//tensorflow/python:framework_ops", "//tensorflow/python:function", "//tensorflow/python:math_ops", @@ -25,8 +26,12 @@ py_library( "//tensorflow/python:tensor_shape", "//tensorflow/python:tensor_util", "//tensorflow/python:util", + "//tensorflow/python/data/experimental/ops:filter_for_shard_ops", + "//tensorflow/python/data/experimental/ops:optimization_options", "//tensorflow/python/data/experimental/ops:stats_options", + "//tensorflow/python/data/experimental/ops:threading_options", "//tensorflow/python/data/util:nest", + "//tensorflow/python/data/util:options", "//tensorflow/python/data/util:random_seed", "//tensorflow/python/data/util:sparse", "//tensorflow/python/data/util:structure", diff --git a/tensorflow/python/data/ops/dataset_ops.py b/tensorflow/python/data/ops/dataset_ops.py index 3836a68e7d4..ba6d20373e4 100644 --- a/tensorflow/python/data/ops/dataset_ops.py +++ b/tensorflow/python/data/ops/dataset_ops.py @@ -18,6 +18,7 @@ from __future__ import division from __future__ import print_function import abc +import functools import threading import warnings @@ -25,11 +26,16 @@ import numpy as np import six from tensorflow.python.compat import compat +from tensorflow.python.data.experimental.ops import filter_for_shard_ops +from tensorflow.python.data.experimental.ops import optimization_options from tensorflow.python.data.experimental.ops import stats_options +from tensorflow.python.data.experimental.ops import threading_options from tensorflow.python.data.ops import iterator_ops from tensorflow.python.data.util import nest +from tensorflow.python.data.util import options as options_lib from tensorflow.python.data.util import random_seed from tensorflow.python.data.util import sparse +from tensorflow.python.data.util import structure as structure_lib from tensorflow.python.eager import context from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes @@ -43,6 +49,7 @@ from tensorflow.python.framework import tensor_util from tensorflow.python.ops import array_ops from tensorflow.python.ops import control_flow_ops from tensorflow.python.ops import gen_dataset_ops +from tensorflow.python.ops import gen_experimental_dataset_ops as ged_ops from tensorflow.python.ops import gen_io_ops from tensorflow.python.ops import math_ops from tensorflow.python.ops import script_ops @@ -52,9 +59,12 @@ from tensorflow.python.util import function_utils from tensorflow.python.util.tf_export import tf_export -@tf_export("data.Dataset") +ops.NotDifferentiable("ReduceDataset") + + +@tf_export("data.Dataset", v1=[]) @six.add_metaclass(abc.ABCMeta) -class Dataset(object): +class DatasetV2(object): """Represents a potentially large set of elements. A `Dataset` can be used to represent an input pipeline as a @@ -62,9 +72,6 @@ class Dataset(object): plan" of transformations that act on those elements. """ - def __init__(self): - pass - def _as_serialized_graph(self): """Produces serialized graph representation of the dataset. @@ -89,6 +96,37 @@ class Dataset(object): raise NotImplementedError("Dataset._inputs") + def _has_captured_ref(self): + """Whether this dataset uses a function that captures ref variables. + + Returns: + A boolean, which if true indicates that the dataset or one of its inputs + uses a function that captures ref variables. + """ + if context.executing_eagerly(): + # RefVariables are not supported in eager mode + return False + + def is_tensor_or_parent_ref(tensor): + if tensor.dtype._is_ref_dtype: # pylint: disable=protected-access + return True + return any([is_tensor_or_parent_ref(x) for x in tensor.op.inputs]) + + for fn in self._functions(): + if any([is_tensor_or_parent_ref(t) for t in fn.function.captured_inputs]): + return True + + return any( + [input_dataset._has_captured_ref() for input_dataset in self._inputs()]) # pylint: disable=protected-access + + def _functions(self): + """Returns a list of functions associated with this dataset. + + Returns: + A list of `StructuredFunctionWrapper` objects. + """ + return [] + def options(self): """Returns the options for this dataset and its inputs. @@ -107,9 +145,26 @@ class Dataset(object): dataset = self options = self.options() + if options.experimental_threading is not None: + t_options = options.experimental_threading + if t_options.private_threadpool_size is not None: + dataset = _PrivateThreadPoolDataset(dataset, + t_options.private_threadpool_size) + if t_options.max_intra_op_parallelism is not None: + dataset = _MaxIntraOpParallelismDataset( + dataset, t_options.max_intra_op_parallelism) static_optimizations = options._static_optimizations() # pylint: disable=protected-access if static_optimizations: - dataset = _OptimizeDataset(dataset, static_optimizations) + if self._has_captured_ref(): + warnings.warn( + "tf.data static optimizations are not compatible with tf.Variable. " + "The following optimizations will be disabled: %s. To enable " + "optimizations, use resource variables instead by calling " + "`tf.enable_resource_variables()` at the start of the program." % + ", ".join(static_optimizations)) + else: + dataset = _OptimizeDataset(dataset, static_optimizations) + if options.experimental_autotune is not False: dataset = _ModelDataset(dataset) if options.experimental_stats and options.experimental_stats.aggregator: # pylint: disable=line-too-long @@ -119,51 +174,6 @@ class Dataset(object): options.experimental_stats.counter_prefix) return dataset - def make_initializable_iterator(self, shared_name=None): - """Creates an `Iterator` for enumerating the elements of this dataset. - - Note: The returned iterator will be in an uninitialized state, - and you must run the `iterator.initializer` operation before using it: - - ```python - dataset = ... - iterator = dataset.make_initializable_iterator() - # ... - sess.run(iterator.initializer) - ``` - - Args: - shared_name: (Optional.) If non-empty, the returned iterator will be - shared under the given name across multiple sessions that share the - same devices (e.g. when using a remote server). - - Returns: - An `Iterator` over the elements of this dataset. - - Raises: - RuntimeError: If eager execution is enabled. - """ - if context.executing_eagerly(): - raise RuntimeError( - "dataset.make_initializable_iterator is not supported when eager " - "execution is enabled.") - dataset = self._apply_options() - if shared_name is None: - shared_name = "" - if compat.forward_compatible(2018, 8, 3): - iterator_resource = gen_dataset_ops.iterator_v2( - container="", shared_name=shared_name, **flat_structure(self)) - else: - iterator_resource = gen_dataset_ops.iterator( - container="", shared_name=shared_name, **flat_structure(self)) - with ops.colocate_with(iterator_resource): - initializer = gen_dataset_ops.make_iterator( - dataset._as_variant_tensor(), # pylint: disable=protected-access - iterator_resource) - return iterator_ops.Iterator(iterator_resource, initializer, - dataset.output_types, dataset.output_shapes, - dataset.output_classes) - def __iter__(self): """Creates an `Iterator` for enumerating the elements of this dataset. @@ -183,55 +193,6 @@ class Dataset(object): raise RuntimeError("dataset.__iter__() is only supported when eager " "execution is enabled.") - def make_one_shot_iterator(self): - """Creates an `Iterator` for enumerating the elements of this dataset. - - Note: The returned iterator will be initialized automatically. - A "one-shot" iterator does not currently support re-initialization. - - Returns: - An `Iterator` over the elements of this dataset. - """ - if context.executing_eagerly(): - dataset = self._apply_options() - return iterator_ops.EagerIterator(dataset) - - graph_level_seed, op_level_seed = core_random_seed.get_seed(None) - - # NOTE(mrry): We capture by value here to ensure that `_make_dataset()` is - # a 0-argument function. - @function.Defun(capture_by_value=True) - def _make_dataset(): - """Factory function for a dataset.""" - # NOTE(mrry): `Defun` does not capture the graph-level seed from the - # enclosing graph, so if a graph-level seed is present we set the local - # graph seed based on a combination of the graph- and op-level seeds. - if graph_level_seed is not None: - assert op_level_seed is not None - core_random_seed.set_random_seed( - (graph_level_seed + 87654321 * op_level_seed) % (2 ** 63 - 1)) - - dataset = self._apply_options() - return dataset._as_variant_tensor() # pylint: disable=protected-access - - try: - _make_dataset.add_to_graph(ops.get_default_graph()) - except ValueError as err: - if "Cannot capture a stateful node" in str(err): - raise ValueError( - "Failed to create a one-shot iterator for a dataset. " - "`Dataset.make_one_shot_iterator()` does not support datasets that " - "capture stateful objects, such as a `Variable` or `LookupTable`. " - "In these cases, use `Dataset.make_initializable_iterator()`. " - "(Original error: %s)" % err) - else: - six.reraise(ValueError, err) - - return iterator_ops.Iterator( - gen_dataset_ops.one_shot_iterator( - dataset_factory=_make_dataset, **flat_structure(self)), - None, self.output_types, self.output_shapes, self.output_classes) - @abc.abstractproperty def output_classes(self): """Returns the class of each component of an element of this dataset. @@ -279,9 +240,10 @@ class Dataset(object): Note that if `tensors` contains a NumPy array, and eager execution is not enabled, the values will be embedded in the graph as one or more `tf.constant` operations. For large datasets (> 1 GB), this can waste - memory and run into byte limits of graph serialization. If tensors contains - one or more large NumPy arrays, consider the alternative described in - [this guide](https://tensorflow.org/guide/datasets#consuming_numpy_arrays). + memory and run into byte limits of graph serialization. If `tensors` + contains one or more large NumPy arrays, consider the alternative described + in [this + guide](https://tensorflow.org/guide/datasets#consuming_numpy_arrays). Args: tensors: A nested structure of tensors. @@ -298,9 +260,10 @@ class Dataset(object): Note that if `tensors` contains a NumPy array, and eager execution is not enabled, the values will be embedded in the graph as one or more `tf.constant` operations. For large datasets (> 1 GB), this can waste - memory and run into byte limits of graph serialization. If tensors contains - one or more large NumPy arrays, consider the alternative described in - [this guide](https://tensorflow.org/guide/datasets#consuming_numpy_arrays). + memory and run into byte limits of graph serialization. If `tensors` + contains one or more large NumPy arrays, consider the alternative described + in [this guide]( + https://tensorflow.org/guide/datasets#consuming_numpy_arrays). Args: tensors: A nested structure of tensors, each having the same size in the @@ -311,19 +274,6 @@ class Dataset(object): """ return TensorSliceDataset(tensors) - @staticmethod - @deprecation.deprecated(None, "Use `tf.data.Dataset.from_tensor_slices()`.") - def from_sparse_tensor_slices(sparse_tensor): - """Splits each rank-N `tf.SparseTensor` in this dataset row-wise. - - Args: - sparse_tensor: A `tf.SparseTensor`. - - Returns: - Dataset: A `Dataset` of rank-(N-1) sparse tensors. - """ - return SparseTensorSliceDataset(sparse_tensor) - class _GeneratorState(object): """Stores outstanding iterators created from a Python generator. @@ -373,17 +323,19 @@ class Dataset(object): ```python import itertools + tf.enable_eager_execution() def gen(): for i in itertools.count(1): yield (i, [1] * i) - ds = Dataset.from_generator( + ds = tf.data.Dataset.from_generator( gen, (tf.int64, tf.int64), (tf.TensorShape([]), tf.TensorShape([None]))) - value = ds.make_one_shot_iterator().get_next() - sess.run(value) # (1, array([1])) - sess.run(value) # (2, array([1, 1])) + for value in ds.take(2): + print value + # (1, array([1])) + # (2, array([1, 1])) ``` NOTE: The current implementation of `Dataset.from_generator()` uses @@ -435,7 +387,7 @@ class Dataset(object): flattened_types = [dtypes.as_dtype(dt) for dt in nest.flatten(output_types)] flattened_shapes = nest.flatten(output_shapes) - generator_state = Dataset._GeneratorState(generator) + generator_state = DatasetV2._GeneratorState(generator) def get_iterator_id_fn(unused_dummy): """Creates a unique `iterator_id` for each pass over the dataset. @@ -580,7 +532,7 @@ class Dataset(object): ``` Args: - *args: follow same semantics as python's xrange. + *args: follows the same semantics as python's xrange. len(args) == 1 -> start = 0, stop = args[0], step = 1 len(args) == 2 -> start = args[0], stop = args[1], step = 1 len(args) == 3 -> start = args[0], stop = args[1, stop = args[2] @@ -820,78 +772,6 @@ class Dataset(object): """ return SkipDataset(self, count) - def shard(self, num_shards, index): - """Creates a `Dataset` that includes only 1/`num_shards` of this dataset. - - This dataset operator is very useful when running distributed training, as - it allows each worker to read a unique subset. - - When reading a single input file, you can skip elements as follows: - - ```python - d = tf.data.TFRecordDataset(FLAGS.input_file) - d = d.shard(FLAGS.num_workers, FLAGS.worker_index) - d = d.repeat(FLAGS.num_epochs) - d = d.shuffle(FLAGS.shuffle_buffer_size) - d = d.map(parser_fn, num_parallel_calls=FLAGS.num_map_threads) - ``` - - Important caveats: - - - Be sure to shard before you use any randomizing operator (such as - shuffle). - - Generally it is best if the shard operator is used early in the dataset - pipeline. For example, when reading from a set of TFRecord files, shard - before converting the dataset to input samples. This avoids reading every - file on every worker. The following is an example of an efficient - sharding strategy within a complete pipeline: - - ```python - d = Dataset.list_files(FLAGS.pattern) - d = d.shard(FLAGS.num_workers, FLAGS.worker_index) - d = d.repeat(FLAGS.num_epochs) - d = d.shuffle(FLAGS.shuffle_buffer_size) - d = d.interleave(tf.data.TFRecordDataset, - cycle_length=FLAGS.num_readers, block_length=1) - d = d.map(parser_fn, num_parallel_calls=FLAGS.num_map_threads) - ``` - - Args: - num_shards: A `tf.int64` scalar `tf.Tensor`, representing the number of - shards operating in parallel. - index: A `tf.int64` scalar `tf.Tensor`, representing the worker index. - - Returns: - Dataset: A `Dataset`. - - Raises: - ValueError: if `num_shards` or `index` are illegal values. Note: error - checking is done on a best-effort basis, and aren't guaranteed to be - caught upon dataset creation. (e.g. providing in a placeholder tensor - bypasses the early checking, and will instead result in an error during - a session.run call.) - """ - num_shards = ops.convert_to_tensor( - num_shards, name="num_shards", dtype=dtypes.int64) - num_shards_static = tensor_util.constant_value(num_shards) - index = ops.convert_to_tensor(index, name="index", dtype=dtypes.int64) - index_static = tensor_util.constant_value(index) - - if num_shards_static is not None and num_shards_static < 1: - raise ValueError("num_shards must be >= 1; got: %s" % num_shards_static) - if index_static is not None and index_static < 0: - raise ValueError("index must be >= 0; got: %s" % index_static) - if (index_static is not None and num_shards_static is not None and - index_static >= num_shards_static): - raise ValueError("index must be <= num_shards; %s is not < %s" % - (index_static, num_shards_static)) - - def filter_fn(elem_index, _): - mod_result = math_ops.mod(elem_index, num_shards) - return math_ops.equal(mod_result, index) - - return self._enumerate().filter(filter_fn).map(lambda _, elem: elem) - def batch(self, batch_size, drop_remainder=False): """Combines consecutive elements of this dataset into batches. @@ -906,7 +786,7 @@ class Dataset(object): batch_size: A `tf.int64` scalar `tf.Tensor`, representing the number of consecutive elements of this dataset to combine in a single batch. drop_remainder: (Optional.) A `tf.bool` scalar `tf.Tensor`, representing - whether the last batch should be dropped in the case its has fewer than + whether the last batch should be dropped in the case it has fewer than `batch_size` elements; the default behavior is not to drop the smaller batch. @@ -963,7 +843,7 @@ class Dataset(object): respective components. Defaults are `0` for numeric types and the empty string for string types. drop_remainder: (Optional.) A `tf.bool` scalar `tf.Tensor`, representing - whether the last batch should be dropped in the case its has fewer than + whether the last batch should be dropped in the case it has fewer than `batch_size` elements; the default behavior is not to drop the smaller batch. @@ -1217,7 +1097,7 @@ class Dataset(object): dataset. """ dataset = transformation_func(self) - if not isinstance(dataset, Dataset): + if not isinstance(dataset, DatasetV2): raise TypeError("`transformation_func` must return a Dataset.") dataset._input_datasets = [self] # pylint: disable=protected-access return dataset @@ -1384,10 +1264,9 @@ class Dataset(object): def with_options(self, options): """Returns a new `tf.data.Dataset` with the given options set. - The options are "global" in the sense they apply to the entire input - pipeline in which the `with_options` transformation is used. If options are - set multiple times, they are merged if possible (see - `tf.data.Options.merge()` for details). + The options are "global" in the sense they apply to the entire dataset. + If options are set multiple times, they are merged as long as different + options do not use different non-default values. Args: options: A `tf.data.Options` that identifies the options the use. @@ -1396,109 +1275,467 @@ class Dataset(object): Dataset: A `Dataset` with the given options. Raises: - ValueError: if options are set more than once + ValueError: when an option is set more than once to a non-default value """ return _OptionsDataset(self, options) -@tf_export("data.Options") -class Options(object): - """Represents options for tf.data.Dataset. +@tf_export(v1=["data.Dataset"]) +class DatasetV1(DatasetV2): + """Represents a potentially large set of elements. - An `Options` object can be for instance used to control which static - optimizations to apply or whether to use performance modeling to dynamically - tune the parallelism of operations such as `tf.data.Dataset.map` or - `tf.data.Dataset.interleave`. + A `Dataset` can be used to represent an input pipeline as a + collection of elements (nested structures of tensors) and a "logical + plan" of transformations that act on those elements. """ - for _name, _ty, _docstring in [ - ("experimental_autotune", bool, - "Whether to dynamically adjust the values of tunable parameters (e.g. " - "degrees of parallelism)."), - ("experimental_deterministic", bool, - "Whether the outputs need to be produced in deterministic order."), - ("experimental_filter_fusion", bool, - "Whether to fuse filter transformations."), - ("experimental_hoist_random_uniform", bool, - "Whether to hoist `tf.random_uniform()` ops out of map transformations." - ), - ("experimental_stats", stats_options.StatsOptions, - "Associate the given statistics options with the dataset pipeline."), - ("experimental_map_and_batch_fusion", bool, - "Whether to fuse map and batch transformations."), - ("experimental_map_and_filter_fusion", bool, - "Whether to fuse map and filter transformations."), - ("experimental_map_fusion", bool, "Whether to fuse map transformations."), - ("experimental_map_parallelization", bool, - "Whether to parallelize stateless map transformations."), - ("experimental_map_vectorization", bool, - "Whether to vectorize map transformations."), - ("experimental_noop_elimination", bool, - "Whether to eliminate no-op transformations."), - ("experimental_shuffle_and_repeat_fusion", bool, - "Whether to fuse shuffle and repeat transformations."), - ("experimental_numa_aware", bool, - "Whether to use NUMA-aware operations."), - ]: - - def _make_getter(name): # pylint: disable=no-self-argument - - def getter(self): - return getattr(self, "_" + name) - - return getter - - def _make_setter(name, ty): # pylint: disable=no-self-argument - - def setter(self, value): - if not isinstance(value, ty): - raise TypeError( - "Attempting to set the option %s to incompatible value: %r when " - "it expects %r" % (name, value, ty)) - setattr(self, "_" + name, value) - - return setter - - vars()["_" + _name] = None - vars()[_name] = property( - _make_getter(_name), _make_setter(_name, _ty), None, _docstring) def __init__(self): pass - def __eq__(self, other): - if isinstance(other, self.__class__): - return self.__dict__ == other.__dict__ - else: - return False + @deprecation.deprecated( + None, "Use `for ... in dataset:` to iterate over a dataset. If using " + "`tf.estimator`, return the `Dataset` object directly from your input " + "function. As a last resort, you can use " + "`tf.compat.v1.data.make_one_shot_iterator(dataset)`.") + def make_one_shot_iterator(self): + """Creates an `Iterator` for enumerating the elements of this dataset. - def __ne__(self, other): - return not self.__eq__(other) + Note: The returned iterator will be initialized automatically. + A "one-shot" iterator does not currently support re-initialization. + + Returns: + An `Iterator` over the elements of this dataset. + """ + if context.executing_eagerly(): + dataset = self._apply_options() + return iterator_ops.EagerIterator(dataset) + + graph_level_seed, op_level_seed = core_random_seed.get_seed(None) + + # NOTE(mrry): We capture by value here to ensure that `_make_dataset()` is + # a 0-argument function. + @function.Defun(capture_by_value=True) + def _make_dataset(): + """Factory function for a dataset.""" + # NOTE(mrry): `Defun` does not capture the graph-level seed from the + # enclosing graph, so if a graph-level seed is present we set the local + # graph seed based on a combination of the graph- and op-level seeds. + if graph_level_seed is not None: + assert op_level_seed is not None + core_random_seed.set_random_seed( + (graph_level_seed + 87654321 * op_level_seed) % (2 ** 63 - 1)) + + dataset = self._apply_options() + return dataset._as_variant_tensor() # pylint: disable=protected-access + + try: + _make_dataset.add_to_graph(ops.get_default_graph()) + except ValueError as err: + if "Cannot capture a stateful node" in str(err): + raise ValueError( + "Failed to create a one-shot iterator for a dataset. " + "`Dataset.make_one_shot_iterator()` does not support datasets that " + "capture stateful objects, such as a `Variable` or `LookupTable`. " + "In these cases, use `Dataset.make_initializable_iterator()`. " + "(Original error: %s)" % err) + else: + six.reraise(ValueError, err) + + return iterator_ops.Iterator( + gen_dataset_ops.one_shot_iterator( + dataset_factory=_make_dataset, **flat_structure(self)), + None, self.output_types, self.output_shapes, self.output_classes) + + @deprecation.deprecated( + None, "Use `for ... in dataset:` to iterate over a dataset. If using " + "`tf.estimator`, return the `Dataset` object directly from your input " + "function. As a last resort, you can use " + "`tf.compat.v1.data.make_initializable_iterator(dataset)`.") + def make_initializable_iterator(self, shared_name=None): + """Creates an `Iterator` for enumerating the elements of this dataset. + + Note: The returned iterator will be in an uninitialized state, + and you must run the `iterator.initializer` operation before using it: + + ```python + dataset = ... + iterator = dataset.make_initializable_iterator() + # ... + sess.run(iterator.initializer) + ``` + + Args: + shared_name: (Optional.) If non-empty, the returned iterator will be + shared under the given name across multiple sessions that share the + same devices (e.g. when using a remote server). + + Returns: + An `Iterator` over the elements of this dataset. + + Raises: + RuntimeError: If eager execution is enabled. + """ + if context.executing_eagerly(): + raise RuntimeError( + "dataset.make_initializable_iterator is not supported when eager " + "execution is enabled.") + dataset = self._apply_options() + if shared_name is None: + shared_name = "" + if compat.forward_compatible(2018, 8, 3): + iterator_resource = gen_dataset_ops.iterator_v2( + container="", shared_name=shared_name, **flat_structure(self)) + else: + iterator_resource = gen_dataset_ops.iterator( + container="", shared_name=shared_name, **flat_structure(self)) + with ops.colocate_with(iterator_resource): + initializer = gen_dataset_ops.make_iterator( + dataset._as_variant_tensor(), # pylint: disable=protected-access + iterator_resource) + return iterator_ops.Iterator(iterator_resource, initializer, + dataset.output_types, dataset.output_shapes, + dataset.output_classes) + + @staticmethod + @functools.wraps(DatasetV2.from_tensors) + def from_tensors(tensors): + return DatasetV1Adapter(DatasetV2.from_tensors(tensors)) + + @staticmethod + @functools.wraps(DatasetV2.from_tensor_slices) + def from_tensor_slices(tensors): + return DatasetV1Adapter(DatasetV2.from_tensor_slices(tensors)) + + @staticmethod + @deprecation.deprecated(None, "Use `tf.data.Dataset.from_tensor_slices()`.") + def from_sparse_tensor_slices(sparse_tensor): + """Splits each rank-N `tf.SparseTensor` in this dataset row-wise. + + Args: + sparse_tensor: A `tf.SparseTensor`. + + Returns: + Dataset: A `Dataset` of rank-(N-1) sparse tensors. + """ + return DatasetV1Adapter(SparseTensorSliceDataset(sparse_tensor)) + + @staticmethod + @functools.wraps(DatasetV2.from_generator) + def from_generator(generator, output_types, output_shapes=None, args=None): + return DatasetV1Adapter(DatasetV2.from_generator( + generator, output_types, output_shapes, args)) + + @staticmethod + @functools.wraps(DatasetV2.range) + def range(*args): + return DatasetV1Adapter(DatasetV2.range(*args)) + + @staticmethod + @functools.wraps(DatasetV2.zip) + def zip(datasets): + return DatasetV1Adapter(DatasetV2.zip(datasets)) + + @functools.wraps(DatasetV2.concatenate) + def concatenate(self, dataset): + return DatasetV1Adapter(super(DatasetV1, self).concatenate(dataset)) + + @functools.wraps(DatasetV2.prefetch) + def prefetch(self, buffer_size): + return DatasetV1Adapter(super(DatasetV1, self).prefetch(buffer_size)) + + @staticmethod + @functools.wraps(DatasetV2.list_files) + def list_files(file_pattern, shuffle=None, seed=None): + return DatasetV1Adapter(DatasetV2.list_files(file_pattern, shuffle, seed)) + + @functools.wraps(DatasetV2.repeat) + def repeat(self, count=None): + return DatasetV1Adapter(super(DatasetV1, self).repeat(count)) + + @functools.wraps(DatasetV2.shuffle) + def shuffle(self, buffer_size, seed=None, reshuffle_each_iteration=None): + return DatasetV1Adapter(super(DatasetV1, self).shuffle( + buffer_size, seed, reshuffle_each_iteration)) + + @functools.wraps(DatasetV2.cache) + def cache(self, filename=""): + return DatasetV1Adapter(super(DatasetV1, self).cache(filename)) + + @functools.wraps(DatasetV2.take) + def take(self, count): + return DatasetV1Adapter(super(DatasetV1, self).take(count)) + + @functools.wraps(DatasetV2.skip) + def skip(self, count): + return DatasetV1Adapter(super(DatasetV1, self).skip(count)) + + @deprecation.deprecated( + None, "Use `dataset.apply(tf.data.experimental.filter_for_shard(...))`.") + def shard(self, num_shards, index): + """Creates a `Dataset` that includes only 1/`num_shards` of this dataset. + + This dataset operator is very useful when running distributed training, as + it allows each worker to read a unique subset. + + When reading a single input file, you can skip elements as follows: + + ```python + d = tf.data.TFRecordDataset(FLAGS.input_file) + d = d.shard(FLAGS.num_workers, FLAGS.worker_index) + d = d.repeat(FLAGS.num_epochs) + d = d.shuffle(FLAGS.shuffle_buffer_size) + d = d.map(parser_fn, num_parallel_calls=FLAGS.num_map_threads) + ``` + + Important caveats: + + - Be sure to shard before you use any randomizing operator (such as + shuffle). + - Generally it is best if the shard operator is used early in the dataset + pipeline. For example, when reading from a set of TFRecord files, shard + before converting the dataset to input samples. This avoids reading every + file on every worker. The following is an example of an efficient + sharding strategy within a complete pipeline: + + ```python + d = Dataset.list_files(FLAGS.pattern) + d = d.shard(FLAGS.num_workers, FLAGS.worker_index) + d = d.repeat(FLAGS.num_epochs) + d = d.shuffle(FLAGS.shuffle_buffer_size) + d = d.interleave(tf.data.TFRecordDataset, + cycle_length=FLAGS.num_readers, block_length=1) + d = d.map(parser_fn, num_parallel_calls=FLAGS.num_map_threads) + ``` + + Args: + num_shards: A `tf.int64` scalar `tf.Tensor`, representing the number of + shards operating in parallel. + index: A `tf.int64` scalar `tf.Tensor`, representing the worker index. + + Returns: + Dataset: A `Dataset`. + + Raises: + ValueError: if `num_shards` or `index` are illegal values. Note: error + checking is done on a best-effort basis, and errors aren't guaranteed + to be caught upon dataset creation. (e.g. providing in a placeholder + tensor bypasses the early checking, and will instead result in an error + during a session.run call.) + """ + return self.apply(filter_for_shard_ops.filter_for_shard(num_shards, index)) + + @functools.wraps(DatasetV2.batch) + def batch(self, batch_size, drop_remainder=False): + return DatasetV1Adapter(super(DatasetV1, self).batch( + batch_size, drop_remainder)) + + @functools.wraps(DatasetV2.padded_batch) + def padded_batch(self, + batch_size, + padded_shapes, + padding_values=None, + drop_remainder=False): + return DatasetV1Adapter(super(DatasetV1, self).padded_batch( + batch_size, padded_shapes, padding_values, drop_remainder)) + + @functools.wraps(DatasetV2.map) + def map(self, map_func, num_parallel_calls=None): + return DatasetV1Adapter(super(DatasetV1, self).map( + map_func, num_parallel_calls)) + + @functools.wraps(DatasetV2.flat_map) + def flat_map(self, map_func): + return DatasetV1Adapter(super(DatasetV1, self).flat_map(map_func)) + + @functools.wraps(DatasetV2.interleave) + def interleave(self, + map_func, + cycle_length, + block_length=1, + num_parallel_calls=None): + return DatasetV1Adapter(super(DatasetV1, self).interleave( + map_func, cycle_length, block_length, num_parallel_calls)) + + @functools.wraps(DatasetV2.filter) + def filter(self, predicate): + return DatasetV1Adapter(super(DatasetV1, self).filter(predicate)) + + @functools.wraps(DatasetV2.apply) + def apply(self, transformation_func): + return DatasetV1Adapter(super(DatasetV1, self).apply(transformation_func)) + + @functools.wraps(DatasetV2.window) + def window(self, size, shift=None, stride=1, drop_remainder=False): + return DatasetV1Adapter(super(DatasetV1, self).window( + size, shift, stride, drop_remainder)) + + @functools.wraps(DatasetV2.with_options) + def with_options(self, options): + return DatasetV1Adapter(super(DatasetV1, self).with_options(options)) + + +# TODO(b/119044825): Until all `tf.data` unit tests are converted to V2, keep +# this alias in place. +Dataset = DatasetV1 + + +class DatasetV1Adapter(DatasetV1): + """Wraps a V2 `Dataset` object in the `tf.compat.v1.data.Dataset` API.""" + + def __init__(self, dataset): + super(DatasetV1Adapter, self).__init__() + self._dataset = dataset + + def _as_variant_tensor(self): + return self._dataset._as_variant_tensor() # pylint: disable=protected-access + + def _has_captured_ref(self): + return self._dataset._has_captured_ref() # pylint: disable=protected-access + + def _inputs(self): + return self._dataset._inputs() # pylint: disable=protected-access + + def options(self): + return self._dataset.options() + + @property + def output_classes(self): + return self._dataset.output_classes + + @property + def output_shapes(self): + return self._dataset.output_shapes + + @property + def output_types(self): + return self._dataset.output_types + + def __iter__(self): + return iter(self._dataset) + + +@tf_export(v1=["data.make_one_shot_iterator"]) +def make_one_shot_iterator(dataset): + """Creates a `tf.data.Iterator` for enumerating the elements of a dataset. + + Note: The returned iterator will be initialized automatically. + A "one-shot" iterator does not support re-initialization. + + Args: + dataset: A `tf.data.Dataset`. + + Returns: + A `tf.data.Iterator` over the elements of this dataset. + """ + try: + # Call the defined `make_one_shot_iterator()` if there is one, because some + # datasets (e.g. for prefetching) override its behavior. + return dataset.make_one_shot_iterator() + except AttributeError: + return DatasetV1Adapter(dataset).make_one_shot_iterator() + + +@tf_export(v1=["data.make_initializable_iterator"]) +def make_initializable_iterator(dataset): + """Creates a `tf.data.Iterator` for enumerating the elements of a dataset. + + Note: The returned iterator will be in an uninitialized state, + and you must run the `iterator.initializer` operation before using it: + + ```python + dataset = ... + iterator = dataset.make_initializable_iterator() + # ... + sess.run(iterator.initializer) + ``` + + Args: + dataset: A `tf.data.Dataset`. + + Returns: + A `tf.data.Iterator` over the elements of `dataset`. + + Raises: + RuntimeError: If eager execution is enabled. + """ + try: + # Call the defined `make_one_shot_iterator()` if there is one, because some + # datasets (e.g. for prefetching) override its behavior. + return dataset.make_initializable_iterator() + except AttributeError: + return DatasetV1Adapter(dataset).make_initializable_iterator() + + +@tf_export("data.Options") +class Options(options_lib.OptionsBase): + """Represents options for tf.data.Dataset. + + An `Options` object can be, for instance, used to control which static + optimizations to apply or whether to use performance modeling to dynamically + tune the parallelism of operations such as `tf.data.Dataset.map` or + `tf.data.Dataset.interleave`. + """ + + experimental_autotune = options_lib.create_option( + name="experimental_autotune", + ty=bool, + docstring= + "Whether to dynamically adjust the values of tunable parameters (e.g. " + "degrees of parallelism).") + + experimental_deterministic = options_lib.create_option( + name="experimental_deterministic", + ty=bool, + docstring= + "Whether to dynamically adjust the values of tunable parameters (e.g. " + "degrees of parallelism).") + + experimental_numa_aware = options_lib.create_option( + name="experimental_numa_aware", + ty=bool, + docstring="Whether to use NUMA-aware operations.") + + experimental_optimization = options_lib.create_option( + name="experimental_optimization", + ty=optimization_options.OptimizationOptions, + docstring="Associates the given optimization options with the dataset.") + + experimental_stats = options_lib.create_option( + name="experimental_stats", + ty=stats_options.StatsOptions, + docstring="Associates the given statistics options with the dataset.") + + experimental_threading = options_lib.create_option( + name="experimental_threading", + ty=threading_options.ThreadingOptions, + docstring="Associates the given threading options with the dataset.") def _static_optimizations(self): """Produces the list of enabled static optimizations.""" - experimental_optimizations = [ - "filter_fusion", - "hoist_random_uniform", - "map_and_batch_fusion", - "map_and_filter_fusion", - "map_fusion", - "map_parallelization", - "map_vectorization", - "noop_elimination", - "shuffle_and_repeat_fusion", - ] - result = [] - for exp_opt in experimental_optimizations: - if getattr(self, "experimental_" + exp_opt): - result.append(exp_opt) - if getattr(self, "experimental_numa_aware"): + result = [] + exp_optimization_options = self.experimental_optimization + if exp_optimization_options: + optimizations = [ + "filter_fusion", + "hoist_random_uniform", + "map_and_batch_fusion", + "map_and_filter_fusion", + "map_fusion", + "map_parallelization", + "map_vectorization", + "noop_elimination", + "shuffle_and_repeat_fusion", + ] + for optimization in optimizations: + if getattr(exp_optimization_options, optimization): + result.append(optimization) + if self.experimental_numa_aware: result.append("make_numa_aware") - if getattr(self, "experimental_deterministic") is False: + if self.experimental_deterministic is False: result.append("make_sloppy") - experimental_stats_options = getattr(self, "experimental_stats") - if experimental_stats_options and getattr(experimental_stats_options, - "latency_all_edges"): + exp_stats_options = self.experimental_stats + if exp_stats_options and exp_stats_options.latency_all_edges: result.append("latency_all_edges") return result @@ -1518,42 +1755,17 @@ class Options(object): New `tf.data.Options()` object which is the result of merging self with the input `tf.data.Options`. """ - result = Options() - for other in [self, options]: - for name in [ - "experimental_autotune", - "experimental_deterministic", - "experimental_filter_fusion", - "experimental_hoist_random_uniform", - "experimental_map_and_batch_fusion", - "experimental_map_and_filter_fusion", - "experimental_map_fusion", - "experimental_map_parallelization", - "experimental_map_vectorization", - "experimental_noop_elimination", - "experimental_numa_aware", - "experimental_shuffle_and_repeat_fusion", - "experimental_stats", - ]: - this = getattr(result, name) - that = getattr(other, name) - if that is not None: - if this is None: - setattr(result, name, that) - elif this != that: - raise ValueError( - "Cannot merge incompatible values of option: %s" % (name)) - return result + return options_lib.merge_options(self, options) -class DatasetSource(Dataset): +class DatasetSource(DatasetV2): """Abstract class representing a dataset with no inputs.""" def _inputs(self): return [] -class UnaryDataset(Dataset): +class UnaryDataset(DatasetV2): """Abstract class representing a dataset with one input.""" def __init__(self, input_dataset): @@ -1564,6 +1776,22 @@ class UnaryDataset(Dataset): return [self._input_dataset] +class UnaryUnchangedStructureDataset(UnaryDataset): + """Represents a unary dataset with the same input and output structure.""" + + @property + def output_classes(self): + return self._input_dataset.output_classes # pylint: disable=protected-access + + @property + def output_shapes(self): + return self._input_dataset.output_shapes # pylint: disable=protected-access + + @property + def output_types(self): + return self._input_dataset.output_types # pylint: disable=protected-access + + class TensorDataset(DatasetSource): """A `Dataset` with a single element, viz. a nested structure of tensors.""" @@ -1684,63 +1912,7 @@ class SparseTensorSliceDataset(DatasetSource): return (dtypes.int64, self._sparse_tensor.dtype, dtypes.int64) -class _NestedDatasetComponent(object): - """The structure of a `Dataset` nested in a component of another `Dataset`. - - A `StructuredFunctionWrapper` around a function that returns a `Dataset` as - one of its components will have a `NestedDatasetComponent` in the - corresponding position in the `output_classes`, `output_shapes`, and - `output_types` properties. - - NOTE(mrry): This class is not currently exposed via the public API. Support - for nested datasets can be enabled on a function-by-function basis by setting - `experimental_nested_dataset_support=True` in the `StructuredFunctionWrapper` - initializer. - - TODO(b/110122868): Add this class, or something equivalent, to the public API. - We are considering revising the public API for accessing Dataset structure - (`output_classes` etc.) based on experience with nested datasets and other - custom component types. - """ - - def __init__(self, - dataset=None, - output_shapes=None, - output_types=None, - output_classes=None): - if dataset is None: - if (output_classes is None or output_shapes is None or - output_types is None): - raise ValueError( - "Either `dataset`, or all of `output_classes`, " - "`output_shapes`, and `output_types` must be specified.") - self._output_classes = output_classes - self._output_shapes = output_shapes - self._output_types = output_types - else: - if not (output_classes is None and output_shapes is None and - output_types is None): - raise ValueError( - "Either `dataset`, or all of `output_classes`, " - "`output_shapes`, and `output_types` must be specified.") - self._output_classes = dataset.output_classes - self._output_shapes = dataset.output_shapes - self._output_types = dataset.output_types - - @property - def output_classes(self): - return self._output_classes - - @property - def output_shapes(self): - return self._output_shapes - - @property - def output_types(self): - return self._output_types - - -class _VariantDataset(Dataset): +class _VariantDataset(DatasetV2): """A Dataset wrapper around a `tf.variant`-typed function argument.""" def __init__(self, dataset_variant, structure): @@ -1756,15 +1928,76 @@ class _VariantDataset(Dataset): @property def output_classes(self): - return self._structure.output_classes + return self._structure._to_legacy_output_classes() # pylint: disable=protected-access @property def output_shapes(self): - return self._structure.output_shapes + return self._structure._to_legacy_output_shapes() # pylint: disable=protected-access @property def output_types(self): - return self._structure.output_types + return self._structure._to_legacy_output_types() # pylint: disable=protected-access + + +class DatasetStructure(structure_lib.Structure): + """Represents a `Dataset` of structured values.""" + + def __init__(self, element_structure): + self._element_structure = element_structure + + @property + def _flat_shapes(self): + return [tensor_shape.scalar()] + + @property + def _flat_types(self): + return [dtypes.variant] + + def is_compatible_with(self, other): + # pylint: disable=protected-access + return (isinstance(other, DatasetStructure) and + self._element_structure.is_compatible_with( + other._element_structure)) + + def _to_tensor_list(self, value): + return [value._as_variant_tensor()] # pylint: disable=protected-access + + def _from_tensor_list(self, flat_value): + if (len(flat_value) != 1 or flat_value[0].dtype != dtypes.variant or + not flat_value[0].shape.is_compatible_with(tensor_shape.scalar())): + raise ValueError( + "DatasetStructure corresponds to a single tf.variant scalar.") + return self._from_compatible_tensor_list(flat_value) + + def _from_compatible_tensor_list(self, flat_value): + # pylint: disable=protected-access + return _VariantDataset(flat_value[0], self._element_structure) + + @staticmethod + def from_value(value): + # TODO(b/110122868): We can simplify this when a `Dataset` object has a + # `Structure`-valued property. + element_structure = structure_lib.Structure._from_legacy_structure( + value.output_types, value.output_shapes, value.output_classes) + return DatasetStructure(element_structure) + + def _to_legacy_output_types(self): + return self + + def _to_legacy_output_shapes(self): + return self + + def _to_legacy_output_classes(self): + return self + + def _batch(self, batch_size): + raise NotImplementedError("Batching for `tf.data.Dataset` objects.") + + +# pylint: disable=protected-access +structure_lib.Structure._register_custom_converter(DatasetV2, + DatasetStructure.from_value) +# pylint: enable=protected-access class StructuredFunctionWrapper(object): @@ -1779,7 +2012,6 @@ class StructuredFunctionWrapper(object): input_shapes=None, input_types=None, add_to_graph=True, - experimental_nested_dataset_support=False, defun_kwargs=None): """Creates a new `StructuredFunctionWrapper` for the given function. @@ -1799,8 +2031,6 @@ class StructuredFunctionWrapper(object): argument defines the element types and structure for `func` arguments. add_to_graph: (Optional.) If `True`, the function will be added to the default graph. - experimental_nested_dataset_support: (Optional.) If `True`, the function - will support `tf.data.Dataset` objects as arguments and return values. defun_kwargs: (Optional.) A dictionary mapping string argument names to values. If supplied, will be passed to `function.Defun()` as keyword arguments. @@ -1825,6 +2055,9 @@ class StructuredFunctionWrapper(object): self._input_types = dataset.output_types self._input_classes = dataset.output_classes + self._input_structure = structure_lib.Structure._from_legacy_structure( # pylint: disable=protected-access + self._input_types, self._input_shapes, self._input_classes) + self._transformation_name = transformation_name readable_transformation_name = transformation_name.replace( ".", "_")[:-2] if len(transformation_name) > 2 else "" @@ -1832,39 +2065,18 @@ class StructuredFunctionWrapper(object): readable_transformation_name, function_utils.get_func_name(func), str(ops.uid()) - ]) - # TODO(b/110122868): Enable this support for all `tf.data` functions. - self._nested_dataset_support = experimental_nested_dataset_support - if defun_kwargs is None: defun_kwargs = {} @function.Defun( - *self._defun_args(), func_name=self._func_name, **defun_kwargs) + *self._input_structure._flat_types, func_name=self._func_name, # pylint: disable=protected-access + **defun_kwargs) def tf_data_structured_function_wrapper(*args): """Wrapper for passing nested structures to and from tf.data functions.""" - flat_args = [] - for arg, arg_class, arg_shape, arg_type in zip( - args, - nest.flatten(self._input_classes), - nest.flatten(self._input_shapes), - nest.flatten(self._input_types)): - # TODO(b/110122868): Add a registration mechanism for new component - # types. - if arg_class is sparse_tensor_lib.SparseTensor: - arg = sparse.deserialize_sparse_tensors( - arg, arg_type, arg_shape, arg_class) - arg.indices.set_shape([None, arg_shape.ndims]) - arg.dense_shape.set_shape([arg_shape.ndims]) - elif isinstance(arg_class, _NestedDatasetComponent): - assert self._nested_dataset_support - arg = _VariantDataset(arg, arg_class) - else: - arg.set_shape(arg_shape) - flat_args.append(arg) - nested_args = nest.pack_sequence_as(self._input_classes, flat_args) + # pylint: disable=protected-access + nested_args = self._input_structure._from_compatible_tensor_list(args) if not _should_unpack_args(nested_args): nested_args = (nested_args,) @@ -1882,55 +2094,14 @@ class StructuredFunctionWrapper(object): if isinstance(ret, list): ret = tuple(ret) - # Convert any `SparseTensorValue`s to `SparseTensor`s and all other - # values to tensors. - flat_ret = [] - flat_classes = [] - flat_shapes = [] - flat_types = [] - for t in nest.flatten(ret): - # TODO(b/110122868): Add a registration mechanism for new component - # types. - if sparse_tensor_lib.is_sparse(t): - t = sparse_tensor_lib.SparseTensor.from_value(t) - flat_ret.append(sparse.serialize_sparse_tensors(t)) - flat_classes.append(sparse_tensor_lib.SparseTensor) - flat_shapes.append(t.get_shape()) - flat_types.append(t.dtype) - elif isinstance(t, Dataset): - if not self._nested_dataset_support: - raise NotImplementedError( - "The %s transformation does not currently support nested " - "datasets as outputs." % self._transformation_name) - - flat_ret.append(t._as_variant_tensor()) # pylint: disable=protected-access - component = _NestedDatasetComponent(t) - flat_classes.append(component) - flat_shapes.append(component) - flat_types.append(component) - if t.options() != Options(): - warnings.warn("Encountered a nested dataset with non-default " - "options. These options will not be propagated to " - "the outer dataset.") - else: - try: - t = ops.convert_to_tensor(t) - except (ValueError, TypeError): - raise TypeError("Unsupported return value from function passed to " - "%s: %s." % (transformation_name, t)) - flat_ret.append(t) - flat_classes.append(ops.Tensor) - flat_shapes.append(t.get_shape()) - flat_types.append(t.dtype) - - ret = nest.pack_sequence_as(ret, flat_ret) - self._output_classes = nest.pack_sequence_as(ret, flat_classes) - self._output_shapes = nest.pack_sequence_as(ret, flat_shapes) - self._output_types = nest.pack_sequence_as(ret, flat_types) + try: + self._output_structure = structure_lib.Structure.from_value(ret) + except (ValueError, TypeError): + raise TypeError("Unsupported return value from function passed to " + "%s: %s." % (transformation_name, ret)) _warn_if_collections(transformation_name) - - return flat_ret + return self._output_structure._to_tensor_list(ret) self._function = tf_data_structured_function_wrapper if add_to_graph: @@ -1941,45 +2112,32 @@ class StructuredFunctionWrapper(object): # in case (e.g.) we need to rerun the function. self._function._create_definition_if_needed() # pylint: disable=protected-access - def _defun_args(self): - """Returns a flat list of `tf.DType` for the input element structure.""" - ret = [] - for input_type, input_class in zip(nest.flatten(self._input_types), - nest.flatten(self._input_classes)): - # TODO(b/110122868): Add a registration mechanism for new component types. - if input_class is sparse_tensor_lib.SparseTensor: - ret.append(dtypes.variant) - elif isinstance(input_class, _NestedDatasetComponent): - if not self._nested_dataset_support: - raise NotImplementedError( - "The %s transformation does not currently support nested " - "datasets as inputs." % self._transformation_name) - ret.append(dtypes.variant) - else: - assert isinstance(input_type, dtypes.DType) - ret.append(input_type) - return ret + @property + def output_structure(self): + return self._output_structure @property def output_classes(self): - return self._output_classes + return self._output_structure._to_legacy_output_classes() # pylint: disable=protected-access @property def output_shapes(self): - return self._output_shapes + return self._output_structure._to_legacy_output_shapes() # pylint: disable=protected-access @property def output_types(self): - return self._output_types + return self._output_structure._to_legacy_output_types() # pylint: disable=protected-access @property def function(self): return self._function -def flat_structure(dataset): +def flat_structure(dataset=None, structure=None): """Helper for setting `output_shapes` and `output_types` attrs of Dataset ops. + Either `dataset` or `structure` must be passed to this function. + Most Dataset op constructors expect `output_shapes` and `output_types` arguments that represent the flattened structure of an element. This helper function generates these attrs as a keyword argument dictionary, allowing @@ -1987,36 +2145,20 @@ def flat_structure(dataset): `**flat_structure(self)` to the op constructor. Args: - dataset: A `tf.data.Dataset`. + dataset: (Optional.) A `tf.data.Dataset`. + structure: (Optional.) A `Structure`. Returns: A dictionary of keyword arguments that can be passed to many Dataset op constructors. """ - output_classes = [] - output_shapes = [] - output_types = [] - for output_class, output_shape, output_type in zip( - nest.flatten(dataset.output_classes), nest.flatten(dataset.output_shapes), - nest.flatten(dataset.output_types)): - if isinstance(output_class, _NestedDatasetComponent): - output_classes.append(output_class.output_classes) - output_shapes.append(output_shape.output_shapes) - output_types.append(output_type.output_types) - else: - output_classes.append(output_class) - output_shapes.append(output_shape) - output_types.append(output_type) - - output_classes = nest.pack_sequence_as(dataset.output_classes, output_classes) - output_shapes = nest.pack_sequence_as(dataset.output_shapes, output_shapes) - output_types = nest.pack_sequence_as(dataset.output_types, output_types) - + # pylint: disable=protected-access + if structure is None: + structure = structure_lib.Structure._from_legacy_structure( + dataset.output_types, dataset.output_shapes, dataset.output_classes) return { - "output_shapes": - nest.flatten(sparse.as_dense_shapes(output_shapes, output_classes)), - "output_types": - nest.flatten(sparse.as_dense_types(output_types, output_classes)), + "output_shapes": structure._flat_shapes, + "output_types": structure._flat_types, } @@ -2108,14 +2250,14 @@ class _GeneratorDataset(DatasetSource): return "Dataset.from_generator()" -class ZipDataset(Dataset): +class ZipDataset(DatasetV2): """A `Dataset` that zips its inputs together.""" def __init__(self, datasets): """See `Dataset.zip()` for details.""" super(ZipDataset, self).__init__() for ds in nest.flatten(datasets): - if not isinstance(ds, Dataset): + if not isinstance(ds, DatasetV2): if isinstance(ds, list): message = ("The argument to `Dataset.zip()` must be a nested " "structure of `Dataset` objects. Nested structures do not " @@ -2155,7 +2297,7 @@ class ZipDataset(Dataset): [ds.output_types for ds in nest.flatten(self._datasets)]) -class ConcatenateDataset(Dataset): +class ConcatenateDataset(DatasetV2): """A `Dataset` that concatenates its input with given dataset.""" def __init__(self, input_dataset, dataset_to_concatenate): @@ -2210,7 +2352,7 @@ class ConcatenateDataset(Dataset): return self._output_types -class RepeatDataset(UnaryDataset): +class RepeatDataset(UnaryUnchangedStructureDataset): """A `Dataset` that repeats its input several times.""" def __init__(self, input_dataset, count): @@ -2229,18 +2371,6 @@ class RepeatDataset(UnaryDataset): count=self._count, **flat_structure(self)) - @property - def output_classes(self): - return self._input_dataset.output_classes - - @property - def output_shapes(self): - return self._input_dataset.output_shapes - - @property - def output_types(self): - return self._input_dataset.output_types - class RangeDataset(DatasetSource): """A `Dataset` of a step separated range of values.""" @@ -2290,7 +2420,7 @@ class RangeDataset(DatasetSource): return dtypes.int64 -class CacheDataset(UnaryDataset): +class CacheDataset(UnaryUnchangedStructureDataset): """A `Dataset` that caches elements of its input.""" def __init__(self, input_dataset, filename): @@ -2306,20 +2436,8 @@ class CacheDataset(UnaryDataset): filename=self._filename, **flat_structure(self)) - @property - def output_classes(self): - return self._input_dataset.output_classes - @property - def output_shapes(self): - return self._input_dataset.output_shapes - - @property - def output_types(self): - return self._input_dataset.output_types - - -class ShuffleDataset(UnaryDataset): +class ShuffleDataset(UnaryUnchangedStructureDataset): """A `Dataset` that randomly shuffles the elements of its input.""" def __init__(self, @@ -2367,20 +2485,8 @@ class ShuffleDataset(UnaryDataset): reshuffle_each_iteration=self._reshuffle_each_iteration, **flat_structure(self)) - @property - def output_classes(self): - return self._input_dataset.output_classes - @property - def output_shapes(self): - return self._input_dataset.output_shapes - - @property - def output_types(self): - return self._input_dataset.output_types - - -class TakeDataset(UnaryDataset): +class TakeDataset(UnaryUnchangedStructureDataset): """A `Dataset` containing the first `count` elements from its input.""" def __init__(self, input_dataset, count): @@ -2395,20 +2501,8 @@ class TakeDataset(UnaryDataset): count=self._count, **flat_structure(self)) - @property - def output_classes(self): - return self._input_dataset.output_classes - @property - def output_shapes(self): - return self._input_dataset.output_shapes - - @property - def output_types(self): - return self._input_dataset.output_types - - -class SkipDataset(UnaryDataset): +class SkipDataset(UnaryUnchangedStructureDataset): """A `Dataset` skipping the first `count` elements from its input.""" def __init__(self, input_dataset, count): @@ -2423,18 +2517,6 @@ class SkipDataset(UnaryDataset): count=self._count, **flat_structure(self)) - @property - def output_classes(self): - return self._input_dataset.output_classes - - @property - def output_shapes(self): - return self._input_dataset.output_shapes - - @property - def output_types(self): - return self._input_dataset.output_types - class BatchDataset(UnaryDataset): """A `Dataset` that batches contiguous elements from its input.""" @@ -2448,37 +2530,37 @@ class BatchDataset(UnaryDataset): self._drop_remainder = ops.convert_to_tensor( drop_remainder, dtype=dtypes.bool, name="drop_remainder") - def _as_variant_tensor(self): - # TODO(jsimsa): Switch to using v2 only any time after 6/30/2018. - if smart_cond.smart_constant_value(self._drop_remainder) is False: - return gen_dataset_ops.batch_dataset( - self._input_dataset._as_variant_tensor(), # pylint: disable=protected-access - batch_size=self._batch_size, - **flat_structure(self)) + # pylint: disable=protected-access + input_structure = structure_lib.Structure._from_legacy_structure( + input_dataset.output_types, input_dataset.output_shapes, + input_dataset.output_classes) + constant_drop_remainder = tensor_util.constant_value(self._drop_remainder) + if constant_drop_remainder: + # NOTE(mrry): `constant_drop_remainder` may be `None` (unknown statically) + # or `False` (explicitly retaining the remainder). + self._output_structure = input_structure._batch( + tensor_util.constant_value(self._batch_size)) else: - return gen_dataset_ops.batch_dataset_v2( - self._input_dataset._as_variant_tensor(), # pylint: disable=protected-access - batch_size=self._batch_size, - drop_remainder=self._drop_remainder, - **flat_structure(self)) + self._output_structure = input_structure._batch(None) + + def _as_variant_tensor(self): + return gen_dataset_ops.batch_dataset_v2( + self._input_dataset._as_variant_tensor(), # pylint: disable=protected-access + batch_size=self._batch_size, + drop_remainder=self._drop_remainder, + **flat_structure(structure=self._output_structure)) @property def output_classes(self): - return self._input_dataset.output_classes + return self._output_structure._to_legacy_output_classes() # pylint: disable=protected-access @property def output_shapes(self): - input_shapes = self._input_dataset.output_shapes - return nest.pack_sequence_as(input_shapes, [ - tensor_shape.vector( - tensor_util.constant_value(self._batch_size) if smart_cond. - smart_constant_value(self._drop_remainder) else None).concatenate(s) - for s in nest.flatten(self._input_dataset.output_shapes) - ]) + return self._output_structure._to_legacy_output_shapes() # pylint: disable=protected-access @property def output_types(self): - return self._input_dataset.output_types + return self._output_structure._to_legacy_output_types() # pylint: disable=protected-access def _is_padded_shape_compatible_with(padded_shape, input_component_shape): @@ -2704,66 +2786,37 @@ class MapDataset(UnaryDataset): super(MapDataset, self).__init__(input_dataset) self._input_dataset = input_dataset self._use_inter_op_parallelism = use_inter_op_parallelism - - wrapped_func = StructuredFunctionWrapper( + self._map_func = StructuredFunctionWrapper( map_func, self._transformation_name(), dataset=input_dataset) - self._output_classes = wrapped_func.output_classes - self._output_shapes = wrapped_func.output_shapes - self._output_types = wrapped_func.output_types - self._map_func = wrapped_func.function def _as_variant_tensor(self): input_t = self._input_dataset._as_variant_tensor() # pylint: disable=protected-access return gen_dataset_ops.map_dataset( input_t, - self._map_func.captured_inputs, - f=self._map_func, + self._map_func.function.captured_inputs, + f=self._map_func.function, use_inter_op_parallelism=self._use_inter_op_parallelism, - **flat_structure(self)) + **flat_structure(structure=self._map_func.output_structure)) + + def _functions(self): + return [self._map_func] @property def output_classes(self): - return self._output_classes + return self._map_func.output_classes @property def output_shapes(self): - return self._output_shapes + return self._map_func.output_shapes @property def output_types(self): - return self._output_types + return self._map_func.output_types def _transformation_name(self): return "Dataset.map()" -class MatchingFilesDataset(Dataset): - """A `Dataset` that list the files according to the input patterns.""" - - def __init__(self, patterns): - super(MatchingFilesDataset, self).__init__() - self._patterns = ops.convert_to_tensor( - patterns, dtype=dtypes.string, name="patterns") - - def _as_variant_tensor(self): - return gen_dataset_ops.matching_files_dataset(self._patterns) - - def _inputs(self): - return [] - - @property - def output_classes(self): - return ops.Tensor - - @property - def output_shapes(self): - return tensor_shape.scalar() - - @property - def output_types(self): - return dtypes.string - - class ParallelMapDataset(MapDataset): """A `Dataset` that maps a function over elements in its input in parallel.""" @@ -2780,16 +2833,15 @@ class ParallelMapDataset(MapDataset): num_parallel_calls, dtype=dtypes.int32, name="num_parallel_calls") def _as_variant_tensor(self): - input_t = self._input_dataset._as_variant_tensor() # pylint: disable=protected-access # pylint: disable=protected-access + input_t = self._input_dataset._as_variant_tensor() return gen_dataset_ops.parallel_map_dataset( input_t, - self._map_func.captured_inputs, - f=self._map_func, + self._map_func.function.captured_inputs, + f=self._map_func.function, num_parallel_calls=self._num_parallel_calls, use_inter_op_parallelism=self._use_inter_op_parallelism, - **flat_structure(self)) - # pylint: enable=protected-access + **flat_structure(structure=self._map_func.output_structure)) class FlatMapDataset(UnaryDataset): @@ -2800,36 +2852,33 @@ class FlatMapDataset(UnaryDataset): super(FlatMapDataset, self).__init__(input_dataset) self._input_dataset = input_dataset - wrapped_func = StructuredFunctionWrapper( - map_func, - self._transformation_name(), - dataset=input_dataset, - experimental_nested_dataset_support=True) - if not isinstance(wrapped_func.output_classes, _NestedDatasetComponent): + self._map_func = StructuredFunctionWrapper( + map_func, self._transformation_name(), dataset=input_dataset) + if not isinstance(self._map_func.output_structure, DatasetStructure): raise TypeError("`map_func` must return a `Dataset` object.") - self._output_classes = wrapped_func.output_classes.output_classes - self._output_types = wrapped_func.output_types.output_types - self._output_shapes = wrapped_func.output_shapes.output_shapes - self._map_func = wrapped_func.function + self._output_structure = self._map_func.output_structure._element_structure # pylint: disable=protected-access + + def _functions(self): + return [self._map_func] def _as_variant_tensor(self): return gen_dataset_ops.flat_map_dataset( self._input_dataset._as_variant_tensor(), # pylint: disable=protected-access - self._map_func.captured_inputs, - f=self._map_func, - **flat_structure(self)) + self._map_func.function.captured_inputs, + f=self._map_func.function, + **flat_structure(structure=self._output_structure)) @property def output_classes(self): - return self._output_classes + return self._output_structure._to_legacy_output_classes() # pylint: disable=protected-access @property def output_shapes(self): - return self._output_shapes + return self._output_structure._to_legacy_output_shapes() # pylint: disable=protected-access @property def output_types(self): - return self._output_types + return self._output_structure._to_legacy_output_types() # pylint: disable=protected-access def _transformation_name(self): return "Dataset.flat_map()" @@ -2848,13 +2897,14 @@ class InterleaveDataset(FlatMapDataset): block_length, dtype=dtypes.int64, name="block_length") def _as_variant_tensor(self): + # pylint: disable=protected-access return gen_dataset_ops.interleave_dataset( - self._input_dataset._as_variant_tensor(), # pylint: disable=protected-access - self._map_func.captured_inputs, # pylint: disable=protected-access + self._input_dataset._as_variant_tensor(), + self._map_func.function.captured_inputs, self._cycle_length, self._block_length, - f=self._map_func, # pylint: disable=protected-access - **flat_structure(self)) + f=self._map_func.function, + **flat_structure(structure=self._output_structure)) def _transformation_name(self): return "Dataset.interleave()" @@ -2877,20 +2927,21 @@ class ParallelInterleaveDataset(FlatMapDataset): num_parallel_calls, dtype=dtypes.int64, name="num_parallel_calls") def _as_variant_tensor(self): + # pylint: disable=protected-access return gen_dataset_ops.parallel_interleave_dataset_v2( - self._input_dataset._as_variant_tensor(), # pylint: disable=protected-access - self._map_func.captured_inputs, # pylint: disable=protected-access + self._input_dataset._as_variant_tensor(), + self._map_func.function.captured_inputs, self._cycle_length, self._block_length, self._num_parallel_calls, - f=self._map_func, # pylint: disable=protected-access - **flat_structure(self)) + f=self._map_func.function, + **flat_structure(structure=self._output_structure)) def _transformation_name(self): return "Dataset.interleave()" -class FilterDataset(UnaryDataset): +class FilterDataset(UnaryUnchangedStructureDataset): """A `Dataset` that filters its input according to a predicate function.""" def __init__(self, input_dataset, predicate): @@ -2903,32 +2954,23 @@ class FilterDataset(UnaryDataset): wrapped_func.output_types == dtypes.bool and wrapped_func.output_shapes.is_compatible_with(tensor_shape.scalar())): raise ValueError("`predicate` must return a scalar boolean tensor.") - self._predicate = wrapped_func.function + self._predicate = wrapped_func + + def _functions(self): + return [self._predicate] def _as_variant_tensor(self): return gen_dataset_ops.filter_dataset( self._input_dataset._as_variant_tensor(), # pylint: disable=protected-access - other_arguments=self._predicate.captured_inputs, - predicate=self._predicate, + other_arguments=self._predicate.function.captured_inputs, + predicate=self._predicate.function, **flat_structure(self)) - @property - def output_classes(self): - return self._input_dataset.output_classes - - @property - def output_shapes(self): - return self._input_dataset.output_shapes - - @property - def output_types(self): - return self._input_dataset.output_types - def _transformation_name(self): return "Dataset.filter()" -class PrefetchDataset(UnaryDataset): +class PrefetchDataset(UnaryUnchangedStructureDataset): """A `Dataset` that asynchronously prefetches its input.""" def __init__(self, input_dataset, buffer_size): @@ -2946,18 +2988,6 @@ class PrefetchDataset(UnaryDataset): buffer_size=self._buffer_size, **flat_structure(self)) - @property - def output_classes(self): - return self._input_dataset.output_classes - - @property - def output_shapes(self): - return self._input_dataset.output_shapes - - @property - def output_types(self): - return self._input_dataset.output_types - class WindowDataset(UnaryDataset): """A dataset that creates window datasets from the input elements.""" @@ -2975,10 +3005,9 @@ class WindowDataset(UnaryDataset): self._output_classes = nest.pack_sequence_as( input_dataset.output_classes, [ - _NestedDatasetComponent( # pylint: disable=protected-access - output_classes=output_class, - output_shapes=output_shape, - output_types=output_type) + DatasetStructure( + structure_lib.Structure._from_legacy_structure( # pylint: disable=protected-access + output_type, output_shape, output_class)) for output_class, output_shape, output_type in zip( nest.flatten(input_dataset.output_classes), nest.flatten(input_dataset.output_shapes), @@ -3009,7 +3038,7 @@ class WindowDataset(UnaryDataset): return self._output_types -class _OptionsDataset(UnaryDataset): +class _OptionsDataset(UnaryUnchangedStructureDataset): """An identity `Dataset` that stores options.""" def __init__(self, input_dataset, options): @@ -3027,20 +3056,8 @@ class _OptionsDataset(UnaryDataset): def options(self): return self._options - @property - def output_classes(self): - return self._input_dataset.output_classes - @property - def output_shapes(self): - return self._input_dataset.output_shapes - - @property - def output_types(self): - return self._input_dataset.output_types - - -class _ModelDataset(UnaryDataset): +class _ModelDataset(UnaryUnchangedStructureDataset): """A `Dataset` that acts as an identity, and models performance.""" def __init__(self, input_dataset): @@ -3053,20 +3070,8 @@ class _ModelDataset(UnaryDataset): self._input_dataset._as_variant_tensor(), # pylint: disable=protected-access **flat_structure(self)) - @property - def output_classes(self): - return self._input_dataset.output_classes - @property - def output_shapes(self): - return self._input_dataset.output_shapes - - @property - def output_types(self): - return self._input_dataset.output_types - - -class _OptimizeDataset(UnaryDataset): +class _OptimizeDataset(UnaryUnchangedStructureDataset): """A `Dataset` that acts as an identity, and applies optimizations.""" def __init__(self, input_dataset, optimizations): @@ -3084,21 +3089,9 @@ class _OptimizeDataset(UnaryDataset): self._optimizations, **flat_structure(self)) - @property - def output_classes(self): - return self._input_dataset.output_classes - @property - def output_shapes(self): - return self._input_dataset.output_shapes - - @property - def output_types(self): - return self._input_dataset.output_types - - -class _SetStatsAggregatorDataset(UnaryDataset): - """A `Dataset` that acts as an identity, and sets stats aggregator.""" +class _SetStatsAggregatorDataset(UnaryUnchangedStructureDataset): + """A `Dataset` that acts as an identity, and sets a stats aggregator.""" def __init__(self, input_dataset, aggregator, prefix, counter_prefix): super(_SetStatsAggregatorDataset, self).__init__(input_dataset) @@ -3108,21 +3101,43 @@ class _SetStatsAggregatorDataset(UnaryDataset): self._counter_prefix = counter_prefix def _as_variant_tensor(self): - return gen_dataset_ops.set_stats_aggregator_dataset( + return ged_ops.experimental_set_stats_aggregator_dataset( self._input_dataset._as_variant_tensor(), # pylint: disable=protected-access self._stats_aggregator._resource, # pylint: disable=protected-access self._prefix, self._counter_prefix, **flat_structure(self)) - @property - def output_shapes(self): - return self._input_dataset.output_shapes - @property - def output_types(self): - return self._input_dataset.output_types +class _MaxIntraOpParallelismDataset(UnaryUnchangedStructureDataset): + """A `Dataset` that acts as an identity, overriding intra-op parallelism.""" - @property - def output_classes(self): - return self._input_dataset.output_classes + def __init__(self, input_dataset, max_intra_op_parallelism): + super(_MaxIntraOpParallelismDataset, self).__init__(input_dataset) + self._input_dataset = input_dataset + self._max_intra_op_parallelism = ops.convert_to_tensor( + max_intra_op_parallelism, + dtype=dtypes.int64, + name="max_intra_op_parallelism") + + def _as_variant_tensor(self): + return ged_ops.experimental_max_intra_op_parallelism_dataset( + self._input_dataset._as_variant_tensor(), # pylint: disable=protected-access + self._max_intra_op_parallelism, + **flat_structure(self)) + + +class _PrivateThreadPoolDataset(UnaryUnchangedStructureDataset): + """A `Dataset` that acts as an identity, setting a private threadpool.""" + + def __init__(self, input_dataset, num_threads): + super(_PrivateThreadPoolDataset, self).__init__(input_dataset) + self._input_dataset = input_dataset + self._num_threads = ops.convert_to_tensor( + num_threads, dtype=dtypes.int64, name="num_threads") + + def _as_variant_tensor(self): + return ged_ops.experimental_private_thread_pool_dataset( + self._input_dataset._as_variant_tensor(), # pylint: disable=protected-access + self._num_threads, + **flat_structure(self)) diff --git a/tensorflow/python/data/ops/iterator_ops.py b/tensorflow/python/data/ops/iterator_ops.py index 68b03ba93be..e2ca64c8025 100644 --- a/tensorflow/python/data/ops/iterator_ops.py +++ b/tensorflow/python/data/ops/iterator_ops.py @@ -68,7 +68,7 @@ def _device_stack_is_empty(): return not bool(device_stack) -@tf_export("data.Iterator") +@tf_export(v1=["data.Iterator"]) class Iterator(checkpointable.CheckpointableBase): """Represents the state of iterating through a `Dataset`.""" diff --git a/tensorflow/python/data/ops/optional_ops.py b/tensorflow/python/data/ops/optional_ops.py index 91cf883ce94..15ec755c676 100644 --- a/tensorflow/python/data/ops/optional_ops.py +++ b/tensorflow/python/data/ops/optional_ops.py @@ -183,19 +183,17 @@ class OptionalStructure(structure.Structure): return OptionalStructure(value.value_structure) def _to_legacy_output_types(self): - raise NotImplementedError("The `output_types` property is not supported on " - "structured objects containing an `Optional`. " - "Use the corresponding `structure` property.") + return self def _to_legacy_output_shapes(self): - raise NotImplementedError("The `output_shapes` property is not supported on" - " structured objects containing an `Optional`. " - "Use the corresponding `structure` property.") + return self def _to_legacy_output_classes(self): - raise NotImplementedError("The `output_classes` property is not supported " - "on structured objects containing an `Optional`. " - "Use the corresponding `structure` property.") + return self + + def _batch(self, batch_size): + raise NotImplementedError( + "Batching for `tf.data.experimental.Optional` objects.") # pylint: disable=protected-access diff --git a/tensorflow/python/data/ops/readers.py b/tensorflow/python/data/ops/readers.py index 7e165a052d7..70a3b1b1cb3 100644 --- a/tensorflow/python/data/ops/readers.py +++ b/tensorflow/python/data/ops/readers.py @@ -25,6 +25,7 @@ from tensorflow.python.framework import ops from tensorflow.python.framework import tensor_shape from tensorflow.python.ops import array_ops from tensorflow.python.ops import gen_dataset_ops +from tensorflow.python.ops import gen_experimental_dataset_ops as ged_ops from tensorflow.python.util.tf_export import tf_export @@ -32,8 +33,8 @@ from tensorflow.python.util.tf_export import tf_export _DEFAULT_READER_BUFFER_SIZE_BYTES = 256 * 1024 # 256 KB -@tf_export("data.TextLineDataset") -class TextLineDataset(dataset_ops.DatasetSource): +@tf_export("data.TextLineDataset", v1=[]) +class TextLineDatasetV2(dataset_ops.DatasetSource): """A `Dataset` comprising lines from one or more text files.""" def __init__(self, filenames, compression_type=None, buffer_size=None): @@ -47,7 +48,7 @@ class TextLineDataset(dataset_ops.DatasetSource): to buffer. A value of 0 results in the default buffering values chosen based on the compression type. """ - super(TextLineDataset, self).__init__() + super(TextLineDatasetV2, self).__init__() self._filenames = ops.convert_to_tensor( filenames, dtype=dtypes.string, name="filenames") self._compression_type = convert.optional_param_to_tensor( @@ -75,6 +76,24 @@ class TextLineDataset(dataset_ops.DatasetSource): return dtypes.string +@tf_export(v1=["data.TextLineDataset"]) +class TextLineDatasetV1(dataset_ops.DatasetV1Adapter): + """A `Dataset` comprising lines from one or more text files.""" + + def __init__(self, filenames, compression_type=None, buffer_size=None): + wrapped = TextLineDatasetV2(filenames, compression_type, buffer_size) + super(TextLineDatasetV1, self).__init__(wrapped) + __init__.__doc__ = TextLineDatasetV2.__init__.__doc__ + + @property + def _filenames(self): + return self._dataset._filenames # pylint: disable=protected-access + + @_filenames.setter + def _filenames(self, value): + self._dataset._filenames = value # pylint: disable=protected-access + + class _TFRecordDataset(dataset_ops.DatasetSource): """A `Dataset` comprising records from one or more TFRecord files.""" @@ -140,29 +159,28 @@ class ParallelInterleaveDataset(dataset_ops.InterleaveDataset): def _as_variant_tensor(self): # pylint: disable=protected-access - return gen_dataset_ops.parallel_interleave_dataset( + return ged_ops.experimental_parallel_interleave_dataset( self._input_dataset._as_variant_tensor(), - self._map_func.captured_inputs, + self._map_func.function.captured_inputs, self._cycle_length, self._block_length, self._sloppy, self._buffer_output_elements, self._prefetch_input_elements, - f=self._map_func, - **dataset_ops.flat_structure(self)) - # pylint: enable=protected-access + f=self._map_func.function, + **dataset_ops.flat_structure(structure=self._output_structure)) def _transformation_name(self): return "tf.data.experimental.parallel_interleave()" -@tf_export("data.TFRecordDataset") -class TFRecordDataset(dataset_ops.Dataset): +@tf_export("data.TFRecordDataset", v1=[]) +class TFRecordDatasetV2(dataset_ops.DatasetV2): """A `Dataset` comprising records from one or more TFRecord files.""" def __init__(self, filenames, compression_type=None, buffer_size=None, num_parallel_reads=None): - """Creates a `TFRecordDataset` to read for one or more TFRecord files. + """Creates a `TFRecordDataset` to read one or more TFRecord files. NOTE: The `num_parallel_reads` argument can be used to improve performance when reading from a remote filesystem. @@ -182,8 +200,8 @@ class TFRecordDataset(dataset_ops.Dataset): TypeError: If any argument does not have the expected type. ValueError: If any argument does not have the expected shape. """ - super(TFRecordDataset, self).__init__() - if isinstance(filenames, dataset_ops.Dataset): + super(TFRecordDatasetV2, self).__init__() + if isinstance(filenames, dataset_ops.DatasetV2): if filenames.output_types != dtypes.string: raise TypeError( "`filenames` must be a `tf.data.Dataset` of `tf.string` elements.") @@ -194,7 +212,7 @@ class TFRecordDataset(dataset_ops.Dataset): else: filenames = ops.convert_to_tensor(filenames, dtype=dtypes.string) filenames = array_ops.reshape(filenames, [-1], name="flat_filenames") - filenames = dataset_ops.Dataset.from_tensor_slices(filenames) + filenames = dataset_ops.DatasetV2.from_tensor_slices(filenames) self._filenames = filenames self._compression_type = compression_type @@ -217,10 +235,10 @@ class TFRecordDataset(dataset_ops.Dataset): compression_type=None, buffer_size=None, num_parallel_reads=None): - return TFRecordDataset(filenames or self._filenames, - compression_type or self._compression_type, - buffer_size or self._buffer_size, - num_parallel_reads or self._num_parallel_reads) + return TFRecordDatasetV2(filenames or self._filenames, + compression_type or self._compression_type, + buffer_size or self._buffer_size, + num_parallel_reads or self._num_parallel_reads) def _as_variant_tensor(self): return self._impl._as_variant_tensor() # pylint: disable=protected-access @@ -241,8 +259,40 @@ class TFRecordDataset(dataset_ops.Dataset): return self._impl.output_types -@tf_export("data.FixedLengthRecordDataset") -class FixedLengthRecordDataset(dataset_ops.DatasetSource): +@tf_export(v1=["data.TFRecordDataset"]) +class TFRecordDatasetV1(dataset_ops.DatasetV1Adapter): + """A `Dataset` comprising records from one or more TFRecord files.""" + + def __init__(self, filenames, compression_type=None, buffer_size=None, + num_parallel_reads=None): + wrapped = TFRecordDatasetV2( + filenames, compression_type, buffer_size, num_parallel_reads) + super(TFRecordDatasetV1, self).__init__(wrapped) + __init__.__doc__ = TFRecordDatasetV2.__init__.__doc__ + + def _clone(self, + filenames=None, + compression_type=None, + buffer_size=None, + num_parallel_reads=None): + # pylint: disable=protected-access + return TFRecordDatasetV1( + filenames or self._dataset._filenames, + compression_type or self._dataset._compression_type, + buffer_size or self._dataset._buffer_size, + num_parallel_reads or self._dataset._num_parallel_reads) + + @property + def _filenames(self): + return self._dataset._filenames # pylint: disable=protected-access + + @_filenames.setter + def _filenames(self, value): + self._dataset._filenames = value # pylint: disable=protected-access + + +@tf_export("data.FixedLengthRecordDataset", v1=[]) +class FixedLengthRecordDatasetV2(dataset_ops.DatasetSource): """A `Dataset` of fixed-length records from one or more binary files.""" def __init__(self, @@ -267,7 +317,7 @@ class FixedLengthRecordDataset(dataset_ops.DatasetSource): compression_type: (Optional.) A `tf.string` scalar evaluating to one of `""` (no compression), `"ZLIB"`, or `"GZIP"`. """ - super(FixedLengthRecordDataset, self).__init__() + super(FixedLengthRecordDatasetV2, self).__init__() self._filenames = ops.convert_to_tensor( filenames, dtype=dtypes.string, name="filenames") self._record_bytes = ops.convert_to_tensor( @@ -296,7 +346,6 @@ class FixedLengthRecordDataset(dataset_ops.DatasetSource): self._filenames, self._header_bytes, self._record_bytes, self._footer_bytes, self._buffer_size) - @property def output_classes(self): return ops.Tensor @@ -308,3 +357,36 @@ class FixedLengthRecordDataset(dataset_ops.DatasetSource): @property def output_types(self): return dtypes.string + + +@tf_export(v1=["data.FixedLengthRecordDataset"]) +class FixedLengthRecordDatasetV1(dataset_ops.DatasetV1Adapter): + """A `Dataset` of fixed-length records from one or more binary files.""" + + def __init__(self, + filenames, + record_bytes, + header_bytes=None, + footer_bytes=None, + buffer_size=None, + compression_type=None): + wrapped = FixedLengthRecordDatasetV2( + filenames, record_bytes, header_bytes, footer_bytes, buffer_size, + compression_type) + super(FixedLengthRecordDatasetV1, self).__init__(wrapped) + __init__.__doc__ = FixedLengthRecordDatasetV2.__init__.__doc__ + + @property + def _filenames(self): + return self._dataset._filenames # pylint: disable=protected-access + + @_filenames.setter + def _filenames(self, value): + self._dataset._filenames = value # pylint: disable=protected-access + + +# TODO(b/119044825): Until all `tf.data` unit tests are converted to V2, keep +# these aliases in place. +FixedLengthRecordDataset = FixedLengthRecordDatasetV1 +TFRecordDataset = TFRecordDatasetV1 +TextLineDataset = TextLineDatasetV1 diff --git a/tensorflow/python/data/util/BUILD b/tensorflow/python/data/util/BUILD index 39082ce3707..f15ebc32a83 100644 --- a/tensorflow/python/data/util/BUILD +++ b/tensorflow/python/data/util/BUILD @@ -97,6 +97,23 @@ py_test( ], ) +py_library( + name = "options", + srcs = ["options.py"], + srcs_version = "PY2AND3", +) + +py_test( + name = "options_test", + size = "small", + srcs = ["options_test.py"], + srcs_version = "PY2AND3", + deps = [ + ":options", + "//tensorflow/python:client_testlib", + ], +) + py_library( name = "convert", srcs = ["convert.py"], diff --git a/tensorflow/python/data/util/convert_test.py b/tensorflow/python/data/util/convert_test.py index 89c3afb2969..78ca6e95139 100644 --- a/tensorflow/python/data/util/convert_test.py +++ b/tensorflow/python/data/util/convert_test.py @@ -22,6 +22,7 @@ from tensorflow.python.data.util import convert from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes from tensorflow.python.framework import tensor_shape +from tensorflow.python.framework import test_util from tensorflow.python.platform import test from tensorflow.python.util import compat @@ -30,47 +31,53 @@ class ConvertTest(test.TestCase): def testInteger(self): resp = convert.optional_param_to_tensor("foo", 3) - with self.cached_session() as sess: - self.assertEqual(3, sess.run(resp)) + self.assertEqual(3, self.evaluate(resp)) def testIntegerDefault(self): resp = convert.optional_param_to_tensor("foo", None) - with self.cached_session() as sess: - self.assertEqual(0, sess.run(resp)) + self.assertEqual(0, self.evaluate(resp)) def testStringDefault(self): resp = convert.optional_param_to_tensor("bar", None, "default", dtypes.string) - with self.cached_session() as sess: - self.assertEqual(compat.as_bytes("default"), sess.run(resp)) + self.assertEqual(compat.as_bytes("default"), self.evaluate(resp)) def testString(self): resp = convert.optional_param_to_tensor("bar", "value", "default", dtypes.string) - with self.cached_session() as sess: - self.assertEqual(compat.as_bytes("value"), sess.run(resp)) + self.assertEqual(compat.as_bytes("value"), self.evaluate(resp)) def testPartialShapeToTensorKnownDimension(self): - with self.cached_session() as sess: - self.assertAllEqual([1], sess.run(convert.partial_shape_to_tensor( - tensor_shape.TensorShape([1])))) - self.assertAllEqual([1], sess.run(convert.partial_shape_to_tensor((1,)))) - self.assertAllEqual([1], sess.run(convert.partial_shape_to_tensor([1]))) - self.assertAllEqual([1], sess.run(convert.partial_shape_to_tensor( - constant_op.constant([1], dtype=dtypes.int64)))) + self.assertAllEqual([1], + self.evaluate( + convert.partial_shape_to_tensor( + tensor_shape.TensorShape([1])))) + self.assertAllEqual([1], self.evaluate( + convert.partial_shape_to_tensor((1,)))) + self.assertAllEqual([1], self.evaluate( + convert.partial_shape_to_tensor([1]))) + self.assertAllEqual([1], + self.evaluate( + convert.partial_shape_to_tensor( + constant_op.constant([1], dtype=dtypes.int64)))) + @test_util.run_deprecated_v1 def testPartialShapeToTensorUnknownDimension(self): - with self.cached_session() as sess: - self.assertAllEqual([-1], sess.run(convert.partial_shape_to_tensor( - tensor_shape.TensorShape([None])))) - self.assertAllEqual([-1], sess.run(convert.partial_shape_to_tensor( - (None,)))) - self.assertAllEqual([-1], sess.run(convert.partial_shape_to_tensor( - [None]))) - self.assertAllEqual([-1], sess.run(convert.partial_shape_to_tensor( - [-1]))) - self.assertAllEqual([-1], sess.run(convert.partial_shape_to_tensor( - constant_op.constant([-1], dtype=dtypes.int64)))) + self.assertAllEqual([-1], + self.evaluate( + convert.partial_shape_to_tensor( + tensor_shape.TensorShape([None])))) + self.assertAllEqual([-1], + self.evaluate(convert.partial_shape_to_tensor((None,)))) + self.assertAllEqual([-1], + self.evaluate(convert.partial_shape_to_tensor([None]))) + self.assertAllEqual([-1], + self.evaluate(convert.partial_shape_to_tensor([-1]))) + self.assertAllEqual([-1], + self.evaluate( + convert.partial_shape_to_tensor( + constant_op.constant([-1], + dtype=dtypes.int64)))) with self.assertRaisesRegexp( ValueError, r"The given shape .* must be a 1-D tensor of tf.int64 " @@ -84,42 +91,63 @@ class ConvertTest(test.TestCase): convert.partial_shape_to_tensor(constant_op.constant([1., 1.])) def testPartialShapeToTensorMultipleDimensions(self): - with self.cached_session() as sess: - self.assertAllEqual([3, 6], sess.run(convert.partial_shape_to_tensor( - tensor_shape.TensorShape([3, 6])))) - self.assertAllEqual([3, 6], sess.run(convert.partial_shape_to_tensor( - (3, 6)))) - self.assertAllEqual([3, 6], sess.run(convert.partial_shape_to_tensor( - [3, 6]))) - self.assertAllEqual([3, 6], sess.run(convert.partial_shape_to_tensor( - constant_op.constant([3, 6], dtype=dtypes.int64)))) + self.assertAllEqual([3, 6], + self.evaluate( + convert.partial_shape_to_tensor( + tensor_shape.TensorShape([3, 6])))) + self.assertAllEqual([3, 6], + self.evaluate(convert.partial_shape_to_tensor((3, 6)))) + self.assertAllEqual([3, 6], + self.evaluate(convert.partial_shape_to_tensor([3, 6]))) + self.assertAllEqual([3, 6], + self.evaluate( + convert.partial_shape_to_tensor( + constant_op.constant([3, 6], + dtype=dtypes.int64)))) - self.assertAllEqual([3, -1], sess.run(convert.partial_shape_to_tensor( - tensor_shape.TensorShape([3, None])))) - self.assertAllEqual([3, -1], sess.run(convert.partial_shape_to_tensor( - (3, None)))) - self.assertAllEqual([3, -1], sess.run(convert.partial_shape_to_tensor( - [3, None]))) - self.assertAllEqual([3, -1], sess.run(convert.partial_shape_to_tensor( - constant_op.constant([3, -1], dtype=dtypes.int64)))) + self.assertAllEqual([3, -1], + self.evaluate( + convert.partial_shape_to_tensor( + tensor_shape.TensorShape([3, None])))) + self.assertAllEqual([3, -1], + self.evaluate( + convert.partial_shape_to_tensor((3, None)))) + self.assertAllEqual([3, -1], + self.evaluate( + convert.partial_shape_to_tensor([3, None]))) + self.assertAllEqual([3, -1], + self.evaluate( + convert.partial_shape_to_tensor( + constant_op.constant([3, -1], + dtype=dtypes.int64)))) - self.assertAllEqual([-1, -1], sess.run(convert.partial_shape_to_tensor( - tensor_shape.TensorShape([None, None])))) - self.assertAllEqual([-1, -1], sess.run(convert.partial_shape_to_tensor( - (None, None)))) - self.assertAllEqual([-1, -1], sess.run(convert.partial_shape_to_tensor( - [None, None]))) - self.assertAllEqual([-1, -1], sess.run(convert.partial_shape_to_tensor( - constant_op.constant([-1, -1], dtype=dtypes.int64)))) + self.assertAllEqual([-1, -1], + self.evaluate( + convert.partial_shape_to_tensor( + tensor_shape.TensorShape([None, None])))) + self.assertAllEqual([-1, -1], + self.evaluate( + convert.partial_shape_to_tensor((None, None)))) + self.assertAllEqual([-1, -1], + self.evaluate( + convert.partial_shape_to_tensor([None, None]))) + self.assertAllEqual([-1, -1], + self.evaluate( + convert.partial_shape_to_tensor( + constant_op.constant([-1, -1], + dtype=dtypes.int64)))) def testPartialShapeToTensorScalar(self): - with self.cached_session() as sess: - self.assertAllEqual([], sess.run(convert.partial_shape_to_tensor( - tensor_shape.TensorShape([])))) - self.assertAllEqual([], sess.run(convert.partial_shape_to_tensor(()))) - self.assertAllEqual([], sess.run(convert.partial_shape_to_tensor([]))) - self.assertAllEqual([], sess.run(convert.partial_shape_to_tensor( - constant_op.constant([], dtype=dtypes.int64)))) + self.assertAllEqual([], + self.evaluate( + convert.partial_shape_to_tensor( + tensor_shape.TensorShape([])))) + self.assertAllEqual([], self.evaluate(convert.partial_shape_to_tensor(()))) + self.assertAllEqual([], self.evaluate(convert.partial_shape_to_tensor([]))) + self.assertAllEqual([], + self.evaluate( + convert.partial_shape_to_tensor( + constant_op.constant([], dtype=dtypes.int64)))) if __name__ == "__main__": diff --git a/tensorflow/python/data/util/options.py b/tensorflow/python/data/util/options.py new file mode 100644 index 00000000000..9badba8e567 --- /dev/null +++ b/tensorflow/python/data/util/options.py @@ -0,0 +1,131 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Utilities for tf.data options.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + + +def _internal_attr_name(name): + return "_" + name + + +class OptionsBase(object): + """Base class for representing a set of tf.data options. + + Attributes: + _options: Stores the option values. + """ + + def __init__(self): + self._options = {} + + def __eq__(self, other): + if not isinstance(other, self.__class__): + return NotImplemented + for name in set(self._options) | set(other._options): # pylint: disable=protected-access + if getattr(self, name) != getattr(other, name): + return False + return True + + def __ne__(self, other): + if isinstance(other, self.__class__): + return not self.__eq__(other) + else: + return NotImplemented + + +def create_option(name, ty, docstring, default=None): + """Creates a type-checked property. + + Args: + name: the name to use + ty: the type to use + docstring: the docstring to use + default: the default value to use + + Returns: + A type-checked property. + """ + + def get_fn(self): + return self._options.get(name, default) # pylint: disable=protected-access + + def set_fn(self, value): + if not isinstance(value, ty): + raise TypeError("Property \"%s\" must be of type %s, got: %r (type: %r)" % + (name, ty, value, type(value))) + self._options[name] = value # pylint: disable=protected-access + + return property(get_fn, set_fn, None, docstring) + + +def merge_options(*options_list): + """Merges the given options, returning the result as a new options object. + + The input arguments are expected to have a matching type that derives from + `OptionsBase` (and thus each represent a set of options). The method outputs + an object of the same type created by merging the sets of options represented + by the input arguments. + + The sets of options can be merged as long as there does not exist an option + with different non-default values. + + If an option is an instance of `OptionsBase` itself, then this method is + applied recursively to the set of options represented by this option. + + Args: + *options_list: options to merge + + Raises: + TypeError: if the input arguments are incompatible or not derived from + `OptionsBase` + ValueError: if the given options cannot be merged + + Returns: + A new options object which is the result of merging the given options. + """ + if len(options_list) < 1: + raise ValueError("At least one options should be provided") + result_type = type(options_list[0]) + + for options in options_list: + if not isinstance(options, result_type): + raise TypeError("Incompatible options type: %r vs %r" % (type(options), + result_type)) + + if not isinstance(options_list[0], OptionsBase): + raise TypeError("The inputs should inherit from `OptionsBase`") + + default_options = result_type() + result = result_type() + for options in options_list: + # Iterate over all set options and merge the into the result. + for name in options._options: # pylint: disable=protected-access + this = getattr(result, name) + that = getattr(options, name) + default = getattr(default_options, name) + if that == default: + continue + elif this == default: + setattr(result, name, that) + elif isinstance(this, OptionsBase): + setattr(result, name, merge_options(this, that)) + elif this != that: + raise ValueError( + "Cannot merge incompatible values (%r and %r) of option: %s" % + (this, that, name)) + return result diff --git a/tensorflow/python/data/util/options_test.py b/tensorflow/python/data/util/options_test.py new file mode 100644 index 00000000000..c5169835a32 --- /dev/null +++ b/tensorflow/python/data/util/options_test.py @@ -0,0 +1,96 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Tests for dataset options utilities.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from tensorflow.python.data.util import options +from tensorflow.python.platform import test + + +class _TestOptions(options.OptionsBase): + x = options.create_option( + name="x", ty=int, docstring="the answer to everything", default=42) + y = options.create_option( + name="y", ty=float, docstring="a tasty pie", default=3.14) + + +class _NestedTestOptions(options.OptionsBase): + opts = options.create_option( + name="opts", ty=_TestOptions, docstring="nested options") + + +class OptionsTest(test.TestCase): + + def testDocumentation(self): + self.assertEqual(_TestOptions.x.__doc__, "the answer to everything") + self.assertEqual(_TestOptions.y.__doc__, "a tasty pie") + + def testCreateOption(self): + opts = _TestOptions() + self.assertEqual(opts.x, 42) + self.assertEqual(opts.y, 3.14) + self.assertIsInstance(opts.x, int) + self.assertIsInstance(opts.y, float) + opts.x = 0 + self.assertEqual(opts.x, 0) + with self.assertRaises(TypeError): + opts.x = 3.14 + opts.y = 0.0 + self.assertEqual(opts.y, 0.0) + with self.assertRaises(TypeError): + opts.y = 42 + + def testMergeOptions(self): + options1, options2 = _TestOptions(), _TestOptions() + with self.assertRaises(ValueError): + options.merge_options() + merged_options = options.merge_options(options1, options2) + self.assertEqual(merged_options.x, 42) + self.assertEqual(merged_options.y, 3.14) + options1.x = 0 + options2.y = 0.0 + merged_options = options.merge_options(options1, options2) + self.assertEqual(merged_options.x, 0) + self.assertEqual(merged_options.y, 0.0) + + def testMergeNestedOptions(self): + options1, options2 = _NestedTestOptions(), _NestedTestOptions() + merged_options = options.merge_options(options1, options2) + self.assertEqual(merged_options.opts, None) + options1.opts = _TestOptions() + merged_options = options.merge_options(options1, options2) + self.assertEqual(merged_options.opts, _TestOptions()) + options2.opts = _TestOptions() + merged_options = options.merge_options(options1, options2) + self.assertEqual(merged_options.opts, _TestOptions()) + options1.opts.x = 0 + options2.opts.y = 0.0 + merged_options = options.merge_options(options1, options2) + self.assertEqual(merged_options.opts.x, 0) + self.assertEqual(merged_options.opts.y, 0.0) + + def testMergeOptionsInvalid(self): + with self.assertRaises(TypeError): + options.merge_options(0) + options1, options2 = _TestOptions(), _NestedTestOptions() + with self.assertRaises(TypeError): + options.merge_options(options1, options2) + + +if __name__ == "__main__": + test.main() diff --git a/tensorflow/python/data/util/sparse.py b/tensorflow/python/data/util/sparse.py index 5e6d2247097..f2e22fefd31 100644 --- a/tensorflow/python/data/util/sparse.py +++ b/tensorflow/python/data/util/sparse.py @@ -34,7 +34,7 @@ def any_sparse(classes): Returns: `True` if `classes` contains a sparse tensor type and `False` otherwise. """ - return any([c is sparse_tensor.SparseTensor for c in nest.flatten(classes)]) + return any(c is sparse_tensor.SparseTensor for c in nest.flatten(classes)) def as_dense_shapes(shapes, classes): diff --git a/tensorflow/python/data/util/sparse_test.py b/tensorflow/python/data/util/sparse_test.py index 056b32480f3..06acf55ab9d 100644 --- a/tensorflow/python/data/util/sparse_test.py +++ b/tensorflow/python/data/util/sparse_test.py @@ -25,6 +25,7 @@ from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops from tensorflow.python.framework import sparse_tensor from tensorflow.python.framework import tensor_shape +from tensorflow.python.framework import test_util from tensorflow.python.platform import test @@ -292,10 +293,11 @@ class SparseTest(test.TestCase): return self.assertTrue(isinstance(b, sparse_tensor.SparseTensor)) with self.cached_session(): - self.assertAllEqual(a.eval().indices, b.eval().indices) - self.assertAllEqual(a.eval().values, b.eval().values) - self.assertAllEqual(a.eval().dense_shape, b.eval().dense_shape) + self.assertAllEqual(a.eval().indices, self.evaluate(b).indices) + self.assertAllEqual(a.eval().values, self.evaluate(b).values) + self.assertAllEqual(a.eval().dense_shape, self.evaluate(b).dense_shape) + @test_util.run_deprecated_v1 def testSerializeDeserialize(self): test_cases = ( (), @@ -325,6 +327,7 @@ class SparseTest(test.TestCase): for a, e in zip(nest.flatten(actual), nest.flatten(expected)): self.assertSparseValuesEqual(a, e) + @test_util.run_deprecated_v1 def testSerializeManyDeserialize(self): test_cases = ( (), diff --git a/tensorflow/python/data/util/structure.py b/tensorflow/python/data/util/structure.py index 9a3118297db..874e8abc6df 100644 --- a/tensorflow/python/data/util/structure.py +++ b/tensorflow/python/data/util/structure.py @@ -144,6 +144,19 @@ class Structure(object): """ return self._from_tensor_list(flat_value) + @abc.abstractmethod + def _batch(self, batch_size): + """Returns a structure representing a batch of objects with this structure. + + Args: + batch_size: An `int` representing the number of elements in a batch, + or `None` if the batch size may vary. + + Returns: + A `Structure` representing a batch of objects with this structure. + """ + raise NotImplementedError("Structure._batch()") + @staticmethod def from_value(value): """Returns a `Structure` that represents the given `value`. @@ -208,14 +221,16 @@ class Structure(object): flat_ret = [] for flat_type, flat_shape, flat_class in zip(flat_types, flat_shapes, flat_classes): - if issubclass(flat_class, sparse_tensor_lib.SparseTensor): + if isinstance(flat_class, Structure): + flat_ret.append(flat_class) + elif issubclass(flat_class, sparse_tensor_lib.SparseTensor): flat_ret.append(SparseTensorStructure(flat_type, flat_shape)) elif issubclass(flat_class, ops.Tensor): flat_ret.append(TensorStructure(flat_type, flat_shape)) else: # NOTE(mrry): Since legacy structures produced by iterators only - # comprise Tensors, SparseTensors, and nests, we do not need to support - # all structure types here. + # comprise Tensors, SparseTensors, and nests, we do not need to + # support all structure types here. raise TypeError( "Could not build a structure for output class %r" % flat_type) @@ -314,15 +329,23 @@ class NestedStructure(Structure): % (len(self._flat_types), len(flat_value))) flat_ret = [] - for sub_value, structure in zip(flat_value, self._flat_nested_structure): - flat_ret.append(structure._from_tensor_list([sub_value])) + i = 0 + for structure in self._flat_nested_structure: + num_flat_values = len(structure._flat_types) + sub_value = flat_value[i:i + num_flat_values] + flat_ret.append(structure._from_tensor_list(sub_value)) + i += num_flat_values return nest.pack_sequence_as(self._nested_structure, flat_ret) def _from_compatible_tensor_list(self, flat_value): flat_ret = [] - for sub_value, structure in zip(flat_value, self._flat_nested_structure): - flat_ret.append(structure._from_compatible_tensor_list([sub_value])) + i = 0 + for structure in self._flat_nested_structure: + num_flat_values = len(structure._flat_types) + sub_value = flat_value[i:i + num_flat_values] + flat_ret.append(structure._from_compatible_tensor_list(sub_value)) + i += num_flat_values return nest.pack_sequence_as(self._nested_structure, flat_ret) @@ -345,6 +368,10 @@ class NestedStructure(Structure): return nest.map_structure( lambda s: s._to_legacy_output_classes(), self._nested_structure) + def _batch(self, batch_size): + return NestedStructure(nest.map_structure( + lambda s: s._batch(batch_size), self._nested_structure)) + class TensorStructure(Structure): """Represents structural information about a `tf.Tensor`.""" @@ -381,6 +408,13 @@ class TensorStructure(Structure): return self._from_compatible_tensor_list(flat_value) def _from_compatible_tensor_list(self, flat_value): + # TODO(b/112266545): It would be cleaner to create a new `ensure_shape()` + # op here and return that, instead of mutating the input's shape using + # `Tensor.set_shape()`. However, that would add extra ops on the arguments + # of each `tf.data` function, which could impact performance. When this + # bug is resolved, we should be able to add the `ensure_shape()` ops and + # optimize them away using contextual shape information. + flat_value[0].set_shape(self._shape) return flat_value[0] @staticmethod @@ -396,6 +430,11 @@ class TensorStructure(Structure): def _to_legacy_output_classes(self): return ops.Tensor + def _batch(self, batch_size): + return TensorStructure( + self._dtype, + tensor_shape.TensorShape([batch_size]).concatenate(self._shape)) + class SparseTensorStructure(Structure): """Represents structural information about a `tf.SparseTensor`.""" @@ -406,7 +445,11 @@ class SparseTensorStructure(Structure): @property def _flat_shapes(self): - return [tensor_shape.vector(3)] + # NOTE(mrry): The default flat shape of a boxed `SparseTensor` is `(3,)`, + # but a `SparseTensorStructure` can also represent a batch of boxed + # `SparseTensor` objects with shape `(?, 3)` (and batches of batches, etc.), + # so the flat shape must be unknown. + return [tensor_shape.unknown_shape(None)] @property def _flat_types(self): @@ -428,8 +471,11 @@ class SparseTensorStructure(Structure): return self._from_compatible_tensor_list(flat_value) def _from_compatible_tensor_list(self, flat_value): - return sparse_ops.deserialize_sparse( + ret = sparse_ops.deserialize_sparse( flat_value[0], dtype=self._dtype, rank=self._dense_shape.ndims) + ret.indices.set_shape([None, self._dense_shape.ndims]) + ret.dense_shape.set_shape([self._dense_shape.ndims]) + return ret @staticmethod def from_value(value): @@ -446,3 +492,8 @@ class SparseTensorStructure(Structure): def _to_legacy_output_classes(self): return sparse_tensor_lib.SparseTensor + + def _batch(self, batch_size): + return SparseTensorStructure( + self._dtype, + tensor_shape.TensorShape([batch_size]).concatenate(self._dense_shape)) diff --git a/tensorflow/python/data/util/structure_test.py b/tensorflow/python/data/util/structure_test.py index 630a0c912bc..314a459effb 100644 --- a/tensorflow/python/data/util/structure_test.py +++ b/tensorflow/python/data/util/structure_test.py @@ -28,6 +28,7 @@ from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops from tensorflow.python.framework import sparse_tensor from tensorflow.python.framework import tensor_shape +from tensorflow.python.framework import test_util from tensorflow.python.ops import array_ops from tensorflow.python.ops import variables from tensorflow.python.platform import test @@ -44,7 +45,7 @@ class StructureTest(test.TestCase, parameterized.TestCase): [dtypes.float32], [[]]), (lambda: sparse_tensor.SparseTensor( indices=[[3, 4]], values=[-1], dense_shape=[4, 5]), - structure.SparseTensorStructure, [dtypes.variant], [[3]]), + structure.SparseTensorStructure, [dtypes.variant], [None]), (lambda: (constant_op.constant(37.0), constant_op.constant([1, 2, 3])), structure.NestedStructure, [dtypes.float32, dtypes.int32], [[], [3]]), (lambda: { @@ -58,14 +59,17 @@ class StructureTest(test.TestCase, parameterized.TestCase): sparse_tensor.SparseTensor( indices=[[3, 4]], values=[-1], dense_shape=[4, 5])) }, structure.NestedStructure, - [dtypes.float32, dtypes.variant, dtypes.variant], [[], [3], [3]])) + [dtypes.float32, dtypes.variant, dtypes.variant], [[], None, None])) def testFlatStructure(self, value_fn, expected_structure, expected_types, expected_shapes): value = value_fn() s = structure.Structure.from_value(value) self.assertIsInstance(s, expected_structure) self.assertEqual(expected_types, s._flat_types) - self.assertEqual(expected_shapes, s._flat_shapes) + for expected, actual in zip(expected_shapes, s._flat_shapes): + self.assertTrue(actual.is_compatible_with(expected)) + self.assertTrue( + tensor_shape.as_shape(expected).is_compatible_with(actual)) @parameterized.parameters( (lambda: constant_op.constant(37.0), lambda: [ @@ -112,6 +116,7 @@ class StructureTest(test.TestCase, parameterized.TestCase): indices=[[0], [1], [2]], values=[4, 5, 6], dense_shape=[3]) }, (constant_op.constant(15.0), constant_op.constant([4, 5, 6]))]), ) + @test_util.run_deprecated_v1 def testIsCompatibleWithStructure( self, original_value_fn, compatible_values_fn, incompatible_values_fn): original_value = original_value_fn() @@ -354,5 +359,65 @@ class StructureTest(test.TestCase, parameterized.TestCase): self.assertTrue(expected_structure.is_compatible_with(actual_structure)) self.assertTrue(actual_structure.is_compatible_with(expected_structure)) + def testNestedNestedStructure(self): + # Although `Structure.from_value()` will not construct one, a nested + # structure containing nested `NestedStructure` objects can occur if a + # structure is constructed manually. + s = structure.NestedStructure( + (structure.TensorStructure(dtypes.int64, []), + structure.NestedStructure( + (structure.TensorStructure(dtypes.float32, []), + structure.TensorStructure(dtypes.string, []))))) + + int64_t = constant_op.constant(37, dtype=dtypes.int64) + float32_t = constant_op.constant(42.0) + string_t = constant_op.constant("Foo") + + nested_tensors = (int64_t, (float32_t, string_t)) + + tensor_list = s._to_tensor_list(nested_tensors) + for expected, actual in zip([int64_t, float32_t, string_t], tensor_list): + self.assertIs(expected, actual) + + (actual_int64_t, (actual_float32_t, actual_string_t)) = s._from_tensor_list( + tensor_list) + self.assertIs(int64_t, actual_int64_t) + self.assertIs(float32_t, actual_float32_t) + self.assertIs(string_t, actual_string_t) + + (actual_int64_t, (actual_float32_t, actual_string_t)) = ( + s._from_compatible_tensor_list(tensor_list)) + self.assertIs(int64_t, actual_int64_t) + self.assertIs(float32_t, actual_float32_t) + self.assertIs(string_t, actual_string_t) + + @parameterized.named_parameters( + ("Tensor", structure.TensorStructure(dtypes.float32, []), 32, + structure.TensorStructure(dtypes.float32, [32])), + ("TensorUnknown", structure.TensorStructure(dtypes.float32, []), None, + structure.TensorStructure(dtypes.float32, [None])), + ("SparseTensor", structure.SparseTensorStructure(dtypes.float32, [None]), + 32, structure.SparseTensorStructure(dtypes.float32, [32, None])), + ("SparseTensorUnknown", + structure.SparseTensorStructure(dtypes.float32, [4]), None, + structure.SparseTensorStructure(dtypes.float32, [None, 4])), + ("Nest", structure.NestedStructure({ + "a": structure.TensorStructure(dtypes.float32, []), + "b": (structure.SparseTensorStructure(dtypes.int32, [2, 2]), + structure.TensorStructure(dtypes.string, []))}), 128, + structure.NestedStructure({ + "a": structure.TensorStructure(dtypes.float32, [128]), + "b": (structure.SparseTensorStructure(dtypes.int32, [128, 2, 2]), + structure.TensorStructure(dtypes.string, [128]))})), + ) + def testBatch(self, element_structure, batch_size, + expected_batched_structure): + batched_structure = element_structure._batch(batch_size) + self.assertTrue( + batched_structure.is_compatible_with(expected_batched_structure)) + self.assertTrue( + expected_batched_structure.is_compatible_with(batched_structure)) + + if __name__ == "__main__": test.main() diff --git a/tensorflow/python/debug/BUILD b/tensorflow/python/debug/BUILD index 79951232097..c6abd476d9d 100644 --- a/tensorflow/python/debug/BUILD +++ b/tensorflow/python/debug/BUILD @@ -557,6 +557,7 @@ py_test( ":source_utils", "//tensorflow/core:protos_all_py", "//tensorflow/python:client", + "//tensorflow/python:cond_v2", "//tensorflow/python:constant_op", "//tensorflow/python:control_flow_ops", "//tensorflow/python:framework_ops", @@ -566,6 +567,7 @@ py_test( "//tensorflow/python:resource_variable_ops", "//tensorflow/python:util", "//tensorflow/python:variables", + "//tensorflow/python:while_v2", "//third_party/py/numpy", ], ) diff --git a/tensorflow/python/debug/cli/analyzer_cli_test.py b/tensorflow/python/debug/cli/analyzer_cli_test.py index f197a9e4dce..322ecf94667 100644 --- a/tensorflow/python/debug/cli/analyzer_cli_test.py +++ b/tensorflow/python/debug/cli/analyzer_cli_test.py @@ -645,6 +645,7 @@ class AnalyzerCLISimpleMulAddTest(test_util.TensorFlowTestCase): self.assertEqual(len("Size (B)") + 1, dump_size_col_width) self.assertEqual(len("Op type") + 1, op_type_col_width) + @test_util.run_deprecated_v1 def testMeasureTensorListColumnWidthsGivesRightAnswerForData(self): dump = self._debug_dump.dumped_tensor_data[0] self.assertLess(dump.dump_size_bytes, 1000) @@ -660,6 +661,7 @@ class AnalyzerCLISimpleMulAddTest(test_util.TensorFlowTestCase): # column should be determined by the length of "VariableV2". self.assertEqual(len("VariableV2") + 1, op_type_col_width) + @test_util.run_deprecated_v1 def testListTensors(self): # Use shorthand alias for the command prefix. out = self._registry.dispatch_command("lt", []) @@ -673,6 +675,7 @@ class AnalyzerCLISimpleMulAddTest(test_util.TensorFlowTestCase): # Check the main menu. check_main_menu(self, out, list_tensors_enabled=False) + @test_util.run_deprecated_v1 def testListTensorsInReverseTimeOrderWorks(self): # Use shorthand alias for the command prefix. out = self._registry.dispatch_command("lt", ["-s", "timestamp", "-r"]) @@ -688,6 +691,7 @@ class AnalyzerCLISimpleMulAddTest(test_util.TensorFlowTestCase): reverse=True) check_main_menu(self, out, list_tensors_enabled=False) + @test_util.run_deprecated_v1 def testListTensorsInDumpSizeOrderWorks(self): out = self._registry.dispatch_command("lt", ["-s", "dump_size"]) assert_listed_tensors( @@ -701,6 +705,7 @@ class AnalyzerCLISimpleMulAddTest(test_util.TensorFlowTestCase): sort_by="dump_size") check_main_menu(self, out, list_tensors_enabled=False) + @test_util.run_deprecated_v1 def testListTensorsInReverseDumpSizeOrderWorks(self): out = self._registry.dispatch_command("lt", ["-s", "dump_size", "-r"]) assert_listed_tensors( @@ -720,6 +725,7 @@ class AnalyzerCLISimpleMulAddTest(test_util.TensorFlowTestCase): self.assertIn("ValueError: Unsupported key to sort tensors by: foobar", out.lines) + @test_util.run_deprecated_v1 def testListTensorsInOpTypeOrderWorks(self): # Use shorthand alias for the command prefix. out = self._registry.dispatch_command("lt", ["-s", "op_type"]) @@ -735,6 +741,7 @@ class AnalyzerCLISimpleMulAddTest(test_util.TensorFlowTestCase): reverse=False) check_main_menu(self, out, list_tensors_enabled=False) + @test_util.run_deprecated_v1 def testListTensorsInReverseOpTypeOrderWorks(self): # Use shorthand alias for the command prefix. out = self._registry.dispatch_command("lt", ["-s", "op_type", "-r"]) @@ -750,6 +757,7 @@ class AnalyzerCLISimpleMulAddTest(test_util.TensorFlowTestCase): reverse=True) check_main_menu(self, out, list_tensors_enabled=False) + @test_util.run_deprecated_v1 def testListTensorsInTensorNameOrderWorks(self): # Use shorthand alias for the command prefix. out = self._registry.dispatch_command("lt", ["-s", "tensor_name"]) @@ -765,6 +773,7 @@ class AnalyzerCLISimpleMulAddTest(test_util.TensorFlowTestCase): reverse=False) check_main_menu(self, out, list_tensors_enabled=False) + @test_util.run_deprecated_v1 def testListTensorsInReverseTensorNameOrderWorks(self): # Use shorthand alias for the command prefix. out = self._registry.dispatch_command("lt", ["-s", "tensor_name", "-r"]) @@ -780,6 +789,7 @@ class AnalyzerCLISimpleMulAddTest(test_util.TensorFlowTestCase): reverse=True) check_main_menu(self, out, list_tensors_enabled=False) + @test_util.run_deprecated_v1 def testListTensorsFilterByNodeNameRegex(self): out = self._registry.dispatch_command("list_tensors", ["--node_name_filter", ".*read.*"]) @@ -793,6 +803,7 @@ class AnalyzerCLISimpleMulAddTest(test_util.TensorFlowTestCase): assert_listed_tensors(self, out, [], [], node_name_regex="^read") check_main_menu(self, out, list_tensors_enabled=False) + @test_util.run_deprecated_v1 def testListTensorFilterByOpTypeRegex(self): out = self._registry.dispatch_command("list_tensors", ["--op_type_filter", "Identity"]) @@ -821,6 +832,7 @@ class AnalyzerCLISimpleMulAddTest(test_util.TensorFlowTestCase): op_type_regex="(Add|MatMul)") check_main_menu(self, out, list_tensors_enabled=False) + @test_util.run_deprecated_v1 def testListTensorWithFilterAndNodeNameExclusionWorks(self): # First, create and register the filter. def is_2x1_vector(datum, tensor): @@ -877,6 +889,7 @@ class AnalyzerCLISimpleMulAddTest(test_util.TensorFlowTestCase): out = self._registry.dispatch_command("list_tensors", ["--bar"]) check_syntax_error_output(self, out, "list_tensors") + @test_util.run_deprecated_v1 def testNodeInfoByNodeName(self): node_name = "simple_mul_add/matmul" out = self._registry.dispatch_command("node_info", [node_name]) @@ -901,6 +914,7 @@ class AnalyzerCLISimpleMulAddTest(test_util.TensorFlowTestCase): [(len(out.lines[0]) - len(node_name), len(out.lines[0]), "bold")], out.font_attr_segs[0]) + @test_util.run_deprecated_v1 def testNodeInfoShowAttributes(self): node_name = "simple_mul_add/matmul" out = self._registry.dispatch_command("node_info", ["-a", node_name]) @@ -924,6 +938,7 @@ class AnalyzerCLISimpleMulAddTest(test_util.TensorFlowTestCase): print_tensor_node_name=node_name, list_outputs_node_name=node_name) + @test_util.run_deprecated_v1 def testNodeInfoShowDumps(self): node_name = "simple_mul_add/matmul" out = self._registry.dispatch_command("node_info", ["-d", node_name]) @@ -948,6 +963,7 @@ class AnalyzerCLISimpleMulAddTest(test_util.TensorFlowTestCase): len(out.lines[16]) - len(out.lines[16].strip()), len(out.lines[16]), "pt %s:0 -n 0" % node_name) + @test_util.run_deprecated_v1 def testNodeInfoShowStackTraceUnavailableIsIndicated(self): self._debug_dump.set_python_graph(None) @@ -971,6 +987,7 @@ class AnalyzerCLISimpleMulAddTest(test_util.TensorFlowTestCase): print_tensor_node_name=node_name, list_outputs_node_name=node_name) + @test_util.run_deprecated_v1 def testNodeInfoShowStackTraceAvailableWorks(self): self._debug_dump.set_python_graph(self._sess.graph) @@ -994,6 +1011,7 @@ class AnalyzerCLISimpleMulAddTest(test_util.TensorFlowTestCase): print_tensor_node_name=node_name, list_outputs_node_name=node_name) + @test_util.run_deprecated_v1 def testNodeInfoByTensorName(self): node_name = "simple_mul_add/u/read" tensor_name = node_name + ":0" @@ -1363,6 +1381,7 @@ class AnalyzerCLISimpleMulAddTest(test_util.TensorFlowTestCase): break return index + @test_util.run_deprecated_v1 def testPrintSourceForOpNamesWholeFileWorks(self): self._debug_dump.set_python_graph(self._sess.graph) out = self._registry.dispatch_command( @@ -1415,6 +1434,7 @@ class AnalyzerCLISimpleMulAddTest(test_util.TensorFlowTestCase): self.assertEqual("pt simple_mul_add/add", out.font_attr_segs[index + 1][0][2].content) + @test_util.run_deprecated_v1 def testPrintSourceForTensorNamesWholeFileWorks(self): self._debug_dump.set_python_graph(self._sess.graph) out = self._registry.dispatch_command( @@ -1435,6 +1455,7 @@ class AnalyzerCLISimpleMulAddTest(test_util.TensorFlowTestCase): self.assertEqual("pt simple_mul_add/u:0", out.font_attr_segs[index + 2][0][2].content) + @test_util.run_deprecated_v1 def testPrintSourceForOpNamesStartingAtSpecifiedLineWorks(self): self._debug_dump.set_python_graph(self._sess.graph) out = self._registry.dispatch_command( @@ -1461,6 +1482,7 @@ class AnalyzerCLISimpleMulAddTest(test_util.TensorFlowTestCase): self.assertEqual("pt simple_mul_add/u/read", out.font_attr_segs[index + 3][0][2].content) + @test_util.run_deprecated_v1 def testPrintSourceForOpNameSettingMaximumElementCountWorks(self): self._debug_dump.set_python_graph(self._sess.graph) out = self._registry.dispatch_command( @@ -1505,6 +1527,7 @@ class AnalyzerCLISimpleMulAddTest(test_util.TensorFlowTestCase): self.assertTrue(cli_shared.COLOR_GRAY in attr_seg[2] or attr_seg[2] == cli_shared.COLOR_GRAY) + @test_util.run_deprecated_v1 def testListSourceWithNodeNameFilterWithMatchesWorks(self): self._debug_dump.set_python_graph(self._sess.graph) out = self._registry.dispatch_command("list_source", ["-n", ".*/read"]) @@ -1583,7 +1606,7 @@ class AnalyzerCLISimpleMulAddTest(test_util.TensorFlowTestCase): x = variables.VariableV1([1, 3, 3, 7], name="x") _, idx = array_ops.unique(x, name="x_unique") idx_times_two = math_ops.multiply(idx, 2, name="idx_times_two") - sess.run(x.initializer) + self.evaluate(x.initializer) run_options = config_pb2.RunOptions(output_partition_graphs=True) debug_utils.watch_graph( @@ -1719,6 +1742,7 @@ class AnalyzerCLIControlDepTest(test_util.TensorFlowTestCase): # Tear down temporary dump directory. shutil.rmtree(cls._dump_root) + @test_util.run_deprecated_v1 def testNodeInfoWithControlDependencies(self): # Call node_info on a node with control inputs. out = self._registry.dispatch_command("node_info", @@ -1759,6 +1783,7 @@ class AnalyzerCLIControlDepTest(test_util.TensorFlowTestCase): len(out.lines[z_line]), "ni -a -d -t control_deps/ctrl_dep_z") + @test_util.run_deprecated_v1 def testListInputsNonRecursiveNoControl(self): """List inputs non-recursively, without any control inputs.""" @@ -1801,6 +1826,7 @@ class AnalyzerCLIControlDepTest(test_util.TensorFlowTestCase): len(out.lines[3]) - len("control_deps/ctrl_dep_y"), len(out.lines[3]), "li -c -r control_deps/ctrl_dep_y") + @test_util.run_deprecated_v1 def testListInputsNonRecursiveNoControlUsingTensorName(self): """List inputs using the name of an output tensor of the node.""" @@ -1829,6 +1855,7 @@ class AnalyzerCLIControlDepTest(test_util.TensorFlowTestCase): len(out.lines[3]) - len("control_deps/ctrl_dep_y"), len(out.lines[3]), "li -c -r control_deps/ctrl_dep_y") + @test_util.run_deprecated_v1 def testListInputsNonRecursiveWithControls(self): """List inputs non-recursively, with control inputs.""" node_name = "control_deps/ctrl_dep_z" @@ -1859,6 +1886,7 @@ class AnalyzerCLIControlDepTest(test_util.TensorFlowTestCase): len(out.lines[5]) - len("control_deps/x"), len(out.lines[5]), "li -c -r control_deps/x") + @test_util.run_deprecated_v1 def testListInputsRecursiveWithControls(self): """List inputs recursively, with control inputs.""" node_name = "control_deps/ctrl_dep_z" @@ -1904,6 +1932,7 @@ class AnalyzerCLIControlDepTest(test_util.TensorFlowTestCase): len(out.lines[18]) - len("control_deps/x"), len(out.lines[18]), "li -c -r control_deps/x") + @test_util.run_deprecated_v1 def testListInputsRecursiveWithControlsWithDepthLimit(self): """List inputs recursively, with control inputs and a depth limit.""" node_name = "control_deps/ctrl_dep_z" @@ -1963,6 +1992,7 @@ class AnalyzerCLIControlDepTest(test_util.TensorFlowTestCase): "ERROR: There is no node named \"control_deps/z/foo\" in the " "partition graphs"], out.lines) + @test_util.run_deprecated_v1 def testListRecipientsRecursiveWithControlsWithDepthLimit(self): """List recipients recursively, with control inputs and a depth limit.""" @@ -2034,6 +2064,7 @@ class AnalyzerCLIWhileLoopTest(test_util.TensorFlowTestCase): # Tear down temporary dump directory. shutil.rmtree(cls._dump_root) + @test_util.run_deprecated_v1 def testMultipleDumpsPrintTensorNoNumber(self): output = self._registry.dispatch_command("pt", ["while/Identity:0"]) @@ -2051,6 +2082,7 @@ class AnalyzerCLIWhileLoopTest(test_util.TensorFlowTestCase): self.assertEqual("For example:", output.lines[-2]) self.assertEqual(" print_tensor while/Identity:0 -n 0", output.lines[-1]) + @test_util.run_deprecated_v1 def testMultipleDumpsPrintTensorWithNumber(self): for i in xrange(5): output = self._registry.dispatch_command( @@ -2064,6 +2096,7 @@ class AnalyzerCLIWhileLoopTest(test_util.TensorFlowTestCase): self.assertTrue(output.lines[4].startswith("array(%d" % i)) self.assertTrue(output.lines[4].endswith(")")) + @test_util.run_deprecated_v1 def testMultipleDumpsPrintTensorInvalidNumber(self): output = self._registry.dispatch_command("pt", ["while/Identity:0", "-n", "10"]) diff --git a/tensorflow/python/debug/cli/cli_shared_test.py b/tensorflow/python/debug/cli/cli_shared_test.py index 07b364db9f2..d191a234fde 100644 --- a/tensorflow/python/debug/cli/cli_shared_test.py +++ b/tensorflow/python/debug/cli/cli_shared_test.py @@ -118,6 +118,7 @@ class GetRunStartIntroAndDescriptionTest(test_util.TensorFlowTestCase): def tearDown(self): ops.reset_default_graph() + @test_util.run_deprecated_v1 def testSingleFetchNoFeeds(self): run_start_intro = cli_shared.get_run_start_intro(12, self.const_a, None, {}) @@ -181,6 +182,7 @@ class GetRunStartIntroAndDescriptionTest(test_util.TensorFlowTestCase): run_start_intro = cli_shared.get_run_start_intro(1, self.sparse_d, None, {}) self.assertEqual(str(self.sparse_d), run_start_intro.lines[4].strip()) + @test_util.run_deprecated_v1 def testTwoFetchesListNoFeeds(self): fetches = [self.const_a, self.const_b] run_start_intro = cli_shared.get_run_start_intro(1, fetches, None, {}) @@ -197,6 +199,7 @@ class GetRunStartIntroAndDescriptionTest(test_util.TensorFlowTestCase): description = cli_shared.get_run_short_description(1, fetches, None) self.assertEqual("run #1: 2 fetches; 0 feeds", description) + @test_util.run_deprecated_v1 def testNestedListAsFetches(self): fetches = [self.const_c, [self.const_a, self.const_b]] run_start_intro = cli_shared.get_run_start_intro(1, fetches, None, {}) @@ -210,6 +213,7 @@ class GetRunStartIntroAndDescriptionTest(test_util.TensorFlowTestCase): description = cli_shared.get_run_short_description(1, fetches, None) self.assertEqual("run #1: 3 fetches; 0 feeds", description) + @test_util.run_deprecated_v1 def testNestedDictAsFetches(self): fetches = {"c": self.const_c, "ab": {"a": self.const_a, "b": self.const_b}} run_start_intro = cli_shared.get_run_start_intro(1, fetches, None, {}) @@ -227,6 +231,7 @@ class GetRunStartIntroAndDescriptionTest(test_util.TensorFlowTestCase): description = cli_shared.get_run_short_description(1, fetches, None) self.assertEqual("run #1: 3 fetches; 0 feeds", description) + @test_util.run_deprecated_v1 def testTwoFetchesAsTupleNoFeeds(self): fetches = (self.const_a, self.const_b) run_start_intro = cli_shared.get_run_start_intro(1, fetches, None, {}) @@ -243,6 +248,7 @@ class GetRunStartIntroAndDescriptionTest(test_util.TensorFlowTestCase): description = cli_shared.get_run_short_description(1, fetches, None) self.assertEqual("run #1: 2 fetches; 0 feeds", description) + @test_util.run_deprecated_v1 def testTwoFetchesAsNamedTupleNoFeeds(self): fetches_namedtuple = namedtuple("fetches", "x y") fetches = fetches_namedtuple(self.const_b, self.const_c) @@ -260,6 +266,7 @@ class GetRunStartIntroAndDescriptionTest(test_util.TensorFlowTestCase): description = cli_shared.get_run_short_description(1, fetches, None) self.assertEqual("run #1: 2 fetches; 0 feeds", description) + @test_util.run_deprecated_v1 def testWithFeedDict(self): feed_dict = { self.const_a: 10.0, @@ -283,6 +290,7 @@ class GetRunStartIntroAndDescriptionTest(test_util.TensorFlowTestCase): feed_dict) self.assertEqual("run #1: 1 fetch (c:0); 2 feeds", description) + @test_util.run_deprecated_v1 def testTensorFilters(self): feed_dict = {self.const_a: 10.0} tensor_filters = { @@ -313,11 +321,13 @@ class GetRunStartIntroAndDescriptionTest(test_util.TensorFlowTestCase): command_set.add(annot[2].content) self.assertEqual({"run -f filter_a", "run -f filter_b"}, command_set) + @test_util.run_deprecated_v1 def testGetRunShortDescriptionWorksForTensorFeedKey(self): short_description = cli_shared.get_run_short_description( 1, self.const_a, {self.const_a: 42.0}) self.assertEqual("run #1: 1 fetch (a:0); 1 feed (a:0)", short_description) + @test_util.run_deprecated_v1 def testGetRunShortDescriptionWorksForUnicodeFeedKey(self): short_description = cli_shared.get_run_short_description( 1, self.const_a, {u"foo": 42.0}) @@ -332,6 +342,7 @@ class GetErrorIntroTest(test_util.TensorFlowTestCase): def tearDown(self): ops.reset_default_graph() + @test_util.run_deprecated_v1 def testShapeError(self): tf_error = errors.OpError(None, self.var_a.initializer, "foo description", None) diff --git a/tensorflow/python/debug/cli/profile_analyzer_cli_test.py b/tensorflow/python/debug/cli/profile_analyzer_cli_test.py index 60b60479707..effcd500c70 100644 --- a/tensorflow/python/debug/cli/profile_analyzer_cli_test.py +++ b/tensorflow/python/debug/cli/profile_analyzer_cli_test.py @@ -348,6 +348,7 @@ class ProfileAnalyzerPrintSourceTest(test_util.TensorFlowTestCase): ops.reset_default_graph() super(ProfileAnalyzerPrintSourceTest, self).tearDown() + @test_util.run_deprecated_v1 def testPrintSourceForWhileLoop(self): prof_output = self.prof_analyzer.print_source([__file__]) @@ -361,6 +362,7 @@ class ProfileAnalyzerPrintSourceTest(test_util.TensorFlowTestCase): r"\[(\|)+(\s)*\] .*us .*7\(55\) .*L%d.*(\S)+" % self.loop_lineno, prof_output.lines) + @test_util.run_deprecated_v1 def testPrintSourceOutputContainsClickableLinks(self): prof_output = self.prof_analyzer.print_source([__file__]) any_match, line_index = _at_least_one_line_matches( @@ -377,6 +379,7 @@ class ProfileAnalyzerPrintSourceTest(test_util.TensorFlowTestCase): break self.assertTrue(any_menu_item_match) + @test_util.run_deprecated_v1 def testPrintSourceWithNonDefaultTimeUnit(self): prof_output = self.prof_analyzer.print_source([ __file__, "--time_unit", "ms"]) @@ -391,6 +394,7 @@ class ProfileAnalyzerPrintSourceTest(test_util.TensorFlowTestCase): r"\[(\|)+(\s)*\] .*ms .*7\(55\) .*L%d.*(\S)+" % self.loop_lineno, prof_output.lines) + @test_util.run_deprecated_v1 def testPrintSourceWithNodeNameFilter(self): prof_output = self.prof_analyzer.print_source([ __file__, "--node_name_filter", "x$"]) @@ -423,6 +427,7 @@ class ProfileAnalyzerPrintSourceTest(test_util.TensorFlowTestCase): break self.assertTrue(any_menu_item_match) + @test_util.run_deprecated_v1 def testPrintSourceWithOpTypeFilter(self): prof_output = self.prof_analyzer.print_source([ __file__, "--op_type_filter", "Less"]) diff --git a/tensorflow/python/debug/lib/common_test.py b/tensorflow/python/debug/lib/common_test.py index 5af0dafcf9f..f6413f6b7b3 100644 --- a/tensorflow/python/debug/lib/common_test.py +++ b/tensorflow/python/debug/lib/common_test.py @@ -27,6 +27,7 @@ from tensorflow.python.platform import googletest class CommonTest(test_util.TensorFlowTestCase): + @test_util.run_deprecated_v1 def testOnFeedOneFetch(self): a = constant_op.constant(10.0, name="a") b = constant_op.constant(20.0, name="b") @@ -35,6 +36,7 @@ class CommonTest(test_util.TensorFlowTestCase): self.assertItemsEqual(["a:0"], loaded[0]) self.assertItemsEqual(["b:0"], loaded[1]) + @test_util.run_deprecated_v1 def testGetRunKeyFlat(self): a = constant_op.constant(10.0, name="a") b = constant_op.constant(20.0, name="b") @@ -43,6 +45,7 @@ class CommonTest(test_util.TensorFlowTestCase): self.assertItemsEqual(["a:0"], loaded[0]) self.assertItemsEqual(["a:0", "b:0"], loaded[1]) + @test_util.run_deprecated_v1 def testGetRunKeyNestedFetches(self): a = constant_op.constant(10.0, name="a") b = constant_op.constant(20.0, name="b") diff --git a/tensorflow/python/debug/lib/debug_gradients_test.py b/tensorflow/python/debug/lib/debug_gradients_test.py index 01867fc69d0..1c531478638 100644 --- a/tensorflow/python/debug/lib/debug_gradients_test.py +++ b/tensorflow/python/debug/lib/debug_gradients_test.py @@ -54,6 +54,7 @@ class IdentifyGradientTest(test_util.TensorFlowTestCase): ops.reset_default_graph() debug_gradients.clear_gradient_debuggers() + @test_util.run_deprecated_v1 def testIdentifyGradientGivesCorrectTensorObjectWithoutContextManager(self): grad_debugger = debug_gradients.GradientsDebugger() id_grad_w = grad_debugger.identify_gradient(self.w) @@ -84,6 +85,7 @@ class IdentifyGradientTest(test_util.TensorFlowTestCase): self.assertIsInstance(w_grad, ops.Tensor) self.assertAllClose(1.0, self.sess.run(w_grad)) + @test_util.run_deprecated_v1 def testIdentifyGradientGivesCorrectTensorObjectWithTfGradients(self): grad_debugger = debug_gradients.GradientsDebugger() id_grad_w = grad_debugger.identify_gradient(self.w) @@ -115,6 +117,7 @@ class IdentifyGradientTest(test_util.TensorFlowTestCase): self.assertIsInstance(w_grad, ops.Tensor) self.assertAllClose(1.0, self.sess.run(w_grad)) + @test_util.run_deprecated_v1 def testCallingIdentifyGradientTwiceWithTheSameGradientsDebuggerErrors(self): grad_debugger = debug_gradients.GradientsDebugger() grad_debugger.identify_gradient(self.w) @@ -122,6 +125,7 @@ class IdentifyGradientTest(test_util.TensorFlowTestCase): "The graph already contains an op named .*"): grad_debugger.identify_gradient(self.w) + @test_util.run_deprecated_v1 def testIdentifyGradientWorksOnMultipleLosses(self): grad_debugger_1 = debug_gradients.GradientsDebugger() grad_debugger_2 = debug_gradients.GradientsDebugger() @@ -150,6 +154,7 @@ class IdentifyGradientTest(test_util.TensorFlowTestCase): self.assertAllClose(2.0 * 5.0, self.sess.run(dz1_dy)) self.assertAllClose(0.5 * (5.0**-0.5), self.sess.run(dz2_dy)) + @test_util.run_deprecated_v1 def testIdentifyGradientRaisesLookupErrorForUnknownXTensor(self): grad_debugger_1 = debug_gradients.GradientsDebugger() grad_debugger_2 = debug_gradients.GradientsDebugger() @@ -170,6 +175,7 @@ class IdentifyGradientTest(test_util.TensorFlowTestCase): r"This GradientsDebugger has not received any gradient tensor for "): grad_debugger_2.gradient_tensor(self.w) + @test_util.run_deprecated_v1 def testIdentifyGradientRaisesTypeErrorForNonTensorOrTensorNameInput(self): grad_debugger = debug_gradients.GradientsDebugger() with self.assertRaisesRegexp( @@ -178,6 +184,7 @@ class IdentifyGradientTest(test_util.TensorFlowTestCase): r"has type .*Operation.*"): grad_debugger.gradient_tensor(variables.global_variables_initializer()) + @test_util.run_deprecated_v1 def testIdentifyGradientTensorWorksWithGradientDescentOptimizer(self): grad_debugger = debug_gradients.GradientsDebugger() id_grad_w = grad_debugger.identify_gradient(self.w) @@ -193,6 +200,7 @@ class IdentifyGradientTest(test_util.TensorFlowTestCase): self.assertIsInstance(w_grad, ops.Tensor) self.assertAllClose(1.0, self.sess.run(w_grad)) + @test_util.run_deprecated_v1 def testWatchGradientsByXTensorNamesWorks(self): y = math_ops.add(self.w, -1.0, name="y") @@ -219,6 +227,7 @@ class IdentifyGradientTest(test_util.TensorFlowTestCase): self.assertIsInstance(w_grad, ops.Tensor) self.assertAllClose(1.0, self.sess.run(w_grad)) + @test_util.run_deprecated_v1 def testWatchGradientsByXTensorNamesWorksWithoutContextManager(self): y = math_ops.add(self.w, -1.0, name="y") @@ -245,6 +254,7 @@ class IdentifyGradientTest(test_util.TensorFlowTestCase): self.assertIsInstance(w_grad, ops.Tensor) self.assertAllClose(1.0, self.sess.run(w_grad)) + @test_util.run_deprecated_v1 def testWatchGradientsWorksOnRefTensor(self): y = math_ops.add(self.w, -1.0, name="y") @@ -263,6 +273,7 @@ class IdentifyGradientTest(test_util.TensorFlowTestCase): self.assertAllClose(3.0, self.sess.run( grad_debugger.gradient_tensor("u:0"))) + @test_util.run_deprecated_v1 def testWatchGradientsWorksOnMultipleTensors(self): y = math_ops.add(self.w, -1.0, name="y") @@ -283,6 +294,7 @@ class IdentifyGradientTest(test_util.TensorFlowTestCase): self.assertAllClose(3.0, self.sess.run( grad_debugger.gradient_tensor("u:0"))) + @test_util.run_deprecated_v1 def testWatchGradientsByXTensorsWorks(self): y = math_ops.add(self.w, -1.0, name="foo/y") z = math_ops.square(y, name="foo/z") @@ -305,6 +317,7 @@ class IdentifyGradientTest(test_util.TensorFlowTestCase): self.assertAllClose(10.0, self.sess.run(w_grad)) self.assertAllClose(30.0, self.sess.run(u_grad)) + @test_util.run_deprecated_v1 def testWatchGradientsByTensorCanWorkOnMultipleLosses(self): y = math_ops.add(self.w, -1.0, name="y") z1 = math_ops.square(y, name="z1") @@ -330,6 +343,7 @@ class IdentifyGradientTest(test_util.TensorFlowTestCase): self.assertAllClose(2.0 * 5.0, self.sess.run(dz1_dy)) self.assertAllClose(0.5 * (5.0**-0.5), self.sess.run(dz2_dy)) + @test_util.run_deprecated_v1 def testGradientsValuesFromDumpWorks(self): y = math_ops.add(self.w, -1.0, name="y") z = math_ops.square(y, name="z") diff --git a/tensorflow/python/debug/lib/debug_graph_reconstruction_test.py b/tensorflow/python/debug/lib/debug_graph_reconstruction_test.py index 1f67f8a0d4e..34030c0adca 100644 --- a/tensorflow/python/debug/lib/debug_graph_reconstruction_test.py +++ b/tensorflow/python/debug/lib/debug_graph_reconstruction_test.py @@ -126,8 +126,8 @@ class ReconstructNonDebugGraphTest(test_util.TensorFlowTestCase): u = variables.Variable([12.0], name="u") v = variables.Variable([30.0], name="v") w = math_ops.add(u, v, name="w") - sess.run(u.initializer) - sess.run(v.initializer) + self.evaluate(u.initializer) + self.evaluate(v.initializer) self._compareOriginalAndReconstructedGraphDefs( sess, w, expected_output=[42.0]) @@ -139,7 +139,7 @@ class ReconstructNonDebugGraphTest(test_util.TensorFlowTestCase): b = math_ops.add(a, a, name="b") with ops.control_dependencies([a, b]): c = math_ops.multiply(b, b, name="c") - sess.run(a.initializer) + self.evaluate(a.initializer) self._compareOriginalAndReconstructedGraphDefs( sess, c, expected_output=400.0) @@ -150,8 +150,8 @@ class ReconstructNonDebugGraphTest(test_util.TensorFlowTestCase): y = variables.Variable(20.0, name="y") cond = control_flow_ops.cond( x > y, lambda: math_ops.add(x, 1), lambda: math_ops.add(y, 1)) - sess.run(x.initializer) - sess.run(y.initializer) + self.evaluate(x.initializer) + self.evaluate(y.initializer) self._compareOriginalAndReconstructedGraphDefs( sess, cond, expected_output=21.0) @@ -173,8 +173,8 @@ class ReconstructNonDebugGraphTest(test_util.TensorFlowTestCase): toy_loss = x * (u - v) train_op = gradient_descent.GradientDescentOptimizer( learning_rate=0.1).minimize(toy_loss, name="train_op") - sess.run(u.initializer) - sess.run(v.initializer) + self.evaluate(u.initializer) + self.evaluate(v.initializer) self._compareOriginalAndReconstructedGraphDefs(sess, train_op) diff --git a/tensorflow/python/debug/lib/debug_utils_test.py b/tensorflow/python/debug/lib/debug_utils_test.py index 23ab98444cd..cf59b30e3da 100644 --- a/tensorflow/python/debug/lib/debug_utils_test.py +++ b/tensorflow/python/debug/lib/debug_utils_test.py @@ -185,6 +185,7 @@ class DebugUtilsTest(test_util.TensorFlowTestCase): self.assertEqual(["file:///tmp/tfdbg_1", "file:///tmp/tfdbg_2"], watch_0.debug_urls) + @test_util.run_deprecated_v1 def testWatchGraph_allNodes(self): debug_utils.watch_graph( self._run_options, @@ -216,6 +217,7 @@ class DebugUtilsTest(test_util.TensorFlowTestCase): self.assertTrue("p1" in node_names) self.assertTrue("s" in node_names) + @test_util.run_deprecated_v1 def testWatchGraph_nodeNameWhitelist(self): debug_utils.watch_graph( self._run_options, @@ -230,6 +232,7 @@ class DebugUtilsTest(test_util.TensorFlowTestCase): sorted(["a1_init", "a1", "a1/Assign", "a1/read", "p1"]), sorted(node_names)) + @test_util.run_deprecated_v1 def testWatchGraph_opTypeWhitelist(self): debug_utils.watch_graph( self._run_options, @@ -255,6 +258,7 @@ class DebugUtilsTest(test_util.TensorFlowTestCase): ["DebugIdentity"], ["file:///tmp/tfdbg_1"]) self.assertEqual(["p1"], node_names) + @test_util.run_deprecated_v1 def testWatchGraph_tensorDTypeWhitelist(self): debug_utils.watch_graph( self._run_options, @@ -267,6 +271,7 @@ class DebugUtilsTest(test_util.TensorFlowTestCase): ["DebugIdentity"], ["file:///tmp/tfdbg_1"]) self.assertItemsEqual(["a1", "a1/Assign", "b", "b/Assign"], node_names) + @test_util.run_deprecated_v1 def testWatchGraph_nodeNameAndTensorDTypeWhitelists(self): debug_utils.watch_graph( self._run_options, @@ -280,6 +285,7 @@ class DebugUtilsTest(test_util.TensorFlowTestCase): ["DebugIdentity"], ["file:///tmp/tfdbg_1"]) self.assertItemsEqual(["a1", "a1/Assign"], node_names) + @test_util.run_deprecated_v1 def testWatchGraph_nodeNameBlacklist(self): debug_utils.watch_graph_with_blacklists( self._run_options, @@ -294,6 +300,7 @@ class DebugUtilsTest(test_util.TensorFlowTestCase): sorted(["b_init", "b", "b/Assign", "b/read", "c", "s"]), sorted(node_names)) + @test_util.run_deprecated_v1 def testWatchGraph_opTypeBlacklist(self): debug_utils.watch_graph_with_blacklists( self._run_options, @@ -306,6 +313,7 @@ class DebugUtilsTest(test_util.TensorFlowTestCase): ["DebugIdentity"], ["file:///tmp/tfdbg_1"]) self.assertEqual(sorted(["p1", "s"]), sorted(node_names)) + @test_util.run_deprecated_v1 def testWatchGraph_nodeNameAndOpTypeBlacklists(self): debug_utils.watch_graph_with_blacklists( self._run_options, @@ -319,6 +327,7 @@ class DebugUtilsTest(test_util.TensorFlowTestCase): ["DebugIdentity"], ["file:///tmp/tfdbg_1"]) self.assertEqual(["s"], node_names) + @test_util.run_deprecated_v1 def testWatchGraph_tensorDTypeBlacklists(self): debug_utils.watch_graph_with_blacklists( self._run_options, @@ -335,6 +344,7 @@ class DebugUtilsTest(test_util.TensorFlowTestCase): self.assertNotIn("b/Assign", node_names) self.assertIn("s", node_names) + @test_util.run_deprecated_v1 def testWatchGraph_nodeNameAndTensorDTypeBlacklists(self): debug_utils.watch_graph_with_blacklists( self._run_options, diff --git a/tensorflow/python/debug/lib/session_debug_file_test.py b/tensorflow/python/debug/lib/session_debug_file_test.py index 1874160dd63..f5f9ba29ab5 100644 --- a/tensorflow/python/debug/lib/session_debug_file_test.py +++ b/tensorflow/python/debug/lib/session_debug_file_test.py @@ -28,6 +28,7 @@ from tensorflow.python.debug.lib import debug_utils from tensorflow.python.debug.lib import session_debug_testlib from tensorflow.python.framework import constant_op from tensorflow.python.framework import ops +from tensorflow.python.framework import test_util from tensorflow.python.ops import math_ops from tensorflow.python.ops import variables from tensorflow.python.platform import googletest @@ -44,6 +45,7 @@ class SessionDebugFileTest(session_debug_testlib.SessionDebugTestBase): else: return os.path.join(self._dump_root, "run_%d" % run_number) + @test_util.run_deprecated_v1 def testAllowsDifferentWatchesOnDifferentRuns(self): """Test watching different tensors on different runs of the same graph.""" diff --git a/tensorflow/python/debug/lib/session_debug_multi_gpu_test.py b/tensorflow/python/debug/lib/session_debug_multi_gpu_test.py index b0dc25851ca..8eef45392f2 100644 --- a/tensorflow/python/debug/lib/session_debug_multi_gpu_test.py +++ b/tensorflow/python/debug/lib/session_debug_multi_gpu_test.py @@ -67,7 +67,7 @@ class SessionDebugMultiGPUTest(test_util.TensorFlowTestCase): u1 = math_ops.multiply(v, v, name="u1") w = math_ops.subtract(u1, u0, name="w") - sess.run(v.initializer) + self.evaluate(v.initializer) run_options = config_pb2.RunOptions(output_partition_graphs=True) debug_utils.watch_graph(run_options, sess.graph, diff --git a/tensorflow/python/debug/lib/source_utils_test.py b/tensorflow/python/debug/lib/source_utils_test.py index 4a8d4eaa99f..9083297fdbb 100644 --- a/tensorflow/python/debug/lib/source_utils_test.py +++ b/tensorflow/python/debug/lib/source_utils_test.py @@ -65,6 +65,7 @@ class GuessIsTensorFlowLibraryTest(test_util.TensorFlowTestCase): self.assertTrue( source_utils.guess_is_tensorflow_py_library(source_utils.__file__)) + @test_util.run_deprecated_v1 def testFileInPythonKernelsPathReturnsTrue(self): x = constant_op.constant(42.0, name="x") self.assertTrue( @@ -109,8 +110,8 @@ class SourceHelperTest(test_util.TensorFlowTestCase): self.w = math_ops.matmul(self.u, self.v, name="w") self.w_line_number = line_number_above() - sess.run(self.u.initializer) - sess.run(self.v.initializer) + self.evaluate(self.u.initializer) + self.evaluate(self.v.initializer) run_options = config_pb2.RunOptions(output_partition_graphs=True) debug_utils.watch_graph( diff --git a/tensorflow/python/debug/wrappers/framework_test.py b/tensorflow/python/debug/wrappers/framework_test.py index 73e08ce7d59..68584b4ede4 100644 --- a/tensorflow/python/debug/wrappers/framework_test.py +++ b/tensorflow/python/debug/wrappers/framework_test.py @@ -339,7 +339,7 @@ class DebugWrapperSessionTest(test_util.TensorFlowTestCase): with wrapper.as_default(): foo = constant_op.constant(42, name="foo") - self.assertEqual(42, foo.eval()) + self.assertEqual(42, self.evaluate(foo)) self.assertEqual(foo, self._observer["run_fetches"]) def testWrapperShouldSupportSessionClose(self): diff --git a/tensorflow/python/distribute/BUILD b/tensorflow/python/distribute/BUILD index 1c3d8ea67e5..2d9a1764db7 100644 --- a/tensorflow/python/distribute/BUILD +++ b/tensorflow/python/distribute/BUILD @@ -7,15 +7,140 @@ licenses(["notice"]) # Apache 2.0 exports_files(["LICENSE"]) load("//tensorflow:tensorflow.bzl", "py_test") +load("//tensorflow:tensorflow.bzl", "tf_py_test") +load("//tensorflow:tensorflow.bzl", "cuda_py_test") py_library( - name = "distribute", + name = "all_reduce", + srcs = [ + "all_reduce.py", + ], srcs_version = "PY2AND3", - visibility = ["//visibility:public"], deps = [ - ":distribute_config", - ":distribute_coordinator", - ":distribute_coordinator_context", + "//tensorflow/python:array_ops", + "//tensorflow/python:framework_ops", + "//tensorflow/python:math_ops", + "//tensorflow/python:nccl_ops", + ], +) + +tf_py_test( + name = "all_reduce_test", + srcs = ["all_reduce_test.py"], + additional_deps = [ + ":all_reduce", + "//third_party/py/numpy", + "//tensorflow/core:protos_all_py", + "//tensorflow/python:array_ops", + "//tensorflow/python:client", + "//tensorflow/python:framework_ops", + "//tensorflow/python:framework_test_lib", + "//tensorflow/python:math_ops", + "//tensorflow/python:constant_op", + "//tensorflow/python:client_testlib", + "//tensorflow/python:platform", + "//tensorflow/python:platform_test", + "//tensorflow/python:state_ops", + ], +) + +py_library( + name = "cross_device_ops", + srcs = ["cross_device_ops.py"], + srcs_version = "PY2AND3", + deps = [ + ":cross_device_utils", + ":device_util", + ":reduce_util", + ":values", + "//tensorflow/python:array_ops", + "//tensorflow/python:device_lib", + "//tensorflow/python:framework_ops", + "//tensorflow/python:math_ops", + "//tensorflow/python:platform", + "//tensorflow/python:resource_variable_ops", + "//tensorflow/python/eager:context", + "@six_archive//:six", + ], +) + +py_library( + name = "cross_device_utils", + srcs = ["cross_device_utils.py"], + srcs_version = "PY2AND3", + deps = [ + ":all_reduce", + ":values", + "//tensorflow/python:array_ops", + "//tensorflow/python:collective_ops", + "//tensorflow/python:device", + "//tensorflow/python:dtypes", + "//tensorflow/python:framework_ops", + "//tensorflow/python:gradients", + "//tensorflow/python:math_ops", + "//tensorflow/python:nccl_ops", + ], +) + +py_library( + name = "device_util", + srcs = ["device_util.py"], + srcs_version = "PY2AND3", + deps = [ + "//tensorflow/python:device", + "//tensorflow/python:framework_ops", + "//tensorflow/python/eager:context", + ], +) + +cuda_py_test( + name = "device_util_test", + srcs = ["device_util_test.py"], + additional_deps = [ + ":device_util", + "//tensorflow/python:client_testlib", + "//tensorflow/python:framework_ops", + ], +) + +py_library( + name = "distribute_lib", + srcs = [ + "distribute_lib.py", + "distribution_strategy_context.py", + ], + srcs_version = "PY2AND3", + deps = [ + ":device_util", + ":reduce_util", + "//tensorflow/python:array_ops", + "//tensorflow/python:constant_op", + "//tensorflow/python:control_flow_ops", + "//tensorflow/python:dtypes", + "//tensorflow/python:framework_ops", + "//tensorflow/python:platform", + "//tensorflow/python:resource_variable_ops", + "//tensorflow/python:state_ops", + "//tensorflow/python:util", + "//tensorflow/python:variable_scope", + "//tensorflow/python/data", + "//tensorflow/python/distribute/cluster_resolver:cluster_resolver_lib", + "//tensorflow/python/ops/losses", + "//tensorflow/tools/docs:doc_controls", + ], +) + +py_test( + name = "distribute_lib_test", + size = "small", + srcs = ["distribute_lib_test.py"], + srcs_version = "PY2AND3", + deps = [ + ":distribute_lib", + "//tensorflow/python:client_testlib", + "//tensorflow/python:constant_op", + "//tensorflow/python:dtypes", + "//tensorflow/python:variable_scope", ], ) @@ -45,7 +170,6 @@ py_library( py_test( name = "distribute_coordinator_test", - size = "large", srcs = ["distribute_coordinator_test.py"], srcs_version = "PY2AND3", tags = [ @@ -76,6 +200,35 @@ py_library( deps = [], ) +py_library( + name = "mirrored_strategy", + srcs = ["mirrored_strategy.py"], + deps = [ + ":cross_device_ops", + ":device_util", + ":distribute_lib", + ":multi_worker_util", + ":reduce_util", + ":shared_variable_creator", + ":values", + "//tensorflow/core:protos_all_py", + "//tensorflow/python:array_ops", + "//tensorflow/python:constant_op", + "//tensorflow/python:control_flow_ops", + "//tensorflow/python:device", + "//tensorflow/python:dtypes", + "//tensorflow/python:framework_ops", + "//tensorflow/python:pywrap_tensorflow", + "//tensorflow/python:tensor_util", + "//tensorflow/python:training", + "//tensorflow/python:util", + "//tensorflow/python:variable_scope", + "//tensorflow/python:variables", + "//tensorflow/python/eager:context", + "//tensorflow/python/eager:tape", + ], +) + py_library( name = "multi_worker_util", srcs = [ @@ -88,6 +241,35 @@ py_library( ], ) +py_library( + name = "input_ops", + srcs = ["input_ops.py"], + deps = [ + "//tensorflow/python:framework_ops", + "//tensorflow/python/data/experimental/ops:filter_for_shard_ops", + "//tensorflow/python/data/util:nest", + ], +) + +cuda_py_test( + name = "input_ops_test", + srcs = ["input_ops_test.py"], + additional_deps = [ + ":input_ops", + "//tensorflow/python/data/ops:dataset_ops", + "//tensorflow/python/data/ops:readers", + "//tensorflow/python:errors", + "//tensorflow/python:client_testlib", + "//tensorflow/python:framework_ops", + "//tensorflow/python:framework_test_lib", + "//tensorflow/python:io_ops", + "//tensorflow/python:util", + ], + tags = [ + "no_pip", + ], +) + py_test( name = "multi_worker_util_test", srcs = ["multi_worker_util_test.py"], @@ -120,3 +302,49 @@ py_library( "//tensorflow/python:training", ], ) + +py_library( + name = "reduce_util", + srcs = ["reduce_util.py"], + deps = [ + "//tensorflow/python:util", + "//tensorflow/python:variable_scope", + ], +) + +py_library( + name = "shared_variable_creator", + srcs = ["shared_variable_creator.py"], +) + +py_test( + name = "shared_variable_creator_test", + srcs = ["shared_variable_creator_test.py"], + srcs_version = "PY2AND3", + deps = [ + ":shared_variable_creator", + "//tensorflow/python:framework_test_lib", + "//tensorflow/python:variable_scope", + "//tensorflow/python/eager:test", + ], +) + +py_library( + name = "values", + srcs = ["values.py"], + deps = [ + ":device_util", + ":distribute_lib", + ":input_ops", + "//tensorflow/python:array_ops", + "//tensorflow/python:control_flow_ops", + "//tensorflow/python:framework_ops", + "//tensorflow/python:resource_variable_ops", + "//tensorflow/python:training", + "//tensorflow/python:util", + "//tensorflow/python/data/ops:multi_device_iterator_ops", + "//tensorflow/python/eager:context", + "//tensorflow/python/training/checkpointable:base", + "@six_archive//:six", + ], +) diff --git a/tensorflow/python/distribute/all_reduce.py b/tensorflow/python/distribute/all_reduce.py new file mode 100644 index 00000000000..bd7c45ae27a --- /dev/null +++ b/tensorflow/python/distribute/all_reduce.py @@ -0,0 +1,860 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Utilities to construct a TF subgraph implementing distributed All-Reduce.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import collections +import math + +from tensorflow.python.framework import device as device_lib +from tensorflow.python.framework import ops +from tensorflow.python.ops import array_ops +from tensorflow.python.ops import math_ops +from tensorflow.python.ops import nccl_ops + + +def _flatten_tensors(tensors): + """Check tensors for isomorphism and flatten. + + Args: + tensors: list of T `tf.Tensor` which must all have the same shape. + + Returns: + tensors: a list of T `tf.Tensor` which are flattened (1D) views of tensors + shape: the original shape of each element of input tensors + + Raises: + ValueError: tensors are empty or non-isomorphic or have unknown shape. + """ + if not tensors: + raise ValueError("tensors cannot be empty") + shape = tensors[0].shape + for tensor in tensors: + shape = shape.merge_with(tensor.shape) + if not shape.is_fully_defined(): + raise ValueError("Tensors must have statically known shape.") + if len(shape) != 1: + reshaped = [] + for t in tensors: + with ops.colocate_with(t): + reshaped.append(array_ops.reshape(t, [-1])) + tensors = reshaped + return tensors, shape + + +def _reshape_tensors(tensors, shape): + """Reshape tensors flattened by _flatten_tensors. + + Args: + tensors: list of T `tf.Tensor` of identical length 1D tensors. + shape: list of integers describing the desired shape. Product of + the elements must equal the length of each tensor. + + Returns: + list of T `tf.Tensor` which are the reshaped inputs. + """ + reshaped = [] + for t in tensors: + with ops.colocate_with(t): + reshaped.append(array_ops.reshape(t, shape)) + return reshaped + + +def _padded_split(tensor, pieces): + """Like split for 1D tensors but pads-out case where len % pieces != 0. + + Args: + tensor: T `tf.Tensor` that must be 1D. + pieces: a positive integer specifying the number of pieces into which + tensor should be split. + + Returns: + list of T `tf.Tensor` of length pieces, which hold the values of + thin input tensor, in order. The final tensor may + be zero-padded on the end to make its size equal to those of all + of the other tensors. + + Raises: + ValueError: The input tensor is not 1D. + """ + shape = tensor.shape + if 1 != len(shape): + raise ValueError("input tensor must be 1D") + tensor_len = shape.dims[0].value + with ops.colocate_with(tensor): + if tensor_len % pieces != 0: + # pad to an even length + chunk_size = 1 + tensor_len // pieces + if pieces > tensor_len: + # This is an edge case that should not come up in practice, + # i.e. a different reduction algorithm would be better, + # but we'll make it work just for completeness. + pad_len = pieces - tensor_len + extended_whole = array_ops.concat( + [tensor, array_ops.zeros([pad_len], dtype=tensor.dtype)], 0) + parts = array_ops.split(extended_whole, pieces) + return parts, pad_len + elif (pieces - 1) * chunk_size >= tensor_len: + # Another edge case of limited real interest. + pad_len = (pieces * chunk_size) % tensor_len + extended_whole = array_ops.concat( + [tensor, array_ops.zeros([pad_len], dtype=tensor.dtype)], 0) + parts = array_ops.split(extended_whole, pieces) + return parts, pad_len + else: + last_chunk_size = tensor_len - (pieces - 1) * chunk_size + pad_len = chunk_size - last_chunk_size + piece_lens = [chunk_size for _ in range(pieces - 1)] + [last_chunk_size] + parts = array_ops.split(tensor, piece_lens) + parts[-1] = array_ops.concat( + [parts[-1], array_ops.zeros([pad_len], dtype=tensor.dtype)], 0) + return parts, pad_len + else: + return array_ops.split(tensor, pieces), 0 + + +def _strip_padding(tensors, pad_len): + """Strip the suffix padding added by _padded_split. + + Args: + tensors: list of T `tf.Tensor` of identical length 1D tensors. + pad_len: number of elements to be stripped from the end of each tensor. + + Returns: + list of T `tf.Tensor` which are the stripped inputs. + + Raises: + ValueError: tensors must be a non-empty list of 1D tensors, and + each must be longer than pad_len. + """ + if not tensors: + raise ValueError("tensors cannot be empty") + shape = tensors[0].shape + if len(shape) > 1: + raise ValueError("tensors must be 1D") + prefix_len = int(shape[0] - pad_len) + if prefix_len < 0: + raise ValueError("pad_len longer than tensor") + stripped = [] + for t in tensors: + with ops.colocate_with(t): + stripped.append(array_ops.slice(t, [0], [prefix_len])) + return stripped + + +def _ragged_split(tensor, pieces): + """Like split for 1D tensors but allows case where len % pieces != 0. + + Args: + tensor: T `tf.Tensor` that must be 1D. + pieces: a positive integer specifying the number of pieces into which + tensor should be split. + + Returns: + list of T `tf.Tensor` of length pieces, which hold the values of + the input tensor, in order. The final tensor may be shorter + than the others, which will all be of equal length. + + Raises: + ValueError: input tensor must be 1D. + """ + shape = tensor.shape + if 1 != len(shape): + raise ValueError("input tensor must be 1D") + tensor_len = shape.dims[0].value + chunk_size = tensor_len // pieces + with ops.colocate_with(tensor): + if tensor_len != (pieces * chunk_size): + # last piece will be short + assert pieces > 1 + last_chunk_size = tensor_len - ((pieces - 1) * chunk_size) + assert last_chunk_size > 0 + piece_lens = [chunk_size for _ in range(pieces - 1)] + [last_chunk_size] + return array_ops.split(tensor, piece_lens) + else: + return array_ops.split(tensor, pieces) + + +def _ring_permutations(num_workers, num_subchunks, gpu_perm): + """"Generate an array of device index arrays, one for each subchunk. + + In the basic ring reduction algorithm there are size(T)/num_devices + data chunks and each device process one chunk per tick, i.e. sending + one chunk and receiving one chunk. The idea of subchunking is that + each device processes num_subchunks smaller data regions per tick, + and the ring rank permutation is different for each subchunk index + so that a device is potentially sending to and receiving from + num_subchunks different other devices at each tick. Where multiple + independent data channels exist between devices, this strategy + supplies a method of using them in parallel. + + Args: + num_workers: number of worker tasks + num_subchunks: number of subchunks into which to divide each per-GPU chunk. + gpu_perm: an array of integers in [0, num_gpus-1] giving the default + ring order of GPUs at each worker. Other permutations will be generated + by rotating this array and splicing together per-worker instances. + + Raises: + ValueError: the number of subchunks may not exceed the number of GPUs. + + Returns: + pred_by_s_d: list of lists that maps (by index) from (subchunk, dev) to + preceding device in the permutation for that subchunk. The + device index of GPU i at worker j is i + (j * num_gpus). + rank_by_s_d: list of lists that maps (by index) from (subchunk, dev) to + local rank of device d in the permutation for that subchunk. + """ + num_gpus = len(gpu_perm) + devices = num_workers * num_gpus + if devices == 0: + return [], [] + if num_subchunks > num_gpus: + raise ValueError( + "num_subchunks %d must be <= num_gpus %d" % (num_subchunks, num_gpus)) + rotation_interval = max(1, int(num_gpus / num_subchunks)) + perms_by_s = [] + for s in range(0, num_subchunks): + full_order = [] + offset = s * rotation_interval + for w in range(0, num_workers): + default_order = [(w * num_gpus) + i for i in gpu_perm] + dev_order = default_order[offset:] + default_order[:offset] + full_order += dev_order + perms_by_s.append(full_order) + pred_by_s_d = [[-1 for d in range(0, devices)] + for s in range(0, num_subchunks)] + rank_by_s_d = [[-1 for d in range(0, devices)] + for s in range(0, num_subchunks)] + for s in range(0, num_subchunks): + for d in range(0, devices): + for t in range(0, devices): + if d == perms_by_s[s][t]: + rank_by_s_d[s][d] = t + pred_by_s_d[s][d] = perms_by_s[s][(t + devices - 1) % devices] + break + return (pred_by_s_d, rank_by_s_d) + + +def build_ring_all_reduce(input_tensors, num_workers, num_subchunks, + gpu_perm, red_op, un_op=None): + """Construct a subgraph performing a ring-style all-reduce of input_tensors. + + Args: + input_tensors: a list of T `tf.Tensor` objects, which must all + have the same shape and type. + num_workers: number of worker tasks spanned by input_tensors. + num_subchunks: number of subchunks each device should process in one tick. + gpu_perm: a list of ints giving a ring-wise rank ordering of GPUs at + each worker. All workers must have the same number of + GPUs with the same rank ordering. If NVLINK is available, this should + be a ring order supported by NVLINK edges. + red_op: a binary operator for elementwise reduction. + un_op: an optional unary operator to apply to fully reduced values. + + Raises: + ValueError: empty input_tensors or they don't all have same + size. + + Returns: + a list of T `tf.Tensor` identical sum-reductions of input_tensors. + """ + if len(input_tensors) < 2: + raise ValueError("input_tensors must be length 2 or longer") + input_tensors, shape = _flatten_tensors(input_tensors) + devices = [t.device for t in input_tensors] + (pred_by_s_d, rank_by_s_d) = _ring_permutations( + num_workers, num_subchunks, gpu_perm) + chunks_by_dev, pad_len = _build_ring_gather( + input_tensors, devices, + num_subchunks, pred_by_s_d, rank_by_s_d, red_op) + if un_op: + chunks_by_dev = _apply_unary_to_chunks(un_op, chunks_by_dev) + output_tensors = _build_ring_scatter(pred_by_s_d, rank_by_s_d, + chunks_by_dev) + if pad_len > 0: + output_tensors = _strip_padding(output_tensors, pad_len) + if len(shape) != 1: + output_tensors = _reshape_tensors(output_tensors, shape) + return output_tensors + + +def _build_ring_gather(input_tensors, devices, num_subchunks, + pred_by_s_d, rank_by_s_d, red_op): + """Construct a subgraph for the first (reduction) pass of ring all-reduce. + + Args: + input_tensors: a list of T `tf.Tensor` 1D input tensors of same + shape and type. + devices: array of device name strings + num_subchunks: number of subchunks each device should process in one tick. + pred_by_s_d: as produced by _ring_permutations + rank_by_s_d: as produced by _ring_permutations + red_op: a binary operator for elementwise reduction + + Raises: + ValueError: tensors must all be one dimensional. + + Returns: + list of list of T `tf.Tensor` of (partially) reduced values where + exactly num_subchunks chunks at each device are fully reduced. + """ + num_devices = len(input_tensors) + if num_devices == 0: + return [] + if num_devices == 1: + return input_tensors + shape = input_tensors[0].shape + if 1 != len(shape): + raise ValueError("input tensors must be 1D") + num_chunks = num_devices * num_subchunks + num_ticks = num_devices - 1 + # Initialize chunks_by_dev with splits of the input tensors. + chunks_by_dev = [] + split_pad_len = 0 + for d in range(0, num_devices): + with ops.device(devices[d]): + splits, split_pad_len = _padded_split(input_tensors[d], num_chunks) + chunks_by_dev.append(splits) + # Reduction phase + for tick in range(0, num_ticks): + # One new partial reduction for every chunk + new_partial_reductions = [None for _ in range(0, num_chunks)] + # Compute reductions with respect to last tick's values + for d in range(0, num_devices): + with ops.device(devices[d]): + for s in range(0, num_subchunks): + rank = rank_by_s_d[s][d] + seg_index = (rank + num_devices - (2 + tick)) % num_devices + pred_dev = pred_by_s_d[s][d] + chunk_index = (seg_index * num_subchunks) + s + new_partial_reductions[chunk_index] = red_op( + chunks_by_dev[pred_dev][chunk_index], + chunks_by_dev[d][chunk_index]) + # Update chunks_by_dev with the new values at the end of the tick. + for d in range(0, num_devices): + for s in range(0, num_subchunks): + rank = rank_by_s_d[s][d] + seg_index = (rank + num_devices - (2 + tick)) % num_devices + chunk_index = (seg_index * num_subchunks) + s + chunks_by_dev[d][chunk_index] = new_partial_reductions[chunk_index] + return chunks_by_dev, split_pad_len + + +def _apply_unary_to_chunks(f, chunks_by_dev): + """Apply a unary op to each tensor in chunks_by_dev, on same device. + + Args: + f: a unary function over T `tf.Tensor`. + chunks_by_dev: list of lists of T `tf.Tensor`. + + Returns: + new list of lists of T `tf.Tensor` with the same structure as + chunks_by_dev containing the derived tensors. + """ + output = [] + for x in chunks_by_dev: + with ops.colocate_with(x[0]): + output.append([f(t) for t in x]) + return output + + +def _build_ring_scatter(pred_by_s_d, rank_by_s_d, + chunks_by_dev): + """Construct subgraph for second (scatter) pass of ring all-reduce. + + Args: + pred_by_s_d: as produced by _ring_permutations + rank_by_s_d: as produced by _ring_permutations + chunks_by_dev: list of list of T `tf.Tensor` indexed by ints + (device, chunk) + + Raises: + ValueError: chunks_by_dev is not well-formed + + Returns: + list of T `tf.Tensor` which are the fully reduced tensors, one + at each device corresponding to the outer dimension of chunks_by_dev. + """ + num_devices = len(chunks_by_dev) + num_chunks = len(chunks_by_dev[0]) + if 0 != num_chunks % num_devices: + raise ValueError( + "Expect number of chunks per device to be divisible by num_devices") + num_subchunks = int(num_chunks / num_devices) + num_ticks = num_devices - 1 + for tick in range(0, num_ticks): + passed_values = [None for _ in range(0, num_chunks)] + for d in range(0, num_devices): + with ops.colocate_with(chunks_by_dev[d][0]): + for s in range(0, num_subchunks): + rank = rank_by_s_d[s][d] + seg_index = (rank + num_devices - (1 + tick)) % num_devices + pred_dev = pred_by_s_d[s][d] + chunk_index = (seg_index * num_subchunks) + s + passed_values[chunk_index] = array_ops.identity( + chunks_by_dev[pred_dev][chunk_index]) + for d in range(0, num_devices): + for s in range(0, num_subchunks): + rank = rank_by_s_d[s][d] + seg_index = (rank + num_devices - (1 + tick)) % num_devices + chunk_index = (seg_index * num_subchunks) + s + chunks_by_dev[d][chunk_index] = passed_values[chunk_index] + # Join chunks at each device. + output = [] + for x in chunks_by_dev: + with ops.colocate_with(x[0]): + output.append(array_ops.concat(x, 0)) + return output + + +def build_recursive_hd_all_reduce(input_tensors, red_op, un_op=None): + """Construct a subgraph for recursive halving-doubling all-reduce. + + The recursive halving-doubling algorithm is described in + http://www.mcs.anl.gov/~thakur/papers/ijhpca-coll.pdf + + The concept is to arrange the participating n devices in + a linear sequence where devices exchange data pairwise + with one other device in each round. During the gather + phase there are lg(n) rounds where devices exchange + increasingly smaller sub-tensors with another device + at increasingly greater distances, until at the top + each device has 1/n of the fully reduced values. During the + scatter phase each device exchanges its fully reduced + sub-tensor (which doubles in length at each round) + with one other device at increasingly smaller distances + until each device has all of the fully reduced values. + + Note: this preliminary version requires that len(input_tensors) be a + power of 2. TODO(tucker): relax this restriction. Also, the + number of elements in each tensor must be divisible by 2^h where h + is the number of hops in each phase. This will also be relaxed in + the future with edge-case specific logic. + + Args: + input_tensors: list of T `tf.Tensor` to be elementwise reduced. + red_op: a binary elementwise reduction Op. + un_op: an optional unary elementwise Op to apply to reduced values. + + Returns: + list of T `tf.Tensor` which are the fully reduced tensors, one + at each device of input_tensors. + + Raises: + ValueError: num_devices not a power of 2, or tensor len not divisible + by 2 the proper number of times. + """ + devices = [t.device for t in input_tensors] + input_tensors, shape = _flatten_tensors(input_tensors) + reduced_shards = _build_recursive_hd_gather(input_tensors, devices, red_op) + if un_op: + reduced_shards = [un_op(t) for t in reduced_shards] + output_tensors = _build_recursive_hd_scatter(reduced_shards, devices) + if len(shape) != 1: + output_tensors = _reshape_tensors(output_tensors, shape) + return output_tensors + + +def _build_recursive_hd_gather(input_tensors, devices, red_op): + """Construct the gather phase of recursive halving-doubling all-reduce. + + Args: + input_tensors: list of T `tf.Tensor` to be elementwise reduced. + devices: a list of strings naming the devices hosting input_tensors, + which will also be used to host the (partial) reduction values. + red_op: a binary elementwise reduction Op. + + Returns: + list of T `tf.Tensor` which are the fully reduced tensor shards. + + Raises: + ValueError: num_devices not a power of 2, or tensor len not divisible + by 2 the proper number of times. + """ + num_devices = len(devices) + num_hops = int(math.log(num_devices, 2)) + if num_devices != (2 ** num_hops): + raise ValueError("num_devices must be a power of 2") + chunks = input_tensors + for h in range(0, num_hops): + span = 2 ** h + group_size = span * 2 + new_chunks = [[] for _ in devices] + for d in range(0, num_devices): + if (d % group_size) >= (group_size / 2): + # skip right half of a pair + continue + left_dev = devices[d] + right_dev = devices[d + span] + left_split = array_ops.split(chunks[d], 2) + right_split = array_ops.split(chunks[d+span], 2) + with ops.device(left_dev): + new_chunks[d] = red_op(left_split[0], right_split[0]) + with ops.device(right_dev): + new_chunks[d + span] = red_op(left_split[1], right_split[1]) + chunks = new_chunks + return chunks + + +def _build_recursive_hd_scatter(input_tensors, devices): + """Construct the scatter phase of recursive halving-doublng all-reduce. + + Args: + input_tensors: list of T `tf.Tensor` that are fully-reduced shards. + devices: a list of strings naming the devices on which the reconstituted + full tensors should be placed. + + Returns: + list of T `tf.Tensor` which are the fully reduced tensors. + """ + num_devices = len(devices) + num_hops = int(math.log(num_devices, 2)) + assert num_devices == (2 ** num_hops), "num_devices must be a power of 2" + chunks = input_tensors + for h in reversed(range(0, num_hops)): + span = 2 ** h + group_size = span * 2 + new_chunks = [[] for _ in devices] + for d in range(0, num_devices): + if (d % group_size) >= (group_size / 2): + # skip right half of a pair + continue + left_idx = d + right_idx = d + span + left_dev = devices[left_idx] + right_dev = devices[right_idx] + with ops.device(left_dev): + new_chunks[left_idx] = array_ops.concat([chunks[left_idx], + chunks[right_idx]], 0) + with ops.device(right_dev): + new_chunks[right_idx] = array_ops.concat([chunks[left_idx], + chunks[right_idx]], 0) + chunks = new_chunks + return chunks + + +def build_shuffle_all_reduce(input_tensors, gather_devices, red_op, un_op=None): + """Construct a subgraph for shuffle all-reduce. + + Shuffle reduce is essentially the algorithm implemented when using + parameter servers. Suppose tensor length is n, there are d devices + and g gather shards. Each device sends a n/g length sub-tensor to + each gather shard. The gather shards perform a reduction across d + fragments, then broadcast the result back to each device. The + devices then join the g fully reduced fragments they receive from + the shards. The gather shards could perform d-1 pairwise + reductions, or one d-way reduction. The first is better where + reduction Op time is low compared to transmission time, the second + better in the other case. + + Args: + input_tensors: list of T @(tf.Tensor} values to be reduced. + gather_devices: list of names of devices on which reduction shards + should be placed. + red_op: an n-array elementwise reduction Op + un_op: optional elementwise unary Op to be applied to fully-reduced values. + + Returns: + list of T `tf.Tensor` which are the fully reduced tensors. + """ + input_tensors, shape = _flatten_tensors(input_tensors) + dst_devices = [t.device for t in input_tensors] + reduced_shards = _build_shuffle_gather(input_tensors, gather_devices, + red_op, un_op) + output_tensors = _build_shuffle_scatter(reduced_shards, dst_devices) + if len(shape) != 1: + output_tensors = _reshape_tensors(output_tensors, shape) + return output_tensors + + +def _build_shuffle_gather(input_tensors, gather_devices, red_op, un_op=None): + """Construct the gather (concentrate and reduce) phase of shuffle all-reduce. + + Args: + input_tensors: list of T @(tf.Tensor} values to be reduced. + gather_devices: list of names of devices on which reduction shards + should be placed. + red_op: the binary reduction Op + un_op: optional elementwise unary Op to be applied to fully-reduced values. + + Returns: + list of T `tf.Tensor` which are the fully reduced shards. + + Raises: + ValueError: inputs not well-formed. + """ + num_source_devices = len(input_tensors) + num_gather_devices = len(gather_devices) + shape = input_tensors[0].shape + if len(shape) != 1: + raise ValueError("input_tensors must be 1D") + shards_by_source = [] + for d in range(0, num_source_devices): + with ops.colocate_with(input_tensors[d]): + shards_by_source.append( + _ragged_split(input_tensors[d], num_gather_devices)) + reduced_shards = [] + for d in range(0, num_gather_devices): + with ops.device(gather_devices[d]): + values = [s[d] for s in shards_by_source] + red_shard = red_op(values) + if un_op: + red_shard = un_op(red_shard) + reduced_shards.append(red_shard) + return reduced_shards + + +def _build_shuffle_scatter(reduced_shards, dst_devices): + """Build the scatter phase of shuffle all-reduce. + + Args: + reduced_shards: list of T @(tf.Tensor} fully reduced shards + dst_devices: list of names of devices at which the fully-reduced value + should be reconstituted. + + Returns: + list of T `tf.Tensor` scattered tensors. + """ + num_devices = len(dst_devices) + out_tensors = [] + for d in range(0, num_devices): + with ops.device(dst_devices[d]): + out_tensors.append(array_ops.concat(reduced_shards, 0)) + return out_tensors + + +def _split_by_task(devices, values): + """Partition devices and values by common task. + + Args: + devices: list of device name strings + values: list of T `tf.tensor` of same length as devices. + + Returns: + (per_task_devices, per_task_values) where both values are + lists of lists with isomorphic structure: the outer list is + indexed by task, and the inner list has length of the number + of values belonging to that task. per_task_devices contains + the specific devices to which the values are local, and + per_task_values contains the corresponding values. + + Raises: + ValueError: devices must be same length as values. + """ + num_devices = len(devices) + if num_devices != len(values): + raise ValueError("len(devices) must equal len(values)") + per_task_devices = collections.OrderedDict() + per_task_values = collections.OrderedDict() + for d in range(num_devices): + d_spec = device_lib.DeviceSpec.from_string(devices[d]) + if not hasattr(d_spec, "task") or d_spec.task is None: + assert False, "failed to parse device %s" % devices[d] + index = (d_spec.job or "localhost", d_spec.replica or 0, d_spec.task) + if index not in per_task_devices: + per_task_devices[index] = [] + per_task_values[index] = [] + per_task_devices[index].append(devices[d]) + per_task_values[index].append(values[d]) + + return (list(per_task_devices.values()), list(per_task_values.values())) + + +def build_nccl_all_reduce(input_tensors, red_op, un_op=None): + """Build a subgraph that does one full all-reduce, using NCCL. + + Args: + input_tensors: list of T `tf.Tensor` of same-shape and type values to + be reduced. + red_op: binary elementwise reduction operator. Must be one of + {tf.add} + un_op: optional unary elementwise Op to apply to fully-reduce values. + + Returns: + list of T `tf.Tensor` of reduced values. + + Raises: + ValueError: red_op not supported. + """ + if red_op == math_ops.add: + output_tensors = nccl_ops.all_sum(input_tensors) + else: + raise ValueError("red_op not supported by NCCL all-reduce: ", red_op) + if un_op: + un_op_wrapped = [] + for t in output_tensors: + with ops.colocate_with(t): + un_op_wrapped.append(un_op(t)) + output_tensors = un_op_wrapped + return output_tensors + + +def _build_nccl_hybrid(input_tensors, red_op, upper_level_f): + """Construct a subgraph for NCCL hybrid all-reduce. + + Args: + input_tensors: list of T `tf.Tensor` of same-shape and type values to + be reduced. + red_op: binary elementwise reduction operator. + upper_level_f: function for reducing one value per worker, across + workers. + + Returns: + list of T `tf.Tensor` of reduced values. + + Raises: + ValueError: inputs not well-formed. + """ + input_tensors, shape = _flatten_tensors(input_tensors) + devices = [t.device for t in input_tensors] + per_worker_devices, per_worker_values = _split_by_task(devices, input_tensors) + num_workers = len(per_worker_devices) + up_values = [None for w in range(0, num_workers)] + up_devices = up_values[:] + down_values = up_values[:] + # First stage: reduce within each worker using NCCL + for w in range(0, num_workers): + worker_values = build_nccl_all_reduce(per_worker_values[w], red_op) + # NOTE: these reductions will not run to completion unless + # every output value is used. Since we only need one, we + # need to put control dependencies on the rest. + with ops.control_dependencies(worker_values): + with ops.device(worker_values[0].device): + up_values[w] = array_ops.identity(worker_values[0]) + up_devices[w] = per_worker_devices[w][0] + # Second stage: Apply upper_level_f to reduce across first device at + # each worker + level_2_output = upper_level_f(up_values) + # Third stage: propagate within each worker using NCCL Broadcast + for w in range(0, num_workers): + dst_tensors = [] + with ops.device(per_worker_devices[w][0]): + broadcast_src = nccl_ops.broadcast(array_ops.identity(level_2_output[w])) + for d in per_worker_devices[w]: + with ops.device(d): + dst_tensors.append(array_ops.identity(broadcast_src)) + down_values[w] = dst_tensors + output_tensors = [v for sublist in down_values for v in sublist] + if len(shape) != 1: + output_tensors = _reshape_tensors(output_tensors, shape) + return output_tensors + + +def _reduce_non_singleton(input_tensors, red_f, un_op): + """If len(input_tensors) > 1, apply red_f, else apply un_op.""" + if len(input_tensors) > 1: + return red_f(input_tensors) + else: + if not un_op: + return input_tensors + output_tensors = [] + for t in input_tensors: + with ops.colocate_with(t): + output_tensors.append(un_op(t)) + return output_tensors + + +def build_nccl_then_ring(input_tensors, subdiv, red_op, un_op=None): + """Construct hybrid of NCCL within workers, Ring across workers.""" + def upper_builder(y): + return build_ring_all_reduce(y, len(y), subdiv, [0], red_op, un_op) + def upper_level_f(x): + return _reduce_non_singleton(x, upper_builder, un_op) + return _build_nccl_hybrid(input_tensors, red_op, upper_level_f) + + +def build_nccl_then_recursive_hd(input_tensors, red_op, un_op=None): + """Construct hybrid of NCCL within workers, Recursive-HD across workers.""" + upper_level_f = lambda x: build_recursive_hd_all_reduce(x, red_op, un_op) + return _build_nccl_hybrid(input_tensors, red_op, upper_level_f) + + +def build_nccl_then_shuffle(input_tensors, gather_devices, nccl_red_op, + shuffle_red_op, un_op=None): + """Construct hybrid of NCCL within workers, Shuffle across workers.""" + def upper_level_f(x): + return build_shuffle_all_reduce(x, gather_devices, shuffle_red_op, un_op) + + return _build_nccl_hybrid(input_tensors, nccl_red_op, upper_level_f) + + +def _build_shuffle_hybrid(input_tensors, gather_devices, red_op, upper_level_f): + """Construct a subgraph for Shuffle hybrid all-reduce. + + Args: + input_tensors: list of T `tf.Tensor` of same-shape and type values to + be reduced. + gather_devices: list of device names on which to host gather shards. + red_op: binary elementwise reduction operator. + upper_level_f: function for reducing one value per worker, across + workers. + + Returns: + list of T `tf.Tensor` of reduced values. + + Raises: + ValueError: inputs not well-formed. + """ + input_tensors, shape = _flatten_tensors(input_tensors) + # First stage, reduce across each worker using gather_devices. + devices = [t.device for t in input_tensors] + per_worker_devices, per_worker_values = _split_by_task(devices, input_tensors) + num_workers = len(per_worker_devices) + up_values = [] + if len(gather_devices) != num_workers: + raise ValueError("For shuffle hybrid, gather_devices must contain one " + "device per worker. ") + for w in range(0, num_workers): + reduced_shards = _build_shuffle_gather( + per_worker_values[w], [gather_devices[w]], red_op) + up_values.append(reduced_shards[0]) + # Second stage, apply upper_level_f. + level_2_output = upper_level_f(up_values) + # Third stage, apply shuffle scatter at each worker. + output_tensors = [] + for w in range(0, num_workers): + output_tensors += _build_shuffle_scatter( + [level_2_output[w]], per_worker_devices[w]) + if len(shape) != 1: + output_tensors = _reshape_tensors(output_tensors, shape) + return output_tensors + + +def build_shuffle_then_ring(input_tensors, gather_devices, subdiv, + red_n_op, red_op, un_op=None): + """Construct hybrid of Shuffle within workers, Ring across workers.""" + def upper_builder(tensors): + return build_ring_all_reduce(tensors, len(tensors), subdiv, [0], + red_op, un_op) + def upper_level_f(tensors): + return _reduce_non_singleton(tensors, upper_builder, un_op) + return _build_shuffle_hybrid( + input_tensors, gather_devices, red_n_op, upper_level_f) + + +def build_shuffle_then_shuffle(input_tensors, first_gather_devices, + second_gather_devices, red_op, un_op=None): + """Construct hybrid of Shuffle within workers, Shuffle across workers.""" + def upper_builder(tensors): + return build_shuffle_all_reduce(tensors, second_gather_devices, + red_op, un_op) + def upper_level_f(tensors): + return _reduce_non_singleton(tensors, upper_builder, un_op) + return _build_shuffle_hybrid( + input_tensors, first_gather_devices, red_op, upper_level_f) diff --git a/tensorflow/contrib/all_reduce/python/all_reduce_test.py b/tensorflow/python/distribute/all_reduce_test.py similarity index 97% rename from tensorflow/contrib/all_reduce/python/all_reduce_test.py rename to tensorflow/python/distribute/all_reduce_test.py index 304fd7fb8a3..2c6b853124c 100644 --- a/tensorflow/contrib/all_reduce/python/all_reduce_test.py +++ b/tensorflow/python/distribute/all_reduce_test.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================== -"""Tests for tensorflow.contrib.all_reduce.python..all_reduce.""" +"""Tests for all_reduce.""" from __future__ import absolute_import from __future__ import division @@ -22,8 +22,8 @@ import time import numpy as np -from tensorflow.contrib.all_reduce.python import all_reduce as ar from tensorflow.core.framework import types_pb2 +from tensorflow.python.distribute import all_reduce as ar from tensorflow.python.framework import constant_op from tensorflow.python.framework import ops from tensorflow.python.framework import tensor_shape @@ -37,6 +37,7 @@ from tensorflow.python.platform import tf_logging class AllReduceTest(test_util.TensorFlowTestCase): + @test_util.run_deprecated_v1 def testFlattenTensorsShapesDefined(self): x = array_ops.placeholder(types_pb2.DT_FLOAT, [None]) with self.assertRaisesRegexp(ValueError, @@ -100,6 +101,7 @@ class AllReduceTest(test_util.TensorFlowTestCase): input_tensors.append(array_ops.identity(t8)) return input_tensors, device_names + @test_util.run_deprecated_v1 def testBuildRingGatherPassStructure(self): # 1 worker, 1 device input_tensors, device_names = self._buildInput(1, 1) @@ -159,7 +161,7 @@ class AllReduceTest(test_util.TensorFlowTestCase): output_tensors = build_f(input_tensors, un_op) sum_reduced = math_ops.add_n(output_tensors) sum_reduced.op.run() - self.assertAllClose(sum_reduced.eval(), simple_sum.eval()) + self.assertAllClose(sum_reduced.eval(), self.evaluate(simple_sum)) def _testRingAllReduce(self, num_workers, num_gpus, shape, subdiv): start_time = time.time() @@ -170,6 +172,7 @@ class AllReduceTest(test_util.TensorFlowTestCase): "subdiv=%d elapsed=%f" % (num_workers, num_gpus, shape, subdiv, elapsed)) + @test_util.run_deprecated_v1 def testRingAllReduce(self): self._testRingAllReduce(1, 2, [], 1) self._testRingAllReduce(1, 2, [8], 1) @@ -199,6 +202,7 @@ class AllReduceTest(test_util.TensorFlowTestCase): tf_logging.info("ShuffleAllReduce num_workers=%d num_gpus=%d shape=%s " "elapsed=%f" % (num_workers, num_gpus, shape, elapsed)) + @test_util.run_deprecated_v1 def testShuffleAllReduce(self): self._testShuffleAllReduce(1, 2, [], 1) self._testShuffleAllReduce(1, 2, [8], 1) @@ -225,6 +229,7 @@ class AllReduceTest(test_util.TensorFlowTestCase): "shape=%s elapsed=%f" % (num_workers, num_gpus, shape, elapsed)) + @test_util.run_deprecated_v1 def testRecursiveHDAllReduce(self): self._testRecursiveHDAllReduce(1, 2, [8]) self._testRecursiveHDAllReduce(1, 2, [4, 4]) diff --git a/tensorflow/python/distribute/cluster_resolver/BUILD b/tensorflow/python/distribute/cluster_resolver/BUILD new file mode 100644 index 00000000000..360a2993cd9 --- /dev/null +++ b/tensorflow/python/distribute/cluster_resolver/BUILD @@ -0,0 +1,180 @@ +# Description: Operations defined for Cluster Resolvers + +load("//tensorflow:tensorflow.bzl", "tf_py_test") + +package( + default_visibility = [ + "//tensorflow:__subpackages__", + ], +) + +licenses(["notice"]) # Apache 2.0 + +py_library( + name = "cluster_resolver_lib", + srcs = [ + "__init__.py", + ], + srcs_version = "PY2AND3", + visibility = ["//visibility:public"], + deps = [ + ":base_cluster_resolver_py", + ":gce_cluster_resolver_py", + ":kubernetes_cluster_resolver_py", + ":slurm_cluster_resolver_py", + ":tfconfig_cluster_resolver_py", + ":tpu_cluster_resolver_py", + "//tensorflow/python:util", + ], +) + +py_library( + name = "base_cluster_resolver_py", + srcs = ["cluster_resolver.py"], + srcs_version = "PY2AND3", + deps = [ + "//tensorflow/python:training_server_lib", + ], +) + +py_library( + name = "gce_cluster_resolver_py", + srcs = ["gce_cluster_resolver.py"], + srcs_version = "PY2AND3", + deps = [ + ":base_cluster_resolver_py", + "//tensorflow/python:training_server_lib", + ], +) + +py_library( + name = "tfconfig_cluster_resolver_py", + srcs = ["tfconfig_cluster_resolver.py"], + srcs_version = "PY2AND3", + deps = [ + ":base_cluster_resolver_py", + "//tensorflow/python:training_server_lib", + ], +) + +py_library( + name = "tpu_cluster_resolver_py", + srcs = ["tpu_cluster_resolver.py"], + srcs_version = "PY2AND3", + deps = [ + ":base_cluster_resolver_py", + "//tensorflow/python:training_server_lib", + ], +) + +py_library( + name = "slurm_cluster_resolver_py", + srcs = ["slurm_cluster_resolver.py"], + srcs_version = "PY2AND3", + deps = [ + ":base_cluster_resolver_py", + "//tensorflow/python:training_server_lib", + ], +) + +py_library( + name = "kubernetes_cluster_resolver_py", + srcs = ["kubernetes_cluster_resolver.py"], + srcs_version = "PY2AND3", + deps = [ + ":base_cluster_resolver_py", + "//tensorflow/python:training_server_lib", + ], +) + +tf_py_test( + name = "base_cluster_resolver_py_test", + srcs = ["cluster_resolver_test.py"], + additional_deps = [ + ":base_cluster_resolver_py", + "//tensorflow/python:client_testlib", + "//tensorflow/python:framework_for_generated_wrappers", + "//tensorflow/python:framework_test_lib", + "//tensorflow/python:platform_test", + "//tensorflow/python:training_server_lib", + ], + main = "cluster_resolver_test.py", +) + +tf_py_test( + name = "gce_cluster_resolver_py_test", + size = "small", + srcs = ["gce_cluster_resolver_test.py"], + additional_deps = [ + ":gce_cluster_resolver_py", + "//tensorflow/python:client_testlib", + "//tensorflow/python:framework_for_generated_wrappers", + "//tensorflow/python:framework_test_lib", + "//tensorflow/python:platform_test", + "//tensorflow/python:training_server_lib", + ], + main = "gce_cluster_resolver_test.py", +) + +tf_py_test( + name = "tfconfig_cluster_resolver_py_test", + size = "small", + srcs = ["tfconfig_cluster_resolver_test.py"], + additional_deps = [ + ":tfconfig_cluster_resolver_py", + "//tensorflow/python:client_testlib", + "//tensorflow/python:framework_for_generated_wrappers", + "//tensorflow/python:framework_test_lib", + "//tensorflow/python:platform_test", + "//tensorflow/python:training_server_lib", + ], + grpc_enabled = True, + main = "tfconfig_cluster_resolver_test.py", +) + +tf_py_test( + name = "tpu_cluster_resolver_py_test", + size = "small", + srcs = ["tpu_cluster_resolver_test.py"], + additional_deps = [ + ":tpu_cluster_resolver_py", + "//tensorflow/python:client_testlib", + "//tensorflow/python:framework_for_generated_wrappers", + "//tensorflow/python:framework_test_lib", + "//tensorflow/python:platform_test", + "//tensorflow/python:training_server_lib", + ], + grpc_enabled = True, + main = "tpu_cluster_resolver_test.py", +) + +tf_py_test( + name = "slurm_cluster_resolver_py_test", + size = "small", + srcs = ["slurm_cluster_resolver_test.py"], + additional_deps = [ + ":slurm_cluster_resolver_py", + "//tensorflow/python:client_testlib", + "//tensorflow/python:framework_for_generated_wrappers", + "//tensorflow/python:framework_test_lib", + "//tensorflow/python:platform_test", + "//tensorflow/python:training_server_lib", + ], + main = "slurm_cluster_resolver_test.py", + tags = [], +) + +tf_py_test( + name = "kubernetes_cluster_resolver_py_test", + size = "small", + srcs = ["kubernetes_cluster_resolver_test.py"], + additional_deps = [ + ":kubernetes_cluster_resolver_py", + "//tensorflow/python:client_testlib", + "//tensorflow/python:framework_for_generated_wrappers", + "//tensorflow/python:framework_test_lib", + "//tensorflow/python:platform_test", + "//tensorflow/python:training_server_lib", + ], + main = "kubernetes_cluster_resolver_test.py", +) diff --git a/tensorflow/contrib/cluster_resolver/README.md b/tensorflow/python/distribute/cluster_resolver/README.md similarity index 100% rename from tensorflow/contrib/cluster_resolver/README.md rename to tensorflow/python/distribute/cluster_resolver/README.md diff --git a/tensorflow/contrib/cluster_resolver/python/training/README.slurm b/tensorflow/python/distribute/cluster_resolver/README.slurm similarity index 100% rename from tensorflow/contrib/cluster_resolver/python/training/README.slurm rename to tensorflow/python/distribute/cluster_resolver/README.slurm diff --git a/tensorflow/python/distribute/cluster_resolver/__init__.py b/tensorflow/python/distribute/cluster_resolver/__init__.py new file mode 100644 index 00000000000..ef87f59b7fd --- /dev/null +++ b/tensorflow/python/distribute/cluster_resolver/__init__.py @@ -0,0 +1,57 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Library Imports for Cluster Resolvers.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from tensorflow.python.distribute.cluster_resolver import cluster_resolver +from tensorflow.python.distribute.cluster_resolver import gce_cluster_resolver +from tensorflow.python.distribute.cluster_resolver import kubernetes_cluster_resolver +from tensorflow.python.distribute.cluster_resolver import slurm_cluster_resolver +from tensorflow.python.distribute.cluster_resolver import tfconfig_cluster_resolver +from tensorflow.python.distribute.cluster_resolver import tpu_cluster_resolver + +from tensorflow.python.distribute.cluster_resolver.cluster_resolver import ClusterResolver +from tensorflow.python.distribute.cluster_resolver.cluster_resolver import SimpleClusterResolver +from tensorflow.python.distribute.cluster_resolver.cluster_resolver import UnionClusterResolver +from tensorflow.python.distribute.cluster_resolver.gce_cluster_resolver import GceClusterResolver +from tensorflow.python.distribute.cluster_resolver.kubernetes_cluster_resolver import KubernetesClusterResolver +from tensorflow.python.distribute.cluster_resolver.slurm_cluster_resolver import SlurmClusterResolver +from tensorflow.python.distribute.cluster_resolver.tfconfig_cluster_resolver import TFConfigClusterResolver +from tensorflow.python.distribute.cluster_resolver.tpu_cluster_resolver import TPUClusterResolver + +from tensorflow.python.util.all_util import remove_undocumented + +_allowed_symbols = [ + 'cluster_resolver', + 'gce_cluster_resolver', + 'kubernetes_cluster_resolver', + 'slurm_cluster_resolver', + 'tfconfig_cluster_resolver', + 'tpu_cluster_resolver', + 'ClusterResolver', + 'SimpleClusterResolver', + 'UnionClusterResolver', + 'GceClusterResolver', + 'KubernetesClusterResolver', + 'TFConfigClusterResolver', + 'TPUClusterResolver', + 'SlurmClusterResolver', +] + +remove_undocumented(__name__, _allowed_symbols) + diff --git a/tensorflow/python/distribute/cluster_resolver/cluster_resolver.py b/tensorflow/python/distribute/cluster_resolver/cluster_resolver.py new file mode 100644 index 00000000000..7774ac0e122 --- /dev/null +++ b/tensorflow/python/distribute/cluster_resolver/cluster_resolver.py @@ -0,0 +1,374 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Cluster Resolvers are used for dynamic cluster IP/hostname resolution.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import abc + +import six + +from tensorflow.python.training.server_lib import ClusterSpec + + +def format_master_url(master, rpc_layer=None): + if rpc_layer: + return '%s://%s' % (rpc_layer, master) + else: + return master + + +@six.add_metaclass(abc.ABCMeta) +class ClusterResolver(object): + """Abstract class for all implementations of ClusterResolvers. + + This defines the skeleton for all implementations of ClusterResolvers. + ClusterResolvers are a way for TensorFlow to communicate with various cluster + management systems (e.g. GCE, AWS, etc...). + + By letting TensorFlow communicate with these systems, we will be able to + automatically discover and resolve IP addresses for various TensorFlow + workers. This will eventually allow us to automatically recover from + underlying machine failures and scale TensorFlow worker clusters up and down. + + Note to Implementors: In addition to these abstract methods, you must also + implement the task_type, task_index, and rpc_layer attributes. You may choose + to implement them either as properties with getters or setters or directly + set the attributes. + + - task_type is the name of the server's current named job (e.g. 'worker', + 'ps' in a distributed parameterized training job). + - task_index is the ordinal index of the server within the task type. + - rpc_layer is the protocol used by TensorFlow to communicate with other + TensorFlow servers in a distributed environment. + """ + + @abc.abstractmethod + def cluster_spec(self): + """Retrieve the current state of the cluster and returns a ClusterSpec. + + Returns: + A ClusterSpec representing the state of the cluster at the moment this + function is called. + + Implementors of this function must take care in ensuring that the + ClusterSpec returned is up-to-date at the time of calling this function. + This usually means retrieving the information from the underlying cluster + management system every time this function is invoked and reconstructing + a cluster_spec, rather than attempting to cache anything. + """ + raise NotImplementedError() + + @abc.abstractmethod + def master(self, task_type=None, task_index=None, rpc_layer=None): + """Retrieves the name or URL of the session master. + + Args: + task_type: (Optional) The type of the TensorFlow task of the master. + task_index: (Optional) The index of the TensorFlow task of the master. + rpc_layer: (Optional) The RPC protocol for the given cluster. + + Returns: + The name or URL of the session master. + + Implementors of this function must take care in ensuring that the master + returned is up-to-date at the time to calling this function. This usually + means retrieving the master every time this function is invoked. + """ + raise NotImplementedError() + + @abc.abstractmethod + def num_accelerators_per_worker(self, session_config=None): + """Returns the number of accelerator cores per worker. + + This returns the number of accelerator cores (such as GPUs and TPUs) + available per worker. If workers only has CPU cores available, then this + should return 0. This method will query the master for this information + if it is not otherwise known. + + Args: + session_config: (Optional) Configuration for starting a new session to + query how many accelerator cores it has. + """ + raise NotImplementedError() + + @abc.abstractproperty + def environment(self): + """Returns the current environment which TensorFlow is running in.""" + raise NotImplementedError() + + +class SimpleClusterResolver(ClusterResolver): + """Simple implementation of ClusterResolver that accepts a ClusterSpec.""" + + def __init__(self, cluster_spec, master='', task_type=None, task_index=None, + environment='', num_accelerators_per_worker=0, + rpc_layer=None): + """Creates a SimpleClusterResolver from a ClusterSpec.""" + super(SimpleClusterResolver, self).__init__() + + self._task_type = task_type + self._task_index = task_index + self._environment = environment + self._num_accelerators_per_worker = num_accelerators_per_worker + self._rpc_layer = rpc_layer + + if not isinstance(cluster_spec, ClusterSpec): + raise TypeError('cluster_spec must be a ClusterSpec.') + self._cluster_spec = cluster_spec + + if not isinstance(master, str): + raise TypeError('master must be a string.') + self._master = master + + def cluster_spec(self): + """Returns the ClusterSpec passed into the constructor.""" + return self._cluster_spec + + def master(self, task_type=None, task_index=None, rpc_layer=None): + """Returns the master address to use when creating a session. + + Args: + task_type: (Optional) The type of the TensorFlow task of the master. + task_index: (Optional) The index of the TensorFlow task of the master. + rpc_layer: (Optional) The RPC used by distributed TensorFlow. + + Returns: + The name or URL of the session master. + + If a task_type and task_index is given, this will override the `master` + string passed into the initialization function. + """ + if task_type is not None and task_index is not None: + master = self.cluster_spec().task_address(task_type, task_index) + else: + master = self._master + + return format_master_url(master, rpc_layer=rpc_layer or self._rpc_layer) + + @property + def task_type(self): + return self._task_type + + @property + def task_index(self): + return self._task_index + + @task_type.setter + def task_type(self, task_type): + self._task_type = task_type + + @task_index.setter + def task_index(self, task_index): + self._task_index = task_index + + @property + def environment(self): + return self._environment + + def num_accelerators_per_worker(self, session_config=None): + """Returns the number of accelerator cores per worker. + + Args: + session_config: Unused. The SimpleClusterResolver does not do automatic + detection of accelerators, so a TensorFlow session will never be + created, and thus a `session_config` is never necessary here, and will + be ignored. + """ + del session_config + return self._num_accelerators_per_worker + + @property + def rpc_layer(self): + return self._rpc_layer + + @rpc_layer.setter + def rpc_layer(self, rpc_layer): + self._rpc_layer = rpc_layer + + +class UnionClusterResolver(ClusterResolver): + """Performs a union on underlying ClusterResolvers. + + This class performs a union given two or more existing ClusterResolvers. It + merges the underlying ClusterResolvers, and returns one unified ClusterSpec + when cluster_spec is called. The details of the merge function is + documented in the cluster_spec function. + + For additional Cluster Resolver properties such as task type, task index, + rpc layer, environment, etc..., we will return the value from the first + ClusterResolver in the union. + """ + + def __init__(self, *args, **kwargs): + """Initializes a UnionClusterResolver with other ClusterResolvers. + + Args: + *args: `ClusterResolver` objects to be unionized. + **kwargs: + rpc_layer - (Optional) Override value for the RPC layer used by + TensorFlow. + task_type - (Optional) Override value for the current task type. + task_index - (Optional) Override value for the current task index. + + Raises: + TypeError: If any argument is not a subclass of `ClusterResolvers`. + ValueError: If there are no arguments passed. + """ + super(UnionClusterResolver, self).__init__() + + self._rpc_layer = kwargs.pop('rpc_layer', None) + self._task_type = kwargs.pop('task_type', None) + self._task_index = kwargs.pop('task_index', None) + + if kwargs: + raise ValueError('Unexpected kwargs provided {!r}'.format(kwargs)) + + if not args: + raise ValueError('At least one ClusterResolver is required.') + + for cluster_resolver in args: + if not isinstance(cluster_resolver, ClusterResolver): + raise TypeError('All arguments must be a sub-class of ' + '`ClusterResolver.`') + self._cluster_resolvers = args + + def cluster_spec(self): + """Returns a union of all the ClusterSpecs from the ClusterResolvers. + + Returns: + A ClusterSpec containing host information merged from all the underlying + ClusterResolvers. + + Raises: + KeyError: If there are conflicting keys detected when merging two or + more dictionaries, this exception is raised. + + Note: If there are multiple ClusterResolvers exposing ClusterSpecs with the + same job name, we will merge the list/dict of workers. + + If *all* underlying ClusterSpecs expose the set of workers as lists, we will + concatenate the lists of workers, starting with the list of workers from + the first ClusterResolver passed into the constructor. + + If *any* of the ClusterSpecs expose the set of workers as a dict, we will + treat all the sets of workers as dicts (even if they are returned as lists) + and will only merge them into a dict if there is no conflicting keys. If + there is a conflicting key, we will raise a `KeyError`. + """ + + merged_cluster = {} + + # We figure out whether it is all lists for a particular job, or whether + # there are dicts inside. + for cluster_resolver in self._cluster_resolvers: + cluster_spec = cluster_resolver.cluster_spec() + cluster_dict = cluster_spec.as_dict() + + for job_name, tasks in cluster_dict.items(): + if job_name in merged_cluster: + # If we see a dict, then we write a dict out regardless. + if isinstance(tasks, dict): + merged_cluster[job_name] = {} + else: + # We take whichever type is present. + if isinstance(tasks, list): + merged_cluster[job_name] = [] + else: + merged_cluster[job_name] = {} + + # We then do the merge as appropriate in merged_cluster[job]. + for cluster_resolver in self._cluster_resolvers: + cluster_spec = cluster_resolver.cluster_spec() + cluster_dict = cluster_spec.as_dict() + + for job_name, tasks in cluster_dict.items(): + if isinstance(merged_cluster[job_name], list): + # We all have lists, we can just concatenate and be done. + merged_cluster[job_name].extend(tasks) + else: + if isinstance(tasks, list): + # We convert to a dictionary if the type is a list. + task_dict = dict(zip(range(0, len(tasks)), tasks)) + else: + # We can simply make a copy (for update) and be done. + task_dict = tasks.copy() + + # We detect if there are duplicates, and raise an error if so. + task_keys = set(task_dict) + merged_keys = set(merged_cluster[job_name].keys()) + intersected_keys = task_keys.intersection(merged_keys) + if intersected_keys: + raise KeyError('Duplicate keys detected when merging two ' + 'ClusterSpecs: %s' % repr(intersected_keys)) + + # We do the merge after all the processing. + merged_cluster[job_name].update(task_dict) + + return ClusterSpec(merged_cluster) + + def master(self, task_type=None, task_index=None, rpc_layer=None): + """Returns the master address to use when creating a session. + + This usually returns the master from the first ClusterResolver passed in, + but you can override this by specifying the task_type and task_index. + + Args: + task_type: (Optional) The type of the TensorFlow task of the master. + task_index: (Optional) The index of the TensorFlow task of the master. + rpc_layer: (Optional) The RPC protocol for the given cluster. + + Returns: + The name or URL of the session master. + """ + if task_type is not None and task_index is not None: + master = self.cluster_spec().task_address(task_type, task_index) + return format_master_url(master, rpc_layer or self._rpc_layer) + + return self._cluster_resolvers[0].master(rpc_layer=rpc_layer) + + @property + def task_type(self): + return self._task_type or self._cluster_resolvers[0].task_type + + @property + def task_index(self): + return self._task_index or self._cluster_resolvers[0].task_index + + @task_type.setter + def task_type(self, task_type): + self._task_type = task_type + + @task_index.setter + def task_index(self, task_index): + self._task_index = task_index + + @property + def environment(self): + return self._cluster_resolvers[0].environment + + def num_accelerators_per_worker(self, session_config=None): + return self._cluster_resolvers[0].num_accelerators_per_worker( + session_config) + + @property + def rpc_layer(self): + return self._rpc_layer or self._cluster_resolvers[0].rpc_layer + + @rpc_layer.setter + def rpc_layer(self, rpc_layer): + self._rpc_layer = rpc_layer diff --git a/tensorflow/contrib/cluster_resolver/python/training/cluster_resolver_test.py b/tensorflow/python/distribute/cluster_resolver/cluster_resolver_test.py similarity index 98% rename from tensorflow/contrib/cluster_resolver/python/training/cluster_resolver_test.py rename to tensorflow/python/distribute/cluster_resolver/cluster_resolver_test.py index b94c9612b5b..b5448faec6b 100644 --- a/tensorflow/contrib/cluster_resolver/python/training/cluster_resolver_test.py +++ b/tensorflow/python/distribute/cluster_resolver/cluster_resolver_test.py @@ -18,8 +18,8 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function -from tensorflow.contrib.cluster_resolver.python.training.cluster_resolver import SimpleClusterResolver -from tensorflow.contrib.cluster_resolver.python.training.cluster_resolver import UnionClusterResolver +from tensorflow.python.distribute.cluster_resolver import SimpleClusterResolver +from tensorflow.python.distribute.cluster_resolver import UnionClusterResolver from tensorflow.python.platform import test from tensorflow.python.training import server_lib diff --git a/tensorflow/python/distribute/cluster_resolver/gce_cluster_resolver.py b/tensorflow/python/distribute/cluster_resolver/gce_cluster_resolver.py new file mode 100644 index 00000000000..b167bc8fc85 --- /dev/null +++ b/tensorflow/python/distribute/cluster_resolver/gce_cluster_resolver.py @@ -0,0 +1,206 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Implementation of Cluster Resolvers for GCE Instance Groups.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from tensorflow.python.distribute.cluster_resolver.cluster_resolver import ClusterResolver +from tensorflow.python.training.server_lib import ClusterSpec + +_GOOGLE_API_CLIENT_INSTALLED = True +try: + from googleapiclient import discovery # pylint: disable=g-import-not-at-top + from oauth2client.client import GoogleCredentials # pylint: disable=g-import-not-at-top +except ImportError: + _GOOGLE_API_CLIENT_INSTALLED = False + + +def _format_master_url(master, rpc_layer=None): + return '%s://%s' % (rpc_layer, master) if rpc_layer else master + + +class GceClusterResolver(ClusterResolver): + """Cluster Resolver for Google Compute Engine. + + This is an implementation of cluster resolvers for the Google Compute Engine + instance group platform. By specifying a project, zone, and instance group, + this will retrieve the IP address of all the instances within the instance + group and return a Cluster Resolver object suitable for use for distributed + TensorFlow. + """ + + def __init__(self, + project, + zone, + instance_group, + port, + task_type='worker', + task_index=0, + rpc_layer='grpc', + num_accelerators_per_worker=0, + credentials='default', + service=None): + """Creates a new GceClusterResolver object. + + This takes in a few parameters and creates a GceClusterResolver project. It + will then use these parameters to query the GCE API for the IP addresses of + each instance in the instance group. + + Args: + project: Name of the GCE project. + zone: Zone of the GCE instance group. + instance_group: Name of the GCE instance group. + port: Port of the listening TensorFlow server (default: 8470) + task_type: Name of the TensorFlow job this GCE instance group of VM + instances belong to. + task_index: The task index for this particular VM, within the GCE + instance group. In particular, every single instance should be assigned + a unique ordinal index within an instance group manually so that they + can be distinguished from each other. + rpc_layer: The RPC layer TensorFlow should use to communicate across + instances. + num_accelerators_per_worker: Number of accelerators (GPUs) present per + instance. + credentials: GCE Credentials. If nothing is specified, this defaults to + GoogleCredentials.get_application_default(). + service: The GCE API object returned by the googleapiclient.discovery + function. (Default: discovery.build('compute', 'v1')). If you specify a + custom service object, then the credentials parameter will be ignored. + + Raises: + ImportError: If the googleapiclient is not installed. + """ + self._project = project + self._zone = zone + self._instance_group = instance_group + self._task_type = task_type + self._task_index = task_index + self._rpc_layer = rpc_layer + self._port = port + self._credentials = credentials + + if credentials == 'default': + if _GOOGLE_API_CLIENT_INSTALLED: + self._credentials = GoogleCredentials.get_application_default() + + if service is None: + if not _GOOGLE_API_CLIENT_INSTALLED: + raise ImportError('googleapiclient must be installed before using the ' + 'GCE cluster resolver') + self._service = discovery.build( + 'compute', 'v1', + credentials=self._credentials) + else: + self._service = service + + def cluster_spec(self): + """Returns a ClusterSpec object based on the latest instance group info. + + This returns a ClusterSpec object for use based on information from the + specified instance group. We will retrieve the information from the GCE APIs + every time this method is called. + + Returns: + A ClusterSpec containing host information retrieved from GCE. + """ + request_body = {'instanceState': 'RUNNING'} + request = self._service.instanceGroups().listInstances( + project=self._project, + zone=self._zone, + instanceGroups=self._instance_group, + body=request_body, + orderBy='name') + + worker_list = [] + + while request is not None: + response = request.execute() + + items = response['items'] + for instance in items: + instance_name = instance['instance'].split('/')[-1] + + instance_request = self._service.instances().get( + project=self._project, + zone=self._zone, + instance=instance_name) + + if instance_request is not None: + instance_details = instance_request.execute() + ip_address = instance_details['networkInterfaces'][0]['networkIP'] + instance_url = '%s:%s' % (ip_address, self._port) + worker_list.append(instance_url) + + request = self._service.instanceGroups().listInstances_next( + previous_request=request, + previous_response=response) + + worker_list.sort() + return ClusterSpec({self._task_type: worker_list}) + + def master(self, task_type=None, task_index=None, rpc_layer=None): + task_type = task_type if task_type is not None else self._task_type + task_index = task_index if task_index is not None else self._task_index + + if task_type is not None and task_index is not None: + master = self.cluster_spec().task_address(task_type, task_index) + if rpc_layer or self._rpc_layer: + return '%s://%s' % (rpc_layer or self._rpc_layer, master) + else: + return master + + return '' + + @property + def task_type(self): + return self._task_type + + @property + def task_index(self): + return self._task_index + + @task_type.setter + def task_type(self, task_type): + raise RuntimeError( + 'You cannot reset the task_type of the GceClusterResolver after it has ' + 'been created.') + + @task_index.setter + def task_index(self, task_index): + self._task_index = task_index + + @property + def environment(self): + """Returns the current environment which TensorFlow is running in. + + For users in the GCE environment, the environment property is always an + empty string, and Google users will not use this ClusterResolver for running + on internal systems. + """ + return '' + + @property + def rpc_layer(self): + return self._rpc_layer + + @rpc_layer.setter + def rpc_layer(self, rpc_layer): + self._rpc_layer = rpc_layer + + def num_accelerators_per_worker(self, session_config=None): + del session_config # Unused, since this is set manually in __init__. + return self._num_accelerators_per_worker diff --git a/tensorflow/contrib/cluster_resolver/python/training/gce_cluster_resolver_test.py b/tensorflow/python/distribute/cluster_resolver/gce_cluster_resolver_test.py similarity index 98% rename from tensorflow/contrib/cluster_resolver/python/training/gce_cluster_resolver_test.py rename to tensorflow/python/distribute/cluster_resolver/gce_cluster_resolver_test.py index c691552e860..d4f0660c922 100644 --- a/tensorflow/contrib/cluster_resolver/python/training/gce_cluster_resolver_test.py +++ b/tensorflow/python/distribute/cluster_resolver/gce_cluster_resolver_test.py @@ -18,8 +18,8 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function -from tensorflow.contrib.cluster_resolver.python.training.cluster_resolver import UnionClusterResolver -from tensorflow.contrib.cluster_resolver.python.training.gce_cluster_resolver import GceClusterResolver +from tensorflow.python.distribute.cluster_resolver import GceClusterResolver +from tensorflow.python.distribute.cluster_resolver import UnionClusterResolver from tensorflow.python.platform import test from tensorflow.python.training import server_lib diff --git a/tensorflow/python/distribute/cluster_resolver/kubernetes_cluster_resolver.py b/tensorflow/python/distribute/cluster_resolver/kubernetes_cluster_resolver.py new file mode 100644 index 00000000000..041c0815409 --- /dev/null +++ b/tensorflow/python/distribute/cluster_resolver/kubernetes_cluster_resolver.py @@ -0,0 +1,173 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Implementation of Cluster Resolvers for Kubernetes.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from tensorflow.python.client import device_lib +from tensorflow.python.distribute.cluster_resolver.cluster_resolver import ClusterResolver +from tensorflow.python.distribute.cluster_resolver.cluster_resolver import format_master_url +from tensorflow.python.training import server_lib + +_KUBERNETES_API_CLIENT_INSTALLED = True +try: + from kubernetes import client as k8sclient # pylint: disable=g-import-not-at-top + from kubernetes import config as k8sconfig # pylint: disable=g-import-not-at-top +except ImportError: + _KUBERNETES_API_CLIENT_INSTALLED = False + + +class KubernetesClusterResolver(ClusterResolver): + """Cluster Resolver for Kubernetes. + + This is an implementation of cluster resolvers for Kubernetes. When given the + the Kubernetes namespace and label selector for pods, we will retrieve the + pod IP addresses of all running pods matching the selector, and return a + ClusterSpec based on that information. + """ + + def __init__(self, + job_to_label_mapping=None, + tf_server_port=8470, + rpc_layer='grpc', + override_client=None): + """Initializes a new KubernetesClusterResolver. + + This initializes a new Kubernetes Cluster Resolver. The Cluster Resolver + will attempt to talk to the Kubernetes master to retrieve all the instances + of pods matching a label selector. + + Args: + job_to_label_mapping: A mapping of TensorFlow jobs to label selectors. + This allows users to specify many TensorFlow jobs in one Cluster + Resolver, and each job can have pods belong with different label + selectors. For example, a sample mapping might be + ``` + {'worker': ['job-name=worker-cluster-a', 'job-name=worker-cluster-b'], + 'ps': ['job-name=ps-1', 'job-name=ps-2']} + ``` + tf_server_port: The port the TensorFlow server is listening on. + rpc_layer: (Optional) The RPC layer TensorFlow should use to communicate + between tasks in Kubernetes. Defaults to 'grpc'. + override_client: The Kubernetes client (usually automatically retrieved + using `from kubernetes import client as k8sclient`). If you pass this + in, you are responsible for setting Kubernetes credentials manually. + + Raises: + ImportError: If the Kubernetes Python client is not installed and no + `override_client` is passed in. + RuntimeError: If autoresolve_task is not a boolean or a callable. + """ + if _KUBERNETES_API_CLIENT_INSTALLED: + k8sconfig.load_kube_config() + + if not job_to_label_mapping: + job_to_label_mapping = {'worker': ['job-name=tensorflow']} + + if not override_client and not _KUBERNETES_API_CLIENT_INSTALLED: + raise ImportError('The Kubernetes Python client must be installed before' + 'using the Kubernetes Cluster Resolver. To install the' + 'Kubernetes Python client, run `pip install ' + 'kubernetes` on your command line.') + + self._job_to_label_mapping = job_to_label_mapping + self._tf_server_port = tf_server_port + self._override_client = override_client + + self.task_type = None + self.task_index = None + self.rpc_layer = rpc_layer + + def master(self, task_type=None, task_index=None, rpc_layer=None): + """Returns the master address to use when creating a session. + + You must have set the task_type and task_index object properties before + calling this function, or pass in the `task_type` and `task_index` + parameters when using this function. If you do both, the function parameters + will override the object properties. + + Args: + task_type: (Optional) The type of the TensorFlow task of the master. + task_index: (Optional) The index of the TensorFlow task of the master. + rpc_layer: (Optional) The RPC protocol for the given cluster. + + Returns: + The name or URL of the session master. + """ + if task_type is not None and task_index is not None: + return format_master_url( + self.cluster_spec().task_address(task_type, task_index), + rpc_layer or self.rpc_layer) + + if self.task_type is not None and self.task_index is not None: + return format_master_url( + self.cluster_spec().task_address(self.task_type, self.task_index), + rpc_layer or self.rpc_layer) + + return '' + + def cluster_spec(self): + """Returns a ClusterSpec object based on the latest info from Kubernetes. + + We retrieve the information from the Kubernetes master every time this + method is called. + + Returns: + A ClusterSpec containing host information returned from Kubernetes. + + Raises: + RuntimeError: If any of the pods returned by the master is not in the + `Running` phase. + """ + if not self._override_client: + k8sconfig.load_kube_config() + + client = self._override_client or k8sclient.CoreV1Api() + cluster_map = {} + + for tf_job in self._job_to_label_mapping: + all_pods = [] + for selector in self._job_to_label_mapping[tf_job]: + ret = client.list_pod_for_all_namespaces(label_selector=selector) + selected_pods = [] + + # Sort the list by the name to make sure it doesn't change call to call. + for pod in sorted(ret.items, key=lambda x: x.metadata.name): + if pod.status.phase == 'Running': + selected_pods.append( + '%s:%s' % (pod.status.host_ip, self._tf_server_port)) + else: + raise RuntimeError('Pod "%s" is not running; phase: "%s"' % + (pod.metadata.name, pod.status.phase)) + all_pods.extend(selected_pods) + cluster_map[tf_job] = all_pods + + return server_lib.ClusterSpec(cluster_map) + + @property + def environment(self): + """Returns the current environment which TensorFlow is running in. + + For users in the Cloud environment, the environment property is always an + empty string, and Google users will not use this ClusterResolver for running + on internal systems. + """ + return '' + + def num_accelerators_per_worker(self, session_config=None): + local_devices = device_lib.list_local_devices(session_config) + return len([d for d in local_devices if d.device_type == 'GPU']) diff --git a/tensorflow/contrib/cluster_resolver/python/training/kubernetes_cluster_resolver_test.py b/tensorflow/python/distribute/cluster_resolver/kubernetes_cluster_resolver_test.py similarity index 87% rename from tensorflow/contrib/cluster_resolver/python/training/kubernetes_cluster_resolver_test.py rename to tensorflow/python/distribute/cluster_resolver/kubernetes_cluster_resolver_test.py index fbb26e803d7..a9750fa60b9 100644 --- a/tensorflow/contrib/cluster_resolver/python/training/kubernetes_cluster_resolver_test.py +++ b/tensorflow/python/distribute/cluster_resolver/kubernetes_cluster_resolver_test.py @@ -18,7 +18,7 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function -from tensorflow.contrib.cluster_resolver.python.training import KubernetesClusterResolver +from tensorflow.python.distribute.cluster_resolver import KubernetesClusterResolver from tensorflow.python.platform import test from tensorflow.python.training import server_lib @@ -109,6 +109,23 @@ class KubernetesClusterResolverTest(test.TestCase): """ self._verifyClusterSpecEquality(actual_cluster_spec, str(expected_proto)) + def testGetMasterWithOverrideParameters(self): + ret = _create_pod_list( + ('worker-0', 'Running', '10.1.2.3'), + ('worker-1', 'Running', '10.1.2.4'), + ('worker-2', 'Running', '10.1.2.5')) + + cluster_resolver = KubernetesClusterResolver( + override_client=_mock_kubernetes_client( + {'job-name=tensorflow': ret})) + cluster_resolver.task_type = 'worker' + cluster_resolver.task_index = 0 + self.assertEqual(cluster_resolver.task_type, 'worker') + self.assertEqual(cluster_resolver.task_index, 0) + self.assertEqual(cluster_resolver.master(), 'grpc://10.1.2.3:8470') + self.assertEqual(cluster_resolver.master('worker', 2), + 'grpc://10.1.2.5:8470') + def testNonRunningPod(self): ret = _create_pod_list(('tensorflow-abc123', 'Failed', '10.1.2.3'),) diff --git a/tensorflow/python/distribute/cluster_resolver/slurm_cluster_resolver.py b/tensorflow/python/distribute/cluster_resolver/slurm_cluster_resolver.py new file mode 100644 index 00000000000..fd3c6d6a18f --- /dev/null +++ b/tensorflow/python/distribute/cluster_resolver/slurm_cluster_resolver.py @@ -0,0 +1,226 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Implementation of Cluster Resolvers for Slurm workload manager.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import collections +import os +import subprocess + +from tensorflow.python.distribute.cluster_resolver.cluster_resolver import ClusterResolver +from tensorflow.python.training.server_lib import ClusterSpec + + +class SlurmClusterResolver(ClusterResolver): + """Cluster Resolver for system with Slurm workload manager. + + This is an implementation of cluster resolvers for Slurm clusters. This allows + the specification of jobs and task counts, number of tasks per node, number of + GPUs on each node and number of GPUs for each task, It retrieves system + attributes by Slurm environment variables, resolves allocated computing node + names, construct a cluster and return a Cluster Resolver object which an be + use for distributed TensorFlow. + """ + + def _resolve_hostnames(self): + """Resolve host names of nodes allocated in current jobs. + + Returns: + A list of node names as strings. + """ + hostlist = (subprocess.check_output(['scontrol', 'show', 'hostname']). + decode('utf-8').strip().split('\n')) + return hostlist + + def __init__(self, + jobs, + port_base=8888, + gpus_per_node=1, + gpus_per_task=1, + tasks_per_node=None, + auto_set_gpu=True, + rpc_layer='grpc'): + """Creates a new SlurmClusterResolver object. + + This takes in parameters and creates a SlurmClusterResolver object. It uses + those parameters to check which nodes will processes reside and resolves + their hostnames. With the number of the GPUs on each node and number of GPUs + for each task it offsets the port number for each processes and allocate + GPUs to tasks by setting environment variables. The resolver currently + supports homogeneous tasks and default Slurm process allocation. + + Args: + jobs: Dictionary with job names as key and number of tasks in the job as + value + port_base: The first port number to start with for processes on a node. + gpus_per_node: Number of GPUs available on each node. + gpus_per_task: Number of GPUs to be used for each task. + tasks_per_node: Number of tasks to run on each node, if not set defaults + to Slurm's output environment variable SLURM_NTASKS_PER_NODE. + auto_set_gpu: Set the visible CUDA devices automatically while resolving + the cluster by setting CUDA_VISIBLE_DEVICES environment variable. + Defaults to True. + rpc_layer: (Optional) The protocol TensorFlow uses to communicate between + nodes. Defaults to 'grpc'. + + Returns: + A ClusterResolver object which can be used with distributed TensorFlow. + + Raises: + RuntimeError: If requested more GPUs per node then available or requested + more tasks then assigned tasks. + """ + + # check if launched by mpirun + if 'OMPI_COMM_WORLD_RANK' in os.environ: + self._rank = int(os.environ['OMPI_COMM_WORLD_RANK']) + num_tasks = int(os.environ['OMPI_COMM_WORLD_SIZE']) + else: + self._rank = int(os.environ['SLURM_PROCID']) + num_tasks = int(os.environ['SLURM_NTASKS']) + + self._jobs = collections.OrderedDict(sorted(jobs.items())) + self._port_base = port_base + + # user specification overrides SLURM specification + if tasks_per_node is not None: + self._tasks_per_node = tasks_per_node + elif tasks_per_node is None and 'SLURM_NTASKS_PER_NODE' in os.environ: + self._tasks_per_node = int(os.environ['SLURM_NTASKS_PER_NODE']) + else: + raise RuntimeError('Neither `tasks_per_node` or ' + 'SLURM_NTASKS_PER_NODE is set.') + + self._gpus_per_node = gpus_per_node + self._gpus_per_task = gpus_per_task + + self._auto_set_gpu = auto_set_gpu + self.task_type = None + self.task_index = None + self.rpc_layer = rpc_layer + + self._gpu_allocation = [] + self._cluster_allocation = {} + + if self._tasks_per_node * self._gpus_per_task > self._gpus_per_node: + raise RuntimeError('Requested more GPUs per node then available.') + + if sum(self._jobs.values()) != num_tasks: + raise RuntimeError('Requested more tasks then assigned tasks.') + + def cluster_spec(self): + """Returns a ClusterSpec object based on the latest instance group info. + + This returns a ClusterSpec object for use based on information from the + specified initialization parameters and Slurm environment variables. The + cluster specification is resolved each time this function is called. The + resolver extract hostnames of nodes by scontrol and pack tasks in that + order until a node a has number of tasks that is equal to specification. + GPUs on nodes are allocated to tasks by specification through setting + CUDA_VISIBLE_DEVICES environment variable. + + Returns: + A ClusterSpec containing host information retrieved from Slurm's + environment variables. + """ + hostlist = self._resolve_hostnames() + + task_list = [] + self._gpu_allocation = [] + self._cluster_allocation = {} + + for host in hostlist: + for port_offset, gpu_offset in zip( + range(self._tasks_per_node), + range(0, self._gpus_per_node, self._gpus_per_task)): + + host_addr = '%s:%d' % (host, self._port_base + port_offset) + task_list.append(host_addr) + gpu_id_list = [] + + for gpu_id in range(gpu_offset, gpu_offset + self._gpus_per_task): + gpu_id_list.append(str(gpu_id)) + + self._gpu_allocation.append(','.join(gpu_id_list)) + + cluster_rank_offset_start = 0 + cluster_rank_offset_end = 0 + + for task_type, num_tasks in self._jobs.items(): + cluster_rank_offset_end = cluster_rank_offset_start + num_tasks + + self._cluster_allocation[task_type] = ( + task_list[cluster_rank_offset_start:cluster_rank_offset_end]) + + if cluster_rank_offset_start <= self._rank < cluster_rank_offset_end: + self.task_type = task_type + self.task_index = self._rank - cluster_rank_offset_start + + cluster_rank_offset_start = cluster_rank_offset_end + + if self._auto_set_gpu is True: + os.environ['CUDA_VISIBLE_DEVICES'] = self._gpu_allocation[self._rank] + + return ClusterSpec(self._cluster_allocation) + + def get_task_info(self): + """Returns job name and task_index for the process which calls this. + + This returns the job name and task index for the process which calls this + function according to its rank and cluster specification. The job name and + task index are set after a cluster is constructed by cluster_spec otherwise + defaults to None. + + Returns: + A string specifying job name the process belongs to and an integner + specifying the task index the process belongs to in that job. + """ + return self.task_type, self.task_index + + def master(self, task_type=None, task_index=None, rpc_layer=None): + """Returns the master string for connecting to a TensorFlow master. + + Args: + task_type: (Optional) Overrides the default auto-selected task type. + task_index: (Optional) Overrides the default auto-slected task index. + rpc_layer: (Optional) Overrides the default RPC protocol TensorFlow uses + to communicate across nodes. + + Returns: + A connection string for connecting to a TensorFlow master. + """ + task_type = task_type if task_type is not None else self.task_type + task_index = task_index if task_index is not None else self.task_index + rpc_layer = rpc_layer or self.rpc_layer + master = self.cluster_spec().task_address(task_type, task_index) + + return '%s://%s' % (rpc_layer, master) if rpc_layer else master + + @property + def environment(self): + """Returns the current environment which TensorFlow is running in. + + For users in the Slurm environment, the environment property is always an + empty string, and Google users will not use this ClusterResolver for running + on internal systems. + """ + return '' + + def num_accelerators_per_worker(self, session_config=None): + del session_config # Unused, since this is set in __init__ manually. + return self._gpus_per_node diff --git a/tensorflow/contrib/cluster_resolver/python/training/slurm_cluster_resolver_test.py b/tensorflow/python/distribute/cluster_resolver/slurm_cluster_resolver_test.py similarity index 85% rename from tensorflow/contrib/cluster_resolver/python/training/slurm_cluster_resolver_test.py rename to tensorflow/python/distribute/cluster_resolver/slurm_cluster_resolver_test.py index 9aa7df745eb..076539d16f1 100644 --- a/tensorflow/contrib/cluster_resolver/python/training/slurm_cluster_resolver_test.py +++ b/tensorflow/python/distribute/cluster_resolver/slurm_cluster_resolver_test.py @@ -20,7 +20,7 @@ from __future__ import print_function import os -from tensorflow.contrib.cluster_resolver.python.training.slurm_cluster_resolver import SlurmClusterResolver +from tensorflow.python.distribute.cluster_resolver import SlurmClusterResolver from tensorflow.python.platform import test from tensorflow.python.training import server_lib @@ -67,6 +67,31 @@ class SlurmClusterResolverTest(test.TestCase): """ self._verifyClusterSpecEquality(actual_cluster_spec, expected_proto) + @mock.patch.dict(os.environ, {'SLURM_PROCID': '0', 'SLURM_NTASKS': '3'}) + @mock.patch.object(SlurmClusterResolver, '_resolve_hostnames', + mock_resolve_hostnames_output) + def testSimpleMasterRetrieval(self): + slurm_cluster_resolver = SlurmClusterResolver( + jobs={ + 'ps': 1, + 'worker': 2 + }, + port_base=8888, + tasks_per_node=1, + gpus_per_node=1, + gpus_per_task=1, + auto_set_gpu=False) + + slurm_cluster_resolver.task_type = 'worker' + slurm_cluster_resolver.task_index = 1 + self.assertEqual(slurm_cluster_resolver.master(), 'grpc://t02n43:8888') + + slurm_cluster_resolver.rpc_layer = 'ab' + self.assertEqual(slurm_cluster_resolver.master('ps', 0), 'ab://t02n13:8888') + self.assertEqual( + slurm_cluster_resolver.master('ps', 0, rpc_layer='test'), + 'test://t02n13:8888') + @mock.patch.dict(os.environ, { 'SLURM_PROCID': '0', 'SLURM_NTASKS': '3', diff --git a/tensorflow/python/distribute/cluster_resolver/tfconfig_cluster_resolver.py b/tensorflow/python/distribute/cluster_resolver/tfconfig_cluster_resolver.py new file mode 100644 index 00000000000..a3246e77f4d --- /dev/null +++ b/tensorflow/python/distribute/cluster_resolver/tfconfig_cluster_resolver.py @@ -0,0 +1,171 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Implementation of Cluster Resolvers for TF_CONFIG Environment Variables.""" + + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import json +import os + +from tensorflow.python.distribute.cluster_resolver.cluster_resolver import ClusterResolver +from tensorflow.python.training.server_lib import ClusterSpec + +_TF_CONFIG_ENV = 'TF_CONFIG' +_SESSION_MASTER_KEY = 'session_master' +_RPC_LAYER_KEY = 'rpc_layer' +_TASK_KEY = 'task' + + +def format_master_url(master, rpc_layer=None): + if rpc_layer: + return '%s://%s' % (rpc_layer, master) + else: + return master + + +def _load_tf_config(): + return json.loads(os.environ.get(_TF_CONFIG_ENV, '{}')) + + +def _get_value_in_tfconfig(key, default=None): + tf_config = _load_tf_config() + return tf_config[key] if key in tf_config else default + + +class TFConfigClusterResolver(ClusterResolver): + """Implementation of a ClusterResolver which reads the TF_CONFIG EnvVar.""" + + def __init__(self, + task_type=None, + task_index=None, + rpc_layer=None, + environment=None, + num_accelerators_per_worker=0): + """Creates a new TFConfigClusterResolver. + + Args: + task_type: (String, optional) Overrides the task type specified in the + TF_CONFIG environment variable. + task_index: (Integer, optional) Overrides the task index specified in the + TF_CONFIG environment variable. + rpc_layer: (String, optional) Overrides the rpc layer TensorFlow uses. + environment: (String, optional) Overrides the environment TensorFlow + operates in. + num_accelerators_per_worker: (Integer, optional) Specifies the number of + accelerators (e.g. GPUs, TPUs, others) that each node has. + """ + + self._task_type = task_type + self._task_index = task_index + self._rpc_layer = rpc_layer + self._environment = environment + self._num_accelerators_per_worker = num_accelerators_per_worker + + @property + def task_type(self): + if self._task_type is None: + task_info = _get_value_in_tfconfig(_TASK_KEY, {}) + return task_info['type'] if 'type' in task_info else None + else: + return self._task_type + + @property + def task_index(self): + if self._task_type is None: + task_info = _get_value_in_tfconfig(_TASK_KEY, {}) + return task_info['index'] if 'index' in task_info else None + else: + return self._task_index + + @task_type.setter + def task_type(self, task_type): + self._task_type = task_type + + @task_index.setter + def task_index(self, task_index): + self._task_index = task_index + + @property + def environment(self): + return self._environment + + @property + def rpc_layer(self): + if self._rpc_layer is None: + return _get_value_in_tfconfig(_RPC_LAYER_KEY) + else: + return self._rpc_layer + + @rpc_layer.setter + def rpc_layer(self, rpc_layer): + self._rpc_layer = rpc_layer + + def num_accelerators_per_worker(self, session_config=None): + # TODO(frankchn): Connect to server (w/ session_config) in the future. + del session_config # Unused, we do not connect to another server here. + return self._num_accelerators_per_worker + + def cluster_spec(self): + """Returns a ClusterSpec based on the TF_CONFIG environment variable. + + Returns: + A ClusterSpec with information from the TF_CONFIG environment variable. + """ + tf_config = _load_tf_config() + if 'cluster' not in tf_config: + return ClusterSpec({}) + return ClusterSpec(tf_config['cluster']) + + def master(self, task_type=None, task_index=None, rpc_layer=None): + """Returns the master address to use when creating a TensorFlow session. + + Args: + task_type: (String, optional) Overrides and sets the task_type of the + master. + task_index: (Integer, optional) Overrides and sets the task id of the + master. + rpc_layer: (String, optional) Overrides and sets the protocol over which + TensorFlow nodes communicate with each other. + + Returns: + The address of the master. + + Raises: + RuntimeError: If the task_type or task_id is not specified and the + `TF_CONFIG` environment variable does not contain a task section. + """ + + # If `session_master` is set, just use that. + session_master = _get_value_in_tfconfig(_SESSION_MASTER_KEY) + if session_master is not None: + return session_master + + # Return an empty string if we are the only job in the ClusterSpec. + cluster_spec = self.cluster_spec() + if (not cluster_spec.jobs or + (len(cluster_spec.jobs) == 1 and + len(cluster_spec.job_tasks(cluster_spec.jobs[0])) == 1)): + return '' + + # We try to auto-detect the task type and id, but uses the user-supplied one + # where available + task_type = task_type if task_type is not None else self.task_type + task_index = task_index if task_index is not None else self.task_index + + return format_master_url(cluster_spec.task_address(task_type, task_index), + self.rpc_layer) diff --git a/tensorflow/contrib/cluster_resolver/python/training/tfconfig_cluster_resolver_test.py b/tensorflow/python/distribute/cluster_resolver/tfconfig_cluster_resolver_test.py similarity index 72% rename from tensorflow/contrib/cluster_resolver/python/training/tfconfig_cluster_resolver_test.py rename to tensorflow/python/distribute/cluster_resolver/tfconfig_cluster_resolver_test.py index 468161d2aa4..c20e51bc0bb 100644 --- a/tensorflow/contrib/cluster_resolver/python/training/tfconfig_cluster_resolver_test.py +++ b/tensorflow/python/distribute/cluster_resolver/tfconfig_cluster_resolver_test.py @@ -20,7 +20,7 @@ from __future__ import print_function import os -from tensorflow.contrib.cluster_resolver.python.training.tfconfig_cluster_resolver import TFConfigClusterResolver +from tensorflow.python.distribute.cluster_resolver import TFConfigClusterResolver from tensorflow.python.platform import test from tensorflow.python.training import server_lib @@ -133,6 +133,58 @@ class TFConfigClusterResolverTest(test.TestCase): cluster_resolver = TFConfigClusterResolver() self.assertEqual('grpc://ps0:2222', cluster_resolver.master()) + def testTaskTypeIndexRpcRead(self): + os.environ['TF_CONFIG'] = """ + { + "cluster": { + "ps": ["ps0:2222", "ps1:2222"], + "worker": ["worker0:2222", "worker1:2222", "worker2:2222"] + }, + "rpc_layer": "grpc", + "task": { + "type": "ps", + "index": 0 + } + } + """ + + cluster_resolver = TFConfigClusterResolver() + self.assertEqual('ps', cluster_resolver.task_type) + self.assertEqual(0, cluster_resolver.task_index) + self.assertEqual('grpc', cluster_resolver.rpc_layer) + + def testParameterOverrides(self): + os.environ['TF_CONFIG'] = """ + { + "cluster": { + "ps": ["ps0:2222", "ps1:2222"], + "worker": ["worker0:2222", "worker1:2222", "worker2:2222"] + }, + "rpc_layer": "grpc", + "task": { + "type": "ps", + "index": 1 + } + } + """ + + cluster_resolver = TFConfigClusterResolver(task_type='ps', task_index=0, + num_accelerators_per_worker=8) + + self.assertEqual('grpc://ps0:2222', cluster_resolver.master()) + self.assertEqual('ps', cluster_resolver.task_type) + self.assertEqual(0, cluster_resolver.task_index) + self.assertEqual(8, cluster_resolver.num_accelerators_per_worker()) + + cluster_resolver.task_type = 'worker' + cluster_resolver.task_index = 1 + cluster_resolver.rpc_layer = 'test' + + self.assertEqual('test://worker1:2222', cluster_resolver.master()) + self.assertEqual('worker', cluster_resolver.task_type) + self.assertEqual(1, cluster_resolver.task_index) + self.assertEqual('test', cluster_resolver.rpc_layer) + def testZeroItemsInClusterSpecMasterRead(self): os.environ['TF_CONFIG'] = """ {} diff --git a/tensorflow/python/distribute/cluster_resolver/tpu_cluster_resolver.py b/tensorflow/python/distribute/cluster_resolver/tpu_cluster_resolver.py new file mode 100644 index 00000000000..1956bd75a87 --- /dev/null +++ b/tensorflow/python/distribute/cluster_resolver/tpu_cluster_resolver.py @@ -0,0 +1,423 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Implementation of Cluster Resolvers for Cloud TPUs.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import os + +from six.moves.urllib.request import Request +from six.moves.urllib.request import urlopen + +from tensorflow.python.distribute.cluster_resolver.cluster_resolver import ClusterResolver +from tensorflow.python.distribute.cluster_resolver.cluster_resolver import format_master_url +from tensorflow.python.training import server_lib +from tensorflow.python.util import compat + +_GOOGLE_API_CLIENT_INSTALLED = True +try: + from googleapiclient import discovery # pylint: disable=g-import-not-at-top + from oauth2client.client import GoogleCredentials # pylint: disable=g-import-not-at-top +except ImportError: + _GOOGLE_API_CLIENT_INSTALLED = False + + +_GKE_ENV_VARIABLE = 'KUBE_GOOGLE_CLOUD_TPU_ENDPOINTS' +_ENDPOINTS_SEPARATOR = ',' +_DEFAULT_ENV_VARIABLE = 'TPU_NAME' +_DISCOVERY_SERVICE_URL_ENV_VARIABLE = 'TPU_API_DISCOVERY_URL' + + +class TPUClusterResolver(ClusterResolver): + """Cluster Resolver for Google Cloud TPUs. + + This is an implementation of cluster resolvers for the Google Cloud TPU + service. As Cloud TPUs are in alpha, you will need to specify a API definition + file for this to consume, in addition to a list of Cloud TPUs in your Google + Cloud Platform project. + """ + + def _tpuService(self): + """Creates a new Cloud TPU API object. + + This works around an issue where the underlying HTTP connection sometimes + times out when the script has been running for too long. Other methods in + this object calls this method to get a new API object whenever they need + to communicate with the Cloud API. + + Returns: + A Google Cloud TPU API object. + """ + if self._service: + return self._service + + credentials = self._credentials + if credentials is None or credentials == 'default': + credentials = GoogleCredentials.get_application_default() + + if self._discovery_url: + return discovery.build( + 'tpu', 'v1alpha1', + credentials=credentials, + discoveryServiceUrl=self._discovery_url) + else: + return discovery.build( + 'tpu', 'v1alpha1', + credentials=credentials) + + def _requestComputeMetadata(self, path): + req = Request('http://metadata/computeMetadata/v1/%s' % path, + headers={'Metadata-Flavor': 'Google'}) + resp = urlopen(req) + return compat.as_bytes(resp.read()) + + def _shouldResolve(self): + if isinstance(self._should_resolve_override, bool): + return self._should_resolve_override + if (self._tpu == compat.as_bytes('') or + self._tpu == compat.as_bytes('local') or + self._tpu.startswith(compat.as_bytes('/bns')) or + self._tpu.startswith(compat.as_bytes('localhost:')) or + self._tpu.startswith(compat.as_bytes('grpc://'))): + return False + return True + + @staticmethod + def _inGke(): + """When running in GKE, the environment variable will be set.""" + return _GKE_ENV_VARIABLE in os.environ + + @staticmethod + def _gkeEndpoints(): + return os.environ[_GKE_ENV_VARIABLE] + + @staticmethod + def _envVarFallback(): + if _DEFAULT_ENV_VARIABLE in os.environ: + return os.environ[_DEFAULT_ENV_VARIABLE] + return None + + @staticmethod + def _environmentDiscoveryUrl(): + return os.environ.get(_DISCOVERY_SERVICE_URL_ENV_VARIABLE) + + def __init__(self, + tpu=None, + zone=None, + project=None, + job_name='worker', + coordinator_name=None, + coordinator_address=None, + credentials='default', + service=None, + discovery_url=None): + """Creates a new TPUClusterResolver object. + + The ClusterResolver will then use the parameters to query the Cloud TPU APIs + for the IP addresses and ports of each Cloud TPU listed. + + Args: + tpu: Either a string, or a list of strings corresponding to the TPUs to + use. If the single string is the empty string, the string 'local', or a + string that begins with 'grpc://' or '/bns', then it is assumed to not + correspond with a Cloud TPU and will instead be passed as the session + master and no ClusterSpec propagation will be done. + zone: Zone where the TPUs are located. If omitted or empty, we will assume + that the zone of the TPU is the same as the zone of the GCE VM, which we + will try to discover from the GCE metadata service. + project: Name of the GCP project containing Cloud TPUs. If omitted or + empty, we will try to discover the project name of the GCE VM from the + GCE metadata service. + job_name: Name of the TensorFlow job the TPUs belong to. + coordinator_name: The name to use for the coordinator. Set to None if the + coordinator should not be included in the computed ClusterSpec. + coordinator_address: The address of the coordinator (typically an ip:port + pair). If set to None, a TF server will be started. If coordinator_name + is None, a TF server will not be started even if coordinator_address is + None. + credentials: GCE Credentials. If None, then we use default credentials + from the oauth2client + service: The GCE API object returned by the googleapiclient.discovery + function. If you specify a custom service object, then the credentials + parameter will be ignored. + discovery_url: A URL template that points to the location of + the discovery service. It should have two parameters {api} and + {apiVersion} that when filled in produce an absolute URL to the + discovery document for that service. The environment variable + 'TPU_API_DISCOVERY_URL' will override this. + + Raises: + ImportError: If the googleapiclient is not installed. + ValueError: If no TPUs are specified. + """ + if isinstance(tpu, list): + if not tpu: + raise ValueError('At least one TPU must be specified.') + if len(tpu) != 1: + raise NotImplementedError( + 'Using multiple TPUs in a single session is not yet implemented') + tpu = tpu[0] + + in_gke = self._inGke() + # When using GKE with Cloud TPUs, the env variable will be set. + if tpu is None: + if in_gke: + tpu = self._gkeEndpoints() + else: + tpu = self._envVarFallback() + + if tpu is None: + raise ValueError('Please provide a TPU Name to connect to.') + + self._tpu = compat.as_bytes(tpu) # self._tpu is always bytes + + # By default the task_type is 'worker` and the task_index is 0 (which is the + # first worker in the task). + self.task_type = job_name + self.task_index = 0 + + if tpu.startswith('grpc://'): + # Cloud environment, where we are using GRPC to communicate to TPUs. + self._environment = '' + elif tpu == 'local' or not tpu: + # Google environment, where the TPU is attached to the host. + self._environment = 'google' + elif tpu.startswith('/bns'): + # Google environment, where we reach the TPU through BNS. + self._environment = 'google' + + # If TPU is in the Google environment or exists locally, we don't use any + # RPC layer. + if tpu.startswith('/bns') or tpu == 'local' or not tpu: + self.rpc_layer = None + else: + self.rpc_layer = 'grpc' + + # Setting this overrides the return value of self._shouldResolve() + self._should_resolve_override = None + + # We strip out the protocol if it is included, and override the + # shouldResolve function to never resolve. We are adding the protocol back + # in later in self.master(). + if self.rpc_layer is not None and tpu.startswith(self.rpc_layer + '://'): + tpu = tpu[len(self.rpc_layer + '://'):] + self._tpu = tpu + self._should_resolve_override = False + + # Whether we should actually attempt to contact Cloud APIs + should_resolve = self._shouldResolve() + + # We error out if we are in a non-Cloud environment which cannot talk to the + # Cloud APIs using the standard class and a special object is not passed in. + self._service = service + if (self._service is None and should_resolve and + not _GOOGLE_API_CLIENT_INSTALLED): + raise ImportError('googleapiclient and oauth2client must be installed ' + 'before using the TPU cluster resolver. Execute: ' + '`pip install --upgrade google-api-python-client` ' + 'and `pip install --upgrade oauth2client` to ' + 'install with pip.') + + # We save user-passed credentials, unless the user didn't pass in anything. + self._credentials = credentials + if (credentials == 'default' and should_resolve and + _GOOGLE_API_CLIENT_INSTALLED): + self._credentials = None + + # Automatically detect project and zone if unspecified. + if not project and should_resolve: + project = compat.as_str( + self._requestComputeMetadata('project/project-id')) + if not zone and should_resolve: + zone_path = compat.as_str(self._requestComputeMetadata('instance/zone')) + zone = zone_path.split('/')[-1] + self._project = project + self._zone = zone + + self._discovery_url = self._environmentDiscoveryUrl() or discovery_url + + self._coordinator_name = coordinator_name + if (coordinator_name and not coordinator_address and + (should_resolve or in_gke)): + self._start_local_server() + else: + self._coordinator_address = coordinator_address + + def master(self, task_type=None, task_index=None, rpc_layer=None): + """Get the Master string to be used for the session. + + In the normal case, this returns the grpc path (grpc://1.2.3.4:8470) of + first instance in the ClusterSpec returned by the cluster_spec function. + + If a non-TPU name is used when constructing a TPUClusterResolver, that will + be returned instead (e.g. If the tpus argument's value when constructing + this TPUClusterResolver was 'grpc://10.240.1.2:8470', + 'grpc://10.240.1.2:8470' will be returned). + + Args: + task_type: (Optional, string) The type of the TensorFlow task of the + master. + task_index: (Optional, integer) The index of the TensorFlow task of the + master. + rpc_layer: (Optional, string) The RPC protocol TensorFlow should use to + communicate with TPUs. + + Returns: + string, the connection string to use when creating a session. + + Raises: + ValueError: If none of the TPUs specified exists. + """ + if self._shouldResolve(): + # We are going to communicate with the Cloud TPU APIs to get a Cluster. + cluster_spec = self.cluster_spec() + if task_type is not None and task_index is not None: + # task_type and task_index is from the function parameter + master = cluster_spec.task_address(task_type, task_index) + elif self.task_type is not None and self.task_index is not None: + # task_type and task_index is from the object + master = cluster_spec.task_address(self.task_type, self.task_index) + else: + # by default we take the first item in the cluster with the right name + job_tasks = cluster_spec.job_tasks(self.task_type) + if not job_tasks: + raise ValueError('No TPUs with the specified names exist.') + master = job_tasks[0] + else: + if isinstance(self._tpu, (bytes, bytearray)): + master = self._tpu.split(compat.as_bytes(_ENDPOINTS_SEPARATOR))[0] + else: + master = self._tpu.split(_ENDPOINTS_SEPARATOR)[0] + return format_master_url(master, rpc_layer or self.rpc_layer) + + def get_master(self): + return self.master() + + def get_job_name(self): + if self._shouldResolve(): + return self.task_type + + def cluster_spec(self): + """Returns a ClusterSpec object based on the latest TPU information. + + We retrieve the information from the GCE APIs every time this method is + called. + + Returns: + A ClusterSpec containing host information returned from Cloud TPUs. + + Raises: + RuntimeError: If the provided TPU is not healthy. + """ + ############################################################################ + # There are 5 potential cases this code must handle: + # 1. [Normal case.] We should resolve the TPU name to a set of tasks, and + # a. Create a ClusterSpec that includes the coordinator job + # b. Create a ClusterSpec without the coordinator job. + # 2. [GKE / No API Access.] We should not resolve the TPU name to a set of + # tasks and + # a. Create a ClusterSpec with the coordinator + # b. Create a ClusterSpec without the coordinator + # 3. [Other (legacy non-gRPC).] We should return an empty ClusterSpec. + ############################################################################ + + if self._shouldResolve(): + # Case 1. + full_name = 'projects/%s/locations/%s/nodes/%s' % ( + self._project, self._zone, compat.as_text(self._tpu)) + service = self._tpuService() + request = service.projects().locations().nodes().get(name=full_name) + response = request.execute() + + if 'state' in response and response['state'] != 'READY': + raise RuntimeError('TPU "%s" is not yet ready; state: "%s"' % + (compat.as_text(self._tpu), response['state'])) + + if 'health' in response and response['health'] != 'HEALTHY': + raise RuntimeError('TPU "%s" is unhealthy: "%s"' % + (compat.as_text(self._tpu), response['health'])) + + if 'networkEndpoints' in response: + worker_list = [ + '%s:%s' % (endpoint['ipAddress'], endpoint['port']) + for endpoint in response['networkEndpoints'] + ] + else: + # Fall back to the deprecated response format + instance_url = '%s:%s' % (response['ipAddress'], response['port']) + worker_list = [instance_url] + + cluster_spec = {self.task_type: worker_list} + else: + if self.rpc_layer is None: + # Case 3. + return None + # Case 2. + tpus = [] + for tpu in self._tpu.split(_ENDPOINTS_SEPARATOR): + # We are working around the fact that GKE environment variable that is + # supplied to us has the protocol string embedded in it, but we want + # to strip it out for the ClusterSpec. + if (self.rpc_layer is not None and + tpu.startswith(self.rpc_layer + '://')): + tpus.append(tpu[len(self.rpc_layer + '://'):]) + else: + tpus.append(tpu) + cluster_spec = {self.task_type: tpus} + + if self._coordinator_address: + # {1, 2}.a + cluster_spec[self._coordinator_name] = [self._coordinator_address] + + return server_lib.ClusterSpec(cluster_spec) + + def num_accelerators_per_worker(self, session_config=None): + """Returns the number of TPU cores per worker. + + This defaults to 8 for all current TPU configurations, and we do not need + to query any remote systems for this. + + Args: + session_config: Unused. Not currently necessary to query anything as this + number is 8 for all TPU configurations. + """ + del session_config # Unused. Not necessary to query anything. + return 8 + + @property + def environment(self): + """Returns the current environment which TensorFlow is running in.""" + return self._environment + + def _start_local_server(self): + address = self._requestComputeMetadata('instance/network-interfaces/0/ip') + self._server = server_lib.Server( + { + 'local': ['0.0.0.0:0'] + }, protocol='grpc', config=None, start=True) + # self._server.target is of the form: grpc://ipaddress:port + target = compat.as_bytes(self._server.target) + splits = target.split(compat.as_bytes(':')) + assert len(splits) == 3, self._server.target + assert splits[0] == compat.as_bytes('grpc'), self._server.target + self._coordinator_port = compat.as_text(splits[2]) + self._coordinator_address = '%s:%s' % ( + address, compat.as_text(self._coordinator_port)) + + def __deepcopy__(self, memo): + # TODO(b/73668574): Remove this once RunConfig avoids performing deepcopy. + return self diff --git a/tensorflow/contrib/cluster_resolver/python/training/tpu_cluster_resolver_test.py b/tensorflow/python/distribute/cluster_resolver/tpu_cluster_resolver_test.py similarity index 86% rename from tensorflow/contrib/cluster_resolver/python/training/tpu_cluster_resolver_test.py rename to tensorflow/python/distribute/cluster_resolver/tpu_cluster_resolver_test.py index 478c82967ba..0f22ede3d9b 100644 --- a/tensorflow/contrib/cluster_resolver/python/training/tpu_cluster_resolver_test.py +++ b/tensorflow/python/distribute/cluster_resolver/tpu_cluster_resolver_test.py @@ -20,7 +20,7 @@ from __future__ import print_function import os -from tensorflow.contrib.cluster_resolver.python.training.tpu_cluster_resolver import TPUClusterResolver +from tensorflow.python.distribute.cluster_resolver import TPUClusterResolver from tensorflow.python.platform import test from tensorflow.python.training import server_lib from tensorflow.python.util import compat @@ -132,6 +132,7 @@ class TPUClusterResolverTest(test.TestCase): } """ % tpu_cluster_resolver._coordinator_port self._verifyClusterSpecEquality(actual_cluster_spec, str(expected_proto)) + self.assertEqual(tpu_cluster_resolver.master(), 'grpc://10.1.2.3:8470') @mock.patch.object(TPUClusterResolver, '_requestComputeMetadata', mock_request_compute_metadata) @@ -157,6 +158,7 @@ class TPUClusterResolverTest(test.TestCase): job { name: 'worker' tasks { key: 0 value: '10.1.2.3:8470' } } """ self._verifyClusterSpecEquality(actual_cluster_spec, expected_proto) + self.assertEqual(tpu_cluster_resolver.master(), 'grpc://10.1.2.3:8470') @mock.patch.object(TPUClusterResolver, '_requestComputeMetadata', mock_request_compute_metadata) @@ -226,6 +228,7 @@ class TPUClusterResolverTest(test.TestCase): job { name: 'worker' tasks { key: 0 value: '10.1.2.3:8470' } } """ self._verifyClusterSpecEquality(actual_cluster_spec, expected_proto) + self.assertEqual(tpu_cluster_resolver.master(), 'grpc://10.1.2.3:8470') def testNewNetworkEndpointFormat(self): tpu_map = { @@ -304,6 +307,7 @@ class TPUClusterResolverTest(test.TestCase): } """ % tpu_cluster_resolver._coordinator_port self._verifyClusterSpecEquality(actual_cluster_spec, str(expected_proto)) + self.assertEqual(tpu_cluster_resolver.master(), 'grpc://10.2.3.4:8470') def testPodResolutionNoCoordinator(self): tpu_map = { @@ -350,6 +354,7 @@ class TPUClusterResolverTest(test.TestCase): } """ self._verifyClusterSpecEquality(actual_cluster_spec, expected_proto) + self.assertEqual(tpu_cluster_resolver.master(), 'grpc://10.2.3.4:8470') def testGetMasterNoEntries(self): tpu_map = {} @@ -464,5 +469,62 @@ class TPUClusterResolverTest(test.TestCase): self.assertEqual('https://{api}.internal/{apiVersion}', TPUClusterResolver._environmentDiscoveryUrl()) + def testEnvironmentAndRpcDetectionForGoogle(self): + tpu_cluster_resolver = TPUClusterResolver(tpu='/bns/ab/cd/ef') + self.assertEqual(tpu_cluster_resolver.environment, 'google') + self.assertEqual(tpu_cluster_resolver.rpc_layer, None) + + def testEnvironmentAndRpcDetectionForGrpcString(self): + tpu_cluster_resolver = TPUClusterResolver(tpu='grpc://10.1.2.3:8470') + self.assertEqual(tpu_cluster_resolver.environment, '') + self.assertEqual(tpu_cluster_resolver.rpc_layer, 'grpc') + self.assertEqual(tpu_cluster_resolver.master(), 'grpc://10.1.2.3:8470') + + def testOverrideTaskTypeAndIndexAndGetMaster(self): + tpu_map = { + 'projects/test-project/locations/us-central1-c/nodes/test-tpu-1': { + 'health': + 'HEALTHY', + 'networkEndpoints': [ + { + 'ipAddress': '10.2.3.4', + 'port': 8470, + }, + { + 'ipAddress': '10.2.3.5', + 'port': 8470, + }, + { + 'ipAddress': '10.2.3.6', + 'port': 8470, + }, + { + 'ipAddress': '10.2.3.7', + 'port': 8470, + }, + ] + } + } + + tpu_cluster_resolver = TPUClusterResolver( + project='test-project', + zone='us-central1-c', + tpu='test-tpu-1', + coordinator_name=None, + credentials=None, + service=self.mock_service_client(tpu_map=tpu_map)) + + self.assertEqual(tpu_cluster_resolver.master(), 'grpc://10.2.3.4:8470') + + tpu_cluster_resolver.task_type = 'worker' + tpu_cluster_resolver.task_index = 3 + self.assertEqual(tpu_cluster_resolver.master(), 'grpc://10.2.3.7:8470') + + self.assertEqual( + tpu_cluster_resolver.master( + task_type='worker', task_index=2, rpc_layer='test'), + 'test://10.2.3.6:8470') + + if __name__ == '__main__': test.main() diff --git a/tensorflow/contrib/distribute/python/cross_tower_ops.py b/tensorflow/python/distribute/cross_device_ops.py similarity index 86% rename from tensorflow/contrib/distribute/python/cross_tower_ops.py rename to tensorflow/python/distribute/cross_device_ops.py index b5b349aa64e..a88ed625331 100644 --- a/tensorflow/contrib/distribute/python/cross_tower_ops.py +++ b/tensorflow/python/distribute/cross_device_ops.py @@ -21,17 +21,17 @@ from __future__ import print_function import collections import six -from tensorflow.contrib.distribute.python import cross_tower_utils -from tensorflow.contrib.distribute.python import values as value_lib from tensorflow.python.client import device_lib +from tensorflow.python.distribute import cross_device_utils +from tensorflow.python.distribute import device_util +from tensorflow.python.distribute import reduce_util +from tensorflow.python.distribute import values as value_lib from tensorflow.python.eager import context from tensorflow.python.framework import ops from tensorflow.python.ops import array_ops from tensorflow.python.ops import math_ops from tensorflow.python.ops import resource_variable_ops -from tensorflow.python.ops import variable_scope as vs from tensorflow.python.platform import tf_logging as logging -from tensorflow.python.training import device_util def check_destinations(destinations): @@ -103,10 +103,10 @@ def _validate_value_destination_pairs(value_destination_pairs): # pylint: disable=g-missing-docstring if not value_destination_pairs: return False if not isinstance(value_destination_pairs, (list, tuple)): return False - if not all([isinstance(pair, tuple) for pair in value_destination_pairs]): + if not all(isinstance(pair, tuple) for pair in value_destination_pairs): return False - if not all([isinstance(v[0], value_lib.PerReplica) - for v in value_destination_pairs]): + if not all(isinstance(v[0], value_lib.PerReplica) + for v in value_destination_pairs): return False return True @@ -132,10 +132,10 @@ def _devices_match(left, right): def _all_devices_match(value_destination_pairs): - if not all([_devices_match(v, d) for v, d in value_destination_pairs]): + if not all(_devices_match(v, d) for v, d in value_destination_pairs): return False - if not all([_devices_match(v, value_destination_pairs[0][0]) - for v, _ in value_destination_pairs[1:]]): + if not all(_devices_match(v, value_destination_pairs[0][0]) + for v, _ in value_destination_pairs[1:]): return False return True @@ -144,13 +144,13 @@ def _simple_broadcast(value, destinations): index = {} devices = get_devices_from(destinations) for d in devices: - index[d] = cross_tower_utils.copy_tensor_or_indexed_slices_to_device( + index[d] = cross_device_utils.copy_tensor_or_indexed_slices_to_device( value, d) return value_lib.Mirrored(index) def _simple_reduce(per_replica_value, reduce_to_device, accumulation_fn, - aggregation): + reduce_op): # pylint: disable=g-missing-docstring all_values = [] count = 0 @@ -162,14 +162,13 @@ def _simple_reduce(per_replica_value, reduce_to_device, accumulation_fn, with ops.device(reduce_to_device): with context.context().device_policy(context.DEVICE_PLACEMENT_SILENT): - reduced = cross_tower_utils.aggregate_tensors_or_indexed_slices( + reduced = cross_device_utils.aggregate_tensors_or_indexed_slices( all_values, accumulation_fn) - if aggregation == vs.VariableAggregation.MEAN: - reduced = cross_tower_utils.divide_by_n_tensors_or_indexed_slices( + if reduce_op == reduce_util.ReduceOp.MEAN: + reduced = cross_device_utils.divide_by_n_tensors_or_indexed_slices( reduced, count) - elif aggregation != vs.VariableAggregation.SUM: - raise ValueError("`aggregation` must be VariableAggregation.SUM " - "or VariableAggregation.MEAN.") + elif reduce_op != reduce_util.ReduceOp.SUM: + raise ValueError("`reduce_op` must be Reduce.SUM or Reduce.MEAN.") return reduced @@ -179,15 +178,15 @@ class CrossDeviceOps(object): def __init__(self): pass - def reduce(self, aggregation, per_replica_value, destinations): + def reduce(self, reduce_op, per_replica_value, destinations): """Reduce `per_replica_value` to `destinations`. - It runs the reduction operation defined by `aggregation` and put the + It runs the reduction operation defined by `reduce_op` and put the result on `destinations`. Args: - aggregation: Indicates how a variable will be aggregated. Accepted values - are `tf.VariableAggregation.SUM`, `tf.VariableAggregation.MEAN`. + reduce_op: Indicates how per_replica_value will be reduced. Accepted + values are `tf.distribute.ReduceOp.SUM`, `tf.distribute.ReduceOp.MEAN`. per_replica_value: a PerReplica object or a tensor with device set. destinations: the reduction destinations. @@ -201,17 +200,17 @@ class CrossDeviceOps(object): per_replica_value = _make_tensor_into_per_replica(per_replica_value) validate_destinations(destinations) - return self._reduce(aggregation, per_replica_value, destinations) + return self._reduce(reduce_op, per_replica_value, destinations) - def batch_reduce(self, aggregation, value_destination_pairs): + def batch_reduce(self, reduce_op, value_destination_pairs): """Reduce PerReplica objects in a batch. Reduce each first element in `value_destination_pairs` to each second element which indicates the destinations. Args: - aggregation: Indicates how a variable will be aggregated. Accepted values - are `tf.VariableAggregation.SUM`, `tf.VariableAggregation.MEAN`. + reduce_op: Indicates how per_replica_value will be reduced. Accepted + values are `tf.distribute.ReduceOp.SUM`, `tf.distribute.ReduceOp.MEAN`. value_destination_pairs: a list or a tuple of tuples of PerReplica objects (or tensors with device set if there is one device) and destinations. @@ -231,7 +230,7 @@ class CrossDeviceOps(object): for _, d in value_destination_pairs: validate_destinations(d) - return self._batch_reduce(aggregation, value_destination_pairs) + return self._batch_reduce(reduce_op, value_destination_pairs) def broadcast(self, tensor, destinations): """Broadcast the `tensor` to destinations. @@ -246,11 +245,11 @@ class CrossDeviceOps(object): validate_destinations(destinations) return self._broadcast(tensor, destinations) - def _reduce(self, aggregation, per_replica_value, destinations): + def _reduce(self, reduce_op, per_replica_value, destinations): raise NotImplementedError( "_reduce method must be implemented in descendants.") - def _batch_reduce(self, aggregation, value_destination_pairs): + def _batch_reduce(self, reduce_op, value_destination_pairs): raise NotImplementedError( "_batch_reduce method must be implemented in descendants.") @@ -276,19 +275,19 @@ class ReductionToOneDeviceCrossDeviceOps(CrossDeviceOps): self.accumulation_fn = accumulation_fn super(ReductionToOneDeviceCrossDeviceOps, self).__init__() - def _reduce(self, aggregation, per_replica_value, destinations): + def _reduce(self, reduce_op, per_replica_value, destinations): if check_destinations(destinations): devices = get_devices_from(destinations) else: devices = get_devices_from(per_replica_value) reduce_to_device = self.reduce_to_device or devices[0] reduced = _simple_reduce(per_replica_value, reduce_to_device, - self.accumulation_fn, aggregation) + self.accumulation_fn, reduce_op) return self.broadcast(reduced, devices) - def _batch_reduce(self, aggregation, value_destination_pairs): + def _batch_reduce(self, reduce_op, value_destination_pairs): return [ - self._reduce(aggregation, t, destinations=v) + self._reduce(reduce_op, t, destinations=v) for t, v in value_destination_pairs ] @@ -323,20 +322,20 @@ def _group_value_by_device(per_replica_values): def _ungroup_and_make_mirrored(grouped_reduced, destinations, - aggregation, + reduce_op, num_between_graph_workers=1): """Ungroup results from all-reduce and make Mirrored objects. Each all-reduce result will be divided by the number of destinations before - Mirrored objects are created if aggregation is "mean". + Mirrored objects are created if reduce_op is "mean". Args: grouped_reduced: a list of lists, each sublist has components for each device, paired with a None. It is the result from - cross_tower_utils.aggregate_gradients_using*. + cross_device_utils.aggregate_gradients_using*. destinations: a list of device strings for returned Mirrored objects. - aggregation: Indicates how a variable will be aggregated. Accepted values - are `tf.VariableAggregation.SUM`, `tf.VariableAggregation.MEAN`. + reduce_op: Indicates how values will be aggregated. Accepted values + are `tf.distribute.ReduceOp.SUM`, `tf.distribute.ReduceOp.MEAN`. num_between_graph_workers: number of workers in the between-graph replication. @@ -346,7 +345,7 @@ def _ungroup_and_make_mirrored(grouped_reduced, index = [{} for _ in range(len(grouped_reduced[0]))] for d, per_replica_reduced in enumerate(grouped_reduced): for i, (v, _) in enumerate(per_replica_reduced): - if aggregation == vs.VariableAggregation.MEAN: + if reduce_op == reduce_util.ReduceOp.MEAN: index[i][destinations[d]] = v / ( len(destinations) * num_between_graph_workers) else: @@ -402,7 +401,7 @@ class ConcatAndSplitPacker(object): # all gradient shapes are defined, we use another method to get the # total size. # TODO(yuefengz): move this logic to array_ops.size. - if all([g.shape.is_fully_defined() for g, _ in device_grads_and_vars]): + if all(g.shape.is_fully_defined() for g, _ in device_grads_and_vars): total_grad_size = sum( [g.shape.num_elements() for g, _ in device_grads_and_vars]) else: @@ -486,7 +485,7 @@ class AggregateSmallTensorPacker(object): """Aggregate small tensors.""" if (self.agg_small_grads_max_bytes > 0 and self.agg_small_grads_max_group > 0): - device_grads, self.packing = cross_tower_utils.pack_small_tensors( + device_grads, self.packing = cross_device_utils.pack_small_tensors( grouped_grads_and_vars, max_bytes=self.agg_small_grads_max_bytes, max_group=self.agg_small_grads_max_group) @@ -494,8 +493,8 @@ class AggregateSmallTensorPacker(object): def unpack(self, summed_device_grad_packs): """Reverse the aggregation process.""" - return cross_tower_utils.unpack_small_tensors(summed_device_grad_packs, - self.packing) + return cross_device_utils.unpack_small_tensors(summed_device_grad_packs, + self.packing) def _pack_tensors(device_grads, @@ -557,13 +556,13 @@ class AllReduceCrossDeviceOps(CrossDeviceOps): self._agg_small_grads_max_group = agg_small_grads_max_group super(AllReduceCrossDeviceOps, self).__init__() - def _reduce(self, aggregation, per_replica_value, destinations): - contains_indexed_slices = cross_tower_utils.contains_indexed_slices( + def _reduce(self, reduce_op, per_replica_value, destinations): + contains_indexed_slices = cross_device_utils.contains_indexed_slices( per_replica_value) if (_devices_match(per_replica_value, destinations) and not context.executing_eagerly() and not contains_indexed_slices): - return self._batch_all_reduce(aggregation, [per_replica_value])[0] + return self._batch_all_reduce(reduce_op, [per_replica_value])[0] else: if contains_indexed_slices: logging.log_first_n( @@ -576,16 +575,16 @@ class AllReduceCrossDeviceOps(CrossDeviceOps): devices = get_devices_from(per_replica_value) reduce_to_device = devices[0] reduced = _simple_reduce(per_replica_value, reduce_to_device, - math_ops.add_n, aggregation) + math_ops.add_n, reduce_op) return self.broadcast(reduced, devices) - def _batch_reduce(self, aggregation, value_destination_pairs): + def _batch_reduce(self, reduce_op, value_destination_pairs): all_devices_match = _all_devices_match(value_destination_pairs) - contains_indexed_slices = cross_tower_utils.contains_indexed_slices( + contains_indexed_slices = cross_device_utils.contains_indexed_slices( value_destination_pairs) if (all_devices_match and not context.executing_eagerly() and not contains_indexed_slices): - return self._batch_all_reduce(aggregation, + return self._batch_all_reduce(reduce_op, [v[0] for v in value_destination_pairs]) else: if not all_devices_match: @@ -595,11 +594,11 @@ class AllReduceCrossDeviceOps(CrossDeviceOps): 10) return [ - self._reduce(aggregation, t, destinations=v) + self._reduce(reduce_op, t, destinations=v) for t, v in value_destination_pairs ] - def _batch_all_reduce(self, aggregation, per_replica_values): + def _batch_all_reduce(self, reduce_op, per_replica_values): """All reduce algorithm in a batch.""" logging.log_first_n( logging.INFO, "batch_all_reduce invoked for batches size = %d with " @@ -619,18 +618,18 @@ class AllReduceCrossDeviceOps(CrossDeviceOps): # the balance on num_splits. if self._all_reduce_alg == "nccl": # TODO(yuefengz): merge this into the all-reduce library. - reduced = cross_tower_utils.aggregate_gradients_using_nccl( + reduced = cross_device_utils.aggregate_gradients_using_nccl( device_grad_packs) else: # TODO(yuefengz): check that gpu ids in `destinations` are in ascending # order. reduced = ( - cross_tower_utils.aggregate_gradients_using_hierarchical_copy( + cross_device_utils.aggregate_gradients_using_hierarchical_copy( destinations, device_grad_packs)) reduced = _unpack_tensors(reduced, tensor_packer) return _ungroup_and_make_mirrored(reduced, per_replica_values[0].devices, - aggregation) + reduce_op) # For compatibility with code using the old name of `AllReduceCrossDeviceOps`. @@ -713,7 +712,7 @@ class MultiWorkerAllReduce(AllReduceCrossDeviceOps): validate_and_complete_spec(spec) for spec in all_reduce_spec ] - def _batch_all_reduce(self, aggregation, per_replica_values): + def _batch_all_reduce(self, reduce_op, per_replica_values): """All reduce algorithm in a batch.""" logging.log_first_n( logging.INFO, @@ -741,13 +740,13 @@ class MultiWorkerAllReduce(AllReduceCrossDeviceOps): this_grads = remaining_grads remaining_grads = [] else: - (this_grads, remaining_grads) = cross_tower_utils.split_grads_by_size( + (this_grads, remaining_grads) = cross_device_utils.split_grads_by_size( spec_tuple.limit, remaining_grads) if this_grads: device_grad_packs, tensor_packer = _pack_tensors( this_grads, self._num_packs, self._agg_small_grads_max_bytes, self._agg_small_grads_max_group) - range_agg_grads = cross_tower_utils.sum_gradients_all_reduce( + range_agg_grads = cross_device_utils.sum_gradients_all_reduce( self._worker_devices, device_grad_packs, len(self._worker_devices), spec_tuple.alg, spec_tuple.shards, range(self._num_gpus_per_worker)) range_agg_grads = _unpack_tensors(range_agg_grads, tensor_packer) @@ -761,7 +760,7 @@ class MultiWorkerAllReduce(AllReduceCrossDeviceOps): assert not remaining_grads return _ungroup_and_make_mirrored(aggregated_grads, destinations, - aggregation) + reduce_op) # TODO(yuefengz): support in-graph collective all-reduce. @@ -790,20 +789,20 @@ class CollectiveAllReduce(CrossDeviceOps): self._num_workers = num_workers self._num_gpus_per_worker = num_gpus_per_worker self._all_reduce_merge_scope = all_reduce_merge_scope - self._collective_keys = collective_keys or cross_tower_utils.CollectiveKeys( - ) + self._collective_keys = (collective_keys or + cross_device_utils.CollectiveKeys()) super(CollectiveAllReduce, self).__init__() # TODO(yuefengz, tucker): is indexed slices supported by collective ops? - def _reduce(self, aggregation, per_replica_value, destinations): - if cross_tower_utils.contains_indexed_slices(per_replica_value): + def _reduce(self, reduce_op, per_replica_value, destinations): + if cross_device_utils.contains_indexed_slices(per_replica_value): raise ValueError( "`IndexSlices` is not supported for Collective All-Reduce.") if context.executing_eagerly(): raise ValueError( "Eager execution is not supported for Collective All-Reduce") - all_reduced = self._batch_all_reduce(aggregation, [per_replica_value])[0] + all_reduced = self._batch_all_reduce(reduce_op, [per_replica_value])[0] if _devices_match(per_replica_value, destinations): return all_reduced else: @@ -819,8 +818,8 @@ class CollectiveAllReduce(CrossDeviceOps): return value_lib.Mirrored(index) - def _batch_reduce(self, aggregation, value_destination_pairs): - if cross_tower_utils.contains_indexed_slices(value_destination_pairs): + def _batch_reduce(self, reduce_op, value_destination_pairs): + if cross_device_utils.contains_indexed_slices(value_destination_pairs): raise ValueError( "`IndexSlices` is not supported for Collective All-Reduce.") if context.executing_eagerly(): @@ -829,7 +828,7 @@ class CollectiveAllReduce(CrossDeviceOps): all_devices_match = _all_devices_match(value_destination_pairs) if all_devices_match: - return self._batch_all_reduce(aggregation, + return self._batch_all_reduce(reduce_op, [v[0] for v in value_destination_pairs]) else: if not all_devices_match: @@ -838,11 +837,11 @@ class CollectiveAllReduce(CrossDeviceOps): "destinations are different.", 10) return [ - self._reduce(aggregation, t, destinations=v) + self._reduce(reduce_op, t, destinations=v) for t, v in value_destination_pairs ] - def _batch_all_reduce(self, aggregation, per_replica_values): + def _batch_all_reduce(self, reduce_op, per_replica_values): """All-reduce across all workers in a batch.""" if context.executing_eagerly(): raise ValueError( @@ -871,7 +870,7 @@ class CollectiveAllReduce(CrossDeviceOps): with ops.name_scope("allreduce"): for grad_and_vars in chunk: scaled_grads = [g for g, _ in grad_and_vars] - collective_reduced = cross_tower_utils.build_collective_reduce( + collective_reduced = cross_device_utils.build_collective_reduce( scaled_grads, self._num_workers, self._collective_keys, "Add", "Id") result = [] @@ -883,7 +882,7 @@ class CollectiveAllReduce(CrossDeviceOps): return _ungroup_and_make_mirrored( new_device_grads, per_replica_values[0].devices, - aggregation, + reduce_op, num_between_graph_workers=self._num_workers) @@ -917,15 +916,15 @@ def _choose_all_reduce_algorithm(device_links): def choose_the_best(devices, session_config=None): - """Find the best subclass of CrossDeviceOps given a tensorflow session. + """Find the best subclass of CrossDeviceOps given a session config. Args: - devices: a list of devices passed for distribute strategy. - session_config: a tensorflow session config or None. If None, it will make - deciesion based on all local devices. + devices: a list of devices passed to `tf.distribute.Strategy`. + session_config: a `tf.ConfigProto` or `None`. If `None`, it will make + decision based on all local devices. Returns: - a subclass of CrossDeviceOps. + A subclass of `CrossDeviceOps`. """ requested_devices = set([device_util.canonicalize(d) for d in devices]) machine_devices = device_lib.list_local_devices(session_config=session_config) @@ -938,13 +937,13 @@ def choose_the_best(devices, session_config=None): "Device is available but not used by distribute strategy: %s", d.name) if len(using_devices) != len(requested_devices): - logging.warning("Not all devices in distribute strategy are visible by " - "TensorFlow sessions.") + logging.warning("Not all devices in `tf.distribute.Strategy` are visible " + "to TensorFlow.") return ReductionToOneDeviceCrossDeviceOps() - if any([d.device_type.lower() != "gpu" for d in using_devices]): - logging.warning("Not all devices in DistributionStrategy are visible to " - "TensorFlow session.") + if any(d.device_type.lower() != "gpu" for d in using_devices): + logging.warning("Not all devices in `tf.distribute.Strategy` are visible " + "to TensorFlow.") return ReductionToOneDeviceCrossDeviceOps() device_links = [[] for _ in range(len(using_devices))] diff --git a/tensorflow/contrib/distribute/python/cross_tower_utils.py b/tensorflow/python/distribute/cross_device_utils.py similarity index 99% rename from tensorflow/contrib/distribute/python/cross_tower_utils.py rename to tensorflow/python/distribute/cross_device_utils.py index 50b3cf31e59..0faadd7e0cf 100644 --- a/tensorflow/contrib/distribute/python/cross_tower_utils.py +++ b/tensorflow/python/distribute/cross_device_utils.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================== -"""Utilities for cross_tower_ops.""" +"""Utilities for cross_device_ops.""" from __future__ import absolute_import from __future__ import division @@ -21,8 +21,8 @@ from __future__ import print_function import collections as pycoll import threading -from tensorflow.contrib.all_reduce.python import all_reduce -from tensorflow.contrib.distribute.python import values as value_lib +from tensorflow.python.distribute import all_reduce +from tensorflow.python.distribute import values as value_lib from tensorflow.python.framework import device as pydev from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops @@ -420,7 +420,7 @@ def sum_gradients_all_reduce(dev_prefixes, replica_grads, num_workers, alg, Returns: list of reduced tensors """ - alg_contains_shuffle = any([n in alg for n in ['pscpu', 'psgpu']]) + alg_contains_shuffle = any(n in alg for n in ['pscpu', 'psgpu']) is_hierarchical = '/' in alg if 'pscpu' in alg: aux_devices = [prefix + '/cpu:0' for prefix in dev_prefixes] diff --git a/tensorflow/python/training/device_util.py b/tensorflow/python/distribute/device_util.py similarity index 98% rename from tensorflow/python/training/device_util.py rename to tensorflow/python/distribute/device_util.py index 70e1ca4b5d7..34474582adf 100644 --- a/tensorflow/python/training/device_util.py +++ b/tensorflow/python/distribute/device_util.py @@ -50,7 +50,7 @@ def canonicalize(d, default=None): # Fill in missing device fields using defaults. result = tf_device.DeviceSpec( replica=0, task=0, device_type="CPU", device_index=0) - if context.executing_eagerly(): + if ops.executing_eagerly_outside_functions(): result.job = "localhost" if default: result.merge_from(tf_device.DeviceSpec.from_string(default)) diff --git a/tensorflow/python/training/device_util_test.py b/tensorflow/python/distribute/device_util_test.py similarity index 95% rename from tensorflow/python/training/device_util_test.py rename to tensorflow/python/distribute/device_util_test.py index cdbb08229d2..2f0d7ed3b31 100644 --- a/tensorflow/python/training/device_util_test.py +++ b/tensorflow/python/distribute/device_util_test.py @@ -18,14 +18,16 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function +from tensorflow.python.distribute import device_util from tensorflow.python.eager import context from tensorflow.python.framework import ops +from tensorflow.python.framework import test_util from tensorflow.python.platform import test -from tensorflow.python.training import device_util class DeviceUtilTest(test.TestCase): + @test_util.run_deprecated_v1 def testCurrentDeviceWithGlobalGraph(self): with ops.device("/cpu:0"): self.assertEqual(device_util.current(), "/device:CPU:0") @@ -49,6 +51,7 @@ class DeviceUtilTest(test.TestCase): self.assertEqual(device_util.current(), "/job:localhost/replica:0/task:0/device:CPU:0") + @test_util.run_deprecated_v1 def testCanonicalizeWithoutDefaultDevice(self): self.assertEqual( device_util.canonicalize("/cpu:0"), diff --git a/tensorflow/python/distribute/distribute_coordinator.py b/tensorflow/python/distribute/distribute_coordinator.py index 520413102be..c0f9b8a1fdf 100644 --- a/tensorflow/python/distribute/distribute_coordinator.py +++ b/tensorflow/python/distribute/distribute_coordinator.py @@ -245,7 +245,7 @@ class _WorkerContext(object): else: session_config = self._session_config - if not self._strategy or self._strategy.should_init: + if not self._strategy or self._strategy.extended.experimental_should_init: logging.info("Creating chief session creator with config: %r", config) return monitored_session.ChiefSessionCreator( scaffold, @@ -261,6 +261,10 @@ class _WorkerContext(object): config=session_config, max_wait_secs=max_wait_secs) + @property + def session_config(self): + return copy.deepcopy(self._session_config) + @property def has_barrier(self): """Whether the barrier is set or not.""" @@ -301,15 +305,20 @@ class _WorkerContext(object): """Returns number of workers in the cluster, including chief.""" return self._num_workers + @property + def experimental_should_init(self): + """Whether to run init ops.""" + return self._strategy.extended.experimental_should_init + @property def should_checkpoint(self): """Whether to save checkpoint.""" - return self._strategy.should_checkpoint + return self._strategy.extended.should_checkpoint @property def should_save_summary(self): """Whether to save summaries.""" - return self._strategy.should_save_summary + return self._strategy.extended.should_save_summary def _run_single_worker(worker_fn, @@ -623,10 +632,10 @@ def run_distribute_coordinator(worker_fn, The `strategy` object is expected to be a DistributionStrategy object which has implemented methods needed by distributed coordinator such as `configure(session_config, cluster_spec, task_type, task_id)` which configures - the strategy object for a specific task and `should_init` property which - instructs the distribute coordinator whether to run init ops for a task. The - distribute coordinator will make a copy of the `strategy` object, call its - `configure` method and pass it to `worker_fn` as an argument. + the strategy object for a specific task and `experimental_should_init` + property which instructs the distribute coordinator whether to run init ops + for a task. The distribute coordinator will make a copy of the `strategy` + object, call its `configure` method and pass it to `worker_fn` as an argument. The `worker_fn` defines the training logic and is called under a its own worker context which can be accessed to via `get_current_worker_context`. A @@ -749,7 +758,7 @@ def run_distribute_coordinator(worker_fn, # The client must know the cluster but servers in the cluster don't have to # know the client. if task_type in [_TaskType.CLIENT, None]: - if strategy.between_graph: + if strategy.extended.experimental_between_graph: return _run_between_graph_client(worker_fn, strategy, eval_fn, eval_strategy, cluster_spec, session_config, rpc_layer) @@ -795,7 +804,7 @@ def run_distribute_coordinator(worker_fn, environment=environment) if task_type in [_TaskType.CHIEF, _TaskType.WORKER]: - if strategy.between_graph: + if strategy.extended.experimental_between_graph: # All jobs run `worker_fn` if between-graph. _run_single_worker(worker_fn, strategy, cluster_spec, task_type, task_id, session_config, rpc_layer) diff --git a/tensorflow/python/distribute/distribute_coordinator_test.py b/tensorflow/python/distribute/distribute_coordinator_test.py index 5d336648ce9..f2cb950aada 100644 --- a/tensorflow/python/distribute/distribute_coordinator_test.py +++ b/tensorflow/python/distribute/distribute_coordinator_test.py @@ -47,6 +47,7 @@ from tensorflow.python.ops import variable_scope from tensorflow.python.ops import variables from tensorflow.python.platform import test from tensorflow.python.training import monitored_session +from tensorflow.python.training import session_manager CHIEF = distribute_coordinator._TaskType.CHIEF @@ -78,6 +79,19 @@ def _strip_protocol(target): return target +class MockExtended(object): + + def __init__(self, + between_graph=False, + should_init=None, + should_checkpoint=None, + should_save_summary=None): + self.experimental_between_graph = between_graph + self.experimental_should_init = should_init + self.should_checkpoint = should_checkpoint + self.should_save_summary = should_save_summary + + class MockStrategy(object): def __init__(self, @@ -85,39 +99,33 @@ class MockStrategy(object): should_init=None, should_checkpoint=None, should_save_summary=None): - self._between_graph = between_graph - self._should_init = should_init - self._should_checkpoint = should_checkpoint - self._should_save_summary = should_save_summary - - @property - def between_graph(self): - return self._between_graph + self.extended = MockExtended(between_graph, should_init, should_checkpoint, + should_save_summary) def configure(self, session_config=None, cluster_spec=None, task_type=None, task_id=None): - if self._should_init is None: + if self.extended.experimental_should_init is None: if task_id == 0: - self._should_init = True + self.extended.experimental_should_init = True else: - self._should_init = False - if self._should_checkpoint is None: + self.extended.experimental_should_init = False + if self.extended.should_checkpoint is None: if task_id == 0: - self._should_checkpoint = True + self.extended.should_checkpoint = True else: - self._should_checkpoint = False - if self._should_save_summary is None: + self.extended.should_checkpoint = False + if self.extended.should_save_summary is None: if task_id == 0: - self._should_save_summary = True + self.extended.should_save_summary = True else: - self._should_save_summary = False + self.extended.should_save_summary = False if session_config: if (cluster_spec and task_type and task_id is not None and - self._between_graph): + self.extended.experimental_between_graph): session_config.intra_op_parallelism_threads += 1 if task_type in ["chief", "worker"]: session_config.device_filters.extend( @@ -126,18 +134,6 @@ class MockStrategy(object): session_config.inter_op_parallelism_threads += 1 session_config.device_filters.append("/job:somejob") - @property - def should_init(self): - return self._should_init - - @property - def should_checkpoint(self): - return self._should_checkpoint - - @property - def should_save_summary(self): - return self._should_save_summary - class MockServer(object): @@ -372,9 +368,12 @@ class DistributeCoordinatorTestBase(test.TestCase): context = distribute_coordinator_context.get_current_worker_context() self.assertTrue(context is not None) - self.assertEqual(context._strategy.should_init, strategy.should_init) - self.assertEqual(context.should_checkpoint, strategy.should_checkpoint) - self.assertEqual(context.should_save_summary, strategy.should_save_summary) + self.assertEqual(context._strategy.extended.experimental_should_init, + strategy.extended.experimental_should_init) + self.assertEqual(context.should_checkpoint, + strategy.extended.should_checkpoint) + self.assertEqual(context.should_save_summary, + strategy.extended.should_save_summary) task_type = str(context.task_type) task_id = context.task_id or 0 @@ -384,7 +383,8 @@ class DistributeCoordinatorTestBase(test.TestCase): while len(self._strategy_property[task_type]) <= task_id: self._strategy_property[task_type].append(None) self._strategy_property[task_type][task_id] = ( - context._strategy.should_init, context.should_checkpoint, + context._strategy.extended.experimental_should_init, + context.should_checkpoint, context.should_save_summary) def _run_mock_std_server(self, @@ -930,4 +930,14 @@ class RunStandardTensorflowServerTest(test.TestCase): if __name__ == "__main__": # TODO(yuefengz): find a smart way to terminite std server threads. with test.mock.patch.object(sys, "exit", os._exit): + # Reduce `recovery_wait_secs` from 30 seconds so the test completes quickly. + orig_init = session_manager.SessionManager.__init__ + + def new_init(*args, **kwargs): + kwargs.pop("recovery_wait_secs", None) + kwargs["recovery_wait_secs"] = 0.5 + orig_init(*args, **kwargs) + + session_manager.SessionManager.__init__ = new_init + test.main() diff --git a/tensorflow/python/distribute/distribute_lib.py b/tensorflow/python/distribute/distribute_lib.py new file mode 100644 index 00000000000..eddd6ff8b16 --- /dev/null +++ b/tensorflow/python/distribute/distribute_lib.py @@ -0,0 +1,1682 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Library for running a computation across multiple devices.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import copy +import threading +import weakref +import enum + +from tensorflow.python.data.ops import dataset_ops +from tensorflow.python.distribute import device_util +from tensorflow.python.distribute import distribution_strategy_context +from tensorflow.python.distribute import reduce_util +from tensorflow.python.eager import context as eager_context +from tensorflow.python.framework import constant_op +from tensorflow.python.framework import dtypes +from tensorflow.python.framework import ops +from tensorflow.python.ops import array_ops +from tensorflow.python.ops import control_flow_ops +from tensorflow.python.ops import resource_variable_ops +from tensorflow.python.ops import variable_scope +from tensorflow.python.ops.losses import losses_impl +from tensorflow.python.platform import tf_logging +from tensorflow.python.util import nest +from tensorflow.python.util.tf_export import tf_export +from tensorflow.tools.docs import doc_controls + + +# ------------------------------------------------------------------------------ +# Context tracking whether in a strategy.update() or .update_non_slot() call. + + +_update_device = threading.local() + + +def get_update_device(): + """Get the current device if in a `tf.distribute.Strategy.update()` call.""" + try: + return _update_device.current + except AttributeError: + return None + + +class UpdateContext(object): + """Context manager when you are in `update()` or `update_non_slot()`.""" + + def __init__(self, device): + self._device = device + self._old_device = None + + def __enter__(self): + self._old_device = get_update_device() + _update_device.current = self._device + + def __exit__(self, exception_type, exception_value, traceback): + del exception_type, exception_value, traceback + _update_device.current = self._old_device + + +# ------------------------------------------------------------------------------ +# Public utility functions. + + +@tf_export("distribute.get_loss_reduction") +def get_loss_reduction(): + """`tf.distribute.ReduceOp` corresponding to the last loss reduction.""" + loss_reduction = ops.get_default_graph()._last_loss_reduction # pylint: disable=protected-access + if (loss_reduction == losses_impl.Reduction.SUM or + loss_reduction == losses_impl.ReductionV2.SUM): + return reduce_util.ReduceOp.SUM + return reduce_util.ReduceOp.MEAN + + +# ------------------------------------------------------------------------------ +# Internal API for validating the current thread mode + + +def _require_cross_replica_context_extended(extended): + """Verify in cross-replica context.""" + context = _get_per_thread_mode() + cross_replica = context.cross_replica_context + if cross_replica is not None and cross_replica.extended is extended: + return + strategy = extended._container_strategy() # pylint: disable=protected-access + # We have an error to report, figure out the right message. + if context.distribution_strategy is not strategy: + _wrong_strategy_scope(strategy, context) + assert cross_replica is None + raise RuntimeError("Method requires being in cross-replica context, use " + "get_replica_context().merge_call()") + + +def _wrong_strategy_scope(strategy, context): + # Figure out the right error message. + if not distribution_strategy_context.has_distribution_strategy(): + raise RuntimeError( + 'Need to be inside "with strategy.scope()" for %s' % + (strategy,)) + else: + raise RuntimeError( + "Mixing different tf.distribute.Strategy objects: %s is not %s" % + (context.distribution_strategy, strategy)) + + +def require_replica_context(replica_ctx): + """Verify in `replica_ctx` replica context.""" + context = _get_per_thread_mode() + if context.replica_context is replica_ctx: return + # We have an error to report, figure out the right message. + if context.replica_context is None: + raise RuntimeError("Need to be inside `call_for_each_replica()`") + if context.distribution_strategy is replica_ctx.distribution_strategy: + # Two different ReplicaContexts with the same tf.distribute.Strategy. + raise RuntimeError("Mismatching ReplicaContext.") + raise RuntimeError( + "Mismatching tf.distribute.Strategy objects: %s is not %s." % + (context.distribution_strategy, replica_ctx.distribution_strategy)) + + +def _require_distribution_strategy_scope_strategy(strategy): + """Verify in a `strategy.scope()` in this thread.""" + context = _get_per_thread_mode() + if context.distribution_strategy is strategy: return + _wrong_strategy_scope(strategy, context) + + +def _require_distribution_strategy_scope_extended(extended): + """Verify in a `distribution_strategy.scope()` in this thread.""" + context = _get_per_thread_mode() + if context.distribution_strategy.extended is extended: return + # Report error. + strategy = extended._container_strategy() # pylint: disable=protected-access + _wrong_strategy_scope(strategy, context) + + +# ------------------------------------------------------------------------------ +# Internal context managers used to implement the DistributionStrategy +# base class + + +class _CurrentDistributionContext(object): + """Context manager setting the current `tf.distribute.Strategy`. + + Also: overrides the variable creator and optionally the current device. + """ + + def __init__(self, + strategy, + var_creator_scope, + var_scope=None, + default_device=None): + self._context = distribution_strategy_context._CrossReplicaThreadMode( # pylint: disable=protected-access + strategy) + self._var_creator_scope = var_creator_scope + self._var_scope = var_scope + if default_device: + self._device_scope = ops.device(default_device) + else: + self._device_scope = None + + def __enter__(self): + _push_per_thread_mode(self._context) + if self._var_scope: + self._var_scope.__enter__() + self._var_creator_scope.__enter__() + if self._device_scope: + self._device_scope.__enter__() + return self._context.distribution_strategy + + def __exit__(self, exception_type, exception_value, traceback): + if self._device_scope: + self._device_scope.__exit__(exception_type, exception_value, traceback) + self._var_creator_scope.__exit__(exception_type, exception_value, traceback) + if self._var_scope: + self._var_scope.__exit__(exception_type, exception_value, traceback) + _pop_per_thread_mode() + + +class _SameScopeAgainContext(object): + """Trivial context manager when you are already in `scope()`.""" + + def __init__(self, strategy): + self._distribution_strategy = strategy + + def __enter__(self): + return self._distribution_strategy + + def __exit__(self, exception_type, exception_value, traceback): + del exception_type, exception_value, traceback + + +# TODO(yuefengz): add more replication modes. +@tf_export("distribute.InputReplicationMode") +class InputReplicationMode(enum.Enum): + """Replication mode for input function.""" + + # The input function will be called on each worker independently, creating as + # many input pipelines as number of workers. Replicas will dequeue from the + # local Dataset on their worker. Distribution Strategy doesn't manage any + # state sharing between such separate input pipelines. + PER_WORKER = "PER_WORKER" + + +@tf_export("distribute.InputContext") +class InputContext(object): + """A class wrapping information needed by an input function. + + This is a context class that is passed to the user's input fn and contains + information about the compute replicas and input pipelines. The number of + compute replicas (in sync training) helps compute per input pipeline batch + size from the desired global batch size. Input pipeline information can be + used to return a different subset of the input in each input pipeline (for + e.g. shard the input pipeline, use a different input source etc). + """ + + def __init__(self, + num_input_pipelines=1, + input_pipeline_id=0, + num_replicas_in_sync=1): + """Initializes an InputContext object. + + Args: + num_input_pipelines: the number of input pipelines in a cluster. + input_pipeline_id: the current input pipeline id, should be an int in + [0,`num_input_pipelines`). + num_replicas_in_sync: the number of replicas that are in sync. + """ + self._num_input_pipelines = num_input_pipelines + self._input_pipeline_id = input_pipeline_id + self._num_replicas_in_sync = num_replicas_in_sync + + @property + def num_replicas_in_sync(self): + """Returns the number of compute replicas in sync.""" + return self._num_replicas_in_sync + + @property + def input_pipeline_id(self): + """Returns the input pipeline ID.""" + return self._input_pipeline_id + + @property + def num_input_pipelines(self): + """Returns the number of input pipelines.""" + return self._num_input_pipelines + + def get_per_replica_batch_size(self, global_batch_size): + """Returns the per-replica batch size. + + Args: + global_batch_size: the global batch size which should be divisible by + `num_replicas_in_sync`. + + Returns: + the per-replica batch size. + + Raises: + ValueError: if `global_batch_size` not divisible by + `num_replicas_in_sync`. + """ + if global_batch_size % self._num_replicas_in_sync != 0: + raise ValueError("The `global_batch_size` %r is not divisible by " + "`num_replicas_in_sync` %r " % + (global_batch_size, self._num_replicas_in_sync)) + return global_batch_size // self._num_replicas_in_sync + + +# ------------------------------------------------------------------------------ +# Base classes for all distribution strategies. + + +@tf_export("distribute.Strategy") +class DistributionStrategy(object): + """A list of devices with a state & compute distribution policy. + + See [tensorflow/contrib/distribute/README.md]( + https://www.tensorflow.org/code/tensorflow/contrib/distribute/README.md) + for overview and examples. + """ + + # TODO(josh11b): Raise an exception if variable partitioning requested before + # we add support. + # TODO(josh11b): Also `parameter_device_index` property? + # TODO(josh11b): `map()` + # TODO(josh11b): ClusterSpec/ClusterResolver + # TODO(josh11b): Partitioned computations, state; sharding + # TODO(josh11b): Model parallelism: "replicas" with multiple devices; shuffling + # TODO(josh11b): List of replicas with their worker and parameter devices + # (where the parameter devices may overlap in the ps case). + + def __init__(self, extended): + self._extended = extended + + @property + def extended(self): + """`tf.distribute.StrategyExtended` with additional methods.""" + return self._extended + + def scope(self): + """Returns a context manager selecting this Strategy as current. + + Inside a `with strategy.scope():` code block, this thread + will use a variable creator set by `strategy`, and will + enter its "cross-replica context". + + Returns: + A context manager. + """ + return self._extended._scope(self) # pylint: disable=protected-access + + @doc_controls.do_not_generate_docs # DEPRECATED, moving to `extended` + def read_var(self, v): + """DEPRECATED: use extended.read_var() instead.""" + return self._extended.read_var(v) + + @doc_controls.do_not_generate_docs # DEPRECATED, moving to `extended` + def colocate_vars_with(self, colocate_with_variable): + """DEPRECATED: use extended.colocate_vars_with() instead.""" + return self._extended.colocate_vars_with(colocate_with_variable) + + @doc_controls.do_not_generate_docs # DEPRECATED + def distribute_dataset(self, dataset_fn): + """Return a `dataset` split across all replicas. DEPRECATED. + + DEPRECATED: Please use `make_dataset_iterator` or + `make_input_fn_iterator` instead. + + Suitable for providing input to `extended.call_for_each_replica()` by + creating an iterator: + + ``` + def dataset_fn(): + return tf.data.Dataset.from_tensors([[1.]]).repeat() + + with strategy.scope(): + distributed_dataset = strategy.distribute_dataset(dataset_fn) + iterator = distributed_dataset.make_initializable_iterator() + replica_results = strategy.extended.call_for_each_replica( + replica_fn, args=(iterator.get_next(),)) + ``` + + Args: + dataset_fn: A function that returns a `tf.data.Dataset`. + + Returns: + A `PerReplicaDataset` that will produce data for each replica. + """ + return self._extended._distribute_dataset(dataset_fn) # pylint: disable=protected-access + + def make_dataset_iterator(self, dataset): + """Makes an iterator for input provided via input_dataset. + + Data from the given dataset will be distributed evenly across all the + compute replicas. We will assume that the input dataset is batched by the + global batch size. With this assumption, we will make a best effort to + divide each batch across all the replicas (one or more workers). + If this effort fails, an error will be thrown, and the user should instead + use `make_input_fn_iterator` which provides more control to the user, and + does not try to divide a batch across replicas. + + The user could also use `make_input_fn_iterator` if they want to + customize which input is fed to which replica/worker etc. + + Args: + dataset: `tf.data.Dataset` that will be distributed evenly across all + replicas. + + Returns: + An `tf.distribute.InputIterator` which returns inputs for each step of the + computation. User should call `initialize` on the returned iterator. + """ + return self._extended._make_dataset_iterator(dataset) # pylint: disable=protected-access + + def make_input_fn_iterator(self, + input_fn, + replication_mode=InputReplicationMode.PER_WORKER): + """Returns an iterator split across replicas created from an input function. + + The `input_fn` should take an `tf.distribute.InputContext` object where + information about input sharding can be accessed: + + ``` + def input_fn(input_context): + d = tf.data.Dataset.from_tensors([[1.]]).repeat() + return d.shard(input_context.num_input_pipelines, + input_context.input_pipeline_id) + with strategy.scope(): + iterator = strategy.make_input_fn_iterator( + input_fn) + replica_results = strategy.extended.call_for_each_replica( + replica_fn, iterator.get_next()) + ``` + + Args: + input_fn: A function that returns a `tf.data.Dataset`. This function is + expected to take an `tf.distribute.InputContext` object. + replication_mode: an enum value of `tf.distribute.InputReplicationMode`. + Only `PER_WORKER` is supported currently. + + Returns: + An iterator object that can be initialized and fetched next element. + """ + if replication_mode != InputReplicationMode.PER_WORKER: + raise ValueError( + "Input replication mode not supported: %r" % replication_mode) + return self.extended._make_input_fn_iterator( # pylint: disable=protected-access + input_fn, replication_mode=replication_mode) + + @doc_controls.do_not_generate_docs # DEPRECATED, moving to `extended` + def broadcast(self, tensor, destinations=None): + """DEPRECATED: use extended.broadcast_to() instead.""" + return self._extended.broadcast_to(tensor, destinations) + + @doc_controls.do_not_generate_docs # Use experimental_initialize() instead. + def initialize(self): + """DEPRECATED: Use `experimental_initialize()` instead.""" + return self._extended._initialize() # pylint: disable=protected-access + + def experimental_initialize(self): + """Any initialization to be done before running any computations. + + In eager mode, it executes any initialization as a side effect. + In graph mode, it creates the initialization ops and returns them. + + For example, TPU initialize_system ops. + + Returns: + A list of ops to execute. + """ + return self._extended._initialize() # pylint: disable=protected-access + + @doc_controls.do_not_generate_docs # Use experimental_finalize() instead. + def finalize(self): + """DEPRECATED: Use `experimental_finalize()` instead.""" + return self._extended._finalize() # pylint: disable=protected-access + + def experimental_finalize(self): + """Any final actions to be done at the end of all computations. + + In eager mode, it executes any finalize actions as a side effect. + In graph mode, it creates the finalize ops and returns them. + + For example, TPU shutdown ops. + + Returns: + A list of ops to execute. + """ + return self._extended._finalize() # pylint: disable=protected-access + + @doc_controls.do_not_generate_docs # DEPRECATED, moving to `extended` + def run_steps_on_dataset(self, fn, iterator, iterations=1, + initial_loop_values=None): + """DEPRECATED: use extended.experimental_run_steps_on_iterator() instead.""" + return self._extended.experimental_run_steps_on_iterator( + fn, iterator, iterations, initial_loop_values) + + @doc_controls.do_not_generate_docs # DEPRECATED, moving to `extended` + def call_for_each_replica(self, fn, *args, **kwargs): + """DEPRECATED: use extended.call_for_each_replica() instead.""" + # Handle old *args, **kwargs, and new args=(...), kwargs={...}, to + # allow transition. + a = kwargs.pop("args", None) + if a is not None: + if args: + raise ValueError( + "Can't pass *args and args=... to call_for_each_replica") + args = a + k = kwargs.pop("kwargs", None) + if k is not None: + if kwargs: + raise ValueError( + "Can't pass **kwargs and kwargs=... to call_for_each_replica") + kwargs = k + kwargs.pop("run_concurrently", None) # Ignore old option. + return self._extended.call_for_each_replica(fn, args, kwargs) + + def reduce(self, reduce_op, value): + """Reduce `value` across replicas. + + Args: + reduce_op: A `tf.distribute.ReduceOp` value specifying how values should + be combined. + value: A "per replica" value to be combined into a single tensor. + + Returns: + A `Tensor`. + """ + _require_cross_replica_context_extended(self._extended) + return self._extended._reduce(reduce_op, value) # pylint: disable=protected-access + + @doc_controls.do_not_generate_docs # DEPRECATED, moving to `extended` + def batch_reduce(self, aggregation, value_destination_pairs): + """DEPRECATED: use extended.batch_reduce_to() instead.""" + return self._extended.batch_reduce_to(aggregation, value_destination_pairs) + + @doc_controls.do_not_generate_docs # DEPRECATED, moving to `extended` + def update(self, var, fn, *args, **kwargs): + """DEPRECATED: use extended.update() instead.""" + group = kwargs.pop("group", True) + # We temporarily support "grouped" in addition to "group" for backward- + # compatibility. + group = kwargs.pop("grouped", True) and group + # Handle old *args, **kwargs, and new args=(...), kwargs={...}, to + # allow transition. + a = kwargs.pop("args", None) + if a is not None: + if args: + raise ValueError( + "Can't pass *args and args=... to update") + args = a + k = kwargs.pop("kwargs", None) + if k is not None: + if kwargs: + raise ValueError( + "Can't pass **kwargs and kwargs=... to update") + kwargs = k + return self._extended.update(var, fn, args, kwargs, group) + + @doc_controls.do_not_generate_docs # DEPRECATED, moving to `extended` + def update_non_slot(self, colocate_with, fn, *args, **kwargs): + """DEPRECATED: use extended.update_non_slot() instead.""" + group = kwargs.pop("group", True) + # We temporarily support "grouped" in addition to "group" for backward- + # compatibility. + group = kwargs.pop("grouped", True) and group + # Handle old *args, **kwargs, and new args=(...), kwargs={...}, to + # allow transition. + a = kwargs.pop("args", None) + if a is not None: + if args: + raise ValueError( + "Can't pass *args and args=... to update_non_slot") + args = a + k = kwargs.pop("kwargs", None) + if k is not None: + if kwargs: + raise ValueError( + "Can't pass **kwargs and kwargs=... to update_non_slot") + kwargs = k + return self._extended.update_non_slot( + colocate_with, fn, args, kwargs, group) + + @doc_controls.do_not_generate_docs # DEPRECATED, -> `DistributedValues` + def unwrap(self, value): + """Returns the list of all per-replica values contained in `value`. + + Args: + value: A value returned by `extended.call_for_each_replica()` or a + variable created in `scope`. + + Returns: + A list of values contained in `value`. If `value` represents a single + value, this returns `[value].` + """ + return self._extended._unwrap(value) # pylint: disable=protected-access + + @doc_controls.do_not_generate_docs # DEPRECATED, moving to `extended` + def value_container(self, value): + """DEPRECATED: use extended.value_container() instead.""" + return self._extended.value_container(value) + + @doc_controls.do_not_generate_docs # DEPRECATED, -> `DistributedValues` + def group(self, value, name=None): + """Shortcut for `tf.group(self.unwrap(value))`.""" + return self._extended._group(value, name) # pylint: disable=protected-access + + @property + @doc_controls.do_not_generate_docs # DEPRECATED, moving to `extended` + def require_static_shapes(self): + """DEPRECATED: use extended.require_static_shapes instead.""" + return self._extended.experimental_require_static_shapes + + @property + def num_replicas_in_sync(self): + """Returns number of replicas over which gradients are aggregated.""" + return self._extended._num_replicas_in_sync # pylint: disable=protected-access + + @property + @doc_controls.do_not_generate_docs # DEPRECATED, moving to `extended` + def worker_devices(self): + """DEPRECATED: use extended.worker_devices instead.""" + return self._extended.worker_devices + + @property + @doc_controls.do_not_generate_docs # DEPRECATED, moving to `extended` + def parameter_devices(self): + """DEPRECATED: use extended.parameter_devices instead.""" + return self._extended.parameter_devices + + @doc_controls.do_not_generate_docs # DEPRECATED, moving to `extended` + def non_slot_devices(self, var_list): + """DEPRECATED: use extended.non_slot_devices instead.""" + return self._extended.non_slot_devices(var_list) + + @property + @doc_controls.do_not_generate_docs # DEPRECATED, moving to `extended` + def between_graph(self): + """DEPRECATED: use extended.experimental_between_graph instead.""" + return self._extended.experimental_between_graph + + @doc_controls.do_not_generate_docs # DEPRECATED, being replaced by a new API. + def configure(self, + session_config=None, + cluster_spec=None, + task_type=None, + task_id=None): + # pylint: disable=g-doc-return-or-yield,g-doc-args + """DEPRECATED: use `update_config_proto` instead. + + Configures the strategy class. + + DEPRECATED: This method's functionality has been split into the strategy + constructor and `update_config_proto`. In the future, we will allow passing + cluster and config_proto to the constructor to configure the strategy. And + `update_config_proto` can be used to update the config_proto based on the + specific strategy. + """ + return self._extended._configure( # pylint: disable=protected-access + session_config, cluster_spec, task_type, task_id) + + def update_config_proto(self, config_proto): + """Returns a copy of `config_proto` modified for use with this strategy. + + The updated config has something needed to run a strategy, e.g. + configuration to run collective ops, or device filters to improve + distributed training performance. + + Args: + config_proto: a `tf.ConfigProto` object. + + Returns: + The updated copy of the `config_proto`. + """ + return self._extended._update_config_proto(config_proto) # pylint: disable=protected-access + + @property + @doc_controls.do_not_generate_docs # DEPRECATED, moving to `extended` + def should_init(self): + """DEPRECATED: use extended.should_init instead.""" + return self._extended.experimental_should_init + + @property + @doc_controls.do_not_generate_docs # DEPRECATED, moving to `extended` + def should_checkpoint(self): + """DEPRECATED: use extended.should_checkpoint instead.""" + return self._extended.should_checkpoint + + @property + @doc_controls.do_not_generate_docs # DEPRECATED, moving to `extended` + def should_save_summary(self): + """DEPRECATED: use extended.should_save_summary instead.""" + return self._extended.should_save_summary + + def __deepcopy__(self, memo): + # First do a regular deepcopy of `self`. + cls = self.__class__ + result = cls.__new__(cls) + memo[id(self)] = result + for k, v in self.__dict__.items(): + setattr(result, k, copy.deepcopy(v, memo)) + # One little fix-up: we want `result._extended` to reference `result` + # instead of `self`. + result._extended._container_strategy_weakref = weakref.ref(result) # pylint: disable=protected-access + return result + + def __copy__(self): + raise RuntimeError("Must only deepcopy DistributionStrategy.") + + +@tf_export("distribute.StrategyExtended") +class DistributionStrategyExtended(object): + """Additional APIs for algorithms that need to be distribution-aware. + + The intent is that you can write an algorithm in a stylized way and + it will be usable with a variety of different + `tf.distribute.Strategy` + implementations. Each descendant will implement a different strategy + for distributing the algorithm across multiple devices/machines. + Furthermore, these changes can be hidden inside the specific layers + and other library classes that need special treatment to run in a + distributed setting, so that most users' model definition code can + run unchanged. The `tf.distribute.Strategy` API works the same way + with eager and graph execution. + + First let's introduce a few high-level concepts: + + * _Data parallelism_ is where we run multiple copies of the model + on different slices of the input data. This is in contrast to + _model parallelism_ where we divide up a single copy of a model + across multiple devices. + Note: we only support data parallelism for now, but + hope to add support for model parallelism in the future. + * A _replica_ is one copy of the model, running on one slice of the + input data. + * _Synchronous_, or more commonly _sync_, training is where the + updates from each replica are aggregated together before updating + the model variables. This is in contrast to _asynchronous_, or + _async_ training, where each replica updates the model variables + independently. + * Furthermore you might run your computation on multiple devices + on one machine (or "host"), or on multiple machines/hosts. + If you are running on multiple machines, you might have a + single master host that drives computation across all of them, + or you might have multiple clients driving the computation + asynchronously. + + To distribute an algorithm, we might use some of these ingredients: + + * Parameter servers: These are hosts that hold a single copy of + parameters/variables. All replicas that want to operate on a variable + retrieve it at the beginning of a step and send an update to be + applied at the end of the step. Can support either sync or async + training. + * Mirrored variables: These are variables that are copied to multiple + devices, where we keep the copies in sync by applying the same + updates to every copy. Normally would only be used with sync training. + * Reductions and Allreduce: A _reduction_ is some method of + aggregating multiple values into one value, like "sum" or + "mean". If doing sync training, we will perform a reduction on the + gradients to a parameter from all replicas before applying the + update. Allreduce is an algorithm for performing a reduction on + values from multiple devices and making the result available on + all of those devices. + * In the future we will have support for TensorFlow's partitioned + variables, where a single variable is split across multiple + devices. + + We have then a few approaches we want to support: + + * Code written (as if) with no knowledge of class `tf.distribute.Strategy`. + This code should work as before, even if some of the layers, etc. + used by that code are written to be distribution-aware. This is done + by having a default `tf.distribute.Strategy` that gives ordinary behavior, + and by default being in a single replica context. + * Ordinary model code that you want to run using a specific + `tf.distribute.Strategy`. This can be as simple as: + + ``` + with my_strategy.scope(): + iterator = my_strategy.make_dataset_iterator(dataset) + session.run(iterator.initialize()) + replica_train_ops = my_strategy.extended.call_for_each_replica( + replica_fn, args=(iterator.get_next(),)) + train_op = my_strategy.group(replica_train_ops) + ``` + + This takes an ordinary `dataset` and `replica_fn` and runs it + distributed using a particular `tf.distribute.Strategy` in + `my_strategy`. Any variables created in `replica_fn` are created + using `my_strategy`'s policy, and library functions called by + `replica_fn` can use the `get_replica_context()` API to get enhanced + behavior in this case. + + * If you want to write a distributed algorithm, you may use any of + the `tf.distribute.Strategy` APIs inside a + `with my_strategy.scope():` block of code. + + Lower-level concepts: + + * Wrapped values: In order to represent values parallel across devices + (either replicas or the devices associated with a particular value), we + wrap them in a "PerReplica" or "Mirrored" object that contains a map + from device to values. "PerReplica" is used when the value may be + different across replicas, and "Mirrored" when the value are the same. + * Unwrapping and merging: Consider calling a function `fn` on multiple + replicas, like `extended.call_for_each_replica(fn, args=[w])` with an + argument `w` that is a wrapped value. This means `w` will have a map taking + replica device `d0` to `w0`, replica device `d1` to `w1`, + etc. `extended.call_for_each_replica()` unwraps `w` before calling `fn`, so + it calls `fn(w0)` on `d0`, `fn(w1)` on `d1`, etc. It then merges the return + values from `fn()`, which can possibly result in wrapped values. For + example, let's say `fn()` returns a tuple with three components: `(x, a, + v0)` from replica 0, `(x, b, v1)` on replica 1, etc. If the first component + is the same object `x` from every replica, then the first component of the + merged result will also be `x`. If the second component is different (`a`, + `b`, ...) from each replica, then the merged value will have a wrapped map + from replica device to the different values. If the third component is the + members of a mirrored variable (`v` maps `d0` to `v0`, `d1` to `v1`, etc.), + then the merged result will be that mirrored variable (`v`). + * Replica context vs. Cross-replica context: _replica context_ is when we + are in some function that is being called once for each replica. + Otherwise we are in cross-replica context, which is useful for + calling `tf.distribute.Strategy` methods which operate across the + replicas (like `reduce_to()`). By default you start in a replica context + (the default "single replica context") and then some methods can + switch you back and forth, as described below. + * Worker devices vs. parameter devices: Most replica computations will + happen on worker devices. Since we don't yet support model + parallelism, there will be one worker device per replica. When using + parameter servers (see above), the set of devices holding + variables may be different, otherwise the parameter devices might + match the worker devices. + * Non-slot devices are some subset of the parameter devices where we + put all the non-slot variables. We need to ensure that all + non-slot variables are allocated on the same device, or mirrored + across the same set of devices. If you have some variable you want + to colocate all the non-slot variables with, you can use + `colocate_vars_with()` to get the remaining non-slot variables on + the same device. Otherwise you can use `non_slot_devices()` to + pick a consistent set of devices to pass to both + `colocate_vars_with()` and `update_non_slot()`. + + When using a `tf.distribute.Strategy`, we have a new type dimension + called _locality_ that says what values are compatible with which + APIs: + + * T: different value for each replica (e.g. a PerReplica-wrapped value). + * M: value is "mirrored" across replicas, i.e. there are copies with the + same value on each replica (e.g. a Mirrored-wrapped value). + * V(`v`): value is "mirrored" across all the devices which have a + copy of variable `v` (also a Mirrored-wrapped value, but over + parameter devices instead of worker devices). + * N: value is "mirrored" across all the "non-slot" devices + + Rules for methods with respect to locality and single-replica vs. + cross-replica context: + + * `with d.scope()`: default single-replica context -> cross-replica context + for `d` + * `with d.extended.colocate_vars_with(v)`: in replica/cross-replica context, + variables will be created with locality V(`v`). That is, if we write + `with d.extended.colocate_vars_with(v1): v2 = tf.get_variable(...)`, + then `v2` will have locality V(`v1`), i.e. locality V(`v2`) will equal + V(`v1`). + * `with d.extended.colocate_vars_with(d.extended.non_slot_devices(...))`: in + replica/cross-replica context, variables will be created with locality N + * `v = tf.get_variable(...)`: in replica/cross-replica context, creates + a variable (which by definition will have locality V(`v`), though + will match another locality if inside a `colocate_vars_with` + scope). + * `d.make_dataset_iterator(dataset)` (or the deprecated + `d.distribute_dataset(dataset).make_one_shot_iterator()`): in cross-replica + context, produces an iterator with locality T + * `d.extended.broadcast_to(t)`: in cross-replica context, produces a value + with locality M + * `d.extended.broadcast_to(t, v)`: in cross-replica context, produces a value + with locality V(`v`) + * `d.extended.call_for_each_replica(fn, ...)`: in cross-replica context, runs + `fn()` in a replica context (and so may call `get_replica_context()` and + use its API, including `merge_call()` to get back to cross-replica + context), once for each replica. May use values with locality T or + M, and any variable. + * `d.extended.reduce_to(m, t, t)`: in cross-replica context, accepts t with + locality T and produces a value with locality M. + * `d.extended.reduce_to(m, t, v)`: in cross-replica context, accepts t with + locality T and produces a value with locality V(`v`). + * `d.extended.batch_reduce_to(m, [(t, v)]): see `d.extended.reduce_to()` + * `d.extended.update(v, fn, ...)`: in cross-replica context, runs `fn()` once + for each device `v` is copied to, all inputs should have locality + V(`v`), output will have locality V(`v`) as well. + * `d.extended.update_non_slot(d.extended.non_slot_devices(), fn)`: in + cross-replica context, like `d.extended.update()` except with locality N. + * `d.extended.read_var(v)`: Gets the (read-only) value of the variable `v` (on + the device determined by the current device scope), aggregating + across replicas for replica-local variables. Frequently, this will be + done automatically when using `v` in an expression or fetching it in + a cross-replica context, but this function can be used to force that + conversion happens at a particular point in time (for example, to + add the result of the conversion to a graph collection). + + The standard pattern for updating variables is to: + + 1. Create an input iterator with `d.make_dataset_iterator()`. + 2. Define each replica `d.extended.call_for_each_replica()` up to the point of + getting a list of gradient, variable pairs. + 3. Call `d.extended.reduce_to(VariableAggregation.SUM, t, v)` or + `d.extended.batch_reduce_to()` to sum the gradients (with locality T) + into values with locality V(`v`). + 4. Call `d.extended.update(v)` for each variable to update its value. + + Steps 3 and 4 are done automatically by class `Optimizer` if you call + its `apply_gradients` method in a replica context. Otherwise you can + manually call its `_distributed_apply` method in a cross-replica context. + + Another thing you might want to do in the middle of your replica function is + an all-reduce of some intermediate value, using `d.extended.reduce_to()` or + `d.extended.batch_reduce_to()`. You simply provide the same tensor as the + input and destination. + + Layers should expect to be called in a replica context, and can use + the `tf.distribute.get_replica_context` function to get a + `tf.distribute.ReplicaContext` object. The + `ReplicaContext` object has a `merge_call()` method for entering + cross-replica context where you can use `reduce_to()` (or + `batch_reduce_to()`) and then optionally `update()` to update state. + + You may use this API whether or not a `tf.distribute.Strategy` is + being used, since there is a default implementation of + `ReplicaContext` and `tf.distribute.Strategy`. + + NOTE for new `tf.distribute.Strategy` implementations: Please put all logic + in a subclass of `tf.distribute.StrategyExtended`. The only code needed for + the `tf.distribute.Strategy` subclass is for instantiating your subclass of + `tf.distribute.StrategyExtended` in the `__init__` method. + """ + + def __init__(self, container_strategy): + self._container_strategy_weakref = weakref.ref(container_strategy) + self._default_device = None + # This property is used to determine if we should set drop_remainder=True + # when creating Datasets from numpy array inputs. + self._require_static_shapes = False + + def _container_strategy(self): + """Get the containing `DistributionStrategy`. + + This should not generally be needed except when creating a new + `ReplicaContext` and to validate that the caller is in the correct + `scope()`. + + Returns: + The `DistributionStrategy` such that `strategy.extended` is `self`. + """ + container_strategy = self._container_strategy_weakref() + assert container_strategy is not None + return container_strategy + + def _scope(self, strategy): + """Implementation of DistributionStrategy.scope().""" + if distribution_strategy_context.has_distribution_strategy(): + _require_cross_replica_context_extended(self) + return _SameScopeAgainContext(strategy) + + def creator_with_resource_vars(*args, **kwargs): + _require_distribution_strategy_scope_extended(self) + kwargs["use_resource"] = True + return self._create_variable(*args, **kwargs) + + def distributed_getter(getter, *args, **kwargs): + if not self._allow_variable_partition(): + if kwargs.pop("partitioner", None) is not None: + tf_logging.log_first_n( + tf_logging.WARN, "Partitioned variables are disabled when using " + "current tf.distribute.Strategy.", 1) + return getter(*args, **kwargs) + + return _CurrentDistributionContext( + strategy, + variable_scope.variable_creator_scope(creator_with_resource_vars), + variable_scope.variable_scope( + variable_scope.get_variable_scope(), + custom_getter=distributed_getter), self._default_device) + + def _allow_variable_partition(self): + return False + + def _create_variable(self, next_creator, *args, **kwargs): + # Note: should support "colocate_with" argument. + raise NotImplementedError("must be implemented in descendants") + + def read_var(self, v): + """Reads the value of a variable. + + Returns the aggregate value of a replica-local variable, or the + (read-only) value of any other variable. + + Args: + v: A variable allocated within the scope of this `tf.distribute.Strategy`. + + Returns: + A tensor representing the value of `v`, aggregated across replicas if + necessary. + """ + raise NotImplementedError("must be implemented in descendants") + + def colocate_vars_with(self, colocate_with_variable): + """Scope that controls which devices variables will be created on. + + No operations should be added to the graph inside this scope, it + should only be used when creating variables (some implementations + work by changing variable creation, others work by using a + tf.colocate_with() scope). + + This may only be used inside `self.scope()`. + + Example usage: + + ``` + with strategy.scope(): + var1 = tf.get_variable(...) + with strategy.extended.colocate_vars_with(v1): + # var2 and var3 will be created on the same device(s) as var1 + var2 = tf.get_variable(...) + var3 = tf.get_variable(...) + + def fn(v1, v2, v3): + # operates on v1 from var1, v2 from var2, and v3 from var3 + + # `fn` runs on every device `v1` is on, `v2` and `v3` will be there too. + strategy.extended.update(v1, fn, args=(v2, v3)) + ``` + + Args: + colocate_with_variable: A created in `self.scope()`. Variables created + while in the returned context manager will be on the same set of + devices as `colocate_with_variable`. + + Returns: + A context manager. + """ + def create_colocated_variable(next_creator, *args, **kwargs): + _require_distribution_strategy_scope_extended(self) + kwargs["use_resource"] = True + kwargs["colocate_with"] = colocate_with_variable + return next_creator(*args, **kwargs) + + _require_distribution_strategy_scope_extended(self) + return variable_scope.variable_creator_scope(create_colocated_variable) + + def _call_dataset_fn(self, dataset_fn): + """Call the `dataset_fn` with `input_context` as argument.""" + result = dataset_fn() + if not isinstance(result, dataset_ops.DatasetV2): + raise ValueError( + "dataset_fn() must return a tf.data.Dataset when using a " + "tf.distribute.Strategy.") + return result + + # TODO(josh11b): `PerReplicaDataset` currently only implements a few methods of + # Dataset API such as make_one_shot_iterator and make_initializable_iterator. + # Extend to implement more functionality of datasets. + def _distribute_dataset(self, dataset_fn): + raise NotImplementedError("must be implemented in descendants") + + def _make_dataset_iterator(self, dataset): + raise NotImplementedError("must be implemented in descendants") + + def _make_input_fn_iterator(self, input_fn, replication_mode): + raise NotImplementedError("must be implemented in descendants") + + def broadcast_to(self, tensor, destinations): + """Mirror a tensor on one device to all worker devices. + + Args: + tensor: A Tensor value to broadcast. + destinations: A mirrored variable or device string specifying the + destination devices to copy `tensor` to. + + Returns: + A value mirrored to `destinations` devices. + """ + # TODO(josh11b): More docstring + _require_cross_replica_context_extended(self) + assert not isinstance(destinations, (list, tuple)) + return self._broadcast_to(tensor, destinations) + + def _broadcast_to(self, tensor, destinations): + raise NotImplementedError("must be implemented in descendants") + + def _initialize(self): + return [] + + def _finalize(self): + return [] + + def experimental_run_steps_on_iterator(self, fn, iterator, iterations=1, + initial_loop_values=None): + """Run `fn` with input from `iterator` for `iterations` times. + + This method can be used to run a step function for training a number of + times using input from a dataset. + + Args: + fn: function to run using this distribution strategy. The function must + have the following signature: `def fn(context, inputs)`. + `context` is an instance of `MultiStepContext` that will be passed when + `fn` is run. `context` can be used to specify the outputs to be returned + from `fn` by calling `context.set_last_step_output`. It can also be used + to capture non tensor outputs by `context.set_non_tensor_output`. + See `MultiStepContext` documentation for more information. + `inputs` will have same type/structure as `iterator.get_next()`. + Typically, `fn` will use `call_for_each_replica` method of the strategy + to distribute the computation over multiple replicas. + iterator: Iterator of a dataset that represents the input for `fn`. The + caller is responsible for initializing the iterator as needed. + iterations: (Optional) Number of iterations that `fn` should be run. + Defaults to 1. + initial_loop_values: (Optional) Initial values to be passed into the + loop that runs `fn`. Defaults to `None`. # TODO(priyag): Remove + initial_loop_values argument when we have a mechanism to infer the + outputs of `fn`. + + Returns: + Returns the `MultiStepContext` object which has the following properties, + among other things: + - run_op: An op that runs `fn` `iterations` times. + - last_step_outputs: A dictionary containing tensors set using + `context.set_last_step_output`. Evaluating this returns the value of + the tensors after the last iteration. + - non_tensor_outputs: A dictionatry containing anything that was set by + `fn` by calling `context.set_non_tensor_output`. + """ + _require_cross_replica_context_extended(self) + return self._experimental_run_steps_on_iterator( + fn, iterator, iterations, initial_loop_values) + + def _experimental_run_steps_on_iterator(self, fn, iterator, iterations, + initial_loop_values): + raise NotImplementedError("must be implemented in descendants") + + def call_for_each_replica(self, fn, args=(), kwargs=None): + """Run `fn` once per replica. + + `fn` may call `tf.get_replica_context()` to access methods such as + `replica_id_in_sync_group` and `merge_call()`. + + `merge_call()` is used to communicate between the replicas and + re-enter the cross-replica context. All replicas pause their execution + having encountered a `merge_call()` call. After that the + `merge_fn`-function is executed. Its results are then unwrapped and + given back to each replica call. After that execution resumes until + `fn` is complete or encounters another `merge_call()`. Example: + + ```python + # Called once in "cross-replica" context. + def merge_fn(distribution, three_plus_replica_id): + # sum the values across replicas + return sum(distribution.unwrap(three_plus_replica_id)) + + # Called once per replica in `distribution`, in a "replica" context. + def fn(three): + replica_ctx = tf.get_replica_context() + v = three + replica_ctx.replica_id_in_sync_group + # Computes the sum of the `v` values across all replicas. + s = replica_ctx.merge_call(merge_fn, args=(v,)) + return s + v + + with distribution.scope(): + # in "cross-replica" context + ... + merged_results = distribution.call_for_each_replica(fn, args=[3]) + # merged_results has the values from every replica execution of `fn`. + print(distribution.unwrap(merged_results)) # Prints a list + ``` + + Args: + fn: function to run (will be run once per replica). + args: Tuple or list with positional arguments for `fn`. + kwargs: Dict with keyword arguments for `fn`. + + Returns: + Merged return value of `fn` across all replicas. + """ + _require_cross_replica_context_extended(self) + if kwargs is None: + kwargs = {} + return self._call_for_each_replica(fn, args, kwargs) + + def _call_for_each_replica(self, fn, args, kwargs): + raise NotImplementedError("must be implemented in descendants") + + def _reduce(self, reduce_op, value): + # Default implementation until we have an implementation for each strategy. + return self._unwrap(self._reduce_to( + reduce_op, value, device_util.current() or "/device:CPU:0"))[0] + + def reduce_to(self, reduce_op, value, destinations): + """Combine (via e.g. sum or mean) values across replicas. + + Args: + reduce_op: Reduction type, an instance of `tf.distribute.ReduceOp` enum. + DEPRECATED but still accepted values: + `tf.VariableAggregation.SUM`, + `tf.VariableAggregation.MEAN`, + value: A per-replica value with one value per replica. + destinations: A mirrored variable, a per-replica tensor, or a device + string. The return value will be copied to all destination devices (or + all the devices where the `destinations` value resides). To perform an + all-reduction, pass `value` to `destinations`. + + Returns: + A value mirrored to `destinations`. + """ + # TODO(josh11b): More docstring + _require_cross_replica_context_extended(self) + assert not isinstance(destinations, (list, tuple)) + + # TODO(priyag): Remove this when all callers have been updated. + if isinstance(reduce_op, variable_scope.VariableAggregation): + assert reduce_op in ( + variable_scope.VariableAggregation.SUM, + variable_scope.VariableAggregation.MEAN, + ) + reduce_op = reduce_util.ReduceOp.from_variable_aggregation(reduce_op) + assert (reduce_op == reduce_util.ReduceOp.SUM or + reduce_op == reduce_util.ReduceOp.MEAN) + return self._reduce_to(reduce_op, value, destinations) + + def _reduce_to(self, reduce_op, value, destinations): + raise NotImplementedError("must be implemented in descendants") + + def batch_reduce_to(self, reduce_op, value_destination_pairs): + """Combine multiple `reduce_to` calls into one for faster execution. + + Args: + reduce_op: Reduction type, an instance of `tf.distribute.ReduceOp` enum. + DEPRECATED but still accepted values: + `tf.VariableAggregation.SUM`, + `tf.VariableAggregation.MEAN`, + value_destination_pairs: A sequence of (value, destinations) + pairs. See `reduce_to()` for a description. + + Returns: + A list of mirrored values, one per pair in `value_destination_pairs`. + """ + # TODO(josh11b): More docstring + _require_cross_replica_context_extended(self) + + # TODO(priyag): Remove this when all callers have been updated. + if isinstance(reduce_op, variable_scope.VariableAggregation): + assert reduce_op in [ + variable_scope.VariableAggregation.SUM, + variable_scope.VariableAggregation.MEAN, + ] + reduce_op = reduce_util.ReduceOp.from_variable_aggregation(reduce_op) + return self._batch_reduce_to(reduce_op, value_destination_pairs) + + def _batch_reduce_to(self, reduce_op, value_destination_pairs): + return [ + self.reduce_to(reduce_op, t, destinations=v) + for t, v in value_destination_pairs + ] + + def update(self, var, fn, args=(), kwargs=None, group=True): + """Run `fn` to update `var` using inputs mirrored to the same devices. + + If `var` is mirrored across multiple devices, then this implements + logic like: + + ``` + results = {} + for device, v in var: + with tf.device(device): + # args and kwargs will be unwrapped if they are mirrored. + results[device] = fn(v, *args, **kwargs) + return merged(results) + ``` + + Otherwise this returns `fn(var, *args, **kwargs)` colocated with `var`. + + Neither `args` nor `kwargs` may contain per-replica values. + If they contain mirrored values, they will be unwrapped before + calling `fn`. + + Args: + var: Variable, possibly mirrored to multiple devices, to operate on. + fn: Function to call. Should take the variable as the first argument. + args: Tuple or list. Additional positional arguments to pass to `fn()`. + kwargs: Dict with keyword arguments to pass to `fn()`. + group: Boolean. Defaults to True. If False, the return value will be + unwrapped. + + Returns: + By default, the merged return value of `fn` across all replicas. The + merged result has dependencies to make sure that if it is evaluated at + all, the side effects (updates) will happen on every replica. If instead + "group=False" is specified, this function will return a nest of lists + where each list has an element per replica, and the caller is responsible + for ensuring all elements are executed. + """ + _require_cross_replica_context_extended(self) + if kwargs is None: + kwargs = {} + return self._update(var, fn, args, kwargs, group) + + def _update(self, var, fn, args, kwargs, group): + raise NotImplementedError("must be implemented in descendants") + + def update_non_slot( + self, colocate_with, fn, args=(), kwargs=None, group=True): + """Runs `fn(*args, **kwargs)` on `colocate_with` devices. + + Args: + colocate_with: The return value of `non_slot_devices()`. + fn: Function to execute. + args: Tuple or list. Positional arguments to pass to `fn()`. + kwargs: Dict with keyword arguments to pass to `fn()`. + group: Boolean. Defaults to True. If False, the return value will be + unwrapped. + + Returns: + Return value of `fn`, possibly merged across devices. + """ + _require_cross_replica_context_extended(self) + if kwargs is None: + kwargs = {} + return self._update_non_slot(colocate_with, fn, args, kwargs, group) + + def _update_non_slot(self, colocate_with, fn, args, kwargs, group): + raise NotImplementedError("must be implemented in descendants") + + def _unwrap(self, distributed_value): + raise NotImplementedError("must be implemented in descendants") + + def value_container(self, value): + """Returns the container that this per-replica `value` belongs to. + + Args: + value: A value returned by `call_for_each_replica()` or a variable + created in `scope()`. + + Returns: + A container that `value` belongs to. + If value does not belong to any container (including the case of + container having been destroyed), returns the value itself. + `value in unwrap(value_container(value))` will always be true. + """ + raise NotImplementedError("must be implemented in descendants") + + def _group(self, value, name=None): + """Shortcut for `tf.group(distribution.unwrap(value))`.""" + value = nest.flatten(self._unwrap(value)) + + if len(value) != 1 or name is not None: + return control_flow_ops.group(value, name=name) + # Special handling for the common case of one op. + v, = value + if hasattr(v, "op"): + v = v.op + return v + + @property + def experimental_require_static_shapes(self): + return self._require_static_shapes + + @property + def _num_replicas_in_sync(self): + """Returns number of replicas over which gradients are aggregated.""" + raise NotImplementedError("must be implemented in descendants") + + @property + def worker_devices(self): + """Returns the list of devices used to run `call_for_each_replica()` calls. + """ + # TODO(josh11b): More docstring + raise NotImplementedError("must be implemented in descendants") + + @property + def parameter_devices(self): + """Returns the list of devices used for variable and `update` placement.""" + # TODO(josh11b): More docstring + raise NotImplementedError("must be implemented in descendants") + + def non_slot_devices(self, var_list): + """Device(s) for non-slot variables. + + Create variables on these devices in a + `with colocate_vars_with(non_slot_devices(...)):` block. + Update those using `update_non_slot()`. + + Args: + var_list: The list of variables being optimized, needed with the + default `tf.distribute.Strategy`. + """ + raise NotImplementedError("must be implemented in descendants") + + @property + def experimental_between_graph(self): + """Whether the strategy uses between-graph replication or not. + + This is expected to return a constant value that will not be changed + throughout its life cycle. + """ + raise NotImplementedError("must be implemented in descendants") + + def _configure(self, + session_config=None, + cluster_spec=None, + task_type=None, + task_id=None): + """Configures the strategy class.""" + del session_config, cluster_spec, task_type, task_id + + def _update_config_proto(self, config_proto): + return copy.deepcopy(config_proto) + + @property + def experimental_should_init(self): + """Whether initialization is needed.""" + raise NotImplementedError("must be implemented in descendants") + + @property + def should_checkpoint(self): + """Whether checkpointing is needed.""" + raise NotImplementedError("must be implemented in descendants") + + @property + def should_save_summary(self): + """Whether saving summaries is needed.""" + raise NotImplementedError("must be implemented in descendants") + + +# A note about the difference between the context managers +# `ReplicaContext` (defined here) and `_CurrentDistributionContext` +# (defined above) used by `DistributionStrategy.scope()`: +# +# * a ReplicaContext is only present during a `call_for_each_replica()` +# call (except during a `merge_run` call) and in such a scope it +# will be returned by calls to `get_replica_context()`. Implementers of new +# DistributionStrategy descendants will frequently also need to +# define a descendant of ReplicaContext, and are responsible for +# entering and exiting this context. +# +# * DistributionStrategy.scope() sets up a variable_creator scope that +# changes variable creation calls (e.g. to make mirrored +# variables). This is intended as an outer scope that users enter once +# around their model creation and graph definition. There is no +# anticipated need to define descendants of _CurrentDistributionContext. +# It sets the current DistributionStrategy for purposes of +# `get_strategy()` and `has_strategy()` +# and switches the thread mode to a "cross-replica context". +@tf_export("distribute.ReplicaContext") +class ReplicaContext(object): + """`tf.distribute.Strategy` API when in a replica context. + + To be used inside your replicated step function, such as in a + `tf.distribute.StrategyExtended.call_for_each_replica` call. + """ + + def __init__(self, strategy, replica_id_in_sync_group): + self._distribution_strategy = strategy + self._thread_context = distribution_strategy_context._InReplicaThreadMode( # pylint: disable=protected-access + self) + self._replica_id_in_sync_group = replica_id_in_sync_group + + def __enter__(self): + _push_per_thread_mode(self._thread_context) + + def __exit__(self, exception_type, exception_value, traceback): + _pop_per_thread_mode() + + def merge_call(self, merge_fn, args=(), kwargs=None): + """Merge args across replicas and run `merge_fn` in a cross-replica context. + + This allows communication and coordination when there are multiple calls + to a model function triggered by a call to + `strategy.extended.call_for_each_replica(model_fn, ...)`. + + See `tf.distribute.StrategyExtended.call_for_each_replica` for an + explanation. + + If not inside a distributed scope, this is equivalent to: + + ``` + strategy = tf.distribute.get_strategy() + with cross-replica-context(strategy): + return merge_fn(strategy, *args, **kwargs) + ``` + + Args: + merge_fn: function that joins arguments from threads that are given as + PerReplica. It accepts `tf.distribute.Strategy` object as + the first argument. + args: List or tuple with positional per-thread arguments for `merge_fn`. + kwargs: Dict with keyword per-thread arguments for `merge_fn`. + + Returns: + The return value of `merge_fn`, except for `PerReplica` values which are + unpacked. + """ + require_replica_context(self) + if kwargs is None: + kwargs = {} + return self._merge_call(merge_fn, args, kwargs) + + def _merge_call(self, merge_fn, args, kwargs): + """Default implementation for single replica.""" + _push_per_thread_mode( # thread-local, so not needed with multiple threads + distribution_strategy_context._CrossReplicaThreadMode( # pylint: disable=protected-access + self._distribution_strategy)) + try: + return merge_fn(self._distribution_strategy, *args, **kwargs) + finally: + _pop_per_thread_mode() + + @property + def num_replicas_in_sync(self): + """Returns number of replicas over which gradients are aggregated.""" + return self._distribution_strategy.num_replicas_in_sync + + @property + def replica_id_in_sync_group(self): + """Which replica is being defined, from 0 to `num_replicas_in_sync - 1`.""" + require_replica_context(self) + return self._replica_id_in_sync_group + + @property + @doc_controls.do_not_generate_docs # DEPRECATED, use `strategy` + def distribution_strategy(self): + """DEPRECATED: use `self.stratgey` instead.""" + return self._distribution_strategy + + @property + def strategy(self): + """The current `tf.distribute.Strategy` object.""" + return self._distribution_strategy + + @property + def devices(self): + """The devices this replica is to be executed on, as a list of strings.""" + require_replica_context(self) + return [device_util.current()] + + # TODO(josh11b): Implement `start_all_reduce(method, t)` for efficient + # all-reduce. It would return a function returning the result of reducing `t` + # across all replicas. The caller would wait to call this function until they + # needed the reduce result, allowing an efficient implementation: + # * With eager execution, the reduction could be performed asynchronously + # in the background, not blocking until the result was needed. + # * When constructing a graph, it could batch up all reduction requests up + # to that point that the first result is needed. Most likely this can be + # implemented in terms of `merge_call()` and `batch_reduce_to()`. + +# ------------------------------------------------------------------------------ + + +class _DefaultDistributionStrategy(DistributionStrategy): + """Default `tf.distribute.Strategy` if none is explicitly selected.""" + + def __init__(self): + super(_DefaultDistributionStrategy, self).__init__( + _DefaultDistributionExtended(self)) + + +class _DefaultDistributionExtended(DistributionStrategyExtended): + """Implementation of _DefaultDistributionStrategy.""" + + def _scope(self, strategy): + """Context manager setting a variable creator and `self` as current.""" + if distribution_strategy_context.has_distribution_strategy(): + raise RuntimeError("Must not nest tf.distribute.Strategy scopes.") + + def creator(next_creator, *args, **kwargs): + _require_distribution_strategy_scope_strategy(strategy) + return next_creator(*args, **kwargs) + + return _CurrentDistributionContext( + strategy, variable_scope.variable_creator_scope(creator)) + + def colocate_vars_with(self, colocate_with_variable): + """Does not require `self.scope`.""" + _require_distribution_strategy_scope_extended(self) + return ops.colocate_with(colocate_with_variable) + + def _distribute_dataset(self, dataset_fn): + return self._call_dataset_fn(dataset_fn) + + def _make_dataset_iterator(self, dataset): + return _DefaultDistributionExtended.DefaultInputIterator(dataset) + + def _make_input_fn_iterator(self, + input_fn, + replication_mode=InputReplicationMode.PER_WORKER): + return input_fn(InputContext()).make_initializable_iterator() + + def _broadcast_to(self, tensor, destinations): + if destinations is None: + return tensor + else: + raise NotImplementedError("TODO") + + def _call_for_each_replica(self, fn, args, kwargs): + with ReplicaContext( + self._container_strategy(), + replica_id_in_sync_group=constant_op.constant(0, dtypes.int32)): + return fn(*args, **kwargs) + + def _reduce_to(self, reduce_op, value, destinations): + # TODO(josh11b): Use destinations? + del reduce_op, destinations + return value + + def _update(self, var, fn, args, kwargs, group): + # The implementations of _update() and _update_non_slot() are identical + # except _update() passes `var` as the first argument to `fn()`. + return self._update_non_slot(var, fn, (var,) + tuple(args), kwargs, group) + + def _update_non_slot(self, colocate_with, fn, args, kwargs, should_group): + # TODO(josh11b): Figure out what we should be passing to UpdateContext() + # once that value is used for something. + with ops.colocate_with(colocate_with), UpdateContext(colocate_with): + result = fn(*args, **kwargs) + if should_group: + return result + else: + return nest.map_structure(self._unwrap, result) + + def read_var(self, replica_local_var): + return array_ops.identity(replica_local_var) + + def _unwrap(self, distributed_value): + return [distributed_value] + + def value_container(self, value): + return value + + @property + def _num_replicas_in_sync(self): + return 1 + + @property + def worker_devices(self): + raise RuntimeError("worker_devices() method unsupported by default " + "tf.distribute.Strategy.") + + @property + def parameter_devices(self): + raise RuntimeError("parameter_devices() method unsupported by default " + "tf.distribute.Strategy.") + + def non_slot_devices(self, var_list): + return min(var_list, key=lambda x: x.name) + + # TODO(priyag): This should inherit from `InputIterator`, once dependency + # issues have been resolved. + class DefaultInputIterator(object): + """Default implementation of `InputIterator` for default strategy.""" + + def __init__(self, dataset): + self._dataset = dataset + if eager_context.executing_eagerly(): + self._iterator = dataset.make_one_shot_iterator() + else: + self._iterator = dataset.make_initializable_iterator() + + def get_next(self): + return self._iterator.get_next() + + def initialize(self): + if eager_context.executing_eagerly(): + self._iterator = self._dataset.make_one_shot_iterator() + return [] + else: + return [self._iterator.initializer] + + # TODO(priyag): Delete this once all strategies use global batch size. + @property + def _global_batch_size(self): + return True + + +# ------------------------------------------------------------------------------ +# We haven't yet implemented deserialization for DistributedVariables. +# So here we catch any attempts to deserialize variables +# when using distribution strategies. +# pylint: disable=protected-access +_original_from_proto = resource_variable_ops._from_proto_fn + + +def _from_proto_fn(v, import_scope=None): + if distribution_strategy_context.has_distribution_strategy(): + raise NotImplementedError( + "Deserialization of variables is not yet supported when using a " + "tf.distribute.Strategy.") + else: + return _original_from_proto(v, import_scope=import_scope) + +resource_variable_ops._from_proto_fn = _from_proto_fn +# pylint: enable=protected-access + + +#------------------------------------------------------------------------------- +# Shorthand for some methods from distribution_strategy_context. +_push_per_thread_mode = distribution_strategy_context._push_per_thread_mode # pylint: disable=protected-access +_get_per_thread_mode = distribution_strategy_context._get_per_thread_mode # pylint: disable=protected-access +_pop_per_thread_mode = distribution_strategy_context._pop_per_thread_mode # pylint: disable=protected-access diff --git a/tensorflow/python/training/distribute_test.py b/tensorflow/python/distribute/distribute_lib_test.py similarity index 75% rename from tensorflow/python/training/distribute_test.py rename to tensorflow/python/distribute/distribute_lib_test.py index 0a7bbd56870..d63d1fe3c32 100644 --- a/tensorflow/python/training/distribute_test.py +++ b/tensorflow/python/distribute/distribute_lib_test.py @@ -18,13 +18,15 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function +from tensorflow.python.distribute import distribute_lib +from tensorflow.python.distribute import distribution_strategy_context +from tensorflow.python.framework import constant_op +from tensorflow.python.framework import dtypes from tensorflow.python.ops import variable_scope from tensorflow.python.platform import test -from tensorflow.python.training import distribute -from tensorflow.python.training import distribution_strategy_context -class _TestReplicaContext(distribute.ReplicaContext): +class _TestReplicaContext(distribute_lib.ReplicaContext): def merge_call(self, fn, *args, **kwargs): return kwargs["test_arg"] @@ -38,10 +40,18 @@ def _get_test_variable(name, synchronization, aggregation): } -class _TestStrategy(distribute.DistributionStrategy): +class _TestStrategy(distribute_lib.DistributionStrategy): + + def __init__(self): + super(_TestStrategy, self).__init__(_TestExtended(self)) + + +class _TestExtended(distribute_lib.DistributionStrategyExtended): def _call_for_each_replica(self, fn, args, kwargs): - with _TestReplicaContext(self, replica_id=0): + with _TestReplicaContext( + self._container_strategy(), + replica_id_in_sync_group=constant_op.constant(0, dtypes.int32)): return fn(*args, **kwargs) def _create_variable(self, next_creator, *args, **kwargs): @@ -53,6 +63,7 @@ def _assert_in_default_state(t): t.assertIs(distribution_strategy_context._get_default_replica_context(), distribution_strategy_context.get_replica_context()) t.assertIs(None, distribution_strategy_context.get_cross_replica_context()) + t.assertFalse(distribution_strategy_context.in_cross_replica_context()) t.assertIs(distribution_strategy_context._get_default_distribution_strategy(), distribution_strategy_context.get_distribution_strategy()) t.assertFalse(distribution_strategy_context.has_distribution_strategy()) @@ -69,6 +80,7 @@ class TestStrategyTest(test.TestCase): self.assertTrue(replica_context is not None) self.assertIs(None, distribution_strategy_context.get_cross_replica_context()) + self.assertFalse(distribution_strategy_context.in_cross_replica_context()) self.assertTrue(distribution_strategy_context.has_distribution_strategy()) self.assertIs(dist, distribution_strategy_context.get_distribution_strategy()) @@ -80,9 +92,9 @@ class TestStrategyTest(test.TestCase): variable_scope.variable(1.0, name="bar")) with self.assertRaises(RuntimeError): - dist.call_for_each_replica(run_fn) + dist.extended.call_for_each_replica(run_fn) with dist.scope(): - dist.call_for_each_replica(run_fn) + dist.extended.call_for_each_replica(run_fn) _assert_in_default_state(self) def testScope(self): @@ -92,6 +104,7 @@ class TestStrategyTest(test.TestCase): self.assertIs(None, distribution_strategy_context.get_replica_context()) self.assertIs(dist, distribution_strategy_context.get_cross_replica_context()) + self.assertTrue(distribution_strategy_context.in_cross_replica_context()) self.assertTrue(distribution_strategy_context.has_distribution_strategy()) self.assertIs(dist, distribution_strategy_context.get_distribution_strategy()) @@ -131,6 +144,7 @@ class DefaultDistributionStrategyTest(test.TestCase): self.assertIs(None, distribution_strategy_context.get_replica_context()) self.assertIs(dist, distribution_strategy_context.get_cross_replica_context()) + self.assertTrue(distribution_strategy_context.in_cross_replica_context()) self.assertIs(dist, distribution_strategy_context.get_distribution_strategy()) self.assertFalse( @@ -140,9 +154,26 @@ class DefaultDistributionStrategyTest(test.TestCase): replica_ctx = distribution_strategy_context.get_replica_context() self.assertIs(distribution_strategy_context._get_default_replica_context(), replica_ctx) - self.assertEqual("foo_bar", replica_ctx.merge_call(merge_fn, "bar")) + self.assertEqual("foo_bar", replica_ctx.merge_call(merge_fn, args=("bar",))) _assert_in_default_state(self) +class InputContextTest(test.TestCase): + + def testProperties(self): + input_context = distribute_lib.InputContext( + num_input_pipelines=2, input_pipeline_id=1, num_replicas_in_sync=6) + self.assertEqual(6, input_context.num_replicas_in_sync) + self.assertEqual(1, input_context.input_pipeline_id) + self.assertEqual(2, input_context.num_input_pipelines) + + def testPerReplicaBatchSize(self): + input_context = distribute_lib.InputContext( + num_input_pipelines=2, input_pipeline_id=1, num_replicas_in_sync=6) + self.assertEqual(2, input_context.get_per_replica_batch_size(12)) + with self.assertRaises(ValueError): + input_context.get_per_replica_batch_size(13) + + if __name__ == "__main__": test.main() diff --git a/tensorflow/python/distribute/distribution_strategy_context.py b/tensorflow/python/distribute/distribution_strategy_context.py new file mode 100644 index 00000000000..78e096e2867 --- /dev/null +++ b/tensorflow/python/distribute/distribution_strategy_context.py @@ -0,0 +1,236 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Utility to get distribution strategy related contexts.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from tensorflow.python.framework import ops +from tensorflow.python.util.lazy_loader import LazyLoader +from tensorflow.python.util.tf_export import tf_export + + +# There is a circular dependency between this and `distribute` module. So we +# load it lazily to workaround this. +distribute_lib = LazyLoader( + "distribute_lib", globals(), + "tensorflow.python.distribute.distribute_lib") + +# ------------------------------------------------------------------------------ +# Internal API for setting the current thread mode as being either in a +# replica or cross-replica context for a particular distribution strategy. + + +class _ThreadMode(object): + + def __init__(self, dist, cross, replica): + self.distribution_strategy = dist + self.cross_replica_context = cross + self.replica_context = replica + + +class _CrossReplicaThreadMode(_ThreadMode): + + def __init__(self, distribution_strategy): + _ThreadMode.__init__( + self, distribution_strategy, distribution_strategy, None) + + +class _InReplicaThreadMode(_ThreadMode): + + def __init__(self, replica_ctx): + _ThreadMode.__init__( + self, replica_ctx.distribution_strategy, None, replica_ctx) + + +def _push_per_thread_mode(context): + ops.get_default_graph()._distribution_strategy_stack.append(context) # pylint: disable=protected-access + + +def _pop_per_thread_mode(): + ops.get_default_graph()._distribution_strategy_stack.pop(-1) # pylint: disable=protected-access + + +class _DefaultReplicaThreadMode(_ThreadMode): + """Type of default value returned by `_get_per_thread_mode()`. + + Used when the thread-local stack is empty. + """ + + def __init__(self): + _ThreadMode.__init__(self, _get_default_distribution_strategy(), None, + _get_default_replica_context()) + + +def _get_per_thread_mode(): + try: + return ops.get_default_graph()._distribution_strategy_stack[-1] # pylint: disable=protected-access + except (AttributeError, IndexError): + return _get_default_replica_mode() + + +# ------------------------------------------------------------------------------ +# Public API for accessing the current thread mode + + +@tf_export("distribute.get_replica_context") +def get_replica_context(): + """Returns the current `tf.distribute.ReplicaContext` or `None`. + + Returns `None` if in a cross-replica context. + + Note that execution: + + 1. starts in the default (single-replica) replica context (this function + will return the default `ReplicaContext` object); + 2. switches to cross-replica context (in which case this will return + `None`) when entering a `with tf.distribute.Strategy.scope():` block; + 3. switches to a (non-default) replica context inside + `extended.call_for_each_replica(fn, ...)`; + 4. if `fn` calls `get_replica_context().merge_call(merge_fn, ...)`, then + inside `merge_fn` you are back in the cross-replica context (and again + this function will return `None`). + + Note that you can also go directly from step 1 to 4 to switch to a + cross-replica context for the default `tf.distribute.Strategy`. You may + also switch from the cross-replica context of 4 to a replica context by + calling `extended.call_for_each_replica()`, jumping back to step 3. + + Most `tf.distribute.Strategy` methods may only be executed in + a cross-replica context, in a replica context you should use the + `ReplicaContext` API instead. + + Returns: + The current `ReplicaContext` object when in a replica context scope, + else `None`. + + Within a particular block, exactly one of these two things will be true: + + * `get_replica_context()` returns non-`None`, or + * `tf.distribute.is_cross_replica_context()` returns True. + """ + return _get_per_thread_mode().replica_context + + +def get_cross_replica_context(): + """Returns the current tf.distribute.Strategy if in a cross-replica context. + + DEPRECATED: Please use `in_cross_replica_context()` and + `get_distribution_strategy()` instead. + + Note that execution: + + 1. starts in the default (single-replica) replica context; + 2. switches to cross-replica context when entering a + `with tf.distribute.Strategy.scope():` block; + 3. switches to a (non-default) replica context inside + `call_for_each_replica(fn, ...)`; + 4. if `fn` calls `get_replica_context()->merge_call(merge_fn, ...)`, then + inside `merge_fn` you are back in the cross-replica context. + + Note that you can also go directly from step 1 to 4 to switch to a + cross-replica context for the default `tf.distribute.Strategy`. You may + also switch from the cross-replica context of 4 to a replica context by + calling `call_for_each_replica()`, jumping back to step 3. + + Most `tf.distribute.Strategy` methods may only be executed in + a cross-replica context. + + Returns: + Returns the current `tf.distribute.Strategy` object in a cross-replica + context, or `None`. + + Exactly one of `get_replica_context()` and `get_cross_replica_context()` + will return `None` in a particular block. + """ + return _get_per_thread_mode().cross_replica_context + + +@tf_export("distribute.in_cross_replica_context") +def in_cross_replica_context(): + """Returns True if in a cross-replica context. + + See `tf.distribute.get_replica_context` for details. + + Returns: + True if in a cross-replica context (`get_replica_context()` returns + `None`), or False if in a replica context (`get_replica_context()` returns + non-`None`). + """ + return _get_per_thread_mode().cross_replica_context is not None + + +@tf_export("distribute.get_strategy") +def get_distribution_strategy(): + """Returns the current `tf.distribute.Strategy` object. + + Typically only used in a cross-replica context: + + ``` + if tf.distribute.in_cross_replica_context(): + strategy = tf.distribute.get_strategy() + ... + ``` + + Returns: + A `tf.distribute.Strategy` object. Inside a + `with distribution_strategy.scope()` block, it returns + `distribution_strategy`, otherwise it returns the default + (single-replica) `tf.distribute.Strategy` object. + """ + return _get_per_thread_mode().distribution_strategy + + +@tf_export("distribute.has_strategy") +def has_distribution_strategy(): + """Return if there is a current non-default `tf.distribute.Strategy`. + + Returns: + True if inside a `with strategy.scope():`. + """ + return get_distribution_strategy() is not _get_default_distribution_strategy() + + +# ------------------------------------------------------------------------------ +# Defaults that are used when no distribution strategy is explicitly created. +# We create them lazily in a function so that we can workaround the circular +# dependency on distribute_lib. See lazy loader at the top of this file. + +_defaults = { + "distribution_strategy": None, + "replica_context": None, + "replica_mode": None +} + + +def _get_default_distribution_strategy(): + if _defaults["distribution_strategy"] is None: + _defaults["distribution_strategy"] = ( + distribute_lib._DefaultDistributionStrategy()) # pylint: disable=protected-access + return _defaults["distribution_strategy"] + + +def _get_default_replica_context(): + if _defaults["replica_context"] is None: + _defaults["replica_context"] = distribute_lib.ReplicaContext( + _get_default_distribution_strategy(), replica_id_in_sync_group=0) + return _defaults["replica_context"] + + +def _get_default_replica_mode(): + if _defaults["replica_mode"] is None: + _defaults["replica_mode"] = _DefaultReplicaThreadMode() + return _defaults["replica_mode"] diff --git a/tensorflow/python/distribute/estimator_training.py b/tensorflow/python/distribute/estimator_training.py index 227b00fb3e5..549fa8fb8aa 100644 --- a/tensorflow/python/distribute/estimator_training.py +++ b/tensorflow/python/distribute/estimator_training.py @@ -308,7 +308,7 @@ def estimator_train(estimator, train_distributed_fn, hooks): raise ValueError('Only `STANDALONE_CLIENT` mode is supported when you call ' '`estimator.train`') - if estimator._config._train_distribute.between_graph: + if estimator._config._train_distribute.extended.experimental_between_graph: # TODO(yuefengz): remove this limitation once we figure out how to merge # return values from `_worker_fn`s. raise ValueError('`Estimator.train` API is not supported for %s with ' @@ -356,7 +356,7 @@ def estimator_evaluate(estimator, evaluate_distributed_fn, hooks): raise ValueError('Only `STANDALONE_CLIENT` mode is supported when you call ' '`Estimator.train`') - if estimator._config._eval_distribute.between_graph: + if estimator._config._eval_distribute.extended.experimental_between_graph: # TODO(yuefengz): remove this limitation once we figure out how to merge # return values from `_worker_fn`s. raise ValueError('`Estimator.evaluate` API is not supported for %s with ' diff --git a/tensorflow/contrib/distribute/python/input_ops.py b/tensorflow/python/distribute/input_ops.py similarity index 89% rename from tensorflow/contrib/distribute/python/input_ops.py rename to tensorflow/python/distribute/input_ops.py index ac1ccd64b32..2ded209701e 100644 --- a/tensorflow/contrib/distribute/python/input_ops.py +++ b/tensorflow/python/distribute/input_ops.py @@ -18,6 +18,7 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function +from tensorflow.python.data.experimental.ops import filter_for_shard_ops from tensorflow.python.data.ops import dataset_ops from tensorflow.python.data.ops import readers from tensorflow.python.data.util import nest @@ -41,7 +42,8 @@ def auto_shard_dataset(dataset, num_shards, index): dataset: A `tf.data.Dataset` instance, typically the result of a bunch of dataset transformations. num_shards: A `tf.int64` scalar `tf.Tensor`, representing the number of - shards operating in parallel. Same usage as in `Dataset.shard`. + shards operating in parallel. Same usage as in + `tf.data.experimental.filter_for_shard`. index: A `tf.int64` scalar `tf.Tensor`, representing the worker index. Same usage as in `Dataset.shard`. @@ -74,13 +76,15 @@ def auto_shard_dataset(dataset, num_shards, index): # constructor. Eventually we will change all cases to clone datasets # instead of updating in-place. return dataset._clone( - filenames=dataset._filenames.shard(num_shards, index)) + filenames=dataset._filenames.apply( + filter_for_shard_ops.filter_for_shard(num_shards, index))) elif isinstance(dataset, dataset_ops.RangeDataset): - return dataset.shard(num_shards, index) + return dataset.apply( + filter_for_shard_ops.filter_for_shard(num_shards, index)) elif hasattr(dataset, "_map_func"): # TODO(priyag): Make this check more robust by enforcing some common # property on all map/flatmap/interleave datasets. - map_func_def = dataset._map_func.definition + map_func_def = dataset._map_func.function.definition for node in map_func_def.node_def: if node.op in _READER_DATASET_OPS: found_reader_op = True @@ -102,6 +106,11 @@ def auto_shard_dataset(dataset, num_shards, index): dataset._input_dataset, found_reader_op) return dataset + if isinstance(dataset, dataset_ops.DatasetV1Adapter): + dataset._dataset = _auto_shard_impl( + dataset._dataset, found_reader_op) + return dataset + # TODO(priyag): Make _input_dataset(s) a common property of all datasets to # make this check more robust. if hasattr(dataset, "_input_dataset"): @@ -137,6 +146,7 @@ def auto_shard_dataset(dataset, num_shards, index): # TODO(priyag): This will shard the filenames before any shuffling of the # filename dataset. It might be desirable to shard after shuffling # filenames? If so, how do we achieve that? - return dataset.shard(num_shards, index) + return dataset.apply( + filter_for_shard_ops.filter_for_shard(num_shards, index)) return _auto_shard_impl(dataset=dataset, found_reader_op=False) diff --git a/tensorflow/contrib/distribute/python/input_ops_test.py b/tensorflow/python/distribute/input_ops_test.py similarity index 89% rename from tensorflow/contrib/distribute/python/input_ops_test.py rename to tensorflow/python/distribute/input_ops_test.py index 559de97bb1f..dcf946ba477 100644 --- a/tensorflow/contrib/distribute/python/input_ops_test.py +++ b/tensorflow/python/distribute/input_ops_test.py @@ -20,10 +20,11 @@ from __future__ import print_function import os -from tensorflow.contrib.distribute.python import input_ops from tensorflow.python.data.ops import dataset_ops from tensorflow.python.data.ops import readers +from tensorflow.python.distribute import input_ops from tensorflow.python.framework import errors +from tensorflow.python.framework import test_util from tensorflow.python.lib.io import python_io from tensorflow.python.platform import test from tensorflow.python.util import compat @@ -92,10 +93,11 @@ class AutoShardDatasetTest(test.TestCase): with self.cached_session() as sess: for f in range(self._shard_index, self._num_files, self._num_shards): for r in range(self._num_records): - self.assertAllEqual(record_fn(r, f), sess.run(next_element)) + self.assertAllEqual(record_fn(r, f), self.evaluate(next_element)) with self.assertRaises(errors.OutOfRangeError): - sess.run(next_element) + self.evaluate(next_element) + @test_util.run_deprecated_v1 def testTFRecordDataset(self): dataset = readers.TFRecordDataset(self._createTFRecordFiles()) dataset = input_ops.auto_shard_dataset( @@ -103,6 +105,7 @@ class AutoShardDatasetTest(test.TestCase): self._verifySimpleShardingOutput(dataset, self._record) + @test_util.run_deprecated_v1 def testFlatMap(self): dataset = dataset_ops.Dataset.from_tensor_slices( self._createTFRecordFiles()) @@ -112,6 +115,7 @@ class AutoShardDatasetTest(test.TestCase): self._verifySimpleShardingOutput(dataset, self._record) + @test_util.run_deprecated_v1 def testInterleave(self): dataset = dataset_ops.Dataset.from_tensor_slices( self._createTFRecordFiles()) @@ -124,9 +128,10 @@ class AutoShardDatasetTest(test.TestCase): # contain records in order of files. self._verifySimpleShardingOutput(dataset, self._record) + @test_util.run_deprecated_v1 def testListfiles(self): filenames = self._createTFRecordFiles() - file_pattern = filenames[0].rsplit("/", 1)[0] + "/tf_record.*.txt" + file_pattern = filenames[0].rsplit(os.sep, 1)[0] + "/tf_record.*.txt" dataset = dataset_ops.Dataset.list_files(file_pattern, shuffle=False) dataset = dataset.flat_map(readers.TFRecordDataset) dataset = input_ops.auto_shard_dataset( @@ -138,12 +143,13 @@ class AutoShardDatasetTest(test.TestCase): actual, expected = [], [] for f in range(self._shard_index, self._num_files, self._num_shards): for r in range(self._num_records): - actual.append(sess.run(next_element)) + actual.append(self.evaluate(next_element)) expected.append(self._record(r, f)) with self.assertRaises(errors.OutOfRangeError): - sess.run(next_element) + self.evaluate(next_element) self.assertAllEqual(expected, actual) + @test_util.run_deprecated_v1 def testComplexPipeline(self): # Setup a complex input pipeline. batch_size = 2 @@ -171,9 +177,9 @@ class AutoShardDatasetTest(test.TestCase): num_iterations = (self._num_files * self._num_records * num_epochs) // ( self._num_shards * batch_size) for _ in range(num_iterations): - actual.extend(sess.run(next_element)) + actual.extend(self.evaluate(next_element)) with self.assertRaises(errors.OutOfRangeError): - sess.run(next_element) + self.evaluate(next_element) expected = [] for f in range(0, self._num_files, self._num_shards): @@ -183,6 +189,7 @@ class AutoShardDatasetTest(test.TestCase): self.assertAllEqual(sorted(expected), sorted(actual)) + @test_util.run_deprecated_v1 def testZip(self): dataset1 = readers.TFRecordDataset(self._createTFRecordFiles()) dataset2 = readers.TextLineDataset(self._createTextFiles()) @@ -193,6 +200,7 @@ class AutoShardDatasetTest(test.TestCase): record_fn = lambda r, f: (self._record(r, f), self._text_line(r, f)) self._verifySimpleShardingOutput(dataset, record_fn) + @test_util.run_deprecated_v1 def testConcat(self): dataset1 = readers.TFRecordDataset(self._createTFRecordFiles()) dataset2 = readers.TextLineDataset(self._createTextFiles()) @@ -205,13 +213,15 @@ class AutoShardDatasetTest(test.TestCase): with self.cached_session() as sess: for f in range(self._shard_index, self._num_files, self._num_shards): for r in range(self._num_records): - self.assertAllEqual(self._record(r, f), sess.run(next_element)) + self.assertAllEqual(self._record(r, f), self.evaluate(next_element)) for f in range(self._shard_index, self._num_files, self._num_shards): for r in range(self._num_records): - self.assertAllEqual(self._text_line(r, f), sess.run(next_element)) + self.assertAllEqual( + self._text_line(r, f), self.evaluate(next_element)) with self.assertRaises(errors.OutOfRangeError): - sess.run(next_element) + self.evaluate(next_element) + @test_util.run_deprecated_v1 def testTextLineReader(self): dataset = readers.TextLineDataset(self._createTextFiles()) dataset = input_ops.auto_shard_dataset( @@ -219,6 +229,7 @@ class AutoShardDatasetTest(test.TestCase): self._verifySimpleShardingOutput(dataset, self._text_line) + @test_util.run_deprecated_v1 def testTextLineReaderWithFlatMap(self): dataset = dataset_ops.Dataset.from_tensor_slices(self._createTextFiles()) dataset = dataset.flat_map(readers.TextLineDataset) @@ -227,6 +238,7 @@ class AutoShardDatasetTest(test.TestCase): self._verifySimpleShardingOutput(dataset, self._text_line) + @test_util.run_deprecated_v1 def testFixedLengthReader(self): dataset = readers.FixedLengthRecordDataset( self._createFixedLengthRecordFiles(), self._record_bytes) @@ -235,6 +247,7 @@ class AutoShardDatasetTest(test.TestCase): self._verifySimpleShardingOutput(dataset, self._fixed_length_record) + @test_util.run_deprecated_v1 def testFixedLengthReaderWithFlatMap(self): dataset = dataset_ops.Dataset.from_tensor_slices( self._createFixedLengthRecordFiles()) diff --git a/tensorflow/python/distribute/mirrored_strategy.py b/tensorflow/python/distribute/mirrored_strategy.py new file mode 100644 index 00000000000..0775920a7d6 --- /dev/null +++ b/tensorflow/python/distribute/mirrored_strategy.py @@ -0,0 +1,908 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Class MirroredStrategy implementing DistributionStrategy.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import contextlib +import copy +import functools +import threading + +from tensorflow.python import pywrap_tensorflow +from tensorflow.python.distribute import cross_device_ops as cross_device_ops_lib +from tensorflow.python.distribute import device_util +from tensorflow.python.distribute import distribute_lib +from tensorflow.python.distribute import multi_worker_util +from tensorflow.python.distribute import reduce_util +from tensorflow.python.distribute import shared_variable_creator +from tensorflow.python.distribute import values +from tensorflow.python.eager import context +from tensorflow.python.eager import tape +from tensorflow.python.framework import constant_op +from tensorflow.python.framework import device as tf_device +from tensorflow.python.framework import dtypes +from tensorflow.python.framework import ops +from tensorflow.python.framework import tensor_util +from tensorflow.python.ops import array_ops +from tensorflow.python.ops import control_flow_ops +from tensorflow.python.ops import variable_scope +from tensorflow.python.training import coordinator +from tensorflow.python.util import nest + + +# TODO(josh11b): Replace asserts in this file with if ...: raise ... + + +@contextlib.contextmanager +def _enter_graph(g): + if context.executing_eagerly(): + with g.as_default(), context.eager_mode(): + yield + else: + with g.as_default(): + yield + + +def _cpu_device(device): + cpu_device = tf_device.DeviceSpec.from_string(device) + cpu_device.merge_from(tf_device.DeviceSpec(device_type="CPU", device_index=0)) + return cpu_device.to_string() + + +class _RequestedStop(Exception): # pylint: disable=g-bad-exception-name + pass + + +# _call_for_each_replica and _reduce_non_distributed_value are not members of +# MirroredStrategy so that they are generally not allowed to use anything +# specific to MirroredStrategy and thus can be shared with other distribution +# strategies. + + +# TODO(yuefengz): maybe create a common class for those who need to call this +# _call_for_each_replica. +def _call_for_each_replica(distribution, fn, args, kwargs): + """Run `fn` in separate threads, once per replica/worker device. + + Args: + distribution: the DistributionStrategy object. + fn: function to run (will be run once per device, each in its own thread). + args: positional arguments for `fn` + kwargs: keyword arguments for `fn`. + + Returns: + Merged return value of `fn` across all replicas. + + Raises: + RuntimeError: If fn() calls get_replica_context().merge_call() a different + number of times from the available devices. + """ + # TODO(josh11b): Add this option once we add synchronization to variable + # creation. Until then, this is pretty unsafe to use. + run_concurrently = False + if not context.executing_eagerly(): + # Needed for per-thread device, etc. contexts in graph mode. + ops.get_default_graph().switch_to_thread_local() + + coord = coordinator.Coordinator(clean_stop_exception_types=(_RequestedStop,)) + + shared_variable_store = {} + + # TODO(isaprykin): Create these threads once instead of during every run() + # call. + threads = [] + for index, d in enumerate(distribution.extended.worker_devices): + variable_creator_fn = shared_variable_creator.make_fn( + shared_variable_store, index) + t = MirroredExtended._MirroredReplicaThread( # pylint: disable=protected-access + distribution, coord, d, variable_creator_fn, fn, + *values.select_device(d, args), **values.select_device(d, kwargs)) + threads.append(t) + + for t in threads: + t.start() + + # When `fn` starts `should_run` event is set on _MirroredReplicaThread + # (`MRT`) threads. The execution waits until + # `MRT.has_paused` is set, which indicates that either `fn` is + # complete or a `get_replica_context().merge_call()` is called. If `fn` is + # complete, then `MRT.done` is set to True. Otherwise, arguments + # of `get_replica_context().merge_call` from all paused threads are grouped + # and the `merge_fn` is performed. Results of the + # `get_replica_context().merge_call` are then set to `MRT.merge_result`. + # Each such `get_replica_context().merge_call` call returns the + # `MRT.merge_result` for that thread when `MRT.should_run` event + # is reset again. Execution of `fn` resumes. + + try: + with coord.stop_on_exception(): + all_done = False + while not all_done and not coord.should_stop(): + done = [] + if run_concurrently: + for t in threads: + t.should_run.set() + for t in threads: + t.has_paused.wait() + t.has_paused.clear() + if coord.should_stop(): + return None + done.append(t.done) + else: + for t in threads: + t.should_run.set() + t.has_paused.wait() + t.has_paused.clear() + if coord.should_stop(): + return None + done.append(t.done) + if coord.should_stop(): + return None + all_done = all(done) + if not all_done: + if any(done): + raise RuntimeError("Some replicas made a different number of " + "replica_context().merge_call() calls.") + # get_replica_context().merge_call() case + merge_args = values.regroup({t.device: t.merge_args for t in threads}) + merge_kwargs = values.regroup( + {t.device: t.merge_kwargs for t in threads}) + # We capture the name_scope of the MRT when we call merge_fn + # to ensure that if we have opened a name scope in the MRT, + # it will be respected when executing the merge function. We only + # capture the name_scope from the first MRT and assume it is + # the same for all other MRTs. + mtt_captured_name_scope = threads[0].captured_name_scope + with ops.name_scope(mtt_captured_name_scope): + merge_result = threads[0].merge_fn(distribution, *merge_args, + **merge_kwargs) + for t in threads: + t.merge_result = values.select_device(t.device, merge_result) + finally: + for t in threads: + t.should_run.set() + coord.join(threads) + + return values.regroup({t.device: t.main_result for t in threads}) + + +def _reduce_non_distributed_value(extended, reduce_op, value, destinations): + """Reduce a non-DistributedValue `value` to `destinations`.""" + if isinstance(value, values.DistributedValues): + raise ValueError("You are passing a `DistributedValue` to " + "`_reduce_non_distributed_value`, which is not allowed.") + + # If the same value is present on all replicas then the PerReplica value will + # be a single value. We also handle the case when `value` is a single value + # and equal to 0. + if value == 0: + return 0 + # If there is only a single value and the reduce op is MEAN, + # that value should be on all destinations. + if reduce_op == reduce_util.ReduceOp.MEAN: + return value + + cross_device_ops_lib.validate_destinations(destinations) + # We do not support a reduce op of SUM if the value is the same across + # all replicas. We call this as part of assign functions for MirroredVariables + # and summing up identical values across replicas is not clearly defined. + if (len(extended.worker_devices) != 1 or + not cross_device_ops_lib.check_destinations(destinations)): + raise ValueError("A non-DistributedValues value %s cannot be reduced with " + "the given reduce op %s." % (value, reduce_op)) + # TODO(anjalisridhar): Moves these methods to a device utility file? + devices = cross_device_ops_lib.get_devices_from(destinations) + if len(devices) == 1: + with ops.device(devices[0]): + return array_ops.identity(value) + else: + value_updates = {} + for d in devices: + with ops.device(d): + value_updates[d] = array_ops.identity(value) + return values.Mirrored(value_updates) + + +def _create_mirrored_variable(devices, real_mirrored_creator, *args, **kwargs): # pylint: disable=g-missing-docstring + # Figure out what collections this variable should be added to. + # We'll add the MirroredVariable to those collections instead. + collections = kwargs.pop("collections", None) + if collections is None: + collections = [ops.GraphKeys.GLOBAL_VARIABLES] + kwargs["collections"] = [] + + # Get synchronization value + synchronization = kwargs.get("synchronization", + variable_scope.VariableSynchronization.ON_WRITE) + if synchronization == variable_scope.VariableSynchronization.NONE: + raise ValueError("`NONE` variable synchronization mode is not " + "supported with `Mirrored` distribution strategy. Please" + " change the `synchronization` for variable: " + + kwargs["name"]) + elif synchronization == variable_scope.VariableSynchronization.ON_READ: + # Variables that are to be synced on read are replica local. + is_replica_local = True + kwargs["trainable"] = False + elif (synchronization == variable_scope.VariableSynchronization.ON_WRITE or + synchronization == variable_scope.VariableSynchronization.AUTO): + # `AUTO` synchronization for `MirroredStrategy` is `ON_WRITE`. + is_replica_local = False + else: + raise ValueError("Invalid variable synchronization mode: " + + synchronization + " for variable: " + kwargs["name"]) + + # Get aggregation value + aggregation = kwargs.pop("aggregation", + variable_scope.VariableAggregation.NONE) + if aggregation not in ( + variable_scope.VariableAggregation.NONE, + variable_scope.VariableAggregation.SUM, + variable_scope.VariableAggregation.MEAN, + variable_scope.VariableAggregation.ONLY_FIRST_REPLICA + ): + raise ValueError("Invalid variable aggregation mode: " + aggregation + + " for variable: " + kwargs["name"]) + + # Ignore user-specified caching device, not needed for mirrored variables. + kwargs.pop("caching_device", None) + + # TODO(josh11b,apassos): It would be better if variable initialization + # was never recorded on the tape instead of having to do this manually + # here. + with tape.stop_recording(): + index = real_mirrored_creator(devices, *args, **kwargs) + + if is_replica_local: + result = values.ReplicaLocalVariable( + index, index[devices[0]], aggregation) + else: + result = values.MirroredVariable(index, index[devices[0]], aggregation) + + # Add the wrapped variable to the requested collections. + # The handling of eager mode and the global step matches + # ResourceVariable._init_from_args(). + if not context.executing_eagerly(): + g = ops.get_default_graph() + # If "trainable" is True, next_creator() will add the member variables + # to the TRAINABLE_VARIABLES collection, so we manually remove + # them and replace with the MirroredVariable. We can't set + # "trainable" to False for next_creator() since that causes functions + # like implicit_gradients to skip those variables. + if kwargs.get("trainable", True): + collections.append(ops.GraphKeys.TRAINABLE_VARIABLES) + l = g.get_collection_ref(ops.GraphKeys.TRAINABLE_VARIABLES) + for v in index.values(): + if v in l: + l.remove(v) + g.add_to_collections(collections, result) + elif ops.GraphKeys.GLOBAL_STEP in collections: + ops.add_to_collections(ops.GraphKeys.GLOBAL_STEP, result) + + return result + + +def _is_device_list_local(devices): + """Checks whether the devices list is for local or multi-worker. + + Args: + devices: a list of device strings, either local for remote devices. + + Returns: + a boolean indicating whether these device strings are for local or for + remote. + + Raises: + ValueError: if device strings are not consistent. + """ + all_local = None + for d in devices: + d_spec = tf_device.DeviceSpec().parse_from_string(d) + is_local = d_spec.job in (None, "localhost") + + if all_local is None: # Determine all_local from first device. + all_local = is_local + + if all_local: + if not is_local: + raise ValueError("Local device string cannot have job specified other " + "than 'localhost'") + else: + if is_local: + raise ValueError("Remote device string must have job specified.") + if d_spec.task is None: + raise ValueError("Remote device string must have task specified.") + return all_local + + +def _cluster_spec_to_device_list(cluster_spec, num_gpus_per_worker): + """Returns a device list given a cluster spec.""" + cluster_spec = multi_worker_util.normalize_cluster_spec(cluster_spec) + devices = [] + for task_type in ("chief", "worker"): + for task_id in range(len(cluster_spec.as_dict().get(task_type, []))): + if num_gpus_per_worker is 0: + devices.append("/job:%s/task:%d" % (task_type, task_id)) + else: + devices.extend([ + "/job:%s/task:%d/device:GPU:%i" % (task_type, task_id, gpu_id) + for gpu_id in range(num_gpus_per_worker) + ]) + return devices + + +def _group_device_list(devices): + """Groups the devices list by task_type and task_id. + + Args: + devices: a list of device strings for remote devices. + + Returns: + a dict of list of device strings mapping from task_type to a list of devices + for the task_type in the asceding order of task_id. + """ + assert not _is_device_list_local(devices) + device_dict = {} + + for d in devices: + d_spec = tf_device.DeviceSpec().parse_from_string(d) + + # Create an entry for the task_type. + if d_spec.job not in device_dict: + device_dict[d_spec.job] = [] + + # Fill the device list for task_type until it covers the task_id. + while len(device_dict[d_spec.job]) <= d_spec.task: + device_dict[d_spec.job].append([]) + + device_dict[d_spec.job][d_spec.task].append(d) + + return device_dict + + +def _infer_num_gpus_per_worker(devices): + """Infers the number of GPUs on each worker. + + Currently to make multi-worker cross device ops work, we need all workers to + have the same number of GPUs. + + Args: + devices: a list of device strings, can be either local devices or remote + devices. + + Returns: + number of GPUs per worker. + + Raises: + ValueError if workers have different number of GPUs or GPU indices are not + consecutive and starting from 0. + """ + if _is_device_list_local(devices): + return len([d for d in devices if "GPU" in d.upper()]) + else: + device_dict = _group_device_list(devices) + num_gpus = None + for _, devices_in_task in device_dict.items(): + for device_in_task in devices_in_task: + if num_gpus is None: + num_gpus = len([d for d in device_in_task if "GPU" in d.upper()]) + + # Verify other workers have the same number of GPUs. + elif ( + num_gpus != len([d for d in device_in_task if "GPU" in d.upper()])): + raise ValueError("All workers should have the same number of GPUs.") + + for d in device_in_task: + d_spec = tf_device.DeviceSpec().parse_from_string(d) + if (d_spec.device_type.upper() == "GPU" and + d_spec.device_index >= num_gpus): + raise ValueError("Device_index on a worker should be consecutive " + "and start from 0.") + return num_gpus + + +def all_local_devices(num_gpus=None): + if num_gpus is None: + num_gpus = context.num_gpus() + return (tuple("/device:GPU:%d" % i for i in range(num_gpus)) or + ("/device:CPU:0",)) + + +class MirroredStrategy(distribute_lib.DistributionStrategy): + """Mirrors vars to distribute across multiple devices and machines. + + This strategy uses one replica per device and sync replication for its + multi-GPU version. + + The multi-worker version will be added in the fture. + + Args: + devices: a list of device strings. + cross_device_ops: optional, a descedant of `CrossDeviceOps`. If this is not + set, nccl will be use by default. + """ + + def __init__(self, devices=None, cross_device_ops=None): + extended = MirroredExtended( + self, devices=devices, cross_device_ops=cross_device_ops) + super(MirroredStrategy, self).__init__(extended) + + +class MirroredExtended(distribute_lib.DistributionStrategyExtended): + """Implementation of MirroredStrategy.""" + + def __init__(self, container_strategy, devices=None, cross_device_ops=None): + super(MirroredExtended, self).__init__(container_strategy) + if devices is None: + devices = all_local_devices() + if not devices: + raise ValueError("Got an empty `devices` list. Please make sure the " + "`devices` you pass in is not empty.") + self._cross_device_ops = cross_device_ops + self._initialize_strategy(devices) + + def _initialize_strategy(self, devices): + # The _initialize_strategy method is intended to be used by distribute + # coordinator as well. + if _is_device_list_local(devices): + self._initialize_local(devices) + else: + self._initialize_multi_worker(devices) + + def _initialize_local(self, devices): + """Initializes the object for local training.""" + self._local_mode = True + assert devices, "Must specify at least one device." + assert len(set(devices)) == len(devices), ( + "No duplicates allowed in `devices` argument.") + # TODO(josh11b): Require at least 2 devices? + self._devices = [device_util.resolve(d) for d in devices] + self._canonical_device_set = set(self._devices) + self._device_index = values.PerReplica( + {d: i for i, d in enumerate(devices)}) + + self._inferred_cross_device_ops = cross_device_ops_lib.choose_the_best( + devices) + + def _initialize_multi_worker(self, devices): + """Initializes the object for multi-worker training.""" + self._local_mode = False + + assert devices, "Must specify at least one device." + assert len(set(devices)) == len(devices), ( + "No duplicates allowed in `devices` argument.") + # TODO(josh11b): Require at least 2 devices? + self._devices = [device_util.resolve(d) for d in devices] + self._canonical_device_set = set(self._devices) + self._device_index = values.PerReplica( + {d: i for i, d in enumerate(devices)}) + + device_dict = _group_device_list(devices) + self._workers = [] + self._worker_devices = [] + for job in ["chief", "worker"]: + for task in range(len(device_dict.get(job, []))): + worker = "/job:%s/task:%d" % (job, task) + self._workers.append(worker) + self._worker_devices.append((worker, device_dict[job][task])) + + # Setting `_default_device` will add a device scope in the + # distribution.scope. We set the default device to the first worker. When + # users specify device under distribution.scope by + # with tf.device("/cpu:0"): + # ... + # their ops will end up on the cpu device of its first worker, e.g. + # "/job:worker/task:0/device:CPU:0". Note this is not used in replica mode. + self._default_device = self._workers[0] + + self._inferred_cross_device_ops = cross_device_ops_lib.MultiWorkerAllReduce( + self._workers, _infer_num_gpus_per_worker(self._devices)) + + def _create_variable(self, next_creator, *args, **kwargs): + """Create a mirrored variable. See `DistributionStrategy.scope`.""" + colocate_with = kwargs.pop("colocate_with", None) + devices = self._get_devices_from(colocate_with) + + def _real_mirrored_creator(devices, *args, **kwargs): # pylint: disable=g-missing-docstring + index = {} + for i, d in enumerate(devices): + with ops.init_scope(), ops.device(d): + if i > 0: + # Give replicas meaningful distinct names: + var0name = index[devices[0]].name.split(":")[0] + # We append a / to variable names created on replicas with id > 0 to + # ensure that we ignore the name scope and instead use the given + # name as the absolute name of the variable. + kwargs["name"] = "%s/replica_%d/" % (var0name, i) + # Initialize replicas with the same value: + def initial_value_fn(device=d): + if context.executing_eagerly(): + init_value = index[devices[0]].value() + return array_ops.identity(init_value) + else: + with ops.device(device): + init_value = index[devices[0]].initial_value + return array_ops.identity(init_value) + kwargs["initial_value"] = initial_value_fn + with context.context().device_policy(context.DEVICE_PLACEMENT_SILENT): + # Don't record operations (e.g. other variable reads) during + # variable creation. + with tape.stop_recording(): + v = next_creator(*args, **kwargs) + assert not isinstance(v, values.DistributedVariable) + index[d] = v + return index + + return _create_mirrored_variable(devices, _real_mirrored_creator, *args, + **kwargs) + + def _distribute_dataset(self, dataset_fn): + if self._local_mode: + return values.PerReplicaDataset( + self._call_dataset_fn(dataset_fn), self._devices) + else: + return values.MultiWorkerDataset( + functools.partial(self._call_dataset_fn, dataset_fn), + self._worker_devices, + auto_shard=False) + + def _make_dataset_iterator(self, dataset): + if self._local_mode: + worker = device_util.canonicalize("/device:CPU:0") + worker_device_pairs = [(worker, self._devices)] + else: + worker_device_pairs = self._worker_devices + + return values.DatasetIterator(dataset, worker_device_pairs, + self._num_replicas_in_sync) + + def _make_input_fn_iterator( + self, + input_fn, + replication_mode=distribute_lib.InputReplicationMode.PER_WORKER): + input_contexts = [] + if self._local_mode: + num_workers = 1 + worker = device_util.canonicalize("/device:CPU:0") + worker_device_pairs = [(worker, self._devices)] + else: + num_workers = len(self._worker_devices) + worker_device_pairs = self._worker_devices + + for i in range(num_workers): + input_contexts.append(distribute_lib.InputContext( + num_input_pipelines=num_workers, + input_pipeline_id=i, + num_replicas_in_sync=self._num_replicas_in_sync)) + return values.InputFunctionIterator( + input_fn, worker_device_pairs, input_contexts) + + # TODO(priyag): Deal with OutOfRange errors once b/111349762 is fixed. + def _experimental_run_steps_on_iterator(self, fn, iterator, iterations, + initial_loop_values=None): + if initial_loop_values is None: + initial_loop_values = {} + initial_loop_values = nest.flatten(initial_loop_values) + + ctx = values.MultiStepContext() + def body(i, *args): + """A wrapper around `fn` to create the while loop body.""" + del args + fn_inputs = iterator.get_next() + if not isinstance(fn_inputs, tuple): + fn_inputs = (fn_inputs,) + fn_result = fn(ctx, fn_inputs) + for (name, output) in ctx.last_step_outputs.items(): + # Convert all outputs to tensors, potentially from `DistributedValues`. + ctx.last_step_outputs[name] = self._unwrap(output) + flat_last_step_outputs = nest.flatten(ctx.last_step_outputs) + with ops.control_dependencies([fn_result]): + return [i + 1] + flat_last_step_outputs + + # We capture the control_flow_context at this point, before we run `fn` + # inside a while_loop. This is useful in cases where we might need to exit + # these contexts and get back to the outer context to do some things, for + # e.g. create an op which should be evaluated only once at the end of the + # loop on the host. One such usage is in creating metrics' value op. + self._outer_control_flow_context = ( + ops.get_default_graph()._get_control_flow_context()) # pylint: disable=protected-access + + cond = lambda i, *args: i < iterations + i = constant_op.constant(0) + loop_result = control_flow_ops.while_loop( + cond, body, [i] + initial_loop_values, name="", + parallel_iterations=1, back_prop=False, swap_memory=False, + return_same_structure=True) + del self._outer_control_flow_context + + ctx.run_op = control_flow_ops.group(loop_result) + + # Convert the last_step_outputs from a list to the original dict structure + # of last_step_outputs. + last_step_tensor_outputs = loop_result[1:] + last_step_tensor_outputs_dict = nest.pack_sequence_as( + ctx.last_step_outputs, last_step_tensor_outputs) + + for name, reduce_op in ctx._last_step_outputs_reduce_ops.items(): # pylint: disable=protected-access + output = last_step_tensor_outputs_dict[name] + # For outputs that have already been reduced, wrap them in a Mirrored + # container, else in a PerReplica container. + if reduce_op is None: + last_step_tensor_outputs_dict[name] = values.regroup( + {d: t for d, t in zip(self._devices, output)}, values.PerReplica) + else: + assert len(output) == 1 + last_step_tensor_outputs_dict[name] = output[0] + + ctx._set_last_step_outputs(last_step_tensor_outputs_dict) # pylint: disable=protected-access + return ctx + + def _broadcast_to(self, tensor, destinations): + # This is both a fast path for Python constants, and a way to delay + # converting Python values to a tensor until we know what type it + # should be converted to. Otherwise we have trouble with: + # global_step.assign_add(1) + # since the `1` gets broadcast as an int32 but global_step is int64. + if isinstance(tensor, (float, int)): + return tensor + # TODO(josh11b): In eager mode, use one thread per device, or async mode. + return self._get_cross_device_ops().broadcast( + tensor, destinations or self._devices) + + def _call_for_each_replica(self, fn, args, kwargs): + return _call_for_each_replica(self._container_strategy(), fn, args, kwargs) + + def _configure(self, + session_config=None, + cluster_spec=None, + task_type=None, + task_id=None): + del task_type, task_id + + if session_config: + session_config.CopyFrom(self._update_config_proto(session_config)) + + if cluster_spec: + # TODO(yuefengz): remove the following code once cluster_resolver is + # added. + num_gpus_per_worker = _infer_num_gpus_per_worker(self._devices) + multi_worker_devices = _cluster_spec_to_device_list( + cluster_spec, num_gpus_per_worker) + self._initialize_multi_worker(multi_worker_devices) + + def _update_config_proto(self, config_proto): + updated_config = copy.deepcopy(config_proto) + updated_config.isolate_session_state = True + return updated_config + + def _get_cross_device_ops(self): + return self._cross_device_ops or self._inferred_cross_device_ops + + def _reduce_to(self, reduce_op, value, destinations): + assert not isinstance(value, values.Mirrored) + if not isinstance(value, values.DistributedValues): + # This function handles reducing values that are not PerReplica or + # Mirrored values. For example, the same value could be present on all + # replicas in which case `value` would be a single value or value could + # be 0. + return _reduce_non_distributed_value(self, reduce_op, value, + destinations) + return self._get_cross_device_ops().reduce( + reduce_op, value, destinations=destinations) + + def _batch_reduce_to(self, reduce_op, value_destination_pairs): + return self._get_cross_device_ops().batch_reduce(reduce_op, + value_destination_pairs) + + def _update(self, var, fn, args, kwargs, group): + # TODO(josh11b): In eager mode, use one thread per device. + assert isinstance(var, values.DistributedVariable) + updates = {} + for d, v in var._index.items(): # pylint: disable=protected-access + name = "update_%d" % self._device_index.get(d) + with ops.device(d), distribute_lib.UpdateContext(d), ops.name_scope(name): + # If args and kwargs are not mirrored, the value is returned as is. + updates[d] = fn(v, + *values.select_device_mirrored(d, args), + **values.select_device_mirrored(d, kwargs)) + return values.update_regroup(self, updates, group) + + def _update_non_slot(self, colocate_with, fn, args, kwargs, group): + assert isinstance(colocate_with, list) + # TODO(josh11b): In eager mode, use one thread per device. + updates = {} + for d in colocate_with: + name = "update_%d" % self._device_index.get(d) + with ops.device(d), distribute_lib.UpdateContext(d), ops.name_scope(name): + updates[d] = fn(*values.select_device_mirrored(d, args), + **values.select_device_mirrored(d, kwargs)) + return values.update_regroup(self, updates, group) + + def read_var(self, replica_local_var): + """Read the aggregate value of a replica-local variable.""" + if isinstance(replica_local_var, values.ReplicaLocalVariable): + return replica_local_var._get_cross_replica() # pylint: disable=protected-access + assert isinstance(replica_local_var, values.Mirrored) + return array_ops.identity(replica_local_var.get()) + + def _unwrap(self, val): + if isinstance(val, values.DistributedValues): + # Return in a deterministic order. + if set(val.devices) == self._canonical_device_set: + return [val.get(device=d) for d in self._devices] + return [val.get(device=d) for d in sorted(val.devices)] + return [val] + + def value_container(self, val): + return values.value_container(val) + + @property + def _num_replicas_in_sync(self): + return len(self._devices) + + @property + def worker_devices(self): + # Make a copy to prevent users from accidentally mutating our copy. + return list(self._devices) + + @property + def parameter_devices(self): + return list(self._devices) + + @property + def experimental_between_graph(self): + return False + + @property + def experimental_should_init(self): + return True + + @property + def should_checkpoint(self): + return True + + @property + def should_save_summary(self): + return True + + def non_slot_devices(self, var_list): + del var_list + return list(self._devices) + + def _get_devices_from(self, colocate_with=None): + if colocate_with is None: + return self._devices + else: + return cross_device_ops_lib.get_devices_from(colocate_with) + + # TODO(priyag): Delete this once all strategies use global batch size. + @property + def _global_batch_size(self): + return True + + class _MirroredReplicaThread(threading.Thread): + """A thread that runs() a function on a device.""" + + def __init__(self, dist, coord, device, variable_creator_fn, fn, *args, + **kwargs): + super(MirroredExtended._MirroredReplicaThread, self).__init__() # pylint: disable=protected-access + self.coord = coord + self.distribution = dist + self.device = device + self.replica_id = dist.extended.worker_devices.index(device) + self.variable_creator_fn = variable_creator_fn + # State needed to run and return the results of `fn`. + self.main_fn = fn + self.main_args = args + self.main_kwargs = kwargs + self.main_result = None + self.done = False + # State needed to run the next merge_call() (if any) requested via + # ReplicaContext. + self.merge_fn = None + self.merge_args = None + self.merge_kwargs = None + self.merge_result = None + self.captured_name_scope = None + # We use a thread.Event for the main thread to signal when this + # thread should start running (`should_run`), and another for + # this thread to transfer control back to the main thread + # (`has_paused`, either when it gets to a + # `get_replica_context().merge_call` or when `fn` returns). In + # either case the event starts cleared, is signaled by calling + # set(). The receiving thread waits for the signal by calling + # wait() and then immediately clearing the event using clear(). + self.should_run = threading.Event() + self.has_paused = threading.Event() + # These fields have to do with inheriting various contexts from the + # parent thread: + # pylint: disable=protected-access + self.context_mode = context.context()._eager_context.mode + if not context.context()._context_handle: + context.context()._initialize_handle_and_devices() + self.context_device_policy = ( + pywrap_tensorflow.TFE_ContextGetDevicePlacementPolicy( + context.context()._context_handle)) + self.graph = ops.get_default_graph() + self._variable_creator_stack = self.graph._variable_creator_stack[:] + self._captured_var_scope = variable_scope.get_variable_scope() + # Adding a "/" at end lets us re-enter this scope later. + self._name_scope = self.graph.get_name_scope() + if self._name_scope: + self._name_scope += "/" + if self.replica_id > 0: + if not self._name_scope: + self._name_scope = "" + self._name_scope += "replica_%d/" % self.replica_id + + def run(self): + # pylint: disable=protected-access + self.graph._variable_creator_stack = self._variable_creator_stack + self.should_run.wait() + self.should_run.clear() + try: + if self.coord.should_stop(): + return + with self.coord.stop_on_exception(), \ + context.context()._mode(self.context_mode), \ + context.context().device_policy(self.context_device_policy), \ + _enter_graph(self.graph), \ + MirroredReplicaContext(self.distribution, constant_op.constant( + self.replica_id, dtypes.int32)), \ + ops.device(self.device), \ + ops.name_scope(self._name_scope), \ + variable_scope.variable_scope( + self._captured_var_scope, reuse=self.replica_id > 0), \ + variable_scope.variable_creator_scope(self.variable_creator_fn): + self.main_result = self.main_fn(*self.main_args, **self.main_kwargs) + self.done = True + finally: + self.has_paused.set() + + +class MirroredReplicaContext(distribute_lib.ReplicaContext): + """ReplicaContext used in MirroredStrategy.call_for_each_replica(). + + Opened in `_MirroredReplicaThread`, to allow the user to invoke + `MirroredStrategy`'s specific implementation of `merge_call()`, + which works by delegating the function and its arguments to + the main thread (the one that invoked + `MirroredStrategy.call_for_each_replica()`). + """ + + def _merge_call(self, fn, args, kwargs): + """Delegate to the main thread to actually perform merge_call().""" + t = threading.current_thread() # a _MirroredReplicaThread + t.merge_fn = fn + t.merge_args = args + t.merge_kwargs = kwargs + t.captured_name_scope = t.graph.get_name_scope() + # Adding a "/" at end lets us re-enter this scope later. + if t.captured_name_scope: + t.captured_name_scope += "/" + t.has_paused.set() + t.should_run.wait() + t.should_run.clear() + if t.coord.should_stop(): + raise _RequestedStop() + return t.merge_result + + @property + def devices(self): + distribute_lib.require_replica_context(self) + replica_id = tensor_util.constant_value(self._replica_id_in_sync_group) + return [self._distribution_strategy.extended.worker_devices[replica_id]] diff --git a/tensorflow/python/distribute/multi_worker_util.py b/tensorflow/python/distribute/multi_worker_util.py index 360733eff64..2986a6726a5 100644 --- a/tensorflow/python/distribute/multi_worker_util.py +++ b/tensorflow/python/distribute/multi_worker_util.py @@ -45,6 +45,33 @@ def normalize_cluster_spec(cluster_spec): return cluster_spec +# TODO(yuefengz): add more validations. +def _validate_cluster_spec(cluster_spec, task_type, task_id): + """Validates `cluster_spec`. + + It checks + 1) whether there is such a task type as `task_type` in the + `cluster_spec`. + 2) whether there is at most one "chief" job. + 3) whether the `task_id` is smaller than the number of `task_type`. + + Args: + cluster_spec: a dict, `ClusterDef` or `ClusterSpec` object to be validated. + task_type: string indicating the type of the task. + task_id: task_id: the id of the `task_type` in this cluster. + Throws: + ValueError: if `cluster_spec` fails any check. + """ + cluster_spec = normalize_cluster_spec(cluster_spec).as_dict() + if task_type and task_type not in cluster_spec: + raise ValueError("`task_type` %r not found in cluster_spec." % task_type) + if len(cluster_spec.get("chief", [])) > 1: + raise ValueError("There must be at most one 'chief' job.") + if task_id >= len(cluster_spec[task_type]): + raise ValueError( + "The `task_id` %d exceeds the maximum id of %s." % (task_id, task_type)) + + def is_chief(cluster_spec, task_type, task_id): """Returns whether the given task is chief in the cluster. @@ -61,20 +88,73 @@ def is_chief(cluster_spec, task_type, task_id): ValueError: if `task_type` is not in the `cluster_spec` or `task_id` exceeds the maximum id of the `task_type`. """ - cluster_spec = normalize_cluster_spec(cluster_spec) - if task_type not in cluster_spec.jobs: - raise ValueError( - "The task_type \"%s\" is not in the `cluster_spec`." % task_type) - if task_id >= cluster_spec.num_tasks(task_type): - raise ValueError("The `task_id` %d exceeds the maximum id of %s." % ( - task_id, task_type)) + _validate_cluster_spec(cluster_spec, task_type, task_id) + cluster_spec = normalize_cluster_spec(cluster_spec).as_dict() if task_type == "chief": return True # If chief not in the cluster_spec, use the first worker as chief. This is # common in CollectiveAllReduceStrategy. - if ("chief" not in cluster_spec.jobs and task_type == "worker" and - task_id == 0): + if ("chief" not in cluster_spec and task_type == "worker" and task_id == 0): return True return False + + +def worker_count(cluster_spec, task_type): + """Returns the number of workers in the cluster.""" + _validate_cluster_spec(cluster_spec, task_type, task_id=0) + cluster_spec = normalize_cluster_spec(cluster_spec).as_dict() + + # Other jobs such as "ps" shouldn't call this function. + if task_type not in ["chief", "worker", "evaluator"]: + raise ValueError("Unexpected `task_type` %r" % task_type) + + if task_type == "evaluator": + # The "evaluator" is in its own cluster or its own partition of a cluster. + # So we don't have to count "chief" or "worker" if the current task is an + # "evaluator". + return len(cluster_spec["evaluator"]) + else: + # In the non-evaluator case, we return the total number of "chief" and + # "worker" tasks as the "chief" is also a worker. + return (len(cluster_spec.get("chief", [])) + len( + cluster_spec.get("worker", []))) + + +def id_in_cluster(cluster_spec, task_type, task_id): + """Returns a unique id for the task in the `task_type`'s cluster. + + It returns an id ranging from [0, `worker_count(task_type, task_id)`). + + Note: this function assumes that "evaluate" job is in its own cluster or its + own partition of a cluster. + + Args: + cluster_spec: a dict, `ClusterDef` or `ClusterSpec` object to be validated. + task_type: string indicating the type of the task. + task_id: the id of the `task_type` in this cluster. + + Returns: + an int indicating the unique id. + + Throws: + ValueError: if `task_type` is not "chief", "worker" or "evaluator". + """ + _validate_cluster_spec(cluster_spec, task_type, task_id) + cluster_spec = normalize_cluster_spec(cluster_spec).as_dict() + + # The "chief" job has always id 0 and there is at most one and "worker" jobs + # come after it. + if task_type == "chief": + return 0 + + if task_type == "worker": + return task_id + len(cluster_spec.get("chief", [])) + + # The "evaluator" is in its own cluster or its own partition of a cluster. + if task_type == "evaluator": + return task_id + + # We currently don't assign ids to other tasks. + raise ValueError("There is no id for task_type %r" % task_type) diff --git a/tensorflow/python/distribute/multi_worker_util_test.py b/tensorflow/python/distribute/multi_worker_util_test.py index bdc49725c77..9e1596eefdf 100644 --- a/tensorflow/python/distribute/multi_worker_util_test.py +++ b/tensorflow/python/distribute/multi_worker_util_test.py @@ -95,7 +95,7 @@ class IsChiefTest(test.TestCase): self.assertFalse(multi_worker_util.is_chief(cluster_spec, "worker", 1)) with self.assertRaisesRegexp( - ValueError, "The task_type \"chief\" is not in the `cluster_spec`."): + ValueError, "`task_type` 'chief' not found in cluster_spec."): multi_worker_util.is_chief(cluster_spec, "chief", 0) with self.assertRaisesRegexp( @@ -103,5 +103,94 @@ class IsChiefTest(test.TestCase): multi_worker_util.is_chief(cluster_spec, "worker", 2) +class NumWorkersTest(test.TestCase): + + def testCountWorker(self): + cluster_spec = { + "chief": ["127.0.0.1:1234"], + "worker": ["127.0.0.1:8964", "127.0.0.1:2333"], + "ps": ["127.0.0.1:1926", "127.0.0.1:3141"] + } + self.assertEqual( + multi_worker_util.worker_count(cluster_spec, task_type="chief"), 3) + self.assertEqual( + multi_worker_util.worker_count(cluster_spec, task_type="worker"), 3) + + def testCountEvaluator(self): + cluster_spec = { + "chief": ["127.0.0.1:1234"], + "worker": ["127.0.0.1:8964", "127.0.0.1:2333"], + "evaluator": ["127.0.0.1:7566"] + } + self.assertEqual( + multi_worker_util.worker_count(cluster_spec, task_type="evaluator"), 1) + + def testTaskTypeNotFound(self): + cluster_spec = {} + with self.assertRaisesRegexp( + ValueError, "`task_type` 'worker' not found in cluster_spec."): + multi_worker_util.worker_count(cluster_spec, task_type="worker") + + def testCountPs(self): + cluster_spec = { + "chief": ["127.0.0.1:1234"], + "ps": ["127.0.0.1:1926", "127.0.0.1:3141"] + } + # A "ps" job shouldn't call this method. + with self.assertRaisesRegexp(ValueError, "Unexpected `task_type` 'ps'"): + multi_worker_util.worker_count(cluster_spec, task_type="ps") + + +class IdInClusterTest(test.TestCase): + + def testChiefId(self): + cluster_spec = { + "chief": ["127.0.0.1:1234"], + "worker": ["127.0.0.1:8964", "127.0.0.1:2333"], + "ps": ["127.0.0.1:1926", "127.0.0.1:3141"] + } + self.assertEqual( + multi_worker_util.id_in_cluster(cluster_spec, "chief", 0), 0) + + def testWorkerId(self): + cluster_spec = { + "chief": ["127.0.0.1:1234"], + "worker": ["127.0.0.1:8964", "127.0.0.1:2333"], + "ps": ["127.0.0.1:1926", "127.0.0.1:3141"] + } + self.assertEqual( + multi_worker_util.id_in_cluster(cluster_spec, "worker", 1), 2) + + cluster_spec = { + "worker": ["127.0.0.1:8964", "127.0.0.1:2333"], + "ps": ["127.0.0.1:1926", "127.0.0.1:3141"] + } + self.assertEqual( + multi_worker_util.id_in_cluster(cluster_spec, "worker", 1), 1) + + def testEvaluatorId(self): + cluster_spec = { + "chief": ["127.0.0.1:1234"], + "worker": ["127.0.0.1:8964", "127.0.0.1:2333"], + "evaluator": ["127.0.0.1:7566"] + } + self.assertEqual( + multi_worker_util.id_in_cluster(cluster_spec, "evaluator", 0), 0) + + def testPsId(self): + cluster_spec = {"chief": ["127.0.0.1:1234"], "ps": ["127.0.0.1:7566"]} + with self.assertRaisesRegexp(ValueError, + "There is no id for task_type 'ps'"): + multi_worker_util.id_in_cluster(cluster_spec, "ps", 0) + + def testMultipleChiefs(self): + cluster_spec = { + "chief": ["127.0.0.1:8258", "127.0.0.1:7566"], + } + with self.assertRaisesRegexp(ValueError, + "There must be at most one 'chief' job."): + multi_worker_util.id_in_cluster(cluster_spec, "chief", 0) + + if __name__ == "__main__": test.main() diff --git a/tensorflow/python/distribute/reduce_util.py b/tensorflow/python/distribute/reduce_util.py new file mode 100644 index 00000000000..2b2a4e9dba8 --- /dev/null +++ b/tensorflow/python/distribute/reduce_util.py @@ -0,0 +1,53 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Utilites for reduce operations.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import enum + +from tensorflow.python.ops import variable_scope +from tensorflow.python.util.tf_export import tf_export + + +@tf_export("distribute.ReduceOp") +class ReduceOp(enum.Enum): + """Indicates how a set of values should be reduced. + + * `SUM`: Add all the values. + * `MEAN`: Take the arithmetic mean ("average") of the values. + + TODO(priyag): Add the following types: + * `MIN`: Return the minimum of all values. + * `MAX`: Return the maximum of all values. + """ + + SUM = "SUM" + MEAN = "MEAN" + + @staticmethod + def from_variable_aggregation(aggregation): + mapping = { + variable_scope.VariableAggregation.SUM: ReduceOp.SUM, + variable_scope.VariableAggregation.MEAN: ReduceOp.MEAN, + } + + reduce_op = mapping.get(aggregation) + if not reduce_op: + raise ValueError("Could not convert from `tf.VariableAggregation` %s to" + "`tf.distribute.ReduceOp` type" % aggregation) + return reduce_op diff --git a/tensorflow/contrib/distribute/python/shared_variable_creator.py b/tensorflow/python/distribute/shared_variable_creator.py similarity index 100% rename from tensorflow/contrib/distribute/python/shared_variable_creator.py rename to tensorflow/python/distribute/shared_variable_creator.py diff --git a/tensorflow/contrib/distribute/python/shared_variable_creator_test.py b/tensorflow/python/distribute/shared_variable_creator_test.py similarity index 97% rename from tensorflow/contrib/distribute/python/shared_variable_creator_test.py rename to tensorflow/python/distribute/shared_variable_creator_test.py index 2a9ab51fcfd..4ddc29f2567 100644 --- a/tensorflow/contrib/distribute/python/shared_variable_creator_test.py +++ b/tensorflow/python/distribute/shared_variable_creator_test.py @@ -18,7 +18,7 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function -from tensorflow.contrib.distribute.python import shared_variable_creator +from tensorflow.python.distribute import shared_variable_creator from tensorflow.python.eager import test from tensorflow.python.framework import test_util from tensorflow.python.ops import variable_scope diff --git a/tensorflow/contrib/distribute/python/values.py b/tensorflow/python/distribute/values.py similarity index 77% rename from tensorflow/contrib/distribute/python/values.py rename to tensorflow/python/distribute/values.py index a1629735353..01a1680a246 100644 --- a/tensorflow/contrib/distribute/python/values.py +++ b/tensorflow/python/distribute/values.py @@ -23,11 +23,18 @@ from __future__ import print_function import collections import contextlib +import operator import weakref import six -from tensorflow.contrib.distribute.python import input_ops +from tensorflow.python.data.experimental.ops import batching +from tensorflow.python.data.ops import dataset_ops from tensorflow.python.data.ops import multi_device_iterator_ops +from tensorflow.python.distribute import device_util +from tensorflow.python.distribute import distribute_lib +from tensorflow.python.distribute import distribution_strategy_context +from tensorflow.python.distribute import input_ops +from tensorflow.python.distribute import reduce_util from tensorflow.python.eager import context from tensorflow.python.eager import tape from tensorflow.python.framework import device as tf_device @@ -38,10 +45,6 @@ from tensorflow.python.ops import control_flow_ops from tensorflow.python.ops import gen_resource_variable_ops from tensorflow.python.ops import math_ops from tensorflow.python.ops import variable_scope as vs -from tensorflow.python.ops import variables as variables_lib -from tensorflow.python.training import device_util -from tensorflow.python.training import distribute as distribute_lib -from tensorflow.python.training import distribution_strategy_context from tensorflow.python.training import saver from tensorflow.python.training.checkpointable import base as checkpointable from tensorflow.python.util import nest @@ -97,10 +100,21 @@ class DistributedValues(object): # DistributionStrategy implementations. +# NOTE(josh11b,apassos): It would be great if we could inspect the values this was +# initialized with and use that to generate the overloaded operators here. +# Unfortunately, Python's rules for special methods don't allow this, see +# https://docs.python.org/3/reference/datamodel.html#special-method-names +# "if a class defines a method named __getitem__(), and x is an instance of +# this class, then x[i] is roughly equivalent to type(x).__getitem__(x, i)." +# In particular, these special methods don't go through __getattr__, and +# it will only use those methods if they are defined in the class, not the +# object. class DistributedDelegate(DistributedValues): """A map from device to values; acts as the same type as the values.""" def __getattr__(self, name): + # TODO(priyag): This needs to be made robust against pitfalls from mix use + # __getattr__ and @property. See b/120402273. return getattr(self.get(), name) # pylint: disable=multiple-statements @@ -316,6 +330,14 @@ class DistributedVariable(DistributedDelegate): ops.register_dense_tensor_like_type(DistributedVariable) +def _apply_aggregation(strategy, value, aggregation, destinations): + if aggregation == vs.VariableAggregation.ONLY_FIRST_REPLICA: + return strategy.broadcast(strategy.unwrap(value)[0], + destinations=destinations) + reduce_op = reduce_util.ReduceOp.from_variable_aggregation(aggregation) + return strategy.extended.reduce_to(reduce_op, value, destinations) + + class _MirroredSaveable(saver.BaseSaverBuilder.ResourceVariableSaveable): """Class for defining how to restore a MirroredVariable.""" @@ -373,14 +395,11 @@ class MirroredVariable(DistributedVariable, Mirrored, "MirroredVariable in Replica Context.") def merge_fn(strategy, value, *other_args, **other_kwargs): - return strategy.update( - self, f, - strategy.reduce( - aggregation=self._aggregation, value=value, destinations=self), - *other_args, **other_kwargs) + v = _apply_aggregation(strategy, value, self._aggregation, self) + return strategy.update(self, f, v, *other_args, **other_kwargs) return distribution_strategy_context.get_replica_context().merge_call( - merge_fn, *args, **kwargs) + merge_fn, args=args, kwargs=kwargs) def assign_sub(self, *args, **kwargs): assign_sub_fn = lambda var, *a, **kw: var.assign_sub(*a, **kw) @@ -614,14 +633,11 @@ class TPUMirroredVariable(checkpointable.CheckpointableBase): "TPUMirroredVariable in Replica Context.") def merge_fn(strategy, value, *other_args, **other_kwargs): - return strategy.update( - self, f, - strategy.reduce( - aggregation=self._aggregation, value=value, destinations=self), - *other_args, **other_kwargs) + v = _apply_aggregation(strategy, value, self._aggregation, self) + return strategy.update(self, f, v, *other_args, **other_kwargs) return distribution_strategy_context.get_replica_context().merge_call( - merge_fn, *args, **kwargs) + merge_fn, args=args, kwargs=kwargs) @contextlib.contextmanager def _handle_graph(self, handle): @@ -1058,18 +1074,18 @@ def select_device_mirrored(device, structured): return nest.map_structure(_get_mirrored, structured) -def update_regroup(strategy, updates, should_group): +def update_regroup(extended, updates, group): """Regroup for an update, with dependencies to ensure all updates execute.""" regrouped = regroup(updates, Mirrored) - if not should_group: - return nest.map_structure(strategy.unwrap, regrouped) + if not group: + return nest.map_structure(extended._unwrap, regrouped) # pylint: disable=protected-access grouped_flat = [] for u in nest.flatten(regrouped): if isinstance(u, DistributedValues): - g = strategy.group(u) + g = extended._group(u) # pylint: disable=protected-access if u.is_tensor_like: # Make sure we run all updates. Without this, something like - # session.run(strategy.update(...)) may only update one replica. + # session.run(extended.update(...)) may only update one replica. index = {} for d in u.devices: with ops.device(d), ops.control_dependencies([g]): @@ -1155,7 +1171,7 @@ class PerReplicaDataset(object): # Eager mode prefetching would error out in constructor. Only remaining # case is non-prefetching in eager mode. We delegate to # PerReplicaDataIterator to handle that case. - dataset_iterator = self._dataset.make_one_shot_iterator() + dataset_iterator = dataset_ops.make_one_shot_iterator(self._dataset) return PerReplicaDataIterator( dataset_iterator, self._devices, prefetch_on_device=False) @@ -1170,7 +1186,7 @@ class PerReplicaDataset(object): dataset_iterator = multi_device_iterator_ops.MultiDeviceIterator( self._dataset, self._devices) else: - dataset_iterator = self._dataset.make_initializable_iterator() + dataset_iterator = dataset_ops.make_initializable_iterator(self._dataset) return PerReplicaDataIterator( dataset_iterator, self._devices, @@ -1252,22 +1268,34 @@ class MultiWorkerDataset(object): """Initialize the MultiWorkerDataset object. Args: - dataset_fn: a function that returns a `tf.data.Dataset`. + dataset_fn: a function or a list of functions that returns a + `tf.data.Dataset`. worker_device_pairs: a list of (worker, list of devices on that worker) - pairs. + pairs; it must have same length with `dataset_fn` if `dataset_fn` is a + list. prefetch_on_device: whether to prefetch to devices. auto_shard: whether to auto-shard the dataset. """ + if isinstance(dataset_fn, list): + if len(dataset_fn) != len(worker_device_pairs): + raise ValueError("If `dataset_fn` is a list, it must have same length " + "as `worker_device_pairs`") + if auto_shard: + raise ValueError( + "If `dataset_fn` is a list, `auto_shard` is not supported.") self._worker_device_pairs = worker_device_pairs self._datasets = [] # TODO(yuefengz, priyag): support different set of jobs for input # processing. for i, (worker, worker_devices) in enumerate(worker_device_pairs): with ops.device(worker): - worker_input = dataset_fn() - if auto_shard: - worker_input = input_ops.auto_shard_dataset( - worker_input, len(worker_device_pairs), i) + if isinstance(dataset_fn, list): + worker_input = dataset_fn[i]() + else: + worker_input = dataset_fn() + if auto_shard: + worker_input = input_ops.auto_shard_dataset( + worker_input, len(worker_device_pairs), i) dataset = PerReplicaDataset( worker_input, worker_devices, prefetch_on_device=prefetch_on_device) self._datasets.append((worker, dataset)) @@ -1276,36 +1304,337 @@ class MultiWorkerDataset(object): iterators = [] for worker, dataset in self._datasets: with ops.device(worker): - iterators.append((worker, dataset.make_one_shot_iterator())) + iterators.append((worker, dataset_ops.make_one_shot_iterator(dataset))) return MultiWorkerDataIterator(iterators, self._worker_device_pairs) def make_initializable_iterator(self): iterators = [] for worker, dataset in self._datasets: with ops.device(worker): - iterators.append((worker, dataset.make_initializable_iterator())) + iterators.append( + (worker, dataset_ops.make_initializable_iterator(dataset))) return MultiWorkerDataIterator(iterators, self._worker_device_pairs) +class InputIterator(object): + """An input iterator, intended to be passed to `DistributionStrategy.run`.""" + + def get_next(self): + """Returns the next inputs for all replicas.""" + raise NotImplementedError("must be implemented in descendants") + + def initialize(self): + """Initialize the underlying input dataset, when applicable. + + In eager mode, this will create a new iterator and return it. + In graph mode, this will initialize the same underlying iterator(s). + + Users are required to call this if + - This iterator was returned from a call to `make_input_fn_iterator` with an + input function that returns a dataset. + - Or this iterator was returned from a call to `make_dataset_iterator`. + + Returns: + A list of initialization ops to be executed. + """ + raise NotImplementedError("must be implemented in descendants") + + +class InputIteratorImpl(InputIterator): + """Common implementation for all input iterators.""" + + def __init__(self, worker_device_pairs, iterators): + if not worker_device_pairs: + raise ValueError("Should have at least one worker for input iterator.") + + self._iterators = iterators + self._worker_device_pairs = worker_device_pairs + self._is_eager = context.executing_eagerly() + + def get_next(self, name=None): + """Returns the next input from the iterator for all replicas.""" + assert self._is_eager == context.executing_eagerly(), ( + "Iterator should be created and used in same execution mode.") + + index = {} + for i, (worker, worker_devices) in enumerate(self._worker_device_pairs): + if name is not None: + d = tf_device.DeviceSpec.from_string(worker) + new_name = "%s_%s_%d" % (name, d.job, d.task) + else: + new_name = None + with ops.device(worker): + data_per_worker = self._iterators[i].get_next(new_name) + + # Ungroup these per-replica value so as to get a flat map from devices to + # values. + for d in worker_devices: + v = select_device(d, data_per_worker) + if d in index: + raise ValueError("Duplicated devices in worker_device_pairs: %r" % v) + index[d] = v + + return regroup(index) + + def initialize(self): + """Initialze underlying iterators. + + Returns: + A list of any initializer ops that should be run. + """ + assert self._is_eager == context.executing_eagerly(), ( + "Iterator should be created and used in same execution mode.") + + init_ops = [] + for it in self._iterators: + init_ops.extend(it.initialize()) + return init_ops + + # TODO(priyag): Remove when we switch to using `MultiDeviceIterator` for TPUs. + @property + def output_classes(self): + return self._iterators[0].output_classes + + # TODO(priyag): Remove when we switch to using `MultiDeviceIterator` for TPUs. + @property + def output_shapes(self): + return self._iterators[0].output_shapes + + # TODO(priyag): Remove when we switch to using `MultiDeviceIterator` for TPUs. + @property + def output_types(self): + return self._iterators[0].output_types + + # TODO(priyag): Remove when we switch to using `MultiDeviceIterator` for TPUs. + def get_iterator(self, worker): + for i, (w, _) in enumerate(self._worker_device_pairs): + if worker == w: + return self._iterators[i] + return None + + +class InputFunctionIterator(InputIteratorImpl): + """Iterator created from input function.""" + + def __init__(self, input_fn, worker_device_pairs, input_contexts): + """Make an iterator for input provided via an input function. + + Currently implements PER_WORKER mode, in which the `input_fn` is called + once on each worker. + + TODO(priyag): Add other replication modes. + TODO(priyag): Allow taking input function that returns a callable that + returns nest of tensors. + + Args: + input_fn: Input function that returns a `tf.data.Dataset` object. + worker_device_pairs: A list of (worker, list of devices on that worker) + pairs. + input_contexts: A list of `InputContext` instances to be passed to call(s) + to `input_fn`. Length and order should match worker order in + `worker_device_pairs`. + """ + if len(worker_device_pairs) != len(input_contexts): + raise ValueError( + "Number of worker_device_pairs (%d) is not same as number of" + "input_contexts (%d)" % ( + len(worker_device_pairs), len(input_contexts))) + + iterators = [] + for (worker, devices), ctx in zip(worker_device_pairs, input_contexts): + # TODO(priyag): We should probably explicitly specify CPU device on worker. + with ops.device(worker): + result = input_fn(ctx) + if not isinstance(result, dataset_ops.DatasetV2): + raise ValueError("input_fn must return a tf.data.Dataset.") + iterator = _SingleWorkerDatasetIterator(result, worker, devices) + iterators.append(iterator) + + super(InputFunctionIterator, self).__init__( + worker_device_pairs, iterators) + + +class DatasetIterator(InputIteratorImpl): + """Iterator created from input dataset.""" + + def __init__(self, dataset, worker_device_pairs, split_batch_by=None): + """Make an iterator for the dataset on given devices. + + If `split_batch_by` is not None, we "split" each batch of the + dataset by `split_batch_by` value. To achieve this, we first unbatch the + input dataset and then rebatch it with the per replica batch size that is + calculated using `global_batch_size // split_batch_by`. + The currently supported datasets are as follows: + `dataset.batch()` is the last operation on the dataset OR + `dataset.apply(map_and_batch)` is the last operation on the dataset OR + `dataset.batch().prefetch()` are the last 2 operations on the dataset OR + `dataset.apply(map_and_batch).prefetch()` are the last 2 operations. + + TODO(priyag): Support multi worker / host cases properly by cloning + and sharding the dataset on each worker. Current setup will only work in + some cases, such as in-graph multi worker GPU case. If the input pipeline + has random shuffling (with a different seed on each worker), each worker + will see random input from the same overall dataset in each step. Otherwise, + each worker will see the same input in each step. + + Args: + dataset: `tf.data.Dataset` that will be used as the input source. + worker_device_pairs: A list of (worker, list of devices on that worker) + pairs. + split_batch_by: Optional integer. If present, we "split" each batch of the + dataset by `split_batch_by` value. + """ + if split_batch_by: + dataset = _split_dataset_batch(dataset, split_batch_by) + + iterators = [] + for worker, worker_devices in worker_device_pairs: + with ops.device(worker): + iterator = _SingleWorkerDatasetIterator(dataset, worker, worker_devices) + iterators.append(iterator) + + super(DatasetIterator, self).__init__(worker_device_pairs, iterators) + + +class _SingleWorkerDatasetIterator(object): + """Iterator for a single `tf.data.Dataset`.""" + + def __init__(self, dataset, worker, devices): + """Create iterator for the `dataset` to fetch data to worker's `devices` . + + `MultiDeviceIterator` is used to prefetch input to the devices on the + given worker. `MultiDeviceIterator` doesn't work in eager mode yet. + + Args: + dataset: A `tf.data.Dataset` instance. + worker: Worker on which ops should be created. + devices: Distribute data from `dataset` to these devices. + """ + self._dataset = dataset + self._worker = worker + self._devices = devices + self._is_eager = context.executing_eagerly() + self._make_iterator() + + def _make_iterator(self): + """Make appropriate iterator on the dataset.""" + with ops.device(self._worker): + if self._is_eager: + # TODO(rohanj): Enable prefetching in eager mode. + # TODO(priyag): Measure the performance of this approach vs calling + # get_next on the original dataset N times. + dataset = self._dataset.batch(len(self._devices), drop_remainder=True) + iterator = dataset_ops.make_one_shot_iterator(dataset) + else: + iterator = multi_device_iterator_ops.MultiDeviceIterator( + self._dataset, self._devices) + self._iterator = iterator + + def get_next(self, name=None): + """Get next element from the underlying iterator.""" + with ops.device(self._worker): + if self._is_eager: + # Batched dataset case. + batch = self._iterator.get_next(name=name) + index = {} + for i, d in enumerate(self._devices): + index[d] = nest.map_structure(operator.itemgetter(i), batch) + with ops.device(d): + index[d] = nest.map_structure(array_ops.identity, index[d]) + else: + # MultiDeviceIterator case. + data_list = self._iterator.get_next() + index = dict(zip(self._devices, data_list)) + + return regroup(index) + + def initialize(self): + """Initialze underlying iterator. + + In eager execution, this simply recreates the underlying iterator. + In graph execution, it returns the initializer ops for the underlying + iterator. + + Returns: + A list of any initializer ops that should be run. + """ + if self._is_eager: + self._make_iterator() + return [] + else: + return [self._iterator.initializer] + + @property + def output_classes(self): + return self._iterator.output_classes + + @property + def output_shapes(self): + return self._iterator.output_shapes + + @property + def output_types(self): + return self._iterator.output_types + + +def _split_dataset_batch(dataset, split_batch_by): + """Divide a batch-ed dataset's batches into smaller batches.""" + # TODO(sourabhbajaj): Remove this in lieu of distributed datasets + # pylint: disable=protected-access + def _get_batch_dataset(d): + """Get the underlying batch dataset from the dataset object.""" + if isinstance(d, dataset_ops.DatasetV1Adapter): + d = d._dataset + + if isinstance(d, (dataset_ops.BatchDataset, batching._MapAndBatchDataset)): + return d + elif isinstance(d, dataset_ops.PrefetchDataset): + return _get_batch_dataset(d._input_dataset) + raise ValueError( + "Unable to get batched dataset from the input dataset. `batch` " + "`map_and_batch` need to be the last operations on the dataset. " + "The batch operations can be followed by a prefetch.") + + batched_dataset = _get_batch_dataset(dataset) + batch_size = batched_dataset._batch_size + drop_remainder = batched_dataset._drop_remainder + # pylint: enable=protected-access + + if tensor_util.is_tensor(batch_size): + batch_size = tensor_util.constant_value(batch_size) + + if tensor_util.is_tensor(drop_remainder): + drop_remainder = tensor_util.constant_value(drop_remainder) + + if batch_size % split_batch_by: + raise ValueError( + "Batch size %s cannot be sharded evenly across replicas %s" % ( + batch_size, split_batch_by)) + new_batch_size = batch_size // split_batch_by + + dataset = dataset.apply(batching.unbatch()) + return dataset.batch(new_batch_size, drop_remainder=drop_remainder) + + class MultiStepContext(object): """A context object that can be used to capture things when running steps. This context object is useful when running multiple steps at a time using the - `run_steps_on_dataset` API. For e.g. it allows the user's step function to - specify which outputs to emit at what frequency. Currently it supports - capturing output from the last step, as well as capturing non tensor outputs. - In the future it will be augmented to support other use cases such as output - each N steps. + `experimental_run_steps_on_iterator` API. For e.g. it allows the user's step + function to specify which outputs to emit at what frequency. Currently it + supports capturing output from the last step, as well as capturing non tensor + outputs. In the future it will be augmented to support other use cases such + as output each N steps. """ def __init__(self): - """Initializes an output context. + """Initialize an output context. Returns: A context object. """ self._last_step_outputs = {} - self._last_step_outputs_aggregations = {} + self._last_step_outputs_reduce_ops = {} self._non_tensor_outputs = {} @property @@ -1315,8 +1644,8 @@ class MultiStepContext(object): Keys in the dictionary are names of tensors to be captured, as specified when `set_last_step_output` is called. Values in the dictionary are the tensors themselves. If - `set_last_step_output` was called with an `aggregation` for this output, - then the value is the aggregated value. + `set_last_step_output` was called with a `reduce_op` for this output, + then the value is the reduced value. Returns: A dictionary with last step outputs. @@ -1329,8 +1658,7 @@ class MultiStepContext(object): raise ValueError("Need a dictionary to set last_step_outputs.") self._last_step_outputs = outputs - def set_last_step_output(self, name, output, - aggregation=variables_lib.VariableAggregation.NONE): + def set_last_step_output(self, name, output, reduce_op=None): """Set `output` with `name` to be outputted from the last step. Args: @@ -1338,39 +1666,36 @@ class MultiStepContext(object): name. output: The tensors that should be outputted with `name`. See below for actual types supported. - aggregation: Aggregation method to use to aggregate outputs from multiple + reduce_op: Reduction method to use to reduce outputs from multiple replicas. Required if `set_last_step_output` is called in a replica context. Optional in cross_replica_context. - When present, the outputs from all the replicas are aggregated using the + When present, the outputs from all the replicas are reduced using the current distribution strategy's `reduce` method. Hence, the type of `output` must be what's supported by the corresponding `reduce` method. - For e.g. if using MirroredStrategy and aggregation is set, output + For e.g. if using MirroredStrategy and reduction is set, output must be a `PerReplica` value. - The aggregation method is also recorded in a dictionary - `_last_step_outputs_aggregations` for later interpreting of the + The reduce method is also recorded in a dictionary + `_last_step_outputs_reduce_ops` for later interpreting of the outputs as already reduced or not. - """ if distribution_strategy_context.get_cross_replica_context(): - self._last_step_outputs_aggregations[name] = aggregation - if aggregation is variables_lib.VariableAggregation.NONE: + self._last_step_outputs_reduce_ops[name] = reduce_op + if reduce_op is None: self._last_step_outputs[name] = output else: distribution = distribution_strategy_context.get_distribution_strategy() - self._last_step_outputs[name] = distribution.reduce( - aggregation, output, destinations="/device:CPU:0") + self._last_step_outputs[name] = distribution.reduce(reduce_op, output) else: - assert aggregation is not variables_lib.VariableAggregation.NONE + assert reduce_op is not None def merge_fn(distribution, value): - self._last_step_outputs[name] = distribution.reduce( - aggregation, value, destinations="/device:CPU:0") + self._last_step_outputs[name] = distribution.reduce(reduce_op, value) # Setting this inside the `merge_fn` because all replicas share the same # context object, so it's more robust to set it only once (even if all # the replicas are trying to set the same value). - self._last_step_outputs_aggregations[name] = aggregation + self._last_step_outputs_reduce_ops[name] = reduce_op distribution_strategy_context.get_replica_context().merge_call( - merge_fn, output) + merge_fn, args=(output,)) @property def non_tensor_outputs(self): @@ -1384,10 +1709,10 @@ class MultiStepContext(object): else: def merge_fn(distribution, value): # NOTE(priyag): For non tensor outputs, we simply return all the values - # in a list as aggregation doesn't make sense on non tensors. + # in a list as reduction doesn't make sense on non tensors. self._non_tensor_outputs[name] = distribution.unwrap(value) distribution_strategy_context.get_replica_context().merge_call( - merge_fn, output) + merge_fn, args=(output,)) def value_container(val): @@ -1452,14 +1777,11 @@ class AggregatingVariable(checkpointable.CheckpointableBase): "a variable in Replica Context.") def merge_fn(strategy, value, *other_args, **other_kwargs): - return strategy.update( - self, f, - strategy.reduce( - aggregation=self._aggregation, value=value, destinations=self), - *other_args, **other_kwargs) + v = _apply_aggregation(strategy, value, self._aggregation, self) + return strategy.update(self, f, v, *other_args, **other_kwargs) return distribution_strategy_context.get_replica_context().merge_call( - merge_fn, *args, **kwargs) + merge_fn, args=args, kwargs=kwargs) def assign_sub(self, *args, **kwargs): assign_sub_fn = lambda var, *a, **kw: var.assign_sub(*a, **kw) diff --git a/tensorflow/python/eager/BUILD b/tensorflow/python/eager/BUILD index 362e8e3b832..f43cf9327a1 100644 --- a/tensorflow/python/eager/BUILD +++ b/tensorflow/python/eager/BUILD @@ -174,6 +174,23 @@ cuda_py_test( ], ) +cuda_py_test( + name = "function_gradients_test", + size = "medium", + srcs = ["function_gradients_test.py"], + additional_deps = [ + ":backprop", + ":context", + ":def_function", + ":function", + ":test", + "@absl_py//absl/testing:parameterized", + "//tensorflow/python:math_ops", + "//tensorflow/python:resource_variable_ops", + ], + shard_count = 5, +) + cuda_py_test( name = "function_test", size = "medium", @@ -193,7 +210,7 @@ cuda_py_test( "//tensorflow/python:math_ops", "//tensorflow/python:resource_variable_ops", ], - shard_count = 20, + shard_count = 15, ) py_library( @@ -238,6 +255,18 @@ py_library( ], ) +py_test( + name = "execution_callbacks_test", + srcs = ["execution_callbacks_test.py"], + srcs_version = "PY2AND3", + deps = [ + ":execution_callbacks", + "//tensorflow/python:client_testlib", + "//tensorflow/python:framework_ops", + "//tensorflow/python:math_ops", + ], +) + py_library( name = "graph_only_ops", srcs = ["graph_only_ops.py"], @@ -318,6 +347,7 @@ py_library( "//tensorflow/python/eager:context", "//tensorflow/python/eager:execute", "//tensorflow/python/eager:tape", + "//tensorflow/python/ops/parallel_for:control_flow_ops", "@six_archive//:six", ], ) diff --git a/tensorflow/python/eager/backprop.py b/tensorflow/python/eager/backprop.py index 844c9b52e7f..29f9b2cda3a 100644 --- a/tensorflow/python/eager/backprop.py +++ b/tensorflow/python/eager/backprop.py @@ -20,6 +20,7 @@ from __future__ import print_function import functools import operator +import sys import six @@ -33,6 +34,7 @@ from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops from tensorflow.python.framework import tensor_shape from tensorflow.python.ops import array_ops +from tensorflow.python.ops import check_ops from tensorflow.python.ops import gen_array_ops from tensorflow.python.ops import gen_math_ops from tensorflow.python.ops import math_ops @@ -42,9 +44,20 @@ from tensorflow.python.platform import tf_logging as logging from tensorflow.python.util import nest from tensorflow.python.util import tf_contextlib from tensorflow.python.util import tf_inspect +from tensorflow.python.util.lazy_loader import LazyLoader from tensorflow.python.util.tf_export import tf_export +# Note that we need to lazy load the following two modules to avoid creating +# circular dependencies. +# TODO(b/119775953): fix the circular dependencies. +pfor_ops = LazyLoader( + "pfor_ops", globals(), + "tensorflow.python.ops.parallel_for.control_flow_ops") + +function = LazyLoader("function", globals(), + "tensorflow.python.eager.function") + _op_attr_type_cache = {} @@ -536,11 +549,11 @@ def _aggregate_grads(gradients): if len(gradients) == 1: return gradients[0] - if all([isinstance(g, ops.Tensor) for g in gradients]): + if all(isinstance(g, ops.Tensor) for g in gradients): return gen_math_ops.add_n(gradients) else: - assert all([isinstance(g, (ops.Tensor, ops.IndexedSlices)) - for g in gradients]) + assert all(isinstance(g, (ops.Tensor, ops.IndexedSlices)) + for g in gradients) indexed_slices_list = [] for grad in gradients: # TODO(xpan): Support nested IndexedSlices and core IndexedSlices @@ -776,6 +789,8 @@ class GradientTape(object): context.context().end_step() except AttributeError: pass + except TypeError: + pass def watch(self, tensor): """Ensures that `tensor` is being traced by this tape. @@ -935,3 +950,213 @@ class GradientTape(object): grad = nest.pack_sequence_as(sources, flat_grad) return grad + + def jacobian(self, + target, + sources, + unconnected_gradients=UnconnectedGradients.NONE, + parallel_iterations=None, + experimental_use_pfor=True): + """Computes the jacobian using operations recorded in context of this tape. + + See http://en.wikipedia.org/wiki/jacobian_matrix_and_determinant for the + definition of a Jacobian. + + Example usage: + + with tf.GradientTape() as g: + x = tf.constant([1.0, 2.0]) + g.watch(x) + y = x * x + jacobian = g.jacobian(y, x) + # jacobian value is [[2., 0.], [0., 4.]] + + Args: + target: Tensor to be differentiated. + sources: a list or nested structure of Tensors or Variables. `target` + will be differentiated against elements in `sources`. + unconnected_gradients: a value which can either hold 'none' or 'zero' and + alters the value which will be returned if the target and sources are + unconnected. The possible values and effects are detailed in + 'UnconnectedGradients' and it defaults to 'none'. + parallel_iterations: A knob to control how many iterations are dispatched + in parallel. This knob can be used to control the total memory usage. + experimental_use_pfor: If true, vectorizes the jacobian computation. Else + falls back to a sequential while_loop. Vectorization can sometimes fail + or lead to excessive memory usage. This option can be used to disable + vectorization in such cases. + + Returns: + a list or nested structure of Tensors (or IndexedSlices, or None), + one for each element in `sources`. Returned structure is the same as + the structure of `sources`. + + Raises: + RuntimeError: If called on a non-persistent tape with eager execution + enabled and without enabling experimental_use_pfor. + ValueError: If vectorization of jacobian computation fails. + """ + flat_sources = nest.flatten(sources) + target_static_shape = target.shape + target_shape = array_ops.shape(target) + # Note that we push and pop the tape here and below. This is needed since we + # need gradients through the enclosed operations. + self._push_tape() + target = array_ops.reshape(target, [-1]) + self._pop_tape() + + def loop_fn(i): + self._push_tape() + y = array_ops.gather(target, i) + self._pop_tape() + return self.gradient(y, flat_sources, + unconnected_gradients=unconnected_gradients) + + try: + target_size = int(target.shape[0]) + except TypeError: + target_size = array_ops.shape(target)[0] + + if experimental_use_pfor: + try: + output = pfor_ops.pfor(loop_fn, target_size, + parallel_iterations=parallel_iterations) + except ValueError as err: + six.reraise( + ValueError, + ValueError( + str(err) + "\nEncountered an exception while vectorizing the " + "jacobian computation. Vectorization can be disabled by setting" + " experimental_use_pfor to False."), + sys.exc_info()[2]) + else: + if context.executing_eagerly() and not self._persistent: + raise RuntimeError( + "GradientTape must be created with persistent=True" + " to compute the jacobian with eager execution enabled and with " + " experimental_use_pfor set to False.") + output = pfor_ops.for_loop( + loop_fn, [target.dtype] * len(flat_sources), target_size, + parallel_iterations=parallel_iterations) + + for i, out in enumerate(output): + if out is not None: + new_shape = array_ops.concat( + [target_shape, array_ops.shape(out)[1:]], axis=0) + out = array_ops.reshape(out, new_shape) + if context.executing_eagerly(): + out.set_shape(target_static_shape.concatenate(flat_sources[i].shape)) + output[i] = out + + return nest.pack_sequence_as(sources, output) + + def batch_jacobian(self, + target, + source, + unconnected_gradients=UnconnectedGradients.NONE, + parallel_iterations=None, + experimental_use_pfor=True): + """Computes and stacks per-example jacobians. + + See http://en.wikipedia.org/wiki/jacobian_matrix_and_determinant for the + definition of a Jacobian. This function is essentially an efficient + implementation of the following: + `tf.stack([self.jacobian(y[i], x[i]) for i in range(x.shape[0])])`. + + Note that compared to `GradientTape.jacobian` which computes gradient of + each output value w.r.t each input value, this function is useful when + `target[i,...] is independent of `source[j,...]` for `j != i`. This + independence assumption allows more efficient computation as compared to + `GradientTape.jacobian`. The output, as well as intermediate activations, + are lower dimensional and avoid a bunch of redundant zeros which would + result in the jacobian computation given the independence assumption. + + Example usage: + with tf.GradientTape() as g: + x = tf.constant([[1, 2], [3, 4]], dtype=tf.float32) + g.watch(x) + y = x * x + batch_jacobian = g.batch_jacobian(y, x) + # batch_jacobian is [[[2, 0], [0, 4]], [[6, 0], [0, 8]]] + + Args: + target: A tensor with rank 2 or higher and with shape [b, y1, ..., y_n]. + `target[i,...]` should only depend on `source[i,...]`. + source: A tensor with rank 2 or higher and with shape [b, x1, ..., x_m]. + unconnected_gradients: a value which can either hold 'none' or 'zero' and + alters the value which will be returned if the target and sources are + unconnected. The possible values and effects are detailed in + 'UnconnectedGradients' and it defaults to 'none'. + parallel_iterations: A knob to control how many iterations are dispatched + in parallel. This knob can be used to control the total memory usage. + experimental_use_pfor: If true, uses pfor for computing the Jacobian. Else + uses a tf.while_loop. + + Returns: + A tensor `t` with shape [b, y_1, ..., y_n, x1, ..., x_m] where `t[i, ...]` + is the jacobian of `target[i, ...]` w.r.t. `source[i, ...]`, i.e. stacked + per-example jacobians. + + Raises: + RuntimeError: If called on a non-persistent tape with eager execution + enabled and without enabling experimental_use_pfor. + ValueError: If vectorization of jacobian computation fails or if first + dimension of `target` and `source` do not match. + """ + target_shape = target.shape + if not target_shape.with_rank_at_least(2)[0].is_compatible_with( + source.shape.with_rank_at_least(2)[0]): + raise ValueError( + "Need first dimension of target shape (%s) and " + "source shape (%s) to match." % (target.shape, source.shape)) + if target_shape.is_fully_defined(): + batch_size = int(target_shape[0]) + target_row_size = target_shape.num_elements() // batch_size + else: + target_shape = array_ops.shape(target) + batch_size = target_shape[0] + target_row_size = array_ops.size(target) // batch_size + source_shape = array_ops.shape(source) + # Flatten target to 2-D. + # Note that we push and pop the tape here and below. This is needed since we + # need gradients through the enclosed operations. + self._push_tape() + with ops.control_dependencies( + [check_ops.assert_equal(batch_size, source_shape[0])]): + target = array_ops.reshape(target, [batch_size, target_row_size]) + self._pop_tape() + + def loop_fn(i): + self._push_tape() + y = array_ops.gather(target, i, axis=1) + self._pop_tape() + return self.gradient(y, source, + unconnected_gradients=unconnected_gradients) + + if experimental_use_pfor: + try: + output = pfor_ops.pfor(loop_fn, target_row_size, + parallel_iterations=parallel_iterations) + except ValueError as err: + six.reraise( + ValueError, + ValueError( + str(err) + "\nEncountered an exception while vectorizing the " + "batch_jacobian computation. Vectorization can be disabled by " + "setting experimental_use_pfor to False."), + sys.exc_info()[2]) + else: + if context.executing_eagerly() and not self._persistent: + raise RuntimeError( + "GradientTape must be created with persistent=True" + " to compute the batch_jacobian with eager execution enabled and " + " with experimental_use_pfor set to False.") + output = pfor_ops.for_loop(loop_fn, target.dtype, target_row_size, + parallel_iterations=parallel_iterations) + if output is None: + return None + output = array_ops.reshape(output, + [target_row_size, batch_size, -1]) + output = array_ops.transpose(output, [1, 0, 2]) + new_shape = array_ops.concat([target_shape, source_shape[1:]], axis=0) + return array_ops.reshape(output, new_shape) diff --git a/tensorflow/python/eager/backprop_test.py b/tensorflow/python/eager/backprop_test.py index 274d5320df0..3cec40a48f7 100644 --- a/tensorflow/python/eager/backprop_test.py +++ b/tensorflow/python/eager/backprop_test.py @@ -74,7 +74,7 @@ class BackpropTest(test.TestCase): tf_g1 = embedding_ops.embedding_lookup(tf_var, tf_ind1) tf_g2 = embedding_ops.embedding_lookup(tf_var, tf_ind2) tf_g3 = embedding_ops.embedding_lookup(tf_var, tf_ind3) - tf_g4 = math_ops.reduce_sum(tf_var * 2.0, reduction_indices=(0, 1)) + tf_g4 = math_ops.reduce_sum(tf_var * 2.0, axis=(0, 1)) tf_y = tf_g1 * tf_g2 * tf_g3 * tf_g4 tf_grad = gradients.gradients(tf_y, [tf_var])[0] @@ -215,7 +215,7 @@ class BackpropTest(test.TestCase): self.assertAllClose(tf_grad.values.eval(), grad.values) tf_opt.apply_gradients([(tf_grad, tf_embedding)]).run() - expected = tf_embedding.eval() + expected = self.evaluate(tf_embedding) opt.apply_gradients([(grad, embedding)]) self.assertAllClose(expected, embedding.read_value()) @@ -233,6 +233,68 @@ class BackpropTest(test.TestCase): self.assertTrue(ordered_variables[0] is v0) self.assertTrue(ordered_variables[1] is v1) + def testTapeNoOpGradient(self): + x = constant_op.constant(3.0) + with backprop.GradientTape() as t: + t.watch(x) + y = x + self.assertEqual(t.gradient(y, x).numpy(), 1.0) + + def testTapeIdentityGradientIsIdentity(self): + x = constant_op.constant(3.0) + with backprop.GradientTape() as t: + t.watch(x) + y = array_ops.identity(x) + self.assertEqual(t.gradient(y, x).numpy(), 1.0) + + def testTapeGradientMultiTargetOneIsSource(self): + x = constant_op.constant(2.0) + with backprop.GradientTape() as t: + t.watch(x) + y = x*x + self.assertEqual(t.gradient([x, y], x).numpy(), 5.0) + + def testTapeNoOpGradientWithMultiTargetAllSource(self): + x = constant_op.constant(3.0) + with backprop.GradientTape() as t: + t.watch(x) + y = x + self.assertEqual(t.gradient([y, y], x).numpy(), 2.0) + + def testTapeNoOpGradientWithMultiTargetMultiSource(self): + x = constant_op.constant(3.0) + y = constant_op.constant(5.0) + with backprop.GradientTape() as t: + t.watch(x) + t.watch(y) + z = y * y + self.assertAllEqual(t.gradient([x, y, z], [x, y]), [1.0, 11.0]) + + def testTapeNoOpOnVariableIsIdentity(self): + v0 = resource_variable_ops.ResourceVariable(1.0) + with backprop.GradientTape() as t: + y = v0.read_value() + self.assertEqual(t.gradient(y, v0).numpy(), 1.0) + + @test_util.assert_no_new_tensors + @test_util.assert_no_garbage_created + def testTapeNoOpGradient2By2(self): + a_2_by_2 = constant_op.constant(2.0, shape=[2, 2]) + with backprop.GradientTape(persistent=True) as tape: + tape.watch(a_2_by_2) + dy_dy = tape.gradient(a_2_by_2, [a_2_by_2])[0] + self.assertAllEqual(dy_dy.numpy(), + constant_op.constant(1.0, shape=[2, 2]).numpy()) + + @test_util.assert_no_new_pyobjects_executing_eagerly + def testTapeNoOpGradientMultiTarget2By2(self): + a_2_by_2 = constant_op.constant(2.0, shape=[2, 2]) + with backprop.GradientTape(persistent=True) as tape: + tape.watch(a_2_by_2) + dy_dy = tape.gradient([a_2_by_2, a_2_by_2], [a_2_by_2])[0] + self.assertAllEqual(dy_dy.numpy(), + constant_op.constant(2.0, shape=[2, 2]).numpy()) + def testTapeStopRecording(self): with backprop.GradientTape() as t: x = resource_variable_ops.ResourceVariable(1.0) @@ -1165,5 +1227,192 @@ class BackpropTest(test.TestCase): self.assertAllEqual(da[0], tf_da[0].eval()) +@test_util.run_all_in_graph_and_eager_modes +class JacobianTest(test.TestCase): + + def _jacobian(self, experimental_use_pfor): + persistent = context.executing_eagerly and not experimental_use_pfor + with backprop.GradientTape(persistent=persistent) as g: + x = constant_op.constant([1., 2.]) + y = constant_op.constant([3., 4.]) + g.watch(x) + g.watch(y) + z = x * x * y + jacobian = g.jacobian(z, [x, y], + experimental_use_pfor=experimental_use_pfor) + answer = [array_ops.diag(2 * x * y), array_ops.diag(x * x)] + return jacobian, answer + + def testPfor(self): + jacobian, answer = self._jacobian(experimental_use_pfor=True) + for j, a in zip(jacobian, answer): + self.assertAllEqual(a, j) + + def testWhileLoop(self): + jacobian, answer = self._jacobian(experimental_use_pfor=False) + for j, a in zip(jacobian, answer): + self.assertAllEqual(a, j) + + def testPforDefun(self): + + @function.defun + def _f(): + return self._jacobian(experimental_use_pfor=True) + + jacobian, answer = _f() + for j, a in zip(jacobian, answer): + self.assertAllEqual(a, j) + + def testWhileLoopDefun(self): + + @function.defun + def _f(): + return self._jacobian(experimental_use_pfor=False) + + jacobian, answer = _f() + for j, a in zip(jacobian, answer): + self.assertAllEqual(a, j) + + def testPersistentTape(self): + if not context.executing_eagerly(): + return + with backprop.GradientTape() as g: + x = constant_op.constant([1.0, 2.0]) + g.watch(x) + y = x * x + with self.assertRaisesRegexp(RuntimeError, 'persistent'): + g.jacobian(y, x, experimental_use_pfor=False) + + def testPforException(self): + var = variables.Variable([1.]) + + @custom_gradient.custom_gradient + def op(x): + def grad(_): + # Note that we perform a stateful operation here that will not be + # compatible with parallel for construct. + with ops.control_dependencies( + [var.assign(random_ops.random_uniform([1]))]): + return constant_op.constant(1.) + return x, grad + + with backprop.GradientTape() as g: + x = constant_op.constant([1., 2.]) + g.watch(x) + y = op(x) + with self.assertRaisesRegexp(ValueError, 'No converter'): + g.jacobian(y, x, experimental_use_pfor=True) + + def test_parallel_iterations(self): + with backprop.GradientTape(persistent=True) as g: + x = constant_op.constant([[1., 2], [3, 4]]) + g.watch(x) + y = math_ops.matmul(x, x) + self.assertAllClose(g.jacobian(y, x, parallel_iterations=2), + g.jacobian(y, x, parallel_iterations=3)) + + +@test_util.run_all_in_graph_and_eager_modes +class BatchJacobianTest(test.TestCase): + + def _batch_jacobian(self, experimental_use_pfor): + persistent = context.executing_eagerly and not experimental_use_pfor + with backprop.GradientTape(persistent=persistent) as g: + x = constant_op.constant([[1., 2.], [3., 4.]]) + y = constant_op.constant([[3., 4.], [5., 6.]]) + g.watch(x) + z = x * x * y + batch_jacobian = g.batch_jacobian( + z, x, experimental_use_pfor=experimental_use_pfor) + answer = array_ops.stack([array_ops.diag(2 * x[0] * y[0]), + array_ops.diag(2 * x[1] * y[1])]) + return batch_jacobian, answer + + def testPfor(self): + batch_jacobian, answer = self._batch_jacobian(experimental_use_pfor=True) + self.assertAllEqual(answer, batch_jacobian) + + def testWhileLoop(self): + batch_jacobian, answer = self._batch_jacobian(experimental_use_pfor=False) + self.assertAllEqual(answer, batch_jacobian) + + def testPforDefun(self): + + @function.defun + def _f(): + return self._batch_jacobian(experimental_use_pfor=True) + + batch_jacobian, answer = _f() + self.assertAllEqual(answer, batch_jacobian) + + def testWhileLoopDefun(self): + + @function.defun + def _f(): + return self._batch_jacobian(experimental_use_pfor=False) + + batch_jacobian, answer = _f() + self.assertAllEqual(answer, batch_jacobian) + + def testPersistentTape(self): + if not context.executing_eagerly(): + return + with backprop.GradientTape() as g: + x = constant_op.constant([[1.0, 2.0]]) + g.watch(x) + y = x * x + with self.assertRaisesRegexp(RuntimeError, 'persistent'): + g.batch_jacobian(y, x, experimental_use_pfor=False) + + def testBadShape(self): + x = random_ops.random_uniform([2, 3]) + with backprop.GradientTape() as g: + y = array_ops.concat([x, x], axis=0) + with self.assertRaisesRegexp(ValueError, 'Need first dimension'): + g.batch_jacobian(y, x) + + def testBadInputRank(self): + x = random_ops.random_uniform([2]) + with backprop.GradientTape() as g: + y = random_ops.random_uniform([2, 2]) + with self.assertRaisesRegexp(ValueError, 'must have rank at least 2'): + g.batch_jacobian(y, x) + + def testBadOutputRank(self): + x = random_ops.random_uniform([2, 2]) + with backprop.GradientTape() as g: + y = random_ops.random_uniform([2]) + with self.assertRaisesRegexp(ValueError, 'must have rank at least 2'): + g.batch_jacobian(y, x) + + def testPforException(self): + var = variables.Variable([1.]) + + @custom_gradient.custom_gradient + def op(x): + def grad(_): + # Note that we perform a stateful operation here that will not be + # compatible with parallel for construct. + with ops.control_dependencies( + [var.assign(random_ops.random_uniform([1]))]): + return constant_op.constant(1.) + return x, grad + + with backprop.GradientTape() as g: + x = constant_op.constant([[1.], [2.]]) + g.watch(x) + y = op(x) + with self.assertRaisesRegexp(ValueError, 'No converter'): + g.batch_jacobian(y, x, experimental_use_pfor=True) + + def test_parallel_iterations(self): + with backprop.GradientTape(persistent=True) as g: + x = constant_op.constant([[1., 2], [3, 4]]) + g.watch(x) + w = constant_op.constant([[1., 2, 3, 4], [5, 6, 7, 8]]) + y = math_ops.matmul(x, w) + self.assertAllClose(g.batch_jacobian(y, x, parallel_iterations=2), + g.batch_jacobian(y, x, parallel_iterations=3)) + if __name__ == '__main__': test.main() diff --git a/tensorflow/python/eager/benchmarks_test.py b/tensorflow/python/eager/benchmarks_test.py index 886715867c8..31a7efca82b 100644 --- a/tensorflow/python/eager/benchmarks_test.py +++ b/tensorflow/python/eager/benchmarks_test.py @@ -80,7 +80,6 @@ class SubclassedKerasModel(keras.Model): def __init__(self, initializer="ones"): super(SubclassedKerasModel, self).__init__() - self._can_use_graph_functions = True self.layer_a = keras.layers.Dense( 64, kernel_initializer=initializer, bias_initializer="zeros") self.layer_b = keras.layers.Dense( @@ -733,38 +732,38 @@ class MicroBenchmarks(test.Benchmark): assert np.equal(func(), make_keras_model()(data)).all() self._run(func, 30000) - def _benchmark_keras_model_fit(self, model): + def _benchmark_keras_model_fit(self, model, run_eagerly=False): data = random_ops.random_uniform((10, 10), minval=-1, maxval=1) labels = random_ops.random_uniform((10, 10), minval=-1, maxval=1) dataset = dataset_ops.Dataset.from_tensors((data, labels)).repeat() model.compile( gradient_descent.GradientDescentOptimizer(learning_rate=0.001), - loss="mse") + loss="mse", run_eagerly=run_eagerly) func = lambda: model.fit(dataset, epochs=1, steps_per_epoch=1000, verbose=0) # First call is more expensive (creates variables etc.), discount that. model.fit(dataset, epochs=1, steps_per_epoch=1, verbose=0) self._run(func, 1) - def _benchmark_keras_model_evaluate(self, model): + def _benchmark_keras_model_evaluate(self, model, run_eagerly=False): data = random_ops.random_uniform((10, 10), minval=-1, maxval=1) labels = random_ops.random_uniform((10, 10), minval=-1, maxval=1) dataset = dataset_ops.Dataset.from_tensors((data, labels)).repeat() model.compile( gradient_descent.GradientDescentOptimizer(learning_rate=0.001), - loss="mse") + loss="mse", run_eagerly=run_eagerly) func = lambda: model.evaluate(dataset, steps=1000, verbose=0) # First call is more expensive (creates variables etc.), discount that. model.evaluate(dataset, steps=1, verbose=0) self._run(func, 1) - def _benchmark_keras_model_predict(self, model): + def _benchmark_keras_model_predict(self, model, run_eagerly=False): data = random_ops.random_uniform((10, 10), minval=-1, maxval=1) dataset = dataset_ops.Dataset.from_tensors(tuple([data])).repeat() model.compile( gradient_descent.GradientDescentOptimizer(learning_rate=0.001), - loss="mse") + loss="mse", run_eagerly=run_eagerly) func = lambda: model.predict(dataset, steps=1000, verbose=0) # First call is more expensive (creates variables etc.), discount that. model.predict(dataset, steps=1, verbose=0) @@ -780,10 +779,9 @@ class MicroBenchmarks(test.Benchmark): model = SubclassedKerasModel(initializer="glorot_uniform") self._benchmark_keras_model_fit(model) - def benchmark_keras_model_subclassed_fit_disable_defun(self): + def benchmark_keras_model_subclassed_fit_run_model_eagerly(self): model = SubclassedKerasModel(initializer="glorot_uniform") - model._can_use_graph_functions = False - self._benchmark_keras_model_fit(model) + self._benchmark_keras_model_fit(model, run_eagerly=True) def benchmark_keras_model_functional_fit(self): model = make_keras_model(initializer="glorot_uniform") @@ -794,10 +792,9 @@ class MicroBenchmarks(test.Benchmark): model = make_keras_model(initializer="glorot_uniform") self._benchmark_keras_model_fit(model) - def benchmark_keras_model_functional_fit_disable_defun(self): + def benchmark_keras_model_functional_fit_run_model_eagerly(self): model = make_keras_model(initializer="glorot_uniform") - model._can_use_graph_functions = False - self._benchmark_keras_model_fit(model) + self._benchmark_keras_model_fit(model, run_eagerly=True) def benchmark_keras_model_sequential_fit(self): model = make_sequential_keras_model(initializer="glorot_uniform") @@ -808,64 +805,57 @@ class MicroBenchmarks(test.Benchmark): model = make_sequential_keras_model(initializer="glorot_uniform") self._benchmark_keras_model_fit(model) - def benchmark_keras_model_sequential_fit_disable_defun(self): + def benchmark_keras_model_sequential_fit_run_model_eagerly(self): model = make_sequential_keras_model(initializer="glorot_uniform") - model._can_use_graph_functions = False - self._benchmark_keras_model_fit(model) + self._benchmark_keras_model_fit(model, run_eagerly=True) def benchmark_keras_model_subclassed_evaluate(self): model = SubclassedKerasModel(initializer="glorot_uniform") self._benchmark_keras_model_evaluate(model) - def benchmark_keras_model_subclassed_evaluate_disable_defun(self): + def benchmark_keras_model_subclassed_evaluate_run_model_eagerly(self): model = SubclassedKerasModel(initializer="glorot_uniform") - model._can_use_graph_functions = False - self._benchmark_keras_model_evaluate(model) + self._benchmark_keras_model_evaluate(model, run_eagerly=True) def benchmark_keras_model_functional_evaluate(self): model = make_keras_model(initializer="glorot_uniform") self._benchmark_keras_model_evaluate(model) - def benchmark_keras_model_functional_evaluate_disable_defun(self): + def benchmark_keras_model_functional_evaluate_run_model_eagerly(self): model = make_keras_model(initializer="glorot_uniform") - model._can_use_graph_functions = False - self._benchmark_keras_model_evaluate(model) + self._benchmark_keras_model_evaluate(model, run_eagerly=True) def benchmark_keras_model_sequential_evaluate(self): model = make_sequential_keras_model(initializer="glorot_uniform") self._benchmark_keras_model_evaluate(model) - def benchmark_keras_model_sequential_evaluate_disable_defun(self): + def benchmark_keras_model_sequential_evaluate_run_model_eagerly(self): model = make_sequential_keras_model(initializer="glorot_uniform") - model._can_use_graph_functions = False - self._benchmark_keras_model_evaluate(model) + self._benchmark_keras_model_evaluate(model, run_eagerly=True) def benchmark_keras_model_subclassed_predict(self): model = SubclassedKerasModel(initializer="glorot_uniform") self._benchmark_keras_model_predict(model) - def benchmark_keras_model_subclassed_predict_disable_defun(self): + def benchmark_keras_model_subclassed_predict_run_model_eagerly(self): model = SubclassedKerasModel(initializer="glorot_uniform") - model._can_use_graph_functions = False - self._benchmark_keras_model_predict(model) + self._benchmark_keras_model_predict(model, run_eagerly=True) def benchmark_keras_model_functional_predict(self): model = make_keras_model(initializer="glorot_uniform") self._benchmark_keras_model_predict(model) - def benchmark_keras_model_functional_predict_disable_defun(self): + def benchmark_keras_model_functional_predict_run_model_eagerly(self): model = make_keras_model(initializer="glorot_uniform") - model._can_use_graph_functions = False - self._benchmark_keras_model_predict(model) + self._benchmark_keras_model_predict(model, run_eagerly=True) def benchmark_keras_model_sequential_predict(self): model = make_sequential_keras_model(initializer="glorot_uniform") self._benchmark_keras_model_predict(model) - def benchmark_keras_model_sequential_predict_disable_defun(self): + def benchmark_keras_model_sequential_predict_run_model_eagerly(self): model = make_sequential_keras_model(initializer="glorot_uniform") - model._can_use_graph_functions = False - self._benchmark_keras_model_predict(model) + self._benchmark_keras_model_predict(model, run_eagerly=True) def benchmarkScan(self): elems = math_ops.range(1600) diff --git a/tensorflow/python/eager/context.py b/tensorflow/python/eager/context.py index e3fef524bf9..2f6b038dda9 100644 --- a/tensorflow/python/eager/context.py +++ b/tensorflow/python/eager/context.py @@ -25,7 +25,6 @@ import random import threading from tensorflow.core.protobuf import config_pb2 -from tensorflow.core.protobuf import rewriter_config_pb2 from tensorflow.python import pywrap_tensorflow from tensorflow.python import tf2 from tensorflow.python.framework import c_api_util @@ -86,21 +85,21 @@ class FunctionCallOptions(object): Eager functions are functions decorated with tf.contrib.eager.defun. """ - def __init__(self, executor_type=None, rewriter_config=None): + def __init__(self, executor_type=None, config_proto=None): """Constructor. Args: executor_type: (optional) name of the executor to be used to execute the eager function. If None or an empty string, the default Tensorflow executor will be used. - rewriter_config: (optional) a rewriter_config_pb2.RewriterConfig proto or + config_proto: (optional) a `config_pb2.ConfigProto` proto or a serialized string of that proto. The config used by Grappler when optimizing the function graph. Each concrete function is optimized the first time is called. Changing - rewriter_config after the first call has no effect. - If rewriter_config is None, an empty RewriterConfig will be used. + config_proto after the first call has no effect. + If config_proto is None, an empty RewriterConfig will be used. """ - self.rewriter_config_serialized = rewriter_config + self.config_proto_serialized = config_proto self.executor_type = executor_type @property @@ -112,24 +111,22 @@ class FunctionCallOptions(object): self._executor_type = executor_type @property - def rewriter_config_serialized(self): - return self._rewriter_config_serialized + def config_proto_serialized(self): + return self._config_proto_serialized - @rewriter_config_serialized.setter - def rewriter_config_serialized(self, config): - if isinstance(config, rewriter_config_pb2.RewriterConfig): - self._rewriter_config_serialized = config.SerializeToString() + @config_proto_serialized.setter + def config_proto_serialized(self, config): + if isinstance(config, config_pb2.ConfigProto): + self._config_proto_serialized = config.SerializeToString() elif isinstance(config, str): - self._rewriter_config_serialized = config + self._config_proto_serialized = config elif config is None: - self._rewriter_config_serialized = rewriter_config_pb2.RewriterConfig( - ).SerializeToString() + self._config_proto_serialized = ( + config_pb2.ConfigProto().SerializeToString()) else: - raise ValueError( - "the rewriter config must be either a " - "rewriter_config_pb2.RewriterConfig, or a serialized string of that " - "proto or None. got: {}" - .format(type(config))) + raise ValueError("the rewriter config must be either a " + "config_pb2.ConfigProto, or a serialized string of that " + "proto or None. got: {}".format(type(config))) # TODO(agarwal): better name ? @@ -152,14 +149,12 @@ class _EagerContext(threading.local): # Default rewriter config corresponds to turning all default grappler # optimizations on. - base_config = rewriter_config_pb2.RewriterConfig() + base_config = config_pb2.ConfigProto() - if config is not None and config.HasField( - "graph_options") and config.graph_options.HasField("rewrite_options"): - base_config.Merge(config.graph_options.rewrite_options) + if config is not None: + base_config.MergeFrom(config) - self.function_call_options = FunctionCallOptions( - rewriter_config=base_config) + self.function_call_options = FunctionCallOptions(config_proto=base_config) ContextSwitch = collections.namedtuple( @@ -897,21 +892,21 @@ def export_run_metadata(): return context().export_run_metadata() -def function_rewriter_config(rewriter_config): +def function_config_proto(config_proto): """Context manager for setting the grappler rewrite config. This config is used by Grappler when optimizing the function graph. Args: - rewriter_config: a rewriter_config_pb2.RewriterConfig proto or + config_proto: a `config_pb2.ConfigProto` proto or a serialized string of that proto or None. If None, the default instance - of rewriter_config_pb2.RewriterConfig will be used. + of `config_pb2.ConfigProto` will be used. Returns: A context manager. """ def _set_options_func(options): - options.rewriter_config_serialized = rewriter_config + options.config_proto_serialized = config_proto return context().function_call_options(_set_options_func) diff --git a/tensorflow/python/eager/def_function.py b/tensorflow/python/eager/def_function.py index cad6721c702..6bacd7a962f 100644 --- a/tensorflow/python/eager/def_function.py +++ b/tensorflow/python/eager/def_function.py @@ -214,7 +214,7 @@ class PolymorphicFunction(object): python_function, name, input_signature=None, - autograph=False, + autograph=True, experimental_autograph_options=None): """Initializes a polymorphic function. @@ -503,7 +503,7 @@ class PolymorphicFunction(object): @tf_export("function", v1=[]) def function(func=None, input_signature=None, - autograph=False, + autograph=True, experimental_autograph_options=None): """Creates a callable TensorFlow graph from a Python function. @@ -552,9 +552,9 @@ def function(func=None, return x + tf.to_float(c) assert int(c) == 0 - assert f(1.0) == 3.0 + assert f(1.0) == 2.0 assert int(c) == 1 - assert f(1.0) == 4.0 + assert f(1.0) == 3.0 assert int(c) == 2 ``` diff --git a/tensorflow/python/eager/def_function_test.py b/tensorflow/python/eager/def_function_test.py index f0f71a219e6..4100a10044c 100644 --- a/tensorflow/python/eager/def_function_test.py +++ b/tensorflow/python/eager/def_function_test.py @@ -17,6 +17,7 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function +import functools from tensorflow.python.eager import backprop from tensorflow.python.eager import def_function @@ -149,9 +150,9 @@ class DefFunctionTest(test.TestCase): result = fn(3.0) - sess.run(variables.global_variables_initializer()) + self.evaluate(variables.global_variables_initializer()) self.assertAllEqual(sess.run(state[0]), 2.0) - self.assertAllEqual(sess.run(result), 6.0) + self.assertAllEqual(self.evaluate(result), 6.0) def testLegacyGraphModeVariablesNonTrivialInitializer(self): with ops.Graph().as_default(), self.test_session() as sess: @@ -168,9 +169,9 @@ class DefFunctionTest(test.TestCase): result = fn(3.0) - sess.run(variables.global_variables_initializer()) + self.evaluate(variables.global_variables_initializer()) self.assertAllEqual(sess.run(state[0]), 6.0) - self.assertAllEqual(sess.run(result), 18.0) + self.assertAllEqual(self.evaluate(result), 18.0) def testLegacyGraphModeInputDependentInitializerFails(self): with ops.Graph().as_default(): @@ -207,6 +208,18 @@ class DefFunctionTest(test.TestCase): m1 = MyModel() self.assertAllEqual(m1.apply(3.0), 6.0) + def test_functools_partial(self): + self.assertAllClose( + 3., + def_function.function(functools.partial(lambda x, y: x + y, 1.))( + constant_op.constant(2.))) + + def test_unspecified_default_argument(self): + wrapped = def_function.function( + lambda x, y=2: x + y, + input_signature=[tensor_spec.TensorSpec((), dtypes.int32)]) + self.assertEqual(3, wrapped(constant_op.constant(1)).numpy()) + def test_optimizer(self): x = constant_op.constant([[3., 4.]]) y = constant_op.constant([2.]) diff --git a/tensorflow/python/eager/execution_callbacks.py b/tensorflow/python/eager/execution_callbacks.py index 80ff4459d60..28b6b84a82c 100644 --- a/tensorflow/python/eager/execution_callbacks.py +++ b/tensorflow/python/eager/execution_callbacks.py @@ -18,6 +18,7 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function +import contextlib import functools import numpy as np @@ -28,8 +29,13 @@ from tensorflow.python.eager import core from tensorflow.python.eager import execute from tensorflow.python.platform import tf_logging as logging -_DEFAULT_CALLBACK_ACTION = "raise" -_VALID_CALLBACK_ACTIONS = (None, "ignore", "print", "raise", "warn") +IGNORE = "ignore" +PRINT = "print" +RAISE = "raise" +WARN = "warn" + +_DEFAULT_CALLBACK_ACTION = RAISE +_VALID_CALLBACK_ACTIONS = (None, IGNORE, PRINT, RAISE, WARN) # TODO(cais): Consider moving this exception class to errors_impl.py. @@ -335,3 +341,38 @@ def seterr(inf_or_nan=None): functools.partial(inf_nan_callback, action=inf_or_nan)) return old_settings + + +@contextlib.contextmanager +def errstate(inf_or_nan=None): + """Context manager setting error state. + + Example: + ``` + c = tf.log(0.) # -inf + + with errstate(inf_or_nan="raise"): + tf.log(0.) # <-- Raises InfOrNanError. + ``` + + Args: + inf_or_nan: Set action for infinity (`inf`) and NaN (`nan`) values. + Possible values: `{IGNORE, PRINT, RAISE, WARN}`. + `IGNORE`: take no action when `inf` values appear. + `PRINT`: print a warning to `stdout`. + `RAISE`: raise an `InfOrNanError`. + `WARN`: print a warning using `tf.logging.warn`. + A value of `None` leads to no change in the action of the condition. + + Yields: + None. + + Raises: + ValueError: If the value of any keyword arguments is invalid. + """ + if not context.executing_eagerly(): + yield + else: + old_settings = seterr(inf_or_nan=inf_or_nan) + yield + seterr(**old_settings) diff --git a/tensorflow/python/eager/execution_callbacks_test.py b/tensorflow/python/eager/execution_callbacks_test.py new file mode 100644 index 00000000000..5594ab5f12a --- /dev/null +++ b/tensorflow/python/eager/execution_callbacks_test.py @@ -0,0 +1,55 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Tests for eager execution_callbacks.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from tensorflow.python.eager import execution_callbacks +from tensorflow.python.framework import constant_op +from tensorflow.python.framework import ops +from tensorflow.python.ops import math_ops +from tensorflow.python.platform import test + + +def log_zero(): + """Computes `log(0.0)`.""" + return math_ops.log(constant_op.constant(0.)) + + +class ExecutionCallbacksTest(test.TestCase): + + def test_errstate_inf_raise(self): + with execution_callbacks.errstate(inf_or_nan=execution_callbacks.RAISE): + with self.assertRaises(execution_callbacks.InfOrNanError): + log_zero() + + def test_errstate_inf_ignore(self): + with execution_callbacks.errstate(inf_or_nan=execution_callbacks.IGNORE): + self.assertEqual(-float("inf"), log_zero().numpy()) + + def test_errstate_nesting(self): + with execution_callbacks.errstate(inf_or_nan=execution_callbacks.RAISE): + with execution_callbacks.errstate(inf_or_nan=execution_callbacks.IGNORE): + self.assertEqual(-float("inf"), log_zero().numpy()) + + with self.assertRaises(execution_callbacks.InfOrNanError): + log_zero() + + +if __name__ == "__main__": + ops.enable_eager_execution() + test.main() diff --git a/tensorflow/python/eager/function.py b/tensorflow/python/eager/function.py index c429dd359bb..9d05a660b1f 100644 --- a/tensorflow/python/eager/function.py +++ b/tensorflow/python/eager/function.py @@ -48,6 +48,7 @@ from tensorflow.python.ops import custom_gradient from tensorflow.python.ops import functional_ops from tensorflow.python.ops import gradients_impl from tensorflow.python.ops import resource_variable_ops +from tensorflow.python.platform import tf_logging as logging from tensorflow.python.util import compat from tensorflow.python.util import nest from tensorflow.python.util import tf_decorator @@ -66,6 +67,11 @@ WHITELIST_FUNCTION_ATTRIBUTE_REGEX = [ BACKWARD_FUNCTION_ATTRIBUTE_NAME ] +CacheKey = collections.namedtuple("CacheKey", [ + "input_signature", "parent_graph", "device_functions", "colocation_stack", + "uses_xla" +]) + def _parse_func_attrs(attributes): """Convert the keyword arguments into function_def attributes. @@ -83,8 +89,8 @@ def _parse_func_attrs(attributes): """ attrs = {} for key, value in attributes.items(): - if not any([re.match(reg, key) - for reg in WHITELIST_FUNCTION_ATTRIBUTE_REGEX]): + if not any(re.match(reg, key) + for reg in WHITELIST_FUNCTION_ATTRIBUTE_REGEX): raise ValueError("Attribute name is not whitelisted. " "Whitelisted: prefix %s, got: %s" % (WHITELIST_FUNCTION_ATTRIBUTE_REGEX, key)) @@ -260,7 +266,7 @@ class _EagerDefinedFunction(object): f=self, tout=self._output_types, executing_eagerly=executing_eagerly, - config=function_call_options.rewriter_config_serialized, + config=function_call_options.config_proto_serialized, executor_type=function_call_options.executor_type) if executing_eagerly: @@ -418,7 +424,10 @@ class Function(object): if (tape.should_record(tensor_inputs) or tape.should_record(self._captured_inputs)): - return self._backprop_call(args) + if context.executing_eagerly(): + return self._eager_backprop_call(args) + else: + return self._backprop_call_with_delayed_rewrite(args) # Only need to override the gradient in graph mode and when we have outputs. if context.executing_eagerly() or not self.outputs: @@ -444,37 +453,34 @@ class Function(object): name: The name to register the gradient as. """ @ops.RegisterGradient(name) - def grad_fn(op, *doutputs): # pylint: disable=unused-variable - """Gradients of this function.""" - if self._backward_graph_function is None: - self._construct_backprop_function() + def _registered_grad_fn(op, *doutputs): # pylint: disable=unused-variable + return self._grad_fn(op, *doutputs) - # pylint: disable=protected-access - self._forward_function.add_to_graph(op.graph) - num_inference_outputs = self._inference_function._num_outputs + def _grad_fn(self, op, *doutputs): + """Gradients of this function.""" + if self._backward_graph_function is None: + self._construct_backprop_function() - # Rewrite an inference call op to be a forward call op - if op.get_attr("f").name.encode() == self._inference_function.name: - func = attr_value_pb2.AttrValue( - func=attr_value_pb2.NameAttrList( - name=self._forward_function.name)) - op._set_attr("f", func) - types = attr_value_pb2.AttrValue.ListValue( - type=self._forward_function._output_types) - op._set_attr("Tout", attr_value_pb2.AttrValue(list=types)) - for i in range( - num_inference_outputs, len(self._forward_function._output_types)): - t = ops.Tensor(op, i, self._forward_function._output_types[i]) - t.set_shape(self._forward_function._output_shapes[i]) - func_graph_output = self._forward_function._func_graph_outputs[i] - custom_gradient.copy_handle_data(func_graph_output, t) - op._outputs.append(t) - # pylint: enable=protected-access - # Compute the gradients using the side outputs - side_outputs = op.outputs[num_inference_outputs:] - args = list(doutputs[:num_inference_outputs]) + list(side_outputs) - return self._backward_graph_function._call_flat( # pylint: disable=protected-access - (a for a in args if a is not None)) + # pylint: disable=protected-access + self._forward_function.add_to_graph(op.graph) + num_inference_outputs = self._inference_function._num_outputs + + # Rewrite an inference call op to be a forward call op + if op.get_attr("f").name.encode() == self._inference_function.name: + op._set_func_attr("f", self._forward_function.name) + op._set_type_list_attr("Tout", self._forward_function._output_types) + op._add_outputs( + self._forward_function._output_types[num_inference_outputs:], + self._forward_function._output_shapes[num_inference_outputs:]) + for i in range(num_inference_outputs, len(op.outputs)): + func_graph_output = self._forward_function._func_graph_outputs[i] + custom_gradient.copy_handle_data(func_graph_output, op.outputs[i]) + # pylint: enable=protected-access + # Compute the gradients using the side outputs + side_outputs = op.outputs[num_inference_outputs:] + args = list(doutputs[:num_inference_outputs]) + list(side_outputs) + return self._backward_graph_function._call_flat( # pylint: disable=protected-access + (a for a in args if a is not None)) @property def name(self): @@ -617,10 +623,13 @@ class Function(object): self._func_graph.outputs + backwards_graph_captures, forward_function_attr) - def _backprop_call(self, args): + def _eager_backprop_call(self, args): """Calls the forward function and records the result on a tape. - (Only records results on a tape if the function has outputs) + This method fully constructs the forward and backward functions before + calling the function and recording them on the tape. + + (Only records results on a tape if the function has outputs). Args: args: All inputs to the function, including resolved captured inputs @@ -662,6 +671,46 @@ class Function(object): args, backward_function) return self._build_call_outputs(real_outputs) + def _backprop_call_with_delayed_rewrite(self, args): + """Calls the inference function and records the result on a tape. + + The recorded backwards function will construct the backwards graph and + rewrite the inference function to the forward function. This only happens + if the recorded backwards function ends up being used to compute gradients. + + This approach avoids constructing unnecessary graphs, but it only works if + we are calling this function when not executing eagerly. + + (Only records results on a tape if the function has outputs) + + Args: + args: All inputs to the function, including resolved captured inputs + + Returns: + The call output. + """ + ctx = context.context() + + if not self._gradient_name: + self._gradient_name = "PartitionedCall-%s" % ops.uid() + self._register_gradient(self._gradient_name) + with ops.get_default_graph().gradient_override_map( + {"PartitionedCall": self._gradient_name, + "StatefulPartitionedCall": self._gradient_name}): + outputs = self._inference_function.call(ctx, args) + + if isinstance(outputs, ops.Operation) or outputs is None: + return outputs + + call_op = outputs[0].op + + def backward_function(*args): + return self._grad_fn(call_op, *args) + + tape.record_operation(self._inference_function.signature.name, outputs, + args, backward_function) + return self._build_call_outputs(outputs) + def _build_call_outputs(self, result): """Maps the fdef output list to actual output structure. @@ -724,7 +773,7 @@ class PolymorphicFunction(object): name, input_signature=None, attributes=None, - autograph=False): + autograph=True): """Initializes a polymorphic function. Args: @@ -927,17 +976,17 @@ class PolymorphicFunction(object): """Computes the cache key given inputs and execution context.""" if self._input_signature is None: inputs = (args, kwargs) if kwargs else args - cache_key = pywrap_tensorflow.TFE_Py_EncodeArg(inputs) + input_signature = pywrap_tensorflow.TFE_Py_EncodeArg(inputs) else: del args, kwargs - cache_key = self._flat_input_signature + input_signature = self._flat_input_signature ctx = context.context() with ops.init_scope(): # The graph, or whether we're executing eagerly, should be a part of the # cache key so we don't improperly capture tensors such as variables. executing_eagerly = ctx.executing_eagerly() - execution_context = executing_eagerly or ops.get_default_graph() + parent_graph = None if executing_eagerly else ops.get_default_graph() # pylint: disable=protected-access default_graph = ops.get_default_graph() @@ -966,8 +1015,8 @@ class PolymorphicFunction(object): else: device_functions = () # pylint: enable=protected-access - return (cache_key, execution_context, device_functions, colocation_stack, - uses_xla) + return CacheKey(input_signature, parent_graph, device_functions, + colocation_stack, uses_xla) def _canonicalize_function_inputs(self, *args, **kwargs): """Canonicalizes `args` and `kwargs`. @@ -1039,16 +1088,21 @@ class PolymorphicFunction(object): return inputs, kwargs else: assert not kwargs + signature_relevant_inputs = inputs[:len(self._input_signature)] try: - nest.assert_same_structure(self._input_signature, inputs) + nest.assert_same_structure(self._input_signature, + signature_relevant_inputs) except (ValueError, TypeError): raise ValueError("Structure of Python function inputs does not match " "input_signature.") - if any(not pywrap_tensorflow.IsTensor(arg) for arg in flat_inputs): + signature_inputs_flat = nest.flatten(signature_relevant_inputs) + if any(not pywrap_tensorflow.IsTensor(arg) + for arg in signature_inputs_flat): raise ValueError("When input_signature is provided, all inputs to " "the Python function must be Tensors.") if any(not spec.is_compatible_with(other) - for spec, other in zip(self._flat_input_signature, flat_inputs)): + for spec, other in zip(self._flat_input_signature, + signature_inputs_flat)): raise ValueError("Python inputs incompatible with input_signature: " "inputs (%s), input_signature (%s)" % (str(inputs), str(self._input_signature))) @@ -1083,6 +1137,9 @@ class PolymorphicFunction(object): "must be hashable.") if graph_function is None: + logging.vlog(1, + "Creating new FuncGraph for Python function %r (key: %r)", + self._python_function, cache_key) if self._input_signature is None: arglen = len(args) else: @@ -1137,7 +1194,7 @@ def validate_signature(signature): "a possibly nested sequence of TensorSpec objects.") -def defun(func=None, input_signature=None, autograph=False): +def defun(func=None, input_signature=None, autograph=True): """Compiles a Python function into a callable TensorFlow graph. `defun` (short for "define function") trace-compiles a Python function @@ -1470,7 +1527,7 @@ def defun(func=None, input_signature=None, autograph=False): def defun_with_attributes(func=None, input_signature=None, attributes=None, - autograph=False): + autograph=True): """Compiles a Python function into a callable TensorFlow graph. This function supports adding extra function attributes. See detailed diff --git a/tensorflow/python/eager/function_gradients_test.py b/tensorflow/python/eager/function_gradients_test.py new file mode 100644 index 00000000000..9b83f57089a --- /dev/null +++ b/tensorflow/python/eager/function_gradients_test.py @@ -0,0 +1,756 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from absl.testing import parameterized + +from tensorflow.core.protobuf import config_pb2 +from tensorflow.python.eager import backprop +from tensorflow.python.eager import context +from tensorflow.python.eager import def_function +from tensorflow.python.eager import function +from tensorflow.python.framework import constant_op +from tensorflow.python.framework import dtypes +from tensorflow.python.framework import ops +from tensorflow.python.framework import tensor_shape +from tensorflow.python.framework import test_util +from tensorflow.python.ops import array_ops +from tensorflow.python.ops import control_flow_ops +from tensorflow.python.ops import gradients_impl +from tensorflow.python.ops import math_ops +from tensorflow.python.ops import resource_variable_ops +from tensorflow.python.ops import variable_scope +from tensorflow.python.ops import variables +from tensorflow.python.platform import test +from tensorflow.python.util import nest + + +class FunctionGradientsTest(test.TestCase, parameterized.TestCase): + + def testGraphModeWithGradients(self): + v = resource_variable_ops.ResourceVariable(1.0, name='v') + + @def_function.function + def step(): + def inner(): + return v * v + + return backprop.implicit_grad(inner)()[0][0] + + self.assertAllEqual(step(), 2.0) + + def testGraphGradientVariable(self): + with ops.Graph().as_default(), self.cached_session(): + v = variables.Variable(1.0) + + @def_function.function + def f(): + return 2.0 * v + + node = f() + grads, = gradients_impl.gradients(node, v) + v.initializer.run() + self.assertAllEqual(grads.eval(), 2.0) + self.assertEqual(grads.shape, v.shape) + + def testSymGradGatherNd(self): + with ops.Graph().as_default(), self.cached_session() as sess: + + @def_function.function + def f(x): + return array_ops.gather_nd(x, [[0]]) + + c = constant_op.constant([[2.]]) + f_c = f(c) + g, = gradients_impl.gradients(f_c, c) + self.assertAllEqual(self.evaluate(g).values, [[1.0]]) + + def testNoSymGradNestedDefun(self): + + @def_function.function + def outer(): + + @def_function.function + def f(x): + return array_ops.gather_nd(x, [[0]]) + + c = constant_op.constant([[2.]]) + f_c = f(c) + g, = gradients_impl.gradients(f_c, c) + self.assertIsInstance(g, ops.IndexedSlices) + + outer() + + def testGraphFunctionWithGradients(self): + v = resource_variable_ops.ResourceVariable(1.0, name='v') + + @def_function.function + def step(): + def inner(): + return v * v + + return backprop.implicit_grad(inner)()[0][0] + + step_op = step.get_concrete_function() + self.assertEqual(step_op.output_dtypes, dtypes.float32) + self.assertEqual(step_op.output_shapes, tensor_shape.TensorShape([])) + self.assertAllEqual(step_op(), 2.0) + + @test_util.run_in_graph_and_eager_modes() + def testDefunCondGradient(self): + + @def_function.function + def f(x): + return control_flow_ops.cond(x > 0.5, lambda: 2 * x, lambda: 3 * x) + + with backprop.GradientTape() as t: + x = constant_op.constant(1.0) + t.watch(x) + y = f(x) + self.assertAllEqual(self.evaluate(t.gradient(y, x)), 2.0) + + @test_util.run_in_graph_and_eager_modes() + def testGraphLoopGradient(self): + + @def_function.function + def f(x): + return control_flow_ops.while_loop(lambda _, i: i < 2, + lambda x, i: (2*x, i + 1), + [x, 0])[0] + + with backprop.GradientTape() as t: + x = constant_op.constant(1.0) + t.watch(x) + y = f(x) + self.assertAllEqual(self.evaluate(t.gradient(y, x)), 4.0) + + def testDefunDifferentiable(self): + v = resource_variable_ops.ResourceVariable(1.0) + + @def_function.function + def f(): + return v * v + + self.assertAllEqual(backprop.implicit_grad(f)()[0][0], 2.0) + + def testDefunCanBeDifferentiatedTwice(self): + v = resource_variable_ops.ResourceVariable(1.0) + + @def_function.function + def f(): + return v * v + + self.assertAllEqual(backprop.implicit_grad(f)()[0][0], 2.0) + # Ensure that v is watched again. + self.assertAllEqual(backprop.implicit_grad(f)()[0][0], 2.0) + + def testSymbolicGradientVariableNoneNotZerosLike(self): + with ops.Graph().as_default(): + v = variables.Variable(1.0) + + @def_function.function + def f(x, v): + v.read_value() + return x * x + + x = constant_op.constant(1.0) + l = f(x, v) + _, dv = gradients_impl.gradients(l, [x, v]) + with self.cached_session(): + v.initializer.run() + self.assertEqual(dv, None) + + def testDefunCallBackprop(self): + + @def_function.function + def f(x): + return math_ops.add(x, x) + + @def_function.function + def g(x): + return backprop.gradients_function(f, [0])(x)[0] + + self.assertAllEqual(2, g(constant_op.constant(2.))) + + @test_util.run_deprecated_v1 + def testGraphModeEagerGradError(self): + with context.graph_mode(): + def f(): + x = variable_scope.get_variable( + 'v', initializer=constant_op.constant(1.0)) + return x * constant_op.constant(2.0) + + with self.assertRaisesRegexp(ValueError, + 'No trainable variables were accessed'): + backprop.implicit_val_and_grad(f)() + + def testDefunCallBackpropUsingSameObjectForMultipleArguments(self): + + @def_function.function + def g(x): + return backprop.gradients_function(math_ops.multiply, [0, 1])(x, x) + + def np_g(x): + return [d.numpy() for d in g(x)] + + x = constant_op.constant(1.) + self.assertAllEqual([1., 1.], np_g(x)) + self.assertAllEqual([1., 1.], np_g(1.)) + + def testGradientTensorConversionWithDefun(self): + three = resource_variable_ops.ResourceVariable(3.0, name='v') + + @def_function.function + def f(x): + return math_ops.add(x, three) + + def g(x): + return f(x) + + g = backprop.implicit_grad(g)(constant_op.constant(1.0))[0][0] + self.assertAllEqual(g, 1.0) + + def testGradient(self): + matmul = def_function.function(math_ops.matmul) + + def sq(x): + return matmul(x, x, transpose_a=True) + + t = constant_op.constant([[1.0, 2.0], [3.0, 4.0]]) + grad_t, = backprop.gradients_function(sq, [0])(t) + self.assertAllEqual(grad_t, [[6, 6], [14, 14]]) + + def testGradientInFunction(self): + + @def_function.function + def f(x): + return backprop.gradients_function(lambda y: y * y, [0])(x)[0] + + self.assertAllEqual(f(constant_op.constant(1.0)), 2.0) + + def testGradientOfGatherWithDefun(self): + v = resource_variable_ops.ResourceVariable([0.0, 1.0, 2.0]) + + def sum_gather(): + return math_ops.reduce_sum(array_ops.gather(v, [1, 2])) + + grad_fn = backprop.implicit_grad(sum_gather) + gradient = grad_fn() + defun_grad_fn = backprop.implicit_grad(def_function.function(sum_gather)) + defun_gradient = defun_grad_fn() + self.assertEqual(len(gradient), len(defun_gradient)) + + gradient = gradient[0][0] + defun_gradient = defun_gradient[0][0] + self.assertAllEqual(gradient.values, defun_gradient.values) + self.assertAllEqual(gradient.indices, defun_gradient.indices) + self.assertAllEqual(gradient.dense_shape, defun_gradient.dense_shape) + + def testDifferentiableFunctionNoneOutputs(self): + + @def_function.function + def my_function(x): + return x, None + + def wrapper(x): + return my_function(x)[0] + + g = backprop.gradients_function(wrapper, [0])(constant_op.constant(0.0)) + self.assertAllEqual(g[0], 1.) + + @def_function.function + def foo(a): + return None, a * a + + x = constant_op.constant(5.0) + with backprop.GradientTape() as tp: + tp.watch(x) + none, r = foo(x) + g = tp.gradient(r, x) + + self.assertIs(none, None) + self.assertAllEqual(r, 25.0) + self.assertAllEqual(g, 2 * 5.0) + + @test_util.run_in_graph_and_eager_modes + def testNestedDifferentiableFunction(self): + @def_function.function + def inner_fn(a, b): + return a * math_ops.add(a, b) + + @def_function.function + def outer_fn(x): + return inner_fn(x, 1.0) + + x = constant_op.constant(5.0) + with backprop.GradientTape() as tp: + tp.watch(x) + result = outer_fn(x) + grad = tp.gradient(result, x) + + self.assertAllEqual(grad, 2 * 5.0 + 1.0) + + @test_util.run_in_graph_and_eager_modes + def testDeeplyNestedDifferentiableFunction(self): + @def_function.function + def inner_inner_fn(a, b): + return math_ops.add(a, b) + + @def_function.function + def inner_fn(a, b): + return inner_inner_fn(a, b) + + @def_function.function + def middle_fn(a, b): + return a * inner_fn(a, b) + + @def_function.function + def outer_fn(x): + return middle_fn(x, 1.0) + + x = constant_op.constant(5.0) + with backprop.GradientTape() as tp: + tp.watch(x) + result = outer_fn(x) + grad = tp.gradient(result, x) + + self.assertAllEqual(grad, 2 * 5.0 + 1.0) + + @test_util.run_in_graph_and_eager_modes + def testDeeplyNestedDifferentiableFunctionWithMultipleGradCalls(self): + @def_function.function + def inner_fn(a, b): + return math_ops.add(a, b) + + @def_function.function + def middle_fn(a, b): + return math_ops.mul(a, inner_fn(a, b)) + + @def_function.function + def outer_fn(x): + return middle_fn(x, 3.0) + + x = constant_op.constant(5.0) + self.assertAllEqual(outer_fn(x), 5.0 * (5.0 + 3.0)) + + with backprop.GradientTape() as tp: + tp.watch(x) + result = outer_fn(x) + grad = tp.gradient(result, x) + + self.assertAllEqual(grad, 2 * 5.0 + 3.0) + self.assertAllEqual(outer_fn(x), 5.0 * (5.0 + 3.0)) + self.assertAllEqual(middle_fn(3.0, x), 3.0 * (3.0 + 5.0)) + + with backprop.GradientTape() as tp: + tp.watch(x) + result = outer_fn(x) + grad = tp.gradient(result, x) + + self.assertAllEqual(grad, 2 * 5.0 + 3.0) + + y = constant_op.constant(4.0) + with backprop.GradientTape() as tp: + tp.watch(y) + result = outer_fn(y) + grad = tp.gradient(result, y) + + self.assertAllEqual(grad, 2 * 4.0 + 3.0) + + with backprop.GradientTape() as tp: + tp.watch(y) + result = inner_fn(y, y) + grad = tp.gradient(result, y) + + self.assertAllEqual(grad, 2.0) + + @test_util.run_in_graph_and_eager_modes + def testDeeplyNestedDifferentiableFunctionGradientTapeInDefun(self): + @def_function.function + def inner_inner_fn(a, b): + return math_ops.add(a, b) + + @def_function.function + def inner_fn(a, b): + return inner_inner_fn(a, b) + + @def_function.function + def middle_fn(a, b): + return a * inner_fn(a, b) + + @def_function.function + def outer_fn(x): + with backprop.GradientTape() as tp: + tp.watch(x) + result = middle_fn(x, 1.0) + grad = tp.gradient(result, x) + return grad + + x = constant_op.constant(5.0) + grad = outer_fn(x) + self.assertAllEqual(grad, 2 * 5.0 + 1.0) + + @test_util.run_in_graph_and_eager_modes + def testDeeplyNestedDifferentiableFunctionGradientTapeInNestedDefun(self): + @def_function.function + def inner_inner_fn(a, b): + return math_ops.add(a, b) + + @def_function.function + def inner_fn(a, b): + return inner_inner_fn(a, b) + + @def_function.function + def middle_fn(a, b): + return a * inner_fn(a, b) + + @def_function.function + def almost_outer_fn(x): + with backprop.GradientTape() as tp: + tp.watch(x) + result = middle_fn(x, 1.0) + grad = tp.gradient(result, x) + return grad + + @def_function.function + def outer_fn(x): + return almost_outer_fn(x) + + x = constant_op.constant(5.0) + grad = outer_fn(x) + self.assertAllEqual(grad, 2 * 5.0 + 1.0) + + @test_util.run_in_graph_and_eager_modes + def testDeeplyNestedDifferentiableFunctionGradientTapeInMultNestedDefun(self): + @def_function.function + def inner_inner_fn(a, b): + return math_ops.add(a, b) + + @def_function.function + def inner_fn(a, b): + return inner_inner_fn(a, b) + + @def_function.function + def middle_fn(a, b): + return a * inner_fn(a, b) + + @def_function.function + def almost_outer_fn(x): + with backprop.GradientTape() as tp: + tp.watch(x) + result = middle_fn(x, 1.0) + grad = tp.gradient(result, x) + return grad + + @def_function.function + def outer_fn(x): + return almost_outer_fn(x) + + @def_function.function + def outer_outer_fn(x): + return outer_fn(x) + + x = constant_op.constant(5.0) + grad = outer_outer_fn(x) + self.assertAllEqual(grad, 2 * 5.0 + 1.0) + + @test_util.run_in_graph_and_eager_modes + def testDeeplyNestedDifferentiableFunctionTFGradientInDefun(self): + @def_function.function + def inner_inner_fn(a, b): + return math_ops.add(a, b) + + @def_function.function + def inner_fn(a, b): + return inner_inner_fn(a, b) + + @def_function.function + def middle_fn(a, b): + return a * inner_fn(a, b) + + @def_function.function + def outer_fn(x): + result = middle_fn(x, 1.0) + return gradients_impl.gradients(result, [x])[0] + + x = constant_op.constant(5.0) + grad = outer_fn(x) + self.assertAllEqual(grad, 2 * 5.0 + 1.0) + + @test_util.run_in_graph_and_eager_modes + def testDeeplyNestedDifferentiableFunctionTFGradientInNestedDefun(self): + @def_function.function + def inner_inner_fn(a, b): + return math_ops.add(a, b) + + @def_function.function + def inner_fn(a, b): + return inner_inner_fn(a, b) + + @def_function.function + def middle_fn(a, b): + return a * inner_fn(a, b) + + @def_function.function + def almost_outer_fn(x): + result = middle_fn(x, 1.0) + return gradients_impl.gradients(result, [x])[0] + + @def_function.function + def outer_fn(x): + return almost_outer_fn(x) + + x = constant_op.constant(5.0) + grad = outer_fn(x) + self.assertAllEqual(grad, 2 * 5.0 + 1.0) + + @test_util.run_in_graph_and_eager_modes + def testDeeplyNestedDifferentiableFunctionTFGradientInMultNestedDefun(self): + @def_function.function + def inner_inner_fn(a, b): + return math_ops.add(a, b) + + @def_function.function + def inner_fn(a, b): + return inner_inner_fn(a, b) + + @def_function.function + def middle_fn(a, b): + return a * inner_fn(a, b) + + @def_function.function + def almost_outer_fn(x): + result = middle_fn(x, 1.0) + return gradients_impl.gradients(result, [x])[0] + + @def_function.function + def outer_fn(x): + return almost_outer_fn(x) + + @def_function.function + def outer_outer_fn(x): + return outer_fn(x) + + x = constant_op.constant(5.0) + grad = outer_outer_fn(x) + self.assertAllEqual(grad, 2 * 5.0 + 1.0) + + def testDeeplyNestedDifferentiableFunctionWithVariable(self): + var = variables.Variable(constant_op.constant(1.0)) + + @def_function.function + def inner_fn(a, b): + return math_ops.add(a, b) + + @def_function.function + def middle_fn(a, b): + return a * inner_fn(a, b) + + @def_function.function + def outer_fn(x): + return middle_fn(x, var) + + x = constant_op.constant(5.0) + with backprop.GradientTape() as tp: + tp.watch(x) + result = outer_fn(x) + grad = tp.gradient(result, x) + + self.assertAllEqual(grad, 2 * 5.0 + 1.0) + + def testDeeplyNestedDifferentiableFunctionWithVariableMultipleGradCalls(self): + v = variables.Variable(constant_op.constant(3.0)) + + @def_function.function + def inner_fn(a, b): + return math_ops.add(a, b) + + @def_function.function + def middle_fn(a, b): + return math_ops.mul(a, inner_fn(a, b)) + + @def_function.function + def outer_fn(x): + return middle_fn(x, v) + + x = constant_op.constant(5.0) + self.assertAllEqual(outer_fn(x), 5.0 * (5.0 + 3.0)) + + with backprop.GradientTape() as tp: + tp.watch(x) + result = outer_fn(x) + grad = tp.gradient(result, x) + + self.assertAllEqual(grad, 2 * 5.0 + 3.0) + self.assertAllEqual(outer_fn(x), 5.0 * (5.0 + 3.0)) + self.assertAllEqual(middle_fn(v, x), 3.0 * (3.0 + 5.0)) + + with backprop.GradientTape() as tp: + tp.watch(x) + result = outer_fn(x) + grad = tp.gradient(result, x) + + self.assertAllEqual(grad, 2 * 5.0 + 3.0) + + y = constant_op.constant(4.0) + with backprop.GradientTape() as tp: + tp.watch(y) + result = outer_fn(y) + grad = tp.gradient(result, y) + + self.assertAllEqual(grad, 2 * 4.0 + 3.0) + + v.assign(constant_op.constant(1.5)) + with backprop.GradientTape() as tp: + tp.watch(y) + result = outer_fn(y) + grad = tp.gradient(result, y) + + self.assertAllEqual(grad, 2 * 4.0 + 1.5) + + with backprop.GradientTape() as tp: + tp.watch(y) + result = inner_fn(y, v) + grad = tp.gradient(result, y) + + self.assertAllEqual(grad, 1.0) + + def testDeeplyNestedDifferentiableFunctionWithVariableMultipleTFGrads(self): + with context.graph_mode(), self.cached_session(): + v = resource_variable_ops.ResourceVariable(3.0) + v.initializer.run() + + @def_function.function + def inner_fn(a, b): + return math_ops.add(a, b) + + @def_function.function + def middle_fn(a, b): + return math_ops.mul(a, inner_fn(a, b)) + + @def_function.function + def outer_fn(x): + return middle_fn(x, v) + + x = constant_op.constant(5.0) + self.assertAllEqual(outer_fn(x).eval(), 5.0 * (5.0 + 3.0)) + + grad, = gradients_impl.gradients(outer_fn(x), x) + + self.assertAllEqual(grad, 2 * 5.0 + 3.0) + self.assertAllEqual(outer_fn(x), 5.0 * (5.0 + 3.0)) + self.assertAllEqual(middle_fn(v, x), 3.0 * (3.0 + 5.0)) + + grad, = gradients_impl.gradients(outer_fn(x), x) + + self.assertAllEqual(grad, 2 * 5.0 + 3.0) + + y = constant_op.constant(4.0) + grad, = gradients_impl.gradients(outer_fn(y), y) + self.assertAllEqual(grad, 2 * 4.0 + 3.0) + + self.evaluate(v.assign(constant_op.constant(1.5))) + grad, = gradients_impl.gradients(outer_fn(y), y) + + self.assertAllEqual(grad, 2 * 4.0 + 1.5) + + grad, = gradients_impl.gradients(inner_fn(y, v), y) + self.assertAllEqual(grad, 1.0) + + def testNestedDifferentiableFunctionNoneOutputs(self): + @def_function.function + def foo(a, b): + return None, a * math_ops.add(a, b), None, 2*a + + @def_function.function + def bar(x): + return foo(x, 1.0) + + x = constant_op.constant(5.0) + with backprop.GradientTape(persistent=True) as tp: + tp.watch(x) + none1, r1, none2, r2 = bar(x) + g1 = tp.gradient(r1, x) + g2 = tp.gradient(r2, x) + + self.assertAllEqual(r1, 30.0) + self.assertAllEqual(r2, 10.0) + self.assertIs(none1, None) + self.assertIs(none2, None) + self.assertAllEqual(g1, 2 * 5.0 + 1.0) + self.assertAllEqual(g2, 2.0) + + def testGradientWithKeywordArguments(self): + matmul = def_function.function(math_ops.matmul) + + def sq(x): + return matmul(a=x, b=x, transpose_a=True) + + t = constant_op.constant([[1.0, 2.0], [3.0, 4.0]]) + grad_t, = backprop.gradients_function(sq, [0])(t) + self.assertAllEqual(grad_t, [[6, 6], [14, 14]]) + + with backprop.GradientTape(persistent=True) as tape: + tape.watch(t) + one = matmul(t, b=t, transpose_a=True) + two = matmul(b=t, a=t, transpose_a=True) + three = matmul(a=t, b=t, transpose_a=True) + + for output in [one, two, three]: + self.assertAllEqual(tape.gradient(output, t), [[6, 6], [14, 14]]) + + def testGradientInFunctionWithKeywordArguments(self): + + @def_function.function + def f(x): + return backprop.gradients_function(lambda y: y * y, [0])(x)[0] + + self.assertAllEqual(f(x=constant_op.constant(1.0)), 2.0) + + @test_util.run_in_graph_and_eager_modes + def testBackwardNone(self): + model = variables.Variable(1.0, name='model') + count = variables.Variable(0) + + @function.defun + def forward_pass(value): + count.assign_add(1) + residuals = value - model + loss = 0.5 * math_ops.reduce_mean(math_ops.pow(residuals, 2)) + # Note: count is an integer, so its doutput will be None + return loss, count + + def reduce_fn(x): + if context.executing_eagerly(): + with backprop.GradientTape() as t: + loss, count = forward_pass(x) + return t.gradient(loss, model), count + loss, count = forward_pass(x) + grad_only = gradients_impl.gradients(loss, model) + return grad_only, count + + g, _ = reduce_fn(constant_op.constant([7.0])) + + self.evaluate(variables.global_variables_initializer()) + self.assertAllEqual(nest.flatten(self.evaluate(g)), [-6.0]) + + +if __name__ == '__main__': + ops.enable_eager_execution( + config=config_pb2.ConfigProto(device_count={'CPU': 4})) + test.main() diff --git a/tensorflow/python/eager/function_test.py b/tensorflow/python/eager/function_test.py index 2af08689f87..71afbd24d8d 100644 --- a/tensorflow/python/eager/function_test.py +++ b/tensorflow/python/eager/function_test.py @@ -29,7 +29,6 @@ import numpy from tensorflow.core.protobuf import config_pb2 from tensorflow.core.protobuf import rewriter_config_pb2 from tensorflow.python import keras -from tensorflow.python.eager import backprop from tensorflow.python.eager import context from tensorflow.python.eager import def_function from tensorflow.python.eager import function @@ -48,7 +47,6 @@ from tensorflow.python.layers import convolutional from tensorflow.python.ops import array_ops from tensorflow.python.ops import clip_ops from tensorflow.python.ops import control_flow_ops -from tensorflow.python.ops import gradients_impl from tensorflow.python.ops import init_ops from tensorflow.python.ops import list_ops from tensorflow.python.ops import math_ops @@ -102,10 +100,10 @@ class FunctionTest(test.TestCase, parameterized.TestCase): _ = x * y return x + y - # The default config allows everything. - rewrites = rewriter_config_pb2.RewriterConfig() + # The default config allows all rewrites. + config_proto = config_pb2.ConfigProto() - with context.function_rewriter_config(rewrites): + with context.function_config_proto(config_proto): t = constant_op.constant(1.0) self.assertAllEqual(add(t, t).numpy(), 2.0) @@ -149,31 +147,22 @@ class FunctionTest(test.TestCase, parameterized.TestCase): out = a_times_b(pair({'a': t}, {'b': t})) self.assertAllEqual(out, math_ops.matmul(t, t).numpy()) - def testGraphModeWithGradients(self): - v = resource_variable_ops.ResourceVariable(1.0, name='v') + def testNestedOutputsGraphMode(self): + matmul = def_function.function(math_ops.matmul) - @def_function.function - def step(): - def inner(): - return v * v + pair = collections.namedtuple('pair', ['a', 'b']) - return backprop.implicit_grad(inner)()[0][0] + @def_function.function() + def pairs_mul(pair_a, pair_b): + return pair(matmul(pair_a.a, pair_b.a), matmul(pair_a.b, pair_b.b)) - self.assertAllEqual(step(), 2.0) + a = constant_op.constant([[1.0, 2.0], [1.0, 2.0]]) + b = constant_op.constant([[3.0, 4.0], [3.0, 4.0]]) - def testGraphGradientVariable(self): - with ops.Graph().as_default(), self.cached_session(): - v = variables.Variable(1.0) - - @def_function.function - def f(): - return 2.0 * v - - node = f() - grads, = gradients_impl.gradients(node, v) - v.initializer.run() - self.assertAllEqual(grads.eval(), 2.0) - self.assertEqual(grads.shape, v.shape) + out = pairs_mul(pair(a, b), pair(b, a)) + expected = pair(math_ops.matmul(a, b).numpy(), + math_ops.matmul(b, a).numpy()) + self.assertAllClose(out, expected) def testGraphEagerIsolation(self): @@ -314,34 +303,6 @@ class FunctionTest(test.TestCase, parameterized.TestCase): random_seed.set_random_seed(1) self.assertAllEqual(f(), x) - def testSymGradGatherNd(self): - with ops.Graph().as_default(), self.cached_session() as sess: - - @def_function.function - def f(x): - return array_ops.gather_nd(x, [[0]]) - - c = constant_op.constant([[2.]]) - f_c = f(c) - g, = gradients_impl.gradients(f_c, c) - self.assertAllEqual(sess.run(g).values, [[1.0]]) - - def testNoSymGradNestedDefun(self): - - @def_function.function - def outer(): - - @def_function.function - def f(x): - return array_ops.gather_nd(x, [[0]]) - - c = constant_op.constant([[2.]]) - f_c = f(c) - g, = gradients_impl.gradients(f_c, c) - self.assertIsInstance(g, ops.IndexedSlices) - - outer() - def testNestedInputsGraphFunction(self): matmul = def_function.function(math_ops.matmul) @@ -378,21 +339,6 @@ class FunctionTest(test.TestCase, parameterized.TestCase): self.assertAllEqual(a, math_ops.matmul(t, t).numpy()) self.assertAllEqual(b['b'].numpy(), 1.0) - def testGraphFunctionWithGradients(self): - v = resource_variable_ops.ResourceVariable(1.0, name='v') - - @def_function.function - def step(): - def inner(): - return v * v - - return backprop.implicit_grad(inner)()[0][0] - - step_op = step.get_concrete_function() - self.assertEqual(step_op.output_dtypes, dtypes.float32) - self.assertEqual(step_op.output_shapes, tensor_shape.TensorShape([])) - self.assertAllEqual(step_op(), 2.0) - def testGraphFunctionNoneOutput(self): @def_function.function def fn(unused_a, unused_b): @@ -404,34 +350,6 @@ class FunctionTest(test.TestCase, parameterized.TestCase): self.assertEqual(fn_op.output_shapes, None) self.assertAllEqual(fn_op(x, x), None) - @test_util.run_in_graph_and_eager_modes() - def testDefunCondGradient(self): - - @def_function.function - def f(x): - return control_flow_ops.cond(x > 0.5, lambda: 2 * x, lambda: 3 * x) - - with backprop.GradientTape() as t: - x = constant_op.constant(1.0) - t.watch(x) - y = f(x) - self.assertAllEqual(self.evaluate(t.gradient(y, x)), 2.0) - - @test_util.run_in_graph_and_eager_modes() - def testGraphLoopGradient(self): - - @def_function.function - def f(x): - return control_flow_ops.while_loop(lambda _, i: i < 2, - lambda x, i: (2*x, i + 1), - [x, 0])[0] - - with backprop.GradientTape() as t: - x = constant_op.constant(1.0) - t.watch(x) - y = f(x) - self.assertAllEqual(self.evaluate(t.gradient(y, x)), 4.0) - def testDefunNumpyArraysConvertedToTensors(self): def f(x): @@ -625,27 +543,7 @@ class FunctionTest(test.TestCase, parameterized.TestCase): self.assertIsInstance( self.v, resource_variable_ops.ResourceVariable) - def testDefunDifferentiable(self): - v = resource_variable_ops.ResourceVariable(1.0) - - @def_function.function - def f(): - return v * v - - self.assertAllEqual(backprop.implicit_grad(f)()[0][0], 2.0) - - def testDefunCanBeDifferentiatedTwice(self): - v = resource_variable_ops.ResourceVariable(1.0) - - @def_function.function - def f(): - return v * v - - self.assertAllEqual(backprop.implicit_grad(f)()[0][0], 2.0) - # Ensure that v is watched again. - self.assertAllEqual(backprop.implicit_grad(f)()[0][0], 2.0) - - def testRunMetadata(self): + def disabled_testRunMetadata(self): @def_function.function def f(x): @@ -683,23 +581,7 @@ class FunctionTest(test.TestCase, parameterized.TestCase): variables.global_variables_initializer().run() call = def_function.function(o.call) op = call() - self.assertAllEqual(sess.run(op), 2.0) - - def testSymbolicGradientVariableNoneNotZerosLike(self): - with ops.Graph().as_default(): - v = variables.Variable(1.0) - - @def_function.function - def f(x, v): - v.read_value() - return x * x - - x = constant_op.constant(1.0) - l = f(x, v) - _, dv = gradients_impl.gradients(l, [x, v]) - with self.cached_session(): - v.initializer.run() - self.assertEqual(dv, None) + self.assertAllEqual(self.evaluate(op), 2.0) def testGraphModeManyFunctions(self): with ops.Graph().as_default(), self.cached_session(): @@ -742,42 +624,6 @@ class FunctionTest(test.TestCase, parameterized.TestCase): self.assertAllEqual(8, g(constant_op.constant(2))) - def testDefunCallBackprop(self): - - @def_function.function - def f(x): - return math_ops.add(x, x) - - @def_function.function - def g(x): - return backprop.gradients_function(f, [0])(x)[0] - - self.assertAllEqual(2, g(constant_op.constant(2.))) - - def testGraphModeEagerGradError(self): - with context.graph_mode(): - def f(): - x = variable_scope.get_variable( - 'v', initializer=constant_op.constant(1.0)) - return x * constant_op.constant(2.0) - - with self.assertRaisesRegexp(ValueError, - 'No trainable variables were accessed'): - backprop.implicit_val_and_grad(f)() - - def testDefunCallBackpropUsingSameObjectForMultipleArguments(self): - - @def_function.function - def g(x): - return backprop.gradients_function(math_ops.multiply, [0, 1])(x, x) - - def np_g(x): - return [d.numpy() for d in g(x)] - - x = constant_op.constant(1.) - self.assertAllEqual([1., 1.], np_g(x)) - self.assertAllEqual([1., 1.], np_g(1.)) - def testCallShape(self): @def_function.function @@ -808,37 +654,6 @@ class FunctionTest(test.TestCase, parameterized.TestCase): g(three) - def testGradientTensorConversionWithDefun(self): - three = resource_variable_ops.ResourceVariable(3.0, name='v') - - @def_function.function - def f(x): - return math_ops.add(x, three) - - def g(x): - return f(x) - - g = backprop.implicit_grad(g)(constant_op.constant(1.0))[0][0] - self.assertAllEqual(g, 1.0) - - def testGradient(self): - matmul = def_function.function(math_ops.matmul) - - def sq(x): - return matmul(x, x, transpose_a=True) - - t = constant_op.constant([[1.0, 2.0], [3.0, 4.0]]) - grad_t, = backprop.gradients_function(sq, [0])(t) - self.assertAllEqual(grad_t, [[6, 6], [14, 14]]) - - def testGradientInFunction(self): - - @def_function.function - def f(x): - return backprop.gradients_function(lambda y: y * y, [0])(x)[0] - - self.assertAllEqual(f(constant_op.constant(1.0)), 2.0) - def testGatherResourceWithDefun(self): with ops.device('cpu:0'): v = resource_variable_ops.ResourceVariable([0.0, 1.0, 2.0]) @@ -849,24 +664,6 @@ class FunctionTest(test.TestCase, parameterized.TestCase): defined = def_function.function(sum_gather) self.assertAllEqual(sum_gather(), defined()) - def testGradientOfGatherWithDefun(self): - v = resource_variable_ops.ResourceVariable([0.0, 1.0, 2.0]) - - def sum_gather(): - return math_ops.reduce_sum(array_ops.gather(v, [1, 2])) - - grad_fn = backprop.implicit_grad(sum_gather) - gradient = grad_fn() - defun_grad_fn = backprop.implicit_grad(def_function.function(sum_gather)) - defun_gradient = defun_grad_fn() - self.assertEqual(len(gradient), len(defun_gradient)) - - gradient = gradient[0][0] - defun_gradient = defun_gradient[0][0] - self.assertAllEqual(gradient.values, defun_gradient.values) - self.assertAllEqual(gradient.indices, defun_gradient.indices) - self.assertAllEqual(gradient.dense_shape, defun_gradient.dense_shape) - def testReturningIndexedSlicesWithDefun(self): def validate(indexed_slice): @@ -1012,440 +809,6 @@ class FunctionTest(test.TestCase, parameterized.TestCase): shape = constant_op.constant([2, 1]).gpu() reshape(value, shape) # No error is raised - def testDifferentiableFunctionNoneOutputs(self): - - @def_function.function - def my_function(x): - return x, None - - def wrapper(x): - return my_function(x)[0] - - g = backprop.gradients_function(wrapper, [0])(constant_op.constant(0.0)) - self.assertAllEqual(g[0], 1.) - - @def_function.function - def foo(a): - return None, a * a - - x = constant_op.constant(5.0) - with backprop.GradientTape() as tp: - tp.watch(x) - none, r = foo(x) - g = tp.gradient(r, x) - - self.assertIs(none, None) - self.assertAllEqual(r, 25.0) - self.assertAllEqual(g, 2 * 5.0) - - @test_util.run_in_graph_and_eager_modes - def testNestedDifferentiableFunction(self): - @def_function.function - def inner_fn(a, b): - return a * math_ops.add(a, b) - - @def_function.function - def outer_fn(x): - return inner_fn(x, 1.0) - - x = constant_op.constant(5.0) - with backprop.GradientTape() as tp: - tp.watch(x) - result = outer_fn(x) - grad = tp.gradient(result, x) - - self.assertAllEqual(grad, 2 * 5.0 + 1.0) - - @test_util.run_in_graph_and_eager_modes - def testDeeplyNestedDifferentiableFunction(self): - @def_function.function - def inner_inner_fn(a, b): - return math_ops.add(a, b) - - @def_function.function - def inner_fn(a, b): - return inner_inner_fn(a, b) - - @def_function.function - def middle_fn(a, b): - return a * inner_fn(a, b) - - @def_function.function - def outer_fn(x): - return middle_fn(x, 1.0) - - x = constant_op.constant(5.0) - with backprop.GradientTape() as tp: - tp.watch(x) - result = outer_fn(x) - grad = tp.gradient(result, x) - - self.assertAllEqual(grad, 2 * 5.0 + 1.0) - - @test_util.run_in_graph_and_eager_modes - def testDeeplyNestedDifferentiableFunctionWithMultipleGradCalls(self): - @def_function.function - def inner_fn(a, b): - return math_ops.add(a, b) - - @def_function.function - def middle_fn(a, b): - return math_ops.mul(a, inner_fn(a, b)) - - @def_function.function - def outer_fn(x): - return middle_fn(x, 3.0) - - x = constant_op.constant(5.0) - self.assertAllEqual(outer_fn(x), 5.0 * (5.0 + 3.0)) - - with backprop.GradientTape() as tp: - tp.watch(x) - result = outer_fn(x) - grad = tp.gradient(result, x) - - self.assertAllEqual(grad, 2 * 5.0 + 3.0) - self.assertAllEqual(outer_fn(x), 5.0 * (5.0 + 3.0)) - self.assertAllEqual(middle_fn(3.0, x), 3.0 * (3.0 + 5.0)) - - with backprop.GradientTape() as tp: - tp.watch(x) - result = outer_fn(x) - grad = tp.gradient(result, x) - - self.assertAllEqual(grad, 2 * 5.0 + 3.0) - - y = constant_op.constant(4.0) - with backprop.GradientTape() as tp: - tp.watch(y) - result = outer_fn(y) - grad = tp.gradient(result, y) - - self.assertAllEqual(grad, 2 * 4.0 + 3.0) - - with backprop.GradientTape() as tp: - tp.watch(y) - result = inner_fn(y, y) - grad = tp.gradient(result, y) - - self.assertAllEqual(grad, 2.0) - - @test_util.run_in_graph_and_eager_modes - def testDeeplyNestedDifferentiableFunctionGradientTapeInDefun(self): - @def_function.function - def inner_inner_fn(a, b): - return math_ops.add(a, b) - - @def_function.function - def inner_fn(a, b): - return inner_inner_fn(a, b) - - @def_function.function - def middle_fn(a, b): - return a * inner_fn(a, b) - - @def_function.function - def outer_fn(x): - with backprop.GradientTape() as tp: - tp.watch(x) - result = middle_fn(x, 1.0) - grad = tp.gradient(result, x) - return grad - - x = constant_op.constant(5.0) - grad = outer_fn(x) - self.assertAllEqual(grad, 2 * 5.0 + 1.0) - - @test_util.run_in_graph_and_eager_modes - def testDeeplyNestedDifferentiableFunctionGradientTapeInNestedDefun(self): - @def_function.function - def inner_inner_fn(a, b): - return math_ops.add(a, b) - - @def_function.function - def inner_fn(a, b): - return inner_inner_fn(a, b) - - @def_function.function - def middle_fn(a, b): - return a * inner_fn(a, b) - - @def_function.function - def almost_outer_fn(x): - with backprop.GradientTape() as tp: - tp.watch(x) - result = middle_fn(x, 1.0) - grad = tp.gradient(result, x) - return grad - - @def_function.function - def outer_fn(x): - return almost_outer_fn(x) - - x = constant_op.constant(5.0) - grad = outer_fn(x) - self.assertAllEqual(grad, 2 * 5.0 + 1.0) - - @test_util.run_in_graph_and_eager_modes - def testDeeplyNestedDifferentiableFunctionGradientTapeInMultNestedDefun(self): - @def_function.function - def inner_inner_fn(a, b): - return math_ops.add(a, b) - - @def_function.function - def inner_fn(a, b): - return inner_inner_fn(a, b) - - @def_function.function - def middle_fn(a, b): - return a * inner_fn(a, b) - - @def_function.function - def almost_outer_fn(x): - with backprop.GradientTape() as tp: - tp.watch(x) - result = middle_fn(x, 1.0) - grad = tp.gradient(result, x) - return grad - - @def_function.function - def outer_fn(x): - return almost_outer_fn(x) - - @def_function.function - def outer_outer_fn(x): - return outer_fn(x) - - x = constant_op.constant(5.0) - grad = outer_outer_fn(x) - self.assertAllEqual(grad, 2 * 5.0 + 1.0) - - @test_util.run_in_graph_and_eager_modes - def testDeeplyNestedDifferentiableFunctionTFGradientInDefun(self): - @def_function.function - def inner_inner_fn(a, b): - return math_ops.add(a, b) - - @def_function.function - def inner_fn(a, b): - return inner_inner_fn(a, b) - - @def_function.function - def middle_fn(a, b): - return a * inner_fn(a, b) - - @def_function.function - def outer_fn(x): - result = middle_fn(x, 1.0) - return gradients_impl.gradients(result, [x])[0] - - x = constant_op.constant(5.0) - grad = outer_fn(x) - self.assertAllEqual(grad, 2 * 5.0 + 1.0) - - @test_util.run_in_graph_and_eager_modes - def testDeeplyNestedDifferentiableFunctionTFGradientInNestedDefun(self): - @def_function.function - def inner_inner_fn(a, b): - return math_ops.add(a, b) - - @def_function.function - def inner_fn(a, b): - return inner_inner_fn(a, b) - - @def_function.function - def middle_fn(a, b): - return a * inner_fn(a, b) - - @def_function.function - def almost_outer_fn(x): - result = middle_fn(x, 1.0) - return gradients_impl.gradients(result, [x])[0] - - @def_function.function - def outer_fn(x): - return almost_outer_fn(x) - - x = constant_op.constant(5.0) - grad = outer_fn(x) - self.assertAllEqual(grad, 2 * 5.0 + 1.0) - - @test_util.run_in_graph_and_eager_modes - def testDeeplyNestedDifferentiableFunctionTFGradientInMultNestedDefun(self): - @def_function.function - def inner_inner_fn(a, b): - return math_ops.add(a, b) - - @def_function.function - def inner_fn(a, b): - return inner_inner_fn(a, b) - - @def_function.function - def middle_fn(a, b): - return a * inner_fn(a, b) - - @def_function.function - def almost_outer_fn(x): - result = middle_fn(x, 1.0) - return gradients_impl.gradients(result, [x])[0] - - @def_function.function - def outer_fn(x): - return almost_outer_fn(x) - - @def_function.function - def outer_outer_fn(x): - return outer_fn(x) - - x = constant_op.constant(5.0) - grad = outer_outer_fn(x) - self.assertAllEqual(grad, 2 * 5.0 + 1.0) - - def testDeeplyNestedDifferentiableFunctionWithVariable(self): - var = variables.Variable(constant_op.constant(1.0)) - - @def_function.function - def inner_fn(a, b): - return math_ops.add(a, b) - - @def_function.function - def middle_fn(a, b): - return a * inner_fn(a, b) - - @def_function.function - def outer_fn(x): - return middle_fn(x, var) - - x = constant_op.constant(5.0) - with backprop.GradientTape() as tp: - tp.watch(x) - result = outer_fn(x) - grad = tp.gradient(result, x) - - self.assertAllEqual(grad, 2 * 5.0 + 1.0) - - def testDeeplyNestedDifferentiableFunctionWithVariableMultipleGradCalls(self): - v = variables.Variable(constant_op.constant(3.0)) - - @def_function.function - def inner_fn(a, b): - return math_ops.add(a, b) - - @def_function.function - def middle_fn(a, b): - return math_ops.mul(a, inner_fn(a, b)) - - @def_function.function - def outer_fn(x): - return middle_fn(x, v) - - x = constant_op.constant(5.0) - self.assertAllEqual(outer_fn(x), 5.0 * (5.0 + 3.0)) - - with backprop.GradientTape() as tp: - tp.watch(x) - result = outer_fn(x) - grad = tp.gradient(result, x) - - self.assertAllEqual(grad, 2 * 5.0 + 3.0) - self.assertAllEqual(outer_fn(x), 5.0 * (5.0 + 3.0)) - self.assertAllEqual(middle_fn(v, x), 3.0 * (3.0 + 5.0)) - - with backprop.GradientTape() as tp: - tp.watch(x) - result = outer_fn(x) - grad = tp.gradient(result, x) - - self.assertAllEqual(grad, 2 * 5.0 + 3.0) - - y = constant_op.constant(4.0) - with backprop.GradientTape() as tp: - tp.watch(y) - result = outer_fn(y) - grad = tp.gradient(result, y) - - self.assertAllEqual(grad, 2 * 4.0 + 3.0) - - v.assign(constant_op.constant(1.5)) - with backprop.GradientTape() as tp: - tp.watch(y) - result = outer_fn(y) - grad = tp.gradient(result, y) - - self.assertAllEqual(grad, 2 * 4.0 + 1.5) - - with backprop.GradientTape() as tp: - tp.watch(y) - result = inner_fn(y, v) - grad = tp.gradient(result, y) - - self.assertAllEqual(grad, 1.0) - - def testDeeplyNestedDifferentiableFunctionWithVariableMultipleTFGrads(self): - with context.graph_mode(), self.cached_session(): - v = resource_variable_ops.ResourceVariable(3.0) - v.initializer.run() - - @def_function.function - def inner_fn(a, b): - return math_ops.add(a, b) - - @def_function.function - def middle_fn(a, b): - return math_ops.mul(a, inner_fn(a, b)) - - @def_function.function - def outer_fn(x): - return middle_fn(x, v) - - x = constant_op.constant(5.0) - self.assertAllEqual(outer_fn(x).eval(), 5.0 * (5.0 + 3.0)) - - grad, = gradients_impl.gradients(outer_fn(x), x) - - self.assertAllEqual(grad, 2 * 5.0 + 3.0) - self.assertAllEqual(outer_fn(x), 5.0 * (5.0 + 3.0)) - self.assertAllEqual(middle_fn(v, x), 3.0 * (3.0 + 5.0)) - - grad, = gradients_impl.gradients(outer_fn(x), x) - - self.assertAllEqual(grad, 2 * 5.0 + 3.0) - - y = constant_op.constant(4.0) - grad, = gradients_impl.gradients(outer_fn(y), y) - self.assertAllEqual(grad, 2 * 4.0 + 3.0) - - self.evaluate(v.assign(constant_op.constant(1.5))) - grad, = gradients_impl.gradients(outer_fn(y), y) - - self.assertAllEqual(grad, 2 * 4.0 + 1.5) - - grad, = gradients_impl.gradients(inner_fn(y, v), y) - self.assertAllEqual(grad, 1.0) - - def testNestedDifferentiableFunctionNoneOutputs(self): - @def_function.function - def foo(a, b): - return None, a * math_ops.add(a, b), None, 2*a - - @def_function.function - def bar(x): - return foo(x, 1.0) - - x = constant_op.constant(5.0) - with backprop.GradientTape(persistent=True) as tp: - tp.watch(x) - none1, r1, none2, r2 = bar(x) - g1 = tp.gradient(r1, x) - g2 = tp.gradient(r2, x) - - self.assertAllEqual(r1, 30.0) - self.assertAllEqual(r2, 10.0) - self.assertIs(none1, None) - self.assertIs(none2, None) - self.assertAllEqual(g1, 2 * 5.0 + 1.0) - self.assertAllEqual(g2, 2.0) - def testNoneOutput(self): @def_function.function @@ -1925,8 +1288,9 @@ class FunctionTest(test.TestCase, parameterized.TestCase): defined(array_ops.ones([2, 1])) # Wrong number of arguments. - with self.assertRaisesRegexp(ValueError, - 'Structure of Python function inputs.*'): + with self.assertRaisesRegexp( + ValueError, + 'Arguments and signature arguments do not match.*'): defined(array_ops.ones([2]), array_ops.ones([2])) with self.assertRaisesRegexp(ValueError, 'Structure of Python function inputs.*'): @@ -1945,10 +1309,16 @@ class FunctionTest(test.TestCase, parameterized.TestCase): else: return -1.0 * a - signature = [tensor_spec.TensorSpec([], dtypes.float32)] * 2 + signature = [ + tensor_spec.TensorSpec([], dtypes.float32), + tensor_spec.TensorSpec([], dtypes.bool), + ] defined = def_function.function(foo, input_signature=signature) a = constant_op.constant(1.0) - with self.assertRaises(TypeError): + with self.assertRaisesRegexp( + ValueError, + 'When input_signature is provided, all inputs to ' + 'the Python function must be Tensors.'): defined(a, training=True) def testInputSignatureWithKeywordPositionalArgs(self): @@ -2039,33 +1409,6 @@ class FunctionTest(test.TestCase, parameterized.TestCase): self.assertAllEqual(six, 2.0) self.assertAllEqual(seven, 2.0) - def testGradientWithKeywordArguments(self): - matmul = def_function.function(math_ops.matmul) - - def sq(x): - return matmul(a=x, b=x, transpose_a=True) - - t = constant_op.constant([[1.0, 2.0], [3.0, 4.0]]) - grad_t, = backprop.gradients_function(sq, [0])(t) - self.assertAllEqual(grad_t, [[6, 6], [14, 14]]) - - with backprop.GradientTape(persistent=True) as tape: - tape.watch(t) - one = matmul(t, b=t, transpose_a=True) - two = matmul(b=t, a=t, transpose_a=True) - three = matmul(a=t, b=t, transpose_a=True) - - for output in [one, two, three]: - self.assertAllEqual(tape.gradient(output, t), [[6, 6], [14, 14]]) - - def testGradientInFunctionWithKeywordArguments(self): - - @def_function.function - def f(x): - return backprop.gradients_function(lambda y: y * y, [0])(x)[0] - - self.assertAllEqual(f(x=constant_op.constant(1.0)), 2.0) - def testDefuningInstanceMethod(self): integer = constant_op.constant(2, dtypes.int64) @@ -2339,33 +1682,6 @@ class FunctionTest(test.TestCase, parameterized.TestCase): # pylint: disable=protected-access self.assertEqual(len(graph._functions), 3) - @test_util.run_in_graph_and_eager_modes - def testBackwardNone(self): - model = variables.Variable(1.0, name='model') - count = variables.Variable(0) - - @function.defun - def forward_pass(value): - count.assign_add(1) - residuals = value - model - loss = 0.5 * math_ops.reduce_mean(math_ops.pow(residuals, 2)) - # Note: count is an integer, so its doutput will be None - return loss, count - - def reduce_fn(x): - if context.executing_eagerly(): - with backprop.GradientTape() as t: - loss, count = forward_pass(x) - return t.gradient(loss, model), count - loss, count = forward_pass(x) - grad_only = gradients_impl.gradients(loss, model) - return grad_only, count - - g, _ = reduce_fn(constant_op.constant([7.0])) - - self.evaluate(variables.global_variables_initializer()) - self.assertAllEqual(nest.flatten(self.evaluate(g)), [-6.0]) - def testCallingFunctionWithDifferentVariables(self): @function.defun @@ -2403,8 +1719,7 @@ class FunctionTest(test.TestCase, parameterized.TestCase): 'be Tensors;.*'): graph_function('Not a Tensor.') - # TODO(scottzhu): Revive the test once the grappler plugin is updated. - def disabled_testSwapImplementationWithGrapplerPlugin(self): + def testSwapImplementationWithGrapplerPlugin(self): rewrites = rewriter_config_pb2.RewriterConfig() # function_optimizer has to be turn off, otherwise it will delete the # registered function if it does not get called. @@ -2441,7 +1756,7 @@ class FunctionTest(test.TestCase, parameterized.TestCase): function.register(cpu_boost, x) y = gpu_boost(x) - y_value = sess.run(y) + y_value = self.evaluate(y) if test.is_gpu_available(): self.assertEqual(y_value, 5.0) diff --git a/tensorflow/python/eager/graph_only_ops_test.py b/tensorflow/python/eager/graph_only_ops_test.py index 3cf3a61a62b..914b4d9a95a 100644 --- a/tensorflow/python/eager/graph_only_ops_test.py +++ b/tensorflow/python/eager/graph_only_ops_test.py @@ -29,12 +29,14 @@ from tensorflow.python.platform import test class GraphOnlyOpsTest(test_util.TensorFlowTestCase): + @test_util.run_deprecated_v1 def testGraphZerosLike(self): x = np.array([[1, 2, 3], [4, 5, 6]], dtype=np.int32) z_tf = graph_only_ops.graph_zeros_like(x) with self.cached_session(): - self.assertAllClose(np.zeros((2, 3)), z_tf.eval()) + self.assertAllClose(np.zeros((2, 3)), self.evaluate(z_tf)) + @test_util.run_deprecated_v1 def testGraphPlaceholder(self): x_tf = graph_only_ops.graph_placeholder(dtypes.int32, shape=(1,)) y_tf = math_ops.square(x_tf) diff --git a/tensorflow/python/eager/pywrap_tensor.cc b/tensorflow/python/eager/pywrap_tensor.cc index 55f0896e3b4..206b96eef60 100644 --- a/tensorflow/python/eager/pywrap_tensor.cc +++ b/tensorflow/python/eager/pywrap_tensor.cc @@ -220,6 +220,14 @@ TFE_TensorHandle* ConvertToEagerTensor(PyObject* value, PyObject* dtype) { return nullptr; } } + tensorflow::Safe_PyObjectPtr value_decrefer; + if (PyArray_CheckAnyScalarExact(value)) { + // Convert numpy scalars to numpy arrays. + value = PyArray_FromScalar(value, nullptr); + // The returned value needs to be DECREF'd, but the original value was + // created in python code, and doesn't need to be DECREF'd. + value_decrefer.reset(value); + } if (PyArray_Check(value)) { int desired_np_dtype = -1; if (desired_dtype >= 0) { @@ -439,8 +447,8 @@ int EagerTensor_init(EagerTensor* self, PyObject* args, PyObject* kwds) { PyErr_SetString( PyExc_TypeError, tensorflow::strings::StrCat( - "Cannot convert value ", TFE_GetPythonString(value_str.get()), - " to EagerTensor with requested dtype: ", + "Cannot convert provided value to EagerTensor. Provided value: ", + TFE_GetPythonString(value_str.get()), " Requested dtype: ", tensorflow::DataTypeString( static_cast(desired_dtype))) .c_str()); @@ -672,11 +680,29 @@ static PyObject* EagerTensor_device(EagerTensor* self) { #endif } +// Getter `backing_device`. +static PyObject* EagerTensor_backing_device(EagerTensor* self) { + const char* device = + TFE_TensorHandleBackingDeviceName(self->handle, self->status); + if (MaybeRaiseExceptionFromTFStatus(self->status, PyExc_ValueError)) { + // Cleanup self->status before returning. + TF_SetStatus(self->status, TF_OK, ""); + return nullptr; + } +#if PY_MAJOR_VERSION >= 3 + return PyUnicode_FromString(device); +#else + return PyBytes_FromString(device); +#endif +} + static PyGetSetDef EagerTensor_getseters[] = { {const_cast("_id"), (getter)EagerTensor_getid, nullptr, const_cast("_id"), nullptr}, {const_cast("device"), (getter)EagerTensor_device, nullptr, const_cast("device"), nullptr}, + {const_cast("backing_device"), (getter)EagerTensor_backing_device, + nullptr, const_cast("backing_device"), nullptr}, {const_cast("_handle_data"), (getter)EagerTensor_tensor_handle, (setter)EagerTensor_settensor_handle, const_cast("_tensor_handle"), nullptr}, diff --git a/tensorflow/python/eager/pywrap_tfe_src.cc b/tensorflow/python/eager/pywrap_tfe_src.cc index 6ca8eadbdeb..9ce500bc08e 100644 --- a/tensorflow/python/eager/pywrap_tfe_src.cc +++ b/tensorflow/python/eager/pywrap_tfe_src.cc @@ -1645,6 +1645,29 @@ PyObject* TFE_Py_TapeGradient(PyObject* tape, PyObject* target, if (PyErr_Occurred()) { return nullptr; } + tensorflow::gtl::FlatSet sources_set(sources_vec.begin(), + sources_vec.end()); + + tensorflow::Safe_PyObjectPtr seq = + tensorflow::make_safe(PySequence_Fast(target, "expected a sequence")); + int len = PySequence_Fast_GET_SIZE(seq.get()); + tensorflow::gtl::FlatMap + source_tensors_that_are_targets; + for (int i = 0; i < len; ++i) { + tensorflow::int64 target_id = target_vec[i]; + if (sources_set.find(target_id) != sources_set.end()) { + auto tensor = PySequence_Fast_GET_ITEM(seq.get(), i); + source_tensors_that_are_targets.insert( + std::make_pair(target_id, TapeTensorFromTensor(tensor))); + } + if (PyErr_Occurred()) { + return nullptr; + } + } + if (PyErr_Occurred()) { + return nullptr; + } + std::vector outgrad_vec; if (output_gradients != Py_None) { outgrad_vec = MakeTensorList(output_gradients); @@ -1659,7 +1682,8 @@ PyObject* TFE_Py_TapeGradient(PyObject* tape, PyObject* target, } std::vector result; status->status = tape_obj->tape->ComputeGradient( - *py_vspace, target_vec, sources_vec, outgrad_vec, &result); + *py_vspace, target_vec, sources_vec, source_tensors_that_are_targets, + outgrad_vec, &result); if (!status->status.ok()) { if (PyErr_Occurred()) { // Do not propagate the erroneous status as that would swallow the @@ -2279,8 +2303,10 @@ bool ConvertToTensor( PyErr_SetString( PyExc_TypeError, tensorflow::strings::StrCat( - "Cannot convert value ", TFE_GetPythonString(input_str.get()), - " to EagerTensor with requested dtype: ", desired_dtype) + "Cannot convert provided value to EagerTensor. Provided value: ", + TFE_GetPythonString(input_str.get()), " Requested dtype: ", + tensorflow::DataTypeString( + static_cast(desired_dtype))) .c_str()); return false; } diff --git a/tensorflow/python/eager/tape.py b/tensorflow/python/eager/tape.py index 1326f097130..e501b403a39 100644 --- a/tensorflow/python/eager/tape.py +++ b/tensorflow/python/eager/tape.py @@ -63,7 +63,7 @@ def watch_variable(tape, variable): """Marks this variable to be watched by the given tape.""" strategy = distribution_strategy_context.get_distribution_strategy() if distribution_strategy_context.get_replica_context(): - variables = [strategy.value_container(variable)] + variables = [strategy.extended.value_container(variable)] else: variables = strategy.unwrap(variable) for var in variables: @@ -78,7 +78,7 @@ def variable_accessed(variable): """ strategy = distribution_strategy_context.get_distribution_strategy() if distribution_strategy_context.get_replica_context(): - variables = [strategy.value_container(variable)] + variables = [strategy.extended.value_container(variable)] else: variables = strategy.unwrap(variable) for var in variables: diff --git a/tensorflow/python/eager/tape_test.py b/tensorflow/python/eager/tape_test.py index acd0e569f11..48d3b8ac6ee 100644 --- a/tensorflow/python/eager/tape_test.py +++ b/tensorflow/python/eager/tape_test.py @@ -80,8 +80,8 @@ class TapeTest(test.TestCase): tf_e = tf_d + tf_f tf_da, tf_db = gradients_impl.gradients(tf_e, [tf_a, tf_b]) - self.assertAllEqual(da, tf_da.eval()) - self.assertAllEqual(db, tf_db.eval()) + self.assertAllEqual(da, self.evaluate(tf_da)) + self.assertAllEqual(db, self.evaluate(tf_db)) def testBasicFunctional(self): @@ -142,8 +142,8 @@ class TapeTest(test.TestCase): tf_rr = 2 * math_ops.reduce_sum(tf_mm) tf_da, tf_db = gradients_impl.gradients(tf_rr, [tf_a, tf_b]) - self.assertAllEqual(da, tf_da.eval()) - self.assertAllEqual(db, tf_db.eval()) + self.assertAllEqual(da, self.evaluate(tf_da)) + self.assertAllEqual(db, self.evaluate(tf_db)) def testGcTwoOutputs(self): diff --git a/tensorflow/python/eager/tensor_test.py b/tensorflow/python/eager/tensor_test.py index f61d8478177..0ee2ff68c20 100644 --- a/tensorflow/python/eager/tensor_test.py +++ b/tensorflow/python/eager/tensor_test.py @@ -95,6 +95,18 @@ class TFETensorTest(test_util.TensorFlowTestCase): t = _create_tensor(values) self.assertAllEqual(values, t) + @test_util.assert_no_new_pyobjects_executing_eagerly + def testNumpyDtypeSurvivesThroughTensorConversion(self): + scalar_creators = [np.int32, np.int64, np.float32, np.float64] + conversion_functions = [ops.convert_to_tensor, constant_op.constant] + + for scalar_creator in scalar_creators: + for conversion_function in conversion_functions: + np_val = scalar_creator(3) + tensor_val = conversion_function(np_val) + self.assertEqual(tensor_val.numpy().dtype, np_val.dtype) + self.assertEqual(tensor_val.numpy(), np_val) + def testNumpyValueWithCast(self): values = np.array([3.0], dtype=np.float32) t = _create_tensor(values, dtype=dtypes.float64) @@ -128,6 +140,23 @@ class TFETensorTest(test_util.TensorFlowTestCase): tensor = constant_op.constant(numpy_tensor) self.assertAllEqual(numpy_tensor.ndim, tensor.ndim) + def testLenAgreesWithNumpy(self): + numpy_tensor = np.asarray(1.0) + tensor = constant_op.constant(numpy_tensor) + with self.assertRaises(TypeError): + len(numpy_tensor) + with self.assertRaisesRegexp( + TypeError, r"Scalar tensor has no `len[(][)]`"): + len(tensor) + + numpy_tensor = np.asarray([1.0, 2.0, 3.0]) + tensor = constant_op.constant(numpy_tensor) + self.assertAllEqual(len(numpy_tensor), len(tensor)) + + numpy_tensor = np.asarray([[1.0, 2.0, 3.0], [1.0, 2.0, 3.0]]) + tensor = constant_op.constant(numpy_tensor) + self.assertAllEqual(len(numpy_tensor), len(tensor)) + def testCopy(self): t = constant_op.constant(1.0) tt = copy.copy(t) @@ -158,9 +187,13 @@ class TFETensorTest(test_util.TensorFlowTestCase): self.assertEqual(dtypes.float64, t.dtype) def testBool(self): - t = _create_tensor(False) - if t: - self.assertFalse(True) + self.assertFalse(bool(_create_tensor(False))) + self.assertFalse(bool(_create_tensor([False]))) + self.assertFalse(bool(_create_tensor([[False]]))) + self.assertFalse(bool(_create_tensor([0]))) + self.assertFalse(bool(_create_tensor([0.]))) + self.assertTrue(bool(_create_tensor([1]))) + self.assertTrue(bool(_create_tensor([1.]))) def testIntDowncast(self): t = _create_tensor(3) @@ -306,6 +339,14 @@ class TFETensorTest(test_util.TensorFlowTestCase): def testConvertToTensorAllowsOverflow(self): _ = ops.convert_to_tensor(123456789, dtype=dtypes.uint8) + def testEagerTensorError(self): + with self.assertRaisesRegexp( + TypeError, + "Cannot convert provided value to EagerTensor. " + "Provided value.*Requested dtype.*"): + _ = ops.convert_to_tensor(1., dtype=dtypes.int32) + + class TFETensorUtilTest(test_util.TensorFlowTestCase): diff --git a/tensorflow/python/eager/test.py b/tensorflow/python/eager/test.py index 33ee797678e..a45deac962d 100644 --- a/tensorflow/python/eager/test.py +++ b/tensorflow/python/eager/test.py @@ -24,6 +24,6 @@ from tensorflow.python.platform.test import * # pylint: disable=wildcard-import # TODO(akshayka): Do away with this file. -def main(argv=None): +def main(argv=None): # pylint: disable=function-redefined _ops.enable_eager_execution() _test.main(argv) diff --git a/tensorflow/python/eager/wrap_function.py b/tensorflow/python/eager/wrap_function.py index 48266437ef5..2b39e99a4ea 100644 --- a/tensorflow/python/eager/wrap_function.py +++ b/tensorflow/python/eager/wrap_function.py @@ -26,6 +26,7 @@ from tensorflow.python.framework import ops from tensorflow.python.ops import array_ops from tensorflow.python.ops import variable_scope from tensorflow.python.util import nest +from tensorflow.python.util.tf_export import tf_export class VariableHolder(object): @@ -45,6 +46,7 @@ class VariableHolder(object): return self._fn(*args, **kwargs) +# TODO(allenl): make this checkpointable class WrappedFunction(function.Function): """Wraps a tf V1 piece of code in a function.""" @@ -77,6 +79,7 @@ class WrappedFunction(function.Function): return pruned_fn +@tf_export(v1=["wrap_function"]) def wrap_function(fn, signature, name=None): """Wraps the TF 1.x function fn into a graph function. @@ -109,6 +112,21 @@ def wrap_function(fn, signature, name=None): assert float(f_sub(1.0)) == 3.0 ``` + Both `tf.compat.v1.wrap_function` and `tf.function` create a callable + TensorFlow graph. But while `tf.function` runs all stateful operations + (e.g. `tf.print`) and sequences operations to provide the same semantics as + eager execution, `wrap_function` is closer to the behavior of `session.run` in + TensorFlow 1.x. It will not run any operations unless they are required to + compute the function's outputs, either through a data dependency or a control + dependency. Nor will it sequence operations. + + Unlike `tf.function`, `wrap_function` will only trace the Python function + once. As with placeholders in TF 1.x, shapes and dtypes must be provided to + `wrap_function`'s `signature` argument. + + Since it is only traced once, variables and state may be created inside the + function and owned by the function wrapper object. + Args: fn: python function to be wrapped signature: the placeholder and python arguments to be passed to the diff --git a/tensorflow/python/feature_column/feature_column.py b/tensorflow/python/feature_column/feature_column.py index b7a6a88535f..a858d92608d 100644 --- a/tensorflow/python/feature_column/feature_column.py +++ b/tensorflow/python/feature_column/feature_column.py @@ -230,7 +230,7 @@ def _internal_input_layer(features, return _get_logits() -@tf_export('feature_column.input_layer') +@tf_export(v1=['feature_column.input_layer']) def input_layer(features, feature_columns, weight_collections=None, @@ -365,7 +365,7 @@ class InputLayer(object): return self._input_layer_template.weights -@tf_export('feature_column.linear_model') +@tf_export(v1=['feature_column.linear_model']) def linear_model(features, feature_columns, units=1, @@ -746,7 +746,7 @@ def _transform_features(features, feature_columns): return outputs -@tf_export('feature_column.make_parse_example_spec') +@tf_export(v1=['feature_column.make_parse_example_spec']) def make_parse_example_spec(feature_columns): """Creates parsing spec dictionary from input feature_columns. @@ -807,11 +807,14 @@ def make_parse_example_spec(feature_columns): return result -@tf_export('feature_column.embedding_column') -def embedding_column( - categorical_column, dimension, combiner='mean', initializer=None, - ckpt_to_load_from=None, tensor_name_in_ckpt=None, max_norm=None, - trainable=True): +def _embedding_column(categorical_column, + dimension, + combiner='mean', + initializer=None, + ckpt_to_load_from=None, + tensor_name_in_ckpt=None, + max_norm=None, + trainable=True): """`_DenseColumn` that converts from sparse, categorical input. Use this when your inputs are sparse, but you want to convert them to a dense @@ -919,178 +922,11 @@ def embedding_column( trainable=trainable) -@tf_export('feature_column.shared_embedding_columns') -def shared_embedding_columns( - categorical_columns, dimension, combiner='mean', initializer=None, - shared_embedding_collection_name=None, ckpt_to_load_from=None, - tensor_name_in_ckpt=None, max_norm=None, trainable=True): - """List of dense columns that convert from sparse, categorical input. - - This is similar to `embedding_column`, except that it produces a list of - embedding columns that share the same embedding weights. - - Use this when your inputs are sparse and of the same type (e.g. watched and - impression video IDs that share the same vocabulary), and you want to convert - them to a dense representation (e.g., to feed to a DNN). - - Inputs must be a list of categorical columns created by any of the - `categorical_column_*` function. They must all be of the same type and have - the same arguments except `key`. E.g. they can be - categorical_column_with_vocabulary_file with the same vocabulary_file. Some or - all columns could also be weighted_categorical_column. - - Here is an example embedding of two features for a DNNClassifier model: - - ```python - watched_video_id = categorical_column_with_vocabulary_file( - 'watched_video_id', video_vocabulary_file, video_vocabulary_size) - impression_video_id = categorical_column_with_vocabulary_file( - 'impression_video_id', video_vocabulary_file, video_vocabulary_size) - columns = shared_embedding_columns( - [watched_video_id, impression_video_id], dimension=10) - - estimator = tf.estimator.DNNClassifier(feature_columns=columns, ...) - - label_column = ... - def input_fn(): - features = tf.parse_example( - ..., features=make_parse_example_spec(columns + [label_column])) - labels = features.pop(label_column.name) - return features, labels - - estimator.train(input_fn=input_fn, steps=100) - ``` - - Here is an example using `shared_embedding_columns` with model_fn: - - ```python - def model_fn(features, ...): - watched_video_id = categorical_column_with_vocabulary_file( - 'watched_video_id', video_vocabulary_file, video_vocabulary_size) - impression_video_id = categorical_column_with_vocabulary_file( - 'impression_video_id', video_vocabulary_file, video_vocabulary_size) - columns = shared_embedding_columns( - [watched_video_id, impression_video_id], dimension=10) - dense_tensor = input_layer(features, columns) - # Form DNN layers, calculate loss, and return EstimatorSpec. - ... - ``` - - Args: - categorical_columns: List of categorical columns created by a - `categorical_column_with_*` function. These columns produce the sparse IDs - that are inputs to the embedding lookup. All columns must be of the same - type and have the same arguments except `key`. E.g. they can be - categorical_column_with_vocabulary_file with the same vocabulary_file. - Some or all columns could also be weighted_categorical_column. - dimension: An integer specifying dimension of the embedding, must be > 0. - combiner: A string specifying how to reduce if there are multiple entries - in a single row. Currently 'mean', 'sqrtn' and 'sum' are supported, with - 'mean' the default. 'sqrtn' often achieves good accuracy, in particular - with bag-of-words columns. Each of this can be thought as example level - normalizations on the column. For more information, see - `tf.embedding_lookup_sparse`. - initializer: A variable initializer function to be used in embedding - variable initialization. If not specified, defaults to - `tf.truncated_normal_initializer` with mean `0.0` and standard deviation - `1/sqrt(dimension)`. - shared_embedding_collection_name: Optional name of the collection where - shared embedding weights are added. If not given, a reasonable name will - be chosen based on the names of `categorical_columns`. This is also used - in `variable_scope` when creating shared embedding weights. - ckpt_to_load_from: String representing checkpoint name/pattern from which to - restore column weights. Required if `tensor_name_in_ckpt` is not `None`. - tensor_name_in_ckpt: Name of the `Tensor` in `ckpt_to_load_from` from - which to restore the column weights. Required if `ckpt_to_load_from` is - not `None`. - max_norm: If not `None`, each embedding is clipped if its l2-norm is - larger than this value, before combining. - trainable: Whether or not the embedding is trainable. Default is True. - - Returns: - A list of dense columns that converts from sparse input. The order of - results follows the ordering of `categorical_columns`. - - Raises: - ValueError: if `dimension` not > 0. - ValueError: if any of the given `categorical_columns` is of different type - or has different arguments than the others. - ValueError: if exactly one of `ckpt_to_load_from` and `tensor_name_in_ckpt` - is specified. - ValueError: if `initializer` is specified and is not callable. - RuntimeError: if eager execution is enabled. - """ - if context.executing_eagerly(): - raise RuntimeError('shared_embedding_columns are not supported when eager ' - 'execution is enabled.') - - if (dimension is None) or (dimension < 1): - raise ValueError('Invalid dimension {}.'.format(dimension)) - if (ckpt_to_load_from is None) != (tensor_name_in_ckpt is None): - raise ValueError('Must specify both `ckpt_to_load_from` and ' - '`tensor_name_in_ckpt` or none of them.') - - if (initializer is not None) and (not callable(initializer)): - raise ValueError('initializer must be callable if specified.') - if initializer is None: - initializer = init_ops.truncated_normal_initializer( - mean=0.0, stddev=1. / math.sqrt(dimension)) - - # Sort the columns so the default collection name is deterministic even if the - # user passes columns from an unsorted collection, such as dict.values(). - sorted_columns = sorted(categorical_columns, key=lambda x: x.name) - - c0 = sorted_columns[0] - num_buckets = c0._num_buckets # pylint: disable=protected-access - if not isinstance(c0, _CategoricalColumn): - raise ValueError( - 'All categorical_columns must be subclasses of _CategoricalColumn. ' - 'Given: {}, of type: {}'.format(c0, type(c0))) - if isinstance(c0, _WeightedCategoricalColumn): - c0 = c0.categorical_column - for c in sorted_columns[1:]: - if isinstance(c, _WeightedCategoricalColumn): - c = c.categorical_column - if not isinstance(c, type(c0)): - raise ValueError( - 'To use shared_embedding_column, all categorical_columns must have ' - 'the same type, or be weighted_categorical_column of the same type. ' - 'Given column: {} of type: {} does not match given column: {} of ' - 'type: {}'.format(c0, type(c0), c, type(c))) - if num_buckets != c._num_buckets: # pylint: disable=protected-access - raise ValueError( - 'To use shared_embedding_column, all categorical_columns must have ' - 'the same number of buckets. Given column: {} with buckets: {} does ' - 'not match column: {} with buckets: {}'.format( - c0, num_buckets, c, c._num_buckets)) # pylint: disable=protected-access - - if not shared_embedding_collection_name: - shared_embedding_collection_name = '_'.join(c.name for c in sorted_columns) - shared_embedding_collection_name += '_shared_embedding' - - result = [] - for column in categorical_columns: - result.append( - _SharedEmbeddingColumn( - categorical_column=column, - initializer=initializer, - dimension=dimension, - combiner=combiner, - shared_embedding_collection_name=shared_embedding_collection_name, - ckpt_to_load_from=ckpt_to_load_from, - tensor_name_in_ckpt=tensor_name_in_ckpt, - max_norm=max_norm, - trainable=trainable)) - - return result - - -@tf_export('feature_column.numeric_column') -def numeric_column(key, - shape=(1,), - default_value=None, - dtype=dtypes.float32, - normalizer_fn=None): +def _numeric_column(key, + shape=(1,), + default_value=None, + dtype=dtypes.float32, + normalizer_fn=None): """Represents real valued or numerical features. Example: @@ -1161,8 +997,7 @@ def numeric_column(key, normalizer_fn=normalizer_fn) -@tf_export('feature_column.bucketized_column') -def bucketized_column(source_column, boundaries): +def _bucketized_column(source_column, boundaries): """Represents discretized dense input. Buckets include the left boundary, and exclude the right boundary. Namely, @@ -1258,10 +1093,9 @@ def _assert_key_is_string(key): type(key), key)) -@tf_export('feature_column.categorical_column_with_hash_bucket') -def categorical_column_with_hash_bucket(key, - hash_bucket_size, - dtype=dtypes.string): +def _categorical_column_with_hash_bucket(key, + hash_bucket_size, + dtype=dtypes.string): """Represents sparse feature where ids are set by hashing. Use this when your sparse features are in string or integer format, and you @@ -1317,13 +1151,12 @@ def categorical_column_with_hash_bucket(key, return _HashedCategoricalColumn(key, hash_bucket_size, dtype) -@tf_export('feature_column.categorical_column_with_vocabulary_file') -def categorical_column_with_vocabulary_file(key, - vocabulary_file, - vocabulary_size=None, - num_oov_buckets=0, - default_value=None, - dtype=dtypes.string): +def _categorical_column_with_vocabulary_file(key, + vocabulary_file, + vocabulary_size=None, + num_oov_buckets=0, + default_value=None, + dtype=dtypes.string): """A `_CategoricalColumn` with a vocabulary file. Use this when your inputs are in string or integer format, and you have a @@ -1437,9 +1270,11 @@ def categorical_column_with_vocabulary_file(key, dtype=dtype) -@tf_export('feature_column.categorical_column_with_vocabulary_list') -def categorical_column_with_vocabulary_list( - key, vocabulary_list, dtype=None, default_value=-1, num_oov_buckets=0): +def _categorical_column_with_vocabulary_list(key, + vocabulary_list, + dtype=None, + default_value=-1, + num_oov_buckets=0): """A `_CategoricalColumn` with in-memory vocabulary. Use this when your inputs are in string or integer format, and you have an @@ -1548,8 +1383,7 @@ def categorical_column_with_vocabulary_list( default_value=default_value, num_oov_buckets=num_oov_buckets) -@tf_export('feature_column.categorical_column_with_identity') -def categorical_column_with_identity(key, num_buckets, default_value=None): +def _categorical_column_with_identity(key, num_buckets, default_value=None): """A `_CategoricalColumn` that returns identity values. Use this when your inputs are integers in the range `[0, num_buckets)`, and @@ -1616,8 +1450,7 @@ def categorical_column_with_identity(key, num_buckets, default_value=None): key=key, num_buckets=num_buckets, default_value=default_value) -@tf_export('feature_column.indicator_column') -def indicator_column(categorical_column): +def _indicator_column(categorical_column): """Represents multi-hot representation of given categorical column. - For DNN model, `indicator_column` can be used to wrap any @@ -1651,9 +1484,9 @@ def indicator_column(categorical_column): return _IndicatorColumn(categorical_column) -@tf_export('feature_column.weighted_categorical_column') -def weighted_categorical_column( - categorical_column, weight_feature_key, dtype=dtypes.float32): +def _weighted_categorical_column(categorical_column, + weight_feature_key, + dtype=dtypes.float32): """Applies weight values to a `_CategoricalColumn`. Use this when each of your sparse inputs has both an ID and a value. For @@ -1726,8 +1559,7 @@ def weighted_categorical_column( dtype=dtype) -@tf_export('feature_column.crossed_column') -def crossed_column(keys, hash_bucket_size, hash_key=None): +def _crossed_column(keys, hash_bucket_size, hash_key=None): """Returns a column for performing crosses of categorical features. Crossed features will be hashed according to `hash_bucket_size`. Conceptually, diff --git a/tensorflow/python/feature_column/feature_column_lib.py b/tensorflow/python/feature_column/feature_column_lib.py index 3b818f18b5b..68a2712425c 100644 --- a/tensorflow/python/feature_column/feature_column_lib.py +++ b/tensorflow/python/feature_column/feature_column_lib.py @@ -20,4 +20,5 @@ from __future__ import print_function # pylint: disable=unused-import,line-too-long,wildcard-import from tensorflow.python.feature_column.feature_column import * +from tensorflow.python.feature_column.feature_column_v2 import * # pylint: enable=unused-import,line-too-long diff --git a/tensorflow/python/feature_column/feature_column_test.py b/tensorflow/python/feature_column/feature_column_test.py index 1ae510250cf..daa0a3b3a4b 100644 --- a/tensorflow/python/feature_column/feature_column_test.py +++ b/tensorflow/python/feature_column/feature_column_test.py @@ -30,7 +30,8 @@ from tensorflow.core.protobuf import rewriter_config_pb2 from tensorflow.python.client import session from tensorflow.python.eager import backprop from tensorflow.python.eager import context -from tensorflow.python.feature_column import feature_column_lib as fc +from tensorflow.python.feature_column import feature_column as fc +from tensorflow.python.feature_column import feature_column_v2 as fc_new from tensorflow.python.feature_column.feature_column import _CategoricalColumn from tensorflow.python.feature_column.feature_column import _DenseColumn from tensorflow.python.feature_column.feature_column import _FeatureColumn @@ -169,6 +170,7 @@ class LazyColumnTest(test.TestCase): TypeError, '"key" must be either a "str" or "_FeatureColumn".'): builder.get(NotAFeatureColumn()) + @test_util.run_deprecated_v1 def test_expand_dim_rank_1_sparse_tensor_empty_batch(self): # empty 1-D sparse tensor: builder = _LazyBuilder(features={'a': sparse_tensor.SparseTensor( @@ -184,8 +186,9 @@ class LazyColumnTest(test.TestCase): class NumericColumnTest(test.TestCase): + @test_util.run_deprecated_v1 def test_defaults(self): - a = fc.numeric_column('aaa') + a = fc._numeric_column('aaa') self.assertEqual('aaa', a.key) self.assertEqual('aaa', a.name) self.assertEqual('aaa', a._var_scope_name) @@ -196,53 +199,53 @@ class NumericColumnTest(test.TestCase): def test_key_should_be_string(self): with self.assertRaisesRegexp(ValueError, 'key must be a string.'): - fc.numeric_column(key=('aaa',)) + fc._numeric_column(key=('aaa',)) def test_shape_saved_as_tuple(self): - a = fc.numeric_column('aaa', shape=[1, 2], default_value=[[3, 2.]]) + a = fc._numeric_column('aaa', shape=[1, 2], default_value=[[3, 2.]]) self.assertEqual((1, 2), a.shape) def test_default_value_saved_as_tuple(self): - a = fc.numeric_column('aaa', default_value=4.) + a = fc._numeric_column('aaa', default_value=4.) self.assertEqual((4.,), a.default_value) - a = fc.numeric_column('aaa', shape=[1, 2], default_value=[[3, 2.]]) + a = fc._numeric_column('aaa', shape=[1, 2], default_value=[[3, 2.]]) self.assertEqual(((3., 2.),), a.default_value) def test_shape_and_default_value_compatibility(self): - fc.numeric_column('aaa', shape=[2], default_value=[1, 2.]) + fc._numeric_column('aaa', shape=[2], default_value=[1, 2.]) with self.assertRaisesRegexp(ValueError, 'The shape of default_value'): - fc.numeric_column('aaa', shape=[2], default_value=[1, 2, 3.]) - fc.numeric_column( + fc._numeric_column('aaa', shape=[2], default_value=[1, 2, 3.]) + fc._numeric_column( 'aaa', shape=[3, 2], default_value=[[2, 3], [1, 2], [2, 3.]]) with self.assertRaisesRegexp(ValueError, 'The shape of default_value'): - fc.numeric_column( + fc._numeric_column( 'aaa', shape=[3, 1], default_value=[[2, 3], [1, 2], [2, 3.]]) with self.assertRaisesRegexp(ValueError, 'The shape of default_value'): - fc.numeric_column( + fc._numeric_column( 'aaa', shape=[3, 3], default_value=[[2, 3], [1, 2], [2, 3.]]) def test_default_value_type_check(self): - fc.numeric_column( + fc._numeric_column( 'aaa', shape=[2], default_value=[1, 2.], dtype=dtypes.float32) - fc.numeric_column( + fc._numeric_column( 'aaa', shape=[2], default_value=[1, 2], dtype=dtypes.int32) with self.assertRaisesRegexp(TypeError, 'must be compatible with dtype'): - fc.numeric_column( + fc._numeric_column( 'aaa', shape=[2], default_value=[1, 2.], dtype=dtypes.int32) with self.assertRaisesRegexp(TypeError, 'default_value must be compatible with dtype'): - fc.numeric_column('aaa', default_value=['string']) + fc._numeric_column('aaa', default_value=['string']) def test_shape_must_be_positive_integer(self): with self.assertRaisesRegexp(TypeError, 'shape dimensions must be integer'): - fc.numeric_column( + fc._numeric_column( 'aaa', shape=[ 1.0, ]) with self.assertRaisesRegexp(ValueError, 'shape dimensions must be greater than 0'): - fc.numeric_column( + fc._numeric_column( 'aaa', shape=[ 0, ]) @@ -250,20 +253,21 @@ class NumericColumnTest(test.TestCase): def test_dtype_is_convertible_to_float(self): with self.assertRaisesRegexp(ValueError, 'dtype must be convertible to float'): - fc.numeric_column('aaa', dtype=dtypes.string) + fc._numeric_column('aaa', dtype=dtypes.string) def test_scalar_default_value_fills_the_shape(self): - a = fc.numeric_column('aaa', shape=[2, 3], default_value=2.) + a = fc._numeric_column('aaa', shape=[2, 3], default_value=2.) self.assertEqual(((2., 2., 2.), (2., 2., 2.)), a.default_value) def test_parse_spec(self): - a = fc.numeric_column('aaa', shape=[2, 3], dtype=dtypes.int32) + a = fc._numeric_column('aaa', shape=[2, 3], dtype=dtypes.int32) self.assertEqual({ 'aaa': parsing_ops.FixedLenFeature((2, 3), dtype=dtypes.int32) }, a._parse_example_spec) + @test_util.run_deprecated_v1 def test_parse_example_no_default_value(self): - price = fc.numeric_column('price', shape=[2]) + price = fc._numeric_column('price', shape=[2]) data = example_pb2.Example(features=feature_pb2.Features( feature={ 'price': @@ -277,8 +281,9 @@ class NumericColumnTest(test.TestCase): with self.cached_session(): self.assertAllEqual([[20., 110.]], features['price'].eval()) + @test_util.run_deprecated_v1 def test_parse_example_with_default_value(self): - price = fc.numeric_column('price', shape=[2], default_value=11.) + price = fc._numeric_column('price', shape=[2], default_value=11.) data = example_pb2.Example(features=feature_pb2.Features( feature={ 'price': @@ -301,29 +306,31 @@ class NumericColumnTest(test.TestCase): def test_normalizer_fn_must_be_callable(self): with self.assertRaisesRegexp(TypeError, 'must be a callable'): - fc.numeric_column('price', normalizer_fn='NotACallable') + fc._numeric_column('price', normalizer_fn='NotACallable') + @test_util.run_deprecated_v1 def test_normalizer_fn_transform_feature(self): def _increment_two(input_tensor): return input_tensor + 2. - price = fc.numeric_column('price', shape=[2], normalizer_fn=_increment_two) + price = fc._numeric_column('price', shape=[2], normalizer_fn=_increment_two) output = _transform_features({'price': [[1., 2.], [5., 6.]]}, [price]) with self.cached_session(): self.assertAllEqual([[3., 4.], [7., 8.]], output[price].eval()) + @test_util.run_deprecated_v1 def test_get_dense_tensor(self): def _increment_two(input_tensor): return input_tensor + 2. - price = fc.numeric_column('price', shape=[2], normalizer_fn=_increment_two) + price = fc._numeric_column('price', shape=[2], normalizer_fn=_increment_two) builder = _LazyBuilder({'price': [[1., 2.], [5., 6.]]}) self.assertEqual(builder.get(price), price._get_dense_tensor(builder)) def test_sparse_tensor_not_supported(self): - price = fc.numeric_column('price') + price = fc._numeric_column('price') builder = _LazyBuilder({ 'price': sparse_tensor.SparseTensor( @@ -332,109 +339,113 @@ class NumericColumnTest(test.TestCase): with self.assertRaisesRegexp(ValueError, 'must be a Tensor'): price._transform_feature(builder) + @test_util.run_deprecated_v1 def test_deep_copy(self): - a = fc.numeric_column('aaa', shape=[1, 2], default_value=[[3., 2.]]) + a = fc._numeric_column('aaa', shape=[1, 2], default_value=[[3., 2.]]) a_copy = copy.deepcopy(a) self.assertEqual(a_copy.name, 'aaa') self.assertEqual(a_copy.shape, (1, 2)) self.assertEqual(a_copy.default_value, ((3., 2.),)) def test_numpy_default_value(self): - a = fc.numeric_column( + a = fc._numeric_column( 'aaa', shape=[1, 2], default_value=np.array([[3., 2.]])) self.assertEqual(a.default_value, ((3., 2.),)) + @test_util.run_deprecated_v1 def test_linear_model(self): - price = fc.numeric_column('price') + price = fc._numeric_column('price') with ops.Graph().as_default(): features = {'price': [[1.], [5.]]} predictions = fc.linear_model(features, [price]) bias = get_linear_model_bias() price_var = get_linear_model_column_var(price) with _initialized_session() as sess: - self.assertAllClose([0.], bias.eval()) - self.assertAllClose([[0.]], price_var.eval()) - self.assertAllClose([[0.], [0.]], predictions.eval()) + self.assertAllClose([0.], self.evaluate(bias)) + self.assertAllClose([[0.]], self.evaluate(price_var)) + self.assertAllClose([[0.], [0.]], self.evaluate(predictions)) sess.run(price_var.assign([[10.]])) - self.assertAllClose([[10.], [50.]], predictions.eval()) + self.assertAllClose([[10.], [50.]], self.evaluate(predictions)) + @test_util.run_deprecated_v1 def test_keras_linear_model(self): - price = fc.numeric_column('price') + price = fc._numeric_column('price') with ops.Graph().as_default(): features = {'price': [[1.], [5.]]} predictions = get_keras_linear_model_predictions(features, [price]) bias = get_linear_model_bias() price_var = get_linear_model_column_var(price) with _initialized_session() as sess: - self.assertAllClose([0.], bias.eval()) - self.assertAllClose([[0.]], price_var.eval()) - self.assertAllClose([[0.], [0.]], predictions.eval()) + self.assertAllClose([0.], self.evaluate(bias)) + self.assertAllClose([[0.]], self.evaluate(price_var)) + self.assertAllClose([[0.], [0.]], self.evaluate(predictions)) sess.run(price_var.assign([[10.]])) - self.assertAllClose([[10.], [50.]], predictions.eval()) + self.assertAllClose([[10.], [50.]], self.evaluate(predictions)) class BucketizedColumnTest(test.TestCase): def test_invalid_source_column_type(self): - a = fc.categorical_column_with_hash_bucket('aaa', hash_bucket_size=10) + a = fc._categorical_column_with_hash_bucket('aaa', hash_bucket_size=10) with self.assertRaisesRegexp( ValueError, 'source_column must be a column generated with numeric_column'): - fc.bucketized_column(a, boundaries=[0, 1]) + fc._bucketized_column(a, boundaries=[0, 1]) def test_invalid_source_column_shape(self): - a = fc.numeric_column('aaa', shape=[2, 3]) + a = fc._numeric_column('aaa', shape=[2, 3]) with self.assertRaisesRegexp( ValueError, 'source_column must be one-dimensional column'): - fc.bucketized_column(a, boundaries=[0, 1]) + fc._bucketized_column(a, boundaries=[0, 1]) def test_invalid_boundaries(self): - a = fc.numeric_column('aaa') + a = fc._numeric_column('aaa') with self.assertRaisesRegexp( ValueError, 'boundaries must be a sorted list'): - fc.bucketized_column(a, boundaries=None) + fc._bucketized_column(a, boundaries=None) with self.assertRaisesRegexp( ValueError, 'boundaries must be a sorted list'): - fc.bucketized_column(a, boundaries=1.) + fc._bucketized_column(a, boundaries=1.) with self.assertRaisesRegexp( ValueError, 'boundaries must be a sorted list'): - fc.bucketized_column(a, boundaries=[1, 0]) + fc._bucketized_column(a, boundaries=[1, 0]) with self.assertRaisesRegexp( ValueError, 'boundaries must be a sorted list'): - fc.bucketized_column(a, boundaries=[1, 1]) + fc._bucketized_column(a, boundaries=[1, 1]) def test_name(self): - a = fc.numeric_column('aaa', dtype=dtypes.int32) - b = fc.bucketized_column(a, boundaries=[0, 1]) + a = fc._numeric_column('aaa', dtype=dtypes.int32) + b = fc._bucketized_column(a, boundaries=[0, 1]) self.assertEqual('aaa_bucketized', b.name) def test_var_scope_name(self): - a = fc.numeric_column('aaa', dtype=dtypes.int32) - b = fc.bucketized_column(a, boundaries=[0, 1]) + a = fc._numeric_column('aaa', dtype=dtypes.int32) + b = fc._bucketized_column(a, boundaries=[0, 1]) self.assertEqual('aaa_bucketized', b._var_scope_name) def test_parse_spec(self): - a = fc.numeric_column('aaa', shape=[2], dtype=dtypes.int32) - b = fc.bucketized_column(a, boundaries=[0, 1]) + a = fc._numeric_column('aaa', shape=[2], dtype=dtypes.int32) + b = fc._bucketized_column(a, boundaries=[0, 1]) self.assertEqual({ 'aaa': parsing_ops.FixedLenFeature((2,), dtype=dtypes.int32) }, b._parse_example_spec) def test_variable_shape(self): - a = fc.numeric_column('aaa', shape=[2], dtype=dtypes.int32) - b = fc.bucketized_column(a, boundaries=[0, 1]) + a = fc._numeric_column('aaa', shape=[2], dtype=dtypes.int32) + b = fc._bucketized_column(a, boundaries=[0, 1]) # Column 'aaa` has shape [2] times three buckets -> variable_shape=[2, 3]. self.assertAllEqual((2, 3), b._variable_shape) def test_num_buckets(self): - a = fc.numeric_column('aaa', shape=[2], dtype=dtypes.int32) - b = fc.bucketized_column(a, boundaries=[0, 1]) + a = fc._numeric_column('aaa', shape=[2], dtype=dtypes.int32) + b = fc._bucketized_column(a, boundaries=[0, 1]) # Column 'aaa` has shape [2] times three buckets -> num_buckets=6. self.assertEqual(6, b._num_buckets) + @test_util.run_deprecated_v1 def test_parse_example(self): - price = fc.numeric_column('price', shape=[2]) - bucketized_price = fc.bucketized_column(price, boundaries=[0, 50]) + price = fc._numeric_column('price', shape=[2]) + bucketized_price = fc._bucketized_column(price, boundaries=[0, 50]) data = example_pb2.Example(features=feature_pb2.Features( feature={ 'price': @@ -448,9 +459,10 @@ class BucketizedColumnTest(test.TestCase): with self.cached_session(): self.assertAllEqual([[20., 110.]], features['price'].eval()) + @test_util.run_deprecated_v1 def test_transform_feature(self): - price = fc.numeric_column('price', shape=[2]) - bucketized_price = fc.bucketized_column(price, boundaries=[0, 2, 4, 6]) + price = fc._numeric_column('price', shape=[2]) + bucketized_price = fc._bucketized_column(price, boundaries=[0, 2, 4, 6]) with ops.Graph().as_default(): transformed_tensor = _transform_features({ 'price': [[-1., 1.], [5., 6.]] @@ -461,24 +473,22 @@ class BucketizedColumnTest(test.TestCase): def test_get_dense_tensor_one_input_value(self): """Tests _get_dense_tensor() for input with shape=[1].""" - price = fc.numeric_column('price', shape=[1]) - bucketized_price = fc.bucketized_column(price, boundaries=[0, 2, 4, 6]) + price = fc._numeric_column('price', shape=[1]) + bucketized_price = fc._bucketized_column(price, boundaries=[0, 2, 4, 6]) with ops.Graph().as_default(): builder = _LazyBuilder({'price': [[-1.], [1.], [5.], [6.]]}) with _initialized_session(): bucketized_price_tensor = bucketized_price._get_dense_tensor(builder) self.assertAllClose( # One-hot tensor. - [[[1., 0., 0., 0., 0.]], - [[0., 1., 0., 0., 0.]], - [[0., 0., 0., 1., 0.]], - [[0., 0., 0., 0., 1.]]], - bucketized_price_tensor.eval()) + [[[1., 0., 0., 0., 0.]], [[0., 1., 0., 0., 0.]], + [[0., 0., 0., 1., 0.]], [[0., 0., 0., 0., 1.]]], + self.evaluate(bucketized_price_tensor)) def test_get_dense_tensor_two_input_values(self): """Tests _get_dense_tensor() for input with shape=[2].""" - price = fc.numeric_column('price', shape=[2]) - bucketized_price = fc.bucketized_column(price, boundaries=[0, 2, 4, 6]) + price = fc._numeric_column('price', shape=[2]) + bucketized_price = fc._bucketized_column(price, boundaries=[0, 2, 4, 6]) with ops.Graph().as_default(): builder = _LazyBuilder({'price': [[-1., 1.], [5., 6.]]}) with _initialized_session(): @@ -487,12 +497,12 @@ class BucketizedColumnTest(test.TestCase): # One-hot tensor. [[[1., 0., 0., 0., 0.], [0., 1., 0., 0., 0.]], [[0., 0., 0., 1., 0.], [0., 0., 0., 0., 1.]]], - bucketized_price_tensor.eval()) + self.evaluate(bucketized_price_tensor)) def test_get_sparse_tensors_one_input_value(self): """Tests _get_sparse_tensors() for input with shape=[1].""" - price = fc.numeric_column('price', shape=[1]) - bucketized_price = fc.bucketized_column(price, boundaries=[0, 2, 4, 6]) + price = fc._numeric_column('price', shape=[1]) + bucketized_price = fc._bucketized_column(price, boundaries=[0, 2, 4, 6]) with ops.Graph().as_default(): builder = _LazyBuilder({'price': [[-1.], [1.], [5.], [6.]]}) with _initialized_session() as sess: @@ -506,8 +516,8 @@ class BucketizedColumnTest(test.TestCase): def test_get_sparse_tensors_two_input_values(self): """Tests _get_sparse_tensors() for input with shape=[2].""" - price = fc.numeric_column('price', shape=[2]) - bucketized_price = fc.bucketized_column(price, boundaries=[0, 2, 4, 6]) + price = fc._numeric_column('price', shape=[2]) + bucketized_price = fc._bucketized_column(price, boundaries=[0, 2, 4, 6]) with ops.Graph().as_default(): builder = _LazyBuilder({'price': [[-1., 1.], [5., 6.]]}) with _initialized_session() as sess: @@ -522,8 +532,8 @@ class BucketizedColumnTest(test.TestCase): self.assertAllEqual([2, 2], id_tensor_value.dense_shape) def test_sparse_tensor_input_not_supported(self): - price = fc.numeric_column('price') - bucketized_price = fc.bucketized_column(price, boundaries=[0, 1]) + price = fc._numeric_column('price') + bucketized_price = fc._bucketized_column(price, boundaries=[0, 1]) builder = _LazyBuilder({ 'price': sparse_tensor.SparseTensor( @@ -532,9 +542,10 @@ class BucketizedColumnTest(test.TestCase): with self.assertRaisesRegexp(ValueError, 'must be a Tensor'): bucketized_price._transform_feature(builder) + @test_util.run_deprecated_v1 def test_deep_copy(self): - a = fc.numeric_column('aaa', shape=[2]) - a_bucketized = fc.bucketized_column(a, boundaries=[0, 1]) + a = fc._numeric_column('aaa', shape=[2]) + a_bucketized = fc._bucketized_column(a, boundaries=[0, 1]) a_bucketized_copy = copy.deepcopy(a_bucketized) self.assertEqual(a_bucketized_copy.name, 'aaa_bucketized') self.assertAllEqual(a_bucketized_copy._variable_shape, (2, 3)) @@ -542,45 +553,48 @@ class BucketizedColumnTest(test.TestCase): def test_linear_model_one_input_value(self): """Tests linear_model() for input with shape=[1].""" - price = fc.numeric_column('price', shape=[1]) - bucketized_price = fc.bucketized_column(price, boundaries=[0, 2, 4, 6]) + price = fc._numeric_column('price', shape=[1]) + bucketized_price = fc._bucketized_column(price, boundaries=[0, 2, 4, 6]) with ops.Graph().as_default(): features = {'price': [[-1.], [1.], [5.], [6.]]} predictions = fc.linear_model(features, [bucketized_price]) bias = get_linear_model_bias() bucketized_price_var = get_linear_model_column_var(bucketized_price) with _initialized_session() as sess: - self.assertAllClose([0.], bias.eval()) + self.assertAllClose([0.], self.evaluate(bias)) # One weight variable per bucket, all initialized to zero. - self.assertAllClose( - [[0.], [0.], [0.], [0.], [0.]], bucketized_price_var.eval()) - self.assertAllClose([[0.], [0.], [0.], [0.]], predictions.eval()) + self.assertAllClose([[0.], [0.], [0.], [0.], [0.]], + self.evaluate(bucketized_price_var)) + self.assertAllClose([[0.], [0.], [0.], [0.]], + self.evaluate(predictions)) sess.run(bucketized_price_var.assign( [[10.], [20.], [30.], [40.], [50.]])) # price -1. is in the 0th bucket, whose weight is 10. # price 1. is in the 1st bucket, whose weight is 20. # price 5. is in the 3rd bucket, whose weight is 40. # price 6. is in the 4th bucket, whose weight is 50. - self.assertAllClose([[10.], [20.], [40.], [50.]], predictions.eval()) + self.assertAllClose([[10.], [20.], [40.], [50.]], + self.evaluate(predictions)) sess.run(bias.assign([1.])) - self.assertAllClose([[11.], [21.], [41.], [51.]], predictions.eval()) + self.assertAllClose([[11.], [21.], [41.], [51.]], + self.evaluate(predictions)) def test_linear_model_two_input_values(self): """Tests linear_model() for input with shape=[2].""" - price = fc.numeric_column('price', shape=[2]) - bucketized_price = fc.bucketized_column(price, boundaries=[0, 2, 4, 6]) + price = fc._numeric_column('price', shape=[2]) + bucketized_price = fc._bucketized_column(price, boundaries=[0, 2, 4, 6]) with ops.Graph().as_default(): features = {'price': [[-1., 1.], [5., 6.]]} predictions = fc.linear_model(features, [bucketized_price]) bias = get_linear_model_bias() bucketized_price_var = get_linear_model_column_var(bucketized_price) with _initialized_session() as sess: - self.assertAllClose([0.], bias.eval()) + self.assertAllClose([0.], self.evaluate(bias)) # One weight per bucket per input column, all initialized to zero. self.assertAllClose( [[0.], [0.], [0.], [0.], [0.], [0.], [0.], [0.], [0.], [0.]], - bucketized_price_var.eval()) - self.assertAllClose([[0.], [0.]], predictions.eval()) + self.evaluate(bucketized_price_var)) + self.assertAllClose([[0.], [0.]], self.evaluate(predictions)) sess.run(bucketized_price_var.assign( [[10.], [20.], [30.], [40.], [50.], [60.], [70.], [80.], [90.], [100.]])) @@ -590,14 +604,14 @@ class BucketizedColumnTest(test.TestCase): # 2nd example: # price 5. is in the 3rd bucket, whose weight is 40. # price 6. is in the 9th bucket, whose weight is 100. - self.assertAllClose([[80.], [140.]], predictions.eval()) + self.assertAllClose([[80.], [140.]], self.evaluate(predictions)) sess.run(bias.assign([1.])) - self.assertAllClose([[81.], [141.]], predictions.eval()) + self.assertAllClose([[81.], [141.]], self.evaluate(predictions)) def test_keras_linear_model_one_input_value(self): """Tests _LinearModel for input with shape=[1].""" - price = fc.numeric_column('price', shape=[1]) - bucketized_price = fc.bucketized_column(price, boundaries=[0, 2, 4, 6]) + price = fc._numeric_column('price', shape=[1]) + bucketized_price = fc._bucketized_column(price, boundaries=[0, 2, 4, 6]) with ops.Graph().as_default(): features = {'price': [[-1.], [1.], [5.], [6.]]} predictions = get_keras_linear_model_predictions(features, @@ -605,25 +619,28 @@ class BucketizedColumnTest(test.TestCase): bias = get_linear_model_bias() bucketized_price_var = get_linear_model_column_var(bucketized_price) with _initialized_session() as sess: - self.assertAllClose([0.], bias.eval()) + self.assertAllClose([0.], self.evaluate(bias)) # One weight variable per bucket, all initialized to zero. self.assertAllClose([[0.], [0.], [0.], [0.], [0.]], - bucketized_price_var.eval()) - self.assertAllClose([[0.], [0.], [0.], [0.]], predictions.eval()) + self.evaluate(bucketized_price_var)) + self.assertAllClose([[0.], [0.], [0.], [0.]], + self.evaluate(predictions)) sess.run( bucketized_price_var.assign([[10.], [20.], [30.], [40.], [50.]])) # price -1. is in the 0th bucket, whose weight is 10. # price 1. is in the 1st bucket, whose weight is 20. # price 5. is in the 3rd bucket, whose weight is 40. # price 6. is in the 4th bucket, whose weight is 50. - self.assertAllClose([[10.], [20.], [40.], [50.]], predictions.eval()) + self.assertAllClose([[10.], [20.], [40.], [50.]], + self.evaluate(predictions)) sess.run(bias.assign([1.])) - self.assertAllClose([[11.], [21.], [41.], [51.]], predictions.eval()) + self.assertAllClose([[11.], [21.], [41.], [51.]], + self.evaluate(predictions)) def test_keras_linear_model_two_input_values(self): """Tests _LinearModel for input with shape=[2].""" - price = fc.numeric_column('price', shape=[2]) - bucketized_price = fc.bucketized_column(price, boundaries=[0, 2, 4, 6]) + price = fc._numeric_column('price', shape=[2]) + bucketized_price = fc._bucketized_column(price, boundaries=[0, 2, 4, 6]) with ops.Graph().as_default(): features = {'price': [[-1., 1.], [5., 6.]]} predictions = get_keras_linear_model_predictions(features, @@ -631,12 +648,12 @@ class BucketizedColumnTest(test.TestCase): bias = get_linear_model_bias() bucketized_price_var = get_linear_model_column_var(bucketized_price) with _initialized_session() as sess: - self.assertAllClose([0.], bias.eval()) + self.assertAllClose([0.], self.evaluate(bias)) # One weight per bucket per input column, all initialized to zero. self.assertAllClose( [[0.], [0.], [0.], [0.], [0.], [0.], [0.], [0.], [0.], [0.]], - bucketized_price_var.eval()) - self.assertAllClose([[0.], [0.]], predictions.eval()) + self.evaluate(bucketized_price_var)) + self.assertAllClose([[0.], [0.]], self.evaluate(predictions)) sess.run( bucketized_price_var.assign([[10.], [20.], [30.], [40.], [50.], [60.], [70.], [80.], [90.], [100.]])) @@ -646,15 +663,16 @@ class BucketizedColumnTest(test.TestCase): # 2nd example: # price 5. is in the 3rd bucket, whose weight is 40. # price 6. is in the 9th bucket, whose weight is 100. - self.assertAllClose([[80.], [140.]], predictions.eval()) + self.assertAllClose([[80.], [140.]], self.evaluate(predictions)) sess.run(bias.assign([1.])) - self.assertAllClose([[81.], [141.]], predictions.eval()) + self.assertAllClose([[81.], [141.]], self.evaluate(predictions)) class HashedCategoricalColumnTest(test.TestCase): + @test_util.run_deprecated_v1 def test_defaults(self): - a = fc.categorical_column_with_hash_bucket('aaa', 10) + a = fc._categorical_column_with_hash_bucket('aaa', 10) self.assertEqual('aaa', a.name) self.assertEqual('aaa', a._var_scope_name) self.assertEqual('aaa', a.key) @@ -663,25 +681,26 @@ class HashedCategoricalColumnTest(test.TestCase): def test_key_should_be_string(self): with self.assertRaisesRegexp(ValueError, 'key must be a string.'): - fc.categorical_column_with_hash_bucket(('key',), 10) + fc._categorical_column_with_hash_bucket(('key',), 10) def test_bucket_size_should_be_given(self): with self.assertRaisesRegexp(ValueError, 'hash_bucket_size must be set.'): - fc.categorical_column_with_hash_bucket('aaa', None) + fc._categorical_column_with_hash_bucket('aaa', None) def test_bucket_size_should_be_positive(self): with self.assertRaisesRegexp(ValueError, 'hash_bucket_size must be at least 1'): - fc.categorical_column_with_hash_bucket('aaa', 0) + fc._categorical_column_with_hash_bucket('aaa', 0) def test_dtype_should_be_string_or_integer(self): - fc.categorical_column_with_hash_bucket('aaa', 10, dtype=dtypes.string) - fc.categorical_column_with_hash_bucket('aaa', 10, dtype=dtypes.int32) + fc._categorical_column_with_hash_bucket('aaa', 10, dtype=dtypes.string) + fc._categorical_column_with_hash_bucket('aaa', 10, dtype=dtypes.int32) with self.assertRaisesRegexp(ValueError, 'dtype must be string or integer'): - fc.categorical_column_with_hash_bucket('aaa', 10, dtype=dtypes.float32) + fc._categorical_column_with_hash_bucket('aaa', 10, dtype=dtypes.float32) + @test_util.run_deprecated_v1 def test_deep_copy(self): - original = fc.categorical_column_with_hash_bucket('aaa', 10) + original = fc._categorical_column_with_hash_bucket('aaa', 10) for column in (original, copy.deepcopy(original)): self.assertEqual('aaa', column.name) self.assertEqual(10, column.hash_bucket_size) @@ -689,19 +708,20 @@ class HashedCategoricalColumnTest(test.TestCase): self.assertEqual(dtypes.string, column.dtype) def test_parse_spec_string(self): - a = fc.categorical_column_with_hash_bucket('aaa', 10) + a = fc._categorical_column_with_hash_bucket('aaa', 10) self.assertEqual({ 'aaa': parsing_ops.VarLenFeature(dtypes.string) }, a._parse_example_spec) def test_parse_spec_int(self): - a = fc.categorical_column_with_hash_bucket('aaa', 10, dtype=dtypes.int32) + a = fc._categorical_column_with_hash_bucket('aaa', 10, dtype=dtypes.int32) self.assertEqual({ 'aaa': parsing_ops.VarLenFeature(dtypes.int32) }, a._parse_example_spec) + @test_util.run_deprecated_v1 def test_parse_example(self): - a = fc.categorical_column_with_hash_bucket('aaa', 10) + a = fc._categorical_column_with_hash_bucket('aaa', 10) data = example_pb2.Example(features=feature_pb2.Features( feature={ 'aaa': @@ -721,8 +741,9 @@ class HashedCategoricalColumnTest(test.TestCase): dense_shape=[1, 2]), features['aaa'].eval()) + @test_util.run_deprecated_v1 def test_strings_should_be_hashed(self): - hashed_sparse = fc.categorical_column_with_hash_bucket('wire', 10) + hashed_sparse = fc._categorical_column_with_hash_bucket('wire', 10) wire_tensor = sparse_tensor.SparseTensor( values=['omar', 'stringer', 'marlo'], indices=[[0, 0], [1, 0], [1, 1]], @@ -739,11 +760,11 @@ class HashedCategoricalColumnTest(test.TestCase): output.dense_shape.eval()) def test_tensor_dtype_should_be_string_or_integer(self): - string_fc = fc.categorical_column_with_hash_bucket( + string_fc = fc._categorical_column_with_hash_bucket( 'a_string', 10, dtype=dtypes.string) - int_fc = fc.categorical_column_with_hash_bucket( + int_fc = fc._categorical_column_with_hash_bucket( 'a_int', 10, dtype=dtypes.int32) - float_fc = fc.categorical_column_with_hash_bucket( + float_fc = fc._categorical_column_with_hash_bucket( 'a_float', 10, dtype=dtypes.string) int_tensor = sparse_tensor.SparseTensor( values=[101], @@ -768,7 +789,7 @@ class HashedCategoricalColumnTest(test.TestCase): builder.get(float_fc) def test_dtype_should_match_with_tensor(self): - hashed_sparse = fc.categorical_column_with_hash_bucket( + hashed_sparse = fc._categorical_column_with_hash_bucket( 'wire', 10, dtype=dtypes.int64) wire_tensor = sparse_tensor.SparseTensor( values=['omar'], indices=[[0, 0]], dense_shape=[1, 1]) @@ -776,8 +797,9 @@ class HashedCategoricalColumnTest(test.TestCase): with self.assertRaisesRegexp(ValueError, 'dtype must be compatible'): builder.get(hashed_sparse) + @test_util.run_deprecated_v1 def test_ints_should_be_hashed(self): - hashed_sparse = fc.categorical_column_with_hash_bucket( + hashed_sparse = fc._categorical_column_with_hash_bucket( 'wire', 10, dtype=dtypes.int64) wire_tensor = sparse_tensor.SparseTensor( values=[101, 201, 301], @@ -790,8 +812,9 @@ class HashedCategoricalColumnTest(test.TestCase): with self.cached_session(): self.assertAllEqual(expected_values, output.values.eval()) + @test_util.run_deprecated_v1 def test_int32_64_is_compatible(self): - hashed_sparse = fc.categorical_column_with_hash_bucket( + hashed_sparse = fc._categorical_column_with_hash_bucket( 'wire', 10, dtype=dtypes.int64) wire_tensor = sparse_tensor.SparseTensor( values=constant_op.constant([101, 201, 301], dtype=dtypes.int32), @@ -804,8 +827,9 @@ class HashedCategoricalColumnTest(test.TestCase): with self.cached_session(): self.assertAllEqual(expected_values, output.values.eval()) + @test_util.run_deprecated_v1 def test_get_sparse_tensors(self): - hashed_sparse = fc.categorical_column_with_hash_bucket('wire', 10) + hashed_sparse = fc._categorical_column_with_hash_bucket('wire', 10) builder = _LazyBuilder({ 'wire': sparse_tensor.SparseTensor( @@ -818,7 +842,7 @@ class HashedCategoricalColumnTest(test.TestCase): self.assertEqual(builder.get(hashed_sparse), id_weight_pair.id_tensor) def test_get_sparse_tensors_weight_collections(self): - column = fc.categorical_column_with_hash_bucket('aaa', 10) + column = fc._categorical_column_with_hash_bucket('aaa', 10) inputs = sparse_tensor.SparseTensor( values=['omar', 'stringer', 'marlo'], indices=[[0, 0], [1, 0], [1, 1]], @@ -832,15 +856,17 @@ class HashedCategoricalColumnTest(test.TestCase): [], ops.get_collection(ops.GraphKeys.GLOBAL_VARIABLES)) self.assertItemsEqual([], ops.get_collection('my_weights')) + @test_util.run_deprecated_v1 def test_get_sparse_tensors_dense_input(self): - hashed_sparse = fc.categorical_column_with_hash_bucket('wire', 10) + hashed_sparse = fc._categorical_column_with_hash_bucket('wire', 10) builder = _LazyBuilder({'wire': (('omar', ''), ('stringer', 'marlo'))}) id_weight_pair = hashed_sparse._get_sparse_tensors(builder) self.assertIsNone(id_weight_pair.weight_tensor) self.assertEqual(builder.get(hashed_sparse), id_weight_pair.id_tensor) + @test_util.run_deprecated_v1 def test_linear_model(self): - wire_column = fc.categorical_column_with_hash_bucket('wire', 4) + wire_column = fc._categorical_column_with_hash_bucket('wire', 4) self.assertEqual(4, wire_column._num_buckets) with ops.Graph().as_default(): predictions = fc.linear_model({ @@ -852,16 +878,18 @@ class HashedCategoricalColumnTest(test.TestCase): bias = get_linear_model_bias() wire_var = get_linear_model_column_var(wire_column) with _initialized_session(): - self.assertAllClose((0.,), bias.eval()) - self.assertAllClose(((0.,), (0.,), (0.,), (0.,)), wire_var.eval()) - self.assertAllClose(((0.,), (0.,)), predictions.eval()) + self.assertAllClose((0.,), self.evaluate(bias)) + self.assertAllClose(((0.,), (0.,), (0.,), (0.,)), + self.evaluate(wire_var)) + self.assertAllClose(((0.,), (0.,)), self.evaluate(predictions)) wire_var.assign(((1.,), (2.,), (3.,), (4.,))).eval() # 'marlo' -> 3: wire_var[3] = 4 # 'skywalker' -> 2, 'omar' -> 2: wire_var[2] + wire_var[2] = 3+3 = 6 - self.assertAllClose(((4.,), (6.,)), predictions.eval()) + self.assertAllClose(((4.,), (6.,)), self.evaluate(predictions)) + @test_util.run_deprecated_v1 def test_keras_linear_model(self): - wire_column = fc.categorical_column_with_hash_bucket('wire', 4) + wire_column = fc._categorical_column_with_hash_bucket('wire', 4) self.assertEqual(4, wire_column._num_buckets) with ops.Graph().as_default(): predictions = get_keras_linear_model_predictions({ @@ -874,13 +902,14 @@ class HashedCategoricalColumnTest(test.TestCase): bias = get_linear_model_bias() wire_var = get_linear_model_column_var(wire_column) with _initialized_session(): - self.assertAllClose((0.,), bias.eval()) - self.assertAllClose(((0.,), (0.,), (0.,), (0.,)), wire_var.eval()) - self.assertAllClose(((0.,), (0.,)), predictions.eval()) + self.assertAllClose((0.,), self.evaluate(bias)) + self.assertAllClose(((0.,), (0.,), (0.,), (0.,)), + self.evaluate(wire_var)) + self.assertAllClose(((0.,), (0.,)), self.evaluate(predictions)) wire_var.assign(((1.,), (2.,), (3.,), (4.,))).eval() # 'marlo' -> 3: wire_var[3] = 4 # 'skywalker' -> 2, 'omar' -> 2: wire_var[2] + wire_var[2] = 3+3 = 6 - self.assertAllClose(((4.,), (6.,)), predictions.eval()) + self.assertAllClose(((4.,), (6.,)), self.evaluate(predictions)) class CrossedColumnTest(test.TestCase): @@ -888,100 +917,102 @@ class CrossedColumnTest(test.TestCase): def test_keys_empty(self): with self.assertRaisesRegexp( ValueError, 'keys must be a list with length > 1'): - fc.crossed_column([], 10) + fc._crossed_column([], 10) def test_keys_length_one(self): with self.assertRaisesRegexp( ValueError, 'keys must be a list with length > 1'): - fc.crossed_column(['a'], 10) + fc._crossed_column(['a'], 10) def test_key_type_unsupported(self): with self.assertRaisesRegexp(ValueError, 'Unsupported key type'): - fc.crossed_column(['a', fc.numeric_column('c')], 10) + fc._crossed_column(['a', fc._numeric_column('c')], 10) with self.assertRaisesRegexp( ValueError, 'categorical_column_with_hash_bucket is not supported'): - fc.crossed_column( - ['a', fc.categorical_column_with_hash_bucket('c', 10)], 10) + fc._crossed_column( + ['a', fc._categorical_column_with_hash_bucket('c', 10)], 10) def test_hash_bucket_size_negative(self): with self.assertRaisesRegexp( ValueError, 'hash_bucket_size must be > 1'): - fc.crossed_column(['a', 'c'], -1) + fc._crossed_column(['a', 'c'], -1) def test_hash_bucket_size_zero(self): with self.assertRaisesRegexp( ValueError, 'hash_bucket_size must be > 1'): - fc.crossed_column(['a', 'c'], 0) + fc._crossed_column(['a', 'c'], 0) def test_hash_bucket_size_none(self): with self.assertRaisesRegexp( ValueError, 'hash_bucket_size must be > 1'): - fc.crossed_column(['a', 'c'], None) + fc._crossed_column(['a', 'c'], None) def test_name(self): - a = fc.numeric_column('a', dtype=dtypes.int32) - b = fc.bucketized_column(a, boundaries=[0, 1]) - crossed1 = fc.crossed_column(['d1', 'd2'], 10) + a = fc._numeric_column('a', dtype=dtypes.int32) + b = fc._bucketized_column(a, boundaries=[0, 1]) + crossed1 = fc._crossed_column(['d1', 'd2'], 10) - crossed2 = fc.crossed_column([b, 'c', crossed1], 10) + crossed2 = fc._crossed_column([b, 'c', crossed1], 10) self.assertEqual('a_bucketized_X_c_X_d1_X_d2', crossed2.name) def test_name_ordered_alphabetically(self): """Tests that the name does not depend on the order of given columns.""" - a = fc.numeric_column('a', dtype=dtypes.int32) - b = fc.bucketized_column(a, boundaries=[0, 1]) - crossed1 = fc.crossed_column(['d1', 'd2'], 10) + a = fc._numeric_column('a', dtype=dtypes.int32) + b = fc._bucketized_column(a, boundaries=[0, 1]) + crossed1 = fc._crossed_column(['d1', 'd2'], 10) - crossed2 = fc.crossed_column([crossed1, 'c', b], 10) + crossed2 = fc._crossed_column([crossed1, 'c', b], 10) self.assertEqual('a_bucketized_X_c_X_d1_X_d2', crossed2.name) def test_name_leaf_keys_ordered_alphabetically(self): """Tests that the name does not depend on the order of given columns.""" - a = fc.numeric_column('a', dtype=dtypes.int32) - b = fc.bucketized_column(a, boundaries=[0, 1]) - crossed1 = fc.crossed_column(['d2', 'c'], 10) + a = fc._numeric_column('a', dtype=dtypes.int32) + b = fc._bucketized_column(a, boundaries=[0, 1]) + crossed1 = fc._crossed_column(['d2', 'c'], 10) - crossed2 = fc.crossed_column([crossed1, 'd1', b], 10) + crossed2 = fc._crossed_column([crossed1, 'd1', b], 10) self.assertEqual('a_bucketized_X_c_X_d1_X_d2', crossed2.name) def test_var_scope_name(self): - a = fc.numeric_column('a', dtype=dtypes.int32) - b = fc.bucketized_column(a, boundaries=[0, 1]) - crossed1 = fc.crossed_column(['d1', 'd2'], 10) + a = fc._numeric_column('a', dtype=dtypes.int32) + b = fc._bucketized_column(a, boundaries=[0, 1]) + crossed1 = fc._crossed_column(['d1', 'd2'], 10) - crossed2 = fc.crossed_column([b, 'c', crossed1], 10) + crossed2 = fc._crossed_column([b, 'c', crossed1], 10) self.assertEqual('a_bucketized_X_c_X_d1_X_d2', crossed2._var_scope_name) def test_parse_spec(self): - a = fc.numeric_column('a', shape=[2], dtype=dtypes.int32) - b = fc.bucketized_column(a, boundaries=[0, 1]) - crossed = fc.crossed_column([b, 'c'], 10) + a = fc._numeric_column('a', shape=[2], dtype=dtypes.int32) + b = fc._bucketized_column(a, boundaries=[0, 1]) + crossed = fc._crossed_column([b, 'c'], 10) self.assertEqual({ 'a': parsing_ops.FixedLenFeature((2,), dtype=dtypes.int32), 'c': parsing_ops.VarLenFeature(dtypes.string), }, crossed._parse_example_spec) def test_num_buckets(self): - a = fc.numeric_column('a', shape=[2], dtype=dtypes.int32) - b = fc.bucketized_column(a, boundaries=[0, 1]) - crossed = fc.crossed_column([b, 'c'], 15) + a = fc._numeric_column('a', shape=[2], dtype=dtypes.int32) + b = fc._bucketized_column(a, boundaries=[0, 1]) + crossed = fc._crossed_column([b, 'c'], 15) self.assertEqual(15, crossed._num_buckets) + @test_util.run_deprecated_v1 def test_deep_copy(self): - a = fc.numeric_column('a', dtype=dtypes.int32) - b = fc.bucketized_column(a, boundaries=[0, 1]) - crossed1 = fc.crossed_column(['d1', 'd2'], 10) - crossed2 = fc.crossed_column([b, 'c', crossed1], 15, hash_key=5) + a = fc._numeric_column('a', dtype=dtypes.int32) + b = fc._bucketized_column(a, boundaries=[0, 1]) + crossed1 = fc._crossed_column(['d1', 'd2'], 10) + crossed2 = fc._crossed_column([b, 'c', crossed1], 15, hash_key=5) crossed2_copy = copy.deepcopy(crossed2) self.assertEqual('a_bucketized_X_c_X_d1_X_d2', crossed2_copy.name,) self.assertEqual(15, crossed2_copy.hash_bucket_size) self.assertEqual(5, crossed2_copy.hash_key) + @test_util.run_deprecated_v1 def test_parse_example(self): - price = fc.numeric_column('price', shape=[2]) - bucketized_price = fc.bucketized_column(price, boundaries=[0, 50]) - price_cross_wire = fc.crossed_column([bucketized_price, 'wire'], 10) + price = fc._numeric_column('price', shape=[2]) + bucketized_price = fc._bucketized_column(price, boundaries=[0, 50]) + price_cross_wire = fc._crossed_column([bucketized_price, 'wire'], 10) data = example_pb2.Example(features=feature_pb2.Features( feature={ 'price': @@ -1004,12 +1035,13 @@ class CrossedColumnTest(test.TestCase): self.assertAllEqual([b'omar', b'stringer'], wire_sparse.values.eval()) self.assertAllEqual([1, 2], wire_sparse.dense_shape.eval()) + @test_util.run_deprecated_v1 def test_transform_feature(self): - price = fc.numeric_column('price', shape=[2]) - bucketized_price = fc.bucketized_column(price, boundaries=[0, 50]) + price = fc._numeric_column('price', shape=[2]) + bucketized_price = fc._bucketized_column(price, boundaries=[0, 50]) hash_bucket_size = 10 - price_cross_wire = fc.crossed_column( - [bucketized_price, 'wire'], hash_bucket_size) + price_cross_wire = fc._crossed_column([bucketized_price, 'wire'], + hash_bucket_size) features = { 'price': constant_op.constant([[1., 2.], [5., 6.]]), 'wire': sparse_tensor.SparseTensor( @@ -1020,18 +1052,19 @@ class CrossedColumnTest(test.TestCase): outputs = _transform_features(features, [price_cross_wire]) output = outputs[price_cross_wire] with self.cached_session() as sess: - output_val = sess.run(output) + output_val = self.evaluate(output) self.assertAllEqual( [[0, 0], [0, 1], [1, 0], [1, 1], [1, 2], [1, 3]], output_val.indices) for val in output_val.values: self.assertIn(val, list(range(hash_bucket_size))) self.assertAllEqual([2, 4], output_val.dense_shape) + @test_util.run_deprecated_v1 def test_get_sparse_tensors(self): - a = fc.numeric_column('a', dtype=dtypes.int32, shape=(2,)) - b = fc.bucketized_column(a, boundaries=(0, 1)) - crossed1 = fc.crossed_column(['d1', 'd2'], 10) - crossed2 = fc.crossed_column([b, 'c', crossed1], 15, hash_key=5) + a = fc._numeric_column('a', dtype=dtypes.int32, shape=(2,)) + b = fc._bucketized_column(a, boundaries=(0, 1)) + crossed1 = fc._crossed_column(['d1', 'd2'], 10) + crossed2 = fc._crossed_column([b, 'c', crossed1], 15, hash_key=5) with ops.Graph().as_default(): builder = _LazyBuilder({ 'a': @@ -1069,9 +1102,9 @@ class CrossedColumnTest(test.TestCase): def test_get_sparse_tensors_simple(self): """Same as test_get_sparse_tensors, but with simpler values.""" - a = fc.numeric_column('a', dtype=dtypes.int32, shape=(2,)) - b = fc.bucketized_column(a, boundaries=(0, 1)) - crossed = fc.crossed_column([b, 'c'], hash_bucket_size=5, hash_key=5) + a = fc._numeric_column('a', dtype=dtypes.int32, shape=(2,)) + b = fc._bucketized_column(a, boundaries=(0, 1)) + crossed = fc._crossed_column([b, 'c'], hash_bucket_size=5, hash_key=5) with ops.Graph().as_default(): builder = _LazyBuilder({ 'a': @@ -1094,14 +1127,15 @@ class CrossedColumnTest(test.TestCase): self.assertAllEqual(expected_values, id_tensor_eval.values) self.assertAllEqual((2, 4), id_tensor_eval.dense_shape) + @test_util.run_deprecated_v1 def test_linear_model(self): """Tests linear_model. Uses data from test_get_sparse_tesnsors_simple. """ - a = fc.numeric_column('a', dtype=dtypes.int32, shape=(2,)) - b = fc.bucketized_column(a, boundaries=(0, 1)) - crossed = fc.crossed_column([b, 'c'], hash_bucket_size=5, hash_key=5) + a = fc._numeric_column('a', dtype=dtypes.int32, shape=(2,)) + b = fc._bucketized_column(a, boundaries=(0, 1)) + crossed = fc._crossed_column([b, 'c'], hash_bucket_size=5, hash_key=5) with ops.Graph().as_default(): predictions = fc.linear_model({ 'a': constant_op.constant(((-1., .5), (.5, 1.))), @@ -1113,15 +1147,15 @@ class CrossedColumnTest(test.TestCase): bias = get_linear_model_bias() crossed_var = get_linear_model_column_var(crossed) with _initialized_session() as sess: - self.assertAllClose((0.,), bias.eval()) - self.assertAllClose( - ((0.,), (0.,), (0.,), (0.,), (0.,)), crossed_var.eval()) - self.assertAllClose(((0.,), (0.,)), predictions.eval()) + self.assertAllClose((0.,), self.evaluate(bias)) + self.assertAllClose(((0.,), (0.,), (0.,), (0.,), (0.,)), + self.evaluate(crossed_var)) + self.assertAllClose(((0.,), (0.,)), self.evaluate(predictions)) sess.run(crossed_var.assign(((1.,), (2.,), (3.,), (4.,), (5.,)))) # Expected ids after cross = (1, 0, 1, 3, 4, 2) - self.assertAllClose(((3.,), (14.,)), predictions.eval()) + self.assertAllClose(((3.,), (14.,)), self.evaluate(predictions)) sess.run(bias.assign((.1,))) - self.assertAllClose(((3.1,), (14.1,)), predictions.eval()) + self.assertAllClose(((3.1,), (14.1,)), self.evaluate(predictions)) def test_linear_model_with_weights(self): class _TestColumnWithWeights(_CategoricalColumn): @@ -1155,7 +1189,7 @@ class CrossedColumnTest(test.TestCase): id_tensor=ids_and_weights[0], weight_tensor=ids_and_weights[1]) t = _TestColumnWithWeights() - crossed = fc.crossed_column([t, 'c'], hash_bucket_size=5, hash_key=5) + crossed = fc._crossed_column([t, 'c'], hash_bucket_size=5, hash_key=5) with ops.Graph().as_default(): with self.assertRaisesRegexp( ValueError, @@ -1175,14 +1209,15 @@ class CrossedColumnTest(test.TestCase): dense_shape=(2, 2)), }, (crossed,)) + @test_util.run_deprecated_v1 def test_keras_linear_model(self): """Tests _LinearModel. Uses data from test_get_sparse_tesnsors_simple. """ - a = fc.numeric_column('a', dtype=dtypes.int32, shape=(2,)) - b = fc.bucketized_column(a, boundaries=(0, 1)) - crossed = fc.crossed_column([b, 'c'], hash_bucket_size=5, hash_key=5) + a = fc._numeric_column('a', dtype=dtypes.int32, shape=(2,)) + b = fc._bucketized_column(a, boundaries=(0, 1)) + crossed = fc._crossed_column([b, 'c'], hash_bucket_size=5, hash_key=5) with ops.Graph().as_default(): predictions = get_keras_linear_model_predictions({ 'a': @@ -1196,15 +1231,15 @@ class CrossedColumnTest(test.TestCase): bias = get_linear_model_bias() crossed_var = get_linear_model_column_var(crossed) with _initialized_session() as sess: - self.assertAllClose((0.,), bias.eval()) + self.assertAllClose((0.,), self.evaluate(bias)) self.assertAllClose(((0.,), (0.,), (0.,), (0.,), (0.,)), - crossed_var.eval()) - self.assertAllClose(((0.,), (0.,)), predictions.eval()) + self.evaluate(crossed_var)) + self.assertAllClose(((0.,), (0.,)), self.evaluate(predictions)) sess.run(crossed_var.assign(((1.,), (2.,), (3.,), (4.,), (5.,)))) # Expected ids after cross = (1, 0, 1, 3, 4, 2) - self.assertAllClose(((3.,), (14.,)), predictions.eval()) + self.assertAllClose(((3.,), (14.,)), self.evaluate(predictions)) sess.run(bias.assign((.1,))) - self.assertAllClose(((3.1,), (14.1,)), predictions.eval()) + self.assertAllClose(((3.1,), (14.1,)), self.evaluate(predictions)) def test_keras_linear_model_with_weights(self): @@ -1242,7 +1277,7 @@ class CrossedColumnTest(test.TestCase): id_tensor=ids_and_weights[0], weight_tensor=ids_and_weights[1]) t = _TestColumnWithWeights() - crossed = fc.crossed_column([t, 'c'], hash_bucket_size=5, hash_key=5) + crossed = fc._crossed_column([t, 'c'], hash_bucket_size=5, hash_key=5) with ops.Graph().as_default(): with self.assertRaisesRegexp( ValueError, @@ -1331,31 +1366,31 @@ class LinearModelTest(test.TestCase): with self.assertRaisesRegexp( ValueError, 'Expected feature_columns to be iterable, found dict.'): fc.linear_model( - features={'a': [[0]]}, feature_columns={'a': fc.numeric_column('a')}) + features={'a': [[0]]}, feature_columns={'a': fc._numeric_column('a')}) def test_raises_if_duplicate_name(self): with self.assertRaisesRegexp( ValueError, 'Duplicate feature column name found for columns'): fc.linear_model( features={'a': [[0]]}, - feature_columns=[fc.numeric_column('a'), - fc.numeric_column('a')]) + feature_columns=[fc._numeric_column('a'), + fc._numeric_column('a')]) def test_dense_bias(self): - price = fc.numeric_column('price') + price = fc._numeric_column('price') with ops.Graph().as_default(): features = {'price': [[1.], [5.]]} predictions = fc.linear_model(features, [price]) bias = get_linear_model_bias() price_var = get_linear_model_column_var(price) with _initialized_session() as sess: - self.assertAllClose([0.], bias.eval()) + self.assertAllClose([0.], self.evaluate(bias)) sess.run(price_var.assign([[10.]])) sess.run(bias.assign([5.])) - self.assertAllClose([[15.], [55.]], predictions.eval()) + self.assertAllClose([[15.], [55.]], self.evaluate(predictions)) def test_sparse_bias(self): - wire_cast = fc.categorical_column_with_hash_bucket('wire_cast', 4) + wire_cast = fc._categorical_column_with_hash_bucket('wire_cast', 4) with ops.Graph().as_default(): wire_tensor = sparse_tensor.SparseTensor( values=['omar', 'stringer', 'marlo'], # hashed to = [2, 0, 3] @@ -1366,15 +1401,16 @@ class LinearModelTest(test.TestCase): bias = get_linear_model_bias() wire_cast_var = get_linear_model_column_var(wire_cast) with _initialized_session() as sess: - self.assertAllClose([0.], bias.eval()) - self.assertAllClose([[0.], [0.], [0.], [0.]], wire_cast_var.eval()) + self.assertAllClose([0.], self.evaluate(bias)) + self.assertAllClose([[0.], [0.], [0.], [0.]], + self.evaluate(wire_cast_var)) sess.run(wire_cast_var.assign([[10.], [100.], [1000.], [10000.]])) sess.run(bias.assign([5.])) - self.assertAllClose([[1005.], [10015.]], predictions.eval()) + self.assertAllClose([[1005.], [10015.]], self.evaluate(predictions)) def test_dense_and_sparse_bias(self): - wire_cast = fc.categorical_column_with_hash_bucket('wire_cast', 4) - price = fc.numeric_column('price') + wire_cast = fc._categorical_column_with_hash_bucket('wire_cast', 4) + price = fc._numeric_column('price') with ops.Graph().as_default(): wire_tensor = sparse_tensor.SparseTensor( values=['omar', 'stringer', 'marlo'], # hashed to = [2, 0, 3] @@ -1389,7 +1425,7 @@ class LinearModelTest(test.TestCase): sess.run(wire_cast_var.assign([[10.], [100.], [1000.], [10000.]])) sess.run(bias.assign([5.])) sess.run(price_var.assign([[10.]])) - self.assertAllClose([[1015.], [10065.]], predictions.eval()) + self.assertAllClose([[1015.], [10065.]], self.evaluate(predictions)) def test_dense_and_sparse_column(self): """When the column is both dense and sparse, uses sparse tensors.""" @@ -1442,25 +1478,25 @@ class LinearModelTest(test.TestCase): sess.run(dense_and_sparse_column_var.assign( [[10.], [100.], [1000.], [10000.]])) sess.run(bias.assign([5.])) - self.assertAllClose([[1005.], [10015.]], predictions.eval()) + self.assertAllClose([[1005.], [10015.]], self.evaluate(predictions)) def test_dense_multi_output(self): - price = fc.numeric_column('price') + price = fc._numeric_column('price') with ops.Graph().as_default(): features = {'price': [[1.], [5.]]} predictions = fc.linear_model(features, [price], units=3) bias = get_linear_model_bias() price_var = get_linear_model_column_var(price) with _initialized_session() as sess: - self.assertAllClose(np.zeros((3,)), bias.eval()) - self.assertAllClose(np.zeros((1, 3)), price_var.eval()) + self.assertAllClose(np.zeros((3,)), self.evaluate(bias)) + self.assertAllClose(np.zeros((1, 3)), self.evaluate(price_var)) sess.run(price_var.assign([[10., 100., 1000.]])) sess.run(bias.assign([5., 6., 7.])) self.assertAllClose([[15., 106., 1007.], [55., 506., 5007.]], - predictions.eval()) + self.evaluate(predictions)) def test_sparse_multi_output(self): - wire_cast = fc.categorical_column_with_hash_bucket('wire_cast', 4) + wire_cast = fc._categorical_column_with_hash_bucket('wire_cast', 4) with ops.Graph().as_default(): wire_tensor = sparse_tensor.SparseTensor( values=['omar', 'stringer', 'marlo'], # hashed to = [2, 0, 3] @@ -1471,29 +1507,29 @@ class LinearModelTest(test.TestCase): bias = get_linear_model_bias() wire_cast_var = get_linear_model_column_var(wire_cast) with _initialized_session() as sess: - self.assertAllClose(np.zeros((3,)), bias.eval()) - self.assertAllClose(np.zeros((4, 3)), wire_cast_var.eval()) + self.assertAllClose(np.zeros((3,)), self.evaluate(bias)) + self.assertAllClose(np.zeros((4, 3)), self.evaluate(wire_cast_var)) sess.run( wire_cast_var.assign([[10., 11., 12.], [100., 110., 120.], [ 1000., 1100., 1200. ], [10000., 11000., 12000.]])) sess.run(bias.assign([5., 6., 7.])) self.assertAllClose([[1005., 1106., 1207.], [10015., 11017., 12019.]], - predictions.eval()) + self.evaluate(predictions)) def test_dense_multi_dimension(self): - price = fc.numeric_column('price', shape=2) + price = fc._numeric_column('price', shape=2) with ops.Graph().as_default(): features = {'price': [[1., 2.], [5., 6.]]} predictions = fc.linear_model(features, [price]) price_var = get_linear_model_column_var(price) with _initialized_session() as sess: - self.assertAllClose([[0.], [0.]], price_var.eval()) + self.assertAllClose([[0.], [0.]], self.evaluate(price_var)) sess.run(price_var.assign([[10.], [100.]])) - self.assertAllClose([[210.], [650.]], predictions.eval()) + self.assertAllClose([[210.], [650.]], self.evaluate(predictions)) def test_sparse_multi_rank(self): - wire_cast = fc.categorical_column_with_hash_bucket('wire_cast', 4) + wire_cast = fc._categorical_column_with_hash_bucket('wire_cast', 4) with ops.Graph().as_default(): wire_tensor = array_ops.sparse_placeholder(dtypes.string) wire_value = sparse_tensor.SparseTensorValue( @@ -1504,7 +1540,7 @@ class LinearModelTest(test.TestCase): predictions = fc.linear_model(features, [wire_cast]) wire_cast_var = get_linear_model_column_var(wire_cast) with _initialized_session() as sess: - self.assertAllClose(np.zeros((4, 1)), wire_cast_var.eval()) + self.assertAllClose(np.zeros((4, 1)), self.evaluate(wire_cast_var)) self.assertAllClose( np.zeros((2, 1)), predictions.eval(feed_dict={wire_tensor: wire_value})) @@ -1514,7 +1550,7 @@ class LinearModelTest(test.TestCase): predictions.eval(feed_dict={wire_tensor: wire_value})) def test_sparse_combiner(self): - wire_cast = fc.categorical_column_with_hash_bucket('wire_cast', 4) + wire_cast = fc._categorical_column_with_hash_bucket('wire_cast', 4) with ops.Graph().as_default(): wire_tensor = sparse_tensor.SparseTensor( values=['omar', 'stringer', 'marlo'], # hashed to = [2, 0, 3] @@ -1528,11 +1564,11 @@ class LinearModelTest(test.TestCase): with _initialized_session() as sess: sess.run(wire_cast_var.assign([[10.], [100.], [1000.], [10000.]])) sess.run(bias.assign([5.])) - self.assertAllClose([[1005.], [5010.]], predictions.eval()) + self.assertAllClose([[1005.], [5010.]], self.evaluate(predictions)) def test_sparse_combiner_with_negative_weights(self): - wire_cast = fc.categorical_column_with_hash_bucket('wire_cast', 4) - wire_cast_weights = fc.weighted_categorical_column(wire_cast, 'weights') + wire_cast = fc._categorical_column_with_hash_bucket('wire_cast', 4) + wire_cast_weights = fc._weighted_categorical_column(wire_cast, 'weights') with ops.Graph().as_default(): wire_tensor = sparse_tensor.SparseTensor( @@ -1550,25 +1586,25 @@ class LinearModelTest(test.TestCase): with _initialized_session() as sess: sess.run(wire_cast_var.assign([[10.], [100.], [1000.], [10000.]])) sess.run(bias.assign([5.])) - self.assertAllClose([[1005.], [-9985.]], predictions.eval()) + self.assertAllClose([[1005.], [-9985.]], self.evaluate(predictions)) def test_dense_multi_dimension_multi_output(self): - price = fc.numeric_column('price', shape=2) + price = fc._numeric_column('price', shape=2) with ops.Graph().as_default(): features = {'price': [[1., 2.], [5., 6.]]} predictions = fc.linear_model(features, [price], units=3) bias = get_linear_model_bias() price_var = get_linear_model_column_var(price) with _initialized_session() as sess: - self.assertAllClose(np.zeros((3,)), bias.eval()) - self.assertAllClose(np.zeros((2, 3)), price_var.eval()) + self.assertAllClose(np.zeros((3,)), self.evaluate(bias)) + self.assertAllClose(np.zeros((2, 3)), self.evaluate(price_var)) sess.run(price_var.assign([[1., 2., 3.], [10., 100., 1000.]])) sess.run(bias.assign([2., 3., 4.])) self.assertAllClose([[23., 205., 2007.], [67., 613., 6019.]], - predictions.eval()) + self.evaluate(predictions)) def test_raises_if_shape_mismatch(self): - price = fc.numeric_column('price', shape=2) + price = fc._numeric_column('price', shape=2) with ops.Graph().as_default(): features = {'price': [[1.], [5.]]} with self.assertRaisesRegexp( @@ -1577,22 +1613,22 @@ class LinearModelTest(test.TestCase): fc.linear_model(features, [price]) def test_dense_reshaping(self): - price = fc.numeric_column('price', shape=[1, 2]) + price = fc._numeric_column('price', shape=[1, 2]) with ops.Graph().as_default(): features = {'price': [[[1., 2.]], [[5., 6.]]]} predictions = fc.linear_model(features, [price]) bias = get_linear_model_bias() price_var = get_linear_model_column_var(price) with _initialized_session() as sess: - self.assertAllClose([0.], bias.eval()) - self.assertAllClose([[0.], [0.]], price_var.eval()) - self.assertAllClose([[0.], [0.]], predictions.eval()) + self.assertAllClose([0.], self.evaluate(bias)) + self.assertAllClose([[0.], [0.]], self.evaluate(price_var)) + self.assertAllClose([[0.], [0.]], self.evaluate(predictions)) sess.run(price_var.assign([[10.], [100.]])) - self.assertAllClose([[210.], [650.]], predictions.eval()) + self.assertAllClose([[210.], [650.]], self.evaluate(predictions)) def test_dense_multi_column(self): - price1 = fc.numeric_column('price1', shape=2) - price2 = fc.numeric_column('price2') + price1 = fc._numeric_column('price1', shape=2) + price2 = fc._numeric_column('price2') with ops.Graph().as_default(): features = { 'price1': [[1., 2.], [5., 6.]], @@ -1603,18 +1639,18 @@ class LinearModelTest(test.TestCase): price1_var = get_linear_model_column_var(price1) price2_var = get_linear_model_column_var(price2) with _initialized_session() as sess: - self.assertAllClose([0.], bias.eval()) - self.assertAllClose([[0.], [0.]], price1_var.eval()) - self.assertAllClose([[0.]], price2_var.eval()) - self.assertAllClose([[0.], [0.]], predictions.eval()) + self.assertAllClose([0.], self.evaluate(bias)) + self.assertAllClose([[0.], [0.]], self.evaluate(price1_var)) + self.assertAllClose([[0.]], self.evaluate(price2_var)) + self.assertAllClose([[0.], [0.]], self.evaluate(predictions)) sess.run(price1_var.assign([[10.], [100.]])) sess.run(price2_var.assign([[1000.]])) sess.run(bias.assign([7.])) - self.assertAllClose([[3217.], [4657.]], predictions.eval()) + self.assertAllClose([[3217.], [4657.]], self.evaluate(predictions)) def test_fills_cols_to_vars(self): - price1 = fc.numeric_column('price1', shape=2) - price2 = fc.numeric_column('price2') + price1 = fc._numeric_column('price1', shape=2) + price2 = fc._numeric_column('price2') with ops.Graph().as_default(): features = {'price1': [[1., 2.], [5., 6.]], 'price2': [[3.], [4.]]} cols_to_vars = {} @@ -1627,8 +1663,8 @@ class LinearModelTest(test.TestCase): self.assertAllEqual(cols_to_vars[price2], [price2_var]) def test_fills_cols_to_vars_partitioned_variables(self): - price1 = fc.numeric_column('price1', shape=2) - price2 = fc.numeric_column('price2', shape=3) + price1 = fc._numeric_column('price1', shape=2) + price2 = fc._numeric_column('price2', shape=3) with ops.Graph().as_default(): features = { 'price1': [[1., 2.], [6., 7.]], @@ -1653,13 +1689,13 @@ class LinearModelTest(test.TestCase): # Provide three _DenseColumn's to input_layer: a _NumericColumn, a # _BucketizedColumn, and an _EmbeddingColumn. Only the _EmbeddingColumn # creates a Variable. - apple_numeric_column = fc.numeric_column('apple_numeric_column') - banana_dense_feature = fc.numeric_column('banana_dense_feature') - banana_dense_feature_bucketized = fc.bucketized_column( + apple_numeric_column = fc._numeric_column('apple_numeric_column') + banana_dense_feature = fc._numeric_column('banana_dense_feature') + banana_dense_feature_bucketized = fc._bucketized_column( banana_dense_feature, boundaries=[0.]) - cherry_sparse_column = fc.categorical_column_with_hash_bucket( + cherry_sparse_column = fc._categorical_column_with_hash_bucket( 'cherry_sparse_feature', hash_bucket_size=5) - dragonfruit_embedding_column = fc.embedding_column( + dragonfruit_embedding_column = fc._embedding_column( cherry_sparse_column, dimension=10) with ops.Graph().as_default(): features = { @@ -1684,7 +1720,7 @@ class LinearModelTest(test.TestCase): self.assertItemsEqual(input_layer_inputs, output_tensors) def test_dense_collection(self): - price = fc.numeric_column('price') + price = fc._numeric_column('price') with ops.Graph().as_default() as g: features = {'price': [[1.], [5.]]} fc.linear_model(features, [price], weight_collections=['my-vars']) @@ -1695,7 +1731,7 @@ class LinearModelTest(test.TestCase): self.assertIn(price_var, my_vars) def test_sparse_collection(self): - wire_cast = fc.categorical_column_with_hash_bucket('wire_cast', 4) + wire_cast = fc._categorical_column_with_hash_bucket('wire_cast', 4) with ops.Graph().as_default() as g: wire_tensor = sparse_tensor.SparseTensor( values=['omar'], indices=[[0, 0]], dense_shape=[1, 1]) @@ -1709,7 +1745,7 @@ class LinearModelTest(test.TestCase): self.assertIn(wire_cast_var, my_vars) def test_dense_trainable_default(self): - price = fc.numeric_column('price') + price = fc._numeric_column('price') with ops.Graph().as_default() as g: features = {'price': [[1.], [5.]]} fc.linear_model(features, [price]) @@ -1720,7 +1756,7 @@ class LinearModelTest(test.TestCase): self.assertIn(price_var, trainable_vars) def test_sparse_trainable_default(self): - wire_cast = fc.categorical_column_with_hash_bucket('wire_cast', 4) + wire_cast = fc._categorical_column_with_hash_bucket('wire_cast', 4) with ops.Graph().as_default() as g: wire_tensor = sparse_tensor.SparseTensor( values=['omar'], indices=[[0, 0]], dense_shape=[1, 1]) @@ -1733,7 +1769,7 @@ class LinearModelTest(test.TestCase): self.assertIn(wire_cast_var, trainable_vars) def test_dense_trainable_false(self): - price = fc.numeric_column('price') + price = fc._numeric_column('price') with ops.Graph().as_default() as g: features = {'price': [[1.], [5.]]} fc.linear_model(features, [price], trainable=False) @@ -1741,7 +1777,7 @@ class LinearModelTest(test.TestCase): self.assertEqual([], trainable_vars) def test_sparse_trainable_false(self): - wire_cast = fc.categorical_column_with_hash_bucket('wire_cast', 4) + wire_cast = fc._categorical_column_with_hash_bucket('wire_cast', 4) with ops.Graph().as_default() as g: wire_tensor = sparse_tensor.SparseTensor( values=['omar'], indices=[[0, 0]], dense_shape=[1, 1]) @@ -1751,9 +1787,9 @@ class LinearModelTest(test.TestCase): self.assertEqual([], trainable_vars) def test_column_order(self): - price_a = fc.numeric_column('price_a') - price_b = fc.numeric_column('price_b') - wire_cast = fc.categorical_column_with_hash_bucket('wire_cast', 4) + price_a = fc._numeric_column('price_a') + price_b = fc._numeric_column('price_b') + wire_cast = fc._categorical_column_with_hash_bucket('wire_cast', 4) with ops.Graph().as_default() as g: features = { 'price_a': [[1.]], @@ -1787,8 +1823,8 @@ class LinearModelTest(test.TestCase): self.assertIn('wire_cast', my_vars[2].name) def test_static_batch_size_mismatch(self): - price1 = fc.numeric_column('price1') - price2 = fc.numeric_column('price2') + price1 = fc._numeric_column('price1') + price2 = fc._numeric_column('price2') with ops.Graph().as_default(): features = { 'price1': [[1.], [5.], [7.]], # batchsize = 3 @@ -1800,9 +1836,9 @@ class LinearModelTest(test.TestCase): fc.linear_model(features, [price1, price2]) def test_subset_of_static_batch_size_mismatch(self): - price1 = fc.numeric_column('price1') - price2 = fc.numeric_column('price2') - price3 = fc.numeric_column('price3') + price1 = fc._numeric_column('price1') + price2 = fc._numeric_column('price2') + price3 = fc._numeric_column('price3') with ops.Graph().as_default(): features = { 'price1': array_ops.placeholder(dtype=dtypes.int64), # batchsize = 3 @@ -1815,8 +1851,8 @@ class LinearModelTest(test.TestCase): fc.linear_model(features, [price1, price2, price3]) def test_runtime_batch_size_mismatch(self): - price1 = fc.numeric_column('price1') - price2 = fc.numeric_column('price2') + price1 = fc._numeric_column('price1') + price2 = fc._numeric_column('price2') with ops.Graph().as_default(): features = { 'price1': array_ops.placeholder(dtype=dtypes.int64), # batchsize = 3 @@ -1830,8 +1866,8 @@ class LinearModelTest(test.TestCase): predictions, feed_dict={features['price1']: [[1.], [5.], [7.]]}) def test_runtime_batch_size_matches(self): - price1 = fc.numeric_column('price1') - price2 = fc.numeric_column('price2') + price1 = fc._numeric_column('price1') + price2 = fc._numeric_column('price2') with ops.Graph().as_default(): features = { 'price1': array_ops.placeholder(dtype=dtypes.int64), # batchsize = 2 @@ -1846,10 +1882,16 @@ class LinearModelTest(test.TestCase): features['price2']: [[1.], [5.]], }) + @test_util.run_deprecated_v1 def test_with_1d_sparse_tensor(self): - price = fc.numeric_column('price') - price_buckets = fc.bucketized_column(price, boundaries=[0., 10., 100.,]) - body_style = fc.categorical_column_with_vocabulary_list( + price = fc._numeric_column('price') + price_buckets = fc._bucketized_column( + price, boundaries=[ + 0., + 10., + 100., + ]) + body_style = fc._categorical_column_with_vocabulary_list( 'body-style', vocabulary_list=['hardtop', 'wagon', 'sedan']) # Provides 1-dim tensor and dense tensor. @@ -1873,14 +1915,21 @@ class LinearModelTest(test.TestCase): sess.run(body_style_var.assign([[-10.], [-100.], [-1000.]])) sess.run(bias.assign([5.])) - self.assertAllClose([[10 - 1000 + 5.], [1000 - 10 + 5.]], sess.run(net)) + self.assertAllClose([[10 - 1000 + 5.], [1000 - 10 + 5.]], + self.evaluate(net)) + @test_util.run_deprecated_v1 def test_with_1d_unknown_shape_sparse_tensor(self): - price = fc.numeric_column('price') - price_buckets = fc.bucketized_column(price, boundaries=[0., 10., 100.,]) - body_style = fc.categorical_column_with_vocabulary_list( + price = fc._numeric_column('price') + price_buckets = fc._bucketized_column( + price, boundaries=[ + 0., + 10., + 100., + ]) + body_style = fc._categorical_column_with_vocabulary_list( 'body-style', vocabulary_list=['hardtop', 'wagon', 'sedan']) - country = fc.categorical_column_with_vocabulary_list( + country = fc._categorical_column_with_vocabulary_list( 'country', vocabulary_list=['US', 'JP', 'CA']) # Provides 1-dim tensor and dense tensor. @@ -1917,8 +1966,9 @@ class LinearModelTest(test.TestCase): features['country']: country_data })) + @test_util.run_deprecated_v1 def test_with_rank_0_feature(self): - price = fc.numeric_column('price') + price = fc._numeric_column('price') features = { 'price': constant_op.constant(0), } @@ -1939,7 +1989,7 @@ class LinearModelTest(test.TestCase): sess.run(net, feed_dict={features['price']: np.array(1)}) def test_multiple_linear_models(self): - price = fc.numeric_column('price') + price = fc._numeric_column('price') with ops.Graph().as_default(): features1 = {'price': [[1.], [5.]]} features2 = {'price': [[2.], [10.]]} @@ -1950,14 +2000,14 @@ class LinearModelTest(test.TestCase): price_var1 = get_linear_model_column_var(price, name='linear_model') price_var2 = get_linear_model_column_var(price, name='linear_model_1') with _initialized_session() as sess: - self.assertAllClose([0.], bias1.eval()) + self.assertAllClose([0.], self.evaluate(bias1)) sess.run(price_var1.assign([[10.]])) sess.run(bias1.assign([5.])) - self.assertAllClose([[15.], [55.]], predictions1.eval()) - self.assertAllClose([0.], bias2.eval()) + self.assertAllClose([[15.], [55.]], self.evaluate(predictions1)) + self.assertAllClose([0.], self.evaluate(bias2)) sess.run(price_var2.assign([[10.]])) sess.run(bias2.assign([5.])) - self.assertAllClose([[25.], [105.]], predictions2.eval()) + self.assertAllClose([[25.], [105.]], self.evaluate(predictions2)) class _LinearModelTest(test.TestCase): @@ -1996,31 +2046,31 @@ class _LinearModelTest(test.TestCase): with self.assertRaisesRegexp( ValueError, 'Expected feature_columns to be iterable, found dict.'): fc.linear_model( - features={'a': [[0]]}, feature_columns={'a': fc.numeric_column('a')}) + features={'a': [[0]]}, feature_columns={'a': fc._numeric_column('a')}) def test_raises_if_duplicate_name(self): with self.assertRaisesRegexp( ValueError, 'Duplicate feature column name found for columns'): get_keras_linear_model_predictions( features={'a': [[0]]}, - feature_columns=[fc.numeric_column('a'), - fc.numeric_column('a')]) + feature_columns=[fc._numeric_column('a'), + fc._numeric_column('a')]) def test_dense_bias(self): - price = fc.numeric_column('price') + price = fc._numeric_column('price') with ops.Graph().as_default(): features = {'price': [[1.], [5.]]} predictions = get_keras_linear_model_predictions(features, [price]) bias = get_linear_model_bias() price_var = get_linear_model_column_var(price) with _initialized_session() as sess: - self.assertAllClose([0.], bias.eval()) + self.assertAllClose([0.], self.evaluate(bias)) sess.run(price_var.assign([[10.]])) sess.run(bias.assign([5.])) - self.assertAllClose([[15.], [55.]], predictions.eval()) + self.assertAllClose([[15.], [55.]], self.evaluate(predictions)) def test_sparse_bias(self): - wire_cast = fc.categorical_column_with_hash_bucket('wire_cast', 4) + wire_cast = fc._categorical_column_with_hash_bucket('wire_cast', 4) with ops.Graph().as_default(): wire_tensor = sparse_tensor.SparseTensor( values=['omar', 'stringer', 'marlo'], # hashed to = [2, 0, 3] @@ -2031,15 +2081,16 @@ class _LinearModelTest(test.TestCase): bias = get_linear_model_bias() wire_cast_var = get_linear_model_column_var(wire_cast) with _initialized_session() as sess: - self.assertAllClose([0.], bias.eval()) - self.assertAllClose([[0.], [0.], [0.], [0.]], wire_cast_var.eval()) + self.assertAllClose([0.], self.evaluate(bias)) + self.assertAllClose([[0.], [0.], [0.], [0.]], + self.evaluate(wire_cast_var)) sess.run(wire_cast_var.assign([[10.], [100.], [1000.], [10000.]])) sess.run(bias.assign([5.])) - self.assertAllClose([[1005.], [10015.]], predictions.eval()) + self.assertAllClose([[1005.], [10015.]], self.evaluate(predictions)) def test_dense_and_sparse_bias(self): - wire_cast = fc.categorical_column_with_hash_bucket('wire_cast', 4) - price = fc.numeric_column('price') + wire_cast = fc._categorical_column_with_hash_bucket('wire_cast', 4) + price = fc._numeric_column('price') with ops.Graph().as_default(): wire_tensor = sparse_tensor.SparseTensor( values=['omar', 'stringer', 'marlo'], # hashed to = [2, 0, 3] @@ -2055,7 +2106,7 @@ class _LinearModelTest(test.TestCase): sess.run(wire_cast_var.assign([[10.], [100.], [1000.], [10000.]])) sess.run(bias.assign([5.])) sess.run(price_var.assign([[10.]])) - self.assertAllClose([[1015.], [10065.]], predictions.eval()) + self.assertAllClose([[1015.], [10065.]], self.evaluate(predictions)) def test_dense_and_sparse_column(self): """When the column is both dense and sparse, uses sparse tensors.""" @@ -2114,10 +2165,10 @@ class _LinearModelTest(test.TestCase): dense_and_sparse_column_var.assign([[10.], [100.], [1000.], [10000.]])) sess.run(bias.assign([5.])) - self.assertAllClose([[1005.], [10015.]], predictions.eval()) + self.assertAllClose([[1005.], [10015.]], self.evaluate(predictions)) def test_dense_multi_output(self): - price = fc.numeric_column('price') + price = fc._numeric_column('price') with ops.Graph().as_default(): features = {'price': [[1.], [5.]]} predictions = get_keras_linear_model_predictions( @@ -2125,15 +2176,15 @@ class _LinearModelTest(test.TestCase): bias = get_linear_model_bias() price_var = get_linear_model_column_var(price) with _initialized_session() as sess: - self.assertAllClose(np.zeros((3,)), bias.eval()) - self.assertAllClose(np.zeros((1, 3)), price_var.eval()) + self.assertAllClose(np.zeros((3,)), self.evaluate(bias)) + self.assertAllClose(np.zeros((1, 3)), self.evaluate(price_var)) sess.run(price_var.assign([[10., 100., 1000.]])) sess.run(bias.assign([5., 6., 7.])) self.assertAllClose([[15., 106., 1007.], [55., 506., 5007.]], - predictions.eval()) + self.evaluate(predictions)) def test_sparse_multi_output(self): - wire_cast = fc.categorical_column_with_hash_bucket('wire_cast', 4) + wire_cast = fc._categorical_column_with_hash_bucket('wire_cast', 4) with ops.Graph().as_default(): wire_tensor = sparse_tensor.SparseTensor( values=['omar', 'stringer', 'marlo'], # hashed to = [2, 0, 3] @@ -2145,29 +2196,29 @@ class _LinearModelTest(test.TestCase): bias = get_linear_model_bias() wire_cast_var = get_linear_model_column_var(wire_cast) with _initialized_session() as sess: - self.assertAllClose(np.zeros((3,)), bias.eval()) - self.assertAllClose(np.zeros((4, 3)), wire_cast_var.eval()) + self.assertAllClose(np.zeros((3,)), self.evaluate(bias)) + self.assertAllClose(np.zeros((4, 3)), self.evaluate(wire_cast_var)) sess.run( wire_cast_var.assign([[10., 11., 12.], [100., 110., 120.], [1000., 1100., 1200.], [10000., 11000., 12000.]])) sess.run(bias.assign([5., 6., 7.])) self.assertAllClose([[1005., 1106., 1207.], [10015., 11017., 12019.]], - predictions.eval()) + self.evaluate(predictions)) def test_dense_multi_dimension(self): - price = fc.numeric_column('price', shape=2) + price = fc._numeric_column('price', shape=2) with ops.Graph().as_default(): features = {'price': [[1., 2.], [5., 6.]]} predictions = get_keras_linear_model_predictions(features, [price]) price_var = get_linear_model_column_var(price) with _initialized_session() as sess: - self.assertAllClose([[0.], [0.]], price_var.eval()) + self.assertAllClose([[0.], [0.]], self.evaluate(price_var)) sess.run(price_var.assign([[10.], [100.]])) - self.assertAllClose([[210.], [650.]], predictions.eval()) + self.assertAllClose([[210.], [650.]], self.evaluate(predictions)) def test_sparse_multi_rank(self): - wire_cast = fc.categorical_column_with_hash_bucket('wire_cast', 4) + wire_cast = fc._categorical_column_with_hash_bucket('wire_cast', 4) with ops.Graph().as_default(): wire_tensor = array_ops.sparse_placeholder(dtypes.string) wire_value = sparse_tensor.SparseTensorValue( @@ -2178,7 +2229,7 @@ class _LinearModelTest(test.TestCase): predictions = get_keras_linear_model_predictions(features, [wire_cast]) wire_cast_var = get_linear_model_column_var(wire_cast) with _initialized_session() as sess: - self.assertAllClose(np.zeros((4, 1)), wire_cast_var.eval()) + self.assertAllClose(np.zeros((4, 1)), self.evaluate(wire_cast_var)) self.assertAllClose( np.zeros((2, 1)), predictions.eval(feed_dict={wire_tensor: wire_value})) @@ -2188,7 +2239,7 @@ class _LinearModelTest(test.TestCase): predictions.eval(feed_dict={wire_tensor: wire_value})) def test_sparse_combiner(self): - wire_cast = fc.categorical_column_with_hash_bucket('wire_cast', 4) + wire_cast = fc._categorical_column_with_hash_bucket('wire_cast', 4) with ops.Graph().as_default(): wire_tensor = sparse_tensor.SparseTensor( values=['omar', 'stringer', 'marlo'], # hashed to = [2, 0, 3] @@ -2202,10 +2253,10 @@ class _LinearModelTest(test.TestCase): with _initialized_session() as sess: sess.run(wire_cast_var.assign([[10.], [100.], [1000.], [10000.]])) sess.run(bias.assign([5.])) - self.assertAllClose([[1005.], [5010.]], predictions.eval()) + self.assertAllClose([[1005.], [5010.]], self.evaluate(predictions)) def test_dense_multi_dimension_multi_output(self): - price = fc.numeric_column('price', shape=2) + price = fc._numeric_column('price', shape=2) with ops.Graph().as_default(): features = {'price': [[1., 2.], [5., 6.]]} predictions = get_keras_linear_model_predictions( @@ -2213,15 +2264,15 @@ class _LinearModelTest(test.TestCase): bias = get_linear_model_bias() price_var = get_linear_model_column_var(price) with _initialized_session() as sess: - self.assertAllClose(np.zeros((3,)), bias.eval()) - self.assertAllClose(np.zeros((2, 3)), price_var.eval()) + self.assertAllClose(np.zeros((3,)), self.evaluate(bias)) + self.assertAllClose(np.zeros((2, 3)), self.evaluate(price_var)) sess.run(price_var.assign([[1., 2., 3.], [10., 100., 1000.]])) sess.run(bias.assign([2., 3., 4.])) self.assertAllClose([[23., 205., 2007.], [67., 613., 6019.]], - predictions.eval()) + self.evaluate(predictions)) def test_raises_if_shape_mismatch(self): - price = fc.numeric_column('price', shape=2) + price = fc._numeric_column('price', shape=2) with ops.Graph().as_default(): features = {'price': [[1.], [5.]]} with self.assertRaisesRegexp( @@ -2230,22 +2281,22 @@ class _LinearModelTest(test.TestCase): get_keras_linear_model_predictions(features, [price]) def test_dense_reshaping(self): - price = fc.numeric_column('price', shape=[1, 2]) + price = fc._numeric_column('price', shape=[1, 2]) with ops.Graph().as_default(): features = {'price': [[[1., 2.]], [[5., 6.]]]} predictions = get_keras_linear_model_predictions(features, [price]) bias = get_linear_model_bias() price_var = get_linear_model_column_var(price) with _initialized_session() as sess: - self.assertAllClose([0.], bias.eval()) - self.assertAllClose([[0.], [0.]], price_var.eval()) - self.assertAllClose([[0.], [0.]], predictions.eval()) + self.assertAllClose([0.], self.evaluate(bias)) + self.assertAllClose([[0.], [0.]], self.evaluate(price_var)) + self.assertAllClose([[0.], [0.]], self.evaluate(predictions)) sess.run(price_var.assign([[10.], [100.]])) - self.assertAllClose([[210.], [650.]], predictions.eval()) + self.assertAllClose([[210.], [650.]], self.evaluate(predictions)) def test_dense_multi_column(self): - price1 = fc.numeric_column('price1', shape=2) - price2 = fc.numeric_column('price2') + price1 = fc._numeric_column('price1', shape=2) + price2 = fc._numeric_column('price2') with ops.Graph().as_default(): features = {'price1': [[1., 2.], [5., 6.]], 'price2': [[3.], [4.]]} predictions = get_keras_linear_model_predictions(features, @@ -2254,18 +2305,18 @@ class _LinearModelTest(test.TestCase): price1_var = get_linear_model_column_var(price1) price2_var = get_linear_model_column_var(price2) with _initialized_session() as sess: - self.assertAllClose([0.], bias.eval()) - self.assertAllClose([[0.], [0.]], price1_var.eval()) - self.assertAllClose([[0.]], price2_var.eval()) - self.assertAllClose([[0.], [0.]], predictions.eval()) + self.assertAllClose([0.], self.evaluate(bias)) + self.assertAllClose([[0.], [0.]], self.evaluate(price1_var)) + self.assertAllClose([[0.]], self.evaluate(price2_var)) + self.assertAllClose([[0.], [0.]], self.evaluate(predictions)) sess.run(price1_var.assign([[10.], [100.]])) sess.run(price2_var.assign([[1000.]])) sess.run(bias.assign([7.])) - self.assertAllClose([[3217.], [4657.]], predictions.eval()) + self.assertAllClose([[3217.], [4657.]], self.evaluate(predictions)) def test_fills_cols_to_vars(self): - price1 = fc.numeric_column('price1', shape=2) - price2 = fc.numeric_column('price2') + price1 = fc._numeric_column('price1', shape=2) + price2 = fc._numeric_column('price2') with ops.Graph().as_default(): features = {'price1': [[1., 2.], [5., 6.]], 'price2': [[3.], [4.]]} cols_to_vars = {} @@ -2279,8 +2330,8 @@ class _LinearModelTest(test.TestCase): self.assertAllEqual(cols_to_vars[price2], [price2_var]) def test_fills_cols_to_vars_partitioned_variables(self): - price1 = fc.numeric_column('price1', shape=2) - price2 = fc.numeric_column('price2', shape=3) + price1 = fc._numeric_column('price1', shape=2) + price2 = fc._numeric_column('price2', shape=3) with ops.Graph().as_default(): features = { 'price1': [[1., 2.], [6., 7.]], @@ -2303,7 +2354,7 @@ class _LinearModelTest(test.TestCase): self.assertAllEqual([[0.]], cols_to_vars[price2][1].eval()) def test_dense_collection(self): - price = fc.numeric_column('price') + price = fc._numeric_column('price') with ops.Graph().as_default() as g: features = {'price': [[1.], [5.]]} get_keras_linear_model_predictions( @@ -2315,7 +2366,7 @@ class _LinearModelTest(test.TestCase): self.assertIn(price_var, my_vars) def test_sparse_collection(self): - wire_cast = fc.categorical_column_with_hash_bucket('wire_cast', 4) + wire_cast = fc._categorical_column_with_hash_bucket('wire_cast', 4) with ops.Graph().as_default() as g: wire_tensor = sparse_tensor.SparseTensor( values=['omar'], indices=[[0, 0]], dense_shape=[1, 1]) @@ -2329,7 +2380,7 @@ class _LinearModelTest(test.TestCase): self.assertIn(wire_cast_var, my_vars) def test_dense_trainable_default(self): - price = fc.numeric_column('price') + price = fc._numeric_column('price') with ops.Graph().as_default() as g: features = {'price': [[1.], [5.]]} get_keras_linear_model_predictions(features, [price]) @@ -2340,7 +2391,7 @@ class _LinearModelTest(test.TestCase): self.assertIn(price_var, trainable_vars) def test_sparse_trainable_default(self): - wire_cast = fc.categorical_column_with_hash_bucket('wire_cast', 4) + wire_cast = fc._categorical_column_with_hash_bucket('wire_cast', 4) with ops.Graph().as_default() as g: wire_tensor = sparse_tensor.SparseTensor( values=['omar'], indices=[[0, 0]], dense_shape=[1, 1]) @@ -2353,7 +2404,7 @@ class _LinearModelTest(test.TestCase): self.assertIn(wire_cast_var, trainable_vars) def test_dense_trainable_false(self): - price = fc.numeric_column('price') + price = fc._numeric_column('price') with ops.Graph().as_default() as g: features = {'price': [[1.], [5.]]} get_keras_linear_model_predictions(features, [price], trainable=False) @@ -2361,7 +2412,7 @@ class _LinearModelTest(test.TestCase): self.assertEqual([], trainable_vars) def test_sparse_trainable_false(self): - wire_cast = fc.categorical_column_with_hash_bucket('wire_cast', 4) + wire_cast = fc._categorical_column_with_hash_bucket('wire_cast', 4) with ops.Graph().as_default() as g: wire_tensor = sparse_tensor.SparseTensor( values=['omar'], indices=[[0, 0]], dense_shape=[1, 1]) @@ -2371,9 +2422,9 @@ class _LinearModelTest(test.TestCase): self.assertEqual([], trainable_vars) def test_column_order(self): - price_a = fc.numeric_column('price_a') - price_b = fc.numeric_column('price_b') - wire_cast = fc.categorical_column_with_hash_bucket('wire_cast', 4) + price_a = fc._numeric_column('price_a') + price_b = fc._numeric_column('price_b') + wire_cast = fc._categorical_column_with_hash_bucket('wire_cast', 4) with ops.Graph().as_default() as g: features = { 'price_a': [[1.]], @@ -2407,8 +2458,8 @@ class _LinearModelTest(test.TestCase): self.assertIn('wire_cast', my_vars[2].name) def test_static_batch_size_mismatch(self): - price1 = fc.numeric_column('price1') - price2 = fc.numeric_column('price2') + price1 = fc._numeric_column('price1') + price2 = fc._numeric_column('price2') with ops.Graph().as_default(): features = { 'price1': [[1.], [5.], [7.]], # batchsize = 3 @@ -2420,9 +2471,9 @@ class _LinearModelTest(test.TestCase): get_keras_linear_model_predictions(features, [price1, price2]) def test_subset_of_static_batch_size_mismatch(self): - price1 = fc.numeric_column('price1') - price2 = fc.numeric_column('price2') - price3 = fc.numeric_column('price3') + price1 = fc._numeric_column('price1') + price2 = fc._numeric_column('price2') + price3 = fc._numeric_column('price3') with ops.Graph().as_default(): features = { 'price1': array_ops.placeholder(dtype=dtypes.int64), # batchsize = 3 @@ -2435,8 +2486,8 @@ class _LinearModelTest(test.TestCase): get_keras_linear_model_predictions(features, [price1, price2, price3]) def test_runtime_batch_size_mismatch(self): - price1 = fc.numeric_column('price1') - price2 = fc.numeric_column('price2') + price1 = fc._numeric_column('price1') + price2 = fc._numeric_column('price2') with ops.Graph().as_default(): features = { 'price1': array_ops.placeholder(dtype=dtypes.int64), # batchsize = 3 @@ -2451,8 +2502,8 @@ class _LinearModelTest(test.TestCase): predictions, feed_dict={features['price1']: [[1.], [5.], [7.]]}) def test_runtime_batch_size_matches(self): - price1 = fc.numeric_column('price1') - price2 = fc.numeric_column('price2') + price1 = fc._numeric_column('price1') + price2 = fc._numeric_column('price2') with ops.Graph().as_default(): features = { 'price1': array_ops.placeholder(dtype=dtypes.int64), # batchsize = 2 @@ -2468,15 +2519,16 @@ class _LinearModelTest(test.TestCase): features['price2']: [[1.], [5.]], }) + @test_util.run_deprecated_v1 def test_with_1d_sparse_tensor(self): - price = fc.numeric_column('price') - price_buckets = fc.bucketized_column( + price = fc._numeric_column('price') + price_buckets = fc._bucketized_column( price, boundaries=[ 0., 10., 100., ]) - body_style = fc.categorical_column_with_vocabulary_list( + body_style = fc._categorical_column_with_vocabulary_list( 'body-style', vocabulary_list=['hardtop', 'wagon', 'sedan']) # Provides 1-dim tensor and dense tensor. @@ -2506,19 +2558,21 @@ class _LinearModelTest(test.TestCase): sess.run(body_style_var.assign([[-10.], [-100.], [-1000.]])) sess.run(bias.assign([5.])) - self.assertAllClose([[10 - 1000 + 5.], [1000 - 10 + 5.]], sess.run(net)) + self.assertAllClose([[10 - 1000 + 5.], [1000 - 10 + 5.]], + self.evaluate(net)) + @test_util.run_deprecated_v1 def test_with_1d_unknown_shape_sparse_tensor(self): - price = fc.numeric_column('price') - price_buckets = fc.bucketized_column( + price = fc._numeric_column('price') + price_buckets = fc._bucketized_column( price, boundaries=[ 0., 10., 100., ]) - body_style = fc.categorical_column_with_vocabulary_list( + body_style = fc._categorical_column_with_vocabulary_list( 'body-style', vocabulary_list=['hardtop', 'wagon', 'sedan']) - country = fc.categorical_column_with_vocabulary_list( + country = fc._categorical_column_with_vocabulary_list( 'country', vocabulary_list=['US', 'JP', 'CA']) # Provides 1-dim tensor and dense tensor. @@ -2554,8 +2608,9 @@ class _LinearModelTest(test.TestCase): features['country']: country_data })) + @test_util.run_deprecated_v1 def test_with_rank_0_feature(self): - price = fc.numeric_column('price') + price = fc._numeric_column('price') features = { 'price': constant_op.constant(0), } @@ -2581,7 +2636,7 @@ class InputLayerTest(test.TestCase): @test_util.run_in_graph_and_eager_modes def test_retrieving_input(self): features = {'a': [0.]} - input_layer = InputLayer(fc.numeric_column('a')) + input_layer = InputLayer(fc._numeric_column('a')) inputs = self.evaluate(input_layer(features)) self.assertAllClose([[0.]], inputs) @@ -2593,8 +2648,8 @@ class InputLayerTest(test.TestCase): dense_shape=(3, 3)) # Create feature columns (categorical and embedding). - categorical_column = fc.categorical_column_with_identity(key='a', - num_buckets=3) + categorical_column = fc._categorical_column_with_identity( + key='a', num_buckets=3) embedding_dimension = 2 def _embedding_column_initializer(shape, dtype, partition_info): del shape # unused @@ -2605,7 +2660,8 @@ class InputLayerTest(test.TestCase): (0, 1), # id 1 (1, 1)) # id 2 return embedding_values - embedding_column = fc.embedding_column( + + embedding_column = fc._embedding_column( categorical_column, dimension=embedding_dimension, initializer=_embedding_column_initializer) @@ -2636,8 +2692,8 @@ class InputLayerTest(test.TestCase): dense_shape=(3, 3)) # Create feature columns (categorical and embedding). - categorical_column = fc.categorical_column_with_identity(key='a', - num_buckets=3) + categorical_column = fc._categorical_column_with_identity( + key='a', num_buckets=3) embedding_dimension = 2 def _embedding_column_initializer(shape, dtype, partition_info): @@ -2650,7 +2706,7 @@ class InputLayerTest(test.TestCase): (1, 1)) # id 2 return embedding_values - embedding_column = fc.embedding_column( + embedding_column = fc._embedding_column( categorical_column, dimension=embedding_dimension, initializer=_embedding_column_initializer) @@ -2687,56 +2743,56 @@ class FunctionalInputLayerTest(test.TestCase): fc.input_layer( features={'a': [[0]]}, feature_columns=[ - fc.categorical_column_with_hash_bucket('wire_cast', 4) + fc._categorical_column_with_hash_bucket('wire_cast', 4) ]) def test_does_not_support_dict_columns(self): with self.assertRaisesRegexp( ValueError, 'Expected feature_columns to be iterable, found dict.'): fc.input_layer( - features={'a': [[0]]}, feature_columns={'a': fc.numeric_column('a')}) + features={'a': [[0]]}, feature_columns={'a': fc._numeric_column('a')}) def test_bare_column(self): with ops.Graph().as_default(): features = features = {'a': [0.]} - net = fc.input_layer(features, fc.numeric_column('a')) + net = fc.input_layer(features, fc._numeric_column('a')) with _initialized_session(): - self.assertAllClose([[0.]], net.eval()) + self.assertAllClose([[0.]], self.evaluate(net)) def test_column_generator(self): with ops.Graph().as_default(): features = features = {'a': [0.], 'b': [1.]} - columns = (fc.numeric_column(key) for key in features) + columns = (fc._numeric_column(key) for key in features) net = fc.input_layer(features, columns) with _initialized_session(): - self.assertAllClose([[0., 1.]], net.eval()) + self.assertAllClose([[0., 1.]], self.evaluate(net)) def test_raises_if_duplicate_name(self): with self.assertRaisesRegexp( ValueError, 'Duplicate feature column name found for columns'): fc.input_layer( features={'a': [[0]]}, - feature_columns=[fc.numeric_column('a'), - fc.numeric_column('a')]) + feature_columns=[fc._numeric_column('a'), + fc._numeric_column('a')]) def test_one_column(self): - price = fc.numeric_column('price') + price = fc._numeric_column('price') with ops.Graph().as_default(): features = {'price': [[1.], [5.]]} net = fc.input_layer(features, [price]) with _initialized_session(): - self.assertAllClose([[1.], [5.]], net.eval()) + self.assertAllClose([[1.], [5.]], self.evaluate(net)) def test_multi_dimension(self): - price = fc.numeric_column('price', shape=2) + price = fc._numeric_column('price', shape=2) with ops.Graph().as_default(): features = {'price': [[1., 2.], [5., 6.]]} net = fc.input_layer(features, [price]) with _initialized_session(): - self.assertAllClose([[1., 2.], [5., 6.]], net.eval()) + self.assertAllClose([[1., 2.], [5., 6.]], self.evaluate(net)) def test_raises_if_shape_mismatch(self): - price = fc.numeric_column('price', shape=2) + price = fc._numeric_column('price', shape=2) with ops.Graph().as_default(): features = {'price': [[1.], [5.]]} with self.assertRaisesRegexp( @@ -2745,16 +2801,16 @@ class FunctionalInputLayerTest(test.TestCase): fc.input_layer(features, [price]) def test_reshaping(self): - price = fc.numeric_column('price', shape=[1, 2]) + price = fc._numeric_column('price', shape=[1, 2]) with ops.Graph().as_default(): features = {'price': [[[1., 2.]], [[5., 6.]]]} net = fc.input_layer(features, [price]) with _initialized_session(): - self.assertAllClose([[1., 2.], [5., 6.]], net.eval()) + self.assertAllClose([[1., 2.], [5., 6.]], self.evaluate(net)) def test_multi_column(self): - price1 = fc.numeric_column('price1', shape=2) - price2 = fc.numeric_column('price2') + price1 = fc._numeric_column('price1', shape=2) + price2 = fc._numeric_column('price2') with ops.Graph().as_default(): features = { 'price1': [[1., 2.], [5., 6.]], @@ -2762,19 +2818,19 @@ class FunctionalInputLayerTest(test.TestCase): } net = fc.input_layer(features, [price1, price2]) with _initialized_session(): - self.assertAllClose([[1., 2., 3.], [5., 6., 4.]], net.eval()) + self.assertAllClose([[1., 2., 3.], [5., 6., 4.]], self.evaluate(net)) def test_fills_cols_to_vars(self): # Provide three _DenseColumn's to input_layer: a _NumericColumn, a # _BucketizedColumn, and an _EmbeddingColumn. Only the _EmbeddingColumn # creates a Variable. - price1 = fc.numeric_column('price1') - dense_feature = fc.numeric_column('dense_feature') - dense_feature_bucketized = fc.bucketized_column( + price1 = fc._numeric_column('price1') + dense_feature = fc._numeric_column('dense_feature') + dense_feature_bucketized = fc._bucketized_column( dense_feature, boundaries=[0.]) - some_sparse_column = fc.categorical_column_with_hash_bucket( + some_sparse_column = fc._categorical_column_with_hash_bucket( 'sparse_feature', hash_bucket_size=5) - some_embedding_column = fc.embedding_column( + some_embedding_column = fc._embedding_column( some_sparse_column, dimension=10) with ops.Graph().as_default(): features = { @@ -2793,24 +2849,25 @@ class FunctionalInputLayerTest(test.TestCase): variables_lib.Variable) self.assertAllEqual(cols_to_vars[some_embedding_column][0].shape, [5, 10]) + @test_util.run_deprecated_v1 def test_fills_cols_to_vars_shared_embedding(self): # Provide 5 DenseColumn's to input_layer: a NumericColumn, a # BucketizedColumn, an EmbeddingColumn, two SharedEmbeddingColumns. The # EmbeddingColumn creates a Variable and the two SharedEmbeddingColumns # shared one variable. - price1 = fc.numeric_column('price1') - dense_feature = fc.numeric_column('dense_feature') - dense_feature_bucketized = fc.bucketized_column( + price1 = fc._numeric_column('price1') + dense_feature = fc._numeric_column('dense_feature') + dense_feature_bucketized = fc._bucketized_column( dense_feature, boundaries=[0.]) - some_sparse_column = fc.categorical_column_with_hash_bucket( + some_sparse_column = fc._categorical_column_with_hash_bucket( 'sparse_feature', hash_bucket_size=5) - some_embedding_column = fc.embedding_column( + some_embedding_column = fc._embedding_column( some_sparse_column, dimension=10) - categorical_column_a = fc.categorical_column_with_identity( + categorical_column_a = fc._categorical_column_with_identity( key='aaa', num_buckets=3) - categorical_column_b = fc.categorical_column_with_identity( + categorical_column_b = fc._categorical_column_with_identity( key='bbb', num_buckets=3) - shared_embedding_a, shared_embedding_b = fc.shared_embedding_columns( + shared_embedding_a, shared_embedding_b = fc_new.shared_embedding_columns( [categorical_column_a, categorical_column_b], dimension=2) with ops.Graph().as_default(): features = { @@ -2850,13 +2907,13 @@ class FunctionalInputLayerTest(test.TestCase): self.assertAllEqual(cols_to_vars[shared_embedding_a][0].shape, [3, 2]) def test_fills_cols_to_vars_partitioned_variables(self): - price1 = fc.numeric_column('price1') - dense_feature = fc.numeric_column('dense_feature') - dense_feature_bucketized = fc.bucketized_column( + price1 = fc._numeric_column('price1') + dense_feature = fc._numeric_column('dense_feature') + dense_feature_bucketized = fc._bucketized_column( dense_feature, boundaries=[0.]) - some_sparse_column = fc.categorical_column_with_hash_bucket( + some_sparse_column = fc._categorical_column_with_hash_bucket( 'sparse_feature', hash_bucket_size=5) - some_embedding_column = fc.embedding_column( + some_embedding_column = fc._embedding_column( some_sparse_column, dimension=10) with ops.Graph().as_default(): features = { @@ -2883,8 +2940,8 @@ class FunctionalInputLayerTest(test.TestCase): self.assertAllEqual(cols_to_vars[some_embedding_column][2].shape, [1, 10]) def test_column_order(self): - price_a = fc.numeric_column('price_a') - price_b = fc.numeric_column('price_b') + price_a = fc._numeric_column('price_a') + price_b = fc._numeric_column('price_b') with ops.Graph().as_default(): features = { 'price_a': [[1.]], @@ -2893,11 +2950,11 @@ class FunctionalInputLayerTest(test.TestCase): net1 = fc.input_layer(features, [price_a, price_b]) net2 = fc.input_layer(features, [price_b, price_a]) with _initialized_session(): - self.assertAllClose([[1., 3.]], net1.eval()) - self.assertAllClose([[1., 3.]], net2.eval()) + self.assertAllClose([[1., 3.]], self.evaluate(net1)) + self.assertAllClose([[1., 3.]], self.evaluate(net2)) def test_fails_for_categorical_column(self): - animal = fc.categorical_column_with_identity('animal', num_buckets=4) + animal = fc._categorical_column_with_identity('animal', num_buckets=4) with ops.Graph().as_default(): features = { 'animal': @@ -2908,8 +2965,8 @@ class FunctionalInputLayerTest(test.TestCase): fc.input_layer(features, [animal]) def test_static_batch_size_mismatch(self): - price1 = fc.numeric_column('price1') - price2 = fc.numeric_column('price2') + price1 = fc._numeric_column('price1') + price2 = fc._numeric_column('price2') with ops.Graph().as_default(): features = { 'price1': [[1.], [5.], [7.]], # batchsize = 3 @@ -2921,9 +2978,9 @@ class FunctionalInputLayerTest(test.TestCase): fc.input_layer(features, [price1, price2]) def test_subset_of_static_batch_size_mismatch(self): - price1 = fc.numeric_column('price1') - price2 = fc.numeric_column('price2') - price3 = fc.numeric_column('price3') + price1 = fc._numeric_column('price1') + price2 = fc._numeric_column('price2') + price3 = fc._numeric_column('price3') with ops.Graph().as_default(): features = { 'price1': array_ops.placeholder(dtype=dtypes.int64), # batchsize = 3 @@ -2936,8 +2993,8 @@ class FunctionalInputLayerTest(test.TestCase): fc.input_layer(features, [price1, price2, price3]) def test_runtime_batch_size_mismatch(self): - price1 = fc.numeric_column('price1') - price2 = fc.numeric_column('price2') + price1 = fc._numeric_column('price1') + price2 = fc._numeric_column('price2') with ops.Graph().as_default(): features = { 'price1': array_ops.placeholder(dtype=dtypes.int64), # batchsize = 3 @@ -2950,8 +3007,8 @@ class FunctionalInputLayerTest(test.TestCase): sess.run(net, feed_dict={features['price1']: [[1.], [5.], [7.]]}) def test_runtime_batch_size_matches(self): - price1 = fc.numeric_column('price1') - price2 = fc.numeric_column('price2') + price1 = fc._numeric_column('price1') + price2 = fc._numeric_column('price2') with ops.Graph().as_default(): features = { 'price1': array_ops.placeholder(dtype=dtypes.int64), # batchsize = 2 @@ -2967,9 +3024,9 @@ class FunctionalInputLayerTest(test.TestCase): }) def test_multiple_layers_with_same_embedding_column(self): - some_sparse_column = fc.categorical_column_with_hash_bucket( + some_sparse_column = fc._categorical_column_with_hash_bucket( 'sparse_feature', hash_bucket_size=5) - some_embedding_column = fc.embedding_column( + some_embedding_column = fc._embedding_column( some_sparse_column, dimension=10) with ops.Graph().as_default(): @@ -2990,13 +3047,14 @@ class FunctionalInputLayerTest(test.TestCase): expected_var_names, [v.name for v in ops.get_collection(ops.GraphKeys.GLOBAL_VARIABLES)]) + @test_util.run_deprecated_v1 def test_multiple_layers_with_same_shared_embedding_column(self): - categorical_column_a = fc.categorical_column_with_identity( + categorical_column_a = fc._categorical_column_with_identity( key='aaa', num_buckets=3) - categorical_column_b = fc.categorical_column_with_identity( + categorical_column_b = fc._categorical_column_with_identity( key='bbb', num_buckets=3) embedding_dimension = 2 - embedding_column_b, embedding_column_a = fc.shared_embedding_columns( + embedding_column_b, embedding_column_a = fc_new.shared_embedding_columns( [categorical_column_b, categorical_column_a], dimension=embedding_dimension) @@ -3023,13 +3081,14 @@ class FunctionalInputLayerTest(test.TestCase): ['input_layer/aaa_bbb_shared_embedding/embedding_weights:0'], [v.name for v in ops.get_collection(ops.GraphKeys.GLOBAL_VARIABLES)]) + @test_util.run_deprecated_v1 def test_multiple_layers_with_same_shared_embedding_column_diff_graphs(self): - categorical_column_a = fc.categorical_column_with_identity( + categorical_column_a = fc._categorical_column_with_identity( key='aaa', num_buckets=3) - categorical_column_b = fc.categorical_column_with_identity( + categorical_column_b = fc._categorical_column_with_identity( key='bbb', num_buckets=3) embedding_dimension = 2 - embedding_column_b, embedding_column_a = fc.shared_embedding_columns( + embedding_column_b, embedding_column_a = fc_new.shared_embedding_columns( [categorical_column_b, categorical_column_a], dimension=embedding_dimension) all_cols = [embedding_column_a, embedding_column_b] @@ -3074,6 +3133,7 @@ class FunctionalInputLayerTest(test.TestCase): ['input_layer/aaa_bbb_shared_embedding/embedding_weights:0'], [v.name for v in ops.get_collection(ops.GraphKeys.GLOBAL_VARIABLES)]) + @test_util.run_deprecated_v1 def test_with_1d_sparse_tensor(self): embedding_values = ( (1., 2., 3., 4., 5.), # id 0 @@ -3085,18 +3145,18 @@ class FunctionalInputLayerTest(test.TestCase): return embedding_values # price has 1 dimension in input_layer - price = fc.numeric_column('price') + price = fc._numeric_column('price') # one_hot_body_style has 3 dims in input_layer. - body_style = fc.categorical_column_with_vocabulary_list( + body_style = fc._categorical_column_with_vocabulary_list( 'body-style', vocabulary_list=['hardtop', 'wagon', 'sedan']) - one_hot_body_style = fc.indicator_column(body_style) + one_hot_body_style = fc._indicator_column(body_style) # embedded_body_style has 5 dims in input_layer. - country = fc.categorical_column_with_vocabulary_list( + country = fc._categorical_column_with_vocabulary_list( 'country', vocabulary_list=['US', 'JP', 'CA']) - embedded_country = fc.embedding_column(country, dimension=5, - initializer=_initializer) + embedded_country = fc._embedding_column( + country, dimension=5, initializer=_initializer) # Provides 1-dim tensor and dense tensor. features = { @@ -3124,6 +3184,7 @@ class FunctionalInputLayerTest(test.TestCase): [1., 0., 0., 1., 2., 3., 4., 5., 12.]], sess.run(net)) + @test_util.run_deprecated_v1 def test_with_1d_unknown_shape_sparse_tensor(self): embedding_values = ( (1., 2.), # id 0 @@ -3135,17 +3196,17 @@ class FunctionalInputLayerTest(test.TestCase): return embedding_values # price has 1 dimension in input_layer - price = fc.numeric_column('price') + price = fc._numeric_column('price') # one_hot_body_style has 3 dims in input_layer. - body_style = fc.categorical_column_with_vocabulary_list( + body_style = fc._categorical_column_with_vocabulary_list( 'body-style', vocabulary_list=['hardtop', 'wagon', 'sedan']) - one_hot_body_style = fc.indicator_column(body_style) + one_hot_body_style = fc._indicator_column(body_style) # embedded_body_style has 5 dims in input_layer. - country = fc.categorical_column_with_vocabulary_list( + country = fc._categorical_column_with_vocabulary_list( 'country', vocabulary_list=['US', 'JP', 'CA']) - embedded_country = fc.embedding_column( + embedded_country = fc._embedding_column( country, dimension=2, initializer=_initializer) # Provides 1-dim tensor and dense tensor. @@ -3183,9 +3244,10 @@ class FunctionalInputLayerTest(test.TestCase): features['country']: country_data })) + @test_util.run_deprecated_v1 def test_with_rank_0_feature(self): # price has 1 dimension in input_layer - price = fc.numeric_column('price') + price = fc._numeric_column('price') features = { 'price': constant_op.constant(0), } @@ -3313,8 +3375,9 @@ class VocabularyFileCategoricalColumnTest(test.TestCase): 'python/feature_column/testdata/wire_vocabulary.txt') self._wire_vocabulary_size = 3 + @test_util.run_deprecated_v1 def test_defaults(self): - column = fc.categorical_column_with_vocabulary_file( + column = fc._categorical_column_with_vocabulary_file( key='aaa', vocabulary_file='path_to_file', vocabulary_size=3) self.assertEqual('aaa', column.name) self.assertEqual('aaa', column._var_scope_name) @@ -3326,22 +3389,30 @@ class VocabularyFileCategoricalColumnTest(test.TestCase): def test_key_should_be_string(self): with self.assertRaisesRegexp(ValueError, 'key must be a string.'): - fc.categorical_column_with_vocabulary_file( + fc._categorical_column_with_vocabulary_file( key=('aaa',), vocabulary_file='path_to_file', vocabulary_size=3) + @test_util.run_deprecated_v1 def test_all_constructor_args(self): - column = fc.categorical_column_with_vocabulary_file( - key='aaa', vocabulary_file='path_to_file', vocabulary_size=3, - num_oov_buckets=4, dtype=dtypes.int32) + column = fc._categorical_column_with_vocabulary_file( + key='aaa', + vocabulary_file='path_to_file', + vocabulary_size=3, + num_oov_buckets=4, + dtype=dtypes.int32) self.assertEqual(7, column._num_buckets) self.assertEqual({ 'aaa': parsing_ops.VarLenFeature(dtypes.int32) }, column._parse_example_spec) + @test_util.run_deprecated_v1 def test_deep_copy(self): - original = fc.categorical_column_with_vocabulary_file( - key='aaa', vocabulary_file='path_to_file', vocabulary_size=3, - num_oov_buckets=4, dtype=dtypes.int32) + original = fc._categorical_column_with_vocabulary_file( + key='aaa', + vocabulary_file='path_to_file', + vocabulary_size=3, + num_oov_buckets=4, + dtype=dtypes.int32) for column in (original, copy.deepcopy(original)): self.assertEqual('aaa', column.name) self.assertEqual(7, column._num_buckets) @@ -3351,16 +3422,17 @@ class VocabularyFileCategoricalColumnTest(test.TestCase): def test_vocabulary_file_none(self): with self.assertRaisesRegexp(ValueError, 'Missing vocabulary_file'): - fc.categorical_column_with_vocabulary_file( + fc._categorical_column_with_vocabulary_file( key='aaa', vocabulary_file=None, vocabulary_size=3) def test_vocabulary_file_empty_string(self): with self.assertRaisesRegexp(ValueError, 'Missing vocabulary_file'): - fc.categorical_column_with_vocabulary_file( + fc._categorical_column_with_vocabulary_file( key='aaa', vocabulary_file='', vocabulary_size=3) + @test_util.run_deprecated_v1 def test_invalid_vocabulary_file(self): - column = fc.categorical_column_with_vocabulary_file( + column = fc._categorical_column_with_vocabulary_file( key='aaa', vocabulary_file='file_does_not_exist', vocabulary_size=10) inputs = sparse_tensor.SparseTensorValue( indices=((0, 0), (1, 0), (1, 1)), @@ -3373,16 +3445,19 @@ class VocabularyFileCategoricalColumnTest(test.TestCase): def test_invalid_vocabulary_size(self): with self.assertRaisesRegexp(ValueError, 'Invalid vocabulary_size'): - fc.categorical_column_with_vocabulary_file( - key='aaa', vocabulary_file=self._wire_vocabulary_file_name, + fc._categorical_column_with_vocabulary_file( + key='aaa', + vocabulary_file=self._wire_vocabulary_file_name, vocabulary_size=-1) with self.assertRaisesRegexp(ValueError, 'Invalid vocabulary_size'): - fc.categorical_column_with_vocabulary_file( - key='aaa', vocabulary_file=self._wire_vocabulary_file_name, + fc._categorical_column_with_vocabulary_file( + key='aaa', + vocabulary_file=self._wire_vocabulary_file_name, vocabulary_size=0) + @test_util.run_deprecated_v1 def test_too_large_vocabulary_size(self): - column = fc.categorical_column_with_vocabulary_file( + column = fc._categorical_column_with_vocabulary_file( key='aaa', vocabulary_file=self._wire_vocabulary_file_name, vocabulary_size=self._wire_vocabulary_size + 1) @@ -3397,20 +3472,24 @@ class VocabularyFileCategoricalColumnTest(test.TestCase): def test_invalid_num_oov_buckets(self): with self.assertRaisesRegexp(ValueError, 'Invalid num_oov_buckets'): - fc.categorical_column_with_vocabulary_file( - key='aaa', vocabulary_file='path', vocabulary_size=3, + fc._categorical_column_with_vocabulary_file( + key='aaa', + vocabulary_file='path', + vocabulary_size=3, num_oov_buckets=-1) def test_invalid_dtype(self): with self.assertRaisesRegexp(ValueError, 'dtype must be string or integer'): - fc.categorical_column_with_vocabulary_file( - key='aaa', vocabulary_file='path', vocabulary_size=3, + fc._categorical_column_with_vocabulary_file( + key='aaa', + vocabulary_file='path', + vocabulary_size=3, dtype=dtypes.float64) def test_invalid_buckets_and_default_value(self): with self.assertRaisesRegexp( ValueError, 'both num_oov_buckets and default_value'): - fc.categorical_column_with_vocabulary_file( + fc._categorical_column_with_vocabulary_file( key='aaa', vocabulary_file=self._wire_vocabulary_file_name, vocabulary_size=self._wire_vocabulary_size, @@ -3418,7 +3497,7 @@ class VocabularyFileCategoricalColumnTest(test.TestCase): default_value=2) def test_invalid_input_dtype_int32(self): - column = fc.categorical_column_with_vocabulary_file( + column = fc._categorical_column_with_vocabulary_file( key='aaa', vocabulary_file=self._wire_vocabulary_file_name, vocabulary_size=self._wire_vocabulary_size, @@ -3431,7 +3510,7 @@ class VocabularyFileCategoricalColumnTest(test.TestCase): column._get_sparse_tensors(_LazyBuilder({'aaa': inputs})) def test_invalid_input_dtype_string(self): - column = fc.categorical_column_with_vocabulary_file( + column = fc._categorical_column_with_vocabulary_file( key='aaa', vocabulary_file=self._warriors_vocabulary_file_name, vocabulary_size=self._warriors_vocabulary_size, @@ -3443,8 +3522,9 @@ class VocabularyFileCategoricalColumnTest(test.TestCase): with self.assertRaisesRegexp(ValueError, 'dtype must be compatible'): column._get_sparse_tensors(_LazyBuilder({'aaa': inputs})) + @test_util.run_deprecated_v1 def test_parse_example(self): - a = fc.categorical_column_with_vocabulary_file( + a = fc._categorical_column_with_vocabulary_file( key='aaa', vocabulary_file='path_to_file', vocabulary_size=3) data = example_pb2.Example(features=feature_pb2.Features( feature={ @@ -3465,8 +3545,9 @@ class VocabularyFileCategoricalColumnTest(test.TestCase): dense_shape=[1, 2]), features['aaa'].eval()) + @test_util.run_deprecated_v1 def test_get_sparse_tensors(self): - column = fc.categorical_column_with_vocabulary_file( + column = fc._categorical_column_with_vocabulary_file( key='aaa', vocabulary_file=self._wire_vocabulary_file_name, vocabulary_size=self._wire_vocabulary_size) @@ -3485,8 +3566,9 @@ class VocabularyFileCategoricalColumnTest(test.TestCase): dense_shape=inputs.dense_shape), id_weight_pair.id_tensor.eval()) + @test_util.run_deprecated_v1 def test_get_sparse_tensors_none_vocabulary_size(self): - column = fc.categorical_column_with_vocabulary_file( + column = fc._categorical_column_with_vocabulary_file( key='aaa', vocabulary_file=self._wire_vocabulary_file_name) inputs = sparse_tensor.SparseTensorValue( indices=((0, 0), (1, 0), (1, 1)), @@ -3503,8 +3585,9 @@ class VocabularyFileCategoricalColumnTest(test.TestCase): dense_shape=inputs.dense_shape), id_weight_pair.id_tensor.eval()) + @test_util.run_deprecated_v1 def test_transform_feature(self): - column = fc.categorical_column_with_vocabulary_file( + column = fc._categorical_column_with_vocabulary_file( key='aaa', vocabulary_file=self._wire_vocabulary_file_name, vocabulary_size=self._wire_vocabulary_size) @@ -3514,16 +3597,15 @@ class VocabularyFileCategoricalColumnTest(test.TestCase): dense_shape=(2, 2)) id_tensor = _transform_features({'aaa': inputs}, [column])[column] with _initialized_session(): - _assert_sparse_tensor_value(self, - sparse_tensor.SparseTensorValue( - indices=inputs.indices, - values=np.array( - (2, -1, 0), dtype=np.int64), - dense_shape=inputs.dense_shape), - id_tensor.eval()) + _assert_sparse_tensor_value( + self, + sparse_tensor.SparseTensorValue( + indices=inputs.indices, + values=np.array((2, -1, 0), dtype=np.int64), + dense_shape=inputs.dense_shape), self.evaluate(id_tensor)) def test_get_sparse_tensors_weight_collections(self): - column = fc.categorical_column_with_vocabulary_file( + column = fc._categorical_column_with_vocabulary_file( key='aaa', vocabulary_file=self._wire_vocabulary_file_name, vocabulary_size=self._wire_vocabulary_size) @@ -3540,8 +3622,9 @@ class VocabularyFileCategoricalColumnTest(test.TestCase): [], ops.get_collection(ops.GraphKeys.GLOBAL_VARIABLES)) self.assertItemsEqual([], ops.get_collection('my_weights')) + @test_util.run_deprecated_v1 def test_get_sparse_tensors_dense_input(self): - column = fc.categorical_column_with_vocabulary_file( + column = fc._categorical_column_with_vocabulary_file( key='aaa', vocabulary_file=self._wire_vocabulary_file_name, vocabulary_size=self._wire_vocabulary_size) @@ -3559,8 +3642,9 @@ class VocabularyFileCategoricalColumnTest(test.TestCase): dense_shape=(2, 2)), id_weight_pair.id_tensor.eval()) + @test_util.run_deprecated_v1 def test_get_sparse_tensors_default_value_in_vocabulary(self): - column = fc.categorical_column_with_vocabulary_file( + column = fc._categorical_column_with_vocabulary_file( key='aaa', vocabulary_file=self._wire_vocabulary_file_name, vocabulary_size=self._wire_vocabulary_size, @@ -3580,8 +3664,9 @@ class VocabularyFileCategoricalColumnTest(test.TestCase): dense_shape=inputs.dense_shape), id_weight_pair.id_tensor.eval()) + @test_util.run_deprecated_v1 def test_get_sparse_tensors_with_oov_buckets(self): - column = fc.categorical_column_with_vocabulary_file( + column = fc._categorical_column_with_vocabulary_file( key='aaa', vocabulary_file=self._wire_vocabulary_file_name, vocabulary_size=self._wire_vocabulary_size, @@ -3601,11 +3686,12 @@ class VocabularyFileCategoricalColumnTest(test.TestCase): dense_shape=inputs.dense_shape), id_weight_pair.id_tensor.eval()) + @test_util.run_deprecated_v1 def test_get_sparse_tensors_small_vocabulary_size(self): # 'marlo' is the last entry in our vocabulary file, so be setting # `vocabulary_size` to 1 less than number of entries in file, we take # 'marlo' out of the vocabulary. - column = fc.categorical_column_with_vocabulary_file( + column = fc._categorical_column_with_vocabulary_file( key='aaa', vocabulary_file=self._wire_vocabulary_file_name, vocabulary_size=self._wire_vocabulary_size - 1) @@ -3624,8 +3710,9 @@ class VocabularyFileCategoricalColumnTest(test.TestCase): dense_shape=inputs.dense_shape), id_weight_pair.id_tensor.eval()) + @test_util.run_deprecated_v1 def test_get_sparse_tensors_int32(self): - column = fc.categorical_column_with_vocabulary_file( + column = fc._categorical_column_with_vocabulary_file( key='aaa', vocabulary_file=self._warriors_vocabulary_file_name, vocabulary_size=self._warriors_vocabulary_size, @@ -3645,9 +3732,10 @@ class VocabularyFileCategoricalColumnTest(test.TestCase): dense_shape=inputs.dense_shape), id_weight_pair.id_tensor.eval()) + @test_util.run_deprecated_v1 def test_get_sparse_tensors_int32_dense_input(self): default_value = -100 - column = fc.categorical_column_with_vocabulary_file( + column = fc._categorical_column_with_vocabulary_file( key='aaa', vocabulary_file=self._warriors_vocabulary_file_name, vocabulary_size=self._warriors_vocabulary_size, @@ -3667,8 +3755,9 @@ class VocabularyFileCategoricalColumnTest(test.TestCase): dense_shape=(3, 3)), id_weight_pair.id_tensor.eval()) + @test_util.run_deprecated_v1 def test_get_sparse_tensors_int32_with_oov_buckets(self): - column = fc.categorical_column_with_vocabulary_file( + column = fc._categorical_column_with_vocabulary_file( key='aaa', vocabulary_file=self._warriors_vocabulary_file_name, vocabulary_size=self._warriors_vocabulary_size, @@ -3689,8 +3778,9 @@ class VocabularyFileCategoricalColumnTest(test.TestCase): dense_shape=inputs.dense_shape), id_weight_pair.id_tensor.eval()) + @test_util.run_deprecated_v1 def test_linear_model(self): - wire_column = fc.categorical_column_with_vocabulary_file( + wire_column = fc._categorical_column_with_vocabulary_file( key='wire', vocabulary_file=self._wire_vocabulary_file_name, vocabulary_size=self._wire_vocabulary_size, @@ -3706,16 +3796,18 @@ class VocabularyFileCategoricalColumnTest(test.TestCase): bias = get_linear_model_bias() wire_var = get_linear_model_column_var(wire_column) with _initialized_session(): - self.assertAllClose((0.,), bias.eval()) - self.assertAllClose(((0.,), (0.,), (0.,), (0.,)), wire_var.eval()) - self.assertAllClose(((0.,), (0.,)), predictions.eval()) + self.assertAllClose((0.,), self.evaluate(bias)) + self.assertAllClose(((0.,), (0.,), (0.,), (0.,)), + self.evaluate(wire_var)) + self.assertAllClose(((0.,), (0.,)), self.evaluate(predictions)) wire_var.assign(((1.,), (2.,), (3.,), (4.,))).eval() # 'marlo' -> 2: wire_var[2] = 3 # 'skywalker' -> 3, 'omar' -> 0: wire_var[3] + wire_var[0] = 4+1 = 5 - self.assertAllClose(((3.,), (5.,)), predictions.eval()) + self.assertAllClose(((3.,), (5.,)), self.evaluate(predictions)) + @test_util.run_deprecated_v1 def test_keras_linear_model(self): - wire_column = fc.categorical_column_with_vocabulary_file( + wire_column = fc._categorical_column_with_vocabulary_file( key='wire', vocabulary_file=self._wire_vocabulary_file_name, vocabulary_size=self._wire_vocabulary_size, @@ -3732,19 +3824,20 @@ class VocabularyFileCategoricalColumnTest(test.TestCase): bias = get_linear_model_bias() wire_var = get_linear_model_column_var(wire_column) with _initialized_session(): - self.assertAllClose((0.,), bias.eval()) - self.assertAllClose(((0.,), (0.,), (0.,), (0.,)), wire_var.eval()) - self.assertAllClose(((0.,), (0.,)), predictions.eval()) + self.assertAllClose((0.,), self.evaluate(bias)) + self.assertAllClose(((0.,), (0.,), (0.,), (0.,)), + self.evaluate(wire_var)) + self.assertAllClose(((0.,), (0.,)), self.evaluate(predictions)) wire_var.assign(((1.,), (2.,), (3.,), (4.,))).eval() # 'marlo' -> 2: wire_var[2] = 3 # 'skywalker' -> 3, 'omar' -> 0: wire_var[3] + wire_var[0] = 4+1 = 5 - self.assertAllClose(((3.,), (5.,)), predictions.eval()) + self.assertAllClose(((3.,), (5.,)), self.evaluate(predictions)) class VocabularyListCategoricalColumnTest(test.TestCase): def test_defaults_string(self): - column = fc.categorical_column_with_vocabulary_list( + column = fc._categorical_column_with_vocabulary_list( key='aaa', vocabulary_list=('omar', 'stringer', 'marlo')) self.assertEqual('aaa', column.name) self.assertEqual('aaa', column.key) @@ -3756,11 +3849,11 @@ class VocabularyListCategoricalColumnTest(test.TestCase): def test_key_should_be_string(self): with self.assertRaisesRegexp(ValueError, 'key must be a string.'): - fc.categorical_column_with_vocabulary_list( + fc._categorical_column_with_vocabulary_list( key=('aaa',), vocabulary_list=('omar', 'stringer', 'marlo')) def test_defaults_int(self): - column = fc.categorical_column_with_vocabulary_list( + column = fc._categorical_column_with_vocabulary_list( key='aaa', vocabulary_list=(12, 24, 36)) self.assertEqual('aaa', column.name) self.assertEqual('aaa', column.key) @@ -3770,17 +3863,21 @@ class VocabularyListCategoricalColumnTest(test.TestCase): 'aaa': parsing_ops.VarLenFeature(dtypes.int64) }, column._parse_example_spec) + @test_util.run_deprecated_v1 def test_all_constructor_args(self): - column = fc.categorical_column_with_vocabulary_list( - key='aaa', vocabulary_list=(12, 24, 36), dtype=dtypes.int32, + column = fc._categorical_column_with_vocabulary_list( + key='aaa', + vocabulary_list=(12, 24, 36), + dtype=dtypes.int32, default_value=-99) self.assertEqual(3, column._num_buckets) self.assertEqual({ 'aaa': parsing_ops.VarLenFeature(dtypes.int32) }, column._parse_example_spec) + @test_util.run_deprecated_v1 def test_deep_copy(self): - original = fc.categorical_column_with_vocabulary_list( + original = fc._categorical_column_with_vocabulary_list( key='aaa', vocabulary_list=(12, 24, 36), dtype=dtypes.int32) for column in (original, copy.deepcopy(original)): self.assertEqual('aaa', column.name) @@ -3791,65 +3888,65 @@ class VocabularyListCategoricalColumnTest(test.TestCase): def test_invalid_dtype(self): with self.assertRaisesRegexp(ValueError, 'dtype must be string or integer'): - fc.categorical_column_with_vocabulary_list( - key='aaa', vocabulary_list=('omar', 'stringer', 'marlo'), + fc._categorical_column_with_vocabulary_list( + key='aaa', + vocabulary_list=('omar', 'stringer', 'marlo'), dtype=dtypes.float32) def test_invalid_mapping_dtype(self): with self.assertRaisesRegexp( ValueError, r'vocabulary dtype must be string or integer'): - fc.categorical_column_with_vocabulary_list( + fc._categorical_column_with_vocabulary_list( key='aaa', vocabulary_list=(12., 24., 36.)) def test_mismatched_int_dtype(self): with self.assertRaisesRegexp( ValueError, r'dtype.*and vocabulary dtype.*do not match'): - fc.categorical_column_with_vocabulary_list( - key='aaa', vocabulary_list=('omar', 'stringer', 'marlo'), + fc._categorical_column_with_vocabulary_list( + key='aaa', + vocabulary_list=('omar', 'stringer', 'marlo'), dtype=dtypes.int32) def test_mismatched_string_dtype(self): with self.assertRaisesRegexp( ValueError, r'dtype.*and vocabulary dtype.*do not match'): - fc.categorical_column_with_vocabulary_list( + fc._categorical_column_with_vocabulary_list( key='aaa', vocabulary_list=(12, 24, 36), dtype=dtypes.string) def test_none_mapping(self): with self.assertRaisesRegexp( ValueError, r'vocabulary_list.*must be non-empty'): - fc.categorical_column_with_vocabulary_list( + fc._categorical_column_with_vocabulary_list( key='aaa', vocabulary_list=None) def test_empty_mapping(self): with self.assertRaisesRegexp( ValueError, r'vocabulary_list.*must be non-empty'): - fc.categorical_column_with_vocabulary_list( + fc._categorical_column_with_vocabulary_list( key='aaa', vocabulary_list=tuple([])) def test_duplicate_mapping(self): with self.assertRaisesRegexp(ValueError, 'Duplicate keys'): - fc.categorical_column_with_vocabulary_list( + fc._categorical_column_with_vocabulary_list( key='aaa', vocabulary_list=(12, 24, 12)) def test_invalid_num_oov_buckets(self): with self.assertRaisesRegexp(ValueError, 'Invalid num_oov_buckets'): - fc.categorical_column_with_vocabulary_list( - key='aaa', vocabulary_list=(12, 24, 36), - num_oov_buckets=-1) + fc._categorical_column_with_vocabulary_list( + key='aaa', vocabulary_list=(12, 24, 36), num_oov_buckets=-1) def test_invalid_buckets_and_default_value(self): with self.assertRaisesRegexp( ValueError, 'both num_oov_buckets and default_value'): - fc.categorical_column_with_vocabulary_list( + fc._categorical_column_with_vocabulary_list( key='aaa', vocabulary_list=(12, 24, 36), num_oov_buckets=100, default_value=2) def test_invalid_input_dtype_int32(self): - column = fc.categorical_column_with_vocabulary_list( - key='aaa', - vocabulary_list=('omar', 'stringer', 'marlo')) + column = fc._categorical_column_with_vocabulary_list( + key='aaa', vocabulary_list=('omar', 'stringer', 'marlo')) inputs = sparse_tensor.SparseTensorValue( indices=((0, 0), (1, 0), (1, 1)), values=(12, 24, 36), @@ -3858,9 +3955,8 @@ class VocabularyListCategoricalColumnTest(test.TestCase): column._get_sparse_tensors(_LazyBuilder({'aaa': inputs})) def test_invalid_input_dtype_string(self): - column = fc.categorical_column_with_vocabulary_list( - key='aaa', - vocabulary_list=(12, 24, 36)) + column = fc._categorical_column_with_vocabulary_list( + key='aaa', vocabulary_list=(12, 24, 36)) inputs = sparse_tensor.SparseTensorValue( indices=((0, 0), (1, 0), (1, 1)), values=('omar', 'stringer', 'marlo'), @@ -3868,8 +3964,9 @@ class VocabularyListCategoricalColumnTest(test.TestCase): with self.assertRaisesRegexp(ValueError, 'dtype must be compatible'): column._get_sparse_tensors(_LazyBuilder({'aaa': inputs})) + @test_util.run_deprecated_v1 def test_parse_example_string(self): - a = fc.categorical_column_with_vocabulary_list( + a = fc._categorical_column_with_vocabulary_list( key='aaa', vocabulary_list=('omar', 'stringer', 'marlo')) data = example_pb2.Example(features=feature_pb2.Features( feature={ @@ -3890,8 +3987,9 @@ class VocabularyListCategoricalColumnTest(test.TestCase): dense_shape=[1, 2]), features['aaa'].eval()) + @test_util.run_deprecated_v1 def test_parse_example_int(self): - a = fc.categorical_column_with_vocabulary_list( + a = fc._categorical_column_with_vocabulary_list( key='aaa', vocabulary_list=(11, 21, 31)) data = example_pb2.Example(features=feature_pb2.Features( feature={ @@ -3912,10 +4010,10 @@ class VocabularyListCategoricalColumnTest(test.TestCase): dense_shape=[1, 2]), features['aaa'].eval()) + @test_util.run_deprecated_v1 def test_get_sparse_tensors(self): - column = fc.categorical_column_with_vocabulary_list( - key='aaa', - vocabulary_list=('omar', 'stringer', 'marlo')) + column = fc._categorical_column_with_vocabulary_list( + key='aaa', vocabulary_list=('omar', 'stringer', 'marlo')) inputs = sparse_tensor.SparseTensorValue( indices=((0, 0), (1, 0), (1, 1)), values=('marlo', 'skywalker', 'omar'), @@ -3931,10 +4029,10 @@ class VocabularyListCategoricalColumnTest(test.TestCase): dense_shape=inputs.dense_shape), id_weight_pair.id_tensor.eval()) + @test_util.run_deprecated_v1 def test_transform_feature(self): - column = fc.categorical_column_with_vocabulary_list( - key='aaa', - vocabulary_list=('omar', 'stringer', 'marlo')) + column = fc._categorical_column_with_vocabulary_list( + key='aaa', vocabulary_list=('omar', 'stringer', 'marlo')) inputs = sparse_tensor.SparseTensorValue( indices=((0, 0), (1, 0), (1, 1)), values=('marlo', 'skywalker', 'omar'), @@ -3946,13 +4044,11 @@ class VocabularyListCategoricalColumnTest(test.TestCase): sparse_tensor.SparseTensorValue( indices=inputs.indices, values=np.array((2, -1, 0), dtype=np.int64), - dense_shape=inputs.dense_shape), - id_tensor.eval()) + dense_shape=inputs.dense_shape), self.evaluate(id_tensor)) def test_get_sparse_tensors_weight_collections(self): - column = fc.categorical_column_with_vocabulary_list( - key='aaa', - vocabulary_list=('omar', 'stringer', 'marlo')) + column = fc._categorical_column_with_vocabulary_list( + key='aaa', vocabulary_list=('omar', 'stringer', 'marlo')) inputs = sparse_tensor.SparseTensor( values=['omar', 'stringer', 'marlo'], indices=[[0, 0], [1, 0], [1, 1]], @@ -3966,10 +4062,10 @@ class VocabularyListCategoricalColumnTest(test.TestCase): [], ops.get_collection(ops.GraphKeys.GLOBAL_VARIABLES)) self.assertItemsEqual([], ops.get_collection('my_weights')) + @test_util.run_deprecated_v1 def test_get_sparse_tensors_dense_input(self): - column = fc.categorical_column_with_vocabulary_list( - key='aaa', - vocabulary_list=('omar', 'stringer', 'marlo')) + column = fc._categorical_column_with_vocabulary_list( + key='aaa', vocabulary_list=('omar', 'stringer', 'marlo')) id_weight_pair = column._get_sparse_tensors( _LazyBuilder({ 'aaa': (('marlo', ''), ('skywalker', 'omar')) @@ -3984,8 +4080,9 @@ class VocabularyListCategoricalColumnTest(test.TestCase): dense_shape=(2, 2)), id_weight_pair.id_tensor.eval()) + @test_util.run_deprecated_v1 def test_get_sparse_tensors_default_value_in_vocabulary(self): - column = fc.categorical_column_with_vocabulary_list( + column = fc._categorical_column_with_vocabulary_list( key='aaa', vocabulary_list=('omar', 'stringer', 'marlo'), default_value=2) @@ -4004,8 +4101,9 @@ class VocabularyListCategoricalColumnTest(test.TestCase): dense_shape=inputs.dense_shape), id_weight_pair.id_tensor.eval()) + @test_util.run_deprecated_v1 def test_get_sparse_tensors_with_oov_buckets(self): - column = fc.categorical_column_with_vocabulary_list( + column = fc._categorical_column_with_vocabulary_list( key='aaa', vocabulary_list=('omar', 'stringer', 'marlo'), num_oov_buckets=100) @@ -4024,8 +4122,9 @@ class VocabularyListCategoricalColumnTest(test.TestCase): dense_shape=inputs.dense_shape), id_weight_pair.id_tensor.eval()) + @test_util.run_deprecated_v1 def test_get_sparse_tensors_int32(self): - column = fc.categorical_column_with_vocabulary_list( + column = fc._categorical_column_with_vocabulary_list( key='aaa', vocabulary_list=np.array((30, 35, 11, 23, 22), dtype=np.int32), dtype=dtypes.int32) @@ -4044,9 +4143,10 @@ class VocabularyListCategoricalColumnTest(test.TestCase): dense_shape=inputs.dense_shape), id_weight_pair.id_tensor.eval()) + @test_util.run_deprecated_v1 def test_get_sparse_tensors_int32_dense_input(self): default_value = -100 - column = fc.categorical_column_with_vocabulary_list( + column = fc._categorical_column_with_vocabulary_list( key='aaa', vocabulary_list=np.array((30, 35, 11, 23, 22), dtype=np.int32), dtype=dtypes.int32, @@ -4067,8 +4167,9 @@ class VocabularyListCategoricalColumnTest(test.TestCase): dense_shape=(3, 3)), id_weight_pair.id_tensor.eval()) + @test_util.run_deprecated_v1 def test_get_sparse_tensors_int32_with_oov_buckets(self): - column = fc.categorical_column_with_vocabulary_list( + column = fc._categorical_column_with_vocabulary_list( key='aaa', vocabulary_list=np.array((30, 35, 11, 23, 22), dtype=np.int32), dtype=dtypes.int32, @@ -4088,8 +4189,9 @@ class VocabularyListCategoricalColumnTest(test.TestCase): dense_shape=inputs.dense_shape), id_weight_pair.id_tensor.eval()) + @test_util.run_deprecated_v1 def test_linear_model(self): - wire_column = fc.categorical_column_with_vocabulary_list( + wire_column = fc._categorical_column_with_vocabulary_list( key='aaa', vocabulary_list=('omar', 'stringer', 'marlo'), num_oov_buckets=1) @@ -4104,16 +4206,18 @@ class VocabularyListCategoricalColumnTest(test.TestCase): bias = get_linear_model_bias() wire_var = get_linear_model_column_var(wire_column) with _initialized_session(): - self.assertAllClose((0.,), bias.eval()) - self.assertAllClose(((0.,), (0.,), (0.,), (0.,)), wire_var.eval()) - self.assertAllClose(((0.,), (0.,)), predictions.eval()) + self.assertAllClose((0.,), self.evaluate(bias)) + self.assertAllClose(((0.,), (0.,), (0.,), (0.,)), + self.evaluate(wire_var)) + self.assertAllClose(((0.,), (0.,)), self.evaluate(predictions)) wire_var.assign(((1.,), (2.,), (3.,), (4.,))).eval() # 'marlo' -> 2: wire_var[2] = 3 # 'skywalker' -> 3, 'omar' -> 0: wire_var[3] + wire_var[0] = 4+1 = 5 - self.assertAllClose(((3.,), (5.,)), predictions.eval()) + self.assertAllClose(((3.,), (5.,)), self.evaluate(predictions)) + @test_util.run_deprecated_v1 def test_keras_linear_model(self): - wire_column = fc.categorical_column_with_vocabulary_list( + wire_column = fc._categorical_column_with_vocabulary_list( key='aaa', vocabulary_list=('omar', 'stringer', 'marlo'), num_oov_buckets=1) @@ -4129,19 +4233,20 @@ class VocabularyListCategoricalColumnTest(test.TestCase): bias = get_linear_model_bias() wire_var = get_linear_model_column_var(wire_column) with _initialized_session(): - self.assertAllClose((0.,), bias.eval()) - self.assertAllClose(((0.,), (0.,), (0.,), (0.,)), wire_var.eval()) - self.assertAllClose(((0.,), (0.,)), predictions.eval()) + self.assertAllClose((0.,), self.evaluate(bias)) + self.assertAllClose(((0.,), (0.,), (0.,), (0.,)), + self.evaluate(wire_var)) + self.assertAllClose(((0.,), (0.,)), self.evaluate(predictions)) wire_var.assign(((1.,), (2.,), (3.,), (4.,))).eval() # 'marlo' -> 2: wire_var[2] = 3 # 'skywalker' -> 3, 'omar' -> 0: wire_var[3] + wire_var[0] = 4+1 = 5 - self.assertAllClose(((3.,), (5.,)), predictions.eval()) + self.assertAllClose(((3.,), (5.,)), self.evaluate(predictions)) class IdentityCategoricalColumnTest(test.TestCase): def test_constructor(self): - column = fc.categorical_column_with_identity(key='aaa', num_buckets=3) + column = fc._categorical_column_with_identity(key='aaa', num_buckets=3) self.assertEqual('aaa', column.name) self.assertEqual('aaa', column.key) self.assertEqual('aaa', column._var_scope_name) @@ -4152,10 +4257,11 @@ class IdentityCategoricalColumnTest(test.TestCase): def test_key_should_be_string(self): with self.assertRaisesRegexp(ValueError, 'key must be a string.'): - fc.categorical_column_with_identity(key=('aaa',), num_buckets=3) + fc._categorical_column_with_identity(key=('aaa',), num_buckets=3) + @test_util.run_deprecated_v1 def test_deep_copy(self): - original = fc.categorical_column_with_identity(key='aaa', num_buckets=3) + original = fc._categorical_column_with_identity(key='aaa', num_buckets=3) for column in (original, copy.deepcopy(original)): self.assertEqual('aaa', column.name) self.assertEqual(3, column._num_buckets) @@ -4165,24 +4271,24 @@ class IdentityCategoricalColumnTest(test.TestCase): def test_invalid_num_buckets_zero(self): with self.assertRaisesRegexp(ValueError, 'num_buckets 0 < 1'): - fc.categorical_column_with_identity(key='aaa', num_buckets=0) + fc._categorical_column_with_identity(key='aaa', num_buckets=0) def test_invalid_num_buckets_negative(self): with self.assertRaisesRegexp(ValueError, 'num_buckets -1 < 1'): - fc.categorical_column_with_identity(key='aaa', num_buckets=-1) + fc._categorical_column_with_identity(key='aaa', num_buckets=-1) def test_invalid_default_value_too_small(self): with self.assertRaisesRegexp(ValueError, 'default_value -1 not in range'): - fc.categorical_column_with_identity( + fc._categorical_column_with_identity( key='aaa', num_buckets=3, default_value=-1) def test_invalid_default_value_too_big(self): with self.assertRaisesRegexp(ValueError, 'default_value 3 not in range'): - fc.categorical_column_with_identity( + fc._categorical_column_with_identity( key='aaa', num_buckets=3, default_value=3) def test_invalid_input_dtype(self): - column = fc.categorical_column_with_identity(key='aaa', num_buckets=3) + column = fc._categorical_column_with_identity(key='aaa', num_buckets=3) inputs = sparse_tensor.SparseTensorValue( indices=((0, 0), (1, 0), (1, 1)), values=('omar', 'stringer', 'marlo'), @@ -4190,8 +4296,9 @@ class IdentityCategoricalColumnTest(test.TestCase): with self.assertRaisesRegexp(ValueError, 'Invalid input, not integer'): column._get_sparse_tensors(_LazyBuilder({'aaa': inputs})) + @test_util.run_deprecated_v1 def test_parse_example(self): - a = fc.categorical_column_with_identity(key='aaa', num_buckets=30) + a = fc._categorical_column_with_identity(key='aaa', num_buckets=30) data = example_pb2.Example(features=feature_pb2.Features( feature={ 'aaa': @@ -4211,8 +4318,9 @@ class IdentityCategoricalColumnTest(test.TestCase): dense_shape=[1, 2]), features['aaa'].eval()) + @test_util.run_deprecated_v1 def test_get_sparse_tensors(self): - column = fc.categorical_column_with_identity(key='aaa', num_buckets=3) + column = fc._categorical_column_with_identity(key='aaa', num_buckets=3) inputs = sparse_tensor.SparseTensorValue( indices=((0, 0), (1, 0), (1, 1)), values=(0, 1, 0), @@ -4228,8 +4336,9 @@ class IdentityCategoricalColumnTest(test.TestCase): dense_shape=inputs.dense_shape), id_weight_pair.id_tensor.eval()) + @test_util.run_deprecated_v1 def test_transform_feature(self): - column = fc.categorical_column_with_identity(key='aaa', num_buckets=3) + column = fc._categorical_column_with_identity(key='aaa', num_buckets=3) inputs = sparse_tensor.SparseTensorValue( indices=((0, 0), (1, 0), (1, 1)), values=(0, 1, 0), @@ -4241,11 +4350,10 @@ class IdentityCategoricalColumnTest(test.TestCase): sparse_tensor.SparseTensorValue( indices=inputs.indices, values=np.array((0, 1, 0), dtype=np.int64), - dense_shape=inputs.dense_shape), - id_tensor.eval()) + dense_shape=inputs.dense_shape), self.evaluate(id_tensor)) def test_get_sparse_tensors_weight_collections(self): - column = fc.categorical_column_with_identity(key='aaa', num_buckets=3) + column = fc._categorical_column_with_identity(key='aaa', num_buckets=3) inputs = sparse_tensor.SparseTensorValue( indices=((0, 0), (1, 0), (1, 1)), values=(0, 1, 0), @@ -4259,8 +4367,9 @@ class IdentityCategoricalColumnTest(test.TestCase): [], ops.get_collection(ops.GraphKeys.GLOBAL_VARIABLES)) self.assertItemsEqual([], ops.get_collection('my_weights')) + @test_util.run_deprecated_v1 def test_get_sparse_tensors_dense_input(self): - column = fc.categorical_column_with_identity(key='aaa', num_buckets=3) + column = fc._categorical_column_with_identity(key='aaa', num_buckets=3) id_weight_pair = column._get_sparse_tensors( _LazyBuilder({ 'aaa': ((0, -1), (1, 0)) @@ -4275,8 +4384,9 @@ class IdentityCategoricalColumnTest(test.TestCase): dense_shape=(2, 2)), id_weight_pair.id_tensor.eval()) + @test_util.run_deprecated_v1 def test_get_sparse_tensors_with_inputs_too_small(self): - column = fc.categorical_column_with_identity(key='aaa', num_buckets=3) + column = fc._categorical_column_with_identity(key='aaa', num_buckets=3) inputs = sparse_tensor.SparseTensorValue( indices=((0, 0), (1, 0), (1, 1)), values=(1, -1, 0), @@ -4288,8 +4398,9 @@ class IdentityCategoricalColumnTest(test.TestCase): errors.OpError, 'assert_greater_or_equal_0'): id_weight_pair.id_tensor.eval() + @test_util.run_deprecated_v1 def test_get_sparse_tensors_with_inputs_too_big(self): - column = fc.categorical_column_with_identity(key='aaa', num_buckets=3) + column = fc._categorical_column_with_identity(key='aaa', num_buckets=3) inputs = sparse_tensor.SparseTensorValue( indices=((0, 0), (1, 0), (1, 1)), values=(1, 99, 0), @@ -4301,8 +4412,9 @@ class IdentityCategoricalColumnTest(test.TestCase): errors.OpError, 'assert_less_than_num_buckets'): id_weight_pair.id_tensor.eval() + @test_util.run_deprecated_v1 def test_get_sparse_tensors_with_default_value(self): - column = fc.categorical_column_with_identity( + column = fc._categorical_column_with_identity( key='aaa', num_buckets=4, default_value=3) inputs = sparse_tensor.SparseTensorValue( indices=((0, 0), (1, 0), (1, 1)), @@ -4319,8 +4431,9 @@ class IdentityCategoricalColumnTest(test.TestCase): dense_shape=inputs.dense_shape), id_weight_pair.id_tensor.eval()) + @test_util.run_deprecated_v1 def test_get_sparse_tensors_with_default_value_and_placeholder_inputs(self): - column = fc.categorical_column_with_identity( + column = fc._categorical_column_with_identity( key='aaa', num_buckets=4, default_value=3) input_indices = array_ops.placeholder(dtype=dtypes.int64) input_values = array_ops.placeholder(dtype=dtypes.int32) @@ -4344,8 +4457,9 @@ class IdentityCategoricalColumnTest(test.TestCase): input_shape: (2, 2), })) + @test_util.run_deprecated_v1 def test_linear_model(self): - column = fc.categorical_column_with_identity(key='aaa', num_buckets=3) + column = fc._categorical_column_with_identity(key='aaa', num_buckets=3) self.assertEqual(3, column._num_buckets) with ops.Graph().as_default(): predictions = fc.linear_model({ @@ -4357,16 +4471,17 @@ class IdentityCategoricalColumnTest(test.TestCase): bias = get_linear_model_bias() weight_var = get_linear_model_column_var(column) with _initialized_session(): - self.assertAllClose((0.,), bias.eval()) - self.assertAllClose(((0.,), (0.,), (0.,)), weight_var.eval()) - self.assertAllClose(((0.,), (0.,)), predictions.eval()) + self.assertAllClose((0.,), self.evaluate(bias)) + self.assertAllClose(((0.,), (0.,), (0.,)), self.evaluate(weight_var)) + self.assertAllClose(((0.,), (0.,)), self.evaluate(predictions)) weight_var.assign(((1.,), (2.,), (3.,))).eval() # weight_var[0] = 1 # weight_var[2] + weight_var[1] = 3+2 = 5 - self.assertAllClose(((1.,), (5.,)), predictions.eval()) + self.assertAllClose(((1.,), (5.,)), self.evaluate(predictions)) + @test_util.run_deprecated_v1 def test_keras_linear_model(self): - column = fc.categorical_column_with_identity(key='aaa', num_buckets=3) + column = fc._categorical_column_with_identity(key='aaa', num_buckets=3) self.assertEqual(3, column._num_buckets) with ops.Graph().as_default(): predictions = get_keras_linear_model_predictions({ @@ -4379,13 +4494,13 @@ class IdentityCategoricalColumnTest(test.TestCase): bias = get_linear_model_bias() weight_var = get_linear_model_column_var(column) with _initialized_session(): - self.assertAllClose((0.,), bias.eval()) - self.assertAllClose(((0.,), (0.,), (0.,)), weight_var.eval()) - self.assertAllClose(((0.,), (0.,)), predictions.eval()) + self.assertAllClose((0.,), self.evaluate(bias)) + self.assertAllClose(((0.,), (0.,), (0.,)), self.evaluate(weight_var)) + self.assertAllClose(((0.,), (0.,)), self.evaluate(predictions)) weight_var.assign(((1.,), (2.,), (3.,))).eval() # weight_var[0] = 1 # weight_var[2] + weight_var[1] = 3+2 = 5 - self.assertAllClose(((1.,), (5.,)), predictions.eval()) + self.assertAllClose(((1.,), (5.,)), self.evaluate(predictions)) class TransformFeaturesTest(test.TestCase): @@ -4393,9 +4508,9 @@ class TransformFeaturesTest(test.TestCase): # All transform tests are distributed in column test. # Here we only test multi column case and naming def transform_multi_column(self): - bucketized_price = fc.bucketized_column( - fc.numeric_column('price'), boundaries=[0, 2, 4, 6]) - hashed_sparse = fc.categorical_column_with_hash_bucket('wire', 10) + bucketized_price = fc._bucketized_column( + fc._numeric_column('price'), boundaries=[0, 2, 4, 6]) + hashed_sparse = fc._categorical_column_with_hash_bucket('wire', 10) with ops.Graph().as_default(): features = { 'price': [[-1.], [5.]], @@ -4452,32 +4567,33 @@ class TransformFeaturesTest(test.TestCase): class IndicatorColumnTest(test.TestCase): def test_indicator_column(self): - a = fc.categorical_column_with_hash_bucket('a', 4) - indicator_a = fc.indicator_column(a) + a = fc._categorical_column_with_hash_bucket('a', 4) + indicator_a = fc._indicator_column(a) self.assertEqual(indicator_a.categorical_column.name, 'a') self.assertEqual(indicator_a.name, 'a_indicator') self.assertEqual(indicator_a._var_scope_name, 'a_indicator') self.assertEqual(indicator_a._variable_shape, [1, 4]) - b = fc.categorical_column_with_hash_bucket('b', hash_bucket_size=100) - indicator_b = fc.indicator_column(b) + b = fc._categorical_column_with_hash_bucket('b', hash_bucket_size=100) + indicator_b = fc._indicator_column(b) self.assertEqual(indicator_b.categorical_column.name, 'b') self.assertEqual(indicator_b.name, 'b_indicator') self.assertEqual(indicator_b._var_scope_name, 'b_indicator') self.assertEqual(indicator_b._variable_shape, [1, 100]) def test_1D_shape_succeeds(self): - animal = fc.indicator_column( - fc.categorical_column_with_hash_bucket('animal', 4)) + animal = fc._indicator_column( + fc._categorical_column_with_hash_bucket('animal', 4)) builder = _LazyBuilder({'animal': ['fox', 'fox']}) output = builder.get(animal) with self.cached_session(): - self.assertAllEqual([[0., 0., 1., 0.], [0., 0., 1., 0.]], output.eval()) + self.assertAllEqual([[0., 0., 1., 0.], [0., 0., 1., 0.]], + self.evaluate(output)) def test_2D_shape_succeeds(self): # TODO(ispir/cassandrax): Swith to categorical_column_with_keys when ready. - animal = fc.indicator_column( - fc.categorical_column_with_hash_bucket('animal', 4)) + animal = fc._indicator_column( + fc._categorical_column_with_hash_bucket('animal', 4)) builder = _LazyBuilder({ 'animal': sparse_tensor.SparseTensor( @@ -4487,11 +4603,12 @@ class IndicatorColumnTest(test.TestCase): }) output = builder.get(animal) with self.cached_session(): - self.assertAllEqual([[0., 0., 1., 0.], [0., 0., 1., 0.]], output.eval()) + self.assertAllEqual([[0., 0., 1., 0.], [0., 0., 1., 0.]], + self.evaluate(output)) def test_multi_hot(self): - animal = fc.indicator_column( - fc.categorical_column_with_identity('animal', num_buckets=4)) + animal = fc._indicator_column( + fc._categorical_column_with_identity('animal', num_buckets=4)) builder = _LazyBuilder({ 'animal': @@ -4500,11 +4617,11 @@ class IndicatorColumnTest(test.TestCase): }) output = builder.get(animal) with self.cached_session(): - self.assertAllEqual([[0., 2., 0., 0.]], output.eval()) + self.assertAllEqual([[0., 2., 0., 0.]], self.evaluate(output)) def test_multi_hot2(self): - animal = fc.indicator_column( - fc.categorical_column_with_identity('animal', num_buckets=4)) + animal = fc._indicator_column( + fc._categorical_column_with_identity('animal', num_buckets=4)) builder = _LazyBuilder({ 'animal': sparse_tensor.SparseTensor( @@ -4512,20 +4629,22 @@ class IndicatorColumnTest(test.TestCase): }) output = builder.get(animal) with self.cached_session(): - self.assertAllEqual([[0., 1., 1., 0.]], output.eval()) + self.assertAllEqual([[0., 1., 1., 0.]], self.evaluate(output)) + @test_util.run_deprecated_v1 def test_deep_copy(self): - a = fc.categorical_column_with_hash_bucket('a', 4) - column = fc.indicator_column(a) + a = fc._categorical_column_with_hash_bucket('a', 4) + column = fc._indicator_column(a) column_copy = copy.deepcopy(column) self.assertEqual(column_copy.categorical_column.name, 'a') self.assertEqual(column.name, 'a_indicator') self.assertEqual(column._variable_shape, [1, 4]) + @test_util.run_deprecated_v1 def test_parse_example(self): - a = fc.categorical_column_with_vocabulary_list( + a = fc._categorical_column_with_vocabulary_list( key='aaa', vocabulary_list=('omar', 'stringer', 'marlo')) - a_indicator = fc.indicator_column(a) + a_indicator = fc._indicator_column(a) data = example_pb2.Example(features=feature_pb2.Features( feature={ 'aaa': @@ -4545,10 +4664,11 @@ class IndicatorColumnTest(test.TestCase): dense_shape=[1, 2]), features['aaa'].eval()) + @test_util.run_deprecated_v1 def test_transform(self): - a = fc.categorical_column_with_vocabulary_list( + a = fc._categorical_column_with_vocabulary_list( key='aaa', vocabulary_list=('omar', 'stringer', 'marlo')) - a_indicator = fc.indicator_column(a) + a_indicator = fc._indicator_column(a) features = { 'aaa': sparse_tensor.SparseTensorValue( indices=((0, 0), (1, 0), (1, 1)), @@ -4557,51 +4677,56 @@ class IndicatorColumnTest(test.TestCase): } indicator_tensor = _transform_features(features, [a_indicator])[a_indicator] with _initialized_session(): - self.assertAllEqual([[0, 0, 1], [1, 0, 0]], indicator_tensor.eval()) + self.assertAllEqual([[0, 0, 1], [1, 0, 0]], + self.evaluate(indicator_tensor)) + @test_util.run_deprecated_v1 def test_transform_with_weighted_column(self): # Github issue 12557 - ids = fc.categorical_column_with_vocabulary_list( + ids = fc._categorical_column_with_vocabulary_list( key='ids', vocabulary_list=('a', 'b', 'c')) - weights = fc.weighted_categorical_column(ids, 'weights') - indicator = fc.indicator_column(weights) + weights = fc._weighted_categorical_column(ids, 'weights') + indicator = fc._indicator_column(weights) features = { 'ids': constant_op.constant([['c', 'b', 'a', 'c']]), 'weights': constant_op.constant([[2., 4., 6., 1.]]) } indicator_tensor = _transform_features(features, [indicator])[indicator] with _initialized_session(): - self.assertAllEqual([[6., 4., 3.]], indicator_tensor.eval()) + self.assertAllEqual([[6., 4., 3.]], self.evaluate(indicator_tensor)) + @test_util.run_deprecated_v1 def test_transform_with_missing_value_in_weighted_column(self): # Github issue 12583 - ids = fc.categorical_column_with_vocabulary_list( + ids = fc._categorical_column_with_vocabulary_list( key='ids', vocabulary_list=('a', 'b', 'c')) - weights = fc.weighted_categorical_column(ids, 'weights') - indicator = fc.indicator_column(weights) + weights = fc._weighted_categorical_column(ids, 'weights') + indicator = fc._indicator_column(weights) features = { 'ids': constant_op.constant([['c', 'b', 'unknown']]), 'weights': constant_op.constant([[2., 4., 6.]]) } indicator_tensor = _transform_features(features, [indicator])[indicator] with _initialized_session(): - self.assertAllEqual([[0., 4., 2.]], indicator_tensor.eval()) + self.assertAllEqual([[0., 4., 2.]], self.evaluate(indicator_tensor)) + @test_util.run_deprecated_v1 def test_transform_with_missing_value_in_categorical_column(self): # Github issue 12583 - ids = fc.categorical_column_with_vocabulary_list( + ids = fc._categorical_column_with_vocabulary_list( key='ids', vocabulary_list=('a', 'b', 'c')) - indicator = fc.indicator_column(ids) + indicator = fc._indicator_column(ids) features = { 'ids': constant_op.constant([['c', 'b', 'unknown']]), } indicator_tensor = _transform_features(features, [indicator])[indicator] with _initialized_session(): - self.assertAllEqual([[0., 1., 1.]], indicator_tensor.eval()) + self.assertAllEqual([[0., 1., 1.]], self.evaluate(indicator_tensor)) + @test_util.run_deprecated_v1 def test_linear_model(self): - animal = fc.indicator_column( - fc.categorical_column_with_identity('animal', num_buckets=4)) + animal = fc._indicator_column( + fc._categorical_column_with_identity('animal', num_buckets=4)) with ops.Graph().as_default(): features = { 'animal': @@ -4613,14 +4738,15 @@ class IndicatorColumnTest(test.TestCase): weight_var = get_linear_model_column_var(animal) with _initialized_session(): # All should be zero-initialized. - self.assertAllClose([[0.], [0.], [0.], [0.]], weight_var.eval()) - self.assertAllClose([[0.]], predictions.eval()) + self.assertAllClose([[0.], [0.], [0.], [0.]], self.evaluate(weight_var)) + self.assertAllClose([[0.]], self.evaluate(predictions)) weight_var.assign([[1.], [2.], [3.], [4.]]).eval() - self.assertAllClose([[2. + 3.]], predictions.eval()) + self.assertAllClose([[2. + 3.]], self.evaluate(predictions)) + @test_util.run_deprecated_v1 def test_keras_linear_model(self): - animal = fc.indicator_column( - fc.categorical_column_with_identity('animal', num_buckets=4)) + animal = fc._indicator_column( + fc._categorical_column_with_identity('animal', num_buckets=4)) with ops.Graph().as_default(): features = { 'animal': @@ -4632,14 +4758,15 @@ class IndicatorColumnTest(test.TestCase): weight_var = get_linear_model_column_var(animal) with _initialized_session(): # All should be zero-initialized. - self.assertAllClose([[0.], [0.], [0.], [0.]], weight_var.eval()) - self.assertAllClose([[0.]], predictions.eval()) + self.assertAllClose([[0.], [0.], [0.], [0.]], self.evaluate(weight_var)) + self.assertAllClose([[0.]], self.evaluate(predictions)) weight_var.assign([[1.], [2.], [3.], [4.]]).eval() - self.assertAllClose([[2. + 3.]], predictions.eval()) + self.assertAllClose([[2. + 3.]], self.evaluate(predictions)) + @test_util.run_deprecated_v1 def test_input_layer(self): - animal = fc.indicator_column( - fc.categorical_column_with_identity('animal', num_buckets=4)) + animal = fc._indicator_column( + fc._categorical_column_with_identity('animal', num_buckets=4)) with ops.Graph().as_default(): features = { 'animal': @@ -4648,16 +4775,17 @@ class IndicatorColumnTest(test.TestCase): } net = fc.input_layer(features, [animal]) with _initialized_session(): - self.assertAllClose([[0., 1., 1., 0.]], net.eval()) + self.assertAllClose([[0., 1., 1., 0.]], self.evaluate(net)) class EmbeddingColumnTest(test.TestCase): + @test_util.run_deprecated_v1 def test_defaults(self): - categorical_column = fc.categorical_column_with_identity( + categorical_column = fc._categorical_column_with_identity( key='aaa', num_buckets=3) embedding_dimension = 2 - embedding_column = fc.embedding_column( + embedding_column = fc._embedding_column( categorical_column, dimension=embedding_dimension) self.assertIs(categorical_column, embedding_column.categorical_column) self.assertEqual(embedding_dimension, embedding_column.dimension) @@ -4674,15 +4802,20 @@ class EmbeddingColumnTest(test.TestCase): 'aaa': parsing_ops.VarLenFeature(dtypes.int64) }, embedding_column._parse_example_spec) + @test_util.run_deprecated_v1 def test_all_constructor_args(self): - categorical_column = fc.categorical_column_with_identity( + categorical_column = fc._categorical_column_with_identity( key='aaa', num_buckets=3) embedding_dimension = 2 - embedding_column = fc.embedding_column( - categorical_column, dimension=embedding_dimension, - combiner='my_combiner', initializer=lambda: 'my_initializer', - ckpt_to_load_from='my_ckpt', tensor_name_in_ckpt='my_ckpt_tensor', - max_norm=42., trainable=False) + embedding_column = fc._embedding_column( + categorical_column, + dimension=embedding_dimension, + combiner='my_combiner', + initializer=lambda: 'my_initializer', + ckpt_to_load_from='my_ckpt', + tensor_name_in_ckpt='my_ckpt_tensor', + max_norm=42., + trainable=False) self.assertIs(categorical_column, embedding_column.categorical_column) self.assertEqual(embedding_dimension, embedding_column.dimension) self.assertEqual('my_combiner', embedding_column.combiner) @@ -4698,15 +4831,20 @@ class EmbeddingColumnTest(test.TestCase): 'aaa': parsing_ops.VarLenFeature(dtypes.int64) }, embedding_column._parse_example_spec) + @test_util.run_deprecated_v1 def test_deep_copy(self): - categorical_column = fc.categorical_column_with_identity( + categorical_column = fc._categorical_column_with_identity( key='aaa', num_buckets=3) embedding_dimension = 2 - original = fc.embedding_column( - categorical_column, dimension=embedding_dimension, - combiner='my_combiner', initializer=lambda: 'my_initializer', - ckpt_to_load_from='my_ckpt', tensor_name_in_ckpt='my_ckpt_tensor', - max_norm=42., trainable=False) + original = fc._embedding_column( + categorical_column, + dimension=embedding_dimension, + combiner='my_combiner', + initializer=lambda: 'my_initializer', + ckpt_to_load_from='my_ckpt', + tensor_name_in_ckpt='my_ckpt_tensor', + max_norm=42., + trainable=False) for embedding_column in (original, copy.deepcopy(original)): self.assertEqual('aaa', embedding_column.categorical_column.name) self.assertEqual(3, embedding_column.categorical_column._num_buckets) @@ -4727,16 +4865,19 @@ class EmbeddingColumnTest(test.TestCase): 'aaa': parsing_ops.VarLenFeature(dtypes.int64) }, embedding_column._parse_example_spec) + @test_util.run_deprecated_v1 def test_invalid_initializer(self): - categorical_column = fc.categorical_column_with_identity( + categorical_column = fc._categorical_column_with_identity( key='aaa', num_buckets=3) with self.assertRaisesRegexp(ValueError, 'initializer must be callable'): - fc.embedding_column(categorical_column, dimension=2, initializer='not_fn') + fc._embedding_column( + categorical_column, dimension=2, initializer='not_fn') + @test_util.run_deprecated_v1 def test_parse_example(self): - a = fc.categorical_column_with_vocabulary_list( + a = fc._categorical_column_with_vocabulary_list( key='aaa', vocabulary_list=('omar', 'stringer', 'marlo')) - a_embedded = fc.embedding_column(a, dimension=2) + a_embedded = fc._embedding_column(a, dimension=2) data = example_pb2.Example(features=feature_pb2.Features( feature={ 'aaa': @@ -4756,9 +4897,10 @@ class EmbeddingColumnTest(test.TestCase): dense_shape=[1, 2]), features['aaa'].eval()) + @test_util.run_deprecated_v1 def test_transform_feature(self): - a = fc.categorical_column_with_identity(key='aaa', num_buckets=3) - a_embedded = fc.embedding_column(a, dimension=2) + a = fc._categorical_column_with_identity(key='aaa', num_buckets=3) + a_embedded = fc._embedding_column(a, dimension=2) features = { 'aaa': sparse_tensor.SparseTensor( indices=((0, 0), (1, 0), (1, 1)), @@ -4769,9 +4911,10 @@ class EmbeddingColumnTest(test.TestCase): output_a = outputs[a] output_embedded = outputs[a_embedded] with _initialized_session(): - _assert_sparse_tensor_value( - self, output_a.eval(), output_embedded.eval()) + _assert_sparse_tensor_value(self, self.evaluate(output_a), + self.evaluate(output_embedded)) + @test_util.run_deprecated_v1 def test_get_dense_tensor(self): # Inputs. vocabulary_size = 3 @@ -4810,10 +4953,11 @@ class EmbeddingColumnTest(test.TestCase): ) # Build columns. - categorical_column = fc.categorical_column_with_identity( + categorical_column = fc._categorical_column_with_identity( key='aaa', num_buckets=vocabulary_size) - embedding_column = fc.embedding_column( - categorical_column, dimension=embedding_dimension, + embedding_column = fc._embedding_column( + categorical_column, + dimension=embedding_dimension, initializer=_initializer) # Provide sparse input and get dense result. @@ -4828,8 +4972,9 @@ class EmbeddingColumnTest(test.TestCase): tuple([v.name for v in global_vars])) with _initialized_session(): self.assertAllEqual(embedding_values, global_vars[0].eval()) - self.assertAllEqual(expected_lookups, embedding_lookup.eval()) + self.assertAllEqual(expected_lookups, self.evaluate(embedding_lookup)) + @test_util.run_deprecated_v1 def test_get_dense_tensor_3d(self): # Inputs. vocabulary_size = 4 @@ -4870,10 +5015,11 @@ class EmbeddingColumnTest(test.TestCase): ) # Build columns. - categorical_column = fc.categorical_column_with_identity( + categorical_column = fc._categorical_column_with_identity( key='aaa', num_buckets=vocabulary_size) - embedding_column = fc.embedding_column( - categorical_column, dimension=embedding_dimension, + embedding_column = fc._embedding_column( + categorical_column, + dimension=embedding_dimension, initializer=_initializer) # Provide sparse input and get dense result. @@ -4888,8 +5034,9 @@ class EmbeddingColumnTest(test.TestCase): tuple([v.name for v in global_vars])) with _initialized_session(): self.assertAllEqual(embedding_values, global_vars[0].eval()) - self.assertAllEqual(expected_lookups, embedding_lookup.eval()) + self.assertAllEqual(expected_lookups, self.evaluate(embedding_lookup)) + @test_util.run_deprecated_v1 def test_get_dense_tensor_weight_collections(self): sparse_input = sparse_tensor.SparseTensorValue( # example 0, ids [2] @@ -4901,9 +5048,9 @@ class EmbeddingColumnTest(test.TestCase): dense_shape=(4, 5)) # Build columns. - categorical_column = fc.categorical_column_with_identity( + categorical_column = fc._categorical_column_with_identity( key='aaa', num_buckets=3) - embedding_column = fc.embedding_column(categorical_column, dimension=2) + embedding_column = fc._embedding_column(categorical_column, dimension=2) # Provide sparse input and get dense result. embedding_column._get_dense_tensor( @@ -4919,6 +5066,7 @@ class EmbeddingColumnTest(test.TestCase): self.assertItemsEqual( ('embedding_weights:0',), tuple([v.name for v in my_vars])) + @test_util.run_deprecated_v1 def test_get_dense_tensor_placeholder_inputs(self): # Inputs. vocabulary_size = 3 @@ -4957,10 +5105,11 @@ class EmbeddingColumnTest(test.TestCase): ) # Build columns. - categorical_column = fc.categorical_column_with_identity( + categorical_column = fc._categorical_column_with_identity( key='aaa', num_buckets=vocabulary_size) - embedding_column = fc.embedding_column( - categorical_column, dimension=embedding_dimension, + embedding_column = fc._embedding_column( + categorical_column, + dimension=embedding_dimension, initializer=_initializer) # Provide sparse input and get dense result. @@ -4989,6 +5138,7 @@ class EmbeddingColumnTest(test.TestCase): input_shape: sparse_input.dense_shape, })) + @test_util.run_deprecated_v1 def test_get_dense_tensor_restore_from_ckpt(self): # Inputs. vocabulary_size = 3 @@ -5025,10 +5175,11 @@ class EmbeddingColumnTest(test.TestCase): ) # Build columns. - categorical_column = fc.categorical_column_with_identity( + categorical_column = fc._categorical_column_with_identity( key='aaa', num_buckets=vocabulary_size) - embedding_column = fc.embedding_column( - categorical_column, dimension=embedding_dimension, + embedding_column = fc._embedding_column( + categorical_column, + dimension=embedding_dimension, ckpt_to_load_from=ckpt_path, tensor_name_in_ckpt=ckpt_tensor) @@ -5044,8 +5195,9 @@ class EmbeddingColumnTest(test.TestCase): ('embedding_weights:0',), tuple([v.name for v in global_vars])) with _initialized_session(): self.assertAllEqual(embedding_values, global_vars[0].eval()) - self.assertAllEqual(expected_lookups, embedding_lookup.eval()) + self.assertAllEqual(expected_lookups, self.evaluate(embedding_lookup)) + @test_util.run_deprecated_v1 def test_linear_model(self): # Inputs. batch_size = 4 @@ -5070,10 +5222,11 @@ class EmbeddingColumnTest(test.TestCase): return zeros_embedding_values # Build columns. - categorical_column = fc.categorical_column_with_identity( + categorical_column = fc._categorical_column_with_identity( key='aaa', num_buckets=vocabulary_size) - embedding_column = fc.embedding_column( - categorical_column, dimension=embedding_dimension, + embedding_column = fc._embedding_column( + categorical_column, + dimension=embedding_dimension, initializer=_initializer) with ops.Graph().as_default(): @@ -5100,11 +5253,13 @@ class EmbeddingColumnTest(test.TestCase): 'linear_model/aaa_embedding/weights:0'] with _initialized_session(): # Predictions with all zero weights. - self.assertAllClose(np.zeros((1,)), bias.eval()) - self.assertAllClose(zeros_embedding_values, embedding_weights.eval()) + self.assertAllClose(np.zeros((1,)), self.evaluate(bias)) + self.assertAllClose(zeros_embedding_values, + self.evaluate(embedding_weights)) self.assertAllClose( - np.zeros((embedding_dimension, 1)), linear_weights.eval()) - self.assertAllClose(np.zeros((batch_size, 1)), predictions.eval()) + np.zeros((embedding_dimension, 1)), self.evaluate(linear_weights)) + self.assertAllClose( + np.zeros((batch_size, 1)), self.evaluate(predictions)) # Predictions with all non-zero weights. embedding_weights.assign(( @@ -5119,8 +5274,10 @@ class EmbeddingColumnTest(test.TestCase): # example 3, ids [1], embedding[3] = [3, 5] # sum(embeddings * linear_weights) # = [4*7 + 6*11, 4*2 + 6*3.5, 4*0 + 6*0, 4*3 + 6*5] = [94, 29, 0, 42] - self.assertAllClose(((94.,), (29.,), (0.,), (42.,)), predictions.eval()) + self.assertAllClose(((94.,), (29.,), (0.,), (42.,)), + self.evaluate(predictions)) + @test_util.run_deprecated_v1 def test_keras_linear_model(self): # Inputs. batch_size = 4 @@ -5146,9 +5303,9 @@ class EmbeddingColumnTest(test.TestCase): return zeros_embedding_values # Build columns. - categorical_column = fc.categorical_column_with_identity( + categorical_column = fc._categorical_column_with_identity( key='aaa', num_buckets=vocabulary_size) - embedding_column = fc.embedding_column( + embedding_column = fc._embedding_column( categorical_column, dimension=embedding_dimension, initializer=_initializer) @@ -5176,11 +5333,13 @@ class EmbeddingColumnTest(test.TestCase): linear_weights = trainable_vars['linear_model/aaa_embedding/weights:0'] with _initialized_session(): # Predictions with all zero weights. - self.assertAllClose(np.zeros((1,)), bias.eval()) - self.assertAllClose(zeros_embedding_values, embedding_weights.eval()) + self.assertAllClose(np.zeros((1,)), self.evaluate(bias)) + self.assertAllClose(zeros_embedding_values, + self.evaluate(embedding_weights)) self.assertAllClose( - np.zeros((embedding_dimension, 1)), linear_weights.eval()) - self.assertAllClose(np.zeros((batch_size, 1)), predictions.eval()) + np.zeros((embedding_dimension, 1)), self.evaluate(linear_weights)) + self.assertAllClose( + np.zeros((batch_size, 1)), self.evaluate(predictions)) # Predictions with all non-zero weights. embedding_weights.assign(( @@ -5195,8 +5354,10 @@ class EmbeddingColumnTest(test.TestCase): # example 3, ids [1], embedding[3] = [3, 5] # sum(embeddings * linear_weights) # = [4*7 + 6*11, 4*2 + 6*3.5, 4*0 + 6*0, 4*3 + 6*5] = [94, 29, 0, 42] - self.assertAllClose(((94.,), (29.,), (0.,), (42.,)), predictions.eval()) + self.assertAllClose(((94.,), (29.,), (0.,), (42.,)), + self.evaluate(predictions)) + @test_util.run_deprecated_v1 def test_input_layer(self): # Inputs. vocabulary_size = 3 @@ -5235,10 +5396,11 @@ class EmbeddingColumnTest(test.TestCase): ) # Build columns. - categorical_column = fc.categorical_column_with_identity( + categorical_column = fc._categorical_column_with_identity( key='aaa', num_buckets=vocabulary_size) - embedding_column = fc.embedding_column( - categorical_column, dimension=embedding_dimension, + embedding_column = fc._embedding_column( + categorical_column, + dimension=embedding_dimension, initializer=_initializer) # Provide sparse input and get dense result. @@ -5255,8 +5417,9 @@ class EmbeddingColumnTest(test.TestCase): tuple([v.name for v in trainable_vars])) with _initialized_session(): self.assertAllEqual(embedding_values, trainable_vars[0].eval()) - self.assertAllEqual(expected_lookups, input_layer.eval()) + self.assertAllEqual(expected_lookups, self.evaluate(input_layer)) + @test_util.run_deprecated_v1 def test_input_layer_not_trainable(self): # Inputs. vocabulary_size = 3 @@ -5295,11 +5458,13 @@ class EmbeddingColumnTest(test.TestCase): ) # Build columns. - categorical_column = fc.categorical_column_with_identity( + categorical_column = fc._categorical_column_with_identity( key='aaa', num_buckets=vocabulary_size) - embedding_column = fc.embedding_column( - categorical_column, dimension=embedding_dimension, - initializer=_initializer, trainable=False) + embedding_column = fc._embedding_column( + categorical_column, + dimension=embedding_dimension, + initializer=_initializer, + trainable=False) # Provide sparse input and get dense result. input_layer = fc.input_layer({'aaa': sparse_input}, (embedding_column,)) @@ -5313,18 +5478,19 @@ class EmbeddingColumnTest(test.TestCase): [], ops.get_collection(ops.GraphKeys.TRAINABLE_VARIABLES)) with _initialized_session(): self.assertAllEqual(embedding_values, global_vars[0].eval()) - self.assertAllEqual(expected_lookups, input_layer.eval()) + self.assertAllEqual(expected_lookups, self.evaluate(input_layer)) class SharedEmbeddingColumnTest(test.TestCase): + @test_util.run_deprecated_v1 def test_defaults(self): - categorical_column_a = fc.categorical_column_with_identity( + categorical_column_a = fc._categorical_column_with_identity( key='aaa', num_buckets=3) - categorical_column_b = fc.categorical_column_with_identity( + categorical_column_b = fc._categorical_column_with_identity( key='bbb', num_buckets=3) embedding_dimension = 2 - embedding_column_b, embedding_column_a = fc.shared_embedding_columns( + embedding_column_b, embedding_column_a = fc_new.shared_embedding_columns( [categorical_column_b, categorical_column_a], dimension=embedding_dimension) self.assertIs(categorical_column_a, embedding_column_a.categorical_column) @@ -5362,13 +5528,14 @@ class SharedEmbeddingColumnTest(test.TestCase): 'bbb': parsing_ops.VarLenFeature(dtypes.int64) }, embedding_column_b._parse_example_spec) + @test_util.run_deprecated_v1 def test_all_constructor_args(self): - categorical_column_a = fc.categorical_column_with_identity( + categorical_column_a = fc._categorical_column_with_identity( key='aaa', num_buckets=3) - categorical_column_b = fc.categorical_column_with_identity( + categorical_column_b = fc._categorical_column_with_identity( key='bbb', num_buckets=3) embedding_dimension = 2 - embedding_column_a, embedding_column_b = fc.shared_embedding_columns( + embedding_column_a, embedding_column_b = fc_new.shared_embedding_columns( [categorical_column_a, categorical_column_b], dimension=embedding_dimension, combiner='my_combiner', @@ -5413,13 +5580,14 @@ class SharedEmbeddingColumnTest(test.TestCase): 'bbb': parsing_ops.VarLenFeature(dtypes.int64) }, embedding_column_b._parse_example_spec) + @test_util.run_deprecated_v1 def test_deep_copy(self): - categorical_column_a = fc.categorical_column_with_identity( + categorical_column_a = fc._categorical_column_with_identity( key='aaa', num_buckets=3) - categorical_column_b = fc.categorical_column_with_identity( + categorical_column_b = fc._categorical_column_with_identity( key='bbb', num_buckets=3) embedding_dimension = 2 - original_a, _ = fc.shared_embedding_columns( + original_a, _ = fc_new.shared_embedding_columns( [categorical_column_a, categorical_column_b], dimension=embedding_dimension, combiner='my_combiner', @@ -5427,7 +5595,8 @@ class SharedEmbeddingColumnTest(test.TestCase): shared_embedding_collection_name='shared_embedding_collection_name', ckpt_to_load_from='my_ckpt', tensor_name_in_ckpt='my_ckpt_tensor', - max_norm=42., trainable=False) + max_norm=42., + trainable=False) for embedding_column_a in (original_a, copy.deepcopy(original_a)): self.assertEqual('aaa', embedding_column_a.categorical_column.name) self.assertEqual(3, embedding_column_a.categorical_column._num_buckets) @@ -5450,55 +5619,60 @@ class SharedEmbeddingColumnTest(test.TestCase): 'aaa': parsing_ops.VarLenFeature(dtypes.int64) }, embedding_column_a._parse_example_spec) + @test_util.run_deprecated_v1 def test_invalid_initializer(self): - categorical_column_a = fc.categorical_column_with_identity( + categorical_column_a = fc._categorical_column_with_identity( key='aaa', num_buckets=3) - categorical_column_b = fc.categorical_column_with_identity( + categorical_column_b = fc._categorical_column_with_identity( key='bbb', num_buckets=3) with self.assertRaisesRegexp(ValueError, 'initializer must be callable'): - fc.shared_embedding_columns( - [categorical_column_a, categorical_column_b], dimension=2, + fc_new.shared_embedding_columns( + [categorical_column_a, categorical_column_b], + dimension=2, initializer='not_fn') + @test_util.run_deprecated_v1 def test_incompatible_column_type(self): - categorical_column_a = fc.categorical_column_with_identity( + categorical_column_a = fc._categorical_column_with_identity( key='aaa', num_buckets=3) - categorical_column_b = fc.categorical_column_with_identity( + categorical_column_b = fc._categorical_column_with_identity( key='bbb', num_buckets=3) - categorical_column_c = fc.categorical_column_with_hash_bucket( + categorical_column_c = fc._categorical_column_with_hash_bucket( key='ccc', hash_bucket_size=3) with self.assertRaisesRegexp( ValueError, 'all categorical_columns must have the same type.*' '_IdentityCategoricalColumn.*_HashedCategoricalColumn'): - fc.shared_embedding_columns( + fc_new.shared_embedding_columns( [categorical_column_a, categorical_column_b, categorical_column_c], dimension=2) + @test_util.run_deprecated_v1 def test_weighted_categorical_column_ok(self): - categorical_column_a = fc.categorical_column_with_identity( + categorical_column_a = fc._categorical_column_with_identity( key='aaa', num_buckets=3) - weighted_categorical_column_a = fc.weighted_categorical_column( + weighted_categorical_column_a = fc._weighted_categorical_column( categorical_column_a, weight_feature_key='aaa_weights') - categorical_column_b = fc.categorical_column_with_identity( + categorical_column_b = fc._categorical_column_with_identity( key='bbb', num_buckets=3) - weighted_categorical_column_b = fc.weighted_categorical_column( + weighted_categorical_column_b = fc._weighted_categorical_column( categorical_column_b, weight_feature_key='bbb_weights') - fc.shared_embedding_columns( + fc_new.shared_embedding_columns( [weighted_categorical_column_a, categorical_column_b], dimension=2) - fc.shared_embedding_columns( + fc_new.shared_embedding_columns( [categorical_column_a, weighted_categorical_column_b], dimension=2) - fc.shared_embedding_columns( + fc_new.shared_embedding_columns( [weighted_categorical_column_a, weighted_categorical_column_b], dimension=2) + @test_util.run_deprecated_v1 def test_parse_example(self): - a = fc.categorical_column_with_vocabulary_list( + a = fc._categorical_column_with_vocabulary_list( key='aaa', vocabulary_list=('omar', 'stringer', 'marlo')) - b = fc.categorical_column_with_vocabulary_list( + b = fc._categorical_column_with_vocabulary_list( key='bbb', vocabulary_list=('omar', 'stringer', 'marlo')) - a_embedded, b_embedded = fc.shared_embedding_columns( - [a, b], dimension=2) + a_embedded, b_embedded = fc_new.shared_embedding_columns([a, b], + dimension=2) data = example_pb2.Example(features=feature_pb2.Features( feature={ 'aaa': @@ -5529,11 +5703,12 @@ class SharedEmbeddingColumnTest(test.TestCase): dense_shape=[1, 2]), features['bbb'].eval()) + @test_util.run_deprecated_v1 def test_transform_feature(self): - a = fc.categorical_column_with_identity(key='aaa', num_buckets=3) - b = fc.categorical_column_with_identity(key='bbb', num_buckets=3) - a_embedded, b_embedded = fc.shared_embedding_columns( - [a, b], dimension=2) + a = fc._categorical_column_with_identity(key='aaa', num_buckets=3) + b = fc._categorical_column_with_identity(key='bbb', num_buckets=3) + a_embedded, b_embedded = fc_new.shared_embedding_columns([a, b], + dimension=2) features = { 'aaa': sparse_tensor.SparseTensor( indices=((0, 0), (1, 0), (1, 1)), @@ -5550,11 +5725,12 @@ class SharedEmbeddingColumnTest(test.TestCase): output_b = outputs[b] output_b_embedded = outputs[b_embedded] with _initialized_session(): - _assert_sparse_tensor_value( - self, output_a.eval(), output_a_embedded.eval()) - _assert_sparse_tensor_value( - self, output_b.eval(), output_b_embedded.eval()) + _assert_sparse_tensor_value(self, self.evaluate(output_a), + self.evaluate(output_a_embedded)) + _assert_sparse_tensor_value(self, self.evaluate(output_b), + self.evaluate(output_b_embedded)) + @test_util.run_deprecated_v1 def test_get_dense_tensor(self): # Inputs. vocabulary_size = 3 @@ -5598,13 +5774,14 @@ class SharedEmbeddingColumnTest(test.TestCase): ) # Build columns. - categorical_column_a = fc.categorical_column_with_identity( + categorical_column_a = fc._categorical_column_with_identity( key='aaa', num_buckets=vocabulary_size) - categorical_column_b = fc.categorical_column_with_identity( + categorical_column_b = fc._categorical_column_with_identity( key='bbb', num_buckets=vocabulary_size) - embedding_column_a, embedding_column_b = fc.shared_embedding_columns( + embedding_column_a, embedding_column_b = fc_new.shared_embedding_columns( [categorical_column_a, categorical_column_b], - dimension=embedding_dimension, initializer=_initializer) + dimension=embedding_dimension, + initializer=_initializer) # Provide sparse input and get dense result. embedding_lookup_a = embedding_column_a._get_dense_tensor( @@ -5618,10 +5795,11 @@ class SharedEmbeddingColumnTest(test.TestCase): tuple([v.name for v in global_vars])) embedding_var = global_vars[0] with _initialized_session(): - self.assertAllEqual(embedding_values, embedding_var.eval()) - self.assertAllEqual(expected_lookups_a, embedding_lookup_a.eval()) - self.assertAllEqual(expected_lookups_b, embedding_lookup_b.eval()) + self.assertAllEqual(embedding_values, self.evaluate(embedding_var)) + self.assertAllEqual(expected_lookups_a, self.evaluate(embedding_lookup_a)) + self.assertAllEqual(expected_lookups_b, self.evaluate(embedding_lookup_b)) + @test_util.run_deprecated_v1 def test_get_dense_tensor_weight_collections(self): # Inputs. vocabulary_size = 3 @@ -5651,11 +5829,11 @@ class SharedEmbeddingColumnTest(test.TestCase): return embedding_values # Build columns. - categorical_column_a = fc.categorical_column_with_identity( + categorical_column_a = fc._categorical_column_with_identity( key='aaa', num_buckets=vocabulary_size) - categorical_column_b = fc.categorical_column_with_identity( + categorical_column_b = fc._categorical_column_with_identity( key='bbb', num_buckets=vocabulary_size) - embedding_column_a, embedding_column_b = fc.shared_embedding_columns( + embedding_column_a, embedding_column_b = fc_new.shared_embedding_columns( [categorical_column_a, categorical_column_b], dimension=embedding_dimension, initializer=_initializer) @@ -5674,6 +5852,7 @@ class SharedEmbeddingColumnTest(test.TestCase): ('input_layer/aaa_bbb_shared_embedding/embedding_weights:0',), tuple(v.name for v in my_vars)) + @test_util.run_deprecated_v1 def test_get_dense_tensor_placeholder_inputs(self): # Inputs. vocabulary_size = 3 @@ -5712,13 +5891,14 @@ class SharedEmbeddingColumnTest(test.TestCase): return embedding_values # Build columns. - categorical_column_a = fc.categorical_column_with_identity( + categorical_column_a = fc._categorical_column_with_identity( key='aaa', num_buckets=vocabulary_size) - categorical_column_b = fc.categorical_column_with_identity( + categorical_column_b = fc._categorical_column_with_identity( key='bbb', num_buckets=vocabulary_size) - embedding_column_a, embedding_column_b = fc.shared_embedding_columns( + embedding_column_a, embedding_column_b = fc_new.shared_embedding_columns( [categorical_column_a, categorical_column_b], - dimension=embedding_dimension, initializer=_initializer) + dimension=embedding_dimension, + initializer=_initializer) # Provide sparse input and get dense result. embedding_lookup_a = embedding_column_a._get_dense_tensor( @@ -5729,6 +5909,7 @@ class SharedEmbeddingColumnTest(test.TestCase): with _initialized_session() as sess: sess.run([embedding_lookup_a, embedding_lookup_b], feed_dict=feed_dict) + @test_util.run_deprecated_v1 def test_linear_model(self): # Inputs. batch_size = 2 @@ -5752,13 +5933,14 @@ class SharedEmbeddingColumnTest(test.TestCase): return zeros_embedding_values # Build columns. - categorical_column_a = fc.categorical_column_with_identity( + categorical_column_a = fc._categorical_column_with_identity( key='aaa', num_buckets=vocabulary_size) - categorical_column_b = fc.categorical_column_with_identity( + categorical_column_b = fc._categorical_column_with_identity( key='bbb', num_buckets=vocabulary_size) - embedding_column_a, embedding_column_b = fc.shared_embedding_columns( + embedding_column_a, embedding_column_b = fc_new.shared_embedding_columns( [categorical_column_a, categorical_column_b], - dimension=embedding_dimension, initializer=_initializer) + dimension=embedding_dimension, + initializer=_initializer) with ops.Graph().as_default(): predictions = fc.linear_model({ @@ -5790,13 +5972,15 @@ class SharedEmbeddingColumnTest(test.TestCase): 'linear_model/aaa_bbb_shared_embedding_1/weights:0'] with _initialized_session(): # Predictions with all zero weights. - self.assertAllClose(np.zeros((1,)), bias.eval()) - self.assertAllClose(zeros_embedding_values, embedding_weights.eval()) + self.assertAllClose(np.zeros((1,)), self.evaluate(bias)) + self.assertAllClose(zeros_embedding_values, + self.evaluate(embedding_weights)) self.assertAllClose( - np.zeros((embedding_dimension, 1)), linear_weights_a.eval()) + np.zeros((embedding_dimension, 1)), self.evaluate(linear_weights_a)) self.assertAllClose( - np.zeros((embedding_dimension, 1)), linear_weights_b.eval()) - self.assertAllClose(np.zeros((batch_size, 1)), predictions.eval()) + np.zeros((embedding_dimension, 1)), self.evaluate(linear_weights_b)) + self.assertAllClose( + np.zeros((batch_size, 1)), self.evaluate(predictions)) # Predictions with all non-zero weights. embedding_weights.assign(( @@ -5814,8 +5998,9 @@ class SharedEmbeddingColumnTest(test.TestCase): # example 1, ids [], embedding[1] = 0, 0] # sum(embeddings * linear_weights) # = [3*1 + 5*2, 3*0 +5*0] = [13, 0] - self.assertAllClose([[94. + 13.], [29.]], predictions.eval()) + self.assertAllClose([[94. + 13.], [29.]], self.evaluate(predictions)) + @test_util.run_deprecated_v1 def test_keras_linear_model(self): # Inputs. batch_size = 2 @@ -5842,11 +6027,11 @@ class SharedEmbeddingColumnTest(test.TestCase): return zeros_embedding_values # Build columns. - categorical_column_a = fc.categorical_column_with_identity( + categorical_column_a = fc._categorical_column_with_identity( key='aaa', num_buckets=vocabulary_size) - categorical_column_b = fc.categorical_column_with_identity( + categorical_column_b = fc._categorical_column_with_identity( key='bbb', num_buckets=vocabulary_size) - embedding_column_a, embedding_column_b = fc.shared_embedding_columns( + embedding_column_a, embedding_column_b = fc_new.shared_embedding_columns( [categorical_column_a, categorical_column_b], dimension=embedding_dimension, initializer=_initializer) @@ -5881,13 +6066,15 @@ class SharedEmbeddingColumnTest(test.TestCase): 'linear_model/aaa_bbb_shared_embedding_1/weights:0'] with _initialized_session(): # Predictions with all zero weights. - self.assertAllClose(np.zeros((1,)), bias.eval()) - self.assertAllClose(zeros_embedding_values, embedding_weights.eval()) + self.assertAllClose(np.zeros((1,)), self.evaluate(bias)) + self.assertAllClose(zeros_embedding_values, + self.evaluate(embedding_weights)) self.assertAllClose( - np.zeros((embedding_dimension, 1)), linear_weights_a.eval()) + np.zeros((embedding_dimension, 1)), self.evaluate(linear_weights_a)) self.assertAllClose( - np.zeros((embedding_dimension, 1)), linear_weights_b.eval()) - self.assertAllClose(np.zeros((batch_size, 1)), predictions.eval()) + np.zeros((embedding_dimension, 1)), self.evaluate(linear_weights_b)) + self.assertAllClose( + np.zeros((batch_size, 1)), self.evaluate(predictions)) # Predictions with all non-zero weights. embedding_weights.assign(( @@ -5905,7 +6092,7 @@ class SharedEmbeddingColumnTest(test.TestCase): # example 1, ids [], embedding[1] = 0, 0] # sum(embeddings * linear_weights) # = [3*1 + 5*2, 3*0 +5*0] = [13, 0] - self.assertAllClose([[94. + 13.], [29.]], predictions.eval()) + self.assertAllClose([[94. + 13.], [29.]], self.evaluate(predictions)) def _test_input_layer(self, trainable=True): # Inputs. @@ -5949,13 +6136,14 @@ class SharedEmbeddingColumnTest(test.TestCase): ) # Build columns. - categorical_column_a = fc.categorical_column_with_identity( + categorical_column_a = fc._categorical_column_with_identity( key='aaa', num_buckets=vocabulary_size) - categorical_column_b = fc.categorical_column_with_identity( + categorical_column_b = fc._categorical_column_with_identity( key='bbb', num_buckets=vocabulary_size) - embedding_column_a, embedding_column_b = fc.shared_embedding_columns( + embedding_column_a, embedding_column_b = fc_new.shared_embedding_columns( [categorical_column_a, categorical_column_b], - dimension=embedding_dimension, initializer=_initializer, + dimension=embedding_dimension, + initializer=_initializer, trainable=trainable) # Provide sparse input and get dense result. @@ -5978,20 +6166,23 @@ class SharedEmbeddingColumnTest(test.TestCase): shared_embedding_vars = global_vars with _initialized_session(): self.assertAllEqual(embedding_values, shared_embedding_vars[0].eval()) - self.assertAllEqual(expected_lookups, input_layer.eval()) + self.assertAllEqual(expected_lookups, self.evaluate(input_layer)) + @test_util.run_deprecated_v1 def test_input_layer(self): self._test_input_layer() + @test_util.run_deprecated_v1 def test_input_layer_no_trainable(self): self._test_input_layer(trainable=False) class WeightedCategoricalColumnTest(test.TestCase): + @test_util.run_deprecated_v1 def test_defaults(self): - column = fc.weighted_categorical_column( - categorical_column=fc.categorical_column_with_identity( + column = fc._weighted_categorical_column( + categorical_column=fc._categorical_column_with_identity( key='ids', num_buckets=3), weight_feature_key='values') self.assertEqual('ids_weighted_by_values', column.name) @@ -6002,10 +6193,11 @@ class WeightedCategoricalColumnTest(test.TestCase): 'values': parsing_ops.VarLenFeature(dtypes.float32) }, column._parse_example_spec) + @test_util.run_deprecated_v1 def test_deep_copy(self): """Tests deepcopy of categorical_column_with_hash_bucket.""" - original = fc.weighted_categorical_column( - categorical_column=fc.categorical_column_with_identity( + original = fc._weighted_categorical_column( + categorical_column=fc._categorical_column_with_identity( key='ids', num_buckets=3), weight_feature_key='values') for column in (original, copy.deepcopy(original)): @@ -6018,23 +6210,23 @@ class WeightedCategoricalColumnTest(test.TestCase): def test_invalid_dtype_none(self): with self.assertRaisesRegexp(ValueError, 'is not convertible to float'): - fc.weighted_categorical_column( - categorical_column=fc.categorical_column_with_identity( + fc._weighted_categorical_column( + categorical_column=fc._categorical_column_with_identity( key='ids', num_buckets=3), weight_feature_key='values', dtype=None) def test_invalid_dtype_string(self): with self.assertRaisesRegexp(ValueError, 'is not convertible to float'): - fc.weighted_categorical_column( - categorical_column=fc.categorical_column_with_identity( + fc._weighted_categorical_column( + categorical_column=fc._categorical_column_with_identity( key='ids', num_buckets=3), weight_feature_key='values', dtype=dtypes.string) def test_invalid_input_dtype(self): - column = fc.weighted_categorical_column( - categorical_column=fc.categorical_column_with_identity( + column = fc._weighted_categorical_column( + categorical_column=fc._categorical_column_with_identity( key='ids', num_buckets=3), weight_feature_key='values') strings = sparse_tensor.SparseTensorValue( @@ -6046,14 +6238,14 @@ class WeightedCategoricalColumnTest(test.TestCase): def test_column_name_collision(self): with self.assertRaisesRegexp(ValueError, r'Parse config.*already exists'): - fc.weighted_categorical_column( - categorical_column=fc.categorical_column_with_identity( + fc._weighted_categorical_column( + categorical_column=fc._categorical_column_with_identity( key='aaa', num_buckets=3), weight_feature_key='aaa')._parse_example_spec() def test_missing_weights(self): - column = fc.weighted_categorical_column( - categorical_column=fc.categorical_column_with_identity( + column = fc._weighted_categorical_column( + categorical_column=fc._categorical_column_with_identity( key='ids', num_buckets=3), weight_feature_key='values') inputs = sparse_tensor.SparseTensorValue( @@ -6064,10 +6256,12 @@ class WeightedCategoricalColumnTest(test.TestCase): ValueError, 'values is not in features dictionary'): _transform_features({'ids': inputs}, (column,)) + @test_util.run_deprecated_v1 def test_parse_example(self): - a = fc.categorical_column_with_vocabulary_list( + a = fc._categorical_column_with_vocabulary_list( key='aaa', vocabulary_list=('omar', 'stringer', 'marlo')) - a_weighted = fc.weighted_categorical_column(a, weight_feature_key='weights') + a_weighted = fc._weighted_categorical_column( + a, weight_feature_key='weights') data = example_pb2.Example(features=feature_pb2.Features( feature={ 'aaa': @@ -6098,9 +6292,10 @@ class WeightedCategoricalColumnTest(test.TestCase): dense_shape=[1, 2]), features['weights'].eval()) + @test_util.run_deprecated_v1 def test_transform_features(self): - column = fc.weighted_categorical_column( - categorical_column=fc.categorical_column_with_identity( + column = fc._weighted_categorical_column( + categorical_column=fc._categorical_column_with_identity( key='ids', num_buckets=3), weight_feature_key='values') inputs = sparse_tensor.SparseTensorValue( @@ -6121,19 +6316,18 @@ class WeightedCategoricalColumnTest(test.TestCase): sparse_tensor.SparseTensorValue( indices=inputs.indices, values=np.array(inputs.values, dtype=np.int64), - dense_shape=inputs.dense_shape), - id_tensor.eval()) + dense_shape=inputs.dense_shape), self.evaluate(id_tensor)) _assert_sparse_tensor_value( self, sparse_tensor.SparseTensorValue( indices=weights.indices, values=np.array(weights.values, dtype=np.float32), - dense_shape=weights.dense_shape), - weight_tensor.eval()) + dense_shape=weights.dense_shape), self.evaluate(weight_tensor)) + @test_util.run_deprecated_v1 def test_transform_features_dense_input(self): - column = fc.weighted_categorical_column( - categorical_column=fc.categorical_column_with_identity( + column = fc._weighted_categorical_column( + categorical_column=fc._categorical_column_with_identity( key='ids', num_buckets=3), weight_feature_key='values') weights = sparse_tensor.SparseTensorValue( @@ -6150,19 +6344,18 @@ class WeightedCategoricalColumnTest(test.TestCase): sparse_tensor.SparseTensorValue( indices=((0, 0), (1, 0), (1, 1)), values=np.array((0, 1, 0), dtype=np.int64), - dense_shape=(2, 2)), - id_tensor.eval()) + dense_shape=(2, 2)), self.evaluate(id_tensor)) _assert_sparse_tensor_value( self, sparse_tensor.SparseTensorValue( indices=weights.indices, values=np.array(weights.values, dtype=np.float32), - dense_shape=weights.dense_shape), - weight_tensor.eval()) + dense_shape=weights.dense_shape), self.evaluate(weight_tensor)) + @test_util.run_deprecated_v1 def test_transform_features_dense_weights(self): - column = fc.weighted_categorical_column( - categorical_column=fc.categorical_column_with_identity( + column = fc._weighted_categorical_column( + categorical_column=fc._categorical_column_with_identity( key='ids', num_buckets=3), weight_feature_key='values') inputs = sparse_tensor.SparseTensorValue( @@ -6179,19 +6372,18 @@ class WeightedCategoricalColumnTest(test.TestCase): sparse_tensor.SparseTensorValue( indices=inputs.indices, values=np.array(inputs.values, dtype=np.int64), - dense_shape=inputs.dense_shape), - id_tensor.eval()) + dense_shape=inputs.dense_shape), self.evaluate(id_tensor)) _assert_sparse_tensor_value( self, sparse_tensor.SparseTensorValue( indices=((0, 0), (1, 0), (1, 1)), values=np.array((.5, 1., .1), dtype=np.float32), - dense_shape=(2, 2)), - weight_tensor.eval()) + dense_shape=(2, 2)), self.evaluate(weight_tensor)) + @test_util.run_deprecated_v1 def test_keras_linear_model(self): - column = fc.weighted_categorical_column( - categorical_column=fc.categorical_column_with_identity( + column = fc._weighted_categorical_column( + categorical_column=fc._categorical_column_with_identity( key='ids', num_buckets=3), weight_feature_key='values') with ops.Graph().as_default(): @@ -6210,18 +6402,18 @@ class WeightedCategoricalColumnTest(test.TestCase): bias = get_linear_model_bias() weight_var = get_linear_model_column_var(column) with _initialized_session(): - self.assertAllClose((0.,), bias.eval()) - self.assertAllClose(((0.,), (0.,), (0.,)), weight_var.eval()) - self.assertAllClose(((0.,), (0.,)), predictions.eval()) + self.assertAllClose((0.,), self.evaluate(bias)) + self.assertAllClose(((0.,), (0.,), (0.,)), self.evaluate(weight_var)) + self.assertAllClose(((0.,), (0.,)), self.evaluate(predictions)) weight_var.assign(((1.,), (2.,), (3.,))).eval() # weight_var[0] * weights[0, 0] = 1 * .5 = .5 # weight_var[2] * weights[1, 0] + weight_var[1] * weights[1, 1] # = 3*1 + 2*.1 = 3+.2 = 3.2 - self.assertAllClose(((.5,), (3.2,)), predictions.eval()) + self.assertAllClose(((.5,), (3.2,)), self.evaluate(predictions)) def test_keras_linear_model_mismatched_shape(self): - column = fc.weighted_categorical_column( - categorical_column=fc.categorical_column_with_identity( + column = fc._weighted_categorical_column( + categorical_column=fc._categorical_column_with_identity( key='ids', num_buckets=3), weight_feature_key='values') with ops.Graph().as_default(): @@ -6241,8 +6433,8 @@ class WeightedCategoricalColumnTest(test.TestCase): }, (column,)) def test_keras_linear_model_mismatched_dense_values(self): - column = fc.weighted_categorical_column( - categorical_column=fc.categorical_column_with_identity( + column = fc._weighted_categorical_column( + categorical_column=fc._categorical_column_with_identity( key='ids', num_buckets=3), weight_feature_key='values') with ops.Graph().as_default(): @@ -6263,11 +6455,11 @@ class WeightedCategoricalColumnTest(test.TestCase): rewriter_config_pb2.RewriterConfig.OFF) with _initialized_session(config): with self.assertRaisesRegexp(errors.OpError, 'Incompatible shapes'): - predictions.eval() + self.evaluate(predictions) def test_keras_linear_model_mismatched_dense_shape(self): - column = fc.weighted_categorical_column( - categorical_column=fc.categorical_column_with_identity( + column = fc._weighted_categorical_column( + categorical_column=fc._categorical_column_with_identity( key='ids', num_buckets=3), weight_feature_key='values') with ops.Graph().as_default(): @@ -6282,18 +6474,19 @@ class WeightedCategoricalColumnTest(test.TestCase): bias = get_linear_model_bias() weight_var = get_linear_model_column_var(column) with _initialized_session(): - self.assertAllClose((0.,), bias.eval()) - self.assertAllClose(((0.,), (0.,), (0.,)), weight_var.eval()) - self.assertAllClose(((0.,), (0.,)), predictions.eval()) + self.assertAllClose((0.,), self.evaluate(bias)) + self.assertAllClose(((0.,), (0.,), (0.,)), self.evaluate(weight_var)) + self.assertAllClose(((0.,), (0.,)), self.evaluate(predictions)) weight_var.assign(((1.,), (2.,), (3.,))).eval() # weight_var[0] * weights[0, 0] = 1 * .5 = .5 # weight_var[2] * weights[1, 0] + weight_var[1] * weights[1, 1] # = 3*1 + 2*.1 = 3+.2 = 3.2 - self.assertAllClose(((.5,), (3.2,)), predictions.eval()) + self.assertAllClose(((.5,), (3.2,)), self.evaluate(predictions)) + @test_util.run_deprecated_v1 def test_linear_model(self): - column = fc.weighted_categorical_column( - categorical_column=fc.categorical_column_with_identity( + column = fc._weighted_categorical_column( + categorical_column=fc._categorical_column_with_identity( key='ids', num_buckets=3), weight_feature_key='values') with ops.Graph().as_default(): @@ -6310,18 +6503,18 @@ class WeightedCategoricalColumnTest(test.TestCase): bias = get_linear_model_bias() weight_var = get_linear_model_column_var(column) with _initialized_session(): - self.assertAllClose((0.,), bias.eval()) - self.assertAllClose(((0.,), (0.,), (0.,)), weight_var.eval()) - self.assertAllClose(((0.,), (0.,)), predictions.eval()) + self.assertAllClose((0.,), self.evaluate(bias)) + self.assertAllClose(((0.,), (0.,), (0.,)), self.evaluate(weight_var)) + self.assertAllClose(((0.,), (0.,)), self.evaluate(predictions)) weight_var.assign(((1.,), (2.,), (3.,))).eval() # weight_var[0] * weights[0, 0] = 1 * .5 = .5 # weight_var[2] * weights[1, 0] + weight_var[1] * weights[1, 1] # = 3*1 + 2*.1 = 3+.2 = 3.2 - self.assertAllClose(((.5,), (3.2,)), predictions.eval()) + self.assertAllClose(((.5,), (3.2,)), self.evaluate(predictions)) def test_linear_model_mismatched_shape(self): - column = fc.weighted_categorical_column( - categorical_column=fc.categorical_column_with_identity( + column = fc._weighted_categorical_column( + categorical_column=fc._categorical_column_with_identity( key='ids', num_buckets=3), weight_feature_key='values') with ops.Graph().as_default(): @@ -6339,8 +6532,8 @@ class WeightedCategoricalColumnTest(test.TestCase): }, (column,)) def test_linear_model_mismatched_dense_values(self): - column = fc.weighted_categorical_column( - categorical_column=fc.categorical_column_with_identity( + column = fc._weighted_categorical_column( + categorical_column=fc._categorical_column_with_identity( key='ids', num_buckets=3), weight_feature_key='values') with ops.Graph().as_default(): @@ -6361,11 +6554,11 @@ class WeightedCategoricalColumnTest(test.TestCase): rewriter_config_pb2.RewriterConfig.OFF) with _initialized_session(config): with self.assertRaisesRegexp(errors.OpError, 'Incompatible shapes'): - predictions.eval() + self.evaluate(predictions) def test_linear_model_mismatched_dense_shape(self): - column = fc.weighted_categorical_column( - categorical_column=fc.categorical_column_with_identity( + column = fc._weighted_categorical_column( + categorical_column=fc._categorical_column_with_identity( key='ids', num_buckets=3), weight_feature_key='values') with ops.Graph().as_default(): @@ -6379,14 +6572,14 @@ class WeightedCategoricalColumnTest(test.TestCase): bias = get_linear_model_bias() weight_var = get_linear_model_column_var(column) with _initialized_session(): - self.assertAllClose((0.,), bias.eval()) - self.assertAllClose(((0.,), (0.,), (0.,)), weight_var.eval()) - self.assertAllClose(((0.,), (0.,)), predictions.eval()) + self.assertAllClose((0.,), self.evaluate(bias)) + self.assertAllClose(((0.,), (0.,), (0.,)), self.evaluate(weight_var)) + self.assertAllClose(((0.,), (0.,)), self.evaluate(predictions)) weight_var.assign(((1.,), (2.,), (3.,))).eval() # weight_var[0] * weights[0, 0] = 1 * .5 = .5 # weight_var[2] * weights[1, 0] + weight_var[1] * weights[1, 1] # = 3*1 + 2*.1 = 3+.2 = 3.2 - self.assertAllClose(((.5,), (3.2,)), predictions.eval()) + self.assertAllClose(((.5,), (3.2,)), self.evaluate(predictions)) # TODO(ptucker): Add test with embedding of weighted categorical. diff --git a/tensorflow/python/feature_column/feature_column_v2.py b/tensorflow/python/feature_column/feature_column_v2.py index 9b4a7e882f9..63089264942 100644 --- a/tensorflow/python/feature_column/feature_column_v2.py +++ b/tensorflow/python/feature_column/feature_column_v2.py @@ -165,6 +165,7 @@ from tensorflow.python.training import checkpoint_utils from tensorflow.python.training.checkpointable import tracking from tensorflow.python.util import deprecation from tensorflow.python.util import nest +from tensorflow.python.util.tf_export import tf_export _FEATURE_COLUMN_DEPRECATION_DATE = '2018-11-30' @@ -258,7 +259,7 @@ class StateManager(object): class _StateManagerImpl(StateManager): - """Manages the state of FeatureLayer and LinearModel.""" + """Manages the state of DenseFeatures and LinearLayer.""" def __init__(self, layer, trainable): """Creates an _StateManagerImpl object. @@ -302,7 +303,8 @@ class _StateManagerImpl(StateManager): raise ValueError('Variable does not exist.') -class FeatureLayer(Layer): +@tf_export('keras.layers.DenseFeatures', v1=[]) +class DenseFeatures(Layer): """A layer that produces a dense `Tensor` based on given `feature_columns`. Generally a single example in training data is described with FeatureColumns. @@ -318,7 +320,7 @@ class FeatureLayer(Layer): keywords_embedded = embedding_column( categorical_column_with_hash_bucket("keywords", 10K), dimensions=16) columns = [price, keywords_embedded, ...] - feature_layer = FeatureLayer(columns) + feature_layer = DenseFeatures(columns) features = tf.parse_example(..., features=make_parse_example_spec(columns)) dense_tensor = feature_layer(features) @@ -333,7 +335,7 @@ class FeatureLayer(Layer): trainable=True, name=None, **kwargs): - """Constructs a FeatureLayer. + """Constructs a DenseFeatures. Args: feature_columns: An iterable containing the FeatureColumns to use as @@ -344,13 +346,14 @@ class FeatureLayer(Layer): `indicator_column`. trainable: If `True` also add the variable to the graph collection `GraphKeys.TRAINABLE_VARIABLES` (see `tf.Variable`). - name: Name to give to the FeatureLayer. + name: Name to give to the DenseFeatures. **kwargs: Keyword arguments to construct a layer. Raises: ValueError: if an item in `feature_columns` is not a `DenseColumn`. """ - super(FeatureLayer, self).__init__(name=name, trainable=trainable, **kwargs) + super(DenseFeatures, self).__init__( + name=name, trainable=trainable, **kwargs) self._feature_columns = _normalize_feature_columns(feature_columns) self._feature_columns = sorted(self._feature_columns, key=lambda x: x.name) @@ -371,7 +374,7 @@ class FeatureLayer(Layer): with variable_scope._pure_variable_scope(self.name): # pylint: disable=protected-access with variable_scope._pure_variable_scope(column.name): # pylint: disable=protected-access column.create_state(self._state_manager) - super(FeatureLayer, self).build(None) + super(DenseFeatures, self).build(None) def call(self, features, cols_to_output_tensors=None): """Returns a dense tensor corresponding to the `feature_columns`. @@ -515,6 +518,7 @@ class _LinearModelLayer(Layer): return predictions +@tf_export('keras.layers.LinearModel', v1=[]) class LinearModel(training.Model): """Produces a linear prediction `Tensor` based on given `feature_columns`. @@ -522,7 +526,7 @@ class LinearModel(training.Model): Weighted sum refers to logits in classification problems. It refers to the prediction itself for linear regression problems. - Note on supported columns: `LinearModel` treats categorical columns as + Note on supported columns: `LinearLayer` treats categorical columns as `indicator_column`s. To be specific, assume the input as `SparseTensor` looks like: @@ -547,7 +551,7 @@ class LinearModel(training.Model): keywords = categorical_column_with_hash_bucket("keywords", 10K) keywords_price = crossed_column('keywords', price_buckets, ...) columns = [price_buckets, keywords, keywords_price ...] - linear_model = LinearModel(columns) + linear_model = LinearLayer(columns) features = tf.parse_example(..., features=make_parse_example_spec(columns)) prediction = linear_model(features) @@ -561,7 +565,7 @@ class LinearModel(training.Model): trainable=True, name=None, **kwargs): - """Constructs a LinearModel. + """Constructs a LinearLayer. Args: feature_columns: An iterable containing the FeatureColumns to use as @@ -650,7 +654,7 @@ class LinearModel(training.Model): return self.layer.bias -def _transform_features(features, feature_columns, state_manager): +def _transform_features_v2(features, feature_columns, state_manager): """Returns transformed features based on features columns passed in. Please note that most probably you would not need to use this function. Please @@ -695,7 +699,8 @@ def _transform_features(features, feature_columns, state_manager): return outputs -def make_parse_example_spec(feature_columns): +@tf_export('feature_column.make_parse_example_spec', v1=[]) +def make_parse_example_spec_v2(feature_columns): """Creates parsing spec dictionary from input feature_columns. The returned dictionary can be used as arg 'features' in `tf.parse_example`. @@ -754,10 +759,15 @@ def make_parse_example_spec(feature_columns): return result -def embedding_column( - categorical_column, dimension, combiner='mean', initializer=None, - ckpt_to_load_from=None, tensor_name_in_ckpt=None, max_norm=None, - trainable=True): +@tf_export('feature_column.embedding_column') +def embedding_column(categorical_column, + dimension, + combiner='mean', + initializer=None, + ckpt_to_load_from=None, + tensor_name_in_ckpt=None, + max_norm=None, + trainable=True): """`DenseColumn` that converts from sparse, categorical input. Use this when your inputs are sparse, but you want to convert them to a dense @@ -854,6 +864,180 @@ def embedding_column( trainable=trainable) +@tf_export(v1=['feature_column.shared_embedding_columns']) +def shared_embedding_columns(categorical_columns, + dimension, + combiner='mean', + initializer=None, + shared_embedding_collection_name=None, + ckpt_to_load_from=None, + tensor_name_in_ckpt=None, + max_norm=None, + trainable=True): + """List of dense columns that convert from sparse, categorical input. + + This is similar to `embedding_column`, except that it produces a list of + embedding columns that share the same embedding weights. + + Use this when your inputs are sparse and of the same type (e.g. watched and + impression video IDs that share the same vocabulary), and you want to convert + them to a dense representation (e.g., to feed to a DNN). + + Inputs must be a list of categorical columns created by any of the + `categorical_column_*` function. They must all be of the same type and have + the same arguments except `key`. E.g. they can be + categorical_column_with_vocabulary_file with the same vocabulary_file. Some or + all columns could also be weighted_categorical_column. + + Here is an example embedding of two features for a DNNClassifier model: + + ```python + watched_video_id = categorical_column_with_vocabulary_file( + 'watched_video_id', video_vocabulary_file, video_vocabulary_size) + impression_video_id = categorical_column_with_vocabulary_file( + 'impression_video_id', video_vocabulary_file, video_vocabulary_size) + columns = shared_embedding_columns( + [watched_video_id, impression_video_id], dimension=10) + + estimator = tf.estimator.DNNClassifier(feature_columns=columns, ...) + + label_column = ... + def input_fn(): + features = tf.parse_example( + ..., features=make_parse_example_spec(columns + [label_column])) + labels = features.pop(label_column.name) + return features, labels + + estimator.train(input_fn=input_fn, steps=100) + ``` + + Here is an example using `shared_embedding_columns` with model_fn: + + ```python + def model_fn(features, ...): + watched_video_id = categorical_column_with_vocabulary_file( + 'watched_video_id', video_vocabulary_file, video_vocabulary_size) + impression_video_id = categorical_column_with_vocabulary_file( + 'impression_video_id', video_vocabulary_file, video_vocabulary_size) + columns = shared_embedding_columns( + [watched_video_id, impression_video_id], dimension=10) + dense_tensor = input_layer(features, columns) + # Form DNN layers, calculate loss, and return EstimatorSpec. + ... + ``` + + Args: + categorical_columns: List of categorical columns created by a + `categorical_column_with_*` function. These columns produce the sparse IDs + that are inputs to the embedding lookup. All columns must be of the same + type and have the same arguments except `key`. E.g. they can be + categorical_column_with_vocabulary_file with the same vocabulary_file. + Some or all columns could also be weighted_categorical_column. + dimension: An integer specifying dimension of the embedding, must be > 0. + combiner: A string specifying how to reduce if there are multiple entries in + a single row. Currently 'mean', 'sqrtn' and 'sum' are supported, with + 'mean' the default. 'sqrtn' often achieves good accuracy, in particular + with bag-of-words columns. Each of this can be thought as example level + normalizations on the column. For more information, see + `tf.embedding_lookup_sparse`. + initializer: A variable initializer function to be used in embedding + variable initialization. If not specified, defaults to + `tf.truncated_normal_initializer` with mean `0.0` and standard deviation + `1/sqrt(dimension)`. + shared_embedding_collection_name: Optional name of the collection where + shared embedding weights are added. If not given, a reasonable name will + be chosen based on the names of `categorical_columns`. This is also used + in `variable_scope` when creating shared embedding weights. + ckpt_to_load_from: String representing checkpoint name/pattern from which to + restore column weights. Required if `tensor_name_in_ckpt` is not `None`. + tensor_name_in_ckpt: Name of the `Tensor` in `ckpt_to_load_from` from which + to restore the column weights. Required if `ckpt_to_load_from` is not + `None`. + max_norm: If not `None`, each embedding is clipped if its l2-norm is larger + than this value, before combining. + trainable: Whether or not the embedding is trainable. Default is True. + + Returns: + A list of dense columns that converts from sparse input. The order of + results follows the ordering of `categorical_columns`. + + Raises: + ValueError: if `dimension` not > 0. + ValueError: if any of the given `categorical_columns` is of different type + or has different arguments than the others. + ValueError: if exactly one of `ckpt_to_load_from` and `tensor_name_in_ckpt` + is specified. + ValueError: if `initializer` is specified and is not callable. + RuntimeError: if eager execution is enabled. + """ + if context.executing_eagerly(): + raise RuntimeError('shared_embedding_columns are not supported when eager ' + 'execution is enabled.') + + if (dimension is None) or (dimension < 1): + raise ValueError('Invalid dimension {}.'.format(dimension)) + if (ckpt_to_load_from is None) != (tensor_name_in_ckpt is None): + raise ValueError('Must specify both `ckpt_to_load_from` and ' + '`tensor_name_in_ckpt` or none of them.') + + if (initializer is not None) and (not callable(initializer)): + raise ValueError('initializer must be callable if specified.') + if initializer is None: + initializer = init_ops.truncated_normal_initializer( + mean=0.0, stddev=1. / math.sqrt(dimension)) + + # Sort the columns so the default collection name is deterministic even if the + # user passes columns from an unsorted collection, such as dict.values(). + sorted_columns = sorted(categorical_columns, key=lambda x: x.name) + + c0 = sorted_columns[0] + num_buckets = c0._num_buckets # pylint: disable=protected-access + if not isinstance(c0, fc_old._CategoricalColumn): # pylint: disable=protected-access + raise ValueError( + 'All categorical_columns must be subclasses of _CategoricalColumn. ' + 'Given: {}, of type: {}'.format(c0, type(c0))) + if isinstance(c0, + (fc_old._WeightedCategoricalColumn, WeightedCategoricalColumn)): # pylint: disable=protected-access + c0 = c0.categorical_column + for c in sorted_columns[1:]: + if isinstance( + c, (fc_old._WeightedCategoricalColumn, WeightedCategoricalColumn)): # pylint: disable=protected-access + c = c.categorical_column + if not isinstance(c, type(c0)): + raise ValueError( + 'To use shared_embedding_column, all categorical_columns must have ' + 'the same type, or be weighted_categorical_column of the same type. ' + 'Given column: {} of type: {} does not match given column: {} of ' + 'type: {}'.format(c0, type(c0), c, type(c))) + if num_buckets != c._num_buckets: # pylint: disable=protected-access + raise ValueError( + 'To use shared_embedding_column, all categorical_columns must have ' + 'the same number of buckets. Given column: {} with buckets: {} does ' + 'not match column: {} with buckets: {}'.format( + c0, num_buckets, c, c._num_buckets)) # pylint: disable=protected-access + + if not shared_embedding_collection_name: + shared_embedding_collection_name = '_'.join(c.name for c in sorted_columns) + shared_embedding_collection_name += '_shared_embedding' + + result = [] + for column in categorical_columns: + result.append( + fc_old._SharedEmbeddingColumn( # pylint: disable=protected-access + categorical_column=column, + initializer=initializer, + dimension=dimension, + combiner=combiner, + shared_embedding_collection_name=shared_embedding_collection_name, + ckpt_to_load_from=ckpt_to_load_from, + tensor_name_in_ckpt=tensor_name_in_ckpt, + max_norm=max_norm, + trainable=trainable)) + + return result + + +@tf_export('feature_column.shared_embedding_columns', v1=[]) def shared_embedding_columns_v2(categorical_columns, dimension, combiner='mean', @@ -1019,6 +1203,7 @@ def shared_embedding_columns_v2(categorical_columns, return result +@tf_export('feature_column.numeric_column') def numeric_column(key, shape=(1,), default_value=None, @@ -1094,6 +1279,7 @@ def numeric_column(key, normalizer_fn=normalizer_fn) +@tf_export('feature_column.bucketized_column') def bucketized_column(source_column, boundaries): """Represents discretized dense input. @@ -1190,6 +1376,7 @@ def _assert_key_is_string(key): type(key), key)) +@tf_export('feature_column.categorical_column_with_hash_bucket') def categorical_column_with_hash_bucket(key, hash_bucket_size, dtype=dtypes.string): @@ -1248,6 +1435,7 @@ def categorical_column_with_hash_bucket(key, return HashedCategoricalColumn(key, hash_bucket_size, dtype) +@tf_export(v1=['feature_column.categorical_column_with_vocabulary_file']) def categorical_column_with_vocabulary_file(key, vocabulary_file, vocabulary_size=None, @@ -1325,6 +1513,97 @@ def categorical_column_with_vocabulary_file(key, Returns: A `CategoricalColumn` with a vocabulary file. + Raises: + ValueError: `vocabulary_file` is missing or cannot be opened. + ValueError: `vocabulary_size` is missing or < 1. + ValueError: `num_oov_buckets` is a negative integer. + ValueError: `num_oov_buckets` and `default_value` are both specified. + ValueError: `dtype` is neither string nor integer. + """ + return categorical_column_with_vocabulary_file_v2( + key, vocabulary_file, vocabulary_size, + dtype, default_value, + num_oov_buckets) + + +@tf_export('feature_column.categorical_column_with_vocabulary_file', v1=[]) +def categorical_column_with_vocabulary_file_v2(key, + vocabulary_file, + vocabulary_size=None, + dtype=dtypes.string, + default_value=None, + num_oov_buckets=0): + """A `CategoricalColumn` with a vocabulary file. + + Use this when your inputs are in string or integer format, and you have a + vocabulary file that maps each value to an integer ID. By default, + out-of-vocabulary values are ignored. Use either (but not both) of + `num_oov_buckets` and `default_value` to specify how to include + out-of-vocabulary values. + + For input dictionary `features`, `features[key]` is either `Tensor` or + `SparseTensor`. If `Tensor`, missing values can be represented by `-1` for int + and `''` for string, which will be dropped by this feature column. + + Example with `num_oov_buckets`: + File '/us/states.txt' contains 50 lines, each with a 2-character U.S. state + abbreviation. All inputs with values in that file are assigned an ID 0-49, + corresponding to its line number. All other values are hashed and assigned an + ID 50-54. + + ```python + states = categorical_column_with_vocabulary_file( + key='states', vocabulary_file='/us/states.txt', vocabulary_size=50, + num_oov_buckets=5) + columns = [states, ...] + features = tf.parse_example(..., features=make_parse_example_spec(columns)) + linear_prediction = linear_model(features, columns) + ``` + + Example with `default_value`: + File '/us/states.txt' contains 51 lines - the first line is 'XX', and the + other 50 each have a 2-character U.S. state abbreviation. Both a literal 'XX' + in input, and other values missing from the file, will be assigned ID 0. All + others are assigned the corresponding line number 1-50. + + ```python + states = categorical_column_with_vocabulary_file( + key='states', vocabulary_file='/us/states.txt', vocabulary_size=51, + default_value=0) + columns = [states, ...] + features = tf.parse_example(..., features=make_parse_example_spec(columns)) + linear_prediction, _, _ = linear_model(features, columns) + ``` + + And to make an embedding with either: + + ```python + columns = [embedding_column(states, 3),...] + features = tf.parse_example(..., features=make_parse_example_spec(columns)) + dense_tensor = input_layer(features, columns) + ``` + + Args: + key: A unique string identifying the input feature. It is used as the + column name and the dictionary key for feature parsing configs, feature + `Tensor` objects, and feature columns. + vocabulary_file: The vocabulary file name. + vocabulary_size: Number of the elements in the vocabulary. This must be no + greater than length of `vocabulary_file`, if less than length, later + values are ignored. If None, it is set to the length of `vocabulary_file`. + dtype: The type of features. Only string and integer types are supported. + default_value: The integer ID value to return for out-of-vocabulary feature + values, defaults to `-1`. This can not be specified with a positive + `num_oov_buckets`. + num_oov_buckets: Non-negative integer, the number of out-of-vocabulary + buckets. All out-of-vocabulary inputs will be assigned IDs in the range + `[vocabulary_size, vocabulary_size+num_oov_buckets)` based on a hash of + the input value. A positive `num_oov_buckets` can not be specified with + `default_value`. + + Returns: + A `CategoricalColumn` with a vocabulary file. + Raises: ValueError: `vocabulary_file` is missing or cannot be opened. ValueError: `vocabulary_size` is missing or < 1. @@ -1367,8 +1646,12 @@ def categorical_column_with_vocabulary_file(key, dtype=dtype) -def categorical_column_with_vocabulary_list( - key, vocabulary_list, dtype=None, default_value=-1, num_oov_buckets=0): +@tf_export('feature_column.categorical_column_with_vocabulary_list') +def categorical_column_with_vocabulary_list(key, + vocabulary_list, + dtype=None, + default_value=-1, + num_oov_buckets=0): """A `CategoricalColumn` with in-memory vocabulary. Use this when your inputs are in string or integer format, and you have an @@ -1480,6 +1763,7 @@ def categorical_column_with_vocabulary_list( num_oov_buckets=num_oov_buckets) +@tf_export('feature_column.categorical_column_with_identity') def categorical_column_with_identity(key, num_buckets, default_value=None): """A `CategoricalColumn` that returns identity values. @@ -1547,6 +1831,7 @@ def categorical_column_with_identity(key, num_buckets, default_value=None): key=key, number_buckets=num_buckets, default_value=default_value) +@tf_export('feature_column.indicator_column') def indicator_column(categorical_column): """Represents multi-hot representation of given categorical column. @@ -1581,8 +1866,10 @@ def indicator_column(categorical_column): return IndicatorColumn(categorical_column) -def weighted_categorical_column( - categorical_column, weight_feature_key, dtype=dtypes.float32): +@tf_export('feature_column.weighted_categorical_column') +def weighted_categorical_column(categorical_column, + weight_feature_key, + dtype=dtypes.float32): """Applies weight values to a `CategoricalColumn`. Use this when each of your sparse inputs has both an ID and a value. For @@ -1655,6 +1942,7 @@ def weighted_categorical_column( dtype=dtype) +@tf_export('feature_column.crossed_column') def crossed_column(keys, hash_bucket_size, hash_key=None): """Returns a column for performing crosses of categorical features. @@ -2120,7 +2408,7 @@ def _create_categorical_column_weighted_sum( weight_tensor = sparse_ops.sparse_reshape( weight_tensor, [array_ops.shape(weight_tensor)[0], -1]) - return _safe_embedding_lookup_sparse( + return embedding_ops.safe_embedding_lookup_sparse( weight_var, id_tensor, sparse_weights=weight_tensor, @@ -2731,7 +3019,7 @@ class EmbeddingColumn( }) # Return embedding lookup result. - return _safe_embedding_lookup_sparse( + return embedding_ops.safe_embedding_lookup_sparse( embedding_weights=embedding_weights, sparse_ids=sparse_ids, sparse_weights=sparse_weights, @@ -2890,7 +3178,7 @@ class EmbeddingColumn( def _raise_shared_embedding_column_error(): raise ValueError('SharedEmbeddingColumns are not supported in ' '`linear_model` or `input_layer`. Please use ' - '`FeatureLayer` or `LinearModel` instead.') + '`DenseFeatures` or `LinearModel` instead.') class SharedEmbeddingColumnCreator(tracking.Checkpointable): @@ -3002,7 +3290,7 @@ class SharedEmbeddingColumn( embedding_weights = self.shared_embedding_column_creator.embedding_weights # Return embedding lookup result. - return _safe_embedding_lookup_sparse( + return embedding_ops.safe_embedding_lookup_sparse( embedding_weights=embedding_weights, sparse_ids=sparse_ids, sparse_weights=sparse_weights, @@ -3687,9 +3975,13 @@ class WeightedCategoricalColumn( def transform_feature(self, transformation_cache, state_manager): """Applies weights to tensor generated from `categorical_column`'.""" + print('WeightedCategoricalColumn.transform_feature: ', self.name) + print('Weight feature key: ', self.weight_feature_key) weight_tensor = transformation_cache.get(self.weight_feature_key, state_manager) + print('Weight tensor before: ', weight_tensor) weight_tensor = self._transform_weight_tensor(weight_tensor) + print('Weight tensor after: ', weight_tensor) return (transformation_cache.get(self.categorical_column, state_manager), weight_tensor) @@ -3703,7 +3995,9 @@ class WeightedCategoricalColumn( def get_sparse_tensors(self, transformation_cache, state_manager): """See `CategoricalColumn` base class.""" + print('WeightedCategoricalColumn.get_sparse_tensors: ', self.name) tensors = transformation_cache.get(self, state_manager) + print('tensors[1]: ', tensors[1]) return CategoricalColumn.IdWeightPair(tensors[0], tensors[1]) @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, @@ -3898,142 +4192,6 @@ def _collect_leaf_level_keys(cross): return leaf_level_keys -# TODO(zakaria): Move this to embedding_ops and make it public. -def _safe_embedding_lookup_sparse(embedding_weights, - sparse_ids, - sparse_weights=None, - combiner='mean', - default_id=None, - name=None, - partition_strategy='div', - max_norm=None): - """Lookup embedding results, accounting for invalid IDs and empty features. - - The partitioned embedding in `embedding_weights` must all be the same shape - except for the first dimension. The first dimension is allowed to vary as the - vocabulary size is not necessarily a multiple of `P`. `embedding_weights` - may be a `PartitionedVariable` as returned by using `tf.get_variable()` with a - partitioner. - - Invalid IDs (< 0) are pruned from input IDs and weights, as well as any IDs - with non-positive weight. For an entry with no features, the embedding vector - for `default_id` is returned, or the 0-vector if `default_id` is not supplied. - - The ids and weights may be multi-dimensional. Embeddings are always aggregated - along the last dimension. - - Args: - embedding_weights: A list of `P` float `Tensor`s or values representing - partitioned embedding `Tensor`s. Alternatively, a `PartitionedVariable` - created by partitioning along dimension 0. The total unpartitioned - shape should be `[e_0, e_1, ..., e_m]`, where `e_0` represents the - vocab size and `e_1, ..., e_m` are the embedding dimensions. - sparse_ids: `SparseTensor` of shape `[d_0, d_1, ..., d_n]` containing the - ids. `d_0` is typically batch size. - sparse_weights: `SparseTensor` of same shape as `sparse_ids`, containing - float weights corresponding to `sparse_ids`, or `None` if all weights - are be assumed to be 1.0. - combiner: A string specifying how to combine embedding results for each - entry. Currently "mean", "sqrtn" and "sum" are supported, with "mean" - the default. - default_id: The id to use for an entry with no features. - name: A name for this operation (optional). - partition_strategy: A string specifying the partitioning strategy. - Currently `"div"` and `"mod"` are supported. Default is `"div"`. - max_norm: If not `None`, all embeddings are l2-normalized to max_norm before - combining. - - - Returns: - Dense `Tensor` of shape `[d_0, d_1, ..., d_{n-1}, e_1, ..., e_m]`. - - Raises: - ValueError: if `embedding_weights` is empty. - """ - if embedding_weights is None: - raise ValueError('Missing embedding_weights %s.' % embedding_weights) - if isinstance(embedding_weights, variables.PartitionedVariable): - embedding_weights = list(embedding_weights) # get underlying Variables. - if not isinstance(embedding_weights, list): - embedding_weights = [embedding_weights] - if len(embedding_weights) < 1: - raise ValueError('Missing embedding_weights %s.' % embedding_weights) - - dtype = sparse_weights.dtype if sparse_weights is not None else None - # TODO(rohanj): Look into removing this convert_to_tensor call. - embedding_weights = [ - ops.convert_to_tensor(w, dtype=dtype) for w in embedding_weights - ] - - with ops.name_scope(name, 'embedding_lookup', - embedding_weights + [sparse_ids, - sparse_weights]) as scope: - # Reshape higher-rank sparse ids and weights to linear segment ids. - original_shape = sparse_ids.dense_shape - original_rank_dim = tensor_shape.dimension_value( - sparse_ids.dense_shape.get_shape()[0]) - original_rank = ( - array_ops.size(original_shape) - if original_rank_dim is None - else original_rank_dim) - sparse_ids = sparse_ops.sparse_reshape(sparse_ids, [ - math_ops.reduce_prod( - array_ops.slice(original_shape, [0], [original_rank - 1])), - array_ops.gather(original_shape, original_rank - 1)]) - if sparse_weights is not None: - sparse_weights = sparse_tensor_lib.SparseTensor( - sparse_ids.indices, - sparse_weights.values, sparse_ids.dense_shape) - - # Prune invalid ids and weights. - sparse_ids, sparse_weights = _prune_invalid_ids(sparse_ids, sparse_weights) - if combiner != 'sum': - sparse_ids, sparse_weights = _prune_invalid_weights( - sparse_ids, sparse_weights) - - # Fill in dummy values for empty features, if necessary. - sparse_ids, is_row_empty = sparse_ops.sparse_fill_empty_rows(sparse_ids, - default_id or - 0) - if sparse_weights is not None: - sparse_weights, _ = sparse_ops.sparse_fill_empty_rows(sparse_weights, 1.0) - - result = embedding_ops.embedding_lookup_sparse( - embedding_weights, - sparse_ids, - sparse_weights, - combiner=combiner, - partition_strategy=partition_strategy, - name=None if default_id is None else scope, - max_norm=max_norm) - - if default_id is None: - # Broadcast is_row_empty to the same shape as embedding_lookup_result, - # for use in Select. - is_row_empty = array_ops.tile( - array_ops.reshape(is_row_empty, [-1, 1]), - array_ops.stack([1, array_ops.shape(result)[1]])) - - result = array_ops.where(is_row_empty, - array_ops.zeros_like(result), - result, - name=scope) - - # Reshape back from linear ids back into higher-dimensional dense result. - final_result = array_ops.reshape( - result, - array_ops.concat([ - array_ops.slice( - math_ops.cast(original_shape, dtypes.int32), [0], - [original_rank - 1]), - array_ops.slice(array_ops.shape(result), [1], [-1]) - ], 0)) - final_result.set_shape(tensor_shape.unknown_shape( - (tensor_shape.Dimension(original_rank_dim) - 1).value).concatenate( - result.get_shape()[1:])) - return final_result - - def _prune_invalid_ids(sparse_ids, sparse_weights): """Prune invalid IDs (< 0) from the input ids and weights.""" is_id_valid = math_ops.greater_equal(sparse_ids.values, 0) @@ -4089,10 +4247,14 @@ class IndicatorColumn( sp_ids=id_tensor, sp_values=weight_tensor, vocab_size=int(self._variable_shape[-1])) - # Remove (?, -1) index + # Remove (?, -1) index. weighted_column = sparse_ops.sparse_slice(weighted_column, [0, 0], weighted_column.dense_shape) - return sparse_ops.sparse_tensor_to_dense(weighted_column) + # Use scatter_nd to merge duplicated indices if existed, + # instead of sparse_tensor_to_dense. + return array_ops.scatter_nd(weighted_column.indices, + weighted_column.values, + weighted_column.dense_shape) dense_id_tensor = sparse_ops.sparse_tensor_to_dense( id_tensor, default_value=-1) @@ -4534,7 +4696,10 @@ def deserialize_feature_column(config, 'Expected FeatureColumn class, instead found: {}'.format(cls)) # Always deserialize the FeatureColumn, in order to get the name. - new_instance = cls._from_config(cls_config, columns_by_name=columns_by_name) # pylint: disable=protected-access + new_instance = cls._from_config( # pylint: disable=protected-access + cls_config, + custom_objects=custom_objects, + columns_by_name=columns_by_name) # If the name already exists, re-use the column from columns_by_name, # (new_instance remains unused). diff --git a/tensorflow/python/feature_column/feature_column_v2_test.py b/tensorflow/python/feature_column/feature_column_v2_test.py index a26b8600568..0755c0b6ac2 100644 --- a/tensorflow/python/feature_column/feature_column_v2_test.py +++ b/tensorflow/python/feature_column/feature_column_v2_test.py @@ -31,7 +31,6 @@ from tensorflow.python import keras from tensorflow.python.client import session from tensorflow.python.eager import backprop from tensorflow.python.eager import context -from tensorflow.python.estimator.inputs import numpy_io from tensorflow.python.feature_column import feature_column as fc_old from tensorflow.python.feature_column import feature_column_v2 as fc from tensorflow.python.framework import constant_op @@ -50,6 +49,7 @@ from tensorflow.python.platform import test from tensorflow.python.training import coordinator from tensorflow.python.training import queue_runner_impl from tensorflow.python.training import rmsprop +from tensorflow_estimator.python.estimator.inputs import numpy_io def _initialized_session(config=None): @@ -218,6 +218,7 @@ class LazyColumnTest(test.TestCase): TypeError, '"key" must be either a "str" or "FeatureColumn".'): transformation_cache.get(NotAFeatureColumn(), None) + @test_util.run_deprecated_v1 def test_expand_dim_rank_1_sparse_tensor_empty_batch(self): # empty 1-D sparse tensor: transformation_cache = fc.FeatureTransformationCache( @@ -228,15 +229,16 @@ class LazyColumnTest(test.TestCase): dense_shape=[0], values=np.array([])) }) - with self.cached_session(): - spv = transformation_cache.get('a', None).eval() - self.assertAllEqual(np.array([0, 1], dtype=np.int64), spv.dense_shape) - self.assertAllEqual( - np.reshape(np.array([], dtype=np.int64), (0, 2)), spv.indices) + + spv = self.evaluate(transformation_cache.get('a', None)) + self.assertAllEqual(np.array([0, 1], dtype=np.int64), spv.dense_shape) + self.assertAllEqual( + np.reshape(np.array([], dtype=np.int64), (0, 2)), spv.indices) class NumericColumnTest(test.TestCase): + @test_util.run_deprecated_v1 def test_defaults(self): a = fc.numeric_column('aaa') self.assertEqual('aaa', a.key) @@ -315,59 +317,67 @@ class NumericColumnTest(test.TestCase): 'aaa': parsing_ops.FixedLenFeature((2, 3), dtype=dtypes.int32) }, a.parse_example_spec) + @test_util.run_deprecated_v1 def test_parse_example_no_default_value(self): price = fc.numeric_column('price', shape=[2]) - data = example_pb2.Example(features=feature_pb2.Features( - feature={ - 'price': - feature_pb2.Feature(float_list=feature_pb2.FloatList( - value=[20., 110.])) - })) + data = example_pb2.Example( + features=feature_pb2.Features( + feature={ + 'price': + feature_pb2.Feature( + float_list=feature_pb2.FloatList(value=[20., 110.])) + })) features = parsing_ops.parse_example( serialized=[data.SerializeToString()], - features=fc.make_parse_example_spec([price])) + features=fc.make_parse_example_spec_v2([price])) self.assertIn('price', features) - with self.cached_session(): - self.assertAllEqual([[20., 110.]], features['price'].eval()) + self.assertAllEqual([[20., 110.]], self.evaluate(features['price'])) + + @test_util.run_deprecated_v1 def test_parse_example_with_default_value(self): price = fc.numeric_column('price', shape=[2], default_value=11.) - data = example_pb2.Example(features=feature_pb2.Features( - feature={ - 'price': - feature_pb2.Feature(float_list=feature_pb2.FloatList( - value=[20., 110.])) - })) - no_data = example_pb2.Example(features=feature_pb2.Features( - feature={ - 'something_else': - feature_pb2.Feature(float_list=feature_pb2.FloatList( - value=[20., 110.])) - })) + data = example_pb2.Example( + features=feature_pb2.Features( + feature={ + 'price': + feature_pb2.Feature( + float_list=feature_pb2.FloatList(value=[20., 110.])) + })) + no_data = example_pb2.Example( + features=feature_pb2.Features( + feature={ + 'something_else': + feature_pb2.Feature( + float_list=feature_pb2.FloatList(value=[20., 110.])) + })) features = parsing_ops.parse_example( serialized=[data.SerializeToString(), no_data.SerializeToString()], - features=fc.make_parse_example_spec([price])) + features=fc.make_parse_example_spec_v2([price])) self.assertIn('price', features) - with self.cached_session(): - self.assertAllEqual([[20., 110.], [11., 11.]], features['price'].eval()) + + self.assertAllEqual([[20., 110.], [11., 11.]], + self.evaluate(features['price'])) def test_normalizer_fn_must_be_callable(self): with self.assertRaisesRegexp(TypeError, 'must be a callable'): fc.numeric_column('price', normalizer_fn='NotACallable') + @test_util.run_deprecated_v1 def test_normalizer_fn_transform_feature(self): def _increment_two(input_tensor): return input_tensor + 2. price = fc.numeric_column('price', shape=[2], normalizer_fn=_increment_two) - output = fc._transform_features({ + output = fc._transform_features_v2({ 'price': [[1., 2.], [5., 6.]] }, [price], None) - with self.cached_session(): - self.assertAllEqual([[3., 4.], [7., 8.]], output[price].eval()) + self.assertAllEqual([[3., 4.], [7., 8.]], self.evaluate(output[price])) + + @test_util.run_deprecated_v1 def test_get_dense_tensor(self): def _increment_two(input_tensor): @@ -391,6 +401,7 @@ class NumericColumnTest(test.TestCase): with self.assertRaisesRegexp(ValueError, 'must be a Tensor'): price.transform_feature(transformation_cache, None) + @test_util.run_deprecated_v1 def test_deep_copy(self): a = fc.numeric_column('aaa', shape=[1, 2], default_value=[[3., 2.]]) a_copy = copy.deepcopy(a) @@ -403,6 +414,7 @@ class NumericColumnTest(test.TestCase): 'aaa', shape=[1, 2], default_value=np.array([[3., 2.]])) self.assertEqual(a.default_value, ((3., 2.),)) + @test_util.run_deprecated_v1 def test_linear_model(self): price = fc.numeric_column('price') with ops.Graph().as_default(): @@ -411,11 +423,11 @@ class NumericColumnTest(test.TestCase): predictions = model(features) price_var, bias = model.variables with _initialized_session() as sess: - self.assertAllClose([0.], bias.eval()) - self.assertAllClose([[0.]], price_var.eval()) - self.assertAllClose([[0.], [0.]], predictions.eval()) + self.assertAllClose([0.], self.evaluate(bias)) + self.assertAllClose([[0.]], self.evaluate(price_var)) + self.assertAllClose([[0.], [0.]], self.evaluate(predictions)) sess.run(price_var.assign([[10.]])) - self.assertAllClose([[10.], [50.]], predictions.eval()) + self.assertAllClose([[10.], [50.]], self.evaluate(predictions)) def test_old_linear_model(self): price = fc.numeric_column('price') @@ -425,12 +437,13 @@ class NumericColumnTest(test.TestCase): bias = get_linear_model_bias() price_var = get_linear_model_column_var(price) with _initialized_session() as sess: - self.assertAllClose([0.], bias.eval()) - self.assertAllClose([[0.]], price_var.eval()) - self.assertAllClose([[0.], [0.]], predictions.eval()) + self.assertAllClose([0.], self.evaluate(bias)) + self.assertAllClose([[0.]], self.evaluate(price_var)) + self.assertAllClose([[0.], [0.]], self.evaluate(predictions)) sess.run(price_var.assign([[10.]])) - self.assertAllClose([[10.], [50.]], predictions.eval()) + self.assertAllClose([[10.], [50.]], self.evaluate(predictions)) + @test_util.run_deprecated_v1 def test_serialization(self): def _increment_two(input_tensor): @@ -471,17 +484,17 @@ class BucketizedColumnTest(test.TestCase): def test_invalid_boundaries(self): a = fc.numeric_column('aaa') - with self.assertRaisesRegexp( - ValueError, 'boundaries must be a sorted list'): + with self.assertRaisesRegexp(ValueError, + 'boundaries must be a sorted list'): fc.bucketized_column(a, boundaries=None) - with self.assertRaisesRegexp( - ValueError, 'boundaries must be a sorted list'): + with self.assertRaisesRegexp(ValueError, + 'boundaries must be a sorted list'): fc.bucketized_column(a, boundaries=1.) - with self.assertRaisesRegexp( - ValueError, 'boundaries must be a sorted list'): + with self.assertRaisesRegexp(ValueError, + 'boundaries must be a sorted list'): fc.bucketized_column(a, boundaries=[1, 0]) - with self.assertRaisesRegexp( - ValueError, 'boundaries must be a sorted list'): + with self.assertRaisesRegexp(ValueError, + 'boundaries must be a sorted list'): fc.bucketized_column(a, boundaries=[1, 1]) def test_name(self): @@ -491,7 +504,7 @@ class BucketizedColumnTest(test.TestCase): self.assertEqual('aaa_bucketized', b.name) def test_is_v2_column_old_numeric(self): - a = fc_old.numeric_column('aaa', dtype=dtypes.int32) + a = fc_old._numeric_column('aaa', dtype=dtypes.int32) b = fc.bucketized_column(a, boundaries=[0, 1]) self.assertFalse(b._is_v2_column) self.assertEqual('aaa_bucketized', b.name) @@ -515,32 +528,38 @@ class BucketizedColumnTest(test.TestCase): # Column 'aaa` has shape [2] times three buckets -> num_buckets=6. self.assertEqual(6, b.num_buckets) + @test_util.run_deprecated_v1 def test_parse_example(self): price = fc.numeric_column('price', shape=[2]) bucketized_price = fc.bucketized_column(price, boundaries=[0, 50]) - data = example_pb2.Example(features=feature_pb2.Features( - feature={ - 'price': - feature_pb2.Feature(float_list=feature_pb2.FloatList( - value=[20., 110.])) - })) + data = example_pb2.Example( + features=feature_pb2.Features( + feature={ + 'price': + feature_pb2.Feature( + float_list=feature_pb2.FloatList(value=[20., 110.])) + })) features = parsing_ops.parse_example( serialized=[data.SerializeToString()], - features=fc.make_parse_example_spec([bucketized_price])) + features=fc.make_parse_example_spec_v2([bucketized_price])) self.assertIn('price', features) - with self.cached_session(): - self.assertAllEqual([[20., 110.]], features['price'].eval()) + self.assertAllEqual([[20., 110.]], self.evaluate(features['price'])) + + @test_util.run_deprecated_v1 def test_transform_feature(self): price = fc.numeric_column('price', shape=[2]) bucketized_price = fc.bucketized_column(price, boundaries=[0, 2, 4, 6]) with ops.Graph().as_default(): - transformed_tensor = fc._transform_features({ + transformed_tensor = fc._transform_features_v2({ 'price': [[-1., 1.], [5., 6.]] }, [bucketized_price], None) - with _initialized_session(): - self.assertAllEqual([[0, 1], [3, 4]], - transformed_tensor[bucketized_price].eval()) + + self.evaluate(variables_lib.global_variables_initializer()) + self.evaluate(lookup_ops.tables_initializer()) + + self.assertAllEqual([[0, 1], [3, 4]], + self.evaluate(transformed_tensor[bucketized_price])) def test_get_dense_tensor_one_input_value(self): """Tests _get_dense_tensor() for input with shape=[1].""" @@ -550,16 +569,17 @@ class BucketizedColumnTest(test.TestCase): transformation_cache = fc.FeatureTransformationCache({ 'price': [[-1.], [1.], [5.], [6.]] }) - with _initialized_session(): - bucketized_price_tensor = bucketized_price.get_dense_tensor( - transformation_cache, None) - self.assertAllClose( - # One-hot tensor. - [[[1., 0., 0., 0., 0.]], - [[0., 1., 0., 0., 0.]], - [[0., 0., 0., 1., 0.]], - [[0., 0., 0., 0., 1.]]], - bucketized_price_tensor.eval()) + + self.evaluate(variables_lib.global_variables_initializer()) + self.evaluate(lookup_ops.tables_initializer()) + + bucketized_price_tensor = bucketized_price.get_dense_tensor( + transformation_cache, None) + self.assertAllClose( + # One-hot tensor. + [[[1., 0., 0., 0., 0.]], [[0., 1., 0., 0., 0.]], + [[0., 0., 0., 1., 0.]], [[0., 0., 0., 0., 1.]]], + self.evaluate(bucketized_price_tensor)) def test_get_dense_tensor_two_input_values(self): """Tests _get_dense_tensor() for input with shape=[2].""" @@ -569,14 +589,17 @@ class BucketizedColumnTest(test.TestCase): transformation_cache = fc.FeatureTransformationCache({ 'price': [[-1., 1.], [5., 6.]] }) - with _initialized_session(): - bucketized_price_tensor = bucketized_price.get_dense_tensor( - transformation_cache, None) - self.assertAllClose( - # One-hot tensor. - [[[1., 0., 0., 0., 0.], [0., 1., 0., 0., 0.]], - [[0., 0., 0., 1., 0.], [0., 0., 0., 0., 1.]]], - bucketized_price_tensor.eval()) + + self.evaluate(variables_lib.global_variables_initializer()) + self.evaluate(lookup_ops.tables_initializer()) + + bucketized_price_tensor = bucketized_price.get_dense_tensor( + transformation_cache, None) + self.assertAllClose( + # One-hot tensor. + [[[1., 0., 0., 0., 0.], [0., 1., 0., 0., 0.]], + [[0., 0., 0., 1., 0.], [0., 0., 0., 0., 1.]]], + self.evaluate(bucketized_price_tensor)) def test_get_sparse_tensors_one_input_value(self): """Tests _get_sparse_tensors() for input with shape=[1].""" @@ -591,8 +614,8 @@ class BucketizedColumnTest(test.TestCase): transformation_cache, None) self.assertIsNone(id_weight_pair.weight_tensor) id_tensor_value = sess.run(id_weight_pair.id_tensor) - self.assertAllEqual( - [[0, 0], [1, 0], [2, 0], [3, 0]], id_tensor_value.indices) + self.assertAllEqual([[0, 0], [1, 0], [2, 0], [3, 0]], + id_tensor_value.indices) self.assertAllEqual([0, 1, 3, 4], id_tensor_value.values) self.assertAllEqual([4, 1], id_tensor_value.dense_shape) @@ -609,8 +632,8 @@ class BucketizedColumnTest(test.TestCase): transformation_cache, None) self.assertIsNone(id_weight_pair.weight_tensor) id_tensor_value = sess.run(id_weight_pair.id_tensor) - self.assertAllEqual( - [[0, 0], [0, 1], [1, 0], [1, 1]], id_tensor_value.indices) + self.assertAllEqual([[0, 0], [0, 1], [1, 0], [1, 1]], + id_tensor_value.indices) # Values 0-4 correspond to the first column of the input price. # Values 5-9 correspond to the second column of the input price. self.assertAllEqual([0, 6, 3, 9], id_tensor_value.values) @@ -627,6 +650,7 @@ class BucketizedColumnTest(test.TestCase): with self.assertRaisesRegexp(ValueError, 'must be a Tensor'): bucketized_price.transform_feature(transformation_cache, None) + @test_util.run_deprecated_v1 def test_deep_copy(self): a = fc.numeric_column('aaa', shape=[2]) a_bucketized = fc.bucketized_column(a, boundaries=[0, 1]) @@ -645,20 +669,23 @@ class BucketizedColumnTest(test.TestCase): predictions = model(features) bucketized_price_var, bias = model.variables with _initialized_session() as sess: - self.assertAllClose([0.], bias.eval()) + self.assertAllClose([0.], self.evaluate(bias)) # One weight variable per bucket, all initialized to zero. - self.assertAllClose( - [[0.], [0.], [0.], [0.], [0.]], bucketized_price_var.eval()) - self.assertAllClose([[0.], [0.], [0.], [0.]], predictions.eval()) - sess.run(bucketized_price_var.assign( - [[10.], [20.], [30.], [40.], [50.]])) + self.assertAllClose([[0.], [0.], [0.], [0.], [0.]], + self.evaluate(bucketized_price_var)) + self.assertAllClose([[0.], [0.], [0.], [0.]], + self.evaluate(predictions)) + sess.run( + bucketized_price_var.assign([[10.], [20.], [30.], [40.], [50.]])) # price -1. is in the 0th bucket, whose weight is 10. # price 1. is in the 1st bucket, whose weight is 20. # price 5. is in the 3rd bucket, whose weight is 40. # price 6. is in the 4th bucket, whose weight is 50. - self.assertAllClose([[10.], [20.], [40.], [50.]], predictions.eval()) + self.assertAllClose([[10.], [20.], [40.], [50.]], + self.evaluate(predictions)) sess.run(bias.assign([1.])) - self.assertAllClose([[11.], [21.], [41.], [51.]], predictions.eval()) + self.assertAllClose([[11.], [21.], [41.], [51.]], + self.evaluate(predictions)) def test_linear_model_two_input_values(self): """Tests linear_model() for input with shape=[2].""" @@ -670,24 +697,24 @@ class BucketizedColumnTest(test.TestCase): predictions = model(features) bucketized_price_var, bias = model.variables with _initialized_session() as sess: - self.assertAllClose([0.], bias.eval()) + self.assertAllClose([0.], self.evaluate(bias)) # One weight per bucket per input column, all initialized to zero. self.assertAllClose( [[0.], [0.], [0.], [0.], [0.], [0.], [0.], [0.], [0.], [0.]], - bucketized_price_var.eval()) - self.assertAllClose([[0.], [0.]], predictions.eval()) - sess.run(bucketized_price_var.assign( - [[10.], [20.], [30.], [40.], [50.], - [60.], [70.], [80.], [90.], [100.]])) + self.evaluate(bucketized_price_var)) + self.assertAllClose([[0.], [0.]], self.evaluate(predictions)) + sess.run( + bucketized_price_var.assign([[10.], [20.], [30.], [40.], [50.], + [60.], [70.], [80.], [90.], [100.]])) # 1st example: # price -1. is in the 0th bucket, whose weight is 10. # price 1. is in the 6th bucket, whose weight is 70. # 2nd example: # price 5. is in the 3rd bucket, whose weight is 40. # price 6. is in the 9th bucket, whose weight is 100. - self.assertAllClose([[80.], [140.]], predictions.eval()) + self.assertAllClose([[80.], [140.]], self.evaluate(predictions)) sess.run(bias.assign([1.])) - self.assertAllClose([[81.], [141.]], predictions.eval()) + self.assertAllClose([[81.], [141.]], self.evaluate(predictions)) def test_old_linear_model_one_input_value(self): """Tests linear_model() for input with shape=[1].""" @@ -699,20 +726,23 @@ class BucketizedColumnTest(test.TestCase): bias = get_linear_model_bias() bucketized_price_var = get_linear_model_column_var(bucketized_price) with _initialized_session() as sess: - self.assertAllClose([0.], bias.eval()) + self.assertAllClose([0.], self.evaluate(bias)) # One weight variable per bucket, all initialized to zero. self.assertAllClose([[0.], [0.], [0.], [0.], [0.]], - bucketized_price_var.eval()) - self.assertAllClose([[0.], [0.], [0.], [0.]], predictions.eval()) + self.evaluate(bucketized_price_var)) + self.assertAllClose([[0.], [0.], [0.], [0.]], + self.evaluate(predictions)) sess.run( bucketized_price_var.assign([[10.], [20.], [30.], [40.], [50.]])) # price -1. is in the 0th bucket, whose weight is 10. # price 1. is in the 1st bucket, whose weight is 20. # price 5. is in the 3rd bucket, whose weight is 40. # price 6. is in the 4th bucket, whose weight is 50. - self.assertAllClose([[10.], [20.], [40.], [50.]], predictions.eval()) + self.assertAllClose([[10.], [20.], [40.], [50.]], + self.evaluate(predictions)) sess.run(bias.assign([1.])) - self.assertAllClose([[11.], [21.], [41.], [51.]], predictions.eval()) + self.assertAllClose([[11.], [21.], [41.], [51.]], + self.evaluate(predictions)) def test_old_linear_model_two_input_values(self): """Tests linear_model() for input with shape=[2].""" @@ -724,12 +754,12 @@ class BucketizedColumnTest(test.TestCase): bias = get_linear_model_bias() bucketized_price_var = get_linear_model_column_var(bucketized_price) with _initialized_session() as sess: - self.assertAllClose([0.], bias.eval()) + self.assertAllClose([0.], self.evaluate(bias)) # One weight per bucket per input column, all initialized to zero. self.assertAllClose( [[0.], [0.], [0.], [0.], [0.], [0.], [0.], [0.], [0.], [0.]], - bucketized_price_var.eval()) - self.assertAllClose([[0.], [0.]], predictions.eval()) + self.evaluate(bucketized_price_var)) + self.assertAllClose([[0.], [0.]], self.evaluate(predictions)) sess.run( bucketized_price_var.assign([[10.], [20.], [30.], [40.], [50.], [60.], [70.], [80.], [90.], [100.]])) @@ -739,13 +769,13 @@ class BucketizedColumnTest(test.TestCase): # 2nd example: # price 5. is in the 3rd bucket, whose weight is 40. # price 6. is in the 9th bucket, whose weight is 100. - self.assertAllClose([[80.], [140.]], predictions.eval()) + self.assertAllClose([[80.], [140.]], self.evaluate(predictions)) sess.run(bias.assign([1.])) - self.assertAllClose([[81.], [141.]], predictions.eval()) + self.assertAllClose([[81.], [141.]], self.evaluate(predictions)) def test_old_linear_model_one_input_value_old_numeric(self): """Tests linear_model() for input with shape=[1].""" - price = fc_old.numeric_column('price', shape=[1]) + price = fc_old._numeric_column('price', shape=[1]) bucketized_price = fc.bucketized_column(price, boundaries=[0, 2, 4, 6]) with ops.Graph().as_default(): features = {'price': [[-1.], [1.], [5.], [6.]]} @@ -753,21 +783,25 @@ class BucketizedColumnTest(test.TestCase): bias = get_linear_model_bias() bucketized_price_var = get_linear_model_column_var(bucketized_price) with _initialized_session() as sess: - self.assertAllClose([0.], bias.eval()) + self.assertAllClose([0.], self.evaluate(bias)) # One weight variable per bucket, all initialized to zero. self.assertAllClose([[0.], [0.], [0.], [0.], [0.]], - bucketized_price_var.eval()) - self.assertAllClose([[0.], [0.], [0.], [0.]], predictions.eval()) + self.evaluate(bucketized_price_var)) + self.assertAllClose([[0.], [0.], [0.], [0.]], + self.evaluate(predictions)) sess.run( bucketized_price_var.assign([[10.], [20.], [30.], [40.], [50.]])) # price -1. is in the 0th bucket, whose weight is 10. # price 1. is in the 1st bucket, whose weight is 20. # price 5. is in the 3rd bucket, whose weight is 40. # price 6. is in the 4th bucket, whose weight is 50. - self.assertAllClose([[10.], [20.], [40.], [50.]], predictions.eval()) + self.assertAllClose([[10.], [20.], [40.], [50.]], + self.evaluate(predictions)) sess.run(bias.assign([1.])) - self.assertAllClose([[11.], [21.], [41.], [51.]], predictions.eval()) + self.assertAllClose([[11.], [21.], [41.], [51.]], + self.evaluate(predictions)) + @test_util.run_deprecated_v1 def test_serialization(self): price = fc.numeric_column('price', shape=[2]) bucketized_price = fc.bucketized_column(price, boundaries=[0, 2, 4, 6]) @@ -800,6 +834,7 @@ class BucketizedColumnTest(test.TestCase): class HashedCategoricalColumnTest(test.TestCase): + @test_util.run_deprecated_v1 def test_defaults(self): a = fc.categorical_column_with_hash_bucket('aaa', 10) self.assertEqual('aaa', a.name) @@ -827,6 +862,7 @@ class HashedCategoricalColumnTest(test.TestCase): with self.assertRaisesRegexp(ValueError, 'dtype must be string or integer'): fc.categorical_column_with_hash_bucket('aaa', 10, dtype=dtypes.float32) + @test_util.run_deprecated_v1 def test_deep_copy(self): original = fc.categorical_column_with_hash_bucket('aaa', 10) for column in (original, copy.deepcopy(original)): @@ -847,45 +883,50 @@ class HashedCategoricalColumnTest(test.TestCase): 'aaa': parsing_ops.VarLenFeature(dtypes.int32) }, a.parse_example_spec) + @test_util.run_deprecated_v1 def test_parse_example(self): a = fc.categorical_column_with_hash_bucket('aaa', 10) - data = example_pb2.Example(features=feature_pb2.Features( - feature={ - 'aaa': - feature_pb2.Feature(bytes_list=feature_pb2.BytesList( - value=[b'omar', b'stringer'])) - })) + data = example_pb2.Example( + features=feature_pb2.Features( + feature={ + 'aaa': + feature_pb2.Feature( + bytes_list=feature_pb2.BytesList( + value=[b'omar', b'stringer'])) + })) features = parsing_ops.parse_example( serialized=[data.SerializeToString()], - features=fc.make_parse_example_spec([a])) + features=fc.make_parse_example_spec_v2([a])) self.assertIn('aaa', features) - with self.cached_session(): - _assert_sparse_tensor_value( - self, - sparse_tensor.SparseTensorValue( - indices=[[0, 0], [0, 1]], - values=np.array([b'omar', b'stringer'], dtype=np.object_), - dense_shape=[1, 2]), - features['aaa'].eval()) + _assert_sparse_tensor_value( + self, + sparse_tensor.SparseTensorValue( + indices=[[0, 0], [0, 1]], + values=np.array([b'omar', b'stringer'], dtype=np.object_), + dense_shape=[1, 2]), self.evaluate(features['aaa'])) + + @test_util.run_deprecated_v1 def test_strings_should_be_hashed(self): hashed_sparse = fc.categorical_column_with_hash_bucket('wire', 10) wire_tensor = sparse_tensor.SparseTensor( values=['omar', 'stringer', 'marlo'], indices=[[0, 0], [1, 0], [1, 1]], dense_shape=[2, 2]) - outputs = fc._transform_features({ + outputs = fc._transform_features_v2({ 'wire': wire_tensor }, [hashed_sparse], None) output = outputs[hashed_sparse] # Check exact hashed output. If hashing changes this test will break. expected_values = [6, 4, 1] - with self.cached_session(): - self.assertEqual(dtypes.int64, output.values.dtype) - self.assertAllEqual(expected_values, output.values.eval()) - self.assertAllEqual(wire_tensor.indices.eval(), output.indices.eval()) - self.assertAllEqual(wire_tensor.dense_shape.eval(), - output.dense_shape.eval()) + + self.assertEqual(dtypes.int64, output.values.dtype) + self.assertAllEqual(expected_values, self.evaluate(output.values)) + self.assertAllEqual( + self.evaluate(wire_tensor.indices), self.evaluate(output.indices)) + self.assertAllEqual( + self.evaluate(wire_tensor.dense_shape), + self.evaluate(output.dense_shape)) def test_tensor_dtype_should_be_string_or_integer(self): string_fc = fc.categorical_column_with_hash_bucket( @@ -895,17 +936,11 @@ class HashedCategoricalColumnTest(test.TestCase): float_fc = fc.categorical_column_with_hash_bucket( 'a_float', 10, dtype=dtypes.string) int_tensor = sparse_tensor.SparseTensor( - values=[101], - indices=[[0, 0]], - dense_shape=[1, 1]) + values=[101], indices=[[0, 0]], dense_shape=[1, 1]) string_tensor = sparse_tensor.SparseTensor( - values=['101'], - indices=[[0, 0]], - dense_shape=[1, 1]) + values=['101'], indices=[[0, 0]], dense_shape=[1, 1]) float_tensor = sparse_tensor.SparseTensor( - values=[101.], - indices=[[0, 0]], - dense_shape=[1, 1]) + values=[101.], indices=[[0, 0]], dense_shape=[1, 1]) transformation_cache = fc.FeatureTransformationCache({ 'a_int': int_tensor, 'a_string': string_tensor, @@ -925,6 +960,7 @@ class HashedCategoricalColumnTest(test.TestCase): with self.assertRaisesRegexp(ValueError, 'dtype must be compatible'): transformation_cache.get(hashed_sparse, None) + @test_util.run_deprecated_v1 def test_ints_should_be_hashed(self): hashed_sparse = fc.categorical_column_with_hash_bucket( 'wire', 10, dtype=dtypes.int64) @@ -936,9 +972,10 @@ class HashedCategoricalColumnTest(test.TestCase): output = transformation_cache.get(hashed_sparse, None) # Check exact hashed output. If hashing changes this test will break. expected_values = [3, 7, 5] - with self.cached_session(): - self.assertAllEqual(expected_values, output.values.eval()) + self.assertAllEqual(expected_values, self.evaluate(output.values)) + + @test_util.run_deprecated_v1 def test_int32_64_is_compatible(self): hashed_sparse = fc.categorical_column_with_hash_bucket( 'wire', 10, dtype=dtypes.int64) @@ -950,9 +987,10 @@ class HashedCategoricalColumnTest(test.TestCase): output = transformation_cache.get(hashed_sparse, None) # Check exact hashed output. If hashing changes this test will break. expected_values = [3, 7, 5] - with self.cached_session(): - self.assertAllEqual(expected_values, output.values.eval()) + self.assertAllEqual(expected_values, self.evaluate(output.values)) + + @test_util.run_deprecated_v1 def test_get_sparse_tensors(self): hashed_sparse = fc.categorical_column_with_hash_bucket('wire', 10) transformation_cache = fc.FeatureTransformationCache({ @@ -968,6 +1006,7 @@ class HashedCategoricalColumnTest(test.TestCase): self.assertEqual( transformation_cache.get(hashed_sparse, None), id_weight_pair.id_tensor) + @test_util.run_deprecated_v1 def test_get_sparse_tensors_dense_input(self): hashed_sparse = fc.categorical_column_with_hash_bucket('wire', 10) transformation_cache = fc.FeatureTransformationCache({ @@ -979,6 +1018,7 @@ class HashedCategoricalColumnTest(test.TestCase): self.assertEqual( transformation_cache.get(hashed_sparse, None), id_weight_pair.id_tensor) + @test_util.run_deprecated_v1 def test_linear_model(self): wire_column = fc.categorical_column_with_hash_bucket('wire', 4) self.assertEqual(4, wire_column.num_buckets) @@ -992,14 +1032,17 @@ class HashedCategoricalColumnTest(test.TestCase): dense_shape=(2, 2)) }) wire_var, bias = model.variables - with _initialized_session(): - self.assertAllClose((0.,), bias.eval()) - self.assertAllClose(((0.,), (0.,), (0.,), (0.,)), wire_var.eval()) - self.assertAllClose(((0.,), (0.,)), predictions.eval()) - wire_var.assign(((1.,), (2.,), (3.,), (4.,))).eval() - # 'marlo' -> 3: wire_var[3] = 4 - # 'skywalker' -> 2, 'omar' -> 2: wire_var[2] + wire_var[2] = 3+3 = 6 - self.assertAllClose(((4.,), (6.,)), predictions.eval()) + + self.evaluate(variables_lib.global_variables_initializer()) + self.evaluate(lookup_ops.tables_initializer()) + + self.assertAllClose((0.,), self.evaluate(bias)) + self.assertAllClose(((0.,), (0.,), (0.,), (0.,)), self.evaluate(wire_var)) + self.assertAllClose(((0.,), (0.,)), self.evaluate(predictions)) + self.evaluate(wire_var.assign(((1.,), (2.,), (3.,), (4.,)))) + # 'marlo' -> 3: wire_var[3] = 4 + # 'skywalker' -> 2, 'omar' -> 2: wire_var[2] + wire_var[2] = 3+3 = 6 + self.assertAllClose(((4.,), (6.,)), self.evaluate(predictions)) def test_old_linear_model(self): wire_column = fc.categorical_column_with_hash_bucket('wire', 4) @@ -1014,15 +1057,19 @@ class HashedCategoricalColumnTest(test.TestCase): }, (wire_column,)) bias = get_linear_model_bias() wire_var = get_linear_model_column_var(wire_column) - with _initialized_session(): - self.assertAllClose((0.,), bias.eval()) - self.assertAllClose(((0.,), (0.,), (0.,), (0.,)), wire_var.eval()) - self.assertAllClose(((0.,), (0.,)), predictions.eval()) - wire_var.assign(((1.,), (2.,), (3.,), (4.,))).eval() - # 'marlo' -> 3: wire_var[3] = 4 - # 'skywalker' -> 2, 'omar' -> 2: wire_var[2] + wire_var[2] = 3+3 = 6 - self.assertAllClose(((4.,), (6.,)), predictions.eval()) + self.evaluate(variables_lib.global_variables_initializer()) + self.evaluate(lookup_ops.tables_initializer()) + + self.assertAllClose((0.,), self.evaluate(bias)) + self.assertAllClose(((0.,), (0.,), (0.,), (0.,)), self.evaluate(wire_var)) + self.assertAllClose(((0.,), (0.,)), self.evaluate(predictions)) + self.evaluate(wire_var.assign(((1.,), (2.,), (3.,), (4.,)))) + # 'marlo' -> 3: wire_var[3] = 4 + # 'skywalker' -> 2, 'omar' -> 2: wire_var[2] + wire_var[2] = 3+3 = 6 + self.assertAllClose(((4.,), (6.,)), self.evaluate(predictions)) + + @test_util.run_deprecated_v1 def test_serialization(self): wire_column = fc.categorical_column_with_hash_bucket('wire', 4) self.assertEqual(['wire'], wire_column.parents) @@ -1041,13 +1088,13 @@ class HashedCategoricalColumnTest(test.TestCase): class CrossedColumnTest(test.TestCase): def test_keys_empty(self): - with self.assertRaisesRegexp( - ValueError, 'keys must be a list with length > 1'): + with self.assertRaisesRegexp(ValueError, + 'keys must be a list with length > 1'): fc.crossed_column([], 10) def test_keys_length_one(self): - with self.assertRaisesRegexp( - ValueError, 'keys must be a list with length > 1'): + with self.assertRaisesRegexp(ValueError, + 'keys must be a list with length > 1'): fc.crossed_column(['a'], 10) def test_key_type_unsupported(self): @@ -1060,18 +1107,15 @@ class CrossedColumnTest(test.TestCase): ['a', fc.categorical_column_with_hash_bucket('c', 10)], 10) def test_hash_bucket_size_negative(self): - with self.assertRaisesRegexp( - ValueError, 'hash_bucket_size must be > 1'): + with self.assertRaisesRegexp(ValueError, 'hash_bucket_size must be > 1'): fc.crossed_column(['a', 'c'], -1) def test_hash_bucket_size_zero(self): - with self.assertRaisesRegexp( - ValueError, 'hash_bucket_size must be > 1'): + with self.assertRaisesRegexp(ValueError, 'hash_bucket_size must be > 1'): fc.crossed_column(['a', 'c'], 0) def test_hash_bucket_size_none(self): - with self.assertRaisesRegexp( - ValueError, 'hash_bucket_size must be > 1'): + with self.assertRaisesRegexp(ValueError, 'hash_bucket_size must be > 1'): fc.crossed_column(['a', 'c'], None) def test_name(self): @@ -1085,7 +1129,7 @@ class CrossedColumnTest(test.TestCase): self.assertEqual('a_bucketized_X_c_X_d1_X_d2', crossed2.name) def test_is_v2_column(self): - a = fc_old.numeric_column('a', dtype=dtypes.int32) + a = fc_old._numeric_column('a', dtype=dtypes.int32) b = fc.bucketized_column(a, boundaries=[0, 1]) crossed1 = fc.crossed_column(['d1', 'd2'], 10) self.assertTrue(crossed1._is_v2_column) @@ -1127,65 +1171,76 @@ class CrossedColumnTest(test.TestCase): crossed = fc.crossed_column([b, 'c'], 15) self.assertEqual(15, crossed.num_buckets) + @test_util.run_deprecated_v1 def test_deep_copy(self): a = fc.numeric_column('a', dtype=dtypes.int32) b = fc.bucketized_column(a, boundaries=[0, 1]) crossed1 = fc.crossed_column(['d1', 'd2'], 10) crossed2 = fc.crossed_column([b, 'c', crossed1], 15, hash_key=5) crossed2_copy = copy.deepcopy(crossed2) - self.assertEqual('a_bucketized_X_c_X_d1_X_d2', crossed2_copy.name,) + self.assertEqual( + 'a_bucketized_X_c_X_d1_X_d2', + crossed2_copy.name, + ) self.assertEqual(15, crossed2_copy.hash_bucket_size) self.assertEqual(5, crossed2_copy.hash_key) + @test_util.run_deprecated_v1 def test_parse_example(self): price = fc.numeric_column('price', shape=[2]) bucketized_price = fc.bucketized_column(price, boundaries=[0, 50]) price_cross_wire = fc.crossed_column([bucketized_price, 'wire'], 10) - data = example_pb2.Example(features=feature_pb2.Features( - feature={ - 'price': - feature_pb2.Feature(float_list=feature_pb2.FloatList( - value=[20., 110.])), - 'wire': - feature_pb2.Feature(bytes_list=feature_pb2.BytesList( - value=[b'omar', b'stringer'])), - })) + data = example_pb2.Example( + features=feature_pb2.Features( + feature={ + 'price': + feature_pb2.Feature( + float_list=feature_pb2.FloatList(value=[20., 110.])), + 'wire': + feature_pb2.Feature( + bytes_list=feature_pb2.BytesList( + value=[b'omar', b'stringer'])), + })) features = parsing_ops.parse_example( serialized=[data.SerializeToString()], - features=fc.make_parse_example_spec([price_cross_wire])) + features=fc.make_parse_example_spec_v2([price_cross_wire])) self.assertIn('price', features) self.assertIn('wire', features) - with self.cached_session(): - self.assertAllEqual([[20., 110.]], features['price'].eval()) - wire_sparse = features['wire'] - self.assertAllEqual([[0, 0], [0, 1]], wire_sparse.indices.eval()) - # Use byte constants to pass the open-source test. - self.assertAllEqual([b'omar', b'stringer'], wire_sparse.values.eval()) - self.assertAllEqual([1, 2], wire_sparse.dense_shape.eval()) + self.assertAllEqual([[20., 110.]], self.evaluate(features['price'])) + wire_sparse = features['wire'] + self.assertAllEqual([[0, 0], [0, 1]], self.evaluate(wire_sparse.indices)) + # Use byte constants to pass the open-source test. + self.assertAllEqual([b'omar', b'stringer'], + self.evaluate(wire_sparse.values)) + self.assertAllEqual([1, 2], self.evaluate(wire_sparse.dense_shape)) + + @test_util.run_deprecated_v1 def test_transform_feature(self): price = fc.numeric_column('price', shape=[2]) bucketized_price = fc.bucketized_column(price, boundaries=[0, 50]) hash_bucket_size = 10 - price_cross_wire = fc.crossed_column( - [bucketized_price, 'wire'], hash_bucket_size) + price_cross_wire = fc.crossed_column([bucketized_price, 'wire'], + hash_bucket_size) features = { - 'price': constant_op.constant([[1., 2.], [5., 6.]]), - 'wire': sparse_tensor.SparseTensor( - values=['omar', 'stringer', 'marlo'], - indices=[[0, 0], [1, 0], [1, 1]], - dense_shape=[2, 2]), + 'price': + constant_op.constant([[1., 2.], [5., 6.]]), + 'wire': + sparse_tensor.SparseTensor( + values=['omar', 'stringer', 'marlo'], + indices=[[0, 0], [1, 0], [1, 1]], + dense_shape=[2, 2]), } - outputs = fc._transform_features(features, [price_cross_wire], None) + outputs = fc._transform_features_v2(features, [price_cross_wire], None) output = outputs[price_cross_wire] - with self.cached_session() as sess: - output_val = sess.run(output) - self.assertAllEqual( - [[0, 0], [0, 1], [1, 0], [1, 1], [1, 2], [1, 3]], output_val.indices) - for val in output_val.values: - self.assertIn(val, list(range(hash_bucket_size))) - self.assertAllEqual([2, 4], output_val.dense_shape) + output_val = self.evaluate(output) + self.assertAllEqual([[0, 0], [0, 1], [1, 0], [1, 1], [1, 2], [1, 3]], + output_val.indices) + for val in output_val.values: + self.assertIn(val, list(range(hash_bucket_size))) + self.assertAllEqual([2, 4], output_val.dense_shape) + @test_util.run_deprecated_v1 def test_get_sparse_tensors(self): a = fc.numeric_column('a', dtype=dtypes.int32, shape=(2,)) b = fc.bucketized_column(a, boundaries=(0, 1)) @@ -1212,19 +1267,21 @@ class CrossedColumnTest(test.TestCase): dense_shape=(2, 2)), }) id_weight_pair = crossed2.get_sparse_tensors(transformation_cache, None) - with _initialized_session(): - id_tensor_eval = id_weight_pair.id_tensor.eval() - self.assertAllEqual( - ((0, 0), (0, 1), (1, 0), (1, 1), (1, 2), (1, 3), (1, 4), (1, 5), - (1, 6), (1, 7), (1, 8), (1, 9), (1, 10), (1, 11), (1, 12), (1, 13), - (1, 14), (1, 15)), - id_tensor_eval.indices) - # Check exact hashed output. If hashing changes this test will break. - # All values are within [0, hash_bucket_size). - expected_values = ( - 6, 14, 0, 13, 8, 8, 10, 12, 2, 0, 1, 9, 8, 12, 2, 0, 10, 11) - self.assertAllEqual(expected_values, id_tensor_eval.values) - self.assertAllEqual((2, 16), id_tensor_eval.dense_shape) + + self.evaluate(variables_lib.global_variables_initializer()) + self.evaluate(lookup_ops.tables_initializer()) + + id_tensor_eval = self.evaluate(id_weight_pair.id_tensor) + self.assertAllEqual( + ((0, 0), (0, 1), (1, 0), (1, 1), (1, 2), (1, 3), (1, 4), (1, 5), + (1, 6), (1, 7), (1, 8), (1, 9), (1, 10), (1, 11), (1, 12), (1, 13), + (1, 14), (1, 15)), id_tensor_eval.indices) + # Check exact hashed output. If hashing changes this test will break. + # All values are within [0, hash_bucket_size). + expected_values = (6, 14, 0, 13, 8, 8, 10, 12, 2, 0, 1, 9, 8, 12, 2, 0, + 10, 11) + self.assertAllEqual(expected_values, id_tensor_eval.values) + self.assertAllEqual((2, 16), id_tensor_eval.dense_shape) def test_get_sparse_tensors_simple(self): """Same as test_get_sparse_tensors, but with simpler values.""" @@ -1242,17 +1299,20 @@ class CrossedColumnTest(test.TestCase): dense_shape=(2, 2)), }) id_weight_pair = crossed.get_sparse_tensors(transformation_cache, None) - with _initialized_session(): - id_tensor_eval = id_weight_pair.id_tensor.eval() - self.assertAllEqual( - ((0, 0), (0, 1), (1, 0), (1, 1), (1, 2), (1, 3)), - id_tensor_eval.indices) - # Check exact hashed output. If hashing changes this test will break. - # All values are within [0, hash_bucket_size). - expected_values = (1, 0, 1, 3, 4, 2) - self.assertAllEqual(expected_values, id_tensor_eval.values) - self.assertAllEqual((2, 4), id_tensor_eval.dense_shape) + self.evaluate(variables_lib.global_variables_initializer()) + self.evaluate(lookup_ops.tables_initializer()) + + id_tensor_eval = self.evaluate(id_weight_pair.id_tensor) + self.assertAllEqual(((0, 0), (0, 1), (1, 0), (1, 1), (1, 2), (1, 3)), + id_tensor_eval.indices) + # Check exact hashed output. If hashing changes this test will break. + # All values are within [0, hash_bucket_size). + expected_values = (1, 0, 1, 3, 4, 2) + self.assertAllEqual(expected_values, id_tensor_eval.values) + self.assertAllEqual((2, 4), id_tensor_eval.dense_shape) + + @test_util.run_deprecated_v1 def test_linear_model(self): """Tests linear_model. @@ -1274,15 +1334,15 @@ class CrossedColumnTest(test.TestCase): }) crossed_var, bias = model.variables with _initialized_session() as sess: - self.assertAllClose((0.,), bias.eval()) - self.assertAllClose( - ((0.,), (0.,), (0.,), (0.,), (0.,)), crossed_var.eval()) - self.assertAllClose(((0.,), (0.,)), predictions.eval()) + self.assertAllClose((0.,), self.evaluate(bias)) + self.assertAllClose(((0.,), (0.,), (0.,), (0.,), (0.,)), + self.evaluate(crossed_var)) + self.assertAllClose(((0.,), (0.,)), self.evaluate(predictions)) sess.run(crossed_var.assign(((1.,), (2.,), (3.,), (4.,), (5.,)))) # Expected ids after cross = (1, 0, 1, 3, 4, 2) - self.assertAllClose(((3.,), (14.,)), predictions.eval()) + self.assertAllClose(((3.,), (14.,)), self.evaluate(predictions)) sess.run(bias.assign((.1,))) - self.assertAllClose(((3.1,), (14.1,)), predictions.eval()) + self.assertAllClose(((3.1,), (14.1,)), self.evaluate(predictions)) def test_linear_model_with_weights(self): @@ -1301,10 +1361,11 @@ class CrossedColumnTest(test.TestCase): @property def parse_example_spec(self): return { - self.name: parsing_ops.VarLenFeature(dtypes.int32), - '{}_weights'.format(self.name): parsing_ops.VarLenFeature( - dtypes.float32), - } + self.name: + parsing_ops.VarLenFeature(dtypes.int32), + '{}_weights'.format(self.name): + parsing_ops.VarLenFeature(dtypes.float32), + } @property def num_buckets(self): @@ -1367,15 +1428,15 @@ class CrossedColumnTest(test.TestCase): bias = get_linear_model_bias() crossed_var = get_linear_model_column_var(crossed) with _initialized_session() as sess: - self.assertAllClose((0.,), bias.eval()) + self.assertAllClose((0.,), self.evaluate(bias)) self.assertAllClose(((0.,), (0.,), (0.,), (0.,), (0.,)), - crossed_var.eval()) - self.assertAllClose(((0.,), (0.,)), predictions.eval()) + self.evaluate(crossed_var)) + self.assertAllClose(((0.,), (0.,)), self.evaluate(predictions)) sess.run(crossed_var.assign(((1.,), (2.,), (3.,), (4.,), (5.,)))) # Expected ids after cross = (1, 0, 1, 3, 4, 2) - self.assertAllClose(((3.,), (14.,)), predictions.eval()) + self.assertAllClose(((3.,), (14.,)), self.evaluate(predictions)) sess.run(bias.assign((.1,))) - self.assertAllClose(((3.1,), (14.1,)), predictions.eval()) + self.assertAllClose(((3.1,), (14.1,)), self.evaluate(predictions)) def test_old_linear_model_with_weights(self): @@ -1461,7 +1522,7 @@ class CrossedColumnTest(test.TestCase): Uses data from test_get_sparse_tesnsors_simple. """ - a = fc_old.numeric_column('a', dtype=dtypes.int32, shape=(2,)) + a = fc_old._numeric_column('a', dtype=dtypes.int32, shape=(2,)) b = fc.bucketized_column(a, boundaries=(0, 1)) crossed = fc.crossed_column([b, 'c'], hash_bucket_size=5, hash_key=5) with ops.Graph().as_default(): @@ -1477,16 +1538,17 @@ class CrossedColumnTest(test.TestCase): bias = get_linear_model_bias() crossed_var = get_linear_model_column_var(crossed) with _initialized_session() as sess: - self.assertAllClose((0.,), bias.eval()) + self.assertAllClose((0.,), self.evaluate(bias)) self.assertAllClose(((0.,), (0.,), (0.,), (0.,), (0.,)), - crossed_var.eval()) - self.assertAllClose(((0.,), (0.,)), predictions.eval()) + self.evaluate(crossed_var)) + self.assertAllClose(((0.,), (0.,)), self.evaluate(predictions)) sess.run(crossed_var.assign(((1.,), (2.,), (3.,), (4.,), (5.,)))) # Expected ids after cross = (1, 0, 1, 3, 4, 2) - self.assertAllClose(((3.,), (14.,)), predictions.eval()) + self.assertAllClose(((3.,), (14.,)), self.evaluate(predictions)) sess.run(bias.assign((.1,))) - self.assertAllClose(((3.1,), (14.1,)), predictions.eval()) + self.assertAllClose(((3.1,), (14.1,)), self.evaluate(predictions)) + @test_util.run_deprecated_v1 def test_serialization(self): a = fc.numeric_column('a', dtype=dtypes.int32, shape=(2,)) b = fc.bucketized_column(a, boundaries=(0, 1)) @@ -1528,7 +1590,6 @@ class CrossedColumnTest(test.TestCase): self.assertIs(b, new_crossed.keys[0]) - class LinearModelTest(test.TestCase): def test_raises_if_empty_feature_columns(self): @@ -1581,7 +1642,7 @@ class LinearModelTest(test.TestCase): features = [[1.], [5.]] model = fc.LinearModel([price]) with self.assertRaisesRegexp(ValueError, 'We expected a dictionary here'): - predictions = model(features) + model(features) def test_dense_bias(self): price = fc.numeric_column('price') @@ -1591,10 +1652,10 @@ class LinearModelTest(test.TestCase): predictions = model(features) price_var, bias = model.variables with _initialized_session() as sess: - self.assertAllClose([0.], bias.eval()) + self.assertAllClose([0.], self.evaluate(bias)) sess.run(price_var.assign([[10.]])) sess.run(bias.assign([5.])) - self.assertAllClose([[15.], [55.]], predictions.eval()) + self.assertAllClose([[15.], [55.]], self.evaluate(predictions)) def test_sparse_bias(self): wire_cast = fc.categorical_column_with_hash_bucket('wire_cast', 4) @@ -1608,11 +1669,12 @@ class LinearModelTest(test.TestCase): predictions = model(features) wire_cast_var, bias = model.variables with _initialized_session() as sess: - self.assertAllClose([0.], bias.eval()) - self.assertAllClose([[0.], [0.], [0.], [0.]], wire_cast_var.eval()) + self.assertAllClose([0.], self.evaluate(bias)) + self.assertAllClose([[0.], [0.], [0.], [0.]], + self.evaluate(wire_cast_var)) sess.run(wire_cast_var.assign([[10.], [100.], [1000.], [10000.]])) sess.run(bias.assign([5.])) - self.assertAllClose([[1005.], [10015.]], predictions.eval()) + self.assertAllClose([[1005.], [10015.]], self.evaluate(predictions)) def test_dense_and_sparse_bias(self): wire_cast = fc.categorical_column_with_hash_bucket('wire_cast', 4) @@ -1630,7 +1692,7 @@ class LinearModelTest(test.TestCase): sess.run(wire_cast_var.assign([[10.], [100.], [1000.], [10000.]])) sess.run(bias.assign([5.])) sess.run(price_var.assign([[10.]])) - self.assertAllClose([[1015.], [10065.]], predictions.eval()) + self.assertAllClose([[1015.], [10065.]], self.evaluate(predictions)) def test_dense_and_sparse_column(self): """When the column is both dense and sparse, uses sparse tensors.""" @@ -1682,10 +1744,11 @@ class LinearModelTest(test.TestCase): predictions = model(features) dense_and_sparse_column_var, bias = model.variables with _initialized_session() as sess: - sess.run(dense_and_sparse_column_var.assign( - [[10.], [100.], [1000.], [10000.]])) + sess.run( + dense_and_sparse_column_var.assign([[10.], [100.], [1000.], + [10000.]])) sess.run(bias.assign([5.])) - self.assertAllClose([[1005.], [10015.]], predictions.eval()) + self.assertAllClose([[1005.], [10015.]], self.evaluate(predictions)) def test_dense_multi_output(self): price = fc.numeric_column('price') @@ -1695,12 +1758,12 @@ class LinearModelTest(test.TestCase): predictions = model(features) price_var, bias = model.variables with _initialized_session() as sess: - self.assertAllClose(np.zeros((3,)), bias.eval()) - self.assertAllClose(np.zeros((1, 3)), price_var.eval()) + self.assertAllClose(np.zeros((3,)), self.evaluate(bias)) + self.assertAllClose(np.zeros((1, 3)), self.evaluate(price_var)) sess.run(price_var.assign([[10., 100., 1000.]])) sess.run(bias.assign([5., 6., 7.])) self.assertAllClose([[15., 106., 1007.], [55., 506., 5007.]], - predictions.eval()) + self.evaluate(predictions)) def test_sparse_multi_output(self): wire_cast = fc.categorical_column_with_hash_bucket('wire_cast', 4) @@ -1714,15 +1777,15 @@ class LinearModelTest(test.TestCase): predictions = model(features) wire_cast_var, bias = model.variables with _initialized_session() as sess: - self.assertAllClose(np.zeros((3,)), bias.eval()) - self.assertAllClose(np.zeros((4, 3)), wire_cast_var.eval()) + self.assertAllClose(np.zeros((3,)), self.evaluate(bias)) + self.assertAllClose(np.zeros((4, 3)), self.evaluate(wire_cast_var)) sess.run( - wire_cast_var.assign([[10., 11., 12.], [100., 110., 120.], [ - 1000., 1100., 1200. - ], [10000., 11000., 12000.]])) + wire_cast_var.assign([[10., 11., 12.], [100., 110., 120.], + [1000., 1100., 1200.], + [10000., 11000., 12000.]])) sess.run(bias.assign([5., 6., 7.])) self.assertAllClose([[1005., 1106., 1207.], [10015., 11017., 12019.]], - predictions.eval()) + self.evaluate(predictions)) def test_dense_multi_dimension(self): price = fc.numeric_column('price', shape=2) @@ -1732,9 +1795,9 @@ class LinearModelTest(test.TestCase): predictions = model(features) price_var, _ = model.variables with _initialized_session() as sess: - self.assertAllClose([[0.], [0.]], price_var.eval()) + self.assertAllClose([[0.], [0.]], self.evaluate(price_var)) sess.run(price_var.assign([[10.], [100.]])) - self.assertAllClose([[210.], [650.]], predictions.eval()) + self.assertAllClose([[210.], [650.]], self.evaluate(predictions)) def test_sparse_multi_rank(self): wire_cast = fc.categorical_column_with_hash_bucket('wire_cast', 4) @@ -1749,7 +1812,7 @@ class LinearModelTest(test.TestCase): predictions = model(features) wire_cast_var, _ = model.variables with _initialized_session() as sess: - self.assertAllClose(np.zeros((4, 1)), wire_cast_var.eval()) + self.assertAllClose(np.zeros((4, 1)), self.evaluate(wire_cast_var)) self.assertAllClose( np.zeros((2, 1)), predictions.eval(feed_dict={wire_tensor: wire_value})) @@ -1772,7 +1835,7 @@ class LinearModelTest(test.TestCase): with _initialized_session() as sess: sess.run(wire_cast_var.assign([[10.], [100.], [1000.], [10000.]])) sess.run(bias.assign([5.])) - self.assertAllClose([[1005.], [5010.]], predictions.eval()) + self.assertAllClose([[1005.], [5010.]], self.evaluate(predictions)) def test_sparse_combiner_with_negative_weights(self): wire_cast = fc.categorical_column_with_hash_bucket('wire_cast', 4) @@ -1793,7 +1856,7 @@ class LinearModelTest(test.TestCase): with _initialized_session() as sess: sess.run(wire_cast_var.assign([[10.], [100.], [1000.], [10000.]])) sess.run(bias.assign([5.])) - self.assertAllClose([[1005.], [-9985.]], predictions.eval()) + self.assertAllClose([[1005.], [-9985.]], self.evaluate(predictions)) def test_dense_multi_dimension_multi_output(self): price = fc.numeric_column('price', shape=2) @@ -1803,12 +1866,12 @@ class LinearModelTest(test.TestCase): predictions = model(features) price_var, bias = model.variables with _initialized_session() as sess: - self.assertAllClose(np.zeros((3,)), bias.eval()) - self.assertAllClose(np.zeros((2, 3)), price_var.eval()) + self.assertAllClose(np.zeros((3,)), self.evaluate(bias)) + self.assertAllClose(np.zeros((2, 3)), self.evaluate(price_var)) sess.run(price_var.assign([[1., 2., 3.], [10., 100., 1000.]])) sess.run(bias.assign([2., 3., 4.])) self.assertAllClose([[23., 205., 2007.], [67., 613., 6019.]], - predictions.eval()) + self.evaluate(predictions)) def test_raises_if_shape_mismatch(self): price = fc.numeric_column('price', shape=2) @@ -1828,32 +1891,29 @@ class LinearModelTest(test.TestCase): predictions = model(features) price_var, bias = model.variables with _initialized_session() as sess: - self.assertAllClose([0.], bias.eval()) - self.assertAllClose([[0.], [0.]], price_var.eval()) - self.assertAllClose([[0.], [0.]], predictions.eval()) + self.assertAllClose([0.], self.evaluate(bias)) + self.assertAllClose([[0.], [0.]], self.evaluate(price_var)) + self.assertAllClose([[0.], [0.]], self.evaluate(predictions)) sess.run(price_var.assign([[10.], [100.]])) - self.assertAllClose([[210.], [650.]], predictions.eval()) + self.assertAllClose([[210.], [650.]], self.evaluate(predictions)) def test_dense_multi_column(self): price1 = fc.numeric_column('price1', shape=2) price2 = fc.numeric_column('price2') with ops.Graph().as_default(): - features = { - 'price1': [[1., 2.], [5., 6.]], - 'price2': [[3.], [4.]] - } + features = {'price1': [[1., 2.], [5., 6.]], 'price2': [[3.], [4.]]} model = fc.LinearModel([price1, price2]) predictions = model(features) price1_var, price2_var, bias = model.variables with _initialized_session() as sess: - self.assertAllClose([0.], bias.eval()) - self.assertAllClose([[0.], [0.]], price1_var.eval()) - self.assertAllClose([[0.]], price2_var.eval()) - self.assertAllClose([[0.], [0.]], predictions.eval()) + self.assertAllClose([0.], self.evaluate(bias)) + self.assertAllClose([[0.], [0.]], self.evaluate(price1_var)) + self.assertAllClose([[0.]], self.evaluate(price2_var)) + self.assertAllClose([[0.], [0.]], self.evaluate(predictions)) sess.run(price1_var.assign([[10.], [100.]])) sess.run(price2_var.assign([[1000.]])) sess.run(bias.assign([7.])) - self.assertAllClose([[3217.], [4657.]], predictions.eval()) + self.assertAllClose([[3217.], [4657.]], self.evaluate(predictions)) def test_dense_trainable_default(self): price = fc.numeric_column('price') @@ -2046,6 +2106,7 @@ class LinearModelTest(test.TestCase): features['price2']: [[1.], [5.]], }) + @test_util.run_deprecated_v1 def test_with_numpy_input_fn(self): price = fc.numeric_column('price') price_buckets = fc.bucketized_column( @@ -2078,11 +2139,13 @@ class LinearModelTest(test.TestCase): sess.run(body_style_var.assign([[-10.], [-100.], [-1000.]])) sess.run(bias.assign([5.])) - self.assertAllClose([[10 - 1000 + 5.], [100 - 10 + 5.]], sess.run(net)) + self.assertAllClose([[10 - 1000 + 5.], [100 - 10 + 5.]], + self.evaluate(net)) coord.request_stop() coord.join(threads) + @test_util.run_deprecated_v1 def test_with_1d_sparse_tensor(self): price = fc.numeric_column('price') price_buckets = fc.bucketized_column( @@ -2096,11 +2159,16 @@ class LinearModelTest(test.TestCase): # Provides 1-dim tensor and dense tensor. features = { - 'price': constant_op.constant([-1., 12.,]), - 'body-style': sparse_tensor.SparseTensor( - indices=((0,), (1,)), - values=('sedan', 'hardtop'), - dense_shape=(2,)), + 'price': + constant_op.constant([ + -1., + 12., + ]), + 'body-style': + sparse_tensor.SparseTensor( + indices=((0,), (1,)), + values=('sedan', 'hardtop'), + dense_shape=(2,)), } self.assertEqual(1, features['price'].shape.ndims) self.assertEqual(1, features['body-style'].dense_shape.get_shape()[0]) @@ -2114,8 +2182,10 @@ class LinearModelTest(test.TestCase): sess.run(body_style_var.assign([[-10.], [-100.], [-1000.]])) sess.run(bias.assign([5.])) - self.assertAllClose([[10 - 1000 + 5.], [1000 - 10 + 5.]], sess.run(net)) + self.assertAllClose([[10 - 1000 + 5.], [1000 - 10 + 5.]], + self.evaluate(net)) + @test_util.run_deprecated_v1 def test_with_1d_unknown_shape_sparse_tensor(self): price = fc.numeric_column('price') price_buckets = fc.bucketized_column( @@ -2140,9 +2210,7 @@ class LinearModelTest(test.TestCase): price_data = np.array([-1., 12.]) body_style_data = sparse_tensor.SparseTensorValue( - indices=((0,), (1,)), - values=('sedan', 'hardtop'), - dense_shape=(2,)) + indices=((0,), (1,)), values=('sedan', 'hardtop'), dense_shape=(2,)) country_data = np.array(['US', 'CA']) model = fc.LinearModel([price_buckets, body_style, country]) @@ -2162,6 +2230,7 @@ class LinearModelTest(test.TestCase): features['country']: country_data })) + @test_util.run_deprecated_v1 def test_with_rank_0_feature(self): price = fc.numeric_column('price') features = { @@ -2197,14 +2266,14 @@ class LinearModelTest(test.TestCase): price_var1, bias1 = model1.variables price_var2, bias2 = model2.variables with _initialized_session() as sess: - self.assertAllClose([0.], bias1.eval()) + self.assertAllClose([0.], self.evaluate(bias1)) sess.run(price_var1.assign([[10.]])) sess.run(bias1.assign([5.])) - self.assertAllClose([[15.], [55.]], predictions1.eval()) - self.assertAllClose([0.], bias2.eval()) + self.assertAllClose([[15.], [55.]], self.evaluate(predictions1)) + self.assertAllClose([0.], self.evaluate(bias2)) sess.run(price_var2.assign([[10.]])) sess.run(bias2.assign([5.])) - self.assertAllClose([[25.], [105.]], predictions2.eval()) + self.assertAllClose([[25.], [105.]], self.evaluate(predictions2)) class OldLinearModelTest(test.TestCase): @@ -2272,10 +2341,10 @@ class OldLinearModelTest(test.TestCase): bias = get_linear_model_bias() price_var = get_linear_model_column_var(price) with _initialized_session() as sess: - self.assertAllClose([0.], bias.eval()) + self.assertAllClose([0.], self.evaluate(bias)) sess.run(price_var.assign([[10.]])) sess.run(bias.assign([5.])) - self.assertAllClose([[15.], [55.]], predictions.eval()) + self.assertAllClose([[15.], [55.]], self.evaluate(predictions)) def test_sparse_bias(self): wire_cast = fc.categorical_column_with_hash_bucket('wire_cast', 4) @@ -2289,11 +2358,12 @@ class OldLinearModelTest(test.TestCase): bias = get_linear_model_bias() wire_cast_var = get_linear_model_column_var(wire_cast) with _initialized_session() as sess: - self.assertAllClose([0.], bias.eval()) - self.assertAllClose([[0.], [0.], [0.], [0.]], wire_cast_var.eval()) + self.assertAllClose([0.], self.evaluate(bias)) + self.assertAllClose([[0.], [0.], [0.], [0.]], + self.evaluate(wire_cast_var)) sess.run(wire_cast_var.assign([[10.], [100.], [1000.], [10000.]])) sess.run(bias.assign([5.])) - self.assertAllClose([[1005.], [10015.]], predictions.eval()) + self.assertAllClose([[1005.], [10015.]], self.evaluate(predictions)) def test_dense_and_sparse_bias(self): wire_cast = fc.categorical_column_with_hash_bucket('wire_cast', 4) @@ -2312,7 +2382,7 @@ class OldLinearModelTest(test.TestCase): sess.run(wire_cast_var.assign([[10.], [100.], [1000.], [10000.]])) sess.run(bias.assign([5.])) sess.run(price_var.assign([[10.]])) - self.assertAllClose([[1015.], [10065.]], predictions.eval()) + self.assertAllClose([[1015.], [10065.]], self.evaluate(predictions)) def test_dense_and_sparse_column(self): """When the column is both dense and sparse, uses sparse tensors.""" @@ -2394,7 +2464,7 @@ class OldLinearModelTest(test.TestCase): dense_and_sparse_column_var.assign([[10.], [100.], [1000.], [10000.]])) sess.run(bias.assign([5.])) - self.assertAllClose([[1005.], [10015.]], predictions.eval()) + self.assertAllClose([[1005.], [10015.]], self.evaluate(predictions)) def test_dense_multi_output(self): price = fc.numeric_column('price') @@ -2404,12 +2474,12 @@ class OldLinearModelTest(test.TestCase): bias = get_linear_model_bias() price_var = get_linear_model_column_var(price) with _initialized_session() as sess: - self.assertAllClose(np.zeros((3,)), bias.eval()) - self.assertAllClose(np.zeros((1, 3)), price_var.eval()) + self.assertAllClose(np.zeros((3,)), self.evaluate(bias)) + self.assertAllClose(np.zeros((1, 3)), self.evaluate(price_var)) sess.run(price_var.assign([[10., 100., 1000.]])) sess.run(bias.assign([5., 6., 7.])) self.assertAllClose([[15., 106., 1007.], [55., 506., 5007.]], - predictions.eval()) + self.evaluate(predictions)) def test_sparse_multi_output(self): wire_cast = fc.categorical_column_with_hash_bucket('wire_cast', 4) @@ -2423,15 +2493,15 @@ class OldLinearModelTest(test.TestCase): bias = get_linear_model_bias() wire_cast_var = get_linear_model_column_var(wire_cast) with _initialized_session() as sess: - self.assertAllClose(np.zeros((3,)), bias.eval()) - self.assertAllClose(np.zeros((4, 3)), wire_cast_var.eval()) + self.assertAllClose(np.zeros((3,)), self.evaluate(bias)) + self.assertAllClose(np.zeros((4, 3)), self.evaluate(wire_cast_var)) sess.run( wire_cast_var.assign([[10., 11., 12.], [100., 110., 120.], [1000., 1100., 1200.], [10000., 11000., 12000.]])) sess.run(bias.assign([5., 6., 7.])) self.assertAllClose([[1005., 1106., 1207.], [10015., 11017., 12019.]], - predictions.eval()) + self.evaluate(predictions)) def test_dense_multi_dimension(self): price = fc.numeric_column('price', shape=2) @@ -2440,9 +2510,9 @@ class OldLinearModelTest(test.TestCase): predictions = fc_old.linear_model(features, [price]) price_var = get_linear_model_column_var(price) with _initialized_session() as sess: - self.assertAllClose([[0.], [0.]], price_var.eval()) + self.assertAllClose([[0.], [0.]], self.evaluate(price_var)) sess.run(price_var.assign([[10.], [100.]])) - self.assertAllClose([[210.], [650.]], predictions.eval()) + self.assertAllClose([[210.], [650.]], self.evaluate(predictions)) def test_sparse_multi_rank(self): wire_cast = fc.categorical_column_with_hash_bucket('wire_cast', 4) @@ -2456,7 +2526,7 @@ class OldLinearModelTest(test.TestCase): predictions = fc_old.linear_model(features, [wire_cast]) wire_cast_var = get_linear_model_column_var(wire_cast) with _initialized_session() as sess: - self.assertAllClose(np.zeros((4, 1)), wire_cast_var.eval()) + self.assertAllClose(np.zeros((4, 1)), self.evaluate(wire_cast_var)) self.assertAllClose( np.zeros((2, 1)), predictions.eval(feed_dict={wire_tensor: wire_value})) @@ -2480,7 +2550,7 @@ class OldLinearModelTest(test.TestCase): with _initialized_session() as sess: sess.run(wire_cast_var.assign([[10.], [100.], [1000.], [10000.]])) sess.run(bias.assign([5.])) - self.assertAllClose([[1005.], [5010.]], predictions.eval()) + self.assertAllClose([[1005.], [5010.]], self.evaluate(predictions)) def test_sparse_combiner_with_negative_weights(self): wire_cast = fc.categorical_column_with_hash_bucket('wire_cast', 4) @@ -2502,7 +2572,7 @@ class OldLinearModelTest(test.TestCase): with _initialized_session() as sess: sess.run(wire_cast_var.assign([[10.], [100.], [1000.], [10000.]])) sess.run(bias.assign([5.])) - self.assertAllClose([[1005.], [-9985.]], predictions.eval()) + self.assertAllClose([[1005.], [-9985.]], self.evaluate(predictions)) def test_dense_multi_dimension_multi_output(self): price = fc.numeric_column('price', shape=2) @@ -2512,12 +2582,12 @@ class OldLinearModelTest(test.TestCase): bias = get_linear_model_bias() price_var = get_linear_model_column_var(price) with _initialized_session() as sess: - self.assertAllClose(np.zeros((3,)), bias.eval()) - self.assertAllClose(np.zeros((2, 3)), price_var.eval()) + self.assertAllClose(np.zeros((3,)), self.evaluate(bias)) + self.assertAllClose(np.zeros((2, 3)), self.evaluate(price_var)) sess.run(price_var.assign([[1., 2., 3.], [10., 100., 1000.]])) sess.run(bias.assign([2., 3., 4.])) self.assertAllClose([[23., 205., 2007.], [67., 613., 6019.]], - predictions.eval()) + self.evaluate(predictions)) def test_raises_if_shape_mismatch(self): price = fc.numeric_column('price', shape=2) @@ -2536,11 +2606,11 @@ class OldLinearModelTest(test.TestCase): bias = get_linear_model_bias() price_var = get_linear_model_column_var(price) with _initialized_session() as sess: - self.assertAllClose([0.], bias.eval()) - self.assertAllClose([[0.], [0.]], price_var.eval()) - self.assertAllClose([[0.], [0.]], predictions.eval()) + self.assertAllClose([0.], self.evaluate(bias)) + self.assertAllClose([[0.], [0.]], self.evaluate(price_var)) + self.assertAllClose([[0.], [0.]], self.evaluate(predictions)) sess.run(price_var.assign([[10.], [100.]])) - self.assertAllClose([[210.], [650.]], predictions.eval()) + self.assertAllClose([[210.], [650.]], self.evaluate(predictions)) def test_dense_multi_column(self): price1 = fc.numeric_column('price1', shape=2) @@ -2552,14 +2622,14 @@ class OldLinearModelTest(test.TestCase): price1_var = get_linear_model_column_var(price1) price2_var = get_linear_model_column_var(price2) with _initialized_session() as sess: - self.assertAllClose([0.], bias.eval()) - self.assertAllClose([[0.], [0.]], price1_var.eval()) - self.assertAllClose([[0.]], price2_var.eval()) - self.assertAllClose([[0.], [0.]], predictions.eval()) + self.assertAllClose([0.], self.evaluate(bias)) + self.assertAllClose([[0.], [0.]], self.evaluate(price1_var)) + self.assertAllClose([[0.]], self.evaluate(price2_var)) + self.assertAllClose([[0.], [0.]], self.evaluate(predictions)) sess.run(price1_var.assign([[10.], [100.]])) sess.run(price2_var.assign([[1000.]])) sess.run(bias.assign([7.])) - self.assertAllClose([[3217.], [4657.]], predictions.eval()) + self.assertAllClose([[3217.], [4657.]], self.evaluate(predictions)) def test_fills_cols_to_vars(self): price1 = fc.numeric_column('price1', shape=2) @@ -2589,15 +2659,18 @@ class OldLinearModelTest(test.TestCase): partitioner=partitioned_variables.fixed_size_partitioner(2, axis=0)): fc_old.linear_model( features, [price1, price2], cols_to_vars=cols_to_vars) - with _initialized_session(): - self.assertEqual([0.], cols_to_vars['bias'][0].eval()) - # Partitioning shards the [2, 1] price1 var into 2 [1, 1] Variables. - self.assertAllEqual([[0.]], cols_to_vars[price1][0].eval()) - self.assertAllEqual([[0.]], cols_to_vars[price1][1].eval()) - # Partitioning shards the [3, 1] price2 var into a [2, 1] Variable and - # a [1, 1] Variable. - self.assertAllEqual([[0.], [0.]], cols_to_vars[price2][0].eval()) - self.assertAllEqual([[0.]], cols_to_vars[price2][1].eval()) + + self.evaluate(variables_lib.global_variables_initializer()) + self.evaluate(lookup_ops.tables_initializer()) + + self.assertEqual([0.], self.evaluate(cols_to_vars['bias'][0])) + # Partitioning shards the [2, 1] price1 var into 2 [1, 1] Variables. + self.assertAllEqual([[0.]], self.evaluate(cols_to_vars[price1][0])) + self.assertAllEqual([[0.]], self.evaluate(cols_to_vars[price1][1])) + # Partitioning shards the [3, 1] price2 var into a [2, 1] Variable and + # a [1, 1] Variable. + self.assertAllEqual([[0.], [0.]], self.evaluate(cols_to_vars[price2][0])) + self.assertAllEqual([[0.]], self.evaluate(cols_to_vars[price2][1])) def test_fills_cols_to_output_tensors(self): # Provide three _DenseColumn's to input_layer: a _NumericColumn, a @@ -2795,6 +2868,7 @@ class OldLinearModelTest(test.TestCase): features['price2']: [[1.], [5.]], }) + @test_util.run_deprecated_v1 def test_with_1d_sparse_tensor(self): price = fc.numeric_column('price') price_buckets = fc.bucketized_column( @@ -2832,8 +2906,10 @@ class OldLinearModelTest(test.TestCase): sess.run(body_style_var.assign([[-10.], [-100.], [-1000.]])) sess.run(bias.assign([5.])) - self.assertAllClose([[10 - 1000 + 5.], [1000 - 10 + 5.]], sess.run(net)) + self.assertAllClose([[10 - 1000 + 5.], [1000 - 10 + 5.]], + self.evaluate(net)) + @test_util.run_deprecated_v1 def test_with_1d_unknown_shape_sparse_tensor(self): price = fc.numeric_column('price') price_buckets = fc.bucketized_column( @@ -2879,6 +2955,7 @@ class OldLinearModelTest(test.TestCase): features['country']: country_data })) + @test_util.run_deprecated_v1 def test_with_rank_0_feature(self): price = fc.numeric_column('price') features = { @@ -2912,26 +2989,27 @@ class OldLinearModelTest(test.TestCase): price_var1 = get_linear_model_column_var(price, name='linear_model') price_var2 = get_linear_model_column_var(price, name='linear_model_1') with _initialized_session() as sess: - self.assertAllClose([0.], bias1.eval()) + self.assertAllClose([0.], self.evaluate(bias1)) sess.run(price_var1.assign([[10.]])) sess.run(bias1.assign([5.])) - self.assertAllClose([[15.], [55.]], predictions1.eval()) - self.assertAllClose([0.], bias2.eval()) + self.assertAllClose([[15.], [55.]], self.evaluate(predictions1)) + self.assertAllClose([0.], self.evaluate(bias2)) sess.run(price_var2.assign([[10.]])) sess.run(bias2.assign([5.])) - self.assertAllClose([[25.], [105.]], predictions2.eval()) + self.assertAllClose([[25.], [105.]], self.evaluate(predictions2)) + @test_util.run_deprecated_v1 def test_linear_model_v1_shared_embedding_all_other_v2(self): price = fc.numeric_column('price') # v2 some_sparse_column = fc.categorical_column_with_hash_bucket( 'sparse_feature', hash_bucket_size=5) # v2 some_embedding_column = fc.embedding_column( some_sparse_column, dimension=10) # v2 - categorical_column_a = fc_old.categorical_column_with_identity( + categorical_column_a = fc.categorical_column_with_identity( key='aaa', num_buckets=3) # v2 - categorical_column_b = fc_old.categorical_column_with_identity( + categorical_column_b = fc.categorical_column_with_identity( key='bbb', num_buckets=3) # v2 - shared_embedding_a, shared_embedding_b = fc_old.shared_embedding_columns( + shared_embedding_a, shared_embedding_b = fc.shared_embedding_columns( [categorical_column_a, categorical_column_b], dimension=2) # v1 all_cols = [ price, some_embedding_column, shared_embedding_a, shared_embedding_b @@ -2954,9 +3032,13 @@ class OldLinearModelTest(test.TestCase): } fc_old.linear_model(features, all_cols) bias = get_linear_model_bias() - with _initialized_session(): - self.assertAllClose([0.], bias.eval()) + self.evaluate(variables_lib.global_variables_initializer()) + self.evaluate(lookup_ops.tables_initializer()) + + self.assertAllClose([0.], self.evaluate(bias)) + + @test_util.run_deprecated_v1 def test_linear_model_v1_shared_embedding_with_v2_cat_all_other_v2(self): price = fc.numeric_column('price') # v2 some_sparse_column = fc.categorical_column_with_hash_bucket( @@ -2967,7 +3049,7 @@ class OldLinearModelTest(test.TestCase): key='aaa', num_buckets=3) # v2 categorical_column_b = fc.categorical_column_with_identity( key='bbb', num_buckets=3) # v2 - shared_embedding_a, shared_embedding_b = fc_old.shared_embedding_columns( + shared_embedding_a, shared_embedding_b = fc.shared_embedding_columns( [categorical_column_a, categorical_column_b], dimension=2) # v1 all_cols = [ price, some_embedding_column, shared_embedding_a, shared_embedding_b @@ -2990,20 +3072,24 @@ class OldLinearModelTest(test.TestCase): } fc_old.linear_model(features, all_cols) bias = get_linear_model_bias() - with _initialized_session(): - self.assertAllClose([0.], bias.eval()) + self.evaluate(variables_lib.global_variables_initializer()) + self.evaluate(lookup_ops.tables_initializer()) + + self.assertAllClose([0.], self.evaluate(bias)) + + @test_util.run_deprecated_v1 def test_linear_model_v1_v2_mix(self): price = fc.numeric_column('price') # v2 - some_sparse_column = fc_old.categorical_column_with_hash_bucket( + some_sparse_column = fc.categorical_column_with_hash_bucket( 'sparse_feature', hash_bucket_size=5) # v1 - some_embedding_column = fc_old.embedding_column( + some_embedding_column = fc.embedding_column( some_sparse_column, dimension=10) # v1 - categorical_column_a = fc_old.categorical_column_with_identity( + categorical_column_a = fc.categorical_column_with_identity( key='aaa', num_buckets=3) # v2 - categorical_column_b = fc_old.categorical_column_with_identity( + categorical_column_b = fc.categorical_column_with_identity( key='bbb', num_buckets=3) # v2 - shared_embedding_a, shared_embedding_b = fc_old.shared_embedding_columns( + shared_embedding_a, shared_embedding_b = fc.shared_embedding_columns( [categorical_column_a, categorical_column_b], dimension=2) # v1 all_cols = [ price, some_embedding_column, shared_embedding_a, shared_embedding_b @@ -3026,14 +3112,18 @@ class OldLinearModelTest(test.TestCase): } fc_old.linear_model(features, all_cols) bias = get_linear_model_bias() - with _initialized_session(): - self.assertAllClose([0.], bias.eval()) + self.evaluate(variables_lib.global_variables_initializer()) + self.evaluate(lookup_ops.tables_initializer()) + + self.assertAllClose([0.], self.evaluate(bias)) + + @test_util.run_deprecated_v1 def test_linear_model_v2_shared_embedding_all_other_v1(self): - price = fc_old.numeric_column('price') # v1 - some_sparse_column = fc_old.categorical_column_with_hash_bucket( + price = fc.numeric_column('price') # v1 + some_sparse_column = fc.categorical_column_with_hash_bucket( 'sparse_feature', hash_bucket_size=5) # v1 - some_embedding_column = fc_old.embedding_column( + some_embedding_column = fc.embedding_column( some_sparse_column, dimension=10) # v1 categorical_column_a = fc.categorical_column_with_identity( key='aaa', num_buckets=3) # v2 @@ -3065,13 +3155,13 @@ class OldLinearModelTest(test.TestCase): fc_old.linear_model(features, all_cols) -class FeatureLayerTest(test.TestCase): +class DenseFeaturesTest(test.TestCase): @test_util.run_in_graph_and_eager_modes() def test_retrieving_input(self): features = {'a': [0.]} - feature_layer = fc.FeatureLayer(fc.numeric_column('a')) - inputs = self.evaluate(feature_layer(features)) + dense_features = fc.DenseFeatures(fc.numeric_column('a')) + inputs = self.evaluate(dense_features(features)) self.assertAllClose([[0.]], inputs) def test_reuses_variables(self): @@ -3085,6 +3175,7 @@ class FeatureLayerTest(test.TestCase): categorical_column = fc.categorical_column_with_identity( key='a', num_buckets=3) embedding_dimension = 2 + def _embedding_column_initializer(shape, dtype, partition_info): del shape # unused del dtype # unused @@ -3100,11 +3191,11 @@ class FeatureLayerTest(test.TestCase): dimension=embedding_dimension, initializer=_embedding_column_initializer) - feature_layer = fc.FeatureLayer([embedding_column]) + dense_features = fc.DenseFeatures([embedding_column]) features = {'a': sparse_input} - inputs = feature_layer(features) - variables = feature_layer.variables + inputs = dense_features(features) + variables = dense_features.variables # Sanity check: test that the inputs are correct. self.assertAllEqual([[1, 0], [0, 1], [1, 1]], inputs) @@ -3112,13 +3203,13 @@ class FeatureLayerTest(test.TestCase): # Check that only one variable was created. self.assertEqual(1, len(variables)) - # Check that invoking feature_layer on the same features does not create + # Check that invoking dense_features on the same features does not create # additional variables - _ = feature_layer(features) + _ = dense_features(features) self.assertEqual(1, len(variables)) - self.assertEqual(variables[0], feature_layer.variables[0]) + self.assertEqual(variables[0], dense_features.variables[0]) - def test_feature_column_feature_layer_gradient(self): + def test_feature_column_dense_features_gradient(self): with context.eager_mode(): sparse_input = sparse_tensor.SparseTensor( indices=((0, 0), (1, 0), (2, 0)), @@ -3145,11 +3236,11 @@ class FeatureLayerTest(test.TestCase): dimension=embedding_dimension, initializer=_embedding_column_initializer) - feature_layer = fc.FeatureLayer([embedding_column]) + dense_features = fc.DenseFeatures([embedding_column]) features = {'a': sparse_input} def scale_matrix(): - matrix = feature_layer(features) + matrix = dense_features(features) return 2 * matrix # Sanity check: Verify that scale_matrix returns the correct output. @@ -3167,11 +3258,11 @@ class FeatureLayerTest(test.TestCase): def test_raises_if_empty_feature_columns(self): with self.assertRaisesRegexp(ValueError, 'feature_columns must not be empty'): - fc.FeatureLayer(feature_columns=[])(features={}) + fc.DenseFeatures(feature_columns=[])(features={}) def test_should_be_dense_column(self): with self.assertRaisesRegexp(ValueError, 'must be a DenseColumn'): - fc.FeatureLayer(feature_columns=[ + fc.DenseFeatures(feature_columns=[ fc.categorical_column_with_hash_bucket('wire_cast', 4) ])( features={ @@ -3181,7 +3272,7 @@ class FeatureLayerTest(test.TestCase): def test_does_not_support_dict_columns(self): with self.assertRaisesRegexp( ValueError, 'Expected feature_columns to be iterable, found dict.'): - fc.FeatureLayer(feature_columns={'a': fc.numeric_column('a')})( + fc.DenseFeatures(feature_columns={'a': fc.numeric_column('a')})( features={ 'a': [[0]] }) @@ -3189,22 +3280,28 @@ class FeatureLayerTest(test.TestCase): def test_bare_column(self): with ops.Graph().as_default(): features = features = {'a': [0.]} - net = fc.FeatureLayer(fc.numeric_column('a'))(features) - with _initialized_session(): - self.assertAllClose([[0.]], net.eval()) + net = fc.DenseFeatures(fc.numeric_column('a'))(features) + + self.evaluate(variables_lib.global_variables_initializer()) + self.evaluate(lookup_ops.tables_initializer()) + + self.assertAllClose([[0.]], self.evaluate(net)) def test_column_generator(self): with ops.Graph().as_default(): features = features = {'a': [0.], 'b': [1.]} columns = (fc.numeric_column(key) for key in features) - net = fc.FeatureLayer(columns)(features) - with _initialized_session(): - self.assertAllClose([[0., 1.]], net.eval()) + net = fc.DenseFeatures(columns)(features) + + self.evaluate(variables_lib.global_variables_initializer()) + self.evaluate(lookup_ops.tables_initializer()) + + self.assertAllClose([[0., 1.]], self.evaluate(net)) def test_raises_if_duplicate_name(self): with self.assertRaisesRegexp( ValueError, 'Duplicate feature column name found for columns'): - fc.FeatureLayer( + fc.DenseFeatures( feature_columns=[fc.numeric_column('a'), fc.numeric_column('a')])( features={ @@ -3215,17 +3312,23 @@ class FeatureLayerTest(test.TestCase): price = fc.numeric_column('price') with ops.Graph().as_default(): features = {'price': [[1.], [5.]]} - net = fc.FeatureLayer([price])(features) - with _initialized_session(): - self.assertAllClose([[1.], [5.]], net.eval()) + net = fc.DenseFeatures([price])(features) + + self.evaluate(variables_lib.global_variables_initializer()) + self.evaluate(lookup_ops.tables_initializer()) + + self.assertAllClose([[1.], [5.]], self.evaluate(net)) def test_multi_dimension(self): price = fc.numeric_column('price', shape=2) with ops.Graph().as_default(): features = {'price': [[1., 2.], [5., 6.]]} - net = fc.FeatureLayer([price])(features) - with _initialized_session(): - self.assertAllClose([[1., 2.], [5., 6.]], net.eval()) + net = fc.DenseFeatures([price])(features) + + self.evaluate(variables_lib.global_variables_initializer()) + self.evaluate(lookup_ops.tables_initializer()) + + self.assertAllClose([[1., 2.], [5., 6.]], self.evaluate(net)) def test_compute_output_shape(self): price1 = fc.numeric_column('price1', shape=2) @@ -3235,12 +3338,15 @@ class FeatureLayerTest(test.TestCase): 'price1': [[1., 2.], [5., 6.]], 'price2': [[3., 4., 5., 6.], [7., 8., 9., 10.]] } - feature_layer = fc.FeatureLayer([price1, price2]) - self.assertEqual((None, 6), feature_layer.compute_output_shape((None,))) - net = feature_layer(features) - with _initialized_session(): - self.assertAllClose( - [[1., 2., 3., 4., 5., 6.], [5., 6., 7., 8., 9., 10.]], net.eval()) + dense_features = fc.DenseFeatures([price1, price2]) + self.assertEqual((None, 6), dense_features.compute_output_shape((None,))) + net = dense_features(features) + + self.evaluate(variables_lib.global_variables_initializer()) + self.evaluate(lookup_ops.tables_initializer()) + + self.assertAllClose([[1., 2., 3., 4., 5., 6.], [5., 6., 7., 8., 9., 10.]], + self.evaluate(net)) def test_raises_if_shape_mismatch(self): price = fc.numeric_column('price', shape=2) @@ -3249,27 +3355,30 @@ class FeatureLayerTest(test.TestCase): with self.assertRaisesRegexp( Exception, r'Cannot reshape a tensor with 2 elements to shape \[2,2\]'): - fc.FeatureLayer([price])(features) + fc.DenseFeatures([price])(features) def test_reshaping(self): price = fc.numeric_column('price', shape=[1, 2]) with ops.Graph().as_default(): features = {'price': [[[1., 2.]], [[5., 6.]]]} - net = fc.FeatureLayer([price])(features) - with _initialized_session(): - self.assertAllClose([[1., 2.], [5., 6.]], net.eval()) + net = fc.DenseFeatures([price])(features) + + self.evaluate(variables_lib.global_variables_initializer()) + self.evaluate(lookup_ops.tables_initializer()) + + self.assertAllClose([[1., 2.], [5., 6.]], self.evaluate(net)) def test_multi_column(self): price1 = fc.numeric_column('price1', shape=2) price2 = fc.numeric_column('price2') with ops.Graph().as_default(): - features = { - 'price1': [[1., 2.], [5., 6.]], - 'price2': [[3.], [4.]] - } - net = fc.FeatureLayer([price1, price2])(features) - with _initialized_session(): - self.assertAllClose([[1., 2., 3.], [5., 6., 4.]], net.eval()) + features = {'price1': [[1., 2.], [5., 6.]], 'price2': [[3.], [4.]]} + net = fc.DenseFeatures([price1, price2])(features) + + self.evaluate(variables_lib.global_variables_initializer()) + self.evaluate(lookup_ops.tables_initializer()) + + self.assertAllClose([[1., 2., 3.], [5., 6., 4.]], self.evaluate(net)) def test_cols_to_output_tensors(self): price1 = fc.numeric_column('price1', shape=2) @@ -3277,12 +3386,16 @@ class FeatureLayerTest(test.TestCase): with ops.Graph().as_default(): cols_dict = {} features = {'price1': [[1., 2.], [5., 6.]], 'price2': [[3.], [4.]]} - feature_layer = fc.FeatureLayer([price1, price2]) - net = feature_layer(features, cols_dict) - with _initialized_session(): - self.assertAllClose([[1., 2.], [5., 6.]], cols_dict[price1].eval()) - self.assertAllClose([[3.], [4.]], cols_dict[price2].eval()) - self.assertAllClose([[1., 2., 3.], [5., 6., 4.]], net.eval()) + dense_features = fc.DenseFeatures([price1, price2]) + net = dense_features(features, cols_dict) + + self.evaluate(variables_lib.global_variables_initializer()) + self.evaluate(lookup_ops.tables_initializer()) + + self.assertAllClose([[1., 2.], [5., 6.]], + self.evaluate(cols_dict[price1])) + self.assertAllClose([[3.], [4.]], self.evaluate(cols_dict[price2])) + self.assertAllClose([[1., 2., 3.], [5., 6., 4.]], self.evaluate(net)) def test_column_order(self): price_a = fc.numeric_column('price_a') @@ -3292,11 +3405,14 @@ class FeatureLayerTest(test.TestCase): 'price_a': [[1.]], 'price_b': [[3.]], } - net1 = fc.FeatureLayer([price_a, price_b])(features) - net2 = fc.FeatureLayer([price_b, price_a])(features) - with _initialized_session(): - self.assertAllClose([[1., 3.]], net1.eval()) - self.assertAllClose([[1., 3.]], net2.eval()) + net1 = fc.DenseFeatures([price_a, price_b])(features) + net2 = fc.DenseFeatures([price_b, price_a])(features) + + self.evaluate(variables_lib.global_variables_initializer()) + self.evaluate(lookup_ops.tables_initializer()) + + self.assertAllClose([[1., 3.]], self.evaluate(net1)) + self.assertAllClose([[1., 3.]], self.evaluate(net2)) def test_fails_for_categorical_column(self): animal = fc.categorical_column_with_identity('animal', num_buckets=4) @@ -3307,7 +3423,7 @@ class FeatureLayerTest(test.TestCase): indices=[[0, 0], [0, 1]], values=[1, 2], dense_shape=[1, 2]) } with self.assertRaisesRegexp(Exception, 'must be a DenseColumn'): - fc.FeatureLayer([animal])(features) + fc.DenseFeatures([animal])(features) def test_static_batch_size_mismatch(self): price1 = fc.numeric_column('price1') @@ -3320,7 +3436,7 @@ class FeatureLayerTest(test.TestCase): with self.assertRaisesRegexp( ValueError, 'Batch size \(first dimension\) of each feature must be same.'): # pylint: disable=anomalous-backslash-in-string - fc.FeatureLayer([price1, price2])(features) + fc.DenseFeatures([price1, price2])(features) def test_subset_of_static_batch_size_mismatch(self): price1 = fc.numeric_column('price1') @@ -3335,7 +3451,7 @@ class FeatureLayerTest(test.TestCase): with self.assertRaisesRegexp( ValueError, 'Batch size \(first dimension\) of each feature must be same.'): # pylint: disable=anomalous-backslash-in-string - fc.FeatureLayer([price1, price2, price3])(features) + fc.DenseFeatures([price1, price2, price3])(features) def test_runtime_batch_size_mismatch(self): price1 = fc.numeric_column('price1') @@ -3345,7 +3461,7 @@ class FeatureLayerTest(test.TestCase): 'price1': array_ops.placeholder(dtype=dtypes.int64), # batchsize = 3 'price2': [[3.], [4.]] # batchsize = 2 } - net = fc.FeatureLayer([price1, price2])(features) + net = fc.DenseFeatures([price1, price2])(features) with _initialized_session() as sess: with self.assertRaisesRegexp(errors.OpError, 'Dimensions of inputs should match'): @@ -3359,7 +3475,7 @@ class FeatureLayerTest(test.TestCase): 'price1': array_ops.placeholder(dtype=dtypes.int64), # batchsize = 2 'price2': array_ops.placeholder(dtype=dtypes.int64), # batchsize = 2 } - net = fc.FeatureLayer([price1, price2])(features) + net = fc.DenseFeatures([price1, price2])(features) with _initialized_session() as sess: sess.run( net, @@ -3379,19 +3495,20 @@ class FeatureLayerTest(test.TestCase): 'sparse_feature': [['a'], ['x']], } all_cols = [some_embedding_column] - fc.FeatureLayer(all_cols)(features) - fc.FeatureLayer(all_cols)(features) + fc.DenseFeatures(all_cols)(features) + fc.DenseFeatures(all_cols)(features) # Make sure that 2 variables get created in this case. self.assertEqual(2, len( ops.get_collection(ops.GraphKeys.GLOBAL_VARIABLES))) expected_var_names = [ - 'feature_layer/sparse_feature_embedding/embedding_weights:0', - 'feature_layer_1/sparse_feature_embedding/embedding_weights:0' + 'dense_features/sparse_feature_embedding/embedding_weights:0', + 'dense_features_1/sparse_feature_embedding/embedding_weights:0' ] self.assertItemsEqual( expected_var_names, [v.name for v in ops.get_collection(ops.GraphKeys.GLOBAL_VARIABLES)]) + @test_util.run_deprecated_v1 def test_multiple_layers_with_same_shared_embedding_column(self): categorical_column_a = fc.categorical_column_with_identity( key='aaa', num_buckets=3) @@ -3416,8 +3533,8 @@ class FeatureLayerTest(test.TestCase): dense_shape=(2, 2)), } all_cols = [embedding_column_a, embedding_column_b] - fc.FeatureLayer(all_cols)(features) - fc.FeatureLayer(all_cols)(features) + fc.DenseFeatures(all_cols)(features) + fc.DenseFeatures(all_cols)(features) # Make sure that only 1 variable gets created in this case. self.assertEqual(1, len( ops.get_collection(ops.GraphKeys.GLOBAL_VARIABLES))) @@ -3425,6 +3542,7 @@ class FeatureLayerTest(test.TestCase): ['aaa_bbb_shared_embedding:0'], [v.name for v in ops.get_collection(ops.GraphKeys.GLOBAL_VARIABLES)]) + @test_util.run_deprecated_v1 def test_multiple_layers_with_same_shared_embedding_column_diff_graphs(self): categorical_column_a = fc.categorical_column_with_identity( key='aaa', num_buckets=3) @@ -3449,7 +3567,7 @@ class FeatureLayerTest(test.TestCase): values=(1, 2, 1), dense_shape=(2, 2)), } - fc.FeatureLayer(all_cols)(features) + fc.DenseFeatures(all_cols)(features) # Make sure that only 1 variable gets created in this case. self.assertEqual(1, len( ops.get_collection(ops.GraphKeys.GLOBAL_VARIABLES))) @@ -3468,7 +3586,7 @@ class FeatureLayerTest(test.TestCase): dense_shape=(2, 2)), } - fc.FeatureLayer(all_cols)(features1) + fc.DenseFeatures(all_cols)(features1) # Make sure that only 1 variable gets created in this case. self.assertEqual(1, len( ops.get_collection(ops.GraphKeys.GLOBAL_VARIABLES))) @@ -3476,23 +3594,25 @@ class FeatureLayerTest(test.TestCase): ['aaa_bbb_shared_embedding:0'], [v.name for v in ops.get_collection(ops.GraphKeys.GLOBAL_VARIABLES)]) + @test_util.run_deprecated_v1 def test_with_numpy_input_fn(self): embedding_values = ( (1., 2., 3., 4., 5.), # id 0 (6., 7., 8., 9., 10.), # id 1 (11., 12., 13., 14., 15.) # id 2 ) + def _initializer(shape, dtype, partition_info): del shape, dtype, partition_info return embedding_values - # price has 1 dimension in feature_layer + # price has 1 dimension in dense_features price = fc.numeric_column('price') body_style = fc.categorical_column_with_vocabulary_list( 'body-style', vocabulary_list=['hardtop', 'wagon', 'sedan']) - # one_hot_body_style has 3 dims in feature_layer. + # one_hot_body_style has 3 dims in dense_features. one_hot_body_style = fc.indicator_column(body_style) - # embedded_body_style has 5 dims in feature_layer. + # embedded_body_style has 5 dims in dense_features. embedded_body_style = fc.embedding_column( body_style, dimension=5, initializer=_initializer) @@ -3504,7 +3624,7 @@ class FeatureLayerTest(test.TestCase): batch_size=2, shuffle=False) features = input_fn() - net = fc.FeatureLayer([price, one_hot_body_style, embedded_body_style])( + net = fc.DenseFeatures([price, one_hot_body_style, embedded_body_style])( features) self.assertEqual(1 + 3 + 5, net.shape[1]) with _initialized_session() as sess: @@ -3513,33 +3633,33 @@ class FeatureLayerTest(test.TestCase): # Each row is formed by concatenating `embedded_body_style`, # `one_hot_body_style`, and `price` in order. - self.assertAllEqual( - [[11., 12., 13., 14., 15., 0., 0., 1., 11.], - [1., 2., 3., 4., 5., 1., 0., 0., 12]], - sess.run(net)) + self.assertAllEqual([[11., 12., 13., 14., 15., 0., 0., 1., 11.], + [1., 2., 3., 4., 5., 1., 0., 0., 12]], sess.run(net)) coord.request_stop() coord.join(threads) + @test_util.run_deprecated_v1 def test_with_1d_sparse_tensor(self): embedding_values = ( (1., 2., 3., 4., 5.), # id 0 (6., 7., 8., 9., 10.), # id 1 (11., 12., 13., 14., 15.) # id 2 ) + def _initializer(shape, dtype, partition_info): del shape, dtype, partition_info return embedding_values - # price has 1 dimension in feature_layer + # price has 1 dimension in dense_features price = fc.numeric_column('price') - # one_hot_body_style has 3 dims in feature_layer. + # one_hot_body_style has 3 dims in dense_features. body_style = fc.categorical_column_with_vocabulary_list( 'body-style', vocabulary_list=['hardtop', 'wagon', 'sedan']) one_hot_body_style = fc.indicator_column(body_style) - # embedded_body_style has 5 dims in feature_layer. + # embedded_body_style has 5 dims in dense_features. country = fc.categorical_column_with_vocabulary_list( 'country', vocabulary_list=['US', 'JP', 'CA']) embedded_country = fc.embedding_column( @@ -3547,49 +3667,56 @@ class FeatureLayerTest(test.TestCase): # Provides 1-dim tensor and dense tensor. features = { - 'price': constant_op.constant([11., 12.,]), - 'body-style': sparse_tensor.SparseTensor( - indices=((0,), (1,)), - values=('sedan', 'hardtop'), - dense_shape=(2,)), + 'price': + constant_op.constant([ + 11., + 12., + ]), + 'body-style': + sparse_tensor.SparseTensor( + indices=((0,), (1,)), + values=('sedan', 'hardtop'), + dense_shape=(2,)), # This is dense tensor for the categorical_column. - 'country': constant_op.constant(['CA', 'US']), + 'country': + constant_op.constant(['CA', 'US']), } self.assertEqual(1, features['price'].shape.ndims) self.assertEqual(1, features['body-style'].dense_shape.get_shape()[0]) self.assertEqual(1, features['country'].shape.ndims) - net = fc.FeatureLayer([price, one_hot_body_style, embedded_country])( + net = fc.DenseFeatures([price, one_hot_body_style, embedded_country])( features) self.assertEqual(1 + 3 + 5, net.shape[1]) with _initialized_session() as sess: # Each row is formed by concatenating `embedded_body_style`, # `one_hot_body_style`, and `price` in order. - self.assertAllEqual( - [[0., 0., 1., 11., 12., 13., 14., 15., 11.], - [1., 0., 0., 1., 2., 3., 4., 5., 12.]], - sess.run(net)) + self.assertAllEqual([[0., 0., 1., 11., 12., 13., 14., 15., 11.], + [1., 0., 0., 1., 2., 3., 4., 5., 12.]], + sess.run(net)) + @test_util.run_deprecated_v1 def test_with_1d_unknown_shape_sparse_tensor(self): embedding_values = ( (1., 2.), # id 0 (6., 7.), # id 1 (11., 12.) # id 2 ) + def _initializer(shape, dtype, partition_info): del shape, dtype, partition_info return embedding_values - # price has 1 dimension in feature_layer + # price has 1 dimension in dense_features price = fc.numeric_column('price') - # one_hot_body_style has 3 dims in feature_layer. + # one_hot_body_style has 3 dims in dense_features. body_style = fc.categorical_column_with_vocabulary_list( 'body-style', vocabulary_list=['hardtop', 'wagon', 'sedan']) one_hot_body_style = fc.indicator_column(body_style) - # embedded_body_style has 5 dims in feature_layer. + # embedded_body_style has 5 dims in dense_features. country = fc.categorical_column_with_vocabulary_list( 'country', vocabulary_list=['US', 'JP', 'CA']) embedded_country = fc.embedding_column( @@ -3608,12 +3735,10 @@ class FeatureLayerTest(test.TestCase): price_data = np.array([11., 12.]) body_style_data = sparse_tensor.SparseTensorValue( - indices=((0,), (1,)), - values=('sedan', 'hardtop'), - dense_shape=(2,)) + indices=((0,), (1,)), values=('sedan', 'hardtop'), dense_shape=(2,)) country_data = np.array([['US'], ['CA']]) - net = fc.FeatureLayer([price, one_hot_body_style, embedded_country])( + net = fc.DenseFeatures([price, one_hot_body_style, embedded_country])( features) self.assertEqual(1 + 3 + 2, net.shape[1]) with _initialized_session() as sess: @@ -3630,8 +3755,9 @@ class FeatureLayerTest(test.TestCase): features['country']: country_data })) + @test_util.run_deprecated_v1 def test_with_rank_0_feature(self): - # price has 1 dimension in feature_layer + # price has 1 dimension in dense_features price = fc.numeric_column('price') features = { 'price': constant_op.constant(0), @@ -3640,13 +3766,13 @@ class FeatureLayerTest(test.TestCase): # Static rank 0 should fail with self.assertRaisesRegexp(ValueError, 'Feature .* cannot have rank 0'): - fc.FeatureLayer([price])(features) + fc.DenseFeatures([price])(features) # Dynamic rank 0 should fail features = { 'price': array_ops.placeholder(dtypes.float32), } - net = fc.FeatureLayer([price])(features) + net = fc.DenseFeatures([price])(features) self.assertEqual(1, net.shape[1]) with _initialized_session() as sess: with self.assertRaisesOpError('Feature .* cannot have rank 0'): @@ -3779,16 +3905,22 @@ class FunctionalInputLayerTest(test.TestCase): with ops.Graph().as_default(): features = features = {'a': [0.]} net = fc_old.input_layer(features, fc.numeric_column('a')) - with _initialized_session(): - self.assertAllClose([[0.]], net.eval()) + + self.evaluate(variables_lib.global_variables_initializer()) + self.evaluate(lookup_ops.tables_initializer()) + + self.assertAllClose([[0.]], self.evaluate(net)) def test_column_generator(self): with ops.Graph().as_default(): features = features = {'a': [0.], 'b': [1.]} columns = (fc.numeric_column(key) for key in features) net = fc_old.input_layer(features, columns) - with _initialized_session(): - self.assertAllClose([[0., 1.]], net.eval()) + + self.evaluate(variables_lib.global_variables_initializer()) + self.evaluate(lookup_ops.tables_initializer()) + + self.assertAllClose([[0., 1.]], self.evaluate(net)) def test_raises_if_duplicate_name(self): with self.assertRaisesRegexp( @@ -3803,16 +3935,22 @@ class FunctionalInputLayerTest(test.TestCase): with ops.Graph().as_default(): features = {'price': [[1.], [5.]]} net = fc_old.input_layer(features, [price]) - with _initialized_session(): - self.assertAllClose([[1.], [5.]], net.eval()) + + self.evaluate(variables_lib.global_variables_initializer()) + self.evaluate(lookup_ops.tables_initializer()) + + self.assertAllClose([[1.], [5.]], self.evaluate(net)) def test_multi_dimension(self): price = fc.numeric_column('price', shape=2) with ops.Graph().as_default(): features = {'price': [[1., 2.], [5., 6.]]} net = fc_old.input_layer(features, [price]) - with _initialized_session(): - self.assertAllClose([[1., 2.], [5., 6.]], net.eval()) + + self.evaluate(variables_lib.global_variables_initializer()) + self.evaluate(lookup_ops.tables_initializer()) + + self.assertAllClose([[1., 2.], [5., 6.]], self.evaluate(net)) def test_raises_if_shape_mismatch(self): price = fc.numeric_column('price', shape=2) @@ -3828,8 +3966,11 @@ class FunctionalInputLayerTest(test.TestCase): with ops.Graph().as_default(): features = {'price': [[[1., 2.]], [[5., 6.]]]} net = fc_old.input_layer(features, [price]) - with _initialized_session(): - self.assertAllClose([[1., 2.], [5., 6.]], net.eval()) + + self.evaluate(variables_lib.global_variables_initializer()) + self.evaluate(lookup_ops.tables_initializer()) + + self.assertAllClose([[1., 2.], [5., 6.]], self.evaluate(net)) def test_multi_column(self): price1 = fc.numeric_column('price1', shape=2) @@ -3837,8 +3978,11 @@ class FunctionalInputLayerTest(test.TestCase): with ops.Graph().as_default(): features = {'price1': [[1., 2.], [5., 6.]], 'price2': [[3.], [4.]]} net = fc_old.input_layer(features, [price1, price2]) - with _initialized_session(): - self.assertAllClose([[1., 2., 3.], [5., 6., 4.]], net.eval()) + + self.evaluate(variables_lib.global_variables_initializer()) + self.evaluate(lookup_ops.tables_initializer()) + + self.assertAllClose([[1., 2., 3.], [5., 6., 4.]], self.evaluate(net)) def test_fills_cols_to_vars(self): # Provide three _DenseColumn's to input_layer: a _NumericColumn, a @@ -3869,6 +4013,7 @@ class FunctionalInputLayerTest(test.TestCase): variables_lib.Variable) self.assertAllEqual(cols_to_vars[some_embedding_column][0].shape, [5, 10]) + @test_util.run_deprecated_v1 def test_fills_cols_to_vars_shared_embedding(self): # Provide 5 DenseColumn's to input_layer: a NumericColumn, a # BucketizedColumn, an EmbeddingColumn, two SharedEmbeddingColumns. The @@ -3882,11 +4027,11 @@ class FunctionalInputLayerTest(test.TestCase): 'sparse_feature', hash_bucket_size=5) some_embedding_column = fc.embedding_column( some_sparse_column, dimension=10) - categorical_column_a = fc_old.categorical_column_with_identity( + categorical_column_a = fc.categorical_column_with_identity( key='aaa', num_buckets=3) - categorical_column_b = fc_old.categorical_column_with_identity( + categorical_column_b = fc.categorical_column_with_identity( key='bbb', num_buckets=3) - shared_embedding_a, shared_embedding_b = fc_old.shared_embedding_columns( + shared_embedding_a, shared_embedding_b = fc.shared_embedding_columns( [categorical_column_a, categorical_column_b], dimension=2) with ops.Graph().as_default(): features = { @@ -3968,9 +4113,12 @@ class FunctionalInputLayerTest(test.TestCase): } net1 = fc_old.input_layer(features, [price_a, price_b]) net2 = fc_old.input_layer(features, [price_b, price_a]) - with _initialized_session(): - self.assertAllClose([[1., 3.]], net1.eval()) - self.assertAllClose([[1., 3.]], net2.eval()) + + self.evaluate(variables_lib.global_variables_initializer()) + self.evaluate(lookup_ops.tables_initializer()) + + self.assertAllClose([[1., 3.]], self.evaluate(net1)) + self.assertAllClose([[1., 3.]], self.evaluate(net2)) def test_fails_for_categorical_column(self): animal = fc.categorical_column_with_identity('animal', num_buckets=4) @@ -4066,6 +4214,7 @@ class FunctionalInputLayerTest(test.TestCase): expected_var_names, [v.name for v in ops.get_collection(ops.GraphKeys.GLOBAL_VARIABLES)]) + @test_util.run_deprecated_v1 def test_with_1d_sparse_tensor(self): embedding_values = ( (1., 2., 3., 4., 5.), # id 0 @@ -4122,6 +4271,7 @@ class FunctionalInputLayerTest(test.TestCase): [1., 0., 0., 1., 2., 3., 4., 5., 12.]], sess.run(net)) + @test_util.run_deprecated_v1 def test_with_1d_unknown_shape_sparse_tensor(self): embedding_values = ( (1., 2.), # id 0 @@ -4180,6 +4330,7 @@ class FunctionalInputLayerTest(test.TestCase): features['country']: country_data })) + @test_util.run_deprecated_v1 def test_with_rank_0_feature(self): # price has 1 dimension in input_layer price = fc.numeric_column('price') @@ -4220,12 +4371,19 @@ class MakeParseExampleSpecTest(test.TestCase): def transform_feature(self, transformation_cache, state_manager): pass + def _transform_feature(self, inputs): + pass + @property def parse_example_spec(self): return self.parse_spec + @property + def _parse_example_spec(self): + return self.parse_spec + def test_no_feature_columns(self): - actual = fc.make_parse_example_spec([]) + actual = fc.make_parse_example_spec_v2([]) self.assertDictEqual({}, actual) def test_invalid_type(self): @@ -4235,15 +4393,17 @@ class MakeParseExampleSpecTest(test.TestCase): with self.assertRaisesRegexp( ValueError, 'All feature_columns must be FeatureColumn instances.*invalid_column'): - fc.make_parse_example_spec( - (self._TestFeatureColumn({key1: parse_spec1}), 'invalid_column')) + fc.make_parse_example_spec_v2((self._TestFeatureColumn({ + key1: parse_spec1 + }), 'invalid_column')) def test_one_feature_column(self): key1 = 'key1' parse_spec1 = parsing_ops.FixedLenFeature( shape=(2,), dtype=dtypes.float32, default_value=0.) - actual = fc.make_parse_example_spec( - (self._TestFeatureColumn({key1: parse_spec1}),)) + actual = fc.make_parse_example_spec_v2((self._TestFeatureColumn({ + key1: parse_spec1 + }),)) self.assertDictEqual({key1: parse_spec1}, actual) def test_two_feature_columns(self): @@ -4252,9 +4412,11 @@ class MakeParseExampleSpecTest(test.TestCase): shape=(2,), dtype=dtypes.float32, default_value=0.) key2 = 'key2' parse_spec2 = parsing_ops.VarLenFeature(dtype=dtypes.string) - actual = fc.make_parse_example_spec( - (self._TestFeatureColumn({key1: parse_spec1}), - self._TestFeatureColumn({key2: parse_spec2}))) + actual = fc.make_parse_example_spec_v2((self._TestFeatureColumn({ + key1: parse_spec1 + }), self._TestFeatureColumn({ + key2: parse_spec2 + }))) self.assertDictEqual({key1: parse_spec1, key2: parse_spec2}, actual) def test_equal_keys_different_parse_spec(self): @@ -4265,17 +4427,21 @@ class MakeParseExampleSpecTest(test.TestCase): with self.assertRaisesRegexp( ValueError, 'feature_columns contain different parse_spec for key key1'): - fc.make_parse_example_spec( - (self._TestFeatureColumn({key1: parse_spec1}), - self._TestFeatureColumn({key1: parse_spec2}))) + fc.make_parse_example_spec_v2((self._TestFeatureColumn({ + key1: parse_spec1 + }), self._TestFeatureColumn({ + key1: parse_spec2 + }))) def test_equal_keys_equal_parse_spec(self): key1 = 'key1' parse_spec1 = parsing_ops.FixedLenFeature( shape=(2,), dtype=dtypes.float32, default_value=0.) - actual = fc.make_parse_example_spec( - (self._TestFeatureColumn({key1: parse_spec1}), - self._TestFeatureColumn({key1: parse_spec1}))) + actual = fc.make_parse_example_spec_v2((self._TestFeatureColumn({ + key1: parse_spec1 + }), self._TestFeatureColumn({ + key1: parse_spec1 + }))) self.assertDictEqual({key1: parse_spec1}, actual) def test_multiple_features_dict(self): @@ -4287,11 +4453,17 @@ class MakeParseExampleSpecTest(test.TestCase): parse_spec2 = parsing_ops.VarLenFeature(dtype=dtypes.string) key3 = 'key3' parse_spec3 = parsing_ops.VarLenFeature(dtype=dtypes.int32) - actual = fc.make_parse_example_spec( - (self._TestFeatureColumn({key1: parse_spec1}), - self._TestFeatureColumn({key2: parse_spec2, key3: parse_spec3}))) - self.assertDictEqual( - {key1: parse_spec1, key2: parse_spec2, key3: parse_spec3}, actual) + actual = fc.make_parse_example_spec_v2((self._TestFeatureColumn({ + key1: parse_spec1 + }), self._TestFeatureColumn({ + key2: parse_spec2, + key3: parse_spec3 + }))) + self.assertDictEqual({ + key1: parse_spec1, + key2: parse_spec2, + key3: parse_spec3 + }, actual) def _assert_sparse_tensor_value(test_case, expected, actual): @@ -4299,7 +4471,8 @@ def _assert_sparse_tensor_value(test_case, expected, actual): test_case.assertAllEqual(expected.indices, actual.indices) test_case.assertEqual( - np.array(expected.values).dtype, np.array(actual.values).dtype) + np.array(expected.values).dtype, + np.array(actual.values).dtype) test_case.assertAllEqual(expected.values, actual.values) test_case.assertEqual(np.int64, np.array(actual.dense_shape).dtype) @@ -4321,6 +4494,7 @@ class VocabularyFileCategoricalColumnTest(test.TestCase): 'python/feature_column/testdata/wire_vocabulary.txt') self._wire_vocabulary_size = 3 + @test_util.run_deprecated_v1 def test_defaults(self): column = fc.categorical_column_with_vocabulary_file( key='aaa', vocabulary_file='path_to_file', vocabulary_size=3) @@ -4337,19 +4511,27 @@ class VocabularyFileCategoricalColumnTest(test.TestCase): fc.categorical_column_with_vocabulary_file( key=('aaa',), vocabulary_file='path_to_file', vocabulary_size=3) + @test_util.run_deprecated_v1 def test_all_constructor_args(self): column = fc.categorical_column_with_vocabulary_file( - key='aaa', vocabulary_file='path_to_file', vocabulary_size=3, - num_oov_buckets=4, dtype=dtypes.int32) + key='aaa', + vocabulary_file='path_to_file', + vocabulary_size=3, + num_oov_buckets=4, + dtype=dtypes.int32) self.assertEqual(7, column.num_buckets) self.assertEqual({ 'aaa': parsing_ops.VarLenFeature(dtypes.int32) }, column.parse_example_spec) + @test_util.run_deprecated_v1 def test_deep_copy(self): original = fc.categorical_column_with_vocabulary_file( - key='aaa', vocabulary_file='path_to_file', vocabulary_size=3, - num_oov_buckets=4, dtype=dtypes.int32) + key='aaa', + vocabulary_file='path_to_file', + vocabulary_size=3, + num_oov_buckets=4, + dtype=dtypes.int32) for column in (original, copy.deepcopy(original)): self.assertEqual('aaa', column.name) self.assertEqual(7, column.num_buckets) @@ -4367,6 +4549,7 @@ class VocabularyFileCategoricalColumnTest(test.TestCase): fc.categorical_column_with_vocabulary_file( key='aaa', vocabulary_file='', vocabulary_size=3) + @test_util.run_deprecated_v1 def test_invalid_vocabulary_file(self): column = fc.categorical_column_with_vocabulary_file( key='aaa', vocabulary_file='file_does_not_exist', vocabulary_size=10) @@ -4379,19 +4562,21 @@ class VocabularyFileCategoricalColumnTest(test.TestCase): 'aaa': inputs }), None) with self.assertRaisesRegexp(errors.OpError, 'file_does_not_exist'): - with self.cached_session(): - lookup_ops.tables_initializer().run() + self.evaluate(lookup_ops.tables_initializer()) def test_invalid_vocabulary_size(self): with self.assertRaisesRegexp(ValueError, 'Invalid vocabulary_size'): fc.categorical_column_with_vocabulary_file( - key='aaa', vocabulary_file=self._wire_vocabulary_file_name, + key='aaa', + vocabulary_file=self._wire_vocabulary_file_name, vocabulary_size=-1) with self.assertRaisesRegexp(ValueError, 'Invalid vocabulary_size'): fc.categorical_column_with_vocabulary_file( - key='aaa', vocabulary_file=self._wire_vocabulary_file_name, + key='aaa', + vocabulary_file=self._wire_vocabulary_file_name, vocabulary_size=0) + @test_util.run_deprecated_v1 def test_too_large_vocabulary_size(self): column = fc.categorical_column_with_vocabulary_file( key='aaa', @@ -4406,24 +4591,27 @@ class VocabularyFileCategoricalColumnTest(test.TestCase): 'aaa': inputs }), None) with self.assertRaisesRegexp(errors.OpError, 'Invalid vocab_size'): - with self.cached_session(): - lookup_ops.tables_initializer().run() + self.evaluate(lookup_ops.tables_initializer()) def test_invalid_num_oov_buckets(self): with self.assertRaisesRegexp(ValueError, 'Invalid num_oov_buckets'): fc.categorical_column_with_vocabulary_file( - key='aaa', vocabulary_file='path', vocabulary_size=3, + key='aaa', + vocabulary_file='path', + vocabulary_size=3, num_oov_buckets=-1) def test_invalid_dtype(self): with self.assertRaisesRegexp(ValueError, 'dtype must be string or integer'): fc.categorical_column_with_vocabulary_file( - key='aaa', vocabulary_file='path', vocabulary_size=3, + key='aaa', + vocabulary_file='path', + vocabulary_size=3, dtype=dtypes.float64) def test_invalid_buckets_and_default_value(self): - with self.assertRaisesRegexp( - ValueError, 'both num_oov_buckets and default_value'): + with self.assertRaisesRegexp(ValueError, + 'both num_oov_buckets and default_value'): fc.categorical_column_with_vocabulary_file( key='aaa', vocabulary_file=self._wire_vocabulary_file_name, @@ -4463,28 +4651,31 @@ class VocabularyFileCategoricalColumnTest(test.TestCase): 'aaa': inputs }), None) + @test_util.run_deprecated_v1 def test_parse_example(self): a = fc.categorical_column_with_vocabulary_file( key='aaa', vocabulary_file='path_to_file', vocabulary_size=3) - data = example_pb2.Example(features=feature_pb2.Features( - feature={ - 'aaa': - feature_pb2.Feature(bytes_list=feature_pb2.BytesList( - value=[b'omar', b'stringer'])) - })) + data = example_pb2.Example( + features=feature_pb2.Features( + feature={ + 'aaa': + feature_pb2.Feature( + bytes_list=feature_pb2.BytesList( + value=[b'omar', b'stringer'])) + })) features = parsing_ops.parse_example( serialized=[data.SerializeToString()], - features=fc.make_parse_example_spec([a])) + features=fc.make_parse_example_spec_v2([a])) self.assertIn('aaa', features) - with self.cached_session(): - _assert_sparse_tensor_value( - self, - sparse_tensor.SparseTensorValue( - indices=[[0, 0], [0, 1]], - values=np.array([b'omar', b'stringer'], dtype=np.object_), - dense_shape=[1, 2]), - features['aaa'].eval()) + _assert_sparse_tensor_value( + self, + sparse_tensor.SparseTensorValue( + indices=[[0, 0], [0, 1]], + values=np.array([b'omar', b'stringer'], dtype=np.object_), + dense_shape=[1, 2]), self.evaluate(features['aaa'])) + + @test_util.run_deprecated_v1 def test_get_sparse_tensors(self): column = fc.categorical_column_with_vocabulary_file( key='aaa', @@ -4499,15 +4690,19 @@ class VocabularyFileCategoricalColumnTest(test.TestCase): 'aaa': inputs }), None) self.assertIsNone(id_weight_pair.weight_tensor) - with _initialized_session(): - _assert_sparse_tensor_value( - self, - sparse_tensor.SparseTensorValue( - indices=inputs.indices, - values=np.array((2, -1, 0), dtype=np.int64), - dense_shape=inputs.dense_shape), - id_weight_pair.id_tensor.eval()) + self.evaluate(variables_lib.global_variables_initializer()) + self.evaluate(lookup_ops.tables_initializer()) + + _assert_sparse_tensor_value( + self, + sparse_tensor.SparseTensorValue( + indices=inputs.indices, + values=np.array((2, -1, 0), dtype=np.int64), + dense_shape=inputs.dense_shape), + self.evaluate(id_weight_pair.id_tensor)) + + @test_util.run_deprecated_v1 def test_get_sparse_tensors_none_vocabulary_size(self): column = fc.categorical_column_with_vocabulary_file( key='aaa', vocabulary_file=self._wire_vocabulary_file_name) @@ -4520,15 +4715,19 @@ class VocabularyFileCategoricalColumnTest(test.TestCase): 'aaa': inputs }), None) self.assertIsNone(id_weight_pair.weight_tensor) - with _initialized_session(): - _assert_sparse_tensor_value(self, - sparse_tensor.SparseTensorValue( - indices=inputs.indices, - values=np.array( - (2, -1, 0), dtype=np.int64), - dense_shape=inputs.dense_shape), - id_weight_pair.id_tensor.eval()) + self.evaluate(variables_lib.global_variables_initializer()) + self.evaluate(lookup_ops.tables_initializer()) + + _assert_sparse_tensor_value( + self, + sparse_tensor.SparseTensorValue( + indices=inputs.indices, + values=np.array((2, -1, 0), dtype=np.int64), + dense_shape=inputs.dense_shape), + self.evaluate(id_weight_pair.id_tensor)) + + @test_util.run_deprecated_v1 def test_transform_feature(self): column = fc.categorical_column_with_vocabulary_file( key='aaa', @@ -4538,16 +4737,21 @@ class VocabularyFileCategoricalColumnTest(test.TestCase): indices=((0, 0), (1, 0), (1, 1)), values=('marlo', 'skywalker', 'omar'), dense_shape=(2, 2)) - id_tensor = fc._transform_features({'aaa': inputs}, [column], None)[column] - with _initialized_session(): - _assert_sparse_tensor_value(self, - sparse_tensor.SparseTensorValue( - indices=inputs.indices, - values=np.array( - (2, -1, 0), dtype=np.int64), - dense_shape=inputs.dense_shape), - id_tensor.eval()) + id_tensor = fc._transform_features_v2({ + 'aaa': inputs + }, [column], None)[column] + self.evaluate(variables_lib.global_variables_initializer()) + self.evaluate(lookup_ops.tables_initializer()) + + _assert_sparse_tensor_value( + self, + sparse_tensor.SparseTensorValue( + indices=inputs.indices, + values=np.array((2, -1, 0), dtype=np.int64), + dense_shape=inputs.dense_shape), self.evaluate(id_tensor)) + + @test_util.run_deprecated_v1 def test_get_sparse_tensors_dense_input(self): column = fc.categorical_column_with_vocabulary_file( key='aaa', @@ -4558,15 +4762,18 @@ class VocabularyFileCategoricalColumnTest(test.TestCase): 'aaa': (('marlo', ''), ('skywalker', 'omar')) }), None) self.assertIsNone(id_weight_pair.weight_tensor) - with _initialized_session(): - _assert_sparse_tensor_value( - self, - sparse_tensor.SparseTensorValue( - indices=((0, 0), (1, 0), (1, 1)), - values=np.array((2, -1, 0), dtype=np.int64), - dense_shape=(2, 2)), - id_weight_pair.id_tensor.eval()) + self.evaluate(variables_lib.global_variables_initializer()) + self.evaluate(lookup_ops.tables_initializer()) + + _assert_sparse_tensor_value( + self, + sparse_tensor.SparseTensorValue( + indices=((0, 0), (1, 0), (1, 1)), + values=np.array((2, -1, 0), dtype=np.int64), + dense_shape=(2, 2)), self.evaluate(id_weight_pair.id_tensor)) + + @test_util.run_deprecated_v1 def test_get_sparse_tensors_default_value_in_vocabulary(self): column = fc.categorical_column_with_vocabulary_file( key='aaa', @@ -4582,15 +4789,19 @@ class VocabularyFileCategoricalColumnTest(test.TestCase): 'aaa': inputs }), None) self.assertIsNone(id_weight_pair.weight_tensor) - with _initialized_session(): - _assert_sparse_tensor_value( - self, - sparse_tensor.SparseTensorValue( - indices=inputs.indices, - values=np.array((2, 2, 0), dtype=np.int64), - dense_shape=inputs.dense_shape), - id_weight_pair.id_tensor.eval()) + self.evaluate(variables_lib.global_variables_initializer()) + self.evaluate(lookup_ops.tables_initializer()) + + _assert_sparse_tensor_value( + self, + sparse_tensor.SparseTensorValue( + indices=inputs.indices, + values=np.array((2, 2, 0), dtype=np.int64), + dense_shape=inputs.dense_shape), + self.evaluate(id_weight_pair.id_tensor)) + + @test_util.run_deprecated_v1 def test_get_sparse_tensors_with_oov_buckets(self): column = fc.categorical_column_with_vocabulary_file( key='aaa', @@ -4606,15 +4817,19 @@ class VocabularyFileCategoricalColumnTest(test.TestCase): 'aaa': inputs }), None) self.assertIsNone(id_weight_pair.weight_tensor) - with _initialized_session(): - _assert_sparse_tensor_value( - self, - sparse_tensor.SparseTensorValue( - indices=inputs.indices, - values=np.array((2, 33, 0, 62), dtype=np.int64), - dense_shape=inputs.dense_shape), - id_weight_pair.id_tensor.eval()) + self.evaluate(variables_lib.global_variables_initializer()) + self.evaluate(lookup_ops.tables_initializer()) + + _assert_sparse_tensor_value( + self, + sparse_tensor.SparseTensorValue( + indices=inputs.indices, + values=np.array((2, 33, 0, 62), dtype=np.int64), + dense_shape=inputs.dense_shape), + self.evaluate(id_weight_pair.id_tensor)) + + @test_util.run_deprecated_v1 def test_get_sparse_tensors_small_vocabulary_size(self): # 'marlo' is the last entry in our vocabulary file, so be setting # `vocabulary_size` to 1 less than number of entries in file, we take @@ -4632,15 +4847,19 @@ class VocabularyFileCategoricalColumnTest(test.TestCase): 'aaa': inputs }), None) self.assertIsNone(id_weight_pair.weight_tensor) - with _initialized_session(): - _assert_sparse_tensor_value( - self, - sparse_tensor.SparseTensorValue( - indices=inputs.indices, - values=np.array((-1, -1, 0), dtype=np.int64), - dense_shape=inputs.dense_shape), - id_weight_pair.id_tensor.eval()) + self.evaluate(variables_lib.global_variables_initializer()) + self.evaluate(lookup_ops.tables_initializer()) + + _assert_sparse_tensor_value( + self, + sparse_tensor.SparseTensorValue( + indices=inputs.indices, + values=np.array((-1, -1, 0), dtype=np.int64), + dense_shape=inputs.dense_shape), + self.evaluate(id_weight_pair.id_tensor)) + + @test_util.run_deprecated_v1 def test_get_sparse_tensors_int32(self): column = fc.categorical_column_with_vocabulary_file( key='aaa', @@ -4656,15 +4875,19 @@ class VocabularyFileCategoricalColumnTest(test.TestCase): 'aaa': inputs }), None) self.assertIsNone(id_weight_pair.weight_tensor) - with _initialized_session(): - _assert_sparse_tensor_value( - self, - sparse_tensor.SparseTensorValue( - indices=inputs.indices, - values=np.array((2, -1, 0, 4), dtype=np.int64), - dense_shape=inputs.dense_shape), - id_weight_pair.id_tensor.eval()) + self.evaluate(variables_lib.global_variables_initializer()) + self.evaluate(lookup_ops.tables_initializer()) + + _assert_sparse_tensor_value( + self, + sparse_tensor.SparseTensorValue( + indices=inputs.indices, + values=np.array((2, -1, 0, 4), dtype=np.int64), + dense_shape=inputs.dense_shape), + self.evaluate(id_weight_pair.id_tensor)) + + @test_util.run_deprecated_v1 def test_get_sparse_tensors_int32_dense_input(self): default_value = -100 column = fc.categorical_column_with_vocabulary_file( @@ -4678,15 +4901,18 @@ class VocabularyFileCategoricalColumnTest(test.TestCase): 'aaa': ((11, -1, -1), (100, 30, -1), (-1, -1, 22)) }), None) self.assertIsNone(id_weight_pair.weight_tensor) - with _initialized_session(): - _assert_sparse_tensor_value( - self, - sparse_tensor.SparseTensorValue( - indices=((0, 0), (1, 0), (1, 1), (2, 2)), - values=np.array((2, default_value, 0, 4), dtype=np.int64), - dense_shape=(3, 3)), - id_weight_pair.id_tensor.eval()) + self.evaluate(variables_lib.global_variables_initializer()) + self.evaluate(lookup_ops.tables_initializer()) + + _assert_sparse_tensor_value( + self, + sparse_tensor.SparseTensorValue( + indices=((0, 0), (1, 0), (1, 1), (2, 2)), + values=np.array((2, default_value, 0, 4), dtype=np.int64), + dense_shape=(3, 3)), self.evaluate(id_weight_pair.id_tensor)) + + @test_util.run_deprecated_v1 def test_get_sparse_tensors_int32_with_oov_buckets(self): column = fc.categorical_column_with_vocabulary_file( key='aaa', @@ -4703,15 +4929,19 @@ class VocabularyFileCategoricalColumnTest(test.TestCase): 'aaa': inputs }), None) self.assertIsNone(id_weight_pair.weight_tensor) - with _initialized_session(): - _assert_sparse_tensor_value( - self, - sparse_tensor.SparseTensorValue( - indices=inputs.indices, - values=np.array((2, 60, 0, 4), dtype=np.int64), - dense_shape=inputs.dense_shape), - id_weight_pair.id_tensor.eval()) + self.evaluate(variables_lib.global_variables_initializer()) + self.evaluate(lookup_ops.tables_initializer()) + + _assert_sparse_tensor_value( + self, + sparse_tensor.SparseTensorValue( + indices=inputs.indices, + values=np.array((2, 60, 0, 4), dtype=np.int64), + dense_shape=inputs.dense_shape), + self.evaluate(id_weight_pair.id_tensor)) + + @test_util.run_deprecated_v1 def test_linear_model(self): wire_column = fc.categorical_column_with_vocabulary_file( key='wire', @@ -4729,14 +4959,17 @@ class VocabularyFileCategoricalColumnTest(test.TestCase): dense_shape=(2, 2)) }) wire_var, bias = model.variables - with _initialized_session(): - self.assertAllClose((0.,), bias.eval()) - self.assertAllClose(((0.,), (0.,), (0.,), (0.,)), wire_var.eval()) - self.assertAllClose(((0.,), (0.,)), predictions.eval()) - wire_var.assign(((1.,), (2.,), (3.,), (4.,))).eval() - # 'marlo' -> 2: wire_var[2] = 3 - # 'skywalker' -> 3, 'omar' -> 0: wire_var[3] + wire_var[0] = 4+1 = 5 - self.assertAllClose(((3.,), (5.,)), predictions.eval()) + + self.evaluate(variables_lib.global_variables_initializer()) + self.evaluate(lookup_ops.tables_initializer()) + + self.assertAllClose((0.,), self.evaluate(bias)) + self.assertAllClose(((0.,), (0.,), (0.,), (0.,)), self.evaluate(wire_var)) + self.assertAllClose(((0.,), (0.,)), self.evaluate(predictions)) + self.evaluate(wire_var.assign(((1.,), (2.,), (3.,), (4.,)))) + # 'marlo' -> 2: wire_var[2] = 3 + # 'skywalker' -> 3, 'omar' -> 0: wire_var[3] + wire_var[0] = 4+1 = 5 + self.assertAllClose(((3.,), (5.,)), self.evaluate(predictions)) def test_old_linear_model(self): wire_column = fc.categorical_column_with_vocabulary_file( @@ -4755,15 +4988,19 @@ class VocabularyFileCategoricalColumnTest(test.TestCase): }, (wire_column,)) bias = get_linear_model_bias() wire_var = get_linear_model_column_var(wire_column) - with _initialized_session(): - self.assertAllClose((0.,), bias.eval()) - self.assertAllClose(((0.,), (0.,), (0.,), (0.,)), wire_var.eval()) - self.assertAllClose(((0.,), (0.,)), predictions.eval()) - wire_var.assign(((1.,), (2.,), (3.,), (4.,))).eval() - # 'marlo' -> 2: wire_var[2] = 3 - # 'skywalker' -> 3, 'omar' -> 0: wire_var[3] + wire_var[0] = 4+1 = 5 - self.assertAllClose(((3.,), (5.,)), predictions.eval()) + self.evaluate(variables_lib.global_variables_initializer()) + self.evaluate(lookup_ops.tables_initializer()) + + self.assertAllClose((0.,), self.evaluate(bias)) + self.assertAllClose(((0.,), (0.,), (0.,), (0.,)), self.evaluate(wire_var)) + self.assertAllClose(((0.,), (0.,)), self.evaluate(predictions)) + self.evaluate(wire_var.assign(((1.,), (2.,), (3.,), (4.,)))) + # 'marlo' -> 2: wire_var[2] = 3 + # 'skywalker' -> 3, 'omar' -> 0: wire_var[3] + wire_var[0] = 4+1 = 5 + self.assertAllClose(((3.,), (5.,)), self.evaluate(predictions)) + + @test_util.run_deprecated_v1 def test_serialization(self): wire_column = fc.categorical_column_with_vocabulary_file( key='wire', @@ -4815,15 +5052,19 @@ class VocabularyListCategoricalColumnTest(test.TestCase): 'aaa': parsing_ops.VarLenFeature(dtypes.int64) }, column.parse_example_spec) + @test_util.run_deprecated_v1 def test_all_constructor_args(self): column = fc.categorical_column_with_vocabulary_list( - key='aaa', vocabulary_list=(12, 24, 36), dtype=dtypes.int32, + key='aaa', + vocabulary_list=(12, 24, 36), + dtype=dtypes.int32, default_value=-99) self.assertEqual(3, column.num_buckets) self.assertEqual({ 'aaa': parsing_ops.VarLenFeature(dtypes.int32) }, column.parse_example_spec) + @test_util.run_deprecated_v1 def test_deep_copy(self): original = fc.categorical_column_with_vocabulary_list( key='aaa', vocabulary_list=(12, 24, 36), dtype=dtypes.int32) @@ -4837,37 +5078,39 @@ class VocabularyListCategoricalColumnTest(test.TestCase): def test_invalid_dtype(self): with self.assertRaisesRegexp(ValueError, 'dtype must be string or integer'): fc.categorical_column_with_vocabulary_list( - key='aaa', vocabulary_list=('omar', 'stringer', 'marlo'), + key='aaa', + vocabulary_list=('omar', 'stringer', 'marlo'), dtype=dtypes.float32) def test_invalid_mapping_dtype(self): - with self.assertRaisesRegexp( - ValueError, r'vocabulary dtype must be string or integer'): + with self.assertRaisesRegexp(ValueError, + r'vocabulary dtype must be string or integer'): fc.categorical_column_with_vocabulary_list( key='aaa', vocabulary_list=(12., 24., 36.)) def test_mismatched_int_dtype(self): - with self.assertRaisesRegexp( - ValueError, r'dtype.*and vocabulary dtype.*do not match'): + with self.assertRaisesRegexp(ValueError, + r'dtype.*and vocabulary dtype.*do not match'): fc.categorical_column_with_vocabulary_list( - key='aaa', vocabulary_list=('omar', 'stringer', 'marlo'), + key='aaa', + vocabulary_list=('omar', 'stringer', 'marlo'), dtype=dtypes.int32) def test_mismatched_string_dtype(self): - with self.assertRaisesRegexp( - ValueError, r'dtype.*and vocabulary dtype.*do not match'): + with self.assertRaisesRegexp(ValueError, + r'dtype.*and vocabulary dtype.*do not match'): fc.categorical_column_with_vocabulary_list( key='aaa', vocabulary_list=(12, 24, 36), dtype=dtypes.string) def test_none_mapping(self): - with self.assertRaisesRegexp( - ValueError, r'vocabulary_list.*must be non-empty'): + with self.assertRaisesRegexp(ValueError, + r'vocabulary_list.*must be non-empty'): fc.categorical_column_with_vocabulary_list( key='aaa', vocabulary_list=None) def test_empty_mapping(self): - with self.assertRaisesRegexp( - ValueError, r'vocabulary_list.*must be non-empty'): + with self.assertRaisesRegexp(ValueError, + r'vocabulary_list.*must be non-empty'): fc.categorical_column_with_vocabulary_list( key='aaa', vocabulary_list=tuple([])) @@ -4879,12 +5122,11 @@ class VocabularyListCategoricalColumnTest(test.TestCase): def test_invalid_num_oov_buckets(self): with self.assertRaisesRegexp(ValueError, 'Invalid num_oov_buckets'): fc.categorical_column_with_vocabulary_list( - key='aaa', vocabulary_list=(12, 24, 36), - num_oov_buckets=-1) + key='aaa', vocabulary_list=(12, 24, 36), num_oov_buckets=-1) def test_invalid_buckets_and_default_value(self): - with self.assertRaisesRegexp( - ValueError, 'both num_oov_buckets and default_value'): + with self.assertRaisesRegexp(ValueError, + 'both num_oov_buckets and default_value'): fc.categorical_column_with_vocabulary_list( key='aaa', vocabulary_list=(12, 24, 36), @@ -4893,8 +5135,7 @@ class VocabularyListCategoricalColumnTest(test.TestCase): def test_invalid_input_dtype_int32(self): column = fc.categorical_column_with_vocabulary_list( - key='aaa', - vocabulary_list=('omar', 'stringer', 'marlo')) + key='aaa', vocabulary_list=('omar', 'stringer', 'marlo')) inputs = sparse_tensor.SparseTensorValue( indices=((0, 0), (1, 0), (1, 1)), values=(12, 24, 36), @@ -4907,8 +5148,7 @@ class VocabularyListCategoricalColumnTest(test.TestCase): def test_invalid_input_dtype_string(self): column = fc.categorical_column_with_vocabulary_list( - key='aaa', - vocabulary_list=(12, 24, 36)) + key='aaa', vocabulary_list=(12, 24, 36)) inputs = sparse_tensor.SparseTensorValue( indices=((0, 0), (1, 0), (1, 1)), values=('omar', 'stringer', 'marlo'), @@ -4919,54 +5159,56 @@ class VocabularyListCategoricalColumnTest(test.TestCase): 'aaa': inputs }), None) + @test_util.run_deprecated_v1 def test_parse_example_string(self): a = fc.categorical_column_with_vocabulary_list( key='aaa', vocabulary_list=('omar', 'stringer', 'marlo')) - data = example_pb2.Example(features=feature_pb2.Features( - feature={ - 'aaa': - feature_pb2.Feature(bytes_list=feature_pb2.BytesList( - value=[b'omar', b'stringer'])) - })) + data = example_pb2.Example( + features=feature_pb2.Features( + feature={ + 'aaa': + feature_pb2.Feature( + bytes_list=feature_pb2.BytesList( + value=[b'omar', b'stringer'])) + })) features = parsing_ops.parse_example( serialized=[data.SerializeToString()], - features=fc.make_parse_example_spec([a])) + features=fc.make_parse_example_spec_v2([a])) self.assertIn('aaa', features) - with self.cached_session(): - _assert_sparse_tensor_value( - self, - sparse_tensor.SparseTensorValue( - indices=[[0, 0], [0, 1]], - values=np.array([b'omar', b'stringer'], dtype=np.object_), - dense_shape=[1, 2]), - features['aaa'].eval()) + _assert_sparse_tensor_value( + self, + sparse_tensor.SparseTensorValue( + indices=[[0, 0], [0, 1]], + values=np.array([b'omar', b'stringer'], dtype=np.object_), + dense_shape=[1, 2]), self.evaluate(features['aaa'])) + + @test_util.run_deprecated_v1 def test_parse_example_int(self): a = fc.categorical_column_with_vocabulary_list( key='aaa', vocabulary_list=(11, 21, 31)) - data = example_pb2.Example(features=feature_pb2.Features( - feature={ - 'aaa': - feature_pb2.Feature(int64_list=feature_pb2.Int64List( - value=[11, 21])) - })) + data = example_pb2.Example( + features=feature_pb2.Features( + feature={ + 'aaa': + feature_pb2.Feature( + int64_list=feature_pb2.Int64List(value=[11, 21])) + })) features = parsing_ops.parse_example( serialized=[data.SerializeToString()], - features=fc.make_parse_example_spec([a])) + features=fc.make_parse_example_spec_v2([a])) self.assertIn('aaa', features) - with self.cached_session(): - _assert_sparse_tensor_value( - self, - sparse_tensor.SparseTensorValue( - indices=[[0, 0], [0, 1]], - values=[11, 21], - dense_shape=[1, 2]), - features['aaa'].eval()) + _assert_sparse_tensor_value( + self, + sparse_tensor.SparseTensorValue( + indices=[[0, 0], [0, 1]], values=[11, 21], dense_shape=[1, 2]), + self.evaluate(features['aaa'])) + + @test_util.run_deprecated_v1 def test_get_sparse_tensors(self): column = fc.categorical_column_with_vocabulary_list( - key='aaa', - vocabulary_list=('omar', 'stringer', 'marlo')) + key='aaa', vocabulary_list=('omar', 'stringer', 'marlo')) inputs = sparse_tensor.SparseTensorValue( indices=((0, 0), (1, 0), (1, 1)), values=('marlo', 'skywalker', 'omar'), @@ -4976,51 +5218,61 @@ class VocabularyListCategoricalColumnTest(test.TestCase): 'aaa': inputs }), None) self.assertIsNone(id_weight_pair.weight_tensor) - with _initialized_session(): - _assert_sparse_tensor_value( - self, - sparse_tensor.SparseTensorValue( - indices=inputs.indices, - values=np.array((2, -1, 0), dtype=np.int64), - dense_shape=inputs.dense_shape), - id_weight_pair.id_tensor.eval()) + self.evaluate(variables_lib.global_variables_initializer()) + self.evaluate(lookup_ops.tables_initializer()) + + _assert_sparse_tensor_value( + self, + sparse_tensor.SparseTensorValue( + indices=inputs.indices, + values=np.array((2, -1, 0), dtype=np.int64), + dense_shape=inputs.dense_shape), + self.evaluate(id_weight_pair.id_tensor)) + + @test_util.run_deprecated_v1 def test_transform_feature(self): column = fc.categorical_column_with_vocabulary_list( - key='aaa', - vocabulary_list=('omar', 'stringer', 'marlo')) + key='aaa', vocabulary_list=('omar', 'stringer', 'marlo')) inputs = sparse_tensor.SparseTensorValue( indices=((0, 0), (1, 0), (1, 1)), values=('marlo', 'skywalker', 'omar'), dense_shape=(2, 2)) - id_tensor = fc._transform_features({'aaa': inputs}, [column], None)[column] - with _initialized_session(): - _assert_sparse_tensor_value( - self, - sparse_tensor.SparseTensorValue( - indices=inputs.indices, - values=np.array((2, -1, 0), dtype=np.int64), - dense_shape=inputs.dense_shape), - id_tensor.eval()) + id_tensor = fc._transform_features_v2({ + 'aaa': inputs + }, [column], None)[column] + self.evaluate(variables_lib.global_variables_initializer()) + self.evaluate(lookup_ops.tables_initializer()) + + _assert_sparse_tensor_value( + self, + sparse_tensor.SparseTensorValue( + indices=inputs.indices, + values=np.array((2, -1, 0), dtype=np.int64), + dense_shape=inputs.dense_shape), self.evaluate(id_tensor)) + + @test_util.run_deprecated_v1 def test_get_sparse_tensors_dense_input(self): column = fc.categorical_column_with_vocabulary_list( - key='aaa', - vocabulary_list=('omar', 'stringer', 'marlo')) + key='aaa', vocabulary_list=('omar', 'stringer', 'marlo')) id_weight_pair = column.get_sparse_tensors( fc.FeatureTransformationCache({ 'aaa': (('marlo', ''), ('skywalker', 'omar')) }), None) self.assertIsNone(id_weight_pair.weight_tensor) - with _initialized_session(): - _assert_sparse_tensor_value( - self, - sparse_tensor.SparseTensorValue( - indices=((0, 0), (1, 0), (1, 1)), - values=np.array((2, -1, 0), dtype=np.int64), - dense_shape=(2, 2)), - id_weight_pair.id_tensor.eval()) + self.evaluate(variables_lib.global_variables_initializer()) + self.evaluate(lookup_ops.tables_initializer()) + + _assert_sparse_tensor_value( + self, + sparse_tensor.SparseTensorValue( + indices=((0, 0), (1, 0), (1, 1)), + values=np.array((2, -1, 0), dtype=np.int64), + dense_shape=(2, 2)), self.evaluate(id_weight_pair.id_tensor)) + + @test_util.run_deprecated_v1 def test_get_sparse_tensors_default_value_in_vocabulary(self): column = fc.categorical_column_with_vocabulary_list( key='aaa', @@ -5035,15 +5287,19 @@ class VocabularyListCategoricalColumnTest(test.TestCase): 'aaa': inputs }), None) self.assertIsNone(id_weight_pair.weight_tensor) - with _initialized_session(): - _assert_sparse_tensor_value( - self, - sparse_tensor.SparseTensorValue( - indices=inputs.indices, - values=np.array((2, 2, 0), dtype=np.int64), - dense_shape=inputs.dense_shape), - id_weight_pair.id_tensor.eval()) + self.evaluate(variables_lib.global_variables_initializer()) + self.evaluate(lookup_ops.tables_initializer()) + + _assert_sparse_tensor_value( + self, + sparse_tensor.SparseTensorValue( + indices=inputs.indices, + values=np.array((2, 2, 0), dtype=np.int64), + dense_shape=inputs.dense_shape), + self.evaluate(id_weight_pair.id_tensor)) + + @test_util.run_deprecated_v1 def test_get_sparse_tensors_with_oov_buckets(self): column = fc.categorical_column_with_vocabulary_list( key='aaa', @@ -5058,15 +5314,19 @@ class VocabularyListCategoricalColumnTest(test.TestCase): 'aaa': inputs }), None) self.assertIsNone(id_weight_pair.weight_tensor) - with _initialized_session(): - _assert_sparse_tensor_value( - self, - sparse_tensor.SparseTensorValue( - indices=inputs.indices, - values=np.array((2, 33, 0, 62), dtype=np.int64), - dense_shape=inputs.dense_shape), - id_weight_pair.id_tensor.eval()) + self.evaluate(variables_lib.global_variables_initializer()) + self.evaluate(lookup_ops.tables_initializer()) + + _assert_sparse_tensor_value( + self, + sparse_tensor.SparseTensorValue( + indices=inputs.indices, + values=np.array((2, 33, 0, 62), dtype=np.int64), + dense_shape=inputs.dense_shape), + self.evaluate(id_weight_pair.id_tensor)) + + @test_util.run_deprecated_v1 def test_get_sparse_tensors_int32(self): column = fc.categorical_column_with_vocabulary_list( key='aaa', @@ -5081,15 +5341,19 @@ class VocabularyListCategoricalColumnTest(test.TestCase): 'aaa': inputs }), None) self.assertIsNone(id_weight_pair.weight_tensor) - with _initialized_session(): - _assert_sparse_tensor_value( - self, - sparse_tensor.SparseTensorValue( - indices=inputs.indices, - values=np.array((2, -1, 0, 4), dtype=np.int64), - dense_shape=inputs.dense_shape), - id_weight_pair.id_tensor.eval()) + self.evaluate(variables_lib.global_variables_initializer()) + self.evaluate(lookup_ops.tables_initializer()) + + _assert_sparse_tensor_value( + self, + sparse_tensor.SparseTensorValue( + indices=inputs.indices, + values=np.array((2, -1, 0, 4), dtype=np.int64), + dense_shape=inputs.dense_shape), + self.evaluate(id_weight_pair.id_tensor)) + + @test_util.run_deprecated_v1 def test_get_sparse_tensors_int32_dense_input(self): default_value = -100 column = fc.categorical_column_with_vocabulary_list( @@ -5104,15 +5368,18 @@ class VocabularyListCategoricalColumnTest(test.TestCase): dtype=np.int32) }), None) self.assertIsNone(id_weight_pair.weight_tensor) - with _initialized_session(): - _assert_sparse_tensor_value( - self, - sparse_tensor.SparseTensorValue( - indices=((0, 0), (1, 0), (1, 1), (2, 2)), - values=np.array((2, default_value, 0, 4), dtype=np.int64), - dense_shape=(3, 3)), - id_weight_pair.id_tensor.eval()) + self.evaluate(variables_lib.global_variables_initializer()) + self.evaluate(lookup_ops.tables_initializer()) + + _assert_sparse_tensor_value( + self, + sparse_tensor.SparseTensorValue( + indices=((0, 0), (1, 0), (1, 1), (2, 2)), + values=np.array((2, default_value, 0, 4), dtype=np.int64), + dense_shape=(3, 3)), self.evaluate(id_weight_pair.id_tensor)) + + @test_util.run_deprecated_v1 def test_get_sparse_tensors_int32_with_oov_buckets(self): column = fc.categorical_column_with_vocabulary_list( key='aaa', @@ -5128,15 +5395,19 @@ class VocabularyListCategoricalColumnTest(test.TestCase): 'aaa': inputs }), None) self.assertIsNone(id_weight_pair.weight_tensor) - with _initialized_session(): - _assert_sparse_tensor_value( - self, - sparse_tensor.SparseTensorValue( - indices=inputs.indices, - values=np.array((2, 60, 0, 4), dtype=np.int64), - dense_shape=inputs.dense_shape), - id_weight_pair.id_tensor.eval()) + self.evaluate(variables_lib.global_variables_initializer()) + self.evaluate(lookup_ops.tables_initializer()) + + _assert_sparse_tensor_value( + self, + sparse_tensor.SparseTensorValue( + indices=inputs.indices, + values=np.array((2, 60, 0, 4), dtype=np.int64), + dense_shape=inputs.dense_shape), + self.evaluate(id_weight_pair.id_tensor)) + + @test_util.run_deprecated_v1 def test_linear_model(self): wire_column = fc.categorical_column_with_vocabulary_list( key='aaa', @@ -5153,14 +5424,17 @@ class VocabularyListCategoricalColumnTest(test.TestCase): dense_shape=(2, 2)) }) wire_var, bias = model.variables - with _initialized_session(): - self.assertAllClose((0.,), bias.eval()) - self.assertAllClose(((0.,), (0.,), (0.,), (0.,)), wire_var.eval()) - self.assertAllClose(((0.,), (0.,)), predictions.eval()) - wire_var.assign(((1.,), (2.,), (3.,), (4.,))).eval() - # 'marlo' -> 2: wire_var[2] = 3 - # 'skywalker' -> 3, 'omar' -> 0: wire_var[3] + wire_var[0] = 4+1 = 5 - self.assertAllClose(((3.,), (5.,)), predictions.eval()) + + self.evaluate(variables_lib.global_variables_initializer()) + self.evaluate(lookup_ops.tables_initializer()) + + self.assertAllClose((0.,), self.evaluate(bias)) + self.assertAllClose(((0.,), (0.,), (0.,), (0.,)), self.evaluate(wire_var)) + self.assertAllClose(((0.,), (0.,)), self.evaluate(predictions)) + self.evaluate(wire_var.assign(((1.,), (2.,), (3.,), (4.,)))) + # 'marlo' -> 2: wire_var[2] = 3 + # 'skywalker' -> 3, 'omar' -> 0: wire_var[3] + wire_var[0] = 4+1 = 5 + self.assertAllClose(((3.,), (5.,)), self.evaluate(predictions)) def test_old_linear_model(self): wire_column = fc.categorical_column_with_vocabulary_list( @@ -5178,15 +5452,19 @@ class VocabularyListCategoricalColumnTest(test.TestCase): }, (wire_column,)) bias = get_linear_model_bias() wire_var = get_linear_model_column_var(wire_column) - with _initialized_session(): - self.assertAllClose((0.,), bias.eval()) - self.assertAllClose(((0.,), (0.,), (0.,), (0.,)), wire_var.eval()) - self.assertAllClose(((0.,), (0.,)), predictions.eval()) - wire_var.assign(((1.,), (2.,), (3.,), (4.,))).eval() - # 'marlo' -> 2: wire_var[2] = 3 - # 'skywalker' -> 3, 'omar' -> 0: wire_var[3] + wire_var[0] = 4+1 = 5 - self.assertAllClose(((3.,), (5.,)), predictions.eval()) + self.evaluate(variables_lib.global_variables_initializer()) + self.evaluate(lookup_ops.tables_initializer()) + + self.assertAllClose((0.,), self.evaluate(bias)) + self.assertAllClose(((0.,), (0.,), (0.,), (0.,)), self.evaluate(wire_var)) + self.assertAllClose(((0.,), (0.,)), self.evaluate(predictions)) + self.evaluate(wire_var.assign(((1.,), (2.,), (3.,), (4.,)))) + # 'marlo' -> 2: wire_var[2] = 3 + # 'skywalker' -> 3, 'omar' -> 0: wire_var[3] + wire_var[0] = 4+1 = 5 + self.assertAllClose(((3.,), (5.,)), self.evaluate(predictions)) + + @test_util.run_deprecated_v1 def test_serialization(self): wire_column = fc.categorical_column_with_vocabulary_list( key='aaa', @@ -5208,7 +5486,6 @@ class VocabularyListCategoricalColumnTest(test.TestCase): fc.VocabularyListCategoricalColumn._from_config(config)) - class IdentityCategoricalColumnTest(test.TestCase): def test_constructor(self): @@ -5225,6 +5502,7 @@ class IdentityCategoricalColumnTest(test.TestCase): with self.assertRaisesRegexp(ValueError, 'key must be a string.'): fc.categorical_column_with_identity(key=('aaa',), num_buckets=3) + @test_util.run_deprecated_v1 def test_deep_copy(self): original = fc.categorical_column_with_identity(key='aaa', num_buckets=3) for column in (original, copy.deepcopy(original)): @@ -5264,63 +5542,70 @@ class IdentityCategoricalColumnTest(test.TestCase): 'aaa': inputs }), None) + @test_util.run_deprecated_v1 def test_parse_example(self): a = fc.categorical_column_with_identity(key='aaa', num_buckets=30) - data = example_pb2.Example(features=feature_pb2.Features( - feature={ - 'aaa': - feature_pb2.Feature(int64_list=feature_pb2.Int64List( - value=[11, 21])) - })) + data = example_pb2.Example( + features=feature_pb2.Features( + feature={ + 'aaa': + feature_pb2.Feature( + int64_list=feature_pb2.Int64List(value=[11, 21])) + })) features = parsing_ops.parse_example( serialized=[data.SerializeToString()], - features=fc.make_parse_example_spec([a])) + features=fc.make_parse_example_spec_v2([a])) self.assertIn('aaa', features) - with self.cached_session(): - _assert_sparse_tensor_value( - self, - sparse_tensor.SparseTensorValue( - indices=[[0, 0], [0, 1]], - values=np.array([11, 21], dtype=np.int64), - dense_shape=[1, 2]), - features['aaa'].eval()) + _assert_sparse_tensor_value( + self, + sparse_tensor.SparseTensorValue( + indices=[[0, 0], [0, 1]], + values=np.array([11, 21], dtype=np.int64), + dense_shape=[1, 2]), self.evaluate(features['aaa'])) + + @test_util.run_deprecated_v1 def test_get_sparse_tensors(self): column = fc.categorical_column_with_identity(key='aaa', num_buckets=3) inputs = sparse_tensor.SparseTensorValue( - indices=((0, 0), (1, 0), (1, 1)), - values=(0, 1, 0), - dense_shape=(2, 2)) + indices=((0, 0), (1, 0), (1, 1)), values=(0, 1, 0), dense_shape=(2, 2)) id_weight_pair = column.get_sparse_tensors( fc.FeatureTransformationCache({ 'aaa': inputs }), None) self.assertIsNone(id_weight_pair.weight_tensor) - with _initialized_session(): - _assert_sparse_tensor_value( - self, - sparse_tensor.SparseTensorValue( - indices=inputs.indices, - values=np.array((0, 1, 0), dtype=np.int64), - dense_shape=inputs.dense_shape), - id_weight_pair.id_tensor.eval()) + self.evaluate(variables_lib.global_variables_initializer()) + self.evaluate(lookup_ops.tables_initializer()) + + _assert_sparse_tensor_value( + self, + sparse_tensor.SparseTensorValue( + indices=inputs.indices, + values=np.array((0, 1, 0), dtype=np.int64), + dense_shape=inputs.dense_shape), + self.evaluate(id_weight_pair.id_tensor)) + + @test_util.run_deprecated_v1 def test_transform_feature(self): column = fc.categorical_column_with_identity(key='aaa', num_buckets=3) inputs = sparse_tensor.SparseTensorValue( - indices=((0, 0), (1, 0), (1, 1)), - values=(0, 1, 0), - dense_shape=(2, 2)) - id_tensor = fc._transform_features({'aaa': inputs}, [column], None)[column] - with _initialized_session(): - _assert_sparse_tensor_value( - self, - sparse_tensor.SparseTensorValue( - indices=inputs.indices, - values=np.array((0, 1, 0), dtype=np.int64), - dense_shape=inputs.dense_shape), - id_tensor.eval()) + indices=((0, 0), (1, 0), (1, 1)), values=(0, 1, 0), dense_shape=(2, 2)) + id_tensor = fc._transform_features_v2({ + 'aaa': inputs + }, [column], None)[column] + self.evaluate(variables_lib.global_variables_initializer()) + self.evaluate(lookup_ops.tables_initializer()) + + _assert_sparse_tensor_value( + self, + sparse_tensor.SparseTensorValue( + indices=inputs.indices, + values=np.array((0, 1, 0), dtype=np.int64), + dense_shape=inputs.dense_shape), self.evaluate(id_tensor)) + + @test_util.run_deprecated_v1 def test_get_sparse_tensors_dense_input(self): column = fc.categorical_column_with_identity(key='aaa', num_buckets=3) id_weight_pair = column.get_sparse_tensors( @@ -5328,47 +5613,53 @@ class IdentityCategoricalColumnTest(test.TestCase): 'aaa': ((0, -1), (1, 0)) }), None) self.assertIsNone(id_weight_pair.weight_tensor) - with _initialized_session(): - _assert_sparse_tensor_value( - self, - sparse_tensor.SparseTensorValue( - indices=((0, 0), (1, 0), (1, 1)), - values=np.array((0, 1, 0), dtype=np.int64), - dense_shape=(2, 2)), - id_weight_pair.id_tensor.eval()) + self.evaluate(variables_lib.global_variables_initializer()) + self.evaluate(lookup_ops.tables_initializer()) + + _assert_sparse_tensor_value( + self, + sparse_tensor.SparseTensorValue( + indices=((0, 0), (1, 0), (1, 1)), + values=np.array((0, 1, 0), dtype=np.int64), + dense_shape=(2, 2)), self.evaluate(id_weight_pair.id_tensor)) + + @test_util.run_deprecated_v1 def test_get_sparse_tensors_with_inputs_too_small(self): column = fc.categorical_column_with_identity(key='aaa', num_buckets=3) inputs = sparse_tensor.SparseTensorValue( - indices=((0, 0), (1, 0), (1, 1)), - values=(1, -1, 0), - dense_shape=(2, 2)) + indices=((0, 0), (1, 0), (1, 1)), values=(1, -1, 0), dense_shape=(2, 2)) id_weight_pair = column.get_sparse_tensors( fc.FeatureTransformationCache({ 'aaa': inputs }), None) self.assertIsNone(id_weight_pair.weight_tensor) - with _initialized_session(): - with self.assertRaisesRegexp( - errors.OpError, 'assert_greater_or_equal_0'): - id_weight_pair.id_tensor.eval() + self.evaluate(variables_lib.global_variables_initializer()) + self.evaluate(lookup_ops.tables_initializer()) + + with self.assertRaisesRegexp(errors.OpError, 'assert_greater_or_equal_0'): + self.evaluate(id_weight_pair.id_tensor) + + @test_util.run_deprecated_v1 def test_get_sparse_tensors_with_inputs_too_big(self): column = fc.categorical_column_with_identity(key='aaa', num_buckets=3) inputs = sparse_tensor.SparseTensorValue( - indices=((0, 0), (1, 0), (1, 1)), - values=(1, 99, 0), - dense_shape=(2, 2)) + indices=((0, 0), (1, 0), (1, 1)), values=(1, 99, 0), dense_shape=(2, 2)) id_weight_pair = column.get_sparse_tensors( fc.FeatureTransformationCache({ 'aaa': inputs }), None) self.assertIsNone(id_weight_pair.weight_tensor) - with _initialized_session(): - with self.assertRaisesRegexp( - errors.OpError, 'assert_less_than_num_buckets'): - id_weight_pair.id_tensor.eval() + self.evaluate(variables_lib.global_variables_initializer()) + self.evaluate(lookup_ops.tables_initializer()) + + with self.assertRaisesRegexp(errors.OpError, + 'assert_less_than_num_buckets'): + self.evaluate(id_weight_pair.id_tensor) + + @test_util.run_deprecated_v1 def test_get_sparse_tensors_with_default_value(self): column = fc.categorical_column_with_identity( key='aaa', num_buckets=4, default_value=3) @@ -5381,15 +5672,19 @@ class IdentityCategoricalColumnTest(test.TestCase): 'aaa': inputs }), None) self.assertIsNone(id_weight_pair.weight_tensor) - with _initialized_session(): - _assert_sparse_tensor_value( - self, - sparse_tensor.SparseTensorValue( - indices=inputs.indices, - values=np.array((1, 3, 3), dtype=np.int64), - dense_shape=inputs.dense_shape), - id_weight_pair.id_tensor.eval()) + self.evaluate(variables_lib.global_variables_initializer()) + self.evaluate(lookup_ops.tables_initializer()) + + _assert_sparse_tensor_value( + self, + sparse_tensor.SparseTensorValue( + indices=inputs.indices, + values=np.array((1, 3, 3), dtype=np.int64), + dense_shape=inputs.dense_shape), + self.evaluate(id_weight_pair.id_tensor)) + + @test_util.run_deprecated_v1 def test_get_sparse_tensors_with_default_value_and_placeholder_inputs(self): column = fc.categorical_column_with_identity( key='aaa', num_buckets=4, default_value=3) @@ -5397,14 +5692,15 @@ class IdentityCategoricalColumnTest(test.TestCase): input_values = array_ops.placeholder(dtype=dtypes.int32) input_shape = array_ops.placeholder(dtype=dtypes.int64) inputs = sparse_tensor.SparseTensorValue( - indices=input_indices, - values=input_values, - dense_shape=input_shape) + indices=input_indices, values=input_values, dense_shape=input_shape) id_weight_pair = column.get_sparse_tensors( fc.FeatureTransformationCache({ 'aaa': inputs }), None) self.assertIsNone(id_weight_pair.weight_tensor) + + self.evaluate(variables_lib.global_variables_initializer()) + self.evaluate(lookup_ops.tables_initializer()) with _initialized_session(): _assert_sparse_tensor_value( self, @@ -5412,12 +5708,14 @@ class IdentityCategoricalColumnTest(test.TestCase): indices=np.array(((0, 0), (1, 0), (1, 1)), dtype=np.int64), values=np.array((1, 3, 3), dtype=np.int64), dense_shape=np.array((2, 2), dtype=np.int64)), - id_weight_pair.id_tensor.eval(feed_dict={ - input_indices: ((0, 0), (1, 0), (1, 1)), - input_values: (1, -1, 99), - input_shape: (2, 2), - })) + id_weight_pair.id_tensor.eval( + feed_dict={ + input_indices: ((0, 0), (1, 0), (1, 1)), + input_values: (1, -1, 99), + input_shape: (2, 2), + })) + @test_util.run_deprecated_v1 def test_linear_model(self): column = fc.categorical_column_with_identity(key='aaa', num_buckets=3) self.assertEqual(3, column.num_buckets) @@ -5431,14 +5729,17 @@ class IdentityCategoricalColumnTest(test.TestCase): dense_shape=(2, 2)) }) weight_var, bias = model.variables - with _initialized_session(): - self.assertAllClose((0.,), bias.eval()) - self.assertAllClose(((0.,), (0.,), (0.,)), weight_var.eval()) - self.assertAllClose(((0.,), (0.,)), predictions.eval()) - weight_var.assign(((1.,), (2.,), (3.,))).eval() - # weight_var[0] = 1 - # weight_var[2] + weight_var[1] = 3+2 = 5 - self.assertAllClose(((1.,), (5.,)), predictions.eval()) + + self.evaluate(variables_lib.global_variables_initializer()) + self.evaluate(lookup_ops.tables_initializer()) + + self.assertAllClose((0.,), self.evaluate(bias)) + self.assertAllClose(((0.,), (0.,), (0.,)), self.evaluate(weight_var)) + self.assertAllClose(((0.,), (0.,)), self.evaluate(predictions)) + self.evaluate(weight_var.assign(((1.,), (2.,), (3.,)))) + # weight_var[0] = 1 + # weight_var[2] + weight_var[1] = 3+2 = 5 + self.assertAllClose(((1.,), (5.,)), self.evaluate(predictions)) def test_old_linear_model(self): column = fc.categorical_column_with_identity(key='aaa', num_buckets=3) @@ -5453,15 +5754,19 @@ class IdentityCategoricalColumnTest(test.TestCase): }, (column,)) bias = get_linear_model_bias() weight_var = get_linear_model_column_var(column) - with _initialized_session(): - self.assertAllClose((0.,), bias.eval()) - self.assertAllClose(((0.,), (0.,), (0.,)), weight_var.eval()) - self.assertAllClose(((0.,), (0.,)), predictions.eval()) - weight_var.assign(((1.,), (2.,), (3.,))).eval() - # weight_var[0] = 1 - # weight_var[2] + weight_var[1] = 3+2 = 5 - self.assertAllClose(((1.,), (5.,)), predictions.eval()) + self.evaluate(variables_lib.global_variables_initializer()) + self.evaluate(lookup_ops.tables_initializer()) + + self.assertAllClose((0.,), self.evaluate(bias)) + self.assertAllClose(((0.,), (0.,), (0.,)), self.evaluate(weight_var)) + self.assertAllClose(((0.,), (0.,)), self.evaluate(predictions)) + self.evaluate(weight_var.assign(((1.,), (2.,), (3.,)))) + # weight_var[0] = 1 + # weight_var[2] + weight_var[1] = 3+2 = 5 + self.assertAllClose(((1.,), (5.,)), self.evaluate(predictions)) + + @test_util.run_deprecated_v1 def test_serialization(self): column = fc.categorical_column_with_identity(key='aaa', num_buckets=3) @@ -5494,13 +5799,18 @@ class TransformFeaturesTest(test.TestCase): indices=[[0, 0], [1, 0], [1, 1]], dense_shape=[2, 2]) } - transformed = fc._transform_features( + transformed = fc._transform_features_v2( features, [bucketized_price, hashed_sparse], None) - with _initialized_session(): - self.assertIn(bucketized_price.name, transformed[bucketized_price].name) - self.assertAllEqual([[0], [3]], transformed[bucketized_price].eval()) - self.assertIn(hashed_sparse.name, transformed[hashed_sparse].name) - self.assertAllEqual([6, 4, 1], transformed[hashed_sparse].values.eval()) + + self.evaluate(variables_lib.global_variables_initializer()) + self.evaluate(lookup_ops.tables_initializer()) + + self.assertIn(bucketized_price.name, transformed[bucketized_price].name) + self.assertAllEqual([[0], [3]], + self.evaluate(transformed[bucketized_price])) + self.assertIn(hashed_sparse.name, transformed[hashed_sparse].name) + self.assertAllEqual([6, 4, 1], + self.evaluate(transformed[hashed_sparse].values)) def test_column_order(self): """When the column is both dense and sparse, uses sparse tensors.""" @@ -5531,12 +5841,12 @@ class TransformFeaturesTest(test.TestCase): column1 = _LoggerColumn('1') column2 = _LoggerColumn('2') call_logger = {'count': 0} - fc._transform_features({}, [column1, column2], None) + fc._transform_features_v2({}, [column1, column2], None) self.assertEqual(0, column1.call_order) self.assertEqual(1, column2.call_order) call_logger = {'count': 0} - fc._transform_features({}, [column2, column1], None) + fc._transform_features_v2({}, [column2, column1], None) self.assertEqual(0, column1.call_order) self.assertEqual(1, column2.call_order) @@ -5551,7 +5861,7 @@ class IndicatorColumnTest(test.TestCase): self.assertEqual(indicator_a.variable_shape, [1, 4]) self.assertTrue(indicator_a._is_v2_column) - b = fc_old.categorical_column_with_hash_bucket('b', hash_bucket_size=100) + b = fc_old._categorical_column_with_hash_bucket('b', hash_bucket_size=100) indicator_b = fc.indicator_column(b) self.assertEqual(indicator_b.categorical_column.name, 'b') self.assertEqual(indicator_b.name, 'b_indicator') @@ -5565,8 +5875,9 @@ class IndicatorColumnTest(test.TestCase): 'animal': ['fox', 'fox'] }) output = transformation_cache.get(animal, None) - with self.cached_session(): - self.assertAllEqual([[0., 0., 1., 0.], [0., 0., 1., 0.]], output.eval()) + + self.assertAllEqual([[0., 0., 1., 0.], [0., 0., 1., 0.]], + self.evaluate(output)) def test_2D_shape_succeeds(self): # TODO(ispir/cassandrax): Swith to categorical_column_with_keys when ready. @@ -5580,8 +5891,9 @@ class IndicatorColumnTest(test.TestCase): dense_shape=[2, 1]) }) output = transformation_cache.get(animal, None) - with self.cached_session(): - self.assertAllEqual([[0., 0., 1., 0.], [0., 0., 1., 0.]], output.eval()) + + self.assertAllEqual([[0., 0., 1., 0.], [0., 0., 1., 0.]], + self.evaluate(output)) def test_multi_hot(self): animal = fc.indicator_column( @@ -5593,8 +5905,8 @@ class IndicatorColumnTest(test.TestCase): indices=[[0, 0], [0, 1]], values=[1, 1], dense_shape=[1, 2]) }) output = transformation_cache.get(animal, None) - with self.cached_session(): - self.assertAllEqual([[0., 2., 0., 0.]], output.eval()) + + self.assertAllEqual([[0., 2., 0., 0.]], self.evaluate(output)) def test_multi_hot2(self): animal = fc.indicator_column( @@ -5605,9 +5917,10 @@ class IndicatorColumnTest(test.TestCase): indices=[[0, 0], [0, 1]], values=[1, 2], dense_shape=[1, 2]) }) output = transformation_cache.get(animal, None) - with self.cached_session(): - self.assertAllEqual([[0., 1., 1., 0.]], output.eval()) + self.assertAllEqual([[0., 1., 1., 0.]], self.evaluate(output)) + + @test_util.run_deprecated_v1 def test_deep_copy(self): a = fc.categorical_column_with_hash_bucket('a', 4) column = fc.indicator_column(a) @@ -5616,44 +5929,52 @@ class IndicatorColumnTest(test.TestCase): self.assertEqual(column.name, 'a_indicator') self.assertEqual(column.variable_shape, [1, 4]) + @test_util.run_deprecated_v1 def test_parse_example(self): a = fc.categorical_column_with_vocabulary_list( key='aaa', vocabulary_list=('omar', 'stringer', 'marlo')) a_indicator = fc.indicator_column(a) - data = example_pb2.Example(features=feature_pb2.Features( - feature={ - 'aaa': - feature_pb2.Feature(bytes_list=feature_pb2.BytesList( - value=[b'omar', b'stringer'])) - })) + data = example_pb2.Example( + features=feature_pb2.Features( + feature={ + 'aaa': + feature_pb2.Feature( + bytes_list=feature_pb2.BytesList( + value=[b'omar', b'stringer'])) + })) features = parsing_ops.parse_example( serialized=[data.SerializeToString()], - features=fc.make_parse_example_spec([a_indicator])) + features=fc.make_parse_example_spec_v2([a_indicator])) self.assertIn('aaa', features) - with self.cached_session(): - _assert_sparse_tensor_value( - self, - sparse_tensor.SparseTensorValue( - indices=[[0, 0], [0, 1]], - values=np.array([b'omar', b'stringer'], dtype=np.object_), - dense_shape=[1, 2]), - features['aaa'].eval()) + _assert_sparse_tensor_value( + self, + sparse_tensor.SparseTensorValue( + indices=[[0, 0], [0, 1]], + values=np.array([b'omar', b'stringer'], dtype=np.object_), + dense_shape=[1, 2]), self.evaluate(features['aaa'])) + + @test_util.run_deprecated_v1 def test_transform(self): a = fc.categorical_column_with_vocabulary_list( key='aaa', vocabulary_list=('omar', 'stringer', 'marlo')) a_indicator = fc.indicator_column(a) features = { - 'aaa': sparse_tensor.SparseTensorValue( - indices=((0, 0), (1, 0), (1, 1)), - values=('marlo', 'skywalker', 'omar'), - dense_shape=(2, 2)) + 'aaa': + sparse_tensor.SparseTensorValue( + indices=((0, 0), (1, 0), (1, 1)), + values=('marlo', 'skywalker', 'omar'), + dense_shape=(2, 2)) } - indicator_tensor = fc._transform_features(features, [a_indicator], - None)[a_indicator] - with _initialized_session(): - self.assertAllEqual([[0, 0, 1], [1, 0, 0]], indicator_tensor.eval()) + indicator_tensor = fc._transform_features_v2(features, [a_indicator], + None)[a_indicator] + self.evaluate(variables_lib.global_variables_initializer()) + self.evaluate(lookup_ops.tables_initializer()) + + self.assertAllEqual([[0, 0, 1], [1, 0, 0]], self.evaluate(indicator_tensor)) + + @test_util.run_deprecated_v1 def test_transform_with_weighted_column(self): # Github issue 12557 ids = fc.categorical_column_with_vocabulary_list( @@ -5661,14 +5982,18 @@ class IndicatorColumnTest(test.TestCase): weights = fc.weighted_categorical_column(ids, 'weights') indicator = fc.indicator_column(weights) features = { - 'ids': constant_op.constant([['c', 'b', 'a']]), - 'weights': constant_op.constant([[2., 4., 6.]]) + 'ids': constant_op.constant([['c', 'b', 'a', 'c']]), + 'weights': constant_op.constant([[2., 4., 6., 1.]]) } - indicator_tensor = fc._transform_features(features, [indicator], - None)[indicator] - with _initialized_session(): - self.assertAllEqual([[6., 4., 2.]], indicator_tensor.eval()) + indicator_tensor = fc._transform_features_v2(features, [indicator], + None)[indicator] + self.evaluate(variables_lib.global_variables_initializer()) + self.evaluate(lookup_ops.tables_initializer()) + + self.assertAllEqual([[6., 4., 3.]], self.evaluate(indicator_tensor)) + + @test_util.run_deprecated_v1 def test_transform_with_missing_value_in_weighted_column(self): # Github issue 12583 ids = fc.categorical_column_with_vocabulary_list( @@ -5679,11 +6004,15 @@ class IndicatorColumnTest(test.TestCase): 'ids': constant_op.constant([['c', 'b', 'unknown']]), 'weights': constant_op.constant([[2., 4., 6.]]) } - indicator_tensor = fc._transform_features(features, [indicator], - None)[indicator] - with _initialized_session(): - self.assertAllEqual([[0., 4., 2.]], indicator_tensor.eval()) + indicator_tensor = fc._transform_features_v2(features, [indicator], + None)[indicator] + self.evaluate(variables_lib.global_variables_initializer()) + self.evaluate(lookup_ops.tables_initializer()) + + self.assertAllEqual([[0., 4., 2.]], self.evaluate(indicator_tensor)) + + @test_util.run_deprecated_v1 def test_transform_with_missing_value_in_categorical_column(self): # Github issue 12583 ids = fc.categorical_column_with_vocabulary_list( @@ -5692,11 +6021,15 @@ class IndicatorColumnTest(test.TestCase): features = { 'ids': constant_op.constant([['c', 'b', 'unknown']]), } - indicator_tensor = fc._transform_features(features, [indicator], - None)[indicator] - with _initialized_session(): - self.assertAllEqual([[0., 1., 1.]], indicator_tensor.eval()) + indicator_tensor = fc._transform_features_v2(features, [indicator], + None)[indicator] + self.evaluate(variables_lib.global_variables_initializer()) + self.evaluate(lookup_ops.tables_initializer()) + + self.assertAllEqual([[0., 1., 1.]], self.evaluate(indicator_tensor)) + + @test_util.run_deprecated_v1 def test_linear_model(self): animal = fc.indicator_column( fc.categorical_column_with_identity('animal', num_buckets=4)) @@ -5710,12 +6043,15 @@ class IndicatorColumnTest(test.TestCase): model = fc.LinearModel([animal]) predictions = model(features) weight_var, _ = model.variables - with _initialized_session(): - # All should be zero-initialized. - self.assertAllClose([[0.], [0.], [0.], [0.]], weight_var.eval()) - self.assertAllClose([[0.]], predictions.eval()) - weight_var.assign([[1.], [2.], [3.], [4.]]).eval() - self.assertAllClose([[2. + 3.]], predictions.eval()) + + self.evaluate(variables_lib.global_variables_initializer()) + self.evaluate(lookup_ops.tables_initializer()) + + # All should be zero-initialized. + self.assertAllClose([[0.], [0.], [0.], [0.]], self.evaluate(weight_var)) + self.assertAllClose([[0.]], self.evaluate(predictions)) + self.evaluate(weight_var.assign([[1.], [2.], [3.], [4.]])) + self.assertAllClose([[2. + 3.]], self.evaluate(predictions)) def test_old_linear_model(self): animal = fc.indicator_column( @@ -5729,16 +6065,19 @@ class IndicatorColumnTest(test.TestCase): predictions = fc_old.linear_model(features, [animal]) weight_var = get_linear_model_column_var(animal) - with _initialized_session(): - # All should be zero-initialized. - self.assertAllClose([[0.], [0.], [0.], [0.]], weight_var.eval()) - self.assertAllClose([[0.]], predictions.eval()) - weight_var.assign([[1.], [2.], [3.], [4.]]).eval() - self.assertAllClose([[2. + 3.]], predictions.eval()) + + self.evaluate(variables_lib.global_variables_initializer()) + self.evaluate(lookup_ops.tables_initializer()) + + # All should be zero-initialized. + self.assertAllClose([[0.], [0.], [0.], [0.]], self.evaluate(weight_var)) + self.assertAllClose([[0.]], self.evaluate(predictions)) + self.evaluate(weight_var.assign([[1.], [2.], [3.], [4.]])) + self.assertAllClose([[2. + 3.]], self.evaluate(predictions)) def test_old_linear_model_old_categorical(self): animal = fc.indicator_column( - fc_old.categorical_column_with_identity('animal', num_buckets=4)) + fc_old._categorical_column_with_identity('animal', num_buckets=4)) with ops.Graph().as_default(): features = { 'animal': @@ -5748,14 +6087,18 @@ class IndicatorColumnTest(test.TestCase): predictions = fc_old.linear_model(features, [animal]) weight_var = get_linear_model_column_var(animal) - with _initialized_session(): - # All should be zero-initialized. - self.assertAllClose([[0.], [0.], [0.], [0.]], weight_var.eval()) - self.assertAllClose([[0.]], predictions.eval()) - weight_var.assign([[1.], [2.], [3.], [4.]]).eval() - self.assertAllClose([[2. + 3.]], predictions.eval()) - def test_feature_layer(self): + self.evaluate(variables_lib.global_variables_initializer()) + self.evaluate(lookup_ops.tables_initializer()) + + # All should be zero-initialized. + self.assertAllClose([[0.], [0.], [0.], [0.]], self.evaluate(weight_var)) + self.assertAllClose([[0.]], self.evaluate(predictions)) + self.evaluate(weight_var.assign([[1.], [2.], [3.], [4.]])) + self.assertAllClose([[2. + 3.]], self.evaluate(predictions)) + + @test_util.run_deprecated_v1 + def test_dense_features(self): animal = fc.indicator_column( fc.categorical_column_with_identity('animal', num_buckets=4)) with ops.Graph().as_default(): @@ -5764,10 +6107,14 @@ class IndicatorColumnTest(test.TestCase): sparse_tensor.SparseTensor( indices=[[0, 0], [0, 1]], values=[1, 2], dense_shape=[1, 2]) } - net = fc.FeatureLayer([animal])(features) - with _initialized_session(): - self.assertAllClose([[0., 1., 1., 0.]], net.eval()) + net = fc.DenseFeatures([animal])(features) + self.evaluate(variables_lib.global_variables_initializer()) + self.evaluate(lookup_ops.tables_initializer()) + + self.assertAllClose([[0., 1., 1., 0.]], self.evaluate(net)) + + @test_util.run_deprecated_v1 def test_input_layer(self): animal = fc.indicator_column( fc.categorical_column_with_identity('animal', num_buckets=4)) @@ -5778,12 +6125,15 @@ class IndicatorColumnTest(test.TestCase): indices=[[0, 0], [0, 1]], values=[1, 2], dense_shape=[1, 2]) } net = fc_old.input_layer(features, [animal]) - with _initialized_session(): - self.assertAllClose([[0., 1., 1., 0.]], net.eval()) + + self.evaluate(variables_lib.global_variables_initializer()) + self.evaluate(lookup_ops.tables_initializer()) + + self.assertAllClose([[0., 1., 1., 0.]], self.evaluate(net)) def test_input_layer_old_categorical(self): animal = fc.indicator_column( - fc_old.categorical_column_with_identity('animal', num_buckets=4)) + fc_old._categorical_column_with_identity('animal', num_buckets=4)) with ops.Graph().as_default(): features = { 'animal': @@ -5791,9 +6141,13 @@ class IndicatorColumnTest(test.TestCase): indices=[[0, 0], [0, 1]], values=[1, 2], dense_shape=[1, 2]) } net = fc_old.input_layer(features, [animal]) - with _initialized_session(): - self.assertAllClose([[0., 1., 1., 0.]], net.eval()) + self.evaluate(variables_lib.global_variables_initializer()) + self.evaluate(lookup_ops.tables_initializer()) + + self.assertAllClose([[0., 1., 1., 0.]], self.evaluate(net)) + + @test_util.run_deprecated_v1 def test_serialization(self): parent = fc.categorical_column_with_identity('animal', num_buckets=4) animal = fc.indicator_column(parent) @@ -5822,7 +6176,6 @@ class IndicatorColumnTest(test.TestCase): self.assertIs(parent, new_animal.categorical_column) - class _TestStateManager(fc.StateManager): def __init__(self, trainable=True): @@ -5864,6 +6217,7 @@ class _TestStateManager(fc.StateManager): class EmbeddingColumnTest(test.TestCase): + @test_util.run_deprecated_v1 def test_defaults(self): categorical_column = fc.categorical_column_with_identity( key='aaa', num_buckets=3) @@ -5885,22 +6239,27 @@ class EmbeddingColumnTest(test.TestCase): self.assertTrue(embedding_column._is_v2_column) def test_is_v2_column(self): - categorical_column = fc_old.categorical_column_with_identity( + categorical_column = fc_old._categorical_column_with_identity( key='aaa', num_buckets=3) embedding_dimension = 2 embedding_column = fc.embedding_column( categorical_column, dimension=embedding_dimension) self.assertFalse(embedding_column._is_v2_column) + @test_util.run_deprecated_v1 def test_all_constructor_args(self): categorical_column = fc.categorical_column_with_identity( key='aaa', num_buckets=3) embedding_dimension = 2 embedding_column = fc.embedding_column( - categorical_column, dimension=embedding_dimension, - combiner='my_combiner', initializer=lambda: 'my_initializer', - ckpt_to_load_from='my_ckpt', tensor_name_in_ckpt='my_ckpt_tensor', - max_norm=42., trainable=False) + categorical_column, + dimension=embedding_dimension, + combiner='my_combiner', + initializer=lambda: 'my_initializer', + ckpt_to_load_from='my_ckpt', + tensor_name_in_ckpt='my_ckpt_tensor', + max_norm=42., + trainable=False) self.assertIs(categorical_column, embedding_column.categorical_column) self.assertEqual(embedding_dimension, embedding_column.dimension) self.assertEqual('my_combiner', embedding_column.combiner) @@ -5914,15 +6273,20 @@ class EmbeddingColumnTest(test.TestCase): 'aaa': parsing_ops.VarLenFeature(dtypes.int64) }, embedding_column.parse_example_spec) + @test_util.run_deprecated_v1 def test_deep_copy(self): categorical_column = fc.categorical_column_with_identity( key='aaa', num_buckets=3) embedding_dimension = 2 original = fc.embedding_column( - categorical_column, dimension=embedding_dimension, - combiner='my_combiner', initializer=lambda: 'my_initializer', - ckpt_to_load_from='my_ckpt', tensor_name_in_ckpt='my_ckpt_tensor', - max_norm=42., trainable=False) + categorical_column, + dimension=embedding_dimension, + combiner='my_combiner', + initializer=lambda: 'my_initializer', + ckpt_to_load_from='my_ckpt', + tensor_name_in_ckpt='my_ckpt_tensor', + max_norm=42., + trainable=False) for embedding_column in (original, copy.deepcopy(original)): self.assertEqual('aaa', embedding_column.categorical_column.name) self.assertEqual(3, embedding_column.categorical_column.num_buckets) @@ -5942,51 +6306,60 @@ class EmbeddingColumnTest(test.TestCase): 'aaa': parsing_ops.VarLenFeature(dtypes.int64) }, embedding_column.parse_example_spec) + @test_util.run_deprecated_v1 def test_invalid_initializer(self): categorical_column = fc.categorical_column_with_identity( key='aaa', num_buckets=3) with self.assertRaisesRegexp(ValueError, 'initializer must be callable'): fc.embedding_column(categorical_column, dimension=2, initializer='not_fn') + @test_util.run_deprecated_v1 def test_parse_example(self): a = fc.categorical_column_with_vocabulary_list( key='aaa', vocabulary_list=('omar', 'stringer', 'marlo')) a_embedded = fc.embedding_column(a, dimension=2) - data = example_pb2.Example(features=feature_pb2.Features( - feature={ - 'aaa': - feature_pb2.Feature(bytes_list=feature_pb2.BytesList( - value=[b'omar', b'stringer'])) - })) + data = example_pb2.Example( + features=feature_pb2.Features( + feature={ + 'aaa': + feature_pb2.Feature( + bytes_list=feature_pb2.BytesList( + value=[b'omar', b'stringer'])) + })) features = parsing_ops.parse_example( serialized=[data.SerializeToString()], - features=fc.make_parse_example_spec([a_embedded])) + features=fc.make_parse_example_spec_v2([a_embedded])) self.assertIn('aaa', features) - with self.cached_session(): - _assert_sparse_tensor_value( - self, - sparse_tensor.SparseTensorValue( - indices=[[0, 0], [0, 1]], - values=np.array([b'omar', b'stringer'], dtype=np.object_), - dense_shape=[1, 2]), - features['aaa'].eval()) + _assert_sparse_tensor_value( + self, + sparse_tensor.SparseTensorValue( + indices=[[0, 0], [0, 1]], + values=np.array([b'omar', b'stringer'], dtype=np.object_), + dense_shape=[1, 2]), self.evaluate(features['aaa'])) + + @test_util.run_deprecated_v1 def test_transform_feature(self): a = fc.categorical_column_with_identity(key='aaa', num_buckets=3) a_embedded = fc.embedding_column(a, dimension=2) features = { - 'aaa': sparse_tensor.SparseTensor( - indices=((0, 0), (1, 0), (1, 1)), - values=(0, 1, 0), - dense_shape=(2, 2)) + 'aaa': + sparse_tensor.SparseTensor( + indices=((0, 0), (1, 0), (1, 1)), + values=(0, 1, 0), + dense_shape=(2, 2)) } - outputs = fc._transform_features(features, [a, a_embedded], None) + outputs = fc._transform_features_v2(features, [a, a_embedded], None) output_a = outputs[a] output_embedded = outputs[a_embedded] - with _initialized_session(): - _assert_sparse_tensor_value( - self, output_a.eval(), output_embedded.eval()) + self.evaluate(variables_lib.global_variables_initializer()) + self.evaluate(lookup_ops.tables_initializer()) + + _assert_sparse_tensor_value(self, self.evaluate(output_a), + self.evaluate(output_embedded)) + + @test_util.run_deprecated_v1 def test_get_dense_tensor(self): # Inputs. vocabulary_size = 3 @@ -6006,6 +6379,7 @@ class EmbeddingColumnTest(test.TestCase): (3., 5.), # id 1 (7., 11.) # id 2 ) + def _initializer(shape, dtype, partition_info): self.assertAllEqual((vocabulary_size, embedding_dimension), shape) self.assertEqual(dtypes.float32, dtype) @@ -6028,7 +6402,8 @@ class EmbeddingColumnTest(test.TestCase): categorical_column = fc.categorical_column_with_identity( key='aaa', num_buckets=vocabulary_size) embedding_column = fc.embedding_column( - categorical_column, dimension=embedding_dimension, + categorical_column, + dimension=embedding_dimension, initializer=_initializer) state_manager = _TestStateManager() embedding_column.create_state(state_manager) @@ -6043,10 +6418,14 @@ class EmbeddingColumnTest(test.TestCase): global_vars = ops.get_collection(ops.GraphKeys.GLOBAL_VARIABLES) self.assertItemsEqual(('embedding_weights:0',), tuple([v.name for v in global_vars])) - with _initialized_session(): - self.assertAllEqual(embedding_values, global_vars[0].eval()) - self.assertAllEqual(expected_lookups, embedding_lookup.eval()) + self.evaluate(variables_lib.global_variables_initializer()) + self.evaluate(lookup_ops.tables_initializer()) + + self.assertAllEqual(embedding_values, self.evaluate(global_vars[0])) + self.assertAllEqual(expected_lookups, self.evaluate(embedding_lookup)) + + @test_util.run_deprecated_v1 def test_get_dense_tensor_old_categorical(self): # Inputs. vocabulary_size = 3 @@ -6086,7 +6465,7 @@ class EmbeddingColumnTest(test.TestCase): ) # Build columns. - categorical_column = fc_old.categorical_column_with_identity( + categorical_column = fc_old._categorical_column_with_identity( key='aaa', num_buckets=vocabulary_size) embedding_column = fc.embedding_column( categorical_column, @@ -6103,10 +6482,14 @@ class EmbeddingColumnTest(test.TestCase): global_vars = ops.get_collection(ops.GraphKeys.GLOBAL_VARIABLES) self.assertItemsEqual(('embedding_weights:0',), tuple([v.name for v in global_vars])) - with _initialized_session(): - self.assertAllEqual(embedding_values, global_vars[0].eval()) - self.assertAllEqual(expected_lookups, embedding_lookup.eval()) + self.evaluate(variables_lib.global_variables_initializer()) + self.evaluate(lookup_ops.tables_initializer()) + + self.assertAllEqual(embedding_values, self.evaluate(global_vars[0])) + self.assertAllEqual(expected_lookups, self.evaluate(embedding_lookup)) + + @test_util.run_deprecated_v1 def test_get_dense_tensor_3d(self): # Inputs. vocabulary_size = 4 @@ -6122,11 +6505,12 @@ class EmbeddingColumnTest(test.TestCase): # Embedding variable. embedding_dimension = 3 embedding_values = ( - (1., 2., 4.), # id 0 - (3., 5., 1.), # id 1 + (1., 2., 4.), # id 0 + (3., 5., 1.), # id 1 (7., 11., 2.), # id 2 - (2., 7., 12.) # id 3 + (2., 7., 12.) # id 3 ) + def _initializer(shape, dtype, partition_info): self.assertAllEqual((vocabulary_size, embedding_dimension), shape) self.assertEqual(dtypes.float32, dtype) @@ -6150,7 +6534,8 @@ class EmbeddingColumnTest(test.TestCase): categorical_column = fc.categorical_column_with_identity( key='aaa', num_buckets=vocabulary_size) embedding_column = fc.embedding_column( - categorical_column, dimension=embedding_dimension, + categorical_column, + dimension=embedding_dimension, initializer=_initializer) state_manager = _TestStateManager() embedding_column.create_state(state_manager) @@ -6165,10 +6550,14 @@ class EmbeddingColumnTest(test.TestCase): global_vars = ops.get_collection(ops.GraphKeys.GLOBAL_VARIABLES) self.assertItemsEqual(('embedding_weights:0',), tuple([v.name for v in global_vars])) - with _initialized_session(): - self.assertAllEqual(embedding_values, global_vars[0].eval()) - self.assertAllEqual(expected_lookups, embedding_lookup.eval()) + self.evaluate(variables_lib.global_variables_initializer()) + self.evaluate(lookup_ops.tables_initializer()) + + self.assertAllEqual(embedding_values, self.evaluate(global_vars[0])) + self.assertAllEqual(expected_lookups, self.evaluate(embedding_lookup)) + + @test_util.run_deprecated_v1 def test_get_dense_tensor_placeholder_inputs(self): # Inputs. vocabulary_size = 3 @@ -6188,6 +6577,7 @@ class EmbeddingColumnTest(test.TestCase): (3., 5.), # id 1 (7., 11.) # id 2 ) + def _initializer(shape, dtype, partition_info): self.assertAllEqual((vocabulary_size, embedding_dimension), shape) self.assertEqual(dtypes.float32, dtype) @@ -6210,7 +6600,8 @@ class EmbeddingColumnTest(test.TestCase): categorical_column = fc.categorical_column_with_identity( key='aaa', num_buckets=vocabulary_size) embedding_column = fc.embedding_column( - categorical_column, dimension=embedding_dimension, + categorical_column, + dimension=embedding_dimension, initializer=_initializer) state_manager = _TestStateManager() embedding_column.create_state(state_manager) @@ -6230,17 +6621,23 @@ class EmbeddingColumnTest(test.TestCase): # Assert expected embedding variable and lookups. global_vars = ops.get_collection(ops.GraphKeys.GLOBAL_VARIABLES) - self.assertItemsEqual( - ('embedding_weights:0',), tuple([v.name for v in global_vars])) - with _initialized_session(): - self.assertAllEqual(embedding_values, global_vars[0].eval()) - self.assertAllEqual(expected_lookups, embedding_lookup.eval( - feed_dict={ - input_indices: sparse_input.indices, - input_values: sparse_input.values, - input_shape: sparse_input.dense_shape, - })) + self.assertItemsEqual(('embedding_weights:0',), + tuple([v.name for v in global_vars])) + self.evaluate(variables_lib.global_variables_initializer()) + self.evaluate(lookup_ops.tables_initializer()) + with _initialized_session(): + self.assertAllEqual(embedding_values, self.evaluate(global_vars[0])) + self.assertAllEqual( + expected_lookups, + embedding_lookup.eval( + feed_dict={ + input_indices: sparse_input.indices, + input_values: sparse_input.values, + input_shape: sparse_input.dense_shape, + })) + + @test_util.run_deprecated_v1 def test_get_dense_tensor_restore_from_ckpt(self): # Inputs. vocabulary_size = 3 @@ -6280,7 +6677,8 @@ class EmbeddingColumnTest(test.TestCase): categorical_column = fc.categorical_column_with_identity( key='aaa', num_buckets=vocabulary_size) embedding_column = fc.embedding_column( - categorical_column, dimension=embedding_dimension, + categorical_column, + dimension=embedding_dimension, ckpt_to_load_from=ckpt_path, tensor_name_in_ckpt=ckpt_tensor) state_manager = _TestStateManager() @@ -6294,12 +6692,16 @@ class EmbeddingColumnTest(test.TestCase): # Assert expected embedding variable and lookups. global_vars = ops.get_collection(ops.GraphKeys.GLOBAL_VARIABLES) - self.assertItemsEqual( - ('embedding_weights:0',), tuple([v.name for v in global_vars])) - with _initialized_session(): - self.assertAllEqual(embedding_values, global_vars[0].eval()) - self.assertAllEqual(expected_lookups, embedding_lookup.eval()) + self.assertItemsEqual(('embedding_weights:0',), + tuple([v.name for v in global_vars])) + self.evaluate(variables_lib.global_variables_initializer()) + self.evaluate(lookup_ops.tables_initializer()) + + self.assertAllEqual(embedding_values, self.evaluate(global_vars[0])) + self.assertAllEqual(expected_lookups, self.evaluate(embedding_lookup)) + + @test_util.run_deprecated_v1 def test_linear_model(self): # Inputs. batch_size = 4 @@ -6317,6 +6719,7 @@ class EmbeddingColumnTest(test.TestCase): embedding_dimension = 2 embedding_shape = (vocabulary_size, embedding_dimension) zeros_embedding_values = np.zeros(embedding_shape) + def _initializer(shape, dtype, partition_info): self.assertAllEqual(embedding_shape, shape) self.assertEqual(dtypes.float32, dtype) @@ -6343,39 +6746,45 @@ class EmbeddingColumnTest(test.TestCase): expected_var_names, [v.name for v in ops.get_collection(ops.GraphKeys.GLOBAL_VARIABLES)]) trainable_vars = { - v.name: v for v in ops.get_collection( - ops.GraphKeys.TRAINABLE_VARIABLES) + v.name: v + for v in ops.get_collection(ops.GraphKeys.TRAINABLE_VARIABLES) } self.assertItemsEqual(expected_var_names, trainable_vars.keys()) bias = trainable_vars['linear_model/bias_weights:0'] embedding_weights = trainable_vars[ 'linear_model/aaa_embedding/embedding_weights:0'] - linear_weights = trainable_vars[ - 'linear_model/aaa_embedding/weights:0'] - with _initialized_session(): - # Predictions with all zero weights. - self.assertAllClose(np.zeros((1,)), bias.eval()) - self.assertAllClose(zeros_embedding_values, embedding_weights.eval()) - self.assertAllClose( - np.zeros((embedding_dimension, 1)), linear_weights.eval()) - self.assertAllClose(np.zeros((batch_size, 1)), predictions.eval()) + linear_weights = trainable_vars['linear_model/aaa_embedding/weights:0'] - # Predictions with all non-zero weights. - embedding_weights.assign(( - (1., 2.), # id 0 - (3., 5.), # id 1 - (7., 11.) # id 2 - )).eval() - linear_weights.assign(((4.,), (6.,))).eval() - # example 0, ids [2], embedding[0] = [7, 11] - # example 1, ids [0, 1], embedding[1] = mean([1, 2] + [3, 5]) = [2, 3.5] - # example 2, ids [], embedding[2] = [0, 0] - # example 3, ids [1], embedding[3] = [3, 5] - # sum(embeddings * linear_weights) - # = [4*7 + 6*11, 4*2 + 6*3.5, 4*0 + 6*0, 4*3 + 6*5] = [94, 29, 0, 42] - self.assertAllClose(((94.,), (29.,), (0.,), (42.,)), predictions.eval()) + self.evaluate(variables_lib.global_variables_initializer()) + self.evaluate(lookup_ops.tables_initializer()) - def test_feature_layer(self): + # Predictions with all zero weights. + self.assertAllClose(np.zeros((1,)), self.evaluate(bias)) + self.assertAllClose(zeros_embedding_values, + self.evaluate(embedding_weights)) + self.assertAllClose( + np.zeros((embedding_dimension, 1)), self.evaluate(linear_weights)) + self.assertAllClose(np.zeros((batch_size, 1)), self.evaluate(predictions)) + + # Predictions with all non-zero weights. + self.evaluate( + embedding_weights.assign(( + (1., 2.), # id 0 + (3., 5.), # id 1 + (7., 11.) # id 2 + ))) + self.evaluate(linear_weights.assign(((4.,), (6.,)))) + # example 0, ids [2], embedding[0] = [7, 11] + # example 1, ids [0, 1], embedding[1] = mean([1, 2] + [3, 5]) = [2, 3.5] + # example 2, ids [], embedding[2] = [0, 0] + # example 3, ids [1], embedding[3] = [3, 5] + # sum(embeddings * linear_weights) + # = [4*7 + 6*11, 4*2 + 6*3.5, 4*0 + 6*0, 4*3 + 6*5] = [94, 29, 0, 42] + self.assertAllClose(((94.,), (29.,), (0.,), (42.,)), + self.evaluate(predictions)) + + @test_util.run_deprecated_v1 + def test_dense_features(self): # Inputs. vocabulary_size = 3 sparse_input = sparse_tensor.SparseTensorValue( @@ -6394,6 +6803,7 @@ class EmbeddingColumnTest(test.TestCase): (3., 5.), # id 1 (7., 11.) # id 2 ) + def _initializer(shape, dtype, partition_info): self.assertAllEqual((vocabulary_size, embedding_dimension), shape) self.assertEqual(dtypes.float32, dtype) @@ -6421,23 +6831,27 @@ class EmbeddingColumnTest(test.TestCase): initializer=_initializer) # Provide sparse input and get dense result. - l = fc.FeatureLayer((embedding_column,)) - feature_layer = l({'aaa': sparse_input}) + l = fc.DenseFeatures((embedding_column,)) + dense_features = l({'aaa': sparse_input}) # Assert expected embedding variable and lookups. global_vars = ops.get_collection(ops.GraphKeys.GLOBAL_VARIABLES) - self.assertItemsEqual(('feature_layer/aaa_embedding/embedding_weights:0',), + self.assertItemsEqual(('dense_features/aaa_embedding/embedding_weights:0',), tuple([v.name for v in global_vars])) for v in global_vars: self.assertTrue(isinstance(v, variables_lib.RefVariable)) trainable_vars = ops.get_collection(ops.GraphKeys.TRAINABLE_VARIABLES) - self.assertItemsEqual(('feature_layer/aaa_embedding/embedding_weights:0',), + self.assertItemsEqual(('dense_features/aaa_embedding/embedding_weights:0',), tuple([v.name for v in trainable_vars])) - with _initialized_session(): - self.assertAllEqual(embedding_values, trainable_vars[0].eval()) - self.assertAllEqual(expected_lookups, feature_layer.eval()) - def test_feature_layer_not_trainable(self): + self.evaluate(variables_lib.global_variables_initializer()) + self.evaluate(lookup_ops.tables_initializer()) + + self.assertAllEqual(embedding_values, self.evaluate(trainable_vars[0])) + self.assertAllEqual(expected_lookups, self.evaluate(dense_features)) + + @test_util.run_deprecated_v1 + def test_dense_features_not_trainable(self): # Inputs. vocabulary_size = 3 sparse_input = sparse_tensor.SparseTensorValue( @@ -6456,6 +6870,7 @@ class EmbeddingColumnTest(test.TestCase): (3., 5.), # id 1 (7., 11.) # id 2 ) + def _initializer(shape, dtype, partition_info): self.assertAllEqual((vocabulary_size, embedding_dimension), shape) self.assertEqual(dtypes.float32, dtype) @@ -6484,18 +6899,24 @@ class EmbeddingColumnTest(test.TestCase): trainable=False) # Provide sparse input and get dense result. - feature_layer = fc.FeatureLayer((embedding_column,))({'aaa': sparse_input}) + dense_features = fc.DenseFeatures((embedding_column,))({ + 'aaa': sparse_input + }) # Assert expected embedding variable and lookups. global_vars = ops.get_collection(ops.GraphKeys.GLOBAL_VARIABLES) - self.assertItemsEqual(('feature_layer/aaa_embedding/embedding_weights:0',), + self.assertItemsEqual(('dense_features/aaa_embedding/embedding_weights:0',), tuple([v.name for v in global_vars])) - self.assertItemsEqual( - [], ops.get_collection(ops.GraphKeys.TRAINABLE_VARIABLES)) - with _initialized_session(): - self.assertAllEqual(embedding_values, global_vars[0].eval()) - self.assertAllEqual(expected_lookups, feature_layer.eval()) + self.assertItemsEqual([], + ops.get_collection(ops.GraphKeys.TRAINABLE_VARIABLES)) + self.evaluate(variables_lib.global_variables_initializer()) + self.evaluate(lookup_ops.tables_initializer()) + + self.assertAllEqual(embedding_values, self.evaluate(global_vars[0])) + self.assertAllEqual(expected_lookups, self.evaluate(dense_features)) + + @test_util.run_deprecated_v1 def test_input_layer(self): # Inputs. vocabulary_size = 3 @@ -6554,9 +6975,12 @@ class EmbeddingColumnTest(test.TestCase): trainable_vars = ops.get_collection(ops.GraphKeys.TRAINABLE_VARIABLES) self.assertItemsEqual(('input_layer/aaa_embedding/embedding_weights:0',), tuple([v.name for v in trainable_vars])) - with _initialized_session(): - self.assertAllEqual(embedding_values, trainable_vars[0].eval()) - self.assertAllEqual(expected_lookups, feature_layer.eval()) + + self.evaluate(variables_lib.global_variables_initializer()) + self.evaluate(lookup_ops.tables_initializer()) + + self.assertAllEqual(embedding_values, self.evaluate(trainable_vars[0])) + self.assertAllEqual(expected_lookups, self.evaluate(feature_layer)) def test_old_linear_model(self): # Inputs. @@ -6611,28 +7035,34 @@ class EmbeddingColumnTest(test.TestCase): embedding_weights = trainable_vars[ 'linear_model/aaa_embedding/embedding_weights:0'] linear_weights = trainable_vars['linear_model/aaa_embedding/weights:0'] - with _initialized_session(): - # Predictions with all zero weights. - self.assertAllClose(np.zeros((1,)), bias.eval()) - self.assertAllClose(zeros_embedding_values, embedding_weights.eval()) - self.assertAllClose( - np.zeros((embedding_dimension, 1)), linear_weights.eval()) - self.assertAllClose(np.zeros((batch_size, 1)), predictions.eval()) - # Predictions with all non-zero weights. - embedding_weights.assign(( - (1., 2.), # id 0 - (3., 5.), # id 1 - (7., 11.) # id 2 - )).eval() - linear_weights.assign(((4.,), (6.,))).eval() - # example 0, ids [2], embedding[0] = [7, 11] - # example 1, ids [0, 1], embedding[1] = mean([1, 2] + [3, 5]) = [2, 3.5] - # example 2, ids [], embedding[2] = [0, 0] - # example 3, ids [1], embedding[3] = [3, 5] - # sum(embeddings * linear_weights) - # = [4*7 + 6*11, 4*2 + 6*3.5, 4*0 + 6*0, 4*3 + 6*5] = [94, 29, 0, 42] - self.assertAllClose(((94.,), (29.,), (0.,), (42.,)), predictions.eval()) + self.evaluate(variables_lib.global_variables_initializer()) + self.evaluate(lookup_ops.tables_initializer()) + + # Predictions with all zero weights. + self.assertAllClose(np.zeros((1,)), self.evaluate(bias)) + self.assertAllClose(zeros_embedding_values, + self.evaluate(embedding_weights)) + self.assertAllClose( + np.zeros((embedding_dimension, 1)), self.evaluate(linear_weights)) + self.assertAllClose(np.zeros((batch_size, 1)), self.evaluate(predictions)) + + # Predictions with all non-zero weights. + self.evaluate( + embedding_weights.assign(( + (1., 2.), # id 0 + (3., 5.), # id 1 + (7., 11.) # id 2 + ))) + self.evaluate(linear_weights.assign(((4.,), (6.,)))) + # example 0, ids [2], embedding[0] = [7, 11] + # example 1, ids [0, 1], embedding[1] = mean([1, 2] + [3, 5]) = [2, 3.5] + # example 2, ids [], embedding[2] = [0, 0] + # example 3, ids [1], embedding[3] = [3, 5] + # sum(embeddings * linear_weights) + # = [4*7 + 6*11, 4*2 + 6*3.5, 4*0 + 6*0, 4*3 + 6*5] = [94, 29, 0, 42] + self.assertAllClose(((94.,), (29.,), (0.,), (42.,)), + self.evaluate(predictions)) def test_old_linear_model_old_categorical(self): # Inputs. @@ -6659,7 +7089,7 @@ class EmbeddingColumnTest(test.TestCase): return zeros_embedding_values # Build columns. - categorical_column = fc_old.categorical_column_with_identity( + categorical_column = fc_old._categorical_column_with_identity( key='aaa', num_buckets=vocabulary_size) embedding_column = fc.embedding_column( categorical_column, @@ -6687,29 +7117,36 @@ class EmbeddingColumnTest(test.TestCase): embedding_weights = trainable_vars[ 'linear_model/aaa_embedding/embedding_weights:0'] linear_weights = trainable_vars['linear_model/aaa_embedding/weights:0'] - with _initialized_session(): - # Predictions with all zero weights. - self.assertAllClose(np.zeros((1,)), bias.eval()) - self.assertAllClose(zeros_embedding_values, embedding_weights.eval()) - self.assertAllClose( - np.zeros((embedding_dimension, 1)), linear_weights.eval()) - self.assertAllClose(np.zeros((batch_size, 1)), predictions.eval()) - # Predictions with all non-zero weights. - embedding_weights.assign(( - (1., 2.), # id 0 - (3., 5.), # id 1 - (7., 11.) # id 2 - )).eval() - linear_weights.assign(((4.,), (6.,))).eval() - # example 0, ids [2], embedding[0] = [7, 11] - # example 1, ids [0, 1], embedding[1] = mean([1, 2] + [3, 5]) = [2, 3.5] - # example 2, ids [], embedding[2] = [0, 0] - # example 3, ids [1], embedding[3] = [3, 5] - # sum(embeddings * linear_weights) - # = [4*7 + 6*11, 4*2 + 6*3.5, 4*0 + 6*0, 4*3 + 6*5] = [94, 29, 0, 42] - self.assertAllClose(((94.,), (29.,), (0.,), (42.,)), predictions.eval()) + self.evaluate(variables_lib.global_variables_initializer()) + self.evaluate(lookup_ops.tables_initializer()) + # Predictions with all zero weights. + self.assertAllClose(np.zeros((1,)), self.evaluate(bias)) + self.assertAllClose(zeros_embedding_values, + self.evaluate(embedding_weights)) + self.assertAllClose( + np.zeros((embedding_dimension, 1)), self.evaluate(linear_weights)) + self.assertAllClose(np.zeros((batch_size, 1)), self.evaluate(predictions)) + + # Predictions with all non-zero weights. + self.evaluate( + embedding_weights.assign(( + (1., 2.), # id 0 + (3., 5.), # id 1 + (7., 11.) # id 2 + ))) + self.evaluate(linear_weights.assign(((4.,), (6.,)))) + # example 0, ids [2], embedding[0] = [7, 11] + # example 1, ids [0, 1], embedding[1] = mean([1, 2] + [3, 5]) = [2, 3.5] + # example 2, ids [], embedding[2] = [0, 0] + # example 3, ids [1], embedding[3] = [3, 5] + # sum(embeddings * linear_weights) + # = [4*7 + 6*11, 4*2 + 6*3.5, 4*0 + 6*0, 4*3 + 6*5] = [94, 29, 0, 42] + self.assertAllClose(((94.,), (29.,), (0.,), (42.,)), + self.evaluate(predictions)) + + @test_util.run_deprecated_v1 def test_serialization(self): def _initializer(shape, dtype, partition_info): @@ -6763,6 +7200,7 @@ class EmbeddingColumnTest(test.TestCase): class SharedEmbeddingColumnTest(test.TestCase): + @test_util.run_deprecated_v1 def test_defaults(self): categorical_column_a = fc.categorical_column_with_identity( key='aaa', num_buckets=3) @@ -6787,6 +7225,7 @@ class SharedEmbeddingColumnTest(test.TestCase): 'bbb': parsing_ops.VarLenFeature(dtypes.int64) }, embedding_column_b.parse_example_spec) + @test_util.run_deprecated_v1 def test_all_constructor_args(self): categorical_column_a = fc.categorical_column_with_identity( key='aaa', num_buckets=3) @@ -6818,6 +7257,7 @@ class SharedEmbeddingColumnTest(test.TestCase): 'bbb': parsing_ops.VarLenFeature(dtypes.int64) }, embedding_column_b.parse_example_spec) + @test_util.run_deprecated_v1 def test_deep_copy(self): categorical_column_a = fc.categorical_column_with_identity( key='aaa', num_buckets=3) @@ -6849,6 +7289,7 @@ class SharedEmbeddingColumnTest(test.TestCase): 'aaa': parsing_ops.VarLenFeature(dtypes.int64) }, embedding_column_a.parse_example_spec) + @test_util.run_deprecated_v1 def test_invalid_initializer(self): categorical_column_a = fc.categorical_column_with_identity( key='aaa', num_buckets=3) @@ -6860,6 +7301,7 @@ class SharedEmbeddingColumnTest(test.TestCase): dimension=2, initializer='not_fn') + @test_util.run_deprecated_v1 def test_incompatible_column_type(self): categorical_column_a = fc.categorical_column_with_identity( key='aaa', num_buckets=3) @@ -6874,6 +7316,7 @@ class SharedEmbeddingColumnTest(test.TestCase): [categorical_column_a, categorical_column_b, categorical_column_c], dimension=2) + @test_util.run_deprecated_v1 def test_weighted_categorical_column_ok(self): categorical_column_a = fc.categorical_column_with_identity( key='aaa', num_buckets=3) @@ -6891,82 +7334,90 @@ class SharedEmbeddingColumnTest(test.TestCase): [weighted_categorical_column_a, weighted_categorical_column_b], dimension=2) + @test_util.run_deprecated_v1 def test_parse_example(self): a = fc.categorical_column_with_vocabulary_list( key='aaa', vocabulary_list=('omar', 'stringer', 'marlo')) b = fc.categorical_column_with_vocabulary_list( key='bbb', vocabulary_list=('omar', 'stringer', 'marlo')) a_embedded, b_embedded = fc.shared_embedding_columns_v2([a, b], dimension=2) - data = example_pb2.Example(features=feature_pb2.Features( - feature={ - 'aaa': - feature_pb2.Feature(bytes_list=feature_pb2.BytesList( - value=[b'omar', b'stringer'])), - 'bbb': - feature_pb2.Feature(bytes_list=feature_pb2.BytesList( - value=[b'stringer', b'marlo'])), - })) + data = example_pb2.Example( + features=feature_pb2.Features( + feature={ + 'aaa': + feature_pb2.Feature( + bytes_list=feature_pb2.BytesList( + value=[b'omar', b'stringer'])), + 'bbb': + feature_pb2.Feature( + bytes_list=feature_pb2.BytesList( + value=[b'stringer', b'marlo'])), + })) features = parsing_ops.parse_example( serialized=[data.SerializeToString()], - features=fc.make_parse_example_spec([a_embedded, b_embedded])) + features=fc.make_parse_example_spec_v2([a_embedded, b_embedded])) self.assertIn('aaa', features) self.assertIn('bbb', features) - with self.cached_session(): - _assert_sparse_tensor_value( - self, - sparse_tensor.SparseTensorValue( - indices=[[0, 0], [0, 1]], - values=np.array([b'omar', b'stringer'], dtype=np.object_), - dense_shape=[1, 2]), - features['aaa'].eval()) - _assert_sparse_tensor_value( - self, - sparse_tensor.SparseTensorValue( - indices=[[0, 0], [0, 1]], - values=np.array([b'stringer', b'marlo'], dtype=np.object_), - dense_shape=[1, 2]), - features['bbb'].eval()) + _assert_sparse_tensor_value( + self, + sparse_tensor.SparseTensorValue( + indices=[[0, 0], [0, 1]], + values=np.array([b'omar', b'stringer'], dtype=np.object_), + dense_shape=[1, 2]), self.evaluate(features['aaa'])) + _assert_sparse_tensor_value( + self, + sparse_tensor.SparseTensorValue( + indices=[[0, 0], [0, 1]], + values=np.array([b'stringer', b'marlo'], dtype=np.object_), + dense_shape=[1, 2]), self.evaluate(features['bbb'])) + + @test_util.run_deprecated_v1 def test_transform_feature(self): a = fc.categorical_column_with_identity(key='aaa', num_buckets=3) b = fc.categorical_column_with_identity(key='bbb', num_buckets=3) a_embedded, b_embedded = fc.shared_embedding_columns_v2([a, b], dimension=2) features = { - 'aaa': sparse_tensor.SparseTensor( - indices=((0, 0), (1, 0), (1, 1)), - values=(0, 1, 0), - dense_shape=(2, 2)), - 'bbb': sparse_tensor.SparseTensor( - indices=((0, 0), (1, 0), (1, 1)), - values=(1, 2, 1), - dense_shape=(2, 2)), + 'aaa': + sparse_tensor.SparseTensor( + indices=((0, 0), (1, 0), (1, 1)), + values=(0, 1, 0), + dense_shape=(2, 2)), + 'bbb': + sparse_tensor.SparseTensor( + indices=((0, 0), (1, 0), (1, 1)), + values=(1, 2, 1), + dense_shape=(2, 2)), } - outputs = fc._transform_features(features, [a, a_embedded, b, b_embedded], - None) + outputs = fc._transform_features_v2(features, + [a, a_embedded, b, b_embedded], None) output_a = outputs[a] output_a_embedded = outputs[a_embedded] output_b = outputs[b] output_b_embedded = outputs[b_embedded] - with _initialized_session(): - _assert_sparse_tensor_value( - self, output_a.eval(), output_a_embedded.eval()) - _assert_sparse_tensor_value( - self, output_b.eval(), output_b_embedded.eval()) + self.evaluate(variables_lib.global_variables_initializer()) + self.evaluate(lookup_ops.tables_initializer()) + + _assert_sparse_tensor_value(self, self.evaluate(output_a), + self.evaluate(output_a_embedded)) + _assert_sparse_tensor_value(self, self.evaluate(output_b), + self.evaluate(output_b_embedded)) + + @test_util.run_deprecated_v1 def test_get_dense_tensor(self): # Inputs. vocabulary_size = 3 # -1 values are ignored. - input_a = np.array( - [[2, -1, -1], # example 0, ids [2] - [0, 1, -1]]) # example 1, ids [0, 1] - input_b = np.array( - [[0, -1, -1], # example 0, ids [0] - [-1, -1, -1]]) # example 1, ids [] - input_features = { - 'aaa': input_a, - 'bbb': input_b - } + input_a = np.array([ + [2, -1, -1], # example 0, ids [2] + [0, 1, -1] + ]) # example 1, ids [0, 1] + input_b = np.array([ + [0, -1, -1], # example 0, ids [0] + [-1, -1, -1] + ]) # example 1, ids [] + input_features = {'aaa': input_a, 'bbb': input_b} # Embedding variable. embedding_dimension = 2 @@ -6975,6 +7426,7 @@ class SharedEmbeddingColumnTest(test.TestCase): (3., 5.), # id 1 (7., 11.) # id 2 ) + def _initializer(shape, dtype, partition_info): self.assertAllEqual((vocabulary_size, embedding_dimension), shape) self.assertEqual(dtypes.float32, dtype) @@ -7016,21 +7468,27 @@ class SharedEmbeddingColumnTest(test.TestCase): self.assertItemsEqual(('aaa_bbb_shared_embedding:0',), tuple([v.name for v in global_vars])) embedding_var = global_vars[0] - with _initialized_session(): - self.assertAllEqual(embedding_values, embedding_var.eval()) - self.assertAllEqual(expected_lookups_a, embedding_lookup_a.eval()) - self.assertAllEqual(expected_lookups_b, embedding_lookup_b.eval()) + self.evaluate(variables_lib.global_variables_initializer()) + self.evaluate(lookup_ops.tables_initializer()) + + self.assertAllEqual(embedding_values, self.evaluate(embedding_var)) + self.assertAllEqual(expected_lookups_a, self.evaluate(embedding_lookup_a)) + self.assertAllEqual(expected_lookups_b, self.evaluate(embedding_lookup_b)) + + @test_util.run_deprecated_v1 def test_get_dense_tensor_placeholder_inputs(self): # Inputs. vocabulary_size = 3 # -1 values are ignored. - input_a = np.array( - [[2, -1, -1], # example 0, ids [2] - [0, 1, -1]]) # example 1, ids [0, 1] - input_b = np.array( - [[0, -1, -1], # example 0, ids [0] - [-1, -1, -1]]) # example 1, ids [] + input_a = np.array([ + [2, -1, -1], # example 0, ids [2] + [0, 1, -1] + ]) # example 1, ids [0, 1] + input_b = np.array([ + [0, -1, -1], # example 0, ids [0] + [-1, -1, -1] + ]) # example 1, ids [] # Specify shape, because dense input must have rank specified. input_a_placeholder = array_ops.placeholder( dtype=dtypes.int64, shape=[None, 3]) @@ -7052,6 +7510,7 @@ class SharedEmbeddingColumnTest(test.TestCase): (3., 5.), # id 1 (7., 11.) # id 2 ) + def _initializer(shape, dtype, partition_info): self.assertAllEqual((vocabulary_size, embedding_dimension), shape) self.assertEqual(dtypes.float32, dtype) @@ -7077,22 +7536,26 @@ class SharedEmbeddingColumnTest(test.TestCase): with _initialized_session() as sess: sess.run([embedding_lookup_a, embedding_lookup_b], feed_dict=feed_dict) + @test_util.run_deprecated_v1 def test_linear_model(self): # Inputs. batch_size = 2 vocabulary_size = 3 # -1 values are ignored. - input_a = np.array( - [[2, -1, -1], # example 0, ids [2] - [0, 1, -1]]) # example 1, ids [0, 1] - input_b = np.array( - [[0, -1, -1], # example 0, ids [0] - [-1, -1, -1]]) # example 1, ids [] + input_a = np.array([ + [2, -1, -1], # example 0, ids [2] + [0, 1, -1] + ]) # example 1, ids [0, 1] + input_b = np.array([ + [0, -1, -1], # example 0, ids [0] + [-1, -1, -1] + ]) # example 1, ids [] # Embedding variable. embedding_dimension = 2 embedding_shape = (vocabulary_size, embedding_dimension) zeros_embedding_values = np.zeros(embedding_shape) + def _initializer(shape, dtype, partition_info): self.assertAllEqual(embedding_shape, shape) self.assertEqual(dtypes.float32, dtype) @@ -7128,8 +7591,8 @@ class SharedEmbeddingColumnTest(test.TestCase): expected_var_names, [v.name for v in ops.get_collection(ops.GraphKeys.GLOBAL_VARIABLES)]) trainable_vars = { - v.name: v for v in ops.get_collection( - ops.GraphKeys.TRAINABLE_VARIABLES) + v.name: v + for v in ops.get_collection(ops.GraphKeys.TRAINABLE_VARIABLES) } self.assertItemsEqual(expected_var_names, trainable_vars.keys()) bias = trainable_vars['linear_model/bias_weights:0'] @@ -7138,35 +7601,40 @@ class SharedEmbeddingColumnTest(test.TestCase): 'linear_model/aaa_shared_embedding/weights:0'] linear_weights_b = trainable_vars[ 'linear_model/bbb_shared_embedding/weights:0'] - with _initialized_session(): - # Predictions with all zero weights. - self.assertAllClose(np.zeros((1,)), bias.eval()) - self.assertAllClose(zeros_embedding_values, embedding_weights.eval()) - self.assertAllClose( - np.zeros((embedding_dimension, 1)), linear_weights_a.eval()) - self.assertAllClose( - np.zeros((embedding_dimension, 1)), linear_weights_b.eval()) - self.assertAllClose(np.zeros((batch_size, 1)), predictions.eval()) - # Predictions with all non-zero weights. - embedding_weights.assign(( - (1., 2.), # id 0 - (3., 5.), # id 1 - (7., 11.) # id 2 - )).eval() - linear_weights_a.assign(((4.,), (6.,))).eval() - # example 0, ids [2], embedding[0] = [7, 11] - # example 1, ids [0, 1], embedding[1] = mean([1, 2] + [3, 5]) = [2, 3.5] - # sum(embeddings * linear_weights) - # = [4*7 + 6*11, 4*2 + 6*3.5] = [94, 29] - linear_weights_b.assign(((3.,), (5.,))).eval() - # example 0, ids [0], embedding[0] = [1, 2] - # example 1, ids [], embedding[1] = 0, 0] - # sum(embeddings * linear_weights) - # = [3*1 + 5*2, 3*0 +5*0] = [13, 0] - self.assertAllClose([[94. + 13.], [29.]], predictions.eval()) + self.evaluate(variables_lib.global_variables_initializer()) + self.evaluate(lookup_ops.tables_initializer()) - def _test_feature_layer(self, trainable=True): + # Predictions with all zero weights. + self.assertAllClose(np.zeros((1,)), self.evaluate(bias)) + self.assertAllClose(zeros_embedding_values, + self.evaluate(embedding_weights)) + self.assertAllClose( + np.zeros((embedding_dimension, 1)), self.evaluate(linear_weights_a)) + self.assertAllClose( + np.zeros((embedding_dimension, 1)), self.evaluate(linear_weights_b)) + self.assertAllClose(np.zeros((batch_size, 1)), self.evaluate(predictions)) + + # Predictions with all non-zero weights. + self.evaluate( + embedding_weights.assign(( + (1., 2.), # id 0 + (3., 5.), # id 1 + (7., 11.) # id 2 + ))) + self.evaluate(linear_weights_a.assign(((4.,), (6.,)))) + # example 0, ids [2], embedding[0] = [7, 11] + # example 1, ids [0, 1], embedding[1] = mean([1, 2] + [3, 5]) = [2, 3.5] + # sum(embeddings * linear_weights) + # = [4*7 + 6*11, 4*2 + 6*3.5] = [94, 29] + self.evaluate(linear_weights_b.assign(((3.,), (5.,)))) + # example 0, ids [0], embedding[0] = [1, 2] + # example 1, ids [], embedding[1] = 0, 0] + # sum(embeddings * linear_weights) + # = [3*1 + 5*2, 3*0 +5*0] = [13, 0] + self.assertAllClose([[94. + 13.], [29.]], self.evaluate(predictions)) + + def _test_dense_features(self, trainable=True): # Inputs. vocabulary_size = 3 sparse_input_a = sparse_tensor.SparseTensorValue( @@ -7201,6 +7669,7 @@ class SharedEmbeddingColumnTest(test.TestCase): (3., 5.), # id 1 (7., 11.) # id 2 ) + def _initializer(shape, dtype, partition_info): self.assertAllEqual((vocabulary_size, embedding_dimension), shape) self.assertEqual(dtypes.float32, dtype) @@ -7252,7 +7721,7 @@ class SharedEmbeddingColumnTest(test.TestCase): } # Provide sparse input and get dense result. - feature_layer = fc.FeatureLayer( + dense_features = fc.DenseFeatures( feature_columns=(embedding_column_b, embedding_column_a, embedding_column_c, embedding_column_d))( features) @@ -7272,16 +7741,23 @@ class SharedEmbeddingColumnTest(test.TestCase): else: self.assertItemsEqual([], tuple([v.name for v in trainable_vars])) shared_embedding_vars = global_vars - with _initialized_session(): - self.assertAllEqual(embedding_values, shared_embedding_vars[0].eval()) - self.assertAllEqual(expected_lookups, feature_layer.eval()) - def test_feature_layer(self): - self._test_feature_layer() + self.evaluate(variables_lib.global_variables_initializer()) + self.evaluate(lookup_ops.tables_initializer()) - def test_feature_layer_no_trainable(self): - self._test_feature_layer(trainable=False) + self.assertAllEqual(embedding_values, + self.evaluate(shared_embedding_vars[0])) + self.assertAllEqual(expected_lookups, self.evaluate(dense_features)) + @test_util.run_deprecated_v1 + def test_dense_features(self): + self._test_dense_features() + + @test_util.run_deprecated_v1 + def test_dense_features_no_trainable(self): + self._test_dense_features(trainable=False) + + @test_util.run_deprecated_v1 def test_serialization(self): def _initializer(shape, dtype, partition_info): @@ -7302,9 +7778,9 @@ class SharedEmbeddingColumnTest(test.TestCase): # TODO(rohanj): Add tests for (from|get)_config once implemented - class WeightedCategoricalColumnTest(test.TestCase): + @test_util.run_deprecated_v1 def test_defaults(self): column = fc.weighted_categorical_column( categorical_column=fc.categorical_column_with_identity( @@ -7320,11 +7796,12 @@ class WeightedCategoricalColumnTest(test.TestCase): def test_is_v2_column(self): column = fc.weighted_categorical_column( - categorical_column=fc_old.categorical_column_with_identity( + categorical_column=fc_old._categorical_column_with_identity( key='ids', num_buckets=3), weight_feature_key='values') self.assertFalse(column._is_v2_column) + @test_util.run_deprecated_v1 def test_deep_copy(self): """Tests deepcopy of categorical_column_with_hash_bucket.""" original = fc.weighted_categorical_column( @@ -7365,7 +7842,7 @@ class WeightedCategoricalColumnTest(test.TestCase): values=('omar', 'stringer', 'marlo'), dense_shape=(2, 2)) with self.assertRaisesRegexp(ValueError, 'Bad dtype'): - fc._transform_features({ + fc._transform_features_v2({ 'ids': strings, 'values': strings }, (column,), None) @@ -7386,77 +7863,79 @@ class WeightedCategoricalColumnTest(test.TestCase): indices=((0, 0), (1, 0), (1, 1)), values=('omar', 'stringer', 'marlo'), dense_shape=(2, 2)) - with self.assertRaisesRegexp( - ValueError, 'values is not in features dictionary'): - fc._transform_features({'ids': inputs}, (column,), None) + with self.assertRaisesRegexp(ValueError, + 'values is not in features dictionary'): + fc._transform_features_v2({'ids': inputs}, (column,), None) + @test_util.run_deprecated_v1 def test_parse_example(self): a = fc.categorical_column_with_vocabulary_list( key='aaa', vocabulary_list=('omar', 'stringer', 'marlo')) a_weighted = fc.weighted_categorical_column(a, weight_feature_key='weights') - data = example_pb2.Example(features=feature_pb2.Features( - feature={ - 'aaa': - feature_pb2.Feature(bytes_list=feature_pb2.BytesList( - value=[b'omar', b'stringer'])), - 'weights': - feature_pb2.Feature(float_list=feature_pb2.FloatList( - value=[1., 10.])) - })) + data = example_pb2.Example( + features=feature_pb2.Features( + feature={ + 'aaa': + feature_pb2.Feature( + bytes_list=feature_pb2.BytesList( + value=[b'omar', b'stringer'])), + 'weights': + feature_pb2.Feature( + float_list=feature_pb2.FloatList(value=[1., 10.])) + })) features = parsing_ops.parse_example( serialized=[data.SerializeToString()], - features=fc.make_parse_example_spec([a_weighted])) + features=fc.make_parse_example_spec_v2([a_weighted])) self.assertIn('aaa', features) self.assertIn('weights', features) - with self.cached_session(): - _assert_sparse_tensor_value( - self, - sparse_tensor.SparseTensorValue( - indices=[[0, 0], [0, 1]], - values=np.array([b'omar', b'stringer'], dtype=np.object_), - dense_shape=[1, 2]), - features['aaa'].eval()) - _assert_sparse_tensor_value( - self, - sparse_tensor.SparseTensorValue( - indices=[[0, 0], [0, 1]], - values=np.array([1., 10.], dtype=np.float32), - dense_shape=[1, 2]), - features['weights'].eval()) + _assert_sparse_tensor_value( + self, + sparse_tensor.SparseTensorValue( + indices=[[0, 0], [0, 1]], + values=np.array([b'omar', b'stringer'], dtype=np.object_), + dense_shape=[1, 2]), self.evaluate(features['aaa'])) + _assert_sparse_tensor_value( + self, + sparse_tensor.SparseTensorValue( + indices=[[0, 0], [0, 1]], + values=np.array([1., 10.], dtype=np.float32), + dense_shape=[1, 2]), self.evaluate(features['weights'])) + + @test_util.run_deprecated_v1 def test_transform_features(self): column = fc.weighted_categorical_column( categorical_column=fc.categorical_column_with_identity( key='ids', num_buckets=3), weight_feature_key='values') inputs = sparse_tensor.SparseTensorValue( - indices=((0, 0), (1, 0), (1, 1)), - values=(0, 1, 0), - dense_shape=(2, 2)) + indices=((0, 0), (1, 0), (1, 1)), values=(0, 1, 0), dense_shape=(2, 2)) weights = sparse_tensor.SparseTensorValue( indices=((0, 0), (1, 0), (1, 1)), values=(0.5, 1.0, 0.1), dense_shape=(2, 2)) - id_tensor, weight_tensor = fc._transform_features({ + id_tensor, weight_tensor = fc._transform_features_v2({ 'ids': inputs, 'values': weights, }, (column,), None)[column] - with _initialized_session(): - _assert_sparse_tensor_value( - self, - sparse_tensor.SparseTensorValue( - indices=inputs.indices, - values=np.array(inputs.values, dtype=np.int64), - dense_shape=inputs.dense_shape), - id_tensor.eval()) - _assert_sparse_tensor_value( - self, - sparse_tensor.SparseTensorValue( - indices=weights.indices, - values=np.array(weights.values, dtype=np.float32), - dense_shape=weights.dense_shape), - weight_tensor.eval()) + self.evaluate(variables_lib.global_variables_initializer()) + self.evaluate(lookup_ops.tables_initializer()) + + _assert_sparse_tensor_value( + self, + sparse_tensor.SparseTensorValue( + indices=inputs.indices, + values=np.array(inputs.values, dtype=np.int64), + dense_shape=inputs.dense_shape), self.evaluate(id_tensor)) + _assert_sparse_tensor_value( + self, + sparse_tensor.SparseTensorValue( + indices=weights.indices, + values=np.array(weights.values, dtype=np.float32), + dense_shape=weights.dense_shape), self.evaluate(weight_tensor)) + + @test_util.run_deprecated_v1 def test_transform_features_dense_input(self): column = fc.weighted_categorical_column( categorical_column=fc.categorical_column_with_identity( @@ -7466,55 +7945,57 @@ class WeightedCategoricalColumnTest(test.TestCase): indices=((0, 0), (1, 0), (1, 1)), values=(0.5, 1.0, 0.1), dense_shape=(2, 2)) - id_tensor, weight_tensor = fc._transform_features({ + id_tensor, weight_tensor = fc._transform_features_v2({ 'ids': ((0, -1), (1, 0)), 'values': weights, }, (column,), None)[column] - with _initialized_session(): - _assert_sparse_tensor_value( - self, - sparse_tensor.SparseTensorValue( - indices=((0, 0), (1, 0), (1, 1)), - values=np.array((0, 1, 0), dtype=np.int64), - dense_shape=(2, 2)), - id_tensor.eval()) - _assert_sparse_tensor_value( - self, - sparse_tensor.SparseTensorValue( - indices=weights.indices, - values=np.array(weights.values, dtype=np.float32), - dense_shape=weights.dense_shape), - weight_tensor.eval()) + self.evaluate(variables_lib.global_variables_initializer()) + self.evaluate(lookup_ops.tables_initializer()) + + _assert_sparse_tensor_value( + self, + sparse_tensor.SparseTensorValue( + indices=((0, 0), (1, 0), (1, 1)), + values=np.array((0, 1, 0), dtype=np.int64), + dense_shape=(2, 2)), self.evaluate(id_tensor)) + _assert_sparse_tensor_value( + self, + sparse_tensor.SparseTensorValue( + indices=weights.indices, + values=np.array(weights.values, dtype=np.float32), + dense_shape=weights.dense_shape), self.evaluate(weight_tensor)) + + @test_util.run_deprecated_v1 def test_transform_features_dense_weights(self): column = fc.weighted_categorical_column( categorical_column=fc.categorical_column_with_identity( key='ids', num_buckets=3), weight_feature_key='values') inputs = sparse_tensor.SparseTensorValue( - indices=((0, 0), (1, 0), (1, 1)), - values=(2, 1, 0), - dense_shape=(2, 2)) - id_tensor, weight_tensor = fc._transform_features({ + indices=((0, 0), (1, 0), (1, 1)), values=(2, 1, 0), dense_shape=(2, 2)) + id_tensor, weight_tensor = fc._transform_features_v2({ 'ids': inputs, 'values': ((.5, 0.), (1., .1)), }, (column,), None)[column] - with _initialized_session(): - _assert_sparse_tensor_value( - self, - sparse_tensor.SparseTensorValue( - indices=inputs.indices, - values=np.array(inputs.values, dtype=np.int64), - dense_shape=inputs.dense_shape), - id_tensor.eval()) - _assert_sparse_tensor_value( - self, - sparse_tensor.SparseTensorValue( - indices=((0, 0), (1, 0), (1, 1)), - values=np.array((.5, 1., .1), dtype=np.float32), - dense_shape=(2, 2)), - weight_tensor.eval()) + self.evaluate(variables_lib.global_variables_initializer()) + self.evaluate(lookup_ops.tables_initializer()) + + _assert_sparse_tensor_value( + self, + sparse_tensor.SparseTensorValue( + indices=inputs.indices, + values=np.array(inputs.values, dtype=np.int64), + dense_shape=inputs.dense_shape), self.evaluate(id_tensor)) + _assert_sparse_tensor_value( + self, + sparse_tensor.SparseTensorValue( + indices=((0, 0), (1, 0), (1, 1)), + values=np.array((.5, 1., .1), dtype=np.float32), + dense_shape=(2, 2)), self.evaluate(weight_tensor)) + + @test_util.run_deprecated_v1 def test_linear_model(self): column = fc.weighted_categorical_column( categorical_column=fc.categorical_column_with_identity( @@ -7535,15 +8016,18 @@ class WeightedCategoricalColumnTest(test.TestCase): dense_shape=(2, 2)) }) weight_var, bias = model.variables - with _initialized_session(): - self.assertAllClose((0.,), bias.eval()) - self.assertAllClose(((0.,), (0.,), (0.,)), weight_var.eval()) - self.assertAllClose(((0.,), (0.,)), predictions.eval()) - weight_var.assign(((1.,), (2.,), (3.,))).eval() - # weight_var[0] * weights[0, 0] = 1 * .5 = .5 - # weight_var[2] * weights[1, 0] + weight_var[1] * weights[1, 1] - # = 3*1 + 2*.1 = 3+.2 = 3.2 - self.assertAllClose(((.5,), (3.2,)), predictions.eval()) + + self.evaluate(variables_lib.global_variables_initializer()) + self.evaluate(lookup_ops.tables_initializer()) + + self.assertAllClose((0.,), self.evaluate(bias)) + self.assertAllClose(((0.,), (0.,), (0.,)), self.evaluate(weight_var)) + self.assertAllClose(((0.,), (0.,)), self.evaluate(predictions)) + self.evaluate(weight_var.assign(((1.,), (2.,), (3.,)))) + # weight_var[0] * weights[0, 0] = 1 * .5 = .5 + # weight_var[2] * weights[1, 0] + weight_var[1] * weights[1, 1] + # = 3*1 + 2*.1 = 3+.2 = 3.2 + self.assertAllClose(((.5,), (3.2,)), self.evaluate(predictions)) def test_linear_model_mismatched_shape(self): column = fc.weighted_categorical_column( @@ -7589,7 +8073,7 @@ class WeightedCategoricalColumnTest(test.TestCase): rewriter_config_pb2.RewriterConfig.OFF) with _initialized_session(config): with self.assertRaisesRegexp(errors.OpError, 'Incompatible shapes'): - predictions.eval() + self.evaluate(predictions) def test_linear_model_mismatched_dense_shape(self): column = fc.weighted_categorical_column( @@ -7607,15 +8091,18 @@ class WeightedCategoricalColumnTest(test.TestCase): 'values': ((.5,), (1.,), (.1,)) }) weight_var, bias = model.variables - with _initialized_session(): - self.assertAllClose((0.,), bias.eval()) - self.assertAllClose(((0.,), (0.,), (0.,)), weight_var.eval()) - self.assertAllClose(((0.,), (0.,)), predictions.eval()) - weight_var.assign(((1.,), (2.,), (3.,))).eval() - # weight_var[0] * weights[0, 0] = 1 * .5 = .5 - # weight_var[2] * weights[1, 0] + weight_var[1] * weights[1, 1] - # = 3*1 + 2*.1 = 3+.2 = 3.2 - self.assertAllClose(((.5,), (3.2,)), predictions.eval()) + + self.evaluate(variables_lib.global_variables_initializer()) + self.evaluate(lookup_ops.tables_initializer()) + + self.assertAllClose((0.,), self.evaluate(bias)) + self.assertAllClose(((0.,), (0.,), (0.,)), self.evaluate(weight_var)) + self.assertAllClose(((0.,), (0.,)), self.evaluate(predictions)) + self.evaluate(weight_var.assign(((1.,), (2.,), (3.,)))) + # weight_var[0] * weights[0, 0] = 1 * .5 = .5 + # weight_var[2] * weights[1, 0] + weight_var[1] * weights[1, 1] + # = 3*1 + 2*.1 = 3+.2 = 3.2 + self.assertAllClose(((.5,), (3.2,)), self.evaluate(predictions)) def test_old_linear_model(self): column = fc.weighted_categorical_column( @@ -7637,15 +8124,18 @@ class WeightedCategoricalColumnTest(test.TestCase): }, (column,)) bias = get_linear_model_bias() weight_var = get_linear_model_column_var(column) - with _initialized_session(): - self.assertAllClose((0.,), bias.eval()) - self.assertAllClose(((0.,), (0.,), (0.,)), weight_var.eval()) - self.assertAllClose(((0.,), (0.,)), predictions.eval()) - weight_var.assign(((1.,), (2.,), (3.,))).eval() - # weight_var[0] * weights[0, 0] = 1 * .5 = .5 - # weight_var[2] * weights[1, 0] + weight_var[1] * weights[1, 1] - # = 3*1 + 2*.1 = 3+.2 = 3.2 - self.assertAllClose(((.5,), (3.2,)), predictions.eval()) + + self.evaluate(variables_lib.global_variables_initializer()) + self.evaluate(lookup_ops.tables_initializer()) + + self.assertAllClose((0.,), self.evaluate(bias)) + self.assertAllClose(((0.,), (0.,), (0.,)), self.evaluate(weight_var)) + self.assertAllClose(((0.,), (0.,)), self.evaluate(predictions)) + self.evaluate(weight_var.assign(((1.,), (2.,), (3.,)))) + # weight_var[0] * weights[0, 0] = 1 * .5 = .5 + # weight_var[2] * weights[1, 0] + weight_var[1] * weights[1, 1] + # = 3*1 + 2*.1 = 3+.2 = 3.2 + self.assertAllClose(((.5,), (3.2,)), self.evaluate(predictions)) def test_old_linear_model_mismatched_shape(self): column = fc.weighted_categorical_column( @@ -7690,7 +8180,7 @@ class WeightedCategoricalColumnTest(test.TestCase): rewriter_config_pb2.RewriterConfig.OFF) with _initialized_session(config): with self.assertRaisesRegexp(errors.OpError, 'Incompatible shapes'): - predictions.eval() + self.evaluate(predictions) def test_old_linear_model_mismatched_dense_shape(self): column = fc.weighted_categorical_column( @@ -7708,19 +8198,22 @@ class WeightedCategoricalColumnTest(test.TestCase): }, (column,)) bias = get_linear_model_bias() weight_var = get_linear_model_column_var(column) - with _initialized_session(): - self.assertAllClose((0.,), bias.eval()) - self.assertAllClose(((0.,), (0.,), (0.,)), weight_var.eval()) - self.assertAllClose(((0.,), (0.,)), predictions.eval()) - weight_var.assign(((1.,), (2.,), (3.,))).eval() - # weight_var[0] * weights[0, 0] = 1 * .5 = .5 - # weight_var[2] * weights[1, 0] + weight_var[1] * weights[1, 1] - # = 3*1 + 2*.1 = 3+.2 = 3.2 - self.assertAllClose(((.5,), (3.2,)), predictions.eval()) + + self.evaluate(variables_lib.global_variables_initializer()) + self.evaluate(lookup_ops.tables_initializer()) + + self.assertAllClose((0.,), self.evaluate(bias)) + self.assertAllClose(((0.,), (0.,), (0.,)), self.evaluate(weight_var)) + self.assertAllClose(((0.,), (0.,)), self.evaluate(predictions)) + self.evaluate(weight_var.assign(((1.,), (2.,), (3.,)))) + # weight_var[0] * weights[0, 0] = 1 * .5 = .5 + # weight_var[2] * weights[1, 0] + weight_var[1] * weights[1, 1] + # = 3*1 + 2*.1 = 3+.2 = 3.2 + self.assertAllClose(((.5,), (3.2,)), self.evaluate(predictions)) def test_old_linear_model_old_categorical(self): column = fc.weighted_categorical_column( - categorical_column=fc_old.categorical_column_with_identity( + categorical_column=fc_old._categorical_column_with_identity( key='ids', num_buckets=3), weight_feature_key='values') with ops.Graph().as_default(): @@ -7738,18 +8231,22 @@ class WeightedCategoricalColumnTest(test.TestCase): }, (column,)) bias = get_linear_model_bias() weight_var = get_linear_model_column_var(column) - with _initialized_session(): - self.assertAllClose((0.,), bias.eval()) - self.assertAllClose(((0.,), (0.,), (0.,)), weight_var.eval()) - self.assertAllClose(((0.,), (0.,)), predictions.eval()) - weight_var.assign(((1.,), (2.,), (3.,))).eval() - # weight_var[0] * weights[0, 0] = 1 * .5 = .5 - # weight_var[2] * weights[1, 0] + weight_var[1] * weights[1, 1] - # = 3*1 + 2*.1 = 3+.2 = 3.2 - self.assertAllClose(((.5,), (3.2,)), predictions.eval()) + + self.evaluate(variables_lib.global_variables_initializer()) + self.evaluate(lookup_ops.tables_initializer()) + + self.assertAllClose((0.,), self.evaluate(bias)) + self.assertAllClose(((0.,), (0.,), (0.,)), self.evaluate(weight_var)) + self.assertAllClose(((0.,), (0.,)), self.evaluate(predictions)) + self.evaluate(weight_var.assign(((1.,), (2.,), (3.,)))) + # weight_var[0] * weights[0, 0] = 1 * .5 = .5 + # weight_var[2] * weights[1, 0] + weight_var[1] * weights[1, 1] + # = 3*1 + 2*.1 = 3+.2 = 3.2 + self.assertAllClose(((.5,), (3.2,)), self.evaluate(predictions)) # TODO(ptucker): Add test with embedding of weighted categorical. + @test_util.run_deprecated_v1 def test_serialization(self): categorical_column = fc.categorical_column_with_identity( key='ids', num_buckets=3) diff --git a/tensorflow/python/framework/auto_control_deps.py b/tensorflow/python/framework/auto_control_deps.py index 9a9ee46aabb..30dc959e9a9 100644 --- a/tensorflow/python/framework/auto_control_deps.py +++ b/tensorflow/python/framework/auto_control_deps.py @@ -21,9 +21,11 @@ from __future__ import print_function from tensorflow.python.eager import context from tensorflow.python.framework import dtypes as dtypes_module from tensorflow.python.framework import ops +from tensorflow.python.framework import sparse_tensor from tensorflow.python.ops import array_ops from tensorflow.python.ops import control_flow_ops from tensorflow.python.ops import control_flow_util +from tensorflow.python.ops import tensor_array_ops from tensorflow.python.util import nest from tensorflow.python.util import tf_decorator @@ -70,6 +72,17 @@ class AutomaticControlDependencies(object): self._returned_tensors.add(indices) self._returned_tensors.add(values) return ops.IndexedSlices(values, indices, dense_shape=tensor.dense_shape) + elif isinstance(tensor, sparse_tensor.SparseTensor): + values = array_ops.identity(tensor.values) + indices = array_ops.identity(tensor.indices) + self._returned_tensors.add(indices) + self._returned_tensors.add(values) + return sparse_tensor.SparseTensor( + indices, values, dense_shape=tensor.dense_shape) + elif isinstance(tensor, tensor_array_ops.TensorArray): + flow = array_ops.identity(tensor.flow) + self._returned_tensors.add(flow) + return tensor_array_ops.build_ta_with_new_flow(tensor, flow) # We want to make the return values depend on the stateful operations, but # we don't want to introduce a cycle, so we make the return value the result # of a new identity operation that the stateful operations definitely don't diff --git a/tensorflow/python/framework/constant_op.py b/tensorflow/python/framework/constant_op.py index 53d84b2dc76..ade0797dcdb 100644 --- a/tensorflow/python/framework/constant_op.py +++ b/tensorflow/python/framework/constant_op.py @@ -114,8 +114,9 @@ def convert_to_eager_tensor(value, ctx, dtype=None): return ops.EagerTensor(value, handle, device, dtype) -@tf_export("constant") -def constant(value, dtype=None, shape=None, name="Const", verify_shape=False): +@tf_export(v1=["constant"]) +def constant_v1( + value, dtype=None, shape=None, name="Const", verify_shape=False): """Creates a constant tensor. The resulting tensor is populated with values of type `dtype`, as @@ -174,6 +175,79 @@ def constant(value, dtype=None, shape=None, name="Const", verify_shape=False): Raises: TypeError: if shape is incorrectly specified or unsupported. """ + return _constant_impl(value, dtype, shape, name, verify_shape=verify_shape, + allow_broadcast=False) + + +@tf_export("constant", v1=[]) +def constant(value, dtype=None, shape=None, name="Const"): + """Creates a constant tensor. + + The resulting tensor is populated with values of type `dtype`, as + specified by arguments `value` and (optionally) `shape` (see examples + below). + + The argument `value` can be a constant value, or a list of values of type + `dtype`. If `value` is a list, then the length of the list must be less + than or equal to the number of elements implied by the `shape` argument (if + specified). In the case where the list length is less than the number of + elements specified by `shape`, the last element in the list will be used + to fill the remaining entries. + + The argument `shape` is optional. If present, it specifies the dimensions of + the resulting tensor. If not present, the shape of `value` is used. + + If the argument `dtype` is not specified, then the type is inferred from + the type of `value`. + + For example: + + ```python + # Constant 1-D Tensor populated with value list. + tensor = tf.constant([1, 2, 3, 4, 5, 6]) => [1 2 3 4 5 6] + + # Constant 1-D Tensor populated with value list. + tensor = tf.constant([1, 2, 3, 4, 5, 6], shape=(2,3)) + => [[1 2 3], [4 5 6]] + + # Constant 2-D tensor populated with scalar value -1. + tensor = tf.constant(-1.0, shape=[2, 3]) => [[-1. -1. -1.] + [-1. -1. -1.]] + ``` + + `tf.constant` differs from `tf.fill` in a few ways: + + * `tf.constant` supports arbitrary constants, not just uniform scalar + Tensors like `tf.fill`. + * `tf.constant` creates a `Const` node in the computation graph with the + exact value at graph construction time. On the other hand, `tf.fill` + creates an Op in the graph that is expanded at runtime. + * Because `tf.constant` only embeds constant values in the graph, it does + not support dynamic shapes based on other runtime Tensors, whereas + `tf.fill` does. + + Args: + value: A constant value (or list) of output type `dtype`. + + dtype: The type of the elements of the resulting tensor. + + shape: Optional dimensions of resulting tensor. + + name: Optional name for the tensor. + + Returns: + A Constant Tensor. + + Raises: + TypeError: if shape is incorrectly specified or unsupported. + """ + return _constant_impl(value, dtype, shape, name, verify_shape=False, + allow_broadcast=True) + + +def _constant_impl( + value, dtype, shape, name, verify_shape, allow_broadcast): + """Implementation of constant.""" ctx = context.context() if ctx.executing_eagerly(): t = convert_to_eager_tensor(value, ctx, dtype) @@ -205,7 +279,8 @@ def constant(value, dtype=None, shape=None, name="Const", verify_shape=False): tensor_value = attr_value_pb2.AttrValue() tensor_value.tensor.CopyFrom( tensor_util.make_tensor_proto( - value, dtype=dtype, shape=shape, verify_shape=verify_shape)) + value, dtype=dtype, shape=shape, verify_shape=verify_shape, + allow_broadcast=allow_broadcast)) dtype_value = attr_value_pb2.AttrValue(type=tensor_value.tensor.dtype) const_tensor = g.create_op( "Const", [], [dtype_value.type], diff --git a/tensorflow/python/framework/device.py b/tensorflow/python/framework/device.py index 7f6e0a75a5c..e7ac6444a4a 100644 --- a/tensorflow/python/framework/device.py +++ b/tensorflow/python/framework/device.py @@ -23,7 +23,7 @@ import threading from tensorflow.python.util.tf_export import tf_export -@tf_export("DeviceSpec") +@tf_export(v1=["DeviceSpec"]) class DeviceSpec(object): """Represents a (possibly partial) specification for a TensorFlow device. diff --git a/tensorflow/python/framework/dtypes.py b/tensorflow/python/framework/dtypes.py index 48e9f0524e8..f7a12d27df7 100644 --- a/tensorflow/python/framework/dtypes.py +++ b/tensorflow/python/framework/dtypes.py @@ -18,6 +18,7 @@ from __future__ import division from __future__ import print_function import numpy as np +from six.moves import builtins from tensorflow.core.framework import types_pb2 from tensorflow.python import pywrap_tensorflow @@ -548,8 +549,8 @@ _NP_TO_TF = frozenset([ (np.int8, int8), (np.complex64, complex64), (np.complex128, complex128), - (np.object, string), - (np.bool, bool), + (np.object_, string), + (np.bool_, bool), (_np_qint8, qint8), (_np_quint8, quint8), (_np_qint16, qint16), @@ -658,8 +659,9 @@ tf_export( __name__, "QUANTIZED_DTYPES") _PYTHON_TO_TF = { - float: float32, - bool: bool, + builtins.float: float32, + builtins.bool: bool, + builtins.object: string } diff --git a/tensorflow/python/framework/dtypes_test.py b/tensorflow/python/framework/dtypes_test.py index a873670e046..719fdc0953a 100644 --- a/tensorflow/python/framework/dtypes_test.py +++ b/tensorflow/python/framework/dtypes_test.py @@ -81,10 +81,10 @@ class TypesTest(test_util.TensorFlowTestCase): self.assertIs(dtypes.int8, dtypes.as_dtype(np.int8)) self.assertIs(dtypes.complex64, dtypes.as_dtype(np.complex64)) self.assertIs(dtypes.complex128, dtypes.as_dtype(np.complex128)) - self.assertIs(dtypes.string, dtypes.as_dtype(np.object)) + self.assertIs(dtypes.string, dtypes.as_dtype(np.object_)) self.assertIs(dtypes.string, dtypes.as_dtype(np.array(["foo", "bar"]).dtype)) - self.assertIs(dtypes.bool, dtypes.as_dtype(np.bool)) + self.assertIs(dtypes.bool, dtypes.as_dtype(np.bool_)) with self.assertRaises(TypeError): dtypes.as_dtype(np.dtype([("f1", np.uint), ("f2", np.int32)])) diff --git a/tensorflow/python/framework/file_system_test.py b/tensorflow/python/framework/file_system_test.py index 6901715e5d0..8687bc5a785 100644 --- a/tensorflow/python/framework/file_system_test.py +++ b/tensorflow/python/framework/file_system_test.py @@ -21,6 +21,7 @@ from __future__ import print_function import os from tensorflow.python.framework import dtypes +from tensorflow.python.framework import test_util from tensorflow.python.framework import load_library from tensorflow.python.ops import data_flow_ops from tensorflow.python.ops import io_ops @@ -36,13 +37,14 @@ class FileSystemTest(test.TestCase): "test_file_system.so") load_library.load_file_system_library(file_system_library) + @test_util.run_deprecated_v1 def testBasic(self): with self.cached_session() as sess: reader = io_ops.WholeFileReader("test_reader") queue = data_flow_ops.FIFOQueue(99, [dtypes.string], shapes=()) queue.enqueue_many([["test://foo"]]).run() queue.close().run() - key, value = sess.run(reader.read(queue)) + key, value = self.evaluate(reader.read(queue)) self.assertEqual(key, compat.as_bytes("test://foo")) self.assertEqual(value, compat.as_bytes("AAAAAAAAAA")) diff --git a/tensorflow/python/framework/func_graph.py b/tensorflow/python/framework/func_graph.py index c7a5d1ee201..f74d072e8e2 100644 --- a/tensorflow/python/framework/func_graph.py +++ b/tensorflow/python/framework/func_graph.py @@ -26,11 +26,13 @@ from tensorflow.python.eager import context from tensorflow.python.eager import tape from tensorflow.python.eager.graph_only_ops import graph_placeholder from tensorflow.python.framework import ops +from tensorflow.python.framework import sparse_tensor from tensorflow.python.framework import tensor_spec from tensorflow.python.framework.auto_control_deps import AutomaticControlDependencies from tensorflow.python.ops import array_ops from tensorflow.python.ops import custom_gradient from tensorflow.python.ops import resource_variable_ops +from tensorflow.python.ops import tensor_array_ops from tensorflow.python.ops import variable_scope from tensorflow.python.util import compat from tensorflow.python.util import nest @@ -111,7 +113,7 @@ class FuncGraph(ops.Graph): # this stack from the default graph even in eager mode. Maybe it should be # part of the eager context? This would also allow us to remove a # get_default_graph() call from the function cache lookup. - self._distribution_strategy_stack = graph._distribution_strategy_stack + self._distribution_strategy_stack = list(graph._distribution_strategy_stack) # We ignore device placements from any outer scopes while tracing the # function when possible, to avoid hard-coding them in the function # graph. "Default" placements come from the PartitionedCallOp's placement, @@ -372,7 +374,7 @@ def func_graph_from_py_func(name, # captured Operations). with ops.control_dependencies([x]): x = array_ops.identity(op_return_value) - else: + elif not isinstance(x, tensor_array_ops.TensorArray): try: x = ops.convert_to_tensor_or_indexed_slices(x) except (ValueError, TypeError): @@ -395,9 +397,9 @@ def func_graph_from_py_func(name, return autograph.converted_call( original_func, None, autograph.ConversionOptions( - verbose=True, + verbose=autograph.Verbosity.BRIEF, recursive=True, - strip_decorators=(function.defun, def_function.function), + strip_decorators=(def_function.function,), optional_features=(), ), *args, **kwargs) @@ -408,7 +410,8 @@ def func_graph_from_py_func(name, func_outputs = python_func(*func_args, **func_kwargs) - # invariant: `func_outputs` contains only Tensors and `None`s. + # invariant: `func_outputs` contains only Tensors, IndexedSlices, + # SparseTensors, TensorArrays and `None`s. func_outputs = nest.map_structure(convert, func_outputs) check_mutation(func_args_before, func_args) @@ -495,7 +498,17 @@ def check_mutation(n1, n2): def flatten(sequence): - """A wrapper around `nest.flatten` that also unpacks `IndexedSlices`.""" + """Like `nest.flatten` but also unpacks other Tensor-like objects. + + Flattens non-tensor objects into their constituent tensors. + + Args: + sequence: A nested structure of Tensors, IndexedSlices, SparseTensors and + TensorArrays. + + Returns: + A list of tensors. + """ # TODO(akshayka): Support `SparseTensor` in a similar fashion. flat_sequence = nest.flatten(sequence) outputs = [] @@ -505,11 +518,58 @@ def flatten(sequence): outputs.extend([item.values, item.indices, item.dense_shape]) else: outputs.extend([item.values, item.indices]) + elif isinstance(item, sparse_tensor.SparseTensor): + outputs.extend([item.indices, item.values, item.dense_shape]) + elif isinstance(item, tensor_array_ops.TensorArray): + outputs.append(item.flow) else: outputs.append(item) return outputs +def pack_sequence_as(structure, flat_sequence): + """Like `nest.pack_sequence_as` but also packs other Tensor-like objects. + + Args: + structure: The structure to pack into. May contain Tensors, IndexedSlices, + TensorArrays or SparseTensors. + flat_sequence: An iterable containing tensors. + + Returns: + A nested structure. + + Raises: + AssertionError if `structure` and `flat_sequence` are not compatible. + """ + flattened_structure = nest.flatten(structure) + flat_sequence_with_slices_and_tas = [] + index = 0 + for t in flattened_structure: + if isinstance(t, ops.IndexedSlices): + if t.dense_shape is not None: + flat_sequence_with_slices_and_tas.append( + ops.IndexedSlices(*flat_sequence[index:index + 3])) + index += 3 + else: + flat_sequence_with_slices_and_tas.append( + ops.IndexedSlices(*flat_sequence[index:index + 2])) + index += 2 + elif isinstance(t, sparse_tensor.SparseTensor): + flat_sequence_with_slices_and_tas.append( + sparse_tensor.SparseTensor(*flat_sequence[index:index + 3])) + index += 3 + elif isinstance(t, tensor_array_ops.TensorArray): + flow = flat_sequence[index] + ta = tensor_array_ops.build_ta_with_new_flow(t, flow) + flat_sequence_with_slices_and_tas.append(ta) + index += 1 + else: + flat_sequence_with_slices_and_tas.append(flat_sequence[index]) + index += 1 + assert len(flattened_structure) == len(flat_sequence_with_slices_and_tas) + return nest.pack_sequence_as(structure, flat_sequence_with_slices_and_tas) + + def _create_substitute_placeholder(value, name=None, dtype=None): """Creates a placeholder for `value` and propagates shape info to it.""" # Note: setting ops.control_dependencies(None) ensures we always put diff --git a/tensorflow/python/framework/function.py b/tensorflow/python/framework/function.py index 230a5546414..622686ce005 100644 --- a/tensorflow/python/framework/function.py +++ b/tensorflow/python/framework/function.py @@ -874,7 +874,7 @@ def func_graph_from_py_func(func, arg_names, arg_types, name=None, # If func only returned one value, make it a tuple. if not isinstance(outputs, (list, tuple)): outputs = (outputs,) - if any([_ is None for _ in outputs]): + if any(_ is None for _ in outputs): raise ValueError("Function %s can not return None." % name) # Ensures each output is a Tensor in the function graph. outputs = [ops.convert_to_tensor(t) for t in outputs] @@ -1190,7 +1190,7 @@ def get_extra_args(): def _type_list_to_str(types): - if any([_ not in _DTYPE_TO_STR for _ in types]): + if any(_ not in _DTYPE_TO_STR for _ in types): raise ValueError("Unsupported dtypes: %s" % types) return "".join([_DTYPE_TO_STR[_] for _ in types]) diff --git a/tensorflow/python/framework/function_def_to_graph_test.py b/tensorflow/python/framework/function_def_to_graph_test.py index b2ef64f8730..ddf1a6e74d2 100644 --- a/tensorflow/python/framework/function_def_to_graph_test.py +++ b/tensorflow/python/framework/function_def_to_graph_test.py @@ -25,6 +25,7 @@ from tensorflow.python.framework import function_def_to_graph from tensorflow.python.framework import graph_to_function_def from tensorflow.python.framework import ops from tensorflow.python.framework import tensor_shape +from tensorflow.python.framework import test_util from tensorflow.python.framework import test_ops from tensorflow.python.ops import array_ops from tensorflow.python.ops import math_ops @@ -52,6 +53,7 @@ class FunctionDefToGraphTest(test.TestCase): fdef.signature.name = "_whats_in_a_name" return fdef + @test_util.run_deprecated_v1 def testInputsAndOutputs(self): fdef = self._build_function_def() g = function_def_to_graph.function_def_to_graph(fdef) @@ -186,6 +188,7 @@ class FunctionDefToGraphDefTest(test.TestCase): self.assertEqual(g.node[0].attr["shape"].shape.unknown_rank, False) self.assertFalse("shape" in g.node[2].attr) + @test_util.run_deprecated_v1 def testFunctionCallsFromFunction(self): x = constant_op.constant(5.0) y = constant_op.constant(10.0) diff --git a/tensorflow/python/framework/function_test.py b/tensorflow/python/framework/function_test.py index 13ee6c5d2d7..d71f06ea528 100644 --- a/tensorflow/python/framework/function_test.py +++ b/tensorflow/python/framework/function_test.py @@ -35,6 +35,7 @@ from tensorflow.python.framework import function from tensorflow.python.framework import graph_to_function_def from tensorflow.python.framework import ops from tensorflow.python.framework import tensor_shape +from tensorflow.python.framework import test_util from tensorflow.python.framework.errors import InvalidArgumentError from tensorflow.python.ops import array_ops from tensorflow.python.ops import control_flow_ops @@ -102,8 +103,9 @@ class FunctionTest(test.TestCase): call = MyIdentityFunc([18.0]) self.assertEqual("MyIdentity", call.op.name) with session.Session() as sess: - self.assertAllEqual([18.0], sess.run(call)) + self.assertAllEqual([18.0], self.evaluate(call)) + @test_util.run_deprecated_v1 def testIdentityImplicitDeref(self): @function.Defun(dtypes.float32, func_name="MyIdentity") @@ -116,8 +118,8 @@ class FunctionTest(test.TestCase): self.assertEqual("MyIdentity", call.op.name) for cfg in _OptimizerOptions(): with session.Session(config=cfg) as sess: - sess.run(var.initializer) - self.assertAllEqual([18.0], sess.run(call)) + self.evaluate(var.initializer) + self.assertAllEqual([18.0], self.evaluate(call)) def testIdentityOutputName(self): @@ -130,7 +132,7 @@ class FunctionTest(test.TestCase): call = MyIdentityFunc([18.0]) self.assertEqual("MyIdentity", call.op.name) with session.Session() as sess: - self.assertAllEqual([18.0], sess.run(call)) + self.assertAllEqual([18.0], self.evaluate(call)) def testTooManyOutputNames(self): @@ -158,7 +160,7 @@ class FunctionTest(test.TestCase): call = APlus2B([1.0], [2.0]) self.assertEqual("APlus2B", call.op.name) with session.Session() as sess: - self.assertAllEqual([5.0], sess.run(call)) + self.assertAllEqual([5.0], self.evaluate(call)) def testFunctionWithNoOutput(self): @@ -187,7 +189,7 @@ class FunctionTest(test.TestCase): call = APlus2B([1.0], [2.0]) self.assertEqual("APlus2B", call.op.name) with session.Session() as sess: - self.assertAllEqual([5.0], sess.run(call)) + self.assertAllEqual([5.0], self.evaluate(call)) def testDefineFunctionDuplicateOutputs(self): @@ -224,8 +226,8 @@ class FunctionTest(test.TestCase): call_g = XSquarePlusOneGrad([2.0], [0.1]) with session.Session() as sess: - self.assertAllClose([5.0], sess.run(call_f)) - self.assertAllClose([0.4], sess.run(call_g)) + self.assertAllClose([5.0], self.evaluate(call_f)) + self.assertAllClose([0.4], self.evaluate(call_g)) def testTanhSymGrad(self): @@ -322,6 +324,7 @@ class FunctionTest(test.TestCase): self.assertEqual(x.get_shape(), dx.get_shape()) self.assertEqual(y.get_shape(), dy.get_shape()) + @test_util.run_deprecated_v1 def testSymGradAttr(self): @function.Defun(noinline=True) @@ -365,7 +368,7 @@ class FunctionTest(test.TestCase): else: dx, dy = gradients_impl.gradients([z], [x, y]) with session.Session() as sess: - dx_val, dy_val = sess.run([dx, dy]) + dx_val, dy_val = self.evaluate([dx, dy]) self.assertEqual([2.0], dx_val) self.assertEqual([0.0], dy_val) @@ -387,7 +390,7 @@ class FunctionTest(test.TestCase): call = AConstant() self.assertEqual("AConstant", call.op.name) with session.Session() as sess: - self.assertAllEqual([42], sess.run(call)) + self.assertAllEqual([42], self.evaluate(call)) def testDefineFunctionNames(self): @@ -438,6 +441,7 @@ class FunctionTest(test.TestCase): "assertion failed.*-3"): self.assertAllEqual(Foo(constant_op.constant(-3.0)).eval(), 6.0) + @test_util.run_deprecated_v1 def testAssertWrapper(self): @function.Defun(dtypes.float32) @@ -452,6 +456,7 @@ class FunctionTest(test.TestCase): "assertion"): _ = MyFn(100.0).eval() + @test_util.run_deprecated_v1 def testWhileLoopCallsFunc(self): with self.session(use_gpu=True) as sess: @@ -468,9 +473,10 @@ class FunctionTest(test.TestCase): loop = control_flow_ops.while_loop(lambda x: x < 1e5, Body, [1.0]) - ans = sess.run(loop) + ans = self.evaluate(loop) self.assertAllClose(ans, 131072.) + @test_util.run_deprecated_v1 def testControlFlowStrictness(self): """Inlined functions must not execute in a untaken control flow branch.""" @@ -517,6 +523,7 @@ class FunctionTest(test.TestCase): "assertion"): sess.run(loop, {pred: True, x: 3}) + @test_util.run_deprecated_v1 def testVar(self): @function.Defun(dtypes.float32) @@ -532,6 +539,7 @@ class FunctionTest(test.TestCase): variables.global_variables_initializer().run() self.assertAllEqual(z.eval(), 101.) + @test_util.run_deprecated_v1 def testResourceVarAsImplicitInput(self): g = ops.Graph() with g.as_default(), ops.device("cpu:0"): @@ -552,8 +560,8 @@ class FunctionTest(test.TestCase): with self.session(graph=g): v.initializer.run() - self.assertAllEqual(expected_val.eval(), actual_val.eval()) - self.assertAllEqual(expected_shape, actual_shape.eval()) + self.assertAllEqual(expected_val.eval(), self.evaluate(actual_val)) + self.assertAllEqual(expected_shape, self.evaluate(actual_shape)) def testDefineErrors(self): with ops.Graph().as_default(): @@ -650,8 +658,8 @@ class FunctionTest(test.TestCase): # pylint: enable=unexpected-keyword-arg self.assertEqual("next", call2.op.name) with session.Session() as sess: - self.assertAllEqual([1], sess.run(call1)) - self.assertAllEqual([0], sess.run(call2)) + self.assertAllEqual([1], self.evaluate(call1)) + self.assertAllEqual([0], self.evaluate(call2)) def testNestedFunction(self): @@ -707,6 +715,7 @@ class FunctionTest(test.TestCase): gdef = g.as_graph_def() self.assertEqual(0, len(gdef.library.function)) + @test_util.run_deprecated_v1 def testReduction(self): g = ops.Graph() @@ -735,6 +744,7 @@ class FunctionTest(test.TestCase): self.assertAllClose(vals[0], vals[1]) self.assertAllClose(vals[2], vals[3]) + @test_util.run_deprecated_v1 def testCapture(self): g = ops.Graph() with g.as_default(): @@ -781,6 +791,7 @@ class FunctionTest(test.TestCase): # NOTE: We still do not support capturing control deps. _ = Foo(x) + @test_util.run_deprecated_v1 def testCaptureInWhileLoop(self): g = ops.Graph() with g.as_default(): @@ -794,8 +805,9 @@ class FunctionTest(test.TestCase): y = Foo() with self.session(graph=g) as sess: - self.assertEqual(sess.run(y), 10) + self.assertEqual(self.evaluate(y), 10) + @test_util.run_deprecated_v1 def testCaptureInCond(self): g = ops.Graph() with g.as_default(): @@ -809,8 +821,8 @@ class FunctionTest(test.TestCase): z = Foo(False) with self.session(graph=g) as sess: - self.assertEqual(sess.run(y), 1) - self.assertEqual(sess.run(z), 2) + self.assertEqual(self.evaluate(y), 1) + self.assertEqual(self.evaluate(z), 2) def testStableName(self): @@ -825,6 +837,7 @@ class FunctionTest(test.TestCase): self.assertEqual("Foo_aCYSbwBkR5A", Foo.instantiate([dtypes.float32] * 3).name) + @test_util.run_deprecated_v1 def testSignatureHash(self): # Foo.Inner and Bar.Inner have identical function body but have # different signatures. They should be treated as two different functions. @@ -854,7 +867,7 @@ class FunctionTest(test.TestCase): z = Bar(x) with self.session(graph=g) as sess: - v0, v1 = sess.run([y, z]) + v0, v1 = self.evaluate([y, z]) self.assertAllEqual(v0, 20.) self.assertAllEqual(v1, 20.) @@ -877,6 +890,7 @@ class FunctionTest(test.TestCase): y = Bar(array_ops.zeros([1, 2, 3])) self.assertAllEqual(y.get_shape().as_list(), [1, 1, 2, 3]) + @test_util.run_deprecated_v1 def testVariableReuse(self): def LinearWithReuse(input_tensor, reuse=None): @@ -900,11 +914,12 @@ class FunctionTest(test.TestCase): self.assertEqual(global_vars[0].name, "linear/w:0") with session.Session() as sess: - sess.run(variables.global_variables_initializer()) + self.evaluate(variables.global_variables_initializer()) output_val = sess.run( output_op, feed_dict={input_op: np.random.rand(32, 100)}) self.assertEqual(output_val.shape, (32, 100)) + @test_util.run_deprecated_v1 def testFunctionCallInDifferentVariableScopes(self): @function.Defun(dtypes.float32) @@ -928,7 +943,7 @@ class FunctionTest(test.TestCase): self.assertEqual(global_vars[0].name, "vs1/var:0") with session.Session() as sess: - sess.run(variables.global_variables_initializer()) + self.evaluate(variables.global_variables_initializer()) out1, out2 = sess.run( [out1_op, out2_op], feed_dict={input_op: np.linspace(1, 10, 10)}) self.assertAllEqual(out1, np.linspace(2, 11, 10)) @@ -968,6 +983,7 @@ class FunctionTest(test.TestCase): self.assertAllClose( np.array([1.0, 0.0]).astype(np.float32), sess.run(dinp, {inp: x})) + @test_util.run_deprecated_v1 def testFunctionMarkedStateful(self): @function.Defun(dtypes.int32, dtypes.float32) @@ -991,10 +1007,11 @@ class FunctionTest(test.TestCase): result_2 = Bar(constant_op.constant(100, dtype=dtypes.int64)) with session.Session() as sess: - self.assertEqual(4.0, sess.run(result_1)) - self.assertEqual(100, sess.run(result_2)) + self.assertEqual(4.0, self.evaluate(result_1)) + self.assertEqual(100, self.evaluate(result_2)) self.assertEqual((4.0, 100), sess.run((result_1, result_2))) + @test_util.run_deprecated_v1 def testStatefulFunction(self): @function.Defun() @@ -1037,6 +1054,7 @@ class FunctionTest(test.TestCase): self.assertFalse(all(val3 == val1)) self.assertFalse(all(val4 == val2)) + @test_util.run_deprecated_v1 def testSameFunctionOnTwoDevices(self): @function.Defun(dtypes.float32) @@ -1052,10 +1070,11 @@ class FunctionTest(test.TestCase): for config in _OptimizerOptions(): config.device_count["CPU"] = 2 with session.Session(config=config) as sess: - self.assertEqual(42.0, sess.run(f_0)) - self.assertEqual(44.0, sess.run(f_1)) + self.assertEqual(42.0, self.evaluate(f_0)) + self.assertEqual(44.0, self.evaluate(f_1)) self.assertEqual((42.0, 44.0), sess.run((f_0, f_1))) + @test_util.run_deprecated_v1 def testGuaranteedConstsAreCaptured(self): var = variables.Variable(1.0) const = array_ops.guarantee_const(var) @@ -1076,9 +1095,10 @@ class FunctionTest(test.TestCase): return output with self.session(use_gpu=False) as sess: - sess.run(var.initializer) + self.evaluate(var.initializer) _ = sess.run(CapturesGuaranteedConst(), {also_not_const: 1.0}) + @test_util.run_deprecated_v1 def testSameFunctionDifferentGrads(self): def PartOne(x): @@ -1127,7 +1147,7 @@ class FunctionTest(test.TestCase): dx2, = gradients_impl.gradients(ys=[y2], xs=[x2]) with self.session(graph=g) as sess: - v0, v1, v2 = sess.run([dx0, dx1, dx2]) + v0, v1, v2 = self.evaluate([dx0, dx1, dx2]) self.assertAllEqual(v0, 2.) self.assertAllEqual(v1, 101.) @@ -1150,6 +1170,7 @@ class FunctionsFromProtos(test.TestCase): self.assertEqual(func.declared_input_types, new_func.declared_input_types) self.assertEqual(func.captured_inputs, new_func.captured_inputs) + @test_util.run_deprecated_v1 def testBasic(self): @function.Defun(dtypes.float32, dtypes.float32) @@ -1359,6 +1380,7 @@ class FunctionsFromProtos(test.TestCase): class FunctionOverloadTest(test.TestCase): + @test_util.run_deprecated_v1 def testBasic(self): @function.Defun() @@ -1411,6 +1433,7 @@ class FunctionOverloadTest(test.TestCase): class FunctionCaptureByValueTest(test.TestCase): + @test_util.run_deprecated_v1 def testCaptureByValue(self): g = ops.Graph() with g.as_default(): @@ -1532,7 +1555,7 @@ class UnrollLSTMTest(test.TestCase): tf_logging.info("time: %f txt size: %d gdef bin size: %d", finish - start, len(str(gdef)), len(gdef.SerializeToString())) with g.as_default(), session.Session(config=cfg) as sess: - return sess.run(m) + return self.evaluate(m) mv0 = RunForward("complete") for cfg in _OptimizerOptions(): @@ -1561,7 +1584,7 @@ class UnrollLSTMTest(test.TestCase): tf_logging.info("time: %f txt size: %d gdef bin size: %d", finish - start, len(str(gdef)), len(gdef.SerializeToString())) with g.as_default(), session.Session(config=cfg) as sess: - return sess.run(dw) + return self.evaluate(dw) d0 = RunForwardBackward("complete") for cfg in _OptimizerOptions(): @@ -1634,6 +1657,7 @@ class FunctionInlineControlTest(test.TestCase): class ModuleFunctionTest(test.TestCase): + @test_util.run_deprecated_v1 def testBasic(self): @function.Defun(*[dtypes.float32] * 3) @@ -1651,8 +1675,8 @@ class ModuleFunctionTest(test.TestCase): y = LinearWithCApi(a, b, c) z = Linear2WithCApi(a, b, c, d, e) with session.Session() as sess: - self.assertAllEqual([[1]], sess.run(y)) - self.assertAllEqual([[5]], sess.run(z)) + self.assertAllEqual([[1]], self.evaluate(y)) + self.assertAllEqual([[5]], self.evaluate(z)) class VariableHoistingTest(test.TestCase): @@ -1704,8 +1728,8 @@ class VariableHoistingTest(test.TestCase): self.assertEqual("Foo/b", b.op.name) with self.session(graph=g) as sess: - sess.run(variables.global_variables_initializer()) - w, b, x, y0, loss, dw, db = sess.run([w, b, x, y0, loss, dw, db]) + self.evaluate(variables.global_variables_initializer()) + w, b, x, y0, loss, dw, db = self.evaluate([w, b, x, y0, loss, dw, db]) self.assertAllEqual(w.shape, (64, 64)) self.assertAllClose(np.sum(w), 2050.44) @@ -1717,10 +1741,12 @@ class VariableHoistingTest(test.TestCase): self.assertAllEqual(db.shape, (64,)) self.assertAllClose(np.sum(db), 0.509, rtol=1e-2) + @test_util.run_deprecated_v1 def testBasic(self): self._testSimpleModel(True) self._testSimpleModel(False) + @test_util.run_deprecated_v1 def testBasicResource(self): self._testSimpleModel(True, use_resource=True) self._testSimpleModel(False, use_resource=True) diff --git a/tensorflow/python/framework/graph_util_test.py b/tensorflow/python/framework/graph_util_test.py index 563a177dd06..4e7408ad49f 100644 --- a/tensorflow/python/framework/graph_util_test.py +++ b/tensorflow/python/framework/graph_util_test.py @@ -29,6 +29,7 @@ from tensorflow.python.framework import graph_util from tensorflow.python.framework import importer from tensorflow.python.framework import ops from tensorflow.python.framework import tensor_util +from tensorflow.python.framework import test_util from tensorflow.python.ops import gen_state_ops from tensorflow.python.ops import math_ops # pylint: disable=unused-import from tensorflow.python.ops import math_ops as math_ops_lib @@ -102,6 +103,7 @@ class DeviceFunctionsTest(test.TestCase): self.assertDeviceEqual(var_5.device, "/device:GPU:0") self.assertDeviceEqual(var_6.device, "/device:CPU:0") + @test_util.run_deprecated_v1 def testNestedDeviceFunctions(self): with ops.Graph().as_default(): var_0 = variables.VariableV1(0) @@ -210,8 +212,8 @@ class DeviceFunctionsTest(test.TestCase): with session.Session() as sess: init = variables.variables_initializer([variable_node]) - sess.run(init) - output = sess.run(output_node) + self.evaluate(init) + output = self.evaluate(output_node) self.assertNear(4.0, output, 0.00001) variable_graph_def = sess.graph.as_graph_def() @@ -242,8 +244,8 @@ class DeviceFunctionsTest(test.TestCase): output_node = math_ops_lib.multiply( variable_node, 2.0, name="output_node") with session.Session() as sess: - sess.run(variable_node.initializer) - output = sess.run(output_node) + self.evaluate(variable_node.initializer) + output = self.evaluate(output_node) self.assertNear(2.0, output, 0.00001) variable_graph_def = sess.graph.as_graph_def() # First get the constant_graph_def when variable_names_whitelist is @@ -256,7 +258,7 @@ class DeviceFunctionsTest(test.TestCase): # Then initialize the unused variable, and get another # constant_graph_def when variable_names_whitelist is not set. - sess.run(another_variable.initializer) + self.evaluate(another_variable.initializer) constant_graph_def_without_variable_whitelist = ( graph_util.convert_variables_to_constants( sess, variable_graph_def, ["output_node"])) @@ -295,7 +297,7 @@ class DeviceFunctionsTest(test.TestCase): ["Variable", "VariableV2", "VarHandleOp", "ReadVariableOp"]) with session.Session() as sess: output_node = sess.graph.get_tensor_by_name("output_node:0") - output = sess.run(output_node) + output = self.evaluate(output_node) self.assertNear(2.0, output, 0.00001) def create_node_def(self, op, name, inputs): diff --git a/tensorflow/python/framework/importer.py b/tensorflow/python/framework/importer.py index c9ac27e7887..98c7aeccc4b 100644 --- a/tensorflow/python/framework/importer.py +++ b/tensorflow/python/framework/importer.py @@ -21,6 +21,7 @@ import contextlib from tensorflow.core.framework import graph_pb2 from tensorflow.python import pywrap_tensorflow as c_api +from tensorflow.python import tf2 from tensorflow.python.framework import c_api_util from tensorflow.python.framework import device as pydev from tensorflow.python.framework import errors @@ -253,7 +254,9 @@ def _ProcessNewOps(graph): # Find any device in the list of colocated ops that have a device, if it # exists. We assume that if multiple ops have devices, they refer to the # same device. Otherwise, a runtime error will occur since the colocation - # property cannot be guaranteed. + # property cannot be guaranteed. Note in TF2 colocations have been removed + # from the public API and will be considered a hint, so there is no runtime + # error. # # One possible improvement is to try to check for compatibility of all # devices in this list at import time here, which would require @@ -262,6 +265,10 @@ def _ProcessNewOps(graph): try: coloc_op = graph._get_operation_by_name_unsafe(coloc_op_name) # pylint: disable=protected-access except KeyError: + # Do not error in TF2 if the colocation cannot be guaranteed + if tf2.enabled(): + continue + raise ValueError('Specified colocation to an op that ' 'does not exist during import: %s in %s' % (coloc_op_name, op.name)) @@ -431,17 +438,16 @@ def import_graph_def(graph_def, # # TODO(skyewm): fetch the TF_Functions directly from the TF_Graph # TODO(skyewm): avoid sending serialized FunctionDefs back to the TF_Graph - # TODO(b/74620627): move this after _ProcessNewOps outside the lock once - # _USE_C_SHAPES is removed. - if graph_def.library and graph_def.library.function: - # pylint: disable=protected-access - functions = function._from_library(graph_def.library) - for f in functions: - f.add_to_graph(graph) - # pylint: enable=protected-access _ProcessNewOps(graph) + if graph_def.library and graph_def.library.function: + # pylint: disable=protected-access + functions = function._from_library(graph_def.library) + for f in functions: + f.add_to_graph(graph) + # pylint: enable=protected-access + # Treat input mappings that don't appear in the graph as an error, because # they are likely to be due to a typo. missing_unused_input_keys = ( diff --git a/tensorflow/python/framework/importer_test.py b/tensorflow/python/framework/importer_test.py index 2b4d8e72995..66e80b55852 100644 --- a/tensorflow/python/framework/importer_test.py +++ b/tensorflow/python/framework/importer_test.py @@ -397,11 +397,11 @@ class ImportGraphDefTest(test.TestCase): # Run the imported graph. # TODO(b/76173421): make this work (currently DCHECKS) # with self.cached_session() as sess: - # sess.run(imported_init) - # self.assertEqual(sess.run(imported_var), 1.0) - # self.assertEqual(sess.run(imported_assign), 2.0) - # self.assertEqual(list(sess.run(imported_shape)), []) - # self.assertEqual(list(sess.run(new_var_shape)), []) + # self.evaluate(imported_init) + # self.assertEqual(self.evaluate(imported_var), 1.0) + # self.assertEqual(self.evaluate(imported_assign), 2.0) + # self.assertEqual(list(self.evaluate(imported_shape)), []) + # self.assertEqual(list(self.evaluate(new_var_shape)), []) def testWhileLoop(self): # Produce GraphDef containing while loop. @@ -418,7 +418,7 @@ class ImportGraphDefTest(test.TestCase): return_elements=[r.name]) self.assertEqual(imported_r.name, "import/" + r.name) with self.cached_session() as sess: - self.assertEqual(sess.run(imported_r), 10) + self.assertEqual(self.evaluate(imported_r), 10) def testImportWhileLoopInCond(self): # Produce GraphDef containing while loop. @@ -458,7 +458,7 @@ class ImportGraphDefTest(test.TestCase): lambda i: i < 2, ImportFn, [0], shape_invariants=[tensor_shape.TensorShape(None)]) with self.cached_session() as sess: - self.assertEqual(sess.run(out), 10) + self.assertEqual(self.evaluate(out), 10) def testTypeMismatchInGraphDef(self): # TODO(skyewm): improve error message @@ -930,7 +930,7 @@ class ImportGraphDefTest(test.TestCase): name="", return_elements=["id:0"]) with self.cached_session(): - self.assertEqual(5.0, t.eval()) + self.assertEqual(5.0, self.evaluate(t)) def testInvalidInputForReturnOperations(self): with ops.Graph().as_default(): @@ -1071,7 +1071,7 @@ class ImportGraphDefTest(test.TestCase): tensor_input = np.ones(input_shape, dtype=np.float32) t = constant_op.constant(tensor_input, shape=input_shape) g = array_ops.identity(t) - g.eval() + self.evaluate(g) def testVersion(self): v0 = versions.GRAPH_DEF_VERSION_MIN_CONSUMER @@ -1255,7 +1255,7 @@ class ImportGraphDefTest(test.TestCase): z = TestFunc() with self.cached_session(): - z_val = z.eval() + z_val = self.evaluate(z) self.assertEqual(z_val, -2.0) def testImportGraphWithFunctionTwice(self): diff --git a/tensorflow/python/framework/load_library.py b/tensorflow/python/framework/load_library.py index 908a5f521e1..727f6aa44c2 100644 --- a/tensorflow/python/framework/load_library.py +++ b/tensorflow/python/framework/load_library.py @@ -31,6 +31,7 @@ from tensorflow.core.lib.core import error_codes_pb2 # pylint: disable=unused-i from tensorflow.python import pywrap_tensorflow as py_tf from tensorflow.python.lib.io import file_io from tensorflow.python.util import compat +from tensorflow.python.util import deprecation from tensorflow.python.util.tf_export import tf_export @@ -83,7 +84,8 @@ def load_op_library(library_filename): return module -@tf_export('load_file_system_library') +@deprecation.deprecated(date=None, instructions='Use tf.load_library instead.') +@tf_export(v1=['load_file_system_library']) def load_file_system_library(library_filename): """Loads a TensorFlow plugin, containing file system implementation. diff --git a/tensorflow/python/framework/meta_graph_test.py b/tensorflow/python/framework/meta_graph_test.py index fc98b91a016..46ce4616a50 100644 --- a/tensorflow/python/framework/meta_graph_test.py +++ b/tensorflow/python/framework/meta_graph_test.py @@ -63,6 +63,7 @@ def _TestDir(test_name): class SimpleMetaGraphTest(test.TestCase): + @test_util.run_deprecated_v1 def testNoVariables(self): test_dir = _TestDir("no_variables") filename = os.path.join(test_dir, "metafile") @@ -116,6 +117,7 @@ class SimpleMetaGraphTest(test.TestCase): {new_input_tensor: input_feed_value}) self.assertEqual(new_output_value, output_value) + @test_util.run_deprecated_v1 def testStrippedOpListNestedFunctions(self): with self.cached_session(): # Square two levels deep @@ -158,6 +160,7 @@ class SimpleMetaGraphTest(test.TestCase): op_list = meta_graph.stripped_op_list_for_graph(graph) self.assertEqual(["Const"], [op.name for op in op_list.op]) + @test_util.run_deprecated_v1 def testDefaultAttrStripping(self): """Verifies that default attributes are stripped from a graph def.""" @@ -210,6 +213,7 @@ class SimpleMetaGraphTest(test.TestCase): self.assertEqual(node_def.attr["Tout"].type, dtypes.complex128) self.assertTrue(meta_graph_def.meta_info_def.stripped_default_attrs) + @test_util.run_deprecated_v1 def testDefaultAttrStrippingNestedFunctions(self): """Verifies that default attributes are stripped from function node defs.""" with self.cached_session(): @@ -261,6 +265,7 @@ class SimpleMetaGraphTest(test.TestCase): self.assertEqual(node_def.attr["attr_1"].i, 1) self.assertTrue(meta_graph_def.meta_info_def.stripped_default_attrs) + @test_util.run_deprecated_v1 def testVariableObjectsAreSharedAmongCollections(self): with ops.Graph().as_default() as graph1: v = variables.Variable(3.0) @@ -454,6 +459,7 @@ class ScopedMetaGraphTest(test.TestCase): # Verifies that we can export the subgraph under each layer and import # them into new layers in a new graph. + @test_util.run_deprecated_v1 def testScopedExportAndImport(self): test_dir = _TestDir("scoped_export_import") filenames = [ @@ -492,8 +498,8 @@ class ScopedMetaGraphTest(test.TestCase): init_op = variables.global_variables_initializer() grad = gradients_impl.gradients([output], [var]) with session.Session() as sess: - sess.run(init_op) - expected_grad_value = sess.run(grad) + self.evaluate(init_op) + expected_grad_value = self.evaluate(grad) # Restore the MetaGraphDef into a new Graph with an import scope. with ops.Graph().as_default(): @@ -518,10 +524,11 @@ class ScopedMetaGraphTest(test.TestCase): init_op = variables.global_variables_initializer() with session.Session() as sess: - sess.run(init_op) - actual_grad_value = sess.run(grad) + self.evaluate(init_op) + actual_grad_value = self.evaluate(grad) self.assertEqual(expected_grad_value, actual_grad_value) + @test_util.run_deprecated_v1 def testImportWhileLoopInWhileLoop(self): # Create a simple while loop. with ops.Graph().as_default(): @@ -544,9 +551,10 @@ class ScopedMetaGraphTest(test.TestCase): _, x = control_flow_ops.while_loop(lambda i, x: i < 2, body, [0, 0.0], name="") with session.Session() as sess: - sess.run(variables.global_variables_initializer()) - sess.run(x) + self.evaluate(variables.global_variables_initializer()) + self.evaluate(x) + @test_util.run_deprecated_v1 def testScopedImportUnderNameScope(self): graph = ops.Graph() with graph.as_default(): @@ -562,6 +570,7 @@ class ScopedMetaGraphTest(test.TestCase): self.assertEqual(list(imported_variables.values())[0].name, "foo/bar/myvar:0") + @test_util.run_deprecated_v1 def testScopedImportUnderNameScopeNoVarScope(self): graph = ops.Graph() with graph.as_default(): @@ -590,6 +599,7 @@ class ScopedMetaGraphTest(test.TestCase): self.assertEqual(list(imported_variables.values())[0].name, "s" + suffix + "/v:0") + @test_util.run_deprecated_v1 def testScopedImportWithSelectedCollections(self): meta_graph_filename = os.path.join( _TestDir("selected_collections_import"), "meta_graph.pb") @@ -600,11 +610,11 @@ class ScopedMetaGraphTest(test.TestCase): with graph.as_default(): variables.Variable(initial_value=1.0, trainable=True) self.assertTrue( - all([ + all( graph.get_collection(key) for key in [ops.GraphKeys.GLOBAL_VARIABLES, ops.GraphKeys.TRAINABLE_VARIABLES] - ])) + )) meta_graph.export_scoped_meta_graph( filename=meta_graph_filename, graph=graph) @@ -687,6 +697,7 @@ class ScopedMetaGraphTest(test.TestCase): # Verifies that we can export the subgraph containing a FIFOQueue under # "queue1" and import it into "new_queue1" in a new graph. + @test_util.run_deprecated_v1 def testScopedWithQueue(self): test_dir = _TestDir("scoped_with_queue") orig_meta_graph = self._testScopedExportWithQueue(test_dir, @@ -749,12 +760,15 @@ class ScopedMetaGraphTest(test.TestCase): for n, e in zip(nodes, expected): self.assertEqual([e], graph2.get_operation_by_name(n).get_attr("_class")) + @test_util.run_deprecated_v1 def testExportNestedNames(self): self.doTestExportNestedNames(use_resource=False) + @test_util.run_deprecated_v1 def testExportNestedNamesResource(self): self.doTestExportNestedNames(use_resource=True) + @test_util.run_deprecated_v1 def testPotentialCycle(self): graph1 = ops.Graph() with graph1.as_default(): @@ -783,6 +797,7 @@ class ScopedMetaGraphTest(test.TestCase): 4.0, shape=[2, 2]) }) + @test_util.run_deprecated_v1 def testClearDevices(self): graph1 = ops.Graph() with graph1.as_default(): @@ -842,6 +857,7 @@ class ScopedMetaGraphTest(test.TestCase): class MetaGraphWithVariableScopeTest(test.TestCase): + @test_util.run_deprecated_v1 def testMetricsCollection(self): def _enqueue_vector(sess, queue, values, shape=None): @@ -868,8 +884,8 @@ class MetaGraphWithVariableScopeTest(test.TestCase): _, update_op = metrics.mean(values) initializer = variables.local_variables_initializer() - sess.run(initializer) - sess.run(update_op) + self.evaluate(initializer) + self.evaluate(update_op) meta_graph.export_scoped_meta_graph( filename=meta_graph_filename, graph=graph) @@ -880,7 +896,7 @@ class MetaGraphWithVariableScopeTest(test.TestCase): with self.session(graph=graph) as sess: meta_graph.import_scoped_meta_graph(meta_graph_filename) initializer = variables.local_variables_initializer() - sess.run(initializer) + self.evaluate(initializer) # Verifies that importing an old meta_graph where "local_variables" # collection is of node_list type works, but cannot build initializer @@ -899,6 +915,7 @@ class MetaGraphWithVariableScopeTest(test.TestCase): class ExportImportAcrossScopesTest(test.TestCase): + @test_util.run_deprecated_v1 def testPartionedVariables(self): def make_graph_with_partitioned_variables(use_resource): diff --git a/tensorflow/python/framework/op_def_library.py b/tensorflow/python/framework/op_def_library.py index 9955a9a2cdd..2318b32ef10 100644 --- a/tensorflow/python/framework/op_def_library.py +++ b/tensorflow/python/framework/op_def_library.py @@ -570,7 +570,7 @@ class OpDefLibrary(object): "than minimum length %d." % (input_name, op_type_name, len(values), num_attr.minimum)) # All tensors must have the same base type. - if any([bt != base_types[0] for bt in base_types]): + if any(bt != base_types[0] for bt in base_types): raise TypeError( "All tensors passed to '%s' of '%s' Op " "must have the same type." % diff --git a/tensorflow/python/framework/ops.py b/tensorflow/python/framework/ops.py index aaa12bf71ff..1a26984809b 100644 --- a/tensorflow/python/framework/ops.py +++ b/tensorflow/python/framework/ops.py @@ -36,14 +36,13 @@ from tensorflow.core.framework import op_def_pb2 from tensorflow.core.framework import versions_pb2 from tensorflow.core.protobuf import config_pb2 from tensorflow.python import pywrap_tensorflow as c_api +from tensorflow.python import tf2 from tensorflow.python.eager import context from tensorflow.python.eager import core from tensorflow.python.eager import tape from tensorflow.python.framework import c_api_util -from tensorflow.python.framework import cpp_shape_inference_pb2 from tensorflow.python.framework import device as pydev from tensorflow.python.framework import dtypes -from tensorflow.python.framework import error_interpolation from tensorflow.python.framework import errors from tensorflow.python.framework import op_def_registry from tensorflow.python.framework import registry @@ -318,22 +317,13 @@ class Tensor(_TensorLike): self._op = op self._value_index = value_index self._dtype = dtypes.as_dtype(dtype) - # This will be set by self._as_tf_output(). self._tf_output = None - # This will be set by self.shape(). self._shape_val = None - # List of operations that use this Tensor as input. We maintain this list # to easily navigate a computation graph. self._consumers = [] - - if not _USE_C_SHAPES: - # Attributes used for C++ shape inference. Not inspected, only forwarded. - # If set, will be a HandleData object from cpp_shape_inference.proto. - self._handle_data = None - self._id = uid() @property @@ -408,17 +398,7 @@ class Tensor(_TensorLike): """ if self._shape_val is None: - if _USE_C_SHAPES: - self._shape_val = self._c_api_shape() - else: - # Call set_shape_and_handle_data_for_outputs in topological order on all - # ops that are needed to compute self.op's shape. We do this instead of - # having set_shape_and_handle_data_for_outputs recursively call - # Operation.shape on self.op.inputs to overflowing the call stack. - need_shapes = self._get_input_ops_without_shapes(self.op) - need_shapes.sort(key=lambda op: op._id) - for op in need_shapes: - set_shape_and_handle_data_for_outputs(op) + self._shape_val = self._c_api_shape() return self._shape_val def _get_input_ops_without_shapes(self, target_op): @@ -533,14 +513,10 @@ class Tensor(_TensorLike): ValueError: If `shape` is not compatible with the current shape of this tensor. """ - if _USE_C_SHAPES: # pylint: disable=protected-access - # Reset cached shape. - self._shape_val = None - else: - self._shape_val = self.shape.merge_with(shape) + # Reset cached shape. + self._shape_val = None - # Update C shape even if _USE_C_SHAPES = False, since we still want - # set_shape to be reflected in the C API graph for when we run it. + # We want set_shape to be reflected in the C API graph for when we run it. if not isinstance(shape, tensor_shape.TensorShape): shape = tensor_shape.TensorShape(shape) dim_list = [] @@ -634,10 +610,7 @@ class Tensor(_TensorLike): return id(self) == id(other) def __copy__(self): - # Make sure _shape_val is computed before we copy. # TODO(b/77597810): get rid of Tensor copies. - if self._shape_val is None: - set_shape_and_handle_data_for_outputs(self.op) cls = self.__class__ result = cls.__new__(cls) result.__dict__.update(self.__dict__) @@ -774,6 +747,18 @@ class _EagerTensorBase(Tensor): def _numpy(self): raise NotImplementedError() + @property + def backing_device(self): + """Returns the name of the device holding this tensor's memory. + + `.backing_device` is usually the same as `.device`, which returns + the device on which the kernel of the operation that produced this tensor + ran. However, some operations can produce tensors on a different device + (e.g., an operation that executes on the GPU but produces output tensors + in host memory). + """ + raise NotImplementedError() + def __copy__(self): # Eager Tensors are immutable so it's safe to return themselves as a copy. return self @@ -890,6 +875,12 @@ class _EagerTensorBase(Tensor): """Returns the number of Tensor dimensions.""" return self.shape.ndims + def __len__(self): + """Returns the length of the first dimension in the Tensor.""" + if not self.shape.ndims: + raise TypeError("Scalar tensor has no `len()`") + return self._shape_tuple()[0] + def _cpu_nograd(self): """A copy of this Tensor with contents backed by host memory. @@ -918,13 +909,7 @@ class _EagerTensorBase(Tensor): return self._copy(context.context(), "GPU:" + str(gpu_index)) def __bool__(self): - if self._shape_tuple() != (): # pylint: disable=g-explicit-bool-comparison - raise ValueError( - "Non-scalar tensor %s cannot be converted to boolean." % repr(self)) - if self.dtype != dtypes.bool: - raise ValueError( - "Non-boolean tensor %s cannot be converted to boolean." % repr(self)) - return bool(self.cpu().numpy()) + return bool(self.numpy()) def __nonzero__(self): return self.__bool__() @@ -1044,12 +1029,12 @@ def convert_to_tensor(value, dtype=None, name=None, preferred_dtype=None): `preferred_dtype` is not possible, this argument has no effect. Returns: - An `Output` based on `value`. + An `Tensor` based on `value`. Raises: - TypeError: If no conversion function is registered for `value`. + TypeError: If no conversion function is registered for `value` to `dtype`. RuntimeError: If a registered conversion function returns an invalid value. - + ValueError: If the `value` is a tensor not of given `dtype` in graph mode. """ return convert_to_tensor_v2(value, dtype, preferred_dtype, name) @@ -1097,12 +1082,12 @@ def convert_to_tensor_v2(value, dtype=None, dtype_hint=None, name=None): name: Optional name to use if a new `Tensor` is created. Returns: - An `Output` based on `value`. + An `Tensor` based on `value`. Raises: - TypeError: If no conversion function is registered for `value`. + TypeError: If no conversion function is registered for `value` to `dtype`. RuntimeError: If a registered conversion function returns an invalid value. - + ValueError: If the `value` is a tensor not of given `dtype` in graph mode. """ return internal_convert_to_tensor( value=value, @@ -1123,49 +1108,13 @@ def internal_convert_to_tensor(value, preferred_dtype=None, ctx=None, accept_symbolic_tensors=True): - """Converts the given `value` to an `Tensor`. - - This function converts Python objects of various types to `Tensor` - objects. It accepts `Tensor` objects, numpy arrays, Python lists, - and Python scalars. For example: - - This function can be useful when composing a new operation in Python - All standard Python op constructors apply this function to each of their - Tensor-valued inputs, which allows those ops to accept numpy arrays, Python - lists, and scalars in addition to `Tensor` objects. - - Args: - value: An object whose type has a registered `Tensor` conversion function. - dtype: Optional element type for the returned tensor. If missing, the - type is inferred from the type of `value`. - name: Optional name to use if a new `Tensor` is created. - as_ref: True if we want the mutable view of Variables, if applicable. - preferred_dtype: Optional element type for the returned tensor, - used when dtype is None. In some cases, a caller may not have a - dtype in mind when converting to a tensor, so preferred_dtype - can be used as a soft preference. If the conversion to - `preferred_dtype` is not possible, this argument has no effect. - ctx: Optional: The value of context.context(). - accept_symbolic_tensors: Whether Keras graph tensors should be accepted as - a valid tensor type during eager execution. - If False, this function will raise an exception if it is passed such - a tensor during eager eager execution. - - Returns: - A `Tensor` based on `value`. - - Raises: - TypeError: If no conversion function is registered for `value`. - RuntimeError: If a registered conversion function returns an invalid value. - - """ + """Implementation of the public convert_to_tensor.""" if ctx is None: ctx = context.context() if isinstance(value, EagerTensor): if ctx.executing_eagerly(): - # Fast path for EagerTensors that don't need any conversion. - # Note that we don't check that value's dtype matches the dtype - # argument. We expect that the C runtime will do that checking - # when we execute the kernel. + if dtype is not None: + dtype = dtypes.as_dtype(dtype) + value = _TensorTensorConversionFunction(value, dtype=dtype) return value else: graph = get_default_graph() @@ -2129,12 +2078,6 @@ class Operation(object): raise TypeError("tensor must be a Tensor: %s" % tensor) _assert_same_graph(self, tensor) - # Make sure output shapes are already computed for this op in case we create - # a cycle (we cannot compute shapes for cycles). Usually shapes are computed - # lazily upon request. - if not _USE_C_SHAPES: - set_shape_and_handle_data_for_outputs(self) - # Reset cached inputs. self._inputs_val = None c_api.UpdateEdge( @@ -2142,6 +2085,31 @@ class Operation(object): tensor._as_tf_output(), # pylint: disable=protected-access self._tf_input(index)) + def _add_while_inputs(self, tensors): + """See AddWhileInputHack in python_api.h. + + NOTE: This is for TF internal use only. Please don't use it. + + Args: + tensors: list of Tensors + + Raises: + TypeError: if tensor is not a Tensor, + or if input tensor type is not convertible to dtype. + ValueError: if the Tensor is from a different graph. + """ + for tensor in tensors: + if not isinstance(tensor, Tensor): + raise TypeError("tensor must be a Tensor: %s" % tensor) + _assert_same_graph(self, tensor) + + # Reset cached inputs. + self._inputs_val = None + c_api.AddWhileInputHack( + self._graph._c_graph, # pylint: disable=protected-access + tensor._as_tf_output(), # pylint: disable=protected-access + self._c_op) + def _add_control_inputs(self, ops): """Add a list of new control inputs to this operation. @@ -2175,6 +2143,23 @@ class Operation(object): """Removes any control inputs to this operation.""" c_api.RemoveAllControlInputs(self._graph._c_graph, self._c_op) # pylint: disable=protected-access + def _add_outputs(self, types, shapes): + """Adds new Tensors to self.outputs. + + Note: this is generally unsafe to use. This is used in certain situations in + conjunction with _set_type_list_attr. + + Arguments: + types: list of DTypes + shapes: list of TensorShapes + """ + assert len(types) == len(shapes) + orig_num_outputs = len(self.outputs) + for i in range(len(types)): + t = Tensor(self, orig_num_outputs + i, types[i]) + self._outputs.append(t) + t.set_shape(shapes[i]) + def __str__(self): return str(self.node_def) @@ -2387,6 +2372,25 @@ class Operation(object): finally: c_api.TF_DeleteBuffer(buf) + def _set_func_attr(self, attr_name, func_name): + """Private method used to set a function attribute in the node_def.""" + func = attr_value_pb2.NameAttrList(name=func_name) + self._set_attr(attr_name, attr_value_pb2.AttrValue(func=func)) + + def _set_type_list_attr(self, attr_name, types): + """Private method used to set a function attribute in the node_def.""" + if not types: return + if isinstance(types[0], dtypes.DType): + types = [dt.as_datatype_enum for dt in types] + types_list = attr_value_pb2.AttrValue.ListValue(type=types) + self._set_attr(attr_name, attr_value_pb2.AttrValue(list=types_list)) + + def _set_shape_list_attr(self, attr_name, shapes): + """Private method used to set a function attribute in the node_def.""" + shapes = [s.as_proto() for s in shapes] + shapes_list = attr_value_pb2.AttrValue.ListValue(shape=shapes) + self._set_attr(attr_name, attr_value_pb2.AttrValue(list=shapes_list)) + def get_attr(self, name): """Returns the value of the attr of this op with the given `name`. @@ -2399,7 +2403,7 @@ class Operation(object): Raises: ValueError: If this op does not have an attr with the given `name`. """ - fields = ["s", "i", "f", "b", "type", "shape", "tensor", "func"] + fields = ("s", "i", "f", "b", "type", "shape", "tensor", "func") try: with c_api_util.tf_buffer() as buf: c_api.TF_OperationGetAttrValueProto(self._c_op, name, buf) @@ -2410,25 +2414,21 @@ class Operation(object): x = attr_value_pb2.AttrValue() x.ParseFromString(data) - # Treat an empty oneof value as an empty list. - if not x.WhichOneof("value"): + oneof_value = x.WhichOneof("value") + if oneof_value is None: return [] - if x.HasField("list"): + if oneof_value == "list": for f in fields: if getattr(x.list, f): if f == "type": - return [dtypes.as_dtype(x) for x in list(getattr(x.list, f))] + return [dtypes.as_dtype(t) for t in x.list.type] else: return list(getattr(x.list, f)) return [] - else: - for f in fields: - if x.HasField(f): - if f == "type": - return dtypes.as_dtype(getattr(x, f)) - else: - return getattr(x, f) - assert False, "Unsupported field type in " + str(x) + if oneof_value == "type": + return dtypes.as_dtype(x.type) + assert oneof_value in fields, "Unsupported field type in " + str(x) + return getattr(x, oneof_value) def run(self, feed_dict=None, session=None): """Runs this operation in a `Session`. @@ -2608,72 +2608,9 @@ class RegisterShape(object): return f -# TODO(b/74620627): remove when _USE_C_SHAPES is removed -def _set_shape_and_handle_data_for_outputs_c_api(op): - """Set shapes and resource handle data using info from the C API.""" - assert not _USE_C_SHAPES - for output in op.outputs: - output._shape_val = output._c_api_shape() - # Set the resource handle data for compatibility with the Python shape - # inference code. - serialized = c_api.GetHandleShapeAndType(op._graph._c_graph, # pylint: disable=protected-access - output._as_tf_output()) - if serialized: - output._handle_data = ( - cpp_shape_inference_pb2.CppShapeInferenceResult.HandleData - .FromString(compat.as_bytes(serialized))) - else: - output._handle_data = None - - -# TODO(b/74620627): remove when _USE_C_SHAPES is removed -def set_shape_and_handle_data_for_outputs(op): - """Set the shapes and resource handle data for op's outputs. - - When _USE_C_SHAPES = False, this is lazily called when a tensor's shape is - first requested. Usually this should work automatically, but some edge cases - may require manually calling this first to make sure Tensor._shape_val and - Tensor._handle_data are set (e.g. manually overriding _handle_data, copying a - Tensor). - """ - if _USE_C_SHAPES: return - - if op.graph._is_function(op.type): - for output in op.outputs: - output._shape_val = tensor_shape.unknown_shape() - return - - try: - shape_func = _shape_registry.lookup(op.type) - except LookupError: - try: - shape_func = _default_shape_function_registry.lookup(op.type) - except LookupError: - shape_func = _call_cpp_shape_fn_and_require_op - - shapes = shape_func(op) - if shapes is None: - raise RuntimeError( - "Shape function for op %s did not return any shapes" % op) - elif isinstance(shapes, dict): - # Returned by call_cpp_shape_fn - shapes_dict = shapes - shapes = shapes_dict["shapes"] - handle_datas = shapes_dict["handle_data"] - for output, handle_data in zip(op.outputs, handle_datas): - # Don't override any existing handle data that may have been manually set. - # pylint: disable=protected-access - if output._handle_data is None: - output._handle_data = handle_data - # pylint: enable=protected-access - - if len(op.outputs) != len(shapes): - raise RuntimeError( - "Shape function for op %s returned %d shapes but expected %d %s %s" % - (op, len(shapes), len(op.outputs), shape_func.__name__, str(shapes))) - for output, s in zip(op.outputs, shapes): - output._shape_val = tensor_shape.unknown_shape() - output._shape_val = output._shape_val.merge_with(s) +def set_shape_and_handle_data_for_outputs(_): + """No op. TODO(b/74620627): Remove this.""" + pass class OpStats(object): @@ -2901,8 +2838,8 @@ class Graph(object): self._stack_state_is_thread_local = False self._thread_local = threading.local() # Functions that will be applied to choose a device if none is specified. - # After switch_to_thread_local(), self._thread_local._device_function_stack - # is used instead. + # In TF2.x or after switch_to_thread_local(), + # self._thread_local._device_function_stack is used instead. self._graph_device_function_stack = traceable_stack.TraceableStack() # Default original_op applied to new ops. self._default_original_op = None @@ -2910,7 +2847,7 @@ class Graph(object): # WhileContext defined in ops/control_flow_ops.py self._control_flow_context = None # A new node will depend of the union of all of the nodes in the stack. - # After switch_to_thread_local(), + # In TF2.x or after switch_to_thread_local(), # self._thread_local._control_dependencies_stack is used instead. self._graph_control_dependencies_stack = [] # Arbitrary collections of objects. @@ -2934,7 +2871,7 @@ class Graph(object): producer=versions.GRAPH_DEF_VERSION, min_consumer=versions.GRAPH_DEF_VERSION_MIN_CONSUMER) self._building_function = False - # Stack of colocate_with ops. After switch_to_thread_local(), + # Stack of colocate_with ops. In TF2.x or after switch_to_thread_local(), # self._thread_local._colocation_stack is used instead. self._graph_colocation_stack = traceable_stack.TraceableStack() # Set of tensors that are dangerous to feed! @@ -2967,6 +2904,8 @@ class Graph(object): # requirement (many custom ops do not have shape functions, and we don't # want to break these existing cases). c_api.SetRequireShapeInferenceFns(self._c_graph, False) + if tf2.enabled(): + self.switch_to_thread_local() # Note: this method is private because the API of tf.Graph() is public and # frozen, and this functionality is still not ready for public visibility. @@ -3391,36 +3330,6 @@ class Graph(object): self._create_op_helper(ret, compute_device=compute_device) return ret - def _make_colocation_conflict_message(self, op, colocation_op): - """Return detailed error message about device conflict due to colocation.""" - # Example error message: - # Tried to colocate op 'a' (defined at file1.py:149) having device - # '/device:GPU:0' with op 'b' (defined at file2:96) which had an - # incompatible device '/device:CPU:0'. - # - # No node-device colocations were active during op 'a' creation. - # Device assignments active during op 'a' creation: - # with tf.device(/device:GPU:0): file1.py:148> - # - # Node-device colocations active during op 'b' creation: - # with tf.colocate_with(a): file2.py:93> - # Device assignments active during op 'b' creation: - # with tf.device(/cpu:0): file2.py:94 - op_info = error_interpolation.compute_field_dict(op) - coloc_op_info = error_interpolation.compute_field_dict(colocation_op) - msg = ("Tried to colocate op '{op_name}'{op_loc} having device '{op_dev}' " - "with op '{coloc_op_name}'{coloc_op_loc} which had an incompatible " - "device '{coloc_op_dev}'.\n\n{op_summary}\n\n{coloc_op_summary}" - .format(op_name=op.name, - op_loc=op_info["defined_at"], - op_dev=op.device, - op_summary=op_info["devs_and_colocs"], - coloc_op_name=colocation_op.name, - coloc_op_loc=coloc_op_info["defined_at"], - coloc_op_dev=colocation_op.device, - coloc_op_summary=coloc_op_info["devs_and_colocs"])) - return msg - def _create_op_helper(self, op, compute_device=True): """Common logic for creating an op in this graph.""" # Apply any additional attributes requested. Do not overwrite any existing @@ -3473,12 +3382,9 @@ class Graph(object): for colocation_op in self._colocation_stack.peek_objs(): all_colocation_groups.extend(colocation_op.colocation_groups()) if colocation_op.device: - if (op.device and pydev.canonical_name(op.device) != - pydev.canonical_name(colocation_op.device)): - msg = self._make_colocation_conflict_message(op, colocation_op) - logging.warning(msg) - else: - op._set_device(colocation_op.device) # pylint: disable=protected-access + # pylint: disable=protected-access + op._set_device(colocation_op.device) + # pylint: enable=protected-access all_colocation_groups = sorted(set(all_colocation_groups)) # pylint: disable=protected-access @@ -3526,11 +3432,6 @@ class Graph(object): # pylint: disable=protected-access for op in new_ops: - # Operations created by the C API always retrieve shapes from the C API so - # we preserve the shapes of ops created in import_graph_def (from the - # "_output_shapes" attr of the imported NodeDef). - if not _USE_C_SHAPES: - _set_shape_and_handle_data_for_outputs_c_api(op) new_control_inputs = self._control_dependencies_for_inputs(op.inputs) op._add_control_inputs(new_control_inputs) op._control_flow_post_processing() @@ -5482,7 +5383,7 @@ def inside_function(): return get_default_graph().building_function -@tf_export("enable_eager_execution") +@tf_export(v1=["enable_eager_execution"]) def enable_eager_execution(config=None, device_policy=None, execution_mode=None): @@ -5553,6 +5454,17 @@ def enable_eager_execution(config=None, server_def=None) +@tf_export(v1=["disable_eager_execution"]) +def disable_eager_execution(): + """Disables eager execution. + + This function can only be called before any Graphs, Ops, or Tensors have been + created. It can be used at the beginning of the program for complex migration + projects from TensorFlow 1.x to 2.x. + """ + context.default_execution_mode = context.GRAPH_MODE + + def enable_eager_execution_internal(config=None, device_policy=None, execution_mode=None, @@ -5560,6 +5472,7 @@ def enable_eager_execution_internal(config=None, """Enables eager execution for the lifetime of this program. Most of the doc string for enable_eager_execution is relevant here as well. + Args: config: See enable_eager_execution doc string device_policy: See enable_eager_execution doc string @@ -5652,7 +5565,7 @@ def eager_run(main=None, argv=None): app.run(main, argv) -@tf_export("reset_default_graph") +@tf_export(v1=["reset_default_graph"]) def reset_default_graph(): """Clears the default graph stack and resets the global default graph. @@ -5671,7 +5584,7 @@ def reset_default_graph(): _default_graph_stack.reset() -@tf_export("get_default_graph") +@tf_export(v1=["get_default_graph"]) def get_default_graph(): """Returns the default graph for the current thread. @@ -5798,7 +5711,7 @@ def _get_graph_from_inputs(op_input_list, graph=None): return graph or get_default_graph() -@tf_export("GraphKeys") +@tf_export(v1=["GraphKeys"]) class GraphKeys(object): """Standard names to use for graph collections. @@ -6004,7 +5917,7 @@ def add_to_collections(names, value): get_default_graph().add_to_collections(names, value) -@tf_export("get_collection_ref") +@tf_export(v1=["get_collection_ref"]) def get_collection_ref(key): """Wrapper for `Graph.get_collection_ref()` using the default graph. @@ -6028,7 +5941,7 @@ def get_collection_ref(key): return get_default_graph().get_collection_ref(key) -@tf_export("get_collection") +@tf_export(v1=["get_collection"]) def get_collection(key, scope=None): """Wrapper for `Graph.get_collection()` using the default graph. diff --git a/tensorflow/python/framework/ops_test.py b/tensorflow/python/framework/ops_test.py index 0fb17081e75..7baa02b446b 100644 --- a/tensorflow/python/framework/ops_test.py +++ b/tensorflow/python/framework/ops_test.py @@ -57,11 +57,13 @@ ops._set_call_cpp_shape_fn(common_shapes.call_cpp_shape_fn) class ResourceTest(test_util.TensorFlowTestCase): + @test_util.run_deprecated_v1 def testBuildGraph(self): with self.cached_session(): pt = test_ops.stub_resource_handle_op(container="a", shared_name="b") test_ops.resource_create_op(pt).run() + @test_util.run_deprecated_v1 def testInitialize(self): with self.cached_session(): handle = test_ops.stub_resource_handle_op(container="a", shared_name="b") @@ -106,6 +108,7 @@ class TensorAndShapeTest(test_util.TensorFlowTestCase): c = a + b self.assertEqual([2, 3], c.shape) + @test_util.run_deprecated_v1 def testUnknownDim(self): with self.cached_session(): a = array_ops.placeholder(dtype=dtypes.float32, shape=[2, None, 3]) @@ -113,6 +116,7 @@ class TensorAndShapeTest(test_util.TensorFlowTestCase): c = a + b self.assertEqual([2, None, 3], c.shape.as_list()) + @test_util.run_deprecated_v1 def testUnknownShape(self): with self.cached_session(): a = array_ops.placeholder(dtype=dtypes.float32, shape=None) @@ -120,6 +124,7 @@ class TensorAndShapeTest(test_util.TensorFlowTestCase): c = a + b self.assertEqual(tensor_shape.unknown_shape(), c.shape) + @test_util.run_deprecated_v1 def testScalarShape(self): with self.cached_session(): a = array_ops.placeholder(dtype=dtypes.float32, shape=[]) @@ -127,6 +132,7 @@ class TensorAndShapeTest(test_util.TensorFlowTestCase): c = a + b self.assertEqual(tensor_shape.scalar(), c.shape) + @test_util.run_deprecated_v1 def testShapeFunctionError(self): with self.cached_session(): a = array_ops.ones([1, 2, 3]) @@ -140,15 +146,16 @@ class TensorAndShapeTest(test_util.TensorFlowTestCase): class IndexedSlicesTest(test_util.TensorFlowTestCase): + @test_util.run_in_graph_and_eager_modes def testToTensor(self): - with self.cached_session(): - values = constant_op.constant([2, 3, 5, 7], shape=[2, 2]) - indices = constant_op.constant([0, 2]) - dense_shape = constant_op.constant([3, 2]) - x = ops.IndexedSlices(values, indices, dense_shape) - tensor = ops.convert_to_tensor(x, name="tensor") - self.assertAllEqual(tensor.eval(), [[2, 3], [0, 0], [5, 7]]) + values = constant_op.constant([2, 3, 5, 7], shape=[2, 2]) + indices = constant_op.constant([0, 2]) + dense_shape = constant_op.constant([3, 2]) + x = ops.IndexedSlices(values, indices, dense_shape) + tensor = ops.convert_to_tensor(x, name="tensor") + self.assertAllEqual(self.evaluate(tensor), [[2, 3], [0, 0], [5, 7]]) + @test_util.run_deprecated_v1 def testNegation(self): with self.cached_session(): values = constant_op.constant([2, 3, 5, 7], shape=[2, 2]) @@ -157,6 +164,7 @@ class IndexedSlicesTest(test_util.TensorFlowTestCase): self.assertAllEqual(x.values.eval(), [[-2, -3], [-5, -7]]) self.assertAllEqual(x.indices.eval(), [0, 2]) + @test_util.run_deprecated_v1 def testScalarMul(self): with self.cached_session(): values = constant_op.constant([2, 3, 5, 7], shape=[2, 2]) @@ -190,6 +198,7 @@ def _apply_op(g, *args, **kwargs): class OperationTest(test_util.TensorFlowTestCase): + @test_util.run_deprecated_v1 def testNoInputs(self): op = test_ops.float_output_string_output(name="myop").a.op self.assertEqual(2, len(op.values())) @@ -212,6 +221,7 @@ class OperationTest(test_util.TensorFlowTestCase): self.assertProtoEquals("op:'FloatOutputStringOutput' name:'myop'", op.node_def) + @test_util.run_deprecated_v1 def testNoOutputs(self): op1 = test_ops.float_output(name="myop1").op float_t, = op1.values() @@ -227,6 +237,7 @@ class OperationTest(test_util.TensorFlowTestCase): self.assertProtoEquals("op:'FloatInput' name:'myop2' input:'myop1'", op2.node_def) + @test_util.run_deprecated_v1 def testInputsAndOutputs(self): op1 = test_ops.float_output(name="myop1").op self.assertEqual(1, len(op1.values())) @@ -308,16 +319,17 @@ class OperationTest(test_util.TensorFlowTestCase): with self.assertRaises(ValueError): ops.Operation(ops._NodeDef("op", "invalid:0"), g) + @test_util.run_deprecated_v1 def testNoShapeFunction(self): op = test_ops.a() self.assertEqual(tensor_shape.unknown_shape(), op.get_shape()) + @test_util.run_in_graph_and_eager_modes def testConvertToTensorNestedArray(self): - with self.cached_session(): - values = [[2], [3], [5], [7]] - tensor = ops.convert_to_tensor(values) - self.assertAllEqual((4, 1), tensor.get_shape().as_list()) - self.assertAllEqual(values, tensor.eval()) + values = [[2], [3], [5], [7]] + tensor = ops.convert_to_tensor(values) + self.assertAllEqual((4, 1), tensor.get_shape().as_list()) + self.assertAllEqual(values, self.evaluate(tensor)) def testShapeTuple(self): with self.cached_session(): @@ -333,57 +345,63 @@ class OperationTest(test_util.TensorFlowTestCase): converted = ops.convert_to_tensor(1) self.assertTrue(isinstance(converted, ops.EagerTensor)) + @test_util.run_in_graph_and_eager_modes def testConvertToTensorNestedTuple(self): - with self.cached_session(): - values = ((2,), (3,), (5,), (7,)) - tensor = ops.convert_to_tensor(values) - self.assertAllEqual((4, 1), tensor.get_shape().as_list()) - self.assertAllEqual(values, ops.convert_to_tensor(values).eval()) + values = ((2,), (3,), (5,), (7,)) + tensor = ops.convert_to_tensor(values) + self.assertAllEqual((4, 1), tensor.get_shape().as_list()) + self.assertAllEqual(values, self.evaluate(ops.convert_to_tensor(values))) + @test_util.run_in_graph_and_eager_modes def testConvertToTensorNestedTensors(self): - with self.cached_session(): - values = ((2,), (3,), (5,), (7,)) - tensor = ops.convert_to_tensor( - [constant_op.constant(row) for row in values]) - self.assertAllEqual((4, 1), tensor.get_shape().as_list()) - self.assertAllEqual(values, tensor.eval()) - tensor = ops.convert_to_tensor( - [[constant_op.constant(v) for v in row] for row in values]) - self.assertAllEqual((4, 1), tensor.get_shape().as_list()) - self.assertAllEqual(values, tensor.eval()) + values = ((2,), (3,), (5,), (7,)) + tensor = ops.convert_to_tensor( + [constant_op.constant(row) for row in values]) + self.assertAllEqual((4, 1), tensor.get_shape().as_list()) + self.assertAllEqual(values, self.evaluate(tensor)) + tensor = ops.convert_to_tensor( + [[constant_op.constant(v) for v in row] for row in values]) + self.assertAllEqual((4, 1), tensor.get_shape().as_list()) + self.assertAllEqual(values, self.evaluate(tensor)) + @test_util.run_in_graph_and_eager_modes def testConvertToTensorNestedMix(self): - with self.cached_session(): - values = ([2], (3,), [constant_op.constant(5)], constant_op.constant([7])) - tensor = ops.convert_to_tensor(values) - self.assertAllEqual((4, 1), tensor.get_shape().as_list()) - self.assertAllEqual(((2,), (3,), (5,), (7,)), tensor.eval()) + values = ([2], (3,), [constant_op.constant(5)], constant_op.constant([7])) + tensor = ops.convert_to_tensor(values) + self.assertAllEqual((4, 1), tensor.get_shape().as_list()) + self.assertAllEqual(((2,), (3,), (5,), (7,)), self.evaluate(tensor)) + @test_util.run_in_graph_and_eager_modes def testConvertToTensorPreferred(self): - with self.cached_session(): - values = [2, 3, 5, 7] - tensor = ops.convert_to_tensor(values, preferred_dtype=dtypes.float32) - self.assertEqual(dtypes.float32, tensor.dtype) + values = [2, 3, 5, 7] + tensor = ops.convert_to_tensor(values, preferred_dtype=dtypes.float32) + self.assertEqual(dtypes.float32, tensor.dtype) - with self.cached_session(): - # Convert empty tensor to anything. - values = [] - tensor = ops.convert_to_tensor(values, preferred_dtype=dtypes.int64) - self.assertEqual(dtypes.int64, tensor.dtype) + # Convert empty tensor to anything. + values = [] + tensor = ops.convert_to_tensor(values, preferred_dtype=dtypes.int64) + self.assertEqual(dtypes.int64, tensor.dtype) - with self.cached_session(): - # The preferred dtype is a type error and will convert to - # float32 instead. - values = [1.23] - tensor = ops.convert_to_tensor(values, preferred_dtype=dtypes.int64) - self.assertEqual(dtypes.float32, tensor.dtype) + # The preferred dtype is a type error and will convert to + # float32 instead. + values = [1.23] + tensor = ops.convert_to_tensor(values, preferred_dtype=dtypes.int64) + self.assertEqual(dtypes.float32, tensor.dtype) + @test_util.run_in_graph_and_eager_modes def testConvertToInvalidTensorType(self): with self.assertRaises(TypeError): # Forcing an invalid dtype should fail with a type error. values = [1.23] - _ = ops.convert_to_tensor(values, dtype=dtypes.int64) + ops.convert_to_tensor(values, dtype=dtypes.int64) + @test_util.run_in_graph_and_eager_modes + def testConvertToTensorFromInvalidTensor(self): + tensor = constant_op.constant(42.0, dtype=dtypes.float32) + with self.assertRaises(ValueError): + ops.convert_to_tensor(tensor, dtype=dtypes.int32) + + @test_util.run_deprecated_v1 def testNoConvert(self): # Operation cannot be converted to Tensor. op = control_flow_ops.no_op() @@ -401,6 +419,7 @@ class OperationTest(test_util.TensorFlowTestCase): ops._NodeDef("None", "op1"), ops.Graph(), [], [dtypes.float32]) self.assertEqual("", repr(op)) + @test_util.run_deprecated_v1 def testGetAttr(self): op = test_ops.default_attrs() self.assertEqual(op.get_attr("string_val"), b"abc") @@ -446,6 +465,7 @@ class OperationTest(test_util.TensorFlowTestCase): # TODO(b/65162920): remove this test when users who are directly mutating the # node_def have been updated to proper usage. + @test_util.run_deprecated_v1 def testSetAttr(self): op = test_ops.int_attr().op op._set_attr("foo", attr_value_pb2.AttrValue(i=2)) @@ -466,6 +486,7 @@ class OperationTest(test_util.TensorFlowTestCase): self.assertEqual(z.control_inputs, [x, y]) self.assertEqual(x._control_outputs, [z]) + @test_util.run_deprecated_v1 def testRemoveAllControlInputs(self): a = constant_op.constant(1) with ops.control_dependencies([a]): @@ -490,6 +511,7 @@ class OperationTest(test_util.TensorFlowTestCase): self.assertEqual(f.op.control_inputs, []) self.assertEqual(list(f.op.inputs), [d, e]) + @test_util.run_deprecated_v1 def testControlInputCycle(self): graph = ops.Graph() with graph.as_default(): @@ -503,7 +525,7 @@ class OperationTest(test_util.TensorFlowTestCase): with self.assertRaisesRegexp( errors.InvalidArgumentError, "Graph is invalid, contains a cycle with 2 nodes"): - sess.run(x) + self.evaluate(x) def testUpdateInput(self): g = ops.Graph() @@ -517,21 +539,21 @@ class OperationTest(test_util.TensorFlowTestCase): self.assertEquals(x.consumers(), []) self.assertEquals(y.consumers(), [z.op, z.op]) with session.Session(graph=g) as sess: - self.assertEquals(sess.run(z), 4) + self.assertEquals(self.evaluate(z), 4) z.op._update_input(0, x) # pylint: disable=protected-access self.assertEquals(list(z.op.inputs), [x, y]) self.assertEquals(x.consumers(), [z.op]) self.assertEquals(y.consumers(), [z.op]) with session.Session(graph=g) as sess: - self.assertEquals(sess.run(z), 3) + self.assertEquals(self.evaluate(z), 3) z.op._update_input(1, y) # pylint: disable=protected-access self.assertEquals(list(z.op.inputs), [x, y]) self.assertEquals(x.consumers(), [z.op]) self.assertEquals(y.consumers(), [z.op]) with session.Session(graph=g) as sess: - self.assertEquals(sess.run(z), 3) + self.assertEquals(self.evaluate(z), 3) def testUpdateInputGraphError(self): g_0 = ops.Graph() @@ -557,7 +579,7 @@ class OperationTest(test_util.TensorFlowTestCase): errors.InvalidArgumentError, "Input 0 of node add was passed string from Const_1:0 incompatible " "with expected int32"): - sess.run(z) + self.evaluate(z) def testUpdateInputShapeError(self): g = ops.Graph() @@ -582,6 +604,32 @@ class OperationTest(test_util.TensorFlowTestCase): ): x.op._update_input(1, x) # pylint: disable=protected-access + @test_util.enable_control_flow_v2 + def testAddWhileInput(self): + @eager_function.defun + def test(): + output = control_flow_ops.while_loop(lambda x: x < 3, lambda x: x + 1, + [1]) + while_op = output.op.inputs[0].op + self.assertEqual(while_op.type, "While") + orig_num_inputs = len(while_op.inputs) + + new_input1 = constant_op.constant(1.0) + new_input2 = constant_op.constant(True) + + while_op._set_type_list_attr("T", + [t.dtype for t in while_op.inputs] + + [new_input1.dtype, new_input2.dtype]) + + while_op._add_while_inputs([new_input1, new_input2]) + # Can't add an edge beyond what's specified by "T" + with self.assertRaises(errors.OutOfRangeError): + while_op._add_while_inputs([new_input2]) + self.assertEqual(len(while_op.inputs), orig_num_inputs + 2) # pylint: disable=g-deprecated-assert + + test() + + @test_util.run_deprecated_v1 def testOpDef(self): x = constant_op.constant(0) y = constant_op.constant(1) @@ -681,6 +729,7 @@ class CreateOpTest(test_util.TensorFlowTestCase): # the low-level behavior. class CreateOpFromTFOperationTest(test_util.TensorFlowTestCase): + @test_util.run_deprecated_v1 def testBasic(self): g = ops.Graph() with g.as_default(): @@ -701,7 +750,6 @@ class CreateOpFromTFOperationTest(test_util.TensorFlowTestCase): self.assertEqual(g.get_operation_by_name("myop"), op) self.assertEqual(g.get_tensor_by_name("myop:0"), op.outputs[0]) - @test_util.enable_c_shapes def testShape(self): g = ops.Graph() with g.as_default(): @@ -732,6 +780,7 @@ class CreateOpFromTFOperationTest(test_util.TensorFlowTestCase): self.assertEqual(op3.name, "myop_2") self.assertEqual(op4.name, "myop_1_1") + @test_util.run_deprecated_v1 def testCond(self): g = ops.Graph() with g.as_default(): @@ -761,6 +810,7 @@ class CreateOpFromTFOperationTest(test_util.TensorFlowTestCase): "cond/cond_text") # pylint: enable=protected-access + @test_util.run_deprecated_v1 def testWhileLoop(self): g = ops.Graph() with g.as_default(): @@ -790,6 +840,7 @@ class CreateOpFromTFOperationTest(test_util.TensorFlowTestCase): "myloop/while_context") # pylint: enable=protected-access + @test_util.run_deprecated_v1 def testWhileLoopWithInternalControlDep(self): g = ops.Graph() with g.as_default(): @@ -813,6 +864,7 @@ class CreateOpFromTFOperationTest(test_util.TensorFlowTestCase): # Internal control dep is preserved self.assertEqual(op.control_inputs, [c]) + @test_util.run_deprecated_v1 def testWhileLoopWithExternalControlDep(self): g = ops.Graph() with g.as_default(): @@ -946,6 +998,7 @@ class NameStackTest(test_util.TensorFlowTestCase): self.assertEqual("bar_2", g.unique_name("bar", mark_as_used=False)) self.assertEqual("bar_2", g.unique_name("bar")) + @test_util.run_deprecated_v1 def testNameAndVariableScope(self): with self.cached_session() as sess: with sess.graph.name_scope("l0"): @@ -1076,6 +1129,13 @@ class DeviceTest(test_util.TensorFlowTestCase): node { name: "FloatOutput" op: "FloatOutput" } """, gd) + def testEagerBackingDevice(self): + with context.eager_mode(): + with ops.device("/device:CPU:0"): + t = constant_op.constant(1.0) + self.assertRegexpMatches(t.device, "/device:CPU:0") + self.assertRegexpMatches(t.backing_device, "/device:CPU:0") + def testDevicePartialString(self): g = ops.Graph() with g.device("/job:worker/replica:2"): @@ -1665,6 +1725,7 @@ def _CopyOverrideGrad(op, x_grad): # pylint: disable=invalid-name class RegistrationTest(test_util.TensorFlowTestCase): + @test_util.run_deprecated_v1 def testRegisterGradients(self): x = test_ops.float_output() y = test_ops.copy_op(x) @@ -1704,6 +1765,7 @@ class ComparisonTest(test_util.TensorFlowTestCase): class ControlDependenciesTest(test_util.TensorFlowTestCase): + @test_util.run_deprecated_v1 def testBasic(self): g = ops.Graph() with g.as_default(): @@ -1947,6 +2009,7 @@ class OpScopeTest(test_util.TensorFlowTestCase): with ops.name_scope(None, "default2") as scope2: self.assertEqual(scope2, "default/default2/") + @test_util.run_deprecated_v1 def testNoScopeName(self): g0 = ops.Graph() values = [ @@ -1960,6 +2023,7 @@ class OpScopeTest(test_util.TensorFlowTestCase): with ops.name_scope(None, None, values): pass + @test_util.run_deprecated_v1 def testEmptyScopeName(self): g0 = ops.Graph() a = g0.create_op("A", [], [dtypes.float32]) @@ -1971,6 +2035,7 @@ class OpScopeTest(test_util.TensorFlowTestCase): self.assertEqual("", scope) self.assertEqual(g0, ops.get_default_graph()) + @test_util.run_deprecated_v1 def testDefaultScopeName(self): g0 = ops.Graph() a = g0.create_op("A", [], [dtypes.float32]) @@ -1995,12 +2060,14 @@ class OpScopeTest(test_util.TensorFlowTestCase): with ops.name_scope(scope_name, values=graph_elements + [a]): pass + @test_util.run_deprecated_v1 def testTensor(self): g0 = ops.Graph() a = g0.create_op("A", [], [dtypes.float32]) b = g0.create_op("B", [], [dtypes.float32]) self._testGraphElements([a, b]) + @test_util.run_deprecated_v1 def testSparseTensor(self): g0 = ops.Graph() a = g0.create_op("A", [], [dtypes.float32]) @@ -2011,6 +2078,7 @@ class OpScopeTest(test_util.TensorFlowTestCase): _apply_op(g0, "Int64Output", [], [dtypes.int64])) self._testGraphElements([a, sparse, b]) + @test_util.run_deprecated_v1 def testVariable(self): g0 = ops.Graph() with g0.as_default(): @@ -2215,6 +2283,7 @@ class InitScopeTest(test_util.TensorFlowTestCase): self.assertEqual(4, int(compiled_outer(inner=compiled_inner))) self.assertEqual(7, int(compiled_outer(inner=compiled_inner))) + @test_util.run_deprecated_v1 def testFallsBackToGlobalGraphWhenAllGraphsAreBuildingFunctions(self): with context.graph_mode(): ops.reset_default_graph() @@ -2351,6 +2420,7 @@ class GraphTest(test_util.TensorFlowTestCase): g.prevent_feeding(a) self.assertFalse(g.is_feedable(a)) + @test_util.run_deprecated_v1 def testPreventFetching(self): g = ops.Graph() a = constant_op.constant(2.0) @@ -2391,7 +2461,7 @@ class GraphTest(test_util.TensorFlowTestCase): c = math_ops.add(a, b) # Create a session we can delete with session.Session(graph=g) as sess: - sess.run(c) + self.evaluate(c) # Delete all references and trigger gc del g del a @@ -2407,7 +2477,7 @@ class GraphTest(test_util.TensorFlowTestCase): math_ops.add([1, 2], [1, 2, 3]) a = constant_op.constant(1) with session.Session() as sess: - sess.run(a) + self.evaluate(a) def testRunnableAfterInvalidShapeWithKernelLabelMap(self): g = ops.Graph() @@ -2417,7 +2487,7 @@ class GraphTest(test_util.TensorFlowTestCase): test_ops.kernel_label_required(1) a = constant_op.constant(1) with session.Session() as sess: - sess.run(a) + self.evaluate(a) class AttrScopeTest(test_util.TensorFlowTestCase): @@ -2434,10 +2504,12 @@ class AttrScopeTest(test_util.TensorFlowTestCase): b = None return (a, b) + @test_util.run_deprecated_v1 def testNoLabel(self): with self.cached_session(): self.assertAllEqual((None, None), self._get_test_attrs()) + @test_util.run_deprecated_v1 def testLabelMap(self): with self.cached_session() as sess: a1 = self._get_test_attrs() @@ -2472,11 +2544,13 @@ ops.RegisterShape("KernelLabel")(common_shapes.scalar_shape) class KernelLabelTest(test_util.TensorFlowTestCase): + @test_util.run_deprecated_v1 def testNoLabel(self): with self.cached_session(): self.assertAllEqual(b"My label is: default", test_ops.kernel_label().eval()) + @test_util.run_deprecated_v1 def testLabelMap(self): with self.cached_session() as sess: default_1 = test_ops.kernel_label() @@ -2491,12 +2565,14 @@ class KernelLabelTest(test_util.TensorFlowTestCase): # pylint: enable=protected-access default_3 = test_ops.kernel_label() - self.assertAllEqual(b"My label is: default", default_1.eval()) - self.assertAllEqual(b"My label is: default", default_2.eval()) - self.assertAllEqual(b"My label is: default", default_3.eval()) - self.assertAllEqual(b"My label is: overload_1", overload_1_1.eval()) - self.assertAllEqual(b"My label is: overload_1", overload_1_2.eval()) - self.assertAllEqual(b"My label is: overload_2", overload_2.eval()) + self.assertAllEqual(b"My label is: default", self.evaluate(default_1)) + self.assertAllEqual(b"My label is: default", self.evaluate(default_2)) + self.assertAllEqual(b"My label is: default", self.evaluate(default_3)) + self.assertAllEqual(b"My label is: overload_1", + self.evaluate(overload_1_1)) + self.assertAllEqual(b"My label is: overload_1", + self.evaluate(overload_1_2)) + self.assertAllEqual(b"My label is: overload_2", self.evaluate(overload_2)) class AsGraphDefTest(test_util.TensorFlowTestCase): @@ -2591,6 +2667,7 @@ class StatisticsTest(test_util.TensorFlowTestCase): class DeviceStackTest(test_util.TensorFlowTestCase): + @test_util.run_deprecated_v1 def testBasicDeviceAssignmentMetadata(self): def device_func(unused_op): @@ -2622,6 +2699,7 @@ class DeviceStackTest(test_util.TensorFlowTestCase): expected_regex = r"device_func<.*ops_test.py, [0-9]+" self.assertRegexpMatches(func_description, expected_regex) + @test_util.run_deprecated_v1 def testDeviceAssignmentMetadataForGraphDeviceAndTfDeviceFunctions(self): with ops.device("/cpu"): @@ -2641,6 +2719,7 @@ class DeviceStackTest(test_util.TensorFlowTestCase): class ColocationGroupTest(test_util.TensorFlowTestCase): + @test_util.run_deprecated_v1 def testBasic(self): a = constant_op.constant([2.0], name="a") with ops.colocate_with(a.op): @@ -2651,6 +2730,7 @@ class ColocationGroupTest(test_util.TensorFlowTestCase): with self.assertRaises(ValueError): c.op.get_attr("_class") + @test_util.run_deprecated_v1 def testBasicColocationMetadata(self): const_two = constant_op.constant([2.0], name="two") with ops.colocate_with(const_two.op): @@ -2663,6 +2743,7 @@ class ColocationGroupTest(test_util.TensorFlowTestCase): # colocation statement. self.assertEqual("ops_test.py", os.path.basename(metadata.filename)) + @test_util.run_deprecated_v1 def testColocationDeviceInteraction(self): with ops.device("/cpu:0"): with ops.device("/device:GPU:0"): @@ -2675,6 +2756,7 @@ class ColocationGroupTest(test_util.TensorFlowTestCase): self.assertEqual([b"loc:@a"], b.op.colocation_groups()) self.assertEqual(a.op.device, b.op.device) + @test_util.run_deprecated_v1 def testColocationCanonicalization(self): with ops.device("/device:GPU:0"): _ = constant_op.constant(2.0) @@ -2690,6 +2772,7 @@ class ColocationGroupTest(test_util.TensorFlowTestCase): # inherits B's device name, after canonicalizing the names. self.assertEqual(b.op.device, c.op.device) + @test_util.run_deprecated_v1 def testLocationOverrides(self): with ops.device("/cpu:0"): with ops.device("/device:GPU:0"): @@ -2711,6 +2794,7 @@ class ColocationGroupTest(test_util.TensorFlowTestCase): self.assertEqual("/device:GPU:0", c.op.device) self.assertEqual("/device:CPU:0", d.op.device) + @test_util.run_deprecated_v1 def testNestedColocateWith(self): a = constant_op.constant([2.0], name="a") with ops.colocate_with(a.op): @@ -2720,6 +2804,7 @@ class ColocationGroupTest(test_util.TensorFlowTestCase): self.assertEqual([b"loc:@a"], b.op.colocation_groups()) self.assertEqual([b"loc:@a"], c.op.colocation_groups()) + @test_util.run_deprecated_v1 def testMultiColocationGroups(self): a = constant_op.constant([2.0], name="a") b = constant_op.constant(3.0, name="b") @@ -2728,6 +2813,7 @@ class ColocationGroupTest(test_util.TensorFlowTestCase): c = constant_op.constant(4.0) self.assertEqual(set([b"loc:@a", b"loc:@b"]), set(c.op.colocation_groups())) + @test_util.run_deprecated_v1 def testColocationIgnoreStack(self): a = constant_op.constant([2.0], name="a") b = constant_op.constant(3.0, name="b") @@ -2736,6 +2822,7 @@ class ColocationGroupTest(test_util.TensorFlowTestCase): c = constant_op.constant(4.0) self.assertEqual(set([b"loc:@b"]), set(c.op.colocation_groups())) + @test_util.run_deprecated_v1 def testColocateWithReset(self): a = constant_op.constant([2.0], name="a") with ops.colocate_with(a.op): @@ -2745,6 +2832,7 @@ class ColocationGroupTest(test_util.TensorFlowTestCase): self.assertEqual([b"loc:@a"], b.op.colocation_groups()) self.assertEqual([b"loc:@c"], c.op.colocation_groups()) + @test_util.run_deprecated_v1 def testColocateWithInitialNoneThenNested(self): a = constant_op.constant([2.0], name="a") with ops.colocate_with(a.op): @@ -2755,47 +2843,13 @@ class ColocationGroupTest(test_util.TensorFlowTestCase): self.assertEqual([b"loc:@b"], b.op.colocation_groups()) self.assertEqual([b"loc:@b"], c.op.colocation_groups()) + @test_util.run_deprecated_v1 def testColocateVariables(self): a = variables.Variable([2.0], name="a") with ops.colocate_with(a.op): b = variables.Variable([3.0], name="b") self.assertEqual([b"loc:@a"], b.op.colocation_groups()) - def testInconsistentDeviceWithinColocate(self): - with ops.device("/device:GPU:0"): - a = constant_op.constant([2.0], name="a") - with ops.colocate_with(a.op): - # This is allowed due to legacy but clearly wrong, since we - # should really be colocating with 'a'. We allow devices to - # override colocate_with, but we log warnings to suggest that - # this is probably unintentional or misguided. - with ops.device("/cpu:0"): - b = constant_op.constant([3.0], name="b") - - self.assertEqual("/device:CPU:0", b.device) - - def testMakeColocationConflictMessage(self): - """Test that provides an example of a complicated error message.""" - # We could test the message with any ops, but this test will be more - # instructive with a real colocation conflict. - with ops.device("/device:GPU:0"): - a = constant_op.constant([2.0], name="a") - with ops.colocate_with(a.op): - with ops.device("/cpu:0"): - b = constant_op.constant([3.0], name="b") - # The definition-location of the nodes will be wrong because of running - # from within a TF unittest. The rest of the info should be correct. - message = ops.get_default_graph()._make_colocation_conflict_message(a.op, - b.op) - self.assertRegexpMatches(message, - r"Tried to colocate op 'a' \(defined at.*\)") - self.assertRegexpMatches(message, "No node-device.*'a'") - self.assertRegexpMatches(message, "Device assignments active.*'a'") - self.assertRegexpMatches(message, "GPU:0") - self.assertRegexpMatches(message, "Node-device colocations active.*'b'") - self.assertRegexpMatches(message, "Device assignments active.*'b'") - self.assertRegexpMatches(message, "cpu:0") - class DeprecatedTest(test_util.TensorFlowTestCase): @@ -2918,6 +2972,7 @@ class NameScopeTest(test_util.TensorFlowTestCase): class TracebackTest(test_util.TensorFlowTestCase): + @test_util.run_deprecated_v1 def testTracebackWithStartLines(self): with self.cached_session() as sess: a = constant_op.constant(2.0) @@ -2939,6 +2994,7 @@ class TracebackTest(test_util.TensorFlowTestCase): class EnableEagerExecutionTest(test_util.TensorFlowTestCase): + @test_util.run_deprecated_v1 def testBadArgumentsToEnableEagerExecution(self): with self.assertRaisesRegexp(TypeError, "config must be a tf.ConfigProto"): ops.enable_eager_execution(context.DEVICE_PLACEMENT_SILENT) diff --git a/tensorflow/python/framework/python_op_gen.cc b/tensorflow/python/framework/python_op_gen.cc index 465016b8087..d91f7b0bdde 100644 --- a/tensorflow/python/framework/python_op_gen.cc +++ b/tensorflow/python/framework/python_op_gen.cc @@ -142,6 +142,7 @@ class GenEagerPythonOp : public python_op_gen_internal::GenPythonOp { void AddEagerAttrs(const string& indentation); void AddEagerExecute(const string& indentation, const string& num_outputs_expr); + void AddDispatch(const string& prefix); void AddAttrForArg(const string& attr, int arg_index) { gtl::InsertIfNotPresent(&inferred_attrs_, attr, @@ -356,9 +357,14 @@ string GenEagerPythonOp::Code() { void GenEagerPythonOp::HandleGraphMode(const string& function_setup) { strings::StrAppend(&result_, " # Add nodes to the TensorFlow graph.\n"); - strings::StrAppend(&result_, function_setup, - " _, _, _op = _op_def_lib._apply_op_helper(\n"); - AddBodyNoReturn(" "); + strings::StrAppend(&result_, function_setup); + if (api_def_.visibility() == ApiDef::VISIBLE) { + strings::StrAppend(&result_, " try:\n "); + } + strings::StrAppend(&result_, " _, _, _op = _op_def_lib._apply_op_helper(\n"); + AddBodyNoReturn(strings::StrCat(" \"", op_def_.name(), "\", ")); + AddDispatch(" "); + if (num_outs_ > 0) { strings::StrAppend(&result_, " _result = _op.outputs[:]\n"); // Special case handling for stateful op with single list output @@ -628,6 +634,7 @@ void GenEagerPythonOp::AddEagerFunctionTeardown( bool GenEagerPythonOp::AddEagerFastPathAndGraphCode( const string& parameters, const std::vector& output_sizes, const string& eager_not_allowed_error) { + strings::StrAppend(&result_, "@_dispatch.add_dispatch_list\n"); AddExport(); AddDefLine(function_name_, parameters); AddDocStringDescription(); @@ -758,6 +765,7 @@ void GenEagerPythonOp::AddEagerFastPathExecute() { strings::StrAppend(&result_, " except _core._SymbolicException:\n"); strings::StrAppend(&result_, " pass # Add nodes to the TensorFlow graph.\n"); + AddDispatch(" "); // Any errors thrown from execute need to be unwrapped from // _NotOkStatusException. @@ -898,6 +906,19 @@ void GenEagerPythonOp::AddEagerExecute(const string& indentation, WordWrap(return_prefix, return_args, kRightMargin), "\n"); } +void GenEagerPythonOp::AddDispatch(const string& prefix) { + if (api_def_.visibility() != ApiDef::VISIBLE) return; + + strings::StrAppend(&result_, prefix, "except (TypeError, ValueError):\n"); + strings::StrAppend(&result_, prefix, " result = _dispatch.dispatch(\n"); + AddBodyNoReturn(strings::StrCat(prefix, " ", function_name_, ", ")); + strings::StrAppend(&result_, prefix, + " if result is not " + "_dispatch.OpDispatcher.NOT_SUPPORTED:\n"); + strings::StrAppend(&result_, prefix, " return result\n"); + strings::StrAppend(&result_, prefix, " raise\n"); +} + string GetPythonOps(const OpList& ops, const ApiDefMap& api_defs, const std::vector& hidden_ops, bool require_shapes, const string& source_file_name = "") { @@ -937,6 +958,7 @@ from tensorflow.python.framework import op_def_registry as _op_def_registry from tensorflow.python.framework import ops as _ops from tensorflow.python.framework import op_def_library as _op_def_library from tensorflow.python.util.deprecation import deprecated_endpoints +from tensorflow.python.util import dispatch as _dispatch from tensorflow.python.util.tf_export import tf_export )"); diff --git a/tensorflow/python/framework/python_op_gen_internal.cc b/tensorflow/python/framework/python_op_gen_internal.cc index 65b9ad5c6a2..cbdeecfbfb9 100644 --- a/tensorflow/python/framework/python_op_gen_internal.cc +++ b/tensorflow/python/framework/python_op_gen_internal.cc @@ -804,8 +804,8 @@ void GenPythonOp::AddDocStringOutputs() { } void GenPythonOp::AddBody(const string& prefix) { - const string apply_prefix = - strings::StrCat(prefix, "_result = _op_def_lib.apply_op("); + const string apply_prefix = strings::StrCat( + prefix, "_result = _op_def_lib.apply_op(\"", op_def_.name(), "\", "); AddBodyNoReturn(apply_prefix); if (num_outs_ > 1) { strings::StrAppend(&result_, prefix, "_result = _", op_def_.name(), @@ -815,7 +815,7 @@ void GenPythonOp::AddBody(const string& prefix) { } void GenPythonOp::AddBodyNoReturn(const string& apply_prefix) { - string args = strings::StrCat("\"", op_def_.name(), "\", "); + string args; for (size_t i = 0; i < param_names_.size(); ++i) { strings::StrAppend(&args, AvoidPythonReserved(param_names_[i].GetName()), "=", param_names_[i].GetRenameTo(), ", "); diff --git a/tensorflow/python/framework/random_seed.py b/tensorflow/python/framework/random_seed.py index 777bb2fe8c5..6b7f56a92cc 100644 --- a/tensorflow/python/framework/random_seed.py +++ b/tensorflow/python/framework/random_seed.py @@ -34,7 +34,7 @@ def _truncate_seed(seed): return seed % _MAXINT32 # Truncate to fit into 32-bit integer -@tf_export('random.get_seed', v1=['random.get_seed', 'get_seed']) +@tf_export(v1=['random.get_seed', 'get_seed']) @deprecation.deprecated_endpoints('get_seed') def get_seed(op_seed): """Returns the local seeds an operation should use given an op-specific seed. @@ -45,7 +45,7 @@ def get_seed(op_seed): graph, or for only specific operations. For details on how the graph-level seed interacts with op seeds, see - `tf.set_random_seed`. + `tf.random.set_random_seed`. Args: op_seed: integer. @@ -82,7 +82,7 @@ def get_seed(op_seed): return seeds -@tf_export('random.set_random_seed', 'set_random_seed') +@tf_export(v1=['random.set_random_seed', 'set_random_seed']) def set_random_seed(seed): """Sets the graph-level random seed. @@ -154,7 +154,7 @@ def set_random_seed(seed): sessions, set a graph-level seed: ```python - tf.set_random_seed(1234) + tf.random.set_random_seed(1234) a = tf.random_uniform([1]) b = tf.random_normal([1]) @@ -182,3 +182,103 @@ def set_random_seed(seed): context.set_global_seed(seed) else: ops.get_default_graph().seed = seed + + +@tf_export('random.set_seed', v1=[]) +def set_seed(seed): + """Sets the graph-level random seed. + + Operations that rely on a random seed actually derive it from two seeds: + the graph-level and operation-level seeds. This sets the graph-level seed. + + Its interactions with operation-level seeds is as follows: + + 1. If neither the graph-level nor the operation seed is set: + A random seed is used for this op. + 2. If the graph-level seed is set, but the operation seed is not: + The system deterministically picks an operation seed in conjunction + with the graph-level seed so that it gets a unique random sequence. + 3. If the graph-level seed is not set, but the operation seed is set: + A default graph-level seed and the specified operation seed are used to + determine the random sequence. + 4. If both the graph-level and the operation seed are set: + Both seeds are used in conjunction to determine the random sequence. + + To illustrate the user-visible effects, consider these examples: + + To generate different sequences across sessions, set neither + graph-level nor op-level seeds: + + ```python + a = tf.random_uniform([1]) + b = tf.random_normal([1]) + + print("Session 1") + with tf.Session() as sess1: + print(sess1.run(a)) # generates 'A1' + print(sess1.run(a)) # generates 'A2' + print(sess1.run(b)) # generates 'B1' + print(sess1.run(b)) # generates 'B2' + + print("Session 2") + with tf.Session() as sess2: + print(sess2.run(a)) # generates 'A3' + print(sess2.run(a)) # generates 'A4' + print(sess2.run(b)) # generates 'B3' + print(sess2.run(b)) # generates 'B4' + ``` + + To generate the same repeatable sequence for an op across sessions, set the + seed for the op: + + ```python + a = tf.random_uniform([1], seed=1) + b = tf.random_normal([1]) + + # Repeatedly running this block with the same graph will generate the same + # sequence of values for 'a', but different sequences of values for 'b'. + print("Session 1") + with tf.Session() as sess1: + print(sess1.run(a)) # generates 'A1' + print(sess1.run(a)) # generates 'A2' + print(sess1.run(b)) # generates 'B1' + print(sess1.run(b)) # generates 'B2' + + print("Session 2") + with tf.Session() as sess2: + print(sess2.run(a)) # generates 'A1' + print(sess2.run(a)) # generates 'A2' + print(sess2.run(b)) # generates 'B3' + print(sess2.run(b)) # generates 'B4' + ``` + + To make the random sequences generated by all ops be repeatable across + sessions, set a graph-level seed: + + ```python + tf.random.set_seed(1234) + a = tf.random_uniform([1]) + b = tf.random_normal([1]) + + # Repeatedly running this block with the same graph will generate the same + # sequences of 'a' and 'b'. + print("Session 1") + with tf.Session() as sess1: + print(sess1.run(a)) # generates 'A1' + print(sess1.run(a)) # generates 'A2' + print(sess1.run(b)) # generates 'B1' + print(sess1.run(b)) # generates 'B2' + + print("Session 2") + with tf.Session() as sess2: + print(sess2.run(a)) # generates 'A1' + print(sess2.run(a)) # generates 'A2' + print(sess2.run(b)) # generates 'B1' + print(sess2.run(b)) # generates 'B2' + ``` + + Args: + seed: integer. + """ + # TODO(go/tf2-random): change doc, update to match design doc + set_random_seed(seed) diff --git a/tensorflow/python/framework/registry.py b/tensorflow/python/framework/registry.py index 2e45acb4995..4357c76bd6c 100644 --- a/tensorflow/python/framework/registry.py +++ b/tensorflow/python/framework/registry.py @@ -23,10 +23,9 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function -import traceback - from tensorflow.python.platform import tf_logging as logging from tensorflow.python.util import compat +from tensorflow.python.util import tf_stack # Registry mechanism below is based on mapreduce.python.mrpython.Register. @@ -57,15 +56,17 @@ class Registry(object): if name in self._registry: (filename, line_number, function_name, _) = ( self._registry[name][_LOCATION_TAG]) - raise KeyError("Registering two %s with name '%s' !" + raise KeyError("Registering two %s with name '%s'! " "(Previous registration was in %s %s:%d)" % (self._name, name, function_name, filename, line_number)) logging.vlog(1, "Registering %s (%s) in %s.", name, candidate, self._name) # stack trace is [this_function, Register(), user_function,...] # so the user function is #2. - stack = traceback.extract_stack() - self._registry[name] = {_TYPE_TAG: candidate, _LOCATION_TAG: stack[2]} + stack = tf_stack.extract_stack() + user_function = stack[2] + location_tag = tf_stack.convert_stack([user_function])[0] + self._registry[name] = {_TYPE_TAG: candidate, _LOCATION_TAG: location_tag} def list(self): """Lists registered items. diff --git a/tensorflow/python/framework/registry_test.py b/tensorflow/python/framework/registry_test.py index a821e16f260..1a0d3f200d9 100644 --- a/tensorflow/python/framework/registry_test.py +++ b/tensorflow/python/framework/registry_test.py @@ -45,7 +45,9 @@ class RegistryTest(test.TestCase): def testDuplicate(self): myreg = registry.Registry('testbar') myreg.register(bar, 'Bar') - with self.assertRaises(KeyError): + with self.assertRaisesRegexp( + KeyError, r'Registering two testbar with name \'Bar\'! ' + r'\(Previous registration was in [^ ]+ .*.py:[0-9]+\)'): myreg.register(bar, 'Bar') diff --git a/tensorflow/python/framework/smart_cond_test.py b/tensorflow/python/framework/smart_cond_test.py index b8a9672b06d..f964c87f024 100644 --- a/tensorflow/python/framework/smart_cond_test.py +++ b/tensorflow/python/framework/smart_cond_test.py @@ -35,6 +35,7 @@ def raise_exception(): class SmartCondTest(test_util.TensorFlowTestCase): + @test_util.run_deprecated_v1 def testTrue(self): with ops.Graph().as_default(): with session.Session(): @@ -44,6 +45,7 @@ class SmartCondTest(test_util.TensorFlowTestCase): lambda: math_ops.multiply(y, 5)) self.assertEqual(z.eval(), 32) + @test_util.run_deprecated_v1 def testFalse(self): with ops.Graph().as_default(): with session.Session(): @@ -99,6 +101,7 @@ class SmartCondTest(test_util.TensorFlowTestCase): class SmartCaseTest(test_util.TensorFlowTestCase): + @test_util.run_deprecated_v1 def testTrue(self): x = array_ops.placeholder(dtype=dtypes.int32, shape=[]) conditions = [(True, lambda: constant_op.constant(1)), @@ -109,9 +112,10 @@ class SmartCaseTest(test_util.TensorFlowTestCase): exclusive=True) with session.Session() as sess: # No feed_dict necessary - self.assertEqual(sess.run(y), 1) - self.assertEqual(sess.run(z), 1) + self.assertEqual(self.evaluate(y), 1) + self.assertEqual(self.evaluate(z), 1) + @test_util.run_deprecated_v1 def testFalse(self): conditions = [(False, raise_exception)] y = smart_cond.smart_case(conditions, @@ -121,9 +125,10 @@ class SmartCaseTest(test_util.TensorFlowTestCase): default=lambda: constant_op.constant(1), exclusive=True) with session.Session() as sess: - self.assertEqual(sess.run(y), 1) - self.assertEqual(sess.run(z), 1) + self.assertEqual(self.evaluate(y), 1) + self.assertEqual(self.evaluate(z), 1) + @test_util.run_deprecated_v1 def testMix(self): x = array_ops.placeholder(dtype=dtypes.int32, shape=[]) y = constant_op.constant(10) diff --git a/tensorflow/python/framework/sparse_tensor_test.py b/tensorflow/python/framework/sparse_tensor_test.py index 22423c4f58c..a999c12ca89 100644 --- a/tensorflow/python/framework/sparse_tensor_test.py +++ b/tensorflow/python/framework/sparse_tensor_test.py @@ -46,11 +46,11 @@ class SparseTensorTest(test_util.TensorFlowTestCase): self.assertEqual(sp.get_shape(), (4, 5)) with self.cached_session() as sess: - value = sp.eval() + value = self.evaluate(sp) self.assertAllEqual(indices, value.indices) self.assertAllEqual(values, value.values) self.assertAllEqual(shape, value.dense_shape) - sess_run_value = sess.run(sp) + sess_run_value = self.evaluate(sp) self.assertAllEqual(sess_run_value.indices, value.indices) self.assertAllEqual(sess_run_value.values, value.values) self.assertAllEqual(sess_run_value.dense_shape, value.dense_shape) @@ -65,6 +65,7 @@ class SparseTensorTest(test_util.TensorFlowTestCase): sparse_tensor.is_sparse( sparse_tensor.SparseTensorValue([[0]], [0], [1]))) + @test_util.run_deprecated_v1 def testConsumers(self): sp = sparse_tensor.SparseTensor([[0, 0], [1, 2]], [1.0, 3.0], [3, 4]) w = ops.convert_to_tensor(np.ones([4, 1], np.float32)) @@ -85,8 +86,9 @@ class ConvertToTensorOrSparseTensorTest(test_util.TensorFlowTestCase): value = [42, 43] from_value = sparse_tensor.convert_to_tensor_or_sparse_tensor( value) - self.assertAllEqual(value, from_value.eval()) + self.assertAllEqual(value, self.evaluate(from_value)) + @test_util.run_deprecated_v1 def test_convert_sparse(self): with self.cached_session(): indices = [[0, 1], [1, 0]] diff --git a/tensorflow/python/framework/subscribe_test.py b/tensorflow/python/framework/subscribe_test.py index cab426844d4..61c6ea65190 100644 --- a/tensorflow/python/framework/subscribe_test.py +++ b/tensorflow/python/framework/subscribe_test.py @@ -43,6 +43,7 @@ class SubscribeTest(test_util.TensorFlowTestCase): self.assertTrue( all(subscribe._is_subscribed_identity(x) for x in container)) + @test_util.run_deprecated_v1 def testSideEffect(self): a = constant_op.constant(1) b = constant_op.constant(1) @@ -66,15 +67,16 @@ class SubscribeTest(test_util.TensorFlowTestCase): self.assertTrue(c.op in d.op.control_inputs) with self.cached_session() as sess: - c_out = sess.run([c]) - n_out = sess.run([n]) - d_out = sess.run([d]) + c_out = self.evaluate([c]) + n_out = self.evaluate([n]) + d_out = self.evaluate([d]) self.assertEqual(n_out, [-2]) self.assertEqual(c_out, [2]) self.assertEqual(d_out, [42]) self.assertEqual(shared, [2, 2, 2]) + @test_util.run_deprecated_v1 def testSupportedTypes(self): """Confirm that supported types are correctly detected and handled.""" @@ -120,6 +122,7 @@ class SubscribeTest(test_util.TensorFlowTestCase): subscribe.subscribe(c.name, lambda t: script_ops.py_func(sub, [t], [t.dtype])) + @test_util.run_deprecated_v1 def testCaching(self): """Confirm caching of control output is recalculated between calls.""" a = constant_op.constant(1) @@ -145,13 +148,14 @@ class SubscribeTest(test_util.TensorFlowTestCase): lambda t: script_ops.py_func(sub, [t], [t.dtype])) with self.cached_session() as sess: - c_out = sess.run([c]) - d_out = sess.run([d]) + c_out = self.evaluate([c]) + d_out = self.evaluate([d]) self.assertEqual(c_out, [42]) self.assertEqual(d_out, [11]) self.assertEqual(shared, {2: 1, 1: 1}) + @test_util.run_deprecated_v1 def testIsSubscribedIdentity(self): """Confirm subscribed identity ops are correctly detected.""" a = constant_op.constant(1) @@ -165,6 +169,7 @@ class SubscribeTest(test_util.TensorFlowTestCase): self.assertFalse(subscribe._is_subscribed_identity(idop)) self.assertTrue(subscribe._is_subscribed_identity(c_sub)) + @test_util.run_deprecated_v1 def testSubscribeExtend(self): """Confirm side effect are correctly added for different input types.""" a = constant_op.constant(1) @@ -205,11 +210,12 @@ class SubscribeTest(test_util.TensorFlowTestCase): # Expect the three side effect graphs to have been evaluated. with self.cached_session() as sess: - sess.run([c_sub]) + self.evaluate([c_sub]) self.assertIn('graph1', shared) self.assertIn('graph2', shared) self.assertIn('graph3', shared) + @test_util.run_deprecated_v1 def testSubscribeVariable(self): """Confirm that variables can be subscribed.""" v1 = variables.VariableV1(0.0) @@ -229,25 +235,26 @@ class SubscribeTest(test_util.TensorFlowTestCase): with self.cached_session() as sess: # Initialize the variables first. - sess.run([v1.initializer]) - sess.run([v2.initializer]) + self.evaluate([v1.initializer]) + self.evaluate([v2.initializer]) # Expect the side effects to be triggered when evaluating the add op as # it will read the value of the variable. - sess.run([add]) + self.evaluate([add]) self.assertEqual(1, len(shared)) # Expect the side effect not to be triggered when evaluating the assign # op as it will not access the 'read' output of the variable. - sess.run([assign_v1]) + self.evaluate([assign_v1]) self.assertEqual(1, len(shared)) - sess.run([add]) + self.evaluate([add]) self.assertEqual(2, len(shared)) # Make sure the values read from the variable match the expected ones. self.assertEqual([0.0, 3.0], shared) + @test_util.run_deprecated_v1 def testResourceType(self): """Confirm that subscribe correctly handles tensors with 'resource' type.""" tensor_array = tensor_array_ops.TensorArray( @@ -273,9 +280,10 @@ class SubscribeTest(test_util.TensorFlowTestCase): self.assertFalse(subscribe._is_subscribed_identity(tensor_array.handle)) with self.cached_session() as sess: - sess.run([reader]) + self.evaluate([reader]) self.assertEqual(0, len(shared)) + @test_util.run_deprecated_v1 def testMultipleOutputs(self): """Handle subscriptions to multiple outputs from the same op.""" sparse_tensor_1 = sparse_tensor.SparseTensor( @@ -304,11 +312,12 @@ class SubscribeTest(test_util.TensorFlowTestCase): lambda t: script_ops.py_func(sub, [t], [t.dtype])) with self.cached_session() as sess: - sess.run([neg]) + self.evaluate([neg]) # All three ops have been processed. self.assertEqual(3, len(shared)) + @test_util.run_deprecated_v1 def test_subscribe_tensors_on_different_devices(self): """Side effect ops are added with the same device of the subscribed op.""" c1 = constant_op.constant(10) @@ -335,6 +344,7 @@ class SubscribeTest(test_util.TensorFlowTestCase): self.assertEqual(add.device, add_sub.device) self.assertEqual(mul.device, mul_sub.device) + @test_util.run_deprecated_v1 def test_subscribe_tensors_within_control_flow_context(self): """Side effect ops are added with the same control flow context.""" c1 = constant_op.constant(10) @@ -375,7 +385,7 @@ class SubscribeTest(test_util.TensorFlowTestCase): self.assertIsNot(context(subscriptions[0]), context(subscriptions[1])) with self.cached_session() as sess: - sess.run(cond) + self.evaluate(cond) self.assertEqual(3, len(results)) diff --git a/tensorflow/python/framework/tensor_shape.py b/tensorflow/python/framework/tensor_shape.py index 5a58d271488..960a3dad738 100644 --- a/tensorflow/python/framework/tensor_shape.py +++ b/tensorflow/python/framework/tensor_shape.py @@ -169,7 +169,7 @@ def dimension_at_index(shape, index): return shape.dims[index] -@tf_export("Dimension") +@tf_export(v1=["Dimension"]) class Dimension(object): """Represents the value of one dimension in a TensorShape.""" diff --git a/tensorflow/python/framework/tensor_spec.py b/tensorflow/python/framework/tensor_spec.py index fbea930fe0e..c44636edc4e 100644 --- a/tensorflow/python/framework/tensor_spec.py +++ b/tensorflow/python/framework/tensor_spec.py @@ -24,14 +24,15 @@ from tensorflow.python.framework import common_shapes from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops from tensorflow.python.framework import tensor_shape +from tensorflow.python.util.tf_export import tf_export +@tf_export("TensorSpec") class TensorSpec(object): """Describes a tf.Tensor. - A TensorSpec allows an API to describe the Tensors that it accepts or - returns, before that Tensor exists. This allows dynamic and flexible graph - construction and configuration. + Metadata for describing the `tf.Tensor` objects accepted or returned + by some TensorFlow APIs. """ __slots__ = ["_shape", "_shape_tuple", "_dtype", "_name"] @@ -69,11 +70,6 @@ class TensorSpec(object): else: raise ValueError("`tensor` should be a tf.Tensor") - @classmethod - def is_bounded(cls): - del cls - return False - @property def shape(self): """Returns the `TensorShape` that represents the shape of the tensor.""" @@ -86,21 +82,21 @@ class TensorSpec(object): @property def name(self): - """Returns the name of the described tensor.""" + """Returns the (optionally provided) name of the described tensor.""" return self._name - @property - def is_discrete(self): - """Whether spec is discrete.""" - return self.dtype.is_integer - - @property - def is_continuous(self): - """Whether spec is continuous.""" - return self.dtype.is_floating - def is_compatible_with(self, spec_or_tensor): - """True if the shape and dtype of `spec_or_tensor` are compatible.""" + """Returns True if spec_or_tensor is compatible with this TensorSpec. + + Two tensors are considered compatible if they have the same dtype + and their shapes are compatible (see `tf.TensorShape.is_compatible_with`). + + Args: + spec_or_tensor: A tf.TensorSpec or a tf.Tensor + + Returns: + True if spec_or_tensor is compatible with self. + """ return (self._dtype.is_compatible_with(spec_or_tensor.dtype) and self._shape.is_compatible_with(spec_or_tensor.shape)) @@ -188,11 +184,6 @@ class BoundedTensorSpec(TensorSpec): self._maximum = np.array(maximum, dtype=self.dtype.as_numpy_dtype()) self._maximum.setflags(write=False) - @classmethod - def is_bounded(cls): - del cls - return True - @classmethod def from_spec(cls, spec): dtype = dtypes.as_dtype(spec.dtype) @@ -223,4 +214,3 @@ class BoundedTensorSpec(TensorSpec): def __reduce__(self): return BoundedTensorSpec, (self._shape, self._dtype, self._minimum, self._maximum, self._name) - diff --git a/tensorflow/python/framework/tensor_spec_test.py b/tensorflow/python/framework/tensor_spec_test.py index 40611e5f840..75c197df09e 100644 --- a/tensorflow/python/framework/tensor_spec_test.py +++ b/tensorflow/python/framework/tensor_spec_test.py @@ -45,6 +45,7 @@ class TensorSpecTest(test_util.TensorFlowTestCase): desc = tensor_spec.TensorSpec(shape=None, dtype=dtypes.float32) self.assertEqual(desc.shape, tensor_shape.TensorShape(None)) + @test_util.run_deprecated_v1 def testShapeCompatibility(self): unknown = array_ops.placeholder(dtypes.int64) partial = array_ops.placeholder(dtypes.int64, shape=[None, 1]) @@ -75,6 +76,7 @@ class TensorSpecTest(test_util.TensorFlowTestCase): self.assertFalse(desc_rank3.is_compatible_with(full)) self.assertTrue(desc_rank3.is_compatible_with(rank3)) + @test_util.run_deprecated_v1 def testTypeCompatibility(self): floats = array_ops.placeholder(dtypes.float32, shape=[10, 10]) ints = array_ops.placeholder(dtypes.int32, shape=[10, 10]) @@ -106,6 +108,7 @@ class TensorSpecTest(test_util.TensorFlowTestCase): spec_2 = tensor_spec.TensorSpec.from_spec(spec_1) self.assertEqual(spec_1, spec_2) + @test_util.run_deprecated_v1 def testFromTensor(self): zero = constant_op.constant(0) spec = tensor_spec.TensorSpec.from_tensor(zero) @@ -113,6 +116,7 @@ class TensorSpecTest(test_util.TensorFlowTestCase): self.assertEqual(spec.shape, []) self.assertEqual(spec.name, "Const") + @test_util.run_deprecated_v1 def testFromPlaceholder(self): unknown = array_ops.placeholder(dtypes.int64, name="unknown") partial = array_ops.placeholder(dtypes.float32, @@ -134,22 +138,6 @@ class TensorSpecTest(test_util.TensorFlowTestCase): self.assertEqual(bounded_spec.dtype, spec.dtype) self.assertEqual(bounded_spec.name, spec.name) - def testIsDiscrete(self): - discrete_spec = tensor_spec.TensorSpec((1, 2), dtypes.int32) - continuous_spec = tensor_spec.TensorSpec((1, 2), dtypes.float32) - self.assertTrue(discrete_spec.is_discrete) - self.assertFalse(continuous_spec.is_discrete) - - def testIsContinuous(self): - discrete_spec = tensor_spec.TensorSpec((1, 2), dtypes.int32) - continuous_spec = tensor_spec.TensorSpec((1, 2), dtypes.float32) - self.assertFalse(discrete_spec.is_continuous) - self.assertTrue(continuous_spec.is_continuous) - - def testIsBounded(self): - unbounded_spec = tensor_spec.TensorSpec((1, 2), dtypes.int32) - self.assertFalse(unbounded_spec.is_bounded()) - def testSerialization(self): desc = tensor_spec.TensorSpec([1, 5], dtypes.float32, "test") self.assertEqual(pickle.loads(pickle.dumps(desc)), desc) @@ -165,11 +153,6 @@ class BoundedTensorSpecTest(test_util.TensorFlowTestCase): with self.assertRaisesRegexp(ValueError, "not compatible"): tensor_spec.BoundedTensorSpec((3, 5), dtypes.uint8, 0, (1, 1, 1)) - def testIsBounded(self): - bounded_spec = tensor_spec.BoundedTensorSpec( - (1, 2), dtypes.int32, minimum=0, maximum=1) - self.assertTrue(bounded_spec.is_bounded()) - def testMinimumMaximumAttributes(self): spec = tensor_spec.BoundedTensorSpec( (1, 2, 3), dtypes.float32, 0, (5, 5, 5)) diff --git a/tensorflow/python/framework/tensor_util.py b/tensorflow/python/framework/tensor_util.py index 9db94f5288c..f98f301b38a 100644 --- a/tensorflow/python/framework/tensor_util.py +++ b/tensorflow/python/framework/tensor_util.py @@ -371,8 +371,10 @@ def _AssertCompatible(values, dtype): (dtype.name, repr(mismatch), type(mismatch).__name__)) +# pylint: disable=invalid-name @tf_export(v1=["make_tensor_proto"]) -def make_tensor_proto(values, dtype=None, shape=None, verify_shape=False): +def make_tensor_proto(values, dtype=None, shape=None, verify_shape=False, + allow_broadcast=False): """Create a TensorProto. Args: @@ -380,6 +382,8 @@ def make_tensor_proto(values, dtype=None, shape=None, verify_shape=False): dtype: Optional tensor_pb2 DataType value. shape: List of integers representing the dimensions of tensor. verify_shape: Boolean that enables verification of a shape of values. + allow_broadcast:Boolean that enables allowing scalars and 1 length vector + broadcasting. Cannot be true when verify_shape is true. Returns: A `TensorProto`. Depending on the type, it may contain data in the @@ -416,6 +420,8 @@ def make_tensor_proto(values, dtype=None, shape=None, verify_shape=False): can not have more elements than what "shape" specifies. """ + if allow_broadcast and verify_shape: + raise ValueError("allow_broadcast and verify_shape are not both allowed.") if isinstance(values, tensor_pb2.TensorProto): return values @@ -504,15 +510,22 @@ def make_tensor_proto(values, dtype=None, shape=None, verify_shape=False): shape_size = np.prod(shape, dtype=np.int64) is_same_size = shape_size == nparray.size - if verify_shape: - if not nparray.shape == tuple(shape): + if allow_broadcast: + if nparray.shape == (1,) or nparray.shape == tuple(): + pass + elif nparray.size != shape_size: raise TypeError("Expected Tensor's shape: %s, got %s." % (tuple(shape), nparray.shape)) - if nparray.size > shape_size: - raise ValueError( - "Too many elements provided. Needed at most %d, but received %d" % - (shape_size, nparray.size)) + else: + if verify_shape and nparray.shape != tuple(shape): + raise TypeError("Expected Tensor's shape: %s, got %s." % + (tuple(shape), nparray.shape)) + + if nparray.size > shape_size: + raise ValueError( + "Too many elements provided. Needed at most %d, but received %d" % + (shape_size, nparray.size)) tensor_proto = tensor_pb2.TensorProto( dtype=numpy_dtype.as_datatype_enum, @@ -560,6 +573,7 @@ def make_tensor_proto(values, dtype=None, shape=None, verify_shape=False): append_fn(tensor_proto, proto_values) return tensor_proto +# pylint: enable=invalid-name @tf_export("make_ndarray") diff --git a/tensorflow/python/framework/tensor_util_test.py b/tensorflow/python/framework/tensor_util_test.py index bdf759f2204..00337546186 100644 --- a/tensorflow/python/framework/tensor_util_test.py +++ b/tensorflow/python/framework/tensor_util_test.py @@ -758,6 +758,7 @@ class TensorUtilTest(test.TestCase): self.assertFalse(tensor_util.ShapeEquals(t, [1, 4])) self.assertFalse(tensor_util.ShapeEquals(t, [4])) + @test_util.run_deprecated_v1 def testMockArray(self): class MockArray(object): @@ -771,7 +772,7 @@ class TensorUtilTest(test.TestCase): with self.cached_session() as sess: ma = MockArray(np.array([10, 20, 30])) t = ops.convert_to_tensor(ma) - a = sess.run(t) + a = self.evaluate(t) self.assertEquals(np.int64, a.dtype) self.assertAllClose(np.array([10, 20, 30], dtype=np.int64), a) @@ -787,6 +788,7 @@ class ConstantValueTest(test.TestCase): tf_val = constant_op.constant(np_val) self.assertAllClose(np_val, tensor_util.constant_value(tf_val)) + @test_util.run_deprecated_v1 def testUnknown(self): tf_val = gen_state_ops.variable( shape=[3, 4, 7], @@ -815,12 +817,14 @@ class ConstantValueTest(test.TestCase): c_val = tensor_util.constant_value(tf_val) self.assertEqual(6, c_val) + @test_util.run_deprecated_v1 def testSizeOfScalar(self): tf_val = array_ops.size(constant_op.constant(0.0)) c_val = tensor_util.constant_value(tf_val) self.assertEqual(1, c_val) self.assertEqual(np.ndarray, type(c_val)) + @test_util.run_deprecated_v1 def testRank(self): tf_val = array_ops.rank(constant_op.constant(0.0, shape=[1, 2, 3])) c_val = tensor_util.constant_value(tf_val) @@ -852,6 +856,7 @@ class ConstantValueTest(test.TestCase): c_val = tensor_util.constant_value(tf_val) self.assertAllClose(np_val.astype(np.float64), c_val) + @test_util.run_deprecated_v1 def testConcat(self): np_val = np.random.rand(3, 4, 7).astype(np.float32) tf_val = array_ops.concat( @@ -871,6 +876,7 @@ class ConstantValueTest(test.TestCase): c_val = tensor_util.constant_value(tf_val) self.assertIs(None, c_val) + @test_util.run_deprecated_v1 def testPack_Axis0(self): inputs = [np.random.rand(4, 7) for _ in range(3)] np_val = np.array(inputs) @@ -883,6 +889,7 @@ class ConstantValueTest(test.TestCase): c_val = tensor_util.constant_value(tf_val) self.assertIs(None, c_val) + @test_util.run_deprecated_v1 def testPack_Axis1(self): inputs = [np.random.rand(4, 7) for _ in range(3)] tf_val = array_ops.stack(inputs, axis=1) @@ -894,6 +901,7 @@ class ConstantValueTest(test.TestCase): c_val = tensor_util.constant_value(tf_val) self.assertIs(None, c_val) + @test_util.run_deprecated_v1 def testPack_Partial_Axis0(self): input_ = np.random.rand(4, 7) tf_val = array_ops.stack([input_, array_ops.placeholder(dtypes.float32)]) @@ -901,6 +909,7 @@ class ConstantValueTest(test.TestCase): self.assertAllClose(input_, c_val[0]) self.assertIsNone(c_val[1]) + @test_util.run_deprecated_v1 def testPack_Partial_Axis1(self): input_ = np.random.rand(4, 7) tf_val = array_ops.stack([input_, array_ops.placeholder(dtypes.float32)], @@ -966,12 +975,14 @@ class ConstantValueAsShapeTest(test.TestCase): c_val = tensor_util.constant_value_as_shape(tf_val) self.assertEqual([None, 1, None], c_val.as_list()) + @test_util.run_deprecated_v1 def testPack(self): tf_val = array_ops.stack( [constant_op.constant(16), 37, array_ops.placeholder(dtypes.int32)]) c_val = tensor_util.constant_value_as_shape(tf_val) self.assertEqual([16, 37, None], c_val.as_list()) + @test_util.run_deprecated_v1 def testConcat(self): tf_val = array_ops.concat( [[16, 37], array_ops.placeholder( @@ -985,6 +996,7 @@ class ConstantValueAsShapeTest(test.TestCase): c_val = tensor_util.constant_value_as_shape(tf_val) self.assertEqual([16, 37, None, 48], c_val.as_list()) + @test_util.run_deprecated_v1 def testSlice(self): tf_val = array_ops.placeholder(dtypes.int32, shape=(4,))[0:2] c_val = tensor_util.constant_value_as_shape(tf_val) diff --git a/tensorflow/python/framework/test_util.py b/tensorflow/python/framework/test_util.py index fd55ad2af9e..fc1a5fbe856 100644 --- a/tensorflow/python/framework/test_util.py +++ b/tensorflow/python/framework/test_util.py @@ -50,6 +50,7 @@ from tensorflow.core.framework import graph_pb2 from tensorflow.core.protobuf import config_pb2 from tensorflow.core.protobuf import rewriter_config_pb2 from tensorflow.python import pywrap_tensorflow +from tensorflow.python import tf2 from tensorflow.python.client import device_lib from tensorflow.python.client import session from tensorflow.python.eager import context @@ -66,6 +67,7 @@ from tensorflow.python.framework import tensor_shape from tensorflow.python.framework import versions from tensorflow.python.ops import array_ops from tensorflow.python.ops import control_flow_ops +from tensorflow.python.ops import tensor_array_ops from tensorflow.python.ops import variables from tensorflow.python.platform import googletest from tensorflow.python.platform import tf_logging as logging @@ -114,8 +116,28 @@ def assert_ops_in_graph(expected_ops, graph): return actual_ops -@tf_export("test.assert_equal_graph_def") -def assert_equal_graph_def(actual, expected, checkpoint_v2=False): +@tf_export("test.assert_equal_graph_def", v1=[]) +def assert_equal_graph_def_v2(actual, expected): + """Asserts that two `GraphDef`s are (mostly) the same. + + Compares two `GraphDef` protos for equality, ignoring versions and ordering of + nodes, attrs, and control inputs. Node names are used to match up nodes + between the graphs, so the naming of nodes must be consistent. This function + ignores randomized attribute values that may appear in V2 checkpoints. + + Args: + actual: The `GraphDef` we have. + expected: The `GraphDef` we expected. + + Raises: + AssertionError: If the `GraphDef`s do not match. + TypeError: If either argument is not a `GraphDef`. + """ + assert_equal_graph_def(actual, expected, checkpoint_v2=True) + + +@tf_export(v1=["test.assert_equal_graph_def"]) +def assert_equal_graph_def_v1(actual, expected, checkpoint_v2=False): """Asserts that two `GraphDef`s are (mostly) the same. Compares two `GraphDef` protos for equality, ignoring versions and ordering of @@ -132,6 +154,10 @@ def assert_equal_graph_def(actual, expected, checkpoint_v2=False): AssertionError: If the `GraphDef`s do not match. TypeError: If either argument is not a `GraphDef`. """ + assert_equal_graph_def(actual, expected, checkpoint_v2) + + +def assert_equal_graph_def(actual, expected, checkpoint_v2=False): if not isinstance(actual, graph_pb2.GraphDef): raise TypeError( "Expected tf.GraphDef for actual, got %s" % type(actual).__name__) @@ -354,53 +380,12 @@ def skip_if(condition): def enable_c_shapes(fn): - """Decorator for enabling C shapes on a test. - - Note this enables the C shapes after running the test class's setup/teardown - methods. - - Args: - fn: the function to be wrapped - - Returns: - The wrapped function - """ - - # pylint: disable=protected-access - def wrapper(*args, **kwargs): - prev_value = ops._USE_C_SHAPES - ops._USE_C_SHAPES = True - try: - fn(*args, **kwargs) - finally: - ops._USE_C_SHAPES = prev_value - - # pylint: enable=protected-access - - return wrapper + """No-op. TODO(b/74620627): Remove this.""" + return fn def with_c_shapes(cls): - """Adds methods that call original methods but with C API shapes enabled. - - Note this enables C shapes in new methods after running the test class's - setup method. - - Args: - cls: class to decorate - - Returns: - cls with new test methods added - """ - # If C shapes are already enabled, don't do anything. Some tests break if the - # same test is run twice, so this allows us to turn on the C shapes by default - # without breaking these tests. - if ops._USE_C_SHAPES: - return cls - - for name, value in cls.__dict__.copy().items(): - if callable(value) and name.startswith("test"): - setattr(cls, name + "WithCShapes", enable_c_shapes(value)) + """No-op. TODO(b/74620627): Remove this.""" return cls @@ -423,13 +408,40 @@ def enable_control_flow_v2(fn): def wrapper(*args, **kwargs): enable_cond_v2_old = control_flow_ops.ENABLE_COND_V2 enable_while_v2_old = control_flow_ops.ENABLE_WHILE_V2 + enable_tensor_array_v2_old = tensor_array_ops.ENABLE_TENSOR_ARRAY_V2 control_flow_ops.ENABLE_COND_V2 = True control_flow_ops.ENABLE_WHILE_V2 = True + tensor_array_ops.ENABLE_TENSOR_ARRAY_V2 = True try: fn(*args, **kwargs) finally: control_flow_ops.ENABLE_COND_V2 = enable_cond_v2_old control_flow_ops.ENABLE_WHILE_V2 = enable_while_v2_old + tensor_array_ops.ENABLE_TENSOR_ARRAY_V2 = enable_tensor_array_v2_old + + return wrapper + + +def enable_tensor_array_v2(fn): + """Decorator for enabling _GraphTensorArrayV2 on a test. + + Note this enables _GraphTensorArrayV2 after running the test class's + setup/teardown methods. + + Args: + fn: the function to be wrapped + + Returns: + The wrapped function + """ + + def wrapper(*args, **kwargs): + enable_tensor_array_v2_old = tensor_array_ops.ENABLE_TENSOR_ARRAY_V2 + tensor_array_ops.ENABLE_TENSOR_ARRAY_V2 = True + try: + fn(*args, **kwargs) + finally: + tensor_array_ops.ENABLE_TENSOR_ARRAY_V2 = enable_tensor_array_v2_old return wrapper @@ -881,8 +893,8 @@ def run_all_in_graph_and_eager_modes(cls): """Execute all test methods in the given class with and without eager.""" base_decorator = run_in_graph_and_eager_modes for name, value in cls.__dict__.copy().items(): - if callable(value) and name.startswith( - "test") and not name.startswith("testSkipEager"): + if callable(value) and name.startswith("test") and not ( + name.startswith("testSkipEager") or name.startswith("test_skip_eager")): setattr(cls, name, base_decorator(value)) return cls @@ -949,7 +961,7 @@ def run_in_graph_and_eager_modes(func=None, def decorator(f): if tf_inspect.isclass(f): raise ValueError( - "`run_test_in_graph_and_eager_modes` only supports test methods. " + "`run_in_graph_and_eager_modes` only supports test methods. " "Did you mean to use `run_all_in_graph_and_eager_modes`?") def decorated(self, *args, **kwargs): @@ -994,6 +1006,174 @@ def run_in_graph_and_eager_modes(func=None, return decorator +def run_deprecated_v1(func=None): + """Execute the decorated test in graph mode. + + This function returns a decorator intended to be applied to tests that have + not been updated to a style that is compatible with both TensorFlow 1.x and + 2.x. When this decorated is applied, the test body will be run in + an environment where API calls construct graphs instead of executing eagerly. + + Args: + func: function to be annotated. If `func` is None, this method returns a + decorator the can be applied to a function. If `func` is not None this + returns the decorator applied to `func`. + Returns: + Returns a decorator that will run the decorated test method in graph mode. + """ + + def decorator(f): + if tf_inspect.isclass(f): + raise ValueError("`run_deprecated_v1` only supports test methods.") + + def decorated(self, *args, **kwargs): + if tf2.enabled(): + with context.graph_mode(): + f(self, *args, **kwargs) + else: + f(self, *args, **kwargs) + + return decorated + + if func is not None: + return decorator(func) + + return decorator + + +def run_v1_only(reason, func=None): + """Execute the decorated test only if running in v1 mode. + + This function is intended to be applied to tests that exercise v1 only + functionality. If the test is run in v2 mode it will simply be skipped. + + Args: + reason: string giving a reason for limiting the test to v1 only. + func: function to be annotated. If `func` is None, this method returns a + decorator the can be applied to a function. If `func` is not None this + returns the decorator applied to `func`. + + Returns: + Returns a decorator that will conditionally skip the decorated test method. + """ + + def decorator(f): + if tf_inspect.isclass(f): + raise ValueError("`run_v1_only` only supports test methods.") + + def decorated(self, *args, **kwargs): + if tf2.enabled(): + self.skipTest(reason) + + f(self, *args, **kwargs) + + return decorated + + if func is not None: + return decorator(func) + + return decorator + + +def run_v2_only(func=None): + """Execute the decorated test only if running in v2 mode. + + This function is intended to be applied to tests that exercise v2 only + functionality. If the test is run in v1 mode it will simply be skipped. + + Args: + func: function to be annotated. If `func` is None, this method returns a + decorator the can be applied to a function. If `func` is not None this + returns the decorator applied to `func`. + + Returns: + Returns a decorator that will conditionally skip the decorated test method. + """ + + def decorator(f): + if tf_inspect.isclass(f): + raise ValueError("`run_v2_only` only supports test methods.") + + def decorated(self, *args, **kwargs): + if not tf2.enabled(): + self.skipTest("Test is only comptaible in v2") + + f(self, *args, **kwargs) + + return decorated + + if func is not None: + return decorator(func) + + return decorator + + +def run_gpu_only(func=None): + """Execute the decorated test only if a GPU is available. + + This function is intended to be applied to tests that require the precense + of a GPU. If a GPU is absent, it will simply be skipped. + + Args: + func: function to be annotated. If `func` is None, this method returns a + decorator the can be applied to a function. If `func` is not None this + returns the decorator applied to `func`. + + Returns: + Returns a decorator that will conditionally skip the decorated test method. + """ + + def decorator(f): + if tf_inspect.isclass(f): + raise ValueError("`run_gpu_only` only supports test methods.") + + def decorated(self, *args, **kwargs): + if not is_gpu_available(): + self.skipTest("Test requires GPU") + + f(self, *args, **kwargs) + + return decorated + + if func is not None: + return decorator(func) + + return decorator + + +def run_cuda_only(func=None): + """Execute the decorated test only if a GPU is available. + + This function is intended to be applied to tests that require the precense + of a CUDA GPU. If a CUDA GPU is absent, it will simply be skipped. + + Args: + func: function to be annotated. If `func` is None, this method returns a + decorator the can be applied to a function. If `func` is not None this + returns the decorator applied to `func`. + + Returns: + Returns a decorator that will conditionally skip the decorated test method. + """ + + def decorator(f): + if tf_inspect.isclass(f): + raise ValueError("`run_cuda_only` only supports test methods.") + + def decorated(self, *args, **kwargs): + if not is_gpu_available(cuda_only=True): + self.skipTest("Test requires CUDA GPU") + + f(self, *args, **kwargs) + + return decorated + + if func is not None: + return decorator(func) + + return decorator + + @tf_export("test.is_gpu_available") def is_gpu_available(cuda_only=False, min_cuda_compute_capability=None): """Returns whether TensorFlow can access a GPU. @@ -1033,7 +1213,7 @@ def is_gpu_available(cuda_only=False, min_cuda_compute_capability=None): return True return False except errors_impl.NotFoundError as e: - if not all([x in str(e) for x in ["CUDA", "not find"]]): + if not all(x in str(e) for x in ["CUDA", "not find"]): raise e else: logging.error(str(e)) @@ -1051,6 +1231,27 @@ def device(use_gpu): yield +@contextlib.contextmanager +def use_gpu(): + """Uses gpu when requested and available.""" + with device(use_gpu=True): + yield + + +@contextlib.contextmanager +def force_gpu(): + """Force the gpu to be used.""" + with ops.device("/device:GPU:0"): + yield + + +@contextlib.contextmanager +def force_cpu(): + """Force the cpu to be used.""" + with ops.device("/device:CPU:0"): + yield + + class CapturedWrites(object): """A utility class to load the captured writes made to a stream.""" diff --git a/tensorflow/python/framework/test_util_test.py b/tensorflow/python/framework/test_util_test.py index cbefe864814..dfdced5a988 100644 --- a/tensorflow/python/framework/test_util_test.py +++ b/tensorflow/python/framework/test_util_test.py @@ -49,6 +49,7 @@ from tensorflow.python.platform import googletest class TestUtilTest(test_util.TensorFlowTestCase, parameterized.TestCase): + @test_util.run_deprecated_v1 def test_assert_ops_in_graph(self): with self.test_session(): constant_op.constant(["hello", "taffy"], name="hello") @@ -60,6 +61,7 @@ class TestUtilTest(test_util.TensorFlowTestCase, parameterized.TestCase): self.assertRaises(ValueError, test_util.assert_ops_in_graph, {"hello": "Variable"}, ops.get_default_graph()) + @test_util.run_deprecated_v1 def test_session_functions(self): with self.test_session() as sess: sess_ref = weakref.ref(sess) @@ -551,6 +553,7 @@ class TestUtilTest(test_util.TensorFlowTestCase, parameterized.TestCase): with self.assertRaises(AssertionError): self.assertAllLessEqual(x, 95.0) + @test_util.run_deprecated_v1 def testAssertAllInRangeWithNonNumericValuesFails(self): s1 = constant_op.constant("Hello, ", name="s1") c = constant_op.constant([1 + 2j, -3 + 5j], name="c") @@ -614,6 +617,7 @@ class TestUtilTest(test_util.TensorFlowTestCase, parameterized.TestCase): with self.assertRaises(AssertionError): self.assertAllInSet(x, (42,)) + @test_util.run_deprecated_v1 def testRandomSeed(self): # Call setUp again for WithCApi case (since it makes a new defeault graph # after setup). @@ -681,7 +685,7 @@ class TestUtilTest(test_util.TensorFlowTestCase, parameterized.TestCase): self.assertIsNone(test_util.get_node_def_from_graph("bar", graph_def)) def test_run_in_eager_and_graph_modes_test_class(self): - msg = "`run_test_in_graph_and_eager_modes` only supports test methods.*" + msg = "`run_in_graph_and_eager_modes` only supports test methods.*" with self.assertRaisesRegexp(ValueError, msg): @test_util.run_in_graph_and_eager_modes() class Foo(object): @@ -706,6 +710,7 @@ class TestUtilTest(test_util.TensorFlowTestCase, parameterized.TestCase): test_util.run_in_graph_and_eager_modes(_test)(self) self.assertEqual(modes, ["graph"]) + @test_util.run_deprecated_v1 def test_run_in_graph_and_eager_modes_setup_in_same_mode(self): modes = [] mode_name = lambda: "eager" if context.executing_eagerly() else "graph" diff --git a/tensorflow/python/grappler/constant_folding_test.py b/tensorflow/python/grappler/constant_folding_test.py index ab1d0ed25b9..30c1e146814 100644 --- a/tensorflow/python/grappler/constant_folding_test.py +++ b/tensorflow/python/grappler/constant_folding_test.py @@ -61,7 +61,7 @@ class ConstantFoldingTest(test.TestCase): back_prop=False, parallel_iterations=1) with session.Session() as sess: - y_v = sess.run(y) + y_v = self.evaluate(y) self.assertAllEqual(np.zeros([10, 20, 30]), y_v) diff --git a/tensorflow/python/grappler/cost_analyzer_test.py b/tensorflow/python/grappler/cost_analyzer_test.py index b8225b81a52..ee3e289f65d 100644 --- a/tensorflow/python/grappler/cost_analyzer_test.py +++ b/tensorflow/python/grappler/cost_analyzer_test.py @@ -38,6 +38,7 @@ from tensorflow.python.training import adam class CostAnalysisTest(test.TestCase): + @test_util.run_deprecated_v1 def testBasicCost(self): """Make sure arguments can be passed correctly.""" a = constant_op.constant(10, name="a") @@ -62,6 +63,7 @@ class CostAnalysisTest(test.TestCase): # Also print the report to make it easier to debug print("{}".format(report)) + @test_util.run_deprecated_v1 def testVerbose(self): """Make sure the full report is generated with verbose=True.""" a = constant_op.constant(10, name="a") @@ -81,6 +83,7 @@ class CostAnalysisTest(test.TestCase): # Also print the report to make it easier to debug print("{}".format(report)) + @test_util.run_deprecated_v1 def testSmallNetworkCost(self): image = array_ops.placeholder(dtypes.float32, shape=[1, 28, 28, 1]) label = array_ops.placeholder(dtypes.float32, shape=[1, 10]) @@ -96,8 +99,8 @@ class CostAnalysisTest(test.TestCase): b_fc = variables.Variable(random_ops.truncated_normal([10], stddev=0.1)) y_conv = nn_ops.softmax(math_ops.matmul(h_conv_flat, w_fc) + b_fc) - cross_entropy = math_ops.reduce_mean(-math_ops.reduce_sum( - label * math_ops.log(y_conv), reduction_indices=[1])) + cross_entropy = math_ops.reduce_mean( + -math_ops.reduce_sum(label * math_ops.log(y_conv), axis=[1])) _ = adam.AdamOptimizer(1e-4).minimize(cross_entropy) mg = meta_graph.create_meta_graph_def(graph=ops.get_default_graph()) @@ -129,6 +132,7 @@ class CostAnalysisTest(test.TestCase): # self.assertTrue(0 < upper) # self.assertTrue(lower <= upper) + @test_util.run_deprecated_v1 def testBasicMemory(self): """Make sure arguments can be passed correctly.""" with test_util.device(use_gpu=False): diff --git a/tensorflow/python/grappler/cost_analyzer_tool.py b/tensorflow/python/grappler/cost_analyzer_tool.py index e6229e18566..7dbaf449cad 100644 --- a/tensorflow/python/grappler/cost_analyzer_tool.py +++ b/tensorflow/python/grappler/cost_analyzer_tool.py @@ -25,8 +25,8 @@ from google.protobuf import message from google.protobuf import text_format from tensorflow.contrib.fused_conv.ops import gen_fused_conv2d_bias_activation_op # pylint: disable=unused-import from tensorflow.core.framework import graph_pb2 +from tensorflow.core.protobuf import config_pb2 from tensorflow.core.protobuf import meta_graph_pb2 -from tensorflow.core.protobuf import rewriter_config_pb2 from tensorflow.core.protobuf import saved_model_pb2 from tensorflow.python.framework import importer from tensorflow.python.framework import ops @@ -79,10 +79,11 @@ def get_metagraph(): def main(_): metagraph = get_metagraph() - rewriter_config = rewriter_config_pb2.RewriterConfig() + config = config_pb2.ConfigProto() if FLAGS.rewriter_config is not None: - text_format.Merge(FLAGS.rewriter_config, rewriter_config) - optimized_graph = tf_optimizer.OptimizeGraph(rewriter_config, metagraph) + text_format.Merge(FLAGS.rewriter_config, + config.graph_options.rewrite_options) + optimized_graph = tf_optimizer.OptimizeGraph(config, metagraph) metagraph.graph_def.CopyFrom(optimized_graph) report = cost_analyzer.GenerateCostReport(metagraph, FLAGS.per_node_report, diff --git a/tensorflow/python/grappler/datasets_test.py b/tensorflow/python/grappler/datasets_test.py index bd870ad8de4..6937301ab25 100644 --- a/tensorflow/python/grappler/datasets_test.py +++ b/tensorflow/python/grappler/datasets_test.py @@ -48,7 +48,7 @@ class GrapplerTest(test.TestCase): for test_case in test_cases: with ops.Graph().as_default() as g: dataset = dataset_ops.Dataset.from_tensors(test_case['tensor']) - iterator = dataset.make_one_shot_iterator() + iterator = dataset_ops.make_one_shot_iterator(dataset) get_next = iterator.get_next() train_op = ops.get_collection_ref(ops.GraphKeys.TRAIN_OP) train_op.append(get_next) @@ -73,7 +73,7 @@ class GrapplerTest(test.TestCase): for test_case in test_cases: with ops.Graph().as_default() as g: dataset = dataset_ops.Dataset.from_tensor_slices(test_case['tensor']) - iterator = dataset.make_one_shot_iterator() + iterator = dataset_ops.make_one_shot_iterator(dataset) get_next = iterator.get_next() train_op = ops.get_collection_ref(ops.GraphKeys.TRAIN_OP) train_op.append(get_next) @@ -109,7 +109,7 @@ class GrapplerTest(test.TestCase): make_generator(test_case['tensor']), dtypes.int64, output_shapes=test_case['shape']) - iterator = dataset.make_one_shot_iterator() + iterator = dataset_ops.make_one_shot_iterator(dataset) get_next = iterator.get_next() train_op = ops.get_collection_ref(ops.GraphKeys.TRAIN_OP) train_op.append(get_next) @@ -122,7 +122,7 @@ class GrapplerTest(test.TestCase): def testRange(self): with ops.Graph().as_default() as g: dataset = dataset_ops.Dataset.range(42) - iterator = dataset.make_one_shot_iterator() + iterator = dataset_ops.make_one_shot_iterator(dataset) get_next = iterator.get_next() train_op = ops.get_collection_ref(ops.GraphKeys.TRAIN_OP) train_op.append(get_next) @@ -148,7 +148,7 @@ class GrapplerTest(test.TestCase): with ops.Graph().as_default() as g: dataset = dataset_ops.Dataset.from_tensors(test_case['tensor']) dataset = fn(dataset, test_case['tensor'], test_case['shape']) - iterator = dataset.make_one_shot_iterator() + iterator = dataset_ops.make_one_shot_iterator(dataset) get_next = iterator.get_next() train_op = ops.get_collection_ref(ops.GraphKeys.TRAIN_OP) train_op.append(get_next) @@ -252,7 +252,7 @@ class GrapplerTest(test.TestCase): with ops.Graph().as_default() as g: dataset = dataset_ops.Dataset.from_tensors(test_case['tensor']) dataset = dataset.batch(42) - iterator = dataset.make_one_shot_iterator() + iterator = dataset_ops.make_one_shot_iterator(dataset) get_next = iterator.get_next() train_op = ops.get_collection_ref(ops.GraphKeys.TRAIN_OP) train_op.append(get_next) @@ -281,7 +281,7 @@ class GrapplerTest(test.TestCase): with ops.Graph().as_default() as g: dataset = dataset_ops.Dataset.from_tensors(test_case['tensor']) dataset = dataset.padded_batch(42, padded_shapes=test_case['shape'][1:]) - iterator = dataset.make_one_shot_iterator() + iterator = dataset_ops.make_one_shot_iterator(dataset) get_next = iterator.get_next() train_op = ops.get_collection_ref(ops.GraphKeys.TRAIN_OP) train_op.append(get_next) @@ -318,7 +318,7 @@ class GrapplerTest(test.TestCase): return dataset_fn dataset = dataset.flat_map(make_dataset(test_case['tensor'])) - iterator = dataset.make_one_shot_iterator() + iterator = dataset_ops.make_one_shot_iterator(dataset) get_next = iterator.get_next() train_op = ops.get_collection_ref(ops.GraphKeys.TRAIN_OP) train_op.append(get_next) @@ -353,7 +353,7 @@ class GrapplerTest(test.TestCase): dataset = dataset.interleave( make_dataset(test_case['tensor']), cycle_length=42) - iterator = dataset.make_one_shot_iterator() + iterator = dataset_ops.make_one_shot_iterator(dataset) get_next = iterator.get_next() train_op = ops.get_collection_ref(ops.GraphKeys.TRAIN_OP) train_op.append(get_next) @@ -382,7 +382,7 @@ class GrapplerTest(test.TestCase): with ops.Graph().as_default() as g: dataset = dataset_ops.Dataset.from_tensors(test_case['tensor']) dataset = dataset.map(array_ops.transpose) - iterator = dataset.make_one_shot_iterator() + iterator = dataset_ops.make_one_shot_iterator(dataset) get_next = iterator.get_next() train_op = ops.get_collection_ref(ops.GraphKeys.TRAIN_OP) train_op.append(get_next) diff --git a/tensorflow/python/grappler/graph_placer.py b/tensorflow/python/grappler/graph_placer.py index 654013b23c5..9c05ad81790 100644 --- a/tensorflow/python/grappler/graph_placer.py +++ b/tensorflow/python/grappler/graph_placer.py @@ -19,8 +19,8 @@ from __future__ import division from __future__ import print_function import time +from tensorflow.core.protobuf import config_pb2 from tensorflow.core.protobuf import meta_graph_pb2 -from tensorflow.core.protobuf import rewriter_config_pb2 from tensorflow.python.framework import errors from tensorflow.python.framework import ops as tf_ops from tensorflow.python.grappler import cluster as gcluster @@ -54,9 +54,9 @@ def PlaceGraph(metagraph, cluster = gcluster.Cluster() # Optimize the metagraph to speedup the placement - rewriter_config = rewriter_config_pb2.RewriterConfig() + config = config_pb2.ConfigProto() optimized_graph = tf_optimizer.OptimizeGraph( - rewriter_config, metagraph, verbose=verbose, cluster=cluster) + config, metagraph, verbose=verbose, cluster=cluster) optimized_metagraph = meta_graph_pb2.MetaGraphDef() optimized_metagraph.CopyFrom(metagraph) optimized_metagraph.graph_def.CopyFrom(optimized_graph) diff --git a/tensorflow/python/grappler/item_test.py b/tensorflow/python/grappler/item_test.py index d3d96c646cd..78604b259ca 100644 --- a/tensorflow/python/grappler/item_test.py +++ b/tensorflow/python/grappler/item_test.py @@ -24,6 +24,7 @@ from tensorflow.python.framework import errors_impl from tensorflow.python.framework import meta_graph from tensorflow.python.framework import ops from tensorflow.python.framework import tensor_shape +from tensorflow.python.framework import test_util from tensorflow.python.grappler import item from tensorflow.python.ops import control_flow_ops from tensorflow.python.ops import gen_array_ops @@ -107,6 +108,7 @@ class ItemTest(test.TestCase): newest_tf_item = grappler_item.tf_item self.assertEqual(new_tf_item, newest_tf_item) + @test_util.run_deprecated_v1 def testColocationContraints(self): with ops.Graph().as_default() as g: c = constant_op.constant([10]) diff --git a/tensorflow/python/grappler/layout_optimizer_test.py b/tensorflow/python/grappler/layout_optimizer_test.py index 8cc971c61d5..98f2e6d7181 100644 --- a/tensorflow/python/grappler/layout_optimizer_test.py +++ b/tensorflow/python/grappler/layout_optimizer_test.py @@ -29,6 +29,7 @@ from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops from tensorflow.python.framework import random_seed +from tensorflow.python.framework import test_util from tensorflow.python.grappler import cluster as gcluster from tensorflow.python.grappler import tf_optimizer from tensorflow.python.layers import convolutional as conv_layers @@ -241,7 +242,7 @@ class LayoutOptimizerTest(test.TestCase): if restore: saver.restore(sess, checkpoint_path) else: - sess.run(variables.global_variables_initializer()) + self.evaluate(variables.global_variables_initializer()) np.random.seed(0) for _ in range(2): @@ -262,7 +263,7 @@ class LayoutOptimizerTest(test.TestCase): output = _two_layer_model(x) with session.Session(config=_get_config(False)) as sess: - output_val_ref = sess.run(output) + output_val_ref = self.evaluate(output) with session.Session(config=_get_config()) as sess: metadata = config_pb2.RunMetadata() @@ -365,7 +366,7 @@ class LayoutOptimizerTest(test.TestCase): output = array_ops.identity(pad) with session.Session(config=_get_config(False)) as sess: - output_val_ref = sess.run(output) + output_val_ref = self.evaluate(output) with session.Session(config=_get_config()) as sess: metadata = config_pb2.RunMetadata() @@ -396,7 +397,7 @@ class LayoutOptimizerTest(test.TestCase): output = array_ops.identity(reduce_sum) with session.Session(config=_get_config(False)) as sess: - output_val_ref = sess.run(output) + output_val_ref = self.evaluate(output) with session.Session(config=_get_config()) as sess: metadata = config_pb2.RunMetadata() @@ -425,7 +426,7 @@ class LayoutOptimizerTest(test.TestCase): output = array_ops.identity(cast) with session.Session(config=_get_config(False)) as sess: - output_val_ref = sess.run(output) + output_val_ref = self.evaluate(output) with session.Session(config=_get_config()) as sess: metadata = config_pb2.RunMetadata() @@ -456,7 +457,7 @@ class LayoutOptimizerTest(test.TestCase): output = array_ops.identity(squeeze) with session.Session(config=_get_config(False)) as sess: - output_val_ref = sess.run(output) + output_val_ref = self.evaluate(output) with session.Session(config=_get_config()) as sess: metadata = config_pb2.RunMetadata() @@ -486,7 +487,7 @@ class LayoutOptimizerTest(test.TestCase): output = array_ops.identity(squeeze) with session.Session(config=_get_config(False)) as sess: - output_val_ref = sess.run(output) + output_val_ref = self.evaluate(output) with session.Session(config=_get_config()) as sess: metadata = config_pb2.RunMetadata() @@ -516,7 +517,7 @@ class LayoutOptimizerTest(test.TestCase): output = array_ops.identity(squeeze) with session.Session(config=_get_config(False)) as sess: - output_val_ref = sess.run(output) + output_val_ref = self.evaluate(output) with session.Session(config=_get_config()) as sess: metadata = config_pb2.RunMetadata() @@ -545,7 +546,7 @@ class LayoutOptimizerTest(test.TestCase): output = array_ops.identity(reduce_sum) with session.Session(config=_get_config(False)) as sess: - output_val_ref = sess.run(output) + output_val_ref = self.evaluate(output) with session.Session(config=_get_config()) as sess: metadata = config_pb2.RunMetadata() @@ -574,7 +575,7 @@ class LayoutOptimizerTest(test.TestCase): output = array_ops.identity(reduce_sum) with session.Session(config=_get_config(False)) as sess: - output_val_ref = sess.run(output) + output_val_ref = self.evaluate(output) with session.Session(config=_get_config()) as sess: metadata = config_pb2.RunMetadata() @@ -603,7 +604,7 @@ class LayoutOptimizerTest(test.TestCase): output = array_ops.identity(reduce_sum) with session.Session(config=_get_config(False)) as sess: - output_val_ref = sess.run(output) + output_val_ref = self.evaluate(output) with session.Session(config=_get_config()) as sess: metadata = config_pb2.RunMetadata() @@ -632,7 +633,7 @@ class LayoutOptimizerTest(test.TestCase): output = array_ops.identity(reduce_sum) with session.Session(config=_get_config(False)) as sess: - output_val_ref = sess.run(output) + output_val_ref = self.evaluate(output) with session.Session(config=_get_config()) as sess: metadata = config_pb2.RunMetadata() @@ -662,7 +663,7 @@ class LayoutOptimizerTest(test.TestCase): output = array_ops.identity(reduce_sum) with session.Session(config=_get_config(False)) as sess: - output_val_ref = sess.run(output) + output_val_ref = self.evaluate(output) with session.Session(config=_get_config()) as sess: metadata = config_pb2.RunMetadata() @@ -691,7 +692,7 @@ class LayoutOptimizerTest(test.TestCase): output = array_ops.identity(reduce_sum) with session.Session(config=_get_config(False)) as sess: - output_val_ref = sess.run(output) + output_val_ref = self.evaluate(output) with session.Session(config=_get_config()) as sess: metadata = config_pb2.RunMetadata() @@ -724,7 +725,7 @@ class LayoutOptimizerTest(test.TestCase): output = array_ops.identity(concat) with session.Session(config=_get_config(False)) as sess: - output_val_ref = sess.run(output) + output_val_ref = self.evaluate(output) with session.Session(config=_get_config()) as sess: metadata = config_pb2.RunMetadata() @@ -835,7 +836,7 @@ class LayoutOptimizerTest(test.TestCase): output = array_ops.identity(reverse) with session.Session(config=_get_config(False)) as sess: - output_val_ref = sess.run(output) + output_val_ref = self.evaluate(output) with session.Session(config=_get_config()) as sess: metadata = config_pb2.RunMetadata() @@ -905,7 +906,7 @@ class LayoutOptimizerTest(test.TestCase): output = array_ops.identity(select) with session.Session(config=_get_config(False)) as sess: - output_val_ref = sess.run(output) + output_val_ref = self.evaluate(output) with session.Session(config=_get_config()) as sess: metadata = config_pb2.RunMetadata() @@ -966,7 +967,7 @@ class LayoutOptimizerTest(test.TestCase): output = array_ops.identity(select) with session.Session(config=_get_config(False)) as sess: - output_val_ref = sess.run(output) + output_val_ref = self.evaluate(output) with session.Session(config=_get_config()) as sess: metadata = config_pb2.RunMetadata() @@ -1179,7 +1180,7 @@ class LayoutOptimizerTest(test.TestCase): output = array_ops.identity(s) with session.Session(config=_get_config(False)) as sess: - output_val_ref = sess.run(output) + output_val_ref = self.evaluate(output) with session.Session(config=_get_config()) as sess: metadata = config_pb2.RunMetadata() @@ -1214,7 +1215,7 @@ class LayoutOptimizerTest(test.TestCase): output = array_ops.identity(s) with session.Session(config=_get_config(False)) as sess: - output_val_ref = sess.run(output) + output_val_ref = self.evaluate(output) with session.Session(config=_get_config()) as sess: metadata = config_pb2.RunMetadata() @@ -1347,7 +1348,7 @@ class LayoutOptimizerTest(test.TestCase): output = _loop() with session.Session(config=_get_config(False)) as sess: - output_val_ref = sess.run(output) + output_val_ref = self.evaluate(output) with session.Session(config=_get_config()) as sess: metadata = config_pb2.RunMetadata() @@ -1374,7 +1375,7 @@ class LayoutOptimizerTest(test.TestCase): output = _loop_with_branch() with session.Session(config=_get_config(False)) as sess: - output_val_ref = sess.run(output) + output_val_ref = self.evaluate(output) with session.Session(config=_get_config()) as sess: metadata = config_pb2.RunMetadata() @@ -1398,7 +1399,7 @@ class LayoutOptimizerTest(test.TestCase): output = _loop_with_vec_and_4d() with session.Session(config=_get_config(False)) as sess: - output_val_ref = sess.run(output) + output_val_ref = self.evaluate(output) with session.Session(config=_get_config()) as sess: metadata = config_pb2.RunMetadata() @@ -1422,7 +1423,7 @@ class LayoutOptimizerTest(test.TestCase): output = _model_with_second_port() with session.Session(config=_get_config(False)) as sess: - output_val_ref = sess.run(output) + output_val_ref = self.evaluate(output) with session.Session(config=_get_config()) as sess: metadata = config_pb2.RunMetadata() @@ -1441,13 +1442,16 @@ class LayoutOptimizerTest(test.TestCase): self._assert_trans_nchw_to_nhwc('Add-0-0', nodes) self.assertAllClose(output_val_ref, output_val, atol=1e-3) + @test_util.run_deprecated_v1 def testGradient(self): meta_graph = _simple_metagraph() - rewrite_options = rewriter_config_pb2.RewriterConfig( - layout_optimizer=rewriter_config_pb2.RewriterConfig.ON, - min_graph_nodes=-1) + config = config_pb2.ConfigProto() + config.graph_options.rewrite_options.CopyFrom( + rewriter_config_pb2.RewriterConfig( + layout_optimizer=rewriter_config_pb2.RewriterConfig.ON, + min_graph_nodes=-1)) optimized_graph = tf_optimizer.OptimizeGraph( - rewrite_options, meta_graph, cluster=_get_cluster()) + config, meta_graph, cluster=_get_cluster()) found = 0 for node in optimized_graph.node: @@ -1456,13 +1460,16 @@ class LayoutOptimizerTest(test.TestCase): self.assertEqual(node.attr['data_format'].s, b'NCHW') self.assertEqual(found, 5) + @test_util.run_deprecated_v1 def testDepthwise(self): meta_graph = _simple_metagraph(depthwise=True) - rewrite_options = rewriter_config_pb2.RewriterConfig( - layout_optimizer=rewriter_config_pb2.RewriterConfig.ON, - min_graph_nodes=-1) + config = config_pb2.ConfigProto() + config.graph_options.rewrite_options.CopyFrom( + rewriter_config_pb2.RewriterConfig( + layout_optimizer=rewriter_config_pb2.RewriterConfig.ON, + min_graph_nodes=-1)) optimized_graph = tf_optimizer.OptimizeGraph( - rewrite_options, meta_graph, cluster=_get_cluster()) + config, meta_graph, cluster=_get_cluster()) found = 0 for node in optimized_graph.node: diff --git a/tensorflow/python/grappler/memory_optimizer_test.py b/tensorflow/python/grappler/memory_optimizer_test.py index 03b42f64539..6eb16fbd39e 100644 --- a/tensorflow/python/grappler/memory_optimizer_test.py +++ b/tensorflow/python/grappler/memory_optimizer_test.py @@ -25,6 +25,7 @@ from tensorflow.python.client import session from tensorflow.python.framework import meta_graph from tensorflow.python.framework import ops from tensorflow.python.framework import random_seed +from tensorflow.python.framework import test_util from tensorflow.python.grappler import tf_optimizer from tensorflow.python.ops import math_ops from tensorflow.python.ops import nn @@ -37,6 +38,7 @@ from tensorflow.python.training import training as train class MemoryOptimizerSwapTest(test.TestCase): """Tests the Grappler memory optimizer.""" + @test_util.run_deprecated_v1 def testNoSwapping(self): """Make sure the graph is preserved when there is nothing to swap.""" a = variables.VariableV1(10, name='a') @@ -49,15 +51,18 @@ class MemoryOptimizerSwapTest(test.TestCase): graph_size = len(mg.graph_def.node) nodes = [node.name for node in mg.graph_def.node] - rewriter_config = rewriter_config_pb2.RewriterConfig( - disable_model_pruning=True, - constant_folding=rewriter_config_pb2.RewriterConfig.OFF, - memory_optimization=rewriter_config_pb2.RewriterConfig.MANUAL) - graph = tf_optimizer.OptimizeGraph(rewriter_config, mg) + config = config_pb2.ConfigProto() + config.graph_options.rewrite_options.CopyFrom( + rewriter_config_pb2.RewriterConfig( + disable_model_pruning=True, + constant_folding=rewriter_config_pb2.RewriterConfig.OFF, + memory_optimization=rewriter_config_pb2.RewriterConfig.MANUAL)) + graph = tf_optimizer.OptimizeGraph(config, mg) self.assertEqual(len(graph.node), graph_size) self.assertItemsEqual([node.name for node in graph.node], nodes) + @test_util.run_deprecated_v1 def testSimpleSwap(self): """Check that the swap annotations are followed.""" a = variables.VariableV1(10, name='a') @@ -72,13 +77,15 @@ class MemoryOptimizerSwapTest(test.TestCase): mg = meta_graph.create_meta_graph_def(graph=ops.get_default_graph()) graph_size = len(mg.graph_def.node) - rewriter_config = rewriter_config_pb2.RewriterConfig( - disable_model_pruning=True, - meta_optimizer_iterations=rewriter_config_pb2.RewriterConfig.ONE, - constant_folding=rewriter_config_pb2.RewriterConfig.OFF, - memory_optimization=rewriter_config_pb2.RewriterConfig.MANUAL, - min_graph_nodes=-1) - graph = tf_optimizer.OptimizeGraph(rewriter_config, mg) + config = config_pb2.ConfigProto() + config.graph_options.rewrite_options.CopyFrom( + rewriter_config_pb2.RewriterConfig( + disable_model_pruning=True, + meta_optimizer_iterations=rewriter_config_pb2.RewriterConfig.ONE, + constant_folding=rewriter_config_pb2.RewriterConfig.OFF, + memory_optimization=rewriter_config_pb2.RewriterConfig.MANUAL, + min_graph_nodes=-1)) + graph = tf_optimizer.OptimizeGraph(config, mg) self.assertEqual(len(graph.node), graph_size + 2) self.assertTrue( @@ -127,7 +134,8 @@ class MemoryOptimizerRecomputeTest(test.TestCase): def testRewritingDefaultGradientNames(self): """Tests that rewriting occurs with default gradient names.""" (original_metagraph, _, _, _) = self._GetMetaGraph() - rewritten_graph_def = tf_optimizer.OptimizeGraph( + config = config_pb2.ConfigProto() + config.graph_options.rewrite_options.CopyFrom( rewriter_config_pb2.RewriterConfig( disable_model_pruning=True, constant_folding=rewriter_config_pb2.RewriterConfig.OFF, @@ -135,8 +143,9 @@ class MemoryOptimizerRecomputeTest(test.TestCase): layout_optimizer=rewriter_config_pb2.RewriterConfig.OFF, arithmetic_optimization=rewriter_config_pb2.RewriterConfig.OFF, min_graph_nodes=-1, - memory_optimization=rewriter_config_pb2.RewriterConfig. - RECOMPUTATION_HEURISTICS), original_metagraph) + memory_optimization=( + rewriter_config_pb2.RewriterConfig.RECOMPUTATION_HEURISTICS))) + rewritten_graph_def = tf_optimizer.OptimizeGraph(config, original_metagraph) self.assertGreater( len(rewritten_graph_def.node), len(original_metagraph.graph_def.node)) @@ -153,7 +162,8 @@ class MemoryOptimizerRecomputeTest(test.TestCase): """Tests that rewriting occurs with non-standard gradient names.""" (original_metagraph, _, _, _) = self._GetMetaGraph( optimizer_scope_name='optimizer') - rewritten_graph_def = tf_optimizer.OptimizeGraph( + config = config_pb2.ConfigProto() + config.graph_options.rewrite_options.CopyFrom( rewriter_config_pb2.RewriterConfig( disable_model_pruning=True, constant_folding=rewriter_config_pb2.RewriterConfig.OFF, @@ -161,11 +171,11 @@ class MemoryOptimizerRecomputeTest(test.TestCase): layout_optimizer=rewriter_config_pb2.RewriterConfig.OFF, arithmetic_optimization=rewriter_config_pb2.RewriterConfig.OFF, min_graph_nodes=-1, - memory_optimization=rewriter_config_pb2.RewriterConfig. - RECOMPUTATION_HEURISTICS, + memory_optimization=rewriter_config_pb2.RewriterConfig + .RECOMPUTATION_HEURISTICS, # Checks that name scope "gradients/" also match sub-scope. - memory_optimizer_target_node_name_scope='gradients/'), - original_metagraph) + memory_optimizer_target_node_name_scope='gradients/')) + rewritten_graph_def = tf_optimizer.OptimizeGraph(config, original_metagraph) self.assertGreater( len(rewritten_graph_def.node), len(original_metagraph.graph_def.node)) @@ -182,18 +192,19 @@ class MemoryOptimizerRecomputeTest(test.TestCase): """Tests that rewriting occurs with non-standard gradient names.""" (original_metagraph, _, _, _) = self._GetMetaGraph(optimizer_scope_name='foo/bar') - rewritten_graph_def = tf_optimizer.OptimizeGraph( + config = config_pb2.ConfigProto() + config.graph_options.rewrite_options.CopyFrom( rewriter_config_pb2.RewriterConfig( disable_model_pruning=True, constant_folding=rewriter_config_pb2.RewriterConfig.OFF, dependency_optimization=rewriter_config_pb2.RewriterConfig.OFF, layout_optimizer=rewriter_config_pb2.RewriterConfig.OFF, arithmetic_optimization=rewriter_config_pb2.RewriterConfig.OFF, - memory_optimization=rewriter_config_pb2.RewriterConfig. - RECOMPUTATION_HEURISTICS, + memory_optimization=rewriter_config_pb2.RewriterConfig + .RECOMPUTATION_HEURISTICS, # This should not match anything. - memory_optimizer_target_node_name_scope='r/gradients/'), - original_metagraph) + memory_optimizer_target_node_name_scope='r/gradients/')) + rewritten_graph_def = tf_optimizer.OptimizeGraph(config, original_metagraph) self.assertEqual( len(rewritten_graph_def.node), len(original_metagraph.graph_def.node)) self.assertEqual(0, @@ -223,10 +234,10 @@ class MemoryOptimizerRecomputeTest(test.TestCase): train_op = graph.get_operation_by_name(train_op_name) loss_op = graph.get_tensor_by_name(loss_op_name) with session.Session(config=config, graph=graph) as sess: - sess.run(init_op) - sess.run(train_op) - sess.run(train_op) - return sess.run(loss_op) + self.evaluate(init_op) + self.evaluate(train_op) + self.evaluate(train_op) + return self.evaluate(loss_op) def testRecomputationRewritingNoErrors(self): """Tests that graph output is not significantly different with rewriting.""" @@ -287,8 +298,8 @@ class MemoryOptimizerRecomputeTest(test.TestCase): rewrite_options=manual_memory_config) session_config = config_pb2.ConfigProto(graph_options=graph_options) with session.Session(config=session_config) as sess: - sess.run(init_op) - sess.run(train_op) + self.evaluate(init_op) + self.evaluate(train_op) def testHintDoesRewrite(self): graph = self._annotated_graph()[0] @@ -298,11 +309,12 @@ class MemoryOptimizerRecomputeTest(test.TestCase): 0, len([node for node in metagraph.graph_def.node if 'Recomputed/' in node.name])) - rewritten_graph_def = tf_optimizer.OptimizeGraph( + config = config_pb2.ConfigProto() + config.graph_options.rewrite_options.CopyFrom( rewriter_config_pb2.RewriterConfig( min_graph_nodes=-1, - memory_optimization=rewriter_config_pb2.RewriterConfig.MANUAL), - metagraph) + memory_optimization=rewriter_config_pb2.RewriterConfig.MANUAL)) + rewritten_graph_def = tf_optimizer.OptimizeGraph(config, metagraph) self.assertEqual( 9, len([node for node in rewritten_graph_def.node diff --git a/tensorflow/python/grappler/model_analyzer_test.py b/tensorflow/python/grappler/model_analyzer_test.py index ec172755f1a..d000cfa1ba2 100644 --- a/tensorflow/python/grappler/model_analyzer_test.py +++ b/tensorflow/python/grappler/model_analyzer_test.py @@ -21,6 +21,7 @@ from __future__ import print_function from tensorflow.python.framework import constant_op from tensorflow.python.framework import meta_graph from tensorflow.python.framework import ops +from tensorflow.python.framework import test_util from tensorflow.python.grappler import model_analyzer from tensorflow.python.ops import math_ops from tensorflow.python.platform import test @@ -28,6 +29,7 @@ from tensorflow.python.platform import test class PyWrapOptimizeGraphTest(test.TestCase): + @test_util.run_deprecated_v1 def testBasic(self): """Make sure arguments can be passed correctly.""" a = constant_op.constant([10, 11], name="a") @@ -49,6 +51,7 @@ class PyWrapOptimizeGraphTest(test.TestCase): # Also print the report to make it easier to debug print("{}".format(report)) + @test_util.run_deprecated_v1 def testDebugMode(self): """Make sure arguments can be passed correctly.""" a = constant_op.constant([10, 11], name="a") diff --git a/tensorflow/python/grappler/tf_optimizer.i b/tensorflow/python/grappler/tf_optimizer.i index 39ca71e99af..b746c3ec261 100644 --- a/tensorflow/python/grappler/tf_optimizer.i +++ b/tensorflow/python/grappler/tf_optimizer.i @@ -34,8 +34,8 @@ limitations under the License. $1 = &temp; } -%typemap(in) const tensorflow::RewriterConfig& ( - tensorflow::RewriterConfig temp) { +%typemap(in) const tensorflow::ConfigProto& ( + tensorflow::ConfigProto temp) { char* c_string; Py_ssize_t py_size; if (PyBytes_AsStringAndSize($input, &c_string, &py_size) == -1) { @@ -46,7 +46,7 @@ limitations under the License. if (!temp.ParseFromString(string(c_string, py_size))) { PyErr_SetString( PyExc_TypeError, - "The RewriterConfig could not be parsed as a valid protocol buffer"); + "The ConfigProto could not be parsed as a valid protocol buffer"); SWIG_fail; } $1 = &temp; @@ -67,20 +67,20 @@ limitations under the License. #include "tensorflow/core/grappler/clusters/utils.h" #include "tensorflow/core/grappler/clusters/virtual_cluster.h" #include "tensorflow/core/grappler/optimizers/meta_optimizer.h" + #include "tensorflow/core/protobuf/config.pb.h" #include "tensorflow/core/protobuf/meta_graph.pb.h" - #include "tensorflow/core/protobuf/rewriter_config.pb.h" #include "tensorflow/core/public/session_options.h" void DetectDevices(std::unordered_map* device_map) { tensorflow::SessionOptions options; - std::vector devices; + std::vector> devices; tensorflow::Status status = tensorflow::DeviceFactory::AddDevices(options, "", &devices); if (!status.ok()) { return; } - for (const tensorflow::Device* device : devices) { + for (const std::unique_ptr& device : devices) { tensorflow::DeviceProperties& prop = (*device_map)[device->name()]; prop = tensorflow::grappler::GetDeviceInfo(device->parsed_name()); @@ -88,13 +88,12 @@ void DetectDevices(std::unordered_map* dev // available device memory. const tensorflow::DeviceAttributes& attr = device->attributes(); prop.set_memory_size(attr.memory_limit()); - delete device; } } PyObject* TF_OptimizeGraph( GCluster cluster, - const tensorflow::RewriterConfig& rewriter_config, + const tensorflow::ConfigProto& config_proto, const tensorflow::MetaGraphDef& metagraph, bool verbose, const string& graph_id, TF_Status* out_status) { tensorflow::grappler::ItemConfig item_config; @@ -110,7 +109,7 @@ PyObject* TF_OptimizeGraph( tensorflow::DeviceBase* cpu_device = nullptr; tensorflow::GraphDef out_graph; - tensorflow::grappler::MetaOptimizer optimizer(cpu_device, rewriter_config); + tensorflow::grappler::MetaOptimizer optimizer(cpu_device, config_proto); tensorflow::Status status = optimizer.Optimize(cluster.get(), *grappler_item, &out_graph); if (verbose) { optimizer.PrintResult(); @@ -127,7 +126,7 @@ PyObject* TF_OptimizeGraph( // Wrap this function PyObject* TF_OptimizeGraph( GCluster cluster, - const tensorflow::RewriterConfig& rewriter_config, + const tensorflow::ConfigProto& config_proto, const tensorflow::MetaGraphDef& metagraph, bool verbose, const string& graph_id, TF_Status* out_status); diff --git a/tensorflow/python/grappler/tf_optimizer.py b/tensorflow/python/grappler/tf_optimizer.py index a73a4a98fc5..e72667b6f31 100644 --- a/tensorflow/python/grappler/tf_optimizer.py +++ b/tensorflow/python/grappler/tf_optimizer.py @@ -19,22 +19,26 @@ from __future__ import division from __future__ import print_function from tensorflow.core.framework import graph_pb2 +from tensorflow.core.protobuf import config_pb2 from tensorflow.python import pywrap_tensorflow as tf_opt from tensorflow.python.framework import errors from tensorflow.python.grappler import cluster as gcluster -def OptimizeGraph(rewriter_config, +def OptimizeGraph(config_proto, metagraph, verbose=True, graph_id=b'graph_to_optimize', cluster=None): """Optimize the provided metagraph.""" + if not isinstance(config_proto, config_pb2.ConfigProto): + raise TypeError('Expected config_proto to be a ConfigProto, saw type %s' % + type(config_proto)) with errors.raise_exception_on_not_ok_status() as status: if cluster is None: cluster = gcluster.Cluster() ret_from_swig = tf_opt.TF_OptimizeGraph(cluster.tf_cluster, - rewriter_config.SerializeToString(), + config_proto.SerializeToString(), metagraph.SerializeToString(), verbose, graph_id, status) if ret_from_swig is None: diff --git a/tensorflow/python/grappler/tf_optimizer_test.py b/tensorflow/python/grappler/tf_optimizer_test.py index eca0f679829..06ccaa813f2 100644 --- a/tensorflow/python/grappler/tf_optimizer_test.py +++ b/tensorflow/python/grappler/tf_optimizer_test.py @@ -17,12 +17,13 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function -from tensorflow.core.protobuf import rewriter_config_pb2 +from tensorflow.core.protobuf import config_pb2 from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes from tensorflow.python.framework import meta_graph from tensorflow.python.framework import ops from tensorflow.python.framework import tensor_shape +from tensorflow.python.framework import test_util from tensorflow.python.grappler import item as gitem from tensorflow.python.grappler import tf_optimizer from tensorflow.python.ops import array_ops @@ -34,6 +35,7 @@ from tensorflow.python.platform import test class PyWrapOptimizeGraphTest(test.TestCase): + @test_util.run_deprecated_v1 def testBasic(self): """Make sure arguments can be passed correctly.""" a = constant_op.constant(10, name='a') @@ -45,15 +47,17 @@ class PyWrapOptimizeGraphTest(test.TestCase): train_op.append(d) mg = meta_graph.create_meta_graph_def(graph=ops.get_default_graph()) - rewriter_config = rewriter_config_pb2.RewriterConfig() + config = config_pb2.ConfigProto() + rewriter_config = config.graph_options.rewrite_options rewriter_config.optimizers.append('constfold') rewriter_config.min_graph_nodes = -1 - graph = tf_optimizer.OptimizeGraph(rewriter_config, mg) + graph = tf_optimizer.OptimizeGraph(config, mg) self.assertEqual(len(graph.node), 1) self.assertItemsEqual([node.name for node in graph.node], ['d']) + @test_util.run_deprecated_v1 def testKeepNodes(self): g = ops.Graph() with g.as_default(): @@ -68,18 +72,21 @@ class PyWrapOptimizeGraphTest(test.TestCase): # Optimize the graph. mg = meta_graph.create_meta_graph_def(graph=g) - rewriter_config = rewriter_config_pb2.RewriterConfig() + config = config_pb2.ConfigProto() + rewriter_config = config.graph_options.rewrite_options rewriter_config.min_graph_nodes = -1 - optimized_graph = tf_optimizer.OptimizeGraph(rewriter_config, mg) + optimized_graph = tf_optimizer.OptimizeGraph(config, mg) # Check that the nodes referenced in various collections have been preserved - self.assertEqual(len(optimized_graph.node), 5) - self.assertEqual(d.op.name, optimized_graph.node[0].name) - self.assertEqual(a1.op.name, optimized_graph.node[1].name) - self.assertEqual('Variable/initial_value', optimized_graph.node[2].name) - self.assertEqual(a2.op.name, optimized_graph.node[3].name) - self.assertEqual('Variable/Assign', optimized_graph.node[4].name) + optimized_graph_nodes = [node.name for node in optimized_graph.node] + expected_nodes = [ + d.op.name, a1.op.name, a2.op.name, 'Variable/initial_value', + 'Variable/Assign' + ] + self.assertEqual(len(optimized_graph_nodes), len(expected_nodes)) + self.assertAllInSet(optimized_graph_nodes, expected_nodes) + @test_util.run_deprecated_v1 def testLoops(self): g = ops.Graph() with g.as_default(): @@ -110,9 +117,10 @@ class PyWrapOptimizeGraphTest(test.TestCase): # Optimize the graph. mg = meta_graph.create_meta_graph_def(graph=g) - rewriter_config = rewriter_config_pb2.RewriterConfig() + config = config_pb2.ConfigProto() + rewriter_config = config.graph_options.rewrite_options rewriter_config.min_graph_nodes = -1 - optimized_graph = tf_optimizer.OptimizeGraph(rewriter_config, mg) + optimized_graph = tf_optimizer.OptimizeGraph(config, mg) mg.graph_def.CopyFrom(optimized_graph) # Check that the nodes referenced in various collections have been preserved diff --git a/tensorflow/python/keras/BUILD b/tensorflow/python/keras/BUILD index 6f38d822e70..ab5628b1d23 100755 --- a/tensorflow/python/keras/BUILD +++ b/tensorflow/python/keras/BUILD @@ -3,10 +3,10 @@ licenses(["notice"]) # Apache 2.0 -exports_files(["LICENSE"]) - package(default_visibility = ["//visibility:public"]) +exports_files(["LICENSE"]) + load("//tensorflow:tensorflow.bzl", "py_test") load("//tensorflow:tensorflow.bzl", "cuda_py_test") @@ -122,8 +122,10 @@ py_library( "constraints.py", "engine/__init__.py", "engine/base_layer.py", + "engine/base_layer_utils.py", "engine/distributed_training_utils.py", "engine/input_layer.py", + "engine/input_spec.py", "engine/network.py", "engine/saving.py", "engine/sequential.py", @@ -141,11 +143,14 @@ py_library( "regularizers.py", "utils/data_utils.py", "utils/io_utils.py", + "utils/losses_utils.py", ], srcs_version = "PY2AND3", deps = [ ":backend", "//tensorflow/python/data", + "//tensorflow/python/distribute:reduce_util", + "//tensorflow/python/keras/optimizer_v2", "//tensorflow/python/training/checkpointable:data_structures", "//tensorflow/tools/docs:doc_controls", "@six_archive//:six", @@ -180,7 +185,6 @@ py_library( ":engine", "//tensorflow/python:array_ops", "//tensorflow/python:cudnn_rnn_ops_gen", - "//tensorflow/python:distribute", "//tensorflow/python:dtypes", "//tensorflow/python:embedding_ops", "//tensorflow/python:framework_ops", @@ -194,6 +198,7 @@ py_library( "//tensorflow/python:tensor_array_ops", "//tensorflow/python:tensor_shape", "//tensorflow/python:util", + "//tensorflow/python/distribute:distribute_lib", ], ) @@ -264,6 +269,7 @@ py_test( name = "optimizers_test", size = "medium", srcs = ["optimizers_test.py"], + shard_count = 2, srcs_version = "PY2AND3", tags = ["notsan"], deps = [ @@ -271,6 +277,7 @@ py_test( "//tensorflow/python:client_testlib", "//tensorflow/python:training", "//third_party/py/numpy", + "@absl_py//absl/testing:parameterized", ], ) @@ -300,6 +307,7 @@ py_test( ":keras", "//tensorflow/python:client_testlib", "//third_party/py/numpy", + "@absl_py//absl/testing:parameterized", ], ) @@ -318,7 +326,7 @@ py_test( py_test( name = "advanced_activations_test", - size = "small", + size = "medium", srcs = ["layers/advanced_activations_test.py"], srcs_version = "PY2AND3", deps = [ @@ -369,7 +377,7 @@ cuda_py_test( py_test( name = "pooling_test", - size = "medium", + size = "large", srcs = ["layers/pooling_test.py"], srcs_version = "PY2AND3", deps = [ @@ -491,12 +499,13 @@ py_test( ":keras", "//tensorflow/python:client_testlib", "//third_party/py/numpy", + "@absl_py//absl/testing:parameterized", ], ) py_test( name = "recurrent_test", - size = "medium", + size = "large", srcs = ["layers/recurrent_test.py"], srcs_version = "PY2AND3", deps = [ @@ -506,6 +515,19 @@ py_test( ], ) +cuda_py_test( + name = "unified_lstm_test", + size = "medium", + srcs = ["layers/unified_lstm_test.py"], + additional_deps = [ + ":keras", + "@absl_py//absl/testing:parameterized", + "//third_party/py/numpy", + "//tensorflow/python:client_testlib", + ], + shard_count = 4, +) + py_test( name = "serialization_test", size = "small", @@ -577,6 +599,17 @@ py_test( ], ) +py_test( + name = "tf_utils_test", + size = "small", + srcs = ["utils/tf_utils_test.py"], + srcs_version = "PY2AND3", + deps = [ + ":keras", + "//tensorflow/python:client_testlib", + ], +) + py_test( name = "io_utils_test", size = "small", @@ -707,15 +740,33 @@ py_test( ) py_test( - name = "training_generator_test", - size = "enormous", - srcs = ["engine/training_generator_test.py"], + name = "training_dataset_test", + size = "medium", + srcs = ["engine/training_dataset_test.py"], srcs_version = "PY2AND3", - tags = ["notsan"], deps = [ ":keras", "//tensorflow/python:client_testlib", "//third_party/py/numpy", + "@absl_py//absl/testing:parameterized", + ], +) + +py_test( + name = "training_generator_test", + size = "enormous", + srcs = ["engine/training_generator_test.py"], + shard_count = 3, + srcs_version = "PY2AND3", + tags = [ + "no_oss", + "notsan", + ], + deps = [ + ":keras", + "//tensorflow/python:client_testlib", + "//third_party/py/numpy", + "@absl_py//absl/testing:parameterized", ], ) @@ -730,6 +781,7 @@ py_test( "//tensorflow/python:client_testlib", "//tensorflow/python/feature_column:feature_column_py", "//third_party/py/numpy", + "@absl_py//absl/testing:parameterized", ], ) @@ -763,6 +815,7 @@ py_test( name = "model_subclassing_test", size = "medium", srcs = ["model_subclassing_test.py"], + shard_count = 2, srcs_version = "PY2AND3", tags = ["notsan"], deps = [ diff --git a/tensorflow/python/keras/activations_test.py b/tensorflow/python/keras/activations_test.py index ad238cb0a9b..6b7bfb698b8 100644 --- a/tensorflow/python/keras/activations_test.py +++ b/tensorflow/python/keras/activations_test.py @@ -21,6 +21,7 @@ from __future__ import print_function import numpy as np from tensorflow.python import keras +from tensorflow.python.framework import test_util from tensorflow.python.platform import test @@ -67,6 +68,7 @@ class KerasActivationsTest(test.TestCase): expected = _ref_softmax(test_values[0, 0]) self.assertAllClose(result[0, 0], expected, rtol=1e-05) + @test_util.run_deprecated_v1 def test_selu(self): x = keras.backend.placeholder(ndim=2) f = keras.backend.function([x], [keras.activations.selu(x)]) @@ -124,6 +126,7 @@ class KerasActivationsTest(test.TestCase): expected = sigmoid(test_values) self.assertAllClose(result, expected, rtol=1e-05) + @test_util.run_deprecated_v1 def test_hard_sigmoid(self): def ref_hard_sigmoid(x): x = (x * 0.2) + 0.5 @@ -147,6 +150,7 @@ class KerasActivationsTest(test.TestCase): # No negative values in test values... self.assertAllClose(result, test_values, rtol=1e-05) + @test_util.run_deprecated_v1 def test_elu(self): with self.cached_session(): x = keras.backend.placeholder(ndim=2) diff --git a/tensorflow/python/keras/backend.py b/tensorflow/python/keras/backend.py index dd9b0c07e70..1277746716a 100644 --- a/tensorflow/python/keras/backend.py +++ b/tensorflow/python/keras/backend.py @@ -25,6 +25,7 @@ import collections import itertools import json import os +import threading import weakref import numpy as np @@ -73,9 +74,9 @@ py_sum = sum # while executing eagerly (such as the functional API for model-building). _GRAPH = None -# This is the default internal TF session used by Keras. -# It can be set manually via `set_session(sess)`. -_SESSION = None +# This is a thread local object that will hold the default internal TF session +# used by Keras. It can be set manually via `set_session(sess)`. +_SESSION = threading.local() # This dictionary holds a mapping {graph: learning_phase}. # A learning phase is a bool tensor used to run Keras models in @@ -337,7 +338,7 @@ def clear_session(): global _GRAPH_TF_OPTIMIZERS # pylint: disable=global-variable-not-assigned ops.reset_default_graph() reset_uids() - _SESSION = None + _SESSION.session = None graph = get_graph() with graph.as_default(): phase = array_ops.placeholder_with_default( @@ -376,27 +377,22 @@ def learning_phase(): Returns: Learning phase (scalar integer tensor or Python integer). """ - with ops.init_scope(): - # We always check & set the learning phase inside the init_scope, - # otherwise the wrong default_graph will be used to look up the learning - # phase inside of functions & defuns. - # - # This is because functions & defuns (both in graph & in eager mode) - # will always execute non-eagerly using a function-specific default - # subgraph. - if context.executing_eagerly(): - if _DUMMY_EAGER_GRAPH not in _GRAPH_LEARNING_PHASES: - # Fallback to inference mode as default. - return 0 - return _GRAPH_LEARNING_PHASES[_DUMMY_EAGER_GRAPH] + if context.executing_eagerly(): + if _DUMMY_EAGER_GRAPH not in _GRAPH_LEARNING_PHASES: + # Fallback to inference mode as default. + return 0 + return _GRAPH_LEARNING_PHASES[_DUMMY_EAGER_GRAPH] + return symbolic_learning_phase() - graph = get_graph() - with graph.as_default(): - if graph not in _GRAPH_LEARNING_PHASES: - phase = array_ops.placeholder_with_default( - False, shape=(), name='keras_learning_phase') - _GRAPH_LEARNING_PHASES[graph] = phase - return _GRAPH_LEARNING_PHASES[graph] + +def symbolic_learning_phase(): + graph = get_graph() + with graph.as_default(): + if graph not in _GRAPH_LEARNING_PHASES: + phase = array_ops.placeholder_with_default( + False, shape=(), name='keras_learning_phase') + _GRAPH_LEARNING_PHASES[graph] = phase + return _GRAPH_LEARNING_PHASES[graph] @tf_export('keras.backend.set_learning_phase') @@ -449,6 +445,20 @@ def learning_phase_scope(value): _GRAPH_LEARNING_PHASES[get_graph()] = previous_value +def _get_session(): + """Returns the session object for the current thread.""" + global _SESSION + default_session = ops.get_default_session() + if default_session is not None: + session = default_session + else: + if getattr(_SESSION, 'session', None) is None: + _SESSION.session = session_module.Session( + config=get_default_session_config()) + session = _SESSION.session + return session + + @tf_export(v1=['keras.backend.get_session']) def get_session(): """Returns the TF session to be used by the backend. @@ -466,14 +476,7 @@ def get_session(): Returns: A TensorFlow session. """ - global _SESSION - default_session = ops.get_default_session() - if default_session is not None: - session = default_session - else: - if _SESSION is None: - _SESSION = session_module.Session(config=get_default_session_config()) - session = _SESSION + session = _get_session() if not _MANUAL_VAR_INIT: with session.graph.as_default(): _initialize_variables(session) @@ -498,7 +501,7 @@ def set_session(session): session: A TF Session. """ global _SESSION - _SESSION = session + _SESSION.session = session def get_default_session_config(): @@ -2322,7 +2325,7 @@ def concatenate(tensors, axis=-1): else: axis = 0 - if py_all([is_sparse(x) for x in tensors]): + if py_all(is_sparse(x) for x in tensors): return sparse_ops.sparse_concat(axis, tensors) else: return array_ops.concat([to_dense(x) for x in tensors], axis) @@ -2552,7 +2555,7 @@ def arange(start, stop=None, step=1, dtype='int32'): result = cast(result, dtype) return result - +@tf_export('keras.backend.tile') def tile(x, n): """Creates a tensor by tiling `x` by `n`. @@ -3123,20 +3126,20 @@ class EagerExecutionFunction(object): updates_ops.append(update) # We set the update ops to run at the end by conditioning it on output[0] - if updates and not outputs: + if updates and not self.outputs: # Edge case; never happens in practice raise ValueError('Cannot create a Keras backend function with updates' ' but no outputs during eager execution.') with ops.control_dependencies(updates_ops): - outputs[0] = array_ops.identity(outputs[0]) + self.outputs[0] = array_ops.identity(self.outputs[0]) # Prepare graph function # TODO(fchollet): can we restrict `captures` to variables actually used in # the relevant subgraph? - graph.inputs = inputs + list(graph.captures.values()) - graph.outputs = outputs + graph.inputs = self.inputs + list(graph.captures.values()) + graph.outputs = self.outputs graph_fn = eager_function.Function(graph) - graph_fn._num_positional_args = len(inputs) + graph_fn._num_positional_args = len(self.inputs) graph_fn._arg_keywords = [] self._graph_fn = graph_fn @@ -3158,8 +3161,13 @@ class EagerExecutionFunction(object): if value is None: raise ValueError( 'You must feed a value for placeholder %s' % (tensor,)) - converted_inputs.append( - ops.convert_to_tensor(value, dtype=tensor.dtype)) + if not isinstance(value, ops.Tensor): + value = ops.convert_to_tensor(value, dtype=tensor.dtype) + if value.dtype != tensor.dtype: + # Temporary workaround due to `convert_to_tensor` not casting floats. + # See b/119637405 + value = math_ops.cast(value, tensor.dtype) + converted_inputs.append(value) outputs = self._graph_fn(*converted_inputs) return [x.numpy() for x in outputs] @@ -3181,7 +3189,7 @@ def function(inputs, outputs, updates=None, name=None, **kwargs): Raises: ValueError: if invalid kwargs are passed in or if in eager execution. """ - if context.executing_eagerly(): + if ops.executing_eagerly_outside_functions(): if kwargs: raise ValueError('Session keyword arguments are not support during ' 'eager execution. You passed: %s' % (kwargs,)) @@ -3242,7 +3250,8 @@ def rnn(step_function, constants=None, unroll=False, input_length=None, - time_major=False): + time_major=False, + zero_output_for_mask=False): """Iterates over the time dimension of a tensor. Arguments: @@ -3280,7 +3289,9 @@ def rnn(step_function, RNN calculation. However, most TensorFlow data is batch-major, so by default this function accepts input and emits output in batch-major form. - + zero_output_for_mask: Boolean. If True, the output for masked timestep + will be zeros, whereas in the False case, output from previous + timestep is returned. Returns: A tuple, `(last_output, outputs, new_states)`. last_output: the latest output of the rnn, of shape `(samples, ...)` @@ -3332,13 +3343,13 @@ def rnn(step_function, # So we need to broadcast the mask to match the shape of inputs. # That's what the tile call does, it just repeats the mask along its # second dimension n times. - def _expand_mask(mask_t, input_t): + def _expand_mask(mask_t, input_t, fixed_dim=1): assert not nest.is_sequence(mask_t) assert not nest.is_sequence(input_t) rank_diff = len(input_t.shape) - len(mask_t.shape) for _ in range(rank_diff): mask_t = array_ops.expand_dims(mask_t) - expand_dims = [1] + input_t.shape.as_list()[1:] + expand_dims = [1] * fixed_dim + input_t.shape.as_list()[fixed_dim:] return array_ops.tile(mask_t, expand_dims) if unroll: @@ -3397,6 +3408,17 @@ def rnn(step_function, last_output = successive_outputs[-1] new_states = successive_states[-1] outputs = array_ops.stack(successive_outputs) + + if zero_output_for_mask: + last_output = array_ops.where( + _expand_mask(mask_list[-1], last_output), + last_output, + zeros_like(last_output)) + outputs = array_ops.where( + _expand_mask(mask, outputs, fixed_dim=2), + outputs, + zeros_like(outputs)) + else: for i in range(time_steps): inp = _get_input_tensor(i) @@ -3490,11 +3512,12 @@ def rnn(step_function, tuple(states) + tuple(constants)) # mask output flat_output = nest.flatten(output) - flat_previous_output = nest.flatten(prev_output) + flat_mask_output = (flat_zero_output if zero_output_for_mask + else nest.flatten(prev_output)) tiled_mask_t = tuple(_expand_mask(mask_t, o) for o in flat_output) flat_new_output = tuple( - array_ops.where(m, o, po) for m, o, po in zip( - tiled_mask_t, flat_output, flat_previous_output)) + array_ops.where(m, o, zo) for m, o, zo in zip( + tiled_mask_t, flat_output, flat_mask_output)) # mask states flat_state = nest.flatten(states) @@ -3503,8 +3526,8 @@ def rnn(step_function, new_state.set_shape(state.shape) tiled_mask_t = tuple(_expand_mask(mask_t, s) for s in flat_state) flat_final_state = tuple( - array_ops.where(m, o, po) - for m, o, po in zip(tiled_mask_t, flat_new_state, flat_state)) + array_ops.where(m, s, ps) + for m, s, ps in zip(tiled_mask_t, flat_new_state, flat_state)) new_states = nest.pack_sequence_as(new_states, flat_final_state) output_ta_t = tuple( @@ -3552,12 +3575,12 @@ def rnn(step_function, **while_loop_kwargs) new_states = final_outputs[2:] - last_time = final_outputs[0] output_ta = final_outputs[1] outputs = tuple(o.stack() for o in output_ta) + last_output = tuple(o[-1] for o in outputs) + outputs = nest.pack_sequence_as(output_time_zero, outputs) - last_output = tuple(o.read(last_time - 1) for o in output_ta) last_output = nest.pack_sequence_as(output_time_zero, last_output) # static shape inference @@ -3662,13 +3685,13 @@ def in_train_phase(x, alt, training=None): if training is None: training = learning_phase() - if training is 1 or training is True: + if training == 1 or training is True: if callable(x): return x() else: return x - elif training is 0 or training is False: + elif training == 0 or training is False: if callable(alt): return alt() else: diff --git a/tensorflow/python/keras/backend_test.py b/tensorflow/python/keras/backend_test.py index d8aa3e9b529..fdb9b281cb2 100644 --- a/tensorflow/python/keras/backend_test.py +++ b/tensorflow/python/keras/backend_test.py @@ -136,7 +136,7 @@ class BackendUtilsTest(test.TestCase): x = keras.Input((3,)) y = keras.layers.BatchNormalization()(x) if not context.executing_eagerly(): - sess.run(variables.global_variables_initializer()) + self.evaluate(variables.global_variables_initializer()) sess.run(y, feed_dict={x: np.random.random((2, 3))}) def test_learning_phase_scope(self): @@ -1069,13 +1069,13 @@ class BackendNNOpsTest(test.TestCase, parameterized.TestCase): initial_states, **kwargs) # check static shape inference - self.assertEquals(last_output.get_shape().as_list(), - [num_samples, output_dim]) - self.assertEquals(outputs.get_shape().as_list(), - [num_samples, timesteps, output_dim]) + self.assertEqual(last_output.get_shape().as_list(), + [num_samples, output_dim]) + self.assertEqual(outputs.get_shape().as_list(), + [num_samples, timesteps, output_dim]) for state in new_states: - self.assertEquals(state.get_shape().as_list(), - [num_samples, output_dim]) + self.assertEqual(state.get_shape().as_list(), + [num_samples, output_dim]) last_output_list[i].append(keras.backend.eval(last_output)) outputs_list[i].append(keras.backend.eval(outputs)) @@ -1173,7 +1173,7 @@ class BackendNNOpsTest(test.TestCase, parameterized.TestCase): self.assertEqual(outputs.get_shape().as_list(), [num_samples, timesteps, output_dim]) # for state in new_states: - # self.assertEquals(state.get_shape().as_list(), + # self.assertEqual(state.get_shape().as_list(), # [num_samples, output_dim]) self.assertEqual(new_states[0].get_shape().as_list(), [num_samples, output_dim]) @@ -1223,6 +1223,121 @@ class BackendNNOpsTest(test.TestCase, parameterized.TestCase): for s, u_s in zip(additional_state_list[2], additional_state_list[3]): self.assertAllClose(s, u_s, atol=1e-04) + def test_rnn_output_and_state_masking_independent(self): + num_samples = 2 + num_timesteps = 4 + state_and_io_size = 2 + mask_last_num_timesteps = 2 # for second sample only + + # a step function that just outputs inputs, + # but increments states +1 per timestep + def step_function(inputs, states): + return inputs, [s + 1 for s in states] + + inputs_vals = np.random.random( + (num_samples, num_timesteps, state_and_io_size)) + initial_state_vals = np.random.random((num_samples, state_and_io_size)) + # masking of two last timesteps for second sample only + mask_vals = np.ones((num_samples, num_timesteps)) + mask_vals[1, -mask_last_num_timesteps:] = 0 + + # outputs expected to be same as inputs for the first sample + expected_outputs = inputs_vals.copy() + # but for the second sample all outputs in masked region should be the same + # as last output before masked region + expected_outputs[1, -mask_last_num_timesteps:] = \ + expected_outputs[1, -(mask_last_num_timesteps + 1)] + + expected_last_state = initial_state_vals.copy() + # first state should be incremented for every timestep (no masking) + expected_last_state[0] += num_timesteps + # second state should not be incremented for last two timesteps + expected_last_state[1] += (num_timesteps - mask_last_num_timesteps) + + # verify same expected output for `unroll=true/false` + inputs = keras.backend.variable(inputs_vals) + initial_states = [keras.backend.variable(initial_state_vals)] + mask = keras.backend.variable(mask_vals) + for unroll in [True, False]: + _, outputs, last_states = keras.backend.rnn( + step_function, + inputs, + initial_states, + mask=mask, + unroll=unroll, + input_length=num_timesteps if unroll else None) + + self.assertAllClose(keras.backend.eval(outputs), expected_outputs) + self.assertAllClose(keras.backend.eval( + last_states[0]), expected_last_state) + + def test_rnn_output_num_dim_larger_than_2_masking(self): + num_samples = 3 + num_timesteps = 4 + num_features = 5 + + def step_function(inputs, states): + outputs = keras.backend.tile(keras.backend.expand_dims(inputs), [1, 1, 2]) + return outputs, [keras.backend.identity(s) for s in states] + # Note: cannot just return states (which can be a problem) -> + # tensorflow/python/ops/resource_variable_ops.py", line 824, in set_shape + # NotImplementedError: ResourceVariable does not implement set_shape() + + inputs_vals = np.random.random((num_samples, num_timesteps, num_features)) + initial_state_vals = np.random.random((num_samples, 6)) + mask_vals = np.ones((num_samples, num_timesteps)) + mask_vals[-1, -1] = 0 # final timestep masked for last sample + + expected_outputs = np.repeat(inputs_vals[..., None], repeats=2, axis=-1) + # for the last sample, the final timestep (in masked region) should be the + # same as the second to final output (before masked region) + expected_outputs[-1, -1] = expected_outputs[-1, -2] + + inputs = keras.backend.variable(inputs_vals) + initial_states = [keras.backend.variable(initial_state_vals)] + mask = keras.backend.variable(mask_vals) + for unroll in [True, False]: + _, outputs, _ = keras.backend.rnn( + step_function, + inputs, + initial_states, + mask=mask, + unroll=unroll, + input_length=num_timesteps if unroll else None) + + self.assertAllClose(keras.backend.eval(outputs), expected_outputs) + + def test_rnn_state_num_dim_larger_than_2_masking(self): + num_samples = 3 + num_timesteps = 4 + + def step_function(inputs, states): + return inputs, [s + 1 for s in states] + + inputs_vals = np.random.random((num_samples, num_timesteps, 5)) + initial_state_vals = np.random.random((num_samples, 6, 7)) + mask_vals = np.ones((num_samples, num_timesteps)) + mask_vals[0, -2:] = 0 # final two timesteps masked for first sample + + expected_last_state = initial_state_vals.copy() + expected_last_state[0] += (num_timesteps - 2) + expected_last_state[1:] += num_timesteps + + inputs = keras.backend.variable(inputs_vals) + initial_states = [keras.backend.variable(initial_state_vals)] + mask = keras.backend.variable(mask_vals) + for unroll in [True, False]: + _, _, last_states = keras.backend.rnn( + step_function, + inputs, + initial_states, + mask=mask, + unroll=unroll, + input_length=num_timesteps if unroll else None) + + self.assertAllClose( + keras.backend.eval(last_states[0]), expected_last_state) + def test_normalize_batch_in_training(self): val = np.random.random((10, 3, 10, 10)) x = keras.backend.variable(val) @@ -1307,6 +1422,7 @@ class TestCTC(test.TestCase): decode_truth[i] == keras.backend.eval(decode_pred_tf[i]))) self.assertAllClose(log_prob_truth, log_prob_pred) + @test_util.run_deprecated_v1 def test_ctc_batch_cost(self): with self.cached_session(): label_lens = np.expand_dims(np.asarray([5, 4]), 1) @@ -1392,6 +1508,7 @@ class TestRandomOps(test.TestCase): class BackendGraphTests(test.TestCase): + @test_util.run_deprecated_v1 def test_is_placeholder(self): x = keras.backend.placeholder(shape=(1,)) self.assertEqual(keras.backend.is_placeholder(x), True) @@ -1431,6 +1548,7 @@ class BackendGraphTests(test.TestCase): output_values = f([None, None]) self.assertEqual(output_values, [5., 6.]) + @test_util.run_deprecated_v1 def test_function_tf_feed_symbols(self): # Test Keras backend functions with TF tensor inputs. with self.cached_session(): @@ -1464,6 +1582,7 @@ class BackendGraphTests(test.TestCase): outs = f([y5, y2, None]) self.assertEqual(outs, [11., 2.]) + @test_util.run_deprecated_v1 def test_function_tf_fetches(self): # Additional operations can be passed to tf.Session().run() via its # `fetches` arguments. In contrast to `updates` argument of @@ -1486,6 +1605,7 @@ class BackendGraphTests(test.TestCase): self.assertEqual(keras.backend.get_session().run(fetches=[x, y]), [11., 5.]) + @test_util.run_deprecated_v1 def test_function_tf_feed_dict(self): # Additional substitutions can be passed to `tf.Session().run()` via its # `feed_dict` arguments. Note that the feed_dict is passed once in the @@ -1518,6 +1638,7 @@ class BackendGraphTests(test.TestCase): self.assertEqual(keras.backend.get_session().run(fetches=[x, y]), [30., 40.]) + @test_util.run_deprecated_v1 def test_function_tf_run_options_with_run_metadata(self): with self.cached_session(): x_placeholder = keras.backend.placeholder(shape=()) @@ -1543,6 +1664,7 @@ class BackendGraphTests(test.TestCase): self.assertEqual(output1, [30.]) self.assertEqual(len(run_metadata.partition_graphs), 0) + @test_util.run_deprecated_v1 def test_function_fetch_callbacks(self): class CallbackStub(object): @@ -1579,6 +1701,7 @@ class BackendGraphTests(test.TestCase): x = keras.backend.placeholder(shape=(3, 4), sparse=True) self.assertEqual(x.get_shape().as_list(), [3, 4]) + @test_util.run_deprecated_v1 def test_batch_normalization(self): # No eager CPU kernel. g_val = np.random.random((3,)) diff --git a/tensorflow/python/keras/callbacks.py b/tensorflow/python/keras/callbacks.py index fde17cb6bc4..2d7d5a415d4 100644 --- a/tensorflow/python/keras/callbacks.py +++ b/tensorflow/python/keras/callbacks.py @@ -24,7 +24,6 @@ import copy import csv import io import json -import math import os import time @@ -35,7 +34,6 @@ from tensorflow.python.data.ops import iterator_ops from tensorflow.python.eager import context from tensorflow.python.framework import dtypes from tensorflow.python.keras import backend as K -from tensorflow.python.keras.engine.training_utils import standardize_input_data from tensorflow.python.keras.utils.data_utils import Sequence from tensorflow.python.keras.utils.generic_utils import Progbar from tensorflow.python.ops import array_ops @@ -54,17 +52,14 @@ except ImportError: requests = None +# pylint: disable=protected-access def configure_callbacks(callbacks, model, do_validation=False, - val_inputs=None, - val_targets=None, - val_sample_weights=None, batch_size=None, epochs=None, steps_per_epoch=None, samples=None, - validation_steps=None, verbose=1, count_mode='steps', mode='train'): @@ -74,17 +69,10 @@ def configure_callbacks(callbacks, callbacks: List of Callbacks. model: Model being trained. do_validation: Whether or not validation loop will be run. - val_inputs: Inputs to Model for validation loop. Can be any - data format Keras accepts. - val_targets: Targets for Model for validation loop. Can be any - data format Keras accepts. - val_sample_weights: Sample weights for Model for validation loop. - Can be any data format Keras accepts. batch_size: Number of samples per batch. epochs: Number of epoch to train. steps_per_epoch: Number of batches to run per training epoch. samples: Number of training samples. - validation_steps: Number of batches to run per validation epoch. verbose: int, 0 or 1. Keras logging verbosity to pass to ProgbarLogger. count_mode: One of 'steps' or 'samples'. Per-batch or per-sample count. mode: String. One of 'train', 'test', or 'predict'. Which loop mode to @@ -114,24 +102,17 @@ def configure_callbacks(callbacks, callback_list = CallbackList(callbacks) # Set callback model - callback_model = model._get_callback_model() # pylint: disable=protected-access - if do_validation and val_inputs and not context.executing_eagerly(): - # Need to create the eval_function before start of the first epoch - # because TensorBoard callback on_epoch_begin adds summary to the - # list of fetches of the eval_function - callback_model._make_eval_function() # pylint: disable=protected-access + callback_model = model._get_callback_model() callback_list.set_model(callback_model) # Set callback parameters callback_metrics = [] # When we have deferred build scenario with iterator input, we will compile # when we standardize first batch of data. - if mode != 'predict' and model._is_compiled: # pylint: disable=protected-access + if mode != 'predict' and hasattr(model, 'metrics_names'): callback_metrics = copy.copy(model.metrics_names) if do_validation: callback_metrics += ['val_' + n for n in model.metrics_names] - if validation_steps is None and isinstance(val_inputs, Sequence): - validation_steps = len(val_inputs) callback_params = { 'batch_size': batch_size, 'epochs': epochs, @@ -140,27 +121,19 @@ def configure_callbacks(callbacks, 'verbose': verbose, 'do_validation': do_validation, 'metrics': callback_metrics, - 'validation_steps': validation_steps } callback_list.set_params(callback_params) - # Pass validation data to callbacks - # TODO(omalleyt): remove this once val hooks are ready. - if not val_inputs: - val_data = [] - elif _is_generator_like(val_inputs): - val_data = val_inputs - else: - val_data = val_inputs + val_targets - if val_sample_weights: - val_data += val_sample_weights - if not isinstance(K.learning_phase(), int): - val_data += [0.] - for cbk in callbacks: - cbk.validation_data = val_data + if (do_validation and not model._distribution_strategy and + not model.run_eagerly): + # Need to create the eval_function before start of the first epoch + # because TensorBoard callback on_epoch_begin adds summary to the + # list of fetches of the eval_function + callback_model._make_eval_function() callback_list.model.stop_training = False return callback_list +# pylint: enable=protected-access def _is_generator_like(data): @@ -491,7 +464,8 @@ class ProgbarLogger(Callback): self.progbar = Progbar( target=self.target, verbose=self.verbose, - stateful_metrics=self.stateful_metrics) + stateful_metrics=self.stateful_metrics, + unit_name='step' if self.use_steps else 'sample') def on_batch_begin(self, batch, logs=None): if self.seen < self.target: @@ -953,6 +927,7 @@ class TensorBoard(Callback): self.batch_size = batch_size self._current_batch = 0 self._total_batches_seen = 0 + self._total_val_batches_seen = 0 self.embeddings_freq = embeddings_freq self.embeddings_layer_names = embeddings_layer_names self.embeddings_metadata = embeddings_metadata @@ -1041,8 +1016,10 @@ class TensorBoard(Callback): # If both embedding_freq and embeddings_data are available, we will # visualize embeddings. if self.embeddings_freq and self.embeddings_data is not None: - self.embeddings_data = standardize_input_data(self.embeddings_data, - model.input_names) + # Avoid circular dependency. + from tensorflow.python.keras.engine import training_utils # pylint: disable=g-import-not-at-top + self.embeddings_data = training_utils.standardize_input_data( + self.embeddings_data, model.input_names) # If embedding_layer_names are not provided, get all of the embedding # layers from the model. @@ -1107,10 +1084,8 @@ class TensorBoard(Callback): projector.visualize_embeddings(self.writer, config) def _fetch_callback(self, summary): - self.writer.add_summary( - summary, - self._epoch + self._current_val_batch / self._validation_batches) - self._current_val_batch += 1 + self.writer.add_summary(summary, self._total_val_batches_seen) + self._total_val_batches_seen += 1 def _write_custom_summaries(self, step, logs=None): """Writes metrics out as custom scalar summaries. @@ -1141,22 +1116,6 @@ class TensorBoard(Callback): self.writer.add_summary(summary, step) self.writer.flush() - def on_train_begin(self, logs=None): - """Checks if histogram summaries can be run.""" - # will never be set when in eager - if self.histogram_freq: - if self.params.get('validation_steps', None) is not None: - self._validation_batches = self.params['validation_steps'] - elif self.validation_data: - self._validation_batches = math.ceil( - self.validation_data[0].shape[0] / self.batch_size) - else: - raise ValueError('If printing histograms, validation data must be ' - 'provided.') - if self._validation_batches == 0: - raise ValueError( - 'If printing histograms, validation data must have length > 0.') - def on_batch_end(self, batch, logs=None): """Writes scalar summaries for metrics on every training batch.""" # Don't output batch_size and batch number as Tensorboard summaries @@ -1177,7 +1136,6 @@ class TensorBoard(Callback): # check if histogram summary should be run for this epoch if self.histogram_freq and epoch % self.histogram_freq == 0: self._epoch = epoch - self._current_val_batch = 0 # pylint: disable=protected-access # add the histogram summary op if it should run this epoch if self.merged not in self.model._eval_function.fetches: diff --git a/tensorflow/python/keras/callbacks_test.py b/tensorflow/python/keras/callbacks_test.py index 9d9ede22c01..6c9a382b327 100644 --- a/tensorflow/python/keras/callbacks_test.py +++ b/tensorflow/python/keras/callbacks_test.py @@ -36,7 +36,6 @@ from tensorflow.python.framework import test_util from tensorflow.python.keras import testing_utils from tensorflow.python.platform import test from tensorflow.python.platform import tf_logging as logging -from tensorflow.python.summary.writer import writer_cache from tensorflow.python.training import adam try: @@ -404,6 +403,7 @@ class KerasCallbacksTest(test.TestCase): float(keras.backend.get_value( model.optimizer.lr)) - 0.01 / 4) < keras.backend.epsilon() + @test_util.run_deprecated_v1 def test_ReduceLROnPlateau(self): with self.cached_session(): np.random.seed(1337) @@ -675,6 +675,7 @@ class KerasCallbacksTest(test.TestCase): self.assertEqual(len(loss), 1) self.assertEqual(loss[0], np.inf) + @test_util.run_deprecated_v1 def test_TensorBoard(self): np.random.seed(1337) @@ -778,78 +779,7 @@ class KerasCallbacksTest(test.TestCase): data_generator(True), len(x_train), epochs=2, callbacks=cbks) assert os.path.exists(temp_dir) - def test_TensorBoard_histogram_freq_must_have_validation_data(self): - np.random.seed(1337) - tmpdir = self.get_temp_dir() - self.addCleanup(shutil.rmtree, tmpdir, ignore_errors=True) - - with self.cached_session(): - filepath = os.path.join(tmpdir, 'logs') - - (x_train, y_train), (x_test, y_test) = testing_utils.get_test_data( - train_samples=TRAIN_SAMPLES, - test_samples=TEST_SAMPLES, - input_shape=(INPUT_DIM,), - num_classes=NUM_CLASSES) - y_test = keras.utils.to_categorical(y_test) - y_train = keras.utils.to_categorical(y_train) - - def data_generator(train): - if train: - max_batch_index = len(x_train) // BATCH_SIZE - else: - max_batch_index = len(x_test) // BATCH_SIZE - i = 0 - while 1: - if train: - # simulate multi-input/output models - yield (x_train[i * BATCH_SIZE: (i + 1) * BATCH_SIZE], - y_train[i * BATCH_SIZE: (i + 1) * BATCH_SIZE]) - else: - yield (x_test[i * BATCH_SIZE: (i + 1) * BATCH_SIZE], - y_test[i * BATCH_SIZE: (i + 1) * BATCH_SIZE]) - i += 1 - i %= max_batch_index - - inp = keras.Input((INPUT_DIM,)) - hidden = keras.layers.Dense(2, activation='relu')(inp) - hidden = keras.layers.Dropout(0.1)(hidden) - output = keras.layers.Dense(NUM_CLASSES, activation='softmax')(hidden) - model = keras.models.Model(inputs=inp, outputs=output) - model.compile(loss='categorical_crossentropy', - optimizer='sgd', - metrics=['accuracy']) - - # we must generate new callbacks for each test, as they aren't stateless - def callbacks_factory(histogram_freq): - return [keras.callbacks.TensorBoard( - log_dir=filepath, - histogram_freq=histogram_freq, - write_images=True, write_grads=True, - batch_size=5)] - - # fit w/o validation data should raise ValueError if histogram_freq > 0 - cbs = callbacks_factory(histogram_freq=1) - with self.assertRaises(ValueError): - model.fit( - x_train, y_train, batch_size=BATCH_SIZE, callbacks=cbs, epochs=3) - - for cb in cbs: - cb.on_train_end() - - # fit generator without validation data should raise ValueError if - # histogram_freq > 0 - cbs = callbacks_factory(histogram_freq=1) - with self.assertRaises(ValueError): - model.fit_generator( - data_generator(True), len(x_train), epochs=2, callbacks=cbs) - - for cb in cbs: - cb.on_train_end() - - # Make sure file writer cache is clear to avoid failures during cleanup. - writer_cache.FileWriterCache.clear() - + @test_util.run_deprecated_v1 def test_TensorBoard_multi_input_output(self): np.random.seed(1337) tmpdir = self.get_temp_dir() @@ -921,6 +851,7 @@ class KerasCallbacksTest(test.TestCase): callbacks=callbacks_factory(histogram_freq=1)) assert os.path.isdir(filepath) + @test_util.run_deprecated_v1 def test_Tensorboard_histogram_summaries_in_test_function(self): class FileWriterStub(object): @@ -996,8 +927,9 @@ class KerasCallbacksTest(test.TestCase): epochs=3, verbose=0) - self.assertAllEqual(tsb.writer.steps_seen, [0, 0.5, 1, 1.5, 2, 2.5]) + self.assertAllEqual(tsb.writer.steps_seen, [0, 1, 2, 3, 4, 5]) + @test_util.run_deprecated_v1 def test_Tensorboard_histogram_summaries_with_generator(self): np.random.seed(1337) tmpdir = self.get_temp_dir() @@ -1129,6 +1061,7 @@ class KerasCallbacksTest(test.TestCase): assert os.path.exists(temp_dir) + @test_util.run_deprecated_v1 def test_Tensorboard_batch_logging(self): class FileWriterStub(object): @@ -1163,6 +1096,7 @@ class KerasCallbacksTest(test.TestCase): self.assertEqual(tb_cbk.writer.summary_values, [0., 1., 2., 3., 4.]) self.assertEqual(tb_cbk.writer.summary_tags, ['batch_acc'] * 5) + @test_util.run_deprecated_v1 def test_Tensorboard_epoch_and_batch_logging(self): class FileWriterStub(object): @@ -1234,6 +1168,7 @@ class KerasCallbacksTest(test.TestCase): self.assertTrue(os.path.exists(temp_dir)) + @test_util.run_deprecated_v1 def test_TensorBoard_update_freq(self): class FileWriterStub(object): @@ -1325,6 +1260,7 @@ class KerasCallbacksTest(test.TestCase): callbacks=cbks, epochs=1) + @test_util.run_deprecated_v1 def test_fit_generator_with_callback(self): class TestCallback(keras.callbacks.Callback): diff --git a/tensorflow/python/keras/engine/__init__.py b/tensorflow/python/keras/engine/__init__.py index 26aed34766f..005f6462ffa 100644 --- a/tensorflow/python/keras/engine/__init__.py +++ b/tensorflow/python/keras/engine/__init__.py @@ -20,10 +20,10 @@ from __future__ import print_function # TODO(fchollet): Remove hourglass imports once external code is done importing # non-public APIs. -from tensorflow.python.keras.engine.base_layer import InputSpec from tensorflow.python.keras.engine.base_layer import Layer from tensorflow.python.keras.engine.input_layer import Input from tensorflow.python.keras.engine.input_layer import InputLayer +from tensorflow.python.keras.engine.input_spec import InputSpec from tensorflow.python.keras.utils.layer_utils import get_source_inputs del absolute_import diff --git a/tensorflow/python/keras/engine/base_layer.py b/tensorflow/python/keras/engine/base_layer.py index 23419ae1503..8e353003429 100644 --- a/tensorflow/python/keras/engine/base_layer.py +++ b/tensorflow/python/keras/engine/base_layer.py @@ -18,8 +18,6 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function -import collections as collections_lib -import enum # pylint: disable=g-bad-import-order import functools import inspect # Necessary supplement to tf_inspect to deal with variadic args. @@ -36,13 +34,14 @@ from tensorflow.python.keras import backend from tensorflow.python.keras import constraints from tensorflow.python.keras import initializers from tensorflow.python.keras import regularizers +from tensorflow.python.keras.engine import base_layer_utils +from tensorflow.python.keras.engine import input_spec from tensorflow.python.keras.utils import generic_utils from tensorflow.python.keras.utils import tf_utils # A module that only depends on `keras.layers` import these from here. from tensorflow.python.keras.utils.generic_utils import to_snake_case # pylint: disable=unused-import from tensorflow.python.keras.utils.tf_utils import is_tensor_or_tensor_list # pylint: disable=unused-import from tensorflow.python.ops import array_ops -from tensorflow.python.ops import init_ops from tensorflow.python.ops import math_ops from tensorflow.python.ops import variables as tf_variables from tensorflow.python.training.checkpointable import base as checkpointable @@ -54,20 +53,6 @@ from tensorflow.python.util.tf_export import tf_export from tensorflow.tools.docs import doc_controls -class CallConvention(enum.Enum): - """Calling conventions for passing `Layer` inputs to `Layer.call`.""" - # The Layer takes inputs as its first argument, named "inputs" for - # compatibility with the signature of Layer.__call__. This is the mode assumed - # for Layers which are not subclassed Models. - EXPLICIT_INPUTS_ARGUMENT = 1 - # The Layer takes a single positional argument, not named "inputs". It's - # treated like an "inputs" argument. - SINGLE_POSITIONAL_ARGUMENT = 2 - # The Layer has multiple positional arguments to which its inputs should be - # bound. - POSITIONAL_ARGUMENTS_ARE_INPUTS = 3 - - @tf_export('keras.layers.Layer') class Layer(checkpointable.CheckpointableBase): """Base layer class. @@ -102,10 +87,6 @@ class Layer(checkpointable.CheckpointableBase): name: The name of the layer (string). dtype: Default dtype of the layer's weights (default of `None` means use the type of the first input). - trainable_variables: List of trainable variables. - non_trainable_variables: List of non-trainable variables. - variables: List of all variables of this layer, trainable and - non-trainable. updates: List of update ops of this layer. losses: List of losses added by this layer. trainable_weights: List of variables to be included in backprop. @@ -150,9 +131,9 @@ class Layer(checkpointable.CheckpointableBase): self.built = False # Provides information about which inputs are compatible with the layer. self.input_spec = None + self.supports_masking = False self._init_set_name(name) - self._activity_regularizer = kwargs.pop('activity_regularizer', None) self._trainable_weights = [] self._non_trainable_weights = [] @@ -170,29 +151,25 @@ class Layer(checkpointable.CheckpointableBase): # in eager mode or graph mode alternatively, we need to keep track of # eager losses and symbolic losses via separate attributes. self._eager_losses = [] + # A list of metric instances corresponding to the symbolic metric tensors + # added using the `add_metric` API. + self._metrics = [] + # TODO(psv): Remove this property. + # A dictionary that maps metric names to metric result tensors. The results + # are the running averages of metric values over an epoch. + self._metrics_tensors = {} self._dtype = None if dtype is None else dtypes.as_dtype(dtype).name self._call_fn_args = function_utils.fn_args(self.call) self._compute_previous_mask = ('mask' in self._call_fn_args or hasattr(self, 'compute_mask')) - self._call_convention = CallConvention.EXPLICIT_INPUTS_ARGUMENT + self._call_convention = (base_layer_utils + .CallConvention.EXPLICIT_INPUTS_ARGUMENT) # These lists will be filled via successive calls # to self._add_inbound_node(). self._inbound_nodes = [] self._outbound_nodes = [] - self.supports_masking = False - - # Mark if a layer supports using graph functions in the eager - # fit/predict/evaluate loop - # TODO(kaftan): merge this with the _static_graph_friendly flag once - # enough eager function bugs involving control flow / tensorarrays have - # been fixed, and static-graph-friendly layers will almost always work in - # eager graph functions. - # We conservatively make this flag opt-in for now to avoid causing existing - # custom layers to crash. - self._can_use_graph_functions = False - call_argspec = tf_inspect.getfullargspec(self.call) if 'training' in call_argspec.args: self._expects_training_arg = True @@ -200,7 +177,7 @@ class Layer(checkpointable.CheckpointableBase): self._expects_training_arg = False # Whether the `call` method can be used to build a TF graph without issues. - self._static_graph_friendly = True + self._call_is_graph_friendly = True # Manage input shape information if passed. if 'input_shape' in kwargs or 'batch_input_shape' in kwargs: @@ -222,286 +199,34 @@ class Layer(checkpointable.CheckpointableBase): else: self._initial_weights = None - @property - def _is_static_graph_friendly(self): - return self._static_graph_friendly - - @_is_static_graph_friendly.setter - def _is_static_graph_friendly(self, value): - if value not in {True, False}: - raise ValueError('`static_graph_friendly` requires a boolean value. ' - 'Received: {}'.format(value)) - self._static_graph_friendly = value - - def _init_set_name(self, name, zero_based=True): - if not name: - self._name = unique_layer_name( - generic_utils.to_snake_case(self.__class__.__name__), - zero_based=zero_based) - else: - self._name = name - - @property - def dtype(self): - return self._dtype - - @property - def name(self): - return self._name - - @property - def activity_regularizer(self): - """Optional regularizer function for the output of this layer.""" - return self._activity_regularizer - - @activity_regularizer.setter - def activity_regularizer(self, regularizer): - """Optional regularizer function for the output of this layer.""" - self._activity_regularizer = self._no_dependency(regularizer) - - @property - def trainable_weights(self): - return self._trainable_weights if self.trainable else [] - - @property - def non_trainable_weights(self): - if self.trainable: - return self._non_trainable_weights - else: - return self._trainable_weights + self._non_trainable_weights - - @property - def trainable_variables(self): - return self.trainable_weights - - @property - def non_trainable_variables(self): - return self.non_trainable_weights - - @property - def weights(self): - """Returns the list of all layer variables/weights. - - Returns: - A list of variables. - """ - return self.trainable_weights + self.non_trainable_weights - - @property - def variables(self): - """Returns the list of all layer variables/weights. - - Returns: - A list of variables. - """ - return self.weights - - @property - def updates(self): - if context.executing_eagerly(): - raise RuntimeError('Layer.updates not supported in Eager mode.') - if not self.trainable and not self.stateful: - return [] - return self._updates - - @doc_controls.for_subclass_implementers - def add_update(self, updates, inputs=None): - """Add update op(s), potentially dependent on layer inputs. - - Weight updates (for instance, the updates of the moving mean and variance - in a BatchNormalization layer) may be dependent on the inputs passed - when calling a layer. Hence, when reusing the same layer on - different inputs `a` and `b`, some entries in `layer.updates` may be - dependent on `a` and some on `b`. This method automatically keeps track - of dependencies. - - The `get_updates_for` method allows to retrieve the updates relevant to a - specific set of inputs. - - This call is ignored when eager execution is enabled (in that case, variable - updates are run on the fly and thus do not need to be tracked for later - execution). - - Arguments: - updates: Update op, or list/tuple of update ops. - inputs: If anything other than None is passed, it signals the updates - are conditional on some of the layer's inputs, - and thus they should only be run where these inputs are available. - This is the case for BatchNormalization updates, for instance. - If None, the updates will be taken into account unconditionally, - and you are responsible for making sure that any dependency they might - have is available at runtime. - A step counter might fall into this category. - """ - if context.executing_eagerly(): - return # Updates already applied when in eager mode. - - def process_update(x): - if isinstance(x, ops.Operation): - return x - elif hasattr(x, 'op'): - return x.op - else: - return ops.convert_to_tensor(x) - - updates = generic_utils.to_list(updates) - updates = [process_update(x) for x in updates] - self._updates += updates - if inputs is None: - for u in updates: - u._unconditional_update = True # pylint: disable=protected-access - else: - for u in updates: - u._unconditional_update = False # pylint: disable=protected-access - - def get_updates_for(self, inputs): - """Retrieves updates relevant to a specific set of inputs. - - Arguments: - inputs: Input tensor or list/tuple of input tensors. - - Returns: - List of update ops of the layer that depend on `inputs`. - - Raises: - RuntimeError: If called in Eager mode. - """ - if context.executing_eagerly(): - raise RuntimeError('`get_updates_for()` not supported in Eager mode.') - - # Updates disabled if layer is not trainable and not explicitly stateful. - if not self.trainable and not self.stateful: - return [] - - if inputs is None: - # Requesting unconditional updates. - return [x for x in self.updates if x._unconditional_update] # pylint: disable=protected-access - - # Requesting input-conditional updates. - inputs = nest.flatten(inputs) - reachable = tf_utils.get_reachable_from_inputs(inputs, self.updates) - updates = [] - for update in self.updates: - if update in reachable: - updates.append(update) - return updates - - @property - def losses(self): - """Losses which are associated with this `Layer`. - - Variable regularization tensors are created when this property is accessed, - so it is eager safe: accessing `losses` under a `tf.GradientTape` will - propagate gradients back to the corresponding variables. - - Returns: - A list of tensors. - """ - collected_losses = [] - if context.executing_eagerly(): - collected_losses.extend(self._eager_losses) - else: - collected_losses.extend(self._losses) - for regularizer in self._callable_losses: - loss_tensor = regularizer() - if loss_tensor is not None: - collected_losses.append(loss_tensor) - return collected_losses - - @doc_controls.for_subclass_implementers - def add_loss(self, losses, inputs=None): - """Add loss tensor(s), potentially dependent on layer inputs. - - Some losses (for instance, activity regularization losses) may be dependent - on the inputs passed when calling a layer. Hence, when reusing the same - layer on different inputs `a` and `b`, some entries in `layer.losses` may - be dependent on `a` and some on `b`. This method automatically keeps track - of dependencies. - - The `get_losses_for` method allows to retrieve the losses relevant to a - specific set of inputs. - - Note that `add_loss` is not supported when executing eagerly. Instead, - variable regularizers may be added through `add_variable`. Activity - regularization is not supported directly (but such losses may be returned - from `Layer.call()`). - - Arguments: - losses: Loss tensor, or list/tuple of tensors. Rather than tensors, losses - may also be zero-argument callables which create a loss tensor. - inputs: Ignored when executing eagerly. If anything other than None is - passed, it signals the losses are conditional on some of the layer's - inputs, and thus they should only be run where these inputs are - available. This is the case for activity regularization losses, for - instance. If `None` is passed, the losses are assumed - to be unconditional, and will apply across all dataflows of the layer - (e.g. weight regularization losses). - """ - losses = generic_utils.to_list(losses) - - def _tag_unconditional(loss): - if callable(loss): - loss = loss() - if loss is None: - return None # Will be filtered out when computing the .losses property - if not tensor_util.is_tensor(loss): - loss = ops.convert_to_tensor(loss, dtype=backend.floatx()) - loss._unconditional_loss = (inputs is None) # pylint: disable=protected-access - return loss - - for loss in losses: - if callable(loss): - self._callable_losses.append( - functools.partial(_tag_unconditional, loss)) - else: - if context.executing_eagerly(): - self._eager_losses.append(_tag_unconditional(loss)) - else: - self._losses.append(_tag_unconditional(loss)) - - def get_losses_for(self, inputs): - """Retrieves losses relevant to a specific set of inputs. - - Arguments: - inputs: Input tensor or list/tuple of input tensors. - - Returns: - List of loss tensors of the layer that depend on `inputs`. - - Raises: - RuntimeError: If called in Eager mode. - """ - if context.executing_eagerly(): - raise RuntimeError('Layer.get_losses_for not supported in Eager mode.') - - if inputs is None: - # Requesting unconditional losses. - return [x for x in self.losses if x._unconditional_loss] # pylint: disable=protected-access - - # Requesting input-conditional losses. - inputs = nest.flatten(inputs) - # Retrieve the set of tensors in the TF graph that depend on `inputs`. - # The losses we want to return will be part of this set. - # To avoid unnecessary work, we stop the search in case all of - # `self.losses` have been retrieved. - reachable = tf_utils.get_reachable_from_inputs(inputs, self.losses) - losses = [] - for loss in self.losses: - if loss in reachable: - losses.append(loss) - return losses - - def _name_scope(self): - return self.name - def build(self, input_shape): - """Creates the variables of the layer.""" + """Creates the variables of the layer (optional, for subclass implementers). + + This is a method that implementers of subclasses of `Layer` or `Model` + can override if they need a state-creation step in-between + layer instantiation and layer call. + + This is typically used to create the weights of `Layer` subclasses. + + Arguments: + input_shape: Instance of `TensorShape`, or list of instances of + `TensorShape` if the layer expects a list of inputs + (one instance per input). + """ self.built = True @doc_controls.for_subclass_implementers - def add_variable(self, *args, **kwargs): - """Alias for `add_weight`.""" - return self.add_weight(*args, **kwargs) + def call(self, inputs, **kwargs): # pylint: disable=unused-argument + """This is where the layer's logic lives. + + Arguments: + inputs: Input tensor, or list/tuple of input tensors. + **kwargs: Additional keyword arguments. + + Returns: + A tensor or list/tuple of tensors. + """ + return inputs @doc_controls.for_subclass_implementers def add_weight(self, @@ -604,7 +329,7 @@ class Layer(checkpointable.CheckpointableBase): shape=shape, # TODO(allenl): a `make_variable` equivalent should be added as a # `Checkpointable` method. - getter=getter or make_variable, + getter=getter or base_layer_utils.make_variable, # Manage errors in Layer rather than Checkpointable. overwrite=True, initializer=initializer, @@ -630,341 +355,45 @@ class Layer(checkpointable.CheckpointableBase): self._non_trainable_weights.append(variable) return variable - def _handle_weight_regularization(self, name, variable, regularizer): - """Create lambdas which compute regularization losses.""" + def get_config(self): + """Returns the config of the layer. - def _loss_for_variable(v): - """Creates a regularization loss `Tensor` for variable `v`.""" - with ops.colocate_with(v): - with ops.name_scope(name + '/Regularizer'): - regularization = regularizer(v) - return regularization + A layer config is a Python dictionary (serializable) + containing the configuration of a layer. + The same layer can be reinstantiated later + (without its trained weights) from this configuration. - if isinstance(variable, tf_variables.PartitionedVariable): - for v in variable: - self.add_loss(functools.partial(_loss_for_variable, v)) - else: - self.add_loss(functools.partial(_loss_for_variable, variable)) + The config of a layer does not include connectivity + information, nor the layer class name. These are handled + by `Network` (one layer of abstraction above). - def _handle_activity_regularization(self, inputs, outputs): - # Apply activity regularization. - # Note that it should be applied every time the layer creates a new - # output, since it is output-specific. - if self._activity_regularizer: - output_list = nest.flatten(outputs) - with ops.name_scope('ActivityRegularizer'): - for output in output_list: - activity_loss = self._activity_regularizer(output) - batch_size = math_ops.cast( - array_ops.shape(output)[0], activity_loss.dtype) - # Make activity regularization strength batch-agnostic. - mean_activity_loss = activity_loss / batch_size - self.add_loss(mean_activity_loss, inputs=inputs) + Returns: + Python dictionary. + """ + config = {'name': self.name, 'trainable': self.trainable} + if hasattr(self, '_batch_input_shape'): + config['batch_input_shape'] = self._batch_input_shape + if hasattr(self, 'dtype'): + config['dtype'] = self.dtype + return config - @doc_controls.for_subclass_implementers - def call(self, inputs, **kwargs): # pylint: disable=unused-argument - """This is where the layer's logic lives. + @classmethod + def from_config(cls, config): + """Creates a layer from its config. + + This method is the reverse of `get_config`, + capable of instantiating the same layer from the config + dictionary. It does not handle layer connectivity + (handled by Network), nor weights (handled by `set_weights`). Arguments: - inputs: Input tensor, or list/tuple of input tensors. - **kwargs: Additional keyword arguments. + config: A Python dictionary, typically the + output of get_config. Returns: - A tensor or list/tuple of tensors. + A layer instance. """ - return inputs - - def __call__(self, inputs, *args, **kwargs): - """Wraps `call`, applying pre- and post-processing steps. - - Arguments: - inputs: input tensor(s). - *args: additional positional arguments to be passed to `self.call`. - **kwargs: additional keyword arguments to be passed to `self.call`. - - Returns: - Output tensor(s). - - Note: - - The following optional keyword arguments are reserved for specific uses: - * `training`: Boolean scalar tensor of Python boolean indicating - whether the `call` is meant for training or inference. - * `mask`: Boolean input mask. - - If the layer's `call` method takes a `mask` argument (as some Keras - layers do), its default value will be set to the mask generated - for `inputs` by the previous layer (if `input` did come from - a layer that generated a corresponding mask, i.e. if it came from - a Keras layer with masking support. - - Raises: - ValueError: if the layer's `call` method returns None (an invalid value). - """ - input_list = nest.flatten(inputs) - - if context.executing_eagerly(): - # Accept NumPy inputs by converting to Tensors when executing eagerly. - if all([isinstance(x, (np.ndarray, float, int)) for x in input_list]): - inputs = nest.map_structure(ops.convert_to_tensor, inputs) - input_list = nest.flatten(inputs) - - # We will attempt to build a TF graph if & only if all inputs are symbolic. - # This is always the case in graph mode. It can also be the case in eager - # mode when all inputs can be traced back to `keras.Input()` (when building - # models using the functional API). - build_graph = tf_utils.are_all_symbolic_tensors(input_list) - executing_eagerly = context.executing_eagerly() - - # Handle Keras mask propagation from previous layer to current layer. - previous_mask = None - if build_graph and (not hasattr(self, '_compute_previous_mask') or - self._compute_previous_mask): - previous_mask = collect_previous_mask(inputs) - if not hasattr(self, '_call_fn_args'): - self._call_fn_args = self._no_dependency( - function_utils.fn_args(self.call)) - if ('mask' in self._call_fn_args and 'mask' not in kwargs and - not generic_utils.is_all_none(previous_mask)): - # The previous layer generated a mask, and mask was not explicitly pass - # to __call__, hence we set previous_mask as the default value. - kwargs['mask'] = previous_mask - - input_shapes = None - - with ops.name_scope(self._name_scope()): - if not self.built: - # Check input assumptions set before layer building, e.g. input rank. - self._assert_input_compatibility(inputs) - if input_list and self._dtype is None: - try: - self._dtype = input_list[0].dtype.base_dtype.name - except AttributeError: - pass - - if all(hasattr(x, 'shape') for x in input_list): - input_shapes = nest.map_structure(lambda x: x.shape, inputs) - - if (not hasattr(self, '_is_graph_network') or - self.__class__.__name__ == 'Sequential' or - not hasattr(self.build, '_is_default')): - # Only if self is a layer, an instance of a sequential model, or - # the user has manually overwritten the build method do we need to - # build it. - self.build(input_shapes) - # We must set self.built since user defined build functions are not - # constrained to set self.built. - self.built = True - - # Check input assumptions set after layer building, e.g. input shape. - if build_graph: - # Symbolic execution on symbolic tensors. We will attempt to build - # the corresponding TF subgraph inside `backend.get_graph()` - self._assert_input_compatibility(inputs) - graph = backend.get_graph() - with graph.as_default(): - if not executing_eagerly: - # In graph mode, failure to build the layer's graph - # implies a user-side bug. We don't catch exceptions. - outputs = self.call(inputs, *args, **kwargs) - else: - try: - outputs = self.call(inputs, *args, **kwargs) - except Exception: # pylint: disable=broad-except - # Any issue during graph-building means we will later run the - # model in eager mode, whether the issue was related to - # graph mode or not. This provides a nice debugging experience. - self._is_static_graph_friendly = False - # We will use static shape inference to return symbolic tensors - # matching the specifications of the layer outputs. - # Since we have set `self._is_static_graph_friendly = False`, - # we will never attempt to run the underlying TF graph (which is - # disconnected). - # TODO(fchollet): consider py_func as an alternative, which - # would enable us to run the underlying graph if needed. - input_shapes = nest.map_structure(lambda x: x.shape, inputs) - output_shapes = self.compute_output_shape(input_shapes) - outputs = nest.map_structure( - lambda shape: backend.placeholder(shape, dtype=self.dtype), - output_shapes) - - if outputs is None: - raise ValueError('A layer\'s `call` method should return a ' - 'Tensor or a list of Tensors, not None ' - '(layer: ' + self.name + ').') - self._handle_activity_regularization(inputs, outputs) - self._set_mask_metadata(inputs, outputs, previous_mask) - if have_all_keras_metadata(inputs): - inputs, outputs = self._set_connectivity_metadata_( - inputs, outputs, args, kwargs) - if hasattr(self, '_set_inputs') and not self.inputs: - # Subclassed network: explicitly set metadata normally set by - # a call to self._set_inputs(). - # This is not relevant in eager execution. - self._set_inputs(inputs, outputs) - else: - # Eager execution on data tensors. - outputs = self.call(inputs, *args, **kwargs) - self._handle_activity_regularization(inputs, outputs) - return outputs - - if not context.executing_eagerly(): - # Optionally load weight values specified at layer instantiation. - # TODO(fchollet): consider enabling this with eager execution too. - if (hasattr(self, '_initial_weights') and - self._initial_weights is not None): - self.set_weights(self._initial_weights) - del self._initial_weights - return outputs - - def apply(self, inputs, *args, **kwargs): - """Apply the layer on a input. - - This simply wraps `self.__call__`. - - Arguments: - inputs: Input tensor(s). - *args: additional positional arguments to be passed to `self.call`. - **kwargs: additional keyword arguments to be passed to `self.call`. - - Returns: - Output tensor(s). - """ - return self.__call__(inputs, *args, **kwargs) - - def _set_mask_metadata(self, inputs, outputs, previous_mask): - # In some cases the mask of the outputs has already been computed by - # inner layers and does not need to be recomputed by this layer. - mask_already_computed = all( - hasattr(x, '_keras_mask') for x in generic_utils.to_list(outputs)) - if hasattr(self, 'compute_mask') and not mask_already_computed: - output_mask = self.compute_mask(inputs, previous_mask) - else: - output_mask = None - if isinstance(outputs, (list, tuple)): - if output_mask is None: - output_mask = [None for _ in range(len(outputs))] - for x, m in zip(outputs, output_mask): - try: - x._keras_mask = m # pylint: disable=protected-access - except AttributeError: - pass # C type such as dict. Masking not supported in this case. - else: - try: - outputs._keras_mask = output_mask # pylint: disable=protected-access - except AttributeError: - pass # C type such as dict. Masking not supported in this case. - - def _set_connectivity_metadata_(self, inputs, outputs, args, kwargs): - call_convention = getattr(self, '_call_convention', - CallConvention.EXPLICIT_INPUTS_ARGUMENT) - if args: - if call_convention == CallConvention.EXPLICIT_INPUTS_ARGUMENT: - raise TypeError( - 'This layer ("{}") takes an `inputs` argument in `call()`, ' - 'and only the `inputs` argument may be specified as a positional ' - 'argument. Pass everything else as a keyword argument ' - '(those arguments will not be tracked ' - 'as inputs to the layer).'.format(self.name)) - elif call_convention == CallConvention.SINGLE_POSITIONAL_ARGUMENT: - raise TypeError( - 'This layer ("{}") takes a single positional argument in `call()`,' - ' which is by convention the `inputs` argument, ' - 'and only this argument may be specified as a positional argument. ' - 'Pass everything else as a keyword argument ' - '(those arguments will not be tracked ' - 'as inputs to the layer).'.format(self.name)) - - # If the layer returns tensors from its inputs, unmodified, - # we copy them to avoid loss of tensor metadata. - output_ls = nest.flatten(outputs) - output_ls_copy = [] - for x in output_ls: - if x in nest.flatten(inputs): - with ops.name_scope(self.name): - x = array_ops.identity(x) - output_ls_copy.append(x) - if len(output_ls_copy) == 1: - outputs = output_ls_copy[0] - else: - outputs = output_ls_copy - - inputs, kwargs = self._inputs_from_call_args( - call_args=(inputs,) + args, call_kwargs=kwargs) - # Add an inbound node to the layer, so it can keep track of this call. - # This updates the layer history of the output tensor(s). - kwargs.pop('mask', None) # `mask` should not be serialized. - self._add_inbound_node( - input_tensors=inputs, output_tensors=outputs, arguments=kwargs) - return inputs, outputs - - def _inputs_from_call_args(self, call_args, call_kwargs): - """Get Layer inputs from __call__ *args and **kwargs. - - Args: - call_args: The positional arguments passed to __call__. - call_kwargs: The keyword argument dict passed to __call__. - - Returns: - A tuple of (inputs, non_input_kwargs). These may be the same objects as - were passed in (call_args and call_kwargs). - """ - call_convention = getattr(self, '_call_convention', - CallConvention.EXPLICIT_INPUTS_ARGUMENT) - if (call_convention in ( - CallConvention.EXPLICIT_INPUTS_ARGUMENT, - CallConvention.SINGLE_POSITIONAL_ARGUMENT)): - assert len(call_args) == 1 # TypeError raised earlier in __call__. - return call_args[0], call_kwargs - else: - call_arg_spec = tf_inspect.getfullargspec(self.call) - # There is no explicit "inputs" argument expected or provided to - # call(). Arguments which have default values are considered non-inputs, - # and arguments without are considered inputs. - if call_arg_spec.defaults: - if call_arg_spec.varargs is not None: - raise TypeError( - 'Layers may not accept both positional arguments and ' - 'arguments with default values (unable to determine which ' - 'are inputs to the layer). ' - 'Issue occurred with layer "%s"' % (self.name)) - keyword_arg_names = set( - call_arg_spec.args[-len(call_arg_spec.defaults):]) - else: - keyword_arg_names = set() - # Training is never an input argument name, to allow signatures like - # call(x, training). - keyword_arg_names.add('training') - _, unwrapped_call = tf_decorator.unwrap(self.call) - bound_args = inspect.getcallargs( - unwrapped_call, *call_args, **call_kwargs) - if call_arg_spec.varkw is not None: - var_kwargs = bound_args.pop(call_arg_spec.varkw) - bound_args.update(var_kwargs) - keyword_arg_names = keyword_arg_names.union(var_kwargs.keys()) - all_args = call_arg_spec.args - if all_args and bound_args[all_args[0]] is self: - # Ignore the 'self' argument of methods - bound_args.pop(call_arg_spec.args[0]) - all_args = all_args[1:] - non_input_arg_values = {} - input_arg_values = [] - remaining_args_are_keyword = False - for argument_name in all_args: - if argument_name in keyword_arg_names: - remaining_args_are_keyword = True - else: - if remaining_args_are_keyword: - raise TypeError( - 'Found a positional argument in a layer call after a non-input ' - 'argument. All arguments after "training" must be keyword ' - 'arguments, and are not tracked as inputs to the layer. ' - 'Issue occurred with layer "%s"' % (self.name)) - if remaining_args_are_keyword: - non_input_arg_values[argument_name] = bound_args[argument_name] - else: - input_arg_values.append(bound_args[argument_name]) - if call_arg_spec.varargs is not None: - input_arg_values.extend(bound_args[call_arg_spec.varargs]) - return input_arg_values, non_input_arg_values + return cls(**config) def compute_output_shape(self, input_shape): """Computes the output shape of the layer. @@ -989,15 +418,15 @@ class Layer(checkpointable.CheckpointableBase): # use `compute_output_shape` manually (these users will have to # implement `compute_output_shape` themselves). self.build(input_shape) - with context.graph_mode(): graph = func_graph.FuncGraph('graph') with graph.as_default(): if isinstance(input_shape, list): - inputs = [generate_placeholders_from_shape(shape) + inputs = [base_layer_utils.generate_placeholders_from_shape(shape) for shape in input_shape] else: - inputs = generate_placeholders_from_shape(input_shape) + inputs = base_layer_utils.generate_placeholders_from_shape( + input_shape) try: if self._expects_training_arg: @@ -1042,86 +471,425 @@ class Layer(checkpointable.CheckpointableBase): # carry over the input mask return mask - def _add_inbound_node(self, - input_tensors, - output_tensors, - arguments=None): - """Internal method to create an inbound node for the layer. + def __call__(self, inputs, *args, **kwargs): + """Wraps `call`, applying pre- and post-processing steps. Arguments: - input_tensors: list of input tensors. - output_tensors: list of output tensors. - arguments: dictionary of keyword arguments that were passed to the - `call` method of the layer at the call that created the node. - """ - input_tensors = nest.flatten(input_tensors) - output_tensors = nest.flatten(output_tensors) - - # Collect input tensor(s) coordinates. - inbound_layers = [] - node_indices = [] - tensor_indices = [] - for x in input_tensors: - assert hasattr(x, '_keras_history') - inbound_layer, node_index, tensor_index = x._keras_history # pylint: disable=protected-access - inbound_layers.append(inbound_layer) - node_indices.append(node_index) - tensor_indices.append(tensor_index) - - # Create node, add it to inbound nodes. - Node( - self, - inbound_layers=inbound_layers, - node_indices=node_indices, - tensor_indices=tensor_indices, - input_tensors=input_tensors, - output_tensors=output_tensors, - arguments=arguments) - - # Update tensor history metadata. - for i in range(len(output_tensors)): - # The metadata attribute consists of 1) a layer instance - # 2) a node index for the layer, 3) a tensor index for the node. - # The allows layer reuse (multiple nodes per layer) and multi-output - # or multi-input layers (e.g. a layer can return multiple tensors, - # and each can be sent to a different layer). - output_tensors[i]._keras_history = (self, len(self._inbound_nodes) - 1, i) # pylint: disable=protected-access - - def _get_node_attribute_at_index(self, node_index, attr, attr_name): - """Private utility to retrieves an attribute (e.g. inputs) from a node. - - This is used to implement the methods: - - get_input_shape_at - - get_output_shape_at - - get_input_at - etc... - - Arguments: - node_index: Integer index of the node from which - to retrieve the attribute. - attr: Exact node attribute name. - attr_name: Human-readable attribute name, for error messages. + inputs: input tensor(s). + *args: additional positional arguments to be passed to `self.call`. + **kwargs: additional keyword arguments to be passed to `self.call`. Returns: - The layer's attribute `attr` at the node of index `node_index`. + Output tensor(s). + + Note: + - The following optional keyword arguments are reserved for specific uses: + * `training`: Boolean scalar tensor of Python boolean indicating + whether the `call` is meant for training or inference. + * `mask`: Boolean input mask. + - If the layer's `call` method takes a `mask` argument (as some Keras + layers do), its default value will be set to the mask generated + for `inputs` by the previous layer (if `input` did come from + a layer that generated a corresponding mask, i.e. if it came from + a Keras layer with masking support. Raises: - RuntimeError: If the layer has no inbound nodes, or if called in Eager - mode. - ValueError: If the index provided does not match any node. + ValueError: if the layer's `call` method returns None (an invalid value). """ - if not self._inbound_nodes: - raise RuntimeError('The layer has never been called ' - 'and thus has no defined ' + attr_name + '.') - if not len(self._inbound_nodes) > node_index: - raise ValueError('Asked to get ' + attr_name + ' at node ' + - str(node_index) + ', but the layer has only ' + - str(len(self._inbound_nodes)) + ' inbound nodes.') - values = getattr(self._inbound_nodes[node_index], attr) - if len(values) == 1: - return values[0] + input_list = nest.flatten(inputs) + + if context.executing_eagerly(): + # Accept NumPy inputs by converting to Tensors when executing eagerly. + if all(isinstance(x, (np.ndarray, float, int)) for x in input_list): + inputs = nest.map_structure(ops.convert_to_tensor, inputs) + input_list = nest.flatten(inputs) + + # We will attempt to build a TF graph if & only if all inputs are symbolic. + # This is always the case in graph mode. It can also be the case in eager + # mode when all inputs can be traced back to `keras.Input()` (when building + # models using the functional API). + build_graph = tf_utils.are_all_symbolic_tensors(input_list) + executing_eagerly = context.executing_eagerly() + + # Handle Keras mask propagation from previous layer to current layer. + previous_mask = None + if build_graph and (not hasattr(self, '_compute_previous_mask') or + self._compute_previous_mask): + previous_mask = base_layer_utils.collect_previous_mask(inputs) + if not hasattr(self, '_call_fn_args'): + self._call_fn_args = self._no_dependency( + function_utils.fn_args(self.call)) + if ('mask' in self._call_fn_args and 'mask' not in kwargs and + not generic_utils.is_all_none(previous_mask)): + # The previous layer generated a mask, and mask was not explicitly pass + # to __call__, hence we set previous_mask as the default value. + kwargs['mask'] = previous_mask + + input_shapes = None + + with ops.name_scope(self._name_scope()): + if not self.built: + # Build layer if applicable (if the `build` method has been overridden). + self._maybe_build(inputs) + # We must set self.built since user defined build functions are not + # constrained to set self.built. + self.built = True + + # Check input assumptions set after layer building, e.g. input shape. + if build_graph: + # Symbolic execution on symbolic tensors. We will attempt to build + # the corresponding TF subgraph inside `backend.get_graph()` + input_spec.assert_input_compatibility( + self.input_spec, inputs, self.name) + graph = backend.get_graph() + with graph.as_default(): + if not executing_eagerly: + # In graph mode, failure to build the layer's graph + # implies a user-side bug. We don't catch exceptions. + outputs = self.call(inputs, *args, **kwargs) + else: + try: + outputs = self.call(inputs, *args, **kwargs) + except Exception: # pylint: disable=broad-except + # Any issue during graph-building means we will later run the + # model in eager mode, whether the issue was related to + # graph mode or not. This provides a nice debugging experience. + self._call_is_graph_friendly = False + # We will use static shape inference to return symbolic tensors + # matching the specifications of the layer outputs. + # Since we have set `self._call_is_graph_friendly = False`, + # we will never attempt to run the underlying TF graph (which is + # disconnected). + # TODO(fchollet): consider py_func as an alternative, which + # would enable us to run the underlying graph if needed. + input_shapes = nest.map_structure(lambda x: x.shape, inputs) + output_shapes = self.compute_output_shape(input_shapes) + outputs = nest.map_structure( + lambda shape: backend.placeholder(shape, dtype=self.dtype), + output_shapes) + + if outputs is None: + raise ValueError('A layer\'s `call` method should return a ' + 'Tensor or a list of Tensors, not None ' + '(layer: ' + self.name + ').') + self._handle_activity_regularization(inputs, outputs) + self._set_mask_metadata(inputs, outputs, previous_mask) + if base_layer_utils.have_all_keras_metadata(inputs): + inputs, outputs = self._set_connectivity_metadata_( + inputs, outputs, args, kwargs) + if hasattr(self, '_set_inputs') and not self.inputs: + # Subclassed network: explicitly set metadata normally set by + # a call to self._set_inputs(). + # This is not relevant in eager execution. + self._set_inputs(inputs, outputs) + else: + # Eager execution on data tensors. + outputs = self.call(inputs, *args, **kwargs) + self._handle_activity_regularization(inputs, outputs) + return outputs + + if not context.executing_eagerly(): + # Optionally load weight values specified at layer instantiation. + # TODO(fchollet): consider enabling this with eager execution too. + if (hasattr(self, '_initial_weights') and + self._initial_weights is not None): + self.set_weights(self._initial_weights) + del self._initial_weights + return outputs + + @property + def dtype(self): + return self._dtype + + @property + def name(self): + return self._name + + @property + def activity_regularizer(self): + """Optional regularizer function for the output of this layer.""" + return self._activity_regularizer + + @activity_regularizer.setter + def activity_regularizer(self, regularizer): + """Optional regularizer function for the output of this layer.""" + self._activity_regularizer = self._no_dependency(regularizer) + + @property + def trainable_weights(self): + return self._trainable_weights if self.trainable else [] + + @property + def non_trainable_weights(self): + if self.trainable: + return self._non_trainable_weights else: - return values + return self._trainable_weights + self._non_trainable_weights + + @property + def weights(self): + """Returns the list of all layer variables/weights. + + Returns: + A list of variables. + """ + return self.trainable_weights + self.non_trainable_weights + + @property + def updates(self): + if not self.trainable and not self.stateful: + return [] + return self._updates + + @property + def losses(self): + """Losses which are associated with this `Layer`. + + Variable regularization tensors are created when this property is accessed, + so it is eager safe: accessing `losses` under a `tf.GradientTape` will + propagate gradients back to the corresponding variables. + + Returns: + A list of tensors. + """ + collected_losses = [] + if context.executing_eagerly(): + collected_losses.extend(self._eager_losses) + else: + collected_losses.extend(self._losses) + for regularizer in self._callable_losses: + loss_tensor = regularizer() + if loss_tensor is not None: + collected_losses.append(loss_tensor) + return collected_losses + + @doc_controls.for_subclass_implementers + def add_loss(self, losses, inputs=None): + """Add loss tensor(s), potentially dependent on layer inputs. + + Some losses (for instance, activity regularization losses) may be dependent + on the inputs passed when calling a layer. Hence, when reusing the same + layer on different inputs `a` and `b`, some entries in `layer.losses` may + be dependent on `a` and some on `b`. This method automatically keeps track + of dependencies. + + The `get_losses_for` method allows to retrieve the losses relevant to a + specific set of inputs. + + Note that `add_loss` is not supported when executing eagerly. Instead, + variable regularizers may be added through `add_variable`. Activity + regularization is not supported directly (but such losses may be returned + from `Layer.call()`). + + Arguments: + losses: Loss tensor, or list/tuple of tensors. Rather than tensors, losses + may also be zero-argument callables which create a loss tensor. + inputs: Ignored when executing eagerly. If anything other than None is + passed, it signals the losses are conditional on some of the layer's + inputs, and thus they should only be run where these inputs are + available. This is the case for activity regularization losses, for + instance. If `None` is passed, the losses are assumed + to be unconditional, and will apply across all dataflows of the layer + (e.g. weight regularization losses). + """ + losses = generic_utils.to_list(losses) + + def _tag_unconditional(loss): + if callable(loss): + loss = loss() + if loss is None: + return None # Will be filtered out when computing the .losses property + if not tensor_util.is_tensor(loss): + loss = ops.convert_to_tensor(loss, dtype=backend.floatx()) + loss._unconditional_loss = (inputs is None) # pylint: disable=protected-access + return loss + + for loss in losses: + if callable(loss): + self._callable_losses.append( + functools.partial(_tag_unconditional, loss)) + else: + if context.executing_eagerly(): + self._eager_losses.append(_tag_unconditional(loss)) + else: + self._losses.append(_tag_unconditional(loss)) + + @doc_controls.for_subclass_implementers + def add_metric(self, value, aggregation=None, name=None): + """Adds metric tensor to the layer. + + Args: + value: Metric tensor. + aggregation: Sample-wise metric reduction function. If `aggregation=None`, + it indicates that the metric tensor provided has been aggregated + already. eg, `model.add_metric(BinaryAccuracy(name='acc')(y_true, + y_pred))`. If aggregation='mean', the given metric tensor will be + sample-wise reduced using `mean` function. eg, `model.add_metric( + tf.reduce_mean(outputs), name='output_mean', aggregation='mean')`. + name: String metric name. + + Raises: + ValueError: If `aggregation` is anything other than None or `mean`. + """ + if aggregation is not None and aggregation != 'mean': + raise ValueError( + 'We currently support only `mean` sample-wise metric aggregation. ' + 'You provided aggregation=`%s`' % aggregation) + + if tf_utils.is_symbolic_tensor(value): + self._symbolic_add_metric(value, aggregation, name) + else: + self._eager_add_metric(value, aggregation, name) + + @doc_controls.for_subclass_implementers + def add_update(self, updates, inputs=None): + """Add update op(s), potentially dependent on layer inputs. + + Weight updates (for instance, the updates of the moving mean and variance + in a BatchNormalization layer) may be dependent on the inputs passed + when calling a layer. Hence, when reusing the same layer on + different inputs `a` and `b`, some entries in `layer.updates` may be + dependent on `a` and some on `b`. This method automatically keeps track + of dependencies. + + The `get_updates_for` method allows to retrieve the updates relevant to a + specific set of inputs. + + This call is ignored when eager execution is enabled (in that case, variable + updates are run on the fly and thus do not need to be tracked for later + execution). + + Arguments: + updates: Update op, or list/tuple of update ops. + inputs: If anything other than None is passed, it signals the updates + are conditional on some of the layer's inputs, + and thus they should only be run where these inputs are available. + This is the case for BatchNormalization updates, for instance. + If None, the updates will be taken into account unconditionally, + and you are responsible for making sure that any dependency they might + have is available at runtime. + A step counter might fall into this category. + """ + if context.executing_eagerly(): + return # Updates already applied when in eager mode. + + def process_update(x): + if isinstance(x, ops.Operation): + return x + elif hasattr(x, 'op'): + return x.op + else: + return ops.convert_to_tensor(x) + + updates = generic_utils.to_list(updates) + updates = [process_update(x) for x in updates] + self._updates += updates + if inputs is None: + for u in updates: + u._unconditional_update = True # pylint: disable=protected-access + else: + for u in updates: + u._unconditional_update = False # pylint: disable=protected-access + + def set_weights(self, weights): + """Sets the weights of the layer, from Numpy arrays. + + Arguments: + weights: a list of Numpy arrays. The number + of arrays and their shape must match + number of the dimensions of the weights + of the layer (i.e. it should match the + output of `get_weights`). + + Raises: + ValueError: If the provided weights list does not match the + layer's specifications. + """ + params = self.weights + if len(params) != len(weights): + raise ValueError('You called `set_weights(weights)` on layer "' + + self.name + '" with a weight list of length ' + + str(len(weights)) + ', but the layer was expecting ' + + str(len(params)) + ' weights. Provided weights: ' + + str(weights)[:50] + '...') + if not params: + return + weight_value_tuples = [] + param_values = backend.batch_get_value(params) + for pv, p, w in zip(param_values, params, weights): + if pv.shape != w.shape: + raise ValueError('Layer weight shape ' + str(pv.shape) + + ' not compatible with ' + 'provided weight shape ' + str(w.shape)) + weight_value_tuples.append((p, w)) + backend.batch_set_value(weight_value_tuples) + + def get_weights(self): + """Returns the current weights of the layer. + + Returns: + Weights values as a list of numpy arrays. + """ + params = self.weights + return backend.batch_get_value(params) + + def get_updates_for(self, inputs): + """Retrieves updates relevant to a specific set of inputs. + + Arguments: + inputs: Input tensor or list/tuple of input tensors. + + Returns: + List of update ops of the layer that depend on `inputs`. + + Raises: + RuntimeError: If called in Eager mode. + """ + # Updates disabled if layer is not trainable and not explicitly stateful. + if not self.trainable and not self.stateful: + return [] + + if inputs is None: + # Requesting unconditional updates. + return [x for x in self.updates if x._unconditional_update] # pylint: disable=protected-access + + # Requesting input-conditional updates. + inputs = nest.flatten(inputs) + reachable = tf_utils.get_reachable_from_inputs(inputs, self.updates) + updates = [] + for update in self.updates: + if update in reachable: + updates.append(update) + return updates + + def get_losses_for(self, inputs): + """Retrieves losses relevant to a specific set of inputs. + + Arguments: + inputs: Input tensor or list/tuple of input tensors. + + Returns: + List of loss tensors of the layer that depend on `inputs`. + + Raises: + RuntimeError: If called in Eager mode. + """ + if inputs is None: + # Requesting unconditional losses. + return [x for x in self.losses if x._unconditional_loss] # pylint: disable=protected-access + + # Requesting input-conditional losses. + inputs = nest.flatten(inputs) + # Retrieve the set of tensors in the TF graph that depend on `inputs`. + # The losses we want to return will be part of this set. + # To avoid unnecessary work, we stop the search in case all of + # `self.losses` have been retrieved. + reachable = tf_utils.get_reachable_from_inputs(inputs, self.losses) + losses = [] + for loss in self.losses: + if loss in reachable: + losses.append(loss) + return losses def get_input_mask_at(self, node_index): """Retrieves the input mask tensor(s) of a layer at a given node. @@ -1376,8 +1144,7 @@ class Layer(checkpointable.CheckpointableBase): ', but the layer isn\'t built. ' 'You can build it manually via: `' + self.name + '.build(batch_input_shape)`.') - weight_shapes = [w.shape.as_list() for w in self.weights] - return int(sum([np.prod(w) for w in weight_shapes])) + return int(sum(np.prod(w.shape.as_list()) for w in self.weights)) @property def output_shape(self): @@ -1429,231 +1196,401 @@ class Layer(checkpointable.CheckpointableBase): """Deprecated, do NOT use! Only for compatibility with external Keras.""" return self._outbound_nodes - def _assert_input_compatibility(self, inputs): - """Checks compatibility between the layer and provided inputs. + ############################################################################## + # Methods & attributes below are public aliases of other methods. # + ############################################################################## - This checks that the tensor(s) `inputs` verify the input assumptions - of the layer (if any). If not, a clear and actional exception gets raised. + def apply(self, inputs, *args, **kwargs): + """Apply the layer on a input. + + This is an alias of `self.__call__`. Arguments: - inputs: input tensor or list of input tensors. + inputs: Input tensor(s). + *args: additional positional arguments to be passed to `self.call`. + **kwargs: additional keyword arguments to be passed to `self.call`. + + Returns: + Output tensor(s). + """ + return self.__call__(inputs, *args, **kwargs) + + @doc_controls.for_subclass_implementers + def add_variable(self, *args, **kwargs): + """Alias for `add_weight`.""" + return self.add_weight(*args, **kwargs) + + @property + def variables(self): + """Returns the list of all layer variables/weights. + + Alias of `self.weights`. + + Returns: + A list of variables. + """ + return self.weights + + @property + def trainable_variables(self): + return self.trainable_weights + + @property + def non_trainable_variables(self): + return self.non_trainable_weights + + ############################################################################## + # Methods & attributes below are all private and only used by the framework. # + ############################################################################## + + def _name_scope(self): + return self.name + + def _init_set_name(self, name, zero_based=True): + if not name: + self._name = base_layer_utils.unique_layer_name( + generic_utils.to_snake_case(self.__class__.__name__), + zero_based=zero_based) + else: + self._name = name + + def _get_existing_metric(self, name=None): + match = [m for m in self._metrics if m.name == name] + if not match: + return + if len(match) > 1: + raise ValueError( + 'Please provide different names for the metrics you have added. ' + 'We found {} metrics with the name: "{}"'.format(len(match), name)) + return match[0] + + def _eager_add_metric(self, value, aggregation=None, name=None): + # If the given metric is available in `metrics` list we just update state + # on it, otherwise we create a new metric instance and + # add it to the `metrics` list. + match = self._get_existing_metric(name) + if match: + match(value) # Update the metric state. + return + else: + if aggregation is None: + raise ValueError('We do not support adding an aggregated metric tensor ' + 'in `call` in eager execution.') + metric_obj, _ = base_layer_utils.create_mean_metric(value, name) + self._metrics.append(metric_obj) + + def _symbolic_add_metric(self, value, aggregation=None, name=None): + if aggregation is None: + # Iterate over the metrics and check if the given metric exists already. + # This can happen when a metric instance is created in subclassed model + # layer `__init__` and we have tracked that instance already in + # model.__setattr__. + match = self._get_existing_metric(name) + if match: + result_tensor = value + if match.name not in self._metrics_tensors: + self._metrics_tensors[match.name] = result_tensor + return + else: + raise ValueError( + 'We currently do not support reusing a metric instance.') + else: + # We track the instance using the metadata on the result tensor. + result_tensor = value + metric_obj = result_tensor._metric_obj + else: + # If a non-aggregated tensor is given as input (ie. `aggregation` is + # explicitly set to `mean`), we wrap the tensor in `Mean` metric. + metric_obj, result_tensor = base_layer_utils.create_mean_metric( + value, name) + self._metrics.append(metric_obj) + self._metrics_tensors[metric_obj.name] = result_tensor + + def _handle_weight_regularization(self, name, variable, regularizer): + """Create lambdas which compute regularization losses.""" + + def _loss_for_variable(v): + """Creates a regularization loss `Tensor` for variable `v`.""" + with ops.colocate_with(v): + with ops.name_scope(name + '/Regularizer'): + regularization = regularizer(v) + return regularization + + if isinstance(variable, tf_variables.PartitionedVariable): + for v in variable: + self.add_loss(functools.partial(_loss_for_variable, v)) + else: + self.add_loss(functools.partial(_loss_for_variable, variable)) + + def _handle_activity_regularization(self, inputs, outputs): + # Apply activity regularization. + # Note that it should be applied every time the layer creates a new + # output, since it is output-specific. + if self._activity_regularizer: + output_list = nest.flatten(outputs) + with ops.name_scope('ActivityRegularizer'): + for output in output_list: + activity_loss = self._activity_regularizer(output) + batch_size = math_ops.cast( + array_ops.shape(output)[0], activity_loss.dtype) + # Make activity regularization strength batch-agnostic. + mean_activity_loss = activity_loss / batch_size + self.add_loss(mean_activity_loss, inputs=inputs) + + def _set_mask_metadata(self, inputs, outputs, previous_mask): + # In some cases the mask of the outputs has already been computed by + # inner layers and does not need to be recomputed by this layer. + mask_already_computed = all( + hasattr(x, '_keras_mask') for x in generic_utils.to_list(outputs)) + if hasattr(self, 'compute_mask') and not mask_already_computed: + output_mask = self.compute_mask(inputs, previous_mask) + else: + output_mask = None + if isinstance(outputs, (list, tuple)): + if output_mask is None: + output_mask = [None for _ in range(len(outputs))] + for x, m in zip(outputs, output_mask): + try: + x._keras_mask = m # pylint: disable=protected-access + except AttributeError: + pass # C type such as dict. Masking not supported in this case. + else: + try: + outputs._keras_mask = output_mask # pylint: disable=protected-access + except AttributeError: + pass # C type such as dict. Masking not supported in this case. + + def _set_connectivity_metadata_(self, inputs, outputs, args, kwargs): + call_convention = getattr( + self, '_call_convention', + base_layer_utils.CallConvention.EXPLICIT_INPUTS_ARGUMENT) + if args: + if call_convention == (base_layer_utils + .CallConvention.EXPLICIT_INPUTS_ARGUMENT): + raise TypeError( + 'This layer ("{}") takes an `inputs` argument in `call()`, ' + 'and only the `inputs` argument may be specified as a positional ' + 'argument. Pass everything else as a keyword argument ' + '(those arguments will not be tracked ' + 'as inputs to the layer).'.format(self.name)) + elif call_convention == (base_layer_utils + .CallConvention.SINGLE_POSITIONAL_ARGUMENT): + raise TypeError( + 'This layer ("{}") takes a single positional argument in `call()`,' + ' which is by convention the `inputs` argument, ' + 'and only this argument may be specified as a positional argument. ' + 'Pass everything else as a keyword argument ' + '(those arguments will not be tracked ' + 'as inputs to the layer).'.format(self.name)) + + # If the layer returns tensors from its inputs, unmodified, + # we copy them to avoid loss of tensor metadata. + output_ls = nest.flatten(outputs) + output_ls_copy = [] + for x in output_ls: + if x in nest.flatten(inputs): + with ops.name_scope(self.name): + x = array_ops.identity(x) + output_ls_copy.append(x) + if len(output_ls_copy) == 1: + outputs = output_ls_copy[0] + else: + outputs = output_ls_copy + + inputs, kwargs = self._inputs_from_call_args( + call_args=(inputs,) + args, call_kwargs=kwargs) + # Add an inbound node to the layer, so it can keep track of this call. + # This updates the layer history of the output tensor(s). + kwargs.pop('mask', None) # `mask` should not be serialized. + self._add_inbound_node( + input_tensors=inputs, output_tensors=outputs, arguments=kwargs) + return inputs, outputs + + def _inputs_from_call_args(self, call_args, call_kwargs): + """Get Layer inputs from __call__ *args and **kwargs. + + Args: + call_args: The positional arguments passed to __call__. + call_kwargs: The keyword argument dict passed to __call__. + + Returns: + A tuple of (inputs, non_input_kwargs). These may be the same objects as + were passed in (call_args and call_kwargs). + """ + call_convention = getattr( + self, '_call_convention', + base_layer_utils.CallConvention.EXPLICIT_INPUTS_ARGUMENT) + if (call_convention in ( + base_layer_utils.CallConvention.EXPLICIT_INPUTS_ARGUMENT, + base_layer_utils.CallConvention.SINGLE_POSITIONAL_ARGUMENT)): + assert len(call_args) == 1 # TypeError raised earlier in __call__. + return call_args[0], call_kwargs + else: + call_arg_spec = tf_inspect.getfullargspec(self.call) + # There is no explicit "inputs" argument expected or provided to + # call(). Arguments which have default values are considered non-inputs, + # and arguments without are considered inputs. + if call_arg_spec.defaults: + if call_arg_spec.varargs is not None: + raise TypeError( + 'Layers may not accept both positional arguments and ' + 'arguments with default values (unable to determine which ' + 'are inputs to the layer). ' + 'Issue occurred with layer "%s"' % (self.name)) + keyword_arg_names = set( + call_arg_spec.args[-len(call_arg_spec.defaults):]) + else: + keyword_arg_names = set() + # Training is never an input argument name, to allow signatures like + # call(x, training). + keyword_arg_names.add('training') + _, unwrapped_call = tf_decorator.unwrap(self.call) + bound_args = inspect.getcallargs( + unwrapped_call, *call_args, **call_kwargs) + if call_arg_spec.varkw is not None: + var_kwargs = bound_args.pop(call_arg_spec.varkw) + bound_args.update(var_kwargs) + keyword_arg_names = keyword_arg_names.union(var_kwargs.keys()) + all_args = call_arg_spec.args + if all_args and bound_args[all_args[0]] is self: + # Ignore the 'self' argument of methods + bound_args.pop(call_arg_spec.args[0]) + all_args = all_args[1:] + non_input_arg_values = {} + input_arg_values = [] + remaining_args_are_keyword = False + for argument_name in all_args: + if argument_name in keyword_arg_names: + remaining_args_are_keyword = True + else: + if remaining_args_are_keyword: + raise TypeError( + 'Found a positional argument in a layer call after a non-input ' + 'argument. All arguments after "training" must be keyword ' + 'arguments, and are not tracked as inputs to the layer. ' + 'Issue occurred with layer "%s"' % (self.name)) + if remaining_args_are_keyword: + non_input_arg_values[argument_name] = bound_args[argument_name] + else: + input_arg_values.append(bound_args[argument_name]) + if call_arg_spec.varargs is not None: + input_arg_values.extend(bound_args[call_arg_spec.varargs]) + return input_arg_values, non_input_arg_values + + def _add_inbound_node(self, + input_tensors, + output_tensors, + arguments=None): + """Internal method to create an inbound node for the layer. + + Arguments: + input_tensors: list of input tensors. + output_tensors: list of output tensors. + arguments: dictionary of keyword arguments that were passed to the + `call` method of the layer at the call that created the node. + """ + input_tensors = nest.flatten(input_tensors) + output_tensors = nest.flatten(output_tensors) + + # Collect input tensor(s) coordinates. + inbound_layers = [] + node_indices = [] + tensor_indices = [] + for x in input_tensors: + assert hasattr(x, '_keras_history') + inbound_layer, node_index, tensor_index = x._keras_history # pylint: disable=protected-access + inbound_layers.append(inbound_layer) + node_indices.append(node_index) + tensor_indices.append(tensor_index) + + # Create node, add it to inbound nodes. + Node( + self, + inbound_layers=inbound_layers, + node_indices=node_indices, + tensor_indices=tensor_indices, + input_tensors=input_tensors, + output_tensors=output_tensors, + arguments=arguments) + + # Update tensor history metadata. + for i in range(len(output_tensors)): + # The metadata attribute consists of 1) a layer instance + # 2) a node index for the layer, 3) a tensor index for the node. + # The allows layer reuse (multiple nodes per layer) and multi-output + # or multi-input layers (e.g. a layer can return multiple tensors, + # and each can be sent to a different layer). + output_tensors[i]._keras_history = (self, len(self._inbound_nodes) - 1, i) # pylint: disable=protected-access + + def _get_node_attribute_at_index(self, node_index, attr, attr_name): + """Private utility to retrieves an attribute (e.g. inputs) from a node. + + This is used to implement the methods: + - get_input_shape_at + - get_output_shape_at + - get_input_at + etc... + + Arguments: + node_index: Integer index of the node from which + to retrieve the attribute. + attr: Exact node attribute name. + attr_name: Human-readable attribute name, for error messages. + + Returns: + The layer's attribute `attr` at the node of index `node_index`. Raises: - ValueError: in case of mismatch between - the provided inputs and the expectations of the layer. + RuntimeError: If the layer has no inbound nodes, or if called in Eager + mode. + ValueError: If the index provided does not match any node. """ - if not self.input_spec: - return - if not isinstance(self.input_spec, (list, tuple)): - input_spec = nest.flatten(self.input_spec) + if not self._inbound_nodes: + raise RuntimeError('The layer has never been called ' + 'and thus has no defined ' + attr_name + '.') + if not len(self._inbound_nodes) > node_index: + raise ValueError('Asked to get ' + attr_name + ' at node ' + + str(node_index) + ', but the layer has only ' + + str(len(self._inbound_nodes)) + ' inbound nodes.') + values = getattr(self._inbound_nodes[node_index], attr) + if len(values) == 1: + return values[0] else: - input_spec = self.input_spec - inputs = nest.flatten(inputs) - if len(inputs) != len(input_spec): - raise ValueError('Layer ' + self.name + ' expects ' + - str(len(input_spec)) + ' inputs, ' - 'but it received ' + str(len(inputs)) + - ' input tensors. Inputs received: ' + str(inputs)) - for input_index, (x, spec) in enumerate(zip(inputs, input_spec)): - if spec is None: - continue + return values - if (spec.ndim is not None or - spec.min_ndim is not None or - spec.max_ndim is not None): - if x.shape.ndims is None: - raise ValueError('Input ' + str(input_index) + ' of layer ' + - self.name + ' is incompatible with the layer: ' - 'its rank is undefined, but the layer requires a ' - 'defined rank.') + @property + def _static_graph_friendly(self): + """Whether the layer can be called to create a static graph. - # Check ndim. - if spec.ndim is not None: - ndim = x.shape.ndims - if ndim != spec.ndim: - raise ValueError('Input ' + str(input_index) + ' of layer ' + - self.name + ' is incompatible with the layer: ' - 'expected ndim=' + str(spec.ndim) + ', found ndim=' + - str(ndim) + '. Full shape received: ' + - str(x.shape.as_list())) - if spec.max_ndim is not None: - ndim = x.shape.ndims - if ndim is not None and ndim > spec.max_ndim: - raise ValueError('Input ' + str(input_index) + ' of layer ' + - self.name + ' is incompatible with the layer: ' - 'expected max_ndim=' + str(spec.max_ndim) + - ', found ndim=' + str(ndim)) - if spec.min_ndim is not None: - ndim = x.shape.ndims - if ndim is not None and ndim < spec.min_ndim: - raise ValueError('Input ' + str(input_index) + ' of layer ' + - self.name + ' is incompatible with the layer: ' - ': expected min_ndim=' + str(spec.min_ndim) + - ', found ndim=' + str(ndim) + - '. Full shape received: ' + - str(x.shape.as_list())) - # Check dtype. - if spec.dtype is not None: - if x.dtype != spec.dtype: - raise ValueError('Input ' + str(input_index) + ' of layer ' + - self.name + ' is incompatible with the layer: ' - 'expected dtype=' + str(spec.dtype) + - ', found dtype=' + str(x.dtype)) - # Check specific shape axes. - if spec.axes: - shape = x.shape.as_list() - if shape is not None: - for axis, value in spec.axes.items(): - if hasattr(value, 'value'): - value = value.value - if value is not None and shape[int(axis)] not in {value, None}: - raise ValueError( - 'Input ' + str(input_index) + ' of layer ' + self.name + ' is' - ' incompatible with the layer: expected axis ' + str(axis) + - ' of input shape to have value ' + str(value) + - ' but received input with shape ' + str(shape)) - # Check shape. - if spec.shape is not None: - shape = x.shape.as_list() - if shape is not None: - for spec_dim, dim in zip(spec.shape, shape): - if spec_dim is not None and dim is not None: - if spec_dim != dim: - raise ValueError('Input ' + str(input_index) + - ' is incompatible with layer ' + self.name + - ': expected shape=' + str(spec.shape) + - ', found shape=' + str(shape)) - - def set_weights(self, weights): - """Sets the weights of the layer, from Numpy arrays. - - Arguments: - weights: a list of Numpy arrays. The number - of arrays and their shape must match - number of the dimensions of the weights - of the layer (i.e. it should match the - output of `get_weights`). - - Raises: - ValueError: If the provided weights list does not match the - layer's specifications. - """ - params = self.weights - if len(params) != len(weights): - raise ValueError('You called `set_weights(weights)` on layer "' + - self.name + '" with a weight list of length ' + - str(len(weights)) + ', but the layer was expecting ' + - str(len(params)) + ' weights. Provided weights: ' + - str(weights)[:50] + '...') - if not params: - return - weight_value_tuples = [] - param_values = backend.batch_get_value(params) - for pv, p, w in zip(param_values, params, weights): - if pv.shape != w.shape: - raise ValueError('Layer weight shape ' + str(pv.shape) + - ' not compatible with ' - 'provided weight shape ' + str(w.shape)) - weight_value_tuples.append((p, w)) - backend.batch_set_value(weight_value_tuples) - - def get_weights(self): - """Returns the current weights of the layer. + Because of nesting, there are two components to being "graph-friendly": + 1) all inner layers are graph-friendly + 2) the way they are composed is graph-friendly. + We denote the latter as "_call_is_graph_friendly", and define + "_static_graph_friendly" as being the combination of + "_call_is_graph_friendly" and "all inner layers are _static_graph_friendly". + For atomic layers (no inner layers), this is just "_call_is_graph_friendly". Returns: - Weights values as a list of numpy arrays. + Boolean. """ - params = self.weights - return backend.batch_get_value(params) + return self._call_is_graph_friendly - def get_config(self): - """Returns the config of the layer. - - A layer config is a Python dictionary (serializable) - containing the configuration of a layer. - The same layer can be reinstantiated later - (without its trained weights) from this configuration. - - The config of a layer does not include connectivity - information, nor the layer class name. These are handled - by `Network` (one layer of abstraction above). - - Returns: - Python dictionary. - """ - config = {'name': self.name, 'trainable': self.trainable} - if hasattr(self, '_batch_input_shape'): - config['batch_input_shape'] = self._batch_input_shape - if hasattr(self, 'dtype'): - config['dtype'] = self.dtype - return config - - @classmethod - def from_config(cls, config): - """Creates a layer from its config. - - This method is the reverse of `get_config`, - capable of instantiating the same layer from the config - dictionary. It does not handle layer connectivity - (handled by Network), nor weights (handled by `set_weights`). - - Arguments: - config: A Python dictionary, typically the - output of get_config. - - Returns: - A layer instance. - """ - return cls(**config) - - -@tf_export( - 'keras.layers.InputSpec', v1=['keras.layers.InputSpec', 'layers.InputSpec']) -class InputSpec(object): - """Specifies the ndim, dtype and shape of every input to a layer. - - Every layer should expose (if appropriate) an `input_spec` attribute: - a list of instances of InputSpec (one per input tensor). - - A None entry in a shape is compatible with any dimension, - a None shape is compatible with any shape. - - Arguments: - dtype: Expected DataType of the input. - shape: Shape tuple, expected shape of the input - (may include None for unchecked axes). - ndim: Integer, expected rank of the input. - max_ndim: Integer, maximum rank of the input. - min_ndim: Integer, minimum rank of the input. - axes: Dictionary mapping integer axes to - a specific dimension value. - """ - - def __init__(self, - dtype=None, - shape=None, - ndim=None, - max_ndim=None, - min_ndim=None, - axes=None): - self.dtype = dtype - self.shape = shape - if shape is not None: - self.ndim = len(shape) - else: - self.ndim = ndim - self.max_ndim = max_ndim - self.min_ndim = min_ndim - self.axes = axes or {} - - def __repr__(self): - spec = [('dtype=' + str(self.dtype)) if self.dtype else '', - ('shape=' + str(self.shape)) if self.shape else '', - ('ndim=' + str(self.ndim)) if self.ndim else '', - ('max_ndim=' + str(self.max_ndim)) if self.max_ndim else '', - ('min_ndim=' + str(self.min_ndim)) if self.min_ndim else '', - ('axes=' + str(self.axes)) if self.axes else ''] - return 'InputSpec(%s)' % ', '.join(x for x in spec if x) + def _maybe_build(self, inputs): + # Check input assumptions set before layer building, e.g. input rank. + input_spec.assert_input_compatibility( + self.input_spec, inputs, self.name) + input_list = nest.flatten(inputs) + if input_list and self._dtype is None: + try: + self._dtype = input_list[0].dtype.base_dtype.name + except AttributeError: + pass + input_shapes = None + if all(hasattr(x, 'shape') for x in input_list): + input_shapes = nest.map_structure(lambda x: x.shape, inputs) + # Only call `build` if the user has manually overridden the build method. + if not hasattr(self.build, '_is_default'): + self.build(input_shapes) class Node(object): @@ -1768,192 +1705,12 @@ class Node(object): } -def unique_layer_name(name, name_uid_map=None, avoid_names=None, namespace='', - zero_based=False): - """Makes a layer name (or arbitrary string) unique within a TensorFlow graph. - - Arguments: - name: String name to make unique. - name_uid_map: An optional defaultdict(int) to use when creating unique - names. If None (default), uses a per-Graph dictionary. - avoid_names: An optional set or dict with names which should not be used. If - None (default) does not avoid any names. - namespace: Gets a name which is unique within the (graph, namespace). Layers - which are not Networks use a blank namespace and so get graph-global - names. - zero_based: If True, name sequences start with no suffix (e.g. "dense", - "dense_1"). If False, naming is one-based ("dense_1", "dense_2"). - - Returns: - Unique string name. - - Example: - - ```python - _unique_layer_name('dense') # dense_1 - _unique_layer_name('dense') # dense_2 - ``` - """ - if name_uid_map is None: - name_uid_map = get_default_graph_uid_map() - if avoid_names is None: - avoid_names = set() - proposed_name = None - while proposed_name is None or proposed_name in avoid_names: - name_key = (namespace, name) - if zero_based: - number = name_uid_map[name_key] - if number: - proposed_name = name + '_' + str(number) - else: - proposed_name = name - name_uid_map[name_key] += 1 - else: - name_uid_map[name_key] += 1 - proposed_name = name + '_' + str(name_uid_map[name_key]) - return proposed_name - - -def have_all_keras_metadata(iterable_or_element): - if not isinstance(iterable_or_element, (list, tuple)): - iterable = [iterable_or_element] - else: - iterable = nest.flatten(iterable_or_element) - return all([hasattr(x, '_keras_history') for x in iterable]) - - -def collect_previous_mask(input_tensors): - """Retrieves the output mask(s) of the previous node. - - Arguments: - input_tensors: A tensor or list of tensors. - - Returns: - A mask tensor or list of mask tensors. - """ - input_tensors = nest.flatten(input_tensors) - masks = [] - for x in input_tensors: - if hasattr(x, '_keras_mask'): - mask = x._keras_mask # pylint: disable=protected-access - masks.append(mask) - else: - masks.append(None) - if len(masks) == 1: - return masks[0] - return masks - - -def get_default_graph_uid_map(): - # TODO(fchollet): refactor this into backend. - graph = ops.get_default_graph() - name_uid_map = backend.PER_GRAPH_LAYER_NAME_UIDS.get(graph, None) - if name_uid_map is None: - name_uid_map = collections_lib.defaultdict(int) - backend.PER_GRAPH_LAYER_NAME_UIDS[graph] = name_uid_map - return name_uid_map - - -def make_variable(name, - shape=None, - dtype=dtypes.float32, - initializer=None, - partition_info=None, - trainable=None, - caching_device=None, - validate_shape=True, - constraint=None, - use_resource=None, - collections=None, - synchronization=tf_variables.VariableSynchronization.AUTO, - aggregation=tf_variables.VariableAggregation.NONE, - partitioner=None): # pylint: disable=unused-argument - """Temporary util to create a variable (relies on `variable_scope.variable`). - - Some reuse-related technicalities prevent us from using - `variable_scope.get_variable()` directly, so we use a subcomponent - that has fewer constraints (`variable_scope.variable()`). - - In the longer term, it seems like a similar "default variable creator" method - should exist in `CheckpointableBase` instead. When this happens, we can get - rid of this temporary solution. - - TODO(fchollet): remove this method when no longer needed. - TODO(fchollet): handle `partitioner` argument. - - Arguments: - name: Variable name. - shape: Variable shape. - dtype: The type of the variable. Defaults to `self.dtype` or `float32`. - initializer: Initializer instance (callable). - partition_info: Not handled at this time. - trainable: Whether the variable should be part of the layer's - "trainable_variables" (e.g. variables, biases) - or "non_trainable_variables" (e.g. BatchNorm mean, stddev). - Note, if the current variable scope is marked as non-trainable - then this parameter is ignored and any added variables are also - marked as non-trainable. `trainable` defaults to `True` unless - `synchronization` is set to `ON_READ`. - caching_device: Passed to `tf.Variable`. - validate_shape: Passed to `tf.Variable`. - constraint: Constraint instance (callable). - use_resource: Whether to use a `ResourceVariable`. - collections: List of graph collections keys. The new variable is added to - these collections. Defaults to `[GraphKeys.GLOBAL_VARIABLES]`. - synchronization: Indicates when a distributed a variable will be - aggregated. Accepted values are constants defined in the class - `tf.VariableSynchronization`. By default the synchronization is set to - `AUTO` and the current `DistributionStrategy` chooses - when to synchronize. If `synchronization` is set to `ON_READ`, - `trainable` must not be set to `True`. - aggregation: Indicates how a distributed variable will be aggregated. - Accepted values are constants defined in the class - `tf.VariableAggregation`. - partitioner: Not handled at this time. - - Returns: - Variable instance. - """ - initializing_from_value = False - if initializer is not None and not callable(initializer): - initializing_from_value = True - - with ops.init_scope(): - if initializing_from_value: - init_val = initializer - variable_dtype = None - else: - # Instantiate initializer if provided initializer is a type object. - if isinstance(initializer, type(init_ops.Initializer)): - initializer = initializer(dtype=dtype) - init_val = lambda: initializer( # pylint: disable=g-long-lambda - shape, dtype=dtype, partition_info=partition_info) - variable_dtype = dtype.base_dtype - if use_resource is None: - use_resource = True - - # TODO(apassos,rohanj) figure out how to remove collections from here so we - # can remove the V1. - v = tf_variables.VariableV1( - initial_value=init_val, - name=name, - trainable=trainable, - caching_device=caching_device, - dtype=variable_dtype, - validate_shape=validate_shape, - constraint=constraint, - use_resource=use_resource, - collections=collections, - synchronization=synchronization, - aggregation=aggregation) - return v - - def default(method): """Decorates a method to detect overrides in subclasses.""" method._is_default = True return method -def generate_placeholders_from_shape(shape): - return array_ops.placeholder(shape=shape, dtype=backend.floatx()) +# Avoid breaking users who directly import this symbol from this file. +# TODO(fchollet): remove this. +InputSpec = input_spec.InputSpec # pylint:disable=invalid-name diff --git a/tensorflow/python/keras/engine/base_layer_test.py b/tensorflow/python/keras/engine/base_layer_test.py index 704589349a8..798775b6a5b 100644 --- a/tensorflow/python/keras/engine/base_layer_test.py +++ b/tensorflow/python/keras/engine/base_layer_test.py @@ -81,14 +81,14 @@ class BaseLayerTest(test.TestCase): inputs = keras.Input((3,)) outputs = DynamicLayer1()(inputs) model = keras.Model(inputs, outputs) - self.assertEqual(model._is_static_graph_friendly, False) + self.assertEqual(model._static_graph_friendly, False) model.compile(RMSPropOptimizer(0.001), loss='mse') model.train_on_batch(np.random.random((2, 3)), np.random.random((2, 3))) inputs = keras.Input((3,)) outputs = DynamicLayer2()(inputs) model = keras.Model(inputs, outputs) - self.assertEqual(model._is_static_graph_friendly, False) + self.assertEqual(model._static_graph_friendly, False) model.compile(RMSPropOptimizer(0.001), loss='mse') model.train_on_batch(np.random.random((2, 3)), np.random.random((2, 3))) @@ -102,7 +102,7 @@ class BaseLayerTest(test.TestCase): outputs = inner_model(x) model = keras.Model(inputs, outputs) - self.assertEqual(model._is_static_graph_friendly, False) + self.assertEqual(model._static_graph_friendly, False) model.compile(RMSPropOptimizer(0.001), loss='mse') model.train_on_batch(np.random.random((2, 3)), np.random.random((2, 3))) @@ -116,7 +116,7 @@ class BaseLayerTest(test.TestCase): inputs = keras.Input((3,)) outputs = InvalidLayer()(inputs) model = keras.Model(inputs, outputs) - self.assertEqual(model._is_static_graph_friendly, False) + self.assertEqual(model._static_graph_friendly, False) model.compile(RMSPropOptimizer(0.001), loss='mse') with self.assertRaisesRegexp(ValueError, 'You did something wrong!'): model.train_on_batch(np.random.random((2, 3)), np.random.random((2, 3))) diff --git a/tensorflow/python/keras/engine/base_layer_utils.py b/tensorflow/python/keras/engine/base_layer_utils.py new file mode 100644 index 00000000000..d2f947f1772 --- /dev/null +++ b/tensorflow/python/keras/engine/base_layer_utils.py @@ -0,0 +1,236 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Contains private utilities used mainly by the base Layer class.""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import collections as collections_lib +import enum + +from tensorflow.python.framework import dtypes +from tensorflow.python.framework import ops +from tensorflow.python.keras import backend +from tensorflow.python.ops import array_ops +from tensorflow.python.ops import init_ops +from tensorflow.python.ops import variables as tf_variables +from tensorflow.python.util import nest + + +class CallConvention(enum.Enum): + """Calling conventions for passing `Layer` inputs to `Layer.call`.""" + # The Layer takes inputs as its first argument, named "inputs" for + # compatibility with the signature of Layer.__call__. This is the mode assumed + # for Layers which are not subclassed Models. + EXPLICIT_INPUTS_ARGUMENT = 1 + # The Layer takes a single positional argument, not named "inputs". It's + # treated like an "inputs" argument. + SINGLE_POSITIONAL_ARGUMENT = 2 + # The Layer has multiple positional arguments to which its inputs should be + # bound. + POSITIONAL_ARGUMENTS_ARE_INPUTS = 3 + + +def create_mean_metric(value, name=None): + # TODO(psv): Remove this import when b/110718070 is fixed. + from tensorflow.python.keras import metrics as metrics_module # pylint: disable=g-import-not-at-top + metric_obj = metrics_module.Mean(name=name) + result = metric_obj(value) + return metric_obj, result + + +def make_variable(name, + shape=None, + dtype=dtypes.float32, + initializer=None, + partition_info=None, + trainable=None, + caching_device=None, + validate_shape=True, + constraint=None, + use_resource=None, + collections=None, + synchronization=tf_variables.VariableSynchronization.AUTO, + aggregation=tf_variables.VariableAggregation.NONE, + partitioner=None): # pylint: disable=unused-argument + """Temporary util to create a variable (relies on `variable_scope.variable`). + + Some reuse-related technicalities prevent us from using + `variable_scope.get_variable()` directly, so we use a subcomponent + that has fewer constraints (`variable_scope.variable()`). + + In the longer term, it seems like a similar "default variable creator" method + should exist in `CheckpointableBase` instead. When this happens, we can get + rid of this temporary solution. + + TODO(fchollet): remove this method when no longer needed. + TODO(fchollet): handle `partitioner` argument. + + Arguments: + name: Variable name. + shape: Variable shape. + dtype: The type of the variable. Defaults to `self.dtype` or `float32`. + initializer: Initializer instance (callable). + partition_info: Not handled at this time. + trainable: Whether the variable should be part of the layer's + "trainable_variables" (e.g. variables, biases) + or "non_trainable_variables" (e.g. BatchNorm mean, stddev). + Note, if the current variable scope is marked as non-trainable + then this parameter is ignored and any added variables are also + marked as non-trainable. `trainable` defaults to `True` unless + `synchronization` is set to `ON_READ`. + caching_device: Passed to `tf.Variable`. + validate_shape: Passed to `tf.Variable`. + constraint: Constraint instance (callable). + use_resource: Whether to use a `ResourceVariable`. + collections: List of graph collections keys. The new variable is added to + these collections. Defaults to `[GraphKeys.GLOBAL_VARIABLES]`. + synchronization: Indicates when a distributed a variable will be + aggregated. Accepted values are constants defined in the class + `tf.VariableSynchronization`. By default the synchronization is set to + `AUTO` and the current `DistributionStrategy` chooses + when to synchronize. If `synchronization` is set to `ON_READ`, + `trainable` must not be set to `True`. + aggregation: Indicates how a distributed variable will be aggregated. + Accepted values are constants defined in the class + `tf.VariableAggregation`. + partitioner: Not handled at this time. + + Returns: + Variable instance. + """ + initializing_from_value = False + if initializer is not None and not callable(initializer): + initializing_from_value = True + + with ops.init_scope(): + if initializing_from_value: + init_val = initializer + variable_dtype = None + else: + # Instantiate initializer if provided initializer is a type object. + if isinstance(initializer, type(init_ops.Initializer)): + initializer = initializer(dtype=dtype) + init_val = lambda: initializer( # pylint: disable=g-long-lambda + shape, dtype=dtype, partition_info=partition_info) + variable_dtype = dtype.base_dtype + if use_resource is None: + use_resource = True + + # TODO(apassos,rohanj) figure out how to remove collections from here so we + # can remove the V1. + v = tf_variables.VariableV1( + initial_value=init_val, + name=name, + trainable=trainable, + caching_device=caching_device, + dtype=variable_dtype, + validate_shape=validate_shape, + constraint=constraint, + use_resource=use_resource, + collections=collections, + synchronization=synchronization, + aggregation=aggregation) + return v + + +def get_default_graph_uid_map(): + # TODO(fchollet): refactor this into backend. + graph = ops.get_default_graph() + name_uid_map = backend.PER_GRAPH_LAYER_NAME_UIDS.get(graph, None) + if name_uid_map is None: + name_uid_map = collections_lib.defaultdict(int) + backend.PER_GRAPH_LAYER_NAME_UIDS[graph] = name_uid_map + return name_uid_map + + +def unique_layer_name(name, name_uid_map=None, avoid_names=None, namespace='', + zero_based=False): + """Makes a layer name (or arbitrary string) unique within a TensorFlow graph. + + Arguments: + name: String name to make unique. + name_uid_map: An optional defaultdict(int) to use when creating unique + names. If None (default), uses a per-Graph dictionary. + avoid_names: An optional set or dict with names which should not be used. If + None (default) does not avoid any names. + namespace: Gets a name which is unique within the (graph, namespace). Layers + which are not Networks use a blank namespace and so get graph-global + names. + zero_based: If True, name sequences start with no suffix (e.g. "dense", + "dense_1"). If False, naming is one-based ("dense_1", "dense_2"). + + Returns: + Unique string name. + + Example: + + ```python + _unique_layer_name('dense') # dense_1 + _unique_layer_name('dense') # dense_2 + ``` + """ + if name_uid_map is None: + name_uid_map = get_default_graph_uid_map() + if avoid_names is None: + avoid_names = set() + proposed_name = None + while proposed_name is None or proposed_name in avoid_names: + name_key = (namespace, name) + if zero_based: + number = name_uid_map[name_key] + if number: + proposed_name = name + '_' + str(number) + else: + proposed_name = name + name_uid_map[name_key] += 1 + else: + name_uid_map[name_key] += 1 + proposed_name = name + '_' + str(name_uid_map[name_key]) + return proposed_name + + +def collect_previous_mask(input_tensors): + """Retrieves the output mask(s) of the previous node. + + Arguments: + input_tensors: A tensor or list of tensors. + + Returns: + A mask tensor or list of mask tensors. + """ + input_tensors = nest.flatten(input_tensors) + masks = [] + for x in input_tensors: + if hasattr(x, '_keras_mask'): + mask = x._keras_mask # pylint: disable=protected-access + masks.append(mask) + else: + masks.append(None) + if len(masks) == 1: + return masks[0] + return masks + + +def have_all_keras_metadata(iterable_or_element): + if not isinstance(iterable_or_element, (list, tuple)): + iterable = [iterable_or_element] + else: + iterable = nest.flatten(iterable_or_element) + return all(hasattr(x, '_keras_history') for x in iterable) + + +def generate_placeholders_from_shape(shape): + return array_ops.placeholder(shape=shape, dtype=backend.floatx()) diff --git a/tensorflow/python/keras/engine/distributed_training_utils.py b/tensorflow/python/keras/engine/distributed_training_utils.py index f939b7565a8..d100182381e 100644 --- a/tensorflow/python/keras/engine/distributed_training_utils.py +++ b/tensorflow/python/keras/engine/distributed_training_utils.py @@ -22,15 +22,17 @@ import numpy as np from tensorflow.python.client import session as session_module from tensorflow.python.data.ops import dataset_ops from tensorflow.python.data.ops import iterator_ops +from tensorflow.python.distribute import distribute_coordinator_context as dc_context +from tensorflow.python.distribute import distribute_lib from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops from tensorflow.python.framework import tensor_util from tensorflow.python.keras import backend as K from tensorflow.python.keras import callbacks +from tensorflow.python.keras.optimizer_v2 import optimizer_v2 from tensorflow.python.ops import array_ops from tensorflow.python.ops import variables from tensorflow.python.platform import tf_logging as logging -from tensorflow.python.training import distribute as distribute_lib from tensorflow.python.util import nest @@ -52,14 +54,18 @@ def set_weights(distribution_strategy, dist_model, weights): num_param = len(layer.weights) layer_weights = weights[:num_param] for sw, w in zip(layer.weights, layer_weights): - assign_ops.append(distribution_strategy.unwrap(sw.assign(w))) - + if ops.executing_eagerly_outside_functions(): + sw.assign(w) + else: + assign_ops.append(distribution_strategy.unwrap(sw.assign(w))) weights = weights[num_param:] - K.get_session().run(assign_ops) + + if not ops.executing_eagerly_outside_functions(): + K.get_session().run(assign_ops) def unwrap_values(distribution_strategy, grouped_inputs, grouped_outputs, - grouped_updates, grouped_session_args, + grouped_updates=None, grouped_session_args=None, with_loss_tensor=False): """Unwrap and return the list of values contained in the PerDevice parameters. @@ -92,11 +98,8 @@ def unwrap_values(distribution_strategy, grouped_inputs, grouped_outputs, grouped_inputs) if with_loss_tensor: # reduce loss tensor before adding it to the list of fetches - loss = distribution_strategy.unwrap( - distribution_strategy.reduce(distribute_lib.get_loss_reduction(), - grouped_outputs[0], - destinations='/device:CPU:0'))[0] - + loss = distribution_strategy.reduce(distribute_lib.get_loss_reduction(), + grouped_outputs[0]) all_outputs = flatten_perdevice_values(distribution_strategy, grouped_outputs[1:]) all_outputs = [loss] + all_outputs @@ -104,20 +107,25 @@ def unwrap_values(distribution_strategy, grouped_inputs, grouped_outputs, all_outputs = flatten_perdevice_values(distribution_strategy, grouped_outputs) - all_updates = flatten_perdevice_values(distribution_strategy, - grouped_updates) + if grouped_updates: + all_updates = flatten_perdevice_values(distribution_strategy, + grouped_updates) + else: + all_updates = None all_session_args = {} - grouped_feed_dict = grouped_session_args.get('feed_dict') - if grouped_feed_dict: - all_session_args['feed_dict'] = flatten_perdevice_values( - distribution_strategy, grouped_feed_dict) + if grouped_session_args: + grouped_feed_dict = grouped_session_args.get('feed_dict') + if grouped_feed_dict: + all_session_args['feed_dict'] = flatten_perdevice_values( + distribution_strategy, grouped_feed_dict) - grouped_fetches = grouped_session_args.get('fetches') - if grouped_fetches: - all_session_args['fetches'] = flatten_perdevice_values( - distribution_strategy, grouped_fetches) + grouped_fetches = grouped_session_args.get('fetches') + if grouped_fetches: + all_session_args['fetches'] = flatten_perdevice_values( + distribution_strategy, grouped_fetches) + # TODO(priyag): Return only non empty/None values return all_inputs, all_outputs, all_updates, all_session_args @@ -144,11 +152,14 @@ def flatten_perdevice_values(distribution_strategy, perdevice_values): for e in distribution_strategy.unwrap(flattened)] -def validate_callbacks(input_callbacks): +def validate_callbacks(input_callbacks, optimizer, current_strategy): """Validate whether given callbacks are supported by DistributionStrategy. Args: input_callbacks: List of callbacks passed by the user to fit. + optimizer: Optimizer instance used to train the model. + current_strategy: The DistributionStrategy used to distribute training + and validation. Raises: ValueError: If `LearningRateScheduler` or `ReduceLROnPlateau` is one of the @@ -170,12 +181,18 @@ def validate_callbacks(input_callbacks): 'these attributes are not set. You can access each of ' 'the individual distributed models using the ' '`_grouped_model` attribute of your original model.') - if isinstance(callback, callbacks.LearningRateScheduler): - raise ValueError('LearningRateScheduler callback is not supported with ' - 'DistributionStrategy.') - if isinstance(callback, callbacks.ReduceLROnPlateau): - raise ValueError('ReduceLROnPlateau callback is not supported with ' - 'DistributionStrategy.') + if isinstance(callback, (callbacks.LearningRateScheduler, + callbacks.ReduceLROnPlateau)): + strategy_name = current_strategy.__class__.__name__ + # TODO(anjalisridhar): We might need to add a condition for multi + # worker strategy when we support it in Keras. + if is_tpu_strategy(current_strategy): + raise ValueError('%s callback is not supported with %s.' % + (callback, strategy_name)) + + if not isinstance(optimizer, optimizer_v2.OptimizerV2): + raise ValueError('You must specify a Keras Optimizer V2 when using ' + '%s callback with DistributionStrategy.' % callback) # If users want to use the TensorBoard callback they cannot use certain # features of the callback that involve accessing model attributes and @@ -293,19 +310,64 @@ def validate_all_tensor_shapes(x, x_values): ' inputs {}'.format(x)) +def _wait_for_variable_initialization(session): + """Utility to wait for variables to be initialized.""" + all_variables = K._get_variables(K.get_graph()) # pylint: disable=protected-access + candidate_vars = [] + for v in all_variables: + if not getattr(v, '_keras_initialized', False): + candidate_vars.append(v) + + if not candidate_vars: + return + + while True: + is_initialized = session.run( + [variables.is_variable_initialized(v) for v in candidate_vars]) + uninitialized_vars = [] + for flag, v in zip(is_initialized, candidate_vars): + if not flag: + uninitialized_vars.append(v) + v._keras_initialized = True # pylint: disable=protected-access + if not uninitialized_vars: + break + + +def init_restore_or_wait_for_variables(): + """Initialize or restore variables or wait for variables to be initialized.""" + session = K._get_session() # pylint: disable=protected-access + worker_context = dc_context.get_current_worker_context() + if not worker_context or worker_context.experimental_should_init: + # TODO(yuefengz): if checkpoints exit, restore from checkpoint. + K._initialize_variables(session) # pylint: disable=protected-access + else: + _wait_for_variable_initialization(session) + + def configure_and_create_session(distribution_strategy): """Configure session config and create a session with it.""" # TODO(priyag): Throw error if a session already exists. session_config = K.get_default_session_config() - distribution_strategy.configure(session_config) - if distribution_strategy.__class__.__name__ == 'TPUStrategy': - # TODO(priyag): Remove this workaround when Distributed Coordinator is - # integrated with keras and we can create a session from there. - master = distribution_strategy._tpu_cluster_resolver.master() # pylint: disable=protected-access + if is_tpu_strategy(distribution_strategy): + # TODO(priyag, yuefengz): Remove this workaround when Distribute + # Coordinator is integrated with keras and we can create a session from + # there. + distribution_strategy.configure(session_config) + master = distribution_strategy.extended._tpu_cluster_resolver.master() # pylint: disable=protected-access session = session_module.Session(config=session_config, target=master) else: - session = session_module.Session(config=session_config) + worker_context = dc_context.get_current_worker_context() + if worker_context: + dc_session_config = worker_context.session_config + # Merge the default session config to the one from distribute coordinator, + # which is fine for now since they don't have conflicting configurations. + dc_session_config.MergeFrom(session_config) + session = session_module.Session( + config=dc_session_config, target=worker_context.master_target) + else: + distribution_strategy.configure(session_config) + session = session_module.Session(config=session_config) K.set_session(session) @@ -334,11 +396,15 @@ def validate_inputs(x, y, distribution_strategy): 'Iterator. You must pass a `tf.data.Dataset` object or a ' 'numpy array as input.') - if distribution_strategy.__class__.__name__ == 'TPUStrategy': + if is_tpu_strategy(distribution_strategy): for i in [x, y]: - if isinstance(i, dataset_ops.Dataset): + if isinstance(i, dataset_ops.DatasetV2): shapes = nest.flatten(i.output_shapes) - if any([not s.is_fully_defined() for s in shapes]): + try: + s = next(s for s in shapes if not s.is_fully_defined()) + except StopIteration: + continue + else: raise ValueError( 'Using TPUs currently requires fully defined shapes. Either use ' 'set_shape() on the input tensors or use ' @@ -346,37 +412,97 @@ def validate_inputs(x, y, distribution_strategy): 'Found unknown shape {} in input {}.'.format(s, i)) -def get_input_batch_params(first_x_value, batch_size, distribution_strategy): +# TODO(b/118776054): Currently we support global batch size for TPUStrategy and +# core MirroredStrategy only. Remove this check when contrib MirroredStrategy is +# no longer needed. +def global_batch_size_supported(distribution_strategy): + return distribution_strategy.extended._global_batch_size # pylint: disable=protected-access + + +# TODO(sourabhbajaj): Remove this once we use the same API for all strategies. +def is_tpu_strategy(strategy): + """We're executing TPU Strategy.""" + return strategy is not None and strategy.__class__.__name__ == 'TPUStrategy' + + +def get_input_params(distribution_strategy, first_x_value, steps, batch_size, + is_training=False): """Calculate the number of batches and steps/steps_per_epoch. Args: + distribution_strategy: The DistributionStrategy used to compile the model. first_x_value: This is the first input numpy array that is passed in as the model input. - batch_size: The specified batch_size or the default batch_size of 32. - distribution_strategy: The current DistributionStrategy used to compile the - model. + steps: The specified number of steps. + batch_size: The specified batch_size. + is_training: Boolean to relax the constraints on consuming all the training + samples to keep compatibility till we support partial batches. Returns: - The steps or steps_per_epoch argument depending on if a user is - calling `fit`, `evaluate` or `predict`. + steps: The steps or steps_per_epoch argument depending on if a user is + calling `fit`, `evaluate` or `predict`. If the is_training flag is set + we don't require the number of samples to be used completely. + batch_size: The batch size to be used in model iterations. Raises: ValueError: If the number of batches or steps evaluates to 0. """ - num_batches = first_x_value.shape[0] // batch_size - if not num_batches: - raise ValueError('Please specify a batch_size that is smaller than' - 'the number of input samples %d.' % first_x_value.shape[0]) - steps = num_batches // distribution_strategy.num_replicas_in_sync - if not steps: - # TODO(anjalisridhar): Number of replicas in the error message may not - # convey what we want to the user. Is there another terminology that we can - # use that is consistent across different strategies? - raise ValueError('The number of batches %d is smaller than the number ' - 'of replicas %d used for DistributionStrategy. ' % - (num_batches, distribution_strategy.num_replicas_in_sync)) - return steps + num_samples = first_x_value.shape[0] + # TODO(b/118776054): Use global batch size for Keras/DS support. + # Currently this is only supported in TPUStrategy and CoreMirroredStrategy. + use_per_replica_batch = not global_batch_size_supported( + distribution_strategy) + + if steps is None: + if batch_size is None: + # If neither the batch size or number of steps are set. We choose the + # global batch size as the minimum of number of samples and 32. 32 is + # chosen to provide backward compatibility. + global_batch_size = min(num_samples, 32) + else: + # If the user provided the batch size we need to handle the case + # between different strategies that use the global/per-replica batch size + global_batch_size = batch_size + if use_per_replica_batch: + global_batch_size *= distribution_strategy.num_replicas_in_sync + if not is_training and num_samples % global_batch_size: + raise ValueError('The number of samples %s is not divisible by ' + 'batch size %s.' % (num_samples, global_batch_size)) + steps = num_samples // global_batch_size + else: + if batch_size is None: + # We calculate the batch size based on the number of steps specified + if num_samples % steps: + raise ValueError('The number of samples %s is not divisible by ' + 'steps %s. Please change the number of steps to a ' + 'value that can consume all the samples' % ( + num_samples, steps)) + global_batch_size = num_samples // steps + else: + # If the user provided the batch size we need to handle the case + # between different strategies that use the global/per-replica batch size + global_batch_size = batch_size + if use_per_replica_batch: + global_batch_size *= distribution_strategy.num_replicas_in_sync + + if num_samples < (global_batch_size * steps): + raise ValueError('Number of samples %s is less than samples required ' + 'for specified batch_size %s and steps %s' % ( + num_samples, global_batch_size, steps)) + + # We need to return the per replica or global batch size based on the strategy + if use_per_replica_batch: + if global_batch_size % distribution_strategy.num_replicas_in_sync: + raise ValueError( + 'The batch size (%s) could not be sharded evenly across the sync ' + 'replicas (%s) in the distribution strategy.' % ( + global_batch_size, distribution_strategy.num_replicas_in_sync)) + batch_size = global_batch_size // distribution_strategy.num_replicas_in_sync + else: + batch_size = global_batch_size + + return steps, batch_size def get_batch_dimension(iterator): @@ -387,33 +513,6 @@ def get_batch_dimension(iterator): return dims[0] if dims else None -def get_batch_size(num_replicas, num_samples, steps): - """Calculate and return batch size for numpy inputs. - - Args: - num_replicas: Number of devices over which the model input is distributed. - num_samples: Total number of input samples in the input numpy arrays. - steps: Number of steps that we run the model for. - - Returns: - batch size used to create the Dataset object from the input numpy arrays. - - """ - if num_samples % steps != 0: - logging.warning('The number of input samples %d is not evenly ' - 'divisible by the number of steps %d. ' - 'Some samples will not be processed as expected.' % - (num_samples, steps)) - global_batch_size = num_samples // steps - if global_batch_size % num_replicas != 0: - logging.warning('The total number of batches per step %d is not evenly ' - 'divisible by the number of replicas %d used in ' - 'DistributionStrategy. Some samples will not be processed ' - 'as expected.' % - (global_batch_size, num_replicas)) - return global_batch_size // num_replicas - - def get_cpu_device(distribution_strategy): """Returns the CPU device of the TPU host or the default CPU device string. @@ -429,12 +528,12 @@ def get_cpu_device(distribution_strategy): NotImplementedError: We currently don't support copying numpy data to multiple hosts in the case of Cloud TPU pods. """ - if distribution_strategy.__class__.__name__ == 'TPUStrategy': - if distribution_strategy.num_hosts > 1: + if is_tpu_strategy(distribution_strategy): + if distribution_strategy.extended.num_hosts > 1: raise NotImplementedError('TPUDistributionStrategy does not ' 'support numpy inputs when running on Cloud' 'TPU pods.') - return distribution_strategy.get_host_cpu_device(0) + return distribution_strategy.extended.get_host_cpu_device(0) else: # For all strategies except TPUDistributionStrategy # TODO(anjalisridhar): We may need to modify this when we add support for diff --git a/tensorflow/python/keras/engine/feature_columns_integration_test.py b/tensorflow/python/keras/engine/feature_columns_integration_test.py index e0478ee357b..b7549e013c9 100644 --- a/tensorflow/python/keras/engine/feature_columns_integration_test.py +++ b/tensorflow/python/keras/engine/feature_columns_integration_test.py @@ -18,11 +18,13 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function +from absl.testing import parameterized import numpy as np from tensorflow.python import keras from tensorflow.python.data.ops import dataset_ops -from tensorflow.python.feature_column import feature_column_v2 as fc +from tensorflow.python.eager import context +from tensorflow.python.feature_column import feature_column_lib as fc from tensorflow.python.framework import test_util as tf_test_util from tensorflow.python.keras import metrics as metrics_module from tensorflow.python.platform import test @@ -33,7 +35,7 @@ class TestDNNModel(keras.models.Model): def __init__(self, feature_columns, units, name=None, **kwargs): super(TestDNNModel, self).__init__(name=name, **kwargs) - self._input_layer = fc.FeatureLayer(feature_columns, name='input_layer') + self._input_layer = fc.DenseFeatures(feature_columns, name='input_layer') self._dense_layer = keras.layers.Dense(units, name='dense_layer') def call(self, features): @@ -42,7 +44,7 @@ class TestDNNModel(keras.models.Model): return net -class FeatureColumnsIntegrationTest(test.TestCase): +class FeatureColumnsIntegrationTest(test.TestCase, parameterized.TestCase): """Most Sequential model API tests are covered in `training_test.py`. """ @@ -51,7 +53,7 @@ class FeatureColumnsIntegrationTest(test.TestCase): def test_sequential_model(self): columns = [fc.numeric_column('a')] model = keras.models.Sequential([ - fc.FeatureLayer(columns), + fc.DenseFeatures(columns), keras.layers.Dense(64, activation='relu'), keras.layers.Dense(20, activation='softmax') ]) @@ -72,7 +74,7 @@ class FeatureColumnsIntegrationTest(test.TestCase): def test_sequential_model_with_ds_input(self): columns = [fc.numeric_column('a')] model = keras.models.Sequential([ - fc.FeatureLayer(columns), + fc.DenseFeatures(columns), keras.layers.Dense(64, activation='relu'), keras.layers.Dense(20, activation='softmax') ]) @@ -112,8 +114,10 @@ class FeatureColumnsIntegrationTest(test.TestCase): dnn_model.evaluate(x=x, y=y, batch_size=5) dnn_model.predict(x=x, batch_size=5) + @parameterized.parameters(True, False) @tf_test_util.run_in_graph_and_eager_modes - def test_subclassed_model_with_feature_columns_with_ds_input(self): + def test_subclassed_model_with_feature_columns_with_ds_input(self, + run_eagerly): col_a = fc.numeric_column('a') col_b = fc.numeric_column('b') @@ -122,7 +126,8 @@ class FeatureColumnsIntegrationTest(test.TestCase): dnn_model.compile( optimizer=rmsprop.RMSPropOptimizer(learning_rate=0.001), loss='categorical_crossentropy', - metrics=['accuracy']) + metrics=['accuracy'], + run_eagerly=run_eagerly and context.executing_eagerly()) y = np.random.randint(20, size=(100, 1)) y = keras.utils.to_categorical(y, num_classes=20) @@ -140,10 +145,10 @@ class FeatureColumnsIntegrationTest(test.TestCase): col_a = fc.numeric_column('a') col_b = fc.numeric_column('b') - feature_layer = fc.FeatureLayer([col_a, col_b], name='fc') + feature_layer = fc.DenseFeatures([col_a, col_b], name='fc') dense = keras.layers.Dense(4) - # This seems problematic.... We probably need something for FeatureLayer + # This seems problematic.... We probably need something for DenseFeatures # the way Input is for InputLayer. output = dense(feature_layer) @@ -167,11 +172,11 @@ class FeatureColumnsIntegrationTest(test.TestCase): col_b = fc.numeric_column('b') col_c = fc.numeric_column('c') - fc1 = fc.FeatureLayer([col_a, col_b], name='fc1') - fc2 = fc.FeatureLayer([col_b, col_c], name='fc2') + fc1 = fc.DenseFeatures([col_a, col_b], name='fc1') + fc2 = fc.DenseFeatures([col_b, col_c], name='fc2') dense = keras.layers.Dense(4) - # This seems problematic.... We probably need something for FeatureLayer + # This seems problematic.... We probably need something for DenseFeatures # the way Input is for InputLayer. output = dense(fc1) + dense(fc2) diff --git a/tensorflow/python/keras/engine/input_layer.py b/tensorflow/python/keras/engine/input_layer.py index 4e96106004f..9874efe2bcc 100644 --- a/tensorflow/python/keras/engine/input_layer.py +++ b/tensorflow/python/keras/engine/input_layer.py @@ -19,12 +19,10 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function -from tensorflow.python.eager import context from tensorflow.python.framework import tensor_shape from tensorflow.python.keras import backend from tensorflow.python.keras.engine import base_layer from tensorflow.python.keras.utils import tf_utils -from tensorflow.python.ops import array_ops from tensorflow.python.util.tf_export import tf_export @@ -84,7 +82,6 @@ class InputLayer(base_layer.Layer): self.sparse = sparse self.batch_size = batch_size self.supports_masking = True - self._can_use_graph_functions = True if isinstance(input_shape, tensor_shape.TensorShape): input_shape = tuple(input_shape.as_list()) @@ -95,19 +92,19 @@ class InputLayer(base_layer.Layer): else: batch_input_shape = None graph = backend.get_graph() - with context.graph_mode(): - with graph.as_default(): - # In graph mode, create a graph placeholder to call the layer on. - if sparse: - input_tensor = array_ops.sparse_placeholder( - shape=batch_input_shape, - dtype=dtype, - name=self.name) - else: - input_tensor = array_ops.placeholder( - shape=batch_input_shape, - dtype=dtype, - name=self.name) + with graph.as_default(): + # In graph mode, create a graph placeholder to call the layer on. + if sparse: + input_tensor = backend.placeholder( + shape=batch_input_shape, + dtype=dtype, + name=self.name, + sparse=True) + else: + input_tensor = backend.placeholder( + shape=batch_input_shape, + dtype=dtype, + name=self.name) self.is_placeholder = True self._batch_input_shape = batch_input_shape diff --git a/tensorflow/python/keras/engine/input_spec.py b/tensorflow/python/keras/engine/input_spec.py new file mode 100644 index 00000000000..7277c16fe51 --- /dev/null +++ b/tensorflow/python/keras/engine/input_spec.py @@ -0,0 +1,170 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +# pylint: disable=protected-access +"""Contains the InputSpec class.""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from six.moves import zip # pylint: disable=redefined-builtin + +from tensorflow.python.util import nest +from tensorflow.python.util.tf_export import tf_export + + +@tf_export('keras.layers.InputSpec', + v1=['keras.layers.InputSpec', 'layers.InputSpec']) +class InputSpec(object): + """Specifies the ndim, dtype and shape of every input to a layer. + + Every layer should expose (if appropriate) an `input_spec` attribute: + a list of instances of InputSpec (one per input tensor). + + A None entry in a shape is compatible with any dimension, + a None shape is compatible with any shape. + + Arguments: + dtype: Expected DataType of the input. + shape: Shape tuple, expected shape of the input + (may include None for unchecked axes). + ndim: Integer, expected rank of the input. + max_ndim: Integer, maximum rank of the input. + min_ndim: Integer, minimum rank of the input. + axes: Dictionary mapping integer axes to + a specific dimension value. + """ + + def __init__(self, + dtype=None, + shape=None, + ndim=None, + max_ndim=None, + min_ndim=None, + axes=None): + self.dtype = dtype + self.shape = shape + if shape is not None: + self.ndim = len(shape) + else: + self.ndim = ndim + self.max_ndim = max_ndim + self.min_ndim = min_ndim + self.axes = axes or {} + + def __repr__(self): + spec = [('dtype=' + str(self.dtype)) if self.dtype else '', + ('shape=' + str(self.shape)) if self.shape else '', + ('ndim=' + str(self.ndim)) if self.ndim else '', + ('max_ndim=' + str(self.max_ndim)) if self.max_ndim else '', + ('min_ndim=' + str(self.min_ndim)) if self.min_ndim else '', + ('axes=' + str(self.axes)) if self.axes else ''] + return 'InputSpec(%s)' % ', '.join(x for x in spec if x) + + +def assert_input_compatibility(input_spec, inputs, layer_name): + """Checks compatibility between the layer and provided inputs. + + This checks that the tensor(s) `inputs` verify the input assumptions + of a layer (if any). If not, a clear and actional exception gets raised. + + Arguments: + input_spec: An InputSpec instance, or None. + inputs: Input tensor or list of input tensors. + layer_name: String, name of the layer (for error message formatting). + + Raises: + ValueError: in case of mismatch between + the provided inputs and the expectations of the layer. + """ + if not input_spec: + return + if not isinstance(input_spec, (list, tuple)): + input_spec = nest.flatten(input_spec) + + inputs = nest.flatten(inputs) + if len(inputs) != len(input_spec): + raise ValueError('Layer ' + layer_name + ' expects ' + + str(len(input_spec)) + ' inputs, ' + 'but it received ' + str(len(inputs)) + + ' input tensors. Inputs received: ' + str(inputs)) + for input_index, (x, spec) in enumerate(zip(inputs, input_spec)): + if spec is None: + continue + + if (spec.ndim is not None or + spec.min_ndim is not None or + spec.max_ndim is not None): + if x.shape.ndims is None: + raise ValueError('Input ' + str(input_index) + ' of layer ' + + layer_name + ' is incompatible with the layer: ' + 'its rank is undefined, but the layer requires a ' + 'defined rank.') + + # Check ndim. + if spec.ndim is not None: + ndim = x.shape.ndims + if ndim != spec.ndim: + raise ValueError('Input ' + str(input_index) + ' of layer ' + + layer_name + ' is incompatible with the layer: ' + 'expected ndim=' + str(spec.ndim) + ', found ndim=' + + str(ndim) + '. Full shape received: ' + + str(x.shape.as_list())) + if spec.max_ndim is not None: + ndim = x.shape.ndims + if ndim is not None and ndim > spec.max_ndim: + raise ValueError('Input ' + str(input_index) + ' of layer ' + + layer_name + ' is incompatible with the layer: ' + 'expected max_ndim=' + str(spec.max_ndim) + + ', found ndim=' + str(ndim)) + if spec.min_ndim is not None: + ndim = x.shape.ndims + if ndim is not None and ndim < spec.min_ndim: + raise ValueError('Input ' + str(input_index) + ' of layer ' + + layer_name + ' is incompatible with the layer: ' + ': expected min_ndim=' + str(spec.min_ndim) + + ', found ndim=' + str(ndim) + + '. Full shape received: ' + + str(x.shape.as_list())) + # Check dtype. + if spec.dtype is not None: + if x.dtype != spec.dtype: + raise ValueError('Input ' + str(input_index) + ' of layer ' + + layer_name + ' is incompatible with the layer: ' + 'expected dtype=' + str(spec.dtype) + + ', found dtype=' + str(x.dtype)) + # Check specific shape axes. + if spec.axes: + shape = x.shape.as_list() + if shape is not None: + for axis, value in spec.axes.items(): + if hasattr(value, 'value'): + value = value.value + if value is not None and shape[int(axis)] not in {value, None}: + raise ValueError( + 'Input ' + str(input_index) + ' of layer ' + layer_name + ' is' + ' incompatible with the layer: expected axis ' + str(axis) + + ' of input shape to have value ' + str(value) + + ' but received input with shape ' + str(shape)) + # Check shape. + if spec.shape is not None: + shape = x.shape.as_list() + if shape is not None: + for spec_dim, dim in zip(spec.shape, shape): + if spec_dim is not None and dim is not None: + if spec_dim != dim: + raise ValueError('Input ' + str(input_index) + + ' is incompatible with layer ' + layer_name + + ': expected shape=' + str(spec.shape) + + ', found shape=' + str(shape)) diff --git a/tensorflow/python/keras/engine/network.py b/tensorflow/python/keras/engine/network.py index 9b58180e3d3..7e6cc7bfeef 100644 --- a/tensorflow/python/keras/engine/network.py +++ b/tensorflow/python/keras/engine/network.py @@ -36,7 +36,9 @@ from tensorflow.python.framework import ops from tensorflow.python.framework import tensor_shape from tensorflow.python.keras import backend from tensorflow.python.keras.engine import base_layer +from tensorflow.python.keras.engine import base_layer_utils from tensorflow.python.keras.engine import saving +from tensorflow.python.keras.engine import training_utils from tensorflow.python.keras.utils import generic_utils from tensorflow.python.keras.utils import layer_utils from tensorflow.python.keras.utils import tf_utils @@ -112,11 +114,6 @@ class Network(base_layer.Layer): self.trainable = True self._is_compiled = False self._expects_training_arg = False - # A list of "extra" variables assigned to attributes of this class, included - # in self.weights and self.variables. Always empty for graph networks (but - # included in base_init to avoid excessive special casing when retrieving - # the value). - self._extra_variables = [] # In many internal cases one needs to compute both the model's output # and its output mask without relying on `__call__` (which would do both and # set mask metadata), but for models, computing the mask requires to @@ -134,12 +131,19 @@ class Network(base_layer.Layer): self.optimizer = None # Private attributes to implement compatibility with Layer. + self._trainable_weights = [] + self._non_trainable_weights = [] self._updates = [] # Used in symbolic mode only. self._losses = [] self._eager_losses = [] + # A list of metric instances corresponding to the symbolic metric tensors + # added using the `add_metric` API. + self._metrics = [] + # A dictionary that maps metric names to metric result tensors. + self._metrics_tensors = {} self._scope = None # Never used. self._reuse = None # Never used. - self._can_use_graph_functions = False + self._call_is_graph_friendly = True if context.executing_eagerly(): self._graph = None else: @@ -160,7 +164,8 @@ class Network(base_layer.Layer): @checkpointable.no_automatic_dependency_tracking def _init_graph_network(self, inputs, outputs, name=None): - self._call_convention = base_layer.CallConvention.EXPLICIT_INPUTS_ARGUMENT + self._call_convention = (base_layer_utils + .CallConvention.EXPLICIT_INPUTS_ARGUMENT) # Normalize and set self.inputs, self.outputs. if isinstance(inputs, (list, tuple)): self.inputs = list(inputs) # Tensor or list of tensors. @@ -170,43 +175,7 @@ class Network(base_layer.Layer): self.outputs = list(outputs) else: self.outputs = [outputs] - - # Check for redundancy in inputs. - if len(set(self.inputs)) != len(self.inputs): - raise ValueError('The list of inputs passed to the model ' - 'is redundant. ' - 'All inputs should only appear once.' - ' Found: ' + str(self.inputs)) - for x in self.inputs: - # Check that x has appropriate `_keras_history` metadata. - if not hasattr(x, '_keras_history'): - cls_name = self.__class__.__name__ - raise ValueError('Input tensors to a ' + cls_name + ' ' + - 'must come from `tf.keras.Input`. ' - 'Received: ' + str(x) + - ' (missing previous layer metadata).') - # Check that x is an input tensor. - # pylint: disable=protected-access - layer, node_index, tensor_index = x._keras_history - if len(layer._inbound_nodes) > 1 or ( - layer._inbound_nodes and layer._inbound_nodes[0].inbound_layers): - cls_name = self.__class__.__name__ - logging.warning(cls_name + ' inputs must come from ' - '`tf.keras.Input` (thus holding past layer metadata), ' - 'they cannot be the output of ' - 'a previous non-Input layer. ' - 'Here, a tensor specified as ' - 'input to "' + self.name + '" was not an Input tensor, ' - 'it was generated by layer ' + layer.name + '.\n' - 'Note that input tensors are ' - 'instantiated via `tensor = tf.keras.Input(shape)`.\n' - 'The tensor that caused the issue was: ' + str(x.name)) - for x in self.outputs: - if not hasattr(x, '_keras_history'): - cls_name = self.__class__.__name__ - raise ValueError('Output tensors to a ' + cls_name + ' must be ' - 'the output of a TensorFlow `Layer` ' - '(thus holding past layer metadata). Found: ' + str(x)) + self._validate_graph_inputs_and_outputs() self._base_init(name=name) self._compute_previous_mask = ( @@ -258,10 +227,6 @@ class Network(base_layer.Layer): self._track_layers(layers) - # A Graph network supports defun-ed eager loops if all of its layers do. - self._can_use_graph_functions = all( - layer._can_use_graph_functions for layer in layers) - # Create the node linking internal inputs to internal outputs. base_layer.Node( outbound_layer=self, @@ -282,9 +247,7 @@ class Network(base_layer.Layer): if layer.is_placeholder: self._feed_input_names.append(layer.name) self._feed_input_shapes.append(backend.int_shape(self.inputs[i])) - # layer.input gives an error in eager mode - if not context.executing_eagerly(): - self._feed_inputs.append(layer.input) + self._feed_inputs.append(layer.input) for layer in self._output_layers: self.output_names.append(layer.name) @@ -301,16 +264,15 @@ class Network(base_layer.Layer): self.outputs = [] self.inputs = [] self.built = False - self._static_graph_friendly = True @property - def _is_static_graph_friendly(self): + def _static_graph_friendly(self): if self._is_graph_network: - return all(layer._is_static_graph_friendly for layer in self.layers) - return self._static_graph_friendly + return all(layer._static_graph_friendly for layer in self.layers) + return self._call_is_graph_friendly def _determine_call_convention(self, call_argspec): - """Decides how `self.call()` is invoked. See base_layer.CallConvention.""" + """Decides how `self.call()` is invoked. See `CallConvention`.""" if call_argspec.varargs: may_take_single_argument = False else: @@ -342,11 +304,11 @@ class Network(base_layer.Layer): "Model.call() takes a single positional argument (to which " "inputs are passed by convention) and a separate 'inputs' " "argument. Unable to determine which arguments are inputs.") - return base_layer.CallConvention.SINGLE_POSITIONAL_ARGUMENT + return base_layer_utils.CallConvention.SINGLE_POSITIONAL_ARGUMENT if 'inputs' in call_argspec.args: - return base_layer.CallConvention.EXPLICIT_INPUTS_ARGUMENT + return base_layer_utils.CallConvention.EXPLICIT_INPUTS_ARGUMENT else: - return base_layer.CallConvention.POSITIONAL_ARGUMENTS_ARE_INPUTS + return base_layer_utils.CallConvention.POSITIONAL_ARGUMENTS_ARE_INPUTS def _track_layers(self, layers): """Add Checkpointable dependencies on a list of Layers.""" @@ -415,44 +377,26 @@ class Network(base_layer.Layer): # simply by assigning them to attributes. not self._is_graph_network and isinstance(value, variables.Variable)): - self._extra_variables.append(value) + if value.trainable: + # Could already be added via `add_weight`. + if value not in self._trainable_weights: + self._trainable_weights.append(value) + else: + if value not in self._non_trainable_weights: + self._non_trainable_weights.append(value) + + # Keeping track of metric instance created in subclassed model/layer. + # We do this so that we can maintain the correct order of metrics by adding + # the instance to the `metrics` list as soon as it is created. + from tensorflow.python.keras import metrics as metrics_module # pylint: disable=g-import-not-at-top + if isinstance(value, metrics_module.Metric): + self._metrics.append(value) super(Network, self).__setattr__(name, value) - def add_variable(self, name, shape, dtype=None, initializer=None, - regularizer=None, trainable=True, constraint=None): - if self._is_graph_network: - raise NotImplementedError('`add_variable` is not supported on Networks.') - else: - raise NotImplementedError( - '`add_variable` is not supported on Networks. However, you may ' - 'assign variables to attributes and they will show up in the weights ' - 'and variables properties.') - - def add_weight(self, - name, - shape, - dtype=None, - initializer=None, - regularizer=None, - trainable=None, - constraint=None, - partitioner=None, - use_resource=None, - synchronization=variables.VariableSynchronization.AUTO, - aggregation=variables.VariableAggregation.NONE, - **kwargs): - if self._is_graph_network: - raise NotImplementedError('`add_weight` is not supported on Networks.') - else: - raise NotImplementedError( - '`add_weight` is not supported on Networks. However, you may ' - 'assign variables to attributes and they will show up in the weights ' - 'and variables properties.') - @property def stateful(self): - return any([(hasattr(layer, 'stateful') and layer.stateful) - for layer in self.layers]) + return any((hasattr(layer, 'stateful') and layer.stateful) + for layer in self.layers) def reset_states(self): for layer in self.layers: @@ -557,14 +501,13 @@ class Network(base_layer.Layer): @property def _unfiltered_updates(self): - if context.executing_eagerly(): - return [] updates = [] for layer in self.layers: if isinstance(layer, Network): updates += layer._unfiltered_updates else: updates += layer.updates + updates += self._updates return updates @property @@ -641,9 +584,6 @@ class Network(base_layer.Layer): Returns: A list of update ops. """ - if context.executing_eagerly(): - return [] - if not self.trainable and not self.stateful: return [] @@ -659,7 +599,7 @@ class Network(base_layer.Layer): else: relevant_inputs.append(inputs) if not relevant_inputs: - return updates + return list(set(updates)) reachable = tf_utils.get_reachable_from_inputs(relevant_inputs, updates) relevant_conditional_updates = [x for x in updates if x in reachable] @@ -667,8 +607,7 @@ class Network(base_layer.Layer): x for x in updates if x._unconditional_update] # pylint: disable=protected-access # A layer could be used multiple times in a nested structure, # so the updates list must be de-duped. - return list(set( - relevant_conditional_updates + unconditional_updates + self._updates)) + return list(set(relevant_conditional_updates + unconditional_updates)) @property def losses(self): @@ -728,14 +667,38 @@ class Network(base_layer.Layer): return checkpointable_layer_utils.gather_trainable_weights( trainable=self.trainable, sub_layers=self._layers, - extra_variables=self._extra_variables) + extra_variables=self._trainable_weights) @property def non_trainable_weights(self): return checkpointable_layer_utils.gather_non_trainable_weights( trainable=self.trainable, sub_layers=self._layers, - extra_variables=self._extra_variables) + extra_variables=self._non_trainable_weights + self._trainable_weights) + + @property + def metrics(self): + """Returns the network's symbolic metrics. + + Model overrides this function to include the metrics from `compile` API. + """ + metrics = [] + for layer in self.layers: + metrics += layer._metrics # pylint: disable=protected-access + return metrics + self._metrics + + @property + def _all_metrics_tensors(self): + """Returns the network's symbolic metric tensors.""" + # TODO(psv): Remove this property. + metrics_tensors = {} + for layer in self.layers: + if isinstance(layer, Network): + metrics_tensors.update(layer._all_metrics_tensors) + else: + metrics_tensors.update(layer._metrics_tensors) + metrics_tensors.update(self._metrics_tensors) + return metrics_tensors @property def input_spec(self): @@ -771,6 +734,11 @@ class Network(base_layer.Layer): This is to be used for subclassed models, which do not know at instantiation time what their inputs look like. + This method only exists for users who want to call `model.build()` in a + standalone way (as a substitute for calling the model on real data to + build it). It will never be called by the framework (and thus it will + never throw unexpected errors in an unrelated workflow). + Args: input_shape: Single tuple, TensorShape, or list of shapes, where shapes are tuples, integers, or TensorShapes. @@ -807,48 +775,53 @@ class Network(base_layer.Layer): # in a Graph. Since tf.Variable is compatible with both eager execution # and graph building, the variables created after building the model in # a Graph are still valid when executing eagerly. - with context.graph_mode(): - graph = func_graph.FuncGraph('graph') - with graph.as_default(): - if isinstance(input_shape, list): - x = [base_layer.generate_placeholders_from_shape(shape) - for shape in input_shape] + if context.executing_eagerly(): + graph = func_graph.FuncGraph('build_graph') + else: + graph = backend.get_graph() + with graph.as_default(): + if isinstance(input_shape, list): + x = [base_layer_utils.generate_placeholders_from_shape(shape) + for shape in input_shape] + else: + x = base_layer_utils.generate_placeholders_from_shape(input_shape) + + kwargs = {} + call_signature = tf_inspect.getfullargspec(self.call) + call_args = call_signature.args + # Exclude `self`, `inputs`, and any argument with a default value. + if len(call_args) > 2: + if call_signature.defaults: + call_args = call_args[2:-len(call_signature.defaults)] else: - x = base_layer.generate_placeholders_from_shape(input_shape) - - kwargs = {} - num_call_args = len(tf_inspect.getfullargspec(self.call).args) - if self._expects_training_arg and num_call_args == 3: - # Has call signature of call(self, input, training) - kwargs['training'] = False - elif num_call_args > 2: - # Has invalid call signature of call(self, input, *args, **kwargs) - raise ValueError('Currently, you cannot build your model if it has ' - 'positional or keyword arguments that are not ' - 'inputs to the model, but are required for its ' - '`call` method. Instead, in order to instantiate ' - 'and build your model, `call` your model on real ' - 'tensor data with all expected call arguments.') - - try: - self.call(x, **kwargs) - except (errors.InvalidArgumentError, TypeError): - raise ValueError('You cannot build your model by calling `build` ' - 'if your layers do not support float type inputs. ' - 'Instead, in order to instantiate and build your ' - 'model, `call` your model on real tensor data (of ' - 'the correct dtype).') - + call_args = call_args[2:] + for arg in call_args: + if arg == 'training': + # Case where `training` is a positional arg with no default. + kwargs['training'] = False + else: + # Has invalid call signature with unknown positional arguments. + raise ValueError( + 'Currently, you cannot build your model if it has ' + 'positional or keyword arguments that are not ' + 'inputs to the model, but are required for its ' + '`call` method. Instead, in order to instantiate ' + 'and build your model, `call` your model on real ' + 'tensor data with all expected call arguments.') + elif len(call_args) < 2: + # Signature without `inputs`. + raise ValueError('You can only call `build` on a model if its `call` ' + 'method accepts an `inputs` argument.') + try: + self.call(x, **kwargs) + except (errors.InvalidArgumentError, TypeError): + raise ValueError('You cannot build your model by calling `build` ' + 'if your layers do not support float type inputs. ' + 'Instead, in order to instantiate and build your ' + 'model, `call` your model on real tensor data (of ' + 'the correct dtype).') if self._layers: self._track_layers(self._layers) - if self.layers: - for layer in self.layers: - if not layer.built: - raise ValueError('Layer: {} was not built in your model. Calling ' - '`build` manually on a subclassed model is only ' - 'allowed for models with a static topology. ' - 'In this case, you can build your model by ' - 'calling it on real tensor data.'.format(layer)) self.built = True def call(self, inputs, training=None, mask=None): @@ -895,9 +868,7 @@ class Network(base_layer.Layer): def compute_output_shape(self, input_shape): if not self._is_graph_network: - if context.executing_eagerly(): - return super(Network, self).compute_output_shape(input_shape) - raise NotImplementedError + return super(Network, self).compute_output_shape(input_shape) if isinstance(input_shape, list): input_shapes = [] @@ -1686,6 +1657,62 @@ class Network(base_layer.Layer): positions=positions, print_fn=print_fn) + def _validate_graph_inputs_and_outputs(self): + """Validates the inputs and outputs of a Graph Network.""" + # Check for redundancy in inputs. + if len(set(self.inputs)) != len(self.inputs): + raise ValueError('The list of inputs passed to the model ' + 'is redundant. ' + 'All inputs should only appear once.' + ' Found: ' + str(self.inputs)) + + for x in self.inputs: + # Check that x has appropriate `_keras_history` metadata. + if not hasattr(x, '_keras_history'): + cls_name = self.__class__.__name__ + raise ValueError('Input tensors to a ' + cls_name + ' ' + + 'must come from `tf.keras.Input`. ' + 'Received: ' + str(x) + + ' (missing previous layer metadata).') + # Check that x is an input tensor. + # pylint: disable=protected-access + layer, _, _ = x._keras_history + if len(layer._inbound_nodes) > 1 or ( + layer._inbound_nodes and layer._inbound_nodes[0].inbound_layers): + cls_name = self.__class__.__name__ + logging.warning(cls_name + ' inputs must come from ' + '`tf.keras.Input` (thus holding past layer metadata), ' + 'they cannot be the output of ' + 'a previous non-Input layer. ' + 'Here, a tensor specified as ' + 'input to "' + self.name + '" was not an Input tensor, ' + 'it was generated by layer ' + layer.name + '.\n' + 'Note that input tensors are ' + 'instantiated via `tensor = tf.keras.Input(shape)`.\n' + 'The tensor that caused the issue was: ' + str(x.name)) + + # Check compatibility of batch sizes of Input Layers. + input_batch_sizes = [ + training_utils.get_static_batch_size(x._keras_history[0]) + for x in self.inputs + ] + consistent_batch_size = None + for batch_size in input_batch_sizes: + if batch_size is not None: + if (consistent_batch_size is not None and + batch_size != consistent_batch_size): + raise ValueError('The specified batch sizes of the Input Layers' + ' are incompatible. Found batch sizes: {}'.format( + input_batch_sizes)) + consistent_batch_size = batch_size + + for x in self.outputs: + if not hasattr(x, '_keras_history'): + cls_name = self.__class__.__name__ + raise ValueError('Output tensors to a ' + cls_name + ' must be ' + 'the output of a TensorFlow `Layer` ' + '(thus holding past layer metadata). Found: ' + str(x)) + def _is_hdf5_filepath(filepath): return (filepath.endswith('.h5') or filepath.endswith('.keras') or diff --git a/tensorflow/python/keras/engine/saving.py b/tensorflow/python/keras/engine/saving.py index 61bff7fff23..54d9e32fb25 100644 --- a/tensorflow/python/keras/engine/saving.py +++ b/tensorflow/python/keras/engine/saving.py @@ -79,6 +79,10 @@ def save_model(model, filepath, overwrite=True, include_optimizer=True): from tensorflow.python.keras import __version__ as keras_version # pylint: disable=g-import-not-at-top + # TODO(psv) Add warning when we save models that contain non-serializable + # entities like metrics added using `add_metric` and losses added using + # `add_loss.` + if not isinstance(filepath, h5py.File): # If file exists and should not be overwritten. if not overwrite and os.path.isfile(filepath): @@ -126,8 +130,8 @@ def save_model(model, filepath, overwrite=True, include_optimizer=True): 'config': model.optimizer.get_config() }, 'loss': model.loss, - 'metrics': model.metrics, - 'weighted_metrics': model.weighted_metrics, + 'metrics': model._compile_metrics, + 'weighted_metrics': model._compile_weighted_metrics, 'sample_weight_mode': model.sample_weight_mode, 'loss_weights': model.loss_weights, }, @@ -913,7 +917,7 @@ def save_attributes_to_hdf5_group(group, name, data): chunked_data = np.array_split(data_npy, num_chunks) # This will never loop forever thanks to the test above. - while any([x.nbytes > HDF5_OBJECT_HEADER_LIMIT for x in chunked_data]): + while any(x.nbytes > HDF5_OBJECT_HEADER_LIMIT for x in chunked_data): num_chunks += 1 chunked_data = np.array_split(data_npy, num_chunks) diff --git a/tensorflow/python/keras/engine/saving_test.py b/tensorflow/python/keras/engine/saving_test.py index f376f081cfe..6d9d9a2fcae 100644 --- a/tensorflow/python/keras/engine/saving_test.py +++ b/tensorflow/python/keras/engine/saving_test.py @@ -32,6 +32,7 @@ from tensorflow.python.framework import ops from tensorflow.python.framework import test_util from tensorflow.python.keras.engine import saving from tensorflow.python.keras.engine import training +from tensorflow.python.lib.io import file_io from tensorflow.python.ops import array_ops from tensorflow.python.ops import random_ops from tensorflow.python.platform import test @@ -288,6 +289,7 @@ class TestWeightSavingAndLoading(test.TestCase, parameterized.TestCase): r'element\(s\)\.'): saving.load_weights_from_hdf5_group_by_name(f_model, model.layers) + @test_util.run_deprecated_v1 def test_sequential_weight_loading_group_name_with_incorrect_shape(self): if h5py is None: return @@ -330,6 +332,7 @@ class TestWeightSavingAndLoading(test.TestCase, parameterized.TestCase): class TestWholeModelSaving(test.TestCase): + @test_util.run_deprecated_v1 def test_sequential_model_saving(self): if h5py is None: self.skipTest('h5py required to run this test') @@ -382,6 +385,7 @@ class TestWholeModelSaving(test.TestCase): out2 = new_model.predict(x) self.assertAllClose(out, out2, atol=1e-05) + @test_util.run_deprecated_v1 def test_sequential_model_saving_without_input_shape(self): if h5py is None: self.skipTest('h5py required to run this test') @@ -442,6 +446,7 @@ class TestWholeModelSaving(test.TestCase): out2 = new_model.predict(x) self.assertAllClose(out, out2, atol=1e-05) + @test_util.run_deprecated_v1 def test_sequential_model_saving_2(self): if h5py is None: self.skipTest('h5py required to run this test') @@ -478,6 +483,7 @@ class TestWholeModelSaving(test.TestCase): out2 = model.predict(x) self.assertAllClose(out, out2, atol=1e-05) + @test_util.run_deprecated_v1 def test_functional_model_saving(self): if h5py is None: self.skipTest('h5py required to run this test') @@ -629,6 +635,7 @@ class TestWholeModelSaving(test.TestCase): os.close(fd) os.remove(fname) + @test_util.run_deprecated_v1 def test_saving_model_with_long_weights_names(self): if h5py is None: self.skipTest('h5py required to run this test') @@ -674,6 +681,7 @@ class TestWholeModelSaving(test.TestCase): os.close(fd) os.remove(fname) + @test_util.run_deprecated_v1 def test_model_saving_to_pre_created_h5py_file(self): if h5py is None: self.skipTest('h5py required to run this test') @@ -715,7 +723,6 @@ class TestWholeModelSaving(test.TestCase): os.close(fd) os.remove(fname) - def test_saving_constant_initializer_with_numpy(self): if h5py is None: self.skipTest('h5py required to run this test') @@ -749,6 +756,7 @@ class SubclassedModel(training.Model): class TestWeightSavingAndLoadingTFFormat(test.TestCase): + @test_util.run_deprecated_v1 def test_keras_optimizer_warning(self): graph = ops.Graph() with graph.as_default(), self.session(graph): @@ -992,5 +1000,57 @@ class TestWeightSavingAndLoadingTFFormat(test.TestCase): AssertionError, 'Nothing except the root object matched'): m.load_weights(save_path) + @test_util.run_in_graph_and_eager_modes + def test_directory_passed(self): + m = keras.Model() + v = m.add_weight(name='v', shape=[]) + self.evaluate(v.assign(42.)) + prefix = os.path.join(self.get_temp_dir(), '{}'.format(ops.uid()), 'ckpt/') + m.save_weights(prefix) + self.evaluate(v.assign(2.)) + m.load_weights(prefix) + self.assertEqual(42., self.evaluate(v)) + + @test_util.run_in_graph_and_eager_modes + def test_relative_path(self): + m = keras.Model() + v = m.add_weight(name='v', shape=[]) + os.chdir(self.get_temp_dir()) + + prefix = 'ackpt' + self.evaluate(v.assign(42.)) + m.save_weights(prefix) + self.assertTrue(file_io.file_exists('ackpt.index')) + self.evaluate(v.assign(1.)) + m.load_weights(prefix) + self.assertEqual(42., self.evaluate(v)) + + prefix = 'subdir/ackpt' + self.evaluate(v.assign(43.)) + m.save_weights(prefix) + self.assertTrue(file_io.file_exists('subdir/ackpt.index')) + self.evaluate(v.assign(2.)) + m.load_weights(prefix) + self.assertEqual(43., self.evaluate(v)) + + prefix = 'ackpt/' + self.evaluate(v.assign(44.)) + m.save_weights(prefix) + self.assertTrue(file_io.file_exists('ackpt/.index')) + self.evaluate(v.assign(3.)) + m.load_weights(prefix) + self.assertEqual(44., self.evaluate(v)) + + @test_util.run_in_graph_and_eager_modes + def test_nonexistant_prefix_directory(self): + m = keras.Model() + v = m.add_weight(name='v', shape=[]) + self.evaluate(v.assign(42.)) + prefix = os.path.join(self.get_temp_dir(), '{}'.format(ops.uid()), 'bckpt') + m.save_weights(prefix) + self.evaluate(v.assign(2.)) + m.load_weights(prefix) + self.assertEqual(42., self.evaluate(v)) + if __name__ == '__main__': test.main() diff --git a/tensorflow/python/keras/engine/sequential.py b/tensorflow/python/keras/engine/sequential.py index 5ce4ca4df41..3255613f6af 100644 --- a/tensorflow/python/keras/engine/sequential.py +++ b/tensorflow/python/keras/engine/sequential.py @@ -25,6 +25,7 @@ from tensorflow.python.eager import context from tensorflow.python.framework import ops from tensorflow.python.keras import layers as layer_module from tensorflow.python.keras.engine import base_layer +from tensorflow.python.keras.engine import training_utils from tensorflow.python.keras.engine.input_layer import Input from tensorflow.python.keras.engine.input_layer import InputLayer from tensorflow.python.keras.engine.network import Network @@ -120,8 +121,8 @@ class Sequential(Model): return layers[:] @property - def _is_static_graph_friendly(self): - return all(layer._is_static_graph_friendly for layer in self.layers) + def _static_graph_friendly(self): + return all(layer._static_graph_friendly for layer in self.layers) @checkpointable.no_automatic_dependency_tracking def add(self, layer): @@ -150,7 +151,7 @@ class Sequential(Model): assert len(layer._inbound_nodes[-1].output_tensors) == 1 set_inputs = True else: - batch_shape, dtype = get_input_shape_and_dtype(layer) + batch_shape, dtype = training_utils.get_input_shape_and_dtype(layer) if batch_shape: # Instantiate an input layer. x = Input( @@ -190,8 +191,6 @@ class Sequential(Model): self._layers.append(layer) if self._layers: self._track_layers(self._layers) - self._can_use_graph_functions = all( - layer._can_use_graph_functions for layer in self.layers) @checkpointable.no_automatic_dependency_tracking def pop(self): @@ -213,23 +212,17 @@ class Sequential(Model): self.outputs = [self.layers[-1].output] self._init_graph_network(self.inputs, self.outputs, name=self.name) self.built = True - self._can_use_graph_functions = all( - layer._can_use_graph_functions for layer in self.layers) + @base_layer.default def build(self, input_shape=None): if self._is_graph_network: self._init_graph_network(self.inputs, self.outputs, name=self.name) else: if input_shape is None: raise ValueError('You must provide an `input_shape` argument.') + input_shape = tuple(input_shape) self._build_input_shape = input_shape - shape = input_shape - for layer in self.layers: - if not layer.built: - with ops.name_scope(layer._name_scope()): - layer.build(shape) - layer.built = True - shape = layer.compute_output_shape(shape) + super(Sequential, self).build(input_shape) self.built = True def call(self, inputs, training=None, mask=None): @@ -241,8 +234,8 @@ class Sequential(Model): return outputs def _call_and_compute_mask(self, inputs, training=None, mask=None): - if not self.built: - self.build(inputs.shape) + if not self.built and self._is_graph_network: + self._init_graph_network(self.inputs, self.outputs, name=self.name) x = inputs for layer in self.layers: @@ -255,6 +248,11 @@ class Sequential(Model): if isinstance(layer, Network) and layer._compute_output_and_mask_jointly: x, mask = layer._call_and_compute_mask(x, **kwargs) else: + if not layer.built: + # Build layer if applicable. + with ops.name_scope(layer._name_scope()): + layer._maybe_build(x) + layer.built = True x = layer.call(x, **kwargs) if layer.supports_masking: mask = layer.compute_mask(x, mask) @@ -362,38 +360,3 @@ class Sequential(Model): if self.layers and hasattr(self.layers[0], 'input_spec'): return self.layers[0].input_spec return None - - -def get_input_shape_and_dtype(layer): - """Retrieve input shape and input dtype of layer if applicable. - - Args: - layer: Layer (or model) instance. - - Returns: - Tuple (input_shape, input_dtype). Both could be None if the layer - does not have a defined input shape. - - Raises: - ValueError: in case an empty Sequential or Graph Network is passed. - """ - if ((isinstance(layer, Model) and layer._is_graph_network) - or isinstance(layer, Sequential)): - # We were passed a model as first layer. - # This requires a specific way to figure out the - # input shape and dtype. - if not layer.layers: - raise ValueError('Cannot add an empty model ' - 'to a `Sequential` model.') - # In case of nested models: recover the first layer - # of the deepest model to infer input shape and dtype. - layer = layer.layers[0] - while ((isinstance(layer, Model) and layer._is_graph_network) - or isinstance(layer, Sequential)): - layer = layer.layers[0] - - if hasattr(layer, '_batch_input_shape'): - batch_shape = layer._batch_input_shape - dtype = layer.dtype - return batch_shape, dtype - return None, None diff --git a/tensorflow/python/keras/engine/sequential_test.py b/tensorflow/python/keras/engine/sequential_test.py index ea8fdf675a0..b6d2510897f 100644 --- a/tensorflow/python/keras/engine/sequential_test.py +++ b/tensorflow/python/keras/engine/sequential_test.py @@ -124,7 +124,7 @@ class TestSequential(test.TestCase, parameterized.TestCase): dataset = dataset_ops.Dataset.from_tensor_slices((x, y)) dataset = dataset.repeat(100) dataset = dataset.batch(10) - iterator = dataset.make_one_shot_iterator() + iterator = dataset_ops.make_one_shot_iterator(dataset) model.fit(iterator, epochs=1, steps_per_epoch=steps_per_epoch) self.assertTrue(model.built) @@ -132,6 +132,7 @@ class TestSequential(test.TestCase, parameterized.TestCase): self.assertFalse(model._is_graph_network) @parameterized.parameters((True,), (False,)) + @tf_test_util.run_deprecated_v1 def test_training_and_eval_methods_on_symbolic_tensors(self, deferred): with self.cached_session(): @@ -219,6 +220,7 @@ class TestSequential(test.TestCase, parameterized.TestCase): inner_model.trainable = True self.assertEqual(len(model.trainable_weights), 4) + @tf_test_util.run_deprecated_v1 def test_sequential_update_disabling(self): val_a = np.random.random((10, 4)) val_out = np.random.random((10, 4)) @@ -294,7 +296,6 @@ class TestSequential(test.TestCase, parameterized.TestCase): model.build((None, 10)) self.assertTrue(model.built) - self.assertTrue(model.layers[-1].built) self.assertEqual(len(model.weights), 8) @tf_test_util.run_in_graph_and_eager_modes @@ -362,29 +363,6 @@ class TestSequentialEagerIntegration(test.TestCase): y = np.random.random((2, 5)) model.fit(x, y, epochs=1) - @tf_test_util.run_in_graph_and_eager_modes - def test_sequential_can_use_graph_functions(self): - model = testing_utils.get_small_sequential_mlp(4, 3) - self.assertTrue(model._can_use_graph_functions) - inner_model = testing_utils.get_small_sequential_mlp(4, 5) - model.add(inner_model) - - self.assertTrue(model._can_use_graph_functions) - - inner_model_two = testing_utils.get_small_sequential_mlp(5, 7) - self.assertTrue(inner_model_two._can_use_graph_functions) - - layer = keras.layers.Lambda(lambda x: x) - layer._can_use_graph_functions = False - inner_model_two.add(layer) - self.assertFalse(inner_model_two._can_use_graph_functions) - - model.add(inner_model_two) - self.assertFalse(model._can_use_graph_functions) - - model.pop() - self.assertTrue(model._can_use_graph_functions) - @tf_test_util.run_in_graph_and_eager_modes def test_sequential_model_fails_with_dict_inputs(self): num_classes = 5 diff --git a/tensorflow/python/keras/engine/topology_test.py b/tensorflow/python/keras/engine/topology_test.py index b4a4babf259..03bfd35589c 100644 --- a/tensorflow/python/keras/engine/topology_test.py +++ b/tensorflow/python/keras/engine/topology_test.py @@ -42,6 +42,7 @@ except ImportError: class TopologyConstructionTest(test.TestCase): + @test_util.run_deprecated_v1 def test_get_updates(self): class MyLayer(keras.layers.Layer): @@ -115,6 +116,7 @@ class TopologyConstructionTest(test.TestCase): self.assertEqual(len(layer.get_updates_for(x1)), 2) self.assertEqual(len(layer.get_updates_for(None)), 0) + @test_util.run_deprecated_v1 def test_get_losses(self): class MyLayer(keras.layers.Layer): @@ -268,6 +270,7 @@ class TopologyConstructionTest(test.TestCase): self.assertEqual(test_layer.input_shape, [(None, 32), (None, 32)]) self.assertEqual(test_layer.output_shape, (None, 32)) + @test_util.run_deprecated_v1 def testBasicNetwork(self): # minimum viable network x = input_layer_lib.Input(shape=(32,)) @@ -341,6 +344,7 @@ class TopologyConstructionTest(test.TestCase): self.assertListEqual(model.trainable_weights, []) self.assertListEqual(model.non_trainable_weights, weights) + @test_util.run_deprecated_v1 def test_layer_call_arguments(self): # Test the ability to pass and serialize arguments to `call`. inp = keras.layers.Input(shape=(2,)) @@ -491,6 +495,7 @@ class TopologyConstructionTest(test.TestCase): fn_outputs = fn([input_a_np, input_b_np]) self.assertListEqual([x.shape for x in fn_outputs], [(10, 64), (10, 5)]) + @test_util.run_deprecated_v1 def test_recursion(self): with self.cached_session(): a = keras.layers.Input(shape=(32,), name='input_a') @@ -675,6 +680,7 @@ class TopologyConstructionTest(test.TestCase): with self.assertRaises(Exception): keras.models.Model([j, k], [m, n, 0]) + @test_util.run_deprecated_v1 def test_raw_tf_compatibility(self): # test calling layers/models on TF tensors a = keras.layers.Input(shape=(32,), name='input_a') @@ -719,6 +725,7 @@ class TopologyConstructionTest(test.TestCase): model = keras.models.Model(a, b) self.assertEqual(model.output_mask.get_shape().as_list(), [None, 10]) + @test_util.run_deprecated_v1 def testMaskingSingleInput(self): class MaskedLayer(keras.layers.Layer): @@ -756,6 +763,7 @@ class TopologyConstructionTest(test.TestCase): y_2 = network(x_2) self.assertEqual(y_2.get_shape().as_list(), [None, 32]) + @test_util.run_deprecated_v1 def test_activity_regularization_with_model_composition(self): def reg(x): @@ -825,6 +833,7 @@ class TopologyConstructionTest(test.TestCase): output_val_2 = m2.predict(x_val) self.assertAllClose(output_val, output_val_2, atol=1e-6) + @test_util.run_deprecated_v1 def test_explicit_training_argument(self): with self.cached_session(): a = keras.layers.Input(shape=(2,)) @@ -1145,6 +1154,7 @@ class DefaultShapeInferenceBehaviorTest(test.TestCase): class GraphUtilsTest(test.TestCase): + @test_util.run_deprecated_v1 def testGetReachableFromInputs(self): with self.cached_session(): diff --git a/tensorflow/python/keras/engine/training.py b/tensorflow/python/keras/engine/training.py index cb96e3e5d20..2236bcf27c5 100644 --- a/tensorflow/python/keras/engine/training.py +++ b/tensorflow/python/keras/engine/training.py @@ -18,6 +18,7 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function +import collections import weakref import numpy as np @@ -26,6 +27,7 @@ from tensorflow.python.data.ops import iterator_ops from tensorflow.python.eager import context from tensorflow.python.framework import errors from tensorflow.python.framework import ops +from tensorflow.python.framework import tensor_shape from tensorflow.python.framework import tensor_util from tensorflow.python.keras import backend as K from tensorflow.python.keras import losses @@ -40,6 +42,8 @@ from tensorflow.python.keras.engine import training_utils from tensorflow.python.keras.engine.network import Network from tensorflow.python.keras.utils import data_utils from tensorflow.python.keras.utils.generic_utils import slice_arrays +from tensorflow.python.keras.utils.losses_utils import squeeze_or_expand_dimensions +from tensorflow.python.ops import math_ops from tensorflow.python.platform import tf_logging as logging from tensorflow.python.training import optimizer as tf_optimizer_module from tensorflow.python.training.checkpointable import base as checkpointable @@ -121,11 +125,8 @@ class Model(Network): # initializing _distribution_strategy here since it is possible to call # predict on a model without compiling it. self._distribution_strategy = None - # This flag must be disabled upon model mutation, such as changing the model - # layers or recompiling the model to use a different optimizer. New function - # definitions are generated whenever this flag is disabled, ensuring that - # internal graph functions are always using the current model structure. - self._built_graph_functions = False + + self.run_eagerly = None def _set_sample_weight_attributes(self, sample_weight_mode, skip_target_weighing_indices): @@ -177,25 +178,66 @@ class Model(Network): metric_name = '%s_%s' % (self.output_names[output_index], metric_name) j = 1 base_metric_name = metric_name - while metric_name in self.metrics_names: + while metric_name in self._compile_metrics_names: metric_name = '%s_%d' % (base_metric_name, j) j += 1 return metric_name + @property + def metrics(self): + """Returns the model's metrics added using `compile`, `add_metric` APIs.""" + metrics = [] + if self._is_compiled: + metrics += self._compile_stateful_metric_functions + return metrics + super(Model, self).metrics + + @property + def metrics_names(self): + """Returns the model's display labels for all outputs.""" + metrics_names = [] + if self._is_compiled: + metrics_names += self._compile_metrics_names # Includes names of losses. + + # Add metric names from layers. + for layer in self.layers: + metrics_names += [m.name for m in layer._metrics] # pylint: disable=protected-access + metrics_names += [m.name for m in self._metrics] + return metrics_names + + @property + def _all_metrics_tensors(self): + """Returns the network's symbolic metric tensors.""" + metrics_tensors = {} + if self._is_compiled: + metrics_tensors.update(self._compile_metrics_tensors) + metrics_tensors.update(super(Model, self)._all_metrics_tensors) + return metrics_tensors + + @property + def _all_stateful_metrics_tensors(self): + """Returns the network's symbolic metric tensors.""" + metrics_tensors = {} + if self._is_compiled: + metrics_tensors.update(self._compile_stateful_metrics_tensors) + metrics_tensors.update(super(Model, self)._all_metrics_tensors) + return metrics_tensors + def _init_metric_attributes(self): """Initialized model metric attributes.""" # List of all metric names in the model. - self.metrics_names = ['loss'] - # List of all aggregated metric result tensors. This includes aggregated - # loss result tensors. - self._stateful_metrics_tensors = [] - # List of all metric result tensors (aggregated or not - based on the - # values given in compile.) - self.metrics_tensors = [] + self._compile_metrics_names = ['loss'] # List of stateful metric functions. Used for resetting metric state during - # training/eval. This includes loss functions. - self.stateful_metric_functions = [] + # training/eval. + # This includes loss functions when there are multiple outputs. + self._compile_stateful_metric_functions = [] + # Dict of all aggregated metric result tensors. This includes aggregated + # loss result tensors when there are multiple outputs. + self._compile_stateful_metrics_tensors = {} + # Dict of all metric result tensors (aggregated or not - based on the + # values given in compile.). This includes aggregated loss result tensors + # when there are multiple outputs. + self._compile_metrics_tensors = {} def _set_per_output_metric_attributes(self, metrics_dict, output_index): """Sets the metric attributes on the model for the given output. @@ -204,24 +246,39 @@ class Model(Network): metrics_dict: A dict with metric names as keys and metric fns as values. output_index: The index of the model output for which the metric attributes are added. - """ - for metric_name, (_, stateful_metric_fn) in metrics_dict.items(): - metric_name = self._add_unique_metric_name(metric_name, output_index) - # Keep track of metric name. - self.metrics_names.append(metric_name) - # Keep track of stateful metric function. - self.stateful_metric_functions.append(stateful_metric_fn) + Returns: + Metrics dict updated with unique metric names as keys. + """ + updated_metrics_dict = collections.OrderedDict() + for metric_name, (metric_fn, stateful_metric_fn) in metrics_dict.items(): + metric_name = self._add_unique_metric_name(metric_name, output_index) + updated_metrics_dict[metric_name] = (metric_fn, stateful_metric_fn) + # Keep track of metric name, function and stateful function. + self._compile_metrics_names.append(metric_name) + self._compile_stateful_metric_functions.append(stateful_metric_fn) + return updated_metrics_dict def _set_metric_attributes(self, outputs, skip_target_indices=None): """Sets the metric attributes on the model for all the model outputs.""" skip_target_indices = skip_target_indices or [] + updated_per_output_metrics = [] + updated_per_output_weighted_metrics = [] for i in range(len(outputs)): if i in skip_target_indices: + updated_per_output_metrics.append(self._per_output_metrics[i]) + updated_per_output_weighted_metrics.append( + self._per_output_weighted_metrics[i]) continue - self._set_per_output_metric_attributes(self._per_output_metrics[i], i) - self._set_per_output_metric_attributes( - self._per_output_weighted_metrics[i], i) + updated_per_output_metrics.append( + self._set_per_output_metric_attributes(self._per_output_metrics[i], + i)) + updated_per_output_weighted_metrics.append( + self._set_per_output_metric_attributes( + self._per_output_weighted_metrics[i], i)) + + self._per_output_metrics = updated_per_output_metrics + self._per_output_weighted_metrics = updated_per_output_weighted_metrics def _handle_per_output_metrics(self, metrics_dict, @@ -256,17 +313,17 @@ class Model(Network): weighted_metric_fn = training_utils.weighted_masked_objective(fn) return weighted_metric_fn(y_true, y_pred, weights=weights, mask=mask) - def _track_metric_tensors(stateless_result, stateful_result): - self.metrics_tensors.append(stateless_result) - self._stateful_metrics_tensors.append(stateful_result) + def _track_metric_tensors(name, stateless_result, stateful_result): + self._compile_metrics_tensors[name] = stateless_result + self._compile_stateful_metrics_tensors[name] = stateful_result if isinstance(metric_fn, metrics_module.Metric): # If the given metric fn is stateful, call the fn and return result. metric_result = _call_stateful_fn(metric_fn) metric_results.append(metric_result) - if not context.executing_eagerly(): - _track_metric_tensors(metric_result, metric_result) - elif context.executing_eagerly(): + if not self.run_eagerly: + _track_metric_tensors(metric_name, metric_result, metric_result) + elif self.run_eagerly: # In eager mode, if the given metric fn is not stateful, we invoke the # given fn or its stateful version based on the given flag. if return_stateful_result: @@ -279,7 +336,8 @@ class Model(Network): # stateless fns. stateful_metric_result = _call_stateful_fn(stateful_fn) metric_result = _call_stateless_fn(metric_fn) - _track_metric_tensors(metric_result, stateful_metric_result) + _track_metric_tensors(metric_name, metric_result, + stateful_metric_result) return metric_results @@ -307,6 +365,7 @@ class Model(Network): skip_target_indices = skip_target_indices or [] metric_results = [] with K.name_scope('metrics'): + # Invoke all metrics added using `compile`. for i in range(len(outputs)): if i in skip_target_indices: continue @@ -328,8 +387,48 @@ class Model(Network): output_mask, weights=sample_weights[i], return_stateful_result=return_stateful_result)) + + # Add metric results from the `add_metric` metrics in eager mode. + if context.executing_eagerly(): + for m in self.metrics: + if m not in self._compile_stateful_metric_functions: + metric_results.append(m.result()) return metric_results + @property + def run_eagerly(self): + """Settable attribute indicating whether the model should run eagerly. + + Running eagerly means that your model will be run step by step, + like Python code. Your model might run slower, but it should become easier + for you to debug it by stepping into individual layer calls. + + By default, we will attempt to compile your model to a static graph to + deliver the best execution performance. + + Returns: + Boolean, whether the model should run eagerly. + """ + if self._run_eagerly is True and not context.executing_eagerly(): + raise ValueError('You can only set `run_eagerly=True` if eager execution ' + 'is enabled.') + if self._static_graph_friendly: + if self._run_eagerly is None: + return False + else: + return self._run_eagerly + else: + if self._run_eagerly is False: + # TODO(fchollet): consider using py_func to enable this. + raise ValueError('Your model contains layers that can only be ' + 'successfully run in eager execution. ' + 'You cannot set `run_eagerly=False`.') + return context.executing_eagerly() + + @run_eagerly.setter + def run_eagerly(self, value): + self._run_eagerly = value + @checkpointable.no_automatic_dependency_tracking def compile(self, optimizer, @@ -391,9 +490,8 @@ class Model(Network): ValueError: In case of invalid arguments for `optimizer`, `loss`, `metrics` or `sample_weight_mode`. """ - # The correct graph function may have changed, - # already-built ones must be updated - self._built_graph_functions = False + run_eagerly = kwargs.pop('run_eagerly', None) + self._run_eagerly = run_eagerly # Validate that arguments passed by the user to `compile` are supported by # DistributionStrategy. @@ -403,9 +501,6 @@ class Model(Network): raise NotImplementedError( 'optimizer must be an instance of ' 'tf.train.Optimizer, not a %s' % type(optimizer)) - if context.executing_eagerly(): - raise NotImplementedError('DistributionStrategy is not supported ' - 'when eager execution is enabled.') if sample_weight_mode: raise NotImplementedError('sample_weight_mode is not supported with ' 'DistributionStrategy.') @@ -417,11 +512,12 @@ class Model(Network): 'DistributionStrategy.') loss = loss or {} - if context.executing_eagerly() and not isinstance( + if self.run_eagerly and not isinstance( optimizer, (tf_optimizer_module.Optimizer, optimizers.TFOptimizer)): raise ValueError( - 'optimizer must be an instance of tf.train.Optimizer, not ' - 'a %s' % type(optimizer)) + 'When running a model in eager execution, the optimizer must be an ' + 'instance of tf.train.Optimizer. Received: ' + '%s' % optimizer) self.optimizer = optimizers.get(optimizer) # We've disabled automatic dependency tracking for this method, but do want @@ -430,12 +526,14 @@ class Model(Network): self._track_checkpointable( self.optimizer, name='optimizer', overwrite=True) self.loss = loss - self.metrics = metrics or [] + self._compile_metrics = metrics or [] self.loss_weights = loss_weights self.sample_weight_mode = sample_weight_mode - self.weighted_metrics = weighted_metrics - if context.executing_eagerly() and target_tensors is not None: - raise ValueError('target_tensors is not supported in Eager mode.') + self._compile_weighted_metrics = weighted_metrics + if self.run_eagerly and target_tensors is not None: + raise ValueError( + 'target_tensors argument is not supported when ' + 'running a model eagerly.') self.target_tensors = target_tensors # Set DistributionStrategy specific parameters. @@ -445,6 +543,8 @@ class Model(Network): if self._distribution_strategy is not None: distributed_training_utils.configure_and_create_session( self._distribution_strategy) + # Initialize model metric attributes. + self._init_metric_attributes() if not self.built: # Model is not compilable because it does not know its number of inputs # and outputs, nor their shapes and names. We will compile after the first @@ -468,16 +568,16 @@ class Model(Network): '" missing from loss dictionary. We assume ' 'this was done on purpose. The fit and evaluate APIs will not be ' 'expecting any data to be passed to "' + name + '".') - loss_functions.append(losses.get(loss.get(name))) + loss_functions.append(training_utils.get_loss_function(loss.get(name))) elif isinstance(loss, list): if len(loss) != len(self.outputs): raise ValueError('When passing a list as loss, ' 'it should have one entry per model outputs. ' 'The model has ' + str(len(self.outputs)) + ' outputs, but you passed loss=' + str(loss)) - loss_functions = [losses.get(l) for l in loss] + loss_functions = [training_utils.get_loss_function(l) for l in loss] else: - loss_function = losses.get(loss) + loss_function = training_utils.get_loss_function(loss) loss_functions = [loss_function for _ in range(len(self.outputs))] self.loss_functions = loss_functions @@ -493,7 +593,7 @@ class Model(Network): skip_target_weighing_indices.append(i) # Prepare output masks. - if not context.executing_eagerly(): + if not self.run_eagerly: masks = [getattr(x, '_keras_mask', None) for x in self.outputs] if not isinstance(masks, list): masks = [masks] @@ -524,11 +624,8 @@ class Model(Network): str(loss_weights) + ' - expected a list of dicts.') self.loss_weights_list = loss_weights_list - # Initialize model metric attributes. - self._init_metric_attributes() - # Initialization for Eager mode execution. - if context.executing_eagerly(): + if self.run_eagerly: # Prepare sample weights. self._set_sample_weight_attributes(sample_weight_mode, skip_target_weighing_indices) @@ -541,7 +638,7 @@ class Model(Network): self.total_loss = None for i in range(len(self.outputs)): if len(self.outputs) > 1: - self.metrics_names.append(self.output_names[i] + '_loss') + self._compile_metrics_names.append(self.output_names[i] + '_loss') # Set metric attributes on model. self._set_metric_attributes( @@ -555,145 +652,167 @@ class Model(Network): self._collected_trainable_weights = self.trainable_weights return - # Prepare targets of model. - self.targets = [] - self._feed_targets = [] - if target_tensors not in (None, []): - if isinstance(target_tensors, list): - if len(target_tensors) != len(self.outputs): - raise ValueError( - 'When passing a list as `target_tensors`, ' - 'it should have one entry per model output. ' - 'The model has ' + str(len(self.outputs)) + - ' outputs, but you passed target_tensors=' + str(target_tensors)) - elif isinstance(target_tensors, dict): - for name in target_tensors: - if name not in self.output_names: + with K.get_graph().as_default(): + # Prepare targets of model. + self.targets = [] + self._feed_targets = [] + if target_tensors not in (None, []): + if isinstance(target_tensors, list): + if len(target_tensors) != len(self.outputs): raise ValueError( - 'Unknown entry in `target_tensors` ' - 'dictionary: "' + name + '". ' - 'Only expected the following keys: ' + str(self.output_names)) - tmp_target_tensors = [] - for name in self.output_names: - tmp_target_tensors.append(target_tensors.get(name, None)) - target_tensors = tmp_target_tensors - elif tensor_util.is_tensor(target_tensors): - target_tensors = [target_tensors] - else: - raise TypeError('Expected `target_tensors` to be a list or tuple or ' - 'dict or a single tensor, but got:', target_tensors) - - for i in range(len(self.outputs)): - if i in skip_target_indices: - self.targets.append(None) - else: - shape = K.int_shape(self.outputs[i]) - name = self.output_names[i] - if target_tensors not in (None, []): - target = target_tensors[i] + 'When passing a list as `target_tensors`, ' + 'it should have one entry per model output. ' + 'The model has %s outputs, but you passed target_tensors=%s' % + (len(self.outputs), target_tensors)) + elif isinstance(target_tensors, dict): + for name in target_tensors: + if name not in self.output_names: + raise ValueError( + 'Unknown entry in `target_tensors` ' + 'dictionary: "' + name + '". ' + 'Only expected the following keys: ' + str(self.output_names)) + tmp_target_tensors = [] + for name in self.output_names: + tmp_target_tensors.append(target_tensors.get(name, None)) + target_tensors = tmp_target_tensors + elif tensor_util.is_tensor(target_tensors): + target_tensors = [target_tensors] else: - target = None - if target is None or K.is_placeholder(target): - if target is None: - target = K.placeholder( - ndim=len(shape), - name=name + '_target', - sparse=K.is_sparse(self.outputs[i]), - dtype=K.dtype(self.outputs[i])) - self._feed_targets.append(target) - self._feed_outputs.append(self.outputs[i]) - self._feed_output_names.append(name) - self._feed_output_shapes.append(shape) - self._feed_loss_fns.append(self.loss_functions[i]) - else: - skip_target_weighing_indices.append(i) - self.targets.append(target) + raise TypeError('Expected `target_tensors` to be a list or tuple or ' + 'dict or a single tensor, but got:', target_tensors) - # Prepare sample weights. - self._set_sample_weight_attributes(sample_weight_mode, - skip_target_weighing_indices) - # Save all metric attributes per output of the model. - self._cache_output_metric_attributes(metrics, weighted_metrics) - - # Compute total loss. - total_loss = None - with K.name_scope('loss'): for i in range(len(self.outputs)): if i in skip_target_indices: - continue - y_true = self.targets[i] - y_pred = self.outputs[i] - loss_fn = loss_functions[i] - sample_weight = self.sample_weights[i] - mask = masks[i] - loss_weight = loss_weights_list[i] - with K.name_scope(self.output_names[i] + '_loss'): - weighted_loss = training_utils.weighted_masked_objective(loss_fn) - output_loss = weighted_loss(y_true, y_pred, sample_weight, mask) + self.targets.append(None) + else: + shape = K.int_shape(self.outputs[i]) + name = self.output_names[i] + if target_tensors not in (None, []): + target = target_tensors[i] + else: + target = None + if target is None or K.is_placeholder(target): + if target is None: + target_dtype = losses.LABEL_DTYPES_FOR_LOSSES.get( + self.loss_functions[i], + K.dtype(self.outputs[i])) - if len(self.outputs) > 1: - # Keep track of the un-aggregated loss result tensor. - self.metrics_tensors.append(output_loss) + target = K.placeholder( + ndim=len(shape), + name=name + '_target', + sparse=K.is_sparse(self.outputs[i]), + dtype=target_dtype) + self._feed_targets.append(target) + self._feed_outputs.append(self.outputs[i]) + self._feed_output_names.append(name) + self._feed_output_shapes.append(shape) + self._feed_loss_fns.append(self.loss_functions[i]) + else: + skip_target_weighing_indices.append(i) + self.targets.append(target) - # Keep track of stateful result tensor and function for the loss. - mean_wrapped_loss = metrics_module.MeanMetricWrapper( - loss_fn, name=loss_fn.__name__) - result_tensor = training_utils.call_metric_function( - mean_wrapped_loss, - y_true, - y_pred, - weights=sample_weight, - mask=mask) - self._stateful_metrics_tensors.append(result_tensor) - self.stateful_metric_functions.append(mean_wrapped_loss) + # Prepare sample weights. + self._set_sample_weight_attributes(sample_weight_mode, + skip_target_weighing_indices) + # Save all metric attributes per output of the model. + self._cache_output_metric_attributes(metrics, weighted_metrics) - self.metrics_names.append(self.output_names[i] + '_loss') + # Compute total loss. + total_loss = None + with K.name_scope('loss'): + for i in range(len(self.outputs)): + if i in skip_target_indices: + continue + y_true = self.targets[i] + y_pred = self.outputs[i] + loss_fn = loss_functions[i] + sample_weight = self.sample_weights[i] + mask = masks[i] + loss_weight = loss_weights_list[i] + with K.name_scope(self.output_names[i] + '_loss'): + if isinstance(loss_fn, losses.Loss): + if mask is not None: + mask = math_ops.cast(mask, y_pred.dtype) + # Update weights with mask. + if sample_weight is None: + sample_weight = mask + else: + # Update dimensions of weights to match with mask if possible. + mask, _, sample_weight = squeeze_or_expand_dimensions( + mask, None, sample_weight) + sample_weight *= mask + output_loss = loss_fn(y_true, y_pred, sample_weight=sample_weight) + else: + weighted_loss = training_utils.weighted_masked_objective(loss_fn) + output_loss = weighted_loss(y_true, y_pred, sample_weight, mask) + + if len(self.outputs) > 1: + # Keep track of the un-aggregated loss result tensor. + self._compile_metrics_tensors[self.output_names[i] + + '_loss'] = output_loss + + # Keep track of stateful result tensor and function for the loss. + loss_name = loss_fn.name if isinstance( + loss_fn, losses.Loss) else loss_fn.__name__ + mean_wrapped_loss = metrics_module.MeanMetricWrapper( + loss_fn, name=loss_name) + result_tensor = training_utils.call_metric_function( + mean_wrapped_loss, + y_true, + y_pred, + weights=sample_weight, + mask=mask) + self._compile_stateful_metrics_tensors[self.output_names[i] + + '_loss'] = result_tensor + self._compile_stateful_metric_functions.append(mean_wrapped_loss) + + self._compile_metrics_names.append(self.output_names[i] + '_loss') + if total_loss is None: + total_loss = loss_weight * output_loss + else: + total_loss += loss_weight * output_loss if total_loss is None: - total_loss = loss_weight * output_loss - else: - total_loss += loss_weight * output_loss - if total_loss is None: - if not self.losses: - raise ValueError('The model cannot be compiled ' - 'because it has no loss to optimize.') - else: - total_loss = 0. + if not self.losses: + raise ValueError('The model cannot be compiled ' + 'because it has no loss to optimize.') + else: + total_loss = 0. - # Add regularization penalties - # and other layer-specific losses. - for loss_tensor in self.losses: - total_loss += loss_tensor + # Add regularization penalties + # and other layer-specific losses. + for loss_tensor in self.losses: + total_loss += loss_tensor - # Set metric attributes on model. - self._set_metric_attributes( - self.outputs, - skip_target_indices=skip_target_indices, - ) - # Invoke metric functions for all the outputs. - self._handle_metrics( - self.outputs, - masks=masks, - targets=self.targets, - skip_target_indices=skip_target_indices, - sample_weights=self.sample_weights) + # Set metric attributes on model. + self._set_metric_attributes( + self.outputs, + skip_target_indices=skip_target_indices, + ) + # Invoke metric functions for all the outputs. + self._handle_metrics( + self.outputs, + masks=masks, + targets=self.targets, + skip_target_indices=skip_target_indices, + sample_weights=self.sample_weights) - # Prepare gradient updates and state updates. - self.total_loss = total_loss + # Prepare gradient updates and state updates. + self.total_loss = total_loss - # Functions for train, test and predict will - # be compiled lazily when required. - # This saves time when the user is not using all functions. - self._function_kwargs = kwargs + # Functions for train, test and predict will + # be compiled lazily when required. + # This saves time when the user is not using all functions. + self._function_kwargs = kwargs - self._fit_function = None - self._eval_function = None - self.train_function = None - self.test_function = None - self.predict_function = None + self._fit_function = None + self._eval_function = None + self.train_function = None + self.test_function = None + self.predict_function = None - # Collected trainable weights, sorted in topological order. - trainable_weights = self.trainable_weights - self._collected_trainable_weights = trainable_weights + # Collected trainable weights, sorted in topological order. + trainable_weights = self.trainable_weights + self._collected_trainable_weights = trainable_weights def _check_trainable_weights_consistency(self): """Check trainable weights count consistency. @@ -721,21 +840,24 @@ class Model(Network): inputs = (self._feed_inputs + self._feed_targets + self._feed_sample_weights) - if not isinstance(K.learning_phase(), int): - inputs += [K.learning_phase()] + if not isinstance(K.symbolic_learning_phase(), int): + inputs += [K.symbolic_learning_phase()] + + with K.get_graph().as_default(): + with K.name_scope('training'): + with K.name_scope(self.optimizer.__class__.__name__): + # Training updates + updates = self.optimizer.get_updates( + params=self._collected_trainable_weights, loss=self.total_loss) + # Unconditional updates + updates += self.get_updates_for(None) + # Conditional updates relevant to this model + updates += self.get_updates_for(self.inputs) + # Add stateful metrics updates. + if metric_updates is not None: + updates += metric_updates with K.name_scope('training'): - with K.name_scope(self.optimizer.__class__.__name__): - # Training updates - updates = self.optimizer.get_updates( - params=self._collected_trainable_weights, loss=self.total_loss) - # Unconditional updates - updates += self.get_updates_for(None) - # Conditional updates relevant to this model - updates += self.get_updates_for(self.inputs) - # Add stateful metrics updates. - if metric_updates is not None: - updates += metric_updates # Gets loss and metrics. Updates weights at each call. fn = K.function( inputs, @@ -746,18 +868,24 @@ class Model(Network): setattr(self, fn_name, fn) def _make_train_function(self): + metrics_tensors = [ + self._all_metrics_tensors[m] for m in self.metrics_names[1:] + ] self._make_train_function_helper('train_function', - [self.total_loss] + self.metrics_tensors) + [self.total_loss] + metrics_tensors) def _make_fit_function(self): # TODO(psv/anjalisridhar): Remove updates after we fix b/118841692 # Stateful metrics updates metric_updates = [] - for m in self.stateful_metric_functions: + for m in self.metrics: metric_updates += m.updates + + metrics_tensors = [ + self._all_stateful_metrics_tensors[m] for m in self.metrics_names[1:] + ] self._make_train_function_helper( - '_fit_function', [self.total_loss] + self._stateful_metrics_tensors, - metric_updates) + '_fit_function', [self.total_loss] + metrics_tensors, metric_updates) def _make_test_function_helper(self, fn_name, outputs, metric_updates=None): if not hasattr(self, fn_name): @@ -766,49 +894,53 @@ class Model(Network): inputs = (self._feed_inputs + self._feed_targets + self._feed_sample_weights) - if not isinstance(K.learning_phase(), int): - inputs += [K.learning_phase()] - updates = self.state_updates - # Add stateful metrics updates. - if metric_updates is not None: - updates += metric_updates - # Return loss and metrics, no gradient updates. - # Does update the network states. - fn = K.function( - inputs, - outputs, - updates=updates, - name='test_function', - **self._function_kwargs) - setattr(self, fn_name, fn) + + with K.name_scope('evaluation'): + updates = self.state_updates + # Add stateful metrics updates. + if metric_updates is not None: + updates += metric_updates + # Return loss and metrics, no gradient updates. + # Does update the network states. + fn = K.function( + inputs, + outputs, + updates=updates, + name='test_function', + **self._function_kwargs) + setattr(self, fn_name, fn) def _make_test_function(self): + metrics_tensors = [ + self._all_metrics_tensors[m] for m in self.metrics_names[1:] + ] self._make_test_function_helper('test_function', - [self.total_loss] + self.metrics_tensors) + [self.total_loss] + metrics_tensors) def _make_eval_function(self): - self._make_test_function_helper( - '_eval_function', [self.total_loss] + self._stateful_metrics_tensors) + metrics_tensors = [ + self._all_stateful_metrics_tensors[m] for m in self.metrics_names[1:] + ] + self._make_test_function_helper('_eval_function', + [self.total_loss] + metrics_tensors) def _make_predict_function(self): if not hasattr(self, 'predict_function'): self.predict_function = None if self.predict_function is None: - if not isinstance(K.learning_phase(), int): - inputs = self._feed_inputs + [K.learning_phase()] - else: - inputs = self._feed_inputs + inputs = self._feed_inputs # Gets network outputs. Does not update weights. # Does update the network states. kwargs = getattr(self, '_function_kwargs', {}) - self.predict_function = K.function( - inputs, - self.outputs, - updates=self.state_updates, - name='predict_function', - **kwargs) + with K.name_scope('predict'): + self.predict_function = K.function( + inputs, + self.outputs, + updates=self.state_updates, + name='predict_function', + **kwargs) - def _get_execution_function(self, mode): + def _make_execution_function(self, mode): if mode == 'train': self._make_fit_function() return self._fit_function @@ -873,7 +1005,8 @@ class Model(Network): 'when using DistributionStrategy.') if (sample_weight is not None and sample_weight.all() and - self._distribution_strategy.__class__.__name__ == 'TPUStrategy'): + distributed_training_utils.is_tpu_strategy( + self._distribution_strategy)): raise NotImplementedError('`sample_weight` is currently not supported ' 'when using TPUStrategy.') @@ -882,18 +1015,13 @@ class Model(Network): # TODO(anjalisridhar): Remove this check once we refactor the # _standardize_user_data code path. This check is already present elsewhere # in the codebase. - if check_steps and isinstance(x, dataset_ops.Dataset) and steps is None: + if check_steps and isinstance(x, dataset_ops.DatasetV2) and steps is None: raise ValueError('When using Datasets as input, ' 'you should specify the `{steps_name}` argument.' .format(steps_name=steps_name)) first_x_value = nest.flatten(x)[0] if isinstance(first_x_value, np.ndarray): - assert steps is not None - x_shape = first_x_value.shape - if batch_size is None: - batch_size = distributed_training_utils.get_batch_size( - self._distribution_strategy.num_replicas_in_sync, x_shape[0], steps) # We need to use the drop_remainder argument to allow for a static # input shape which is required for TPUs. drop_remainder = self._distribution_strategy.require_static_shapes @@ -928,19 +1056,15 @@ class Model(Network): var_x = distributed_training_utils.get_var_for_numpy( self._distribution_strategy, x) x = dataset_ops.Dataset.from_tensor_slices(var_x) - x = x.repeat() x = x.batch(batch_size, drop_remainder=drop_remainder) - assert isinstance(x, dataset_ops.Dataset) + assert isinstance(x, dataset_ops.DatasetV2) - # TODO(anjalisridhar): We want distribute_dataset() to accept a Dataset or a - # function which returns a Dataset. Currently distribute_dataset() only - # accepts a function that returns a Dataset. Once we add support for being - # able to clone a Dataset on multiple workers we can remove this lambda. - result = self._distribution_strategy.distribute_dataset(lambda: x) - iterator = result.make_initializable_iterator() with self._distribution_strategy.scope(): - K.get_session().run(iterator.initializer) + iterator = self._distribution_strategy.make_dataset_iterator(x) + init_op = iterator.initialize() + if not context.executing_eagerly(): + K.get_session().run(init_op) training_utils.validate_iterator_input(x, y, sample_weight, validation_split) @@ -1025,14 +1149,14 @@ class Model(Network): shuffle=shuffle) return iterator, None, None - if isinstance(x, dataset_ops.Dataset): + if isinstance(x, dataset_ops.DatasetV2): if context.executing_eagerly(): - x = x.make_one_shot_iterator() + x = iter(x) else: if x in self._dataset_iterator_cache: x = self._dataset_iterator_cache[x] else: - iterator = x.make_initializable_iterator() + iterator = dataset_ops.make_initializable_iterator(x) self._dataset_iterator_cache[x] = iterator x = iterator K.get_session().run(x.initializer) @@ -1052,7 +1176,7 @@ class Model(Network): # For eager iterators, when we have to process multiple batches of samples, # we will standardize the data when we actually loop over iterator and get # the batches. For now, we just return the iterator as is. - if is_x_eager_iterator and steps is not None: + if is_x_eager_iterator: return x, y, sample_weight # If input data is a dataset iterator in graph mode or if it is an eager @@ -1096,6 +1220,8 @@ class Model(Network): all_inputs = [] is_build_called = False is_compile_called = False + # Whether this is a subclassed model that expects dictionary inputs + # rather than list inputs (e.g. FeatureColumn-based models). dict_inputs = False if not self.inputs: # We need to use `x` to set the model inputs. @@ -1122,9 +1248,16 @@ class Model(Network): # to match the value shapes. if not self.inputs: is_build_called = True - self._set_inputs(x) + cast_inputs = x + if training_utils.has_tensors(x): + cast_inputs = training_utils.cast_if_floating_dtype(x) + self._set_inputs(cast_inputs) else: dict_inputs = isinstance(self.inputs, dict) + if dict_inputs and context.executing_eagerly(): + # No support for graph functions when the model expects dictionary inputs + # (i.e. FeatureColumn-based models). + self.run_eagerly = True if y is not None: if not self.optimizer: @@ -1134,6 +1267,8 @@ class Model(Network): if not self._is_compiled: # On-the-fly compilation of the model. # We need to use `y` to set the model targets. + if training_utils.has_tensors(y): + y = training_utils.cast_if_floating_dtype(y) if isinstance(y, (list, tuple)): if not all(isinstance(v, np.ndarray) or tensor_util.is_tensor(v) for v in y): @@ -1158,19 +1293,22 @@ class Model(Network): 'TensorFlow tensors. ' 'You passed: x=' + str(x) + '; y=' + str(y)) - if context.executing_eagerly(): + if self.run_eagerly: target_tensors = None else: # Handle target tensors if any passed. if not isinstance(y, (list, tuple)): y = [y] - target_tensors = [v for v in y if tensor_util.is_tensor(v)] + target_tensors = [v for v in y if _is_symbolic_tensor(v)] is_compile_called = True - self.compile(optimizer=self.optimizer, - loss=self.loss, - metrics=self.metrics, - loss_weights=self.loss_weights, - target_tensors=target_tensors) + self.compile( + optimizer=self.optimizer, + loss=self.loss, + metrics=self._compile_metrics, + weighted_metrics=self._compile_weighted_metrics, + loss_weights=self.loss_weights, + target_tensors=target_tensors, + run_eagerly=self.run_eagerly) # In graph mode, if we had just set inputs and targets as symbolic tensors # by invoking build and compile on the model respectively, we do not have to @@ -1178,15 +1316,15 @@ class Model(Network): # part of the graph. # Note: in this case, `any` and `all` are equivalent since we disallow # mixed symbolic/value inputs. - if (not context.executing_eagerly() and is_build_called and + if (not self.run_eagerly and is_build_called and is_compile_called and - any(tensor_util.is_tensor(v) for v in all_inputs)): + any(_is_symbolic_tensor(v) for v in all_inputs)): return [], [], [] # What follows is input validation and standardization to list format, # in the case where all inputs are value arrays. - if context.executing_eagerly(): + if self.run_eagerly: # In eager mode, do not do shape validation # since the network has no input nodes (placeholders) to be fed. feed_input_names = self.input_names @@ -1242,7 +1380,9 @@ class Model(Network): y = training_utils.standardize_input_data( y, feed_output_names, - feed_output_shapes, + # Don't enforce target shapes to match output shapes. + # Precise checks will be run in `check_loss_and_target_compatibility`. + shapes=None, check_batch_axis=False, # Don't enforce the batch size. exception_prefix='target') @@ -1260,7 +1400,7 @@ class Model(Network): # Check that all arrays have the same length. if not self._distribution_strategy: training_utils.check_array_lengths(x, y, sample_weights) - if self._is_graph_network and not context.executing_eagerly(): + if self._is_graph_network and not self.run_eagerly: # Additional checks to avoid users mistakenly using improper loss fns. training_utils.check_loss_and_target_compatibility( y, self._feed_loss_fns, feed_output_shapes) @@ -1315,8 +1455,7 @@ class Model(Network): if self.__class__.__name__ == 'Sequential' and not self.built: if tensor_util.is_tensor(inputs): - input_shape = (None,) + tuple(inputs.get_shape().as_list()[1:]) - self.build(input_shape=input_shape) + input_shape = (None,) + tuple(inputs.shape.as_list()[1:]) elif isinstance(inputs, dict): # We assert that the first layer is a FeatureLayer. if not training_utils.is_feature_layer(self.layers[0]): @@ -1324,10 +1463,9 @@ class Model(Network): 'which doesn\'t have FeatureLayer as the first layer' ' is an error.') input_shape = (None,) - self.build(input_shape=input_shape) else: - input_shape = (None,) + inputs.shape[1:] - self.build(input_shape=input_shape) + input_shape = (None,) + tuple(inputs.shape[1:]) + self._build_input_shape = input_shape # On-the-fly setting of symbolic model inputs (either by using the tensor # provided, or by creating a placeholder if Numpy data was provided). @@ -1346,10 +1484,11 @@ class Model(Network): self._feed_input_names.append(k) self._feed_input_shapes.append(K.int_shape(v)) + # TODO(fchollet): consider calling `_maybe_build` before calling the model. + if outputs is None: # Obtain symbolic outputs by calling the model. - graph = K.get_graph() - with graph.as_default(): + with K.get_graph().as_default(): if self._expects_training_arg: outputs = self.call(inputs, training=training) else: @@ -1509,7 +1648,6 @@ class Model(Network): """ # TODO(fchollet): this method may be creating reference cycles, which would # lead to accumulating garbage in memory when called in a loop. Investigate. - if data_utils.is_generator_or_sequence(x): training_utils.check_generator_arguments(y, sample_weight) return self.fit_generator( @@ -1527,9 +1665,6 @@ class Model(Network): shuffle=shuffle, initial_epoch=initial_epoch) - # Backwards compatibility - if batch_size is None and steps_per_epoch is None: - batch_size = 32 # Legacy support if 'nb_epoch' in kwargs: logging.warning( @@ -1541,15 +1676,21 @@ class Model(Network): # Validate and standardize user data. if self._distribution_strategy: - distributed_training_utils.validate_callbacks(callbacks) + distributed_training_utils.validate_callbacks(callbacks, self.optimizer, + self._distribution_strategy) distributed_training_utils.validate_inputs( x, y, self._distribution_strategy) first_x_value = nest.flatten(x)[0] - if not steps_per_epoch and isinstance(first_x_value, np.ndarray): - steps_per_epoch = distributed_training_utils.get_input_batch_params( - first_x_value, batch_size, self._distribution_strategy) + if isinstance(first_x_value, np.ndarray): + steps_per_epoch, batch_size = ( + distributed_training_utils.get_input_params( + self._distribution_strategy, first_x_value, steps_per_epoch, + batch_size, is_training=True)) + + batch_size = self._validate_or_infer_batch_size(batch_size, steps_per_epoch, + x) x, y, sample_weights = self._standardize_user_data( x, @@ -1567,7 +1708,7 @@ class Model(Network): if validation_data: if (isinstance(validation_data, iterator_ops.Iterator) or isinstance(validation_data, iterator_ops.EagerIterator) or - isinstance(validation_data, dataset_ops.Dataset)): + isinstance(validation_data, dataset_ops.DatasetV2)): val_x = validation_data val_y = None val_sample_weight = None @@ -1590,9 +1731,10 @@ class Model(Network): distributed_training_utils.validate_inputs( val_x, val_y, self._distribution_strategy) first_valx_value = nest.flatten(val_x)[0] - if not validation_steps and isinstance(first_valx_value, np.ndarray): - validation_steps = distributed_training_utils.get_input_batch_params( - first_valx_value, batch_size, self._distribution_strategy) + if isinstance(first_valx_value, np.ndarray): + validation_steps, _ = distributed_training_utils.get_input_params( + self._distribution_strategy, first_valx_value, validation_steps, + batch_size) val_x, val_y, val_sample_weights = self._standardize_user_data( val_x, @@ -1622,27 +1764,25 @@ class Model(Network): val_y = None val_sample_weights = None - if context.executing_eagerly(): - return training_eager.fit_loop( - self, - inputs=x, - targets=y, - sample_weights=sample_weights, - class_weight=class_weight, + if (self.run_eagerly or (isinstance(x, iterator_ops.EagerIterator) and + not self._distribution_strategy)): + return training_generator.fit_generator( + self, (x, y, sample_weights), + steps_per_epoch=steps_per_epoch, batch_size=batch_size, epochs=epochs, + shuffle=shuffle, verbose=verbose, callbacks=callbacks, - val_inputs=val_x, - val_targets=val_y, - val_sample_weights=val_sample_weights, - shuffle=shuffle, - initial_epoch=initial_epoch, - steps_per_epoch=steps_per_epoch, - validation_steps=validation_steps) - elif self._distribution_strategy: - return training_distributed.fit_loop( - self, x, + validation_data=validation_data, + validation_steps=validation_steps, + workers=0, + initial_epoch=initial_epoch) + elif distributed_training_utils.is_tpu_strategy( + self._distribution_strategy): + return training_distributed.experimental_fit_loop( + self, + x, epochs=epochs, verbose=verbose, callbacks=callbacks, @@ -1757,19 +1897,16 @@ class Model(Network): max_queue_size=max_queue_size, workers=workers, use_multiprocessing=use_multiprocessing) - - # Backwards compatibility. - if batch_size is None and steps is None: - batch_size = 32 - # Validate and standardize user data. if self._distribution_strategy: distributed_training_utils.validate_inputs( x, y, self._distribution_strategy) first_x_value = nest.flatten(x)[0] - if isinstance(first_x_value, np.ndarray) and not steps: - steps = distributed_training_utils.get_input_batch_params( - first_x_value, batch_size, self._distribution_strategy) + if isinstance(first_x_value, np.ndarray): + steps, batch_size = distributed_training_utils.get_input_params( + self._distribution_strategy, first_x_value, steps, batch_size) + + batch_size = self._validate_or_infer_batch_size(batch_size, steps, x) x, y, sample_weights = self._standardize_user_data( x, @@ -1780,21 +1917,18 @@ class Model(Network): steps_name='steps', steps=steps) - if context.executing_eagerly(): - return training_eager.test_loop( - self, - inputs=x, - targets=y, - sample_weights=sample_weights, + if (self.run_eagerly or (isinstance(x, iterator_ops.EagerIterator) and + not self._distribution_strategy)): + return training_generator.evaluate_generator( + self, (x, y, sample_weights), + steps=steps, batch_size=batch_size, verbose=verbose, - steps=steps) - elif self._distribution_strategy: - return training_distributed.test_loop( - self, - iterator=x, - verbose=verbose, - steps=steps) + workers=0) + elif distributed_training_utils.is_tpu_strategy( + self._distribution_strategy): + return training_distributed.experimental_test_loop( + self, iterator=x, verbose=verbose, steps=steps) else: return training_arrays.test_loop( self, @@ -1868,37 +2002,57 @@ class Model(Network): max_queue_size=max_queue_size, workers=workers, use_multiprocessing=use_multiprocessing) - - # Backwards compatibility. - if batch_size is None and steps is None: - batch_size = 32 - if self._distribution_strategy: distributed_training_utils.validate_inputs( x, None, self._distribution_strategy) first_x_value = nest.flatten(x)[0] - if isinstance(first_x_value, np.ndarray) and not steps: - steps = distributed_training_utils.get_input_batch_params( - first_x_value, batch_size, self._distribution_strategy) + if isinstance(first_x_value, np.ndarray): + steps, batch_size = distributed_training_utils.get_input_params( + self._distribution_strategy, first_x_value, steps, batch_size) + + batch_size = self._validate_or_infer_batch_size(batch_size, steps, x) # Validate and standardize user data. - # TODO(anjalisridhar): We don't pass batch_size here for some reason. This - # means that we end up calculating it twice which we should avoid. - x, _, _ = self._standardize_user_data( - x, check_steps=True, steps_name='steps', steps=steps) + if self._distribution_strategy: + x, _, _ = self._standardize_user_data( + x, check_steps=True, steps_name='steps', steps=steps, + batch_size=batch_size) + else: + # TODO(anjalisridhar): We don't pass batch_size here for some reason. This + # means we need to special case distribution strategy which needs the + # batch size. + x, _, _ = self._standardize_user_data( + x, check_steps=True, steps_name='steps', steps=steps) - if context.executing_eagerly(): - return training_eager.predict_loop( - self, x, batch_size=batch_size, verbose=verbose, steps=steps) - elif self._distribution_strategy: - results = training_distributed.predict_loop( + if (self.run_eagerly or (isinstance(x, iterator_ops.EagerIterator) and + not self._distribution_strategy)): + return training_generator.predict_generator( + self, + x, + steps=steps, + batch_size=batch_size, + verbose=verbose, + workers=0) + elif distributed_training_utils.is_tpu_strategy( + self._distribution_strategy): + return training_distributed.experimental_predict_loop( self, x, verbose=verbose, steps=steps) - return results else: return training_arrays.predict_loop( self, x, batch_size=batch_size, verbose=verbose, steps=steps) - def train_on_batch(self, x, y=None, sample_weight=None, class_weight=None): + def reset_metrics(self): + """Resets the state of metrics.""" + if hasattr(self, 'metrics'): + for m in self.metrics: + m.reset_states() + + def train_on_batch(self, + x, + y=None, + sample_weight=None, + class_weight=None, + reset_metrics=True): """Runs a single gradient update on a single batch of data. Arguments: @@ -1926,6 +2080,9 @@ class Model(Network): weight (float) to apply to the model's loss for the samples from this class during training. This can be useful to tell the model to "pay more attention" to samples from an under-represented class. + reset_metrics: If `True`, the metrics returned will be only for this + batch. If `False`, the metrics will be statefully accumulated across + batches. Returns: Scalar training loss @@ -1944,23 +2101,30 @@ class Model(Network): x, y, sample_weights = self._standardize_user_data( x, y, sample_weight=sample_weight, class_weight=class_weight) - if context.executing_eagerly(): + if self.run_eagerly: outputs = training_eager.train_on_batch( self, x, y, sample_weights=sample_weights) else: - if not isinstance(K.learning_phase(), int): - ins = x + y + sample_weights + [1] + if not isinstance(K.symbolic_learning_phase(), int): + ins = x + y + sample_weights + [True] else: ins = x + y + sample_weights - self._make_train_function() - outputs = self.train_function(ins) # pylint: disable=not-callable + if reset_metrics: + self._make_train_function() + outputs = self.train_function(ins) # pylint: disable=not-callable + else: + self._make_fit_function() + outputs = self._fit_function(ins) # pylint: disable=not-callable + + if reset_metrics: + self.reset_metrics() if len(outputs) == 1: return outputs[0] return outputs - def test_on_batch(self, x, y=None, sample_weight=None): + def test_on_batch(self, x, y=None, sample_weight=None, reset_metrics=True): """Test the model on a single batch of samples. Arguments: @@ -1986,6 +2150,9 @@ class Model(Network): In this case you should make sure to specify sample_weight_mode="temporal" in compile(). This argument is not supported when `x` is a dataset or a dataset iterator. + reset_metrics: If `True`, the metrics returned will be only for this + batch. If `False`, the metrics will be statefully accumulated across + batches. Returns: Scalar test loss (if the model has a single output and no metrics) @@ -2003,16 +2170,20 @@ class Model(Network): x, y, sample_weights = self._standardize_user_data( x, y, sample_weight=sample_weight) - if context.executing_eagerly(): + if self.run_eagerly: outputs = training_eager.test_on_batch( self, x, y, sample_weights=sample_weights) else: - if not isinstance(K.learning_phase(), int): - ins = x + y + sample_weights + [0] + inputs = x + y + sample_weights + if reset_metrics: + self._make_test_function() + outputs = self.test_function(inputs) # pylint: disable=not-callable else: - ins = x + y + sample_weights - self._make_test_function() - outputs = self.test_function(ins) # pylint: disable=not-callable + self._make_eval_function() + outputs = self._eval_function(inputs) # pylint: disable=not-callable + + if reset_metrics: + self.reset_metrics() if len(outputs) == 1: return outputs[0] @@ -2041,28 +2212,21 @@ class Model(Network): 'models compiled with DistributionStrategy.') # Validate and standardize user data. inputs, _, _ = self._standardize_user_data(x) - if context.executing_eagerly(): - if (isinstance(x, iterator_ops.EagerIterator) or - (isinstance(x, dataset_ops.Dataset) and context.executing_eagerly())): + if self.run_eagerly: + if (isinstance(inputs, iterator_ops.EagerIterator) or + (isinstance(inputs, dataset_ops.DatasetV2))): inputs = training_utils.cast_if_floating_dtype(inputs) - else: + elif isinstance(inputs, collections.Sequence): inputs = [ - ops.convert_to_tensor(val, dtype=K.floatx()) for val in inputs - ] + ops.convert_to_tensor(val, dtype=K.floatx()) for val in inputs] return self(inputs) # pylint: disable=not-callable - if not context.executing_eagerly(): - if not isinstance(K.learning_phase(), int): - ins = inputs + [0] - else: - ins = inputs + self._make_predict_function() + outputs = self.predict_function(inputs) - self._make_predict_function() - outputs = self.predict_function(ins) - - if len(outputs) == 1: - return outputs[0] - return outputs + if len(outputs) == 1: + return outputs[0] + return outputs def fit_generator(self, generator, @@ -2172,11 +2336,6 @@ class Model(Network): if self._distribution_strategy: raise NotImplementedError('`fit_generator` is not supported for ' 'models compiled with DistributionStrategy.') - - if not self.built and not self._is_graph_network: - raise NotImplementedError( - '`fit_generator` is not yet enabled for unbuilt Model subclasses') - return training_generator.fit_generator( self, generator, @@ -2243,12 +2402,6 @@ class Model(Network): if self._distribution_strategy: raise NotImplementedError('`evaluate_generator` is not supported for ' 'models compiled with DistributionStrategy.') - - if not self.built and not self._is_graph_network: - raise NotImplementedError( - '`evaluate_generator` is not yet enabled for ' - 'unbuilt Model subclasses') - return training_generator.evaluate_generator( self, generator, @@ -2300,11 +2453,6 @@ class Model(Network): if self._distribution_strategy: raise NotImplementedError('`predict_generator` is not supported for ' 'models compiled with DistributionStrategy.') - - if not self.built and not self._is_graph_network: - raise NotImplementedError( - '`predict_generator` is not yet enabled for unbuilt Model subclasses') - return training_generator.predict_generator( self, generator, @@ -2336,15 +2484,63 @@ class Model(Network): self._replicated_model = DistributedCallbackModel(first_replicated_model) self._replicated_model.set_original_model(self) + def _validate_or_infer_batch_size(self, batch_size, steps, x): + """Validates that the `batch_size` provided is consistent with InputLayer. + + It's possible that the user specified a static batch size in their + InputLayer. If so, this method checks the provided `batch_size` and `x` + arguments are consistent with this static batch size. Also, if + `batch_size` is `None`, this method will attempt to infer the batch size + from the static batch size of the InputLayer. + + Arguments: + batch_size: The batch_size provided as an argument to + fit/evaluate/predict. + steps: The steps provided as an argument to fit/evaluate/predict. + x: The data passed as `x` to fit/evaluate/predict. + + Returns: + The validated batch_size, auto-inferred from the first layer if not + provided. + """ + first_layer = super(Model, + self).layers[0] # Avoids the override in Sequential. + static_batch_size = training_utils.get_static_batch_size(first_layer) + if static_batch_size is not None: + + # Check `batch_size` argument is consistent with InputLayer. + if batch_size is not None and batch_size != static_batch_size: + raise ValueError('The `batch_size` argument value ' + str(batch_size) + + ' is incompatible with the specified batch size ' + 'of your Input Layer: ' + str(static_batch_size)) + + # Check Dataset/Iterator batch size is consistent with InputLayer. + if isinstance(x, (dataset_ops.DatasetV2, iterator_ops.Iterator, + iterator_ops.EagerIterator)): + ds_batch_size = tensor_shape.as_dimension( + nest.flatten(x.output_shapes)[0][0]).value + if ds_batch_size is not None and ds_batch_size != static_batch_size: + raise ValueError('The batch output shape of your `Dataset` is ' + + str(ds_batch_size) + ' which is incompatible ' + 'with the specified batch size of your Input ' + 'Layer: ' + str(static_batch_size)) + + # Set inferred batch size from the InputLayer. + if steps is None: + batch_size = static_batch_size + + if batch_size is None and steps is None: + # Backwards compatibility + batch_size = 32 + return batch_size + class DistributedCallbackModel(Model): """Model that is used for callbacks with DistributionStrategy.""" def __init__(self, model): super(DistributedCallbackModel, self).__init__() - # TODO(anjalisridhar): Right now the only attributes set are the layer and - # weights. We may need to set additional attributes as needed since we have - # not called compile on this model. + self.optimizer = model.optimizer def set_original_model(self, orig_model): self._original_model = orig_model @@ -2376,3 +2572,7 @@ class DistributedCallbackModel(Model): logging.warning('You are accessing attribute ' + item + ' of the ' 'DistributedCallbackModel that may not have been set ' 'correctly.') + + +def _is_symbolic_tensor(x): + return tensor_util.is_tensor(x) and not isinstance(x, ops.EagerTensor) diff --git a/tensorflow/python/keras/engine/training_arrays.py b/tensorflow/python/keras/engine/training_arrays.py index a2a13b9bd60..e9dfbcbcc07 100644 --- a/tensorflow/python/keras/engine/training_arrays.py +++ b/tensorflow/python/keras/engine/training_arrays.py @@ -23,9 +23,11 @@ import functools import numpy as np +from tensorflow.python.eager import context from tensorflow.python.framework import errors from tensorflow.python.keras import backend as K from tensorflow.python.keras import callbacks as cbks +from tensorflow.python.keras.engine import training_distributed from tensorflow.python.keras.engine import training_utils from tensorflow.python.keras.utils.generic_utils import make_batches from tensorflow.python.keras.utils.generic_utils import slice_arrays @@ -37,91 +39,6 @@ except ImportError: issparse = None -class Aggregator(object): - """Abstract base class used to aggregate batch-level outputs of a loop. - - Arguments: - use_steps: Whether the loop is using `step` or `batch_size`. - num_samples_or_steps: Either `batch_size*num_batches` or `steps`. - """ - - def __init__(self, use_steps, num_samples_or_steps): - self.use_steps = use_steps - self.num_samples_or_steps = num_samples_or_steps - self.results = [] - - def create(self, batch_outs): - """Create the initial results from the first batch outputs. - - Arguments: - batch_outs: A list of batch-level outputs. - """ - raise NotImplementedError - - def aggregate(self, batch_outs, batch_start=None, batch_end=None): - """Aggregate batch-level results into total results. - - Arguments: - batch_outs: A list of batch-level outputs. - batch_start: The start index of this batch. Always `None` if `use_steps` - is `True`. - batch_end: The end index of this batch. Always `None` if `use_steps` is - `True`. - """ - raise NotImplementedError - - def finalize(self): - """Prepare the total results to be returned.""" - raise NotImplementedError - - -class MetricsAggregator(Aggregator): - """Aggregator that calculates loss and metrics info.""" - - def create(self, batch_outs): - self.results = [0.] * len(batch_outs) - - def aggregate(self, batch_outs, batch_start=None, batch_end=None): - # Loss. - if self.use_steps: - self.results[0] += batch_outs[0] - else: - self.results[0] += batch_outs[0] * (batch_end - batch_start) - # Metrics (always stateful, just grab current values.) - self.results[1:] = batch_outs[1:] - - def finalize(self): - self.results[0] /= self.num_samples_or_steps - - -class OutputsAggregator(Aggregator): - """Aggregator that concatenates outputs.""" - - def create(self, batch_outs): - if self.use_steps: - # Cannot pre-allocate the returned NumPy arrays bc - # batch sizes are unknown. Concatenate batches at the end. - for _ in batch_outs: - self.results.append([]) - else: - # Pre-allocate NumPy arrays. - for batch_out in batch_outs: - shape = (self.num_samples_or_steps,) + batch_out.shape[1:] - self.results.append(np.zeros(shape, dtype=batch_out.dtype)) - - def aggregate(self, batch_outs, batch_start=None, batch_end=None): - if self.use_steps: - for i, batch_out in enumerate(batch_outs): - self.results[i].append(batch_out) - else: - for i, batch_out in enumerate(batch_outs): - self.results[i][batch_start:batch_end] = batch_out - - def finalize(self): - if self.use_steps: - self.results = [np.concatenate(result, axis=0) for result in self.results] - - def _get_model_feed(model, mode): if mode == 'predict': feed = model._feed_inputs @@ -151,13 +68,6 @@ def _print_train_info(inputs, val_inputs, steps_per_epoch, verbose): (inputs[0].shape[0], val_inputs[0].shape[0])) -def _get_progbar(model, count_mode): - stateful_metric_names = None - if hasattr(model, 'metrics_names'): - stateful_metric_names = model.metrics_names[1:] # Exclude `loss` - return cbks.ProgbarLogger(count_mode, stateful_metrics=stateful_metric_names) - - def _get_num_samples_or_steps(ins, batch_size, steps_per_epoch): """Returns total number of samples (when training in batch mode) or steps.""" if steps_per_epoch: @@ -166,16 +76,50 @@ def _get_num_samples_or_steps(ins, batch_size, steps_per_epoch): 'steps_per_epoch') -def _make_logs(model, outputs, mode, prefix=''): - """Used to make logs to send to `on_batch_end` methods.""" - logs = {} - # TODO(omalleyt): handle outputs in prediction when Callback - # hooks are ready. - if mode in ['train', 'test']: - if hasattr(model, 'metrics_names'): - for label, output in zip(model.metrics_names, outputs): - logs[prefix + label] = output - return logs +def _prepare_feed_values(model, inputs, targets, sample_weights, mode): + """Prepare feed values to the model execution function. + + Arguments: + model: Model to prepare feed values for. + inputs: List or dict of model inputs. + targets: Optional list of model targets. + sample_weights: Optional list of sample weight arrays. + mode: One of 'train'/'test'/'predict'. + + Returns: + Feed values for the model in the given mode. + """ + if model._distribution_strategy: + def get_distributed_inputs(): + return training_distributed._prepare_feed_values( + model, inputs, targets, sample_weights, mode) + + # In the eager case, we want to call the input method per step, so return + # a lambda from here that can be called. Note that this is applicable only + # in Distribution Strategy case as it follows the same code path for both + # eager and graph modes. + # TODO(priyag,omalleyt): Either we should move the training DS with + # EagerIterator to use training_generator code path, or figure out how to + # set a symbolic Iterator out of a Dataset when in eager mode. + if context.executing_eagerly(): + return get_distributed_inputs + else: + return get_distributed_inputs() + + inputs = training_utils.ModelInputs(inputs).as_list() + targets = targets or [] + sample_weights = sample_weights or [] + ins = inputs + targets + sample_weights + if mode == 'train' and not isinstance(K.symbolic_learning_phase(), int): + ins += [True] + return ins + + +def _make_execution_function(model, mode): + """Makes function to run one step of model execution.""" + if model._distribution_strategy: + return training_distributed._make_execution_function(model, mode) + return model._make_execution_function(mode) def model_iteration(model, @@ -238,19 +182,18 @@ def model_iteration(model, if mode == 'train': _print_train_info(inputs, val_inputs, steps_per_epoch, verbose) + # Enter DistributionStrategy scope. + if model._distribution_strategy: + scope = model._distribution_strategy.scope() + scope.__enter__() + # Get step function and loop type. - f = model._get_execution_function(mode) + f = _make_execution_function(model, mode) use_steps = steps_per_epoch is not None do_validation = val_inputs is not None # Prepare input data. - inputs = training_utils.ModelInputs(inputs).as_list() - targets = targets or [] - sample_weights = sample_weights or [] - learning_phase_input = [] - if not isinstance(K.learning_phase(), int): - learning_phase_input = [1] if mode == 'train' else [0] - ins = inputs + targets + sample_weights + learning_phase_input + ins = _prepare_feed_values(model, inputs, targets, sample_weights, mode) num_samples_or_steps = _get_num_samples_or_steps(ins, batch_size, steps_per_epoch) @@ -260,24 +203,19 @@ def model_iteration(model, callbacks, model, do_validation=do_validation, - val_inputs=val_inputs, - val_targets=val_targets, - val_sample_weights=val_sample_weights, batch_size=batch_size, epochs=epochs, steps_per_epoch=steps_per_epoch, samples=num_samples_or_steps, - validation_steps=validation_steps, verbose=0, # Handle ProgBarLogger separately in this loop. - count_mode=count_mode, mode=mode) # TODO(omalleyt): Handle ProgBar as part of Callbacks once hooks are ready. - progbar = _get_progbar(model, count_mode) + progbar = training_utils.get_progbar(model, count_mode) progbar.params = callbacks.params progbar.params['verbose'] = verbose # Find beforehand arrays that need sparse-to-dense conversion. - if issparse is not None: + if issparse is not None and not use_steps: indices_for_conversion_to_dense = [] feed = _get_model_feed(model, mode) for i, (input_data, feed_tensor) in enumerate(zip(ins, feed)): @@ -286,9 +224,14 @@ def model_iteration(model, # Select aggregation method. if mode == 'predict': - aggregator = OutputsAggregator(use_steps, num_samples_or_steps) + aggregator = training_utils.OutputsAggregator(use_steps, + num_samples_or_steps) else: - aggregator = MetricsAggregator(use_steps, num_samples_or_steps) + aggregator = training_utils.MetricsAggregator(use_steps, + num_samples_or_steps) + + if model._distribution_strategy: + training_distributed._copy_weights_to_distributed_model(model) callbacks.model.stop_training = False callbacks._call_begin_hook(mode) @@ -298,10 +241,9 @@ def model_iteration(model, break # Setup work for each epoch - results = [] epoch_logs = {} - if hasattr(model, 'stateful_metric_functions'): - for m in model.stateful_metric_functions: + if hasattr(model, 'metrics'): + for m in model.metrics: m.reset_states() callbacks.on_epoch_begin(epoch, epoch_logs, mode=mode) progbar.on_epoch_begin(epoch, epoch_logs) @@ -315,26 +257,31 @@ def model_iteration(model, # Get outputs. try: - batch_outs = f(ins) + # `ins` can be callable in DistributionStrategy + eager case. + actual_inputs = ins() if callable(ins) else ins + batch_outs = f(actual_inputs) except errors.OutOfRangeError: logging.warning('Your dataset iterator ran out of data; ' 'interrupting training. Make sure that your dataset ' 'can generate at least `steps_per_epoch * epochs` ' 'batches (in this case, %d batches). You may need to' 'use the repeat() function when building your ' - 'dataset.' % - steps_per_epoch * epochs) + 'dataset.' % steps_per_epoch * epochs) break if not isinstance(batch_outs, list): batch_outs = [batch_outs] + if model._distribution_strategy: + batch_outs = training_distributed._per_device_aggregate_batch( + batch_outs, model, mode) + # Aggregate results. if step == 0: aggregator.create(batch_outs) aggregator.aggregate(batch_outs) # Callbacks batch end. - batch_logs.update(_make_logs(model, batch_outs, mode)) + batch_logs.update(training_utils.make_logs(model, batch_outs, mode)) callbacks._call_batch_hook(mode, 'end', step, batch_logs) progbar.on_batch_end(step, batch_logs) @@ -365,8 +312,9 @@ def model_iteration(model, 'pass shuffle="batch".') # Sparse to dense conversion. - for i in indices_for_conversion_to_dense: - ins_batch[i] = ins_batch[i].toarray() + if issparse is not None: + for i in indices_for_conversion_to_dense: + ins_batch[i] = ins_batch[i].toarray() # Callbacks batch_begin. batch_logs = {'batch': batch_index, 'size': len(batch_ids)} @@ -384,7 +332,7 @@ def model_iteration(model, aggregator.aggregate(batch_outs, batch_start, batch_end) # Callbacks batch end. - batch_logs.update(_make_logs(model, batch_outs, mode)) + batch_logs.update(training_utils.make_logs(model, batch_outs, mode)) callbacks._call_batch_hook(mode, 'end', batch_index, batch_logs) progbar.on_batch_end(batch_index, batch_logs) @@ -393,7 +341,7 @@ def model_iteration(model, aggregator.finalize() results = aggregator.results - epoch_logs.update(_make_logs(model, results, mode)) + epoch_logs.update(training_utils.make_logs(model, results, mode)) if len(results) == 1: results = results[0] @@ -411,12 +359,17 @@ def model_iteration(model, mode='test') if not isinstance(val_results, list): val_results = [val_results] - epoch_logs.update(_make_logs(model, val_results, mode, prefix='val_')) + epoch_logs.update( + training_utils.make_logs(model, val_results, mode, prefix='val_')) callbacks.on_epoch_end(epoch, epoch_logs, mode=mode) progbar.on_epoch_end(epoch, epoch_logs) callbacks._call_end_hook(mode) + if model._distribution_strategy: + training_distributed._copy_weights_to_original_model(model, mode) + scope.__exit__(None, None, None) + if mode == 'train': return model.history return results diff --git a/tensorflow/python/keras/engine/training_dataset_test.py b/tensorflow/python/keras/engine/training_dataset_test.py new file mode 100644 index 00000000000..1b2d0e88e36 --- /dev/null +++ b/tensorflow/python/keras/engine/training_dataset_test.py @@ -0,0 +1,344 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Tests for training routines.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import logging + +from absl.testing import parameterized + +import numpy as np + +from tensorflow.python import keras +from tensorflow.python.data.ops import dataset_ops +from tensorflow.python.framework import ops +from tensorflow.python.framework import test_util as tf_test_util +from tensorflow.python.keras import metrics as metrics_module +from tensorflow.python.keras import testing_utils +from tensorflow.python.ops.losses import losses_impl +from tensorflow.python.platform import test +from tensorflow.python.platform import tf_logging as logging +from tensorflow.python.training.rmsprop import RMSPropOptimizer + + +class TestTrainingWithDatasetIterators(test.TestCase, parameterized.TestCase): + + @parameterized.parameters( + {'model': 'functional'}, + {'model': 'subclass'}, + ) + @tf_test_util.run_in_graph_and_eager_modes + def test_training_and_eval_methods_on_iterators_single_io(self, model): + if model == 'functional': + model = testing_utils.get_small_functional_mlp(1, 4, input_dim=3) + elif model == 'subclass': + model = testing_utils.get_small_sequential_mlp(1, 4) + optimizer = RMSPropOptimizer(learning_rate=0.001) + loss = 'mse' + metrics = ['mae', metrics_module.CategoricalAccuracy()] + model.compile(optimizer, loss, metrics=metrics) + + inputs = np.zeros((10, 3), np.float32) + targets = np.zeros((10, 4), np.float32) + dataset = dataset_ops.Dataset.from_tensor_slices((inputs, targets)) + dataset = dataset.repeat(100) + dataset = dataset.batch(10) + iterator = dataset_ops.make_one_shot_iterator(dataset) + + model.fit(iterator, epochs=1, steps_per_epoch=2, verbose=1) + model.evaluate(iterator, steps=2, verbose=1) + model.predict(iterator, steps=2) + + # Test with validation data + model.fit(iterator, + epochs=1, steps_per_epoch=2, verbose=0, + validation_data=iterator, validation_steps=2) + # Test with validation split + with self.assertRaisesRegexp( + ValueError, '`validation_split` argument is not supported ' + 'when input `x` is a dataset or a dataset iterator'): + model.fit(iterator, + epochs=1, steps_per_epoch=2, verbose=0, + validation_split=0.5, validation_steps=2) + + # Test with sample weight. + sample_weight = np.random.random((10,)) + with self.assertRaisesRegexp( + ValueError, '`sample_weight` argument is not supported ' + 'when input `x` is a dataset or a dataset iterator'): + model.fit( + iterator, + epochs=1, + steps_per_epoch=2, + verbose=0, + sample_weight=sample_weight) + + # Test invalid usage + with self.assertRaisesRegexp(ValueError, + 'you should not specify a target'): + model.fit(iterator, iterator, + epochs=1, steps_per_epoch=2, verbose=0) + + with self.assertRaisesRegexp( + ValueError, 'you should specify the `steps_per_epoch` argument'): + model.fit(iterator, epochs=1, verbose=0) + with self.assertRaisesRegexp(ValueError, + 'you should specify the `steps` argument'): + model.evaluate(iterator, verbose=0) + with self.assertRaisesRegexp(ValueError, + 'you should specify the `steps` argument'): + model.predict(iterator, verbose=0) + + @tf_test_util.run_in_graph_and_eager_modes + def test_get_next_op_created_once(self): + model = testing_utils.get_small_functional_mlp(1, 4, input_dim=3) + optimizer = RMSPropOptimizer(learning_rate=0.001) + loss = 'mse' + metrics = ['mae'] + model.compile(optimizer, loss, metrics=metrics) + + inputs = np.zeros((10, 3), np.float32) + targets = np.zeros((10, 4), np.float32) + dataset = dataset_ops.Dataset.from_tensor_slices((inputs, targets)) + dataset = dataset.repeat(100) + dataset = dataset.batch(10) + iterator = dataset_ops.make_one_shot_iterator(dataset) + + model.fit(iterator, epochs=1, steps_per_epoch=2, verbose=1) + # Finalize graph to make sure we are not appending another iterator + # get_next op in the graph. + ops.get_default_graph().finalize() + model.fit(iterator, epochs=1, steps_per_epoch=2, verbose=1) + + @tf_test_util.run_in_graph_and_eager_modes + def test_iterators_running_out_of_data(self): + model = testing_utils.get_small_functional_mlp(1, 4, input_dim=3) + optimizer = RMSPropOptimizer(learning_rate=0.001) + loss = 'mse' + metrics = ['mae'] + model.compile(optimizer, loss, metrics=metrics) + + inputs = np.zeros((10, 3), np.float32) + targets = np.zeros((10, 4), np.float32) + dataset = dataset_ops.Dataset.from_tensor_slices((inputs, targets)) + dataset = dataset.repeat(2) + dataset = dataset.batch(10) + iterator = dataset_ops.make_one_shot_iterator(dataset) + + with test.mock.patch.object(logging, 'warning') as mock_log: + model.fit(iterator, epochs=1, steps_per_epoch=3, verbose=0) + self.assertRegexpMatches( + str(mock_log.call_args), + 'dataset iterator ran out of data') + + +class TestTrainingWithDataset(test.TestCase, parameterized.TestCase): + + @tf_test_util.run_in_graph_and_eager_modes + def test_calling_model_on_same_dataset(self): + model = testing_utils.get_small_functional_mlp(1, 4, input_dim=3) + optimizer = RMSPropOptimizer(learning_rate=0.001) + loss = 'mse' + metrics = ['mae'] + model.compile(optimizer, loss, metrics=metrics) + + inputs = np.zeros((10, 3), np.float32) + targets = np.zeros((10, 4), np.float32) + dataset = dataset_ops.Dataset.from_tensor_slices((inputs, targets)) + dataset = dataset.repeat(100) + dataset = dataset.batch(10) + + # Call fit with validation data + model.fit(dataset, epochs=1, steps_per_epoch=2, verbose=0, + validation_data=dataset, validation_steps=2) + # Finalize the graph to make sure new ops aren't added when calling on the + # same dataset + ops.get_default_graph().finalize() + model.fit(dataset, epochs=1, steps_per_epoch=2, verbose=0, + validation_data=dataset, validation_steps=2) + + @tf_test_util.run_in_graph_and_eager_modes + def test_training_and_eval_methods_on_dataset(self): + model = testing_utils.get_small_functional_mlp(1, 4, input_dim=3) + optimizer = RMSPropOptimizer(learning_rate=0.001) + loss = 'mse' + metrics = ['mae', metrics_module.CategoricalAccuracy()] + model.compile(optimizer, loss, metrics=metrics) + + inputs = np.zeros((10, 3), np.float32) + targets = np.zeros((10, 4), np.float32) + dataset = dataset_ops.Dataset.from_tensor_slices((inputs, targets)) + dataset = dataset.repeat(100) + dataset = dataset.batch(10) + + model.fit(dataset, epochs=1, steps_per_epoch=2, verbose=1) + model.evaluate(dataset, steps=2, verbose=1) + model.predict(dataset, steps=2) + + # Test with validation data + model.fit(dataset, epochs=1, steps_per_epoch=2, verbose=0, + validation_data=dataset, validation_steps=2) + + # Test with validation split + with self.assertRaisesRegexp( + ValueError, '`validation_split` argument is not supported ' + 'when input `x` is a dataset or a dataset iterator'): + model.fit(dataset, + epochs=1, steps_per_epoch=2, verbose=0, + validation_split=0.5, validation_steps=2) + + # Test with sample weight. + sample_weight = np.random.random((10,)) + with self.assertRaisesRegexp( + ValueError, '`sample_weight` argument is not supported ' + 'when input `x` is a dataset or a dataset iterator'): + model.fit( + dataset, + epochs=1, + steps_per_epoch=2, + verbose=0, + sample_weight=sample_weight) + + # Test invalid usage + with self.assertRaisesRegexp(ValueError, + 'you should not specify a target'): + model.fit(dataset, dataset, + epochs=1, steps_per_epoch=2, verbose=0) + + with self.assertRaisesRegexp( + ValueError, 'you should specify the `steps_per_epoch` argument'): + model.fit(dataset, epochs=1, verbose=0) + with self.assertRaisesRegexp(ValueError, + 'you should specify the `steps` argument'): + model.evaluate(dataset, verbose=0) + with self.assertRaisesRegexp(ValueError, + 'you should specify the `steps` argument'): + model.predict(dataset, verbose=0) + + @tf_test_util.run_in_graph_and_eager_modes + def test_dataset_with_sample_weights(self): + model = testing_utils.get_small_functional_mlp(1, 4, input_dim=3) + optimizer = RMSPropOptimizer(learning_rate=0.001) + loss = 'mse' + metrics = ['mae', metrics_module.CategoricalAccuracy()] + model.compile(optimizer, loss, metrics=metrics) + + inputs = np.zeros((10, 3), np.float32) + targets = np.zeros((10, 4), np.float32) + sample_weights = np.ones((10), np.float32) + dataset = dataset_ops.Dataset.from_tensor_slices((inputs, targets, + sample_weights)) + dataset = dataset.repeat(100) + dataset = dataset.batch(10) + + model.fit(dataset, epochs=1, steps_per_epoch=2, verbose=1) + model.evaluate(dataset, steps=2, verbose=1) + model.predict(dataset, steps=2) + + @parameterized.parameters( + {'model': 'functional'}, + {'model': 'subclass'}, + ) + @tf_test_util.run_in_graph_and_eager_modes + def test_dataset_with_sparse_labels(self, model): + if model == 'functional': + model = testing_utils.get_small_functional_mlp(1, 4, input_dim=3) + elif model == 'subclass': + model = testing_utils.get_small_sequential_mlp(1, 4) + + for loss in ['sparse_categorical_crossentropy', + losses_impl.sparse_softmax_cross_entropy]: + optimizer = RMSPropOptimizer(learning_rate=0.001) + model.compile(optimizer, loss) + + inputs = np.zeros((10, 3), dtype=np.float32) + targets = np.random.randint(0, 4, size=10, dtype=np.int32) + dataset = dataset_ops.Dataset.from_tensor_slices((inputs, targets)) + dataset = dataset.repeat(100) + dataset = dataset.batch(10) + + model.fit(dataset, epochs=1, steps_per_epoch=2, verbose=1) + + @tf_test_util.run_deprecated_v1 + def test_dataset_input_shape_validation(self): + with self.cached_session(): + model = testing_utils.get_small_functional_mlp(1, 4, input_dim=3) + model.compile(optimizer=RMSPropOptimizer(learning_rate=0.001), loss='mse') + + # User forgets to batch the dataset + inputs = np.zeros((10, 3)) + targets = np.zeros((10, 4)) + dataset = dataset_ops.Dataset.from_tensor_slices((inputs, targets)) + dataset = dataset.repeat(100) + + with self.assertRaisesRegexp( + ValueError, + r'expected (.*?) to have shape \(3,\) but got array with shape \(1,\)' + ): + model.train_on_batch(dataset) + + # Wrong input shape + inputs = np.zeros((10, 5)) + targets = np.zeros((10, 4)) + dataset = dataset_ops.Dataset.from_tensor_slices((inputs, targets)) + dataset = dataset.repeat(100) + dataset = dataset.batch(10) + + with self.assertRaisesRegexp(ValueError, + r'expected (.*?) to have shape \(3,\)'): + model.train_on_batch(dataset) + + +class TestMetricsWithDatasetIterators(test.TestCase): + + @tf_test_util.run_in_graph_and_eager_modes + def test_metrics_correctness_with_iterator(self): + model = keras.Sequential() + model.add( + keras.layers.Dense( + 8, activation='relu', input_dim=4, kernel_initializer='ones')) + model.add( + keras.layers.Dense( + 1, activation='sigmoid', kernel_initializer='ones')) + model.compile( + loss='binary_crossentropy', + metrics=['accuracy', metrics_module.BinaryAccuracy()], + optimizer=RMSPropOptimizer(learning_rate=0.001)) + + np.random.seed(123) + x = np.random.randint(10, size=(100, 4)).astype(np.float32) + y = np.random.randint(2, size=(100, 1)).astype(np.float32) + dataset = dataset_ops.Dataset.from_tensor_slices((x, y)) + dataset = dataset.batch(10) + iterator = dataset_ops.make_one_shot_iterator(dataset) + outs = model.evaluate(iterator, steps=10) + self.assertEqual(np.around(outs[1], decimals=1), 0.5) + self.assertEqual(np.around(outs[2], decimals=1), 0.5) + + y = np.zeros((100, 1), dtype=np.float32) + dataset = dataset_ops.Dataset.from_tensor_slices((x, y)) + dataset = dataset.repeat(100) + dataset = dataset.batch(10) + iterator = dataset_ops.make_one_shot_iterator(dataset) + outs = model.evaluate(iterator, steps=10) + self.assertEqual(outs[1], 0.) + self.assertEqual(outs[2], 0.) + + +if __name__ == '__main__': + test.main() diff --git a/tensorflow/python/keras/engine/training_distributed.py b/tensorflow/python/keras/engine/training_distributed.py index 808d7c9f333..473f06ded70 100644 --- a/tensorflow/python/keras/engine/training_distributed.py +++ b/tensorflow/python/keras/engine/training_distributed.py @@ -19,9 +19,12 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function -import enum +import enum # pylint: disable=g-bad-import-order import numpy as np +from tensorflow.python.distribute import distribute_lib +from tensorflow.python.distribute import reduce_util as ds_reduce_util +from tensorflow.python.eager import context from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes from tensorflow.python.framework import errors @@ -34,9 +37,7 @@ from tensorflow.python.keras.engine import distributed_training_utils from tensorflow.python.keras.utils.generic_utils import Progbar from tensorflow.python.ops import array_ops from tensorflow.python.ops import math_ops -from tensorflow.python.ops import variable_scope from tensorflow.python.platform import tf_logging as logging -from tensorflow.python.training import distribute as distribute_lib from tensorflow.python.util import nest @@ -48,180 +49,15 @@ class _Mode(enum.Enum): # TODO(priyag, sourabhbajaj): Refactor this file to address code duplication. -def fit_loop( - model, - iterator, - epochs=100, - verbose=1, - callbacks=None, - val_iterator=None, - initial_epoch=0, - steps_per_epoch=None, - validation_steps=None): - """Fit loop for training with DistributionStrategy. - - Arguments: - model: Keras Model instance. - iterator: Iterator for input data. - epochs: Number of times to iterate over the data - verbose: Integer, Verbosity mode, 0, 1 or 2 - callbacks: List of callbacks to be called during training - val_iterator: Iterator for validation data. - initial_epoch: Epoch at which to start training - (useful for resuming a previous training run) - steps_per_epoch: Total number of steps (batches of samples) - before declaring one epoch finished and starting the - next epoch. Ignored with the default value of `None`. - validation_steps: Number of steps to run validation for - (only if doing validation from data tensors). - Ignored with the default value of `None`. - - Returns: - `History` object. - - Raises: - ValueError: in case of invalid arguments. - """ - current_strategy = model._distribution_strategy - - # TODO(priyag, sourabhbajaj): Remove this when the codepaths are merged. - if current_strategy.__class__.__name__ == 'TPUStrategy': - return _experimental_fit_loop( - model, iterator, epochs, verbose, callbacks, initial_epoch, - steps_per_epoch, val_iterator, validation_steps) - - if not model._grouped_model: - clone_model_on_replicas(model, current_strategy, make_callback_model=True) - - def _per_device_fit_function(model): - model._make_fit_function() - return (model._fit_function.inputs, model._fit_function.outputs, - model._fit_function.updates_op, model._fit_function.session_kwargs) - - inputs, targets, sample_weights = _get_input_from_iterator(iterator, model) - with current_strategy.scope(): - # Create train ops on each of the devices when we call - # `_per_device_fit_function`. - (grouped_inputs, grouped_outputs, grouped_updates, - grouped_session_args) = current_strategy.call_for_each_replica( - _per_device_fit_function, args=(model._grouped_model,)) - # Unwrap all the per device values returned from `call_for_each_replica`. - # Unwrapping per device values gives you a list of values that can be - # used to construct a new train function that is composed of update ops on - # all the devices over which the model is distributed. - (all_inputs, all_outputs, all_updates, - all_session_args) = distributed_training_utils.unwrap_values( - current_strategy, grouped_inputs, grouped_outputs, - grouped_updates, grouped_session_args, with_loss_tensor=True) - - # Dataset inputs and targets are also per devices values that need to be - # unwrapped. - dataset_inputs = distributed_training_utils.flatten_perdevice_values( - current_strategy, inputs) - dataset_targets = distributed_training_utils.flatten_perdevice_values( - current_strategy, targets) - - # Create a train function that is composed of all the parameters above. - distributed_fit_function = K.function( - all_inputs, - all_outputs, - updates=all_updates, - name='distributed_fit_function', - **all_session_args) - - # We need to set sample_weights to None since there are sample weight - # placeholders that are created with default values. - sample_weights = [None for _ in range( - len(model.outputs) * current_strategy.num_replicas_in_sync)] - if not isinstance(K.learning_phase(), int): - ins = dataset_inputs + dataset_targets + sample_weights + [1] - else: - ins = dataset_inputs + dataset_targets - - do_validation = False - if validation_steps: - do_validation = True - - # Copy the weights from the original model to each of the replicated models. - orig_model_weights = model.get_weights() - distributed_model = current_strategy.unwrap(model._grouped_model)[0] - distributed_training_utils.set_weights( - current_strategy, distributed_model, orig_model_weights) - - callbacks = cbks.configure_callbacks( - callbacks, - model, - do_validation=do_validation, - val_inputs=None, - val_targets=None, - epochs=epochs, - steps_per_epoch=steps_per_epoch, - verbose=verbose) - out_labels = model.metrics_names or [] - callbacks.on_train_begin() - - assert steps_per_epoch is not None - - for epoch in range(initial_epoch, epochs): - # Reset stateful metrics - for m in model.stateful_metric_functions: - m.reset_states() - callbacks.on_epoch_begin(epoch) - epoch_logs = {} - for step_index in range(steps_per_epoch): - batch_logs = {'batch': step_index, 'size': 1} - callbacks.on_batch_begin(step_index, batch_logs) - try: - outs = distributed_fit_function(ins) - except errors.OutOfRangeError: - logging.warning('Your dataset iterator ran out of data; ' - 'interrupting training. Make sure that your dataset ' - 'can generate at least `steps_per_epoch * epochs` ' - 'batches (in this case, %d batches).' % - steps_per_epoch * epochs) - break - - if not isinstance(outs, list): - outs = [outs] - for l, o in zip(out_labels, outs): - batch_logs[l] = o - callbacks.on_batch_end(step_index, batch_logs) - if callbacks.model.stop_training: - break - if do_validation: - val_outs = test_loop( - model, - val_iterator, - steps=validation_steps, - verbose=0) - if not isinstance(val_outs, list): - val_outs = [val_outs] - # Same labels assumed. - for l, o in zip(out_labels, val_outs): - epoch_logs['val_' + l] = o - - callbacks.on_epoch_end(epoch, epoch_logs) - if callbacks.model.stop_training: - break - callbacks.on_train_end() - - # Copy the weights back from the replicated model to the original model. - updated_weights = current_strategy.unwrap( - model._grouped_model)[0].get_weights() - model.set_weights(updated_weights) - return model.history - - -def _experimental_fit_loop( - model, - iterator, - epochs=100, - verbose=1, - callbacks=None, - initial_epoch=0, - steps_per_epoch=None, - val_iterator=None, - validation_steps=None): +def experimental_fit_loop(model, + iterator, + epochs=100, + verbose=1, + callbacks=None, + initial_epoch=0, + steps_per_epoch=None, + val_iterator=None, + validation_steps=None): """Fit loop for training with TPU DistributionStrategy. Arguments: @@ -259,11 +95,12 @@ def _experimental_fit_loop( K.set_learning_phase(1) out_labels = model.metrics_names or [] - def step_fn(ctx, inputs, targets): + def step_fn(ctx, inputs): """Clones the model and calls make_fit_function.""" # TODO(priyag, sourabhbajaj): The model gets cloned every time # fit/test/predict is called. We should look into caching this keyed on # input shapes. + inputs, targets = inputs clone_model_on_replicas( model, current_strategy, @@ -273,7 +110,7 @@ def _experimental_fit_loop( mode=_Mode.TRAIN) (grouped_inputs, grouped_outputs, grouped_updates, - grouped_session_args) = current_strategy.call_for_each_replica( + grouped_session_args) = current_strategy.extended.call_for_each_replica( _per_device_fit_function, args=(model._grouped_model_train,)) (all_inputs, all_outputs, all_updates, all_session_args) = distributed_training_utils.unwrap_values( @@ -288,12 +125,12 @@ def _experimental_fit_loop( for label, output in zip(out_labels, combined_fn.outputs): if label == 'loss': - aggregation = distribute_lib.get_loss_reduction() + reduce_op = distribute_lib.get_loss_reduction() else: - # We aggregate all other metrics using mean for now. This is temporary + # We reduce all other metrics using mean for now. This is temporary # workaround until new metrics are in place. - aggregation = variable_scope.VariableAggregation.MEAN - ctx.set_last_step_output(label, output, aggregation) + reduce_op = ds_reduce_util.ReduceOp.MEAN + ctx.set_last_step_output(label, output, reduce_op) # TODO(priyag, sourabhbajaj): Ignoring these things from the combined_fn: # feed_dict, session kwargs, run options, run_metadata for now. These should @@ -303,19 +140,20 @@ def _experimental_fit_loop( # Add initial dummy values for loss and other metric tensors. initial_loop_values = {} initial_loop_values['loss'] = constant_op.constant(1e7) - for name, tensor in zip(model.metrics_names[1:], model.metrics_tensors): + for name in model.metrics_names[1:]: + tensor = model._all_stateful_metrics_tensors[name] initial_loop_values[name] = array_ops.zeros(tensor.shape, tensor.dtype) if steps_per_epoch is None: raise ValueError('`steps_per_epoch` should be specified when calling ' '`fit` on the model.') steps_per_run = K.variable( - value=min(steps_per_epoch, current_strategy.steps_per_run), + value=min(steps_per_epoch, current_strategy.extended.steps_per_run), dtype='int32', name='steps_per_run') with current_strategy.scope(): - ctx = current_strategy.run_steps_on_dataset( + ctx = current_strategy.extended.experimental_run_steps_on_iterator( step_fn, iterator, iterations=steps_per_run, initial_loop_values=initial_loop_values) @@ -334,17 +172,16 @@ def _experimental_fit_loop( callbacks, model, do_validation=do_validation, - val_inputs=None, - val_targets=None, epochs=epochs, steps_per_epoch=steps_per_epoch, verbose=verbose) # Calculate the steps each time on the device. - steps_to_run = [current_strategy.steps_per_run] * ( - steps_per_epoch // current_strategy.steps_per_run) - if steps_per_epoch % current_strategy.steps_per_run: - steps_to_run.append(steps_per_epoch % current_strategy.steps_per_run) + steps_to_run = [current_strategy.extended.steps_per_run] * ( + steps_per_epoch // current_strategy.extended.steps_per_run) + if steps_per_epoch % current_strategy.extended.steps_per_run: + steps_to_run.append( + steps_per_epoch % current_strategy.extended.steps_per_run) callbacks.on_train_begin() for epoch in range(initial_epoch, epochs): @@ -384,7 +221,7 @@ def _experimental_fit_loop( model._grouped_model_train)[0].get_weights() model.set_weights(updated_weights) - val_outs = _experimental_test_loop( + val_outs = experimental_test_loop( # pylint: disable=undefined-variable model, val_iterator, steps=validation_steps, @@ -411,105 +248,11 @@ def _experimental_fit_loop( return model.history -def test_loop(model, iterator, verbose=0, steps=None): - """Test loop for evaluating with DistributionStrategy. - - Arguments: - model: Keras Model instance. - iterator: Iterator for input data. - verbose: Integer, Verbosity mode 0 or 1. - steps: Total number of steps (batches of samples) - before declaring predictions finished. - Ignored with the default value of `None`. - - Returns: - Scalar loss (if the model has a single output and no metrics) - or list of scalars (if the model has multiple outputs - and/or metrics). The attribute `model.metrics_names` will give you - the display labels for the outputs. - """ - current_strategy = model._distribution_strategy - - # TODO(priyag, sourabhbajaj): Remove this when the codepaths are merged. - if current_strategy.__class__.__name__ == 'TPUStrategy': - return _experimental_test_loop(model, iterator, verbose, steps) - - if not model._grouped_model: - clone_model_on_replicas(model, current_strategy) - - def _per_device_eval_function(model): - model._make_eval_function() - return (model._eval_function.inputs, model._eval_function.outputs, - model._eval_function.updates_op, - model._eval_function.session_kwargs) - - inputs, targets, sample_weights = _get_input_from_iterator(iterator, model) - with current_strategy.scope(): - (grouped_inputs, grouped_outputs, grouped_updates, - grouped_session_args) = current_strategy.call_for_each_replica( - _per_device_eval_function, args=(model._grouped_model,)) - - (all_inputs, all_outputs, all_updates, - all_session_args) = distributed_training_utils.unwrap_values( - current_strategy, grouped_inputs, grouped_outputs, grouped_updates, - grouped_session_args, with_loss_tensor=True) - - dataset_inputs = distributed_training_utils.flatten_perdevice_values( - current_strategy, inputs) - dataset_targets = distributed_training_utils.flatten_perdevice_values( - current_strategy, targets) - - distributed_test_function = K.function( - all_inputs, all_outputs, - updates=all_updates, - name='distributed_test_function', - **all_session_args) - - # We need to set sample_weights to None since there are sample weight - # placeholders that are created with default values. - sample_weights = [None for _ in range( - len(model.outputs) * current_strategy.num_replicas_in_sync)] - if not isinstance(K.learning_phase(), int): - ins = dataset_inputs + dataset_targets + sample_weights + [0] - else: - ins = dataset_inputs + dataset_targets - - for m in model.stateful_metric_functions: - m.reset_states() - - outs = [] - if verbose == 1: - progbar = Progbar(target=steps) - - # Copy the weights from the original model to each of the replicated models. - orig_model_weights = model.get_weights() - distributed_model = current_strategy.unwrap(model._grouped_model)[0] - distributed_training_utils.set_weights( - current_strategy, distributed_model, orig_model_weights) - - assert steps is not None - for step in range(steps): - batch_outs = distributed_test_function(ins) - if isinstance(batch_outs, list): - if step == 0: - outs = [0.] * len(batch_outs) - outs[0] += batch_outs[0] # index 0 = 'loss' - outs[1:] = batch_outs[1:] - else: - if step == 0: - outs.append(0.) - outs[0] += batch_outs # index 0 = 'loss' - if verbose >= 1: - progbar.update(step + 1) - outs[0] /= steps # index 0 = 'loss' - - if len(outs) == 1: - return outs[0] - return outs - - -def _experimental_test_loop(model, iterator, verbose=0, steps=None, - initialize_finalize_strategy=True): +def experimental_test_loop(model, + iterator, + verbose=0, + steps=None, + initialize_finalize_strategy=True): """Test loop for evaluating with TPU DistributionStrategy. Arguments: @@ -541,11 +284,12 @@ def _experimental_test_loop(model, iterator, verbose=0, steps=None, # TODO(priyag, sourabhbajaj): This should likely not be hardcoded here. K.set_learning_phase(0) - def step_fn(ctx, inputs, targets): + def step_fn(ctx, inputs): """Clones the model and calls make_eval_function.""" # TODO(priyag, sourabhbajaj): The model gets cloned every time # fit/test/predict is called. We should look into caching this keyed on # input shapes. + inputs, targets = inputs clone_model_on_replicas( model, current_strategy, @@ -555,7 +299,7 @@ def _experimental_test_loop(model, iterator, verbose=0, steps=None, mode=_Mode.TEST) (grouped_inputs, grouped_outputs, grouped_updates, - grouped_session_args) = current_strategy.call_for_each_replica( + grouped_session_args) = current_strategy.extended.call_for_each_replica( _per_device_eval_function, args=(model._grouped_model_test,)) (all_inputs, all_outputs, all_updates, @@ -571,25 +315,26 @@ def _experimental_test_loop(model, iterator, verbose=0, steps=None, for label, output in zip(model.metrics_names, combined_fn.outputs): if label == 'loss': - aggregation = distribute_lib.get_loss_reduction() + reduce_op = distribute_lib.get_loss_reduction() else: - # We aggregate all other metrics using mean for now. This is temporary + # We reduce all other metrics using mean for now. This is temporary # workaround until new metrics are in place. - aggregation = variable_scope.VariableAggregation.MEAN - ctx.set_last_step_output(label, output, aggregation) + reduce_op = ds_reduce_util.ReduceOp.MEAN + ctx.set_last_step_output(label, output, reduce_op) return combined_fn.updates_op # Add initial dummy values for loss and other metric tensors. initial_loop_values = {} initial_loop_values['loss'] = constant_op.constant(1e7) - for name, tensor in zip(model.metrics_names[1:], model.metrics_tensors): + for name in model.metrics_names[1:]: + tensor = model._all_stateful_metrics_tensors[name] initial_loop_values[name] = array_ops.zeros(tensor.shape, tensor.dtype) with current_strategy.scope(): # TODO(priyag): Use steps_per_run when we use new metrics as they will # allow handling metric computation at each step using variables. - ctx = current_strategy.run_steps_on_dataset( + ctx = current_strategy.extended.experimental_run_steps_on_iterator( step_fn, iterator, iterations=1, initial_loop_values=initial_loop_values) @@ -625,103 +370,7 @@ def _experimental_test_loop(model, iterator, verbose=0, steps=None, return outs -def predict_loop(model, iterator, verbose=0, steps=None): - """Predict loop for predicting with DistributionStrategy. - - Arguments: - model: Keras Model instance. - iterator: Iterator for input data. - verbose: Integer, Verbosity mode 0 or 1. - steps: Total number of steps (batches of samples) - before declaring `_predict_loop` finished. - Ignored with the default value of `None`. - - Returns: - Array of predictions (if the model has a single output) - or list of arrays of predictions - (if the model has multiple outputs). - """ - current_strategy = model._distribution_strategy - - # TODO(priyag, sourabhbajaj): Remove this when the codepaths are merged. - if current_strategy.__class__.__name__ == 'TPUStrategy': - return _experimental_predict_loop(model, iterator, verbose, steps) - - if not model._grouped_model: - clone_model_on_replicas(model, current_strategy) - - def _per_device_predict_function(model): - model._make_predict_function() - return (model.predict_function.inputs, - model.predict_function.outputs, - model.predict_function.updates_op, - model.predict_function.session_kwargs) - - inputs, _, _ = _get_input_from_iterator(iterator, model) - with current_strategy.scope(): - (grouped_inputs, grouped_outputs, grouped_updates, - grouped_session_args) = current_strategy.call_for_each_replica( - _per_device_predict_function, args=(model._grouped_model,)) - - (all_inputs, all_outputs, all_updates, - all_session_args) = distributed_training_utils.unwrap_values( - current_strategy, grouped_inputs, grouped_outputs, grouped_updates, - grouped_session_args) - - dataset_inputs = distributed_training_utils.flatten_perdevice_values( - current_strategy, inputs) - - distributed_predict_function = K.function( - all_inputs, all_outputs, - updates=all_updates, - name='distributed_predict_function', - **all_session_args) - - if not isinstance(K.learning_phase(), int): - ins = dataset_inputs + [0] - else: - ins = dataset_inputs - - if verbose == 1: - progbar = Progbar(target=steps) - - # Copy the weights from the original model to each of the replicated models. - orig_model_weights = model.get_weights() - distributed_model = current_strategy.unwrap(model._grouped_model)[0] - distributed_training_utils.set_weights( - current_strategy, distributed_model, orig_model_weights) - - num_replicas = current_strategy.num_replicas_in_sync - # Since we do not know how many samples we will see, we cannot - # pre-allocate the returned Numpy arrays. Instead, we store one array per - # batch seen and concatenate them upon returning. - unconcatenated_outs = [] - assert steps is not None - for step in range(steps): - batch_outs = distributed_predict_function(ins) - if not isinstance(batch_outs, list): - batch_outs = [batch_outs] - if step == 0: - # batch_outs gives you the number of model outputs. In the distributed - # case this will be number of model_outputs * num_replicas. - for _ in range(len(model.outputs)): - unconcatenated_outs.append([]) - for i in range(len(model.outputs)): - nested_outs = batch_outs[i * num_replicas: - i * num_replicas + num_replicas] - outs = nest.flatten(nested_outs) - unconcatenated_outs[i].extend(outs) - if verbose >= 1: - progbar.update(step + 1) - if len(unconcatenated_outs) == 1: - return np.concatenate(unconcatenated_outs[0], axis=0) - return [ - np.concatenate(unconcatenated_outs[i], axis=0) - for i in range(len(unconcatenated_outs)) - ] - - -def _experimental_predict_loop(model, iterator, verbose=0, steps=None): +def experimental_predict_loop(model, iterator, verbose=0, steps=None): """Predict loop for predicting with TPU DistributionStrategy. Arguments: @@ -750,7 +399,7 @@ def _experimental_predict_loop(model, iterator, verbose=0, steps=None): model.predict_function.updates_op, model.predict_function.session_kwargs) - def step_fn(ctx, *inputs): + def step_fn(ctx, inputs): """Clones the model and calls make_predict_function.""" # TODO(priyag, sourabhbajaj): The model gets cloned every time @@ -764,7 +413,7 @@ def _experimental_predict_loop(model, iterator, verbose=0, steps=None): mode=_Mode.PREDICT) (grouped_inputs, grouped_outputs, grouped_updates, - grouped_session_args) = current_strategy.call_for_each_replica( + grouped_session_args) = current_strategy.extended.call_for_each_replica( _per_device_predict_function, args=(model._grouped_model_predict,)) (all_inputs, all_outputs, all_updates, @@ -795,7 +444,7 @@ def _experimental_predict_loop(model, iterator, verbose=0, steps=None): with current_strategy.scope(): # TODO(priyag, sourabhbajaj): Support steps_per_run if/when we add outfeed. - ctx = current_strategy.run_steps_on_dataset( + ctx = current_strategy.extended.experimental_run_steps_on_iterator( step_fn, iterator, iterations=1, initial_loop_values=initial_loop_values) @@ -835,7 +484,17 @@ def _experimental_predict_loop(model, iterator, verbose=0, steps=None): ] -def _clone_and_build_model(model, inputs=None, targets=None): +def _custom_compile_for_predict(model): + """Custom compile for TPU predict mode.""" + model.total_loss = None + model._fit_function = None + model._eval_function = None + model.train_function = None + model.test_function = None + model.predict_function = None + + +def _clone_and_build_model(model, inputs=None, targets=None, mode=None): """Clone and build the given keras_model.""" # We need to set the import here since we run into a circular dependency # error. @@ -862,23 +521,27 @@ def _clone_and_build_model(model, inputs=None, targets=None): if isinstance(targets, tuple): targets = nest.flatten(targets) - cloned_model.compile( - optimizer, - model.loss, - metrics=metrics_module.clone_metrics(model.metrics), - loss_weights=model.loss_weights, - sample_weight_mode=model.sample_weight_mode, - weighted_metrics=metrics_module.clone_metrics(model.weighted_metrics), - target_tensors=targets) + if mode == _Mode.PREDICT: + _custom_compile_for_predict(cloned_model) + else: + cloned_model.compile( + optimizer, + model.loss, + metrics=metrics_module.clone_metrics(model._compile_metrics), + loss_weights=model.loss_weights, + sample_weight_mode=model.sample_weight_mode, + weighted_metrics=metrics_module.clone_metrics( + model._compile_weighted_metrics), + target_tensors=targets) return cloned_model def clone_model_on_replicas(model, strategy, make_callback_model=False, inputs=None, targets=None, mode=None): """Create a cloned model on each replica.""" - with strategy.scope(): - grouped_model = strategy.call_for_each_replica( - _clone_and_build_model, args=(model, inputs, targets)) + with K.get_graph().as_default(), strategy.scope(): + grouped_model = strategy.extended.call_for_each_replica( + _clone_and_build_model, args=(model, inputs, targets, mode)) if mode is _Mode.TRAIN: model._grouped_model_train = grouped_model elif mode is _Mode.TEST: @@ -915,3 +578,149 @@ def _get_input_from_iterator(iterator, model): model._standardize_weights(x_values, y_values, sample_weight=sample_weights_values) return x, y, sample_weights + + +def _make_execution_function(model, mode): + """Makes function to run one step of distributed model execution.""" + if context.executing_eagerly(): + return _make_eager_execution_function(model, mode) + + strategy = model._distribution_strategy + if not model._grouped_model: + clone_model_on_replicas( + model, strategy, make_callback_model=(mode == 'train')) + + def _per_device_function(model): + f = model._make_execution_function(mode) + return (f.inputs, f.outputs, f.updates_op, f.session_kwargs) + + with strategy.scope(): + # Create train ops on each of the devices when we call + # `_per_device_fit_function`. + (grouped_inputs, grouped_outputs, grouped_updates, + grouped_session_args) = strategy.extended.call_for_each_replica( + _per_device_function, args=(model._grouped_model,)) + + if mode == 'train': + # Initialize the variables in the replicated model. This is necessary for + # multi-worker training because on some workers, initialization is not + # needed. This method does initialization or waiting for initialization + # according to the context object of distribute coordinator. + distributed_training_utils.init_restore_or_wait_for_variables() + + # Unwrap all the per device values returned from `call_for_each_replica`. + # Unwrapping per device values gives you a list of values that can be + # used to construct a new train function that is composed of update ops on + # all the devices over which the model is distributed. + (all_inputs, all_outputs, all_updates, + all_session_args) = distributed_training_utils.unwrap_values( + strategy, + grouped_inputs, + grouped_outputs, + grouped_updates, + grouped_session_args, + with_loss_tensor=(mode != 'predict')) + + return K.function( + all_inputs, + all_outputs, + updates=all_updates, + name='distributed_{}_function'.format(mode), + **all_session_args) + + +def _make_eager_execution_function(model, mode): + """Makes function to run one step of distributed model eager execution.""" + strategy = model._distribution_strategy + if not model._grouped_model: + clone_model_on_replicas( + model, strategy, make_callback_model=(mode == 'train')) + + def _per_device_function(model): + f = model._make_execution_function(mode) + return (f.inputs, f.outputs) + + # NOTE(priyag): Try creating a new FuncGraph within DS scope instead of using + # the global one. + with K.get_graph().as_default(), strategy.scope(): + # Create train ops on each of the devices when we call + # `_per_device_fit_function`. + (grouped_inputs, grouped_outputs) = strategy.call_for_each_replica( + _per_device_function, args=(model._grouped_model,)) + + # Unwrap all the per device values returned from `call_for_each_replica`. + # Unwrapping per device values gives you a list of values that can be + # used to construct a new train function that is composed of inptus/outputs + # on all the devices over which the model is distributed. + (all_inputs, all_outputs, _, _) = distributed_training_utils.unwrap_values( + strategy, + grouped_inputs, + grouped_outputs, + with_loss_tensor=(mode != 'predict')) + + return K.function( + all_inputs, + all_outputs, + name='eager_distributed_{}_function'.format(mode)) + + +def _prepare_feed_values(model, inputs, targets, sample_weights, mode): + """Prepare feed values to the model execution function. + + Arguments: + model: Model to prepare feed values for. + inputs: List or dict of model inputs. + targets: Optional list of model targets. + sample_weights: Optional list of sample weight arrays. + mode: One of 'train'/'test'/'predict'. + + Returns: + Feed values for the model in the given mode. + """ + strategy = model._distribution_strategy + inputs, targets, sample_weights = _get_input_from_iterator(inputs, model) + inputs = distributed_training_utils.flatten_perdevice_values(strategy, inputs) + targets = distributed_training_utils.flatten_perdevice_values( + strategy, targets) + if mode == 'predict': + sample_weights = [] + targets = [] + else: + sample_weights = [ + None for _ in range(len(model.outputs) * strategy.num_replicas_in_sync) + ] + ins = inputs + targets + sample_weights + if mode == 'train' and not isinstance(K.symbolic_learning_phase(), int): + ins += [True] + return ins + + +def _copy_weights_to_distributed_model(model): + """Copies weights from original model to distributed models.""" + if model._distribution_strategy: + # Copy the weights from the original model to each of the replicated models. + orig_model_weights = model.get_weights() + distributed_model = model._distribution_strategy.unwrap( + model._grouped_model)[0] + distributed_training_utils.set_weights( + model._distribution_strategy, distributed_model, orig_model_weights) + + +def _copy_weights_to_original_model(model, mode): + """Copies weights from first distributed model back to original model.""" + if model._distribution_strategy and mode == 'train': + updated_weights = model._distribution_strategy.unwrap( + model._grouped_model)[0].get_weights() + model.set_weights(updated_weights) + + +def _per_device_aggregate_batch(batch_outs, model, mode): + """Aggregates the per-device batch-level outputs from a distributed step.""" + if model._distribution_strategy is not None and mode == 'predict': + total_batch_outs = [] + for i in range(len(model.outputs)): + num_replicas = model._distribution_strategy.num_replicas_in_sync + nested_outs = batch_outs[i * num_replicas:i * num_replicas + num_replicas] + total_batch_outs.append(np.concatenate(nest.flatten(nested_outs))) + return total_batch_outs + return batch_outs diff --git a/tensorflow/python/keras/engine/training_eager.py b/tensorflow/python/keras/engine/training_eager.py index 9131df5cd0a..1d1cec1c507 100644 --- a/tensorflow/python/keras/engine/training_eager.py +++ b/tensorflow/python/keras/engine/training_eager.py @@ -19,30 +19,20 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function -import copy -import threading +import collections -import numpy as np - -from tensorflow.python.data.ops import iterator_ops -from tensorflow.python.eager import function as eager_function from tensorflow.python.eager.backprop import GradientTape -from tensorflow.python.framework import errors from tensorflow.python.framework import ops from tensorflow.python.framework import tensor_util from tensorflow.python.keras import backend -from tensorflow.python.keras import callbacks as cbks -from tensorflow.python.keras import metrics as metrics_module +from tensorflow.python.keras import losses as losses_module from tensorflow.python.keras.engine import training_utils from tensorflow.python.keras.utils import generic_utils +from tensorflow.python.keras.utils.losses_utils import squeeze_or_expand_dimensions from tensorflow.python.ops import math_ops from tensorflow.python.platform import tf_logging as logging -# A lock for assigning polymorphic functions to models in a thread-safe way -_graph_function_building_lock = threading.Lock() - - def _eager_loss_fn(outputs, targets, loss_fn, output_name): with backend.name_scope(output_name + '_loss'): loss = loss_fn(targets, outputs) @@ -133,11 +123,24 @@ def _model_loss(model, else: weights = None mask = masks[i] - - weighted_masked_fn = training_utils.weighted_masked_objective(loss_fn) with backend.name_scope(model.output_names[i] + '_loss'): - output_loss = weighted_masked_fn( - targets[i], outs[i], weights, mask=mask) + if isinstance(loss_fn, losses_module.Loss): + if mask is not None: + mask = math_ops.cast(mask, outs[i].dtype) + # Update weights with mask. + if weights is None: + weights = mask + else: + # Update dimensions of weights to match with mask if possible. + mask, _, weights = squeeze_or_expand_dimensions( + mask, None, weights) + weights *= mask + output_loss = loss_fn(targets[i], outs[i], sample_weight=weights) + else: + weighted_masked_fn = training_utils.weighted_masked_objective(loss_fn) + output_loss = weighted_masked_fn( + targets[i], outs[i], weights, mask=mask) + # If the number of outputs is 1 then we don't append the loss metric # associated with each model output. When there are multiple outputs # associated with a model, each output's loss is calculated and returned @@ -171,412 +174,6 @@ def _model_loss(model, return outs, total_loss, loss_metrics, aggregated_loss_metrics, masks -def _maybe_build_graph_functions(model): - """Constructs polymorphic functions to use for fit, evaluate and predict.""" - # We lock this function to ensure thread-safety in case users are - # hypothetically trying to call '.predict' on a model in multiple threads - # at once when the graph functions were never previously built. - with _graph_function_building_lock: - if not model._built_graph_functions: - model._eager_process_single_batch_graph_function = eager_function.defun( - _process_single_batch - ) - model._eager_model_loss_graph_function = eager_function.defun(_model_loss) - model._eager_call_graph_function = eager_function.defun(model.call) - model._built_graph_functions = True - - -def _maybe_graph_function_model_loss(model, - inputs, - targets, - output_loss_metrics=None, - sample_weights=None, - training=False): - """Compute model loss, using defun if the model supports it.""" - if model._can_use_graph_functions: - _maybe_build_graph_functions(model) - return model._eager_model_loss_graph_function( - model, - inputs, - targets, - output_loss_metrics=output_loss_metrics, - sample_weights=sample_weights, - training=training) - else: - return _model_loss( - model, - inputs, - targets, - output_loss_metrics=output_loss_metrics, - sample_weights=sample_weights, - training=training) - - -def _maybe_graph_function_model_call(model, *args, **kwargs): - """Compute model loss, using defun if the model supports it.""" - if model._can_use_graph_functions: - _maybe_build_graph_functions(model) - return model._eager_call_graph_function(*args, **kwargs) - else: - return model.call(*args, **kwargs) - - -def iterator_fit_loop(model, - inputs, - class_weight, - steps_per_epoch, - epoch_logs, - val_inputs=None, - val_targets=None, - val_sample_weights=None, - epochs=1, - verbose=1, - callbacks=None, - validation_steps=None, - do_validation=False, - batch_size=None, - output_loss_metrics=None): - """Fit function for eager execution when input is given as dataset iterator. - - Updates the given epoch logs. - - Arguments: - model: Instance of the `Model`. - inputs: Input dataset iterator. - class_weight: Optional class-weight array to weight the importance of - samples in `inputs` based on the class they belong to, as conveyed by - the targets from the `inputs` iterator. - steps_per_epoch: Total number of steps (batches of samples) - before declaring one epoch finished and starting the - next epoch. - epoch_logs: Dictionary of logs from every epoch. - val_inputs: Input data for validation. - val_targets: Target data for validation. - val_sample_weights: Sample weight data for validation. - epochs: Number of times to iterate over the data - verbose: Verbosity mode, 0, 1 or 2 - callbacks: CallbackList instance. Controls callbacks during training. - validation_steps: Number of steps to run validation for (only if doing - validation from data tensors). Ignored with default value of `None`. - do_validation: Boolean value indicating whether we should do validation. - batch_size: int, val_inputs and val_targets will be evaled batch by - batch with size batch_size if they are array. - output_loss_metrics: List of metrics that are used to aggregated output - loss values. - - Raises: - ValueError: In case of mismatch between given number of inputs and - expectations of the model. - """ - assert isinstance(inputs, iterator_ops.EagerIterator) - - # make sure either x,y or x,y,sample_weights is provided - if (not isinstance(inputs.output_shapes, (list, tuple)) or - len(inputs.output_shapes) not in (2, 3)): - raise ValueError('Please provide either inputs and targets ' - 'or inputs, targets, and sample_weights') - - for step_index in range(steps_per_epoch): - batch_logs = {'batch': step_index, 'size': 1} - callbacks.on_batch_begin(step_index, batch_logs) - - # Get data from the iterator. - try: - next_element = inputs.get_next() - except errors.OutOfRangeError: - logging.warning( - 'Your dataset iterator ran out of data; interrupting training. Make ' - 'sure that your dataset can generate at least ' - '`steps_per_epoch * epochs` batches (in this case, %d batches). You ' - 'may need to use the repeat() function when building your ' - 'dataset.' % steps_per_epoch * epochs) - break - - if len(inputs.output_shapes) == 2: - x, y = next_element - sample_weights = None - else: - x, y, sample_weights = next_element - - # Validate and standardize data. - x, y, sample_weights = model._standardize_user_data( - x, y, sample_weight=sample_weights, class_weight=class_weight) - x = training_utils.cast_if_floating_dtype(x) - y = training_utils.cast_if_floating_dtype(y) - if sample_weights: - sample_weights = [ - training_utils.cast_if_floating_dtype( - ops.convert_to_tensor(val, dtype=backend.floatx())) - if val is not None else None for val in sample_weights - ] - - # Set stateful_metrics in callbacks. We do not do this before the - # `steps_per_epoch` loop because model will be compiled only in the first - # iteration of this loop in the deferred build scenario. - if step_index == 0: - for cbk in callbacks: - if (isinstance(cbk, cbks.BaseLogger) or - isinstance(cbk, cbks.ProgbarLogger)): - cbk.stateful_metrics = model.metrics_names[1:] # Exclude `loss` - - if step_index == 0 and not callbacks.params['metrics']: - callback_metrics = copy.copy(model.metrics_names) - if do_validation: - callback_metrics += ['val_' + n for n in model.metrics_names] - callbacks.set_params({ - 'batch_size': batch_size, - 'epochs': epochs, - 'steps': steps_per_epoch, - 'verbose': verbose, - 'do_validation': do_validation, - 'metrics': callback_metrics or [], - 'validation_steps': validation_steps - }) - - # Train model. - outs, loss, _, aggregated_loss_metrics, masks = \ - _maybe_graph_function_process_single_batch( - model, - x, - y, - output_loss_metrics=output_loss_metrics, - sample_weights=sample_weights, - training=True) - outs = generic_utils.to_list(outs) - - # Calculate metrics. - for l, o in zip(model.metrics_names, outs): - batch_logs[l] = o - metrics_results = _eager_metrics_fn( - model, outs, y, sample_weights=sample_weights, masks=masks) - batch_logs['loss'] = tensor_util.constant_value(backend.mean(loss)) - - for k, v in zip( - model.metrics_names, - [backend.mean(loss)] + aggregated_loss_metrics + metrics_results): - batch_logs[k] = tensor_util.constant_value(v) - callbacks.on_batch_end(step_index, batch_logs) - if callbacks.model.stop_training: - break - - if step_index == steps_per_epoch - 1: - if do_validation: - val_outs = test_loop( - model, - val_inputs, - val_targets, - sample_weights=val_sample_weights, - steps=validation_steps, - verbose=0, - batch_size=batch_size) - if not isinstance(val_outs, list): - val_outs = [val_outs] - # Same labels assumed. - for l, o in zip(model.metrics_names, val_outs): - epoch_logs['val_' + l] = o - - -def iterator_test_loop(model, inputs, steps, verbose=0): - """Test function for eager execution when input is given as dataset iterator. - - Arguments: - model: Model instance that is being evaluated in Eager mode. - inputs: Input dataset iterator. - steps: Total number of steps (batches of samples) before declaring - predictions finished. - verbose: Verbosity mode. - - Returns: - Scalar loss (if the model has a single output and no metrics) - or list of scalars (if the model has multiple outputs - and/or metrics). The attribute `model.metrics_names` will give you - the display labels for the scalar outputs. - - Raises: - ValueError: In case of mismatch between given number of inputs and - expectations of the model. - """ - assert isinstance(inputs, iterator_ops.EagerIterator) - # make sure either x,y or x,y,sample_weights is provided - if (not isinstance(inputs.output_shapes, (list, tuple)) or - len(inputs.output_shapes) < 2 or len(inputs.output_shapes) > 3): - raise ValueError('Please provide either inputs and targets' - 'or inputs, targets, and sample_weights') - outs = [] - - # Create metric wrapper for the losses. - output_loss_metrics = [] - for i in range(len(model.outputs)): - loss_fn = model.loss_functions[i] - mean_wrapped_loss = metrics_module.MeanMetricWrapper( - loss_fn, name=loss_fn.__name__) - output_loss_metrics.append(mean_wrapped_loss) - - num_samples = 0 - if verbose == 1: - progbar = generic_utils.Progbar(target=steps) - for step_index in range(steps): - # Get data from the iterator. - try: - next_element = inputs.get_next() - except errors.OutOfRangeError: - logging.warning( - 'Your dataset iterator ran out of data interrupting testing. ' - 'Make sure that your dataset can generate at least `steps` batches ' - '(in this case, %d batches). You may need to use the repeat() ' - 'function when building your dataset.', steps) - break - - if len(inputs.output_shapes) == 2: - x, y = next_element - sample_weights = None - else: - x, y, sample_weights = next_element - - # Validate and standardize data. - x, y, sample_weights = model._standardize_user_data( - x, y, sample_weight=sample_weights) - x = training_utils.cast_if_floating_dtype(x) - y = training_utils.cast_if_floating_dtype(y) - if sample_weights: - sample_weights = [ - training_utils.cast_if_floating_dtype( - ops.convert_to_tensor(val, dtype=backend.floatx())) - if val is not None else None for val in sample_weights - ] - - if step_index == 0: - # Get stateful metrics indices. We do not do this before the `steps` loop - # because model will be compiled only in the first iteration of this loop - # in the deferred build scenario. - if hasattr(model, 'metrics'): - for m in model.stateful_metric_functions: - m.reset_states() - for m in output_loss_metrics: - m.reset_states() - - # Calculate model output, loss values. - loss_outs, loss, _, aggregated_loss_metrics, masks = \ - _maybe_graph_function_model_loss( - model, - x, - y, - output_loss_metrics=output_loss_metrics, - sample_weights=sample_weights, - training=False) - metrics_results = _eager_metrics_fn( - model, loss_outs, y, sample_weights=sample_weights, masks=masks) - batch_outs = [] - for _, v in zip( - model.metrics_names, - [backend.mean(loss)] + aggregated_loss_metrics + metrics_results): - batch_outs.append(tensor_util.constant_value(v)) - - # Get current step size. - if isinstance(x, list): - step_size = x[0].get_shape().as_list()[0] - elif isinstance(x, dict): - step_size = list(x.values())[0].get_shape().as_list()[0] - else: - step_size = x.get_shape().as_list()[0] - - # Accumulate results in output array. - if not isinstance(batch_outs, list): - batch_outs = [batch_outs] - if step_index == 0: - for _ in enumerate(batch_outs): - outs.append(0.) - outs[0] += batch_outs[0] * step_size # index 0 = 'loss' - outs[1:] = batch_outs[1:] - - # Calculate sample size. - num_samples += step_size - if verbose == 1: - progbar.update(step_index + 1) - - outs[0] /= num_samples # index 0 = 'loss' - if len(outs) == 1: - return outs[0] - return outs - - -def iterator_predict_loop(model, inputs, steps, verbose=0): - """Predict function for eager execution when input is dataset iterator. - - Arguments: - model: Instance of `Model`. - inputs: Input dataset iterator. - steps: Total number of steps (batches of samples) before declaring - `_predict_loop` finished. - verbose: Verbosity mode. - - Returns: - Array of predictions (if the model has a single output) - or list of arrays of predictions (if the model has multiple outputs). - - Raises: - ValueError: In case of mismatch between given number of inputs and - expectations of the model. - """ - assert isinstance(inputs, iterator_ops.EagerIterator) - if not isinstance(inputs.output_shapes, - (list, tuple)) or len(inputs.output_shapes) > 3: - raise ValueError( - 'Please provide data as a list or tuple of 1, 2, or 3 elements ' - ' - `(input)`, or `(input, target)`, or `(input, target,' - 'sample_weights)`. Received %s. We do not use the `target` or' - '`sample_weights` value here.' % inputs.output_shapes) - outs = [] - if verbose == 1: - progbar = generic_utils.Progbar(target=steps) - - for step_index in range(steps): - # Get data from the iterator. - try: - next_element = inputs.get_next() - except errors.OutOfRangeError: - logging.warning( - 'Your dataset iterator ran out of data; interrupting prediction. ' - 'Make sure that your dataset can generate at least `steps` batches ' - '(in this case, %d batches). You may need to use the repeat() ' - 'function when building your dataset.', steps) - break - - # expects a tuple, where first element of tuple represents inputs - x = next_element[0] - - # Validate and standardize data. - x, _, _ = model._standardize_user_data(x) - x = training_utils.cast_if_floating_dtype(x) - - if isinstance(x, list) and len(x) == 1: - x = x[0] - - if model._expects_training_arg: - batch_outs = _maybe_graph_function_model_call(model, x, training=False) - else: - batch_outs = _maybe_graph_function_model_call(model, x) - if not isinstance(batch_outs, list): - batch_outs = [batch_outs] - - # We collect the results from every step and then concatenate them once - # in the end. This is an expensive process. We are doing this because we - # do not know the number of samples beforehand. - if step_index == 0: - for _ in batch_outs: - outs.append([]) - for i, batch_out in enumerate(batch_outs): - outs[i].append(backend.get_value(batch_out)) - - if verbose == 1: - progbar.update(step_index + 1) - for i, out in enumerate(outs): - outs[i] = np.concatenate(tuple(out), axis=0) - if len(outs) == 1: - return outs[0] - return outs - - def _process_single_batch(model, inputs, targets, @@ -630,32 +227,6 @@ def _process_single_batch(model, return outs, loss, loss_metrics, aggregated_loss_metrics, masks -def _maybe_graph_function_process_single_batch(model, - inputs, - targets, - output_loss_metrics=None, - sample_weights=None, - training=False): - """Process a single batch, using defun if the model supports it.""" - if model._can_use_graph_functions: - _maybe_build_graph_functions(model) - return model._eager_process_single_batch_graph_function( - model, - inputs, - targets, - output_loss_metrics=output_loss_metrics, - sample_weights=sample_weights, - training=training) - else: - return _process_single_batch( - model, - inputs, - targets, - output_loss_metrics=output_loss_metrics, - sample_weights=sample_weights, - training=training) - - def train_on_batch(model, inputs, targets, sample_weights=None): """Calculates the loss and gradient updates for one input batch. @@ -668,25 +239,25 @@ def train_on_batch(model, inputs, targets, sample_weights=None): Returns: total loss and the loss associated with each output. """ - if len(inputs) and tensor_util.is_tensor(inputs[0]): - inputs = training_utils.cast_if_floating_dtype(inputs) - targets = training_utils.cast_if_floating_dtype(targets) - else: - inputs = [ - ops.convert_to_tensor(val, dtype=backend.floatx()) for val in inputs - ] - targets = [ - ops.convert_to_tensor(val, dtype=backend.floatx()) for val in targets - ] + if isinstance(inputs, collections.Sequence): + if len(inputs) and tensor_util.is_tensor(inputs[0]): + inputs = training_utils.cast_if_floating_dtype(inputs) + targets = training_utils.cast_if_floating_dtype(targets) + else: + inputs = [ + ops.convert_to_tensor(val, dtype=backend.floatx()) for val in inputs + ] + targets = [ + ops.convert_to_tensor(val, dtype=backend.floatx()) for val in targets + ] if sample_weights: sample_weights = [ ops.convert_to_tensor(val, dtype=backend.floatx()) if val is not None else None for val in sample_weights ] - outs, loss, loss_metrics, _, masks = \ - _maybe_graph_function_process_single_batch( - model, inputs, targets, sample_weights=sample_weights, training=True) + outs, loss, loss_metrics, _, masks = _process_single_batch( + model, inputs, targets, sample_weights=sample_weights, training=True) if not isinstance(outs, list): outs = [outs] metrics_results = _eager_metrics_fn( @@ -695,7 +266,7 @@ def train_on_batch(model, inputs, targets, sample_weights=None): targets, sample_weights=sample_weights, masks=masks, - return_stateful_result=False) + return_stateful_result=True) loss = generic_utils.to_list(loss) return [ @@ -716,22 +287,23 @@ def test_on_batch(model, inputs, targets, sample_weights=None): Returns: total loss, loss and metrics associated with each output. """ - if len(inputs) and tensor_util.is_tensor(inputs[0]): - inputs = training_utils.cast_if_floating_dtype(inputs) - targets = training_utils.cast_if_floating_dtype(targets) - else: - inputs = [ - ops.convert_to_tensor(val, dtype=backend.floatx()) for val in inputs - ] - targets = [ - ops.convert_to_tensor(val, dtype=backend.floatx()) for val in targets - ] + if isinstance(inputs, collections.Sequence): + if len(inputs) and tensor_util.is_tensor(inputs[0]): + inputs = training_utils.cast_if_floating_dtype(inputs) + targets = training_utils.cast_if_floating_dtype(targets) + else: + inputs = [ + ops.convert_to_tensor(val, dtype=backend.floatx()) for val in inputs + ] + targets = [ + ops.convert_to_tensor(val, dtype=backend.floatx()) for val in targets + ] if sample_weights: sample_weights = [ ops.convert_to_tensor(val, dtype=backend.floatx()) if val is not None else None for val in sample_weights ] - outs, loss, loss_metrics, _, masks = _maybe_graph_function_model_loss( + outs, loss, loss_metrics, _, masks = _model_loss( model, inputs, targets, sample_weights=sample_weights, training=False) if not isinstance(outs, list): outs = [outs] @@ -741,184 +313,10 @@ def test_on_batch(model, inputs, targets, sample_weights=None): targets, sample_weights=sample_weights, masks=masks, - return_stateful_result=False) + return_stateful_result=True) loss = generic_utils.to_list(loss) return [ tensor_util.constant_value(v) for v in loss + loss_metrics + metrics_results ] - - -def fit_loop(model, - inputs, - targets, - sample_weights=None, - class_weight=None, - val_inputs=None, - val_targets=None, - val_sample_weights=None, - batch_size=None, - epochs=1, - verbose=1, - callbacks=None, - shuffle=True, - initial_epoch=0, - steps_per_epoch=None, - validation_steps=None): - """Fit function for eager execution. - - Arguments: - model: Instance of the model that is being executed in Eager mode. - inputs: List of input arrays. - targets: List of target arrays. - sample_weights: Optional list of sample weight arrays. - class_weight: Optional class-weight array to weight the importance of - samples in `inputs` based on the class they belong to, as conveyed by - `targets`. - val_inputs: Input data for validation. - val_targets: Target data for validation. - val_sample_weights: Sample weight data for validation. - batch_size: Integer batch size or None if unknown. - epochs: Number of times to iterate over the data - verbose: Verbosity mode, 0, 1 or 2 - callbacks: List of callbacks to be called during training - shuffle: Whether to shuffle the data at the beginning of each epoch - initial_epoch: Epoch at which to start training - (useful for resuming a previous training run) - steps_per_epoch: Total number of steps (batches of samples) - before declaring one epoch finished and starting the - next epoch. Ignored with the default value of `None`. - validation_steps: Number of steps to run validation for (only if doing - validation from data tensors). Ignored with default value of `None`. - - Returns: - `History` object. - - Raises: - ValueError: In case of invalid argument values. - """ - # Convert training inputs to an EagerIterator - inputs, steps_per_epoch = training_utils.convert_to_iterator( - x=inputs, - y=targets, - sample_weights=sample_weights, - batch_size=batch_size, - steps_per_epoch=steps_per_epoch, - epochs=epochs, - shuffle=shuffle) - # Required for eager execution - with backend.learning_phase_scope(1): - do_validation = val_inputs is not None - callbacks = cbks.configure_callbacks( - callbacks, - model, - do_validation=do_validation, - batch_size=batch_size, - epochs=epochs, - steps_per_epoch=steps_per_epoch, - val_inputs=val_inputs, - val_targets=val_targets, - val_sample_weights=val_sample_weights, - validation_steps=validation_steps, - verbose=verbose) - - # Create metric wrapper for the losses. - output_loss_metrics = [] - for i in range(len(model.outputs)): - loss_fn = model.loss_functions[i] - mean_wrapped_loss = metrics_module.MeanMetricWrapper( - loss_fn, name=loss_fn.__name__) - output_loss_metrics.append(mean_wrapped_loss) - - callbacks.on_train_begin() - for epoch in range(initial_epoch, epochs): - if model._is_compiled: # Model may not be compiled the first time. - # Reset stateful metrics - for m in model.stateful_metric_functions: - m.reset_states() - - for m in output_loss_metrics: - m.reset_states() - - callbacks.on_epoch_begin(epoch) - epoch_logs = {} - iterator_fit_loop( - model, - inputs, - class_weight, - steps_per_epoch=steps_per_epoch, - epoch_logs=epoch_logs, - val_inputs=val_inputs, - val_targets=val_targets, - val_sample_weights=val_sample_weights, - epochs=epochs, - verbose=verbose, - callbacks=callbacks, - validation_steps=validation_steps, - do_validation=do_validation, - batch_size=batch_size, - output_loss_metrics=output_loss_metrics) - callbacks.on_epoch_end(epoch, epoch_logs) - if callbacks.model.stop_training: - break - callbacks.on_train_end() - return model.history - - -def test_loop(model, inputs, targets, - sample_weights=None, - batch_size=None, - verbose=0, - steps=None): - """Test function for eager execution. - - Arguments: - model: Model instance that is being evaluated in Eager mode. - inputs: List of input arrays. - targets: List of target arrays. - sample_weights: Optional list of sample weight arrays. - batch_size: integer batch size or `None`. - verbose: verbosity mode. - steps: Total number of steps (batches of samples) - before declaring predictions finished. - Ignored with the default value of `None`. - - Returns: - Scalar loss (if the model has a single output and no metrics) - or list of scalars (if the model has multiple outputs - and/or metrics). The attribute `model.metrics_names` will give you - the display labels for the scalar outputs. - """ - inputs, steps = training_utils.convert_to_iterator( - x=inputs, - y=targets, - sample_weights=sample_weights, - batch_size=batch_size, - steps_per_epoch=steps, - is_validation=True) - with backend.learning_phase_scope(0): - return iterator_test_loop(model, inputs, steps, verbose=verbose) - - -def predict_loop(model, inputs, batch_size=32, verbose=0, steps=None): - """Predict function for eager execution. - - Arguments: - model: Instance of `Model`. - inputs: List of input arrays. - batch_size: integer batch size. - verbose: verbosity mode. - steps: Total number of steps (batches of samples) - before declaring `_predict_loop` finished. - Ignored with the default value of `None`. - - Returns: - Array of predictions (if the model has a single output) - or list of arrays of predictions - (if the model has multiple outputs). - """ - with backend.learning_phase_scope(0): - inputs, steps = training_utils.convert_to_iterator( - x=inputs, batch_size=batch_size, steps_per_epoch=steps) - return iterator_predict_loop(model, inputs, steps, verbose=verbose) diff --git a/tensorflow/python/keras/engine/training_eager_test.py b/tensorflow/python/keras/engine/training_eager_test.py index 76aaf1643b0..3fabbb17edc 100644 --- a/tensorflow/python/keras/engine/training_eager_test.py +++ b/tensorflow/python/keras/engine/training_eager_test.py @@ -20,10 +20,10 @@ from __future__ import print_function import numpy as np -from tensorflow.python.data.ops import dataset_ops from tensorflow.python import keras +from tensorflow.python.data.ops import dataset_ops +from tensorflow.python.eager import context from tensorflow.python.framework import ops -from tensorflow.python.framework import test_util as tf_test_util from tensorflow.python.keras import metrics as metrics_module from tensorflow.python.platform import test from tensorflow.python.training.rmsprop import RMSPropOptimizer @@ -51,6 +51,7 @@ class TrainingTest(test.TestCase): loss, metrics=metrics, loss_weights=loss_weights, + run_eagerly=True, sample_weight_mode=None) input_a = keras.backend.zeros(shape=(10, 3)) @@ -111,7 +112,7 @@ class TrainingTest(test.TestCase): optimizer = RMSPropOptimizer(learning_rate=0.001) loss = 'mse' metrics = ['mae', metrics_module.CategoricalAccuracy()] - model.compile(optimizer, loss, metrics=metrics) + model.compile(optimizer, loss, metrics=metrics, run_eagerly=True) inputs = keras.backend.zeros(shape=(10, 3)) targets = keras.backend.zeros(shape=(10, 4)) @@ -129,29 +130,34 @@ class TrainingTest(test.TestCase): x = keras.layers.Input(shape=(3,), name='input') y = keras.layers.Dense(4, name='dense')(x) model = keras.Model(x, y) - model.compile(optimizer=RMSPropOptimizer(learning_rate=0.001), loss='mse') + model.compile(optimizer=RMSPropOptimizer(learning_rate=0.001), + loss='mse', + run_eagerly=True) x = keras.backend.zeros(shape=(10, 3)) y = keras.backend.zeros(shape=(10, 4)) dataset = dataset_ops.Dataset.from_tensor_slices((x, y)).repeat(10).batch(5) - iterator = dataset.make_one_shot_iterator() + iterator = dataset_ops.make_one_shot_iterator(dataset) validation_dataset = dataset_ops.Dataset.from_tensor_slices( (x, y)).repeat(10).batch(5) - validation_iterator = validation_dataset.make_one_shot_iterator() + validation_iterator = dataset_ops.make_one_shot_iterator(validation_dataset) with self.assertRaisesRegexp( ValueError, r'specify .* `steps_per_epoch`'): model.fit(iterator, epochs=1, verbose=0) - with self.assertRaisesRegexp( - ValueError, r'provide either `batch_size` or `validation_steps`'): - model.fit(iterator, steps_per_epoch=2, epochs=1, verbose=0, - validation_data=(x, y)) - with self.assertRaisesRegexp( - ValueError, r'provide either `batch_size` or `validation_steps`'): + if not context.executing_eagerly(): + # In eager execution, `keras.backend.zeros` returns value tensors + # which can be used for validation without a `validation_steps` argument. + with self.assertRaisesRegexp( + ValueError, r'provide either `batch_size` or `validation_steps`'): + model.fit(iterator, steps_per_epoch=2, epochs=1, verbose=0, + validation_data=(x, y)) + with self.assertRaisesRegexp(ValueError, + 'specify the `validation_steps` argument.'): model.fit(iterator, steps_per_epoch=2, epochs=1, verbose=0, validation_data=validation_dataset) - with self.assertRaisesRegexp( - ValueError, r'provide either `batch_size` or `validation_steps`'): + with self.assertRaisesRegexp(ValueError, + 'specify the `validation_steps` argument.'): model.fit(iterator, steps_per_epoch=2, epochs=1, verbose=0, validation_data=validation_iterator) @@ -160,25 +166,31 @@ class TrainingTest(test.TestCase): model.add(keras.layers.Dense(4, input_shape=(3,))) optimizer = RMSPropOptimizer(learning_rate=0.001) model.compile( - optimizer, 'mse', metrics=['mae', - metrics_module.CategoricalAccuracy()]) + optimizer, + loss='mse', + metrics=['mae', metrics_module.CategoricalAccuracy()], + run_eagerly=True) x = np.random.random((10, 3)) y = np.random.random((10, 4)) - def iterator(): + def numpy_iterator(): while True: yield x, y - model.fit_generator(iterator(), steps_per_epoch=3, epochs=1) - model.evaluate_generator(iterator(), steps=3) - out = model.predict_generator(iterator(), steps=3) + model.fit_generator(numpy_iterator(), steps_per_epoch=3, epochs=1) + model.evaluate_generator(numpy_iterator(), steps=3) + + def inference_numpy_iterator(): + while True: + yield x + + out = model.predict_generator(inference_numpy_iterator(), steps=3) self.assertEqual(out.shape, (30, 4)) class CorrectnessTest(test.TestCase): - @tf_test_util.run_in_graph_and_eager_modes def test_loss_correctness(self): # Test that training loss is the same in eager and graph # (by comparing it to a reference value in a deterministic case) @@ -191,14 +203,14 @@ class CorrectnessTest(test.TestCase): activation='softmax', kernel_initializer='ones')) model.compile(loss='sparse_categorical_crossentropy', - optimizer=RMSPropOptimizer(learning_rate=0.001)) + optimizer=RMSPropOptimizer(learning_rate=0.001), + run_eagerly=False) x = np.ones((100, 4)) np.random.seed(123) y = np.random.randint(0, 1, size=(100, 1)) history = model.fit(x, y, epochs=1, batch_size=10) self.assertAlmostEqual(history.history['loss'][-1], 0.6173, 4) - @tf_test_util.run_in_graph_and_eager_modes def test_loss_correctness_with_iterator(self): # Test that training loss is the same in eager and graph # (by comparing it to a reference value in a deterministic case) @@ -210,14 +222,15 @@ class CorrectnessTest(test.TestCase): keras.layers.Dense(2, activation='softmax', kernel_initializer='ones')) model.compile( loss='sparse_categorical_crossentropy', - optimizer=RMSPropOptimizer(learning_rate=0.001)) + optimizer=RMSPropOptimizer(learning_rate=0.001), + run_eagerly=True) x = np.ones((100, 4), dtype=np.float32) np.random.seed(123) y = np.random.randint(0, 1, size=(100, 1)) dataset = dataset_ops.Dataset.from_tensor_slices((x, y)) dataset = dataset.repeat(100) dataset = dataset.batch(10) - iterator = dataset.make_one_shot_iterator() + iterator = dataset_ops.make_one_shot_iterator(dataset) history = model.fit(iterator, epochs=1, steps_per_epoch=10) self.assertAlmostEqual(history.history['loss'][-1], 0.6173, 4) diff --git a/tensorflow/python/keras/engine/training_generator.py b/tensorflow/python/keras/engine/training_generator.py index b5e3a039767..0abf0b82709 100644 --- a/tensorflow/python/keras/engine/training_generator.py +++ b/tensorflow/python/keras/engine/training_generator.py @@ -19,412 +19,433 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function +import functools +import math + import numpy as np +from tensorflow.python.data.ops import dataset_ops +from tensorflow.python.data.ops import iterator_ops from tensorflow.python.eager import context +from tensorflow.python.framework import errors +from tensorflow.python.keras import backend from tensorflow.python.keras import callbacks as cbks -from tensorflow.python.keras.utils.data_utils import GeneratorEnqueuer -from tensorflow.python.keras.utils.data_utils import iter_sequence_infinite -from tensorflow.python.keras.utils.data_utils import OrderedEnqueuer -from tensorflow.python.keras.utils.data_utils import Sequence -from tensorflow.python.keras.utils.generic_utils import Progbar +from tensorflow.python.keras.engine import training_utils +from tensorflow.python.keras.utils import data_utils +from tensorflow.python.keras.utils import generic_utils from tensorflow.python.platform import tf_logging as logging +from tensorflow.python.util import nest -def fit_generator(model, - generator, - steps_per_epoch=None, - epochs=1, - verbose=1, - callbacks=None, - validation_data=None, - validation_steps=None, - class_weight=None, - max_queue_size=10, - workers=1, - use_multiprocessing=False, - shuffle=True, - initial_epoch=0): - """See docstring for `Model.fit_generator`.""" - epoch = initial_epoch +def model_iteration(model, + data, + steps_per_epoch=None, + epochs=1, + verbose=1, + callbacks=None, + validation_data=None, + validation_steps=None, + class_weight=None, + max_queue_size=10, + workers=1, + use_multiprocessing=False, + shuffle=True, + initial_epoch=0, + mode='train', + batch_size=None, + **kwargs): + """Loop function for arrays of data with modes 'train'/'test'/'predict'. - do_validation = bool(validation_data) - if not context.executing_eagerly(): - model._make_train_function() - if do_validation: - model._make_test_function() + Arguments: + model: Keras Model instance. + data: Either a tuple of NumPy/Tensor inputs (i.e. `(x,)` or `(x, y)` or + `(x, y, sample_weights)`) or a generator or + `keras.utils.data_utils.Sequence` object or Eager Iterator or Dataset. + steps_per_epoch: Total number of steps (batches of samples) before + declaring one epoch finished and starting the next epoch. Ignored with + the default value of `None`. + epochs: Number of times to iterate over the data. + verbose: Verbosity mode, 0, 1 or 2. + callbacks: List of callbacks to be called during training. + validation_data: Either a tuple of NumPy/Tensor inputs (i.e. `(x,)` or + `(x, y)` or `(x, y, sample_weights)`) or a generator or + `keras.utils.data_utils.Sequence` object or Eager Iterator or Dataset. + validation_steps: Total number of steps (batches of samples) before + declaring validation finished. + class_weight: Dictionary mapping class indices to a weight for the class. + max_queue_size: Integer. Maximum size for the generator queue. If + unspecified, `max_queue_size` will default to 10. + workers: Integer. Maximum number of processes to spin up when using + process-based threading. If unspecified, `workers` will default to 1. If + 0, will execute the generator on the main thread. + use_multiprocessing: Boolean. If `True`, use process-based threading. If + unspecified, `use_multiprocessing` will default to `False`. Note that + because this implementation relies on multiprocessing, you should not + pass non-picklable arguments to the generator as they can't be passed + easily to children processes. + shuffle: Boolean. Whether to shuffle the order of the batches at the + beginning of each epoch. Only used with instances of `Sequence` + (`keras.utils.Sequence`). Has no effect when `steps_per_epoch` is not + `None`. + initial_epoch: Epoch at which to start training (useful for resuming a + previous training run). + mode: One of 'train'/'test'/'predict'. + batch_size: Integer batch size or None if unknown. Will only be used if + `data` is in NumPy/Tensor format. + **kwargs: Additional arguments for backwards compatibility. `steps` is + accepted as an alias for `steps_per_epoch`. - is_sequence = isinstance(generator, Sequence) - if not is_sequence and use_multiprocessing and workers > 1: - logging.warning( - UserWarning('Using a generator with `use_multiprocessing=True`' - ' and multiple workers may duplicate your data.' - ' Please consider using the`keras.utils.Sequence' - ' class.')) - if steps_per_epoch is None: - if is_sequence: - steps_per_epoch = len(generator) - else: - raise ValueError('`steps_per_epoch=None` is only valid for a' - ' generator based on the `keras.utils.Sequence`' - ' class. Please specify `steps_per_epoch` or use' - ' the `keras.utils.Sequence` class.') + Returns: + - In 'train' mode: `History` object. + - In 'test' mode: Evaluation metrics. + - In 'predict' mode: Outputs of the Model called on inputs. - # python 2 has 'next', 3 has '__next__' - # avoid any explicit version checks - val_gen = ( - hasattr(validation_data, 'next') or - hasattr(validation_data, '__next__') or - isinstance(validation_data, Sequence)) - if (val_gen and not isinstance(validation_data, Sequence) and - not validation_steps): - raise ValueError('`validation_steps=None` is only valid for a' - ' generator based on the `keras.utils.Sequence`' - ' class. Please specify `validation_steps` or use' - ' the `keras.utils.Sequence` class.') + Raises: + ValueError: in case of invalid arguments. + """ + if 'steps' in kwargs: + steps_per_epoch = kwargs['steps'] - enqueuer = None - val_enqueuer = None + # Convert to a format that supports `next(generator)`. + generator, steps_per_epoch = convert_to_generator_like( + data, + steps_per_epoch=steps_per_epoch, + batch_size=batch_size, + epochs=epochs - initial_epoch, + shuffle=shuffle) - try: - val_x, val_y, val_sample_weights = validation_data, None, None - if do_validation and not val_gen: - # Prepare data for validation - if len(validation_data) == 2: - val_x, val_y = validation_data # pylint: disable=unpacking-non-sequence - val_sample_weights = None - elif len(validation_data) == 3: - val_x, val_y, val_sample_weights = validation_data # pylint: disable=unpacking-non-sequence - else: - raise ValueError( - '`validation_data` should be a tuple ' - '`(val_x, val_y, val_sample_weight)` ' - 'or `(val_x, val_y)`. Found: ' + str(validation_data)) - val_x, val_y, val_sample_weights = model._standardize_user_data( - val_x, val_y, val_sample_weights) + do_validation = validation_data is not None + should_set_learning_phase = context.executing_eagerly() and model.run_eagerly + is_sequence = isinstance(generator, data_utils.Sequence) + _validate_arguments(is_sequence, use_multiprocessing, workers, + steps_per_epoch, validation_data, validation_steps, mode, + kwargs) - callbacks = cbks.configure_callbacks( - callbacks, - model, - do_validation=do_validation, - val_inputs=val_x, - val_targets=val_y, - val_sample_weights=val_sample_weights, - epochs=epochs, - validation_steps=validation_steps, - steps_per_epoch=steps_per_epoch, - verbose=verbose) + batch_function = _make_execution_function( + model, mode, class_weight=class_weight) - if workers > 0: - if is_sequence: - enqueuer = OrderedEnqueuer( - generator, - use_multiprocessing=use_multiprocessing, - shuffle=shuffle) - else: - enqueuer = GeneratorEnqueuer( - generator, - use_multiprocessing=use_multiprocessing) - enqueuer.start(workers=workers, max_queue_size=max_queue_size) - output_generator = enqueuer.get() - else: - if is_sequence: - output_generator = iter_sequence_infinite(generator) - else: - output_generator = generator + # Create the queue for the generator. + output_generator, enqueuer = _make_enqueued_generator( + generator, + workers=workers, + use_multiprocessing=use_multiprocessing, + max_queue_size=max_queue_size, + shuffle=shuffle) - callbacks.on_train_begin() - # Construct epoch logs. + num_samples_or_steps, use_steps = _get_num_samples_or_steps( + data, steps_per_epoch) + + count_mode = 'steps' if use_steps else 'samples' + callbacks = cbks.configure_callbacks( + callbacks, + model, + do_validation=do_validation, + epochs=epochs, + steps_per_epoch=steps_per_epoch, + batch_size=batch_size, + samples=num_samples_or_steps, + verbose=0, # Handle ProgBar as part of Callbacks once hooks are ready. + mode=mode) + # TODO(omalleyt): Handle ProgBar as part of Callbacks once hooks are ready. + progbar = training_utils.get_progbar(model, count_mode) + progbar.params = callbacks.params + progbar.params['verbose'] = verbose + + if mode == 'predict': + aggregator = training_utils.OutputsAggregator(True, steps_per_epoch) + else: + aggregator = training_utils.MetricsAggregator(True, steps_per_epoch) + + if should_set_learning_phase: + old_learning_phase = backend.learning_phase() + backend.set_learning_phase(1 if mode == 'train' else 0) + + callbacks.model.stop_training = False + callbacks._call_begin_hook(mode) + progbar.on_train_begin() + for epoch in range(initial_epoch, epochs): + if callbacks.model.stop_training: + break + + # Setup work for each epoch. + model.reset_metrics() epoch_logs = {} - while epoch < epochs: - for m in model.stateful_metric_functions: - m.reset_states() - callbacks.on_epoch_begin(epoch) - steps_done = 0 - batch_index = 0 - while steps_done < steps_per_epoch: - generator_output = next(output_generator) + callbacks.on_epoch_begin(epoch, epoch_logs, mode=mode) + progbar.on_epoch_begin(epoch, epoch_logs) - if not hasattr(generator_output, '__len__'): - raise ValueError('Output of generator should be ' - 'a tuple `(x, y, sample_weight)` ' - 'or `(x, y)`. Found: ' + str(generator_output)) + for step in range(steps_per_epoch): + batch_data = _get_next_batch(output_generator, mode) + if batch_data is None: + callbacks.model.stop_training = True + break - if len(generator_output) == 2: - x, y = generator_output - sample_weight = None - elif len(generator_output) == 3: - x, y, sample_weight = generator_output - else: - raise ValueError('Output of generator should be ' - 'a tuple `(x, y, sample_weight)` ' - 'or `(x, y)`. Found: ' + str(generator_output)) - # build batch logs - batch_logs = {} - if isinstance(x, list): - batch_size = x[0].shape[0] - elif isinstance(x, dict): - batch_size = list(x.values())[0].shape[0] - else: - batch_size = x.shape[0] - batch_logs['batch'] = batch_index - batch_logs['size'] = batch_size - callbacks.on_batch_begin(batch_index, batch_logs) + # `batch_size` used for validation data if validation + # data is NumPy/EagerTensors. + batch_size = int(nest.flatten(batch_data)[0].shape[0]) - outs = model.train_on_batch( - x, y, sample_weight=sample_weight, class_weight=class_weight) + # Callbacks batch begin. + batch_logs = {'batch': step, 'size': batch_size} + callbacks._call_batch_hook(mode, 'begin', step, batch_logs) + progbar.on_batch_begin(step, batch_logs) - if not isinstance(outs, list): - outs = [outs] - for l, o in zip(model.metrics_names, outs): - batch_logs[l] = o + batch_outs = batch_function(*batch_data) + if not isinstance(batch_outs, list): + batch_outs = [batch_outs] - callbacks.on_batch_end(batch_index, batch_logs) + # Aggregate results. + if step == 0: + aggregator.create(batch_outs) + aggregator.aggregate(batch_outs) - batch_index += 1 - steps_done += 1 + # Callbacks batch end. + batch_logs.update(training_utils.make_logs(model, batch_outs, mode)) + callbacks._call_batch_hook(mode, 'end', step, batch_logs) + progbar.on_batch_end(step, batch_logs) - # Epoch finished. - if steps_done >= steps_per_epoch and do_validation: - if val_gen: - val_outs = evaluate_generator( - model, - validation_data, - validation_steps, - workers=workers, - use_multiprocessing=use_multiprocessing, - max_queue_size=max_queue_size) - else: - # No need for try/except because - # data has already been validated. - val_outs = model.evaluate( - val_x, - val_y, - batch_size=batch_size, - sample_weight=val_sample_weights, - verbose=0) - if not isinstance(val_outs, list): - val_outs = [val_outs] - # Same labels assumed. - for l, o in zip(model.metrics_names, val_outs): - epoch_logs['val_' + l] = o - - if callbacks.model.stop_training: - break - - callbacks.on_epoch_end(epoch, epoch_logs) - epoch += 1 if callbacks.model.stop_training: break - finally: - try: - if enqueuer is not None: - enqueuer.stop() - finally: - if val_enqueuer is not None: - val_enqueuer.stop() + aggregator.finalize() + results = aggregator.results + epoch_logs.update(training_utils.make_logs(model, results, mode)) + if len(results) == 1: + results = results[0] - callbacks.on_train_end() - return model.history + # Run the test loop every epoch during training. + if do_validation and not callbacks.model.stop_training: + val_results = model_iteration( + model, + validation_data, + steps_per_epoch=validation_steps, + batch_size=batch_size, + class_weight=class_weight, + workers=workers, + use_multiprocessing=use_multiprocessing, + max_queue_size=max_queue_size, + mode='test') + + if not isinstance(val_results, list): + val_results = [val_results] + epoch_logs.update( + training_utils.make_logs(model, val_results, mode, prefix='val_')) + + callbacks.on_epoch_end(epoch, epoch_logs, mode=mode) + progbar.on_epoch_end(epoch, epoch_logs) + callbacks._call_end_hook(mode) + + if enqueuer is not None: + enqueuer.stop() + + if should_set_learning_phase: + backend.set_learning_phase(old_learning_phase) + + if mode == 'train': + return model.history + return results -def evaluate_generator(model, - generator, - steps=None, - max_queue_size=10, - workers=1, - use_multiprocessing=False, - verbose=0): - """See docstring for `Model.evaluate_generator`.""" - if not context.executing_eagerly(): - model._make_test_function() +# Maintain compatibility with the existing names. +fit_generator = functools.partial(model_iteration, mode='train') +evaluate_generator = functools.partial(model_iteration, mode='test') +predict_generator = functools.partial(model_iteration, mode='predict') - if hasattr(model, 'metrics'): - for m in model.stateful_metric_functions: - m.reset_states() - steps_done = 0 - all_outs = [] - batch_sizes = [] - is_sequence = isinstance(generator, Sequence) +def _get_next_batch(output_generator, mode): + """Retrieves the next batch of input data.""" + try: + generator_output = next(output_generator) + except (errors.OutOfRangeError, StopIteration): + # Returning `None` will trigger looping to stop. + logging.warning('Your dataset iterator ran out of data.') + return None + if not isinstance(generator_output, tuple): + if mode == 'predict': + # Always wrap in a tuple. + return (generator_output,) + else: + raise ValueError('Output of generator should be ' + 'a tuple `(x, y, sample_weight)` ' + 'or `(x, y)`. Found: ' + str(generator_output)) + + if len(generator_output) < 1 or len(generator_output) > 3: + raise ValueError('Output of generator should be ' + 'a tuple `(x, y, sample_weight)` ' + 'or `(x, y)` or (x,). Found: ' + str(generator_output)) + return generator_output + + +def _validate_arguments(is_sequence, use_multiprocessing, workers, + steps_per_epoch, validation_data, validation_steps, + mode, kwargs): + """Raises errors if arguments are invalid. + + Arguments: + is_sequence: Boolean, whether data is a `keras.utils.data_utils.Sequence` + instance. + use_multiprocessing: Boolean. If `True`, use process-based threading. If + unspecified, `use_multiprocessing` will default to `False`. Note that + because this implementation relies on multiprocessing, you should not pass + non-picklable arguments to the generator as they can't be passed easily to + children processes. + workers: Integer. Maximum number of processes to spin up when using + process-based threading. If unspecified, `workers` will default to 1. If + 0, will execute the generator on the main thread. + steps_per_epoch: Total number of steps (batches of samples) before declaring + one epoch finished and starting the next epoch. Ignored with the default + value of `None`. + validation_data: Either a tuple of NumPy/Tensor inputs (i.e. `(x,)` or `(x, + y)` or `(x, y, sample_weights)`) or a generator or + `keras.utils.data_utils.Sequence` object or Eager Iterator or Dataset. + validation_steps: Total number of steps (batches of samples) before + declaring validation finished. + mode: One of 'train'/'test'/'predict'. + kwargs: Additional arguments for backwards compatibility. + + Raises: + ValueError: If `steps_per_epoch` or `validation_steps` are not passed + for data types that require them, or if unrecognized keyword + arguments are passed. + """ if not is_sequence and use_multiprocessing and workers > 1: logging.warning( UserWarning('Using a generator with `use_multiprocessing=True`' ' and multiple workers may duplicate your data.' - ' Please consider using the`keras.utils.Sequence' + ' Please consider using the `keras.utils.Sequence`' ' class.')) - if steps is None: - if is_sequence: - steps = len(generator) - else: - raise ValueError('`steps=None` is only valid for a generator' - ' based on the `keras.utils.Sequence` class.' - ' Please specify `steps` or use the' - ' `keras.utils.Sequence` class.') + + if steps_per_epoch is None: + arg_name = 'steps_per_epoch' if mode == 'train' else 'steps' + raise ValueError('Please specify the number of steps via the ' + '`{}` argument.'.format(arg_name)) + + val_gen = ( + data_utils.is_generator_or_sequence(validation_data) or + isinstance(validation_data, iterator_ops.EagerIterator) or + isinstance(validation_data, dataset_ops.DatasetV2)) + if (val_gen and not isinstance(validation_data, data_utils.Sequence) and + not validation_steps): + raise ValueError('Please specify the `validation_steps` argument.') + + if any(k != 'steps' for k in kwargs): + raise ValueError('Invalid arguments passed: {}'.format( + [k for k in kwargs if k != 'steps'])) + + +def convert_to_generator_like(data, + batch_size=None, + steps_per_epoch=None, + epochs=1, + shuffle=False): + """Make a generator out of NumPy or EagerTensor inputs. + + Arguments: + data: Either a generator or `keras.utils.data_utils.Sequence` object or + `Dataset` or `EagerIterator` or a {1,2,3}-tuple of NumPy arrays or + EagerTensors. If a tuple, the elements represent `(x, y, sample_weights)` + and may be `None` or `[None]`. + batch_size: Used when creating a generator out of tuples of NumPy arrays or + EagerTensors. + steps_per_epoch: Steps of the generator to run each epoch. + epochs: Total number of epochs to run. + shuffle: Whether the data should be shuffled. + + Returns: + - Generator or `keras.utils.data_utils.Sequence` or EagerIterator. + + Raises: + - ValueError: If `batch_size` is not provided for NumPy or EagerTensor + inputs. + """ + if isinstance(data, tuple): + # Scrub `Nones` that might have been passed for `targets`, `sample_weights`. + data = tuple( + ele for ele in data if not all(e is None for e in nest.flatten(ele))) + if len(data) == 1: + data = data[0] + + if data_utils.is_generator_or_sequence(data) or isinstance( + data, iterator_ops.EagerIterator): + if isinstance(data, data_utils.Sequence): + steps_per_epoch = len(data) + return data, steps_per_epoch + if isinstance(data, dataset_ops.DatasetV2): + return dataset_ops.make_one_shot_iterator(data), steps_per_epoch + + # Create generator from NumPy or EagerTensor Input. + num_samples = int(nest.flatten(data)[0].shape[0]) + if batch_size is None: + raise ValueError('You must specify `batch_size`') + steps_per_epoch = int(math.ceil(num_samples / batch_size)) + + def _gen(data): + """Makes a generator out of a structure of NumPy/EagerTensors.""" + index_array = np.arange(num_samples) + for _ in range(epochs): + if shuffle: + np.random.shuffle(index_array) + batches = generic_utils.make_batches(num_samples, batch_size) + for (batch_start, batch_end) in batches: + batch_ids = index_array[batch_start:batch_end] + flat_batch_data = training_utils.slice_arrays( + nest.flatten(data), batch_ids, contiguous=(not shuffle)) + yield nest.pack_sequence_as(data, flat_batch_data) + + return _gen(data), steps_per_epoch + + +def _make_enqueued_generator(generator, + workers=1, + use_multiprocessing=False, + max_queue_size=10, + shuffle=False): + """Create a buffered queue of next elements of the generator.""" + is_sequence = isinstance(generator, data_utils.Sequence) enqueuer = None - - try: - if workers > 0: - if is_sequence: - enqueuer = OrderedEnqueuer( - generator, use_multiprocessing=use_multiprocessing) - else: - enqueuer = GeneratorEnqueuer( - generator, - use_multiprocessing=use_multiprocessing) - enqueuer.start(workers=workers, max_queue_size=max_queue_size) - output_generator = enqueuer.get() - else: - if is_sequence: - output_generator = iter_sequence_infinite(generator) - else: - output_generator = generator - - if verbose == 1: - progbar = Progbar(target=steps) - - while steps_done < steps: - generator_output = next(output_generator) - if not hasattr(generator_output, '__len__'): - raise ValueError('Output of generator should be a tuple ' - '(x, y, sample_weight) ' - 'or (x, y). Found: ' + str(generator_output)) - if len(generator_output) == 2: - x, y = generator_output - sample_weight = None - elif len(generator_output) == 3: - x, y, sample_weight = generator_output - else: - raise ValueError('Output of generator should be a tuple ' - '(x, y, sample_weight) ' - 'or (x, y). Found: ' + str(generator_output)) - outs = model.test_on_batch(x, y, sample_weight=sample_weight) - - if isinstance(x, list): - batch_size = x[0].shape[0] - elif isinstance(x, dict): - batch_size = list(x.values())[0].shape[0] - else: - batch_size = x.shape[0] - if batch_size == 0: - raise ValueError('Received an empty batch. ' - 'Batches should at least contain one item.') - all_outs.append(outs) - - steps_done += 1 - batch_sizes.append(batch_size) - if verbose == 1: - progbar.update(steps_done) - - finally: - if enqueuer is not None: - enqueuer.stop() - - if not isinstance(outs, list): - return np.average(np.asarray(all_outs), weights=batch_sizes) - else: - averages = [float(all_outs[-1][0])] # index 0 = 'loss' - averages.extend([ - np.average([out[i] - for out in all_outs], weights=batch_sizes) - for i in range(1, len(outs)) - ]) - return averages - - -def predict_generator(model, - generator, - steps=None, - max_queue_size=10, - workers=1, - use_multiprocessing=False, - verbose=0): - """See docstring for `Model.predict_generator`.""" - if not context.executing_eagerly(): - model._make_predict_function() - - steps_done = 0 - all_outs = [] - is_sequence = isinstance(generator, Sequence) - if not is_sequence and use_multiprocessing and workers > 1: - logging.warning( - UserWarning('Using a generator with `use_multiprocessing=True`' - ' and multiple workers may duplicate your data.' - ' Please consider using the`keras.utils.Sequence' - ' class.')) - if steps is None: + if workers > 0: if is_sequence: - steps = len(generator) + enqueuer = data_utils.OrderedEnqueuer( + generator, use_multiprocessing=use_multiprocessing, shuffle=shuffle) else: - raise ValueError('`steps=None` is only valid for a generator' - ' based on the `keras.utils.Sequence` class.' - ' Please specify `steps` or use the' - ' `keras.utils.Sequence` class.') - enqueuer = None - - try: - if workers > 0: - if is_sequence: - enqueuer = OrderedEnqueuer( - generator, use_multiprocessing=use_multiprocessing) - else: - enqueuer = GeneratorEnqueuer( - generator, - use_multiprocessing=use_multiprocessing) - enqueuer.start(workers=workers, max_queue_size=max_queue_size) - output_generator = enqueuer.get() - else: - if is_sequence: - output_generator = iter_sequence_infinite(generator) - else: - output_generator = generator - - if verbose == 1: - progbar = Progbar(target=steps) - - while steps_done < steps: - generator_output = next(output_generator) - if isinstance(generator_output, tuple): - # Compatibility with the generators - # used for training. - if len(generator_output) == 2: - x, _ = generator_output - elif len(generator_output) == 3: - x, _, _ = generator_output - else: - raise ValueError('Output of generator should be ' - 'a tuple `(x, y, sample_weight)` ' - 'or `(x, y)`. Found: ' + str(generator_output)) - else: - # Assumes a generator that only - # yields inputs (not targets and sample weights). - x = generator_output - - outs = model.predict_on_batch(x) - if not isinstance(outs, list): - outs = [outs] - - if not all_outs: - for out in outs: - all_outs.append([]) - - for i, out in enumerate(outs): - all_outs[i].append(out) - steps_done += 1 - if verbose == 1: - progbar.update(steps_done) - - finally: - if enqueuer is not None: - enqueuer.stop() - - if len(all_outs) == 1: - if steps_done == 1: - return all_outs[0][0] - else: - return np.concatenate(all_outs[0]) - if steps_done == 1: - return [out[0] for out in all_outs] + enqueuer = data_utils.GeneratorEnqueuer( + generator, use_multiprocessing=use_multiprocessing) + enqueuer.start(workers=workers, max_queue_size=max_queue_size) + output_generator = enqueuer.get() else: - return [np.concatenate(out) for out in all_outs] + if is_sequence: + output_generator = data_utils.iter_sequence_infinite(generator) + else: + output_generator = generator + return output_generator, enqueuer + + +def _make_execution_function(model, mode, class_weight=None): + """Makes function to run one step of model execution.""" + if mode == 'train': + if not context.executing_eagerly(): + model._make_fit_function() + f = functools.partial(model.train_on_batch, class_weight=class_weight) + elif mode == 'test': + if not context.executing_eagerly(): + model._make_eval_function() + f = model.test_on_batch + else: + # Match signature of other modes to allow + # 1, 2, or 3-tuples from generator + def predict_on_batch(x, y=None, sample_weights=None): # pylint: disable=unused-argument + return model.predict_on_batch(x) + + f = predict_on_batch + + # Maintain stateful metrics across batch-level calls. + if mode != 'predict': + f = functools.partial(f, reset_metrics=False) + + return f + + +def _get_num_samples_or_steps(data, steps_per_epoch): + """Returns number of samples or steps, and whether to use steps count mode.""" + flat_inputs = nest.flatten(data) + if hasattr(flat_inputs[0], 'shape'): + return int(flat_inputs[0].shape[0]), False + return steps_per_epoch, True diff --git a/tensorflow/python/keras/engine/training_generator_test.py b/tensorflow/python/keras/engine/training_generator_test.py index 88e89434242..8941428e43a 100644 --- a/tensorflow/python/keras/engine/training_generator_test.py +++ b/tensorflow/python/keras/engine/training_generator_test.py @@ -21,220 +21,274 @@ from __future__ import print_function import os import unittest +from absl.testing import parameterized import numpy as np from tensorflow.python import keras +from tensorflow.python.data.ops import dataset_ops +from tensorflow.python.data.ops import iterator_ops +from tensorflow.python.eager import context from tensorflow.python.framework import test_util as tf_test_util from tensorflow.python.keras import metrics as metrics_module +from tensorflow.python.keras import testing_utils +from tensorflow.python.keras.engine import training_generator from tensorflow.python.platform import test from tensorflow.python.training.rmsprop import RMSPropOptimizer +from tensorflow.python.util import nest -class TestGeneratorMethods(test.TestCase): +def custom_generator(mode=2): + batch_size = 10 + num_samples = 50 + arr_data = np.random.random((num_samples, 2)) + arr_labels = np.random.random((num_samples, 4)) + arr_weights = np.random.random((num_samples,)) + i = 0 + while True: + batch_index = i * batch_size % num_samples + i += 1 + start = batch_index + end = start + batch_size + x = arr_data[start: end] + y = arr_labels[start: end] + w = arr_weights[start: end] + if mode == 1: + yield x + elif mode == 2: + yield x, y + else: + yield x, y, w + + +@tf_test_util.run_all_in_graph_and_eager_modes +class TestGeneratorMethods(test.TestCase, parameterized.TestCase): @unittest.skipIf( os.name == 'nt', 'use_multiprocessing=True does not work on windows properly.') - def test_generator_methods(self): - arr_data = np.random.random((50, 2)) - arr_labels = np.random.random((50,)) + @parameterized.parameters('sequential', 'functional') + def test_fit_generator_method(self, model_type): + if model_type == 'sequential': + model = testing_utils.get_small_sequential_mlp( + num_hidden=3, num_classes=4, input_dim=2) + else: + model = testing_utils.get_small_functional_mlp( + num_hidden=3, num_classes=4, input_dim=2) + model.compile( + loss='mse', + optimizer='sgd', + metrics=['mae', metrics_module.CategoricalAccuracy()]) - def custom_generator(): - batch_size = 10 - num_samples = 50 - while True: - batch_index = np.random.randint(0, num_samples - batch_size) - start = batch_index - end = start + batch_size - x = arr_data[start: end] - y = arr_labels[start: end] - yield x, y + model.fit_generator(custom_generator(), + steps_per_epoch=5, + epochs=1, + verbose=1, + max_queue_size=10, + workers=4, + use_multiprocessing=True) + model.fit_generator(custom_generator(), + steps_per_epoch=5, + epochs=1, + verbose=1, + max_queue_size=10, + use_multiprocessing=False) + model.fit_generator(custom_generator(), + steps_per_epoch=5, + epochs=1, + verbose=1, + max_queue_size=10, + use_multiprocessing=False, + validation_data=custom_generator(), + validation_steps=10) + model.fit_generator(custom_generator(), + steps_per_epoch=5, + validation_data=custom_generator(), + validation_steps=1, + workers=0) - with self.cached_session(): - x = keras.Input((2,)) - y = keras.layers.Dense(1)(x) - fn_model = keras.models.Model(x, y) - fn_model.compile( - loss='mse', - optimizer='sgd', - metrics=['mae', metrics_module.CategoricalAccuracy()]) + @unittest.skipIf( + os.name == 'nt', + 'use_multiprocessing=True does not work on windows properly.') + @parameterized.parameters('sequential', 'functional') + def test_evaluate_generator_method(self, model_type): + if model_type == 'sequential': + model = testing_utils.get_small_sequential_mlp( + num_hidden=3, num_classes=4, input_dim=2) + else: + model = testing_utils.get_small_functional_mlp( + num_hidden=3, num_classes=4, input_dim=2) + model.compile( + loss='mse', + optimizer='sgd', + metrics=['mae', metrics_module.CategoricalAccuracy()]) + model.summary() - seq_model = keras.models.Sequential() - seq_model.add(keras.layers.Dense(1, input_shape=(2,))) - seq_model.compile(loss='mse', optimizer='sgd') + model.evaluate_generator(custom_generator(), + steps=5, + max_queue_size=10, + workers=2, + verbose=1, + use_multiprocessing=True) + model.evaluate_generator(custom_generator(), + steps=5, + max_queue_size=10, + use_multiprocessing=False) + model.evaluate_generator(custom_generator(), + steps=5, + max_queue_size=10, + use_multiprocessing=False, + workers=0) - for model in [fn_model, seq_model]: - model.fit_generator(custom_generator(), - steps_per_epoch=5, - epochs=1, - verbose=1, + @unittest.skipIf( + os.name == 'nt', + 'use_multiprocessing=True does not work on windows properly.') + @parameterized.parameters('sequential', 'functional') + def test_predict_generator_method(self, model_type): + if model_type == 'sequential': + model = testing_utils.get_small_sequential_mlp( + num_hidden=3, num_classes=4, input_dim=2) + else: + model = testing_utils.get_small_functional_mlp( + num_hidden=3, num_classes=4, input_dim=2) + model.compile( + loss='mse', + optimizer='sgd', + metrics=['mae', metrics_module.CategoricalAccuracy()]) + + model.predict_generator(custom_generator(), + steps=5, max_queue_size=10, - workers=4, + workers=2, use_multiprocessing=True) - model.fit_generator(custom_generator(), - steps_per_epoch=5, - epochs=1, - verbose=1, + model.predict_generator(custom_generator(), + steps=5, max_queue_size=10, use_multiprocessing=False) - model.fit_generator(custom_generator(), - steps_per_epoch=5, - epochs=1, - verbose=1, + model.predict_generator(custom_generator(), + steps=5, + max_queue_size=10, + workers=0) + # Test generator with just inputs (no targets) + model.predict_generator(custom_generator(mode=1), + steps=5, + max_queue_size=10, + workers=2, + use_multiprocessing=True) + model.predict_generator(custom_generator(mode=1), + steps=5, + max_queue_size=10, + use_multiprocessing=False) + model.predict_generator(custom_generator(mode=1), + steps=5, max_queue_size=10, - use_multiprocessing=False, - validation_data=custom_generator(), - validation_steps=10) - model.fit_generator(custom_generator(), - steps_per_epoch=5, - validation_data=custom_generator(), - validation_steps=1, workers=0) - model.predict_generator(custom_generator(), - steps=5, - max_queue_size=10, - workers=2, - use_multiprocessing=True) - model.predict_generator(custom_generator(), - steps=5, - max_queue_size=10, - use_multiprocessing=False) - model.predict_generator(custom_generator(), - steps=5, - max_queue_size=10, - workers=0) - model.evaluate_generator(custom_generator(), - steps=5, - max_queue_size=10, - workers=2, - verbose=1, - use_multiprocessing=True) - model.evaluate_generator(custom_generator(), - steps=5, - max_queue_size=10, - use_multiprocessing=False) - model.evaluate_generator(custom_generator(), - steps=5, - max_queue_size=10, - use_multiprocessing=False, - workers=0) def test_generator_methods_with_sample_weights(self): - arr_data = np.random.random((50, 2)) - arr_labels = np.random.random((50,)) - arr_sample_weights = np.random.random((50,)) + model = keras.models.Sequential() + model.add(keras.layers.Dense(4, input_shape=(2,))) + model.compile( + loss='mse', + optimizer='sgd', + metrics=['mae', metrics_module.CategoricalAccuracy()]) - def custom_generator(): - batch_size = 10 - num_samples = 50 - while True: - batch_index = np.random.randint(0, num_samples - batch_size) - start = batch_index - end = start + batch_size - x = arr_data[start: end] - y = arr_labels[start: end] - w = arr_sample_weights[start: end] - yield x, y, w + model.fit_generator(custom_generator(mode=3), + steps_per_epoch=5, + epochs=1, + verbose=1, + max_queue_size=10, + use_multiprocessing=False) + model.fit_generator(custom_generator(mode=3), + steps_per_epoch=5, + epochs=1, + verbose=1, + max_queue_size=10, + use_multiprocessing=False, + validation_data=custom_generator(mode=3), + validation_steps=10) + model.predict_generator(custom_generator(mode=3), + steps=5, + max_queue_size=10, + use_multiprocessing=False) + model.evaluate_generator(custom_generator(mode=3), + steps=5, + max_queue_size=10, + use_multiprocessing=False) - with self.cached_session(): - model = keras.models.Sequential() - model.add(keras.layers.Dense(1, input_shape=(2,))) - model.compile( - loss='mse', - optimizer='sgd', - metrics=['mae', metrics_module.CategoricalAccuracy()]) + def test_generator_methods_invalid_use_case(self): - model.fit_generator(custom_generator(), + def invalid_generator(): + while 1: + yield 0 + + model = keras.models.Sequential() + model.add(keras.layers.Dense(4, input_shape=(2,))) + model.compile(loss='mse', optimizer='sgd') + + with self.assertRaises(ValueError): + model.fit_generator(invalid_generator(), steps_per_epoch=5, epochs=1, verbose=1, max_queue_size=10, use_multiprocessing=False) + with self.assertRaises(ValueError): model.fit_generator(custom_generator(), steps_per_epoch=5, epochs=1, verbose=1, max_queue_size=10, use_multiprocessing=False, - validation_data=custom_generator(), + validation_data=invalid_generator(), validation_steps=10) - model.predict_generator(custom_generator(), + with self.assertRaises(AttributeError): + model.predict_generator(invalid_generator(), steps=5, max_queue_size=10, use_multiprocessing=False) - model.evaluate_generator(custom_generator(), + with self.assertRaises(ValueError): + model.evaluate_generator(invalid_generator(), steps=5, max_queue_size=10, use_multiprocessing=False) - def test_generator_methods_invalid_use_case(self): + def test_generator_input_to_fit_eval_predict(self): + val_data = np.ones([10, 10], np.float32), np.ones([10, 1], np.float32) - def custom_generator(): - while 1: - yield 0 + def ones_generator(): + while True: + yield np.ones([10, 10], np.float32), np.ones([10, 1], np.float32) - with self.cached_session(): - model = keras.models.Sequential() - model.add(keras.layers.Dense(1, input_shape=(2,))) - model.compile(loss='mse', optimizer='sgd') + inputs = keras.layers.Input(shape=(10,)) + x = keras.layers.Dense(10, activation='relu')(inputs) + outputs = keras.layers.Dense(1, activation='sigmoid')(x) + model = keras.Model(inputs, outputs) - with self.assertRaises(ValueError): - model.fit_generator(custom_generator(), - steps_per_epoch=5, - epochs=1, - verbose=1, - max_queue_size=10, - use_multiprocessing=False) - with self.assertRaises(ValueError): - model.fit_generator(custom_generator(), - steps_per_epoch=5, - epochs=1, - verbose=1, - max_queue_size=10, - use_multiprocessing=False, - validation_data=custom_generator(), - validation_steps=10) - with self.assertRaises(AttributeError): - model.predict_generator(custom_generator(), - steps=5, - max_queue_size=10, - use_multiprocessing=False) - with self.assertRaises(ValueError): - model.evaluate_generator(custom_generator(), - steps=5, - max_queue_size=10, - use_multiprocessing=False) + model.compile(RMSPropOptimizer(0.001), 'binary_crossentropy') + model.fit( + ones_generator(), + steps_per_epoch=2, + validation_data=val_data, + epochs=2) + model.evaluate(ones_generator(), steps=2) + model.predict(ones_generator(), steps=2) + + +@tf_test_util.run_all_in_graph_and_eager_modes +class TestGeneratorMethodsWithSequences(test.TestCase): def test_training_with_sequences(self): class DummySequence(keras.utils.Sequence): def __getitem__(self, idx): - return np.zeros([10, 2]), np.ones([10]) + return np.zeros([10, 2]), np.ones([10, 4]) def __len__(self): return 10 - arr_data = np.random.random((50, 2)) - arr_labels = np.random.random((50,)) - arr_sample_weights = np.random.random((50,)) - - def custom_generator(): - batch_size = 10 - num_samples = 50 - while True: - batch_index = np.random.randint(0, num_samples - batch_size) - start = batch_index - end = start + batch_size - x = arr_data[start: end] - y = arr_labels[start: end] - w = arr_sample_weights[start: end] - yield x, y, w - - with self.cached_session(): - model = keras.models.Sequential() - model.add(keras.layers.Dense(1, input_shape=(2,))) - model.compile(loss='mse', optimizer='sgd') + model = keras.models.Sequential() + model.add(keras.layers.Dense(4, input_shape=(2,))) + model.compile(loss='mse', optimizer='sgd') model.fit_generator(DummySequence(), steps_per_epoch=10, @@ -251,29 +305,6 @@ class TestGeneratorMethods(test.TestCase): workers=0, use_multiprocessing=False) - @tf_test_util.run_in_graph_and_eager_modes - def test_generator_input_to_fit_eval_predict(self): - val_data = np.ones([10, 10], np.float32), np.ones([10, 1], np.float32) - - def custom_generator(): - while True: - yield np.ones([10, 10], np.float32), np.ones([10, 1], np.float32) - - inputs = keras.layers.Input(shape=(10,)) - x = keras.layers.Dense(10, activation='relu')(inputs) - outputs = keras.layers.Dense(1, activation='sigmoid')(x) - model = keras.Model(inputs, outputs) - - model.compile(RMSPropOptimizer(0.001), 'binary_crossentropy') - model.fit( - custom_generator(), - steps_per_epoch=2, - validation_data=val_data, - epochs=2) - model.evaluate(custom_generator(), steps=2) - model.predict(custom_generator(), steps=2) - - @tf_test_util.run_in_graph_and_eager_modes def test_sequence_input_to_fit_eval_predict(self): val_data = np.ones([10, 10], np.float32), np.ones([10, 1], np.float32) @@ -303,5 +334,56 @@ class TestGeneratorMethods(test.TestCase): model.fit(CustomSequence(), sample_weight=np.ones([10, 1])) +@tf_test_util.run_all_in_graph_and_eager_modes +class TestConvertToGeneratorLike(test.TestCase, parameterized.TestCase): + simple_inputs = (np.ones((10, 10)), np.ones((10, 1))) + nested_inputs = ((np.ones((10, 10)), np.ones((10, 20))), (np.ones((10, 1)), + np.ones((10, 3)))) + + def _make_dataset(self, inputs, batches): + return dataset_ops.DatasetV2.from_tensors(inputs).repeat(batches) + + def _make_iterator(self, inputs, batches): + return dataset_ops.make_one_shot_iterator( + self._make_dataset(inputs, batches)) + + def _make_generator(self, inputs, batches): + + def _gen(): + for _ in range(batches): + yield inputs + + return _gen() + + def _make_numpy(self, inputs, _): + return inputs + + @parameterized.named_parameters( + ('simple_dataset', _make_dataset, simple_inputs), + ('simple_iterator', _make_iterator, simple_inputs), + ('simple_generator', _make_generator, simple_inputs), + ('simple_numpy', _make_numpy, simple_inputs), + ('nested_dataset', _make_dataset, nested_inputs), + ('nested_iterator', _make_iterator, nested_inputs), + ('nested_generator', _make_generator, nested_inputs), + ('nested_numpy', _make_numpy, nested_inputs)) + def test_convert_to_generator_like(self, input_fn, inputs): + expected_batches = 5 + data = input_fn(self, inputs, expected_batches) + + # Dataset and Iterator not supported in Legacy Graph mode. + if (not context.executing_eagerly() and + isinstance(data, (dataset_ops.DatasetV2, iterator_ops.Iterator))): + return + + generator, steps = training_generator.convert_to_generator_like( + data, batch_size=2, steps_per_epoch=expected_batches) + self.assertEqual(steps, expected_batches) + + for _ in range(expected_batches): + outputs = next(generator) + nest.assert_same_structure(outputs, inputs) + + if __name__ == '__main__': test.main() diff --git a/tensorflow/python/keras/engine/training_gpu_test.py b/tensorflow/python/keras/engine/training_gpu_test.py index 596d085f3fa..45dcfe43995 100644 --- a/tensorflow/python/keras/engine/training_gpu_test.py +++ b/tensorflow/python/keras/engine/training_gpu_test.py @@ -69,7 +69,7 @@ class TrainingGPUTest(test.TestCase): return simple_model if test.is_gpu_available(cuda_only=True): - with self.session(use_gpu=True): + with test_util.use_gpu(): losses_to_test = ['sparse_categorical_crossentropy', 'categorical_crossentropy', 'binary_crossentropy'] diff --git a/tensorflow/python/keras/engine/training_test.py b/tensorflow/python/keras/engine/training_test.py index 3cb24255d15..8e3b61be0c1 100644 --- a/tensorflow/python/keras/engine/training_test.py +++ b/tensorflow/python/keras/engine/training_test.py @@ -37,6 +37,7 @@ from tensorflow.python.keras import testing_utils from tensorflow.python.keras.callbacks import Callback from tensorflow.python.keras.engine.training_utils import weighted_masked_objective from tensorflow.python.ops import array_ops +from tensorflow.python.ops import math_ops from tensorflow.python.ops import sparse_ops from tensorflow.python.ops import variables as variables_lib from tensorflow.python.platform import test @@ -448,6 +449,7 @@ class TrainingTest(test.TestCase): optimizer=keras.optimizers.Adam(lr=0.0001), metrics=['accuracy']) + @tf_test_util.run_deprecated_v1 def test_that_trainable_disables_updates(self): val_a = np.random.random((10, 4)) val_out = np.random.random((10, 4)) @@ -544,6 +546,152 @@ class TrainingTest(test.TestCase): 'val_loss', 'val_weighted_mean_absolute_error' ])) + @tf_test_util.run_in_graph_and_eager_modes + def test_mismatched_output_shape_and_target_shape(self): + model = keras.Sequential([ + keras.layers.Dense(2, input_shape=(3, 4)), + keras.layers.Dense(5), + ]) + model.compile(RMSPropOptimizer(learning_rate=0.001), + loss='sparse_categorical_crossentropy') + # Test with Numpy data + x_train = np.random.random((10, 3, 4)) + y_train = np.random.randint(0, 5, size=(10, 3)) + model.fit(x_train, y_train, batch_size=5, epochs=1) + + # Test with iterator + dataset = dataset_ops.Dataset.from_tensor_slices((x_train, y_train)) + dataset = dataset.repeat(10) + dataset = dataset.batch(10) + iterator = dataset_ops.make_one_shot_iterator(dataset) + model.fit(iterator, epochs=1, steps_per_epoch=2) + + if context.executing_eagerly(): + # Test with eager execution + model.compile(RMSPropOptimizer(learning_rate=0.001), + loss='sparse_categorical_crossentropy', + run_eagerly=True) + model.fit(x_train, y_train, batch_size=5, epochs=1) + + # Test with eager execution and iterator + model.fit(iterator, epochs=1, steps_per_epoch=2) + + def test_losses_in_defun(self): + with context.eager_mode(): + layer = keras.layers.Dense(1, kernel_regularizer='l1') + layer(array_ops.ones([1, 10])) + + @function.defun + def get_losses(): + return layer.losses + + self.assertAllEqual( + self.evaluate(layer.losses), self.evaluate(get_losses())) + + @tf_test_util.run_in_graph_and_eager_modes + def test_logging(self): + mock_stdout = io.BytesIO() if six.PY2 else io.StringIO() + model = keras.models.Sequential() + model.add(keras.layers.Dense(10, activation='relu')) + model.add(keras.layers.Dense(1, activation='sigmoid')) + model.compile( + RMSPropOptimizer(learning_rate=0.001), loss='binary_crossentropy') + with test.mock.patch.object(sys, 'stdout', mock_stdout): + model.fit( + np.ones((10, 10), 'float32'), np.ones((10, 1), 'float32'), epochs=10) + self.assertTrue('Epoch 5/10' in mock_stdout.getvalue()) + + @tf_test_util.run_in_graph_and_eager_modes + def test_training_with_loss_instance(self): + a = keras.layers.Input(shape=(3,), name='input_a') + b = keras.layers.Input(shape=(3,), name='input_b') + + dense = keras.layers.Dense(4, name='dense') + c = dense(a) + d = dense(b) + e = keras.layers.Dropout(0.5, name='dropout')(c) + + model = keras.models.Model([a, b], [d, e]) + loss_weights = [1., 0.5] + model.compile( + RMSPropOptimizer(learning_rate=0.001), + loss=keras.losses.MeanSquaredError(), + metrics=[metrics_module.CategoricalAccuracy(), 'mae'], + loss_weights=loss_weights) + + input_a_np = np.random.random((10, 3)) + input_b_np = np.random.random((10, 3)) + + output_d_np = np.random.random((10, 4)) + output_e_np = np.random.random((10, 4)) + + model.fit([input_a_np, input_b_np], [output_d_np, output_e_np], + epochs=1, + batch_size=5) + + @tf_test_util.run_in_graph_and_eager_modes + def test_static_batch_in_input_layer(self): + + class Counter(keras.callbacks.Callback): + + def __init__(self): + self.batches = 0 + + def on_batch_end(self, batch, logs=None): + self.batches += 1 + + x, y = np.ones((64, 10), 'float32'), np.ones((64, 1), 'float32') + + for batch_size, expected_batches in [(None, 2), (4, 16)]: + inputs = keras.Input(batch_size=batch_size, shape=(10,)) + outputs = keras.layers.Dense(1, activation='sigmoid')(inputs) + model = keras.Model(inputs, outputs) + + model.compile(keras.optimizer_v2.adam.Adam(0.001), 'binary_crossentropy') + counter = Counter() + model.fit(x, y, callbacks=[counter]) + self.assertEqual(counter.batches, expected_batches) + + model = keras.Sequential( + [keras.layers.Dense(1, batch_input_shape=(batch_size, 10))]) + model.compile(keras.optimizer_v2.adam.Adam(0.001), 'binary_crossentropy') + counter = Counter() + model.fit(x, y, callbacks=[counter]) + self.assertEqual(counter.batches, expected_batches) + + @tf_test_util.run_in_graph_and_eager_modes + def test_static_batch_in_input_layer_consistency_checks(self): + x, y = np.ones((64, 10), 'float32'), np.ones((64, 1), 'float32') + + inputs = keras.Input(batch_size=2, shape=(10,)) + outputs = keras.layers.Dense(1, activation='sigmoid')(inputs) + model = keras.Model(inputs, outputs) + model.compile(keras.optimizer_v2.adam.Adam(0.001), 'binary_crossentropy') + with self.assertRaisesRegexp(ValueError, + 'incompatible with the specified batch size'): + model.fit(x, y, batch_size=4) + + data = dataset_ops.DatasetV2.from_tensor_slices((x, y)) + data = data.batch(4, drop_remainder=True) + with self.assertRaisesRegexp(ValueError, + 'incompatible with the specified batch size'): + model.fit(data, steps_per_epoch=16) + + @tf_test_util.run_in_graph_and_eager_modes + def test_compatible_batch_size_functional_model(self): + + class MyLayer(keras.layers.Layer): + + def call(self, inputs): + return array_ops.concat(inputs, axis=0) + + input1 = keras.Input(batch_size=2, shape=(10,)) + input2 = keras.Input(batch_size=3, shape=(10,)) + outputs = MyLayer()([input1, input2]) + with self.assertRaisesRegexp(ValueError, + 'specified batch sizes of the Input Layers'): + keras.Model([input1, input2], outputs) + class TestExceptionsAndWarnings(test.TestCase): @@ -1062,6 +1210,7 @@ class LossMaskingTest(test.TestCase): class TestDynamicTrainability(test.TestCase): + @tf_test_util.run_deprecated_v1 def test_trainable_warning(self): with self.cached_session(): x = np.random.random((5, 3)) @@ -1075,6 +1224,7 @@ class TestDynamicTrainability(test.TestCase): model.train_on_batch(x, y) self.assertRaises(Warning) + @tf_test_util.run_deprecated_v1 def test_trainable_argument(self): with self.cached_session(): x = np.random.random((5, 3)) @@ -1205,6 +1355,7 @@ class TestDynamicTrainability(test.TestCase): class TestTrainingWithDataTensors(test.TestCase): + @tf_test_util.run_deprecated_v1 def test_training_and_eval_methods_on_symbolic_tensors_single_io(self): with self.cached_session(): x = keras.layers.Input(shape=(3,), name='input') @@ -1245,6 +1396,7 @@ class TestTrainingWithDataTensors(test.TestCase): epochs=1, steps_per_epoch=2, verbose=0, validation_data=(inputs, targets), validation_steps=2) + @tf_test_util.run_deprecated_v1 def test_training_and_eval_methods_on_symbolic_tensors_multi_io(self): with self.cached_session(): a = keras.layers.Input(shape=(3,), name='input_a') @@ -1340,6 +1492,7 @@ class TestTrainingWithDataTensors(test.TestCase): model.predict([input_a_tf, input_b_tf], steps=2) model.test_on_batch([input_a_tf, input_b_tf], [output_d_tf, output_e_tf]) + @tf_test_util.run_deprecated_v1 def test_model_with_input_feed_tensor(self): """We test building a model with a TF variable as input. @@ -1518,6 +1671,7 @@ class TestTrainingWithDataTensors(test.TestCase): # evaluate _ = model.evaluate(input_a_np, [output_a_np]) + @tf_test_util.run_deprecated_v1 def test_model_with_external_loss(self): with self.cached_session(): # None loss, only regularization loss. @@ -1713,6 +1867,7 @@ class TestTrainingWithDataTensors(test.TestCase): model.train_on_batch(input_val, None, sample_weight={'dense_a': np.random.random((10,))}) + @tf_test_util.run_deprecated_v1 def test_model_custom_target_tensors(self): with self.cached_session(): a = keras.Input(shape=(3,), name='input_a') @@ -1774,264 +1929,6 @@ class TestTrainingWithDataTensors(test.TestCase): [output_a_np, output_b_np]) -class TestTrainingWithDatasetIterators(test.TestCase): - - @tf_test_util.run_in_graph_and_eager_modes - def test_training_and_eval_methods_on_iterators_single_io(self): - model = testing_utils.get_small_functional_mlp(1, 4, input_dim=3) - optimizer = RMSPropOptimizer(learning_rate=0.001) - loss = 'mse' - metrics = ['mae', metrics_module.CategoricalAccuracy()] - model.compile(optimizer, loss, metrics=metrics) - - inputs = np.zeros((10, 3)) - targets = np.zeros((10, 4)) - dataset = dataset_ops.Dataset.from_tensor_slices((inputs, targets)) - dataset = dataset.repeat(100) - dataset = dataset.batch(10) - iterator = dataset.make_one_shot_iterator() - - model.fit(iterator, epochs=1, steps_per_epoch=2, verbose=1) - model.evaluate(iterator, steps=2, verbose=1) - model.predict(iterator, steps=2) - model.train_on_batch(iterator) - model.test_on_batch(iterator) - model.predict_on_batch(iterator) - - # Test with validation data - model.fit(iterator, - epochs=1, steps_per_epoch=2, verbose=0, - validation_data=iterator, validation_steps=2) - # Test with validation split - with self.assertRaisesRegexp( - ValueError, '`validation_split` argument is not supported ' - 'when input `x` is a dataset or a dataset iterator'): - model.fit(iterator, - epochs=1, steps_per_epoch=2, verbose=0, - validation_split=0.5, validation_steps=2) - - # Test with sample weight. - sample_weight = np.random.random((10,)) - with self.assertRaisesRegexp( - ValueError, '`sample_weight` argument is not supported ' - 'when input `x` is a dataset or a dataset iterator'): - model.fit( - iterator, - epochs=1, - steps_per_epoch=2, - verbose=0, - sample_weight=sample_weight) - - # Test invalid usage - with self.assertRaisesRegexp(ValueError, - 'you should not specify a target'): - model.fit(iterator, iterator, - epochs=1, steps_per_epoch=2, verbose=0) - - with self.assertRaisesRegexp( - ValueError, 'you should specify the `steps_per_epoch` argument'): - model.fit(iterator, epochs=1, verbose=0) - with self.assertRaisesRegexp(ValueError, - 'you should specify the `steps` argument'): - model.evaluate(iterator, verbose=0) - with self.assertRaisesRegexp(ValueError, - 'you should specify the `steps` argument'): - model.predict(iterator, verbose=0) - - @tf_test_util.run_in_graph_and_eager_modes - def test_get_next_op_created_once(self): - model = testing_utils.get_small_functional_mlp(1, 4, input_dim=3) - optimizer = RMSPropOptimizer(learning_rate=0.001) - loss = 'mse' - metrics = ['mae'] - model.compile(optimizer, loss, metrics=metrics) - - inputs = np.zeros((10, 3)) - targets = np.zeros((10, 4)) - dataset = dataset_ops.Dataset.from_tensor_slices((inputs, targets)) - dataset = dataset.repeat(100) - dataset = dataset.batch(10) - iterator = dataset.make_one_shot_iterator() - - model.fit(iterator, epochs=1, steps_per_epoch=2, verbose=1) - # Finalize graph to make sure we are not appending another iterator - # get_next op in the graph. - ops.get_default_graph().finalize() - model.fit(iterator, epochs=1, steps_per_epoch=2, verbose=1) - - @tf_test_util.run_in_graph_and_eager_modes - def test_iterators_running_out_of_data(self): - model = testing_utils.get_small_functional_mlp(1, 4, input_dim=3) - optimizer = RMSPropOptimizer(learning_rate=0.001) - loss = 'mse' - metrics = ['mae'] - model.compile(optimizer, loss, metrics=metrics) - - inputs = np.zeros((10, 3)) - targets = np.zeros((10, 4)) - dataset = dataset_ops.Dataset.from_tensor_slices((inputs, targets)) - dataset = dataset.repeat(2) - dataset = dataset.batch(10) - iterator = dataset.make_one_shot_iterator() - - with test.mock.patch.object(logging, 'warning') as mock_log: - model.fit(iterator, epochs=1, steps_per_epoch=3, verbose=0) - self.assertRegexpMatches( - str(mock_log.call_args), - 'dataset iterator ran out of data') - - -class TestTrainingWithDataset(test.TestCase): - - @tf_test_util.run_in_graph_and_eager_modes - def test_calling_model_on_same_dataset(self): - model = testing_utils.get_small_functional_mlp(1, 4, input_dim=3) - optimizer = RMSPropOptimizer(learning_rate=0.001) - loss = 'mse' - metrics = ['mae'] - model.compile(optimizer, loss, metrics=metrics) - - inputs = np.zeros((10, 3)) - targets = np.zeros((10, 4)) - dataset = dataset_ops.Dataset.from_tensor_slices((inputs, targets)) - dataset = dataset.repeat(100) - dataset = dataset.batch(10) - - # Call fit with validation data - model.fit(dataset, epochs=1, steps_per_epoch=2, verbose=0, - validation_data=dataset, validation_steps=2) - # Finalize the graph to make sure new ops aren't added when calling on the - # same dataset - ops.get_default_graph().finalize() - model.fit(dataset, epochs=1, steps_per_epoch=2, verbose=0, - validation_data=dataset, validation_steps=2) - - @tf_test_util.run_in_graph_and_eager_modes - def test_training_and_eval_methods_on_dataset(self): - model = testing_utils.get_small_functional_mlp(1, 4, input_dim=3) - optimizer = RMSPropOptimizer(learning_rate=0.001) - loss = 'mse' - metrics = ['mae', metrics_module.CategoricalAccuracy()] - model.compile(optimizer, loss, metrics=metrics) - - inputs = np.zeros((10, 3)) - targets = np.zeros((10, 4)) - dataset = dataset_ops.Dataset.from_tensor_slices((inputs, targets)) - dataset = dataset.repeat(100) - dataset = dataset.batch(10) - - model.fit(dataset, epochs=1, steps_per_epoch=2, verbose=1) - model.evaluate(dataset, steps=2, verbose=1) - model.predict(dataset, steps=2) - model.train_on_batch(dataset) - model.predict_on_batch(dataset) - - # Test with validation data - model.fit(dataset, epochs=1, steps_per_epoch=2, verbose=0, - validation_data=dataset, validation_steps=2) - - # Test with validation split - with self.assertRaisesRegexp( - ValueError, '`validation_split` argument is not supported ' - 'when input `x` is a dataset or a dataset iterator'): - model.fit(dataset, - epochs=1, steps_per_epoch=2, verbose=0, - validation_split=0.5, validation_steps=2) - - # Test with sample weight. - sample_weight = np.random.random((10,)) - with self.assertRaisesRegexp( - ValueError, '`sample_weight` argument is not supported ' - 'when input `x` is a dataset or a dataset iterator'): - model.fit( - dataset, - epochs=1, - steps_per_epoch=2, - verbose=0, - sample_weight=sample_weight) - - # Test invalid usage - with self.assertRaisesRegexp(ValueError, - 'you should not specify a target'): - model.fit(dataset, dataset, - epochs=1, steps_per_epoch=2, verbose=0) - - with self.assertRaisesRegexp( - ValueError, 'you should specify the `steps_per_epoch` argument'): - model.fit(dataset, epochs=1, verbose=0) - with self.assertRaisesRegexp(ValueError, - 'you should specify the `steps` argument'): - model.evaluate(dataset, verbose=0) - with self.assertRaisesRegexp(ValueError, - 'you should specify the `steps` argument'): - model.predict(dataset, verbose=0) - - @tf_test_util.run_in_graph_and_eager_modes - def test_dataset_with_sample_weights(self): - model = testing_utils.get_small_functional_mlp(1, 4, input_dim=3) - optimizer = RMSPropOptimizer(learning_rate=0.001) - loss = 'mse' - metrics = ['mae', metrics_module.CategoricalAccuracy()] - model.compile(optimizer, loss, metrics=metrics) - - inputs = np.zeros((10, 3), np.float32) - targets = np.zeros((10, 4), np.float32) - sample_weights = np.ones((10), np.float32) - dataset = dataset_ops.Dataset.from_tensor_slices((inputs, targets, - sample_weights)) - dataset = dataset.repeat(100) - dataset = dataset.batch(10) - - model.fit(dataset, epochs=1, steps_per_epoch=2, verbose=1) - model.evaluate(dataset, steps=2, verbose=1) - model.predict(dataset, steps=2) - model.train_on_batch(dataset) - model.predict_on_batch(dataset) - - @tf_test_util.run_in_graph_and_eager_modes - def test_dataset_with_sparse_labels(self): - model = testing_utils.get_small_functional_mlp(1, 4, input_dim=3) - optimizer = RMSPropOptimizer(learning_rate=0.001) - loss = 'sparse_categorical_crossentropy' - model.compile(optimizer, loss) - - inputs = np.zeros((10, 3)) - targets = np.random.randint(0, 4, size=10, dtype=np.int32) - dataset = dataset_ops.Dataset.from_tensor_slices((inputs, targets)) - dataset = dataset.repeat(100) - dataset = dataset.batch(10) - - model.fit(dataset, epochs=1, steps_per_epoch=2, verbose=1) - - def test_dataset_input_shape_validation(self): - with self.cached_session(): - model = testing_utils.get_small_functional_mlp(1, 4, input_dim=3) - model.compile(optimizer=RMSPropOptimizer(learning_rate=0.001), loss='mse') - - # User forgets to batch the dataset - inputs = np.zeros((10, 3)) - targets = np.zeros((10, 4)) - dataset = dataset_ops.Dataset.from_tensor_slices((inputs, targets)) - dataset = dataset.repeat(100) - - with self.assertRaisesRegexp( - ValueError, - r'expected (.*?) to have shape \(3,\) but got array with shape \(1,\)' - ): - model.train_on_batch(dataset) - - # Wrong input shape - inputs = np.zeros((10, 5)) - targets = np.zeros((10, 4)) - dataset = dataset_ops.Dataset.from_tensor_slices((inputs, targets)) - dataset = dataset.repeat(100) - dataset = dataset.batch(10) - - with self.assertRaisesRegexp(ValueError, - r'expected (.*?) to have shape \(3,\)'): - model.train_on_batch(dataset) - - class TestTrainingWithMetrics(test.TestCase): """Training tests related to metrics.""" @@ -2095,39 +1992,6 @@ class TestTrainingWithMetrics(test.TestCase): self.assertEqual(outs[1], 0.) self.assertEqual(outs[2], 0.) - @tf_test_util.run_in_graph_and_eager_modes - def test_metrics_correctness_with_iterator(self): - model = keras.Sequential() - model.add( - keras.layers.Dense( - 8, activation='relu', input_dim=4, kernel_initializer='ones')) - model.add( - keras.layers.Dense( - 1, activation='sigmoid', kernel_initializer='ones')) - model.compile( - loss='binary_crossentropy', - metrics=['accuracy', metrics_module.BinaryAccuracy()], - optimizer=RMSPropOptimizer(learning_rate=0.001)) - - np.random.seed(123) - x = np.random.randint(10, size=(100, 4)).astype(np.float32) - y = np.random.randint(2, size=(100, 1)).astype(np.float32) - dataset = dataset_ops.Dataset.from_tensor_slices((x, y)) - dataset = dataset.batch(10) - iterator = dataset.make_one_shot_iterator() - outs = model.evaluate(iterator, steps=10) - self.assertEqual(np.around(outs[1], decimals=1), 0.5) - self.assertEqual(np.around(outs[2], decimals=1), 0.5) - - y = np.zeros((100, 1), dtype=np.float32) - dataset = dataset_ops.Dataset.from_tensor_slices((x, y)) - dataset = dataset.repeat(100) - dataset = dataset.batch(10) - iterator = dataset.make_one_shot_iterator() - outs = model.evaluate(iterator, steps=10) - self.assertEqual(outs[1], 0.) - self.assertEqual(outs[2], 0.) - @tf_test_util.run_in_graph_and_eager_modes def test_metrics_correctness_with_weighted_metrics(self): np.random.seed(1337) @@ -2153,7 +2017,7 @@ class TestTrainingWithMetrics(test.TestCase): w = np.array([[3., 4.], [1., 2.]]) outs = model.evaluate(x, y, sample_weight=w) - self.assertArrayNear(outs, [0.3, 0.7, 0.3], .001) + self.assertArrayNear(outs, [0.75, 0.7, 0.3], .001) # Verify that metric value is same with arbitrary weights and batch size. x = np.random.random((50, 2, 1)) @@ -2223,32 +2087,331 @@ class TestTrainingWithMetrics(test.TestCase): # verify that masking is combined with sample weights. w = np.array([3, 2, 4]) scores = model.train_on_batch(x, y, sample_weight=w) - self.assertArrayNear(scores, [0.2, 0.8], 0.1) + self.assertArrayNear(scores, [0.3328, 0.8], 0.001) + + @tf_test_util.run_deprecated_v1 + def test_add_metric_with_tensor_on_model_in_graph_mode(self): + with self.cached_session(): + x = keras.layers.Input(shape=(1,)) + y = keras.layers.Dense(1, kernel_initializer='ones')(x) + model = keras.models.Model(x, y) + model.add_metric( + math_ops.reduce_sum(y), name='metric_1', aggregation='mean') + + # test with a metric which does not have the standard signature: + # (y_true, y_pred, sample_Weight) + model.add_metric(metrics_module.Mean(name='metric_2')(y)) + model.compile('sgd', loss='mse') + + inputs = np.ones(shape=(10, 1)) + targets = np.ones(shape=(10, 1)) + history = model.fit( + inputs, + targets, + epochs=2, + batch_size=5, + validation_data=(inputs, targets)) + self.assertEqual(history.history['metric_1'][-1], 5) + self.assertEqual(history.history['metric_2'][-1], 1) + self.assertEqual(history.history['val_metric_1'][-1], 5) + self.assertEqual(history.history['val_metric_2'][-1], 1) + + eval_results = model.evaluate(inputs, targets, batch_size=5) + self.assertEqual(eval_results[-1], 1) + self.assertEqual(eval_results[-2], 5) + + model.predict(inputs, batch_size=5) + model.train_on_batch(inputs, targets) + model.test_on_batch(inputs, targets) @tf_test_util.run_in_graph_and_eager_modes - def test_logging(self): - mock_stdout = io.BytesIO() if six.PY2 else io.StringIO() - model = keras.models.Sequential() - model.add(keras.layers.Dense(10, activation='relu')) - model.add(keras.layers.Dense(1, activation='sigmoid')) - model.compile( - RMSPropOptimizer(learning_rate=0.001), loss='binary_crossentropy') - with test.mock.patch.object(sys, 'stdout', mock_stdout): - model.fit( - np.ones((10, 10), 'float32'), np.ones((10, 1), 'float32'), epochs=10) - self.assertTrue('Epoch 5/10' in mock_stdout.getvalue()) + def test_add_metric_in_model_call(self): - def test_losses_in_defun(self): + class TestModel(keras.Model): + + def __init__(self): + super(TestModel, self).__init__(name='test_model') + self.dense1 = keras.layers.Dense(2, kernel_initializer='ones') + self.mean = metrics_module.Mean(name='metric_1') + + def call(self, x): + self.add_metric( + math_ops.reduce_sum(x), name='metric_2', aggregation='mean') + # Provide same name as in the instance created in __init__ + # for eager mode + self.add_metric(self.mean(x), name='metric_1') + return self.dense1(x) + + model = TestModel() + model.compile(loss='mse', optimizer=RMSPropOptimizer(0.01)) + + x = np.ones(shape=(10, 1)) + y = np.ones(shape=(10, 2)) + history = model.fit(x, y, epochs=2, batch_size=5, validation_data=(x, y)) + self.assertAlmostEqual(history.history['metric_1'][-1], 1, 0) + self.assertAlmostEqual(history.history['val_metric_1'][-1], 1, 0) + self.assertAlmostEqual(history.history['metric_2'][-1], 5, 0) + self.assertAlmostEqual(history.history['val_metric_2'][-1], 5, 0) + + eval_results = model.evaluate(x, y, batch_size=5) + self.assertAlmostEqual(eval_results[1], 1, 0) + self.assertAlmostEqual(eval_results[2], 5, 0) + + model.predict(x, batch_size=5) + model.train_on_batch(x, y) + model.test_on_batch(x, y) + + def test_add_metric_in_model_call_run_eagerly(self): with context.eager_mode(): - layer = keras.layers.Dense(1, kernel_regularizer='l1') - layer(array_ops.ones([1, 10])) - @function.defun - def get_losses(): - return layer.losses + class TestModel(keras.Model): + + def __init__(self): + super(TestModel, self).__init__(name='test_model') + self.dense1 = keras.layers.Dense(2, kernel_initializer='ones') + self.mean = metrics_module.Mean(name='metric_1') + + def call(self, x): + self.add_metric( + math_ops.reduce_sum(x), name='metric_2', aggregation='mean') + # Provide same name as in the instance created in __init__ + # for eager mode + self.add_metric(self.mean(x), name='metric_1') + return self.dense1(x) + + model = TestModel() + model.compile( + loss='mse', optimizer=RMSPropOptimizer(0.01), run_eagerly=True) + + x = np.ones(shape=(10, 1)) + y = np.ones(shape=(10, 2)) + history = model.fit(x, y, epochs=2, batch_size=5, validation_data=(x, y)) + self.assertAlmostEqual(history.history['metric_1'][-1], 1, 0) + self.assertAlmostEqual(history.history['val_metric_1'][-1], 1, 0) + self.assertAlmostEqual(history.history['metric_2'][-1], 5, 0) + self.assertAlmostEqual(history.history['val_metric_2'][-1], 5, 0) + + eval_results = model.evaluate(x, y, batch_size=5) + self.assertAlmostEqual(eval_results[1], 1, 0) + self.assertAlmostEqual(eval_results[2], 5, 0) + + model.predict(x, batch_size=5) + model.train_on_batch(x, y) + model.test_on_batch(x, y) + + @tf_test_util.run_in_graph_and_eager_modes + def test_add_metric_in_layer_call(self): + + class TestLayer(keras.layers.Layer): + + def build(self, input_shape): + self.a = self.add_variable( + 'a', (1, 1), initializer='ones', trainable=False) + self.built = True + + def call(self, inputs): + self.add_metric( + math_ops.reduce_sum(inputs), name='metric_1', aggregation='mean') + return inputs + 1 + + model = keras.Sequential() + model.add(TestLayer(input_shape=(1,))) + model.add(keras.layers.Dense(2, kernel_initializer='ones')) + model.compile(loss='mse', optimizer=RMSPropOptimizer(0.01)) + + x = np.ones(shape=(10, 1)) + y = np.ones(shape=(10, 2)) + history = model.fit(x, y, epochs=2, batch_size=5, validation_data=(x, y)) + self.assertEqual(history.history['metric_1'][-1], 5) + self.assertAlmostEqual(history.history['val_metric_1'][-1], 5, 0) + + def test_add_metric_in_layer_call_run_eagerly(self): + with context.eager_mode(): + + class TestLayer(keras.layers.Layer): + + def build(self, input_shape): + self.a = self.add_variable( + 'a', (1, 1), initializer='ones', trainable=False) + self.built = True + + def call(self, inputs): + self.add_metric( + math_ops.reduce_sum(inputs), name='metric_1', aggregation='mean') + return inputs + 1 + + model = keras.Sequential() + model.add(TestLayer(input_shape=(1,))) + model.add(keras.layers.Dense(2, kernel_initializer='ones')) + model.compile( + loss='mse', optimizer=RMSPropOptimizer(0.01), run_eagerly=True) + + x = np.ones(shape=(10, 1)) + y = np.ones(shape=(10, 2)) + history = model.fit(x, y, epochs=2, batch_size=5, validation_data=(x, y)) + self.assertEqual(history.history['metric_1'][-1], 5) + self.assertAlmostEqual(history.history['val_metric_1'][-1], 5, 0) + + @tf_test_util.run_deprecated_v1 + def test_model_metrics_list(self): + with self.cached_session(): + x = keras.layers.Input(shape=(1,)) + y = keras.layers.Dense(1, kernel_initializer='ones')(x) + model = keras.models.Model(x, y) + model.add_metric( + math_ops.reduce_sum(y), name='metric_1', aggregation='mean') + model.add_metric(metrics_module.Mean(name='metric_2')(y)) + model.compile('sgd', loss='mse', metrics=['acc']) + + # Verify that the metrics added using `compile` and `add_metric` API are + # included + self.assertEqual(model._compile_metrics, ['acc']) + names = [] + for m in model.metrics: + if isinstance(m, metrics_module.Metric): + names.append(m.name) + else: + names.append(m.__name__) + self.assertEqual(names, ['binary_accuracy', 'metric_1', 'metric_2']) + + def test_model_eager_metrics_list(self): + with context.eager_mode(): + + class TestModel(keras.Model): + + def __init__(self): + super(TestModel, self).__init__(name='test_model') + self.dense1 = keras.layers.Dense(2, kernel_initializer='ones') + + def call(self, x): + self.add_metric( + math_ops.reduce_sum(x), name='metric_1', aggregation='mean') + return self.dense1(x) + + model = TestModel() + model.compile( + loss='mse', + optimizer=RMSPropOptimizer(0.01), + metrics=['acc'], + run_eagerly=True) + x = np.ones(shape=(10, 1)) + y = np.ones(shape=(10, 2)) + model.fit(x, y, epochs=2, batch_size=5, validation_data=(x, y)) + + self.assertEqual(model._compile_metrics, ['acc']) + names = [] + for m in model.metrics: + if isinstance(m, metrics_module.Metric): + names.append(m.name) + else: + names.append(m.__name__) + self.assertEqual(names, ['categorical_accuracy', 'metric_1']) + + @tf_test_util.run_in_graph_and_eager_modes + def test_multiple_add_metric_calls(self): + + class TestModel(keras.Model): + + def __init__(self): + super(TestModel, self).__init__(name='test_model') + self.dense1 = keras.layers.Dense(2, kernel_initializer='ones') + self.mean1 = metrics_module.Mean(name='metric_1') + self.mean2 = metrics_module.Mean(name='metric_2') + + def call(self, x): + self.add_metric(self.mean2(x), name='metric_2') + self.add_metric(self.mean1(x), name='metric_1') + self.add_metric( + math_ops.reduce_sum(x), name='metric_3', aggregation='mean') + return self.dense1(x) + + model = TestModel() + model.compile(loss='mse', optimizer=RMSPropOptimizer(0.01)) + + x = np.ones(shape=(10, 1)) + y = np.ones(shape=(10, 2)) + history = model.fit(x, y, epochs=2, batch_size=5, validation_data=(x, y)) + self.assertAlmostEqual(history.history['metric_1'][-1], 1, 0) + self.assertAlmostEqual(history.history['metric_2'][-1], 1, 0) + self.assertAlmostEqual(history.history['metric_3'][-1], 5, 0) + + eval_results = model.evaluate(x, y, batch_size=5) + self.assertArrayNear(eval_results[1:4], [1, 1, 5], 0.1) + + model.predict(x, batch_size=5) + model.train_on_batch(x, y) + model.test_on_batch(x, y) + + def test_invalid_metric_tensor_in_call(self): + with context.eager_mode(): + + class TestLayer(keras.layers.Layer): + + def call(self, inputs): + self.add_metric(metrics_module.Mean(name='metric_1')(inputs)) + return inputs + 1 + + model = keras.Sequential() + model.add(TestLayer(input_shape=(1,))) + model.add(keras.layers.Dense(2, kernel_initializer='ones')) + model.compile( + loss='mse', optimizer=RMSPropOptimizer(0.01), run_eagerly=True) + + x = np.ones(shape=(10, 1)) + y = np.ones(shape=(10, 2)) + with self.assertRaisesRegexp( + ValueError, + 'We do not support adding an aggregated metric tensor in `call` in ' + 'eager execution.'): + model.fit(x, y, epochs=2, batch_size=5, validation_data=(x, y)) + + @tf_test_util.run_in_graph_and_eager_modes + def test_duplicate_metric_name_in_add_metric(self): + + class TestModel(keras.Model): + + def __init__(self): + super(TestModel, self).__init__(name='test_model') + self.dense1 = keras.layers.Dense(2, kernel_initializer='ones') + self.mean = metrics_module.Mean(name='metric_1') + self.mean2 = metrics_module.Mean(name='metric_1') + + def call(self, x): + self.add_metric(self.mean(x), name='metric_1') + return self.dense1(x) + + model = TestModel() + model.compile(loss='mse', optimizer=RMSPropOptimizer(0.01)) + + x = np.ones(shape=(10, 1)) + y = np.ones(shape=(10, 2)) + with self.assertRaisesRegexp( + ValueError, + 'Please provide different names for the metrics you have added. ' + 'We found 2 metrics with the name: "metric_1"'): + model.fit(x, y, epochs=2, batch_size=5, validation_data=(x, y)) + + @tf_test_util.run_in_graph_and_eager_modes + def test_multiple_no_name_input_to_add_metric(self): + + class TestModel(keras.Model): + + def __init__(self): + super(TestModel, self).__init__(name='test_model') + self.dense1 = keras.layers.Dense(2, kernel_initializer='ones') + + def call(self, x): + self.add_metric(math_ops.reduce_sum(x), aggregation='mean') + self.add_metric(math_ops.reduce_sum(x), aggregation='mean') + return self.dense1(x) + + model = TestModel() + model.compile(loss='mse', optimizer=RMSPropOptimizer(0.01)) + x = np.ones(shape=(10, 1)) + y = np.ones(shape=(10, 2)) + model.fit(x, y, epochs=2, batch_size=5, validation_data=(x, y)) + self.assertEqual([m.name for m in model.metrics], ['mean', 'mean_1']) - self.assertAllEqual(self.evaluate(layer.losses), - self.evaluate(get_losses())) if __name__ == '__main__': test.main() diff --git a/tensorflow/python/keras/engine/training_utils.py b/tensorflow/python/keras/engine/training_utils.py index e563b7a23df..0157fe084c2 100644 --- a/tensorflow/python/keras/engine/training_utils.py +++ b/tensorflow/python/keras/engine/training_utils.py @@ -18,163 +18,176 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function +import abc from collections import OrderedDict import copy -import math import numpy as np import six -from tensorflow.python.data.ops import dataset_ops from tensorflow.python.data.ops import iterator_ops from tensorflow.python.eager import context from tensorflow.python.framework import constant_op from tensorflow.python.framework import ops +from tensorflow.python.framework import tensor_shape from tensorflow.python.framework import tensor_util from tensorflow.python.keras import backend as K +from tensorflow.python.keras import callbacks as cbks from tensorflow.python.keras import losses from tensorflow.python.keras import metrics as metrics_module from tensorflow.python.keras.engine import base_layer +from tensorflow.python.keras.utils import generic_utils +from tensorflow.python.keras.utils.losses_utils import squeeze_or_expand_dimensions from tensorflow.python.ops import array_ops from tensorflow.python.ops import math_ops from tensorflow.python.ops import weights_broadcast_ops from tensorflow.python.util import nest -def _map_nested(data, func): - """Maps each nested element using func.""" - if isinstance(data, list): - return [_map_nested(nested_data, func) for nested_data in data] - elif isinstance(data, tuple): - return tuple(_map_nested(nested_data, func) for nested_data in data) - elif isinstance(data, dict): - return { - k: _map_nested(nested_data, func) for k, nested_data in data.items() - } - else: - return func(data) +@six.add_metaclass(abc.ABCMeta) +class Aggregator(object): + """Abstract base class used to aggregate batch-level outputs of a loop. + + Attributes: + use_steps: Whether the loop is using `step` or `batch_size`. + num_samples_or_steps: Either `batch_size*num_batches` or `steps`. + results: What to return at the end of the aggregation loop. + """ + + def __init__(self, use_steps, num_samples_or_steps): + self.use_steps = use_steps + self.num_samples_or_steps = num_samples_or_steps + self.results = [] + + @abc.abstractmethod + def create(self, batch_outs): + """Creates the initial results from the first batch outputs. + + Arguments: + batch_outs: A list of batch-level outputs. + """ + NotImplementedError('Must be implemented in subclasses.') + + @abc.abstractmethod + def aggregate(self, batch_outs, batch_start=None, batch_end=None): + """Aggregates batch-level results into total results. + + Arguments: + batch_outs: A list of batch-level outputs. + batch_start: The start index of this batch. Always `None` if `use_steps` + is `True`. + batch_end: The end index of this batch. Always `None` if `use_steps` is + `True`. + """ + NotImplementedError('Must be implemented in subclasses.') + + @abc.abstractmethod + def finalize(self): + """Prepares the total results to be returned.""" + NotImplementedError('Must be implemented in subclasses.') -def _nested_all(data, cond_func): - """Checks if all elements in a nested structure satisfy cond_func.""" - if isinstance(data, (tuple, list)): - return all([_nested_all(nested_data, cond_func) for nested_data in data]) - elif isinstance(data, dict): - return all( - [_nested_all(nested_data, cond_func) for nested_data in data.values()]) - else: - return cond_func(data) +class MetricsAggregator(Aggregator): + """Aggregator that calculates loss and metrics info.""" + + def create(self, batch_outs): + self.results = [0.] * len(batch_outs) + + def aggregate(self, batch_outs, batch_start=None, batch_end=None): + # Loss. + if self.use_steps: + self.results[0] += batch_outs[0] + else: + self.results[0] += batch_outs[0] * (batch_end - batch_start) + # Metrics (always stateful, just grab current values.) + self.results[1:] = batch_outs[1:] + + def finalize(self): + self.results[0] /= self.num_samples_or_steps -def _nested_any(data, cond_func): - """Checks if any nested_elements in a nested structure satisfy cond_func.""" - if isinstance(data, (tuple, list)): - return any([_nested_any(nested_data, cond_func) for nested_data in data]) - elif isinstance(data, dict): - return any( - [_nested_any(nested_data, cond_func) for nested_data in data.values()]) - else: - return cond_func(data) +class OutputsAggregator(Aggregator): + """Aggregator that concatenates outputs.""" + + def create(self, batch_outs): + if self.use_steps: + # Cannot pre-allocate the returned NumPy arrays bc + # batch sizes are unknown. Concatenate batches at the end. + for _ in batch_outs: + self.results.append([]) + else: + # Pre-allocate NumPy arrays. + for batch_out in batch_outs: + shape = (self.num_samples_or_steps,) + batch_out.shape[1:] + self.results.append(np.zeros(shape, dtype=batch_out.dtype)) + + def aggregate(self, batch_outs, batch_start=None, batch_end=None): + if self.use_steps: + for i, batch_out in enumerate(batch_outs): + self.results[i].append(batch_out) + else: + for i, batch_out in enumerate(batch_outs): + self.results[i][batch_start:batch_end] = batch_out + + def finalize(self): + if self.use_steps: + self.results = [np.concatenate(result, axis=0) for result in self.results] -def _convert_lists_to_tuples(data): - """Converts all lists to tuples, since Datasets expect tuples.""" - if isinstance(data, (tuple, list)): - return tuple(_convert_lists_to_tuples(nested_data) for nested_data in data) - elif isinstance(data, dict): - return { - k: _convert_lists_to_tuples(nested_data) - for k, nested_data in data.items() - } - else: - return data +def make_logs(model, outputs, mode, prefix=''): + """Computes logs for sending to `on_batch_end` methods.""" + logs = {} + # TODO(omalleyt): handle outputs in prediction when Callback + # hooks are ready. + if mode in ['train', 'test']: + if hasattr(model, 'metrics_names'): + for label, output in zip(model.metrics_names, outputs): + logs[prefix + label] = output + return logs -def _get_batch_axis_size(data): - """Returns batch axis shape for nested data.""" - if isinstance(data, (tuple, list)): - return _get_batch_axis_size(data[0]) - elif isinstance(data, dict): - return _get_batch_axis_size(list(data.values())) - else: - return int(data.shape[0]) +def get_progbar(model, count_mode): + """Get Progbar.""" + stateful_metric_names = None + if hasattr(model, 'metrics_names'): + stateful_metric_names = model.metrics_names[1:] # Exclude `loss` + return cbks.ProgbarLogger(count_mode, stateful_metrics=stateful_metric_names) -def convert_to_iterator(x=None, - y=None, - sample_weights=None, - batch_size=None, - steps_per_epoch=None, - epochs=1, - shuffle=False, - is_validation=False): - """Converts NumPy arrays or EagerTensors to an EagerIterator. +def slice_arrays(arrays, indices, contiguous=True): + """Slices batches out of provided arrays (workaround for eager tensors). - Combines all provided data into a single EagerIterator. + Unfortunately eager tensors don't have the same slicing behavior as + Numpy arrays (they follow the same slicing behavior as symbolic TF tensors), + hence we cannot use `generic_utils.slice_arrays` directly + and we have to implement this workaround based on `concat`. This has a + performance cost. Arguments: - x: NumPy array or EagerTensor, or list of Numpy arrays or EagerTensors - representing inputs to a model. - y: Optional. NumPy array or EagerTensor, or list of Numpy arrays or - EagerTensors representing targets of a model. - sample_weights: Optional NumPy array or EagerTensor representing sample - weights. - batch_size: Used to batch data and calculate how many steps EagerIterator - should take per epoch. - steps_per_epoch: If provided, how many steps EagerIterator should take per - epoch. - epochs: Epochs to repeat iterator for. - shuffle: Whether to shuffle data after each epoch. - is_validation: Whether this call is for validation during a training - (e.g., `fit()`) call. This info is used to construct error messages - (if any). - - Raises: - ValueError: if steps_per_epoch cannot be calculated from the data - provided. + arrays: Single array or list of arrays. + indices: List of indices in the array that should be included in the output + batch. + contiguous: Boolean flag indicating whether the indices are contiguous. Returns: - (Iterator, steps_per_epoch). - + Slice of data (either single array or list of arrays). """ - if isinstance(x, iterator_ops.EagerIterator): - return x, steps_per_epoch - - if not _nested_any(sample_weights, lambda x: x is None): - data = (x, y, sample_weights) - elif not _nested_any(y, lambda x: x is None): - data = (x, y) + converted_to_list = False + if not isinstance(arrays, list): + converted_to_list = True + arrays = [arrays] + if any(tensor_util.is_tensor(x) for x in arrays): + if not contiguous: + entries = [[x[i:i + 1] for i in indices] for x in arrays] + slices = [array_ops.concat(x, axis=0) for x in entries] + else: + slices = [x[indices[0]:indices[-1] + 1] for x in arrays] else: - # always wrap in a tuple, so we know y, sample_weights weren't set - # even when x has multiple elements - data = (x,) + slices = generic_utils.slice_arrays(arrays, indices) - data = _convert_lists_to_tuples(data) - if steps_per_epoch is None and batch_size is not None: - num_samples = _get_batch_axis_size(data) - steps_per_epoch = int(math.ceil(num_samples / batch_size)) - - if steps_per_epoch is None: - alternative_arg_name = ( - 'validation_steps' if is_validation else 'steps_per_epoch') - raise ValueError( - 'Could not determine how to convert EagerTensors into EagerIterator. ' - 'Please provide either `batch_size` or ' - '`%s`.' % alternative_arg_name) - - # TODO(omalleyt) for NumPy arrays in graph mode - # placeholder ops should be used - # this is only ideal for eager mode - dataset = dataset_ops.Dataset.from_tensor_slices(data) - - if batch_size is not None: - dataset = dataset.batch(batch_size) - if shuffle: - dataset = dataset.shuffle(buffer_size=10000) - dataset = dataset.repeat(epochs) - iterator = dataset.make_one_shot_iterator() - - return iterator, steps_per_epoch + if converted_to_list: + slices = slices[0] + return slices def check_num_samples(ins, @@ -224,9 +237,9 @@ def standardize_single_array(x): return None if x.shape is not None and len(x.shape) == 1: if tensor_util.is_tensor(x): - return array_ops.expand_dims(x, axis=1) + x = array_ops.expand_dims(x, axis=1) else: - return np.expand_dims(x, 1) + x = np.expand_dims(x, 1) return x @@ -629,15 +642,14 @@ def weighted_masked_objective(fn): weights = mask else: # Update dimensions of weights to match with mask if possible. - mask, _, weights = metrics_module.squeeze_or_expand_dimensions( - mask, None, weights) + mask, _, weights = squeeze_or_expand_dimensions(mask, None, weights) weights *= mask # Apply sample weighting. if weights is not None: # Update dimensions of weights to match with values if possible. - score_array, _, weights = metrics_module.squeeze_or_expand_dimensions( + score_array, _, weights = squeeze_or_expand_dimensions( score_array, None, weights) try: # Broadcast weights if possible. @@ -651,7 +663,7 @@ def weighted_masked_objective(fn): score_array = math_ops.multiply(score_array, weights) score_array = math_ops.reduce_sum(score_array) weights = math_ops.reduce_sum(weights) - score_array = metrics_module.safe_div(score_array, weights) + score_array = math_ops.div_no_nan(score_array, weights) return K.mean(score_array) return weighted @@ -835,12 +847,22 @@ def call_metric_function(metric_fn, y_true, y_pred, weights=None, mask=None): return metric_fn(y_true, y_pred, sample_weight=mask) # Update dimensions of weights to match with mask. - mask, _, weights = metrics_module.squeeze_or_expand_dimensions( - mask, None, weights) + mask, _, weights = squeeze_or_expand_dimensions(mask, None, weights) weights *= mask return metric_fn(y_true, y_pred, sample_weight=weights) +def get_loss_function(loss): + """Returns the loss function corresponding to the given loss input.""" + if loss is None or isinstance(loss, losses.Loss): + return loss + + # TODO(psv): After we have added all V2 losses, update this function. + if loss in ['mse', 'MSE', 'mean_squared_error']: + return losses.MeanSquaredError() + return losses.get(loss) + + def validate_iterator_input(x, y, sample_weight, validation_split=None): """Validates user input arguments when a dataset iterator is passed. @@ -1053,9 +1075,11 @@ class ModelInputs(object): self._inputs = inputs self._is_dict = isinstance(self._inputs, dict) self._is_single_input = not isinstance(self._inputs, (list, tuple, dict)) + self._flattened_inputs = [] self._input_names = [] - if isinstance(self._inputs, dict): + + if self._is_dict: for k in sorted(self._inputs.keys()): self._flattened_inputs.append(self._inputs[k]) self._input_names.append(k) @@ -1064,7 +1088,6 @@ class ModelInputs(object): self._input_names = [ 'input_%d' % (i + 1) for i in range(len(self._flattened_inputs)) ] - assert len(self._input_names) == len(self._flattened_inputs) def get_input_names(self): """Returns keys to name inputs by. @@ -1074,56 +1097,29 @@ class ModelInputs(object): """ return self._input_names - def _get(self, return_single_as_list=False): - """Returns provided inputs, potentially transformed. - - Inputs are returned in the same format they were provided i.e. lists - are returned as lists, single entries as single entries (unless - `return_single_as_list` is true), dictionaries as dictionaries. - - Args: - return_single_as_list: Returns a list of size 1 for single entry case. - """ - if self._is_dict: - return dict(zip(self._input_names, self._flattened_inputs)) - if self._is_single_input and not return_single_as_list: - return self._flattened_inputs[0] - return self._flattened_inputs - - def get_input_values(self): - """Returns input values passed in.""" - if context.executing_eagerly(): - for i in range(len(self._flattened_inputs)): - v = self._flattened_inputs[i] - if tensor_util.is_tensor(v): - v = cast_single_tensor(v) - else: - v = ops.convert_to_tensor(v, dtype=K.floatx()) - self._flattened_inputs[i] = v - return self._get(return_single_as_list=False) - def get_symbolic_inputs(self, return_single_as_list=False): """Returns inputs to be set as self.inputs for a model.""" for i in range(len(self._flattened_inputs)): k = self._input_names[i] v = self._flattened_inputs[i] - if context.executing_eagerly(): - v = K.placeholder((None,) + tuple(v.shape[1:]), name=k) - else: - if isinstance(v, list): - v = np.asarray(v) - if v.ndim == 1: - v = np.expand_dims(v, 1) - if isinstance(v, (np.ndarray)): - # We fix the placeholder shape except the batch size. - # This is suboptimal, but it is the best we can do with the info - # we have. The user should call `model._set_inputs(placeholders)` - # to specify custom placeholders if the need arises. - shape = (None,) + v.shape[1:] - v = K.placeholder(shape=shape, name=k) + if isinstance(v, (list, float, int)): + v = np.asarray(v) + if v.ndim == 1: + v = np.expand_dims(v, 1) + if isinstance(v, (np.ndarray, ops.EagerTensor)): + # We fix the placeholder shape except the batch size. + # This is suboptimal, but it is the best we can do with the info + # we have. The user should call `model._set_inputs(placeholders)` + # to specify custom placeholders if the need arises. + shape = (None,) + tuple(v.shape[1:]) + v = K.placeholder(shape=shape, name=k) self._flattened_inputs[i] = v - return self._get(return_single_as_list) + if self._is_dict: + return dict(zip(self._input_names, self._flattened_inputs)) + if self._is_single_input and not return_single_as_list: + return self._flattened_inputs[0] + return self._flattened_inputs def as_dict(self): """An iterable over a dictionary version of inputs.""" @@ -1133,3 +1129,54 @@ class ModelInputs(object): def as_list(self): """Returning the inputs as a list.""" return self._flattened_inputs + + +# Allow use of methods not exposed to the user. +# pylint: disable=protected-access +def get_input_shape_and_dtype(layer): + """Retrieves input shape and input dtype of layer if applicable. + + Args: + layer: Layer (or model) instance. + + Returns: + Tuple (input_shape, input_dtype). Both could be None if the layer + does not have a defined input shape. + + Raises: + ValueError: in case an empty Sequential or Graph Network is passed. + """ + + def _is_graph_model(layer): + return ((hasattr(layer, '_is_graph_network') and layer._is_graph_network) or + layer.__class__.__name__ == 'Sequential') + + # In case of nested models: recover the first layer + # of the deepest model to infer input shape and dtype. + # Subclassed Models may not have been built so can't be checked. + while _is_graph_model(layer): + if not layer.layers: + raise ValueError('An empty Model cannot be used as a Layer.') + layer = layer.layers[0] + + if hasattr(layer, '_batch_input_shape'): + return layer._batch_input_shape, layer.dtype + return None, None + + +# pylint: enable=protected-access + + +def get_static_batch_size(layer): + """Gets the static batch size of a Layer. + + Arguments: + layer: a `Layer` instance. + + Returns: + The static batch size of a Layer. + """ + batch_input_shape, _ = get_input_shape_and_dtype(layer) + if batch_input_shape is not None: + return tensor_shape.as_dimension(batch_input_shape[0]).value + return None diff --git a/tensorflow/python/keras/engine/training_utils_test.py b/tensorflow/python/keras/engine/training_utils_test.py index 7b217cf3732..44ea23998fe 100644 --- a/tensorflow/python/keras/engine/training_utils_test.py +++ b/tensorflow/python/keras/engine/training_utils_test.py @@ -21,185 +21,39 @@ from __future__ import print_function import numpy as np from tensorflow.python.eager import context -from tensorflow.python.framework import ops from tensorflow.python.framework import tensor_util -from tensorflow.python.framework import test_util from tensorflow.python.keras.engine import training_utils from tensorflow.python.keras.utils import tf_utils from tensorflow.python.platform import test -class TrainingUtilTest(test.TestCase): - - @test_util.run_in_graph_and_eager_modes - def test_convert_to_iterator_single_numpy(self): - batch_size = 2 - a = np.ones([10, 10]) - iterator, steps_per_epoch = training_utils.convert_to_iterator( - x=a, batch_size=batch_size) - self.assertEquals(steps_per_epoch, 5) - - expected_batch = a[:batch_size, :] - actual_batch, = iterator.get_next() - self.assertAllEqual(expected_batch, actual_batch) - - @test_util.run_in_graph_and_eager_modes - def test_convert_to_iterator_single_tensor(self): - batch_size = 2 - a = ops.convert_to_tensor(np.ones([10, 10])) - iterator, steps_per_epoch = training_utils.convert_to_iterator( - x=a, batch_size=batch_size) - self.assertEquals(steps_per_epoch, 5) - - expected_batch = a[:batch_size, :] - actual_batch, = iterator.get_next() - self.assertAllEqual(expected_batch, actual_batch) - - @test_util.run_in_graph_and_eager_modes - def test_convert_to_iterator_y(self): - batch_size = 2 - a = np.ones([10, 100]) - b = np.ones([10, 10]) - iterator, steps_per_epoch = training_utils.convert_to_iterator( - x=a, y=b, batch_size=batch_size) - self.assertEquals(steps_per_epoch, 5) - - expected_x = a[:batch_size, :] - expected_y = b[:batch_size, :] - actual_x, actual_y = iterator.get_next() - self.assertAllEqual(expected_x, actual_x) - self.assertAllEqual(expected_y, actual_y) - - @test_util.run_in_graph_and_eager_modes - def test_convert_to_iterator_sample_weights(self): - batch_size = 2 - a = ops.convert_to_tensor(np.ones([10, 100])) - b = ops.convert_to_tensor(np.ones([10, 10])) - sw = ops.convert_to_tensor(np.ones([10])) - iterator, steps_per_epoch = training_utils.convert_to_iterator( - x=a, y=b, sample_weights=sw, batch_size=batch_size) - self.assertEquals(steps_per_epoch, 5) - - expected_x = a[:batch_size, :] - expected_y = b[:batch_size, :] - expected_sw = sw[:batch_size] - actual_x, actual_y, actual_sw = iterator.get_next() - self.assertAllEqual(expected_x, actual_x) - self.assertAllEqual(expected_y, actual_y) - self.assertAllEqual(expected_sw, actual_sw) - - @test_util.run_in_graph_and_eager_modes - def test_convert_to_iterator_nested(self): - batch_size = 2 - x = {'1': np.ones([10, 100]), '2': [np.zeros([10, 10]), np.ones([10, 20])]} - iterator, steps_per_epoch = training_utils.convert_to_iterator( - x=x, batch_size=batch_size) - self.assertEquals(steps_per_epoch, 5) - - expected_x1 = x['1'][:batch_size, :] - expected_x2_0 = x['2'][0][:batch_size, :] - expected_x2_1 = x['2'][1][:batch_size, :] - - actual_x, = iterator.get_next() - actual_x1 = actual_x['1'][:batch_size, :] - actual_x2_0 = actual_x['2'][0][:batch_size, :] - actual_x2_1 = actual_x['2'][1][:batch_size, :] - - self.assertAllEqual(expected_x1, actual_x1) - self.assertAllEqual(expected_x2_0, actual_x2_0) - self.assertAllEqual(expected_x2_1, actual_x2_1) - - @test_util.run_in_graph_and_eager_modes - def test_convert_to_iterator_epochs(self): - batch_size = 2 - a = np.ones([10, 10]) - iterator, steps_per_epoch = training_utils.convert_to_iterator( - x=a, batch_size=batch_size, epochs=2) - self.assertEquals(steps_per_epoch, 5) - - expected_batch = a[:batch_size, :] - # loop through one whole epoch - for _ in range(6): - actual_batch, = iterator.get_next() - self.assertAllEqual(expected_batch, actual_batch) - - @test_util.run_in_graph_and_eager_modes - def test_convert_to_iterator_insufficient_info(self): - # with batch_size and steps_per_epoch not set - with self.assertRaises(ValueError): - a = np.ones([10, 10]) - _ = training_utils.convert_to_iterator(x=a) - - def test_nested_all(self): - nested_data = {'a': True, 'b': [True, True, (False, True)]} - all_true = training_utils._nested_all(nested_data, lambda x: x) - self.assertEquals(all_true, False) - - nested_data = {'a': True, 'b': [True, True, (True, True)]} - all_true = training_utils._nested_all(nested_data, lambda x: x) - self.assertEquals(all_true, True) - - def test_nested_any(self): - nested_data = [False, {'a': False, 'b': (False, True)}] - any_true = training_utils._nested_any(nested_data, lambda x: x) - self.assertEquals(any_true, True) - - nested_data = [False, {'a': False, 'b': (False, False)}] - any_true = training_utils._nested_any(nested_data, lambda x: x) - self.assertEquals(any_true, False) - - def test_check_array_lengths(self): - training_utils.check_array_lengths(None, None, None) - a_np = np.random.random((4, 3, 3)) - training_utils.check_array_lengths(a_np, a_np, a_np) - training_utils.check_array_lengths( - [a_np, a_np], [a_np, a_np], [a_np, a_np]) - training_utils.check_array_lengths([None], [None], [None]) - - b_np = np.random.random((3, 4)) - with self.assertRaises(ValueError): - training_utils.check_array_lengths([a_np], [b_np], None) - - class ModelInputsTest(test.TestCase): def test_single_thing(self): a = np.ones(10) model_inputs = training_utils.ModelInputs(a) - self.assertEquals(['input_1'], model_inputs.get_input_names()) - vals = model_inputs.get_input_values() - self.assertAllEqual(np.ones(10), vals) - self.assertFalse(tensor_util.is_tensor(vals)) + self.assertEqual(['input_1'], model_inputs.get_input_names()) vals = model_inputs.get_symbolic_inputs() self.assertTrue(tensor_util.is_tensor(vals)) vals = model_inputs.get_symbolic_inputs(return_single_as_list=True) - self.assertEquals(1, len(vals)) + self.assertEqual(1, len(vals)) self.assertTrue(tensor_util.is_tensor(vals[0])) def test_single_thing_eager(self): with context.eager_mode(): a = np.ones(10) model_inputs = training_utils.ModelInputs(a) - self.assertEquals(['input_1'], model_inputs.get_input_names()) - val = model_inputs.get_input_values() - self.assertAllEqual(np.ones(10), val) - self.assertTrue(tensor_util.is_tensor(val)) + self.assertEqual(['input_1'], model_inputs.get_input_names()) val = model_inputs.get_symbolic_inputs() self.assertTrue(tf_utils.is_symbolic_tensor(val)) vals = model_inputs.get_symbolic_inputs(return_single_as_list=True) - self.assertEquals(1, len(vals)) + self.assertEqual(1, len(vals)) self.assertTrue(tf_utils.is_symbolic_tensor(vals[0])) def test_list(self): a = [np.ones(10), np.ones(20)] model_inputs = training_utils.ModelInputs(a) - self.assertEquals(['input_1', 'input_2'], model_inputs.get_input_names()) - vals = model_inputs.get_input_values() - self.assertEqual(2, len(vals)) - self.assertAllEqual(np.ones(10), vals[0]) - self.assertAllEqual(np.ones(20), vals[1]) - self.assertFalse(tensor_util.is_tensor(vals[0])) - self.assertFalse(tensor_util.is_tensor(vals[1])) + self.assertEqual(['input_1', 'input_2'], model_inputs.get_input_names()) vals = model_inputs.get_symbolic_inputs() self.assertTrue(tensor_util.is_tensor(vals[0])) self.assertTrue(tensor_util.is_tensor(vals[1])) @@ -208,13 +62,7 @@ class ModelInputsTest(test.TestCase): with context.eager_mode(): a = [np.ones(10), np.ones(20)] model_inputs = training_utils.ModelInputs(a) - self.assertEquals(['input_1', 'input_2'], model_inputs.get_input_names()) - vals = model_inputs.get_input_values() - self.assertEqual(2, len(vals)) - self.assertAllEqual(np.ones(10), vals[0]) - self.assertAllEqual(np.ones(20), vals[1]) - self.assertTrue(tensor_util.is_tensor(vals[0])) - self.assertTrue(tensor_util.is_tensor(vals[1])) + self.assertEqual(['input_1', 'input_2'], model_inputs.get_input_names()) vals = model_inputs.get_symbolic_inputs() self.assertTrue(tf_utils.is_symbolic_tensor(vals[0])) self.assertTrue(tf_utils.is_symbolic_tensor(vals[1])) @@ -222,12 +70,7 @@ class ModelInputsTest(test.TestCase): def test_dict(self): a = {'b': np.ones(10), 'a': np.ones(20)} model_inputs = training_utils.ModelInputs(a) - self.assertEquals(['a', 'b'], model_inputs.get_input_names()) - vals = model_inputs.get_input_values() - self.assertAllEqual(np.ones(20), vals['a']) - self.assertAllEqual(np.ones(10), vals['b']) - self.assertFalse(tensor_util.is_tensor(vals['a'])) - self.assertFalse(tensor_util.is_tensor(vals['b'])) + self.assertEqual(['a', 'b'], model_inputs.get_input_names()) vals = model_inputs.get_symbolic_inputs() self.assertTrue(tensor_util.is_tensor(vals['a'])) self.assertTrue(tensor_util.is_tensor(vals['b'])) @@ -236,12 +79,7 @@ class ModelInputsTest(test.TestCase): with context.eager_mode(): a = {'b': np.ones(10), 'a': np.ones(20)} model_inputs = training_utils.ModelInputs(a) - self.assertEquals(['a', 'b'], model_inputs.get_input_names()) - vals = model_inputs.get_input_values() - self.assertAllEqual(np.ones(20), vals['a']) - self.assertAllEqual(np.ones(10), vals['b']) - self.assertTrue(tensor_util.is_tensor(vals['a'])) - self.assertTrue(tensor_util.is_tensor(vals['b'])) + self.assertEqual(['a', 'b'], model_inputs.get_input_names()) vals = model_inputs.get_symbolic_inputs() self.assertTrue(tf_utils.is_symbolic_tensor(vals['a'])) self.assertTrue(tf_utils.is_symbolic_tensor(vals['b'])) diff --git a/tensorflow/python/keras/estimator/__init__.py b/tensorflow/python/keras/estimator/__init__.py index b244beb5b58..dcd06008970 100644 --- a/tensorflow/python/keras/estimator/__init__.py +++ b/tensorflow/python/keras/estimator/__init__.py @@ -24,23 +24,54 @@ from tensorflow.python.util.tf_export import tf_export # As long as you depend //third_party/py/tensorflow:tensorflow target # everything will work as normal. -try: - from tensorflow.python.estimator import keras as keras_lib # pylint: disable=g-import-not-at-top - model_to_estimator = tf_export('keras.estimator.model_to_estimator')( - keras_lib.model_to_estimator) -except Exception: # pylint: disable=broad-except - # pylint: disable=unused-argument - def stub_model_to_estimator(keras_model=None, - keras_model_path=None, - custom_objects=None, - model_dir=None, - config=None): +# LINT.IfChange +@tf_export('keras.estimator.model_to_estimator') +def model_to_estimator( + keras_model=None, + keras_model_path=None, + custom_objects=None, + model_dir=None, + config=None): + """Constructs an `Estimator` instance from given keras model. + + For usage example, please see: + [Creating estimators from Keras + Models](https://tensorflow.org/guide/estimators#model_to_estimator). + + Args: + keras_model: A compiled Keras model object. This argument is mutually + exclusive with `keras_model_path`. + keras_model_path: Path to a compiled Keras model saved on disk, in HDF5 + format, which can be generated with the `save()` method of a Keras model. + This argument is mutually exclusive with `keras_model`. + custom_objects: Dictionary for custom objects. + model_dir: Directory to save `Estimator` model parameters, graph, summary + files for TensorBoard, etc. + config: `RunConfig` to config `Estimator`. + + Returns: + An Estimator from given keras model. + + Raises: + ValueError: if neither keras_model nor keras_model_path was given. + ValueError: if both keras_model and keras_model_path was given. + ValueError: if the keras_model_path is a GCS URI. + ValueError: if keras_model has not been compiled. + """ + try: + from tensorflow_estimator.python.estimator import keras as keras_lib # pylint: disable=g-import-not-at-top + except ImportError: raise NotImplementedError( 'tf.keras.estimator.model_to_estimator function not available in your ' 'installation.') - # pylint: enable=unused-argument + return keras_lib.model_to_estimator( + keras_model=keras_model, + keras_model_path=keras_model_path, + custom_objects=custom_objects, + model_dir=model_dir, + config=config) + +# LINT.ThenChange(//third_party/tensorflow_estimator/python/estimator/keras.py) - model_to_estimator = tf_export('keras.estimator.model_to_estimator')( - stub_model_to_estimator) diff --git a/tensorflow/python/keras/initializers_test.py b/tensorflow/python/keras/initializers_test.py index 2b758a98f30..4f91bea1e33 100644 --- a/tensorflow/python/keras/initializers_test.py +++ b/tensorflow/python/keras/initializers_test.py @@ -22,6 +22,7 @@ import numpy as np from tensorflow.python import keras from tensorflow.python.ops import init_ops +from tensorflow.python.framework import test_util from tensorflow.python.platform import test @@ -38,6 +39,7 @@ class KerasInitializersTest(test.TestCase): output_2 = keras.backend.get_value(variable) self.assertAllClose(output, output_2, atol=1e-4) + @test_util.run_deprecated_v1 def test_uniform(self): tensor_shape = (9, 6, 7) with self.cached_session(): @@ -47,6 +49,7 @@ class KerasInitializersTest(test.TestCase): tensor_shape, target_mean=0., target_max=1, target_min=-1) + @test_util.run_deprecated_v1 def test_normal(self): tensor_shape = (8, 12, 99) with self.cached_session(): @@ -54,6 +57,7 @@ class KerasInitializersTest(test.TestCase): tensor_shape, target_mean=0., target_std=1) + @test_util.run_deprecated_v1 def test_truncated_normal(self): tensor_shape = (12, 99, 7) with self.cached_session(): @@ -69,6 +73,7 @@ class KerasInitializersTest(test.TestCase): self._runner(keras.initializers.Constant(2), tensor_shape, target_mean=2, target_max=2, target_min=2) + @test_util.run_deprecated_v1 def test_lecun_uniform(self): tensor_shape = (5, 6, 4, 2) with self.cached_session(): @@ -77,6 +82,7 @@ class KerasInitializersTest(test.TestCase): self._runner(keras.initializers.lecun_uniform(seed=123), tensor_shape, target_mean=0., target_std=std) + @test_util.run_deprecated_v1 def test_glorot_uniform(self): tensor_shape = (5, 6, 4, 2) with self.cached_session(): @@ -85,6 +91,7 @@ class KerasInitializersTest(test.TestCase): self._runner(keras.initializers.glorot_uniform(seed=123), tensor_shape, target_mean=0., target_std=std) + @test_util.run_deprecated_v1 def test_he_uniform(self): tensor_shape = (5, 6, 4, 2) with self.cached_session(): @@ -93,6 +100,7 @@ class KerasInitializersTest(test.TestCase): self._runner(keras.initializers.he_uniform(seed=123), tensor_shape, target_mean=0., target_std=std) + @test_util.run_deprecated_v1 def test_lecun_normal(self): tensor_shape = (5, 6, 4, 2) with self.cached_session(): @@ -101,6 +109,7 @@ class KerasInitializersTest(test.TestCase): self._runner(keras.initializers.lecun_normal(seed=123), tensor_shape, target_mean=0., target_std=std) + @test_util.run_deprecated_v1 def test_glorot_normal(self): tensor_shape = (5, 6, 4, 2) with self.cached_session(): @@ -109,6 +118,7 @@ class KerasInitializersTest(test.TestCase): self._runner(keras.initializers.glorot_normal(seed=123), tensor_shape, target_mean=0., target_std=std) + @test_util.run_deprecated_v1 def test_he_normal(self): tensor_shape = (5, 6, 4, 2) with self.cached_session(): @@ -117,6 +127,7 @@ class KerasInitializersTest(test.TestCase): self._runner(keras.initializers.he_normal(seed=123), tensor_shape, target_mean=0., target_std=std) + @test_util.run_deprecated_v1 def test_orthogonal(self): tensor_shape = (20, 20) with self.cached_session(): diff --git a/tensorflow/python/keras/integration_test.py b/tensorflow/python/keras/integration_test.py index 3c0f73b1c3a..23f54385057 100644 --- a/tensorflow/python/keras/integration_test.py +++ b/tensorflow/python/keras/integration_test.py @@ -22,6 +22,7 @@ import numpy as np from tensorflow.python import keras from tensorflow.python.framework import dtypes +from tensorflow.python.framework import test_util from tensorflow.python.keras import testing_utils from tensorflow.python.layers import core as tf_core_layers from tensorflow.python.ops import nn @@ -34,6 +35,7 @@ class KerasIntegrationTest(test.TestCase): def test_version(self): self.assertTrue(keras.__version__.endswith('-tf')) + @test_util.run_deprecated_v1 def test_vector_classification_sequential(self): with self.cached_session(): np.random.seed(1337) @@ -59,6 +61,7 @@ class KerasIntegrationTest(test.TestCase): verbose=2) self.assertGreater(history.history['val_acc'][-1], 0.7) + @test_util.run_deprecated_v1 def test_vector_classification_functional(self): with self.cached_session(): np.random.seed(1337) @@ -83,6 +86,7 @@ class KerasIntegrationTest(test.TestCase): verbose=2) self.assertGreater(history.history['val_acc'][-1], 0.7) + @test_util.run_deprecated_v1 def test_temporal_classification_sequential(self): with self.cached_session(): np.random.seed(1337) @@ -105,6 +109,7 @@ class KerasIntegrationTest(test.TestCase): verbose=2) self.assertGreater(history.history['val_acc'][-1], 0.7) + @test_util.run_deprecated_v1 def test_temporal_classification_sequential_tf_rnn(self): with self.cached_session(): np.random.seed(1337) @@ -163,6 +168,7 @@ class KerasIntegrationTest(test.TestCase): verbose=2) self.assertGreater(history.history['val_acc'][-1], 0.7) + @test_util.run_deprecated_v1 def test_video_classification_functional(self): with self.cached_session(): np.random.seed(1337) @@ -191,6 +197,7 @@ class KerasIntegrationTest(test.TestCase): verbose=2) self.assertGreater(history.history['val_acc'][-1], 0.7) + @test_util.run_deprecated_v1 def test_vector_classification_shared_sequential(self): # Test that Sequential models that feature internal updates # and internal losses can be shared. @@ -225,6 +232,7 @@ class KerasIntegrationTest(test.TestCase): verbose=2) self.assertGreater(history.history['val_acc'][-1], 0.7) + @test_util.run_deprecated_v1 def test_vector_classification_shared_model(self): # Test that functional models that feature internal updates # and internal losses can be shared. @@ -312,6 +320,15 @@ class KerasIntegrationTest(test.TestCase): verbose=0) self.assertGreater(history.history['val_acc'][-1], 0.7) + def test_regularizers_with_get_variable(self): + # Test case for GitHub issue 22470. + with self.cached_session(): + v = variable_scope.get_variable( + "v", + shape=[4, 4], + initializer=keras.initializers.glorot_uniform(), + regularizer=keras.regularizers.l2(0.)) + if __name__ == '__main__': test.main() diff --git a/tensorflow/python/keras/layers/__init__.py b/tensorflow/python/keras/layers/__init__.py index 7268040b028..49990b6bf4f 100644 --- a/tensorflow/python/keras/layers/__init__.py +++ b/tensorflow/python/keras/layers/__init__.py @@ -22,7 +22,7 @@ from __future__ import print_function # pylint: disable=g-bad-import-order from tensorflow.python.keras.engine.input_layer import Input from tensorflow.python.keras.engine.input_layer import InputLayer -from tensorflow.python.keras.engine.base_layer import InputSpec +from tensorflow.python.keras.engine.input_spec import InputSpec from tensorflow.python.keras.engine.base_layer import Layer # Advanced activations. diff --git a/tensorflow/python/keras/layers/advanced_activations.py b/tensorflow/python/keras/layers/advanced_activations.py index b0dffced3ea..35ac7830b2e 100644 --- a/tensorflow/python/keras/layers/advanced_activations.py +++ b/tensorflow/python/keras/layers/advanced_activations.py @@ -22,8 +22,8 @@ from tensorflow.python.keras import backend as K from tensorflow.python.keras import constraints from tensorflow.python.keras import initializers from tensorflow.python.keras import regularizers -from tensorflow.python.keras.engine.base_layer import InputSpec from tensorflow.python.keras.engine.base_layer import Layer +from tensorflow.python.keras.engine.input_spec import InputSpec from tensorflow.python.keras.utils import tf_utils from tensorflow.python.ops import math_ops from tensorflow.python.util.tf_export import tf_export @@ -54,7 +54,6 @@ class LeakyReLU(Layer): super(LeakyReLU, self).__init__(**kwargs) self.supports_masking = True self.alpha = K.cast_to_floatx(alpha) - self._can_use_graph_functions = True def call(self, inputs): return K.relu(inputs, alpha=self.alpha) @@ -118,7 +117,6 @@ class PReLU(Layer): self.shared_axes = [shared_axes] else: self.shared_axes = list(shared_axes) - self._can_use_graph_functions = True @tf_utils.shape_type_conversion def build(self, input_shape): @@ -193,7 +191,6 @@ class ELU(Layer): super(ELU, self).__init__(**kwargs) self.supports_masking = True self.alpha = K.cast_to_floatx(alpha) - self._can_use_graph_functions = True def call(self, inputs): return K.elu(inputs, self.alpha) @@ -233,7 +230,6 @@ class ThresholdedReLU(Layer): super(ThresholdedReLU, self).__init__(**kwargs) self.supports_masking = True self.theta = K.cast_to_floatx(theta) - self._can_use_graph_functions = True def call(self, inputs, mask=None): return inputs * math_ops.cast( @@ -269,7 +265,6 @@ class Softmax(Layer): super(Softmax, self).__init__(**kwargs) self.supports_masking = True self.axis = axis - self._can_use_graph_functions = True def call(self, inputs): return K.softmax(inputs, axis=self.axis) @@ -324,7 +319,6 @@ class ReLU(Layer): self.max_value = max_value self.negative_slope = K.cast_to_floatx(negative_slope) self.threshold = K.cast_to_floatx(threshold) - self._can_use_graph_functions = True def call(self, inputs): # alpha is used for leaky relu slope in activations instead of diff --git a/tensorflow/python/keras/layers/convolutional.py b/tensorflow/python/keras/layers/convolutional.py index 0671a5a36d6..6564d6e8fdb 100644 --- a/tensorflow/python/keras/layers/convolutional.py +++ b/tensorflow/python/keras/layers/convolutional.py @@ -26,8 +26,8 @@ from tensorflow.python.keras import backend from tensorflow.python.keras import constraints from tensorflow.python.keras import initializers from tensorflow.python.keras import regularizers -from tensorflow.python.keras.engine.base_layer import InputSpec from tensorflow.python.keras.engine.base_layer import Layer +from tensorflow.python.keras.engine.input_spec import InputSpec # imports for backwards namespace compatibility # pylint: disable=unused-import from tensorflow.python.keras.layers.pooling import AveragePooling1D @@ -120,7 +120,6 @@ class Conv(Layer): name=name, activity_regularizer=regularizers.get(activity_regularizer), **kwargs) - self._can_use_graph_functions = True self.rank = rank self.filters = filters self.kernel_size = conv_utils.normalize_tuple( @@ -1916,7 +1915,6 @@ class UpSampling1D(Layer): super(UpSampling1D, self).__init__(**kwargs) self.size = int(size) self.input_spec = InputSpec(ndim=3) - self._can_use_graph_functions = True def compute_output_shape(self, input_shape): input_shape = tensor_shape.TensorShape(input_shape).as_list() @@ -1983,7 +1981,6 @@ class UpSampling2D(Layer): 'or `"bilinear"`.') self.interpolation = interpolation self.input_spec = InputSpec(ndim=4) - self._can_use_graph_functions = True def compute_output_shape(self, input_shape): input_shape = tensor_shape.TensorShape(input_shape).as_list() @@ -2054,7 +2051,6 @@ class UpSampling3D(Layer): self.size = conv_utils.normalize_tuple(size, 3, 'size') self.input_spec = InputSpec(ndim=5) super(UpSampling3D, self).__init__(**kwargs) - self._can_use_graph_functions = True def compute_output_shape(self, input_shape): input_shape = tensor_shape.TensorShape(input_shape).as_list() @@ -2109,7 +2105,6 @@ class ZeroPadding1D(Layer): def __init__(self, padding=1, **kwargs): super(ZeroPadding1D, self).__init__(**kwargs) - self._can_use_graph_functions = True self.padding = conv_utils.normalize_tuple(padding, 2, 'padding') self.input_spec = InputSpec(ndim=3) @@ -2175,7 +2170,6 @@ class ZeroPadding2D(Layer): def __init__(self, padding=(1, 1), data_format=None, **kwargs): super(ZeroPadding2D, self).__init__(**kwargs) - self._can_use_graph_functions = True self.data_format = conv_utils.normalize_data_format(data_format) if isinstance(padding, int): self.padding = ((padding, padding), (padding, padding)) @@ -2280,7 +2274,6 @@ class ZeroPadding3D(Layer): def __init__(self, padding=(1, 1, 1), data_format=None, **kwargs): super(ZeroPadding3D, self).__init__(**kwargs) - self._can_use_graph_functions = True self.data_format = conv_utils.normalize_data_format(data_format) if isinstance(padding, int): self.padding = ((padding, padding), (padding, padding), (padding, @@ -2375,7 +2368,6 @@ class Cropping1D(Layer): super(Cropping1D, self).__init__(**kwargs) self.cropping = conv_utils.normalize_tuple(cropping, 2, 'cropping') self.input_spec = InputSpec(ndim=3) - self._can_use_graph_functions = True def compute_output_shape(self, input_shape): input_shape = tensor_shape.TensorShape(input_shape).as_list() @@ -2475,7 +2467,6 @@ class Cropping2D(Layer): '((top_crop, bottom_crop), (left_crop, right_crop)). ' 'Found: ' + str(cropping)) self.input_spec = InputSpec(ndim=4) - self._can_use_graph_functions = True def compute_output_shape(self, input_shape): input_shape = tensor_shape.TensorShape(input_shape).as_list() @@ -2609,7 +2600,6 @@ class Cropping3D(Layer): ' (left_dim3_crop, right_dim2_crop)). ' 'Found: ' + str(cropping)) self.input_spec = InputSpec(ndim=5) - self._can_use_graph_functions = True def compute_output_shape(self, input_shape): input_shape = tensor_shape.TensorShape(input_shape).as_list() diff --git a/tensorflow/python/keras/layers/convolutional_recurrent.py b/tensorflow/python/keras/layers/convolutional_recurrent.py index 100542129bf..cf3861da218 100644 --- a/tensorflow/python/keras/layers/convolutional_recurrent.py +++ b/tensorflow/python/keras/layers/convolutional_recurrent.py @@ -26,8 +26,8 @@ from tensorflow.python.keras import backend as K from tensorflow.python.keras import constraints from tensorflow.python.keras import initializers from tensorflow.python.keras import regularizers -from tensorflow.python.keras.engine.base_layer import InputSpec from tensorflow.python.keras.engine.base_layer import Layer +from tensorflow.python.keras.engine.input_spec import InputSpec from tensorflow.python.keras.layers.recurrent import _generate_dropout_mask from tensorflow.python.keras.layers.recurrent import _standardize_args from tensorflow.python.keras.layers.recurrent import RNN diff --git a/tensorflow/python/keras/layers/core.py b/tensorflow/python/keras/layers/core.py index e5c37be0aa1..56dd70558cc 100644 --- a/tensorflow/python/keras/layers/core.py +++ b/tensorflow/python/keras/layers/core.py @@ -34,8 +34,8 @@ from tensorflow.python.keras import backend as K from tensorflow.python.keras import constraints from tensorflow.python.keras import initializers from tensorflow.python.keras import regularizers -from tensorflow.python.keras.engine.base_layer import InputSpec from tensorflow.python.keras.engine.base_layer import Layer +from tensorflow.python.keras.engine.input_spec import InputSpec from tensorflow.python.keras.utils import conv_utils from tensorflow.python.keras.utils import generic_utils from tensorflow.python.keras.utils import tf_utils @@ -81,7 +81,6 @@ class Masking(Layer): super(Masking, self).__init__(**kwargs) self.supports_masking = True self.mask_value = mask_value - self._can_use_graph_functions = True def compute_mask(self, inputs, mask=None): return K.any(math_ops.not_equal(inputs, self.mask_value), axis=-1) @@ -125,7 +124,6 @@ class Dropout(Layer): self.noise_shape = noise_shape self.seed = seed self.supports_masking = True - self._can_use_graph_functions = True def _get_noise_shape(self, inputs): # Subclasses of `Dropout` may implement `_get_noise_shape(self, inputs)`, @@ -326,7 +324,6 @@ class Activation(Layer): super(Activation, self).__init__(**kwargs) self.supports_masking = True self.activation = activations.get(activation) - self._can_use_graph_functions = True def call(self, inputs): return self.activation(inputs) @@ -379,7 +376,6 @@ class Reshape(Layer): def __init__(self, target_shape, **kwargs): super(Reshape, self).__init__(**kwargs) self.target_shape = tuple(target_shape) - self._can_use_graph_functions = True def _fix_unknown_dimension(self, input_shape, output_shape): """Find and replace a missing dimension in an output shape. @@ -488,7 +484,6 @@ class Permute(Layer): 'The set of indices in `dims` must be consecutive and start from 1.' % (dims,)) self.input_spec = InputSpec(ndim=len(self.dims) + 1) - self._can_use_graph_functions = True def compute_output_shape(self, input_shape): input_shape = tensor_shape.TensorShape(input_shape).as_list() @@ -540,7 +535,6 @@ class Flatten(Layer): super(Flatten, self).__init__(**kwargs) self.data_format = conv_utils.normalize_data_format(data_format) self.input_spec = InputSpec(min_ndim=2) - self._can_use_graph_functions = True def call(self, inputs): if self.data_format == 'channels_first': @@ -600,7 +594,6 @@ class RepeatVector(Layer): super(RepeatVector, self).__init__(**kwargs) self.n = n self.input_spec = InputSpec(ndim=2) - self._can_use_graph_functions = True def compute_output_shape(self, input_shape): input_shape = tensor_shape.TensorShape(input_shape).as_list() @@ -929,7 +922,6 @@ class Dense(Layer): self.supports_masking = True self.input_spec = InputSpec(min_ndim=2) - self._can_use_graph_functions = True def build(self, input_shape): input_shape = tensor_shape.TensorShape(input_shape) @@ -1029,7 +1021,6 @@ class ActivityRegularization(Layer): self.supports_masking = True self.l1 = l1 self.l2 = l2 - self._can_use_graph_functions = True def compute_output_shape(self, input_shape): return input_shape diff --git a/tensorflow/python/keras/layers/cudnn_recurrent.py b/tensorflow/python/keras/layers/cudnn_recurrent.py index beacdf25156..16692753afb 100644 --- a/tensorflow/python/keras/layers/cudnn_recurrent.py +++ b/tensorflow/python/keras/layers/cudnn_recurrent.py @@ -25,7 +25,8 @@ from tensorflow.python.keras import backend as K from tensorflow.python.keras import constraints from tensorflow.python.keras import initializers from tensorflow.python.keras import regularizers -from tensorflow.python.keras.engine.base_layer import InputSpec +from tensorflow.python.keras.engine.input_spec import InputSpec +from tensorflow.python.keras.layers import recurrent from tensorflow.python.keras.layers.recurrent import RNN from tensorflow.python.ops import array_ops from tensorflow.python.ops import gen_cudnn_rnn_ops @@ -80,11 +81,6 @@ class _CuDNNRNN(RNN): self._num_inputs = None self._vector_shape = constant_op.constant([-1]) - def _canonical_to_params(self, weights, biases): - weights = [array_ops.reshape(x, self._vector_shape) for x in weights] - biases = [array_ops.reshape(x, self._vector_shape) for x in biases] - return array_ops.concat(weights + biases, axis=0) - def call(self, inputs, mask=None, training=None, initial_state=None): if isinstance(mask, list): mask = mask[0] @@ -279,7 +275,7 @@ class CuDNNGRU(_CuDNNRNN): input_h = initial_state[0] input_h = array_ops.expand_dims(input_h, axis=0) - params = self._canonical_to_params( + params = recurrent._canonical_to_params( # pylint: disable=protected-access weights=[ self.kernel[:, self.units:self.units * 2], self.kernel[:, :self.units], @@ -296,7 +292,7 @@ class CuDNNGRU(_CuDNNRNN): self.bias[self.units * 3:self.units * 4], self.bias[self.units * 5:], ], - ) + shape=self._vector_shape) outputs, h, _, _ = gen_cudnn_rnn_ops.cudnn_rnn( inputs, @@ -474,7 +470,7 @@ class CuDNNLSTM(_CuDNNRNN): input_h = array_ops.expand_dims(input_h, axis=0) input_c = array_ops.expand_dims(input_c, axis=0) - params = self._canonical_to_params( + params = recurrent._canonical_to_params( # pylint: disable=protected-access weights=[ self.kernel[:, :self.units], self.kernel[:, self.units:self.units * 2], @@ -495,7 +491,7 @@ class CuDNNLSTM(_CuDNNRNN): self.bias[self.units * 6:self.units * 7], self.bias[self.units * 7:], ], - ) + shape=self._vector_shape) outputs, h, c, _ = gen_cudnn_rnn_ops.cudnn_rnn( inputs, diff --git a/tensorflow/python/keras/layers/embeddings.py b/tensorflow/python/keras/layers/embeddings.py index 5d805ea684f..e8a8575705a 100644 --- a/tensorflow/python/keras/layers/embeddings.py +++ b/tensorflow/python/keras/layers/embeddings.py @@ -45,11 +45,11 @@ class Embedding(Layer): model = Sequential() model.add(Embedding(1000, 64, input_length=10)) # the model will take as input an integer matrix of size (batch, - input_length). + # input_length). # the largest integer (i.e. word index) in the input should be no larger - than 999 (vocabulary size). + # than 999 (vocabulary size). # now model.output_shape == (None, 10, 64), where None is the batch - dimension. + # dimension. input_array = np.random.randint(1000, size=(32, 10)) @@ -116,7 +116,6 @@ class Embedding(Layer): self.mask_zero = mask_zero self.supports_masking = mask_zero self.input_length = input_length - self._can_use_graph_functions = True @tf_utils.shape_type_conversion def build(self, input_shape): diff --git a/tensorflow/python/keras/layers/local.py b/tensorflow/python/keras/layers/local.py index 30b83eaf50c..d2c4aaa125e 100644 --- a/tensorflow/python/keras/layers/local.py +++ b/tensorflow/python/keras/layers/local.py @@ -23,8 +23,8 @@ from tensorflow.python.keras import backend as K from tensorflow.python.keras import constraints from tensorflow.python.keras import initializers from tensorflow.python.keras import regularizers -from tensorflow.python.keras.engine.base_layer import InputSpec from tensorflow.python.keras.engine.base_layer import Layer +from tensorflow.python.keras.engine.input_spec import InputSpec from tensorflow.python.keras.utils import conv_utils from tensorflow.python.keras.utils import tf_utils from tensorflow.python.util.tf_export import tf_export @@ -154,7 +154,6 @@ class LocallyConnected1D(Layer): self.bias_constraint = constraints.get(bias_constraint) self.implementation = implementation self.input_spec = InputSpec(ndim=3) - self._can_use_graph_functions = True @tf_utils.shape_type_conversion def build(self, input_shape): @@ -430,7 +429,6 @@ class LocallyConnected2D(Layer): self.bias_constraint = constraints.get(bias_constraint) self.implementation = implementation self.input_spec = InputSpec(ndim=4) - self._can_use_graph_functions = True @tf_utils.shape_type_conversion def build(self, input_shape): diff --git a/tensorflow/python/keras/layers/local_test.py b/tensorflow/python/keras/layers/local_test.py index 300e7c96545..e4f4d0a639a 100644 --- a/tensorflow/python/keras/layers/local_test.py +++ b/tensorflow/python/keras/layers/local_test.py @@ -28,39 +28,43 @@ from tensorflow.python.training.rmsprop import RMSPropOptimizer class LocallyConnected1DLayersTest(test.TestCase): + # TODO(fchollet): investigate why LocallyConnected1D + # fails inside a graph function in an eager context (fails with error + # "Incompatible shapes between op input and calculated input gradient"). - @tf_test_util.run_in_graph_and_eager_modes + @tf_test_util.run_deprecated_v1 def test_locallyconnected_1d(self): - num_samples = 2 - num_steps = 8 - input_dim = 5 - filter_length = 3 - filters = 4 + with self.cached_session(): + num_samples = 2 + num_steps = 8 + input_dim = 5 + filter_length = 3 + filters = 4 - for padding in ['valid', 'same']: - for strides in [1]: - if padding == 'same' and strides != 1: - continue - for data_format in ['channels_first', 'channels_last']: - for implementation in [1, 2]: - kwargs = { - 'filters': filters, - 'kernel_size': filter_length, - 'padding': padding, - 'strides': strides, - 'data_format': data_format, - 'implementation': implementation - } + for padding in ['valid', 'same']: + for strides in [1]: + if padding == 'same' and strides != 1: + continue + for data_format in ['channels_first', 'channels_last']: + for implementation in [1, 2]: + kwargs = { + 'filters': filters, + 'kernel_size': filter_length, + 'padding': padding, + 'strides': strides, + 'data_format': data_format, + 'implementation': implementation + } - if padding == 'same' and implementation == 1: - self.assertRaises(ValueError, - keras.layers.LocallyConnected1D, - **kwargs) - else: - testing_utils.layer_test( - keras.layers.LocallyConnected1D, - kwargs=kwargs, - input_shape=(num_samples, num_steps, input_dim)) + if padding == 'same' and implementation == 1: + self.assertRaises(ValueError, + keras.layers.LocallyConnected1D, + **kwargs) + else: + testing_utils.layer_test( + keras.layers.LocallyConnected1D, + kwargs=kwargs, + input_shape=(num_samples, num_steps, input_dim)) def test_locallyconnected_1d_regularization(self): num_samples = 2 @@ -113,30 +117,63 @@ class LocallyConnected1DLayersTest(test.TestCase): class LocallyConnected2DLayersTest(test.TestCase): + # TODO(fchollet): investigate why LocallyConnected2D + # fails inside a graph function in an eager context (fails with error + # "Incompatible shapes between op input and calculated input gradient"). - @tf_test_util.run_in_graph_and_eager_modes + @tf_test_util.run_deprecated_v1 def test_locallyconnected_2d(self): - num_samples = 8 - filters = 3 - stack_size = 4 - num_row = 6 - num_col = 10 + with self.cached_session(): + num_samples = 8 + filters = 3 + stack_size = 4 + num_row = 6 + num_col = 10 - for padding in ['valid', 'same']: - for strides in [(1, 1), (2, 2)]: - for implementation in [1, 2]: - if padding == 'same' and strides != (1, 1): - continue + for padding in ['valid', 'same']: + for strides in [(1, 1), (2, 2)]: + for implementation in [1, 2]: + if padding == 'same' and strides != (1, 1): + continue + kwargs = { + 'filters': filters, + 'kernel_size': 3, + 'padding': padding, + 'kernel_regularizer': 'l2', + 'bias_regularizer': 'l2', + 'strides': strides, + 'data_format': 'channels_last', + 'implementation': implementation + } + + if padding == 'same' and implementation == 1: + self.assertRaises(ValueError, + keras.layers.LocallyConnected2D, + **kwargs) + else: + testing_utils.layer_test( + keras.layers.LocallyConnected2D, + kwargs=kwargs, + input_shape=(num_samples, num_row, num_col, stack_size)) + + @tf_test_util.run_deprecated_v1 + def test_locallyconnected_2d_channels_first(self): + with self.cached_session(): + num_samples = 8 + filters = 3 + stack_size = 4 + num_row = 6 + num_col = 10 + + for implementation in [1, 2]: + for padding in ['valid', 'same']: kwargs = { 'filters': filters, 'kernel_size': 3, - 'padding': padding, - 'kernel_regularizer': 'l2', - 'bias_regularizer': 'l2', - 'strides': strides, - 'data_format': 'channels_last', - 'implementation': implementation + 'data_format': 'channels_first', + 'implementation': implementation, + 'padding': padding } if padding == 'same' and implementation == 1: @@ -149,34 +186,6 @@ class LocallyConnected2DLayersTest(test.TestCase): kwargs=kwargs, input_shape=(num_samples, num_row, num_col, stack_size)) - @tf_test_util.run_in_graph_and_eager_modes - def test_locallyconnected_2d_channels_first(self): - num_samples = 8 - filters = 3 - stack_size = 4 - num_row = 6 - num_col = 10 - - for implementation in [1, 2]: - for padding in ['valid', 'same']: - kwargs = { - 'filters': filters, - 'kernel_size': 3, - 'data_format': 'channels_first', - 'implementation': implementation, - 'padding': padding - } - - if padding == 'same' and implementation == 1: - self.assertRaises(ValueError, - keras.layers.LocallyConnected2D, - **kwargs) - else: - testing_utils.layer_test( - keras.layers.LocallyConnected2D, - kwargs=kwargs, - input_shape=(num_samples, num_row, num_col, stack_size)) - def test_locallyconnected_2d_regularization(self): num_samples = 2 filters = 3 @@ -226,64 +235,67 @@ class LocallyConnected2DLayersTest(test.TestCase): class LocallyConnectedImplementationModeTest(test.TestCase): - @tf_test_util.run_in_graph_and_eager_modes + @tf_test_util.run_deprecated_v1 def test_locallyconnected_implementation(self): - num_samples = 4 - num_classes = 3 - num_epochs = 2 + with self.cached_session(): + num_samples = 4 + num_classes = 3 + num_epochs = 2 - np.random.seed(1) - targets = np.random.randint(0, num_classes, (num_samples,)) + np.random.seed(1) + targets = np.random.randint(0, num_classes, (num_samples,)) - for width in [1, 6]: - for height in [7]: - for filters in [2]: - for data_format in ['channels_first', 'channels_last']: - inputs = get_inputs( - data_format, filters, height, num_samples, width) + for width in [1, 6]: + for height in [7]: + for filters in [2]: + for data_format in ['channels_first', 'channels_last']: + inputs = get_inputs( + data_format, filters, height, num_samples, width) - for kernel_x in [(3,)]: - for kernel_y in [()] if width == 1 else [(2,)]: - for stride_x in [(1,)]: - for stride_y in [()] if width == 1 else [(3,)]: - for layers in [2]: - kwargs = { - 'layers': layers, - 'filters': filters, - 'kernel_size': kernel_x + kernel_y, - 'strides': stride_x + stride_y, - 'data_format': data_format, - 'num_classes': num_classes, - 'input_shape': inputs.shape - } + for kernel_x in [(3,)]: + for kernel_y in [()] if width == 1 else [(2,)]: + for stride_x in [(1,)]: + for stride_y in [()] if width == 1 else [(3,)]: + for layers in [2]: + kwargs = { + 'layers': layers, + 'filters': filters, + 'kernel_size': kernel_x + kernel_y, + 'strides': stride_x + stride_y, + 'data_format': data_format, + 'num_classes': num_classes + } + model_1 = get_model(implementation=1, **kwargs) + model_2 = get_model(implementation=2, **kwargs) - model_1 = get_model(implementation=1, **kwargs) - model_2 = get_model(implementation=2, **kwargs) + # Build models. + model_1.train_on_batch(inputs, targets) + model_2.train_on_batch(inputs, targets) - copy_model_weights(model_2, model_1) + # Copy weights. + copy_model_weights(model_2, model_1) - # Compare outputs at initialization. - out_1 = model_1.call(inputs) - out_2 = model_2.call(inputs) - self.assertAllCloseAccordingToType(out_1, out_2, - rtol=1e-5, atol=1e-5) + # Compare outputs at initialization. + out_1 = model_1.call(inputs) + out_2 = model_2.call(inputs) + self.assertAllCloseAccordingToType(out_1, out_2, + rtol=1e-5, atol=1e-5) - # Train. - model_1.fit(x=inputs, - y=targets, - epochs=num_epochs, - batch_size=num_samples) + # Train. + model_1.fit(x=inputs, + y=targets, + epochs=num_epochs, + batch_size=num_samples) + model_2.fit(x=inputs, + y=targets, + epochs=num_epochs, + batch_size=num_samples) - model_2.fit(x=inputs, - y=targets, - epochs=num_epochs, - batch_size=num_samples) - - # Compare outputs after a few training steps. - out_1 = model_1.call(inputs) - out_2 = model_2.call(inputs) - self.assertAllCloseAccordingToType(out_1, out_2, - rtol=1e-5, atol=1e-5) + # Compare outputs after a few training steps. + out_1 = model_1.call(inputs) + out_2 = model_2.call(inputs) + self.assertAllCloseAccordingToType( + out_1, out_2, atol=2e-4) @tf_test_util.run_in_graph_and_eager_modes def test_make_2d(self): @@ -360,8 +372,7 @@ def get_model(implementation, strides, layers, num_classes, - data_format, - input_shape): + data_format): model = keras.Sequential() if len(kernel_size) == 1: @@ -390,7 +401,6 @@ def get_model(implementation, metrics=[keras.metrics.categorical_accuracy], loss=xent ) - model.build(input_shape) return model diff --git a/tensorflow/python/keras/layers/lstm_test.py b/tensorflow/python/keras/layers/lstm_test.py index 9db697871fe..3f89cc398ed 100644 --- a/tensorflow/python/keras/layers/lstm_test.py +++ b/tensorflow/python/keras/layers/lstm_test.py @@ -18,6 +18,7 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function +from absl.testing import parameterized import numpy as np from tensorflow.python import keras @@ -30,7 +31,7 @@ from tensorflow.python.training.rmsprop import RMSPropOptimizer @tf_test_util.run_all_in_graph_and_eager_modes -class LSTMLayerTest(test.TestCase): +class LSTMLayerTest(test.TestCase, parameterized.TestCase): def test_return_sequences_LSTM(self): num_samples = 2 @@ -56,7 +57,7 @@ class LSTMLayerTest(test.TestCase): layer = keras.layers.LSTM(units, return_sequences=True) model.add(layer) outputs = model.layers[-1].output - self.assertEquals(outputs.get_shape().as_list(), [None, timesteps, units]) + self.assertEqual(outputs.get_shape().as_list(), [None, timesteps, units]) def test_dynamic_behavior_LSTM(self): num_samples = 2 @@ -83,17 +84,17 @@ class LSTMLayerTest(test.TestCase): 'recurrent_dropout': 0.1}, input_shape=(num_samples, timesteps, embedding_dim)) - def test_implementation_mode_LSTM(self): + @parameterized.parameters([0, 1, 2]) + def test_implementation_mode_LSTM(self, implementation_mode): num_samples = 2 timesteps = 3 embedding_dim = 4 units = 2 - for mode in [0, 1, 2]: - testing_utils.layer_test( - keras.layers.LSTM, - kwargs={'units': units, - 'implementation': mode}, - input_shape=(num_samples, timesteps, embedding_dim)) + testing_utils.layer_test( + keras.layers.LSTM, + kwargs={'units': units, + 'implementation': implementation_mode}, + input_shape=(num_samples, timesteps, embedding_dim)) def test_constraints_LSTM(self): embedding_dim = 4 @@ -114,6 +115,7 @@ class LSTMLayerTest(test.TestCase): self.assertEqual(layer.cell.recurrent_kernel.constraint, r_constraint) self.assertEqual(layer.cell.bias.constraint, b_constraint) + @tf_test_util.run_deprecated_v1 def test_with_masking_layer_LSTM(self): layer_class = keras.layers.LSTM inputs = np.random.random((2, 3, 4)) @@ -126,6 +128,7 @@ class LSTMLayerTest(test.TestCase): optimizer=RMSPropOptimizer(0.01)) model.fit(inputs, targets, epochs=1, batch_size=2, verbose=1) + @tf_test_util.run_deprecated_v1 def test_masking_with_stacking_LSTM(self): inputs = np.random.random((2, 3, 4)) targets = np.abs(np.random.random((2, 3, 5))) @@ -311,6 +314,7 @@ class LSTMLayerTest(test.TestCase): class LSTMLayerGraphOnlyTest(test.TestCase): + @tf_test_util.run_deprecated_v1 def test_statefulness_LSTM(self): num_samples = 2 timesteps = 3 @@ -374,6 +378,7 @@ class LSTMLayerGraphOnlyTest(test.TestCase): self.assertAllClose(out7, out6, atol=1e-5) + @tf_test_util.run_deprecated_v1 def test_regularizers_LSTM(self): embedding_dim = 4 layer_class = keras.layers.LSTM diff --git a/tensorflow/python/keras/layers/merge.py b/tensorflow/python/keras/layers/merge.py index 0ded0e42ed3..45e705c6960 100644 --- a/tensorflow/python/keras/layers/merge.py +++ b/tensorflow/python/keras/layers/merge.py @@ -40,7 +40,6 @@ class _Merge(Layer): def __init__(self, **kwargs): super(_Merge, self).__init__(**kwargs) - self._can_use_graph_functions = True self.supports_masking = True def _merge_function(self, inputs): @@ -213,7 +212,7 @@ class _Merge(Layer): if len(mask) != len(inputs): raise ValueError('The lists `inputs` and `mask` ' 'should have the same length.') - if all([m is None for m in mask]): + if all(m is None for m in mask): return None masks = [array_ops.expand_dims(m, axis=0) for m in mask if m is not None] return K.all(K.concatenate(masks, axis=0), axis=0, keepdims=False) @@ -369,7 +368,6 @@ class Concatenate(_Merge): def __init__(self, axis=-1, **kwargs): super(Concatenate, self).__init__(**kwargs) - self._can_use_graph_functions = True self.axis = axis self.supports_masking = True self._reshape_required = False @@ -380,7 +378,7 @@ class Concatenate(_Merge): if not isinstance(input_shape, list) or len(input_shape) < 2: raise ValueError('A `Concatenate` layer should be called ' 'on a list of at least 2 inputs') - if all([shape is None for shape in input_shape]): + if all(shape is None for shape in input_shape): return reduced_inputs_shapes = [list(shape) for shape in input_shape] shape_set = set() @@ -420,7 +418,7 @@ class Concatenate(_Merge): if len(mask) != len(inputs): raise ValueError('The lists `inputs` and `mask` ' 'should have the same length.') - if all([m is None for m in mask]): + if all(m is None for m in mask): return None # Make a list of masks while making sure # the dimensionality of each mask @@ -467,7 +465,6 @@ class Dot(_Merge): def __init__(self, axes, normalize=False, **kwargs): super(Dot, self).__init__(**kwargs) - self._can_use_graph_functions = True if not isinstance(axes, int): if not isinstance(axes, (list, tuple)): raise TypeError('Invalid type for `axes` - ' diff --git a/tensorflow/python/keras/layers/merge_test.py b/tensorflow/python/keras/layers/merge_test.py index 698c5662b6f..fcb161ae20a 100644 --- a/tensorflow/python/keras/layers/merge_test.py +++ b/tensorflow/python/keras/layers/merge_test.py @@ -207,6 +207,7 @@ class MergeLayersGraphOnlyTest(test.TestCase): mask = layer.output_mask self.assertListEqual(mask.get_shape().as_list(), [None, 4]) + @tf_test_util.run_deprecated_v1 def test_merge_add_dynamic_shape(self): with self.cached_session(): i1 = array_ops.placeholder(shape=(4, None), dtype='float32') diff --git a/tensorflow/python/keras/layers/noise.py b/tensorflow/python/keras/layers/noise.py index e7c0478513d..cb7cee3ebc3 100644 --- a/tensorflow/python/keras/layers/noise.py +++ b/tensorflow/python/keras/layers/noise.py @@ -55,7 +55,6 @@ class GaussianNoise(Layer): super(GaussianNoise, self).__init__(**kwargs) self.supports_masking = True self.stddev = stddev - self._can_use_graph_functions = True def call(self, inputs, training=None): @@ -100,7 +99,6 @@ class GaussianDropout(Layer): super(GaussianDropout, self).__init__(**kwargs) self.supports_masking = True self.rate = rate - self._can_use_graph_functions = True def call(self, inputs, training=None): if 0 < self.rate < 1: @@ -155,7 +153,6 @@ class AlphaDropout(Layer): self.noise_shape = noise_shape self.seed = seed self.supports_masking = True - self._can_use_graph_functions = True def _get_noise_shape(self, inputs): return self.noise_shape if self.noise_shape else array_ops.shape(inputs) diff --git a/tensorflow/python/keras/layers/normalization.py b/tensorflow/python/keras/layers/normalization.py index 3d3bf647e6f..d9584976555 100644 --- a/tensorflow/python/keras/layers/normalization.py +++ b/tensorflow/python/keras/layers/normalization.py @@ -18,6 +18,7 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function +from tensorflow.python import tf2 from tensorflow.python.eager import context from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops @@ -26,8 +27,8 @@ from tensorflow.python.keras import backend as K from tensorflow.python.keras import constraints from tensorflow.python.keras import initializers from tensorflow.python.keras import regularizers -from tensorflow.python.keras.engine.base_layer import InputSpec from tensorflow.python.keras.engine.base_layer import Layer +from tensorflow.python.keras.engine.input_spec import InputSpec from tensorflow.python.keras.utils import tf_utils from tensorflow.python.ops import array_ops from tensorflow.python.ops import init_ops @@ -40,8 +41,8 @@ from tensorflow.python.training import distribution_strategy_context from tensorflow.python.util.tf_export import tf_export -@tf_export('keras.layers.BatchNormalization') -class BatchNormalization(Layer): +@tf_export('keras.layers.BatchNormalization', v1=[]) +class BatchNormalizationV2(Layer): """Batch normalization layer (Ioffe and Szegedy, 2014). Normalize the activations of the previous layer at each batch, @@ -84,8 +85,10 @@ class BatchNormalization(Layer): and should be neither too small (which would add noise) nor too large (which would give stale estimates). Note that `momentum` is still applied to get the means and variances for inference. - fused: if `None` or `True`, use a faster, fused implementation if possible. - If `False`, use the system recommended implementation. + fused: if `True`, use a faster, fused implementation, or raise a ValueError + if the fused implementation cannot be used. If `None`, use the faster + implementation if possible. If False, do not used the fused + implementation. trainable: Boolean, if `True` also add variables to the graph collection `GraphKeys.TRAINABLE_VARIABLES` (see tf.Variable). virtual_batch_size: An `int`. By default, `virtual_batch_size` is `None`, @@ -120,6 +123,9 @@ class BatchNormalization(Layer): Internal Covariate Shift](https://arxiv.org/abs/1502.03167) """ + # The BatchNormalizationV1 subclass sets this to False to use the V1 behavior. + _USE_V2_BEHAVIOR = True + def __init__(self, axis=-1, momentum=0.99, @@ -143,13 +149,15 @@ class BatchNormalization(Layer): adjustment=None, name=None, **kwargs): - super(BatchNormalization, self).__init__( + super(BatchNormalizationV2, self).__init__( name=name, trainable=trainable, **kwargs) - self._can_use_graph_functions = True if isinstance(axis, list): self.axis = axis[:] - else: + elif isinstance(axis, int): self.axis = axis + else: + raise TypeError('axis must be int or list, type given: %s' + % type(self.axis)) self.momentum = momentum self.epsilon = epsilon self.center = center @@ -166,7 +174,14 @@ class BatchNormalization(Layer): self.renorm = renorm self.virtual_batch_size = virtual_batch_size self.adjustment = adjustment - if fused is None: + if self._USE_V2_BEHAVIOR: + if fused: + self._raise_if_fused_cannot_be_used() + # We leave fused as None if self._fused_can_be_used()==True, since we + # still may set it to False in self.build() if the input rank is not 4. + elif fused is None and not self._fused_can_be_used(): + fused = False + elif fused is None: fused = True self.supports_masking = True @@ -182,6 +197,38 @@ class BatchNormalization(Layer): self.renorm_clipping = renorm_clipping self.renorm_momentum = renorm_momentum + def _raise_if_fused_cannot_be_used(self): + """Raises a ValueError if fused implementation cannot be used. + + In addition to the checks done in this function, the input tensors rank must + be 4. The input rank check can only be done once the input shape is known. + """ + # Currently fused batch norm doesn't support renorm. It also only supports a + # channel dimension on axis 1 or 3, when no virtual batch size or adjustment + # is used. + if self.renorm: + raise ValueError('Passing both fused=True and renorm=True is ' + 'unsupported') + axis = [self.axis] if isinstance(self.axis, int) else self.axis + # Axis -3 is equivalent to 1, and axis -1 is equivalent to 3, because the + # input rank is required to be 4 (which is checked later). + if len(axis) > 1 or axis[0] not in (-3, -1, 1, 3): + raise ValueError('Passing fused=True is only supported when axis is 1 ' + 'or 3') + if self.virtual_batch_size is not None: + raise ValueError('Passing fused=True is unsupported when ' + 'virtual_batch_size is specified.') + if self.adjustment is not None: + raise ValueError('Passing fused=True is unsupported when ' + 'adjustment is specified.') + + def _fused_can_be_used(self): + try: + self._raise_if_fused_cannot_be_used() + return True + except ValueError: + return False + def build(self, input_shape): input_shape = tensor_shape.TensorShape(input_shape) if not input_shape.ndims: @@ -192,10 +239,6 @@ class BatchNormalization(Layer): if isinstance(self.axis, int): self.axis = [self.axis] - if not isinstance(self.axis, list): - raise TypeError('axis must be int or list, type given: %s' - % type(self.axis)) - for idx, x in enumerate(self.axis): if x < 0: self.axis[idx] = ndims + x @@ -220,16 +263,18 @@ class BatchNormalization(Layer): raise ValueError('When using virtual_batch_size, adjustment cannot ' 'be specified') - if self.fused: - # Currently fused batch norm doesn't support renorm. It also only supports - # an input tensor of rank 4 and a channel dimension on axis 1 or 3. + if self.fused in (None, True): # TODO(yaozhang): if input is not 4D, reshape it to 4D and reshape the # output back to its original shape accordingly. - self.fused = (not self.renorm and - ndims == 4 and - self.axis in [[1], [3]] and - self.virtual_batch_size is None and - self.adjustment is None) + if self._USE_V2_BEHAVIOR: + if self.fused is None: + self.fused = (ndims == 4) + elif self.fused and ndims != 4: + raise ValueError('Batch normalization layers with fused=True only ' + 'support 4D input tensors.') + else: + assert self.fused is not None + self.fused = (ndims == 4 and self._fused_can_be_used()) # TODO(chrisying): fused batch norm is currently not supported for # multi-axis batch norm and by extension virtual batches. In some cases, # it might be possible to use fused batch norm but would require reshaping @@ -492,6 +537,9 @@ class BatchNormalization(Layer): return (r, d, new_mean, new_variance) + def _moments(self, inputs, reduction_axes, keep_dims): + return nn.moments(inputs, reduction_axes, keep_dims=keep_dims) + def call(self, inputs, training=None): if training is None: training = K.learning_phase() @@ -563,7 +611,8 @@ class BatchNormalization(Layer): # Some of the computations here are not necessary when training==False # but not a constant. However, this makes the code simpler. keep_dims = self.virtual_batch_size is not None or len(self.axis) > 1 - mean, variance = nn.moments(inputs, reduction_axes, keep_dims=keep_dims) + mean, variance = self._moments( + inputs, reduction_axes, keep_dims=keep_dims) moving_mean = self.moving_mean moving_variance = self.moving_variance @@ -669,5 +718,36 @@ class BatchNormalization(Layer): 'layer cannot be serialized and has been omitted from ' 'the layer config. It will not be included when ' 're-creating the layer from the saved config.') - base_config = super(BatchNormalization, self).get_config() + base_config = super(BatchNormalizationV2, self).get_config() return dict(list(base_config.items()) + list(config.items())) + + +def _replace_in_v2_docstring(old, new): + string = BatchNormalizationV2.__doc__ + if old not in string: + raise ValueError('Could not find following string in BatchNormalizationV2 ' + 'docstring: "{}"'.format(old)) + return string.replace(old, new) + + +@tf_export(v1=['keras.layers.BatchNormalization']) # pylint: disable=missing-docstring +class BatchNormalizationV1(BatchNormalizationV2): + + __doc__ = _replace_in_v2_docstring( + ''' + fused: if `True`, use a faster, fused implementation, or raise a ValueError + if the fused implementation cannot be used. If `None`, use the faster + implementation if possible. If False, do not used the fused + implementation.''', + + ''' + fused: if `None` or `True`, use a faster, fused implementation if possible. + If `False`, use the system recommended implementation.''') + + _USE_V2_BEHAVIOR = False + + +if tf2.enabled(): + BatchNormalization = BatchNormalizationV2 +else: + BatchNormalization = BatchNormalizationV1 diff --git a/tensorflow/python/keras/layers/normalization_test.py b/tensorflow/python/keras/layers/normalization_test.py index 92e41287077..9138c0a08a3 100644 --- a/tensorflow/python/keras/layers/normalization_test.py +++ b/tensorflow/python/keras/layers/normalization_test.py @@ -23,6 +23,7 @@ import numpy as np from tensorflow.python import keras from tensorflow.python.framework import test_util as tf_test_util from tensorflow.python.keras import testing_utils +from tensorflow.python.keras.layers import normalization from tensorflow.python.platform import test from tensorflow.python.training import gradient_descent @@ -54,6 +55,14 @@ class NormalizationLayersTest(test.TestCase): kwargs={'scale': False, 'center': False}, input_shape=(3, 3)) + testing_utils.layer_test( + normalization.BatchNormalizationV2, + kwargs={'fused': True}, + input_shape=(3, 3, 3, 3)) + testing_utils.layer_test( + normalization.BatchNormalizationV2, + kwargs={'fused': None}, + input_shape=(3, 3, 3)) def test_batchnorm_weights(self): layer = keras.layers.BatchNormalization(scale=False, center=False) @@ -78,15 +87,18 @@ class NormalizationLayersTest(test.TestCase): self.assertEqual(layer.gamma.constraint, max_norm) self.assertEqual(layer.beta.constraint, max_norm) - def test_batchnorm_correctness(self): + def _test_batchnorm_correctness(self, dtype, use_v2=True, fused=False): model = keras.models.Sequential() - norm = keras.layers.BatchNormalization(input_shape=(10,), momentum=0.8) + layer_ctor = (normalization.BatchNormalizationV2 if use_v2 + else normalization.BatchNormalizationV1) + norm = layer_ctor(input_shape=(2, 2, 2), momentum=0.8, fused=fused) model.add(norm) model.compile(loss='mse', optimizer=gradient_descent.GradientDescentOptimizer(0.01)) # centered on 5.0, variance 10.0 - x = np.random.normal(loc=5.0, scale=10.0, size=(1000, 10)) + x = (np.random.normal(loc=5.0, scale=10.0, size=(1000, 2, 2, 2)) + .astype(dtype)) model.fit(x, x, epochs=4, verbose=0) out = model.predict(x) out -= keras.backend.eval(norm.beta) @@ -95,23 +107,15 @@ class NormalizationLayersTest(test.TestCase): np.testing.assert_allclose(out.mean(), 0.0, atol=1e-1) np.testing.assert_allclose(out.std(), 1.0, atol=1e-1) + def test_batchnorm_correctness(self): + self._test_batchnorm_correctness(np.float32) + self._test_batchnorm_correctness(np.float32, fused=True) + self._test_batchnorm_correctness(np.float32, use_v2=False) + def test_batchnorm_mixed_precision(self): - model = keras.models.Sequential() - norm = keras.layers.BatchNormalization(input_shape=(10,), momentum=0.8) - model.add(norm) - model.compile(loss='mse', - optimizer=gradient_descent.GradientDescentOptimizer(0.01)) - - # centered on 5.0, variance 10.0 - x = np.random.normal( - loc=5.0, scale=10.0, size=(1000, 10)).astype(np.float16) - model.fit(x, x, epochs=4, verbose=0) - out = model.predict(x) - out -= keras.backend.eval(norm.beta) - out /= keras.backend.eval(norm.gamma) - - np.testing.assert_allclose(out.mean(), 0.0, atol=1e-1) - np.testing.assert_allclose(out.std(), 1.0, atol=1e-1) + self._test_batchnorm_correctness(np.float16) + self._test_batchnorm_correctness(np.float16, fused=True) + self._test_batchnorm_correctness(np.float16, use_v2=False) def test_batchnorm_convnet(self): if test.is_gpu_available(cuda_only=True): @@ -151,6 +155,77 @@ class NormalizationLayersTest(test.TestCase): np.testing.assert_allclose(np.mean(out, axis=(0, 1, 2)), 0.0, atol=1e-1) np.testing.assert_allclose(np.std(out, axis=(0, 1, 2)), 1.0, atol=1e-1) + def test_v1_fused_attribute(self): + norm = normalization.BatchNormalizationV1() + inp = keras.layers.Input((4, 4, 4)) + norm(inp) + self.assertEqual(norm.fused, True) + + norm = normalization.BatchNormalizationV1(fused=False) + self.assertEqual(norm.fused, False) + inp = keras.layers.Input(shape=(4, 4, 4)) + norm(inp) + self.assertEqual(norm.fused, False) + + norm = normalization.BatchNormalizationV1(virtual_batch_size=2) + self.assertEqual(norm.fused, True) + inp = keras.layers.Input(shape=(2, 2, 2)) + norm(inp) + self.assertEqual(norm.fused, False) + + def test_v2_fused_attribute(self): + norm = normalization.BatchNormalizationV2() + self.assertEqual(norm.fused, None) + inp = keras.layers.Input(shape=(4, 4, 4)) + norm(inp) + self.assertEqual(norm.fused, True) + + norm = normalization.BatchNormalizationV2() + self.assertEqual(norm.fused, None) + inp = keras.layers.Input(shape=(4, 4)) + norm(inp) + self.assertEqual(norm.fused, False) + + norm = normalization.BatchNormalizationV2(virtual_batch_size=2) + self.assertEqual(norm.fused, False) + inp = keras.layers.Input(shape=(4, 4, 4)) + norm(inp) + self.assertEqual(norm.fused, False) + + norm = normalization.BatchNormalizationV2(fused=False) + self.assertEqual(norm.fused, False) + inp = keras.layers.Input(shape=(4, 4, 4)) + norm(inp) + self.assertEqual(norm.fused, False) + + norm = normalization.BatchNormalizationV2(fused=True, axis=[3]) + self.assertEqual(norm.fused, True) + inp = keras.layers.Input(shape=(4, 4, 4)) + norm(inp) + self.assertEqual(norm.fused, True) + + with self.assertRaisesRegexp(ValueError, 'fused.*renorm'): + normalization.BatchNormalizationV2(fused=True, renorm=True) + + with self.assertRaisesRegexp(ValueError, 'fused.*when axis is 1 or 3'): + normalization.BatchNormalizationV2(fused=True, axis=2) + + with self.assertRaisesRegexp(ValueError, 'fused.*when axis is 1 or 3'): + normalization.BatchNormalizationV2(fused=True, axis=[1, 3]) + + with self.assertRaisesRegexp(ValueError, 'fused.*virtual_batch_size'): + normalization.BatchNormalizationV2(fused=True, virtual_batch_size=2) + + with self.assertRaisesRegexp(ValueError, 'fused.*adjustment'): + normalization.BatchNormalizationV2(fused=True, + adjustment=lambda _: (1, 0)) + + norm = normalization.BatchNormalizationV2(fused=True) + self.assertEqual(norm.fused, True) + inp = keras.layers.Input(shape=(4, 4)) + with self.assertRaisesRegexp(ValueError, '4D input tensors'): + norm(inp) + class NormalizationLayersGraphModeOnlyTest(test.TestCase): @@ -226,6 +301,7 @@ class NormalizationLayersGraphModeOnlyTest(test.TestCase): x2 = model.predict(val_a) self.assertAllClose(x1, x2, atol=1e-7) + @tf_test_util.run_deprecated_v1 def test_batchnorm_trainable(self): """Tests that batchnorm layer is trainable when learning phase is enabled. diff --git a/tensorflow/python/keras/layers/pooling.py b/tensorflow/python/keras/layers/pooling.py index b8d6b03664f..a0744cddad6 100644 --- a/tensorflow/python/keras/layers/pooling.py +++ b/tensorflow/python/keras/layers/pooling.py @@ -22,8 +22,8 @@ import functools from tensorflow.python.framework import tensor_shape from tensorflow.python.keras import backend -from tensorflow.python.keras.engine.base_layer import InputSpec from tensorflow.python.keras.engine.base_layer import Layer +from tensorflow.python.keras.engine.input_spec import InputSpec from tensorflow.python.keras.utils import conv_utils from tensorflow.python.ops import array_ops from tensorflow.python.ops import math_ops @@ -58,7 +58,6 @@ class Pooling1D(Layer): padding='valid', data_format='channels_last', name=None, **kwargs): super(Pooling1D, self).__init__(name=name, **kwargs) - self._can_use_graph_functions = True if data_format is None: data_format = backend.image_data_format() if strides is None: @@ -231,7 +230,6 @@ class Pooling2D(Layer): padding='valid', data_format=None, name=None, **kwargs): super(Pooling2D, self).__init__(name=name, **kwargs) - self._can_use_graph_functions = True if data_format is None: data_format = backend.image_data_format() if strides is None: @@ -427,7 +425,6 @@ class Pooling3D(Layer): padding='valid', data_format='channels_last', name=None, **kwargs): super(Pooling3D, self).__init__(name=name, **kwargs) - self._can_use_graph_functions = True if data_format is None: data_format = backend.image_data_format() if strides is None: @@ -599,7 +596,6 @@ class GlobalPooling1D(Layer): def __init__(self, data_format='channels_last', **kwargs): super(GlobalPooling1D, self).__init__(**kwargs) - self._can_use_graph_functions = True self.input_spec = InputSpec(ndim=3) self.data_format = conv_utils.normalize_data_format(data_format) @@ -705,7 +701,6 @@ class GlobalPooling2D(Layer): def __init__(self, data_format=None, **kwargs): super(GlobalPooling2D, self).__init__(**kwargs) - self._can_use_graph_functions = True self.data_format = conv_utils.normalize_data_format(data_format) self.input_spec = InputSpec(ndim=4) @@ -804,7 +799,6 @@ class GlobalPooling3D(Layer): def __init__(self, data_format=None, **kwargs): super(GlobalPooling3D, self).__init__(**kwargs) - self._can_use_graph_functions = True self.data_format = conv_utils.normalize_data_format(data_format) self.input_spec = InputSpec(ndim=5) diff --git a/tensorflow/python/keras/layers/recurrent.py b/tensorflow/python/keras/layers/recurrent.py index 979187c719f..d9502dfc5b7 100644 --- a/tensorflow/python/keras/layers/recurrent.py +++ b/tensorflow/python/keras/layers/recurrent.py @@ -19,20 +19,27 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function +import uuid + import numpy as np from tensorflow.python.eager import context +from tensorflow.python.eager import function +from tensorflow.python.framework import constant_op +from tensorflow.python.framework import dtypes +from tensorflow.python.framework import ops from tensorflow.python.framework import tensor_shape from tensorflow.python.keras import activations from tensorflow.python.keras import backend as K from tensorflow.python.keras import constraints from tensorflow.python.keras import initializers from tensorflow.python.keras import regularizers -from tensorflow.python.keras.engine.base_layer import InputSpec from tensorflow.python.keras.engine.base_layer import Layer +from tensorflow.python.keras.engine.input_spec import InputSpec from tensorflow.python.keras.utils import generic_utils from tensorflow.python.keras.utils import tf_utils from tensorflow.python.ops import array_ops +from tensorflow.python.ops import gen_cudnn_rnn_ops from tensorflow.python.ops import state_ops from tensorflow.python.platform import tf_logging as logging from tensorflow.python.training.checkpointable import base as checkpointable @@ -446,6 +453,9 @@ class RNN(Layer): 'an attribute `state_size` ' '(tuple of integers, ' 'one integer per RNN state).') + # If True, the output for masked timestep will be zeros, whereas in the + # False case, output from previous timestep is returned for masked timestep. + self.zero_output_for_mask = kwargs.pop('zero_output_for_mask', False) super(RNN, self).__init__(**kwargs) self.cell = cell if isinstance(cell, checkpointable.CheckpointableBase): @@ -743,35 +753,12 @@ class RNN(Layer): training=None, initial_state=None, constants=None): - # input shape: `(samples, time (padded with zeros), input_dim)` - # note that the .build() method of subclasses MUST define - # self.input_spec and self.state_spec with complete input shapes. - if isinstance(inputs, list): - # get initial_state from full input spec - # as they could be copied to multiple GPU. - if self._num_constants is None: - initial_state = inputs[1:] - else: - initial_state = inputs[1:-self._num_constants] - constants = inputs[-self._num_constants:] - if len(initial_state) == 0: - initial_state = None - inputs = inputs[0] - if initial_state is not None: - pass - elif self.stateful: - initial_state = self.states - else: - initial_state = self.get_initial_state(inputs) + inputs, initial_state, constants = self._process_inputs( + inputs, initial_state, constants) if isinstance(mask, list): mask = mask[0] - if len(initial_state) != len(self.states): - raise ValueError( - 'Layer has ' + str(len(self.states)) + ' states but was passed ' + - str(len(initial_state)) + ' initial states.') - if nest.is_sequence(inputs): # In the case of nested input, use the first element for shape check. input_shape = K.int_shape(nest.flatten(inputs)[0]) @@ -829,7 +816,8 @@ class RNN(Layer): mask=mask, unroll=self.unroll, input_length=timesteps, - time_major=self.time_major) + time_major=self.time_major, + zero_output_for_mask=self.zero_output_for_mask) if self.stateful: updates = [] for i in range(len(states)): @@ -850,6 +838,34 @@ class RNN(Layer): else: return output + def _process_inputs(self, inputs, initial_state, constants): + # input shape: `(samples, time (padded with zeros), input_dim)` + # note that the .build() method of subclasses MUST define + # self.input_spec and self.state_spec with complete input shapes. + if isinstance(inputs, list): + # get initial_state from full input spec + # as they could be copied to multiple GPU. + if self._num_constants is None: + initial_state = inputs[1:] + else: + initial_state = inputs[1:-self._num_constants] + constants = inputs[-self._num_constants:] + if len(initial_state) == 0: + initial_state = None + inputs = inputs[0] + if initial_state is not None: + pass + elif self.stateful: + initial_state = self.states + else: + initial_state = self.get_initial_state(inputs) + + if len(initial_state) != len(self.states): + raise ValueError('Layer has ' + str(len(self.states)) + + ' states but was passed ' + str(len(initial_state)) + + ' initial states.') + return inputs, initial_state, constants + def reset_states(self, states=None): if not self.stateful: raise AttributeError('Layer must be stateful.') @@ -923,6 +939,8 @@ class RNN(Layer): } if self._num_constants is not None: config['num_constants'] = self._num_constants + if self.zero_output_for_mask: + config['zero_output_for_mask'] = self.zero_output_for_mask cell_config = self.cell.get_config() config['cell'] = { @@ -2515,6 +2533,416 @@ class LSTM(RNN): return cls(**config) +class UnifiedLSTM(LSTM): + """Long Short-Term Memory layer - Hochreiter 1997. + + `UnifiedLSTM` unifies the implementations between standard `LSTM` layer and + `CuDNNLSTM` layer. Based on available runtime hardware and constrains, + `UnifiedLSTM` will choose different implementations to maximize the + performance. For instance, if GPU is available and all the parameters meet the + requirement of CuDNN kernel, `UnifiedLSTM` will use CuDNN kernel for the + calculation. + + Arguments: + units: Positive integer, dimensionality of the output space. + activation: Activation function to use. + Default: hyperbolic tangent (`tanh`). If you pass `None`, no activation + is applied + (ie. "linear" activation: `a(x) = x`). + recurrent_activation: Activation function to use for the recurrent step. + Default: hard sigmoid (`hard_sigmoid`). If you pass `None`, no + activation is applied + (ie. "linear" activation: `a(x) = x`). + use_bias: Boolean, whether the layer uses a bias vector. + kernel_initializer: Initializer for the `kernel` weights matrix, used for + the linear transformation of the inputs.. + recurrent_initializer: Initializer for the `recurrent_kernel` weights + matrix, used for the linear transformation of the recurrent state.. + bias_initializer: Initializer for the bias vector. + unit_forget_bias: Boolean. If True, add 1 to the bias of the forget gate at + initialization. Setting it to true will also force + `bias_initializer="zeros"`. This is recommended in [Jozefowicz et + al.](http://www.jmlr.org/proceedings/papers/v37/jozefowicz15.pdf) + kernel_regularizer: Regularizer function applied to the `kernel` weights + matrix. + recurrent_regularizer: Regularizer function applied to the + `recurrent_kernel` weights matrix. + bias_regularizer: Regularizer function applied to the bias vector. + activity_regularizer: Regularizer function applied to the output of the + layer (its "activation").. + kernel_constraint: Constraint function applied to the `kernel` weights + matrix. + recurrent_constraint: Constraint function applied to the `recurrent_kernel` + weights matrix. + bias_constraint: Constraint function applied to the bias vector. + dropout: Float between 0 and 1. Fraction of the units to drop for the linear + transformation of the inputs. + recurrent_dropout: Float between 0 and 1. Fraction of the units to drop for + the linear transformation of the recurrent state. + implementation: Implementation mode, either 1 or 2. Mode 1 will structure + its operations as a larger number of smaller dot products and additions, + whereas mode 2 will batch them into fewer, larger operations. These modes + will have different performance profiles on different hardware and for + different applications. + return_sequences: Boolean. Whether to return the last output. in the output + sequence, or the full sequence. + return_state: Boolean. Whether to return the last state in addition to the + output. + go_backwards: Boolean (default False). If True, process the input sequence + backwards and return the reversed sequence. + stateful: Boolean (default False). If True, the last state for each sample + at index i in a batch will be used as initial state for the sample of + index i in the following batch. + unroll: Boolean (default False). If True, the network will be unrolled, else + a symbolic loop will be used. Unrolling can speed-up a RNN, although it + tends to be more memory-intensive. Unrolling is only suitable for short + sequences. + """ + + def __init__(self, + units, + activation='tanh', + recurrent_activation='hard_sigmoid', + use_bias=True, + kernel_initializer='glorot_uniform', + recurrent_initializer='orthogonal', + bias_initializer='zeros', + unit_forget_bias=True, + kernel_regularizer=None, + recurrent_regularizer=None, + bias_regularizer=None, + activity_regularizer=None, + kernel_constraint=None, + recurrent_constraint=None, + bias_constraint=None, + dropout=0., + recurrent_dropout=0., + implementation=1, + return_sequences=False, + return_state=False, + go_backwards=False, + stateful=False, + time_major=False, + unroll=False, + **kwargs): + # return_runtime is a flag for testing, which shows the real backend + # implementation chosen by grappler in graph mode. + self.return_runtime = kwargs.pop('return_runtime', False) + + super(UnifiedLSTM, self).__init__( + units, + activation=activation, + recurrent_activation=recurrent_activation, + use_bias=use_bias, + kernel_initializer=kernel_initializer, + recurrent_initializer=recurrent_initializer, + bias_initializer=bias_initializer, + unit_forget_bias=unit_forget_bias, + kernel_regularizer=kernel_regularizer, + recurrent_regularizer=recurrent_regularizer, + bias_regularizer=bias_regularizer, + activity_regularizer=activity_regularizer, + kernel_constraint=kernel_constraint, + recurrent_constraint=recurrent_constraint, + bias_constraint=bias_constraint, + dropout=dropout, + recurrent_dropout=recurrent_dropout, + implementation=implementation, + return_sequences=return_sequences, + return_state=return_state, + go_backwards=go_backwards, + stateful=stateful, + time_major=time_major, + unroll=unroll, + **kwargs) + + self.state_spec = [ + InputSpec(shape=(None, dim)) for dim in (self.units, self.units) + ] + self._num_constants = None + self._num_inputs = None + self.could_use_cudnn = ( + activation == 'tanh' and dropout == 0 and not unroll and use_bias and + unit_forget_bias) + + def build(self, input_shape): + super(UnifiedLSTM, self).build(input_shape) + if self.could_use_cudnn: + # Add a new set of bias for CuDNN implementation only. Standard LSTM only + # has bias for recurrent kernel, while CuDNN LSTM has an extra set for + # input gate as well. + self.cudnn_bias = self.add_weight( + shape=(self.units * 4,), + name='cudnn_bias', + use_resource=True, + initializer=self.bias_initializer, + regularizer=self.bias_regularizer, + constraint=self.bias_constraint) + self.built = True + + def call(self, inputs, mask=None, training=None, initial_state=None): + # LSTM does not support constants. Ignore it during process. + inputs, initial_state, _ = self._process_inputs(inputs, initial_state, None) + + if isinstance(mask, list): + mask = mask[0] + + input_shape = K.int_shape(inputs) + timesteps = input_shape[0] if self.time_major else input_shape[1] + + if mask is not None or not self.could_use_cudnn: + # CuDNN does not support masking, fall back to use the normal LSTM. + kwargs = {'training': training} + + def step(inputs, states): + return self.cell.call(inputs, states, **kwargs) + + last_output, outputs, states = K.rnn( + step, + inputs, + initial_state, + constants=None, + go_backwards=self.go_backwards, + mask=mask, + unroll=self.unroll, + input_length=timesteps, + time_major=self.time_major, + zero_output_for_mask=self.zero_output_for_mask) + runtime = constant_op.constant( + 'unknown', dtype=dtypes.string, name='runtime') + else: + # Use the new defun approach for backend implementation swap. + # Note that different implementations need to have same function + # signature, eg, the tensor parameters need to have same shape and dtypes. + # Since the CuDNN has an extra set of bias, those bias will be passed to + # both normal and CuDNN implementations. + if self.go_backwards: + # Reverse time axis. + inputs = K.reverse(inputs, 1) + + combined_bias = array_ops.concat([self.cudnn_bias, self.cell.bias], 0) + + # Each time a defun function is called, we will give a unique identifiable + # API name, so that the grappler won't get confused when it sees multiple + # LSTM layer added into same graph, and it will be able to pair up the + # different implementations across them. + experimental_api_name = 'lstm_' + str(uuid.uuid4()) + standard_lstm_attributes = { + 'experimental_api_implements': experimental_api_name, + 'experimental_api_preferred_device': 'CPU', + } + cudnn_lstm_attributes = { + 'experimental_api_implements': experimental_api_name, + 'experimental_api_preferred_device': 'GPU', + } + defun_standard_lstm = function.defun_with_attributes( + standard_lstm, attributes=standard_lstm_attributes) + defun_cudnn_lstm = function.defun_with_attributes( + cudnn_lstm, attributes=cudnn_lstm_attributes) + + if ops.executing_eagerly_outside_functions(): + # Under eager context, the device placement is already known. Prefer the + # GPU implementation here. + if context.num_gpus() > 0: + last_output, outputs, new_h, new_c, runtime = defun_cudnn_lstm( + inputs, initial_state[0], initial_state[1], self.cell.kernel, + self.cell.recurrent_kernel, combined_bias, self.time_major) + else: + last_output, outputs, new_h, new_c, runtime = defun_standard_lstm( + inputs, initial_state[0], initial_state[1], self.cell.kernel, + self.cell.recurrent_kernel, combined_bias, self.activation, + self.recurrent_activation, self.time_major) + else: + # Call the normal LSTM impl and register the CuDNN impl function. The + # grappler will kick in during session execution to optimize the graph. + last_output, outputs, new_h, new_c, runtime = defun_standard_lstm( + inputs, initial_state[0], initial_state[1], self.cell.kernel, + self.cell.recurrent_kernel, combined_bias, self.activation, + self.recurrent_activation, self.time_major) + + function.register(defun_cudnn_lstm, inputs, initial_state[0], + initial_state[1], self.cell.kernel, + self.cell.recurrent_kernel, combined_bias, + self.time_major) + states = [new_h, new_c] + + if self.stateful: + updates = [] + for i in range(len(states)): + updates.append(state_ops.assign(self.states[i], states[i])) + self.add_update(updates, inputs) + + if self.return_sequences: + output = outputs + else: + output = last_output + + if self.return_state: + return [output] + states + elif self.return_runtime: + return output, runtime + else: + return output + + @property + def trainable_weights(self): + if self.trainable: + weights = [] + weights += self.cell.trainable_weights + if getattr(self, 'cudnn_bias', None) is not None: + weights += [self.cudnn_bias] + return weights + return [] + + @property + def non_trainable_weights(self): + if not self.trainable: + weights = [] + weights += self.cell.non_trainable_weights + if getattr(self, 'cudnn_bias', None) is not None: + weights += [self.cudnn_bias] + return weights + return [] + + @property + def losses(self): + losses = [] + losses += self.cell.losses + return losses + self._losses + + @property + def updates(self): + updates = [] + updates += self.cell.updates + return updates + self._updates + + def get_weights(self): + weights = [] + weights += self.cell.weights + if getattr(self, 'cudnn_bias', None) is not None: + weights += [self.cudnn_bias] + return K.batch_get_value(weights) + + def set_weights(self, weights): + tuples = [] + cell_weights = weights[:len(self.cell.weights)] + if cell_weights: + tuples.append((self.cell.weights, cell_weights)) + if getattr(self, 'cudnn_bias', None) is not None: + cudnn_bias_weights = weights[len(self.cell.weights):] + if cudnn_bias_weights: + tuples.append((self.cudnn_bias, cudnn_bias_weights)) + K.batch_set_value(tuples) + + +def _canonical_to_params(weights, biases, shape): + """Utility function convert variable to CuDNN compatible parameter.""" + weights = [array_ops.reshape(x, shape) for x in weights] + biases = [array_ops.reshape(x, shape) for x in biases] + return array_ops.concat(weights + biases, axis=0) + + +def standard_lstm(inputs, init_h, init_c, kernel, recurrent_kernel, bias, + activation, recurrent_activation, time_major): + """LSTM with standard kernel implementation. + + This implementation can be run on all types for hardware. + + This implementation lifts out all the layer weights and make them function + parameters. It has same number of tensor input params as the CuDNN + counterpart. The RNN step logic has been simplified, eg dropout and mask is + removed since CuDNN implementation does not support that. + + Note that the first half of the bias tensor should be ignored by this impl. + The CuDNN impl need an extra set of input gate bias. In order to make the both + function take same shape of parameter, that extra set of bias is also feed + here. + + Args: + inputs: input tensor of LSTM layer. + init_h: initial state tensor for the cell output. + init_c: initial state tensor for the cell hidden state. + kernel: weights for cell kernel. + recurrent_kernel: weights for cell recurrent kernel. + bias: weights for cell kernel bias and recurrent bias. Only recurrent bias + is used in this case. + activation: Activation function to use for output. + recurrent_activation: Activation function to use for hidden recurrent state. + time_major: boolean, whether the inputs are in the format of + [time, batch, feature] or [batch, time, feature]. + + Returns: + last_output: output tensor for the last timestep, which has shape + [batch, units]. + outputs: output tensor for all timesteps, which has shape + [batch, time, units]. + state_0: the cell output, which has same shape as init_h. + state_1: the cell hidden state, which has same shape as init_c. + runtime: constant string tensor which indicate real runtime hardware. This + value is for testing purpose and should be used by user. + """ + input_shape = K.int_shape(inputs) + timesteps = input_shape[0] if time_major else input_shape[1] + + # Only use the second half of the bias weights. + _, real_bias = array_ops.split(bias, 2) + + def step(cell_inputs, cell_states): + """Step function that will be used by Keras RNN backend.""" + h_tm1 = cell_states[0] # previous memory state + c_tm1 = cell_states[1] # previous carry state + + z = K.dot(cell_inputs, kernel) + z += K.dot(h_tm1, recurrent_kernel) + z = K.bias_add(z, real_bias) + + z0, z1, z2, z3 = array_ops.split(z, 4, axis=1) + + i = recurrent_activation(z0) + f = recurrent_activation(z1) + c = f * c_tm1 + i * activation(z2) + o = recurrent_activation(z3) + + h = o * activation(c) + return h, [h, c] + + last_output, outputs, new_states = K.rnn( + step, + inputs, [init_h, init_c], + constants=None, + unroll=False, + time_major=time_major, + input_length=timesteps) + return last_output, outputs, new_states[0], new_states[ + 1], constant_op.constant('cpu', dtype=dtypes.string, name='runtime') + + +def cudnn_lstm(inputs, input_h, input_c, kernel, recurrent_kernel, bias, + time_major): + """LSTM with CuDNN implementation which is only available for GPU.""" + if not time_major: + inputs = array_ops.transpose(inputs, perm=(1, 0, 2)) + input_h = array_ops.expand_dims(input_h, axis=0) + input_c = array_ops.expand_dims(input_c, axis=0) + + weights = array_ops.split(kernel, 4, axis=1) + weights += array_ops.split(recurrent_kernel, 4, axis=1) + params = _canonical_to_params( + weights=weights, + biases=array_ops.split(bias, 8), + shape=constant_op.constant([-1])) + + outputs, h, c, _ = gen_cudnn_rnn_ops.cudnn_rnn( + inputs, input_h=input_h, input_c=input_c, params=params) + if not time_major: + outputs = array_ops.transpose(outputs, perm=[1, 0, 2]) + h = h[0] + c = c[0] + last_output = outputs[:, -1, :] + return last_output, outputs, h, c, constant_op.constant( + 'cudnn', dtype=dtypes.string, name='runtime') + + def _generate_dropout_mask(ones, rate, training=None, count=1): def dropped_inputs(): return K.dropout(ones, rate) diff --git a/tensorflow/python/keras/layers/recurrent_test.py b/tensorflow/python/keras/layers/recurrent_test.py index bb14a7a5056..b1449069e32 100644 --- a/tensorflow/python/keras/layers/recurrent_test.py +++ b/tensorflow/python/keras/layers/recurrent_test.py @@ -1013,8 +1013,8 @@ class RNNTest(test.TestCase): inputs, _ = cell(inputs, initial_state) output = inputs if not context.executing_eagerly(): - sess.run(variables_lib.global_variables_initializer()) - output = sess.run(output) + self.evaluate(variables_lib.global_variables_initializer()) + output = self.evaluate(output) return output random_seed.set_random_seed(12345) @@ -1079,6 +1079,32 @@ class RNNTest(test.TestCase): # Expect last output to be the same as last output before masking self.assertAllClose(y_np, x_np[:, 1, :]) + def test_zero_output_for_masking(self): + + for unroll in [True, False]: + cell = keras.layers.SimpleRNNCell(5) + x = keras.Input((5, 5)) + mask = keras.layers.Masking() + layer = keras.layers.RNN( + cell, return_sequences=True, zero_output_for_mask=True, unroll=unroll) + masked_input = mask(x) + y = layer(masked_input) + model = keras.models.Model(x, y) + model.compile(optimizer=rmsprop.RMSPropOptimizer(learning_rate=0.001), + loss='mse') + + np_x = np.ones((6, 5, 5)) + result_1 = model.predict(np_x) + + # set the time 4 and 5 for last record to be zero (masked). + np_x[5, 3:] = 0 + result_2 = model.predict(np_x) + + # expect the result_2 has same output, except the time 4,5 for last + # record. + result_1[5, 3:] = 0 + self.assertAllClose(result_1, result_2) + class Minimal2DRNNCell(keras.layers.Layer): """The minimal 2D RNN cell is a simple combination of 2 1-D RNN cell. diff --git a/tensorflow/python/keras/layers/simplernn_test.py b/tensorflow/python/keras/layers/simplernn_test.py index 93456b5e3a7..b49b159b719 100644 --- a/tensorflow/python/keras/layers/simplernn_test.py +++ b/tensorflow/python/keras/layers/simplernn_test.py @@ -98,6 +98,7 @@ class SimpleRNNLayerTest(test.TestCase): self.assertEqual(layer.cell.recurrent_kernel.constraint, r_constraint) self.assertEqual(layer.cell.bias.constraint, b_constraint) + @tf_test_util.run_deprecated_v1 def test_with_masking_layer_SimpleRNN(self): layer_class = keras.layers.SimpleRNN inputs = np.random.random((2, 3, 4)) @@ -120,6 +121,7 @@ class SimpleRNNLayerTest(test.TestCase): class SimpleRNNLayerGraphOnlyTest(test.TestCase): + @tf_test_util.run_deprecated_v1 def test_statefulness_SimpleRNN(self): num_samples = 2 timesteps = 3 @@ -183,6 +185,7 @@ class SimpleRNNLayerGraphOnlyTest(test.TestCase): np.testing.assert_allclose(out7, out6, atol=1e-5) + @tf_test_util.run_deprecated_v1 def test_regularizers_SimpleRNN(self): embedding_dim = 4 layer_class = keras.layers.SimpleRNN diff --git a/tensorflow/python/keras/layers/unified_lstm_test.py b/tensorflow/python/keras/layers/unified_lstm_test.py new file mode 100644 index 00000000000..d229d14312f --- /dev/null +++ b/tensorflow/python/keras/layers/unified_lstm_test.py @@ -0,0 +1,724 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Tests for UnifiedLSTM layer.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import time + +from absl.testing import parameterized +import numpy as np + +from tensorflow.core.protobuf import config_pb2 +from tensorflow.core.protobuf import rewriter_config_pb2 +from tensorflow.python import keras +from tensorflow.python.framework import constant_op +from tensorflow.python.framework import dtypes +from tensorflow.python.framework import test_util +from tensorflow.python.keras import testing_utils +from tensorflow.python.keras.layers.cudnn_recurrent import CuDNNLSTM +from tensorflow.python.keras.layers.recurrent import UnifiedLSTM +from tensorflow.python.ops import array_ops +from tensorflow.python.ops import control_flow_ops +from tensorflow.python.ops import gen_math_ops +from tensorflow.python.ops import variables +from tensorflow.python.ops.losses import losses +from tensorflow.python.platform import test +from tensorflow.python.platform import tf_logging as logging +from tensorflow.python.training import gradient_descent + + +# Global config for grappler setting that is used for graph mode test. +_rewrites = rewriter_config_pb2.RewriterConfig() +_rewrites.function_optimization = rewriter_config_pb2.RewriterConfig.OFF +_customer_optimizer = _rewrites.custom_optimizers.add() +_customer_optimizer.name = 'ExperimentalImplementationSelector' +_rewrites.min_graph_nodes = -1 +_graph_options = config_pb2.GraphOptions(rewrite_options=_rewrites) +_config = config_pb2.ConfigProto(graph_options=_graph_options) + + +class UnifiedLSTMTest(test.TestCase, parameterized.TestCase): + + @test_util.run_deprecated_v1 + def test_unifiedLSTM(self): + input_shape = 10 + rnn_state_size = 8 + output_shape = 8 + timestep = 4 + batch = 100 + epoch = 1 + + with self.cached_session(config=_config, use_gpu=True) as sess: + (x_train, y_train), _ = testing_utils.get_test_data( + train_samples=batch, + test_samples=0, + input_shape=(timestep, input_shape), + num_classes=output_shape) + y_train = keras.utils.to_categorical(y_train, output_shape) + + layer = UnifiedLSTM(rnn_state_size, return_runtime=True) + + inputs = array_ops.placeholder( + dtypes.float32, shape=(None, timestep, input_shape), name='inputs') + predict = array_ops.placeholder( + dtypes.float32, shape=(None, output_shape), name='predict') + + outputs, runtime = layer(inputs) + loss = losses.softmax_cross_entropy(predict, outputs) + optimizer = gradient_descent.GradientDescentOptimizer(0.001) + train_op = optimizer.minimize(loss) + + sess.run([variables.global_variables_initializer()]) + existing_loss = 0 + for _ in range(epoch): + loss_value, _, runtime_value = sess.run([loss, train_op, runtime], { + inputs: x_train, + predict: y_train + }) + if test.is_gpu_available(): + self.assertEqual(runtime_value, b'cudnn') + else: + self.assertEqual(runtime_value, b'cpu') + # Make sure the loss is updated for every epoch + # (layer weights properly updated). + self.assertNotEqual(existing_loss, loss_value) + existing_loss = loss_value + + @test_util.run_deprecated_v1 + def test_unifiedLSTM_with_cond(self): + # This test is to demonstrate the graph rewrite of grappler plugin under + # the condition that the function returns different number of internal + # states. + input_shape = 10 + rnn_state_size = 8 + output_shape = 8 + timestep = 4 + batch = 100 + epoch = 1 + + with self.cached_session(config=_config, use_gpu=True) as sess: + (x_train, y_train), _ = testing_utils.get_test_data( + train_samples=batch, + test_samples=0, + input_shape=(timestep, input_shape), + num_classes=output_shape) + y_train = keras.utils.to_categorical(y_train, output_shape) + + layer = UnifiedLSTM(rnn_state_size, return_runtime=True) + + inputs = array_ops.placeholder( + dtypes.float32, shape=(None, timestep, input_shape), name='inputs') + predict = array_ops.placeholder( + dtypes.float32, shape=(None, output_shape), name='predict') + + zeros = array_ops.zeros([batch, output_shape]) + dummy_runtime = constant_op.constant( + 'unknown', dtype=dtypes.string, name='runtime') + a = constant_op.constant(0) + b = constant_op.constant(1) + # Will always run the lstm layer. + outputs, runtime = control_flow_ops.cond( + gen_math_ops.less(a, b), + lambda: layer(inputs), + lambda: (zeros, dummy_runtime)) + loss = losses.softmax_cross_entropy(predict, outputs) + optimizer = gradient_descent.GradientDescentOptimizer(0.001) + train_op = optimizer.minimize(loss) + + sess.run([variables.global_variables_initializer()]) + existing_loss = 0 + + for _ in range(epoch): + loss_value, _, runtime_value = sess.run([loss, train_op, runtime], { + inputs: x_train, + predict: y_train + }) + if test.is_gpu_available(): + self.assertEqual(runtime_value, b'cudnn') + else: + self.assertEqual(runtime_value, b'cpu') + # Make sure the loss is updated for every epoch + # (layer weights properly updated). + self.assertNotEqual(existing_loss, loss_value) + existing_loss = loss_value + + @test_util.run_in_graph_and_eager_modes(config=_config) + def test_keras_model_with_lstm(self): + input_shape = 10 + rnn_state_size = 8 + output_shape = 8 + timestep = 4 + batch = 100 + epoch = 10 + + (x_train, y_train), _ = testing_utils.get_test_data( + train_samples=batch, + test_samples=0, + input_shape=(timestep, input_shape), + num_classes=output_shape) + y_train = keras.utils.to_categorical(y_train, output_shape) + + layer = UnifiedLSTM(rnn_state_size) + + inputs = keras.layers.Input( + shape=[timestep, input_shape], dtype=dtypes.float32) + + outputs = layer(inputs) + model = keras.models.Model(inputs, outputs) + model.compile('rmsprop', loss='mse') + model.fit(x_train, y_train, epochs=epoch) + model.evaluate(x_train, y_train) + + @test_util.run_in_graph_and_eager_modes(config=_config) + def test_return_sequences_LSTM(self): + num_samples = 2 + timesteps = 3 + embedding_dim = 4 + units = 2 + testing_utils.layer_test( + UnifiedLSTM, + kwargs={ + 'units': units, + 'return_sequences': True + }, + input_shape=(num_samples, timesteps, embedding_dim)) + + @test_util.run_in_graph_and_eager_modes(config=_config) + def test_static_shape_inference_LSTM(self): + # Github issue: 15165 + timesteps = 3 + embedding_dim = 4 + units = 2 + + model = keras.models.Sequential() + inputs = keras.layers.Dense( + embedding_dim, input_shape=(timesteps, embedding_dim)) + model.add(inputs) + layer = UnifiedLSTM(units, return_sequences=True) + model.add(layer) + outputs = model.layers[-1].output + self.assertEqual(outputs.get_shape().as_list(), [None, timesteps, units]) + + @test_util.run_in_graph_and_eager_modes(config=_config) + def test_dynamic_behavior_LSTM(self): + num_samples = 2 + timesteps = 3 + embedding_dim = 4 + units = 2 + layer = UnifiedLSTM(units, input_shape=(None, embedding_dim)) + model = keras.models.Sequential() + model.add(layer) + model.compile(gradient_descent.GradientDescentOptimizer(0.001), 'mse') + x = np.random.random((num_samples, timesteps, embedding_dim)) + y = np.random.random((num_samples, units)) + model.train_on_batch(x, y) + + @test_util.run_in_graph_and_eager_modes(config=_config) + def test_dropout_LSTM(self): + num_samples = 2 + timesteps = 3 + embedding_dim = 4 + units = 2 + testing_utils.layer_test( + UnifiedLSTM, + kwargs={ + 'units': units, + 'dropout': 0.1, + 'recurrent_dropout': 0.1 + }, + input_shape=(num_samples, timesteps, embedding_dim)) + + @parameterized.parameters([0, 1, 2]) + @test_util.run_in_graph_and_eager_modes(config=_config) + def test_implementation_mode_LSTM(self, implementation_mode): + num_samples = 2 + timesteps = 3 + embedding_dim = 4 + units = 2 + testing_utils.layer_test( + UnifiedLSTM, + kwargs={ + 'units': units, + 'implementation': implementation_mode + }, + input_shape=(num_samples, timesteps, embedding_dim)) + + @test_util.run_in_graph_and_eager_modes(config=_config) + def test_constraints_LSTM(self): + embedding_dim = 4 + layer_class = UnifiedLSTM + k_constraint = keras.constraints.max_norm(0.01) + r_constraint = keras.constraints.max_norm(0.01) + b_constraint = keras.constraints.max_norm(0.01) + layer = layer_class( + 5, + return_sequences=False, + weights=None, + input_shape=(None, embedding_dim), + kernel_constraint=k_constraint, + recurrent_constraint=r_constraint, + bias_constraint=b_constraint) + layer.build((None, None, embedding_dim)) + self.assertEqual(layer.cell.kernel.constraint, k_constraint) + self.assertEqual(layer.cell.recurrent_kernel.constraint, r_constraint) + self.assertEqual(layer.cell.bias.constraint, b_constraint) + + @test_util.run_in_graph_and_eager_modes(config=_config) + def test_with_masking_layer_LSTM(self): + layer_class = UnifiedLSTM + inputs = np.random.random((2, 3, 4)) + targets = np.abs(np.random.random((2, 3, 5))) + targets /= targets.sum(axis=-1, keepdims=True) + model = keras.models.Sequential() + model.add(keras.layers.Masking(input_shape=(3, 4))) + model.add(layer_class(units=5, return_sequences=True, unroll=False)) + model.compile( + loss='categorical_crossentropy', + optimizer=gradient_descent.GradientDescentOptimizer(0.01)) + model.fit(inputs, targets, epochs=1, batch_size=2, verbose=1) + + @test_util.run_in_graph_and_eager_modes(config=_config) + def test_stacking_LSTM(self): + inputs = np.random.random((2, 3, 4)) + targets = np.abs(np.random.random((2, 3, 5))) + targets /= targets.sum(axis=-1, keepdims=True) + model = keras.models.Sequential() + model.add(UnifiedLSTM(10, return_sequences=True, unroll=False)) + model.add(UnifiedLSTM(5, return_sequences=True, unroll=False)) + model.compile( + loss='categorical_crossentropy', + optimizer=gradient_descent.GradientDescentOptimizer(0.01)) + model.fit(inputs, targets, epochs=1, batch_size=2, verbose=1) + + @test_util.run_in_graph_and_eager_modes(config=_config) + def test_masking_with_stacking_LSTM(self): + inputs = np.random.random((2, 3, 4)) + targets = np.abs(np.random.random((2, 3, 5))) + targets /= targets.sum(axis=-1, keepdims=True) + model = keras.models.Sequential() + model.add(keras.layers.Masking(input_shape=(3, 4))) + model.add(UnifiedLSTM(10, return_sequences=True, unroll=False)) + model.add(UnifiedLSTM(5, return_sequences=True, unroll=False)) + model.compile( + loss='categorical_crossentropy', + optimizer=gradient_descent.GradientDescentOptimizer(0.01)) + model.fit(inputs, targets, epochs=1, batch_size=2, verbose=1) + + @test_util.run_in_graph_and_eager_modes(config=_config) + def test_from_config_LSTM(self): + layer_class = UnifiedLSTM + for stateful in (False, True): + l1 = layer_class(units=1, stateful=stateful) + l2 = layer_class.from_config(l1.get_config()) + assert l1.get_config() == l2.get_config() + + @test_util.run_in_graph_and_eager_modes(config=_config) + def test_specify_initial_state_keras_tensor(self): + num_states = 2 + timesteps = 3 + embedding_dim = 4 + units = 3 + num_samples = 2 + + # Test with Keras tensor + inputs = keras.Input((timesteps, embedding_dim)) + initial_state = [keras.Input((units,)) for _ in range(num_states)] + layer = UnifiedLSTM(units) + if len(initial_state) == 1: + output = layer(inputs, initial_state=initial_state[0]) + else: + output = layer(inputs, initial_state=initial_state) + assert initial_state[0] in layer._inbound_nodes[0].input_tensors + + model = keras.models.Model([inputs] + initial_state, output) + model.compile( + loss='categorical_crossentropy', + optimizer=gradient_descent.GradientDescentOptimizer(0.01)) + + inputs = np.random.random((num_samples, timesteps, embedding_dim)) + initial_state = [ + np.random.random((num_samples, units)) for _ in range(num_states) + ] + targets = np.random.random((num_samples, units)) + model.train_on_batch([inputs] + initial_state, targets) + + @test_util.run_in_graph_and_eager_modes(config=_config) + def DISABLED_test_specify_initial_state_non_keras_tensor(self): + num_states = 2 + timesteps = 3 + embedding_dim = 4 + units = 3 + num_samples = 2 + + # Test with non-Keras tensor + inputs = keras.Input((timesteps, embedding_dim)) + initial_state = [ + keras.backend.random_normal_variable((num_samples, units), 0, 1) + for _ in range(num_states) + ] + layer = UnifiedLSTM(units) + output = layer(inputs, initial_state=initial_state) + + model = keras.models.Model(inputs, output) + model.compile( + loss='categorical_crossentropy', + optimizer=gradient_descent.GradientDescentOptimizer(0.01)) + + inputs = np.random.random((num_samples, timesteps, embedding_dim)) + targets = np.random.random((num_samples, units)) + model.train_on_batch(inputs, targets) + + @test_util.run_in_graph_and_eager_modes(config=_config) + def test_reset_states_with_values(self): + num_states = 2 + timesteps = 3 + embedding_dim = 4 + units = 3 + num_samples = 2 + + layer = UnifiedLSTM(units, stateful=True) + layer.build((num_samples, timesteps, embedding_dim)) + layer.reset_states() + assert len(layer.states) == num_states + assert layer.states[0] is not None + self.assertAllClose( + keras.backend.eval(layer.states[0]), + np.zeros(keras.backend.int_shape(layer.states[0])), + atol=1e-4) + state_shapes = [keras.backend.int_shape(state) for state in layer.states] + values = [np.ones(shape) for shape in state_shapes] + if len(values) == 1: + values = values[0] + layer.reset_states(values) + self.assertAllClose( + keras.backend.eval(layer.states[0]), + np.ones(keras.backend.int_shape(layer.states[0])), + atol=1e-4) + + # Test with invalid data + with self.assertRaises(ValueError): + layer.reset_states([1] * (len(layer.states) + 1)) + + @test_util.run_in_graph_and_eager_modes(config=_config) + def test_specify_state_with_masking(self): + num_states = 2 + timesteps = 3 + embedding_dim = 4 + units = 3 + num_samples = 2 + + inputs = keras.Input((timesteps, embedding_dim)) + _ = keras.layers.Masking()(inputs) + initial_state = [keras.Input((units,)) for _ in range(num_states)] + output = UnifiedLSTM(units)(inputs, initial_state=initial_state) + + model = keras.models.Model([inputs] + initial_state, output) + model.compile( + loss='categorical_crossentropy', + optimizer=gradient_descent.GradientDescentOptimizer(0.01)) + + inputs = np.random.random((num_samples, timesteps, embedding_dim)) + initial_state = [ + np.random.random((num_samples, units)) for _ in range(num_states) + ] + targets = np.random.random((num_samples, units)) + model.train_on_batch([inputs] + initial_state, targets) + + @test_util.run_in_graph_and_eager_modes(config=_config) + def test_return_state(self): + num_states = 2 + timesteps = 3 + embedding_dim = 4 + units = 3 + num_samples = 2 + + inputs = keras.Input(batch_shape=(num_samples, timesteps, embedding_dim)) + layer = UnifiedLSTM(units, return_state=True, stateful=True) + outputs = layer(inputs) + state = outputs[1:] + assert len(state) == num_states + model = keras.models.Model(inputs, state[0]) + + inputs = np.random.random((num_samples, timesteps, embedding_dim)) + state = model.predict(inputs) + self.assertAllClose(keras.backend.eval(layer.states[0]), state, atol=1e-4) + + @test_util.run_in_graph_and_eager_modes(config=_config) + def test_state_reuse(self): + timesteps = 3 + embedding_dim = 4 + units = 3 + num_samples = 2 + + inputs = keras.Input(batch_shape=(num_samples, timesteps, embedding_dim)) + layer = UnifiedLSTM(units, return_state=True, return_sequences=True) + outputs = layer(inputs) + output, state = outputs[0], outputs[1:] + output = UnifiedLSTM(units)(output, initial_state=state) + model = keras.models.Model(inputs, output) + + inputs = np.random.random((num_samples, timesteps, embedding_dim)) + model.predict(inputs) + + @test_util.run_in_graph_and_eager_modes(config=_config) + def test_initial_states_as_other_inputs(self): + timesteps = 3 + embedding_dim = 4 + units = 3 + num_samples = 2 + num_states = 2 + layer_class = UnifiedLSTM + + # Test with Keras tensor + main_inputs = keras.Input((timesteps, embedding_dim)) + initial_state = [keras.Input((units,)) for _ in range(num_states)] + inputs = [main_inputs] + initial_state + + layer = layer_class(units) + output = layer(inputs) + assert initial_state[0] in layer._inbound_nodes[0].input_tensors + + model = keras.models.Model(inputs, output) + model.compile( + loss='categorical_crossentropy', + optimizer=gradient_descent.GradientDescentOptimizer(0.01)) + + main_inputs = np.random.random((num_samples, timesteps, embedding_dim)) + initial_state = [ + np.random.random((num_samples, units)) for _ in range(num_states) + ] + targets = np.random.random((num_samples, units)) + model.train_on_batch([main_inputs] + initial_state, targets) + + +class LSTMLayerGraphOnlyTest(test.TestCase): + + def test_statefulness_LSTM(self): + num_samples = 2 + timesteps = 3 + embedding_dim = 4 + units = 2 + layer_class = UnifiedLSTM + with self.cached_session(config=_config): + model = keras.models.Sequential() + model.add( + keras.layers.Embedding( + 4, + embedding_dim, + mask_zero=True, + input_length=timesteps, + batch_input_shape=(num_samples, timesteps))) + layer = layer_class( + units, return_sequences=False, stateful=True, weights=None) + model.add(layer) + model.compile( + optimizer=gradient_descent.GradientDescentOptimizer(0.01), loss='mse') + out1 = model.predict(np.ones((num_samples, timesteps))) + self.assertEqual(out1.shape, (num_samples, units)) + + # train once so that the states change + model.train_on_batch( + np.ones((num_samples, timesteps)), np.ones((num_samples, units))) + out2 = model.predict(np.ones((num_samples, timesteps))) + + # if the state is not reset, output should be different + self.assertNotEqual(out1.max(), out2.max()) + + # check that output changes after states are reset + # (even though the model itself didn't change) + layer.reset_states() + out3 = model.predict(np.ones((num_samples, timesteps))) + self.assertNotEqual(out2.max(), out3.max()) + + # check that container-level reset_states() works + model.reset_states() + out4 = model.predict(np.ones((num_samples, timesteps))) + self.assertAllClose(out3, out4, atol=1e-5) + + # check that the call to `predict` updated the states + out5 = model.predict(np.ones((num_samples, timesteps))) + self.assertNotEqual(out4.max(), out5.max()) + + # Check masking + layer.reset_states() + + left_padded_input = np.ones((num_samples, timesteps)) + left_padded_input[0, :1] = 0 + left_padded_input[1, :2] = 0 + out6 = model.predict(left_padded_input) + + layer.reset_states() + + right_padded_input = np.ones((num_samples, timesteps)) + right_padded_input[0, -1:] = 0 + right_padded_input[1, -2:] = 0 + out7 = model.predict(right_padded_input) + + self.assertAllClose(out7, out6, atol=1e-5) + + def test_regularizers_LSTM(self): + embedding_dim = 4 + layer_class = UnifiedLSTM + with self.cached_session(config=_config): + layer = layer_class( + 5, + return_sequences=False, + weights=None, + input_shape=(None, embedding_dim), + kernel_regularizer=keras.regularizers.l1(0.01), + recurrent_regularizer=keras.regularizers.l1(0.01), + bias_regularizer='l2', + activity_regularizer='l1') + layer.build((None, None, 2)) + self.assertEqual(len(layer.losses), 3) + x = keras.backend.variable(np.ones((2, 3, 2))) + layer(x) + self.assertEqual(len(layer.get_losses_for(x)), 1) + + +class UnifiedLSTMPerformanceTest(test.TestCase): + + def _measure_performance(self, test_config, model, x_train, y_train): + batch = test_config['batch'] + epoch = test_config['epoch'] + warmup_epoch = test_config['warmup_epoch'] + + # warm up the model + model.fit(x_train, y_train, batch_size=batch, epochs=warmup_epoch) + start_time = time.time() + model.fit(x_train, y_train, batch_size=batch, epochs=epoch - warmup_epoch) + end_time = time.time() + return (end_time - start_time) / (epoch - warmup_epoch) + + def _time_performance_run_cudnn_lstm(self, test_config, x_train, y_train): + # Get the performance number for standard Cudnn LSTM + input_shape = test_config['input_shape'] + rnn_state_size = test_config['rnn_state_size'] + timestep = test_config['timestep'] + + cudnn_lstm_layer = CuDNNLSTM(rnn_state_size) + inputs = keras.layers.Input( + shape=[timestep, input_shape], dtype=dtypes.float32) + + outputs = cudnn_lstm_layer(inputs) + model = keras.models.Model(inputs, outputs) + model.compile('sgd', 'mse') + + sec_per_epoch = self._measure_performance( + test_config, model, x_train, y_train) + logging.info('Average performance for %s per epoch is: %s', + 'CuDNN LSTM', sec_per_epoch) + return sec_per_epoch + + def _time_performance_run_unifed_lstm_gpu( + self, test_config, x_train, y_train): + # Get performance number for Unified_LSTM with grappler swap the impl + input_shape = test_config['input_shape'] + rnn_state_size = test_config['rnn_state_size'] + timestep = test_config['timestep'] + + layer = UnifiedLSTM(rnn_state_size) + inputs = keras.layers.Input( + shape=[timestep, input_shape], dtype=dtypes.float32) + + outputs = layer(inputs) + model = keras.models.Model(inputs, outputs) + model.compile('sgd', 'mse') + + sec_per_epoch = self._measure_performance( + test_config, model, x_train, y_train) + logging.info('Average performance for %s per epoch is: %s', + 'Unified LSTM', sec_per_epoch) + return sec_per_epoch + + def _time_performance_run_normal_lstm( + self, test_config, x_train, y_train): + # Get performance number for standard LSTM on GPU. + input_shape = test_config['input_shape'] + rnn_state_size = test_config['rnn_state_size'] + timestep = test_config['timestep'] + + layer = keras.layers.LSTM(rnn_state_size) + inputs = keras.layers.Input( + shape=[timestep, input_shape], dtype=dtypes.float32) + + outputs = layer(inputs) + model = keras.models.Model(inputs, outputs) + model.compile('sgd', 'mse') + + sec_per_epoch = self._measure_performance( + test_config, model, x_train, y_train) + logging.info('Average performance for %s per epoch is: %s', + 'Normal LSTM', sec_per_epoch) + return sec_per_epoch + + @test_util.run_in_graph_and_eager_modes(config=_config, use_gpu=True) + def test_performance_with_standard_cudnn_impl(self): + if not test.is_gpu_available(): + self.skipTest('performance test will only run on GPU') + + batch = 64 + num_batch = 10 + test_config = { + 'input_shape': 128, + 'rnn_state_size': 64, + 'output_shape': 64, + 'timestep': 50, + 'batch': batch, + 'epoch': 20, + # The performance for warmup epoch is ignored. + 'warmup_epoch': 1, + } + (x_train, y_train), _ = testing_utils.get_test_data( + train_samples=(batch * num_batch), + test_samples=0, + input_shape=(test_config['timestep'], test_config['input_shape']), + num_classes=test_config['output_shape']) + y_train = keras.utils.to_categorical(y_train, test_config['output_shape']) + + cudnn_duration = self._time_performance_run_cudnn_lstm( + test_config, x_train, y_train) + unified_lstm_gpu_duration = self._time_performance_run_unifed_lstm_gpu( + test_config, x_train, y_train) + normal_lstm_duration = self._time_performance_run_normal_lstm( + test_config, x_train, y_train) + + cudnn_vs_unified = cudnn_duration / unified_lstm_gpu_duration + unified_vs_normal = normal_lstm_duration / unified_lstm_gpu_duration + + # TODO(scottzhu): reeanble the test after moving it to benchmark test suite. + # The current test has performance flakiness issue. + logging.info('Expect the performance of Unified LSTM is within 80% of ' + 'CuDNN LSTM, got {0:.2f}%'.format(cudnn_vs_unified * 100)) + logging.info('Expect the performance of Unified LSTM is more than 5 times' + ' of normal LSTM, got {0:.2f}'.format(unified_vs_normal)) + + # Assert the performance diff should be within 80% of the native cudnn. + # self.assertGreaterEqual( + # cudnn_vs_unified, 0.80, + # 'Expect the performance of Unified LSTM is within 80% of CuDNN LSTM, ' + # 'but got {0:.2f}%'.format(cudnn_vs_unified * 100)) + # # Assert the performance diff between CPU impl and GPU impl should be more + # # than 5 times. + # self.assertGreaterEqual( + # unified_vs_normal, 5, + # 'Expect the performance of Unified LSTM is more than 5 times of ' + # 'normal LSTM, but got {0:.2f}'.format(unified_vs_normal)) + +if __name__ == '__main__': + test.main() diff --git a/tensorflow/python/keras/layers/wrappers.py b/tensorflow/python/keras/layers/wrappers.py index d40f7a2e809..67b154141ef 100644 --- a/tensorflow/python/keras/layers/wrappers.py +++ b/tensorflow/python/keras/layers/wrappers.py @@ -23,8 +23,8 @@ import copy from tensorflow.python.framework import tensor_shape from tensorflow.python.keras import backend as K -from tensorflow.python.keras.engine.base_layer import InputSpec from tensorflow.python.keras.engine.base_layer import Layer +from tensorflow.python.keras.engine.input_spec import InputSpec from tensorflow.python.keras.layers.recurrent import _standardize_args from tensorflow.python.keras.utils import generic_utils from tensorflow.python.keras.utils import tf_utils @@ -389,6 +389,10 @@ class Bidirectional(Wrapper): raise ValueError('Invalid merge mode. ' 'Merge mode should be one of ' '{"sum", "mul", "ave", "concat", None}') + if getattr(layer, 'zero_output_for_mask', None) is not None: + # Force the zero_output_for_mask to be True if it presents. + layer.zero_output_for_mask = True + self.forward_layer = copy.copy(layer) config = layer.get_config() config['go_backwards'] = not config['go_backwards'] diff --git a/tensorflow/python/keras/layers/wrappers_test.py b/tensorflow/python/keras/layers/wrappers_test.py index 9584b0186c4..b9196416ddd 100644 --- a/tensorflow/python/keras/layers/wrappers_test.py +++ b/tensorflow/python/keras/layers/wrappers_test.py @@ -192,8 +192,8 @@ class TimeDistributedTest(test.TestCase): x = keras.layers.Input(shape=(3, 2)) layer = keras.layers.TimeDistributed(keras.layers.BatchNormalization()) _ = layer(x) - self.assertEquals(len(layer.updates), 2) - self.assertEquals(len(layer.trainable_weights), 2) + self.assertEqual(len(layer.updates), 2) + self.assertEqual(len(layer.trainable_weights), 2) layer.trainable = False assert not layer.updates assert not layer.trainable_weights @@ -201,6 +201,7 @@ class TimeDistributedTest(test.TestCase): assert len(layer.updates) == 2 assert len(layer.trainable_weights) == 2 + @tf_test_util.run_deprecated_v1 def test_TimeDistributed_with_masked_embedding_and_unspecified_shape(self): with self.cached_session(): # test with unspecified shape and Embeddings with mask_zero @@ -233,6 +234,7 @@ class TimeDistributedTest(test.TestCase): self.assertAllEqual(mask_outputs_val[i], ref_mask_val[i]) self.assertIs(mask_outputs[-1], None) # final layer + @tf_test_util.run_deprecated_v1 def test_TimeDistributed_with_masking_layer(self): with self.cached_session(): # test with Masking layer @@ -375,6 +377,7 @@ class BidirectionalTest(test.TestCase): model.compile(loss='mse', optimizer='sgd') model.fit(x, y, epochs=1, batch_size=1) + @tf_test_util.run_deprecated_v1 def test_Bidirectional_merged_value(self): rnn = keras.layers.LSTM samples = 2 @@ -505,6 +508,7 @@ class BidirectionalTest(test.TestCase): layer.trainable = True assert len(layer.trainable_weights) == 6 + @tf_test_util.run_deprecated_v1 def test_Bidirectional_updates(self): with self.cached_session(): x = keras.layers.Input(shape=(3, 2)) @@ -635,6 +639,34 @@ class BidirectionalTest(test.TestCase): y_np_3 = model.predict([x_np, s_fw_np, s_bk_np, c_np]) self.assertAllClose(y_np, y_np_3, atol=1e-4) + def test_Bidirectional_with_masking(self): + rnn = keras.layers.LSTM + samples = 2 + dim = 5 + timesteps = 3 + units = 3 + merge_mode = 'concat' + x = np.random.rand(samples, timesteps, dim) + # clear the first record's timestep 2, and expect the output of timestep 2 + # is also 0s. + x[0, 2] = 0 + + with self.cached_session(): + inputs = keras.Input((timesteps, dim)) + masked_inputs = keras.layers.Masking()(inputs) + wrapped = keras.layers.Bidirectional( + rnn(units, return_sequences=True), + merge_mode=merge_mode) + outputs = _to_list(wrapped(masked_inputs, training=True)) + self.assertEqual(len(outputs), 1) + self.assertEqual(outputs[0].get_shape().as_list(), + [None, timesteps, units * 2]) + + model = keras.Model(inputs, outputs) + y = _to_list(model.predict(x)) + self.assertEqual(len(y), 1) + self.assertAllClose(y[0][0, 2], np.zeros(units * 2)) + def _to_list(ls): if isinstance(ls, list): diff --git a/tensorflow/python/keras/losses.py b/tensorflow/python/keras/losses.py index 9f548bfe040..4c584d0ff05 100644 --- a/tensorflow/python/keras/losses.py +++ b/tensorflow/python/keras/losses.py @@ -19,16 +19,382 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function +import abc + import six +from tensorflow.python.framework import ops from tensorflow.python.keras import backend as K from tensorflow.python.keras.utils.generic_utils import deserialize_keras_object from tensorflow.python.keras.utils.generic_utils import serialize_keras_object +from tensorflow.python.keras.utils.losses_utils import compute_weighted_loss +from tensorflow.python.ops import array_ops from tensorflow.python.ops import math_ops from tensorflow.python.ops import nn +from tensorflow.python.ops.losses import losses_impl from tensorflow.python.util.tf_export import tf_export +class Loss(object): + """Loss base class. + + To be implemented by subclasses: + * `call()`: Contains the logic for loss calculation using `y_true`, `y_pred`. + + Example subclass implementation: + ``` + class MeanSquaredError(Loss): + def call(self, y_true, y_pred): + y_pred = ops.convert_to_tensor(y_pred) + y_true = math_ops.cast(y_true, y_pred.dtype) + return K.mean(math_ops.square(y_pred - y_true), axis=-1) + ``` + + Args: + reduction: Type of `tf.losses.Reduction` to apply to loss. Default value is + `SUM_OVER_BATCH_SIZE`. + name: Optional name for the op. + """ + + def __init__(self, + reduction=losses_impl.ReductionV2.SUM_OVER_BATCH_SIZE, + name=None): + self.reduction = reduction + self.name = name + + def __call__(self, y_true, y_pred, sample_weight=None): + """Invokes the `Loss` instance. + + Args: + y_true: Ground truth values. + y_pred: The predicted values. + sample_weight: Optional `Tensor` whose rank is either 0, or the same rank + as `y_true`, or is broadcastable to `y_true`. `sample_weight` acts as a + coefficient for the loss. If a scalar is provided, then the loss is + simply scaled by the given value. If `sample_weight` is a tensor of size + `[batch_size]`, then the total loss for each sample of the batch is + rescaled by the corresponding element in the `sample_weight` vector. If + the shape of `sample_weight` matches the shape of `y_pred`, then the + loss of each measurable element of `y_pred` is scaled by the + corresponding value of `sample_weight`. + + Returns: + Weighted loss float `Tensor`. If `reduction` is `NONE`, this has the same + shape as `y_true`; otherwise, it is scalar. + + Raises: + ValueError: If the shape of `sample_weight` is invalid. + """ + with ops.name_scope(self.name, format(self.__class__.__name__), + (y_pred, y_true, sample_weight)): + losses = self.call(y_true, y_pred) + return compute_weighted_loss( + losses, sample_weight, reduction=self.reduction) + + @classmethod + def from_config(cls, config): + """Instantiates a `Loss` from its config (output of `get_config()`). + + Args: + config: Output of `get_config()`. + + Returns: + A `Loss` instance. + """ + return cls(**config) + + def get_config(self): + return {'reduction': self.reduction, 'name': self.name} + + @abc.abstractmethod + def call(self, y_true, y_pred): + """Invokes the `Loss` instance. + + Args: + y_true: Ground truth values, with the same shape as 'y_pred'. + y_pred: The predicted values. + """ + NotImplementedError('Must be implemented in subclasses.') + + +@tf_export('keras.losses.MeanSquaredError') +class MeanSquaredError(Loss): + """Computes the mean of squares of errors between labels and predictions. + + For example, if `y_true` is [0., 0., 1., 1.] and `y_pred` is [1., 1., 1., 0.] + then the mean squared error value is 3/4 (0.75). + + Usage: + + ```python + mse = tf.keras.losses.MeanSquaredError() + loss = mse([0., 0., 1., 1.], [1., 1., 1., 0.]) + print('Loss: ', loss.numpy()) # Loss: 0.75 + ``` + + Usage with tf.keras API: + + ```python + model = keras.models.Model(inputs, outputs) + model.compile('sgd', loss=tf.keras.losses.MeanSquaredError()) + ``` + """ + + def call(self, y_true, y_pred): + """Invokes the `MeanSquaredError` instance. + + Args: + y_true: Ground truth values. + y_pred: The predicted values. + + Returns: + Mean squared error losses. + """ + y_pred = ops.convert_to_tensor(y_pred) + y_true = math_ops.cast(y_true, y_pred.dtype) + return mean_squared_error(y_true, y_pred) + + +@tf_export('keras.losses.MeanAbsoluteError') +class MeanAbsoluteError(Loss): + """Computes the mean of absolute difference between labels and predictions. + + For example, if `y_true` is [0., 0., 1., 1.] and `y_pred` is [1., 1., 1., 0.] + then the mean absolute error value is 3/4 (0.75). + + Usage: + + ```python + mae = tf.keras.losses.MeanAbsoluteError() + loss = mae([0., 0., 1., 1.], [1., 1., 1., 0.]) + print('Loss: ', loss.numpy()) # Loss: 0.75 + ``` + + Usage with tf.keras API: + + ```python + model = keras.models.Model(inputs, outputs) + model.compile('sgd', loss=tf.keras.losses.MeanAbsoluteError()) + ``` + """ + + def call(self, y_true, y_pred): + """Invokes the `MeanAbsoluteError` instance. + + Args: + y_true: Ground truth values. + y_pred: The predicted values. + + Returns: + Mean absolute error losses. + """ + y_pred = ops.convert_to_tensor(y_pred) + y_true = math_ops.cast(y_true, y_pred.dtype) + return mean_absolute_error(y_true, y_pred) + + +@tf_export('keras.losses.MeanAbsolutePercentageError') +class MeanAbsolutePercentageError(Loss): + """Computes the mean absolute percentage error between `y_true` and `y_pred`. + + For example, if `y_true` is [0., 0., 1., 1.] and `y_pred` is [1., 1., 1., 0.] + then the mean absolute percentage error value is 5e+08. + + Usage: + + ```python + mape = tf.keras.losses.MeanAbsolutePercentageError() + loss = mape([0., 0., 1., 1.], [1., 1., 1., 0.]) + print('Loss: ', loss.numpy()) # Loss: 5e+08 + ``` + + Usage with tf.keras API: + + ```python + model = keras.models.Model(inputs, outputs) + model.compile('sgd', loss=tf.keras.losses.MeanAbsolutePercentageError()) + ``` + """ + + def call(self, y_true, y_pred): + """Invokes the `MeanAbsolutePercentageError` instance. + + Args: + y_true: Ground truth values. + y_pred: The predicted values. + + Returns: + Mean absolute percentage error losses. + """ + y_pred = ops.convert_to_tensor(y_pred) + y_true = math_ops.cast(y_true, y_pred.dtype) + return mean_absolute_percentage_error(y_true, y_pred) + + +@tf_export('keras.losses.MeanSquaredLogarithmicError') +class MeanSquaredLogarithmicError(Loss): + """Computes the mean squared logarithmic error between `y_true` and `y_pred`. + + For example, if `y_true` is [0., 0., 1., 1.] and `y_pred` is [1., 1., 1., 0.] + then the mean squared logarithmic error value is 0.36034. + + Usage: + + ```python + msle = tf.keras.losses.MeanSquaredLogarithmicError() + loss = msle([0., 0., 1., 1.], [1., 1., 1., 0.]) + print('Loss: ', loss.numpy()) # Loss: 0.36034 + ``` + + Usage with tf.keras API: + + ```python + model = keras.models.Model(inputs, outputs) + model.compile('sgd', loss=tf.keras.losses.MeanSquaredLogarithmicError()) + ``` + """ + + def call(self, y_true, y_pred): + """Invokes the `MeanSquaredLogarithmicError` instance. + + Args: + y_true: Ground truth values. + y_pred: The predicted values. + + Returns: + Mean squared logarithmic error losses. + """ + y_pred = ops.convert_to_tensor(y_pred) + y_true = math_ops.cast(y_true, y_pred.dtype) + return mean_squared_logarithmic_error(y_true, y_pred) + + +@tf_export('keras.losses.BinaryCrossentropy') +class BinaryCrossentropy(Loss): + """Computes the binary cross entropy loss between the labels and predictions. + + Usage: + + ```python + bce = tf.keras.losses.BinaryCrossentropy() + loss = bce([0., 0., 1., 1.], [1., 1., 1., 0.]) + print('Loss: ', loss.numpy()) # Loss: 12.007 + ``` + + Usage with tf.keras API: + + ```python + model = keras.models.Model(inputs, outputs) + model.compile('sgd', loss=tf.keras.losses.BinaryCrossentropy()) + ```` + + Args: + from_logits: Whether `output` is expected to be a logits tensor. By default, + we consider that `output` encodes a probability distribution. + label_smoothing: If greater than `0` then smooth the labels. + reduction: Type of `tf.losses.Reduction` to apply to loss. Default value is + `SUM_OVER_BATCH_SIZE`. + name: Optional name for the op. + """ + + def __init__(self, + from_logits=False, + label_smoothing=0, + reduction=losses_impl.ReductionV2.SUM_OVER_BATCH_SIZE, + name=None): + super(BinaryCrossentropy, self).__init__(reduction=reduction, name=name) + self.from_logits = from_logits + self.label_smoothing = label_smoothing + + def call(self, y_true, y_pred): + """Invokes the `BinaryCrossentropy` instance. + + Args: + y_true: Ground truth values. + y_pred: The predicted values. + + Returns: + Binary cross entropy losses. + """ + y_pred = ops.convert_to_tensor(y_pred) + y_true = math_ops.cast(y_true, y_pred.dtype) + + if self.label_smoothing > 0: + y_true = y_true * (1 - self.label_smoothing) + 0.5 * self.label_smoothing + + return binary_crossentropy(y_true, y_pred, from_logits=self.from_logits) + + +@tf_export('keras.losses.CategoricalCrossentropy') +class CategoricalCrossentropy(Loss): + """Computes categorical cross entropy loss between the `y_true` and `y_pred`. + + Usage: + + ```python + cce = tf.keras.losses.CategoricalCrossentropy() + loss = cce( + [[1., 0., 0.], [0., 1., 0.], [0., 0., 1.]], + [[.9, .05, .05], [.5, .89, .6], [.05, .01, .94]]) + print('Loss: ', loss.numpy()) # Loss: 0.3239 + ``` + + Usage with tf.keras API: + + ```python + model = keras.models.Model(inputs, outputs) + model.compile('sgd', loss=tf.keras.losses.CategoricalCrossentropy()) + ```` + + Args: + from_logits: Whether `output` is expected to be a logits tensor. By default, + we consider that `output` encodes a probability distribution. + label_smoothing: If greater than `0` then smooth the labels. This option is + currently not supported when `y_pred` is a sparse input (not one-hot). + reduction: Type of `tf.losses.Reduction` to apply to loss. Default value is + `SUM_OVER_BATCH_SIZE`. + name: Optional name for the op. + """ + + def __init__(self, + from_logits=False, + label_smoothing=0, + reduction=losses_impl.ReductionV2.SUM_OVER_BATCH_SIZE, + name=None): + super(CategoricalCrossentropy, self).__init__( + reduction=reduction, name=name) + self.from_logits = from_logits + self.label_smoothing = label_smoothing + + def call(self, y_true, y_pred): + """Invokes the `CategoricalCrossentropy` instance. + + Args: + y_true: Ground truth values. + y_pred: The predicted values. + + Returns: + Categorical cross entropy losses. + """ + y_pred = ops.convert_to_tensor(y_pred) + y_true = ops.convert_to_tensor(y_true) + is_sparse = y_pred.shape != y_true.shape + + if is_sparse: + return sparse_categorical_crossentropy( + y_true, y_pred, from_logits=self.from_logits) + else: + y_true = math_ops.cast(y_true, y_pred.dtype) + if self.label_smoothing > 0: + num_classes = math_ops.cast(array_ops.shape(y_true)[1], y_pred.dtype) + smooth_positives = 1.0 - self.label_smoothing + smooth_negatives = self.label_smoothing / num_classes + y_true = y_true * smooth_positives + smooth_negatives + + return categorical_crossentropy( + y_true, y_pred, from_logits=self.from_logits) + + @tf_export('keras.metrics.mean_squared_error', 'keras.metrics.mse', 'keras.metrics.MSE', @@ -116,20 +482,22 @@ def logcosh(y_true, y_pred): @tf_export('keras.metrics.categorical_crossentropy', 'keras.losses.categorical_crossentropy') -def categorical_crossentropy(y_true, y_pred): - return K.categorical_crossentropy(y_true, y_pred) +def categorical_crossentropy(y_true, y_pred, from_logits=False): + return K.categorical_crossentropy(y_true, y_pred, from_logits=from_logits) @tf_export('keras.metrics.sparse_categorical_crossentropy', 'keras.losses.sparse_categorical_crossentropy') -def sparse_categorical_crossentropy(y_true, y_pred): - return K.sparse_categorical_crossentropy(y_true, y_pred) +def sparse_categorical_crossentropy(y_true, y_pred, from_logits=False): + return K.sparse_categorical_crossentropy( + y_true, y_pred, from_logits=from_logits) @tf_export('keras.metrics.binary_crossentropy', 'keras.losses.binary_crossentropy') -def binary_crossentropy(y_true, y_pred): - return K.mean(K.binary_crossentropy(y_true, y_pred), axis=-1) +def binary_crossentropy(y_true, y_pred, from_logits=False): + return K.mean( + K.binary_crossentropy(y_true, y_pred, from_logits=from_logits), axis=-1) @tf_export('keras.metrics.kullback_leibler_divergence', @@ -159,6 +527,40 @@ def cosine_proximity(y_true, y_pred): return -math_ops.reduce_sum(y_true * y_pred, axis=-1) +class CosineProximity(Loss): + """Computes the cosine distance between `y_true` and `y_pred`. + + Usage: + + ```python + cosine_loss = tf.losses.CosineProximity() + loss = cosine_loss([0., 1., 1.], [1., 0., 1.]) + print('Loss: ', loss.numpy()) # Loss: -0.5 + ``` + + Usage with tf.keras API: + + ```python + model = keras.models.Model(inputs, outputs) + model.compile('sgd', loss=tf.losses.CosineProximity()) + ``` + """ + + def call(self, y_true, y_pred): + """Calculates the cosine proximity loss. + + Args: + y_true: Ground truth values. + y_pred: The predicted values. + + Returns: + Cosine distance loss. + """ + y_pred = ops.convert_to_tensor(y_pred) + y_true = math_ops.cast(y_true, y_pred.dtype) + return cosine_proximity(y_true, y_pred) + + # Aliases. mse = MSE = mean_squared_error @@ -197,3 +599,9 @@ def get(identifier): else: raise ValueError('Could not interpret ' 'loss function identifier:', identifier) + + +LABEL_DTYPES_FOR_LOSSES = { + losses_impl.sparse_softmax_cross_entropy: 'int32', + sparse_categorical_crossentropy: 'int32' +} diff --git a/tensorflow/python/keras/losses_test.py b/tensorflow/python/keras/losses_test.py index c7015270acc..d2791cdcd3b 100644 --- a/tensorflow/python/keras/losses_test.py +++ b/tensorflow/python/keras/losses_test.py @@ -24,6 +24,11 @@ import shutil import numpy as np from tensorflow.python import keras +from tensorflow.python.framework import constant_op +from tensorflow.python.framework import dtypes +from tensorflow.python.framework import test_util +from tensorflow.python.ops import array_ops +from tensorflow.python.ops.losses import losses_impl from tensorflow.python.platform import test try: @@ -138,5 +143,633 @@ class KerasLossesTest(test.TestCase): loaded_model.predict(np.random.rand(128, 2)) +@test_util.run_all_in_graph_and_eager_modes +class MeanSquaredErrorTest(test.TestCase): + + def test_config(self): + mse_obj = keras.losses.MeanSquaredError( + reduction=losses_impl.ReductionV2.SUM, name='mse_1') + self.assertEqual(mse_obj.name, 'mse_1') + self.assertEqual(mse_obj.reduction, losses_impl.ReductionV2.SUM) + + def test_all_correct_unweighted(self): + mse_obj = keras.losses.MeanSquaredError() + y_true = constant_op.constant([4, 8, 12, 8, 1, 3], shape=(2, 3)) + loss = mse_obj(y_true, y_true) + self.assertAlmostEqual(self.evaluate(loss), 0.0, 3) + + def test_unweighted(self): + mse_obj = keras.losses.MeanSquaredError() + y_true = constant_op.constant([1, 9, 2, -5, -2, 6], shape=(2, 3)) + y_pred = constant_op.constant([4, 8, 12, 8, 1, 3], + shape=(2, 3), + dtype=dtypes.float32) + loss = mse_obj(y_true, y_pred) + self.assertAlmostEqual(self.evaluate(loss), 49.5, 3) + + def test_scalar_weighted(self): + mse_obj = keras.losses.MeanSquaredError() + y_true = constant_op.constant([1, 9, 2, -5, -2, 6], shape=(2, 3)) + y_pred = constant_op.constant([4, 8, 12, 8, 1, 3], + shape=(2, 3), + dtype=dtypes.float32) + loss = mse_obj(y_true, y_pred, sample_weight=2.3) + self.assertAlmostEqual(self.evaluate(loss), 113.85, 3) + + def test_sample_weighted(self): + mse_obj = keras.losses.MeanSquaredError() + y_true = constant_op.constant([1, 9, 2, -5, -2, 6], shape=(2, 3)) + y_pred = constant_op.constant([4, 8, 12, 8, 1, 3], + shape=(2, 3), + dtype=dtypes.float32) + sample_weight = constant_op.constant([1.2, 3.4], shape=(2, 1)) + loss = mse_obj(y_true, y_pred, sample_weight=sample_weight) + self.assertAlmostEqual(self.evaluate(loss), 767.8 / 6, 3) + + def test_timestep_weighted(self): + mse_obj = keras.losses.MeanSquaredError() + y_true = constant_op.constant([1, 9, 2, -5, -2, 6], shape=(2, 3, 1)) + y_pred = constant_op.constant([4, 8, 12, 8, 1, 3], + shape=(2, 3, 1), + dtype=dtypes.float32) + sample_weight = constant_op.constant([3, 6, 5, 0, 4, 2], shape=(2, 3)) + loss = mse_obj(y_true, y_pred, sample_weight=sample_weight) + self.assertAlmostEqual(self.evaluate(loss), 587 / 6, 3) + + def test_zero_weighted(self): + mse_obj = keras.losses.MeanSquaredError() + y_true = constant_op.constant([1, 9, 2, -5, -2, 6], shape=(2, 3)) + y_pred = constant_op.constant([4, 8, 12, 8, 1, 3], + shape=(2, 3), + dtype=dtypes.float32) + loss = mse_obj(y_true, y_pred, sample_weight=0) + self.assertAlmostEqual(self.evaluate(loss), 0.0, 3) + + def test_invalid_sample_weight(self): + mse_obj = keras.losses.MeanSquaredError() + y_true = constant_op.constant([1, 9, 2, -5, -2, 6], shape=(2, 3, 1)) + y_pred = constant_op.constant([4, 8, 12, 8, 1, 3], shape=(2, 3, 1)) + sample_weight = constant_op.constant([3, 6, 5, 0], shape=(2, 2)) + with self.assertRaisesRegexp( + ValueError, r'Shapes \(2, 2\) and \(2, 3\) are incompatible'): + mse_obj(y_true, y_pred, sample_weight=sample_weight) + + def test_no_reduction(self): + mse_obj = keras.losses.MeanSquaredError( + reduction=losses_impl.ReductionV2.NONE) + y_true = constant_op.constant([1, 9, 2, -5, -2, 6], shape=(2, 3)) + y_pred = constant_op.constant([4, 8, 12, 8, 1, 3], + shape=(2, 3), + dtype=dtypes.float32) + loss = mse_obj(y_true, y_pred, sample_weight=2.3) + loss = self.evaluate(loss) + self.assertArrayNear(loss, [84.3333, 143.3666], 1e-3) + + def test_sum_reduction(self): + mse_obj = keras.losses.MeanSquaredError( + reduction=losses_impl.ReductionV2.SUM) + y_true = constant_op.constant([1, 9, 2, -5, -2, 6], shape=(2, 3)) + y_pred = constant_op.constant([4, 8, 12, 8, 1, 3], + shape=(2, 3), + dtype=dtypes.float32) + loss = mse_obj(y_true, y_pred, sample_weight=2.3) + self.assertAlmostEqual(self.evaluate(loss), 227.69998, 3) + + +@test_util.run_all_in_graph_and_eager_modes +class MeanAbsoluteErrorTest(test.TestCase): + + def test_config(self): + mae_obj = keras.losses.MeanAbsoluteError( + reduction=losses_impl.ReductionV2.SUM, name='mae_1') + self.assertEqual(mae_obj.name, 'mae_1') + self.assertEqual(mae_obj.reduction, losses_impl.ReductionV2.SUM) + + def test_all_correct_unweighted(self): + mae_obj = keras.losses.MeanAbsoluteError() + y_true = constant_op.constant([4, 8, 12, 8, 1, 3], shape=(2, 3)) + loss = mae_obj(y_true, y_true) + self.assertAlmostEqual(self.evaluate(loss), 0.0, 3) + + def test_unweighted(self): + mae_obj = keras.losses.MeanAbsoluteError() + y_true = constant_op.constant([1, 9, 2, -5, -2, 6], shape=(2, 3)) + y_pred = constant_op.constant([4, 8, 12, 8, 1, 3], + shape=(2, 3), + dtype=dtypes.float32) + loss = mae_obj(y_true, y_pred) + self.assertAlmostEqual(self.evaluate(loss), 5.5, 3) + + def test_scalar_weighted(self): + mae_obj = keras.losses.MeanAbsoluteError() + y_true = constant_op.constant([1, 9, 2, -5, -2, 6], shape=(2, 3)) + y_pred = constant_op.constant([4, 8, 12, 8, 1, 3], + shape=(2, 3), + dtype=dtypes.float32) + loss = mae_obj(y_true, y_pred, sample_weight=2.3) + self.assertAlmostEqual(self.evaluate(loss), 12.65, 3) + + def test_sample_weighted(self): + mae_obj = keras.losses.MeanAbsoluteError() + y_true = constant_op.constant([1, 9, 2, -5, -2, 6], shape=(2, 3)) + y_pred = constant_op.constant([4, 8, 12, 8, 1, 3], + shape=(2, 3), + dtype=dtypes.float32) + sample_weight = constant_op.constant([1.2, 3.4], shape=(2, 1)) + loss = mae_obj(y_true, y_pred, sample_weight=sample_weight) + self.assertAlmostEqual(self.evaluate(loss), 81.4 / 6, 3) + + def test_timestep_weighted(self): + mae_obj = keras.losses.MeanAbsoluteError() + y_true = constant_op.constant([1, 9, 2, -5, -2, 6], shape=(2, 3, 1)) + y_pred = constant_op.constant([4, 8, 12, 8, 1, 3], + shape=(2, 3, 1), + dtype=dtypes.float32) + sample_weight = constant_op.constant([3, 6, 5, 0, 4, 2], shape=(2, 3)) + loss = mae_obj(y_true, y_pred, sample_weight=sample_weight) + self.assertAlmostEqual(self.evaluate(loss), 83 / 6, 3) + + def test_zero_weighted(self): + mae_obj = keras.losses.MeanAbsoluteError() + y_true = constant_op.constant([1, 9, 2, -5, -2, 6], shape=(2, 3)) + y_pred = constant_op.constant([4, 8, 12, 8, 1, 3], + shape=(2, 3), + dtype=dtypes.float32) + loss = mae_obj(y_true, y_pred, sample_weight=0) + self.assertAlmostEqual(self.evaluate(loss), 0.0, 3) + + def test_invalid_sample_weight(self): + mae_obj = keras.losses.MeanAbsoluteError() + y_true = constant_op.constant([1, 9, 2, -5, -2, 6], shape=(2, 3, 1)) + y_pred = constant_op.constant([4, 8, 12, 8, 1, 3], shape=(2, 3, 1)) + sample_weight = constant_op.constant([3, 6, 5, 0], shape=(2, 2)) + with self.assertRaisesRegexp( + ValueError, r'Shapes \(2, 2\) and \(2, 3\) are incompatible'): + mae_obj(y_true, y_pred, sample_weight=sample_weight) + + def test_no_reduction(self): + mae_obj = keras.losses.MeanAbsoluteError( + reduction=losses_impl.ReductionV2.NONE) + y_true = constant_op.constant([1, 9, 2, -5, -2, 6], shape=(2, 3)) + y_pred = constant_op.constant([4, 8, 12, 8, 1, 3], + shape=(2, 3), + dtype=dtypes.float32) + loss = mae_obj(y_true, y_pred, sample_weight=2.3) + loss = self.evaluate(loss) + self.assertArrayNear(loss, [10.7333, 14.5666], 1e-3) + + def test_sum_reduction(self): + mae_obj = keras.losses.MeanAbsoluteError( + reduction=losses_impl.ReductionV2.SUM) + y_true = constant_op.constant([1, 9, 2, -5, -2, 6], shape=(2, 3)) + y_pred = constant_op.constant([4, 8, 12, 8, 1, 3], + shape=(2, 3), + dtype=dtypes.float32) + loss = mae_obj(y_true, y_pred, sample_weight=2.3) + self.assertAlmostEqual(self.evaluate(loss), 25.29999, 3) + + +@test_util.run_all_in_graph_and_eager_modes +class MeanAbsolutePercentageErrorTest(test.TestCase): + + def test_config(self): + mape_obj = keras.losses.MeanAbsolutePercentageError( + reduction=losses_impl.ReductionV2.SUM, name='mape_1') + self.assertEqual(mape_obj.name, 'mape_1') + self.assertEqual(mape_obj.reduction, losses_impl.ReductionV2.SUM) + + def test_unweighted(self): + mape_obj = keras.losses.MeanAbsolutePercentageError() + y_true = constant_op.constant([1, 9, 2, -5, -2, 6], shape=(2, 3)) + y_pred = constant_op.constant([4, 8, 12, 8, 1, 3], + shape=(2, 3), + dtype=dtypes.float32) + loss = mape_obj(y_true, y_pred) + self.assertAlmostEqual(self.evaluate(loss), 211.8518, 3) + + def test_scalar_weighted(self): + mape_obj = keras.losses.MeanAbsolutePercentageError() + y_true = constant_op.constant([1, 9, 2, -5, -2, 6], shape=(2, 3)) + y_pred = constant_op.constant([4, 8, 12, 8, 1, 3], + shape=(2, 3), + dtype=dtypes.float32) + loss = mape_obj(y_true, y_pred, sample_weight=2.3) + self.assertAlmostEqual(self.evaluate(loss), 487.259, 3) + + def test_sample_weighted(self): + mape_obj = keras.losses.MeanAbsolutePercentageError() + y_true = constant_op.constant([1, 9, 2, -5, -2, 6], shape=(2, 3)) + y_pred = constant_op.constant([4, 8, 12, 8, 1, 3], + shape=(2, 3), + dtype=dtypes.float32) + sample_weight = constant_op.constant([1.2, 3.4], shape=(2, 1)) + loss = mape_obj(y_true, y_pred, sample_weight=sample_weight) + self.assertAlmostEqual(self.evaluate(loss), 422.8888, 3) + + def test_timestep_weighted(self): + mape_obj = keras.losses.MeanAbsolutePercentageError() + y_true = constant_op.constant([1, 9, 2, -5, -2, 6], shape=(2, 3, 1)) + y_pred = constant_op.constant([4, 8, 12, 8, 1, 3], + shape=(2, 3, 1), + dtype=dtypes.float32) + sample_weight = constant_op.constant([3, 6, 5, 0, 4, 2], shape=(2, 3)) + loss = mape_obj(y_true, y_pred, sample_weight=sample_weight) + self.assertAlmostEqual(self.evaluate(loss), 694.4445, 3) + + def test_zero_weighted(self): + mape_obj = keras.losses.MeanAbsolutePercentageError() + y_true = constant_op.constant([1, 9, 2, -5, -2, 6], shape=(2, 3)) + y_pred = constant_op.constant([4, 8, 12, 8, 1, 3], + shape=(2, 3), + dtype=dtypes.float32) + loss = mape_obj(y_true, y_pred, sample_weight=0) + self.assertAlmostEqual(self.evaluate(loss), 0.0, 3) + + +@test_util.run_all_in_graph_and_eager_modes +class MeanSquaredLogarithmicErrorTest(test.TestCase): + + def test_config(self): + msle_obj = keras.losses.MeanSquaredLogarithmicError( + reduction=losses_impl.ReductionV2.SUM, name='mape_1') + self.assertEqual(msle_obj.name, 'mape_1') + self.assertEqual(msle_obj.reduction, losses_impl.ReductionV2.SUM) + + def test_unweighted(self): + msle_obj = keras.losses.MeanSquaredLogarithmicError() + y_true = constant_op.constant([1, 9, 2, -5, -2, 6], shape=(2, 3)) + y_pred = constant_op.constant([4, 8, 12, 8, 1, 3], + shape=(2, 3), + dtype=dtypes.float32) + loss = msle_obj(y_true, y_pred) + self.assertAlmostEqual(self.evaluate(loss), 1.4370, 3) + + def test_scalar_weighted(self): + msle_obj = keras.losses.MeanSquaredLogarithmicError() + y_true = constant_op.constant([1, 9, 2, -5, -2, 6], shape=(2, 3)) + y_pred = constant_op.constant([4, 8, 12, 8, 1, 3], + shape=(2, 3), + dtype=dtypes.float32) + loss = msle_obj(y_true, y_pred, sample_weight=2.3) + self.assertAlmostEqual(self.evaluate(loss), 3.3051, 3) + + def test_sample_weighted(self): + msle_obj = keras.losses.MeanSquaredLogarithmicError() + y_true = constant_op.constant([1, 9, 2, -5, -2, 6], shape=(2, 3)) + y_pred = constant_op.constant([4, 8, 12, 8, 1, 3], + shape=(2, 3), + dtype=dtypes.float32) + sample_weight = constant_op.constant([1.2, 3.4], shape=(2, 1)) + loss = msle_obj(y_true, y_pred, sample_weight=sample_weight) + self.assertAlmostEqual(self.evaluate(loss), 3.7856, 3) + + def test_timestep_weighted(self): + msle_obj = keras.losses.MeanSquaredLogarithmicError() + y_true = constant_op.constant([1, 9, 2, -5, -2, 6], shape=(2, 3, 1)) + y_pred = constant_op.constant([4, 8, 12, 8, 1, 3], + shape=(2, 3, 1), + dtype=dtypes.float32) + sample_weight = constant_op.constant([3, 6, 5, 0, 4, 2], shape=(2, 3)) + loss = msle_obj(y_true, y_pred, sample_weight=sample_weight) + self.assertAlmostEqual(self.evaluate(loss), 2.6473, 3) + + def test_zero_weighted(self): + msle_obj = keras.losses.MeanSquaredLogarithmicError() + y_true = constant_op.constant([1, 9, 2, -5, -2, 6], shape=(2, 3)) + y_pred = constant_op.constant([4, 8, 12, 8, 1, 3], + shape=(2, 3), + dtype=dtypes.float32) + loss = msle_obj(y_true, y_pred, sample_weight=0) + self.assertAlmostEqual(self.evaluate(loss), 0.0, 3) + + +@test_util.run_all_in_graph_and_eager_modes +class CosineProximityTest(test.TestCase): + + def test_config(self): + cosine_obj = keras.losses.CosineProximity( + reduction=losses_impl.ReductionV2.SUM, name='cosine_loss') + self.assertEqual(cosine_obj.name, 'cosine_loss') + self.assertEqual(cosine_obj.reduction, losses_impl.ReductionV2.SUM) + + def test_unweighted(self): + cosine_obj = keras.losses.CosineProximity() + y_true = constant_op.constant([1, 9, 2, -5, -2, 6], shape=(2, 3)) + y_pred = constant_op.constant([4, 8, 12, 8, 1, 3], + shape=(2, 3), + dtype=dtypes.float32) + loss = cosine_obj(y_true, y_pred) + self.assertAlmostEqual(self.evaluate(loss), -0.18722, 3) + + def test_scalar_weighted(self): + cosine_obj = keras.losses.CosineProximity() + y_true = constant_op.constant([1, 9, 2, -5, -2, 6], shape=(2, 3)) + y_pred = constant_op.constant([4, 8, 12, 8, 1, 3], + shape=(2, 3), + dtype=dtypes.float32) + loss = cosine_obj(y_true, y_pred, sample_weight=2.3) + self.assertAlmostEqual(self.evaluate(loss), -0.43060, 3) + + def test_sample_weighted(self): + cosine_obj = keras.losses.CosineProximity() + y_true = constant_op.constant([1, 9, 2, -5, -2, 6], shape=(2, 3)) + y_pred = constant_op.constant([4, 8, 12, 8, 1, 3], + shape=(2, 3), + dtype=dtypes.float32) + sample_weight = constant_op.constant([1.2, 3.4], shape=(2, 1)) + loss = cosine_obj(y_true, y_pred, sample_weight=sample_weight) + self.assertAlmostEqual(self.evaluate(loss), 0.15599, 3) + + def test_timestep_weighted(self): + cosine_obj = keras.losses.CosineProximity() + y_true = constant_op.constant([1, 9, 2, -5, -2, 6], shape=(2, 3, 1)) + y_pred = constant_op.constant([4, 8, 12, 8, 1, 3], + shape=(2, 3, 1), + dtype=dtypes.float32) + sample_weight = constant_op.constant([3, 6, 5, 0, 4, 2], shape=(2, 3)) + loss = cosine_obj(y_true, y_pred, sample_weight=sample_weight) + self.assertAlmostEqual(self.evaluate(loss), -2.0000, 3) + + def test_zero_weighted(self): + cosine_obj = keras.losses.CosineProximity() + y_true = constant_op.constant([1, 9, 2, -5, -2, 6], shape=(2, 3)) + y_pred = constant_op.constant([4, 8, 12, 8, 1, 3], + shape=(2, 3), + dtype=dtypes.float32) + loss = cosine_obj(y_true, y_pred, sample_weight=0) + self.assertAlmostEqual(self.evaluate(loss), 0., 3) + + +@test_util.run_all_in_graph_and_eager_modes +class BinaryCrossentropyTest(test.TestCase): + + def test_config(self): + bce_obj = keras.losses.BinaryCrossentropy( + reduction=losses_impl.ReductionV2.SUM, name='bce_1') + self.assertEqual(bce_obj.name, 'bce_1') + self.assertEqual(bce_obj.reduction, losses_impl.ReductionV2.SUM) + + def test_all_correct_unweighted(self): + y_true = constant_op.constant([[1, 0, 0], [0, 1, 0], [0, 0, 1]], + dtype=dtypes.float32) + bce_obj = keras.losses.BinaryCrossentropy() + loss = bce_obj(y_true, y_true) + self.assertAlmostEqual(self.evaluate(loss), 0.0, 3) + + # Test with logits. + logits = constant_op.constant([[100.0, -100.0, -100.0], + [-100.0, 100.0, -100.0], + [-100.0, -100.0, 100.0]]) + bce_obj = keras.losses.BinaryCrossentropy(from_logits=True) + loss = bce_obj(y_true, logits) + self.assertAlmostEqual(self.evaluate(loss), 0.0, 3) + + def test_unweighted(self): + bce_obj = keras.losses.BinaryCrossentropy() + y_true = constant_op.constant([1, 0, 1, 0, 0, 1], shape=(2, 3)) + y_pred = constant_op.constant([1, 1, 1, 0, 1, 0], + shape=(2, 3), + dtype=dtypes.float32) + loss = bce_obj(y_true, y_pred) + self.assertAlmostEqual(self.evaluate(loss), 8.0004, 3) + + # Test with logits. + logits = constant_op.constant([10., 10., 10., -10., 10, -10], + shape=(2, 3), + dtype=dtypes.float32) + bce_obj = keras.losses.BinaryCrossentropy(from_logits=True) + loss = bce_obj(y_true, logits) + self.assertAlmostEqual(self.evaluate(loss), 5., 3) + + def test_scalar_weighted(self): + bce_obj = keras.losses.BinaryCrossentropy() + y_true = constant_op.constant([1, 0, 1, 0, 0, 1], shape=(2, 3)) + y_pred = constant_op.constant([1, 1, 1, 0, 1, 0], + shape=(2, 3), + dtype=dtypes.float32) + loss = bce_obj(y_true, y_pred, sample_weight=2.3) + self.assertAlmostEqual(self.evaluate(loss), 18.4010, 3) + + # Test with logits. + y_true = array_ops.ones((32, 1)) + logits = array_ops.ones((32, 1), dtype=dtypes.float32) + bce_obj = keras.losses.BinaryCrossentropy(from_logits=True) + loss = bce_obj(y_true, logits, sample_weight=2.3) + self.assertAlmostEqual(self.evaluate(loss), 0.7205, 3) + + def test_sample_weighted(self): + bce_obj = keras.losses.BinaryCrossentropy() + y_true = constant_op.constant([1, 0, 1, 0, 0, 1], shape=(2, 3)) + y_pred = constant_op.constant([1, 1, 1, 0, 1, 0], + shape=(2, 3), + dtype=dtypes.float64) + sample_weight = constant_op.constant([1.2, 3.4], shape=(2, 1)) + loss = bce_obj(y_true, y_pred, sample_weight=sample_weight) + self.assertAlmostEqual(self.evaluate(loss), 21.4907, 3) + + # Test with logits. + y_true = constant_op.constant([[0, 0, 1], [1, 0, 0], [0, 1, 0]]) + logits = constant_op.constant( + [[100.0, -100.0, -100.0], [-100.0, 100.0, -100.0], + [-100.0, -100.0, 100.0]], + dtype=dtypes.float64) + weights = constant_op.constant([3, 2, 8]) + bce_obj = keras.losses.BinaryCrossentropy(from_logits=True) + loss = bce_obj(y_true, logits, sample_weight=weights) + self.assertAlmostEqual(self.evaluate(loss), 288.8888, 3) + + def test_no_reduction(self): + y_true = constant_op.constant(((1, 0, 1), (1, 1, 0), (0, 1, 1))) + logits = constant_op.constant(((100.0, -100.0, 100.0), + (100.0, -100.0, 100.0), + (100.0, 100.0, -100.0))) + bce_obj = keras.losses.BinaryCrossentropy( + from_logits=True, reduction=losses_impl.ReductionV2.NONE) + loss = bce_obj(y_true, logits) + self.assertAllClose((0., 66.6666, 66.6666), self.evaluate(loss), 3) + + def test_label_smoothing(self): + logits = constant_op.constant([[100.0, -100.0, -100.0]]) + y_true = constant_op.constant([[1, 0, 1]]) + label_smoothing = 0.1 + # Loss: max(x, 0) - x * z + log(1 + exp(-abs(x))) + # Label smoothing: z' = z * (1 - L) + 0.5L + # 1 = 1 - 0.5L + # 0 = 0.5L + # Applying the above two fns to the given input: + # (100 - 100 * (1 - 0.5 L) + 0 + + # 0 + 100 * (0.5 L) + 0 + + # 0 + 100 * (1 - 0.5 L) + 0) * (1/3) + # = (100 + 50L) * 1/3 + bce_obj = keras.losses.BinaryCrossentropy( + from_logits=True, label_smoothing=label_smoothing) + loss = bce_obj(y_true, logits) + expected_value = (100.0 + 50.0 * label_smoothing) / 3.0 + self.assertAlmostEqual(self.evaluate(loss), expected_value, 3) + + +@test_util.run_all_in_graph_and_eager_modes +class CategoricalCrossentropyTest(test.TestCase): + + def test_config(self): + cce_obj = keras.losses.CategoricalCrossentropy( + reduction=losses_impl.ReductionV2.SUM, name='bce_1') + self.assertEqual(cce_obj.name, 'bce_1') + self.assertEqual(cce_obj.reduction, losses_impl.ReductionV2.SUM) + + def test_all_correct_unweighted(self): + y_true = constant_op.constant([[1, 0, 0], [0, 1, 0], [0, 0, 1]], + dtype=dtypes.int64) + y_pred = constant_op.constant([[1., 0., 0.], [0., 1., 0.], [0., 0., 1.]], + dtype=dtypes.float32) + cce_obj = keras.losses.CategoricalCrossentropy() + loss = cce_obj(y_true, y_pred) + self.assertAlmostEqual(self.evaluate(loss), 0.0, 3) + + # Test with logits. + logits = constant_op.constant([[10., 0., 0.], [0., 10., 0.], [0., 0., 10.]]) + cce_obj = keras.losses.CategoricalCrossentropy(from_logits=True) + loss = cce_obj(y_true, logits) + self.assertAlmostEqual(self.evaluate(loss), 0.0, 3) + + def test_unweighted(self): + cce_obj = keras.losses.CategoricalCrossentropy() + y_true = constant_op.constant([[1, 0, 0], [0, 1, 0], [0, 0, 1]]) + y_pred = constant_op.constant( + [[.9, .05, .05], [.5, .89, .6], [.05, .01, .94]], dtype=dtypes.float32) + loss = cce_obj(y_true, y_pred) + self.assertAlmostEqual(self.evaluate(loss), .3239, 3) + + # Test with logits. + logits = constant_op.constant([[8., 1., 1.], [0., 9., 1.], [2., 3., 5.]]) + cce_obj = keras.losses.CategoricalCrossentropy(from_logits=True) + loss = cce_obj(y_true, logits) + self.assertAlmostEqual(self.evaluate(loss), .0573, 3) + + def test_scalar_weighted(self): + cce_obj = keras.losses.CategoricalCrossentropy() + y_true = constant_op.constant([[1, 0, 0], [0, 1, 0], [0, 0, 1]]) + y_pred = constant_op.constant( + [[.9, .05, .05], [.5, .89, .6], [.05, .01, .94]], dtype=dtypes.float32) + loss = cce_obj(y_true, y_pred, sample_weight=2.3) + self.assertAlmostEqual(self.evaluate(loss), .7449, 3) + + # Test with logits. + logits = constant_op.constant([[8., 1., 1.], [0., 9., 1.], [2., 3., 5.]]) + cce_obj = keras.losses.CategoricalCrossentropy(from_logits=True) + loss = cce_obj(y_true, logits, sample_weight=2.3) + self.assertAlmostEqual(self.evaluate(loss), .1317, 3) + + def test_sample_weighted(self): + cce_obj = keras.losses.CategoricalCrossentropy() + y_true = constant_op.constant([[1, 0, 0], [0, 1, 0], [0, 0, 1]]) + y_pred = constant_op.constant( + [[.9, .05, .05], [.5, .89, .6], [.05, .01, .94]], dtype=dtypes.float32) + sample_weight = constant_op.constant([[1.2], [3.4], [5.6]], shape=(3, 1)) + loss = cce_obj(y_true, y_pred, sample_weight=sample_weight) + self.assertAlmostEqual(self.evaluate(loss), 1.0696, 3) + + # Test with logits. + logits = constant_op.constant([[8., 1., 1.], [0., 9., 1.], [2., 3., 5.]]) + cce_obj = keras.losses.CategoricalCrossentropy(from_logits=True) + loss = cce_obj(y_true, logits, sample_weight=sample_weight) + self.assertAlmostEqual(self.evaluate(loss), 0.31829, 3) + + def test_no_reduction(self): + y_true = constant_op.constant([[1, 0, 0], [0, 1, 0], [0, 0, 1]]) + logits = constant_op.constant([[8., 1., 1.], [0., 9., 1.], [2., 3., 5.]]) + cce_obj = keras.losses.CategoricalCrossentropy( + from_logits=True, reduction=losses_impl.ReductionV2.NONE) + loss = cce_obj(y_true, logits) + self.assertAllClose((0.001822, 0.000459, 0.169846), self.evaluate(loss), 3) + + def test_label_smoothing(self): + logits = constant_op.constant([[100.0, -100.0, -100.0]]) + y_true = constant_op.constant([[1, 0, 0]]) + label_smoothing = 0.1 + # Softmax Cross Entropy Loss: -\sum_i p_i \log q_i + # where for a softmax activation + # \log q_i = x_i - \log \sum_j \exp x_j + # = x_i - x_max - \log \sum_j \exp (x_j - x_max) + # For our activations, [100, -100, -100] + # \log ( exp(0) + exp(-200) + exp(-200) ) = 0 + # so our log softmaxes become: [0, -200, -200] + # Label smoothing: z' = z * (1 - L) + L/n + # 1 = 1 - L + L/n + # 0 = L/n + # Applying the above two fns to the given input: + # -0 * (1 - L + L/n) + 200 * L/n + 200 * L/n = 400 L/n + cce_obj = keras.losses.CategoricalCrossentropy( + from_logits=True, label_smoothing=label_smoothing) + loss = cce_obj(y_true, logits) + expected_value = 400.0 * label_smoothing / 3.0 + self.assertAlmostEqual(self.evaluate(loss), expected_value, 3) + + def test_all_correct_unweighted_sparse(self): + y_true = constant_op.constant([[0], [1], [2]], dtype=dtypes.int64) + y_pred = constant_op.constant([[1., 0., 0.], [0., 1., 0.], [0., 0., 1.]], + dtype=dtypes.float32) + cce_obj = keras.losses.CategoricalCrossentropy() + loss = cce_obj(y_true, y_pred) + self.assertAlmostEqual(self.evaluate(loss), 0.0, 3) + + # Test with logits. + logits = constant_op.constant([[10., 0., 0.], [0., 10., 0.], [0., 0., 10.]]) + cce_obj = keras.losses.CategoricalCrossentropy(from_logits=True) + loss = cce_obj(y_true, logits) + self.assertAlmostEqual(self.evaluate(loss), 0.0, 3) + + def test_unweighted_sparse(self): + cce_obj = keras.losses.CategoricalCrossentropy() + y_true = constant_op.constant([0, 1, 2]) + y_pred = constant_op.constant( + [[.9, .05, .05], [.5, .89, .6], [.05, .01, .94]], dtype=dtypes.float32) + loss = cce_obj(y_true, y_pred) + self.assertAlmostEqual(self.evaluate(loss), .3239, 3) + + # Test with logits. + logits = constant_op.constant([[8., 1., 1.], [0., 9., 1.], [2., 3., 5.]]) + cce_obj = keras.losses.CategoricalCrossentropy(from_logits=True) + loss = cce_obj(y_true, logits) + self.assertAlmostEqual(self.evaluate(loss), .0573, 3) + + def test_scalar_weighted_sparse(self): + cce_obj = keras.losses.CategoricalCrossentropy() + y_true = constant_op.constant([[0], [1], [2]]) + y_pred = constant_op.constant( + [[.9, .05, .05], [.5, .89, .6], [.05, .01, .94]], dtype=dtypes.float32) + loss = cce_obj(y_true, y_pred, sample_weight=2.3) + self.assertAlmostEqual(self.evaluate(loss), .7449, 3) + + # Test with logits. + logits = constant_op.constant([[8., 1., 1.], [0., 9., 1.], [2., 3., 5.]]) + cce_obj = keras.losses.CategoricalCrossentropy(from_logits=True) + loss = cce_obj(y_true, logits, sample_weight=2.3) + self.assertAlmostEqual(self.evaluate(loss), .1317, 3) + + def test_sample_weighted_sparse(self): + cce_obj = keras.losses.CategoricalCrossentropy() + y_true = constant_op.constant([[0], [1], [2]]) + y_pred = constant_op.constant( + [[.9, .05, .05], [.5, .89, .6], [.05, .01, .94]], dtype=dtypes.float32) + sample_weight = constant_op.constant([[1.2], [3.4], [5.6]], shape=(3, 1)) + loss = cce_obj(y_true, y_pred, sample_weight=sample_weight) + self.assertAlmostEqual(self.evaluate(loss), 1.0696, 3) + + # Test with logits. + logits = constant_op.constant([[8., 1., 1.], [0., 9., 1.], [2., 3., 5.]]) + cce_obj = keras.losses.CategoricalCrossentropy(from_logits=True) + loss = cce_obj(y_true, logits, sample_weight=sample_weight) + self.assertAlmostEqual(self.evaluate(loss), 0.31829, 3) + + def test_no_reduction_sparse(self): + y_true = constant_op.constant([[0], [1], [2]]) + logits = constant_op.constant([[8., 1., 1.], [0., 9., 1.], [2., 3., 5.]]) + cce_obj = keras.losses.CategoricalCrossentropy( + from_logits=True, reduction=losses_impl.ReductionV2.NONE) + loss = cce_obj(y_true, logits) + self.assertAllClose((0.001822, 0.000459, 0.169846), self.evaluate(loss), 3) + + if __name__ == '__main__': test.main() diff --git a/tensorflow/python/keras/metrics.py b/tensorflow/python/keras/metrics.py index 2ea64055979..3c2682e4c6f 100644 --- a/tensorflow/python/keras/metrics.py +++ b/tensorflow/python/keras/metrics.py @@ -24,9 +24,10 @@ import functools import sys import types import weakref +from enum import Enum +import numpy as np import six -from tensorflow.python.compat import compat from tensorflow.python.eager import context from tensorflow.python.eager import function from tensorflow.python.framework import dtypes @@ -48,8 +49,10 @@ from tensorflow.python.keras.losses import sparse_categorical_crossentropy from tensorflow.python.keras.losses import squared_hinge from tensorflow.python.keras.utils.generic_utils import deserialize_keras_object from tensorflow.python.keras.utils.generic_utils import serialize_keras_object +from tensorflow.python.keras.utils.generic_utils import to_list +from tensorflow.python.keras.utils.losses_utils import squeeze_or_expand_dimensions from tensorflow.python.ops import array_ops -from tensorflow.python.ops import confusion_matrix +from tensorflow.python.ops import check_ops from tensorflow.python.ops import control_flow_ops from tensorflow.python.ops import init_ops from tensorflow.python.ops import math_ops @@ -63,13 +66,6 @@ from tensorflow.python.util.tf_export import tf_export from tensorflow.tools.docs import doc_controls -def check_is_tensor_or_operation(x, name): - """Raises type error if the given input is not a tensor or operation.""" - if not (isinstance(x, ops.Tensor) or isinstance(x, ops.Operation)): - raise TypeError('{0} must be a Tensor or Operation, given: {1}'.format( - name, x)) - - def clone_metric(metric): """Returns a clone of the metric if stateful, otherwise returns it as is.""" if isinstance(metric, Metric): @@ -102,8 +98,6 @@ def update_state_wrapper(update_state_fn): update_op = update_state_fn(*args, **kwargs) if update_op is not None: # update_op will be None in eager execution. metric_obj.add_update(update_op, inputs=True) - check_is_tensor_or_operation( - update_op, 'Metric {0}\'s update'.format(metric_obj.name)) return update_op return tf_decorator.make_decorator(update_state_fn, decorated) @@ -128,7 +122,7 @@ def result_wrapper(result_fn): `merge_call()`. """ - def decorated(metric_obj, *args): + def decorated(_, *args): """Decorated function with merge_call.""" replica_context = distribution_strategy_context.get_replica_context() if replica_context is None: # if in cross replica context already @@ -147,9 +141,8 @@ def result_wrapper(result_fn): # Wrapping result in merge_call. merge_call is used when we want to leave # replica mode and compute a value in cross replica mode. - result_t = replica_context.merge_call(merge_fn_wrapper, result_fn, *args) - check_is_tensor_or_operation(result_t, - 'Metric {0}\'s result'.format(metric_obj.name)) + result_t = replica_context.merge_call( + merge_fn_wrapper, args=(result_fn,) + args) return result_t return tf_decorator.make_decorator(result_fn, decorated) @@ -170,108 +163,169 @@ def weakmethod(method): return inner -def safe_div(numerator, denominator): - """Computes a safe divide which returns 0 if the denominator is zero. +class _ConfusionMatrix(Enum): + TRUE_POSITIVES = 'tp' + FALSE_POSITIVES = 'fp' + TRUE_NEGATIVES = 'tn' + FALSE_NEGATIVES = 'fn' - Note that the function contains an additional conditional check that is - necessary for avoiding situations where the loss is zero causing NaNs to - creep into the gradient computation. + +def _assert_thresholds_range(thresholds): + invalid_thresholds = [t for t in thresholds if t < 0 or t > 1] + if any(invalid_thresholds): + raise ValueError('Threshold values must be in [0, 1]. Invalid values: {}' + .format(invalid_thresholds)) + + +def _update_confusion_matrix_variables(variables_to_update, + y_true, + y_pred, + thresholds, + sample_weight=None): + """Returns op to update the given confusion matrix variables. + + For every pair of values in y_true and y_pred: + + true_positive: y_true == True and y_pred > thresholds + false_negatives: y_true == True and y_pred <= thresholds + true_negatives: y_true == False and y_pred <= thresholds + false_positive: y_true == False and y_pred > thresholds + + The results will be weighted and added together. When multiple thresholds are + provided, we will repeat the same for every threshold. + + For estimation of these metrics over a stream of data, the function creates an + `update_op` operation that updates the given variables. + + If `sample_weight` is `None`, weights default to 1. + Use weights of 0 to mask values. Args: - numerator: An arbitrary `Tensor`. - denominator: A `Tensor` whose shape matches `numerator` and whose values are - assumed to be non-negative. + variables_to_update: Dictionary with 'tp', 'fn', 'tn', 'fp' as valid keys + and corresponding variables to update as values. + y_true: A `Tensor` whose shape matches `y_pred`. Will be cast to `bool`. + y_pred: A floating point `Tensor` of arbitrary shape and whose values are in + the range `[0, 1]`. + thresholds: A float value or a python list or tuple of float thresholds in + `[0, 1]`. + sample_weight: Optional `Tensor` whose rank is either 0, or the same rank as + `y_true`, and must be broadcastable to `y_true` (i.e., all dimensions must + be either `1`, or the same as the corresponding `y_true` dimension). Returns: - The element-wise value of the numerator divided by the denominator. + Update op. + + Raises: + ValueError: If `y_pred` and `y_true` have mismatched shapes, or if + `sample_weight` is not `None` and its shape doesn't match `y_pred`, or if + `variables_to_update` contains invalid keys. """ - if compat.forward_compatible(2018, 11, 1): - return math_ops.div_no_nan(numerator, denominator) - return array_ops.where( - math_ops.greater(denominator, 0), - math_ops.div(numerator, - array_ops.where( - math_ops.equal(denominator, 0), - array_ops.ones_like(denominator), denominator)), - array_ops.zeros_like(numerator)) + if variables_to_update is None: + return + y_true = ops.convert_to_tensor(y_true) + y_pred = ops.convert_to_tensor(y_pred) + y_pred.shape.assert_is_compatible_with(y_true.shape) + if not any( + key for key in variables_to_update if key in list(_ConfusionMatrix)): + raise ValueError( + 'Please provide at least one valid confusion matrix ' + 'variable to update. Valid variable key options are: "{}". ' + 'Received: "{}"'.format( + list(_ConfusionMatrix), variables_to_update.keys())) -def squeeze_or_expand_dimensions(y_pred, y_true, sample_weight): - """Squeeze or expand last dimension if needed. + invalid_keys = [ + key for key in variables_to_update if key not in list(_ConfusionMatrix) + ] + if invalid_keys: + raise ValueError( + 'Invalid keys: {}. Valid variable key options are: "{}"'.format( + invalid_keys, list(_ConfusionMatrix))) - 1. Squeezes last dim of `y_pred` or `y_true` if their rank differs by 1 - (using `confusion_matrix.remove_squeezable_dimensions`). - 2. Squeezes or expands last dim of `sample_weight` if its rank differs by 1 - from the new rank of `y_pred`. - If `sample_weight` is scalar, it is kept scalar. + with ops.control_dependencies([ + check_ops.assert_greater_equal( + y_pred, + math_ops.cast(0.0, dtype=y_pred.dtype), + message='predictions must be >= 0'), + check_ops.assert_less_equal( + y_pred, + math_ops.cast(1.0, dtype=y_pred.dtype), + message='predictions must be <= 1') + ]): + y_pred, y_true, sample_weight = squeeze_or_expand_dimensions( + math_ops.cast(y_pred, dtype=dtypes.float32), + math_ops.cast(y_true, dtype=dtypes.bool), sample_weight) - This will use static shape if available. Otherwise, it will add graph - operations, which could result in a performance hit. + thresholds = to_list(thresholds) + num_thresholds = len(thresholds) + num_predictions = array_ops.size(y_pred) - Args: - y_pred: Predicted values, a `Tensor` of arbitrary dimensions. - y_true: Optional label `Tensor` whose dimensions match `y_pred`. - sample_weight: Optional weight scalar or `Tensor` whose dimensions match - `y_pred`. + # Reshape predictions and labels. + predictions_2d = array_ops.reshape(y_pred, [1, -1]) + labels_2d = array_ops.reshape( + math_ops.cast(y_true, dtype=dtypes.bool), [1, -1]) - Returns: - Tuple of `y_pred`, `y_true` and `sample_weight`. Each of them possibly has - the last dimension squeezed, - `sample_weight` could be extended by one dimension. - """ - if y_true is not None: - # squeeze last dim of `y_pred` or `y_true` if their rank differs by 1 - y_true, y_pred = confusion_matrix.remove_squeezable_dimensions( - y_true, y_pred) + # Tile the thresholds for every prediction. + thresh_tiled = array_ops.tile( + array_ops.expand_dims(array_ops.constant(thresholds), 1), + array_ops.stack([1, num_predictions])) - if sample_weight is None: - return y_pred, y_true, None + # Tile the predictions for every threshold. + preds_tiled = array_ops.tile(predictions_2d, [num_thresholds, 1]) - sample_weight = ops.convert_to_tensor(sample_weight) - weights_shape = sample_weight.get_shape() - weights_rank = weights_shape.ndims - if weights_rank == 0: # If weights is scalar, do nothing. - return y_pred, y_true, sample_weight + # Compare predictions and threshold. + pred_is_pos = math_ops.greater(preds_tiled, thresh_tiled) - y_pred_shape = y_pred.get_shape() - y_pred_rank = y_pred_shape.ndims - if (y_pred_rank is not None) and (weights_rank is not None): - # Use static rank. - if weights_rank - y_pred_rank == 1: - sample_weight = array_ops.squeeze(sample_weight, [-1]) - elif y_pred_rank - weights_rank == 1: - sample_weight = array_ops.expand_dims(sample_weight, [-1]) - return y_pred, y_true, sample_weight + # Tile labels by number of thresholds + label_is_pos = array_ops.tile(labels_2d, [num_thresholds, 1]) - # Use dynamic rank. - weights_rank_tensor = array_ops.rank(sample_weight) - rank_diff = weights_rank_tensor - array_ops.rank(y_pred) - maybe_squeeze_weights = lambda: array_ops.squeeze(sample_weight, [-1]) + if sample_weight is not None: + weights = weights_broadcast_ops.broadcast_weights( + math_ops.cast(sample_weight, dtype=dtypes.float32), y_pred) + weights_tiled = array_ops.tile( + array_ops.reshape(weights, [1, -1]), [num_thresholds, 1]) + else: + weights_tiled = None - def _maybe_expand_weights(): - return control_flow_ops.cond( - math_ops.equal(rank_diff, - -1), lambda: array_ops.expand_dims(sample_weight, [-1]), - lambda: sample_weight) + update_ops = [] - def _maybe_adjust_weights(): - return control_flow_ops.cond( - math_ops.equal(rank_diff, 1), maybe_squeeze_weights, - _maybe_expand_weights) + def weighted_assign_add(label, pred, weights, var): + label_and_pred = math_ops.cast( + math_ops.logical_and(label, pred), dtype=dtypes.float32) + if weights is not None: + label_and_pred *= weights + return state_ops.assign_add(var, math_ops.reduce_sum(label_and_pred, 1)) - # squeeze or expand last dim of `sample_weight` if its rank differs by 1 - # from the new rank of `y_pred`. - sample_weight = control_flow_ops.cond( - math_ops.equal(weights_rank_tensor, 0), lambda: sample_weight, - _maybe_adjust_weights) - return y_pred, y_true, sample_weight + loop_vars = { + _ConfusionMatrix.TRUE_POSITIVES: (label_is_pos, pred_is_pos), + } + update_tn = _ConfusionMatrix.TRUE_NEGATIVES in variables_to_update + update_fp = _ConfusionMatrix.FALSE_POSITIVES in variables_to_update + update_fn = _ConfusionMatrix.FALSE_NEGATIVES in variables_to_update + + if update_fn or update_tn: + pred_is_neg = math_ops.logical_not(pred_is_pos) + loop_vars[_ConfusionMatrix.FALSE_NEGATIVES] = (label_is_pos, pred_is_neg) + + if update_fp or update_tn: + label_is_neg = math_ops.logical_not(label_is_pos) + loop_vars[_ConfusionMatrix.FALSE_POSITIVES] = (label_is_neg, pred_is_pos) + if update_tn: + loop_vars[_ConfusionMatrix.TRUE_NEGATIVES] = (label_is_neg, pred_is_neg) + + for matrix_cond, (label, pred) in loop_vars.items(): + if matrix_cond in variables_to_update: + update_ops.append( + weighted_assign_add(label, pred, weights_tiled, + variables_to_update[matrix_cond])) + return control_flow_ops.group(update_ops) @six.add_metaclass(abc.ABCMeta) class Metric(Layer): """Encapsulates metric logic and state. - Usage with eager execution: + Usage: ```python m = SomeMetric(...) @@ -280,19 +334,6 @@ class Metric(Layer): print('Final result: ', m.result().numpy()) ``` - Usage with graph execution: - - ```python - m = SomeMetric(...) - init_op = tf.variables_initializer(m.variables) # Initialize variables - with tf.Session() as sess: - sess.run(init_op) - for input in ...: - update_op = m.update_state(input) - sess.run(update_op) - print('Final result: ', sess.run(m.result())) - ``` - Usage with tf.keras API: ```python @@ -388,9 +429,20 @@ class Metric(Layer): Returns: The metric value tensor. """ - update_op = self.update_state(*args, **kwargs) # pylint: disable=not-callable + update_op = self.update_state(*args, **kwargs) with ops.control_dependencies([update_op]): - return self.result() # pylint: disable=not-callable + result_t = self.result() + + # We are adding the metric object as metadata on the result tensor. + # This is required when we want to use a metric with `add_metric` API on + # a Model/Layer in graph mode. This metric instance will later be used + # to reset variable state after each epoch of training. + # Example: + # model = Model() + # model.add_metric(Mean()(values), name='mean') + if not context.executing_eagerly(): + result_t._metric_obj = self # pylint: disable=protected-access + return result_t def reset_states(self): """Resets all of the metric state variables. @@ -459,15 +511,35 @@ class Metric(Layer): ### End: For use by subclasses ### +@tf_export('metrics.Mean', 'keras.metrics.Mean') class Mean(Metric): """Computes the (weighted) mean of the given values. + For example, if values is [1, 3, 5, 7] then the mean is 4. + If the weights were specified as [1, 1, 0, 0] then the mean would be 2. + This metric creates two variables, `total` and `count` that are used to compute the average of `values`. This average is ultimately returned as `mean` which is an idempotent operation that simply divides `total` by `count`. If `sample_weight` is `None`, weights default to 1. Use `sample_weight` of 0 to mask values. + + Usage: + + ```python + m = tf.metrics.Mean() + m.update_state([1, 3, 5, 7]) + print('Final result: ', m.result().numpy()) # Final result: 4.0 + ``` + + Usage with tf.keras API: + + ```python + model = keras.models.Model(inputs, outputs) + model.add_metric(metrics_module.Mean(name='mean_1')(outputs)) + model.compile('sgd', loss='mse') + ``` """ def __init__(self, name='mean', dtype=None): @@ -525,11 +597,10 @@ class Mean(Metric): # updated. update_total_op = state_ops.assign_add(self.total, values) with ops.control_dependencies([update_total_op]): - update_count_op = state_ops.assign_add(self.count, num_values) - return ops.convert_to_tensor(update_count_op) + return state_ops.assign_add(self.count, num_values) def result(self): - return safe_div(self.total, self.count) + return math_ops.div_no_nan(self.total, self.count) class MeanMetricWrapper(Mean): @@ -574,14 +645,20 @@ class MeanMetricWrapper(Mean): matches, sample_weight=sample_weight) def get_config(self): - config = self._fn_kwargs + config = {'fn': self._fn} + config.update(self._fn_kwargs) base_config = super(MeanMetricWrapper, self).get_config() return dict(list(base_config.items()) + list(config.items())) -class BinaryAccuracy(MeanMetricWrapper): +@tf_export('metrics.Accuracy', 'keras.metrics.Accuracy') +class Accuracy(MeanMetricWrapper): """Calculates how often predictions matches labels. + For example, if `y_true` is [1, 2, 3, 4] and `y_pred` is [0, 2, 3, 4] + then the accuracy is 3/4 or .75. If the weights were specified as + [1, 1, 0, 0] then the accuracy would be 1/2 or .5. + This metric creates two local variables, `total` and `count` that are used to compute the frequency with which `y_pred` matches `y_true`. This frequency is ultimately returned as `binary accuracy`: an idempotent operation that simply @@ -589,6 +666,63 @@ class BinaryAccuracy(MeanMetricWrapper): If `sample_weight` is `None`, weights default to 1. Use `sample_weight` of 0 to mask values. + + Usage: + + ```python + m = tf.metrics.Accuracy() + m.update_state([1, 2, 3, 4], [0, 2, 3, 4]) + print('Final result: ', m.result().numpy()) # Final result: 0.75 + ``` + + Usage with tf.keras API: + + ```python + model = keras.models.Model(inputs, outputs) + model.compile('sgd', loss='mse', metrics=[tf.metrics.Accuracy()]) + ``` + """ + + def __init__(self, name='accuracy', dtype=None): + super(Accuracy, self).__init__(accuracy, name, dtype=dtype) + + @classmethod + def from_config(cls, config): + if 'fn' in config: + config.pop('fn') + return super(Accuracy, cls).from_config(config) + + +@tf_export('metrics.BinaryAccuracy', 'keras.metrics.BinaryAccuracy') +class BinaryAccuracy(MeanMetricWrapper): + """Calculates how often predictions matches labels. + + For example, if `y_true` is [1, 1, 0, 0] and `y_pred` is [0.98, 1, 0, 0.6] + then the binary accuracy is 3/4 or .75. If the weights were specified as + [1, 0, 0, 1] then the binary accuracy would be 1/2 or .5. + + This metric creates two local variables, `total` and `count` that are used to + compute the frequency with which `y_pred` matches `y_true`. This frequency is + ultimately returned as `binary accuracy`: an idempotent operation that simply + divides `total` by `count`. + + If `sample_weight` is `None`, weights default to 1. + Use `sample_weight` of 0 to mask values. + + Usage: + + ```python + m = tf.metrics.BinaryAccuracy() + m.update_state([1, 1, 0, 0], [0.98, 1, 0, 0.6]) + print('Final result: ', m.result().numpy()) # Final result: 0.75 + ``` + + Usage with tf.keras API: + + ```python + model = keras.models.Model(inputs, outputs) + model.compile('sgd', loss='mse', metrics=[tf.metrics.BinaryAccuracy()]) + ``` """ def __init__(self, name='binary_accuracy', dtype=None, threshold=0.5): @@ -603,17 +737,48 @@ class BinaryAccuracy(MeanMetricWrapper): super(BinaryAccuracy, self).__init__( binary_accuracy, name, dtype=dtype, threshold=threshold) + @classmethod + def from_config(cls, config): + if 'fn' in config: + config.pop('fn') + return super(BinaryAccuracy, cls).from_config(config) + +@tf_export( + 'metrics.CategoricalAccuracy', 'keras.metrics.CategoricalAccuracy') class CategoricalAccuracy(MeanMetricWrapper): """Calculates how often predictions matches labels. + For example, if `y_true` is [[0, 0, 1], [0, 1, 0]] and `y_pred` is + [[0.1, 0.9, 0.8], [0.05, 0.95, 0]] then the categorical accuracy is 1/2 or .5. + If the weights were specified as [0.7, 0.3] then the categorical accuracy + would be .3. + This metric creates two local variables, `total` and `count` that are used to compute the frequency with which `y_pred` matches `y_true`. This frequency is ultimately returned as `categorical accuracy`: an idempotent operation that simply divides `total` by `count`. + `y_pred` and `y_true` should be passed in as vectors of probabilities, rather + than as labels. If necessary, use `tf.one_hot` to expand `y_true` as a vector. + If `sample_weight` is `None`, weights default to 1. Use `sample_weight` of 0 to mask values. + + Usage: + + ```python + m = tf.metrics.CategoricalAccuracy() + m.update_state([[0, 0, 1], [0, 1, 0]], [[0.1, 0.9, 0.8], [0.05, 0.95, 0]]) + print('Final result: ', m.result().numpy()) # Final result: 0.5 + ``` + + Usage with tf.keras API: + + ```python + model = keras.models.Model(inputs, outputs) + model.compile('sgd', loss='mse', metrics=[tf.metrics.CategoricalAccuracy()]) + ``` """ def __init__(self, name='categorical_accuracy', dtype=None): @@ -626,10 +791,24 @@ class CategoricalAccuracy(MeanMetricWrapper): super(CategoricalAccuracy, self).__init__( categorical_accuracy, name, dtype=dtype) + @classmethod + def from_config(cls, config): + if 'fn' in config: + config.pop('fn') + return super(CategoricalAccuracy, cls).from_config(config) + +@tf_export( + 'metrics.SparseCategoricalAccuracy', + 'keras.metrics.SparseCategoricalAccuracy') class SparseCategoricalAccuracy(MeanMetricWrapper): """Calculates how often predictions matches integer labels. + For example, if `y_true` is [[2], [1]] and `y_pred` is + [[0.1, 0.9, 0.8], [0.05, 0.95, 0]] then the categorical accuracy is 1/2 or .5. + If the weights were specified as [0.7, 0.3] then the categorical accuracy + would be .3. + This metric creates two local variables, `total` and `count` that are used to compute the frequency with which `y_pred` matches `y_true`. This frequency is ultimately returned as `sparse categorical accuracy`: an idempotent operation @@ -637,12 +816,710 @@ class SparseCategoricalAccuracy(MeanMetricWrapper): If `sample_weight` is `None`, weights default to 1. Use `sample_weight` of 0 to mask values. + + Usage: + + ```python + m = tf.metrics.SparseCategoricalAccuracy() + m.update_state([[2], [1]], [[0.1, 0.9, 0.8], [0.05, 0.95, 0]]) + print('Final result: ', m.result().numpy()) # Final result: 0.5 + ``` + + Usage with tf.keras API: + + ```python + model = keras.models.Model(inputs, outputs) + model.compile( + 'sgd', + loss='mse', + metrics=[tf.metrics.SparseCategoricalAccuracy()]) + ``` """ def __init__(self, name='sparse_categorical_accuracy', dtype=None): super(SparseCategoricalAccuracy, self).__init__( sparse_categorical_accuracy, name, dtype=dtype) + @classmethod + def from_config(cls, config): + if 'fn' in config: + config.pop('fn') + return super(SparseCategoricalAccuracy, cls).from_config(config) + + +class _ConfusionMatrixConditionCount(Metric): + """Calculates the number of the given confusion matrix condition.""" + + def __init__(self, + confusion_matrix_cond, + thresholds=None, + name=None, + dtype=None): + """Creates a `_ConfusionMatrixConditionCount` instance. + + Args: + confusion_matrix_cond: One of `_ConfusionMatrix` conditions. + thresholds: (Optional) Defaults to 0.5. A float value or a python + list/tuple of float threshold values in [0, 1]. A threshold is compared + with prediction values to determine the truth value of predictions + (i.e., above the threshold is `true`, below is `false`). One metric + value is generated for each threshold value. + name: (Optional) string name of the metric instance. + dtype: (Optional) data type of the metric result. + """ + super(_ConfusionMatrixConditionCount, self).__init__(name=name, dtype=dtype) + self._confusion_matrix_cond = confusion_matrix_cond + self.thresholds = 0.5 if thresholds is None else thresholds + thresholds = to_list(thresholds) + _assert_thresholds_range(thresholds) + self.accumulator = self.add_weight( + 'accumulator', + shape=(len(thresholds),), + initializer=init_ops.zeros_initializer) + + def update_state(self, y_true, y_pred, sample_weight=None): + """Accumulates the given confusion matrix condition statistics. + + Args: + y_true: The ground truth values. + y_pred: The predicted values. + sample_weight: Optional weighting of each example. Defaults to 1. Can be a + `Tensor` whose rank is either 0, or the same rank as `y_true`, and must + be broadcastable to `y_true`. + + Returns: + Update op. + """ + return _update_confusion_matrix_variables({ + self._confusion_matrix_cond: self.accumulator + }, y_true, y_pred, self.thresholds, sample_weight) + + def result(self): + if isinstance(self.thresholds, (list, tuple)): + result = self.accumulator + else: + result = self.accumulator[0] + return ops.convert_to_tensor(result) + + def reset_states(self): + num_thresholds = len(to_list(self.thresholds)) + for v in self.variables: + K.set_value(v, np.zeros((num_thresholds,))) + + +@tf_export('metrics.FalsePositives', 'keras.metrics.FalsePositives') +class FalsePositives(_ConfusionMatrixConditionCount): + """Calculates the number of false positives. + + For example, if `y_true` is [0, 1, 0, 0] and `y_pred` is [0, 0, 1, 1] + then the false positives value is 2. If the weights were specified as + [0, 0, 1, 0] then the false positives value would be 1. + + If `sample_weight` is given, calculates the sum of the weights of + false positives. This metric creates one local variable, `accumulator` + that is used to keep track of the number of false positives. + + If `sample_weight` is `None`, weights default to 1. + Use `sample_weight` of 0 to mask values. + + Usage: + + ```python + m = tf.metrics.FalsePositives() + m.update_state([0, 1, 0, 0], [0, 0, 1, 1]) + print('Final result: ', m.result().numpy()) # Final result: 2 + ``` + + Usage with tf.keras API: + + ```python + model = keras.models.Model(inputs, outputs) + model.compile('sgd', loss='mse', metrics=[tf.metrics.FalsePositives()]) + ``` + """ + + def __init__(self, thresholds=None, name=None, dtype=None): + """Creates a `FalsePositives` instance. + + Args: + thresholds: (Optional) Defaults to 0.5. A float value or a python + list/tuple of float threshold values in [0, 1]. A threshold is compared + with prediction values to determine the truth value of predictions + (i.e., above the threshold is `true`, below is `false`). One metric + value is generated for each threshold value. + name: (Optional) string name of the metric instance. + dtype: (Optional) data type of the metric result. + """ + super(FalsePositives, self).__init__( + confusion_matrix_cond=_ConfusionMatrix.FALSE_POSITIVES, + thresholds=thresholds, + name=name, + dtype=dtype) + + +@tf_export('metrics.FalseNegatives', 'keras.metrics.FalseNegatives') +class FalseNegatives(_ConfusionMatrixConditionCount): + """Calculates the number of false negatives. + + For example, if `y_true` is [0, 1, 1, 1] and `y_pred` is [0, 1, 0, 0] + then the false negatives value is 2. If the weights were specified as + [0, 0, 1, 0] then the false negatives value would be 1. + + If `sample_weight` is given, calculates the sum of the weights of + false negatives. This metric creates one local variable, `accumulator` + that is used to keep track of the number of false negatives. + + If `sample_weight` is `None`, weights default to 1. + Use `sample_weight` of 0 to mask values. + + Usage: + + ```python + m = tf.metrics.FalseNegatives() + m.update_state([0, 1, 1, 1], [0, 1, 0, 0]) + print('Final result: ', m.result().numpy()) # Final result: 2 + ``` + + Usage with tf.keras API: + + ```python + model = keras.models.Model(inputs, outputs) + model.compile('sgd', loss='mse', metrics=[tf.metrics.FalseNegatives()]) + ``` + """ + + def __init__(self, thresholds=None, name=None, dtype=None): + """Creates a `FalseNegatives` instance. + + Args: + thresholds: (Optional) Defaults to 0.5. A float value or a python + list/tuple of float threshold values in [0, 1]. A threshold is compared + with prediction values to determine the truth value of predictions + (i.e., above the threshold is `true`, below is `false`). One metric + value is generated for each threshold value. + name: (Optional) string name of the metric instance. + dtype: (Optional) data type of the metric result. + """ + super(FalseNegatives, self).__init__( + confusion_matrix_cond=_ConfusionMatrix.FALSE_NEGATIVES, + thresholds=thresholds, + name=name, + dtype=dtype) + + +@tf_export('metrics.TrueNegatives', 'keras.metrics.TrueNegatives') +class TrueNegatives(_ConfusionMatrixConditionCount): + """Calculates the number of true negatives. + + For example, if `y_true` is [0, 1, 0, 0] and `y_pred` is [1, 1, 0, 0] + then the true negatives value is 2. If the weights were specified as + [0, 0, 1, 0] then the true negatives value would be 1. + + If `sample_weight` is given, calculates the sum of the weights of + true negatives. This metric creates one local variable, `accumulator` + that is used to keep track of the number of true negatives. + + If `sample_weight` is `None`, weights default to 1. + Use `sample_weight` of 0 to mask values. + + Usage: + + ```python + m = tf.metrics.TrueNegatives() + m.update_state([0, 1, 0, 0], [1, 1, 0, 0]) + print('Final result: ', m.result().numpy()) # Final result: 2 + ``` + + Usage with tf.keras API: + + ```python + model = keras.models.Model(inputs, outputs) + model.compile('sgd', loss='mse', metrics=[tf.metrics.TrueNegatives()]) + ``` + """ + + def __init__(self, thresholds=None, name=None, dtype=None): + """Creates a `TrueNegatives` instance. + + Args: + thresholds: (Optional) Defaults to 0.5. A float value or a python + list/tuple of float threshold values in [0, 1]. A threshold is compared + with prediction values to determine the truth value of predictions + (i.e., above the threshold is `true`, below is `false`). One metric + value is generated for each threshold value. + name: (Optional) string name of the metric instance. + dtype: (Optional) data type of the metric result. + """ + super(TrueNegatives, self).__init__( + confusion_matrix_cond=_ConfusionMatrix.TRUE_NEGATIVES, + thresholds=thresholds, + name=name, + dtype=dtype) + + +@tf_export('metrics.TruePositives', 'keras.metrics.TruePositives') +class TruePositives(_ConfusionMatrixConditionCount): + """Calculates the number of true positives. + + For example, if `y_true` is [0, 1, 1, 1] and `y_pred` is [1, 0, 1, 1] + then the true positives value is 2. If the weights were specified as + [0, 0, 1, 0] then the true positives value would be 1. + + If `sample_weight` is given, calculates the sum of the weights of + true positives. This metric creates one local variable, `true_positives` + that is used to keep track of the number of true positives. + + If `sample_weight` is `None`, weights default to 1. + Use `sample_weight` of 0 to mask values. + + Usage: + + ```python + m = tf.metrics.TruePositives() + m.update_state([0, 1, 1, 1], [1, 0, 1, 1]) + print('Final result: ', m.result().numpy()) # Final result: 2 + ``` + + Usage with tf.keras API: + + ```python + model = keras.models.Model(inputs, outputs) + model.compile('sgd', loss='mse', metrics=[tf.metrics.TruePositives()]) + ``` + """ + + def __init__(self, thresholds=None, name=None, dtype=None): + """Creates a `TruePositives` instance. + + Args: + thresholds: (Optional) Defaults to 0.5. A float value or a python + list/tuple of float threshold values in [0, 1]. A threshold is compared + with prediction values to determine the truth value of predictions + (i.e., above the threshold is `true`, below is `false`). One metric + value is generated for each threshold value. + name: (Optional) string name of the metric instance. + dtype: (Optional) data type of the metric result. + """ + super(TruePositives, self).__init__( + confusion_matrix_cond=_ConfusionMatrix.TRUE_POSITIVES, + thresholds=thresholds, + name=name, + dtype=dtype) + + +@tf_export('metrics.Precision', 'keras.metrics.Precision') +class Precision(Metric): + """Computes the precision of the predictions with respect to the labels. + + For example, if `y_true` is [0, 1, 1, 1] and `y_pred` is [1, 0, 1, 1] + then the precision value is 2/(2+1) ie. 0.66. If the weights were specified as + [0, 0, 1, 0] then the precision value would be 1. + + The metric creates two local variables, `true_positives` and `false_positives` + that are used to compute the precision. This value is ultimately returned as + `precision`, an idempotent operation that simply divides `true_positives` + by the sum of `true_positives` and `false_positives`. + + If `sample_weight` is `None`, weights default to 1. + Use `sample_weight` of 0 to mask values. + + Usage: + + ```python + m = tf.metrics.Precision() + m.update_state([0, 1, 1, 1], [1, 0, 1, 1]) + print('Final result: ', m.result().numpy()) # Final result: 0.66 + ``` + + Usage with tf.keras API: + + ```python + model = keras.models.Model(inputs, outputs) + model.compile('sgd', loss='mse', metrics=[tf.metrics.Precision()]) + ``` + """ + + def __init__(self, thresholds=None, name=None, dtype=None): + """Creates a `Precision` instance. + + Args: + thresholds: (Optional) Defaults to 0.5. A float value or a python + list/tuple of float threshold values in [0, 1]. A threshold is compared + with prediction values to determine the truth value of predictions + (i.e., above the threshold is `true`, below is `false`). One metric + value is generated for each threshold value. + name: (Optional) string name of the metric instance. + dtype: (Optional) data type of the metric result. + """ + super(Precision, self).__init__(name=name, dtype=dtype) + self.thresholds = 0.5 if thresholds is None else thresholds + thresholds = to_list(thresholds) + _assert_thresholds_range(thresholds) + self.tp = self.add_weight( + 'true_positives', + shape=(len(thresholds),), + initializer=init_ops.zeros_initializer) + self.fp = self.add_weight( + 'false_positives', + shape=(len(thresholds),), + initializer=init_ops.zeros_initializer) + + def update_state(self, y_true, y_pred, sample_weight=None): + """Accumulates true positive and false positive statistics. + + Args: + y_true: The ground truth values. + y_pred: The predicted values. + sample_weight: Optional weighting of each example. Defaults to 1. Can be a + `Tensor` whose rank is either 0, or the same rank as `y_true`, and must + be broadcastable to `y_true`. + + Returns: + Update op. + """ + return _update_confusion_matrix_variables({ + _ConfusionMatrix.TRUE_POSITIVES: self.tp, + _ConfusionMatrix.FALSE_POSITIVES: self.fp + }, y_true, y_pred, self.thresholds, sample_weight) + + def result(self): + result = math_ops.div_no_nan(self.tp, self.tp + self.fp) + return result if isinstance(self.thresholds, (list, tuple)) else result[0] + + def reset_states(self): + num_thresholds = len(to_list(self.thresholds)) + for v in self.variables: + K.set_value(v, np.zeros((num_thresholds,))) + + +@tf_export('metrics.Recall', 'keras.metrics.Recall') +class Recall(Metric): + """Computes the recall of the predictions with respect to the labels. + + For example, if `y_true` is [0, 1, 1, 1] and `y_pred` is [1, 0, 1, 1] + then the recall value is 2/(2+1) ie. 0.66. If the weights were specified as + [0, 0, 1, 0] then the recall value would be 1. + + This metric creates two local variables, `true_positives` and + `false_negatives`, that are used to compute the recall. This value is + ultimately returned as `recall`, an idempotent operation that simply divides + `true_positives` by the sum of `true_positives` and `false_negatives`. + + If `sample_weight` is `None`, weights default to 1. + Use `sample_weight` of 0 to mask values. + + Usage: + + ```python + m = tf.metrics.Recall() + m.update_state([0, 1, 1, 1], [1, 0, 1, 1]) + print('Final result: ', m.result().numpy()) # Final result: 0.66 + ``` + + Usage with tf.keras API: + + ```python + model = keras.models.Model(inputs, outputs) + model.compile('sgd', loss='mse', metrics=[tf.metrics.Recall()]) + ``` + """ + + def __init__(self, thresholds=None, name=None, dtype=None): + """Creates a `Recall` instance. + + Args: + thresholds: (Optional) Defaults to 0.5. A float value or a python + list/tuple of float threshold values in [0, 1]. A threshold is compared + with prediction values to determine the truth value of predictions + (i.e., above the threshold is `true`, below is `false`). One metric + value is generated for each threshold value. + name: (Optional) string name of the metric instance. + dtype: (Optional) data type of the metric result. + """ + super(Recall, self).__init__(name=name, dtype=dtype) + self.thresholds = 0.5 if thresholds is None else thresholds + thresholds = to_list(thresholds) + _assert_thresholds_range(thresholds) + self.tp = self.add_weight( + 'true_positives', + shape=(len(thresholds),), + initializer=init_ops.zeros_initializer) + self.fn = self.add_weight( + 'false_negatives', + shape=(len(thresholds),), + initializer=init_ops.zeros_initializer) + + def update_state(self, y_true, y_pred, sample_weight=None): + """Accumulates true positive and false negative statistics. + + Args: + y_true: The ground truth values. + y_pred: The predicted values. + sample_weight: Optional weighting of each example. Defaults to 1. Can be a + `Tensor` whose rank is either 0, or the same rank as `y_true`, and must + be broadcastable to `y_true`. + + Returns: + Update op. + """ + return _update_confusion_matrix_variables({ + _ConfusionMatrix.TRUE_POSITIVES: self.tp, + _ConfusionMatrix.FALSE_NEGATIVES: self.fn + }, y_true, y_pred, self.thresholds, sample_weight) + + def result(self): + result = math_ops.div_no_nan(self.tp, self.tp + self.fn) + return result if isinstance(self.thresholds, (list, tuple)) else result[0] + + def reset_states(self): + num_thresholds = len(to_list(self.thresholds)) + for v in self.variables: + K.set_value(v, np.zeros((num_thresholds,))) + + +@six.add_metaclass(abc.ABCMeta) +class SensitivitySpecificityBase(Metric): + """Abstract base class for computing sensitivity and specificity. + + For additional information about specificity and sensitivity, see the + following: https://en.wikipedia.org/wiki/Sensitivity_and_specificity + """ + + def __init__(self, value, num_thresholds=200, name=None, dtype=None): + super(SensitivitySpecificityBase, self).__init__(name=name, dtype=dtype) + if num_thresholds <= 0: + raise ValueError('`num_thresholds` must be > 0.') + self.value = value + self.tp = self.add_weight( + 'true_positives', + shape=(num_thresholds,), + initializer=init_ops.zeros_initializer) + self.tn = self.add_weight( + 'true_negatives', + shape=(num_thresholds,), + initializer=init_ops.zeros_initializer) + self.fp = self.add_weight( + 'false_positives', + shape=(num_thresholds,), + initializer=init_ops.zeros_initializer) + self.fn = self.add_weight( + 'false_negatives', + shape=(num_thresholds,), + initializer=init_ops.zeros_initializer) + + # Compute `num_thresholds` thresholds in [0, 1] + if num_thresholds == 1: + self.thresholds = [0.5] + else: + thresholds = [(i + 1) * 1.0 / (num_thresholds - 1) + for i in range(num_thresholds - 2)] + self.thresholds = [0.0] + thresholds + [1.0] + + def update_state(self, y_true, y_pred, sample_weight=None): + """Accumulates confusion matrix statistics. + + Args: + y_true: The ground truth values. + y_pred: The predicted values. + sample_weight: Optional weighting of each example. Defaults to 1. Can be a + `Tensor` whose rank is either 0, or the same rank as `y_true`, and must + be broadcastable to `y_true`. + + Returns: + Update op. + """ + return _update_confusion_matrix_variables({ + _ConfusionMatrix.TRUE_POSITIVES: self.tp, + _ConfusionMatrix.TRUE_NEGATIVES: self.tn, + _ConfusionMatrix.FALSE_POSITIVES: self.fp, + _ConfusionMatrix.FALSE_NEGATIVES: self.fn, + }, y_true, y_pred, self.thresholds, sample_weight) + + def reset_states(self): + num_thresholds = len(self.thresholds) + for v in self.variables: + K.set_value(v, np.zeros((num_thresholds,))) + + +class SensitivityAtSpecificity(SensitivitySpecificityBase): + """Computes the sensitivity at a given specificity. + + `Sensitivity` measures the proportion of actual positives that are correctly + identified as such (tp / (tp + fn)). + `Specificity` measures the proportion of actual negatives that are correctly + identified as such (tn / (tn + fp)). + + This metric creates four local variables, `true_positives`, `true_negatives`, + `false_positives` and `false_negatives` that are used to compute the + sensitivity at the given specificity. The threshold for the given specificity + value is computed and used to evaluate the corresponding sensitivity. + + If `sample_weight` is `None`, weights default to 1. + Use `sample_weight` of 0 to mask values. + + For additional information about specificity and sensitivity, see the + following: https://en.wikipedia.org/wiki/Sensitivity_and_specificity + + Usage: + + ```python + m = tf.metrics.SensitivityAtSpecificity(0.4, num_thresholds=1) + m.update_state([0, 0, 1, 1], [0, 0.5, 0.3, 0.9]) + print('Final result: ', m.result().numpy()) # Final result: 0.5 + ``` + + Usage with tf.keras API: + + ```python + model = keras.models.Model(inputs, outputs) + model.compile( + 'sgd', + loss='mse', + metrics=[tf.metrics.SensitivityAtSpecificity()]) + ``` + """ + + def __init__(self, specificity, num_thresholds=200, name=None, dtype=None): + """Creates a `SensitivityAtSpecificity` instance. + + Args: + specificity: A scalar value in range `[0, 1]`. + num_thresholds: (Optional) Defaults to 200. The number of thresholds to + use for matching the given specificity. + name: (Optional) string name of the metric instance. + dtype: (Optional) data type of the metric result. + """ + if specificity < 0 or specificity > 1: + raise ValueError('`specificity` must be in the range [0, 1].') + super(SensitivityAtSpecificity, self).__init__( + specificity, num_thresholds=num_thresholds, name=name, dtype=dtype) + + def result(self): + # Calculate specificities at all the thresholds. + specificities = math_ops.div_no_nan(self.tn, self.tn + self.fp) + + # Find the index of the threshold where the specificity is closest to the + # given specificity. + min_index = math_ops.argmin( + math_ops.abs(specificities - self.value), axis=0) + min_index = math_ops.cast(min_index, dtypes.int32) + + # Compute sensitivity at that index. + return math_ops.div_no_nan(self.tp[min_index], + self.tp[min_index] + self.fn[min_index]) + + +class SpecificityAtSensitivity(SensitivitySpecificityBase): + """Computes the specificity at a given sensitivity. + + `Sensitivity` measures the proportion of actual positives that are correctly + identified as such (tp / (tp + fn)). + `Specificity` measures the proportion of actual negatives that are correctly + identified as such (tn / (tn + fp)). + + This metric creates four local variables, `true_positives`, `true_negatives`, + `false_positives` and `false_negatives` that are used to compute the + specificity at the given sensitivity. The threshold for the given sensitivity + value is computed and used to evaluate the corresponding specificity. + + If `sample_weight` is `None`, weights default to 1. + Use `sample_weight` of 0 to mask values. + + For additional information about specificity and sensitivity, see the + following: https://en.wikipedia.org/wiki/Sensitivity_and_specificity + + Usage: + + ```python + m = tf.metrics.SpecificityAtSensitivity(0.8, num_thresholds=1) + m.update_state([0, 0, 1, 1], [0, 0.5, 0.3, 0.9]) + print('Final result: ', m.result().numpy()) # Final result: 1.0 + ``` + + Usage with tf.keras API: + + ```python + model = keras.models.Model(inputs, outputs) + model.compile( + 'sgd', + loss='mse', + metrics=[tf.metrics.SpecificityAtSensitivity()]) + ``` + """ + + def __init__(self, sensitivity, num_thresholds=200, name=None, dtype=None): + """Creates a `SpecificityAtSensitivity` instance. + + Args: + sensitivity: A scalar value in range `[0, 1]`. + num_thresholds: (Optional) Defaults to 200. The number of thresholds to + use for matching the given specificity. + name: (Optional) string name of the metric instance. + dtype: (Optional) data type of the metric result. + """ + if sensitivity < 0 or sensitivity > 1: + raise ValueError('`sensitivity` must be in the range [0, 1].') + super(SpecificityAtSensitivity, self).__init__( + sensitivity, num_thresholds=num_thresholds, name=name, dtype=dtype) + + def result(self): + # Calculate sensitivities at all the thresholds. + sensitivities = math_ops.div_no_nan(self.tp, self.tp + self.fn) + + # Find the index of the threshold where the sensitivity is closest to the + # given specificity. + min_index = math_ops.argmin( + math_ops.abs(sensitivities - self.value), axis=0) + min_index = math_ops.cast(min_index, dtypes.int32) + + # Compute specificity at that index. + return math_ops.div_no_nan(self.tn[min_index], + self.tn[min_index] + self.fp[min_index]) + + +class CosineProximity(MeanMetricWrapper): + """Computes the cosine distance between the labels and predictions. + + For example, if `y_true` is [0, 1, 1], and `y_pred` is [1, 0, 1], the cosine + proximity is -0.5. + + This metric keeps the average cosine distance between `predictions` and + `labels` over a stream of data. + + Usage: + ```python + m = tf.metrics.CosineProximity() + m.update_state([0, 1, 1], [1, 0, 1]) + print('Final result: ', m.result().numpy()) # Final result: -0.5 + ``` + + Usage with tf.keras API: + + ```python + model = keras.models.Model(inputs, outputs) + model.compile( + 'sgd', + loss='mse', + metrics=[tf.metrics.CosineProximity()]) + ``` + """ + + def __init__(self, name='cosine_proximity', dtype=None): + super(CosineProximity, self).__init__(cosine, name, dtype=dtype) + + @classmethod + def from_config(cls, config): + if 'fn' in config: + config.pop('fn') + return super(CosineProximity, cls).from_config(config) + + +def accuracy(y_true, y_pred): + y_pred.get_shape().assert_is_compatible_with(y_true.get_shape()) + if y_true.dtype != y_pred.dtype: + y_pred = math_ops.cast(y_pred, y_true.dtype) + return math_ops.cast(math_ops.equal(y_true, y_pred), K.floatx()) + @tf_export('keras.metrics.binary_accuracy') def binary_accuracy(y_true, y_pred, threshold=0.5): diff --git a/tensorflow/python/keras/metrics_test.py b/tensorflow/python/keras/metrics_test.py index 5f5565d4d5a..92398acd8e6 100644 --- a/tensorflow/python/keras/metrics_test.py +++ b/tensorflow/python/keras/metrics_test.py @@ -19,22 +19,25 @@ from __future__ import division from __future__ import print_function import os +from absl.testing import parameterized import numpy as np from tensorflow.python.eager import context +from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops from tensorflow.python.framework import test_util from tensorflow.python.keras import backend as K from tensorflow.python.keras import layers from tensorflow.python.keras import metrics -from tensorflow.python.keras.engine.training import Model +from tensorflow.python.keras.models import Sequential from tensorflow.python.ops import array_ops from tensorflow.python.ops import math_ops -from tensorflow.python.ops import state_ops +from tensorflow.python.ops import random_ops from tensorflow.python.ops import variables from tensorflow.python.platform import test from tensorflow.python.training.checkpointable import util as checkpointable_utils +from tensorflow.python.training.rmsprop import RMSPropOptimizer class KerasMetricsTest(test.TestCase): @@ -47,7 +50,7 @@ class KerasMetricsTest(test.TestCase): output = metric(y_a, y_b) self.assertEqual(K.eval(output).shape, (6,)) - def test_sparse_categorical_accuracy(self): + def test_sparse_categorical_accuracy_int(self): with self.cached_session(): metric = metrics.sparse_categorical_accuracy y_true = K.variable(np.random.randint(0, 7, (6,))) @@ -128,116 +131,6 @@ class KerasMetricsTest(test.TestCase): result = K.eval(metrics.top_k_categorical_accuracy(y_true, y_pred, k=1)) self.assertEqual(result, 0.) - def test_stateful_metrics(self): - with self.cached_session(): - np.random.seed(1334) - - class BinaryTruePositives(layers.Layer): - """Stateful Metric to count the total true positives over all batches. - - Assumes predictions and targets of shape `(samples, 1)`. - - Arguments: - threshold: Float, lower limit on prediction value that counts as a - positive class prediction. - name: String, name for the metric. - """ - - def __init__(self, name='true_positives', **kwargs): - super(BinaryTruePositives, self).__init__(name=name, **kwargs) - self.true_positives = K.variable(value=0, dtype='int32') - self.stateful = True - - def reset_states(self): - K.set_value(self.true_positives, 0) - - def __call__(self, y_true, y_pred): - """Computes the number of true positives in a batch. - - Args: - y_true: Tensor, batch_wise labels - y_pred: Tensor, batch_wise predictions - - Returns: - The total number of true positives seen this epoch at the - completion of the batch. - """ - y_true = math_ops.cast(y_true, 'int32') - y_pred = math_ops.cast(math_ops.round(y_pred), 'int32') - correct_preds = math_ops.cast(math_ops.equal(y_pred, y_true), 'int32') - true_pos = math_ops.cast( - math_ops.reduce_sum(correct_preds * y_true), 'int32') - current_true_pos = self.true_positives * 1 - self.add_update( - state_ops.assign_add(self.true_positives, true_pos), - inputs=[y_true, y_pred]) - return current_true_pos + true_pos - - metric_fn = BinaryTruePositives() - config = metrics.serialize(metric_fn) - metric_fn = metrics.deserialize( - config, custom_objects={'BinaryTruePositives': BinaryTruePositives}) - - # Test on simple model - inputs = layers.Input(shape=(2,)) - outputs = layers.Dense(1, activation='sigmoid')(inputs) - model = Model(inputs, outputs) - model.compile(optimizer='sgd', - loss='binary_crossentropy', - metrics=['acc', metric_fn]) - - # Test fit, evaluate - samples = 100 - x = np.random.random((samples, 2)) - y = np.random.randint(2, size=(samples, 1)) - val_samples = 10 - val_x = np.random.random((val_samples, 2)) - val_y = np.random.randint(2, size=(val_samples, 1)) - - history = model.fit(x, y, - epochs=1, - batch_size=10, - validation_data=(val_x, val_y)) - outs = model.evaluate(x, y, batch_size=10) - preds = model.predict(x) - - def ref_true_pos(y_true, y_pred): - return np.sum(np.logical_and(y_pred > 0.5, y_true == 1)) - - # Test correctness (e.g. updates should have been run) - self.assertAllClose(outs[2], ref_true_pos(y, preds), atol=1e-5) - - # Test correctness of the validation metric computation - val_preds = model.predict(val_x) - val_outs = model.evaluate(val_x, val_y, batch_size=10) - self.assertAllClose( - val_outs[2], ref_true_pos(val_y, val_preds), atol=1e-5) - self.assertAllClose( - val_outs[2], history.history['val_true_positives'][-1], atol=1e-5) - - # Test with generators - gen = [(np.array([x0]), np.array([y0])) for x0, y0 in zip(x, y)] - val_gen = [(np.array([x0]), np.array([y0])) - for x0, y0 in zip(val_x, val_y)] - history = model.fit_generator(iter(gen), - epochs=1, - steps_per_epoch=samples, - validation_data=iter(val_gen), - validation_steps=val_samples) - outs = model.evaluate_generator(iter(gen), steps=samples) - preds = model.predict_generator(iter(gen), steps=samples) - - # Test correctness of the metric results - self.assertAllClose(outs[2], ref_true_pos(y, preds), atol=1e-5) - - # Test correctness of the validation metric computation - val_preds = model.predict_generator(iter(val_gen), steps=val_samples) - val_outs = model.evaluate_generator(iter(val_gen), steps=val_samples) - self.assertAllClose( - val_outs[2], ref_true_pos(val_y, val_preds), atol=1e-5) - self.assertAllClose( - val_outs[2], history.history['val_true_positives'][-1], atol=1e-5) - @test_util.run_in_graph_and_eager_modes(assert_no_eager_garbage=True) def test_mean(self): m = metrics.Mean(name='my_mean') @@ -319,19 +212,19 @@ class KerasMetricsTest(test.TestCase): m = metrics.Mean() v = array_ops.placeholder(dtypes.float32) w = array_ops.placeholder(dtypes.float32) - sess.run(variables.variables_initializer(m.variables)) + self.evaluate(variables.variables_initializer(m.variables)) # check __call__() result_t = m(v, sample_weight=w) result = sess.run(result_t, feed_dict=({v: 100, w: 0.5})) - self.assertEqual(sess.run(m.total), 50) - self.assertEqual(sess.run(m.count), 0.5) + self.assertEqual(self.evaluate(m.total), 50) + self.assertEqual(self.evaluate(m.count), 0.5) self.assertEqual(result, 50 / 0.5) # check update_state() and result() result = sess.run(result_t, feed_dict=({v: [1, 5], w: [1, 0.2]})) - self.assertAlmostEqual(sess.run(m.total), 52, 2) # 50 + 1 + 5 * 0.2 - self.assertAlmostEqual(sess.run(m.count), 1.7, 2) # 0.5 + 1.2 + self.assertAlmostEqual(self.evaluate(m.total), 52, 2) # 50 + 1 + 5 * 0.2 + self.assertAlmostEqual(self.evaluate(m.count), 1.7, 2) # 0.5 + 1.2 self.assertAlmostEqual(result, 52 / 1.7, 2) @test_util.run_in_graph_and_eager_modes @@ -365,6 +258,28 @@ class KerasMetricsTest(test.TestCase): self.assertEqual(200., self.evaluate(restore_mean.result())) self.assertEqual(3, self.evaluate(restore_mean.count)) + @test_util.run_in_graph_and_eager_modes + def test_accuracy(self): + acc_obj = metrics.Accuracy(name='my acc') + + # check config + self.assertEqual(acc_obj.name, 'my acc') + self.assertTrue(acc_obj.stateful) + self.assertEqual(len(acc_obj.variables), 2) + self.assertEqual(acc_obj.dtype, dtypes.float32) + self.evaluate(variables.variables_initializer(acc_obj.variables)) + + # verify that correct value is returned + update_op = acc_obj.update_state([[1], [2], [3], [4]], [[1], [2], [3], [4]]) + self.evaluate(update_op) + result = self.evaluate(acc_obj.result()) + self.assertEqual(result, 1) # 2/2 + + # check with sample_weight + result_t = acc_obj([[2], [1]], [[2], [0]], sample_weight=[[0.5], [0.2]]) + result = self.evaluate(result_t) + self.assertAlmostEqual(result, 0.96, 2) # 4.5/4.7 + @test_util.run_in_graph_and_eager_modes def test_binary_accuracy(self): acc_obj = metrics.BinaryAccuracy(name='my acc') @@ -398,11 +313,6 @@ class KerasMetricsTest(test.TestCase): result = self.evaluate(result_t) self.assertAlmostEqual(result, 0.67, 2) # 4.5/6.7 - # check incompatible shapes - with self.assertRaisesRegexp(ValueError, - r'Shapes \(1,\) and \(2,\) are incompatible'): - acc_obj.update_state([1, 1], [1]) - @test_util.run_in_graph_and_eager_modes def test_binary_accuracy_threshold(self): acc_obj = metrics.BinaryAccuracy(threshold=0.7) @@ -436,47 +346,830 @@ class KerasMetricsTest(test.TestCase): self.assertAlmostEqual(result, 0.93, 2) # 2.5/2.7 @test_util.run_in_graph_and_eager_modes - def test_invalid_result(self): + def test_sparse_categorical_accuracy(self): + acc_obj = metrics.SparseCategoricalAccuracy(name='my acc') - class InvalidResult(metrics.Metric): + # check config + self.assertEqual(acc_obj.name, 'my acc') + self.assertTrue(acc_obj.stateful) + self.assertEqual(len(acc_obj.variables), 2) + self.assertEqual(acc_obj.dtype, dtypes.float32) + self.evaluate(variables.variables_initializer(acc_obj.variables)) - def __init__(self, name='invalid-result', dtype=dtypes.float64): - super(InvalidResult, self).__init__(name=name, dtype=dtype) + # verify that correct value is returned + update_op = acc_obj.update_state([[2], [1]], + [[0.1, 0.1, 0.8], [0.05, 0.95, 0]]) + self.evaluate(update_op) + result = self.evaluate(acc_obj.result()) + self.assertEqual(result, 1) # 2/2 - def update_state(self, *args, **kwargs): - pass + # check with sample_weight + result_t = acc_obj([[2], [1]], [[0.1, 0.1, 0.8], [0.05, 0, 0.95]], + [[0.5], [0.2]]) + result = self.evaluate(result_t) + self.assertAlmostEqual(result, 0.93, 2) # 2.5/2.7 - def result(self): - return 1 - invalid_result_obj = InvalidResult() +def _get_simple_sequential_model(compile_metrics): + model = Sequential() + model.add( + layers.Dense( + 3, activation='relu', input_dim=4, kernel_initializer='ones')) + model.add(layers.Dense(1, activation='sigmoid', kernel_initializer='ones')) + model.compile( + loss='mae', + metrics=compile_metrics, + optimizer=RMSPropOptimizer(learning_rate=0.001)) + return model + + +@test_util.run_all_in_graph_and_eager_modes +class FalsePositivesTest(test.TestCase): + + def test_config(self): + fp_obj = metrics.FalsePositives(name='my_fp', thresholds=[0.4, 0.9]) + self.assertEqual(fp_obj.name, 'my_fp') + self.assertEqual(len(fp_obj.variables), 1) + self.assertEqual(fp_obj.thresholds, [0.4, 0.9]) + + def test_unweighted(self): + fp_obj = metrics.FalsePositives() + self.evaluate(variables.variables_initializer(fp_obj.variables)) + + y_true = constant_op.constant(((0, 1, 0, 1, 0), (0, 0, 1, 1, 1), + (1, 1, 1, 1, 0), (0, 0, 0, 0, 1))) + y_pred = constant_op.constant(((0, 0, 1, 1, 0), (1, 1, 1, 1, 1), + (0, 1, 0, 1, 0), (1, 1, 1, 1, 1))) + + update_op = fp_obj.update_state(y_true, y_pred) + self.evaluate(update_op) + result = fp_obj.result() + self.assertAllClose(7., result) + + def test_weighted(self): + fp_obj = metrics.FalsePositives() + self.evaluate(variables.variables_initializer(fp_obj.variables)) + y_true = constant_op.constant(((0, 1, 0, 1, 0), (0, 0, 1, 1, 1), + (1, 1, 1, 1, 0), (0, 0, 0, 0, 1))) + y_pred = constant_op.constant(((0, 0, 1, 1, 0), (1, 1, 1, 1, 1), + (0, 1, 0, 1, 0), (1, 1, 1, 1, 1))) + sample_weight = constant_op.constant((1., 1.5, 2., 2.5)) + result = fp_obj(y_true, y_pred, sample_weight=sample_weight) + self.assertAllClose(14., self.evaluate(result)) + + def test_unweighted_with_thresholds(self): + fp_obj = metrics.FalsePositives(thresholds=[0.15, 0.5, 0.85]) + self.evaluate(variables.variables_initializer(fp_obj.variables)) + + y_pred = constant_op.constant(((0.9, 0.2, 0.8, 0.1), (0.2, 0.9, 0.7, 0.6), + (0.1, 0.2, 0.4, 0.3), (0, 1, 0.7, 0.3))) + y_true = constant_op.constant(((0, 1, 1, 0), (1, 0, 0, 0), (0, 0, 0, 0), + (1, 1, 1, 1))) + + update_op = fp_obj.update_state(y_true, y_pred) + self.evaluate(update_op) + result = fp_obj.result() + self.assertAllClose([7., 4., 2.], result) + + def test_weighted_with_thresholds(self): + fp_obj = metrics.FalsePositives(thresholds=[0.15, 0.5, 0.85]) + self.evaluate(variables.variables_initializer(fp_obj.variables)) + + y_pred = constant_op.constant(((0.9, 0.2, 0.8, 0.1), (0.2, 0.9, 0.7, 0.6), + (0.1, 0.2, 0.4, 0.3), (0, 1, 0.7, 0.3))) + y_true = constant_op.constant(((0, 1, 1, 0), (1, 0, 0, 0), (0, 0, 0, 0), + (1, 1, 1, 1))) + sample_weight = ((1.0, 2.0, 3.0, 5.0), (7.0, 11.0, 13.0, 17.0), + (19.0, 23.0, 29.0, 31.0), (5.0, 15.0, 10.0, 0)) + + result = fp_obj(y_true, y_pred, sample_weight=sample_weight) + self.assertAllClose([125., 42., 12.], self.evaluate(result)) + + def test_threshold_limit(self): with self.assertRaisesRegexp( - TypeError, - 'Metric invalid-result\'s result must be a Tensor or Operation, given:' - ): - invalid_result_obj.result() + ValueError, + r'Threshold values must be in \[0, 1\]. Invalid values: \[-1, 2\]'): + metrics.FalsePositives(thresholds=[-1, 0.5, 2]) - @test_util.run_in_graph_and_eager_modes - def test_invalid_update(self): + def test_reset_states(self): + fp_obj = metrics.FalsePositives() + model = _get_simple_sequential_model([fp_obj]) + x = np.ones((100, 4)) + y = np.zeros((100, 1)) + model.evaluate(x, y) + self.assertEqual(self.evaluate(fp_obj.accumulator), 100.) + model.evaluate(x, y) + self.assertEqual(self.evaluate(fp_obj.accumulator), 100.) - class InvalidUpdate(metrics.Metric): - def __init__(self, name='invalid-update', dtype=dtypes.float64): - super(InvalidUpdate, self).__init__(name=name, dtype=dtype) +@test_util.run_all_in_graph_and_eager_modes +class FalseNegativesTest(test.TestCase): - def update_state(self, *args, **kwargs): - return [1] + def test_config(self): + fn_obj = metrics.FalseNegatives(name='my_fn', thresholds=[0.4, 0.9]) + self.assertEqual(fn_obj.name, 'my_fn') + self.assertEqual(len(fn_obj.variables), 1) + self.assertEqual(fn_obj.thresholds, [0.4, 0.9]) - def result(self): - pass + def test_unweighted(self): + fn_obj = metrics.FalseNegatives() + self.evaluate(variables.variables_initializer(fn_obj.variables)) - invalid_update_obj = InvalidUpdate() + y_true = constant_op.constant(((0, 1, 0, 1, 0), (0, 0, 1, 1, 1), + (1, 1, 1, 1, 0), (0, 0, 0, 0, 1))) + y_pred = constant_op.constant(((0, 0, 1, 1, 0), (1, 1, 1, 1, 1), + (0, 1, 0, 1, 0), (1, 1, 1, 1, 1))) + + update_op = fn_obj.update_state(y_true, y_pred) + self.evaluate(update_op) + result = fn_obj.result() + self.assertAllClose(3., result) + + def test_weighted(self): + fn_obj = metrics.FalseNegatives() + self.evaluate(variables.variables_initializer(fn_obj.variables)) + y_true = constant_op.constant(((0, 1, 0, 1, 0), (0, 0, 1, 1, 1), + (1, 1, 1, 1, 0), (0, 0, 0, 0, 1))) + y_pred = constant_op.constant(((0, 0, 1, 1, 0), (1, 1, 1, 1, 1), + (0, 1, 0, 1, 0), (1, 1, 1, 1, 1))) + sample_weight = constant_op.constant((1., 1.5, 2., 2.5)) + result = fn_obj(y_true, y_pred, sample_weight=sample_weight) + self.assertAllClose(5., self.evaluate(result)) + + def test_unweighted_with_thresholds(self): + fn_obj = metrics.FalseNegatives(thresholds=[0.15, 0.5, 0.85]) + self.evaluate(variables.variables_initializer(fn_obj.variables)) + + y_pred = constant_op.constant(((0.9, 0.2, 0.8, 0.1), (0.2, 0.9, 0.7, 0.6), + (0.1, 0.2, 0.4, 0.3), (0, 1, 0.7, 0.3))) + y_true = constant_op.constant(((0, 1, 1, 0), (1, 0, 0, 0), (0, 0, 0, 0), + (1, 1, 1, 1))) + + update_op = fn_obj.update_state(y_true, y_pred) + self.evaluate(update_op) + result = fn_obj.result() + self.assertAllClose([1., 4., 6.], result) + + def test_weighted_with_thresholds(self): + fn_obj = metrics.FalseNegatives(thresholds=[0.15, 0.5, 0.85]) + self.evaluate(variables.variables_initializer(fn_obj.variables)) + + y_pred = constant_op.constant(((0.9, 0.2, 0.8, 0.1), (0.2, 0.9, 0.7, 0.6), + (0.1, 0.2, 0.4, 0.3), (0, 1, 0.7, 0.3))) + y_true = constant_op.constant(((0, 1, 1, 0), (1, 0, 0, 0), (0, 0, 0, 0), + (1, 1, 1, 1))) + sample_weight = ((3.0,), (5.0,), (7.0,), (4.0,)) + + result = fn_obj(y_true, y_pred, sample_weight=sample_weight) + self.assertAllClose([4., 16., 23.], self.evaluate(result)) + + def test_reset_states(self): + fn_obj = metrics.FalseNegatives() + model = _get_simple_sequential_model([fn_obj]) + x = np.zeros((100, 4)) + y = np.ones((100, 1)) + model.evaluate(x, y) + self.assertEqual(self.evaluate(fn_obj.accumulator), 100.) + model.evaluate(x, y) + self.assertEqual(self.evaluate(fn_obj.accumulator), 100.) + + +@test_util.run_all_in_graph_and_eager_modes +class TrueNegativesTest(test.TestCase): + + def test_config(self): + tn_obj = metrics.TrueNegatives(name='my_tn', thresholds=[0.4, 0.9]) + self.assertEqual(tn_obj.name, 'my_tn') + self.assertEqual(len(tn_obj.variables), 1) + self.assertEqual(tn_obj.thresholds, [0.4, 0.9]) + + def test_unweighted(self): + tn_obj = metrics.TrueNegatives() + self.evaluate(variables.variables_initializer(tn_obj.variables)) + + y_true = constant_op.constant(((0, 1, 0, 1, 0), (0, 0, 1, 1, 1), + (1, 1, 1, 1, 0), (0, 0, 0, 0, 1))) + y_pred = constant_op.constant(((0, 0, 1, 1, 0), (1, 1, 1, 1, 1), + (0, 1, 0, 1, 0), (1, 1, 1, 1, 1))) + + update_op = tn_obj.update_state(y_true, y_pred) + self.evaluate(update_op) + result = tn_obj.result() + self.assertAllClose(3., result) + + def test_weighted(self): + tn_obj = metrics.TrueNegatives() + self.evaluate(variables.variables_initializer(tn_obj.variables)) + y_true = constant_op.constant(((0, 1, 0, 1, 0), (0, 0, 1, 1, 1), + (1, 1, 1, 1, 0), (0, 0, 0, 0, 1))) + y_pred = constant_op.constant(((0, 0, 1, 1, 0), (1, 1, 1, 1, 1), + (0, 1, 0, 1, 0), (1, 1, 1, 1, 1))) + sample_weight = constant_op.constant((1., 1.5, 2., 2.5)) + result = tn_obj(y_true, y_pred, sample_weight=sample_weight) + self.assertAllClose(4., self.evaluate(result)) + + def test_unweighted_with_thresholds(self): + tn_obj = metrics.TrueNegatives(thresholds=[0.15, 0.5, 0.85]) + self.evaluate(variables.variables_initializer(tn_obj.variables)) + + y_pred = constant_op.constant(((0.9, 0.2, 0.8, 0.1), (0.2, 0.9, 0.7, 0.6), + (0.1, 0.2, 0.4, 0.3), (0, 1, 0.7, 0.3))) + y_true = constant_op.constant(((0, 1, 1, 0), (1, 0, 0, 0), (0, 0, 0, 0), + (1, 1, 1, 1))) + + update_op = tn_obj.update_state(y_true, y_pred) + self.evaluate(update_op) + result = tn_obj.result() + self.assertAllClose([2., 5., 7.], result) + + def test_weighted_with_thresholds(self): + tn_obj = metrics.TrueNegatives(thresholds=[0.15, 0.5, 0.85]) + self.evaluate(variables.variables_initializer(tn_obj.variables)) + + y_pred = constant_op.constant(((0.9, 0.2, 0.8, 0.1), (0.2, 0.9, 0.7, 0.6), + (0.1, 0.2, 0.4, 0.3), (0, 1, 0.7, 0.3))) + y_true = constant_op.constant(((0, 1, 1, 0), (1, 0, 0, 0), (0, 0, 0, 0), + (1, 1, 1, 1))) + sample_weight = ((0.0, 2.0, 3.0, 5.0),) + + result = tn_obj(y_true, y_pred, sample_weight=sample_weight) + self.assertAllClose([5., 15., 23.], self.evaluate(result)) + + def test_reset_states(self): + tn_obj = metrics.TrueNegatives() + model = _get_simple_sequential_model([tn_obj]) + x = np.zeros((100, 4)) + y = np.zeros((100, 1)) + model.evaluate(x, y) + self.assertEqual(self.evaluate(tn_obj.accumulator), 100.) + model.evaluate(x, y) + self.assertEqual(self.evaluate(tn_obj.accumulator), 100.) + + +@test_util.run_all_in_graph_and_eager_modes +class TruePositivesTest(test.TestCase): + + def test_config(self): + tp_obj = metrics.TruePositives(name='my_tp', thresholds=[0.4, 0.9]) + self.assertEqual(tp_obj.name, 'my_tp') + self.assertEqual(len(tp_obj.variables), 1) + self.assertEqual(tp_obj.thresholds, [0.4, 0.9]) + + def test_unweighted(self): + tp_obj = metrics.TruePositives() + self.evaluate(variables.variables_initializer(tp_obj.variables)) + + y_true = constant_op.constant(((0, 1, 0, 1, 0), (0, 0, 1, 1, 1), + (1, 1, 1, 1, 0), (0, 0, 0, 0, 1))) + y_pred = constant_op.constant(((0, 0, 1, 1, 0), (1, 1, 1, 1, 1), + (0, 1, 0, 1, 0), (1, 1, 1, 1, 1))) + + update_op = tp_obj.update_state(y_true, y_pred) + self.evaluate(update_op) + result = tp_obj.result() + self.assertAllClose(7., result) + + def test_weighted(self): + tp_obj = metrics.TruePositives() + self.evaluate(variables.variables_initializer(tp_obj.variables)) + y_true = constant_op.constant(((0, 1, 0, 1, 0), (0, 0, 1, 1, 1), + (1, 1, 1, 1, 0), (0, 0, 0, 0, 1))) + y_pred = constant_op.constant(((0, 0, 1, 1, 0), (1, 1, 1, 1, 1), + (0, 1, 0, 1, 0), (1, 1, 1, 1, 1))) + sample_weight = constant_op.constant((1., 1.5, 2., 2.5)) + result = tp_obj(y_true, y_pred, sample_weight=sample_weight) + self.assertAllClose(12., self.evaluate(result)) + + def test_unweighted_with_thresholds(self): + tp_obj = metrics.TruePositives(thresholds=[0.15, 0.5, 0.85]) + self.evaluate(variables.variables_initializer(tp_obj.variables)) + + y_pred = constant_op.constant(((0.9, 0.2, 0.8, 0.1), (0.2, 0.9, 0.7, 0.6), + (0.1, 0.2, 0.4, 0.3), (0, 1, 0.7, 0.3))) + y_true = constant_op.constant(((0, 1, 1, 0), (1, 0, 0, 0), (0, 0, 0, 0), + (1, 1, 1, 1))) + + update_op = tp_obj.update_state(y_true, y_pred) + self.evaluate(update_op) + result = tp_obj.result() + self.assertAllClose([6., 3., 1.], result) + + def test_weighted_with_thresholds(self): + tp_obj = metrics.TruePositives(thresholds=[0.15, 0.5, 0.85]) + self.evaluate(variables.variables_initializer(tp_obj.variables)) + + y_pred = constant_op.constant(((0.9, 0.2, 0.8, 0.1), (0.2, 0.9, 0.7, 0.6), + (0.1, 0.2, 0.4, 0.3), (0, 1, 0.7, 0.3))) + y_true = constant_op.constant(((0, 1, 1, 0), (1, 0, 0, 0), (0, 0, 0, 0), + (1, 1, 1, 1))) + + result = tp_obj(y_true, y_pred, sample_weight=37.) + self.assertAllClose([222., 111., 37.], self.evaluate(result)) + + def test_reset_states(self): + tp_obj = metrics.TruePositives() + model = _get_simple_sequential_model([tp_obj]) + x = np.ones((100, 4)) + y = np.ones((100, 1)) + model.evaluate(x, y) + self.assertEqual(self.evaluate(tp_obj.accumulator), 100.) + model.evaluate(x, y) + self.assertEqual(self.evaluate(tp_obj.accumulator), 100.) + + +@test_util.run_all_in_graph_and_eager_modes +class PrecisionTest(test.TestCase): + + def test_config(self): + p_obj = metrics.Precision(name='my_precision', thresholds=[0.4, 0.9]) + self.assertEqual(p_obj.name, 'my_precision') + self.assertLen(p_obj.variables, 2) + self.assertEqual([v.name for v in p_obj.variables], + ['true_positives:0', 'false_positives:0']) + self.assertEqual(p_obj.thresholds, [0.4, 0.9]) + + def test_value_is_idempotent(self): + p_obj = metrics.Precision(thresholds=[0.3, 0.72]) + y_pred = random_ops.random_uniform(shape=(10, 3)) + y_true = random_ops.random_uniform(shape=(10, 3)) + update_op = p_obj.update_state(y_true, y_pred) + self.evaluate(variables.variables_initializer(p_obj.variables)) + + # Run several updates. + for _ in range(10): + self.evaluate(update_op) + + # Then verify idempotency. + initial_precision = self.evaluate(p_obj.result()) + for _ in range(10): + self.assertArrayNear(initial_precision, self.evaluate(p_obj.result()), + 1e-3) + + def test_unweighted(self): + p_obj = metrics.Precision() + y_pred = constant_op.constant([1, 0, 1, 0], shape=(1, 4)) + y_true = constant_op.constant([0, 1, 1, 0], shape=(1, 4)) + self.evaluate(variables.variables_initializer(p_obj.variables)) + result = p_obj(y_true, y_pred) + self.assertAlmostEqual(0.5, self.evaluate(result)) + + def test_unweighted_all_incorrect(self): + p_obj = metrics.Precision(thresholds=[0.5]) + inputs = np.random.randint(0, 2, size=(100, 1)) + y_pred = constant_op.constant(inputs) + y_true = constant_op.constant(1 - inputs) + self.evaluate(variables.variables_initializer(p_obj.variables)) + result = p_obj(y_true, y_pred) + self.assertAlmostEqual(0, self.evaluate(result)) + + def test_weighted(self): + p_obj = metrics.Precision() + y_pred = constant_op.constant([[1, 0, 1, 0], [1, 0, 1, 0]]) + y_true = constant_op.constant([[0, 1, 1, 0], [1, 0, 0, 1]]) + self.evaluate(variables.variables_initializer(p_obj.variables)) + result = p_obj( + y_true, + y_pred, + sample_weight=constant_op.constant([[1, 2, 3, 4], [4, 3, 2, 1]])) + weighted_tp = 3.0 + 4.0 + weighted_positives = (1.0 + 3.0) + (4.0 + 2.0) + expected_precision = weighted_tp / weighted_positives + self.assertAlmostEqual(expected_precision, self.evaluate(result)) + + def test_div_by_zero(self): + p_obj = metrics.Precision() + y_pred = constant_op.constant([0, 0, 0, 0]) + y_true = constant_op.constant([0, 0, 0, 0]) + self.evaluate(variables.variables_initializer(p_obj.variables)) + result = p_obj(y_true, y_pred) + self.assertEqual(0, self.evaluate(result)) + + def test_unweighted_with_threshold(self): + p_obj = metrics.Precision(thresholds=[0.5, 0.7]) + y_pred = constant_op.constant([1, 0, 0.6, 0], shape=(1, 4)) + y_true = constant_op.constant([0, 1, 1, 0], shape=(1, 4)) + self.evaluate(variables.variables_initializer(p_obj.variables)) + result = p_obj(y_true, y_pred) + self.assertArrayNear([0.5, 0.], self.evaluate(result), 0) + + def test_weighted_with_threshold(self): + p_obj = metrics.Precision(thresholds=[0.5, 1.]) + y_true = constant_op.constant([[0, 1], [1, 0]], shape=(2, 2)) + y_pred = constant_op.constant([[1, 0], [0.6, 0]], + shape=(2, 2), + dtype=dtypes.float32) + weights = constant_op.constant([[4, 0], [3, 1]], + shape=(2, 2), + dtype=dtypes.float32) + self.evaluate(variables.variables_initializer(p_obj.variables)) + result = p_obj(y_true, y_pred, sample_weight=weights) + weighted_tp = 0 + 3. + weighted_positives = (0 + 3.) + (4. + 0.) + expected_precision = weighted_tp / weighted_positives + self.assertArrayNear([expected_precision, 0], self.evaluate(result), 1e-3) + + def test_multiple_updates(self): + p_obj = metrics.Precision(thresholds=[0.5, 1.]) + y_true = constant_op.constant([[0, 1], [1, 0]], shape=(2, 2)) + y_pred = constant_op.constant([[1, 0], [0.6, 0]], + shape=(2, 2), + dtype=dtypes.float32) + weights = constant_op.constant([[4, 0], [3, 1]], + shape=(2, 2), + dtype=dtypes.float32) + self.evaluate(variables.variables_initializer(p_obj.variables)) + update_op = p_obj.update_state(y_true, y_pred, sample_weight=weights) + for _ in range(2): + self.evaluate(update_op) + + weighted_tp = (0 + 3.) + (0 + 3.) + weighted_positives = ((0 + 3.) + (4. + 0.)) + ((0 + 3.) + (4. + 0.)) + expected_precision = weighted_tp / weighted_positives + self.assertArrayNear([expected_precision, 0], self.evaluate(p_obj.result()), + 1e-3) + + def test_reset_states(self): + p_obj = metrics.Precision() + model = _get_simple_sequential_model([p_obj]) + x = np.concatenate((np.ones((50, 4)), np.ones((50, 4)))) + y = np.concatenate((np.ones((50, 1)), np.zeros((50, 1)))) + model.evaluate(x, y) + self.assertEqual(self.evaluate(p_obj.tp), 50.) + self.assertEqual(self.evaluate(p_obj.fp), 50.) + model.evaluate(x, y) + self.assertEqual(self.evaluate(p_obj.tp), 50.) + self.assertEqual(self.evaluate(p_obj.fp), 50.) + + +@test_util.run_all_in_graph_and_eager_modes +class RecallTest(test.TestCase): + + def test_config(self): + r_obj = metrics.Recall(name='my_recall', thresholds=[0.4, 0.9]) + self.assertEqual(r_obj.name, 'my_recall') + self.assertLen(r_obj.variables, 2) + self.assertEqual([v.name for v in r_obj.variables], + ['true_positives:0', 'false_negatives:0']) + self.assertEqual(r_obj.thresholds, [0.4, 0.9]) + + def test_value_is_idempotent(self): + r_obj = metrics.Recall(thresholds=[0.3, 0.72]) + y_pred = random_ops.random_uniform(shape=(10, 3)) + y_true = random_ops.random_uniform(shape=(10, 3)) + update_op = r_obj.update_state(y_true, y_pred) + self.evaluate(variables.variables_initializer(r_obj.variables)) + + # Run several updates. + for _ in range(10): + self.evaluate(update_op) + + # Then verify idempotency. + initial_recall = self.evaluate(r_obj.result()) + for _ in range(10): + self.assertArrayNear(initial_recall, self.evaluate(r_obj.result()), 1e-3) + + def test_unweighted(self): + r_obj = metrics.Recall() + y_pred = constant_op.constant([1, 0, 1, 0], shape=(1, 4)) + y_true = constant_op.constant([0, 1, 1, 0], shape=(1, 4)) + self.evaluate(variables.variables_initializer(r_obj.variables)) + result = r_obj(y_true, y_pred) + self.assertAlmostEqual(0.5, self.evaluate(result)) + + def test_unweighted_all_incorrect(self): + r_obj = metrics.Recall(thresholds=[0.5]) + inputs = np.random.randint(0, 2, size=(100, 1)) + y_pred = constant_op.constant(inputs) + y_true = constant_op.constant(1 - inputs) + self.evaluate(variables.variables_initializer(r_obj.variables)) + result = r_obj(y_true, y_pred) + self.assertAlmostEqual(0, self.evaluate(result)) + + def test_weighted(self): + r_obj = metrics.Recall() + y_pred = constant_op.constant([[1, 0, 1, 0], [0, 1, 0, 1]]) + y_true = constant_op.constant([[0, 1, 1, 0], [1, 0, 0, 1]]) + self.evaluate(variables.variables_initializer(r_obj.variables)) + result = r_obj( + y_true, + y_pred, + sample_weight=constant_op.constant([[1, 2, 3, 4], [4, 3, 2, 1]])) + weighted_tp = 3.0 + 1.0 + weighted_t = (2.0 + 3.0) + (4.0 + 1.0) + expected_recall = weighted_tp / weighted_t + self.assertAlmostEqual(expected_recall, self.evaluate(result)) + + def test_div_by_zero(self): + r_obj = metrics.Recall() + y_pred = constant_op.constant([0, 0, 0, 0]) + y_true = constant_op.constant([0, 0, 0, 0]) + self.evaluate(variables.variables_initializer(r_obj.variables)) + result = r_obj(y_true, y_pred) + self.assertEqual(0, self.evaluate(result)) + + def test_unweighted_with_threshold(self): + r_obj = metrics.Recall(thresholds=[0.5, 0.7]) + y_pred = constant_op.constant([1, 0, 0.6, 0], shape=(1, 4)) + y_true = constant_op.constant([0, 1, 1, 0], shape=(1, 4)) + self.evaluate(variables.variables_initializer(r_obj.variables)) + result = r_obj(y_true, y_pred) + self.assertArrayNear([0.5, 0.], self.evaluate(result), 0) + + def test_weighted_with_threshold(self): + r_obj = metrics.Recall(thresholds=[0.5, 1.]) + y_true = constant_op.constant([[0, 1], [1, 0]], shape=(2, 2)) + y_pred = constant_op.constant([[1, 0], [0.6, 0]], + shape=(2, 2), + dtype=dtypes.float32) + weights = constant_op.constant([[1, 4], [3, 2]], + shape=(2, 2), + dtype=dtypes.float32) + self.evaluate(variables.variables_initializer(r_obj.variables)) + result = r_obj(y_true, y_pred, sample_weight=weights) + weighted_tp = 0 + 3. + weighted_positives = (0 + 3.) + (4. + 0.) + expected_recall = weighted_tp / weighted_positives + self.assertArrayNear([expected_recall, 0], self.evaluate(result), 1e-3) + + def test_multiple_updates(self): + r_obj = metrics.Recall(thresholds=[0.5, 1.]) + y_true = constant_op.constant([[0, 1], [1, 0]], shape=(2, 2)) + y_pred = constant_op.constant([[1, 0], [0.6, 0]], + shape=(2, 2), + dtype=dtypes.float32) + weights = constant_op.constant([[1, 4], [3, 2]], + shape=(2, 2), + dtype=dtypes.float32) + self.evaluate(variables.variables_initializer(r_obj.variables)) + update_op = r_obj.update_state(y_true, y_pred, sample_weight=weights) + for _ in range(2): + self.evaluate(update_op) + + weighted_tp = (0 + 3.) + (0 + 3.) + weighted_positives = ((0 + 3.) + (4. + 0.)) + ((0 + 3.) + (4. + 0.)) + expected_recall = weighted_tp / weighted_positives + self.assertArrayNear([expected_recall, 0], self.evaluate(r_obj.result()), + 1e-3) + + def test_reset_states(self): + r_obj = metrics.Recall() + model = _get_simple_sequential_model([r_obj]) + x = np.concatenate((np.ones((50, 4)), np.zeros((50, 4)))) + y = np.concatenate((np.ones((50, 1)), np.ones((50, 1)))) + model.evaluate(x, y) + self.assertEqual(self.evaluate(r_obj.tp), 50.) + self.assertEqual(self.evaluate(r_obj.fn), 50.) + model.evaluate(x, y) + self.assertEqual(self.evaluate(r_obj.tp), 50.) + self.assertEqual(self.evaluate(r_obj.fn), 50.) + + +@test_util.run_all_in_graph_and_eager_modes +class SensitivityAtSpecificityTest(test.TestCase, parameterized.TestCase): + + def test_config(self): + s_obj = metrics.SensitivityAtSpecificity( + 0.4, num_thresholds=100, name='sensitivity_at_specificity_1') + self.assertEqual(s_obj.name, 'sensitivity_at_specificity_1') + self.assertLen(s_obj.variables, 4) + self.assertEqual(s_obj.value, 0.4) + self.assertLen(s_obj.thresholds, 100) + + def test_value_is_idempotent(self): + s_obj = metrics.SensitivityAtSpecificity(0.7) + y_pred = random_ops.random_uniform((10, 3), + maxval=1, + dtype=dtypes.float32, + seed=1) + y_true = random_ops.random_uniform((10, 3), + maxval=2, + dtype=dtypes.int64, + seed=1) + update_op = s_obj.update_state(y_true, y_pred) + self.evaluate(variables.variables_initializer(s_obj.variables)) + + # Run several updates. + for _ in range(10): + self.evaluate(update_op) + + # Then verify idempotency. + initial_sensitivity = self.evaluate(s_obj.result()) + for _ in range(10): + self.assertAlmostEqual(initial_sensitivity, self.evaluate(s_obj.result()), + 1e-3) + + def test_unweighted_all_correct(self): + s_obj = metrics.SensitivityAtSpecificity(0.7) + inputs = np.random.randint(0, 2, size=(100, 1)) + y_pred = constant_op.constant(inputs, dtype=dtypes.float32) + y_true = constant_op.constant(inputs) + self.evaluate(variables.variables_initializer(s_obj.variables)) + result = s_obj(y_true, y_pred) + self.assertAlmostEqual(1, self.evaluate(result)) + + def test_unweighted_high_specificity(self): + s_obj = metrics.SensitivityAtSpecificity(0.8) + pred_values = [0.0, 0.1, 0.2, 0.3, 0.4, 0.1, 0.45, 0.5, 0.8, 0.9] + label_values = [0, 0, 0, 0, 0, 1, 1, 1, 1, 1] + + y_pred = constant_op.constant(pred_values, dtype=dtypes.float32) + y_true = constant_op.constant(label_values) + self.evaluate(variables.variables_initializer(s_obj.variables)) + result = s_obj(y_true, y_pred) + self.assertAlmostEqual(0.8, self.evaluate(result)) + + def test_unweighted_low_specificity(self): + s_obj = metrics.SensitivityAtSpecificity(0.4) + pred_values = [0.0, 0.1, 0.2, 0.3, 0.4, 0.01, 0.02, 0.25, 0.26, 0.26] + label_values = [0, 0, 0, 0, 0, 1, 1, 1, 1, 1] + + y_pred = constant_op.constant(pred_values, dtype=dtypes.float32) + y_true = constant_op.constant(label_values) + self.evaluate(variables.variables_initializer(s_obj.variables)) + result = s_obj(y_true, y_pred) + self.assertAlmostEqual(0.6, self.evaluate(result)) + + @parameterized.parameters([dtypes.bool, dtypes.int32, dtypes.float32]) + def test_weighted(self, label_dtype): + s_obj = metrics.SensitivityAtSpecificity(0.4) + pred_values = [0.0, 0.1, 0.2, 0.3, 0.4, 0.01, 0.02, 0.25, 0.26, 0.26] + label_values = [0, 0, 0, 0, 0, 1, 1, 1, 1, 1] + weight_values = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] + + y_pred = constant_op.constant(pred_values, dtype=dtypes.float32) + y_true = math_ops.cast(label_values, dtype=label_dtype) + weights = constant_op.constant(weight_values) + self.evaluate(variables.variables_initializer(s_obj.variables)) + result = s_obj(y_true, y_pred, sample_weight=weights) + self.assertAlmostEqual(0.675, self.evaluate(result)) + + def test_invalid_specificity(self): with self.assertRaisesRegexp( - TypeError, - 'Metric invalid-update\'s update must be a Tensor or Operation, given:' - ): - invalid_update_obj.update_state() + ValueError, r'`specificity` must be in the range \[0, 1\].'): + metrics.SensitivityAtSpecificity(-1) + def test_invalid_num_thresholds(self): + with self.assertRaisesRegexp(ValueError, '`num_thresholds` must be > 0.'): + metrics.SensitivityAtSpecificity(0.4, num_thresholds=-1) + + def test_reset_states(self): + s_obj = metrics.SensitivityAtSpecificity(0.5, num_thresholds=1) + model = _get_simple_sequential_model([s_obj]) + x = np.concatenate((np.ones((25, 4)), np.zeros((25, 4)), np.zeros((25, 4)), + np.ones((25, 4)))) + y = np.concatenate((np.ones((25, 1)), np.zeros((25, 1)), np.ones((25, 1)), + np.zeros((25, 1)))) + model.evaluate(x, y) + self.assertEqual(self.evaluate(s_obj.tp), 25.) + self.assertEqual(self.evaluate(s_obj.fp), 25.) + self.assertEqual(self.evaluate(s_obj.fn), 25.) + self.assertEqual(self.evaluate(s_obj.tn), 25.) + model.evaluate(x, y) + self.assertEqual(self.evaluate(s_obj.tp), 25.) + self.assertEqual(self.evaluate(s_obj.fp), 25.) + self.assertEqual(self.evaluate(s_obj.fn), 25.) + self.assertEqual(self.evaluate(s_obj.tn), 25.) + + +@test_util.run_all_in_graph_and_eager_modes +class SpecificityAtSensitivityTest(test.TestCase, parameterized.TestCase): + + def test_config(self): + s_obj = metrics.SpecificityAtSensitivity( + 0.4, num_thresholds=100, name='specificity_at_sensitivity_1') + self.assertEqual(s_obj.name, 'specificity_at_sensitivity_1') + self.assertLen(s_obj.variables, 4) + self.assertEqual(s_obj.value, 0.4) + self.assertLen(s_obj.thresholds, 100) + + def test_value_is_idempotent(self): + s_obj = metrics.SpecificityAtSensitivity(0.7) + y_pred = random_ops.random_uniform((10, 3), + maxval=1, + dtype=dtypes.float32, + seed=1) + y_true = random_ops.random_uniform((10, 3), + maxval=2, + dtype=dtypes.int64, + seed=1) + update_op = s_obj.update_state(y_true, y_pred) + self.evaluate(variables.variables_initializer(s_obj.variables)) + + # Run several updates. + for _ in range(10): + self.evaluate(update_op) + + # Then verify idempotency. + initial_specificity = self.evaluate(s_obj.result()) + for _ in range(10): + self.assertAlmostEqual(initial_specificity, self.evaluate(s_obj.result()), + 1e-3) + + def test_unweighted_all_correct(self): + s_obj = metrics.SpecificityAtSensitivity(0.7) + inputs = np.random.randint(0, 2, size=(100, 1)) + y_pred = constant_op.constant(inputs, dtype=dtypes.float32) + y_true = constant_op.constant(inputs) + self.evaluate(variables.variables_initializer(s_obj.variables)) + result = s_obj(y_true, y_pred) + self.assertAlmostEqual(1, self.evaluate(result)) + + def test_unweighted_high_sensitivity(self): + s_obj = metrics.SpecificityAtSensitivity(0.8) + pred_values = [0.0, 0.1, 0.2, 0.3, 0.4, 0.1, 0.45, 0.5, 0.8, 0.9] + label_values = [0, 0, 0, 0, 0, 1, 1, 1, 1, 1] + + y_pred = constant_op.constant(pred_values, dtype=dtypes.float32) + y_true = constant_op.constant(label_values) + self.evaluate(variables.variables_initializer(s_obj.variables)) + result = s_obj(y_true, y_pred) + self.assertAlmostEqual(0.4, self.evaluate(result)) + + def test_unweighted_low_sensitivity(self): + s_obj = metrics.SpecificityAtSensitivity(0.4) + pred_values = [0.0, 0.1, 0.2, 0.3, 0.4, 0.01, 0.02, 0.25, 0.26, 0.26] + label_values = [0, 0, 0, 0, 0, 1, 1, 1, 1, 1] + + y_pred = constant_op.constant(pred_values, dtype=dtypes.float32) + y_true = constant_op.constant(label_values) + self.evaluate(variables.variables_initializer(s_obj.variables)) + result = s_obj(y_true, y_pred) + self.assertAlmostEqual(0.6, self.evaluate(result)) + + @parameterized.parameters([dtypes.bool, dtypes.int32, dtypes.float32]) + def test_weighted(self, label_dtype): + s_obj = metrics.SpecificityAtSensitivity(0.4) + pred_values = [0.0, 0.1, 0.2, 0.3, 0.4, 0.01, 0.02, 0.25, 0.26, 0.26] + label_values = [0, 0, 0, 0, 0, 1, 1, 1, 1, 1] + weight_values = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] + + y_pred = constant_op.constant(pred_values, dtype=dtypes.float32) + y_true = math_ops.cast(label_values, dtype=label_dtype) + weights = constant_op.constant(weight_values) + self.evaluate(variables.variables_initializer(s_obj.variables)) + result = s_obj(y_true, y_pred, sample_weight=weights) + self.assertAlmostEqual(0.4, self.evaluate(result)) + + def test_invalid_sensitivity(self): + with self.assertRaisesRegexp( + ValueError, r'`sensitivity` must be in the range \[0, 1\].'): + metrics.SpecificityAtSensitivity(-1) + + def test_invalid_num_thresholds(self): + with self.assertRaisesRegexp(ValueError, '`num_thresholds` must be > 0.'): + metrics.SpecificityAtSensitivity(0.4, num_thresholds=-1) + + def test_reset_states(self): + s_obj = metrics.SpecificityAtSensitivity(0.5, num_thresholds=1) + model = _get_simple_sequential_model([s_obj]) + x = np.concatenate((np.ones((25, 4)), np.zeros((25, 4)), np.zeros((25, 4)), + np.ones((25, 4)))) + y = np.concatenate((np.ones((25, 1)), np.zeros((25, 1)), np.ones((25, 1)), + np.zeros((25, 1)))) + model.evaluate(x, y) + self.assertEqual(self.evaluate(s_obj.tp), 25.) + self.assertEqual(self.evaluate(s_obj.fp), 25.) + self.assertEqual(self.evaluate(s_obj.fn), 25.) + self.assertEqual(self.evaluate(s_obj.tn), 25.) + model.evaluate(x, y) + self.assertEqual(self.evaluate(s_obj.tp), 25.) + self.assertEqual(self.evaluate(s_obj.fp), 25.) + self.assertEqual(self.evaluate(s_obj.fn), 25.) + self.assertEqual(self.evaluate(s_obj.tn), 25.) + + +@test_util.run_all_in_graph_and_eager_modes +class CosineProximityTest(test.TestCase): + + def test_config(self): + cosine_obj = metrics.CosineProximity(name='my_cos', dtype=dtypes.int32) + self.assertEqual(cosine_obj.name, 'my_cos') + self.assertEqual(cosine_obj._dtype, dtypes.int32) + + def test_unweighted(self): + cosine_obj = metrics.CosineProximity() + self.evaluate(variables.variables_initializer(cosine_obj.variables)) + + y_true = constant_op.constant(((0, 1, 0, 1, 0), (0, 0, 1, 1, 1), + (1, 1, 1, 1, 0), (0, 0, 0, 0, 1))) + y_pred = constant_op.constant(((0, 0, 1, 1, 0), (1, 1, 1, 1, 1), + (0, 1, 0, 1, 0), (1, 1, 1, 1, 1))) + + update_op = cosine_obj.update_state(y_true, y_pred) + self.evaluate(update_op) + result = cosine_obj.result() + self.assertAllClose(-0.60723, result, atol=1e-5) + + def test_weighted(self): + cosine_obj = metrics.CosineProximity() + self.evaluate(variables.variables_initializer(cosine_obj.variables)) + y_true = constant_op.constant(((0, 1, 0, 1, 0), (0, 0, 1, 1, 1), + (1, 1, 1, 1, 0), (0, 0, 0, 0, 1))) + y_pred = constant_op.constant(((0, 0, 1, 1, 0), (1, 1, 1, 1, 1), + (0, 1, 0, 1, 0), (1, 1, 1, 1, 1))) + sample_weight = constant_op.constant((1., 1.5, 2., 2.5)) + result = cosine_obj(y_true, y_pred, sample_weight=sample_weight) + self.assertAllClose(-0.59916, self.evaluate(result), atol=1e-5) if __name__ == '__main__': test.main() diff --git a/tensorflow/python/keras/model_subclassing_test.py b/tensorflow/python/keras/model_subclassing_test.py index aca058b1111..620275e50f8 100644 --- a/tensorflow/python/keras/model_subclassing_test.py +++ b/tensorflow/python/keras/model_subclassing_test.py @@ -25,6 +25,7 @@ import numpy as np from tensorflow.python import keras from tensorflow.python.data.ops import dataset_ops from tensorflow.python.eager import context +from tensorflow.python.framework import ops from tensorflow.python.framework import tensor_shape from tensorflow.python.framework import test_util from tensorflow.python.ops import array_ops @@ -455,12 +456,12 @@ class ModelSubclassingTest(test.TestCase): model = SimpleTestModel(num_classes=num_classes, use_dp=True, use_bn=True) model.compile(loss='mse', optimizer=RMSPropOptimizer(learning_rate=0.001)) - x = np.ones((num_samples, input_dim)) - y = np.zeros((num_samples, num_classes)) + x = np.ones((num_samples, input_dim), dtype=np.float32) + y = np.zeros((num_samples, num_classes), dtype=np.float32) dataset = dataset_ops.Dataset.from_tensor_slices((x, y)) dataset = dataset.repeat(100) dataset = dataset.batch(10) - iterator = dataset.make_one_shot_iterator() + iterator = dataset_ops.make_one_shot_iterator(dataset) model.fit(iterator, epochs=2, steps_per_epoch=10, verbose=0) _ = model.evaluate(iterator, steps=10, verbose=0) @@ -725,10 +726,41 @@ class ModelSubclassingTest(test.TestCase): _ = model.evaluate(x, y, verbose=0) self.assertEqual(len(model.weights), 16) - self.assertEqual( - len(model.non_trainable_weights), 4) + self.assertEqual(len(model.non_trainable_weights), 4) self.assertEqual(len(model.trainable_weights), 12) + def test_subclass_nested_in_sequential(self): + num_classes = 2 + num_samples = 100 + input_dim = 50 + + class Inner(keras.Model): + + def __init__(self): + super(Inner, self).__init__() + self.dense1 = keras.layers.Dense(32, activation='relu') + self.dense2 = keras.layers.Dense(num_classes, activation='relu') + self.bn = keras.layers.BatchNormalization() + + def call(self, inputs): + x = self.dense1(inputs) + x = self.dense2(x) + return self.bn(x) + + model = keras.Sequential([Inner()]) + model.compile(loss='mse', + optimizer=RMSPropOptimizer(learning_rate=0.001), + metrics=['acc']) + + x = np.ones((num_samples, input_dim)) + y = np.zeros((num_samples, num_classes)) + model.fit(x, y, epochs=2, batch_size=32, verbose=0) + _ = model.evaluate(x, y, verbose=0) + + self.assertEqual(len(model.weights), 8) + self.assertEqual(len(model.non_trainable_weights), 2) + self.assertEqual(len(model.trainable_weights), 6) + def test_support_for_manual_training_arg(self): # In most cases, the `training` argument is left unspecified, in which # case it defaults to value corresponding to the Model method being used @@ -819,9 +851,73 @@ class ModelSubclassingTest(test.TestCase): self.assertEqual([m.dense.kernel, m.dense.bias, m.not_trainable_var], m.non_trainable_variables) + @test_util.run_in_graph_and_eager_modes + def test_add_weight_in_model(self): + + class MyModel(keras.Model): + + def __init__(self): + super(MyModel, self).__init__() + self.b = self.add_weight('bias', (10,)) + self.c = self.add_weight('bias2', (10,), trainable=False) + + def call(self, inputs): + return inputs + self.b + self.c + + x = ops.convert_to_tensor(np.ones((10, 10), 'float32')) + model = MyModel() + model(x) + self.assertEqual(1, len(model.trainable_weights)) + self.assertEqual(1, len(model.non_trainable_weights)) + self.assertEqual(2, len(model.weights)) + + class MyModelCustomBuild(keras.Model): + + def build(self, input_shape): + self.b = self.add_weight('bias', (10,)) + self.c = self.add_weight('bias2', (10,), trainable=False) + + def call(self, inputs): + return inputs + self.b + self.c + + x = ops.convert_to_tensor(np.ones((10, 10), 'float32')) + model = MyModelCustomBuild() + model(x) + self.assertEqual(1, len(model.trainable_weights)) + self.assertEqual(1, len(model.non_trainable_weights)) + self.assertEqual(2, len(model.weights)) + + def test_add_update_in_model(self): + + class MyModel(keras.Model): + + def __init__(self): + super(MyModel, self).__init__() + self.b = self.add_weight('bias', (10,)) + self.c = self.add_weight('bias2', (10,)) + + def call(self, inputs): + # Unconditional + self.add_update(self.b.assign(self.b * 2)) + # Conditional + self.add_update(self.c.assign(inputs[1, :]), inputs) + return inputs + self.b + self.c + + x = ops.convert_to_tensor(np.ones((10, 10), 'float32')) + model = MyModel() + model(x) + + if context.executing_eagerly(): + self.assertEqual(0, len(model.updates)) + else: + self.assertEqual(2, len(model.updates)) + self.assertEqual(1, len(model.get_updates_for(None))) + self.assertEqual(1, len(model.get_updates_for(x))) + class GraphSpecificModelSubclassingTests(test.TestCase): + @test_util.run_deprecated_v1 def test_single_io_workflow_with_tensors(self): num_classes = 2 num_samples = 10 @@ -839,6 +935,7 @@ class GraphSpecificModelSubclassingTests(test.TestCase): model.fit(x, y, epochs=2, steps_per_epoch=10, verbose=0) _ = model.evaluate(steps=10, verbose=0) + @test_util.run_deprecated_v1 def test_multi_io_workflow_with_tensors(self): num_classes = (2, 3) num_samples = 10 @@ -858,6 +955,7 @@ class GraphSpecificModelSubclassingTests(test.TestCase): model.fit([x1, x2], [y1, y2], epochs=2, steps_per_epoch=10, verbose=0) _ = model.evaluate(steps=10, verbose=0) + @test_util.run_deprecated_v1 def test_updates_and_losses_for_nested_models_in_subclassed_model(self): # Case 1: deferred-build sequential nested in subclass. @@ -925,6 +1023,7 @@ class GraphSpecificModelSubclassingTests(test.TestCase): self.assertEqual(len(model.get_updates_for(x)), 2) self.assertEqual(len(model.get_losses_for(x)), 1) + @test_util.run_deprecated_v1 def test_multi_io_workflow_with_numpy_arrays_and_custom_placeholders(self): num_classes = (2, 3) num_samples = 1000 @@ -974,6 +1073,16 @@ class TrainingNoDefaultModel(keras.Model): return self.dense1(x) +class TrainingMaskingModel(keras.Model): + + def __init__(self): + super(TrainingMaskingModel, self).__init__() + self.dense1 = keras.layers.Dense(1) + + def call(self, x, training=False, mask=None): + return self.dense1(x) + + class CustomCallSignatureTests(test.TestCase): @test_util.run_in_graph_and_eager_modes @@ -1003,6 +1112,19 @@ class CustomCallSignatureTests(test.TestCase): 'has been properly built.')) self.assertTrue(model.built, 'Model should be built after calling `build`.') + @test_util.run_in_graph_and_eager_modes + def test_training_and_mask_args_call_build(self): + input_dim = 2 + + model = TrainingMaskingModel() + self.assertFalse(model.built, 'Model should not have been built') + self.assertFalse(model.weights, ('Model should have no weights since it ' + 'has not been built.')) + model.build((None, input_dim)) + self.assertTrue(model.weights, ('Model should have weights now that it ' + 'has been properly built.')) + self.assertTrue(model.built, 'Model should be built after calling `build`.') + @test_util.run_in_graph_and_eager_modes def test_custom_call_kwargs_and_build(self): first_input_shape = (2, 3) diff --git a/tensorflow/python/keras/models.py b/tensorflow/python/keras/models.py index 225c6c6af8e..2637191bb75 100644 --- a/tensorflow/python/keras/models.py +++ b/tensorflow/python/keras/models.py @@ -100,17 +100,19 @@ def _clone_functional_model(model, input_tensors=None): input_tensors = list(input_tensors) input_tensors = generic_utils.to_list(input_tensors) input_tensors_ = [] - for i, x in enumerate(input_tensors): - if not K.is_keras_tensor(x): - name = model._input_layers[i].name - input_tensor = Input(tensor=x, name='input_wrapper_for_' + name) + for i in range(len(input_tensors)): + input_tensor = input_tensors[i] + if not K.is_keras_tensor(input_tensor): + original_input_layer = model._input_layers[i] + name = original_input_layer.name + input_tensor = Input(tensor=input_tensor, + name='input_wrapper_for_' + name) input_tensors_.append(input_tensor) # Cache newly created input layer. - original_input_layer = x._keras_history[0] newly_created_input_layer = input_tensor._keras_history[0] layer_map[original_input_layer] = newly_created_input_layer else: - input_tensors_.append(x) + input_tensors_.append(input_tensor) input_tensors = input_tensors_ for x, y in zip(model.inputs, input_tensors): @@ -209,14 +211,17 @@ def _clone_sequential_model(model, input_tensors=None): # Use model._layers to ensure that all layers are cloned. The model's layers # property will exclude the initial InputLayer (if it exists) in the model, # resulting in a different Sequential model structure. - layers = [clone(layer) for layer in model._layers] if input_tensors is None: + layers = [clone(layer) for layer in model._layers] return Sequential(layers=layers, name=model.name) else: # If input tensors are provided, the original model's InputLayer is # overwritten with a different InputLayer. - if isinstance(layers[0], InputLayer): - layers = layers[1:] + layers = [ + clone(layer) + for layer in model._layers + if not isinstance(layer, InputLayer) + ] if len(generic_utils.to_list(input_tensors)) != 1: raise ValueError('To clone a `Sequential` model, we expect ' ' at most one tensor ' @@ -304,8 +309,9 @@ def _in_place_subclassed_model_reset(model): attributes_cache[name] = value assert value in model._layers elif isinstance( - value, (list, tuple)) and name not in ('layers', '_layers', - 'stateful_metric_functions'): + value, + (list, tuple)) and name not in ('layers', '_layers', 'metrics', + '_compile_stateful_metric_functions'): # Handle case: list/tuple of layers (also tracked by the Network API). if value and all(isinstance(val, Layer) for val in value): raise ValueError('We do not support the use of list-of-layers ' @@ -345,9 +351,6 @@ def _in_place_subclassed_model_reset(model): 'targets', '_feed_targets', 'sample_weight_modes', - 'weighted_metrics', - 'metrics_names', - 'metrics_tensors', 'total_loss', 'sample_weights', '_feed_sample_weights', @@ -495,10 +498,11 @@ def clone_and_build_model( clone.compile( optimizer, model.loss, - metrics=metrics_module.clone_metrics(model.metrics), + metrics=metrics_module.clone_metrics(model._compile_metrics), loss_weights=model.loss_weights, sample_weight_mode=model.sample_weight_mode, - weighted_metrics=metrics_module.clone_metrics(model.weighted_metrics), + weighted_metrics=metrics_module.clone_metrics( + model._compile_weighted_metrics), target_tensors=target_tensors) return clone diff --git a/tensorflow/python/keras/models_test.py b/tensorflow/python/keras/models_test.py index bf778f14971..b0872ae3abf 100644 --- a/tensorflow/python/keras/models_test.py +++ b/tensorflow/python/keras/models_test.py @@ -26,10 +26,12 @@ import numpy as np from tensorflow.python import keras from tensorflow.python.eager import context from tensorflow.python.framework import dtypes +from tensorflow.python.framework import ops from tensorflow.python.framework import test_util from tensorflow.python.keras import backend as K from tensorflow.python.keras import metrics from tensorflow.python.keras import models +from tensorflow.python.ops import array_ops from tensorflow.python.ops import random_ops from tensorflow.python.ops import resource_variable_ops from tensorflow.python.platform import test @@ -67,6 +69,7 @@ def sequential_model(add_input_layer, include_input_shape=True): class TestModelCloning(test.TestCase): + @test_util.run_deprecated_v1 def test_clone_sequential_model(self): with self.cached_session(): val_a = np.random.random((10, 4)) @@ -81,25 +84,28 @@ class TestModelCloning(test.TestCase): # With placeholder creation new_model = keras.models.clone_model(model) # update ops from batch norm needs to be included - self.assertEquals(len(new_model.get_updates_for(new_model.inputs)), 2) + self.assertEqual(len(new_model.get_updates_for(new_model.inputs)), 2) new_model.compile('rmsprop', 'mse') new_model.train_on_batch(val_a, val_out) # On top of new tensor input_a = keras.Input(shape=(4,)) new_model = keras.models.clone_model(model, input_tensors=input_a) - self.assertEquals(len(new_model.get_updates_for(new_model.inputs)), 2) + self.assertEqual(len(new_model.get_updates_for(new_model.inputs)), 2) new_model.compile('rmsprop', 'mse') new_model.train_on_batch(val_a, val_out) # On top of new, non-Keras tensor input_a = keras.backend.variable(val_a) new_model = keras.models.clone_model(model, input_tensors=input_a) - self.assertEquals(len(new_model.get_updates_for(new_model.inputs)), 2) + self.assertEqual(len(new_model.get_updates_for(new_model.inputs)), 2) new_model.compile('rmsprop', 'mse') new_model.train_on_batch(None, val_out) + @test_util.run_deprecated_v1 def test_clone_sequential_model_input_layer(self): + + @test_util.run_deprecated_v1 def test_input_layer(include_inputs): with self.cached_session(): val_a = np.random.random((10, 4)) @@ -136,6 +142,7 @@ class TestModelCloning(test.TestCase): test_input_layer(True) test_input_layer(False) + @test_util.run_deprecated_v1 def test_clone_functional_model(self): with self.cached_session(): val_a = np.random.random((10, 4)) @@ -161,7 +168,7 @@ class TestModelCloning(test.TestCase): with self.cached_session(): # With placeholder creation new_model = keras.models.clone_model(model) - self.assertEquals(len(new_model.get_updates_for(new_model.inputs)), 2) + self.assertEqual(len(new_model.get_updates_for(new_model.inputs)), 2) new_model.compile('rmsprop', 'mse') new_model.train_on_batch([val_a, val_b], val_out) @@ -170,7 +177,7 @@ class TestModelCloning(test.TestCase): input_b = keras.Input(shape=(4,), name='b') new_model = keras.models.clone_model( model, input_tensors=[input_a, input_b]) - self.assertEquals(len(new_model.get_updates_for(new_model.inputs)), 2) + self.assertEqual(len(new_model.get_updates_for(new_model.inputs)), 2) new_model.compile('rmsprop', 'mse') new_model.train_on_batch([val_a, val_b], val_out) @@ -179,7 +186,7 @@ class TestModelCloning(test.TestCase): input_b = keras.backend.variable(val_b) new_model = keras.models.clone_model( model, input_tensors=[input_a, input_b]) - self.assertEquals(len(new_model.get_updates_for(new_model.inputs)), 2) + self.assertEqual(len(new_model.get_updates_for(new_model.inputs)), 2) new_model.compile('rmsprop', 'mse') new_model.train_on_batch(None, val_out) @@ -219,6 +226,34 @@ class TestModelCloning(test.TestCase): with self.assertRaises(ValueError): keras.models._clone_sequential_model(seq_model, input_tensors=y) + def test_functional_cloning_does_not_create_unnecessary_placeholders(self): + with ops.Graph().as_default(): + x = keras.Input((4,)) + y = keras.layers.Dense(4)(x) + model = keras.models.Model(x, y) + graph = ops.Graph() + with graph.as_default(): + x = array_ops.ones((10, 4)) + _ = keras.models.clone_model(model, input_tensors=[x]) + has_placeholder = _has_placeholder(graph) + self.assertFalse(has_placeholder) + + def test_sequential_cloning_does_not_create_unnecessary_placeholders(self): + with ops.Graph().as_default(): + model = keras.models.Sequential() + model.add(keras.layers.Dense(4, input_shape=(4,))) + graph = ops.Graph() + with graph.as_default(): + x = array_ops.ones((10, 4)) + _ = keras.models.clone_model(model, input_tensors=[x]) + has_placeholder = _has_placeholder(graph) + self.assertFalse(has_placeholder) + + +def _has_placeholder(graph): + ops_types = [op.type for op in graph.get_operations()] + return any('Placeholder' in s for s in ops_types) + class CheckpointingTests(test.TestCase): @@ -331,7 +366,8 @@ class TestCloneAndBuildModel(test.TestCase): self.assertEqual('mse', model.loss) self.assertTrue( isinstance(model.optimizer, keras.optimizers.RMSprop)) - self.assertEqual(['acc', metrics.categorical_accuracy], model.metrics) + self.assertEqual(['acc', metrics.categorical_accuracy], + model._compile_metrics) def _clone_and_build_test_helper(self, model, is_subclassed=False): inp = np.random.random((10, 4)) @@ -366,6 +402,7 @@ class TestCloneAndBuildModel(test.TestCase): new_model.train_on_batch(inp, out) new_model.evaluate(inp, out) + @test_util.run_deprecated_v1 def test_clone_and_build_compiled_sequential_model(self): with self.cached_session(): model = keras.models.Sequential() @@ -378,6 +415,7 @@ class TestCloneAndBuildModel(test.TestCase): self._clone_and_build_test_helper(model) + @test_util.run_deprecated_v1 def test_clone_and_build_functional_model(self): with self.cached_session(): input_a = keras.Input(shape=(4,)) @@ -394,6 +432,7 @@ class TestCloneAndBuildModel(test.TestCase): self._clone_and_build_test_helper(model) + @test_util.run_deprecated_v1 def test_clone_and_build_subclassed_model(self): class SubclassedModel(keras.Model): @@ -442,9 +481,11 @@ class TestCloneAndBuildModel(test.TestCase): def test_replace_tf_optimizer_iterations_variable(self): self.assert_optimizer_iterations_increases(adam.AdamOptimizer(0.01)) + @test_util.run_deprecated_v1 def test_replace_keras_optimizer_iterations_variable(self): self.assert_optimizer_iterations_increases('adam') + @test_util.run_deprecated_v1 def test_clone_and_build_sequential_model_without_inputs_defined(self): with self.cached_session(): model = sequential_model(False, False) diff --git a/tensorflow/python/keras/optimizer_v2/BUILD b/tensorflow/python/keras/optimizer_v2/BUILD index fb43775fdc4..b8f01249419 100644 --- a/tensorflow/python/keras/optimizer_v2/BUILD +++ b/tensorflow/python/keras/optimizer_v2/BUILD @@ -7,14 +7,21 @@ licenses(["notice"]) # Apache 2.0 exports_files(["LICENSE"]) +load("//tensorflow:tensorflow.bzl", "py_test") load("//tensorflow:tensorflow.bzl", "cuda_py_test") py_library( name = "optimizer_v2", srcs = [ + "adadelta.py", + "adagrad.py", "adam.py", + "adamax.py", + "ftrl.py", "gradient_descent.py", + "nadam.py", "optimizer_v2.py", + "rmsprop.py", ], srcs_version = "PY2AND3", deps = [ @@ -24,12 +31,107 @@ py_library( "//tensorflow/python:math_ops", "//tensorflow/python:resource_variable_ops", "//tensorflow/python:state_ops", - "//tensorflow/python:training", "//tensorflow/python:variable_scope", "//tensorflow/python:variables", + "//tensorflow/python/distribute:reduce_util", ], ) +cuda_py_test( + name = "adagrad_test", + size = "medium", + srcs = ["adagrad_test.py"], + additional_deps = [ + ":optimizer_v2", + "//tensorflow/python:client_testlib", + "//tensorflow/python:embedding_ops", + "//tensorflow/python:platform_test", + "//tensorflow/python:framework", + "//tensorflow/python:math_ops", + "//tensorflow/python:resource_variable_ops", + "//tensorflow/python:resources", + "//tensorflow/python:variables", + "//tensorflow/python/eager:context", + ], + shard_count = 4, +) + +cuda_py_test( + name = "adam_test", + size = "medium", + srcs = ["adam_test.py"], + additional_deps = [ + ":optimizer_v2", + "//tensorflow/python:client_testlib", + "//tensorflow/python:embedding_ops", + "//tensorflow/python:platform_test", + "//tensorflow/python:framework", + "//tensorflow/python:math_ops", + "//tensorflow/python:resource_variable_ops", + "//tensorflow/python:resources", + "//tensorflow/python:variables", + "//tensorflow/python/eager:context", + ], + shard_count = 4, +) + +cuda_py_test( + name = "adamax_test", + size = "medium", + srcs = ["adamax_test.py"], + additional_deps = [ + ":optimizer_v2", + "//tensorflow/python:client_testlib", + "//tensorflow/python:embedding_ops", + "//tensorflow/python:platform_test", + "//tensorflow/python:framework", + "//tensorflow/python:math_ops", + "//tensorflow/python:resource_variable_ops", + "//tensorflow/python:resources", + "//tensorflow/python:variables", + "//tensorflow/python/eager:context", + ], + shard_count = 4, +) + +cuda_py_test( + name = "adadelta_test", + size = "medium", + srcs = ["adadelta_test.py"], + additional_deps = [ + ":optimizer_v2", + "//tensorflow/python:client_testlib", + "//tensorflow/python:embedding_ops", + "//tensorflow/python:platform_test", + "//tensorflow/python:framework", + "//tensorflow/python:math_ops", + "//tensorflow/python:resource_variable_ops", + "//tensorflow/python:resources", + "//tensorflow/python:variables", + "//tensorflow/python/eager:context", + ], + shard_count = 4, +) + +cuda_py_test( + name = "ftrl_test", + size = "medium", + srcs = ["ftrl_test.py"], + additional_deps = [ + ":optimizer_v2", + "//tensorflow/python:client_testlib", + "//tensorflow/python:embedding_ops", + "//tensorflow/python:platform_test", + "//tensorflow/python:framework", + "//tensorflow/python:math_ops", + "//tensorflow/python:resource_variable_ops", + "//tensorflow/python:resources", + "//tensorflow/python:variables", + "//tensorflow/python/eager:context", + ], + shard_count = 4, +) + cuda_py_test( name = "gradient_descent_test", size = "medium", @@ -50,9 +152,53 @@ cuda_py_test( ) cuda_py_test( - name = "optimizer_v2_test", + name = "nadam_test", size = "medium", + srcs = ["nadam_test.py"], + additional_deps = [ + ":optimizer_v2", + "//tensorflow/python:client_testlib", + "//tensorflow/python:embedding_ops", + "//tensorflow/python:platform_test", + "//tensorflow/python:framework", + "//tensorflow/python:math_ops", + "//tensorflow/python:resource_variable_ops", + "//tensorflow/python:resources", + "//tensorflow/python:variables", + "//tensorflow/python/eager:context", + ], + shard_count = 4, +) + +py_test( + name = "optimizer_v2_test", + size = "large", srcs = ["optimizer_v2_test.py"], + shard_count = 4, + tags = [ + "no_windows", + ], + deps = [ + ":optimizer_v2", + "//tensorflow/python:array_ops", + "//tensorflow/python:client_testlib", + "//tensorflow/python:clip_ops", + "//tensorflow/python:framework", + "//tensorflow/python:framework_test_lib", + "//tensorflow/python:gradients", + "//tensorflow/python:resource_variable_ops", + "//tensorflow/python:state_ops", + "//tensorflow/python:variables", + "//tensorflow/python/eager:def_function", + "//tensorflow/python/keras", + "@absl_py//absl/testing:parameterized", + ], +) + +cuda_py_test( + name = "rmsprop_test", + size = "medium", + srcs = ["rmsprop_test.py"], additional_deps = [ ":optimizer_v2", "//tensorflow/python/eager:def_function", diff --git a/tensorflow/python/keras/optimizer_v2/adadelta.py b/tensorflow/python/keras/optimizer_v2/adadelta.py new file mode 100644 index 00000000000..55b4eba1051 --- /dev/null +++ b/tensorflow/python/keras/optimizer_v2/adadelta.py @@ -0,0 +1,148 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +"""Adadelta for TensorFlow.""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import numpy as np + +from tensorflow.python.keras.optimizer_v2 import optimizer_v2 +from tensorflow.python.training import training_ops + + +class Adadelta(optimizer_v2.OptimizerV2): + r"""Optimizer that implements the Adadelta algorithm. + + Adadelta optimization is a stochastic gradient descent method that is based on + adaptive learning rate per dimension to address two drawbacks: + 1) the continual decay of learning rates throughout training + 2) the need for a manually selected global learning rate + + Two accumulation steps are required: + 1) the accumulation of gradients squared, + 2) the accumulation of updates squared. + + Initialization: + + $$accum_g_0 := 0 \text{(Initialize gradient 2nd order moment vector)}$$ + $$accum_x_0 := 0 \text{(Initialize variable update 2nd order moment vector)}$$ + + $$t := t + 1$$ + $$accum_g_t := rho * accum_g_{t-1} + (1 - rho) * g * g$$ + $$delta = -\sqrt{accum_x_{t-1}} / (\sqrt{accum_g_{t-1}} + \epsilon)$$ + $$accum_x_t := rho * accum_x_{t-1} + (1 - rho) * delta * delta$$ + + References + See [M. D. Zeiler](http://arxiv.org/abs/1212.5701) + ([pdf](http://arxiv.org/pdf/1212.5701v1.pdf)) + + """ + + def __init__(self, + learning_rate=0.001, + rho=0.95, + epsilon=1e-7, + name='Adadelta', + **kwargs): + """Construct a new Adadelta optimizer. + + Adadelta is a more robust extension of Adagrad that adapts learning rates + based on a moving window of gradient updates, instead of accumulating all + past gradients. This way, Adadelta continues learning even when many updates + have been done. Compared to Adagrad, in the original version of Adadelta you + don't have to set an initial learning rate. In this version, initial + learning rate can be set, as in most other Keras optimizers. + + Args: + learning_rate: A `Tensor` or a floating point value. The learning rate. + To match the exact form in the original paper use 1.0. + rho: A `Tensor` or a floating point value. The decay rate. + epsilon: A `Tensor` or a floating point value. A constant epsilon used + to better conditioning the grad update. + name: Optional name prefix for the operations created when applying + gradients. Defaults to "Adadelta". + **kwargs: keyword arguments. Allowed to be {`decay`} + + @compatibility(eager) + When eager execution is enabled, `learning_rate`, `rho`, and `epsilon` can + each be a callable that takes no arguments and returns the actual value to + use. This can be useful for changing these values across different + invocations of optimizer functions. + @end_compatibility + """ + super(Adadelta, self).__init__(name, **kwargs) + self._set_hyper('learning_rate', learning_rate) + self._set_hyper('decay', self._initial_decay) + self._set_hyper('rho', rho) + self._set_hyper('epsilon', epsilon) + + def _create_slots(self, var_list): + # Separate for-loops to respect the ordering of slot variables from v1. + for v in var_list: + self.add_slot(v, 'accum_grad') + for v in var_list: + self.add_slot(v, 'accum_var') + + def set_weights(self, weights): + params = self.weights + # Override set_weights for backward compatibility of Keras V1 optimizer + # since it does not include iteration at head of the weight list. Set + # iteration to 0. + if len(params) == len(weights) + 1: + weights = [np.array(0)] + weights + super(Adadelta, self).set_weights(weights) + + def _resource_apply_dense(self, grad, var): + var_dtype = var.dtype.base_dtype + lr_t = self._decayed_lr(var_dtype) + accum_grad = self.get_slot(var, 'accum_grad') + accum_var = self.get_slot(var, 'accum_var') + return training_ops.resource_apply_adadelta( + var.handle, + accum_grad.handle, + accum_var.handle, + lr_t, + self._get_hyper('rho', var_dtype), + self._get_hyper('epsilon', var_dtype), + grad, + use_locking=self._use_locking) + + def _resource_apply_sparse(self, grad, var, indices): + var_dtype = var.dtype.base_dtype + lr_t = self._decayed_lr(var_dtype) + accum_grad = self.get_slot(var, 'accum_grad') + accum_var = self.get_slot(var, 'accum_var') + return training_ops.resource_sparse_apply_adadelta( + var.handle, + accum_grad.handle, + accum_var.handle, + lr_t, + self._get_hyper('rho', var_dtype), + self._get_hyper('epsilon', var_dtype), + grad, + indices, + use_locking=self._use_locking) + + def get_config(self): + config = super(Adadelta, self).get_config() + config.update({ + 'learning_rate': self._serialize_hyperparameter('learning_rate'), + 'decay': self._serialize_hyperparameter('decay'), + 'rho': self._serialize_hyperparameter('rho'), + 'epsilon': self._serialize_hyperparameter('epsilon'), + }) + return config diff --git a/tensorflow/python/keras/optimizer_v2/adadelta_test.py b/tensorflow/python/keras/optimizer_v2/adadelta_test.py new file mode 100644 index 00000000000..0fb67d0cd16 --- /dev/null +++ b/tensorflow/python/keras/optimizer_v2/adadelta_test.py @@ -0,0 +1,170 @@ +# Copyright 2015 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Tests for Adadelta Optimizer.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import numpy as np + +from tensorflow.python.eager import context +from tensorflow.python.framework import constant_op +from tensorflow.python.framework import dtypes +from tensorflow.python.framework import test_util +from tensorflow.python.keras.optimizer_v2 import adadelta +from tensorflow.python.ops import embedding_ops +from tensorflow.python.ops import math_ops +from tensorflow.python.ops import resource_variable_ops +from tensorflow.python.ops import variables +from tensorflow.python.platform import test + + +class AdadeltaOptimizerTest(test.TestCase): + + def doTestBasic(self, use_resource=False, use_callable_params=False): + num_updates = 4 # number of ADADELTA steps to perform + for dtype in [dtypes.half, dtypes.float32]: + for grad in [0.2, 0.1, 0.01]: + for lr in [1.0, 0.5, 0.1]: + var0_init = [1.0, 2.0] + var1_init = [3.0, 4.0] + if use_resource: + var0 = resource_variable_ops.ResourceVariable( + var0_init, dtype=dtype) + var1 = resource_variable_ops.ResourceVariable( + var1_init, dtype=dtype) + else: + var0 = variables.Variable(var0_init, dtype=dtype) + var1 = variables.Variable(var1_init, dtype=dtype) + + grads = constant_op.constant([grad, grad], dtype=dtype) + + accum = 0.0 + accum_update = 0.0 + + # ADADELTA gradient optimizer + rho = 0.95 + epsilon = 1e-8 + if use_callable_params: + adadelta_opt = adadelta.Adadelta( + learning_rate=lambda: lr, # pylint: disable=cell-var-from-loop + rho=lambda: rho, # pylint: disable=cell-var-from-loop + epsilon=lambda: epsilon) # pylint: disable=cell-var-from-loop + else: + adadelta_opt = adadelta.Adadelta( + learning_rate=lr, rho=rho, epsilon=epsilon) + if not context.executing_eagerly(): + adadelta_update = adadelta_opt.apply_gradients( + zip([grads, grads], [var0, var1])) + self.evaluate(variables.global_variables_initializer()) + + # Assign slots + slot = [None] * 2 + slot_update = [None] * 2 + slot[0] = adadelta_opt.get_slot(var0, "accum_grad") + self.assertEqual(slot[0].get_shape(), var0.get_shape()) + + slot_update[0] = adadelta_opt.get_slot(var0, "accum_var") + self.assertEqual(slot_update[0].get_shape(), var0.get_shape()) + + slot[1] = adadelta_opt.get_slot(var1, "accum_grad") + self.assertEqual(slot[1].get_shape(), var1.get_shape()) + + slot_update[1] = adadelta_opt.get_slot(var1, "accum_var") + self.assertEqual(slot_update[1].get_shape(), var1.get_shape()) + + # Fetch params to validate initial values + self.assertAllClose(var0_init, self.evaluate(var0)) + self.assertAllClose(var1_init, self.evaluate(var1)) + + update = [None] * num_updates + tot_update = 0 + for step in range(num_updates): + # Run adadelta update for comparison + if not context.executing_eagerly(): + self.evaluate(adadelta_update) + else: + adadelta_opt.apply_gradients(zip([grads, grads], [var0, var1])) + + # Perform initial update without previous accum values + accum = accum * rho + (grad**2) * (1 - rho) + update[step] = ( + np.sqrt(accum_update + epsilon) * + (1. / np.sqrt(accum + epsilon)) * grad) + accum_update = ( + accum_update * rho + (update[step]**2) * (1.0 - rho)) + tot_update += update[step] * lr + + if not context.executing_eagerly(): + # Check that the accumulators have been updated + # TODO(lxuechen): This is hard to test in eager mode + for slot_idx in range(2): + self.assertAllCloseAccordingToType( + np.array([accum, accum], dtype=dtype.as_numpy_dtype()), + self.evaluate(slot[slot_idx]), + rtol=1e-5) + + self.assertAllCloseAccordingToType( + np.array( + [accum_update, accum_update], + dtype=dtype.as_numpy_dtype()), + self.evaluate(slot_update[slot_idx]), + rtol=1e-5) + + # Check that the parameters have been updated + self.assertAllCloseAccordingToType( + np.array( + [var0_init[0] - tot_update, var0_init[1] - tot_update], + dtype=dtype.as_numpy_dtype()), + self.evaluate(var0), + rtol=1e-5) + + self.assertAllCloseAccordingToType( + np.array( + [var1_init[0] - tot_update, var1_init[1] - tot_update], + dtype=dtype.as_numpy_dtype()), + self.evaluate(var1), + rtol=1e-5) + + @test_util.run_in_graph_and_eager_modes(reset_test=True) + def testResourceBasic(self): + self.doTestBasic(use_resource=True) + + def testBasicCallableParams(self): + with context.eager_mode(): + self.doTestBasic(use_resource=True, use_callable_params=True) + + @test_util.run_deprecated_v1 + def testMinimizeSparseResourceVariable(self): + for dtype in [dtypes.half, dtypes.float32, dtypes.float64]: + with self.cached_session(): + var0 = resource_variable_ops.ResourceVariable([[1.0, 2.0]], dtype=dtype) + x = constant_op.constant([[4.0], [5.0]], dtype=dtype) + pred = math_ops.matmul(embedding_ops.embedding_lookup([var0], [0]), x) + loss = pred * pred + sgd_op = adadelta.Adadelta(1.0, 1.0, 1.0).minimize( + loss, var_list=[var0]) + variables.global_variables_initializer().run() + # Fetch params to validate initial values + self.assertAllCloseAccordingToType([[1.0, 2.0]], self.evaluate(var0)) + # Run 1 step of sgd + sgd_op.run() + # Validate updated params + self.assertAllCloseAccordingToType([[-111, -138]], self.evaluate(var0)) + + +if __name__ == "__main__": + test.main() diff --git a/tensorflow/python/keras/optimizer_v2/adagrad.py b/tensorflow/python/keras/optimizer_v2/adagrad.py new file mode 100644 index 00000000000..670cad70e63 --- /dev/null +++ b/tensorflow/python/keras/optimizer_v2/adagrad.py @@ -0,0 +1,171 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +"""Adagrad for TensorFlow.""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import numpy as np + +from tensorflow.python.framework import ops +from tensorflow.python.keras.optimizer_v2 import optimizer_v2 +from tensorflow.python.ops import array_ops +from tensorflow.python.ops import init_ops +from tensorflow.python.ops import math_ops +from tensorflow.python.ops import resource_variable_ops +from tensorflow.python.ops import state_ops + + +class Adagrad(optimizer_v2.OptimizerV2): + r"""Optimizer that implements the Adagrad algorithm. + + Adagrad is an optimizer with parameter-specific learning rates, + which are adapted relative to how frequently a parameter gets + updated during training. The more updates a parameter receives, + the smaller the updates. + + Initialization: + + $$accum_g_0 := initial_accumulator_value$$ + + $$t := t + 1$$ + $$accum_g_t := accum_g_{t-1} + g * g$$ + $$theta_t := theta_{t-1} - lr * g / (\sqrt{accum_g_t} + \epsilon)$$ + + References + See [paper] + (http://www.jmlr.org/papers/volume12/duchi11a/duchi11a.pdf) + or this + [intro](https://ppasupat.github.io/a9online/uploads/proximal_notes.pdf). + """ + + def __init__(self, + learning_rate=0.001, + initial_accumulator_value=0.1, + epsilon=1e-7, + name='Adagrad', + **kwargs): + """Construct a new Adagrad optimizer. + + Args: + learning_rate: A `Tensor` or a floating point value. The learning rate. + initial_accumulator_value: A floating point value. + Starting value for the accumulators, must be positive. + epsilon: A floating point value. + Starting value for the accumulators, must be positive. + name: Optional name prefix for the operations created when applying + gradients. Defaults to "Adagrad". + **kwargs: keyword arguments. Allowed to be {`decay`} + + Raises: + ValueError: If the `initial_accumulator_value` or `epsilon` is invalid. + + @compatibility(eager) + When eager execution is enabled, `learning_rate` can be a callable that + takes no arguments and returns the actual value to use. This can be useful + for changing these values across different invocations of optimizer + functions. + @end_compatibility + """ + if initial_accumulator_value < 0.0: + raise ValueError('initial_accumulator_value must be non-negative: %s' % + initial_accumulator_value) + if epsilon < 1e-7: + raise ValueError('epsilon must be larger than 1e-7: %s' % epsilon) + super(Adagrad, self).__init__(name, **kwargs) + self._set_hyper('learning_rate', learning_rate) + self._set_hyper('decay', self._initial_decay) + self._initial_accumulator_value = initial_accumulator_value + self._set_hyper('epsilon', epsilon) + + def _create_slots(self, var_list): + for var in var_list: + dtype = var.dtype.base_dtype + init = init_ops.constant_initializer( + self._initial_accumulator_value, dtype=dtype) + self.add_slot(var, 'accumulator', init) + + def set_weights(self, weights): + params = self.weights + # Override set_weights for backward compatibility of Keras V1 optimizer + # since it does not include iteration at head of the weight list. Set + # iteration to 0. + if len(params) == len(weights) + 1: + weights = [np.array(0)] + weights + super(Adagrad, self).set_weights(weights) + + @classmethod + def from_config(cls, config, custom_objects=None): + """Creates an optimizer from its config. + + This method is the reverse of `get_config`, + capable of instantiating the same optimizer from the config + dictionary. + + Arguments: + config: A Python dictionary, typically the output of get_config. + custom_objects: A Python dictionary mapping names to additional Python + objects used to create this optimizer, such as a function used for a + hyperparameter. + + Returns: + An optimizer instance. + """ + if 'initial_accumulator_value' not in config: + config['initial_accumulator_value'] = 0. + if 'lr' in config: + config['learning_rate'] = config.pop('lr') + return cls(**config) + + def _resource_apply_dense(self, grad, var): + var_dtype = var.dtype.base_dtype + lr_t = self._decayed_lr(var_dtype) + epsilon = self._get_hyper('epsilon', var_dtype) + acc = self.get_slot(var, 'accumulator') + + acc_t = state_ops.assign_add( + acc, math_ops.square(grad), use_locking=self._use_locking) + var_update = state_ops.assign_sub( + var, lr_t * grad / (math_ops.sqrt(acc_t) + epsilon)) + return var_update + + def _resource_apply_sparse(self, grad, var, indices): + + def _resource_scatter_add(x, i, v): + with ops.control_dependencies( + [resource_variable_ops.resource_scatter_add(x.handle, i, v)]): + return x.value() + + var_dtype = var.dtype.base_dtype + lr_t = self._decayed_lr(var_dtype) + epsilon = self._get_hyper('epsilon', var_dtype) + acc = self.get_slot(var, 'accumulator') + + acc_t = _resource_scatter_add(acc, indices, math_ops.square(grad)) + acc_t_slice = array_ops.gather(acc_t, indices) + var_update = _resource_scatter_add( + var, indices, -lr_t * grad / (math_ops.sqrt(acc_t_slice) + epsilon)) + return var_update + + def get_config(self): + config = super(Adagrad, self).get_config() + config.update({ + 'learning_rate': self._serialize_hyperparameter('learning_rate'), + 'decay': self._serialize_hyperparameter('decay'), + 'initial_accumulator_value': self._initial_accumulator_value, + 'epsilon': self._serialize_hyperparameter('epsilon'), + }) + return config diff --git a/tensorflow/python/keras/optimizer_v2/adagrad_test.py b/tensorflow/python/keras/optimizer_v2/adagrad_test.py new file mode 100644 index 00000000000..b2c290178fe --- /dev/null +++ b/tensorflow/python/keras/optimizer_v2/adagrad_test.py @@ -0,0 +1,400 @@ +# Copyright 2015 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Functional tests for aggregate operations.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import copy + +import numpy as np + +from tensorflow.python.eager import context +from tensorflow.python.framework import constant_op +from tensorflow.python.framework import dtypes +from tensorflow.python.framework import ops +from tensorflow.python.framework import test_util +from tensorflow.python.keras.optimizer_v2 import adagrad +from tensorflow.python.ops import embedding_ops +from tensorflow.python.ops import math_ops +from tensorflow.python.ops import resource_variable_ops +from tensorflow.python.ops import variables +from tensorflow.python.platform import test + + +def adagrad_update_numpy(param, accum, g_t, lr=0.001, epsilon=1e-7): + accum_t = accum + g_t * g_t + param_t = param - lr * g_t / (np.sqrt(accum_t) + epsilon) + return param_t, accum_t + + +def sparse_adagrad_update_numpy(param, + accum, + gindexs, + gvalues, + lr=0.001, + epsilon=1e-7): + accum_t = copy.deepcopy(accum) + param_t = copy.deepcopy(param) + # first loop accumulates repeated indices if necessary. + for i in range(len(gindexs)): + gindex = gindexs[i] + gvalue = gvalues[i] + accum_t[gindex] = accum_t[gindex] + gvalue * gvalue + for i in range(len(gindexs)): + gindex = gindexs[i] + gvalue = gvalues[i] + param_t[gindex] = param_t[gindex] - lr * gvalue / ( + np.sqrt(accum_t[gindex]) + epsilon) + return param_t, accum_t + + +class AdagradOptimizerTest(test.TestCase): + + def doTestBasic(self, use_callable_params=False): + for dtype in [dtypes.float32, dtypes.float64]: + with self.cached_session(): + var0_np = np.array([1.0, 2.0], dtype=dtype.as_numpy_dtype) + var1_np = np.array([3.0, 4.0], dtype=dtype.as_numpy_dtype) + grads0_np = np.array([0.1, 0.1], dtype=dtype.as_numpy_dtype) + grads1_np = np.array([0.01, 0.01], dtype=dtype.as_numpy_dtype) + var0 = resource_variable_ops.ResourceVariable(var0_np) + var1 = resource_variable_ops.ResourceVariable(var1_np) + grads0 = constant_op.constant(grads0_np) + grads1 = constant_op.constant(grads1_np) + + learning_rate = lambda: 3.0 + if not use_callable_params: + learning_rate = learning_rate() + + ada_opt = adagrad.Adagrad(learning_rate) + + accum0_np = np.array([0.1, 0.1], dtype=dtype.as_numpy_dtype) + accum1_np = np.array([0.1, 0.1], dtype=dtype.as_numpy_dtype) + + if not context.executing_eagerly(): + ada_update = ada_opt.apply_gradients( + zip([grads0, grads1], [var0, var1])) + self.evaluate(variables.global_variables_initializer()) + + # Fetch params to validate initial values + v0_val, v1_val = self.evaluate([var0, var1]) + self.assertAllClose([1.0, 2.0], v0_val) + self.assertAllClose([3.0, 4.0], v1_val) + + # Run 3 steps of adagrad + for _ in range(3): + if not context.executing_eagerly(): + self.evaluate(ada_update) + else: + ada_opt.apply_gradients(zip([grads0, grads1], [var0, var1])) + var0_np, accum0_np = adagrad_update_numpy(var0_np, accum0_np, + grads0_np, 3.0) + var1_np, accum1_np = adagrad_update_numpy(var1_np, accum1_np, + grads1_np, 3.0) + self.assertAllCloseAccordingToType(var0_np, self.evaluate(var0)) + self.assertAllCloseAccordingToType(var1_np, self.evaluate(var1)) + + @test_util.run_in_graph_and_eager_modes(reset_test=True) + def testBasic(self): + self.doTestBasic() + + def testBasicCallableParams(self): + with context.eager_mode(): + self.doTestBasic(use_callable_params=True) + + def testBasicWithLearningRateDecay(self): + for dtype in [dtypes.float32, dtypes.float64]: + with self.cached_session(): + var0_np = np.array([1.0, 2.0], dtype=dtype.as_numpy_dtype) + var1_np = np.array([3.0, 4.0], dtype=dtype.as_numpy_dtype) + grads0_np = np.array([0.1, 0.1], dtype=dtype.as_numpy_dtype) + grads1_np = np.array([0.01, 0.01], dtype=dtype.as_numpy_dtype) + var0 = resource_variable_ops.ResourceVariable(var0_np) + var1 = resource_variable_ops.ResourceVariable(var1_np) + grads0 = constant_op.constant(grads0_np) + grads1 = constant_op.constant(grads1_np) + + learning_rate = 3.0 + decay = 0.5 + + ada_opt = adagrad.Adagrad(learning_rate, decay=decay) + + accum0_np = np.array([0.1, 0.1], dtype=dtype.as_numpy_dtype) + accum1_np = np.array([0.1, 0.1], dtype=dtype.as_numpy_dtype) + + if not context.executing_eagerly(): + ada_update = ada_opt.apply_gradients( + zip([grads0, grads1], [var0, var1])) + self.evaluate(variables.global_variables_initializer()) + + # Fetch params to validate initial values + v0_val, v1_val = self.evaluate([var0, var1]) + self.assertAllClose([1.0, 2.0], v0_val) + self.assertAllClose([3.0, 4.0], v1_val) + + # Run 3 steps of adagrad + for t in range(3): + if not context.executing_eagerly(): + self.evaluate(ada_update) + else: + ada_opt.apply_gradients(zip([grads0, grads1], [var0, var1])) + lr_np = learning_rate / (1 + decay * t) + var0_np, accum0_np = adagrad_update_numpy(var0_np, accum0_np, + grads0_np, lr_np) + var1_np, accum1_np = adagrad_update_numpy(var1_np, accum1_np, + grads1_np, lr_np) + self.assertAllCloseAccordingToType(var0_np, self.evaluate(var0)) + self.assertAllCloseAccordingToType(var1_np, self.evaluate(var1)) + + @test_util.run_deprecated_v1 + def testMinimizeSparseResourceVariable(self): + for dtype in [dtypes.half, dtypes.float32, dtypes.float64]: + with self.cached_session(): + var0 = resource_variable_ops.ResourceVariable( + [[1.0, 2.0], [3.0, 4.0]], dtype=dtype) + x = constant_op.constant([[4.0], [5.0]], dtype=dtype) + pred = math_ops.matmul(embedding_ops.embedding_lookup([var0], [0]), x) + loss = pred * pred + sgd_op = adagrad.Adagrad(1.0).minimize(loss, var_list=[var0]) + variables.global_variables_initializer().run() + # Fetch params to validate initial values + self.assertAllCloseAccordingToType( + [[1.0, 2.0], [3.0, 4.0]], var0.eval()) + # Run 1 step of sgd + sgd_op.run() + # Validate updated params + self.assertAllCloseAccordingToType( + [[0, 1], [3, 4]], var0.eval(), atol=0.01) + + @test_util.run_deprecated_v1 + def testTensorLearningRate(self): + for dtype in [dtypes.half, dtypes.float32, dtypes.float64]: + with self.cached_session(): + var0_np = np.array([1.0, 2.0], dtype=dtype.as_numpy_dtype) + var1_np = np.array([3.0, 4.0], dtype=dtype.as_numpy_dtype) + grads0_np = np.array([0.1, 0.1], dtype=dtype.as_numpy_dtype) + grads1_np = np.array([0.01, 0.01], dtype=dtype.as_numpy_dtype) + var0 = resource_variable_ops.ResourceVariable(var0_np) + var1 = resource_variable_ops.ResourceVariable(var1_np) + grads0 = constant_op.constant(grads0_np) + grads1 = constant_op.constant(grads1_np) + + learning_rate = constant_op.constant(3.0) + ada_opt = adagrad.Adagrad(learning_rate) + ada_update = ada_opt.apply_gradients( + zip([grads0, grads1], [var0, var1])) + variables.global_variables_initializer().run() + # Fetch params to validate initial values + self.assertAllClose([1.0, 2.0], var0.eval()) + self.assertAllClose([3.0, 4.0], var1.eval()) + accum0_np = np.array([0.1, 0.1], dtype=dtype.as_numpy_dtype) + accum1_np = np.array([0.1, 0.1], dtype=dtype.as_numpy_dtype) + # Run 3 steps of adagrad + for _ in range(3): + ada_update.run() + var0_np, accum0_np = adagrad_update_numpy(var0_np, accum0_np, + grads0_np, learning_rate) + var1_np, accum1_np = adagrad_update_numpy(var1_np, accum1_np, + grads1_np, learning_rate) + self.assertAllCloseAccordingToType(var0_np, self.evaluate(var0)) + self.assertAllCloseAccordingToType(var1_np, self.evaluate(var1)) + + @test_util.run_deprecated_v1 + def testSparseBasic(self): + for dtype in [dtypes.half, dtypes.float32, dtypes.float64]: + with self.cached_session(): + var0_np = np.array([1.0, 1.0, 2.0], dtype=dtype.as_numpy_dtype) + grads0_np = np.array([0.1, 0, 0.1], dtype=dtype.as_numpy_dtype) + var1_np = np.array([3.0, 3.0, 4.0], dtype=dtype.as_numpy_dtype) + grads1_np = np.array([0.01, 0, 0.01], dtype=dtype.as_numpy_dtype) + + var0 = resource_variable_ops.ResourceVariable(var0_np) + var1 = resource_variable_ops.ResourceVariable(var1_np) + grads0_np_indices = np.array([0, 2], dtype=np.int32) + grads0 = ops.IndexedSlices( + constant_op.constant(grads0_np[grads0_np_indices]), + constant_op.constant(grads0_np_indices), constant_op.constant([3])) + grads1_np_indices = np.array([0, 2], dtype=np.int32) + grads1 = ops.IndexedSlices( + constant_op.constant(grads1_np[grads1_np_indices]), + constant_op.constant(grads1_np_indices), constant_op.constant([3])) + learning_rate = 3.0 + ada_opt = adagrad.Adagrad(learning_rate) + ada_update = ada_opt.apply_gradients( + zip([grads0, grads1], [var0, var1])) + variables.global_variables_initializer().run() + + # Fetch params to validate initial values + self.assertAllClose([1.0, 1.0, 2.0], var0.eval()) + self.assertAllClose([3.0, 3.0, 4.0], var1.eval()) + + accum0_np = np.array([0.1, 0.1, 0.1], dtype=dtype.as_numpy_dtype) + accum1_np = np.array([0.1, 0.1, 0.1], dtype=dtype.as_numpy_dtype) + + # Run 3 step of sgd + for _ in range(3): + ada_update.run() + + var0_np, accum0_np = sparse_adagrad_update_numpy( + var0_np, accum0_np, grads0_np_indices, + grads0_np[grads0_np_indices], learning_rate) + var1_np, accum1_np = sparse_adagrad_update_numpy( + var1_np, accum1_np, grads1_np_indices, + grads1_np[grads1_np_indices], learning_rate) + self.assertAllCloseAccordingToType(var0_np, self.evaluate(var0)) + self.assertAllCloseAccordingToType(var1_np, self.evaluate(var1)) + + @test_util.run_deprecated_v1 + def testSparseRepeatedIndices(self): + for dtype in [dtypes.half, dtypes.float32, dtypes.float64]: + with self.cached_session(): + var_np = np.array([[1.0], [2.0]], dtype=dtype.as_numpy_dtype) + + repeated_index_update_var = resource_variable_ops.ResourceVariable( + var_np, dtype=dtype) + aggregated_update_var = resource_variable_ops.ResourceVariable( + var_np, dtype=dtype) + grad_repeated_index = ops.IndexedSlices( + constant_op.constant( + [0.1, 0.1], shape=[2, 1], dtype=dtype), + constant_op.constant([1, 1]), + constant_op.constant([2, 1])) + grad_aggregated = ops.IndexedSlices( + constant_op.constant( + [0.2], shape=[1, 1], dtype=dtype), + constant_op.constant([1]), + constant_op.constant([2, 1])) + repeated_update = adagrad.Adagrad(3.0).apply_gradients( + [(grad_repeated_index, repeated_index_update_var)]) + aggregated_update = adagrad.Adagrad(3.0).apply_gradients( + [(grad_aggregated, aggregated_update_var)]) + variables.global_variables_initializer().run() + self.assertAllClose(aggregated_update_var.eval(), + repeated_index_update_var.eval()) + for _ in range(3): + repeated_update.run() + aggregated_update.run() + self.assertAllClose(aggregated_update_var.eval(), + repeated_index_update_var.eval()) + + @test_util.run_deprecated_v1 + def testSparseRepeatedIndicesByEmbeddingLookUp(self): + for dtype in [dtypes.half, dtypes.float32, dtypes.float64]: + with self.cached_session(): + var_repeated = resource_variable_ops.ResourceVariable( + [1.0, 2.0], dtype=dtype) + loss_repeated = math_ops.reduce_sum( + embedding_ops.embedding_lookup(var_repeated, [0, 0])) + var_aggregated = resource_variable_ops.ResourceVariable( + [1.0, 2.0], dtype=dtype) + loss_aggregated = 2 * math_ops.reduce_sum( + embedding_ops.embedding_lookup(var_aggregated, [0])) + update_op_repeated = adagrad.Adagrad(2.0).minimize( + loss_repeated, var_list=[var_repeated]) + update_op_aggregated = adagrad.Adagrad(2.0).minimize( + loss_aggregated, var_list=[var_aggregated]) + variables.global_variables_initializer().run() + self.assertAllCloseAccordingToType( + var_repeated.eval(), var_aggregated.eval()) + for _ in range(3): + update_op_repeated.run() + update_op_aggregated.run() + self.assertAllCloseAccordingToType( + var_repeated.eval(), var_aggregated.eval()) + + @test_util.run_deprecated_v1 + def testSparseStability(self): + for dtype in [dtypes.half]: + with self.cached_session(): + shape = [1, 6] + var0_np = np.array([[ + 0.00872496, -0.106952, 0.110467, 0.226505, -0.0147257, -0.0105945 + ]], + dtype=dtype.as_numpy_dtype) + var0 = resource_variable_ops.ResourceVariable(var0_np) + grads0_np = np.array([[ + -5.91278e-05, 5.31673e-05, -2.5779e-06, 4.29153e-05, -8.4877e-05, + -9.48906e-05 + ]], + dtype=dtype.as_numpy_dtype) + grads0 = ops.IndexedSlices( + constant_op.constant(grads0_np), constant_op.constant([0]), + constant_op.constant(shape)) + ada_opt = adagrad.Adagrad(1.0) + ada_update = ada_opt.apply_gradients(zip([grads0], [var0])) + slot0 = ada_opt.get_slot(var0, "accumulator") + init = variables.global_variables_initializer() + for _ in range(100): + init.run() + ada_update.run() + self.assertAllCloseAccordingToType( + np.array([[0.1, 0.1, 0.1, 0.1, 0.1, 0.1]]), slot0.eval()) + self.assertAllCloseAccordingToType( + np.array([[ + 0.00891194, -0.10712013, 0.11047515, 0.22636929, -0.0144573, + -0.01029443 + ]]), var0.eval()) + + @test_util.run_deprecated_v1 + def testSharing(self): + for dtype in [dtypes.half, dtypes.float32, dtypes.float64]: + with self.cached_session(): + var0_np = np.array([1.0, 2.0], dtype=dtype.as_numpy_dtype) + grads0_np = np.array([0.1, 0.1], dtype=dtype.as_numpy_dtype) + var1_np = np.array([3.0, 4.0], dtype=dtype.as_numpy_dtype) + grads1_np = np.array([0.01, 0.01], dtype=dtype.as_numpy_dtype) + + var0 = resource_variable_ops.ResourceVariable(var0_np) + var1 = resource_variable_ops.ResourceVariable(var1_np) + grads0 = constant_op.constant(grads0_np) + grads1 = constant_op.constant(grads1_np) + + learning_rate = 3.0 + ada_opt = adagrad.Adagrad(learning_rate) + # Apply the optimizer twice. Both applications will use + # the same accums. + ada_update1 = ada_opt.apply_gradients( + zip([grads0, grads1], [var0, var1])) + ada_update2 = ada_opt.apply_gradients( + zip([grads0, grads1], [var0, var1])) + slot0 = ada_opt.get_slot(var0, "accumulator") + self.assertEqual(slot0.get_shape(), var0.get_shape()) + slot1 = ada_opt.get_slot(var1, "accumulator") + self.assertEqual(slot1.get_shape(), var1.get_shape()) + variables.global_variables_initializer().run() + + # Fetch params to validate initial values. + self.assertAllClose([1.0, 2.0], var0.eval()) + self.assertAllClose([3.0, 4.0], var1.eval()) + # Mix the first and the second adagrad for 3 steps. + ada_update1.run() + ada_update2.run() + ada_update1.run() + + accum0_np = np.array([0.1, 0.1], dtype=dtype.as_numpy_dtype) + accum1_np = np.array([0.1, 0.1], dtype=dtype.as_numpy_dtype) + for _ in range(3): + var0_np, accum0_np = adagrad_update_numpy(var0_np, accum0_np, + grads0_np, learning_rate) + var1_np, accum1_np = adagrad_update_numpy(var1_np, accum1_np, + grads1_np, learning_rate) + self.assertAllCloseAccordingToType(var0_np, self.evaluate(var0)) + self.assertAllCloseAccordingToType(var1_np, self.evaluate(var1)) + + +if __name__ == "__main__": + test.main() diff --git a/tensorflow/python/keras/optimizer_v2/adam.py b/tensorflow/python/keras/optimizer_v2/adam.py index b05811c419f..ef3d783f891 100644 --- a/tensorflow/python/keras/optimizer_v2/adam.py +++ b/tensorflow/python/keras/optimizer_v2/adam.py @@ -1,4 +1,4 @@ -# Copyright 2015 The TensorFlow Authors. All Rights Reserved. +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -17,8 +17,12 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function +from tensorflow.python.framework import ops from tensorflow.python.keras.optimizer_v2 import optimizer_v2 +from tensorflow.python.ops import control_flow_ops from tensorflow.python.ops import math_ops +from tensorflow.python.ops import resource_variable_ops +from tensorflow.python.ops import state_ops from tensorflow.python.training import training_ops @@ -31,36 +35,62 @@ class Adam(optimizer_v2.OptimizerV2): requirement, invariant to diagonal rescaling of gradients, and is well suited for problems that are large in terms of data/parameters'. + Note, amsgrad is currently not supported and the argument can only be False. + # References See [Kingma et al., 2014](http://arxiv.org/abs/1412.6980) ([pdf](http://arxiv.org/pdf/1412.6980.pdf)). + For AMSGrad see [Reddi et al., 2-18] + (https://openreview.net/pdf?id=ryQu7f-RZ) """ def __init__(self, learning_rate=0.001, beta_1=0.9, beta_2=0.999, - epsilon=1e-8, - name='Adam'): + epsilon=1e-7, + amsgrad=False, + name='Adam', + **kwargs): r"""Construct a new Adam optimizer. - Initialization: + If amsgrad = False: + Initialization: - $$m_0 := 0 \text{(Initialize initial 1st moment vector)}$$ - $$v_0 := 0 \text{(Initialize initial 2nd moment vector)}$$ - $$t := 0 \text{(Initialize timestep)}$$ + $$m_0 := 0 \text{(Initialize initial 1st moment vector)}$$ + $$v_0 := 0 \text{(Initialize initial 2nd moment vector)}$$ + $$t := 0 \text{(Initialize timestep)}$$ - The update rule for `variable` with gradient `g` uses an optimization - described at the end of section2 of the paper: + The update rule for `variable` with gradient `g` uses an optimization + described at the end of section2 of the paper: - $$t := t + 1$$ - $$lr_t := \text{learning\_rate} * \sqrt{1 - beta_2^t} / (1 - beta_1^t)$$ + $$t := t + 1$$ + $$lr_t := \text{learning\_rate} * \sqrt{1 - beta_2^t} / (1 - beta_1^t)$$ - $$m_t := beta_1 * m_{t-1} + (1 - beta_1) * g$$ - $$v_t := beta_2 * v_{t-1} + (1 - beta_2) * g * g$$ - $$variable := variable - lr_t * m_t / (\sqrt{v_t} + \epsilon)$$ + $$m_t := beta_1 * m_{t-1} + (1 - beta_1) * g$$ + $$v_t := beta_2 * v_{t-1} + (1 - beta_2) * g * g$$ + $$variable := variable - lr_t * m_t / (\sqrt{v_t} + \epsilon)$$ - The default value of 1e-8 for epsilon might not be a good default in + If amsgrad = True: + Initialization: + + $$m_0 := 0 \text{(Initialize initial 1st moment vector)}$$ + $$v_0 := 0 \text{(Initialize initial 2nd moment vector)}$$ + $$v_hat_0 := 0 \text{(Initialize initial 2nd moment vector)}$$ + $$t := 0 \text{(Initialize timestep)}$$ + + The update rule for `variable` with gradient `g` uses an optimization + described at the end of section2 of the paper: + + $$t := t + 1$$ + $$lr_t := \text{learning\_rate} * \sqrt{1 - beta_2^t} / (1 - beta_1^t)$$ + + $$m_t := beta_1 * m_{t-1} + (1 - beta_1) * g$$ + $$v_t := beta_2 * v_{t-1} + (1 - beta_2) * g * g$$ + $$v_hat_t := max(v_hat_{t-1}, v_t) + $$variable := variable - lr_t * m_t / (\sqrt{v_hat_t} + \epsilon)$$ + + The default value of 1e-7 for epsilon might not be a good default in general. For example, when training an Inception network on ImageNet a current good choice is 1.0 or 0.1. Note that since AdamOptimizer uses the formulation just before Section 2.1 of the Kingma and Ba paper rather than @@ -85,50 +115,142 @@ class Adam(optimizer_v2.OptimizerV2): epsilon: A small constant for numerical stability. This epsilon is "epsilon hat" in the Kingma and Ba paper (in the formula just before Section 2.1), not the epsilon in Algorithm 1 of the paper. + amsgrad: boolean. Whether to apply AMSGrad variant of this algorithm from + the paper "On the Convergence of Adam and beyond". name: Optional name for the operations created when applying gradients. Defaults to "Adam". @compatibility(eager) When eager execution is enabled, `learning_rate`, `beta_1`, `beta_2`, and `epsilon` can each be a callable that takes no arguments and returns the actual value to use. This can be useful for changing these values across different invocations of optimizer functions. @end_compatibility + **kwargs: keyword arguments. Allowed to be {`decay`} """ - super(Adam, self).__init__(name) + super(Adam, self).__init__(name, **kwargs) self._set_hyper('learning_rate', learning_rate) + self._set_hyper('decay', self._initial_decay) self._set_hyper('beta_1', beta_1) self._set_hyper('beta_2', beta_2) self._set_hyper('epsilon', epsilon) + self._amsgrad = amsgrad def _create_slots(self, var_list): # Create slots for the first and second moments. + # Separate for-loops to respect the ordering of slot variables from v1. for var in var_list: self.add_slot(var, 'm') + for var in var_list: self.add_slot(var, 'v') + if self._amsgrad: + for var in var_list: + self.add_slot(var, 'vhat') + + def set_weights(self, weights): + params = self.weights + # If the weights are generated by Keras V1 optimizer, it includes vhats + # even without amsgrad, i.e, V1 optimizer has 3x + 1 variables, while V2 + # optimizer has 2x + 1 variables. Filter vhats out for compatibility. + num_vars = int((len(params) - 1) / 2) + if len(weights) == 3 * num_vars + 1: + weights = weights[:len(params)] + super(Adam, self).set_weights(weights) def _resource_apply_dense(self, grad, var): + var_dtype = var.dtype.base_dtype + lr_t = self._decayed_lr(var_dtype) m = self.get_slot(var, 'm') v = self.get_slot(var, 'v') - # TODO(tanzheny): let optimizer have its own step counter, and let - # beta1_power and beta2_power depend on it. - return training_ops.resource_apply_adam( - var.handle, - m.handle, - v.handle, - math_ops.cast(self._get_hyper('beta_1'), grad.dtype.base_dtype), - math_ops.cast(self._get_hyper('beta_2'), grad.dtype.base_dtype), - math_ops.cast(self._get_hyper('learning_rate'), grad.dtype.base_dtype), - math_ops.cast(self._get_hyper('beta_1'), grad.dtype.base_dtype), - math_ops.cast(self._get_hyper('beta_2'), grad.dtype.base_dtype), - math_ops.cast(self._get_hyper('epsilon'), grad.dtype.base_dtype), - grad, - use_locking=self._use_locking) + beta_1_t = self._get_hyper('beta_1', var_dtype) + beta_2_t = self._get_hyper('beta_2', var_dtype) + epsilon = self._get_hyper('epsilon', var_dtype) + local_step = math_ops.cast(self.iterations + 1, var_dtype) + beta_1_power = math_ops.pow(beta_1_t, local_step) + beta_2_power = math_ops.pow(beta_2_t, local_step) + if not self._amsgrad: + return training_ops.resource_apply_adam( + var.handle, + m.handle, + v.handle, + beta_1_power, + beta_2_power, + lr_t, + beta_1_t, + beta_2_t, + epsilon, + grad, + use_locking=self._use_locking) + else: + vhat = self.get_slot(var, 'vhat') + return training_ops.resource_apply_adam_with_amsgrad( + var.handle, + m.handle, + v.handle, + vhat.handle, + beta_1_power, + beta_2_power, + lr_t, + beta_1_t, + beta_2_t, + epsilon, + grad, + use_locking=self._use_locking) + + def _resource_apply_sparse(self, grad, var, indices): + var_dtype = var.dtype.base_dtype + lr_t = self._decayed_lr(var_dtype) + beta_1_t = self._get_hyper('beta_1', var_dtype) + beta_2_t = self._get_hyper('beta_2', var_dtype) + local_step = math_ops.cast(self.iterations + 1, var_dtype) + beta_1_power = math_ops.pow(beta_1_t, local_step) + beta_2_power = math_ops.pow(beta_2_t, local_step) + epsilon_t = self._get_hyper('epsilon', var_dtype) + lr = (lr_t * math_ops.sqrt(1 - beta_2_power) / (1 - beta_1_power)) + + # m_t = beta1 * m + (1 - beta1) * g_t + m = self.get_slot(var, 'm') + m_scaled_g_values = grad * (1 - beta_1_t) + m_t = state_ops.assign(m, m * beta_1_t, use_locking=self._use_locking) + with ops.control_dependencies([m_t]): + m_t = self._resource_scatter_add(m, indices, m_scaled_g_values) + + # v_t = beta2 * v + (1 - beta2) * (g_t * g_t) + v = self.get_slot(var, 'v') + v_scaled_g_values = (grad * grad) * (1 - beta_2_t) + v_t = state_ops.assign(v, v * beta_2_t, use_locking=self._use_locking) + with ops.control_dependencies([v_t]): + v_t = self._resource_scatter_add(v, indices, v_scaled_g_values) + + if not self._amsgrad: + v_sqrt = math_ops.sqrt(v_t) + var_update = state_ops.assign_sub( + var, lr * m_t / (v_sqrt + epsilon_t), use_locking=self._use_locking) + return control_flow_ops.group(*[var_update, m_t, v_t]) + else: + v_hat = self.get_slot(var, 'vhat') + v_hat_t = math_ops.maximum(v_hat, v_t) + with ops.control_dependencies([v_hat_t]): + v_hat_t = state_ops.assign( + v_hat, v_hat_t, use_locking=self._use_locking) + v_hat_sqrt = math_ops.sqrt(v_hat_t) + var_update = state_ops.assign_sub( + var, + lr * m_t / (v_hat_sqrt + epsilon_t), + use_locking=self._use_locking) + return control_flow_ops.group(*[var_update, m_t, v_t, v_hat_t]) + + def _resource_scatter_add(self, x, i, v): + with ops.control_dependencies( + [resource_variable_ops.resource_scatter_add(x.handle, i, v)]): + return x.value() def get_config(self): config = super(Adam, self).get_config() config.update({ 'learning_rate': self._serialize_hyperparameter('learning_rate'), + 'decay': self._serialize_hyperparameter('decay'), 'beta_1': self._serialize_hyperparameter('beta_1'), 'beta_2': self._serialize_hyperparameter('beta_2'), 'epsilon': self._serialize_hyperparameter('epsilon'), + 'amsgrad': self._amsgrad, }) return config diff --git a/tensorflow/python/keras/optimizer_v2/adam_test.py b/tensorflow/python/keras/optimizer_v2/adam_test.py new file mode 100644 index 00000000000..3bbafe12f8e --- /dev/null +++ b/tensorflow/python/keras/optimizer_v2/adam_test.py @@ -0,0 +1,508 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Tests for Adam.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import numpy as np + +from tensorflow.python.eager import context +from tensorflow.python.framework import constant_op +from tensorflow.python.framework import dtypes +from tensorflow.python.framework import ops +from tensorflow.python.framework import test_util +from tensorflow.python.keras import optimizers +from tensorflow.python.keras.optimizer_v2 import adam +from tensorflow.python.ops import array_ops +from tensorflow.python.ops import math_ops +from tensorflow.python.ops import resource_variable_ops +from tensorflow.python.ops import variables +from tensorflow.python.platform import test + + +def adam_update_numpy(param, + g_t, + t, + m, + v, + lr=0.001, + beta1=0.9, + beta2=0.999, + epsilon=1e-7): + lr_t = lr * np.sqrt(1 - beta2**(t + 1)) / (1 - beta1**(t + 1)) + + m_t = beta1 * m + (1 - beta1) * g_t + v_t = beta2 * v + (1 - beta2) * g_t * g_t + + param_t = param - lr_t * m_t / (np.sqrt(v_t) + epsilon) + return param_t, m_t, v_t + + +def adam_update_numpy_amsgrad(param, + g_t, + t, + m, + v, + vhat, + lr=0.001, + beta1=0.9, + beta2=0.999, + epsilon=1e-7): + lr_t = lr * np.sqrt(1 - beta2**(t + 1)) / (1 - beta1**(t + 1)) + + m_t = beta1 * m + (1 - beta1) * g_t + v_t = beta2 * v + (1 - beta2) * g_t * g_t + vhat_t = np.maximum(vhat, v_t) + + param_t = param - lr_t * m_t / (np.sqrt(vhat_t) + epsilon) + return param_t, m_t, v_t, vhat_t + + +def adam_sparse_update_numpy_amsgrad(param, + indices, + g_t, + t, + m, + v, + vhat, + lr=0.001, + beta1=0.9, + beta2=0.999, + epsilon=1e-7): + m_t, v_t, vhat_t, param_t = (np.copy(m), np.copy(v), np.copy(vhat), + np.copy(param)) + lr_t = lr * np.sqrt(1 - beta2**(t + 1)) / (1 - beta1**(t + 1)) + m_t_slice = beta1 * m[indices] + (1 - beta1) * g_t + v_t_slice = beta2 * v[indices] + (1 - beta2) * g_t * g_t + m_t[indices] = m_t_slice + v_t[indices] = v_t_slice + v_hat_t = np.maximum(vhat_t, v_t) + v_hat_t_slice = v_hat_t[indices] + param_t_slice = param[indices] - ( + lr_t * (m_t_slice / (np.sqrt(v_hat_t_slice) + epsilon))) + param_t[indices] = param_t_slice + return param_t, m_t, v_t, vhat_t + + +def get_beta_accumulators(opt, dtype): + local_step = math_ops.cast(opt.iterations + 1, dtype) + beta_1_t = math_ops.cast(opt._get_hyper("beta_1"), dtype) + beta_1_power = math_ops.pow(beta_1_t, local_step) + beta_2_t = math_ops.cast(opt._get_hyper("beta_2"), dtype) + beta_2_power = math_ops.pow(beta_2_t, local_step) + return (beta_1_power, beta_2_power) + + +class AdamOptimizerTest(test.TestCase): + + @test_util.run_deprecated_v1 + def testSparse(self): + for dtype in [dtypes.half, dtypes.float32, dtypes.float64]: + with self.cached_session(): + # Initialize variables for numpy implementation. + m0, v0, m1, v1 = 0.0, 0.0, 0.0, 0.0 + var0_np = np.array([1.0, 1.0, 2.0], dtype=dtype.as_numpy_dtype) + grads0_np = np.array([0.1, 0.0, 0.1], dtype=dtype.as_numpy_dtype) + var1_np = np.array([3.0, 3.0, 4.0], dtype=dtype.as_numpy_dtype) + grads1_np = np.array([0.01, 0.0, 0.01], dtype=dtype.as_numpy_dtype) + + var0 = resource_variable_ops.ResourceVariable(var0_np) + var1 = resource_variable_ops.ResourceVariable(var1_np) + grads0_np_indices = np.array([0, 2], dtype=np.int32) + grads0 = ops.IndexedSlices( + constant_op.constant(grads0_np[grads0_np_indices]), + constant_op.constant(grads0_np_indices), constant_op.constant([3])) + grads1_np_indices = np.array([0, 2], dtype=np.int32) + grads1 = ops.IndexedSlices( + constant_op.constant(grads1_np[grads1_np_indices]), + constant_op.constant(grads1_np_indices), constant_op.constant([3])) + opt = adam.Adam() + update = opt.apply_gradients(zip([grads0, grads1], [var0, var1])) + variables.global_variables_initializer().run() + + # Fetch params to validate initial values + self.assertAllClose([1.0, 1.0, 2.0], self.evaluate(var0)) + self.assertAllClose([3.0, 3.0, 4.0], self.evaluate(var1)) + + beta_1_power, beta_2_power = get_beta_accumulators(opt, dtype) + # Run 3 steps of Adam + for t in range(3): + self.assertAllCloseAccordingToType(0.9**(t + 1), + self.evaluate(beta_1_power)) + self.assertAllCloseAccordingToType(0.999**(t + 1), + self.evaluate(beta_2_power)) + update.run() + + var0_np, m0, v0 = adam_update_numpy(var0_np, grads0_np, t, m0, v0) + var1_np, m1, v1 = adam_update_numpy(var1_np, grads1_np, t, m1, v1) + + # Validate updated params + self.assertAllCloseAccordingToType(var0_np, self.evaluate(var0)) + self.assertAllCloseAccordingToType(var1_np, self.evaluate(var1)) + + @test_util.run_deprecated_v1 + def testSparseDevicePlacement(self): + for index_dtype in [dtypes.int32, dtypes.int64]: + with self.cached_session(force_gpu=test.is_gpu_available()): + # If a GPU is available, tests that all optimizer ops can be placed on + # it (i.e. they have GPU kernels). + var = variables.Variable([[1.0], [2.0]]) + indices = constant_op.constant([0, 1], dtype=index_dtype) + gathered_sum = math_ops.reduce_sum(array_ops.gather(var, indices)) + optimizer = adam.Adam(3.0) + minimize_op = optimizer.minimize(gathered_sum, var_list=[var]) + variables.global_variables_initializer().run() + minimize_op.run() + + @test_util.run_deprecated_v1 + def testSparseRepeatedIndices(self): + for dtype in [dtypes.half, dtypes.float32, dtypes.float64]: + with self.cached_session(): + repeated_index_update_var = variables.Variable( + [[1.0], [2.0]], dtype=dtype) + aggregated_update_var = variables.Variable( + [[1.0], [2.0]], dtype=dtype) + grad_repeated_index = ops.IndexedSlices( + constant_op.constant( + [0.1, 0.1], shape=[2, 1], dtype=dtype), + constant_op.constant([1, 1]), + constant_op.constant([2, 1])) + grad_aggregated = ops.IndexedSlices( + constant_op.constant( + [0.2], shape=[1, 1], dtype=dtype), + constant_op.constant([1]), + constant_op.constant([2, 1])) + repeated_update = adam.Adam().apply_gradients( + [(grad_repeated_index, repeated_index_update_var)]) + aggregated_update = adam.Adam().apply_gradients( + [(grad_aggregated, aggregated_update_var)]) + variables.global_variables_initializer().run() + self.assertAllClose(aggregated_update_var.eval(), + self.evaluate(repeated_index_update_var)) + for _ in range(3): + repeated_update.run() + aggregated_update.run() + self.assertAllClose(aggregated_update_var.eval(), + self.evaluate(repeated_index_update_var)) + + def doTestBasic(self, use_callable_params=False): + for i, dtype in enumerate([dtypes.half, dtypes.float32, dtypes.float64]): + with self.session(graph=ops.Graph()): + # Initialize variables for numpy implementation. + m0, v0, m1, v1 = 0.0, 0.0, 0.0, 0.0 + var0_np = np.array([1.0, 2.0], dtype=dtype.as_numpy_dtype) + grads0_np = np.array([0.1, 0.1], dtype=dtype.as_numpy_dtype) + var1_np = np.array([3.0, 4.0], dtype=dtype.as_numpy_dtype) + grads1_np = np.array([0.01, 0.01], dtype=dtype.as_numpy_dtype) + + var0 = resource_variable_ops.ResourceVariable( + var0_np, name="var0_%d" % i) + var1 = resource_variable_ops.ResourceVariable( + var1_np, name="var1_%d" % i) + grads0 = constant_op.constant(grads0_np) + grads1 = constant_op.constant(grads1_np) + + learning_rate = lambda: 0.001 + beta1 = lambda: 0.9 + beta2 = lambda: 0.999 + epsilon = lambda: 1e-8 + if not use_callable_params: + learning_rate = learning_rate() + beta1 = beta1() + beta2 = beta2() + epsilon = epsilon() + + opt = adam.Adam(learning_rate=learning_rate) + if not context.executing_eagerly(): + update = opt.apply_gradients(zip([grads0, grads1], [var0, var1])) + + self.evaluate(variables.global_variables_initializer()) + # Run 3 steps of Adam + for t in range(3): + beta_1_power, beta_2_power = get_beta_accumulators(opt, dtype) + self.assertAllCloseAccordingToType(0.9**(t + 1), + self.evaluate(beta_1_power)) + self.assertAllCloseAccordingToType(0.999**(t + 1), + self.evaluate(beta_2_power)) + if not context.executing_eagerly(): + self.evaluate(update) + else: + opt.apply_gradients(zip([grads0, grads1], [var0, var1])) + + var0_np, m0, v0 = adam_update_numpy(var0_np, grads0_np, t, m0, v0) + var1_np, m1, v1 = adam_update_numpy(var1_np, grads1_np, t, m1, v1) + + # Validate updated params + self.assertAllCloseAccordingToType(var0_np, self.evaluate(var0)) + self.assertAllCloseAccordingToType(var1_np, self.evaluate(var1)) + + @test_util.run_in_graph_and_eager_modes(reset_test=True) + def testResourceBasic(self): + self.doTestBasic() + + def testBasicCallableParams(self): + with context.eager_mode(): + self.doTestBasic(use_callable_params=True) + + @test_util.run_in_graph_and_eager_modes(reset_test=True) + def testBasicWithAmsgrad(self): + for i, dtype in enumerate([dtypes.half, dtypes.float32, dtypes.float64]): + with self.session(graph=ops.Graph()): + # Initialize variables for numpy implementation. + m0, v0, v0hat, m1, v1, v1hat = 0.0, 0.0, 0.0, 0.0, 0.0, 0.0 + var0_np = np.array([1.0, 2.0], dtype=dtype.as_numpy_dtype) + grads0_np = np.array([0.1, 0.1], dtype=dtype.as_numpy_dtype) + var1_np = np.array([3.0, 4.0], dtype=dtype.as_numpy_dtype) + grads1_np = np.array([0.01, 0.01], dtype=dtype.as_numpy_dtype) + + var0 = resource_variable_ops.ResourceVariable( + var0_np, name="var0_%d" % i) + var1 = resource_variable_ops.ResourceVariable( + var1_np, name="var1_%d" % i) + grads0 = constant_op.constant(grads0_np) + grads1 = constant_op.constant(grads1_np) + + opt = adam.Adam(amsgrad=True) + if not context.executing_eagerly(): + update = opt.apply_gradients(zip([grads0, grads1], [var0, var1])) + + self.evaluate(variables.global_variables_initializer()) + # Run 3 steps of Adam + for t in range(3): + beta_1_power, beta_2_power = get_beta_accumulators(opt, dtype) + self.assertAllCloseAccordingToType(0.9**(t + 1), + self.evaluate(beta_1_power)) + self.assertAllCloseAccordingToType(0.999**(t + 1), + self.evaluate(beta_2_power)) + if not context.executing_eagerly(): + self.evaluate(update) + else: + opt.apply_gradients(zip([grads0, grads1], [var0, var1])) + + var0_np, m0, v0, v0hat = adam_update_numpy_amsgrad( + var0_np, grads0_np, t, m0, v0, v0hat) + var1_np, m1, v1, v1hat = adam_update_numpy_amsgrad( + var1_np, grads1_np, t, m1, v1, v1hat) + + # Validate updated params + self.assertAllCloseAccordingToType(var0_np, self.evaluate(var0)) + self.assertAllCloseAccordingToType(var1_np, self.evaluate(var1)) + + @test_util.run_in_graph_and_eager_modes + def testSparseWithAmsgrad(self): + # dtypes.half does not work on gpu + eager. + for dtype in [dtypes.float32, dtypes.float64]: + with self.cached_session(): + m0 = np.array([[0.0], [0.0]]) + v0 = np.array([[0.0], [0.0]]) + v0hat = np.array([[0.0], [0.0]]) + indices_np = np.array([1]) + indices = constant_op.constant(indices_np, dtype=dtypes.int32) + var0_np = np.array([[1.0], [2.0]], dtype=dtype.as_numpy_dtype) + repeated_index_update_var = variables.Variable(var0_np, dtype=dtype) + aggregated_update_var = variables.Variable(var0_np, dtype=dtype) + grads0_np = np.array([[0.2]], dtype=dtype.as_numpy_dtype) + grad_repeated_index = ops.IndexedSlices( + constant_op.constant([0.1, 0.1], shape=[2, 1], dtype=dtype), + constant_op.constant([1, 1]), constant_op.constant([2, 1])) + grad_aggregated = ops.IndexedSlices(grads0_np, indices, + constant_op.constant([2, 1])) + opt_repeated = adam.Adam(amsgrad=True) + opt_aggregated = adam.Adam(amsgrad=True) + if not context.executing_eagerly(): + repeated_update = opt_repeated.apply_gradients( + [(grad_repeated_index, repeated_index_update_var)]) + aggregated_update = opt_aggregated.apply_gradients( + [(grad_aggregated, aggregated_update_var)]) + self.evaluate(variables.global_variables_initializer()) + self.assertAllClose( + self.evaluate(aggregated_update_var), + self.evaluate(repeated_index_update_var)) + for t in range(3): + if not context.executing_eagerly(): + self.evaluate(repeated_update) + self.evaluate(aggregated_update) + else: + opt_repeated.apply_gradients( + [(grad_repeated_index, repeated_index_update_var)]) + opt_aggregated.apply_gradients( + [(grad_aggregated, aggregated_update_var)]) + + var0_np, m0, v0, v0hat = adam_sparse_update_numpy_amsgrad( + var0_np, indices_np, grads0_np, t, m0, v0, v0hat) + + # Validate updated params + self.assertAllCloseAccordingToType( + var0_np, self.evaluate(aggregated_update_var)) + self.assertAllCloseAccordingToType( + self.evaluate(aggregated_update_var), + self.evaluate(repeated_index_update_var)) + + @test_util.run_deprecated_v1 + def testBasicWithLearningRateDecay(self): + for i, dtype in enumerate([dtypes.half, dtypes.float32, dtypes.float64]): + with self.session(graph=ops.Graph()): + # Initialize variables for numpy implementation. + m0, v0, m1, v1 = 0.0, 0.0, 0.0, 0.0 + var0_np = np.array([1.0, 2.0], dtype=dtype.as_numpy_dtype) + grads0_np = np.array([0.1, 0.1], dtype=dtype.as_numpy_dtype) + var1_np = np.array([3.0, 4.0], dtype=dtype.as_numpy_dtype) + grads1_np = np.array([0.01, 0.01], dtype=dtype.as_numpy_dtype) + + var0 = resource_variable_ops.ResourceVariable( + var0_np, name="var0_%d" % i) + var1 = resource_variable_ops.ResourceVariable( + var1_np, name="var1_%d" % i) + grads0 = constant_op.constant(grads0_np) + grads1 = constant_op.constant(grads1_np) + + learning_rate = 0.001 + beta_1 = 0.9 + beta_2 = 0.999 + epsilon = 1e-7 + decay = 0.5 + + opt = adam.Adam( + learning_rate=learning_rate, + beta_1=beta_1, + beta_2=beta_2, + epsilon=epsilon, + decay=decay) + update = opt.apply_gradients(zip([grads0, grads1], [var0, var1])) + + self.evaluate(variables.global_variables_initializer()) + # Run 3 steps of Adam + for t in range(3): + self.evaluate(update) + lr_np = learning_rate / (1 + decay * t) + + var0_np, m0, v0 = adam_update_numpy( + var0_np, grads0_np, t, m0, v0, lr=lr_np) + var1_np, m1, v1 = adam_update_numpy( + var1_np, grads1_np, t, m1, v1, lr=lr_np) + + # Validate updated params + self.assertAllCloseAccordingToType(var0_np, self.evaluate(var0)) + self.assertAllCloseAccordingToType(var1_np, self.evaluate(var1)) + + @test_util.run_deprecated_v1 + def testTensorLearningRate(self): + for dtype in [dtypes.half, dtypes.float32, dtypes.float64]: + with self.cached_session(): + # Initialize variables for numpy implementation. + m0, v0, m1, v1 = 0.0, 0.0, 0.0, 0.0 + var0_np = np.array([1.0, 2.0], dtype=dtype.as_numpy_dtype) + grads0_np = np.array([0.1, 0.1], dtype=dtype.as_numpy_dtype) + var1_np = np.array([3.0, 4.0], dtype=dtype.as_numpy_dtype) + grads1_np = np.array([0.01, 0.01], dtype=dtype.as_numpy_dtype) + + var0 = variables.Variable(var0_np) + var1 = variables.Variable(var1_np) + grads0 = constant_op.constant(grads0_np) + grads1 = constant_op.constant(grads1_np) + opt = adam.Adam(constant_op.constant(0.001)) + update = opt.apply_gradients(zip([grads0, grads1], [var0, var1])) + variables.global_variables_initializer().run() + + # Fetch params to validate initial values + self.assertAllClose([1.0, 2.0], self.evaluate(var0)) + self.assertAllClose([3.0, 4.0], self.evaluate(var1)) + + beta_1_power, beta_2_power = get_beta_accumulators(opt, dtype) + # Run 3 steps of Adam + for t in range(3): + self.assertAllCloseAccordingToType(0.9**(t + 1), + self.evaluate(beta_1_power)) + self.assertAllCloseAccordingToType(0.999**(t + 1), + self.evaluate(beta_2_power)) + update.run() + + var0_np, m0, v0 = adam_update_numpy(var0_np, grads0_np, t, m0, v0) + var1_np, m1, v1 = adam_update_numpy(var1_np, grads1_np, t, m1, v1) + + # Validate updated params + self.assertAllCloseAccordingToType(var0_np, self.evaluate(var0)) + self.assertAllCloseAccordingToType(var1_np, self.evaluate(var1)) + + @test_util.run_deprecated_v1 + def testSharing(self): + for dtype in [dtypes.half, dtypes.float32, dtypes.float64]: + with self.cached_session(): + # Initialize variables for numpy implementation. + m0, v0, m1, v1 = 0.0, 0.0, 0.0, 0.0 + var0_np = np.array([1.0, 2.0], dtype=dtype.as_numpy_dtype) + grads0_np = np.array([0.1, 0.1], dtype=dtype.as_numpy_dtype) + var1_np = np.array([3.0, 4.0], dtype=dtype.as_numpy_dtype) + grads1_np = np.array([0.01, 0.01], dtype=dtype.as_numpy_dtype) + + var0 = variables.Variable(var0_np) + var1 = variables.Variable(var1_np) + grads0 = constant_op.constant(grads0_np) + grads1 = constant_op.constant(grads1_np) + opt = adam.Adam() + update1 = opt.apply_gradients(zip([grads0, grads1], [var0, var1])) + update2 = opt.apply_gradients(zip([grads0, grads1], [var0, var1])) + variables.global_variables_initializer().run() + + beta_1_power, beta_2_power = get_beta_accumulators(opt, dtype) + + # Fetch params to validate initial values + self.assertAllClose([1.0, 2.0], self.evaluate(var0)) + self.assertAllClose([3.0, 4.0], self.evaluate(var1)) + + # Run 3 steps of intertwined Adam1 and Adam2. + for t in range(3): + self.assertAllCloseAccordingToType(0.9**(t + 1), + self.evaluate(beta_1_power)) + self.assertAllCloseAccordingToType(0.999**(t + 1), + self.evaluate(beta_2_power)) + if t % 2 == 0: + update1.run() + else: + update2.run() + + var0_np, m0, v0 = adam_update_numpy(var0_np, grads0_np, t, m0, v0) + var1_np, m1, v1 = adam_update_numpy(var1_np, grads1_np, t, m1, v1) + + # Validate updated params + self.assertAllCloseAccordingToType(var0_np, self.evaluate(var0)) + self.assertAllCloseAccordingToType(var1_np, self.evaluate(var1)) + + def testSlotsUniqueEager(self): + with context.eager_mode(): + v1 = resource_variable_ops.ResourceVariable(1.) + v2 = resource_variable_ops.ResourceVariable(1.) + opt = adam.Adam(1.) + opt.minimize(lambda: v1 + v2, var_list=[v1, v2]) + # There should be iteration, and two unique slot variables for v1 and v2. + self.assertEqual(5, len(set(opt.variables()))) + self.assertEqual( + self.evaluate(opt.variables()[0]), self.evaluate(opt.iterations)) + + def testSetWeightsFromV1AdamWithoutMinimize(self): + keras_v1_adam = optimizers.Adam() + keras_v2_adam = adam.Adam() + keras_v2_adam.set_weights(keras_v1_adam.get_weights()) + keras_v1_iteration = keras_v1_adam.iterations + keras_v2_iteration = keras_v2_adam.iterations + self.evaluate(variables.global_variables_initializer()) + self.assertEqual( + self.evaluate(keras_v1_iteration), self.evaluate(keras_v2_iteration)) + + +if __name__ == "__main__": + test.main() diff --git a/tensorflow/python/keras/optimizer_v2/adamax.py b/tensorflow/python/keras/optimizer_v2/adamax.py new file mode 100644 index 00000000000..ddd78584f85 --- /dev/null +++ b/tensorflow/python/keras/optimizer_v2/adamax.py @@ -0,0 +1,159 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +"""Adamax for TensorFlow.""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from tensorflow.python.framework import ops +from tensorflow.python.keras.optimizer_v2 import adam +from tensorflow.python.ops import array_ops +from tensorflow.python.ops import control_flow_ops +from tensorflow.python.ops import math_ops +from tensorflow.python.ops import resource_variable_ops +from tensorflow.python.training import training_ops + + +class Adamax(adam.Adam): + """Optimizer that implements the Adamax algorithm. + + It is a variant of Adam based on the infinity norm. + Default parameters follow those provided in the paper. + Adamax is sometimes superior to adam, specially in models with embeddings. + + References + see Section 7 of [Kingma et al., 2014](http://arxiv.org/abs/1412.6980) + ([pdf](http://arxiv.org/pdf/1412.6980.pdf)). + """ + + def __init__(self, + learning_rate=0.001, + beta_1=0.9, + beta_2=0.999, + epsilon=1e-7, + name='Adamax', + **kwargs): + """Construct a new Adamax optimizer. + + Initialization: + + ``` + m_0 <- 0 (Initialize initial 1st moment vector) + v_0 <- 0 (Initialize the exponentially weighted infinity norm) + t <- 0 (Initialize timestep) + ``` + + The update rule for `variable` with gradient `g` uses an optimization + described at the end of section 7.1 of the paper: + + ``` + t <- t + 1 + + m_t <- beta1 * m_{t-1} + (1 - beta1) * g + v_t <- max(beta2 * v_{t-1}, abs(g)) + variable <- variable - learning_rate / (1 - beta1^t) * m_t / (v_t + epsilon) + ``` + + Similar to AdamOptimizer, the epsilon is added for numerical stability + (especially to get rid of division by zero when v_t = 0). + + Contrast to AdamOptimizer, the sparse implementation of this algorithm + (used when the gradient is an IndexedSlices object, typically because of + `tf.gather` or an embedding lookup in the forward pass) only updates + variable slices and corresponding `m_t`, `v_t` terms when that part of + the variable was used in the forward pass. This means that the sparse + behavior is contrast to the dense behavior (similar to some momentum + implementations which ignore momentum unless a variable slice was actually + used). + + Args: + learning_rate: A Tensor or a floating point value. The learning rate. + beta_1: A float value or a constant float tensor. The exponential decay + rate for the 1st moment estimates. + beta_2: A float value or a constant float tensor. The exponential decay + rate for the exponentially weighted infinity norm. + epsilon: A small constant for numerical stability. + name: Optional name for the operations created when applying gradients. + Defaults to "Adamax". + **kwargs: keyword arguments. Allowed to be {`decay`} + """ + # pylint: disable=useless-super-delegation + super(Adamax, self).__init__( + learning_rate=learning_rate, + beta_1=beta_1, + beta_2=beta_2, + epsilon=epsilon, + amsgrad=False, + name=name, + **kwargs) + # pylint: enable=useless-super-delegation + + def _resource_apply_dense(self, grad, var): + var_dtype = var.dtype.base_dtype + lr_t = self._decayed_lr(var_dtype) + m = self.get_slot(var, 'm') + v = self.get_slot(var, 'v') + beta_1_t = self._get_hyper('beta_1', var_dtype) + beta_2_t = self._get_hyper('beta_2', var_dtype) + local_step = math_ops.cast(self.iterations + 1, var_dtype) + beta_1_power = math_ops.pow(beta_1_t, local_step) + return training_ops.resource_apply_ada_max( + var.handle, + m.handle, + v.handle, + beta_1_power, + lr_t, + beta_1_t, + beta_2_t, + self._get_hyper('epsilon', var_dtype), + grad, + use_locking=self._use_locking) + + def _resource_apply_sparse(self, grad, var, indices): + var_dtype = var.dtype.base_dtype + lr_t = self._decayed_lr(var_dtype) + + beta_1_t = self._get_hyper('beta_1', var_dtype) + beta_2_t = self._get_hyper('beta_2', var_dtype) + local_step = math_ops.cast(self.iterations + 1, var_dtype) + beta_1_power = math_ops.pow(beta_1_t, local_step) + epsilon_t = self._get_hyper('epsilon', var_dtype) + + # m_t = beta1 * m + (1 - beta1) * g_t + m = self.get_slot(var, 'm') + m_slice = array_ops.gather(m, indices) + m_t_slice = m_slice * beta_1_t + grad * (1 - beta_1_t) + with ops.control_dependencies([m_t_slice]): + m_t = self._resource_scatter_update(m, indices, m_t_slice) + + # u_t = max(beta2 * u, abs(g_t)) + v = self.get_slot(var, 'v') + v_slice = array_ops.gather(v, indices) + v_t_slice = math_ops.maximum(v_slice * beta_2_t, math_ops.abs(grad)) + with ops.control_dependencies([v_t_slice]): + v_t = self._resource_scatter_update(v, indices, v_t_slice) + # theta_t = theta - lr / (1 - beta1^t) * m_t / u_t + var_slice = -lr_t / (1 - beta_1_power) * ( + m_t_slice / (v_t_slice + epsilon_t)) + with ops.control_dependencies([var_slice]): + var_update = self._resource_scatter_add(var, indices, var_slice) + return control_flow_ops.group(*[var_update, m_t, v_t]) + + def _resource_scatter_update(self, x, i, v): + with ops.control_dependencies( + [resource_variable_ops.resource_scatter_update( + x.handle, i, v)]): + return x.value() diff --git a/tensorflow/python/keras/optimizer_v2/adamax_test.py b/tensorflow/python/keras/optimizer_v2/adamax_test.py new file mode 100644 index 00000000000..baf131fbb0c --- /dev/null +++ b/tensorflow/python/keras/optimizer_v2/adamax_test.py @@ -0,0 +1,367 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Tests for Adamax.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import numpy as np + +from tensorflow.python.eager import context +from tensorflow.python.framework import constant_op +from tensorflow.python.framework import dtypes +from tensorflow.python.framework import ops +from tensorflow.python.framework import test_util +from tensorflow.python.keras.optimizer_v2 import adamax +from tensorflow.python.ops import array_ops +from tensorflow.python.ops import math_ops +from tensorflow.python.ops import resource_variable_ops +from tensorflow.python.ops import variables +from tensorflow.python.platform import test + + +def adamax_update_numpy(param, + g_t, + t, + m, + v, + alpha=0.001, + beta1=0.9, + beta2=0.999, + epsilon=1e-8): + m_t = beta1 * m + (1 - beta1) * g_t + v_t = np.maximum(beta2 * v, np.abs(g_t)) + param_t = param - (alpha / (1 - beta1**(t + 1))) * (m_t / (v_t + epsilon)) + return param_t, m_t, v_t + + +def adamax_sparse_update_numpy(param, + indices, + g_t, + t, + m, + v, + alpha=0.001, + beta1=0.9, + beta2=0.999, + epsilon=1e-8): + m_t, v_t, param_t = np.copy(m), np.copy(v), np.copy(param) + m_t_slice = beta1 * m[indices] + (1 - beta1) * g_t + v_t_slice = np.maximum(beta2 * v[indices], np.abs(g_t)) + param_t_slice = param[indices] - ( + (alpha / (1 - beta1**(t + 1))) * (m_t_slice / (v_t_slice + epsilon))) + m_t[indices] = m_t_slice + v_t[indices] = v_t_slice + param_t[indices] = param_t_slice + return param_t, m_t, v_t + + +def get_beta_accumulators(opt, dtype): + local_step = math_ops.cast(opt.iterations + 1, dtype) + beta_1_t = math_ops.cast(opt._get_hyper("beta_1"), dtype) + beta_1_power = math_ops.pow(beta_1_t, local_step) + return beta_1_power + + +class AdamaxOptimizerTest(test.TestCase): + + def doTestSparse(self, use_resource=False): + for dtype in [dtypes.half, dtypes.float32, dtypes.float64]: + with self.cached_session(): + # Initialize variables for numpy implementation. + zero_slots = lambda: np.zeros((3), dtype=dtype.as_numpy_dtype) # pylint: disable=cell-var-from-loop + m0, v0, m1, v1 = zero_slots(), zero_slots(), zero_slots(), zero_slots() + var0_np = np.array([1.0, 2.0, 3.0], dtype=dtype.as_numpy_dtype) + grads0_np = np.array([0.1, 0.1], dtype=dtype.as_numpy_dtype) + var1_np = np.array([4.0, 5.0, 6.0], dtype=dtype.as_numpy_dtype) + grads1_np = np.array([0.01, 0.01], dtype=dtype.as_numpy_dtype) + + var0 = resource_variable_ops.ResourceVariable(var0_np) + var1 = resource_variable_ops.ResourceVariable(var1_np) + + grads0_np_indices = np.array([0, 1], dtype=np.int32) + grads0 = ops.IndexedSlices( + constant_op.constant(grads0_np), + constant_op.constant(grads0_np_indices), constant_op.constant([3])) + grads1_np_indices = np.array([2, 1], dtype=np.int32) + grads1 = ops.IndexedSlices( + constant_op.constant(grads1_np), + constant_op.constant(grads1_np_indices), constant_op.constant([3])) + opt = adamax.Adamax() + update = opt.apply_gradients(zip([grads0, grads1], [var0, var1])) + variables.global_variables_initializer().run() + + # Fetch params to validate initial values + self.assertAllClose([1.0, 2.0, 3.0], var0.eval()) + self.assertAllClose([4.0, 5.0, 6.0], var1.eval()) + + beta1_power = get_beta_accumulators(opt, dtype) + + # Run 3 steps of Adamax + for t in range(3): + self.assertAllCloseAccordingToType(0.9**(t + 1), beta1_power.eval()) + update.run() + + var0_np, m0, v0 = adamax_sparse_update_numpy( + var0_np, grads0_np_indices, grads0_np, t, m0, v0) + var1_np, m1, v1 = adamax_sparse_update_numpy( + var1_np, grads1_np_indices, grads1_np, t, m1, v1) + + # Validate updated params + self.assertAllCloseAccordingToType(var0_np, var0.eval()) + self.assertAllCloseAccordingToType(var1_np, var1.eval()) + + @test_util.run_deprecated_v1 + def testResourceSparse(self): + self.doTestSparse(use_resource=True) + + @test_util.run_deprecated_v1 + def testSparseDevicePlacement(self): + for index_dtype in [dtypes.int32, dtypes.int64]: + with self.cached_session(force_gpu=test.is_gpu_available()): + # If a GPU is available, tests that all optimizer ops can be placed on + # it (i.e. they have GPU kernels). + var = variables.Variable([[1.0], [2.0]]) + indices = constant_op.constant([0, 1], dtype=index_dtype) + gathered_sum = math_ops.reduce_sum(array_ops.gather(var, indices)) + optimizer = adamax.Adamax(3.0) + minimize_op = optimizer.minimize(gathered_sum, var_list=[var]) + variables.global_variables_initializer().run() + minimize_op.run() + + @test_util.run_deprecated_v1 + def testSparseRepeatedIndices(self): + for dtype in [dtypes.half, dtypes.float32, dtypes.float64]: + with self.cached_session(): + repeated_index_update_var = variables.Variable( + [[1.0], [2.0]], dtype=dtype) + aggregated_update_var = variables.Variable( + [[1.0], [2.0]], dtype=dtype) + grad_repeated_index = ops.IndexedSlices( + constant_op.constant( + [0.1, 0.1], shape=[2, 1], dtype=dtype), + constant_op.constant([1, 1]), + constant_op.constant([2, 1])) + grad_aggregated = ops.IndexedSlices( + constant_op.constant( + [0.2], shape=[1, 1], dtype=dtype), + constant_op.constant([1]), + constant_op.constant([2, 1])) + repeated_update = adamax.Adamax().apply_gradients( + [(grad_repeated_index, repeated_index_update_var)]) + aggregated_update = adamax.Adamax().apply_gradients( + [(grad_aggregated, aggregated_update_var)]) + variables.global_variables_initializer().run() + self.assertAllClose(aggregated_update_var.eval(), + repeated_index_update_var.eval()) + for _ in range(3): + repeated_update.run() + aggregated_update.run() + self.assertAllClose(aggregated_update_var.eval(), + repeated_index_update_var.eval()) + + @test_util.run_in_graph_and_eager_modes(reset_test=True) + def testBasic(self): + for i, dtype in enumerate([dtypes.half, dtypes.float32, dtypes.float64]): + with self.session(graph=ops.Graph()): + # Initialize variables for numpy implementation. + m0, v0, m1, v1 = 0.0, 0.0, 0.0, 0.0 + var0_np = np.array([1.0, 2.0], dtype=dtype.as_numpy_dtype) + grads0_np = np.array([0.1, 0.1], dtype=dtype.as_numpy_dtype) + var1_np = np.array([3.0, 4.0], dtype=dtype.as_numpy_dtype) + grads1_np = np.array([0.01, 0.01], dtype=dtype.as_numpy_dtype) + + var0 = resource_variable_ops.ResourceVariable( + var0_np, name="var0_%d" % i) + var1 = resource_variable_ops.ResourceVariable( + var1_np, name="var1_%d" % i) + + grads0 = constant_op.constant(grads0_np) + grads1 = constant_op.constant(grads1_np) + + opt = adamax.Adamax() + if not context.executing_eagerly(): + update = opt.apply_gradients(zip([grads0, grads1], [var0, var1])) + + if not context.executing_eagerly(): + self.evaluate(variables.global_variables_initializer()) + # Fetch params to validate initial values + self.assertAllClose([1.0, 2.0], self.evaluate(var0)) + self.assertAllClose([3.0, 4.0], self.evaluate(var1)) + + # Run 3 steps of Adamax + for t in range(3): + beta_1_power = get_beta_accumulators(opt, dtype) + self.assertAllCloseAccordingToType(0.9**(t + 1), + self.evaluate(beta_1_power)) + if not context.executing_eagerly(): + self.evaluate(update) + else: + opt.apply_gradients(zip([grads0, grads1], [var0, var1])) + + var0_np, m0, v0 = adamax_update_numpy(var0_np, grads0_np, t, m0, v0) + var1_np, m1, v1 = adamax_update_numpy(var1_np, grads1_np, t, m1, v1) + + # Validate updated params + self.assertAllCloseAccordingToType( + var0_np, self.evaluate(var0), rtol=1e-2) + self.assertAllCloseAccordingToType( + var1_np, self.evaluate(var1), rtol=1e-2) + + @test_util.run_in_graph_and_eager_modes(reset_test=True) + def testBasicWithLearningRateDecay(self): + for i, dtype in enumerate([dtypes.half, dtypes.float32, dtypes.float64]): + with self.session(graph=ops.Graph()): + # Initialize variables for numpy implementation. + m0, v0, m1, v1 = 0.0, 0.0, 0.0, 0.0 + var0_np = np.array([1.0, 2.0], dtype=dtype.as_numpy_dtype) + grads0_np = np.array([0.1, 0.1], dtype=dtype.as_numpy_dtype) + var1_np = np.array([3.0, 4.0], dtype=dtype.as_numpy_dtype) + grads1_np = np.array([0.01, 0.01], dtype=dtype.as_numpy_dtype) + + var0 = resource_variable_ops.ResourceVariable( + var0_np, name="var0_%d" % i) + var1 = resource_variable_ops.ResourceVariable( + var1_np, name="var1_%d" % i) + + grads0 = constant_op.constant(grads0_np) + grads1 = constant_op.constant(grads1_np) + + learning_rate = 0.001 + decay = 0.002 + opt = adamax.Adamax(learning_rate=learning_rate, decay=decay) + if not context.executing_eagerly(): + update = opt.apply_gradients(zip([grads0, grads1], [var0, var1])) + + if not context.executing_eagerly(): + self.evaluate(variables.global_variables_initializer()) + # Fetch params to validate initial values + self.assertAllClose([1.0, 2.0], self.evaluate(var0)) + self.assertAllClose([3.0, 4.0], self.evaluate(var1)) + + # Run 3 steps of Adamax + for t in range(3): + beta_1_power = get_beta_accumulators(opt, dtype) + self.assertAllCloseAccordingToType(0.9**(t + 1), + self.evaluate(beta_1_power)) + if not context.executing_eagerly(): + self.evaluate(update) + else: + opt.apply_gradients(zip([grads0, grads1], [var0, var1])) + + lr = learning_rate / (1 + decay * t) + + var0_np, m0, v0 = adamax_update_numpy( + var0_np, grads0_np, t, m0, v0, alpha=lr) + var1_np, m1, v1 = adamax_update_numpy( + var1_np, grads1_np, t, m1, v1, alpha=lr) + + # Validate updated params + self.assertAllCloseAccordingToType(var0_np, self.evaluate(var0), + rtol=1e-2) + self.assertAllCloseAccordingToType(var1_np, self.evaluate(var1), + rtol=1e-2) + + @test_util.run_deprecated_v1 + def testTensorLearningRate(self): + for dtype in [dtypes.half, dtypes.float32, dtypes.float64]: + with self.cached_session(): + # Initialize variables for numpy implementation. + m0, v0, m1, v1 = 0.0, 0.0, 0.0, 0.0 + var0_np = np.array([1.0, 2.0], dtype=dtype.as_numpy_dtype) + grads0_np = np.array([0.1, 0.1], dtype=dtype.as_numpy_dtype) + var1_np = np.array([3.0, 4.0], dtype=dtype.as_numpy_dtype) + grads1_np = np.array([0.01, 0.01], dtype=dtype.as_numpy_dtype) + + var0 = variables.Variable(var0_np) + var1 = variables.Variable(var1_np) + grads0 = constant_op.constant(grads0_np) + grads1 = constant_op.constant(grads1_np) + opt = adamax.Adamax(constant_op.constant(0.001)) + update = opt.apply_gradients(zip([grads0, grads1], [var0, var1])) + variables.global_variables_initializer().run() + + # Fetch params to validate initial values + self.assertAllClose([1.0, 2.0], var0.eval()) + self.assertAllClose([3.0, 4.0], var1.eval()) + + beta1_power = get_beta_accumulators(opt, dtype) + + # Run 3 steps of Adamax + for t in range(3): + self.assertAllCloseAccordingToType(0.9**(t + 1), beta1_power.eval()) + update.run() + + var0_np, m0, v0 = adamax_update_numpy(var0_np, grads0_np, t, m0, v0) + var1_np, m1, v1 = adamax_update_numpy(var1_np, grads1_np, t, m1, v1) + + # Validate updated params + self.assertAllCloseAccordingToType(var0_np, var0.eval()) + self.assertAllCloseAccordingToType(var1_np, var1.eval()) + + @test_util.run_deprecated_v1 + def testSharing(self): + for dtype in [dtypes.half, dtypes.float32, dtypes.float64]: + with self.cached_session(): + # Initialize variables for numpy implementation. + m0, v0, m1, v1 = 0.0, 0.0, 0.0, 0.0 + var0_np = np.array([1.0, 2.0], dtype=dtype.as_numpy_dtype) + grads0_np = np.array([0.1, 0.1], dtype=dtype.as_numpy_dtype) + var1_np = np.array([3.0, 4.0], dtype=dtype.as_numpy_dtype) + grads1_np = np.array([0.01, 0.01], dtype=dtype.as_numpy_dtype) + + var0 = variables.Variable(var0_np) + var1 = variables.Variable(var1_np) + grads0 = constant_op.constant(grads0_np) + grads1 = constant_op.constant(grads1_np) + opt = adamax.Adamax() + update1 = opt.apply_gradients(zip([grads0, grads1], [var0, var1])) + update2 = opt.apply_gradients(zip([grads0, grads1], [var0, var1])) + variables.global_variables_initializer().run() + + beta1_power = get_beta_accumulators(opt, dtype) + + # Fetch params to validate initial values + self.assertAllClose([1.0, 2.0], var0.eval()) + self.assertAllClose([3.0, 4.0], var1.eval()) + + # Run 3 steps of intertwined Adamax1 and Adamax2. + for t in range(3): + self.assertAllCloseAccordingToType(0.9**(t + 1), beta1_power.eval()) + if t % 2 == 0: + update1.run() + else: + update2.run() + + var0_np, m0, v0 = adamax_update_numpy(var0_np, grads0_np, t, m0, v0) + var1_np, m1, v1 = adamax_update_numpy(var1_np, grads1_np, t, m1, v1) + + # Validate updated params + self.assertAllCloseAccordingToType(var0_np, var0.eval()) + self.assertAllCloseAccordingToType(var1_np, var1.eval()) + + def testSlotsUniqueEager(self): + with context.eager_mode(): + v1 = resource_variable_ops.ResourceVariable(1.) + v2 = resource_variable_ops.ResourceVariable(1.) + opt = adamax.Adamax(1.) + opt.minimize(lambda: v1 + v2, var_list=[v1, v2]) + # There should be iteration, and two unique slot variables for v1 and v2. + self.assertEqual(5, len(set(opt.variables()))) + + +if __name__ == "__main__": + test.main() diff --git a/tensorflow/python/keras/optimizer_v2/ftrl.py b/tensorflow/python/keras/optimizer_v2/ftrl.py new file mode 100644 index 00000000000..e278e352f55 --- /dev/null +++ b/tensorflow/python/keras/optimizer_v2/ftrl.py @@ -0,0 +1,210 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Ftrl-proximal for TensorFlow.""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from tensorflow.python.keras.optimizer_v2 import optimizer_v2 +from tensorflow.python.ops import init_ops +from tensorflow.python.ops import math_ops +from tensorflow.python.training import training_ops + + +class Ftrl(optimizer_v2.OptimizerV2): + """Optimizer that implements the FTRL algorithm. + + See this [paper]( + https://www.eecs.tufts.edu/~dsculley/papers/ad-click-prediction.pdf). + This version has support for both online L2 (the L2 penalty given in the paper + above) and shrinkage-type L2 (which is the addition of an L2 penalty to the + loss function). + """ + + def __init__(self, + learning_rate, + learning_rate_power=-0.5, + initial_accumulator_value=0.1, + l1_regularization_strength=0.0, + l2_regularization_strength=0.0, + name='Ftrl', + l2_shrinkage_regularization_strength=0.0, + **kwargs): + r"""Construct a new FTRL optimizer. + + Args: + learning_rate: A float value or a constant float `Tensor`. + learning_rate_power: A float value, must be less or equal to zero. + Controls how the learning rate decreases during training. Use zero for + a fixed learning rate. + initial_accumulator_value: The starting value for accumulators. + Only zero or positive values are allowed. + l1_regularization_strength: A float value, must be greater than or + equal to zero. + l2_regularization_strength: A float value, must be greater than or + equal to zero. + name: Optional name prefix for the operations created when applying + gradients. Defaults to "Ftrl". + l2_shrinkage_regularization_strength: A float value, must be greater than + or equal to zero. This differs from L2 above in that the L2 above is a + stabilization penalty, whereas this L2 shrinkage is a magnitude penalty. + The FTRL formulation can be written as: + w_{t+1} = argmin_w(\hat{g}_{1:t}w + L1*||w||_1 + L2*||w||_2^2), where + \hat{g} = g + (2*L2_shrinkage*w), and g is the gradient of the loss + function w.r.t. the weights w. + Specifically, in the absence of L1 regularization, it is equivalent to + the following update rule: + w_{t+1} = w_t - lr_t / (1 + 2*L2*lr_t) * g_t - + 2*L2_shrinkage*lr_t / (1 + 2*L2*lr_t) * w_t + where lr_t is the learning rate at t. + When input is sparse shrinkage will only happen on the active weights.\ + **kwargs: keyword arguments. Allowed to be {`decay`} + + Raises: + ValueError: If one of the arguments is invalid. + + References + See [paper] + (https://www.eecs.tufts.edu/~dsculley/papers/ad-click-prediction.pdf) + """ + super(Ftrl, self).__init__(name, **kwargs) + + if initial_accumulator_value < 0.0: + raise ValueError( + 'initial_accumulator_value %f needs to be positive or zero' % + initial_accumulator_value) + if learning_rate_power > 0.0: + raise ValueError('learning_rate_power %f needs to be negative or zero' % + learning_rate_power) + if l1_regularization_strength < 0.0: + raise ValueError( + 'l1_regularization_strength %f needs to be positive or zero' % + l1_regularization_strength) + if l2_regularization_strength < 0.0: + raise ValueError( + 'l2_regularization_strength %f needs to be positive or zero' % + l2_regularization_strength) + if l2_shrinkage_regularization_strength < 0.0: + raise ValueError( + 'l2_shrinkage_regularization_strength %f needs to be positive' + ' or zero' % l2_shrinkage_regularization_strength) + + self._set_hyper('learning_rate', learning_rate) + self._set_hyper('decay', self._initial_decay) + self._set_hyper('learning_rate_power', learning_rate_power) + self._set_hyper('l1_regularization_strength', l1_regularization_strength) + self._set_hyper('l2_regularization_strength', l2_regularization_strength) + self._initial_accumulator_value = initial_accumulator_value + self._l2_shrinkage_regularization_strength = ( + l2_shrinkage_regularization_strength) + + def _create_slots(self, var_list): + # Create the "accum" and "linear" slots. + for var in var_list: + dtype = var.dtype.base_dtype + init = init_ops.constant_initializer( + self._initial_accumulator_value, dtype=dtype) + self.add_slot(var, 'accumulator', init) + self.add_slot(var, 'linear') + + def _resource_apply_dense(self, grad, var): + var_dtype = var.dtype.base_dtype + lr_t = self._decayed_lr(var_dtype) + learning_rate_power = self._get_hyper('learning_rate_power', var_dtype) + l1_regularization_strength = self._get_hyper('l1_regularization_strength', + var_dtype) + l2_regularization_strength = self._get_hyper('l2_regularization_strength', + var_dtype) + accum = self.get_slot(var, 'accumulator') + linear = self.get_slot(var, 'linear') + if self._l2_shrinkage_regularization_strength <= 0.0: + return training_ops.resource_apply_ftrl( + var.handle, + accum.handle, + linear.handle, + grad, + lr_t, + l1_regularization_strength, + l2_regularization_strength, + learning_rate_power, + use_locking=self._use_locking) + else: + return training_ops.resource_apply_ftrl_v2( + var.handle, + accum.handle, + linear.handle, + grad, + lr_t, + l1_regularization_strength, + l2_regularization_strength, + math_ops.cast(self._l2_shrinkage_regularization_strength, var_dtype), + learning_rate_power, + use_locking=self._use_locking) + + def _resource_apply_sparse(self, grad, var, indices): + var_dtype = var.dtype.base_dtype + lr_t = self._decayed_lr(var_dtype) + learning_rate_power = self._get_hyper('learning_rate_power', var_dtype) + l1_regularization_strength = self._get_hyper('l1_regularization_strength', + var_dtype) + l2_regularization_strength = self._get_hyper('l2_regularization_strength', + var_dtype) + accum = self.get_slot(var, 'accumulator') + linear = self.get_slot(var, 'linear') + if self._l2_shrinkage_regularization_strength <= 0.0: + return training_ops.resource_sparse_apply_ftrl( + var.handle, + accum.handle, + linear.handle, + grad, + indices, + lr_t, + l1_regularization_strength, + l2_regularization_strength, + learning_rate_power, + use_locking=self._use_locking) + else: + return training_ops.resource_sparse_apply_ftrl_v2( + var.handle, + accum.handle, + linear.handle, + grad, + indices, + lr_t, + l1_regularization_strength, + l2_regularization_strength, + math_ops.cast(self._l2_shrinkage_regularization_strength, var_dtype), + learning_rate_power, + use_locking=self._use_locking) + + def get_config(self): + config = super(Ftrl, self).get_config() + config.update({ + 'learning_rate': + self._serialize_hyperparameter('learning_rate'), + 'decay': + self._serialize_hyperparameter('decay'), + 'initial_accumulator_value': + self._initial_accumulator_value, + 'learning_rate_power': + self._serialize_hyperparameter('learning_rate_power'), + 'l1_regularization_strength': + self._serializer_hyperparameter('l1_regularization_strength'), + 'l2_regularization_strength': + self._serializer_hyperparameter('l2_regularization_strength'), + 'l2_shrinkage_regularization_strength': + self._l2_shrinkage_regularization_strength, + }) + return config diff --git a/tensorflow/python/keras/optimizer_v2/ftrl_test.py b/tensorflow/python/keras/optimizer_v2/ftrl_test.py new file mode 100644 index 00000000000..bec400e8cbb --- /dev/null +++ b/tensorflow/python/keras/optimizer_v2/ftrl_test.py @@ -0,0 +1,440 @@ +# Copyright 2015 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Functional tests for Ftrl operations.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import numpy as np + +from tensorflow.python.framework import constant_op +from tensorflow.python.framework import dtypes +from tensorflow.python.framework import ops +from tensorflow.python.framework import test_util +from tensorflow.python.keras.optimizer_v2 import ftrl +from tensorflow.python.ops import embedding_ops +from tensorflow.python.ops import math_ops +from tensorflow.python.ops import resource_variable_ops +from tensorflow.python.ops import variables +from tensorflow.python.platform import test +from tensorflow.python.training import adagrad +from tensorflow.python.training import gradient_descent + + +class FtrlOptimizerTest(test.TestCase): + + def doTestFtrlwithoutRegularization(self, use_resource=False): + for dtype in [dtypes.half, dtypes.float32]: + with self.cached_session() as sess: + if use_resource: + var0 = resource_variable_ops.ResourceVariable([0.0, 0.0], dtype=dtype) + var1 = resource_variable_ops.ResourceVariable([0.0, 0.0], dtype=dtype) + else: + var0 = variables.Variable([0.0, 0.0], dtype=dtype) + var1 = variables.Variable([0.0, 0.0], dtype=dtype) + grads0 = constant_op.constant([0.1, 0.2], dtype=dtype) + grads1 = constant_op.constant([0.01, 0.02], dtype=dtype) + opt = ftrl.Ftrl( + 3.0, + initial_accumulator_value=0.1, + l1_regularization_strength=0.0, + l2_regularization_strength=0.0) + update = opt.apply_gradients(zip([grads0, grads1], [var0, var1])) + variables.global_variables_initializer().run() + + v0_val, v1_val = self.evaluate([var0, var1]) + self.assertAllClose([0.0, 0.0], v0_val) + self.assertAllClose([0.0, 0.0], v1_val) + + # Run 3 steps FTRL + for _ in range(3): + update.run() + + v0_val, v1_val = self.evaluate([var0, var1]) + self.assertAllCloseAccordingToType( + np.array([-2.60260963, -4.29698515]), v0_val) + self.assertAllCloseAccordingToType( + np.array([-0.28432083, -0.56694895]), v1_val) + + @test_util.run_deprecated_v1 + def testFtrlWithoutRegularization(self): + self.doTestFtrlwithoutRegularization(use_resource=False) + + @test_util.run_deprecated_v1 + def testResourceFtrlWithoutRegularization(self): + self.doTestFtrlwithoutRegularization(use_resource=True) + + @test_util.run_deprecated_v1 + def testFtrlwithoutRegularization2(self): + for dtype in [dtypes.half, dtypes.float32]: + with self.cached_session() as sess: + var0 = variables.Variable([1.0, 2.0], dtype=dtype) + var1 = variables.Variable([4.0, 3.0], dtype=dtype) + grads0 = constant_op.constant([0.1, 0.2], dtype=dtype) + grads1 = constant_op.constant([0.01, 0.02], dtype=dtype) + + opt = ftrl.Ftrl( + 3.0, + initial_accumulator_value=0.1, + l1_regularization_strength=0.0, + l2_regularization_strength=0.0) + update = opt.apply_gradients(zip([grads0, grads1], [var0, var1])) + variables.global_variables_initializer().run() + + v0_val, v1_val = self.evaluate([var0, var1]) + self.assertAllCloseAccordingToType([1.0, 2.0], v0_val) + self.assertAllCloseAccordingToType([4.0, 3.0], v1_val) + + # Run 3 steps FTRL + for _ in range(3): + update.run() + v0_val, v1_val = self.evaluate([var0, var1]) + self.assertAllCloseAccordingToType( + np.array([-2.55607247, -3.98729396]), v0_val) + self.assertAllCloseAccordingToType( + np.array([-0.28232238, -0.56096673]), v1_val) + + @test_util.run_deprecated_v1 + def testMinimizeSparseResourceVariable(self): + for dtype in [dtypes.half, dtypes.float32, dtypes.float64]: + with self.cached_session(): + var0 = resource_variable_ops.ResourceVariable([[1.0, 2.0]], dtype=dtype) + x = constant_op.constant([[4.0], [5.0]], dtype=dtype) + pred = math_ops.matmul(embedding_ops.embedding_lookup([var0], [0]), x) + loss = pred * pred + sgd_op = ftrl.Ftrl(1.0).minimize(loss, var_list=[var0]) + variables.global_variables_initializer().run() + # Fetch params to validate initial values + self.assertAllCloseAccordingToType([[1.0, 2.0]], self.evaluate(var0)) + # Run 1 step of sgd + sgd_op.run() + # Validate updated params + self.assertAllCloseAccordingToType([[0, 1]], + self.evaluate(var0), + atol=0.01) + + @test_util.run_deprecated_v1 + def testFtrlWithL1(self): + for dtype in [dtypes.half, dtypes.float32]: + with self.cached_session() as sess: + var0 = variables.Variable([1.0, 2.0], dtype=dtype) + var1 = variables.Variable([4.0, 3.0], dtype=dtype) + grads0 = constant_op.constant([0.1, 0.2], dtype=dtype) + grads1 = constant_op.constant([0.01, 0.02], dtype=dtype) + + opt = ftrl.Ftrl( + 3.0, + initial_accumulator_value=0.1, + l1_regularization_strength=0.001, + l2_regularization_strength=0.0) + update = opt.apply_gradients(zip([grads0, grads1], [var0, var1])) + variables.global_variables_initializer().run() + + v0_val, v1_val = self.evaluate([var0, var1]) + self.assertAllCloseAccordingToType([1.0, 2.0], v0_val) + self.assertAllCloseAccordingToType([4.0, 3.0], v1_val) + + # Run 10 steps FTRL + for _ in range(10): + update.run() + v0_val, v1_val = self.evaluate([var0, var1]) + self.assertAllCloseAccordingToType( + np.array([-7.66718769, -10.91273689]), v0_val) + self.assertAllCloseAccordingToType( + np.array([-0.93460727, -1.86147261]), v1_val) + + @test_util.run_deprecated_v1 + def testFtrlWithL1_L2(self): + for dtype in [dtypes.half, dtypes.float32]: + with self.cached_session() as sess: + var0 = variables.Variable([1.0, 2.0], dtype=dtype) + var1 = variables.Variable([4.0, 3.0], dtype=dtype) + grads0 = constant_op.constant([0.1, 0.2], dtype=dtype) + grads1 = constant_op.constant([0.01, 0.02], dtype=dtype) + + opt = ftrl.Ftrl( + 3.0, + initial_accumulator_value=0.1, + l1_regularization_strength=0.001, + l2_regularization_strength=2.0) + update = opt.apply_gradients(zip([grads0, grads1], [var0, var1])) + variables.global_variables_initializer().run() + + v0_val, v1_val = self.evaluate([var0, var1]) + self.assertAllCloseAccordingToType([1.0, 2.0], v0_val) + self.assertAllCloseAccordingToType([4.0, 3.0], v1_val) + + # Run 10 steps FTRL + for _ in range(10): + update.run() + + v0_val, v1_val = self.evaluate([var0, var1]) + self.assertAllCloseAccordingToType( + np.array([-0.24059935, -0.46829352]), v0_val) + self.assertAllCloseAccordingToType( + np.array([-0.02406147, -0.04830509]), v1_val) + + @test_util.run_deprecated_v1 + def testFtrlWithL1_L2_L2Shrinkage(self): + """Test the new FTRL op with support for l2 shrinkage. + + The addition of this parameter which places a constant pressure on weights + towards the origin causes the gradient descent trajectory to differ. The + weights will tend to have smaller magnitudes with this parameter set. + """ + for dtype in [dtypes.half, dtypes.float32]: + with self.cached_session() as sess: + var0 = variables.Variable([1.0, 2.0], dtype=dtype) + var1 = variables.Variable([4.0, 3.0], dtype=dtype) + grads0 = constant_op.constant([0.1, 0.2], dtype=dtype) + grads1 = constant_op.constant([0.01, 0.02], dtype=dtype) + + opt = ftrl.Ftrl( + 3.0, + initial_accumulator_value=0.1, + l1_regularization_strength=0.001, + l2_regularization_strength=2.0, + l2_shrinkage_regularization_strength=0.1) + update = opt.apply_gradients(zip([grads0, grads1], [var0, var1])) + variables.global_variables_initializer().run() + + v0_val, v1_val = self.evaluate([var0, var1]) + self.assertAllCloseAccordingToType([1.0, 2.0], v0_val) + self.assertAllCloseAccordingToType([4.0, 3.0], v1_val) + + # Run 10 steps FTRL + for _ in range(10): + update.run() + + v0_val, v1_val = self.evaluate([var0, var1]) + self.assertAllCloseAccordingToType( + np.array([-0.22578995, -0.44345796]), v0_val) + self.assertAllCloseAccordingToType( + np.array([-0.14378493, -0.13229476]), v1_val) + + @test_util.run_deprecated_v1 + def testFtrlWithL1_L2_L2ShrinkageSparse(self): + """Tests the new FTRL op with support for l2 shrinkage on sparse grads.""" + for dtype in [dtypes.half, dtypes.float32]: + with self.cached_session() as sess: + var0 = variables.Variable([[1.0], [2.0]], dtype=dtype) + var1 = variables.Variable([[4.0], [3.0]], dtype=dtype) + grads0 = ops.IndexedSlices( + constant_op.constant([0.1], shape=[1, 1], dtype=dtype), + constant_op.constant([0]), constant_op.constant([2, 1])) + grads1 = ops.IndexedSlices( + constant_op.constant([0.02], shape=[1, 1], dtype=dtype), + constant_op.constant([1]), constant_op.constant([2, 1])) + + opt = ftrl.Ftrl( + 3.0, + initial_accumulator_value=0.1, + l1_regularization_strength=0.001, + l2_regularization_strength=2.0, + l2_shrinkage_regularization_strength=0.1) + update = opt.apply_gradients(zip([grads0, grads1], [var0, var1])) + variables.global_variables_initializer().run() + + v0_val, v1_val = self.evaluate([var0, var1]) + self.assertAllCloseAccordingToType([[1.0], [2.0]], v0_val) + self.assertAllCloseAccordingToType([[4.0], [3.0]], v1_val) + + # Run 10 steps FTRL + for _ in range(10): + update.run() + + v0_val, v1_val = self.evaluate([var0, var1]) + self.assertAllCloseAccordingToType([[-0.22578995], [2.]], v0_val) + self.assertAllCloseAccordingToType([[4.], [-0.13229476]], v1_val) + + @test_util.run_deprecated_v1 + def testFtrlWithL2ShrinkageDoesNotChangeLrSchedule(self): + """Verifies that l2 shrinkage in FTRL does not change lr schedule.""" + for dtype in [dtypes.half, dtypes.float32]: + with self.cached_session() as sess: + var0 = variables.Variable([1.0, 2.0], dtype=dtype) + var1 = variables.Variable([1.0, 2.0], dtype=dtype) + grads0 = constant_op.constant([0.1, 0.2], dtype=dtype) + grads1 = constant_op.constant([0.1, 0.2], dtype=dtype) + + opt0 = ftrl.Ftrl( + 3.0, + initial_accumulator_value=0.1, + l1_regularization_strength=0.001, + l2_regularization_strength=2.0, + l2_shrinkage_regularization_strength=0.1) + opt1 = ftrl.Ftrl( + 3.0, + initial_accumulator_value=0.1, + l1_regularization_strength=0.001, + l2_regularization_strength=2.0) + update0 = opt0.apply_gradients([(grads0, var0)]) + update1 = opt1.apply_gradients([(grads1, var1)]) + variables.global_variables_initializer().run() + + v0_val, v1_val = self.evaluate([var0, var1]) + self.assertAllCloseAccordingToType([1.0, 2.0], v0_val) + self.assertAllCloseAccordingToType([1.0, 2.0], v1_val) + + # Run 10 steps FTRL + for _ in range(10): + update0.run() + update1.run() + + v0_val, v1_val = self.evaluate([var0, var1]) + # var0 is experiencing L2 shrinkage so it should be smaller than var1 + # in magnitude. + self.assertTrue((v0_val**2 < v1_val**2).all()) + accum0 = sess.run(opt0.get_slot(var0, "accumulator")) + accum1 = sess.run(opt1.get_slot(var1, "accumulator")) + # L2 shrinkage should not change how we update grad accumulator. + self.assertAllCloseAccordingToType(accum0, accum1) + + def applyOptimizer(self, opt, dtype, steps=5, is_sparse=False): + if is_sparse: + var0 = variables.Variable([[0.0], [0.0]], dtype=dtype) + var1 = variables.Variable([[0.0], [0.0]], dtype=dtype) + grads0 = ops.IndexedSlices( + constant_op.constant([0.1], shape=[1, 1], dtype=dtype), + constant_op.constant([0]), constant_op.constant([2, 1])) + grads1 = ops.IndexedSlices( + constant_op.constant([0.02], shape=[1, 1], dtype=dtype), + constant_op.constant([1]), constant_op.constant([2, 1])) + else: + var0 = variables.Variable([0.0, 0.0], dtype=dtype) + var1 = variables.Variable([0.0, 0.0], dtype=dtype) + grads0 = constant_op.constant([0.1, 0.2], dtype=dtype) + grads1 = constant_op.constant([0.01, 0.02], dtype=dtype) + + update = opt.apply_gradients(zip([grads0, grads1], [var0, var1])) + variables.global_variables_initializer().run() + + sess = ops.get_default_session() + v0_val, v1_val = self.evaluate([var0, var1]) + if is_sparse: + self.assertAllCloseAccordingToType([[0.0], [0.0]], v0_val) + self.assertAllCloseAccordingToType([[0.0], [0.0]], v1_val) + else: + self.assertAllCloseAccordingToType([0.0, 0.0], v0_val) + self.assertAllCloseAccordingToType([0.0, 0.0], v1_val) + + # Run Ftrl for a few steps + for _ in range(steps): + update.run() + + v0_val, v1_val = self.evaluate([var0, var1]) + return v0_val, v1_val + + # When variables are initialized with Zero, FTRL-Proximal has two properties: + # 1. Without L1&L2 but with fixed learning rate, FTRL-Proximal is identical + # with GradientDescent. + # 2. Without L1&L2 but with adaptive learning rate, FTRL-Proximal is identical + # with Adagrad. + # So, basing on these two properties, we test if our implementation of + # FTRL-Proximal performs same updates as Adagrad or GradientDescent. + @test_util.run_deprecated_v1 + def testEquivAdagradwithoutRegularization(self): + for dtype in [dtypes.half, dtypes.float32]: + with self.cached_session(): + val0, val1 = self.applyOptimizer( + ftrl.Ftrl( + 3.0, + # Adagrad learning rate + learning_rate_power=-0.5, + initial_accumulator_value=0.1, + l1_regularization_strength=0.0, + l2_regularization_strength=0.0), + dtype) + + with self.cached_session(): + val2, val3 = self.applyOptimizer( + adagrad.AdagradOptimizer(3.0, initial_accumulator_value=0.1), dtype) + + self.assertAllCloseAccordingToType(val0, val2) + self.assertAllCloseAccordingToType(val1, val3) + + @test_util.run_deprecated_v1 + def testEquivSparseAdagradwithoutRegularization(self): + for dtype in [dtypes.half, dtypes.float32]: + with self.cached_session(): + val0, val1 = self.applyOptimizer( + ftrl.Ftrl( + 3.0, + # Adagrad learning rate + learning_rate_power=-0.5, + initial_accumulator_value=0.1, + l1_regularization_strength=0.0, + l2_regularization_strength=0.0), + dtype, + is_sparse=True) + + with self.cached_session(): + val2, val3 = self.applyOptimizer( + adagrad.AdagradOptimizer(3.0, initial_accumulator_value=0.1), + dtype, + is_sparse=True) + + self.assertAllCloseAccordingToType(val0, val2) + self.assertAllCloseAccordingToType(val1, val3) + + @test_util.run_deprecated_v1 + def testEquivSparseGradientDescentwithoutRegularization(self): + for dtype in [dtypes.half, dtypes.float32]: + with self.cached_session(): + val0, val1 = self.applyOptimizer( + ftrl.Ftrl( + 3.0, + # Fixed learning rate + learning_rate_power=-0.0, + initial_accumulator_value=0.1, + l1_regularization_strength=0.0, + l2_regularization_strength=0.0), + dtype, + is_sparse=True) + + with self.cached_session(): + val2, val3 = self.applyOptimizer( + gradient_descent.GradientDescentOptimizer(3.0), + dtype, + is_sparse=True) + + self.assertAllCloseAccordingToType(val0, val2) + self.assertAllCloseAccordingToType(val1, val3) + + @test_util.run_deprecated_v1 + def testEquivGradientDescentwithoutRegularization(self): + for dtype in [dtypes.half, dtypes.float32]: + with self.cached_session(): + val0, val1 = self.applyOptimizer( + ftrl.Ftrl( + 3.0, + # Fixed learning rate + learning_rate_power=-0.0, + initial_accumulator_value=0.1, + l1_regularization_strength=0.0, + l2_regularization_strength=0.0), + dtype) + + with self.cached_session(): + val2, val3 = self.applyOptimizer( + gradient_descent.GradientDescentOptimizer(3.0), dtype) + + self.assertAllCloseAccordingToType(val0, val2) + self.assertAllCloseAccordingToType(val1, val3) + + +if __name__ == "__main__": + test.main() diff --git a/tensorflow/python/keras/optimizer_v2/gradient_descent.py b/tensorflow/python/keras/optimizer_v2/gradient_descent.py index 90106c941cc..2b82b5e78de 100644 --- a/tensorflow/python/keras/optimizer_v2/gradient_descent.py +++ b/tensorflow/python/keras/optimizer_v2/gradient_descent.py @@ -19,7 +19,6 @@ from __future__ import print_function from tensorflow.python.framework import ops from tensorflow.python.keras.optimizer_v2 import optimizer_v2 -from tensorflow.python.ops import math_ops from tensorflow.python.ops import resource_variable_ops from tensorflow.python.training import training_ops @@ -62,7 +61,8 @@ class SGD(optimizer_v2.OptimizerV2): learning_rate=0.001, momentum=0.0, nesterov=False, - name="SGD"): + name="SGD", + **kwargs): """Construct a new Stochastic Gradient Descent or Momentum optimizer. Arguments: @@ -72,9 +72,11 @@ class SGD(optimizer_v2.OptimizerV2): nesterov: boolean. Whether to apply Nesterov momentum. name: Optional name prefix for the operations created when applying gradients. Defaults to 'SGD'. + **kwargs: keyword arguments. Allowed to be {`decay`} """ - super(SGD, self).__init__(name) + super(SGD, self).__init__(name, **kwargs) self._set_hyper("learning_rate", learning_rate) + self._set_hyper("decay", self._initial_decay) self._momentum = False if isinstance(momentum, ops.Tensor) or callable(momentum) or momentum > 0: @@ -91,44 +93,44 @@ class SGD(optimizer_v2.OptimizerV2): self.add_slot(var, "momentum") def _resource_apply_dense(self, grad, var): - learning_rate = self._get_hyper("learning_rate") + var_dtype = var.dtype.base_dtype + lr_t = self._decayed_lr(var_dtype) if self._momentum: momentum_var = self.get_slot(var, "momentum") - return training_ops.resource_apply_momentum( + return training_ops.resource_apply_keras_momentum( var.handle, momentum_var.handle, - math_ops.cast(learning_rate, grad.dtype.base_dtype), + lr_t, grad, - math_ops.cast(self._get_hyper("momentum"), grad.dtype.base_dtype), + self._get_hyper("momentum", var_dtype), use_locking=self._use_locking, use_nesterov=self._nesterov) else: return training_ops.resource_apply_gradient_descent( - var.handle, - math_ops.cast(learning_rate, grad.dtype.base_dtype), - grad, - use_locking=self._use_locking) + var.handle, lr_t, grad, use_locking=self._use_locking) def _resource_apply_sparse_duplicate_indices(self, grad, var, indices): if self._momentum: return super(SGD, self)._resource_apply_sparse_duplicate_indices( grad, var, indices) else: - return resource_variable_ops.resource_scatter_add( - var.handle, indices, -grad * math_ops.cast( - self._get_hyper("learning_rate"), grad.dtype.base_dtype)) + var_dtype = var.dtype.base_dtype + lr_t = self._decayed_lr(var_dtype) + return resource_variable_ops.resource_scatter_add(var.handle, indices, + -grad * lr_t) def _resource_apply_sparse(self, grad, var, indices): # This method is only needed for momentum optimization. - learning_rate = self._get_hyper("learning_rate") + var_dtype = var.dtype.base_dtype + lr_t = self._decayed_lr(var_dtype) momentum_var = self.get_slot(var, "momentum") - return training_ops.resource_sparse_apply_momentum( + return training_ops.resource_sparse_apply_keras_momentum( var.handle, momentum_var.handle, - math_ops.cast(learning_rate, grad.dtype.base_dtype), + lr_t, grad, indices, - math_ops.cast(self._get_hyper("momentum"), grad.dtype.base_dtype), + self._get_hyper("momentum", var_dtype), use_locking=self._use_locking, use_nesterov=self._nesterov) @@ -136,6 +138,7 @@ class SGD(optimizer_v2.OptimizerV2): config = super(SGD, self).get_config() config.update({ "learning_rate": self._serialize_hyperparameter("learning_rate"), + "decay": self._serialize_hyperparameter("decay"), "momentum": self._serialize_hyperparameter("momentum"), "nesterov": self._nesterov, }) diff --git a/tensorflow/python/keras/optimizer_v2/gradient_descent_test.py b/tensorflow/python/keras/optimizer_v2/gradient_descent_test.py index b84bf1a6ecc..0c64202da81 100644 --- a/tensorflow/python/keras/optimizer_v2/gradient_descent_test.py +++ b/tensorflow/python/keras/optimizer_v2/gradient_descent_test.py @@ -47,7 +47,6 @@ class GradientDescentOptimizerTest(test.TestCase): grads0 = constant_op.constant([0.1, 0.1], dtype=dtype) grads1 = constant_op.constant([0.01, 0.01], dtype=dtype) sgd = gradient_descent.SGD(3.0) - # self.assertFalse(sgd._initial_decay) sgd_op = sgd.apply_gradients(zip([grads0, grads1], [var0, var1])) self.evaluate(variables.global_variables_initializer()) # Run 1 step of sgd @@ -58,6 +57,43 @@ class GradientDescentOptimizerTest(test.TestCase): self.assertAllCloseAccordingToType([3.0 - 3.0 * 0.01, 4.0 - 3.0 * 0.01], self.evaluate(var1)) + @test_util.run_in_graph_and_eager_modes + def testBasicWithLearningRateDecay(self): + for dtype in [dtypes.half, dtypes.float32, dtypes.float64]: + with self.cached_session(): + var0 = resource_variable_ops.ResourceVariable([1.0, 2.0], dtype=dtype) + var1 = resource_variable_ops.ResourceVariable([3.0, 4.0], dtype=dtype) + grads0 = constant_op.constant([0.1, 0.1], dtype=dtype) + grads1 = constant_op.constant([0.01, 0.01], dtype=dtype) + learning_rate = 3.0 + decay = 0.5 + sgd = gradient_descent.SGD(learning_rate=learning_rate, decay=decay) + if not context.executing_eagerly(): + sgd_op = sgd.apply_gradients(zip([grads0, grads1], [var0, var1])) + self.evaluate(variables.global_variables_initializer()) + # Run 2 steps of sgd + if not context.executing_eagerly(): + self.evaluate(sgd_op) + else: + sgd.apply_gradients(zip([grads0, grads1], [var0, var1])) + # Validate updated params + self.assertAllCloseAccordingToType([1.0 - 3.0 * 0.1, 2.0 - 3.0 * 0.1], + self.evaluate(var0)) + self.assertAllCloseAccordingToType([3.0 - 3.0 * 0.01, 4.0 - 3.0 * 0.01], + self.evaluate(var1)) + + if not context.executing_eagerly(): + self.evaluate(sgd_op) + else: + sgd.apply_gradients(zip([grads0, grads1], [var0, var1])) + # Validate updated params + self.assertAllCloseAccordingToType( + [1.0 - 3.0 * 0.1 - 2.0 * 0.1, 2.0 - 3.0 * 0.1 - 2.0 * 0.1], + self.evaluate(var0)) + self.assertAllCloseAccordingToType( + [3.0 - 3.0 * 0.01 - 2.0 * 0.01, 4.0 - 3.0 * 0.01 - 2.0 * 0.01], + self.evaluate(var1)) + @test_util.run_in_graph_and_eager_modes def testBasicCallableParams(self): for dtype in [dtypes.half, dtypes.float32, dtypes.float64]: @@ -98,6 +134,7 @@ class GradientDescentOptimizerTest(test.TestCase): self.evaluate(var0)) self.assertAllCloseAccordingToType([3.0 - 1.0], self.evaluate(var1)) + @test_util.run_deprecated_v1 def testMinimizeSparseResourceVariable(self): for dtype in [dtypes.half, dtypes.float32, dtypes.float64]: with self.cached_session(): @@ -137,6 +174,7 @@ class GradientDescentOptimizerTest(test.TestCase): self.assertAllCloseAccordingToType([3.0 - 3.0 * 0.01, 4.0 - 3.0 * 0.01], self.evaluate(var1)) + @test_util.run_deprecated_v1 def testGradWrtRef(self): for dtype in [dtypes.half, dtypes.float32, dtypes.float64]: with self.cached_session(): @@ -170,6 +208,37 @@ class GradientDescentOptimizerTest(test.TestCase): self.assertAllCloseAccordingToType([[3.0], [4.0 - 3.0 * 0.01]], self.evaluate(var1)) + @test_util.run_deprecated_v1 + def testSparseBasicWithLearningRateDecay(self): + for dtype in [dtypes.half, dtypes.float32, dtypes.float64]: + with self.cached_session(): + var0 = variables.Variable([[1.0], [2.0]], dtype=dtype) + var1 = variables.Variable([[3.0], [4.0]], dtype=dtype) + grads0 = ops.IndexedSlices( + constant_op.constant([0.1], shape=[1, 1], dtype=dtype), + constant_op.constant([0]), constant_op.constant([2, 1])) + grads1 = ops.IndexedSlices( + constant_op.constant([0.01], shape=[1, 1], dtype=dtype), + constant_op.constant([1]), constant_op.constant([2, 1])) + sgd_op = gradient_descent.SGD( + 3.0, decay=0.5).apply_gradients( + zip([grads0, grads1], [var0, var1])) + self.evaluate(variables.global_variables_initializer()) + # Run 2 steps of sgd + self.evaluate(sgd_op) + # Validate updated params + self.assertAllCloseAccordingToType([[1.0 - 3.0 * 0.1], [2.0]], + self.evaluate(var0)) + self.assertAllCloseAccordingToType([[3.0], [4.0 - 3.0 * 0.01]], + self.evaluate(var1)) + + self.evaluate(sgd_op) + # Validate updated params + self.assertAllCloseAccordingToType( + [[1.0 - 3.0 * 0.1 - 2.0 * 0.1], [2.0]], self.evaluate(var0)) + self.assertAllCloseAccordingToType( + [[3.0], [4.0 - 3.0 * 0.01 - 2.0 * 0.01]], self.evaluate(var1)) + def testCapturingInDefunWhileExecutingEagerly(self): with context.eager_mode(): optimizer = gradient_descent.SGD(1.0) @@ -194,10 +263,8 @@ class GradientDescentOptimizerTest(test.TestCase): class MomentumOptimizerTest(test.TestCase): def _update_nesterov_momentum_numpy(self, var, accum, g, lr, momentum): - var = var + accum * lr * momentum - accum = accum * momentum + g - var = var - lr * accum - var = var - accum * lr * momentum + accum = accum * momentum - g * lr + var += (accum * momentum - g * lr) return var, accum @test_util.run_in_graph_and_eager_modes @@ -222,9 +289,9 @@ class MomentumOptimizerTest(test.TestCase): # Check we have slots slot0 = mom_opt.get_slot(var0, "momentum") - self.assertEquals(slot0.get_shape(), var0.get_shape()) + self.assertEqual(slot0.get_shape(), var0.get_shape()) slot1 = mom_opt.get_slot(var1, "momentum") - self.assertEquals(slot1.get_shape(), var1.get_shape()) + self.assertEqual(slot1.get_shape(), var1.get_shape()) # Step 1: the momentum accumulators where 0. So we should see a normal # update: v -= grad * learning_rate @@ -232,9 +299,9 @@ class MomentumOptimizerTest(test.TestCase): self.evaluate(mom_update) # Check that the momentum accumulators have been updated. self.assertAllCloseAccordingToType( - np.array([0.1, 0.1]), self.evaluate(slot0)) + np.array([-0.2, -0.2]), self.evaluate(slot0)) self.assertAllCloseAccordingToType( - np.array([0.01, 0.01]), self.evaluate(slot1)) + np.array([-0.02, -0.02]), self.evaluate(slot1)) # Check that the parameters have been updated. self.assertAllCloseAccordingToType( np.array([1.0 - (0.1 * 2.0), 2.0 - (0.1 * 2.0)]), @@ -248,11 +315,11 @@ class MomentumOptimizerTest(test.TestCase): mom_opt.apply_gradients(zip([grads0, grads1], [var0, var1])) # Check that the momentum accumulators have been updated. self.assertAllCloseAccordingToType( - np.array([(0.9 * 0.1 + 0.1), (0.9 * 0.1 + 0.1)]), + np.array([(0.9 * (-0.2) - 2.0 * 0.1), (0.9 * (-0.2) - 2.0 * 0.1)]), self.evaluate(slot0)) self.assertAllCloseAccordingToType( - np.array([(0.9 * 0.01 + 0.01), (0.9 * 0.01 + 0.01)]), - self.evaluate(slot1)) + np.array([(0.9 * (-0.02) - 2.0 * 0.01), + (0.9 * (-0.02) - 2.0 * 0.01)]), self.evaluate(slot1)) # Check that the parameters have been updated. self.assertAllCloseAccordingToType( np.array([ @@ -265,6 +332,7 @@ class MomentumOptimizerTest(test.TestCase): 3.98 - ((0.9 * 0.01 + 0.01) * 2.0) ]), self.evaluate(var1)) + @test_util.run_deprecated_v1 def testNesterovMomentum(self): for dtype in [dtypes.float32, dtypes.float64]: with self.cached_session(): @@ -289,9 +357,10 @@ class MomentumOptimizerTest(test.TestCase): var0_np, accum0_np, var0_np * 10, 2.0, 0.9) var1_np, accum1_np = self._update_nesterov_momentum_numpy( var1_np, accum1_np, 3, 2.0, 0.9) - self.assertAllClose(var0_np, var0.eval()) - self.assertAllClose(var1_np, var1.eval()) + self.assertAllClose(var0_np, self.evaluate(var0)) + self.assertAllClose(var1_np, self.evaluate(var1)) + @test_util.run_deprecated_v1 def testSparseNesterovMomentum(self): for dtype in [dtypes.float32, dtypes.float64]: with self.cached_session(): @@ -329,10 +398,11 @@ class MomentumOptimizerTest(test.TestCase): var0_np, accum0_np, var0_np * 10, 2.0, 0.9) var1_np, accum1_np = self._update_nesterov_momentum_numpy( var1_np, accum1_np, 3, 2.0, 0.9) - self.assertAllClose(var0_np, var0.eval()) - self.assertAllClose(var1_np, var1.eval()) + self.assertAllClose(var0_np, self.evaluate(var0)) + self.assertAllClose(var1_np, self.evaluate(var1)) @test_util.run_in_graph_and_eager_modes + @test_util.run_deprecated_v1 def testMinimizeSparseResourceVariable(self): for dtype in [dtypes.half, dtypes.float32, dtypes.float64]: # This test invokes the ResourceSparseApplyMomentum operation, which @@ -386,6 +456,7 @@ class MomentumOptimizerTest(test.TestCase): self.evaluate(sgd_op) self.assertAllCloseAccordingToType([[1, 1], [0, 0]], self.evaluate(var0)) + @test_util.run_deprecated_v1 def testTensorLearningRateAndMomentum(self): for dtype in [dtypes.half, dtypes.float32, dtypes.float64]: with self.cached_session(): @@ -401,43 +472,50 @@ class MomentumOptimizerTest(test.TestCase): variables.global_variables_initializer().run() # Check we have slots slot0 = mom_opt.get_slot(var0, "momentum") - self.assertEquals(slot0.get_shape(), var0.get_shape()) + self.assertEqual(slot0.get_shape(), var0.get_shape()) slot1 = mom_opt.get_slot(var1, "momentum") - self.assertEquals(slot1.get_shape(), var1.get_shape()) + self.assertEqual(slot1.get_shape(), var1.get_shape()) # Fetch params to validate initial values - self.assertAllClose([1.0, 2.0], var0.eval()) - self.assertAllClose([3.0, 4.0], var1.eval()) + self.assertAllClose([1.0, 2.0], self.evaluate(var0)) + self.assertAllClose([3.0, 4.0], self.evaluate(var1)) # Step 1: the momentum accumulators where 0. So we should see a normal # update: v -= grad * learning_rate mom_update.run() # Check that the momentum accumulators have been updated. - self.assertAllCloseAccordingToType(np.array([0.1, 0.1]), slot0.eval()) - self.assertAllCloseAccordingToType(np.array([0.01, 0.01]), slot1.eval()) + self.assertAllCloseAccordingToType( + np.array([-0.2, -0.2]), self.evaluate(slot0)) + self.assertAllCloseAccordingToType( + np.array([-0.02, -0.02]), self.evaluate(slot1)) # Check that the parameters have been updated. self.assertAllCloseAccordingToType( - np.array([1.0 - (0.1 * 2.0), 2.0 - (0.1 * 2.0)]), var0.eval()) + np.array([1.0 - (0.1 * 2.0), 2.0 - (0.1 * 2.0)]), + self.evaluate(var0)) self.assertAllCloseAccordingToType( - np.array([3.0 - (0.01 * 2.0), 4.0 - (0.01 * 2.0)]), var1.eval()) + np.array([3.0 - (0.01 * 2.0), 4.0 - (0.01 * 2.0)]), + self.evaluate(var1)) # Step 2: the momentum accumulators contain the previous update. mom_update.run() # Check that the momentum accumulators have been updated. self.assertAllCloseAccordingToType( - np.array([(0.9 * 0.1 + 0.1), (0.9 * 0.1 + 0.1)]), slot0.eval()) + np.array([(0.9 * (-0.2) - 2.0 * 0.1), (0.9 * (-0.2) - 2.0 * 0.1)]), + self.evaluate(slot0)) self.assertAllCloseAccordingToType( - np.array([(0.9 * 0.01 + 0.01), (0.9 * 0.01 + 0.01)]), slot1.eval()) + np.array([(0.9 * (-0.02) - 2.0 * 0.01), + (0.9 * (-0.02) - 2.0 * 0.01)]), self.evaluate(slot1)) # Check that the parameters have been updated. self.assertAllCloseAccordingToType( np.array([ 1.0 - (0.1 * 2.0) - ((0.9 * 0.1 + 0.1) * 2.0), 2.0 - (0.1 * 2.0) - ((0.9 * 0.1 + 0.1) * 2.0) - ]), var0.eval()) + ]), self.evaluate(var0)) self.assertAllCloseAccordingToType( np.array([ 2.98 - ((0.9 * 0.01 + 0.01) * 2.0), 3.98 - ((0.9 * 0.01 + 0.01) * 2.0) - ]), var1.eval()) + ]), self.evaluate(var1)) + @test_util.run_deprecated_v1 def testSparse(self): for dtype in [dtypes.half, dtypes.float32, dtypes.float64]: with self.cached_session(): @@ -456,57 +534,65 @@ class MomentumOptimizerTest(test.TestCase): # Check we have slots slot0 = mom_opt.get_slot(var0, "momentum") - self.assertEquals(slot0.get_shape(), var0.get_shape()) + self.assertEqual(slot0.get_shape(), var0.get_shape()) slot1 = mom_opt.get_slot(var1, "momentum") - self.assertEquals(slot1.get_shape(), var1.get_shape()) + self.assertEqual(slot1.get_shape(), var1.get_shape()) # Fetch params to validate initial values - self.assertAllClose([0, 0], var0.eval()[0]) - self.assertAllClose([0, 0], var0.eval()[1]) - self.assertAllClose([1, 1], var1.eval()[2]) + self.assertAllClose([0, 0], self.evaluate(var0)[0]) + self.assertAllClose([0, 0], self.evaluate(var0)[1]) + self.assertAllClose([1, 1], self.evaluate(var1)[2]) # Step 1: the momentum accumulators are 0. So we should see a normal # update: v -= grad * learning_rate mom_update.run() # Check that the momentum accumulators have been updated. - self.assertAllCloseAccordingToType(np.array([0, 0]), slot0.eval()[0]) - self.assertAllCloseAccordingToType(np.array([.1, .1]), slot0.eval()[1]) self.assertAllCloseAccordingToType( - np.array([.01, .01]), - slot1.eval()[2]) + np.array([0, 0]), + self.evaluate(slot0)[0]) + self.assertAllCloseAccordingToType( + np.array([-2.0 * .1, -2.0 * .1]), + self.evaluate(slot0)[1]) + self.assertAllCloseAccordingToType( + np.array([-2.0 * .01, -2.0 * .01]), + self.evaluate(slot1)[2]) # Check that the parameters have been updated. - self.assertAllCloseAccordingToType(np.array([0, 0]), var0.eval()[0]) + self.assertAllCloseAccordingToType( + np.array([0, 0]), + self.evaluate(var0)[0]) self.assertAllCloseAccordingToType( np.array([-(0.1 * 2.0), -(0.1 * 2.0)]), - var0.eval()[1]) + self.evaluate(var0)[1]) self.assertAllCloseAccordingToType( np.array([1.0 - (0.01 * 2.0), 1.0 - (0.01 * 2.0)]), - var1.eval()[2]) + self.evaluate(var1)[2]) # Step 2: the momentum accumulators contain the previous update. mom_update.run() # Check that the momentum accumulators have been updated. - self.assertAllClose(np.array([0, 0]), slot0.eval()[0]) + self.assertAllClose(np.array([0, 0]), self.evaluate(slot0)[0]) self.assertAllCloseAccordingToType( - np.array([(0.9 * 0.1 + 0.1), (0.9 * 0.1 + 0.1)]), - slot0.eval()[1]) + np.array([(0.9 * (-0.2) - 2.0 * 0.1), (0.9 * (-0.2) - 2.0 * 0.1)]), + self.evaluate(slot0)[1]) self.assertAllCloseAccordingToType( - np.array([(0.9 * 0.01 + 0.01), (0.9 * 0.01 + 0.01)]), - slot1.eval()[2]) + np.array([(0.9 * (-0.02) - 2.0 * 0.01), + (0.9 * (-0.02) - 2.0 * 0.01)]), + self.evaluate(slot1)[2]) # Check that the parameters have been updated. - self.assertAllClose(np.array([0, 0]), var0.eval()[0]) + self.assertAllClose(np.array([0, 0]), self.evaluate(var0)[0]) self.assertAllCloseAccordingToType( np.array([ -(0.1 * 2.0) - ((0.9 * 0.1 + 0.1) * 2.0), -(0.1 * 2.0) - ((0.9 * 0.1 + 0.1) * 2.0) ]), - var0.eval()[1]) + self.evaluate(var0)[1]) self.assertAllCloseAccordingToType( np.array([ 0.98 - ((0.9 * 0.01 + 0.01) * 2.0), 0.98 - ((0.9 * 0.01 + 0.01) * 2.0) ]), - var1.eval()[2]) + self.evaluate(var1)[2]) + @test_util.run_deprecated_v1 def testSharing(self): for dtype in [dtypes.half, dtypes.float32, dtypes.float64]: with self.cached_session(): @@ -522,42 +608,48 @@ class MomentumOptimizerTest(test.TestCase): variables.global_variables_initializer().run() slot0 = mom_opt.get_slot(var0, "momentum") - self.assertEquals(slot0.get_shape(), var0.get_shape()) + self.assertEqual(slot0.get_shape(), var0.get_shape()) slot1 = mom_opt.get_slot(var1, "momentum") - self.assertEquals(slot1.get_shape(), var1.get_shape()) + self.assertEqual(slot1.get_shape(), var1.get_shape()) # Fetch params to validate initial values - self.assertAllClose([1.0, 2.0], var0.eval()) - self.assertAllClose([3.0, 4.0], var1.eval()) + self.assertAllClose([1.0, 2.0], self.evaluate(var0)) + self.assertAllClose([3.0, 4.0], self.evaluate(var1)) # Step 1: the momentum accumulators where 0. So we should see a normal # update: v -= grad * learning_rate mom_update1.run() # Check that the momentum accumulators have been updated. - self.assertAllCloseAccordingToType(np.array([0.1, 0.1]), slot0.eval()) - self.assertAllCloseAccordingToType(np.array([0.01, 0.01]), slot1.eval()) + self.assertAllCloseAccordingToType( + np.array([-0.2, -0.2]), self.evaluate(slot0)) + self.assertAllCloseAccordingToType( + np.array([-0.02, -0.02]), self.evaluate(slot1)) # Check that the parameters have been updated. self.assertAllCloseAccordingToType( - np.array([1.0 - (0.1 * 2.0), 2.0 - (0.1 * 2.0)]), var0.eval()) + np.array([1.0 - (0.1 * 2.0), 2.0 - (0.1 * 2.0)]), + self.evaluate(var0)) self.assertAllCloseAccordingToType( - np.array([3.0 - (0.01 * 2.0), 4.0 - (0.01 * 2.0)]), var1.eval()) + np.array([3.0 - (0.01 * 2.0), 4.0 - (0.01 * 2.0)]), + self.evaluate(var1)) # Step 2: the second momentum accumulators contain the previous update. mom_update2.run() # Check that the momentum accumulators have been updated. self.assertAllCloseAccordingToType( - np.array([(0.9 * 0.1 + 0.1), (0.9 * 0.1 + 0.1)]), slot0.eval()) + np.array([(0.9 * (-0.2) - 2.0 * 0.1), (0.9 * (-0.2) - 2.0 * 0.1)]), + self.evaluate(slot0)) self.assertAllCloseAccordingToType( - np.array([(0.9 * 0.01 + 0.01), (0.9 * 0.01 + 0.01)]), slot1.eval()) + np.array([(0.9 * (-0.02) - 2.0 * 0.01), + (0.9 * (-0.02) - 2.0 * 0.01)]), self.evaluate(slot1)) # Check that the parameters have been updated. self.assertAllCloseAccordingToType( np.array([ 1.0 - (0.1 * 2.0) - ((0.9 * 0.1 + 0.1) * 2.0), 2.0 - (0.1 * 2.0) - ((0.9 * 0.1 + 0.1) * 2.0) - ]), var0.eval()) + ]), self.evaluate(var0)) self.assertAllCloseAccordingToType( np.array([ 2.98 - ((0.9 * 0.01 + 0.01) * 2.0), 3.98 - ((0.9 * 0.01 + 0.01) * 2.0) - ]), var1.eval()) + ]), self.evaluate(var1)) @test_util.run_in_graph_and_eager_modes def testConfig(self): diff --git a/tensorflow/python/keras/optimizer_v2/nadam.py b/tensorflow/python/keras/optimizer_v2/nadam.py new file mode 100644 index 00000000000..00b095e0dc9 --- /dev/null +++ b/tensorflow/python/keras/optimizer_v2/nadam.py @@ -0,0 +1,143 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Nadam for TensorFlow.""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from tensorflow.python.framework import ops +from tensorflow.python.keras.optimizer_v2 import adam +from tensorflow.python.ops import array_ops +from tensorflow.python.ops import control_flow_ops +from tensorflow.python.ops import math_ops +from tensorflow.python.ops import state_ops +from tensorflow.python.training import training_ops + + +class Nadam(adam.Adam): + r"""Optimizer that implements the NAdam algorithm. + + Much like Adam is essentially RMSprop with momentum, Nadam is Adam with + Nesterov momentum. + + Initialization: + + $$m_0 := 0 \text{(Initialize initial 1st moment vector)}$$ + $$v_0 := 0 \text{(Initialize initial 2nd moment vector)}$$ + $$t := 0 \text{(Initialize timestep)}$$ + + Computes: + $$t := t + 1$$ + $$lr_t := \text{learning\_rate} * \sqrt{1 - beta_2^t} / (1 - beta_1^t)$$ + $$m_t := beta_1 * m_{t-1} + (1 - beta_1) * g$$ + $$v_t := beta_2 * v_{t-1} + (1 - beta_2) * g * g$$ + $$m_bar_t := beta_1 * v_t + (1 - beta_1) * g$$ + $$theta_t := theta_{t-1} - lr_t * m_bar_t / (\sqrt{v_t} + \epsilon)$$ + + gradient is evaluated at theta(t) + momentum * v(t), and the variables always + store theta + beta_1 * m / sqrt(v) instead of theta. + + References + See [Dozat, T., 2015](http://cs229.stanford.edu/proj2015/054_report.pdf). + """ + + def __init__(self, + learning_rate=0.001, + beta_1=0.9, + beta_2=0.999, + epsilon=1e-7, + name='Nadam', + **kwargs): + """Construct a new Nadam optimizer. + + Args: + learning_rate: A Tensor or a floating point value. The learning rate. + beta_1: A float value or a constant float tensor. The exponential decay + rate for the 1st moment estimates. + beta_2: A float value or a constant float tensor. The exponential decay + rate for the exponentially weighted infinity norm. + epsilon: A small constant for numerical stability. + name: Optional name for the operations created when applying gradients. + Defaults to "Adamax". + **kwargs: keyword arguments. Allowed to be {`decay`} + """ + + # pylint: disable=useless-super-delegation + super(Nadam, self).__init__( + learning_rate=learning_rate, + beta_1=beta_1, + beta_2=beta_2, + epsilon=epsilon, + amsgrad=False, + name=name, + **kwargs) + # pylint: enable=useless-super-delegation + + def _resource_apply_dense(self, grad, var): + var_dtype = var.dtype.base_dtype + lr_t = self._decayed_lr(var_dtype) + m = self.get_slot(var, 'm') + v = self.get_slot(var, 'v') + beta_1_t = self._get_hyper('beta_1', var_dtype) + beta_2_t = self._get_hyper('beta_2', var_dtype) + local_step = math_ops.cast(self.iterations + 1, var_dtype) + beta_1_power = math_ops.pow(beta_1_t, local_step) + beta_2_power = math_ops.pow(beta_2_t, local_step) + return training_ops.resource_apply_adam( + var.handle, + m.handle, + v.handle, + beta_1_power, + beta_2_power, + lr_t, + beta_1_t, + beta_2_t, + self._get_hyper('epsilon', var_dtype), + grad, + use_locking=self._use_locking, + use_nesterov=True) + + def _resource_apply_sparse(self, grad, var, indices): + var_dtype = var.dtype.base_dtype + lr_t = self._decayed_lr(var_dtype) + beta_1_t = self._get_hyper('beta_1', var_dtype) + beta_2_t = self._get_hyper('beta_2', var_dtype) + local_step = math_ops.cast(self.iterations + 1, var_dtype) + beta_1_power = math_ops.pow(beta_1_t, local_step) + beta_2_power = math_ops.pow(beta_2_t, local_step) + epsilon_t = self._get_hyper('epsilon', var_dtype) + lr = (lr_t * math_ops.sqrt(1 - beta_2_power) / (1 - beta_1_power)) + + # m_t = beta1 * m + (1 - beta1) * g_t + m = self.get_slot(var, 'm') + m_scaled_g_values = grad * (1 - beta_1_t) + m_t = state_ops.assign(m, m * beta_1_t, use_locking=self._use_locking) + with ops.control_dependencies([m_t]): + m_t = self._resource_scatter_add(m, indices, m_scaled_g_values) + # m_bar = (1 - beta1) * g_t + beta1 * m_t + m_bar = m_scaled_g_values + beta_1_t * array_ops.gather(m_t, indices) + + # v_t = beta2 * v + (1 - beta2) * (g_t * g_t) + v = self.get_slot(var, 'v') + v_scaled_g_values = (grad * grad) * (1 - beta_2_t) + v_t = state_ops.assign(v, v * beta_2_t, use_locking=self._use_locking) + with ops.control_dependencies([v_t]): + v_t = self._resource_scatter_add(v, indices, v_scaled_g_values) + + v_t_slice = array_ops.gather(v_t, indices) + v_sqrt = math_ops.sqrt(v_t_slice) + var_update = self._resource_scatter_add(var, indices, + -lr * m_bar / (v_sqrt + epsilon_t)) + return control_flow_ops.group(*[var_update, m_bar, v_t]) diff --git a/tensorflow/python/keras/optimizer_v2/nadam_test.py b/tensorflow/python/keras/optimizer_v2/nadam_test.py new file mode 100644 index 00000000000..d991e3117ca --- /dev/null +++ b/tensorflow/python/keras/optimizer_v2/nadam_test.py @@ -0,0 +1,213 @@ +# Copyright 2015 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Tests for Nadam.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import numpy as np + +from tensorflow.python.framework import constant_op +from tensorflow.python.framework import dtypes +from tensorflow.python.framework import ops +from tensorflow.python.framework import test_util +from tensorflow.python.keras.optimizer_v2 import nadam +from tensorflow.python.ops import math_ops +from tensorflow.python.ops import resource_variable_ops +from tensorflow.python.ops import variables +from tensorflow.python.platform import test + + +def get_beta_accumulators(opt, dtype): + local_step = math_ops.cast(opt.iterations + 1, dtype) + beta_1_t = math_ops.cast(opt._get_hyper("beta_1"), dtype) + beta_1_power = math_ops.pow(beta_1_t, local_step) + beta_2_t = math_ops.cast(opt._get_hyper("beta_2"), dtype) + beta_2_power = math_ops.pow(beta_2_t, local_step) + return (beta_1_power, beta_2_power) + + +def nadam_update_numpy(param, + g_t, + t, + m, + v, + alpha=0.001, + beta1=0.9, + beta2=0.999, + epsilon=1e-8): + alpha_t = alpha * np.sqrt(1 - beta2**(t + 1)) / (1 - beta1**(t + 1)) + + m_t = beta1 * m + (1 - beta1) * g_t + v_t = beta2 * v + (1 - beta2) * g_t * g_t + + m_bar = (1 - beta1) * g_t + beta1 * m_t + + param_t = param - alpha_t * m_bar / (np.sqrt(v_t) + epsilon) + return param_t, m_t, v_t + + +class NadamOptimizerTest(test.TestCase): + + def doTestSparse(self, use_resource=False): + sparse_epsilon = 1e-7 + for dtype in [dtypes.half, dtypes.float32, dtypes.float64]: + with self.cached_session(): + # Initialize variables for numpy implementation. + m0, v0, m1, v1 = 0.0, 0.0, 0.0, 0.0 + var0_np = np.array([1.0, 1.0, 2.0], dtype=dtype.as_numpy_dtype) + grads0_np = np.array([0.1, 0, 0.1], dtype=dtype.as_numpy_dtype) + var1_np = np.array([3.0, 3.0, 4.0], dtype=dtype.as_numpy_dtype) + grads1_np = np.array([0.01, 0, 0.01], dtype=dtype.as_numpy_dtype) + + if use_resource: + var0 = resource_variable_ops.ResourceVariable(var0_np) + var1 = resource_variable_ops.ResourceVariable(var1_np) + else: + var0 = variables.Variable(var0_np) + var1 = variables.Variable(var1_np) + grads0_np_indices = np.array([0, 2], dtype=np.int32) + grads0 = ops.IndexedSlices( + constant_op.constant(grads0_np[grads0_np_indices]), + constant_op.constant(grads0_np_indices), constant_op.constant([3])) + grads1_np_indices = np.array([0, 2], dtype=np.int32) + grads1 = ops.IndexedSlices( + constant_op.constant(grads1_np[grads1_np_indices]), + constant_op.constant(grads1_np_indices), constant_op.constant([3])) + opt = nadam.Nadam(epsilon=sparse_epsilon) + update = opt.apply_gradients(zip([grads0, grads1], [var0, var1])) + variables.global_variables_initializer().run() + + # Fetch params to validate initial values + self.assertAllClose([1.0, 1.0, 2.0], var0.eval()) + self.assertAllClose([3.0, 3.0, 4.0], var1.eval()) + + beta1_power, beta2_power = get_beta_accumulators(opt, dtype) + + # Run 3 steps of Nadam + for t in range(3): + self.assertAllCloseAccordingToType(0.9**(t + 1), beta1_power.eval()) + self.assertAllCloseAccordingToType(0.999**(t + 1), beta2_power.eval()) + update.run() + + var0_np, m0, v0 = nadam_update_numpy( + var0_np, grads0_np, t, m0, v0, epsilon=sparse_epsilon) + var1_np, m1, v1 = nadam_update_numpy( + var1_np, grads1_np, t, m1, v1, epsilon=sparse_epsilon) + + # Validate updated params + self.assertAllCloseAccordingToType(var0_np, var0.eval()) + self.assertAllCloseAccordingToType(var1_np, var1.eval()) + + @test_util.run_deprecated_v1 + def testSparse(self): + self.doTestSparse(use_resource=False) + + @test_util.run_deprecated_v1 + def testResourceSparse(self): + self.doTestSparse(use_resource=True) + + def doTestBasic(self, use_resource=False): + for dtype in [dtypes.half, dtypes.float32, dtypes.float64]: + with self.cached_session(): + # Initialize variables for numpy implementation. + m0, v0, m1, v1 = 0.0, 0.0, 0.0, 0.0 + var0_np = np.array([1.0, 2.0], dtype=dtype.as_numpy_dtype) + grads0_np = np.array([0.1, 0.1], dtype=dtype.as_numpy_dtype) + var1_np = np.array([3.0, 4.0], dtype=dtype.as_numpy_dtype) + grads1_np = np.array([0.01, 0.01], dtype=dtype.as_numpy_dtype) + + if use_resource: + var0 = resource_variable_ops.ResourceVariable(var0_np) + var1 = resource_variable_ops.ResourceVariable(var1_np) + else: + var0 = variables.Variable(var0_np) + var1 = variables.Variable(var1_np) + grads0 = constant_op.constant(grads0_np) + grads1 = constant_op.constant(grads1_np) + opt = nadam.Nadam() + update = opt.apply_gradients(zip([grads0, grads1], [var0, var1])) + variables.global_variables_initializer().run() + + # Fetch params to validate initial values + self.assertAllClose([1.0, 2.0], var0.eval()) + self.assertAllClose([3.0, 4.0], var1.eval()) + + beta1_power, beta2_power = get_beta_accumulators(opt, dtype) + + # Run 3 steps of Nadam + for t in range(3): + self.assertAllCloseAccordingToType(0.9**(t + 1), beta1_power.eval()) + self.assertAllCloseAccordingToType(0.999**(t + 1), beta2_power.eval()) + update.run() + + var0_np, m0, v0 = nadam_update_numpy(var0_np, grads0_np, t, m0, v0) + var1_np, m1, v1 = nadam_update_numpy(var1_np, grads1_np, t, m1, v1) + + # Validate updated params + self.assertAllCloseAccordingToType(var0_np, var0.eval()) + self.assertAllCloseAccordingToType(var1_np, var1.eval()) + + @test_util.run_deprecated_v1 + def testResourceBasic(self): + self.doTestBasic(use_resource=True) + + @test_util.run_deprecated_v1 + def testBasicWithLearningRateDecay(self): + for dtype in [dtypes.half, dtypes.float32, dtypes.float64]: + with self.cached_session(): + # Initialize variables for numpy implementation. + m0, v0, m1, v1 = 0.0, 0.0, 0.0, 0.0 + var0_np = np.array([1.0, 2.0], dtype=dtype.as_numpy_dtype) + grads0_np = np.array([0.1, 0.1], dtype=dtype.as_numpy_dtype) + var1_np = np.array([3.0, 4.0], dtype=dtype.as_numpy_dtype) + grads1_np = np.array([0.01, 0.01], dtype=dtype.as_numpy_dtype) + + var0 = resource_variable_ops.ResourceVariable(var0_np) + var1 = resource_variable_ops.ResourceVariable(var1_np) + grads0 = constant_op.constant(grads0_np) + grads1 = constant_op.constant(grads1_np) + learning_rate = 0.001 + decay = 0.5 + opt = nadam.Nadam(learning_rate=learning_rate, decay=decay) + update = opt.apply_gradients(zip([grads0, grads1], [var0, var1])) + variables.global_variables_initializer().run() + + # Fetch params to validate initial values + self.assertAllClose([1.0, 2.0], var0.eval()) + self.assertAllClose([3.0, 4.0], var1.eval()) + + beta1_power, beta2_power = get_beta_accumulators(opt, dtype) + + # Run 3 steps of Nadam + for t in range(3): + self.assertAllCloseAccordingToType(0.9**(t + 1), beta1_power.eval()) + self.assertAllCloseAccordingToType(0.999**(t + 1), beta2_power.eval()) + update.run() + + lr = learning_rate / (1 + decay * t) + var0_np, m0, v0 = nadam_update_numpy( + var0_np, grads0_np, t, m0, v0, alpha=lr) + var1_np, m1, v1 = nadam_update_numpy( + var1_np, grads1_np, t, m1, v1, alpha=lr) + + # Validate updated params + self.assertAllCloseAccordingToType(var0_np, var0.eval()) + self.assertAllCloseAccordingToType(var1_np, var1.eval()) + + +if __name__ == "__main__": + test.main() diff --git a/tensorflow/python/keras/optimizer_v2/optimizer_v2.py b/tensorflow/python/keras/optimizer_v2/optimizer_v2.py index c6e1d57c5e4..b26b3cefc8c 100644 --- a/tensorflow/python/keras/optimizer_v2/optimizer_v2.py +++ b/tensorflow/python/keras/optimizer_v2/optimizer_v2.py @@ -24,16 +24,17 @@ import abc import six +from tensorflow.python.distribute import reduce_util as ds_reduce_util from tensorflow.python.eager import backprop from tensorflow.python.eager import context from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops from tensorflow.python.keras import backend from tensorflow.python.keras import initializers -from tensorflow.python.keras.engine import base_layer +from tensorflow.python.keras.engine import base_layer_utils from tensorflow.python.ops import gradients -from tensorflow.python.ops import variable_scope -from tensorflow.python.ops import variables +from tensorflow.python.ops import math_ops +from tensorflow.python.ops import variables as tf_variables from tensorflow.python.platform import tf_logging as logging from tensorflow.python.training import distribution_strategy_context from tensorflow.python.training import optimizer as optimizer_v1 @@ -114,7 +115,7 @@ class OptimizerV2(optimizer_v1.Optimizer): """ - def __init__(self, name): + def __init__(self, name, **kwargs): """Create a new Optimizer. This must be called by the constructors of subclasses. @@ -128,6 +129,7 @@ class OptimizerV2(optimizer_v1.Optimizer): Args: name: A non-empty string. The name to use for accumulators created for the optimizer. + **kwargs: keyword arguments. Allowed to be {`decay`} Raises: ValueError: If name is malformed. @@ -140,6 +142,12 @@ class OptimizerV2(optimizer_v1.Optimizer): # dict: {variable name : {slot name : variable}} self._slots = {} self._weights = [] + + decay = kwargs.pop("decay", 0.0) + if decay < 0.: + raise ValueError("decay cannot be less than 0: {}".format(decay)) + self._initial_decay = decay + self._prepared = False def minimize(self, @@ -296,6 +304,7 @@ class OptimizerV2(optimizer_v1.Optimizer): grads_and_vars = zip(reduced_grads, var_list) with ops.init_scope(): + self._prepare() self._create_slots(var_list) update_ops = [] @@ -317,19 +326,21 @@ class OptimizerV2(optimizer_v1.Optimizer): return update_op with ops.name_scope(name, self._name) as name: - self._prepare() for grad, var in grads_and_vars: scope_name = ("" if ops.executing_eagerly_outside_functions() else "_" + var.op.name) - with ops.name_scope("update" + scope_name), ops.colocate_with(var): + with ops.name_scope("update" + scope_name): update_ops.append(update_grad_to_var(grad, var)) # control dependencies does not work in per replica mode, please change # this once b/118841692 is fixed. # with ops.control_dependencies(update_ops): # apply_updates = self._iterations.assign_add(1).op - apply_updates = merge_update_step(update_ops, self.iteration) + apply_updates = merge_update_step(update_ops, self.iterations) return apply_updates + def get_updates(self, loss, params): + return [self.minimize(loss, params)] + def _set_hyper(self, name, value): """set hyper `name` to value. value can be callable, tensor, numeric.""" if name not in self._hyper: @@ -342,9 +353,14 @@ class OptimizerV2(optimizer_v1.Optimizer): else: backend.set_value(self._hyper[name], value) - def _get_hyper(self, name): + def _get_hyper(self, name, dtype=None): value = self._hyper[name] - return self._call_if_callable(value) + if callable(value): + value = value() + if dtype: + return math_ops.cast(value, dtype) + else: + return value def __getattribute__(self, name): """Overridden to support hyperparameter access.""" @@ -371,12 +387,16 @@ class OptimizerV2(optimizer_v1.Optimizer): else: super(OptimizerV2, self).__setattr__(name, value) - def add_slot(self, var, slot_name): + def add_slot(self, var, slot_name, initializer="zeros"): var_key = _var_key(var) slot_dict = self._slots.setdefault(var_key, {}) if slot_name not in slot_dict: slot_key = _get_slot_key_from_var(var, slot_name) - weight = self.add_weight(name=slot_key, shape=var.shape, dtype=var.dtype) + weight = self.add_weight( + name=slot_key, + shape=var.shape, + dtype=var.dtype, + initializer=initializer) slot_dict[slot_name] = weight self._weights.append(weight) @@ -392,8 +412,10 @@ class OptimizerV2(optimizer_v1.Optimizer): self._iterations = self.add_weight( "iter", shape=[], + dtype=dtypes.int64, trainable=False, - aggregation=variables.VariableAggregation.ONLY_FIRST_REPLICA) + aggregation=tf_variables.VariableAggregation.ONLY_FIRST_REPLICA) + self._weights.append(self._iterations) for name, value in self._hyper.items(): if isinstance(value, ops.Tensor) or callable(value): pass @@ -403,15 +425,24 @@ class OptimizerV2(optimizer_v1.Optimizer): shape=[], trainable=False, initializer=value, - aggregation=variables.VariableAggregation.ONLY_FIRST_REPLICA) + aggregation=tf_variables.VariableAggregation.ONLY_FIRST_REPLICA) self._prepared = True @property - def iteration(self): + def iterations(self): if not self._prepared: self._prepare() return self._iterations + def _decayed_lr(self, var_dtype): + """Get decayed learning rate as a Tensor with dtype=var_dtype.""" + lr_t = self._get_hyper("learning_rate", var_dtype) + if self._initial_decay > 0.: + local_step = math_ops.cast(self.iterations, var_dtype) + decay_t = self._get_hyper("decay", var_dtype) + lr_t = lr_t / (1. + decay_t * local_step) + return lr_t + @abc.abstractmethod def get_config(self): """Returns the config of the optimimizer. @@ -443,6 +474,8 @@ class OptimizerV2(optimizer_v1.Optimizer): Returns: An optimizer instance. """ + if "lr" in config: + config["learning_rate"] = config.pop("lr") return cls(**config) def _serialize_hyperparameter(self, hyperparameter_name): @@ -450,10 +483,14 @@ class OptimizerV2(optimizer_v1.Optimizer): value = self._get_hyper(hyperparameter_name) if callable(value): return value() - if isinstance(value, (ops.Tensor, variables.Variable)): + if isinstance(value, (ops.Tensor, tf_variables.Variable)): return backend.get_value(value) return value + def variables(self): + """Returns variables of this Optimizer based on the order created.""" + return self._weights + @property def weights(self): """Returns variables of this Optimizer based on the order created.""" @@ -490,15 +527,15 @@ class OptimizerV2(optimizer_v1.Optimizer): dtype=None, initializer="zeros", trainable=None, - synchronization=variables.VariableSynchronization.AUTO, - aggregation=variables.VariableAggregation.NONE): + synchronization=tf_variables.VariableSynchronization.AUTO, + aggregation=tf_variables.VariableAggregation.NONE): if dtype is None: dtype = dtypes.float32 if isinstance(initializer, six.string_types) or callable(initializer): initializer = initializers.get(initializer) - if synchronization == variables.VariableSynchronization.ON_READ: + if synchronization == tf_variables.VariableSynchronization.ON_READ: if trainable: raise ValueError( "Synchronization value can be set to " @@ -514,7 +551,7 @@ class OptimizerV2(optimizer_v1.Optimizer): variable = self._add_variable_with_custom_getter( name=name, shape=shape, - getter=base_layer.make_variable, + getter=base_layer_utils.make_variable, overwrite=True, initializer=initializer, dtype=dtype, @@ -522,6 +559,7 @@ class OptimizerV2(optimizer_v1.Optimizer): use_resource=True, synchronization=synchronization, aggregation=aggregation) + backend.track_variable(variable) return variable @@ -561,7 +599,7 @@ def merge_update_step(update_ops, local_step): return incre_op return distribution_strategy_context.get_replica_context().merge_call( - merge_update_step_fn, update_ops, local_step) + merge_update_step_fn, args=(update_ops, local_step)) def merge_grads(grads_and_vars): @@ -569,11 +607,11 @@ def merge_grads(grads_and_vars): def merge_grad_fn(strategy, grads_and_vars): reduced_grads = strategy.batch_reduce( - variable_scope.VariableAggregation.MEAN, grads_and_vars) + ds_reduce_util.ReduceOp.MEAN, grads_and_vars) return reduced_grads return distribution_strategy_context.get_replica_context().merge_call( - merge_grad_fn, grads_and_vars) + merge_grad_fn, args=(grads_and_vars,)) def _var_key(var): diff --git a/tensorflow/python/keras/optimizer_v2/optimizer_v2_test.py b/tensorflow/python/keras/optimizer_v2/optimizer_v2_test.py index 682deda23f0..158577fe64a 100644 --- a/tensorflow/python/keras/optimizer_v2/optimizer_v2_test.py +++ b/tensorflow/python/keras/optimizer_v2/optimizer_v2_test.py @@ -18,6 +18,13 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function +import os +import tempfile + +from absl.testing import parameterized +import numpy as np + +from tensorflow.python import keras from tensorflow.python.eager import context from tensorflow.python.eager import def_function from tensorflow.python.eager import function @@ -25,15 +32,27 @@ from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops from tensorflow.python.framework import test_util +from tensorflow.python.keras import backend +from tensorflow.python.keras import callbacks +from tensorflow.python.keras import optimizers +from tensorflow.python.keras import testing_utils +from tensorflow.python.keras.engine import input_layer +from tensorflow.python.keras.engine import saving +from tensorflow.python.keras.engine import sequential +from tensorflow.python.keras.engine import training +from tensorflow.python.keras.layers import core from tensorflow.python.keras.optimizer_v2 import adam from tensorflow.python.keras.optimizer_v2 import gradient_descent +from tensorflow.python.keras.optimizer_v2 import optimizer_v2 from tensorflow.python.ops import array_ops from tensorflow.python.ops import clip_ops from tensorflow.python.ops import gradients_impl from tensorflow.python.ops import resource_variable_ops from tensorflow.python.ops import state_ops from tensorflow.python.ops import variables +from tensorflow.python.platform import gfile from tensorflow.python.platform import test +from tensorflow.python.training import momentum class OptimizerTest(test.TestCase): @@ -279,8 +298,8 @@ class OptimizerTest(test.TestCase): def testIterationWithoutMinimize(self): with self.cached_session(): sgd = gradient_descent.SGD(3.0) - self.evaluate(sgd.iteration.initializer) - self.assertEqual(0, self.evaluate(sgd.iteration)) + self.evaluate(sgd.iterations.initializer) + self.assertEqual(0, self.evaluate(sgd.iterations)) @test_util.run_in_graph_and_eager_modes def testSerializationWithinDefun(self): @@ -341,8 +360,8 @@ class OptimizerTest(test.TestCase): opt2.set_weights(weights) self.evaluate([opt_op_1, opt_op_2]) self.assertAllClose(self.evaluate(var1), self.evaluate(var2)) - self.assertEqual(1, self.evaluate(opt1.iteration)) - self.assertEqual(1, self.evaluate(opt2.iteration)) + self.assertEqual(1, self.evaluate(opt1.iterations)) + self.assertEqual(1, self.evaluate(opt2.iterations)) var3 = resource_variable_ops.ResourceVariable([1.0, 2.0, 3.0], dtype=dtypes.float32) @@ -394,7 +413,231 @@ class OptimizerTest(test.TestCase): with self.assertRaises(AttributeError): opt.not_an_attr += 3 - def testOptimizerWithFunction(self): + @test_util.run_in_graph_and_eager_modes + def testOptimizerWithKerasModel(self): + a = input_layer.Input(shape=(3,), name='input_a') + b = input_layer.Input(shape=(3,), name='input_b') + + dense = core.Dense(4, name='dense') + c = dense(a) + d = dense(b) + e = core.Dropout(0.5, name='dropout')(c) + + model = training.Model([a, b], [d, e]) + + optimizer = gradient_descent.SGD(learning_rate=0.001) + loss = 'mse' + model.compile(optimizer, loss, metrics=['mae']) + + input_a_np = np.random.random((10, 3)) + input_b_np = np.random.random((10, 3)) + + output_d_np = np.random.random((10, 4)) + output_e_np = np.random.random((10, 4)) + + model.fit([input_a_np, input_b_np], [output_d_np, output_e_np], + epochs=1, + batch_size=5) + + @test_util.run_in_graph_and_eager_modes + def testOptimizerWithCallbacks(self): + input_np = np.random.random((10, 3)) + output_np = np.random.random((10, 4)) + a = input_layer.Input(shape=(3,), name='input_a') + model = sequential.Sequential() + model.add(core.Dense(4, name='dense')) + model.add(core.Dropout(0.5, name='dropout')) + model(a) + optimizer = gradient_descent.SGD(learning_rate=0.1) + model.compile(optimizer, loss='mse', metrics=['mae']) + # This does not reduce the LR after the first epoch (due to low delta). + cbks = [ + callbacks.ReduceLROnPlateau( + monitor='val_loss', factor=0.1, min_delta=0, patience=1, cooldown=5) + ] + model.fit( + input_np, + output_np, + batch_size=10, + validation_data=(input_np, output_np), + callbacks=cbks, + epochs=5, + verbose=0) + self.assertAllClose( + float(backend.get_value(model.optimizer.lr)), 0.1, atol=1e-4) + + # This should reduce the LR after the first epoch (due to high delta). + cbks = [ + callbacks.ReduceLROnPlateau( + monitor='val_loss', + factor=0.1, + min_delta=10, + patience=1, + cooldown=5) + ] + model.fit( + input_np, + output_np, + batch_size=10, + validation_data=(input_np, output_np), + callbacks=cbks, + epochs=5, + verbose=2) + self.assertAllClose( + float(backend.get_value(model.optimizer.lr)), 0.01, atol=1e-4) + + +class OptimizersCompatibilityTest(test.TestCase, parameterized.TestCase): + + # TODO(tanzheny): remove test_numeric after algorithm for Momentum, Adam and + # NAdam has been unified: currently these three algorithms behave differently. + @parameterized.named_parameters( + ('adadelta', 'adadelta', True, True), ('adagrad', 'adagrad', True, True), + ('adam', 'adam', True, True), ('adamax', 'adamax', True, True), + ('nadam', 'nadam', True, False), ('momentum', 'momentum', True, True), + ('sgd', 'sgd', False, True)) + def testOptimizersCompatibility(self, opt_str, test_weights, test_numeric): + np.random.seed(1331) + with self.cached_session(): + train_samples = 20 + input_dim = 3 + num_classes = 2 + (x, y), _ = testing_utils.get_test_data( + train_samples=train_samples, + test_samples=10, + input_shape=(input_dim,), + num_classes=num_classes) + y = keras.utils.to_categorical(y) + + num_hidden = 5 + model = testing_utils.get_small_sequential_mlp( + num_hidden=num_hidden, num_classes=num_classes, input_dim=input_dim) + + old_mode = os.environ.get('TF2_BEHAVIOR', None) + # Disable tf2 to create V1 optimizer. + disable_tf2() + if opt_str == 'momentum': + opt_v1 = optimizers.SGD(momentum=0.9) + else: + opt_v1 = optimizers.get(opt_str) + + # Test compile and fit with v1 optimizer. + model.compile(opt_v1, loss='categorical_crossentropy', metrics=[]) + model.fit(x, y, batch_size=5, epochs=1) + model_dir = tempfile.mkdtemp() + gfile.MakeDirs(model_dir) + file_name = os.path.join(model_dir, 'model.h5') + model.save(file_name) + + enable_tf2() + # Test load and fit with v2 optimizer. + model_2 = saving.load_model(file_name) + opt_v2 = model_2.optimizer + self.assertIsInstance(opt_v2, optimizer_v2.OptimizerV2) + # set_weights is called inside load_model but exception is swallowed, + # this call checks the weights can be set correctly. + if test_weights: + opt_v2.set_weights(opt_v1.get_weights()) + if test_numeric: + hist_1 = model.fit(x, y, batch_size=5, epochs=1, shuffle=False) + hist_2 = model_2.fit(x, y, batch_size=5, epochs=1, shuffle=False) + self.assertAllClose(model.get_weights(), model_2.get_weights()) + self.assertAllClose(model.get_weights(), model_2.get_weights()) + self.assertAllClose(hist_1.history['loss'], hist_2.history['loss']) + + if old_mode is not None: + os.environ['TF2_BEHAVIOR'] = old_mode + + def testNumericEquivalenceForNesterovMomentum(self): + np.random.seed(1331) + with self.cached_session(): + train_samples = 20 + input_dim = 3 + num_classes = 2 + (x, y), _ = testing_utils.get_test_data( + train_samples=train_samples, + test_samples=10, + input_shape=(input_dim,), + num_classes=num_classes) + y = keras.utils.to_categorical(y) + + num_hidden = 5 + model_k_v1 = testing_utils.get_small_sequential_mlp( + num_hidden=num_hidden, num_classes=num_classes, input_dim=input_dim) + model_k_v2 = testing_utils.get_small_sequential_mlp( + num_hidden=num_hidden, num_classes=num_classes, input_dim=input_dim) + model_k_v2.set_weights(model_k_v1.get_weights()) + model_tf = testing_utils.get_small_sequential_mlp( + num_hidden=num_hidden, num_classes=num_classes, input_dim=input_dim) + model_tf.set_weights(model_k_v2.get_weights()) + + opt_k_v1 = optimizers.SGD(lr=0.001, momentum=0.9, nesterov=True) + opt_k_v2 = gradient_descent.SGD(momentum=0.9, nesterov=True) + opt_tf = momentum.MomentumOptimizer( + learning_rate=0.001, momentum=0.9, use_nesterov=True) + + model_k_v1.compile(opt_k_v1, loss='categorical_crossentropy', metrics=[]) + model_k_v2.compile(opt_k_v2, loss='categorical_crossentropy', metrics=[]) + model_tf.compile(opt_tf, loss='categorical_crossentropy', metrics=[]) + + hist_k_v1 = model_k_v1.fit(x, y, batch_size=5, epochs=10, shuffle=False) + hist_k_v2 = model_k_v2.fit(x, y, batch_size=5, epochs=10, shuffle=False) + hist_tf = model_tf.fit(x, y, batch_size=5, epochs=10, shuffle=False) + + self.assertAllClose(model_k_v1.get_weights(), model_tf.get_weights()) + self.assertAllClose(model_k_v1.get_weights(), model_k_v2.get_weights()) + self.assertAllClose(opt_k_v1.get_weights(), opt_k_v2.get_weights()) + self.assertAllClose(hist_k_v1.history['loss'], hist_tf.history['loss']) + self.assertAllClose(hist_k_v1.history['loss'], hist_k_v2.history['loss']) + + def testNumericEquivalenceForAmsgrad(self): + np.random.seed(1331) + with self.cached_session(): + train_samples = 20 + input_dim = 3 + num_classes = 2 + (x, y), _ = testing_utils.get_test_data( + train_samples=train_samples, + test_samples=10, + input_shape=(input_dim,), + num_classes=num_classes) + y = keras.utils.to_categorical(y) + + num_hidden = 5 + model_k_v1 = testing_utils.get_small_sequential_mlp( + num_hidden=num_hidden, num_classes=num_classes, input_dim=input_dim) + model_k_v2 = testing_utils.get_small_sequential_mlp( + num_hidden=num_hidden, num_classes=num_classes, input_dim=input_dim) + model_k_v2.set_weights(model_k_v1.get_weights()) + + opt_k_v1 = optimizers.Adam(amsgrad=True) + opt_k_v2 = adam.Adam(amsgrad=True) + + model_k_v1.compile(opt_k_v1, loss='categorical_crossentropy', metrics=[]) + model_k_v2.compile(opt_k_v2, loss='categorical_crossentropy', metrics=[]) + + hist_k_v1 = model_k_v1.fit(x, y, batch_size=5, epochs=10, shuffle=False) + hist_k_v2 = model_k_v2.fit(x, y, batch_size=5, epochs=10, shuffle=False) + + self.assertAllClose(model_k_v1.get_weights(), model_k_v2.get_weights()) + self.assertAllClose(opt_k_v1.get_weights(), opt_k_v2.get_weights()) + self.assertAllClose(hist_k_v1.history['loss'], hist_k_v2.history['loss']) + + +def disable_tf2(): + if 'TF2_BEHAVIOR' in os.environ: + del os.environ['TF2_BEHAVIOR'] + + +def enable_tf2(): + os.environ['TF2_BEHAVIOR'] = 'enabled' + + +# Note: These tests are kept in a separate class to avoid bugs in some +# distributions of Python that break AutoGraph which is used by tf.function. +class OptimizerWithFunctionTest(test.TestCase): + + def testBasic(self): with context.eager_mode(): var = resource_variable_ops.ResourceVariable([1.0, 2.0], dtype=dtypes.float32) @@ -406,10 +649,8 @@ class OptimizerTest(test.TestCase): opt.minimize(loss, [var]) return var - self.assertAllClose([0., 1.], fn()) - # This is just to test tf.function. The values needs to be updated - # when adam updates beta_1_power. - self.assertAllClose([-1.343838, -0.343838], fn()) + self.assertAllClose([0., 1.], fn(), atol=1e-4) + self.assertAllClose([-1, 0.], fn(), atol=1e-4) if __name__ == '__main__': diff --git a/tensorflow/python/keras/optimizer_v2/rmsprop.py b/tensorflow/python/keras/optimizer_v2/rmsprop.py new file mode 100644 index 00000000000..6a5b334fc46 --- /dev/null +++ b/tensorflow/python/keras/optimizer_v2/rmsprop.py @@ -0,0 +1,196 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""RMSprop for TensorFlow.""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from tensorflow.python.framework import ops +from tensorflow.python.keras.optimizer_v2 import optimizer_v2 +from tensorflow.python.training import training_ops + + +class RMSprop(optimizer_v2.OptimizerV2): + r"""Optimizer that implements the RMSprop algorithm. + + A detailed description of rmsprop. + + - maintain a moving (discounted) average of the square of gradients + - divide gradient by the root of this average + + $$mean_square_t = rho * mean_square{t-1} + (1-rho) * gradient ** 2$$ + $$mom_t = momentum * mom_{t-1} + learning_rate * gradient / \sqrt{ / + mean_square_t + \epsilon}$$ + $$variable_t := variable_{t-1} - mom_t + + This implementation of RMSprop uses plain momentum, not Nesterov momentum. + + The centered version additionally maintains a moving average of the + gradients, and uses that average to estimate the variance: + + $$mean_grad_t = rho * mean_grad_{t-1} + (1-rho) * gradient$$ + $$mean_square_t = rho * mean_square_{t-1} + (1-rho) * gradient ** 2$$ + $$mom_t = momentum * mom_{t-1} + learning_rate * gradient / + sqrt(mean_square_t - mean_grad_t**2 + epsilon)$$ + $$variable_t := variable_{t-1} - mom_t + + References + See ([pdf] + http://www.cs.toronto.edu/~tijmen/csc321/slides/lecture_slides_lec6.pdf). + """ + + def __init__(self, + learning_rate=0.001, + rho=0.9, + momentum=0.0, + epsilon=1e-7, + centered=False, + name="RMSprop", + **kwargs): + """Construct a new RMSprop optimizer. + + Note that in the dense implementation of this algorithm, variables and their + corresponding accumulators (momentum, gradient moving average, square + gradient moving average) will be updated even if the gradient is zero + (i.e. accumulators will decay, momentum will be applied). The sparse + implementation (used when the gradient is an `IndexedSlices` object, + typically because of `tf.gather` or an embedding lookup in the forward pass) + will not update variable slices or their accumulators unless those slices + were used in the forward pass (nor is there an "eventual" correction to + account for these omitted updates). This leads to more efficient updates for + large embedding lookup tables (where most of the slices are not accessed in + a particular graph execution), but differs from the published algorithm. + + Args: + learning_rate: A Tensor or a floating point value. The learning rate. + rho: Discounting factor for the history/coming gradient + momentum: A scalar tensor. + epsilon: Small value to avoid zero denominator. + centered: If True, gradients are normalized by the estimated variance of + the gradient; if False, by the uncentered second moment. Setting this to + True may help with training, but is slightly more expensive in terms of + computation and memory. Defaults to False. + name: Optional name prefix for the operations created when applying + gradients. Defaults to "RMSprop". @compatibility(eager) When eager + execution is enabled, `learning_rate`, `decay`, `momentum`, and + `epsilon` can each be a callable that takes no arguments and returns the + actual value to use. This can be useful for changing these values across + different invocations of optimizer functions. @end_compatibility + **kwargs: keyword arguments. Allowed to be {`decay`} + """ + super(RMSprop, self).__init__(name, **kwargs) + self._set_hyper("learning_rate", learning_rate) + self._set_hyper("decay", self._initial_decay) + self._set_hyper("rho", rho) + + self._momentum = False + if isinstance(momentum, ops.Tensor) or callable(momentum) or momentum > 0: + self._momentum = True + if isinstance(momentum, (int, float)) and (momentum < 0 or momentum > 1): + raise ValueError("`momentum` must be between [0, 1].") + self._set_hyper("momentum", momentum) + + self._set_hyper("epsilon", epsilon) + self._centered = centered + + def _create_slots(self, var_list): + for var in var_list: + self.add_slot(var, "rms") + self.add_slot(var, "momentum") + if self._centered: + self.add_slot(var, "mg") + + def _resource_apply_dense(self, grad, var): + var_dtype = var.dtype.base_dtype + lr_t = self._decayed_lr(var_dtype) + rms = self.get_slot(var, "rms") + mom = self.get_slot(var, "momentum") + rho = self._get_hyper("rho", var_dtype) + momentum = self._get_hyper("momentum", var_dtype) + epsilon = self._get_hyper("epsilon", var_dtype) + if self._centered: + mg = self.get_slot(var, "mg") + return training_ops.resource_apply_centered_rms_prop( + var.handle, + mg.handle, + rms.handle, + mom.handle, + lr_t, + rho, + momentum, + epsilon, + grad, + use_locking=self._use_locking) + else: + return training_ops.resource_apply_rms_prop( + var.handle, + rms.handle, + mom.handle, + lr_t, + rho, + momentum, + epsilon, + grad, + use_locking=self._use_locking) + + def _resource_apply_sparse(self, grad, var, indices): + var_dtype = var.dtype.base_dtype + lr_t = self._decayed_lr(var_dtype) + rms = self.get_slot(var, "rms") + mom = self.get_slot(var, "momentum") + rho = self._get_hyper("rho", var_dtype) + momentum = self._get_hyper("momentum", var_dtype) + epsilon = self._get_hyper("epsilon", var_dtype) + if self._centered: + mg = self.get_slot(var, "mg") + return training_ops.resource_sparse_apply_centered_rms_prop( + var.handle, + mg.handle, + rms.handle, + mom.handle, + lr_t, + rho, + momentum, + epsilon, + grad, + indices, + use_locking=self._use_locking) + else: + return training_ops.resource_sparse_apply_rms_prop( + var.handle, + rms.handle, + mom.handle, + lr_t, + rho, + momentum, + epsilon, + grad, + indices, + use_locking=self._use_locking) + + def get_config(self): + config = super(RMSprop, self).get_config() + config.update({ + "learning_rate": self._serialize_hyperparameter("learning_rate"), + "decay": self._serialize_hyperparameter("decay"), + "rho": self._serialize_hyperparameter("rho"), + "momentum": self._serialize_hyperparameter("momentum"), + "epsilon": self._serialize_hyperparameter("epsilon"), + "centered": self._centered, + }) + return config + + +RMSProp = RMSprop diff --git a/tensorflow/python/keras/optimizer_v2/rmsprop_test.py b/tensorflow/python/keras/optimizer_v2/rmsprop_test.py new file mode 100644 index 00000000000..a8658a85507 --- /dev/null +++ b/tensorflow/python/keras/optimizer_v2/rmsprop_test.py @@ -0,0 +1,410 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Tests for rmsprop.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import copy +import itertools +import math + +import numpy as np + +from tensorflow.python.eager import context +from tensorflow.python.framework import constant_op +from tensorflow.python.framework import dtypes +from tensorflow.python.framework import ops +from tensorflow.python.framework import test_util +from tensorflow.python.keras.optimizer_v2 import rmsprop +from tensorflow.python.ops import embedding_ops +from tensorflow.python.ops import math_ops +from tensorflow.python.ops import resource_variable_ops +from tensorflow.python.ops import variables +from tensorflow.python.platform import test + +_DATA_TYPES = [dtypes.half, dtypes.float32] + +_TEST_PARAM_VALUES = [ + # learning_rate, rho, momentum, epsilon, centered + [0.05, 0.9, 0.0, 1e-3, True], + [0.05, 0.9, 0.0, 1e-3, False], + [0.1, 0.9, 0.0, 1e-3, True], + [0.01, 0.9, 0.0, 1e-5, True], + [0.01, 0.9, 0.9, 1e-5, True], +] + +_TESTPARAMS = [ + [data_type] + values + for data_type, values in itertools.product(_DATA_TYPES, _TEST_PARAM_VALUES) +] + + +class RMSpropOptimizerTest(test.TestCase): + + def _rmsprop_update_numpy(self, var, g, mg, rms, mom, lr, rho, momentum, + epsilon, centered): + rms_t = rms * rho + (1 - rho) * g * g + denom_t = rms_t + epsilon + if centered: + mg_t = mg * rho + (1 - rho) * g + denom_t -= mg_t * mg_t + else: + mg_t = mg + mom_t = momentum * mom + lr * g / np.sqrt(denom_t, dtype=denom_t.dtype) + var_t = var - mom_t + return var_t, mg_t, rms_t, mom_t + + def _sparse_rmsprop_update_numpy(self, var, gindexs, gvalues, mg, rms, mom, + lr, rho, momentum, epsilon, centered): + mg_t = copy.deepcopy(mg) + rms_t = copy.deepcopy(rms) + mom_t = copy.deepcopy(mom) + var_t = copy.deepcopy(var) + for i in range(len(gindexs)): + gindex = gindexs[i] + gvalue = gvalues[i] + rms_t[gindex] = rms[gindex] * rho + (1 - rho) * gvalue * gvalue + denom_t = rms_t[gindex] + epsilon + if centered: + mg_t[gindex] = mg_t[gindex] * rho + (1 - rho) * gvalue + denom_t -= mg_t[gindex] * mg_t[gindex] + mom_t[gindex] = momentum * mom[gindex] + lr * gvalue / np.sqrt(denom_t) + var_t[gindex] = var[gindex] - mom_t[gindex] + return var_t, mg_t, rms_t, mom_t + + @test_util.run_deprecated_v1 + def testDense(self): + for (dtype, learning_rate, rho, momentum, epsilon, centered) in _TESTPARAMS: + with test_util.use_gpu(): + # Initialize variables for numpy implementation. + var0_np = np.array([1.0, 2.0], dtype=dtype.as_numpy_dtype) + grads0_np = np.array([0.1, 0.2], dtype=dtype.as_numpy_dtype) + var1_np = np.array([3.0, 4.0], dtype=dtype.as_numpy_dtype) + grads1_np = np.array([0.01, 0.2], dtype=dtype.as_numpy_dtype) + + var0 = resource_variable_ops.ResourceVariable(var0_np, dtype=dtype) + var1 = resource_variable_ops.ResourceVariable(var1_np, dtype=dtype) + grads0 = constant_op.constant(grads0_np, dtype=dtype) + grads1 = constant_op.constant(grads1_np, dtype=dtype) + opt = rmsprop.RMSprop( + learning_rate=learning_rate, + rho=rho, + momentum=momentum, + epsilon=epsilon, + centered=centered) + + update = opt.apply_gradients(zip([grads0, grads1], [var0, var1])) + self.evaluate(variables.global_variables_initializer()) + + if centered: + mg0 = opt.get_slot(var0, "mg") + mg1 = opt.get_slot(var1, "mg") + else: + mg0 = None + mg1 = None + + rms0 = opt.get_slot(var0, "rms") + self.assertTrue(rms0 is not None) + rms1 = opt.get_slot(var1, "rms") + self.assertTrue(rms1 is not None) + mom0 = opt.get_slot(var0, "momentum") + self.assertTrue(mom0 is not None) + mom1 = opt.get_slot(var1, "momentum") + self.assertTrue(mom1 is not None) + + mg0_np = np.array([0.0, 0.0], dtype=dtype.as_numpy_dtype) + mg1_np = np.array([0.0, 0.0], dtype=dtype.as_numpy_dtype) + rms0_np = np.array([0.0, 0.0], dtype=dtype.as_numpy_dtype) + rms1_np = np.array([0.0, 0.0], dtype=dtype.as_numpy_dtype) + mom0_np = np.array([0.0, 0.0], dtype=dtype.as_numpy_dtype) + mom1_np = np.array([0.0, 0.0], dtype=dtype.as_numpy_dtype) + + # Fetch params to validate initial values + self.assertAllClose([1.0, 2.0], self.evaluate(var0)) + self.assertAllClose([3.0, 4.0], self.evaluate(var1)) + + # Run 4 steps of RMSprop + for _ in range(1, 5): + self.evaluate(update) + + var0_np, mg0_np, rms0_np, mom0_np = self._rmsprop_update_numpy( + var0_np, grads0_np, mg0_np, rms0_np, mom0_np, learning_rate, rho, + momentum, epsilon, centered) + var1_np, mg1_np, rms1_np, mom1_np = self._rmsprop_update_numpy( + var1_np, grads1_np, mg1_np, rms1_np, mom1_np, learning_rate, rho, + momentum, epsilon, centered) + + # Validate updated params + if centered: + self.assertAllCloseAccordingToType(mg0_np, self.evaluate(mg0)) + self.assertAllCloseAccordingToType(mg1_np, self.evaluate(mg1)) + self.assertAllCloseAccordingToType(rms0_np, self.evaluate(rms0)) + self.assertAllCloseAccordingToType(rms1_np, self.evaluate(rms1)) + self.assertAllCloseAccordingToType(mom0_np, self.evaluate(mom0)) + self.assertAllCloseAccordingToType(mom1_np, self.evaluate(mom1)) + self.assertAllCloseAccordingToType(var0_np, self.evaluate(var0)) + self.assertAllCloseAccordingToType(var1_np, self.evaluate(var1)) + + @test_util.run_deprecated_v1 + def testDenseWithLearningRateDecay(self): + var0_np = np.array([1.0, 2.0]) + grads0_np = np.array([0.1, 0.2]) + var1_np = np.array([3.0, 4.0]) + grads1_np = np.array([0.01, 0.2]) + + var0 = resource_variable_ops.ResourceVariable(var0_np) + var1 = resource_variable_ops.ResourceVariable(var1_np) + grads0 = constant_op.constant(grads0_np) + grads1 = constant_op.constant(grads1_np) + learning_rate = 0.01 + rho = 0.9 + momentum = 0.0 + epsilon = 1e-7 + centered = False + decay = 0.5 + opt = rmsprop.RMSprop( + learning_rate=learning_rate, + rho=rho, + momentum=momentum, + epsilon=epsilon, + centered=centered, + decay=decay) + + update = opt.apply_gradients(zip([grads0, grads1], [var0, var1])) + self.evaluate(variables.global_variables_initializer()) + + rms0 = opt.get_slot(var0, "rms") + self.assertTrue(rms0 is not None) + rms1 = opt.get_slot(var1, "rms") + self.assertTrue(rms1 is not None) + mom0 = opt.get_slot(var0, "momentum") + self.assertTrue(mom0 is not None) + mom1 = opt.get_slot(var1, "momentum") + self.assertTrue(mom1 is not None) + + mg0_np = np.array([0.0, 0.0]) + mg1_np = np.array([0.0, 0.0]) + rms0_np = np.array([0.0, 0.0]) + rms1_np = np.array([0.0, 0.0]) + mom0_np = np.array([0.0, 0.0]) + mom1_np = np.array([0.0, 0.0]) + + # Fetch params to validate initial values + self.assertAllClose([1.0, 2.0], self.evaluate(var0)) + self.assertAllClose([3.0, 4.0], self.evaluate(var1)) + + # Run 4 steps of RMSprop + for t in range(2): + self.evaluate(update) + + lr = learning_rate / (1 + decay * t) + var0_np, mg0_np, rms0_np, mom0_np = self._rmsprop_update_numpy( + var0_np, grads0_np, mg0_np, rms0_np, mom0_np, lr, rho, momentum, + epsilon, centered) + var1_np, mg1_np, rms1_np, mom1_np = self._rmsprop_update_numpy( + var1_np, grads1_np, mg1_np, rms1_np, mom1_np, lr, rho, momentum, + epsilon, centered) + + # Validate updated params + self.assertAllCloseAccordingToType(rms0_np, self.evaluate(rms0)) + self.assertAllCloseAccordingToType(rms1_np, self.evaluate(rms1)) + self.assertAllCloseAccordingToType(mom0_np, self.evaluate(mom0)) + self.assertAllCloseAccordingToType(mom1_np, self.evaluate(mom1)) + self.assertAllCloseAccordingToType(var0_np, self.evaluate(var0)) + self.assertAllCloseAccordingToType(var1_np, self.evaluate(var1)) + + @test_util.run_deprecated_v1 + def testMinimizeSparseResourceVariable(self): + for dtype in [dtypes.float32, dtypes.float64]: + with self.cached_session(): + var0 = resource_variable_ops.ResourceVariable([[1.0, 2.0]], dtype=dtype) + x = constant_op.constant([[4.0], [5.0]], dtype=dtype) + pred = math_ops.matmul(embedding_ops.embedding_lookup([var0], [0]), x) + loss = pred * pred + sgd_op = rmsprop.RMSprop( + learning_rate=1.0, + rho=0.0, + momentum=0.0, + epsilon=0.0, + centered=False).minimize( + loss, var_list=[var0]) + self.evaluate(variables.global_variables_initializer()) + # Fetch params to validate initial values + self.assertAllCloseAccordingToType([[1.0, 2.0]], self.evaluate(var0)) + # Run 1 step of sgd + self.evaluate(sgd_op) + # Validate updated params + self.assertAllCloseAccordingToType([[0., 1.]], + self.evaluate(var0), + atol=0.01) + + @test_util.run_deprecated_v1 + def testMinimizeSparseResourceVariableCentered(self): + for dtype in [dtypes.float32, dtypes.float64]: + with self.cached_session(): + var0 = resource_variable_ops.ResourceVariable([[1.0, 2.0]], dtype=dtype) + x = constant_op.constant([[4.0], [5.0]], dtype=dtype) + pred = math_ops.matmul(embedding_ops.embedding_lookup([var0], [0]), x) + loss = pred * pred + sgd_op = rmsprop.RMSprop( + learning_rate=1.0, + rho=0.0, + momentum=0.0, + epsilon=1.0, + centered=True).minimize( + loss, var_list=[var0]) + self.evaluate(variables.global_variables_initializer()) + # Fetch params to validate initial values + self.assertAllCloseAccordingToType([[1.0, 2.0]], self.evaluate(var0)) + # Run 1 step of sgd + self.evaluate(sgd_op) + # Validate updated params + self.assertAllCloseAccordingToType([[-111, -138]], + self.evaluate(var0), + atol=0.01) + + @test_util.run_deprecated_v1 + def testSparse(self): + for (dtype, learning_rate, rho, momentum, epsilon, centered) in _TESTPARAMS: + with test_util.use_gpu(): + # Initialize variables for numpy implementation. + var0_np = np.array([1.0, 2.0], dtype=dtype.as_numpy_dtype) + grads0_np = np.array([0.1], dtype=dtype.as_numpy_dtype) + var1_np = np.array([3.0, 4.0], dtype=dtype.as_numpy_dtype) + grads1_np = np.array([0.01], dtype=dtype.as_numpy_dtype) + + var0 = variables.Variable(var0_np) + var1 = variables.Variable(var1_np) + grads0_np_indices = np.array([0], dtype=np.int32) + grads0 = ops.IndexedSlices( + constant_op.constant(grads0_np), + constant_op.constant(grads0_np_indices), constant_op.constant([1])) + grads1_np_indices = np.array([1], dtype=np.int32) + grads1 = ops.IndexedSlices( + constant_op.constant(grads1_np), + constant_op.constant(grads1_np_indices), constant_op.constant([1])) + opt = rmsprop.RMSprop( + learning_rate=learning_rate, + rho=rho, + momentum=momentum, + epsilon=epsilon, + centered=centered) + update = opt.apply_gradients(zip([grads0, grads1], [var0, var1])) + self.evaluate(variables.global_variables_initializer()) + + if centered: + mg0 = opt.get_slot(var0, "mg") + self.assertEqual(mg0 is not None, centered) + mg1 = opt.get_slot(var1, "mg") + self.assertEqual(mg1 is not None, centered) + else: + mg0 = None + mg1 = None + rms0 = opt.get_slot(var0, "rms") + self.assertTrue(rms0 is not None) + rms1 = opt.get_slot(var1, "rms") + self.assertTrue(rms1 is not None) + mom0 = opt.get_slot(var0, "momentum") + self.assertTrue(mom0 is not None) + mom1 = opt.get_slot(var1, "momentum") + self.assertTrue(mom1 is not None) + + mg0_np = np.array([0.0, 0.0], dtype=dtype.as_numpy_dtype) + mg1_np = np.array([0.0, 0.0], dtype=dtype.as_numpy_dtype) + rms0_np = np.array([0.0, 0.0], dtype=dtype.as_numpy_dtype) + rms1_np = np.array([0.0, 0.0], dtype=dtype.as_numpy_dtype) + mom0_np = np.array([0.0, 0.0], dtype=dtype.as_numpy_dtype) + mom1_np = np.array([0.0, 0.0], dtype=dtype.as_numpy_dtype) + + # Fetch params to validate initial values + self.assertAllClose([1.0, 2.0], self.evaluate(var0)) + self.assertAllClose([3.0, 4.0], self.evaluate(var1)) + + # Run 4 steps of RMSprop + for _ in range(1, 5): + self.evaluate(update) + + var0_np, mg0_np, rms0_np, mom0_np = self._sparse_rmsprop_update_numpy( + var0_np, grads0_np_indices, grads0_np, mg0_np, rms0_np, mom0_np, + learning_rate, rho, momentum, epsilon, centered) + var1_np, mg1_np, rms1_np, mom1_np = self._sparse_rmsprop_update_numpy( + var1_np, grads1_np_indices, grads1_np, mg1_np, rms1_np, mom1_np, + learning_rate, rho, momentum, epsilon, centered) + + # Validate updated params + if centered: + self.assertAllCloseAccordingToType(mg0_np, self.evaluate(mg0)) + self.assertAllCloseAccordingToType(mg1_np, self.evaluate(mg1)) + self.assertAllCloseAccordingToType(rms0_np, self.evaluate(rms0)) + self.assertAllCloseAccordingToType(rms1_np, self.evaluate(rms1)) + self.assertAllCloseAccordingToType(mom0_np, self.evaluate(mom0)) + self.assertAllCloseAccordingToType(mom1_np, self.evaluate(mom1)) + self.assertAllCloseAccordingToType(var0_np, self.evaluate(var0)) + self.assertAllCloseAccordingToType(var1_np, self.evaluate(var1)) + + def testCallableParams(self): + with context.eager_mode(): + for dtype in [dtypes.half, dtypes.float32]: + var0 = resource_variable_ops.ResourceVariable([1.0, 2.0], dtype=dtype) + var1 = resource_variable_ops.ResourceVariable([3.0, 4.0], dtype=dtype) + grads0 = constant_op.constant([0.1, 0.1], dtype=dtype) + grads1 = constant_op.constant([0.01, 0.01], dtype=dtype) + + learning_rate = lambda: 2.0 + rho = lambda: 0.9 + momentum = lambda: 0.0 + epsilon = lambda: 1.0 + opt = rmsprop.RMSprop(learning_rate, rho, momentum, epsilon) + + # Fetch params to validate initial values + self.assertAllClose([1.0, 2.0], self.evaluate(var0)) + self.assertAllClose([3.0, 4.0], self.evaluate(var1)) + # Step 1: the rms accumulators where 1. So we should see a normal + # update: v -= grad * learning_rate + opt.apply_gradients(zip([grads0, grads1], [var0, var1])) + # Check the parameters. + self.assertAllCloseAccordingToType( + np.array([ + 1.0 - (0.1 * 2.0 / math.sqrt(0.001 + 1.0)), + 2.0 - (0.1 * 2.0 / math.sqrt(0.001 + 1.0)) + ]), self.evaluate(var0)) + self.assertAllCloseAccordingToType( + np.array([ + 3.0 - (0.01 * 2.0 / math.sqrt(0.00001 + 1.0)), + 4.0 - (0.01 * 2.0 / math.sqrt(0.00001 + 1.0)) + ]), self.evaluate(var1)) + # Step 2: the root mean square accumulators contain the previous update. + opt.apply_gradients(zip([grads0, grads1], [var0, var1])) + # Check the parameters. + self.assertAllCloseAccordingToType( + np.array([ + 1.0 - (0.1 * 2.0 / math.sqrt(0.001 + 1.0)) - + (0.1 * 2.0 / math.sqrt(0.001 * 0.9 + 0.001 + 1.0)), + 2.0 - (0.1 * 2.0 / math.sqrt(0.001 + 1.0)) - + (0.1 * 2.0 / math.sqrt(0.001 * 0.9 + 0.001 + 1.0)) + ]), self.evaluate(var0)) + self.assertAllCloseAccordingToType( + np.array([ + 3.0 - (0.01 * 2.0 / math.sqrt(0.00001 + 1.0)) - + (0.01 * 2.0 / math.sqrt(0.00001 * 0.9 + 1e-5 + 1.0)), + 4.0 - (0.01 * 2.0 / math.sqrt(0.00001 + 1.0)) - + (0.01 * 2.0 / math.sqrt(0.00001 * 0.9 + 1e-5 + 1.0)) + ]), self.evaluate(var1)) + + +if __name__ == "__main__": + test.main() diff --git a/tensorflow/python/keras/optimizers.py b/tensorflow/python/keras/optimizers.py index 715d80a116c..10466eb573d 100644 --- a/tensorflow/python/keras/optimizers.py +++ b/tensorflow/python/keras/optimizers.py @@ -22,7 +22,17 @@ from __future__ import print_function import six from six.moves import zip # pylint: disable=redefined-builtin +from tensorflow.python import tf2 +from tensorflow.python.framework import ops from tensorflow.python.keras import backend as K +from tensorflow.python.keras.optimizer_v2 import adadelta as adadelta_v2 +from tensorflow.python.keras.optimizer_v2 import adagrad as adagrad_v2 +from tensorflow.python.keras.optimizer_v2 import adam as adam_v2 +from tensorflow.python.keras.optimizer_v2 import adamax as adamax_v2 +from tensorflow.python.keras.optimizer_v2 import gradient_descent as gradient_descent_v2 +from tensorflow.python.keras.optimizer_v2 import nadam as nadam_v2 +from tensorflow.python.keras.optimizer_v2 import optimizer_v2 +from tensorflow.python.keras.optimizer_v2 import rmsprop as rmsprop_v2 from tensorflow.python.keras.utils.generic_utils import deserialize_keras_object from tensorflow.python.keras.utils.generic_utils import serialize_keras_object from tensorflow.python.ops import clip_ops @@ -473,7 +483,7 @@ class Adam(Optimizer): def get_updates(self, loss, params): grads = self.get_gradients(loss, params) - self.updates = [state_ops.assign_add(self.iterations, 1)] + self.updates = [] lr = self.lr if self.initial_decay > 0: @@ -481,7 +491,8 @@ class Adam(Optimizer): 1. / (1. + self.decay * math_ops.cast(self.iterations, K.dtype(self.decay)))) - t = math_ops.cast(self.iterations, K.floatx()) + 1 + with ops.control_dependencies([state_ops.assign_add(self.iterations, 1)]): + t = math_ops.cast(self.iterations, K.floatx()) lr_t = lr * ( K.sqrt(1. - math_ops.pow(self.beta_2, t)) / (1. - math_ops.pow(self.beta_1, t))) @@ -795,16 +806,27 @@ def deserialize(config, custom_objects=None): Returns: A Keras Optimizer instance. """ - all_classes = { - 'sgd': SGD, - 'rmsprop': RMSprop, - 'adagrad': Adagrad, - 'adadelta': Adadelta, - 'adam': Adam, - 'adamax': Adamax, - 'nadam': Nadam, - 'tfoptimizer': TFOptimizer, - } + if tf2.enabled(): + all_classes = { + 'adadelta': adadelta_v2.Adadelta, + 'adagrad': adagrad_v2.Adagrad, + 'adam': adam_v2.Adam, + 'adamax': adamax_v2.Adamax, + 'nadam': nadam_v2.Nadam, + 'rmsprop': rmsprop_v2.RMSprop, + 'sgd': gradient_descent_v2.SGD + } + else: + all_classes = { + 'adadelta': Adadelta, + 'adagrad': Adagrad, + 'adam': Adam, + 'adamax': Adamax, + 'nadam': Nadam, + 'rmsprop': RMSprop, + 'sgd': SGD, + 'tfoptimizer': TFOptimizer + } # Make deserialization case-insensitive for built-in optimizers. if config['class_name'].lower() in all_classes: config['class_name'] = config['class_name'].lower() @@ -833,17 +855,17 @@ def get(identifier): Raises: ValueError: If `identifier` cannot be interpreted. """ + if isinstance(identifier, (Optimizer, optimizer_v2.OptimizerV2)): + return identifier # Wrap TF optimizer instances - if isinstance(identifier, tf_optimizer_module.Optimizer): + elif isinstance(identifier, tf_optimizer_module.Optimizer): opt = TFOptimizer(identifier) K.track_tf_optimizer(opt) return opt - if isinstance(identifier, dict): + elif isinstance(identifier, dict): return deserialize(identifier) elif isinstance(identifier, six.string_types): config = {'class_name': str(identifier), 'config': {}} return deserialize(config) - if isinstance(identifier, Optimizer): - return identifier else: raise ValueError('Could not interpret optimizer identifier:', identifier) diff --git a/tensorflow/python/keras/optimizers_test.py b/tensorflow/python/keras/optimizers_test.py index 9664f09fff0..d3cacb702c9 100644 --- a/tensorflow/python/keras/optimizers_test.py +++ b/tensorflow/python/keras/optimizers_test.py @@ -19,11 +19,14 @@ from __future__ import division from __future__ import print_function import gc +import os import weakref +from absl.testing import parameterized import numpy as np from tensorflow.python import keras +from tensorflow.python import tf2 from tensorflow.python.eager import context from tensorflow.python.framework import ops from tensorflow.python.framework import test_util @@ -88,22 +91,26 @@ def _test_optimizer(optimizer, target=0.75): class KerasOptimizersTest(test.TestCase): + @test_util.run_deprecated_v1 def test_sgd(self): with self.cached_session(): _test_optimizer(keras.optimizers.SGD(lr=0.01, momentum=0.9, nesterov=True)) + @test_util.run_deprecated_v1 def test_rmsprop(self): with self.cached_session(): _test_optimizer(keras.optimizers.RMSprop()) _test_optimizer(keras.optimizers.RMSprop(decay=1e-3)) + @test_util.run_deprecated_v1 def test_adagrad(self): with self.cached_session(): _test_optimizer(keras.optimizers.Adagrad()) _test_optimizer(keras.optimizers.Adagrad(decay=1e-3)) + @test_util.run_deprecated_v1 def test_adadelta(self): with self.cached_session(): _test_optimizer(keras.optimizers.Adadelta(), target=0.6) @@ -112,27 +119,32 @@ class KerasOptimizersTest(test.TestCase): # the accuracy. _test_optimizer(keras.optimizers.Adadelta(decay=1e-3), target=0.4) + @test_util.run_deprecated_v1 def test_adam(self): with self.cached_session(): _test_optimizer(keras.optimizers.Adam()) _test_optimizer(keras.optimizers.Adam(decay=1e-3)) _test_optimizer(keras.optimizers.Adam(amsgrad=True)) + @test_util.run_deprecated_v1 def test_adamax(self): with self.cached_session(): _test_optimizer(keras.optimizers.Adamax()) _test_optimizer(keras.optimizers.Adamax(decay=1e-3)) + @test_util.run_deprecated_v1 def test_nadam(self): with self.cached_session(): _test_optimizer(keras.optimizers.Nadam()) + @test_util.run_deprecated_v1 def test_clipnorm(self): with self.cached_session(): _test_optimizer(keras.optimizers.SGD(lr=0.01, momentum=0.9, clipnorm=0.5)) + @test_util.run_deprecated_v1 def test_clipvalue(self): with self.cached_session(): _test_optimizer(keras.optimizers.SGD(lr=0.01, @@ -208,5 +220,40 @@ class KerasOptimizersTest(test.TestCase): _ = keras.optimizers.Adam(clipnorm=-2.0) +@test_util.run_all_in_graph_and_eager_modes +class KerasV2OptimizersTest(test.TestCase, parameterized.TestCase): + + @parameterized.named_parameters( + ('adadelta_tf2', 'adadelta', True), ('adadelta_tf1', 'adadelta', False), + ('adagrad_tf2', 'adagrad', True), ('adagrad_tf1', 'adagrad', False), + ('adam_tf2', 'adam', True), ('adam_tf1', 'adam', False), + ('adamax_tf2', 'adamax', True), ('adamax_tf1', 'adamax', False), + ('sgd_tf2', 'sgd', True), ('sgd_tf1', 'sgd', False), + ('nadam_tf2', 'nadam', True), ('nadam_tf1', 'nadam', False), + ('rmsprop_tf2', 'rmsprop', True), ('rmsprop_tf1', 'rmsprop', False)) + def test_load_from_string(self, optimizer_string, tf2mode): + old_mode = os.environ.get('TF2_BEHAVIOR', None) + if tf2mode: + os.environ['TF2_BEHAVIOR'] = 'enabled' + else: + if 'TF2_BEHAVIOR' in os.environ: + del os.environ['TF2_BEHAVIOR'] + + # Sanity check. + self.assertEqual(tf2.enabled(), tf2mode) + + model = keras.models.Sequential() + model.add(keras.layers.Dense(1, input_shape=(10,))) + model.compile(optimizer_string, 'binary_crossentropy') + + self.assertEqual(optimizer_string, + model.optimizer.__class__.__name__.lower()) + + model.fit(np.ones((10, 10), 'float32'), np.ones((10, 1), 'float32')) + + if old_mode is not None: + os.environ['TF2_BEHAVIOR'] = old_mode + + if __name__ == '__main__': test.main() diff --git a/tensorflow/python/keras/regularizers_test.py b/tensorflow/python/keras/regularizers_test.py index bba4ebb287b..3d6b259d87d 100644 --- a/tensorflow/python/keras/regularizers_test.py +++ b/tensorflow/python/keras/regularizers_test.py @@ -20,6 +20,7 @@ from __future__ import print_function from tensorflow.python import keras from tensorflow.python.keras import testing_utils +from tensorflow.python.framework import test_util from tensorflow.python.platform import test @@ -61,6 +62,7 @@ class KerasRegularizersTest(test.TestCase): model.fit(x_train, y_train, batch_size=10, epochs=1, verbose=0) + @test_util.run_deprecated_v1 def test_activity_regularization(self): with self.cached_session(): (x_train, y_train), _ = get_data() diff --git a/tensorflow/python/keras/testing_utils.py b/tensorflow/python/keras/testing_utils.py index d342131a521..7827e166273 100644 --- a/tensorflow/python/keras/testing_utils.py +++ b/tensorflow/python/keras/testing_utils.py @@ -149,6 +149,10 @@ def layer_test(layer_cls, kwargs=None, input_shape=None, input_dtype=None, np.testing.assert_allclose(output, actual_output, rtol=1e-3) # test training mode (e.g. useful for dropout tests) + # Rebuild the model to avoid the graph being reused between predict() and + # train(). This was causing some error for layer with Defun as it body. + # See b/120160788 for more details. This should be mitigated after 2.0. + model = keras.models.Model(x, layer(x)) model.compile(RMSPropOptimizer(0.01), 'mse', weighted_metrics=['acc']) model.train_on_batch(input_data, actual_output) diff --git a/tensorflow/python/keras/utils/__init__.py b/tensorflow/python/keras/utils/__init__.py index 8939044f71d..61940ad789c 100644 --- a/tensorflow/python/keras/utils/__init__.py +++ b/tensorflow/python/keras/utils/__init__.py @@ -34,6 +34,7 @@ from tensorflow.python.keras.utils.generic_utils import serialize_keras_object from tensorflow.python.keras.utils.io_utils import HDF5Matrix from tensorflow.python.keras.utils.layer_utils import convert_all_kernels_in_model from tensorflow.python.keras.utils.layer_utils import get_source_inputs +from tensorflow.python.keras.utils.losses_utils import squeeze_or_expand_dimensions from tensorflow.python.keras.utils.multi_gpu_utils import multi_gpu_model from tensorflow.python.keras.utils.np_utils import normalize from tensorflow.python.keras.utils.np_utils import to_categorical diff --git a/tensorflow/python/keras/utils/generic_utils.py b/tensorflow/python/keras/utils/generic_utils.py index 375bd9d196c..c331ce430bd 100644 --- a/tensorflow/python/keras/utils/generic_utils.py +++ b/tensorflow/python/keras/utils/generic_utils.py @@ -319,14 +319,16 @@ class Progbar(object): will be displayed as-is. All others will be averaged by the progbar before display. interval: Minimum visual progress update interval (in seconds). + unit_name: Display name for step counts (usually "step" or "sample"). """ def __init__(self, target, width=30, verbose=1, interval=0.05, - stateful_metrics=None): + stateful_metrics=None, unit_name='step'): self.target = target self.width = width self.verbose = verbose self.interval = interval + self.unit_name = unit_name if stateful_metrics: self.stateful_metrics = set(stateful_metrics) else: @@ -425,12 +427,12 @@ class Progbar(object): info = ' - ETA: %s' % eta_format else: - if time_per_unit >= 1: - info += ' %.0fs/step' % time_per_unit + if time_per_unit >= 1 or time_per_unit == 0: + info += ' %.0fs/%s' % (time_per_unit, self.unit_name) elif time_per_unit >= 1e-3: - info += ' %.0fms/step' % (time_per_unit * 1e3) + info += ' %.0fms/%s' % (time_per_unit * 1e3, self.unit_name) else: - info += ' %.0fus/step' % (time_per_unit * 1e6) + info += ' %.0fus/%s' % (time_per_unit * 1e6, self.unit_name) for k in self._values_order: info += ' - %s:' % k diff --git a/tensorflow/python/keras/utils/layer_utils.py b/tensorflow/python/keras/utils/layer_utils.py index 158a9a5e76d..60677be7351 100644 --- a/tensorflow/python/keras/utils/layer_utils.py +++ b/tensorflow/python/keras/utils/layer_utils.py @@ -77,7 +77,7 @@ def count_params(weights): Returns: The total number of scalars composing the weights """ - return int(np.sum([np.prod(p.get_shape().as_list()) for p in set(weights)])) + return int(sum(np.prod(p.get_shape().as_list()) for p in set(weights))) def print_summary(model, line_length=None, positions=None, print_fn=None): diff --git a/tensorflow/python/keras/utils/losses_utils.py b/tensorflow/python/keras/utils/losses_utils.py new file mode 100644 index 00000000000..fc4b4ac7dfd --- /dev/null +++ b/tensorflow/python/keras/utils/losses_utils.py @@ -0,0 +1,189 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +# pylint: disable=protected-access +"""Utilities related to loss functions.""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from tensorflow.python.framework import ops +from tensorflow.python.keras import backend as K +from tensorflow.python.ops import array_ops +from tensorflow.python.ops import confusion_matrix +from tensorflow.python.ops import control_flow_ops +from tensorflow.python.ops import math_ops +from tensorflow.python.ops import weights_broadcast_ops +from tensorflow.python.ops.losses import losses_impl + + +def squeeze_or_expand_dimensions(y_pred, y_true, sample_weight): + """Squeeze or expand last dimension if needed. + + 1. Squeezes last dim of `y_pred` or `y_true` if their rank differs by 1 + (using `confusion_matrix.remove_squeezable_dimensions`). + 2. Squeezes or expands last dim of `sample_weight` if its rank differs by 1 + from the new rank of `y_pred`. + If `sample_weight` is scalar, it is kept scalar. + + This will use static shape if available. Otherwise, it will add graph + operations, which could result in a performance hit. + + Args: + y_pred: Predicted values, a `Tensor` of arbitrary dimensions. + y_true: Optional label `Tensor` whose dimensions match `y_pred`. + sample_weight: Optional weight scalar or `Tensor` whose dimensions match + `y_pred`. + + Returns: + Tuple of `y_pred`, `y_true` and `sample_weight`. Each of them possibly has + the last dimension squeezed, + `sample_weight` could be extended by one dimension. + """ + if y_true is not None: + # squeeze last dim of `y_pred` or `y_true` if their rank differs by 1 + y_true, y_pred = confusion_matrix.remove_squeezable_dimensions( + y_true, y_pred) + + if sample_weight is None: + return y_pred, y_true, None + + sample_weight = ops.convert_to_tensor(sample_weight) + weights_shape = sample_weight.get_shape() + weights_rank = weights_shape.ndims + if weights_rank == 0: # If weights is scalar, do nothing. + return y_pred, y_true, sample_weight + + y_pred_shape = y_pred.get_shape() + y_pred_rank = y_pred_shape.ndims + if (y_pred_rank is not None) and (weights_rank is not None): + # Use static rank. + if weights_rank - y_pred_rank == 1: + sample_weight = array_ops.squeeze(sample_weight, [-1]) + elif y_pred_rank - weights_rank == 1: + sample_weight = array_ops.expand_dims(sample_weight, [-1]) + return y_pred, y_true, sample_weight + + # Use dynamic rank. + weights_rank_tensor = array_ops.rank(sample_weight) + rank_diff = weights_rank_tensor - array_ops.rank(y_pred) + maybe_squeeze_weights = lambda: array_ops.squeeze(sample_weight, [-1]) + + def _maybe_expand_weights(): + return control_flow_ops.cond( + math_ops.equal(rank_diff, + -1), lambda: array_ops.expand_dims(sample_weight, [-1]), + lambda: sample_weight) + + def _maybe_adjust_weights(): + return control_flow_ops.cond( + math_ops.equal(rank_diff, 1), maybe_squeeze_weights, + _maybe_expand_weights) + + # squeeze or expand last dim of `sample_weight` if its rank differs by 1 + # from the new rank of `y_pred`. + sample_weight = control_flow_ops.cond( + math_ops.equal(weights_rank_tensor, 0), lambda: sample_weight, + _maybe_adjust_weights) + return y_pred, y_true, sample_weight + + +def _safe_mean(losses, num_present): + """Computes a safe mean of the losses. + + Args: + losses: `Tensor` whose elements contain individual loss measurements. + num_present: The number of measurable elements in `losses`. + + Returns: + A scalar representing the mean of `losses`. If `num_present` is zero, + then zero is returned. + """ + total_loss = math_ops.reduce_sum(losses) + return math_ops.div_no_nan(total_loss, num_present, name='value') + + +def _num_elements(losses): + """Computes the number of elements in `losses` tensor.""" + with ops.name_scope(None, 'num_elements', values=[losses]) as scope: + return math_ops.cast(array_ops.size(losses, name=scope), dtype=losses.dtype) + + +def _reduce_weighted_loss( + weighted_losses, reduction=losses_impl.ReductionV2.SUM_OVER_BATCH_SIZE): + """Reduces the individual weighted loss measurements.""" + if reduction == losses_impl.ReductionV2.NONE: + loss = weighted_losses + else: + loss = math_ops.reduce_sum(weighted_losses) + if reduction == losses_impl.ReductionV2.SUM_OVER_BATCH_SIZE: + loss = _safe_mean(loss, _num_elements(weighted_losses)) + return loss + + +def compute_weighted_loss(losses, + sample_weight=None, + reduction=losses_impl.ReductionV2.SUM_OVER_BATCH_SIZE, + name=None): + """Computes the weighted loss. + + Args: + losses: `Tensor` of shape `[batch_size, d1, ... dN]`. + sample_weight: Optional `Tensor` whose rank is either 0, or the same rank as + `losses`, or be broadcastable to `losses`. + reduction: Type of `tf.losses.Reduction` to apply to loss. Default value is + `SUM_OVER_BATCH_SIZE`. + name: Optional name for the op. + + Raises: + ValueError: If the shape of `sample_weight` is not compatible with `losses`. + + Returns: + Weighted loss `Tensor` of the same type as `losses`. If `reduction` is + `NONE`, this has the same shape as `losses`; otherwise, it is scalar. + """ + losses_impl.ReductionV2.validate(reduction) + if sample_weight is None: + sample_weight = 1.0 + with ops.name_scope(name, 'weighted_loss', (losses, sample_weight)): + # Save the `reduction` argument for loss normalization when distributing + # to multiple replicas. + # TODO(josh11b): Associate it with the returned op for more precision. + ops.get_default_graph()._last_loss_reduction = reduction # pylint: disable=protected-access + + # Update dimensions of `sample_weight` to match with `losses` if possible. + losses, _, sample_weight = squeeze_or_expand_dimensions( + losses, None, sample_weight) + losses = ops.convert_to_tensor(losses) + input_dtype = losses.dtype + losses = math_ops.to_float(losses) + sample_weight = math_ops.to_float(sample_weight) + + try: + # Broadcast weights if possible. + sample_weight = weights_broadcast_ops.broadcast_weights( + sample_weight, losses) + except ValueError: + # Reduce values to same ndim as weight array. + ndim = K.ndim(losses) + weight_ndim = K.ndim(sample_weight) + losses = K.mean(losses, axis=list(range(weight_ndim, ndim))) + + sample_weight.get_shape().assert_is_compatible_with(losses.get_shape()) + weighted_losses = math_ops.multiply(losses, sample_weight) + # Apply reduction function to the individual weighted losses. + loss = _reduce_weighted_loss(weighted_losses, reduction) + # Convert the result back to the input type. + loss = math_ops.cast(loss, input_dtype) + return loss diff --git a/tensorflow/python/keras/utils/multi_gpu_utils_test.py b/tensorflow/python/keras/utils/multi_gpu_utils_test.py index 1780ab65871..8c1abd63248 100644 --- a/tensorflow/python/keras/utils/multi_gpu_utils_test.py +++ b/tensorflow/python/keras/utils/multi_gpu_utils_test.py @@ -158,7 +158,7 @@ class TestMultiGPUModel(test.TestCase): dataset = data.Dataset.from_tensor_slices((x_train, y_train)) dataset = dataset.repeat() dataset = dataset.batch(4) - iterator = dataset.make_one_shot_iterator() + iterator = data.make_one_shot_iterator(dataset) inputs, targets = iterator.get_next() diff --git a/tensorflow/python/keras/utils/tf_utils.py b/tensorflow/python/keras/utils/tf_utils.py index 6b7c6c34a26..7b4c9e7239e 100644 --- a/tensorflow/python/keras/utils/tf_utils.py +++ b/tensorflow/python/keras/utils/tf_utils.py @@ -161,6 +161,9 @@ def are_all_symbolic_tensors(tensors): return all(is_symbolic_tensor(tensor) for tensor in tensors) +_user_convertible_tensor_types = set() + + def is_symbolic_tensor(tensor): """Returns whether a tensor is symbolic (from a TF graph) or an eager tensor. @@ -176,9 +179,40 @@ def is_symbolic_tensor(tensor): if isinstance(tensor, variables.Variable): return not context.executing_eagerly() if isinstance(tensor, (ops.Tensor, sparse_tensor.SparseTensor)): - try: - _ = tensor.graph - return True - except AttributeError: - return False + return hasattr(tensor, 'graph') + if isinstance(tensor, tuple(_user_convertible_tensor_types)): + return hasattr(ops.convert_to_tensor(tensor), 'graph') return False + + +def register_symbolic_tensor_type(cls): + """Allows users to specify types regarded as symbolic `Tensor`s. + + Used in conjunction with `tf.register_tensor_conversion_function`, calling + `tf.keras.utils.register_symbolic_tensor_type(cls)` allows non-`Tensor` + objects to be plumbed through Keras layers. + + Example: + + ```python + # One-time setup. + class Foo(object): + def __init__(self, input_): + self._input = input_ + def value(self): + return tf.constant(42.) + + tf.register_tensor_conversion_function( + Foo, lambda x, *args, **kwargs: x.value()) + + tf.keras.utils.register_symbolic_tensor_type(Foo) + + # User-land. + layer = tf.keras.layers.Lambda(lambda input_: Foo(input_)) + ``` + + Arguments: + cls: A `class` type which shall be regarded as a symbolic `Tensor`. + """ + global _user_convertible_tensor_types + _user_convertible_tensor_types.add(cls) diff --git a/tensorflow/python/keras/utils/tf_utils_test.py b/tensorflow/python/keras/utils/tf_utils_test.py new file mode 100644 index 00000000000..9833a492993 --- /dev/null +++ b/tensorflow/python/keras/utils/tf_utils_test.py @@ -0,0 +1,134 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Tests for Keras TF utils.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from tensorflow.python import keras +from tensorflow.python.eager import context +from tensorflow.python.framework import ops +from tensorflow.python.framework import sparse_tensor +from tensorflow.python.framework import test_util +from tensorflow.python.keras.utils import tf_utils +from tensorflow.python.ops import variables +from tensorflow.python.platform import test + + +@test_util.run_all_in_graph_and_eager_modes +class TestIsSymbolicTensor(test.TestCase): + + def test_default_behavior(self): + if context.executing_eagerly(): + self.assertFalse(tf_utils.is_symbolic_tensor( + variables.Variable(name='blah', initial_value=0.))) + self.assertFalse(tf_utils.is_symbolic_tensor( + ops.convert_to_tensor(0.))) + self.assertFalse(tf_utils.is_symbolic_tensor( + sparse_tensor.SparseTensor( + indices=[[0, 0], [1, 2]], values=[1, 2], dense_shape=[3, 4]))) + else: + self.assertTrue(tf_utils.is_symbolic_tensor( + variables.Variable(name='blah', initial_value=0.))) + self.assertTrue(tf_utils.is_symbolic_tensor( + ops.convert_to_tensor(0.))) + self.assertTrue(tf_utils.is_symbolic_tensor( + sparse_tensor.SparseTensor( + indices=[[0, 0], [1, 2]], values=[1, 2], dense_shape=[3, 4]))) + + def test_works_with_registered(self): + + class CustomClass(object): + + def value(self): + return ops.convert_to_tensor(42.) + + ops.register_tensor_conversion_function( + CustomClass, lambda value, **_: value.value()) + + tf_utils.register_symbolic_tensor_type(CustomClass) + + if context.executing_eagerly(): + self.assertFalse(tf_utils.is_symbolic_tensor( + variables.Variable(name='blah', initial_value=0.))) + self.assertFalse(tf_utils.is_symbolic_tensor( + ops.convert_to_tensor(0.))) + self.assertFalse(tf_utils.is_symbolic_tensor( + sparse_tensor.SparseTensor( + indices=[[0, 0], [1, 2]], values=[1, 2], dense_shape=[3, 4]))) + self.assertFalse(tf_utils.is_symbolic_tensor(CustomClass())) + else: + self.assertTrue(tf_utils.is_symbolic_tensor( + variables.Variable(name='blah', initial_value=0.))) + self.assertTrue(tf_utils.is_symbolic_tensor( + ops.convert_to_tensor(0.))) + self.assertTrue(tf_utils.is_symbolic_tensor( + sparse_tensor.SparseTensor( + indices=[[0, 0], [1, 2]], values=[1, 2], dense_shape=[3, 4]))) + self.assertTrue(tf_utils.is_symbolic_tensor(CustomClass())) + + def test_enables_nontensor_plumbing(self): + # Setup. + + class Foo(object): + + def __init__(self, input_): + self._input = input_ + self.value = ops.convert_to_tensor(42.) + + ops.register_tensor_conversion_function( + Foo, lambda x, *args, **kwargs: x.value) + tf_utils.register_symbolic_tensor_type(Foo) + + class PlumbingLayer(keras.layers.Lambda): + + def __init__(self, fn, **kwargs): + def _fn(*fargs, **fkwargs): + d = fn(*fargs, **fkwargs) + x = ops.convert_to_tensor(d) + d.shape = x.shape + d.get_shape = x.get_shape + return d, x + super(PlumbingLayer, self).__init__(_fn, **kwargs) + self._enter_dunder_call = False + + def __call__(self, inputs, *args, **kwargs): + self._enter_dunder_call = True + d, _ = super(PlumbingLayer, self).__call__(inputs, *args, **kwargs) + self._enter_dunder_call = False + return d + + def call(self, inputs, *args, **kwargs): + d, v = super(PlumbingLayer, self).call(inputs, *args, **kwargs) + if self._enter_dunder_call: + return d, v + return d + + # User-land. + model = keras.Sequential([ + keras.layers.InputLayer([]), + PlumbingLayer(Foo), # Makes a `Foo` object. + ]) + # Let's ensure Keras graph history is preserved by composing the models. + model = keras.Model(model.inputs, model(model.outputs)) + # Now we instantiate the model and verify we have a `Foo` object, not a + # `Tensor`. + y = model(ops.convert_to_tensor(7.)) + self.assertIsInstance(y, Foo) + + +if __name__ == '__main__': + test.main() diff --git a/tensorflow/python/kernel_tests/BUILD b/tensorflow/python/kernel_tests/BUILD index 6b990d3a926..97ac21b8adf 100644 --- a/tensorflow/python/kernel_tests/BUILD +++ b/tensorflow/python/kernel_tests/BUILD @@ -121,8 +121,10 @@ cuda_py_test( "@absl_py//absl/testing:parameterized", "//third_party/py/numpy", "//tensorflow/python:array_ops", - "//tensorflow/python:math_ops", + "//tensorflow/python:gradients_impl", "//tensorflow/python:list_ops", + "//tensorflow/python:math_ops", + "//tensorflow/python:tensor_shape", "//tensorflow/python/eager:context", "//tensorflow/python:framework_for_generated_wrappers", "//tensorflow/python:framework_test_lib", @@ -268,7 +270,7 @@ tf_py_test( ], ) -tf_py_test( +cuda_py_test( name = "ctc_loss_op_test", size = "small", srcs = ["ctc_loss_op_test.py"], @@ -659,6 +661,18 @@ cuda_py_test( ], ) +cuda_py_test( + name = "matrix_square_root_op_test", + size = "medium", + srcs = ["matrix_square_root_op_test.py"], + additional_deps = [ + "//third_party/py/numpy", + "//tensorflow/python:client_testlib", + "//tensorflow/python:framework_for_generated_wrappers", + "//tensorflow/python:linalg_ops", + ], +) + cuda_py_test( name = "matrix_solve_op_test", size = "medium", @@ -817,6 +831,7 @@ tf_py_test( "//tensorflow/core:protos_all_py", "//tensorflow/python:client", "//tensorflow/python:client_testlib", + "//tensorflow/python:constant_op", "//tensorflow/python:io_ops", "//tensorflow/python:io_ops_gen", ], @@ -1142,6 +1157,21 @@ cuda_py_test( ], ) +tf_py_test( + name = "unicode_encode_op_test", + size = "small", + srcs = ["unicode_encode_op_test.py"], + additional_deps = [ + "@absl_py//absl/testing:parameterized", + "//third_party/py/numpy", + "//tensorflow/python:client_testlib", + "//tensorflow/python:constant_op", + "//tensorflow/python:errors", + "//tensorflow/python/ops/ragged:ragged_factory_ops", + "//tensorflow/python/ops/ragged:ragged_string_ops", + ], +) + tf_py_test( name = "unicode_transcode_op_test", size = "small", @@ -1154,6 +1184,18 @@ tf_py_test( ], ) +tf_py_test( + name = "unicode_decode_op_test", + size = "small", + srcs = ["unicode_decode_op_test.py"], + additional_deps = [ + "@absl_py//absl/testing:parameterized", + "//tensorflow/python:client_testlib", + "//tensorflow/python:framework_for_generated_wrappers", + "//tensorflow/python:string_ops", + ], +) + tf_py_test( name = "unique_op_test", size = "small", @@ -1338,6 +1380,7 @@ cuda_py_test( "//tensorflow/python:test_ops", "//tensorflow/python:variables", "//tensorflow/python/eager:context", + "//tensorflow/python/eager:def_function", ], shard_count = 10, tags = [ @@ -1449,6 +1492,7 @@ cuda_py_test( additional_deps = [ "//third_party/py/numpy", "//tensorflow/python/eager:context", + "//tensorflow/python/eager:def_function", "//tensorflow/python:array_ops", "//tensorflow/python:check_ops", "//tensorflow/python:math_ops", @@ -1736,9 +1780,11 @@ cuda_py_test( "//tensorflow/python:tensor_array_grad", "//tensorflow/python:variable_scope", "//tensorflow/python:variables", + "//tensorflow/python:while_v2", "//tensorflow/python/data/ops:iterator_ops", ], grpc_enabled = True, + shard_count = 2, tags = ["no_windows"], ) @@ -2371,6 +2417,8 @@ cuda_py_test( "//tensorflow/python:tensor_array_ops", "//tensorflow/python:variables", "//tensorflow/python:variable_scope", + "//tensorflow/python:cond_v2", + "//tensorflow/python:while_v2", "//tensorflow/python/eager:backprop", "//tensorflow/python/eager:context", ], @@ -2614,34 +2662,6 @@ cuda_py_test( tags = ["manual"], ) -cuda_py_test( - name = "dct_ops_test", - srcs = ["dct_ops_test.py"], - additional_deps = [ - "//third_party/py/numpy", - "//tensorflow/python:client_testlib", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:spectral_ops", - "//tensorflow/python:spectral_ops_test_util", - ], -) - -cuda_py_test( - name = "fft_ops_test", - size = "medium", - srcs = ["fft_ops_test.py"], - additional_deps = [ - "//third_party/py/numpy", - "//tensorflow/python:client_testlib", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:math_ops", - "//tensorflow/python:spectral_ops", - "//tensorflow/python:spectral_ops_test_util", - ], - shard_count = 4, - tags = ["optonly"], -) - cuda_py_test( name = "pooling_ops_3d_test", size = "medium", @@ -3330,7 +3350,9 @@ cuda_py_test( "//tensorflow/python:framework", "//tensorflow/python:framework_ops", "//tensorflow/python:gradients", + "//tensorflow/python:tensor_array_ops", "//tensorflow/python:training", + "//tensorflow/python:while_v2", ], grpc_enabled = True, ) diff --git a/tensorflow/python/kernel_tests/accumulate_n_test.py b/tensorflow/python/kernel_tests/accumulate_n_test.py index 7889edc198f..5eece9c9413 100644 --- a/tensorflow/python/kernel_tests/accumulate_n_test.py +++ b/tensorflow/python/kernel_tests/accumulate_n_test.py @@ -32,6 +32,7 @@ from tensorflow.python.platform import googletest class AccumulateNV2Test(test_util.TensorFlowTestCase): """Tests of the new, differentiable version of accumulate_n.""" + @test_util.run_deprecated_v1 def testFloat(self): np.random.seed(12345) x = [np.random.random((1, 2, 3, 4, 5)) - 0.5 for _ in range(5)] @@ -41,6 +42,7 @@ class AccumulateNV2Test(test_util.TensorFlowTestCase): self.assertAllClose(x[0] * 5, math_ops.accumulate_n([tf_x[0]] * 5).eval()) + @test_util.run_deprecated_v1 def testInt(self): np.random.seed(54321) x = [np.random.randint(-128, 128, (5, 4, 3, 2, 1)) for _ in range(6)] @@ -50,12 +52,14 @@ class AccumulateNV2Test(test_util.TensorFlowTestCase): self.assertAllEqual(x[0] * 6, math_ops.accumulate_n([tf_x[0]] * 6).eval()) + @test_util.run_deprecated_v1 def testUnknownShape(self): with self.session(use_gpu=True): x0 = array_ops.placeholder(dtype=dtypes_lib.int32, shape=[None]) acc = math_ops.accumulate_n([x0, x0], shape=[None]) self.assertAllEqual([2, 4], acc.eval(feed_dict={x0: [1, 2]})) + @test_util.run_deprecated_v1 def testGrad(self): np.random.seed(42) for num_inputs in range(1, 10): @@ -65,7 +69,7 @@ class AccumulateNV2Test(test_util.TensorFlowTestCase): for _ in range(0, num_inputs) ] accum_n = math_ops.accumulate_n(input_vars) - sess.run(variables.global_variables_initializer()) + self.evaluate(variables.global_variables_initializer()) accum_n_grad = gradients.gradients(accum_n, input_vars) self.assertAllEqual( np.repeat(1.0, num_inputs), # d/dx (x + y + ...) = 1 @@ -88,13 +92,13 @@ class AccumulateNV2Test(test_util.TensorFlowTestCase): np_val = random_arrays[0] for random_array in random_arrays[1:]: np_val += random_array - self.assertAllClose(np_val, tf_val.eval()) + self.assertAllClose(np_val, self.evaluate(tf_val)) def testZeroArgs(self): with self.cached_session(): with self.assertRaises(ValueError): tf_val = math_ops.accumulate_n([]) - tf_val.eval() + self.evaluate(tf_val) def testWrongShape(self): with self.cached_session(): diff --git a/tensorflow/python/kernel_tests/ackermann_test.py b/tensorflow/python/kernel_tests/ackermann_test.py index d267e497527..6c20b19be9e 100644 --- a/tensorflow/python/kernel_tests/ackermann_test.py +++ b/tensorflow/python/kernel_tests/ackermann_test.py @@ -20,12 +20,14 @@ from __future__ import print_function import os from tensorflow.python.framework import load_library +from tensorflow.python.framework import test_util from tensorflow.python.platform import resource_loader from tensorflow.python.platform import test class AckermannTest(test.TestCase): + @test_util.run_deprecated_v1 def testBasic(self): library_filename = os.path.join(resource_loader.get_data_files_path(), 'ackermann_op.so') diff --git a/tensorflow/python/kernel_tests/aggregate_ops_test.py b/tensorflow/python/kernel_tests/aggregate_ops_test.py index 0f15319cb59..d9787cc3bf6 100644 --- a/tensorflow/python/kernel_tests/aggregate_ops_test.py +++ b/tensorflow/python/kernel_tests/aggregate_ops_test.py @@ -24,6 +24,7 @@ from tensorflow.core.framework import tensor_pb2 from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes from tensorflow.python.framework import tensor_shape +from tensorflow.python.framework import test_util from tensorflow.python.ops import array_ops from tensorflow.python.ops import logging_ops from tensorflow.python.ops import math_ops @@ -61,12 +62,13 @@ class AddNTest(test.TestCase): for dtype in self._supported_types(): for count in range(1, self._MAX_N + 1): data = [self._buildData((2, 2), dtype) for _ in range(count)] - actual = sess.run(math_ops.add_n(data)) + actual = self.evaluate(math_ops.add_n(data)) expected = np.sum(np.vstack( [np.expand_dims(d, 0) for d in data]), axis=0) tol = 5e-3 if dtype == dtypes.float16 else 5e-7 self.assertAllClose(expected, actual, rtol=tol, atol=tol) + @test_util.run_deprecated_v1 def testUnknownShapes(self): np.random.seed(12345) with self.session(use_gpu=True) as sess: @@ -80,6 +82,7 @@ class AddNTest(test.TestCase): tol = 5e-3 if dtype == dtypes.float16 else 5e-7 self.assertAllClose(expected, actual, rtol=tol, atol=tol) + @test_util.run_deprecated_v1 def testVariant(self): def create_constant_variant(value): diff --git a/tensorflow/python/kernel_tests/argmax_op_test.py b/tensorflow/python/kernel_tests/argmax_op_test.py index fa370c17b46..06ec0948c25 100644 --- a/tensorflow/python/kernel_tests/argmax_op_test.py +++ b/tensorflow/python/kernel_tests/argmax_op_test.py @@ -20,6 +20,7 @@ from __future__ import print_function import numpy as np from tensorflow.python.framework import dtypes +from tensorflow.python.framework import test_util from tensorflow.python.ops import array_ops from tensorflow.python.ops import math_ops from tensorflow.python.platform import test @@ -37,14 +38,14 @@ class ArgMaxTest(test.TestCase): with self.session(use_gpu=use_gpu): ans = method(x, axis=axis) if expected_err_re is None: - tf_ans = ans.eval() + tf_ans = self.evaluate(ans) # Defaults to int64 output. self.assertEqual(np.int64, tf_ans.dtype) self.assertAllEqual(tf_ans, expected_values) self.assertShapeEqual(expected_values, ans) else: with self.assertRaisesOpError(expected_err_re): - ans.eval() + self.evaluate(ans) def _testBothArg(self, method, @@ -79,7 +80,7 @@ class ArgMaxTest(test.TestCase): expected_values = x.argmax() with self.session(use_gpu=True): ans = math_ops.argmax(x, axis=0, output_type=dtypes.int32) - tf_ans = ans.eval() + tf_ans = self.evaluate(ans) self.assertEqual(np.int32, tf_ans.dtype) # The values are equal when comparing int32 to int64 because # the values don't have a range that exceeds 32-bit integers. @@ -87,7 +88,7 @@ class ArgMaxTest(test.TestCase): expected_values = x.argmin() with self.session(use_gpu=True): ans = math_ops.argmin(x, axis=0, output_type=dtypes.int32) - tf_ans = ans.eval() + tf_ans = self.evaluate(ans) self.assertEqual(np.int32, tf_ans.dtype) self.assertAllEqual(tf_ans, expected_values) @@ -110,12 +111,14 @@ class ArgMaxTest(test.TestCase): r"Reduction axis 0 is empty in shape \[0\]"): op([], 0).eval() + @test_util.run_deprecated_v1 def testDefaultAxis(self): with self.cached_session(): for op in math_ops.argmin, math_ops.argmax: ans = op([1]).eval() self.assertAllEqual(ans, 0) + @test_util.run_deprecated_v1 def testOutputEmpty(self): with self.cached_session(): for op in math_ops.argmin, math_ops.argmax: diff --git a/tensorflow/python/kernel_tests/array_ops_test.py b/tensorflow/python/kernel_tests/array_ops_test.py index c90794c7892..f4c442b7b19 100644 --- a/tensorflow/python/kernel_tests/array_ops_test.py +++ b/tensorflow/python/kernel_tests/array_ops_test.py @@ -25,6 +25,7 @@ import numpy as np from tensorflow.python.client import session from tensorflow.python.eager import context +from tensorflow.python.eager import def_function from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes from tensorflow.python.framework import errors @@ -32,6 +33,7 @@ from tensorflow.python.framework import errors_impl from tensorflow.python.framework import ops from tensorflow.python.framework import sparse_tensor from tensorflow.python.framework import tensor_shape +from tensorflow.python.framework import tensor_spec from tensorflow.python.framework import test_ops from tensorflow.python.framework import test_util from tensorflow.python.ops import array_ops @@ -46,24 +48,23 @@ from tensorflow.python.ops import variables from tensorflow.python.platform import test as test_lib +@test_util.run_all_in_graph_and_eager_modes class BatchMatrixTransposeTest(test_util.TensorFlowTestCase): def testNonBatchMatrix(self): matrix = [[1, 2, 3], [4, 5, 6]] # Shape (2, 3) expected_transposed = [[1, 4], [2, 5], [3, 6]] # Shape (3, 2) - with self.cached_session(): - transposed = array_ops.matrix_transpose(matrix) - self.assertEqual((3, 2), transposed.get_shape()) - self.assertAllEqual(expected_transposed, transposed.eval()) + transposed = array_ops.matrix_transpose(matrix) + self.assertEqual((3, 2), transposed.get_shape()) + self.assertAllEqual(expected_transposed, transposed) def testConjugate(self): m = [[1 + 1j, 2 + 2j, 3 + 3j], [4 + 4j, 5 + 5j, 6 + 6j]] expected_transposed = [[1 - 1j, 4 - 4j], [2 - 2j, 5 - 5j], [3 - 3j, 6 - 6j]] - with self.cached_session(): - matrix = ops.convert_to_tensor(m) - transposed = array_ops.matrix_transpose(matrix, conjugate=True) - self.assertEqual((3, 2), transposed.get_shape()) - self.assertAllEqual(expected_transposed, transposed.eval()) + matrix = ops.convert_to_tensor(m) + transposed = array_ops.matrix_transpose(matrix, conjugate=True) + self.assertEqual((3, 2), transposed.get_shape()) + self.assertAllEqual(expected_transposed, transposed) def testBatchMatrix(self): matrix_0 = [[1, 2, 3], [4, 5, 6]] @@ -72,43 +73,44 @@ class BatchMatrixTransposeTest(test_util.TensorFlowTestCase): matrix_1_t = [[11, 44], [22, 55], [33, 66]] batch_matrix = [matrix_0, matrix_1] # Shape (2, 2, 3) expected_transposed = [matrix_0_t, matrix_1_t] # Shape (2, 3, 2) - with self.cached_session(): - transposed = array_ops.matrix_transpose(batch_matrix) - self.assertEqual((2, 3, 2), transposed.get_shape()) - self.assertAllEqual(expected_transposed, transposed.eval()) + transposed = array_ops.matrix_transpose(batch_matrix) + self.assertEqual((2, 3, 2), transposed.get_shape()) + self.assertAllEqual(expected_transposed, transposed) def testNonBatchMatrixDynamicallyDefined(self): - matrix = [[1, 2, 3], [4, 5, 6]] # Shape (2, 3) + # needs explicit `constant` because lists are not automatically + # converted to sensors when applying `transpose` below + matrix = constant_op.constant([[1, 2, 3], [4, 5, 6]]) # Shape (2, 3) expected_transposed = [[1, 4], [2, 5], [3, 6]] # Shape (3, 2) - with self.cached_session(): - matrix_ph = array_ops.placeholder(dtypes.int32) - transposed = array_ops.matrix_transpose(matrix_ph) - self.assertAllEqual( - expected_transposed, transposed.eval(feed_dict={ - matrix_ph: matrix - })) + @def_function.function(input_signature= + [tensor_spec.TensorSpec + (shape=None, dtype=dtypes.int32)]) + def transpose(matrix): + self.assertIs(matrix.shape.ndims, None) + return array_ops.matrix_transpose(matrix) + self.assertAllEqual(expected_transposed, transpose(matrix)) def testBatchMatrixDynamicallyDefined(self): matrix_0 = [[1, 2, 3], [4, 5, 6]] matrix_0_t = [[1, 4], [2, 5], [3, 6]] matrix_1 = [[11, 22, 33], [44, 55, 66]] matrix_1_t = [[11, 44], [22, 55], [33, 66]] - batch_matrix = [matrix_0, matrix_1] # Shape (2, 2, 3) + # needs explicit `constant` because lists are not automatically + # converted to sensors when applying `transpose` below + batch_matrix = constant_op.constant([matrix_0, matrix_1]) # Shape (2, 2, 3) expected_transposed = [matrix_0_t, matrix_1_t] # Shape (2, 3, 2) - with self.cached_session(): - batch_matrix_ph = array_ops.placeholder(dtypes.int32) - transposed = array_ops.matrix_transpose(batch_matrix_ph) - self.assertAllEqual( - expected_transposed, - transposed.eval(feed_dict={ - batch_matrix_ph: batch_matrix - })) + @def_function.function(input_signature= + [tensor_spec.TensorSpec + (shape=None, dtype=dtypes.int32)]) + def transpose(matrix): + self.assertIs(matrix.shape.ndims, None) + return array_ops.matrix_transpose(matrix) + self.assertAllEqual(expected_transposed, transpose(batch_matrix)) def testTensorWithStaticRankLessThanTwoRaisesBecauseNotAMatrix(self): vector = [1, 2, 3] - with self.cached_session(): - with self.assertRaisesRegexp(ValueError, "should be a "): - array_ops.matrix_transpose(vector) + with self.assertRaisesRegexp(ValueError, "should be a "): + array_ops.matrix_transpose(vector) class BooleanMaskTest(test_util.TensorFlowTestCase): @@ -141,36 +143,43 @@ class BooleanMaskTest(test_util.TensorFlowTestCase): self.assertAllClose(masked_arr, masked_tensor.eval()) + @test_util.run_deprecated_v1 def testMaskDim1ArrDim2Axis1(self): ndims_mask = 1 for arr_shape in [(1, 1), (2, 2), (2, 5)]: self.CheckVersusNumpy(ndims_mask, arr_shape, axis=1) + @test_util.run_deprecated_v1 def testMaskDim2ArrDim2Axis1(self): ndims_mask = 2 for arr_shape in [(1, 1), (2, 2), (2, 5)]: self.CheckVersusNumpy(ndims_mask, arr_shape, axis=1) + @test_util.run_deprecated_v1 def testMaskDim1ArrDim1(self): ndims_mask = 1 for arr_shape in [(1,), (2,), (3,), (10,)]: self.CheckVersusNumpy(ndims_mask, arr_shape) + @test_util.run_deprecated_v1 def testMaskDim1ArrDim2(self): ndims_mask = 1 for arr_shape in [(1, 1), (2, 2), (2, 5)]: self.CheckVersusNumpy(ndims_mask, arr_shape) + @test_util.run_deprecated_v1 def testMaskDim2ArrDim2(self): ndims_mask = 2 for arr_shape in [(1, 1), (2, 2), (2, 5)]: self.CheckVersusNumpy(ndims_mask, arr_shape) + @test_util.run_deprecated_v1 def testMaskDim2ArrDim3(self): ndims_mask = 2 for arr_shape in [(1, 1, 1), (1, 2, 2), (2, 2, 1)]: self.CheckVersusNumpy(ndims_mask, arr_shape) + @test_util.run_deprecated_v1 def testEmptyInput2D(self): mask = np.array([True, False]) arr = np.array([[], []]).astype(np.float32) @@ -189,6 +198,7 @@ class BooleanMaskTest(test_util.TensorFlowTestCase): with self.cached_session(): self.assertAllClose(numpy_result, tf_result.eval()) + @test_util.run_deprecated_v1 def testEmptyOutput(self): make_mask = lambda shape: np.zeros(shape, dtype=bool) for ndims_mask in range(1, 4): @@ -197,6 +207,7 @@ class BooleanMaskTest(test_util.TensorFlowTestCase): arr_shape = np.random.randint(1, 5, size=ndims_arr) self.CheckVersusNumpy(ndims_mask, arr_shape, make_mask=make_mask) + @test_util.run_deprecated_v1 def testWorksWithDimensionsEqualToNoneDuringGraphBuild(self): # The rank of the mask tensor must be specified. This is explained # in the docstring as well. @@ -215,6 +226,7 @@ class BooleanMaskTest(test_util.TensorFlowTestCase): }) np.testing.assert_allclose(masked_tensor, arr[mask]) + @test_util.run_deprecated_v1 def testMaskDimensionsSetToNoneRaises(self): # The rank of the mask tensor must be specified. This is explained # in the docstring as well. @@ -281,6 +293,7 @@ class OperatorShapeTest(test_util.TensorFlowTestCase): class ReverseV2Test(test_util.TensorFlowTestCase): + @test_util.run_deprecated_v1 def testReverse0DimAuto(self): x_np = 4 for use_gpu in [False, True]: @@ -325,6 +338,7 @@ class ReverseV2Test(test_util.TensorFlowTestCase): # This test covers the axis validation in the shape function # (no eval()) + @test_util.run_deprecated_v1 def testInvalidAxis(self): x_np = np.array([[1, 2, 3], [4, 5, 6]], dtype=np.float32) with self.assertRaisesRegexp(ValueError, @@ -343,6 +357,7 @@ class ReverseV2Test(test_util.TensorFlowTestCase): # # Note: this test passes placeholder as constant axis is validated # in shape function (see testInvalidAxis) + @test_util.run_deprecated_v1 def testInvalid(self): x_np = np.array([[1, 2, 3], [4, 5, 6]], dtype=np.float32) axis = array_ops.placeholder(dtypes.int32) @@ -357,6 +372,7 @@ class ReverseV2Test(test_util.TensorFlowTestCase): "axis 0 specified more than once"): array_ops.reverse_v2(x_np, axis).eval(feed_dict={axis: [0, -2]}) + @test_util.run_deprecated_v1 def testReverse1DimAuto(self): for dtype in [ np.uint8, np.int8, np.uint16, np.int16, np.int32, np.int64, np.bool, @@ -365,6 +381,7 @@ class ReverseV2Test(test_util.TensorFlowTestCase): ]: self._reverse1DimAuto(dtype) + @test_util.run_deprecated_v1 def testReverse2DimAuto(self): for dtype in [ np.uint8, np.int8, np.uint16, np.int16, np.int32, np.int64, np.bool, @@ -373,6 +390,7 @@ class ReverseV2Test(test_util.TensorFlowTestCase): ]: self._reverse2DimAuto(dtype) + @test_util.run_deprecated_v1 def testUnknownDims(self): reverse_v2 = array_ops.reverse_v2 data_t = array_ops.placeholder(dtypes.float32) @@ -390,6 +408,7 @@ class ReverseV2Test(test_util.TensorFlowTestCase): reverse_2d_t = reverse_v2(data_2d_t, axis_2d_t) self.assertEqual(2, reverse_2d_t.get_shape().ndims) + @test_util.run_deprecated_v1 def testReverseRowsOf3Channels(self): """Tests optimized code for reversing rows with last dim size = 3.""" with self.session(use_gpu=True): @@ -403,6 +422,7 @@ class ReverseV2Test(test_util.TensorFlowTestCase): np_answer = x_np[:, ::-1, :] self.assertAllEqual(x_tf, np_answer) + @test_util.run_deprecated_v1 def testReverseRowsOf4Channels(self): with self.session(use_gpu=True): for reverse_f in [array_ops.reverse_v2, array_ops.reverse]: @@ -452,6 +472,7 @@ class MeshgridTest(test_util.TensorFlowTestCase): for x_np, x_tf in zip(numpy_out, tf_out): self.assertAllEqual(x_np, x_tf.eval()) + @test_util.run_deprecated_v1 def testCompare(self): for t in (np.float16, np.float32, np.float64, np.int32, np.int64, np.complex64, np.complex128): @@ -524,6 +545,7 @@ STRIDED_SLICE_TYPES = [ class StridedSliceTest(test_util.TensorFlowTestCase): """Test the strided slice operation with variants of slices.""" + @test_util.run_deprecated_v1 def test_basic_slice(self): for tensor_type in STRIDED_SLICE_TYPES: with self.cached_session(use_gpu=True): @@ -554,7 +576,8 @@ class StridedSliceTest(test_util.TensorFlowTestCase): def testInt64GPU(self): if not test_util.is_gpu_available(): self.skipTest("No GPU available") - with self.session(use_gpu=True, force_gpu=True): + + with test_util.force_gpu(): x = constant_op.constant([1., 2., 3.]) begin = constant_op.constant([2], dtype=dtypes.int64) end = constant_op.constant([3], dtype=dtypes.int64) @@ -578,6 +601,7 @@ class StridedSliceTest(test_util.TensorFlowTestCase): v = variables.Variable([1., 2.]) v[0] # pylint: disable=pointless-statement + @test_util.run_deprecated_v1 def testDegenerateSlices(self): with self.session(use_gpu=True): checker = StridedSliceChecker(self, StridedSliceChecker.REF_TENSOR) @@ -588,6 +612,7 @@ class StridedSliceTest(test_util.TensorFlowTestCase): # empty interval in every dimension _ = checker[-1:0, 2:2, 2:3:-1] + @test_util.run_deprecated_v1 def testEllipsis(self): with self.session(use_gpu=True): raw = [[[[[1, 2], [3, 4], [5, 6]]], [[[7, 8], [9, 10], [11, 12]]]]] @@ -608,6 +633,7 @@ class StridedSliceTest(test_util.TensorFlowTestCase): with self.assertRaisesRegexp(ValueError, "Multiple ellipses"): _ = checker[..., :, ...].eval() + @test_util.run_deprecated_v1 def testShrink(self): with self.session(use_gpu=True): raw = [[[[[1, 2, 4, 5], [5, 6, 7, 8], [9, 10, 11, 12]]], @@ -618,6 +644,7 @@ class StridedSliceTest(test_util.TensorFlowTestCase): _ = checker[:, 0] _ = checker[:, :, 0] + @test_util.run_deprecated_v1 def testBothNewAxisAndShrink(self): with self.session(use_gpu=True): ones = array_ops.placeholder(shape=[2, 2], dtype=dtypes.int16) @@ -626,6 +653,7 @@ class StridedSliceTest(test_util.TensorFlowTestCase): feed_dict={ones: [[1, 1], [1, 1]]}), [[1, 1]]) + @test_util.run_deprecated_v1 def testTensorIndexing(self): with self.session(use_gpu=True): raw = [[[[[1, 2, 4, 5], [5, 6, 7, 8], [9, 10, 11, 12]]], @@ -636,6 +664,7 @@ class StridedSliceTest(test_util.TensorFlowTestCase): _ = checker[..., bar:bar2] _ = checker[..., bar] _ = checker[..., 3] + _ = checker[..., 2 ** 64 // 2**63] # Test longs in Python 2 def testTensorIndexingTypeError(self): with self.session(use_gpu=True): @@ -650,6 +679,7 @@ class StridedSliceTest(test_util.TensorFlowTestCase): with self.assertRaisesRegexp(TypeError, expected): _ = checker[constant_op.constant(0.0)] + @test_util.run_deprecated_v1 def testExpand(self): with self.session(use_gpu=True): raw = [[[[[1, 2, 4, 5], [5, 6, 7, 8], [9, 10, 11, 12]]], @@ -667,6 +697,7 @@ class StridedSliceTest(test_util.TensorFlowTestCase): # Ellipsis in middle of two newaxis _ = checker[np.newaxis, ..., np.newaxis] + @test_util.run_deprecated_v1 def testExpandVariable(self): with self.session(use_gpu=True): x = variables.Variable(7, dtype=dtypes.int32) @@ -675,6 +706,7 @@ class StridedSliceTest(test_util.TensorFlowTestCase): self.assertEqual(y.shape, (1,)) self.assertAllEqual(y, (7,)) + @test_util.run_deprecated_v1 def testOptimizedCases(self): with self.session(use_gpu=True): checker = StridedSliceChecker(self, @@ -704,6 +736,7 @@ class StridedSliceShapeChecker(object): class StridedSliceShapeTest(test_util.TensorFlowTestCase): """Test the shape inference of StridedSliceShapes.""" + @test_util.run_deprecated_v1 def testUnknown(self): with self.session(use_gpu=True): uncertain_tensor = array_ops.placeholder(dtypes.float32) @@ -715,6 +748,7 @@ class StridedSliceShapeTest(test_util.TensorFlowTestCase): self.assertTrue(x is not None and y is not None or x is None and y is None) self.assertEqual(x.as_list(), y.as_list()) + @test_util.run_deprecated_v1 def testTensorShapeUncertain(self): with self.session(use_gpu=True): uncertain_tensor = array_ops.placeholder( @@ -738,6 +772,7 @@ class StridedSliceShapeTest(test_util.TensorFlowTestCase): self.tensorShapeEqual(a[::-1, :, array_ops.newaxis, ::-2], tensor_shape.TensorShape([5, None, 1, 4])) + @test_util.run_deprecated_v1 def testTensorValuedIndexShape(self): with self.session(use_gpu=True): defined_shape_tensor = array_ops.placeholder( @@ -794,6 +829,7 @@ class GradSliceChecker(object): class StridedSliceGradTest(test_util.TensorFlowTestCase): """Test that strided slice's custom gradient produces correct gradients.""" + @test_util.run_deprecated_v1 def testGradient(self): with self.session(use_gpu=True) as sess: var = variables.Variable( @@ -823,18 +859,20 @@ class StridedSliceGradTest(test_util.TensorFlowTestCase): grad = GradSliceChecker(self, sess, var, np.array(8)) _ = grad[tuple()] + @test_util.run_deprecated_v1 def testInt64Indices(self): with self.session(use_gpu=True) as sess: a = math_ops.range(3, dtype=dtypes.float32) index = constant_op.constant(1, dtype=dtypes.int64) b = 2. * a[index] grad, = gradients_impl.gradients(b, a) - self.assertAllEqual(sess.run(grad), [0., 2., 0.]) + self.assertAllEqual(self.evaluate(grad), [0., 2., 0.]) class StridedSliceGradTypeTest(test_util.TensorFlowTestCase): """Test varied index types and host located memory.""" + @test_util.run_deprecated_v1 def testHostVsDevice(self): with self.session(use_gpu=True) as sess: var2 = variables.Variable( @@ -842,20 +880,21 @@ class StridedSliceGradTypeTest(test_util.TensorFlowTestCase): math_ops.cast(math_ops.range(1, 5, 1), dtypes.float32), shape=(4, 1, 1))) varshape = variables.Variable([6, 4, 4], dtype=dtypes.int32) - sess.run(variables.global_variables_initializer()) + self.evaluate(variables.global_variables_initializer()) begin = constant_op.constant([0, 0, 0]) end = constant_op.constant([4, 1, 1]) strides = constant_op.constant([1, 1, 1]) foo = array_ops.strided_slice_grad(varshape, begin, end, strides, var2) sess.run(foo) + @test_util.run_deprecated_v1 def testInt64Shape(self): with self.session(use_gpu=True) as sess: original_dy = array_ops.reshape( math_ops.cast(math_ops.range(1, 5, 1), dtypes.float32), shape=(4, 1, 1)) original_shape = constant_op.constant([6, 4, 4], dtype=dtypes.int64) - sess.run(variables.global_variables_initializer()) + self.evaluate(variables.global_variables_initializer()) begin = constant_op.constant([0, 0, 0], dtype=dtypes.int64) end = constant_op.constant([4, 1, 1], dtype=dtypes.int64) strides = constant_op.constant([1, 1, 1], dtype=dtypes.int64) @@ -863,13 +902,14 @@ class StridedSliceGradTypeTest(test_util.TensorFlowTestCase): original_dy) sess.run(dx) + @test_util.run_deprecated_v1 def testMixedIndexTypes(self): with self.session(use_gpu=True) as sess: original_dy = array_ops.reshape( math_ops.cast(math_ops.range(1, 5, 1), dtypes.float32), shape=(4, 1, 1)) original_shape = constant_op.constant([6, 4, 4], dtype=dtypes.int64) - sess.run(variables.global_variables_initializer()) + self.evaluate(variables.global_variables_initializer()) begin = constant_op.constant([0, 0, 0], dtype=dtypes.int32) end = constant_op.constant([4, 1, 1], dtype=dtypes.int64) strides = constant_op.constant([1, 1, 1], dtype=dtypes.int64) @@ -971,6 +1011,7 @@ class StridedSliceAssignChecker(object): class SliceAssignTest(test_util.TensorFlowTestCase): + @test_util.run_deprecated_v1 def testInvalidSlice(self): with self.cached_session() as sess: foo = constant_op.constant([1, 2, 3]) @@ -1008,12 +1049,15 @@ class SliceAssignTest(test_util.TensorFlowTestCase): checker2[...] = 6 # ellipsis checker2[None] = [6] # new axis + @test_util.run_deprecated_v1 def testSliceAssign(self): self.doTestSliceAssign(use_resource=False) + @test_util.run_deprecated_v1 def testSliceAssignResource(self): self.doTestSliceAssign(use_resource=True) + @test_util.run_deprecated_v1 def testUninitialized(self): with self.assertRaisesRegexp( errors.FailedPreconditionError, @@ -1032,13 +1076,14 @@ class SliceAssignTest(test_util.TensorFlowTestCase): with self.assertRaises(TypeError): v[:].assign(too_large_val) + @test_util.run_deprecated_v1 def testTypeErrorResource(self): init_val = constant_op.constant([1, 2], dtype=dtypes.int32) too_small_val = constant_op.constant([3, 4], dtype=dtypes.int8) too_large_val = constant_op.constant([3, 4], dtype=dtypes.int64) v = resource_variable_ops.ResourceVariable(init_val) with self.cached_session() as sess: - sess.run(v.initializer) + self.evaluate(v.initializer) with self.assertRaises(ValueError): sess.run(v[:].assign(too_large_val)) with self.assertRaises(ValueError): @@ -1088,6 +1133,7 @@ class SequenceMaskTest(test_util.TensorFlowTestCase): with self.assertRaisesRegexp(ValueError, "maxlen must be scalar"): array_ops.sequence_mask([10, 20], [10, 20]) + @test_util.run_deprecated_v1 def testOneDimensionalWithMaxlen(self): with self.cached_session(): res = array_ops.sequence_mask(constant_op.constant([1, 3, 2]), 5) @@ -1097,7 +1143,7 @@ class SequenceMaskTest(test_util.TensorFlowTestCase): [[True, False, False, False, False], [True, True, True, False, False], [True, True, False, False, False]]) - @test_util.enable_c_shapes + @test_util.run_deprecated_v1 def testOneDimensionalDtypeWithoutMaxlen(self): with self.cached_session(): # test dtype and default maxlen: @@ -1108,7 +1154,7 @@ class SequenceMaskTest(test_util.TensorFlowTestCase): res.eval(), [[0.0, 0.0, 0.0, 0.0], [1.0, 0.0, 0.0, 0.0], [1.0, 1.0, 1.0, 1.0]]) - @test_util.enable_c_shapes + @test_util.run_deprecated_v1 def testOneDimensionalWithoutMaxlen(self): with self.cached_session(): res = array_ops.sequence_mask( @@ -1120,7 +1166,6 @@ class SequenceMaskTest(test_util.TensorFlowTestCase): [True, False, False, False], [True, True, True, True]]) - @test_util.enable_c_shapes def testTwoDimensional(self): with self.cached_session(): res = array_ops.sequence_mask(constant_op.constant([[1, 3, 2]]), 5) @@ -1138,11 +1183,13 @@ class SequenceMaskTest(test_util.TensorFlowTestCase): [[[0.0, 0.0, 0.0, 0.0], [1.0, 0.0, 0.0, 0.0], [1.0, 1.0, 1.0, 1.0]], [[1.0, 0.0, 0.0, 0.0], [1.0, 1.0, 0.0, 0.0], [1.0, 1.0, 1.0, 0.0]]]) + @test_util.run_deprecated_v1 def testUnknownShape(self): lengths = array_ops.placeholder(dtype=dtypes.int32) res = array_ops.sequence_mask(lengths) self.assertEqual(res.shape, None) + @test_util.run_deprecated_v1 def testDtypes(self): def check_dtypes(lengths_dtype, maxlen_dtype): @@ -1165,6 +1212,7 @@ class SequenceMaskTest(test_util.TensorFlowTestCase): class ConcatSliceResourceTest(test_util.TensorFlowTestCase): @test_util.run_in_graph_and_eager_modes + @test_util.run_deprecated_v1 def testConcatSlice(self): r1 = test_ops.stub_resource_handle_op(container="a", shared_name="b") r2 = test_ops.stub_resource_handle_op(container="a", shared_name="c") @@ -1187,18 +1235,18 @@ class IdentityTest(test_util.TensorFlowTestCase): self.assertAllEqual(x.numpy(), y.numpy()) self.assertTrue(device in y.device.lower()) - with ops.device("gpu:0"): + with test_util.force_gpu(): a = constant_op.constant([[2], [3]], dtype=dtypes.float32) - with ops.device("gpu:0"): + with test_util.force_gpu(): b = array_ops.identity(a) _test(a, b, "gpu") - with ops.device("cpu:0"): + with test_util.force_cpu(): c = array_ops.identity(b) _test(b, c, "cpu") - with ops.device("cpu:0"): + with test_util.force_cpu(): d = array_ops.identity(c) _test(c, d, "cpu") - with ops.device("gpu:0"): + with test_util.force_gpu(): e = array_ops.identity(d) _test(d, e, "gpu") @@ -1220,6 +1268,7 @@ class PadTest(test_util.TensorFlowTestCase): class InvertPermutationTest(test_util.TensorFlowTestCase): + @test_util.run_deprecated_v1 def testInvertPermutation(self): for dtype in [dtypes.int32, dtypes.int64]: with self.cached_session(use_gpu=True): @@ -1254,12 +1303,14 @@ class UnravelIndexTest(test_util.TensorFlowTestCase): class GuaranteeConstOpTest(test_util.TensorFlowTestCase): + @test_util.run_deprecated_v1 def testSimple(self): with self.cached_session(): a = array_ops.constant(10) guarantee_a = array_ops.guarantee_const(a) self.assertEqual(10, guarantee_a.eval()) + @test_util.run_deprecated_v1 def testVariables(self): with self.cached_session() as sess: for use_resource in [False, True]: @@ -1268,9 +1319,10 @@ class GuaranteeConstOpTest(test_util.TensorFlowTestCase): initializer=init_ops.constant_initializer(10.0), use_resource=use_resource) guarantee_a = array_ops.guarantee_const(a) - sess.run(variables.global_variables_initializer()) + self.evaluate(variables.global_variables_initializer()) self.assertEqual(10.0, guarantee_a.eval()) + @test_util.run_deprecated_v1 def testResourceRejection(self): with self.cached_session() as sess: a = variable_scope.get_variable( @@ -1278,7 +1330,7 @@ class GuaranteeConstOpTest(test_util.TensorFlowTestCase): initializer=init_ops.constant_initializer(10.0), use_resource=True) guarantee_a = array_ops.guarantee_const(a.handle) - sess.run(variables.global_variables_initializer()) + self.evaluate(variables.global_variables_initializer()) with self.assertRaisesWithPredicateMatch(errors.InvalidArgumentError, "cannot be a resource variable"): guarantee_a.eval() @@ -1286,6 +1338,7 @@ class GuaranteeConstOpTest(test_util.TensorFlowTestCase): class SnapshotOpTest(test_util.TensorFlowTestCase): + @test_util.run_deprecated_v1 def testInvertPermutation(self): for dtype in [dtypes.int32, dtypes.int64, dtypes.float32, dtypes.float64]: with self.cached_session(use_gpu=True): diff --git a/tensorflow/python/kernel_tests/as_string_op_test.py b/tensorflow/python/kernel_tests/as_string_op_test.py index dd4a90e5f65..287701a73e4 100644 --- a/tensorflow/python/kernel_tests/as_string_op_test.py +++ b/tensorflow/python/kernel_tests/as_string_op_test.py @@ -20,6 +20,7 @@ from __future__ import print_function import numpy as np from tensorflow.python.framework import dtypes +from tensorflow.python.framework import test_util from tensorflow.python.ops import array_ops from tensorflow.python.ops import string_ops from tensorflow.python.platform import test @@ -27,6 +28,7 @@ from tensorflow.python.platform import test class AsStringOpTest(test.TestCase): + @test_util.run_deprecated_v1 def testFloat(self): float_inputs_ = [ 0, 1, -1, 0.5, 0.25, 0.125, float("INF"), float("NAN"), float("-INF") @@ -78,6 +80,7 @@ class AsStringOpTest(test.TestCase): output = string_ops.as_string(input_, fill="ab") output.eval(feed_dict={input_: float_inputs_}) + @test_util.run_deprecated_v1 def testInt(self): # Cannot use values outside -128..127 for test, because we're also # testing int8 @@ -112,6 +115,7 @@ class AsStringOpTest(test.TestCase): output = string_ops.as_string(input_, precision=0) output.eval(feed_dict={input_: int_inputs_}) + @test_util.run_deprecated_v1 def testLargeInt(self): # Cannot use values outside -128..127 for test, because we're also # testing int8 @@ -130,6 +134,7 @@ class AsStringOpTest(test.TestCase): result = output.eval(feed_dict={input_: int_inputs_}) self.assertAllEqual(s(result), ["%d" % x for x in int_inputs_]) + @test_util.run_deprecated_v1 def testHalfInt(self): s = lambda strs: [x.decode("ascii") for x in strs] @@ -140,6 +145,7 @@ class AsStringOpTest(test.TestCase): result = output.eval(feed_dict={input_: int_inputs_}) self.assertAllEqual(s(result), ["%d" % x for x in int_inputs_]) + @test_util.run_deprecated_v1 def testBool(self): bool_inputs_ = [False, True] s = lambda strs: [x.decode("ascii") for x in strs] @@ -152,6 +158,7 @@ class AsStringOpTest(test.TestCase): result = output.eval(feed_dict={input_: bool_inputs_}) self.assertAllEqual(s(result), ["false", "true"]) + @test_util.run_deprecated_v1 def testComplex(self): float_inputs_ = [ 0, 1, -1, 0.5, 0.25, 0.125, complex("INF"), complex("NAN"), diff --git a/tensorflow/python/kernel_tests/atrous_conv2d_test.py b/tensorflow/python/kernel_tests/atrous_conv2d_test.py index 1d82b3d0588..a13e325835c 100644 --- a/tensorflow/python/kernel_tests/atrous_conv2d_test.py +++ b/tensorflow/python/kernel_tests/atrous_conv2d_test.py @@ -22,6 +22,7 @@ import numpy as np from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes +from tensorflow.python.framework import test_util from tensorflow.python.ops import array_ops from tensorflow.python.ops import gradient_checker from tensorflow.python.ops import nn_impl @@ -58,6 +59,7 @@ def _upsample_filters(filters, rate): class AtrousConv2DTest(test.TestCase): + @test_util.run_deprecated_v1 def testAtrousConv2DForward(self): with self.session(use_gpu=True): # Input: [batch, height, width, input_depth] @@ -79,8 +81,10 @@ class AtrousConv2DTest(test.TestCase): y1 = nn_ops.atrous_conv2d(x, f, rate, padding=padding) y2 = nn_ops.conv2d( x, f_up, strides=[1, 1, 1, 1], padding=padding) - self.assertAllClose(y1.eval(), y2.eval(), rtol=1e-3, atol=1e-3) + self.assertAllClose( + y1.eval(), self.evaluate(y2), rtol=1e-3, atol=1e-3) + @test_util.run_deprecated_v1 def testAtrousSequence(self): """Tests optimization of sequence of atrous convolutions. @@ -131,8 +135,10 @@ class AtrousConv2DTest(test.TestCase): y2 = nn_ops.conv2d(y2, f, strides=[1, 1, 1, 1], padding=padding) y2 = nn_ops.conv2d(y2, f, strides=[1, 1, 1, 1], padding=padding) y2 = array_ops.batch_to_space(y2, crops=pad, block_size=rate) - self.assertAllClose(y1.eval(), y2.eval(), rtol=1e-2, atol=1e-2) + self.assertAllClose( + y1.eval(), self.evaluate(y2), rtol=1e-2, atol=1e-2) + @test_util.run_deprecated_v1 def testGradient(self): with self.session(use_gpu=True): # Input: [batch, height, width, input_depth] @@ -160,6 +166,7 @@ class AtrousConv2DTest(test.TestCase): class AtrousConv2DTransposeTest(test.TestCase): + @test_util.run_deprecated_v1 def testAtrousConv2DTransposeForward(self): with self.session(use_gpu=True): # Input: [batch, height, width, input_depth] @@ -193,11 +200,13 @@ class AtrousConv2DTransposeTest(test.TestCase): padding) y2 = nn_ops.conv2d_transpose( x, f_up, y_shape, strides=[1, 1, 1, 1], padding=padding) - self.assertAllClose(y1.eval(), y2.eval(), rtol=1e-3, atol=1e-3) + self.assertAllClose( + y1.eval(), self.evaluate(y2), rtol=1e-3, atol=1e-3) class AtrousDepthwiseConv2DTest(test.TestCase): + @test_util.run_deprecated_v1 def testAtrousDepthwiseConv2DForward(self): strides = [1, 1, 1, 1] with self.session(use_gpu=True): @@ -220,7 +229,8 @@ class AtrousDepthwiseConv2DTest(test.TestCase): y1 = nn_impl.depthwise_conv2d( x, f, strides, padding, rate=[rate, rate]) y2 = nn_impl.depthwise_conv2d(x, f_up, strides, padding) - self.assertAllClose(y1.eval(), y2.eval(), rtol=1e-3, atol=1e-3) + self.assertAllClose( + y1.eval(), self.evaluate(y2), rtol=1e-3, atol=1e-3) if __name__ == "__main__": diff --git a/tensorflow/python/kernel_tests/attention_ops_test.py b/tensorflow/python/kernel_tests/attention_ops_test.py index 1e09ba5b65c..00dba9996dd 100644 --- a/tensorflow/python/kernel_tests/attention_ops_test.py +++ b/tensorflow/python/kernel_tests/attention_ops_test.py @@ -85,7 +85,7 @@ class ExtractGlimpseTest(test.TestCase): # Evaluate the TensorFlow Graph. with self.cached_session() as sess: - value_rows, value_cols = sess.run([glimpse_rows, glimpse_cols]) + value_rows, value_cols = self.evaluate([glimpse_rows, glimpse_cols]) # Check dimensions of returned glimpse. self.assertEqual(value_rows.shape[1], glimpse_sizes[0]) @@ -121,8 +121,7 @@ class ExtractGlimpseTest(test.TestCase): with self.cached_session(): result = image_ops.extract_glimpse(empty_image, [1, 1], offsets) self.assertAllEqual( - np.zeros( - (0, 1, 1, 0), dtype=np.float32), result.eval()) + np.zeros((0, 1, 1, 0), dtype=np.float32), self.evaluate(result)) def testLargeCenterGlimpse(self): self._VerifyValues( diff --git a/tensorflow/python/kernel_tests/barrier_ops_test.py b/tensorflow/python/kernel_tests/barrier_ops_test.py index 4d36b3a4658..60fe6f0eecd 100644 --- a/tensorflow/python/kernel_tests/barrier_ops_test.py +++ b/tensorflow/python/kernel_tests/barrier_ops_test.py @@ -25,6 +25,7 @@ import numpy as np from tensorflow.python.framework import dtypes from tensorflow.python.framework import errors_impl from tensorflow.python.framework import ops +from tensorflow.python.framework import test_util from tensorflow.python.ops import data_flow_ops from tensorflow.python.platform import test @@ -66,6 +67,7 @@ class BarrierTest(test.TestCase): attr { key: 'shared_name' value: { s: 'B' } } """, b.barrier_ref.op.node_def) + @test_util.run_deprecated_v1 def testInsertMany(self): with self.cached_session(): b = data_flow_ops.Barrier( @@ -90,6 +92,7 @@ class BarrierTest(test.TestCase): data_flow_ops.Barrier( (dtypes.float32, dtypes.float32), shapes=((1,), (0,)), name="B") + @test_util.run_deprecated_v1 def testInsertManyEmptyTensorUnknown(self): with self.cached_session(): b = data_flow_ops.Barrier((dtypes.float32, dtypes.float32), name="B") @@ -102,6 +105,7 @@ class BarrierTest(test.TestCase): ".*Tensors with no elements are not supported.*"): insert_0_op.run() + @test_util.run_deprecated_v1 def testTakeMany(self): with self.cached_session() as sess: b = data_flow_ops.Barrier( @@ -127,6 +131,7 @@ class BarrierTest(test.TestCase): self.assertEqual(values_0_val[idx], v0) self.assertEqual(values_1_val[idx], v1) + @test_util.run_deprecated_v1 def testTakeManySmallBatch(self): with self.cached_session() as sess: b = data_flow_ops.Barrier( @@ -191,6 +196,7 @@ class BarrierTest(test.TestCase): with self.assertRaisesOpError("is closed"): insert_1_3_op.run() + @test_util.run_deprecated_v1 def testUseBarrierWithShape(self): with self.cached_session() as sess: b = data_flow_ops.Barrier( @@ -220,6 +226,7 @@ class BarrierTest(test.TestCase): self.assertAllEqual(values_0_val[idx], v0) self.assertAllEqual(values_1_val[idx], v1) + @test_util.run_deprecated_v1 def testParallelInsertMany(self): with self.cached_session() as sess: b = data_flow_ops.Barrier(dtypes.float32, shapes=()) @@ -229,7 +236,7 @@ class BarrierTest(test.TestCase): insert_ops = [b.insert_many(0, [k], [v]) for k, v in zip(keys, values)] take_t = b.take_many(10) - sess.run(insert_ops) + self.evaluate(insert_ops) self.assertEquals(size_t.eval(), [10]) indices_val, keys_val, values_val = sess.run( @@ -240,6 +247,7 @@ class BarrierTest(test.TestCase): idx = keys_val.tolist().index(k) self.assertEqual(values_val[idx], v) + @test_util.run_deprecated_v1 def testParallelTakeMany(self): with self.cached_session() as sess: b = data_flow_ops.Barrier(dtypes.float32, shapes=()) @@ -274,6 +282,7 @@ class BarrierTest(test.TestCase): self.assertItemsEqual( zip(keys, values), [(k[0], v[0]) for k, v in zip(key_vals, value_vals)]) + @test_util.run_deprecated_v1 def testBlockingTakeMany(self): with self.cached_session() as sess: b = data_flow_ops.Barrier(dtypes.float32, shapes=()) @@ -296,6 +305,7 @@ class BarrierTest(test.TestCase): insert_op.run() t.join() + @test_util.run_deprecated_v1 def testParallelInsertManyTakeMany(self): with self.cached_session() as sess: b = data_flow_ops.Barrier( @@ -375,6 +385,7 @@ class BarrierTest(test.TestCase): 2 + outer_indices_from_keys + inner_indices_from_keys)).T self.assertAllEqual(taken_i["values_1"], expected_values_1) + @test_util.run_deprecated_v1 def testClose(self): with self.cached_session() as sess: b = data_flow_ops.Barrier( @@ -433,6 +444,7 @@ class BarrierTest(test.TestCase): with self.assertRaisesOpError("is closed and has insufficient elements"): sess.run(take_t[0]) + @test_util.run_deprecated_v1 def testCancel(self): with self.cached_session() as sess: b = data_flow_ops.Barrier( @@ -491,10 +503,11 @@ class BarrierTest(test.TestCase): b = data_flow_ops.Barrier( (dtypes.float32, dtypes.float32), shapes=((), ()), name="B") take_t = b.take_many(1, allow_small_batch=True) - sess.run(b.close(cancel)) + self.evaluate(b.close(cancel)) with self.assertRaisesOpError("is closed and has insufficient elements"): - sess.run(take_t) + self.evaluate(take_t) + @test_util.run_deprecated_v1 def testClosedEmptyBarrierTakeManyAllowSmallBatchRaises(self): self._testClosedEmptyBarrierTakeManyAllowSmallBatchRaises(cancel=False) self._testClosedEmptyBarrierTakeManyAllowSmallBatchRaises(cancel=True) @@ -569,9 +582,11 @@ class BarrierTest(test.TestCase): sorted(taken), [0] * (num_iterations // 2) + [10] * (num_iterations // 2)) + @test_util.run_deprecated_v1 def testParallelInsertManyTakeManyCloseHalfwayThrough(self): self._testParallelInsertManyTakeManyCloseHalfwayThrough(cancel=False) + @test_util.run_deprecated_v1 def testParallelInsertManyTakeManyCancelHalfwayThrough(self): self._testParallelInsertManyTakeManyCloseHalfwayThrough(cancel=True) @@ -669,12 +684,15 @@ class BarrierTest(test.TestCase): else: self.assertEqual(taken, [10] * num_iterations) + @test_util.run_deprecated_v1 def testParallelPartialInsertManyTakeManyCloseHalfwayThrough(self): self._testParallelPartialInsertManyTakeManyCloseHalfwayThrough(cancel=False) + @test_util.run_deprecated_v1 def testParallelPartialInsertManyTakeManyCancelHalfwayThrough(self): self._testParallelPartialInsertManyTakeManyCloseHalfwayThrough(cancel=True) + @test_util.run_deprecated_v1 def testIncompatibleSharedBarrierErrors(self): with self.cached_session(): # Do component types and shapes. diff --git a/tensorflow/python/kernel_tests/base64_ops_test.py b/tensorflow/python/kernel_tests/base64_ops_test.py index 1b399942efb..bb903d827f2 100644 --- a/tensorflow/python/kernel_tests/base64_ops_test.py +++ b/tensorflow/python/kernel_tests/base64_ops_test.py @@ -93,7 +93,7 @@ class Base64OpsTest(test_util.TensorFlowTestCase): decoded = string_ops.decode_base64(encoded) with self.cached_session() as sess: - encoded_value, decoded_value = sess.run([encoded, decoded]) + encoded_value, decoded_value = self.evaluate([encoded, decoded]) self.assertEqual(encoded_value.shape, msg.shape) self.assertEqual(decoded_value.shape, msg.shape) diff --git a/tensorflow/python/kernel_tests/basic_gpu_test.py b/tensorflow/python/kernel_tests/basic_gpu_test.py index 225c1b35ae5..1a8513d022d 100644 --- a/tensorflow/python/kernel_tests/basic_gpu_test.py +++ b/tensorflow/python/kernel_tests/basic_gpu_test.py @@ -44,13 +44,13 @@ class GPUBinaryOpsTest(test.TestCase): inx = ops.convert_to_tensor(x) iny = ops.convert_to_tensor(y) out = tf_func(inx, iny) - tf_gpu = sess.run(out) + tf_gpu = self.evaluate(out) with self.cached_session(use_gpu=False) as sess: inx = ops.convert_to_tensor(x) iny = ops.convert_to_tensor(y) out = tf_func(inx, iny) - tf_cpu = sess.run(out) + tf_cpu = self.evaluate(out) self.assertAllClose(tf_cpu, tf_gpu) @@ -96,7 +96,7 @@ class MathBuiltinUnaryTest(test.TestCase): with self.cached_session(use_gpu=use_gpu) as sess: inx = ops.convert_to_tensor(x) ofunc = tf_func(inx) - tf_out = sess.run(ofunc) + tf_out = self.evaluate(ofunc) self.assertAllClose(np_out, tf_out) def _inv(self, x): @@ -148,7 +148,7 @@ class MathBuiltinUnaryTest(test.TestCase): iny = ops.convert_to_tensor(y + 0.1) ofunc = inx / iny out_func2 = math_ops.floor(ofunc) - tf_out = sess.run(out_func2) + tf_out = self.evaluate(out_func2) self.assertAllClose(np_out, tf_out) @@ -159,6 +159,7 @@ class BroadcastSimpleTest(test.TestCase): with self.cached_session(use_gpu=True) as sess: return sess.run(broadcast_gradient_args(xs, ys)) + @test_util.run_deprecated_v1 def testBroadcast(self): r0, r1 = self._GetGradientArgs([2, 3, 5], [1]) self.assertAllEqual(r0, []) @@ -214,11 +215,12 @@ class BroadcastSimpleTest(test.TestCase): inx = ops.convert_to_tensor(x) iny = ops.convert_to_tensor(y) out = tf_func(inx, iny) - tf_gpu = out.eval() + tf_gpu = self.evaluate(out) self.assertAllClose(np_ans, tf_gpu) self.assertShapeEqual(np_ans, out) # TODO(zhifengc/ke): make gradient checker work on GPU. + @test_util.run_deprecated_v1 def testGradient(self): x = (1 + np.linspace(0, 5, np.prod([1, 3, 2]))).astype(np.float32).reshape( [1, 3, 2]) @@ -255,6 +257,7 @@ class GpuMultiSessionMemoryTest(test_util.TensorFlowTestCase): if len(results) != 1: break + @test_util.run_deprecated_v1 def testConcurrentSessions(self): n_threads = 4 threads = [] diff --git a/tensorflow/python/kernel_tests/batch_gather_op_test.py b/tensorflow/python/kernel_tests/batch_gather_op_test.py index 547506d844d..7e0b3e1b5ea 100644 --- a/tensorflow/python/kernel_tests/batch_gather_op_test.py +++ b/tensorflow/python/kernel_tests/batch_gather_op_test.py @@ -23,6 +23,7 @@ import numpy as np from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes +from tensorflow.python.framework import test_util from tensorflow.python.ops import array_ops from tensorflow.python.platform import test @@ -52,7 +53,7 @@ class GatherTest(test.TestCase, parameterized.TestCase): gather_t = array_ops.batch_gather(params, indices_tf) expected_result = np.array([3, 7]) np_val = self._buildParams(expected_result, dtype) - gather_val = gather_t.eval() + gather_val = self.evaluate(gather_t) self.assertAllEqual(np_val, gather_val) self.assertEqual(np_val.shape, gather_t.get_shape()) @@ -68,7 +69,7 @@ class GatherTest(test.TestCase, parameterized.TestCase): gather_t = array_ops.batch_gather(params, indices_tf) expected_result = np.array([[3], [15]]) np_val = self._buildParams(expected_result, dtype) - gather_val = gather_t.eval() + gather_val = self.evaluate(gather_t) self.assertAllEqual(np_val, gather_val) self.assertEqual(np_val.shape, gather_t.get_shape()) @@ -81,12 +82,13 @@ class GatherTest(test.TestCase, parameterized.TestCase): params = constant_op.constant(params_np) indices_tf = constant_op.constant(indices) gather_t = array_ops.batch_gather(params, indices_tf) - gather_val = gather_t.eval() + gather_val = self.evaluate(gather_t) expected_result = np.array([[[2, 0], [7, 5]], [[10, 8], [11, 15]]]) np_val = self._buildParams(expected_result, dtype) self.assertAllEqual(np_val, gather_val) self.assertEqual(np_val.shape, gather_t.get_shape()) + @test_util.run_deprecated_v1 def testString(self): params = np.array([[b"asdf", b"zxcv"], [b"qwer", b"uiop"]]) with self.cached_session(): @@ -94,6 +96,7 @@ class GatherTest(test.TestCase, parameterized.TestCase): self.assertAllEqual([[b"qwer", b"uiop"]], array_ops.batch_gather(params, indices_tf).eval()) + @test_util.run_deprecated_v1 def testUnknownIndices(self): params = constant_op.constant([[0, 1, 2]]) indices = array_ops.placeholder(dtypes.int32, shape=[None, None]) @@ -106,6 +109,7 @@ class GatherTest(test.TestCase, parameterized.TestCase): with self.assertRaisesOpError(r"indices\[0\] = 7 is not in \[0, 2\)"): array_ops.batch_gather(params, [7]).eval() + @test_util.run_deprecated_v1 def testEmptySlices(self): with self.session(use_gpu=True): for dtype in _TEST_TYPES: diff --git a/tensorflow/python/kernel_tests/batch_matmul_op_test.py b/tensorflow/python/kernel_tests/batch_matmul_op_test.py index 8f6c089b423..a0ad8151b26 100644 --- a/tensorflow/python/kernel_tests/batch_matmul_op_test.py +++ b/tensorflow/python/kernel_tests/batch_matmul_op_test.py @@ -86,7 +86,7 @@ class BatchMatmulOpTest(test.TestCase): with self.cached_session(use_gpu=is_floating) as sess: if static_shape: z0 = math_ops.matmul(x, y, adjoint_a=adjoint_a, adjoint_b=adjoint_b) - z0_val = z0.eval() + z0_val = self.evaluate(z0) else: x_ph = array_ops.placeholder(x.dtype) y_ph = array_ops.placeholder(y.dtype) diff --git a/tensorflow/python/kernel_tests/batch_scatter_ops_test.py b/tensorflow/python/kernel_tests/batch_scatter_ops_test.py index 742a2048831..eefcdc508f0 100644 --- a/tensorflow/python/kernel_tests/batch_scatter_ops_test.py +++ b/tensorflow/python/kernel_tests/batch_scatter_ops_test.py @@ -22,6 +22,7 @@ import numpy as np from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes +from tensorflow.python.framework import test_util from tensorflow.python.ops import state_ops from tensorflow.python.ops import variables from tensorflow.python.platform import test @@ -73,6 +74,7 @@ class ScatterTest(test.TestCase): tf_scatter(ref, indices, updates).eval() self.assertAllClose(ref.eval(), new) + @test_util.run_deprecated_v1 def testVariableRankUpdate(self): vtypes = [np.float32, np.float64] for vtype in vtypes: @@ -80,6 +82,7 @@ class ScatterTest(test.TestCase): self._VariableRankTest( state_ops.batch_scatter_update, vtype, itype) + @test_util.run_deprecated_v1 def testBooleanScatterUpdate(self): with self.session(use_gpu=False) as session: var = variables.Variable([True, False]) @@ -91,8 +94,9 @@ class ScatterTest(test.TestCase): session.run([update0, update1]) - self.assertAllEqual([False, True], var.eval()) + self.assertAllEqual([False, True], self.evaluate(var)) + @test_util.run_deprecated_v1 def testScatterOutOfRange(self): params = np.array([1, 2, 3, 4, 5, 6]).astype(np.float32) updates = np.array([-3, -4, -5]).astype(np.float32) diff --git a/tensorflow/python/kernel_tests/batchtospace_op_test.py b/tensorflow/python/kernel_tests/batchtospace_op_test.py index 03f3f64353d..c422df8806f 100644 --- a/tensorflow/python/kernel_tests/batchtospace_op_test.py +++ b/tensorflow/python/kernel_tests/batchtospace_op_test.py @@ -27,6 +27,7 @@ import numpy as np from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops +from tensorflow.python.framework import test_util from tensorflow.python.ops import array_ops from tensorflow.python.ops import gen_array_ops from tensorflow.python.ops import gradient_checker @@ -50,6 +51,7 @@ class CppOpImpl(object): class BatchToSpaceDepthToSpace(test.TestCase, PythonOpImpl): # Verifies that: batch_to_space(x) = transpose(depth_to_space(transpose(x))) + @test_util.run_deprecated_v1 def testDepthToSpaceTranspose(self): x = np.arange(20 * 5 * 8 * 7, dtype=np.float32).reshape([20, 5, 8, 7]) block_size = 2 @@ -70,6 +72,7 @@ class BatchToSpaceDepthToSpaceCpp(BatchToSpaceDepthToSpace, CppOpImpl): class BatchToSpaceErrorHandlingTest(test.TestCase, PythonOpImpl): + @test_util.run_deprecated_v1 def testInputWrongDimMissingBatch(self): # The input is missing the first dimension ("batch") x_np = [[[1], [2]], [[3], [4]]] @@ -78,6 +81,7 @@ class BatchToSpaceErrorHandlingTest(test.TestCase, PythonOpImpl): with self.assertRaises(ValueError): _ = self.batch_to_space(x_np, crops, block_size) + @test_util.run_deprecated_v1 def testBlockSize0(self): # The block size is 0. x_np = [[[[1], [2]], [[3], [4]]]] @@ -87,6 +91,7 @@ class BatchToSpaceErrorHandlingTest(test.TestCase, PythonOpImpl): out_tf = self.batch_to_space(x_np, crops, block_size) out_tf.eval() + @test_util.run_deprecated_v1 def testBlockSizeOne(self): # The block size is 1. The block size needs to be > 1. x_np = [[[[1], [2]], [[3], [4]]]] @@ -96,6 +101,7 @@ class BatchToSpaceErrorHandlingTest(test.TestCase, PythonOpImpl): out_tf = self.batch_to_space(x_np, crops, block_size) out_tf.eval() + @test_util.run_deprecated_v1 def testBlockSizeLarger(self): # The block size is too large for this input. x_np = [[[[1], [2]], [[3], [4]]]] @@ -105,6 +111,7 @@ class BatchToSpaceErrorHandlingTest(test.TestCase, PythonOpImpl): out_tf = self.batch_to_space(x_np, crops, block_size) out_tf.eval() + @test_util.run_deprecated_v1 def testBlockSizeSquaredNotDivisibleBatch(self): # The block size squared does not divide the batch. x_np = [[[[1], [2], [3]], [[3], [4], [7]]]] @@ -113,6 +120,7 @@ class BatchToSpaceErrorHandlingTest(test.TestCase, PythonOpImpl): with self.assertRaises(ValueError): _ = self.batch_to_space(x_np, crops, block_size) + @test_util.run_deprecated_v1 def testUnknownShape(self): t = self.batch_to_space( array_ops.placeholder(dtypes.float32), @@ -160,28 +168,35 @@ class BatchToSpaceNDErrorHandlingTest(test.TestCase): self._testStaticShape(input_shape, block_shape, paddings, error) self._testDynamicShape(input_shape, block_shape, paddings) + @test_util.run_deprecated_v1 def testInputWrongDimMissingBatch(self): self._testShape([2, 2], [2, 2], [[0, 0], [0, 0]], ValueError) self._testShape([2, 2, 3], [2, 2, 3], [[0, 0], [0, 0]], ValueError) + @test_util.run_deprecated_v1 def testBlockSize0(self): # The block size is 0. self._testShape([1, 2, 2, 1], [0, 1], [[0, 0], [0, 0]], ValueError) + @test_util.run_deprecated_v1 def testBlockSizeNegative(self): self._testShape([1, 2, 2, 1], [-1, 1], [[0, 0], [0, 0]], ValueError) + @test_util.run_deprecated_v1 def testNegativePadding(self): self._testShape([1, 2, 2], [1, 1], [[0, -1], [0, 0]], ValueError) + @test_util.run_deprecated_v1 def testCropTooLarge(self): # The amount to crop exceeds the padded size. self._testShape([1 * 2 * 2, 2, 3, 1], [2, 2], [[3, 2], [0, 0]], ValueError) + @test_util.run_deprecated_v1 def testBlockSizeSquaredNotDivisibleBatch(self): # The batch dimension is not divisible by the product of the block_shape. self._testShape([3, 1, 1, 1], [2, 3], [[0, 0], [0, 0]], ValueError) + @test_util.run_deprecated_v1 def testUnknownShape(self): # Verify that input shape and paddings shape can be unknown. _ = array_ops.batch_to_space_nd( @@ -263,18 +278,21 @@ class BatchToSpaceGradientTest(test.TestCase, PythonOpImpl): # Don't use very large numbers as dimensions here as the result is tensor # with cartesian product of the dimensions. + @test_util.run_deprecated_v1 def testSmall(self): block_size = 2 crop_beg = 0 crop_end = 0 self._compare(1, 2, 3, 5, block_size, crop_beg, crop_end) + @test_util.run_deprecated_v1 def testSmall2(self): block_size = 2 crop_beg = 0 crop_end = 0 self._compare(2, 4, 3, 2, block_size, crop_beg, crop_end) + @test_util.run_deprecated_v1 def testSmallCrop1x1(self): block_size = 2 crop_beg = 1 @@ -316,14 +334,17 @@ class BatchToSpaceNDGradientTest(test.TestCase): # Don't use very large numbers as dimensions here as the result is tensor # with cartesian product of the dimensions. + @test_util.run_deprecated_v1 def testSmall(self): for dtype in [dtypes.int64, dtypes.int32]: self._compare([1, 2, 3, 5], [2, 2], [[0, 0], [0, 0]], dtype) + @test_util.run_deprecated_v1 def testSmall2(self): for dtype in [dtypes.int64, dtypes.int32]: self._compare([2, 4, 3, 2], [2, 2], [[0, 0], [0, 0]], dtype) + @test_util.run_deprecated_v1 def testSmallCrop1x1(self): for dtype in [dtypes.int64, dtypes.int32]: self._compare([1, 2, 3, 5], [2, 2], [[1, 1], [1, 1]], dtype) diff --git a/tensorflow/python/kernel_tests/bcast_ops_test.py b/tensorflow/python/kernel_tests/bcast_ops_test.py index 3ec820aeada..ae00955ac29 100644 --- a/tensorflow/python/kernel_tests/bcast_ops_test.py +++ b/tensorflow/python/kernel_tests/bcast_ops_test.py @@ -20,6 +20,7 @@ from __future__ import print_function from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes +from tensorflow.python.framework import test_util from tensorflow.python.ops.gen_array_ops import broadcast_args from tensorflow.python.ops.gen_array_ops import broadcast_gradient_args from tensorflow.python.platform import test @@ -35,6 +36,7 @@ class BcastOpsTest(test.TestCase): with self.cached_session() as sess: return sess.run(broadcast_gradient_args(xs, ys)) + @test_util.run_deprecated_v1 def testBasic(self): r = self._GetBroadcastShape([2, 3, 5], [1]) self.assertAllEqual(r, [2, 3, 5]) @@ -66,6 +68,7 @@ class BcastOpsTest(test.TestCase): r = self._GetBroadcastShape([3, 1], [2, 1, 5]) self.assertAllEqual(r, [2, 3, 5]) + @test_util.run_deprecated_v1 def testBasicGradient(self): r0, r1 = self._GetGradientArgs([2, 3, 5], [1]) self.assertAllEqual(r0, []) @@ -107,6 +110,7 @@ class BcastOpsTest(test.TestCase): self.assertAllEqual(r0, [0, 2]) self.assertAllEqual(r1, [1]) + @test_util.run_deprecated_v1 def testZeroDims(self): r = self._GetBroadcastShape([2, 0, 3, 0, 5], [3, 0, 5]) self.assertAllEqual(r, [2, 0, 3, 0, 5]) @@ -120,6 +124,7 @@ class BcastOpsTest(test.TestCase): r = self._GetBroadcastShape([3, 1, 5], [2, 0, 3, 0, 5]) self.assertAllEqual(r, [2, 0, 3, 0, 5]) + @test_util.run_deprecated_v1 def testZeroDimsGradient(self): r0, r1 = self._GetGradientArgs([2, 0, 3, 0, 5], [3, 0, 5]) self.assertAllEqual(r0, []) @@ -137,6 +142,7 @@ class BcastOpsTest(test.TestCase): self.assertAllEqual(r0, [0, 1, 3]) self.assertAllEqual(r1, []) + @test_util.run_deprecated_v1 def testDataTypes(self): for dtype in [dtypes.int32, dtypes.int64]: r = self._GetBroadcastShape( diff --git a/tensorflow/python/kernel_tests/benchmark_test.py b/tensorflow/python/kernel_tests/benchmark_test.py index 5777a5d0970..bffa5e6e8f4 100644 --- a/tensorflow/python/kernel_tests/benchmark_test.py +++ b/tensorflow/python/kernel_tests/benchmark_test.py @@ -21,9 +21,12 @@ import json import os import random +import numpy as np + from tensorflow.core.util import test_log_pb2 from tensorflow.python.client import session -from tensorflow.python.framework import constant_op +from tensorflow.python.framework import dtypes +from tensorflow.python.ops import array_ops from tensorflow.python.platform import benchmark from tensorflow.python.platform import gfile from tensorflow.python.platform import test @@ -64,11 +67,17 @@ class TestReportingBenchmark(test.Benchmark): "other_key": "string"}) def benchmark_times_an_op(self): + input_size = 5 with session.Session(config=benchmark.benchmark_config()) as sess: - a = constant_op.constant(0.0) + a = array_ops.placeholder(dtype=dtypes.float32, shape=(input_size)) a_plus_a = a + a return self.run_op_benchmark( - sess, a_plus_a, min_iters=1000, store_trace=True, name="op_benchmark") + sess, + a_plus_a, + feed_dict={a: np.arange(input_size)}, + min_iters=1000, + store_trace=True, + name="op_benchmark") class BenchmarkTest(test.TestCase): diff --git a/tensorflow/python/kernel_tests/betainc_op_test.py b/tensorflow/python/kernel_tests/betainc_op_test.py index 92d21462d52..9dc34a60628 100644 --- a/tensorflow/python/kernel_tests/betainc_op_test.py +++ b/tensorflow/python/kernel_tests/betainc_op_test.py @@ -24,6 +24,7 @@ import numpy as np from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes +from tensorflow.python.framework import test_util from tensorflow.python.ops import array_ops from tensorflow.python.ops import gradient_checker from tensorflow.python.ops import gradients_impl @@ -48,7 +49,7 @@ class BetaincTest(test.TestCase): tf_x_s = constant_op.constant(x_s, dtype=dtype) tf_out_t = math_ops.betainc(tf_a_s, tf_b_s, tf_x_s) with self.cached_session(): - tf_out = tf_out_t.eval() + tf_out = self.evaluate(tf_out_t) scipy_out = special.betainc(a_s, b_s, x_s).astype(np_dt) # the scipy version of betainc uses a double-only implementation. @@ -109,36 +110,42 @@ class BetaincTest(test.TestCase): except ImportError as e: tf_logging.warn("Cannot test special functions: %s" % str(e)) + @test_util.run_deprecated_v1 def testBetaIncFloat(self): a_s = np.abs(np.random.randn(10, 10) * 30) # in (0, infty) b_s = np.abs(np.random.randn(10, 10) * 30) # in (0, infty) x_s = np.random.rand(10, 10) # in (0, 1) self._testBetaInc(a_s, b_s, x_s, dtypes.float32) + @test_util.run_deprecated_v1 def testBetaIncDouble(self): a_s = np.abs(np.random.randn(10, 10) * 30) # in (0, infty) b_s = np.abs(np.random.randn(10, 10) * 30) # in (0, infty) x_s = np.random.rand(10, 10) # in (0, 1) self._testBetaInc(a_s, b_s, x_s, dtypes.float64) + @test_util.run_deprecated_v1 def testBetaIncDoubleVeryLargeValues(self): a_s = np.abs(np.random.randn(10, 10) * 1e15) # in (0, infty) b_s = np.abs(np.random.randn(10, 10) * 1e15) # in (0, infty) x_s = np.random.rand(10, 10) # in (0, 1) self._testBetaInc(a_s, b_s, x_s, dtypes.float64) + @test_util.run_deprecated_v1 def testBetaIncDoubleVerySmallValues(self): a_s = np.abs(np.random.randn(10, 10) * 1e-16) # in (0, infty) b_s = np.abs(np.random.randn(10, 10) * 1e-16) # in (0, infty) x_s = np.random.rand(10, 10) # in (0, 1) self._testBetaInc(a_s, b_s, x_s, dtypes.float64) + @test_util.run_deprecated_v1 def testBetaIncFloatVerySmallValues(self): a_s = np.abs(np.random.randn(10, 10) * 1e-8) # in (0, infty) b_s = np.abs(np.random.randn(10, 10) * 1e-8) # in (0, infty) x_s = np.random.rand(10, 10) # in (0, 1) self._testBetaInc(a_s, b_s, x_s, dtypes.float32) + @test_util.run_deprecated_v1 def testBetaIncFpropAndBpropAreNeverNAN(self): with self.cached_session() as sess: space = np.logspace(-8, 5).tolist() @@ -159,6 +166,7 @@ class BetaincTest(test.TestCase): self.assertAllEqual(np.zeros_like(grads_x).astype(np.bool), np.isnan(grads_x)) + @test_util.run_deprecated_v1 def testBetaIncGrads(self): err_tolerance = 1e-3 with self.cached_session(): diff --git a/tensorflow/python/kernel_tests/bias_op_test.py b/tensorflow/python/kernel_tests/bias_op_test.py index 749d6a791e3..66f442dbddb 100644 --- a/tensorflow/python/kernel_tests/bias_op_test.py +++ b/tensorflow/python/kernel_tests/bias_op_test.py @@ -22,6 +22,7 @@ import numpy as np from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes +from tensorflow.python.framework import test_util from tensorflow.python.ops import array_ops from tensorflow.python.ops import gradient_checker from tensorflow.python.ops import gradients_impl @@ -89,10 +90,12 @@ class BiasAddTest(test.TestCase): self._testBiasNCHW(np_inputs, np_bias, use_gpu=True) + @test_util.run_deprecated_v1 def testInputDims(self): with self.assertRaises(ValueError): nn_ops.bias_add([1, 2], [1]) + @test_util.run_deprecated_v1 def testBiasVec(self): with self.assertRaises(ValueError): nn_ops.bias_add( @@ -101,6 +104,7 @@ class BiasAddTest(test.TestCase): array_ops.reshape( [1, 2], shape=[1, 2])) + @test_util.run_deprecated_v1 def testBiasInputsMatch(self): with self.assertRaises(ValueError): nn_ops.bias_add( @@ -109,23 +113,27 @@ class BiasAddTest(test.TestCase): array_ops.reshape( [1], shape=[1])) + @test_util.run_deprecated_v1 def testIntTypes(self): for t in [np.int8, np.int16, np.int32, np.int64]: self._testAll( np.array([[10, 20, 30], [40, 50, 60]]).astype(t), np.array([1, 2, 3]).astype(t)) + @test_util.run_deprecated_v1 def testFloatTypes(self): for t in [np.float16, np.float32, np.float64]: self._testAll( np.random.rand(4, 3, 3).astype(t), np.random.rand(3).astype(t)) + @test_util.run_deprecated_v1 def test4DFloatTypes(self): for t in [np.float16, np.float32, np.float64]: self._testAll( np.random.rand(4, 3, 2, 3).astype(t), np.random.rand(3).astype(t)) + @test_util.run_deprecated_v1 def test5DFloatTypes(self): for t in [np.float16, np.float32, np.float64]: self._testAll( @@ -187,6 +195,7 @@ class BiasAddTest(test.TestCase): self.assertAllClose(bias_jacob_t, bias_jacob_n, threshold, threshold) self.assertAllClose(grad_jacob_t, grad_jacob_n, threshold, threshold) + @test_util.run_deprecated_v1 def testGradientTensor(self): # TODO(yongtang): BiasAddGrad with NCHW only works 4D. Reenable once # all dimensions are supported. @@ -198,6 +207,7 @@ class BiasAddTest(test.TestCase): bias = np.array([1.3, 2.4], dtype=dtype.as_numpy_dtype) self._testGradient(np_input, bias, dtype, data_format, use_gpu) + @test_util.run_deprecated_v1 def testGradientTensor4D(self): # BiasAddGrad with NCHW support 4D so all are enabled. for (data_format, use_gpu) in [("NHWC", False), ("NHWC", True), @@ -209,11 +219,13 @@ class BiasAddTest(test.TestCase): bias = np.array([1.3, 2.4], dtype=dtype.as_numpy_dtype) self._testGradient(np_input, bias, dtype, data_format, use_gpu) + @test_util.run_deprecated_v1 def testEmpty(self): np.random.seed(7) for shape in (0, 0), (2, 0), (0, 2), (4, 3, 0), (4, 0, 3), (0, 4, 3): self._testAll(np.random.randn(*shape), np.random.randn(shape[-1])) + @test_util.run_deprecated_v1 def testEmptyGradient(self): # TODO(yongtang): BiasAddGrad with NCHW only works 4D. Reenable once # all dimensions are supported. diff --git a/tensorflow/python/kernel_tests/bincount_op_test.py b/tensorflow/python/kernel_tests/bincount_op_test.py index 49eb835847e..d064d736cf2 100644 --- a/tensorflow/python/kernel_tests/bincount_op_test.py +++ b/tensorflow/python/kernel_tests/bincount_op_test.py @@ -30,6 +30,7 @@ from tensorflow.python.platform import googletest class BincountTest(test_util.TensorFlowTestCase): + @test_util.run_deprecated_v1 def test_empty(self): with self.session(use_gpu=True): self.assertAllEqual( @@ -43,6 +44,7 @@ class BincountTest(test_util.TensorFlowTestCase): math_ops.bincount([], minlength=3, dtype=np.float64).eval().dtype, np.float64) + @test_util.run_deprecated_v1 def test_values(self): with self.session(use_gpu=True): self.assertAllEqual( @@ -58,12 +60,14 @@ class BincountTest(test_util.TensorFlowTestCase): self.assertAllEqual( math_ops.bincount(np.arange(10000)).eval(), np.ones(10000)) + @test_util.run_deprecated_v1 def test_maxlength(self): with self.session(use_gpu=True): self.assertAllEqual(math_ops.bincount([5], maxlength=3).eval(), [0, 0, 0]) self.assertAllEqual(math_ops.bincount([1], maxlength=3).eval(), [0, 1]) self.assertAllEqual(math_ops.bincount([], maxlength=3).eval(), []) + @test_util.run_deprecated_v1 def test_random_with_weights(self): num_samples = 10000 with self.session(use_gpu=True): @@ -77,6 +81,7 @@ class BincountTest(test_util.TensorFlowTestCase): self.assertAllClose( math_ops.bincount(arr, weights).eval(), np.bincount(arr, weights)) + @test_util.run_deprecated_v1 def test_random_without_weights(self): num_samples = 10000 with self.session(use_gpu=True): @@ -87,6 +92,7 @@ class BincountTest(test_util.TensorFlowTestCase): self.assertAllClose( math_ops.bincount(arr, None).eval(), np.bincount(arr, weights)) + @test_util.run_deprecated_v1 def test_zero_weights(self): with self.session(use_gpu=True): self.assertAllEqual( @@ -99,6 +105,7 @@ class BincountTest(test_util.TensorFlowTestCase): with self.assertRaises(errors.InvalidArgumentError): math_ops.bincount([1, 2, 3, -1, 6, 8]).eval() + @test_util.run_deprecated_v1 def test_shape_function(self): # size must be scalar. with self.assertRaisesRegexp( diff --git a/tensorflow/python/kernel_tests/bitcast_op_test.py b/tensorflow/python/kernel_tests/bitcast_op_test.py index 79e0f36d242..b4f9a21a899 100644 --- a/tensorflow/python/kernel_tests/bitcast_op_test.py +++ b/tensorflow/python/kernel_tests/bitcast_op_test.py @@ -21,6 +21,7 @@ from __future__ import print_function import numpy as np from tensorflow.python.framework import dtypes +from tensorflow.python.framework import test_util from tensorflow.python.ops import array_ops from tensorflow.python.platform import test @@ -28,9 +29,9 @@ from tensorflow.python.platform import test class BitcastTest(test.TestCase): def _testBitcast(self, x, datatype, shape): - with self.session(use_gpu=True): + with test_util.use_gpu(): tf_ans = array_ops.bitcast(x, datatype) - out = tf_ans.eval() + out = self.evaluate(tf_ans) buff_after = memoryview(out).tobytes() buff_before = memoryview(x).tobytes() self.assertEqual(buff_before, buff_after) @@ -59,6 +60,7 @@ class BitcastTest(test.TestCase): shape = [3, 4] self._testBitcast(x, dtypes.int64, shape) + @test_util.run_deprecated_v1 def testErrors(self): x = np.zeros([1, 1], np.int8) datatype = dtypes.int32 @@ -71,6 +73,7 @@ class BitcastTest(test.TestCase): shape = [4] self._testBitcast(x, datatype, shape) + @test_util.run_deprecated_v1 def testUnknown(self): x = array_ops.placeholder(dtypes.float32) datatype = dtypes.int8 diff --git a/tensorflow/python/kernel_tests/boosted_trees/prediction_ops_test.py b/tensorflow/python/kernel_tests/boosted_trees/prediction_ops_test.py index 7cdc67f83f0..6b04e8abf40 100644 --- a/tensorflow/python/kernel_tests/boosted_trees/prediction_ops_test.py +++ b/tensorflow/python/kernel_tests/boosted_trees/prediction_ops_test.py @@ -28,6 +28,7 @@ from tensorflow.python.platform import googletest class TrainingPredictionOpsTest(test_util.TensorFlowTestCase): """Tests prediction ops for training.""" + @test_util.run_deprecated_v1 def testCachedPredictionOnEmptyEnsemble(self): """Tests that prediction on a dummy ensemble does not fail.""" with self.cached_session() as session: @@ -61,6 +62,7 @@ class TrainingPredictionOpsTest(test_util.TensorFlowTestCase): self.assertAllClose(cached_node_ids, new_node_ids) self.assertAllClose([[0], [0]], logits_updates) + @test_util.run_deprecated_v1 def testNoCachedPredictionButTreeExists(self): """Tests that predictions are updated once trees are added.""" with self.cached_session() as session: @@ -127,6 +129,7 @@ class TrainingPredictionOpsTest(test_util.TensorFlowTestCase): self.assertAllClose([2, 1], new_node_ids) self.assertAllClose([[0.1 * 8.79], [0.1 * 1.14]], logits_updates) + @test_util.run_deprecated_v1 def testCachedPredictionIsCurrent(self): """Tests that prediction based on previous node in the tree works.""" with self.cached_session() as session: @@ -199,6 +202,7 @@ class TrainingPredictionOpsTest(test_util.TensorFlowTestCase): self.assertAllClose(cached_node_ids, new_node_ids) self.assertAllClose([[0], [0]], logits_updates) + @test_util.run_deprecated_v1 def testCachedPredictionFromTheSameTree(self): """Tests that prediction based on previous node in the tree works.""" with self.cached_session() as session: @@ -313,6 +317,7 @@ class TrainingPredictionOpsTest(test_util.TensorFlowTestCase): # 1.65 and -3.875, and then multiply them by 0.1 (lr) self.assertAllClose([[0.1 * 1.65], [0.1 * -3.875]], logits_updates) + @test_util.run_deprecated_v1 def testCachedPredictionFromPreviousTree(self): """Tests the predictions work when we have cache from previous trees.""" with self.cached_session() as session: @@ -445,6 +450,7 @@ class TrainingPredictionOpsTest(test_util.TensorFlowTestCase): # change= 0.1(1.14+7.0-7.0) self.assertAllClose([[1], [0.114]], logits_updates) + @test_util.run_deprecated_v1 def testCategoricalSplits(self): """Tests the training prediction work for categorical splits.""" with self.cached_session() as session: @@ -517,6 +523,7 @@ class TrainingPredictionOpsTest(test_util.TensorFlowTestCase): self.assertAllClose([3, 4, 2], new_node_ids) self.assertAllClose([[5.], [6.], [7.]], logits_updates) + @test_util.run_deprecated_v1 def testCachedPredictionFromTheSameTreeWithPostPrunedNodes(self): """Tests that prediction based on previous node in the tree works.""" with self.cached_session() as session: @@ -647,6 +654,7 @@ class TrainingPredictionOpsTest(test_util.TensorFlowTestCase): self.assertAllClose([[0.01], [0.01], [0.0553], [0.0783], [0.01], [0.01]], logits_updates + cached_values) + @test_util.run_deprecated_v1 def testCachedPredictionFromThePreviousTreeWithPostPrunedNodes(self): """Tests that prediction based on previous node in the tree works.""" with self.cached_session() as session: @@ -792,6 +800,7 @@ class TrainingPredictionOpsTest(test_util.TensorFlowTestCase): [root + 0.0783], [root + 0.01], [root + 0.01]], logits_updates + cached_values) + @test_util.run_deprecated_v1 def testCachedPredictionTheWholeTreeWasPruned(self): """Tests that prediction based on previous node in the tree works.""" with self.cached_session() as session: @@ -864,6 +873,7 @@ class TrainingPredictionOpsTest(test_util.TensorFlowTestCase): class PredictionOpsTest(test_util.TensorFlowTestCase): """Tests prediction ops for inference.""" + @test_util.run_deprecated_v1 def testPredictionOnEmptyEnsemble(self): """Tests that prediction on a empty ensemble does not fail.""" with self.cached_session() as session: @@ -886,6 +896,7 @@ class PredictionOpsTest(test_util.TensorFlowTestCase): logits = session.run(predict_op) self.assertAllClose(expected_logits, logits) + @test_util.run_deprecated_v1 def testPredictionMultipleTree(self): """Tests the predictions work when we have multiple trees.""" with self.cached_session() as session: @@ -996,6 +1007,7 @@ class PredictionOpsTest(test_util.TensorFlowTestCase): logits = session.run(predict_op) self.assertAllClose(expected_logits, logits) + @test_util.run_deprecated_v1 def testCategoricalSplits(self): """Tests the predictions work for categorical splits.""" with self.cached_session() as session: @@ -1062,6 +1074,7 @@ class PredictionOpsTest(test_util.TensorFlowTestCase): class FeatureContribsOpsTest(test_util.TensorFlowTestCase): """Tests feature contribs ops for model understanding.""" + @test_util.run_deprecated_v1 def testContribsForOnlyABiasNode(self): """Tests case when, after training, only left with a bias node. @@ -1122,6 +1135,7 @@ class FeatureContribsOpsTest(test_util.TensorFlowTestCase): self.assertAllClose(feature_ids, expected_feature_ids) self.assertAllClose(logits_paths, expected_logits_paths) + @test_util.run_deprecated_v1 def testContribsMultipleTreeWhenFirstTreeIsABiasNode(self): """Tests case when, after training, first tree contains only a bias node.""" with self.cached_session() as session: @@ -1219,6 +1233,7 @@ class FeatureContribsOpsTest(test_util.TensorFlowTestCase): self.assertAllClose(feature_ids, expected_feature_ids) self.assertAllClose(logits_paths, expected_logits_paths) + @test_util.run_deprecated_v1 def testContribsMultipleTree(self): """Tests that the contribs work when we have multiple trees.""" with self.cached_session() as session: diff --git a/tensorflow/python/kernel_tests/boosted_trees/quantile_ops_test.py b/tensorflow/python/kernel_tests/boosted_trees/quantile_ops_test.py index 12afb6a2ad8..390672febeb 100644 --- a/tensorflow/python/kernel_tests/boosted_trees/quantile_ops_test.py +++ b/tensorflow/python/kernel_tests/boosted_trees/quantile_ops_test.py @@ -82,6 +82,7 @@ class QuantileOpsTest(test_util.TensorFlowTestCase): self.max_elements = 1 << 16 self.num_quantiles = constant_op.constant(3, dtype=dtypes.int64) + @test_util.run_deprecated_v1 def testBasicQuantileBucketsSingleResource(self): with self.cached_session() as sess: quantile_accumulator_handle = self.create_resource("floats", self.eps, @@ -98,14 +99,15 @@ class QuantileOpsTest(test_util.TensorFlowTestCase): quantile_accumulator_handle, num_features=2) quantiles = boosted_trees_ops.boosted_trees_bucketize( [self._feature_0, self._feature_1], buckets) - sess.run(summary_op) - sess.run(flush_op) + self.evaluate(summary_op) + self.evaluate(flush_op) self.assertAllClose(self._feature_0_boundaries, buckets[0].eval()) self.assertAllClose(self._feature_1_boundaries, buckets[1].eval()) self.assertAllClose(self._feature_0_quantiles, quantiles[0].eval()) self.assertAllClose(self._feature_1_quantiles, quantiles[1].eval()) + @test_util.run_deprecated_v1 def testBasicQuantileBucketsMultipleResources(self): with self.cached_session() as sess: quantile_accumulator_handle_0 = self.create_resource("float_0", self.eps, @@ -132,14 +134,15 @@ class QuantileOpsTest(test_util.TensorFlowTestCase): quantile_accumulator_handle_1, num_features=1) quantiles = boosted_trees_ops.boosted_trees_bucketize( [self._feature_0, self._feature_1], bucket_0 + bucket_1) - sess.run([summary_op_0, summary_op_1]) - sess.run([flush_op_0, flush_op_1]) + self.evaluate([summary_op_0, summary_op_1]) + self.evaluate([flush_op_0, flush_op_1]) self.assertAllClose(self._feature_0_boundaries, bucket_0[0].eval()) self.assertAllClose(self._feature_1_boundaries, bucket_1[0].eval()) self.assertAllClose(self._feature_0_quantiles, quantiles[0].eval()) self.assertAllClose(self._feature_1_quantiles, quantiles[1].eval()) + @test_util.run_deprecated_v1 def testSaveRestoreAfterFlush(self): save_dir = os.path.join(self.get_temp_dir(), "save_restore") save_path = os.path.join(tempfile.mkdtemp(prefix=save_dir), "hash") @@ -158,7 +161,7 @@ class QuantileOpsTest(test_util.TensorFlowTestCase): self._example_weights) with ops.control_dependencies([summaries]): flush = accumulator.flush() - sess.run(flush) + self.evaluate(flush) self.assertAllClose(self._feature_0_boundaries, buckets[0].eval()) self.assertAllClose(self._feature_1_boundaries, buckets[1].eval()) save.save(sess, save_path) @@ -172,6 +175,7 @@ class QuantileOpsTest(test_util.TensorFlowTestCase): self.assertAllClose(self._feature_0_boundaries, buckets[0].eval()) self.assertAllClose(self._feature_1_boundaries, buckets[1].eval()) + @test_util.run_deprecated_v1 def testSaveRestoreBeforeFlush(self): save_dir = os.path.join(self.get_temp_dir(), "save_restore") save_path = os.path.join(tempfile.mkdtemp(prefix=save_dir), "hash") @@ -185,12 +189,12 @@ class QuantileOpsTest(test_util.TensorFlowTestCase): summaries = accumulator.add_summaries([self._feature_0, self._feature_1], self._example_weights) - sess.run(summaries) + self.evaluate(summaries) buckets = accumulator.get_bucket_boundaries() self.assertAllClose([], buckets[0].eval()) self.assertAllClose([], buckets[1].eval()) save.save(sess, save_path) - sess.run(accumulator.flush()) + self.evaluate(accumulator.flush()) self.assertAllClose(self._feature_0_boundaries, buckets[0].eval()) self.assertAllClose(self._feature_1_boundaries, buckets[1].eval()) diff --git a/tensorflow/python/kernel_tests/boosted_trees/resource_ops_test.py b/tensorflow/python/kernel_tests/boosted_trees/resource_ops_test.py index 65bb9ab55f0..0a34277bbdb 100644 --- a/tensorflow/python/kernel_tests/boosted_trees/resource_ops_test.py +++ b/tensorflow/python/kernel_tests/boosted_trees/resource_ops_test.py @@ -30,19 +30,21 @@ from tensorflow.python.platform import googletest class ResourceOpsTest(test_util.TensorFlowTestCase): """Tests resource_ops.""" + @test_util.run_deprecated_v1 def testCreate(self): with self.cached_session(): ensemble = boosted_trees_ops.TreeEnsemble('ensemble') resources.initialize_resources(resources.shared_resources()).run() stamp_token = ensemble.get_stamp_token() - self.assertEqual(0, stamp_token.eval()) + self.assertEqual(0, self.evaluate(stamp_token)) (_, num_trees, num_finalized_trees, num_attempted_layers, nodes_range) = ensemble.get_states() - self.assertEqual(0, num_trees.eval()) - self.assertEqual(0, num_finalized_trees.eval()) - self.assertEqual(0, num_attempted_layers.eval()) - self.assertAllEqual([0, 1], nodes_range.eval()) + self.assertEqual(0, self.evaluate(num_trees)) + self.assertEqual(0, self.evaluate(num_finalized_trees)) + self.assertEqual(0, self.evaluate(num_attempted_layers)) + self.assertAllEqual([0, 1], self.evaluate(nodes_range)) + @test_util.run_deprecated_v1 def testCreateWithProto(self): with self.cached_session(): ensemble_proto = boosted_trees_pb2.TreeEnsemble() @@ -154,12 +156,13 @@ class ResourceOpsTest(test_util.TensorFlowTestCase): resources.initialize_resources(resources.shared_resources()).run() (stamp_token, num_trees, num_finalized_trees, num_attempted_layers, nodes_range) = ensemble.get_states() - self.assertEqual(7, stamp_token.eval()) - self.assertEqual(2, num_trees.eval()) - self.assertEqual(1, num_finalized_trees.eval()) - self.assertEqual(6, num_attempted_layers.eval()) - self.assertAllEqual([16, 19], nodes_range.eval()) + self.assertEqual(7, self.evaluate(stamp_token)) + self.assertEqual(2, self.evaluate(num_trees)) + self.assertEqual(1, self.evaluate(num_finalized_trees)) + self.assertEqual(6, self.evaluate(num_attempted_layers)) + self.assertAllEqual([16, 19], self.evaluate(nodes_range)) + @test_util.run_deprecated_v1 def testSerializeDeserialize(self): with self.cached_session(): # Initialize. @@ -167,11 +170,11 @@ class ResourceOpsTest(test_util.TensorFlowTestCase): resources.initialize_resources(resources.shared_resources()).run() (stamp_token, num_trees, num_finalized_trees, num_attempted_layers, nodes_range) = ensemble.get_states() - self.assertEqual(5, stamp_token.eval()) - self.assertEqual(0, num_trees.eval()) - self.assertEqual(0, num_finalized_trees.eval()) - self.assertEqual(0, num_attempted_layers.eval()) - self.assertAllEqual([0, 1], nodes_range.eval()) + self.assertEqual(5, self.evaluate(stamp_token)) + self.assertEqual(0, self.evaluate(num_trees)) + self.assertEqual(0, self.evaluate(num_finalized_trees)) + self.assertEqual(0, self.evaluate(num_attempted_layers)) + self.assertAllEqual([0, 1], self.evaluate(nodes_range)) # Deserialize. ensemble_proto = boosted_trees_pb2.TreeEnsemble() @@ -219,18 +222,18 @@ class ResourceOpsTest(test_util.TensorFlowTestCase): ]): (stamp_token, num_trees, num_finalized_trees, num_attempted_layers, nodes_range) = ensemble.get_states() - self.assertEqual(3, stamp_token.eval()) - self.assertEqual(1, num_trees.eval()) + self.assertEqual(3, self.evaluate(stamp_token)) + self.assertEqual(1, self.evaluate(num_trees)) # This reads from metadata, not really counting the layers. - self.assertEqual(5, num_attempted_layers.eval()) - self.assertEqual(0, num_finalized_trees.eval()) - self.assertAllEqual([3, 7], nodes_range.eval()) + self.assertEqual(5, self.evaluate(num_attempted_layers)) + self.assertEqual(0, self.evaluate(num_finalized_trees)) + self.assertAllEqual([3, 7], self.evaluate(nodes_range)) # Serialize. new_ensemble_proto = boosted_trees_pb2.TreeEnsemble() new_stamp_token, new_serialized = ensemble.serialize() - self.assertEqual(3, new_stamp_token.eval()) + self.assertEqual(3, self.evaluate(new_stamp_token)) new_ensemble_proto.ParseFromString(new_serialized.eval()) self.assertProtoEquals(ensemble_proto, new_ensemble_proto) diff --git a/tensorflow/python/kernel_tests/boosted_trees/stats_ops_test.py b/tensorflow/python/kernel_tests/boosted_trees/stats_ops_test.py index 09e9cfa3aff..e2e23486b5a 100644 --- a/tensorflow/python/kernel_tests/boosted_trees/stats_ops_test.py +++ b/tensorflow/python/kernel_tests/boosted_trees/stats_ops_test.py @@ -65,16 +65,16 @@ class StatsOpsTest(test_util.TensorFlowTestCase): min_node_weight=0, max_splits=max_splits) - self.assertAllEqual([[1, 2], [1, 2]], sess.run(node_ids_list)) + self.assertAllEqual([[1, 2], [1, 2]], self.evaluate(node_ids_list)) self.assertAllClose([[0.004775, 0.41184], [0.02823, 0.41184]], - sess.run(gains_list)) - self.assertAllEqual([[1, 1], [1, 1]], sess.run(thresholds_list)) + self.evaluate(gains_list)) + self.assertAllEqual([[1, 1], [1, 1]], self.evaluate(thresholds_list)) # The left node contrib will be later added to the previous node value to # make the left node value, and the same for right node contrib. self.assertAllClose([[[-.416667], [.568966]], [[-.6], [-.75]]], - sess.run(left_node_contribs_list)) + self.evaluate(left_node_contribs_list)) self.assertAllClose([[[-.592593], [-.75]], [[-.076923], [.568966]]], - sess.run(right_node_contribs_list)) + self.evaluate(right_node_contribs_list)) def testCalculateBestGainsWithL2(self): """Testing Gain calculation with L2.""" @@ -113,16 +113,16 @@ class StatsOpsTest(test_util.TensorFlowTestCase): min_node_weight=0, max_splits=max_splits) - self.assertAllEqual([[1, 2], [1, 2]], sess.run(node_ids_list)) + self.assertAllEqual([[1, 2], [1, 2]], self.evaluate(node_ids_list)) self.assertAllClose([[0., 0.33931375], [0.01879096, 0.33931375]], - sess.run(gains_list)) - self.assertAllEqual([[0, 1], [1, 1]], sess.run(thresholds_list)) + self.evaluate(gains_list)) + self.assertAllEqual([[0, 1], [1, 1]], self.evaluate(thresholds_list)) # The left node contrib will be later added to the previous node value to # make the left node value, and the same for right node contrib. self.assertAllClose([[[0.], [.485294]], [[-.5], [-.6]]], - sess.run(left_node_contribs_list)) + self.evaluate(left_node_contribs_list)) self.assertAllClose([[[-.424658], [-.6]], [[-.043478], [.485294]]], - sess.run(right_node_contribs_list)) + self.evaluate(right_node_contribs_list)) def testCalculateBestGainsWithL1(self): """Testing Gain calculation with L1.""" @@ -162,18 +162,18 @@ class StatsOpsTest(test_util.TensorFlowTestCase): min_node_weight=0, max_splits=max_splits) - self.assertAllEqual([[0, 1], [1, 1]], sess.run(thresholds_list)) + self.assertAllEqual([[0, 1], [1, 1]], self.evaluate(thresholds_list)) - self.assertAllEqual([[1, 2], [1, 2]], sess.run(node_ids_list)) + self.assertAllEqual([[1, 2], [1, 2]], self.evaluate(node_ids_list)) self.assertAllClose([[[0.0], [0.3965517]], [[-0.4], [-0.5]]], - sess.run(left_node_contribs_list)) + self.evaluate(left_node_contribs_list)) self.assertAllClose([[[-0.3333333], [-0.5]], [[0.0], [0.396552]]], - sess.run(right_node_contribs_list)) + self.evaluate(right_node_contribs_list)) # Gain should also include an adjustment of the gradient by l1. self.assertAllClose([[0.0, 0.191207], [0.01, 0.191207]], - sess.run(gains_list)) + self.evaluate(gains_list)) def testCalculateBestGainsWithTreeComplexity(self): """Testing Gain calculation with L2.""" @@ -214,18 +214,18 @@ class StatsOpsTest(test_util.TensorFlowTestCase): min_node_weight=0, max_splits=max_splits) - self.assertAllEqual([[1, 2], [1, 2]], sess.run(node_ids_list)) + self.assertAllEqual([[1, 2], [1, 2]], self.evaluate(node_ids_list)) self.assertAllClose([[-3., -2.66068625], [-2.98120904, -2.66068625]], - sess.run(gains_list)) + self.evaluate(gains_list)) - self.assertAllEqual([[0, 1], [1, 1]], sess.run(thresholds_list)) + self.assertAllEqual([[0, 1], [1, 1]], self.evaluate(thresholds_list)) # The left node contrib will be later added to the previous node value to # make the left node value, and the same for right node contrib. self.assertAllClose([[[0.], [.485294]], [[-.5], [-.6]]], - sess.run(left_node_contribs_list)) + self.evaluate(left_node_contribs_list)) self.assertAllClose([[[-.424658], [-.6]], [[-.043478], [.485294]]], - sess.run(right_node_contribs_list)) + self.evaluate(right_node_contribs_list)) def testCalculateBestGainsWithMinNodeWeight(self): """Testing Gain calculation without any regularization.""" @@ -266,13 +266,13 @@ class StatsOpsTest(test_util.TensorFlowTestCase): # We can't split node 1 on feature 1 and node 2 on feature 2 because of # the min node weight. - self.assertAllEqual([[2], [1]], sess.run(node_ids_list)) - self.assertAllClose([[0.384314], [0.098013]], sess.run(gains_list)) - self.assertAllEqual([[1], [1]], sess.run(thresholds_list)) + self.assertAllEqual([[2], [1]], self.evaluate(node_ids_list)) + self.assertAllClose([[0.384314], [0.098013]], self.evaluate(gains_list)) + self.assertAllEqual([[1], [1]], self.evaluate(thresholds_list)) self.assertAllClose([[[0.4852941]], [[-.6]]], - sess.run(left_node_contribs_list)) + self.evaluate(left_node_contribs_list)) self.assertAllClose([[[-0.75]], [[-0.014925]]], - sess.run(right_node_contribs_list)) + self.evaluate(right_node_contribs_list)) def testCalculateBestGainsWithMinNodeWeightNoSplitOnFeturePossible(self): """Testing Gain calculation without any regularization.""" @@ -311,9 +311,9 @@ class StatsOpsTest(test_util.TensorFlowTestCase): max_splits=max_splits) # We can't split either of the nodes on the first feature - self.assertEqual(2, len(sess.run(node_ids_list))) - self.assertAllEqual([], sess.run(node_ids_list)[0]) - self.assertAllEqual([1], sess.run(node_ids_list)[1]) + self.assertEqual(2, len(self.evaluate(node_ids_list))) + self.assertAllEqual([], self.evaluate(node_ids_list)[0]) + self.assertAllEqual([1], self.evaluate(node_ids_list)[1]) # Now check when we can't split on any feature (node_ids_list, _, _, _, @@ -325,8 +325,9 @@ class StatsOpsTest(test_util.TensorFlowTestCase): tree_complexity=0.0, min_node_weight=10, max_splits=max_splits) - self.assertAllEqual([[], []], sess.run(node_ids_list)) + self.assertAllEqual([[], []], self.evaluate(node_ids_list)) + @test_util.run_deprecated_v1 def testMakeStatsSummarySimple(self): """Simple test for MakeStatsSummary.""" with self.cached_session(): @@ -359,7 +360,7 @@ class StatsOpsTest(test_util.TensorFlowTestCase): [[0., 0.], [.15, .36], [.06, .07], [.1, .2]], # node 1 [[-.33, .58], [0., 0.], [.3, .4], [0., 0.]], # node 2 ]], - result.eval()) + self.evaluate(result)) def testMakeStatsSummaryMultipleFeatures(self): """Tests that MakeStatsSummary works for multiple features.""" @@ -389,7 +390,7 @@ class StatsOpsTest(test_util.TensorFlowTestCase): [[.3, .4], [0., 0.], [-.4, .5], [.07, .08]], # node 2 ], # feature 1 ], - result.eval()) + self.evaluate(result)) def _verify_precision(self, length): with self.cached_session(): @@ -408,7 +409,7 @@ class StatsOpsTest(test_util.TensorFlowTestCase): node_ids, gradients, hessians, [bucketized_features], max_splits, num_buckets) # shape=[max_splits, num_buckets, num_features, 2] - self.assertAllClose([[[[2., 0.2]]]], result.eval()) + self.assertAllClose([[[[2., 0.2]]]], self.evaluate(result)) def testMakeStatsSummaryNumericalPrecisionSmallBatch(self): """Tests numeric precision.""" diff --git a/tensorflow/python/kernel_tests/boosted_trees/training_ops_test.py b/tensorflow/python/kernel_tests/boosted_trees/training_ops_test.py index ea022820e44..afc0564fc5a 100644 --- a/tensorflow/python/kernel_tests/boosted_trees/training_ops_test.py +++ b/tensorflow/python/kernel_tests/boosted_trees/training_ops_test.py @@ -30,6 +30,7 @@ from tensorflow.python.platform import googletest class UpdateTreeEnsembleOpTest(test_util.TensorFlowTestCase): """Tests for growing tree ensemble from split candidates.""" + @test_util.run_deprecated_v1 def testGrowWithEmptyEnsemble(self): """Test growing an empty ensemble.""" with self.cached_session() as session: @@ -139,6 +140,7 @@ class UpdateTreeEnsembleOpTest(test_util.TensorFlowTestCase): self.assertEqual(new_stamp, 1) self.assertProtoEquals(expected_result, tree_ensemble) + @test_util.run_deprecated_v1 def testBiasCenteringOnEmptyEnsemble(self): """Test growing with bias centering on an empty ensemble.""" with self.cached_session() as session: @@ -182,6 +184,7 @@ class UpdateTreeEnsembleOpTest(test_util.TensorFlowTestCase): self.assertEqual(new_stamp, 1) self.assertProtoEquals(expected_result, tree_ensemble) + @test_util.run_deprecated_v1 def testGrowExistingEnsembleTreeNotFinalized(self): """Test growing an existing ensemble with the last tree not finalized.""" with self.cached_session() as session: @@ -366,6 +369,7 @@ class UpdateTreeEnsembleOpTest(test_util.TensorFlowTestCase): self.assertEqual(new_stamp, 1) self.assertProtoEquals(expected_result, tree_ensemble) + @test_util.run_deprecated_v1 def testGrowExistingEnsembleTreeFinalized(self): """Test growing an existing ensemble with the last tree finalized.""" with self.cached_session() as session: @@ -515,6 +519,7 @@ class UpdateTreeEnsembleOpTest(test_util.TensorFlowTestCase): self.assertEqual(new_stamp, 1) self.assertProtoEquals(expected_result, tree_ensemble) + @test_util.run_deprecated_v1 def testPrePruning(self): """Test growing an existing ensemble with pre-pruning.""" with self.cached_session() as session: @@ -671,6 +676,7 @@ class UpdateTreeEnsembleOpTest(test_util.TensorFlowTestCase): self.assertEqual(new_stamp, 1) self.assertProtoEquals(expected_result, tree_ensemble) + @test_util.run_deprecated_v1 def testMetadataWhenCantSplitDueToEmptySplits(self): """Test that the metadata is updated even though we can't split.""" with self.cached_session() as session: @@ -782,6 +788,7 @@ class UpdateTreeEnsembleOpTest(test_util.TensorFlowTestCase): self.assertEqual(new_stamp, 1) self.assertProtoEquals(expected_result, tree_ensemble) + @test_util.run_deprecated_v1 def testMetadataWhenCantSplitDuePrePruning(self): """Test metadata is updated correctly when no split due to prepruning.""" with self.cached_session() as session: @@ -917,6 +924,7 @@ class UpdateTreeEnsembleOpTest(test_util.TensorFlowTestCase): self.assertEqual(new_stamp, 1) self.assertProtoEquals(expected_result, tree_ensemble) + @test_util.run_deprecated_v1 def testPostPruningOfSomeNodes(self): """Test growing an ensemble with post-pruning.""" with self.cached_session() as session: @@ -1251,6 +1259,7 @@ class UpdateTreeEnsembleOpTest(test_util.TensorFlowTestCase): self.assertEqual(new_stamp, 3) self.assertProtoEquals(expected_result, res_ensemble) + @test_util.run_deprecated_v1 def testPostPruningOfAllNodes(self): """Test growing an ensemble with post-pruning, with all nodes are pruned.""" with self.cached_session() as session: @@ -1434,6 +1443,7 @@ class UpdateTreeEnsembleOpTest(test_util.TensorFlowTestCase): } """, res_ensemble) + @test_util.run_deprecated_v1 def testPostPruningChangesNothing(self): """Test growing an ensemble with post-pruning with all gains >0.""" with self.cached_session() as session: diff --git a/tensorflow/python/kernel_tests/broadcast_to_ops_test.py b/tensorflow/python/kernel_tests/broadcast_to_ops_test.py index 233c1664052..b9eb2391b49 100644 --- a/tensorflow/python/kernel_tests/broadcast_to_ops_test.py +++ b/tensorflow/python/kernel_tests/broadcast_to_ops_test.py @@ -29,6 +29,7 @@ from tensorflow.python.platform import test as test_lib class BroadcastToTest(test_util.TensorFlowTestCase): + @test_util.run_deprecated_v1 def testBroadcastToBasic(self): for dtype in [np.uint8, np.uint16, np.int8, np.int16, np.int32, np.int64]: with self.session(use_gpu=True): @@ -37,6 +38,7 @@ class BroadcastToTest(test_util.TensorFlowTestCase): v_np = np.broadcast_to(x, [3, 3]) self.assertAllEqual(v_tf.eval(), v_np) + @test_util.run_deprecated_v1 def testBroadcastToString(self): with self.session(use_gpu=True): x = np.array([b"1", b"2", b"3"]) @@ -44,6 +46,7 @@ class BroadcastToTest(test_util.TensorFlowTestCase): v_np = np.broadcast_to(x, [3, 3]) self.assertAllEqual(v_tf.eval(), v_np) + @test_util.run_deprecated_v1 def testBroadcastToBool(self): with self.session(use_gpu=True): x = np.array([True, False, True], dtype=np.bool) @@ -51,6 +54,7 @@ class BroadcastToTest(test_util.TensorFlowTestCase): v_np = np.broadcast_to(x, [3, 3]) self.assertAllEqual(v_tf.eval(), v_np) + @test_util.run_deprecated_v1 def testBroadcastToShape(self): for input_dim in range(1, 6): for output_dim in range(input_dim, 6): @@ -62,6 +66,7 @@ class BroadcastToTest(test_util.TensorFlowTestCase): v_np = np.broadcast_to(x, output_shape) self.assertAllEqual(v_tf.eval(), v_np) + @test_util.run_deprecated_v1 def testBroadcastToScalar(self): with self.session(use_gpu=True): x = np.array(1, dtype=np.int32) @@ -69,6 +74,7 @@ class BroadcastToTest(test_util.TensorFlowTestCase): v_np = np.broadcast_to(x, [3, 3]) self.assertAllEqual(v_tf.eval(), v_np) + @test_util.run_deprecated_v1 def testBroadcastScalarToNonScalar(self): with self.session(use_gpu=True): x = np.array(1.0, dtype=np.float) @@ -76,6 +82,7 @@ class BroadcastToTest(test_util.TensorFlowTestCase): v_np = np.broadcast_to(x, [2, 3, 4]) self.assertAllEqual(v_tf.eval(), v_np) + @test_util.run_deprecated_v1 def testBroadcastToShapeTypeAndInference(self): for dtype in [dtypes.int32, dtypes.int64]: with self.cached_session(use_gpu=True): @@ -89,6 +96,7 @@ class BroadcastToTest(test_util.TensorFlowTestCase): # check shape inference when shape input is constant self.assertAllEqual(shape, v_np.shape) + @test_util.run_deprecated_v1 def testGradientForScalar(self): x = constant_op.constant(1, dtype=dtypes.float32) v = array_ops.broadcast_to(x, [2, 4, 3]) @@ -98,6 +106,7 @@ class BroadcastToTest(test_util.TensorFlowTestCase): out.get_shape()) self.assertLess(err, 1e-4) + @test_util.run_deprecated_v1 def testGradientWithSameRank(self): x = constant_op.constant(np.reshape(np.arange(6), (2, 1, 3)), dtype=dtypes.float32) @@ -108,6 +117,7 @@ class BroadcastToTest(test_util.TensorFlowTestCase): out, out.get_shape()) self.assertLess(err, 1e-4) + @test_util.run_deprecated_v1 def testGradientWithIncreasingRank(self): x = constant_op.constant([[1], [2]], dtype=dtypes.float32) @@ -118,6 +128,7 @@ class BroadcastToTest(test_util.TensorFlowTestCase): out, out.get_shape()) self.assertLess(err, 1e-4) + @test_util.run_deprecated_v1 def testGradientWithBroadcastAllDimensions(self): x = constant_op.constant([[1, 2, 3], [4, 5, 6]], dtype=dtypes.float32) v = array_ops.broadcast_to(x, [5, 4, 6]) diff --git a/tensorflow/python/kernel_tests/bucketize_op_test.py b/tensorflow/python/kernel_tests/bucketize_op_test.py index 57413e6af50..95df6943705 100644 --- a/tensorflow/python/kernel_tests/bucketize_op_test.py +++ b/tensorflow/python/kernel_tests/bucketize_op_test.py @@ -20,6 +20,7 @@ from __future__ import print_function from tensorflow.python.framework import constant_op from tensorflow.python.framework import errors_impl +from tensorflow.python.framework import test_util from tensorflow.python.ops import math_ops from tensorflow.python.platform import test @@ -32,7 +33,7 @@ class BucketizationOpTest(test.TestCase): boundaries=[0, 3, 8, 11]) expected_out = [0, 1, 1, 2, 2, 3, 3, 4, 4] with self.session(use_gpu=True) as sess: - self.assertAllEqual(expected_out, sess.run(op)) + self.assertAllEqual(expected_out, self.evaluate(op)) def testFloat(self): op = math_ops._bucketize( @@ -40,7 +41,7 @@ class BucketizationOpTest(test.TestCase): boundaries=[0., 3., 8., 11.]) expected_out = [0, 1, 1, 2, 2, 3, 3, 4, 4] with self.session(use_gpu=True) as sess: - self.assertAllEqual(expected_out, sess.run(op)) + self.assertAllEqual(expected_out, self.evaluate(op)) def test2DInput(self): op = math_ops._bucketize( @@ -48,15 +49,16 @@ class BucketizationOpTest(test.TestCase): boundaries=[0, 3, 8, 11]) expected_out = [[0, 1, 1, 2, 2], [3, 3, 4, 4, 1]] with self.session(use_gpu=True) as sess: - self.assertAllEqual(expected_out, sess.run(op)) + self.assertAllEqual(expected_out, self.evaluate(op)) + @test_util.run_deprecated_v1 def testInvalidBoundariesOrder(self): op = math_ops._bucketize( constant_op.constant([-5, 0]), boundaries=[0, 8, 3, 11]) with self.session(use_gpu=True) as sess: with self.assertRaisesRegexp( errors_impl.InvalidArgumentError, "Expected sorted boundaries"): - sess.run(op) + self.evaluate(op) def testBoundariesNotList(self): with self.assertRaisesRegexp( diff --git a/tensorflow/python/kernel_tests/candidate_sampler_ops_test.py b/tensorflow/python/kernel_tests/candidate_sampler_ops_test.py index b19077db560..fa6eb5c9689 100644 --- a/tensorflow/python/kernel_tests/candidate_sampler_ops_test.py +++ b/tensorflow/python/kernel_tests/candidate_sampler_ops_test.py @@ -22,6 +22,7 @@ import numpy as np from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes +from tensorflow.python.framework import test_util from tensorflow.python.ops import array_ops from tensorflow.python.ops import candidate_sampling_ops from tensorflow.python.ops import math_ops @@ -37,6 +38,7 @@ class RangeSamplerOpsTest(test.TestCase): TRUE_LABELS = [[1, 2], [0, 4], [3, 3]] + @test_util.run_deprecated_v1 def testTrueCandidates(self): with self.cached_session() as sess: indices = constant_op.constant([0, 0, 1, 1, 2, 2]) @@ -55,7 +57,7 @@ class RangeSamplerOpsTest(test.TestCase): [[1, 2], [0, 4], [3, 3]], dtype=dtypes.int64) sampled_candidates, _, _ = candidate_sampling_ops.all_candidate_sampler( true_classes, self.NUM_TRUE, self.NUM_SAMPLED, True) - result = sampled_candidates.eval() + result = self.evaluate(sampled_candidates) expected_ids = [0, 1, 2, 3, 4] self.assertAllEqual(result, expected_ids) @@ -68,7 +70,7 @@ class RangeSamplerOpsTest(test.TestCase): _, true_expected_count, _ = candidate_sampling_ops.all_candidate_sampler( true_classes, self.NUM_TRUE, self.NUM_SAMPLED, True) true_log_expected_count = math_ops.log(true_expected_count) - result = true_log_expected_count.eval() + result = self.evaluate(true_log_expected_count) self.assertAllEqual(result, [[0.0] * self.NUM_TRUE] * self.BATCH_SIZE) self.assertEqual(true_expected_count.get_shape(), @@ -83,7 +85,7 @@ class RangeSamplerOpsTest(test.TestCase): _, _, sampled_expected_count = candidate_sampling_ops.all_candidate_sampler( # pylint: disable=line-too-long true_classes, self.NUM_TRUE, self.NUM_SAMPLED, True) sampled_log_expected_count = math_ops.log(sampled_expected_count) - result = sampled_log_expected_count.eval() + result = self.evaluate(sampled_log_expected_count) self.assertAllEqual(result, [0.0] * self.NUM_SAMPLED) self.assertEqual(sampled_expected_count.get_shape(), [self.NUM_SAMPLED]) @@ -97,7 +99,7 @@ class RangeSamplerOpsTest(test.TestCase): true_classes, self.NUM_TRUE, self.NUM_SAMPLED, True) accidental_hits = candidate_sampling_ops.compute_accidental_hits( true_classes, sampled_candidates, self.NUM_TRUE) - indices, ids, weights = sess.run(accidental_hits) + indices, ids, weights = self.evaluate(accidental_hits) self.assertEqual(1, accidental_hits[0].get_shape().ndims) self.assertEqual(1, accidental_hits[1].get_shape().ndims) @@ -106,6 +108,7 @@ class RangeSamplerOpsTest(test.TestCase): self.assertTrue(id_ in self.TRUE_LABELS[index]) self.assertLess(weight, -1.0e37) + @test_util.run_deprecated_v1 def testSeed(self): def draw(seed): @@ -114,7 +117,7 @@ class RangeSamplerOpsTest(test.TestCase): [[1, 2], [0, 4], [3, 3]], dtype=dtypes.int64) sampled, _, _ = candidate_sampling_ops.log_uniform_candidate_sampler( true_classes, self.NUM_TRUE, self.NUM_SAMPLED, True, 5, seed=seed) - return sampled.eval() + return self.evaluate(sampled) # Non-zero seed. Repeatable. for seed in [1, 12, 123, 1234]: diff --git a/tensorflow/python/kernel_tests/cast_op_test.py b/tensorflow/python/kernel_tests/cast_op_test.py index a5dff5df629..b3187e16371 100644 --- a/tensorflow/python/kernel_tests/cast_op_test.py +++ b/tensorflow/python/kernel_tests/cast_op_test.py @@ -25,6 +25,7 @@ import platform from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes from tensorflow.python.framework import sparse_tensor +from tensorflow.python.framework import test_util from tensorflow.python.ops import array_ops from tensorflow.python.ops import gradient_checker from tensorflow.python.ops import math_ops @@ -90,10 +91,12 @@ class CastOpTest(test.TestCase): if x.dtype == np.float32 or x.dtype == np.float64: self._testTypes(x, use_gpu=True) + @test_util.run_deprecated_v1 def testBasic(self): self._testAll(np.arange(-10, 10).reshape(2, 10)) self._testAll(np.linspace(-10, 10, 17)) + @test_util.run_deprecated_v1 def testSmallValues(self): f4 = np.finfo(np.float32) f8 = np.finfo(np.float64) @@ -107,11 +110,12 @@ class CastOpTest(test.TestCase): a = np.random.uniform(-100, 100, 100).astype(np.float32) with self.cached_session(use_gpu=False): b = math_ops.cast(math_ops.cast(a, dtypes.bfloat16), dtypes.float32) - self.assertAllClose(a, b.eval(), rtol=1 / 128.) + self.assertAllClose(a, self.evaluate(b), rtol=1 / 128.) with self.cached_session(use_gpu=True): b = math_ops.cast(math_ops.cast(a, dtypes.bfloat16), dtypes.float32) - self.assertAllClose(a, b.eval(), rtol=1 / 128.) + self.assertAllClose(a, self.evaluate(b), rtol=1 / 128.) + @test_util.run_deprecated_v1 def testRandom(self): self._testAll(np.random.normal(0, 10, 210).reshape([2, 3, 5, 7])) self._testAll(np.random.normal(0, 1e6, 210).reshape([2, 3, 5, 7])) @@ -124,6 +128,7 @@ class CastOpTest(test.TestCase): self._cast( x, dst_dtype, use_gpu=use_gpu), dst_dtype(expected)) + @test_util.run_deprecated_v1 def testIntToFloatBoundary(self): i4 = np.iinfo(np.int32) i8 = np.iinfo(np.int64) @@ -138,6 +143,7 @@ class CastOpTest(test.TestCase): self._compare(i8.max, np.float64, i8.max, False) # NOTE: GPU does not support int32/int64 for casting. + @test_util.run_deprecated_v1 def testInfNan(self): i4 = np.iinfo(np.int32) i8 = np.iinfo(np.int64) @@ -181,14 +187,16 @@ class CastOpTest(test.TestCase): def testNotImplemented(self): self._OpError(np.arange(0, 10), dtypes.string, "Cast.*int64.*string.*") + @test_util.run_deprecated_v1 def testCastToTypeOfVariable(self): with self.cached_session() as sess: x = variables.Variable(5, dtype=dtypes.float32) y = variables.Variable(True, dtype=dtypes.bool) cast = math_ops.cast(y, x.dtype) variables.global_variables_initializer().run() - self.assertEqual(1.0, sess.run(cast)) + self.assertEqual(1.0, self.evaluate(cast)) + @test_util.run_deprecated_v1 def testGradients(self): t = [dtypes.float32, dtypes.float64, dtypes.complex64, dtypes.complex128] for src_t in t: @@ -203,6 +211,7 @@ class CastOpTest(test.TestCase): class SparseTensorCastTest(test.TestCase): + @test_util.run_deprecated_v1 def testCast(self): indices = constant_op.constant([[0], [1], [2]], dtypes.int64) values = constant_op.constant(np.array([1, 2, 3], np.int64)) @@ -229,7 +238,7 @@ class SaturateCastTest(test.TestCase): [lo, lo + 1, lo // 2, hi // 2, hi - 1, hi], dtype=in_type) y = math_ops.saturate_cast(x, dtype=out_type) self.assertEqual(y.dtype, out_type) - x, y = sess.run([x, y]) + x, y = self.evaluate([x, y]) correct = np.maximum(out_type.min, np.minimum(out_type.max, x)) self.assertAllEqual(correct, y) diff --git a/tensorflow/python/kernel_tests/check_ops_test.py b/tensorflow/python/kernel_tests/check_ops_test.py index 88f5cd6f223..95bac85027b 100644 --- a/tensorflow/python/kernel_tests/check_ops_test.py +++ b/tensorflow/python/kernel_tests/check_ops_test.py @@ -25,6 +25,7 @@ from tensorflow.core.protobuf import config_pb2 from tensorflow.core.protobuf import rewriter_config_pb2 from tensorflow.python.client import session from tensorflow.python.eager import context +from tensorflow.python.eager import def_function from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes from tensorflow.python.framework import errors @@ -39,6 +40,69 @@ from tensorflow.python.ops import random_ops from tensorflow.python.platform import test +class AssertV2Asserts(test.TestCase): + + def test_passes_when_it_should(self): + # This is a v2 test and need to run eagerly + with context.eager_mode(): + c1 = constant_op.constant(-1, name="minus_one", dtype=dtypes.int32) + c2 = constant_op.constant(2, name="two", dtype=dtypes.int32) + c3 = constant_op.constant([3., 3.], name="three", dtype=dtypes.float32) + c4 = constant_op.constant([3., 3.5], name="three_and_a_half", + dtype=dtypes.float32) + scalar = c1 + non_scalar = c3 + integer = c1 + non_integer = c3 + positive = c2 + negative = c1 + cases = [ + (check_ops.assert_equal_v2, (c1, c1), (c1, c2)), + (check_ops.assert_less_v2, (c1, c2), (c1, c1)), + (check_ops.assert_near_v2, (c3, c3), (c3, c4)), + (check_ops.assert_greater_v2, (c2, c1), (c1, c1)), + (check_ops.assert_negative_v2, (negative,), (positive,)), + (check_ops.assert_positive_v2, (positive,), (negative,)), + (check_ops.assert_less_equal_v2, (c1, c1), (c2, c1)), + (check_ops.assert_none_equal_v2, (c1, c2), (c3, c4)), + (check_ops.assert_non_negative_v2, (positive,), (negative,)), + (check_ops.assert_non_positive_v2, (negative,), (positive,)), + (check_ops.assert_greater_equal_v2, (c1, c1), (c1, c2)), + (check_ops.assert_type_v2, (c1, dtypes.int32), (c1, dtypes.float32), + TypeError), + (check_ops.assert_integer_v2, (integer,), (non_integer,), + TypeError), + (check_ops.assert_scalar_v2, (scalar,), (non_scalar,), + ValueError), + (check_ops.assert_rank_v2, (c1, 0), (c3, 2), ValueError), + (check_ops.assert_rank_in_v2, (c1, [0, 1]), (c1, [1, 2]), + ValueError), + (check_ops.assert_rank_at_least_v2, (non_scalar, 1), (scalar, 1), + ValueError), + ] + + for case in cases: + fn = case[0] + passing_args = case[1] + failing_args = case[2] + error = errors.InvalidArgumentError if len(case) < 4 else case[3] + + print("Testing %s passing properly." % fn) + + fn(*passing_args) + + print("Testing %s failing properly." % fn) + + @def_function.function + def failing_fn(): + fn(*failing_args, message="fail") # pylint: disable=cell-var-from-loop + + with self.assertRaisesRegexp(error, "fail"): + failing_fn() + + del failing_fn + + class AssertProperIterableTest(test.TestCase): @test_util.run_in_graph_and_eager_modes @@ -109,6 +173,7 @@ class AssertEqualTest(test.TestCase): assert x is None @test_util.run_in_graph_and_eager_modes + @test_util.run_deprecated_v1 def test_raises_when_greater(self): # Static check static_small = constant_op.constant([1, 2], name="small") @@ -116,6 +181,7 @@ class AssertEqualTest(test.TestCase): with self.assertRaisesRegexp(errors.InvalidArgumentError, "fail"): check_ops.assert_equal(static_big, static_small, message="fail") + @test_util.run_deprecated_v1 def test_raises_when_greater_dynamic(self): with self.cached_session(): small = array_ops.placeholder(dtypes.int32, name="small") @@ -187,6 +253,7 @@ First 2 elements of y: summarize=2) @test_util.run_in_graph_and_eager_modes + @test_util.run_deprecated_v1 def test_raises_when_less(self): # Static check static_small = constant_op.constant([3, 1], name="small") @@ -194,6 +261,7 @@ First 2 elements of y: with self.assertRaisesRegexp(errors.InvalidArgumentError, "fail"): check_ops.assert_equal(static_big, static_small, message="fail") + @test_util.run_deprecated_v1 def test_raises_when_less_dynamic(self): with self.cached_session(): small = array_ops.placeholder(dtypes.int32, name="small") @@ -253,6 +321,7 @@ class AssertNoneEqualTest(test.TestCase): self.evaluate(out) @test_util.run_in_graph_and_eager_modes + @test_util.run_deprecated_v1 def test_raises_when_equal(self): small = constant_op.constant([3, 1], name="small") with self.assertRaisesOpError("x != y did not hold"): @@ -442,6 +511,7 @@ class AssertAllCloseTest(test.TestCase): class AssertLessTest(test.TestCase): @test_util.run_in_graph_and_eager_modes + @test_util.run_deprecated_v1 def test_raises_when_equal(self): small = constant_op.constant([1, 2], name="small") with self.assertRaisesOpError("failure message.*\n*.* x < y did not hold"): @@ -452,6 +522,7 @@ class AssertLessTest(test.TestCase): self.evaluate(out) @test_util.run_in_graph_and_eager_modes + @test_util.run_deprecated_v1 def test_raises_when_greater(self): small = constant_op.constant([1, 2], name="small") big = constant_op.constant([3, 4], name="big") @@ -518,6 +589,7 @@ class AssertLessEqualTest(test.TestCase): self.evaluate(out) @test_util.run_in_graph_and_eager_modes + @test_util.run_deprecated_v1 def test_raises_when_greater(self): small = constant_op.constant([1, 2], name="small") big = constant_op.constant([3, 4], name="big") @@ -573,6 +645,7 @@ class AssertLessEqualTest(test.TestCase): class AssertGreaterTest(test.TestCase): @test_util.run_in_graph_and_eager_modes + @test_util.run_deprecated_v1 def test_raises_when_equal(self): small = constant_op.constant([1, 2], name="small") with self.assertRaisesOpError("fail"): @@ -583,6 +656,7 @@ class AssertGreaterTest(test.TestCase): self.evaluate(out) @test_util.run_in_graph_and_eager_modes + @test_util.run_deprecated_v1 def test_raises_when_less(self): small = constant_op.constant([1, 2], name="small") big = constant_op.constant([3, 4], name="big") @@ -642,6 +716,7 @@ class AssertGreaterEqualTest(test.TestCase): self.evaluate(out) @test_util.run_in_graph_and_eager_modes + @test_util.run_deprecated_v1 def test_raises_when_less(self): small = constant_op.constant([1, 2], name="small") big = constant_op.constant([3, 4], name="big") @@ -706,6 +781,7 @@ class AssertNegativeTest(test.TestCase): self.evaluate(out) @test_util.run_in_graph_and_eager_modes + @test_util.run_deprecated_v1 def test_raises_when_positive(self): doug = constant_op.constant([1, 2], name="doug") with self.assertRaisesOpError("fail"): @@ -716,6 +792,7 @@ class AssertNegativeTest(test.TestCase): self.evaluate(out) @test_util.run_in_graph_and_eager_modes + @test_util.run_deprecated_v1 def test_raises_when_zero(self): claire = constant_op.constant([0], name="claire") with self.assertRaisesOpError("x < 0 did not hold"): @@ -738,6 +815,7 @@ class AssertNegativeTest(test.TestCase): class AssertPositiveTest(test.TestCase): @test_util.run_in_graph_and_eager_modes + @test_util.run_deprecated_v1 def test_raises_when_negative(self): freddie = constant_op.constant([-1, -2], name="freddie") with self.assertRaisesOpError("fail"): @@ -755,6 +833,7 @@ class AssertPositiveTest(test.TestCase): self.evaluate(out) @test_util.run_in_graph_and_eager_modes + @test_util.run_deprecated_v1 def test_raises_when_zero(self): meechum = constant_op.constant([0], name="meechum") with self.assertRaisesOpError("x > 0 did not hold"): @@ -777,26 +856,31 @@ class AssertPositiveTest(test.TestCase): class EnsureShapeTest(test.TestCase): # Static shape inference + @test_util.run_deprecated_v1 def testStaticShape(self): placeholder = array_ops.placeholder(dtypes.int32) ensure_shape_op = check_ops.ensure_shape(placeholder, (3, 3, 3)) self.assertEqual(ensure_shape_op.get_shape(), (3, 3, 3)) + @test_util.run_deprecated_v1 def testStaticShape_MergesShapes(self): placeholder = array_ops.placeholder(dtypes.int32, shape=(None, None, 3)) ensure_shape_op = check_ops.ensure_shape(placeholder, (5, 4, None)) self.assertEqual(ensure_shape_op.get_shape(), (5, 4, 3)) + @test_util.run_deprecated_v1 def testStaticShape_RaisesErrorWhenRankIncompatible(self): placeholder = array_ops.placeholder(dtypes.int32, shape=(None, None, 3)) with self.assertRaises(ValueError): check_ops.ensure_shape(placeholder, (2, 3)) + @test_util.run_deprecated_v1 def testStaticShape_RaisesErrorWhenDimIncompatible(self): placeholder = array_ops.placeholder(dtypes.int32, shape=(None, None, 3)) with self.assertRaises(ValueError): check_ops.ensure_shape(placeholder, (2, 2, 4)) + @test_util.run_deprecated_v1 def testStaticShape_CanSetUnknownShape(self): placeholder = array_ops.placeholder(dtypes.int32) derived = placeholder / 3 @@ -804,6 +888,7 @@ class EnsureShapeTest(test.TestCase): self.assertEqual(ensure_shape_op.get_shape(), None) # Dynamic shape check + @test_util.run_deprecated_v1 def testEnsuresDynamicShape_RaisesError(self): placeholder = array_ops.placeholder(dtypes.int32) derived = math_ops.divide(placeholder, 3, name="MyDivide") @@ -816,6 +901,7 @@ class EnsureShapeTest(test.TestCase): r"expected shape \[3,3,3\]."): sess.run(derived, feed_dict={placeholder: feed_val}) + @test_util.run_deprecated_v1 def testEnsuresDynamicShape_RaisesErrorDimUnknown(self): placeholder = array_ops.placeholder(dtypes.int32) derived = placeholder / 3 @@ -828,6 +914,7 @@ class EnsureShapeTest(test.TestCase): r"expected shape \[\?,\?,3\]."): sess.run(derived, feed_dict={placeholder: feed_val}) + @test_util.run_deprecated_v1 def testEnsuresDynamicShape(self): placeholder = array_ops.placeholder(dtypes.int32) derived = placeholder / 3 @@ -836,6 +923,7 @@ class EnsureShapeTest(test.TestCase): with self.cached_session() as sess: sess.run(derived, feed_dict={placeholder: feed_val}) + @test_util.run_deprecated_v1 def testEnsuresDynamicShape_WithUnknownDims(self): placeholder = array_ops.placeholder(dtypes.int32) derived = placeholder / 3 @@ -844,6 +932,7 @@ class EnsureShapeTest(test.TestCase): with self.cached_session() as sess: sess.run(derived, feed_dict={placeholder: feed_val}) + @test_util.run_deprecated_v1 def testGradient(self): placeholder = array_ops.placeholder(dtypes.float32) derived = check_ops.ensure_shape(placeholder, (None, None)) @@ -939,6 +1028,7 @@ class AssertRankTest(test.TestCase): tensor, desired_rank, message="fail")]): self.evaluate(array_ops.identity(tensor)) + @test_util.run_deprecated_v1 def test_rank_zero_tensor_raises_if_rank_too_small_dynamic_rank(self): with self.cached_session(): tensor = array_ops.placeholder(dtypes.float32, name="my_tensor") @@ -957,6 +1047,7 @@ class AssertRankTest(test.TestCase): [check_ops.assert_rank(tensor, desired_rank)]): self.evaluate(array_ops.identity(tensor)) + @test_util.run_deprecated_v1 def test_rank_zero_tensor_doesnt_raise_if_rank_just_right_dynamic_rank(self): with self.cached_session(): tensor = array_ops.placeholder(dtypes.float32, name="my_tensor") @@ -974,6 +1065,7 @@ class AssertRankTest(test.TestCase): [check_ops.assert_rank(tensor, desired_rank)]): self.evaluate(array_ops.identity(tensor)) + @test_util.run_deprecated_v1 def test_rank_one_tensor_raises_if_rank_too_large_dynamic_rank(self): with self.cached_session(): tensor = array_ops.placeholder(dtypes.float32, name="my_tensor") @@ -991,6 +1083,7 @@ class AssertRankTest(test.TestCase): [check_ops.assert_rank(tensor, desired_rank)]): self.evaluate(array_ops.identity(tensor)) + @test_util.run_deprecated_v1 def test_rank_one_tensor_doesnt_raise_if_rank_just_right_dynamic_rank(self): with self.cached_session(): tensor = array_ops.placeholder(dtypes.float32, name="my_tensor") @@ -1008,6 +1101,7 @@ class AssertRankTest(test.TestCase): [check_ops.assert_rank(tensor, desired_rank)]): self.evaluate(array_ops.identity(tensor)) + @test_util.run_deprecated_v1 def test_rank_one_tensor_raises_if_rank_too_small_dynamic_rank(self): with self.cached_session(): tensor = array_ops.placeholder(dtypes.float32, name="my_tensor") @@ -1023,6 +1117,7 @@ class AssertRankTest(test.TestCase): with self.assertRaisesRegexp(ValueError, "Rank must be a scalar"): check_ops.assert_rank(tensor, np.array([], dtype=np.int32)) + @test_util.run_deprecated_v1 def test_raises_if_rank_is_not_scalar_dynamic(self): with self.cached_session(): tensor = constant_op.constant( @@ -1040,6 +1135,7 @@ class AssertRankTest(test.TestCase): "must be of type "): check_ops.assert_rank(tensor, .5) + @test_util.run_deprecated_v1 def test_raises_if_rank_is_not_integer_dynamic(self): with self.cached_session(): tensor = constant_op.constant( @@ -1063,6 +1159,7 @@ class AssertRankInTest(test.TestCase): check_ops.assert_rank_in(tensor_rank0, (1, 2), message="fail")]): self.evaluate(array_ops.identity(tensor_rank0)) + @test_util.run_deprecated_v1 def test_rank_zero_tensor_raises_if_rank_mismatch_dynamic_rank(self): with self.cached_session(): tensor_rank0 = array_ops.placeholder(dtypes.float32, name="my_tensor") @@ -1079,6 +1176,7 @@ class AssertRankInTest(test.TestCase): check_ops.assert_rank_in(tensor_rank0, desired_ranks)]): self.evaluate(array_ops.identity(tensor_rank0)) + @test_util.run_deprecated_v1 def test_rank_zero_tensor_doesnt_raise_if_rank_matches_dynamic_rank(self): with self.cached_session(): tensor_rank0 = array_ops.placeholder(dtypes.float32, name="my_tensor") @@ -1095,6 +1193,7 @@ class AssertRankInTest(test.TestCase): check_ops.assert_rank_in(tensor_rank1, desired_ranks)]): self.evaluate(array_ops.identity(tensor_rank1)) + @test_util.run_deprecated_v1 def test_rank_one_tensor_doesnt_raise_if_rank_matches_dynamic_rank(self): with self.cached_session(): tensor_rank1 = array_ops.placeholder(dtypes.float32, name="my_tensor") @@ -1113,6 +1212,7 @@ class AssertRankInTest(test.TestCase): check_ops.assert_rank_in(tensor_rank1, (0, 2))]): self.evaluate(array_ops.identity(tensor_rank1)) + @test_util.run_deprecated_v1 def test_rank_one_tensor_raises_if_rank_mismatches_dynamic_rank(self): with self.cached_session(): tensor_rank1 = array_ops.placeholder(dtypes.float32, name="my_tensor") @@ -1132,6 +1232,7 @@ class AssertRankInTest(test.TestCase): with self.assertRaisesRegexp(ValueError, "Rank must be a scalar"): check_ops.assert_rank_in(tensor, desired_ranks) + @test_util.run_deprecated_v1 def test_raises_if_rank_is_not_scalar_dynamic(self): with self.cached_session(): tensor = constant_op.constant( @@ -1154,6 +1255,7 @@ class AssertRankInTest(test.TestCase): "must be of type "): check_ops.assert_rank_in(tensor, (1, .5,)) + @test_util.run_deprecated_v1 def test_raises_if_rank_is_not_integer_dynamic(self): with self.cached_session(): tensor = constant_op.constant( @@ -1177,6 +1279,7 @@ class AssertRankAtLeastTest(test.TestCase): [check_ops.assert_rank_at_least(tensor, desired_rank)]): self.evaluate(array_ops.identity(tensor)) + @test_util.run_deprecated_v1 def test_rank_zero_tensor_raises_if_rank_too_small_dynamic_rank(self): with self.cached_session(): tensor = array_ops.placeholder(dtypes.float32, name="my_tensor") @@ -1194,6 +1297,7 @@ class AssertRankAtLeastTest(test.TestCase): [check_ops.assert_rank_at_least(tensor, desired_rank)]): self.evaluate(array_ops.identity(tensor)) + @test_util.run_deprecated_v1 def test_rank_zero_tensor_doesnt_raise_if_rank_just_right_dynamic_rank(self): with self.cached_session(): tensor = array_ops.placeholder(dtypes.float32, name="my_tensor") @@ -1210,6 +1314,7 @@ class AssertRankAtLeastTest(test.TestCase): [check_ops.assert_rank_at_least(tensor, desired_rank)]): self.evaluate(array_ops.identity(tensor)) + @test_util.run_deprecated_v1 def test_rank_one_ten_doesnt_raise_if_rank_too_large_dynamic_rank(self): with self.cached_session(): tensor = array_ops.placeholder(dtypes.float32, name="my_tensor") @@ -1226,6 +1331,7 @@ class AssertRankAtLeastTest(test.TestCase): [check_ops.assert_rank_at_least(tensor, desired_rank)]): self.evaluate(array_ops.identity(tensor)) + @test_util.run_deprecated_v1 def test_rank_one_tensor_doesnt_raise_if_rank_just_right_dynamic_rank(self): with self.cached_session(): tensor = array_ops.placeholder(dtypes.float32, name="my_tensor") @@ -1243,6 +1349,7 @@ class AssertRankAtLeastTest(test.TestCase): [check_ops.assert_rank_at_least(tensor, desired_rank)]): self.evaluate(array_ops.identity(tensor)) + @test_util.run_deprecated_v1 def test_rank_one_tensor_raises_if_rank_too_small_dynamic_rank(self): with self.cached_session(): tensor = array_ops.placeholder(dtypes.float32, name="my_tensor") @@ -1256,6 +1363,7 @@ class AssertRankAtLeastTest(test.TestCase): class AssertNonNegativeTest(test.TestCase): @test_util.run_in_graph_and_eager_modes + @test_util.run_deprecated_v1 def test_raises_when_negative(self): zoe = constant_op.constant([-1, -2], name="zoe") with self.assertRaisesOpError("x >= 0 did not hold"): @@ -1292,6 +1400,7 @@ class AssertNonPositiveTest(test.TestCase): self.evaluate(out) @test_util.run_in_graph_and_eager_modes + @test_util.run_deprecated_v1 def test_raises_when_positive(self): rachel = constant_op.constant([0, 2], name="rachel") with self.assertRaisesOpError("x <= 0 did not hold"): diff --git a/tensorflow/python/kernel_tests/checkpoint_ops_test.py b/tensorflow/python/kernel_tests/checkpoint_ops_test.py index 51611b75afb..b8c8c9edb5a 100644 --- a/tensorflow/python/kernel_tests/checkpoint_ops_test.py +++ b/tensorflow/python/kernel_tests/checkpoint_ops_test.py @@ -24,6 +24,7 @@ from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes from tensorflow.python.framework import errors from tensorflow.python.framework import ops +from tensorflow.python.framework import test_util from tensorflow.python.ops import gen_checkpoint_ops from tensorflow.python.ops import partitioned_variables from tensorflow.python.ops import variable_scope @@ -48,6 +49,7 @@ class GenerateVocabRemappingTest(test.TestCase): with open(self.old_vocab_file, 'w') as f: f.write('\n'.join(['knitting', 'eminem', 'MISSING']) + '\n') + @test_util.run_deprecated_v1 def test_generate_remapping_with_no_vocab_changes(self): """Tests where vocab does not change at all.""" remapping, num_present = gen_checkpoint_ops.generate_vocab_remapping( @@ -58,8 +60,8 @@ class GenerateVocabRemappingTest(test.TestCase): expected_remapping = range(0, 3) expected_num_present = 3 with self.cached_session(): - self.assertAllEqual(expected_remapping, remapping.eval()) - self.assertAllEqual(expected_num_present, num_present.eval()) + self.assertAllEqual(expected_remapping, self.evaluate(remapping)) + self.assertAllEqual(expected_num_present, self.evaluate(num_present)) def test_generate_remapping_with_shifted_vocab(self): """Tests where vocab is the same, but shifted / ordered differently.""" @@ -71,8 +73,8 @@ class GenerateVocabRemappingTest(test.TestCase): expected_remapping = [2, 0, 1] expected_num_present = 3 with self.cached_session(): - self.assertAllEqual(expected_remapping, remapping.eval()) - self.assertAllEqual(expected_num_present, num_present.eval()) + self.assertAllEqual(expected_remapping, self.evaluate(remapping)) + self.assertAllEqual(expected_num_present, self.evaluate(num_present)) def test_generate_remapping_with_offset(self): """Tests offset and num_new_vocab logic.""" @@ -84,8 +86,8 @@ class GenerateVocabRemappingTest(test.TestCase): expected_remapping = [0] expected_num_present = 1 with self.cached_session(): - self.assertAllEqual(expected_remapping, remapping.eval()) - self.assertAllEqual(expected_num_present, num_present.eval()) + self.assertAllEqual(expected_remapping, self.evaluate(remapping)) + self.assertAllEqual(expected_num_present, self.evaluate(num_present)) def test_generate_remapping_with_old_vocab_size(self): """Tests where old_vocab_size is specified.""" @@ -99,8 +101,8 @@ class GenerateVocabRemappingTest(test.TestCase): expected_remapping = [-1, 0, 1] expected_num_present = 2 with self.cached_session(): - self.assertAllEqual(expected_remapping, remapping.eval()) - self.assertAllEqual(expected_num_present, num_present.eval()) + self.assertAllEqual(expected_remapping, self.evaluate(remapping)) + self.assertAllEqual(expected_num_present, self.evaluate(num_present)) class LoadAndRemapMatrixTest(test.TestCase): @@ -142,7 +144,7 @@ class LoadAndRemapMatrixTest(test.TestCase): num_cols=self.old_num_cols) with self.cached_session(): self.assertAllClose(self.matrix_value[row_remapping], - remapped_matrix.eval()) + self.evaluate(remapped_matrix)) # No row remapping, new weight matrix has third col, then first col. row_remapping = list(range(self.old_num_rows)) @@ -157,7 +159,7 @@ class LoadAndRemapMatrixTest(test.TestCase): num_cols=len(col_remapping)) with self.cached_session(): self.assertAllClose(self.matrix_value[row_remapping][:, col_remapping], - remapped_matrix.eval()) + self.evaluate(remapped_matrix)) # Both row and column remappings. row_remapping = [1, 0, 4] @@ -172,7 +174,7 @@ class LoadAndRemapMatrixTest(test.TestCase): num_cols=len(col_remapping)) with self.cached_session(): self.assertAllClose(self.matrix_value[row_remapping][:, col_remapping], - remapped_matrix.eval()) + self.evaluate(remapped_matrix)) def test_load_and_remap_with_init(self): """Tests the op's load and remap where there are missing entries.""" @@ -190,7 +192,8 @@ class LoadAndRemapMatrixTest(test.TestCase): [33, init_val, init_val, init_val, 1, init_val], [3, 2]) with self.cached_session(): - self.assertAllClose(expected_remapped_matrix, remapped_matrix.eval()) + self.assertAllClose(expected_remapped_matrix, + self.evaluate(remapped_matrix)) def test_load_and_remap_all_missing_rows(self): """Tests when all the rows are missing and need to be initialized.""" @@ -207,7 +210,7 @@ class LoadAndRemapMatrixTest(test.TestCase): with self.cached_session(): self.assertAllClose( np.reshape(initializing_values, (num_rows, self.old_num_cols)), - remapped_matrix.eval()) + self.evaluate(remapped_matrix)) def test_load_and_remap_all_missing_rows_and_cols(self): """Tests when all the rows & cols are missing and need to be initialized.""" @@ -225,7 +228,7 @@ class LoadAndRemapMatrixTest(test.TestCase): with self.cached_session(): self.assertAllClose( np.reshape(initializing_values, (num_rows, num_cols)), - remapped_matrix.eval()) + self.evaluate(remapped_matrix)) def test_load_and_remap_invalid_remapping(self): """Tests that errors are raised when an ID maps to multiple new IDs. @@ -244,7 +247,7 @@ class LoadAndRemapMatrixTest(test.TestCase): num_rows=len(invalid_remapping), num_cols=self.old_num_cols) with self.cached_session(), self.assertRaises(errors.UnimplementedError): - remapped_matrix.eval() + self.evaluate(remapped_matrix) # Invalid column remapping. remapped_matrix = gen_checkpoint_ops.load_and_remap_matrix( @@ -256,7 +259,7 @@ class LoadAndRemapMatrixTest(test.TestCase): num_rows=self.old_num_rows, num_cols=len(invalid_remapping)) with self.cached_session(), self.assertRaises(errors.UnimplementedError): - remapped_matrix.eval() + self.evaluate(remapped_matrix) def test_load_and_remap_incorrect_initializing_values(self): """Tests that errors are raised with incorrect number of init values.""" @@ -273,7 +276,7 @@ class LoadAndRemapMatrixTest(test.TestCase): num_rows=3, num_cols=2) with self.cached_session(), self.assertRaises(errors.InvalidArgumentError): - remapped_matrix.eval() + self.evaluate(remapped_matrix) remapped_matrix = gen_checkpoint_ops.load_and_remap_matrix( ckpt_path=[self.bundle_file], @@ -285,7 +288,7 @@ class LoadAndRemapMatrixTest(test.TestCase): num_rows=3, num_cols=2) with self.cached_session(), self.assertRaises(errors.InvalidArgumentError): - remapped_matrix.eval() + self.evaluate(remapped_matrix) class LoadAndRemapMatrixWithMaxRowsTest(test.TestCase): @@ -324,7 +327,7 @@ class LoadAndRemapMatrixWithMaxRowsTest(test.TestCase): num_rows=num_rows, num_cols=num_cols, max_rows_in_memory=max_rows_in_memory) - self.assertAllClose(np_value[::-1], remapped_matrix.eval()) + self.assertAllClose(np_value[::-1], self.evaluate(remapped_matrix)) # Tests loading the tensor (except for the first and last rows), with # uninitialized values. Requires num_rows to be at least 3 since we're @@ -348,7 +351,7 @@ class LoadAndRemapMatrixWithMaxRowsTest(test.TestCase): np.vstack([ np.tile(42, [prefix_rows, num_cols]), np_value[1:-1], np.tile(42, [suffix_rows, num_cols]) - ]), remapped_matrix.eval()) + ]), self.evaluate(remapped_matrix)) # Tests when everything is taken from initializing_values. new_rows = 7 @@ -365,8 +368,9 @@ class LoadAndRemapMatrixWithMaxRowsTest(test.TestCase): max_rows_in_memory=max_rows_in_memory) self.assertAllClose( np.reshape(initializing_values, (new_rows, num_cols)), - remapped_matrix.eval()) + self.evaluate(remapped_matrix)) + @test_util.run_deprecated_v1 def test_loading_rows_divisible_by_max_rows(self): """Tests loading normal var when rows are evenly divisible by max_rows.""" self._test_loading_variable_with_max_rows( @@ -375,6 +379,7 @@ class LoadAndRemapMatrixWithMaxRowsTest(test.TestCase): # 9 is evenly divisible by 3. max_rows_in_memory=3) + @test_util.run_deprecated_v1 def test_loading_rows_not_divisible_by_max_rows(self): """Tests loading normal var when rows aren't divisible by max_rows.""" self._test_loading_variable_with_max_rows( @@ -383,6 +388,7 @@ class LoadAndRemapMatrixWithMaxRowsTest(test.TestCase): # 9 is not evenly divisible by 4. max_rows_in_memory=4) + @test_util.run_deprecated_v1 def test_loading_rows_less_than_max_rows(self): """Tests loading normal var as a single slice. @@ -394,6 +400,7 @@ class LoadAndRemapMatrixWithMaxRowsTest(test.TestCase): # 10 > 9. max_rows_in_memory=10) + @test_util.run_deprecated_v1 def test_loading_no_max_rows(self): """Tests loading normal var as a single slice with no valid max_rows.""" self._test_loading_variable_with_max_rows( @@ -401,6 +408,7 @@ class LoadAndRemapMatrixWithMaxRowsTest(test.TestCase): partitioner=None, max_rows_in_memory=-1) + @test_util.run_deprecated_v1 def test_loading_partitions_equals_max_rows(self): """Tests loading partitioned var sliced on partition boundary.""" self._test_loading_variable_with_max_rows( @@ -410,6 +418,7 @@ class LoadAndRemapMatrixWithMaxRowsTest(test.TestCase): # exactly 3 rows. max_rows_in_memory=3) + @test_util.run_deprecated_v1 def test_loading_partitions_greater_than_max_rows(self): """Tests loading partitioned var with more slices than partitions.""" self._test_loading_variable_with_max_rows( @@ -419,6 +428,7 @@ class LoadAndRemapMatrixWithMaxRowsTest(test.TestCase): # row at a time. max_rows_in_memory=1) + @test_util.run_deprecated_v1 def test_loading_partitions_less_than_max_rows(self): """Tests loading partitioned var as a single slice. @@ -429,6 +439,7 @@ class LoadAndRemapMatrixWithMaxRowsTest(test.TestCase): partitioner=partitioned_variables.fixed_size_partitioner(3), max_rows_in_memory=10) + @test_util.run_deprecated_v1 def test_loading_partitions_no_max_rows(self): """Tests loading partitioned var as single slice with no valid max_rows.""" self._test_loading_variable_with_max_rows( diff --git a/tensorflow/python/kernel_tests/cholesky_op_test.py b/tensorflow/python/kernel_tests/cholesky_op_test.py index e96b2772665..f3947236b1f 100644 --- a/tensorflow/python/kernel_tests/cholesky_op_test.py +++ b/tensorflow/python/kernel_tests/cholesky_op_test.py @@ -26,6 +26,7 @@ from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes as dtypes_lib from tensorflow.python.framework import errors_impl from tensorflow.python.framework import ops +from tensorflow.python.framework import test_util from tensorflow.python.ops import array_ops from tensorflow.python.ops import control_flow_ops from tensorflow.python.ops import gen_linalg_ops @@ -97,7 +98,7 @@ def TriAngInvCompositeGrad(l, grad): class CholeskyOpTest(test.TestCase): def _verifyCholeskyBase(self, sess, x, chol, verification): - chol_np, verification_np = sess.run([chol, verification]) + chol_np, verification_np = self.evaluate([chol, verification]) self.assertAllClose(x, verification_np) self.assertShapeEqual(x, chol) # Check that the cholesky is lower triangular, and has positive diagonal @@ -145,6 +146,7 @@ class CholeskyOpTest(test.TestCase): matrices[i] = np.dot(matrices[i].T.conj(), matrices[i]) self._verifyCholesky(matrices) + @test_util.run_deprecated_v1 def testNonSquareMatrix(self): with self.assertRaises(ValueError): linalg_ops.cholesky(np.array([[1., 2., 3.], [3., 4., 5.]])) @@ -175,6 +177,7 @@ class CholeskyOpTest(test.TestCase): self._verifyCholesky(np.empty([0, 2, 2])) self._verifyCholesky(np.empty([2, 0, 0])) + @test_util.run_deprecated_v1 def testConcurrentExecutesWithoutError(self): with self.session(use_gpu=True) as sess: matrix1 = random_ops.random_normal([5, 5], seed=42) @@ -183,8 +186,8 @@ class CholeskyOpTest(test.TestCase): matrix2 = math_ops.matmul(matrix2, matrix2, adjoint_a=True) c1 = linalg_ops.cholesky(matrix1) c2 = linalg_ops.cholesky(matrix2) - c1_val, c2_val = sess.run([c1, c2]) - self.assertAllEqual(c1_val, c2_val) + c1_val, c2_val = self.evaluate([c1, c2]) + self.assertAllClose(c1_val, c2_val) class CholeskyGradTest(test.TestCase): @@ -193,18 +196,21 @@ class CholeskyGradTest(test.TestCase): def getShapes(self, shapeList): return ((elem, int(np.floor(1.2 * elem))) for elem in shapeList) + @test_util.run_deprecated_v1 def testSmallMatrices(self): np.random.seed(0) shapes = self.getShapes([1, 2, 10]) self.runFiniteDifferences( shapes, dtypes=(dtypes_lib.float32, dtypes_lib.float64)) + @test_util.run_deprecated_v1 def testSmallMatricesComplex(self): np.random.seed(0) shapes = self.getShapes([1, 2, 10]) self.runFiniteDifferences( shapes, dtypes=(dtypes_lib.complex64, dtypes_lib.complex128)) + @test_util.run_deprecated_v1 def testOneBlockMatrices(self): np.random.seed(0) shapes = self.getShapes([self._backprop_block_size + 1]) @@ -213,12 +219,14 @@ class CholeskyGradTest(test.TestCase): dtypes=(dtypes_lib.float32, dtypes_lib.float64), scalarTest=True) + @test_util.run_deprecated_v1 def testTwoBlockMatrixFloat(self): np.random.seed(0) shapes = self.getShapes([2 * self._backprop_block_size + 1]) self.runFiniteDifferences( shapes, dtypes=(dtypes_lib.float32,), scalarTest=True) + @test_util.run_deprecated_v1 def testTwoBlockMatrixDouble(self): np.random.seed(0) shapes = self.getShapes([2 * self._backprop_block_size + 1]) @@ -231,6 +239,7 @@ class CholeskyGradTest(test.TestCase): self.runFiniteDifferences( shapes, dtypes=(dtypes_lib.complex64,), scalarTest=True) + @test_util.run_deprecated_v1 def testTwoBlockMatrixComplexDouble(self): np.random.seed(0) shapes = self.getShapes([2 * self._backprop_block_size + 1]) diff --git a/tensorflow/python/kernel_tests/clip_ops_test.py b/tensorflow/python/kernel_tests/clip_ops_test.py index efd7eee8474..45f1e6152a2 100644 --- a/tensorflow/python/kernel_tests/clip_ops_test.py +++ b/tensorflow/python/kernel_tests/clip_ops_test.py @@ -24,10 +24,12 @@ from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes from tensorflow.python.framework import errors from tensorflow.python.framework import ops +from tensorflow.python.framework import test_util from tensorflow.python.ops import array_ops from tensorflow.python.ops import clip_ops from tensorflow.python.ops import gradient_checker from tensorflow.python.ops import gradients_impl +from tensorflow.python.ops import math_ops from tensorflow.python.platform import test @@ -55,7 +57,7 @@ class ClipTest(test.TestCase): np_ans = [[-4.4, 2.0, 3.0], [4.0, 4.4, 4.4]] clip_value = 4.4 ans = clip_ops.clip_by_value(x, -clip_value, clip_value) - tf_ans = ans.eval() + tf_ans = self.evaluate(ans) self.assertAllClose(np_ans, tf_ans) @@ -71,7 +73,7 @@ class ClipTest(test.TestCase): clip_value_min = 2 clip_value_max = 4 ans = clip_ops.clip_by_value(x, clip_value_min, clip_value_max) - tf_ans = ans.eval() + tf_ans = self.evaluate(ans) self.assertAllClose(np_ans, tf_ans) @@ -88,7 +90,7 @@ class ClipTest(test.TestCase): [2, 2, 2, 3, 3, 3], shape=[2, 3], dtype=dtype) clip_value_max = 4 ans = clip_ops.clip_by_value(x, clip_value_min, clip_value_max) - tf_ans = ans.eval() + tf_ans = self.evaluate(ans) self.assertAllClose(np_ans, tf_ans) @@ -105,7 +107,7 @@ class ClipTest(test.TestCase): clip_value_max = constant_op.constant( [6, 6, 6, 6, 6, 6], shape=[2, 3], dtype=dtype) ans = clip_ops.clip_by_value(x, clip_value_min, clip_value_max) - tf_ans = ans.eval() + tf_ans = self.evaluate(ans) self.assertAllClose(np_ans, tf_ans) @@ -123,7 +125,7 @@ class ClipTest(test.TestCase): clip_value_max = constant_op.constant( [5, 5, 5, 7, 7, 7], shape=[2, 3], dtype=dtype) ans = clip_ops.clip_by_value(x, clip_value_min, clip_value_max) - tf_ans = ans.eval() + tf_ans = self.evaluate(ans) self.assertAllClose(np_ans, tf_ans) @@ -144,7 +146,7 @@ class ClipTest(test.TestCase): np_ans = [float('NaN'), 4.0, -4.0] clip_value = 4.0 ans = clip_ops.clip_by_value(x, -clip_value, clip_value) - tf_ans = ans.eval() + tf_ans = self.evaluate(ans) self.assertAllClose(np_ans, tf_ans) @@ -157,14 +159,15 @@ class ClipTest(test.TestCase): np_ans = [[-2.4, 0.0, 0.0], [3.2, 0.0, 0.0]] clip_norm = 4.0 ans = clip_ops.clip_by_norm(x, clip_norm) - tf_ans = ans.eval() + tf_ans = self.evaluate(ans) ans = clip_ops.clip_by_norm(x, clip_norm) - tf_ans_tensor = ans.eval() + tf_ans_tensor = self.evaluate(ans) self.assertAllClose(np_ans, tf_ans) self.assertAllClose(np_ans, tf_ans_tensor) + @test_util.run_deprecated_v1 def testClipByNormGradientZeros(self): with self.session(use_gpu=True): x = array_ops.zeros([3]) @@ -188,7 +191,7 @@ class ClipTest(test.TestCase): np_ans = [[-3.0, 0.0, 0.0], [4.0, 0.0, 0.0]] clip_norm = 6.0 ans = clip_ops.clip_by_norm(x, clip_norm) - tf_ans = ans.eval() + tf_ans = self.evaluate(ans) self.assertAllClose(np_ans, tf_ans) @@ -200,7 +203,7 @@ class ClipTest(test.TestCase): np_ans = [[0.0, 0.0, 0.0], [0.0, 0.0, 0.0]] clip_norm = 6.0 ans = clip_ops.clip_by_norm(x, clip_norm) - tf_ans = ans.eval() + tf_ans = self.evaluate(ans) self.assertAllClose(np_ans, tf_ans) @@ -212,7 +215,7 @@ class ClipTest(test.TestCase): np_ans = [[-2.4, 0.0, 0.0], [3.2, 0.0, 3.0]] clip_norm = 4.0 ans = clip_ops.clip_by_norm(x, clip_norm, [0]) - tf_ans = ans.eval() + tf_ans = self.evaluate(ans) self.assertAllClose(np_ans, tf_ans) @@ -224,7 +227,7 @@ class ClipTest(test.TestCase): np_ans = [[-3.0, 0.0, 0.0], [3.2, 0.0, 2.4]] clip_norm = 4.0 ans = clip_ops.clip_by_norm(x, clip_norm, [1]) - tf_ans = ans.eval() + tf_ans = self.evaluate(ans) self.assertAllClose(np_ans, tf_ans) @@ -236,11 +239,12 @@ class ClipTest(test.TestCase): np_ans = [[-3.0, 0.0, 0.0], [4.0, 0.0, 3.0]] clip_norm = 6.0 ans = clip_ops.clip_by_norm(x, clip_norm, [1]) - tf_ans = ans.eval() + tf_ans = self.evaluate(ans) self.assertAllClose(np_ans, tf_ans) # ClipByGlobalNorm tests + @test_util.run_deprecated_v1 def testClipByGlobalNormClipped(self): # Norm clipping when clip_norm < 5 with self.session(use_gpu=True): @@ -256,12 +260,13 @@ class ClipTest(test.TestCase): ans, norm = clip_ops.clip_by_global_norm((x0, x1), clip_norm) tf_ans_1 = ans[0].eval() tf_ans_2 = ans[1].eval() - tf_norm = norm.eval() + tf_norm = self.evaluate(norm) self.assertAllClose(tf_norm, 5.0) self.assertAllClose(np_ans_0, tf_ans_1) self.assertAllClose(np_ans_1, tf_ans_2) + @test_util.run_deprecated_v1 def testClipByGlobalNormClippedTensor(self): # Norm clipping when clip_norm < 5 with self.session(use_gpu=True): @@ -277,12 +282,13 @@ class ClipTest(test.TestCase): ans, norm = clip_ops.clip_by_global_norm((x0, x1), clip_norm) tf_ans_1 = ans[0].eval() tf_ans_2 = ans[1].eval() - tf_norm = norm.eval() + tf_norm = self.evaluate(norm) self.assertAllClose(tf_norm, 5.0) self.assertAllClose(np_ans_0, tf_ans_1) self.assertAllClose(np_ans_1, tf_ans_2) + @test_util.run_deprecated_v1 def testClipByGlobalNormSupportsNone(self): # Norm clipping when clip_norm < 5 with self.session(use_gpu=True): @@ -300,12 +306,13 @@ class ClipTest(test.TestCase): self.assertTrue(ans[3] is None) tf_ans_1 = ans[0].eval() tf_ans_2 = ans[2].eval() - tf_norm = norm.eval() + tf_norm = self.evaluate(norm) self.assertAllClose(tf_norm, 5.0) self.assertAllClose(np_ans_0, tf_ans_1) self.assertAllClose(np_ans_1, tf_ans_2) + @test_util.run_deprecated_v1 def testClipByGlobalNormWithIndexedSlicesClipped(self): # Norm clipping when clip_norm < 5 with self.session(use_gpu=True): @@ -322,7 +329,7 @@ class ClipTest(test.TestCase): ans, norm = clip_ops.clip_by_global_norm([x0, x1], clip_norm) tf_ans_1 = ans[0].eval() tf_ans_2 = ans[1].values.eval() - tf_norm = norm.eval() + tf_norm = self.evaluate(norm) self.assertAllClose(tf_norm, 5.0) self.assertAllClose(np_ans_0, tf_ans_1) @@ -339,6 +346,7 @@ class ClipTest(test.TestCase): self.assertEqual(dense_shape, slices.dense_shape) self.assertEqual(dense_shape, modified_slices.dense_shape) + @test_util.run_deprecated_v1 def testClipByGlobalNormNotClipped(self): # No norm clipping when clip_norm >= 5 with self.session(use_gpu=True): @@ -352,12 +360,13 @@ class ClipTest(test.TestCase): ans, norm = clip_ops.clip_by_global_norm([x0, x1], clip_norm) tf_ans_1 = ans[0].eval() tf_ans_2 = ans[1].eval() - tf_norm = norm.eval() + tf_norm = self.evaluate(norm) self.assertAllClose(tf_norm, 5.0) self.assertAllClose(np_ans_0, tf_ans_1) self.assertAllClose(np_ans_1, tf_ans_2) + @test_util.run_deprecated_v1 def testClipByGlobalNormZero(self): # No norm clipping when norm = 0 with self.session(use_gpu=True): @@ -371,12 +380,13 @@ class ClipTest(test.TestCase): ans, norm = clip_ops.clip_by_global_norm([x0, x1], clip_norm) tf_ans_1 = ans[0].eval() tf_ans_2 = ans[1].eval() - tf_norm = norm.eval() + tf_norm = self.evaluate(norm) self.assertAllClose(tf_norm, 0.0) self.assertAllClose(np_ans_0, tf_ans_1) self.assertAllClose(np_ans_1, tf_ans_2) + @test_util.run_deprecated_v1 def testClipByGlobalNormInf(self): with self.session(use_gpu=True): x0 = constant_op.constant([-2.0, 0.0, np.inf, 4.0, 0.0, 0.0], @@ -386,7 +396,7 @@ class ClipTest(test.TestCase): ans, norm = clip_ops.clip_by_global_norm([x0, x1], clip_norm) with self.assertRaisesRegexp(errors.InvalidArgumentError, "global norm"): - norm.eval() + self.evaluate(norm) with self.assertRaisesRegexp(errors.InvalidArgumentError, "global norm"): ans[0].eval() with self.assertRaisesRegexp(errors.InvalidArgumentError, "global norm"): @@ -400,7 +410,7 @@ class ClipTest(test.TestCase): np_ans = [[-2.88, 0.0, 0.0], [3.84, 0.0, 0.0]] clip_norm = 0.8 ans = clip_ops.clip_by_average_norm(x, clip_norm) - tf_ans = ans.eval() + tf_ans = self.evaluate(ans) self.assertAllClose(np_ans, tf_ans) @@ -412,7 +422,7 @@ class ClipTest(test.TestCase): np_ans = [[-2.88, 0.0, 0.0], [3.84, 0.0, 0.0]] clip_norm = constant_op.constant(0.8) ans = clip_ops.clip_by_average_norm(x, clip_norm) - tf_ans = ans.eval() + tf_ans = self.evaluate(ans) self.assertAllClose(np_ans, tf_ans) @@ -424,7 +434,7 @@ class ClipTest(test.TestCase): np_ans = [[-3.0, 0.0, 0.0], [4.0, 0.0, 0.0]] clip_norm = 0.9 ans = clip_ops.clip_by_average_norm(x, clip_norm) - tf_ans = ans.eval() + tf_ans = self.evaluate(ans) self.assertAllClose(np_ans, tf_ans) @@ -436,10 +446,26 @@ class ClipTest(test.TestCase): np_ans = [[0.0, 0.0, 0.0], [0.0, 0.0, 0.0]] clip_norm = 0.9 ans = clip_ops.clip_by_average_norm(x, clip_norm) - tf_ans = ans.eval() + tf_ans = self.evaluate(ans) self.assertAllClose(np_ans, tf_ans) + def testClipByAverageNormReplacedWithClipByNorm(self): + # Check clip_by_average_norm(t) is the same as + # clip_by_norm(t, clip_norm * tf.to_float(tf.size(t))) + with self.session(use_gpu=True): + x = constant_op.constant([-3.0, 0.0, 0.0, 4.0, 0.0, 0.0], shape=[2, 3]) + # Average norm of x = sqrt(3^2 + 4^2) / 6 = 0.83333333 + # expected answer [[-2.88, 0.0, 0.0], [3.84, 0.0, 0.0]] + clip_norm = constant_op.constant(0.8) + with_norm = clip_ops.clip_by_average_norm(x, clip_norm) + without_norm = clip_ops.clip_by_norm( + x, clip_norm * math_ops.to_float(array_ops.size(x))) + clip_by_average_norm_ans = self.evaluate(with_norm) + clip_by_norm_ans = self.evaluate(without_norm) + self.assertAllClose(clip_by_average_norm_ans, clip_by_norm_ans) + + @test_util.run_deprecated_v1 def testClipByValueEmptyTensor(self): # Test case for GitHub issue 19337 zero = array_ops.placeholder(dtype=dtypes.float32, shape=None) diff --git a/tensorflow/python/kernel_tests/compare_and_bitpack_op_test.py b/tensorflow/python/kernel_tests/compare_and_bitpack_op_test.py index f27a0fc4722..215ea97f36d 100644 --- a/tensorflow/python/kernel_tests/compare_and_bitpack_op_test.py +++ b/tensorflow/python/kernel_tests/compare_and_bitpack_op_test.py @@ -20,6 +20,7 @@ from __future__ import print_function import numpy as np +from tensorflow.python.framework import test_util from tensorflow.python.ops import math_ops from tensorflow.python.platform import test @@ -30,15 +31,15 @@ class CompareAndBitpackTest(test.TestCase): x, threshold, truth, expected_err_re=None): - with self.cached_session(use_gpu=True): + with test_util.use_gpu(): ans = math_ops.compare_and_bitpack(x, threshold) if expected_err_re is None: - tf_ans = ans.eval() + tf_ans = self.evaluate(ans) self.assertShapeEqual(truth, ans) self.assertAllEqual(tf_ans, truth) else: with self.assertRaisesOpError(expected_err_re): - ans.eval() + self.evaluate(ans) def _testBasic(self, dtype): rows = 371 diff --git a/tensorflow/python/kernel_tests/concat_op_test.py b/tensorflow/python/kernel_tests/concat_op_test.py index 92d09986e6c..474760a93ff 100644 --- a/tensorflow/python/kernel_tests/concat_op_test.py +++ b/tensorflow/python/kernel_tests/concat_op_test.py @@ -23,6 +23,7 @@ import numpy as np from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes from tensorflow.python.framework import errors_impl +from tensorflow.python.framework import test_util from tensorflow.python.ops import array_ops from tensorflow.python.ops import gen_array_ops from tensorflow.python.ops import gradient_checker @@ -34,6 +35,7 @@ from tensorflow.python.platform import test class ConcatOpTest(test.TestCase): + @test_util.run_deprecated_v1 def testHStack(self): with self.session(use_gpu=True): p1 = array_ops.placeholder(dtypes.float32, shape=[4, 4]) @@ -49,6 +51,7 @@ class ConcatOpTest(test.TestCase): self.assertAllEqual(result[:4, :], params[p1]) self.assertAllEqual(result[4:, :], params[p2]) + @test_util.run_deprecated_v1 def testVStack(self): with self.session(use_gpu=True): p1 = array_ops.placeholder(dtypes.float32, shape=[4, 4]) @@ -65,25 +68,25 @@ class ConcatOpTest(test.TestCase): self.assertAllEqual(result[:, 4:], params[p2]) def testInt32GPU(self): - with self.session(use_gpu=True): + with test_util.use_gpu(): p1 = np.random.rand(2, 3).astype("i") p2 = np.random.rand(2, 3).astype("i") x1 = constant_op.constant(p1) x2 = constant_op.constant(p2) c = array_ops.concat([x1, x2], 0) - result = c.eval() + result = self.evaluate(c) self.assertAllEqual(result[:2, :], p1) self.assertAllEqual(result[2:, :], p2) def testRefType(self): - with self.session(use_gpu=True): + with test_util.use_gpu(): p1 = np.random.rand(4, 4).astype("f") p2 = np.random.rand(4, 4).astype("f") v1 = variables.Variable(p1) v2 = variables.Variable(p2) c = array_ops.concat([v1, v2], 0) - variables.global_variables_initializer().run() - result = c.eval() + self.evaluate(variables.global_variables_initializer()) + result = self.evaluate(c) self.assertEqual(result.shape, c.get_shape()) self.assertAllEqual(result[:4, :], p1) @@ -137,6 +140,7 @@ class ConcatOpTest(test.TestCase): else: self.assertAllClose(result[ind], params[p[i]], 0.01) + @test_util.run_deprecated_v1 def testRandom(self): self._testRandom(dtypes.bool) self._testRandom(dtypes.float32) @@ -147,6 +151,7 @@ class ConcatOpTest(test.TestCase): self._testRandom(dtypes.complex64) self._testRandom(dtypes.complex128) + @test_util.run_deprecated_v1 def testInvalidConcatDimTypeAndShape(self): a = variables.Variable(constant_op.constant(1.0, shape=[1])) b = variables.Variable(constant_op.constant(2.0, shape=[1])) @@ -172,7 +177,7 @@ class ConcatOpTest(test.TestCase): # Test both positive and negative concat axis. # -2 and 1 correspond to the same axis for 3-dimensional tensors. for axis in [-2, 1]: - with self.cached_session(use_gpu=True): + with test_util.use_gpu(): inp = [] inp_tensors = [] for x in [1, 2, 6]: @@ -195,15 +200,17 @@ class ConcatOpTest(test.TestCase): grad_inp.flatten(), shape=output_shape) grad = gradients_impl.gradients([c], inp_tensors, [grad_tensor]) concated_grad = array_ops.concat(grad, axis) - result = concated_grad.eval() + result = self.evaluate(concated_grad) self.assertAllEqual(result, grad_inp) + @test_util.run_deprecated_v1 def testGradientsSimple(self): self._testGradientsSimple(dtypes.float32) self._testGradientsSimple(dtypes.complex64) + @test_util.run_deprecated_v1 def testGradientsFirstDim(self): - with self.session(use_gpu=True): + with test_util.use_gpu(): inp = [] inp_tensors = [] for x in [1, 2, 6]: @@ -222,15 +229,16 @@ class ConcatOpTest(test.TestCase): grad_inp.flatten(), shape=output_shape) grad = gradients_impl.gradients([c], inp_tensors, [grad_tensor]) concated_grad = array_ops.concat(grad, 0) - result = concated_grad.eval() + result = self.evaluate(concated_grad) self.assertAllEqual(result, grad_inp) + @test_util.run_deprecated_v1 def testGradientsLastDim(self): # Test both positive and negative concat axis. # -1 and 2 correspond to the same axis for 3-dimensional tensors. for axis in [-1, 2]: - with self.cached_session(use_gpu=True): + with test_util.use_gpu(): inp = [] inp_tensors = [] for x in [1, 2, 6]: @@ -249,7 +257,7 @@ class ConcatOpTest(test.TestCase): grad_inp.flatten(), shape=output_shape) grad = gradients_impl.gradients([c], inp_tensors, [grad_tensor]) concated_grad = array_ops.concat(grad, axis) - result = concated_grad.eval() + result = self.evaluate(concated_grad) self.assertAllEqual(result, grad_inp) @@ -261,7 +269,7 @@ class ConcatOpTest(test.TestCase): # Random dim to concat on concat_dim = np.random.randint(5) concat_dim_sizes = np.random.randint(1, 5, size=num_tensors) - with self.cached_session(use_gpu=True): + with test_util.use_gpu(): inp = [] inp_tensors = [] for x in concat_dim_sizes: @@ -279,14 +287,16 @@ class ConcatOpTest(test.TestCase): grad_tensor = constant_op.constant(grad_inp.flatten(), shape=output_shape) grad = gradients_impl.gradients([c], inp_tensors, [grad_tensor]) concated_grad = array_ops.concat(grad, concat_dim) - result = concated_grad.eval() + result = self.evaluate(concated_grad) self.assertAllEqual(result, grad_inp) + @test_util.run_deprecated_v1 def testGradientsRandom(self): for _ in range(5): self._RunAndVerifyGradientsRandom() + @test_util.run_deprecated_v1 def testGradientWithUnknownInputDim(self): with self.session(use_gpu=True): x = array_ops.placeholder(dtypes.float32) @@ -308,6 +318,7 @@ class ConcatOpTest(test.TestCase): self.assertAllEqual(result, grad_inp) + @test_util.run_deprecated_v1 def testShapeError(self): # Rank doesn't match. with self.assertRaises(ValueError): @@ -337,6 +348,7 @@ class ConcatOpTest(test.TestCase): constant_op.constant(20.0, shape=[4, 4, 4]) ], -4) + @test_util.run_deprecated_v1 def testShapeWithUnknownConcatDim(self): p1 = array_ops.placeholder(dtypes.float32) c1 = constant_op.constant(10.0, shape=[4, 4, 4, 4]) @@ -355,10 +367,11 @@ class ConcatOpTest(test.TestCase): with self.assertRaises(ValueError): array_ops.concat([p1, c1, p2, c3], dim) + @test_util.run_deprecated_v1 def testZeroSize(self): # Verify that concat doesn't crash and burn for zero size inputs np.random.seed(7) - with self.session(use_gpu=True) as sess: + with test_util.use_gpu(): for shape0 in (), (2,): axis = len(shape0) for shape1 in (), (3,): @@ -370,12 +383,13 @@ class ConcatOpTest(test.TestCase): # TODO(irving): Make tf.concat handle map, then drop list(). xs = list(map(constant_op.constant, [x0, x1])) c = array_ops.concat(xs, axis) - self.assertAllEqual(c.eval(), correct) + self.assertAllEqual(self.evaluate(c), correct) # Check gradients dc = np.random.randn(*c.get_shape().as_list()) - dxs = sess.run(gradients_impl.gradients(c, xs, dc)) + dxs = self.evaluate(gradients_impl.gradients(c, xs, dc)) self.assertAllEqual(dc, np.concatenate(dxs, axis=axis)) + @test_util.run_deprecated_v1 def testTensorConcatDim0Grad(self): x_shapes = [[20, 7, 3], [10, 7, 3], [14, 7, 3]] output_shape = [44, 7, 3] @@ -390,6 +404,7 @@ class ConcatOpTest(test.TestCase): output_shape) self.assertLess(err, 1e-11) + @test_util.run_deprecated_v1 def testTensorConcatDim1Grad(self): x_shapes = [[20, 7, 3], [20, 3, 3], [20, 1, 3]] output_shape = [20, 11, 3] @@ -404,6 +419,7 @@ class ConcatOpTest(test.TestCase): output_shape) self.assertLess(err, 1e-11) + @test_util.run_deprecated_v1 def testIndexedSlicesConcatDim0Grad(self): x_shapes = [[20, 7, 3], [10, 7, 3], [14, 7, 3]] output_shape = [4, 7, 3] @@ -419,6 +435,7 @@ class ConcatOpTest(test.TestCase): output_shape) self.assertLess(err, 1e-11) + @test_util.run_deprecated_v1 def testIndexedSlicesConcatDim1Grad(self): x_shapes = [[20, 7, 3], [20, 3, 3], [20, 1, 3]] output_shape = [4, 11, 3] @@ -434,6 +451,7 @@ class ConcatOpTest(test.TestCase): output_shape) self.assertLess(err, 1e-11) + @test_util.run_deprecated_v1 def testIndexedSlicesConcatDim2Grad(self): x_shapes = [[20, 7, 3], [20, 7, 1], [20, 7, 2]] output_shape = [4, 7, 6] @@ -449,6 +467,7 @@ class ConcatOpTest(test.TestCase): output_shape) self.assertLess(err, 1e-11) + @test_util.run_deprecated_v1 def testIndexedSlicesConcatDim1Grad_UnknownInputDim(self): x_shapes = [[20, 7, 3], [20, 3, 3], [20, 1, 3]] output_shape = [4, 11, 3] @@ -473,21 +492,22 @@ class ConcatOpTest(test.TestCase): def testConcatTuple(self): c1 = np.random.rand(4, 4) c2 = np.random.rand(4, 4) - with self.cached_session(): - concat_list_t = array_ops.concat([c1, c2], 0) - concat_tuple_t = array_ops.concat((c1, c2), 0) - self.assertAllEqual(concat_list_t.eval(), concat_tuple_t.eval()) + concat_list_t = array_ops.concat([c1, c2], 0) + concat_tuple_t = array_ops.concat((c1, c2), 0) + self.assertAllEqual( + self.evaluate(concat_list_t), self.evaluate(concat_tuple_t)) + @test_util.run_deprecated_v1 def testConcatNoScalars(self): - with self.cached_session(): - scalar = constant_op.constant(7) - dim = array_ops.placeholder(dtypes.int32) - with self.assertRaisesRegexp( - ValueError, r"Can't concatenate scalars \(use tf\.stack instead\)"): - array_ops.concat([scalar, scalar, scalar], dim) + scalar = constant_op.constant(7) + dim = array_ops.placeholder(dtypes.int32) + with self.assertRaisesRegexp( + ValueError, r"Can't concatenate scalars \(use tf\.stack instead\)"): + array_ops.concat([scalar, scalar, scalar], dim) # important as gpu implementation could fail if # shared memory is not large for all the inputs + @test_util.run_deprecated_v1 def testConcatLargeNumberOfTensors(self): with self.session(use_gpu=True): for concat_dim in range(2): @@ -523,33 +543,34 @@ class ConcatOpTest(test.TestCase): self.assertAllEqual(result[index], params[p[i]]) def testConcatEmpty(self): - with self.session(use_gpu=True): + with test_util.use_gpu(): t1 = [] t2 = [] - output = gen_array_ops.concat_v2([t1, t2], 0).eval() - self.assertFalse(output) # Checks that output is empty + output = gen_array_ops.concat_v2([t1, t2], 0) + self.assertFalse(self.evaluate(output)) # Checks that output is empty + @test_util.run_deprecated_v1 def testConcatInvalidAxis(self): with self.assertRaises(ValueError): - with self.session(use_gpu=True): + with test_util.use_gpu(): t1 = [1] t2 = [2] gen_array_ops.concat_v2([t1, t2], 1).eval() def testConcatNegativeAxis(self): - with self.session(use_gpu=True): + with test_util.use_gpu(): t1 = [[1, 2, 3], [4, 5, 6]] t2 = [[7, 8, 9], [10, 11, 12]] c = gen_array_ops.concat_v2([t1, t2], -2) self.assertEqual([4, 3], c.get_shape().as_list()) - output = c.eval() + output = self.evaluate(c) self.assertAllEqual([[1, 2, 3], [4, 5, 6], [7, 8, 9], [10, 11, 12]], output) c = gen_array_ops.concat_v2([t1, t2], -1) self.assertEqual([2, 6], c.get_shape().as_list()) - output = c.eval() + output = self.evaluate(c) self.assertAllEqual([[1, 2, 3, 7, 8, 9], [4, 5, 6, 10, 11, 12]], output) def _testGradientsForAxis( @@ -578,6 +599,7 @@ class ConcatOpTest(test.TestCase): result = concated_grad.eval(feed_dict=feed_dict) self.assertAllEqual(result, grad_inp) + @test_util.run_deprecated_v1 def testGradientsNegativeAxis(self): x1 = [[1.0, 2.0, 3.0], [4.0, 5.0, 6.0]] x2 = [[7.0, 8.0, 9.0], [10.0, 11.0, 12.0]] @@ -608,78 +630,78 @@ class ConcatOpTest(test.TestCase): def testConcatAxisType(self): for dtype in [dtypes.int32, dtypes.int64]: - with self.cached_session(use_gpu=True): + with test_util.use_gpu(): t1 = [[1, 2, 3], [4, 5, 6]] t2 = [[7, 8, 9], [10, 11, 12]] c = gen_array_ops.concat_v2([t1, t2], constant_op.constant(1, dtype=dtype)) self.assertEqual([2, 6], c.get_shape().as_list()) - output = c.eval() + output = self.evaluate(c) self.assertAllEqual([[1, 2, 3, 7, 8, 9], [4, 5, 6, 10, 11, 12]], output) class ConcatOffsetTest(test.TestCase): def testBasic(self): - with self.session(use_gpu=True) as sess: + with test_util.use_gpu(): cdim = constant_op.constant(1, dtypes.int32) s0 = constant_op.constant([2, 3, 5], dtypes.int32) s1 = constant_op.constant([2, 7, 5], dtypes.int32) s2 = constant_op.constant([2, 20, 5], dtypes.int32) off = gen_array_ops.concat_offset(cdim, [s0, s1, s2]) - ans = sess.run(off) + ans = self.evaluate(off) self.assertAllEqual(ans, [[0, 0, 0], [0, 3, 0], [0, 10, 0]]) + @test_util.run_deprecated_v1 def testNotVector(self): - with self.cached_session() as sess: - cdim = constant_op.constant(1, dtypes.int32) - s0 = constant_op.constant([[2, 3, 5]], dtypes.int32) - s1 = constant_op.constant([[2, 7, 5]], dtypes.int32) - off = gen_array_ops.concat_offset(cdim, [s0, s1]) - with self.assertRaisesRegexp(errors_impl.InvalidArgumentError, - r"should be a vector"): - sess.run(off) + cdim = constant_op.constant(1, dtypes.int32) + s0 = constant_op.constant([[2, 3, 5]], dtypes.int32) + s1 = constant_op.constant([[2, 7, 5]], dtypes.int32) + off = gen_array_ops.concat_offset(cdim, [s0, s1]) + with self.assertRaisesRegexp(errors_impl.InvalidArgumentError, + r"should be a vector"): + self.evaluate(off) + @test_util.run_deprecated_v1 def testConcatDimOutOfRange(self): - with self.cached_session() as sess: - cdim = constant_op.constant(4, dtypes.int32) - s0 = constant_op.constant([2, 3, 5], dtypes.int32) - s1 = constant_op.constant([2, 7, 5], dtypes.int32) - off = gen_array_ops.concat_offset(cdim, [s0, s1]) - with self.assertRaisesRegexp(errors_impl.InvalidArgumentError, - r"Concat dim is out of range: 4 vs. 3"): - sess.run(off) + cdim = constant_op.constant(4, dtypes.int32) + s0 = constant_op.constant([2, 3, 5], dtypes.int32) + s1 = constant_op.constant([2, 7, 5], dtypes.int32) + off = gen_array_ops.concat_offset(cdim, [s0, s1]) + with self.assertRaisesRegexp(errors_impl.InvalidArgumentError, + r"Concat dim is out of range: 4 vs. 3"): + self.evaluate(off) + @test_util.run_deprecated_v1 def testDimMismatch(self): - with self.cached_session() as sess: - cdim = constant_op.constant(1, dtypes.int32) - s0 = constant_op.constant([2, 3, 5], dtypes.int32) - s1 = constant_op.constant([2, 7, 5, 10], dtypes.int32) - off = gen_array_ops.concat_offset(cdim, [s0, s1]) - with self.assertRaisesRegexp(errors_impl.InvalidArgumentError, - r"should contain 3 elem"): - sess.run(off) + cdim = constant_op.constant(1, dtypes.int32) + s0 = constant_op.constant([2, 3, 5], dtypes.int32) + s1 = constant_op.constant([2, 7, 5, 10], dtypes.int32) + off = gen_array_ops.concat_offset(cdim, [s0, s1]) + with self.assertRaisesRegexp(errors_impl.InvalidArgumentError, + r"should contain 3 elem"): + self.evaluate(off) + @test_util.run_deprecated_v1 def testSizeMismatch(self): - with self.cached_session() as sess: - cdim = constant_op.constant(1, dtypes.int32) - s0 = constant_op.constant([2, 3, 5], dtypes.int32) - s1 = constant_op.constant([2, 7, 10], dtypes.int32) - off = gen_array_ops.concat_offset(cdim, [s0, s1]) - with self.assertRaisesRegexp( - errors_impl.InvalidArgumentError, - r"All dimensions except 1 must match. Input 1 has shape \[2 7 10\] " - r"and doesn't match input 0 with shape \[2 3 5\]."): - sess.run(off) + cdim = constant_op.constant(1, dtypes.int32) + s0 = constant_op.constant([2, 3, 5], dtypes.int32) + s1 = constant_op.constant([2, 7, 10], dtypes.int32) + off = gen_array_ops.concat_offset(cdim, [s0, s1]) + with self.assertRaisesRegexp( + errors_impl.InvalidArgumentError, + r"All dimensions except 1 must match. Input 1 has shape \[2 7 10\] " + r"and doesn't match input 0 with shape \[2 3 5\]."): + self.evaluate(off) def testNegativeDim(self): - with self.session(use_gpu=True) as sess: + with test_util.use_gpu(): cdim = constant_op.constant(-2, dtypes.int32) s0 = constant_op.constant([2, 3, 5], dtypes.int32) s1 = constant_op.constant([2, 7, 5], dtypes.int32) s2 = constant_op.constant([2, 20, 5], dtypes.int32) off = gen_array_ops.concat_offset(cdim, [s0, s1, s2]) - ans = sess.run(off) + ans = self.evaluate(off) self.assertAllEqual(ans, [[0, 0, 0], [0, 3, 0], [0, 10, 0]]) cdim = constant_op.constant(-3, dtypes.int32) @@ -687,7 +709,7 @@ class ConcatOffsetTest(test.TestCase): s1 = constant_op.constant([1, 3, 5], dtypes.int32) s2 = constant_op.constant([3, 3, 5], dtypes.int32) off = gen_array_ops.concat_offset(cdim, [s0, s1, s2]) - ans = sess.run(off) + ans = self.evaluate(off) self.assertAllEqual(ans, [[0, 0, 0], [2, 0, 0], [3, 0, 0]]) diff --git a/tensorflow/python/kernel_tests/cond_v2_test.py b/tensorflow/python/kernel_tests/cond_v2_test.py index b077b853edb..1f4b37ce2a4 100644 --- a/tensorflow/python/kernel_tests/cond_v2_test.py +++ b/tensorflow/python/kernel_tests/cond_v2_test.py @@ -20,6 +20,7 @@ from __future__ import division from __future__ import print_function from tensorflow.core.protobuf import config_pb2 +from tensorflow.python.eager import context from tensorflow.python.eager import function from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes @@ -32,6 +33,7 @@ from tensorflow.python.ops import control_flow_ops from tensorflow.python.ops import data_flow_ops from tensorflow.python.ops import gradients_impl from tensorflow.python.ops import math_ops +from tensorflow.python.ops import tensor_array_ops from tensorflow.python.ops import variables from tensorflow.python.platform import test from tensorflow.python.training import saver @@ -66,6 +68,7 @@ class CondV2Test(test.TestCase): self.assertEqual(expected_val, actual_val) self.assertEqual(expected_grad_val, actual_grad_val) + @test_util.run_deprecated_v1 def testBasic(self): x = constant_op.constant(1.0, name="x") y = constant_op.constant(2.0, name="y") @@ -80,6 +83,7 @@ class CondV2Test(test.TestCase): self._testCond(true_fn, false_fn, [x, y]) self._testCond(true_fn, false_fn, [y]) + @test_util.run_deprecated_v1 def testMultipleOutputs(self): x = constant_op.constant(1.0, name="x") y = constant_op.constant(3.0, name="y") @@ -94,6 +98,7 @@ class CondV2Test(test.TestCase): self._testCond(true_fn, false_fn, [x, y]) self._testCond(true_fn, false_fn, [y]) + @test_util.run_deprecated_v1 def testBasic2(self): x = constant_op.constant(1.0, name="x") y = constant_op.constant(2.0, name="y") @@ -108,6 +113,7 @@ class CondV2Test(test.TestCase): self._testCond(true_fn, false_fn, [x, y]) self._testCond(true_fn, false_fn, [y]) + @test_util.run_deprecated_v1 def testNoInputs(self): with self.cached_session() as sess: pred = array_ops.placeholder(dtypes.bool, name="pred") @@ -124,7 +130,7 @@ class CondV2Test(test.TestCase): self.assertEqual(sess.run(out, {pred: False}), (2.0,)) def _createCond(self, name): - """Helper function for testDefaultName.""" + """Creates a cond_v2 call and returns the output tensor and the cond op.""" pred = constant_op.constant(True, name="pred") x = constant_op.constant(1.0, name="x") @@ -137,11 +143,11 @@ class CondV2Test(test.TestCase): output = cond_v2.cond_v2(pred, true_fn, false_fn, name=name) cond_op = output.op.inputs[0].op self.assertEqual(cond_op.type, "If") - return cond_op + return output, cond_op def testDefaultName(self): with ops.Graph().as_default(): - cond_op = self._createCond(None) + _, cond_op = self._createCond(None) self.assertEqual(cond_op.name, "cond") self.assertRegexpMatches( cond_op.get_attr("then_branch").name, r"cond_true_\d*") @@ -150,14 +156,14 @@ class CondV2Test(test.TestCase): with ops.Graph().as_default(): with ops.name_scope("foo"): - cond1_op = self._createCond("") + _, cond1_op = self._createCond("") self.assertEqual(cond1_op.name, "foo/cond") self.assertRegexpMatches( cond1_op.get_attr("then_branch").name, r"foo_cond_true_\d*") self.assertRegexpMatches( cond1_op.get_attr("else_branch").name, r"foo_cond_false_\d*") - cond2_op = self._createCond(None) + _, cond2_op = self._createCond(None) self.assertEqual(cond2_op.name, "foo/cond_1") self.assertRegexpMatches( cond2_op.get_attr("then_branch").name, r"foo_cond_1_true_\d*") @@ -538,6 +544,7 @@ class CondV2Test(test.TestCase): pred_inner: False }), [5., 0.]) + @test_util.run_deprecated_v1 def testSecondDerivative(self): with self.cached_session() as sess: pred = array_ops.placeholder(dtypes.bool, name="pred") @@ -610,11 +617,11 @@ class CondV2Test(test.TestCase): def testLowering(self): with ops.Graph().as_default() as g: with self.session(graph=g) as sess: - out_cond = self._createCond("cond") + cond_output, _ = self._createCond("cond") run_options = config_pb2.RunOptions(output_partition_graphs=True) run_metadata = config_pb2.RunMetadata() - sess.run(out_cond, options=run_options, run_metadata=run_metadata) + sess.run(cond_output, options=run_options, run_metadata=run_metadata) # If lowering was enabled, there should be a `Switch` node switch_found = any( @@ -634,17 +641,18 @@ class CondV2Test(test.TestCase): self.assertFalse(if_found, "An `If` op was found, but it should be lowered.") + @test_util.run_deprecated_v1 def testLoweringDisabledInXLA(self): with self.session(graph=ops.Graph()) as sess: # Build the cond_v2 in an XLA context xla_context = control_flow_ops.XLAControlFlowContext() xla_context.Enter() - out_cond = self._createCond("cond") + cond_output, _ = self._createCond("cond") xla_context.Exit() run_options = config_pb2.RunOptions(output_partition_graphs=True) run_metadata = config_pb2.RunMetadata() - sess.run(out_cond, options=run_options, run_metadata=run_metadata) + sess.run(cond_output, options=run_options, run_metadata=run_metadata) # Lowering disabled in XLA, there should be no `Switch` node switch_found = any( @@ -666,6 +674,110 @@ class CondV2Test(test.TestCase): if_found, "An `If` op was not found, but the graph should not be lowered.") + @test_util.run_deprecated_v1 + def testLoweringDisabledWithSingleThreadedExecutorContext(self): + with self.session(graph=ops.Graph()) as sess: + @function.defun + def _add_cond(x): + return cond_v2.cond_v2( + constant_op.constant(True, name="pred"), + lambda: x, + lambda: x + 1) + + x = array_ops.placeholder(shape=None, dtype=dtypes.float32) + with context.function_executor_type("SINGLE_THREADED_EXECUTOR"): + out_cond = _add_cond(x) + + # The fact that sess.run() succeeds means lowering is disabled, because + # the single threaded executor does not support cond v1 ops. + sess.run(out_cond, feed_dict={x: 1.0}) + + @test_util.enable_control_flow_v2 + def testStructuredOutputs(self): + x = constant_op.constant(1.0, name="x") + y = constant_op.constant(3.0, name="y") + + def true_fn(): + return ((x * y,), y) + + def false_fn(): + return ((x,), y * 3.0) + + output = control_flow_ops.cond( + constant_op.constant(False), true_fn, false_fn) + self.assertEqual(self.evaluate(output[0][0]), 1.) + self.assertEqual(self.evaluate(output[1]), 9.) + + @test_util.enable_control_flow_v2 + @test_util.run_deprecated_v1 + def testRaisesOutputStructuresMismatch(self): + x = constant_op.constant(1.0, name="x") + y = constant_op.constant(3.0, name="y") + + def true_fn(): + return x * y, y + + def false_fn(): + return ((x,), y * 3.0) + + with self.assertRaisesRegexp( + ValueError, "Outputs of true_fn and false_fn must" + " have the same structure"): + control_flow_ops.cond(constant_op.constant(False), true_fn, false_fn) + + @test_util.enable_control_flow_v2 + def testCondAndTensorArray(self): + x = math_ops.range(-5, 5) + output = tensor_array_ops.TensorArray(dtype=dtypes.int32, size=x.shape[0]) + + def loop_body(i, output): + + def if_true(): + return output.write(i, x[i]**2) + + def if_false(): + return output.write(i, x[i]) + + output = control_flow_ops.cond(x[i] > 0, if_true, if_false) + return i + 1, output + + _, output = control_flow_ops.while_loop( + lambda i, arr: i < x.shape[0], + loop_body, + loop_vars=(constant_op.constant(0), output)) + output_t = output.stack() + self.assertAllEqual( + self.evaluate(output_t), [-5, -4, -3, -2, -1, 0, 1, 4, 9, 16]) + + @test_util.enable_control_flow_v2 + def testCondAndTensorArrayInDefun(self): + + @function.defun + def f(): + x = math_ops.range(-5, 5) + output = tensor_array_ops.TensorArray(dtype=dtypes.int32, size=x.shape[0]) + + def loop_body(i, output): + + def if_true(): + return output.write(i, x[i]**2) + + def if_false(): + return output.write(i, x[i]) + + output = control_flow_ops.cond(x[i] > 0, if_true, if_false) + return i + 1, output + + _, output = control_flow_ops.while_loop( + lambda i, arr: i < x.shape[0], + loop_body, + loop_vars=(constant_op.constant(0), output)) + return output.stack() + + output_t = f() + self.assertAllEqual( + self.evaluate(output_t), [-5, -4, -3, -2, -1, 0, 1, 4, 9, 16]) + class CondV2CollectionTest(test.TestCase): diff --git a/tensorflow/python/kernel_tests/conditional_accumulator_test.py b/tensorflow/python/kernel_tests/conditional_accumulator_test.py index 97ab23fe49b..5847e4639bb 100644 --- a/tensorflow/python/kernel_tests/conditional_accumulator_test.py +++ b/tensorflow/python/kernel_tests/conditional_accumulator_test.py @@ -26,6 +26,7 @@ from tensorflow.python.framework import dtypes as dtypes_lib from tensorflow.python.framework import errors_impl from tensorflow.python.framework import ops from tensorflow.python.framework import tensor_shape +from tensorflow.python.framework import test_util from tensorflow.python.ops import array_ops from tensorflow.python.ops import data_flow_ops from tensorflow.python.ops import math_ops @@ -79,11 +80,13 @@ class ConditionalAccumulatorTest(test.TestCase): attr { key: 'reduction_type' value {s: 'MEAN'} } """, q.accumulator_ref.op.node_def) + @test_util.run_deprecated_v1 def testAccumulatorSizeEmpty(self): with self.cached_session(): q = data_flow_ops.ConditionalAccumulator(dtypes_lib.float32, name="Q") self.assertEqual(q.num_accumulated().eval(), 0) + @test_util.run_deprecated_v1 def testAccumulatorSetGlobalStep(self): with self.cached_session(): q = data_flow_ops.ConditionalAccumulator( @@ -91,6 +94,7 @@ class ConditionalAccumulatorTest(test.TestCase): set_global_step_op = q.set_global_step(1) set_global_step_op.run() + @test_util.run_deprecated_v1 def testAccumulatorApplyGradFloat32(self): with self.cached_session(): q = data_flow_ops.ConditionalAccumulator( @@ -98,6 +102,7 @@ class ConditionalAccumulatorTest(test.TestCase): accum_op = q.apply_grad((10.0,)) accum_op.run() + @test_util.run_deprecated_v1 def testDtypes(self): with self.cached_session() as sess: dtypes = [dtypes_lib.float16, dtypes_lib.float32, dtypes_lib.float64] @@ -111,10 +116,11 @@ class ConditionalAccumulatorTest(test.TestCase): for e in elems: q.apply_grad((e,)).run() - result = sess.run(q.take_grad(1)) + result = self.evaluate(q.take_grad(1)) self.assertEqual(sum(elems) / len(elems), result) + @test_util.run_deprecated_v1 def testAccumulatorMultipleAccumulators(self): with self.cached_session(): q_f32_0 = data_flow_ops.ConditionalAccumulator( @@ -134,6 +140,7 @@ class ConditionalAccumulatorTest(test.TestCase): result = accums[i].take_grad(1).eval() self.assertEqual(result, i + 10.0) + @test_util.run_deprecated_v1 def testAccumulatorApplyAndTakeGradWithShape(self): with self.cached_session(): q = data_flow_ops.ConditionalAccumulator( @@ -149,12 +156,13 @@ class ConditionalAccumulatorTest(test.TestCase): accum_op.run() is_all_equal = True - val = takeg_t.eval() + val = self.evaluate(takeg_t) for i in range(len(val)): for j in range(len(val[i])): is_all_equal &= (val[i][j] == elems_ave[i][j]) self.assertTrue(is_all_equal) + @test_util.run_deprecated_v1 def testAccumulatorApplyGradWithWrongShape(self): q = data_flow_ops.ConditionalAccumulator( dtypes_lib.float32, name="Q", shape=(3, 2)) @@ -165,6 +173,7 @@ class ConditionalAccumulatorTest(test.TestCase): with self.assertRaises(ValueError): q.apply_grad([[1.0], [2.0], [3.0]]) + @test_util.run_deprecated_v1 def testAccumulatorDynamicShape(self): with self.cached_session() as sess: q = data_flow_ops.ConditionalAccumulator( @@ -184,12 +193,13 @@ class ConditionalAccumulatorTest(test.TestCase): sess.run(accum_op, feed_dict={x: elem}) is_all_equal = True - val = takeg_t.eval() + val = self.evaluate(takeg_t) for i in range(len(val)): for j in range(len(val[i])): is_all_equal &= (val[i][j] == elems_ave[i][j]) self.assertTrue(is_all_equal) + @test_util.run_deprecated_v1 def testAccumulatorWrongDynamicShape(self): with self.cached_session() as sess: q = data_flow_ops.ConditionalAccumulator( @@ -208,6 +218,7 @@ class ConditionalAccumulatorTest(test.TestCase): with self.assertRaises(errors_impl.InvalidArgumentError): sess.run(accum_op, feed_dict={x: [[1.0], [2.0], [3.0]]}) + @test_util.run_deprecated_v1 def testAccumulatorSizeAfterApplyGrad(self): with self.cached_session(): q = data_flow_ops.ConditionalAccumulator( @@ -219,6 +230,7 @@ class ConditionalAccumulatorTest(test.TestCase): accum_op.run() self.assertEqual(q.num_accumulated().eval(), 2) + @test_util.run_deprecated_v1 def testAccumulatorSizeAfterApplyGradAndTakeGrad(self): with self.cached_session(): q = data_flow_ops.ConditionalAccumulator( @@ -247,6 +259,7 @@ class ConditionalAccumulatorTest(test.TestCase): extract_t.op.run() self.assertEqual(q.num_accumulated().eval(), 0) + @test_util.run_deprecated_v1 def testAccumulatorTakeGradMean(self): with self.cached_session(): q = data_flow_ops.ConditionalAccumulator( @@ -259,7 +272,7 @@ class ConditionalAccumulatorTest(test.TestCase): for accum_op in accum_ops: accum_op.run() - val = takeg_t.eval() + val = self.evaluate(takeg_t) self.assertEqual(15.0, val) accum_ops = [q.apply_grad((x,), local_step=1) for x in elems] @@ -268,9 +281,10 @@ class ConditionalAccumulatorTest(test.TestCase): for accum_op in accum_ops: accum_op.run() - val = takeg_t.eval() + val = self.evaluate(takeg_t) self.assertEqual(15.0, val) + @test_util.run_deprecated_v1 def testAccumulatorTakeGradSum(self): with self.cached_session(): q = data_flow_ops.ConditionalAccumulator( @@ -286,7 +300,7 @@ class ConditionalAccumulatorTest(test.TestCase): for accum_op in accum_ops: accum_op.run() - val = takeg_t.eval() + val = self.evaluate(takeg_t) self.assertEqual(30.0, val) accum_ops = [q.apply_grad((x,), local_step=1) for x in elems] @@ -295,9 +309,10 @@ class ConditionalAccumulatorTest(test.TestCase): for accum_op in accum_ops: accum_op.run() - val = takeg_t.eval() + val = self.evaluate(takeg_t) self.assertEqual(30.0, val) + @test_util.run_deprecated_v1 def testAccumulatorTakeGradInvalidReductionType(self): with self.assertRaises(ValueError): data_flow_ops.ConditionalAccumulator( @@ -306,6 +321,7 @@ class ConditionalAccumulatorTest(test.TestCase): shape=tensor_shape.TensorShape([1]), reduction_type="Invalid") + @test_util.run_deprecated_v1 def testAccumulatorInvalidTakeGrad(self): with self.cached_session(): q = data_flow_ops.ConditionalAccumulator( @@ -319,8 +335,9 @@ class ConditionalAccumulatorTest(test.TestCase): accum_op.run() with self.assertRaises(errors_impl.InvalidArgumentError): - takeg_t.eval() + self.evaluate(takeg_t) + @test_util.run_deprecated_v1 def testAccumulatorRepeatedTakeGradMean(self): with self.cached_session(): q = data_flow_ops.ConditionalAccumulator( @@ -334,7 +351,7 @@ class ConditionalAccumulatorTest(test.TestCase): for accum_op in accum_ops: accum_op.run() - val = takeg_t.eval() + val = self.evaluate(takeg_t) self.assertEqual(elems_ave, val) elems = [20.0, 30.0] @@ -345,9 +362,10 @@ class ConditionalAccumulatorTest(test.TestCase): for accum_op in accum_ops: accum_op.run() - val = takeg_t.eval() + val = self.evaluate(takeg_t) self.assertEqual(elems_ave + 0.0, val) + @test_util.run_deprecated_v1 def testAccumulatorRepeatedTakeGradSum(self): with self.cached_session(): q = data_flow_ops.ConditionalAccumulator( @@ -364,7 +382,7 @@ class ConditionalAccumulatorTest(test.TestCase): for accum_op in accum_ops: accum_op.run() - val = takeg_t.eval() + val = self.evaluate(takeg_t) self.assertEqual(elems_sum, val) elems = [20.0, 30.0] @@ -375,9 +393,10 @@ class ConditionalAccumulatorTest(test.TestCase): for accum_op in accum_ops: accum_op.run() - val = takeg_t.eval() + val = self.evaluate(takeg_t) self.assertEqual(elems_sum, val) + @test_util.run_deprecated_v1 def testAccumulatorIncrementGlobalStep(self): with self.cached_session(): q = data_flow_ops.ConditionalAccumulator( @@ -392,8 +411,9 @@ class ConditionalAccumulatorTest(test.TestCase): variables.global_variables_initializer().run() for _ in range(3): set_global_step_op.run() - inc_global_step.eval() + self.evaluate(inc_global_step) + @test_util.run_deprecated_v1 def testAccumulatorSetGlobalStepPreventsAccumulation(self): with self.cached_session(): q = data_flow_ops.ConditionalAccumulator( @@ -410,11 +430,12 @@ class ConditionalAccumulatorTest(test.TestCase): accum_op.run() takeg_t = q.take_grad(1) - val = takeg_t.eval() + val = self.evaluate(takeg_t) self.assertEqual(0.0 + sum(x for x in local_steps if x >= ls) / sum(1 for x in local_steps if x >= ls), val) + @test_util.run_deprecated_v1 def testParallelApplyGrad(self): with self.cached_session() as sess: q = data_flow_ops.ConditionalAccumulator( @@ -424,7 +445,7 @@ class ConditionalAccumulatorTest(test.TestCase): takeg_t = q.take_grad(1) def apply_grad(accum_op): - sess.run(accum_op) + self.evaluate(accum_op) threads = [ self.checkedThread( @@ -436,10 +457,11 @@ class ConditionalAccumulatorTest(test.TestCase): for thread in threads: thread.join() - val = takeg_t.eval() + val = self.evaluate(takeg_t) self.assertEqual(val, sum(elems) / len(elems)) + @test_util.run_deprecated_v1 def testParallelTakeGrad(self): with self.cached_session() as sess: q = data_flow_ops.ConditionalAccumulator( @@ -451,14 +473,14 @@ class ConditionalAccumulatorTest(test.TestCase): def apply_grad(): for accum_op in accum_ops: time.sleep(1.0) - sess.run(accum_op) + self.evaluate(accum_op) apply_grad_thread = self.checkedThread(target=apply_grad) results = [] def take_grad(): - results.append(sess.run(takeg_t)) + results.append(self.evaluate(takeg_t)) threads = [self.checkedThread(target=take_grad) for _ in range(10)] @@ -472,6 +494,7 @@ class ConditionalAccumulatorTest(test.TestCase): self.assertItemsEqual(elems, results) + @test_util.run_deprecated_v1 def testAccumulatorApplyAndBlockingTake(self): with self.cached_session() as sess: q = data_flow_ops.ConditionalAccumulator( @@ -485,12 +508,12 @@ class ConditionalAccumulatorTest(test.TestCase): def apply_grad(): time.sleep(1.0) for accum_op in accum_ops: - sess.run(accum_op) + self.evaluate(accum_op) return_array = [] def take_grad(): - return_array.append(sess.run(takeg_t)) + return_array.append(self.evaluate(takeg_t)) accum_thread = self.checkedThread(target=apply_grad) takeg_thread = self.checkedThread(target=take_grad) @@ -503,8 +526,9 @@ class ConditionalAccumulatorTest(test.TestCase): def _blocking_takeg(self, sess, takeg_op): with self.assertRaisesOpError("was cancelled"): - sess.run(takeg_op) + self.evaluate(takeg_op) + @test_util.run_deprecated_v1 def testAccumulatorCancel(self): with self.cached_session() as sess: q = data_flow_ops.ConditionalAccumulator( diff --git a/tensorflow/python/kernel_tests/confusion_matrix_test.py b/tensorflow/python/kernel_tests/confusion_matrix_test.py index bc24345261e..ae13c8e32e5 100644 --- a/tensorflow/python/kernel_tests/confusion_matrix_test.py +++ b/tensorflow/python/kernel_tests/confusion_matrix_test.py @@ -71,9 +71,11 @@ class ConfusionMatrixTest(test.TestCase): self._testConfMatrix(labels=labels, predictions=predictions, truth=truth) + @test_util.run_deprecated_v1 def testInt32Basic(self): self._testBasic(dtype=np.int32) + @test_util.run_deprecated_v1 def testInt64Basic(self): self._testBasic(dtype=np.int64) @@ -111,9 +113,11 @@ class ConfusionMatrixTest(test.TestCase): self.assertEqual(cm_out.dtype, np_dtype) self.assertAllClose(cm_out, truth, atol=1e-10) + @test_util.run_deprecated_v1 def testOnTensors_int32(self): self._testConfMatrixOnTensors(dtypes.int32, np.int32) + @test_util.run_deprecated_v1 def testOnTensors_int64(self): self._testConfMatrixOnTensors(dtypes.int64, np.int64) @@ -133,9 +137,11 @@ class ConfusionMatrixTest(test.TestCase): self._testConfMatrix(labels=labels, predictions=predictions, truth=truth) + @test_util.run_deprecated_v1 def testInt32DifferentLabels(self, dtype=np.int32): self._testDifferentLabelsInPredictionAndTarget(dtype) + @test_util.run_deprecated_v1 def testInt64DifferentLabels(self, dtype=np.int64): self._testDifferentLabelsInPredictionAndTarget(dtype) @@ -155,12 +161,15 @@ class ConfusionMatrixTest(test.TestCase): self._testConfMatrix(labels=labels, predictions=predictions, truth=truth) + @test_util.run_deprecated_v1 def testInt32MultipleLabels(self, dtype=np.int32): self._testMultipleLabels(dtype) + @test_util.run_deprecated_v1 def testInt64MultipleLabels(self, dtype=np.int64): self._testMultipleLabels(dtype) + @test_util.run_deprecated_v1 def testWeighted(self): labels = np.arange(5, dtype=np.int32) predictions = np.arange(5, dtype=np.int32) @@ -177,6 +186,7 @@ class ConfusionMatrixTest(test.TestCase): self._testConfMatrix( labels=labels, predictions=predictions, weights=weights, truth=truth) + @test_util.run_deprecated_v1 def testLabelsTooLarge(self): labels = np.asarray([1, 1, 0, 3, 5], dtype=np.int32) predictions = np.asarray([2, 1, 0, 2, 2], dtype=np.int32) @@ -191,6 +201,7 @@ class ConfusionMatrixTest(test.TestCase): self._testConfMatrix( labels=labels, predictions=predictions, num_classes=3, truth=None) + @test_util.run_deprecated_v1 def testPredictionsTooLarge(self): labels = np.asarray([1, 1, 0, 2, 2], dtype=np.int32) predictions = np.asarray([2, 1, 0, 3, 5], dtype=np.int32) @@ -205,6 +216,7 @@ class ConfusionMatrixTest(test.TestCase): self._testConfMatrix( labels=labels, predictions=predictions, num_classes=3, truth=None) + @test_util.run_deprecated_v1 def testInvalidRank_predictionsTooBig(self): labels = np.asarray([1, 2, 3]) predictions = np.asarray([[1, 2, 3]]) @@ -212,6 +224,7 @@ class ConfusionMatrixTest(test.TestCase): confusion_matrix.confusion_matrix, predictions, labels) + @test_util.run_deprecated_v1 def testInvalidRank_predictionsTooSmall(self): labels = np.asarray([[1, 2, 3]]) predictions = np.asarray([1, 2, 3]) @@ -219,6 +232,7 @@ class ConfusionMatrixTest(test.TestCase): confusion_matrix.confusion_matrix, predictions, labels) + @test_util.run_deprecated_v1 def testInputDifferentSize(self): labels = np.asarray([1, 2]) predictions = np.asarray([1, 2, 3]) @@ -232,7 +246,7 @@ class ConfusionMatrixTest(test.TestCase): with self.cached_session(): cm = confusion_matrix.confusion_matrix( labels, predictions, dtype=dtypes.int32) - tf_cm = cm.eval() + tf_cm = self.evaluate(cm) self.assertEqual(tf_cm.dtype, np.int32) def testOutputIsInt64(self): @@ -241,12 +255,13 @@ class ConfusionMatrixTest(test.TestCase): with self.cached_session(): cm = confusion_matrix.confusion_matrix( labels, predictions, dtype=dtypes.int64) - tf_cm = cm.eval() + tf_cm = self.evaluate(cm) self.assertEqual(tf_cm.dtype, np.int64) class RemoveSqueezableDimensionsTest(test.TestCase): + @test_util.run_deprecated_v1 def testBothScalarShape(self): label_values = 1.0 prediction_values = 0.0 @@ -261,8 +276,8 @@ class RemoveSqueezableDimensionsTest(test.TestCase): labels_placeholder, predictions_placeholder)) with self.cached_session(): - self.assertAllEqual(label_values, static_labels.eval()) - self.assertAllEqual(prediction_values, static_predictions.eval()) + self.assertAllEqual(label_values, self.evaluate(static_labels)) + self.assertAllEqual(prediction_values, self.evaluate(static_predictions)) feed_dict = { labels_placeholder: label_values, predictions_placeholder: prediction_values @@ -272,6 +287,7 @@ class RemoveSqueezableDimensionsTest(test.TestCase): self.assertAllEqual( prediction_values, dynamic_predictions.eval(feed_dict=feed_dict)) + @test_util.run_deprecated_v1 def testSameShape(self): label_values = np.ones(shape=(2, 3, 1)) prediction_values = np.zeros_like(label_values) @@ -286,8 +302,8 @@ class RemoveSqueezableDimensionsTest(test.TestCase): labels_placeholder, predictions_placeholder)) with self.cached_session(): - self.assertAllEqual(label_values, static_labels.eval()) - self.assertAllEqual(prediction_values, static_predictions.eval()) + self.assertAllEqual(label_values, self.evaluate(static_labels)) + self.assertAllEqual(prediction_values, self.evaluate(static_predictions)) feed_dict = { labels_placeholder: label_values, predictions_placeholder: prediction_values @@ -297,6 +313,7 @@ class RemoveSqueezableDimensionsTest(test.TestCase): self.assertAllEqual( prediction_values, dynamic_predictions.eval(feed_dict=feed_dict)) + @test_util.run_deprecated_v1 def testSameShapeExpectedRankDiff0(self): label_values = np.ones(shape=(2, 3, 1)) prediction_values = np.zeros_like(label_values) @@ -311,8 +328,8 @@ class RemoveSqueezableDimensionsTest(test.TestCase): labels_placeholder, predictions_placeholder, expected_rank_diff=0)) with self.cached_session(): - self.assertAllEqual(label_values, static_labels.eval()) - self.assertAllEqual(prediction_values, static_predictions.eval()) + self.assertAllEqual(label_values, self.evaluate(static_labels)) + self.assertAllEqual(prediction_values, self.evaluate(static_predictions)) feed_dict = { labels_placeholder: label_values, predictions_placeholder: prediction_values @@ -322,6 +339,7 @@ class RemoveSqueezableDimensionsTest(test.TestCase): self.assertAllEqual( prediction_values, dynamic_predictions.eval(feed_dict=feed_dict)) + @test_util.run_deprecated_v1 def testSqueezableLabels(self): label_values = np.ones(shape=(2, 3, 1)) prediction_values = np.zeros(shape=(2, 3)) @@ -337,8 +355,8 @@ class RemoveSqueezableDimensionsTest(test.TestCase): expected_label_values = np.reshape(label_values, newshape=(2, 3)) with self.cached_session(): - self.assertAllEqual(expected_label_values, static_labels.eval()) - self.assertAllEqual(prediction_values, static_predictions.eval()) + self.assertAllEqual(expected_label_values, self.evaluate(static_labels)) + self.assertAllEqual(prediction_values, self.evaluate(static_predictions)) feed_dict = { labels_placeholder: label_values, predictions_placeholder: prediction_values @@ -348,6 +366,7 @@ class RemoveSqueezableDimensionsTest(test.TestCase): self.assertAllEqual( prediction_values, dynamic_predictions.eval(feed_dict=feed_dict)) + @test_util.run_deprecated_v1 def testSqueezableLabelsExpectedRankDiffPlus1(self): label_values = np.ones(shape=(2, 3, 1)) prediction_values = np.zeros(shape=(2, 3, 5)) @@ -363,8 +382,8 @@ class RemoveSqueezableDimensionsTest(test.TestCase): expected_label_values = np.reshape(label_values, newshape=(2, 3)) with self.cached_session(): - self.assertAllEqual(expected_label_values, static_labels.eval()) - self.assertAllEqual(prediction_values, static_predictions.eval()) + self.assertAllEqual(expected_label_values, self.evaluate(static_labels)) + self.assertAllEqual(prediction_values, self.evaluate(static_predictions)) feed_dict = { labels_placeholder: label_values, predictions_placeholder: prediction_values @@ -374,6 +393,7 @@ class RemoveSqueezableDimensionsTest(test.TestCase): self.assertAllEqual( prediction_values, dynamic_predictions.eval(feed_dict=feed_dict)) + @test_util.run_deprecated_v1 def testSqueezablePredictions(self): label_values = np.ones(shape=(2, 3)) prediction_values = np.zeros(shape=(2, 3, 1)) @@ -389,8 +409,9 @@ class RemoveSqueezableDimensionsTest(test.TestCase): expected_prediction_values = np.reshape(prediction_values, newshape=(2, 3)) with self.cached_session(): - self.assertAllEqual(label_values, static_labels.eval()) - self.assertAllEqual(expected_prediction_values, static_predictions.eval()) + self.assertAllEqual(label_values, self.evaluate(static_labels)) + self.assertAllEqual(expected_prediction_values, + self.evaluate(static_predictions)) feed_dict = { labels_placeholder: label_values, predictions_placeholder: prediction_values @@ -401,6 +422,7 @@ class RemoveSqueezableDimensionsTest(test.TestCase): expected_prediction_values, dynamic_predictions.eval(feed_dict=feed_dict)) + @test_util.run_deprecated_v1 def testSqueezablePredictionsExpectedRankDiffMinus1(self): label_values = np.ones(shape=(2, 3, 5)) prediction_values = np.zeros(shape=(2, 3, 1)) @@ -416,8 +438,9 @@ class RemoveSqueezableDimensionsTest(test.TestCase): expected_prediction_values = np.reshape(prediction_values, newshape=(2, 3)) with self.cached_session(): - self.assertAllEqual(label_values, static_labels.eval()) - self.assertAllEqual(expected_prediction_values, static_predictions.eval()) + self.assertAllEqual(label_values, self.evaluate(static_labels)) + self.assertAllEqual(expected_prediction_values, + self.evaluate(static_predictions)) feed_dict = { labels_placeholder: label_values, predictions_placeholder: prediction_values @@ -428,6 +451,7 @@ class RemoveSqueezableDimensionsTest(test.TestCase): expected_prediction_values, dynamic_predictions.eval(feed_dict=feed_dict)) + @test_util.run_deprecated_v1 def testUnsqueezableLabels(self): label_values = np.ones(shape=(2, 3, 2)) prediction_values = np.zeros(shape=(2, 3)) @@ -453,6 +477,7 @@ class RemoveSqueezableDimensionsTest(test.TestCase): self.assertAllEqual( prediction_values, dynamic_predictions.eval(feed_dict=feed_dict)) + @test_util.run_deprecated_v1 def testUnsqueezablePredictions(self): label_values = np.ones(shape=(2, 3)) prediction_values = np.zeros(shape=(2, 3, 2)) diff --git a/tensorflow/python/kernel_tests/constant_op_test.py b/tensorflow/python/kernel_tests/constant_op_test.py index 38b8c0c146f..583082c2aa2 100644 --- a/tensorflow/python/kernel_tests/constant_op_test.py +++ b/tensorflow/python/kernel_tests/constant_op_test.py @@ -70,6 +70,7 @@ class ConstantTest(test.TestCase): with self.assertRaises(TypeError): constant_op.constant(dtypes_lib.string, "[,]") + @test_util.run_deprecated_v1 def testBFloat16(self): bfloat16 = dtypes_lib.bfloat16.as_numpy_dtype self._testAll(np.arange(-15, 15).reshape([2, 3, 5]).astype(bfloat16)) @@ -77,36 +78,42 @@ class ConstantTest(test.TestCase): np.random.normal(size=30).reshape([2, 3, 5]).astype(bfloat16)) self._testAll(np.empty((2, 0, 5)).astype(bfloat16)) + @test_util.run_deprecated_v1 def testHalf(self): self._testAll(np.arange(-15, 15).reshape([2, 3, 5]).astype(np.float16)) self._testAll( np.random.normal(size=30).reshape([2, 3, 5]).astype(np.float16)) self._testAll(np.empty((2, 0, 5)).astype(np.float16)) + @test_util.run_deprecated_v1 def testFloat(self): self._testAll(np.arange(-15, 15).reshape([2, 3, 5]).astype(np.float32)) self._testAll( np.random.normal(size=30).reshape([2, 3, 5]).astype(np.float32)) self._testAll(np.empty((2, 0, 5)).astype(np.float32)) + @test_util.run_deprecated_v1 def testDouble(self): self._testAll(np.arange(-15, 15).reshape([2, 3, 5]).astype(np.float64)) self._testAll( np.random.normal(size=30).reshape([2, 3, 5]).astype(np.float64)) self._testAll(np.empty((2, 0, 5)).astype(np.float64)) + @test_util.run_deprecated_v1 def testInt32(self): self._testAll(np.arange(-15, 15).reshape([2, 3, 5]).astype(np.int32)) self._testAll((100 * np.random.normal(size=30)).reshape([2, 3, 5]).astype( np.int32)) self._testAll(np.empty((2, 0, 5)).astype(np.int32)) + @test_util.run_deprecated_v1 def testInt64(self): self._testAll(np.arange(-15, 15).reshape([2, 3, 5]).astype(np.int64)) self._testAll((100 * np.random.normal(size=30)).reshape([2, 3, 5]).astype( np.int64)) self._testAll(np.empty((2, 0, 5)).astype(np.int64)) + @test_util.run_deprecated_v1 def testComplex64(self): self._testAll( np.complex(1, 2) * @@ -116,6 +123,7 @@ class ConstantTest(test.TestCase): np.random.normal(size=30).reshape([2, 3, 5]).astype(np.complex64)) self._testAll(np.empty((2, 0, 5)).astype(np.complex64)) + @test_util.run_deprecated_v1 def testComplex128(self): self._testAll( np.complex(1, 2) * @@ -125,12 +133,14 @@ class ConstantTest(test.TestCase): np.random.normal(size=30).reshape([2, 3, 5]).astype(np.complex128)) self._testAll(np.empty((2, 0, 5)).astype(np.complex128)) + @test_util.run_deprecated_v1 def testString(self): self._testCpu( np.array([compat.as_bytes(str(x)) for x in np.arange(-15, 15)]).reshape( [2, 3, 5])) self._testCpu(np.empty((2, 0, 5)).astype(np.str_)) + @test_util.run_deprecated_v1 def testVariant(self): # TODO(ebrevdo): Re-enable use_gpu=True once non-DMA Variant # copying between CPU and GPU is supported. @@ -161,6 +171,7 @@ class ConstantTest(test.TestCase): message="Variant storing an int, decoded const value:").op logging_const_op.run() + @test_util.run_deprecated_v1 def testStringWithNulls(self): with self.cached_session(): val = ops.convert_to_tensor(b"\0\0\0\0").eval() @@ -219,16 +230,28 @@ class ConstantTest(test.TestCase): def testShapeInconsistent(self): with ops.Graph().as_default(): - c = constant_op.constant([1, 2, 3, 4, 5, 6, 7], shape=[10]) + c = constant_op.constant_v1([1, 2, 3, 4, 5, 6, 7], shape=[10]) + self.assertEqual(c.get_shape(), [10]) + + with ops.Graph().as_default(): + with self.assertRaisesRegexp( + TypeError, "Expected Tensor's shape"): + c = constant_op.constant([1, 2, 3, 4, 5, 6, 7], shape=[10]) + + def testPromotionShapes(self): + with ops.Graph().as_default(): + c = constant_op.constant([7], shape=[10]) + self.assertEqual(c.get_shape(), [10]) + with ops.Graph().as_default(): + c = constant_op.constant(3, shape=[10]) self.assertEqual(c.get_shape(), [10]) # pylint: disable=g-long-lambda def testShapeWrong(self): with ops.Graph().as_default(): - with self.assertRaisesWithPredicateMatch( - ValueError, - lambda e: ("Too many elements provided. Needed at most 5, " - "but received 7" == str(e))): + with self.assertRaisesRegexp(ValueError, "Too many elements provided."): + constant_op.constant_v1([1, 2, 3, 4, 5, 6, 7], shape=[5]) + with self.assertRaisesRegexp(TypeError, "Expected Tensor's shape"): constant_op.constant([1, 2, 3, 4, 5, 6, 7], shape=[5]) # pylint: enable=g-long-lambda @@ -253,6 +276,7 @@ class ConstantTest(test.TestCase): "GraphDef cannot be larger than 2GB."): g.as_graph_def() + @test_util.run_deprecated_v1 def testSparseValuesRaiseErrors(self): with self.assertRaisesRegexp(ValueError, "setting an array element with a sequence"): @@ -282,29 +306,29 @@ class AsTensorTest(test.TestCase): with self.cached_session(): x = ops.convert_to_tensor(tensor_shape.TensorShape([])) self.assertEqual(dtypes_lib.int32, x.dtype) - self.assertAllEqual([], x.eval()) + self.assertAllEqual([], self.evaluate(x)) x = ops.convert_to_tensor(tensor_shape.TensorShape([1, 2, 3])) self.assertEqual(dtypes_lib.int32, x.dtype) - self.assertAllEqual([1, 2, 3], x.eval()) + self.assertAllEqual([1, 2, 3], self.evaluate(x)) x = ops.convert_to_tensor(tensor_shape.TensorShape([2**31-1, 2, 3])) self.assertEqual(dtypes_lib.int32, x.dtype) - self.assertAllEqual([2**31-1, 2, 3], x.eval()) + self.assertAllEqual([2**31 - 1, 2, 3], self.evaluate(x)) x = ops.convert_to_tensor(tensor_shape.TensorShape([2**31-1, 2, 3]), dtype=dtypes_lib.int32) self.assertEqual(dtypes_lib.int32, x.dtype) - self.assertAllEqual([2**31-1, 2, 3], x.eval()) + self.assertAllEqual([2**31 - 1, 2, 3], self.evaluate(x)) x = ops.convert_to_tensor(tensor_shape.TensorShape([2**31, 2, 3])) self.assertEqual(dtypes_lib.int64, x.dtype) - self.assertAllEqual([2**31, 2, 3], x.eval()) + self.assertAllEqual([2**31, 2, 3], self.evaluate(x)) x = ops.convert_to_tensor(tensor_shape.TensorShape([2**31, 2, 3]), dtype=dtypes_lib.int64) self.assertEqual(dtypes_lib.int64, x.dtype) - self.assertAllEqual([2**31, 2, 3], x.eval()) + self.assertAllEqual([2**31, 2, 3], self.evaluate(x)) with self.assertRaisesRegexp( ValueError, "a dimension is too large .2147483648."): @@ -314,11 +338,11 @@ class AsTensorTest(test.TestCase): x = ops.convert_to_tensor( tensor_shape.TensorShape([1, 2, 3]), dtype=dtypes_lib.int64) self.assertEqual(dtypes_lib.int64, x.dtype) - self.assertAllEqual([1, 2, 3], x.eval()) + self.assertAllEqual([1, 2, 3], self.evaluate(x)) x = array_ops.reshape( array_ops.zeros([6]), tensor_shape.TensorShape([2, 3])) - self.assertAllEqual([[0.0, 0.0, 0.0], [0.0, 0.0, 0.0]], x.eval()) + self.assertAllEqual([[0.0, 0.0, 0.0], [0.0, 0.0, 0.0]], self.evaluate(x)) with self.assertRaisesRegexp(ValueError, "partially known"): ops.convert_to_tensor(tensor_shape.TensorShape(None)) @@ -330,16 +354,17 @@ class AsTensorTest(test.TestCase): ops.convert_to_tensor( tensor_shape.TensorShape([1, 2, 3]), dtype=dtypes_lib.float32) + @test_util.run_deprecated_v1 def testAsTensorForDimensionInput(self): with self.cached_session(): x = ops.convert_to_tensor(tensor_shape.TensorShape([1, 2, 3])[1]) self.assertEqual(dtypes_lib.int32, x.dtype) - self.assertAllEqual(2, x.eval()) + self.assertAllEqual(2, self.evaluate(x)) x = ops.convert_to_tensor( tensor_shape.TensorShape([1, 2, 3])[1], dtype=dtypes_lib.int64) self.assertEqual(dtypes_lib.int64, x.dtype) - self.assertAllEqual(2, x.eval()) + self.assertAllEqual(2, self.evaluate(x)) shape = tensor_shape.TensorShape(None) if shape._v2_behavior: @@ -372,7 +397,7 @@ class ZerosTest(test.TestCase): with self.cached_session(): ret = array_ops.zeros(shape) self.assertEqual(shape, ret.get_shape()) - return ret.eval() + return self.evaluate(ret) def testConst(self): self.assertTrue( @@ -383,7 +408,7 @@ class ZerosTest(test.TestCase): self.assertEqual(0, self._Zeros(())) with self.cached_session(): scalar = array_ops.zeros(constant_op.constant([], dtype=dtypes_lib.int32)) - self.assertEqual(0, scalar.eval()) + self.assertEqual(0, self.evaluate(scalar)) def testDynamicSizes(self): np_ans = np.array([[0] * 3] * 2) @@ -392,11 +417,12 @@ class ZerosTest(test.TestCase): d = array_ops.fill([2, 3], 12., name="fill") # Constructs a tensor of zeros of the same dimensions as "d". z = array_ops.zeros(array_ops.shape(d)) - out = z.eval() + out = self.evaluate(z) self.assertAllEqual(np_ans, out) self.assertShapeEqual(np_ans, d) self.assertShapeEqual(np_ans, z) + @test_util.run_deprecated_v1 def testDtype(self): with self.cached_session(): d = array_ops.fill([2, 3], 12., name="fill") @@ -420,13 +446,13 @@ class ZerosTest(test.TestCase): z = array_ops.zeros([2, 3], dtype=dtype) self.assertEqual(z.dtype, dtype) self.assertEqual([2, 3], z.get_shape()) - z_value = z.eval() + z_value = self.evaluate(z) self.assertFalse(np.any(z_value)) self.assertEqual((2, 3), z_value.shape) z = array_ops.zeros(array_ops.shape(d), dtype=dtype) self.assertEqual(z.dtype, dtype) self.assertEqual([2, 3], z.get_shape()) - z_value = z.eval() + z_value = self.evaluate(z) self.assertFalse(np.any(z_value)) self.assertEqual((2, 3), z_value.shape) @@ -465,6 +491,7 @@ class ZerosLikeTest(test.TestCase): self.assertFalse(np.any(z_value)) self.assertEqual((2, 3), z_value.shape) + @test_util.run_deprecated_v1 def testZerosLikeCPU(self): for dtype in [ dtypes_lib.half, dtypes_lib.float32, dtypes_lib.float64, @@ -475,6 +502,7 @@ class ZerosLikeTest(test.TestCase): self._compareZeros(dtype, fully_defined_shape=False, use_gpu=False) self._compareZeros(dtype, fully_defined_shape=True, use_gpu=False) + @test_util.run_deprecated_v1 def testZerosLikeGPU(self): for dtype in [ dtypes_lib.half, dtypes_lib.float32, dtypes_lib.float64, @@ -484,11 +512,13 @@ class ZerosLikeTest(test.TestCase): self._compareZeros(dtype, fully_defined_shape=False, use_gpu=True) self._compareZeros(dtype, fully_defined_shape=True, use_gpu=True) + @test_util.run_deprecated_v1 def testZerosLikePartialShape(self): d = array_ops.placeholder(dtypes_lib.float32, shape=[None, 4, None]) z = array_ops.zeros_like(d) self.assertEqual(d.get_shape().as_list(), z.get_shape().as_list()) + @test_util.run_deprecated_v1 def testZerosLikeDtype(self): # Make sure zeros_like works even for dtypes that cannot be cast between with self.cached_session(): @@ -502,6 +532,7 @@ class ZerosLikeTest(test.TestCase): self.assertEqual(y.shape, shape) self.assertAllEqual(y, np.zeros(shape, dtype=out_type)) + @test_util.run_deprecated_v1 def testZerosLikeVariant(self): # TODO(ebrevdo): Re-enable use_gpu=True once non-DMA Variant # copying between CPU and GPU is supported AND we register a @@ -538,7 +569,7 @@ class OnesTest(test.TestCase): with self.cached_session(): ret = array_ops.ones(shape) self.assertEqual(shape, ret.get_shape()) - return ret.eval() + return self.evaluate(ret) def testConst(self): self.assertTrue(np.array_equal(self._Ones([2, 3]), np.array([[1] * 3] * 2))) @@ -548,7 +579,7 @@ class OnesTest(test.TestCase): self.assertEqual(1, self._Ones(())) with self.cached_session(): scalar = array_ops.ones(constant_op.constant([], dtype=dtypes_lib.int32)) - self.assertEqual(1, scalar.eval()) + self.assertEqual(1, self.evaluate(scalar)) def testDynamicSizes(self): np_ans = np.array([[1] * 3] * 2) @@ -557,11 +588,12 @@ class OnesTest(test.TestCase): d = array_ops.fill([2, 3], 12., name="fill") # Constructs a tensor of ones of the same dimensions as "d". z = array_ops.ones(array_ops.shape(d)) - out = z.eval() + out = self.evaluate(z) self.assertAllEqual(np_ans, out) self.assertShapeEqual(np_ans, d) self.assertShapeEqual(np_ans, z) + @test_util.run_deprecated_v1 def testAutoPack(self): with self.cached_session(): h = array_ops.placeholder(dtypes_lib.int32, shape=[]) @@ -570,6 +602,7 @@ class OnesTest(test.TestCase): out = z.eval(feed_dict={h: 4, w: 16}) self.assertAllEqual(out, np.array([[1] * 16] * 4)) + @test_util.run_deprecated_v1 def testDtype(self): with self.cached_session(): d = array_ops.fill([2, 3], 12., name="fill") @@ -617,12 +650,13 @@ class OnesLikeTest(test.TestCase): z_var = array_ops.ones_like(d) # Test that the type is correct self.assertEqual(z_var.dtype, dtype) - z_value = z_var.eval() + z_value = self.evaluate(z_var) # Test that the value is correct self.assertTrue(np.array_equal(z_value, np.array([[1] * 3] * 2))) self.assertEqual([2, 3], z_var.get_shape()) + @test_util.run_deprecated_v1 def testOnesLikePartialShape(self): d = array_ops.placeholder(dtypes_lib.float32, shape=[None, 4, None]) z = array_ops.ones_like(d) @@ -634,7 +668,7 @@ class FillTest(test.TestCase): def _compare(self, dims, val, np_ans, use_gpu): with self.cached_session(use_gpu=use_gpu): tf_ans = array_ops.fill(dims, val, name="fill") - out = tf_ans.eval() + out = self.evaluate(tf_ans) self.assertAllClose(np_ans, out) # Fill does not set the shape. # self.assertShapeEqual(np_ans, tf_ans) @@ -667,12 +701,14 @@ class FillTest(test.TestCase): np_ans = np.array([[0.15 + 0.3j] * 3] * 2).astype(np.complex128) self._compareAll([2, 3], np_ans[0][0], np_ans) + @test_util.run_deprecated_v1 def testFillString(self): np_ans = np.array([[b"yolo"] * 3] * 2) with self.session(use_gpu=False): tf_ans = array_ops.fill([2, 3], np_ans[0][0], name="fill").eval() self.assertAllEqual(np_ans, tf_ans) + @test_util.run_deprecated_v1 def testFillNegative(self): with self.cached_session(): for shape in (-1,), (2, -1), (-1, 2), (-2), (-3): @@ -686,6 +722,7 @@ class FillTest(test.TestCase): with self.assertRaises(errors_impl.InvalidArgumentError): fill_t.eval({dims: shape}) + @test_util.run_deprecated_v1 def testShapeFunctionEdgeCases(self): # Non-vector dimensions. with self.assertRaises(ValueError): @@ -704,6 +741,7 @@ class FillTest(test.TestCase): dtypes_lib.int32, shape=()), 17], 1.0) self.assertEqual([None, 17], f.get_shape().as_list()) + @test_util.run_deprecated_v1 def testGradient(self): with self.cached_session(): in_v = constant_op.constant(5.0) @@ -716,6 +754,7 @@ class FillTest(test.TestCase): class PlaceholderTest(test.TestCase): + @test_util.run_deprecated_v1 def testDtype(self): with self.cached_session(): p = array_ops.placeholder(dtypes_lib.float32, shape=(10, 10), name="p") @@ -726,8 +765,9 @@ class PlaceholderTest(test.TestCase): with self.assertRaisesOpError( "must feed a value for placeholder tensor 'p' with dtype float"): - p_identity.eval() + self.evaluate(p_identity) + @test_util.run_deprecated_v1 def testShape(self): with self.cached_session(): p = array_ops.placeholder(dtypes_lib.float32, shape=(10, 10), name="p") @@ -739,12 +779,13 @@ class PlaceholderTest(test.TestCase): with self.assertRaisesOpError( "must feed a value for placeholder tensor 'p' with dtype float and " r"shape \[10,10\]"): - p_identity.eval() + self.evaluate(p_identity) with self.assertRaisesWithPredicateMatch( ValueError, lambda e: "Cannot feed value of shape" in str(e)): p_identity.eval(feed_dict={p: feed_array[:5, :5]}) + @test_util.run_deprecated_v1 def testUnknownShape(self): with self.cached_session(): p = array_ops.placeholder(dtypes_lib.float32, shape=None, name="p") @@ -757,12 +798,14 @@ class PlaceholderTest(test.TestCase): self.assertAllClose( p_identity.eval(feed_dict={p: feed_array}), feed_array) + @test_util.run_deprecated_v1 def testScalarShape(self): with self.cached_session(): p = array_ops.placeholder(dtypes_lib.float32, shape=[], name="p") p_identity = array_ops.identity(p) self.assertAllClose(p_identity.eval(feed_dict={p: 5}), 5) + @test_util.run_deprecated_v1 def testPartialShape(self): with self.cached_session(): p = array_ops.placeholder(dtypes_lib.float32, shape=[None, 3], name="p") @@ -775,6 +818,7 @@ class PlaceholderTest(test.TestCase): ValueError, lambda e: "Cannot feed value of shape" in str(e)): p_identity.eval(feed_dict={p: feed_array[:5, :2]}) + @test_util.run_deprecated_v1 def testPartialShapeWhenNotFed(self): with self.cached_session(): p = array_ops.placeholder(dtypes_lib.float32, shape=[None, 3], name="p") @@ -783,8 +827,9 @@ class PlaceholderTest(test.TestCase): # Should trigger an operator error, not a shape error. with self.assertRaisesOpError( "must feed a value for placeholder tensor 'p' with dtype float"): - p_identity.eval() + self.evaluate(p_identity) + @test_util.run_deprecated_v1 def testControlDependency(self): with self.cached_session(): p = array_ops.placeholder(dtypes_lib.int32, shape=[], name="p") @@ -794,10 +839,12 @@ class PlaceholderTest(test.TestCase): val = np.array(2).astype(np.int) self.assertEqual(10, d.eval(feed_dict={p: val})) + @test_util.run_deprecated_v1 def testBadShape(self): with self.assertRaises(ValueError): array_ops.placeholder(dtypes_lib.float32, shape=(-1, 10)) + @test_util.run_deprecated_v1 def testTensorStr(self): a = array_ops.placeholder(dtypes_lib.float32, shape=None, name="a") self.assertEqual(" dtype=float32>", repr(a)) @@ -813,6 +860,7 @@ class PlaceholderTest(test.TestCase): self.assertEqual( "", repr(c)) + @test_util.run_deprecated_v1 def testOldGraph(self): # Load graph generated from earlier version of TF where # placeholder shape was not set. @@ -892,36 +940,40 @@ versions { class PlaceholderWithDefaultTest(test.TestCase): + @test_util.run_deprecated_v1 def testFullShape(self): with self.session(force_gpu=test_util.is_gpu_available()): p = array_ops.placeholder_with_default([[2, 2], [2, 2]], shape=[2, 2]) a = array_ops.identity(p) - self.assertAllEqual([[2, 2], [2, 2]], a.eval()) + self.assertAllEqual([[2, 2], [2, 2]], self.evaluate(a)) self.assertAllEqual( [[3, 3], [3, 3]], a.eval(feed_dict={p: [[3, 3], [3, 3]]})) with self.assertRaises(ValueError): a.eval(feed_dict={p: [[6, 6, 6], [6, 6, 6]]}) + @test_util.run_deprecated_v1 def testPartialShape(self): with self.session(force_gpu=test_util.is_gpu_available()): p = array_ops.placeholder_with_default([1, 2, 3], shape=[None]) a = array_ops.identity(p) - self.assertAllEqual([1, 2, 3], a.eval()) + self.assertAllEqual([1, 2, 3], self.evaluate(a)) self.assertAllEqual([3, 37], a.eval(feed_dict={p: [3, 37]})) with self.assertRaises(ValueError): a.eval(feed_dict={p: [[2, 2], [2, 2]]}) + @test_util.run_deprecated_v1 def testNoShape(self): with self.session(force_gpu=test_util.is_gpu_available()): p = array_ops.placeholder_with_default([17], shape=None) a = array_ops.identity(p) - self.assertAllEqual([17], a.eval()) + self.assertAllEqual([17], self.evaluate(a)) self.assertAllEqual([3, 37], a.eval(feed_dict={p: [3, 37]})) self.assertAllEqual( [[3, 3], [3, 3]], a.eval(feed_dict={p: [[3, 3], [3, 3]]})) + @test_util.run_deprecated_v1 def testGradient(self): with self.session(force_gpu=test_util.is_gpu_available()): x = array_ops.placeholder(dtypes_lib.float32, [5, 7]) diff --git a/tensorflow/python/kernel_tests/control_flow_ops_py_test.py b/tensorflow/python/kernel_tests/control_flow_ops_py_test.py index 19b067e4499..c80c95da846 100644 --- a/tensorflow/python/kernel_tests/control_flow_ops_py_test.py +++ b/tensorflow/python/kernel_tests/control_flow_ops_py_test.py @@ -139,8 +139,9 @@ class ControlFlowTest(test.TestCase): self.assertTrue(isinstance(v2, ops.Tensor)) variables.global_variables_initializer().run() - self.assertEqual(9, v2.eval()) + self.assertEqual(9, self.evaluate(v2)) + @test_util.run_deprecated_v1 def testRefEnter(self): with self.cached_session(): v = variables.VariableV1(7) @@ -152,8 +153,9 @@ class ControlFlowTest(test.TestCase): v2 = control_flow_ops.with_dependencies([op], enter_v) v3 = control_flow_ops.exit(v2) variables.global_variables_initializer().run() - self.assertEqual(9, v3.eval()) + self.assertEqual(9, self.evaluate(v3)) + @test_util.run_deprecated_v1 def testRefSwitch(self): with self.cached_session(): v = variables.VariableV1(7) @@ -162,7 +164,7 @@ class ControlFlowTest(test.TestCase): v1 = control_flow_ops._SwitchRefOrTensor(v._ref(), p) # pylint: disable=protected-access v2 = state_ops.assign(v1[1], 9) variables.global_variables_initializer().run() - self.assertEqual(9, v2.eval()) + self.assertEqual(9, self.evaluate(v2)) def testEnterMulExit(self): with self.cached_session(): @@ -173,9 +175,10 @@ class ControlFlowTest(test.TestCase): mul_op = math_ops.multiply(enter_data, enter_five) exit_op = control_flow_ops.exit(mul_op) - result = exit_op.eval() + result = self.evaluate(exit_op) self.assertAllEqual(np.array([x * 5 for x in [1, 2, 3, 4, 5, 6]]), result) + @test_util.run_deprecated_v1 def testEnterShapePropagation(self): with self.cached_session(): v = variables.Variable([0.0, 0.0], dtype=dtypes.float32) @@ -214,7 +217,7 @@ class ControlFlowTest(test.TestCase): with self.assertRaisesWithPredicateMatch( errors_impl.InvalidArgumentError, lambda e: "Retval[0] does not have value" in str(e)): - dead_branch.eval() + self.evaluate(dead_branch) def testSwitchMergeLess(self): with self.cached_session(): @@ -225,7 +228,7 @@ class ControlFlowTest(test.TestCase): switch_op = control_flow_ops.switch(data, less_op) merge_op = control_flow_ops.merge(switch_op)[0] - result = merge_op.eval() + result = self.evaluate(merge_op) self.assertAllEqual(np.arange(1, 7), result) def testSwitchMergeAddIdentity(self): @@ -238,7 +241,7 @@ class ControlFlowTest(test.TestCase): id_op = array_ops.identity(switch_op[1]) merge_op = control_flow_ops.merge([add_op, id_op])[0] - result = merge_op.eval() + result = self.evaluate(merge_op) self.assertAllEqual(np.array([x + 1 for x in [1, 2, 3, 4, 5, 6]]), result) def testSwitchMergeAddMul(self): @@ -252,7 +255,7 @@ class ControlFlowTest(test.TestCase): mul_op = math_ops.multiply(switch_op[1], five) merge_op = control_flow_ops.merge([add_op, mul_op])[0] - result = merge_op.eval() + result = self.evaluate(merge_op) self.assertAllEqual(np.array([x * 5 for x in [1, 2, 3, 4, 5, 6]]), result) def testLoop_false(self): @@ -269,9 +272,10 @@ class ControlFlowTest(test.TestCase): next_n = control_flow_ops.next_iteration(switch_n[0]) merge_n.op._update_input(1, next_n) - result = exit_n.eval() + result = self.evaluate(exit_n) self.assertAllEqual(10, result) + @test_util.run_deprecated_v1 def testLoop_1(self): with self.cached_session(): zero = constant_op.constant(0) @@ -295,7 +299,7 @@ class ControlFlowTest(test.TestCase): merge_i.op._update_input(1, next_i) exit_i = control_flow_ops.exit(switch_i[0]) - result = exit_i.eval() + result = self.evaluate(exit_i) self.assertAllEqual(10, result) def testLoop_2(self): @@ -321,7 +325,7 @@ class ControlFlowTest(test.TestCase): merge_i.op._update_input(1, next_i) exit_i = control_flow_ops.exit(switch_i[0]) - result = exit_i.eval() + result = self.evaluate(exit_i) self.assertAllEqual(10, result) def testDifferentFrame(self): @@ -333,6 +337,7 @@ class ControlFlowTest(test.TestCase): with self.assertRaisesOpError("has inputs from different frames"): res.eval(feed_dict={data: 1.0}) + @test_util.run_deprecated_v1 def testCondBool(self): values = constant_op.constant(10) fn1 = lambda: math_ops.add(values, 1) @@ -340,6 +345,7 @@ class ControlFlowTest(test.TestCase): with self.assertRaisesRegexp(TypeError, "must not be a Python bool"): _ = control_flow_ops.cond(False, fn1, fn2) + @test_util.run_deprecated_v1 def testCondInt(self): p = array_ops.placeholder(dtypes.bool, shape=[]) v = constant_op.constant(10) @@ -389,7 +395,6 @@ class ControlFlowTest(test.TestCase): with self.assertRaisesRegexp(ValueError, "may not be fed"): sess.run(r, feed_dict={t: 3}) - @test_util.disable_control_flow_v2("b/113296180 (IndexedSlices)") def testCondIndexedSlices(self): with self.cached_session(): values = constant_op.constant(10) @@ -405,7 +410,6 @@ class ControlFlowTest(test.TestCase): self.assertAllEqual(11, val) self.assertAllEqual(0, ind) - @test_util.disable_control_flow_v2("b/113296161 (SparseTensors)") def testCondSparseTensor(self): with self.cached_session(): values = constant_op.constant([2.0, 4.0], name="values") @@ -437,6 +441,19 @@ class ControlFlowTest(test.TestCase): self.assertEqual(1.0, control_flow_ops.cond(rv, case, lambda: t).eval()) + def testCondWithTensorArrayGrad(self): + with self.cached_session() as sess: + with ops.device(test.gpu_device_name()): + pred = array_ops.placeholder(dtypes.bool, []) + x = constant_op.constant([1.0, 2.0, 3.0]) + y = control_flow_ops.cond( + pred, lambda: functional_ops.map_fn(lambda z: z * 2.0, x), + lambda: constant_op.constant([1.0, 1.0, 1.0])) + g = gradients_impl.gradients(y, x)[0] + + self.assertAllEqual(sess.run(g, {pred: True}), [2.0, 2.0, 2.0]) + self.assertAllEqual(sess.run(g, {pred: False}), [0.0, 0.0, 0.0]) + @test_util.disable_control_flow_v2("b/113293074") def testCondIndexedSlicesDifferentTypes(self): with self.cached_session(): @@ -478,7 +495,7 @@ class ControlFlowTest(test.TestCase): fn2 = lambda: math_ops.subtract(x, 1) r = control_flow_ops.cond(pred, fn1, fn2) - result = r.eval() + result = self.evaluate(r) self.assertAllEqual(11, result) def testCond_1(self): @@ -494,7 +511,7 @@ class ControlFlowTest(test.TestCase): r = control_flow_ops.cond( math_ops.less(1, 0), lambda: math_ops.add(x, 1), lambda: math_ops.subtract(x, 1)) - result = r.eval() + result = self.evaluate(r) self.assertAllEqual(9, result) def testCond_3(self): @@ -507,7 +524,7 @@ class ControlFlowTest(test.TestCase): fn3 = lambda: math_ops.add(control_flow_ops.cond(pred, fn1, fn2), 1) r = control_flow_ops.cond(pred, fn3, fn2) - result = r.eval() + result = self.evaluate(r) self.assertAllEqual(12, result) @test_util.run_in_graph_and_eager_modes @@ -534,9 +551,9 @@ class ControlFlowTest(test.TestCase): result = f().eval() self.assertEqual(True, result) # Only second cond result was fetched, so v1 assign shouldn't run. - self.assertEqual(7, v1.eval()) - self.assertEqual(2, v2.eval()) - self.assertEqual(7, v3.eval()) + self.assertEqual(7, self.evaluate(v1)) + self.assertEqual(2, self.evaluate(v2)) + self.assertEqual(7, self.evaluate(v3)) result = f_defun() self.assertEqual(True, self.evaluate(result)) @@ -557,10 +574,9 @@ class ControlFlowTest(test.TestCase): for i in range(10): alive, count = body(i) - self.assertAllEqual(4, count.eval()) + self.assertAllEqual(4, self.evaluate(count)) def testCond_6(self): - with self.cached_session(): v1 = variables.Variable([7]) @@ -571,7 +587,7 @@ class ControlFlowTest(test.TestCase): r = control_flow_ops.cond(pred, fn1, fn2) variables.global_variables_initializer().run() - result = r.eval() + result = self.evaluate(r) self.assertAllEqual(np.array([7]), result) def testCond_7(self): @@ -582,8 +598,89 @@ class ControlFlowTest(test.TestCase): fn1 = lambda: [math_ops.add(x, 1), math_ops.add(x, 2)] fn2 = lambda: [y, y] r = control_flow_ops.cond(pred, fn1, fn2) - self.assertAllEqual([11, 12], sess.run(r)) + self.assertAllEqual([11, 12], self.evaluate(r)) + def testCondListOutput(self): + with self.cached_session() as sess: + x = constant_op.constant(10) + y = constant_op.constant(200) + pred = math_ops.less(1, 2) + fn1 = lambda: [math_ops.add(x, y), math_ops.add(x, y)] + fn2 = lambda: [y, y] + r = control_flow_ops.cond(pred, fn1, fn2) + test_result = self.evaluate(r) + self.assertListEqual([210, 210], test_result) + + def testTupleOutput(self): + with self.cached_session() as sess: + x = constant_op.constant(10) + y = constant_op.constant(200) + pred = math_ops.less(1, 2) + fn1 = lambda: (math_ops.add(x, y), math_ops.add(x, y)) + fn2 = lambda: (y, y) + r = control_flow_ops.cond(pred, fn1, fn2) + test_result = self.evaluate(r) + self.assertTupleEqual((210, 210), test_result) + + def testDictOutput(self): + with self.cached_session() as sess: + x = constant_op.constant(10) + y = constant_op.constant(200) + pred = math_ops.less(1, 2) + fn1 = lambda: {"a": math_ops.add(x, y), "b": math_ops.add(x, y)} + fn2 = lambda: {"a": y, "b": y} + r = control_flow_ops.cond(pred, fn1, fn2) + test_result = self.evaluate(r) + self.assertDictEqual({"a": 210, "b": 210}, test_result) + + @test_util.run_deprecated_v1 + def testEmbeddedListOutput(self): + with self.cached_session() as sess: + x = constant_op.constant(10) + y = constant_op.constant(200) + pred = math_ops.less(1, 2) + fn1 = lambda: [[math_ops.add(x, y), math_ops.add(x, y)]] + fn2 = lambda: [[y, y]] + # Pass strict=True flag as cond_v2 allows for tensors to be + # in nested output structures as singletons + r = control_flow_ops.cond(pred, fn1, fn2, strict=True) + test_result = self.evaluate(r) + self.assertListEqual([[210, 210]], test_result) + + def testEmbeddedTupleOutput(self): + with self.cached_session() as sess: + x = constant_op.constant(10) + y = constant_op.constant(200) + pred = math_ops.less(1, 2) + fn1 = lambda: ((math_ops.add(x, y), math_ops.add(x, y))) + fn2 = lambda: ((y, y)) + r = control_flow_ops.cond(pred, fn1, fn2) + test_result = self.evaluate(r) + self.assertTupleEqual(((210, 210)), test_result) + + def testEmbeddedDictOutput(self): + with self.cached_session() as sess: + x = constant_op.constant(10) + y = constant_op.constant(200) + pred = math_ops.less(1, 2) + fn1 = lambda: {"a": {"c": math_ops.add(x, y)}, "b": {"d": math_ops.add(x, y)}} + fn2 = lambda: {"a": {"c": y}, "b": {"d": y}} + r = control_flow_ops.cond(pred, fn1, fn2) + test_result = self.evaluate(r) + self.assertDictEqual({"a": {"c": 210}, "b": {"d": 210}}, test_result) + + def testCheckNestedOutputStruct(self): + with self.cached_session() as sess: + x = constant_op.constant(10) + y = constant_op.constant(200) + pred = math_ops.less(1, 2) + fn1 = lambda: {"a": math_ops.add(x, y), "b": math_ops.add(x, y)} + fn2 = lambda: {"c": y, "d": y} + with self.assertRaisesRegexp(ValueError, "The two structures don't have the same nested structure"): + r = control_flow_ops.cond(pred, fn1, fn2) + self.evaluate(r) + + @test_util.run_deprecated_v1 def testCondRef(self): with self.cached_session(): @@ -596,9 +693,10 @@ class ControlFlowTest(test.TestCase): true_fn = lambda: x false_fn = lambda: constant_op.constant([2.0]) r = control_flow_ops.cond(constant_op.constant(False), true_fn, false_fn) - self.assertAllEqual([2.0], r.eval()) + self.assertAllEqual([2.0], self.evaluate(r)) @test_util.disable_control_flow_v2("b/79881896 (control deps)") + @test_util.run_deprecated_v1 def testCondWithControl(self): with self.cached_session(): control_holder = array_ops.placeholder(dtypes.float32, shape=()) @@ -612,7 +710,7 @@ class ControlFlowTest(test.TestCase): r = control_flow_ops.cond( constant_op.constant(True), true_branch, lambda: constant_op.constant(1)) - self.assertEqual(5, r.eval()) + self.assertEqual(5, self.evaluate(r)) def testUninitializedRefIdentity(self): with self.cached_session() as sess: @@ -636,7 +734,7 @@ class ControlFlowTest(test.TestCase): with ops.control_dependencies([v_t_op]): orig_v = array_ops.identity(v) merged_op = control_flow_ops.merge([assign_v, orig_v]) - self.assertAllEqual([1.0], sess.run(merged_op.output)) + self.assertAllEqual([1.0], self.evaluate(merged_op.output)) def testCondSwitchIdentity(self): # Make sure the recv identity is not removed by optimization. @@ -650,7 +748,7 @@ class ControlFlowTest(test.TestCase): return control_flow_ops.Assert(False, ["Wrong branch!!!"]) r = control_flow_ops.cond(pred, fn1, fn2) - sess.run(r) + self.evaluate(r) def testCondRecvIdentity(self): # Make sure the switch identity is not removed by optimization. @@ -666,7 +764,7 @@ class ControlFlowTest(test.TestCase): return control_flow_ops.Assert(False, ["Wrong branch!!!"]) r = control_flow_ops.cond(pred, fn1, fn2) - sess.run(r) + self.evaluate(r) def testCondGrad_1(self): with self.cached_session(): @@ -677,8 +775,9 @@ class ControlFlowTest(test.TestCase): r = control_flow_ops.cond(pred, fn1, fn2) grad = gradients_impl.gradients(r, [x])[0] - self.assertAllEqual(1.0, grad.eval()) + self.assertAllEqual(1.0, self.evaluate(grad)) + @test_util.run_deprecated_v1 def testCondGrad_2(self): with self.cached_session(): c = array_ops.placeholder(dtypes.int32, shape=[]) @@ -694,6 +793,7 @@ class ControlFlowTest(test.TestCase): @test_util.disable_control_flow_v2( "b/110550782 (gradient w.r.t external variable)") + @test_util.run_deprecated_v1 def testCondGrad_3(self): with self.cached_session(): c = array_ops.placeholder(dtypes.int32, shape=[]) @@ -711,6 +811,35 @@ class ControlFlowTest(test.TestCase): self.assertAllEqual(980.0, r.eval(feed_dict={c: 1})) self.assertAllEqual(30.0, r.eval(feed_dict={c: 3})) + @test_util.run_deprecated_v1 + def testCondGradMultiDevice(self): + config = config_pb2.ConfigProto(device_count={"CPU": 2}, + allow_soft_placement=True) + with self.cached_session(use_gpu=True, config=config) as sess: + pred = array_ops.placeholder(dtypes.bool, []) + x = array_ops.placeholder(dtypes.float32) + y = array_ops.placeholder(dtypes.float32) + + with ops.device("/cpu:0"): + z = control_flow_ops.cond(pred, lambda: x * y * 2.0, lambda: 2.0) + + with ops.device("/cpu:1"): + grad = gradients_impl.gradients(z, x)[0] + + self.assertEqual(sess.run(grad, {pred: True, x: 1.0, y: 2.0}), 4.0) + self.assertEqual(sess.run(grad, {pred: False, x: 1.0, y: 2.0}), 0.0) + + with ops.device("/cpu:0"): + grad_grad = gradients_impl.gradients(grad, x)[0] + + # v1 control flow gets None second derivative for some reason. + if not control_flow_ops.ENABLE_COND_V2: + self.assertIsNone(grad_grad) + return + + self.assertEqual(sess.run(grad_grad, {pred: True, x: 1.0, y: 2.0}), 0.0) + self.assertEqual(sess.run(grad_grad, {pred: False, x: 1.0, y: 2.0}), 0.0) + def testNestedCond_Simple(self): with self.cached_session(): x = constant_op.constant(0., name="X") @@ -718,15 +847,16 @@ class ControlFlowTest(test.TestCase): constant_op.constant(True), lambda: x, lambda: control_flow_ops.cond(x < 1., lambda: x, lambda: x)) result = gradients_impl.gradients(y, x)[0] - self.assertEqual(1.0, result.eval()) + self.assertEqual(1.0, self.evaluate(result)) z = control_flow_ops.cond( constant_op.constant(False), lambda: x, lambda: control_flow_ops.cond(x < 1., lambda: x, lambda: x)) result = gradients_impl.gradients(z, x)[0] - self.assertEqual(1.0, result.eval()) + self.assertEqual(1.0, self.evaluate(result)) @test_util.disable_control_flow_v2("b/113327884") + @test_util.run_deprecated_v1 def testCondGrad_Gather(self): with self.cached_session() as sess: v1 = variables.Variable([1.0, 42.0]) @@ -740,16 +870,26 @@ class ControlFlowTest(test.TestCase): # Should just be [1, 1], but possibly a sparse representation gv, gi = sess.run([grad.values, grad.indices], feed_dict={c: 1}) dense_gv = [ - sum([y for (x, y) in zip(gi, gv) if x == i]) for i in range(2) + sum(y for (x, y) in zip(gi, gv) if x == i) for i in range(2) ] self.assertAllEqual(dense_gv, [1.0, 1.0]) # Should be [0, 2], as the else forwards v1[1] twice gv, gi = sess.run([grad.values, grad.indices], feed_dict={c: 3}) dense_gv = [ - sum([y for (x, y) in zip(gi, gv) if x == i]) for i in range(2) + sum(y for (x, y) in zip(gi, gv) if x == i) for i in range(2) ] self.assertAllEqual(dense_gv, [0.0, 2.0]) + def testCondPredicateTensor(self): + """Regression test for lowering predicate from non-first output of an op.""" + + @eager_function.defun + def foo(): + return constant_op.constant("foo"), constant_op.constant(True) + + r = control_flow_ops.cond(foo()[1], lambda: 1.0, lambda: 2.0) + self.assertEqual(self.evaluate(r), 1.0) + # TODO(b/117945658): reenable @test_util.run_in_graph_and_eager_modes def DISABLED_testCondAutoControlDeps(self): @@ -863,7 +1003,7 @@ class ControlFlowTest(test.TestCase): c = lambda x: math_ops.less(x, 10000) b = lambda x: math_ops.add(x, 1) r = control_flow_ops.while_loop(c, b, [n], parallel_iterations=20) - self.assertEqual(10000, r.eval()) + self.assertEqual(10000, self.evaluate(r)) @test_util.disable_control_flow_v2("b/79881896 (control deps)") def testWhileExternalControlDependencies(self): @@ -894,10 +1034,11 @@ class ControlFlowTest(test.TestCase): result = control_flow_ops.while_loop(cond=lambda i: i < 5, body=body_fn, loop_vars=[0]) - result.eval() + self.evaluate(result) self.assertAllEqual(v.eval(), 1.0) @test_util.disable_control_flow_v2("b/113324949 (RefVariable)") + @test_util.run_deprecated_v1 def testWhileWithRefs_1(self): with self.cached_session() as sess: x = variables.VariableV1(0)._ref() # pylint: disable=protected-access @@ -917,7 +1058,7 @@ class ControlFlowTest(test.TestCase): self.assertEqual(r[0].dtype, dtypes.int32) self.assertEqual(r[1].dtype, dtypes.int32_ref) - value_i, value_x = sess.run(r) + value_i, value_x = self.evaluate(r) self.assertEqual(100, value_i) self.assertEqual(0, value_x) @@ -926,19 +1067,19 @@ class ControlFlowTest(test.TestCase): with self.cached_session(): s = constant_op.constant(0) r = isum(s) - self.assertAllEqual(45, r.eval()) + self.assertAllEqual(45, self.evaluate(r)) def testWhileWithMaximumIterations(self): with self.cached_session(): s = constant_op.constant([1, 2, 3, 4, 5]) r = isum(s, maximum_iterations=3) - self.assertAllEqual([1 + 3, 2 + 3, 3 + 3, 4 + 3, 5 + 3], r.eval()) + self.assertAllEqual([1 + 3, 2 + 3, 3 + 3, 4 + 3, 5 + 3], self.evaluate(r)) def testWhileWithMaximumIterationsAndSingleArgument(self): with self.cached_session(): r = control_flow_ops.while_loop( lambda i: i < 3, lambda i: i + 1, [0], maximum_iterations=1) - self.assertEqual(1, r.eval()) + self.assertEqual(1, self.evaluate(r)) @test_util.disable_control_flow_v2("b/115776323 (max_iters)") def testSingleNestedMaximumIterationsWhileLoopGradientInXLAContext(self): @@ -1137,6 +1278,7 @@ class ControlFlowTest(test.TestCase): # Have more than 10 parallel iterations and hence exercise k-bound # most of the time. + @test_util.run_deprecated_v1 def testWhile_3(self): with self.cached_session(): @@ -1157,6 +1299,7 @@ class ControlFlowTest(test.TestCase): result = r[3].eval() self.assertAllEqual(10100, result) + @test_util.run_deprecated_v1 def testWhile_4(self): with self.cached_session(): @@ -1231,7 +1374,7 @@ class ControlFlowTest(test.TestCase): c = lambda x: math_ops.less(x, 10.0) b = lambda x: math_ops.add(x, 1.0) r = control_flow_ops.while_loop(c, b, [n]) - self.assertAllClose(10.0, r.eval()) + self.assertAllClose(10.0, self.evaluate(r)) def testWhile_Gpu_1(self): self._testWhile_Gpu_1(use_gpu=False) @@ -1247,7 +1390,7 @@ class ControlFlowTest(test.TestCase): return math_ops.add(x, 1.0) r = control_flow_ops.while_loop(c, b, [n]) - self.assertAllClose(10.0, r.eval()) + self.assertAllClose(10.0, self.evaluate(r)) def testWhile_Gpu_2(self): self._testWhile_Gpu_2(use_gpu=False) @@ -1268,15 +1411,16 @@ class ControlFlowTest(test.TestCase): c, _b, [i, m], [i.get_shape(), tensor_shape.unknown_shape()]) r = r[1] * array_ops.ones([8, 8]) - self.assertAllEqual(np.ones((8, 8)), r.eval()) + self.assertAllEqual(np.ones((8, 8)), self.evaluate(r)) + @test_util.run_deprecated_v1 def testWhileWithNonTensorInput_Scalar(self): with self.cached_session(): n = 0 c = lambda x: x < 10000 b = lambda x: x + 1 r = control_flow_ops.while_loop(c, b, [n], parallel_iterations=20) - self.assertEqual(10000, r.eval()) + self.assertEqual(10000, self.evaluate(r)) def testWhileWithNonTensorInput_Vector(self): with self.cached_session(): @@ -1284,7 +1428,7 @@ class ControlFlowTest(test.TestCase): c = lambda x: x[0] < 10000 b = lambda x: array_ops.stack([x[0] + 1]) r = control_flow_ops.while_loop(c, b, [n], parallel_iterations=20) - self.assertEqual([10000], r.eval()) + self.assertEqual([10000], self.evaluate(r)) def testWhileShapeInference(self): with self.cached_session(): @@ -1344,6 +1488,7 @@ class ControlFlowTest(test.TestCase): [i.get_shape(), tensor_shape.TensorShape([5])]) @test_util.disable_control_flow_v2("b/116282023 (IndexedSlices)") + @test_util.run_deprecated_v1 def testWhileShapeInferenceIndexedSlices(self): with self.cached_session(): values = constant_op.constant([[2.0, 4.0], [3.0, 5.0]], name="values") @@ -1396,7 +1541,7 @@ class ControlFlowTest(test.TestCase): c = lambda x: math_ops.less(x, 200) b = lambda x: math_ops.add(x, cpu_sum(n)) r = control_flow_ops.while_loop(c, b, [n]) - self.assertEqual(225, r.eval()) + self.assertEqual(225, self.evaluate(r)) def testNestedWhile_1(self): self._testNestedWhile_1(use_gpu=False) @@ -1428,7 +1573,7 @@ class ControlFlowTest(test.TestCase): r = control_flow_ops.while_loop( outer_c, outer_b, [s0], parallel_iterations=1) - self.assertEqual(1048576.0, r.eval()) + self.assertEqual(1048576.0, self.evaluate(r)) def testNestedWhile_2(self): self._testNestedWhile_2(use_gpu=False) @@ -1450,6 +1595,7 @@ class ControlFlowTest(test.TestCase): condition, body, [n, r], parallel_iterations=1) self.assertAllEqual(12, res[1].eval()) + @test_util.run_deprecated_v1 def testWhileWithControl_2(self): with self.cached_session(): r = constant_op.constant(0) @@ -1462,7 +1608,7 @@ class ControlFlowTest(test.TestCase): res = control_flow_ops.while_loop( condition, body, [r], parallel_iterations=1) - self.assertAllEqual(12, res.eval()) + self.assertAllEqual(12, self.evaluate(res)) def testWhileWithControl_3(self): with self.cached_session() as sess: @@ -1509,7 +1655,7 @@ class ControlFlowTest(test.TestCase): with ops.control_dependencies([control_flow_ops.no_op()]): loop = control_flow_ops.while_loop(cond, body, (constant_op.constant(5),)) - self.assertEqual(0, sess.run(loop)) + self.assertEqual(0, self.evaluate(loop)) @test_util.disable_control_flow_v2("b/113324949 (ref vars)") def testWhileCondWithControl_1(self): @@ -1531,8 +1677,8 @@ class ControlFlowTest(test.TestCase): r = control_flow_ops.while_loop(loop_condition, loop_body, (i0,)) variables.global_variables_initializer().run() - self.assertEqual(4, r.eval()) - self.assertAllClose(65536.0, v.eval()) + self.assertEqual(4, self.evaluate(r)) + self.assertAllClose(65536.0, self.evaluate(v)) @test_util.disable_control_flow_v2("b/113324949 (ref vars)") def testWhileCondExitControl(self): @@ -1556,8 +1702,8 @@ class ControlFlowTest(test.TestCase): constant_op.constant(False), lambda: constant_op.constant(1.0), false_branch) variables.global_variables_initializer().run() - self.assertEqual(6.0, r.eval()) - self.assertEqual(99, v.eval()) + self.assertEqual(6.0, self.evaluate(r)) + self.assertEqual(99, self.evaluate(v)) def testCondWhile_1(self): @@ -1568,7 +1714,7 @@ class ControlFlowTest(test.TestCase): r = control_flow_ops.cond( math_ops.less(0, 1), lambda: control_flow_ops.while_loop(c, b, [n]), lambda: n) - self.assertAllEqual(10, r.eval()) + self.assertAllEqual(10, self.evaluate(r)) def testCondWhile_2(self): @@ -1579,7 +1725,7 @@ class ControlFlowTest(test.TestCase): r = control_flow_ops.cond( math_ops.less(1, 0), lambda: math_ops.add(n, 1), lambda: control_flow_ops.while_loop(c, b, [n])) - self.assertAllEqual(10, r.eval()) + self.assertAllEqual(10, self.evaluate(r)) def _testCondWhile_3(self, use_gpu): with self.cached_session(use_gpu=use_gpu) as sess: @@ -1604,6 +1750,7 @@ class ControlFlowTest(test.TestCase): self.assertEqual([2.0], sess.run(r1, {p: False})) @test_util.disable_control_flow_v2("b/116743589") + @test_util.run_deprecated_v1 def testCondWhile_3(self): self._testCondWhile_3(use_gpu=False) self._testCondWhile_3(use_gpu=True) @@ -1622,7 +1769,7 @@ class ControlFlowTest(test.TestCase): lambda: math_ops.add(x, one), lambda: math_ops.subtract(x, one)) # pylint: enable=undefined-variable r = control_flow_ops.while_loop(c, b, [i]) - self.assertAllEqual(10, r.eval()) + self.assertAllEqual(10, self.evaluate(r)) def testWhileCond_2(self): @@ -1631,7 +1778,7 @@ class ControlFlowTest(test.TestCase): c = lambda x: math_ops.less(x, 10) b = lambda x: control_flow_ops.cond(constant_op.constant(True), lambda: math_ops.add(x, 1), lambda: n) r = control_flow_ops.while_loop(c, b, [n]) - self.assertAllEqual(10, r.eval()) + self.assertAllEqual(10, self.evaluate(r)) def testWhileCond_3(self): @@ -1645,10 +1792,41 @@ class ControlFlowTest(test.TestCase): lambda: math_ops.subtract(x, 1)) # pylint: enable=undefined-variable r = control_flow_ops.while_loop(c, b, [n]) - self.assertAllEqual(10, r.eval()) + self.assertAllEqual(10, self.evaluate(r)) + + @test_util.run_deprecated_v1 + def testWhileCondGradMultiDevice(self): + config = config_pb2.ConfigProto(device_count={"CPU": 2}, + allow_soft_placement=True) + with self.cached_session(use_gpu=True, config=config) as sess: + pred = array_ops.placeholder(dtypes.bool, []) + x_init = constant_op.constant(1.0) + + with ops.device("/cpu:0"): + z = control_flow_ops.while_loop( + lambda i, _: i < 3, + lambda i, x: (i + 1, control_flow_ops.cond( + pred, lambda: x * 2.0, lambda: 10.0)), + [0, x_init]) + + with ops.device("/cpu:1"): + grad = gradients_impl.gradients(z, x_init)[0] + + self.assertEqual(sess.run(grad, {pred: True}), 8.0) + self.assertEqual(sess.run(grad, {pred: False}), 0.0) + + if not control_flow_ops.ENABLE_WHILE_V2: + return + + with ops.device("/cpu:0"): + grad_grad = gradients_impl.gradients(grad, x_init)[0] + + self.assertEqual(sess.run(grad_grad, {pred: True}), 0.0) + self.assertEqual(sess.run(grad_grad, {pred: False}), 0.0) # NOTE: It is ok to have parallel_iterations > 1 @test_util.disable_control_flow_v2("b/113324949 (RefVariable)") + @test_util.run_deprecated_v1 def testWhileUpdateVariable_1(self): with self.cached_session(): select = variables.Variable([3.0, 4.0, 5.0]) @@ -1667,8 +1845,8 @@ class ControlFlowTest(test.TestCase): r = control_flow_ops.while_loop( loop_iterator, loop_body, [n], parallel_iterations=1) variables.global_variables_initializer().run() - self.assertEqual(3, r.eval()) - result = select.eval() + self.assertEqual(3, self.evaluate(r)) + result = self.evaluate(select) self.assertAllClose(np.array([10.0, 10.0, 10.0]), result) @test_util.disable_control_flow_v2("b/113324949 (RefVariable)") @@ -1692,13 +1870,14 @@ class ControlFlowTest(test.TestCase): r = control_flow_ops.while_loop( loop_iterator, loop_body, [n], parallel_iterations=1) variables.global_variables_initializer().run() - self.assertEqual(3, r.eval()) - result1 = select1.eval() + self.assertEqual(3, self.evaluate(r)) + result1 = self.evaluate(select1) self.assertAllClose(np.array([10.0, 10.0, 10.0]), result1) - result2 = select2.eval() + result2 = self.evaluate(select2) self.assertAllClose(np.array([10.0, 10.0, 10.0]), result2) @test_util.disable_control_flow_v2("b/113324949 (RefVariable)") + @test_util.run_deprecated_v1 def testWhileUpdateVariable_3(self): with self.cached_session(): select = variables.Variable([3.0, 4.0, 5.0]) @@ -1721,6 +1900,7 @@ class ControlFlowTest(test.TestCase): self.assertAllClose(np.array([10.0, 10.0, 10.0]), result) @test_util.disable_control_flow_v2("b/113324949 (RefVariable)") + @test_util.run_deprecated_v1 def testWhileUpdateVariable_4(self): with self.cached_session(): var_a = variables.Variable(0, name="a") @@ -1744,11 +1924,12 @@ class ControlFlowTest(test.TestCase): lpa = control_flow_ops.while_loop( pred, loop_body, [c], parallel_iterations=1) - self.assertEqual(0, var_b.eval()) - lpa.eval() # Run the loop - self.assertEqual(10, var_b.eval()) + self.assertEqual(0, self.evaluate(var_b)) + self.evaluate(lpa) # Run the loop + self.assertEqual(10, self.evaluate(var_b)) @test_util.disable_control_flow_v2("b/113324949 (RefVariable)") + @test_util.run_deprecated_v1 def testWhileUpdateVariable_5(self): with self.cached_session(): # Create some variables. @@ -1773,10 +1954,10 @@ class ControlFlowTest(test.TestCase): lpa = control_flow_ops.while_loop( pred, loop_body, [var_b], parallel_iterations=1, name="loop") - self.assertEqual(0, var_b.eval()) - lpa.eval() # Run the loop - self.assertEqual(10, var_a.eval()) - self.assertEqual(10, var_b.eval()) + self.assertEqual(0, self.evaluate(var_b)) + self.evaluate(lpa) # Run the loop + self.assertEqual(10, self.evaluate(var_a)) + self.assertEqual(10, self.evaluate(var_b)) @test_util.disable_control_flow_v2("b/113324949 (RefVariable)") def testWhileUpdateVariable_6(self): @@ -1803,10 +1984,10 @@ class ControlFlowTest(test.TestCase): lpa = control_flow_ops.while_loop( pred, loop_body, [c], parallel_iterations=1, name="loop") - self.assertEqual(0, var_b.eval()) - lpa.eval() # Run the loop - self.assertEqual(55, var_b.eval()) - self.assertEqual(10, var_a.eval()) + self.assertEqual(0, self.evaluate(var_b)) + self.evaluate(lpa) # Run the loop + self.assertEqual(55, self.evaluate(var_b)) + self.assertEqual(10, self.evaluate(var_a)) def testWhileQueue_1(self): with self.cached_session(): @@ -1822,7 +2003,7 @@ class ControlFlowTest(test.TestCase): return ni r = control_flow_ops.while_loop(c, b, [i], parallel_iterations=1) - self.assertEqual([10], r.eval()) + self.assertEqual([10], self.evaluate(r)) for i in xrange(10): self.assertEqual([i], q.dequeue().eval()) @@ -1858,7 +2039,7 @@ class ControlFlowTest(test.TestCase): b1, [r, x], [r.get_shape(), tensor_shape.unknown_shape()], parallel_iterations=1) - self.assertEqual(45, rx.eval()) + self.assertEqual(45, self.evaluate(rx)) def _testWhileGrad_ColocateGradients(self, colocate): gpu_dev_name = test.gpu_device_name() if test.is_gpu_available( @@ -1893,7 +2074,7 @@ class ControlFlowTest(test.TestCase): self.assertFalse(gpu_dev_name in dev) with self.session(graph=graph) as sess: - self.assertAllClose(1024.0, sess.run(r)) + self.assertAllClose(1024.0, self.evaluate(r)) @test_util.disable_control_flow_v2("b/116351701 (colocation)") def testWhileGrad_ColocateGradients(self): @@ -1909,7 +2090,7 @@ class ControlFlowTest(test.TestCase): r = control_flow_ops.cond(math_ops.less(1, 2), lambda: r, lambda: v) r = gradients_impl.gradients(r, v)[0] - self.assertAllClose(1024.0, r.eval()) + self.assertAllClose(1024.0, self.evaluate(r)) def testWhileGrad_Shape(self): with self.cached_session(): @@ -1928,6 +2109,7 @@ class ControlFlowTest(test.TestCase): self.assertEqual([None], r.get_shape().as_list()) self.assertAllClose([810.0, 2560.0], r.eval(feed_dict={x: [3.0, 4.0]})) + @test_util.run_deprecated_v1 def testWhileGrad_BaseShape(self): with self.cached_session() as sess: x = array_ops.placeholder(dtypes.float32, [None]) @@ -1949,7 +2131,7 @@ class ControlFlowTest(test.TestCase): r = math_ops.multiply(r, r) r = gradients_impl.gradients(r, v)[0] - self.assertEqual(524288.0, r.eval()) + self.assertEqual(524288.0, self.evaluate(r)) def testWhileGrad_LoopAdd(self): with self.cached_session(): @@ -1960,7 +2142,7 @@ class ControlFlowTest(test.TestCase): r = math_ops.add(r, r) r = gradients_impl.gradients(r, v)[0] - self.assertAllClose(2048.0, r.eval()) + self.assertAllClose(2048.0, self.evaluate(r)) def _testWhileGrad_Mul(self, use_gpu, p_iters): with self.cached_session(use_gpu=use_gpu) as sess: @@ -1971,11 +2153,12 @@ class ControlFlowTest(test.TestCase): r = control_flow_ops.while_loop(c, b, [v], parallel_iterations=p_iters) grad_a, grad_v = gradients_impl.gradients(r, [a, v]) - grad_a_val, grad_v_val = sess.run([grad_a, grad_v]) + grad_a_val, grad_v_val = self.evaluate([grad_a, grad_v]) self.assertAllClose(216.0, grad_a_val) self.assertAllClose(81.0, grad_v_val) @test_util.disable_control_flow_v2("b/116630618 (parallel_iters: times out)") + @test_util.run_deprecated_v1 def testWhileGrad_Mul(self): self._testWhileGrad_Mul(use_gpu=False, p_iters=1) self._testWhileGrad_Mul(use_gpu=False, p_iters=10) @@ -2003,14 +2186,13 @@ class ControlFlowTest(test.TestCase): r = control_flow_ops.while_loop(c, b, [v]) r = gradients_impl.gradients(r, v)[0] - self.assertAllClose(512.0, r.eval()) + self.assertAllClose(512.0, self.evaluate(r)) + @test_util.run_deprecated_v1 def testNestedWhileCondWhileGrad(self): - if control_flow_ops.ENABLE_WHILE_V2 and test_util.is_gpu_available(): - self.skipTest("b/118459209") self._testNestedWhileCondWhileGrad(use_gpu=False) - @test_util.disable_control_flow_v2("b/118459209") + @test_util.run_deprecated_v1 def testNestedWhileCondWhileGradGpu(self): self._testNestedWhileCondWhileGrad(use_gpu=True) @@ -2026,6 +2208,7 @@ class ControlFlowTest(test.TestCase): variables.global_variables_initializer().run() self.assertAllClose(216.0, r[0].eval()) + @test_util.run_deprecated_v1 def testWhileGrad_ResourceVariable(self): with self.cached_session(): a = resource_variable_ops.ResourceVariable(3.0) @@ -2040,7 +2223,7 @@ class ControlFlowTest(test.TestCase): def testWhileGradInCond(self): - with self.cached_session(): + with self.cached_session() as sess: n = ops.convert_to_tensor(1.0, name="n") x = array_ops.placeholder(dtypes.float32, shape=None) c = lambda n: math_ops.less(n, 10.0) @@ -2051,10 +2234,15 @@ class ControlFlowTest(test.TestCase): [tensor_shape.unknown_shape()]) return gradients_impl.gradients(r, x) - r = control_flow_ops.cond(math_ops.less(1, 2), fn1, lambda: x) - self.assertAllClose(9.0, r.eval(feed_dict={x: 1.0})) + #placed lambda function return tensor in list and set strict flag to True + #as cond_v2 implementation preserves nested output structures even with singeltons + r = control_flow_ops.cond(math_ops.less(1, 2), fn1, lambda: [x], strict=True) + #cannot run eval() on list object so use sess.run() and save output + result = sess.run(r,feed_dict={x: 1.0}) + self.assertAllClose([9.0], result) @test_util.disable_control_flow_v2("b/116340060") + @test_util.run_deprecated_v1 def testGradInWhileWrtInitialLoopVal(self): with self.cached_session(): x = array_ops.placeholder(dtypes.float32, shape=(), name="x") @@ -2102,7 +2290,7 @@ class ControlFlowTest(test.TestCase): i, x = control_flow_ops.while_loop(lambda i, x: i < 3, outer_body, [0, 0.0]) with self.cached_session() as sess: - i_val, x_val = sess.run([i, x]) + i_val, x_val = self.evaluate([i, x]) self.assertEqual(i_val, 3) self.assertAllClose(x_val, 1.0) @@ -2131,7 +2319,7 @@ class ControlFlowTest(test.TestCase): r_flattened = nest.flatten(r) self.assertEqual([100.0, 1.0, 102.0, 3.0, 4.0 + 100 * 2.0], - sess.run(r_flattened)) + self.evaluate(r_flattened)) def testWhile_NestedBadArityFails(self): with self.cached_session(): @@ -2172,6 +2360,7 @@ class ControlFlowTest(test.TestCase): r = gradients_impl.gradients([rx], y) self.assertAllClose(120.0, r[0].eval()) + @test_util.run_deprecated_v1 def testWhileGrad_Dependency(self): with self.cached_session(): i = constant_op.constant(0, name="i") @@ -2223,6 +2412,7 @@ class ControlFlowTest(test.TestCase): variables.global_variables_initializer().run() self.assertAllClose(np.ones([2, 3]), sess.run(grad[0])) + @test_util.run_deprecated_v1 def testWhileGrad_Const(self): with self.cached_session() as sess: c0 = constant_op.constant(0.0, name="c0") @@ -2295,10 +2485,11 @@ class ControlFlowTest(test.TestCase): with ops.control_dependencies([x_f]): y_f_d = array_ops.identity(y_f, name="y_f_d") - self.assertAllClose(2.0, y_f_d.eval()) # y_f_d = 1.0 + 1.0 + self.assertAllClose(2.0, self.evaluate(y_f_d)) # y_f_d = 1.0 + 1.0 g = gradients_impl.gradients([y_f_d], [x])[0] self.assertTrue(g is not None) - self.assertAllClose(1.0, g.eval()) # y_f_d = x + 1.0, dy_f_d/dx = 1.0 + self.assertAllClose(1.0, + self.evaluate(g)) # y_f_d = x + 1.0, dy_f_d/dx = 1.0 def _testNestedWhileGrad_Simple(self, use_gpu): with self.cached_session(use_gpu=use_gpu): @@ -2314,8 +2505,9 @@ class ControlFlowTest(test.TestCase): r = control_flow_ops.while_loop(c, b, [v]) r = gradients_impl.gradients(r, v)[0] - self.assertAllClose(8.0, r.eval()) + self.assertAllClose(8.0, self.evaluate(r)) + @test_util.run_deprecated_v1 def testNestedWhileGrad_Simple(self): self._testNestedWhileGrad_Simple(use_gpu=False) self._testNestedWhileGrad_Simple(use_gpu=True) @@ -2341,8 +2533,9 @@ class ControlFlowTest(test.TestCase): r = control_flow_ops.while_loop(c, b, [v]) r = gradients_impl.gradients(r, v)[0] - self.assertAllClose(256.0, r.eval()) + self.assertAllClose(256.0, self.evaluate(r)) + @test_util.run_deprecated_v1 def testNestedWhileGrad_ParallelInner(self): with self.cached_session(): v = constant_op.constant(1.0) @@ -2364,10 +2557,8 @@ class ControlFlowTest(test.TestCase): r = control_flow_ops.while_loop(c, b, [v]) r = gradients_impl.gradients(r, v)[0] - self.assertAllClose(512.0, r.eval()) + self.assertAllClose(512.0, self.evaluate(r)) - @test_util.disable_control_flow_v2("unsupported: resource creation in body. " - "Enable with new TAs b/117675481") def testNestedWhileGrad_ParallelIterations(self): # Make sure the stack pushes and pops of an inner loop are executed in # the sequential order of the iterations of its outer loop. @@ -2386,9 +2577,9 @@ class ControlFlowTest(test.TestCase): res = outer_loop(inp) optimizer = adam.AdamOptimizer(learning_rate=0.001) train_op = optimizer.minimize(math_ops.reduce_mean(math_ops.square(res))) - sess.run(variables.global_variables_initializer()) - sess.run(train_op) - self.assertAllClose(2.999, var.eval()) + self.evaluate(variables.global_variables_initializer()) + self.evaluate(train_op) + self.assertAllClose(2.999, self.evaluate(var)) def _testWhileCondGrad_Simple(self, use_gpu): with self.cached_session(use_gpu=use_gpu): @@ -2404,14 +2595,16 @@ class ControlFlowTest(test.TestCase): # pylint: enable=undefined-variable r = control_flow_ops.while_loop(c, b, [v]) r = gradients_impl.gradients(r, v)[0] - self.assertAllClose(1024.0, r.eval()) + self.assertAllClose(1024.0, self.evaluate(r)) @test_util.disable_control_flow_v2("b/117519152") + @test_util.run_deprecated_v1 def testWhileCondGrad_Simple(self): self._testWhileCondGrad_Simple(use_gpu=False) self._testWhileCondGrad_Simple(use_gpu=True) @test_util.disable_control_flow_v2("b/117276490") + @test_util.run_deprecated_v1 def testWhileCondGrad_UnknownShape(self): with self.cached_session() as sess: v = array_ops.placeholder(dtypes.float32) @@ -2429,6 +2622,7 @@ class ControlFlowTest(test.TestCase): r = sess.run(r, feed_dict={v: 2.0}) self.assertAllClose(1024.0, r) + @test_util.run_deprecated_v1 def testWhileGrad_Concat(self): with self.cached_session() as sess: x = variable_scope.get_variable("x", initializer=[[1., 2.]]) @@ -2446,11 +2640,11 @@ class ControlFlowTest(test.TestCase): [i0.get_shape(), tensor_shape.TensorShape([None, 2])]) s = math_ops.reduce_sum(h) - sess.run(variables.global_variables_initializer()) + self.evaluate(variables.global_variables_initializer()) optimizer = gradient_descent.GradientDescentOptimizer(0.01) op = optimizer.minimize(s) - sess.run(op) - self.assertAllClose([[0.98000002, 1.98000002]], sess.run(x)) + self.evaluate(op) + self.assertAllClose([[0.98000002, 1.98000002]], self.evaluate(x)) @test_util.disable_control_flow_v2("b/113324949 (RefVariable)") def testWhileWithRefsWithGradients_1(self): @@ -2501,9 +2695,10 @@ class ControlFlowTest(test.TestCase): _, r = control_flow_ops.while_loop(c, b, [i, x]) r = gradients_impl.gradients(r.values, values)[0] - self.assertAllClose(np.array([1024.0, 1024.0]), r.eval()) + self.assertAllClose(np.array([1024.0, 1024.0]), self.evaluate(r)) @test_util.disable_control_flow_v2("b/116328420 (SparseTensor)") + @test_util.run_deprecated_v1 def testWhileGrad_SparseTensor(self): with self.cached_session(): values = constant_op.constant([2.0, 4.0], name="values") @@ -2524,7 +2719,7 @@ class ControlFlowTest(test.TestCase): _, r = control_flow_ops.while_loop(c, b, [i, x]) r = gradients_impl.gradients(r.values, values)[0] - self.assertAllClose(np.array([1024.0, 1024.0]), r.eval()) + self.assertAllClose(np.array([1024.0, 1024.0]), self.evaluate(r)) @test_util.disable_control_flow_v2("b/115920078 (gradients)") def testCallGradInLoop(self): @@ -2544,10 +2739,9 @@ class ControlFlowTest(test.TestCase): output_grad = control_flow_ops.while_loop( c, b, [i0, constant_op.constant(0.0)]) - self.assertAllClose(600.0, sess.run(output_grad)[1]) + self.assertAllClose(600.0, self.evaluate(output_grad)[1]) - @test_util.disable_control_flow_v2("unsupported: resource creation in body. " - "Enable with new TAs b/117675481") + @test_util.run_deprecated_v1 def testWhileAndTensorArray(self): with self.cached_session() as sess: param = constant_op.constant(2.0) @@ -2565,8 +2759,9 @@ class ControlFlowTest(test.TestCase): r = control_flow_ops.while_loop(c, b, [n0, y0], parallel_iterations=1) r = gradients_impl.gradients(r, param)[0] - self.assertAllClose(107520.0, sess.run(r)) + self.assertAllClose(107520.0, self.evaluate(r)) + @test_util.run_deprecated_v1 def testWhileGrad_StopGrad(self): with self.cached_session(): x = constant_op.constant(3.0, name="x") @@ -2582,9 +2777,9 @@ class ControlFlowTest(test.TestCase): rx, ry = control_flow_ops.while_loop(c, b, [x, y]) r = gradients_impl.gradients(rx, y)[0] - self.assertEqual(136.0, r.eval()) + self.assertEqual(136.0, self.evaluate(r)) r = gradients_impl.gradients(ry, y)[0] - self.assertEqual(32.0, r.eval()) + self.assertEqual(32.0, self.evaluate(r)) r = gradients_impl.gradients(array_ops.stop_gradient(rx), y)[0] self.assertEqual(r, None) @@ -2602,14 +2797,15 @@ class ControlFlowTest(test.TestCase): self.assertEqual(r, None) r = gradients_impl.gradients(math_ops.add(rx, ry), y)[0] - self.assertEqual(168.0, r.eval()) + self.assertEqual(168.0, self.evaluate(r)) r = gradients_impl.gradients( math_ops.add(rx, array_ops.stop_gradient(ry)), y)[0] - self.assertEqual(136.0, r.eval()) + self.assertEqual(136.0, self.evaluate(r)) r = gradients_impl.gradients( math_ops.add(array_ops.stop_gradient(rx), ry), y)[0] - self.assertEqual(32.0, r.eval()) + self.assertEqual(32.0, self.evaluate(r)) + @test_util.run_deprecated_v1 def testWhileGrad_StopGradInside(self): with self.cached_session(): x = constant_op.constant(3.0, name="x") @@ -2625,10 +2821,11 @@ class ControlFlowTest(test.TestCase): rx, _ = control_flow_ops.while_loop(c, b, [x, y]) r = gradients_impl.gradients(rx, y)[0] - self.assertAllClose(0.0, r.eval()) + self.assertAllClose(0.0, self.evaluate(r)) r = gradients_impl.gradients(rx, x)[0] - self.assertAllClose(156.0, r.eval()) + self.assertAllClose(156.0, self.evaluate(r)) + @test_util.run_deprecated_v1 def testWhileGrad_StopGradInsideNoShape(self): with self.cached_session() as sess: x = array_ops.placeholder(dtypes.float32) @@ -2650,9 +2847,10 @@ class ControlFlowTest(test.TestCase): self.assertAllClose([156.0, 400.0], sess.run(r, feed_dict=feed_dict)) name = "gradients/while/stopped_grad" all_ops = x.graph.get_operations() - self.assertFalse(any([name in op.name for op in all_ops])) + self.assertFalse(any(name in op.name for op in all_ops)) @test_util.disable_control_flow_v2("b/117954949") + @test_util.run_deprecated_v1 def testWhileGradGradFail(self): theta = variables.Variable(initial_value=1.) @@ -2667,6 +2865,7 @@ class ControlFlowTest(test.TestCase): grad_theta_stopped = array_ops.stop_gradient(grad_theta) gradients_impl.gradients(grad_theta_stopped, theta) + @test_util.run_deprecated_v1 def testStopGradOnWhileGrad(self): with self.cached_session(): x = constant_op.constant(2.0, name="x") @@ -2681,9 +2880,10 @@ class ControlFlowTest(test.TestCase): r = math_ops.add(math_ops.square(y), rx) r = math_ops.add(r, rg) r = gradients_impl.gradients(r, y)[0] - self.assertEqual(388.0, r.eval()) + self.assertEqual(388.0, self.evaluate(r)) @test_util.disable_control_flow_v2("b/113324949 (RefVariable)") + @test_util.run_deprecated_v1 def testWhileGradientWithNontrainablePath1(self): q = variables.Variable([7., 8.]) @@ -2698,8 +2898,8 @@ class ControlFlowTest(test.TestCase): dy_dq, = gradients_impl.gradients(y, q) self.assertIsNotNone(dy_dq) with self.cached_session() as sess: - sess.run(q.initializer) - self.assertAllClose([0., 0.], sess.run(dy_dq)) + self.evaluate(q.initializer) + self.assertAllClose([0., 0.], self.evaluate(dy_dq)) @test_util.disable_control_flow_v2("b/113324949 (RefVariable)") def testWhileGradientWithNontrainablePath2(self): @@ -2716,8 +2916,8 @@ class ControlFlowTest(test.TestCase): dy_dq, = gradients_impl.gradients(y, q) self.assertIsNotNone(dy_dq) with self.cached_session() as sess: - sess.run(q.initializer) - self.assertAllClose([1., 1.], sess.run(dy_dq)) + self.evaluate(q.initializer) + self.assertAllClose([1., 1.], self.evaluate(dy_dq)) @test_util.disable_control_flow_v2("b/115920078 (gradients)") def testIssue16504(self): @@ -2767,7 +2967,7 @@ class ControlFlowTest(test.TestCase): z = math_ops.add(r, array_ops.stop_gradient(math_ops.reduce_sum(grads))) result = gradients_impl.gradients(z, vars_)[0] variables.global_variables_initializer().run() - self.assertEqual(5.0, result.eval()) + self.assertEqual(5.0, self.evaluate(result)) def testOneValueCond(self): @@ -2785,6 +2985,7 @@ class ControlFlowTest(test.TestCase): # False case: c = 0 is not >= 1 self.assertEqual([2], i.eval(feed_dict={c: 0})) + @test_util.run_deprecated_v1 def testExampleCond(self): with self.cached_session(): @@ -2828,7 +3029,7 @@ class ControlFlowTest(test.TestCase): r4 = control_flow_ops.case( [(x < y, f1), (x < y, f2)], default=f3, exclusive=True) with self.assertRaisesOpError("Input error:"): - r4.eval() + self.evaluate(r4) # Check that the default is called if none of the others are r5 = control_flow_ops.case({x > y: f1}, default=f3) @@ -2874,19 +3075,19 @@ class ControlFlowTest(test.TestCase): ((x > y, a), (x > y, b)), default=c, exclusive=True) variables.global_variables_initializer().run() - self.assertAllEqual(sess.run([v0, v1, v2]), [-1] * 3) - self.assertEqual(2, r2.eval()) - self.assertAllEqual(sess.run([v0, v1, v2]), [-1, -1, 2]) + self.assertAllEqual(self.evaluate([v0, v1, v2]), [-1] * 3) + self.assertEqual(2, self.evaluate(r2)) + self.assertAllEqual(self.evaluate([v0, v1, v2]), [-1, -1, 2]) variables.global_variables_initializer().run() - self.assertAllEqual(sess.run([v0, v1, v2]), [-1] * 3) - self.assertEqual(1, r1.eval()) - self.assertAllEqual(sess.run([v0, v1, v2]), [-1, 1, -1]) + self.assertAllEqual(self.evaluate([v0, v1, v2]), [-1] * 3) + self.assertEqual(1, self.evaluate(r1)) + self.assertAllEqual(self.evaluate([v0, v1, v2]), [-1, 1, -1]) variables.global_variables_initializer().run() - self.assertAllEqual(sess.run([v0, v1, v2]), [-1] * 3) - self.assertEqual(0, r0.eval()) - self.assertAllEqual(sess.run([v0, v1, v2]), [0, -1, -1]) + self.assertAllEqual(self.evaluate([v0, v1, v2]), [-1] * 3) + self.assertEqual(0, self.evaluate(r0)) + self.assertAllEqual(self.evaluate([v0, v1, v2]), [0, -1, -1]) @test_util.disable_control_flow_v2("b/113324949 (ref vars)") def testOneOpCond(self): @@ -2907,15 +3108,15 @@ class ControlFlowTest(test.TestCase): self.assertTrue(isinstance(i, ops.Tensor)) variables.global_variables_initializer().run() - self.assertEqual(0, v.eval()) + self.assertEqual(0, self.evaluate(v)) # True case: c = 2 is >= 1, v is set to 1. self.assertEqual(1, i.eval(feed_dict={c.name: 2})) - self.assertEqual(1, v.eval()) + self.assertEqual(1, self.evaluate(v)) # False case: c = 0 is not >= 1, v is set to 2. self.assertEqual(2, i.eval(feed_dict={c.name: 0})) - self.assertEqual(2, v.eval()) + self.assertEqual(2, self.evaluate(v)) def testWithOpsDependencies(self): with self.cached_session() as sess: @@ -2924,7 +3125,7 @@ class ControlFlowTest(test.TestCase): # Fetching v directly will result in an uninitialized error with self.assertRaisesOpError("Attempting to use uninitialized value"): - sess.run([c, v]) + self.evaluate([c, v]) # Use a control dependency to ensure init_variable is run # while asking for c @@ -2932,7 +3133,7 @@ class ControlFlowTest(test.TestCase): name="real_tensor", output_tensor=v._ref(), # pylint: disable=protected-access dependencies=[v.initializer]) - c_val, real_v_val = sess.run([c, real_v]) + c_val, real_v_val = self.evaluate([c, real_v]) # Ensure the result of 'real_c' is the same as 'c' self.assertAllEqual(10, c_val) @@ -2957,14 +3158,14 @@ class ControlFlowTest(test.TestCase): # Fetching v directly will result in an uninitialized error with self.assertRaisesOpError("Attempting to use uninitialized value"): - v.eval() + self.evaluate(v) # Get the value of 'c2_with_c1_dep', which should cause 'v' # to be initialized. - self.assertAllEqual(20, c2_with_c1_dep.eval()) + self.assertAllEqual(20, self.evaluate(c2_with_c1_dep)) # Ensure that 'v' is initialized - self.assertAllClose(0.0, v.eval()) + self.assertAllClose(0.0, self.evaluate(v)) def testWithIndexedSlicesDependencies(self): with self.cached_session(): @@ -2979,13 +3180,15 @@ class ControlFlowTest(test.TestCase): # Fetching gather_v_at_1 will result in an uninitialized error with self.assertRaisesOpError("Attempting to use uninitialized value"): - gather_v_at_1.eval() + self.evaluate(gather_v_at_1) # Getting gather_v_at_1_after_init will work, and initialize v. - self.assertAllEqual([[10.0, 11.0]], gather_v_at_1_after_init.eval()) + self.assertAllEqual([[10.0, 11.0]], + self.evaluate(gather_v_at_1_after_init)) # Double check that 'v' is initialized - self.assertAllClose([[0.0, 1.0], [10.0, 11.0], [20.0, 21.0]], v.eval()) + self.assertAllClose([[0.0, 1.0], [10.0, 11.0], [20.0, 21.0]], + self.evaluate(v)) def testDependenciesDevice(self): with ops.Graph().as_default(): @@ -3019,11 +3222,11 @@ class ControlFlowTest(test.TestCase): init = control_flow_ops.group(v1.initializer, v2.initializer) # Fetching v1 directly will result in an uninitialized error with self.assertRaisesOpError("Attempting to use uninitialized value"): - v1.eval() + self.evaluate(v1) # Runs "init" before fetching v1 and v2. init.run() - v1_val, v2_val = sess.run([v1, v2]) + v1_val, v2_val = self.evaluate([v1, v2]) # Ensure that v1 and v2 are initialized self.assertAllClose([0.0], v1_val) @@ -3034,6 +3237,7 @@ class ControlFlowTest(test.TestCase): self.assertEqual(op.type, "NoOp") self.assertEqual(op.control_inputs, []) + @test_util.run_deprecated_v1 def testMergeShapes(self): # All inputs unknown. p1 = array_ops.placeholder(dtypes.float32) @@ -3088,6 +3292,7 @@ class ControlFlowTest(test.TestCase): self.assertEqual([None, None], m.get_shape().as_list()) self.assertEqual([], index.get_shape()) + @test_util.run_deprecated_v1 def testRefSelect(self): index = array_ops.placeholder(dtypes.int32) @@ -3121,6 +3326,7 @@ class ControlFlowTest(test.TestCase): s = control_flow_ops.ref_select(index, [v1, v2]) self.assertEqual(None, s.get_shape()) + @test_util.run_deprecated_v1 def testRunLoopTensor(self): with self.cached_session() as sess: tensor_list = [] @@ -3134,7 +3340,7 @@ class ControlFlowTest(test.TestCase): result = control_flow_ops.while_loop(condition, body, [constant_op.constant(4)]) - self.assertEqual(10, sess.run(result)) + self.assertEqual(10, self.evaluate(result)) # Ensure that we cannot run a tensor that escapes the loop body # accidentally. @@ -3184,7 +3390,7 @@ class ControlFlowTest(test.TestCase): cond = constant_op.constant(True, dtypes.bool) v_f, v_t = control_flow_ops.switch(constant_qint, cond) result = control_flow_ops.merge([v_f, v_t]) - sess.run(result) + self.evaluate(result) def testQIntRefSwitchMerge(self): with self.cached_session(use_gpu=test.is_gpu_available()) as sess: @@ -3192,13 +3398,22 @@ class ControlFlowTest(test.TestCase): shape=[1], dtype=dtypes.qint8, name="v", container="", shared_name="") assign_op = state_ops.assign( var_qint, constant_op.constant(np.array([42]), dtypes.qint8)) - sess.run(assign_op) + self.evaluate(assign_op) cond = constant_op.constant(True, dtypes.bool) v_f, v_t = control_flow_ops.ref_switch(var_qint, cond) result = control_flow_ops.ref_merge([v_f, v_t]) - sess.run(result) + self.evaluate(result) + def testUInt64SwitchMerge(self): + with self.cached_session(force_gpu=test.is_gpu_available()) as sess: + constant_uint64 = constant_op.constant(np.array([42]), dtypes.uint64) + cond = constant_op.constant(True, dtypes.bool) + v_f, v_t = control_flow_ops.switch(constant_uint64, cond) + result = control_flow_ops.merge([v_f, v_t]) + self.evaluate(result) + + @test_util.run_deprecated_v1 def testQIntArgAndRet(self): @function.Defun(dtypes.qint8) @@ -3208,7 +3423,7 @@ class ControlFlowTest(test.TestCase): with self.cached_session(force_gpu=test.is_gpu_available()) as sess: qint = constant_op.constant(np.array([42]), dtypes.qint8) result = func(qint) - sess.run(result) + self.evaluate(result) class ControlFlowContextCheckTest(test.TestCase): @@ -3246,6 +3461,7 @@ class ControlFlowContextCheckTest(test.TestCase): "is in a while loop. See info log for more details."): math_ops.add(1, while_tensor) + @test_util.run_deprecated_v1 def testInvalidContextInCond(self): # Accessing a while loop tensor in cond is illegal. while_tensor = self._getWhileTensor() @@ -3314,6 +3530,7 @@ class ControlFlowContextCheckTest(test.TestCase): control_flow_ops.while_loop(lambda i: i < 5, body, [0]) + @test_util.run_deprecated_v1 def testInvalidNestedContexts(self): # Accessing a tensor from a while context in a different while context, all # inside a cond context, is illegal. @@ -3347,21 +3564,22 @@ class TupleTest(test.TestCase): # v1 is not initialized. with self.assertRaisesOpError("Attempting to use uninitialized value"): - v1.eval() + self.evaluate(v1) # v2 is not initialized. with self.assertRaisesOpError("Attempting to use uninitialized value"): - v2.eval() + self.evaluate(v2) if v1_first: # Getting t1 initializes v2. - self.assertAllClose([3.0], t1.eval()) - self.assertAllClose([10.0], v2.eval()) + self.assertAllClose([3.0], self.evaluate(t1)) + self.assertAllClose([10.0], self.evaluate(v2)) else: # Getting t2 initializes v1. - self.assertAllClose([30.0], t2.eval()) - self.assertAllClose([1.0], v1.eval()) + self.assertAllClose([30.0], self.evaluate(t2)) + self.assertAllClose([1.0], self.evaluate(v1)) + @test_util.run_deprecated_v1 def testIndexedSlices(self): for v1_first in [True, False]: with self.cached_session(): @@ -3385,22 +3603,22 @@ class TupleTest(test.TestCase): # v1 is not initialized. with self.assertRaisesOpError("Attempting to use uninitialized value"): - v1.eval() + self.evaluate(v1) # v2 is not initialized. with self.assertRaisesOpError("Attempting to use uninitialized value"): - v2.eval() + self.evaluate(v2) if v1_first: # Getting g1 initializes v2. - self.assertAllClose([[10.0, 11.0]], g1.eval()) + self.assertAllClose([[10.0, 11.0]], self.evaluate(g1)) self.assertAllClose([[0.1, 1.1], [10.1, 11.1], [20.1, 21.1]], - v2.eval()) + self.evaluate(v2)) else: # Getting g2 initializes v1. - self.assertAllClose([[10.1, 11.1]], g2.eval()) + self.assertAllClose([[10.1, 11.1]], self.evaluate(g2)) self.assertAllClose([[0.0, 1.0], [10.0, 11.0], [20.0, 21.0]], - v1.eval()) + self.evaluate(v1)) def testAcceptTensorsAsControlInputs(self): with self.cached_session(): @@ -3410,13 +3628,14 @@ class TupleTest(test.TestCase): [constant_op.constant(0)], control_inputs=[assign]) # Should trigger the assign. - t.eval() + self.evaluate(t) - self.assertEquals(1, var.eval()) + self.assertEquals(1, self.evaluate(var)) class AssertTest(test.TestCase): + @test_util.run_deprecated_v1 def testGuardedAssertDoesNotCopyWhenTrue(self): with self.session(use_gpu=True) as sess: with ops.device(test.gpu_device_name()): @@ -3513,7 +3732,7 @@ class WhileOpBenchmark(test.Benchmark): with session.Session() as sess, ops.device(default_device): # Get the initial id i, input x, and kernel. i, x, kernel = self._getInitVariables() - sess.run(variables.global_variables_initializer()) + self.evaluate(variables.global_variables_initializer()) if static_unroll: for _ in xrange(steps): @@ -3532,11 +3751,11 @@ class WhileOpBenchmark(test.Benchmark): for _ in xrange(3): # exclude warm up time - sess.run(r) + self.evaluate(r) start_time = time.time() for _ in xrange(num_iters): - sess.run(r) + self.evaluate(r) return (time.time() - start_time) / num_iters def benchmarkWhileOpCrossDevicePlacement(self): diff --git a/tensorflow/python/kernel_tests/conv1d_test.py b/tensorflow/python/kernel_tests/conv1d_test.py index 8540875d75e..e8463323df9 100644 --- a/tensorflow/python/kernel_tests/conv1d_test.py +++ b/tensorflow/python/kernel_tests/conv1d_test.py @@ -43,7 +43,7 @@ class Conv1DTest(test.TestCase): with self.cached_session(use_gpu=test.is_gpu_available()): c = nn_ops.conv1d(x, filters, stride, padding="VALID") reduced = array_ops.squeeze(c) - output = reduced.eval() + output = self.evaluate(reduced) if stride == 1: self.assertEqual(len(output), 3) self.assertAllClose(output, @@ -69,7 +69,7 @@ class Conv1DTest(test.TestCase): 1.0, shape=f_shape, name="filter", dtype=dtypes.float32) output = nn_ops.conv1d_transpose( x, f, y_shape, stride=stride, padding="VALID") - value = output.eval() + value = self.evaluate(output) cache_values = np.zeros(y_shape, dtype=np.float32) diff --git a/tensorflow/python/kernel_tests/conv2d_backprop_filter_grad_test.py b/tensorflow/python/kernel_tests/conv2d_backprop_filter_grad_test.py index af6ffc1d195..7b3b560b240 100644 --- a/tensorflow/python/kernel_tests/conv2d_backprop_filter_grad_test.py +++ b/tensorflow/python/kernel_tests/conv2d_backprop_filter_grad_test.py @@ -22,6 +22,7 @@ import numpy as np from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes +from tensorflow.python.framework import test_util from tensorflow.python.ops import array_ops from tensorflow.python.ops import gradient_checker from tensorflow.python.ops import nn_ops @@ -31,6 +32,7 @@ from tensorflow.python.platform import test class Conv2DBackpropFilterGradTest(test.TestCase): + @test_util.run_deprecated_v1 def testGradient(self): with self.cached_session(): for padding in ["SAME", "VALID"]: diff --git a/tensorflow/python/kernel_tests/conv2d_transpose_test.py b/tensorflow/python/kernel_tests/conv2d_transpose_test.py index 6f9992a317f..c603c086306 100644 --- a/tensorflow/python/kernel_tests/conv2d_transpose_test.py +++ b/tensorflow/python/kernel_tests/conv2d_transpose_test.py @@ -53,7 +53,7 @@ class Conv2DTransposeTest(test.TestCase): 1.0, shape=f_shape, name="filter", dtype=dtypes.float32) output = nn_ops.conv2d_transpose( x, f, y_shape, strides=strides, padding="SAME") - value = output.eval() + value = self.evaluate(output) # We count the number of cells being added at the locations in the output. # At the center, #cells=kernel_height * kernel_width @@ -91,7 +91,7 @@ class Conv2DTransposeTest(test.TestCase): 1.0, shape=f_shape, name="filter", dtype=dtypes.float32) output = nn_ops.conv2d_transpose( x, f, y_shape, strides=strides, padding="SAME") - value = output.eval() + value = self.evaluate(output) for n in xrange(x_shape[0]): for k in xrange(f_shape[2]): @@ -124,7 +124,7 @@ class Conv2DTransposeTest(test.TestCase): 1.0, shape=f_shape, name="filter", dtype=dtypes.float32) output = nn_ops.conv2d_transpose( x, f, y_shape, strides=strides, padding="VALID") - value = output.eval() + value = self.evaluate(output) cache_values = np.zeros(y_shape, dtype=np.float32) @@ -155,6 +155,7 @@ class Conv2DTransposeTest(test.TestCase): self.assertAllClose(cache_values, value) + @test_util.run_deprecated_v1 def testGradient(self): x_shape = [2, 6, 4, 3] f_shape = [3, 3, 2, 3] @@ -195,7 +196,7 @@ class Conv2DTransposeTest(test.TestCase): output = nn_ops.conv2d_transpose( x, f, y_shape, strides=strides, padding="SAME", data_format="NCHW") - value = output.eval() + value = self.evaluate(output) for n in xrange(x_shape[0]): for k in xrange(f_shape[2]): for w in xrange(y_shape[3]): @@ -230,7 +231,7 @@ class Conv2DTransposeTest(test.TestCase): output = nn_ops.conv2d_transpose( x, f, y_shape, strides=strides, padding="SAME", data_format="NCHW") - value = output.eval() + value = self.evaluate(output) for n in xrange(x_shape[0]): for k in xrange(f_shape[2]): for w in xrange(y_shape[3]): @@ -265,7 +266,7 @@ class Conv2DTransposeTest(test.TestCase): output = nn_ops.conv2d_transpose( x, f, y_shape, strides=strides, padding="VALID", data_format="NCHW") - value = output.eval() + value = self.evaluate(output) cache_values = np.zeros(y_shape, dtype=np.float32) # The amount of padding added pad = 1 @@ -293,7 +294,6 @@ class Conv2DTransposeTest(test.TestCase): self.assertAllClose(cache_values, value) - @test_util.enable_c_shapes def testConv2DTransposeShapeInference(self): # Test case for 8972 initializer = random_ops.truncated_normal( diff --git a/tensorflow/python/kernel_tests/conv3d_backprop_filter_v2_grad_test.py b/tensorflow/python/kernel_tests/conv3d_backprop_filter_v2_grad_test.py index 89b64068ace..7e913febed3 100644 --- a/tensorflow/python/kernel_tests/conv3d_backprop_filter_v2_grad_test.py +++ b/tensorflow/python/kernel_tests/conv3d_backprop_filter_v2_grad_test.py @@ -22,6 +22,7 @@ import numpy as np from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes +from tensorflow.python.framework import test_util from tensorflow.python.ops import array_ops from tensorflow.python.ops import gradient_checker from tensorflow.python.ops import nn_ops @@ -31,6 +32,7 @@ from tensorflow.python.platform import test class Conv3DBackpropFilterV2GradTest(test.TestCase): + @test_util.run_deprecated_v1 def testGradient(self): with self.cached_session(): for padding in ["SAME", "VALID"]: diff --git a/tensorflow/python/kernel_tests/conv3d_transpose_test.py b/tensorflow/python/kernel_tests/conv3d_transpose_test.py index 2527b837692..22ba5b90375 100644 --- a/tensorflow/python/kernel_tests/conv3d_transpose_test.py +++ b/tensorflow/python/kernel_tests/conv3d_transpose_test.py @@ -23,6 +23,7 @@ from six.moves import xrange # pylint: disable=redefined-builtin from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes +from tensorflow.python.framework import test_util from tensorflow.python.ops import gradient_checker from tensorflow.python.ops import nn_ops import tensorflow.python.ops.nn_grad # pylint: disable=unused-import @@ -48,7 +49,7 @@ class Conv3DTransposeTest(test.TestCase): 1.0, shape=f_shape, name="filter", dtype=dtypes.float32) output = nn_ops.conv3d_transpose( x, f, y_shape, strides=strides, padding="SAME") - value = output.eval() + value = self.evaluate(output) # We count the number of cells being added at the locations in the output. # At the center, #cells = kernel_depth * kernel_height * kernel_width @@ -98,7 +99,7 @@ class Conv3DTransposeTest(test.TestCase): 1.0, shape=f_shape, name="filter", dtype=dtypes.float32) output = nn_ops.conv3d_transpose( x, f, y_shape, strides=strides, padding="SAME") - value = output.eval() + value = self.evaluate(output) for n in xrange(x_shape[0]): for k in xrange(f_shape[3]): @@ -119,6 +120,7 @@ class Conv3DTransposeTest(test.TestCase): target = 3.0 self.assertAllClose(target, value[n, d, h, w, k]) + @test_util.run_deprecated_v1 def testConv3DTransposeShapeMismatch(self): # Test case for GitHub issue 18460 x_shape = [2, 2, 3, 4, 3] @@ -146,7 +148,7 @@ class Conv3DTransposeTest(test.TestCase): output = nn_ops.conv3d_transpose( x_value, f_value, constant_op.constant(y_shape, dtype=dtype), strides=strides, padding="SAME") - output.eval() + self.evaluate(output) def testConv3DTransposeValid(self): with self.cached_session(): @@ -165,7 +167,7 @@ class Conv3DTransposeTest(test.TestCase): 1.0, shape=f_shape, name="filter", dtype=dtypes.float32) output = nn_ops.conv3d_transpose( x, f, y_shape, strides=strides, padding="VALID") - value = output.eval() + value = self.evaluate(output) cache_values = np.zeros(y_shape, dtype=np.float32) @@ -201,6 +203,7 @@ class Conv3DTransposeTest(test.TestCase): self.assertAllClose(cache_values, value) + @test_util.run_deprecated_v1 def testGradient(self): x_shape = [2, 3, 4, 3, 2] f_shape = [3, 3, 3, 2, 2] diff --git a/tensorflow/python/kernel_tests/conv_ops_3d_test.py b/tensorflow/python/kernel_tests/conv_ops_3d_test.py index c4a9cdcf8e0..4a689b3fdfa 100644 --- a/tensorflow/python/kernel_tests/conv_ops_3d_test.py +++ b/tensorflow/python/kernel_tests/conv_ops_3d_test.py @@ -52,11 +52,11 @@ class Conv3DTest(test.TestCase): def _DtypesToTest(self, use_gpu): if use_gpu: if not test_util.CudaSupportsHalfMatMulAndConv(): - return [dtypes.float32] + return [dtypes.float64, dtypes.float32] else: # It is important that float32 comes before float16 here, # as we will be using its gradients as reference for fp16 gradients. - return [dtypes.float32, dtypes.float16] + return [dtypes.float64, dtypes.float32, dtypes.float16] else: return [dtypes.float64, dtypes.float32, dtypes.float16] @@ -109,7 +109,7 @@ class Conv3DTest(test.TestCase): results.append(result) with self.cached_session() as sess: - values = sess.run(results) + values = self.evaluate(results) for value in values: print("expected = ", expected) print("actual = ", value) @@ -184,8 +184,8 @@ class Conv3DTest(test.TestCase): computed_results.append(computed) tolerance = 1e-2 if use_gpu else 1e-5 with self.cached_session() as sess: - expected_values = sess.run(expected_results) - computed_values = sess.run(computed_results) + expected_values = self.evaluate(expected_results) + computed_values = self.evaluate(computed_results) for e_value, c_value in zip(expected_values, computed_values): print("expected = ", e_value) print("actual = ", c_value) @@ -462,6 +462,7 @@ class Conv3DTest(test.TestCase): self._ConstructAndTestGradientForConfig(data_format=data_format, use_gpu=use_gpu, **kwargs) + @test_util.run_deprecated_v1 def testInputGradientValidPaddingStrideOne(self): self.ConstructAndTestGradient( batch=2, @@ -473,6 +474,7 @@ class Conv3DTest(test.TestCase): padding="VALID", test_input=True) + @test_util.run_deprecated_v1 def testFilterGradientValidPaddingStrideOne(self): self.ConstructAndTestGradient( batch=4, @@ -484,6 +486,7 @@ class Conv3DTest(test.TestCase): padding="VALID", test_input=False) + @test_util.run_deprecated_v1 def testInputGradientValidPaddingStrideTwo(self): self.ConstructAndTestGradient( batch=2, @@ -495,6 +498,7 @@ class Conv3DTest(test.TestCase): padding="VALID", test_input=True) + @test_util.run_deprecated_v1 def testFilterGradientValidPaddingStrideTwo(self): self.ConstructAndTestGradient( batch=2, @@ -506,6 +510,7 @@ class Conv3DTest(test.TestCase): padding="VALID", test_input=False) + @test_util.run_deprecated_v1 def testInputGradientValidPaddingStrideThree(self): self.ConstructAndTestGradient( batch=2, @@ -517,6 +522,7 @@ class Conv3DTest(test.TestCase): padding="VALID", test_input=True) + @test_util.run_deprecated_v1 def testFilterGradientValidPaddingStrideThree(self): self.ConstructAndTestGradient( batch=2, @@ -528,6 +534,7 @@ class Conv3DTest(test.TestCase): padding="VALID", test_input=False) + @test_util.run_deprecated_v1 def testInputGradientSamePaddingStrideOne(self): self.ConstructAndTestGradient( batch=2, @@ -539,6 +546,7 @@ class Conv3DTest(test.TestCase): padding="SAME", test_input=True) + @test_util.run_deprecated_v1 def testFilterGradientSamePaddingStrideOne(self): self.ConstructAndTestGradient( batch=2, @@ -550,6 +558,7 @@ class Conv3DTest(test.TestCase): padding="SAME", test_input=False) + @test_util.run_deprecated_v1 def testInputGradientSamePaddingStrideTwo(self): self.ConstructAndTestGradient( batch=2, @@ -561,6 +570,7 @@ class Conv3DTest(test.TestCase): padding="SAME", test_input=True) + @test_util.run_deprecated_v1 def testFilterGradientSamePaddingStrideTwo(self): self.ConstructAndTestGradient( batch=4, @@ -572,6 +582,7 @@ class Conv3DTest(test.TestCase): padding="SAME", test_input=False) + @test_util.run_deprecated_v1 def testInputGradientSamePaddingStrideThree(self): self.ConstructAndTestGradient( batch=2, @@ -583,6 +594,7 @@ class Conv3DTest(test.TestCase): padding="SAME", test_input=True) + @test_util.run_deprecated_v1 def testFilterGradientSamePaddingStrideThree(self): self.ConstructAndTestGradient( batch=2, @@ -594,6 +606,7 @@ class Conv3DTest(test.TestCase): padding="SAME", test_input=False) + @test_util.run_deprecated_v1 def testInputGradientSamePaddingDifferentStrides(self): self.ConstructAndTestGradient( batch=1, @@ -605,6 +618,7 @@ class Conv3DTest(test.TestCase): padding="SAME", test_input=True) + @test_util.run_deprecated_v1 def testFilterGradientKernelSizeMatchesInputSize(self): self.ConstructAndTestGradient( batch=2, @@ -616,6 +630,7 @@ class Conv3DTest(test.TestCase): padding="VALID", test_input=False) + @test_util.run_deprecated_v1 def testInputGradientKernelSizeMatchesInputSize(self): self.ConstructAndTestGradient( batch=2, @@ -640,6 +655,7 @@ class Conv3DTest(test.TestCase): # Test the fast path in gemm_pack_rhs/mkldnn_gemm_pack, when channel # dimension is a multiple of packet size. + @test_util.run_deprecated_v1 def testInputGradientValidPaddingStrideOneFastPath(self): self.ConstructAndTestGradient( batch=2, @@ -651,6 +667,7 @@ class Conv3DTest(test.TestCase): padding="VALID", test_input=True) + @test_util.run_deprecated_v1 def testFilterGradientValidPaddingStrideOneFastPath(self): self.ConstructAndTestGradient( batch=2, @@ -715,8 +732,8 @@ class Conv3DTest(test.TestCase): expected_grad = gradients_impl.gradients(expected, t1 if mode == "input" else t2)[0] # "values" consists of two tensors for two backprops - actual_value = sess.run(actual_grad) - expected_value = sess.run(expected_grad) + actual_value = self.evaluate(actual_grad) + expected_value = self.evaluate(expected_grad) self.assertShapeEqual(actual_value, actual_grad) self.assertShapeEqual(expected_value, expected_grad) print("expected = ", expected_value) diff --git a/tensorflow/python/kernel_tests/conv_ops_test.py b/tensorflow/python/kernel_tests/conv_ops_test.py index 0ccbbf155c5..2f6f3bb383b 100644 --- a/tensorflow/python/kernel_tests/conv_ops_test.py +++ b/tensorflow/python/kernel_tests/conv_ops_test.py @@ -908,8 +908,8 @@ class Conv2DTest(test.TestCase): conv = gradients_impl.gradients(conv_forward, t1)[0] conv_2 = gradients_impl.gradients(conv_forward_2, t1)[0] # "values" consists of two tensors for two backprops - value = sess.run(conv) - value_2 = sess.run(conv_2) + value = self.evaluate(conv) + value_2 = self.evaluate(conv_2) self.assertShapeEqual(value, conv) self.assertShapeEqual(value_2, conv_2) tf_logging.info("expected = ", value_2) @@ -961,8 +961,8 @@ class Conv2DTest(test.TestCase): conv_forward_2 = test_util.NCHWToNHWC(conv_forward_2) conv = gradients_impl.gradients(conv_forward, t2)[0] conv_2 = gradients_impl.gradients(conv_forward, t2)[0] - value = sess.run(conv) - value_2 = sess.run(conv_2) + value = self.evaluate(conv) + value_2 = self.evaluate(conv_2) self.assertShapeEqual(value, conv) self.assertShapeEqual(value_2, conv_2) tf_logging.info("expected = ", value_2) @@ -1545,7 +1545,7 @@ class DepthwiseConv2DTest(test.TestCase): t2 = constant_op.constant(x2, shape=filter_in_sizes) conv = nn_impl.depthwise_conv2d( t1, t2, strides=[1, stride, stride, 1], padding=padding) - value = sess.run(conv) + value = self.evaluate(conv) tf_logging.info("value = ", value) self.assertArrayNear(expected, np.ravel(value), 1e-5) self.assertShapeEqual(value, conv) @@ -1667,9 +1667,9 @@ class SeparableConv2DTest(test.TestCase): if data_format == "NCHW": conv = array_ops.transpose(conv, [0, 2, 3, 1]) - value = sess.run(conv) + value = self.evaluate(conv) tf_logging.info("value = ", value) - self.assertArrayNear(expected, np.ravel(value), 1e-5) + self.assertArrayNear(expected, np.ravel(value), 1e-3) self.assertShapeEqual(value, conv) def _testSeparableConv2D(self, data_format): @@ -1774,10 +1774,10 @@ class DeepConv2DTest(test.TestCase): conv = nn_ops.conv2d(t1, t2, strides=strides, padding=padding) os.environ["TF_USE_DEEP_CONV2D"] = "0" - values_expect = sess.run([conv]) + values_expect = self.evaluate([conv]) os.environ["TF_USE_DEEP_CONV2D"] = "1" - values_test = sess.run([conv]) + values_test = self.evaluate([conv]) self.assertAllClose(values_expect, values_test, rtol=1e-5, atol=1e-5) diff --git a/tensorflow/python/kernel_tests/cross_grad_test.py b/tensorflow/python/kernel_tests/cross_grad_test.py index 0bd4006d6ac..b397133fd73 100644 --- a/tensorflow/python/kernel_tests/cross_grad_test.py +++ b/tensorflow/python/kernel_tests/cross_grad_test.py @@ -18,6 +18,7 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function +from tensorflow.python.framework import test_util from tensorflow.python.ops import array_ops from tensorflow.python.ops import gradient_checker from tensorflow.python.ops import math_ops @@ -26,6 +27,7 @@ from tensorflow.python.platform import test class CrossOpTest(test.TestCase): + @test_util.run_deprecated_v1 def testGradientRandomValues(self): with self.cached_session(): us = [2, 3] diff --git a/tensorflow/python/kernel_tests/ctc_decoder_ops_test.py b/tensorflow/python/kernel_tests/ctc_decoder_ops_test.py index d818fbd75ce..0d86d13c715 100644 --- a/tensorflow/python/kernel_tests/ctc_decoder_ops_test.py +++ b/tensorflow/python/kernel_tests/ctc_decoder_ops_test.py @@ -25,6 +25,7 @@ from six.moves import zip_longest from tensorflow.python.framework import errors from tensorflow.python.framework import ops +from tensorflow.python.framework import test_util from tensorflow.python.ops import array_ops from tensorflow.python.ops import ctc_ops from tensorflow.python.platform import test @@ -94,6 +95,7 @@ class CTCGreedyDecoderTest(test.TestCase): with self.assertRaisesOpError(expected_err_re): sess.run(decoded_unwrapped + [log_probability]) + @test_util.run_deprecated_v1 def testCTCGreedyDecoder(self): """Test two batch entries - best path decoder.""" max_time_steps = 6 @@ -170,6 +172,7 @@ class CTCGreedyDecoderTest(test.TestCase): self._testCTCDecoder(ctc_ops.ctc_greedy_decoder, inputs, seq_lens, log_prob_truth, decode_truth) + @test_util.run_deprecated_v1 def testCTCDecoderBeamSearch(self): """Test one batch, two beams - hibernating beam search.""" # max_time_steps == 8 diff --git a/tensorflow/python/kernel_tests/ctc_loss_op_test.py b/tensorflow/python/kernel_tests/ctc_loss_op_test.py index cfc7cb98aa6..e6b5835079e 100644 --- a/tensorflow/python/kernel_tests/ctc_loss_op_test.py +++ b/tensorflow/python/kernel_tests/ctc_loss_op_test.py @@ -23,9 +23,16 @@ import numpy as np from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes from tensorflow.python.framework import errors_impl +from tensorflow.python.framework import ops +from tensorflow.python.framework import random_seed from tensorflow.python.framework import sparse_tensor +from tensorflow.python.framework import test_util +from tensorflow.python.ops import array_ops from tensorflow.python.ops import ctc_ops from tensorflow.python.ops import gradients_impl +from tensorflow.python.ops import math_ops +from tensorflow.python.ops import random_ops +from tensorflow.python.ops import sparse_ops from tensorflow.python.platform import test @@ -52,6 +59,24 @@ def SimpleSparseTensorFrom(x): return sparse_tensor.SparseTensor(x_ix, x_val, x_shape) +def _ctc_loss_v2(labels, inputs, sequence_length, + preprocess_collapse_repeated=False, + ctc_merge_repeated=True, + ignore_longer_outputs_than_inputs=False, + time_major=True): + """Call ctc_loss_v2 with v1 args.""" + assert not preprocess_collapse_repeated + assert ctc_merge_repeated + assert not ignore_longer_outputs_than_inputs + return ctc_ops.ctc_loss_v2( + labels=labels, + logits=inputs, + logit_length=sequence_length, + label_length=None, + blank_index=-1, + logits_time_major=time_major) + + class CTCLossTest(test.TestCase): def _testCTCLoss(self, @@ -66,7 +91,7 @@ class CTCLossTest(test.TestCase): inputs_t = constant_op.constant(inputs) with self.cached_session(use_gpu=False) as sess: - loss = ctc_ops.ctc_loss( + loss = _ctc_loss_v2( inputs=inputs_t, labels=labels, sequence_length=seq_lens) grad = gradients_impl.gradients(loss, [inputs_t])[0] @@ -74,13 +99,14 @@ class CTCLossTest(test.TestCase): self.assertShapeEqual(grad_truth, grad) if expected_err_re is None: - (tf_loss, tf_grad) = sess.run([loss, grad]) + (tf_loss, tf_grad) = self.evaluate([loss, grad]) self.assertAllClose(tf_loss, loss_truth, atol=1e-6) self.assertAllClose(tf_grad, grad_truth, atol=1e-6) else: with self.assertRaisesOpError(expected_err_re): - sess.run([loss, grad]) + self.evaluate([loss, grad]) + @test_util.run_deprecated_v1 def testBasic(self): """Test two batch entries.""" # Input and ground truth from Alex Graves' implementation. @@ -216,6 +242,7 @@ class CTCLossTest(test.TestCase): self._testCTCLoss(inputs, seq_lens, labels, loss_truth, grad_truth) + @test_util.run_deprecated_v1 def test_time_major(self): """Testing time_major param. @@ -234,17 +261,18 @@ class CTCLossTest(test.TestCase): inputs_t_transposed = constant_op.constant(inputs.transpose(1, 0, 2)) with self.session(use_gpu=False) as sess: - loss = ctc_ops.ctc_loss( + loss = _ctc_loss_v2( inputs=inputs_t, labels=labels, sequence_length=seq_lens) - loss_transposed = ctc_ops.ctc_loss( + loss_transposed = _ctc_loss_v2( inputs=inputs_t_transposed, labels=labels, sequence_length=seq_lens, time_major=False) - (tf_loss, tf_loss_transposed) = sess.run([loss, loss_transposed]) + (tf_loss, tf_loss_transposed) = self.evaluate([loss, loss_transposed]) self.assertAllEqual(tf_loss, tf_loss_transposed) + @test_util.run_deprecated_v1 def testInvalidSecondGradient(self): inputs = np.random.randn(2, 2, 3).astype(np.float32) inputs_t = constant_op.constant(inputs) @@ -253,7 +281,7 @@ class CTCLossTest(test.TestCase): v = [1.0] with self.session(use_gpu=False): - loss = ctc_ops.ctc_loss( + loss = _ctc_loss_v2( inputs=inputs_t, labels=labels, sequence_length=seq_lens) # Taking ths second gradient should fail, since it is not # yet supported. @@ -261,6 +289,7 @@ class CTCLossTest(test.TestCase): "explicitly disabled"): _ = gradients_impl._hessian_vector_product(loss, [inputs_t], v) + @test_util.run_deprecated_v1 def testEmptyBatch(self): inputs = constant_op.constant([], dtype=dtypes.float32, shape=(1, 0, 2)) sequence_lengths = constant_op.constant([], dtype=dtypes.int32) @@ -272,7 +301,546 @@ class CTCLossTest(test.TestCase): with self.session(use_gpu=False) as sess: with self.assertRaisesRegexp(errors_impl.InvalidArgumentError, "batch_size must not be 0"): - sess.run(ctc_ops.ctc_loss(labels, inputs, sequence_lengths)) + sess.run(_ctc_loss_v2(labels, inputs, sequence_lengths)) + + +class CTCLossTestV2(test.TestCase): + + @test_util.run_deprecated_v1 + def testCtcLossV2(self): + random_seed.set_random_seed(5) + + batch_size = 8 + num_labels = 6 + max_label_length = 5 + num_frames = 12 + + labels = random_ops.random_uniform( + [batch_size, max_label_length], minval=1, maxval=num_labels, + dtype=dtypes.int64) + logits = random_ops.random_uniform([num_frames, batch_size, num_labels]) + + label_length = random_ops.random_uniform( + [batch_size], minval=2, maxval=max_label_length, dtype=dtypes.int64) + label_mask = array_ops.sequence_mask( + label_length, maxlen=max_label_length, dtype=label_length.dtype) + labels *= label_mask + logit_length = [num_frames] * batch_size + + ref_loss = ctc_ops.ctc_loss_v2( + labels=labels, + logits=logits, + label_length=label_length, + logit_length=logit_length) + ref_grad = gradients_impl.gradients(ref_loss, [logits]) + + sparse_labels = ctc_ops.dense_labels_to_sparse(labels, label_length) + + def assert_same_loss_and_grads(loss): + with self.cached_session() as sess: + self.assertAllClose(*self.evaluate([loss, ref_loss])) + grad = gradients_impl.gradients(loss, [logits]) + self.assertAllClose( + *self.evaluate([grad, ref_grad]), rtol=2e-06, atol=2e-06) + + assert_same_loss_and_grads( + ctc_ops.ctc_loss_v2( + labels=sparse_labels, + logits=logits, + label_length=label_length, + logit_length=logit_length, + blank_index=0)) + + @test_util.run_deprecated_v1 + def testCtcLossDenseIsSameAsCtcLoss(self): + with ops.device("/GPU:0" if test.is_gpu_available() else "/CPU:0"): + random_seed.set_random_seed(5) + + batch_size = 8 + num_labels = 6 + label_length = 5 + num_frames = 12 + logits = random_ops.random_uniform([num_frames, batch_size, num_labels]) + labels = random_ops.random_uniform( + [batch_size, label_length], minval=1, maxval=num_labels, + dtype=dtypes.int64) + + label_lengths = random_ops.random_uniform( + [batch_size], minval=2, maxval=label_length, dtype=dtypes.int64) + label_mask = array_ops.sequence_mask( + label_lengths, maxlen=label_length, dtype=label_lengths.dtype) + labels *= label_mask + + logit_lengths = [num_frames] * batch_size + + ctc_loss = ctc_ops.ctc_loss_dense( + labels=labels, + logits=logits, + label_length=label_lengths, + logit_length=logit_lengths) + ctc_loss_grads = gradients_impl.gradients(ctc_loss, [logits])[0] + + # Shift labels down by one (move blank from 0 to num_labels -1) + tf_ctc_loss_labels = math_ops.cast(labels, dtypes.int32) - 1 + tf_nn_ctc_logits = array_ops.concat([ + logits[:, :, 1:], + logits[:, :, 0:1], + ], axis=2) + + tf_ctc_loss_labels = ctc_ops.dense_labels_to_sparse( + tf_ctc_loss_labels, label_lengths) + + tf_nn_ctc_loss = ctc_ops.ctc_loss( + labels=tf_ctc_loss_labels, + inputs=tf_nn_ctc_logits, + sequence_length=logit_lengths, + time_major=True) + tf_nn_ctc_grads = gradients_impl.gradients(tf_nn_ctc_loss, [logits])[0] + + with self.cached_session() as sess: + for _ in range(32): + self.assertAllClose(*self.evaluate([ctc_loss, tf_nn_ctc_loss])) + self.assertAllClose( + *self.evaluate([ctc_loss_grads, tf_nn_ctc_grads]), + rtol=2e-06, + atol=2e-06) + + @test_util.run_deprecated_v1 + def testCtcLossDenseUniqueFastPathIsSameAsCtcLoss(self): + random_seed.set_random_seed(5) + + batch_size = 8 + num_labels = 6 + label_length = 5 + num_frames = 12 + logits = random_ops.random_uniform([num_frames, batch_size, num_labels]) + labels = random_ops.random_uniform( + [batch_size, label_length], minval=1, maxval=num_labels, + dtype=dtypes.int64) + + label_lengths = random_ops.random_uniform( + [batch_size], minval=2, maxval=label_length, dtype=dtypes.int64) + label_mask = array_ops.sequence_mask( + label_lengths, maxlen=label_length, dtype=label_lengths.dtype) + labels *= label_mask + + logit_lengths = [num_frames] * batch_size + + ctc_loss = ctc_ops.ctc_loss_dense( + labels=labels, + logits=logits, + label_length=label_lengths, + logit_length=logit_lengths, + unique=ctc_ops.ctc_unique_labels(labels)) + ctc_loss_grads = gradients_impl.gradients(ctc_loss, [logits])[0] + + # Shift labels down by one (move blank from 0 to num_labels -1) + tf_ctc_loss_labels = math_ops.cast(labels, dtypes.int32) - 1 + tf_nn_ctc_logits = array_ops.concat([ + logits[:, :, 1:], + logits[:, :, 0:1], + ], axis=2) + + tf_ctc_loss_labels = ctc_ops.dense_labels_to_sparse( + tf_ctc_loss_labels, label_lengths) + + tf_nn_ctc_loss = ctc_ops.ctc_loss( + labels=tf_ctc_loss_labels, + inputs=tf_nn_ctc_logits, + sequence_length=logit_lengths, + time_major=True) + tf_nn_ctc_grads = gradients_impl.gradients(tf_nn_ctc_loss, [logits])[0] + + with self.cached_session() as sess: + for _ in range(32): + self.assertAllClose(*self.evaluate([ctc_loss, tf_nn_ctc_loss])) + self.assertAllClose( + *self.evaluate([ctc_loss_grads, tf_nn_ctc_grads]), + rtol=2e-06, + atol=2e-06) + + @test_util.run_deprecated_v1 + def testCtcLossDenseWithBlankIndexIsSameAsCtcLoss(self): + random_seed.set_random_seed(5) + + batch_size = 8 + num_labels = 6 + label_length = 5 + num_frames = 12 + logits = random_ops.random_uniform([num_frames, batch_size, num_labels]) + labels = random_ops.random_uniform( + [batch_size, label_length], minval=0, maxval=num_labels-1, + dtype=dtypes.int64) + + label_lengths = random_ops.random_uniform( + [batch_size], minval=2, maxval=label_length, dtype=dtypes.int64) + label_mask = array_ops.sequence_mask( + label_lengths, maxlen=label_length, dtype=label_lengths.dtype) + labels *= label_mask + + logit_lengths = [num_frames] * batch_size + + tf_ctc_loss_labels = math_ops.cast(labels, dtypes.int32) + tf_ctc_loss_labels = ctc_ops.dense_labels_to_sparse( + tf_ctc_loss_labels, label_lengths) + + tf_nn_ctc_loss = ctc_ops.ctc_loss( + labels=tf_ctc_loss_labels, + inputs=logits, + sequence_length=logit_lengths, + time_major=True) + tf_nn_ctc_grads = gradients_impl.gradients(tf_nn_ctc_loss, [logits])[0] + + # Shift the blank logits/labels to be somewhere in the middle. + blank_index = 2 + shifted_logits = array_ops.concat([ + logits[:, :, :blank_index], + logits[:, :, -1:], + logits[:, :, blank_index:-1], + ], axis=2) + shifted_labels = array_ops.where(labels < blank_index, labels, labels + 1) + + ctc_loss = ctc_ops.ctc_loss_dense( + labels=shifted_labels, + logits=shifted_logits, + label_length=label_lengths, + logit_length=logit_lengths, + blank_index=blank_index) + ctc_loss_grads = gradients_impl.gradients(ctc_loss, [logits])[0] + + with self.cached_session() as sess: + for _ in range(32): + self.assertAllClose(*self.evaluate([ctc_loss, tf_nn_ctc_loss])) + self.assertAllClose( + *self.evaluate([ctc_loss_grads, tf_nn_ctc_grads]), + rtol=2e-06, + atol=2e-06) + + @test_util.run_deprecated_v1 + def testCtcLossDenseWithNegativeBlankIndexIsSameAsCtcLoss(self): + with ops.device("/GPU:0" if test.is_gpu_available() else "/CPU:0"): + random_seed.set_random_seed(5) + + batch_size = 8 + num_labels = 6 + label_length = 5 + num_frames = 12 + logits = random_ops.random_uniform([num_frames, batch_size, num_labels]) + labels = random_ops.random_uniform( + [batch_size, label_length], minval=0, maxval=num_labels-1, + dtype=dtypes.int64) + + label_lengths = random_ops.random_uniform( + [batch_size], minval=2, maxval=label_length, dtype=dtypes.int64) + label_mask = array_ops.sequence_mask( + label_lengths, maxlen=label_length, dtype=label_lengths.dtype) + labels *= label_mask + + logit_lengths = [num_frames] * batch_size + + ctc_loss = ctc_ops.ctc_loss_dense( + labels=labels, + logits=logits, + label_length=label_lengths, + logit_length=logit_lengths, + blank_index=-1) + ctc_loss_grads = gradients_impl.gradients(ctc_loss, [logits])[0] + + tf_ctc_loss_labels = math_ops.cast(labels, dtypes.int32) + tf_ctc_loss_labels = ctc_ops.dense_labels_to_sparse( + tf_ctc_loss_labels, label_lengths) + + tf_nn_ctc_loss = ctc_ops.ctc_loss( + labels=tf_ctc_loss_labels, + inputs=logits, + sequence_length=logit_lengths, + time_major=True) + tf_nn_ctc_grads = gradients_impl.gradients(tf_nn_ctc_loss, [logits])[0] + + with self.cached_session() as sess: + for _ in range(32): + self.assertAllClose(*self.evaluate([ctc_loss, tf_nn_ctc_loss])) + self.assertAllClose( + *self.evaluate([ctc_loss_grads, tf_nn_ctc_grads]), + rtol=2e-06, + atol=2e-06) + + @test_util.run_deprecated_v1 + def testCollapseRepeated(self): + collapsed, new_seq_lengths = ctc_ops.collapse_repeated( + labels=[[1, 3, 3, 3, 0], + [1, 4, 4, 4, 0], + [4, 2, 2, 9, 4]], + seq_length=[4, 5, 5]) + self.assertAllEqual(new_seq_lengths, [2, 3, 4]) + self.assertAllEqual( + collapsed, + [[1, 3, 0, 0], + [1, 4, 0, 0], + [4, 2, 9, 4]]) + + @test_util.run_deprecated_v1 + def testCollapseRepeatedPreservesDtypes(self): + collapsed, new_seq_lengths = ctc_ops.collapse_repeated( + labels=constant_op.constant( + [[1, 3, 3, 3, 0], + [1, 4, 4, 4, 0], + [4, 2, 2, 9, 4]], + dtype=dtypes.int64), + seq_length=constant_op.constant([4, 5, 5], dtype=dtypes.int64)) + self.assertEqual(new_seq_lengths.dtype, dtypes.int64) + self.assertEqual(collapsed.dtype, dtypes.int64) + self.assertAllEqual(new_seq_lengths, [2, 3, 4]) + self.assertAllEqual( + collapsed, + [[1, 3, 0, 0], + [1, 4, 0, 0], + [4, 2, 9, 4]]) + + @test_util.run_deprecated_v1 + def testCollapseRepeatedExtraPadding(self): + collapsed, new_seq_lengths = ctc_ops.collapse_repeated( + labels=[[1, 3, 3, 3, 0, 0, 0], + [1, 4, 4, 4, 0, 1, 2], + [4, 2, 2, 9, 4, 0, 0]], + seq_length=[4, 5, 5]) + self.assertAllEqual(new_seq_lengths, [2, 3, 4]) + self.assertAllEqual( + collapsed, + [[1, 3, 0, 0], + [1, 4, 0, 0], + [4, 2, 9, 4]]) + + @test_util.run_deprecated_v1 + def testCollapseRepeatedFrontRepeats(self): + collapsed, new_seq_lengths = ctc_ops.collapse_repeated( + labels=[[1, 1, 1, 2, 2], + [1, 1, 1, 2, 2], + [1, 1, 1, 2, 2]], + seq_length=[5, 4, 3]) + self.assertAllEqual(new_seq_lengths, [2, 2, 1]) + self.assertAllEqual( + collapsed, + [[1, 2], + [1, 2], + [1, 0]]) + + @test_util.run_deprecated_v1 + def testCollapseRepeatedAllLabelsTheSame(self): + collapsed, new_seq_lengths = ctc_ops.collapse_repeated( + labels=[[1, 1, 1, 1, 1], + [1, 1, 1, 1, 1], + [1, 1, 1, 1, 1]], + seq_length=[4, 5, 1]) + self.assertAllEqual(new_seq_lengths, [1, 1, 1]) + self.assertAllEqual( + collapsed, + [[1], + [1], + [1]]) + + def testDenseSequencesToSparse(self): + labels = [[1, 3, 3, 3, 0], + [1, 4, 4, 4, 0], + [4, 2, 2, 9, 4]] + length = [4, 5, 5] + sparse = ctc_ops.dense_labels_to_sparse(labels, length) + new_dense = sparse_ops.sparse_tensor_to_dense(sparse) + + self.assertAllEqual(labels, new_dense) + + padded_labels = [[1, 3, 3, 3, 0, 0, 0, 0], + [1, 4, 4, 4, 0, 0, 0, 0], + [4, 2, 2, 9, 4, 0, 0, 0]] + length = [4, 5, 5] + sparse = ctc_ops.dense_labels_to_sparse(padded_labels, length) + padded_dense = sparse_ops.sparse_tensor_to_dense(sparse) + + self.assertAllEqual(padded_dense, new_dense) + + @test_util.run_deprecated_v1 + def testUnique(self): + labels = [ + [3, 4, 4, 3], + [1, 1, 1, 0], + ] + unique, idx = ctc_ops.ctc_unique_labels(labels) + self.assertAllEqual([ + [3, 4, 0, 0], + [1, 0, 0, 0], + ], unique) + self.assertAllEqual([ + [0, 1, 1, 0], + [0, 0, 0, 1], + ], idx) + + @test_util.run_deprecated_v1 + def testSumStates(self): + idx = [ + [0, 1, 0, 1], + [0, 0, 0, 1], + ] + states = math_ops.log([ + [[1.0, 2.0, 3.0, 4.0], + [5.0, 6.0, 7.0, 8.0]], + [[0.1, 0.2, 0.3, 0.4], + [0.5, 0.6, 0.7, 0.8]], + ]) + sum_of_states = math_ops.exp(ctc_ops._sum_states(idx, states)) + self.assertAllClose([ + [[4.0, 6.0, 0.0, 0.0], + [18.0, 8.0, 0.0, 0.0]], + [[0.4, 0.6, 0.0, 0.0], + [1.8, 0.8, 0.0, 0.0]] + ], sum_of_states) + + @test_util.run_deprecated_v1 + def testStateToOlabel(self): + labels = [ + [3, 4, 3, 4], + [1, 1, 1, 0], + ] + num_labels = 8 + + # 3 frames, 2 batch, 10 states (5 label, 5 blank). + states = [ + [[0.11, 0.12, 0.13, 0.14, 0.15, 0.16, 0.17, 0.18, 0.19, 0.20], + [0.21, 0.22, 0.23, 0.24, 0.25, 0.26, 0.27, 0.28, 0.29, 0.30]], + [[1.1, 1.2, 1.3, 1.4, 1.5, 1.6, 1.7, 1.8, 1.9, 2.0], + [2.1, 2.2, 2.3, 2.4, 2.5, 2.6, 2.7, 2.8, 2.9, 3.0]], + [[11.0, 12.0, 13.0, 14.0, 15.0, 16.0, 17.0, 18.0, 19.0, 20.0], + [21.0, 22.0, 23.0, 24.0, 25.0, 26.0, 27.0, 28.0, 29.0, 30.0]], + ] + labels = ops.convert_to_tensor(labels) + states = math_ops.log(states) + olabel = ctc_ops._state_to_olabel(labels, num_labels, states) + olabel = math_ops.exp(olabel) + blank = olabel[:, :, 0] + self.assertAllClose(blank, [ + [0.16 + 0.17 + 0.18 + 0.19 + 0.20, + 0.26 + 0.27 + 0.28 + 0.29 + 0.30], + [1.6 + 1.7 + 1.8 + 1.9 + 2.0, + 2.6 + 2.7 + 2.8 + 2.9 + 3.0], + [16.0 + 17.0 + 18.0 + 19.0 + 20.0, + 26.0 + 27.0 + 28.0 + 29.0 + 30.0] + ]) + self.assertAllClose(olabel[:, :, 1:], [ + [[0.0, 0.0, 0.12 + 0.14, 0.13 + 0.15, 0.0, 0.0, 0.0], + [0.22 + 0.23 + 0.24, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], + [[0.0, 0.0, 1.2 + 1.4, 1.3 + 1.5, 0.0, 0.0, 0.0], + [2.2 + 2.3 + 2.4, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], + [[0.0, 0.0, 12.0 + 14.0, 13.0 + 15.0, 0.0, 0.0, 0.0], + [22.0 + 23.0 + 24.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], + ]) + + @test_util.run_deprecated_v1 + def testStateToOlabelUnique(self): + labels = [ + [3, 4, 3, 4], + [1, 1, 1, 0], + ] + num_labels = 8 + + # 3 frames, 2 batch, 10 states (5 label, 5 blank). + states = [ + [[0.11, 0.12, 0.13, 0.14, 0.15, 0.16, 0.17, 0.18, 0.19, 0.20], + [0.21, 0.22, 0.23, 0.24, 0.25, 0.26, 0.27, 0.28, 0.29, 0.30]], + [[1.1, 1.2, 1.3, 1.4, 1.5, 1.6, 1.7, 1.8, 1.9, 2.0], + [2.1, 2.2, 2.3, 2.4, 2.5, 2.6, 2.7, 2.8, 2.9, 3.0]], + [[11.0, 12.0, 13.0, 14.0, 15.0, 16.0, 17.0, 18.0, 19.0, 20.0], + [21.0, 22.0, 23.0, 24.0, 25.0, 26.0, 27.0, 28.0, 29.0, 30.0]], + ] + labels = ops.convert_to_tensor(labels) + states = math_ops.log(states) + olabel = ctc_ops._state_to_olabel_unique( + labels, num_labels, states, ctc_ops.ctc_unique_labels(labels)) + olabel = math_ops.exp(olabel) + blank = olabel[:, :, 0] + self.assertAllClose(blank, [ + [0.16 + 0.17 + 0.18 + 0.19 + 0.20, + 0.26 + 0.27 + 0.28 + 0.29 + 0.30], + [1.6 + 1.7 + 1.8 + 1.9 + 2.0, + 2.6 + 2.7 + 2.8 + 2.9 + 3.0], + [16.0 + 17.0 + 18.0 + 19.0 + 20.0, + 26.0 + 27.0 + 28.0 + 29.0 + 30.0]]) + self.assertAllClose(olabel[:, :, 1:], [ + [[0.0, 0.0, 0.12 + 0.14, 0.13 + 0.15, 0.0, 0.0, 0.0], + [0.22 + 0.23 + 0.24, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], + [[0.0, 0.0, 1.2 + 1.4, 1.3 + 1.5, 0.0, 0.0, 0.0], + [2.2 + 2.3 + 2.4, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], + [[0.0, 0.0, 12.0 + 14.0, 13.0 + 15.0, 0.0, 0.0, 0.0], + [22.0 + 23.0 + 24.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], + ]) + + @test_util.run_deprecated_v1 + def testScan(self): + with ops.device("/GPU:0" if test.is_gpu_available() else "/CPU:0"): + out = ctc_ops._scan( + lambda accum, elem: accum + elem, + constant_op.constant([1.0, 2.0, 3.0]), 23.0) + self.assertAllEqual([24.0, 26.0, 29.0], out) + + out = ctc_ops._scan( + lambda a, e: a + e, + constant_op.constant([1.0, 2.0, 3.0]), 23.0, + inclusive=True) + self.assertAllEqual([23.0, 24.0, 26.0, 29.0], out) + + out = ctc_ops._scan( + lambda a, e: a + e, + constant_op.constant([1.0, 2.0, 3.0]), 23.0, + reverse=True) + self.assertAllEqual([29.0, 28.0, 26.0], out) + + out = ctc_ops._scan( + lambda a, e: a + e, + constant_op.constant([1.0, 2.0, 3.0]), 23.0, + reverse=True, + inclusive=True) + self.assertAllEqual([29.0, 28.0, 26.0, 23.0], out) + + out = ctc_ops._scan( + lambda a, e: a + e, + constant_op.constant([[0.0, 1.0], [2.0, 3.0], [4.0, 5.0]]), + constant_op.constant([23.0, 24.0])) + self.assertAllEqual([[23.0, 25.0], [25.0, 28.0], [29.0, 33.0]], out) + + @test_util.run_deprecated_v1 + def testScanCapturesVariables(self): + with self.cached_session() as sess: + x = random_ops.random_uniform([]) + fn = lambda accum, elem: accum + x * elem + out = ctc_ops._scan(fn, constant_op.constant([0.0, 1.0, 2.0]), 23.0) + self.assertAllEqual(*sess.run([ + [23.0 + x * 0.0, 23.0 + x * 1.0, 23.0 + x * 3.0], out + ])) + + @test_util.run_deprecated_v1 + def testScanMultipleAccumulators(self): + with ops.device("/GPU:0" if test.is_gpu_available() else "/CPU:0"): + def fn(accum, elem): + accum_a, accum_b = accum + return accum_a + elem, accum_b * elem + out = ctc_ops._scan( + fn, constant_op.constant([1.0, 2.0, 3.0]), + (23.0, constant_op.constant([1.0, 2.0]))) + a, b = out + self.assertAllEqual([24.0, 26.0, 29.0], a) + self.assertAllEqual([[1.0, 2.0], [2.0, 4.0], [6.0, 12.0]], b) + + @test_util.run_deprecated_v1 + def testScanMultipleElements(self): + with ops.device("/GPU:0" if test.is_gpu_available() else "/CPU:0"): + def fn(accum, elem): + elem_a, elem_b = elem + return accum + (elem_a * elem_b) + elems_a = constant_op.constant([1.0, 2.0, 3.0]) + elems_b = constant_op.constant([[1.0, 2.0], [2.0, 3.0], [3.0, 4.0]]) + out = ctc_ops._scan( + fn, (elems_a, elems_b), + initial=constant_op.constant([0.0, 0.0])) + self.assertAllEqual( + [[1.0, 2.0], [5.0, 8.0], [14.0, 20.0]], out) if __name__ == "__main__": test.main() diff --git a/tensorflow/python/kernel_tests/cwise_ops_binary_test.py b/tensorflow/python/kernel_tests/cwise_ops_binary_test.py index 8028f93a8c5..49dbbb125a1 100644 --- a/tensorflow/python/kernel_tests/cwise_ops_binary_test.py +++ b/tensorflow/python/kernel_tests/cwise_ops_binary_test.py @@ -77,23 +77,23 @@ class BinaryOpTest(test.TestCase): def _compareCpu(self, x, y, np_func, tf_func, also_compare_variables=False): np_ans = np_func(x, y) - with self.test_session(use_gpu=False): + with test_util.force_cpu(): inx = ops.convert_to_tensor(x) iny = ops.convert_to_tensor(y) out = tf_func(inx, iny) - tf_cpu = out.eval() + tf_cpu = self.evaluate(out) # Test that the op takes precedence over numpy operators. - np_left = tf_func(x, iny).eval() - np_right = tf_func(inx, y).eval() + np_left = self.evaluate(tf_func(x, iny)) + np_right = self.evaluate(tf_func(inx, y)) if also_compare_variables: var_x = variables.Variable(x) var_y = variables.Variable(y) - variables.global_variables_initializer().run() + self.evaluate(variables.global_variables_initializer()) print(type(x), type(y), type(var_x), type(var_y)) print(type(tf_func(x, var_y)), type(tf_func(var_x, y))) - np_var_left = tf_func(x, var_y).eval() - np_var_right = tf_func(var_x, y).eval() + np_var_left = self.evaluate(tf_func(x, var_y)) + np_var_right = self.evaluate(tf_func(var_x, y)) if np_ans.dtype != np.object: self.assertAllClose(np_ans, tf_cpu) @@ -174,11 +174,11 @@ class BinaryOpTest(test.TestCase): def _compareGpu(self, x, y, np_func, tf_func): np_ans = np_func(x, y) - with self.test_session(force_gpu=test_util.is_gpu_available()): + with test_util.use_gpu(): inx = ops.convert_to_tensor(x) iny = ops.convert_to_tensor(y) out = tf_func(inx, iny) - tf_gpu = out.eval() + tf_gpu = self.evaluate(out) self.assertAllClose(np_ans, tf_gpu) self.assertShapeEqual(np_ans, out) # TODO(zhifengc/ke): make gradient checker work on GPU. @@ -196,6 +196,7 @@ class BinaryOpTest(test.TestCase): self._compareGradientY(x, y, np_func, tf_func) self._compareGpu(x, y, np_func, tf_func) + @test_util.run_deprecated_v1 def testFloatBasic(self): x = np.linspace(-5, 20, 15).reshape(1, 3, 5).astype(np.float32) y = np.linspace(20, -5, 15).reshape(1, 3, 5).astype(np.float32) @@ -233,6 +234,7 @@ class BinaryOpTest(test.TestCase): except ImportError as e: tf_logging.warn("Cannot test special functions: %s" % str(e)) + @test_util.run_deprecated_v1 def testFloatDifferentShapes(self): x = np.array([1, 2, 3, 4]).reshape(2, 2).astype(np.float32) y = np.array([1, 2]).reshape(2, 1).astype(np.float32) @@ -252,14 +254,17 @@ class BinaryOpTest(test.TestCase): y = np.array([1, 2]).reshape(2, 1).astype(np.int32) var_x = variables.Variable(x) var_y = variables.Variable(y) + with self.cached_session() as sess: - sess.run([var_x.initializer, var_y.initializer]) - left_result = (var_x * y).eval() - right_result = (x * var_y).eval() + self.evaluate([var_x.initializer, var_y.initializer]) + left_result = self.evaluate(var_x * y) + right_result = self.evaluate(x * var_y) + np_result = x * y self.assertAllEqual(np_result, left_result) self.assertAllEqual(np_result, right_result) + @test_util.run_deprecated_v1 def testDoubleBasic(self): x = np.linspace(-5, 20, 15).reshape(1, 3, 5).astype(np.float64) y = np.linspace(20, -5, 15).reshape(1, 3, 5).astype(np.float64) @@ -351,6 +356,7 @@ class BinaryOpTest(test.TestCase): self._compareBoth(x, y, np.floor_divide, _FLOORDIV) self._compareBoth(x, y, np.mod, _MOD) + @test_util.run_deprecated_v1 def testComplex64Basic(self): x = np.complex(1, 1) * np.linspace(-10, 10, 6).reshape(1, 3, 2).astype( np.complex64) @@ -365,6 +371,7 @@ class BinaryOpTest(test.TestCase): self._compareBoth(x, y, np.multiply, _MUL) self._compareBoth(x, y + 0.1, np.true_divide, _TRUEDIV) + @test_util.run_deprecated_v1 def testComplex128Basic(self): x = np.complex(1, 1) * np.linspace(-10, 10, 6).reshape(1, 3, 2).astype( np.complex128) @@ -382,10 +389,10 @@ class BinaryOpTest(test.TestCase): def testStringComparison(self): x = np.array([["abc", "bh"], ["c", ""]]) y = np.array([["abc", "bh"], ["def", "hi"]]) - with self.test_session(use_gpu=False) as sess: + with test_util.force_cpu(): cmp_eq = math_ops.equal(x, y) cmp_not_eq = math_ops.not_equal(x, y) - values = sess.run([cmp_eq, cmp_not_eq]) + values = self.evaluate([cmp_eq, cmp_not_eq]) self.assertAllEqual([[True, True], [False, False]], values[0]) self.assertAllEqual([[False, False], [True, True]], values[1]) @@ -478,198 +485,263 @@ class BinaryOpTest(test.TestCase): ] self._testBCastByFunc(funcs, xs, ys) + @test_util.run_deprecated_v1 def testBCast_0A(self): self._testBCastA([1, 3, 2], [1]) + @test_util.run_deprecated_v1 def testBCast_0B(self): self._testBCastB([1, 3, 2], [1]) + @test_util.run_deprecated_v1 def testBCast_0C(self): self._testBCastC([1, 3, 2], [1]) + @test_util.run_deprecated_v1 def testBCast_0D(self): self._testBCastD([1, 3, 2], [1]) + @test_util.run_deprecated_v1 def testBCast_1A(self): self._testBCastA([1, 3, 2], [2]) + @test_util.run_deprecated_v1 def testBCast_1B(self): self._testBCastB([1, 3, 2], [2]) + @test_util.run_deprecated_v1 def testBCast_1C(self): self._testBCastC([1, 3, 2], [2]) + @test_util.run_deprecated_v1 def testBCast_1D(self): self._testBCastD([1, 3, 2], [2]) + @test_util.run_deprecated_v1 def testBCast_2A(self): self._testBCastA([1, 3, 2], [3, 2]) + @test_util.run_deprecated_v1 def testBCast_2B(self): self._testBCastB([1, 3, 2], [3, 2]) + @test_util.run_deprecated_v1 def testBCast_2C(self): self._testBCastC([1, 3, 2], [3, 2]) + @test_util.run_deprecated_v1 def testBCast_2D(self): self._testBCastD([1, 3, 2], [3, 2]) + @test_util.run_deprecated_v1 def testBCast_3A(self): self._testBCastA([1, 3, 2], [3, 1]) + @test_util.run_deprecated_v1 def testBCast_3B(self): self._testBCastB([1, 3, 2], [3, 1]) + @test_util.run_deprecated_v1 def testBCast_3C(self): self._testBCastC([1, 3, 2], [3, 1]) + @test_util.run_deprecated_v1 def testBCast_3D(self): self._testBCastD([1, 3, 2], [3, 1]) + @test_util.run_deprecated_v1 def testBCast_4A(self): self._testBCastA([1, 3, 2], [1, 3, 2]) + @test_util.run_deprecated_v1 def testBCast_4B(self): self._testBCastB([1, 3, 2], [1, 3, 2]) + @test_util.run_deprecated_v1 def testBCast_4C(self): self._testBCastC([1, 3, 2], [1, 3, 2]) + @test_util.run_deprecated_v1 def testBCast_4D(self): self._testBCastD([1, 3, 2], [1, 3, 2]) + @test_util.run_deprecated_v1 def testBCast_5A(self): self._testBCastA([1, 3, 2], [2, 3, 1]) + @test_util.run_deprecated_v1 def testBCast_5B(self): self._testBCastB([1, 3, 2], [2, 3, 1]) + @test_util.run_deprecated_v1 def testBCast_5C(self): self._testBCastC([1, 3, 2], [2, 3, 1]) + @test_util.run_deprecated_v1 def testBCast_5D(self): self._testBCastD([1, 3, 2], [2, 3, 1]) + @test_util.run_deprecated_v1 def testBCast_6A(self): self._testBCastA([1, 3, 2], [2, 1, 1]) + @test_util.run_deprecated_v1 def testBCast_6B(self): self._testBCastB([1, 3, 2], [2, 1, 1]) + @test_util.run_deprecated_v1 def testBCast_6C(self): self._testBCastC([1, 3, 2], [2, 1, 1]) + @test_util.run_deprecated_v1 def testBCast_6D(self): self._testBCastD([1, 3, 2], [2, 1, 1]) + @test_util.run_deprecated_v1 def testBCast_7A(self): self._testBCastA([1, 3, 2], [1, 3, 1]) + @test_util.run_deprecated_v1 def testBCast_7B(self): self._testBCastB([1, 3, 2], [1, 3, 1]) + @test_util.run_deprecated_v1 def testBCast_7C(self): self._testBCastC([1, 3, 2], [1, 3, 1]) + @test_util.run_deprecated_v1 def testBCast_7D(self): self._testBCastD([1, 3, 2], [1, 3, 1]) + @test_util.run_deprecated_v1 def testBCast_8A(self): self._testBCastA([2, 1, 5], [2, 3, 1]) + @test_util.run_deprecated_v1 def testBCast_8B(self): self._testBCastB([2, 1, 5], [2, 3, 1]) + @test_util.run_deprecated_v1 def testBCast_8C(self): self._testBCastC([2, 1, 5], [2, 3, 1]) + @test_util.run_deprecated_v1 def testBCast_8D(self): self._testBCastD([2, 1, 5], [2, 3, 1]) + @test_util.run_deprecated_v1 def testBCast_9A(self): self._testBCastA([2, 0, 5], [2, 0, 1]) + @test_util.run_deprecated_v1 def testBCast_9B(self): self._testBCastB([2, 0, 5], [2, 0, 1]) + @test_util.run_deprecated_v1 def testBCast_9C(self): self._testBCastC([2, 0, 5], [2, 0, 1]) + @test_util.run_deprecated_v1 def testBCast_9D(self): self._testBCastD([2, 0, 5], [2, 0, 1]) + @test_util.run_deprecated_v1 def testBCast_10A(self): self._testBCastA([2, 3, 0], [2, 3, 1]) + @test_util.run_deprecated_v1 def testBCast_10B(self): self._testBCastB([2, 3, 0], [2, 3, 1]) + @test_util.run_deprecated_v1 def testBCast_10C(self): self._testBCastC([2, 3, 0], [2, 3, 1]) + @test_util.run_deprecated_v1 def testBCast_10D(self): self._testBCastD([2, 3, 0], [2, 3, 1]) + @test_util.run_deprecated_v1 def testBCast_11A(self): self._testBCastA([1, 3, 2], [1, 3, 2]) + @test_util.run_deprecated_v1 def testBCast_11B(self): self._testBCastB([1, 3, 2], [1, 3, 2]) + @test_util.run_deprecated_v1 def testBCast_11C(self): self._testBCastC([1, 3, 2], [1, 3, 2]) + @test_util.run_deprecated_v1 def testBCast_11D(self): self._testBCastD([1, 3, 2], [1, 3, 2]) + @test_util.run_deprecated_v1 def testBCast_12A(self): self._testBCastA([1, 1, 1, 1, 3, 2], [1, 3, 2]) + @test_util.run_deprecated_v1 def testBCast_12B(self): self._testBCastB([1, 1, 1, 1, 3, 2], [1, 3, 2]) + @test_util.run_deprecated_v1 def testBCast_12C(self): self._testBCastC([1, 1, 1, 1, 3, 2], [1, 3, 2]) + @test_util.run_deprecated_v1 def testBCast_12D(self): self._testBCastD([1, 1, 1, 1, 3, 2], [1, 3, 2]) + @test_util.run_deprecated_v1 def testBCast_13A(self): self._testBCastA([1, 3, 2, 1, 1], [1]) + @test_util.run_deprecated_v1 def testBCast_13B(self): self._testBCastB([1, 3, 2, 1, 1], [1]) + @test_util.run_deprecated_v1 def testBCast_13C(self): self._testBCastC([1, 3, 2, 1, 1], [1]) + @test_util.run_deprecated_v1 def testBCast_13D(self): self._testBCastD([1, 3, 2, 1, 1], [1]) + @test_util.run_deprecated_v1 def testBCast_14A(self): self._testBCastA([2, 3, 1, 1, 5], [1]) + @test_util.run_deprecated_v1 def testBCast_14B(self): self._testBCastB([2, 3, 1, 1, 5], [1]) + @test_util.run_deprecated_v1 def testBCast_14C(self): self._testBCastC([2, 3, 1, 1, 5], [1]) + @test_util.run_deprecated_v1 def testBCast_14D(self): self._testBCastD([2, 3, 1, 1, 5], [1]) + @test_util.run_deprecated_v1 def testBCast_15A(self): self._testBCastA([10, 3, 1, 2], [3, 1, 2]) + @test_util.run_deprecated_v1 def testBCast_15B(self): self._testBCastB([10, 3, 1, 2], [3, 1, 2]) + @test_util.run_deprecated_v1 def testBCast_15C(self): self._testBCastC([10, 3, 1, 2], [3, 1, 2]) + @test_util.run_deprecated_v1 def testBCast_15D(self): self._testBCastD([10, 3, 1, 2], [3, 1, 2]) + @test_util.run_deprecated_v1 def testMismatchedDimensions(self): for func in [ math_ops.add, math_ops.subtract, math_ops.multiply, math_ops.div, _ADD, @@ -681,6 +753,7 @@ class BinaryOpTest(test.TestCase): ops.convert_to_tensor([10.0, 20.0, 30.0]), ops.convert_to_tensor([[40.0, 50.0], [60.0, 70.0]])) + @test_util.run_deprecated_v1 def testZeroPowGrad(self): with self.cached_session(): for dtype in (np.float16, np.float32, np.float64, np.complex64, @@ -691,6 +764,7 @@ class BinaryOpTest(test.TestCase): error = gradient_checker.compute_gradient_error(y, [], z, []) self.assertEqual(error, 0) + @test_util.run_deprecated_v1 def testComplexPowGrad(self): with self.cached_session(): for dtype in np.complex64, np.complex128: @@ -716,39 +790,39 @@ class BinaryOpTest(test.TestCase): def testPowNegativeExponent(self): for dtype in [np.int32, np.int64]: - with self.test_session(use_gpu=False) as sess: + with test_util.force_cpu(): with self.assertRaisesRegexp( errors_impl.InvalidArgumentError, "Integers to negative integer powers are not allowed"): x = np.array([5, 2]).astype(dtype) y = np.array([-2, 3]).astype(dtype) - sess.run(math_ops.pow(x, y)) + self.evaluate(math_ops.pow(x, y)) - with self.test_session(use_gpu=False) as sess: + with test_util.force_cpu(): with self.assertRaisesRegexp( errors_impl.InvalidArgumentError, "Integers to negative integer powers are not allowed"): x = np.array([5, 2]).astype(dtype) y = np.array([2, -3]).astype(dtype) - sess.run(math_ops.pow(x, y)) + self.evaluate(math_ops.pow(x, y)) - with self.test_session(use_gpu=False) as sess: + with test_util.force_cpu(): with self.assertRaisesRegexp( errors_impl.InvalidArgumentError, "Integers to negative integer powers are not allowed"): x = np.array([5, 2]).astype(dtype) y = -3 - sess.run(math_ops.pow(x, y)) + self.evaluate(math_ops.pow(x, y)) class ComparisonOpTest(test.TestCase): def _compareScalar(self, func, x, y, dtype): - with self.test_session(force_gpu=test_util.is_gpu_available()): + with test_util.use_gpu(): out = func( ops.convert_to_tensor(np.array([x]).astype(dtype)), ops.convert_to_tensor(np.array([y]).astype(dtype))) - ret = out.eval() + ret = self.evaluate(out) return ret[0] def testScalarCompareScalar(self): @@ -777,9 +851,9 @@ class ComparisonOpTest(test.TestCase): def _compare(self, x, y, np_func, tf_func): np_ans = np_func(x, y) - with self.test_session(force_gpu=test_util.is_gpu_available()): + with test_util.use_gpu(): out = tf_func(ops.convert_to_tensor(x), ops.convert_to_tensor(y)) - tf_ans = out.eval() + tf_ans = self.evaluate(out) self.assertAllEqual(np_ans, tf_ans) def testTensorCompareTensor(self): @@ -859,6 +933,7 @@ class ComparisonOpTest(test.TestCase): self._testBCastByFunc( np.not_equal, math_ops.not_equal, include_complex=True) + @test_util.run_deprecated_v1 def testShapeMismatch(self): dtypes = [np.float16, np.float32, np.float64, np.int32, np.int64] funcs = [ diff --git a/tensorflow/python/kernel_tests/cwise_ops_test.py b/tensorflow/python/kernel_tests/cwise_ops_test.py index c5311ad834a..9bb7d8b8b12 100644 --- a/tensorflow/python/kernel_tests/cwise_ops_test.py +++ b/tensorflow/python/kernel_tests/cwise_ops_test.py @@ -84,11 +84,11 @@ def _default_tolerance(dtype): class ComparisonOpTest(test.TestCase): def _compareScalar(self, func, x, y, dtype): - with self.test_session(force_gpu=test_util.is_gpu_available()): + with test_util.use_gpu(): out = func( ops.convert_to_tensor(np.array([x]).astype(dtype)), ops.convert_to_tensor(np.array([y]).astype(dtype))) - ret = out.eval() + ret = self.evaluate(out) return ret[0] def testScalarCompareScalar(self): @@ -117,9 +117,9 @@ class ComparisonOpTest(test.TestCase): def _compare(self, x, y, np_func, tf_func): np_ans = np_func(x, y) - with self.test_session(force_gpu=test_util.is_gpu_available()): + with test_util.use_gpu(): out = tf_func(ops.convert_to_tensor(x), ops.convert_to_tensor(y)) - tf_ans = out.eval() + tf_ans = self.evaluate(out) self.assertAllEqual(np_ans, tf_ans) def testTensorCompareTensor(self): @@ -199,6 +199,7 @@ class ComparisonOpTest(test.TestCase): self._testBCastByFunc( np.not_equal, math_ops.not_equal, include_complex=True) + @test_util.run_deprecated_v1 def testShapeMismatch(self): dtypes = [np.float16, np.float32, np.float64, np.int32, np.int64] funcs = [ @@ -218,22 +219,20 @@ class LogicalOpTest(test.TestCase): def _compareBinary(self, x, y, np_func, tf_func, use_gpu=False): np_ans = np_func(x, y) - with self.test_session(use_gpu=use_gpu, - force_gpu=use_gpu and test_util.is_gpu_available()): + with test_util.device(use_gpu=use_gpu): inx = ops.convert_to_tensor(x) iny = ops.convert_to_tensor(y) out = tf_func(inx, iny) - tf_val = out.eval() + tf_val = self.evaluate(out) self.assertEqual(out.dtype, dtypes_lib.bool) self.assertAllEqual(np_ans, tf_val) self.assertShapeEqual(np_ans, out) def _not(self, x, use_gpu=False): np_ans = np.logical_not(x) - with self.test_session(use_gpu=use_gpu, - force_gpu=use_gpu and test_util.is_gpu_available()): + with test_util.device(use_gpu=use_gpu): out = math_ops.logical_not(ops.convert_to_tensor(x)) - tf_val = out.eval() + tf_val = self.evaluate(out) self.assertEqual(out.dtype, dtypes_lib.bool) self.assertAllEqual(np_ans, tf_val) self.assertShapeEqual(np_ans, out) @@ -282,6 +281,7 @@ class LogicalOpTest(test.TestCase): self._compareBinary(x, y, np.logical_or, math_ops.logical_or, use_gpu) self._compareBinary(x, y, np.logical_xor, math_ops.logical_xor, use_gpu) + @test_util.run_deprecated_v1 def testShapeMismatch(self): x = np.random.randint(0, 2, 6).astype(np.bool).reshape(1, 3, 2) y = np.random.randint(0, 2, 6).astype(np.bool).reshape(3, 2, 1) @@ -290,6 +290,7 @@ class LogicalOpTest(test.TestCase): ValueError, lambda e: "Dimensions must" in str(e)): f(x, y) + @test_util.run_deprecated_v1 def testUsingAsPythonValueFails(self): # Ensure that we raise an error when the user attempts to treat a # `Tensor` as a Python `bool`. @@ -316,10 +317,9 @@ class SelectOpTest(test.TestCase): def _compare(self, c, x, y, use_gpu): np_ans = np.where(c, x, y) - with self.test_session(use_gpu=use_gpu, - force_gpu=use_gpu and test_util.is_gpu_available()): + with test_util.device(use_gpu=use_gpu): out = array_ops.where(c, x, y) - tf_ans = out.eval() + tf_ans = self.evaluate(out) self.assertAllEqual(np_ans, tf_ans) self.assertShapeEqual(np_ans, out) @@ -399,6 +399,7 @@ class SelectOpTest(test.TestCase): if t in [np.float16, np.float32, np.float64]: self._compare(c, xt, yt, use_gpu=True) + @test_util.run_deprecated_v1 def testGradients(self): c = np.random.randint(0, 2, 6).astype(np.bool).reshape(1, 3, 2) x = np.random.rand(1, 3, 2) * 100 @@ -418,6 +419,7 @@ class SelectOpTest(test.TestCase): self._compareGradientX(c, xt, yt) self._compareGradientY(c, xt, yt) + @test_util.run_deprecated_v1 def testShapeMismatch(self): c = np.random.randint(0, 2, 6).astype(np.bool).reshape(1, 3, 2) x = np.random.rand(1, 3, 2) * 100 @@ -431,6 +433,7 @@ class SelectOpTest(test.TestCase): with self.assertRaises(ValueError): array_ops.where(c, xt, yt) + @test_util.run_deprecated_v1 def testEmptyTensor(self): c = np.random.randint(0, 3, 0).astype(np.bool).reshape(1, 3, 0) x = np.random.rand(1, 3, 0) * 100 @@ -442,6 +445,7 @@ class SelectOpTest(test.TestCase): z = array_ops.where(c, xt, yt).eval() self.assertAllEqual(z_expected, z) + @test_util.run_deprecated_v1 def testNan(self): """Verify that nans don't propagate where they shouldn't.""" with self.cached_session(): @@ -460,10 +464,9 @@ class BatchSelectOpTest(test.TestCase): np_ans = np.dstack( [x_i if c_i else y_i for c_i, x_i, y_i in zip(c, x, y)]).transpose( [2, 0, 1]) - with self.test_session(use_gpu=use_gpu, - force_gpu=use_gpu and test_util.is_gpu_available()): + with test_util.device(use_gpu=use_gpu): out = array_ops.where(c, x, y) - tf_ans = out.eval() + tf_ans = self.evaluate(out) self.assertAllEqual(np_ans, tf_ans) self.assertShapeEqual(np_ans, out) @@ -529,6 +532,7 @@ class BatchSelectOpTest(test.TestCase): if t in [np.float16, np.float32, np.float64]: self._compare(c, xt, yt, use_gpu=True) + @test_util.run_deprecated_v1 def testGradients(self): c = np.random.randint(0, 2, 16).astype(np.bool) x = np.random.rand(16, 2, 8) * 100 @@ -548,6 +552,7 @@ class BatchSelectOpTest(test.TestCase): self._compareGradientX(c, xt, yt) self._compareGradientY(c, xt, yt) + @test_util.run_deprecated_v1 def testShapeMismatch(self): c = np.random.randint(0, 2, 8).astype(np.bool) x = np.random.rand(16, 3, 2) * 100 @@ -566,13 +571,11 @@ class MinMaxOpTest(test.TestCase): def _compare(self, x, y, use_gpu): np_min, np_max = np.minimum(x, y), np.maximum(x, y) - with self.test_session( - use_gpu=use_gpu, - force_gpu=use_gpu and test_util.is_gpu_available()) as sess: + with test_util.device(use_gpu=use_gpu): inx = ops.convert_to_tensor(x) iny = ops.convert_to_tensor(y) omin, omax = math_ops.minimum(inx, iny), math_ops.maximum(inx, iny) - tf_min, tf_max = sess.run([omin, omax]) + tf_min, tf_max = self.evaluate([omin, omax]) self.assertAllEqual(np_min, tf_min) self.assertAllEqual(np_max, tf_max) @@ -628,6 +631,7 @@ class MinMaxOpTest(test.TestCase): elif x.dtype == np.float64: self.assertAllClose(jacob_t, jacob_n, rtol=1e-5, atol=1e-5) + @test_util.run_deprecated_v1 def testGradients(self): x = np.random.rand(1, 3, 2) * 100. # ensure x != y @@ -641,16 +645,16 @@ class MinMaxOpTest(test.TestCase): class MathOpsOverloadTest(test.TestCase): def _computeTensorAndLiteral(self, x, y, dtype, func): - with self.test_session(use_gpu=False): + with test_util.force_cpu(): inx = ops.convert_to_tensor(x, dtype=dtype) z = func(inx, y) # Should use __add__, __sub__, etc. - return z.eval() + return self.evaluate(z) def _computeLiteralAndTensor(self, x, y, dtype, func): - with self.test_session(use_gpu=False): + with test_util.force_cpu(): iny = ops.convert_to_tensor(y, dtype=dtype) z = func(x, iny) # Should use __radd__, __rsub__, etc. - return z.eval() + return self.evaluate(z) def _compareBinary(self, x, y, dtype, np_func, tf_func): np_ans = np_func(x, y).astype(dtype.as_numpy_dtype) @@ -661,9 +665,9 @@ class MathOpsOverloadTest(test.TestCase): def _compareUnary(self, x, dtype, np_func, tf_func): np_ans = np_func(x).astype(dtype.as_numpy_dtype) - with self.test_session(use_gpu=False): - self.assertAllClose(np_ans, - tf_func(ops.convert_to_tensor(x, dtype=dtype)).eval()) + with test_util.force_cpu(): + self.assertAllClose( + np_ans, self.evaluate(tf_func(ops.convert_to_tensor(x, dtype=dtype)))) def testOverload(self): dtypes = [ @@ -730,13 +734,11 @@ class IsFiniteInfNanTest(test.TestCase): def _compare(self, x, use_gpu): np_finite, np_inf, np_nan = np.isfinite(x), np.isinf(x), np.isnan(x) - with self.test_session( - use_gpu=use_gpu, - force_gpu=use_gpu and test_util.is_gpu_available()) as sess: + with test_util.device(use_gpu=use_gpu): inx = ops.convert_to_tensor(x) ofinite, oinf, onan = math_ops.is_finite(inx), math_ops.is_inf( inx), math_ops.is_nan(inx) - tf_finite, tf_inf, tf_nan = sess.run([ofinite, oinf, onan]) + tf_finite, tf_inf, tf_nan = self.evaluate([ofinite, oinf, onan]) self.assertAllEqual(np_inf, tf_inf) self.assertAllEqual(np_nan, tf_nan) self.assertAllEqual(np_finite, tf_finite) @@ -773,31 +775,33 @@ class IsFiniteInfNanTest(test.TestCase): x = np.full((size,), value, dtype=dtype) np_y = np.sqrt(x) np_nan = np.isnan(np_y) - with self.test_session(force_gpu=test_util.is_gpu_available()): + with test_util.use_gpu(): tf_y = math_ops.sqrt(x) tf_nan = math_ops.is_nan(tf_y) if value < 0: - self.assertAllEqual(np_nan, tf_nan.eval()) + self.assertAllEqual(np_nan, self.evaluate(tf_nan)) else: - self.assertAllCloseAccordingToType(np_y, tf_y.eval()) + self.assertAllCloseAccordingToType(np_y, self.evaluate(tf_y)) class RoundingTest(test.TestCase): def _compare_values(self, x, y=None): y = np.rint(x) if y is None else np.asarray(y) - with self.cached_session() as sess: - tf_rint = math_ops.rint(x) - np_rint = sess.run(tf_rint) + + tf_rint = math_ops.rint(x) + np_rint = self.evaluate(tf_rint) + self.assertAllEqual(y, np_rint) self.assertShapeEqual(y, tf_rint) def _compare(self, x): np_floor, np_ceil = np.floor(x), np.ceil(x) - with self.cached_session() as sess: - inx = ops.convert_to_tensor(x) - ofloor, oceil = math_ops.floor(inx), math_ops.ceil(inx) - tf_floor, tf_ceil = sess.run([ofloor, oceil]) + + inx = ops.convert_to_tensor(x) + ofloor, oceil = math_ops.floor(inx), math_ops.ceil(inx) + tf_floor, tf_ceil = self.evaluate([ofloor, oceil]) + self.assertAllEqual(np_floor, tf_floor) self.assertAllEqual(np_ceil, tf_ceil) self.assertShapeEqual(np_floor, ofloor) @@ -828,12 +832,13 @@ class ComplexMakeRealImagTest(test.TestCase): def _compareMake(self, real, imag, use_gpu): np_ans = real + (1j) * imag - with self.test_session(use_gpu=use_gpu, - force_gpu=use_gpu and test_util.is_gpu_available()): + + with test_util.device(use_gpu=use_gpu): real = ops.convert_to_tensor(real) imag = ops.convert_to_tensor(imag) tf_ans = math_ops.complex(real, imag) - out = tf_ans.eval() + out = self.evaluate(tf_ans) + self.assertAllEqual(np_ans, out) self.assertShapeEqual(np_ans, tf_ans) @@ -848,17 +853,17 @@ class ComplexMakeRealImagTest(test.TestCase): def _compareRealImag(self, cplx, use_gpu): np_real, np_imag = np.real(cplx), np.imag(cplx) np_zeros = np_real * 0 - with self.test_session(use_gpu=use_gpu, - force_gpu=use_gpu and test_util.is_gpu_available()): + + with test_util.device(use_gpu=use_gpu): inx = ops.convert_to_tensor(cplx) tf_real = math_ops.real(inx) tf_imag = math_ops.imag(inx) tf_real_real = math_ops.real(tf_real) tf_imag_real = math_ops.imag(tf_real) - self.assertAllEqual(np_real, tf_real.eval()) - self.assertAllEqual(np_imag, tf_imag.eval()) - self.assertAllEqual(np_real, tf_real_real.eval()) - self.assertAllEqual(np_zeros, tf_imag_real.eval()) + self.assertAllEqual(np_real, self.evaluate(tf_real)) + self.assertAllEqual(np_imag, self.evaluate(tf_imag)) + self.assertAllEqual(np_real, self.evaluate(tf_real_real)) + self.assertAllEqual(np_zeros, self.evaluate(tf_imag_real)) def testRealImag64(self): real = (np.arange(-3, 3) / 4.).reshape([1, 3, 2]).astype(np.float32) @@ -876,12 +881,12 @@ class ComplexMakeRealImagTest(test.TestCase): def _compareAngle(self, cplx, use_gpu): np_angle = np.angle(cplx) - with self.test_session( - use_gpu=use_gpu, - force_gpu=use_gpu and test_util.is_gpu_available()) as sess: + + with test_util.device(use_gpu=use_gpu): inx = ops.convert_to_tensor(cplx) tf_angle = math_ops.angle(inx) - tf_angle_val = sess.run(tf_angle) + tf_angle_val = self.evaluate(tf_angle) + self.assertAllEqual(np_angle, tf_angle_val) self.assertShapeEqual(np_angle, tf_angle) @@ -903,6 +908,7 @@ class ComplexMakeRealImagTest(test.TestCase): # build failures on GPU (See #10643 for context). # self._compareAngle(cplx, use_gpu=True) + @test_util.run_deprecated_v1 def testRealReal(self): for dtype in (dtypes_lib.int32, dtypes_lib.int64, dtypes_lib.float32, dtypes_lib.float64): @@ -912,11 +918,10 @@ class ComplexMakeRealImagTest(test.TestCase): def _compareConj(self, cplx, use_gpu): np_ans = np.conj(cplx) - with self.test_session(use_gpu=use_gpu, - force_gpu=use_gpu and test_util.is_gpu_available()): + with test_util.device(use_gpu=use_gpu): inx = ops.convert_to_tensor(cplx) tf_conj = math_ops.conj(inx) - tf_ans = tf_conj.eval() + tf_ans = self.evaluate(tf_conj) self.assertAllEqual(np_ans, tf_ans) self.assertShapeEqual(np_ans, tf_conj) @@ -934,6 +939,7 @@ class ComplexMakeRealImagTest(test.TestCase): self._compareConj(cplx, use_gpu=False) self._compareConj(cplx, use_gpu=True) + @test_util.run_deprecated_v1 def testConjReal(self): for dtype in (dtypes_lib.int32, dtypes_lib.int64, dtypes_lib.float16, dtypes_lib.float32, dtypes_lib.float64): @@ -941,6 +947,7 @@ class ComplexMakeRealImagTest(test.TestCase): y = math_ops.conj(x) self.assertEqual(x, y) + @test_util.run_deprecated_v1 def testConjString(self): x = array_ops.placeholder(dtypes_lib.string) with self.assertRaisesRegexp(TypeError, @@ -977,6 +984,7 @@ class ComplexMakeRealImagTest(test.TestCase): x_, list(x.shape), z, [1], x_init_value=x, delta=epsilon) self.assertAllClose(jacob_t, jacob_n, rtol=epsilon, atol=epsilon) + @test_util.run_deprecated_v1 def testGradient(self): # complex64 data = np.arange(1, 2, 0.10).reshape([5, 2]).astype(np.float32) @@ -1012,6 +1020,7 @@ class ComplexMakeRealImagTest(test.TestCase): inp, list(data.shape), loss, [1], x_init_value=data, delta=epsilon) self.assertAllClose(jacob_t, jacob_n, rtol=epsilon, atol=epsilon) + @test_util.run_deprecated_v1 def testMulGradient(self): data = np.arange(1, 2, 0.125).reshape([2, 4]).astype(np.float32) self._compareMulGradient(data) @@ -1032,13 +1041,13 @@ class AccumulateTest(test.TestCase): np_val = random_arrays[0] for random_array in random_arrays[1:]: np_val += random_array - self.assertAllClose(np_val, tf_val.eval()) + self.assertAllClose(np_val, self.evaluate(tf_val)) def testZeroArgs(self): with self.cached_session(): with self.assertRaises(ValueError): tf_val = math_ops.accumulate_n([]) - tf_val.eval() + self.evaluate(tf_val) def testWrongShape(self): with self.cached_session(): @@ -1070,7 +1079,7 @@ class PolyvalTest(test.TestCase): np_val = np.polyval(coeffs, x) with self.cached_session(): tf_val = math_ops.polyval(coeffs, x) - self.assertAllClose(np_val, tf_val.eval()) + self.assertAllClose(np_val, self.evaluate(tf_val)) def testSimple(self): for dtype in [ @@ -1093,7 +1102,7 @@ class PolyvalTest(test.TestCase): np_val = np.polyval(coeffs, x) with self.cached_session(): tf_val = math_ops.polyval(coeffs, x) - self.assertAllClose(np_val, tf_val.eval()) + self.assertAllClose(np_val, self.evaluate(tf_val)) def testEmpty(self): x = np.random.rand(2, 2).astype(np.float32) @@ -1101,7 +1110,7 @@ class PolyvalTest(test.TestCase): np_val = np.polyval(coeffs, x) with self.cached_session(): tf_val = math_ops.polyval(coeffs, x) - self.assertAllClose(np_val, tf_val.eval()) + self.assertAllClose(np_val, self.evaluate(tf_val)) if __name__ == "__main__": diff --git a/tensorflow/python/kernel_tests/cwise_ops_unary_test.py b/tensorflow/python/kernel_tests/cwise_ops_unary_test.py index 77f182784eb..709a20f3d0d 100644 --- a/tensorflow/python/kernel_tests/cwise_ops_unary_test.py +++ b/tensorflow/python/kernel_tests/cwise_ops_unary_test.py @@ -76,7 +76,7 @@ class UnaryOpTest(test.TestCase): if grad_atol is None: grad_atol = _default_tolerance(x.dtype) np_ans = np_func(x) - with self.test_session(use_gpu=False): + with self.cached_session(use_gpu=False): inx = ops.convert_to_tensor(x) if x.dtype in (np.float32, np.float64, dtypes_lib.bfloat16.as_numpy_dtype): @@ -84,7 +84,7 @@ class UnaryOpTest(test.TestCase): np_ans *= 1.1 else: y = tf_func(inx) - tf_cpu = y.eval() + tf_cpu = self.evaluate(y) self.assertShapeEqual(np_ans, y) if x.dtype == np.float16: self.assertAllClose(np_ans, tf_cpu, rtol=1e-3, atol=1e-3) @@ -121,26 +121,24 @@ class UnaryOpTest(test.TestCase): def _check(self, result_tensor, result_np, input_sp_t, tol): self.assertTrue(isinstance(result_tensor, sparse_tensor.SparseTensor)) self.assertTrue(isinstance(input_sp_t, sparse_tensor.SparseTensor)) - self.assertAllEqual(input_sp_t.indices.eval(), result_tensor.indices.eval()) - self.assertAllEqual(input_sp_t.dense_shape.eval(), - result_tensor.dense_shape.eval()) + self.assertAllEqual(input_sp_t.indices, result_tensor.indices) + self.assertAllEqual(input_sp_t.dense_shape, result_tensor.dense_shape) if tol is None: - self.assertAllClose(result_np, result_tensor.values.eval()) + self.assertAllClose(result_np, result_tensor.values) else: - self.assertAllClose( - result_np, result_tensor.values.eval(), rtol=tol, atol=tol) + self.assertAllClose(result_np, result_tensor.values, rtol=tol, atol=tol) def _compareSparseCpu(self, x, np_func, tf_func, tol): x_sp, x_sp_vals = _sparsify(x) res_np = np_func(x_sp_vals) - with self.test_session(use_gpu=False): + with test_util.force_cpu(): self._check(tf_func(x_sp), res_np, x_sp, tol) def _compareGpu(self, x, np_func, tf_func): np_ans = np_func(x) - with self.test_session(force_gpu=test_util.is_gpu_available()): + with test_util.use_gpu(): result = tf_func(ops.convert_to_tensor(x)) - tf_gpu = result.eval() + tf_gpu = self.evaluate(result) if x.dtype == np.float16: self.assertAllClose(np_ans, tf_gpu, rtol=1e-3, atol=1e-3) else: @@ -150,7 +148,7 @@ class UnaryOpTest(test.TestCase): def _compareSparseGpu(self, x, np_func, tf_func, tol): x_sp, x_sp_vals = _sparsify(x) res_np = np_func(x_sp_vals) - with self.test_session(force_gpu=test_util.is_gpu_available()): + with test_util.use_gpu(): self._check(tf_func(x_sp), res_np, x_sp, tol) def _compareBoth(self, x, np_func, tf_func): @@ -186,6 +184,7 @@ class UnaryOpTest(test.TestCase): return func + @test_util.run_deprecated_v1 def testFloatBasic(self): x = np.arange(-3, 3).reshape(1, 3, 2).astype(np.float32) w = x - x.min() + 1.02 # all greater than 1 @@ -240,12 +239,14 @@ class UnaryOpTest(test.TestCase): self._compareBothSparse(y, np.sign, math_ops.sign) self._compareBothSparse(x, np.vectorize(math.erf), math_ops.erf) + @test_util.run_deprecated_v1 def testFloatTanhEdge(self): x = np.arange(40, 40 + 6).reshape(6).astype(np.float32) self._compareBoth(x, np.tanh, math_ops.tanh) x = np.arange(-40, -40 + 6).reshape(6).astype(np.float32) self._compareBoth(x, np.tanh, math_ops.tanh) + @test_util.run_deprecated_v1 def testFloatEmpty(self): x = np.empty((2, 0, 5), dtype=np.float32) self._compareBoth(x, np.abs, math_ops.abs) @@ -291,6 +292,7 @@ class UnaryOpTest(test.TestCase): self._compareBothSparse(x, np.sign, math_ops.sign) self._compareBothSparse(x, np.sign, math_ops.erf) + @test_util.run_deprecated_v1 def testDoubleBasic(self): x = np.arange(-3, 3).reshape(1, 3, 2).astype(np.float64) w = x - x.min() + 1.02 # all greater than 1 @@ -344,6 +346,7 @@ class UnaryOpTest(test.TestCase): self._compareBothSparse(y, np.sign, math_ops.sign) self._compareBothSparse(x, np.vectorize(math.erf), math_ops.erf) + @test_util.run_deprecated_v1 def testHalfBasic(self): x = np.arange(-3, 3).reshape(1, 3, 2).astype(np.float16) y = (x + .5).astype(np.float16) # no zero @@ -416,6 +419,7 @@ class UnaryOpTest(test.TestCase): self._compareCpu(x, np.square, math_ops.square) self._compareBothSparse(x, np.square, math_ops.square) + @test_util.run_deprecated_v1 def testComplex64Basic(self): x = np.complex(1, 1) * np.arange(-3, 3).reshape(1, 3, 2).astype( np.complex64) @@ -460,6 +464,7 @@ class UnaryOpTest(test.TestCase): self._compareBoth(y, complex_sign, math_ops.sign) self._compareBothSparse(y, complex_sign, math_ops.sign) + @test_util.run_deprecated_v1 def testComplex128Basic(self): x = np.complex(1, 1) * np.arange(-3, 3).reshape(1, 3, 2).astype( np.complex128) @@ -499,6 +504,7 @@ class UnaryOpTest(test.TestCase): self._compareBoth(y, complex_sign, math_ops.sign) self._compareBothSparse(y, complex_sign, math_ops.sign) + @test_util.run_deprecated_v1 def testGradGrad(self): np.random.seed(7) shape = (5,) diff --git a/tensorflow/python/kernel_tests/decode_bmp_op_test.py b/tensorflow/python/kernel_tests/decode_bmp_op_test.py index eebaffbe13a..5e7991382ed 100644 --- a/tensorflow/python/kernel_tests/decode_bmp_op_test.py +++ b/tensorflow/python/kernel_tests/decode_bmp_op_test.py @@ -61,7 +61,7 @@ class DecodeBmpOpTest(test.TestCase): decode = array_ops.squeeze(image_ops.decode_bmp(img_in)) with self.cached_session(): - decoded = decode.eval() + decoded = self.evaluate(decode) self.assertAllEqual(decoded, img_bytes) def testGrayscale(self): @@ -136,7 +136,7 @@ class DecodeBmpOpTest(test.TestCase): decode = image_ops.decode_bmp(img_in) with self.cached_session(): - decoded = decode.eval() + decoded = self.evaluate(decode) self.assertAllEqual(decoded, img_bytes) diff --git a/tensorflow/python/kernel_tests/decode_compressed_op_test.py b/tensorflow/python/kernel_tests/decode_compressed_op_test.py index 1cc1c7da30a..fd871c00906 100644 --- a/tensorflow/python/kernel_tests/decode_compressed_op_test.py +++ b/tensorflow/python/kernel_tests/decode_compressed_op_test.py @@ -24,6 +24,7 @@ import zlib from six import BytesIO from tensorflow.python.framework import dtypes +from tensorflow.python.framework import test_util from tensorflow.python.ops import array_ops from tensorflow.python.ops import parsing_ops from tensorflow.python.platform import test @@ -42,6 +43,7 @@ class DecodeCompressedOpTest(test.TestCase): f.write(bytes_in) return out.getvalue() + @test_util.run_deprecated_v1 def testDecompress(self): for compression_type in ["ZLIB", "GZIP", ""]: with self.cached_session(): @@ -55,6 +57,7 @@ class DecodeCompressedOpTest(test.TestCase): self._compress(b"bBbb", compression_type)]}) self.assertAllEqual([b"AaAA", b"bBbb"], result) + @test_util.run_deprecated_v1 def testDecompressWithRaw(self): for compression_type in ["ZLIB", "GZIP", ""]: with self.cached_session(): diff --git a/tensorflow/python/kernel_tests/decode_image_op_test.py b/tensorflow/python/kernel_tests/decode_image_op_test.py index 0975f964b58..ba5770001ad 100644 --- a/tensorflow/python/kernel_tests/decode_image_op_test.py +++ b/tensorflow/python/kernel_tests/decode_image_op_test.py @@ -23,6 +23,7 @@ import os.path import numpy as np from tensorflow.python.framework import errors_impl +from tensorflow.python.framework import test_util from tensorflow.python.ops import image_ops from tensorflow.python.ops import io_ops import tensorflow.python.ops.nn_grad # pylint: disable=unused-import @@ -40,10 +41,11 @@ class DecodeImageOpTest(test.TestCase): bmp0 = io_ops.read_file(path) image0 = image_ops.decode_image(bmp0) image1 = image_ops.decode_bmp(bmp0) - bmp0, image0, image1 = sess.run([bmp0, image0, image1]) + bmp0, image0, image1 = self.evaluate([bmp0, image0, image1]) self.assertEqual(len(bmp0), 4194) self.assertAllEqual(image0, image1) + @test_util.run_deprecated_v1 def testGif(self): # Read some real GIFs path = os.path.join(prefix_path, "gif", "testdata", "scan.gif") @@ -56,7 +58,7 @@ class DecodeImageOpTest(test.TestCase): gif0 = io_ops.read_file(path) image0 = image_ops.decode_image(gif0) image1 = image_ops.decode_gif(gif0) - gif0, image0, image1 = sess.run([gif0, image0, image1]) + gif0, image0, image1 = self.evaluate([gif0, image0, image1]) self.assertEqual(image0.shape, shape) self.assertAllEqual(image0, image1) @@ -76,8 +78,9 @@ class DecodeImageOpTest(test.TestCase): bad_channels = image_ops.decode_image(gif0, channels=1) with self.assertRaises(errors_impl.InvalidArgumentError): - bad_channels.eval() + self.evaluate(bad_channels) + @test_util.run_deprecated_v1 def testJpeg(self): # Read a real jpeg and verify shape path = os.path.join(prefix_path, "jpeg", "testdata", "jpeg_merge_test1.jpg") @@ -85,14 +88,14 @@ class DecodeImageOpTest(test.TestCase): jpeg0 = io_ops.read_file(path) image0 = image_ops.decode_image(jpeg0) image1 = image_ops.decode_jpeg(jpeg0) - jpeg0, image0, image1 = sess.run([jpeg0, image0, image1]) + jpeg0, image0, image1 = self.evaluate([jpeg0, image0, image1]) self.assertEqual(len(jpeg0), 3771) self.assertEqual(image0.shape, (256, 128, 3)) self.assertAllEqual(image0, image1) bad_channels = image_ops.decode_image(jpeg0, channels=4) with self.assertRaises(errors_impl.InvalidArgumentError): - bad_channels.eval() + self.evaluate(bad_channels) def testPng(self): # Read some real PNGs, converting to different channel numbers @@ -104,16 +107,17 @@ class DecodeImageOpTest(test.TestCase): png0 = io_ops.read_file(path) image0 = image_ops.decode_image(png0, channels=channels) image1 = image_ops.decode_png(png0, channels=channels) - png0, image0, image1 = sess.run([png0, image0, image1]) + png0, image0, image1 = self.evaluate([png0, image0, image1]) self.assertEqual(image0.shape, (26, 51, channels or channels_in)) self.assertAllEqual(image0, image1) + @test_util.run_deprecated_v1 def testInvalidBytes(self): image_bytes = b"ThisIsNotAnImage!" decode = image_ops.decode_image(image_bytes) with self.cached_session(): with self.assertRaises(errors_impl.InvalidArgumentError): - decode.eval() + self.evaluate(decode) if __name__ == "__main__": diff --git a/tensorflow/python/kernel_tests/decode_jpeg_op_test.py b/tensorflow/python/kernel_tests/decode_jpeg_op_test.py index 66b3e0f22fd..f8fc28062f4 100644 --- a/tensorflow/python/kernel_tests/decode_jpeg_op_test.py +++ b/tensorflow/python/kernel_tests/decode_jpeg_op_test.py @@ -80,7 +80,7 @@ class DecodeJpegBenchmark(test.Benchmark): initializer=image_ops.encode_jpeg(tiled_image)) with session.Session() as sess: - sess.run(variables.global_variables_initializer()) + self.evaluate(variables.global_variables_initializer()) images = [] for _ in xrange(parallelism): if crop_window is None: @@ -105,11 +105,11 @@ class DecodeJpegBenchmark(test.Benchmark): for _ in xrange(3): # Skip warm up time. - sess.run(r) + self.evaluate(r) start_time = time.time() for _ in xrange(num_iters): - sess.run(r) + self.evaluate(r) end_time = time.time() return end_time - start_time diff --git a/tensorflow/python/kernel_tests/decode_png_op_test.py b/tensorflow/python/kernel_tests/decode_png_op_test.py index 8f36343667f..5a0b742a6a4 100644 --- a/tensorflow/python/kernel_tests/decode_png_op_test.py +++ b/tensorflow/python/kernel_tests/decode_png_op_test.py @@ -47,7 +47,7 @@ class DecodePngOpTest(test.TestCase): img_in, dtype=dtypes.uint16)) with self.cached_session(): - decoded = decode.eval() + decoded = self.evaluate(decode) self.assertAllEqual(decoded, img_bytes) diff --git a/tensorflow/python/kernel_tests/decode_raw_op_test.py b/tensorflow/python/kernel_tests/decode_raw_op_test.py index dcc984811cb..008e59ba3e6 100644 --- a/tensorflow/python/kernel_tests/decode_raw_op_test.py +++ b/tensorflow/python/kernel_tests/decode_raw_op_test.py @@ -21,6 +21,7 @@ from __future__ import print_function import numpy as np from tensorflow.python.framework import dtypes +from tensorflow.python.framework import test_util from tensorflow.python.ops import array_ops from tensorflow.python.ops import parsing_ops from tensorflow.python.platform import test @@ -28,6 +29,7 @@ from tensorflow.python.platform import test class DecodeRawOpTest(test.TestCase): + @test_util.run_deprecated_v1 def testToUint8(self): with self.cached_session(): in_bytes = array_ops.placeholder(dtypes.string, shape=[2]) @@ -46,6 +48,7 @@ class DecodeRawOpTest(test.TestCase): "element 1 has size 5 != 6"): decode.eval(feed_dict={in_bytes: ["short", "longer"]}) + @test_util.run_deprecated_v1 def testToInt16(self): with self.cached_session(): in_bytes = array_ops.placeholder(dtypes.string, shape=[None]) @@ -61,6 +64,7 @@ class DecodeRawOpTest(test.TestCase): "size of int16"): decode.eval(feed_dict={in_bytes: ["123", "456"]}) + @test_util.run_deprecated_v1 def testEndianness(self): with self.cached_session(): in_bytes = array_ops.placeholder(dtypes.string, shape=[None]) @@ -73,6 +77,7 @@ class DecodeRawOpTest(test.TestCase): result = decode_be.eval(feed_dict={in_bytes: ["\x01\x02\x03\x04"]}) self.assertAllEqual([[0x01020304]], result) + @test_util.run_deprecated_v1 def testToFloat16(self): with self.cached_session(): in_bytes = array_ops.placeholder(dtypes.string, shape=[None]) @@ -84,6 +89,7 @@ class DecodeRawOpTest(test.TestCase): self.assertAllEqual(expected_result, result) + @test_util.run_deprecated_v1 def testEmptyStringInput(self): with self.cached_session(): in_bytes = array_ops.placeholder(dtypes.string, shape=[None]) @@ -93,6 +99,7 @@ class DecodeRawOpTest(test.TestCase): result = decode.eval(feed_dict={in_bytes: [""] * num_inputs}) self.assertEqual((num_inputs, 0), result.shape) + @test_util.run_deprecated_v1 def testToUInt16(self): with self.cached_session(): in_bytes = array_ops.placeholder(dtypes.string, shape=[None]) diff --git a/tensorflow/python/kernel_tests/denormal_test.py b/tensorflow/python/kernel_tests/denormal_test.py index 71a528c4aa2..80a3033ecc4 100644 --- a/tensorflow/python/kernel_tests/denormal_test.py +++ b/tensorflow/python/kernel_tests/denormal_test.py @@ -22,6 +22,7 @@ import numpy as np import platform from tensorflow.python.framework import constant_op +from tensorflow.python.framework import test_util from tensorflow.python.ops import array_ops from tensorflow.python.platform import test @@ -50,10 +51,12 @@ class DenormalTest(test.TestCase): # Make sure the flags don't leak out self.testPythonHasDenormals() + @test_util.run_deprecated_v1 def testFlushDenormalsCPU(self): # On CPUs, the processor flags flush for both single and double precision. self._flushDenormalsTest(use_gpu=False, dtypes=(np.float32, np.float64)) + @test_util.run_deprecated_v1 def testFlushDenormalsGPU(self): # On GPUs, only single precision can flush to zero. self._flushDenormalsTest(use_gpu=True, dtypes=(np.float32,)) diff --git a/tensorflow/python/kernel_tests/dense_update_ops_no_tsan_test.py b/tensorflow/python/kernel_tests/dense_update_ops_no_tsan_test.py index affbaf159d8..4f74e1e7412 100644 --- a/tensorflow/python/kernel_tests/dense_update_ops_no_tsan_test.py +++ b/tensorflow/python/kernel_tests/dense_update_ops_no_tsan_test.py @@ -20,6 +20,7 @@ from __future__ import print_function import numpy as np +from tensorflow.python.framework import test_util from tensorflow.python.ops import array_ops from tensorflow.python.ops import math_ops from tensorflow.python.ops import state_ops @@ -32,6 +33,7 @@ class AssignOpTest(test.TestCase): # NOTE(mrry): We exclude thess tests from the TSAN TAP target, because they # contain benign and deliberate data races when multiple threads update # the same parameters without a lock. + @test_util.run_deprecated_v1 def testParallelUpdateWithoutLocking(self): with self.cached_session() as sess: ones_t = array_ops.fill([1024, 1024], 1.0) @@ -43,7 +45,7 @@ class AssignOpTest(test.TestCase): variables.global_variables_initializer().run() def run_add(add_op): - sess.run(add_op) + self.evaluate(add_op) threads = [ self.checkedThread( @@ -54,11 +56,12 @@ class AssignOpTest(test.TestCase): for t in threads: t.join() - vals = p.eval() + vals = self.evaluate(p) ones = np.ones((1024, 1024)).astype(np.float32) self.assertTrue((vals >= ones).all()) self.assertTrue((vals <= ones * 20).all()) + @test_util.run_deprecated_v1 def testParallelAssignWithoutLocking(self): with self.cached_session() as sess: ones_t = array_ops.fill([1024, 1024], float(1)) @@ -70,7 +73,7 @@ class AssignOpTest(test.TestCase): variables.global_variables_initializer().run() def run_assign(assign_op): - sess.run(assign_op) + self.evaluate(assign_op) threads = [ self.checkedThread( @@ -81,7 +84,7 @@ class AssignOpTest(test.TestCase): for t in threads: t.join() - vals = p.eval() + vals = self.evaluate(p) # Assert every element is taken from one of the assignments. self.assertTrue((vals > 0).all()) @@ -91,6 +94,7 @@ class AssignOpTest(test.TestCase): # contain non-benign but known data races between the variable assignment and # returning the output tensors. This issue will be resolved with the new # resource variables. + @test_util.run_deprecated_v1 def testParallelUpdateWithLocking(self): with self.cached_session() as sess: zeros_t = array_ops.fill([1024, 1024], 0.0) @@ -103,7 +107,7 @@ class AssignOpTest(test.TestCase): p.initializer.run() def run_add(add_op): - sess.run(add_op) + self.evaluate(add_op) threads = [ self.checkedThread( @@ -114,10 +118,11 @@ class AssignOpTest(test.TestCase): for t in threads: t.join() - vals = p.eval() + vals = self.evaluate(p) ones = np.ones((1024, 1024)).astype(np.float32) self.assertAllEqual(vals, ones * 20) + @test_util.run_deprecated_v1 def testParallelAssignWithLocking(self): with self.cached_session() as sess: zeros_t = array_ops.fill([1024, 1024], 0.0) @@ -131,7 +136,7 @@ class AssignOpTest(test.TestCase): p.initializer.run() def run_assign(assign_op): - sess.run(assign_op) + self.evaluate(assign_op) threads = [ self.checkedThread( @@ -142,7 +147,7 @@ class AssignOpTest(test.TestCase): for t in threads: t.join() - vals = p.eval() + vals = self.evaluate(p) # Assert every element is the same, and taken from one of the assignments. self.assertTrue(vals[0, 0] > 0) diff --git a/tensorflow/python/kernel_tests/dense_update_ops_test.py b/tensorflow/python/kernel_tests/dense_update_ops_test.py index 3e0a03d634f..309da88bef7 100644 --- a/tensorflow/python/kernel_tests/dense_update_ops_test.py +++ b/tensorflow/python/kernel_tests/dense_update_ops_test.py @@ -21,6 +21,7 @@ from __future__ import print_function import numpy as np from tensorflow.python.framework import dtypes +from tensorflow.python.framework import test_util from tensorflow.python.ops import array_ops from tensorflow.python.ops import state_ops from tensorflow.python.ops import variables @@ -36,8 +37,8 @@ class AssignOpTest(test.TestCase): p = variables.Variable(x) assign = state_ops.assign(p, y) p.initializer.run() - new_value = assign.eval() - return p.eval(), new_value + new_value = self.evaluate(assign) + return self.evaluate(p), new_value def _initAssignAddFetch(self, x, y, use_gpu=False): """Initialize a param to init, and compute param += y.""" @@ -45,8 +46,8 @@ class AssignOpTest(test.TestCase): p = variables.Variable(x) add = state_ops.assign_add(p, y) p.initializer.run() - new_value = add.eval() - return p.eval(), new_value + new_value = self.evaluate(add) + return self.evaluate(p), new_value def _initAssignSubFetch(self, x, y, use_gpu=False): """Initialize a param to init, and compute param -= y.""" @@ -54,8 +55,8 @@ class AssignOpTest(test.TestCase): p = variables.Variable(x) sub = state_ops.assign_sub(p, y) p.initializer.run() - new_value = sub.eval() - return p.eval(), new_value + new_value = self.evaluate(sub) + return self.evaluate(p), new_value def _testTypes(self, vals): for dtype in [np.float32, np.float64, np.int32, np.int64]: @@ -81,23 +82,26 @@ class AssignOpTest(test.TestCase): self.assertAllEqual(x - y, var_value) self.assertAllEqual(x - y, op_value) + @test_util.run_deprecated_v1 def testBasic(self): self._testTypes(np.arange(0, 20).reshape([4, 5])) + @test_util.run_deprecated_v1 def testAssignNonStrictShapeChecking(self): with self.cached_session(): data = array_ops.fill([1024, 1024], 0) p = variables.VariableV1([1]) a = state_ops.assign(p, data, validate_shape=False) a.op.run() - self.assertAllEqual(p.eval(), data.eval()) + self.assertAllEqual(p.eval(), self.evaluate(data)) # Assign to yet another shape data2 = array_ops.fill([10, 10], 1) a2 = state_ops.assign(p, data2, validate_shape=False) a2.op.run() - self.assertAllEqual(p.eval(), data2.eval()) + self.assertAllEqual(p.eval(), self.evaluate(data2)) + @test_util.run_deprecated_v1 def testInitRequiredAssignAdd(self): with self.cached_session(): p = variables.VariableV1(array_ops.fill([1024, 1024], 1), dtypes.int32) @@ -105,6 +109,7 @@ class AssignOpTest(test.TestCase): with self.assertRaisesOpError("use uninitialized"): a.op.run() + @test_util.run_deprecated_v1 def testInitRequiredAssignSub(self): with self.cached_session(): p = variables.VariableV1(array_ops.fill([1024, 1024], 1), dtypes.int32) diff --git a/tensorflow/python/kernel_tests/depthtospace_op_test.py b/tensorflow/python/kernel_tests/depthtospace_op_test.py index 13a28caf1fd..96c9b5258e2 100644 --- a/tensorflow/python/kernel_tests/depthtospace_op_test.py +++ b/tensorflow/python/kernel_tests/depthtospace_op_test.py @@ -53,12 +53,14 @@ class DepthToSpaceTest(test.TestCase): output_nhwc = test_util.NCHWToNHWC(output_nchw) self.assertAllEqual(output_nhwc.eval(), outputs) + @test_util.run_deprecated_v1 def testBasic(self): x_np = [[[[1, 2, 3, 4]]]] block_size = 2 x_out = [[[[1], [2]], [[3], [4]]]] self._testOne(x_np, block_size, x_out) + @test_util.run_deprecated_v1 def testBasicFloat16(self): x_np = [[[[1, 2, 3, 4]]]] block_size = 2 @@ -67,6 +69,7 @@ class DepthToSpaceTest(test.TestCase): # Tests for larger input dimensions. To make sure elements are # correctly ordered spatially. + @test_util.run_deprecated_v1 def testBlockSize2(self): x_np = [[[[1, 2, 3, 4], [5, 6, 7, 8]], @@ -79,6 +82,7 @@ class DepthToSpaceTest(test.TestCase): [[11], [12], [15], [16]]]] self._testOne(x_np, block_size, x_out) + @test_util.run_deprecated_v1 def testBlockSize2Batch10(self): block_size = 2 def batch_input_elt(i): @@ -106,15 +110,16 @@ class DepthToSpaceTest(test.TestCase): # test NHWC (default) on CPU x_tf = array_ops.depth_to_space(input_nhwc, block_size) self.assertAllEqual(x_tf.shape, x_out.shape) - x_tf.eval() + self.evaluate(x_tf) if test.is_gpu_available(): with self.cached_session(use_gpu=True): # test NHWC (default) on GPU x_tf = array_ops.depth_to_space(input_nhwc, block_size) self.assertAllEqual(x_tf.shape, x_out.shape) - x_tf.eval() + self.evaluate(x_tf) # Tests for different width and height. + @test_util.run_deprecated_v1 def testNonSquare(self): x_np = [[[[1, 10, 2, 20, 3, 30, 4, 40]], [[5, 50, 6, 60, 7, 70, 8, 80]], @@ -130,6 +135,7 @@ class DepthToSpaceTest(test.TestCase): # Tests for larger input dimensions. To make sure elements are # correctly ordered spatially. + @test_util.run_deprecated_v1 def testBlockSize4FlatInput(self): x_np = [[[[1, 2, 5, 6, 3, 4, 7, 8, 9, 10, 13, 14, 11, 12, 15, 16]]]] block_size = 4 @@ -141,6 +147,7 @@ class DepthToSpaceTest(test.TestCase): # Tests for larger input depths. # To make sure elements are properly interleaved in depth. + @test_util.run_deprecated_v1 def testDepthInterleaved(self): x_np = [[[[1, 10, 2, 20, 3, 30, 4, 40]]]] block_size = 2 @@ -150,6 +157,7 @@ class DepthToSpaceTest(test.TestCase): # Tests for larger input depths. Here an odd depth. # To make sure elements are properly interleaved in depth. + @test_util.run_deprecated_v1 def testDepthInterleavedDepth3(self): x_np = [[[[1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12]]]] block_size = 2 @@ -159,6 +167,7 @@ class DepthToSpaceTest(test.TestCase): # Tests for larger input depths. # To make sure elements are properly interleaved in depth. + @test_util.run_deprecated_v1 def testDepthInterleavedLarger(self): x_np = [[[[1, 10, 2, 20, 3, 30, 4, 40], [5, 50, 6, 60, 7, 70, 8, 80]], @@ -175,6 +184,7 @@ class DepthToSpaceTest(test.TestCase): # Tests for a block larger for the depth. In this case should raise an # exception. + @test_util.run_deprecated_v1 def testBlockSizeTooLarge(self): x_np = [[[[1, 2, 3, 4], [5, 6, 7, 8]], @@ -185,18 +195,20 @@ class DepthToSpaceTest(test.TestCase): # divisible by 16. with self.assertRaises(ValueError): out_tf = array_ops.depth_to_space(x_np, block_size) - out_tf.eval() + self.evaluate(out_tf) # Test when the block size is 0. + @test_util.run_deprecated_v1 def testBlockSize0(self): x_np = [[[[1], [2]], [[3], [4]]]] block_size = 0 with self.assertRaises(ValueError): out_tf = array_ops.depth_to_space(x_np, block_size) - out_tf.eval() + self.evaluate(out_tf) # Test when the block size is 1. The block size should be > 1. + @test_util.run_deprecated_v1 def testBlockSizeOne(self): x_np = [[[[1, 1, 1, 1], [2, 2, 2, 2]], @@ -205,8 +217,9 @@ class DepthToSpaceTest(test.TestCase): block_size = 1 with self.assertRaises(ValueError): out_tf = array_ops.depth_to_space(x_np, block_size) - out_tf.eval() + self.evaluate(out_tf) + @test_util.run_deprecated_v1 def testBlockSizeLargerThanInput(self): # The block size is too large for this input. x_np = [[[[1], [2]], @@ -214,8 +227,9 @@ class DepthToSpaceTest(test.TestCase): block_size = 10 with self.assertRaises(ValueError): out_tf = array_ops.space_to_depth(x_np, block_size) - out_tf.eval() + self.evaluate(out_tf) + @test_util.run_deprecated_v1 def testBlockSizeNotDivisibleDepth(self): # The depth is not divisible by the square of the block size. x_np = [[[[1, 1, 1, 1], @@ -226,6 +240,7 @@ class DepthToSpaceTest(test.TestCase): with self.assertRaises(ValueError): _ = array_ops.space_to_depth(x_np, block_size) + @test_util.run_deprecated_v1 def testUnknownShape(self): t = array_ops.depth_to_space( array_ops.placeholder(dtypes.float32), block_size=4) @@ -277,7 +292,7 @@ class DepthToSpaceTest(test.TestCase): actual = array_ops.depth_to_space(t, block_size, data_format=data_format) with self.session(use_gpu=use_gpu) as sess: - actual_vals, expected_vals = sess.run([actual, expected]) + actual_vals, expected_vals = self.evaluate([actual, expected]) self.assertTrue(np.array_equal(actual_vals, expected_vals)) def testAgainstTranspose(self): @@ -343,11 +358,13 @@ class DepthToSpaceGradientTest(test.TestCase): # Don't use very large numbers as dimensions here, as the result is tensor # with cartesian product of the dimensions. + @test_util.run_deprecated_v1 def testSmall(self): block_size = 2 self._compare(3, 2, 5, 3, block_size, "NHWC") self._compare(3, 2, 5, 3, block_size, "NCHW") + @test_util.run_deprecated_v1 def testSmall2(self): block_size = 3 self._compare(1, 2, 3, 2, block_size, "NHWC") diff --git a/tensorflow/python/kernel_tests/depthwise_conv_op_test.py b/tensorflow/python/kernel_tests/depthwise_conv_op_test.py index 77b27c6c7e0..f6d834c2f85 100644 --- a/tensorflow/python/kernel_tests/depthwise_conv_op_test.py +++ b/tensorflow/python/kernel_tests/depthwise_conv_op_test.py @@ -162,7 +162,7 @@ class DepthwiseConv2DTest(test.TestCase): conv_native = array_ops.transpose(conv_native, [0, 2, 3, 1]) try: - native_result = sess.run(conv_native) + native_result = self.evaluate(conv_native) except errors.InvalidArgumentError as e: # Grouped convolution kernel is only registered for cuDNN 7. Silently # return when we are running on an earlier version or without GPU. @@ -174,7 +174,7 @@ class DepthwiseConv2DTest(test.TestCase): conv_interface = nn_impl.depthwise_conv2d( t1, t2, strides=[1, stride, stride, 1], padding=padding) - interface_result = sess.run(conv_interface) + interface_result = self.evaluate(conv_interface) tf_logging.info( "data_type: %r, use_gpu: %r, grouped_conv: %r, max diff = %f", @@ -269,7 +269,7 @@ class DepthwiseConv2DTest(test.TestCase): t2 = constant_op.constant(x2, shape=filter_in_sizes) conv = nn_ops.depthwise_conv2d_native( t1, t2, strides=[1, stride, stride, 1], padding=padding) - value = sess.run(conv) + value = self.evaluate(conv) tf_logging.info("value = %r", value) self.assertArrayNear(expected, np.ravel(value), 1e-5) self.assertShapeEqual(value, conv) @@ -528,7 +528,7 @@ class DepthwiseConv2DTest(test.TestCase): t2 = constant_op.constant(x2, shape=output_sizes) backprop = nn_ops.depthwise_conv2d_native_backprop_input( t0, t1, t2, strides=[1, stride, stride, 1], padding=padding) - ret = backprop.eval() + ret = self.evaluate(backprop) self.assertShapeEqual(ret, backprop) return ret @@ -548,7 +548,7 @@ class DepthwiseConv2DTest(test.TestCase): t2 = constant_op.constant(x2, shape=output_sizes) backprop = nn_ops.depthwise_conv2d_native_backprop_input( t0, t1, t2, strides=[1, stride, stride, 1], padding=padding) - ret = backprop.eval() + ret = self.evaluate(backprop) self.assertShapeEqual(ret, backprop) return ret @@ -580,7 +580,7 @@ class DepthwiseConv2DTest(test.TestCase): t2 = constant_op.constant(x2, shape=output_sizes) backprop = nn_ops.depthwise_conv2d_native_backprop_filter( t0, t1, t2, strides=[1, stride, stride, 1], padding=padding) - ret = backprop.eval() + ret = self.evaluate(backprop) self.assertShapeEqual(ret, backprop) return ret @@ -600,7 +600,7 @@ class DepthwiseConv2DTest(test.TestCase): t2 = constant_op.constant(x2, shape=output_sizes) backprop = nn_ops.depthwise_conv2d_native_backprop_filter( t0, t1, t2, strides=[1, stride, stride, 1], padding=padding) - ret = backprop.eval() + ret = self.evaluate(backprop) self.assertShapeEqual(ret, backprop) return ret diff --git a/tensorflow/python/kernel_tests/determinant_op_test.py b/tensorflow/python/kernel_tests/determinant_op_test.py index da33b2848b7..d6ef9e70b83 100644 --- a/tensorflow/python/kernel_tests/determinant_op_test.py +++ b/tensorflow/python/kernel_tests/determinant_op_test.py @@ -23,6 +23,7 @@ import numpy as np from tensorflow.python.client import session from tensorflow.python.framework import constant_op from tensorflow.python.framework import ops +from tensorflow.python.framework import test_util from tensorflow.python.ops import control_flow_ops from tensorflow.python.ops import gen_linalg_ops from tensorflow.python.ops import linalg_ops @@ -35,7 +36,7 @@ from tensorflow.python.platform import test class DeterminantOpTest(test.TestCase): def _compareDeterminantBase(self, matrix_x, tf_ans): - out = tf_ans.eval() + out = self.evaluate(tf_ans) shape = matrix_x.shape if shape[-1] == 0 and shape[-2] == 0: np_ans = np.ones(shape[:-2]).astype(matrix_x.dtype) @@ -54,15 +55,15 @@ class DeterminantOpTest(test.TestCase): np_ans = np_ans.astype(matrix_x.dtype) self.assertShapeEqual(np_ans, abs_log_det_tf) - sign_tf_val = sign_tf.eval() - abs_log_det_tf_val = abs_log_det_tf.eval() + sign_tf_val = self.evaluate(sign_tf) + abs_log_det_tf_val = self.evaluate(abs_log_det_tf) self.assertAllClose( sign_tf_val * np.exp(abs_log_det_tf_val), np_sign * np.exp(np_ans), atol=5e-5) def _compareDeterminant(self, matrix_x): - with self.cached_session(use_gpu=True): + with test_util.use_gpu(): self._compareDeterminantBase(matrix_x, linalg_ops.matrix_determinant(matrix_x)) self._compareLogDeterminantBase( @@ -155,7 +156,7 @@ class DeterminantOpTest(test.TestCase): matrix2 = random_ops.random_normal([5, 5], seed=42) det1 = linalg_ops.matrix_determinant(matrix1) det2 = linalg_ops.matrix_determinant(matrix2) - det1_val, det2_val = sess.run([det1, det2]) + det1_val, det2_val = self.evaluate([det1, det2]) self.assertEqual(det1_val, det2_val) diff --git a/tensorflow/python/kernel_tests/diag_op_test.py b/tensorflow/python/kernel_tests/diag_op_test.py index 9e43258fa2d..ed2a9e8e47e 100644 --- a/tensorflow/python/kernel_tests/diag_op_test.py +++ b/tensorflow/python/kernel_tests/diag_op_test.py @@ -22,6 +22,7 @@ import numpy as np from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes as dtypes_lib from tensorflow.python.framework import ops +from tensorflow.python.framework import test_util from tensorflow.python.ops import array_ops from tensorflow.python.ops import gradient_checker from tensorflow.python.ops import gradients_impl @@ -31,6 +32,7 @@ from tensorflow.python.platform import tf_logging class MatrixDiagTest(test.TestCase): + @test_util.run_deprecated_v1 def testVector(self): with self.session(use_gpu=True): v = np.array([1.0, 2.0, 3.0]) @@ -49,6 +51,7 @@ class MatrixDiagTest(test.TestCase): self.assertEqual((2, 3, 3), v_batch_diag.get_shape()) self.assertAllEqual(v_batch_diag.eval(), mat_batch) + @test_util.run_deprecated_v1 def testBatchVector(self): self._testBatchVector(np.float32) self._testBatchVector(np.float64) @@ -56,16 +59,19 @@ class MatrixDiagTest(test.TestCase): self._testBatchVector(np.int64) self._testBatchVector(np.bool) + @test_util.run_deprecated_v1 def testInvalidShape(self): with self.assertRaisesRegexp(ValueError, "must be at least rank 1"): array_ops.matrix_diag(0) + @test_util.run_deprecated_v1 def testInvalidShapeAtEval(self): with self.session(use_gpu=True): v = array_ops.placeholder(dtype=dtypes_lib.float32) with self.assertRaisesOpError("input must be at least 1-dim"): array_ops.matrix_diag(v).eval(feed_dict={v: 0.0}) + @test_util.run_deprecated_v1 def testGrad(self): shapes = ((3,), (7, 4)) with self.session(use_gpu=True): @@ -81,6 +87,7 @@ class MatrixDiagTest(test.TestCase): class MatrixSetDiagTest(test.TestCase): + @test_util.run_deprecated_v1 def testSquare(self): with self.session(use_gpu=True): v = np.array([1.0, 2.0, 3.0]) @@ -89,8 +96,9 @@ class MatrixSetDiagTest(test.TestCase): [1.0, 1.0, 3.0]]) output = array_ops.matrix_set_diag(mat, v) self.assertEqual((3, 3), output.get_shape()) - self.assertAllEqual(mat_set_diag, output.eval()) + self.assertAllEqual(mat_set_diag, self.evaluate(output)) + @test_util.run_deprecated_v1 def testRectangular(self): with self.session(use_gpu=True): v = np.array([3.0, 4.0]) @@ -98,14 +106,14 @@ class MatrixSetDiagTest(test.TestCase): expected = np.array([[3.0, 1.0, 0.0], [1.0, 4.0, 1.0]]) output = array_ops.matrix_set_diag(mat, v) self.assertEqual((2, 3), output.get_shape()) - self.assertAllEqual(expected, output.eval()) + self.assertAllEqual(expected, self.evaluate(output)) v = np.array([3.0, 4.0]) mat = np.array([[0.0, 1.0], [1.0, 0.0], [1.0, 1.0]]) expected = np.array([[3.0, 1.0], [1.0, 4.0], [1.0, 1.0]]) output = array_ops.matrix_set_diag(mat, v) self.assertEqual((3, 2), output.get_shape()) - self.assertAllEqual(expected, output.eval()) + self.assertAllEqual(expected, self.evaluate(output)) def _testSquareBatch(self, dtype): with self.cached_session(use_gpu=True): @@ -121,8 +129,9 @@ class MatrixSetDiagTest(test.TestCase): output = array_ops.matrix_set_diag(mat_batch, v_batch) self.assertEqual((2, 3, 3), output.get_shape()) - self.assertAllEqual(mat_set_diag_batch, output.eval()) + self.assertAllEqual(mat_set_diag_batch, self.evaluate(output)) + @test_util.run_deprecated_v1 def testSquareBatch(self): self._testSquareBatch(np.float32) self._testSquareBatch(np.float64) @@ -130,6 +139,7 @@ class MatrixSetDiagTest(test.TestCase): self._testSquareBatch(np.int64) self._testSquareBatch(np.bool) + @test_util.run_deprecated_v1 def testRectangularBatch(self): with self.session(use_gpu=True): v_batch = np.array([[-1.0, -2.0], [-4.0, -5.0]]) @@ -140,14 +150,16 @@ class MatrixSetDiagTest(test.TestCase): [[-4.0, 0.0, 4.0], [0.0, -5.0, 0.0]]]) output = array_ops.matrix_set_diag(mat_batch, v_batch) self.assertEqual((2, 2, 3), output.get_shape()) - self.assertAllEqual(mat_set_diag_batch, output.eval()) + self.assertAllEqual(mat_set_diag_batch, self.evaluate(output)) + @test_util.run_deprecated_v1 def testInvalidShape(self): with self.assertRaisesRegexp(ValueError, "must be at least rank 2"): array_ops.matrix_set_diag(0, [0]) with self.assertRaisesRegexp(ValueError, "must be at least rank 1"): array_ops.matrix_set_diag([[0]], 0) + @test_util.run_deprecated_v1 def testInvalidShapeAtEval(self): with self.session(use_gpu=True): v = array_ops.placeholder(dtype=dtypes_lib.float32) @@ -157,6 +169,7 @@ class MatrixSetDiagTest(test.TestCase): r"but received input shape: \[1,1\] and diagonal shape: \[\]"): array_ops.matrix_set_diag([[v]], v).eval(feed_dict={v: 0.0}) + @test_util.run_deprecated_v1 def testGrad(self): shapes = ((3, 4, 4), (3, 3, 4), (3, 4, 3), (7, 4, 8, 8)) with self.session(use_gpu=True): @@ -178,6 +191,7 @@ class MatrixSetDiagTest(test.TestCase): y.get_shape().as_list()) self.assertLess(error_x_diag, 1e-4) + @test_util.run_deprecated_v1 def testGradWithNoShapeInformation(self): with self.session(use_gpu=True) as sess: v = array_ops.placeholder(dtype=dtypes_lib.float32) @@ -200,6 +214,7 @@ class MatrixSetDiagTest(test.TestCase): class MatrixDiagPartTest(test.TestCase): + @test_util.run_deprecated_v1 def testSquare(self): with self.session(use_gpu=True): v = np.array([1.0, 2.0, 3.0]) @@ -208,6 +223,7 @@ class MatrixDiagPartTest(test.TestCase): self.assertEqual((3,), mat_diag.get_shape()) self.assertAllEqual(mat_diag.eval(), v) + @test_util.run_deprecated_v1 def testRectangular(self): with self.session(use_gpu=True): mat = np.array([[1.0, 2.0, 3.0], [4.0, 5.0, 6.0]]) @@ -228,6 +244,7 @@ class MatrixDiagPartTest(test.TestCase): self.assertEqual((2, 3), mat_batch_diag.get_shape()) self.assertAllEqual(mat_batch_diag.eval(), v_batch) + @test_util.run_deprecated_v1 def testSquareBatch(self): self._testSquareBatch(np.float32) self._testSquareBatch(np.float64) @@ -235,6 +252,7 @@ class MatrixDiagPartTest(test.TestCase): self._testSquareBatch(np.int64) self._testSquareBatch(np.bool) + @test_util.run_deprecated_v1 def testRectangularBatch(self): with self.session(use_gpu=True): v_batch = np.array([[1.0, 2.0], [4.0, 5.0]]) @@ -245,16 +263,19 @@ class MatrixDiagPartTest(test.TestCase): self.assertEqual((2, 2), mat_batch_diag.get_shape()) self.assertAllEqual(mat_batch_diag.eval(), v_batch) + @test_util.run_deprecated_v1 def testInvalidShape(self): with self.assertRaisesRegexp(ValueError, "must be at least rank 2"): array_ops.matrix_diag_part(0) + @test_util.run_deprecated_v1 def testInvalidShapeAtEval(self): with self.session(use_gpu=True): v = array_ops.placeholder(dtype=dtypes_lib.float32) with self.assertRaisesOpError("input must be at least 2-dim"): array_ops.matrix_diag_part(v).eval(feed_dict={v: 0.0}) + @test_util.run_deprecated_v1 def testGrad(self): shapes = ((3, 3), (2, 3), (3, 2), (5, 3, 3)) with self.session(use_gpu=True): @@ -273,9 +294,9 @@ class DiagTest(test.TestCase): def _diagOp(self, diag, dtype, expected_ans, use_gpu): with self.cached_session(use_gpu=use_gpu): tf_ans = array_ops.diag(ops.convert_to_tensor(diag.astype(dtype))) - out = tf_ans.eval() + out = self.evaluate(tf_ans) tf_ans_inv = array_ops.diag_part(expected_ans) - inv_out = tf_ans_inv.eval() + inv_out = self.evaluate(tf_ans_inv) self.assertAllClose(out, expected_ans) self.assertAllClose(inv_out, diag) self.assertShapeEqual(expected_ans, tf_ans) @@ -407,6 +428,7 @@ class DiagTest(test.TestCase): dtype=dtype) self.diagOp(x, dtype, expected_ans) + @test_util.run_deprecated_v1 def testInvalidRank(self): with self.assertRaisesRegexp(ValueError, "must be at least rank 1"): array_ops.diag(0.0) @@ -421,7 +443,7 @@ class DiagPartOpTest(test.TestCase): with self.cached_session(use_gpu=use_gpu): tensor = ops.convert_to_tensor(tensor.astype(dtype)) tf_ans_inv = array_ops.diag_part(tensor) - inv_out = tf_ans_inv.eval() + inv_out = self.evaluate(tf_ans_inv) self.assertAllClose(inv_out, expected_ans) self.assertShapeEqual(expected_ans, tf_ans_inv) @@ -445,7 +467,7 @@ class DiagPartOpTest(test.TestCase): t = ops.convert_to_tensor(x.astype(np.float32)) t.set_shape(shape) tf_ans = array_ops.diag_part(t) - out = tf_ans.eval() + out = self.evaluate(tf_ans) self.assertAllClose(out, expected_ans) self.assertShapeEqual(expected_ans, tf_ans) @@ -476,6 +498,7 @@ class DiagPartOpTest(test.TestCase): self.diagPartOp(x, np.complex64, expected_ans) self.diagPartOp(x, np.complex128, expected_ans) + @test_util.run_deprecated_v1 def testOddRank(self): w = np.random.rand(2) x = np.random.rand(2, 2, 2) @@ -484,6 +507,7 @@ class DiagPartOpTest(test.TestCase): with self.assertRaises(ValueError): array_ops.diag_part(0.0) + @test_util.run_deprecated_v1 def testUnevenDimensions(self): w = np.random.rand(2, 5) x = np.random.rand(2, 1, 2, 3) @@ -493,6 +517,7 @@ class DiagPartOpTest(test.TestCase): class DiagGradOpTest(test.TestCase): + @test_util.run_deprecated_v1 def testDiagGrad(self): np.random.seed(0) shapes = ((3,), (3, 3), (3, 3, 3)) @@ -513,6 +538,7 @@ class DiagGradOpTest(test.TestCase): class DiagGradPartOpTest(test.TestCase): + @test_util.run_deprecated_v1 def testDiagPartGrad(self): np.random.seed(0) shapes = ((3, 3), (3, 3, 3, 3)) diff --git a/tensorflow/python/kernel_tests/distributions/bernoulli_test.py b/tensorflow/python/kernel_tests/distributions/bernoulli_test.py index 37b35ba51a8..e6d560b4bc4 100644 --- a/tensorflow/python/kernel_tests/distributions/bernoulli_test.py +++ b/tensorflow/python/kernel_tests/distributions/bernoulli_test.py @@ -151,6 +151,7 @@ class BernoulliTest(test.TestCase): self.assertAllClose(self.evaluate(dist.prob(x)), expected_pmf) self.assertAllClose(self.evaluate(dist.log_prob(x)), np.log(expected_pmf)) + @test_util.run_deprecated_v1 def testPmfCorrectBroadcastDynamicShape(self): with self.cached_session(): p = array_ops.placeholder(dtype=dtypes.float32) @@ -167,6 +168,7 @@ class BernoulliTest(test.TestCase): }), [[0.2, 0.7, 0.4]]) @test_util.run_in_graph_and_eager_modes + @test_util.run_deprecated_v1 def testPmfInvalid(self): p = [0.1, 0.2, 0.7] dist = bernoulli.Bernoulli(probs=p, validate_args=True) @@ -193,6 +195,7 @@ class BernoulliTest(test.TestCase): self.evaluate( bernoulli.Bernoulli(probs=p, validate_args=False).log_prob(samps))) + @test_util.run_deprecated_v1 def testBroadcasting(self): with self.cached_session(): p = array_ops.placeholder(dtypes.float32) @@ -207,6 +210,7 @@ class BernoulliTest(test.TestCase): p: [0.5, 0.5, 0.5] })) + @test_util.run_deprecated_v1 def testPmfShapes(self): with self.cached_session(): p = array_ops.placeholder(dtypes.float32, shape=[None, 1]) @@ -276,6 +280,7 @@ class BernoulliTest(test.TestCase): grad_p = tape.gradient(samples, p) self.assertIsNone(grad_p) + @test_util.run_deprecated_v1 def testSampleActsLikeSampleN(self): with self.cached_session() as sess: p = [0.2, 0.6] diff --git a/tensorflow/python/kernel_tests/distributions/bijector_test.py b/tensorflow/python/kernel_tests/distributions/bijector_test.py index e20f59f48ac..a0e0a36fecc 100644 --- a/tensorflow/python/kernel_tests/distributions/bijector_test.py +++ b/tensorflow/python/kernel_tests/distributions/bijector_test.py @@ -132,6 +132,7 @@ class BijectorTestEventNdims(test.TestCase): with self.assertRaisesRegexp(ValueError, "Expected scalar"): bij.inverse_log_det_jacobian(1., event_ndims=(1, 2)) + @test_util.run_deprecated_v1 def testBijectorDynamicEventNdims(self): bij = BrokenBijector(validate_args=True) event_ndims = array_ops.placeholder(dtype=np.int32, shape=None) @@ -301,6 +302,7 @@ class BijectorReduceEventDimsTest(test.TestCase): 8., self.evaluate(bij.inverse_log_det_jacobian(x, event_ndims=2))) + @test_util.run_deprecated_v1 def testHandlesNonStaticEventNdims(self): x_ = [[[1., 2.], [3., 4.]]] x = array_ops.placeholder_with_default(x_, shape=None) diff --git a/tensorflow/python/kernel_tests/distributions/categorical_test.py b/tensorflow/python/kernel_tests/distributions/categorical_test.py index c6bb06eab30..ec1d4ed2070 100644 --- a/tensorflow/python/kernel_tests/distributions/categorical_test.py +++ b/tensorflow/python/kernel_tests/distributions/categorical_test.py @@ -25,6 +25,7 @@ from tensorflow.python.eager import backprop from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes from tensorflow.python.framework import tensor_util +from tensorflow.python.framework import test_util from tensorflow.python.ops import array_ops from tensorflow.python.ops import gradients_impl from tensorflow.python.ops import math_ops @@ -44,6 +45,7 @@ def make_categorical(batch_shape, num_classes, dtype=dtypes.int32): class CategoricalTest(test.TestCase, parameterized.TestCase): + @test_util.run_deprecated_v1 def testP(self): p = [0.2, 0.8] dist = categorical.Categorical(probs=p) @@ -51,6 +53,7 @@ class CategoricalTest(test.TestCase, parameterized.TestCase): self.assertAllClose(p, dist.probs.eval()) self.assertAllEqual([2], dist.logits.get_shape()) + @test_util.run_deprecated_v1 def testLogits(self): p = np.array([0.2, 0.8], dtype=np.float32) logits = np.log(p) - 50. @@ -61,6 +64,7 @@ class CategoricalTest(test.TestCase, parameterized.TestCase): self.assertAllClose(dist.probs.eval(), p) self.assertAllClose(dist.logits.eval(), logits) + @test_util.run_deprecated_v1 def testShapes(self): with self.cached_session(): for batch_shape in ([], [1], [2, 3, 4]): @@ -107,6 +111,7 @@ class CategoricalTest(test.TestCase, parameterized.TestCase): self.assertEqual(dist.dtype, dtype) self.assertEqual(dist.dtype, dist.sample(5).dtype) + @test_util.run_deprecated_v1 def testUnknownShape(self): with self.cached_session(): logits = array_ops.placeholder(dtype=dtypes.float32) @@ -121,18 +126,21 @@ class CategoricalTest(test.TestCase, parameterized.TestCase): feed_dict={logits: [[-1000.0, 1000.0], [1000.0, -1000.0]]}) self.assertAllEqual([1, 0], sample_value_batch) + @test_util.run_deprecated_v1 def testPMFWithBatch(self): histograms = [[0.2, 0.8], [0.6, 0.4]] dist = categorical.Categorical(math_ops.log(histograms) - 50.) with self.cached_session(): self.assertAllClose(dist.prob([0, 1]).eval(), [0.2, 0.4]) + @test_util.run_deprecated_v1 def testPMFNoBatch(self): histograms = [0.2, 0.8] dist = categorical.Categorical(math_ops.log(histograms) - 50.) with self.cached_session(): self.assertAllClose(dist.prob(0).eval(), 0.2) + @test_util.run_deprecated_v1 def testCDFWithDynamicEventShapeKnownNdims(self): """Test that dynamically-sized events with unknown shape work.""" batch_size = 2 @@ -184,6 +192,7 @@ class CategoricalTest(test.TestCase, parameterized.TestCase): actual_cdf = self.evaluate(cdf_op) self.assertAllClose(actual_cdf, expected_cdf) + @test_util.run_deprecated_v1 def testCDFWithBatch(self): histograms = [[0.1, 0.2, 0.3, 0.25, 0.15], [0.0, 0.75, 0.2, 0.05, 0.0]] @@ -195,6 +204,7 @@ class CategoricalTest(test.TestCase, parameterized.TestCase): with self.cached_session(): self.assertAllClose(cdf_op.eval(), expected_cdf) + @test_util.run_deprecated_v1 def testCDFNoBatch(self): histogram = [0.1, 0.2, 0.3, 0.4] event = 2 @@ -205,6 +215,7 @@ class CategoricalTest(test.TestCase, parameterized.TestCase): with self.cached_session(): self.assertAlmostEqual(cdf_op.eval(), expected_cdf) + @test_util.run_deprecated_v1 def testCDFBroadcasting(self): # shape: [batch=2, n_bins=3] histograms = [[0.2, 0.1, 0.7], @@ -287,7 +298,7 @@ class CategoricalTest(test.TestCase, parameterized.TestCase): } with self.cached_session() as sess: - run_result = sess.run(to_run) + run_result = self.evaluate(to_run) self.assertAllEqual(run_result["cat_prob"].shape, run_result["norm_prob"].shape) @@ -298,6 +309,7 @@ class CategoricalTest(test.TestCase, parameterized.TestCase): self.assertAllEqual(run_result["cat_log_cdf"].shape, run_result["norm_log_cdf"].shape) + @test_util.run_deprecated_v1 def testLogPMF(self): logits = np.log([[0.2, 0.8], [0.6, 0.4]]) - 50. dist = categorical.Categorical(logits) @@ -305,6 +317,7 @@ class CategoricalTest(test.TestCase, parameterized.TestCase): self.assertAllClose(dist.log_prob([0, 1]).eval(), np.log([0.2, 0.4])) self.assertAllClose(dist.log_prob([0.0, 1.0]).eval(), np.log([0.2, 0.4])) + @test_util.run_deprecated_v1 def testEntropyNoBatch(self): logits = np.log([0.2, 0.8]) - 50. dist = categorical.Categorical(logits) @@ -312,6 +325,7 @@ class CategoricalTest(test.TestCase, parameterized.TestCase): self.assertAllClose(dist.entropy().eval(), -(0.2 * np.log(0.2) + 0.8 * np.log(0.8))) + @test_util.run_deprecated_v1 def testEntropyWithBatch(self): logits = np.log([[0.2, 0.8], [0.6, 0.4]]) - 50. dist = categorical.Categorical(logits) @@ -321,6 +335,7 @@ class CategoricalTest(test.TestCase, parameterized.TestCase): -(0.6 * np.log(0.6) + 0.4 * np.log(0.4)) ]) + @test_util.run_deprecated_v1 def testEntropyGradient(self): with self.cached_session() as sess: logits = constant_op.constant([[1., 2., 3.], [2., 5., 1.]]) @@ -355,7 +370,7 @@ class CategoricalTest(test.TestCase, parameterized.TestCase): samples = dist.sample(n, seed=123) samples.set_shape([n, 1, 2]) self.assertEqual(samples.dtype, dtypes.int32) - sample_values = samples.eval() + sample_values = self.evaluate(samples) self.assertFalse(np.any(sample_values < 0)) self.assertFalse(np.any(sample_values > 1)) self.assertAllClose( @@ -371,7 +386,7 @@ class CategoricalTest(test.TestCase, parameterized.TestCase): dist = categorical.Categorical(math_ops.log(histograms) - 50.) samples = dist.sample((100, 100), seed=123) prob = dist.prob(samples) - prob_val = prob.eval() + prob_val = self.evaluate(prob) self.assertAllClose( [0.2**2 + 0.8**2], [prob_val[:, :, :, 0].mean()], atol=1e-2) self.assertAllClose( @@ -393,26 +408,26 @@ class CategoricalTest(test.TestCase, parameterized.TestCase): dist = categorical.Categorical(math_ops.log(histograms) - 50.) prob = dist.prob(1) - self.assertAllClose([[0.8, 0.6]], prob.eval()) + self.assertAllClose([[0.8, 0.6]], self.evaluate(prob)) prob = dist.prob([1]) - self.assertAllClose([[0.8, 0.6]], prob.eval()) + self.assertAllClose([[0.8, 0.6]], self.evaluate(prob)) prob = dist.prob([0, 1]) - self.assertAllClose([[0.2, 0.6]], prob.eval()) + self.assertAllClose([[0.2, 0.6]], self.evaluate(prob)) prob = dist.prob([[0, 1]]) - self.assertAllClose([[0.2, 0.6]], prob.eval()) + self.assertAllClose([[0.2, 0.6]], self.evaluate(prob)) prob = dist.prob([[[0, 1]]]) - self.assertAllClose([[[0.2, 0.6]]], prob.eval()) + self.assertAllClose([[[0.2, 0.6]]], self.evaluate(prob)) prob = dist.prob([[1, 0], [0, 1]]) - self.assertAllClose([[0.8, 0.4], [0.2, 0.6]], prob.eval()) + self.assertAllClose([[0.8, 0.4], [0.2, 0.6]], self.evaluate(prob)) prob = dist.prob([[[1, 1], [1, 0]], [[1, 0], [0, 1]]]) self.assertAllClose([[[0.8, 0.6], [0.8, 0.4]], [[0.8, 0.4], [0.2, 0.6]]], - prob.eval()) + self.evaluate(prob)) def testLogPMFShape(self): with self.cached_session(): @@ -440,12 +455,14 @@ class CategoricalTest(test.TestCase, parameterized.TestCase): self.assertEqual(3, log_prob.get_shape().ndims) self.assertAllEqual([2, 2, 2], log_prob.get_shape()) + @test_util.run_deprecated_v1 def testMode(self): with self.cached_session(): histograms = [[[0.2, 0.8], [0.6, 0.4]]] dist = categorical.Categorical(math_ops.log(histograms) - 50.) self.assertAllEqual(dist.mode().eval(), [[1, 0]]) + @test_util.run_deprecated_v1 def testCategoricalCategoricalKL(self): def np_softmax(logits): @@ -462,7 +479,7 @@ class CategoricalTest(test.TestCase, parameterized.TestCase): b = categorical.Categorical(logits=b_logits) kl = kullback_leibler.kl_divergence(a, b) - kl_val = sess.run(kl) + kl_val = self.evaluate(kl) # Make sure KL(a||a) is 0 kl_same = sess.run(kullback_leibler.kl_divergence(a, a)) diff --git a/tensorflow/python/kernel_tests/distributions/dirichlet_multinomial_test.py b/tensorflow/python/kernel_tests/distributions/dirichlet_multinomial_test.py index d558ca09cc6..c530037e1ed 100644 --- a/tensorflow/python/kernel_tests/distributions/dirichlet_multinomial_test.py +++ b/tensorflow/python/kernel_tests/distributions/dirichlet_multinomial_test.py @@ -22,6 +22,7 @@ from tensorflow.python.eager import backprop from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes from tensorflow.python.framework import tensor_shape +from tensorflow.python.framework import test_util from tensorflow.python.ops import array_ops from tensorflow.python.ops import math_ops from tensorflow.python.ops.distributions import dirichlet_multinomial @@ -36,6 +37,7 @@ class DirichletMultinomialTest(test.TestCase): def setUp(self): self._rng = np.random.RandomState(42) + @test_util.run_deprecated_v1 def testSimpleShapes(self): with self.cached_session(): alpha = np.random.rand(3) @@ -45,6 +47,7 @@ class DirichletMultinomialTest(test.TestCase): self.assertEqual(tensor_shape.TensorShape([3]), dist.event_shape) self.assertEqual(tensor_shape.TensorShape([]), dist.batch_shape) + @test_util.run_deprecated_v1 def testComplexShapes(self): with self.cached_session(): alpha = np.random.rand(3, 2, 2) @@ -55,6 +58,7 @@ class DirichletMultinomialTest(test.TestCase): self.assertEqual(tensor_shape.TensorShape([2]), dist.event_shape) self.assertEqual(tensor_shape.TensorShape([3, 2]), dist.batch_shape) + @test_util.run_deprecated_v1 def testNproperty(self): alpha = [[1., 2, 3]] n = [[5.]] @@ -63,6 +67,7 @@ class DirichletMultinomialTest(test.TestCase): self.assertEqual([1, 1], dist.total_count.get_shape()) self.assertAllClose(n, dist.total_count.eval()) + @test_util.run_deprecated_v1 def testAlphaProperty(self): alpha = [[1., 2, 3]] with self.cached_session(): @@ -70,6 +75,7 @@ class DirichletMultinomialTest(test.TestCase): self.assertEqual([1, 3], dist.concentration.get_shape()) self.assertAllClose(alpha, dist.concentration.eval()) + @test_util.run_deprecated_v1 def testPmfNandCountsAgree(self): alpha = [[1., 2, 3]] n = [[5.]] @@ -83,6 +89,7 @@ class DirichletMultinomialTest(test.TestCase): "last-dimension must sum to `self.total_count`"): dist.prob([3., 3, 0]).eval() + @test_util.run_deprecated_v1 def testPmfNonIntegerCounts(self): alpha = [[1., 2, 3]] n = [[5.]] @@ -110,7 +117,7 @@ class DirichletMultinomialTest(test.TestCase): counts = [1., 0] dist = ds.DirichletMultinomial(1., alpha) pmf = dist.prob(counts) - self.assertAllClose(1 / 3., pmf.eval()) + self.assertAllClose(1 / 3., self.evaluate(pmf)) self.assertEqual((), pmf.get_shape()) def testPmfBothZeroBatchesNontrivialN(self): @@ -122,7 +129,7 @@ class DirichletMultinomialTest(test.TestCase): counts = [3., 2] dist = ds.DirichletMultinomial(5., alpha) pmf = dist.prob(counts) - self.assertAllClose(1 / 7., pmf.eval()) + self.assertAllClose(1 / 7., self.evaluate(pmf)) self.assertEqual((), pmf.get_shape()) def testPmfBothZeroBatchesMultidimensionalN(self): @@ -134,7 +141,7 @@ class DirichletMultinomialTest(test.TestCase): n = np.full([4, 3], 5., dtype=np.float32) dist = ds.DirichletMultinomial(n, alpha) pmf = dist.prob(counts) - self.assertAllClose([[1 / 7., 1 / 7., 1 / 7.]] * 4, pmf.eval()) + self.assertAllClose([[1 / 7., 1 / 7., 1 / 7.]] * 4, self.evaluate(pmf)) self.assertEqual((4, 3), pmf.get_shape()) def testPmfAlphaStretchedInBroadcastWhenSameRank(self): @@ -145,7 +152,7 @@ class DirichletMultinomialTest(test.TestCase): counts = [[1., 0], [0., 1]] dist = ds.DirichletMultinomial([1.], alpha) pmf = dist.prob(counts) - self.assertAllClose([1 / 3., 2 / 3.], pmf.eval()) + self.assertAllClose([1 / 3., 2 / 3.], self.evaluate(pmf)) self.assertAllEqual([2], pmf.get_shape()) def testPmfAlphaStretchedInBroadcastWhenLowerRank(self): @@ -155,7 +162,7 @@ class DirichletMultinomialTest(test.TestCase): alpha = [1., 2] counts = [[1., 0], [0., 1]] pmf = ds.DirichletMultinomial(1., alpha).prob(counts) - self.assertAllClose([1 / 3., 2 / 3.], pmf.eval()) + self.assertAllClose([1 / 3., 2 / 3.], self.evaluate(pmf)) self.assertAllEqual([2], pmf.get_shape()) def testPmfCountsStretchedInBroadcastWhenSameRank(self): @@ -165,7 +172,7 @@ class DirichletMultinomialTest(test.TestCase): alpha = [[1., 2], [2., 3]] counts = [[1., 0]] pmf = ds.DirichletMultinomial([1., 1.], alpha).prob(counts) - self.assertAllClose([1 / 3., 2 / 5.], pmf.eval()) + self.assertAllClose([1 / 3., 2 / 5.], self.evaluate(pmf)) self.assertAllEqual([2], pmf.get_shape()) def testPmfCountsStretchedInBroadcastWhenLowerRank(self): @@ -175,9 +182,10 @@ class DirichletMultinomialTest(test.TestCase): alpha = [[1., 2], [2., 3]] counts = [1., 0] pmf = ds.DirichletMultinomial(1., alpha).prob(counts) - self.assertAllClose([1 / 3., 2 / 5.], pmf.eval()) + self.assertAllClose([1 / 3., 2 / 5.], self.evaluate(pmf)) self.assertAllEqual([2], pmf.get_shape()) + @test_util.run_deprecated_v1 def testPmfForOneVoteIsTheMeanWithOneRecordInput(self): # The probabilities of one vote falling into class k is the mean for class # k. @@ -194,6 +202,7 @@ class DirichletMultinomialTest(test.TestCase): self.assertAllEqual([3], mean.shape) self.assertAllEqual([], pmf.shape) + @test_util.run_deprecated_v1 def testMeanDoubleTwoVotes(self): # The probabilities of two votes falling into class k for # DirichletMultinomial(2, alpha) is twice as much as the probability of one @@ -215,6 +224,7 @@ class DirichletMultinomialTest(test.TestCase): self.assertAllClose(mean2[class_num], 2 * mean1[class_num]) self.assertAllEqual([3], mean1.shape) + @test_util.run_deprecated_v1 def testCovarianceFromSampling(self): # We will test mean, cov, var, stddev on a DirichletMultinomial constructed # via broadcast between alpha, n. @@ -289,7 +299,7 @@ class DirichletMultinomialTest(test.TestCase): expected_covariance = n * (n + alpha_0) / (1 + alpha_0) * shared_matrix self.assertEqual([2, 2], covariance.get_shape()) - self.assertAllClose(expected_covariance, covariance.eval()) + self.assertAllClose(expected_covariance, self.evaluate(covariance)) def testCovarianceNAlphaBroadcast(self): alpha_v = [1., 2, 3] @@ -327,7 +337,7 @@ class DirichletMultinomialTest(test.TestCase): ns * (ns + alpha_0) / (1 + alpha_0))[..., array_ops.newaxis] self.assertEqual([4, 3, 3], covariance.get_shape()) - self.assertAllClose(expected_covariance, covariance.eval()) + self.assertAllClose(expected_covariance, self.evaluate(covariance)) def testCovarianceMultidimensional(self): alpha = np.random.rand(3, 5, 4).astype(np.float32) @@ -353,7 +363,7 @@ class DirichletMultinomialTest(test.TestCase): with self.cached_session(): dist = ds.DirichletMultinomial(0., alpha) pmf = dist.prob(counts) - self.assertAllClose(1.0, pmf.eval()) + self.assertAllClose(1.0, self.evaluate(pmf)) self.assertEqual((), pmf.get_shape()) def testLargeTauGivesPreciseProbabilities(self): @@ -368,7 +378,7 @@ class DirichletMultinomialTest(test.TestCase): with self.cached_session(): dist = ds.DirichletMultinomial(1., alpha) pmf = dist.prob(counts) - self.assertAllClose(0.8, pmf.eval(), atol=1e-4) + self.assertAllClose(0.8, self.evaluate(pmf), atol=1e-4) self.assertEqual((), pmf.get_shape()) # Two (three sided) coin flips. Prob[coin 3] = 0.8. @@ -376,7 +386,7 @@ class DirichletMultinomialTest(test.TestCase): with self.cached_session(): dist = ds.DirichletMultinomial(2., alpha) pmf = dist.prob(counts) - self.assertAllClose(0.8**2, pmf.eval(), atol=1e-2) + self.assertAllClose(0.8**2, self.evaluate(pmf), atol=1e-2) self.assertEqual((), pmf.get_shape()) # Three (three sided) coin flips. @@ -384,7 +394,7 @@ class DirichletMultinomialTest(test.TestCase): with self.cached_session(): dist = ds.DirichletMultinomial(3., alpha) pmf = dist.prob(counts) - self.assertAllClose(3 * 0.1 * 0.8 * 0.8, pmf.eval(), atol=1e-2) + self.assertAllClose(3 * 0.1 * 0.8 * 0.8, self.evaluate(pmf), atol=1e-2) self.assertEqual((), pmf.get_shape()) def testSmallTauPrefersCorrelatedResults(self): @@ -399,7 +409,7 @@ class DirichletMultinomialTest(test.TestCase): with self.cached_session(): dist = ds.DirichletMultinomial(1., alpha) pmf = dist.prob(counts) - self.assertAllClose(0.5, pmf.eval()) + self.assertAllClose(0.5, self.evaluate(pmf)) self.assertEqual((), pmf.get_shape()) # If there are two draws, it is much more likely that they are the same. @@ -409,9 +419,10 @@ class DirichletMultinomialTest(test.TestCase): dist = ds.DirichletMultinomial(2., alpha) pmf_same = dist.prob(counts_same) pmf_different = dist.prob(counts_different) - self.assertLess(5 * pmf_different.eval(), pmf_same.eval()) + self.assertLess(5 * self.evaluate(pmf_different), self.evaluate(pmf_same)) self.assertEqual((), pmf_same.get_shape()) + @test_util.run_deprecated_v1 def testNonStrictTurnsOffAllChecks(self): # Make totally invalid input. with self.cached_session(): @@ -421,6 +432,7 @@ class DirichletMultinomialTest(test.TestCase): dist = ds.DirichletMultinomial(n, alpha, validate_args=False) dist.prob(counts).eval() # Should not raise. + @test_util.run_deprecated_v1 def testSampleUnbiasedNonScalarBatch(self): with self.cached_session() as sess: dist = ds.DirichletMultinomial( @@ -450,6 +462,7 @@ class DirichletMultinomialTest(test.TestCase): self.assertAllClose( actual_covariance_, sample_covariance_, atol=0., rtol=0.20) + @test_util.run_deprecated_v1 def testSampleUnbiasedScalarBatch(self): with self.cached_session() as sess: dist = ds.DirichletMultinomial( diff --git a/tensorflow/python/kernel_tests/distributions/identity_bijector_test.py b/tensorflow/python/kernel_tests/distributions/identity_bijector_test.py index e35a8e1cdd7..62b562387d0 100644 --- a/tensorflow/python/kernel_tests/distributions/identity_bijector_test.py +++ b/tensorflow/python/kernel_tests/distributions/identity_bijector_test.py @@ -18,6 +18,7 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function +from tensorflow.python.framework import test_util from tensorflow.python.ops.distributions import bijector_test_util from tensorflow.python.ops.distributions import identity_bijector from tensorflow.python.platform import test @@ -41,6 +42,7 @@ class IdentityBijectorTest(test.TestCase): self.evaluate( bijector.forward_log_det_jacobian(x, event_ndims=3))) + @test_util.run_deprecated_v1 def testScalarCongruency(self): with self.cached_session(): bijector = identity_bijector.Identity() diff --git a/tensorflow/python/kernel_tests/distributions/kullback_leibler_test.py b/tensorflow/python/kernel_tests/distributions/kullback_leibler_test.py index e77e1117d49..1e967de570f 100644 --- a/tensorflow/python/kernel_tests/distributions/kullback_leibler_test.py +++ b/tensorflow/python/kernel_tests/distributions/kullback_leibler_test.py @@ -18,6 +18,7 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function +from tensorflow.python.framework import test_util from tensorflow.python.ops import array_ops from tensorflow.python.ops.distributions import kullback_leibler from tensorflow.python.ops.distributions import normal @@ -45,6 +46,7 @@ class KLTest(test.TestCase): a = MyDist(loc=0.0, scale=1.0) self.assertEqual("OK", kullback_leibler.kl_divergence(a, a, name="OK")) + @test_util.run_deprecated_v1 def testDomainErrorExceptions(self): class MyDistException(normal.Normal): @@ -63,17 +65,17 @@ class KLTest(test.TestCase): kl = kullback_leibler.kl_divergence(a, a, allow_nan_stats=False) with self.assertRaisesOpError( "KL calculation between .* and .* returned NaN values"): - kl.eval() + self.evaluate(kl) with self.assertRaisesOpError( "KL calculation between .* and .* returned NaN values"): a.kl_divergence(a).eval() a = MyDistException(loc=0.0, scale=1.0, allow_nan_stats=True) kl_ok = kullback_leibler.kl_divergence(a, a) - self.assertAllEqual([float("nan")], kl_ok.eval()) + self.assertAllEqual([float("nan")], self.evaluate(kl_ok)) self_kl_ok = a.kl_divergence(a) - self.assertAllEqual([float("nan")], self_kl_ok.eval()) + self.assertAllEqual([float("nan")], self.evaluate(self_kl_ok)) cross_ok = a.cross_entropy(a) - self.assertAllEqual([float("nan")], cross_ok.eval()) + self.assertAllEqual([float("nan")], self.evaluate(cross_ok)) def testRegistrationFailures(self): diff --git a/tensorflow/python/kernel_tests/distributions/multinomial_test.py b/tensorflow/python/kernel_tests/distributions/multinomial_test.py index 3840d7331ca..b3f3416a52f 100644 --- a/tensorflow/python/kernel_tests/distributions/multinomial_test.py +++ b/tensorflow/python/kernel_tests/distributions/multinomial_test.py @@ -127,7 +127,7 @@ class MultinomialTest(test.TestCase): p = [0.5, 0.5] counts = [1., 0] pmf = multinomial.Multinomial(total_count=1., probs=p).prob(counts) - self.assertAllClose(0.5, pmf.eval()) + self.assertAllClose(0.5, self.evaluate(pmf)) self.assertEqual((), pmf.get_shape()) def testPmfBothZeroBatchesNontrivialN(self): @@ -138,7 +138,7 @@ class MultinomialTest(test.TestCase): dist = multinomial.Multinomial(total_count=5., probs=p) pmf = dist.prob(counts) # 5 choose 3 = 5 choose 2 = 10. 10 * (.9)^2 * (.1)^3 = 81/10000. - self.assertAllClose(81. / 10000, pmf.eval()) + self.assertAllClose(81. / 10000, self.evaluate(pmf)) self.assertEqual((), pmf.get_shape()) def testPmfPStretchedInBroadcastWhenSameRank(self): @@ -146,7 +146,7 @@ class MultinomialTest(test.TestCase): p = [[0.1, 0.9]] counts = [[1., 0], [0, 1]] pmf = multinomial.Multinomial(total_count=1., probs=p).prob(counts) - self.assertAllClose([0.1, 0.9], pmf.eval()) + self.assertAllClose([0.1, 0.9], self.evaluate(pmf)) self.assertEqual((2), pmf.get_shape()) def testPmfPStretchedInBroadcastWhenLowerRank(self): @@ -154,7 +154,7 @@ class MultinomialTest(test.TestCase): p = [0.1, 0.9] counts = [[1., 0], [0, 1]] pmf = multinomial.Multinomial(total_count=1., probs=p).prob(counts) - self.assertAllClose([0.1, 0.9], pmf.eval()) + self.assertAllClose([0.1, 0.9], self.evaluate(pmf)) self.assertEqual((2), pmf.get_shape()) def testPmfCountsStretchedInBroadcastWhenSameRank(self): @@ -182,7 +182,7 @@ class MultinomialTest(test.TestCase): # [2] counts = [2., 1] pmf = multinomial.Multinomial(total_count=n, probs=p).prob(counts) - pmf.eval() + self.evaluate(pmf) self.assertEqual(pmf.get_shape(), (2, 2)) def testPmfShapeCountsPStretchedN(self): @@ -191,7 +191,7 @@ class MultinomialTest(test.TestCase): counts = [3., 2] n = np.full([4, 3], 5., dtype=np.float32) pmf = multinomial.Multinomial(total_count=n, probs=p).prob(counts) - pmf.eval() + self.evaluate(pmf) self.assertEqual((4, 3), pmf.get_shape()) def testMultinomialMean(self): diff --git a/tensorflow/python/kernel_tests/distributions/normal_test.py b/tensorflow/python/kernel_tests/distributions/normal_test.py index 6625a88843f..f2a193e69bd 100644 --- a/tensorflow/python/kernel_tests/distributions/normal_test.py +++ b/tensorflow/python/kernel_tests/distributions/normal_test.py @@ -511,6 +511,7 @@ class NormalTest(test.TestCase): self.assertAllEqual(self.evaluate(normal.event_shape_tensor()), []) self.assertEqual(normal.event_shape, tensor_shape.TensorShape([])) + @test_util.run_deprecated_v1 def testNormalShapeWithPlaceholders(self): mu = array_ops.placeholder(dtype=dtypes.float32) sigma = array_ops.placeholder(dtype=dtypes.float32) diff --git a/tensorflow/python/kernel_tests/distributions/special_math_test.py b/tensorflow/python/kernel_tests/distributions/special_math_test.py index cc43e121686..d97fcfa655f 100644 --- a/tensorflow/python/kernel_tests/distributions/special_math_test.py +++ b/tensorflow/python/kernel_tests/distributions/special_math_test.py @@ -104,6 +104,7 @@ class NdtriTest(test.TestCase): x = special_math.ndtri(p) self.assertAllClose(expected_x, self.evaluate(x), atol=0.) + @test_util.run_deprecated_v1 def testNdtriDynamicShape(self): """Verifies that ndtri computation is correct.""" with self.cached_session() as sess: @@ -213,9 +214,11 @@ class NdtrTest(test.TestCase): rtol=error_spec.rtol, atol=error_spec.atol) + @test_util.run_deprecated_v1 def test_float32(self): self._test_grid(np.float32, self._grid32, self._error32) + @test_util.run_deprecated_v1 def test_float64(self): self._test_grid(np.float64, self._grid64, self._error64) @@ -338,10 +341,12 @@ class NdtrGradientTest(test.TestCase): rtol=error_spec.rtol, atol=error_spec.atol) + @test_util.run_deprecated_v1 def test_float32(self): self._test_grad_accuracy(np.float32, self._grid, self._error32) self._test_grad_finite(np.float32) + @test_util.run_deprecated_v1 def test_float64(self): self._test_grad_accuracy(np.float64, self._grid, self._error64) self._test_grad_finite(np.float64) @@ -362,7 +367,7 @@ class ErfInvTest(test.TestCase): expected_x = special.erfinv(x) x = special_math.erfinv(x) - self.assertAllClose(expected_x, x.eval(), atol=0.) + self.assertAllClose(expected_x, self.evaluate(x), atol=0.) def testErfInvIntegerInput(self): with self.cached_session(): @@ -418,6 +423,7 @@ class LogCDFLaplaceTest(test.TestCase): rtol=error_spec.rtol, atol=error_spec.atol) + @test_util.run_deprecated_v1 def test_float32_lower_and_mid_segment_scipy_float32_ok(self): # Choose values mild enough that we can use scipy in float32, which will # allow for a high accuracy match to scipy (since we both use float32). @@ -427,6 +433,7 @@ class LogCDFLaplaceTest(test.TestCase): GridSpec(min=-10, max=self.CUTOFF_FLOAT32_UPPER - 5, shape=[100]), ErrorSpec(rtol=5e-4, atol=0)) + @test_util.run_deprecated_v1 def test_float32_all_segments_with_scipy_float64_ok(self): # Choose values outside the range where scipy float32 works. # Let scipy use float64. This means we @@ -437,6 +444,7 @@ class LogCDFLaplaceTest(test.TestCase): GridSpec(min=-50, max=self.CUTOFF_FLOAT32_UPPER + 5, shape=[100]), ErrorSpec(rtol=0.05, atol=0)) + @test_util.run_deprecated_v1 def test_float32_extreme_values_result_and_gradient_finite_and_nonzero(self): with self.cached_session() as sess: # On the lower branch, log_cdf_laplace(x) = x, so we know this will be @@ -448,7 +456,7 @@ class LogCDFLaplaceTest(test.TestCase): actual = sm.log_cdf_laplace(grid) grad = gradients_impl.gradients(actual, grid)[0] - actual_, grad_ = sess.run([actual, grad]) + actual_, grad_ = self.evaluate([actual, grad]) # isfinite checks for NaN and Inf. self.assertAllTrue(np.isfinite(actual_)) @@ -456,6 +464,7 @@ class LogCDFLaplaceTest(test.TestCase): self.assertFalse(np.any(actual_ == 0)) self.assertFalse(np.any(grad_ == 0)) + @test_util.run_deprecated_v1 def test_float64_extreme_values_result_and_gradient_finite_and_nonzero(self): with self.cached_session() as sess: # On the lower branch, log_cdf_laplace(x) = x, so we know this will be @@ -467,7 +476,7 @@ class LogCDFLaplaceTest(test.TestCase): actual = sm.log_cdf_laplace(grid) grad = gradients_impl.gradients(actual, grid)[0] - actual_, grad_ = sess.run([actual, grad]) + actual_, grad_ = self.evaluate([actual, grad]) # isfinite checks for NaN and Inf. self.assertAllTrue(np.isfinite(actual_)) diff --git a/tensorflow/python/kernel_tests/distributions/util_test.py b/tensorflow/python/kernel_tests/distributions/util_test.py index f4e651b25bb..030ad601bf4 100644 --- a/tensorflow/python/kernel_tests/distributions/util_test.py +++ b/tensorflow/python/kernel_tests/distributions/util_test.py @@ -59,6 +59,7 @@ def _logit(x): class AssertCloseTest(test.TestCase): + @test_util.run_deprecated_v1 def testAssertIntegerForm(self): # This should only be detected as an integer. x = array_ops.placeholder(dtypes.float32) @@ -112,6 +113,7 @@ class MaybeGetStaticTest(test.TestCase): self.assertAllClose( np.array(2.), du.maybe_get_static_value(x, dtype=np.float64)) + @test_util.run_deprecated_v1 def testGetStaticPlaceholder(self): x = array_ops.placeholder(dtype=dtypes.int32, shape=[1]) self.assertEqual(None, du.maybe_get_static_value(x)) @@ -235,6 +237,7 @@ class GetLogitsAndProbsTest(test.TestCase): probs=p4, multidimensional=True, validate_args=False) self.evaluate(prob) + @test_util.run_deprecated_v1 def testProbsMultidimShape(self): with self.cached_session(): with self.assertRaises(ValueError): @@ -249,6 +252,7 @@ class GetLogitsAndProbsTest(test.TestCase): probs=p, multidimensional=True, validate_args=True) prob.eval(feed_dict={p: np.ones([int(2**11+1)])}) + @test_util.run_deprecated_v1 def testLogitsMultidimShape(self): with self.cached_session(): with self.assertRaises(ValueError): @@ -266,6 +270,7 @@ class GetLogitsAndProbsTest(test.TestCase): class EmbedCheckCategoricalEventShapeTest(test.TestCase): + @test_util.run_deprecated_v1 def testTooSmall(self): with self.cached_session(): with self.assertRaises(ValueError): @@ -280,6 +285,7 @@ class EmbedCheckCategoricalEventShapeTest(test.TestCase): param) checked_param.eval(feed_dict={param: np.ones([1])}) + @test_util.run_deprecated_v1 def testTooLarge(self): with self.cached_session(): with self.assertRaises(ValueError): @@ -305,6 +311,7 @@ class EmbedCheckCategoricalEventShapeTest(test.TestCase): class EmbedCheckIntegerCastingClosedTest(test.TestCase): + @test_util.run_deprecated_v1 def testCorrectlyAssertsNonnegative(self): with self.cached_session(): with self.assertRaisesOpError("Elements must be non-negative"): @@ -313,6 +320,7 @@ class EmbedCheckIntegerCastingClosedTest(test.TestCase): x, target_dtype=dtypes.int16) x_checked.eval(feed_dict={x: np.array([1, -1], dtype=np.float16)}) + @test_util.run_deprecated_v1 def testCorrectlyAssersIntegerForm(self): with self.cached_session(): with self.assertRaisesOpError("Elements must be int16-equivalent."): @@ -321,6 +329,7 @@ class EmbedCheckIntegerCastingClosedTest(test.TestCase): x, target_dtype=dtypes.int16) x_checked.eval(feed_dict={x: np.array([1, 1.5], dtype=np.float16)}) + @test_util.run_deprecated_v1 def testCorrectlyAssertsLargestPossibleInteger(self): with self.cached_session(): with self.assertRaisesOpError("Elements cannot exceed 32767."): @@ -329,6 +338,7 @@ class EmbedCheckIntegerCastingClosedTest(test.TestCase): x, target_dtype=dtypes.int16) x_checked.eval(feed_dict={x: np.array([1, 2**15], dtype=np.int32)}) + @test_util.run_deprecated_v1 def testCorrectlyAssertsSmallestPossibleInteger(self): with self.cached_session(): with self.assertRaisesOpError("Elements cannot be smaller than 0."): @@ -369,6 +379,7 @@ class LogCombinationsTest(test.TestCase): class DynamicShapeTest(test.TestCase): + @test_util.run_deprecated_v1 def testSameDynamicShape(self): with self.cached_session(): scalar = constant_op.constant(2.0) @@ -493,6 +504,7 @@ class RotateTransposeTest(test.TestCase): self._np_rotate_transpose(x, shift), self.evaluate(y)) self.assertAllEqual(np.roll(x.shape, shift), y.get_shape().as_list()) + @test_util.run_deprecated_v1 def testRollDynamic(self): with self.cached_session() as sess: x = array_ops.placeholder(dtypes.float32) @@ -511,6 +523,7 @@ class RotateTransposeTest(test.TestCase): class PickVectorTest(test.TestCase): + @test_util.run_deprecated_v1 def testCorrectlyPicksVector(self): with self.cached_session(): x = np.arange(10, 12) @@ -529,36 +542,42 @@ class PickVectorTest(test.TestCase): class PreferStaticRankTest(test.TestCase): + @test_util.run_deprecated_v1 def testNonEmptyConstantTensor(self): x = array_ops.zeros((2, 3, 4)) rank = du.prefer_static_rank(x) self.assertIsInstance(rank, np.ndarray) self.assertEqual(3, rank) + @test_util.run_deprecated_v1 def testEmptyConstantTensor(self): x = constant_op.constant([]) rank = du.prefer_static_rank(x) self.assertIsInstance(rank, np.ndarray) self.assertEqual(1, rank) + @test_util.run_deprecated_v1 def testScalarTensor(self): x = constant_op.constant(1.) rank = du.prefer_static_rank(x) self.assertIsInstance(rank, np.ndarray) self.assertEqual(0, rank) + @test_util.run_deprecated_v1 def testDynamicRankEndsUpBeingNonEmpty(self): x = array_ops.placeholder(np.float64, shape=None) rank = du.prefer_static_rank(x) with self.cached_session(): self.assertAllEqual(2, rank.eval(feed_dict={x: np.zeros((2, 3))})) + @test_util.run_deprecated_v1 def testDynamicRankEndsUpBeingEmpty(self): x = array_ops.placeholder(np.int32, shape=None) rank = du.prefer_static_rank(x) with self.cached_session(): self.assertAllEqual(1, rank.eval(feed_dict={x: []})) + @test_util.run_deprecated_v1 def testDynamicRankEndsUpBeingScalar(self): x = array_ops.placeholder(np.int32, shape=None) rank = du.prefer_static_rank(x) @@ -568,36 +587,42 @@ class PreferStaticRankTest(test.TestCase): class PreferStaticShapeTest(test.TestCase): + @test_util.run_deprecated_v1 def testNonEmptyConstantTensor(self): x = array_ops.zeros((2, 3, 4)) shape = du.prefer_static_shape(x) self.assertIsInstance(shape, np.ndarray) self.assertAllEqual(np.array([2, 3, 4]), shape) + @test_util.run_deprecated_v1 def testEmptyConstantTensor(self): x = constant_op.constant([]) shape = du.prefer_static_shape(x) self.assertIsInstance(shape, np.ndarray) self.assertAllEqual(np.array([0]), shape) + @test_util.run_deprecated_v1 def testScalarTensor(self): x = constant_op.constant(1.) shape = du.prefer_static_shape(x) self.assertIsInstance(shape, np.ndarray) self.assertAllEqual(np.array([]), shape) + @test_util.run_deprecated_v1 def testDynamicShapeEndsUpBeingNonEmpty(self): x = array_ops.placeholder(np.float64, shape=None) shape = du.prefer_static_shape(x) with self.cached_session(): self.assertAllEqual((2, 3), shape.eval(feed_dict={x: np.zeros((2, 3))})) + @test_util.run_deprecated_v1 def testDynamicShapeEndsUpBeingEmpty(self): x = array_ops.placeholder(np.int32, shape=None) shape = du.prefer_static_shape(x) with self.cached_session(): self.assertAllEqual(np.array([0]), shape.eval(feed_dict={x: []})) + @test_util.run_deprecated_v1 def testDynamicShapeEndsUpBeingScalar(self): x = array_ops.placeholder(np.int32, shape=None) shape = du.prefer_static_shape(x) @@ -607,24 +632,28 @@ class PreferStaticShapeTest(test.TestCase): class PreferStaticValueTest(test.TestCase): + @test_util.run_deprecated_v1 def testNonEmptyConstantTensor(self): x = array_ops.zeros((2, 3, 4)) value = du.prefer_static_value(x) self.assertIsInstance(value, np.ndarray) self.assertAllEqual(np.zeros((2, 3, 4)), value) + @test_util.run_deprecated_v1 def testEmptyConstantTensor(self): x = constant_op.constant([]) value = du.prefer_static_value(x) self.assertIsInstance(value, np.ndarray) self.assertAllEqual(np.array([]), value) + @test_util.run_deprecated_v1 def testScalarTensor(self): x = constant_op.constant(1.) value = du.prefer_static_value(x) self.assertIsInstance(value, np.ndarray) self.assertAllEqual(np.array(1.), value) + @test_util.run_deprecated_v1 def testDynamicValueEndsUpBeingNonEmpty(self): x = array_ops.placeholder(np.float64, shape=None) value = du.prefer_static_value(x) @@ -632,12 +661,14 @@ class PreferStaticValueTest(test.TestCase): self.assertAllEqual(np.zeros((2, 3)), value.eval(feed_dict={x: np.zeros((2, 3))})) + @test_util.run_deprecated_v1 def testDynamicValueEndsUpBeingEmpty(self): x = array_ops.placeholder(np.int32, shape=None) value = du.prefer_static_value(x) with self.cached_session(): self.assertAllEqual(np.array([]), value.eval(feed_dict={x: []})) + @test_util.run_deprecated_v1 def testDynamicValueEndsUpBeingScalar(self): x = array_ops.placeholder(np.int32, shape=None) value = du.prefer_static_value(x) @@ -698,43 +729,55 @@ class FillTriangularTest(test.TestCase): self.assertAllClose(expected, actual_, rtol=1e-8, atol=1e-9) self.assertAllClose(x_, grad_actual_, rtol=1e-8, atol=1e-9) + @test_util.run_deprecated_v1 def testCorrectlyMakes1x1TriLower(self): self._run_test(self._rng.randn(3, int(1*2/2))) + @test_util.run_deprecated_v1 def testCorrectlyMakesNoBatchTriLower(self): self._run_test(self._rng.randn(int(4*5/2))) + @test_util.run_deprecated_v1 def testCorrectlyMakesBatchTriLower(self): self._run_test(self._rng.randn(2, 3, int(3*4/2))) + @test_util.run_deprecated_v1 def testCorrectlyMakesBatchTriLowerUnknownShape(self): self._run_test(self._rng.randn(2, 3, int(3*4/2)), use_deferred_shape=True) + @test_util.run_deprecated_v1 def testCorrectlyMakesBatch7x7TriLowerUnknownShape(self): self._run_test(self._rng.randn(2, 3, int(7*8/2)), use_deferred_shape=True) + @test_util.run_deprecated_v1 def testCorrectlyMakesBatch7x7TriLower(self): self._run_test(self._rng.randn(2, 3, int(7*8/2))) + @test_util.run_deprecated_v1 def testCorrectlyMakes1x1TriUpper(self): self._run_test(self._rng.randn(3, int(1*2/2)), upper=True) + @test_util.run_deprecated_v1 def testCorrectlyMakesNoBatchTriUpper(self): self._run_test(self._rng.randn(int(4*5/2)), upper=True) + @test_util.run_deprecated_v1 def testCorrectlyMakesBatchTriUpper(self): self._run_test(self._rng.randn(2, 2, int(3*4/2)), upper=True) + @test_util.run_deprecated_v1 def testCorrectlyMakesBatchTriUpperUnknownShape(self): self._run_test(self._rng.randn(2, 2, int(3*4/2)), use_deferred_shape=True, upper=True) + @test_util.run_deprecated_v1 def testCorrectlyMakesBatch7x7TriUpperUnknownShape(self): self._run_test(self._rng.randn(2, 3, int(7*8/2)), use_deferred_shape=True, upper=True) + @test_util.run_deprecated_v1 def testCorrectlyMakesBatch7x7TriUpper(self): self._run_test(self._rng.randn(2, 3, int(7*8/2)), upper=True) @@ -773,6 +816,7 @@ class ReduceWeightedLogSumExp(test.TestCase): m = np.squeeze(m, axis=axis) return m + np.log(sgn * sum_), sgn + @test_util.run_deprecated_v1 def testNoWeights(self): logx_ = np.array([[0., -1, 1000.], [0, 1, -1000.], @@ -805,7 +849,7 @@ class ReduceWeightedLogSumExp(test.TestCase): w = constant_op.constant(w_) actual, actual_sgn = du.reduce_weighted_logsumexp( logx, w, axis=-1, return_sign=True) - [actual_, actual_sgn_] = sess.run([actual, actual_sgn]) + [actual_, actual_sgn_] = self.evaluate([actual, actual_sgn]) self.assertAllEqual(expected, actual_) self.assertAllEqual([-1., -1, 1], actual_sgn_) @@ -823,7 +867,7 @@ class ReduceWeightedLogSumExp(test.TestCase): w = constant_op.constant(w_) actual, actual_sgn = du.reduce_weighted_logsumexp( logx, w, axis=-1, return_sign=True, keep_dims=True) - [actual_, actual_sgn_] = sess.run([actual, actual_sgn]) + [actual_, actual_sgn_] = self.evaluate([actual, actual_sgn]) self.assertAllEqual(expected, actual_) self.assertAllEqual([[-1.], [-1], [1]], actual_sgn_) @@ -903,6 +947,7 @@ class SoftplusTest(test.TestCase): self.assertAllEqual(np.ones_like(tf_softplus_inverse).astype(np.bool), np.isfinite(tf_softplus_inverse)) + @test_util.run_deprecated_v1 def testNumbers(self): for t in [np.float16, np.float32, np.float64]: lower = {np.float16: -15, np.float32: -50, np.float64: -50}.get(t, -100) @@ -933,6 +978,7 @@ class SoftplusTest(test.TestCase): ], use_gpu=True) + @test_util.run_deprecated_v1 def testGradient(self): with self.cached_session(): x = constant_op.constant( @@ -949,6 +995,7 @@ class SoftplusTest(test.TestCase): tf_logging.vlog(2, "softplus (float) gradient err = ", err) self.assertLess(err, 1e-4) + @test_util.run_deprecated_v1 def testInverseSoftplusGradientNeverNan(self): with self.cached_session(): # Note that this range contains both zero and inf. @@ -958,6 +1005,7 @@ class SoftplusTest(test.TestCase): # Equivalent to `assertAllFalse` (if it existed). self.assertAllEqual(np.zeros_like(grads).astype(np.bool), np.isnan(grads)) + @test_util.run_deprecated_v1 def testInverseSoftplusGradientFinite(self): with self.cached_session(): # This range of x is all finite, and so is 1 / x. So the diff --git a/tensorflow/python/kernel_tests/division_future_test.py b/tensorflow/python/kernel_tests/division_future_test.py index e477bdc73b9..85c85809d3f 100644 --- a/tensorflow/python/kernel_tests/division_future_test.py +++ b/tensorflow/python/kernel_tests/division_future_test.py @@ -65,7 +65,7 @@ class DivisionTestCase(test.TestCase): tf_floordiv = tf_x // tf_y check(floordiv, tf_floordiv) # Do only one sess.run for speed - for f, (x, y) in zip(checks, sess.run(tensors)): + for f, (x, y) in zip(checks, self.evaluate(tensors)): f(x, y) diff --git a/tensorflow/python/kernel_tests/division_past_test.py b/tensorflow/python/kernel_tests/division_past_test.py index 63951b5b382..38bb18631ab 100644 --- a/tensorflow/python/kernel_tests/division_past_test.py +++ b/tensorflow/python/kernel_tests/division_past_test.py @@ -64,7 +64,7 @@ class DivisionTestCase(test.TestCase): tf_floordiv = tf_x // tf_y check(floordiv, tf_floordiv) # Do only one sess.run for speed - for f, (x, y) in zip(checks, sess.run(tensors)): + for f, (x, y) in zip(checks, self.evaluate(tensors)): f(x, y) diff --git a/tensorflow/python/kernel_tests/draw_bounding_box_op_test.py b/tensorflow/python/kernel_tests/draw_bounding_box_op_test.py index c6558762809..6aa757e293e 100644 --- a/tensorflow/python/kernel_tests/draw_bounding_box_op_test.py +++ b/tensorflow/python/kernel_tests/draw_bounding_box_op_test.py @@ -87,7 +87,7 @@ class DrawBoundingBoxOpTest(test.TestCase): image = array_ops.expand_dims(image, 0) image = image_ops.draw_bounding_boxes(image, bboxes) with self.cached_session(use_gpu=False) as sess: - op_drawn_image = np.squeeze(sess.run(image), 0) + op_drawn_image = np.squeeze(self.evaluate(image), 0) self.assertAllEqual(test_drawn_image, op_drawn_image) def testDrawBoundingBoxRGBColorCycling(self): diff --git a/tensorflow/python/kernel_tests/duplicate_op_test.py b/tensorflow/python/kernel_tests/duplicate_op_test.py index 654267a5825..fef3127d4a8 100644 --- a/tensorflow/python/kernel_tests/duplicate_op_test.py +++ b/tensorflow/python/kernel_tests/duplicate_op_test.py @@ -20,6 +20,7 @@ from __future__ import print_function import os from tensorflow.python.framework import load_library +from tensorflow.python.framework import test_util from tensorflow.python.ops import math_ops from tensorflow.python.platform import resource_loader from tensorflow.python.platform import test @@ -27,6 +28,7 @@ from tensorflow.python.platform import test class DuplicateOpTest(test.TestCase): + @test_util.run_deprecated_v1 def testBasic(self): library_filename = os.path.join(resource_loader.get_data_files_path(), 'duplicate_op.so') diff --git a/tensorflow/python/kernel_tests/dynamic_partition_op_test.py b/tensorflow/python/kernel_tests/dynamic_partition_op_test.py index 07da855a017..8c448194076 100644 --- a/tensorflow/python/kernel_tests/dynamic_partition_op_test.py +++ b/tensorflow/python/kernel_tests/dynamic_partition_op_test.py @@ -25,6 +25,7 @@ from six.moves import xrange # pylint: disable=redefined-builtin from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes +from tensorflow.python.framework import test_util from tensorflow.python.ops import array_ops from tensorflow.python.ops import data_flow_ops from tensorflow.python.ops import gradients_impl @@ -34,13 +35,14 @@ from tensorflow.python.platform import test class DynamicPartitionTest(test.TestCase): + @test_util.run_deprecated_v1 def testSimpleOneDimensional(self): with self.session(use_gpu=True) as sess: data = constant_op.constant([0, 13, 2, 39, 4, 17], dtype=dtypes.float32) indices = constant_op.constant([0, 0, 2, 3, 2, 1]) partitions = data_flow_ops.dynamic_partition( data, indices, num_partitions=4) - partition_vals = sess.run(partitions) + partition_vals = self.evaluate(partitions) self.assertEqual(4, len(partition_vals)) self.assertAllEqual([0, 13], partition_vals[0]) @@ -54,6 +56,7 @@ class DynamicPartitionTest(test.TestCase): self.assertEqual([None], partitions[2].get_shape().as_list()) self.assertEqual([None], partitions[3].get_shape().as_list()) + @test_util.run_deprecated_v1 def testSimpleTwoDimensional(self): with self.session(use_gpu=True) as sess: data = constant_op.constant([[0, 1, 2], [3, 4, 5], [6, 7, 8], [9, 10, 11], @@ -62,7 +65,7 @@ class DynamicPartitionTest(test.TestCase): indices = constant_op.constant([0, 0, 2, 3, 2, 1]) partitions = data_flow_ops.dynamic_partition( data, indices, num_partitions=4) - partition_vals = sess.run(partitions) + partition_vals = self.evaluate(partitions) self.assertEqual(4, len(partition_vals)) self.assertAllEqual([[0, 1, 2], [3, 4, 5]], partition_vals[0]) @@ -87,7 +90,7 @@ class DynamicPartitionTest(test.TestCase): indices = constant_op.constant(indices_list, dtype=dtypes.int32) partitions = data_flow_ops.dynamic_partition( data, indices, num_partitions=2) - partition_vals = sess.run(partitions) + partition_vals = self.evaluate(partitions) self.assertEqual(2, len(partition_vals)) self.assertAllEqual(part1, partition_vals[0]) @@ -109,7 +112,7 @@ class DynamicPartitionTest(test.TestCase): indices = constant_op.constant(indices_list, dtype=dtypes.int32) partitions = data_flow_ops.dynamic_partition( data, indices, num_partitions=num_partitions) - partition_vals = sess.run(partitions) + partition_vals = self.evaluate(partitions) self.assertEqual(num_partitions, len(partition_vals)) for i in range(num_partitions): @@ -125,7 +128,7 @@ class DynamicPartitionTest(test.TestCase): indices = constant_op.constant(indices_list, dtype=dtypes.int32) partitions = data_flow_ops.dynamic_partition( data, indices, num_partitions=2) - partition_vals = sess.run(partitions) + partition_vals = self.evaluate(partitions) self.assertEqual(2, len(partition_vals)) self.assertAllEqual([3 + 4j, 7 + 8j], partition_vals[0]) @@ -138,7 +141,7 @@ class DynamicPartitionTest(test.TestCase): indices = 3 partitions = data_flow_ops.dynamic_partition( data, indices, num_partitions=4) - partition_vals = sess.run(partitions) + partition_vals = self.evaluate(partitions) self.assertEqual(4, len(partition_vals)) self.assertAllEqual(np.array([], dtype=np.float64).reshape(-1, 4), @@ -151,6 +154,7 @@ class DynamicPartitionTest(test.TestCase): dtype=np.float64).reshape(-1, 4), partition_vals[3]) + @test_util.run_deprecated_v1 def testHigherRank(self): np.random.seed(7) with self.session(use_gpu=True) as sess: @@ -164,7 +168,7 @@ class DynamicPartitionTest(test.TestCase): outputs = data_flow_ops.dynamic_partition( data_t, partitions_t, num_partitions=n) self.assertEqual(n, len(outputs)) - outputs_val = sess.run(outputs) + outputs_val = self.evaluate(outputs) for i, output in enumerate(outputs_val): self.assertAllEqual(output, data[partitions == i]) @@ -183,7 +187,7 @@ class DynamicPartitionTest(test.TestCase): indices = constant_op.constant(indices_list, dtype=dtypes.int32) partitions = data_flow_ops.dynamic_partition( data, indices, num_partitions=4) - partition_vals = sess.run(partitions) + partition_vals = self.evaluate(partitions) self.assertEqual(4, len(partition_vals)) self.assertAllEqual([], partition_vals[0]) @@ -199,7 +203,7 @@ class DynamicPartitionTest(test.TestCase): indices = constant_op.constant(indices_list, dtype=dtypes.int32) partitions = data_flow_ops.dynamic_partition( data, indices, num_partitions=3) - partition_vals = sess.run(partitions) + partition_vals = self.evaluate(partitions) self.assertEqual(3, len(partition_vals)) self.assertAllEqual([[]], partition_vals[0]) @@ -215,7 +219,7 @@ class DynamicPartitionTest(test.TestCase): indices = constant_op.constant(indices_list, dtype=dtypes.int32) partitions = data_flow_ops.dynamic_partition( data, indices, num_partitions=2) - partition_vals = sess.run(partitions) + partition_vals = self.evaluate(partitions) self.assertEqual(2, len(partition_vals)) self.assertAllEqual([], partition_vals[0]) @@ -236,7 +240,7 @@ class DynamicPartitionTest(test.TestCase): indices = constant_op.constant(indices_list, dtype=dtypes.int32) partitions = data_flow_ops.dynamic_partition( data, indices, num_partitions=2) - partition_vals = sess.run(partitions) + partition_vals = self.evaluate(partitions) self.assertEqual(2, len(partition_vals)) self.assertAllEqual([6], partition_vals[0]) @@ -257,7 +261,7 @@ class DynamicPartitionTest(test.TestCase): indices = constant_op.constant(indices_list, dtype=dtypes.int32) partitions = data_flow_ops.dynamic_partition( data, indices, num_partitions=5) - partition_vals = sess.run(partitions) + partition_vals = self.evaluate(partitions) self.assertEqual(5, len(partition_vals)) self.assertAllEqual([5], partition_vals[0]) @@ -281,12 +285,13 @@ class DynamicPartitionTest(test.TestCase): indices = constant_op.constant(indices_list, dtype=dtypes.int32) partitions = data_flow_ops.dynamic_partition( data, indices, num_partitions=40) - partition_vals = sess.run(partitions) + partition_vals = self.evaluate(partitions) self.assertEqual(40, len(partition_vals)) for i in range(40): self.assertAllEqual([], partition_vals[i]) + @test_util.run_deprecated_v1 def testErrorIndexOutOfRange(self): with self.cached_session() as sess: data = constant_op.constant([[0, 1, 2], [3, 4, 5], [6, 7, 8], [9, 10, 11], @@ -295,16 +300,18 @@ class DynamicPartitionTest(test.TestCase): partitions = data_flow_ops.dynamic_partition( data, indices, num_partitions=4) with self.assertRaisesOpError(r"partitions\[2\] = 99 is not in \[0, 4\)"): - sess.run(partitions) + self.evaluate(partitions) + @test_util.run_deprecated_v1 def testScalarIndexOutOfRange(self): with self.cached_session() as sess: bad = 17 data = np.zeros(5) partitions = data_flow_ops.dynamic_partition(data, bad, num_partitions=7) with self.assertRaisesOpError(r"partitions = 17 is not in \[0, 7\)"): - sess.run(partitions) + self.evaluate(partitions) + @test_util.run_deprecated_v1 def testHigherRankIndexOutOfRange(self): with self.cached_session() as sess: shape = (2, 3) @@ -320,6 +327,7 @@ class DynamicPartitionTest(test.TestCase): r"partitions\[%d,%d\] = 17 is not in \[0, 7\)" % (i, j)): sess.run(partitions, feed_dict={indices: bad}) + @test_util.run_deprecated_v1 def testErrorWrongDimsIndices(self): data = constant_op.constant([[0], [1], [2]]) indices = constant_op.constant([[0], [0]]) @@ -335,7 +343,7 @@ class DynamicPartitionTest(test.TestCase): self.assertEqual(len(inds), x.shape[0]) partitioned = data_flow_ops.dynamic_partition(x, inds, 16) with self.cached_session() as sess: - res = sess.run(partitioned) + res = self.evaluate(partitioned) self.assertEqual(res[-1].shape[0], 192) diff --git a/tensorflow/python/kernel_tests/dynamic_stitch_op_test.py b/tensorflow/python/kernel_tests/dynamic_stitch_op_test.py index c3f67d29aa4..4f338880aa3 100644 --- a/tensorflow/python/kernel_tests/dynamic_stitch_op_test.py +++ b/tensorflow/python/kernel_tests/dynamic_stitch_op_test.py @@ -22,6 +22,7 @@ import numpy as np from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes +from tensorflow.python.framework import test_util from tensorflow.python.ops import array_ops from tensorflow.python.ops import data_flow_ops from tensorflow.python.ops import math_ops @@ -36,18 +37,19 @@ class DynamicStitchTestBase(object): self.stitch_op = stitch_op def testScalar(self): - with self.session(use_gpu=True): + with test_util.use_gpu(): indices = [constant_op.constant(0), constant_op.constant(1)] data = [constant_op.constant(40), constant_op.constant(60)] for step in -1, 1: stitched_t = self.stitch_op(indices[::step], data) - stitched_val = stitched_t.eval() + stitched_val = self.evaluate(stitched_t) self.assertAllEqual([40, 60][::step], stitched_val) # Dimension 0 is max(flatten(indices))+1. self.assertEqual([2], stitched_t.get_shape().as_list()) + @test_util.run_deprecated_v1 def testShapeInferenceForScalarWithNonConstantIndices(self): - with self.session(use_gpu=True): + with test_util.use_gpu(): indices = [ array_ops.placeholder(dtype=dtypes.int32), constant_op.constant(1) @@ -61,7 +63,7 @@ class DynamicStitchTestBase(object): self.assertEqual([None], stitched_t.get_shape().as_list()) def testSimpleOneDimensional(self): - with self.session(use_gpu=True): + with test_util.use_gpu(): # Test various datatypes in the simple case to ensure that the op was # registered under those types. dtypes_to_test = [ @@ -78,23 +80,23 @@ class DynamicStitchTestBase(object): constant_op.constant([10, 60, 20, 30, 50]), dtype=dtype) ] stitched_t = self.stitch_op(indices, data) - stitched_val = stitched_t.eval() + stitched_val = self.evaluate(stitched_t) self.assertAllEqual([0, 10, 20, 30, 40, 50, 60, 70], stitched_val) # Dimension 0 is max(flatten(indices))+1. self.assertEqual([8], stitched_t.get_shape().as_list()) def testOneListOneDimensional(self): - with self.session(use_gpu=True): + with test_util.use_gpu(): indices = [constant_op.constant([1, 6, 2, 3, 5, 0, 4, 7])] data = [constant_op.constant([10, 60, 20, 30, 50, 0, 40, 70])] stitched_t = self.stitch_op(indices, data) - stitched_val = stitched_t.eval() + stitched_val = self.evaluate(stitched_t) self.assertAllEqual([0, 10, 20, 30, 40, 50, 60, 70], stitched_val) # Dimension 0 is max(flatten(indices))+1. self.assertEqual([8], stitched_t.get_shape().as_list()) def testSimpleTwoDimensional(self): - with self.session(use_gpu=True): + with test_util.use_gpu(): indices = [ constant_op.constant([0, 4, 7]), constant_op.constant([1, 6]), @@ -106,14 +108,14 @@ class DynamicStitchTestBase(object): constant_op.constant([[20, 21], [30, 31], [50, 51]]) ] stitched_t = self.stitch_op(indices, data) - stitched_val = stitched_t.eval() + stitched_val = self.evaluate(stitched_t) self.assertAllEqual([[0, 1], [10, 11], [20, 21], [30, 31], [40, 41], [50, 51], [60, 61], [70, 71]], stitched_val) # Dimension 0 is max(flatten(indices))+1. self.assertEqual([8, 2], stitched_t.get_shape().as_list()) def testZeroSizeTensor(self): - with self.session(use_gpu=True): + with test_util.use_gpu(): indices = [ constant_op.constant([0, 4, 7]), constant_op.constant([1, 6]), @@ -127,12 +129,13 @@ class DynamicStitchTestBase(object): array_ops.zeros([0, 2], dtype=dtypes.int32) ] stitched_t = self.stitch_op(indices, data) - stitched_val = stitched_t.eval() + stitched_val = self.evaluate(stitched_t) self.assertAllEqual([[0, 1], [10, 11], [20, 21], [30, 31], [40, 41], [50, 51], [60, 61], [70, 71]], stitched_val) # Dimension 0 is max(flatten(indices))+1. self.assertEqual([8, 2], stitched_t.get_shape().as_list()) + @test_util.run_deprecated_v1 def testHigherRank(self): with self.session(use_gpu=True) as sess: indices = [ @@ -147,7 +150,7 @@ class DynamicStitchTestBase(object): [[1., 2.], [31., 32.]]]) ] stitched_t = self.stitch_op(indices, data) - stitched_val = stitched_t.eval() + stitched_val = self.evaluate(stitched_t) correct = 10. * np.arange(7)[:, None] + [1., 2.] self.assertAllEqual(correct, stitched_val) self.assertEqual([7, 2], stitched_t.get_shape().as_list()) @@ -157,8 +160,9 @@ class DynamicStitchTestBase(object): stitched_grad) self.assertEqual(grads[:3], [None] * 3) # Indices have no gradients for datum, grad in zip(data, sess.run(grads[3:])): - self.assertAllEqual(7. * datum.eval(), grad) + self.assertAllEqual(7. * self.evaluate(datum), grad) + @test_util.run_deprecated_v1 def testErrorIndicesMultiDimensional(self): indices = [ constant_op.constant([0, 4, 7]), @@ -171,6 +175,7 @@ class DynamicStitchTestBase(object): with self.assertRaises(ValueError): self.stitch_op(indices, data) + @test_util.run_deprecated_v1 def testErrorDataNumDimsMismatch(self): indices = [ constant_op.constant([0, 4, 7]), @@ -183,6 +188,7 @@ class DynamicStitchTestBase(object): with self.assertRaises(ValueError): self.stitch_op(indices, data) + @test_util.run_deprecated_v1 def testErrorDataDimSizeMismatch(self): indices = [ constant_op.constant([0, 4, 5]), @@ -195,6 +201,7 @@ class DynamicStitchTestBase(object): with self.assertRaises(ValueError): self.stitch_op(indices, data) + @test_util.run_deprecated_v1 def testErrorDataAndIndicesSizeMismatch(self): indices = [ constant_op.constant([0, 4, 7]), @@ -222,16 +229,17 @@ class ParallelDynamicStitchTest(DynamicStitchTestBase, test.TestCase): DynamicStitchTestBase.__init__(self, data_flow_ops.parallel_dynamic_stitch) def testScalar(self): - with self.session(use_gpu=True): + with test_util.use_gpu(): indices = [constant_op.constant(0), constant_op.constant(1)] data = [constant_op.constant(40.0), constant_op.constant(60.0)] for step in -1, 1: stitched_t = data_flow_ops.dynamic_stitch(indices[::step], data) - stitched_val = stitched_t.eval() + stitched_val = self.evaluate(stitched_t) self.assertAllEqual([40.0, 60.0][::step], stitched_val) # Dimension 0 is max(flatten(indices))+1. self.assertEqual([2], stitched_t.get_shape().as_list()) + @test_util.run_deprecated_v1 def testHigherRank(self): with self.session(use_gpu=True) as sess: indices = [ @@ -246,7 +254,7 @@ class ParallelDynamicStitchTest(DynamicStitchTestBase, test.TestCase): [[[51, 52], [21, 22]], [[1, 2], [31, 32]]], dtype=dtypes.float32) ] stitched_t = data_flow_ops.dynamic_stitch(indices, data) - stitched_val = stitched_t.eval() + stitched_val = self.evaluate(stitched_t) correct = 10 * np.arange(7)[:, None] + [1.0, 2.0] self.assertAllEqual(correct, stitched_val) self.assertEqual([7, 2], stitched_t.get_shape().as_list()) @@ -256,7 +264,7 @@ class ParallelDynamicStitchTest(DynamicStitchTestBase, test.TestCase): stitched_grad) self.assertEqual(grads[:3], [None] * 3) # Indices have no gradients for datum, grad in zip(data, sess.run(grads[3:])): - self.assertAllEqual(7.0 * datum.eval(), grad) + self.assertAllEqual(7.0 * self.evaluate(datum), grad) # GPU version unit tests def testScalarGPU(self): @@ -265,11 +273,12 @@ class ParallelDynamicStitchTest(DynamicStitchTestBase, test.TestCase): data = [constant_op.constant(40.0), constant_op.constant(60.0)] for step in -1, 1: stitched_t = data_flow_ops.dynamic_stitch(indices[::step], data) - stitched_val = stitched_t.eval() + stitched_val = self.evaluate(stitched_t) self.assertAllEqual([40.0, 60.0][::step], stitched_val) # Dimension 0 is max(flatten(indices))+1. self.assertEqual([2], stitched_t.get_shape().as_list()) + @test_util.run_deprecated_v1 def testHigherRankGPU(self): with self.cached_session() as sess: indices = [ @@ -284,7 +293,7 @@ class ParallelDynamicStitchTest(DynamicStitchTestBase, test.TestCase): [[[51, 52], [21, 22]], [[1, 2], [31, 32]]], dtype=dtypes.float32) ] stitched_t = data_flow_ops.dynamic_stitch(indices, data) - stitched_val = stitched_t.eval() + stitched_val = self.evaluate(stitched_t) correct = 10 * np.arange(7)[:, None] + [1.0, 2.0] self.assertAllEqual(correct, stitched_val) self.assertEqual([7, 2], stitched_t.get_shape().as_list()) @@ -294,7 +303,7 @@ class ParallelDynamicStitchTest(DynamicStitchTestBase, test.TestCase): stitched_grad) self.assertEqual(grads[:3], [None] * 3) # Indices have no gradients for datum, grad in zip(data, sess.run(grads[3:])): - self.assertAllEqual(7.0 * datum.eval(), grad) + self.assertAllEqual(7.0 * self.evaluate(datum), grad) if __name__ == "__main__": diff --git a/tensorflow/python/kernel_tests/edit_distance_op_test.py b/tensorflow/python/kernel_tests/edit_distance_op_test.py index dab5eee7f50..4a06ab770aa 100644 --- a/tensorflow/python/kernel_tests/edit_distance_op_test.py +++ b/tensorflow/python/kernel_tests/edit_distance_op_test.py @@ -49,11 +49,11 @@ class EditDistanceTest(test.TestCase): if expected_err_re is None: self.assertEqual(edit_distance.get_shape(), expected_shape) - output = edit_distance.eval() + output = self.evaluate(edit_distance) self.assertAllClose(output, expected_output) else: with self.assertRaisesOpError(expected_err_re): - edit_distance.eval() + self.evaluate(edit_distance) def _testEditDistance(self, hypothesis, diff --git a/tensorflow/python/kernel_tests/embedding_ops_test.py b/tensorflow/python/kernel_tests/embedding_ops_test.py index 008d6fbf577..6019245d0f8 100644 --- a/tensorflow/python/kernel_tests/embedding_ops_test.py +++ b/tensorflow/python/kernel_tests/embedding_ops_test.py @@ -28,6 +28,7 @@ from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops from tensorflow.python.framework import sparse_tensor +from tensorflow.python.framework import test_util from tensorflow.python.ops import array_ops from tensorflow.python.ops import data_flow_ops from tensorflow.python.ops import embedding_ops @@ -76,7 +77,7 @@ class ScatterAddSubTest(test.TestCase): # p = init variables.global_variables_initializer().run() # p += vals - result = p2.eval() + result = self.evaluate(p2) # Compute the expected 'p' using numpy operations. for i, ind in enumerate(indices): if scatter_op == state_ops.scatter_add: @@ -87,16 +88,19 @@ class ScatterAddSubTest(test.TestCase): vals_shape[0], -1)[i, :]) self.assertTrue(all((p_init == result).ravel())) + @test_util.run_deprecated_v1 def testNoRepetitions(self): self._TestCase([2, 2], [1]) self._TestCase([4, 4, 4], [2, 0]) self._TestCase([43, 20, 10, 10], [42, 5, 6, 1, 3, 5, 7, 9]) + @test_util.run_deprecated_v1 def testWithRepetitions(self): self._TestCase([2, 2], [1, 1]) self._TestCase([5, 3, 9, 5], [2, 0, 4, 1, 3, 1, 4, 0, 4, 3]) self._TestCase([32, 4, 4], [31] * 8) + @test_util.run_deprecated_v1 def testRandom(self): # Random shapes of rank 4, random indices for _ in range(5): @@ -104,6 +108,7 @@ class ScatterAddSubTest(test.TestCase): indices = np.random.randint(shape[0], size=2 * shape[0]) self._TestCase(_AsLong(list(shape)), list(indices)) + @test_util.run_deprecated_v1 def testSubRandom(self): # Random shapes of rank 4, random indices for _ in range(5): @@ -111,6 +116,7 @@ class ScatterAddSubTest(test.TestCase): indices = np.random.randint(shape[0], size=2 * shape[0]) self._TestCase(_AsLong(list(shape)), list(indices), state_ops.scatter_sub) + @test_util.run_deprecated_v1 def testWrongShape(self): # Indices and values mismatch. var = variables.Variable( @@ -241,6 +247,7 @@ class EmbeddingLookupTest(test.TestCase): # both the ids are in the first shard, one of the resulting lookup # vector is going to be empty. The subsequent DivOp fails because of that. # TODO(keveman): Disabling the test until the underlying problem is fixed. + @test_util.run_deprecated_v1 def testSimpleSharded(self): with self.cached_session(): num_shards = 2 @@ -257,6 +264,7 @@ class EmbeddingLookupTest(test.TestCase): self.assertAllEqual(np_result, tf_result) self.assertShapeEqual(np_result, embedding) + @test_util.run_deprecated_v1 def testMaxNorm(self): with self.cached_session(): embeddings = constant_op.constant([[2.0]]) @@ -267,6 +275,7 @@ class EmbeddingLookupTest(test.TestCase): self.assertAllEqual(embedding.eval(), [[1.0]]) + @test_util.run_deprecated_v1 def testMaxNormNontrivial(self): with self.cached_session(): embeddings = constant_op.constant([[2.0, 4.0], [3.0, 1.0]]) @@ -278,8 +287,9 @@ class EmbeddingLookupTest(test.TestCase): norms = math_ops.sqrt( math_ops.reduce_sum(embeddings * embeddings, axis=1)) normalized = embeddings / array_ops.stack([norms, norms], axis=1) - self.assertAllEqual(embedding.eval(), 2 * normalized.eval()) + self.assertAllEqual(embedding.eval(), 2 * self.evaluate(normalized)) + @test_util.run_deprecated_v1 def testSimpleShardedPartitionedVariable(self): with self.cached_session() as sess: num_shards = 2 @@ -294,7 +304,7 @@ class EmbeddingLookupTest(test.TestCase): variables.global_variables_initializer().run() params_values = [params[p_i.name] for p_i in p] # Test that the PartitionedVariable components equal the list in p - p_var_val = sess.run(list(p_variable)) + p_var_val = self.evaluate(list(p_variable)) # Actual test tf_result = embedding.eval(feed_dict=feed_dict) np_result, _, _ = _EmbeddingResult(params, id_vals, num_shards, vocab_size) @@ -302,6 +312,7 @@ class EmbeddingLookupTest(test.TestCase): self.assertAllEqual(np_result, tf_result) self.assertShapeEqual(np_result, embedding) + @test_util.run_deprecated_v1 def testSimpleShardedPartitionedResourceVariable(self): with self.cached_session() as sess: num_shards = 2 @@ -316,15 +327,16 @@ class EmbeddingLookupTest(test.TestCase): variables.global_variables_initializer().run() params_values = [params[p_i.name] for p_i in p] # Test that the PartitionedVariable components equal the list in p - p_var_val = sess.run(list(p_variable)) + p_var_val = self.evaluate(list(p_variable)) # Actual test print(ops.get_default_graph().as_graph_def()) - tf_result = embedding.eval() + tf_result = self.evaluate(embedding) np_result, _, _ = _EmbeddingResult(params, id_vals, num_shards, vocab_size) self.assertAllEqual(params_values, p_var_val) self.assertAllEqual(np_result, tf_result) self.assertShapeEqual(np_result, embedding) + @test_util.run_deprecated_v1 def testShardedModPartitioningInt32Ids(self): with self.cached_session(): num_shards = 5 @@ -347,6 +359,7 @@ class EmbeddingLookupTest(test.TestCase): self.assertAllEqual(np_result, tf_result) self.assertShapeEqual(np_result, embedding) + @test_util.run_deprecated_v1 def testShardedModPartitioningInt64Ids(self): with self.cached_session(): num_shards = 5 @@ -369,6 +382,7 @@ class EmbeddingLookupTest(test.TestCase): self.assertAllEqual(np_result, tf_result) self.assertShapeEqual(np_result, embedding) + @test_util.run_deprecated_v1 def testShardedDivPartitioningInt32Ids(self): with self.cached_session(): num_shards = 5 @@ -393,6 +407,7 @@ class EmbeddingLookupTest(test.TestCase): self.assertAllEqual(np_result, tf_result) self.assertShapeEqual(np_result, embedding) + @test_util.run_deprecated_v1 def testShardedDivPartitioningInt32IdsPartitionedVariable(self): with self.cached_session(): num_shards = 5 @@ -418,6 +433,7 @@ class EmbeddingLookupTest(test.TestCase): self.assertAllEqual(np_result, tf_result) self.assertShapeEqual(np_result, embedding) + @test_util.run_deprecated_v1 def testShardedDivPartitioningInt64Ids(self): with self.cached_session(): num_shards = 5 @@ -442,6 +458,7 @@ class EmbeddingLookupTest(test.TestCase): self.assertAllEqual(np_result, tf_result) self.assertShapeEqual(np_result, embedding) + @test_util.run_deprecated_v1 def testShardedDivPartitioningUnknownParamShape(self): with self.cached_session(): num_shards = 5 @@ -468,6 +485,7 @@ class EmbeddingLookupTest(test.TestCase): params, id_vals, num_shards, vocab_size, partition_strategy="div") self.assertAllEqual(np_result, tf_result) + @test_util.run_deprecated_v1 def testGradientsEmbeddingLookup(self): vocab_size = 9 num_ids = 10 @@ -488,6 +506,7 @@ class EmbeddingLookupTest(test.TestCase): x, x_shape, y, y_shape, x_init_value=x_init_value) self.assertLess(err, 1e-4) + @test_util.run_deprecated_v1 def testGradientsEmbeddingLookupWithComputedParams(self): vocab_size = 9 num_ids = 5 @@ -526,6 +545,7 @@ class EmbeddingLookupTest(test.TestCase): ids = constant_op.constant([0, 1, 1, 17], dtype=dtypes.int32) embedding_ops.embedding_lookup(p, ids) + @test_util.run_deprecated_v1 def testHigherRank(self): np.random.seed(8) with self.cached_session(): @@ -546,6 +566,7 @@ class EmbeddingLookupTest(test.TestCase): sharded = embedding_ops.embedding_lookup(split_params, ids).eval() self.assertAllEqual(simple, sharded) + @test_util.run_deprecated_v1 def testHigherRankMaxNorm(self): np.random.seed(8) with self.cached_session(): @@ -574,6 +595,7 @@ class EmbeddingLookupTest(test.TestCase): split_params, ids, max_norm=1.0).eval() self.assertAllEqual(simple, sharded) + @test_util.run_deprecated_v1 def testTransform(self): # This tests all combinations of: # - ids rank 0, 1, >1 @@ -648,6 +670,7 @@ class EmbeddingLookupSparseTest(test.TestCase): index += num_val return grouped_vals + @test_util.run_deprecated_v1 def testEmbeddingLookupSparse(self): vocab_size = 13 batch_size = 10 @@ -706,6 +729,7 @@ class EmbeddingLookupSparseTest(test.TestCase): atol = rtol self.assertAllClose(np_embedding_sum, tf_embedding_sum, rtol, atol) + @test_util.run_deprecated_v1 def testGradientsEmbeddingLookupSparse(self): vocab_size = 12 batch_size = 4 @@ -733,6 +757,7 @@ class EmbeddingLookupSparseTest(test.TestCase): x, x_shape, y, y_shape, x_init_value=x_init_value) self.assertLess(err, 1e-5 if dtype == dtypes.float64 else 2e-3) + @test_util.run_deprecated_v1 def testIncompatibleShapes(self): with self.cached_session(): x, _, _ = _EmbeddingParams(1, 10, dtype=dtypes.float32) @@ -758,11 +783,13 @@ class SafeEmbeddingLookupSparseTest(test.TestCase): assert num_shards > 0 assert num_shards <= vocab_size - embedding_weights = partitioned_variables.create_partitioned_variables( + initializer = init_ops.truncated_normal_initializer( + mean=0.0, stddev=1.0 / math.sqrt(vocab_size), dtype=dtypes.float32) + embedding_weights = list(variable_scope.get_variable( + name="embedding_weights", shape=[vocab_size, embed_dim], - slicing=[num_shards, 1], - initializer=init_ops.truncated_normal_initializer( - mean=0.0, stddev=1.0 / math.sqrt(vocab_size), dtype=dtypes.float32)) + partitioner=partitioned_variables.fixed_size_partitioner(num_shards), + initializer=initializer)) for w in embedding_weights: w.initializer.run() embedding_weights = [w.eval() for w in embedding_weights] @@ -818,6 +845,7 @@ class SafeEmbeddingLookupSparseTest(test.TestCase): return sparse_ids, sparse_weights + @test_util.run_deprecated_v1 def test_safe_embedding_lookup_sparse_return_zero_vector(self): with self.cached_session(): embedding_weights = self._random_weights() @@ -831,6 +859,7 @@ class SafeEmbeddingLookupSparseTest(test.TestCase): [(1.0 * embedding_weights[0][0] + 2.0 * embedding_weights[0][1]) / 3.0, [0] * 4, [0] * 4, embedding_weights[0][2], [0] * 4]) + @test_util.run_deprecated_v1 def test_safe_embedding_lookup_sparse_return_special_vector(self): with self.cached_session(): embedding_weights = self._random_weights() @@ -845,6 +874,7 @@ class SafeEmbeddingLookupSparseTest(test.TestCase): 3.0, embedding_weights[0][3], embedding_weights[0][3], embedding_weights[0][2], embedding_weights[0][3]]) + @test_util.run_deprecated_v1 def test_safe_embedding_lookup_sparse_no_weights(self): with self.cached_session(): embedding_weights = self._random_weights() @@ -859,6 +889,7 @@ class SafeEmbeddingLookupSparseTest(test.TestCase): [0] * 4, embedding_weights[0][2], ( embedding_weights[0][0] + embedding_weights[0][1]) / 2.0]) + @test_util.run_deprecated_v1 def test_safe_embedding_lookup_sparse_partitioned(self): with self.cached_session(): embedding_weights = self._random_weights(num_shards=3) @@ -873,6 +904,7 @@ class SafeEmbeddingLookupSparseTest(test.TestCase): [0] * 4, [0] * 4, embedding_weights[2], (embedding_weights[0] + embedding_weights[1]) / 2.0]) + @test_util.run_deprecated_v1 def test_safe_embedding_lookup_sparse_partitioned_inconsistent_weights(self): with self.cached_session(): embedding_weights = self._random_weights(num_shards=3) @@ -888,6 +920,7 @@ class SafeEmbeddingLookupSparseTest(test.TestCase): self.assertRaises(ValueError, embedding_ops.safe_embedding_lookup_sparse, embedding_weights, sparse_ids, sparse_weights) + @test_util.run_deprecated_v1 def test_safe_embedding_lookup_sparse_3d_return_zero_vector(self): with self.cached_session(): embedding_weights = self._random_weights() @@ -901,6 +934,7 @@ class SafeEmbeddingLookupSparseTest(test.TestCase): [0] * 4, [0] * 4 ], [embedding_weights[0][2], [0] * 4, [0] * 4]]) + @test_util.run_deprecated_v1 def test_safe_embedding_lookup_sparse_3d_return_special_vector(self): with self.cached_session(): embedding_weights = self._random_weights() @@ -917,6 +951,7 @@ class SafeEmbeddingLookupSparseTest(test.TestCase): embedding_weights[0][3] ]]) + @test_util.run_deprecated_v1 def test_safe_embedding_lookup_sparse_3d_no_weights(self): with self.cached_session(): embedding_weights = self._random_weights() @@ -933,6 +968,7 @@ class SafeEmbeddingLookupSparseTest(test.TestCase): (embedding_weights[0][0] + embedding_weights[0][1]) / 2.0, [0] * 4 ]]) + @test_util.run_deprecated_v1 def test_safe_embedding_lookup_sparse_3d_partitioned(self): with self.cached_session(): embedding_weights = self._random_weights(num_shards=3) @@ -949,6 +985,7 @@ class SafeEmbeddingLookupSparseTest(test.TestCase): (embedding_weights[0] + embedding_weights[1]) / 2.0, [0] * 4 ]]) + @test_util.run_deprecated_v1 def test_safe_embedding_lookup_sparse_3d_partitioned_inconsistent_weights( self): with self.cached_session(): @@ -968,6 +1005,7 @@ class SafeEmbeddingLookupSparseTest(test.TestCase): class DynamicStitchOpTest(test.TestCase): + @test_util.run_deprecated_v1 def testCint32Cpu(self): with self.session(use_gpu=False): indices = [ @@ -981,6 +1019,7 @@ class DynamicStitchOpTest(test.TestCase): self.assertAllEqual( data_flow_ops.dynamic_stitch(indices, values).eval(), [12, 23, 1, 2]) + @test_util.run_deprecated_v1 def testCint32Gpu(self): with self.session(use_gpu=True): indices = [ @@ -994,6 +1033,7 @@ class DynamicStitchOpTest(test.TestCase): self.assertAllEqual( data_flow_ops.dynamic_stitch(indices, values).eval(), [12, 23, 1, 2]) + @test_util.run_deprecated_v1 def testInt32Cpu(self): with self.session(use_gpu=False): indices = [ @@ -1007,6 +1047,7 @@ class DynamicStitchOpTest(test.TestCase): self.assertAllEqual( data_flow_ops.dynamic_stitch(indices, values).eval(), [12, 23, 1, 2]) + @test_util.run_deprecated_v1 def testInt32Gpu(self): with self.session(use_gpu=True): indices = [ @@ -1020,6 +1061,7 @@ class DynamicStitchOpTest(test.TestCase): self.assertAllEqual( data_flow_ops.dynamic_stitch(indices, values).eval(), [12, 23, 1, 2]) + @test_util.run_deprecated_v1 def testSumGradArgs(self): with self.session(use_gpu=False): indices = [ @@ -1034,6 +1076,7 @@ class DynamicStitchOpTest(test.TestCase): data_flow_ops.dynamic_stitch(indices, values).eval(), [2, 3, 1, 1]) # We expect that the values are merged in order. + @test_util.run_deprecated_v1 def testStitchOrder(self): with self.cached_session(): indices = [] @@ -1049,6 +1092,7 @@ class DynamicStitchOpTest(test.TestCase): class ParallelDynamicStitchOpTest(test.TestCase): + @test_util.run_deprecated_v1 def testCint32Cpu(self): with self.session(use_gpu=False): indices = [ @@ -1063,6 +1107,7 @@ class ParallelDynamicStitchOpTest(test.TestCase): data_flow_ops.parallel_dynamic_stitch(indices, values).eval(), [12, 23, 1, 2, 34, 3, 45]) + @test_util.run_deprecated_v1 def testInt32Cpu(self): with self.session(use_gpu=False): indices = [ @@ -1077,6 +1122,7 @@ class ParallelDynamicStitchOpTest(test.TestCase): data_flow_ops.parallel_dynamic_stitch(indices, values).eval(), [12, 23, 1, 2, 3, 34, 45, 56]) + @test_util.run_deprecated_v1 def testSimple(self): with self.session(use_gpu=False): indices = [ops.convert_to_tensor([0, 1]), ops.convert_to_tensor([2, 3])] diff --git a/tensorflow/python/kernel_tests/extract_image_patches_grad_test.py b/tensorflow/python/kernel_tests/extract_image_patches_grad_test.py index 7d9d4e51752..7ba2dc6c209 100644 --- a/tensorflow/python/kernel_tests/extract_image_patches_grad_test.py +++ b/tensorflow/python/kernel_tests/extract_image_patches_grad_test.py @@ -23,6 +23,7 @@ import numpy as np from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes from tensorflow.python.framework import random_seed as random_seed_lib +from tensorflow.python.framework import test_util from tensorflow.python.ops import array_ops from tensorflow.python.ops import gradient_checker from tensorflow.python.ops import gradients_impl @@ -78,6 +79,7 @@ class ExtractImagePatchesGradTest(test.TestCase): }, ] + @test_util.run_deprecated_v1 def testGradient(self): # Set graph seed for determinism. random_seed = 42 @@ -102,6 +104,7 @@ class ExtractImagePatchesGradTest(test.TestCase): print('extract_image_patches gradient err: %.4e' % err) self.assertLess(err, 1e-4) + @test_util.run_deprecated_v1 def testConstructGradientWithLargeImages(self): batch_size = 4 height = 1024 diff --git a/tensorflow/python/kernel_tests/extract_image_patches_op_test.py b/tensorflow/python/kernel_tests/extract_image_patches_op_test.py index 61436f24cfe..bb3c0ae8069 100644 --- a/tensorflow/python/kernel_tests/extract_image_patches_op_test.py +++ b/tensorflow/python/kernel_tests/extract_image_patches_op_test.py @@ -21,6 +21,7 @@ from __future__ import print_function import numpy as np from tensorflow.python.framework import constant_op +from tensorflow.python.framework import test_util from tensorflow.python.ops import array_ops from tensorflow.python.platform import test @@ -43,7 +44,7 @@ class ExtractImagePatches(test.TestCase): strides = [1] + strides + [1] rates = [1] + rates + [1] - with self.session(use_gpu=True): + with test_util.use_gpu(): out_tensor = array_ops.extract_image_patches( constant_op.constant(image), ksizes=ksizes, @@ -51,7 +52,7 @@ class ExtractImagePatches(test.TestCase): rates=rates, padding=padding, name="im2col") - self.assertAllClose(patches, out_tensor.eval()) + self.assertAllClose(patches, self.evaluate(out_tensor)) def testKsize1x1Stride1x1Rate1x1(self): """Verifies that for 1x1 kernel the output equals the input.""" diff --git a/tensorflow/python/kernel_tests/extract_volume_patches_op_test.py b/tensorflow/python/kernel_tests/extract_volume_patches_op_test.py index bbb3fef85b4..88f7df8fbb6 100644 --- a/tensorflow/python/kernel_tests/extract_volume_patches_op_test.py +++ b/tensorflow/python/kernel_tests/extract_volume_patches_op_test.py @@ -21,6 +21,7 @@ from __future__ import print_function import numpy as np from tensorflow.python.framework import constant_op +from tensorflow.python.framework import test_util from tensorflow.python.ops import array_ops from tensorflow.python.platform import test @@ -45,14 +46,14 @@ class ExtractVolumePatches(test.TestCase): ksizes = [1] + ksizes + [1] strides = [1] + strides + [1] - with self.cached_session(use_gpu=True): + with test_util.use_gpu(): out_tensor = array_ops.extract_volume_patches( constant_op.constant(image), ksizes=ksizes, strides=strides, padding=padding, name="im2col_3d") - self.assertAllClose(patches, out_tensor.eval()) + self.assertAllClose(patches, self.evaluate(out_tensor)) # pylint: disable=bad-whitespace def testKsize1x1x1Stride1x1x1(self): diff --git a/tensorflow/python/kernel_tests/fifo_queue_test.py b/tensorflow/python/kernel_tests/fifo_queue_test.py index 8961c4b13c2..9655351a01e 100644 --- a/tensorflow/python/kernel_tests/fifo_queue_test.py +++ b/tensorflow/python/kernel_tests/fifo_queue_test.py @@ -159,7 +159,7 @@ class FIFOQueueTest(test.TestCase): # Run one producer thread for each element in elems. def enqueue(enqueue_op): - sess.run(enqueue_op) + self.evaluate(enqueue_op) threads = [ self.checkedThread( @@ -191,7 +191,7 @@ class FIFOQueueTest(test.TestCase): results = [] def dequeue(): - results.append(sess.run(dequeued_t)) + results.append(self.evaluate(dequeued_t)) threads = [self.checkedThread(target=dequeue) for _ in enqueue_ops] for thread in threads: @@ -211,7 +211,7 @@ class FIFOQueueTest(test.TestCase): enqueue_op.run() for i in xrange(len(elems)): - vals = dequeued_t.eval() + vals = self.evaluate(dequeued_t) self.assertEqual([elems[i]], vals) def testDequeueHalf(self): @@ -225,7 +225,7 @@ class FIFOQueueTest(test.TestCase): enqueue_op.run() for i in xrange(len(elems)): - vals = dequeued_t.eval() + vals = self.evaluate(dequeued_t) self.assertEqual([elems[i]], vals) def testEnqueueAndBlockingDequeue(self): @@ -240,13 +240,13 @@ class FIFOQueueTest(test.TestCase): # TODO(mrry): Figure out how to do this without sleeping. time.sleep(0.1) for enqueue_op in enqueue_ops: - sess.run(enqueue_op) + self.evaluate(enqueue_op) results = [] def dequeue(): for _ in xrange(len(elems)): - results.append(sess.run(dequeued_t)) + results.append(self.evaluate(dequeued_t)) enqueue_thread = self.checkedThread(target=enqueue) dequeue_thread = self.checkedThread(target=dequeue) @@ -269,7 +269,7 @@ class FIFOQueueTest(test.TestCase): enqueue_op.run() for i in xrange(len(elems)): - x_val, y_val = sess.run(dequeued_t) + x_val, y_val = self.evaluate(dequeued_t) x, y = elems[i] self.assertEqual([x], x_val) self.assertEqual([y], y_val) @@ -288,9 +288,9 @@ class FIFOQueueTest(test.TestCase): self.assertEqual([], size.get_shape()) enqueue_op.run() - self.assertEqual(1, size.eval()) + self.assertEqual(1, self.evaluate(size)) dequeued_t.op.run() - self.assertEqual(0, size.eval()) + self.assertEqual(0, self.evaluate(size)) def testEnqueueMany(self): with self.cached_session(): @@ -302,7 +302,7 @@ class FIFOQueueTest(test.TestCase): enqueue_op.run() for i in range(8): - vals = dequeued_t.eval() + vals = self.evaluate(dequeued_t) self.assertEqual([elems[i % 4]], vals) def testEmptyEnqueueMany(self): @@ -313,9 +313,9 @@ class FIFOQueueTest(test.TestCase): enqueue_op = q.enqueue_many((empty_t,)) size_t = q.size() - self.assertEqual([0], size_t.eval()) + self.assertEqual([0], self.evaluate(size_t)) enqueue_op.run() - self.assertEqual([0], size_t.eval()) + self.assertEqual([0], self.evaluate(size_t)) def testEmptyDequeueMany(self): with self.cached_session(): @@ -323,9 +323,9 @@ class FIFOQueueTest(test.TestCase): enqueue_op = q.enqueue((10.0,)) dequeued_t = q.dequeue_many(0) - self.assertEqual([], dequeued_t.eval().tolist()) + self.assertEqual([], self.evaluate(dequeued_t).tolist()) enqueue_op.run() - self.assertEqual([], dequeued_t.eval().tolist()) + self.assertEqual([], self.evaluate(dequeued_t).tolist()) def testEmptyDequeueUpTo(self): with self.cached_session(): @@ -333,9 +333,9 @@ class FIFOQueueTest(test.TestCase): enqueue_op = q.enqueue((10.0,)) dequeued_t = q.dequeue_up_to(0) - self.assertEqual([], dequeued_t.eval().tolist()) + self.assertEqual([], self.evaluate(dequeued_t).tolist()) enqueue_op.run() - self.assertEqual([], dequeued_t.eval().tolist()) + self.assertEqual([], self.evaluate(dequeued_t).tolist()) def testEmptyDequeueManyWithNoShape(self): with self.cached_session(): @@ -356,7 +356,7 @@ class FIFOQueueTest(test.TestCase): enqueue_op.run() for i in range(8): - float_val, int_val = sess.run(dequeued_t) + float_val, int_val = self.evaluate(dequeued_t) self.assertEqual(float_elems[i % 4], float_val) self.assertAllEqual(int_elems[i % 4], int_val) @@ -369,8 +369,8 @@ class FIFOQueueTest(test.TestCase): enqueue_op.run() - self.assertAllEqual(elems[0:4], dequeued_t.eval()) - self.assertAllEqual(elems[4:8], dequeued_t.eval()) + self.assertAllEqual(elems[0:4], self.evaluate(dequeued_t)) + self.assertAllEqual(elems[4:8], self.evaluate(dequeued_t)) def testDequeueUpToNoBlocking(self): with self.cached_session(): @@ -381,8 +381,8 @@ class FIFOQueueTest(test.TestCase): enqueue_op.run() - self.assertAllEqual(elems[0:4], dequeued_t.eval()) - self.assertAllEqual(elems[4:8], dequeued_t.eval()) + self.assertAllEqual(elems[0:4], self.evaluate(dequeued_t)) + self.assertAllEqual(elems[4:8], self.evaluate(dequeued_t)) def testMultiDequeueMany(self): with self.cached_session() as sess: @@ -399,17 +399,17 @@ class FIFOQueueTest(test.TestCase): enqueue_op.run() - float_val, int_val = sess.run(dequeued_t) + float_val, int_val = self.evaluate(dequeued_t) self.assertAllEqual(float_elems[0:4], float_val) self.assertAllEqual(int_elems[0:4], int_val) self.assertEqual(float_val.shape, dequeued_t[0].get_shape()) self.assertEqual(int_val.shape, dequeued_t[1].get_shape()) - float_val, int_val = sess.run(dequeued_t) + float_val, int_val = self.evaluate(dequeued_t) self.assertAllEqual(float_elems[4:8], float_val) self.assertAllEqual(int_elems[4:8], int_val) - float_val, int_val = sess.run(dequeued_single_t) + float_val, int_val = self.evaluate(dequeued_single_t) self.assertAllEqual(float_elems[8], float_val) self.assertAllEqual(int_elems[8], int_val) self.assertEqual(float_val.shape, dequeued_single_t[0].get_shape()) @@ -429,13 +429,13 @@ class FIFOQueueTest(test.TestCase): enqueue_op.run() - float_val, int_val = sess.run(dequeued_t) + float_val, int_val = self.evaluate(dequeued_t) self.assertAllEqual(float_elems[0:4], float_val) self.assertAllEqual(int_elems[0:4], int_val) self.assertEqual([None], dequeued_t[0].get_shape().as_list()) self.assertEqual([None, 2], dequeued_t[1].get_shape().as_list()) - float_val, int_val = sess.run(dequeued_t) + float_val, int_val = self.evaluate(dequeued_t) self.assertAllEqual(float_elems[4:8], float_val) self.assertAllEqual(int_elems[4:8], int_val) @@ -518,7 +518,7 @@ class FIFOQueueTest(test.TestCase): r"Expected \[2,3,3\], got \[2,3,4\]"): sess.run([enqueue_op], feed_dict={elems_bad: np.array([1] * 24).reshape((2, 3, 4))}) - dequeued_t.eval() + self.evaluate(dequeued_t) def testParallelEnqueueMany(self): with self.cached_session() as sess: @@ -529,7 +529,7 @@ class FIFOQueueTest(test.TestCase): # Enqueue 100 items in parallel on 10 threads. def enqueue(): - sess.run(enqueue_op) + self.evaluate(enqueue_op) threads = [self.checkedThread(target=enqueue) for _ in range(10)] for thread in threads: @@ -552,7 +552,7 @@ class FIFOQueueTest(test.TestCase): dequeued_elems = [] def dequeue(): - dequeued_elems.extend(sess.run(dequeued_t)) + dequeued_elems.extend(self.evaluate(dequeued_t)) threads = [self.checkedThread(target=dequeue) for _ in range(10)] for thread in threads: @@ -576,7 +576,7 @@ class FIFOQueueTest(test.TestCase): dequeued_elems = [] def dequeue(): - dequeued_elems.extend(sess.run(dequeued_t)) + dequeued_elems.extend(self.evaluate(dequeued_t)) threads = [self.checkedThread(target=dequeue) for _ in range(10)] for thread in threads: @@ -596,11 +596,11 @@ class FIFOQueueTest(test.TestCase): def enqueue(): for _ in xrange(100): - sess.run(enqueue_op) + self.evaluate(enqueue_op) def dequeue(): for _ in xrange(100): - self.assertTrue(sess.run(dequeued_t) in (10.0, 20.0)) + self.assertTrue(self.evaluate(dequeued_t) in (10.0, 20.0)) enqueue_threads = [self.checkedThread(target=enqueue) for _ in range(10)] dequeue_threads = [self.checkedThread(target=dequeue) for _ in range(10)] @@ -632,7 +632,7 @@ class FIFOQueueTest(test.TestCase): def dequeue(): for i in xrange(250): - self.assertEqual(i, sess.run(dequeued_t)) + self.assertEqual(i, self.evaluate(dequeued_t)) dequeue_thread = self.checkedThread(target=dequeue) dequeue_thread.start() @@ -663,7 +663,7 @@ class FIFOQueueTest(test.TestCase): dequeuemany_t = q.dequeue_many(count_placeholder) def enqueue(): - sess.run(enqueue_op) + self.evaluate(enqueue_op) enqueue_thread = self.checkedThread(target=enqueue) enqueue_thread.start() @@ -672,7 +672,7 @@ class FIFOQueueTest(test.TestCase): while elements_dequeued < 250: # With equal probability, run Dequeue or dequeue_many. if random.random() > 0.5: - self.assertEqual(elements_dequeued, dequeued_t.eval()) + self.assertEqual(elements_dequeued, self.evaluate(dequeued_t)) elements_dequeued += 1 else: count = random.randint(0, min(20, 250 - elements_dequeued)) @@ -701,10 +701,10 @@ class FIFOQueueTest(test.TestCase): # The enqueue_op should run after the dequeue op has blocked. # TODO(mrry): Figure out how to do this without sleeping. time.sleep(0.1) - sess.run(enqueue_op) + self.evaluate(enqueue_op) def dequeue(): - dequeued_elems.extend(sess.run(dequeued_t).tolist()) + dequeued_elems.extend(self.evaluate(dequeued_t).tolist()) enqueue_thread = self.checkedThread(target=enqueue) dequeue_thread = self.checkedThread(target=dequeue) @@ -728,10 +728,10 @@ class FIFOQueueTest(test.TestCase): # The enqueue_op should run after the dequeue op has blocked. # TODO(mrry): Figure out how to do this without sleeping. time.sleep(0.1) - sess.run(enqueue_op) + self.evaluate(enqueue_op) def dequeue(): - dequeued_elems.extend(sess.run(dequeued_t).tolist()) + dequeued_elems.extend(self.evaluate(dequeued_t).tolist()) enqueue_thread = self.checkedThread(target=enqueue) dequeue_thread = self.checkedThread(target=dequeue) @@ -778,12 +778,12 @@ class FIFOQueueTest(test.TestCase): enqueue_op.run() close_op.run() for elem in elems: - self.assertEqual([elem], dequeued_t.eval()) + self.assertEqual([elem], self.evaluate(dequeued_t)) # Expect the operation to fail due to the queue being closed. with self.assertRaisesRegexp(errors_impl.OutOfRangeError, "is closed and has insufficient"): - dequeued_t.eval() + self.evaluate(dequeued_t) def testBlockingDequeueFromClosedQueue(self): with self.cached_session() as sess: @@ -797,11 +797,11 @@ class FIFOQueueTest(test.TestCase): def dequeue(): for elem in elems: - self.assertEqual([elem], sess.run(dequeued_t)) + self.assertEqual([elem], self.evaluate(dequeued_t)) # Expect the operation to fail due to the queue being closed. with self.assertRaisesRegexp(errors_impl.OutOfRangeError, "is closed and has insufficient"): - sess.run(dequeued_t) + self.evaluate(dequeued_t) dequeue_thread = self.checkedThread(target=dequeue) dequeue_thread.start() @@ -821,7 +821,7 @@ class FIFOQueueTest(test.TestCase): # Expect the operation to fail due to the queue being closed. with self.assertRaisesRegexp(errors_impl.OutOfRangeError, "is closed and has insufficient"): - sess.run(dequeued_t) + self.evaluate(dequeued_t) dequeue_thread = self.checkedThread(target=dequeue) dequeue_thread.start() @@ -842,11 +842,11 @@ class FIFOQueueTest(test.TestCase): enqueue_op.run() def dequeue(): - self.assertAllEqual(elems, sess.run(dequeued_t)) + self.assertAllEqual(elems, self.evaluate(dequeued_t)) # Expect the operation to fail due to the queue being closed. with self.assertRaisesRegexp(errors_impl.OutOfRangeError, "is closed and has insufficient"): - sess.run(dequeued_t) + self.evaluate(dequeued_t) dequeue_thread = self.checkedThread(target=dequeue) dequeue_thread.start() @@ -867,11 +867,11 @@ class FIFOQueueTest(test.TestCase): enqueue_op.run() def dequeue(): - self.assertAllEqual(elems[:3], sess.run(dequeued_t)) + self.assertAllEqual(elems[:3], self.evaluate(dequeued_t)) # Expect the operation to fail due to the queue being closed. with self.assertRaisesRegexp(errors_impl.OutOfRangeError, "is closed and has insufficient"): - sess.run(dequeued_t) + self.evaluate(dequeued_t) dequeue_thread = self.checkedThread(target=dequeue) dequeue_thread.start() @@ -892,8 +892,8 @@ class FIFOQueueTest(test.TestCase): enqueue_op.run() def dequeue(): - self.assertAllEqual(elems[:3], sess.run(dequeued_t)) - self.assertAllEqual(elems[3:], sess.run(dequeued_t)) + self.assertAllEqual(elems[:3], self.evaluate(dequeued_t)) + self.assertAllEqual(elems[3:], self.evaluate(dequeued_t)) dequeue_thread = self.checkedThread(target=dequeue) dequeue_thread.start() @@ -913,16 +913,16 @@ class FIFOQueueTest(test.TestCase): cleanup_dequeue_t = q.dequeue() def enqueue(): - sess.run(enqueue_op) + self.evaluate(enqueue_op) def dequeue(): - self.assertAllEqual(elems[0:3], sess.run(dequeued_t)) + self.assertAllEqual(elems[0:3], self.evaluate(dequeued_t)) with self.assertRaises(errors_impl.OutOfRangeError): - sess.run(dequeued_t) - self.assertEqual(elems[3], sess.run(cleanup_dequeue_t)) + self.evaluate(dequeued_t) + self.assertEqual(elems[3], self.evaluate(cleanup_dequeue_t)) def close(): - sess.run(close_op) + self.evaluate(close_op) enqueue_thread = self.checkedThread(target=enqueue) enqueue_thread.start() @@ -955,7 +955,7 @@ class FIFOQueueTest(test.TestCase): def dequeue(): with self.assertRaises(errors_impl.OutOfRangeError): - sess.run([dequeued_a_t, dequeued_b_t]) + self.evaluate([dequeued_a_t, dequeued_b_t]) dequeue_thread = self.checkedThread(target=dequeue) dequeue_thread.start() @@ -968,7 +968,7 @@ class FIFOQueueTest(test.TestCase): # Test that the elements in the partially-dequeued batch are # restored in the correct order. for elem_a, elem_b in zip(elems_a, elems_b): - val_a, val_b = sess.run([cleanup_dequeue_a_t, cleanup_dequeue_b_t]) + val_a, val_b = self.evaluate([cleanup_dequeue_a_t, cleanup_dequeue_b_t]) self.assertEqual(elem_a, val_a) self.assertEqual(elem_b, val_b) self.assertEqual(0, q.size().eval()) @@ -983,7 +983,7 @@ class FIFOQueueTest(test.TestCase): # Expect the operation to fail due to the queue being closed. with self.assertRaisesRegexp(errors_impl.OutOfRangeError, "is closed and has insufficient"): - sess.run(dequeued_t) + self.evaluate(dequeued_t) dequeue_thread = self.checkedThread(target=dequeue) dequeue_thread.start() @@ -1003,7 +1003,7 @@ class FIFOQueueTest(test.TestCase): # Expect the operation to fail due to the queue being closed. with self.assertRaisesRegexp(errors_impl.OutOfRangeError, "is closed and has insufficient"): - sess.run(dequeued_t) + self.evaluate(dequeued_t) dequeue_thread = self.checkedThread(target=dequeue) dequeue_thread.start() @@ -1051,7 +1051,7 @@ class FIFOQueueTest(test.TestCase): enqueue_op.run() def blocking_enqueue(): - sess.run(blocking_enqueue_op) + self.evaluate(blocking_enqueue_op) thread = self.checkedThread(target=blocking_enqueue) thread.start() @@ -1059,8 +1059,8 @@ class FIFOQueueTest(test.TestCase): # TODO(mrry): Figure out how to do this without sleeping. time.sleep(0.1) for elem in elems: - self.assertEqual([elem], dequeued_t.eval()) - self.assertEqual([50.0], dequeued_t.eval()) + self.assertEqual([elem], self.evaluate(dequeued_t)) + self.assertEqual([50.0], self.evaluate(dequeued_t)) thread.join() def testBlockingEnqueueManyToFullQueue(self): @@ -1074,7 +1074,7 @@ class FIFOQueueTest(test.TestCase): enqueue_op.run() def blocking_enqueue(): - sess.run(blocking_enqueue_op) + self.evaluate(blocking_enqueue_op) thread = self.checkedThread(target=blocking_enqueue) thread.start() @@ -1082,10 +1082,10 @@ class FIFOQueueTest(test.TestCase): # TODO(mrry): Figure out how to do this without sleeping. time.sleep(0.1) for elem in elems: - self.assertEqual([elem], dequeued_t.eval()) + self.assertEqual([elem], self.evaluate(dequeued_t)) time.sleep(0.01) - self.assertEqual([50.0], dequeued_t.eval()) - self.assertEqual([60.0], dequeued_t.eval()) + self.assertEqual([50.0], self.evaluate(dequeued_t)) + self.assertEqual([60.0], self.evaluate(dequeued_t)) # Make sure the thread finishes before exiting. thread.join() @@ -1103,7 +1103,7 @@ class FIFOQueueTest(test.TestCase): def blocking_enqueue(): # Expect the operation to succeed once the dequeue op runs. - sess.run(blocking_enqueue_op) + self.evaluate(blocking_enqueue_op) enqueue_thread = self.checkedThread(target=blocking_enqueue) enqueue_thread.start() @@ -1113,18 +1113,18 @@ class FIFOQueueTest(test.TestCase): time.sleep(0.1) def close(): - sess.run(close_op) + self.evaluate(close_op) close_thread = self.checkedThread(target=close) close_thread.start() # The dequeue will unblock both threads. - self.assertEqual(10.0, dequeued_t.eval()) + self.assertEqual(10.0, self.evaluate(dequeued_t)) enqueue_thread.join() close_thread.join() for elem in [20.0, 30.0, 40.0, 50.0]: - self.assertEqual(elem, dequeued_t.eval()) + self.assertEqual(elem, self.evaluate(dequeued_t)) self.assertEqual(0, q.size().eval()) def testBlockingEnqueueManyBeforeClose(self): @@ -1138,7 +1138,7 @@ class FIFOQueueTest(test.TestCase): enqueue_op.run() def blocking_enqueue(): - sess.run(blocking_enqueue_op) + self.evaluate(blocking_enqueue_op) enqueue_thread = self.checkedThread(target=blocking_enqueue) enqueue_thread.start() @@ -1148,17 +1148,17 @@ class FIFOQueueTest(test.TestCase): time.sleep(0.1) def close(): - sess.run(close_op) + self.evaluate(close_op) close_thread = self.checkedThread(target=close) close_thread.start() # The dequeue will unblock both threads. - self.assertEqual(10.0, dequeued_t.eval()) + self.assertEqual(10.0, self.evaluate(dequeued_t)) enqueue_thread.join() close_thread.join() for elem in [20.0, 30.0, 50.0, 60.0]: - self.assertEqual(elem, dequeued_t.eval()) + self.assertEqual(elem, self.evaluate(dequeued_t)) def testDoesNotLoseValue(self): with self.cached_session(): @@ -1266,19 +1266,19 @@ class FIFOQueueTest(test.TestCase): def _blockingDequeue(self, sess, dequeue_op): with self.assertRaisesOpError("was cancelled"): - sess.run(dequeue_op) + self.evaluate(dequeue_op) def _blockingDequeueMany(self, sess, dequeue_many_op): with self.assertRaisesOpError("was cancelled"): - sess.run(dequeue_many_op) + self.evaluate(dequeue_many_op) def _blockingEnqueue(self, sess, enqueue_op): with self.assertRaisesOpError("was cancelled"): - sess.run(enqueue_op) + self.evaluate(enqueue_op) def _blockingEnqueueMany(self, sess, enqueue_many_op): with self.assertRaisesOpError("was cancelled"): - sess.run(enqueue_many_op) + self.evaluate(enqueue_many_op) def testResetOfBlockingOperation(self): with self.cached_session() as sess: @@ -1321,7 +1321,7 @@ class FIFOQueueTest(test.TestCase): def blocking_enqueue(): enq_done.append(False) # This will fill the queue and then block until enough dequeues happen. - sess.run(enq) + self.evaluate(enq) enq_done.append(True) thread = self.checkedThread(target=blocking_enqueue) @@ -1331,14 +1331,14 @@ class FIFOQueueTest(test.TestCase): results = [] results.append(deq.eval()) # Will only complete after the enqueue starts. self.assertEqual(len(enq_done), 1) - self.assertEqual(sess.run(size_op), 5) + self.assertEqual(self.evaluate(size_op), 5) for _ in range(3): results.append(deq.eval()) time.sleep(0.1) self.assertEqual(len(enq_done), 1) - self.assertEqual(sess.run(size_op), 5) + self.assertEqual(self.evaluate(size_op), 5) # This dequeue will unblock the thread. results.append(deq.eval()) @@ -1364,7 +1364,7 @@ class FIFOQueueTest(test.TestCase): def blocking_dequeue(): # Will only complete after 4 enqueues complete. - results.extend(sess.run(deq)) + results.extend(self.evaluate(deq)) thread = self.checkedThread(target=blocking_dequeue) thread.start() @@ -1373,7 +1373,7 @@ class FIFOQueueTest(test.TestCase): # TODO(mrry): Figure out how to do this without sleeping. time.sleep(0.1) self.assertEqual(len(results), 0) - sess.run(enq) + self.evaluate(enq) # Enough enqueued to unblock the dequeue thread.join() @@ -1405,7 +1405,7 @@ class FIFOQueueTest(test.TestCase): q.enqueue_many(input_tuple).run() output_tuple_t = q.dequeue_many(32) - output_tuple = sess.run(output_tuple_t) + output_tuple = self.evaluate(output_tuple_t) for (input_elem, output_elem) in zip(input_tuple, output_tuple): self.assertAllEqual(input_elem, output_elem) @@ -1507,10 +1507,10 @@ class FIFOQueueDictTest(test.TestCase): enqueue_op4 = q.enqueue_many({"f": [40.0, 50.0]}) dequeue = q.dequeue() dequeue_2 = q.dequeue_many(2) - sess.run(enqueue_op) - sess.run(enqueue_op2) - sess.run(enqueue_op3) - sess.run(enqueue_op4) + self.evaluate(enqueue_op) + self.evaluate(enqueue_op2) + self.evaluate(enqueue_op3) + self.evaluate(enqueue_op4) f = sess.run(dequeue["f"]) self.assertEqual(10.0, f) f = sess.run(dequeue_2["f"]) @@ -1565,10 +1565,10 @@ class FIFOQueueDictTest(test.TestCase): }) dequeue = q.dequeue() dequeue_2 = q.dequeue_many(2) - sess.run(enqueue_op) - sess.run(enqueue_op2) - sess.run(enqueue_op3) - sess.run(enqueue_op4) + self.evaluate(enqueue_op) + self.evaluate(enqueue_op2) + self.evaluate(enqueue_op3) + self.evaluate(enqueue_op4) i, f, s = sess.run([dequeue["i"], dequeue["f"], dequeue["s"]]) self.assertEqual(123, i) self.assertEqual(10.0, f) @@ -1597,7 +1597,7 @@ class FIFOQueueWithTimeoutTest(test.TestCase): # until operation_timeout_in_ms. with self.assertRaisesRegexp(errors_impl.DeadlineExceededError, "Timed out waiting for notification"): - sess.run(dequeued_t) + self.evaluate(dequeued_t) def testReusableAfterTimeout(self): with self.cached_session() as sess: @@ -1613,8 +1613,8 @@ class FIFOQueueWithTimeoutTest(test.TestCase): "Timed out waiting for notification"): sess.run(dequeued_t, options=config_pb2.RunOptions(timeout_in_ms=10)) - sess.run(enqueue_op) - self.assertEqual(37, sess.run(dequeued_t)) + self.evaluate(enqueue_op) + self.assertEqual(37, self.evaluate(dequeued_t)) class QueueContainerTest(test.TestCase): diff --git a/tensorflow/python/kernel_tests/fractional_avg_pool_op_test.py b/tensorflow/python/kernel_tests/fractional_avg_pool_op_test.py index f89d2062f1e..0d5928aefac 100644 --- a/tensorflow/python/kernel_tests/fractional_avg_pool_op_test.py +++ b/tensorflow/python/kernel_tests/fractional_avg_pool_op_test.py @@ -24,6 +24,7 @@ import numpy as np from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes +from tensorflow.python.framework import test_util from tensorflow.python.ops import array_ops from tensorflow.python.ops import gen_nn_ops from tensorflow.python.ops import gradient_checker @@ -37,7 +38,6 @@ class FractionalAvgTest(test.TestCase): # Random number generate with seed. _PRNG = np.random.RandomState(341261000) _SEED = 341261001 - _SEED2 = 341261002 def _AvgPoolAlongRows(self, input_matrix, row_seq, overlapping): """Perform average pool along row of a 2-D matrix based on row_seq. @@ -128,15 +128,13 @@ class FractionalAvgTest(test.TestCase): None """ with self.cached_session() as sess: - p, r, c = nn_ops.fractional_avg_pool( + p, r, c = nn_ops.fractional_avg_pool_v2( input_tensor, pooling_ratio, pseudo_random, overlapping, - deterministic=True, - seed=self._SEED, - seed2=self._SEED2) - actual, row_seq, col_seq = sess.run([p, r, c]) + seed=self._SEED) + actual, row_seq, col_seq = self.evaluate([p, r, c]) expected = self._GetExpectedFractionalAvgPoolResult(input_tensor, row_seq, col_seq, overlapping) self.assertShapeEqual(expected, p) @@ -161,15 +159,13 @@ class FractionalAvgTest(test.TestCase): rand_mat = self._PRNG.randint(10, size=tensor_shape) pooling_ratio = [1, math.sqrt(2), math.sqrt(2), 1] with self.cached_session() as sess: - p, r, c = nn_ops.fractional_avg_pool( + p, r, c = nn_ops.fractional_avg_pool_v2( rand_mat.astype(np.float32), pooling_ratio, pseudo_random, overlapping, - deterministic=True, - seed=self._SEED, - seed2=self._SEED2) - tensor_output, row_seq, col_seq = sess.run([p, r, c]) + seed=self._SEED) + tensor_output, row_seq, col_seq = self.evaluate([p, r, c]) expected_result = self._GetExpectedFractionalAvgPoolResult( rand_mat.astype(np.float32), row_seq, col_seq, overlapping) print("row sequence:") @@ -214,12 +210,6 @@ class FractionalAvgTest(test.TestCase): def testIntegerTensorInput(self): """Test FractionalAvgPool works fine when input tensor is integer type. - - I would have used _ValidateFractionalAvgPoolResult function to automate this - process, however, there's rounding issue. It is caused by numpy.mean cast - integer input to numpy.float64 for intermediate use. While for - fractional_avg_pool, the mean operation is integer division (trucated). So, - for this test case, I will hard code a simple matrix. """ pseudo_random = True overlapping = True @@ -234,29 +224,9 @@ class FractionalAvgTest(test.TestCase): [4, 4, 5, 9, 7, 2] ]) # pyformat: enable - with self.cached_session() as sess: - # Since deterministic = True, seed and seed2 are fixed. Therefore r, and c - # are the same each time. We can have an expected result precomputed. - # r = [0, 2, 4, 6] - # c = [0, 1, 3, 4, 6] - - # pyformat: disable - expected = np.array([ - [6, 5, 3, 5], - [5, 5, 4, 5], - [5, 4, 7, 5] - ]).reshape((1, 3, 4, 1)) - # pyformat: enable - p, unused_r, unused_c = nn_ops.fractional_avg_pool( - mat.reshape(tensor_shape), [1, math.sqrt(3), math.sqrt(2), 1], - pseudo_random, - overlapping, - deterministic=True, - seed=self._SEED, - seed2=self._SEED2) - actual = sess.run(p) - self.assertShapeEqual(expected, p) - self.assertAllClose(expected, actual) + self._ValidateFractionalAvgPoolResult(mat.reshape(tensor_shape), + [1, math.sqrt(3), math.sqrt(2), 1], + pseudo_random, overlapping) def testDifferentTensorShapes(self): """Test different shapes of input tensor. @@ -312,6 +282,7 @@ class FractionalAvgTest(test.TestCase): self._ValidateFractionalAvgPoolResult(rand_mat, [1, 2, 2, 1], pseudo_random, overlapping) + @test_util.run_deprecated_v1 def testDifferentInputTensorShape(self): """Runs the operation in one session with different input tensor shapes.""" with self.cached_session() as sess: @@ -320,14 +291,12 @@ class FractionalAvgTest(test.TestCase): pooling_ratio = [1, 1.5, 1.5, 1] pseudo_random = False overlapping = False - p, r, c = nn_ops.fractional_avg_pool( + p, r, c = nn_ops.fractional_avg_pool_v2( input_holder, pooling_ratio, pseudo_random, overlapping, - deterministic=True, - seed=self._SEED, - seed2=self._SEED2) + seed=self._SEED) # First run. input_a = np.zeros([3, 32, 32, 3]) actual, row_seq, col_seq = sess.run([p, r, c], {input_holder: input_a}) @@ -372,7 +341,6 @@ class FractionalAvgPoolGradTest(test.TestCase): """ _PRNG = np.random.RandomState(341261004) _SEED = 341261005 - _SEED2 = 341261006 def _GenerateRandomInputTensor(self, shape): num_elements = 1 @@ -398,7 +366,7 @@ class FractionalAvgPoolGradTest(test.TestCase): padding = "VALID" output_tensor = nn_ops.avg_pool(input_tensor, window_size, stride_size, padding) - output_data = output_tensor.eval() + output_data = self.evaluate(output_tensor) num_elements = 1 for dim_size in output_data.shape: num_elements *= dim_size @@ -407,7 +375,7 @@ class FractionalAvgPoolGradTest(test.TestCase): input_backprop_tensor = gen_nn_ops.avg_pool_grad( input_tensor.get_shape(), output_backprop, window_size, stride_size, padding) - input_backprop = input_backprop_tensor.eval() + input_backprop = self.evaluate(input_backprop_tensor) row_seq = list(range(0, num_rows + 1, row_window_size)) col_seq = list(range(0, num_cols + 1, col_window_size)) fap_input_backprop_tensor = gen_nn_ops.fractional_avg_pool_grad( @@ -416,7 +384,7 @@ class FractionalAvgPoolGradTest(test.TestCase): row_seq, col_seq, overlapping=False) - fap_input_backprop = fap_input_backprop_tensor.eval() + fap_input_backprop = self.evaluate(fap_input_backprop_tensor) self.assertShapeEqual(input_backprop, fap_input_backprop_tensor) self.assertAllClose(input_backprop, fap_input_backprop) @@ -437,7 +405,7 @@ class FractionalAvgPoolGradTest(test.TestCase): padding = "VALID" output_tensor = nn_ops.avg_pool(input_tensor, window_size, stride_size, padding) - output_data = output_tensor.eval() + output_data = self.evaluate(output_tensor) num_elements = 1 for dim_size in output_data.shape: num_elements *= dim_size @@ -446,7 +414,7 @@ class FractionalAvgPoolGradTest(test.TestCase): input_backprop_tensor = gen_nn_ops.avg_pool_grad( input_tensor.get_shape(), output_backprop, window_size, stride_size, padding) - input_backprop = input_backprop_tensor.eval() + input_backprop = self.evaluate(input_backprop_tensor) row_seq = list(range(0, num_rows, row_window_size - 1)) col_seq = list(range(0, num_cols, col_window_size - 1)) row_seq[-1] += 1 @@ -457,10 +425,11 @@ class FractionalAvgPoolGradTest(test.TestCase): row_seq, col_seq, overlapping=True) - fap_input_backprop = fap_input_backprop_tensor.eval() + fap_input_backprop = self.evaluate(fap_input_backprop_tensor) self.assertShapeEqual(input_backprop, fap_input_backprop_tensor) self.assertAllClose(input_backprop, fap_input_backprop) + @test_util.run_deprecated_v1 def testAllInputOptionsThroughGradientError(self): input_shape = (1, 7, 13, 1) input_data = self._GenerateRandomInputTensor(input_shape) @@ -470,15 +439,13 @@ class FractionalAvgPoolGradTest(test.TestCase): for overlapping in True, False: with self.cached_session() as _: input_tensor = constant_op.constant(input_data, shape=input_shape) - output_tensor, unused_a, unused_b = nn_ops.fractional_avg_pool( + output_tensor, unused_a, unused_b = nn_ops.fractional_avg_pool_v2( input_tensor, pooling_ratio, pseudo_random=pseudo_random, overlapping=overlapping, - deterministic=True, - seed=self._SEED, - seed2=self._SEED2) - output_data = output_tensor.eval() + seed=self._SEED) + output_data = self.evaluate(output_tensor) output_shape = output_data.shape # error_margin and delta setting is similar to avg_pool_grad. error_margin = 1e-4 @@ -491,6 +458,7 @@ class FractionalAvgPoolGradTest(test.TestCase): delta=1e-2) self.assertLess(gradient_error, error_margin) + @test_util.run_deprecated_v1 def testDifferentTensorShapesThroughGradientError(self): pseudo_random = True overlapping = True @@ -503,15 +471,13 @@ class FractionalAvgPoolGradTest(test.TestCase): input_data = self._GenerateRandomInputTensor(input_shape) with self.cached_session() as _: input_tensor = constant_op.constant(input_data, shape=input_shape) - output_tensor, unused_a, unused_b = nn_ops.fractional_avg_pool( + output_tensor, unused_a, unused_b = nn_ops.fractional_avg_pool_v2( input_tensor, pooling_ratio, pseudo_random=pseudo_random, overlapping=overlapping, - deterministic=True, - seed=self._SEED, - seed2=self._SEED2) - output_data = output_tensor.eval() + seed=self._SEED) + output_data = self.evaluate(output_tensor) output_shape = output_data.shape # error_margin and delta setting is similar to avg_pool_grad. error_margin = 1e-4 @@ -524,6 +490,7 @@ class FractionalAvgPoolGradTest(test.TestCase): delta=1e-2) self.assertLess(gradient_error, error_margin) + @test_util.run_deprecated_v1 def testLargePoolingRatioThroughGradientError(self): input_shape = (1, 17, 23, 1) input_data = self._GenerateRandomInputTensor(input_shape) @@ -534,14 +501,12 @@ class FractionalAvgPoolGradTest(test.TestCase): with self.cached_session() as _: input_tensor = constant_op.constant(input_data, shape=input_shape) - output_tensor, unused_a, unused_b = nn_ops.fractional_avg_pool( + output_tensor, unused_a, unused_b = nn_ops.fractional_avg_pool_v2( input_tensor, pooling_ratio, pseudo_random=pseudo_random, overlapping=overlapping, - deterministic=True, - seed=self._SEED, - seed2=self._SEED2) + seed=self._SEED) # error_margin and delta setting is similar to avg_pool_grad. error_margin = 1e-4 gradient_error = gradient_checker.compute_gradient_error( diff --git a/tensorflow/python/kernel_tests/fractional_max_pool_op_test.py b/tensorflow/python/kernel_tests/fractional_max_pool_op_test.py index 9b94ca85547..fa886cc215a 100644 --- a/tensorflow/python/kernel_tests/fractional_max_pool_op_test.py +++ b/tensorflow/python/kernel_tests/fractional_max_pool_op_test.py @@ -24,6 +24,7 @@ import numpy as np from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes +from tensorflow.python.framework import test_util from tensorflow.python.ops import array_ops from tensorflow.python.ops import gen_nn_ops from tensorflow.python.ops import gradient_checker @@ -37,7 +38,6 @@ class FractionalMaxPoolTest(test.TestCase): # Random number generate with seed. _PRNG = np.random.RandomState(341261) _SEED = 123456 - _SEED2 = 654321 def _MaxPoolAlongRows(self, input_matrix, row_seq, overlapping): """Perform max pool along row of a 2-D matrix based on row_seq. @@ -128,15 +128,13 @@ class FractionalMaxPoolTest(test.TestCase): None """ with self.cached_session() as sess: - p, r, c = nn_ops.fractional_max_pool( + p, r, c = nn_ops.fractional_max_pool_v2( input_tensor, pooling_ratio, pseudo_random, overlapping, - deterministic=True, - seed=self._SEED, - seed2=self._SEED2) - actual, row_seq, col_seq = sess.run([p, r, c]) + seed=self._SEED) + actual, row_seq, col_seq = self.evaluate([p, r, c]) expected = self._GetExpectedFractionalMaxPoolResult(input_tensor, row_seq, col_seq, overlapping) self.assertShapeEqual(expected, p) @@ -161,15 +159,13 @@ class FractionalMaxPoolTest(test.TestCase): rand_mat = self._PRNG.randint(10, size=tensor_shape) pooling_ratio = [1, math.sqrt(2), math.sqrt(2), 1] with self.cached_session() as sess: - p, r, c = nn_ops.fractional_max_pool( + p, r, c = nn_ops.fractional_max_pool_v2( rand_mat, pooling_ratio, pseudo_random, overlapping, - deterministic=True, - seed=self._SEED, - seed2=self._SEED2) - tensor_output, row_seq, col_seq = sess.run([p, r, c]) + seed=self._SEED) + tensor_output, row_seq, col_seq = self.evaluate([p, r, c]) expected_result = self._GetExpectedFractionalMaxPoolResult(rand_mat, row_seq, col_seq, @@ -283,6 +279,7 @@ class FractionalMaxPoolTest(test.TestCase): self._ValidateFractionalMaxPoolResult(rand_mat, [1, 2, 2, 1], pseudo_random, overlapping) + @test_util.run_deprecated_v1 def testDifferentInputTensorShape(self): """Runs the operation in one session with different input tensor shapes.""" with self.cached_session() as sess: @@ -291,14 +288,12 @@ class FractionalMaxPoolTest(test.TestCase): pooling_ratio = [1, 1.5, 1.5, 1] pseudo_random = False overlapping = False - p, r, c = nn_ops.fractional_max_pool( + p, r, c = nn_ops.fractional_max_pool_v2( input_holder, pooling_ratio, pseudo_random, overlapping, - deterministic=True, - seed=self._SEED, - seed2=self._SEED2) + seed=self._SEED) # First run. input_a = np.zeros([3, 32, 32, 3]) actual, row_seq, col_seq = sess.run([p, r, c], {input_holder: input_a}) @@ -344,7 +339,6 @@ class FractionalMaxPoolGradTest(test.TestCase): _PRNG = np.random.RandomState(341261) _SEED = 123456 - _SEED2 = 654321 def _GenerateUniqueRandomInputTensor(self, shape): """Generate 'unqiue' random input tensor. @@ -382,12 +376,12 @@ class FractionalMaxPoolGradTest(test.TestCase): padding = "VALID" output_tensor = nn_ops.max_pool(input_tensor, window_size, stride_size, padding) - output_data = output_tensor.eval() + output_data = self.evaluate(output_tensor) output_backprop = self._PRNG.randint(100, size=output_data.shape) input_backprop_tensor = gen_nn_ops.max_pool_grad( input_tensor, output_tensor, output_backprop, window_size, stride_size, padding) - input_backprop = input_backprop_tensor.eval() + input_backprop = self.evaluate(input_backprop_tensor) row_seq = list(range(0, num_rows + 1, row_window_size)) col_seq = list(range(0, num_cols + 1, col_window_size)) fmp_input_backprop_tensor = gen_nn_ops.fractional_max_pool_grad( @@ -397,7 +391,7 @@ class FractionalMaxPoolGradTest(test.TestCase): row_seq, col_seq, overlapping=False) - fmp_input_backprop = fmp_input_backprop_tensor.eval() + fmp_input_backprop = self.evaluate(fmp_input_backprop_tensor) self.assertShapeEqual(input_backprop, fmp_input_backprop_tensor) self.assertAllClose(input_backprop, fmp_input_backprop) @@ -417,12 +411,12 @@ class FractionalMaxPoolGradTest(test.TestCase): padding = "VALID" output_tensor = nn_ops.max_pool(input_tensor, window_size, stride_size, padding) - output_data = output_tensor.eval() + output_data = self.evaluate(output_tensor) output_backprop = self._PRNG.randint(100, size=output_data.shape) input_backprop_tensor = gen_nn_ops.max_pool_grad( input_tensor, output_tensor, output_backprop, window_size, stride_size, padding) - input_backprop = input_backprop_tensor.eval() + input_backprop = self.evaluate(input_backprop_tensor) row_seq = list(range(0, num_rows, row_window_size - 1)) col_seq = list(range(0, num_cols, col_window_size - 1)) row_seq[-1] += 1 @@ -434,10 +428,11 @@ class FractionalMaxPoolGradTest(test.TestCase): row_seq, col_seq, overlapping=True) - fmp_input_backprop = fmp_input_backprop_tensor.eval() + fmp_input_backprop = self.evaluate(fmp_input_backprop_tensor) self.assertShapeEqual(input_backprop, fmp_input_backprop_tensor) self.assertAllClose(input_backprop, fmp_input_backprop) + @test_util.run_deprecated_v1 def testAllInputOptionsThroughGradientError(self): input_shape = (1, 7, 13, 1) input_data = self._GenerateUniqueRandomInputTensor(input_shape) @@ -449,15 +444,13 @@ class FractionalMaxPoolGradTest(test.TestCase): for overlapping in True, False: with self.cached_session() as _: input_tensor = constant_op.constant(input_data, shape=input_shape) - output_tensor, unused_a, unused_b = nn_ops.fractional_max_pool( + output_tensor, unused_a, unused_b = nn_ops.fractional_max_pool_v2( input_tensor, pooling_ratio, pseudo_random=pseudo_random, overlapping=overlapping, - deterministic=True, - seed=self._SEED, - seed2=self._SEED2) - output_data = output_tensor.eval() + seed=self._SEED) + output_data = self.evaluate(output_tensor) output_shape = output_data.shape # error_margin and delta setting is similar to max_pool_grad. error_margin = 1e-3 @@ -470,6 +463,7 @@ class FractionalMaxPoolGradTest(test.TestCase): delta=1e-2) self.assertLess(gradient_error, error_margin) + @test_util.run_deprecated_v1 def testDifferentTensorShapesThroughGradientError(self): pseudo_random = True overlapping = True @@ -484,15 +478,13 @@ class FractionalMaxPoolGradTest(test.TestCase): input_data += self._PRNG.random_sample(input_shape) with self.cached_session() as _: input_tensor = constant_op.constant(input_data, shape=input_shape) - output_tensor, unused_a, unused_b = nn_ops.fractional_max_pool( + output_tensor, unused_a, unused_b = nn_ops.fractional_max_pool_v2( input_tensor, pooling_ratio, pseudo_random=pseudo_random, overlapping=overlapping, - deterministic=True, - seed=self._SEED, - seed2=self._SEED2) - output_data = output_tensor.eval() + seed=self._SEED) + output_data = self.evaluate(output_tensor) output_shape = output_data.shape # error_margin and delta setting is similar to max_pool_grad. error_margin = 1e-3 @@ -505,6 +497,7 @@ class FractionalMaxPoolGradTest(test.TestCase): delta=1e-2) self.assertLess(gradient_error, error_margin) + @test_util.run_deprecated_v1 def testLargePoolingRatioThroughGradientError(self): input_shape = (1, 17, 23, 1) input_data = self._GenerateUniqueRandomInputTensor(input_shape) @@ -517,14 +510,12 @@ class FractionalMaxPoolGradTest(test.TestCase): with self.cached_session() as _: input_tensor = constant_op.constant(input_data, shape=input_shape) - output_tensor, unused_a, unused_b = nn_ops.fractional_max_pool( + output_tensor, unused_a, unused_b = nn_ops.fractional_max_pool_v2( input_tensor, pooling_ratio, pseudo_random=pseudo_random, overlapping=overlapping, - deterministic=True, - seed=self._SEED, - seed2=self._SEED2) + seed=self._SEED) # error_margin and delta setting is similar to max_pool_grad. error_margin = 1e-3 gradient_error = gradient_checker.compute_gradient_error( @@ -592,7 +583,7 @@ class FractionalMaxPoolGradTest(test.TestCase): row_seq, col_seq, overlapping=False) - input_backprop_not_overlapping = r.eval() + input_backprop_not_overlapping = self.evaluate(r) self.assertShapeEqual( np.reshape(expected_input_backprop_not_overlapping, input_size), r) self.assertAllClose(expected_input_backprop_not_overlapping, @@ -602,7 +593,7 @@ class FractionalMaxPoolGradTest(test.TestCase): output_data_overlapping, shape=output_size) r = gen_nn_ops.fractional_max_pool_grad( input_tensor, output_tensor, grad, row_seq, col_seq, overlapping=True) - input_backprop_overlapping = r.eval() + input_backprop_overlapping = self.evaluate(r) self.assertShapeEqual( np.reshape(expected_input_backprop_overlapping, input_size), r) self.assertAllClose(expected_input_backprop_overlapping, diff --git a/tensorflow/python/kernel_tests/functional_ops_test.py b/tensorflow/python/kernel_tests/functional_ops_test.py index 04c1032722c..c489623fe56 100644 --- a/tensorflow/python/kernel_tests/functional_ops_test.py +++ b/tensorflow/python/kernel_tests/functional_ops_test.py @@ -56,6 +56,7 @@ def simple_scoped_fn(a, x): return math_ops.multiply(math_ops.add(a, x), two) +@test_util.with_control_flow_v2 class FunctionalOpsTest(test.TestCase): @test_util.run_in_graph_and_eager_modes @@ -100,6 +101,7 @@ class FunctionalOpsTest(test.TestCase): (elems, other_elems), initializer) self.assertAllEqual([1.0, 2.0, 3.0], self.evaluate(r)) + @test_util.run_deprecated_v1 def testFoldl_Scoped(self): with self.cached_session() as sess: with variable_scope.variable_scope("root") as varscope: @@ -152,6 +154,7 @@ class FunctionalOpsTest(test.TestCase): initializer) self.assertAllEqual(1, self.evaluate(r)) + @test_util.run_deprecated_v1 def testFoldr_Scoped(self): with self.cached_session() as sess: with variable_scope.variable_scope("root") as varscope: @@ -172,6 +175,7 @@ class FunctionalOpsTest(test.TestCase): self.assertAllEqual(1282, self.evaluate(r)) # pylint: disable=unnecessary-lambda + @test_util.run_deprecated_v1 def testFold_Grad(self): with self.cached_session(): elems = constant_op.constant([1.0, 2.0, 3.0, 4.0, 5.0, 6.0], name="data") @@ -213,6 +217,7 @@ class FunctionalOpsTest(test.TestCase): with self.assertRaisesRegexp(ValueError, "not a scalar"): functional_ops.map_fn(lambda x: x, 1) + @test_util.run_deprecated_v1 def testMap_Scoped(self): with self.cached_session() as sess: @@ -244,6 +249,7 @@ class FunctionalOpsTest(test.TestCase): self.assertEqual(len(variables.trainable_variables()), 1) self.assertAllEqual(doubles, self.evaluate(r)) + @test_util.run_deprecated_v1 def testMap_Grad(self): with self.cached_session(): param = constant_op.constant(2.0) @@ -380,6 +386,7 @@ class FunctionalOpsTest(test.TestCase): ValueError, "two structures don't have the same nested structure"): functional_ops.scan(lambda a, x: (a, -a), elems, initializer) + @test_util.run_deprecated_v1 def testScan_Scoped(self): with self.cached_session() as sess: with variable_scope.variable_scope("root") as varscope: @@ -424,6 +431,7 @@ class FunctionalOpsTest(test.TestCase): # t_1 == 1, b == 4.5, y == 0.5, returns b * y * x = 9 self.assertAllClose([1., 1., 2.25, 9.], self.evaluate(r)) + @test_util.run_deprecated_v1 def testScan_Control(self): with self.cached_session() as sess: s = array_ops.placeholder(dtypes.float32, shape=[None]) @@ -435,6 +443,7 @@ class FunctionalOpsTest(test.TestCase): np.array([1.0, 3.0, 9.0]), sess.run(c, {s: [1, 3, 3], b: True})) + @test_util.run_deprecated_v1 def testScan_Grad(self): with self.cached_session(): elems = constant_op.constant([1.0, 2.0, 3.0, 4.0, 5.0, 6.0], name="data") @@ -447,6 +456,7 @@ class FunctionalOpsTest(test.TestCase): r = gradients_impl.gradients(r, v)[0] self.assertAllEqual(873.0, self.evaluate(r)) + @test_util.run_deprecated_v1 def testScanGradientWithPartStopGradient(self): a = variables.Variable(0.0, name="a") b = variables.Variable(0.0, name="b") @@ -457,7 +467,7 @@ class FunctionalOpsTest(test.TestCase): grad = gradients_impl.gradients(ys=[loss], xs=[a, b]) with self.test_session(use_gpu=True) as sess: variables.global_variables_initializer().run() - sess.run(grad) + self.evaluate(grad) @test_util.run_in_graph_and_eager_modes def testFoldShape(self): @@ -476,12 +486,15 @@ class FunctionalOpsTest(test.TestCase): y = functional_ops.map_fn(lambda e: e, x) self.assertAllEqual(y.get_shape(), self.evaluate(y).shape) + @test_util.run_deprecated_v1 def testMapUnknownShape(self): x = array_ops.placeholder(dtypes.float32) y = functional_ops.map_fn(lambda e: e, x) self.assertIs(None, y.get_shape().dims) + @test_util.disable_control_flow_v2("b/119323354") @test_util.run_in_graph_and_eager_modes + @test_util.run_deprecated_v1 def testMapEmptyScalar(self): map_return = functional_ops.map_fn(lambda x: 1, constant_op.constant([])) self.assertAllEqual([0], map_return.get_shape().dims) @@ -489,6 +502,8 @@ class FunctionalOpsTest(test.TestCase): # TODO(akshayka): this test fails in eager: the iterable is of length 0 so # so the body of the while loop never executes + @test_util.disable_control_flow_v2("b/119323354") + @test_util.run_deprecated_v1 def testMapEmptyTensor(self): with self.cached_session(): map_return = functional_ops.map_fn(lambda x: array_ops.zeros([3, 2]), @@ -509,6 +524,7 @@ class FunctionalOpsTest(test.TestCase): # TODO(akshayka): this test fails in eager: the iterable is of length 0 so # so the body of the while loop never executes + @test_util.run_deprecated_v1 def testScanEmptyTensor(self): with self.cached_session(): x = functional_ops.scan( @@ -516,6 +532,7 @@ class FunctionalOpsTest(test.TestCase): self.assertAllEqual([0, 2, 4], x.get_shape()) self.assertAllEqual(x.get_shape(), self.evaluate(x).shape) + @test_util.run_deprecated_v1 def testScanUnknownShape(self): x = array_ops.placeholder(dtypes.float32) initializer = array_ops.placeholder(dtypes.float32) @@ -526,6 +543,7 @@ class FunctionalOpsTest(test.TestCase): y = functional_ops.scan(fn, x, initializer=initializer) self.assertIs(None, y.get_shape().dims) + @test_util.run_deprecated_v1 def testScanVaryingShape(self): with self.cached_session() as sess: x = array_ops.placeholder(dtype=dtypes.float32, shape=[None, 2]) @@ -542,6 +560,7 @@ class FunctionalOpsTest(test.TestCase): sess.run([result, result_t, result_grad, result_t_grad], feed_dict={x: [[1.0, 2.0]]}) + @test_util.run_deprecated_v1 def testRemoteFunction(self): worker_config = config_pb2.ConfigProto() worker_config.device_count["CPU"] = 2 @@ -564,10 +583,11 @@ class FunctionalOpsTest(test.TestCase): target="/job:worker/replica:0/task:0/cpu:1") with session.Session(worker[0].target) as sess: - sess.run(variables.global_variables_initializer()) - mul = sess.run(remote_op) + self.evaluate(variables.global_variables_initializer()) + mul = self.evaluate(remote_op) self.assertEqual(mul, [6]) + @test_util.run_deprecated_v1 def testRemoteFunctionDirectSession(self): worker_config = config_pb2.ConfigProto() worker_config.device_count["CPU"] = 2 @@ -588,10 +608,11 @@ class FunctionalOpsTest(test.TestCase): target="/job:localhost/replica:0/task:0/cpu:1") with self.test_session(config=worker_config) as sess: - sess.run(variables.global_variables_initializer()) - mul = sess.run(remote_op) + self.evaluate(variables.global_variables_initializer()) + mul = self.evaluate(remote_op) self.assertEqual(mul, [6]) + @test_util.run_deprecated_v1 def testRemoteFunctionSameDeviceDirectSession(self): @function.Defun(dtypes.int32, dtypes.int32) @@ -607,8 +628,8 @@ class FunctionalOpsTest(test.TestCase): args=[a, b], Tout=[dtypes.int32], f=_remote_fn, target="/cpu:0") with self.cached_session() as sess: - sess.run(variables.global_variables_initializer()) - mul = sess.run(remote_op) + self.evaluate(variables.global_variables_initializer()) + mul = self.evaluate(remote_op) self.assertEqual(mul, [6]) def testRemoteFunctionCPUGPU(self): @@ -631,8 +652,8 @@ class FunctionalOpsTest(test.TestCase): target="/job:localhost/replica:0/task:0/device:GPU:0")[0] + 3.0 with self.cached_session() as sess: - sess.run(variables.global_variables_initializer()) - mul = sess.run(remote_op) + self.evaluate(variables.global_variables_initializer()) + mul = self.evaluate(remote_op) self.assertEqual(mul, 9.0) def testRemoteFunctionGPUCPU(self): @@ -655,8 +676,8 @@ class FunctionalOpsTest(test.TestCase): target="/job:localhost/replica:0/task:0/cpu:0")[0] + 3.0 with self.cached_session() as sess: - sess.run(variables.global_variables_initializer()) - mul = sess.run(remote_op) + self.evaluate(variables.global_variables_initializer()) + mul = self.evaluate(remote_op) self.assertEqual(mul, 9.0) def testRemoteFunctionGPUCPUStrings(self): @@ -674,9 +695,10 @@ class FunctionalOpsTest(test.TestCase): args=[a], Tout=[dtypes.string], f=_remote_fn, target="/cpu:0") with self.cached_session() as sess: - ret = sess.run(remote_op) + ret = self.evaluate(remote_op) self.assertAllEqual(ret, [b"a"]) + @test_util.run_deprecated_v1 def testRemoteFunctionCrossProcess(self): workers, _ = test_util.create_local_cluster(2, 1) @@ -696,10 +718,11 @@ class FunctionalOpsTest(test.TestCase): target="/job:worker/replica:0/task:1/cpu:0")[0] + 3.0 with session.Session(workers[0].target) as sess: - sess.run(variables.global_variables_initializer()) - mul = sess.run(remote_op) + self.evaluate(variables.global_variables_initializer()) + mul = self.evaluate(remote_op) self.assertEqual(mul, 9) + @test_util.run_deprecated_v1 def testIf(self): @function.Defun(dtypes.float32) @@ -739,6 +762,7 @@ class FunctionalOpsTest(test.TestCase): self.assertAllEqual(Run(sess, 20.), 210.) self.assertAllEqual(Run(sess, 100.), 5050.) + @test_util.run_deprecated_v1 def testWhileLowering(self): def Run(n, fetch_by_name): @@ -766,13 +790,14 @@ class FunctionalOpsTest(test.TestCase): else: fetch = "my_while:1" with self.session(graph=g, use_gpu=use_gpu) as sess: - return sess.run(fetch) + return self.evaluate(fetch) self.assertAllEqual(Run(20., False), 210.) self.assertAllEqual(Run(20., True), 210.) self.assertAllEqual(Run(100., False), 5050.) self.assertAllEqual(Run(100., True), 5050.) + @test_util.run_deprecated_v1 def testWhileError(self): for use_gpu in (True, False): with ops.Graph().as_default() as g: @@ -854,11 +879,11 @@ class FunctionalOpsTest(test.TestCase): result_binary = functional_ops.While( [1.0, 0., 0.], function.Defun(*[dtypes.float32] * 3)(TestCond), TestBinary) - sess.run(variables.global_variables_initializer()) + self.evaluate(variables.global_variables_initializer()) assert len(result_unary) == 2 - self.assertEqual([10.0, 54.0], sess.run(result_unary)) + self.assertEqual([10.0, 54.0], self.evaluate(result_unary)) assert len(result_binary) == 3 - self.assertEqual([10.0, 54.0, 9.0], sess.run(result_binary)) + self.assertEqual([10.0, 54.0, 9.0], self.evaluate(result_binary)) def TestCondCapture(n, *args): del args @@ -889,7 +914,7 @@ class FunctionalOpsTest(test.TestCase): 100, 0, -1, [0.], Body, rewrite_with_while=rewrite_with_while) [0], ] - xvals = sess.run(xs) + xvals = self.evaluate(xs) self.assertAllEqual(210, xvals[0]) self.assertAllEqual(5050, xvals[1]) @@ -919,6 +944,7 @@ class FunctionalOpsTest(test.TestCase): self.assertTrue("TestBody_Cond" in names) self.assertTrue("TestBody_Body" in names) + @test_util.run_deprecated_v1 def testForCapturedInputs(self): v = variables.Variable(1.0) @@ -946,16 +972,16 @@ class FunctionalOpsTest(test.TestCase): result_binary = functional_ops.For( 1, 10, 1, [0., 0.], TestBinary, rewrite_with_while=rewrite_with_while) - sess.run(variables.global_variables_initializer()) + self.evaluate(variables.global_variables_initializer()) assert not result_nullary # The nullary variant doesn't return anything so we can't easily run it. # As a total hack, fetch the operation by name and run it. sess.run(ops.get_default_graph().get_operation_by_name( "While" if rewrite_with_while else "For")) assert len(result_unary) == 1 - self.assertEqual([54.0], sess.run(result_unary)) + self.assertEqual([54.0], self.evaluate(result_unary)) assert len(result_binary) == 2 - self.assertEqual([54.0, 9.0], sess.run(result_binary)) + self.assertEqual([54.0, 9.0], self.evaluate(result_binary)) def _tfMLP(self, xval, wsval, bsval, rewrite_with_while): # On GPU, don't rewrite using a while loop. @@ -974,7 +1000,7 @@ class FunctionalOpsTest(test.TestCase): MLP, rewrite_with_while=rewrite_with_while)[0] - return ret.eval() + return self.evaluate(ret) def _npMLP(self, xval, wsval, bsval): for i in range(wsval.shape[0]): @@ -993,12 +1019,15 @@ class FunctionalOpsTest(test.TestCase): tf_for_ans = self._tfMLP(xval, wsval, bsval, rewrite_with_while) self.assertAllClose(np_ans, tf_for_ans) + @test_util.run_deprecated_v1 def testForMLP(self): self._testForMLP(False) + @test_util.run_deprecated_v1 def testForMLPWhile(self): self._testForMLP(True) + @test_util.run_deprecated_v1 def testForError(self): @function.Defun(dtypes.int32, dtypes.float32) @@ -1021,6 +1050,7 @@ class FunctionalOpsTest(test.TestCase): "For loop body returned 2 arguments. Expected: 1"): functional_ops.For(0, 10, 1, [0.0], ReturnsTooManyArgs)[0].eval() + @test_util.run_deprecated_v1 def testGradient(self): @function.Defun(dtypes.float32) @@ -1038,14 +1068,15 @@ class FunctionalOpsTest(test.TestCase): avals = [Poly(a), Grad(a)] b = constant_op.constant(1.) bvals = [Poly(b), Grad(b)] - self.assertAllEqual(sess.run(avals), [8., 4.]) - self.assertAllEqual(sess.run(bvals), [17., 16.]) + self.assertAllEqual(self.evaluate(avals), [8., 4.]) + self.assertAllEqual(self.evaluate(bvals), [17., 16.]) # TODO(akshayka): Replace `function.Defun` with tf.contrib.eager.defun` in the # below test cases. class PartitionedCallTest(test.TestCase): + @test_util.run_deprecated_v1 def testBasicSingleDevice(self): @function.Defun(*[dtypes.float32] * 2) @@ -1061,6 +1092,7 @@ class PartitionedCallTest(test.TestCase): constant_op.constant(2.)], f=Body)) self.assertEqual(output, 6.) + @test_util.run_deprecated_v1 def testBasicMultiDevice(self): config = config_pb2.ConfigProto(device_count={"CPU": 3}) @@ -1104,6 +1136,7 @@ class PartitionedCallTest(test.TestCase): constant_op.constant(2.)], f=Body)) self.assertEqual(output, 6.) + @test_util.run_deprecated_v1 def testBasicNoDeviceAnnotations(self): @function.Defun(*[dtypes.float32] * 2) @@ -1118,6 +1151,7 @@ class PartitionedCallTest(test.TestCase): constant_op.constant(2.)], f=Body)) self.assertEqual(output, 6.) + @test_util.run_deprecated_v1 def testShardsRunOnRequestedDevices(self): config = config_pb2.ConfigProto(device_count={"CPU": 4}) @@ -1147,6 +1181,7 @@ class PartitionedCallTest(test.TestCase): self.assertIn(compat.as_bytes("CPU:1"), outputs[1]) self.assertIn(compat.as_bytes("CPU:2"), outputs[2]) + @test_util.run_deprecated_v1 def testAssignAddResourceVariable(self): v = resource_variable_ops.ResourceVariable(1.0) @@ -1190,14 +1225,15 @@ class PartitionedCallTest(test.TestCase): allow_soft_placement=False, log_device_placement=True, device_count={"CPU": 2})) as sess: - sess.run(variables.global_variables_initializer()) - expected = sess.run(sum_gather()) + self.evaluate(variables.global_variables_initializer()) + expected = self.evaluate(sum_gather()) result = sess.run( functional_ops.partitioned_call( args=defined.captured_inputs, f=defined)) self.assertAllEqual(expected, result) # Use an invalid executor name to test the plumbing of the executor_type attr. + @test_util.run_deprecated_v1 def testExecutorTypeAttrExecutorNotFound(self): @function.Defun(dtypes.int32) def AddFive(x): diff --git a/tensorflow/python/kernel_tests/gather_nd_op_test.py b/tensorflow/python/kernel_tests/gather_nd_op_test.py index ee761435d84..320ffc9674b 100644 --- a/tensorflow/python/kernel_tests/gather_nd_op_test.py +++ b/tensorflow/python/kernel_tests/gather_nd_op_test.py @@ -27,6 +27,7 @@ from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops from tensorflow.python.framework import tensor_shape +from tensorflow.python.framework import test_util from tensorflow.python.ops import array_ops from tensorflow.python.ops import gradients_impl from tensorflow.python.ops import variables @@ -40,7 +41,7 @@ class GatherNdTest(test.TestCase): params = constant_op.constant(np.array([8, 1, 2, 3, 7, 5], dtype=dtype)) indices = constant_op.constant([[4], [4], [0]]) gather_nd_t = array_ops.gather_nd(params, indices) - gather_nd_val = gather_nd_t.eval() + gather_nd_val = self.evaluate(gather_nd_t) self.assertAllEqual(np.array([7, 7, 8], dtype=dtype), gather_nd_val) self.assertEqual([3], gather_nd_t.get_shape()) @@ -54,26 +55,27 @@ class GatherNdTest(test.TestCase): self._testSimpleDtype(np.complex128) self._testSimpleDtype("|S") # byte strings in python2 + 3 + @test_util.run_deprecated_v1 def testEmptyIndicesAndParamsOKButJustEmptyParamsFails(self): with self.session(use_gpu=True): params = np.ones((3, 3), dtype=np.float32) indices_empty = np.empty((0, 2), dtype=np.int32) gather_nd_ok_t = array_ops.gather_nd(params, indices_empty) - gather_nd_ok_val = gather_nd_ok_t.eval() + gather_nd_ok_val = self.evaluate(gather_nd_ok_t) self.assertEqual([0], gather_nd_ok_t.get_shape()) self.assertAllClose(np.empty((0,), dtype=np.float32), gather_nd_ok_val) indices_empty = np.empty((0, 1), dtype=np.int32) gather_nd_ok_t = array_ops.gather_nd(params, indices_empty) - gather_nd_ok_val = gather_nd_ok_t.eval() + gather_nd_ok_val = self.evaluate(gather_nd_ok_t) self.assertEqual([0, 3], gather_nd_ok_t.get_shape()) self.assertAllClose(np.empty((0, 3), dtype=np.float32), gather_nd_ok_val) params_empty = np.empty((0, 3), dtype=np.float32) indices_empty = np.empty((0, 2), dtype=np.int32) gather_nd_ok_t = array_ops.gather_nd(params_empty, indices_empty) - gather_nd_ok_val = gather_nd_ok_t.eval() + gather_nd_ok_val = self.evaluate(gather_nd_ok_t) self.assertEqual([0], gather_nd_ok_t.get_shape()) self.assertAllClose(np.empty((0,), dtype=np.float32), gather_nd_ok_val) @@ -82,7 +84,7 @@ class GatherNdTest(test.TestCase): gather_nd_break_t = array_ops.gather_nd(params_empty, indices_nonempty) with self.assertRaisesOpError( r"Requested more than 0 entries, but params is empty."): - gather_nd_break_t.eval() + self.evaluate(gather_nd_break_t) self.assertAllClose(np.empty((0,), dtype=np.float32), gather_nd_ok_val) def testIndexScalar(self): @@ -91,7 +93,7 @@ class GatherNdTest(test.TestCase): [[-8, -1, -2, -3, -7, -5], [8, 1, 2, 3, 7, 5]], dtype=np.float32).T indices = constant_op.constant([4, 1]) gather_nd_t = array_ops.gather_nd(params, indices) - gather_nd_val = gather_nd_t.eval() + gather_nd_val = self.evaluate(gather_nd_t) self.assertEqual([], gather_nd_t.get_shape()) self.assertAllEqual(np.array(7), gather_nd_val) @@ -101,7 +103,7 @@ class GatherNdTest(test.TestCase): [[-8, -1, -2, -3, -7, -5], [8, 1, 2, 3, 7, 5]], dtype=np.float32).T indices = constant_op.constant([4]) gather_nd_t = array_ops.gather_nd(params, indices) - gather_nd_val = gather_nd_t.eval() + gather_nd_val = self.evaluate(gather_nd_t) self.assertEqual([2], gather_nd_t.get_shape()) self.assertAllEqual(np.array([-7, 7]), gather_nd_val) @@ -111,7 +113,7 @@ class GatherNdTest(test.TestCase): [[-8, -1, -2, -3, -7, -5], [8, 1, 2, 3, 7, 5]], dtype=np.float32).T indices = constant_op.constant([[4], [4], [0]]) gather_nd_t = array_ops.gather_nd(params, indices) - gather_nd_val = gather_nd_t.eval() + gather_nd_val = self.evaluate(gather_nd_t) self.assertEqual([3, 2], gather_nd_t.get_shape()) self.assertAllEqual(np.array([[-7, 7], [-7, 7], [-8, 8]]), gather_nd_val) @@ -125,7 +127,7 @@ class GatherNdTest(test.TestCase): params_t = constant_op.constant(params) indices = constant_op.constant([[4], [4], [0]]) gather_nd_t = array_ops.gather_nd(params_t, indices) - gather_nd_val = gather_nd_t.eval() + gather_nd_val = self.evaluate(gather_nd_t) self.assertEqual([3, 2, 2], gather_nd_t.get_shape()) self.assertAllEqual(params[[4, 4, 0]], gather_nd_val) @@ -140,7 +142,7 @@ class GatherNdTest(test.TestCase): indices = constant_op.constant( [[], []], dtype=dtypes.int32) # Size (2, 0) gather_nd_t = array_ops.gather_nd(params_t, indices) - gather_nd_val = gather_nd_t.eval() + gather_nd_val = self.evaluate(gather_nd_t) self.assertEqual([2, 6, 2, 2], gather_nd_t.get_shape()) self.assertAllEqual( @@ -156,7 +158,7 @@ class GatherNdTest(test.TestCase): params_t = constant_op.constant(params) indices = constant_op.constant([[[3], [2], [1]], [[4], [4], [0]]]) gather_nd_t = array_ops.gather_nd(params_t, indices) - gather_nd_val = gather_nd_t.eval() + gather_nd_val = self.evaluate(gather_nd_t) self.assertEqual([2, 3, 2, 2], gather_nd_t.get_shape()) self.assertAllEqual(params[[3, 2, 1, 4, 4, 0]].reshape(2, 3, 2, 2), @@ -168,7 +170,7 @@ class GatherNdTest(test.TestCase): params = np.random.rand(*shape) indices = np.vstack([np.random.randint(0, s, size=2000) for s in shape]).T gather_nd_t = array_ops.gather_nd(params, indices) - gather_nd_val = gather_nd_t.eval() + gather_nd_val = self.evaluate(gather_nd_t) expected = params[tuple(indices.T)] self.assertAllEqual(expected, gather_nd_val) @@ -181,7 +183,7 @@ class GatherNdTest(test.TestCase): indices = np.vstack([np.random.randint(0, s, size=2000) for s in shape]).T indices_reshaped = indices.reshape([10, 10, 20, 5]) gather_nd_t = array_ops.gather_nd(params, indices_reshaped) - gather_nd_val = gather_nd_t.eval() + gather_nd_val = self.evaluate(gather_nd_t) expected = params[tuple(indices.T)] self.assertAllEqual(expected.reshape([10, 10, 20]), gather_nd_val) @@ -190,6 +192,7 @@ class GatherNdTest(test.TestCase): def assertIndexedSlices(self, t): self.assertIsInstance(t, ops.IndexedSlices) + @test_util.run_deprecated_v1 def testUnknownIndices(self): params = constant_op.constant([[0, 1, 2]]) indices = array_ops.placeholder(dtypes.int32) @@ -198,6 +201,7 @@ class GatherNdTest(test.TestCase): self.assertEqual(None, shape.ndims) self.assertEqual(None, tensor_shape.dimension_value(shape[0])) + @test_util.run_deprecated_v1 def testBadIndicesCPU(self): with self.session(use_gpu=False): params = [0, 1, 2] @@ -205,7 +209,7 @@ class GatherNdTest(test.TestCase): gather_nd = array_ops.gather_nd(params, indices) with self.assertRaisesOpError( r"indices\[0,1\] = \[7\] does not index into param shape \[3\]"): - gather_nd.eval() + self.evaluate(gather_nd) def _disabledTestBadIndicesGPU(self): # TODO disabled due to different behavior on GPU and CPU @@ -218,8 +222,9 @@ class GatherNdTest(test.TestCase): gather_nd = array_ops.gather_nd(params, indices) with self.assertRaisesOpError( r"indices\[0,1\] = \[7\] does not index into param shape \[3\]"): - gather_nd.eval() + self.evaluate(gather_nd) + @test_util.run_deprecated_v1 def testBadIndicesWithSlicesCPU(self): with self.session(use_gpu=False): params = [[0, 1, 2]] @@ -227,7 +232,7 @@ class GatherNdTest(test.TestCase): gather_nd = array_ops.gather_nd(params, indices) with self.assertRaisesOpError( r"indices\[0,2\] = \[1\] does not index into param shape \[1,3\]"): - gather_nd.eval() + self.evaluate(gather_nd) def _disabledTestBadIndicesWithSlicesGPU(self): # TODO disabled due to different behavior on GPU and CPU @@ -240,8 +245,9 @@ class GatherNdTest(test.TestCase): gather_nd = array_ops.gather_nd(params, indices) with self.assertRaisesOpError( r"indices\[0,2\] = \[1\] does not index into param shape \[1,3\]"): - gather_nd.eval() + self.evaluate(gather_nd) + @test_util.run_deprecated_v1 def testGradientsRank2Elements(self): indices = constant_op.constant([[0, 0], [1, 1]], dtype=dtypes.int32) inputs = constant_op.constant([[1, 2], [3, 4]], dtype=dtypes.float64) @@ -251,8 +257,9 @@ class GatherNdTest(test.TestCase): grads = gradients_impl.gradients([outputs], [inputs], [grad_vals])[0] expected_grads = np.array([[1, 0], [0, 2]], dtype=np.float64) with self.session(use_gpu=True): - assert np.array_equal(expected_grads, grads.eval()) + assert np.array_equal(expected_grads, self.evaluate(grads)) + @test_util.run_deprecated_v1 def testGradientsRank2Slices(self): indices = constant_op.constant([[1], [0]], dtype=dtypes.int32) inputs = constant_op.constant([[1, 2], [3, 4]], dtype=dtypes.float64) @@ -265,6 +272,7 @@ class GatherNdTest(test.TestCase): self.assertIndexedSlices(grads) self.assertAllEqual(expected_grads, ops.convert_to_tensor(grads).eval()) + @test_util.run_deprecated_v1 def testGradientsRank3Elements(self): indices = constant_op.constant( [[[0, 1], [1, 0]], [[0, 0], [1, 1]]], dtype=dtypes.int32) @@ -278,8 +286,9 @@ class GatherNdTest(test.TestCase): expected_grads = np.array( [[[5, 6], [1, 2]], [[3, 4], [7, 8]]], dtype=np.float64) with self.session(use_gpu=True): - self.assertAllEqual(expected_grads, grads.eval()) + self.assertAllEqual(expected_grads, self.evaluate(grads)) + @test_util.run_deprecated_v1 def testGradientsRank7Elements(self): # Shape [1,1,2,1,1,2,2] indices = constant_op.constant( @@ -307,8 +316,9 @@ class GatherNdTest(test.TestCase): [[[[3, 4], [7, 8]]]] ]]], dtype=np.float64) with self.session(use_gpu=True): - self.assertAllEqual(expected_grads, grads.eval()) + self.assertAllEqual(expected_grads, self.evaluate(grads)) + @test_util.run_deprecated_v1 def testGradientsInt64Indices(self): indices = constant_op.constant( [[[0, 1], [1, 0]], [[0, 0], [1, 1]]], dtype=dtypes.int64) @@ -322,8 +332,9 @@ class GatherNdTest(test.TestCase): expected_grads = np.array( [[[5, 6], [1, 2]], [[3, 4], [7, 8]]], dtype=np.float64) with self.session(use_gpu=True): - self.assertAllEqual(expected_grads, grads.eval()) + self.assertAllEqual(expected_grads, self.evaluate(grads)) + @test_util.run_deprecated_v1 def testGradientsRank2SlicesWithEmptySpace(self): indices = constant_op.constant([[2], [0], [5]], dtype=dtypes.int32) inputs = constant_op.constant( @@ -361,10 +372,10 @@ class GatherNdOpBenchmark(test.Benchmark): gather_op = array_ops.gather_nd(t_params, t_indices) variables.global_variables_initializer().run() for _ in range(10): - gather_op.eval() + self.evaluate(gather_op) t1 = time.time() for _ in range(1000): - gather_op.eval() + self.evaluate(gather_op) t2 = time.time() self.report_benchmark(iters=1000, wall_time=(t2 - t1) / 1000.0) diff --git a/tensorflow/python/kernel_tests/gather_op_test.py b/tensorflow/python/kernel_tests/gather_op_test.py index bdafc52ab5e..fc86068c3fc 100644 --- a/tensorflow/python/kernel_tests/gather_op_test.py +++ b/tensorflow/python/kernel_tests/gather_op_test.py @@ -23,6 +23,7 @@ import numpy as np from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops +from tensorflow.python.framework import test_util from tensorflow.python.ops import array_ops from tensorflow.python.ops import gradients_impl from tensorflow.python.platform import test @@ -50,7 +51,7 @@ class GatherTest(test.TestCase): params = constant_op.constant(params_np) indices_tf = constant_op.constant(indices) gather_t = array_ops.gather(params, indices_tf) - gather_val = gather_t.eval() + gather_val = self.evaluate(gather_t) np_val = params_np[indices] self.assertAllEqual(np_val, gather_val) self.assertEqual(np_val.shape, gather_t.get_shape()) @@ -65,7 +66,7 @@ class GatherTest(test.TestCase): params = constant_op.constant(params_np) indices = constant_op.constant(2) gather_t = array_ops.gather(params, indices, axis=axis) - gather_val = gather_t.eval() + gather_val = self.evaluate(gather_t) self.assertAllEqual(np.take(params_np, 2, axis=axis), gather_val) expected_shape = data.shape[:axis] + data.shape[axis + 1:] self.assertEqual(expected_shape, gather_t.get_shape()) @@ -81,12 +82,13 @@ class GatherTest(test.TestCase): # The indices must be in bounds for any axis. indices = constant_op.constant([0, 1, 0, 2]) gather_t = array_ops.gather(params, indices, axis=axis) - gather_val = gather_t.eval() + gather_val = self.evaluate(gather_t) self.assertAllEqual(np.take(params_np, [0, 1, 0, 2], axis=axis), gather_val) expected_shape = data.shape[:axis] + (4,) + data.shape[axis + 1:] self.assertEqual(expected_shape, gather_t.get_shape()) + @test_util.run_deprecated_v1 def testHigherRank(self): # We check that scalar and empty indices shapes work as well shape = (2, 1, 3, 2) @@ -142,9 +144,13 @@ class GatherTest(test.TestCase): source_slice = ((slice(None),) * outer_dims + (source_index,) + (slice(None),) * inner_dims) correct_params_grad[dest_slice] += gather_grad[source_slice] - self.assertAllClose(correct_params_grad, params_grad.eval(), - atol=2e-6, rtol=2e-6) + self.assertAllClose( + correct_params_grad, + self.evaluate(params_grad), + atol=2e-6, + rtol=2e-6) + @test_util.run_deprecated_v1 def testString(self): params = np.array([[b"asdf", b"zxcv"], [b"qwer", b"uiop"]]) with self.cached_session(): @@ -153,6 +159,7 @@ class GatherTest(test.TestCase): self.assertAllEqual([b"asdf", b"qwer"], array_ops.gather(params, 0, axis=1).eval()) + @test_util.run_deprecated_v1 def testUInt32AndUInt64(self): for unsigned_type in (dtypes.uint32, dtypes.uint64): params = self._buildParams( @@ -162,12 +169,14 @@ class GatherTest(test.TestCase): array_ops.gather(params, 1, axis=0).eval()) self.assertAllEqual([1, 7], array_ops.gather(params, 0, axis=1).eval()) + @test_util.run_deprecated_v1 def testUnknownIndices(self): params = constant_op.constant([[0, 1, 2]]) indices = array_ops.placeholder(dtypes.int32) gather_t = array_ops.gather(params, indices) self.assertEqual(None, gather_t.get_shape()) + @test_util.run_deprecated_v1 def testUnknownAxis(self): params = constant_op.constant([[0, 1, 2]]) indices = constant_op.constant([[0, 0], [0, 0]]) @@ -201,6 +210,7 @@ class GatherTest(test.TestCase): with self.assertRaisesOpError(r"indices\[0,0\] = 7 is not in \[0, 3\)"): array_ops.gather(params, [[7]], axis=1).eval() + @test_util.run_deprecated_v1 def testBadAxis(self): with self.session(use_gpu=True): params = [0, 1, 2] @@ -217,6 +227,7 @@ class GatherTest(test.TestCase): array_ops.gather(params_ph, indices, axis=bad_axis).eval( feed_dict={params_ph: params}) + @test_util.run_deprecated_v1 def testEmptySlices(self): with self.session(use_gpu=True): for dtype in _TEST_TYPES: diff --git a/tensorflow/python/kernel_tests/gradient_correctness_test.py b/tensorflow/python/kernel_tests/gradient_correctness_test.py index 291a69ebac6..0148de5047a 100644 --- a/tensorflow/python/kernel_tests/gradient_correctness_test.py +++ b/tensorflow/python/kernel_tests/gradient_correctness_test.py @@ -22,6 +22,7 @@ import numpy as np from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes +from tensorflow.python.framework import test_util from tensorflow.python.ops import gradients_impl from tensorflow.python.ops import math_ops from tensorflow.python.platform import test @@ -29,37 +30,42 @@ from tensorflow.python.platform import test class GradientCorrectnessTest(test.TestCase): + @test_util.run_deprecated_v1 def testMultipleOutputChainedGradients(self): with self.cached_session() as sess: x = constant_op.constant(1.0, dtype=dtypes.float32) yexp = math_ops.exp(x) yexplog = math_ops.log(yexp) grads = gradients_impl.gradients([yexp, yexplog], [x]) - grad_vals = sess.run(grads) + grad_vals = self.evaluate(grads) exp1_plus_one = (1.0 + np.exp(1.0)).astype(np.float32) # [dexp(x)/dx + d(log(exp(x)))/dx] @ x=1 == exp(1) + 1 self.assertAllClose(grad_vals[0], exp1_plus_one) + @test_util.run_deprecated_v1 def testIdentityGradient(self): x = constant_op.constant(3.) dx_dx, = gradients_impl.gradients(x, x) with self.cached_session() as sess: - self.assertAllClose(1., sess.run(dx_dx)) + self.assertAllClose(1., self.evaluate(dx_dx)) + @test_util.run_deprecated_v1 def testIntegerIdentityGradient(self): x = constant_op.constant(3) dx_dx, = gradients_impl.gradients(x, x) with self.cached_session() as sess: - self.assertAllClose(1, sess.run(dx_dx)) + self.assertAllClose(1, self.evaluate(dx_dx)) + @test_util.run_deprecated_v1 def testGradientWithIntegerPath(self): x = constant_op.constant([3.9, 4.1]) k = math_ops.to_float(math_ops.to_int32(x)) y = x * k dy_dx, = gradients_impl.gradients(y, x) with self.cached_session() as sess: - self.assertAllClose([3., 4.], sess.run(dy_dx)) + self.assertAllClose([3., 4.], self.evaluate(dy_dx)) + @test_util.run_deprecated_v1 def testNoIntegerGradient1(self): x = constant_op.constant([3.9, 4.1]) k = math_ops.to_float(math_ops.to_int32(x)) @@ -67,6 +73,7 @@ class GradientCorrectnessTest(test.TestCase): dy_dx, = gradients_impl.gradients(y, x) self.assertIsNone(dy_dx) + @test_util.run_deprecated_v1 def testNoIntegerGradient2(self): k = constant_op.constant([3, 4]) x = math_ops.to_float(k) @@ -74,18 +81,21 @@ class GradientCorrectnessTest(test.TestCase): dy_dk, = gradients_impl.gradients(y, k) self.assertIsNone(dy_dk) + @test_util.run_deprecated_v1 def testNoIntegerGradient3(self): k = constant_op.constant([3, 4]) m = k * k dm_dk, = gradients_impl.gradients(m, k) self.assertIsNone(dm_dk) + @test_util.run_deprecated_v1 def testNoIntegerGradient4(self): k = constant_op.constant([3, 4]) m = k * k * k dm_dk, = gradients_impl.gradients(m, k) self.assertIsNone(dm_dk) + @test_util.run_deprecated_v1 def testNoIntegerGradient5(self): k = constant_op.constant([3, 4]) m = k * k @@ -93,6 +103,7 @@ class GradientCorrectnessTest(test.TestCase): dn_dk, = gradients_impl.gradients(n, k) self.assertIsNone(dn_dk) + @test_util.run_deprecated_v1 def testNoIntegerGradient6(self): k = constant_op.constant(3) x = math_ops.to_float(k) diff --git a/tensorflow/python/kernel_tests/huge_slice_op_test.py b/tensorflow/python/kernel_tests/huge_slice_op_test.py index 8646d74c96f..4074946350a 100644 --- a/tensorflow/python/kernel_tests/huge_slice_op_test.py +++ b/tensorflow/python/kernel_tests/huge_slice_op_test.py @@ -33,11 +33,11 @@ class SliceTest(test.TestCase): a_large = array_ops.tile( constant_op.constant(np.array([False, True] * 4)), [2**29 + 3]) slice_t = array_ops.slice(a_large, np.asarray([3]).astype(np.int64), [3]) - slice_val = slice_t.eval() + slice_val = self.evaluate(slice_t) self.assertAllEqual([True, False, True], slice_val) slice_t = array_ops.slice( a_large, constant_op.constant([long(2)**32 + 3], dtype=dtypes.int64), [3]) - slice_val = slice_t.eval() + slice_val = self.evaluate(slice_t) self.assertAllEqual([True, False, True], slice_val) diff --git a/tensorflow/python/kernel_tests/identity_n_op_py_test.py b/tensorflow/python/kernel_tests/identity_n_op_py_test.py index 518733cd8e9..a1110d640f0 100644 --- a/tensorflow/python/kernel_tests/identity_n_op_py_test.py +++ b/tensorflow/python/kernel_tests/identity_n_op_py_test.py @@ -21,12 +21,14 @@ from __future__ import print_function import numpy as np from tensorflow.python.framework import constant_op +from tensorflow.python.framework import test_util from tensorflow.python.ops import array_ops from tensorflow.python.platform import test class IdentityNOpTest(test.TestCase): + @test_util.run_deprecated_v1 def testInt32String_6(self): with self.cached_session() as sess: [value0, value1] = sess.run( @@ -36,6 +38,7 @@ class IdentityNOpTest(test.TestCase): self.assertAllEqual( np.array([b"a", b"b", b"C", b"d", b"E", b"f", b"g"]), value1) + @test_util.run_deprecated_v1 def testInt32_shapes(self): with self.cached_session() as sess: inp0 = constant_op.constant([10, 20, 30, 40, 50, 60], shape=[2, 3]) @@ -50,6 +53,7 @@ class IdentityNOpTest(test.TestCase): np.array([[1, 2, 3], [4, 5, 6], [7, 8, 9], [10, 11, 12], [13, 14, 15]]), value2) + @test_util.run_deprecated_v1 def testString(self): source = [b"A", b"b", b"C", b"d", b"E", b"f"] with self.cached_session() as sess: diff --git a/tensorflow/python/kernel_tests/identity_op_py_test.py b/tensorflow/python/kernel_tests/identity_op_py_test.py index 88ea10c22a3..1a6794e896f 100644 --- a/tensorflow/python/kernel_tests/identity_op_py_test.py +++ b/tensorflow/python/kernel_tests/identity_op_py_test.py @@ -22,6 +22,7 @@ import numpy as np from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes +from tensorflow.python.framework import test_util from tensorflow.python.ops import array_ops from tensorflow.python.ops import gen_array_ops from tensorflow.python.ops import variables @@ -30,17 +31,20 @@ from tensorflow.python.platform import test class IdentityOpTest(test.TestCase): + @test_util.run_deprecated_v1 def testInt32_6(self): with self.cached_session(): value = array_ops.identity([1, 2, 3, 4, 5, 6]).eval() self.assertAllEqual(np.array([1, 2, 3, 4, 5, 6]), value) + @test_util.run_deprecated_v1 def testInt32_2_3(self): with self.cached_session(): inp = constant_op.constant([10, 20, 30, 40, 50, 60], shape=[2, 3]) value = array_ops.identity(inp).eval() self.assertAllEqual(np.array([[10, 20, 30], [40, 50, 60]]), value) + @test_util.run_deprecated_v1 def testString(self): source = [b"A", b"b", b"C", b"d", b"E", b"f"] with self.cached_session(): @@ -58,6 +62,7 @@ class IdentityOpTest(test.TestCase): self.assertEquals(shape, array_ops.identity(np.array(array_2x3)).get_shape()) + @test_util.run_deprecated_v1 def testRefIdentityShape(self): with self.cached_session(): shape = [2, 3] diff --git a/tensorflow/python/kernel_tests/in_topk_op_test.py b/tensorflow/python/kernel_tests/in_topk_op_test.py index 6fdb497bc6f..507822b3142 100644 --- a/tensorflow/python/kernel_tests/in_topk_op_test.py +++ b/tensorflow/python/kernel_tests/in_topk_op_test.py @@ -32,7 +32,7 @@ class InTopKTest(test.TestCase): np_ans = np.array(expected) with self.cached_session(): precision = nn_ops.in_top_k(predictions, target, k) - out = precision.eval() + out = self.evaluate(precision) self.assertAllClose(np_ans, out) self.assertShapeEqual(np_ans, precision) @@ -77,7 +77,7 @@ class InTopKTest(test.TestCase): np_ans = np.array([False, True]) with self.cached_session(): precision = nn_ops.in_top_k(predictions, target, k) - out = precision.eval() + out = self.evaluate(precision) self.assertAllClose(np_ans, out) self.assertShapeEqual(np_ans, precision) diff --git a/tensorflow/python/kernel_tests/init_ops_test.py b/tensorflow/python/kernel_tests/init_ops_test.py index 70bfbf8544a..09b9944baa1 100644 --- a/tensorflow/python/kernel_tests/init_ops_test.py +++ b/tensorflow/python/kernel_tests/init_ops_test.py @@ -25,6 +25,7 @@ from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops from tensorflow.python.framework import random_seed +from tensorflow.python.framework import test_util from tensorflow.python.layers import convolutional from tensorflow.python.ops import array_ops from tensorflow.python.ops import init_ops @@ -106,6 +107,7 @@ def _init_sampler(tc, init, num): class ConstantInitializersTest(test.TestCase): + @test_util.run_deprecated_v1 def testZerosInitializer(self): with self.session(use_gpu=True): shape = [2, 3] @@ -114,6 +116,7 @@ class ConstantInitializersTest(test.TestCase): x.initializer.run() self.assertAllEqual(x.eval(), np.zeros(shape)) + @test_util.run_deprecated_v1 def testOnesInitializer(self): with self.session(use_gpu=True): shape = [2, 3] @@ -122,6 +125,7 @@ class ConstantInitializersTest(test.TestCase): x.initializer.run() self.assertAllEqual(x.eval(), np.ones(shape)) + @test_util.run_deprecated_v1 def testConstantZeroInitializer(self): with self.session(use_gpu=True): shape = [2, 3] @@ -130,6 +134,7 @@ class ConstantInitializersTest(test.TestCase): x.initializer.run() self.assertAllEqual(x.eval(), np.zeros(shape)) + @test_util.run_deprecated_v1 def testConstantOneInitializer(self): with self.session(use_gpu=True): shape = [2, 3] @@ -138,6 +143,7 @@ class ConstantInitializersTest(test.TestCase): x.initializer.run() self.assertAllEqual(x.eval(), np.ones(shape)) + @test_util.run_deprecated_v1 def testConstantIntInitializer(self): with self.session(use_gpu=True): shape = [2, 3] @@ -150,6 +156,7 @@ class ConstantInitializersTest(test.TestCase): self.assertEqual(x.dtype.base_dtype, dtypes.int32) self.assertAllEqual(x.eval(), 7 * np.ones(shape, dtype=np.int32)) + @test_util.run_deprecated_v1 def testConstantTupleInitializer(self): with self.session(use_gpu=True): shape = [3] @@ -173,6 +180,7 @@ class ConstantInitializersTest(test.TestCase): for a, e in zip(actual, expected): self.assertEqual(a, e) + @test_util.run_deprecated_v1 def testNDimConstantInitializer(self): value = [0, 1, 2, 3, 4, 5] shape = [2, 3] @@ -199,6 +207,7 @@ class ConstantInitializersTest(test.TestCase): e = expected[i] if i < len(expected) else expected[-1] self.assertEqual(a, e) + @test_util.run_deprecated_v1 def testNDimConstantInitializerLessValues(self): value = [0, 1, 2, 3, 4, 5] shape = [2, 4] @@ -222,6 +231,7 @@ class ConstantInitializersTest(test.TestCase): shape=shape, initializer=init) + @test_util.run_deprecated_v1 def testNDimConstantInitializerMoreValues(self): value = [0, 1, 2, 3, 4, 5, 6, 7] shape = [2, 3] @@ -243,18 +253,21 @@ class ConstantInitializersTest(test.TestCase): class RandomNormalInitializationTest(test.TestCase): + @test_util.run_deprecated_v1 def testInitializerIdentical(self): for dtype in [dtypes.float32, dtypes.float64]: init1 = init_ops.random_normal_initializer(0.0, 1.0, seed=1, dtype=dtype) init2 = init_ops.random_normal_initializer(0.0, 1.0, seed=1, dtype=dtype) self.assertTrue(identicaltest(self, init1, init2)) + @test_util.run_deprecated_v1 def testInitializerDifferent(self): for dtype in [dtypes.float32, dtypes.float64]: init1 = init_ops.random_normal_initializer(0.0, 1.0, seed=1, dtype=dtype) init2 = init_ops.random_normal_initializer(0.0, 1.0, seed=2, dtype=dtype) self.assertFalse(identicaltest(self, init1, init2)) + @test_util.run_deprecated_v1 def testDuplicatedInitializer(self): init = init_ops.random_normal_initializer(0.0, 1.0) self.assertFalse(duplicated_initializer(self, init, 1)) @@ -270,6 +283,7 @@ class RandomNormalInitializationTest(test.TestCase): class TruncatedNormalInitializationTest(test.TestCase): + @test_util.run_deprecated_v1 def testInitializerIdentical(self): for dtype in [dtypes.float32, dtypes.float64]: init1 = init_ops.truncated_normal_initializer( @@ -278,6 +292,7 @@ class TruncatedNormalInitializationTest(test.TestCase): 0.0, 1.0, seed=1, dtype=dtype) self.assertTrue(identicaltest(self, init1, init2)) + @test_util.run_deprecated_v1 def testInitializerDifferent(self): for dtype in [dtypes.float32, dtypes.float64]: init1 = init_ops.truncated_normal_initializer( @@ -286,6 +301,7 @@ class TruncatedNormalInitializationTest(test.TestCase): 0.0, 1.0, seed=2, dtype=dtype) self.assertFalse(identicaltest(self, init1, init2)) + @test_util.run_deprecated_v1 def testDuplicatedInitializer(self): init = init_ops.truncated_normal_initializer(0.0, 1.0) self.assertFalse(duplicated_initializer(self, init, 1)) @@ -301,18 +317,21 @@ class TruncatedNormalInitializationTest(test.TestCase): class RandomUniformInitializationTest(test.TestCase): + @test_util.run_deprecated_v1 def testInitializerIdentical(self): for dtype in [dtypes.float32, dtypes.float64, dtypes.int64]: init1 = init_ops.random_uniform_initializer(0, 7, seed=1, dtype=dtype) init2 = init_ops.random_uniform_initializer(0, 7, seed=1, dtype=dtype) self.assertTrue(identicaltest(self, init1, init2)) + @test_util.run_deprecated_v1 def testInitializerDifferent(self): for dtype in [dtypes.float32, dtypes.float64, dtypes.int32, dtypes.int64]: init1 = init_ops.random_uniform_initializer(0, 7, seed=1, dtype=dtype) init2 = init_ops.random_uniform_initializer(0, 7, seed=2, dtype=dtype) self.assertFalse(identicaltest(self, init1, init2)) + @test_util.run_deprecated_v1 def testDuplicatedInitializer(self): init = init_ops.random_uniform_initializer(0.0, 1.0) self.assertFalse(duplicated_initializer(self, init, 1)) @@ -320,6 +339,7 @@ class RandomUniformInitializationTest(test.TestCase): class UniformUnitScalingInitializationTest(test.TestCase): + @test_util.run_deprecated_v1 def testInitializerIdentical(self): for dtype in [dtypes.float32, dtypes.float64]: init1 = init_ops.uniform_unit_scaling_initializer(seed=1, dtype=dtype) @@ -331,6 +351,7 @@ class UniformUnitScalingInitializationTest(test.TestCase): 1.5, seed=1, dtype=dtype) self.assertTrue(identicaltest(self, init3, init4)) + @test_util.run_deprecated_v1 def testInitializerDifferent(self): for dtype in [dtypes.float32, dtypes.float64]: init1 = init_ops.uniform_unit_scaling_initializer(seed=1, dtype=dtype) @@ -341,6 +362,7 @@ class UniformUnitScalingInitializationTest(test.TestCase): self.assertFalse(identicaltest(self, init1, init3)) self.assertFalse(identicaltest(self, init2, init3)) + @test_util.run_deprecated_v1 def testZeroSize(self): shape = [0, 2] with self.cached_session(): @@ -349,8 +371,9 @@ class UniformUnitScalingInitializationTest(test.TestCase): shape=shape, initializer=init_ops.uniform_unit_scaling_initializer()) variables.global_variables_initializer().run() - self.assertAllEqual(shape, x.eval().shape) + self.assertAllEqual(shape, self.evaluate(x).shape) + @test_util.run_deprecated_v1 def testDuplicatedInitializer(self): init = init_ops.uniform_unit_scaling_initializer() self.assertFalse(duplicated_initializer(self, init, 1)) @@ -364,6 +387,7 @@ class UniformUnitScalingInitializationTest(test.TestCase): class VarianceScalingInitializationTest(test.TestCase): + @test_util.run_deprecated_v1 def testTruncatedNormalDistribution(self): shape = [100, 100] expect_mean = 0. @@ -381,6 +405,7 @@ class VarianceScalingInitializationTest(test.TestCase): self.assertNear(np.mean(x), expect_mean, err=1e-2) self.assertNear(np.var(x), expect_var, err=1e-2) + @test_util.run_deprecated_v1 def testNormalDistribution(self): shape = [100, 100] expect_mean = 0. @@ -397,6 +422,7 @@ class VarianceScalingInitializationTest(test.TestCase): self.assertNear(np.mean(x), expect_mean, err=1e-2) self.assertNear(np.var(x), expect_var, err=1e-2) + @test_util.run_deprecated_v1 def testUntruncatedNormalDistribution(self): shape = [100, 100] expect_mean = 0. @@ -414,6 +440,7 @@ class VarianceScalingInitializationTest(test.TestCase): self.assertNear(np.mean(x), expect_mean, err=1e-2) self.assertNear(np.var(x), expect_var, err=1e-2) + @test_util.run_deprecated_v1 def testUniformDistribution(self): shape = [100, 100] expect_mean = 0. @@ -435,7 +462,7 @@ class RangeTest(test.TestCase): tf_ans = math_ops.range(start, limit, delta, name="range") self.assertEqual([len(np.arange(start, limit, delta))], tf_ans.get_shape()) - return tf_ans.eval() + return self.evaluate(tf_ans) def testBasic(self): self.assertTrue( @@ -449,6 +476,7 @@ class RangeTest(test.TestCase): self._Range(100, 500, 100), np.array([100, 200, 300, 400]))) self.assertEqual(math_ops.range(0, 5, 1).dtype, dtypes.int32) + @test_util.run_deprecated_v1 def testLimitOnly(self): with self.session(use_gpu=True): self.assertAllEqual(np.arange(5), math_ops.range(5).eval()) @@ -524,7 +552,7 @@ class LinSpaceTest(test.TestCase): with self.session(graph=graph, force_gpu=self.force_gpu): tf_ans = math_ops.linspace(start, stop, num, name="linspace") self.assertEqual([num], tf_ans.get_shape()) - return tf_ans.eval() + return self.evaluate(tf_ans) def testPositive(self): for self.force_gpu in self._gpu_modes(): @@ -583,18 +611,21 @@ class DeviceTest(test.TestCase): class OrthogonalInitializerTest(test.TestCase): + @test_util.run_deprecated_v1 def testInitializerIdentical(self): for dtype in [dtypes.float32, dtypes.float64]: init1 = init_ops.orthogonal_initializer(seed=1, dtype=dtype) init2 = init_ops.orthogonal_initializer(seed=1, dtype=dtype) self.assertTrue(identicaltest(self, init1, init2, (10, 10))) + @test_util.run_deprecated_v1 def testInitializerDifferent(self): for dtype in [dtypes.float32, dtypes.float64]: init1 = init_ops.orthogonal_initializer(seed=1, dtype=dtype) init2 = init_ops.orthogonal_initializer(seed=2, dtype=dtype) self.assertFalse(identicaltest(self, init1, init2, (10, 10))) + @test_util.run_deprecated_v1 def testDuplicatedInitializer(self): init = init_ops.orthogonal_initializer() self.assertFalse(duplicated_initializer(self, init, 1, (10, 10))) @@ -608,6 +639,7 @@ class OrthogonalInitializerTest(test.TestCase): with self.session(graph=ops.Graph(), use_gpu=True): self.assertRaises(ValueError, init1, shape=[5]) + @test_util.run_deprecated_v1 def testGain(self): shape = (10, 10) for dtype in [dtypes.float32, dtypes.float64]: @@ -616,8 +648,9 @@ class OrthogonalInitializerTest(test.TestCase): with self.session(graph=ops.Graph(), use_gpu=True): t1 = init1(shape).eval() t2 = init2(shape).eval() - return np.allclose(t1, t2 / 3.14, rtol=1e-15, atol=1e-15) + self.assertAllClose(t1, t2 / 3.14) + @test_util.run_deprecated_v1 def testShapesValues(self): for dtype in [dtypes.float32, dtypes.float64]: for shape in [(10, 10), (10, 9, 8), (100, 5, 5), (50, 40), (40, 50)]: @@ -639,18 +672,21 @@ class OrthogonalInitializerTest(test.TestCase): class ConvolutionDeltaOrthogonalInitializerTest(test.TestCase): + @test_util.run_deprecated_v1 def testInitializerIdentical(self): for dtype in [dtypes.float32, dtypes.float64]: init1 = init_ops.convolutional_delta_orthogonal(seed=1, dtype=dtype) init2 = init_ops.convolutional_delta_orthogonal(seed=1, dtype=dtype) self.assertTrue(identicaltest(self, init1, init2, (3, 3, 10, 10))) + @test_util.run_deprecated_v1 def testInitializerDifferent(self): for dtype in [dtypes.float32, dtypes.float64]: init1 = init_ops.convolutional_delta_orthogonal(seed=1, dtype=dtype) init2 = init_ops.convolutional_delta_orthogonal(seed=2, dtype=dtype) self.assertFalse(identicaltest(self, init1, init2, (3, 3, 10, 10))) + @test_util.run_deprecated_v1 def testDuplicatedInitializer(self): init = init_ops.convolutional_delta_orthogonal() self.assertFalse(duplicated_initializer(self, init, 1, (3, 3, 10, 10))) @@ -665,6 +701,7 @@ class ConvolutionDeltaOrthogonalInitializerTest(test.TestCase): with self.session(graph=ops.Graph(), use_gpu=True): self.assertRaises(ValueError, init1, shape=[3, 3, 6, 5]) + @test_util.run_deprecated_v1 def testGain(self): shape = (3, 3, 10, 10) for dtype in [dtypes.float32, dtypes.float64]: @@ -674,8 +711,9 @@ class ConvolutionDeltaOrthogonalInitializerTest(test.TestCase): with self.session(graph=ops.Graph(), use_gpu=True): t1 = init1(shape).eval() t2 = init2(shape).eval() - return np.allclose(t1, t2 / 3.14, rtol=1e-15, atol=1e-15) + self.assertAllClose(t1, t2 / 3.14) + @test_util.run_deprecated_v1 def testShapesValues(self): gain = 3.14 for dtype in [dtypes.float32]: @@ -704,14 +742,14 @@ class ConvolutionDeltaOrthogonalInitializerTest(test.TestCase): ratio = outputs_2norm / inputs_2norm my_ops = variables.global_variables_initializer() with self.session(use_gpu=True) as sess: - sess.run(my_ops) + self.evaluate(my_ops) # Check the shape of the outputs - t = outputs.eval() + t = self.evaluate(outputs) self.assertAllEqual(t.shape, outputs_shape) # Check isometry of the delta-orthogonal kernel. - self.assertAllClose(sess.run(ratio), np.sqrt(gain), - rtol=tol, atol=tol) + self.assertAllClose(self.evaluate(ratio), gain, rtol=tol, atol=tol) + @test_util.run_deprecated_v1 def testNonuniformity(self): value = 0 abs_value = 0 @@ -724,7 +762,7 @@ class ConvolutionDeltaOrthogonalInitializerTest(test.TestCase): initializer= init_ops.convolutional_delta_orthogonal) x.initializer.run() - y = x.eval()[1, 1, :, :] + y = self.evaluate(x)[1, 1, :, :] determinant = np.linalg.det(y) value += determinant abs_value += np.abs(determinant) @@ -739,18 +777,21 @@ class ConvolutionDeltaOrthogonalInitializerTest(test.TestCase): class ConvolutionOrthogonal1dInitializerTest(test.TestCase): + @test_util.run_deprecated_v1 def testInitializerIdentical(self): for dtype in [dtypes.float32, dtypes.float64]: init1 = init_ops.convolutional_orthogonal_1d(seed=1, dtype=dtype) init2 = init_ops.convolutional_orthogonal_1d(seed=1, dtype=dtype) self.assertTrue(identicaltest(self, init1, init2, (3, 10, 10))) + @test_util.run_deprecated_v1 def testInitializerDifferent(self): for dtype in [dtypes.float32, dtypes.float64]: init1 = init_ops.convolutional_orthogonal_1d(seed=1, dtype=dtype) init2 = init_ops.convolutional_orthogonal_1d(seed=2, dtype=dtype) self.assertFalse(identicaltest(self, init1, init2, (3, 10, 10))) + @test_util.run_deprecated_v1 def testDuplicatedInitializer(self): init = init_ops.convolutional_orthogonal_1d() self.assertFalse(duplicated_initializer(self, init, 1, (3, 10, 10))) @@ -765,6 +806,7 @@ class ConvolutionOrthogonal1dInitializerTest(test.TestCase): with self.session(graph=ops.Graph(), use_gpu=True): self.assertRaises(ValueError, init1, shape=[3, 6, 5]) + @test_util.run_deprecated_v1 def testGain(self): shape = (3, 10, 10) for dtype in [dtypes.float32, dtypes.float64]: @@ -774,8 +816,9 @@ class ConvolutionOrthogonal1dInitializerTest(test.TestCase): with self.session(graph=ops.Graph(), use_gpu=True): t1 = init1(shape).eval() t2 = init2(shape).eval() - return np.allclose(t1, t2 / 3.14, rtol=1e-15, atol=1e-15) + self.assertAllClose(t1, t2 / 3.14) + @test_util.run_deprecated_v1 def testNonuniformity(self): value = 0 abs_value = 0 @@ -800,6 +843,7 @@ class ConvolutionOrthogonal1dInitializerTest(test.TestCase): # Compute the sum of the absolute values of 'count' determinants self.assertAllClose(abs_value, count, rtol=tol, atol=tol) + @test_util.run_deprecated_v1 def testShapesValues(self): def circular_pad(input_, width, kernel_size): """Pad input_ for computing (circular) convolution. @@ -843,28 +887,31 @@ class ConvolutionOrthogonal1dInitializerTest(test.TestCase): ratio = outputs_2norm / inputs_2norm my_ops = variables.global_variables_initializer() with self.session(use_gpu=True) as sess: - sess.run(my_ops) + self.evaluate(my_ops) # Check the shape of the outputs - t = outputs.eval() + t = self.evaluate(outputs) self.assertAllEqual(t.shape, outputs_shape) # Check isometry of the orthogonal kernel. - self.assertAllClose(sess.run(ratio), np.sqrt(gain), rtol=tol, atol=tol) + self.assertAllClose(self.evaluate(ratio), gain, rtol=tol, atol=tol) class ConvolutionOrthogonal2dInitializerTest(test.TestCase): + @test_util.run_deprecated_v1 def testInitializerIdentical(self): for dtype in [dtypes.float32, dtypes.float64]: init1 = init_ops.convolutional_orthogonal_2d(seed=1, dtype=dtype) init2 = init_ops.convolutional_orthogonal_2d(seed=1, dtype=dtype) self.assertTrue(identicaltest(self, init1, init2, (3, 3, 10, 10))) + @test_util.run_deprecated_v1 def testInitializerDifferent(self): for dtype in [dtypes.float32, dtypes.float64]: init1 = init_ops.convolutional_orthogonal_2d(seed=1, dtype=dtype) init2 = init_ops.convolutional_orthogonal_2d(seed=2, dtype=dtype) self.assertFalse(identicaltest(self, init1, init2, (3, 3, 10, 10))) + @test_util.run_deprecated_v1 def testDuplicatedInitializer(self): init = init_ops.convolutional_orthogonal_2d() self.assertFalse(duplicated_initializer(self, init, 1, (3, 3, 10, 10))) @@ -879,6 +926,7 @@ class ConvolutionOrthogonal2dInitializerTest(test.TestCase): with self.session(graph=ops.Graph(), use_gpu=True): self.assertRaises(ValueError, init1, shape=[3, 3, 6, 5]) + @test_util.run_deprecated_v1 def testGain(self): shape = (3, 3, 10, 10) for dtype in [dtypes.float32, dtypes.float64]: @@ -888,8 +936,9 @@ class ConvolutionOrthogonal2dInitializerTest(test.TestCase): with self.session(graph=ops.Graph(), use_gpu=True): t1 = init1(shape).eval() t2 = init2(shape).eval() - return np.allclose(t1, t2 / 3.14, rtol=1e-15, atol=1e-15) + self.assertAllClose(t1, t2 / 3.14) + @test_util.run_deprecated_v1 def testShapesValues(self): def circular_pad(input_, width, kernel_size): """Pad input_ for computing (circular) convolution. @@ -938,28 +987,31 @@ class ConvolutionOrthogonal2dInitializerTest(test.TestCase): ratio = outputs_2norm / inputs_2norm my_ops = variables.global_variables_initializer() with self.session(use_gpu=True) as sess: - sess.run(my_ops) + self.evaluate(my_ops) # Check the shape of the outputs - t = outputs.eval() + t = self.evaluate(outputs) self.assertAllEqual(t.shape, outputs_shape) # Check isometry of the orthogonal kernel. - self.assertAllClose(sess.run(ratio), np.sqrt(gain), rtol=tol, atol=tol) + self.assertAllClose(self.evaluate(ratio), gain, rtol=tol, atol=tol) class ConvolutionOrthogonal3dInitializerTest(test.TestCase): + @test_util.run_deprecated_v1 def testInitializerIdentical(self): for dtype in [dtypes.float32, dtypes.float64]: init1 = init_ops.convolutional_orthogonal_3d(seed=1, dtype=dtype) init2 = init_ops.convolutional_orthogonal_3d(seed=1, dtype=dtype) self.assertTrue(identicaltest(self, init1, init2, (3, 3, 3, 10, 10))) + @test_util.run_deprecated_v1 def testInitializerDifferent(self): for dtype in [dtypes.float32, dtypes.float64]: init1 = init_ops.convolutional_orthogonal_3d(seed=1, dtype=dtype) init2 = init_ops.convolutional_orthogonal_3d(seed=2, dtype=dtype) self.assertFalse(identicaltest(self, init1, init2, (3, 3, 3, 10, 10))) + @test_util.run_deprecated_v1 def testDuplicatedInitializer(self): init = init_ops.convolutional_orthogonal_3d() self.assertFalse(duplicated_initializer(self, init, 1, (3, 3, 3, 10, 10))) @@ -974,6 +1026,7 @@ class ConvolutionOrthogonal3dInitializerTest(test.TestCase): with self.session(graph=ops.Graph(), use_gpu=True): self.assertRaises(ValueError, init1, shape=[3, 3, 3, 6, 5]) + @test_util.run_deprecated_v1 def testGain(self): shape = (3, 3, 3, 10, 10) for dtype in [dtypes.float32, dtypes.float64]: @@ -983,8 +1036,9 @@ class ConvolutionOrthogonal3dInitializerTest(test.TestCase): with self.session(graph=ops.Graph(), use_gpu=True): t1 = init1(shape).eval() t2 = init2(shape).eval() - return np.allclose(t1, t2 / 3.14, rtol=1e-15, atol=1e-15) + self.assertAllClose(t1, t2 / 3.14) + @test_util.run_deprecated_v1 def testNonuniformity(self): value = 0 abs_value = 0 @@ -1009,6 +1063,7 @@ class ConvolutionOrthogonal3dInitializerTest(test.TestCase): # Compute the sum of the absolute values of 'count' determinants self.assertAllClose(abs_value, count, rtol=tol, atol=tol) + @test_util.run_deprecated_v1 def testShapesValues(self): def circular_pad(input_, width, kernel_size): """Padding input_ for computing circular convolution. @@ -1063,12 +1118,12 @@ class ConvolutionOrthogonal3dInitializerTest(test.TestCase): ratio = outputs_2norm / inputs_2norm my_ops = variables.global_variables_initializer() with self.cached_session(use_gpu=True) as sess: - sess.run(my_ops) + self.evaluate(my_ops) # Check the shape of the outputs - t = outputs.eval() + t = self.evaluate(outputs) self.assertAllEqual(t.shape, outputs_shape) # Check isometry of the orthogonal kernel. - self.assertAllClose(sess.run(ratio), np.sqrt(gain), rtol=tol, atol=tol) + self.assertAllClose(self.evaluate(ratio), gain, rtol=tol, atol=tol) class IdentityInitializerTest(test.TestCase): @@ -1084,12 +1139,14 @@ class IdentityInitializerTest(test.TestCase): self.assertRaises(ValueError, init, shape=[5]) self.assertRaises(ValueError, init, shape=[]) + @test_util.run_deprecated_v1 def testNonSquare(self): init = init_ops.identity_initializer() shape = (10, 5) with self.session(graph=ops.Graph(), use_gpu=True): self.assertAllClose(init(shape).eval(), np.eye(*shape)) + @test_util.run_deprecated_v1 def testGain(self): shape = (10, 10) for dtype in [dtypes.float32, dtypes.float64]: @@ -1100,6 +1157,7 @@ class IdentityInitializerTest(test.TestCase): with self.session(graph=ops.Graph(), use_gpu=True): self.assertAllClose(init_custom(shape).eval(), np.eye(*shape) * 0.9) + @test_util.run_deprecated_v1 def testPartitions(self): shape = (10, 10) init = init_ops.identity_initializer() diff --git a/tensorflow/python/kernel_tests/inplace_ops_test.py b/tensorflow/python/kernel_tests/inplace_ops_test.py index 51d16861dd8..9eaaac7a248 100644 --- a/tensorflow/python/kernel_tests/inplace_ops_test.py +++ b/tensorflow/python/kernel_tests/inplace_ops_test.py @@ -31,6 +31,7 @@ from tensorflow.python.platform import test as test_lib class InplaceOpsTest(test_util.TensorFlowTestCase): + @test_util.run_deprecated_v1 def testBasicUpdate(self): for dtype in [dtypes.float32, dtypes.int32, dtypes.int64]: with self.session(use_gpu=True): @@ -48,6 +49,7 @@ class InplaceOpsTest(test_util.TensorFlowTestCase): y[5, :] = 7 self.assertAllClose(x.eval(), y) + @test_util.run_deprecated_v1 def testBasicUpdateBool(self): with self.session(use_gpu=True): x = array_ops.ones([7, 3], dtypes.bool) @@ -65,6 +67,7 @@ class InplaceOpsTest(test_util.TensorFlowTestCase): y[5, :] = False self.assertAllClose(x.eval(), y) + @test_util.run_deprecated_v1 def testBasicAdd(self): for dtype in [dtypes.float32, dtypes.int32, dtypes.int64]: with self.cached_session(use_gpu=True): @@ -84,6 +87,7 @@ class InplaceOpsTest(test_util.TensorFlowTestCase): y[:, :] += 99 self.assertAllClose(x.eval(), y) + @test_util.run_deprecated_v1 def testBasicSub(self): for dtype in [dtypes.float32, dtypes.int32, dtypes.int64]: with self.cached_session(use_gpu=True): @@ -103,6 +107,7 @@ class InplaceOpsTest(test_util.TensorFlowTestCase): y[:, :] -= 99 self.assertAllClose(x.eval(), y) + @test_util.run_deprecated_v1 def testRandom(self): with self.session(use_gpu=True): d0, d1, d2 = 100, 3, 5 @@ -123,6 +128,7 @@ class InplaceOpsTest(test_util.TensorFlowTestCase): y[idx, :] -= val self.assertAllClose(x.eval(), y) + @test_util.run_deprecated_v1 def testRandom1D(self): with self.session(use_gpu=True): d0 = 100 @@ -149,7 +155,7 @@ class InplaceOpsTest(test_util.TensorFlowTestCase): y = inplace_ops.alias_inplace_add(x, [0], [[1, 2, 3]]) with ops.control_dependencies([y]): z = array_ops.identity(x) - _, vy, vz = sess.run([x, y, z]) + _, vy, vz = self.evaluate([x, y, z]) self.assertAllClose(vy, vz) def testError(self): @@ -164,6 +170,7 @@ class InplaceOpsTest(test_util.TensorFlowTestCase): "i and x shape doesn't match"): _ = inplace_ops.inplace_update([[1.]], [0, 1], [[10]]).eval() + @test_util.run_deprecated_v1 def testEmpty(self): for dtype in [ dtypes.float32, dtypes.float64, dtypes.int32, dtypes.int64, dtypes.bool, diff --git a/tensorflow/python/kernel_tests/io_ops_test.py b/tensorflow/python/kernel_tests/io_ops_test.py index afa24195cb3..c5df5231bf6 100644 --- a/tensorflow/python/kernel_tests/io_ops_test.py +++ b/tensorflow/python/kernel_tests/io_ops_test.py @@ -23,6 +23,7 @@ import os import shutil import tempfile +from tensorflow.python.framework import test_util from tensorflow.python.ops import io_ops from tensorflow.python.platform import test from tensorflow.python.util import compat @@ -30,6 +31,7 @@ from tensorflow.python.util import compat class IoOpsTest(test.TestCase): + @test_util.run_deprecated_v1 def testReadFile(self): cases = ['', 'Some contents', 'Неки садржаји на српском'] for contents in cases: @@ -53,7 +55,7 @@ class IoOpsTest(test.TestCase): pass with self.cached_session() as sess: w = io_ops.write_file(temp.name, contents) - sess.run(w) + self.evaluate(w) with open(temp.name, 'rb') as f: file_contents = f.read() self.assertEqual(file_contents, contents) @@ -67,7 +69,7 @@ class IoOpsTest(test.TestCase): filepath = os.path.join(subdir, 'subdir2', 'filename') with self.cached_session() as sess: w = io_ops.write_file(filepath, contents) - sess.run(w) + self.evaluate(w) with open(filepath, 'rb') as f: file_contents = f.read() self.assertEqual(file_contents, contents) @@ -78,6 +80,7 @@ class IoOpsTest(test.TestCase): compat.as_bytes(files[i].name) for i in range(len(files)) if i in indices) + @test_util.run_deprecated_v1 def testMatchingFiles(self): cases = [ 'ABcDEF.GH', 'ABzDEF.GH', 'ABasdfjklDEF.GH', 'AB3DEF.GH', 'AB4DEF.GH', diff --git a/tensorflow/python/kernel_tests/large_concat_op_test.py b/tensorflow/python/kernel_tests/large_concat_op_test.py index 1b23e747764..bf6fa9ea71f 100644 --- a/tensorflow/python/kernel_tests/large_concat_op_test.py +++ b/tensorflow/python/kernel_tests/large_concat_op_test.py @@ -35,7 +35,7 @@ class LargeConcatOpTest(test.TestCase): with self.session(use_gpu=False): # TODO(dga): Add more depth to this test to validate correctness, # not just non-crashingness, once other large tensor fixes have gone in. - _ = onezeros.eval() + _ = self.evaluate(onezeros) if __name__ == "__main__": diff --git a/tensorflow/python/kernel_tests/linalg/BUILD b/tensorflow/python/kernel_tests/linalg/BUILD index be2e31cb5ad..ba9e64979a4 100644 --- a/tensorflow/python/kernel_tests/linalg/BUILD +++ b/tensorflow/python/kernel_tests/linalg/BUILD @@ -40,6 +40,44 @@ cuda_py_test( ], ) +cuda_py_test( + name = "linear_operator_adjoint_test", + size = "medium", + srcs = ["linear_operator_adjoint_test.py"], + additional_deps = [ + "//tensorflow/python/ops/linalg", + "//third_party/py/numpy", + "//tensorflow/python:array_ops", + "//tensorflow/python:client_testlib", + "//tensorflow/python:framework", + "//tensorflow/python:framework_for_generated_wrappers", + "//tensorflow/python:framework_test_lib", + "//tensorflow/python:math_ops", + "//tensorflow/python:platform_test", + ], + shard_count = 5, + tags = [ + "noasan", # times out, b/63678675 + "optonly", # times out + ], +) + +cuda_py_test( + name = "linear_operator_algebra_test", + size = "small", + srcs = ["linear_operator_algebra_test.py"], + additional_deps = [ + "//tensorflow/python/ops/linalg", + "//third_party/py/numpy", + "//tensorflow/python:array_ops", + "//tensorflow/python:client_testlib", + "//tensorflow/python:framework_for_generated_wrappers", + "//tensorflow/python:framework_test_lib", + "//tensorflow/python:math_ops", + "//tensorflow/python:platform_test", + ], +) + cuda_py_test( name = "linear_operator_block_diag_test", size = "medium", @@ -89,7 +127,6 @@ cuda_py_test( size = "medium", srcs = ["linear_operator_circulant_test.py"], additional_deps = [ - "//tensorflow/python/ops/linalg", "//third_party/py/numpy", "//tensorflow/python:array_ops", "//tensorflow/python:spectral_ops_test_util", @@ -99,6 +136,8 @@ cuda_py_test( "//tensorflow/python:framework_test_lib", "//tensorflow/python:math_ops", "//tensorflow/python:platform_test", + "//tensorflow/python/ops/linalg", + "//tensorflow/python/ops/signal", ], shard_count = 5, tags = [ @@ -150,6 +189,28 @@ cuda_py_test( ], ) +cuda_py_test( + name = "linear_operator_inversion_test", + size = "medium", + srcs = ["linear_operator_inversion_test.py"], + additional_deps = [ + "//tensorflow/python/ops/linalg", + "//third_party/py/numpy", + "//tensorflow/python:array_ops", + "//tensorflow/python:client_testlib", + "//tensorflow/python:framework", + "//tensorflow/python:framework_for_generated_wrappers", + "//tensorflow/python:framework_test_lib", + "//tensorflow/python:math_ops", + "//tensorflow/python:platform_test", + ], + shard_count = 5, + tags = [ + "noasan", # times out, b/63678675 + "optonly", # times out + ], +) + cuda_py_test( name = "linear_operator_full_matrix_test", size = "medium", diff --git a/tensorflow/python/kernel_tests/linalg/linear_operator_addition_test.py b/tensorflow/python/kernel_tests/linalg/linear_operator_addition_test.py index 628ed998c54..627349c69b3 100644 --- a/tensorflow/python/kernel_tests/linalg/linear_operator_addition_test.py +++ b/tensorflow/python/kernel_tests/linalg/linear_operator_addition_test.py @@ -19,6 +19,7 @@ from __future__ import print_function import numpy as np +from tensorflow.python.framework import test_util from tensorflow.python.ops import linalg_ops from tensorflow.python.ops.linalg import linalg as linalg_lib from tensorflow.python.ops.linalg import linear_operator_addition @@ -69,6 +70,7 @@ class LinearOperatorAdditionCorrectnessTest(test.TestCase): with self.assertRaisesRegexp(TypeError, "contain only LinearOperator"): add_operators([1, 2]) + @test_util.run_deprecated_v1 def test_two_diag_operators(self): op_a = linalg.LinearOperatorDiag( [1., 1.], is_positive_definite=True, name="A") @@ -89,6 +91,7 @@ class LinearOperatorAdditionCorrectnessTest(test.TestCase): # Enforce particular name for this simple case self.assertEqual("Add/B__A/", op.name) + @test_util.run_deprecated_v1 def test_three_diag_operators(self): op1 = linalg.LinearOperatorDiag( [1., 1.], is_positive_definite=True, name="op1") @@ -109,6 +112,7 @@ class LinearOperatorAdditionCorrectnessTest(test.TestCase): # Positive definite ==> non-singular self.assertTrue(op.is_non_singular) + @test_util.run_deprecated_v1 def test_diag_tril_diag(self): op1 = linalg.LinearOperatorDiag( [1., 1.], is_non_singular=True, name="diag_a") @@ -134,6 +138,7 @@ class LinearOperatorAdditionCorrectnessTest(test.TestCase): # Since no custom hint was provided, we default to None (unknown). self.assertEqual(None, op.is_non_singular) + @test_util.run_deprecated_v1 def test_matrix_diag_tril_diag_uses_custom_name(self): op0 = linalg.LinearOperatorFullMatrix( [[-1., -1.], [-1., -1.]], name="matrix") @@ -217,6 +222,7 @@ class LinearOperatorOrderOfAdditionTest(test.TestCase): self.assertEqual(1, len(op_sum)) self.assertIsInstance(op_sum[0], linalg.LinearOperatorLowerTriangular) + @test_util.run_deprecated_v1 def test_cannot_add_everything_so_return_more_than_one_operator(self): diag1 = linalg.LinearOperatorDiag([1.]) diag2 = linalg.LinearOperatorDiag([2.]) @@ -261,6 +267,7 @@ class AddAndReturnScaledIdentityTest(test.TestCase): def setUp(self): self._adder = linear_operator_addition._AddAndReturnScaledIdentity() + @test_util.run_deprecated_v1 def test_identity_plus_identity(self): id1 = linalg.LinearOperatorIdentity(num_rows=2) id2 = linalg.LinearOperatorIdentity(num_rows=2, batch_shape=[3]) @@ -279,6 +286,7 @@ class AddAndReturnScaledIdentityTest(test.TestCase): self.assertTrue(operator.is_non_singular) self.assertEqual("my_operator", operator.name) + @test_util.run_deprecated_v1 def test_identity_plus_scaled_identity(self): id1 = linalg.LinearOperatorIdentity(num_rows=2, batch_shape=[3]) id2 = linalg.LinearOperatorScaledIdentity(num_rows=2, multiplier=2.2) @@ -297,6 +305,7 @@ class AddAndReturnScaledIdentityTest(test.TestCase): self.assertTrue(operator.is_non_singular) self.assertEqual("my_operator", operator.name) + @test_util.run_deprecated_v1 def test_scaled_identity_plus_scaled_identity(self): id1 = linalg.LinearOperatorScaledIdentity( num_rows=2, multiplier=[2.2, 2.2, 2.2]) @@ -322,6 +331,7 @@ class AddAndReturnDiagTest(test.TestCase): def setUp(self): self._adder = linear_operator_addition._AddAndReturnDiag() + @test_util.run_deprecated_v1 def test_identity_plus_identity_returns_diag(self): id1 = linalg.LinearOperatorIdentity(num_rows=2) id2 = linalg.LinearOperatorIdentity(num_rows=2, batch_shape=[3]) @@ -340,6 +350,7 @@ class AddAndReturnDiagTest(test.TestCase): self.assertTrue(operator.is_non_singular) self.assertEqual("my_operator", operator.name) + @test_util.run_deprecated_v1 def test_diag_plus_diag(self): diag1 = rng.rand(2, 3, 4) diag2 = rng.rand(4) @@ -366,6 +377,7 @@ class AddAndReturnTriLTest(test.TestCase): def setUp(self): self._adder = linear_operator_addition._AddAndReturnTriL() + @test_util.run_deprecated_v1 def test_diag_plus_tril(self): diag = linalg.LinearOperatorDiag([1., 2.]) tril = linalg.LinearOperatorLowerTriangular([[10., 0.], [30., 0.]]) @@ -389,6 +401,7 @@ class AddAndReturnMatrixTest(test.TestCase): def setUp(self): self._adder = linear_operator_addition._AddAndReturnMatrix() + @test_util.run_deprecated_v1 def test_diag_plus_diag(self): diag1 = linalg.LinearOperatorDiag([1., 2.]) diag2 = linalg.LinearOperatorDiag([-1., 3.]) diff --git a/tensorflow/python/kernel_tests/linalg/linear_operator_adjoint_test.py b/tensorflow/python/kernel_tests/linalg/linear_operator_adjoint_test.py new file mode 100644 index 00000000000..1bed4b5268e --- /dev/null +++ b/tensorflow/python/kernel_tests/linalg/linear_operator_adjoint_test.py @@ -0,0 +1,118 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from tensorflow.python.framework import dtypes +from tensorflow.python.ops import array_ops +from tensorflow.python.ops.linalg import linalg as linalg_lib +from tensorflow.python.ops.linalg import linear_operator_adjoint +from tensorflow.python.ops.linalg import linear_operator_test_util +from tensorflow.python.platform import test + +linalg = linalg_lib + +LinearOperatorAdjoint = linear_operator_adjoint.LinearOperatorAdjoint # pylint: disable=invalid-name + + +class LinearOperatorAdjointTest( + linear_operator_test_util.SquareLinearOperatorDerivedClassTest): + """Most tests done in the base class LinearOperatorDerivedClassTest.""" + + def setUp(self): + self._atol[dtypes.complex64] = 1e-5 + self._rtol[dtypes.complex64] = 1e-5 + + def _operator_and_matrix(self, + build_info, + dtype, + use_placeholder, + ensure_self_adjoint_and_pd=False): + shape = list(build_info.shape) + + if ensure_self_adjoint_and_pd: + matrix = linear_operator_test_util.random_positive_definite_matrix( + shape, dtype, force_well_conditioned=True) + else: + matrix = linear_operator_test_util.random_tril_matrix( + shape, dtype, force_well_conditioned=True, remove_upper=True) + + lin_op_matrix = matrix + + if use_placeholder: + lin_op_matrix = array_ops.placeholder_with_default(matrix, shape=None) + + if ensure_self_adjoint_and_pd: + operator = LinearOperatorAdjoint( + linalg.LinearOperatorFullMatrix( + lin_op_matrix, is_positive_definite=True, is_self_adjoint=True)) + else: + operator = LinearOperatorAdjoint( + linalg.LinearOperatorLowerTriangular(lin_op_matrix)) + + return operator, linalg.adjoint(matrix) + + def test_base_operator_hint_used(self): + # The matrix values do not effect auto-setting of the flags. + matrix = [[1., 0.], [1., 1.]] + operator = linalg.LinearOperatorFullMatrix( + matrix, + is_positive_definite=True, + is_non_singular=True, + is_self_adjoint=False) + operator_adjoint = LinearOperatorAdjoint(operator) + self.assertTrue(operator_adjoint.is_positive_definite) + self.assertTrue(operator_adjoint.is_non_singular) + self.assertFalse(operator_adjoint.is_self_adjoint) + + def test_supplied_hint_used(self): + # The matrix values do not effect auto-setting of the flags. + matrix = [[1., 0.], [1., 1.]] + operator = linalg.LinearOperatorFullMatrix(matrix) + operator_adjoint = LinearOperatorAdjoint( + operator, + is_positive_definite=True, + is_non_singular=True, + is_self_adjoint=False) + self.assertTrue(operator_adjoint.is_positive_definite) + self.assertTrue(operator_adjoint.is_non_singular) + self.assertFalse(operator_adjoint.is_self_adjoint) + + def test_contradicting_hints_raise(self): + # The matrix values do not effect auto-setting of the flags. + matrix = [[1., 0.], [1., 1.]] + operator = linalg.LinearOperatorFullMatrix( + matrix, is_positive_definite=False) + with self.assertRaisesRegexp(ValueError, "positive-definite"): + LinearOperatorAdjoint(operator, is_positive_definite=True) + + operator = linalg.LinearOperatorFullMatrix(matrix, is_self_adjoint=False) + with self.assertRaisesRegexp(ValueError, "self-adjoint"): + LinearOperatorAdjoint(operator, is_self_adjoint=True) + + def test_name(self): + matrix = [[11., 0.], [1., 8.]] + operator = linalg.LinearOperatorFullMatrix( + matrix, name="my_operator", is_non_singular=True) + + operator = LinearOperatorAdjoint(operator) + + self.assertEqual("my_operator_adjoint", operator.name) + + +if __name__ == "__main__": + test.main() diff --git a/tensorflow/python/kernel_tests/linalg/linear_operator_algebra_test.py b/tensorflow/python/kernel_tests/linalg/linear_operator_algebra_test.py new file mode 100644 index 00000000000..8e296c026c0 --- /dev/null +++ b/tensorflow/python/kernel_tests/linalg/linear_operator_algebra_test.py @@ -0,0 +1,133 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Tests for registration mechanisms.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from tensorflow.python.framework import tensor_shape +from tensorflow.python.ops.linalg import cholesky_registrations # pylint: disable=unused-import +from tensorflow.python.ops.linalg import linear_operator +from tensorflow.python.ops.linalg import linear_operator_algebra +from tensorflow.python.ops.linalg import matmul_registrations # pylint: disable=unused-import +from tensorflow.python.platform import test + +# pylint: disable=protected-access +_CHOLESKY_DECOMPS = linear_operator_algebra._CHOLESKY_DECOMPS +_MATMUL = linear_operator_algebra._MATMUL +_registered_cholesky = linear_operator_algebra._registered_cholesky +_registered_matmul = linear_operator_algebra._registered_matmul +# pylint: enable=protected-access + + +class CholeskyTest(test.TestCase): + + def testRegistration(self): + + class CustomLinOp(linear_operator.LinearOperator): + + def _matmul(self, a): + pass + + def _shape(self): + return tensor_shape.TensorShape([1, 1]) + + def _shape_tensor(self): + pass + + # Register Cholesky to a lambda that spits out the name parameter + @linear_operator_algebra.RegisterCholesky(CustomLinOp) + def _cholesky(a): # pylint: disable=unused-argument,unused-variable + return "OK" + + with self.assertRaisesRegexp(ValueError, "positive definite"): + CustomLinOp(dtype=None, is_self_adjoint=True).cholesky() + + with self.assertRaisesRegexp(ValueError, "self adjoint"): + CustomLinOp(dtype=None, is_positive_definite=True).cholesky() + + custom_linop = CustomLinOp( + dtype=None, is_self_adjoint=True, is_positive_definite=True) + self.assertEqual("OK", custom_linop.cholesky()) + + def testRegistrationFailures(self): + + class CustomLinOp(linear_operator.LinearOperator): + pass + + with self.assertRaisesRegexp(TypeError, "must be callable"): + linear_operator_algebra.RegisterCholesky(CustomLinOp)("blah") + + # First registration is OK + linear_operator_algebra.RegisterCholesky(CustomLinOp)(lambda a: None) + + # Second registration fails + with self.assertRaisesRegexp(ValueError, "has already been registered"): + linear_operator_algebra.RegisterCholesky(CustomLinOp)(lambda a: None) + + def testExactCholeskyRegistrationsAllMatch(self): + for (k, v) in _CHOLESKY_DECOMPS.items(): + self.assertEqual(v, _registered_cholesky(k[0])) + + +class MatmulTest(test.TestCase): + + def testRegistration(self): + + class CustomLinOp(linear_operator.LinearOperator): + + def _matmul(self, a): + pass + + def _shape(self): + return tensor_shape.TensorShape([1, 1]) + + def _shape_tensor(self): + pass + + # Register Matmul to a lambda that spits out the name parameter + @linear_operator_algebra.RegisterMatmul(CustomLinOp, CustomLinOp) + def _matmul(a, b): # pylint: disable=unused-argument,unused-variable + return "OK" + + custom_linop = CustomLinOp( + dtype=None, is_self_adjoint=True, is_positive_definite=True) + self.assertEqual("OK", custom_linop.matmul(custom_linop)) + + def testRegistrationFailures(self): + + class CustomLinOp(linear_operator.LinearOperator): + pass + + with self.assertRaisesRegexp(TypeError, "must be callable"): + linear_operator_algebra.RegisterMatmul(CustomLinOp, CustomLinOp)("blah") + + # First registration is OK + linear_operator_algebra.RegisterMatmul( + CustomLinOp, CustomLinOp)(lambda a: None) + + # Second registration fails + with self.assertRaisesRegexp(ValueError, "has already been registered"): + linear_operator_algebra.RegisterMatmul( + CustomLinOp, CustomLinOp)(lambda a: None) + + def testExactMatmulRegistrationsAllMatch(self): + for (k, v) in _MATMUL.items(): + self.assertEqual(v, _registered_matmul(k[0], k[1])) + + +if __name__ == "__main__": + test.main() diff --git a/tensorflow/python/kernel_tests/linalg/linear_operator_block_diag_test.py b/tensorflow/python/kernel_tests/linalg/linear_operator_block_diag_test.py index 30951b1b0eb..f0cc5d709f9 100644 --- a/tensorflow/python/kernel_tests/linalg/linear_operator_block_diag_test.py +++ b/tensorflow/python/kernel_tests/linalg/linear_operator_block_diag_test.py @@ -23,6 +23,7 @@ from tensorflow.python.framework import dtypes from tensorflow.python.ops import array_ops from tensorflow.python.ops.linalg import linalg as linalg_lib from tensorflow.python.ops.linalg import linear_operator_block_diag as block_diag +from tensorflow.python.ops.linalg import linear_operator_lower_triangular as lower_triangular from tensorflow.python.ops.linalg import linear_operator_test_util from tensorflow.python.ops.linalg import linear_operator_util from tensorflow.python.platform import test @@ -78,7 +79,9 @@ class SquareLinearOperatorBlockDiagTest( build_info((2, 1, 5, 5), blocks=[(2, 1, 2, 2), (1, 3, 3)]), ] - def _operator_and_matrix(self, build_info, dtype, use_placeholder): + def _operator_and_matrix( + self, build_info, dtype, use_placeholder, + ensure_self_adjoint_and_pd=False): shape = list(build_info.shape) expected_blocks = ( build_info.__dict__["blocks"] if "blocks" in build_info.__dict__ @@ -98,7 +101,11 @@ class SquareLinearOperatorBlockDiagTest( operator = block_diag.LinearOperatorBlockDiag( [linalg.LinearOperatorFullMatrix( - l, is_square=True) for l in lin_op_matrices]) + l, + is_square=True, + is_self_adjoint=True if ensure_self_adjoint_and_pd else None, + is_positive_definite=True if ensure_self_adjoint_and_pd else None) + for l in lin_op_matrices]) # Should be auto-set. self.assertTrue(operator.is_square) @@ -129,6 +136,40 @@ class SquareLinearOperatorBlockDiagTest( self.assertTrue(operator.is_non_singular) self.assertFalse(operator.is_self_adjoint) + def test_block_diag_cholesky_type(self): + matrix = [[1., 0.], [0., 1.]] + operator = block_diag.LinearOperatorBlockDiag( + [ + linalg.LinearOperatorFullMatrix( + matrix, + is_positive_definite=True, + is_self_adjoint=True, + ), + linalg.LinearOperatorFullMatrix( + matrix, + is_positive_definite=True, + is_self_adjoint=True, + ), + ], + is_positive_definite=True, + is_self_adjoint=True, + ) + cholesky_factor = operator.cholesky() + self.assertTrue(isinstance( + cholesky_factor, + block_diag.LinearOperatorBlockDiag)) + self.assertEqual(2, len(cholesky_factor.operators)) + self.assertTrue( + isinstance( + cholesky_factor.operators[0], + lower_triangular.LinearOperatorLowerTriangular) + ) + self.assertTrue( + isinstance( + cholesky_factor.operators[1], + lower_triangular.LinearOperatorLowerTriangular) + ) + def test_is_non_singular_auto_set(self): # Matrix with two positive eigenvalues, 11 and 8. # The matrix values do not effect auto-setting of the flags. diff --git a/tensorflow/python/kernel_tests/linalg/linear_operator_circulant_test.py b/tensorflow/python/kernel_tests/linalg/linear_operator_circulant_test.py index f1e151ebd86..6366083ac5b 100644 --- a/tensorflow/python/kernel_tests/linalg/linear_operator_circulant_test.py +++ b/tensorflow/python/kernel_tests/linalg/linear_operator_circulant_test.py @@ -21,12 +21,14 @@ import contextlib import numpy as np from tensorflow.python.framework import dtypes +from tensorflow.python.framework import test_util from tensorflow.python.ops import array_ops from tensorflow.python.ops import math_ops from tensorflow.python.ops import spectral_ops_test_util from tensorflow.python.ops.linalg import linalg from tensorflow.python.ops.linalg import linear_operator_circulant from tensorflow.python.ops.linalg import linear_operator_test_util +from tensorflow.python.ops.signal import fft_ops from tensorflow.python.platform import test rng = np.random.RandomState(0) @@ -75,8 +77,8 @@ class LinearOperatorCirculantBaseTest(object): x = np.zeros([domain_dimension]) # x is a basis vector. x[m] = 1.0 - fft_x = math_ops.fft(x.astype(np.complex64)) - h_convolve_x = math_ops.ifft(spectrum * fft_x) + fft_x = fft_ops.fft(x.astype(np.complex64)) + h_convolve_x = fft_ops.ifft(spectrum * fft_x) matrix_rows.append(h_convolve_x) matrix = array_ops.stack(matrix_rows, axis=-1) return math_ops.cast(matrix, dtype) @@ -97,7 +99,9 @@ class LinearOperatorCirculantTestSelfAdjointOperator( # real, the matrix will not be real. return [dtypes.complex64] - def _operator_and_matrix(self, build_info, dtype, use_placeholder): + def _operator_and_matrix( + self, build_info, dtype, use_placeholder, + ensure_self_adjoint_and_pd=False): shape = build_info.shape # For this test class, we are creating real spectrums. # We also want the spectrum to have eigenvalues bounded away from zero. @@ -105,6 +109,8 @@ class LinearOperatorCirculantTestSelfAdjointOperator( # spectrum is bounded away from zero. spectrum = linear_operator_test_util.random_sign_uniform( shape=self._shape_to_spectrum_shape(shape), minval=1., maxval=2.) + if ensure_self_adjoint_and_pd: + spectrum = math_ops.abs(spectrum) # If dtype is complex, cast spectrum to complex. The imaginary part will be # zero, so the operator will still be self-adjoint. spectrum = math_ops.cast(spectrum, dtype) @@ -115,12 +121,16 @@ class LinearOperatorCirculantTestSelfAdjointOperator( lin_op_spectrum = array_ops.placeholder_with_default(spectrum, shape=None) operator = linalg.LinearOperatorCirculant( - lin_op_spectrum, is_self_adjoint=True, input_output_dtype=dtype) + lin_op_spectrum, + is_self_adjoint=True, + is_positive_definite=True if ensure_self_adjoint_and_pd else None, + input_output_dtype=dtype) mat = self._spectrum_to_circulant_1d(spectrum, shape, dtype=dtype) return operator, mat + @test_util.run_deprecated_v1 def test_simple_hermitian_spectrum_gives_operator_with_zero_imag_part(self): with self.cached_session(): spectrum = math_ops.cast([1., 1j, -1j], dtypes.complex64) @@ -129,7 +139,8 @@ class LinearOperatorCirculantTestSelfAdjointOperator( matrix = operator.to_dense() imag_matrix = math_ops.imag(matrix) eps = np.finfo(np.float32).eps - np.testing.assert_allclose(0, imag_matrix.eval(), rtol=0, atol=eps * 3) + np.testing.assert_allclose( + 0, self.evaluate(imag_matrix), rtol=0, atol=eps * 3) class LinearOperatorCirculantTestHermitianSpectrum( @@ -146,7 +157,9 @@ class LinearOperatorCirculantTestHermitianSpectrum( def _dtypes_to_test(self): return [dtypes.float32, dtypes.complex64] - def _operator_and_matrix(self, build_info, dtype, use_placeholder): + def _operator_and_matrix( + self, build_info, dtype, use_placeholder, + ensure_self_adjoint_and_pd=False): shape = build_info.shape # For this test class, we are creating Hermitian spectrums. # We also want the spectrum to have eigenvalues bounded away from zero. @@ -160,14 +173,14 @@ class LinearOperatorCirculantTestHermitianSpectrum( # = IFFT[EvenPartOf[pre_spectrum]] # is the IFFT of something that is also bounded away from zero. # Therefore, FFT[pre_h] would be a well-conditioned spectrum. - pre_h = math_ops.ifft(pre_spectrum_c) + pre_h = fft_ops.ifft(pre_spectrum_c) # A spectrum is Hermitian iff it is the DFT of a real convolution kernel. # So we will make spectrum = FFT[h], for real valued h. h = math_ops.real(pre_h) h_c = _to_complex(h) - spectrum = math_ops.fft(h_c) + spectrum = fft_ops.fft(h_c) lin_op_spectrum = spectrum @@ -175,12 +188,17 @@ class LinearOperatorCirculantTestHermitianSpectrum( lin_op_spectrum = array_ops.placeholder_with_default(spectrum, shape=None) operator = linalg.LinearOperatorCirculant( - lin_op_spectrum, input_output_dtype=dtype) + lin_op_spectrum, + input_output_dtype=dtype, + is_positive_definite=True if ensure_self_adjoint_and_pd else None, + is_self_adjoint=True if ensure_self_adjoint_and_pd else None, + ) mat = self._spectrum_to_circulant_1d(spectrum, shape, dtype=dtype) return operator, mat + @test_util.run_deprecated_v1 def test_simple_hermitian_spectrum_gives_operator_with_zero_imag_part(self): with self.cached_session(): spectrum = math_ops.cast([1., 1j, -1j], dtypes.complex64) @@ -189,7 +207,8 @@ class LinearOperatorCirculantTestHermitianSpectrum( matrix = operator.to_dense() imag_matrix = math_ops.imag(matrix) eps = np.finfo(np.float32).eps - np.testing.assert_allclose(0, imag_matrix.eval(), rtol=0, atol=eps * 3) + np.testing.assert_allclose( + 0, self.evaluate(imag_matrix), rtol=0, atol=eps * 3) class LinearOperatorCirculantTestNonHermitianSpectrum( @@ -205,7 +224,16 @@ class LinearOperatorCirculantTestNonHermitianSpectrum( def _dtypes_to_test(self): return [dtypes.complex64] - def _operator_and_matrix(self, build_info, dtype, use_placeholder): + # Skip Cholesky since we are explicitly testing non-hermitian + # spectra. + @property + def _tests_to_skip(self): + return ["cholesky"] + + def _operator_and_matrix( + self, build_info, dtype, use_placeholder, + ensure_self_adjoint_and_pd=False): + del ensure_self_adjoint_and_pd shape = build_info.shape # Will be well conditioned enough to get accurate solves. spectrum = linear_operator_test_util.random_sign_uniform( @@ -226,6 +254,7 @@ class LinearOperatorCirculantTestNonHermitianSpectrum( return operator, mat + @test_util.run_deprecated_v1 def test_simple_hermitian_spectrum_gives_operator_with_zero_imag_part(self): with self.cached_session(): spectrum = math_ops.cast([1., 1j, -1j], dtypes.complex64) @@ -234,8 +263,10 @@ class LinearOperatorCirculantTestNonHermitianSpectrum( matrix = operator.to_dense() imag_matrix = math_ops.imag(matrix) eps = np.finfo(np.float32).eps - np.testing.assert_allclose(0, imag_matrix.eval(), rtol=0, atol=eps * 3) + np.testing.assert_allclose( + 0, self.evaluate(imag_matrix), rtol=0, atol=eps * 3) + @test_util.run_deprecated_v1 def test_simple_positive_real_spectrum_gives_self_adjoint_pos_def_oper(self): with self.cached_session() as sess: spectrum = math_ops.cast([6., 4, 2], dtypes.complex64) @@ -248,10 +279,11 @@ class LinearOperatorCirculantTestNonHermitianSpectrum( operator.assert_positive_definite().run() # Should not fail operator.assert_self_adjoint().run() # Should not fail + @test_util.run_deprecated_v1 def test_defining_operator_using_real_convolution_kernel(self): with self.cached_session(): convolution_kernel = [1., 2., 1.] - spectrum = math_ops.fft( + spectrum = fft_ops.fft( math_ops.cast(convolution_kernel, dtypes.complex64)) # spectrum is shape [3] ==> operator is shape [3, 3] @@ -269,15 +301,16 @@ class LinearOperatorCirculantTestNonHermitianSpectrum( # Make spectrum the FFT of a real convolution kernel h. This ensures that # spectrum is Hermitian. h = linear_operator_test_util.random_normal(shape=(3, 4)) - spectrum = math_ops.fft(math_ops.cast(h, dtypes.complex64)) + spectrum = fft_ops.fft(math_ops.cast(h, dtypes.complex64)) operator = linalg.LinearOperatorCirculant( spectrum, input_output_dtype=dtypes.complex64) matrix = operator.to_dense() imag_matrix = math_ops.imag(matrix) eps = np.finfo(np.float32).eps np.testing.assert_allclose( - 0, imag_matrix.eval(), rtol=0, atol=eps * 3 * 4) + 0, self.evaluate(imag_matrix), rtol=0, atol=eps * 3 * 4) + @test_util.run_deprecated_v1 def test_convolution_kernel_same_as_first_row_of_to_dense(self): spectrum = [[3., 2., 1.], [2., 1.5, 1.]] with self.cached_session(): @@ -287,8 +320,9 @@ class LinearOperatorCirculantTestNonHermitianSpectrum( self.assertAllEqual((2, 3), h.get_shape()) self.assertAllEqual((2, 3, 3), c.get_shape()) - self.assertAllClose(h.eval(), c.eval()[:, :, 0]) + self.assertAllClose(h.eval(), self.evaluate(c)[:, :, 0]) + @test_util.run_deprecated_v1 def test_assert_non_singular_fails_for_singular_operator(self): spectrum = math_ops.cast([0, 4, 2j + 2], dtypes.complex64) operator = linalg.LinearOperatorCirculant(spectrum) @@ -296,12 +330,14 @@ class LinearOperatorCirculantTestNonHermitianSpectrum( with self.assertRaisesOpError("Singular operator"): operator.assert_non_singular().run() + @test_util.run_deprecated_v1 def test_assert_non_singular_does_not_fail_for_non_singular_operator(self): spectrum = math_ops.cast([-3j, 4, 2j + 2], dtypes.complex64) operator = linalg.LinearOperatorCirculant(spectrum) with self.cached_session(): operator.assert_non_singular().run() # Should not fail + @test_util.run_deprecated_v1 def test_assert_positive_definite_fails_for_non_positive_definite(self): spectrum = math_ops.cast([6., 4, 2j], dtypes.complex64) operator = linalg.LinearOperatorCirculant(spectrum) @@ -309,6 +345,7 @@ class LinearOperatorCirculantTestNonHermitianSpectrum( with self.assertRaisesOpError("Not positive definite"): operator.assert_positive_definite().run() + @test_util.run_deprecated_v1 def test_assert_positive_definite_does_not_fail_when_pos_def(self): spectrum = math_ops.cast([6., 4, 2j + 2], dtypes.complex64) operator = linalg.LinearOperatorCirculant(spectrum) @@ -397,8 +434,8 @@ class LinearOperatorCirculant2DBaseTest(object): x = np.zeros(block_shape) # x is a basis vector. x[n0, n1] = 1.0 - fft_x = math_ops.fft2d(x.astype(np.complex64)) - h_convolve_x = math_ops.ifft2d(spectrum * fft_x) + fft_x = fft_ops.fft2d(x.astype(np.complex64)) + h_convolve_x = fft_ops.ifft2d(spectrum * fft_x) # We want the flat version of the action of the operator on a basis # vector, not the block version. h_convolve_x = array_ops.reshape(h_convolve_x, shape[:-1]) @@ -421,7 +458,9 @@ class LinearOperatorCirculant2DTestHermitianSpectrum( def _dtypes_to_test(self): return [dtypes.float32, dtypes.complex64] - def _operator_and_matrix(self, build_info, dtype, use_placeholder): + def _operator_and_matrix( + self, build_info, dtype, use_placeholder, + ensure_self_adjoint_and_pd=False): shape = build_info.shape # For this test class, we are creating Hermitian spectrums. # We also want the spectrum to have eigenvalues bounded away from zero. @@ -435,14 +474,14 @@ class LinearOperatorCirculant2DTestHermitianSpectrum( # = IFFT[EvenPartOf[pre_spectrum]] # is the IFFT of something that is also bounded away from zero. # Therefore, FFT[pre_h] would be a well-conditioned spectrum. - pre_h = math_ops.ifft2d(pre_spectrum_c) + pre_h = fft_ops.ifft2d(pre_spectrum_c) # A spectrum is Hermitian iff it is the DFT of a real convolution kernel. # So we will make spectrum = FFT[h], for real valued h. h = math_ops.real(pre_h) h_c = _to_complex(h) - spectrum = math_ops.fft2d(h_c) + spectrum = fft_ops.fft2d(h_c) lin_op_spectrum = spectrum @@ -450,7 +489,10 @@ class LinearOperatorCirculant2DTestHermitianSpectrum( lin_op_spectrum = array_ops.placeholder_with_default(spectrum, shape=None) operator = linalg.LinearOperatorCirculant2D( - lin_op_spectrum, input_output_dtype=dtype) + lin_op_spectrum, + is_positive_definite=True if ensure_self_adjoint_and_pd else None, + is_self_adjoint=True if ensure_self_adjoint_and_pd else None, + input_output_dtype=dtype) mat = self._spectrum_to_circulant_2d(spectrum, shape, dtype=dtype) @@ -470,7 +512,14 @@ class LinearOperatorCirculant2DTestNonHermitianSpectrum( def _dtypes_to_test(self): return [dtypes.complex64] - def _operator_and_matrix(self, build_info, dtype, use_placeholder): + @property + def _tests_to_skip(self): + return ["cholesky"] + + def _operator_and_matrix( + self, build_info, dtype, use_placeholder, + ensure_self_adjoint_and_pd=False): + del ensure_self_adjoint_and_pd shape = build_info.shape # Will be well conditioned enough to get accurate solves. spectrum = linear_operator_test_util.random_sign_uniform( @@ -491,6 +540,7 @@ class LinearOperatorCirculant2DTestNonHermitianSpectrum( return operator, mat + @test_util.run_deprecated_v1 def test_real_hermitian_spectrum_gives_real_symmetric_operator(self): with self.cached_session() as sess: # This is a real and hermitian spectrum. @@ -508,6 +558,7 @@ class LinearOperatorCirculant2DTestNonHermitianSpectrum( np.testing.assert_allclose(0, imag_matrix, atol=1e-6) self.assertAllClose(matrix, matrix_transpose, atol=0) + @test_util.run_deprecated_v1 def test_real_spectrum_gives_self_adjoint_operator(self): with self.cached_session() as sess: # This is a real and hermitian spectrum. @@ -519,9 +570,10 @@ class LinearOperatorCirculant2DTestNonHermitianSpectrum( self.assertEqual(matrix_tensor.dtype, linear_operator_circulant._DTYPE_COMPLEX) matrix_h = linalg.adjoint(matrix_tensor) - matrix, matrix_h = sess.run([matrix_tensor, matrix_h]) + matrix, matrix_h = self.evaluate([matrix_tensor, matrix_h]) self.assertAllClose(matrix, matrix_h, atol=0) + @test_util.run_deprecated_v1 def test_assert_non_singular_fails_for_singular_operator(self): spectrum = math_ops.cast([[0, 4], [2j + 2, 3.]], dtypes.complex64) operator = linalg.LinearOperatorCirculant2D(spectrum) @@ -529,12 +581,14 @@ class LinearOperatorCirculant2DTestNonHermitianSpectrum( with self.assertRaisesOpError("Singular operator"): operator.assert_non_singular().run() + @test_util.run_deprecated_v1 def test_assert_non_singular_does_not_fail_for_non_singular_operator(self): spectrum = math_ops.cast([[-3j, 4], [2j + 2, 3.]], dtypes.complex64) operator = linalg.LinearOperatorCirculant2D(spectrum) with self.cached_session(): operator.assert_non_singular().run() # Should not fail + @test_util.run_deprecated_v1 def test_assert_positive_definite_fails_for_non_positive_definite(self): spectrum = math_ops.cast([[6., 4], [2j, 3.]], dtypes.complex64) operator = linalg.LinearOperatorCirculant2D(spectrum) @@ -542,6 +596,7 @@ class LinearOperatorCirculant2DTestNonHermitianSpectrum( with self.assertRaisesOpError("Not positive definite"): operator.assert_positive_definite().run() + @test_util.run_deprecated_v1 def test_assert_positive_definite_does_not_fail_when_pos_def(self): spectrum = math_ops.cast([[6., 4], [2j + 2, 3.]], dtypes.complex64) operator = linalg.LinearOperatorCirculant2D(spectrum) @@ -580,6 +635,7 @@ class LinearOperatorCirculant3DTest(test.TestCase): with spectral_ops_test_util.fft_kernel_label_map(): yield sess + @test_util.run_deprecated_v1 def test_real_spectrum_gives_self_adjoint_operator(self): with self.cached_session() as sess: # This is a real and hermitian spectrum. @@ -593,16 +649,17 @@ class LinearOperatorCirculant3DTest(test.TestCase): linear_operator_circulant._DTYPE_COMPLEX) matrix_h = linalg.adjoint(matrix_tensor) - matrix, matrix_h = sess.run([matrix_tensor, matrix_h]) + matrix, matrix_h = self.evaluate([matrix_tensor, matrix_h]) self.assertAllEqual((2, 2 * 3 * 5, 2 * 3 * 5), matrix.shape) self.assertAllClose(matrix, matrix_h) + @test_util.run_deprecated_v1 def test_defining_operator_using_real_convolution_kernel(self): with self.cached_session(): convolution_kernel = linear_operator_test_util.random_normal( shape=(2, 2, 3, 5), dtype=dtypes.float32) # Convolution kernel is real ==> spectrum is Hermitian. - spectrum = math_ops.fft3d( + spectrum = fft_ops.fft3d( math_ops.cast(convolution_kernel, dtypes.complex64)) # spectrum is Hermitian ==> operator is real. @@ -615,6 +672,7 @@ class LinearOperatorCirculant3DTest(test.TestCase): self.assertAllEqual((2, 2 * 3 * 5, 2 * 3 * 5), matrix.shape) np.testing.assert_allclose(0, np.imag(matrix), atol=1e-6) + @test_util.run_deprecated_v1 def test_defining_spd_operator_by_taking_real_part(self): with self.cached_session() as sess: # S is real and positive. @@ -634,7 +692,7 @@ class LinearOperatorCirculant3DTest(test.TestCase): # = H1 + H2 # where H1 is real since it is Hermitian, # and H2 is imaginary since it is anti-Hermitian. - ifft_s = math_ops.ifft3d(math_ops.cast(s, dtypes.complex64)) + ifft_s = fft_ops.ifft3d(math_ops.cast(s, dtypes.complex64)) # Throw away H2, keep H1. real_ifft_s = math_ops.real(ifft_s) @@ -642,7 +700,7 @@ class LinearOperatorCirculant3DTest(test.TestCase): # This is the perfect spectrum! # spectrum = DFT[H1] # = S1, - fft_real_ifft_s = math_ops.fft3d( + fft_real_ifft_s = fft_ops.fft3d( math_ops.cast(real_ifft_s, dtypes.complex64)) # S1 is Hermitian ==> operator is real. @@ -665,7 +723,7 @@ class LinearOperatorCirculant3DTest(test.TestCase): # S2 is anti-Hermitian ==> operator is imaginary. # S2 is real ==> operator is self-adjoint. imag_ifft_s = math_ops.imag(ifft_s) - fft_imag_ifft_s = math_ops.fft3d( + fft_imag_ifft_s = fft_ops.fft3d( 1j * math_ops.cast(imag_ifft_s, dtypes.complex64)) operator_imag = linalg.LinearOperatorCirculant3D(fft_imag_ifft_s) diff --git a/tensorflow/python/kernel_tests/linalg/linear_operator_composition_test.py b/tensorflow/python/kernel_tests/linalg/linear_operator_composition_test.py index 02f56db5962..214b73aa2f3 100644 --- a/tensorflow/python/kernel_tests/linalg/linear_operator_composition_test.py +++ b/tensorflow/python/kernel_tests/linalg/linear_operator_composition_test.py @@ -21,6 +21,7 @@ import numpy as np from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops +from tensorflow.python.framework import test_util from tensorflow.python.ops import array_ops from tensorflow.python.ops import math_ops from tensorflow.python.ops.linalg import linalg as linalg_lib @@ -42,8 +43,12 @@ class SquareLinearOperatorCompositionTest( self._rtol[dtypes.float32] = 1e-4 self._rtol[dtypes.complex64] = 1e-4 + @property + def _tests_to_skip(self): + # Cholesky not implemented. + return ["cholesky"] + def _operator_and_matrix(self, build_info, dtype, use_placeholder): - sess = ops.get_default_session() shape = list(build_info.shape) # Either 1 or 2 matrices, depending. @@ -175,6 +180,7 @@ class NonSquareLinearOperatorCompositionTest( return operator, mat + @test_util.run_deprecated_v1 def test_static_shapes(self): operators = [ linalg.LinearOperatorFullMatrix(rng.rand(2, 3, 4)), @@ -183,6 +189,7 @@ class NonSquareLinearOperatorCompositionTest( operator = linalg.LinearOperatorComposition(operators) self.assertAllEqual((2, 3, 5), operator.shape) + @test_util.run_deprecated_v1 def test_shape_tensors_when_statically_available(self): operators = [ linalg.LinearOperatorFullMatrix(rng.rand(2, 3, 4)), @@ -192,6 +199,7 @@ class NonSquareLinearOperatorCompositionTest( with self.cached_session(): self.assertAllEqual((2, 3, 5), operator.shape_tensor().eval()) + @test_util.run_deprecated_v1 def test_shape_tensors_when_only_dynamically_available(self): mat_1 = rng.rand(1, 2, 3, 4) mat_2 = rng.rand(1, 2, 4, 5) diff --git a/tensorflow/python/kernel_tests/linalg/linear_operator_diag_test.py b/tensorflow/python/kernel_tests/linalg/linear_operator_diag_test.py index 0758349531e..dcbc0dd7c97 100644 --- a/tensorflow/python/kernel_tests/linalg/linear_operator_diag_test.py +++ b/tensorflow/python/kernel_tests/linalg/linear_operator_diag_test.py @@ -17,6 +17,7 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function +from tensorflow.python.framework import test_util from tensorflow.python.ops import array_ops from tensorflow.python.ops import linalg_ops from tensorflow.python.ops import math_ops @@ -32,17 +33,26 @@ class LinearOperatorDiagTest( linear_operator_test_util.SquareLinearOperatorDerivedClassTest): """Most tests done in the base class LinearOperatorDerivedClassTest.""" - def _operator_and_matrix(self, build_info, dtype, use_placeholder): + def _operator_and_matrix( + self, build_info, dtype, use_placeholder, + ensure_self_adjoint_and_pd=False): shape = list(build_info.shape) diag = linear_operator_test_util.random_sign_uniform( shape[:-1], minval=1., maxval=2., dtype=dtype) + if ensure_self_adjoint_and_pd: + # Abs on complex64 will result in a float32, so we cast back up. + diag = math_ops.cast(math_ops.abs(diag), dtype=dtype) + lin_op_diag = diag if use_placeholder: lin_op_diag = array_ops.placeholder_with_default(diag, shape=None) - operator = linalg.LinearOperatorDiag(lin_op_diag) + operator = linalg.LinearOperatorDiag( + lin_op_diag, + is_self_adjoint=True if ensure_self_adjoint_and_pd else None, + is_positive_definite=True if ensure_self_adjoint_and_pd else None) matrix = array_ops.matrix_diag(diag) @@ -71,6 +81,7 @@ class LinearOperatorDiagTest( with self.assertRaisesOpError("non-positive real.*not positive definite"): operator.assert_positive_definite().run() + @test_util.run_deprecated_v1 def test_assert_positive_definite_does_not_raise_if_pd_and_complex(self): with self.cached_session(): x = [1., 2.] @@ -87,6 +98,7 @@ class LinearOperatorDiagTest( with self.assertRaisesOpError("Singular operator"): operator.assert_non_singular().run() + @test_util.run_deprecated_v1 def test_assert_non_singular_does_not_raise_for_complex_nonsingular(self): with self.cached_session(): x = [1., 0.] @@ -104,6 +116,7 @@ class LinearOperatorDiagTest( with self.assertRaisesOpError("imaginary.*not self-adjoint"): operator.assert_self_adjoint().run() + @test_util.run_deprecated_v1 def test_assert_self_adjoint_does_not_raise_for_diag_with_zero_imag(self): with self.cached_session(): x = [1., 0.] @@ -138,12 +151,52 @@ class LinearOperatorDiagTest( operator_matmul = operator.matmul(x) mat_matmul = math_ops.matmul(mat, x) self.assertAllEqual(operator_matmul.get_shape(), mat_matmul.get_shape()) - self.assertAllClose(*sess.run([operator_matmul, mat_matmul])) + self.assertAllClose(*self.evaluate([operator_matmul, mat_matmul])) operator_solve = operator.solve(x) mat_solve = linalg_ops.matrix_solve(mat, x) self.assertAllEqual(operator_solve.get_shape(), mat_solve.get_shape()) - self.assertAllClose(*sess.run([operator_solve, mat_solve])) + self.assertAllClose(*self.evaluate([operator_solve, mat_solve])) + + def test_diag_matmul(self): + operator1 = linalg_lib.LinearOperatorDiag([2., 3.]) + operator2 = linalg_lib.LinearOperatorDiag([1., 2.]) + operator3 = linalg_lib.LinearOperatorScaledIdentity( + num_rows=2, multiplier=3.) + operator_matmul = operator1.matmul(operator2) + self.assertTrue(isinstance( + operator_matmul, + linalg_lib.LinearOperatorDiag)) + self.assertAllClose([2., 6.], self.evaluate(operator_matmul.diag)) + + operator_matmul = operator2.matmul(operator1) + self.assertTrue(isinstance( + operator_matmul, + linalg_lib.LinearOperatorDiag)) + self.assertAllClose([2., 6.], self.evaluate(operator_matmul.diag)) + + operator_matmul = operator1.matmul(operator3) + self.assertTrue(isinstance( + operator_matmul, + linalg_lib.LinearOperatorDiag)) + self.assertAllClose([6., 9.], self.evaluate(operator_matmul.diag)) + + operator_matmul = operator3.matmul(operator1) + self.assertTrue(isinstance( + operator_matmul, + linalg_lib.LinearOperatorDiag)) + self.assertAllClose([6., 9.], self.evaluate(operator_matmul.diag)) + + def test_diag_cholesky_type(self): + diag = [1., 3., 5., 8.] + operator = linalg.LinearOperatorDiag( + diag, + is_positive_definite=True, + is_self_adjoint=True, + ) + self.assertTrue(isinstance( + operator.cholesky(), + linalg.LinearOperatorDiag)) if __name__ == "__main__": diff --git a/tensorflow/python/kernel_tests/linalg/linear_operator_full_matrix_test.py b/tensorflow/python/kernel_tests/linalg/linear_operator_full_matrix_test.py index 8c2d2cf0774..aff0b1ae14c 100644 --- a/tensorflow/python/kernel_tests/linalg/linear_operator_full_matrix_test.py +++ b/tensorflow/python/kernel_tests/linalg/linear_operator_full_matrix_test.py @@ -20,6 +20,7 @@ from __future__ import print_function import numpy as np from tensorflow.python.framework import dtypes +from tensorflow.python.framework import test_util from tensorflow.python.ops import array_ops from tensorflow.python.ops import math_ops from tensorflow.python.ops.linalg import linalg as linalg_lib @@ -33,7 +34,9 @@ class SquareLinearOperatorFullMatrixTest( linear_operator_test_util.SquareLinearOperatorDerivedClassTest): """Most tests done in the base class LinearOperatorDerivedClassTest.""" - def _operator_and_matrix(self, build_info, dtype, use_placeholder): + def _operator_and_matrix( + self, build_info, dtype, use_placeholder, + ensure_self_adjoint_and_pd=False): shape = list(build_info.shape) matrix = linear_operator_test_util.random_positive_definite_matrix( @@ -44,7 +47,12 @@ class SquareLinearOperatorFullMatrixTest( if use_placeholder: lin_op_matrix = array_ops.placeholder_with_default(matrix, shape=None) - operator = linalg.LinearOperatorFullMatrix(lin_op_matrix, is_square=True) + # Set the hints to none to test non-symmetric PD code paths. + operator = linalg.LinearOperatorFullMatrix( + lin_op_matrix, + is_square=True, + is_self_adjoint=True if ensure_self_adjoint_and_pd else None, + is_positive_definite=True if ensure_self_adjoint_and_pd else None) return operator, matrix @@ -62,6 +70,7 @@ class SquareLinearOperatorFullMatrixTest( # Auto-detected. self.assertTrue(operator.is_square) + @test_util.run_deprecated_v1 def test_assert_non_singular_raises_if_cond_too_big_but_finite(self): with self.cached_session(): tril = linear_operator_test_util.random_tril_matrix( @@ -123,7 +132,13 @@ class SquareLinearOperatorFullMatrixSymmetricPositiveDefiniteTest( def _dtypes_to_test(self): return [dtypes.float32, dtypes.float64] - def _operator_and_matrix(self, build_info, dtype, use_placeholder): + def _operator_and_matrix( + self, build_info, dtype, use_placeholder, + ensure_self_adjoint_and_pd=False): + + # Matrix is always symmetric and positive definite in this class. + del ensure_self_adjoint_and_pd + shape = list(build_info.shape) matrix = linear_operator_test_util.random_positive_definite_matrix( @@ -134,7 +149,11 @@ class SquareLinearOperatorFullMatrixSymmetricPositiveDefiniteTest( if use_placeholder: lin_op_matrix = array_ops.placeholder_with_default(matrix, shape=None) - operator = linalg.LinearOperatorFullMatrix(lin_op_matrix, is_square=True) + operator = linalg.LinearOperatorFullMatrix( + lin_op_matrix, + is_square=True, + is_self_adjoint=True, + is_positive_definite=True) return operator, matrix diff --git a/tensorflow/python/kernel_tests/linalg/linear_operator_identity_test.py b/tensorflow/python/kernel_tests/linalg/linear_operator_identity_test.py index 465a8194dd9..2da5e712d77 100644 --- a/tensorflow/python/kernel_tests/linalg/linear_operator_identity_test.py +++ b/tensorflow/python/kernel_tests/linalg/linear_operator_identity_test.py @@ -20,8 +20,10 @@ from __future__ import print_function import numpy as np from tensorflow.python.framework import dtypes +from tensorflow.python.framework import test_util from tensorflow.python.ops import array_ops from tensorflow.python.ops import linalg_ops +from tensorflow.python.ops import math_ops from tensorflow.python.ops import random_ops from tensorflow.python.ops.linalg import linalg as linalg_lib from tensorflow.python.ops.linalg import linear_operator_test_util @@ -41,7 +43,12 @@ class LinearOperatorIdentityTest( # 16bit. return [dtypes.float32, dtypes.float64, dtypes.complex64, dtypes.complex128] - def _operator_and_matrix(self, build_info, dtype, use_placeholder): + def _operator_and_matrix( + self, build_info, dtype, use_placeholder, + ensure_self_adjoint_and_pd=False): + # Identity matrix is already Hermitian Positive Definite. + del ensure_self_adjoint_and_pd + shape = list(build_info.shape) assert shape[-1] == shape[-2] @@ -54,16 +61,19 @@ class LinearOperatorIdentityTest( return operator, mat + @test_util.run_deprecated_v1 def test_assert_positive_definite(self): with self.cached_session(): operator = linalg_lib.LinearOperatorIdentity(num_rows=2) operator.assert_positive_definite().run() # Should not fail + @test_util.run_deprecated_v1 def test_assert_non_singular(self): with self.cached_session(): operator = linalg_lib.LinearOperatorIdentity(num_rows=2) operator.assert_non_singular().run() # Should not fail + @test_util.run_deprecated_v1 def test_assert_self_adjoint(self): with self.cached_session(): operator = linalg_lib.LinearOperatorIdentity(num_rows=2) @@ -77,7 +87,7 @@ class LinearOperatorIdentityTest( num_rows=2, dtype=dtypes.float16) x = rng.randn(2, 3).astype(np.float16) y = operator.matmul(x) - self.assertAllClose(x, y.eval()) + self.assertAllClose(x, self.evaluate(y)) def test_non_scalar_num_rows_raises_static(self): with self.assertRaisesRegexp(ValueError, "must be a 0-D Tensor"): @@ -103,6 +113,7 @@ class LinearOperatorIdentityTest( with self.assertRaisesRegexp(ValueError, "must be non-negative"): linalg_lib.LinearOperatorIdentity(num_rows=2, batch_shape=[-2]) + @test_util.run_deprecated_v1 def test_non_scalar_num_rows_raises_dynamic(self): with self.cached_session(): num_rows = array_ops.placeholder(dtypes.int32) @@ -111,6 +122,7 @@ class LinearOperatorIdentityTest( with self.assertRaisesOpError("must be a 0-D Tensor"): operator.to_dense().eval(feed_dict={num_rows: [2]}) + @test_util.run_deprecated_v1 def test_negative_num_rows_raises_dynamic(self): with self.cached_session(): num_rows = array_ops.placeholder(dtypes.int32) @@ -119,6 +131,7 @@ class LinearOperatorIdentityTest( with self.assertRaisesOpError("must be non-negative"): operator.to_dense().eval(feed_dict={num_rows: -2}) + @test_util.run_deprecated_v1 def test_non_1d_batch_shape_raises_dynamic(self): with self.cached_session(): batch_shape = array_ops.placeholder(dtypes.int32) @@ -127,6 +140,7 @@ class LinearOperatorIdentityTest( with self.assertRaisesOpError("must be a 1-D"): operator.to_dense().eval(feed_dict={batch_shape: 2}) + @test_util.run_deprecated_v1 def test_negative_batch_shape_raises_dynamic(self): with self.cached_session(): batch_shape = array_ops.placeholder(dtypes.int32) @@ -141,6 +155,7 @@ class LinearOperatorIdentityTest( with self.assertRaisesRegexp(ValueError, "Dimensions.*not compatible"): operator.matmul(x) + @test_util.run_deprecated_v1 def test_wrong_matrix_dimensions_raises_dynamic(self): num_rows = array_ops.placeholder(dtypes.int32) x = array_ops.placeholder(dtypes.float32) @@ -164,8 +179,9 @@ class LinearOperatorIdentityTest( expected = x self.assertAllEqual(operator_matmul.get_shape(), expected.get_shape()) - self.assertAllClose(*sess.run([operator_matmul, expected])) + self.assertAllClose(*self.evaluate([operator_matmul, expected])) + @test_util.run_deprecated_v1 def test_default_batch_shape_broadcasts_with_everything_dynamic(self): # These cannot be done in the automated (base test class) tests since they # test shapes that tf.batch_matmul cannot handle. @@ -201,8 +217,9 @@ class LinearOperatorIdentityTest( operator_matmul = operator.matmul(x) self.assertAllEqual(operator_matmul.get_shape(), expected.get_shape()) - self.assertAllClose(*sess.run([operator_matmul, expected])) + self.assertAllClose(*self.evaluate([operator_matmul, expected])) + @test_util.run_deprecated_v1 def test_broadcast_matmul_dynamic_shapes(self): # These cannot be done in the automated (base test class) tests since they # test shapes that tf.batch_matmul cannot handle. @@ -242,6 +259,16 @@ class LinearOperatorIdentityTest( is_non_singular=None, ) + def test_identity_cholesky_type(self): + operator = linalg_lib.LinearOperatorIdentity( + num_rows=2, + is_positive_definite=True, + is_self_adjoint=True, + ) + self.assertTrue(isinstance( + operator.cholesky(), + linalg_lib.LinearOperatorIdentity)) + class LinearOperatorScaledIdentityTest( linear_operator_test_util.SquareLinearOperatorDerivedClassTest): @@ -253,7 +280,10 @@ class LinearOperatorScaledIdentityTest( # 16bit. return [dtypes.float32, dtypes.float64, dtypes.complex64, dtypes.complex128] - def _operator_and_matrix(self, build_info, dtype, use_placeholder): + def _operator_and_matrix( + self, build_info, dtype, use_placeholder, + ensure_self_adjoint_and_pd=False): + shape = list(build_info.shape) assert shape[-1] == shape[-2] @@ -266,6 +296,9 @@ class LinearOperatorScaledIdentityTest( multiplier = linear_operator_test_util.random_sign_uniform( shape=batch_shape, minval=1., maxval=2., dtype=dtype) + if ensure_self_adjoint_and_pd: + # Abs on complex64 will result in a float32, so we cast back up. + multiplier = math_ops.cast(math_ops.abs(multiplier), dtype=dtype) # Nothing to feed since LinearOperatorScaledIdentity takes no Tensor args. lin_op_multiplier = multiplier @@ -275,7 +308,10 @@ class LinearOperatorScaledIdentityTest( multiplier, shape=None) operator = linalg_lib.LinearOperatorScaledIdentity( - num_rows, lin_op_multiplier) + num_rows, + lin_op_multiplier, + is_self_adjoint=True if ensure_self_adjoint_and_pd else None, + is_positive_definite=True if ensure_self_adjoint_and_pd else None) multiplier_matrix = array_ops.expand_dims( array_ops.expand_dims(multiplier, -1), -1) @@ -284,6 +320,7 @@ class LinearOperatorScaledIdentityTest( return operator, matrix + @test_util.run_deprecated_v1 def test_assert_positive_definite_does_not_raise_when_positive(self): with self.cached_session(): operator = linalg_lib.LinearOperatorScaledIdentity( @@ -297,6 +334,7 @@ class LinearOperatorScaledIdentityTest( with self.assertRaisesOpError("not positive definite"): operator.assert_positive_definite().run() + @test_util.run_deprecated_v1 def test_assert_non_singular_does_not_raise_when_non_singular(self): with self.cached_session(): operator = linalg_lib.LinearOperatorScaledIdentity( @@ -310,6 +348,7 @@ class LinearOperatorScaledIdentityTest( with self.assertRaisesOpError("was singular"): operator.assert_non_singular().run() + @test_util.run_deprecated_v1 def test_assert_self_adjoint_does_not_raise_when_self_adjoint(self): with self.cached_session(): operator = linalg_lib.LinearOperatorScaledIdentity( @@ -332,7 +371,7 @@ class LinearOperatorScaledIdentityTest( num_rows=2, multiplier=multiplier) x = rng.randn(2, 3).astype(np.float16) y = operator.matmul(x) - self.assertAllClose(multiplier[..., None, None] * x, y.eval()) + self.assertAllClose(multiplier[..., None, None] * x, self.evaluate(y)) def test_non_scalar_num_rows_raises_static(self): # Many "test_...num_rows" tests are performed in LinearOperatorIdentity. @@ -347,6 +386,7 @@ class LinearOperatorScaledIdentityTest( with self.assertRaisesRegexp(ValueError, "Dimensions.*not compatible"): operator.matmul(x) + @test_util.run_deprecated_v1 def test_wrong_matrix_dimensions_raises_dynamic(self): num_rows = array_ops.placeholder(dtypes.int32) x = array_ops.placeholder(dtypes.float32) @@ -378,13 +418,13 @@ class LinearOperatorScaledIdentityTest( expected = x * 2.2 + zeros operator_matmul = operator.matmul(x) self.assertAllEqual(operator_matmul.get_shape(), expected.get_shape()) - self.assertAllClose(*sess.run([operator_matmul, expected])) + self.assertAllClose(*self.evaluate([operator_matmul, expected])) # Test solve expected = x / 2.2 + zeros operator_solve = operator.solve(x) self.assertAllEqual(operator_solve.get_shape(), expected.get_shape()) - self.assertAllClose(*sess.run([operator_solve, expected])) + self.assertAllClose(*self.evaluate([operator_solve, expected])) def test_broadcast_matmul_and_solve_scalar_scale_multiplier(self): # These cannot be done in the automated (base test class) tests since they @@ -404,13 +444,13 @@ class LinearOperatorScaledIdentityTest( expected = x * 2.2 operator_matmul = operator.matmul(x) self.assertAllEqual(operator_matmul.get_shape(), expected.get_shape()) - self.assertAllClose(*sess.run([operator_matmul, expected])) + self.assertAllClose(*self.evaluate([operator_matmul, expected])) # Test solve expected = x / 2.2 operator_solve = operator.solve(x) self.assertAllEqual(operator_solve.get_shape(), expected.get_shape()) - self.assertAllClose(*sess.run([operator_solve, expected])) + self.assertAllClose(*self.evaluate([operator_solve, expected])) def test_is_x_flags(self): operator = linalg_lib.LinearOperatorScaledIdentity( @@ -420,6 +460,41 @@ class LinearOperatorScaledIdentityTest( self.assertTrue(operator.is_non_singular) self.assertTrue(operator.is_self_adjoint is None) + def test_identity_matmul(self): + operator1 = linalg_lib.LinearOperatorIdentity(num_rows=2) + operator2 = linalg_lib.LinearOperatorScaledIdentity( + num_rows=2, multiplier=3.) + self.assertTrue(isinstance( + operator1.matmul(operator1), + linalg_lib.LinearOperatorIdentity)) + + self.assertTrue(isinstance( + operator1.matmul(operator1), + linalg_lib.LinearOperatorIdentity)) + + operator_matmul = operator1.matmul(operator2) + self.assertTrue(isinstance( + operator_matmul, + linalg_lib.LinearOperatorScaledIdentity)) + self.assertAllClose(3., self.evaluate(operator_matmul.multiplier)) + + operator_matmul = operator2.matmul(operator1) + self.assertTrue(isinstance( + operator_matmul, + linalg_lib.LinearOperatorScaledIdentity)) + self.assertAllClose(3., self.evaluate(operator_matmul.multiplier)) + + def test_scaled_identity_cholesky_type(self): + operator = linalg_lib.LinearOperatorScaledIdentity( + num_rows=2, + multiplier=3., + is_positive_definite=True, + is_self_adjoint=True, + ) + self.assertTrue(isinstance( + operator.cholesky(), + linalg_lib.LinearOperatorScaledIdentity)) + if __name__ == "__main__": test.main() diff --git a/tensorflow/python/kernel_tests/linalg/linear_operator_inversion_test.py b/tensorflow/python/kernel_tests/linalg/linear_operator_inversion_test.py new file mode 100644 index 00000000000..9344c526ee8 --- /dev/null +++ b/tensorflow/python/kernel_tests/linalg/linear_operator_inversion_test.py @@ -0,0 +1,130 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from tensorflow.python.framework import dtypes +from tensorflow.python.ops import array_ops +from tensorflow.python.ops.linalg import linalg as linalg_lib +from tensorflow.python.ops.linalg import linear_operator_inversion +from tensorflow.python.ops.linalg import linear_operator_test_util +from tensorflow.python.platform import test + +linalg = linalg_lib + +LinearOperatorInversion = linear_operator_inversion.LinearOperatorInversion # pylint: disable=invalid-name + + +class LinearOperatorInversionTest( + linear_operator_test_util.SquareLinearOperatorDerivedClassTest): + """Most tests done in the base class LinearOperatorDerivedClassTest.""" + + def setUp(self): + self._atol[dtypes.complex64] = 1e-5 + self._rtol[dtypes.complex64] = 1e-5 + + def _operator_and_matrix(self, + build_info, + dtype, + use_placeholder, + ensure_self_adjoint_and_pd=False): + shape = list(build_info.shape) + + if ensure_self_adjoint_and_pd: + matrix = linear_operator_test_util.random_positive_definite_matrix( + shape, dtype, force_well_conditioned=True) + else: + matrix = linear_operator_test_util.random_tril_matrix( + shape, dtype, force_well_conditioned=True, remove_upper=True) + + lin_op_matrix = matrix + + if use_placeholder: + lin_op_matrix = array_ops.placeholder_with_default(matrix, shape=None) + + if ensure_self_adjoint_and_pd: + operator = LinearOperatorInversion( + linalg.LinearOperatorFullMatrix( + lin_op_matrix, is_positive_definite=True, is_self_adjoint=True)) + else: + operator = LinearOperatorInversion( + linalg.LinearOperatorLowerTriangular(lin_op_matrix)) + + return operator, linalg.inv(matrix) + + def test_base_operator_hint_used(self): + # The matrix values do not effect auto-setting of the flags. + matrix = [[1., 0.], [1., 1.]] + operator = linalg.LinearOperatorFullMatrix( + matrix, + is_positive_definite=True, + is_non_singular=True, + is_self_adjoint=False) + operator_inv = LinearOperatorInversion(operator) + self.assertTrue(operator_inv.is_positive_definite) + self.assertTrue(operator_inv.is_non_singular) + self.assertFalse(operator_inv.is_self_adjoint) + + def test_supplied_hint_used(self): + # The matrix values do not effect auto-setting of the flags. + matrix = [[1., 0.], [1., 1.]] + operator = linalg.LinearOperatorFullMatrix(matrix) + operator_inv = LinearOperatorInversion( + operator, + is_positive_definite=True, + is_non_singular=True, + is_self_adjoint=False) + self.assertTrue(operator_inv.is_positive_definite) + self.assertTrue(operator_inv.is_non_singular) + self.assertFalse(operator_inv.is_self_adjoint) + + def test_contradicting_hints_raise(self): + # The matrix values do not effect auto-setting of the flags. + matrix = [[1., 0.], [1., 1.]] + operator = linalg.LinearOperatorFullMatrix( + matrix, is_positive_definite=False) + with self.assertRaisesRegexp(ValueError, "positive-definite"): + LinearOperatorInversion(operator, is_positive_definite=True) + + operator = linalg.LinearOperatorFullMatrix(matrix, is_self_adjoint=False) + with self.assertRaisesRegexp(ValueError, "self-adjoint"): + LinearOperatorInversion(operator, is_self_adjoint=True) + + def test_singular_raises(self): + # The matrix values do not effect auto-setting of the flags. + matrix = [[1., 1.], [1., 1.]] + + operator = linalg.LinearOperatorFullMatrix(matrix, is_non_singular=False) + with self.assertRaisesRegexp(ValueError, "is_non_singular"): + LinearOperatorInversion(operator) + + operator = linalg.LinearOperatorFullMatrix(matrix) + with self.assertRaisesRegexp(ValueError, "is_non_singular"): + LinearOperatorInversion(operator, is_non_singular=False) + + def test_name(self): + matrix = [[11., 0.], [1., 8.]] + operator = linalg.LinearOperatorFullMatrix( + matrix, name="my_operator", is_non_singular=True) + + operator = LinearOperatorInversion(operator) + + self.assertEqual("my_operator_inv", operator.name) + + +if __name__ == "__main__": + test.main() diff --git a/tensorflow/python/kernel_tests/linalg/linear_operator_kronecker_test.py b/tensorflow/python/kernel_tests/linalg/linear_operator_kronecker_test.py index f039b60f648..513b2468032 100644 --- a/tensorflow/python/kernel_tests/linalg/linear_operator_kronecker_test.py +++ b/tensorflow/python/kernel_tests/linalg/linear_operator_kronecker_test.py @@ -21,9 +21,11 @@ import numpy as np from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops +from tensorflow.python.framework import test_util from tensorflow.python.ops import array_ops from tensorflow.python.ops.linalg import linalg as linalg_lib from tensorflow.python.ops.linalg import linear_operator_kronecker as kronecker +from tensorflow.python.ops.linalg import linear_operator_lower_triangular as lower_triangular from tensorflow.python.ops.linalg import linear_operator_test_util from tensorflow.python.ops.linalg import linear_operator_util from tensorflow.python.platform import test @@ -52,6 +54,7 @@ def _kronecker_dense(factors): class KroneckerDenseTest(test.TestCase): + @test_util.run_deprecated_v1 def testKroneckerDenseMatrix(self): x = ops.convert_to_tensor([[2., 3.], [1., 2.]], dtype=dtypes.float32) y = ops.convert_to_tensor([[1., 2.], [5., -1.]], dtype=dtypes.float32) @@ -69,8 +72,8 @@ class KroneckerDenseTest(test.TestCase): [5., 10., -1., -2.]], dtype=dtypes.float32) with self.cached_session(): - self.assertAllClose(_kronecker_dense([x, y]).eval(), z.eval()) - self.assertAllClose(_kronecker_dense([y, x]).eval(), w.eval()) + self.assertAllClose(_kronecker_dense([x, y]).eval(), self.evaluate(z)) + self.assertAllClose(_kronecker_dense([y, x]).eval(), self.evaluate(w)) class SquareLinearOperatorKroneckerTest( @@ -99,7 +102,12 @@ class SquareLinearOperatorKroneckerTest( def _tests_to_skip(self): return ["det", "solve", "solve_with_broadcast"] - def _operator_and_matrix(self, build_info, dtype, use_placeholder): + def _operator_and_matrix( + self, build_info, dtype, use_placeholder, + ensure_self_adjoint_and_pd=False): + # Kronecker products constructed below will be from symmetric + # positive-definite matrices. + del ensure_self_adjoint_and_pd shape = list(build_info.shape) expected_factors = build_info.__dict__["factors"] matrices = [ @@ -116,7 +124,11 @@ class SquareLinearOperatorKroneckerTest( operator = kronecker.LinearOperatorKronecker( [linalg.LinearOperatorFullMatrix( - l, is_square=True) for l in lin_op_matrices]) + l, + is_square=True, + is_self_adjoint=True, + is_positive_definite=True) + for l in lin_op_matrices]) matrices = linear_operator_util.broadcast_matrix_batch_dims(matrices) @@ -180,6 +192,40 @@ class SquareLinearOperatorKroneckerTest( with self.assertRaisesRegexp(ValueError, ">=1 operators"): kronecker.LinearOperatorKronecker([]) + def test_kronecker_cholesky_type(self): + matrix = [[1., 0.], [0., 1.]] + operator = kronecker.LinearOperatorKronecker( + [ + linalg.LinearOperatorFullMatrix( + matrix, + is_positive_definite=True, + is_self_adjoint=True, + ), + linalg.LinearOperatorFullMatrix( + matrix, + is_positive_definite=True, + is_self_adjoint=True, + ), + ], + is_positive_definite=True, + is_self_adjoint=True, + ) + cholesky_factor = operator.cholesky() + self.assertTrue(isinstance( + cholesky_factor, + kronecker.LinearOperatorKronecker)) + self.assertEqual(2, len(cholesky_factor.operators)) + self.assertTrue( + isinstance( + cholesky_factor.operators[0], + lower_triangular.LinearOperatorLowerTriangular) + ) + self.assertTrue( + isinstance( + cholesky_factor.operators[1], + lower_triangular.LinearOperatorLowerTriangular) + ) + if __name__ == "__main__": test.main() diff --git a/tensorflow/python/kernel_tests/linalg/linear_operator_low_rank_update_test.py b/tensorflow/python/kernel_tests/linalg/linear_operator_low_rank_update_test.py index 207e5edf818..2920f3ae7eb 100644 --- a/tensorflow/python/kernel_tests/linalg/linear_operator_low_rank_update_test.py +++ b/tensorflow/python/kernel_tests/linalg/linear_operator_low_rank_update_test.py @@ -69,7 +69,8 @@ class BaseLinearOperatorLowRankUpdatetest(object): return linear_operator_test_util.random_uniform( diag_shape, minval=1e-4, maxval=1., dtype=dtype) - def _operator_and_matrix(self, build_info, dtype, use_placeholder): + def _operator_and_matrix(self, build_info, dtype, use_placeholder, + ensure_self_adjoint_and_pd=False): # Recall A = L + UDV^H shape = list(build_info.shape) diag_shape = shape[:-1] @@ -93,7 +94,7 @@ class BaseLinearOperatorLowRankUpdatetest(object): lin_op_v = v # D - if self._is_diag_update_positive: + if self._is_diag_update_positive or ensure_self_adjoint_and_pd: diag_update = self._gen_positive_diag(dtype, diag_update_shape) else: diag_update = linear_operator_test_util.random_normal( @@ -178,6 +179,10 @@ class LinearOperatorLowRankUpdatetestWithDiagCannotUseCholesky( linear_operator_test_util.SquareLinearOperatorDerivedClassTest): """A = L + UDU^H, D !> 0, L > 0 ==> A !> 0 and we cannot use a Cholesky.""" + @property + def _tests_to_skip(self): + return ["cholesky"] + _use_diag_update = True _is_diag_update_positive = False _use_v = False @@ -217,6 +222,10 @@ class LinearOperatorLowRankUpdatetestNoDiagCannotUseCholesky( linear_operator_test_util.SquareLinearOperatorDerivedClassTest): """A = L + UV^H, L > 0 ==> A is not symmetric and we cannot use a Cholesky.""" + @property + def _tests_to_skip(self): + return ["cholesky"] + _use_diag_update = False _is_diag_update_positive = None _use_v = True diff --git a/tensorflow/python/kernel_tests/linalg/linear_operator_lower_triangular_test.py b/tensorflow/python/kernel_tests/linalg/linear_operator_lower_triangular_test.py index e3c8f5cb688..bd41f9ed9d3 100644 --- a/tensorflow/python/kernel_tests/linalg/linear_operator_lower_triangular_test.py +++ b/tensorflow/python/kernel_tests/linalg/linear_operator_lower_triangular_test.py @@ -18,6 +18,7 @@ from __future__ import division from __future__ import print_function from tensorflow.python.ops import array_ops +from tensorflow.python.ops import math_ops from tensorflow.python.ops.linalg import linalg as linalg_lib from tensorflow.python.ops.linalg import linear_operator_test_util from tensorflow.python.platform import test @@ -29,6 +30,11 @@ class LinearOperatorLowerTriangularTest( linear_operator_test_util.SquareLinearOperatorDerivedClassTest): """Most tests done in the base class LinearOperatorDerivedClassTest.""" + @property + def _tests_to_skip(self): + # Cholesky does not make sense for triangular matrices. + return ["cholesky"] + def _operator_and_matrix(self, build_info, dtype, use_placeholder): shape = list(build_info.shape) # Upper triangle will be nonzero, but ignored. @@ -71,6 +77,30 @@ class LinearOperatorLowerTriangularTest( with self.assertRaisesRegexp(ValueError, "at least 2 dimensions"): linalg.LinearOperatorLowerTriangular([1.]) + def test_triangular_diag_matmul(self): + operator1 = linalg_lib.LinearOperatorLowerTriangular( + [[1., 0., 0.], [2., 1., 0.], [2., 3., 3.]]) + operator2 = linalg_lib.LinearOperatorDiag([2., 2., 3.]) + operator_matmul = operator1.matmul(operator2) + self.assertTrue(isinstance( + operator_matmul, + linalg_lib.LinearOperatorLowerTriangular)) + self.assertAllClose( + math_ops.matmul( + operator1.to_dense(), + operator2.to_dense()), + self.evaluate(operator_matmul.to_dense())) + + operator_matmul = operator2.matmul(operator1) + self.assertTrue(isinstance( + operator_matmul, + linalg_lib.LinearOperatorLowerTriangular)) + self.assertAllClose( + math_ops.matmul( + operator2.to_dense(), + operator1.to_dense()), + self.evaluate(operator_matmul.to_dense())) + if __name__ == "__main__": test.main() diff --git a/tensorflow/python/kernel_tests/linalg/linear_operator_test.py b/tensorflow/python/kernel_tests/linalg/linear_operator_test.py index 819347343b1..8f8b15e8ed8 100644 --- a/tensorflow/python/kernel_tests/linalg/linear_operator_test.py +++ b/tensorflow/python/kernel_tests/linalg/linear_operator_test.py @@ -22,6 +22,7 @@ from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops from tensorflow.python.framework import tensor_shape +from tensorflow.python.framework import test_util from tensorflow.python.ops import array_ops from tensorflow.python.ops import linalg_ops from tensorflow.python.ops import math_ops @@ -107,6 +108,7 @@ class LinearOperatorTest(test.TestCase): self.assertAllEqual(4, operator.domain_dimension) self.assertAllEqual(3, operator.range_dimension) + @test_util.run_deprecated_v1 def test_all_shape_methods_defined_by_the_one_method_shape(self): with self.cached_session(): shape = (1, 2, 3, 4) @@ -134,8 +136,9 @@ class LinearOperatorTest(test.TestCase): with self.cached_session(): operator_dense = operator.to_dense() self.assertAllEqual((2, 3, 4), operator_dense.get_shape()) - self.assertAllClose(matrix, operator_dense.eval()) + self.assertAllClose(matrix, self.evaluate(operator_dense)) + @test_util.run_deprecated_v1 def test_generic_to_dense_method_non_square_matrix_tensor(self): matrix = rng.randn(2, 3, 4) matrix_ph = array_ops.placeholder(dtypes.float64) @@ -152,7 +155,7 @@ class LinearOperatorTest(test.TestCase): with self.cached_session(): y = operator.matvec(x) self.assertAllEqual((2,), y.get_shape()) - self.assertAllClose([1., 2.], y.eval()) + self.assertAllClose([1., 2.], self.evaluate(y)) def test_solvevec(self): matrix = [[1., 0], [0., 2.]] @@ -161,7 +164,7 @@ class LinearOperatorTest(test.TestCase): with self.cached_session(): x = operator.solvevec(y) self.assertAllEqual((2,), x.get_shape()) - self.assertAllClose([1., 1 / 2.], x.eval()) + self.assertAllClose([1., 1 / 2.], self.evaluate(x)) def test_is_square_set_to_true_for_square_static_shapes(self): operator = LinearOperatorShape(shape=(2, 4, 4)) @@ -175,6 +178,7 @@ class LinearOperatorTest(test.TestCase): with self.assertRaisesRegexp(ValueError, "but.*was square"): _ = LinearOperatorShape(shape=(2, 4, 4), is_square=False).is_square + @test_util.run_deprecated_v1 def test_is_square_set_inconsistent_with_other_hints_raises(self): with self.assertRaisesRegexp(ValueError, "is always square"): matrix = array_ops.placeholder(dtypes.float32) @@ -185,6 +189,7 @@ class LinearOperatorTest(test.TestCase): LinearOperatorMatmulSolve( matrix, is_positive_definite=True, is_square=False) + @test_util.run_deprecated_v1 def test_non_square_operators_raise_on_determinant_and_solve(self): operator = LinearOperatorShape((2, 3)) with self.assertRaisesRegexp(NotImplementedError, "not be square"): @@ -199,6 +204,7 @@ class LinearOperatorTest(test.TestCase): LinearOperatorMatmulSolve( matrix, is_positive_definite=True, is_square=False) + @test_util.run_deprecated_v1 def test_is_square_manual_set_works(self): matrix = array_ops.placeholder(dtypes.float32) # Default is None. @@ -208,6 +214,80 @@ class LinearOperatorTest(test.TestCase): operator = LinearOperatorMatmulSolve(matrix, is_square=True) self.assertTrue(operator.is_square) + @test_util.run_deprecated_v1 + def test_linear_operator_matmul_hints_closed(self): + matrix = array_ops.placeholder(dtypes.float32) + operator1 = LinearOperatorMatmulSolve(matrix) + + operator_matmul = operator1.matmul(operator1) + + self.assertEqual(None, operator_matmul.is_square) + self.assertEqual(None, operator_matmul.is_non_singular) + self.assertEqual(None, operator_matmul.is_self_adjoint) + self.assertEqual(None, operator_matmul.is_positive_definite) + + operator2 = LinearOperatorMatmulSolve( + matrix, + is_non_singular=True, + is_self_adjoint=True, + is_positive_definite=True, + is_square=True, + ) + + operator_matmul = operator2.matmul(operator2) + + self.assertTrue(operator_matmul.is_square) + self.assertTrue(operator_matmul.is_non_singular) + self.assertTrue(operator_matmul.is_self_adjoint) + self.assertEqual(None, operator_matmul.is_positive_definite) + + @test_util.run_deprecated_v1 + def test_linear_operator_matmul_hints_false(self): + matrix = array_ops.placeholder(dtypes.float32) + operator1 = LinearOperatorMatmulSolve( + matrix, + is_non_singular=False, + is_self_adjoint=False, + is_positive_definite=False, + is_square=True, + ) + + operator_matmul = operator1.matmul(operator1) + + self.assertTrue(operator_matmul.is_square) + self.assertFalse(operator_matmul.is_non_singular) + self.assertEqual(None, operator_matmul.is_self_adjoint) + self.assertEqual(None, operator_matmul.is_positive_definite) + + operator2 = LinearOperatorMatmulSolve( + matrix, + is_non_singular=False, + is_self_adjoint=False, + is_positive_definite=False, + is_square=False, + ) + + operator_matmul = operator2.matmul(operator2) + + self.assertEqual(None, operator_matmul.is_square) + self.assertEqual(None, operator_matmul.is_non_singular) + self.assertEqual(None, operator_matmul.is_self_adjoint) + self.assertEqual(None, operator_matmul.is_positive_definite) + + @test_util.run_deprecated_v1 + def test_linear_operator_matmul_hint_infer_square(self): + matrix1 = array_ops.placeholder(shape=[2, 3], dtype=dtypes.float32) + matrix2 = array_ops.placeholder(shape=[3, 2], dtype=dtypes.float32) + matrix3 = array_ops.placeholder(shape=[3, 4], dtype=dtypes.float32) + + operator1 = LinearOperatorMatmulSolve(matrix1, is_square=False) + operator2 = LinearOperatorMatmulSolve(matrix2, is_square=False) + operator3 = LinearOperatorMatmulSolve(matrix3, is_square=False) + + self.assertTrue(operator1.matmul(operator2).is_square) + self.assertTrue(operator2.matmul(operator1).is_square) + self.assertFalse(operator1.matmul(operator3).is_square) + if __name__ == "__main__": test.main() diff --git a/tensorflow/python/kernel_tests/linalg/linear_operator_util_test.py b/tensorflow/python/kernel_tests/linalg/linear_operator_util_test.py index 31fb19e4a69..d1e6c37e35a 100644 --- a/tensorflow/python/kernel_tests/linalg/linear_operator_util_test.py +++ b/tensorflow/python/kernel_tests/linalg/linear_operator_util_test.py @@ -21,6 +21,7 @@ import numpy as np from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops +from tensorflow.python.framework import test_util from tensorflow.python.ops import array_ops from tensorflow.python.ops import linalg_ops from tensorflow.python.ops import math_ops @@ -32,12 +33,14 @@ rng = np.random.RandomState(0) class AssertZeroImagPartTest(test.TestCase): + @test_util.run_deprecated_v1 def test_real_tensor_doesnt_raise(self): x = ops.convert_to_tensor([0., 2, 3]) with self.cached_session(): # Should not raise. linear_operator_util.assert_zero_imag_part(x, message="ABC123").run() + @test_util.run_deprecated_v1 def test_complex_tensor_with_imag_zero_doesnt_raise(self): x = ops.convert_to_tensor([1., 0, 3]) y = ops.convert_to_tensor([0., 0, 0]) @@ -57,6 +60,7 @@ class AssertZeroImagPartTest(test.TestCase): class AssertNoEntriesWithModulusZeroTest(test.TestCase): + @test_util.run_deprecated_v1 def test_nonzero_real_tensor_doesnt_raise(self): x = ops.convert_to_tensor([1., 2, 3]) with self.cached_session(): @@ -64,6 +68,7 @@ class AssertNoEntriesWithModulusZeroTest(test.TestCase): linear_operator_util.assert_no_entries_with_modulus_zero( x, message="ABC123").run() + @test_util.run_deprecated_v1 def test_nonzero_complex_tensor_doesnt_raise(self): x = ops.convert_to_tensor([1., 0, 3]) y = ops.convert_to_tensor([1., 2, 0]) @@ -102,8 +107,9 @@ class BroadcastMatrixBatchDimsTest(test.TestCase): self.assertTrue(isinstance(tensor, ops.Tensor)) with self.cached_session(): - self.assertAllClose(arr, tensor.eval()) + self.assertAllClose(arr, self.evaluate(tensor)) + @test_util.run_deprecated_v1 def test_static_dims_broadcast(self): # x.batch_shape = [3, 1, 2] # y.batch_shape = [4, 1] @@ -119,7 +125,7 @@ class BroadcastMatrixBatchDimsTest(test.TestCase): with self.cached_session() as sess: self.assertAllEqual(x_bc_expected.shape, x_bc.get_shape()) self.assertAllEqual(y_bc_expected.shape, y_bc.get_shape()) - x_bc_, y_bc_ = sess.run([x_bc, y_bc]) + x_bc_, y_bc_ = self.evaluate([x_bc, y_bc]) self.assertAllClose(x_bc_expected, x_bc_) self.assertAllClose(y_bc_expected, y_bc_) @@ -138,10 +144,11 @@ class BroadcastMatrixBatchDimsTest(test.TestCase): with self.cached_session() as sess: self.assertAllEqual(x_bc_expected.shape, x_bc.get_shape()) self.assertAllEqual(y_bc_expected.shape, y_bc.get_shape()) - x_bc_, y_bc_ = sess.run([x_bc, y_bc]) + x_bc_, y_bc_ = self.evaluate([x_bc, y_bc]) self.assertAllClose(x_bc_expected, x_bc_) self.assertAllClose(y_bc_expected, y_bc_) + @test_util.run_deprecated_v1 def test_dynamic_dims_broadcast_32bit(self): # x.batch_shape = [3, 1, 2] # y.batch_shape = [4, 1] @@ -162,6 +169,7 @@ class BroadcastMatrixBatchDimsTest(test.TestCase): self.assertAllClose(x_bc_expected, x_bc_) self.assertAllClose(y_bc_expected, y_bc_) + @test_util.run_deprecated_v1 def test_dynamic_dims_broadcast_32bit_second_arg_higher_rank(self): # x.batch_shape = [1, 2] # y.batch_shape = [3, 4, 1] @@ -195,6 +203,7 @@ class BroadcastMatrixBatchDimsTest(test.TestCase): class CholeskySolveWithBroadcastTest(test.TestCase): + @test_util.run_deprecated_v1 def test_static_dims_broadcast(self): # batch_shape = [2] chol = rng.rand(3, 3) @@ -205,8 +214,9 @@ class CholeskySolveWithBroadcastTest(test.TestCase): result = linear_operator_util.cholesky_solve_with_broadcast(chol, rhs) self.assertAllEqual((2, 3, 7), result.get_shape()) expected = linalg_ops.cholesky_solve(chol_broadcast, rhs) - self.assertAllClose(expected.eval(), result.eval()) + self.assertAllClose(expected.eval(), self.evaluate(result)) + @test_util.run_deprecated_v1 def test_dynamic_dims_broadcast_64bit(self): # batch_shape = [2, 2] chol = rng.rand(2, 3, 3) @@ -233,6 +243,7 @@ class CholeskySolveWithBroadcastTest(test.TestCase): class MatmulWithBroadcastTest(test.TestCase): + @test_util.run_deprecated_v1 def test_static_dims_broadcast_x_has_extra_dims(self): # batch_shape = [2] # for each batch member, we have a 1x3 matrix times a 3x7 matrix ==> 1x7 @@ -244,8 +255,9 @@ class MatmulWithBroadcastTest(test.TestCase): result = linear_operator_util.matmul_with_broadcast(x, y) self.assertAllEqual((2, 1, 7), result.get_shape()) expected = math_ops.matmul(x, y_broadcast) - self.assertAllClose(expected.eval(), result.eval()) + self.assertAllClose(expected.eval(), self.evaluate(result)) + @test_util.run_deprecated_v1 def test_static_dims_broadcast_y_has_extra_dims(self): # Since the second arg has extra dims, and the domain dim of the first arg # is larger than the number of linear equations, code will "flip" the extra @@ -261,8 +273,9 @@ class MatmulWithBroadcastTest(test.TestCase): result = linear_operator_util.matmul_with_broadcast(x, y) self.assertAllEqual((2, 3, 5, 5), result.get_shape()) expected = math_ops.matmul(x_broadcast, y) - self.assertAllClose(expected.eval(), result.eval()) + self.assertAllClose(expected.eval(), self.evaluate(result)) + @test_util.run_deprecated_v1 def test_static_dims_broadcast_y_has_extra_dims_transpose_a_and_b(self): # Since the second arg has extra dims, and the domain dim of the first arg # is larger than the number of linear equations, code will "flip" the extra @@ -280,8 +293,9 @@ class MatmulWithBroadcastTest(test.TestCase): self.assertAllEqual((2, 3, 5, 1), result.get_shape()) expected = math_ops.matmul( x_broadcast, y, transpose_a=True, transpose_b=True) - self.assertAllClose(expected.eval(), result.eval()) + self.assertAllClose(expected.eval(), self.evaluate(result)) + @test_util.run_deprecated_v1 def test_static_dims_broadcast_y_has_extra_dims_transpose_dynamic(self): # Since the second arg has extra dims, and the domain dim of the first arg # is larger than the number of linear equations, code will "flip" the extra @@ -308,6 +322,7 @@ class MatmulWithBroadcastTest(test.TestCase): y_ph: y })) + @test_util.run_deprecated_v1 def test_dynamic_dims_broadcast_64bit(self): # batch_shape = [2] # for each batch member, we have a 1x3 matrix times a 3x7 matrix ==> 1x7 @@ -333,6 +348,7 @@ class MatmulWithBroadcastTest(test.TestCase): class MatrixSolveWithBroadcastTest(test.TestCase): + @test_util.run_deprecated_v1 def test_static_dims_broadcast_matrix_has_extra_dims(self): # batch_shape = [2] matrix = rng.rand(2, 3, 3) @@ -344,8 +360,9 @@ class MatrixSolveWithBroadcastTest(test.TestCase): matrix, rhs) self.assertAllEqual((2, 3, 7), result.get_shape()) expected = linalg_ops.matrix_solve(matrix, rhs_broadcast) - self.assertAllClose(expected.eval(), result.eval()) + self.assertAllClose(expected.eval(), self.evaluate(result)) + @test_util.run_deprecated_v1 def test_static_dims_broadcast_rhs_has_extra_dims(self): # Since the second arg has extra dims, and the domain dim of the first arg # is larger than the number of linear equations, code will "flip" the extra @@ -362,8 +379,9 @@ class MatrixSolveWithBroadcastTest(test.TestCase): result = linear_operator_util.matrix_solve_with_broadcast(matrix, rhs) self.assertAllEqual((2, 3, 2), result.get_shape()) expected = linalg_ops.matrix_solve(matrix_broadcast, rhs) - self.assertAllClose(expected.eval(), result.eval()) + self.assertAllClose(expected.eval(), self.evaluate(result)) + @test_util.run_deprecated_v1 def test_static_dims_broadcast_rhs_has_extra_dims_dynamic(self): # Since the second arg has extra dims, and the domain dim of the first arg # is larger than the number of linear equations, code will "flip" the extra @@ -385,12 +403,13 @@ class MatrixSolveWithBroadcastTest(test.TestCase): self.assertAllEqual(3, result.shape.ndims) expected = linalg_ops.matrix_solve(matrix_broadcast, rhs) self.assertAllClose( - expected.eval(), + self.evaluate(expected), result.eval(feed_dict={ matrix_ph: matrix, rhs_ph: rhs })) + @test_util.run_deprecated_v1 def test_static_dims_broadcast_rhs_has_extra_dims_and_adjoint(self): # Since the second arg has extra dims, and the domain dim of the first arg # is larger than the number of linear equations, code will "flip" the extra @@ -408,8 +427,9 @@ class MatrixSolveWithBroadcastTest(test.TestCase): matrix, rhs, adjoint=True) self.assertAllEqual((2, 3, 2), result.get_shape()) expected = linalg_ops.matrix_solve(matrix_broadcast, rhs, adjoint=True) - self.assertAllClose(expected.eval(), result.eval()) + self.assertAllClose(expected.eval(), self.evaluate(result)) + @test_util.run_deprecated_v1 def test_dynamic_dims_broadcast_64bit(self): # batch_shape = [2, 2] matrix = rng.rand(2, 3, 3) @@ -436,6 +456,7 @@ class MatrixSolveWithBroadcastTest(test.TestCase): class MatrixTriangularSolveWithBroadcastTest(test.TestCase): + @test_util.run_deprecated_v1 def test_static_dims_broadcast_matrix_has_extra_dims(self): # batch_shape = [2] matrix = rng.rand(2, 3, 3) @@ -447,8 +468,9 @@ class MatrixTriangularSolveWithBroadcastTest(test.TestCase): matrix, rhs) self.assertAllEqual((2, 3, 7), result.get_shape()) expected = linalg_ops.matrix_triangular_solve(matrix, rhs_broadcast) - self.assertAllClose(expected.eval(), result.eval()) + self.assertAllClose(expected.eval(), self.evaluate(result)) + @test_util.run_deprecated_v1 def test_static_dims_broadcast_rhs_has_extra_dims(self): # Since the second arg has extra dims, and the domain dim of the first arg # is larger than the number of linear equations, code will "flip" the extra @@ -466,8 +488,9 @@ class MatrixTriangularSolveWithBroadcastTest(test.TestCase): matrix, rhs) self.assertAllEqual((2, 3, 2), result.get_shape()) expected = linalg_ops.matrix_triangular_solve(matrix_broadcast, rhs) - self.assertAllClose(expected.eval(), result.eval()) + self.assertAllClose(expected.eval(), self.evaluate(result)) + @test_util.run_deprecated_v1 def test_static_dims_broadcast_rhs_has_extra_dims_and_adjoint(self): # Since the second arg has extra dims, and the domain dim of the first arg # is larger than the number of linear equations, code will "flip" the extra @@ -486,8 +509,9 @@ class MatrixTriangularSolveWithBroadcastTest(test.TestCase): self.assertAllEqual((2, 3, 2), result.get_shape()) expected = linalg_ops.matrix_triangular_solve( matrix_broadcast, rhs, adjoint=True) - self.assertAllClose(expected.eval(), result.eval()) + self.assertAllClose(expected.eval(), self.evaluate(result)) + @test_util.run_deprecated_v1 def test_dynamic_dims_broadcast_64bit(self): # batch_shape = [2] matrix = rng.rand(2, 3, 3) @@ -522,6 +546,7 @@ class DomainDimensionStubOperator(object): class AssertCompatibleMatrixDimensionsTest(test.TestCase): + @test_util.run_deprecated_v1 def test_compatible_dimensions_do_not_raise(self): with self.cached_session(): x = ops.convert_to_tensor(rng.rand(2, 3, 4)) diff --git a/tensorflow/python/kernel_tests/linalg/linear_operator_zeros_test.py b/tensorflow/python/kernel_tests/linalg/linear_operator_zeros_test.py index ad97d1a93ea..eb0b8ef1277 100644 --- a/tensorflow/python/kernel_tests/linalg/linear_operator_zeros_test.py +++ b/tensorflow/python/kernel_tests/linalg/linear_operator_zeros_test.py @@ -20,6 +20,7 @@ from __future__ import print_function import numpy as np from tensorflow.python.framework import dtypes +from tensorflow.python.framework import test_util from tensorflow.python.ops import array_ops from tensorflow.python.ops.linalg import linalg as linalg_lib from tensorflow.python.ops.linalg import linear_operator_test_util @@ -35,7 +36,7 @@ class LinearOperatorZerosTest( @property def _tests_to_skip(self): - return ["log_abs_det", "solve", "solve_with_broadcast"] + return ["cholesky", "log_abs_det", "solve", "solve_with_broadcast"] @property def _operator_build_infos(self): @@ -46,7 +47,10 @@ class LinearOperatorZerosTest( build_info((3, 4, 4)), build_info((2, 1, 4, 4))] - def _operator_and_matrix(self, build_info, dtype, use_placeholder): + def _operator_and_matrix( + self, build_info, dtype, use_placeholder, + ensure_self_adjoint_and_pd=False): + del ensure_self_adjoint_and_pd del use_placeholder shape = list(build_info.shape) assert shape[-1] == shape[-2] @@ -70,6 +74,7 @@ class LinearOperatorZerosTest( operator = linalg_lib.LinearOperatorZeros(num_rows=2) operator.assert_non_singular() + @test_util.run_deprecated_v1 def test_assert_self_adjoint(self): with self.cached_session(): operator = linalg_lib.LinearOperatorZeros(num_rows=2) @@ -105,6 +110,7 @@ class LinearOperatorZerosTest( with self.assertRaisesRegexp(ValueError, "must be non-negative"): linalg_lib.LinearOperatorZeros(num_rows=2, batch_shape=[-2]) + @test_util.run_deprecated_v1 def test_non_scalar_num_rows_raises_dynamic(self): with self.cached_session(): num_rows = array_ops.placeholder(dtypes.int32) @@ -113,6 +119,7 @@ class LinearOperatorZerosTest( with self.assertRaisesOpError("must be a 0-D Tensor"): operator.to_dense().eval(feed_dict={num_rows: [2]}) + @test_util.run_deprecated_v1 def test_negative_num_rows_raises_dynamic(self): with self.cached_session(): n = array_ops.placeholder(dtypes.int32) @@ -126,6 +133,7 @@ class LinearOperatorZerosTest( with self.assertRaisesOpError("must be non-negative"): operator.to_dense().eval(feed_dict={n: -2}) + @test_util.run_deprecated_v1 def test_non_1d_batch_shape_raises_dynamic(self): with self.cached_session(): batch_shape = array_ops.placeholder(dtypes.int32) @@ -134,6 +142,7 @@ class LinearOperatorZerosTest( with self.assertRaisesOpError("must be a 1-D"): operator.to_dense().eval(feed_dict={batch_shape: 2}) + @test_util.run_deprecated_v1 def test_negative_batch_shape_raises_dynamic(self): with self.cached_session(): batch_shape = array_ops.placeholder(dtypes.int32) @@ -148,6 +157,7 @@ class LinearOperatorZerosTest( with self.assertRaisesRegexp(ValueError, "Dimensions.*not compatible"): operator.matmul(x) + @test_util.run_deprecated_v1 def test_wrong_matrix_dimensions_raises_dynamic(self): num_rows = array_ops.placeholder(dtypes.int32) x = array_ops.placeholder(dtypes.float32) @@ -166,6 +176,17 @@ class LinearOperatorZerosTest( self.assertFalse(operator.is_non_singular) self.assertTrue(operator.is_self_adjoint) + def test_zeros_matmul(self): + operator1 = linalg_lib.LinearOperatorIdentity(num_rows=2) + operator2 = linalg_lib.LinearOperatorZeros(num_rows=2) + self.assertTrue(isinstance( + operator1.matmul(operator2), + linalg_lib.LinearOperatorZeros)) + + self.assertTrue(isinstance( + operator2.matmul(operator1), + linalg_lib.LinearOperatorZeros)) + class LinearOperatorZerosNotSquareTest( linear_operator_test_util.NonSquareLinearOperatorDerivedClassTest): diff --git a/tensorflow/python/kernel_tests/linalg_grad_test.py b/tensorflow/python/kernel_tests/linalg_grad_test.py index 03b640a85a3..28e1d7e1684 100644 --- a/tensorflow/python/kernel_tests/linalg_grad_test.py +++ b/tensorflow/python/kernel_tests/linalg_grad_test.py @@ -21,6 +21,7 @@ from __future__ import print_function import numpy as np from tensorflow.python.framework import constant_op +from tensorflow.python.framework import test_util from tensorflow.python.ops import array_ops from tensorflow.python.ops import gradient_checker from tensorflow.python.ops import gradients_impl @@ -39,6 +40,7 @@ def _AddTest(test, op_name, testcase_name, fn): class ShapeTest(test_lib.TestCase): + @test_util.run_deprecated_v1 def testBatchGradientUnknownSize(self): with self.cached_session(): batch_size = constant_op.constant(3) @@ -50,7 +52,7 @@ class ShapeTest(test_lib.TestCase): determinants = linalg_ops.matrix_determinant(batch_identity) reduced = math_ops.reduce_sum(determinants) sum_grad = gradients_impl.gradients(reduced, batch_identity)[0] - self.assertAllClose(batch_identity.eval(), sum_grad.eval()) + self.assertAllClose(batch_identity.eval(), self.evaluate(sum_grad)) class MatrixUnaryFunctorGradientTest(test_lib.TestCase): @@ -69,7 +71,7 @@ def _GetMatrixUnaryFunctorGradientTest(functor_, dtype_, shape_, **kwargs_): if functor_.__name__ == 'matrix_square_root': # Square the input matrix to ensure that its matrix square root exists a = math_ops.matmul(a, a) - a_np = a.eval() + a_np = self.evaluate(a) b = functor_(a, **kwargs_) # Optimal stepsize for central difference is O(epsilon^{1/3}). diff --git a/tensorflow/python/kernel_tests/linalg_ops_test.py b/tensorflow/python/kernel_tests/linalg_ops_test.py index 28391aaa878..028167a7860 100644 --- a/tensorflow/python/kernel_tests/linalg_ops_test.py +++ b/tensorflow/python/kernel_tests/linalg_ops_test.py @@ -25,6 +25,7 @@ import numpy as np from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops +from tensorflow.python.framework import test_util from tensorflow.python.ops import array_ops from tensorflow.python.ops import linalg_ops from tensorflow.python.ops import math_ops @@ -52,6 +53,7 @@ class CholeskySolveTest(test.TestCase): def setUp(self): self.rng = np.random.RandomState(0) + @test_util.run_deprecated_v1 def test_works_with_five_different_random_pos_def_matrices(self): for n in range(1, 6): for np_type, atol in [(np.float32, 0.05), (np.float64, 1e-5)]: @@ -73,6 +75,7 @@ class LogdetTest(test.TestCase): def setUp(self): self.rng = np.random.RandomState(42) + @test_util.run_deprecated_v1 def test_works_with_five_different_random_pos_def_matrices(self): for n in range(1, 6): for np_dtype, atol in [(np.float32, 0.05), (np.float64, 1e-5), @@ -85,7 +88,7 @@ class LogdetTest(test.TestCase): # [_RandomPDMatrix(n, self.rng, np_dtype), # _RandomPDMatrix(n, self.rng, np_dtype)]).astype(np_dtype) logdet_tf = linalg.logdet(matrix) - self.assertAllClose(logdet_np, logdet_tf.eval(), atol=atol) + self.assertAllClose(logdet_np, self.evaluate(logdet_tf), atol=atol) def test_works_with_underflow_case(self): for np_dtype, atol in [(np.float32, 0.05), (np.float64, 1e-5), @@ -94,7 +97,7 @@ class LogdetTest(test.TestCase): _, logdet_np = np.linalg.slogdet(matrix) with self.session(use_gpu=True): logdet_tf = linalg.logdet(matrix) - self.assertAllClose(logdet_np, logdet_tf.eval(), atol=atol) + self.assertAllClose(logdet_np, self.evaluate(logdet_tf), atol=atol) class SlogdetTest(test.TestCase): @@ -102,6 +105,7 @@ class SlogdetTest(test.TestCase): def setUp(self): self.rng = np.random.RandomState(42) + @test_util.run_deprecated_v1 def test_works_with_five_different_random_pos_def_matrices(self): for n in range(1, 6): for np_dtype, atol in [(np.float32, 0.05), (np.float64, 1e-5), @@ -110,8 +114,9 @@ class SlogdetTest(test.TestCase): sign_np, log_abs_det_np = np.linalg.slogdet(matrix) with self.session(use_gpu=True): sign_tf, log_abs_det_tf = linalg.slogdet(matrix) - self.assertAllClose(log_abs_det_np, log_abs_det_tf.eval(), atol=atol) - self.assertAllClose(sign_np, sign_tf.eval(), atol=atol) + self.assertAllClose( + log_abs_det_np, self.evaluate(log_abs_det_tf), atol=atol) + self.assertAllClose(sign_np, self.evaluate(sign_tf), atol=atol) def test_works_with_underflow_case(self): for np_dtype, atol in [(np.float32, 0.05), (np.float64, 1e-5), @@ -120,8 +125,9 @@ class SlogdetTest(test.TestCase): sign_np, log_abs_det_np = np.linalg.slogdet(matrix) with self.session(use_gpu=True): sign_tf, log_abs_det_tf = linalg.slogdet(matrix) - self.assertAllClose(log_abs_det_np, log_abs_det_tf.eval(), atol=atol) - self.assertAllClose(sign_np, sign_tf.eval(), atol=atol) + self.assertAllClose( + log_abs_det_np, self.evaluate(log_abs_det_tf), atol=atol) + self.assertAllClose(sign_np, self.evaluate(sign_tf), atol=atol) class AdjointTest(test.TestCase): @@ -135,7 +141,7 @@ class AdjointTest(test.TestCase): matrix = ops.convert_to_tensor(matrix_np) transposed = linalg.adjoint(matrix) self.assertEqual((3, 2), transposed.get_shape()) - self.assertAllEqual(expected_transposed, transposed.eval()) + self.assertAllEqual(expected_transposed, self.evaluate(transposed)) class EyeTest(parameterized.TestCase, test.TestCase): @@ -230,6 +236,7 @@ class EyeTest(parameterized.TestCase, test.TestCase): dtypes.complex128 ]) ) + @test_util.run_deprecated_v1 def test_eye_with_placeholder( self, num_rows, num_columns, batch_shape, dtype): eye_np = np.eye(num_rows, M=num_columns, dtype=dtype.as_numpy_dtype) diff --git a/tensorflow/python/kernel_tests/list_ops_test.py b/tensorflow/python/kernel_tests/list_ops_test.py index 92552854aa6..489f6c9b004 100644 --- a/tensorflow/python/kernel_tests/list_ops_test.py +++ b/tensorflow/python/kernel_tests/list_ops_test.py @@ -29,9 +29,11 @@ from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes from tensorflow.python.framework import errors from tensorflow.python.framework import ops +from tensorflow.python.framework import tensor_shape from tensorflow.python.framework import test_util from tensorflow.python.ops import array_ops from tensorflow.python.ops import control_flow_ops +from tensorflow.python.ops import gradients_impl from tensorflow.python.ops import list_ops from tensorflow.python.ops import math_ops from tensorflow.python.ops import state_ops @@ -39,17 +41,13 @@ from tensorflow.python.ops import variable_scope as vs from tensorflow.python.platform import test -def scalar_shape(): - return ops.convert_to_tensor([], dtype=dtypes.int32) - - @test_util.run_all_in_graph_and_eager_modes class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase): def _testPushPop(self, max_num_elements): l = list_ops.empty_tensor_list( element_dtype=dtypes.float32, - element_shape=scalar_shape(), + element_shape=[], max_num_elements=max_num_elements) l = list_ops.tensor_list_push_back(l, constant_op.constant(1.0)) l, e = list_ops.tensor_list_pop_back(l, element_dtype=dtypes.float32) @@ -68,11 +66,10 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase): with context.device("gpu:0"): self._testPushPop(max_num_elements) + @test_util.run_deprecated_v1 def testPushInFullListFails(self): l = list_ops.empty_tensor_list( - element_dtype=dtypes.float32, - element_shape=scalar_shape(), - max_num_elements=1) + element_dtype=dtypes.float32, element_shape=[], max_num_elements=1) l = list_ops.tensor_list_push_back(l, constant_op.constant(1.0)) with self.assertRaisesRegexp(errors.InvalidArgumentError, "Tried to push item into a full list"): @@ -81,10 +78,11 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase): @parameterized.named_parameters(("NoMaxNumElements", None), ("WithMaxNumElements", 2)) + @test_util.run_deprecated_v1 def testPopFromEmptyTensorListFails(self, max_num_elements): l = list_ops.empty_tensor_list( element_dtype=dtypes.float32, - element_shape=scalar_shape(), + element_shape=[], max_num_elements=max_num_elements) with self.assertRaisesRegexp(errors.InvalidArgumentError, "Trying to pop from an empty list"): @@ -94,11 +92,13 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase): def _testStack(self, max_num_elements): l = list_ops.empty_tensor_list( element_dtype=dtypes.float32, - element_shape=scalar_shape(), + element_shape=[], max_num_elements=max_num_elements) l = list_ops.tensor_list_push_back(l, constant_op.constant(1.0)) l = list_ops.tensor_list_push_back(l, constant_op.constant(2.0)) t = list_ops.tensor_list_stack(l, element_dtype=dtypes.float32) + if not context.executing_eagerly(): + self.assertAllEqual(t.shape.as_list(), [None]) self.assertAllEqual(self.evaluate(t), [1.0, 2.0]) @parameterized.named_parameters(("NoMaxNumElements", None), @@ -116,10 +116,11 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase): @parameterized.named_parameters(("NoMaxNumElements", None), ("WithMaxNumElements", 3)) + @test_util.run_deprecated_v1 def testStackWithUnknownElementShape(self, max_num_elements): l = list_ops.empty_tensor_list( element_dtype=dtypes.float32, - element_shape=-1, + element_shape=None, max_num_elements=max_num_elements) l = list_ops.tensor_list_push_back(l, constant_op.constant(1.0)) l = list_ops.tensor_list_push_back(l, constant_op.constant(2.0)) @@ -136,10 +137,11 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase): @parameterized.named_parameters(("NoMaxNumElements", None), ("WithMaxNumElements", 3)) + @test_util.run_deprecated_v1 def testStackWithPartiallyDefinedElementShape(self, max_num_elements): l = list_ops.empty_tensor_list( element_dtype=dtypes.float32, - element_shape=[-1], + element_shape=[None], max_num_elements=max_num_elements) l = list_ops.tensor_list_push_back(l, constant_op.constant([1.0])) l = list_ops.tensor_list_push_back(l, constant_op.constant([2.0])) @@ -156,6 +158,7 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase): @parameterized.named_parameters(("NoMaxNumElements", None), ("WithMaxNumElements", 2)) + @test_util.run_deprecated_v1 def testStackEmptyList(self, max_num_elements): # Should be able to stack empty lists with fully defined element_shape. l = list_ops.empty_tensor_list( @@ -171,7 +174,7 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase): "non-fully-defined"): l = list_ops.empty_tensor_list( element_dtype=dtypes.float32, - element_shape=[-1, 2], + element_shape=[None, 2], max_num_elements=max_num_elements) t = list_ops.tensor_list_stack(l, element_dtype=dtypes.float32) self.evaluate(t) @@ -181,7 +184,7 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase): "non-fully-defined"): l = list_ops.empty_tensor_list( element_dtype=dtypes.float32, - element_shape=-1, + element_shape=None, max_num_elements=max_num_elements) t = list_ops.tensor_list_stack(l, element_dtype=dtypes.float32) self.evaluate(t) @@ -192,7 +195,7 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase): with backprop.GradientTape() as tape: l = list_ops.empty_tensor_list( element_dtype=dtypes.float32, - element_shape=scalar_shape(), + element_shape=[], max_num_elements=max_num_elements) c0 = constant_op.constant(1.0) tape.watch(c0) @@ -206,10 +209,11 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase): @parameterized.named_parameters(("NoMaxNumElements", None), ("WithMaxNumElements", 3)) + @test_util.run_deprecated_v1 def testGatherWithUnknownElementShape(self, max_num_elements): l = list_ops.empty_tensor_list( element_dtype=dtypes.float32, - element_shape=-1, + element_shape=None, max_num_elements=max_num_elements) l = list_ops.tensor_list_push_back(l, constant_op.constant(1.0)) l = list_ops.tensor_list_push_back(l, constant_op.constant(2.0)) @@ -229,10 +233,11 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase): @parameterized.named_parameters(("NoMaxNumElements", None), ("WithMaxNumElements", 3)) + @test_util.run_deprecated_v1 def testGatherWithPartiallyDefinedElementShape(self, max_num_elements): l = list_ops.empty_tensor_list( element_dtype=dtypes.float32, - element_shape=[-1], + element_shape=[None], max_num_elements=max_num_elements) l = list_ops.tensor_list_push_back(l, constant_op.constant([1.0])) l = list_ops.tensor_list_push_back(l, constant_op.constant([2.0, 3.0])) @@ -252,6 +257,7 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase): @parameterized.named_parameters(("NoMaxNumElements", None), ("WithMaxNumElements", 3)) + @test_util.run_deprecated_v1 def testGatherEmptyList(self, max_num_elements): # Should be able to gather from empty lists with fully defined # element_shape. @@ -268,7 +274,7 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase): "non-fully-defined"): l = list_ops.empty_tensor_list( element_dtype=dtypes.float32, - element_shape=[-1, 2], + element_shape=[None, 2], max_num_elements=max_num_elements) t = list_ops.tensor_list_gather(l, [], element_dtype=dtypes.float32) self.evaluate(t) @@ -279,7 +285,7 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase): "non-fully-defined"): l = list_ops.empty_tensor_list( element_dtype=dtypes.float32, - element_shape=-1, + element_shape=None, max_num_elements=max_num_elements) t = list_ops.tensor_list_gather(l, [], element_dtype=dtypes.float32) self.evaluate(t) @@ -300,7 +306,7 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase): def testTensorListFromTensor(self): t = constant_op.constant([1.0, 2.0]) - l = list_ops.tensor_list_from_tensor(t, element_shape=scalar_shape()) + l = list_ops.tensor_list_from_tensor(t, element_shape=[]) l, e = list_ops.tensor_list_pop_back(l, element_dtype=dtypes.float32) self.assertAllEqual(self.evaluate(e), 2.0) l, e = list_ops.tensor_list_pop_back(l, element_dtype=dtypes.float32) @@ -315,7 +321,7 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase): def testGetSetItem(self): t = constant_op.constant([1.0, 2.0]) - l = list_ops.tensor_list_from_tensor(t, element_shape=scalar_shape()) + l = list_ops.tensor_list_from_tensor(t, element_shape=[]) e0 = list_ops.tensor_list_get_item(l, 0, element_dtype=dtypes.float32) self.assertAllEqual(self.evaluate(e0), 1.0) l = list_ops.tensor_list_set_item(l, 0, 3.0) @@ -333,19 +339,16 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase): t = constant_op.constant(5.) tape.watch(t) l = list_ops.tensor_list_reserve( - element_dtype=dtypes.float32, - element_shape=scalar_shape(), - num_elements=3) + element_dtype=dtypes.float32, element_shape=[], num_elements=3) l = list_ops.tensor_list_set_item(l, 1, 2. * t) e = list_ops.tensor_list_get_item(l, 1, element_dtype=dtypes.float32) self.assertAllEqual(self.evaluate(e), 10.0) self.assertAllEqual(self.evaluate(tape.gradient(e, t)), 2.0) + @test_util.run_deprecated_v1 def testSetOnEmptyListWithMaxNumElementsFails(self): l = list_ops.empty_tensor_list( - element_dtype=dtypes.float32, - element_shape=scalar_shape(), - max_num_elements=3) + element_dtype=dtypes.float32, element_shape=[], max_num_elements=3) with self.assertRaisesRegexp( errors.InvalidArgumentError, "Trying to modify element 0 in a list with 0 elements."): @@ -354,7 +357,7 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase): def testUnknownShape(self): l = list_ops.empty_tensor_list( - element_dtype=dtypes.float32, element_shape=-1) + element_dtype=dtypes.float32, element_shape=None) l = list_ops.tensor_list_push_back(l, constant_op.constant(1.0)) l = list_ops.tensor_list_push_back(l, constant_op.constant([1.0, 2.0])) l, e = list_ops.tensor_list_pop_back(l, element_dtype=dtypes.float32) @@ -366,7 +369,7 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase): if not context.num_gpus(): return t = constant_op.constant([1.0, 2.0]) - l = list_ops.tensor_list_from_tensor(t, element_shape=scalar_shape()) + l = list_ops.tensor_list_from_tensor(t, element_shape=[]) with context.device("gpu:0"): l_gpu = array_ops.identity(l) self.assertAllEqual( @@ -383,7 +386,7 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase): if not context.num_gpus(): return t = constant_op.constant([1.0, 2.0]) - child_l = list_ops.tensor_list_from_tensor(t, element_shape=scalar_shape()) + child_l = list_ops.tensor_list_from_tensor(t, element_shape=[]) l = list_ops.empty_tensor_list( element_shape=constant_op.constant([], dtype=dtypes.int32), element_dtype=dtypes.variant) @@ -495,9 +498,7 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase): with ops.Graph().as_default(), session.Session(target=worker.target): with ops.device("/job:worker"): l = list_ops.tensor_list_reserve( - element_dtype=dtypes.float32, - element_shape=scalar_shape(), - num_elements=2) + element_dtype=dtypes.float32, element_shape=[], num_elements=2) l = list_ops.tensor_list_set_item(l, 0, 1.) with ops.device("/job:ps"): l_ps = array_ops.identity(l) @@ -512,7 +513,7 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase): with ops.Graph().as_default(), session.Session(target=worker.target): with ops.device("/job:worker"): t = constant_op.constant([[1.0], [2.0]]) - l = list_ops.tensor_list_from_tensor(t, element_shape=-1) + l = list_ops.tensor_list_from_tensor(t, element_shape=None) with ops.device("/job:ps"): l_ps = array_ops.identity(l) element_shape = list_ops.tensor_list_element_shape( @@ -529,7 +530,9 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase): with ops.Graph().as_default(), session.Session(target=worker.target): with ops.device("/job:worker"): l = list_ops.empty_tensor_list( - element_shape=-1, element_dtype=dtypes.float32, max_num_elements=2) + element_shape=None, + element_dtype=dtypes.float32, + max_num_elements=2) l = list_ops.tensor_list_push_back(l, 1.) with ops.device("/job:ps"): l_ps = array_ops.identity(l) @@ -543,8 +546,8 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase): def testPushPopGradients(self): with backprop.GradientTape() as tape: - l = list_ops.empty_tensor_list(element_dtype=dtypes.float32, - element_shape=scalar_shape()) + l = list_ops.empty_tensor_list( + element_dtype=dtypes.float32, element_shape=[]) c = constant_op.constant(1.0) tape.watch(c) l = list_ops.tensor_list_push_back(l, c) @@ -556,7 +559,7 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase): with backprop.GradientTape() as tape: c = constant_op.constant([1.0, 2.0]) tape.watch(c) - l = list_ops.tensor_list_from_tensor(c, element_shape=scalar_shape()) + l = list_ops.tensor_list_from_tensor(c, element_shape=[]) c2 = list_ops.tensor_list_stack( l, element_dtype=dtypes.float32, num_elements=2) result = c2 * 2.0 @@ -567,7 +570,7 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase): with backprop.GradientTape() as tape: c = constant_op.constant([1.0, 2.0]) tape.watch(c) - l = list_ops.tensor_list_from_tensor(c, element_shape=scalar_shape()) + l = list_ops.tensor_list_from_tensor(c, element_shape=[]) c2 = constant_op.constant(3.0) tape.watch(c2) l = list_ops.tensor_list_set_item(l, 0, c2) @@ -578,17 +581,19 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase): self.assertAllEqual(self.evaluate(grad_c), [0.0, 4.0]) self.assertAllEqual(self.evaluate(grad_c2), 6.0) + @test_util.run_deprecated_v1 def testSetOutOfBounds(self): c = constant_op.constant([1.0, 2.0]) - l = list_ops.tensor_list_from_tensor(c, element_shape=scalar_shape()) + l = list_ops.tensor_list_from_tensor(c, element_shape=[]) with self.assertRaises(errors.InvalidArgumentError): self.evaluate(list_ops.tensor_list_set_item(l, 20, 3.0)) + @test_util.run_deprecated_v1 def testSkipEagerSetItemWithMismatchedShapeFails(self): with self.cached_session() as sess: ph = array_ops.placeholder(dtypes.float32) c = constant_op.constant([1.0, 2.0]) - l = list_ops.tensor_list_from_tensor(c, element_shape=scalar_shape()) + l = list_ops.tensor_list_from_tensor(c, element_shape=[]) # Set a placeholder with unknown shape to satisfy the shape inference # at graph building time. l = list_ops.tensor_list_set_item(l, 0, ph) @@ -599,7 +604,7 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase): def testResourceVariableScatterGather(self): c = constant_op.constant([1.0, 2.0], dtype=dtypes.float32) - l = list_ops.tensor_list_from_tensor(c, element_shape=scalar_shape()) + l = list_ops.tensor_list_from_tensor(c, element_shape=[]) v = vs.get_variable("var", initializer=[l] * 10, use_resource=True) v_r_0_stacked = list_ops.tensor_list_stack(v[0], dtypes.float32) self.evaluate(v.initializer) @@ -607,10 +612,8 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase): v_r_sparse_stacked = list_ops.tensor_list_stack( v.sparse_read(0), dtypes.float32) self.assertAllEqual([1.0, 2.0], self.evaluate(v_r_sparse_stacked)) - l_new_0 = list_ops.tensor_list_from_tensor( - [3.0, 4.0], element_shape=scalar_shape()) - l_new_1 = list_ops.tensor_list_from_tensor( - [5.0, 6.0], element_shape=scalar_shape()) + l_new_0 = list_ops.tensor_list_from_tensor([3.0, 4.0], element_shape=[]) + l_new_1 = list_ops.tensor_list_from_tensor([5.0, 6.0], element_shape=[]) updated_v = state_ops.scatter_update(v, [3, 5], [l_new_0, l_new_1]) updated_v_elems = array_ops.unstack(updated_v) updated_v_stacked = [ @@ -620,10 +623,11 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase): [[1.0, 2.0]] * 4) self.assertAllEqual(self.evaluate(updated_v_stacked), expected) + @test_util.run_deprecated_v1 def testConcat(self): c = constant_op.constant([1.0, 2.0], dtype=dtypes.float32) - l0 = list_ops.tensor_list_from_tensor(c, element_shape=scalar_shape()) - l1 = list_ops.tensor_list_from_tensor([-1.0], element_shape=scalar_shape()) + l0 = list_ops.tensor_list_from_tensor(c, element_shape=[]) + l1 = list_ops.tensor_list_from_tensor([-1.0], element_shape=[]) l_batch_0 = array_ops.stack([l0, l1]) l_batch_1 = array_ops.stack([l1, l0]) @@ -659,7 +663,7 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase): self.evaluate( list_ops.tensor_list_concat_lists( l_batch_0, - list_ops.empty_tensor_list(scalar_shape(), dtypes.float32), + list_ops.empty_tensor_list([], dtypes.float32), element_dtype=dtypes.float32)) with self.assertRaisesRegexp(errors.InvalidArgumentError, @@ -673,16 +677,16 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase): with self.assertRaisesRegexp(errors.InvalidArgumentError, r"input_b\[0\].dtype != element_dtype."): l_batch_of_int_tls = array_ops.stack( - [list_ops.tensor_list_from_tensor([1], element_shape=scalar_shape())] - * 2) + [list_ops.tensor_list_from_tensor([1], element_shape=[])] * 2) self.evaluate( list_ops.tensor_list_concat_lists(l_batch_0, l_batch_of_int_tls, element_dtype=dtypes.float32)) + @test_util.run_deprecated_v1 def testPushBackBatch(self): c = constant_op.constant([1.0, 2.0], dtype=dtypes.float32) - l0 = list_ops.tensor_list_from_tensor(c, element_shape=scalar_shape()) - l1 = list_ops.tensor_list_from_tensor([-1.0], element_shape=scalar_shape()) + l0 = list_ops.tensor_list_from_tensor(c, element_shape=[]) + l1 = list_ops.tensor_list_from_tensor([-1.0], element_shape=[]) l_batch = array_ops.stack([l0, l1]) l_push = list_ops.tensor_list_push_back_batch(l_batch, [3.0, 4.0]) l_unstack = array_ops.unstack(l_push) @@ -726,7 +730,7 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase): dtypes.float64, dtypes.complex64, dtypes.complex128, dtypes.bool): l_empty = list_ops.empty_tensor_list( - element_dtype=dtype, element_shape=scalar_shape()) + element_dtype=dtype, element_shape=[]) l_empty_zeros = array_ops.zeros_like(l_empty) t_empty_zeros = list_ops.tensor_list_stack( l_empty_zeros, element_dtype=dtype) @@ -750,10 +754,9 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase): dtypes.float64, dtypes.complex64, dtypes.complex128, dtypes.bool): l = list_ops.empty_tensor_list( - element_dtype=dtypes.variant, element_shape=scalar_shape()) + element_dtype=dtypes.variant, element_shape=[]) - sub_l = list_ops.empty_tensor_list( - element_dtype=dtype, element_shape=scalar_shape()) + sub_l = list_ops.empty_tensor_list(element_dtype=dtype, element_shape=[]) l = list_ops.tensor_list_push_back(l, sub_l) sub_l = list_ops.tensor_list_push_back(sub_l, math_ops.cast( 1, dtype=dtype)) @@ -786,13 +789,12 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase): def testElementShape(self): l = list_ops.empty_tensor_list( - element_dtype=dtypes.float32, element_shape=-1) + element_dtype=dtypes.float32, element_shape=None) shape = list_ops.tensor_list_element_shape(l, shape_type=dtypes.int32) self.assertEqual(self.evaluate(shape), -1) def testZerosLikeUninitialized(self): - l0 = list_ops.tensor_list_reserve( - scalar_shape(), 3, element_dtype=dtypes.float32) + l0 = list_ops.tensor_list_reserve([], 3, element_dtype=dtypes.float32) l1 = list_ops.tensor_list_set_item(l0, 0, 1.) # [1., _, _] zeros_1 = array_ops.zeros_like(l1) # [0., _, _] l2 = list_ops.tensor_list_set_item(l1, 2, 2.) # [1., _, 2.] @@ -808,6 +810,292 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase): self.assertAllEqual(self.evaluate(res_1), [0.]) self.assertAllEqual(self.evaluate(res_2), [0., 0.]) + @test_util.run_deprecated_v1 + def testSkipEagerTensorListGetItemGradAggregation(self): + l = list_ops.tensor_list_reserve( + element_shape=[], num_elements=1, element_dtype=dtypes.float32) + x = constant_op.constant(1.0) + l = list_ops.tensor_list_set_item(l, 0, x) + l_read1 = list_ops.tensor_list_get_item(l, 0, element_dtype=dtypes.float32) + l_read2 = list_ops.tensor_list_get_item(l, 0, element_dtype=dtypes.float32) + grad = gradients_impl.gradients([l_read1, l_read2], [x]) + with self.cached_session() as sess: + self.assertSequenceEqual(self.evaluate(grad), [2.]) + + @test_util.run_deprecated_v1 + def testSkipEagerBuildElementShape(self): + fn = list_ops._build_element_shape + # Unknown shape -> -1. + self.assertEqual(fn(None), -1) + self.assertEqual(fn(tensor_shape.unknown_shape()), -1) + # Scalar shape -> [] with type int32. + self.assertEqual(fn([]).dtype, dtypes.int32) + self.assertEqual(fn(tensor_shape.scalar()).dtype, dtypes.int32) + self.assertAllEqual(self.evaluate(fn([])), np.array([], np.int32)) + self.assertAllEqual( + self.evaluate(fn(tensor_shape.scalar())), np.array([], np.int32)) + # Tensor -> Tensor + shape = constant_op.constant(1) + self.assertIs(fn(shape), shape) + # Shape with unknown dims -> shape list with -1's. + shape = [None, 5] + self.assertAllEqual(fn(shape), [-1, 5]) + self.assertAllEqual(fn(tensor_shape.TensorShape(shape)), [-1, 5]) + # Shape with unknown dims and tensor dims -> shape list with -1's and tensor + # dims. + t = array_ops.placeholder(dtypes.int32) + shape = [None, 5, t] + result = fn(shape) + self.assertAllEqual(result[:2], [-1, 5]) + self.assertIs(result[2], t) + + def testAddN(self): + l1 = list_ops.tensor_list_from_tensor([1.0, 2.0], element_shape=[]) + l2 = list_ops.tensor_list_from_tensor([3.0, 4.0], element_shape=[]) + l3 = list_ops.tensor_list_from_tensor([5.0, 6.0], element_shape=[]) + result = math_ops.add_n((l1, l2, l3)) + result_t = list_ops.tensor_list_stack(result, element_dtype=dtypes.float32) + self.assertAllEqual(self.evaluate(result_t), [9., 12.]) + + def testAddNNestedList(self): + l1 = list_ops.tensor_list_from_tensor([1.0, 2.0], element_shape=[]) + l2 = list_ops.tensor_list_from_tensor([3.0, 4.0], element_shape=[]) + l3 = list_ops.tensor_list_from_tensor([5.0, 6.0], element_shape=[]) + l4 = list_ops.tensor_list_from_tensor([7.0, 8.0], element_shape=[]) + a = list_ops.empty_tensor_list( + element_dtype=dtypes.variant, element_shape=[]) + a = list_ops.tensor_list_push_back(a, l1) + a = list_ops.tensor_list_push_back(a, l2) + b = list_ops.empty_tensor_list( + element_dtype=dtypes.variant, element_shape=[]) + b = list_ops.tensor_list_push_back(b, l3) + b = list_ops.tensor_list_push_back(b, l4) + result = math_ops.add_n((a, b)) + result_0 = list_ops.tensor_list_stack( + list_ops.tensor_list_get_item(result, 0, element_dtype=dtypes.variant), + element_dtype=dtypes.float32) + result_1 = list_ops.tensor_list_stack( + list_ops.tensor_list_get_item(result, 1, element_dtype=dtypes.variant), + element_dtype=dtypes.float32) + self.assertAllEqual(self.evaluate(result_0), [6., 8.]) + self.assertAllEqual(self.evaluate(result_1), [10., 12.]) + + @test_util.run_deprecated_v1 + def testSkipEagerConcatShapeInference(self): + + def BuildTensor(element_shape): + l = list_ops.empty_tensor_list( + element_dtype=dtypes.float32, element_shape=element_shape) + return list_ops.tensor_list_concat(l, element_dtype=dtypes.float32) + + self.assertIsNone(BuildTensor(None).shape.rank) + self.assertAllEqual(BuildTensor([None, 2, 3]).shape.as_list(), [None, 2, 3]) + self.assertAllEqual( + BuildTensor([None, 2, None]).shape.as_list(), [None, 2, None]) + self.assertAllEqual(BuildTensor([1, 2, 3]).shape.as_list(), [None, 2, 3]) + + def testConcatWithFullyDefinedElementShape(self): + l = list_ops.empty_tensor_list( + element_dtype=dtypes.float32, element_shape=[2, 2]) + l = list_ops.tensor_list_push_back(l, [[0., 1.], [2., 3.]]) + l = list_ops.tensor_list_push_back(l, [[4., 5.], [6., 7.]]) + t = list_ops.tensor_list_concat(l, element_dtype=dtypes.float32) + self.assertAllEqual( + self.evaluate(t), [[0., 1.], [2., 3.], [4., 5.], [6., 7.]]) + + def testConcatWithNonFullyDefinedElementShape(self): + l = list_ops.empty_tensor_list( + element_dtype=dtypes.float32, element_shape=[None, 2]) + l = list_ops.tensor_list_push_back(l, [[0., 1.]]) + l = list_ops.tensor_list_push_back(l, [[2., 3.], [4., 5.]]) + t = list_ops.tensor_list_concat(l, element_dtype=dtypes.float32) + self.assertAllEqual(self.evaluate(t), [[0., 1.], [2., 3.], [4., 5.]]) + + def testConcatWithMismatchingTensorShapesFails(self): + l = list_ops.empty_tensor_list( + element_dtype=dtypes.float32, element_shape=None) + l = list_ops.tensor_list_push_back(l, [[0., 1.]]) + l = list_ops.tensor_list_push_back(l, [[2.], [4.]]) + with self.assertRaisesRegexp( + errors.InvalidArgumentError, + r"Tried to concat tensors with unequal shapes: " + r"\[2\] vs \[1\]"): + t = list_ops.tensor_list_concat(l, element_dtype=dtypes.float32) + self.evaluate(t) + + def testConcatEmptyListWithFullyDefinedElementShape(self): + l = list_ops.empty_tensor_list( + element_dtype=dtypes.float32, element_shape=[5, 2]) + t = list_ops.tensor_list_concat(l, element_dtype=dtypes.float32) + self.assertAllEqual(self.evaluate(t).shape, (0, 2)) + l = list_ops.empty_tensor_list( + element_dtype=dtypes.float32, element_shape=[None, 2]) + t = list_ops.tensor_list_concat(l, element_dtype=dtypes.float32) + self.assertAllEqual(self.evaluate(t).shape, (0, 2)) + + def testConcatEmptyListWithUnknownElementShapeFails(self): + l = list_ops.empty_tensor_list( + element_dtype=dtypes.float32, element_shape=None) + with self.assertRaisesRegexp( + errors.InvalidArgumentError, + "All except the first dimension must be fully" + " defined when concating an empty tensor list"): + t = list_ops.tensor_list_concat(l, element_dtype=dtypes.float32) + self.evaluate(t) + + def testConcatEmptyListWithPartiallyDefinedElementShapeFails(self): + l = list_ops.empty_tensor_list( + element_dtype=dtypes.float32, element_shape=[2, None]) + with self.assertRaisesRegexp( + errors.InvalidArgumentError, + "All except the first dimension must be fully" + " defined when concating an empty tensor list"): + t = list_ops.tensor_list_concat(l, element_dtype=dtypes.float32) + self.evaluate(t) + + def testConcatListWithScalarElementShapeFails(self): + l = list_ops.empty_tensor_list( + element_dtype=dtypes.float32, element_shape=tensor_shape.scalar()) + with self.assertRaisesRegexp( + errors.InvalidArgumentError, + "Concat requires elements to be at least vectors, " + "found scalars instead"): + t = list_ops.tensor_list_concat(l, element_dtype=dtypes.float32) + self.evaluate(t) + + def testConcatListWithScalarElementsFails(self): + l = list_ops.empty_tensor_list( + element_dtype=dtypes.float32, element_shape=None) + l1 = list_ops.tensor_list_push_back(l, 1.) + with self.assertRaisesRegexp( + errors.InvalidArgumentError, "Concat saw a scalar shape at index 0" + " but requires at least vectors"): + t = list_ops.tensor_list_concat(l1, element_dtype=dtypes.float32) + self.evaluate(t) + l1 = list_ops.tensor_list_push_back(l, [1.]) + l1 = list_ops.tensor_list_push_back(l1, 2.) + with self.assertRaisesRegexp( + errors.InvalidArgumentError, "Concat saw a scalar shape at index 1" + " but requires at least vectors"): + t = list_ops.tensor_list_concat(l1, element_dtype=dtypes.float32) + self.evaluate(t) + + def testEvenSplit(self): + + def RunTest(input_tensor, lengths, expected_stacked_output): + l = list_ops.tensor_list_split( + input_tensor, element_shape=None, lengths=lengths) + self.assertAllEqual( + list_ops.tensor_list_stack(l, element_dtype=dtypes.float32), + expected_stacked_output) + + RunTest([1., 2., 3.], [1, 1, 1], [[1.], [2.], [3.]]) + RunTest([1., 2., 3., 4.], [2, 2], [[1., 2.], [3., 4.]]) + RunTest([[1., 2.], [3., 4.]], [1, 1], [[[1., 2.]], [[3., 4.]]]) + + def testUnevenSplit(self): + l = list_ops.tensor_list_split([1., 2., 3., 4., 5], + element_shape=None, + lengths=[3, 2]) + self.assertAllEqual(list_ops.tensor_list_length(l), 2) + self.assertAllEqual( + list_ops.tensor_list_get_item(l, 0, element_dtype=dtypes.float32), + [1., 2., 3.]) + self.assertAllEqual( + list_ops.tensor_list_get_item(l, 1, element_dtype=dtypes.float32), + [4., 5.]) + + @test_util.run_deprecated_v1 + def testSkipEagerSplitWithInvalidTensorShapeFails(self): + with self.cached_session(): + tensor = array_ops.placeholder(dtype=dtypes.float32) + l = list_ops.tensor_list_split(tensor, element_shape=None, lengths=[1]) + with self.assertRaisesRegexp( + errors.InvalidArgumentError, + r"Tensor must be at least a vector, but saw shape: \[\]"): + l.eval({tensor: 1}) + + @test_util.run_deprecated_v1 + def testSkipEagerSplitWithInvalidLengthsShapeFails(self): + with self.cached_session(): + lengths = array_ops.placeholder(dtype=dtypes.int64) + l = list_ops.tensor_list_split([1., 2.], + element_shape=None, + lengths=lengths) + with self.assertRaisesRegexp( + errors.InvalidArgumentError, + r"Expected lengths to be a vector, received shape: \[\]"): + l.eval({lengths: 1}) + + def testSplitWithInvalidLengthsFails(self): + with self.assertRaisesRegexp(errors.InvalidArgumentError, + r"Invalid value in lengths: -1"): + l = list_ops.tensor_list_split([1., 2.], + element_shape=None, + lengths=[1, -1]) + self.evaluate(l) + with self.assertRaisesRegexp( + errors.InvalidArgumentError, + r"Attempting to slice \[0, 3\] from tensor with length 2"): + l = list_ops.tensor_list_split([1., 2.], element_shape=None, lengths=[3]) + self.evaluate(l) + with self.assertRaisesRegexp( + errors.InvalidArgumentError, + r"Unused values in tensor. Length of tensor: 2 Values used: 1"): + l = list_ops.tensor_list_split([1., 2.], element_shape=None, lengths=[1]) + self.evaluate(l) + + @test_util.run_deprecated_v1 + def testSkipEagerSplitWithScalarElementShapeFails(self): + with self.assertRaisesRegexp(ValueError, + r"Shapes must be equal rank, but are 1 and 0"): + l = list_ops.tensor_list_split([1., 2.], element_shape=[], lengths=[1, 1]) + with self.cached_session(): + with self.assertRaisesRegexp( + errors.InvalidArgumentError, + r"TensorListSplit requires element_shape to be at least of rank 1, " + r"but saw: \[\]"): + element_shape = array_ops.placeholder(dtype=dtypes.int32) + l = list_ops.tensor_list_split([1., 2.], + element_shape=element_shape, + lengths=[1, 1]) + l.eval({element_shape: []}) + + def testEagerOnlySplitWithScalarElementShapeFails(self): + if context.executing_eagerly(): + with self.assertRaisesRegexp( + errors.InvalidArgumentError, + r"TensorListSplit requires element_shape to be at least of rank 1, " + r"but saw: \[\]"): + list_ops.tensor_list_split([1., 2.], element_shape=[], lengths=[1, 1]) + + @test_util.run_deprecated_v1 + def testSkipEagerSplitWithIncompatibleTensorShapeAndElementShapeFails(self): + with self.assertRaisesRegexp(ValueError, + r"Shapes must be equal rank, but are 2 and 1"): + l = list_ops.tensor_list_split([[1.], [2.]], + element_shape=[1], + lengths=[1, 1]) + + with self.cached_session(): + with self.assertRaisesRegexp( + errors.InvalidArgumentError, + r"tensor shape \[2,1\] is not compatible with element_shape \[1\]"): + element_shape = array_ops.placeholder(dtype=dtypes.int32) + l = list_ops.tensor_list_split([[1.], [2.]], + element_shape=element_shape, + lengths=[1, 1]) + l.eval({element_shape: [1]}) + + def testEagerOnlySplitWithIncompatibleTensorShapeAndElementShapeFails(self): + if context.executing_eagerly(): + with self.assertRaisesRegexp( + errors.InvalidArgumentError, + r"tensor shape \[2,1\] is not compatible with element_shape \[1\]"): + list_ops.tensor_list_split([[1.], [2.]], + element_shape=[1], + lengths=[1, 1]) + if __name__ == "__main__": test.main() diff --git a/tensorflow/python/kernel_tests/listdiff_op_test.py b/tensorflow/python/kernel_tests/listdiff_op_test.py index baeb40dd635..28657107980 100644 --- a/tensorflow/python/kernel_tests/listdiff_op_test.py +++ b/tensorflow/python/kernel_tests/listdiff_op_test.py @@ -47,7 +47,7 @@ class ListDiffTest(test.TestCase): y_tensor = ops.convert_to_tensor(y, dtype=dtype) out_tensor, idx_tensor = diff_func(x_tensor, y_tensor, index_dtype=index_dtype) - tf_out, tf_idx = sess.run([out_tensor, idx_tensor]) + tf_out, tf_idx = self.evaluate([out_tensor, idx_tensor]) self.assertAllEqual(tf_out, out) self.assertAllEqual(tf_idx, idx) self.assertEqual(1, out_tensor.get_shape().ndims) diff --git a/tensorflow/python/kernel_tests/logging_ops_test.py b/tensorflow/python/kernel_tests/logging_ops_test.py index 8e9b87f6512..85035e5f7d3 100644 --- a/tensorflow/python/kernel_tests/logging_ops_test.py +++ b/tensorflow/python/kernel_tests/logging_ops_test.py @@ -39,6 +39,7 @@ from tensorflow.python.platform import test class LoggingOpsTest(test.TestCase): + @test_util.run_deprecated_v1 def testAssertDivideByZero(self): with self.cached_session() as sess: epsilon = ops.convert_to_tensor(1e-20) @@ -52,7 +53,7 @@ class LoggingOpsTest(test.TestCase): math_ops.less(epsilon, y), ["Divide-by-zero"]) ]): out = math_ops.div(z, y) - self.assertAllEqual(2.0, out.eval()) + self.assertAllEqual(2.0, self.evaluate(out)) # assert(epsilon < x) # z / x # @@ -63,7 +64,7 @@ class LoggingOpsTest(test.TestCase): ]): out = math_ops.div(z, x) with self.assertRaisesOpError("less than x"): - out.eval() + self.evaluate(out) class PrintV2Test(test.TestCase): @@ -305,12 +306,14 @@ class PrintV2Test(test.TestCase): tensor, output_stream="unknown") self.evaluate(print_op) + @test_util.run_deprecated_v1 def testPrintOpName(self): with self.cached_session(): tensor = math_ops.range(10) print_op = logging_ops.print_v2(tensor, name="print_name") self.assertEqual(print_op.name, "print_name") + @test_util.run_deprecated_v1 def testNoDuplicateFormatOpGraphModeAfterExplicitFormat(self): with self.cached_session(): tensor = math_ops.range(10) @@ -379,6 +382,7 @@ class PrintGradientTest(test.TestCase): inp_printed = logging_ops.Print(inp, ["hello"]) self.assertEqual(inp.get_shape(), inp_printed.get_shape()) + @test_util.run_deprecated_v1 def testPrintGradient(self): with self.cached_session(): inp = constant_op.constant(2.0, shape=[100, 32], name="in") @@ -387,8 +391,8 @@ class PrintGradientTest(test.TestCase): wx_print = logging_ops.Print(wx, [w, w, w]) wx_grad = gradients_impl.gradients(wx, w)[0] wx_print_grad = gradients_impl.gradients(wx_print, w)[0] - wxg = wx_grad.eval() - wxpg = wx_print_grad.eval() + wxg = self.evaluate(wx_grad) + wxpg = self.evaluate(wx_print_grad) self.assertAllEqual(wxg, wxpg) diff --git a/tensorflow/python/kernel_tests/lookup_ops_test.py b/tensorflow/python/kernel_tests/lookup_ops_test.py index bd93942efbd..ad81e0be649 100644 --- a/tensorflow/python/kernel_tests/lookup_ops_test.py +++ b/tensorflow/python/kernel_tests/lookup_ops_test.py @@ -37,6 +37,7 @@ from tensorflow.python.training import server_lib class HashTableOpTest(test.TestCase): + @test_util.run_deprecated_v1 def testHashTable(self): with self.cached_session(): default_val = -1 @@ -52,15 +53,16 @@ class HashTableOpTest(test.TestCase): output = table.lookup(input_string) self.assertAllEqual([3], output.get_shape()) - result = output.eval() + result = self.evaluate(output) self.assertAllEqual([0, 1, -1], result) exported_keys_tensor, exported_values_tensor = table.export() self.assertItemsEqual([b"brain", b"salad", b"surgery"], - exported_keys_tensor.eval()) - self.assertItemsEqual([0, 1, 2], exported_values_tensor.eval()) + self.evaluate(exported_keys_tensor)) + self.assertItemsEqual([0, 1, 2], self.evaluate(exported_values_tensor)) + @test_util.run_deprecated_v1 def testHashTableFindHighRank(self): with self.cached_session(): default_val = -1 @@ -76,9 +78,10 @@ class HashTableOpTest(test.TestCase): [["brain", "salad"], ["tank", "tarkus"]]) output = table.lookup(input_string) - result = output.eval() + result = self.evaluate(output) self.assertAllEqual([[0, 1], [-1, -1]], result) + @test_util.run_deprecated_v1 def testHashTableInitWithPythonArrays(self): with self.cached_session(): default_val = -1 @@ -94,9 +97,10 @@ class HashTableOpTest(test.TestCase): input_string = constant_op.constant(["brain", "salad", "tank"]) output = table.lookup(input_string) - result = output.eval() + result = self.evaluate(output) self.assertAllEqual([0, 1, -1], result) + @test_util.run_deprecated_v1 def testHashTableInitWithNumPyArrays(self): with self.cached_session(): default_val = -1 @@ -111,9 +115,10 @@ class HashTableOpTest(test.TestCase): input_string = constant_op.constant(["brain", "salad", "tank"]) output = table.lookup(input_string) - result = output.eval() + result = self.evaluate(output) self.assertAllEqual([0, 1, -1], result) + @test_util.run_deprecated_v1 def testMultipleHashTables(self): with self.cached_session() as sess: default_val = -1 @@ -137,11 +142,12 @@ class HashTableOpTest(test.TestCase): output2 = table2.lookup(input_string) output3 = table3.lookup(input_string) - out1, out2, out3 = sess.run([output1, output2, output3]) + out1, out2, out3 = self.evaluate([output1, output2, output3]) self.assertAllEqual([0, 1, -1], out1) self.assertAllEqual([0, 1, -1], out2) self.assertAllEqual([0, 1, -1], out3) + @test_util.run_deprecated_v1 def testHashTableWithTensorDefault(self): with self.cached_session(): default_val = constant_op.constant(-1, dtypes.int64) @@ -154,9 +160,10 @@ class HashTableOpTest(test.TestCase): input_string = constant_op.constant(["brain", "salad", "tank"]) output = table.lookup(input_string) - result = output.eval() + result = self.evaluate(output) self.assertAllEqual([0, 1, -1], result) + @test_util.run_deprecated_v1 def testHashTableWithSparseTensorInput(self): with self.cached_session() as sess: default_val = constant_op.constant(-1, dtypes.int64) @@ -174,12 +181,13 @@ class HashTableOpTest(test.TestCase): constant_op.constant(sp_shape, dtypes.int64)) output = table.lookup(input_tensor) - out_indices, out_values, out_shape = sess.run(output) + out_indices, out_values, out_shape = self.evaluate(output) self.assertAllEqual([0, 1, -1], out_values) self.assertAllEqual(sp_indices, out_indices) self.assertAllEqual(sp_shape, out_shape) + @test_util.run_deprecated_v1 def testSignatureMismatch(self): with self.cached_session(): default_val = -1 @@ -210,6 +218,7 @@ class HashTableOpTest(test.TestCase): lookup_ops.KeyValueTensorInitializer(["a"], [1], [dtypes.string], dtypes.int64), default_val) + @test_util.run_deprecated_v1 def testNotInitialized(self): with self.cached_session(): default_val = -1 @@ -221,8 +230,9 @@ class HashTableOpTest(test.TestCase): output = table.lookup(input_string) with self.assertRaisesOpError("Table not initialized"): - output.eval() + self.evaluate(output) + @test_util.run_deprecated_v1 def testInitializeTwice(self): with self.cached_session(): default_val = -1 @@ -235,6 +245,7 @@ class HashTableOpTest(test.TestCase): with self.assertRaisesOpError("Table already initialized"): table.initializer.run() + @test_util.run_deprecated_v1 def testInitializationWithInvalidDimensions(self): with self.cached_session(): default_val = -1 @@ -245,6 +256,7 @@ class HashTableOpTest(test.TestCase): lookup_ops.HashTable( lookup_ops.KeyValueTensorInitializer(keys, values), default_val) + @test_util.run_deprecated_v1 def testMultipleSessions(self): # Start a server server = server_lib.Server( @@ -274,6 +286,7 @@ class HashTableOpTest(test.TestCase): table.initializer.run() self.assertAllEqual(3, table.size().eval()) + @test_util.run_deprecated_v1 def testHashTableInt32String(self): with self.cached_session(): default_val = "n/a" @@ -286,7 +299,7 @@ class HashTableOpTest(test.TestCase): input_tensor = constant_op.constant([0, 1, -1]) output = table.lookup(input_tensor) - result = output.eval() + result = self.evaluate(output) self.assertAllEqual([b"brain", b"salad", b"n/a"], result) @@ -298,6 +311,7 @@ class IndexTableFromFile(test.TestCase): f.write("\n".join(values) + "\n") return vocabulary_file + @test_util.run_deprecated_v1 def test_string_index_table_from_file(self): vocabulary_file = self._createVocabFile("f2i_vocab1.txt") with self.cached_session(): @@ -306,10 +320,11 @@ class IndexTableFromFile(test.TestCase): ids = table.lookup(constant_op.constant(["salad", "surgery", "tarkus"])) with self.assertRaises(errors_impl.OpError): - ids.eval() + self.evaluate(ids) lookup_ops.tables_initializer().run() - self.assertAllEqual((1, 2, 3), ids.eval()) + self.assertAllEqual((1, 2, 3), self.evaluate(ids)) + @test_util.run_deprecated_v1 def test_string_index_table_from_multicolumn_file(self): vocabulary_file = self._createVocabFile( "f2i_vocab1.txt", values=("brain\t300", "salad\t20", "surgery\t1")) @@ -322,10 +337,11 @@ class IndexTableFromFile(test.TestCase): ids = table.lookup(constant_op.constant(["salad", "surgery", "tarkus"])) with self.assertRaises(errors_impl.OpError): - ids.eval() + self.evaluate(ids) lookup_ops.tables_initializer().run() - self.assertAllEqual((1, 2, 3), ids.eval()) + self.assertAllEqual((1, 2, 3), self.evaluate(ids)) + @test_util.run_deprecated_v1 def test_string_index_table_from_multicolumn_file_custom_delimiter(self): vocabulary_file = self._createVocabFile( "f2i_vocab1.txt", values=("brain 300", "salad 20", "surgery 1")) @@ -339,10 +355,11 @@ class IndexTableFromFile(test.TestCase): ids = table.lookup(constant_op.constant(["salad", "surgery", "tarkus"])) with self.assertRaises(errors_impl.OpError): - ids.eval() + self.evaluate(ids) lookup_ops.tables_initializer().run() - self.assertAllEqual((1, 2, 3), ids.eval()) + self.assertAllEqual((1, 2, 3), self.evaluate(ids)) + @test_util.run_deprecated_v1 def test_string_index_table_from_file_tensor_filename(self): vocabulary_file = self._createVocabFile("f2i_vocab1.txt") with self.cached_session(): @@ -352,12 +369,13 @@ class IndexTableFromFile(test.TestCase): ids = table.lookup(constant_op.constant(["salad", "surgery", "tarkus"])) with self.assertRaises(errors_impl.OpError): - ids.eval() + self.evaluate(ids) lookup_ops.tables_initializer().run() - self.assertAllEqual((1, 2, 3), ids.eval()) + self.assertAllEqual((1, 2, 3), self.evaluate(ids)) self.assertEqual(1, len(ops.get_collection(ops.GraphKeys.ASSET_FILEPATHS))) + @test_util.run_deprecated_v1 def test_string_index_table_from_file_placeholder_filename(self): vocabulary_file = self._createVocabFile("f2i_vocab1.txt") with self.cached_session(): @@ -367,14 +385,15 @@ class IndexTableFromFile(test.TestCase): ids = table.lookup(constant_op.constant(["salad", "surgery", "tarkus"])) with self.assertRaises(errors_impl.OpError): - ids.eval() + self.evaluate(ids) feed_dict = {vocabulary_placeholder.name: vocabulary_file} lookup_ops.tables_initializer().run(feed_dict=feed_dict) - self.assertAllEqual((1, 2, 3), ids.eval()) + self.assertAllEqual((1, 2, 3), self.evaluate(ids)) self.assertEqual(0, len(ops.get_collection(ops.GraphKeys.ASSET_FILEPATHS))) + @test_util.run_deprecated_v1 def test_int32_index_table_from_file(self): vocabulary_file = self._createVocabFile( "f2i_vocab2.txt", values=("42", "1", "-1000")) @@ -387,10 +406,11 @@ class IndexTableFromFile(test.TestCase): constant_op.constant((1, -1000, 11), dtype=dtypes.int32)) with self.assertRaises(errors_impl.OpError): - ids.eval() + self.evaluate(ids) lookup_ops.tables_initializer().run() - self.assertAllEqual((1, 2, 3), ids.eval()) + self.assertAllEqual((1, 2, 3), self.evaluate(ids)) + @test_util.run_deprecated_v1 def test_int64_index_table_from_file(self): vocabulary_file = self._createVocabFile( "f2i_vocab3.txt", values=("42", "1", "-1000")) @@ -403,10 +423,11 @@ class IndexTableFromFile(test.TestCase): constant_op.constant((1, -1000, 11), dtype=dtypes.int64)) with self.assertRaises(errors_impl.OpError): - ids.eval() + self.evaluate(ids) lookup_ops.tables_initializer().run() - self.assertAllEqual((1, 2, 3), ids.eval()) + self.assertAllEqual((1, 2, 3), self.evaluate(ids)) + @test_util.run_deprecated_v1 def test_index_table_from_file_with_default_value(self): default_value = -42 vocabulary_file = self._createVocabFile("f2i_vocab4.txt") @@ -416,10 +437,11 @@ class IndexTableFromFile(test.TestCase): ids = table.lookup(constant_op.constant(["salad", "surgery", "tarkus"])) with self.assertRaises(errors_impl.OpError): - ids.eval() + self.evaluate(ids) lookup_ops.tables_initializer().run() - self.assertAllEqual((1, 2, default_value), ids.eval()) + self.assertAllEqual((1, 2, default_value), self.evaluate(ids)) + @test_util.run_deprecated_v1 def test_index_table_from_file_with_oov_buckets(self): vocabulary_file = self._createVocabFile("f2i_vocab5.txt") with self.cached_session(): @@ -429,7 +451,7 @@ class IndexTableFromFile(test.TestCase): constant_op.constant(["salad", "surgery", "tarkus", "toccata"])) with self.assertRaises(errors_impl.OpError): - ids.eval() + self.evaluate(ids) lookup_ops.tables_initializer().run() self.assertAllEqual( ( @@ -437,7 +459,7 @@ class IndexTableFromFile(test.TestCase): 2, # From vocabulary file. 867, # 3 + fingerprint("tarkus") mod 300. 860), # 3 + fingerprint("toccata") mod 300. - ids.eval()) + self.evaluate(ids)) def test_index_table_from_file_fails_with_empty_vocabulary_file_name(self): self.assertRaises( @@ -468,6 +490,7 @@ class IndexTableFromFile(test.TestCase): vocabulary_file=vocabulary_file, vocab_size=0) + @test_util.run_deprecated_v1 def test_index_table_from_file_with_vocab_size_too_small(self): vocabulary_file = self._createVocabFile("f2i_vocab6.txt") with self.cached_session(): @@ -476,11 +499,12 @@ class IndexTableFromFile(test.TestCase): ids = table.lookup(constant_op.constant(["salad", "surgery", "tarkus"])) with self.assertRaises(errors_impl.OpError): - ids.eval() + self.evaluate(ids) lookup_ops.tables_initializer().run() - self.assertAllEqual((1, -1, -1), ids.eval()) + self.assertAllEqual((1, -1, -1), self.evaluate(ids)) self.assertEqual(2, table.size().eval()) + @test_util.run_deprecated_v1 def test_index_table_from_file_with_vocab_size_too_large(self): vocabulary_file = self._createVocabFile("f2i_vocab7.txt") with self.cached_session(): @@ -489,6 +513,7 @@ class IndexTableFromFile(test.TestCase): self.assertRaisesRegexp(errors_impl.InvalidArgumentError, "Invalid vocab_size", table.initializer.run) + @test_util.run_deprecated_v1 def test_index_table_from_file_with_vocab_size(self): vocabulary_file = self._createVocabFile("f2i_vocab8.txt") @@ -504,9 +529,9 @@ class IndexTableFromFile(test.TestCase): ids = table.lookup(constant_op.constant(["salad", "surgery", "tarkus"])) with self.assertRaises(errors_impl.OpError): - ids.eval() + self.evaluate(ids) lookup_ops.tables_initializer().run() - self.assertAllEqual((1, 2, -1), ids.eval()) + self.assertAllEqual((1, 2, -1), self.evaluate(ids)) self.assertEqual(3, table.size().eval()) def test_index_table_from_file_with_invalid_hashers(self): @@ -577,6 +602,7 @@ class KeyValueTensorInitializerTest(test.TestCase): table = lookup_ops.HashTable(init, default_value=-1) table.initializer.run() + @test_util.run_deprecated_v1 def test_int32(self): with ops.Graph().as_default(), self.cached_session(): init = lookup_ops.KeyValueTensorInitializer((42, 1, -1000), (0, 1, 2), @@ -590,6 +616,7 @@ class KeyValueTensorInitializerTest(test.TestCase): class IndexTableFromTensor(test.TestCase): @test_util.run_in_graph_and_eager_modes + @test_util.run_deprecated_v1 def test_index_table_from_tensor_with_tensor_init(self): table = lookup_ops.index_table_from_tensor( vocabulary_list=("brain", "salad", "surgery"), num_oov_buckets=1) @@ -606,6 +633,7 @@ class IndexTableFromTensor(test.TestCase): ids = table.lookup(constant_op.constant(("salad", "surgery", "tarkus"))) self.assertAllEqual((1, 2, 3), self.evaluate(ids)) + @test_util.run_deprecated_v1 def test_int32_index_table_from_tensor_with_tensor_init(self): with self.cached_session(): table = lookup_ops.index_table_from_tensor( @@ -614,10 +642,11 @@ class IndexTableFromTensor(test.TestCase): constant_op.constant((1, -1000, 11), dtype=dtypes.int32)) with self.assertRaises(errors_impl.FailedPreconditionError): - ids.eval() + self.evaluate(ids) lookup_ops.tables_initializer().run() - self.assertAllEqual((1, 2, 3), ids.eval()) + self.assertAllEqual((1, 2, 3), self.evaluate(ids)) + @test_util.run_deprecated_v1 def test_int64_index_table_from_tensor_with_tensor_init(self): with self.cached_session(): table = lookup_ops.index_table_from_tensor( @@ -626,10 +655,11 @@ class IndexTableFromTensor(test.TestCase): constant_op.constant((1, -1000, 11), dtype=dtypes.int64)) with self.assertRaises(errors_impl.FailedPreconditionError): - ids.eval() + self.evaluate(ids) lookup_ops.tables_initializer().run() - self.assertAllEqual((1, 2, 3), ids.eval()) + self.assertAllEqual((1, 2, 3), self.evaluate(ids)) + @test_util.run_deprecated_v1 def test_index_table_from_tensor_with_default_value(self): default_value = -42 with self.cached_session(): @@ -639,9 +669,9 @@ class IndexTableFromTensor(test.TestCase): ids = table.lookup(constant_op.constant(["salad", "surgery", "tarkus"])) with self.assertRaises(errors_impl.FailedPreconditionError): - ids.eval() + self.evaluate(ids) lookup_ops.tables_initializer().run() - self.assertAllEqual((1, 2, default_value), ids.eval()) + self.assertAllEqual((1, 2, default_value), self.evaluate(ids)) def test_index_table_from_tensor_missing_vocabulary_list(self): with self.cached_session(): @@ -650,13 +680,14 @@ class IndexTableFromTensor(test.TestCase): lookup_ops.index_table_from_tensor( vocabulary_list=None, num_oov_buckets=1) + @test_util.run_deprecated_v1 def test_index_table_from_tensor_empty_vocabulary_list(self): with self.cached_session(): table = lookup_ops.index_table_from_tensor( vocabulary_list=np.array([], dtype=np.str_), num_oov_buckets=1) ids = table.lookup(constant_op.constant(["salad", "surgery", "brain"])) with self.assertRaises(errors_impl.OpError): - ids.eval() + self.evaluate(ids) with self.assertRaisesRegexp( errors_impl.OpError, "keys and values cannot be empty"): lookup_ops.tables_initializer().run() @@ -686,6 +717,7 @@ class IndexToStringTableFromFileTest(test.TestCase): f.write("\n".join(values) + "\n") return vocabulary_file + @test_util.run_deprecated_v1 def test_index_to_string_table(self): vocabulary_path = self._createVocabFile("i2f_vocab1.txt") # vocabulary_file supports string and tensor @@ -698,11 +730,12 @@ class IndexToStringTableFromFileTest(test.TestCase): features = table.lookup( constant_op.constant([0, 1, 2, 3], dtypes.int64)) with self.assertRaises(errors_impl.OpError): - features.eval() + self.evaluate(features) lookup_ops.tables_initializer().run() self.assertAllEqual((b"brain", b"salad", b"surgery", b"UNK"), - features.eval()) + self.evaluate(features)) + @test_util.run_deprecated_v1 def test_index_to_string_table_from_multicolumn_file(self): vocabulary_file = self._createVocabFile( "f2i_vocab1.txt", values=("brain\t300", "salad\t20", "surgery\t1")) @@ -713,11 +746,12 @@ class IndexToStringTableFromFileTest(test.TestCase): value_column_index=0) features = table.lookup(constant_op.constant([0, 1, 2, 3], dtypes.int64)) with self.assertRaises(errors_impl.OpError): - features.eval() + self.evaluate(features) lookup_ops.tables_initializer().run() self.assertAllEqual((b"brain", b"salad", b"surgery", b"UNK"), - features.eval()) + self.evaluate(features)) + @test_util.run_deprecated_v1 def test_index_to_string_table_from_multicolumn_file_custom_delimiter(self): vocabulary_file = self._createVocabFile( "f2i_vocab1.txt", values=("brain 300", "salad 20", "surgery 1")) @@ -729,11 +763,12 @@ class IndexToStringTableFromFileTest(test.TestCase): delimiter=" ") features = table.lookup(constant_op.constant([0, 1, 2, 3], dtypes.int64)) with self.assertRaises(errors_impl.OpError): - features.eval() + self.evaluate(features) lookup_ops.tables_initializer().run() self.assertAllEqual((b"brain", b"salad", b"surgery", b"UNK"), - features.eval()) + self.evaluate(features)) + @test_util.run_deprecated_v1 def test_index_to_string_table_with_default_value(self): default_value = b"NONE" vocabulary_file = self._createVocabFile("f2i_vocab2.txt") @@ -742,11 +777,12 @@ class IndexToStringTableFromFileTest(test.TestCase): vocabulary_file=vocabulary_file, default_value=default_value) features = table.lookup(constant_op.constant([1, 2, 4], dtypes.int64)) with self.assertRaises(errors_impl.OpError): - features.eval() + self.evaluate(features) lookup_ops.tables_initializer().run() self.assertAllEqual((b"salad", b"surgery", default_value), - features.eval()) + self.evaluate(features)) + @test_util.run_deprecated_v1 def test_index_to_string_table_with_vocab_size_too_small(self): default_value = b"NONE" vocabulary_file = self._createVocabFile("f2i_vocab2.txt") @@ -757,11 +793,12 @@ class IndexToStringTableFromFileTest(test.TestCase): default_value=default_value) features = table.lookup(constant_op.constant([1, 2, 4], dtypes.int64)) with self.assertRaises(errors_impl.OpError): - features.eval() + self.evaluate(features) lookup_ops.tables_initializer().run() self.assertAllEqual((b"salad", default_value, default_value), - features.eval()) + self.evaluate(features)) + @test_util.run_deprecated_v1 def test_index_to_string_table_with_vocab_size_too_large(self): vocabulary_file = self._createVocabFile("f2i_vocab6.txt") with self.cached_session(): @@ -770,11 +807,12 @@ class IndexToStringTableFromFileTest(test.TestCase): features = table.lookup(constant_op.constant([1, 2, 4], dtypes.int64)) with self.assertRaises(errors_impl.OpError): - features.eval() + self.evaluate(features) init = lookup_ops.tables_initializer() self.assertRaisesRegexp(errors_impl.InvalidArgumentError, "Invalid vocab_size", init.run) + @test_util.run_deprecated_v1 def test_index_to_string_table_with_vocab_size(self): vocabulary_file = self._createVocabFile("f2i_vocab7.txt") with self.cached_session(): @@ -783,13 +821,15 @@ class IndexToStringTableFromFileTest(test.TestCase): features = table.lookup(constant_op.constant([1, 2, 4], dtypes.int64)) with self.assertRaises(errors_impl.OpError): - features.eval() + self.evaluate(features) lookup_ops.tables_initializer().run() - self.assertAllEqual((b"salad", b"surgery", b"UNK"), features.eval()) + self.assertAllEqual((b"salad", b"surgery", b"UNK"), + self.evaluate(features)) class IndexToStringTableFromTensorTest(test.TestCase): + @test_util.run_deprecated_v1 def test_index_to_string_table_from_tensor(self): with self.cached_session(): vocabulary_list = constant_op.constant(["brain", "salad", "surgery"]) @@ -799,12 +839,13 @@ class IndexToStringTableFromTensorTest(test.TestCase): indices = constant_op.constant([0, 1, 2, 3], dtypes.int64) features = table.lookup(indices) with self.assertRaises(errors_impl.OpError): - features.eval() + self.evaluate(features) lookup_ops.tables_initializer().run() self.assertAllEqual((b"brain", b"salad", b"surgery", b"UNK"), - features.eval()) + self.evaluate(features)) + @test_util.run_deprecated_v1 def test_duplicate_entries(self): with self.cached_session(): vocabulary_list = constant_op.constant(["hello", "hello"]) @@ -813,8 +854,9 @@ class IndexToStringTableFromTensorTest(test.TestCase): indices = constant_op.constant([0, 1, 4], dtypes.int64) features = table.lookup(indices) lookup_ops.tables_initializer().run() - self.assertAllEqual((b"hello", b"hello", b"UNK"), features.eval()) + self.assertAllEqual((b"hello", b"hello", b"UNK"), self.evaluate(features)) + @test_util.run_deprecated_v1 def test_index_to_string_with_default_value(self): default_value = b"NONE" with self.cached_session(): @@ -824,11 +866,11 @@ class IndexToStringTableFromTensorTest(test.TestCase): indices = constant_op.constant([1, 2, 4], dtypes.int64) features = table.lookup(indices) with self.assertRaises(errors_impl.OpError): - features.eval() + self.evaluate(features) lookup_ops.tables_initializer().run() self.assertAllEqual((b"salad", b"surgery", default_value), - features.eval()) + self.evaluate(features)) class InitializeTableFromFileOpTest(test.TestCase): @@ -854,6 +896,7 @@ class InitializeTableFromFileOpTest(test.TestCase): result = self.evaluate(output) self.assertAllEqual([0, 1, -1], result) + @test_util.run_deprecated_v1 def testInitializeInt64Table(self): vocabulary_file = self._createVocabFile( "one_column_int64.txt", values=("42", "1", "-1000")) @@ -870,9 +913,10 @@ class InitializeTableFromFileOpTest(test.TestCase): output = table.lookup( constant_op.constant((42, 1, 11), dtype=dtypes.int64)) - result = output.eval() + result = self.evaluate(output) self.assertAllEqual([0, 1, -1], result) + @test_util.run_deprecated_v1 def testInitializeIndexTable(self): vocabulary_file = self._createVocabFile("one_column_2.txt") @@ -889,9 +933,10 @@ class InitializeTableFromFileOpTest(test.TestCase): input_values = constant_op.constant([0, 1, 2, 3], dtypes.int64) output = table.lookup(input_values) - result = output.eval() + result = self.evaluate(output) self.assertAllEqual([b"brain", b"salad", b"surgery", b"UNK"], result) + @test_util.run_deprecated_v1 def testMultiColumn(self): vocabulary_file = os.path.join(self.get_temp_dir(), "three_columns.txt") with open(vocabulary_file, "w") as f: @@ -911,9 +956,10 @@ class InitializeTableFromFileOpTest(test.TestCase): input_string = constant_op.constant(["brain", "salad", "surgery"]) output = table.lookup(input_string) - result = output.eval() + result = self.evaluate(output) self.assertAllEqual([1, 5, 6], result) + @test_util.run_deprecated_v1 def testInvalidDataTypeInMultiColumn(self): vocabulary_file = os.path.join(self.get_temp_dir(), "three_columns.txt") with open(vocabulary_file, "w") as f: @@ -944,6 +990,7 @@ class InitializeTableFromFileOpTest(test.TestCase): key_index, dtypes.string, value_index), default_value) + @test_util.run_deprecated_v1 def testInvalidIndex(self): vocabulary_file = self._createVocabFile("one_column_4.txt") with self.cached_session(): @@ -958,6 +1005,7 @@ class InitializeTableFromFileOpTest(test.TestCase): with self.assertRaisesOpError("Invalid number of columns"): table.initializer.run() + @test_util.run_deprecated_v1 def testInitializeSameTableWithMultipleNodes(self): vocabulary_file = self._createVocabFile("one_column_5.txt") @@ -994,7 +1042,7 @@ class InitializeTableFromFileOpTest(test.TestCase): output2 = table2.lookup(input_string) output3 = table3.lookup(input_string) - out1, out2, out3 = sess.run([output1, output2, output3]) + out1, out2, out3 = self.evaluate([output1, output2, output3]) self.assertAllEqual([0, 1, -1], out1) self.assertAllEqual([0, 1, -1], out2) self.assertAllEqual([0, 1, -1], out3) @@ -1009,6 +1057,7 @@ class InitializeTableFromFileOpTest(test.TestCase): dtypes.int64, lookup_ops.TextFileIndex.LINE_NUMBER), default_value) + @test_util.run_deprecated_v1 def testInitializeWithVocabSize(self): with self.cached_session(): default_value = -1 @@ -1055,6 +1104,7 @@ class InitializeTableFromFileOpTest(test.TestCase): table3.initializer.run() self.assertEquals(vocab_size, table3.size().eval()) + @test_util.run_deprecated_v1 def testFeedVocabularyName(self): vocabulary_file = self._createVocabFile("feed_vocabulary.txt") @@ -1078,9 +1128,10 @@ class InitializeTableFromFileOpTest(test.TestCase): input_string = constant_op.constant(["brain", "salad", "tank"]) output = table.lookup(input_string) - result = output.eval() + result = self.evaluate(output) self.assertAllEqual([0, 1, -1], result) + @test_util.run_deprecated_v1 def testInvalidFilenames(self): vocabulary_file = self._createVocabFile("filename_shape.txt") @@ -1105,6 +1156,7 @@ class InitializeTableFromFileOpTest(test.TestCase): dtypes.int64, lookup_ops.TextFileIndex.LINE_NUMBER), default_value) + @test_util.run_deprecated_v1 def testIdToStringTable(self): vocab_file = self._createVocabFile("feat_to_id_1.txt") with self.cached_session(): @@ -1119,9 +1171,11 @@ class InitializeTableFromFileOpTest(test.TestCase): input_values = constant_op.constant([0, 1, 2, 3], dtypes.int64) out = table.lookup(input_values) - self.assertAllEqual([b"brain", b"salad", b"surgery", b"UNK"], out.eval()) + self.assertAllEqual([b"brain", b"salad", b"surgery", b"UNK"], + self.evaluate(out)) self.assertEquals(vocab_size, table.size().eval()) + @test_util.run_deprecated_v1 def testStringToIdTable(self): vocab_file = self._createVocabFile("feat_to_id_2.txt") with self.cached_session(): @@ -1135,9 +1189,10 @@ class InitializeTableFromFileOpTest(test.TestCase): input_string = constant_op.constant(["brain", "salad", "surgery", "UNK"]) out = table.lookup(input_string) - self.assertAllEqual([0, 1, 2, -1], out.eval()) + self.assertAllEqual([0, 1, 2, -1], self.evaluate(out)) self.assertEquals(vocab_size, table.size().eval()) + @test_util.run_deprecated_v1 def testInt64ToIdTable(self): vocab_file = self._createVocabFile( "feat_to_id_3.txt", values=("42", "1", "-1000")) @@ -1152,7 +1207,7 @@ class InitializeTableFromFileOpTest(test.TestCase): out = table.lookup( constant_op.constant((42, 1, -1000, 11), dtype=dtypes.int64)) - self.assertAllEqual((0, 1, 2, -1), out.eval()) + self.assertAllEqual((0, 1, 2, -1), self.evaluate(out)) self.assertEquals(vocab_size, table.size().eval()) @@ -1164,6 +1219,7 @@ class IdTableWithHashBucketsTest(test.TestCase): f.write("\n".join(values) + "\n") return vocabulary_file + @test_util.run_deprecated_v1 def testStringIdTableWithHashBuckets(self): vocab_file = self._createVocabFile("feat_to_id_1.txt") with self.cached_session(): @@ -1181,9 +1237,10 @@ class IdTableWithHashBucketsTest(test.TestCase): input_string = constant_op.constant(["brain", "salad", "surgery", "UNK"]) out = table.lookup(input_string) - self.assertAllEqual([0, 1, 2, 3], out.eval()) + self.assertAllEqual([0, 1, 2, 3], self.evaluate(out)) self.assertEquals(vocab_size + oov_buckets, table.size().eval()) + @test_util.run_deprecated_v1 def testInt32IdTableWithHashBuckets(self): vocab_file = self._createVocabFile("feat_to_id_2.txt", ("42", "1", "-1000")) with self.cached_session(): @@ -1203,9 +1260,10 @@ class IdTableWithHashBucketsTest(test.TestCase): values = constant_op.constant((42, 1, -1000, 11), dtype=dtypes.int32) out = table.lookup(values) - self.assertAllEqual([0, 1, 2, 3], out.eval()) + self.assertAllEqual([0, 1, 2, 3], self.evaluate(out)) self.assertEquals(vocab_size + oov_buckets, table.size().eval()) + @test_util.run_deprecated_v1 def testInt64IdTableWithHashBuckets(self): vocab_file = self._createVocabFile("feat_to_id_3.txt", ("42", "1", "-1000")) with self.cached_session(): @@ -1223,9 +1281,10 @@ class IdTableWithHashBucketsTest(test.TestCase): values = constant_op.constant((42, 1, -1000, 11), dtype=dtypes.int64) out = table.lookup(values) - self.assertAllEqual([0, 1, 2, 3], out.eval()) + self.assertAllEqual([0, 1, 2, 3], self.evaluate(out)) self.assertEquals(vocab_size + oov_buckets, table.size().eval()) + @test_util.run_deprecated_v1 def testStringIdTableWithOnlyHashBucket(self): with self.cached_session(): oov_buckets = 5 @@ -1244,9 +1303,10 @@ class IdTableWithHashBucketsTest(test.TestCase): 1, # fingerprint("salad") mod 5. 4 # fingerprint("surgery") mod 5 ], - out.eval()) + self.evaluate(out)) self.assertEquals(oov_buckets, table.size().eval()) + @test_util.run_deprecated_v1 def testInt32IdTableWithOnlyHashBucket(self): with self.cached_session(): oov_buckets = 5 @@ -1266,7 +1326,7 @@ class IdTableWithHashBucketsTest(test.TestCase): 4, # fingerprint("1") mod 5. 2 # fingerprint("-1000") mod 5 ], - out.eval()) + self.evaluate(out)) self.assertEquals(oov_buckets, table.size().eval()) def testFloat64IdTableWithOnlyHashBucket(self): @@ -1281,6 +1341,7 @@ class IdTableWithHashBucketsTest(test.TestCase): lookup_ops.IdTableWithHashBuckets( None, num_oov_buckets=5, key_dtype=dtypes.bool) + @test_util.run_deprecated_v1 def testIdTableWithHashBucketsWithMultipleInitializers(self): vocab_file = self._createVocabFile("feat_to_id_4.txt") with self.cached_session() as sess: @@ -1311,7 +1372,7 @@ class IdTableWithHashBucketsTest(test.TestCase): out1 = table1.lookup(input_string) out2 = table2.lookup(input_string) - out1, out2 = sess.run([out1, out2]) + out1, out2 = self.evaluate([out1, out2]) self.assertAllEqual([5, 0, 1, 2, 5], out1) self.assertAllEqual([5, 0, 1, 2, 3], out2) self.assertEquals(vocab_size + oov_buckets, table1.size().eval()) @@ -1321,6 +1382,7 @@ class IdTableWithHashBucketsTest(test.TestCase): "table2_Lookup/hash_bucket": "StringToHashBucketStrong", }, sess.graph) + @test_util.run_deprecated_v1 def testIdTableWithHashBucketsInitializationAcrossSessions(self): vocab_file = self._createVocabFile("feat_to_id_5.txt") shared_name = "across-sessions" @@ -1342,7 +1404,7 @@ class IdTableWithHashBucketsTest(test.TestCase): out1 = table1.lookup(input_string_1) - self.assertAllEqual([0, 1, 2, 3], out1.eval()) + self.assertAllEqual([0, 1, 2, 3], self.evaluate(out1)) self.assertEquals(vocab_size + oov_buckets, table1.size().eval()) with self.cached_session(): @@ -1363,9 +1425,10 @@ class IdTableWithHashBucketsTest(test.TestCase): out2 = table2.lookup(input_string_2) - self.assertAllEqual([3, 1, 3], out2.eval()) + self.assertAllEqual([3, 1, 3], self.evaluate(out2)) self.assertEquals(vocab_size + oov_buckets, table2.size().eval()) + @test_util.run_deprecated_v1 def testIdTableWithHashBucketsWithMultipleInitializersDifferentDefault(self): vocab_file = self._createVocabFile("feat_to_id_6.txt") with self.cached_session() as sess: @@ -1394,12 +1457,13 @@ class IdTableWithHashBucketsTest(test.TestCase): out1 = table1.lookup(input_string_1) out2 = table2.lookup(input_string_2) - out1, out2 = sess.run([out1, out2]) + out1, out2 = self.evaluate([out1, out2]) self.assertAllEqual([0, 1, 2, -1], out1) self.assertAllEqual([-2, 1, -2], out2) self.assertEquals(vocab_size + oov_buckets, table1.size().eval()) self.assertEquals(vocab_size + oov_buckets, table2.size().eval()) + @test_util.run_deprecated_v1 def testSparseTensor(self): vocab_file = self._createVocabFile("feat_to_id_7.txt") input_indices = [[0, 0], [0, 1], [2, 0], [2, 2], [3, 0]] @@ -1428,6 +1492,7 @@ class IdTableWithHashBucketsTest(test.TestCase): self.assertAllEqual([0, 1, 0, 2, 3], sp_ids_val) self.assertAllEqual(input_shape, sp_ids_shape) + @test_util.run_deprecated_v1 def testInt32SparseTensor(self): input_indices = [[0, 0], [0, 1], [2, 0], [2, 2], [3, 0]] input_shape = [4, 4] @@ -1456,6 +1521,7 @@ class IdTableWithHashBucketsTest(test.TestCase): self.assertAllEqual([0, 1, 0, 2, 3], sp_ids_val) self.assertAllEqual(input_shape, sp_ids_shape) + @test_util.run_deprecated_v1 def testInt64SparseTensor(self): input_indices = [[0, 0], [0, 1], [2, 0], [2, 2], [3, 0]] input_shape = [4, 4] diff --git a/tensorflow/python/kernel_tests/losses_test.py b/tensorflow/python/kernel_tests/losses_test.py index b04996f7889..abff61f81b0 100644 --- a/tensorflow/python/kernel_tests/losses_test.py +++ b/tensorflow/python/kernel_tests/losses_test.py @@ -51,58 +51,62 @@ class AbsoluteDifferenceLossTest(test.TestCase): losses.absolute_difference( self._predictions, self._predictions, weights=None) + @test_util.run_deprecated_v1 def testAllCorrectNoLossWeight(self): loss = losses.absolute_difference(self._predictions, self._predictions) with self.cached_session(): - self.assertAlmostEqual(0.0, loss.eval(), 3) + self.assertAlmostEqual(0.0, self.evaluate(loss), 3) + @test_util.run_deprecated_v1 def testNonZeroLoss(self): loss = losses.absolute_difference(self._labels, self._predictions) with self.cached_session(): - self.assertAlmostEqual(5.5, loss.eval(), 3) + self.assertAlmostEqual(5.5, self.evaluate(loss), 3) + @test_util.run_deprecated_v1 def testNonZeroLossWithPythonScalarWeight(self): weights = 2.3 loss = losses.absolute_difference(self._labels, self._predictions, weights) with self.cached_session(): - self.assertAlmostEqual(5.5 * weights, loss.eval(), 3) + self.assertAlmostEqual(5.5 * weights, self.evaluate(loss), 3) + @test_util.run_deprecated_v1 def testNonZeroLossWithScalarTensorWeight(self): weights = 2.3 loss = losses.absolute_difference(self._labels, self._predictions, constant_op.constant(weights)) with self.cached_session(): - self.assertAlmostEqual(5.5 * weights, loss.eval(), 3) + self.assertAlmostEqual(5.5 * weights, self.evaluate(loss), 3) def testNonZeroLossWithOneDimBatchSpecificWeights(self): weights = constant_op.constant((1.2, 0.0), shape=(2, 1)) loss = losses.absolute_difference(self._labels, self._predictions, weights) with self.cached_session(): - self.assertAlmostEqual(5.6, loss.eval(), 3) + self.assertAlmostEqual(5.6, self.evaluate(loss), 3) def testNonZeroLossWithTwoDimBatchSpecificWeights(self): weights = constant_op.constant([1.2, 0.0], shape=[2, 1]) loss = losses.absolute_difference(self._labels, self._predictions, weights) with self.cached_session(): - self.assertAlmostEqual(5.6, loss.eval(), 3) + self.assertAlmostEqual(5.6, self.evaluate(loss), 3) def testNonZeroLossWithSampleSpecificWeights(self): weights = constant_op.constant([3, 6, 5, 0, 4, 2], shape=[2, 3]) loss = losses.absolute_difference(self._labels, self._predictions, weights) with self.cached_session(): - self.assertAlmostEqual(16.6, loss.eval(), 3) + self.assertAlmostEqual(16.6, self.evaluate(loss), 3) def testNonZeroLossWithSampleSpecificWeightsMostZero(self): weights = constant_op.constant([0, 0, 0, 0, 0, 2], shape=[2, 3]) loss = losses.absolute_difference(self._labels, self._predictions, weights) with self.cached_session(): - self.assertAlmostEqual(6.0, loss.eval(), 3) + self.assertAlmostEqual(6.0, self.evaluate(loss), 3) def testLossWithSampleSpecificWeightsAllZero(self): weights = array_ops.zeros((2, 3)) loss = losses.absolute_difference(self._labels, self._predictions, weights) with self.cached_session(): - self.assertAlmostEqual(0.0, loss.eval(), 3) + self.assertAlmostEqual(0.0, self.evaluate(loss), 3) @test_util.assert_no_new_pyobjects_executing_eagerly def testEagerNoMemoryLeaked(self): @@ -123,6 +127,7 @@ class SoftmaxCrossEntropyLossTest(test.TestCase): with self.assertRaises(ValueError): losses.softmax_cross_entropy(labels, logits, weights=None) + @test_util.run_deprecated_v1 def testAllCorrect(self): with self.cached_session(): logits = constant_op.constant([[10.0, 0.0, 0.0], [0.0, 10.0, 0.0], @@ -132,6 +137,7 @@ class SoftmaxCrossEntropyLossTest(test.TestCase): self.assertEquals('softmax_cross_entropy_loss/value', loss.op.name) self.assertAlmostEqual(loss.eval(), 0.0, 3) + @test_util.run_deprecated_v1 def testAllWrong(self): logits = constant_op.constant([[10.0, 0.0, 0.0], [0.0, 10.0, 0.0], [0.0, 0.0, 10.0]]) @@ -142,6 +148,7 @@ class SoftmaxCrossEntropyLossTest(test.TestCase): self.assertEquals(loss.op.name, 'softmax_cross_entropy_loss/value') self.assertAlmostEqual(loss.eval(), 10.0, 3) + @test_util.run_deprecated_v1 def testNonZeroLossWithPythonScalarWeight(self): logits = constant_op.constant([[10.0, 0.0, 0.0], [0.0, 10.0, 0.0], [0.0, 0.0, 10.0]]) @@ -149,8 +156,9 @@ class SoftmaxCrossEntropyLossTest(test.TestCase): weights = 2.3 with self.cached_session(): loss = losses.softmax_cross_entropy(labels, logits, weights) - self.assertAlmostEqual(weights * 10.0, loss.eval(), 3) + self.assertAlmostEqual(weights * 10.0, self.evaluate(loss), 3) + @test_util.run_deprecated_v1 def testNonZeroLossWithScalarTensorWeight(self): logits = constant_op.constant([[10.0, 0.0, 0.0], [0.0, 10.0, 0.0], [0.0, 0.0, 10.0]]) @@ -159,7 +167,7 @@ class SoftmaxCrossEntropyLossTest(test.TestCase): with self.cached_session(): loss = losses.softmax_cross_entropy(labels, logits, constant_op.constant(weights)) - self.assertAlmostEqual(weights * 10.0, loss.eval(), 3) + self.assertAlmostEqual(weights * 10.0, self.evaluate(loss), 3) def testNonZeroLossWithOneDimBatchSpecificWeights(self): logits = constant_op.constant([[10.0, 0.0, 0.0], [0.0, 10.0, 0.0], @@ -168,7 +176,8 @@ class SoftmaxCrossEntropyLossTest(test.TestCase): weights = constant_op.constant((1.2, 3.4, 5.6)) with self.cached_session(): loss = losses.softmax_cross_entropy(labels, logits, weights) - self.assertAlmostEqual((1.2 + 3.4 + 5.6) * 10.0 / 3.0, loss.eval(), 3) + self.assertAlmostEqual((1.2 + 3.4 + 5.6) * 10.0 / 3.0, + self.evaluate(loss), 3) def testAllWrongAllWeightsMissing(self): logits = constant_op.constant([[10.0, 0.0, 0.0], [0.0, 10.0, 0.0], @@ -177,7 +186,7 @@ class SoftmaxCrossEntropyLossTest(test.TestCase): weights = constant_op.constant([0, 0, 0], shape=[3]) with self.cached_session(): loss = losses.softmax_cross_entropy(labels, logits, weights) - self.assertAlmostEqual(0.0, loss.eval(), 3) + self.assertAlmostEqual(0.0, self.evaluate(loss), 3) def testSomeWeightsMissing(self): logits = constant_op.constant([[10.0, 0.0, 0.0], [0.0, 10.0, 0.0], @@ -186,7 +195,7 @@ class SoftmaxCrossEntropyLossTest(test.TestCase): weights = constant_op.constant([1.2, 0, 0], shape=[3]) with self.cached_session(): loss = losses.softmax_cross_entropy(labels, logits, weights) - self.assertAlmostEqual(12.0, loss.eval(), 3) + self.assertAlmostEqual(12.0, self.evaluate(loss), 3) def testSoftmaxWithMeasurementSpecificWeightsRaisesException(self): with self.cached_session(): @@ -199,6 +208,7 @@ class SoftmaxCrossEntropyLossTest(test.TestCase): with self.assertRaises(ValueError): losses.softmax_cross_entropy(labels, logits, weights=weights).eval() + @test_util.run_deprecated_v1 def testSoftmaxLabelSmoothing(self): with self.cached_session(): # Softmax Cross Entropy Loss is: @@ -231,6 +241,7 @@ class SparseSoftmaxCrossEntropyLossTest(test.TestCase): with self.assertRaises(ValueError): losses.sparse_softmax_cross_entropy(labels, logits, weights=None) + @test_util.run_deprecated_v1 def testAllCorrectInt32Labels(self): with self.cached_session(): logits = constant_op.constant([[10.0, 0.0, 0.0], [0.0, 10.0, 0.0], @@ -247,6 +258,7 @@ class SparseSoftmaxCrossEntropyLossTest(test.TestCase): labels = constant_op.constant([[0], [1], [2]], dtype=dtypes.int32) losses.sparse_softmax_cross_entropy(labels, logits) + @test_util.run_deprecated_v1 def testAllCorrectInt64Labels(self): with self.cached_session(): logits = constant_op.constant([[10.0, 0.0, 0.0], [0.0, 10.0, 0.0], @@ -256,6 +268,7 @@ class SparseSoftmaxCrossEntropyLossTest(test.TestCase): self.assertEquals(loss.op.name, 'sparse_softmax_cross_entropy_loss/value') self.assertAlmostEqual(loss.eval(), 0.0, 3) + @test_util.run_deprecated_v1 def testAllCorrectNonColumnLabels(self): with self.cached_session(): logits = constant_op.constant([[10.0, 0.0, 0.0], [0.0, 10.0, 0.0], @@ -265,6 +278,7 @@ class SparseSoftmaxCrossEntropyLossTest(test.TestCase): self.assertEquals(loss.op.name, 'sparse_softmax_cross_entropy_loss/value') self.assertAlmostEqual(loss.eval(), 0.0, 3) + @test_util.run_deprecated_v1 def testAllWrongInt32Labels(self): logits = constant_op.constant([[10.0, 0.0, 0.0], [0.0, 10.0, 0.0], [0.0, 0.0, 10.0]]) @@ -275,6 +289,7 @@ class SparseSoftmaxCrossEntropyLossTest(test.TestCase): self.assertEquals(loss.op.name, 'sparse_softmax_cross_entropy_loss/value') self.assertAlmostEqual(loss.eval(), 10.0, 3) + @test_util.run_deprecated_v1 def testAllWrongInt64Labels(self): logits = constant_op.constant([[10.0, 0.0, 0.0], [0.0, 10.0, 0.0], [0.0, 0.0, 10.0]]) @@ -285,6 +300,7 @@ class SparseSoftmaxCrossEntropyLossTest(test.TestCase): self.assertEquals(loss.op.name, 'sparse_softmax_cross_entropy_loss/value') self.assertAlmostEqual(loss.eval(), 10.0, 3) + @test_util.run_deprecated_v1 def testAllWrongNonColumnLabels(self): logits = constant_op.constant([[10.0, 0.0, 0.0], [0.0, 10.0, 0.0], [0.0, 0.0, 10.0]]) @@ -295,6 +311,7 @@ class SparseSoftmaxCrossEntropyLossTest(test.TestCase): self.assertEquals(loss.op.name, 'sparse_softmax_cross_entropy_loss/value') self.assertAlmostEqual(loss.eval(), 10.0, 3) + @test_util.run_deprecated_v1 def testNonZeroLossWithPythonScalarWeight(self): logits = constant_op.constant([[10.0, 0.0, 0.0], [0.0, 10.0, 0.0], [0.0, 0.0, 10.0]]) @@ -302,8 +319,9 @@ class SparseSoftmaxCrossEntropyLossTest(test.TestCase): weights = 2.3 with self.cached_session(): loss = losses.sparse_softmax_cross_entropy(labels, logits, weights) - self.assertAlmostEqual(weights * 10.0, loss.eval(), 3) + self.assertAlmostEqual(weights * 10.0, self.evaluate(loss), 3) + @test_util.run_deprecated_v1 def testNonZeroLossWithScalarTensorWeight(self): logits = constant_op.constant([[10.0, 0.0, 0.0], [0.0, 10.0, 0.0], [0.0, 0.0, 10.0]]) @@ -312,7 +330,7 @@ class SparseSoftmaxCrossEntropyLossTest(test.TestCase): with self.cached_session(): loss = losses.sparse_softmax_cross_entropy(labels, logits, constant_op.constant(weights)) - self.assertAlmostEqual(weights * 10.0, loss.eval(), 3) + self.assertAlmostEqual(weights * 10.0, self.evaluate(loss), 3) def testNonZeroLossWith1DTensorWeight(self): logits = constant_op.constant([[10.0, 0.0, 0.0], [0.0, 10.0, 0.0], @@ -322,8 +340,9 @@ class SparseSoftmaxCrossEntropyLossTest(test.TestCase): with self.cached_session(): loss = losses.sparse_softmax_cross_entropy( labels, logits, constant_op.constant((weights,))) - self.assertAlmostEqual(weights * 10.0, loss.eval(), 3) + self.assertAlmostEqual(weights * 10.0, self.evaluate(loss), 3) + @test_util.run_deprecated_v1 def testNonZeroLossWithPlaceholderForWeights(self): logits = constant_op.constant([[10.0, 0.0, 0.0], [0.0, 10.0, 0.0], @@ -336,6 +355,7 @@ class SparseSoftmaxCrossEntropyLossTest(test.TestCase): feed_dict={weights: ((1.2,), (3.4,), (5.6,))}) self.assertAlmostEqual((1.2 + 3.4 + 5.6) * 10.0 / 3.0, loss_val, 3) + @test_util.run_deprecated_v1 def testUnknownShapePlaceholderForLogitsLabelsButScalarWeights(self): logits = array_ops.placeholder(dtypes.float32) labels = array_ops.placeholder(dtypes.int32) @@ -351,6 +371,7 @@ class SparseSoftmaxCrossEntropyLossTest(test.TestCase): }) self.assertAlmostEqual((1.0 + 1.0 + 1.0) * 10.0 / 3.0, loss_val, 3) + @test_util.run_deprecated_v1 def testNonZeroLossWithPlaceholderForLogitsLabelsAndWeights(self): logits = array_ops.placeholder(dtypes.float32, shape=(None, 3)) labels = array_ops.placeholder(dtypes.int32, shape=(None, 1)) @@ -374,7 +395,8 @@ class SparseSoftmaxCrossEntropyLossTest(test.TestCase): weights = constant_op.constant([1.2, 3.4, 5.6], shape=(3, 1)) with self.cached_session(): loss = losses.sparse_softmax_cross_entropy(labels, logits, weights) - self.assertAlmostEqual((1.2 + 3.4 + 5.6) * 10.0 / 3.0, loss.eval(), 3) + self.assertAlmostEqual((1.2 + 3.4 + 5.6) * 10.0 / 3.0, + self.evaluate(loss), 3) def testNonZeroLossWithColumnWeights(self): logits = constant_op.constant([[10.0, 0.0, 0.0], [0.0, 10.0, 0.0], @@ -383,7 +405,8 @@ class SparseSoftmaxCrossEntropyLossTest(test.TestCase): weights = constant_op.constant([[1.2], [3.4], [5.6]]) with self.cached_session(): loss = losses.sparse_softmax_cross_entropy(labels, logits, weights) - self.assertAlmostEqual((1.2 + 3.4 + 5.6) * 10.0 / 3.0, loss.eval(), 3) + self.assertAlmostEqual((1.2 + 3.4 + 5.6) * 10.0 / 3.0, + self.evaluate(loss), 3) def testAllWrongAllWeightsMissing(self): logits = constant_op.constant([[10.0, 0.0, 0.0], [0.0, 10.0, 0.0], @@ -392,7 +415,7 @@ class SparseSoftmaxCrossEntropyLossTest(test.TestCase): weights = constant_op.constant([0, 0, 0], shape=(3, 1)) with self.cached_session(): loss = losses.sparse_softmax_cross_entropy(labels, logits, weights) - self.assertAlmostEqual(0.0, loss.eval(), 3) + self.assertAlmostEqual(0.0, self.evaluate(loss), 3) def testSomeWeightsMissing(self): logits = constant_op.constant([[10.0, 0.0, 0.0], [0.0, 10.0, 0.0], @@ -401,8 +424,9 @@ class SparseSoftmaxCrossEntropyLossTest(test.TestCase): weights = constant_op.constant([1.2, 0, 0], shape=(3, 1)) with self.cached_session(): loss = losses.sparse_softmax_cross_entropy(labels, logits, weights) - self.assertAlmostEqual(12.0, loss.eval(), 3) + self.assertAlmostEqual(12.0, self.evaluate(loss), 3) + @test_util.run_deprecated_v1 def testMeasurementSpecificWeightsRaisesException(self): with self.cached_session(): logits = constant_op.constant([[100.0, -100.0, -100.0], @@ -441,6 +465,7 @@ class SparseSoftmaxCrossEntropyLossTest(test.TestCase): losses.sparse_softmax_cross_entropy( labels, logits, weights=weights).eval() + @test_util.run_deprecated_v1 def testInconsistentWeightShapeRaisesException(self): """The weight tensor has incorrect shape.""" with self.cached_session(): @@ -455,6 +480,7 @@ class SparseSoftmaxCrossEntropyLossTest(test.TestCase): losses.sparse_softmax_cross_entropy( labels, logits, weights=weights).eval() + @test_util.run_deprecated_v1 def testInconsistentLabelShapeRaisesException(self): """The label tensor has incorrect shape.""" with self.cached_session(): @@ -472,6 +498,7 @@ class SparseSoftmaxCrossEntropyLossTest(test.TestCase): class SigmoidCrossEntropyLossTest(test.TestCase): + @test_util.run_deprecated_v1 def testAllCorrectSigmoid(self): with self.cached_session(): logits = constant_op.constant([[100.0, -100.0, -100.0], @@ -481,8 +508,9 @@ class SigmoidCrossEntropyLossTest(test.TestCase): loss = losses.sigmoid_cross_entropy(labels, logits) self.assertEquals(logits.dtype, loss.dtype) self.assertEquals('sigmoid_cross_entropy_loss/value', loss.op.name) - self.assertAlmostEqual(0.0, loss.eval(), 3) + self.assertAlmostEqual(0.0, self.evaluate(loss), 3) + @test_util.run_deprecated_v1 def testLossWithSingleDimPlaceholderForLogitsAndWeights1(self): logits = array_ops.placeholder(dtypes.float32, shape=(None, 1)) labels = array_ops.placeholder(dtypes.float32, shape=(None, 1)) @@ -499,6 +527,7 @@ class SigmoidCrossEntropyLossTest(test.TestCase): }) self.assertAlmostEqual(0.313, loss, 3) + @test_util.run_deprecated_v1 def testLossWithSingleDimPlaceholderForLogitsAndWeights2(self): logits = array_ops.placeholder(dtypes.float32, shape=(None, 2)) labels = array_ops.placeholder(dtypes.float32, shape=(None, 2)) @@ -515,6 +544,7 @@ class SigmoidCrossEntropyLossTest(test.TestCase): }) self.assertAlmostEqual(0.313, loss, 3) + @test_util.run_deprecated_v1 def testAllWrongSigmoid(self): with self.cached_session(): logits = constant_op.constant([[100.0, -100.0, -100.0], @@ -526,6 +556,7 @@ class SigmoidCrossEntropyLossTest(test.TestCase): self.assertEquals('sigmoid_cross_entropy_loss/value', loss.op.name) self.assertAlmostEqual(loss.eval(), 600.0 / 9.0, 3) + @test_util.run_deprecated_v1 def testAllWrongSigmoidWithMeasurementSpecificWeights(self): with self.cached_session(): logits = constant_op.constant([[100.0, -100.0, -100.0], @@ -536,8 +567,9 @@ class SigmoidCrossEntropyLossTest(test.TestCase): loss = losses.sigmoid_cross_entropy(labels, logits, weights) self.assertEquals(logits.dtype, loss.dtype) self.assertEquals('sigmoid_cross_entropy_loss/value', loss.op.name) - self.assertAlmostEqual(1700.0 / 7.0, loss.eval(), 3) + self.assertAlmostEqual(1700.0 / 7.0, self.evaluate(loss), 3) + @test_util.run_deprecated_v1 def testMultiCorrectSigmoid(self): logits = constant_op.constant([[100.0, -100.0, 100.0], [100.0, 100.0, -100.0], @@ -548,7 +580,7 @@ class SigmoidCrossEntropyLossTest(test.TestCase): self.assertEquals('sigmoid_cross_entropy_loss/value', loss.op.name) with self.cached_session(): - self.assertAlmostEqual(0.0, loss.eval(), 3) + self.assertAlmostEqual(0.0, self.evaluate(loss), 3) def testSigmoidFloat64(self): logits = constant_op.constant(( @@ -563,7 +595,7 @@ class SigmoidCrossEntropyLossTest(test.TestCase): self.assertEquals(logits.dtype, loss.dtype) with self.cached_session(): - self.assertAlmostEqual(44.444, loss.eval(), 3) + self.assertAlmostEqual(44.444, self.evaluate(loss), 3) def testSigmoidNoReduction(self): logits = constant_op.constant(( @@ -576,12 +608,10 @@ class SigmoidCrossEntropyLossTest(test.TestCase): self.assertEquals(logits.dtype, loss.dtype) with self.cached_session(): - self.assertAllClose(( - (0., 0., 0.), - (0., 100., 100.), - (100., 0., 100.) - ), loss.eval(), 3) + self.assertAllClose(((0., 0., 0.), (0., 100., 100.), (100., 0., 100.)), + self.evaluate(loss), 3) + @test_util.run_deprecated_v1 def testSigmoidLabelSmoothingCorrect(self): with self.cached_session(): logits = constant_op.constant([[100.0, -100.0, -100.0]]) @@ -605,6 +635,7 @@ class SigmoidCrossEntropyLossTest(test.TestCase): expected_value = (100.0 + 50.0 * label_smoothing) / 3.0 self.assertAlmostEqual(loss.eval(), expected_value, 3) + @test_util.run_deprecated_v1 def testSigmoidLabelSmoothingEqualsSoftmaxTwoLabel(self): with self.cached_session(): label_smoothing = 0.1 @@ -619,7 +650,8 @@ class SigmoidCrossEntropyLossTest(test.TestCase): softmax_labels = constant_op.constant([[0, 1], [1, 0], [0, 1]]) softmax_loss = losses.softmax_cross_entropy( softmax_labels, softmax_logits, label_smoothing=label_smoothing) - self.assertAlmostEqual(sigmoid_loss.eval(), softmax_loss.eval(), 3) + self.assertAlmostEqual(sigmoid_loss.eval(), self.evaluate(softmax_loss), + 3) class LogLossTest(test.TestCase): @@ -645,11 +677,13 @@ class LogLossTest(test.TestCase): with self.assertRaises(ValueError): losses.log_loss(self._labels, self._labels, weights=None) + @test_util.run_deprecated_v1 def testAllCorrectNoLossWeight(self): loss = losses.log_loss(self._labels, self._labels) with self.cached_session(): - self.assertAlmostEqual(0.0, loss.eval(), 3) + self.assertAlmostEqual(0.0, self.evaluate(loss), 3) + @test_util.run_deprecated_v1 def testAllCorrectNoLossWeightWithPlaceholder(self): tf_predictions = array_ops.placeholder( dtypes.float32, shape=self._np_labels.shape) @@ -658,27 +692,31 @@ class LogLossTest(test.TestCase): self.assertAlmostEqual( 0.0, loss.eval(feed_dict={tf_predictions: self._np_labels}), 3) + @test_util.run_deprecated_v1 def testNonZeroLoss(self): loss = losses.log_loss(self._labels, self._predictions) with self.cached_session(): self.assertAlmostEqual(-np.sum(self._expected_losses) / 6.0, - loss.eval(), 3) + self.evaluate(loss), 3) + @test_util.run_deprecated_v1 def testNonZeroLossWithPythonScalarWeight(self): weights = 2.3 loss = losses.log_loss(self._labels, self._predictions, weights) with self.cached_session(): self.assertAlmostEqual(weights * -np.sum(self._expected_losses) / 6.0, - loss.eval(), 3) + self.evaluate(loss), 3) + @test_util.run_deprecated_v1 def testNonZeroLossWithScalarTensorWeight(self): weights = 2.3 loss = losses.log_loss(self._labels, self._predictions, constant_op.constant(weights)) with self.cached_session(): self.assertAlmostEqual(weights * -np.sum(self._expected_losses) / 6.0, - loss.eval(), 3) + self.evaluate(loss), 3) + @test_util.run_deprecated_v1 def testNonZeroLossWithScalarTensorWeightAndPlaceholder(self): tf_predictions = array_ops.placeholder( dtypes.float32, shape=self._np_predictions.shape) @@ -690,6 +728,7 @@ class LogLossTest(test.TestCase): self.assertAlmostEqual(weights * -np.sum(self._expected_losses) / 6.0, loss, 3) + @test_util.run_deprecated_v1 def testNonZeroLossWithScalarTensorWeightAndPlaceholderWithRankOnly(self): tf_predictions = array_ops.placeholder(dtypes.float32, shape=[None, None]) weights = 2.3 @@ -707,7 +746,8 @@ class LogLossTest(test.TestCase): np.asarray([1.2, 1.2, 1.2, 3.4, 3.4, 3.4]).reshape((2, 3))) loss = losses.log_loss(self._labels, self._predictions, weights) with self.cached_session(): - self.assertAlmostEqual(-np.sum(expected_losses) / 6.0, loss.eval(), 3) + self.assertAlmostEqual(-np.sum(expected_losses) / 6.0, + self.evaluate(loss), 3) def testNonZeroLossWithOneDimBatchSpecificWeightsSomeZero(self): weights = constant_op.constant((1.2, 0), shape=(2, 1)) @@ -716,7 +756,8 @@ class LogLossTest(test.TestCase): (2, 3))) loss = losses.log_loss(self._labels, self._predictions, weights) with self.cached_session(): - self.assertAlmostEqual(-np.sum(expected_losses) / 3.0, loss.eval(), 3) + self.assertAlmostEqual(-np.sum(expected_losses) / 3.0, + self.evaluate(loss), 3) def testNonZeroLossWithTwoDimBatchSpecificWeightsSomeZero(self): weights = constant_op.constant([1.2, 0], shape=[2, 1]) @@ -725,7 +766,8 @@ class LogLossTest(test.TestCase): (2, 3))) loss = losses.log_loss(self._labels, self._predictions, weights) with self.cached_session(): - self.assertAlmostEqual(-np.sum(expected_losses) / 3.0, loss.eval(), 3) + self.assertAlmostEqual(-np.sum(expected_losses) / 3.0, + self.evaluate(loss), 3) def testWeightsWithSameNumDimsButWrongShapeThrowsException(self): weights = constant_op.constant(np.random.normal(size=(2, 4)), shape=[2, 4]) @@ -743,8 +785,10 @@ class LogLossTest(test.TestCase): constant_op.constant( weights, shape=(2, 3))) with self.cached_session(): - self.assertAlmostEqual(-np.sum(expected_losses) / 5.0, loss.eval(), 3) + self.assertAlmostEqual(-np.sum(expected_losses) / 5.0, + self.evaluate(loss), 3) + @test_util.run_deprecated_v1 def testNonZeroLossWithMeasurementSpecificWeightsWithPlaceholder(self): weights = np.array([3, 6, 5, 0, 4, 2]).reshape((2, 3)) expected_losses = np.multiply(self._expected_losses, weights) @@ -770,8 +814,9 @@ class LogLossTest(test.TestCase): constant_op.constant( weights, shape=(2, 3))) with self.cached_session(): - self.assertAlmostEqual(-np.sum(expected_losses), loss.eval(), 3) + self.assertAlmostEqual(-np.sum(expected_losses), self.evaluate(loss), 3) + @test_util.run_deprecated_v1 def testNonZeroLossWithSampleSpecificWeightsMostZeroWithPlaceholder(self): weights = np.array([0, 0, 0, 0, 0, 2]).reshape((2, 3)) expected_losses = np.multiply(self._expected_losses, weights) @@ -788,7 +833,7 @@ class LogLossTest(test.TestCase): tf_weights = array_ops.zeros(shape=(2, 3)) loss = losses.log_loss(self._labels, self._predictions, tf_weights) with self.cached_session(): - self.assertAlmostEqual(0.0, loss.eval(), 3) + self.assertAlmostEqual(0.0, self.evaluate(loss), 3) class HingeLossTest(test.TestCase): @@ -800,6 +845,7 @@ class HingeLossTest(test.TestCase): with self.assertRaises(ValueError): _ = losses.hinge_loss(labels, logits).eval() + @test_util.run_deprecated_v1 def testAllOutsideMargin(self): with self.cached_session(): logits = constant_op.constant([1.2, -1.4, -1.0, 2.1]) @@ -807,6 +853,7 @@ class HingeLossTest(test.TestCase): loss = losses.hinge_loss(labels, logits) self.assertAllClose(loss.eval(), 0.0, atol=1e-3) + @test_util.run_deprecated_v1 def testSomeInsideMargin(self): with self.cached_session(): logits = constant_op.constant([[-0.7], [-1.4], [1.4], [0.6]]) @@ -816,6 +863,7 @@ class HingeLossTest(test.TestCase): # the margin so they incur some (small) loss. self.assertAllClose(loss.eval(), 0.175, atol=1e-3) + @test_util.run_deprecated_v1 def testSomeMisclassified(self): with self.cached_session(): logits = constant_op.constant([[[1.2], [0.4], [-1.0], [-1.1]]]) @@ -835,6 +883,7 @@ class HuberLossTest(test.TestCase): with self.assertRaises(ValueError): _ = losses.huber_loss(labels, predictions).eval() + @test_util.run_deprecated_v1 def testAllQuadratic(self): with self.cached_session(): predictions = constant_op.constant([1.5, -1.4, -1.0, 0.0]) @@ -843,6 +892,7 @@ class HuberLossTest(test.TestCase): self.assertAllClose(loss.eval(), 0.5 * (0.25 + 0.16 + 1.0 + 0.25) / 4., atol=1e-5) + @test_util.run_deprecated_v1 def testAllLinear(self): with self.cached_session(): predictions = constant_op.constant([1.5, -1.4, -1.0, 0.0]) @@ -851,6 +901,7 @@ class HuberLossTest(test.TestCase): self.assertAllClose(loss.eval(), (1.5 + 2.4 + 1.0 + 1.5) / 4. - 0.5, atol=1e-5) + @test_util.run_deprecated_v1 def testMixedQuadraticLinear(self): with self.cached_session(): predictions = constant_op.constant([[1.5, -1.4, -1.0, 0.0], @@ -870,7 +921,7 @@ class HuberLossTest(test.TestCase): labels = constant_op.constant([1.0, -1.0, 0.0, 0.5]) expected = 0.5 * np.array([0.5**2, 0.4**2, 0.5**2, 0.5**2]).mean() loss = losses.huber_loss(labels, predictions, delta=delta) - self.assertAllClose(expected, loss.eval(), atol=1e-5) + self.assertAllClose(expected, self.evaluate(loss), atol=1e-5) def testAllLinearDelta(self): delta = 0.5 @@ -880,7 +931,7 @@ class HuberLossTest(test.TestCase): expected -= 0.5 * delta**2 loss = losses.huber_loss(labels, predictions, delta=delta) with self.cached_session(): - self.assertAllClose(expected, loss.eval(), atol=1e-5) + self.assertAllClose(expected, self.evaluate(loss), atol=1e-5) class MeanSquaredErrorTest(test.TestCase): @@ -896,6 +947,7 @@ class MeanSquaredErrorTest(test.TestCase): losses.mean_squared_error( self._predictions, self._predictions, weights=None) + @test_util.run_deprecated_v1 def testScalar(self): with self.cached_session(): self.assertEqual( @@ -903,58 +955,62 @@ class MeanSquaredErrorTest(test.TestCase): losses.mean_squared_error(predictions=constant_op.constant(0), labels=constant_op.constant(0)).eval()) + @test_util.run_deprecated_v1 def testAllCorrectNoLossWeight(self): loss = losses.mean_squared_error(self._predictions, self._predictions) with self.cached_session(): - self.assertAlmostEqual(0.0, loss.eval(), 3) + self.assertAlmostEqual(0.0, self.evaluate(loss), 3) + @test_util.run_deprecated_v1 def testNonZeroLoss(self): loss = losses.mean_squared_error(self._labels, self._predictions) with self.cached_session(): - self.assertAlmostEqual(49.5, loss.eval(), 3) + self.assertAlmostEqual(49.5, self.evaluate(loss), 3) + @test_util.run_deprecated_v1 def testNonZeroLossWithPythonScalarWeight(self): weights = 2.3 loss = losses.mean_squared_error(self._labels, self._predictions, weights) with self.cached_session(): - self.assertAlmostEqual(49.5 * weights, loss.eval(), 3) + self.assertAlmostEqual(49.5 * weights, self.evaluate(loss), 3) + @test_util.run_deprecated_v1 def testNonZeroLossWithScalarTensorWeight(self): weights = 2.3 loss = losses.mean_squared_error(self._labels, self._predictions, constant_op.constant(weights)) with self.cached_session(): - self.assertAlmostEqual(49.5 * weights, loss.eval(), 3) + self.assertAlmostEqual(49.5 * weights, self.evaluate(loss), 3) def testNonZeroLossWithOneDimBatchSpecificWeights(self): weights = constant_op.constant([1.2, 3.4], shape=(2, 1)) loss = losses.mean_squared_error(self._labels, self._predictions, weights) with self.cached_session(): - self.assertAlmostEqual(767.8 / 6.0, loss.eval(), 3) + self.assertAlmostEqual(767.8 / 6.0, self.evaluate(loss), 3) def testNonZeroLossWithTwoDimBatchSpecificWeights(self): weights = constant_op.constant([1.2, 3.4], shape=[2, 1]) loss = losses.mean_squared_error(self._labels, self._predictions, weights) with self.cached_session(): - self.assertAlmostEqual(767.8 / 6.0, loss.eval(), 3) + self.assertAlmostEqual(767.8 / 6.0, self.evaluate(loss), 3) def testNonZeroLossWithSampleSpecificWeights(self): weights = constant_op.constant([3, 6, 5, 0, 4, 2], shape=[2, 3]) loss = losses.mean_squared_error(self._labels, self._predictions, weights) with self.cached_session(): - self.assertAlmostEqual(587 / 5.0, loss.eval(), 3) + self.assertAlmostEqual(587 / 5.0, self.evaluate(loss), 3) def testNonZeroLossWithSampleSpecificWeightsMostZero(self): weights = constant_op.constant([0, 0, 0, 0, 0, 2], shape=[2, 3]) loss = losses.mean_squared_error(self._labels, self._predictions, weights) with self.cached_session(): - self.assertAlmostEqual(18.0, loss.eval(), 3) + self.assertAlmostEqual(18.0, self.evaluate(loss), 3) def testLossWithSampleSpecificWeightsAllZero(self): weights = array_ops.zeros((2, 3)) loss = losses.mean_squared_error(self._labels, self._predictions, weights) with self.cached_session(): - self.assertAlmostEqual(0.0, loss.eval(), 3) + self.assertAlmostEqual(0.0, self.evaluate(loss), 3) class MeanPairwiseSquaredErrorTest(test.TestCase): @@ -991,7 +1047,8 @@ class MeanPairwiseSquaredErrorTest(test.TestCase): with self.cached_session(): static_inputs_op = losses.mean_pairwise_squared_error( predictions=predictions, labels=labels, weights=weights) - self.assertAlmostEqual(expected_loss, static_inputs_op.eval(), places=3) + self.assertAlmostEqual( + expected_loss, self.evaluate(static_inputs_op), places=3) predictions_placeholder = array_ops.placeholder( dtypes.float32, shape=np.asarray(predictions.shape)) @@ -1011,10 +1068,12 @@ class MeanPairwiseSquaredErrorTest(test.TestCase): self.assertAlmostEqual( expected_loss, dynamic_inputs_op.eval(feed_dict=feed_dict), places=3) + @test_util.run_deprecated_v1 def testAllCorrectNoLossWeight(self): self._test_valid_weights( self._labels, self._labels, expected_loss=0.0) + @test_util.run_deprecated_v1 def testNonZeroLoss(self): self._test_valid_weights( self._labels, self._predictions, @@ -1040,11 +1099,12 @@ class MeanPairwiseSquaredErrorTest(test.TestCase): init_op = variables.global_variables_initializer() with self.cached_session() as sess: - sess.run(init_op) + self.evaluate(init_op) for grad, _ in gradients_to_variables: - np_grad = sess.run(grad) + np_grad = self.evaluate(grad) self.assertFalse(np.isnan(np_grad).any()) + @test_util.run_deprecated_v1 def testNonZeroLossWithPythonScalarWeight(self): weight = 2.3 self._test_valid_weights( @@ -1052,6 +1112,7 @@ class MeanPairwiseSquaredErrorTest(test.TestCase): expected_loss=weight * np.sum(self._expected_losses), weights=weight) + @test_util.run_deprecated_v1 def testNonZeroLossWithScalarTensorWeight(self): weights = 2.3 loss = losses.mean_pairwise_squared_error( @@ -1060,12 +1121,14 @@ class MeanPairwiseSquaredErrorTest(test.TestCase): weights=constant_op.constant(weights)) with self.cached_session(): self.assertAlmostEqual(weights * np.sum(self._expected_losses), - loss.eval(), 3) + self.evaluate(loss), 3) + @test_util.run_deprecated_v1 def testNonZeroLossWithScalarZeroWeight(self): self._test_valid_weights( self._labels, self._predictions, expected_loss=0.0, weights=0.0) + @test_util.run_deprecated_v1 def test3d(self): labels = np.array([ [[1, 9, 2], [12, 11, 10], [9, 8, 7]], @@ -1077,6 +1140,7 @@ class MeanPairwiseSquaredErrorTest(test.TestCase): ]) self._test_valid_weights(labels, predictions, expected_loss=137.5) + @test_util.run_deprecated_v1 def test3dWeightedScalar(self): labels = np.array([ [[1, 9, 2], [12, 11, 10], [9, 8, 7]], @@ -1115,6 +1179,7 @@ class MeanPairwiseSquaredErrorTest(test.TestCase): weights_placeholder: weights, }) + @test_util.run_deprecated_v1 def testInvalid3dWeighted2x0(self): labels = np.array([ [[1, 9, 2], [12, 11, 10], [9, 8, 7]], @@ -1127,6 +1192,7 @@ class MeanPairwiseSquaredErrorTest(test.TestCase): self._test_invalid_weights( labels, predictions, weights=np.asarray((1.2, 3.4))) + @test_util.run_deprecated_v1 def test3dWeighted2x3x3(self): labels = np.array([ [[1, 9, 2], [12, 11, 10], [9, 8, 7]], @@ -1143,6 +1209,7 @@ class MeanPairwiseSquaredErrorTest(test.TestCase): expected_loss=9 * 137.5, weights=np.ones((2, 3, 3))) + @test_util.run_deprecated_v1 def testLossWithAllZeroBatchSpecificWeights(self): self._test_valid_weights( self._labels, self._predictions, expected_loss=0.0, @@ -1215,7 +1282,7 @@ class CosineDistanceLossTest(test.TestCase): labels=constant_op.constant(self._labels), dim=2) with self.cached_session(): - self.assertAlmostEqual(0, loss.eval(), 5) + self.assertAlmostEqual(0, self.evaluate(loss), 5) def testPartiallyCorrectWithIntegerValues(self): loss = losses.cosine_distance( @@ -1223,7 +1290,7 @@ class CosineDistanceLossTest(test.TestCase): labels=constant_op.constant(self._labels), dim=2) with self.cached_session(): - self.assertAlmostEqual(1, loss.eval(), 5) + self.assertAlmostEqual(1, self.evaluate(loss), 5) def testPartiallyCorrectFloatingPointValues(self): predictions = np.matrix( @@ -1241,7 +1308,7 @@ class CosineDistanceLossTest(test.TestCase): loss = losses.cosine_distance(tf_labels, tf_preds, dim=2) with self.cached_session(): - self.assertAlmostEqual(1.0, loss.eval(), 5) + self.assertAlmostEqual(1.0, self.evaluate(loss), 5) def testSampleSpecificWeights(self): loss = losses.cosine_distance( @@ -1250,7 +1317,7 @@ class CosineDistanceLossTest(test.TestCase): dim=2, weights=np.asarray((1, 0, 0)).reshape((3, 1, 1))) with self.cached_session(): - self.assertEqual(1.0, loss.eval()) + self.assertEqual(1.0, self.evaluate(loss)) def testMeasurementSpecificWeights(self): loss = losses.cosine_distance( @@ -1260,8 +1327,9 @@ class CosineDistanceLossTest(test.TestCase): weights=constant_op.constant( [1, 0, 0, 1, 1, 1], shape=(3, 2, 1))) with self.cached_session(): - self.assertEqual(3.0 / 4.0, loss.eval()) + self.assertEqual(3.0 / 4.0, self.evaluate(loss)) + @test_util.run_deprecated_v1 def testMeasurementSpecificWeightsWithPlaceholderWithShape(self): tf_predictions = array_ops.placeholder( dtypes.float32, shape=self._labels.shape) @@ -1282,7 +1350,7 @@ class CosineDistanceLossTest(test.TestCase): dim=2, weights=array_ops.zeros((3, 1, 1))) with self.cached_session(): - self.assertEqual(0, loss.eval()) + self.assertEqual(0, self.evaluate(loss)) def testZeroLossWhenAllMeasurementSpecificWeightsAreZero(self): loss = losses.cosine_distance( @@ -1291,7 +1359,7 @@ class CosineDistanceLossTest(test.TestCase): dim=2, weights=array_ops.zeros((3, 2, 1))) with self.cached_session(): - self.assertEqual(0, loss.eval()) + self.assertEqual(0, self.evaluate(loss)) class AddLossTest(test.TestCase): @@ -1351,15 +1419,16 @@ class ComputeWeightedLossTest(test.TestCase): with self.session(g): for unweighted_loss in unweighted_losses: if reduction == losses.Reduction.NONE: - self.assertAllClose(self._raw_losses, unweighted_loss.eval()) + self.assertAllClose(self._raw_losses, + self.evaluate(unweighted_loss)) elif reduction == losses.Reduction.SUM: self.assertAllClose( - np.sum(self._raw_losses), unweighted_loss.eval()) + np.sum(self._raw_losses), self.evaluate(unweighted_loss)) else: # reduction one of MEAN, SUM_OVER_NONZERO_WEIGHTS, # SUM_BY_NONZERO_WEIGHTS or SUM_OVER_BATCH_SIZE. self.assertAllClose( - np.mean(self._raw_losses), unweighted_loss.eval()) + np.mean(self._raw_losses), self.evaluate(unweighted_loss)) def testUnweightedFromPlaceholder(self): for reduction in losses.Reduction.all(): @@ -1398,7 +1467,7 @@ class ComputeWeightedLossTest(test.TestCase): self.assertEqual(1, len(util.get_losses())) with self.cached_session(): self.assertAllClose( - np.mean(weight * self._raw_losses), weighted_loss.eval()) + np.mean(weight * self._raw_losses), self.evaluate(weighted_loss)) def _test_invalid_weights(self, weights): with ops.Graph().as_default(): @@ -1470,24 +1539,22 @@ class ComputeWeightedLossTest(test.TestCase): weighted_losses = weights * self._raw_losses weighted_sum = np.sum(weighted_losses) if reduction == losses.Reduction.NONE: - self.assertAllClose(weighted_losses, weighted_loss.eval()) + self.assertAllClose(weighted_losses, self.evaluate(weighted_loss)) elif reduction == losses.Reduction.SUM: - self.assertAllClose(weighted_sum, weighted_loss.eval()) + self.assertAllClose(weighted_sum, self.evaluate(weighted_loss)) else: broadcast_weights = weights * np.ones_like(self._raw_losses) if reduction == losses.Reduction.MEAN: - self.assertAllClose( - weighted_sum / np.sum(broadcast_weights), - weighted_loss.eval()) + self.assertAllClose(weighted_sum / np.sum(broadcast_weights), + self.evaluate(weighted_loss)) elif (reduction == losses.Reduction.SUM_OVER_NONZERO_WEIGHTS or reduction == losses.Reduction.SUM_BY_NONZERO_WEIGHTS): self.assertAllClose( weighted_sum / np.count_nonzero(broadcast_weights), - weighted_loss.eval()) + self.evaluate(weighted_loss)) elif reduction == losses.Reduction.SUM_OVER_BATCH_SIZE: - self.assertAllClose( - weighted_sum / self._raw_losses.size, - weighted_loss.eval()) + self.assertAllClose(weighted_sum / self._raw_losses.size, + self.evaluate(weighted_loss)) def test1x1x1Weight(self): self._test_valid_weights((((17.0,),),)) diff --git a/tensorflow/python/kernel_tests/lrn_op_test.py b/tensorflow/python/kernel_tests/lrn_op_test.py index 7ebeb91d90e..fbe628c3944 100644 --- a/tensorflow/python/kernel_tests/lrn_op_test.py +++ b/tensorflow/python/kernel_tests/lrn_op_test.py @@ -24,6 +24,7 @@ import numpy as np from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes +from tensorflow.python.framework import test_util from tensorflow.python.ops import array_ops from tensorflow.python.ops import gradient_checker from tensorflow.python.ops import gradients_impl @@ -92,6 +93,7 @@ class LRNOpTest(test.TestCase): self.assertTrue(err < 1e-2) self.assertShapeEqual(expected, lrn_t) + @test_util.run_deprecated_v1 def testCompute(self): for _ in range(2): self._RunAndVerify(dtypes.float32) @@ -99,6 +101,7 @@ class LRNOpTest(test.TestCase): if not test.is_gpu_available(): self._RunAndVerify(dtypes.float16) + @test_util.run_deprecated_v1 def testGradientsZeroInput(self): with self.session(use_gpu=True): shape = [4, 4, 4, 4] @@ -147,6 +150,7 @@ class LRNOpTest(test.TestCase): else: self.assertLess(err, 1.0) + @test_util.run_deprecated_v1 def testGradients(self): for _ in range(2): self._RunAndVerifyGradients(dtypes.float32) diff --git a/tensorflow/python/kernel_tests/manip_ops_test.py b/tensorflow/python/kernel_tests/manip_ops_test.py index f71857a3cba..5700db4b950 100644 --- a/tensorflow/python/kernel_tests/manip_ops_test.py +++ b/tensorflow/python/kernel_tests/manip_ops_test.py @@ -62,6 +62,7 @@ class RollTest(test_util.TensorFlowTestCase): if np_input.dtype == np.float32: self._testGradient(np_input, shift, axis) + @test_util.run_deprecated_v1 def testIntTypes(self): for t in [np.int32, np.int64]: self._testAll(np.random.randint(-100, 100, (5)).astype(t), 3, 0) @@ -73,6 +74,7 @@ class RollTest(test_util.TensorFlowTestCase): np.random.randint(-100, 100, (4, 2, 1, 3)).astype(t), [0, 1, -2], [1, 2, 3]) + @test_util.run_deprecated_v1 def testFloatTypes(self): for t in [np.float32, np.float64]: self._testAll(np.random.rand(5).astype(t), 2, 0) @@ -80,6 +82,7 @@ class RollTest(test_util.TensorFlowTestCase): self._testAll(np.random.rand(3, 4).astype(t), [1, 2], [1, 0]) self._testAll(np.random.rand(1, 3, 4).astype(t), [1, 0, -3], [0, 1, 2]) + @test_util.run_deprecated_v1 def testComplexTypes(self): for t in [np.complex64, np.complex128]: x = np.random.rand(4, 4).astype(t) @@ -90,6 +93,7 @@ class RollTest(test_util.TensorFlowTestCase): x = np.random.rand(3, 2, 1, 1).astype(t) self._testAll(x + 1j * x, [2, 1, 1, 0], [0, 3, 1, 2]) + @test_util.run_deprecated_v1 def testNegativeAxis(self): self._testAll(np.random.randint(-100, 100, (5)).astype(np.int32), 3, -1) self._testAll(np.random.randint(-100, 100, (4, 4)).astype(np.int32), 3, -2) @@ -100,12 +104,14 @@ class RollTest(test_util.TensorFlowTestCase): manip_ops.roll(np.random.randint(-100, 100, (4, 4)).astype(np.int32), 3, -10).eval() + @test_util.run_deprecated_v1 def testInvalidInputShape(self): # The input should be 1-D or higher, checked in shape function. with self.assertRaisesRegexp( ValueError, "Shape must be at least rank 1 but is rank 0"): manip_ops.roll(7, 1, 0) + @test_util.run_deprecated_v1 def testRollInputMustVectorHigherRaises(self): # The input should be 1-D or higher, checked in kernel. tensor = array_ops.placeholder(dtype=dtypes.int32) @@ -116,12 +122,14 @@ class RollTest(test_util.TensorFlowTestCase): "input must be 1-D or higher"): manip_ops.roll(tensor, shift, axis).eval(feed_dict={tensor: 7}) + @test_util.run_deprecated_v1 def testInvalidAxisShape(self): # The axis should be a scalar or 1-D, checked in shape function. with self.assertRaisesRegexp( ValueError, "Shape must be at most rank 1 but is rank 2"): manip_ops.roll([[1, 2], [3, 4]], 1, [[0, 1]]) + @test_util.run_deprecated_v1 def testRollAxisMustBeScalarOrVectorRaises(self): # The axis should be a scalar or 1-D, checked in kernel. tensor = [[1, 2], [3, 4]] @@ -132,12 +140,14 @@ class RollTest(test_util.TensorFlowTestCase): "axis must be a scalar or a 1-D vector"): manip_ops.roll(tensor, shift, axis).eval(feed_dict={axis: [[0, 1]]}) + @test_util.run_deprecated_v1 def testInvalidShiftShape(self): # The shift should be a scalar or 1-D, checked in shape function. with self.assertRaisesRegexp( ValueError, "Shape must be at most rank 1 but is rank 2"): manip_ops.roll([[1, 2], [3, 4]], [[0, 1]], 1) + @test_util.run_deprecated_v1 def testRollShiftMustBeScalarOrVectorRaises(self): # The shift should be a scalar or 1-D, checked in kernel. tensor = [[1, 2], [3, 4]] @@ -148,11 +158,13 @@ class RollTest(test_util.TensorFlowTestCase): "shift must be a scalar or a 1-D vector"): manip_ops.roll(tensor, shift, axis).eval(feed_dict={shift: [[0, 1]]}) + @test_util.run_deprecated_v1 def testInvalidShiftAndAxisNotEqualShape(self): # The shift and axis must be same size, checked in shape function. with self.assertRaisesRegexp(ValueError, "both shapes must be equal"): manip_ops.roll([[1, 2], [3, 4]], [1], [0, 1]) + @test_util.run_deprecated_v1 def testRollShiftAndAxisMustBeSameSizeRaises(self): # The shift and axis must be same size, checked in kernel. tensor = [[1, 2], [3, 4]] diff --git a/tensorflow/python/kernel_tests/map_stage_op_test.py b/tensorflow/python/kernel_tests/map_stage_op_test.py index d503f3d7c9f..dd16fad6904 100644 --- a/tensorflow/python/kernel_tests/map_stage_op_test.py +++ b/tensorflow/python/kernel_tests/map_stage_op_test.py @@ -19,6 +19,7 @@ from __future__ import print_function from tensorflow.python.framework import errors from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops +from tensorflow.python.framework import test_util from tensorflow.python.ops import array_ops from tensorflow.python.ops import data_flow_ops from tensorflow.python.ops import math_ops @@ -29,6 +30,7 @@ TIMEOUT = 1 class MapStageTest(test.TestCase): + @test_util.run_deprecated_v1 def testSimple(self): with ops.Graph().as_default() as G: with ops.device('/cpu:0'): @@ -50,6 +52,7 @@ class MapStageTest(test.TestCase): _, yval = sess.run([stage, y], feed_dict={x: i, pi: i + 1, gi: i}) self.assertAllClose(4 * (i - 1) * (i - 1) * 128, yval, rtol=1e-4) + @test_util.run_deprecated_v1 def testMultiple(self): with ops.Graph().as_default() as G: with ops.device('/cpu:0'): @@ -72,6 +75,7 @@ class MapStageTest(test.TestCase): self.assertAllClose( 4 * (i - 1) * (i - 1) * (i - 1) * 128, yval, rtol=1e-4) + @test_util.run_deprecated_v1 def testDictionary(self): with ops.Graph().as_default() as G: with ops.device('/cpu:0'): @@ -121,6 +125,7 @@ class MapStageTest(test.TestCase): G.finalize() + @test_util.run_deprecated_v1 def testPeek(self): with ops.Graph().as_default() as G: with ops.device('/cpu:0'): @@ -150,6 +155,7 @@ class MapStageTest(test.TestCase): self.assertTrue(sess.run(size) == 10) + @test_util.run_deprecated_v1 def testSizeAndClear(self): with ops.Graph().as_default() as G: with ops.device('/cpu:0'): @@ -176,6 +182,7 @@ class MapStageTest(test.TestCase): sess.run(clear) self.assertEqual(sess.run(size), 0) + @test_util.run_deprecated_v1 def testCapacity(self): capacity = 3 @@ -239,6 +246,7 @@ class MapStageTest(test.TestCase): self.assertTrue(sess.run(size) == 0) + @test_util.run_deprecated_v1 def testMemoryLimit(self): memory_limit = 512 * 1024 # 512K chunk = 200 * 1024 # 256K @@ -303,6 +311,7 @@ class MapStageTest(test.TestCase): self.assertTrue(sess.run(size) == 0) + @test_util.run_deprecated_v1 def testOrdering(self): import six import random @@ -341,6 +350,7 @@ class MapStageTest(test.TestCase): self.assertTrue(sess.run(size) == 0) + @test_util.run_deprecated_v1 def testPartialDictInsert(self): with ops.Graph().as_default() as G: with ops.device('/cpu:0'): @@ -400,6 +410,7 @@ class MapStageTest(test.TestCase): 'v': 3 }]) + @test_util.run_deprecated_v1 def testPartialIndexInsert(self): with ops.Graph().as_default() as G: with ops.device('/cpu:0'): @@ -443,6 +454,7 @@ class MapStageTest(test.TestCase): # We can now obtain tuple associated with key 1 self.assertTrue(sess.run([key, ret], feed_dict={gi: 1}) == [1, [1, 3, 2]]) + @test_util.run_deprecated_v1 def testPartialDictGetsAndPeeks(self): with ops.Graph().as_default() as G: with ops.device('/cpu:0'): @@ -540,6 +552,7 @@ class MapStageTest(test.TestCase): # Nothing is left self.assertTrue(sess.run([size, isize]) == [0, 0]) + @test_util.run_deprecated_v1 def testPartialIndexGets(self): with ops.Graph().as_default() as G: with ops.device('/cpu:0'): diff --git a/tensorflow/python/kernel_tests/matmul_op_test.py b/tensorflow/python/kernel_tests/matmul_op_test.py index 1c2822180ac..983f463f5e3 100644 --- a/tensorflow/python/kernel_tests/matmul_op_test.py +++ b/tensorflow/python/kernel_tests/matmul_op_test.py @@ -44,7 +44,7 @@ class MatVecTest(test_lib.TestCase): with self.cached_session(): c = math_ops.matvec(a, b) self.assertAllEqual((2,), c.shape) - c_ = c.eval() + c_ = self.evaluate(c) self.assertAllEqual([5 + 2 * 6, 3 * 5 + 4 * 6], c_) @@ -90,7 +90,7 @@ def _GetMatMulTest(a_np_, b_np_, use_static_shape_, **kwargs_): a = constant_op.constant(effective_a_np) b = constant_op.constant(effective_b_np) res = math_ops.matmul(a, b, **kwargs_) - tf_val = res.eval() + tf_val = self.evaluate(res) else: a = array_ops.placeholder(a_np_.dtype) b = array_ops.placeholder(b_np_.dtype) @@ -194,6 +194,7 @@ except AttributeError: class MatMulInfixOperatorTest(test_lib.TestCase): + @test_util.run_deprecated_v1 def testMismatchedShape(self): with self.assertRaisesWithPredicateMatch(ValueError, lambda e: "Shape must" in str(e)): @@ -201,6 +202,7 @@ class MatMulInfixOperatorTest(test_lib.TestCase): ops.convert_to_tensor([10.0, 20.0, 30.0]), ops.convert_to_tensor([[40.0, 50.0], [60.0, 70.0]])) + @test_util.run_deprecated_v1 def testMismatchedDimensions(self): with self.assertRaisesWithPredicateMatch( ValueError, lambda e: "Dimensions must" in str(e)): @@ -208,19 +210,21 @@ class MatMulInfixOperatorTest(test_lib.TestCase): ops.convert_to_tensor([[10.0, 20.0, 30.0]]), ops.convert_to_tensor([[40.0, 50.0], [60.0, 70.0]])) + @test_util.run_deprecated_v1 def testInfixMatmulIsTfMatmul(self): a = ops.convert_to_tensor([[10.0, 20.0, 30.0]]) b = ops.convert_to_tensor([[40.0, 50.0], [60.0, 70.0], [80.0, 90.0]]) c = infix_matmul(a, b) self.assertEqual(c.op.type, "MatMul") + @test_util.run_deprecated_v1 def testInfixMatmulDoesDotProduct(self): a = ops.convert_to_tensor([[10.0, 20.0, 30.0]]) b = ops.convert_to_tensor([[40.0, 50.0], [60.0, 70.0], [80.0, 90.0]]) c = infix_matmul(a, b) d = math_ops.matmul(a, b) with self.cached_session(): - self.assertAllEqual(c.eval(), d.eval()) + self.assertAllEqual(c.eval(), self.evaluate(d)) if __name__ == "__main__": diff --git a/tensorflow/python/kernel_tests/matrix_band_part_op_test.py b/tensorflow/python/kernel_tests/matrix_band_part_op_test.py index 93a668f1259..129ea40dfe6 100644 --- a/tensorflow/python/kernel_tests/matrix_band_part_op_test.py +++ b/tensorflow/python/kernel_tests/matrix_band_part_op_test.py @@ -62,7 +62,7 @@ def _GetMatrixBandPartTest(dtype_, batch_shape_, shape_): batch_mat, constant_op.constant(lower, index_dtype), constant_op.constant(upper, index_dtype)) - self.assertAllEqual(band_np, band.eval()) + self.assertAllEqual(band_np, self.evaluate(band)) return Test diff --git a/tensorflow/python/kernel_tests/matrix_exponential_op_test.py b/tensorflow/python/kernel_tests/matrix_exponential_op_test.py index 3abdf50ece5..372b6dc17f4 100644 --- a/tensorflow/python/kernel_tests/matrix_exponential_op_test.py +++ b/tensorflow/python/kernel_tests/matrix_exponential_op_test.py @@ -25,6 +25,7 @@ import numpy as np from tensorflow.python.client import session from tensorflow.python.framework import constant_op from tensorflow.python.framework import ops +from tensorflow.python.framework import test_util from tensorflow.python.ops import array_ops from tensorflow.python.ops import control_flow_ops from tensorflow.python.ops import random_ops @@ -50,7 +51,7 @@ class ExponentialOpTest(test.TestCase): def _verifyExponential(self, x, np_type): inp = x.astype(np_type) - with self.cached_session(use_gpu=True): + with test_util.use_gpu(): tf_ans = linalg_impl.matrix_exponential(inp) if x.size == 0: np_ans = np.empty(x.shape, dtype=np_type) @@ -61,7 +62,7 @@ class ExponentialOpTest(test.TestCase): np_ans[i] = np_expm(inp[i]) else: np_ans = np_expm(inp) - out = tf_ans.eval() + out = self.evaluate(tf_ans) self.assertAllClose(np_ans, out, rtol=1e-4, atol=1e-3) def _verifyExponentialReal(self, x): @@ -121,12 +122,14 @@ class ExponentialOpTest(test.TestCase): # Complex batch self._verifyExponentialComplex(self._makeBatch(matrix1, matrix2)) + @test_util.run_deprecated_v1 def testNonSquareMatrix(self): # When the exponential of a non-square matrix is attempted we should return # an error with self.assertRaises(ValueError): linalg_impl.matrix_exponential(np.array([[1., 2., 3.], [3., 4., 5.]])) + @test_util.run_deprecated_v1 def testWrongDimensions(self): # The input to the exponential should be at least a 2-dimensional tensor. tensor3 = constant_op.constant([1., 2.]) @@ -137,6 +140,7 @@ class ExponentialOpTest(test.TestCase): self._verifyExponentialReal(np.empty([0, 2, 2])) self._verifyExponentialReal(np.empty([2, 0, 0])) + @test_util.run_deprecated_v1 def testDynamic(self): with self.session(use_gpu=True) as sess: inp = array_ops.placeholder(ops.dtypes.float32) @@ -144,13 +148,14 @@ class ExponentialOpTest(test.TestCase): matrix = np.array([[1., 2.], [3., 4.]]) sess.run(expm, feed_dict={inp: matrix}) + @test_util.run_deprecated_v1 def testConcurrentExecutesWithoutError(self): with self.session(use_gpu=True) as sess: matrix1 = random_ops.random_normal([5, 5], seed=42) matrix2 = random_ops.random_normal([5, 5], seed=42) expm1 = linalg_impl.matrix_exponential(matrix1) expm2 = linalg_impl.matrix_exponential(matrix2) - expm = sess.run([expm1, expm2]) + expm = self.evaluate([expm1, expm2]) self.assertAllEqual(expm[0], expm[1]) diff --git a/tensorflow/python/kernel_tests/matrix_inverse_op_test.py b/tensorflow/python/kernel_tests/matrix_inverse_op_test.py index 2247f1541e2..5cef4b79a32 100644 --- a/tensorflow/python/kernel_tests/matrix_inverse_op_test.py +++ b/tensorflow/python/kernel_tests/matrix_inverse_op_test.py @@ -46,7 +46,7 @@ class InverseOpTest(test.TestCase): tiling = list(y.shape) tiling[-2:] = [1, 1] np_ans = np.tile(np_ans, tiling) - out = tf_ans.eval() + out = self.evaluate(tf_ans) self.assertAllClose(np_ans, out, rtol=1e-4, atol=1e-3) self.assertShapeEqual(y, tf_ans) @@ -146,7 +146,7 @@ class InverseOpTest(test.TestCase): inv1 = linalg_ops.matrix_inverse(matrix1, adjoint=adjoint_) inv2 = linalg_ops.matrix_inverse(matrix2, adjoint=adjoint_) all_ops += [inv1, inv2] - inv = sess.run(all_ops) + inv = self.evaluate(all_ops) self.assertAllEqual(inv[0], inv[1]) self.assertAllEqual(inv[2], inv[3]) diff --git a/tensorflow/python/kernel_tests/matrix_logarithm_op_test.py b/tensorflow/python/kernel_tests/matrix_logarithm_op_test.py index 2010a4b2a86..b0bce6a1b9b 100644 --- a/tensorflow/python/kernel_tests/matrix_logarithm_op_test.py +++ b/tensorflow/python/kernel_tests/matrix_logarithm_op_test.py @@ -25,6 +25,7 @@ from tensorflow.python.client import session from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops +from tensorflow.python.framework import test_util from tensorflow.python.ops import control_flow_ops from tensorflow.python.ops import gen_linalg_ops from tensorflow.python.ops import math_ops @@ -39,11 +40,11 @@ class LogarithmOpTest(test.TestCase): def _verifyLogarithm(self, x, np_type): inp = x.astype(np_type) - with self.cached_session(use_gpu=True): + with test_util.use_gpu(): # Verify that expm(logm(A)) == A. tf_ans = linalg_impl.matrix_exponential( gen_linalg_ops.matrix_logarithm(inp)) - out = tf_ans.eval() + out = self.evaluate(tf_ans) self.assertAllClose(inp, out, rtol=1e-4, atol=1e-3) def _verifyLogarithmComplex(self, x): @@ -128,7 +129,7 @@ class LogarithmOpTest(test.TestCase): random_ops.random_normal([5, 5], seed=42), dtypes.complex64) logm1 = gen_linalg_ops.matrix_logarithm(matrix1) logm2 = gen_linalg_ops.matrix_logarithm(matrix2) - logm = sess.run([logm1, logm2]) + logm = self.evaluate([logm1, logm2]) self.assertAllEqual(logm[0], logm[1]) diff --git a/tensorflow/python/kernel_tests/matrix_solve_ls_op_test.py b/tensorflow/python/kernel_tests/matrix_solve_ls_op_test.py index 13a7df7f95d..a6f5da9d3d7 100644 --- a/tensorflow/python/kernel_tests/matrix_solve_ls_op_test.py +++ b/tensorflow/python/kernel_tests/matrix_solve_ls_op_test.py @@ -24,6 +24,7 @@ from tensorflow.python.client import session from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops +from tensorflow.python.framework import test_util from tensorflow.python.ops import array_ops from tensorflow.python.ops import control_flow_ops from tensorflow.python.ops import linalg_ops @@ -133,6 +134,7 @@ class MatrixSolveLsOpTest(test_lib.TestCase): self.assertEqual(np_ans.shape, tf_ans_val.shape) self.assertAllClose(np_ans, tf_ans_val, atol=2 * tol, rtol=2 * tol) + @test_util.run_deprecated_v1 def testWrongDimensions(self): # The matrix and right-hand sides should have the same number of rows. with self.session(use_gpu=True): @@ -141,6 +143,7 @@ class MatrixSolveLsOpTest(test_lib.TestCase): with self.assertRaises(ValueError): linalg_ops.matrix_solve_ls(matrix, rhs) + @test_util.run_deprecated_v1 def testEmpty(self): full = np.array([[1., 2.], [3., 4.], [5., 6.]]) empty0 = np.empty([3, 0]) @@ -156,6 +159,7 @@ class MatrixSolveLsOpTest(test_lib.TestCase): tf_ans = linalg_ops.matrix_solve_ls(empty1, empty1, fast=fast).eval() self.assertEqual(tf_ans.shape, (2, 2)) + @test_util.run_deprecated_v1 def testBatchResultSize(self): # 3x3x3 matrices, 3x3x1 right-hand sides. matrix = np.array([1., 2., 3., 4., 5., 6., 7., 8., 9.] * 3).reshape(3, 3, 3) diff --git a/tensorflow/python/kernel_tests/matrix_solve_op_test.py b/tensorflow/python/kernel_tests/matrix_solve_op_test.py index 9e30ae16289..db7c4802f69 100644 --- a/tensorflow/python/kernel_tests/matrix_solve_op_test.py +++ b/tensorflow/python/kernel_tests/matrix_solve_op_test.py @@ -24,6 +24,7 @@ from tensorflow.python.client import session from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops +from tensorflow.python.framework import test_util from tensorflow.python.ops import array_ops from tensorflow.python.ops import control_flow_ops from tensorflow.python.ops import linalg_ops @@ -63,7 +64,7 @@ class MatrixSolveOpTest(test.TestCase): out = sess.run(tf_ans, {a_ph: a, b_ph: b}) else: tf_ans = linalg_ops.matrix_solve(a, b, adjoint=adjoint) - out = tf_ans.eval() + out = self.evaluate(tf_ans) self.assertEqual(tf_ans.get_shape(), out.shape) self.assertEqual(np_ans.shape, out.shape) self.assertAllClose(np_ans, out, atol=tol, rtol=tol) @@ -75,6 +76,7 @@ class MatrixSolveOpTest(test.TestCase): [m, n])) return matrix + @test_util.run_deprecated_v1 def testSolve(self): for n in 1, 2, 4, 9: matrix = self._generateMatrix(n, n) @@ -82,6 +84,7 @@ class MatrixSolveOpTest(test.TestCase): rhs = self._generateMatrix(n, nrhs) self._verifySolve(matrix, rhs) + @test_util.run_deprecated_v1 def testSolveBatch(self): for n in 2, 5: matrix = self._generateMatrix(n, n) @@ -90,6 +93,7 @@ class MatrixSolveOpTest(test.TestCase): for batch_dims in [[2], [2, 2], [7, 4]]: self._verifySolve(matrix, rhs, batch_dims=batch_dims) + @test_util.run_deprecated_v1 def testNonSquareMatrix(self): # When the solve of a non-square matrix is attempted we should return # an error @@ -98,6 +102,7 @@ class MatrixSolveOpTest(test.TestCase): matrix = constant_op.constant([[1., 2., 3.], [3., 4., 5.]]) linalg_ops.matrix_solve(matrix, matrix) + @test_util.run_deprecated_v1 def testWrongDimensions(self): # The matrix and right-hand sides should have the same number of rows. with self.session(use_gpu=True): @@ -115,6 +120,7 @@ class MatrixSolveOpTest(test.TestCase): [0., -1., 1.]]) linalg_ops.matrix_solve(matrix, matrix).eval() + @test_util.run_deprecated_v1 def testConcurrent(self): with self.session(use_gpu=True) as sess: all_ops = [] @@ -126,7 +132,7 @@ class MatrixSolveOpTest(test.TestCase): s1 = linalg_ops.matrix_solve(lhs1, rhs1, adjoint=adjoint_) s2 = linalg_ops.matrix_solve(lhs2, rhs2, adjoint=adjoint_) all_ops += [s1, s2] - val = sess.run(all_ops) + val = self.evaluate(all_ops) self.assertAllEqual(val[0], val[1]) self.assertAllEqual(val[2], val[3]) diff --git a/tensorflow/python/kernel_tests/matrix_square_root_op_test.py b/tensorflow/python/kernel_tests/matrix_square_root_op_test.py index 9212580313c..1e2109b8c41 100644 --- a/tensorflow/python/kernel_tests/matrix_square_root_op_test.py +++ b/tensorflow/python/kernel_tests/matrix_square_root_op_test.py @@ -21,6 +21,7 @@ from __future__ import print_function import numpy as np from tensorflow.python.framework import constant_op +from tensorflow.python.framework import test_util from tensorflow.python.ops import gen_linalg_ops from tensorflow.python.ops import math_ops from tensorflow.python.ops import random_ops @@ -31,7 +32,7 @@ class SquareRootOpTest(test.TestCase): def _verifySquareRoot(self, matrix, np_type): matrix = matrix.astype(np_type) - with self.test_session(use_gpu=True): + with test_util.use_gpu(): # Verify that matmul(sqrtm(A), sqrtm(A)) = A sqrt = gen_linalg_ops.matrix_square_root(matrix) square = math_ops.matmul(sqrt, sqrt) @@ -96,19 +97,20 @@ class SquareRootOpTest(test.TestCase): gen_linalg_ops.matrix_square_root(tensor) def testNotSquare(self): - with self.test_session(): - with self.assertRaises(ValueError): - tensor = constant_op.constant([[1., 0., -1.], [-1., 1., 0.]]) - gen_linalg_ops.matrix_square_root(tensor).eval() + with self.assertRaises(ValueError): + tensor = constant_op.constant([[1., 0., -1.], [-1., 1., 0.]]) + self.evaluate(gen_linalg_ops.matrix_square_root(tensor)) def testConcurrentExecutesWithoutError(self): - with self.test_session(use_gpu=True) as sess: + with test_util.use_gpu(): matrix1 = random_ops.random_normal([5, 5], seed=42) matrix2 = random_ops.random_normal([5, 5], seed=42) - sqrt1 = gen_linalg_ops.matrix_square_root(matrix1) - sqrt2 = gen_linalg_ops.matrix_square_root(matrix2) + square1 = math_ops.matmul(matrix1, matrix1) + square2 = math_ops.matmul(matrix2, matrix2) + sqrt1 = gen_linalg_ops.matrix_square_root(square1) + sqrt2 = gen_linalg_ops.matrix_square_root(square2) all_ops = [sqrt1, sqrt2] - sqrt = sess.run(all_ops) + sqrt = self.evaluate(all_ops) self.assertAllEqual(sqrt[0], sqrt[1]) diff --git a/tensorflow/python/kernel_tests/matrix_triangular_solve_op_test.py b/tensorflow/python/kernel_tests/matrix_triangular_solve_op_test.py index 445faca3ee2..dde83f12f3c 100644 --- a/tensorflow/python/kernel_tests/matrix_triangular_solve_op_test.py +++ b/tensorflow/python/kernel_tests/matrix_triangular_solve_op_test.py @@ -21,6 +21,7 @@ from __future__ import print_function import numpy as np from tensorflow.python.framework import constant_op +from tensorflow.python.framework import test_util from tensorflow.python.ops import array_ops from tensorflow.python.ops import linalg_ops from tensorflow.python.platform import test @@ -87,12 +88,13 @@ class MatrixTriangularSolveOpTest(test.TestCase): b_tf = constant_op.constant(b) tf_ans = linalg_ops.matrix_triangular_solve( a_tf, b_tf, lower=lower, adjoint=adjoint) - tf_val = tf_ans.eval() + tf_val = self.evaluate(tf_ans) np_ans = np.linalg.solve(a_np, b) self.assertEqual(np_ans.shape, tf_ans.get_shape()) self.assertEqual(np_ans.shape, tf_val.shape) self.assertAllClose(np_ans, tf_val) + @test_util.run_deprecated_v1 def testSolve(self): # 1x1 matrix, single rhs. matrix = np.array([[0.1]]) @@ -106,6 +108,7 @@ class MatrixTriangularSolveOpTest(test.TestCase): rhs1 = np.array([[1., 0., 1.], [0., 1., 1.]]) self._verifySolveAllWaysReal(matrix, rhs1) + @test_util.run_deprecated_v1 def testSolveComplex(self): # 1x1 matrix, single rhs. matrix = np.array([[0.1 + 1j * 0.1]]) @@ -122,6 +125,7 @@ class MatrixTriangularSolveOpTest(test.TestCase): rhs1 += 1j * rhs1 self._verifySolveAllWaysComplex(matrix, rhs1) + @test_util.run_deprecated_v1 def testSolveBatch(self): matrix = np.array([[1., 2.], [3., 4.]]) rhs = np.array([[1., 0., 1.], [0., 1., 1.]]) @@ -130,6 +134,7 @@ class MatrixTriangularSolveOpTest(test.TestCase): # Batch of 3x2x2x2 matrices, 3x2x2x3 right-hand sides. self._verifySolveAllWaysReal(matrix, rhs, batch_dims=[3, 2]) + @test_util.run_deprecated_v1 def testSolveBatchComplex(self): matrix = np.array([[1., 2.], [3., 4.]]).astype(np.complex64) matrix += 1j * matrix @@ -140,6 +145,7 @@ class MatrixTriangularSolveOpTest(test.TestCase): # Batch of 3x2x2x2 matrices, 3x2x2x3 right-hand sides. self._verifySolveAllWaysComplex(matrix, rhs, batch_dims=[3, 2]) + @test_util.run_deprecated_v1 def testNonSquareMatrix(self): # A non-square matrix should cause an error. matrix = np.array([[1., 2., 3.], [3., 4., 5.]]) @@ -149,6 +155,7 @@ class MatrixTriangularSolveOpTest(test.TestCase): with self.assertRaises(ValueError): self._verifySolve(matrix, matrix, batch_dims=[2, 3]) + @test_util.run_deprecated_v1 def testWrongDimensions(self): # The matrix should have the same number of rows as the # right-hand sides. diff --git a/tensorflow/python/kernel_tests/metrics_test.py b/tensorflow/python/kernel_tests/metrics_test.py index 5dcdb9e4205..64dd5914552 100644 --- a/tensorflow/python/kernel_tests/metrics_test.py +++ b/tensorflow/python/kernel_tests/metrics_test.py @@ -29,6 +29,7 @@ from tensorflow.python.framework import dtypes as dtypes_lib from tensorflow.python.framework import errors_impl from tensorflow.python.framework import ops from tensorflow.python.framework import sparse_tensor +from tensorflow.python.framework import test_util from tensorflow.python.ops import array_ops from tensorflow.python.ops import data_flow_ops from tensorflow.python.ops import math_ops @@ -175,22 +176,26 @@ class MeanTest(test.TestCase): def setUp(self): ops.reset_default_graph() + @test_util.run_deprecated_v1 def testVars(self): metrics.mean(array_ops.ones([4, 3])) _assert_metric_variables(self, ('mean/count:0', 'mean/total:0')) + @test_util.run_deprecated_v1 def testMetricsCollection(self): my_collection_name = '__metrics__' mean, _ = metrics.mean( array_ops.ones([4, 3]), metrics_collections=[my_collection_name]) self.assertListEqual(ops.get_collection(my_collection_name), [mean]) + @test_util.run_deprecated_v1 def testUpdatesCollection(self): my_collection_name = '__updates__' _, update_op = metrics.mean( array_ops.ones([4, 3]), updates_collections=[my_collection_name]) self.assertListEqual(ops.get_collection(my_collection_name), [update_op]) + @test_util.run_deprecated_v1 def testBasic(self): with self.cached_session() as sess: values_queue = data_flow_ops.FIFOQueue( @@ -203,11 +208,12 @@ class MeanTest(test.TestCase): mean, update_op = metrics.mean(values) - sess.run(variables.local_variables_initializer()) + self.evaluate(variables.local_variables_initializer()) for _ in range(4): - sess.run(update_op) - self.assertAlmostEqual(1.65, sess.run(mean), 5) + self.evaluate(update_op) + self.assertAlmostEqual(1.65, self.evaluate(mean), 5) + @test_util.run_deprecated_v1 def testUpdateOpsReturnsCurrentValue(self): with self.cached_session() as sess: values_queue = data_flow_ops.FIFOQueue( @@ -220,15 +226,16 @@ class MeanTest(test.TestCase): mean, update_op = metrics.mean(values) - sess.run(variables.local_variables_initializer()) + self.evaluate(variables.local_variables_initializer()) - self.assertAlmostEqual(0.5, sess.run(update_op), 5) - self.assertAlmostEqual(1.475, sess.run(update_op), 5) - self.assertAlmostEqual(12.4 / 6.0, sess.run(update_op), 5) - self.assertAlmostEqual(1.65, sess.run(update_op), 5) + self.assertAlmostEqual(0.5, self.evaluate(update_op), 5) + self.assertAlmostEqual(1.475, self.evaluate(update_op), 5) + self.assertAlmostEqual(12.4 / 6.0, self.evaluate(update_op), 5) + self.assertAlmostEqual(1.65, self.evaluate(update_op), 5) - self.assertAlmostEqual(1.65, sess.run(mean), 5) + self.assertAlmostEqual(1.65, self.evaluate(mean), 5) + @test_util.run_deprecated_v1 def testUnweighted(self): values = _test_values((3, 2, 4, 1)) mean_results = ( @@ -271,37 +278,44 @@ class MeanTest(test.TestCase): self.assertAlmostEqual(expected, update_op.eval(), places=5) self.assertAlmostEqual(expected, mean.eval(), places=5) + @test_util.run_deprecated_v1 def test1x1x1Weighted(self): self._test_3d_weighted( _test_values((3, 2, 4)), weights=np.asarray((5,)).reshape((1, 1, 1))) + @test_util.run_deprecated_v1 def test1x1xNWeighted(self): self._test_3d_weighted( _test_values((3, 2, 4)), weights=np.asarray((5, 7, 11, 3)).reshape((1, 1, 4))) + @test_util.run_deprecated_v1 def test1xNx1Weighted(self): self._test_3d_weighted( _test_values((3, 2, 4)), weights=np.asarray((5, 11)).reshape((1, 2, 1))) + @test_util.run_deprecated_v1 def test1xNxNWeighted(self): self._test_3d_weighted( _test_values((3, 2, 4)), weights=np.asarray((5, 7, 11, 3, 2, 13, 7, 5)).reshape((1, 2, 4))) + @test_util.run_deprecated_v1 def testNx1x1Weighted(self): self._test_3d_weighted( _test_values((3, 2, 4)), weights=np.asarray((5, 7, 11)).reshape((3, 1, 1))) + @test_util.run_deprecated_v1 def testNx1xNWeighted(self): self._test_3d_weighted( _test_values((3, 2, 4)), weights=np.asarray(( 5, 7, 11, 3, 2, 12, 7, 5, 2, 17, 11, 3)).reshape((3, 1, 4))) + @test_util.run_deprecated_v1 def testNxNxNWeighted(self): self._test_3d_weighted( _test_values((3, 2, 4)), @@ -309,6 +323,7 @@ class MeanTest(test.TestCase): 5, 7, 11, 3, 2, 12, 7, 5, 2, 17, 11, 3, 2, 17, 11, 3, 5, 7, 11, 3, 2, 12, 7, 5)).reshape((3, 2, 4))) + @test_util.run_deprecated_v1 def testInvalidWeights(self): values_placeholder = array_ops.placeholder(dtype=dtypes_lib.float32) values = _test_values((3, 2, 4, 1)) @@ -341,23 +356,27 @@ class MeanTensorTest(test.TestCase): def setUp(self): ops.reset_default_graph() + @test_util.run_deprecated_v1 def testVars(self): metrics.mean_tensor(array_ops.ones([4, 3])) _assert_metric_variables(self, ('mean/total_tensor:0', 'mean/count_tensor:0')) + @test_util.run_deprecated_v1 def testMetricsCollection(self): my_collection_name = '__metrics__' mean, _ = metrics.mean_tensor( array_ops.ones([4, 3]), metrics_collections=[my_collection_name]) self.assertListEqual(ops.get_collection(my_collection_name), [mean]) + @test_util.run_deprecated_v1 def testUpdatesCollection(self): my_collection_name = '__updates__' _, update_op = metrics.mean_tensor( array_ops.ones([4, 3]), updates_collections=[my_collection_name]) self.assertListEqual(ops.get_collection(my_collection_name), [update_op]) + @test_util.run_deprecated_v1 def testBasic(self): with self.cached_session() as sess: values_queue = data_flow_ops.FIFOQueue( @@ -370,11 +389,12 @@ class MeanTensorTest(test.TestCase): mean, update_op = metrics.mean_tensor(values) - sess.run(variables.local_variables_initializer()) + self.evaluate(variables.local_variables_initializer()) for _ in range(4): - sess.run(update_op) - self.assertAllClose([[-0.9 / 4., 3.525]], sess.run(mean)) + self.evaluate(update_op) + self.assertAllClose([[-0.9 / 4., 3.525]], self.evaluate(mean)) + @test_util.run_deprecated_v1 def testMultiDimensional(self): with self.cached_session() as sess: values_queue = data_flow_ops.FIFOQueue( @@ -391,11 +411,13 @@ class MeanTensorTest(test.TestCase): mean, update_op = metrics.mean_tensor(values) - sess.run(variables.local_variables_initializer()) + self.evaluate(variables.local_variables_initializer()) for _ in range(2): - sess.run(update_op) - self.assertAllClose([[[1, 2], [1, 2]], [[2, 3], [5, 6]]], sess.run(mean)) + self.evaluate(update_op) + self.assertAllClose([[[1, 2], [1, 2]], [[2, 3], [5, 6]]], + self.evaluate(mean)) + @test_util.run_deprecated_v1 def testUpdateOpsReturnsCurrentValue(self): with self.cached_session() as sess: values_queue = data_flow_ops.FIFOQueue( @@ -408,15 +430,16 @@ class MeanTensorTest(test.TestCase): mean, update_op = metrics.mean_tensor(values) - sess.run(variables.local_variables_initializer()) + self.evaluate(variables.local_variables_initializer()) - self.assertAllClose([[0, 1]], sess.run(update_op), 5) - self.assertAllClose([[-2.1, 5.05]], sess.run(update_op), 5) - self.assertAllClose([[2.3 / 3., 10.1 / 3.]], sess.run(update_op), 5) - self.assertAllClose([[-0.9 / 4., 3.525]], sess.run(update_op), 5) + self.assertAllClose([[0, 1]], self.evaluate(update_op), 5) + self.assertAllClose([[-2.1, 5.05]], self.evaluate(update_op), 5) + self.assertAllClose([[2.3 / 3., 10.1 / 3.]], self.evaluate(update_op), 5) + self.assertAllClose([[-0.9 / 4., 3.525]], self.evaluate(update_op), 5) - self.assertAllClose([[-0.9 / 4., 3.525]], sess.run(mean), 5) + self.assertAllClose([[-0.9 / 4., 3.525]], self.evaluate(mean), 5) + @test_util.run_deprecated_v1 def testBinaryWeighted1d(self): with self.cached_session() as sess: # Create the queue that populates the values. @@ -439,11 +462,12 @@ class MeanTensorTest(test.TestCase): mean, update_op = metrics.mean_tensor(values, weights) - sess.run(variables.local_variables_initializer()) + self.evaluate(variables.local_variables_initializer()) for _ in range(4): - sess.run(update_op) - self.assertAllClose([[3.25, 0.5]], sess.run(mean), 5) + self.evaluate(update_op) + self.assertAllClose([[3.25, 0.5]], self.evaluate(mean), 5) + @test_util.run_deprecated_v1 def testWeighted1d(self): with self.cached_session() as sess: # Create the queue that populates the values. @@ -466,11 +490,12 @@ class MeanTensorTest(test.TestCase): mean, update_op = metrics.mean_tensor(values, weights) - sess.run(variables.local_variables_initializer()) + self.evaluate(variables.local_variables_initializer()) for _ in range(4): - sess.run(update_op) - self.assertAllClose([[0.8, 3.52]], sess.run(mean), 5) + self.evaluate(update_op) + self.assertAllClose([[0.8, 3.52]], self.evaluate(mean), 5) + @test_util.run_deprecated_v1 def testWeighted2d_1(self): with self.cached_session() as sess: # Create the queue that populates the values. @@ -493,11 +518,12 @@ class MeanTensorTest(test.TestCase): mean, update_op = metrics.mean_tensor(values, weights) - sess.run(variables.local_variables_initializer()) + self.evaluate(variables.local_variables_initializer()) for _ in range(4): - sess.run(update_op) - self.assertAllClose([[-2.1, 0.5]], sess.run(mean), 5) + self.evaluate(update_op) + self.assertAllClose([[-2.1, 0.5]], self.evaluate(mean), 5) + @test_util.run_deprecated_v1 def testWeighted2d_2(self): with self.cached_session() as sess: # Create the queue that populates the values. @@ -520,10 +546,10 @@ class MeanTensorTest(test.TestCase): mean, update_op = metrics.mean_tensor(values, weights) - sess.run(variables.local_variables_initializer()) + self.evaluate(variables.local_variables_initializer()) for _ in range(4): - sess.run(update_op) - self.assertAllClose([[0, 0.5]], sess.run(mean), 5) + self.evaluate(update_op) + self.assertAllClose([[0, 0.5]], self.evaluate(mean), 5) class AccuracyTest(test.TestCase): @@ -531,6 +557,7 @@ class AccuracyTest(test.TestCase): def setUp(self): ops.reset_default_graph() + @test_util.run_deprecated_v1 def testVars(self): metrics.accuracy( predictions=array_ops.ones((10, 1)), @@ -539,6 +566,7 @@ class AccuracyTest(test.TestCase): _assert_metric_variables(self, ('my_accuracy/count:0', 'my_accuracy/total:0')) + @test_util.run_deprecated_v1 def testMetricsCollection(self): my_collection_name = '__metrics__' mean, _ = metrics.accuracy( @@ -547,6 +575,7 @@ class AccuracyTest(test.TestCase): metrics_collections=[my_collection_name]) self.assertListEqual(ops.get_collection(my_collection_name), [mean]) + @test_util.run_deprecated_v1 def testUpdatesCollection(self): my_collection_name = '__updates__' _, update_op = metrics.accuracy( @@ -555,12 +584,14 @@ class AccuracyTest(test.TestCase): updates_collections=[my_collection_name]) self.assertListEqual(ops.get_collection(my_collection_name), [update_op]) + @test_util.run_deprecated_v1 def testPredictionsAndLabelsOfDifferentSizeRaisesValueError(self): predictions = array_ops.ones((10, 3)) labels = array_ops.ones((10, 4)) with self.assertRaises(ValueError): metrics.accuracy(labels, predictions) + @test_util.run_deprecated_v1 def testPredictionsAndWeightsOfDifferentSizeRaisesValueError(self): predictions = array_ops.ones((10, 3)) labels = array_ops.ones((10, 3)) @@ -568,6 +599,7 @@ class AccuracyTest(test.TestCase): with self.assertRaises(ValueError): metrics.accuracy(labels, predictions, weights) + @test_util.run_deprecated_v1 def testValueTensorIsIdempotent(self): predictions = random_ops.random_uniform( (10, 3), maxval=3, dtype=dtypes_lib.int64, seed=1) @@ -576,17 +608,18 @@ class AccuracyTest(test.TestCase): accuracy, update_op = metrics.accuracy(labels, predictions) with self.cached_session() as sess: - sess.run(variables.local_variables_initializer()) + self.evaluate(variables.local_variables_initializer()) # Run several updates. for _ in range(10): - sess.run(update_op) + self.evaluate(update_op) # Then verify idempotency. initial_accuracy = accuracy.eval() for _ in range(10): self.assertEqual(initial_accuracy, accuracy.eval()) + @test_util.run_deprecated_v1 def testMultipleUpdates(self): with self.cached_session() as sess: # Create the queue that populates the predictions. @@ -609,32 +642,35 @@ class AccuracyTest(test.TestCase): accuracy, update_op = metrics.accuracy(labels, predictions) - sess.run(variables.local_variables_initializer()) + self.evaluate(variables.local_variables_initializer()) for _ in xrange(3): - sess.run(update_op) - self.assertEqual(0.5, sess.run(update_op)) + self.evaluate(update_op) + self.assertEqual(0.5, self.evaluate(update_op)) self.assertEqual(0.5, accuracy.eval()) + @test_util.run_deprecated_v1 def testEffectivelyEquivalentSizes(self): predictions = array_ops.ones((40, 1)) labels = array_ops.ones((40,)) with self.cached_session() as sess: accuracy, update_op = metrics.accuracy(labels, predictions) - sess.run(variables.local_variables_initializer()) + self.evaluate(variables.local_variables_initializer()) self.assertEqual(1.0, update_op.eval()) self.assertEqual(1.0, accuracy.eval()) + @test_util.run_deprecated_v1 def testEffectivelyEquivalentSizesWithScalarWeight(self): predictions = array_ops.ones((40, 1)) labels = array_ops.ones((40,)) with self.cached_session() as sess: accuracy, update_op = metrics.accuracy(labels, predictions, weights=2.0) - sess.run(variables.local_variables_initializer()) + self.evaluate(variables.local_variables_initializer()) self.assertEqual(1.0, update_op.eval()) self.assertEqual(1.0, accuracy.eval()) + @test_util.run_deprecated_v1 def testEffectivelyEquivalentSizesWithStaticShapedWeight(self): predictions = ops.convert_to_tensor([1, 1, 1]) # shape 3, labels = array_ops.expand_dims(ops.convert_to_tensor([1, 0, 0]), @@ -645,13 +681,14 @@ class AccuracyTest(test.TestCase): with self.cached_session() as sess: accuracy, update_op = metrics.accuracy(labels, predictions, weights) - sess.run(variables.local_variables_initializer()) + self.evaluate(variables.local_variables_initializer()) # if streaming_accuracy does not flatten the weight, accuracy would be # 0.33333334 due to an intended broadcast of weight. Due to flattening, # it will be higher than .95 self.assertGreater(update_op.eval(), .95) self.assertGreater(accuracy.eval(), .95) + @test_util.run_deprecated_v1 def testEffectivelyEquivalentSizesWithDynamicallyShapedWeight(self): predictions = ops.convert_to_tensor([1, 1, 1]) # shape 3, labels = array_ops.expand_dims(ops.convert_to_tensor([1, 0, 0]), @@ -666,13 +703,14 @@ class AccuracyTest(test.TestCase): accuracy, update_op = metrics.accuracy(labels, predictions, weights_placeholder) - sess.run(variables.local_variables_initializer()) + self.evaluate(variables.local_variables_initializer()) # if streaming_accuracy does not flatten the weight, accuracy would be # 0.33333334 due to an intended broadcast of weight. Due to flattening, # it will be higher than .95 self.assertGreater(update_op.eval(feed_dict=feed_dict), .95) self.assertGreater(accuracy.eval(feed_dict=feed_dict), .95) + @test_util.run_deprecated_v1 def testMultipleUpdatesWithWeightedValues(self): with self.cached_session() as sess: # Create the queue that populates the predictions. @@ -704,10 +742,10 @@ class AccuracyTest(test.TestCase): accuracy, update_op = metrics.accuracy(labels, predictions, weights) - sess.run(variables.local_variables_initializer()) + self.evaluate(variables.local_variables_initializer()) for _ in xrange(3): - sess.run(update_op) - self.assertEqual(1.0, sess.run(update_op)) + self.evaluate(update_op) + self.assertEqual(1.0, self.evaluate(update_op)) self.assertEqual(1.0, accuracy.eval()) @@ -717,12 +755,14 @@ class PrecisionTest(test.TestCase): np.random.seed(1) ops.reset_default_graph() + @test_util.run_deprecated_v1 def testVars(self): metrics.precision( predictions=array_ops.ones((10, 1)), labels=array_ops.ones((10, 1))) _assert_metric_variables(self, ('precision/false_positives/count:0', 'precision/true_positives/count:0')) + @test_util.run_deprecated_v1 def testMetricsCollection(self): my_collection_name = '__metrics__' mean, _ = metrics.precision( @@ -731,6 +771,7 @@ class PrecisionTest(test.TestCase): metrics_collections=[my_collection_name]) self.assertListEqual(ops.get_collection(my_collection_name), [mean]) + @test_util.run_deprecated_v1 def testUpdatesCollection(self): my_collection_name = '__updates__' _, update_op = metrics.precision( @@ -739,6 +780,7 @@ class PrecisionTest(test.TestCase): updates_collections=[my_collection_name]) self.assertListEqual(ops.get_collection(my_collection_name), [update_op]) + @test_util.run_deprecated_v1 def testValueTensorIsIdempotent(self): predictions = random_ops.random_uniform( (10, 3), maxval=1, dtype=dtypes_lib.int64, seed=1) @@ -747,17 +789,18 @@ class PrecisionTest(test.TestCase): precision, update_op = metrics.precision(labels, predictions) with self.cached_session() as sess: - sess.run(variables.local_variables_initializer()) + self.evaluate(variables.local_variables_initializer()) # Run several updates. for _ in range(10): - sess.run(update_op) + self.evaluate(update_op) # Then verify idempotency. initial_precision = precision.eval() for _ in range(10): self.assertEqual(initial_precision, precision.eval()) + @test_util.run_deprecated_v1 def testAllCorrect(self): inputs = np.random.randint(0, 2, size=(100, 1)) @@ -766,10 +809,11 @@ class PrecisionTest(test.TestCase): precision, update_op = metrics.precision(labels, predictions) with self.cached_session() as sess: - sess.run(variables.local_variables_initializer()) - self.assertAlmostEqual(1, sess.run(update_op)) + self.evaluate(variables.local_variables_initializer()) + self.assertAlmostEqual(1, self.evaluate(update_op)) self.assertAlmostEqual(1, precision.eval()) + @test_util.run_deprecated_v1 def testSomeCorrect_multipleInputDtypes(self): for dtype in (dtypes_lib.bool, dtypes_lib.int32, dtypes_lib.float32): predictions = math_ops.cast( @@ -779,10 +823,11 @@ class PrecisionTest(test.TestCase): precision, update_op = metrics.precision(labels, predictions) with self.cached_session() as sess: - sess.run(variables.local_variables_initializer()) + self.evaluate(variables.local_variables_initializer()) self.assertAlmostEqual(0.5, update_op.eval()) self.assertAlmostEqual(0.5, precision.eval()) + @test_util.run_deprecated_v1 def testWeighted1d(self): predictions = constant_op.constant([[1, 0, 1, 0], [1, 0, 1, 0]]) labels = constant_op.constant([[0, 1, 1, 0], [1, 0, 0, 1]]) @@ -797,6 +842,7 @@ class PrecisionTest(test.TestCase): self.assertAlmostEqual(expected_precision, update_op.eval()) self.assertAlmostEqual(expected_precision, precision.eval()) + @test_util.run_deprecated_v1 def testWeightedScalar_placeholders(self): predictions = array_ops.placeholder(dtype=dtypes_lib.float32) labels = array_ops.placeholder(dtype=dtypes_lib.float32) @@ -816,6 +862,7 @@ class PrecisionTest(test.TestCase): self.assertAlmostEqual( expected_precision, precision.eval(feed_dict=feed_dict)) + @test_util.run_deprecated_v1 def testWeighted1d_placeholders(self): predictions = array_ops.placeholder(dtype=dtypes_lib.float32) labels = array_ops.placeholder(dtype=dtypes_lib.float32) @@ -836,6 +883,7 @@ class PrecisionTest(test.TestCase): self.assertAlmostEqual( expected_precision, precision.eval(feed_dict=feed_dict)) + @test_util.run_deprecated_v1 def testWeighted2d(self): predictions = constant_op.constant([[1, 0, 1, 0], [1, 0, 1, 0]]) labels = constant_op.constant([[0, 1, 1, 0], [1, 0, 0, 1]]) @@ -852,6 +900,7 @@ class PrecisionTest(test.TestCase): self.assertAlmostEqual(expected_precision, update_op.eval()) self.assertAlmostEqual(expected_precision, precision.eval()) + @test_util.run_deprecated_v1 def testWeighted2d_placeholders(self): predictions = array_ops.placeholder(dtype=dtypes_lib.float32) labels = array_ops.placeholder(dtype=dtypes_lib.float32) @@ -874,6 +923,7 @@ class PrecisionTest(test.TestCase): self.assertAlmostEqual( expected_precision, precision.eval(feed_dict=feed_dict)) + @test_util.run_deprecated_v1 def testAllIncorrect(self): inputs = np.random.randint(0, 2, size=(100, 1)) @@ -882,18 +932,19 @@ class PrecisionTest(test.TestCase): precision, update_op = metrics.precision(labels, predictions) with self.cached_session() as sess: - sess.run(variables.local_variables_initializer()) - sess.run(update_op) + self.evaluate(variables.local_variables_initializer()) + self.evaluate(update_op) self.assertAlmostEqual(0, precision.eval()) + @test_util.run_deprecated_v1 def testZeroTrueAndFalsePositivesGivesZeroPrecision(self): predictions = constant_op.constant([0, 0, 0, 0]) labels = constant_op.constant([0, 0, 0, 0]) precision, update_op = metrics.precision(labels, predictions) with self.cached_session() as sess: - sess.run(variables.local_variables_initializer()) - sess.run(update_op) + self.evaluate(variables.local_variables_initializer()) + self.evaluate(update_op) self.assertEqual(0.0, precision.eval()) @@ -903,6 +954,7 @@ class RecallTest(test.TestCase): np.random.seed(1) ops.reset_default_graph() + @test_util.run_deprecated_v1 def testVars(self): metrics.recall( predictions=array_ops.ones((10, 1)), labels=array_ops.ones((10, 1))) @@ -910,6 +962,7 @@ class RecallTest(test.TestCase): self, ('recall/false_negatives/count:0', 'recall/true_positives/count:0')) + @test_util.run_deprecated_v1 def testMetricsCollection(self): my_collection_name = '__metrics__' mean, _ = metrics.recall( @@ -918,6 +971,7 @@ class RecallTest(test.TestCase): metrics_collections=[my_collection_name]) self.assertListEqual(ops.get_collection(my_collection_name), [mean]) + @test_util.run_deprecated_v1 def testUpdatesCollection(self): my_collection_name = '__updates__' _, update_op = metrics.recall( @@ -926,6 +980,7 @@ class RecallTest(test.TestCase): updates_collections=[my_collection_name]) self.assertListEqual(ops.get_collection(my_collection_name), [update_op]) + @test_util.run_deprecated_v1 def testValueTensorIsIdempotent(self): predictions = random_ops.random_uniform( (10, 3), maxval=1, dtype=dtypes_lib.int64, seed=1) @@ -934,17 +989,18 @@ class RecallTest(test.TestCase): recall, update_op = metrics.recall(labels, predictions) with self.cached_session() as sess: - sess.run(variables.local_variables_initializer()) + self.evaluate(variables.local_variables_initializer()) # Run several updates. for _ in range(10): - sess.run(update_op) + self.evaluate(update_op) # Then verify idempotency. initial_recall = recall.eval() for _ in range(10): self.assertEqual(initial_recall, recall.eval()) + @test_util.run_deprecated_v1 def testAllCorrect(self): np_inputs = np.random.randint(0, 2, size=(100, 1)) @@ -953,10 +1009,11 @@ class RecallTest(test.TestCase): recall, update_op = metrics.recall(labels, predictions) with self.cached_session() as sess: - sess.run(variables.local_variables_initializer()) - sess.run(update_op) + self.evaluate(variables.local_variables_initializer()) + self.evaluate(update_op) self.assertEqual(1, recall.eval()) + @test_util.run_deprecated_v1 def testSomeCorrect_multipleInputDtypes(self): for dtype in (dtypes_lib.bool, dtypes_lib.int32, dtypes_lib.float32): predictions = math_ops.cast( @@ -966,10 +1023,11 @@ class RecallTest(test.TestCase): recall, update_op = metrics.recall(labels, predictions) with self.cached_session() as sess: - sess.run(variables.local_variables_initializer()) + self.evaluate(variables.local_variables_initializer()) self.assertAlmostEqual(0.5, update_op.eval()) self.assertAlmostEqual(0.5, recall.eval()) + @test_util.run_deprecated_v1 def testWeighted1d(self): predictions = constant_op.constant([[1, 0, 1, 0], [0, 1, 0, 1]]) labels = constant_op.constant([[0, 1, 1, 0], [1, 0, 0, 1]]) @@ -977,13 +1035,14 @@ class RecallTest(test.TestCase): recall, update_op = metrics.recall(labels, predictions, weights=weights) with self.cached_session() as sess: - sess.run(variables.local_variables_initializer()) + self.evaluate(variables.local_variables_initializer()) weighted_tp = 2.0 + 5.0 weighted_t = (2.0 + 2.0) + (5.0 + 5.0) expected_precision = weighted_tp / weighted_t self.assertAlmostEqual(expected_precision, update_op.eval()) self.assertAlmostEqual(expected_precision, recall.eval()) + @test_util.run_deprecated_v1 def testWeighted2d(self): predictions = constant_op.constant([[1, 0, 1, 0], [0, 1, 0, 1]]) labels = constant_op.constant([[0, 1, 1, 0], [1, 0, 0, 1]]) @@ -991,13 +1050,14 @@ class RecallTest(test.TestCase): recall, update_op = metrics.recall(labels, predictions, weights=weights) with self.cached_session() as sess: - sess.run(variables.local_variables_initializer()) + self.evaluate(variables.local_variables_initializer()) weighted_tp = 3.0 + 1.0 weighted_t = (2.0 + 3.0) + (4.0 + 1.0) expected_precision = weighted_tp / weighted_t self.assertAlmostEqual(expected_precision, update_op.eval()) self.assertAlmostEqual(expected_precision, recall.eval()) + @test_util.run_deprecated_v1 def testAllIncorrect(self): np_inputs = np.random.randint(0, 2, size=(100, 1)) @@ -1006,18 +1066,19 @@ class RecallTest(test.TestCase): recall, update_op = metrics.recall(labels, predictions) with self.cached_session() as sess: - sess.run(variables.local_variables_initializer()) - sess.run(update_op) + self.evaluate(variables.local_variables_initializer()) + self.evaluate(update_op) self.assertEqual(0, recall.eval()) + @test_util.run_deprecated_v1 def testZeroTruePositivesAndFalseNegativesGivesZeroRecall(self): predictions = array_ops.zeros((1, 4)) labels = array_ops.zeros((1, 4)) recall, update_op = metrics.recall(labels, predictions) with self.cached_session() as sess: - sess.run(variables.local_variables_initializer()) - sess.run(update_op) + self.evaluate(variables.local_variables_initializer()) + self.evaluate(update_op) self.assertEqual(0, recall.eval()) @@ -1027,6 +1088,7 @@ class AUCTest(test.TestCase): np.random.seed(1) ops.reset_default_graph() + @test_util.run_deprecated_v1 def testVars(self): metrics.auc(predictions=array_ops.ones((10, 1)), labels=array_ops.ones((10, 1))) @@ -1034,6 +1096,7 @@ class AUCTest(test.TestCase): ('auc/true_positives:0', 'auc/false_negatives:0', 'auc/false_positives:0', 'auc/true_negatives:0')) + @test_util.run_deprecated_v1 def testMetricsCollection(self): my_collection_name = '__metrics__' mean, _ = metrics.auc(predictions=array_ops.ones((10, 1)), @@ -1041,6 +1104,7 @@ class AUCTest(test.TestCase): metrics_collections=[my_collection_name]) self.assertListEqual(ops.get_collection(my_collection_name), [mean]) + @test_util.run_deprecated_v1 def testUpdatesCollection(self): my_collection_name = '__updates__' _, update_op = metrics.auc(predictions=array_ops.ones((10, 1)), @@ -1048,6 +1112,7 @@ class AUCTest(test.TestCase): updates_collections=[my_collection_name]) self.assertListEqual(ops.get_collection(my_collection_name), [update_op]) + @test_util.run_deprecated_v1 def testValueTensorIsIdempotent(self): predictions = random_ops.random_uniform( (10, 3), maxval=1, dtype=dtypes_lib.float32, seed=1) @@ -1056,17 +1121,18 @@ class AUCTest(test.TestCase): auc, update_op = metrics.auc(labels, predictions) with self.cached_session() as sess: - sess.run(variables.local_variables_initializer()) + self.evaluate(variables.local_variables_initializer()) # Run several updates. for _ in range(10): - sess.run(update_op) + self.evaluate(update_op) # Then verify idempotency. initial_auc = auc.eval() for _ in range(10): self.assertAlmostEqual(initial_auc, auc.eval(), 5) + @test_util.run_deprecated_v1 def testAllCorrect(self): self.allCorrectAsExpected('ROC') @@ -1078,11 +1144,12 @@ class AUCTest(test.TestCase): labels = constant_op.constant(inputs) auc, update_op = metrics.auc(labels, predictions, curve=curve) - sess.run(variables.local_variables_initializer()) - self.assertEqual(1, sess.run(update_op)) + self.evaluate(variables.local_variables_initializer()) + self.assertEqual(1, self.evaluate(update_op)) self.assertEqual(1, auc.eval()) + @test_util.run_deprecated_v1 def testSomeCorrect_multipleLabelDtypes(self): with self.cached_session() as sess: for label_dtype in ( @@ -1093,11 +1160,12 @@ class AUCTest(test.TestCase): constant_op.constant([0, 1, 1, 0], shape=(1, 4)), dtype=label_dtype) auc, update_op = metrics.auc(labels, predictions) - sess.run(variables.local_variables_initializer()) - self.assertAlmostEqual(0.5, sess.run(update_op)) + self.evaluate(variables.local_variables_initializer()) + self.assertAlmostEqual(0.5, self.evaluate(update_op)) self.assertAlmostEqual(0.5, auc.eval()) + @test_util.run_deprecated_v1 def testWeighted1d(self): with self.cached_session() as sess: predictions = constant_op.constant( @@ -1106,11 +1174,12 @@ class AUCTest(test.TestCase): weights = constant_op.constant([2], shape=(1, 1)) auc, update_op = metrics.auc(labels, predictions, weights=weights) - sess.run(variables.local_variables_initializer()) - self.assertAlmostEqual(0.5, sess.run(update_op), 5) + self.evaluate(variables.local_variables_initializer()) + self.assertAlmostEqual(0.5, self.evaluate(update_op), 5) self.assertAlmostEqual(0.5, auc.eval(), 5) + @test_util.run_deprecated_v1 def testWeighted2d(self): with self.cached_session() as sess: predictions = constant_op.constant( @@ -1119,13 +1188,14 @@ class AUCTest(test.TestCase): weights = constant_op.constant([1, 2, 3, 4], shape=(1, 4)) auc, update_op = metrics.auc(labels, predictions, weights=weights) - sess.run(variables.local_variables_initializer()) - self.assertAlmostEqual(0.7, sess.run(update_op), 5) + self.evaluate(variables.local_variables_initializer()) + self.assertAlmostEqual(0.7, self.evaluate(update_op), 5) self.assertAlmostEqual(0.7, auc.eval(), 5) # Regarding the AUC-PR tests: note that the preferred method when # calculating AUC-PR is summation_method='careful_interpolation'. + @test_util.run_deprecated_v1 def testCorrectAUCPRSpecialCase(self): with self.cached_session() as sess: predictions = constant_op.constant( @@ -1134,12 +1204,13 @@ class AUCTest(test.TestCase): auc, update_op = metrics.auc(labels, predictions, curve='PR', summation_method='careful_interpolation') - sess.run(variables.local_variables_initializer()) + self.evaluate(variables.local_variables_initializer()) # expected ~= 0.79726744594 expected = 1 - math.log(1.5) / 2 - self.assertAlmostEqual(expected, sess.run(update_op), delta=1e-3) + self.assertAlmostEqual(expected, self.evaluate(update_op), delta=1e-3) self.assertAlmostEqual(expected, auc.eval(), delta=1e-3) + @test_util.run_deprecated_v1 def testCorrectAnotherAUCPRSpecialCase(self): with self.cached_session() as sess: predictions = constant_op.constant( @@ -1150,12 +1221,13 @@ class AUCTest(test.TestCase): auc, update_op = metrics.auc(labels, predictions, curve='PR', summation_method='careful_interpolation') - sess.run(variables.local_variables_initializer()) + self.evaluate(variables.local_variables_initializer()) # expected ~= 0.61350593198 expected = (2.5 - 2 * math.log(4./3) - 0.25 * math.log(7./5)) / 3 - self.assertAlmostEqual(expected, sess.run(update_op), delta=1e-3) + self.assertAlmostEqual(expected, self.evaluate(update_op), delta=1e-3) self.assertAlmostEqual(expected, auc.eval(), delta=1e-3) + @test_util.run_deprecated_v1 def testThirdCorrectAUCPRSpecialCase(self): with self.cached_session() as sess: predictions = constant_op.constant( @@ -1166,12 +1238,13 @@ class AUCTest(test.TestCase): auc, update_op = metrics.auc(labels, predictions, curve='PR', summation_method='careful_interpolation') - sess.run(variables.local_variables_initializer()) + self.evaluate(variables.local_variables_initializer()) # expected ~= 0.90410597584 expected = 1 - math.log(4./3) / 3 - self.assertAlmostEqual(expected, sess.run(update_op), delta=1e-3) + self.assertAlmostEqual(expected, self.evaluate(update_op), delta=1e-3) self.assertAlmostEqual(expected, auc.eval(), delta=1e-3) + @test_util.run_deprecated_v1 def testIncorrectAUCPRSpecialCase(self): with self.cached_session() as sess: predictions = constant_op.constant( @@ -1180,11 +1253,12 @@ class AUCTest(test.TestCase): auc, update_op = metrics.auc(labels, predictions, curve='PR', summation_method='trapezoidal') - sess.run(variables.local_variables_initializer()) - self.assertAlmostEqual(0.79166, sess.run(update_op), delta=1e-3) + self.evaluate(variables.local_variables_initializer()) + self.assertAlmostEqual(0.79166, self.evaluate(update_op), delta=1e-3) self.assertAlmostEqual(0.79166, auc.eval(), delta=1e-3) + @test_util.run_deprecated_v1 def testAnotherIncorrectAUCPRSpecialCase(self): with self.cached_session() as sess: predictions = constant_op.constant( @@ -1195,11 +1269,12 @@ class AUCTest(test.TestCase): auc, update_op = metrics.auc(labels, predictions, curve='PR', summation_method='trapezoidal') - sess.run(variables.local_variables_initializer()) - self.assertAlmostEqual(0.610317, sess.run(update_op), delta=1e-3) + self.evaluate(variables.local_variables_initializer()) + self.assertAlmostEqual(0.610317, self.evaluate(update_op), delta=1e-3) self.assertAlmostEqual(0.610317, auc.eval(), delta=1e-3) + @test_util.run_deprecated_v1 def testThirdIncorrectAUCPRSpecialCase(self): with self.cached_session() as sess: predictions = constant_op.constant( @@ -1210,11 +1285,12 @@ class AUCTest(test.TestCase): auc, update_op = metrics.auc(labels, predictions, curve='PR', summation_method='trapezoidal') - sess.run(variables.local_variables_initializer()) - self.assertAlmostEqual(0.90277, sess.run(update_op), delta=1e-3) + self.evaluate(variables.local_variables_initializer()) + self.assertAlmostEqual(0.90277, self.evaluate(update_op), delta=1e-3) self.assertAlmostEqual(0.90277, auc.eval(), delta=1e-3) + @test_util.run_deprecated_v1 def testAllIncorrect(self): inputs = np.random.randint(0, 2, size=(100, 1)) @@ -1223,30 +1299,32 @@ class AUCTest(test.TestCase): labels = constant_op.constant(1 - inputs, dtype=dtypes_lib.float32) auc, update_op = metrics.auc(labels, predictions) - sess.run(variables.local_variables_initializer()) - self.assertAlmostEqual(0, sess.run(update_op)) + self.evaluate(variables.local_variables_initializer()) + self.assertAlmostEqual(0, self.evaluate(update_op)) self.assertAlmostEqual(0, auc.eval()) + @test_util.run_deprecated_v1 def testZeroTruePositivesAndFalseNegativesGivesOneAUC(self): with self.cached_session() as sess: predictions = array_ops.zeros([4], dtype=dtypes_lib.float32) labels = array_ops.zeros([4]) auc, update_op = metrics.auc(labels, predictions) - sess.run(variables.local_variables_initializer()) - self.assertAlmostEqual(1, sess.run(update_op), 6) + self.evaluate(variables.local_variables_initializer()) + self.assertAlmostEqual(1, self.evaluate(update_op), 6) self.assertAlmostEqual(1, auc.eval(), 6) + @test_util.run_deprecated_v1 def testRecallOneAndPrecisionOneGivesOnePRAUC(self): with self.cached_session() as sess: predictions = array_ops.ones([4], dtype=dtypes_lib.float32) labels = array_ops.ones([4]) auc, update_op = metrics.auc(labels, predictions, curve='PR') - sess.run(variables.local_variables_initializer()) - self.assertAlmostEqual(1, sess.run(update_op), 6) + self.evaluate(variables.local_variables_initializer()) + self.assertAlmostEqual(1, self.evaluate(update_op), 6) self.assertAlmostEqual(1, auc.eval(), 6) @@ -1277,6 +1355,7 @@ class AUCTest(test.TestCase): tp = np.cumsum(sorted_weights * is_positive) / num_positives return np.sum((sorted_weights * tp)[~is_positive]) / num_negatives + @test_util.run_deprecated_v1 def testWithMultipleUpdates(self): num_samples = 1000 batch_size = 10 @@ -1317,9 +1396,9 @@ class AUCTest(test.TestCase): num_thresholds=500, weights=tf_weights) - sess.run(variables.local_variables_initializer()) + self.evaluate(variables.local_variables_initializer()) for i in range(num_batches): - sess.run(update_op) + self.evaluate(update_op) # Since this is only approximate, we can't expect a 6 digits match. # Although with higher number of samples/thresholds we should see the @@ -1333,6 +1412,7 @@ class SpecificityAtSensitivityTest(test.TestCase): np.random.seed(1) ops.reset_default_graph() + @test_util.run_deprecated_v1 def testVars(self): metrics.specificity_at_sensitivity( predictions=array_ops.ones((10, 1)), @@ -1344,6 +1424,7 @@ class SpecificityAtSensitivityTest(test.TestCase): 'specificity_at_sensitivity/false_positives:0', 'specificity_at_sensitivity/true_negatives:0')) + @test_util.run_deprecated_v1 def testMetricsCollection(self): my_collection_name = '__metrics__' mean, _ = metrics.specificity_at_sensitivity( @@ -1353,6 +1434,7 @@ class SpecificityAtSensitivityTest(test.TestCase): metrics_collections=[my_collection_name]) self.assertListEqual(ops.get_collection(my_collection_name), [mean]) + @test_util.run_deprecated_v1 def testUpdatesCollection(self): my_collection_name = '__updates__' _, update_op = metrics.specificity_at_sensitivity( @@ -1362,6 +1444,7 @@ class SpecificityAtSensitivityTest(test.TestCase): updates_collections=[my_collection_name]) self.assertListEqual(ops.get_collection(my_collection_name), [update_op]) + @test_util.run_deprecated_v1 def testValueTensorIsIdempotent(self): predictions = random_ops.random_uniform( (10, 3), maxval=1, dtype=dtypes_lib.float32, seed=1) @@ -1371,17 +1454,18 @@ class SpecificityAtSensitivityTest(test.TestCase): labels, predictions, sensitivity=0.7) with self.cached_session() as sess: - sess.run(variables.local_variables_initializer()) + self.evaluate(variables.local_variables_initializer()) # Run several updates. for _ in range(10): - sess.run(update_op) + self.evaluate(update_op) # Then verify idempotency. initial_specificity = specificity.eval() for _ in range(10): self.assertAlmostEqual(initial_specificity, specificity.eval(), 5) + @test_util.run_deprecated_v1 def testAllCorrect(self): inputs = np.random.randint(0, 2, size=(100, 1)) @@ -1391,10 +1475,11 @@ class SpecificityAtSensitivityTest(test.TestCase): labels, predictions, sensitivity=0.7) with self.cached_session() as sess: - sess.run(variables.local_variables_initializer()) - self.assertEqual(1, sess.run(update_op)) + self.evaluate(variables.local_variables_initializer()) + self.assertEqual(1, self.evaluate(update_op)) self.assertEqual(1, specificity.eval()) + @test_util.run_deprecated_v1 def testSomeCorrectHighSensitivity(self): predictions_values = [0.1, 0.2, 0.4, 0.3, 0.0, 0.1, 0.45, 0.5, 0.8, 0.9] labels_values = [0, 0, 0, 0, 0, 1, 1, 1, 1, 1] @@ -1406,10 +1491,11 @@ class SpecificityAtSensitivityTest(test.TestCase): labels, predictions, sensitivity=0.8) with self.cached_session() as sess: - sess.run(variables.local_variables_initializer()) - self.assertAlmostEqual(1.0, sess.run(update_op)) + self.evaluate(variables.local_variables_initializer()) + self.assertAlmostEqual(1.0, self.evaluate(update_op)) self.assertAlmostEqual(1.0, specificity.eval()) + @test_util.run_deprecated_v1 def testSomeCorrectLowSensitivity(self): predictions_values = [0.1, 0.2, 0.4, 0.3, 0.0, 0.1, 0.2, 0.2, 0.26, 0.26] labels_values = [0, 0, 0, 0, 0, 1, 1, 1, 1, 1] @@ -1421,11 +1507,12 @@ class SpecificityAtSensitivityTest(test.TestCase): labels, predictions, sensitivity=0.4) with self.cached_session() as sess: - sess.run(variables.local_variables_initializer()) + self.evaluate(variables.local_variables_initializer()) - self.assertAlmostEqual(0.6, sess.run(update_op)) + self.assertAlmostEqual(0.6, self.evaluate(update_op)) self.assertAlmostEqual(0.6, specificity.eval()) + @test_util.run_deprecated_v1 def testWeighted1d_multipleLabelDtypes(self): for label_dtype in (dtypes_lib.bool, dtypes_lib.int32, dtypes_lib.float32): predictions_values = [0.1, 0.2, 0.4, 0.3, 0.0, 0.1, 0.2, 0.2, 0.26, 0.26] @@ -1440,11 +1527,12 @@ class SpecificityAtSensitivityTest(test.TestCase): labels, predictions, weights=weights, sensitivity=0.4) with self.cached_session() as sess: - sess.run(variables.local_variables_initializer()) + self.evaluate(variables.local_variables_initializer()) - self.assertAlmostEqual(0.6, sess.run(update_op)) + self.assertAlmostEqual(0.6, self.evaluate(update_op)) self.assertAlmostEqual(0.6, specificity.eval()) + @test_util.run_deprecated_v1 def testWeighted2d(self): predictions_values = [0.1, 0.2, 0.4, 0.3, 0.0, 0.1, 0.2, 0.2, 0.26, 0.26] labels_values = [0, 0, 0, 0, 0, 1, 1, 1, 1, 1] @@ -1458,9 +1546,9 @@ class SpecificityAtSensitivityTest(test.TestCase): labels, predictions, weights=weights, sensitivity=0.4) with self.cached_session() as sess: - sess.run(variables.local_variables_initializer()) + self.evaluate(variables.local_variables_initializer()) - self.assertAlmostEqual(8.0 / 15.0, sess.run(update_op)) + self.assertAlmostEqual(8.0 / 15.0, self.evaluate(update_op)) self.assertAlmostEqual(8.0 / 15.0, specificity.eval()) @@ -1470,6 +1558,7 @@ class SensitivityAtSpecificityTest(test.TestCase): np.random.seed(1) ops.reset_default_graph() + @test_util.run_deprecated_v1 def testVars(self): metrics.sensitivity_at_specificity( predictions=array_ops.ones((10, 1)), @@ -1481,6 +1570,7 @@ class SensitivityAtSpecificityTest(test.TestCase): 'sensitivity_at_specificity/false_positives:0', 'sensitivity_at_specificity/true_negatives:0')) + @test_util.run_deprecated_v1 def testMetricsCollection(self): my_collection_name = '__metrics__' mean, _ = metrics.sensitivity_at_specificity( @@ -1490,6 +1580,7 @@ class SensitivityAtSpecificityTest(test.TestCase): metrics_collections=[my_collection_name]) self.assertListEqual(ops.get_collection(my_collection_name), [mean]) + @test_util.run_deprecated_v1 def testUpdatesCollection(self): my_collection_name = '__updates__' _, update_op = metrics.sensitivity_at_specificity( @@ -1499,6 +1590,7 @@ class SensitivityAtSpecificityTest(test.TestCase): updates_collections=[my_collection_name]) self.assertListEqual(ops.get_collection(my_collection_name), [update_op]) + @test_util.run_deprecated_v1 def testValueTensorIsIdempotent(self): predictions = random_ops.random_uniform( (10, 3), maxval=1, dtype=dtypes_lib.float32, seed=1) @@ -1508,17 +1600,18 @@ class SensitivityAtSpecificityTest(test.TestCase): labels, predictions, specificity=0.7) with self.cached_session() as sess: - sess.run(variables.local_variables_initializer()) + self.evaluate(variables.local_variables_initializer()) # Run several updates. for _ in range(10): - sess.run(update_op) + self.evaluate(update_op) # Then verify idempotency. initial_sensitivity = sensitivity.eval() for _ in range(10): self.assertAlmostEqual(initial_sensitivity, sensitivity.eval(), 5) + @test_util.run_deprecated_v1 def testAllCorrect(self): inputs = np.random.randint(0, 2, size=(100, 1)) @@ -1528,10 +1621,11 @@ class SensitivityAtSpecificityTest(test.TestCase): labels, predictions, specificity=0.7) with self.cached_session() as sess: - sess.run(variables.local_variables_initializer()) - self.assertEqual(1, sess.run(update_op)) + self.evaluate(variables.local_variables_initializer()) + self.assertEqual(1, self.evaluate(update_op)) self.assertEqual(1, specificity.eval()) + @test_util.run_deprecated_v1 def testSomeCorrectHighSpecificity(self): predictions_values = [0.0, 0.1, 0.2, 0.3, 0.4, 0.1, 0.45, 0.5, 0.8, 0.9] labels_values = [0, 0, 0, 0, 0, 1, 1, 1, 1, 1] @@ -1543,10 +1637,11 @@ class SensitivityAtSpecificityTest(test.TestCase): labels, predictions, specificity=0.8) with self.cached_session() as sess: - sess.run(variables.local_variables_initializer()) - self.assertAlmostEqual(0.8, sess.run(update_op)) + self.evaluate(variables.local_variables_initializer()) + self.assertAlmostEqual(0.8, self.evaluate(update_op)) self.assertAlmostEqual(0.8, specificity.eval()) + @test_util.run_deprecated_v1 def testSomeCorrectLowSpecificity(self): predictions_values = [0.0, 0.1, 0.2, 0.3, 0.4, 0.01, 0.02, 0.25, 0.26, 0.26] labels_values = [0, 0, 0, 0, 0, 1, 1, 1, 1, 1] @@ -1558,10 +1653,11 @@ class SensitivityAtSpecificityTest(test.TestCase): labels, predictions, specificity=0.4) with self.cached_session() as sess: - sess.run(variables.local_variables_initializer()) - self.assertAlmostEqual(0.6, sess.run(update_op)) + self.evaluate(variables.local_variables_initializer()) + self.assertAlmostEqual(0.6, self.evaluate(update_op)) self.assertAlmostEqual(0.6, specificity.eval()) + @test_util.run_deprecated_v1 def testWeighted_multipleLabelDtypes(self): for label_dtype in (dtypes_lib.bool, dtypes_lib.int32, dtypes_lib.float32): predictions_values = [ @@ -1577,8 +1673,8 @@ class SensitivityAtSpecificityTest(test.TestCase): labels, predictions, weights=weights, specificity=0.4) with self.cached_session() as sess: - sess.run(variables.local_variables_initializer()) - self.assertAlmostEqual(0.675, sess.run(update_op)) + self.evaluate(variables.local_variables_initializer()) + self.assertAlmostEqual(0.675, self.evaluate(update_op)) self.assertAlmostEqual(0.675, specificity.eval()) @@ -1589,6 +1685,7 @@ class PrecisionRecallThresholdsTest(test.TestCase): np.random.seed(1) ops.reset_default_graph() + @test_util.run_deprecated_v1 def testVars(self): metrics.precision_at_thresholds( predictions=array_ops.ones((10, 1)), @@ -1599,6 +1696,7 @@ class PrecisionRecallThresholdsTest(test.TestCase): 'precision_at_thresholds/false_positives:0', )) + @test_util.run_deprecated_v1 def testMetricsCollection(self): my_collection_name = '__metrics__' prec, _ = metrics.precision_at_thresholds( @@ -1613,6 +1711,7 @@ class PrecisionRecallThresholdsTest(test.TestCase): metrics_collections=[my_collection_name]) self.assertListEqual(ops.get_collection(my_collection_name), [prec, rec]) + @test_util.run_deprecated_v1 def testUpdatesCollection(self): my_collection_name = '__updates__' _, precision_op = metrics.precision_at_thresholds( @@ -1628,6 +1727,7 @@ class PrecisionRecallThresholdsTest(test.TestCase): self.assertListEqual( ops.get_collection(my_collection_name), [precision_op, recall_op]) + @test_util.run_deprecated_v1 def testValueTensorIsIdempotent(self): predictions = random_ops.random_uniform( (10, 3), maxval=1, dtype=dtypes_lib.float32, seed=1) @@ -1639,18 +1739,19 @@ class PrecisionRecallThresholdsTest(test.TestCase): rec, rec_op = metrics.recall_at_thresholds(labels, predictions, thresholds) with self.cached_session() as sess: - sess.run(variables.local_variables_initializer()) + self.evaluate(variables.local_variables_initializer()) # Run several updates, then verify idempotency. - sess.run([prec_op, rec_op]) + self.evaluate([prec_op, rec_op]) initial_prec = prec.eval() initial_rec = rec.eval() for _ in range(10): - sess.run([prec_op, rec_op]) + self.evaluate([prec_op, rec_op]) self.assertAllClose(initial_prec, prec.eval()) self.assertAllClose(initial_rec, rec.eval()) # TODO(nsilberman): fix tests (passing but incorrect). + @test_util.run_deprecated_v1 def testAllCorrect(self): inputs = np.random.randint(0, 2, size=(100, 1)) @@ -1663,12 +1764,13 @@ class PrecisionRecallThresholdsTest(test.TestCase): rec, rec_op = metrics.recall_at_thresholds(labels, predictions, thresholds) - sess.run(variables.local_variables_initializer()) - sess.run([prec_op, rec_op]) + self.evaluate(variables.local_variables_initializer()) + self.evaluate([prec_op, rec_op]) self.assertEqual(1, prec.eval()) self.assertEqual(1, rec.eval()) + @test_util.run_deprecated_v1 def testSomeCorrect_multipleLabelDtypes(self): with self.cached_session() as sess: for label_dtype in ( @@ -1683,12 +1785,13 @@ class PrecisionRecallThresholdsTest(test.TestCase): rec, rec_op = metrics.recall_at_thresholds(labels, predictions, thresholds) - sess.run(variables.local_variables_initializer()) - sess.run([prec_op, rec_op]) + self.evaluate(variables.local_variables_initializer()) + self.evaluate([prec_op, rec_op]) self.assertAlmostEqual(0.5, prec.eval()) self.assertAlmostEqual(0.5, rec.eval()) + @test_util.run_deprecated_v1 def testAllIncorrect(self): inputs = np.random.randint(0, 2, size=(100, 1)) @@ -1701,12 +1804,13 @@ class PrecisionRecallThresholdsTest(test.TestCase): rec, rec_op = metrics.recall_at_thresholds(labels, predictions, thresholds) - sess.run(variables.local_variables_initializer()) - sess.run([prec_op, rec_op]) + self.evaluate(variables.local_variables_initializer()) + self.evaluate([prec_op, rec_op]) self.assertAlmostEqual(0, prec.eval()) self.assertAlmostEqual(0, rec.eval()) + @test_util.run_deprecated_v1 def testWeights1d(self): with self.cached_session() as sess: predictions = constant_op.constant( @@ -1729,14 +1833,15 @@ class PrecisionRecallThresholdsTest(test.TestCase): rec_low = array_ops.reshape(rec_low, shape=()) rec_high = array_ops.reshape(rec_high, shape=()) - sess.run(variables.local_variables_initializer()) - sess.run([prec_op, rec_op]) + self.evaluate(variables.local_variables_initializer()) + self.evaluate([prec_op, rec_op]) self.assertAlmostEqual(1.0, prec_low.eval(), places=5) self.assertAlmostEqual(0.0, prec_high.eval(), places=5) self.assertAlmostEqual(1.0, rec_low.eval(), places=5) self.assertAlmostEqual(0.0, rec_high.eval(), places=5) + @test_util.run_deprecated_v1 def testWeights2d(self): with self.cached_session() as sess: predictions = constant_op.constant( @@ -1759,14 +1864,15 @@ class PrecisionRecallThresholdsTest(test.TestCase): rec_low = array_ops.reshape(rec_low, shape=()) rec_high = array_ops.reshape(rec_high, shape=()) - sess.run(variables.local_variables_initializer()) - sess.run([prec_op, rec_op]) + self.evaluate(variables.local_variables_initializer()) + self.evaluate([prec_op, rec_op]) self.assertAlmostEqual(1.0, prec_low.eval(), places=5) self.assertAlmostEqual(0.0, prec_high.eval(), places=5) self.assertAlmostEqual(1.0, rec_low.eval(), places=5) self.assertAlmostEqual(0.0, rec_high.eval(), places=5) + @test_util.run_deprecated_v1 def testExtremeThresholds(self): with self.cached_session() as sess: predictions = constant_op.constant( @@ -1783,14 +1889,15 @@ class PrecisionRecallThresholdsTest(test.TestCase): [rec_low, rec_high] = array_ops.split( value=rec, num_or_size_splits=2, axis=0) - sess.run(variables.local_variables_initializer()) - sess.run([prec_op, rec_op]) + self.evaluate(variables.local_variables_initializer()) + self.evaluate([prec_op, rec_op]) self.assertAlmostEqual(0.75, prec_low.eval()) self.assertAlmostEqual(0.0, prec_high.eval()) self.assertAlmostEqual(1.0, rec_low.eval()) self.assertAlmostEqual(0.0, rec_high.eval()) + @test_util.run_deprecated_v1 def testZeroLabelsPredictions(self): with self.cached_session() as sess: predictions = array_ops.zeros([4], dtype=dtypes_lib.float32) @@ -1801,12 +1908,13 @@ class PrecisionRecallThresholdsTest(test.TestCase): rec, rec_op = metrics.recall_at_thresholds(labels, predictions, thresholds) - sess.run(variables.local_variables_initializer()) - sess.run([prec_op, rec_op]) + self.evaluate(variables.local_variables_initializer()) + self.evaluate([prec_op, rec_op]) self.assertAlmostEqual(0, prec.eval(), 6) self.assertAlmostEqual(0, rec.eval(), 6) + @test_util.run_deprecated_v1 def testWithMultipleUpdates(self): num_samples = 1000 batch_size = 10 @@ -1869,9 +1977,9 @@ class PrecisionRecallThresholdsTest(test.TestCase): rec, rec_op = metrics.recall_at_thresholds(tf_labels, tf_predictions, thresholds) - sess.run(variables.local_variables_initializer()) + self.evaluate(variables.local_variables_initializer()) for _ in range(int(num_samples / batch_size)): - sess.run([prec_op, rec_op]) + self.evaluate([prec_op, rec_op]) # Since this is only approximate, we can't expect a 6 digits match. # Although with higher number of samples/thresholds we should see the # accuracy improving @@ -1989,6 +2097,7 @@ class SingleLabelPrecisionAtKTest(test.TestCase): self._test_average_precision_at_k = functools.partial( _test_average_precision_at_k, test_case=self) + @test_util.run_deprecated_v1 def test_at_k1_nan(self): for labels in self._labels: # Classes 0,1,2 have 0 predictions, classes -1 and 4 are out of range. @@ -1998,6 +2107,7 @@ class SingleLabelPrecisionAtKTest(test.TestCase): self._test_precision_at_top_k( self._predictions_idx, labels, k=1, expected=NAN, class_id=class_id) + @test_util.run_deprecated_v1 def test_at_k1(self): for labels in self._labels: # Class 3: 1 label, 2 predictions, 1 correct. @@ -2025,6 +2135,7 @@ class MultiLabelPrecisionAtKTest(test.TestCase): self._test_average_precision_at_k = functools.partial( _test_average_precision_at_k, test_case=self) + @test_util.run_deprecated_v1 def test_average_precision(self): # Example 1. # Matches example here: @@ -2100,6 +2211,7 @@ class MultiLabelPrecisionAtKTest(test.TestCase): expected=streaming_average_precision[i], weights=weights) + @test_util.run_deprecated_v1 def test_average_precision_some_labels_out_of_range(self): """Tests that labels outside the [0, n_classes) range are ignored.""" labels_ex1 = (-1, 0, 1, 2, 3, 4, 7) @@ -2119,6 +2231,7 @@ class MultiLabelPrecisionAtKTest(test.TestCase): self._test_average_precision_at_k( predictions, labels, k, expected=avg_precision_ex1[i]) + @test_util.run_deprecated_v1 def test_three_labels_at_k5_no_predictions(self): predictions = [[0.5, 0.1, 0.6, 0.3, 0.8, 0.0, 0.7, 0.2, 0.4, 0.9], [0.3, 0.0, 0.7, 0.2, 0.4, 0.9, 0.5, 0.8, 0.1, 0.6]] @@ -2135,6 +2248,7 @@ class MultiLabelPrecisionAtKTest(test.TestCase): self._test_precision_at_top_k( predictions_idx, labels, k=5, expected=NAN, class_id=class_id) + @test_util.run_deprecated_v1 def test_three_labels_at_k5_no_labels(self): predictions = [[0.5, 0.1, 0.6, 0.3, 0.8, 0.0, 0.7, 0.2, 0.4, 0.9], [0.3, 0.0, 0.7, 0.2, 0.4, 0.9, 0.5, 0.8, 0.1, 0.6]] @@ -2151,6 +2265,7 @@ class MultiLabelPrecisionAtKTest(test.TestCase): self._test_precision_at_top_k( predictions_idx, labels, k=5, expected=0.0, class_id=class_id) + @test_util.run_deprecated_v1 def test_three_labels_at_k5(self): predictions = [[0.5, 0.1, 0.6, 0.3, 0.8, 0.0, 0.7, 0.2, 0.4, 0.9], [0.3, 0.0, 0.7, 0.2, 0.4, 0.9, 0.5, 0.8, 0.1, 0.6]] @@ -2184,6 +2299,7 @@ class MultiLabelPrecisionAtKTest(test.TestCase): self._test_precision_at_top_k( predictions_idx, labels, k=5, expected=3.0 / 10) + @test_util.run_deprecated_v1 def test_three_labels_at_k5_some_out_of_range(self): """Tests that labels outside the [0, n_classes) range are ignored.""" predictions = [[0.5, 0.1, 0.6, 0.3, 0.8, 0.0, 0.7, 0.2, 0.4, 0.9], @@ -2220,6 +2336,7 @@ class MultiLabelPrecisionAtKTest(test.TestCase): self._test_precision_at_top_k( predictions_idx, sp_labels, k=5, expected=3.0 / 10) + @test_util.run_deprecated_v1 def test_3d_nan(self): predictions = [[[0.5, 0.1, 0.6, 0.3, 0.8, 0.0, 0.7, 0.2, 0.4, 0.9], [0.3, 0.0, 0.7, 0.2, 0.4, 0.9, 0.5, 0.8, 0.1, 0.6]], @@ -2238,6 +2355,7 @@ class MultiLabelPrecisionAtKTest(test.TestCase): self._test_precision_at_top_k( predictions_idx, labels, k=5, expected=NAN, class_id=class_id) + @test_util.run_deprecated_v1 def test_3d_no_labels(self): predictions = [[[0.5, 0.1, 0.6, 0.3, 0.8, 0.0, 0.7, 0.2, 0.4, 0.9], [0.3, 0.0, 0.7, 0.2, 0.4, 0.9, 0.5, 0.8, 0.1, 0.6]], @@ -2256,6 +2374,7 @@ class MultiLabelPrecisionAtKTest(test.TestCase): self._test_precision_at_top_k( predictions_idx, labels, k=5, expected=0.0, class_id=class_id) + @test_util.run_deprecated_v1 def test_3d(self): predictions = [[[0.5, 0.1, 0.6, 0.3, 0.8, 0.0, 0.7, 0.2, 0.4, 0.9], [0.3, 0.0, 0.7, 0.2, 0.4, 0.9, 0.5, 0.8, 0.1, 0.6]], @@ -2291,6 +2410,7 @@ class MultiLabelPrecisionAtKTest(test.TestCase): self._test_precision_at_top_k( predictions_idx, labels, k=5, expected=7.0 / 20) + @test_util.run_deprecated_v1 def test_3d_ignore_some(self): predictions = [[[0.5, 0.1, 0.6, 0.3, 0.8, 0.0, 0.7, 0.2, 0.4, 0.9], [0.3, 0.0, 0.7, 0.2, 0.4, 0.9, 0.5, 0.8, 0.1, 0.6]], @@ -2432,6 +2552,7 @@ class SingleLabelRecallAtKTest(test.TestCase): self._test_recall_at_top_k = functools.partial( _test_recall_at_top_k, test_case=self) + @test_util.run_deprecated_v1 def test_at_k1_nan(self): # Classes 0,1 have 0 labels, 0 predictions, classes -1 and 4 are out of # range. @@ -2442,6 +2563,7 @@ class SingleLabelRecallAtKTest(test.TestCase): self._test_recall_at_top_k( self._predictions_idx, labels, k=1, expected=NAN, class_id=class_id) + @test_util.run_deprecated_v1 def test_at_k1_no_predictions(self): for labels in self._labels: # Class 2: 0 predictions. @@ -2450,6 +2572,7 @@ class SingleLabelRecallAtKTest(test.TestCase): self._test_recall_at_top_k( self._predictions_idx, labels, k=1, expected=0.0, class_id=2) + @test_util.run_deprecated_v1 def test_one_label_at_k1(self): for labels in self._labels: # Class 3: 1 label, 2 predictions, 1 correct. @@ -2463,6 +2586,7 @@ class SingleLabelRecallAtKTest(test.TestCase): self._test_recall_at_top_k( self._predictions_idx, labels, k=1, expected=1.0 / 2) + @test_util.run_deprecated_v1 def test_one_label_at_k1_weighted_class_id3(self): predictions = self._predictions predictions_idx = self._predictions_idx @@ -2504,6 +2628,7 @@ class SingleLabelRecallAtKTest(test.TestCase): predictions_idx, labels, k=1, expected=2.0 / 2, class_id=3, weights=(2.0, 3.0)) + @test_util.run_deprecated_v1 def test_one_label_at_k1_weighted(self): predictions = self._predictions predictions_idx = self._predictions_idx @@ -2553,6 +2678,7 @@ class MultiLabel2dRecallAtKTest(test.TestCase): self._test_recall_at_top_k = functools.partial( _test_recall_at_top_k, test_case=self) + @test_util.run_deprecated_v1 def test_at_k5_nan(self): for labels in self._labels: # Classes 0,3,4,6,9 have 0 labels, class 10 is out of range. @@ -2562,6 +2688,7 @@ class MultiLabel2dRecallAtKTest(test.TestCase): self._test_recall_at_top_k( self._predictions_idx, labels, k=5, expected=NAN, class_id=class_id) + @test_util.run_deprecated_v1 def test_at_k5_no_predictions(self): for labels in self._labels: # Class 8: 1 label, no predictions. @@ -2570,6 +2697,7 @@ class MultiLabel2dRecallAtKTest(test.TestCase): self._test_recall_at_top_k( self._predictions_idx, labels, k=5, expected=0.0 / 1, class_id=8) + @test_util.run_deprecated_v1 def test_at_k5(self): for labels in self._labels: # Class 2: 2 labels, both correct. @@ -2595,6 +2723,7 @@ class MultiLabel2dRecallAtKTest(test.TestCase): self._test_recall_at_top_k( self._predictions_idx, labels, k=5, expected=3.0 / 6) + @test_util.run_deprecated_v1 def test_at_k5_some_out_of_range(self): """Tests that labels outside the [0, n_classes) count in denominator.""" labels = sparse_tensor.SparseTensorValue( @@ -2647,6 +2776,7 @@ class MultiLabel3dRecallAtKTest(test.TestCase): self._test_recall_at_top_k = functools.partial( _test_recall_at_top_k, test_case=self) + @test_util.run_deprecated_v1 def test_3d_nan(self): # Classes 0,3,4,6,9 have 0 labels, class 10 is out of range. for class_id in (0, 3, 4, 6, 9, 10): @@ -2656,6 +2786,7 @@ class MultiLabel3dRecallAtKTest(test.TestCase): self._predictions_idx, self._labels, k=5, expected=NAN, class_id=class_id) + @test_util.run_deprecated_v1 def test_3d_no_predictions(self): # Classes 1,8 have 0 predictions, >=1 label. for class_id in (1, 8): @@ -2665,6 +2796,7 @@ class MultiLabel3dRecallAtKTest(test.TestCase): self._predictions_idx, self._labels, k=5, expected=0.0, class_id=class_id) + @test_util.run_deprecated_v1 def test_3d(self): # Class 2: 4 labels, all correct. self._test_recall_at_k( @@ -2693,6 +2825,7 @@ class MultiLabel3dRecallAtKTest(test.TestCase): self._test_recall_at_top_k( self._predictions_idx, self._labels, k=5, expected=7.0 / 12) + @test_util.run_deprecated_v1 def test_3d_ignore_all(self): for class_id in xrange(10): self._test_recall_at_k( @@ -2719,6 +2852,7 @@ class MultiLabel3dRecallAtKTest(test.TestCase): self._predictions_idx, self._labels, k=5, expected=NAN, weights=[[0, 0], [0, 0]]) + @test_util.run_deprecated_v1 def test_3d_ignore_some(self): # Class 2: 2 labels, both correct. self._test_recall_at_k( @@ -2774,12 +2908,14 @@ class MeanAbsoluteErrorTest(test.TestCase): def setUp(self): ops.reset_default_graph() + @test_util.run_deprecated_v1 def testVars(self): metrics.mean_absolute_error( predictions=array_ops.ones((10, 1)), labels=array_ops.ones((10, 1))) _assert_metric_variables( self, ('mean_absolute_error/count:0', 'mean_absolute_error/total:0')) + @test_util.run_deprecated_v1 def testMetricsCollection(self): my_collection_name = '__metrics__' mean, _ = metrics.mean_absolute_error( @@ -2788,6 +2924,7 @@ class MeanAbsoluteErrorTest(test.TestCase): metrics_collections=[my_collection_name]) self.assertListEqual(ops.get_collection(my_collection_name), [mean]) + @test_util.run_deprecated_v1 def testUpdatesCollection(self): my_collection_name = '__updates__' _, update_op = metrics.mean_absolute_error( @@ -2796,23 +2933,25 @@ class MeanAbsoluteErrorTest(test.TestCase): updates_collections=[my_collection_name]) self.assertListEqual(ops.get_collection(my_collection_name), [update_op]) + @test_util.run_deprecated_v1 def testValueTensorIsIdempotent(self): predictions = random_ops.random_normal((10, 3), seed=1) labels = random_ops.random_normal((10, 3), seed=2) error, update_op = metrics.mean_absolute_error(labels, predictions) with self.cached_session() as sess: - sess.run(variables.local_variables_initializer()) + self.evaluate(variables.local_variables_initializer()) # Run several updates. for _ in range(10): - sess.run(update_op) + self.evaluate(update_op) # Then verify idempotency. initial_error = error.eval() for _ in range(10): self.assertEqual(initial_error, error.eval()) + @test_util.run_deprecated_v1 def testSingleUpdateWithErrorAndWeights(self): predictions = constant_op.constant( [2, 4, 6, 8], shape=(1, 4), dtype=dtypes_lib.float32) @@ -2823,8 +2962,8 @@ class MeanAbsoluteErrorTest(test.TestCase): error, update_op = metrics.mean_absolute_error(labels, predictions, weights) with self.cached_session() as sess: - sess.run(variables.local_variables_initializer()) - self.assertEqual(3, sess.run(update_op)) + self.evaluate(variables.local_variables_initializer()) + self.assertEqual(3, self.evaluate(update_op)) self.assertEqual(3, error.eval()) @@ -2833,6 +2972,7 @@ class MeanRelativeErrorTest(test.TestCase): def setUp(self): ops.reset_default_graph() + @test_util.run_deprecated_v1 def testVars(self): metrics.mean_relative_error( predictions=array_ops.ones((10, 1)), @@ -2841,6 +2981,7 @@ class MeanRelativeErrorTest(test.TestCase): _assert_metric_variables( self, ('mean_relative_error/count:0', 'mean_relative_error/total:0')) + @test_util.run_deprecated_v1 def testMetricsCollection(self): my_collection_name = '__metrics__' mean, _ = metrics.mean_relative_error( @@ -2850,6 +2991,7 @@ class MeanRelativeErrorTest(test.TestCase): metrics_collections=[my_collection_name]) self.assertListEqual(ops.get_collection(my_collection_name), [mean]) + @test_util.run_deprecated_v1 def testUpdatesCollection(self): my_collection_name = '__updates__' _, update_op = metrics.mean_relative_error( @@ -2859,6 +3001,7 @@ class MeanRelativeErrorTest(test.TestCase): updates_collections=[my_collection_name]) self.assertListEqual(ops.get_collection(my_collection_name), [update_op]) + @test_util.run_deprecated_v1 def testValueTensorIsIdempotent(self): predictions = random_ops.random_normal((10, 3), seed=1) labels = random_ops.random_normal((10, 3), seed=2) @@ -2867,17 +3010,18 @@ class MeanRelativeErrorTest(test.TestCase): normalizer) with self.cached_session() as sess: - sess.run(variables.local_variables_initializer()) + self.evaluate(variables.local_variables_initializer()) # Run several updates. for _ in range(10): - sess.run(update_op) + self.evaluate(update_op) # Then verify idempotency. initial_error = error.eval() for _ in range(10): self.assertEqual(initial_error, error.eval()) + @test_util.run_deprecated_v1 def testSingleUpdateNormalizedByLabels(self): np_predictions = np.asarray([2, 4, 6, 8], dtype=np.float32) np_labels = np.asarray([1, 3, 2, 3], dtype=np.float32) @@ -2892,10 +3036,11 @@ class MeanRelativeErrorTest(test.TestCase): labels, predictions, normalizer=labels) with self.cached_session() as sess: - sess.run(variables.local_variables_initializer()) - self.assertEqual(expected_error, sess.run(update_op)) + self.evaluate(variables.local_variables_initializer()) + self.assertEqual(expected_error, self.evaluate(update_op)) self.assertEqual(expected_error, error.eval()) + @test_util.run_deprecated_v1 def testSingleUpdateNormalizedByZeros(self): np_predictions = np.asarray([2, 4, 6, 8], dtype=np.float32) @@ -2908,8 +3053,8 @@ class MeanRelativeErrorTest(test.TestCase): labels, predictions, normalizer=array_ops.zeros_like(labels)) with self.cached_session() as sess: - sess.run(variables.local_variables_initializer()) - self.assertEqual(0.0, sess.run(update_op)) + self.evaluate(variables.local_variables_initializer()) + self.assertEqual(0.0, self.evaluate(update_op)) self.assertEqual(0.0, error.eval()) @@ -2918,12 +3063,14 @@ class MeanSquaredErrorTest(test.TestCase): def setUp(self): ops.reset_default_graph() + @test_util.run_deprecated_v1 def testVars(self): metrics.mean_squared_error( predictions=array_ops.ones((10, 1)), labels=array_ops.ones((10, 1))) _assert_metric_variables( self, ('mean_squared_error/count:0', 'mean_squared_error/total:0')) + @test_util.run_deprecated_v1 def testMetricsCollection(self): my_collection_name = '__metrics__' mean, _ = metrics.mean_squared_error( @@ -2932,6 +3079,7 @@ class MeanSquaredErrorTest(test.TestCase): metrics_collections=[my_collection_name]) self.assertListEqual(ops.get_collection(my_collection_name), [mean]) + @test_util.run_deprecated_v1 def testUpdatesCollection(self): my_collection_name = '__updates__' _, update_op = metrics.mean_squared_error( @@ -2940,23 +3088,25 @@ class MeanSquaredErrorTest(test.TestCase): updates_collections=[my_collection_name]) self.assertListEqual(ops.get_collection(my_collection_name), [update_op]) + @test_util.run_deprecated_v1 def testValueTensorIsIdempotent(self): predictions = random_ops.random_normal((10, 3), seed=1) labels = random_ops.random_normal((10, 3), seed=2) error, update_op = metrics.mean_squared_error(labels, predictions) with self.cached_session() as sess: - sess.run(variables.local_variables_initializer()) + self.evaluate(variables.local_variables_initializer()) # Run several updates. for _ in range(10): - sess.run(update_op) + self.evaluate(update_op) # Then verify idempotency. initial_error = error.eval() for _ in range(10): self.assertEqual(initial_error, error.eval()) + @test_util.run_deprecated_v1 def testSingleUpdateZeroError(self): predictions = array_ops.zeros((1, 3), dtype=dtypes_lib.float32) labels = array_ops.zeros((1, 3), dtype=dtypes_lib.float32) @@ -2964,10 +3114,11 @@ class MeanSquaredErrorTest(test.TestCase): error, update_op = metrics.mean_squared_error(labels, predictions) with self.cached_session() as sess: - sess.run(variables.local_variables_initializer()) - self.assertEqual(0, sess.run(update_op)) + self.evaluate(variables.local_variables_initializer()) + self.assertEqual(0, self.evaluate(update_op)) self.assertEqual(0, error.eval()) + @test_util.run_deprecated_v1 def testSingleUpdateWithError(self): predictions = constant_op.constant( [2, 4, 6], shape=(1, 3), dtype=dtypes_lib.float32) @@ -2977,10 +3128,11 @@ class MeanSquaredErrorTest(test.TestCase): error, update_op = metrics.mean_squared_error(labels, predictions) with self.cached_session() as sess: - sess.run(variables.local_variables_initializer()) - self.assertEqual(6, sess.run(update_op)) + self.evaluate(variables.local_variables_initializer()) + self.assertEqual(6, self.evaluate(update_op)) self.assertEqual(6, error.eval()) + @test_util.run_deprecated_v1 def testSingleUpdateWithErrorAndWeights(self): predictions = constant_op.constant( [2, 4, 6, 8], shape=(1, 4), dtype=dtypes_lib.float32) @@ -2991,10 +3143,11 @@ class MeanSquaredErrorTest(test.TestCase): error, update_op = metrics.mean_squared_error(labels, predictions, weights) with self.cached_session() as sess: - sess.run(variables.local_variables_initializer()) - self.assertEqual(13, sess.run(update_op)) + self.evaluate(variables.local_variables_initializer()) + self.assertEqual(13, self.evaluate(update_op)) self.assertEqual(13, error.eval()) + @test_util.run_deprecated_v1 def testMultipleBatchesOfSizeOne(self): with self.cached_session() as sess: # Create the queue that populates the predictions. @@ -3013,12 +3166,13 @@ class MeanSquaredErrorTest(test.TestCase): error, update_op = metrics.mean_squared_error(labels, predictions) - sess.run(variables.local_variables_initializer()) - sess.run(update_op) - self.assertAlmostEqual(208.0 / 6, sess.run(update_op), 5) + self.evaluate(variables.local_variables_initializer()) + self.evaluate(update_op) + self.assertAlmostEqual(208.0 / 6, self.evaluate(update_op), 5) self.assertAlmostEqual(208.0 / 6, error.eval(), 5) + @test_util.run_deprecated_v1 def testMetricsComputedConcurrently(self): with self.cached_session() as sess: # Create the queue that populates one set of predictions. @@ -3054,14 +3208,15 @@ class MeanSquaredErrorTest(test.TestCase): mse1, update_op1 = metrics.mean_squared_error( labels1, predictions1, name='msd1') - sess.run(variables.local_variables_initializer()) - sess.run([update_op0, update_op1]) - sess.run([update_op0, update_op1]) + self.evaluate(variables.local_variables_initializer()) + self.evaluate([update_op0, update_op1]) + self.evaluate([update_op0, update_op1]) - mse0, mse1 = sess.run([mse0, mse1]) + mse0, mse1 = self.evaluate([mse0, mse1]) self.assertAlmostEqual(208.0 / 6, mse0, 5) self.assertAlmostEqual(79.0 / 6, mse1, 5) + @test_util.run_deprecated_v1 def testMultipleMetricsOnMultipleBatchesOfSizeOne(self): with self.cached_session() as sess: # Create the queue that populates the predictions. @@ -3081,9 +3236,9 @@ class MeanSquaredErrorTest(test.TestCase): mae, ma_update_op = metrics.mean_absolute_error(labels, predictions) mse, ms_update_op = metrics.mean_squared_error(labels, predictions) - sess.run(variables.local_variables_initializer()) - sess.run([ma_update_op, ms_update_op]) - sess.run([ma_update_op, ms_update_op]) + self.evaluate(variables.local_variables_initializer()) + self.evaluate([ma_update_op, ms_update_op]) + self.evaluate([ma_update_op, ms_update_op]) self.assertAlmostEqual(32.0 / 6, mae.eval(), 5) self.assertAlmostEqual(208.0 / 6, mse.eval(), 5) @@ -3094,6 +3249,7 @@ class RootMeanSquaredErrorTest(test.TestCase): def setUp(self): ops.reset_default_graph() + @test_util.run_deprecated_v1 def testVars(self): metrics.root_mean_squared_error( predictions=array_ops.ones((10, 1)), labels=array_ops.ones((10, 1))) @@ -3101,6 +3257,7 @@ class RootMeanSquaredErrorTest(test.TestCase): self, ('root_mean_squared_error/count:0', 'root_mean_squared_error/total:0')) + @test_util.run_deprecated_v1 def testMetricsCollection(self): my_collection_name = '__metrics__' mean, _ = metrics.root_mean_squared_error( @@ -3109,6 +3266,7 @@ class RootMeanSquaredErrorTest(test.TestCase): metrics_collections=[my_collection_name]) self.assertListEqual(ops.get_collection(my_collection_name), [mean]) + @test_util.run_deprecated_v1 def testUpdatesCollection(self): my_collection_name = '__updates__' _, update_op = metrics.root_mean_squared_error( @@ -3117,23 +3275,25 @@ class RootMeanSquaredErrorTest(test.TestCase): updates_collections=[my_collection_name]) self.assertListEqual(ops.get_collection(my_collection_name), [update_op]) + @test_util.run_deprecated_v1 def testValueTensorIsIdempotent(self): predictions = random_ops.random_normal((10, 3), seed=1) labels = random_ops.random_normal((10, 3), seed=2) error, update_op = metrics.root_mean_squared_error(labels, predictions) with self.cached_session() as sess: - sess.run(variables.local_variables_initializer()) + self.evaluate(variables.local_variables_initializer()) # Run several updates. for _ in range(10): - sess.run(update_op) + self.evaluate(update_op) # Then verify idempotency. initial_error = error.eval() for _ in range(10): self.assertEqual(initial_error, error.eval()) + @test_util.run_deprecated_v1 def testSingleUpdateZeroError(self): with self.cached_session() as sess: predictions = constant_op.constant( @@ -3142,11 +3302,12 @@ class RootMeanSquaredErrorTest(test.TestCase): rmse, update_op = metrics.root_mean_squared_error(labels, predictions) - sess.run(variables.local_variables_initializer()) - self.assertEqual(0, sess.run(update_op)) + self.evaluate(variables.local_variables_initializer()) + self.assertEqual(0, self.evaluate(update_op)) self.assertEqual(0, rmse.eval()) + @test_util.run_deprecated_v1 def testSingleUpdateWithError(self): with self.cached_session() as sess: predictions = constant_op.constant( @@ -3156,10 +3317,11 @@ class RootMeanSquaredErrorTest(test.TestCase): rmse, update_op = metrics.root_mean_squared_error(labels, predictions) - sess.run(variables.local_variables_initializer()) + self.evaluate(variables.local_variables_initializer()) self.assertAlmostEqual(math.sqrt(6), update_op.eval(), 5) self.assertAlmostEqual(math.sqrt(6), rmse.eval(), 5) + @test_util.run_deprecated_v1 def testSingleUpdateWithErrorAndWeights(self): with self.cached_session() as sess: predictions = constant_op.constant( @@ -3171,8 +3333,8 @@ class RootMeanSquaredErrorTest(test.TestCase): rmse, update_op = metrics.root_mean_squared_error(labels, predictions, weights) - sess.run(variables.local_variables_initializer()) - self.assertAlmostEqual(math.sqrt(13), sess.run(update_op)) + self.evaluate(variables.local_variables_initializer()) + self.assertAlmostEqual(math.sqrt(13), self.evaluate(update_op)) self.assertAlmostEqual(math.sqrt(13), rmse.eval(), 5) @@ -3187,6 +3349,7 @@ class MeanCosineDistanceTest(test.TestCase): def setUp(self): ops.reset_default_graph() + @test_util.run_deprecated_v1 def testVars(self): metrics.mean_cosine_distance( predictions=array_ops.ones((10, 3)), @@ -3197,6 +3360,7 @@ class MeanCosineDistanceTest(test.TestCase): 'mean_cosine_distance/total:0', )) + @test_util.run_deprecated_v1 def testMetricsCollection(self): my_collection_name = '__metrics__' mean, _ = metrics.mean_cosine_distance( @@ -3206,6 +3370,7 @@ class MeanCosineDistanceTest(test.TestCase): metrics_collections=[my_collection_name]) self.assertListEqual(ops.get_collection(my_collection_name), [mean]) + @test_util.run_deprecated_v1 def testUpdatesCollection(self): my_collection_name = '__updates__' _, update_op = metrics.mean_cosine_distance( @@ -3215,23 +3380,25 @@ class MeanCosineDistanceTest(test.TestCase): updates_collections=[my_collection_name]) self.assertListEqual(ops.get_collection(my_collection_name), [update_op]) + @test_util.run_deprecated_v1 def testValueTensorIsIdempotent(self): predictions = random_ops.random_normal((10, 3), seed=1) labels = random_ops.random_normal((10, 3), seed=2) error, update_op = metrics.mean_cosine_distance(labels, predictions, dim=1) with self.cached_session() as sess: - sess.run(variables.local_variables_initializer()) + self.evaluate(variables.local_variables_initializer()) # Run several updates. for _ in range(10): - sess.run(update_op) + self.evaluate(update_op) # Then verify idempotency. initial_error = error.eval() for _ in range(10): self.assertEqual(initial_error, error.eval()) + @test_util.run_deprecated_v1 def testSingleUpdateZeroError(self): np_labels = np.matrix(('1 0 0;' '0 0 1;' '0 1 0')) @@ -3243,10 +3410,11 @@ class MeanCosineDistanceTest(test.TestCase): error, update_op = metrics.mean_cosine_distance(labels, predictions, dim=2) with self.cached_session() as sess: - sess.run(variables.local_variables_initializer()) - self.assertEqual(0, sess.run(update_op)) + self.evaluate(variables.local_variables_initializer()) + self.assertEqual(0, self.evaluate(update_op)) self.assertEqual(0, error.eval()) + @test_util.run_deprecated_v1 def testSingleUpdateWithError1(self): np_labels = np.matrix(('1 0 0;' '0 0 1;' '0 1 0')) np_predictions = np.matrix(('1 0 0;' '0 0 -1;' '1 0 0')) @@ -3259,10 +3427,11 @@ class MeanCosineDistanceTest(test.TestCase): error, update_op = metrics.mean_cosine_distance(labels, predictions, dim=2) with self.cached_session() as sess: - sess.run(variables.local_variables_initializer()) - self.assertAlmostEqual(1, sess.run(update_op), 5) + self.evaluate(variables.local_variables_initializer()) + self.assertAlmostEqual(1, self.evaluate(update_op), 5) self.assertAlmostEqual(1, error.eval(), 5) + @test_util.run_deprecated_v1 def testSingleUpdateWithError2(self): np_predictions = np.matrix( ('0.819031913261206 0.567041924552012 0.087465312324590;' @@ -3280,10 +3449,11 @@ class MeanCosineDistanceTest(test.TestCase): error, update_op = metrics.mean_cosine_distance(labels, predictions, dim=2) with self.cached_session() as sess: - sess.run(variables.local_variables_initializer()) - self.assertAlmostEqual(1.0, sess.run(update_op), 5) + self.evaluate(variables.local_variables_initializer()) + self.assertAlmostEqual(1.0, self.evaluate(update_op), 5) self.assertAlmostEqual(1.0, error.eval(), 5) + @test_util.run_deprecated_v1 def testSingleUpdateWithErrorAndWeights1(self): np_predictions = np.matrix(('1 0 0;' '0 0 -1;' '1 0 0')) np_labels = np.matrix(('1 0 0;' '0 0 1;' '0 1 0')) @@ -3299,10 +3469,11 @@ class MeanCosineDistanceTest(test.TestCase): labels, predictions, dim=2, weights=weights) with self.cached_session() as sess: - sess.run(variables.local_variables_initializer()) - self.assertEqual(0, sess.run(update_op)) + self.evaluate(variables.local_variables_initializer()) + self.assertEqual(0, self.evaluate(update_op)) self.assertEqual(0, error.eval()) + @test_util.run_deprecated_v1 def testSingleUpdateWithErrorAndWeights2(self): np_predictions = np.matrix(('1 0 0;' '0 0 -1;' '1 0 0')) np_labels = np.matrix(('1 0 0;' '0 0 1;' '0 1 0')) @@ -3318,7 +3489,7 @@ class MeanCosineDistanceTest(test.TestCase): labels, predictions, dim=2, weights=weights) with self.cached_session() as sess: - sess.run(variables.local_variables_initializer()) + self.evaluate(variables.local_variables_initializer()) self.assertEqual(1.5, update_op.eval()) self.assertEqual(1.5, error.eval()) @@ -3328,6 +3499,7 @@ class PcntBelowThreshTest(test.TestCase): def setUp(self): ops.reset_default_graph() + @test_util.run_deprecated_v1 def testVars(self): metrics.percentage_below(values=array_ops.ones((10,)), threshold=2) _assert_metric_variables(self, ( @@ -3335,6 +3507,7 @@ class PcntBelowThreshTest(test.TestCase): 'percentage_below_threshold/total:0', )) + @test_util.run_deprecated_v1 def testMetricsCollection(self): my_collection_name = '__metrics__' mean, _ = metrics.percentage_below( @@ -3343,6 +3516,7 @@ class PcntBelowThreshTest(test.TestCase): metrics_collections=[my_collection_name]) self.assertListEqual(ops.get_collection(my_collection_name), [mean]) + @test_util.run_deprecated_v1 def testUpdatesCollection(self): my_collection_name = '__updates__' _, update_op = metrics.percentage_below( @@ -3351,6 +3525,7 @@ class PcntBelowThreshTest(test.TestCase): updates_collections=[my_collection_name]) self.assertListEqual(ops.get_collection(my_collection_name), [update_op]) + @test_util.run_deprecated_v1 def testOneUpdate(self): with self.cached_session() as sess: values = constant_op.constant( @@ -3360,14 +3535,15 @@ class PcntBelowThreshTest(test.TestCase): pcnt1, update_op1 = metrics.percentage_below(values, 7, name='medium') pcnt2, update_op2 = metrics.percentage_below(values, 1, name='low') - sess.run(variables.local_variables_initializer()) - sess.run([update_op0, update_op1, update_op2]) + self.evaluate(variables.local_variables_initializer()) + self.evaluate([update_op0, update_op1, update_op2]) - pcnt0, pcnt1, pcnt2 = sess.run([pcnt0, pcnt1, pcnt2]) + pcnt0, pcnt1, pcnt2 = self.evaluate([pcnt0, pcnt1, pcnt2]) self.assertAlmostEqual(1.0, pcnt0, 5) self.assertAlmostEqual(0.75, pcnt1, 5) self.assertAlmostEqual(0.0, pcnt2, 5) + @test_util.run_deprecated_v1 def testSomePresentOneUpdate(self): with self.cached_session() as sess: values = constant_op.constant( @@ -3382,11 +3558,11 @@ class PcntBelowThreshTest(test.TestCase): pcnt2, update_op2 = metrics.percentage_below( values, 1, weights=weights, name='low') - sess.run(variables.local_variables_initializer()) + self.evaluate(variables.local_variables_initializer()) self.assertListEqual([1.0, 0.5, 0.0], - sess.run([update_op0, update_op1, update_op2])) + self.evaluate([update_op0, update_op1, update_op2])) - pcnt0, pcnt1, pcnt2 = sess.run([pcnt0, pcnt1, pcnt2]) + pcnt0, pcnt1, pcnt2 = self.evaluate([pcnt0, pcnt1, pcnt2]) self.assertAlmostEqual(1.0, pcnt0, 5) self.assertAlmostEqual(0.5, pcnt1, 5) self.assertAlmostEqual(0.0, pcnt2, 5) @@ -3398,6 +3574,7 @@ class MeanIOUTest(test.TestCase): np.random.seed(1) ops.reset_default_graph() + @test_util.run_deprecated_v1 def testVars(self): metrics.mean_iou( predictions=array_ops.ones([10, 1]), @@ -3405,6 +3582,7 @@ class MeanIOUTest(test.TestCase): num_classes=2) _assert_metric_variables(self, ('mean_iou/total_confusion_matrix:0',)) + @test_util.run_deprecated_v1 def testMetricsCollections(self): my_collection_name = '__metrics__' mean_iou, _ = metrics.mean_iou( @@ -3414,6 +3592,7 @@ class MeanIOUTest(test.TestCase): metrics_collections=[my_collection_name]) self.assertListEqual(ops.get_collection(my_collection_name), [mean_iou]) + @test_util.run_deprecated_v1 def testUpdatesCollection(self): my_collection_name = '__updates__' _, update_op = metrics.mean_iou( @@ -3423,12 +3602,14 @@ class MeanIOUTest(test.TestCase): updates_collections=[my_collection_name]) self.assertListEqual(ops.get_collection(my_collection_name), [update_op]) + @test_util.run_deprecated_v1 def testPredictionsAndLabelsOfDifferentSizeRaisesValueError(self): predictions = array_ops.ones([10, 3]) labels = array_ops.ones([10, 4]) with self.assertRaises(ValueError): metrics.mean_iou(labels, predictions, num_classes=2) + @test_util.run_deprecated_v1 def testLabelsAndWeightsOfDifferentSizeRaisesValueError(self): predictions = array_ops.ones([10]) labels = array_ops.ones([10]) @@ -3436,6 +3617,7 @@ class MeanIOUTest(test.TestCase): with self.assertRaises(ValueError): metrics.mean_iou(labels, predictions, num_classes=2, weights=weights) + @test_util.run_deprecated_v1 def testValueTensorIsIdempotent(self): num_classes = 3 predictions = random_ops.random_uniform( @@ -3446,17 +3628,18 @@ class MeanIOUTest(test.TestCase): labels, predictions, num_classes=num_classes) with self.cached_session() as sess: - sess.run(variables.local_variables_initializer()) + self.evaluate(variables.local_variables_initializer()) # Run several updates. for _ in range(10): - sess.run(update_op) + self.evaluate(update_op) # Then verify idempotency. initial_mean_iou = mean_iou.eval() for _ in range(10): self.assertEqual(initial_mean_iou, mean_iou.eval()) + @test_util.run_deprecated_v1 def testMultipleUpdates(self): num_classes = 3 with self.cached_session() as sess: @@ -3482,12 +3665,13 @@ class MeanIOUTest(test.TestCase): miou, update_op = metrics.mean_iou(labels, predictions, num_classes) - sess.run(variables.local_variables_initializer()) + self.evaluate(variables.local_variables_initializer()) for _ in range(5): - sess.run(update_op) + self.evaluate(update_op) desired_output = np.mean([1.0 / 2.0, 1.0 / 4.0, 0.]) self.assertEqual(desired_output, miou.eval()) + @test_util.run_deprecated_v1 def testMultipleUpdatesWithWeights(self): num_classes = 2 with self.cached_session() as sess: @@ -3529,10 +3713,11 @@ class MeanIOUTest(test.TestCase): variables.local_variables_initializer().run() for _ in range(6): - sess.run(update_op) + self.evaluate(update_op) desired_output = np.mean([2.0 / 3.0, 1.0 / 2.0]) self.assertAlmostEqual(desired_output, mean_iou.eval()) + @test_util.run_deprecated_v1 def testMultipleUpdatesWithMissingClass(self): # Test the case where there are no predicions and labels for # one class, and thus there is one row and one column with @@ -3563,12 +3748,13 @@ class MeanIOUTest(test.TestCase): miou, update_op = metrics.mean_iou(labels, predictions, num_classes) - sess.run(variables.local_variables_initializer()) + self.evaluate(variables.local_variables_initializer()) for _ in range(5): - sess.run(update_op) + self.evaluate(update_op) desired_output = np.mean([1.0 / 3.0, 2.0 / 4.0]) self.assertAlmostEqual(desired_output, miou.eval()) + @test_util.run_deprecated_v1 def testUpdateOpEvalIsAccumulatedConfusionMatrix(self): predictions = array_ops.concat( [ @@ -3587,32 +3773,35 @@ class MeanIOUTest(test.TestCase): num_classes = 2 with self.cached_session() as sess: miou, update_op = metrics.mean_iou(labels, predictions, num_classes) - sess.run(variables.local_variables_initializer()) + self.evaluate(variables.local_variables_initializer()) confusion_matrix = update_op.eval() self.assertAllEqual([[3, 0], [2, 5]], confusion_matrix) desired_miou = np.mean([3. / 5., 5. / 7.]) self.assertAlmostEqual(desired_miou, miou.eval()) + @test_util.run_deprecated_v1 def testAllCorrect(self): predictions = array_ops.zeros([40]) labels = array_ops.zeros([40]) num_classes = 1 with self.cached_session() as sess: miou, update_op = metrics.mean_iou(labels, predictions, num_classes) - sess.run(variables.local_variables_initializer()) + self.evaluate(variables.local_variables_initializer()) self.assertEqual(40, update_op.eval()[0]) self.assertEqual(1.0, miou.eval()) + @test_util.run_deprecated_v1 def testAllWrong(self): predictions = array_ops.zeros([40]) labels = array_ops.ones([40]) num_classes = 2 with self.cached_session() as sess: miou, update_op = metrics.mean_iou(labels, predictions, num_classes) - sess.run(variables.local_variables_initializer()) + self.evaluate(variables.local_variables_initializer()) self.assertAllEqual([[0, 0], [40, 0]], update_op.eval()) self.assertEqual(0., miou.eval()) + @test_util.run_deprecated_v1 def testResultsWithSomeMissing(self): predictions = array_ops.concat( [ @@ -3640,11 +3829,12 @@ class MeanIOUTest(test.TestCase): with self.cached_session() as sess: miou, update_op = metrics.mean_iou( labels, predictions, num_classes, weights=weights) - sess.run(variables.local_variables_initializer()) + self.evaluate(variables.local_variables_initializer()) self.assertAllEqual([[2, 0], [2, 4]], update_op.eval()) desired_miou = np.mean([2. / 4., 4. / 6.]) self.assertAlmostEqual(desired_miou, miou.eval()) + @test_util.run_deprecated_v1 def testMissingClassInLabels(self): labels = constant_op.constant([ [[0, 0, 1, 1, 0, 0], @@ -3659,22 +3849,24 @@ class MeanIOUTest(test.TestCase): num_classes = 3 with self.cached_session() as sess: miou, update_op = metrics.mean_iou(labels, predictions, num_classes) - sess.run(variables.local_variables_initializer()) + self.evaluate(variables.local_variables_initializer()) self.assertAllEqual([[7, 4, 3], [3, 5, 2], [0, 0, 0]], update_op.eval()) self.assertAlmostEqual( 1 / 3 * (7 / (7 + 3 + 7) + 5 / (5 + 4 + 5) + 0 / (0 + 5 + 0)), miou.eval()) + @test_util.run_deprecated_v1 def testMissingClassOverallSmall(self): labels = constant_op.constant([0]) predictions = constant_op.constant([0]) num_classes = 2 with self.cached_session() as sess: miou, update_op = metrics.mean_iou(labels, predictions, num_classes) - sess.run(variables.local_variables_initializer()) + self.evaluate(variables.local_variables_initializer()) self.assertAllEqual([[1, 0], [0, 0]], update_op.eval()) self.assertAlmostEqual(1, miou.eval()) + @test_util.run_deprecated_v1 def testMissingClassOverallLarge(self): labels = constant_op.constant([ [[0, 0, 1, 1, 0, 0], @@ -3689,7 +3881,7 @@ class MeanIOUTest(test.TestCase): num_classes = 3 with self.cached_session() as sess: miou, update_op = metrics.mean_iou(labels, predictions, num_classes) - sess.run(variables.local_variables_initializer()) + self.evaluate(variables.local_variables_initializer()) self.assertAllEqual([[9, 5, 0], [3, 7, 0], [0, 0, 0]], update_op.eval()) self.assertAlmostEqual( 1 / 2 * (9 / (9 + 3 + 5) + 7 / (7 + 5 + 3)), miou.eval()) @@ -3701,6 +3893,7 @@ class MeanPerClassAccuracyTest(test.TestCase): np.random.seed(1) ops.reset_default_graph() + @test_util.run_deprecated_v1 def testVars(self): metrics.mean_per_class_accuracy( predictions=array_ops.ones([10, 1]), @@ -3709,6 +3902,7 @@ class MeanPerClassAccuracyTest(test.TestCase): _assert_metric_variables(self, ('mean_accuracy/count:0', 'mean_accuracy/total:0')) + @test_util.run_deprecated_v1 def testMetricsCollections(self): my_collection_name = '__metrics__' mean_accuracy, _ = metrics.mean_per_class_accuracy( @@ -3719,6 +3913,7 @@ class MeanPerClassAccuracyTest(test.TestCase): self.assertListEqual( ops.get_collection(my_collection_name), [mean_accuracy]) + @test_util.run_deprecated_v1 def testUpdatesCollection(self): my_collection_name = '__updates__' _, update_op = metrics.mean_per_class_accuracy( @@ -3728,12 +3923,14 @@ class MeanPerClassAccuracyTest(test.TestCase): updates_collections=[my_collection_name]) self.assertListEqual(ops.get_collection(my_collection_name), [update_op]) + @test_util.run_deprecated_v1 def testPredictionsAndLabelsOfDifferentSizeRaisesValueError(self): predictions = array_ops.ones([10, 3]) labels = array_ops.ones([10, 4]) with self.assertRaises(ValueError): metrics.mean_per_class_accuracy(labels, predictions, num_classes=2) + @test_util.run_deprecated_v1 def testLabelsAndWeightsOfDifferentSizeRaisesValueError(self): predictions = array_ops.ones([10]) labels = array_ops.ones([10]) @@ -3742,6 +3939,7 @@ class MeanPerClassAccuracyTest(test.TestCase): metrics.mean_per_class_accuracy( labels, predictions, num_classes=2, weights=weights) + @test_util.run_deprecated_v1 def testValueTensorIsIdempotent(self): num_classes = 3 predictions = random_ops.random_uniform( @@ -3752,11 +3950,11 @@ class MeanPerClassAccuracyTest(test.TestCase): labels, predictions, num_classes=num_classes) with self.cached_session() as sess: - sess.run(variables.local_variables_initializer()) + self.evaluate(variables.local_variables_initializer()) # Run several updates. for _ in range(10): - sess.run(update_op) + self.evaluate(update_op) # Then verify idempotency. initial_mean_accuracy = mean_accuracy.eval() @@ -3788,12 +3986,13 @@ class MeanPerClassAccuracyTest(test.TestCase): mean_accuracy, update_op = metrics.mean_per_class_accuracy( labels, predictions, num_classes) - sess.run(variables.local_variables_initializer()) + self.evaluate(variables.local_variables_initializer()) for _ in range(5): - sess.run(update_op) + self.evaluate(update_op) desired_output = np.mean([1.0, 1.0 / 3.0, 0.0]) self.assertAlmostEqual(desired_output, mean_accuracy.eval()) + @test_util.run_deprecated_v1 def testMultipleUpdatesWithWeights(self): num_classes = 2 with self.cached_session() as sess: @@ -3835,10 +4034,11 @@ class MeanPerClassAccuracyTest(test.TestCase): variables.local_variables_initializer().run() for _ in range(6): - sess.run(update_op) + self.evaluate(update_op) desired_output = np.mean([2.0 / 2.0, 0.5 / 1.5]) self.assertAlmostEqual(desired_output, mean_accuracy.eval()) + @test_util.run_deprecated_v1 def testMultipleUpdatesWithMissingClass(self): # Test the case where there are no predicions and labels for # one class, and thus there is one row and one column with @@ -3870,12 +4070,13 @@ class MeanPerClassAccuracyTest(test.TestCase): mean_accuracy, update_op = metrics.mean_per_class_accuracy( labels, predictions, num_classes) - sess.run(variables.local_variables_initializer()) + self.evaluate(variables.local_variables_initializer()) for _ in range(5): - sess.run(update_op) + self.evaluate(update_op) desired_output = np.mean([1.0 / 2.0, 2.0 / 3.0, 0.]) self.assertAlmostEqual(desired_output, mean_accuracy.eval()) + @test_util.run_deprecated_v1 def testAllCorrect(self): predictions = array_ops.zeros([40]) labels = array_ops.zeros([40]) @@ -3883,10 +4084,11 @@ class MeanPerClassAccuracyTest(test.TestCase): with self.cached_session() as sess: mean_accuracy, update_op = metrics.mean_per_class_accuracy( labels, predictions, num_classes) - sess.run(variables.local_variables_initializer()) + self.evaluate(variables.local_variables_initializer()) self.assertEqual(1.0, update_op.eval()[0]) self.assertEqual(1.0, mean_accuracy.eval()) + @test_util.run_deprecated_v1 def testAllWrong(self): predictions = array_ops.zeros([40]) labels = array_ops.ones([40]) @@ -3894,10 +4096,11 @@ class MeanPerClassAccuracyTest(test.TestCase): with self.cached_session() as sess: mean_accuracy, update_op = metrics.mean_per_class_accuracy( labels, predictions, num_classes) - sess.run(variables.local_variables_initializer()) + self.evaluate(variables.local_variables_initializer()) self.assertAllEqual([0.0, 0.0], update_op.eval()) self.assertEqual(0., mean_accuracy.eval()) + @test_util.run_deprecated_v1 def testResultsWithSomeMissing(self): predictions = array_ops.concat([ constant_op.constant(0, shape=[5]), constant_op.constant(1, shape=[5]) @@ -3913,7 +4116,7 @@ class MeanPerClassAccuracyTest(test.TestCase): with self.cached_session() as sess: mean_accuracy, update_op = metrics.mean_per_class_accuracy( labels, predictions, num_classes, weights=weights) - sess.run(variables.local_variables_initializer()) + self.evaluate(variables.local_variables_initializer()) desired_accuracy = np.array([2. / 2., 4. / 6.], dtype=np.float32) self.assertAllEqual(desired_accuracy, update_op.eval()) desired_mean_accuracy = np.mean(desired_accuracy) @@ -3926,12 +4129,14 @@ class FalseNegativesTest(test.TestCase): np.random.seed(1) ops.reset_default_graph() + @test_util.run_deprecated_v1 def testVars(self): metrics.false_negatives( labels=(0, 1, 0, 1), predictions=(0, 0, 1, 1)) _assert_metric_variables(self, ('false_negatives/count:0',)) + @test_util.run_deprecated_v1 def testUnweighted(self): labels = constant_op.constant(((0, 1, 0, 1, 0), (0, 0, 1, 1, 1), @@ -3945,11 +4150,12 @@ class FalseNegativesTest(test.TestCase): labels=labels, predictions=predictions) with self.cached_session() as sess: - sess.run(variables.local_variables_initializer()) + self.evaluate(variables.local_variables_initializer()) self.assertAllClose(0., tn.eval()) self.assertAllClose(3., tn_update_op.eval()) self.assertAllClose(3., tn.eval()) + @test_util.run_deprecated_v1 def testWeighted(self): labels = constant_op.constant(((0, 1, 0, 1, 0), (0, 0, 1, 1, 1), @@ -3964,7 +4170,7 @@ class FalseNegativesTest(test.TestCase): labels=labels, predictions=predictions, weights=weights) with self.cached_session() as sess: - sess.run(variables.local_variables_initializer()) + self.evaluate(variables.local_variables_initializer()) self.assertAllClose(0., tn.eval()) self.assertAllClose(5., tn_update_op.eval()) self.assertAllClose(5., tn.eval()) @@ -3976,6 +4182,7 @@ class FalseNegativesAtThresholdsTest(test.TestCase): np.random.seed(1) ops.reset_default_graph() + @test_util.run_deprecated_v1 def testVars(self): metrics.false_negatives_at_thresholds( predictions=array_ops.ones((10, 1)), @@ -3983,6 +4190,7 @@ class FalseNegativesAtThresholdsTest(test.TestCase): thresholds=[0.15, 0.5, 0.85]) _assert_metric_variables(self, ('false_negatives/false_negatives:0',)) + @test_util.run_deprecated_v1 def testUnweighted(self): predictions = constant_op.constant(((0.9, 0.2, 0.8, 0.1), (0.2, 0.9, 0.7, 0.6), @@ -3994,11 +4202,12 @@ class FalseNegativesAtThresholdsTest(test.TestCase): predictions=predictions, labels=labels, thresholds=[0.15, 0.5, 0.85]) with self.cached_session() as sess: - sess.run(variables.local_variables_initializer()) + self.evaluate(variables.local_variables_initializer()) self.assertAllEqual((0, 0, 0), fn.eval()) self.assertAllEqual((0, 2, 3), fn_update_op.eval()) self.assertAllEqual((0, 2, 3), fn.eval()) + @test_util.run_deprecated_v1 def testWeighted(self): predictions = constant_op.constant(((0.9, 0.2, 0.8, 0.1), (0.2, 0.9, 0.7, 0.6), @@ -4013,7 +4222,7 @@ class FalseNegativesAtThresholdsTest(test.TestCase): thresholds=[0.15, 0.5, 0.85]) with self.cached_session() as sess: - sess.run(variables.local_variables_initializer()) + self.evaluate(variables.local_variables_initializer()) self.assertAllEqual((0.0, 0.0, 0.0), fn.eval()) self.assertAllEqual((0.0, 8.0, 11.0), fn_update_op.eval()) self.assertAllEqual((0.0, 8.0, 11.0), fn.eval()) @@ -4025,12 +4234,14 @@ class FalsePositivesTest(test.TestCase): np.random.seed(1) ops.reset_default_graph() + @test_util.run_deprecated_v1 def testVars(self): metrics.false_positives( labels=(0, 1, 0, 1), predictions=(0, 0, 1, 1)) _assert_metric_variables(self, ('false_positives/count:0',)) + @test_util.run_deprecated_v1 def testUnweighted(self): labels = constant_op.constant(((0, 1, 0, 1, 0), (0, 0, 1, 1, 1), @@ -4044,11 +4255,12 @@ class FalsePositivesTest(test.TestCase): labels=labels, predictions=predictions) with self.cached_session() as sess: - sess.run(variables.local_variables_initializer()) + self.evaluate(variables.local_variables_initializer()) self.assertAllClose(0., tn.eval()) self.assertAllClose(7., tn_update_op.eval()) self.assertAllClose(7., tn.eval()) + @test_util.run_deprecated_v1 def testWeighted(self): labels = constant_op.constant(((0, 1, 0, 1, 0), (0, 0, 1, 1, 1), @@ -4063,7 +4275,7 @@ class FalsePositivesTest(test.TestCase): labels=labels, predictions=predictions, weights=weights) with self.cached_session() as sess: - sess.run(variables.local_variables_initializer()) + self.evaluate(variables.local_variables_initializer()) self.assertAllClose(0., tn.eval()) self.assertAllClose(14., tn_update_op.eval()) self.assertAllClose(14., tn.eval()) @@ -4075,6 +4287,7 @@ class FalsePositivesAtThresholdsTest(test.TestCase): np.random.seed(1) ops.reset_default_graph() + @test_util.run_deprecated_v1 def testVars(self): metrics.false_positives_at_thresholds( predictions=array_ops.ones((10, 1)), @@ -4082,6 +4295,7 @@ class FalsePositivesAtThresholdsTest(test.TestCase): thresholds=[0.15, 0.5, 0.85]) _assert_metric_variables(self, ('false_positives/false_positives:0',)) + @test_util.run_deprecated_v1 def testUnweighted(self): predictions = constant_op.constant(((0.9, 0.2, 0.8, 0.1), (0.2, 0.9, 0.7, 0.6), @@ -4093,11 +4307,12 @@ class FalsePositivesAtThresholdsTest(test.TestCase): predictions=predictions, labels=labels, thresholds=[0.15, 0.5, 0.85]) with self.cached_session() as sess: - sess.run(variables.local_variables_initializer()) + self.evaluate(variables.local_variables_initializer()) self.assertAllEqual((0, 0, 0), fp.eval()) self.assertAllEqual((7, 4, 2), fp_update_op.eval()) self.assertAllEqual((7, 4, 2), fp.eval()) + @test_util.run_deprecated_v1 def testWeighted(self): predictions = constant_op.constant(((0.9, 0.2, 0.8, 0.1), (0.2, 0.9, 0.7, 0.6), @@ -4114,7 +4329,7 @@ class FalsePositivesAtThresholdsTest(test.TestCase): thresholds=[0.15, 0.5, 0.85]) with self.cached_session() as sess: - sess.run(variables.local_variables_initializer()) + self.evaluate(variables.local_variables_initializer()) self.assertAllEqual((0.0, 0.0, 0.0), fp.eval()) self.assertAllEqual((125.0, 42.0, 12.0), fp_update_op.eval()) self.assertAllEqual((125.0, 42.0, 12.0), fp.eval()) @@ -4126,12 +4341,14 @@ class TrueNegativesTest(test.TestCase): np.random.seed(1) ops.reset_default_graph() + @test_util.run_deprecated_v1 def testVars(self): metrics.true_negatives( labels=(0, 1, 0, 1), predictions=(0, 0, 1, 1)) _assert_metric_variables(self, ('true_negatives/count:0',)) + @test_util.run_deprecated_v1 def testUnweighted(self): labels = constant_op.constant(((0, 1, 0, 1, 0), (0, 0, 1, 1, 1), @@ -4145,11 +4362,12 @@ class TrueNegativesTest(test.TestCase): labels=labels, predictions=predictions) with self.cached_session() as sess: - sess.run(variables.local_variables_initializer()) + self.evaluate(variables.local_variables_initializer()) self.assertAllClose(0., tn.eval()) self.assertAllClose(3., tn_update_op.eval()) self.assertAllClose(3., tn.eval()) + @test_util.run_deprecated_v1 def testWeighted(self): labels = constant_op.constant(((0, 1, 0, 1, 0), (0, 0, 1, 1, 1), @@ -4164,7 +4382,7 @@ class TrueNegativesTest(test.TestCase): labels=labels, predictions=predictions, weights=weights) with self.cached_session() as sess: - sess.run(variables.local_variables_initializer()) + self.evaluate(variables.local_variables_initializer()) self.assertAllClose(0., tn.eval()) self.assertAllClose(4., tn_update_op.eval()) self.assertAllClose(4., tn.eval()) @@ -4176,6 +4394,7 @@ class TrueNegativesAtThresholdsTest(test.TestCase): np.random.seed(1) ops.reset_default_graph() + @test_util.run_deprecated_v1 def testVars(self): metrics.true_negatives_at_thresholds( predictions=array_ops.ones((10, 1)), @@ -4183,6 +4402,7 @@ class TrueNegativesAtThresholdsTest(test.TestCase): thresholds=[0.15, 0.5, 0.85]) _assert_metric_variables(self, ('true_negatives/true_negatives:0',)) + @test_util.run_deprecated_v1 def testUnweighted(self): predictions = constant_op.constant(((0.9, 0.2, 0.8, 0.1), (0.2, 0.9, 0.7, 0.6), @@ -4194,11 +4414,12 @@ class TrueNegativesAtThresholdsTest(test.TestCase): predictions=predictions, labels=labels, thresholds=[0.15, 0.5, 0.85]) with self.cached_session() as sess: - sess.run(variables.local_variables_initializer()) + self.evaluate(variables.local_variables_initializer()) self.assertAllEqual((0, 0, 0), tn.eval()) self.assertAllEqual((2, 5, 7), tn_update_op.eval()) self.assertAllEqual((2, 5, 7), tn.eval()) + @test_util.run_deprecated_v1 def testWeighted(self): predictions = constant_op.constant(((0.9, 0.2, 0.8, 0.1), (0.2, 0.9, 0.7, 0.6), @@ -4213,7 +4434,7 @@ class TrueNegativesAtThresholdsTest(test.TestCase): thresholds=[0.15, 0.5, 0.85]) with self.cached_session() as sess: - sess.run(variables.local_variables_initializer()) + self.evaluate(variables.local_variables_initializer()) self.assertAllEqual((0.0, 0.0, 0.0), tn.eval()) self.assertAllEqual((5.0, 15.0, 23.0), tn_update_op.eval()) self.assertAllEqual((5.0, 15.0, 23.0), tn.eval()) @@ -4225,12 +4446,14 @@ class TruePositivesTest(test.TestCase): np.random.seed(1) ops.reset_default_graph() + @test_util.run_deprecated_v1 def testVars(self): metrics.true_positives( labels=(0, 1, 0, 1), predictions=(0, 0, 1, 1)) _assert_metric_variables(self, ('true_positives/count:0',)) + @test_util.run_deprecated_v1 def testUnweighted(self): labels = constant_op.constant(((0, 1, 0, 1, 0), (0, 0, 1, 1, 1), @@ -4244,11 +4467,12 @@ class TruePositivesTest(test.TestCase): labels=labels, predictions=predictions) with self.cached_session() as sess: - sess.run(variables.local_variables_initializer()) + self.evaluate(variables.local_variables_initializer()) self.assertAllClose(0., tn.eval()) self.assertAllClose(7., tn_update_op.eval()) self.assertAllClose(7., tn.eval()) + @test_util.run_deprecated_v1 def testWeighted(self): labels = constant_op.constant(((0, 1, 0, 1, 0), (0, 0, 1, 1, 1), @@ -4263,7 +4487,7 @@ class TruePositivesTest(test.TestCase): labels=labels, predictions=predictions, weights=weights) with self.cached_session() as sess: - sess.run(variables.local_variables_initializer()) + self.evaluate(variables.local_variables_initializer()) self.assertAllClose(0., tn.eval()) self.assertAllClose(12., tn_update_op.eval()) self.assertAllClose(12., tn.eval()) @@ -4275,6 +4499,7 @@ class TruePositivesAtThresholdsTest(test.TestCase): np.random.seed(1) ops.reset_default_graph() + @test_util.run_deprecated_v1 def testVars(self): metrics.true_positives_at_thresholds( predictions=array_ops.ones((10, 1)), @@ -4282,6 +4507,7 @@ class TruePositivesAtThresholdsTest(test.TestCase): thresholds=[0.15, 0.5, 0.85]) _assert_metric_variables(self, ('true_positives/true_positives:0',)) + @test_util.run_deprecated_v1 def testUnweighted(self): predictions = constant_op.constant(((0.9, 0.2, 0.8, 0.1), (0.2, 0.9, 0.7, 0.6), @@ -4293,11 +4519,12 @@ class TruePositivesAtThresholdsTest(test.TestCase): predictions=predictions, labels=labels, thresholds=[0.15, 0.5, 0.85]) with self.cached_session() as sess: - sess.run(variables.local_variables_initializer()) + self.evaluate(variables.local_variables_initializer()) self.assertAllEqual((0, 0, 0), tp.eval()) self.assertAllEqual((3, 1, 0), tp_update_op.eval()) self.assertAllEqual((3, 1, 0), tp.eval()) + @test_util.run_deprecated_v1 def testWeighted(self): predictions = constant_op.constant(((0.9, 0.2, 0.8, 0.1), (0.2, 0.9, 0.7, 0.6), @@ -4310,7 +4537,7 @@ class TruePositivesAtThresholdsTest(test.TestCase): thresholds=[0.15, 0.5, 0.85]) with self.cached_session() as sess: - sess.run(variables.local_variables_initializer()) + self.evaluate(variables.local_variables_initializer()) self.assertAllEqual((0.0, 0.0, 0.0), tp.eval()) self.assertAllEqual((111.0, 37.0, 0.0), tp_update_op.eval()) self.assertAllEqual((111.0, 37.0, 0.0), tp.eval()) diff --git a/tensorflow/python/kernel_tests/morphological_ops_test.py b/tensorflow/python/kernel_tests/morphological_ops_test.py index 6d601554b80..f54aaf30d0a 100644 --- a/tensorflow/python/kernel_tests/morphological_ops_test.py +++ b/tensorflow/python/kernel_tests/morphological_ops_test.py @@ -21,6 +21,7 @@ from __future__ import print_function import numpy as np from tensorflow.python.framework import constant_op +from tensorflow.python.framework import test_util from tensorflow.python.ops import gradient_checker from tensorflow.python.ops import nn_ops import tensorflow.python.ops.nn_grad # pylint: disable=unused-import @@ -52,7 +53,7 @@ class DilationTest(test.TestCase): rates=rates, padding=padding, name="dilation2d") - self.assertAllClose(out, out_tensor.eval()) + self.assertAllClose(out, self.evaluate(out_tensor)) def _testDilationValidPadding(self, use_gpu): # [1, 2, 2, 1] @@ -216,7 +217,7 @@ class DilationTest(test.TestCase): rates=rates, padding=padding, name="dilation2d") - out_shape = out_tensor.eval().shape + out_shape = self.evaluate(out_tensor).shape # Small delta is necessary for argmax to remain the same. err = gradient_checker.compute_gradient_error( @@ -291,6 +292,7 @@ class DilationTest(test.TestCase): padding="SAME", use_gpu=use_gpu) + @test_util.run_deprecated_v1 def testDilationGrad(self): for use_gpu in True, False: self._testDilationGradValidPadding_1x1x1(use_gpu) @@ -327,7 +329,7 @@ class ErosionTest(test.TestCase): rates=rates, padding=padding, name="erosion2d") - self.assertAllClose(out, out_tensor.eval()) + self.assertAllClose(out, self.evaluate(out_tensor)) def _testErosionValidPadding(self, use_gpu): # [1, 2, 2, 1] @@ -491,7 +493,7 @@ class ErosionTest(test.TestCase): rates=rates, padding=padding, name="erosion2d") - out_shape = out_tensor.eval().shape + out_shape = self.evaluate(out_tensor).shape # Small delta is necessary for argmax to remain the same. err = gradient_checker.compute_gradient_error( @@ -566,6 +568,7 @@ class ErosionTest(test.TestCase): padding="SAME", use_gpu=use_gpu) + @test_util.run_deprecated_v1 def testErosionGrad(self): for use_gpu in True, False: self._testErosionGradValidPadding_1x1x1(use_gpu) diff --git a/tensorflow/python/kernel_tests/neon_depthwise_conv_op_test.py b/tensorflow/python/kernel_tests/neon_depthwise_conv_op_test.py index 15e38265421..380d2860da4 100644 --- a/tensorflow/python/kernel_tests/neon_depthwise_conv_op_test.py +++ b/tensorflow/python/kernel_tests/neon_depthwise_conv_op_test.py @@ -21,6 +21,7 @@ from __future__ import print_function import numpy as np from tensorflow.python.framework import constant_op +from tensorflow.python.framework import test_util from tensorflow.python.ops import array_ops from tensorflow.python.ops import nn_impl from tensorflow.python.ops import nn_ops @@ -142,8 +143,8 @@ class DepthwiseConv2DTest(test.TestCase): conv_interface = nn_impl.depthwise_conv2d( t1, t2, strides=[1, stride, stride, 1], padding=padding) - native_result = sess.run(conv_native) - interface_result = sess.run(conv_interface) + native_result = self.evaluate(conv_native) + interface_result = self.evaluate(conv_interface) print("depthwise conv_2d: ", tensor_in_sizes, "*", filter_in_sizes, ", stride:", stride, ", padding: ", padding, ", max diff: ", @@ -153,6 +154,7 @@ class DepthwiseConv2DTest(test.TestCase): self.assertShapeEqual(native_result, conv_native) self.assertShapeEqual(native_result, conv_interface) + @test_util.run_deprecated_v1 def testDepthwiseConv2D(self): for index, (input_size, filter_size, _, stride, padding) in enumerate(ConfigsToTest()): @@ -211,11 +213,12 @@ class DepthwiseConv2DTest(test.TestCase): t2 = constant_op.constant(x2, shape=filter_in_sizes) conv = nn_ops.depthwise_conv2d_native( t1, t2, strides=[1, stride, stride, 1], padding=padding) - value = sess.run(conv) + value = self.evaluate(conv) print("value = ", value) self.assertAllClose(expected, np.ravel(value), 1e-5) self.assertShapeEqual(value, conv) + @test_util.run_deprecated_v1 def testConv2D2x2Filter(self): # The inputs look like this (it's a 3 x 2 matrix, each of depth 2): # diff --git a/tensorflow/python/kernel_tests/norm_op_test.py b/tensorflow/python/kernel_tests/norm_op_test.py index e202b6e8a43..5ff0c58bf1b 100644 --- a/tensorflow/python/kernel_tests/norm_op_test.py +++ b/tensorflow/python/kernel_tests/norm_op_test.py @@ -70,7 +70,7 @@ def _GetNormOpTest(dtype_, shape_, ord_, axis_, keep_dims_, use_static_shape_): tf_matrix = constant_op.constant(matrix) tf_norm = linalg_ops.norm( tf_matrix, ord=ord_, axis=axis_, keepdims=keep_dims_) - tf_norm_val = sess.run(tf_norm) + tf_norm_val = self.evaluate(tf_norm) else: tf_matrix = array_ops.placeholder(dtype_) tf_norm = linalg_ops.norm( diff --git a/tensorflow/python/kernel_tests/nth_element_op_test.py b/tensorflow/python/kernel_tests/nth_element_op_test.py index 338b6cec010..4be78b2d5ca 100644 --- a/tensorflow/python/kernel_tests/nth_element_op_test.py +++ b/tensorflow/python/kernel_tests/nth_element_op_test.py @@ -22,6 +22,7 @@ import numpy as np import tensorflow.python.ops.nn_grad # pylint: disable=unused-import from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops +from tensorflow.python.framework import test_util from tensorflow.python.ops import nn_ops from tensorflow.python.ops import array_ops from tensorflow.python.ops import gradients_impl @@ -35,7 +36,7 @@ class NthElementTest(test.TestCase): with self.cached_session(use_gpu=False) as sess: inputs_op = ops.convert_to_tensor(inputs, dtype=dtype) values_op = nn_ops.nth_element(inputs_op, n, reverse=reverse) - values = sess.run(values_op) + values = self.evaluate(values_op) self.assertShapeEqual(np_expected_values, values_op) self.assertAllClose(np_expected_values, values) @@ -111,17 +112,20 @@ class NthElementTest(test.TestCase): self._testEnumerateN([10, 10, 10]) self._testEnumerateN([10, 10, 10, 10]) + @test_util.run_deprecated_v1 def testInvalidInput(self): with self.assertRaisesRegexp(ValueError, "at least rank 1 but is rank 0"): nn_ops.nth_element(5, 0) + @test_util.run_deprecated_v1 def testInvalidInputAtEval(self): with self.session(use_gpu=False): v = array_ops.placeholder(dtype=dtypes.float32) with self.assertRaisesOpError("Input must be >= 1-D"): nn_ops.nth_element(v, 0).eval(feed_dict={v: 5.0}) + @test_util.run_deprecated_v1 def testInvalidN(self): with self.assertRaisesRegexp(ValueError, "non-negative but is -1"): @@ -130,6 +134,7 @@ class NthElementTest(test.TestCase): "scalar but has rank 1"): nn_ops.nth_element([5, 6, 3], [1]) + @test_util.run_deprecated_v1 def testInvalidNAtEval(self): inputs = [[0.1, 0.2], [0.3, 0.4]] with self.session(use_gpu=False): @@ -138,12 +143,14 @@ class NthElementTest(test.TestCase): with self.assertRaisesOpError("Need n >= 0, got -7"): values.eval(feed_dict={n: -7}) + @test_util.run_deprecated_v1 def testNTooLarge(self): inputs = [[0.1, 0.2], [0.3, 0.4]] with self.assertRaisesRegexp(ValueError, "must have last dimension > n = 2"): nn_ops.nth_element(inputs, 2) + @test_util.run_deprecated_v1 def testNTooLargeAtEval(self): inputs = [[0.1, 0.2], [0.3, 0.4]] with self.session(use_gpu=False): @@ -152,6 +159,7 @@ class NthElementTest(test.TestCase): with self.assertRaisesOpError(r"Input must have at least n\+1 columns"): values.eval(feed_dict={n: 2}) + @test_util.run_deprecated_v1 def testGradients(self): with self.session(use_gpu=False) as sess: inputs = array_ops.placeholder(dtypes.float32, shape=[3, 5]) diff --git a/tensorflow/python/kernel_tests/numerics_test.py b/tensorflow/python/kernel_tests/numerics_test.py index 5db591ed304..5751f3fe767 100644 --- a/tensorflow/python/kernel_tests/numerics_test.py +++ b/tensorflow/python/kernel_tests/numerics_test.py @@ -23,6 +23,7 @@ import numpy as np from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops +from tensorflow.python.framework import test_util from tensorflow.python.ops import array_ops from tensorflow.python.ops import control_flow_ops from tensorflow.python.ops import math_ops @@ -35,11 +36,11 @@ class VerifyTensorAllFiniteTest(test.TestCase): def testVerifyTensorAllFiniteSucceeds(self): x_shape = [5, 4] x = np.random.random_sample(x_shape).astype(np.float32) - with self.session(use_gpu=True): + with test_util.use_gpu(): t = constant_op.constant(x, shape=x_shape, dtype=dtypes.float32) t_verified = numerics.verify_tensor_all_finite(t, "Input is not a number.") - self.assertAllClose(x, t_verified.eval()) + self.assertAllClose(x, self.evaluate(t_verified)) def testVerifyTensorAllFiniteFails(self): x_shape = [5, 4] @@ -48,23 +49,24 @@ class VerifyTensorAllFiniteTest(test.TestCase): # Test NaN. x[0] = np.nan - with self.session(use_gpu=True): + with test_util.use_gpu(): with self.assertRaisesOpError(my_msg): t = constant_op.constant(x, shape=x_shape, dtype=dtypes.float32) t_verified = numerics.verify_tensor_all_finite(t, my_msg) - t_verified.eval() + self.evaluate(t_verified) # Test Inf. x[0] = np.inf - with self.session(use_gpu=True): + with test_util.use_gpu(): with self.assertRaisesOpError(my_msg): t = constant_op.constant(x, shape=x_shape, dtype=dtypes.float32) t_verified = numerics.verify_tensor_all_finite(t, my_msg) - t_verified.eval() + self.evaluate(t_verified) class NumericsTest(test.TestCase): + @test_util.run_deprecated_v1 def testInf(self): with self.session(graph=ops.Graph()): t1 = constant_op.constant(1.0) @@ -73,8 +75,9 @@ class NumericsTest(test.TestCase): check = numerics.add_check_numerics_ops() a = control_flow_ops.with_dependencies([check], a) with self.assertRaisesOpError("Inf"): - a.eval() + self.evaluate(a) + @test_util.run_deprecated_v1 def testNaN(self): with self.session(graph=ops.Graph()): t1 = constant_op.constant(0.0) @@ -83,8 +86,9 @@ class NumericsTest(test.TestCase): check = numerics.add_check_numerics_ops() a = control_flow_ops.with_dependencies([check], a) with self.assertRaisesOpError("NaN"): - a.eval() + self.evaluate(a) + @test_util.run_deprecated_v1 def testBoth(self): with self.session(graph=ops.Graph()): t1 = constant_op.constant([1.0, 0.0]) @@ -93,16 +97,17 @@ class NumericsTest(test.TestCase): check = numerics.add_check_numerics_ops() a = control_flow_ops.with_dependencies([check], a) with self.assertRaisesOpError("Inf and NaN"): - a.eval() + self.evaluate(a) def testPassThrough(self): with self.session(graph=ops.Graph()): t1 = constant_op.constant([1.0, 2.0, 3.0, 4.0, 5.0, 6.0], shape=[2, 3]) checked = array_ops.check_numerics(t1, message="pass through test") - value = checked.eval() + value = self.evaluate(checked) self.assertAllEqual(np.array([[1.0, 2.0, 3.0], [4.0, 5.0, 6.0]]), value) self.assertEqual([2, 3], checked.get_shape()) + @test_util.run_deprecated_v1 def testControlFlowCond(self): predicate = array_ops.placeholder(dtypes.bool, shape=[]) _ = control_flow_ops.cond(predicate, @@ -115,6 +120,7 @@ class NumericsTest(test.TestCase): r"or `tf.while_loop\(\)`\."): numerics.add_check_numerics_ops() + @test_util.run_deprecated_v1 def testControlFlowWhile(self): predicate = array_ops.placeholder(dtypes.bool, shape=[]) _ = control_flow_ops.while_loop(lambda _: predicate, diff --git a/tensorflow/python/kernel_tests/one_hot_op_test.py b/tensorflow/python/kernel_tests/one_hot_op_test.py index 377d545c9cd..856ba7bb7f3 100644 --- a/tensorflow/python/kernel_tests/one_hot_op_test.py +++ b/tensorflow/python/kernel_tests/one_hot_op_test.py @@ -41,12 +41,12 @@ class OneHotTest(test.TestCase): else: ans = array_ops.one_hot(**inputs) if expected_err_re is None: - tf_ans = ans.eval() + tf_ans = self.evaluate(ans) self.assertAllEqual(tf_ans, truth) self.assertEqual(tf_ans.shape, ans.get_shape()) else: with self.assertRaisesOpError(expected_err_re): - ans.eval() + self.evaluate(ans) def _testBothOneHot(self, truth, expected_err_re=None, raises=None, **inputs): self._testOneHot(truth, True, expected_err_re, raises, **inputs) diff --git a/tensorflow/python/kernel_tests/pad_op_test.py b/tensorflow/python/kernel_tests/pad_op_test.py index fc302c4141a..7b1b054ae06 100644 --- a/tensorflow/python/kernel_tests/pad_op_test.py +++ b/tensorflow/python/kernel_tests/pad_op_test.py @@ -23,6 +23,7 @@ import numpy as np from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops +from tensorflow.python.framework import test_util from tensorflow.python.ops import array_ops from tensorflow.python.ops import gradient_checker from tensorflow.python.platform import test @@ -88,7 +89,7 @@ class PadOpTest(test.TestCase): with self.cached_session(use_gpu=True): tf_val = array_ops.pad(np_inputs, paddings, mode=mode, constant_values=constant_values) - out = tf_val.eval() + out = self.evaluate(tf_val) self.assertAllEqual(np_val, out) self.assertShapeEqual(np_val, tf_val) @@ -116,6 +117,7 @@ class PadOpTest(test.TestCase): self._testGradient(np_inputs, paddings, mode=mode, constant_values=constant_values) + @test_util.run_deprecated_v1 def testInputDims(self): with self.session(use_gpu=True): with self.assertRaises(ValueError): @@ -124,6 +126,7 @@ class PadOpTest(test.TestCase): array_ops.reshape( [1, 2], shape=[1, 2])) + @test_util.run_deprecated_v1 def testPaddingsDim(self): with self.session(use_gpu=True): with self.assertRaises(ValueError): @@ -132,6 +135,7 @@ class PadOpTest(test.TestCase): array_ops.reshape( [1, 2], shape=[2])) + @test_util.run_deprecated_v1 def testPaddingsDim2(self): with self.session(use_gpu=True): with self.assertRaises(ValueError): @@ -140,6 +144,7 @@ class PadOpTest(test.TestCase): array_ops.reshape( [1, 2], shape=[2, 1])) + @test_util.run_deprecated_v1 def testPaddingsDim3(self): with self.session(use_gpu=True): with self.assertRaises(ValueError): @@ -148,6 +153,7 @@ class PadOpTest(test.TestCase): array_ops.reshape( [1, 2], shape=[1, 2])) + @test_util.run_deprecated_v1 def testPaddingsDim4(self): with self.session(use_gpu=True): with self.assertRaises(ValueError): @@ -156,6 +162,7 @@ class PadOpTest(test.TestCase): array_ops.reshape( [1, 2, 3, 4, 5, 6], shape=[3, 2])) + @test_util.run_deprecated_v1 def testPaddingsNonNegative(self): with self.session(use_gpu=True): with self.assertRaisesRegexp(ValueError, "must be non-negative"): @@ -164,6 +171,7 @@ class PadOpTest(test.TestCase): constant_op.constant( [-1, 0], shape=[1, 2])) + @test_util.run_deprecated_v1 def testPaddingsNonNegative2(self): with self.session(use_gpu=True): with self.assertRaisesRegexp(ValueError, "must be non-negative"): @@ -208,7 +216,7 @@ class PadOpTest(test.TestCase): constant_op.constant(paddings, padding_dtype), mode=mode, constant_values=0) - out = tf_val.eval() + out = self.evaluate(tf_val) self.assertAllEqual(np_val, out) self.assertShapeEqual(np_val, tf_val) @@ -223,6 +231,7 @@ class PadOpTest(test.TestCase): np.random.randint(-100, 100, (4, 2, 1, 3)).astype(t), [[0, 0], [0, 0], [0, 0], [0, 0]], -123) + @test_util.run_deprecated_v1 def testFloatTypes(self): for t in [np.float32, np.float64]: self._testAll(np.random.rand(2, 5).astype(t), [[1, 0], [2, 0]], 0.0) @@ -250,17 +259,18 @@ class PadOpTest(test.TestCase): symmetric = array_ops.pad(x, [[1, 0], [0, 1]], mode="SYMMETRIC", constant_values="PAD") with self.session(use_gpu=True): - self.assertAllEqual([[b"PAD", b"PAD", b"PAD"], - [b"Hello", b"World", b"PAD"], - [b"Goodnight", b"Moon", b"PAD"]], constant.eval()) + self.assertAllEqual( + [[b"PAD", b"PAD", b"PAD"], [b"Hello", b"World", b"PAD"], + [b"Goodnight", b"Moon", b"PAD"]], self.evaluate(constant)) self.assertAllEqual([[b"Goodnight", b"Moon", b"Goodnight"], [b"Hello", b"World", b"Hello"], [b"Goodnight", b"Moon", b"Goodnight"]], - reflect.eval()) - self.assertAllEqual([[b"Hello", b"World", b"World"], - [b"Hello", b"World", b"World"], - [b"Goodnight", b"Moon", b"Moon"]], symmetric.eval()) + self.evaluate(reflect)) + self.assertAllEqual( + [[b"Hello", b"World", b"World"], [b"Hello", b"World", b"World"], + [b"Goodnight", b"Moon", b"Moon"]], self.evaluate(symmetric)) + @test_util.run_deprecated_v1 def testShapeFunctionEdgeCases(self): # Unknown paddings shape. inp = constant_op.constant(0.0, shape=[4, 4, 4, 4]) @@ -277,6 +287,7 @@ class PadOpTest(test.TestCase): padded = array_ops.pad(inp, array_ops.placeholder(dtypes.int32)) self.assertAllEqual(None, padded.get_shape().ndims) + @test_util.run_deprecated_v1 def testPartialShapeInformation(self): unknown = array_ops.placeholder(dtypes.int32) @@ -327,7 +338,7 @@ class PadOpTest(test.TestCase): inp = np.asarray(7) with self.session(use_gpu=True): tf_val = array_ops.pad(inp, paddings) - out = tf_val.eval() + out = self.evaluate(tf_val) self.assertAllEqual(inp, out) self.assertShapeEqual(inp, tf_val) @@ -337,10 +348,11 @@ class PadOpTest(test.TestCase): inp = np.asarray(7) with self.cached_session(use_gpu=True): tf_val = array_ops.pad(inp, constant_op.constant(paddings, dtype=dtype)) - out = tf_val.eval() + out = self.evaluate(tf_val) self.assertAllEqual(inp, out) self.assertShapeEqual(inp, tf_val) + @test_util.run_deprecated_v1 def testCollapseAdjacentNonPaddedDimensions(self): # pyformat: disable paddings_values = [[[0, 0], [0, 0], [0, 0], [0, 1]], @@ -361,11 +373,12 @@ class PadOpTest(test.TestCase): [paddings_value[i][0] + inp.shape.dims[i].value for i in range(4)], [-1, -1, -1, -1]) with self.cached_session(use_gpu=True): - self.assertAllEqual(inp.eval(), middle.eval()) + self.assertAllEqual(inp.eval(), self.evaluate(middle)) self.assertAllEqual( - np.zeros([row[0] for row in paddings_value]), left.eval()) + np.zeros([row[0] for row in paddings_value]), self.evaluate(left)) self.assertAllEqual( - np.zeros([row[1] for row in paddings_value]), right.eval()) + np.zeros([row[1] for row in paddings_value]), + self.evaluate(right)) if __name__ == "__main__": diff --git a/tensorflow/python/kernel_tests/padding_fifo_queue_test.py b/tensorflow/python/kernel_tests/padding_fifo_queue_test.py index 95f3dcceeaa..b4818360d57 100644 --- a/tensorflow/python/kernel_tests/padding_fifo_queue_test.py +++ b/tensorflow/python/kernel_tests/padding_fifo_queue_test.py @@ -126,7 +126,7 @@ class PaddingFIFOQueueTest(test.TestCase): # Run one producer thread for each element in elems. def enqueue(enqueue_op): - sess.run(enqueue_op) + self.evaluate(enqueue_op) threads = [ self.checkedThread( @@ -158,7 +158,7 @@ class PaddingFIFOQueueTest(test.TestCase): results = [] def dequeue(): - results.append(sess.run(dequeued_t)) + results.append(self.evaluate(dequeued_t)) threads = [self.checkedThread(target=dequeue) for _ in enqueue_ops] for thread in threads: @@ -178,7 +178,7 @@ class PaddingFIFOQueueTest(test.TestCase): enqueue_op.run() for i in xrange(len(elems)): - vals = dequeued_t.eval() + vals = self.evaluate(dequeued_t) self.assertEqual([elems[i]], vals) def testEnqueueAndBlockingDequeue(self): @@ -193,13 +193,13 @@ class PaddingFIFOQueueTest(test.TestCase): # TODO(mrry): Figure out how to do this without sleeping. time.sleep(0.1) for enqueue_op in enqueue_ops: - sess.run(enqueue_op) + self.evaluate(enqueue_op) results = [] def dequeue(): for _ in xrange(len(elems)): - results.append(sess.run(dequeued_t)) + results.append(self.evaluate(dequeued_t)) enqueue_thread = self.checkedThread(target=enqueue) dequeue_thread = self.checkedThread(target=dequeue) @@ -224,7 +224,7 @@ class PaddingFIFOQueueTest(test.TestCase): enqueue_op.run() for i in xrange(len(elems)): - x_val, y_val = sess.run(dequeued_t) + x_val, y_val = self.evaluate(dequeued_t) x, y = elems[i] self.assertEqual([x], x_val) self.assertEqual([y], y_val) @@ -243,9 +243,9 @@ class PaddingFIFOQueueTest(test.TestCase): self.assertEqual([], size.get_shape()) enqueue_op.run() - self.assertEqual(1, size.eval()) + self.assertEqual(1, self.evaluate(size)) dequeued_t.op.run() - self.assertEqual(0, size.eval()) + self.assertEqual(0, self.evaluate(size)) def testEnqueueMany(self): with self.cached_session(): @@ -257,7 +257,7 @@ class PaddingFIFOQueueTest(test.TestCase): enqueue_op.run() for i in range(8): - vals = dequeued_t.eval() + vals = self.evaluate(dequeued_t) self.assertEqual([elems[i % 4]], vals) def testEmptyEnqueueMany(self): @@ -269,9 +269,9 @@ class PaddingFIFOQueueTest(test.TestCase): enqueue_op = q.enqueue_many((empty_t,)) size_t = q.size() - self.assertEqual([0], size_t.eval()) + self.assertEqual([0], self.evaluate(size_t)) enqueue_op.run() - self.assertEqual([0], size_t.eval()) + self.assertEqual([0], self.evaluate(size_t)) def testEmptyDequeueMany(self): with self.cached_session(): @@ -279,9 +279,9 @@ class PaddingFIFOQueueTest(test.TestCase): enqueue_op = q.enqueue((10.0,)) dequeued_t = q.dequeue_many(0) - self.assertEqual([], dequeued_t.eval().tolist()) + self.assertEqual([], self.evaluate(dequeued_t).tolist()) enqueue_op.run() - self.assertEqual([], dequeued_t.eval().tolist()) + self.assertEqual([], self.evaluate(dequeued_t).tolist()) def testEmptyDequeueManyWithDynamicShape(self): with self.cached_session(): @@ -290,9 +290,9 @@ class PaddingFIFOQueueTest(test.TestCase): enqueue_op = q.enqueue(([10.0],)) dequeued_t = q.dequeue_many(0) - self.assertEqual([], dequeued_t.eval().tolist()) + self.assertEqual([], self.evaluate(dequeued_t).tolist()) enqueue_op.run() - self.assertEqual([], dequeued_t.eval().tolist()) + self.assertEqual([], self.evaluate(dequeued_t).tolist()) def testEmptyDequeueUpToWithDynamicShape(self): with self.cached_session(): @@ -301,9 +301,9 @@ class PaddingFIFOQueueTest(test.TestCase): enqueue_op = q.enqueue(([10.0],)) dequeued_t = q.dequeue_up_to(0) - self.assertEqual([], dequeued_t.eval().tolist()) + self.assertEqual([], self.evaluate(dequeued_t).tolist()) enqueue_op.run() - self.assertEqual([], dequeued_t.eval().tolist()) + self.assertEqual([], self.evaluate(dequeued_t).tolist()) def testConstructPaddingFIFOQueueWithNoShape(self): with self.cached_session(): @@ -327,7 +327,7 @@ class PaddingFIFOQueueTest(test.TestCase): enqueue_op.run() for i in range(8): - float_val, int_val = sess.run(dequeued_t) + float_val, int_val = self.evaluate(dequeued_t) self.assertEqual(float_elems[i % 4], float_val) self.assertAllEqual(int_elems[i % 4], int_val) @@ -344,7 +344,7 @@ class PaddingFIFOQueueTest(test.TestCase): enqueue_op.run() for i in range(8): - float_val, int_val = sess.run(dequeued_t) + float_val, int_val = self.evaluate(dequeued_t) self.assertEqual(float_elems[i % 4], float_val) self.assertAllEqual(int_elems[i % 4], int_val) @@ -357,8 +357,8 @@ class PaddingFIFOQueueTest(test.TestCase): enqueue_op.run() - self.assertAllEqual(elems[0:4], dequeued_t.eval()) - self.assertAllEqual(elems[4:8], dequeued_t.eval()) + self.assertAllEqual(elems[0:4], self.evaluate(dequeued_t)) + self.assertAllEqual(elems[4:8], self.evaluate(dequeued_t)) def testDequeueUpToNoBlocking(self): with self.cached_session(): @@ -369,8 +369,8 @@ class PaddingFIFOQueueTest(test.TestCase): enqueue_op.run() - self.assertAllEqual(elems[0:4], dequeued_t.eval()) - self.assertAllEqual(elems[4:8], dequeued_t.eval()) + self.assertAllEqual(elems[0:4], self.evaluate(dequeued_t)) + self.assertAllEqual(elems[4:8], self.evaluate(dequeued_t)) def testMultiDequeueMany(self): with self.cached_session() as sess: @@ -387,17 +387,17 @@ class PaddingFIFOQueueTest(test.TestCase): enqueue_op.run() - float_val, int_val = sess.run(dequeued_t) + float_val, int_val = self.evaluate(dequeued_t) self.assertAllEqual(float_elems[0:4], float_val) self.assertAllEqual(int_elems[0:4], int_val) self.assertEqual(float_val.shape, dequeued_t[0].get_shape()) self.assertEqual(int_val.shape, dequeued_t[1].get_shape()) - float_val, int_val = sess.run(dequeued_t) + float_val, int_val = self.evaluate(dequeued_t) self.assertAllEqual(float_elems[4:8], float_val) self.assertAllEqual(int_elems[4:8], int_val) - float_val, int_val = sess.run(dequeued_single_t) + float_val, int_val = self.evaluate(dequeued_single_t) self.assertAllEqual(float_elems[8], float_val) self.assertAllEqual(int_elems[8], int_val) self.assertEqual(float_val.shape, dequeued_single_t[0].get_shape()) @@ -418,7 +418,7 @@ class PaddingFIFOQueueTest(test.TestCase): enqueue_op.run() - float_val, int_val = sess.run(dequeued_t) + float_val, int_val = self.evaluate(dequeued_t) self.assertAllEqual(float_elems[0:4], float_val) self.assertAllEqual(int_elems[0:4], int_val) self.assertTrue( @@ -428,11 +428,11 @@ class PaddingFIFOQueueTest(test.TestCase): tensor_shape.TensorShape(int_val.shape).is_compatible_with(dequeued_t[ 1].get_shape())) - float_val, int_val = sess.run(dequeued_t) + float_val, int_val = self.evaluate(dequeued_t) self.assertAllEqual(float_elems[4:8], float_val) self.assertAllEqual(int_elems[4:8], int_val) - float_val, int_val = sess.run(dequeued_single_t) + float_val, int_val = self.evaluate(dequeued_single_t) self.assertAllEqual(float_elems[8], float_val) self.assertAllEqual(int_elems[8], int_val) self.assertTrue( @@ -459,7 +459,7 @@ class PaddingFIFOQueueTest(test.TestCase): for enqueue_op in enqueue_ops: enqueue_op.run() - string_val, int_val = sess.run(dequeued_t) + string_val, int_val = self.evaluate(dequeued_t) self.assertAllEqual([[b"a", b"", b""], [b"ab", b"", b""], [b"abc", b"", b""], [b"abc", b"d", b""], @@ -473,7 +473,7 @@ class PaddingFIFOQueueTest(test.TestCase): tensor_shape.TensorShape(int_val.shape).is_compatible_with(dequeued_t[ 1].get_shape())) - string_val, int_val = sess.run(dequeued_single_t) + string_val, int_val = self.evaluate(dequeued_single_t) self.assertAllEqual([b"abc", b"d", b"e", b"f"], string_val) self.assertAllEqual([[1, 2, 3, 4]], int_val) self.assertTrue( @@ -500,7 +500,7 @@ class PaddingFIFOQueueTest(test.TestCase): for enqueue_op in enqueue_ops: enqueue_op.run() - string_val, int_val = sess.run(dequeued_t) + string_val, int_val = self.evaluate(dequeued_t) self.assertAllEqual([[b"a", b"", b""], [b"ab", b"", b""], [b"abc", b"", b""], [b"abc", b"d", b""], @@ -514,7 +514,7 @@ class PaddingFIFOQueueTest(test.TestCase): tensor_shape.TensorShape(int_val.shape).is_compatible_with(dequeued_t[ 1].get_shape())) - string_val, int_val = sess.run(dequeued_single_t) + string_val, int_val = self.evaluate(dequeued_single_t) self.assertAllEqual([b"abc", b"d", b"e", b"f"], string_val) self.assertAllEqual([[1, 2, 3, 4]], int_val) self.assertTrue( @@ -622,7 +622,7 @@ class PaddingFIFOQueueTest(test.TestCase): r"Expected \[2,\?,3\], got \[2,3,4\]"): sess.run([enqueue_op], feed_dict={elems_bad: np.array([1] * 24).reshape((2, 3, 4))}) - dequeued_t.eval() + self.evaluate(dequeued_t) def testParallelEnqueueMany(self): with self.cached_session() as sess: @@ -633,7 +633,7 @@ class PaddingFIFOQueueTest(test.TestCase): # Enqueue 100 items in parallel on 10 threads. def enqueue(): - sess.run(enqueue_op) + self.evaluate(enqueue_op) threads = [self.checkedThread(target=enqueue) for _ in range(10)] for thread in threads: @@ -656,7 +656,7 @@ class PaddingFIFOQueueTest(test.TestCase): dequeued_elems = [] def dequeue(): - dequeued_elems.extend(sess.run(dequeued_t)) + dequeued_elems.extend(self.evaluate(dequeued_t)) threads = [self.checkedThread(target=dequeue) for _ in range(10)] for thread in threads: @@ -680,7 +680,7 @@ class PaddingFIFOQueueTest(test.TestCase): dequeued_elems = [] def dequeue(): - dequeued_elems.extend(sess.run(dequeued_t)) + dequeued_elems.extend(self.evaluate(dequeued_t)) threads = [self.checkedThread(target=dequeue) for _ in range(10)] for thread in threads: @@ -700,11 +700,11 @@ class PaddingFIFOQueueTest(test.TestCase): def enqueue(): for _ in xrange(100): - sess.run(enqueue_op) + self.evaluate(enqueue_op) def dequeue(): for _ in xrange(100): - self.assertTrue(sess.run(dequeued_t) in (10.0, 20.0)) + self.assertTrue(self.evaluate(dequeued_t) in (10.0, 20.0)) enqueue_threads = [self.checkedThread(target=enqueue) for _ in range(10)] dequeue_threads = [self.checkedThread(target=dequeue) for _ in range(10)] @@ -736,7 +736,7 @@ class PaddingFIFOQueueTest(test.TestCase): def dequeue(): for i in xrange(250): - self.assertEqual(i, sess.run(dequeued_t)) + self.assertEqual(i, self.evaluate(dequeued_t)) dequeue_thread = self.checkedThread(target=dequeue) dequeue_thread.start() @@ -767,7 +767,7 @@ class PaddingFIFOQueueTest(test.TestCase): dequeuemany_t = q.dequeue_many(count_placeholder) def enqueue(): - sess.run(enqueue_op) + self.evaluate(enqueue_op) enqueue_thread = self.checkedThread(target=enqueue) enqueue_thread.start() @@ -776,7 +776,7 @@ class PaddingFIFOQueueTest(test.TestCase): while elements_dequeued < 250: # With equal probability, run Dequeue or dequeue_many. if random.random() > 0.5: - self.assertEqual(elements_dequeued, dequeued_t.eval()) + self.assertEqual(elements_dequeued, self.evaluate(dequeued_t)) elements_dequeued += 1 else: count = random.randint(0, min(20, 250 - elements_dequeued)) @@ -805,10 +805,10 @@ class PaddingFIFOQueueTest(test.TestCase): # The enqueue_op should run after the dequeue op has blocked. # TODO(mrry): Figure out how to do this without sleeping. time.sleep(0.1) - sess.run(enqueue_op) + self.evaluate(enqueue_op) def dequeue(): - dequeued_elems.extend(sess.run(dequeued_t).tolist()) + dequeued_elems.extend(self.evaluate(dequeued_t).tolist()) enqueue_thread = self.checkedThread(target=enqueue) dequeue_thread = self.checkedThread(target=dequeue) @@ -832,10 +832,10 @@ class PaddingFIFOQueueTest(test.TestCase): # The enqueue_op should run after the dequeue op has blocked. # TODO(mrry): Figure out how to do this without sleeping. time.sleep(0.1) - sess.run(enqueue_op) + self.evaluate(enqueue_op) def dequeue(): - dequeued_elems.extend(sess.run(dequeued_t).tolist()) + dequeued_elems.extend(self.evaluate(dequeued_t).tolist()) enqueue_thread = self.checkedThread(target=enqueue) dequeue_thread = self.checkedThread(target=dequeue) @@ -882,12 +882,12 @@ class PaddingFIFOQueueTest(test.TestCase): enqueue_op.run() close_op.run() for elem in elems: - self.assertEqual([elem], dequeued_t.eval()) + self.assertEqual([elem], self.evaluate(dequeued_t)) # Expect the operation to fail due to the queue being closed. with self.assertRaisesRegexp(errors_impl.OutOfRangeError, "is closed and has insufficient"): - dequeued_t.eval() + self.evaluate(dequeued_t) def testBlockingDequeueFromClosedQueue(self): with self.cached_session() as sess: @@ -901,11 +901,11 @@ class PaddingFIFOQueueTest(test.TestCase): def dequeue(): for elem in elems: - self.assertEqual([elem], sess.run(dequeued_t)) + self.assertEqual([elem], self.evaluate(dequeued_t)) # Expect the operation to fail due to the queue being closed. with self.assertRaisesRegexp(errors_impl.OutOfRangeError, "is closed and has insufficient"): - sess.run(dequeued_t) + self.evaluate(dequeued_t) dequeue_thread = self.checkedThread(target=dequeue) dequeue_thread.start() @@ -926,8 +926,8 @@ class PaddingFIFOQueueTest(test.TestCase): enqueue_op.run() def dequeue(): - self.assertAllEqual(elems[:3], sess.run(dequeued_t)) - self.assertAllEqual(elems[3:], sess.run(dequeued_t)) + self.assertAllEqual(elems[:3], self.evaluate(dequeued_t)) + self.assertAllEqual(elems[3:], self.evaluate(dequeued_t)) dequeue_thread = self.checkedThread(target=dequeue) dequeue_thread.start() @@ -947,7 +947,7 @@ class PaddingFIFOQueueTest(test.TestCase): # Expect the operation to fail due to the queue being closed. with self.assertRaisesRegexp(errors_impl.OutOfRangeError, "is closed and has insufficient"): - sess.run(dequeued_t) + self.evaluate(dequeued_t) dequeue_thread = self.checkedThread(target=dequeue) dequeue_thread.start() @@ -968,11 +968,11 @@ class PaddingFIFOQueueTest(test.TestCase): enqueue_op.run() def dequeue(): - self.assertAllEqual(elems, sess.run(dequeued_t)) + self.assertAllEqual(elems, self.evaluate(dequeued_t)) # Expect the operation to fail due to the queue being closed. with self.assertRaisesRegexp(errors_impl.OutOfRangeError, "is closed and has insufficient"): - sess.run(dequeued_t) + self.evaluate(dequeued_t) dequeue_thread = self.checkedThread(target=dequeue) dequeue_thread.start() @@ -993,11 +993,11 @@ class PaddingFIFOQueueTest(test.TestCase): enqueue_op.run() def dequeue(): - self.assertAllEqual(elems[:3], sess.run(dequeued_t)) + self.assertAllEqual(elems[:3], self.evaluate(dequeued_t)) # Expect the operation to fail due to the queue being closed. with self.assertRaisesRegexp(errors_impl.OutOfRangeError, "is closed and has insufficient"): - sess.run(dequeued_t) + self.evaluate(dequeued_t) dequeue_thread = self.checkedThread(target=dequeue) dequeue_thread.start() @@ -1017,16 +1017,16 @@ class PaddingFIFOQueueTest(test.TestCase): cleanup_dequeue_t = q.dequeue() def enqueue(): - sess.run(enqueue_op) + self.evaluate(enqueue_op) def dequeue(): - self.assertAllEqual(elems[0:3], sess.run(dequeued_t)) + self.assertAllEqual(elems[0:3], self.evaluate(dequeued_t)) with self.assertRaises(errors_impl.OutOfRangeError): - sess.run(dequeued_t) - self.assertEqual(elems[3], sess.run(cleanup_dequeue_t)) + self.evaluate(dequeued_t) + self.assertEqual(elems[3], self.evaluate(cleanup_dequeue_t)) def close(): - sess.run(close_op) + self.evaluate(close_op) enqueue_thread = self.checkedThread(target=enqueue) enqueue_thread.start() @@ -1059,7 +1059,7 @@ class PaddingFIFOQueueTest(test.TestCase): def dequeue(): with self.assertRaises(errors_impl.OutOfRangeError): - sess.run([dequeued_a_t, dequeued_b_t]) + self.evaluate([dequeued_a_t, dequeued_b_t]) dequeue_thread = self.checkedThread(target=dequeue) dequeue_thread.start() @@ -1072,7 +1072,7 @@ class PaddingFIFOQueueTest(test.TestCase): # Test that the elements in the partially-dequeued batch are # restored in the correct order. for elem_a, elem_b in zip(elems_a, elems_b): - val_a, val_b = sess.run([cleanup_dequeue_a_t, cleanup_dequeue_b_t]) + val_a, val_b = self.evaluate([cleanup_dequeue_a_t, cleanup_dequeue_b_t]) self.assertEqual(elem_a, val_a) self.assertEqual(elem_b, val_b) self.assertEqual(0, q.size().eval()) @@ -1087,7 +1087,7 @@ class PaddingFIFOQueueTest(test.TestCase): # Expect the operation to fail due to the queue being closed. with self.assertRaisesRegexp(errors_impl.OutOfRangeError, "is closed and has insufficient"): - sess.run(dequeued_t) + self.evaluate(dequeued_t) dequeue_thread = self.checkedThread(target=dequeue) dequeue_thread.start() @@ -1107,7 +1107,7 @@ class PaddingFIFOQueueTest(test.TestCase): # Expect the operation to fail due to the queue being closed. with self.assertRaisesRegexp(errors_impl.OutOfRangeError, "is closed and has insufficient"): - sess.run(dequeued_t) + self.evaluate(dequeued_t) dequeue_thread = self.checkedThread(target=dequeue) dequeue_thread.start() @@ -1155,7 +1155,7 @@ class PaddingFIFOQueueTest(test.TestCase): enqueue_op.run() def blocking_enqueue(): - sess.run(blocking_enqueue_op) + self.evaluate(blocking_enqueue_op) thread = self.checkedThread(target=blocking_enqueue) thread.start() @@ -1163,8 +1163,8 @@ class PaddingFIFOQueueTest(test.TestCase): # TODO(mrry): Figure out how to do this without sleeping. time.sleep(0.1) for elem in elems: - self.assertEqual([elem], dequeued_t.eval()) - self.assertEqual([50.0], dequeued_t.eval()) + self.assertEqual([elem], self.evaluate(dequeued_t)) + self.assertEqual([50.0], self.evaluate(dequeued_t)) thread.join() def testBlockingEnqueueManyToFullQueue(self): @@ -1178,7 +1178,7 @@ class PaddingFIFOQueueTest(test.TestCase): enqueue_op.run() def blocking_enqueue(): - sess.run(blocking_enqueue_op) + self.evaluate(blocking_enqueue_op) thread = self.checkedThread(target=blocking_enqueue) thread.start() @@ -1186,10 +1186,10 @@ class PaddingFIFOQueueTest(test.TestCase): # TODO(mrry): Figure out how to do this without sleeping. time.sleep(0.1) for elem in elems: - self.assertEqual([elem], dequeued_t.eval()) + self.assertEqual([elem], self.evaluate(dequeued_t)) time.sleep(0.01) - self.assertEqual([50.0], dequeued_t.eval()) - self.assertEqual([60.0], dequeued_t.eval()) + self.assertEqual([50.0], self.evaluate(dequeued_t)) + self.assertEqual([60.0], self.evaluate(dequeued_t)) # Make sure the thread finishes before exiting. thread.join() @@ -1207,7 +1207,7 @@ class PaddingFIFOQueueTest(test.TestCase): def blocking_enqueue(): # Expect the operation to succeed once the dequeue op runs. - sess.run(blocking_enqueue_op) + self.evaluate(blocking_enqueue_op) enqueue_thread = self.checkedThread(target=blocking_enqueue) enqueue_thread.start() @@ -1217,18 +1217,18 @@ class PaddingFIFOQueueTest(test.TestCase): time.sleep(0.1) def close(): - sess.run(close_op) + self.evaluate(close_op) close_thread = self.checkedThread(target=close) close_thread.start() # The dequeue will unblock both threads. - self.assertEqual(10.0, dequeued_t.eval()) + self.assertEqual(10.0, self.evaluate(dequeued_t)) enqueue_thread.join() close_thread.join() for elem in [20.0, 30.0, 40.0, 50.0]: - self.assertEqual(elem, dequeued_t.eval()) + self.assertEqual(elem, self.evaluate(dequeued_t)) self.assertEqual(0, q.size().eval()) def testBlockingEnqueueManyBeforeClose(self): @@ -1242,7 +1242,7 @@ class PaddingFIFOQueueTest(test.TestCase): enqueue_op.run() def blocking_enqueue(): - sess.run(blocking_enqueue_op) + self.evaluate(blocking_enqueue_op) enqueue_thread = self.checkedThread(target=blocking_enqueue) enqueue_thread.start() @@ -1252,17 +1252,17 @@ class PaddingFIFOQueueTest(test.TestCase): time.sleep(0.1) def close(): - sess.run(close_op) + self.evaluate(close_op) close_thread = self.checkedThread(target=close) close_thread.start() # The dequeue will unblock both threads. - self.assertEqual(10.0, dequeued_t.eval()) + self.assertEqual(10.0, self.evaluate(dequeued_t)) enqueue_thread.join() close_thread.join() for elem in [20.0, 30.0, 50.0, 60.0]: - self.assertEqual(elem, dequeued_t.eval()) + self.assertEqual(elem, self.evaluate(dequeued_t)) def testDoesNotLoseValue(self): with self.cached_session(): @@ -1379,19 +1379,19 @@ class PaddingFIFOQueueTest(test.TestCase): def _blockingDequeue(self, sess, dequeue_op): with self.assertRaisesOpError("was cancelled"): - sess.run(dequeue_op) + self.evaluate(dequeue_op) def _blockingDequeueMany(self, sess, dequeue_many_op): with self.assertRaisesOpError("was cancelled"): - sess.run(dequeue_many_op) + self.evaluate(dequeue_many_op) def _blockingEnqueue(self, sess, enqueue_op): with self.assertRaisesOpError("was cancelled"): - sess.run(enqueue_op) + self.evaluate(enqueue_op) def _blockingEnqueueMany(self, sess, enqueue_many_op): with self.assertRaisesOpError("was cancelled"): - sess.run(enqueue_many_op) + self.evaluate(enqueue_many_op) def testResetOfBlockingOperation(self): with self.cached_session() as sess: @@ -1434,7 +1434,7 @@ class PaddingFIFOQueueTest(test.TestCase): def blocking_enqueue(): enq_done.append(False) # This will fill the queue and then block until enough dequeues happen. - sess.run(enq) + self.evaluate(enq) enq_done.append(True) thread = self.checkedThread(target=blocking_enqueue) @@ -1444,14 +1444,14 @@ class PaddingFIFOQueueTest(test.TestCase): results = [] results.append(deq.eval()) # Will only complete after the enqueue starts. self.assertEqual(len(enq_done), 1) - self.assertEqual(sess.run(size_op), 5) + self.assertEqual(self.evaluate(size_op), 5) for _ in range(3): results.append(deq.eval()) time.sleep(0.1) self.assertEqual(len(enq_done), 1) - self.assertEqual(sess.run(size_op), 5) + self.assertEqual(self.evaluate(size_op), 5) # This dequeue will unblock the thread. results.append(deq.eval()) @@ -1477,7 +1477,7 @@ class PaddingFIFOQueueTest(test.TestCase): def blocking_dequeue(): # Will only complete after 4 enqueues complete. - results.extend(sess.run(deq)) + results.extend(self.evaluate(deq)) thread = self.checkedThread(target=blocking_dequeue) thread.start() @@ -1486,7 +1486,7 @@ class PaddingFIFOQueueTest(test.TestCase): # TODO(mrry): Figure out how to do this without sleeping. time.sleep(0.1) self.assertEqual(len(results), 0) - sess.run(enq) + self.evaluate(enq) # Enough enqueued to unblock the dequeue thread.join() @@ -1517,7 +1517,7 @@ class PaddingFIFOQueueTest(test.TestCase): q.enqueue_many(input_tuple).run() output_tuple_t = q.dequeue_many(32) - output_tuple = sess.run(output_tuple_t) + output_tuple = self.evaluate(output_tuple_t) for (input_elem, output_elem) in zip(input_tuple, output_tuple): self.assertAllEqual(input_elem, output_elem) diff --git a/tensorflow/python/kernel_tests/parameterized_truncated_normal_op_test.py b/tensorflow/python/kernel_tests/parameterized_truncated_normal_op_test.py index c9221f8c209..f87f5170539 100644 --- a/tensorflow/python/kernel_tests/parameterized_truncated_normal_op_test.py +++ b/tensorflow/python/kernel_tests/parameterized_truncated_normal_op_test.py @@ -29,6 +29,7 @@ from tensorflow.core.protobuf import config_pb2 from tensorflow.python.client import session from tensorflow.python.framework import ops from tensorflow.python.framework import random_seed +from tensorflow.python.framework import test_util from tensorflow.python.ops import control_flow_ops from tensorflow.python.ops import random_ops from tensorflow.python.platform import test @@ -166,30 +167,39 @@ class ParameterizedTruncatedNormalTest(test.TestCase): except ImportError as e: tf_logging.warn("Cannot test truncated normal op: %s" % str(e)) + @test_util.run_deprecated_v1 def testDefaults(self): self.validateMoments([10**5], 0.0, 1.0, -2.0, 2.0) + @test_util.run_deprecated_v1 def testShifted(self): self.validateMoments([10**5], -1.0, 1.0, -2.0, 2.0) + @test_util.run_deprecated_v1 def testRightTail(self): self.validateMoments([10**5], 0.0, 1.0, 4.0, np.infty) + @test_util.run_deprecated_v1 def testLeftTail(self): self.validateMoments([10**5], 0.0, 1.0, -np.infty, -4.0) + @test_util.run_deprecated_v1 def testLeftTailTwoSidedBounds(self): self.validateMoments([10**5], 0.0, 1.0, -6.0, -3.0) + @test_util.run_deprecated_v1 def testTwoSidedLeftTailShifted(self): self.validateKolmogorovSmirnov([10**5], 6.0, 1.0, -1.0, 1.0) + @test_util.run_deprecated_v1 def testRightTailShifted(self): self.validateMoments([10**5], -5.0, 1.0, 2.0, np.infty) + @test_util.run_deprecated_v1 def testSmallStddev(self): self.validateKolmogorovSmirnov([10**5], 0.0, 0.1, 0.05, 0.10) + @test_util.run_deprecated_v1 def testSamplingWithSmallStdDevFarFromBound(self): sample_op = random_ops.parameterized_truncated_normal( shape=(int(1e5),), means=0.8, stddevs=0.05, minvals=-1., maxvals=1.) @@ -202,6 +212,7 @@ class ParameterizedTruncatedNormalTest(test.TestCase): no_neg_samples = np.sum(samples < 0.) self.assertEqual(no_neg_samples, 0.) + @test_util.run_deprecated_v1 def testSamplingAtRandnSwitchover(self): # The randn sampler is used as the bounds are moved farther from the mean, # and the probability of accepting a sample increases the farther the diff --git a/tensorflow/python/kernel_tests/parse_single_example_op_test.py b/tensorflow/python/kernel_tests/parse_single_example_op_test.py index a84895a287e..43c8fa4ab5c 100644 --- a/tensorflow/python/kernel_tests/parse_single_example_op_test.py +++ b/tensorflow/python/kernel_tests/parse_single_example_op_test.py @@ -29,6 +29,7 @@ from tensorflow.python.framework import errors_impl from tensorflow.python.framework import ops from tensorflow.python.framework import sparse_tensor from tensorflow.python.framework import tensor_shape +from tensorflow.python.framework import test_util from tensorflow.python.ops import parsing_ops from tensorflow.python.platform import test from tensorflow.python.platform import tf_logging @@ -107,7 +108,7 @@ class ParseExampleTest(test.TestCase): for result_dict in [out, out_with_example_name]: result = flatten_values_tensors_or_sparse(result_dict.values()) # Check values. - tf_result = sess.run(result) + tf_result = self.evaluate(result) _compare_output_to_expected(self, result_dict, expected_values, tf_result) @@ -121,6 +122,7 @@ class ParseExampleTest(test.TestCase): self.assertEqual( tuple(out[k].dense_shape.get_shape().as_list()), (1,)) + @test_util.run_deprecated_v1 def testEmptySerializedWithAllDefaults(self): sparse_name = "st_a" a_name = "a" @@ -229,6 +231,7 @@ class ParseExampleTest(test.TestCase): }, expected_err=(ValueError, "Missing shape for feature a")) + @test_util.run_deprecated_v1 def testSerializedContainingSparse(self): original = [ example(features=features({ @@ -552,6 +555,7 @@ class ParseExampleTest(test.TestCase): } }, expected_output) + @test_util.run_deprecated_v1 def testSerializedContainingSparseAndSparseFeatureAndDenseWithNoDefault(self): original = [ example(features=features({ @@ -618,6 +622,7 @@ class ParseExampleTest(test.TestCase): }, expected_output) + @test_util.run_deprecated_v1 def testSerializedContainingSparseAndSparseFeatureWithReuse(self): original = [ example(features=features({ @@ -658,6 +663,7 @@ class ParseExampleTest(test.TestCase): } }, expected_output) + @test_util.run_deprecated_v1 def testSerializedContainingVarLenDense(self): aname = "a" bname = "b" @@ -869,6 +875,7 @@ class ParseSingleExampleTest(test.TestCase): self.assertEqual( tuple(out[k].dense_shape.get_shape().as_list()), (1,)) + @test_util.run_deprecated_v1 def testSingleExampleWithSparseAndSparseFeatureAndDense(self): original = example(features=features({ "c": float_feature([3, 4]), diff --git a/tensorflow/python/kernel_tests/parsing_ops_test.py b/tensorflow/python/kernel_tests/parsing_ops_test.py index 71d8b60d3cc..af76e09f393 100644 --- a/tensorflow/python/kernel_tests/parsing_ops_test.py +++ b/tensorflow/python/kernel_tests/parsing_ops_test.py @@ -33,6 +33,7 @@ from tensorflow.python.framework import errors_impl from tensorflow.python.framework import ops from tensorflow.python.framework import sparse_tensor from tensorflow.python.framework import tensor_shape +from tensorflow.python.framework import test_util from tensorflow.python.framework import tensor_util from tensorflow.python.ops import array_ops from tensorflow.python.ops import parsing_ops @@ -101,15 +102,15 @@ class ParseExampleTest(test.TestCase): out = parsing_ops.parse_example(**kwargs) result = flatten_values_tensors_or_sparse(out.values()) # Check values. - tf_result = sess.run(result) + tf_result = self.evaluate(result) _compare_output_to_expected(self, out, expected_values, tf_result) # Check shapes; if serialized is a Tensor we need its size to # properly check. serialized = kwargs["serialized"] batch_size = ( - serialized.eval().size if isinstance(serialized, ops.Tensor) else - np.asarray(serialized).size) + self.evaluate(serialized).size if isinstance(serialized, ops.Tensor) + else np.asarray(serialized).size) for k, f in kwargs["features"].items(): if isinstance(f, parsing_ops.FixedLenFeature) and f.shape is not None: self.assertEqual( @@ -121,6 +122,7 @@ class ParseExampleTest(test.TestCase): self.assertEqual( tuple(out[k].dense_shape.get_shape().as_list()), (2,)) + @test_util.run_deprecated_v1 def testEmptySerializedWithAllDefaults(self): sparse_name = "st_a" a_name = "a" @@ -243,6 +245,7 @@ class ParseExampleTest(test.TestCase): }, expected_err=(ValueError, "Missing shape for feature a")) + @test_util.run_deprecated_v1 def testSerializedContainingSparse(self): original = [ example(features=features({ @@ -571,6 +574,7 @@ class ParseExampleTest(test.TestCase): } }, expected_output) + @test_util.run_deprecated_v1 def testSerializedContainingSparseAndSparseFeatureAndDenseWithNoDefault(self): expected_st_a = ( # indices, values, shape np.empty((0, 2), dtype=np.int64), # indices @@ -631,6 +635,7 @@ class ParseExampleTest(test.TestCase): }, expected_output) + @test_util.run_deprecated_v1 def testSerializedContainingSparseAndSparseFeatureWithReuse(self): expected_idx = ( # indices, values, shape np.array([[0, 0], [0, 1], [1, 0], [1, 1]], dtype=np.int64), @@ -740,6 +745,7 @@ class ParseExampleTest(test.TestCase): for batch_size in (1, 10, 20, 100, 256): self._testSerializedContainingVarLenDenseLargerBatch(batch_size) + @test_util.run_deprecated_v1 def testSerializedContainingVarLenDense(self): aname = "a" bname = "b" @@ -962,6 +968,7 @@ class ParseSingleExampleTest(test.TestCase): self.assertEqual( tuple(out[k].dense_shape.get_shape().as_list()), (1,)) + @test_util.run_deprecated_v1 def testSingleExampleWithSparseAndSparseFeatureAndDense(self): original = example( features=features({ @@ -1180,6 +1187,7 @@ class ParseSequenceExampleTest(test.TestCase): expected_err=expected_err, batch=True) + @test_util.run_deprecated_v1 def testSequenceExampleWithSparseAndDenseContext(self): original = sequence_example( context=features({ @@ -1223,6 +1231,7 @@ class ParseSequenceExampleTest(test.TestCase): }, expected_context_values=expected_context_output) + @test_util.run_deprecated_v1 def testSequenceExampleWithMultipleSizeFeatureLists(self): original = sequence_example( feature_lists=feature_lists({ @@ -1286,6 +1295,7 @@ class ParseSequenceExampleTest(test.TestCase): }, expected_feat_list_values=expected_feature_list_output) + @test_util.run_deprecated_v1 def testSequenceExampleWithoutDebugName(self): original = sequence_example( feature_lists=feature_lists({ @@ -1343,6 +1353,7 @@ class ParseSequenceExampleTest(test.TestCase): }, expected_feat_list_values=expected_feature_list_output) + @test_util.run_deprecated_v1 def testSequenceExampleWithSparseAndDenseFeatureLists(self): original = sequence_example( feature_lists=feature_lists({ @@ -1401,6 +1412,7 @@ class ParseSequenceExampleTest(test.TestCase): }, expected_feat_list_values=expected_feature_list_output) + @test_util.run_deprecated_v1 def testSequenceExampleWithEmptyFeatureInFeatureLists(self): original = sequence_example( feature_lists=feature_lists({ @@ -1541,6 +1553,7 @@ class ParseSequenceExampleTest(test.TestCase): " feature_list_dense_missing_assumed_empty or" " feature_list_dense_defaults?")) + @test_util.run_deprecated_v1 def testSequenceExampleBatch(self): first = sequence_example( feature_lists=feature_lists({ @@ -1614,7 +1627,7 @@ class DecodeJSONExampleTest(test.TestCase): shape=examples.shape, dtype=dtypes.string) binary_tensor = parsing_ops.decode_json_example(json_tensor) - binary_val = sess.run(binary_tensor) + binary_val = self.evaluate(binary_tensor) if examples.shape: self.assertShapeEqual(binary_val, json_tensor) @@ -1695,16 +1708,18 @@ class DecodeJSONExampleTest(test.TestCase): })), ]) + @test_util.run_deprecated_v1 def testInvalidSyntax(self): with self.cached_session() as sess: json_tensor = constant_op.constant(["{]"]) binary_tensor = parsing_ops.decode_json_example(json_tensor) with self.assertRaisesOpError("Error while parsing JSON"): - sess.run(binary_tensor) + self.evaluate(binary_tensor) class ParseTensorOpTest(test.TestCase): + @test_util.run_deprecated_v1 def testToFloat32(self): with self.cached_session(): expected = np.random.rand(3, 4, 5).astype(np.float32) @@ -1718,6 +1733,7 @@ class ParseTensorOpTest(test.TestCase): self.assertAllEqual(expected, result) + @test_util.run_deprecated_v1 def testToUint8(self): with self.cached_session(): expected = np.random.rand(3, 4, 5).astype(np.uint8) @@ -1731,6 +1747,7 @@ class ParseTensorOpTest(test.TestCase): self.assertAllEqual(expected, result) + @test_util.run_deprecated_v1 def testTypeMismatch(self): with self.cached_session(): expected = np.random.rand(3, 4, 5).astype(np.uint8) @@ -1744,6 +1761,7 @@ class ParseTensorOpTest(test.TestCase): r"\(uint16\)"): tensor.eval(feed_dict={serialized: tensor_proto.SerializeToString()}) + @test_util.run_deprecated_v1 def testInvalidInput(self): with self.cached_session(): serialized = array_ops.placeholder(dtypes.string) diff --git a/tensorflow/python/kernel_tests/partitioned_variables_test.py b/tensorflow/python/kernel_tests/partitioned_variables_test.py index d1f0c6c2a05..48655391fa7 100644 --- a/tensorflow/python/kernel_tests/partitioned_variables_test.py +++ b/tensorflow/python/kernel_tests/partitioned_variables_test.py @@ -26,6 +26,7 @@ from six.moves import xrange # pylint: disable=redefined-builtin from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops +from tensorflow.python.framework import test_util from tensorflow.python.ops import array_ops from tensorflow.python.ops import init_ops from tensorflow.python.ops import partitioned_variables @@ -322,17 +323,19 @@ class PartitionedVariablesTestCase(test.TestCase): for i in xrange(len(expected_specs)): self.assertEquals(expected_specs[i], slices[i]._save_slice_info.spec) + @test_util.run_deprecated_v1 def testVecConstantInit(self): with self.cached_session(): rnd_par = constant_op.constant([1, 2, 3, 4]) vs = partitioned_variables.create_partitioned_variables([4], [4], rnd_par) variables.global_variables_initializer().run() val = array_ops.concat(vs, 0).eval() - rnd = rnd_par.eval() + rnd = self.evaluate(rnd_par) self.assertAllClose(rnd, val) self.assertEqual([dtypes.int32] * 4, [v.dtype.base_dtype for v in vs]) self._TestSaveSpec(vs, ["4 0,1", "4 1,1", "4 2,1", "4 3,1"]) + @test_util.run_deprecated_v1 def testConstantInit(self): with self.cached_session(): rnd_par = constant_op.constant([[1, 2, 3, 4], [5, 6, 7, 8]]) @@ -340,7 +343,7 @@ class PartitionedVariablesTestCase(test.TestCase): rnd_par) variables.global_variables_initializer().run() val = array_ops.concat(vs, 1).eval() - rnd = rnd_par.eval() + rnd = self.evaluate(rnd_par) self.assertAllClose(rnd, val) self.assertEqual([dtypes.int32] * 2, [v.dtype.base_dtype for v in vs]) self._TestSaveSpec(vs, ["2 4 0,2:0,2", "2 4 0,2:2,2"]) @@ -401,12 +404,15 @@ class PartitionedVariablesTestCase(test.TestCase): self.assertEqual(var2_name + "/part_0:0", vs2[0].name) self.assertEqual(var2_name + "/part_1:0", vs2[1].name) + @test_util.run_deprecated_v1 def testName(self): self._testNameHelper(use_resource=False) + @test_util.run_deprecated_v1 def testResourceName(self): self._testNameHelper(use_resource=True) + @test_util.run_deprecated_v1 def testRandomInitValue(self): with self.cached_session(): rnd = variables.Variable(random_ops.random_uniform([200, 40])) @@ -414,7 +420,7 @@ class PartitionedVariablesTestCase(test.TestCase): rnd.get_shape(), [1, 10], rnd.initialized_value()) variables.global_variables_initializer().run() val = array_ops.concat(vs, 1).eval() - rnd = rnd.eval() + rnd = self.evaluate(rnd) self.assertAllClose(rnd, val) self.assertEqual([dtypes.float32] * 10, [v.dtype.base_dtype for v in vs]) self._TestSaveSpec(vs, [ @@ -424,6 +430,7 @@ class PartitionedVariablesTestCase(test.TestCase): "200 40 0,200:36,4" ]) + @test_util.run_deprecated_v1 def testRandomInitUnevenPartitions(self): with self.cached_session(): rnd = variables.Variable( @@ -434,7 +441,7 @@ class PartitionedVariablesTestCase(test.TestCase): for i in xrange(1, 10) ] variables.global_variables_initializer().run() - rnd_val = rnd.eval() + rnd_val = self.evaluate(rnd) # Only check the slice save specs for the first 5 tf. save_specs = [ # One slice @@ -462,6 +469,7 @@ class PartitionedVariablesTestCase(test.TestCase): if i < len(save_specs): self._TestSaveSpec(vs, save_specs[i]) + @test_util.run_deprecated_v1 def testDegenerate(self): with self.cached_session(): rnd = variables.Variable(random_ops.random_uniform([10, 43])) @@ -469,10 +477,11 @@ class PartitionedVariablesTestCase(test.TestCase): rnd.get_shape(), [1, 1], rnd.initialized_value()) variables.global_variables_initializer().run() val = array_ops.concat(vs, 0).eval() - rnd = rnd.eval() + rnd = self.evaluate(rnd) self.assertAllClose(rnd, val) self._TestSaveSpec(vs, ["10 43 0,10:0,43"]) + @test_util.run_deprecated_v1 def testSliceSizeOne(self): with self.cached_session(): rnd = variables.Variable(random_ops.random_uniform([10, 43])) @@ -480,7 +489,7 @@ class PartitionedVariablesTestCase(test.TestCase): rnd.get_shape(), [10, 1], rnd.initialized_value()) variables.global_variables_initializer().run() val = array_ops.concat(vs, 0).eval() - rnd = rnd.eval() + rnd = self.evaluate(rnd) self.assertAllClose(rnd, val) self._TestSaveSpec(vs, [ "10 43 0,1:0,43", "10 43 1,1:0,43", "10 43 2,1:0,43", @@ -488,6 +497,7 @@ class PartitionedVariablesTestCase(test.TestCase): "10 43 6,1:0,43", "10 43 7,1:0,43", "10 43 8,1:0,43", "10 43 9,1:0,43" ]) + @test_util.run_deprecated_v1 def testIotaInitializer(self): self.assertAllClose([0., 1., 2., 3.], _IotaInitializer([4])) self.assertAllClose([[0., 1.], [0., 10.], [0., 100.], [0., 1000.]], @@ -503,6 +513,7 @@ class PartitionedVariablesTestCase(test.TestCase): self.assertAllClose(slice0 + slice1 + slice2, val) self._TestSaveSpec(vs, ["13 5 0,5:0,5", "13 5 5,4:0,5", "13 5 9,4:0,5"]) + @test_util.run_deprecated_v1 def testRandomInitializer(self): # Sanity check that the slices uses a different seed when using a random # initializer function. @@ -510,7 +521,7 @@ class PartitionedVariablesTestCase(test.TestCase): var0, var1 = partitioned_variables.create_partitioned_variables( [20, 12], [1, 2], init_ops.random_uniform_initializer()) variables.global_variables_initializer().run() - val0, val1 = var0.eval().flatten(), var1.eval().flatten() + val0, val1 = self.evaluate(var0).flatten(), self.evaluate(var1).flatten() self.assertTrue(np.linalg.norm(val0 - val1) > 1e-6) # Negative test that proves that slices have the same values if # the random initializer uses a seed. @@ -518,7 +529,7 @@ class PartitionedVariablesTestCase(test.TestCase): var0, var1 = partitioned_variables.create_partitioned_variables( [20, 12], [1, 2], init_ops.random_uniform_initializer(seed=201)) variables.global_variables_initializer().run() - val0, val1 = var0.eval().flatten(), var1.eval().flatten() + val0, val1 = self.evaluate(var0).flatten(), self.evaluate(var1).flatten() self.assertAllClose(val0, val1) def testSomeErrors(self): @@ -546,6 +557,7 @@ class PartitionedVariablesTestCase(test.TestCase): partitioned_variables.create_partitioned_variables( [10, 43], [1, 50], rnd.initialized_value()) + @test_util.run_deprecated_v1 def testControlDepsNone(self): with self.cached_session() as session: c = constant_op.constant(1.0) @@ -572,6 +584,7 @@ class PartitionedVariablesTestCase(test.TestCase): for op in reading_ops: self.assertEqual([], op.control_inputs) + @test_util.run_deprecated_v1 def testConcat(self): with self.cached_session() as session: var_x = variable_scope.get_variable( diff --git a/tensorflow/python/kernel_tests/pool_test.py b/tensorflow/python/kernel_tests/pool_test.py index 372861297fb..78e786f01ca 100644 --- a/tensorflow/python/kernel_tests/pool_test.py +++ b/tensorflow/python/kernel_tests/pool_test.py @@ -24,6 +24,7 @@ import numpy as np from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes +from tensorflow.python.framework import test_util from tensorflow.python.ops import gradient_checker from tensorflow.python.ops import nn_ops import tensorflow.python.ops.nn_grad # pylint: disable=unused-import @@ -151,7 +152,7 @@ class PoolingTest(test.TestCase): np.prod(input_shape), dtype=np.float32).reshape(input_shape) - 1 y1 = pool_direct(input=x, **kwargs) y2 = nn_ops.pool(input=x, **kwargs) - self.assertAllClose(y1, y2.eval(), rtol=1e-2, atol=1e-2) + self.assertAllClose(y1, self.evaluate(y2), rtol=1e-2, atol=1e-2) def testPoolSimple(self): with self.session(use_gpu=test.is_gpu_available()): @@ -301,6 +302,7 @@ class PoolingTest(test.TestCase): err_tolerance = 1e-2 self.assertLess(err, err_tolerance) + @test_util.run_deprecated_v1 def testGradient1D(self): with self.session(use_gpu=test.is_gpu_available()): for padding in ["SAME", "VALID"]: @@ -327,6 +329,7 @@ class PoolingTest(test.TestCase): dilation_rate=[1], strides=strides) + @test_util.run_deprecated_v1 def testGradient2D(self): with self.session(use_gpu=test.is_gpu_available()): for padding in ["SAME", "VALID"]: @@ -353,6 +356,7 @@ class PoolingTest(test.TestCase): dilation_rate=[1, 1], strides=strides) + @test_util.run_deprecated_v1 def testGradient3D(self): with self.session(use_gpu=test.is_gpu_available()): for padding in ["SAME", "VALID"]: diff --git a/tensorflow/python/kernel_tests/pooling_ops_3d_test.py b/tensorflow/python/kernel_tests/pooling_ops_3d_test.py index e393c7a0229..347e092dee3 100644 --- a/tensorflow/python/kernel_tests/pooling_ops_3d_test.py +++ b/tensorflow/python/kernel_tests/pooling_ops_3d_test.py @@ -81,7 +81,7 @@ class PoolingTest(test.TestCase): data_format=data_format) if data_format == "NCDHW": t = test_util.NCHWToNHWC(t) - vals = sess.run(t) + vals = self.evaluate(t) # Verifies values. actual = vals.flatten() self.assertAllClose(expected, actual) @@ -253,6 +253,7 @@ class PoolingTest(test.TestCase): ksize = test_util.NHWCToNCHW(ksize) strides = test_util.NHWCToNCHW(strides) t = test_util.NHWCToNCHW(t) + output_sizes = test_util.NHWCToNCHW(output_sizes) t = pool_func( t, @@ -294,6 +295,7 @@ class PoolingTest(test.TestCase): use_gpu=use_gpu, **kwargs) + @test_util.run_deprecated_v1 def testMaxPoolGradValidPadding1_1_3d(self): self._ConstructAndTestGradient( nn_ops.max_pool3d, @@ -303,6 +305,7 @@ class PoolingTest(test.TestCase): strides=(1, 1, 1), padding="VALID") + @test_util.run_deprecated_v1 def testMaxPoolGradValidPadding2_1_6_3d(self): self._ConstructAndTestGradient( nn_ops.max_pool3d, @@ -312,6 +315,7 @@ class PoolingTest(test.TestCase): strides=(1, 1, 1), padding="VALID") + @test_util.run_deprecated_v1 def testMaxPoolGradValidPadding2_1_7_3d(self): self._ConstructAndTestGradient( nn_ops.max_pool3d, @@ -321,6 +325,7 @@ class PoolingTest(test.TestCase): strides=(1, 1, 1), padding="VALID") + @test_util.run_deprecated_v1 def testMaxPoolGradValidPadding1_2_3d(self): self._ConstructAndTestGradient( nn_ops.max_pool3d, @@ -330,6 +335,7 @@ class PoolingTest(test.TestCase): strides=(2, 2, 2), padding="VALID") + @test_util.run_deprecated_v1 def testMaxPoolGradValidPadding2_2_3d(self): self._ConstructAndTestGradient( nn_ops.max_pool3d, @@ -339,6 +345,7 @@ class PoolingTest(test.TestCase): strides=(2, 2, 2), padding="VALID") + @test_util.run_deprecated_v1 def testMaxPoolGradSamePadding1_1_3d(self): self._ConstructAndTestGradient( nn_ops.max_pool3d, @@ -348,6 +355,7 @@ class PoolingTest(test.TestCase): strides=(1, 1, 1), padding="SAME") + @test_util.run_deprecated_v1 def testMaxPoolGradSamePadding1_2_3d(self): self._ConstructAndTestGradient( nn_ops.max_pool3d, @@ -357,6 +365,7 @@ class PoolingTest(test.TestCase): strides=(2, 2, 2), padding="SAME") + @test_util.run_deprecated_v1 def testMaxPoolGradSamePadding2_1_3d(self): self._ConstructAndTestGradient( nn_ops.max_pool3d, @@ -366,6 +375,7 @@ class PoolingTest(test.TestCase): strides=(1, 1, 1), padding="SAME") + @test_util.run_deprecated_v1 def testMaxPoolGradSamePadding2_2_3d(self): self._ConstructAndTestGradient( nn_ops.max_pool3d, @@ -375,6 +385,7 @@ class PoolingTest(test.TestCase): strides=(2, 2, 2), padding="SAME") + @test_util.run_deprecated_v1 def testMaxPoolGradSamePadding3_1_3d(self): self._ConstructAndTestGradient( nn_ops.max_pool3d, @@ -384,6 +395,7 @@ class PoolingTest(test.TestCase): strides=(1, 1, 1), padding="SAME") + @test_util.run_deprecated_v1 def testAvgPoolGradValidPadding1_1_3d(self): self._ConstructAndTestGradient( nn_ops.avg_pool3d, @@ -393,6 +405,7 @@ class PoolingTest(test.TestCase): strides=(1, 1, 1), padding="VALID") + @test_util.run_deprecated_v1 def testAvgPoolGradValidPadding1_2_3d(self): self._ConstructAndTestGradient( nn_ops.avg_pool3d, @@ -402,6 +415,7 @@ class PoolingTest(test.TestCase): strides=(2, 2, 2), padding="VALID") + @test_util.run_deprecated_v1 def testAvgPoolGradValidPadding2_1_3d(self): self._ConstructAndTestGradient( nn_ops.avg_pool3d, @@ -411,6 +425,7 @@ class PoolingTest(test.TestCase): strides=(1, 1, 1), padding="VALID") + @test_util.run_deprecated_v1 def testAvgPoolGradValidPadding2_2_3d(self): self._ConstructAndTestGradient( nn_ops.avg_pool3d, @@ -420,6 +435,7 @@ class PoolingTest(test.TestCase): strides=(2, 2, 2), padding="VALID") + @test_util.run_deprecated_v1 def testAvgPoolGradSamePadding1_1_3d(self): self._ConstructAndTestGradient( nn_ops.avg_pool3d, @@ -429,6 +445,7 @@ class PoolingTest(test.TestCase): strides=(1, 1, 1), padding="SAME") + @test_util.run_deprecated_v1 def testAvgPoolGradSamePadding1_2_3d(self): self._ConstructAndTestGradient( nn_ops.avg_pool3d, @@ -438,6 +455,7 @@ class PoolingTest(test.TestCase): strides=(2, 2, 2), padding="SAME") + @test_util.run_deprecated_v1 def testAvgPoolGradSamePadding2_1_3d(self): self._ConstructAndTestGradient( nn_ops.avg_pool3d, @@ -447,6 +465,7 @@ class PoolingTest(test.TestCase): strides=(1, 1, 1), padding="SAME") + @test_util.run_deprecated_v1 def testAvgPoolGradSamePadding2_2_3d(self): self._ConstructAndTestGradient( nn_ops.avg_pool3d, @@ -456,6 +475,7 @@ class PoolingTest(test.TestCase): strides=(2, 2, 2), padding="SAME") + @test_util.run_deprecated_v1 def testAvgPoolGradSamePadding3_1_3d(self): self._ConstructAndTestGradient( nn_ops.avg_pool3d, diff --git a/tensorflow/python/kernel_tests/pooling_ops_test.py b/tensorflow/python/kernel_tests/pooling_ops_test.py index 53003a7f284..c33b59bb99b 100644 --- a/tensorflow/python/kernel_tests/pooling_ops_test.py +++ b/tensorflow/python/kernel_tests/pooling_ops_test.py @@ -166,7 +166,7 @@ class PoolingTest(test.TestCase): strides_placeholder: strides }) else: - actual = t.eval() + actual = self.evaluate(t) self.assertShapeEqual(actual, t) self.assertAllCloseAccordingToType(expected, actual.flatten()) @@ -384,6 +384,7 @@ class PoolingTest(test.TestCase): expected=[], use_gpu=use_gpu) + @test_util.run_deprecated_v1 def testAvgPooling(self): for use_gpu in True, False: self._testAvgPoolValidPadding(use_gpu) @@ -577,6 +578,7 @@ class PoolingTest(test.TestCase): expected=[], use_gpu=use_gpu) + @test_util.run_deprecated_v1 def testMaxPooling(self): for use_gpu in True, False: self._testMaxPoolValidPadding(use_gpu) @@ -588,6 +590,7 @@ class PoolingTest(test.TestCase): self._testMaxPoolEmptyInput(use_gpu) # Tests for DepthwiseMaxPooling on CPU only. + @test_util.run_deprecated_v1 def testDepthwiseMaxPool1x1DepthWindow1(self): # input is: # [1.0, ..., 10.0] along depth, @@ -613,6 +616,7 @@ class PoolingTest(test.TestCase): use_gpu=False, v2=v2) + @test_util.run_deprecated_v1 def testDepthwiseMaxPool2x2DepthWindow3(self): # input is: # @@ -639,6 +643,7 @@ class PoolingTest(test.TestCase): use_gpu=False, v2=v2) + @test_util.run_deprecated_v1 def testKernelSmallerThanStrideValid(self): for use_gpu in [True, False]: self._VerifyValues( @@ -670,6 +675,7 @@ class PoolingTest(test.TestCase): expected=[5, 8, 26, 29], use_gpu=use_gpu) + @test_util.run_deprecated_v1 def testKernelSmallerThanStrideSame(self): for use_gpu in [True, False]: for pool_func in [nn_ops.max_pool, nn_ops.avg_pool]: @@ -750,11 +756,11 @@ class PoolingTest(test.TestCase): with self.cached_session(use_gpu=True): t = constant_op.constant(tensor_input, shape=input_shape) out_op, _ = nn_ops.max_pool_with_argmax(t, ksize, strides, padding) - gpu_val = out_op.eval() + gpu_val = self.evaluate(out_op) with self.cached_session(use_gpu=False): t = constant_op.constant(tensor_input, shape=input_shape) out_op = nn_ops.max_pool(t, ksize, strides, padding) - cpu_val = out_op.eval() + cpu_val = self.evaluate(out_op) self.assertAllCloseAccordingToType(cpu_val, gpu_val) def _CompareMaxPoolingBk(self, input_shape, output_shape, ksize, strides, @@ -767,20 +773,20 @@ class PoolingTest(test.TestCase): with self.cached_session(use_gpu=True): t = constant_op.constant(tensor_input, shape=input_shape) _, argmax_op = nn_ops.max_pool_with_argmax(t, ksize, strides, padding) - argmax = argmax_op.eval() + argmax = self.evaluate(argmax_op) grad_in = constant_op.constant(tensor_output, shape=output_shape) out_op = gen_nn_ops.max_pool_grad_with_argmax(t, grad_in, argmax, ksize, strides, padding) - gpu_val = out_op.eval() + gpu_val = self.evaluate(out_op) self.assertShapeEqual(gpu_val, out_op) with self.cached_session(use_gpu=False): t = constant_op.constant(tensor_input, shape=input_shape) out_op = nn_ops.max_pool(t, ksize, strides, padding) - orig_out = out_op.eval() + orig_out = self.evaluate(out_op) grad_in = constant_op.constant(tensor_output, shape=output_shape) out_op = gen_nn_ops.max_pool_grad(t, orig_out, grad_in, ksize, strides, padding) - cpu_val = out_op.eval() + cpu_val = self.evaluate(out_op) self.assertShapeEqual(cpu_val, out_op) # The CPU version accumulates its gradient on fp16, so it's less # accurate than the GPU version that does the accumulation on fp32 @@ -796,20 +802,20 @@ class PoolingTest(test.TestCase): with self.cached_session(use_gpu=True): t = constant_op.constant(tensor_input, shape=input_shape) _, argmax_op = nn_ops.max_pool_with_argmax(t, ksize, strides, padding) - argmax = argmax_op.eval() + argmax = self.evaluate(argmax_op) grad_in = constant_op.constant(tensor_input, shape=input_shape) out_op = gen_nn_ops.max_pool_grad_grad_with_argmax( t, grad_in, argmax, ksize, strides, padding) - gpu_val = out_op.eval() + gpu_val = self.evaluate(out_op) self.assertShapeEqual(gpu_val, out_op) with self.cached_session(use_gpu=False): t = constant_op.constant(tensor_input, shape=input_shape) out_op = nn_ops.max_pool(t, ksize, strides, padding) - orig_out = out_op.eval() + orig_out = self.evaluate(out_op) grad_in = constant_op.constant(tensor_input, shape=input_shape) out_op = gen_nn_ops.max_pool_grad_grad(t, orig_out, grad_in, ksize, strides, padding) - cpu_val = out_op.eval() + cpu_val = self.evaluate(out_op) self.assertShapeEqual(cpu_val, out_op) # The CPU version accumulates its gradient on fp16, so it's less # accurate than the GPU version that does the accumulation on fp32 @@ -826,7 +832,7 @@ class PoolingTest(test.TestCase): strides=[1, 1, 1, 1], Targmax=dtypes.int64, padding="VALID") - out, argmax = sess.run([out_op, argmax_op]) + out, argmax = self.evaluate([out_op, argmax_op]) self.assertShapeEqual(out, out_op) self.assertShapeEqual(argmax, argmax_op) self.assertAllClose(out.ravel(), [1.0, 1.0, 1.0, 1.0]) @@ -848,7 +854,7 @@ class PoolingTest(test.TestCase): ksize=[1, 2, 2, 1], strides=[1, 1, 1, 1], padding="VALID") - out = out_op.eval().flatten() + out = self.evaluate(out_op).flatten() self.assertAllClose(out, [11.0, 12.0, 0.0, 13.0, 0.0, 14.0, 0.0, 0.0, 0.0]) @@ -871,7 +877,7 @@ class PoolingTest(test.TestCase): ksize=[1, 2, 2, 1], strides=[1, 1, 1, 1], padding="VALID") - out = out_op.eval().flatten() + out = self.evaluate(out_op).flatten() self.assertAllClose(out, [11.0, 12.0, 14.0, 16.0]) def _ConstructAndTestGradient(self, @@ -1167,6 +1173,7 @@ class PoolingTest(test.TestCase): data_format=data_format, use_gpu=use_gpu) + @test_util.run_deprecated_v1 def testMaxPoolGrad(self): for (data_format, use_gpu) in GetTestConfigs(): self._testMaxPoolGradValidPadding1_1(data_format, use_gpu) @@ -1221,12 +1228,12 @@ class PoolingTest(test.TestCase): input_tensor, output_tensor, output_backprop_tensor, window_rows, window_cols, row_stride, col_stride, padding, v2) - actual_input_backprop = input_backprop_tensor.eval() + actual_input_backprop = self.evaluate(input_backprop_tensor) self.assertShapeEqual(actual_input_backprop, input_backprop_tensor) actual_input_backprop = actual_input_backprop.flatten() actual_input_backprop = self._GetNdArray(actual_input_backprop) - actual_output = output_tensor.eval().flatten() + actual_output = self.evaluate(output_tensor).flatten() actual_output = self._GetNdArray(actual_output) self.assertAllClose( @@ -1497,6 +1504,7 @@ class PoolingTest(test.TestCase): else: del os.environ["TF_ENABLE_MAXPOOL_NANPROP"] + @test_util.run_deprecated_v1 def testMaxPoolGradDirect(self): self._testMaxPoolGradDirect1_1() self._testMaxPoolGradDirect1_2() @@ -1616,6 +1624,7 @@ class PoolingTest(test.TestCase): data_format=data_format, use_gpu=use_gpu) + @test_util.run_deprecated_v1 def testMaxPoolGradGrad(self): for (data_format, use_gpu) in GetTestConfigs(): self._testMaxPoolGradGradValidPadding1_1(data_format, use_gpu) @@ -1649,6 +1658,7 @@ class PoolingTest(test.TestCase): orig_input, orig_output, grad, [1, window_rows, window_cols, 1], [1, row_stride, col_stride, 1], padding) + @test_util.run_deprecated_v1 def testAvgPoolGrad(self): for (data_format, use_gpu) in GetTestConfigs(): self._testAvgPoolGradValidPadding1_1(data_format, use_gpu) @@ -1778,6 +1788,7 @@ class PoolingTest(test.TestCase): data_format=data_format, use_gpu=use_gpu) + @test_util.run_deprecated_v1 def testShapeFunctionEdgeCases(self): # All shapes unknown. for pool_func in [nn_ops.max_pool, nn_ops.avg_pool]: @@ -1806,6 +1817,7 @@ class PoolingTest(test.TestCase): strides=[1, 1, 1, 1], padding="SAME") + @test_util.run_deprecated_v1 def testOpEdgeCases(self): with self.session(use_gpu=test.is_gpu_available()) as sess: pool_funcs = [nn_ops.max_pool, nn_ops.avg_pool] diff --git a/tensorflow/python/kernel_tests/priority_queue_test.py b/tensorflow/python/kernel_tests/priority_queue_test.py index 73a9c816382..9be682ea52f 100644 --- a/tensorflow/python/kernel_tests/priority_queue_test.py +++ b/tensorflow/python/kernel_tests/priority_queue_test.py @@ -50,7 +50,7 @@ class PriorityQueueTest(test.TestCase): enq.run() deq = q.dequeue_many(100) - deq_elem, deq_value_0, deq_value_1 = sess.run(deq) + deq_elem, deq_value_0, deq_value_1 = self.evaluate(deq) allowed = {} missed = set() @@ -81,7 +81,7 @@ class PriorityQueueTest(test.TestCase): # Run one producer thread for each element in elems. def enqueue(enqueue_op): - sess.run(enqueue_op) + self.evaluate(enqueue_op) dequeue_op = q.dequeue_many(100) @@ -93,7 +93,7 @@ class PriorityQueueTest(test.TestCase): for t in enqueue_threads: t.start() - deq_elem, deq_value_0, deq_value_1 = sess.run(dequeue_op) + deq_elem, deq_value_0, deq_value_1 = self.evaluate(dequeue_op) for t in enqueue_threads: t.join() @@ -132,12 +132,12 @@ class PriorityQueueTest(test.TestCase): # Run one producer thread for each element in elems. def enqueue(enqueue_op): - sess.run(enqueue_op) + self.evaluate(enqueue_op) dequeued = [] def dequeue(dequeue_op): - (dequeue_indices, dequeue_values) = sess.run(dequeue_op) + (dequeue_indices, dequeue_values) = self.evaluate(dequeue_op) self.assertAllEqual(dequeue_indices, dequeue_values) dequeued.extend(dequeue_indices) @@ -184,10 +184,10 @@ class PriorityQueueTest(test.TestCase): # Run one producer thread for each element in elems. def enqueue(enqueue_op): - sess.run(enqueue_op) + self.evaluate(enqueue_op) def dequeue(dequeue_op, dequeued): - (dequeue_indices, dequeue_values) = sess.run(dequeue_op) + (dequeue_indices, dequeue_values) = self.evaluate(dequeue_op) self.assertAllEqual(dequeue_indices, dequeue_values) dequeue_wait.acquire() dequeued.extend(dequeue_indices) @@ -215,7 +215,7 @@ class PriorityQueueTest(test.TestCase): # We can't guarantee full sorting because we can't guarantee # that the dequeued.extend() call runs immediately after the - # sess.run() call. Here we're just happy everything came out. + # self.evaluate() call. Here we're just happy everything came out. self.assertAllEqual(set(dequeued), set(all_enqueued_values)) def testRoundTripInsertManyMultiThreadedReadOnceSorts(self): @@ -236,7 +236,7 @@ class PriorityQueueTest(test.TestCase): # Run one producer thread for each element in elems. def enqueue(enqueue_op): - sess.run(enqueue_op) + self.evaluate(enqueue_op) dequeue_op = q.dequeue_many(100) @@ -248,7 +248,7 @@ class PriorityQueueTest(test.TestCase): for t in enqueue_threads: t.start() - deq_elem, deq_value_0, deq_value_1 = sess.run(dequeue_op) + deq_elem, deq_value_0, deq_value_1 = self.evaluate(dequeue_op) for t in enqueue_threads: t.join() @@ -276,7 +276,7 @@ class PriorityQueueTest(test.TestCase): side_value_1 = np.random.rand(1000).astype(bytes) q.enqueue_many((elem, side_value_0, side_value_1)).run() deq = q.dequeue_many(1000) - deq_elem, deq_value_0, deq_value_1 = sess.run(deq) + deq_elem, deq_value_0, deq_value_1 = self.evaluate(deq) allowed = {} for e, v0, v1 in zip(elem, side_value_0, side_value_1): diff --git a/tensorflow/python/kernel_tests/py_func_test.py b/tensorflow/python/kernel_tests/py_func_test.py index 837f1ec054f..1f3f02a9f01 100644 --- a/tensorflow/python/kernel_tests/py_func_test.py +++ b/tensorflow/python/kernel_tests/py_func_test.py @@ -272,7 +272,7 @@ class PyFuncTest(test.TestCase): with self.assertRaisesRegexp(errors.UnimplementedError, "Unsupported numpy type"): - y.eval() + self.evaluate(y) def testBadReturnType(self): with self.cached_session(): @@ -285,7 +285,7 @@ class PyFuncTest(test.TestCase): with self.assertRaisesRegexp(errors.UnimplementedError, "Unsupported object type"): - z.eval() + self.evaluate(z) def testReturnInput(self): with self.cached_session(): @@ -307,9 +307,9 @@ class PyFuncTest(test.TestCase): with session_lib.Session() as sess: producer = iter(range(3)) x, = script_ops.py_func(lambda: next(producer), [], [dtypes.int64]) - self.assertEqual(sess.run(x), 0) - self.assertEqual(sess.run(x), 1) - self.assertEqual(sess.run(x), 2) + self.assertEqual(self.evaluate(x), 0) + self.assertEqual(self.evaluate(x), 1) + self.assertEqual(self.evaluate(x), 2) def testStateless(self): # Not using self.cached_session(), which disables optimization. @@ -317,9 +317,9 @@ class PyFuncTest(test.TestCase): producer = iter(range(3)) x, = script_ops.py_func( lambda: next(producer), [], [dtypes.int64], stateful=False) - self.assertEqual(sess.run(x), 0) - self.assertEqual(sess.run(x), 0) - self.assertEqual(sess.run(x), 0) + self.assertEqual(self.evaluate(x), 0) + self.assertEqual(self.evaluate(x), 0) + self.assertEqual(self.evaluate(x), 0) def testGradientFunction(self): # Input to tf.py_func is necessary, otherwise get_gradient_function() @@ -335,7 +335,7 @@ class PyFuncTest(test.TestCase): val = [[1, 2], [3, 4]] x, = script_ops.py_func(lambda: np.array(val, order="F"), [], [dtypes.int64]) - self.assertAllEqual(val, x.eval()) + self.assertAllEqual(val, self.evaluate(x)) def testParallel(self): # Tests that tf.py_func's can run in parallel if they release the GIL. @@ -390,7 +390,7 @@ class PyFuncTest(test.TestCase): f = script_ops.py_func( do_nothing, [constant_op.constant(3, dtypes.int64)], [], stateful=False) with self.cached_session() as sess: - self.assertEqual(sess.run(f), []) + self.assertEqual(self.evaluate(f), []) def _testExceptionHandling(self, py_exp, tf_exp, eager=False): @@ -514,6 +514,7 @@ class PyFuncTest(test.TestCase): self.assertAllEqual(ret, [[3.0], [3.0], [3.0]]) @test_util.run_in_graph_and_eager_modes + @test_util.run_deprecated_v1 def testEagerExceptionHandling(self): with test_util.device(use_gpu=True): self._testExceptionHandling( @@ -533,6 +534,7 @@ class PyFuncTest(test.TestCase): self._testExceptionHandling(WeirdError, errors.UnknownError, eager=True) @test_util.run_in_graph_and_eager_modes + @test_util.run_deprecated_v1 def testEagerReturningVariableRaisesError(self): def return_variable(): return resource_variable_ops.ResourceVariable(0.0) diff --git a/tensorflow/python/kernel_tests/qr_op_test.py b/tensorflow/python/kernel_tests/qr_op_test.py index a60237fb25a..0f2537b3711 100644 --- a/tensorflow/python/kernel_tests/qr_op_test.py +++ b/tensorflow/python/kernel_tests/qr_op_test.py @@ -21,6 +21,7 @@ from __future__ import print_function import numpy as np from tensorflow.python.framework import constant_op +from tensorflow.python.framework import test_util from tensorflow.python.ops import array_ops from tensorflow.python.ops import gradient_checker from tensorflow.python.ops import linalg_ops @@ -49,6 +50,7 @@ class QrOpTest(test.TestCase): "Shape must be at least rank 2 but is rank 1"): linalg_ops.qr(vector) + @test_util.run_deprecated_v1 def testConcurrentExecutesWithoutError(self): with self.session(use_gpu=True) as sess: all_ops = [] @@ -60,7 +62,7 @@ class QrOpTest(test.TestCase): q1, r1 = linalg_ops.qr(matrix1, full_matrices=full_matrices_) q2, r2 = linalg_ops.qr(matrix2, full_matrices=full_matrices_) all_ops += [q1, r1, q2, r2] - val = sess.run(all_ops) + val = self.evaluate(all_ops) for i in range(8): q = 4 * i self.assertAllEqual(val[q], val[q + 2]) # q1 == q2 @@ -110,7 +112,7 @@ def _GetQrOpTest(dtype_, shape_, full_matrices_, use_static_shape_): tol = 1e-5 else: tol = 1e-14 - self.assertAllClose(identity.eval(), xx.eval(), atol=tol) + self.assertAllClose(identity.eval(), self.evaluate(xx), atol=tol) def Test(self): np.random.seed(1) @@ -129,7 +131,7 @@ def _GetQrOpTest(dtype_, shape_, full_matrices_, use_static_shape_): q_tf, r_tf = linalg_ops.qr(x_tf, full_matrices=full_matrices_) if use_static_shape_: - q_tf_val, r_tf_val = sess.run([q_tf, r_tf]) + q_tf_val, r_tf_val = self.evaluate([q_tf, r_tf]) else: q_tf_val, r_tf_val = sess.run([q_tf, r_tf], feed_dict={x_tf: x_np}) diff --git a/tensorflow/python/kernel_tests/random/multinomial_op_big_test.py b/tensorflow/python/kernel_tests/random/multinomial_op_big_test.py index 0023506b77a..576720528e2 100644 --- a/tensorflow/python/kernel_tests/random/multinomial_op_big_test.py +++ b/tensorflow/python/kernel_tests/random/multinomial_op_big_test.py @@ -23,6 +23,7 @@ import numpy as np from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes from tensorflow.python.framework import random_seed +from tensorflow.python.framework import test_util from tensorflow.python.ops import random_ops from tensorflow.python.platform import test @@ -39,7 +40,7 @@ class MultinomialTest(test.TestCase): num_samples=1000000, seed=15) for _ in range(100): - x = sess.run(samples) + x = self.evaluate(samples) indices, counts = np.unique(x, return_counts=True) for index, count in zip(indices, counts): if index in counts_by_indices.keys(): @@ -57,7 +58,7 @@ class MultinomialTest(test.TestCase): num_samples=1000000, seed=15) for _ in range(100): - x = sess.run(samples) + x = self.evaluate(samples) indices, counts = np.unique(x, return_counts=True) for index, count in zip(indices, counts): if index in counts_by_indices.keys(): @@ -66,6 +67,7 @@ class MultinomialTest(test.TestCase): counts_by_indices[index] = count self.assertEqual(counts_by_indices[0], 100000000) + @test_util.run_deprecated_v1 def testLargeDynamicRange3(self): random_seed.set_random_seed(10) counts_by_indices = {} @@ -79,7 +81,7 @@ class MultinomialTest(test.TestCase): # we'll run out of memory if we try to draw 1e9 samples directly # really should fit in 12GB of memory... for _ in range(100): - x = sess.run(samples) + x = self.evaluate(samples) indices, counts = np.unique(x, return_counts=True) for index, count in zip(indices, counts): if index in counts_by_indices.keys(): diff --git a/tensorflow/python/kernel_tests/random/multinomial_op_test.py b/tensorflow/python/kernel_tests/random/multinomial_op_test.py index bd64d61af8e..5d123307a8e 100644 --- a/tensorflow/python/kernel_tests/random/multinomial_op_test.py +++ b/tensorflow/python/kernel_tests/random/multinomial_op_test.py @@ -66,12 +66,13 @@ class MultinomialTest(test.TestCase): logits, num_samples, output_dtype=output_dtype)) self.assertAllEqual([[1] * num_samples, [2] * num_samples], samples) + @test_util.run_deprecated_v1 def testOneOpMultipleStepsIndependent(self): - with self.test_session(use_gpu=True) as sess: + with test_util.use_gpu(): sample_op1, _ = self._make_ops(10) # Consecutive runs shouldn't yield identical output. - sample1a = sess.run(sample_op1) - sample1b = sess.run(sample_op1) + sample1a = self.evaluate(sample_op1) + sample1b = self.evaluate(sample_op1) self.assertFalse(np.equal(sample1a, sample1b).all()) def testEagerOneOpMultipleStepsIndependent(self): @@ -81,26 +82,27 @@ class MultinomialTest(test.TestCase): self.assertFalse(np.equal(sample1.numpy(), sample2.numpy()).all()) def testTwoOpsIndependent(self): - with self.test_session(use_gpu=True) as sess: + with test_util.use_gpu(): sample_op1, sample_op2 = self._make_ops(32) - sample1, sample2 = sess.run([sample_op1, sample_op2]) + sample1, sample2 = self.evaluate([sample_op1, sample_op2]) # We expect sample1 and sample2 to be independent. # 1 in 2^32 chance of this assertion failing. self.assertFalse(np.equal(sample1, sample2).all()) + @test_util.run_deprecated_v1 def testTwoOpsSameSeedDrawSameSequences(self): - with self.test_session(use_gpu=True) as sess: + with test_util.use_gpu(): sample_op1, sample_op2 = self._make_ops(1000, seed=1) - sample1, sample2 = sess.run([sample_op1, sample_op2]) + sample1, sample2 = self.evaluate([sample_op1, sample_op2]) self.assertAllEqual(sample1, sample2) def testLargeLogits(self): for neg in [True, False]: - with self.test_session(use_gpu=True): + with test_util.use_gpu(): logits = np.array([[1000.] * 5]) if neg: logits *= -1 - samples = random_ops.multinomial(logits, 10).eval() + samples = self.evaluate(random_ops.multinomial(logits, 10)) # Sampled classes should be in-range. self.assertTrue((samples >= 0).all()) self.assertTrue((samples < 5).all()) @@ -157,10 +159,10 @@ class MultinomialTest(test.TestCase): Returns: Frequencies from sampled classes; shape [batch_size, num_classes]. """ - with self.test_session(use_gpu=True) as sess: + with test_util.use_gpu(): random_seed.set_random_seed(1618) op = sampler(constant_op.constant(logits), num_samples) - d = sess.run(op) + d = self.evaluate(op) batch_size, num_classes = logits.shape freqs_mat = [] @@ -186,25 +188,27 @@ class MultinomialTest(test.TestCase): def testEmpty(self): classes = 5 - with self.test_session(use_gpu=True): + with test_util.use_gpu(): for batch in 0, 3: for samples in 0, 7: - x = random_ops.multinomial( - array_ops.zeros([batch, classes]), samples).eval() + x = self.evaluate( + random_ops.multinomial( + array_ops.zeros([batch, classes]), samples)) self.assertEqual(x.shape, (batch, samples)) + @test_util.run_deprecated_v1 def testEmptyClasses(self): - with self.test_session(use_gpu=True): + with test_util.use_gpu(): x = random_ops.multinomial(array_ops.zeros([5, 0]), 7) with self.assertRaisesOpError("num_classes should be positive"): - x.eval() + self.evaluate(x) def testNegativeMinLogits(self): random_seed.set_random_seed(78844) - with self.test_session(use_gpu=True): + with test_util.use_gpu(): logits = constant_op.constant([[np.finfo(np.float32).min] * 1023 + [0]]) num_samples = 1000 - samples = random_ops.multinomial(logits, num_samples).eval() + samples = self.evaluate(random_ops.multinomial(logits, num_samples)) self.assertAllEqual([[1023] * num_samples], samples) diff --git a/tensorflow/python/kernel_tests/random/random_crop_test.py b/tensorflow/python/kernel_tests/random/random_crop_test.py index 8ded522320b..724bee07157 100644 --- a/tensorflow/python/kernel_tests/random/random_crop_test.py +++ b/tensorflow/python/kernel_tests/random/random_crop_test.py @@ -20,12 +20,14 @@ from __future__ import print_function import numpy as np +from tensorflow.python.framework import test_util from tensorflow.python.ops import random_ops from tensorflow.python.platform import test class RandomCropTest(test.TestCase): + @test_util.run_deprecated_v1 def testNoOp(self): # No random cropping is performed since the size is value.shape. for shape in (2, 1, 1), (2, 1, 3), (4, 5, 3): @@ -44,10 +46,11 @@ class RandomCropTest(test.TestCase): for i in range(2) for j in range(3) for k in range(4)) crop = random_ops.random_crop(value, size=target) for _ in range(20): - y = crop.eval() + y = self.evaluate(crop) self.assertAllEqual(y.shape, target) self.assertTrue(tuple(y.ravel()) in value_set) + @test_util.run_deprecated_v1 def testRandomization(self): # Run 1x1 crop num_samples times in an image and ensure that one finds each # pixel 1/size of the time. @@ -61,7 +64,7 @@ class RandomCropTest(test.TestCase): crop = random_ops.random_crop(value, single, seed=7) counts = np.zeros(size, dtype=np.int32) for _ in range(num_samples): - y = crop.eval() + y = self.evaluate(crop) self.assertAllEqual(y.shape, single) counts[y] += 1 diff --git a/tensorflow/python/kernel_tests/random/random_gamma_test.py b/tensorflow/python/kernel_tests/random/random_gamma_test.py index 606e8862c47..a5952a21968 100644 --- a/tensorflow/python/kernel_tests/random/random_gamma_test.py +++ b/tensorflow/python/kernel_tests/random/random_gamma_test.py @@ -26,6 +26,7 @@ from six.moves import xrange # pylint: disable=redefined-builtin from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops from tensorflow.python.framework import random_seed +from tensorflow.python.framework import test_util from tensorflow.python.ops import array_ops from tensorflow.python.ops import math_ops from tensorflow.python.ops import random_ops @@ -48,14 +49,16 @@ class RandomGammaTest(test.TestCase): [num], alpha, beta=beta, dtype=dtype, seed=seed) ret = np.empty([10, num]) for i in xrange(10): - ret[i, :] = sess.run(rng) + ret[i, :] = self.evaluate(rng) return ret return func + @test_util.run_deprecated_v1 def testMomentsFloat32(self): self._testMoments(dtypes.float32) + @test_util.run_deprecated_v1 def testMomentsFloat64(self): self._testMoments(dtypes.float64) @@ -208,6 +211,7 @@ class RandomGammaTest(test.TestCase): sy = self._Sampler(1000, 0.0, 1.0, dt, use_gpu=use_gpu, seed=345) self.assertAllEqual(sx(), sy()) + @test_util.run_deprecated_v1 def testNoCSE(self): """CSE = constant subexpression eliminator. @@ -222,6 +226,7 @@ class RandomGammaTest(test.TestCase): diff = rnd2 - rnd1 self.assertGreater(np.linalg.norm(diff.eval()), 0.1) + @test_util.run_deprecated_v1 def testShape(self): # Fully known shape. rnd = random_ops.random_gamma([150], 2.0) @@ -253,6 +258,7 @@ class RandomGammaTest(test.TestCase): rnd = random_ops.random_gamma([50], array_ops.placeholder(dtypes.float32)) self.assertIs(None, rnd.get_shape().ndims) + @test_util.run_deprecated_v1 def testPositive(self): n = int(10e3) for dt in [dtypes.float16, dtypes.float32, dtypes.float64]: diff --git a/tensorflow/python/kernel_tests/random/random_grad_test.py b/tensorflow/python/kernel_tests/random/random_grad_test.py index d89056c485a..aac6eeac06a 100644 --- a/tensorflow/python/kernel_tests/random/random_grad_test.py +++ b/tensorflow/python/kernel_tests/random/random_grad_test.py @@ -22,6 +22,7 @@ import numpy as np from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes +from tensorflow.python.framework import test_util from tensorflow.python.ops import array_ops from tensorflow.python.ops import gradients_impl from tensorflow.python.ops import math_ops @@ -45,6 +46,7 @@ class AddLeadingUnitDimensionsTest(test.TestCase): ret = random_grad.add_leading_unit_dimensions(1.0, 2) self.assertAllEqual(ret.shape, [1, 1]) + @test_util.run_deprecated_v1 def testUnknownShape(self): x = array_ops.placeholder(dtypes.float32) num_dimensions = array_ops.placeholder(dtypes.int32) @@ -72,6 +74,7 @@ class RandomGammaGradTest(test.TestCase): some statistical properties of the derivative. """ + @test_util.run_deprecated_v1 def testGradientsShape(self): shape = [2, 3] alpha = array_ops.ones([2, 2]) @@ -81,6 +84,7 @@ class RandomGammaGradTest(test.TestCase): self.assertAllEqual(grads_alpha.shape, alpha.shape) self.assertAllEqual(grads_beta.shape, beta.shape) + @test_util.run_deprecated_v1 def testGradientsShapeWithOneSamplePerParameter(self): shape = [] alpha = array_ops.ones([2, 2]) @@ -90,6 +94,7 @@ class RandomGammaGradTest(test.TestCase): self.assertAllEqual(grads_alpha.shape, alpha.shape) self.assertAllEqual(grads_beta.shape, beta.shape) + @test_util.run_deprecated_v1 def testGradientsUnknownShape(self): shape = array_ops.placeholder(dtypes.int32) alpha = array_ops.placeholder(dtypes.float32) @@ -138,9 +143,11 @@ class RandomGammaGradTest(test.TestCase): except ImportError as e: tf_logging.warn("Cannot use special functions in a test: %s" % str(e)) + @test_util.run_deprecated_v1 def testCompareToExplicitDerivativeFloat(self): self._testCompareToExplicitDerivative(dtypes.float32) + @test_util.run_deprecated_v1 def testCompareToExplicitDerivativeDouble(self): self._testCompareToExplicitDerivative(dtypes.float64) @@ -182,12 +189,15 @@ class RandomGammaGradTest(test.TestCase): self.assertAllClose(actual_val, expected_val, rtol=1e-3, atol=1e-3) + @test_util.run_deprecated_v1 def testCompareToImplicitDerivativeFloat(self): self._testCompareToImplicitDerivative(dtypes.float32) + @test_util.run_deprecated_v1 def testCompareToImplicitDerivativeDouble(self): self._testCompareToImplicitDerivative(dtypes.float64) + @test_util.run_deprecated_v1 def testAverageAlphaGradient(self): """Statistical test for the gradient. @@ -207,6 +217,7 @@ class RandomGammaGradTest(test.TestCase): dsample_dalpha_val = self.evaluate(dsample_dalpha) self.assertAllClose(dsample_dalpha_val, [1.0] * 3, atol=1e-1, rtol=1e-1) + @test_util.run_deprecated_v1 def testQuadraticLoss(self): """Statistical test for the gradient. diff --git a/tensorflow/python/kernel_tests/random/random_ops_test.py b/tensorflow/python/kernel_tests/random/random_ops_test.py index 6de894846bc..1384c3f446f 100644 --- a/tensorflow/python/kernel_tests/random/random_ops_test.py +++ b/tensorflow/python/kernel_tests/random/random_ops_test.py @@ -25,6 +25,7 @@ from tensorflow.python.eager import context from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops from tensorflow.python.framework import random_seed +from tensorflow.python.framework import test_util from tensorflow.python.ops import array_ops from tensorflow.python.ops import random_ops from tensorflow.python.ops import variables @@ -49,9 +50,9 @@ class RandomOpTestCommon(test.TestCase): random_seed.set_random_seed(graph_seed) x = rng_func([num], min_or_mean, max_or_stddev, dtype=dtype, seed=op_seed) - y = sess.run(x) - z = sess.run(x) - w = sess.run(x) + y = self.evaluate(x) + z = self.evaluate(x) + w = self.evaluate(x) # We use exact equality here. If the random-number generator is producing # the same output, all three outputs will be bitwise identical. @@ -69,7 +70,7 @@ class RandomNormalTest(RandomOpTestCommon): [num], mean=mu, stddev=sigma, dtype=dtype, seed=seed) ret = np.empty([10, num]) for i in xrange(10): - ret[i, :] = sess.run(rng) + ret[i, :] = self.evaluate(rng) return ret return func @@ -92,6 +93,7 @@ class RandomNormalTest(RandomOpTestCommon): # Checks that the CPU and GPU implementation returns the same results, # given the same random seed + @test_util.run_deprecated_v1 def testCPUGPUMatch(self): for dt in dtypes.float16, dtypes.float32, dtypes.float64: results = {} @@ -104,12 +106,14 @@ class RandomNormalTest(RandomOpTestCommon): else: self.assertAllClose(results[False], results[True], rtol=1e-6, atol=1e-6) + @test_util.run_deprecated_v1 def testSeed(self): for dt in dtypes.float16, dtypes.float32, dtypes.float64: sx = self._Sampler(1000, 0.0, 1.0, dt, use_gpu=True, seed=345) sy = self._Sampler(1000, 0.0, 1.0, dt, use_gpu=True, seed=345) self.assertAllEqual(sx(), sy()) + @test_util.run_deprecated_v1 def testNoCSE(self): for use_gpu in [False, True]: with self.session(use_gpu=use_gpu): @@ -119,12 +123,14 @@ class RandomNormalTest(RandomOpTestCommon): diff = rnd2 - rnd1 self.assertTrue(np.linalg.norm(diff.eval()) > 0.1) + @test_util.run_deprecated_v1 def testSingleSessionNotConstant(self): for use_gpu in [False, True]: for dt in dtypes.float16, dtypes.float32, dtypes.float64: self._testSingleSessionNotConstant( random_ops.random_normal, 100, dt, 0.0, 1.0, use_gpu=use_gpu) + @test_util.run_deprecated_v1 def testSingleSessionOpSeedNotConstant(self): for use_gpu in [False, True]: for dt in dtypes.float16, dtypes.float32, dtypes.float64: @@ -137,6 +143,7 @@ class RandomNormalTest(RandomOpTestCommon): use_gpu=use_gpu, op_seed=1345) + @test_util.run_deprecated_v1 def testSingleSessionGraphSeedNotConstant(self): for use_gpu in [False, True]: for dt in dtypes.float16, dtypes.float32, dtypes.float64: @@ -160,7 +167,7 @@ class TruncatedNormalTest(test.TestCase): [num], mean=mu, stddev=sigma, dtype=dtype, seed=seed) ret = np.empty([10, num]) for i in xrange(10): - ret[i, :] = sess.run(rng) + ret[i, :] = self.evaluate(rng) return ret return func @@ -185,6 +192,7 @@ class TruncatedNormalTest(test.TestCase): # Checks that the CPU and GPU implementation returns the same results, # given the same random seed + @test_util.run_deprecated_v1 def testCPUGPUMatch(self): # Skip the test if there is no GPU. if not test.is_gpu_available(): @@ -203,6 +211,7 @@ class TruncatedNormalTest(test.TestCase): else: self.assertAllClose(results[False], results[True], rtol=1e-6, atol=1e-6) + @test_util.run_deprecated_v1 def testSeed(self): for dt in dtypes.float16, dtypes.float32, dtypes.float64: sx = self._Sampler(1000, 0.0, 1.0, dt, use_gpu=True, seed=345) @@ -219,6 +228,7 @@ class TruncatedNormalTest(test.TestCase): print("std(x)", np.std(x), abs(np.std(x) / stddev - 0.85)) self.assertTrue(abs(np.std(x) / stddev - 0.85) < 0.04) + @test_util.run_deprecated_v1 def testLargeShape(self): with self.session(use_gpu=True): v = variables.Variable( @@ -226,6 +236,7 @@ class TruncatedNormalTest(test.TestCase): n = random_ops.truncated_normal(v.shape) self.assertEqual([8589934592, 1], n.shape.as_list()) + @test_util.run_deprecated_v1 def testNoCSE(self): with self.session(use_gpu=True): shape = [2, 3, 4] @@ -256,7 +267,7 @@ class RandomUniformTest(RandomOpTestCommon): [num], minval=minv, maxval=maxv, dtype=dtype, seed=seed) ret = np.empty([10, num]) for i in xrange(10): - ret[i, :] = sess.run(rng) + ret[i, :] = self.evaluate(rng) return ret return func @@ -287,6 +298,7 @@ class RandomUniformTest(RandomOpTestCommon): print("count = ", count) self.assertTrue(count < count_limit) + @test_util.run_deprecated_v1 def testUniformIntsWithInvalidShape(self): for dtype in dtypes.int32, dtypes.int64: with self.assertRaisesRegexp( @@ -299,6 +311,7 @@ class RandomUniformTest(RandomOpTestCommon): [1000], minval=1, maxval=[2, 3], dtype=dtype) # Check that uniform ints actually follow a uniform distribution. + @test_util.run_deprecated_v1 def testUniformInts(self): minv = -2 maxv = 15 @@ -331,6 +344,7 @@ class RandomUniformTest(RandomOpTestCommon): # Checks that the CPU and GPU implementation returns the same results, # given the same random seed + @test_util.run_deprecated_v1 def testCPUGPUMatch(self): for dt in (dtypes.float16, dtypes.float32, dtypes.float64, dtypes.int32, dtypes.int64): @@ -342,6 +356,7 @@ class RandomUniformTest(RandomOpTestCommon): results[use_gpu] = sampler() self.assertAllEqual(results[False], results[True]) + @test_util.run_deprecated_v1 def testSeed(self): for dt in (dtypes.float16, dtypes.float32, dtypes.float64, dtypes.int32, dtypes.int64): @@ -350,6 +365,7 @@ class RandomUniformTest(RandomOpTestCommon): sy = self._Sampler(1000, 0, 17, dtype=dt, use_gpu=True, seed=seed) self.assertAllEqual(sx(), sy()) + @test_util.run_deprecated_v1 def testNoCSE(self): shape = [2, 3, 4] for dtype in dtypes.float16, dtypes.float32, dtypes.int32: @@ -359,6 +375,7 @@ class RandomUniformTest(RandomOpTestCommon): diff = (rnd2 - rnd1).eval() self.assertTrue(np.linalg.norm(diff) > 0.1) + @test_util.run_deprecated_v1 def testSingleSessionNotConstant(self): for use_gpu in [False, True]: for dt in (dtypes.float16, dtypes.float32, dtypes.float64, dtypes.int32, @@ -366,6 +383,7 @@ class RandomUniformTest(RandomOpTestCommon): self._testSingleSessionNotConstant( random_ops.random_uniform, 100, dt, 0, 17, use_gpu=use_gpu) + @test_util.run_deprecated_v1 def testSingleSessionOpSeedNotConstant(self): for use_gpu in [False, True]: for dt in (dtypes.float16, dtypes.float32, dtypes.float64, dtypes.int32, @@ -379,6 +397,7 @@ class RandomUniformTest(RandomOpTestCommon): use_gpu=use_gpu, op_seed=1345) + @test_util.run_deprecated_v1 def testSingleSessionGraphSeedNotConstant(self): for use_gpu in [False, True]: for dt in (dtypes.float16, dtypes.float32, dtypes.float64, dtypes.int32, @@ -395,6 +414,7 @@ class RandomUniformTest(RandomOpTestCommon): class RandomShapeTest(test.TestCase): + @test_util.run_deprecated_v1 def testTruncatedNormal(self): # Fully known shape. rnd1 = random_ops.truncated_normal([1, 2, 3]) @@ -407,6 +427,7 @@ class RandomShapeTest(test.TestCase): rnd3 = random_ops.truncated_normal(array_ops.placeholder(dtypes.int32)) self.assertIs(None, rnd3.get_shape().ndims) + @test_util.run_deprecated_v1 def testRandomNormal(self): # Fully known shape. rnd1 = random_ops.random_normal([1, 2, 3]) @@ -419,6 +440,7 @@ class RandomShapeTest(test.TestCase): rnd3 = random_ops.random_normal(array_ops.placeholder(dtypes.int32)) self.assertIs(None, rnd3.get_shape().ndims) + @test_util.run_deprecated_v1 def testRandomUniform(self): # Fully known shape. rnd1 = random_ops.random_uniform([1, 2, 3]) diff --git a/tensorflow/python/kernel_tests/random/random_poisson_test.py b/tensorflow/python/kernel_tests/random/random_poisson_test.py index 417588f8a39..0a6b004d682 100644 --- a/tensorflow/python/kernel_tests/random/random_poisson_test.py +++ b/tensorflow/python/kernel_tests/random/random_poisson_test.py @@ -23,6 +23,7 @@ from six.moves import xrange # pylint: disable=redefined-builtin from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops +from tensorflow.python.framework import test_util from tensorflow.python.ops import array_ops from tensorflow.python.ops import random_ops from tensorflow.python.platform import test @@ -43,7 +44,7 @@ class RandomPoissonTest(test.TestCase): rng = random_ops.random_poisson(lam, [num], dtype=dtype, seed=seed) ret = np.empty([10, num]) for i in xrange(10): - ret[i, :] = sess.run(rng) + ret[i, :] = self.evaluate(rng) return ret return func @@ -104,6 +105,7 @@ class RandomPoissonTest(test.TestCase): # Checks that the CPU and GPU implementation returns the same results, # given the same random seed + @test_util.run_deprecated_v1 def testCPUGPUMatch(self): for dt in _SUPPORTED_DTYPES: results = {} @@ -115,12 +117,14 @@ class RandomPoissonTest(test.TestCase): else: self.assertAllClose(results[False], results[True], rtol=1e-6, atol=1e-6) + @test_util.run_deprecated_v1 def testSeed(self): for dt in dtypes.float16, dtypes.float32, dtypes.float64: sx = self._Sampler(1000, 1.0, dt, use_gpu=True, seed=345) sy = self._Sampler(1000, 1.0, dt, use_gpu=True, seed=345) self.assertAllEqual(sx(), sy()) + @test_util.run_deprecated_v1 def testNoCSE(self): """CSE = constant subexpression eliminator. @@ -140,8 +144,9 @@ class RandomPoissonTest(test.TestCase): with self.cached_session(): rnd = random_ops.random_poisson([], [], seed=12345) self.assertEqual([0], rnd.get_shape().as_list()) - self.assertAllClose(np.array([], dtype=np.float32), rnd.eval()) + self.assertAllClose(np.array([], dtype=np.float32), self.evaluate(rnd)) + @test_util.run_deprecated_v1 def testShape(self): # Fully known shape rnd = random_ops.random_poisson(2.0, [150], seed=12345) @@ -184,6 +189,7 @@ class RandomPoissonTest(test.TestCase): seed=12345) self.assertIs(None, rnd.get_shape().ndims) + @test_util.run_deprecated_v1 def testDTypeCombinationsV2(self): """Tests random_poisson_v2() for all supported dtype combinations.""" with self.cached_session(): diff --git a/tensorflow/python/kernel_tests/random/random_shuffle_queue_test.py b/tensorflow/python/kernel_tests/random/random_shuffle_queue_test.py index 0d85a072d4a..ed4f5434d9f 100644 --- a/tensorflow/python/kernel_tests/random/random_shuffle_queue_test.py +++ b/tensorflow/python/kernel_tests/random/random_shuffle_queue_test.py @@ -84,9 +84,9 @@ class RandomShuffleQueueTest(test.TestCase): dequeue_t = q.dequeue() results = [] for _ in range(2): - a, b = sess.run(dequeue_t) + a, b = self.evaluate(dequeue_t) results.append((a, b)) - a, b = sess.run(q.dequeue_many(3)) + a, b = self.evaluate(q.dequeue_many(3)) for i in range(3): results.append((a[i], b[i])) self.assertItemsEqual([(1, [5]), (2, [6]), (3, [7]), (4, [8]), (9, [10])], @@ -101,7 +101,7 @@ class RandomShuffleQueueTest(test.TestCase): # Run one producer thread for each element in elems. def enqueue(enqueue_op): - sess.run(enqueue_op) + self.evaluate(enqueue_op) threads = [ self.checkedThread( @@ -133,7 +133,7 @@ class RandomShuffleQueueTest(test.TestCase): results = [] def dequeue(): - results.append(sess.run(dequeued_t)) + results.append(self.evaluate(dequeued_t)) threads = [self.checkedThread(target=dequeue) for _ in enqueue_ops] for thread in threads: @@ -167,13 +167,13 @@ class RandomShuffleQueueTest(test.TestCase): # TODO(mrry): Figure out how to do this without sleeping. time.sleep(0.1) for enqueue_op in enqueue_ops: - sess.run(enqueue_op) + self.evaluate(enqueue_op) results = [] def dequeue(): for _ in xrange(len(elems)): - results.append(sess.run(dequeued_t)) + results.append(self.evaluate(dequeued_t)) enqueue_thread = self.checkedThread(target=enqueue) dequeue_thread = self.checkedThread(target=dequeue) @@ -197,7 +197,7 @@ class RandomShuffleQueueTest(test.TestCase): results = [] for _ in xrange(len(elems)): - x, y = sess.run(dequeued_t) + x, y = self.evaluate(dequeued_t) results.append((x, y)) self.assertItemsEqual(elems, results) @@ -215,9 +215,9 @@ class RandomShuffleQueueTest(test.TestCase): self.assertEqual([], size.get_shape()) enqueue_op.run() - self.assertEqual([1], size.eval()) + self.assertEqual([1], self.evaluate(size)) dequeued_t.op.run() - self.assertEqual([0], size.eval()) + self.assertEqual([0], self.evaluate(size)) def testEnqueueMany(self): with self.cached_session(): @@ -241,9 +241,9 @@ class RandomShuffleQueueTest(test.TestCase): enqueue_op = q.enqueue_many((empty_t,)) size_t = q.size() - self.assertEqual(0, size_t.eval()) + self.assertEqual(0, self.evaluate(size_t)) enqueue_op.run() - self.assertEqual(0, size_t.eval()) + self.assertEqual(0, self.evaluate(size_t)) def testEmptyDequeueMany(self): with self.cached_session(): @@ -251,9 +251,9 @@ class RandomShuffleQueueTest(test.TestCase): enqueue_op = q.enqueue((10.0,)) dequeued_t = q.dequeue_many(0) - self.assertEqual([], dequeued_t.eval().tolist()) + self.assertEqual([], self.evaluate(dequeued_t).tolist()) enqueue_op.run() - self.assertEqual([], dequeued_t.eval().tolist()) + self.assertEqual([], self.evaluate(dequeued_t).tolist()) def testEmptyDequeueUpTo(self): with self.cached_session(): @@ -261,9 +261,9 @@ class RandomShuffleQueueTest(test.TestCase): enqueue_op = q.enqueue((10.0,)) dequeued_t = q.dequeue_up_to(0) - self.assertEqual([], dequeued_t.eval().tolist()) + self.assertEqual([], self.evaluate(dequeued_t).tolist()) enqueue_op.run() - self.assertEqual([], dequeued_t.eval().tolist()) + self.assertEqual([], self.evaluate(dequeued_t).tolist()) def testEmptyDequeueManyWithNoShape(self): with self.cached_session(): @@ -275,7 +275,7 @@ class RandomShuffleQueueTest(test.TestCase): # Expect the operation to fail due to the shape not being constrained. with self.assertRaisesOpError( "require the components to have specified shapes"): - dequeued_t.eval() + self.evaluate(dequeued_t) enqueue_op.run() @@ -284,7 +284,7 @@ class RandomShuffleQueueTest(test.TestCase): # elements enqueued. with self.assertRaisesOpError( "require the components to have specified shapes"): - dequeued_t.eval() + self.evaluate(dequeued_t) def testEmptyDequeueUpToWithNoShape(self): with self.cached_session(): @@ -296,7 +296,7 @@ class RandomShuffleQueueTest(test.TestCase): # Expect the operation to fail due to the shape not being constrained. with self.assertRaisesOpError( "require the components to have specified shapes"): - dequeued_t.eval() + self.evaluate(dequeued_t) enqueue_op.run() @@ -305,7 +305,7 @@ class RandomShuffleQueueTest(test.TestCase): # elements enqueued. with self.assertRaisesOpError( "require the components to have specified shapes"): - dequeued_t.eval() + self.evaluate(dequeued_t) def testMultiEnqueueMany(self): with self.cached_session() as sess: @@ -321,7 +321,7 @@ class RandomShuffleQueueTest(test.TestCase): results = [] for _ in range(8): - float_val, int_val = sess.run(dequeued_t) + float_val, int_val = self.evaluate(dequeued_t) results.append((float_val, [int_val[0], int_val[1]])) expected = list(zip(float_elems, int_elems)) * 2 self.assertItemsEqual(expected, results) @@ -335,7 +335,7 @@ class RandomShuffleQueueTest(test.TestCase): enqueue_op.run() - results = dequeued_t.eval().tolist() + results = self.evaluate(dequeued_t).tolist() results.extend(dequeued_t.eval()) self.assertItemsEqual(elems, results) @@ -348,7 +348,7 @@ class RandomShuffleQueueTest(test.TestCase): enqueue_op.run() - results = dequeued_t.eval().tolist() + results = self.evaluate(dequeued_t).tolist() results.extend(dequeued_t.eval()) self.assertItemsEqual(elems, results) @@ -368,20 +368,20 @@ class RandomShuffleQueueTest(test.TestCase): enqueue_op.run() results = [] - float_val, int_val = sess.run(dequeued_t) + float_val, int_val = self.evaluate(dequeued_t) self.assertEqual(float_val.shape, dequeued_t[0].get_shape()) self.assertEqual(int_val.shape, dequeued_t[1].get_shape()) results.extend(zip(float_val, int_val.tolist())) - float_val, int_val = sess.run(dequeued_t) + float_val, int_val = self.evaluate(dequeued_t) results.extend(zip(float_val, int_val.tolist())) - float_val, int_val = sess.run(dequeued_single_t) + float_val, int_val = self.evaluate(dequeued_single_t) self.assertEqual(float_val.shape, dequeued_single_t[0].get_shape()) self.assertEqual(int_val.shape, dequeued_single_t[1].get_shape()) results.append((float_val, int_val.tolist())) - float_val, int_val = sess.run(dequeued_single_t) + float_val, int_val = self.evaluate(dequeued_single_t) results.append((float_val, int_val.tolist())) self.assertItemsEqual(zip(float_elems, int_elems), results) @@ -402,21 +402,21 @@ class RandomShuffleQueueTest(test.TestCase): enqueue_op.run() results = [] - float_val, int_val = sess.run(dequeued_t) + float_val, int_val = self.evaluate(dequeued_t) # dequeue_up_to has undefined shape. self.assertEqual([None], dequeued_t[0].get_shape().as_list()) self.assertEqual([None, 2], dequeued_t[1].get_shape().as_list()) results.extend(zip(float_val, int_val.tolist())) - float_val, int_val = sess.run(dequeued_t) + float_val, int_val = self.evaluate(dequeued_t) results.extend(zip(float_val, int_val.tolist())) - float_val, int_val = sess.run(dequeued_single_t) + float_val, int_val = self.evaluate(dequeued_single_t) self.assertEqual(float_val.shape, dequeued_single_t[0].get_shape()) self.assertEqual(int_val.shape, dequeued_single_t[1].get_shape()) results.append((float_val, int_val.tolist())) - float_val, int_val = sess.run(dequeued_single_t) + float_val, int_val = self.evaluate(dequeued_single_t) results.append((float_val, int_val.tolist())) self.assertItemsEqual(zip(float_elems, int_elems), results) @@ -442,7 +442,7 @@ class RandomShuffleQueueTest(test.TestCase): # Enqueue 100 items in parallel on 10 threads. def enqueue(): - sess.run(enqueue_op) + self.evaluate(enqueue_op) threads = [self.checkedThread(target=enqueue) for _ in range(10)] for thread in threads: @@ -466,7 +466,7 @@ class RandomShuffleQueueTest(test.TestCase): dequeued_elems = [] def dequeue(): - dequeued_elems.extend(sess.run(dequeued_t)) + dequeued_elems.extend(self.evaluate(dequeued_t)) threads = [self.checkedThread(target=dequeue) for _ in range(10)] for thread in threads: @@ -489,7 +489,7 @@ class RandomShuffleQueueTest(test.TestCase): dequeued_elems = [] def dequeue(): - dequeued_elems.extend(sess.run(dequeued_t)) + dequeued_elems.extend(self.evaluate(dequeued_t)) threads = [self.checkedThread(target=dequeue) for _ in range(10)] for thread in threads: @@ -515,7 +515,7 @@ class RandomShuffleQueueTest(test.TestCase): dequeued_elems = [] def dequeue(dequeue_op): - dequeued_elems.extend(sess.run(dequeue_op)) + dequeued_elems.extend(self.evaluate(dequeue_op)) threads = [] for dequeue_op in dequeue_ops: @@ -539,10 +539,10 @@ class RandomShuffleQueueTest(test.TestCase): # The enqueue_op should run after the dequeue op has blocked. # TODO(mrry): Figure out how to do this without sleeping. time.sleep(0.1) - sess.run(enqueue_op) + self.evaluate(enqueue_op) def dequeue(): - dequeued_elems.extend(sess.run(dequeued_t).tolist()) + dequeued_elems.extend(self.evaluate(dequeued_t).tolist()) enqueue_thread = self.checkedThread(target=enqueue) dequeue_thread = self.checkedThread(target=dequeue) @@ -566,10 +566,10 @@ class RandomShuffleQueueTest(test.TestCase): # The enqueue_op should run after the dequeue op has blocked. # TODO(mrry): Figure out how to do this without sleeping. time.sleep(0.1) - sess.run(enqueue_op) + self.evaluate(enqueue_op) def dequeue(): - dequeued_elems.extend(sess.run(dequeued_t).tolist()) + dequeued_elems.extend(self.evaluate(dequeued_t).tolist()) enqueue_thread = self.checkedThread(target=enqueue) dequeue_thread = self.checkedThread(target=dequeue) @@ -649,7 +649,7 @@ class RandomShuffleQueueTest(test.TestCase): # Expect the operation to fail due to the queue being closed. with self.assertRaisesRegexp(errors_impl.OutOfRangeError, "is closed and has insufficient"): - dequeued_t.eval() + self.evaluate(dequeued_t) def testBlockingDequeueFromClosedQueue(self): with self.cached_session() as sess: @@ -665,18 +665,18 @@ class RandomShuffleQueueTest(test.TestCase): results = [] # Manually dequeue until we hit min_size. - results.append(sess.run(dequeued_t)) - results.append(sess.run(dequeued_t)) + results.append(self.evaluate(dequeued_t)) + results.append(self.evaluate(dequeued_t)) def blocking_dequeue(): - results.append(sess.run(dequeued_t)) - results.append(sess.run(dequeued_t)) + results.append(self.evaluate(dequeued_t)) + results.append(self.evaluate(dequeued_t)) self.assertItemsEqual(elems, results) # Expect the operation to fail due to the queue being closed. with self.assertRaisesRegexp(errors_impl.OutOfRangeError, "is closed and has insufficient"): - sess.run(dequeued_t) + self.evaluate(dequeued_t) dequeue_thread = self.checkedThread(target=blocking_dequeue) dequeue_thread.start() @@ -701,7 +701,7 @@ class RandomShuffleQueueTest(test.TestCase): # Expect the operation to fail due to the queue being closed. with self.assertRaisesRegexp(errors_impl.OutOfRangeError, "is closed and has insufficient"): - sess.run(dequeued_t) + self.evaluate(dequeued_t) finished.append(True) dequeue_thread = self.checkedThread(target=dequeue) @@ -727,12 +727,12 @@ class RandomShuffleQueueTest(test.TestCase): progress = [] # Must be mutable def dequeue(): - self.assertItemsEqual(elems, sess.run(dequeued_t)) + self.assertItemsEqual(elems, self.evaluate(dequeued_t)) progress.append(1) # Expect the operation to fail due to the queue being closed. with self.assertRaisesRegexp(errors_impl.OutOfRangeError, "is closed and has insufficient"): - sess.run(dequeued_t) + self.evaluate(dequeued_t) progress.append(2) self.assertEqual(len(progress), 0) @@ -763,9 +763,9 @@ class RandomShuffleQueueTest(test.TestCase): results = [] def dequeue(): - results.extend(sess.run(dequeued_t)) + results.extend(self.evaluate(dequeued_t)) self.assertEquals(3, len(results)) - results.extend(sess.run(dequeued_t)) + results.extend(self.evaluate(dequeued_t)) self.assertEquals(4, len(results)) dequeue_thread = self.checkedThread(target=dequeue) @@ -794,11 +794,11 @@ class RandomShuffleQueueTest(test.TestCase): results = [] def dequeue(): - results.extend(sess.run(dequeued_t)) + results.extend(self.evaluate(dequeued_t)) self.assertEquals(3, len(results)) # min_after_dequeue is 2, we ask for 3 elements, and we end up only # getting the remaining 1. - results.extend(sess.run(dequeued_t)) + results.extend(self.evaluate(dequeued_t)) self.assertEquals(4, len(results)) dequeue_thread = self.checkedThread(target=dequeue) @@ -824,16 +824,16 @@ class RandomShuffleQueueTest(test.TestCase): results = [] def dequeue(): - results.extend(sess.run(dequeued_t)) + results.extend(self.evaluate(dequeued_t)) self.assertEqual(len(results), 3) # Expect the operation to fail due to the queue being closed. with self.assertRaisesRegexp(errors_impl.OutOfRangeError, "is closed and has insufficient"): - sess.run(dequeued_t) + self.evaluate(dequeued_t) # While the last dequeue failed, we want to insure that it returns # any elements that it potentially reserved to dequeue. Thus the # next cleanup should return a single element. - results.extend(sess.run(cleanup_dequeue_t)) + results.extend(self.evaluate(cleanup_dequeue_t)) dequeue_thread = self.checkedThread(target=dequeue) dequeue_thread.start() @@ -854,7 +854,7 @@ class RandomShuffleQueueTest(test.TestCase): # Expect the operation to fail due to the queue being closed. with self.assertRaisesRegexp(errors_impl.OutOfRangeError, "is closed and has insufficient"): - sess.run(dequeued_t) + self.evaluate(dequeued_t) dequeue_thread = self.checkedThread(target=dequeue) dequeue_thread.start() @@ -874,7 +874,7 @@ class RandomShuffleQueueTest(test.TestCase): # Expect the operation to fail due to the queue being closed. with self.assertRaisesRegexp(errors_impl.OutOfRangeError, "is closed and has insufficient"): - sess.run(dequeued_t) + self.evaluate(dequeued_t) dequeue_thread = self.checkedThread(target=dequeue) dequeue_thread.start() @@ -922,7 +922,7 @@ class RandomShuffleQueueTest(test.TestCase): enqueue_op.run() def blocking_enqueue(): - sess.run(blocking_enqueue_op) + self.evaluate(blocking_enqueue_op) thread = self.checkedThread(target=blocking_enqueue) thread.start() @@ -950,7 +950,7 @@ class RandomShuffleQueueTest(test.TestCase): enqueue_op.run() def blocking_enqueue(): - sess.run(blocking_enqueue_op) + self.evaluate(blocking_enqueue_op) thread = self.checkedThread(target=blocking_enqueue) thread.start() @@ -987,11 +987,11 @@ class RandomShuffleQueueTest(test.TestCase): def blocking_enqueue(): # Expect the operation to succeed since it will complete # before the queue is closed. - sess.run(blocking_enqueue_op) + self.evaluate(blocking_enqueue_op) # Expect the operation to fail due to the queue being closed. with self.assertRaisesRegexp(errors_impl.CancelledError, "closed"): - sess.run(blocking_enqueue_op) + self.evaluate(blocking_enqueue_op) thread1 = self.checkedThread(target=blocking_enqueue) thread1.start() @@ -1001,7 +1001,7 @@ class RandomShuffleQueueTest(test.TestCase): time.sleep(0.1) def blocking_close(): - sess.run(close_op) + self.evaluate(close_op) thread2 = self.checkedThread(target=blocking_close) thread2.start() @@ -1032,7 +1032,7 @@ class RandomShuffleQueueTest(test.TestCase): def blocking_enqueue(): # This will block until the dequeue after the close. - sess.run(blocking_enqueue_op) + self.evaluate(blocking_enqueue_op) thread1 = self.checkedThread(target=blocking_enqueue) thread1.start() @@ -1040,7 +1040,7 @@ class RandomShuffleQueueTest(test.TestCase): # First blocking_enqueue_op of blocking_enqueue has enqueued 1 of 2 # elements, and is blocked waiting for one more element to be dequeue. for i in range(50): - queue_size = size_t.eval() + queue_size = self.evaluate(size_t) if queue_size == 4: break elif i == 49: @@ -1050,7 +1050,7 @@ class RandomShuffleQueueTest(test.TestCase): time.sleep(0.1) def blocking_close(): - sess.run(close_op) + self.evaluate(close_op) thread2 = self.checkedThread(target=blocking_close) thread2.start() @@ -1064,7 +1064,7 @@ class RandomShuffleQueueTest(test.TestCase): # At this point the close operation will complete, so the next enqueue # will fail. with self.assertRaisesRegexp(errors_impl.CancelledError, "closed"): - sess.run(blocking_enqueue_op) + self.evaluate(blocking_enqueue_op) def testSharedQueueSameSession(self): with self.cached_session(): @@ -1216,23 +1216,23 @@ class RandomShuffleQueueTest(test.TestCase): def _blockingDequeue(self, sess, dequeue_op): with self.assertRaisesOpError("was cancelled"): - sess.run(dequeue_op) + self.evaluate(dequeue_op) def _blockingDequeueMany(self, sess, dequeue_many_op): with self.assertRaisesOpError("was cancelled"): - sess.run(dequeue_many_op) + self.evaluate(dequeue_many_op) def _blockingDequeueUpTo(self, sess, dequeue_up_to_op): with self.assertRaisesOpError("was cancelled"): - sess.run(dequeue_up_to_op) + self.evaluate(dequeue_up_to_op) def _blockingEnqueue(self, sess, enqueue_op): with self.assertRaisesOpError("was cancelled"): - sess.run(enqueue_op) + self.evaluate(enqueue_op) def _blockingEnqueueMany(self, sess, enqueue_many_op): with self.assertRaisesOpError("was cancelled"): - sess.run(enqueue_many_op) + self.evaluate(enqueue_many_op) def testResetOfBlockingOperation(self): with self.cached_session() as sess: @@ -1383,7 +1383,7 @@ class RandomShuffleQueueTest(test.TestCase): def blocking_enqueue(): enq_done.append(False) # This will fill the queue and then block until enough dequeues happen. - sess.run(enq) + self.evaluate(enq) enq_done.append(True) thread = self.checkedThread(target=blocking_enqueue) @@ -1393,14 +1393,14 @@ class RandomShuffleQueueTest(test.TestCase): results = [] results.append(deq.eval()) # Will only complete after the enqueue starts. self.assertEqual(len(enq_done), 1) - self.assertEqual(sess.run(size_op), 5) + self.assertEqual(self.evaluate(size_op), 5) for _ in range(3): results.append(deq.eval()) time.sleep(0.1) self.assertEqual(len(enq_done), 1) - self.assertEqual(sess.run(size_op), 5) + self.assertEqual(self.evaluate(size_op), 5) # This dequeue will unblock the thread. results.append(deq.eval()) @@ -1426,7 +1426,7 @@ class RandomShuffleQueueTest(test.TestCase): def blocking_dequeue(): # Will only complete after 4 enqueues complete. - results.extend(sess.run(deq)) + results.extend(self.evaluate(deq)) thread = self.checkedThread(target=blocking_dequeue) thread.start() @@ -1435,7 +1435,7 @@ class RandomShuffleQueueTest(test.TestCase): # TODO(mrry): Figure out how to do this without sleeping. time.sleep(0.1) self.assertEqual(len(results), 0) - sess.run(enq) + self.evaluate(enq) # Enough enqueued to unblock the dequeue thread.join() diff --git a/tensorflow/python/kernel_tests/random/stateless_random_ops_test.py b/tensorflow/python/kernel_tests/random/stateless_random_ops_test.py index d57db3c5126..898f38444b7 100644 --- a/tensorflow/python/kernel_tests/random/stateless_random_ops_test.py +++ b/tensorflow/python/kernel_tests/random/stateless_random_ops_test.py @@ -24,6 +24,7 @@ import numpy as np from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes from tensorflow.python.framework import random_seed +from tensorflow.python.framework import test_util from tensorflow.python.ops import array_ops from tensorflow.python.ops import random_ops from tensorflow.python.ops import stateless_random_ops as stateless @@ -58,11 +59,11 @@ class StatelessOpsTest(test.TestCase): preseed = invert_philox(key, (seed[0], 0, seed[1], 0)).astype(np.uint64) preseed = preseed[::2] | preseed[1::2] << 32 random_seed.set_random_seed(seed[0]) - with self.test_session(use_gpu=True): + with test_util.use_gpu(): for stateless_op, stateful_op in cases: stateful = stateful_op(seed=seed[1]) pure = stateless_op(seed=preseed) - self.assertAllEqual(stateful.eval(), pure.eval()) + self.assertAllEqual(self.evaluate(stateful), self.evaluate(pure)) def _test_determinism(self, cases): # Stateless values should be equal iff the seeds are equal (roughly) @@ -128,23 +129,29 @@ class StatelessOpsTest(test.TestCase): yield (functools.partial(stateless.stateless_multinomial, **kwds), functools.partial(random_ops.multinomial, **kwds)) + @test_util.run_deprecated_v1 def testMatchFloat(self): self._test_match(self._float_cases()) + @test_util.run_deprecated_v1 def testMatchInt(self): self._test_match(self._int_cases()) + @test_util.run_deprecated_v1 def testMatchMultinomial(self): self._test_match(self._multinomial_cases()) + @test_util.run_deprecated_v1 def testDeterminismFloat(self): self._test_determinism( self._float_cases(shape_dtypes=(dtypes.int32, dtypes.int64))) + @test_util.run_deprecated_v1 def testDeterminismInt(self): self._test_determinism( self._int_cases(shape_dtypes=(dtypes.int32, dtypes.int64))) + @test_util.run_deprecated_v1 def testDeterminismMultinomial(self): self._test_determinism(self._multinomial_cases()) diff --git a/tensorflow/python/kernel_tests/reader_ops_test.py b/tensorflow/python/kernel_tests/reader_ops_test.py index ac9be56d63f..43d15817e97 100644 --- a/tensorflow/python/kernel_tests/reader_ops_test.py +++ b/tensorflow/python/kernel_tests/reader_ops_test.py @@ -28,6 +28,7 @@ import zlib from tensorflow.core.protobuf import config_pb2 from tensorflow.python.framework import dtypes from tensorflow.python.framework import errors_impl +from tensorflow.python.framework import test_util from tensorflow.python.lib.io import tf_record from tensorflow.python.ops import data_flow_ops from tensorflow.python.ops import io_ops @@ -140,147 +141,147 @@ class TFCompressionTestCase(test.TestCase): class IdentityReaderTest(test.TestCase): - def _ExpectRead(self, sess, key, value, expected): - k, v = sess.run([key, value]) + def _ExpectRead(self, key, value, expected): + k, v = self.evaluate([key, value]) self.assertAllEqual(expected, k) self.assertAllEqual(expected, v) + @test_util.run_deprecated_v1 def testOneEpoch(self): - with self.cached_session() as sess: - reader = io_ops.IdentityReader("test_reader") - work_completed = reader.num_work_units_completed() - produced = reader.num_records_produced() - queue = data_flow_ops.FIFOQueue(99, [dtypes.string], shapes=()) - queued_length = queue.size() - key, value = reader.read(queue) + reader = io_ops.IdentityReader("test_reader") + work_completed = reader.num_work_units_completed() + produced = reader.num_records_produced() + queue = data_flow_ops.FIFOQueue(99, [dtypes.string], shapes=()) + queued_length = queue.size() + key, value = reader.read(queue) - self.assertAllEqual(0, work_completed.eval()) - self.assertAllEqual(0, produced.eval()) - self.assertAllEqual(0, queued_length.eval()) + self.assertAllEqual(0, self.evaluate(work_completed)) + self.assertAllEqual(0, self.evaluate(produced)) + self.assertAllEqual(0, self.evaluate(queued_length)) - queue.enqueue_many([["A", "B", "C"]]).run() - queue.close().run() - self.assertAllEqual(3, queued_length.eval()) + self.evaluate(queue.enqueue_many([["A", "B", "C"]])) + self.evaluate(queue.close()) + self.assertAllEqual(3, self.evaluate(queued_length)) - self._ExpectRead(sess, key, value, b"A") - self.assertAllEqual(1, produced.eval()) + self._ExpectRead(key, value, b"A") + self.assertAllEqual(1, self.evaluate(produced)) - self._ExpectRead(sess, key, value, b"B") + self._ExpectRead(key, value, b"B") - self._ExpectRead(sess, key, value, b"C") - self.assertAllEqual(3, produced.eval()) - self.assertAllEqual(0, queued_length.eval()) + self._ExpectRead(key, value, b"C") + self.assertAllEqual(3, self.evaluate(produced)) + self.assertAllEqual(0, self.evaluate(queued_length)) - with self.assertRaisesOpError("is closed and has insufficient elements " - "\\(requested 1, current size 0\\)"): - sess.run([key, value]) + with self.assertRaisesOpError("is closed and has insufficient elements " + "\\(requested 1, current size 0\\)"): + self.evaluate([key, value]) - self.assertAllEqual(3, work_completed.eval()) - self.assertAllEqual(3, produced.eval()) - self.assertAllEqual(0, queued_length.eval()) + self.assertAllEqual(3, self.evaluate(work_completed)) + self.assertAllEqual(3, self.evaluate(produced)) + self.assertAllEqual(0, self.evaluate(queued_length)) + @test_util.run_deprecated_v1 def testMultipleEpochs(self): - with self.cached_session() as sess: - reader = io_ops.IdentityReader("test_reader") - queue = data_flow_ops.FIFOQueue(99, [dtypes.string], shapes=()) - enqueue = queue.enqueue_many([["DD", "EE"]]) - key, value = reader.read(queue) + reader = io_ops.IdentityReader("test_reader") + queue = data_flow_ops.FIFOQueue(99, [dtypes.string], shapes=()) + enqueue = queue.enqueue_many([["DD", "EE"]]) + key, value = reader.read(queue) - enqueue.run() - self._ExpectRead(sess, key, value, b"DD") - self._ExpectRead(sess, key, value, b"EE") - enqueue.run() - self._ExpectRead(sess, key, value, b"DD") - self._ExpectRead(sess, key, value, b"EE") - enqueue.run() - self._ExpectRead(sess, key, value, b"DD") - self._ExpectRead(sess, key, value, b"EE") - queue.close().run() - with self.assertRaisesOpError("is closed and has insufficient elements " - "\\(requested 1, current size 0\\)"): - sess.run([key, value]) + self.evaluate(enqueue) + self._ExpectRead(key, value, b"DD") + self._ExpectRead(key, value, b"EE") + self.evaluate(enqueue) + self._ExpectRead(key, value, b"DD") + self._ExpectRead(key, value, b"EE") + self.evaluate(enqueue) + self._ExpectRead(key, value, b"DD") + self._ExpectRead(key, value, b"EE") + self.evaluate(queue.close()) + with self.assertRaisesOpError("is closed and has insufficient elements " + "\\(requested 1, current size 0\\)"): + self.evaluate([key, value]) + @test_util.run_deprecated_v1 def testSerializeRestore(self): - with self.cached_session() as sess: - reader = io_ops.IdentityReader("test_reader") - produced = reader.num_records_produced() - queue = data_flow_ops.FIFOQueue(99, [dtypes.string], shapes=()) - queue.enqueue_many([["X", "Y", "Z"]]).run() - key, value = reader.read(queue) + reader = io_ops.IdentityReader("test_reader") + produced = reader.num_records_produced() + queue = data_flow_ops.FIFOQueue(99, [dtypes.string], shapes=()) + self.evaluate(queue.enqueue_many([["X", "Y", "Z"]])) + key, value = reader.read(queue) - self._ExpectRead(sess, key, value, b"X") - self.assertAllEqual(1, produced.eval()) - state = reader.serialize_state().eval() + self._ExpectRead(key, value, b"X") + self.assertAllEqual(1, self.evaluate(produced)) + state = self.evaluate(reader.serialize_state()) - self._ExpectRead(sess, key, value, b"Y") - self._ExpectRead(sess, key, value, b"Z") - self.assertAllEqual(3, produced.eval()) + self._ExpectRead(key, value, b"Y") + self._ExpectRead(key, value, b"Z") + self.assertAllEqual(3, self.evaluate(produced)) - queue.enqueue_many([["Y", "Z"]]).run() - queue.close().run() - reader.restore_state(state).run() - self.assertAllEqual(1, produced.eval()) - self._ExpectRead(sess, key, value, b"Y") - self._ExpectRead(sess, key, value, b"Z") - with self.assertRaisesOpError("is closed and has insufficient elements " - "\\(requested 1, current size 0\\)"): - sess.run([key, value]) - self.assertAllEqual(3, produced.eval()) + self.evaluate(queue.enqueue_many([["Y", "Z"]])) + self.evaluate(queue.close()) + self.evaluate(reader.restore_state(state)) + self.assertAllEqual(1, self.evaluate(produced)) + self._ExpectRead(key, value, b"Y") + self._ExpectRead(key, value, b"Z") + with self.assertRaisesOpError("is closed and has insufficient elements " + "\\(requested 1, current size 0\\)"): + self.evaluate([key, value]) + self.assertAllEqual(3, self.evaluate(produced)) - self.assertEqual(bytes, type(state)) + self.assertEqual(bytes, type(state)) - with self.assertRaises(ValueError): - reader.restore_state([]) + with self.assertRaises(ValueError): + reader.restore_state([]) - with self.assertRaises(ValueError): - reader.restore_state([state, state]) + with self.assertRaises(ValueError): + reader.restore_state([state, state]) - with self.assertRaisesOpError( - "Could not parse state for IdentityReader 'test_reader'"): - reader.restore_state(state[1:]).run() + with self.assertRaisesOpError( + "Could not parse state for IdentityReader 'test_reader'"): + self.evaluate(reader.restore_state(state[1:])) - with self.assertRaisesOpError( - "Could not parse state for IdentityReader 'test_reader'"): - reader.restore_state(state[:-1]).run() + with self.assertRaisesOpError( + "Could not parse state for IdentityReader 'test_reader'"): + self.evaluate(reader.restore_state(state[:-1])) - with self.assertRaisesOpError( - "Could not parse state for IdentityReader 'test_reader'"): - reader.restore_state(state + b"ExtraJunk").run() + with self.assertRaisesOpError( + "Could not parse state for IdentityReader 'test_reader'"): + self.evaluate(reader.restore_state(state + b"ExtraJunk")) - with self.assertRaisesOpError( - "Could not parse state for IdentityReader 'test_reader'"): - reader.restore_state(b"PREFIX" + state).run() + with self.assertRaisesOpError( + "Could not parse state for IdentityReader 'test_reader'"): + self.evaluate(reader.restore_state(b"PREFIX" + state)) - with self.assertRaisesOpError( - "Could not parse state for IdentityReader 'test_reader'"): - reader.restore_state(b"BOGUS" + state[5:]).run() + with self.assertRaisesOpError( + "Could not parse state for IdentityReader 'test_reader'"): + self.evaluate(reader.restore_state(b"BOGUS" + state[5:])) + @test_util.run_deprecated_v1 def testReset(self): - with self.cached_session() as sess: - reader = io_ops.IdentityReader("test_reader") - work_completed = reader.num_work_units_completed() - produced = reader.num_records_produced() - queue = data_flow_ops.FIFOQueue(99, [dtypes.string], shapes=()) - queued_length = queue.size() - key, value = reader.read(queue) + reader = io_ops.IdentityReader("test_reader") + work_completed = reader.num_work_units_completed() + produced = reader.num_records_produced() + queue = data_flow_ops.FIFOQueue(99, [dtypes.string], shapes=()) + queued_length = queue.size() + key, value = reader.read(queue) - queue.enqueue_many([["X", "Y", "Z"]]).run() - self._ExpectRead(sess, key, value, b"X") - self.assertLess(0, queued_length.eval()) - self.assertAllEqual(1, produced.eval()) + self.evaluate(queue.enqueue_many([["X", "Y", "Z"]])) + self._ExpectRead(key, value, b"X") + self.assertLess(0, self.evaluate(queued_length)) + self.assertAllEqual(1, self.evaluate(produced)) - self._ExpectRead(sess, key, value, b"Y") - self.assertLess(0, work_completed.eval()) - self.assertAllEqual(2, produced.eval()) + self._ExpectRead(key, value, b"Y") + self.assertLess(0, self.evaluate(work_completed)) + self.assertAllEqual(2, self.evaluate(produced)) - reader.reset().run() - self.assertAllEqual(0, work_completed.eval()) - self.assertAllEqual(0, produced.eval()) - self.assertAllEqual(1, queued_length.eval()) - self._ExpectRead(sess, key, value, b"Z") + self.evaluate(reader.reset()) + self.assertAllEqual(0, self.evaluate(work_completed)) + self.assertAllEqual(0, self.evaluate(produced)) + self.assertAllEqual(1, self.evaluate(queued_length)) + self._ExpectRead(key, value, b"Z") - queue.enqueue_many([["K", "L"]]).run() - self._ExpectRead(sess, key, value, b"K") + self.evaluate(queue.enqueue_many([["K", "L"]])) + self._ExpectRead(key, value, b"K") class WholeFileReaderTest(test.TestCase): @@ -301,44 +302,44 @@ class WholeFileReaderTest(test.TestCase): os.remove(fn) super(WholeFileReaderTest, self).tearDown() - def _ExpectRead(self, sess, key, value, index): - k, v = sess.run([key, value]) + def _ExpectRead(self, key, value, index): + k, v = self.evaluate([key, value]) self.assertAllEqual(compat.as_bytes(self._filenames[index]), k) self.assertAllEqual(self._content[index], v) + @test_util.run_deprecated_v1 def testOneEpoch(self): - with self.cached_session() as sess: - reader = io_ops.WholeFileReader("test_reader") - queue = data_flow_ops.FIFOQueue(99, [dtypes.string], shapes=()) - queue.enqueue_many([self._filenames]).run() - queue.close().run() - key, value = reader.read(queue) + reader = io_ops.WholeFileReader("test_reader") + queue = data_flow_ops.FIFOQueue(99, [dtypes.string], shapes=()) + self.evaluate(queue.enqueue_many([self._filenames])) + self.evaluate(queue.close()) + key, value = reader.read(queue) - self._ExpectRead(sess, key, value, 0) - self._ExpectRead(sess, key, value, 1) - self._ExpectRead(sess, key, value, 2) + self._ExpectRead(key, value, 0) + self._ExpectRead(key, value, 1) + self._ExpectRead(key, value, 2) - with self.assertRaisesOpError("is closed and has insufficient elements " - "\\(requested 1, current size 0\\)"): - sess.run([key, value]) + with self.assertRaisesOpError("is closed and has insufficient elements " + "\\(requested 1, current size 0\\)"): + self.evaluate([key, value]) + @test_util.run_deprecated_v1 def testInfiniteEpochs(self): - with self.cached_session() as sess: - reader = io_ops.WholeFileReader("test_reader") - queue = data_flow_ops.FIFOQueue(99, [dtypes.string], shapes=()) - enqueue = queue.enqueue_many([self._filenames]) - key, value = reader.read(queue) + reader = io_ops.WholeFileReader("test_reader") + queue = data_flow_ops.FIFOQueue(99, [dtypes.string], shapes=()) + enqueue = queue.enqueue_many([self._filenames]) + key, value = reader.read(queue) - enqueue.run() - self._ExpectRead(sess, key, value, 0) - self._ExpectRead(sess, key, value, 1) - enqueue.run() - self._ExpectRead(sess, key, value, 2) - self._ExpectRead(sess, key, value, 0) - self._ExpectRead(sess, key, value, 1) - enqueue.run() - self._ExpectRead(sess, key, value, 2) - self._ExpectRead(sess, key, value, 0) + self.evaluate(enqueue) + self._ExpectRead(key, value, 0) + self._ExpectRead(key, value, 1) + self.evaluate(enqueue) + self._ExpectRead(key, value, 2) + self._ExpectRead(key, value, 0) + self._ExpectRead(key, value, 1) + self.evaluate(enqueue) + self._ExpectRead(key, value, 2) + self._ExpectRead(key, value, 0) class TextLineReaderTest(test.TestCase): @@ -366,47 +367,48 @@ class TextLineReaderTest(test.TestCase): return filenames def _testOneEpoch(self, files): - with self.cached_session() as sess: - reader = io_ops.TextLineReader(name="test_reader") - queue = data_flow_ops.FIFOQueue(99, [dtypes.string], shapes=()) - key, value = reader.read(queue) + reader = io_ops.TextLineReader(name="test_reader") + queue = data_flow_ops.FIFOQueue(99, [dtypes.string], shapes=()) + key, value = reader.read(queue) - queue.enqueue_many([files]).run() - queue.close().run() - for i in range(self._num_files): - for j in range(self._num_lines): - k, v = sess.run([key, value]) - self.assertAllEqual("%s:%d" % (files[i], j + 1), compat.as_text(k)) - self.assertAllEqual(self._LineText(i, j), v) + self.evaluate(queue.enqueue_many([files])) + self.evaluate(queue.close()) + for i in range(self._num_files): + for j in range(self._num_lines): + k, v = self.evaluate([key, value]) + self.assertAllEqual("%s:%d" % (files[i], j + 1), compat.as_text(k)) + self.assertAllEqual(self._LineText(i, j), v) - with self.assertRaisesOpError("is closed and has insufficient elements " - "\\(requested 1, current size 0\\)"): - k, v = sess.run([key, value]) + with self.assertRaisesOpError("is closed and has insufficient elements " + "\\(requested 1, current size 0\\)"): + k, v = self.evaluate([key, value]) + @test_util.run_deprecated_v1 def testOneEpochLF(self): self._testOneEpoch(self._CreateFiles(crlf=False)) + @test_util.run_deprecated_v1 def testOneEpochCRLF(self): self._testOneEpoch(self._CreateFiles(crlf=True)) + @test_util.run_deprecated_v1 def testSkipHeaderLines(self): files = self._CreateFiles() - with self.cached_session() as sess: - reader = io_ops.TextLineReader(skip_header_lines=1, name="test_reader") - queue = data_flow_ops.FIFOQueue(99, [dtypes.string], shapes=()) - key, value = reader.read(queue) + reader = io_ops.TextLineReader(skip_header_lines=1, name="test_reader") + queue = data_flow_ops.FIFOQueue(99, [dtypes.string], shapes=()) + key, value = reader.read(queue) - queue.enqueue_many([files]).run() - queue.close().run() - for i in range(self._num_files): - for j in range(self._num_lines - 1): - k, v = sess.run([key, value]) - self.assertAllEqual("%s:%d" % (files[i], j + 2), compat.as_text(k)) - self.assertAllEqual(self._LineText(i, j + 1), v) + self.evaluate(queue.enqueue_many([files])) + self.evaluate(queue.close()) + for i in range(self._num_files): + for j in range(self._num_lines - 1): + k, v = self.evaluate([key, value]) + self.assertAllEqual("%s:%d" % (files[i], j + 2), compat.as_text(k)) + self.assertAllEqual(self._LineText(i, j + 1), v) - with self.assertRaisesOpError("is closed and has insufficient elements " - "\\(requested 1, current size 0\\)"): - k, v = sess.run([key, value]) + with self.assertRaisesOpError("is closed and has insufficient elements " + "\\(requested 1, current size 0\\)"): + k, v = self.evaluate([key, value]) class FixedLengthRecordReaderTest(TFCompressionTestCase): @@ -522,56 +524,55 @@ class FixedLengthRecordReaderTest(TFCompressionTestCase): # gap_bytes=hop_bytes-record_bytes def _TestOneEpoch(self, files, num_records, gap_bytes, encoding=None): hop_bytes = 0 if gap_bytes == 0 else self._record_bytes + gap_bytes - with self.cached_session() as sess: - reader = io_ops.FixedLengthRecordReader( - header_bytes=self._header_bytes, - record_bytes=self._record_bytes, - footer_bytes=self._footer_bytes, - hop_bytes=hop_bytes, - encoding=encoding, - name="test_reader") - queue = data_flow_ops.FIFOQueue(99, [dtypes.string], shapes=()) - key, value = reader.read(queue) + reader = io_ops.FixedLengthRecordReader( + header_bytes=self._header_bytes, + record_bytes=self._record_bytes, + footer_bytes=self._footer_bytes, + hop_bytes=hop_bytes, + encoding=encoding, + name="test_reader") + queue = data_flow_ops.FIFOQueue(99, [dtypes.string], shapes=()) + key, value = reader.read(queue) - queue.enqueue_many([files]).run() - queue.close().run() - for i in range(self._num_files): - for j in range(num_records): - k, v = sess.run([key, value]) - self.assertAllEqual("%s:%d" % (files[i], j), compat.as_text(k)) - self.assertAllEqual(self._Record(i, j), v) + self.evaluate(queue.enqueue_many([files])) + self.evaluate(queue.close()) + for i in range(self._num_files): + for j in range(num_records): + k, v = self.evaluate([key, value]) + self.assertAllEqual("%s:%d" % (files[i], j), compat.as_text(k)) + self.assertAllEqual(self._Record(i, j), v) - with self.assertRaisesOpError("is closed and has insufficient elements " - "\\(requested 1, current size 0\\)"): - k, v = sess.run([key, value]) + with self.assertRaisesOpError("is closed and has insufficient elements " + "\\(requested 1, current size 0\\)"): + k, v = self.evaluate([key, value]) def _TestOneEpochWithHopBytes(self, files, num_overlapped_records, encoding=None): - with self.cached_session() as sess: - reader = io_ops.FixedLengthRecordReader( - header_bytes=self._header_bytes, - record_bytes=self._record_bytes, - footer_bytes=self._footer_bytes, - hop_bytes=self._hop_bytes, - encoding=encoding, - name="test_reader") - queue = data_flow_ops.FIFOQueue(99, [dtypes.string], shapes=()) - key, value = reader.read(queue) + reader = io_ops.FixedLengthRecordReader( + header_bytes=self._header_bytes, + record_bytes=self._record_bytes, + footer_bytes=self._footer_bytes, + hop_bytes=self._hop_bytes, + encoding=encoding, + name="test_reader") + queue = data_flow_ops.FIFOQueue(99, [dtypes.string], shapes=()) + key, value = reader.read(queue) - queue.enqueue_many([files]).run() - queue.close().run() - for i in range(self._num_files): - for j in range(num_overlapped_records): - k, v = sess.run([key, value]) - self.assertAllEqual("%s:%d" % (files[i], j), compat.as_text(k)) - self.assertAllEqual(self._OverlappedRecord(i, j), v) + self.evaluate(queue.enqueue_many([files])) + self.evaluate(queue.close()) + for i in range(self._num_files): + for j in range(num_overlapped_records): + k, v = self.evaluate([key, value]) + self.assertAllEqual("%s:%d" % (files[i], j), compat.as_text(k)) + self.assertAllEqual(self._OverlappedRecord(i, j), v) - with self.assertRaisesOpError("is closed and has insufficient elements " - "\\(requested 1, current size 0\\)"): - k, v = sess.run([key, value]) + with self.assertRaisesOpError("is closed and has insufficient elements " + "\\(requested 1, current size 0\\)"): + k, v = self.evaluate([key, value]) + @test_util.run_deprecated_v1 def testOneEpoch(self): for num_records in [0, 7]: # gap_bytes=0: hop_bytes=0 @@ -580,6 +581,7 @@ class FixedLengthRecordReaderTest(TFCompressionTestCase): files = self._CreateFiles(num_records, gap_bytes) self._TestOneEpoch(files, num_records, gap_bytes) + @test_util.run_deprecated_v1 def testGzipOneEpoch(self): for num_records in [0, 7]: # gap_bytes=0: hop_bytes=0 @@ -588,6 +590,7 @@ class FixedLengthRecordReaderTest(TFCompressionTestCase): files = self._CreateGzipFiles(num_records, gap_bytes) self._TestOneEpoch(files, num_records, gap_bytes, encoding="GZIP") + @test_util.run_deprecated_v1 def testZlibOneEpoch(self): for num_records in [0, 7]: # gap_bytes=0: hop_bytes=0 @@ -596,17 +599,20 @@ class FixedLengthRecordReaderTest(TFCompressionTestCase): files = self._CreateZlibFiles(num_records, gap_bytes) self._TestOneEpoch(files, num_records, gap_bytes, encoding="ZLIB") + @test_util.run_deprecated_v1 def testOneEpochWithHopBytes(self): for num_overlapped_records in [0, 2]: files = self._CreateOverlappedRecordFiles(num_overlapped_records) self._TestOneEpochWithHopBytes(files, num_overlapped_records) + @test_util.run_deprecated_v1 def testGzipOneEpochWithHopBytes(self): for num_overlapped_records in [0, 2]: files = self._CreateGzipOverlappedRecordFiles(num_overlapped_records,) self._TestOneEpochWithHopBytes( files, num_overlapped_records, encoding="GZIP") + @test_util.run_deprecated_v1 def testZlibOneEpochWithHopBytes(self): for num_overlapped_records in [0, 2]: files = self._CreateZlibOverlappedRecordFiles(num_overlapped_records) @@ -619,90 +625,91 @@ class TFRecordReaderTest(TFCompressionTestCase): def setUp(self): super(TFRecordReaderTest, self).setUp() + @test_util.run_deprecated_v1 def testOneEpoch(self): files = self._CreateFiles() - with self.cached_session() as sess: - reader = io_ops.TFRecordReader(name="test_reader") - queue = data_flow_ops.FIFOQueue(99, [dtypes.string], shapes=()) - key, value = reader.read(queue) + reader = io_ops.TFRecordReader(name="test_reader") + queue = data_flow_ops.FIFOQueue(99, [dtypes.string], shapes=()) + key, value = reader.read(queue) - queue.enqueue_many([files]).run() - queue.close().run() - for i in range(self._num_files): - for j in range(self._num_records): - k, v = sess.run([key, value]) - self.assertTrue(compat.as_text(k).startswith("%s:" % files[i])) - self.assertAllEqual(self._Record(i, j), v) + self.evaluate(queue.enqueue_many([files])) + self.evaluate(queue.close()) + for i in range(self._num_files): + for j in range(self._num_records): + k, v = self.evaluate([key, value]) + self.assertTrue(compat.as_text(k).startswith("%s:" % files[i])) + self.assertAllEqual(self._Record(i, j), v) - with self.assertRaisesOpError("is closed and has insufficient elements " - "\\(requested 1, current size 0\\)"): - k, v = sess.run([key, value]) + with self.assertRaisesOpError("is closed and has insufficient elements " + "\\(requested 1, current size 0\\)"): + k, v = self.evaluate([key, value]) + @test_util.run_deprecated_v1 def testReadUpTo(self): files = self._CreateFiles() - with self.cached_session() as sess: - reader = io_ops.TFRecordReader(name="test_reader") - queue = data_flow_ops.FIFOQueue(99, [dtypes.string], shapes=()) - batch_size = 3 - key, value = reader.read_up_to(queue, batch_size) + reader = io_ops.TFRecordReader(name="test_reader") + queue = data_flow_ops.FIFOQueue(99, [dtypes.string], shapes=()) + batch_size = 3 + key, value = reader.read_up_to(queue, batch_size) - queue.enqueue_many([files]).run() - queue.close().run() - num_k = 0 - num_v = 0 + self.evaluate(queue.enqueue_many([files])) + self.evaluate(queue.close()) + num_k = 0 + num_v = 0 - while True: - try: - k, v = sess.run([key, value]) - # Test reading *up to* batch_size records - self.assertLessEqual(len(k), batch_size) - self.assertLessEqual(len(v), batch_size) - num_k += len(k) - num_v += len(v) - except errors_impl.OutOfRangeError: - break + while True: + try: + k, v = self.evaluate([key, value]) + # Test reading *up to* batch_size records + self.assertLessEqual(len(k), batch_size) + self.assertLessEqual(len(v), batch_size) + num_k += len(k) + num_v += len(v) + except errors_impl.OutOfRangeError: + break - # Test that we have read everything - self.assertEqual(self._num_files * self._num_records, num_k) - self.assertEqual(self._num_files * self._num_records, num_v) + # Test that we have read everything + self.assertEqual(self._num_files * self._num_records, num_k) + self.assertEqual(self._num_files * self._num_records, num_v) + @test_util.run_deprecated_v1 def testReadZlibFiles(self): options = tf_record.TFRecordOptions(TFRecordCompressionType.ZLIB) files = self._CreateFiles(options) - with self.cached_session() as sess: - reader = io_ops.TFRecordReader(name="test_reader", options=options) - queue = data_flow_ops.FIFOQueue(99, [dtypes.string], shapes=()) - key, value = reader.read(queue) + reader = io_ops.TFRecordReader(name="test_reader", options=options) + queue = data_flow_ops.FIFOQueue(99, [dtypes.string], shapes=()) + key, value = reader.read(queue) - queue.enqueue_many([files]).run() - queue.close().run() - for i in range(self._num_files): - for j in range(self._num_records): - k, v = sess.run([key, value]) - self.assertTrue(compat.as_text(k).startswith("%s:" % files[i])) - self.assertAllEqual(self._Record(i, j), v) + self.evaluate(queue.enqueue_many([files])) + self.evaluate(queue.close()) + for i in range(self._num_files): + for j in range(self._num_records): + k, v = self.evaluate([key, value]) + self.assertTrue(compat.as_text(k).startswith("%s:" % files[i])) + self.assertAllEqual(self._Record(i, j), v) + @test_util.run_deprecated_v1 def testReadGzipFiles(self): options = tf_record.TFRecordOptions(TFRecordCompressionType.GZIP) files = self._CreateFiles(options) - with self.cached_session() as sess: - reader = io_ops.TFRecordReader(name="test_reader", options=options) - queue = data_flow_ops.FIFOQueue(99, [dtypes.string], shapes=()) - key, value = reader.read(queue) + reader = io_ops.TFRecordReader(name="test_reader", options=options) + queue = data_flow_ops.FIFOQueue(99, [dtypes.string], shapes=()) + key, value = reader.read(queue) - queue.enqueue_many([files]).run() - queue.close().run() - for i in range(self._num_files): - for j in range(self._num_records): - k, v = sess.run([key, value]) - self.assertTrue(compat.as_text(k).startswith("%s:" % files[i])) - self.assertAllEqual(self._Record(i, j), v) + self.evaluate(queue.enqueue_many([files])) + self.evaluate(queue.close()) + for i in range(self._num_files): + for j in range(self._num_records): + k, v = self.evaluate([key, value]) + self.assertTrue(compat.as_text(k).startswith("%s:" % files[i])) + self.assertAllEqual(self._Record(i, j), v) class AsyncReaderTest(test.TestCase): + @test_util.run_deprecated_v1 def testNoDeadlockFromQueue(self): """Tests that reading does not block main execution threads.""" config = config_pb2.ConfigProto( @@ -724,7 +731,7 @@ class AsyncReaderTest(test.TestCase): thread_data.append(thread_data_t(t, queue, output)) # Start all readers. They are all blocked waiting for queue entries. - sess.run(variables.global_variables_initializer()) + self.evaluate(variables.global_variables_initializer()) for d in thread_data: d.thread.start() @@ -733,7 +740,7 @@ class AsyncReaderTest(test.TestCase): fname = os.path.join(self.get_temp_dir(), "deadlock.%s.txt" % i) with open(fname, "wb") as f: f.write(("file-%s" % i).encode()) - d.queue.enqueue_many([[fname]]).run() + self.evaluate(d.queue.enqueue_many([[fname]])) d.thread.join() self.assertEqual([[("file-%s" % i).encode()]], d.output) @@ -751,24 +758,25 @@ class LMDBReaderTest(test.TestCase): self.db_path = os.path.join(self.get_temp_dir(), "data.mdb") shutil.copy(path, self.db_path) + @test_util.run_deprecated_v1 def testReadFromFile(self): - with self.cached_session() as sess: - reader = io_ops.LMDBReader(name="test_read_from_file") - queue = data_flow_ops.FIFOQueue(99, [dtypes.string], shapes=()) - key, value = reader.read(queue) + reader = io_ops.LMDBReader(name="test_read_from_file") + queue = data_flow_ops.FIFOQueue(99, [dtypes.string], shapes=()) + key, value = reader.read(queue) - queue.enqueue([self.db_path]).run() - queue.close().run() - for i in range(10): - k, v = sess.run([key, value]) - self.assertAllEqual(compat.as_bytes(k), compat.as_bytes(str(i))) - self.assertAllEqual( - compat.as_bytes(v), compat.as_bytes(str(chr(ord("a") + i)))) + self.evaluate(queue.enqueue([self.db_path])) + self.evaluate(queue.close()) + for i in range(10): + k, v = self.evaluate([key, value]) + self.assertAllEqual(compat.as_bytes(k), compat.as_bytes(str(i))) + self.assertAllEqual( + compat.as_bytes(v), compat.as_bytes(str(chr(ord("a") + i)))) - with self.assertRaisesOpError("is closed and has insufficient elements " - "\\(requested 1, current size 0\\)"): - k, v = sess.run([key, value]) + with self.assertRaisesOpError("is closed and has insufficient elements " + "\\(requested 1, current size 0\\)"): + k, v = self.evaluate([key, value]) + @test_util.run_deprecated_v1 def testReadFromSameFile(self): with self.cached_session() as sess: reader1 = io_ops.LMDBReader(name="test_read_from_same_file1") @@ -782,30 +790,31 @@ class LMDBReaderTest(test.TestCase): threads = queue_runner_impl.start_queue_runners(sess, coord=coord) for _ in range(3): for _ in range(10): - k1, v1, k2, v2 = sess.run([key1, value1, key2, value2]) + k1, v1, k2, v2 = self.evaluate([key1, value1, key2, value2]) self.assertAllEqual(compat.as_bytes(k1), compat.as_bytes(k2)) self.assertAllEqual(compat.as_bytes(v1), compat.as_bytes(v2)) coord.request_stop() coord.join(threads) + @test_util.run_deprecated_v1 def testReadFromFolder(self): - with self.cached_session() as sess: - reader = io_ops.LMDBReader(name="test_read_from_folder") - queue = data_flow_ops.FIFOQueue(99, [dtypes.string], shapes=()) - key, value = reader.read(queue) + reader = io_ops.LMDBReader(name="test_read_from_folder") + queue = data_flow_ops.FIFOQueue(99, [dtypes.string], shapes=()) + key, value = reader.read(queue) - queue.enqueue([self.db_path]).run() - queue.close().run() - for i in range(10): - k, v = sess.run([key, value]) - self.assertAllEqual(compat.as_bytes(k), compat.as_bytes(str(i))) - self.assertAllEqual( - compat.as_bytes(v), compat.as_bytes(str(chr(ord("a") + i)))) + self.evaluate(queue.enqueue([self.db_path])) + self.evaluate(queue.close()) + for i in range(10): + k, v = self.evaluate([key, value]) + self.assertAllEqual(compat.as_bytes(k), compat.as_bytes(str(i))) + self.assertAllEqual( + compat.as_bytes(v), compat.as_bytes(str(chr(ord("a") + i)))) - with self.assertRaisesOpError("is closed and has insufficient elements " - "\\(requested 1, current size 0\\)"): - k, v = sess.run([key, value]) + with self.assertRaisesOpError("is closed and has insufficient elements " + "\\(requested 1, current size 0\\)"): + k, v = self.evaluate([key, value]) + @test_util.run_deprecated_v1 def testReadFromFileRepeatedly(self): with self.cached_session() as sess: reader = io_ops.LMDBReader(name="test_read_from_file_repeated") @@ -819,7 +828,7 @@ class LMDBReaderTest(test.TestCase): for _ in range(3): # Go over all 10 records each time. for j in range(10): - k, v = sess.run([key, value]) + k, v = self.evaluate([key, value]) self.assertAllEqual(compat.as_bytes(k), compat.as_bytes(str(j))) self.assertAllEqual( compat.as_bytes(v), compat.as_bytes(str(chr(ord("a") + j)))) diff --git a/tensorflow/python/kernel_tests/record_input_test.py b/tensorflow/python/kernel_tests/record_input_test.py index ebb9872f226..ad8188b372f 100644 --- a/tensorflow/python/kernel_tests/record_input_test.py +++ b/tensorflow/python/kernel_tests/record_input_test.py @@ -20,6 +20,7 @@ from __future__ import print_function import os +from tensorflow.python.framework import test_util from tensorflow.python.framework.errors_impl import NotFoundError from tensorflow.python.lib.io import tf_record from tensorflow.python.ops import data_flow_ops @@ -54,7 +55,7 @@ class RecordInputOpTest(test.TestCase): batch_size=1, name="record_input").get_yield_op() - self.assertEqual(sess.run(yield_op), b"0000000000") + self.assertEqual(self.evaluate(yield_op), b"0000000000") def testRecordInputSimpleGzip(self): with self.cached_session() as sess: @@ -73,7 +74,7 @@ class RecordInputOpTest(test.TestCase): compression_type=tf_record.TFRecordCompressionType.GZIP).get_yield_op( ) - self.assertEqual(sess.run(yield_op), b"0000000000") + self.assertEqual(self.evaluate(yield_op), b"0000000000") def testRecordInputSimpleZlib(self): with self.cached_session() as sess: @@ -92,8 +93,9 @@ class RecordInputOpTest(test.TestCase): compression_type=tf_record.TFRecordCompressionType.ZLIB).get_yield_op( ) - self.assertEqual(sess.run(yield_op), b"0000000000") + self.assertEqual(self.evaluate(yield_op), b"0000000000") + @test_util.run_deprecated_v1 def testRecordInputEpochs(self): files = 100 records_per_file = 100 @@ -117,7 +119,7 @@ class RecordInputOpTest(test.TestCase): for _ in range(3): epoch_set = set() for _ in range(int(files * records_per_file / batches)): - op_list = sess.run(yield_op) + op_list = self.evaluate(yield_op) self.assertTrue(len(op_list) is batches) for r in op_list: self.assertTrue(r[0] not in epoch_set) @@ -138,16 +140,18 @@ class RecordInputOpTest(test.TestCase): yield_op = records.get_yield_op() for _ in range(50): - sess.run(yield_op) + self.evaluate(yield_op) + @test_util.run_deprecated_v1 def testEmptyGlob(self): with self.cached_session() as sess: record_input = data_flow_ops.RecordInput(file_pattern="foo") yield_op = record_input.get_yield_op() - sess.run(variables.global_variables_initializer()) + self.evaluate(variables.global_variables_initializer()) with self.assertRaises(NotFoundError): - sess.run(yield_op) + self.evaluate(yield_op) + @test_util.run_deprecated_v1 def testBufferTooSmall(self): files = 10 records_per_file = 10 @@ -171,7 +175,7 @@ class RecordInputOpTest(test.TestCase): for _ in range(3): epoch_set = set() for _ in range(int(files * records_per_file / batches)): - op_list = sess.run(yield_op) + op_list = self.evaluate(yield_op) self.assertTrue(len(op_list) is batches) for r in op_list: self.assertTrue(r[0] not in epoch_set) diff --git a/tensorflow/python/kernel_tests/reduce_benchmark_test.py b/tensorflow/python/kernel_tests/reduce_benchmark_test.py index 3a2fb81157d..ef9c4c350fd 100644 --- a/tensorflow/python/kernel_tests/reduce_benchmark_test.py +++ b/tensorflow/python/kernel_tests/reduce_benchmark_test.py @@ -81,7 +81,7 @@ class ReduceBenchmarks(test.Benchmark): grad, = gradients_impl.gradients(reduction, tensor) def fn(): - sess.run(grad.op) + self.evaluate(grad.op) self._run(fn, 10000) @@ -98,7 +98,7 @@ class ReduceBenchmarks(test.Benchmark): grad, = gradients_impl.gradients(reduction, tensor) def fn(): - sess.run(grad.op) + self.evaluate(grad.op) self._run(fn, 10000) diff --git a/tensorflow/python/kernel_tests/reduce_join_op_test.py b/tensorflow/python/kernel_tests/reduce_join_op_test.py index 3bb4986313d..49b6620779e 100644 --- a/tensorflow/python/kernel_tests/reduce_join_op_test.py +++ b/tensorflow/python/kernel_tests/reduce_join_op_test.py @@ -25,6 +25,7 @@ from six.moves import xrange # pylint: disable=redefined-builtin from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes +from tensorflow.python.framework import test_util from tensorflow.python.ops import array_ops from tensorflow.python.ops import string_ops from tensorflow.python.platform import test @@ -119,7 +120,7 @@ class ReduceJoinTest(UnicodeTestCase): axis=axis, keep_dims=keep_dims, separator=separator) - output_array = output.eval() + output_array = self.evaluate(output) self.assertAllEqualUnicode(truth, output_array) self.assertAllEqual(truth_shape, output.get_shape()) @@ -149,10 +150,10 @@ class ReduceJoinTest(UnicodeTestCase): if not axis: truth = constant_op.constant(truth) truth_squeezed = array_ops.squeeze(truth, axis=axis) - output_array = output.eval() - output_keep_dims_array = output_keep_dims.eval() - truth_array = truth.eval() - truth_squeezed_array = truth_squeezed.eval() + output_array = self.evaluate(output) + output_keep_dims_array = self.evaluate(output_keep_dims) + truth_array = self.evaluate(truth) + truth_squeezed_array = self.evaluate(truth_squeezed) self.assertAllEqualUnicode(truth_array, output_keep_dims_array) self.assertAllEqualUnicode(truth_squeezed_array, output_array) self.assertAllEqual(truth.get_shape(), output_keep_dims.get_shape()) @@ -230,6 +231,7 @@ class ReduceJoinTest(UnicodeTestCase): axis=1, separator=" ") + @test_util.run_deprecated_v1 def testUnknownShape(self): input_array = [["a"], ["b"]] truth = ["ab"] @@ -241,6 +243,7 @@ class ReduceJoinTest(UnicodeTestCase): self.assertAllEqualUnicode(truth, output_array) self.assertAllEqual(truth_shape, reduced.get_shape()) + @test_util.run_deprecated_v1 def testUnknownIndices(self): input_array = [["this", "is", "a", "test"], ["please", "do", "not", "panic"]] @@ -297,6 +300,7 @@ class ReduceJoinTest(UnicodeTestCase): for permutation in itertools.permutations(xrange(num_dims), i): self._testMultipleReduceJoin(input_array, axis=permutation) + @test_util.run_deprecated_v1 def testInvalidReductionIndices(self): with self.cached_session(): with self.assertRaisesRegexp(ValueError, "Invalid reduction dim"): @@ -318,13 +322,14 @@ class ReduceJoinTest(UnicodeTestCase): # Reduction that drops the dim of size 0. output = string_ops.reduce_join(inputs=inputs, axis=0) - self.assertAllEqualUnicode([""], output.eval()) + self.assertAllEqualUnicode([""], self.evaluate(output)) # Reduction that keeps the dim of size 0. output = string_ops.reduce_join(inputs=inputs, axis=1) - output_shape = output.eval().shape + output_shape = self.evaluate(output).shape self.assertAllEqual([0], output_shape) + @test_util.run_deprecated_v1 def testInvalidArgsUnknownShape(self): with self.cached_session(): placeholder = array_ops.placeholder(dtypes.string, name="placeholder") @@ -335,6 +340,7 @@ class ReduceJoinTest(UnicodeTestCase): with self.assertRaisesOpError("Duplicate reduction dimension 1"): duplicate_index.eval(feed_dict={placeholder.name: [[""]]}) + @test_util.run_deprecated_v1 def testInvalidArgsUnknownIndices(self): with self.cached_session(): placeholder = array_ops.placeholder(dtypes.int32, name="placeholder") diff --git a/tensorflow/python/kernel_tests/reduction_ops_test.py b/tensorflow/python/kernel_tests/reduction_ops_test.py index 2ac3996e25b..67a89461f3a 100644 --- a/tensorflow/python/kernel_tests/reduction_ops_test.py +++ b/tensorflow/python/kernel_tests/reduction_ops_test.py @@ -27,6 +27,7 @@ from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops from tensorflow.python.framework import tensor_shape +from tensorflow.python.framework import test_util from tensorflow.python.ops import array_ops from tensorflow.python.ops import gradient_checker from tensorflow.python.ops import math_ops @@ -60,6 +61,7 @@ class ReducedShapeTest(test.TestCase): output = math_ops.reduced_shape(shape, axes=axes) self.assertAllEqual(output.eval(), result) + @test_util.run_deprecated_v1 def testSimple(self): with self.cached_session(): self._check([3], [], [3]) @@ -69,6 +71,7 @@ class ReducedShapeTest(test.TestCase): self._check([5, 3], [1], [5, 1]) self._check([5, 3], [0, 1], [1, 1]) + @test_util.run_deprecated_v1 def testZeros(self): """Check that reduced_shape does the right thing with zero dimensions.""" with self.cached_session(): @@ -83,6 +86,7 @@ class ReducedShapeTest(test.TestCase): self._check([3, 0], [1], [3, 1]) self._check([3, 0], [0, 1], [1, 1]) + @test_util.run_deprecated_v1 def testNegAxes(self): with self.cached_session(): self._check([10, 10, 10], [-1], [10, 10, 1]) @@ -94,6 +98,7 @@ class ReducedShapeTest(test.TestCase): class ReductionUnknownShape(test.TestCase): + @test_util.run_deprecated_v1 def testBasic(self): with self.cached_session(): for dtype, reductions in [(dtypes.float32, @@ -185,9 +190,10 @@ class SumReductionTest(BaseReductionTest): for dtype in [dtypes.int64, dtypes.int32]: with self.cached_session(use_gpu=True) as sess: v = math_ops.reduce_sum([0, 0], constant_op.constant(0, dtype=dtype)) - tf_v = sess.run(v) + tf_v = self.evaluate(v) self.assertAllEqual(tf_v, 0) + @test_util.run_deprecated_v1 def testInfinity(self): for dtype in [np.float32, np.float64]: for special_value_x in [-np.inf, np.inf]: @@ -195,11 +201,13 @@ class SumReductionTest(BaseReductionTest): np_arr = np.array([special_value_x, special_value_y]).astype(dtype) self._compareAll(np_arr, None) + @test_util.run_deprecated_v1 def testInt32(self): for rank in range(1, _MAX_RANK + 1): np_arr = self._makeIncremental((2,) * rank, dtypes.int32) self._compareAllAxes(np_arr) + @test_util.run_deprecated_v1 def testFloat16(self): for rank in range(1, _MAX_RANK + 1): np_arr = self._makeIncremental((2,) * rank, dtypes.float16) @@ -216,9 +224,10 @@ class SumReductionTest(BaseReductionTest): tf_arr = variables.Variable(arr) variables.global_variables_initializer().run() tf_mean = math_ops.reduce_mean(tf_arr, 0, False) - tf_out_mean = sess.run(tf_mean) + tf_out_mean = self.evaluate(tf_mean) self.assertAllClose(tf_out_mean, 1.) + @test_util.run_deprecated_v1 def testFloat32(self): for rank in range(1, _MAX_RANK + 1): np_arr = self._makeIncremental((2,) * rank, dtypes.float32) @@ -238,7 +247,7 @@ class SumReductionTest(BaseReductionTest): with self.session(graph=ops.Graph(), use_gpu=True) as sess: tf_row_sum = self._tf_reduce(arr, 1, False) tf_col_sum = self._tf_reduce(arr, 0, False) - tf_out_row, tf_out_col = sess.run([tf_row_sum, tf_col_sum]) + tf_out_row, tf_out_col = self.evaluate([tf_row_sum, tf_col_sum]) self.assertAllClose(col_sum, tf_out_col) self.assertAllClose(row_sum, tf_out_row) @@ -252,25 +261,29 @@ class SumReductionTest(BaseReductionTest): with self.session(graph=ops.Graph(), use_gpu=True) as sess: tf_sum_xz = self._tf_reduce(arr, [0, 2], False) tf_sum_y = self._tf_reduce(arr, 1, False) - tf_out_sum_xz, tf_out_sum_y = sess.run([tf_sum_xz, tf_sum_y]) + tf_out_sum_xz, tf_out_sum_y = self.evaluate([tf_sum_xz, tf_sum_y]) self.assertAllClose(sum_y, tf_out_sum_y) self.assertAllClose(sum_xz, tf_out_sum_xz) + @test_util.run_deprecated_v1 def testFloat64(self): for rank in range(1, _MAX_RANK + 1): np_arr = self._makeIncremental((2,) * rank, dtypes.float64) self._compareAllAxes(np_arr) + @test_util.run_deprecated_v1 def testComplex64(self): for rank in range(1, _MAX_RANK + 1): np_arr = self._makeIncremental((2,) * rank, dtypes.complex64) self._compareAllAxes(np_arr) + @test_util.run_deprecated_v1 def testComplex128(self): for rank in range(1, _MAX_RANK + 1): np_arr = self._makeIncremental((2,) * rank, dtypes.complex128) self._compareAllAxes(np_arr) + @test_util.run_deprecated_v1 def testInvalidIndex(self): np_arr = np.arange(0, 10).reshape([2, 5]).astype(np.float32) input_tensor = ops.convert_to_tensor(np_arr) @@ -284,6 +297,7 @@ class SumReductionTest(BaseReductionTest): ValueError, lambda e: "Invalid reduction dimension" in str(e)): math_ops.reduce_sum(input_tensor, [0, 2]) + @test_util.run_deprecated_v1 def testPartialShapes(self): np.random.seed(1618) @@ -317,6 +331,7 @@ class SumReductionTest(BaseReductionTest): c_unknown_indices, unknown_indices, keepdims=True) self.assertEqual(2, s_unknown_indices_keep.get_shape().rank) + @test_util.run_deprecated_v1 def testWrongShapeForReductionIndices(self): reduction_axes = [[1], [2]] c_unknown = array_ops.placeholder(dtypes.float32) @@ -326,6 +341,7 @@ class SumReductionTest(BaseReductionTest): # Int64?? + @test_util.run_deprecated_v1 def testGradient(self): for dtype in [ dtypes.float32, dtypes.float64, dtypes.complex64, dtypes.complex128 @@ -333,6 +349,7 @@ class SumReductionTest(BaseReductionTest): x = self._makeIncremental([2, 3, 4, 2], dtype) self._compareGradientAxes(x) + @test_util.run_deprecated_v1 def testHighRank(self): # Do a bunch of random high dimensional reductions np.random.seed(42) @@ -350,11 +367,13 @@ class SumReductionTest(BaseReductionTest): np.arange(1, rank, 2)): self._compareAll(data, axes) + @test_util.run_deprecated_v1 def testExpand(self): # Reduce an empty tensor to a nonempty tensor x = np.zeros((5, 0)) self._compareAll(x, [1]) + @test_util.run_deprecated_v1 def testEmptyGradients(self): with self.session(use_gpu=True): x = array_ops.zeros([0, 3]) @@ -362,6 +381,7 @@ class SumReductionTest(BaseReductionTest): error = gradient_checker.compute_gradient_error(x, [0, 3], y, [0]) self.assertEqual(error, 0) + @test_util.run_deprecated_v1 def testDegenerate(self): with self.session(use_gpu=True): for dtype in (dtypes.float16, dtypes.float32, dtypes.float64, @@ -400,9 +420,10 @@ class MeanReductionTest(BaseReductionTest): for dtype in [dtypes.int64, dtypes.int32]: with self.cached_session(use_gpu=True) as sess: v = math_ops.reduce_mean([0, 0], constant_op.constant(0, dtype=dtype)) - tf_v = sess.run(v) + tf_v = self.evaluate(v) self.assertAllEqual(tf_v, 0) + @test_util.run_deprecated_v1 def testInfinity(self): for dtype in [np.float32, np.float64]: for special_value_x in [-np.inf, np.inf]: @@ -410,37 +431,44 @@ class MeanReductionTest(BaseReductionTest): np_arr = np.array([special_value_x, special_value_y]).astype(dtype) self._compareAll(np_arr, None) + @test_util.run_deprecated_v1 def testInt32(self): for rank in range(1, _MAX_RANK + 1): np_arr = self._makeIncremental((2,) * rank, dtypes.int32) self._compareAllAxes(np_arr) + @test_util.run_deprecated_v1 def testFloat32(self): for rank in range(1, _MAX_RANK + 1): np_arr = self._makeIncremental((2,) * rank, dtypes.float32) self._compareAllAxes(np_arr) + @test_util.run_deprecated_v1 def testFloat64(self): for rank in range(1, _MAX_RANK + 1): np_arr = self._makeIncremental((2,) * rank, dtypes.float64) self._compareAllAxes(np_arr) + @test_util.run_deprecated_v1 def testComplex64(self): for rank in range(1, _MAX_RANK + 1): np_arr = self._makeIncremental((2,) * rank, dtypes.complex64) self._compareAllAxes(np_arr) + @test_util.run_deprecated_v1 def testComplex128(self): for rank in range(1, _MAX_RANK + 1): np_arr = self._makeIncremental((2,) * rank, dtypes.complex128) self._compareAllAxes(np_arr) + @test_util.run_deprecated_v1 def testGradient(self): s = [2, 3, 4, 2] for dtype in [dtypes.float32, dtypes.float64]: x = self._makeIncremental(s, dtype) self._compareGradientAxes(x, rtol=1e-3, atol=1e-3) + @test_util.run_deprecated_v1 def testEmptyGradients(self): with self.session(use_gpu=True): x = array_ops.zeros([0, 3]) @@ -448,6 +476,7 @@ class MeanReductionTest(BaseReductionTest): error = gradient_checker.compute_gradient_error(x, [0, 3], y, [0]) self.assertEqual(error, 0) + @test_util.run_deprecated_v1 def testDegenerate(self): with self.session(use_gpu=True): for dtype in (dtypes.float16, dtypes.float32, dtypes.float64): @@ -473,9 +502,10 @@ class ProdReductionTest(BaseReductionTest): for dtype in [dtypes.int64, dtypes.int32]: with self.cached_session(use_gpu=True) as sess: v = math_ops.reduce_prod([0, 0], constant_op.constant(0, dtype=dtype)) - tf_v = sess.run(v) + tf_v = self.evaluate(v) self.assertAllEqual(tf_v, 0) + @test_util.run_deprecated_v1 def testInfinity(self): for dtype in [np.float32, np.float64]: for special_value_x in [-np.inf, np.inf]: @@ -483,6 +513,7 @@ class ProdReductionTest(BaseReductionTest): np_arr = np.array([special_value_x, special_value_y]).astype(dtype) self._compareAll(np_arr, None) + @test_util.run_deprecated_v1 def testInt32(self): # Numpy automatically upgrades the type of np.prod from int32 to int64, so # Numpy does not overflow an int32 np.prod while TensorFlow does. To avoid @@ -491,26 +522,31 @@ class ProdReductionTest(BaseReductionTest): np_arr = self._makeIncremental((2,) * rank, dtypes.int32) / 2 self._compareAllAxes(np_arr) + @test_util.run_deprecated_v1 def testFloat32(self): for rank in range(1, _MAX_RANK + 1): np_arr = self._makeIncremental((2,) * rank, dtypes.float32) self._compareAllAxes(np_arr) + @test_util.run_deprecated_v1 def testFloat64(self): for rank in range(1, _MAX_RANK + 1): np_arr = self._makeIncremental((2,) * rank, dtypes.float64) self._compareAllAxes(np_arr) + @test_util.run_deprecated_v1 def testComplex64(self): for rank in range(1, _MAX_RANK + 1): np_arr = self._makeIncremental((2,) * rank, dtypes.complex64) self._compareAllAxes(np_arr) + @test_util.run_deprecated_v1 def testComplex128(self): for rank in range(1, _MAX_RANK + 1): np_arr = self._makeIncremental((2,) * rank, dtypes.complex128) self._compareAllAxes(np_arr) + @test_util.run_deprecated_v1 def testGradientWithZeros(self): s = [2, 3, 4, 2] x = self._makeIncremental(s, dtypes.float32) / 20. @@ -533,6 +569,7 @@ class ProdReductionTest(BaseReductionTest): x4[:, :, :, :] = 0 self._compareGradientAxes(x4, rtol=1e-3, atol=1e-3) + @test_util.run_deprecated_v1 def testEmptyGradients(self): with self.session(use_gpu=True): x = array_ops.zeros([0, 3]) @@ -540,6 +577,7 @@ class ProdReductionTest(BaseReductionTest): error = gradient_checker.compute_gradient_error(x, [0, 3], y, [0]) self.assertEqual(error, 0) + @test_util.run_deprecated_v1 def testDegenerate(self): with self.session(use_gpu=True): for dtype in (dtypes.float16, dtypes.float32, dtypes.float64): @@ -562,7 +600,7 @@ class MinReductionTest(test.TestCase): if reduction_axes is not None: reduction_axes = np.array(reduction_axes).astype(np.int32) tf_ans = math_ops.reduce_min(x, reduction_axes, keepdims) - out = tf_ans.eval() + out = self.evaluate(tf_ans) self.assertAllClose(np_ans, out) self.assertShapeEqual(np_ans, tf_ans) @@ -576,9 +614,10 @@ class MinReductionTest(test.TestCase): for dtype in [dtypes.int64, dtypes.int32]: with self.cached_session(use_gpu=True) as sess: v = math_ops.reduce_min([0, 0], constant_op.constant(0, dtype=dtype)) - tf_v = sess.run(v) + tf_v = self.evaluate(v) self.assertAllEqual(tf_v, 0) + @test_util.run_deprecated_v1 def testInfinity(self): for dtype in [np.float32, np.float64]: for special_value_x in [-np.inf, np.inf]: @@ -614,6 +653,7 @@ class MinReductionTest(test.TestCase): self._compareAll(np_arr, [0, 2]) self._compareAll(np_arr, [0, 1, 2]) + @test_util.run_deprecated_v1 def testGradient(self): s = [2, 3, 4, 2] x = np.arange(1.0, 49.0).reshape(s).astype(np.float64) @@ -624,6 +664,7 @@ class MinReductionTest(test.TestCase): t, s, su, [2, 2], x_init_value=x, delta=1) self.assertAllClose(jacob_t, jacob_n, rtol=1e-8, atol=1e-8) + @test_util.run_deprecated_v1 def testGradient2(self): s = [2, 3, 4, 2] x = np.arange(1.0, 49.0).reshape(s).astype(np.float64) @@ -634,6 +675,7 @@ class MinReductionTest(test.TestCase): t, s, su, [2, 4, 2], x_init_value=x, delta=1) self.assertAllClose(jacob_t, jacob_n, rtol=1e-8, atol=1e-8) + @test_util.run_deprecated_v1 def testGradient3(self): s = [2, 3, 4, 2] x = np.arange(1.0, 49.0).reshape(s).astype(np.float64) @@ -644,6 +686,7 @@ class MinReductionTest(test.TestCase): t, s, su, [2, 3, 2], x_init_value=x, delta=1) self.assertAllClose(jacob_t, jacob_n, rtol=1e-8, atol=1e-8) + @test_util.run_deprecated_v1 def testGradient4(self): s = [2, 3, 4, 2] x = np.arange(1.0, 49.0).reshape(s).astype(np.float64) @@ -654,6 +697,7 @@ class MinReductionTest(test.TestCase): t, s, su, [1], x_init_value=x, delta=1) self.assertAllClose(jacob_t, jacob_n, rtol=1e-8, atol=1e-8) + @test_util.run_deprecated_v1 def testEmptyGradients(self): with self.cached_session(): x = array_ops.zeros([0, 3]) @@ -675,7 +719,7 @@ class MaxReductionTest(test.TestCase): if reduction_axes is not None: reduction_axes = np.array(reduction_axes).astype(np.int32) tf_ans = math_ops.reduce_max(x, reduction_axes, keepdims) - out = tf_ans.eval() + out = self.evaluate(tf_ans) self.assertAllClose(np_ans, out) self.assertShapeEqual(np_ans, tf_ans) @@ -689,9 +733,10 @@ class MaxReductionTest(test.TestCase): for dtype in [dtypes.int64, dtypes.int32]: with self.cached_session(use_gpu=True) as sess: v = math_ops.reduce_max([0, 0], constant_op.constant(0, dtype=dtype)) - tf_v = sess.run(v) + tf_v = self.evaluate(v) self.assertAllEqual(tf_v, 0) + @test_util.run_deprecated_v1 def testInfinity(self): for dtype in [np.float32, np.float64]: for special_value_x in [-np.inf, np.inf]: @@ -741,6 +786,7 @@ class MaxReductionTest(test.TestCase): self._compareAll(np_arr, [0, 2]) self._compareAll(np_arr, [0, 1, 2]) + @test_util.run_deprecated_v1 def testGradient(self): s = [2, 3, 4, 2] x = np.arange(-49.0, -1.0).reshape(s).astype(np.float64) @@ -751,6 +797,7 @@ class MaxReductionTest(test.TestCase): t, s, su, [2, 2], x_init_value=x, delta=1) self.assertAllClose(jacob_t, jacob_n, rtol=1e-8, atol=1e-8) + @test_util.run_deprecated_v1 def testGradient2(self): s = [2, 3, 4, 2] x = np.arange(-49.0, -1.0).reshape(s).astype(np.float64) @@ -761,6 +808,7 @@ class MaxReductionTest(test.TestCase): t, s, su, [2, 4, 2], x_init_value=x, delta=1) self.assertAllClose(jacob_t, jacob_n, rtol=1e-8, atol=1e-8) + @test_util.run_deprecated_v1 def testGradient3(self): s = [2, 3, 4, 2] x = np.arange(-49.0, -1.0).reshape(s).astype(np.float64) @@ -771,6 +819,7 @@ class MaxReductionTest(test.TestCase): t, s, su, [2, 3, 2], x_init_value=x, delta=1) self.assertAllClose(jacob_t, jacob_n, rtol=1e-8, atol=1e-8) + @test_util.run_deprecated_v1 def testGradient4(self): s = [2, 3, 4, 2] x = np.arange(-49.0, -1.0).reshape(s).astype(np.float64) @@ -781,6 +830,7 @@ class MaxReductionTest(test.TestCase): t, s, su, [1], x_init_value=x, delta=1) self.assertAllClose(jacob_t, jacob_n, rtol=1e-8, atol=1e-8) + @test_util.run_deprecated_v1 def testEmptyGradients(self): with self.cached_session(): x = array_ops.zeros([0, 3]) @@ -802,7 +852,7 @@ class AllReductionTest(test.TestCase): if reduction_axes is not None: reduction_axes = np.array(reduction_axes).astype(np.int32) tf_ans = math_ops.reduce_all(x, reduction_axes, keepdims) - out = tf_ans.eval() + out = self.evaluate(tf_ans) self.assertAllEqual(np_ans, out) self.assertShapeEqual(np_ans, tf_ans) @@ -817,7 +867,7 @@ class AllReductionTest(test.TestCase): with self.session(use_gpu=True) as sess: v = math_ops.reduce_all([True, True], constant_op.constant(0, dtype=dtype)) - tf_v = sess.run(v) + tf_v = self.evaluate(v) self.assertAllEqual(tf_v, True) def testAll3D(self): @@ -851,7 +901,7 @@ class AnyReductionTest(test.TestCase): if reduction_axes is not None: reduction_axes = np.array(reduction_axes).astype(np.int32) tf_ans = math_ops.reduce_any(x, reduction_axes, keepdims) - out = tf_ans.eval() + out = self.evaluate(tf_ans) self.assertAllEqual(np_ans, out) self.assertShapeEqual(np_ans, tf_ans) @@ -866,7 +916,7 @@ class AnyReductionTest(test.TestCase): with self.session(use_gpu=True) as sess: v = math_ops.reduce_any([True, True], constant_op.constant(0, dtype=dtype)) - tf_v = sess.run(v) + tf_v = self.evaluate(v) self.assertAllEqual(tf_v, True) def testAll3D(self): @@ -913,6 +963,7 @@ class CountNonzeroReductionTest(test.TestCase): self._compare(x, reduction_axes, True, use_gpu=True, feed_dict=feed_dict) self._compare(x, reduction_axes, True, use_gpu=False, feed_dict=feed_dict) + @test_util.run_deprecated_v1 def testBoolReduce1D(self): # Create a 1D array of floats np_arr = np.asarray([False, False, True, False, False, True]) @@ -920,11 +971,13 @@ class CountNonzeroReductionTest(test.TestCase): self._compareAll(np_arr, []) self._compareAll(np_arr, [0]) + @test_util.run_deprecated_v1 def testFloatReduce1D(self): # Create a 1D array of floats np_arr = np.asarray([0.0, 1.0, -1.0, 0.0, 0.0, 3.0]).astype(np.float32) self._compareAll(np_arr, [0]) + @test_util.run_deprecated_v1 def testFloatReduce4D(self): # Create a 4D array of floats and reduce across some # dimensions @@ -944,11 +997,13 @@ class CountNonzeroReductionTest(test.TestCase): self._compareAll(np_arr, [1, 2, 3]) self._compareAll(np_arr, [0, 1, 2, 3]) + @test_util.run_deprecated_v1 def testExpand(self): # Reduce an empty tensor to a nonempty tensor x = np.zeros((5, 0)) self._compareAll(x, [1]) + @test_util.run_deprecated_v1 def testDegenerate(self): for use_gpu in False, True: with self.cached_session(use_gpu=use_gpu): @@ -962,8 +1017,9 @@ class CountNonzeroReductionTest(test.TestCase): # Test case for GitHub issue 18712 with self.cached_session() as sess: v = math_ops.count_nonzero(constant_op.constant(["test"])) - self.assertAllClose(sess.run(v), 1) + self.assertAllClose(self.evaluate(v), 1) + @test_util.run_deprecated_v1 def testStringReduce1D(self): # Create a 1D array of strings x = np.asarray(["", "", "a", "", "", "b"]) @@ -974,6 +1030,7 @@ class CountNonzeroReductionTest(test.TestCase): self._compare(x, [], keepdims=True, zero=np.str("")) self._compare(x, [0], keepdims=True, zero=np.str("")) + @test_util.run_deprecated_v1 def testStringReduce2D(self): # Create a 2D array of strings x = np.asarray([["", "", "a", "", "", "b"], diff --git a/tensorflow/python/kernel_tests/regex_full_match_op_test.py b/tensorflow/python/kernel_tests/regex_full_match_op_test.py index 98746e7d9b1..488ec85ab2c 100644 --- a/tensorflow/python/kernel_tests/regex_full_match_op_test.py +++ b/tensorflow/python/kernel_tests/regex_full_match_op_test.py @@ -23,6 +23,7 @@ from absl.testing import parameterized from tensorflow.python.compat import compat from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes +from tensorflow.python.framework import test_util from tensorflow.python.ops import gen_string_ops from tensorflow.python.ops import string_ops from tensorflow.python.platform import test @@ -33,6 +34,7 @@ from tensorflow.python.platform import test (gen_string_ops.static_regex_full_match)) class RegexFullMatchOpVariantsTest(test.TestCase, parameterized.TestCase): + @test_util.run_deprecated_v1 def testRegexFullMatch(self, op): values = ["abaaba", "abcdabcde"] with self.cached_session(): @@ -40,6 +42,7 @@ class RegexFullMatchOpVariantsTest(test.TestCase, parameterized.TestCase): matched = op(input_tensor, "a.*a").eval() self.assertAllEqual([True, False], matched) + @test_util.run_deprecated_v1 def testRegexFullMatchTwoDims(self, op): values = [["abaaba", "abcdabcde"], ["acdcba", "ebcda"]] with self.cached_session(): @@ -47,6 +50,7 @@ class RegexFullMatchOpVariantsTest(test.TestCase, parameterized.TestCase): matched = op(input_tensor, "a.*a").eval() self.assertAllEqual([[True, False], [True, False]], matched) + @test_util.run_deprecated_v1 def testEmptyMatch(self, op): values = ["abc", "1"] with self.cached_session(): @@ -54,6 +58,7 @@ class RegexFullMatchOpVariantsTest(test.TestCase, parameterized.TestCase): matched = op(input_tensor, "").eval() self.assertAllEqual([False, False], matched) + @test_util.run_deprecated_v1 def testInvalidPattern(self, op): values = ["abc", "1"] with self.cached_session(): @@ -61,11 +66,12 @@ class RegexFullMatchOpVariantsTest(test.TestCase, parameterized.TestCase): invalid_pattern = "A[" matched = op(input_tensor, invalid_pattern) with self.assertRaisesOpError("Invalid pattern"): - matched.eval() + self.evaluate(matched) class RegexFullMatchOpTest(test.TestCase): + @test_util.run_deprecated_v1 def testRegexFullMatchDelegation(self): with compat.forward_compatibility_horizon(2018, 11, 1): with self.cached_session(): @@ -78,6 +84,7 @@ class RegexFullMatchOpTest(test.TestCase): op_tensor = string_ops.regex_full_match(input_tensor, pattern_tensor) self.assertTrue(op_tensor.name.startswith("RegexFullMatch"), op.name) + @test_util.run_deprecated_v1 def testStaticRegexFullMatchDelegation(self): with compat.forward_compatibility_horizon(2018, 11, 20): with self.cached_session(): diff --git a/tensorflow/python/kernel_tests/regex_replace_op_test.py b/tensorflow/python/kernel_tests/regex_replace_op_test.py index d9b7ed28d21..6c7dfee7b40 100644 --- a/tensorflow/python/kernel_tests/regex_replace_op_test.py +++ b/tensorflow/python/kernel_tests/regex_replace_op_test.py @@ -22,6 +22,7 @@ from absl.testing import parameterized from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes +from tensorflow.python.framework import test_util from tensorflow.python.ops import gen_string_ops from tensorflow.python.ops import string_ops from tensorflow.python.platform import test @@ -32,6 +33,7 @@ from tensorflow.python.platform import test (gen_string_ops.static_regex_replace)) class RegexReplaceOpVariantsTest(test.TestCase, parameterized.TestCase): + @test_util.run_deprecated_v1 def testForwarding(self, op): with self.cached_session(): # Generate an input that is uniquely consumed by the regex op. @@ -45,6 +47,7 @@ class RegexReplaceOpVariantsTest(test.TestCase, parameterized.TestCase): stripped = op(inp, "\\p{Ll}", ".").eval() self.assertAllEqual([b"A.C.E", b"H.J.L"], stripped) + @test_util.run_deprecated_v1 def testRemovePrefix(self, op): values = ["a:foo", "a:bar", "a:foo", "b:baz", "b:qux", "ca:b"] with self.cached_session(): @@ -53,6 +56,7 @@ class RegexReplaceOpVariantsTest(test.TestCase, parameterized.TestCase): self.assertAllEqual([b"foo", b"bar", b"foo", b"baz", b"qux", b"ca:b"], stripped) + @test_util.run_deprecated_v1 def testRegexReplace(self, op): values = ["aba\naba", "abcdabcde"] with self.cached_session(): @@ -60,6 +64,7 @@ class RegexReplaceOpVariantsTest(test.TestCase, parameterized.TestCase): stripped = op(input_vector, "a.*a", "(\\0)").eval() self.assertAllEqual([b"(aba)\n(aba)", b"(abcda)bcde"], stripped) + @test_util.run_deprecated_v1 def testEmptyMatch(self, op): values = ["abc", "1"] with self.cached_session(): @@ -67,6 +72,7 @@ class RegexReplaceOpVariantsTest(test.TestCase, parameterized.TestCase): stripped = op(input_vector, "", "x").eval() self.assertAllEqual([b"xaxbxcx", b"x1x"], stripped) + @test_util.run_deprecated_v1 def testInvalidPattern(self, op): values = ["abc", "1"] with self.cached_session(): @@ -74,8 +80,9 @@ class RegexReplaceOpVariantsTest(test.TestCase, parameterized.TestCase): invalid_pattern = "A[" replace = op(input_vector, invalid_pattern, "x") with self.assertRaisesOpError("Invalid pattern"): - replace.eval() + self.evaluate(replace) + @test_util.run_deprecated_v1 def testGlobal(self, op): values = ["ababababab", "abcabcabc", ""] with self.cached_session(): @@ -98,6 +105,7 @@ class RegexReplaceTest(test.TestCase, parameterized.TestCase): (as_string, as_tensor), (as_tensor, as_string), (as_tensor, as_tensor)) + @test_util.run_deprecated_v1 def testRegexReplaceDelegation(self, pattern_fn, rewrite_fn): with self.cached_session(): input_vector = constant_op.constant("foo", dtypes.string) @@ -106,6 +114,7 @@ class RegexReplaceTest(test.TestCase, parameterized.TestCase): op = string_ops.regex_replace(input_vector, pattern, replace) self.assertTrue(op.name.startswith("RegexReplace")) + @test_util.run_deprecated_v1 def testStaticRegexReplaceDelegation(self): with self.cached_session(): input_vector = constant_op.constant("foo", dtypes.string) diff --git a/tensorflow/python/kernel_tests/relu_op_test.py b/tensorflow/python/kernel_tests/relu_op_test.py index b0f2796ede1..55e68f4884c 100644 --- a/tensorflow/python/kernel_tests/relu_op_test.py +++ b/tensorflow/python/kernel_tests/relu_op_test.py @@ -25,6 +25,8 @@ from tensorflow.python.compat import compat from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes from tensorflow.python.framework import errors +from tensorflow.python.framework import ops +from tensorflow.python.framework import test_util from tensorflow.python.ops import array_ops from tensorflow.python.ops import gradient_checker from tensorflow.python.ops import gradients_impl @@ -55,55 +57,56 @@ class ReluTest(test.TestCase): np.array([[-0.9, 0.7, -0.5, 0.3, -0.1], [0.1, -0.3, 0.5, -0.7, 0.9]]))) - def _testRelu(self, np_features, use_gpu=False): + def _testRelu(self, np_features): np_relu = self._npRelu(np_features) - with self.cached_session(use_gpu=use_gpu): - relu = nn_ops.relu(np_features) - tf_relu = relu.eval() + tf_relu = nn_ops.relu(np_features) self.assertAllClose(np_relu, tf_relu) - self.assertShapeEqual(np_relu, relu) + self.assertShapeEqual(np_relu, tf_relu) - def testNumbers(self): + def testNumbersCPU(self): for t in [np.int32, np.int64, np.float16, np.float32, np.float64]: - self._testRelu( - np.array([[-9, 7, -5, 3, -1], [1, -3, 5, -7, 9]]).astype(t), - use_gpu=False) - if t in [np.float16, np.float32, np.float64]: + # Force execution on CPU even if a GPU kernel is available for the type. + with ops.device("/device:CPU:0"): self._testRelu( - np.array([[-9, 7, -5, 3, -1], [1, -3, 5, -7, 9]]).astype(t), - use_gpu=True) + np.array([[-9, 7, -5, 3, -1], [1, -3, 5, -7, 9]]).astype(t)) - def _testReluInt8x4(self, np_inputs): - if not test.is_gpu_available(cuda_only=True): - return - np_relu = self._npRelu(np_inputs) - with self.cached_session(use_gpu=True): - relu = nn_ops.relu(constant_op.constant(np_inputs, dtypes.qint8)) - if np_inputs.size % 4 == 0: - tf_relu = relu.eval() - self.assertAllClose(np_relu, tf_relu) - self.assertShapeEqual(np_relu, relu) - else: - with self.assertRaisesRegexp( - errors.InvalidArgumentError, - "Tensor size must be a multiple of 4 for Relu. Got %d" % - np_inputs.size): - tf_relu = relu.eval() + def testNumbersGPU(self): + if not test.is_gpu_available(): + self.skipTest("No GPU available") + for t in [np.float16, np.float32, np.float64]: + self._testRelu( + np.array([[-9, 7, -5, 3, -1], [1, -3, 5, -7, 9]]).astype(t)) def testReluInt8x4GoodShape(self): - self._testReluInt8x4(np.array([[-50, 7, 23, 0], [-1, -5, 6, 11]])) + if not test.is_gpu_available(cuda_only=True): + self.skipTest("No GPU available") + inputs = np.array([[-50, 7, 23, 0], [-1, -5, 6, 11]]) + np_relu = self._npRelu(inputs) + tf_relu = nn_ops.relu(constant_op.constant(inputs, dtypes.qint8)) + self.assertAllClose(np_relu, tf_relu) + self.assertShapeEqual(np_relu, tf_relu) def testReluInt8x4BadShape(self): - np_inputs = np.array([[-50, 7, 23], [0, 1, -5], [6, -2, 11]]) - self.assertEqual(np_inputs.size, 9) - self._testReluInt8x4(np_inputs) - np_inputs = np.array( - [1, -2, 3, -4, 5, -6, 7, -8, 9, -8, 7, -6, 5, -4, 3, -2, 1]) - self.assertEqual(np_inputs.size, 17) - self._testReluInt8x4(np_inputs) + if not test.is_gpu_available(cuda_only=True): + self.skipTest("No GPU available") + inputs = constant_op.constant( + np.array([[-50, 7, 23], [0, 1, -5], [6, -2, 11]]), dtypes.qint8) + with self.assertRaisesRegexp( + errors.InvalidArgumentError, + "Tensor size must be a multiple of 4 for Relu. Got 9"): + self.evaluate(nn_ops.relu(inputs)) + + inputs = constant_op.constant( + np.array([1, -2, 3, -4, 5, -6, 7, -8, 9, -8, 7, -6, 5, -4, 3, -2, 1]), + dtypes.qint8) + with self.assertRaisesRegexp( + errors.InvalidArgumentError, + "Tensor size must be a multiple of 4 for Relu. Got 17"): + self.evaluate(nn_ops.relu(inputs)) # The gradient test for ReLU is a bit tricky as the derivative is not well # defined at around zero and we want to avoid that in terms of input values. + @test_util.run_deprecated_v1 def testGradientFloat32(self): with self.cached_session(): x = constant_op.constant( @@ -123,6 +126,7 @@ class ReluTest(test.TestCase): # The gradient for fp16 is inaccurate due to the low-precision. # Instead of relying on compute_gradient_error, we compare the fp16 analytical # gradient against their fp32 counterpart. + @test_util.run_deprecated_v1 def testGradientFloat16(self): with self.session(use_gpu=True) as sess: # Randomly construct a 1D shape from [1, 40) @@ -146,9 +150,10 @@ class ReluTest(test.TestCase): # Repeat the experiment for 100 times. All tensor shapes and its tensor # values are randomly generated for each run. for _ in xrange(100): - dx_f32_v, dx_f16_v = sess.run([dx_f32, dx_f16]) + dx_f32_v, dx_f16_v = self.evaluate([dx_f32, dx_f16]) self.assertAllClose(dx_f32_v, dx_f16_v, atol=3e-4) + @test_util.run_deprecated_v1 def testGradientFloat64(self): with self.cached_session(): x = constant_op.constant( @@ -166,6 +171,7 @@ class ReluTest(test.TestCase): print("relu (float64) gradient err = ", err) self.assertLess(err, 1e-10) + @test_util.run_deprecated_v1 def testGradGradFloat32(self): with self.cached_session(): x = constant_op.constant( @@ -183,6 +189,7 @@ class ReluTest(test.TestCase): print("relu (float32) gradient of gradient err = ", err) self.assertLess(err, 1e-4) + @test_util.run_deprecated_v1 def testGradGradFloat64(self): with self.cached_session(): x = constant_op.constant( @@ -202,15 +209,15 @@ class ReluTest(test.TestCase): self.assertLess(err, 1e-10) def testGradientScalar(self): - with self.cached_session() as sess: - x = variables.Variable(100.) - y = nn_ops.relu(x) - loss = y**2 - optimizer = gradient_descent.GradientDescentOptimizer(learning_rate=0.25) - train_op = optimizer.minimize(loss) - sess.run(variables.global_variables_initializer()) - sess.run(train_op) - self.assertAllClose(x.eval(), 50.0) + x = variables.Variable(100.) + + def loss(): + return nn_ops.relu(x)**2 + + optimizer = gradient_descent.GradientDescentOptimizer(learning_rate=0.25) + self.evaluate(variables.global_variables_initializer()) + self.evaluate(optimizer.minimize(loss)) + self.assertAllClose(x.read_value(), 50.0) class Relu6Test(test.TestCase): @@ -228,27 +235,30 @@ class Relu6Test(test.TestCase): np.array([[-0.9, 0.7, -0.5, 0.3, 6.0], [0.1, -0.3, 6.5, -0.7, 0.9]]))) - def _testRelu6(self, np_features, use_gpu=False): + def _testRelu6(self, np_features): np_relu6 = self._npRelu6(np_features) - with self.cached_session(use_gpu=use_gpu): - relu6 = nn_ops.relu6(np_features) - tf_relu6 = relu6.eval() + tf_relu6 = nn_ops.relu6(np_features) self.assertAllClose(np_relu6, tf_relu6) - self.assertShapeEqual(np_relu6, relu6) + self.assertShapeEqual(np_relu6, tf_relu6) - def testNumbers(self): + def testNumbersCPU(self): for t in [np.int32, np.int64, np.float16, np.float32, np.float64]: - self._testRelu6( - np.array([[-9, 7, -5, 3, -1], [1, -3, 5, -7, 9]]).astype(t), - use_gpu=False) - if t in [np.float16, np.float, np.double]: + # Force execution on CPU even if a GPU kernel is available for the type. + with ops.device("/device:CPU:0"): self._testRelu6( - np.array([[-9, 7, -5, 3, -1], [1, -3, 5, -7, 9]]).astype(t), - use_gpu=True) + np.array([[-9, 7, -5, 3, -1], [1, -3, 5, -7, 9]]).astype(t)) + + def testNumbersGPU(self): + if not test.is_gpu_available(): + self.skipTest("No GPU available") + for t in [np.float16, np.float, np.double]: + self._testRelu6( + np.array([[-9, 7, -5, 3, -1], [1, -3, 5, -7, 9]]).astype(t)) # The gradient test for ReLU6 is a bit tricky as the derivative is # not well defined at around zero and six and we want to avoid that # in terms of input values. + @test_util.run_deprecated_v1 def testGradientFloat32(self): with self.cached_session(): x = constant_op.constant( @@ -265,6 +275,7 @@ class Relu6Test(test.TestCase): print("relu6 (float32) gradient err = ", err) self.assertLess(err, 1e-4) + @test_util.run_deprecated_v1 def testGradientFloat64(self): with self.cached_session(): x = constant_op.constant( @@ -297,29 +308,32 @@ class LeakyReluTest(test.TestCase): 0.9]]), alpha=0.1)) - def _testLeakyRelu(self, np_features, alpha, use_gpu=False): + def _testLeakyRelu(self, np_features, alpha): np_leaky_relu = self._npLeakyRelu(np_features, alpha) - with self.test_session(use_gpu=use_gpu): - leaky_relu = nn_ops.leaky_relu(np_features, alpha) - tf_leaky_relu = leaky_relu.eval() + tf_leaky_relu = nn_ops.leaky_relu(np_features, alpha) self.assertAllClose(np_leaky_relu, tf_leaky_relu) - self.assertShapeEqual(np_leaky_relu, leaky_relu) + self.assertShapeEqual(np_leaky_relu, tf_leaky_relu) - def testNumbers(self): + def testNumbersCPU(self): for t in [np.int32, np.int64, np.float16, np.float32, np.float64]: - self._testLeakyRelu( - np.array([[-9, 7, -5, 3, -1], [1, -3, 5, -7, 9]]).astype(t), - alpha=0.2, - use_gpu=False) - if t in [np.float16, np.float32, np.float64]: + # Force execution on CPU even if a GPU kernel is available for the type. + with ops.device("/device:CPU:0"): self._testLeakyRelu( np.array([[-9, 7, -5, 3, -1], [1, -3, 5, -7, 9]]).astype(t), - alpha=0.1, - use_gpu=True) + alpha=0.2) + + def testNumbersGPU(self): + if not test.is_gpu_available(): + self.skipTest("No GPU available") + for t in [np.float16, np.float32, np.float64]: + self._testLeakyRelu( + np.array([[-9, 7, -5, 3, -1], [1, -3, 5, -7, 9]]).astype(t), + alpha=0.1) # The gradient test for Leaky ReLU is a bit tricky as the derivative is not # well defined at around zero and we want to avoid that in terms of input # values. + @test_util.run_deprecated_v1 def testGradientFloat32(self): with self.test_session(): x = constant_op.constant( @@ -336,6 +350,7 @@ class LeakyReluTest(test.TestCase): print("leaky_relu (float32) gradient err = ", err) self.assertLess(err, 1e-4) + @test_util.run_deprecated_v1 def testGradientFloat64(self): with self.test_session(): x = constant_op.constant( @@ -353,6 +368,7 @@ class LeakyReluTest(test.TestCase): print("leaky_relu (float64) gradient err = ", err) self.assertLess(err, 1e-10) + @test_util.run_deprecated_v1 def testGradGradFloat32(self): with compat.forward_compatibility_horizon(2018, 11, 2): with self.test_session(): @@ -371,6 +387,7 @@ class LeakyReluTest(test.TestCase): print("leaky_relu (float32) gradient of gradient err = ", err) self.assertLess(err, 1e-4) + @test_util.run_deprecated_v1 def testGradGradFloat64(self): with compat.forward_compatibility_horizon(2018, 11, 2): with self.test_session(): @@ -391,15 +408,15 @@ class LeakyReluTest(test.TestCase): self.assertLess(err, 1e-10) def testGradientScalar(self): - with self.test_session() as sess: - x = variables.Variable(-100.) - y = nn_ops.leaky_relu(x, 0.05) - loss = y**2 - optimizer = gradient_descent.GradientDescentOptimizer(learning_rate=0.2) - train_op = optimizer.minimize(loss) - sess.run(variables.global_variables_initializer()) - sess.run(train_op) - self.assertAllClose(x.eval(), -99.9) + x = variables.Variable(-100.) + + def loss(): + return nn_ops.leaky_relu(x, 0.05)**2 + + optimizer = gradient_descent.GradientDescentOptimizer(learning_rate=0.2) + self.evaluate(variables.global_variables_initializer()) + self.evaluate(optimizer.minimize(loss)) + self.assertAllClose(x.read_value(), -99.9) class EluTest(test.TestCase): @@ -415,23 +432,26 @@ class EluTest(test.TestCase): np.array([[-0.9, 0.7, -0.5, 0.3, -0.1], [0.1, -0.3, 0.5, -0.7, 0.9]]))) - def _testElu(self, np_features, use_gpu=False): + def _testElu(self, np_features): np_elu = self._npElu(np_features) - with self.cached_session(use_gpu=use_gpu): - elu = nn_ops.elu(np_features) - tf_elu = elu.eval() + tf_elu = nn_ops.elu(np_features) self.assertAllClose(np_elu, tf_elu) - self.assertShapeEqual(np_elu, elu) + self.assertShapeEqual(np_elu, tf_elu) - def testNumbers(self): + def testNumbersCPU(self): for t in [np.float16, np.float32, np.float64]: - self._testElu( - np.array([[-9, 7, -5, 3, -1], [1, -3, 5, -7, 9]]).astype(t), - use_gpu=False) - self._testElu( - np.array([[-9, 7, -5, 3, -1], [1, -3, 5, -7, 9]]).astype(t), - use_gpu=True) + # Force execution on CPU even if a GPU kernel is available for the type. + with ops.device("/device:CPU:0"): + self._testElu( + np.array([[-9, 7, -5, 3, -1], [1, -3, 5, -7, 9]]).astype(t)) + def testNumbersGPU(self): + if not test.is_gpu_available(): + self.skipTest("No GPU available") + for t in [np.float16, np.float32, np.float64]: + self._testElu(np.array([[-9, 7, -5, 3, -1], [1, -3, 5, -7, 9]]).astype(t)) + + @test_util.run_deprecated_v1 def testGradientFloat32(self): with self.cached_session(): x_val = [[-0.9, -0.7, -0.5, -0.3, -0.1], [0.1, 0.3, 0.5, 0.7, 0.9]] @@ -443,6 +463,7 @@ class EluTest(test.TestCase): print("elu (float32) gradient err = ", err) self.assertLess(err, 1e-4) + @test_util.run_deprecated_v1 def testGradientFloat64(self): with self.cached_session(): x_val = [[-0.9, -0.7, -0.5, -0.3, -0.1], [0.1, 0.3, 0.5, 0.7, 0.9]] @@ -454,6 +475,7 @@ class EluTest(test.TestCase): print("elu (float64) gradient err = ", err) self.assertLess(err, 1e-6) + @test_util.run_deprecated_v1 def testGradGrad(self): with self.cached_session(): x = array_ops.placeholder(dtype=dtypes.float32) @@ -465,6 +487,7 @@ class EluTest(test.TestCase): err = np.abs(gg.eval(feed_dict={x: x_val}) - _elu_grad_grad(x_val)) self.assertLess(err, 1e-4) + @test_util.run_deprecated_v1 def testGradGradFloat32(self): with self.cached_session(): x = constant_op.constant( @@ -482,6 +505,7 @@ class EluTest(test.TestCase): print("elu (float32) gradient of gradient err = ", err) self.assertLess(err, 1e-4) + @test_util.run_deprecated_v1 def testGradGradFloat64(self): with self.cached_session(): x = constant_op.constant( @@ -517,23 +541,22 @@ class SeluTest(test.TestCase): np.array([[-0.9, 0.7, -0.5, 0.3, -0.1], [0.1, -0.3, 0.5, -0.7, 0.9]]))) - def _testSelu(self, np_features, use_gpu=False): + def _testSelu(self, np_features): np_selu = self._npSelu(np_features) - with self.cached_session(use_gpu=use_gpu): - selu = nn_ops.selu(np_features) - tf_selu = selu.eval() + tf_selu = nn_ops.selu(np_features) self.assertAllClose(np_selu, tf_selu) - self.assertShapeEqual(np_selu, selu) + self.assertShapeEqual(np_selu, tf_selu) def testNumbers(self): for t in [np.float16, np.float32, np.float64]: self._testSelu( - np.array([[-9, 7, -5, 3, -1], [1, -3, 5, -7, 9]]).astype(t), - use_gpu=False) - self._testSelu( - np.array([[-9, 7, -5, 3, -1], [1, -3, 5, -7, 9]]).astype(t), - use_gpu=True) + np.array([[-9, 7, -5, 3, -1], [1, -3, 5, -7, 9]]).astype(t)) + # Force executed on CPU in case GPU kernels are avaiable. + with ops.device("/device:CPU:0"): + self._testSelu( + np.array([[-9, 7, -5, 3, -1], [1, -3, 5, -7, 9]]).astype(t)) + @test_util.run_deprecated_v1 def testGradientFloat32(self): with self.cached_session(): x_val = [[-0.9, -0.7, -0.5, -0.3, -0.1], [0.1, 0.3, 0.5, 0.7, 0.9]] @@ -545,6 +568,7 @@ class SeluTest(test.TestCase): print("selu (float32) gradient err = ", err) self.assertLess(err, 1e-4) + @test_util.run_deprecated_v1 def testGradientFloat64(self): with self.cached_session(): x_val = [[-0.9, -0.7, -0.5, -0.3, -0.1], [0.1, 0.3, 0.5, 0.7, 0.9]] @@ -556,6 +580,7 @@ class SeluTest(test.TestCase): print("selu (float64) gradient err = ", err) self.assertLess(err, 1e-6) + @test_util.run_deprecated_v1 def testGradGradFloat32(self): with self.cached_session(): x = constant_op.constant( @@ -573,6 +598,7 @@ class SeluTest(test.TestCase): print("selu (float32) gradient of gradient err = ", err) self.assertLess(err, 1e-4) + @test_util.run_deprecated_v1 def testGradGradFloat64(self): with self.cached_session(): x = constant_op.constant( @@ -599,46 +625,44 @@ class CreluTest(test.TestCase): t = nn_ops.crelu(f) self.assertEqual([50, 5, 7, 20], t.get_shape()) - def _testCrelu(self, np_features, use_gpu=False): + def _testCrelu(self, np_features): np_relu = np.maximum(np_features, np.zeros_like(np_features)) np_neg_relu = np.maximum(-np_features, np.zeros_like(np_features)) np_crelu = np.concatenate((np_relu, np_neg_relu), len(np_features.shape) - 1) - with self.cached_session(use_gpu=use_gpu): - crelu = nn_ops.crelu(np_features) - tf_relu = crelu.eval() + tf_crelu = nn_ops.crelu(np_features) - self.assertAllClose(np_crelu, tf_relu) - self.assertShapeEqual(np_crelu, crelu) + self.assertAllClose(np_crelu, tf_crelu) + self.assertShapeEqual(np_crelu, tf_crelu) - def testNumbers(self): + def testNumbersCPU(self): for t in [np.int32, np.int64, np.float16, np.float32, np.float64]: - self._testCrelu( - np.array([[-9, 7, -5, 3, -1], [1, -3, 5, -7, 9]]).astype(t), - use_gpu=False) - if t in [np.float16, np.float32, np.float64]: + # Force execution on CPU even if a GPU kernel is available for the type. + with ops.device("/device:CPU:0"): self._testCrelu( - np.array([[-9, 7, -5, 3, -1], [1, -3, 5, -7, 9]]).astype(t), - use_gpu=True) + np.array([[-9, 7, -5, 3, -1], [1, -3, 5, -7, 9]]).astype(t)) + + def testNumbersGPU(self): + if not test.is_gpu_available(): + self.skipTest("No GPU available") + for t in [np.float16, np.float32, np.float64]: + self._testCrelu( + np.array([[-9, 7, -5, 3, -1], [1, -3, 5, -7, 9]]).astype(t)) def testNumbersWithAxis0(self): - with self.cached_session(): - crelu = nn_ops.crelu( - np.array([[-9, 7, -5, 3, -1], [1, -3, 5, -7, 9]]), axis=0) - tf_relu = crelu.eval() - np_crelu = np.array([[0, 7, 0, 3, 0], [1, 0, 5, 0, 9], [9, 0, 5, 0, 1], - [0, 3, 0, 7, 0]]) - self.assertAllEqual(np_crelu, tf_relu) + tf_crelu = nn_ops.crelu( + np.array([[-9, 7, -5, 3, -1], [1, -3, 5, -7, 9]]), axis=0) + np_crelu = np.array([[0, 7, 0, 3, 0], [1, 0, 5, 0, 9], [9, 0, 5, 0, 1], + [0, 3, 0, 7, 0]]) + self.assertAllEqual(np_crelu, tf_crelu) def testNumbersWithAxis1(self): - with self.cached_session(): - crelu = nn_ops.crelu( - np.array([[-9, 7, -5, 3, -1], [1, -3, 5, -7, 9]]), axis=1) - tf_relu = crelu.eval() - np_crelu = np.array([[0, 7, 0, 3, 0, 9, 0, 5, 0, 1], - [1, 0, 5, 0, 9, 0, 3, 0, 7, 0]]) - self.assertAllEqual(np_crelu, tf_relu) + tf_crelu = nn_ops.crelu( + np.array([[-9, 7, -5, 3, -1], [1, -3, 5, -7, 9]]), axis=1) + np_crelu = np.array([[0, 7, 0, 3, 0, 9, 0, 5, 0, 1], + [1, 0, 5, 0, 9, 0, 3, 0, 7, 0]]) + self.assertAllEqual(np_crelu, tf_crelu) if __name__ == "__main__": diff --git a/tensorflow/python/kernel_tests/reshape_op_test.py b/tensorflow/python/kernel_tests/reshape_op_test.py index 14cdae18370..db3e88a104f 100644 --- a/tensorflow/python/kernel_tests/reshape_op_test.py +++ b/tensorflow/python/kernel_tests/reshape_op_test.py @@ -22,6 +22,7 @@ import numpy as np from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes +from tensorflow.python.framework import test_util from tensorflow.python.ops import array_ops from tensorflow.python.ops import gradient_checker from tensorflow.python.platform import test @@ -33,14 +34,14 @@ class ReshapeTest(test.TestCase): with self.cached_session(use_gpu=use_gpu): np_ans = x.reshape(y) tf_ans = array_ops.reshape(x, y) - out = tf_ans.eval() + out = self.evaluate(tf_ans) self.assertEqual(tf_ans.get_shape(), out.shape) self.assertShapeEqual(np_ans, tf_ans) # Repeat with an int64 shape tensor. y64 = constant_op.constant(y, dtype=dtypes.int64) tf_ans = array_ops.reshape(x, y64) - out = tf_ans.eval() + out = self.evaluate(tf_ans) self.assertEqual(tf_ans.get_shape(), out.shape) self.assertShapeEqual(np_ans, tf_ans) @@ -91,6 +92,7 @@ class ReshapeTest(test.TestCase): # TODO(vrv): Add tests for failure conditions once python test_util # reports errors. + @test_util.run_deprecated_v1 def testFloatReshapeGradThreeDimensions(self): x = np.arange(1., 25.).reshape([2, 3, 4]).astype(np.float32) s = list(np.shape(x)) @@ -111,6 +113,7 @@ class ReshapeTest(test.TestCase): self._testBothReshape(x, [0, 0, 0]) self._testBothReshape(x, [1, -1, 5]) + @test_util.run_deprecated_v1 def testErrors(self): y = constant_op.constant(0.0, shape=[23, 29, 31]) with self.assertRaisesRegexp(ValueError, "must be evenly divisible by 17"): @@ -121,6 +124,7 @@ class ReshapeTest(test.TestCase): "Cannot reshape a tensor with 4096 elements"): array_ops.reshape(z, [4095]) + @test_util.run_deprecated_v1 def testPartialShapes(self): x = array_ops.placeholder(dtypes.float32) diff --git a/tensorflow/python/kernel_tests/resource_variable_ops_test.py b/tensorflow/python/kernel_tests/resource_variable_ops_test.py index c8227dc117f..b57d9d47aa3 100644 --- a/tensorflow/python/kernel_tests/resource_variable_ops_test.py +++ b/tensorflow/python/kernel_tests/resource_variable_ops_test.py @@ -29,6 +29,7 @@ from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes from tensorflow.python.framework import errors from tensorflow.python.framework import ops +from tensorflow.python.framework import tensor_util from tensorflow.python.framework import test_util from tensorflow.python.ops import array_ops from tensorflow.python.ops import control_flow_ops @@ -53,6 +54,7 @@ class ResourceVariableOpsTest(test_util.TensorFlowTestCase): # involving objects with __del__ defined. self.assertEqual(0, len(gc.garbage)) + @test_util.run_deprecated_v1 def testHandleDtypeShapeMatch(self): with self.cached_session(): handle = resource_variable_ops.var_handle_op(dtype=dtypes.int32, shape=[]) @@ -122,6 +124,7 @@ class ResourceVariableOpsTest(test_util.TensorFlowTestCase): # values. self.assertFalse(np.allclose(variable.numpy(), copied_variable.numpy())) + @test_util.run_deprecated_v1 def testGraphDeepCopy(self): with self.cached_session(): init_value = np.ones((4, 4, 4)) @@ -137,6 +140,15 @@ class ResourceVariableOpsTest(test_util.TensorFlowTestCase): self.evaluate(v[0].assign(2.0)) self.assertAllEqual(self.evaluate(v), [2.0, 2.0]) + @test_util.run_in_graph_and_eager_modes + def testVariableShape(self): + v = resource_variable_ops.ResourceVariable([1., 1.]) + self.assertAllEqual( + tensor_util.constant_value( + resource_variable_ops.variable_shape(v.handle)), + [2]) + + @test_util.run_deprecated_v1 def testDifferentAssignGraph(self): with ops.Graph().as_default(): v = resource_variable_ops.ResourceVariable(1.0) @@ -144,16 +156,18 @@ class ResourceVariableOpsTest(test_util.TensorFlowTestCase): v.assign(2.0) # Note: this fails if we run convert_to_tensor on not the # variable graph. + @test_util.run_deprecated_v1 def testFetchHandle(self): with self.cached_session(): handle = resource_variable_ops.var_handle_op( dtype=dtypes.int32, shape=[1], name="foo") self.assertGreater(len(handle.eval()), 0) + @test_util.run_deprecated_v1 def testCachedValueReadBeforeWrite(self): with self.cached_session() as sess: v = resource_variable_ops.ResourceVariable(0.0, caching_device="cpu:0") - sess.run(v.initializer) + self.evaluate(v.initializer) value, _ = sess.run([v, v.assign_add(1.0)]) self.assertAllEqual(value, 0.0) @@ -426,6 +440,7 @@ class ResourceVariableOpsTest(test_util.TensorFlowTestCase): read = resource_variable_ops.read_variable_op(handle, dtype=dtypes.int32) self.assertEqual(self.evaluate(read), [[6]]) + @test_util.run_deprecated_v1 def testScatterUpdateString(self): handle = resource_variable_ops.var_handle_op( dtype=dtypes.string, shape=[1, 1]) @@ -437,6 +452,7 @@ class ResourceVariableOpsTest(test_util.TensorFlowTestCase): self.assertEqual(compat.as_bytes(self.evaluate(read)[0][0]), compat.as_bytes("b")) + @test_util.run_deprecated_v1 def testScatterUpdateStringScalar(self): handle = resource_variable_ops.var_handle_op( dtype=dtypes.string, shape=[1, 1]) @@ -456,7 +472,7 @@ class ResourceVariableOpsTest(test_util.TensorFlowTestCase): # TODO(alive): get this to work in Eager mode. def testGPU(self): - with self.test_session(use_gpu=True): + with test_util.use_gpu(): abc = variable_scope.get_variable( "abc", shape=[1], @@ -491,6 +507,7 @@ class ResourceVariableOpsTest(test_util.TensorFlowTestCase): initial_value=lambda: 1, constraint=constraint, name="var1") # TODO(alive): how should this work in Eager mode? + @test_util.run_deprecated_v1 def testInitFn(self): with self.cached_session(): v = resource_variable_ops.ResourceVariable( @@ -568,6 +585,21 @@ class ResourceVariableOpsTest(test_util.TensorFlowTestCase): v.load(2.0) self.assertEqual(2.0, self.evaluate(v.value())) + def testToFromProtoCachedValue(self): + with ops.Graph().as_default(): + v_def = resource_variable_ops.ResourceVariable( + initial_value=constant_op.constant(3.0)).to_proto() + v_prime = resource_variable_ops.ResourceVariable(variable_def=v_def) + self.assertTrue(getattr(v_prime, "_cached_value", None) is None) + + other_v_def = resource_variable_ops.ResourceVariable( + caching_device="cpu:0", + initial_value=constant_op.constant(3.0)).to_proto() + other_v_prime = resource_variable_ops.ResourceVariable( + variable_def=other_v_def) + self.assertTrue(other_v_prime._cached_value is not None) + + @test_util.run_deprecated_v1 def testVariableDefInitializedInstances(self): with ops.Graph().as_default(), self.cached_session() as sess: v_def = resource_variable_ops.ResourceVariable( @@ -576,11 +608,11 @@ class ResourceVariableOpsTest(test_util.TensorFlowTestCase): with ops.Graph().as_default(), self.cached_session() as sess: # v describes a VariableDef-based variable without an initial value. v = resource_variable_ops.ResourceVariable(variable_def=v_def) - self.assertEqual(3.0, sess.run(v.initialized_value())) + self.assertEqual(3.0, self.evaluate(v.initialized_value())) # initialized_value should not rerun the initializer_op if the variable # has already been initialized elsewhere. - sess.run(v.assign(1.0)) + self.evaluate(v.assign(1.0)) self.assertEqual(1.0, v.initialized_value().eval()) v_def.ClearField("initial_value_name") @@ -592,7 +624,7 @@ class ResourceVariableOpsTest(test_util.TensorFlowTestCase): self.assertProtoEquals(v_def, v.to_proto()) # But attempts to use initialized_value will result in errors. with self.assertRaises(ValueError): - sess.run(v.initialized_value()) + self.evaluate(v.initialized_value()) def testTrainableInProto(self): with ops.Graph().as_default(): @@ -623,6 +655,7 @@ class ResourceVariableOpsTest(test_util.TensorFlowTestCase): value = self.evaluate(v.sparse_read([0, 3, 1, 2])) self.assertAllEqual(init_value[[0, 3, 1, 2], ...], value) + @test_util.run_deprecated_v1 def testToFromProto(self): with self.cached_session(): v = resource_variable_ops.ResourceVariable(1.0) @@ -671,6 +704,7 @@ class ResourceVariableOpsTest(test_util.TensorFlowTestCase): self.assertEqual(0.0, self.evaluate(v.value())) @test_util.run_in_graph_and_eager_modes + @test_util.run_deprecated_v1 def testDestroyResource(self): v = resource_variable_ops.ResourceVariable(3.0, name="var0") self.evaluate(variables.global_variables_initializer()) @@ -684,6 +718,7 @@ class ResourceVariableOpsTest(test_util.TensorFlowTestCase): self.evaluate(resource_variable_ops.destroy_resource_op( handle, ignore_lookup_error=True)) + @test_util.run_deprecated_v1 def testAssignDifferentShapes(self): with self.cached_session() as sess, variable_scope.variable_scope( "foo", use_resource=True): @@ -704,6 +739,7 @@ class ResourceVariableOpsTest(test_util.TensorFlowTestCase): assign = var.assign(np.zeros(shape=[2, 2])) self.evaluate(assign) + @test_util.run_deprecated_v1 def testDtypeAfterFromProto(self): v = resource_variable_ops.ResourceVariable(2.0) w = resource_variable_ops.ResourceVariable.from_proto(v.to_proto()) @@ -711,6 +747,7 @@ class ResourceVariableOpsTest(test_util.TensorFlowTestCase): self.assertEqual(v.dtype, w.dtype) # TODO(alive): get caching to work in eager mode. + @test_util.run_deprecated_v1 def testCachingDevice(self): with ops.device("/job:server/task:1"): v = resource_variable_ops.ResourceVariable( @@ -726,6 +763,7 @@ class ResourceVariableOpsTest(test_util.TensorFlowTestCase): with self.assertRaises(ValueError): _ = w.value().op.get_attr("_class") + @test_util.run_deprecated_v1 def testSharedName(self): with self.cached_session(): v = resource_variable_ops.ResourceVariable(300.0, name="var4") @@ -736,7 +774,7 @@ class ResourceVariableOpsTest(test_util.TensorFlowTestCase): # Needed in Eager since we get a unique container name by default. container=ops.get_default_graph()._container) w_read = resource_variable_ops.read_variable_op(w, v.dtype.base_dtype) - self.assertEqual(300.0, w_read.eval()) + self.assertEqual(300.0, self.evaluate(w_read)) x = resource_variable_ops.var_handle_op( dtype=v.dtype.base_dtype, shape=v.get_shape(), shared_name="var5", @@ -744,6 +782,7 @@ class ResourceVariableOpsTest(test_util.TensorFlowTestCase): with self.assertRaisesOpError("Resource .*/var5/.* does not exist"): resource_variable_ops.read_variable_op(x, v.dtype.base_dtype).eval() + @test_util.run_deprecated_v1 def testSharedNameWithNamescope(self): with self.cached_session(): with ops.name_scope("foo"): @@ -772,6 +811,7 @@ class ResourceVariableOpsTest(test_util.TensorFlowTestCase): "", str(v.sparse_read(array_ops.placeholder(dtypes.int32)).shape)) + @test_util.run_deprecated_v1 def testSetInitialValue(self): with self.cached_session(): # Initialize variable with a value different from the initial value passed @@ -780,6 +820,7 @@ class ResourceVariableOpsTest(test_util.TensorFlowTestCase): v.initializer.run(feed_dict={v.initial_value: 3.0}) self.assertEqual(3.0, v.value().eval()) + @test_util.run_deprecated_v1 def testControlFlowInitialization(self): """Expects an error if an initializer is in a control-flow scope.""" @@ -916,6 +957,7 @@ class ResourceVariableOpsTest(test_util.TensorFlowTestCase): self.assertAllEqual(self.evaluate(v.assign_add(1)), [1, 2, 3, 4]) @test_util.run_in_graph_and_eager_modes + @test_util.run_deprecated_v1 def testCopyToGraphUninitialized(self): v = resource_variable_ops.ResourceVariable([0, 1, 2, 3]) copy_to_graph = ops.Graph() diff --git a/tensorflow/python/kernel_tests/reverse_sequence_op_test.py b/tensorflow/python/kernel_tests/reverse_sequence_op_test.py index 56609bd0a5e..05307c9834a 100644 --- a/tensorflow/python/kernel_tests/reverse_sequence_op_test.py +++ b/tensorflow/python/kernel_tests/reverse_sequence_op_test.py @@ -23,6 +23,7 @@ from six.moves import xrange # pylint: disable=redefined-builtin from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes +from tensorflow.python.framework import test_util from tensorflow.python.ops import array_ops from tensorflow.python.ops import gradient_checker from tensorflow.python.platform import test @@ -42,12 +43,12 @@ class ReverseSequenceTest(test.TestCase): ans = array_ops.reverse_sequence( x, batch_axis=batch_axis, seq_axis=seq_axis, seq_lengths=seq_lengths) if expected_err_re is None: - tf_ans = ans.eval() + tf_ans = self.evaluate(ans) self.assertAllClose(tf_ans, truth, atol=1e-10) self.assertShapeEqual(truth, ans) else: with self.assertRaisesOpError(expected_err_re): - ans.eval() + self.evaluate(ans) def _testBothReverseSequence(self, x, @@ -107,6 +108,7 @@ class ReverseSequenceTest(test.TestCase): def testComplex128Basic(self): self._testBasic(np.complex128) + @test_util.run_deprecated_v1 def testFloatReverseSequenceGrad(self): x = np.asarray( [[[1, 2, 3, 4], [5, 6, 7, 8]], [[9, 10, 11, 12], [13, 14, 15, 16]], @@ -133,6 +135,7 @@ class ReverseSequenceTest(test.TestCase): print("ReverseSequence gradient error = %g" % err) self.assertLess(err, 1e-8) + @test_util.run_deprecated_v1 def testShapeFunctionEdgeCases(self): t = array_ops.reverse_sequence( array_ops.placeholder( diff --git a/tensorflow/python/kernel_tests/rnn_test.py b/tensorflow/python/kernel_tests/rnn_test.py index 0090b7332f9..3bc457f8fb6 100644 --- a/tensorflow/python/kernel_tests/rnn_test.py +++ b/tensorflow/python/kernel_tests/rnn_test.py @@ -262,6 +262,7 @@ class RNNTest(test.TestCase): rnn.dynamic_rnn(cell, inputs, dtype=dtypes.float32, sequence_length=[4]) @test_util.run_in_graph_and_eager_modes + @test_util.run_deprecated_v1 def testTensorArrayStateIsAccepted(self): cell = TensorArrayStateRNNCell() in_eager_mode = context.executing_eagerly() @@ -285,6 +286,7 @@ class RNNTest(test.TestCase): self.assertAllEqual(4, state[0]) self.assertAllEqual([[[1]], [[2]], [[3]], [[4]]], state[1]) + @test_util.run_deprecated_v1 def testCellGetInitialState(self): cell = rnn_cell_impl.BasicRNNCell(5) with self.assertRaisesRegexp( @@ -345,6 +347,7 @@ class RNNTest(test.TestCase): self._assert_cell_builds(contrib_rnn.IndyLSTMCell, f32, 5, 7, 3) self._assert_cell_builds(contrib_rnn.IndyLSTMCell, f64, 5, 7, 3) + @test_util.run_deprecated_v1 def testRNNWithKerasSimpleRNNCell(self): with self.cached_session() as sess: input_shape = 10 @@ -378,6 +381,7 @@ class RNNTest(test.TestCase): self.assertEqual(len(outputs), batch) self.assertEqual(len(state), batch) + @test_util.run_deprecated_v1 def testRNNWithKerasGRUCell(self): with self.cached_session() as sess: input_shape = 10 @@ -411,6 +415,7 @@ class RNNTest(test.TestCase): self.assertEqual(len(outputs), batch) self.assertEqual(len(state), batch) + @test_util.run_deprecated_v1 def testRNNWithKerasLSTMCell(self): with self.cached_session() as sess: input_shape = 10 @@ -448,6 +453,7 @@ class RNNTest(test.TestCase): self.assertEqual(len(state[0]), batch) self.assertEqual(len(state[1]), batch) + @test_util.run_deprecated_v1 def testRNNWithStackKerasCell(self): with self.cached_session() as sess: input_shape = 10 @@ -491,6 +497,7 @@ class RNNTest(test.TestCase): for s in state: self.assertEqual(len(s), batch) + @test_util.run_deprecated_v1 def testStaticRNNWithKerasSimpleRNNCell(self): with self.cached_session() as sess: input_shape = 10 @@ -529,6 +536,7 @@ class RNNTest(test.TestCase): self.assertEqual(len(outputs[0]), batch) self.assertEqual(len(state), batch) + @test_util.run_deprecated_v1 def testKerasAndTFRNNLayerOutputComparison(self): input_shape = 10 output_shape = 5 @@ -562,6 +570,7 @@ class RNNTest(test.TestCase): self.assertAllClose(tf_out, k_out) self.assertAllClose(tf_state, k_state) + @test_util.run_deprecated_v1 def testSimpleRNNCellAndBasicRNNCellComparison(self): input_shape = 10 output_shape = 5 @@ -601,6 +610,7 @@ class RNNTest(test.TestCase): self.assertAllClose(tf_out, k_out, atol=1e-5) self.assertAllClose(tf_state, k_state, atol=1e-5) + @test_util.run_deprecated_v1 def testBasicLSTMCellInterchangeWithLSTMCell(self): with self.session(graph=ops_lib.Graph()) as sess: basic_cell = rnn_cell_impl.BasicLSTMCell(1) diff --git a/tensorflow/python/kernel_tests/save_restore_ops_test.py b/tensorflow/python/kernel_tests/save_restore_ops_test.py index cb9aa1e34d6..fecc9a3800f 100644 --- a/tensorflow/python/kernel_tests/save_restore_ops_test.py +++ b/tensorflow/python/kernel_tests/save_restore_ops_test.py @@ -17,14 +17,30 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function +import os + from tensorflow.core.protobuf import config_pb2 from tensorflow.python.client import session +from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes +from tensorflow.python.framework import test_util from tensorflow.python.ops import gen_io_ops from tensorflow.python.ops import io_ops from tensorflow.python.platform import test +class SaveTest(test.TestCase): + + @test_util.run_in_graph_and_eager_modes + def testRelativePath(self): + os.chdir(self.get_temp_dir()) + self.evaluate(io_ops.save_v2( + "ckpt", ["x"], [""], [constant_op.constant(100.)])) + self.assertAllEqual([100.], + self.evaluate(io_ops.restore_v2( + "ckpt", ["x"], [""], [dtypes.float32]))) + + class ShardedFileOpsTest(test.TestCase): def testShardedFileName(self): @@ -39,6 +55,7 @@ class ShardedFileOpsTest(test.TestCase): class ShapeInferenceTest(test.TestCase): + @test_util.run_deprecated_v1 def testRestoreV2WithSliceInput(self): op = io_ops.restore_v2("model", ["var1", "var2"], ["", "3 4 0,1:-"], [dtypes.float32, dtypes.float32]) @@ -46,11 +63,13 @@ class ShapeInferenceTest(test.TestCase): self.assertFalse(op[0].get_shape().is_fully_defined()) self.assertEqual([1, 4], op[1].get_shape()) + @test_util.run_deprecated_v1 def testRestoreV2NumSlicesNotMatch(self): with self.assertRaises(ValueError): io_ops.restore_v2("model", ["var1", "var2", "var3"], ["", "3 4 0,1:-"], [dtypes.float32, dtypes.float32]) + @test_util.run_deprecated_v1 def testRestoreSlice(self): op = gen_io_ops.restore_slice("model", "var", "3 4 0,1:-", dtypes.float32) self.assertEqual([1, 4], op.get_shape()) diff --git a/tensorflow/python/kernel_tests/scan_ops_test.py b/tensorflow/python/kernel_tests/scan_ops_test.py index b3692225652..33e491fee1d 100644 --- a/tensorflow/python/kernel_tests/scan_ops_test.py +++ b/tensorflow/python/kernel_tests/scan_ops_test.py @@ -24,6 +24,7 @@ from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes from tensorflow.python.framework import errors_impl from tensorflow.python.framework import ops +from tensorflow.python.framework import test_util from tensorflow.python.ops import gradient_checker from tensorflow.python.ops import math_ops from tensorflow.python.platform import test @@ -88,12 +89,14 @@ class CumsumTest(test.TestCase): for reverse in [True, False]: self._compare(x, axis, exclusive, reverse) + @test_util.run_deprecated_v1 def testEmpty(self): for dtype in self.valid_dtypes: x = np.zeros([0]).astype(dtype) for axis in (-1, 0): self._compareAll(x, axis) + @test_util.run_deprecated_v1 def testAxisType(self): for dtype in self.valid_dtypes: x = np.arange(1, 6).reshape([5]).astype(dtype) @@ -102,30 +105,40 @@ class CumsumTest(test.TestCase): axis = constant_op.constant(0, axis_dtype) tf_out = math_ops.cumsum(x, axis).eval() + @test_util.run_deprecated_v1 def test1D(self): for dtype in self.valid_dtypes: x = np.arange(1, 6).reshape([5]).astype(dtype) for axis in (-1, 0): self._compareAll(x, axis) + @test_util.run_deprecated_v1 def test2D(self): for dtype in self.valid_dtypes: x = np.arange(0, 10).reshape([2, 5]).astype(dtype) for axis in (-2, -1, 0, 1): self._compareAll(x, axis) + @test_util.run_deprecated_v1 def test3D(self): for dtype in self.valid_dtypes: x = np.arange(0, 20).reshape([2, 2, 5]).astype(dtype) for axis in (-3, -2, -1, 0, 1, 2): self._compareAll(x, axis) + @test_util.run_deprecated_v1 def test6D(self): for dtype in self.valid_dtypes: x = np.arange(1, 145).reshape([2, 2, 3, 3, 2, 2]).astype(dtype) for axis in range(-6, 6, 3): self._compareAll(x, axis) + @test_util.run_deprecated_v1 + def testLarge(self): + for dtype in self.valid_dtypes: + x = np.ones([1000000], dtype=dtype) / 1024 + self._compareAll(x, 0) + def testInvalidAxis(self): x = np.arange(0, 10).reshape([2, 5]).astype(np.float32) input_tensor = ops.convert_to_tensor(x) @@ -152,22 +165,27 @@ class CumsumTest(test.TestCase): t, shape, result, shape, x_init_value=x, delta=1) self.assertAllClose(jacob_t, jacob_n, rtol=1e-8, atol=1e-8) + @test_util.run_deprecated_v1 def testGradient(self): for axis in (-1, 0): self._compareGradient([50], axis, False, False) + @test_util.run_deprecated_v1 def testGradientReverse(self): for axis in (-1, 0): self._compareGradient([50], axis, False, True) + @test_util.run_deprecated_v1 def testGradientExclusive(self): for axis in (-1, 0): self._compareGradient([50], axis, True, False) + @test_util.run_deprecated_v1 def testGradientExclusiveReverse(self): for axis in (-1, 0): self._compareGradient([50], axis, True, True) + @test_util.run_deprecated_v1 def testGradient2D(self): for axis in (-1, 0, 1): for exclusive in [True, False]: @@ -194,12 +212,14 @@ class CumprodTest(test.TestCase): for reverse in [True, False]: self._compare(x, axis, exclusive, reverse) + @test_util.run_deprecated_v1 def testEmpty(self): for dtype in self.valid_dtypes: x = np.zeros([0]).astype(dtype) for axis in (-1, 0): self._compareAll(x, axis) + @test_util.run_deprecated_v1 def testAxisType(self): for dtype in self.valid_dtypes: x = np.arange(1, 6).reshape([5]).astype(dtype) @@ -208,24 +228,28 @@ class CumprodTest(test.TestCase): axis = constant_op.constant(0, axis_dtype) tf_out = math_ops.cumprod(x, axis).eval() + @test_util.run_deprecated_v1 def test1D(self): for dtype in self.valid_dtypes: x = np.arange(1, 6).reshape([5]).astype(dtype) for axis in (-1, 0): self._compareAll(x, axis) + @test_util.run_deprecated_v1 def test2D(self): for dtype in self.valid_dtypes: x = np.arange(1, 11).reshape([2, 5]).astype(dtype) for axis in (-2, -1, 0, 1): self._compareAll(x, axis) + @test_util.run_deprecated_v1 def test3D(self): for dtype in self.valid_dtypes: x = np.arange(1, 21).reshape([2, 2, 5]).astype(dtype) for axis in (-3, -2, -1, 0, 1, 2): self._compareAll(x, axis) + @test_util.run_deprecated_v1 def test6D(self): for dtype in self.valid_dtypes: x = np.arange(1, 145).reshape([2, 2, 3, 3, 2, 2]).astype(dtype) @@ -258,22 +282,27 @@ class CumprodTest(test.TestCase): t, shape, result, shape, x_init_value=x, delta=1) self.assertAllClose(jacob_t, jacob_n, rtol=1e-8, atol=1e-8) + @test_util.run_deprecated_v1 def testGradient(self): for axis in (-1, 0): self._compareGradient([8], axis, False, False) + @test_util.run_deprecated_v1 def testGradientReverse(self): for axis in (-1, 0): self._compareGradient([8], axis, False, True) + @test_util.run_deprecated_v1 def testGradientExclusive(self): for axis in (-1, 0): self._compareGradient([8], axis, True, False) + @test_util.run_deprecated_v1 def testGradientExclusiveReverse(self): for axis in (-1, 0): self._compareGradient([8], axis, True, True) + @test_util.run_deprecated_v1 def testGradient2D(self): for axis in (-2, -1, 0, 1): for exclusive in [True, False]: diff --git a/tensorflow/python/kernel_tests/scatter_nd_ops_test.py b/tensorflow/python/kernel_tests/scatter_nd_ops_test.py index 0ed508b9fe2..c1241ba87ee 100644 --- a/tensorflow/python/kernel_tests/scatter_nd_ops_test.py +++ b/tensorflow/python/kernel_tests/scatter_nd_ops_test.py @@ -29,6 +29,7 @@ from tensorflow.python.framework import dtypes from tensorflow.python.framework import errors from tensorflow.python.framework import test_util from tensorflow.python.ops import array_ops +from tensorflow.python.ops import gradient_checker from tensorflow.python.ops import gradients_impl from tensorflow.python.ops import resource_variable_ops from tensorflow.python.ops import state_ops @@ -144,7 +145,7 @@ class StatefulScatterNdTest(test.TestCase): tf_scatter(ref_var, indices, updates).eval() # Compare - self.assertAllClose(new, ref_var.eval()) + self.assertAllClose(new, self.evaluate(ref_var)) def _VariableRankTests(self, np_scatter, tf_scatter): for vtype in (np.int32, np.float16, np.float32, np.float64, np.complex64, @@ -161,10 +162,11 @@ class StatefulScatterNdTest(test.TestCase): init = variables.global_variables_initializer() with self.session(use_gpu=True) as sess: - sess.run(init) - result = sess.run(scatter) + self.evaluate(init) + result = self.evaluate(scatter) self.assertAllClose(result, expected) + @test_util.run_deprecated_v1 def testSimpleResource(self): indices = constant_op.constant([[4], [3], [1], [7]], dtype=dtypes.int32) updates = constant_op.constant([9, 10, 11, 12], dtype=dtypes.float32) @@ -175,8 +177,8 @@ class StatefulScatterNdTest(test.TestCase): init = variables.global_variables_initializer() with self.session(use_gpu=True) as sess: - sess.run(init) - sess.run(scatter) + self.evaluate(init) + self.evaluate(scatter) self.assertAllClose(ref.eval(), expected) def testSimple2(self): @@ -189,8 +191,8 @@ class StatefulScatterNdTest(test.TestCase): init = variables.global_variables_initializer() with self.session(use_gpu=True) as sess: - sess.run(init) - result = sess.run(scatter) + self.evaluate(init) + result = self.evaluate(scatter) self.assertAllClose(result, expected) def testSimple3(self): @@ -203,16 +205,19 @@ class StatefulScatterNdTest(test.TestCase): init = variables.global_variables_initializer() with self.session(use_gpu=True) as sess: - sess.run(init) - result = sess.run(scatter) + self.evaluate(init) + result = self.evaluate(scatter) self.assertAllClose(result, expected) + @test_util.run_deprecated_v1 def testVariableRankUpdate(self): self._VariableRankTests(_NumpyUpdate, state_ops.scatter_nd_update) + @test_util.run_deprecated_v1 def testVariableRankAdd(self): self._VariableRankTests(_NumpyAdd, state_ops.scatter_nd_add) + @test_util.run_deprecated_v1 def testVariableRankSub(self): self._VariableRankTests(_NumpySub, state_ops.scatter_nd_sub) @@ -230,6 +235,7 @@ class StatefulScatterNdTest(test.TestCase): self._VariableRankTest( np_scatter, tf_scatter, vtype, itype, repeat_indices=True) + @test_util.run_deprecated_v1 def testScatterRepeatIndices(self): """This tests scatter_add using indices that repeat.""" self._ScatterRepeatIndicesTest(_NumpyAdd, state_ops.scatter_nd_add) @@ -249,8 +255,9 @@ class StatefulScatterNdTest(test.TestCase): # [[0]], dtype=tf.int64), [False]) # var.initializer.run() # session.run([update0, update1]) - # self.assertAllEqual([False, True], var.eval()) + # self.assertAllEqual([False, True], self.evaluate(var)) + @test_util.run_deprecated_v1 def testScatterOutOfRangeCpu(self): # TODO(simister): Re-enable once binary size increase due to # scatter_nd ops is under control. @@ -287,6 +294,7 @@ class StatefulScatterNdTest(test.TestCase): state_ops.scatter_nd_update(ref, indices, updates).get_shape().as_list(), shape) + @test_util.run_deprecated_v1 def testResVarInvalidOutputShape(self): res = variables.Variable( initial_value=lambda: array_ops.zeros(shape=[], dtype=dtypes.float32), @@ -296,6 +304,7 @@ class StatefulScatterNdTest(test.TestCase): with self.assertRaisesOpError("Output must be at least 1-D"): state_ops.scatter_nd_update(res, [[0]], [0.22]).eval() + @test_util.run_deprecated_v1 def testExtraIndicesDimensions(self): indices = array_ops.zeros([1, 1, 2], dtypes.int32) updates = array_ops.zeros([1, 1], dtypes.int32) @@ -307,8 +316,9 @@ class StatefulScatterNdTest(test.TestCase): expected_result = np.zeros([2, 2], dtype=np.int32) with self.cached_session(): ref.initializer.run() - self.assertAllEqual(expected_result, scatter_update.eval()) + self.assertAllEqual(expected_result, self.evaluate(scatter_update)) + @test_util.run_deprecated_v1 def testRank3InvalidShape1(self): indices = array_ops.zeros([3, 2, 2], dtypes.int32) updates = array_ops.zeros([2, 2, 2], dtypes.int32) @@ -318,6 +328,7 @@ class StatefulScatterNdTest(test.TestCase): ValueError, "The outer \\d+ dimensions of indices\\.shape="): state_ops.scatter_nd_update(ref, indices, updates) + @test_util.run_deprecated_v1 def testRank3InvalidShape2(self): indices = array_ops.zeros([2, 2, 1], dtypes.int32) updates = array_ops.zeros([2, 2], dtypes.int32) @@ -327,6 +338,7 @@ class StatefulScatterNdTest(test.TestCase): ValueError, "The inner \\d+ dimensions of input\\.shape="): state_ops.scatter_nd_update(ref, indices, updates) + @test_util.run_deprecated_v1 def testConcurrentUpdates(self): num_updates = 10000 update_values = np.random.rand(num_updates) @@ -341,8 +353,8 @@ class StatefulScatterNdTest(test.TestCase): init = variables.global_variables_initializer() with session.Session() as sess: - sess.run(init) - result = sess.run(scatter) + self.evaluate(init) + result = self.evaluate(scatter) assert np.allclose(result, expected_result) # TODO(fpmc): Re-enable this test when gpu_pip test actually runs on a GPU. @@ -421,7 +433,7 @@ class ScatterNdTest(test.TestCase): b"", b"", b"seven"]) scatter = self.scatter_nd(indices, updates, shape=(8,)) with self.cached_session() as sess: - result = sess.run(scatter) + result = self.evaluate(scatter) self.assertAllEqual(expected, result) # Same indice is updated twice by same value. @@ -432,7 +444,7 @@ class ScatterNdTest(test.TestCase): expected = np.array([b"", b"", b"", b"bb", b"a", b"", b"", b"c"]) scatter = self.scatter_nd(indices, updates, shape=(8,)) with self.cached_session() as sess: - result = sess.run(scatter) + result = self.evaluate(scatter) self.assertAllEqual(expected, result) # Same indice is updated twice by different value. @@ -444,7 +456,7 @@ class ScatterNdTest(test.TestCase): np.array([b"", b"", b"", b"cb", b"a", b"", b"", b"d"])] scatter = self.scatter_nd(indices, updates, shape=(8,)) with self.cached_session() as sess: - result = sess.run(scatter) + result = self.evaluate(scatter) self.assertTrue(np.array_equal(result, expected[0]) or np.array_equal(result, expected[1])) @@ -455,6 +467,7 @@ class ScatterNdTest(test.TestCase): self.assertAllEqual( self.scatter_nd(indices, updates, shape).get_shape().as_list(), shape) + @test_util.run_deprecated_v1 def testExtraIndicesDimensions(self): indices = array_ops.zeros([1, 1, 2], dtypes.int32) updates = array_ops.zeros([1, 1], dtypes.int32) @@ -463,26 +476,30 @@ class ScatterNdTest(test.TestCase): self.assertAllEqual(scatter.get_shape().as_list(), shape) expected_result = np.zeros([2, 2], dtype=np.int32) with self.cached_session(): - self.assertAllEqual(expected_result, scatter.eval()) + self.assertAllEqual(expected_result, self.evaluate(scatter)) + @test_util.run_deprecated_v1 def testUndefinedIndicesShape(self): indices = array_ops.placeholder(dtypes.int32, shape=None) updates = array_ops.placeholder(dtypes.int32, shape=[2, 2, 2]) shape = constant_op.constant([2, 2, 2], dtypes.int32) self.scatter_nd(indices, updates, shape) + @test_util.run_deprecated_v1 def testUndefinedUpdatesShape(self): indices = array_ops.placeholder(dtypes.int32, shape=[2, 2, 2]) updates = array_ops.placeholder(dtypes.int32, shape=None) shape = constant_op.constant([2, 2, 2], dtypes.int32) self.scatter_nd(indices, updates, shape) + @test_util.run_deprecated_v1 def testUndefinedOutputShape(self): indices = array_ops.placeholder(dtypes.int32, shape=[2, 2, 2]) updates = array_ops.placeholder(dtypes.int32, shape=[2, 2, 2]) shape = array_ops.placeholder(dtypes.int32, shape=[None]) self.scatter_nd(indices, updates, shape) + @test_util.run_deprecated_v1 def testEmptyOutputShape1(self): indices = array_ops.zeros([2, 2, 2], dtypes.int32) updates = array_ops.zeros([2, 2, 2], dtypes.int32) @@ -492,6 +509,7 @@ class ScatterNdTest(test.TestCase): ValueError, "Indices and updates specified for empty output shape"): self.scatter_nd(indices, updates, shape) + @test_util.run_deprecated_v1 def testEmptyOutputShape2(self): indices = array_ops.placeholder(dtypes.int32, shape=None) updates = array_ops.placeholder(dtypes.int32, shape=None) @@ -505,6 +523,7 @@ class ScatterNdTest(test.TestCase): updates: np.zeros([2, 2, 2], dtype=np.int32) }) + @test_util.run_deprecated_v1 def testEmptyOutputShape3(self): indices = array_ops.zeros([0], dtypes.int32) updates = array_ops.zeros([0], dtypes.int32) @@ -514,6 +533,7 @@ class ScatterNdTest(test.TestCase): with self.cached_session(): self.assertEqual(scatter.eval().size, 0) + @test_util.run_deprecated_v1 def testRank3InvalidShape1(self): indices = array_ops.zeros([3, 2, 2], dtypes.int32) updates = array_ops.zeros([2, 2, 2], dtypes.int32) @@ -522,6 +542,7 @@ class ScatterNdTest(test.TestCase): ValueError, "The outer \\d+ dimensions of indices\\.shape="): self.scatter_nd(indices, updates, shape) + @test_util.run_deprecated_v1 def testRank3InvalidShape2(self): indices = array_ops.zeros([2, 2, 1], dtypes.int32) updates = array_ops.zeros([2, 2], dtypes.int32) @@ -530,6 +551,7 @@ class ScatterNdTest(test.TestCase): ValueError, "The inner \\d+ dimensions of (input|output)\\.shape="): self.scatter_nd(indices, updates, shape) + @test_util.run_deprecated_v1 def testGradientsRank2ElementUpdate(self): for dtype in GRADIENT_TESTS_DTYPES: indices = constant_op.constant([[0, 0], [1, 1]], dtype=dtypes.int32) @@ -545,10 +567,11 @@ class ScatterNdTest(test.TestCase): expected_input_grad = np.array([[1, 2], [3, 4]], dtype=dtype.as_numpy_dtype()) with self.cached_session(): - self.assertAllEqual(expected_updates_grad, updates_grad.eval()) + self.assertAllEqual(expected_updates_grad, self.evaluate(updates_grad)) if self.non_aliasing_add_test: - self.assertAllEqual(expected_input_grad, input_grad.eval()) + self.assertAllEqual(expected_input_grad, self.evaluate(input_grad)) + @test_util.run_deprecated_v1 def testGradientsRank2SliceUpdate(self): for dtype in GRADIENT_TESTS_DTYPES: indices = constant_op.constant([[1], [0]], dtype=dtypes.int32) @@ -565,10 +588,11 @@ class ScatterNdTest(test.TestCase): expected_input_grad = np.array([[3, 4], [1, 2]], dtype=dtype.as_numpy_dtype()) with self.cached_session(): - self.assertAllEqual(expected_updates_grad, updates_grad.eval()) + self.assertAllEqual(expected_updates_grad, self.evaluate(updates_grad)) if self.non_aliasing_add_test: - self.assertAllEqual(expected_input_grad, input_grad.eval()) + self.assertAllEqual(expected_input_grad, self.evaluate(input_grad)) + @test_util.run_deprecated_v1 def testGradientsRank3SliceUpdate(self): for dtype in GRADIENT_TESTS_DTYPES: indices = constant_op.constant([[[0, 1], [1, 0]], [[0, 0], [1, 1]]], @@ -588,10 +612,11 @@ class ScatterNdTest(test.TestCase): expected_input_grad = np.array([[[1, 2], [3, 4]], [[5, 6], [7, 8]]], dtype=dtype.as_numpy_dtype()) with self.cached_session(): - self.assertAllEqual(expected_updates_grad, updates_grad.eval()) + self.assertAllEqual(expected_updates_grad, self.evaluate(updates_grad)) if self.non_aliasing_add_test: - self.assertAllEqual(expected_input_grad, input_grad.eval()) + self.assertAllEqual(expected_input_grad, self.evaluate(input_grad)) + @test_util.run_deprecated_v1 def testGradientsRank7SliceUpdate(self): for dtype in GRADIENT_TESTS_DTYPES: indices = constant_op.constant( @@ -615,10 +640,11 @@ class ScatterNdTest(test.TestCase): [[[[[[[1, 2], [3, 4]]]], [[[[5, 6], [7, 8]]]]]]], dtype=dtype.as_numpy_dtype()) with self.cached_session(): - self.assertAllEqual(expected_updates_grad, updates_grad.eval()) + self.assertAllEqual(expected_updates_grad, self.evaluate(updates_grad)) if self.non_aliasing_add_test: - self.assertAllEqual(expected_input_grad, input_grad.eval()) + self.assertAllEqual(expected_input_grad, self.evaluate(input_grad)) + @test_util.run_deprecated_v1 def testScatterNdRepatedIndicesAdd(self): indices = array_ops.zeros([100000, 1], dtypes.int32) values = np.random.randn(100000) @@ -627,6 +653,7 @@ class ScatterNdTest(test.TestCase): val = self.scatter_nd(indices, values, shape).eval() self.assertAllClose([np.sum(values)], val) + @test_util.run_deprecated_v1 def testSmokeScatterNdBatch2DSliceDim2(self): with self.cached_session(): indices = array_ops.zeros([3, 5, 2], dtype=dtypes.int32) @@ -634,6 +661,7 @@ class ScatterNdTest(test.TestCase): shape = [4, 6, 7] self.scatter_nd(indices, values, shape).eval() + @test_util.run_deprecated_v1 def testSmokeScatterNdBatch1DSliceDim2(self): with self.cached_session(): indices = array_ops.zeros([0, 2], dtype=dtypes.int32) @@ -641,6 +669,7 @@ class ScatterNdTest(test.TestCase): shape = [4, 6, 7] self.scatter_nd(indices, values, shape).eval() + @test_util.run_deprecated_v1 def testSmokeScatterNdBatch1DSliceDim3ShapeRank7(self): with self.cached_session(): indices = array_ops.zeros([1, 3], dtype=dtypes.int32) @@ -648,6 +677,7 @@ class ScatterNdTest(test.TestCase): shape = [3, 4, 5, 6, 7, 8, 9] self.scatter_nd(indices, values, shape).eval() + @test_util.run_deprecated_v1 def testSmokeScatterNdBatch2DSliceDim3ShapeRank7(self): with self.cached_session(): indices = array_ops.zeros([1, 2, 3], dtype=dtypes.int32) @@ -669,5 +699,55 @@ class ScatterNdNonAliasingAddTest(ScatterNdTest): pass +class ScatterNdTensorTest(test.TestCase): + + @test_util.run_in_graph_and_eager_modes + def testUpdateAddSub(self): + indices = constant_op.constant([[4], [3], [1], [7]]) + updates = constant_op.constant([9, 10, 11, 12], dtype=dtypes.float32) + t = array_ops.ones([8], dtype=dtypes.float32) + assigned = array_ops.tensor_scatter_update(t, indices, updates) + added = array_ops.tensor_scatter_add(t, indices, updates) + subbed = array_ops.tensor_scatter_sub(t, indices, updates) + + self.assertAllEqual(assigned, + constant_op.constant([1, 11, 1, 10, 9, 1, 1, 12])) + self.assertAllEqual(added, + constant_op.constant([1, 12, 1, 11, 10, 1, 1, 13])) + self.assertAllEqual(subbed, + constant_op.constant([1, -10, 1, -9, -8, 1, 1, -11])) + + def testUpdateAddSubGradients(self): + + with self.cached_session(): + indices = constant_op.constant([[3], [1]]) + updates = constant_op.constant([9, 10], dtype=dtypes.float32) + x = array_ops.ones([4], dtype=dtypes.float32) + + assigned = array_ops.tensor_scatter_update(x, indices, updates) + added = array_ops.tensor_scatter_add(x, indices, updates) + subbed = array_ops.tensor_scatter_sub(x, indices, updates) + + err_assigned = gradient_checker.compute_gradient_error( + x, [4], assigned, [4]) + err_added = gradient_checker.compute_gradient_error(x, [4], added, [4]) + err_subbed = gradient_checker.compute_gradient_error(x, [4], subbed, [4]) + + self.assertLess(err_assigned, 2e-4) + self.assertLess(err_added, 2e-4) + self.assertLess(err_subbed, 2e-4) + + err_assigned_wrt_updates = gradient_checker.compute_gradient_error( + updates, [2], assigned, [4]) + err_added_wrt_updates = gradient_checker.compute_gradient_error( + updates, [2], added, [4]) + err_subbed_wrt_updates = gradient_checker.compute_gradient_error( + updates, [2], subbed, [4]) + + self.assertLess(err_assigned_wrt_updates, 2e-4) + self.assertLess(err_added_wrt_updates, 2e-4) + self.assertLess(err_subbed_wrt_updates, 2e-4) + + if __name__ == "__main__": test.main() diff --git a/tensorflow/python/kernel_tests/scatter_ops_test.py b/tensorflow/python/kernel_tests/scatter_ops_test.py index 87c345245c1..623c17d373c 100644 --- a/tensorflow/python/kernel_tests/scatter_ops_test.py +++ b/tensorflow/python/kernel_tests/scatter_ops_test.py @@ -22,6 +22,7 @@ import numpy as np from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes +from tensorflow.python.framework import test_util from tensorflow.python.ops import state_ops from tensorflow.python.ops import variables from tensorflow.python.platform import test @@ -196,84 +197,111 @@ class ScatterTest(test.TestCase): self._VariableRankTest(tf_scatter, vtype, itype, repeat_indices, updates_are_scalar) + @test_util.run_deprecated_v1 def testVariableRankUpdate(self): self._VariableRankTests(state_ops.scatter_update, False) + @test_util.run_deprecated_v1 def testVariableRankAdd(self): self._VariableRankTests(state_ops.scatter_add, False) + @test_util.run_deprecated_v1 def testVariableRankSub(self): self._VariableRankTests(state_ops.scatter_sub, False) + @test_util.run_deprecated_v1 def testVariableRankMul(self): self._VariableRankTests(state_ops.scatter_mul, False) + @test_util.run_deprecated_v1 def testVariableRankDiv(self): self._VariableRankTests(state_ops.scatter_div, False) + @test_util.run_deprecated_v1 def testVariableRankMin(self): self._VariableRankTests(state_ops.scatter_min, False) + @test_util.run_deprecated_v1 def testVariableRankMax(self): self._VariableRankTests(state_ops.scatter_max, False) + @test_util.run_deprecated_v1 def testRepeatIndicesAdd(self): self._VariableRankTests(state_ops.scatter_add, True) + @test_util.run_deprecated_v1 def testRepeatIndicesSub(self): self._VariableRankTests(state_ops.scatter_sub, True) + @test_util.run_deprecated_v1 def testRepeatIndicesMul(self): self._VariableRankTests(state_ops.scatter_mul, True) + @test_util.run_deprecated_v1 def testRepeatIndicesDiv(self): self._VariableRankTests(state_ops.scatter_div, True) + @test_util.run_deprecated_v1 def testRepeatIndicesMin(self): self._VariableRankTests(state_ops.scatter_min, True) + @test_util.run_deprecated_v1 def testRepeatIndicesMax(self): self._VariableRankTests(state_ops.scatter_max, True) + @test_util.run_deprecated_v1 def testVariableRankUpdateScalar(self): self._VariableRankTests(state_ops.scatter_update, False, True) + @test_util.run_deprecated_v1 def testVariableRankAddScalar(self): self._VariableRankTests(state_ops.scatter_add, False, True) + @test_util.run_deprecated_v1 def testVariableRankSubScalar(self): self._VariableRankTests(state_ops.scatter_sub, False, True) + @test_util.run_deprecated_v1 def testVariableRankMulScalar(self): self._VariableRankTests(state_ops.scatter_mul, False, True) + @test_util.run_deprecated_v1 def testVariableRankDivScalar(self): self._VariableRankTests(state_ops.scatter_div, False, True) + @test_util.run_deprecated_v1 def testVariableRankMinScalar(self): self._VariableRankTests(state_ops.scatter_min, False, True) + @test_util.run_deprecated_v1 def testVariableRankMaxScalar(self): self._VariableRankTests(state_ops.scatter_max, False, True) + @test_util.run_deprecated_v1 def testRepeatIndicesAddScalar(self): self._VariableRankTests(state_ops.scatter_add, True, True) + @test_util.run_deprecated_v1 def testRepeatIndicesSubScalar(self): self._VariableRankTests(state_ops.scatter_sub, True, True) + @test_util.run_deprecated_v1 def testRepeatIndicesMulScalar(self): self._VariableRankTests(state_ops.scatter_mul, True, True) + @test_util.run_deprecated_v1 def testRepeatIndicesDivScalar(self): self._VariableRankTests(state_ops.scatter_div, True, True) + @test_util.run_deprecated_v1 def testRepeatIndicesMinScalar(self): self._VariableRankTests(state_ops.scatter_min, True, True) + @test_util.run_deprecated_v1 def testRepeatIndicesMaxScalar(self): self._VariableRankTests(state_ops.scatter_max, True, True) + @test_util.run_deprecated_v1 def testBooleanScatterUpdate(self): if not test.is_gpu_available(): with self.session(use_gpu=False) as session: @@ -286,8 +314,9 @@ class ScatterTest(test.TestCase): session.run([update0, update1]) - self.assertAllEqual([False, True], var.eval()) + self.assertAllEqual([False, True], self.evaluate(var)) + @test_util.run_deprecated_v1 def testScatterOutOfRangeCpu(self): for op, _ in _TF_OPS_TO_NUMPY.items(): params = np.array([1, 2, 3, 4, 5, 6]).astype(np.float32) @@ -320,19 +349,19 @@ class ScatterTest(test.TestCase): updates = np.array([-3, -4, -5]).astype(np.float32) # With GPU, the code ignores indices that are out of range. # We don't test the implementation; just test there's no failures. - with self.cached_session(force_gpu=True): + with test_util.force_gpu(): ref = variables.Variable(params) ref.initializer.run() # Indices all in range, no problem. indices = np.array([2, 0, 5]) - op(ref, indices, updates).eval() + self.evaluate(op(ref, indices, updates)) # Indicies out of range should not fail. indices = np.array([-1, 0, 5]) - op(ref, indices, updates).eval() + self.evaluate(op(ref, indices, updates)) indices = np.array([2, 0, 6]) - op(ref, indices, updates).eval() + self.evaluate(op(ref, indices, updates)) if __name__ == '__main__': diff --git a/tensorflow/python/kernel_tests/segment_reduction_ops_test.py b/tensorflow/python/kernel_tests/segment_reduction_ops_test.py index 3f7e43b5335..8af1b47e83c 100644 --- a/tensorflow/python/kernel_tests/segment_reduction_ops_test.py +++ b/tensorflow/python/kernel_tests/segment_reduction_ops_test.py @@ -26,6 +26,7 @@ from tensorflow.python.client import session from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes as dtypes_lib from tensorflow.python.framework import ops +from tensorflow.python.framework import test_util from tensorflow.python.ops import gradient_checker from tensorflow.python.ops import math_ops from tensorflow.python.ops import variables @@ -118,7 +119,7 @@ class SegmentReductionOpTest(SegmentReductionHelper): for np_op1, np_op2, tf_op in curr_ops_list: np_ans = self._segmentReduce(indices, np_x, np_op1, np_op2) s = tf_op(data=tf_x, segment_ids=indices) - tf_ans = s.eval() + tf_ans = self.evaluate(s) self.assertAllClose(np_ans, tf_ans) # NOTE(mrry): The static shape inference that computes # `tf_ans.shape` can only infer that sizes from dimension 1 @@ -126,6 +127,7 @@ class SegmentReductionOpTest(SegmentReductionHelper): # and may therefore vary dynamically. self.assertAllEqual(np_ans.shape[1:], tf_ans.shape[1:]) + @test_util.run_deprecated_v1 def testSegmentIdsShape(self): shape = [4, 4] tf_x, _ = self._input(shape) @@ -133,6 +135,7 @@ class SegmentReductionOpTest(SegmentReductionHelper): with self.assertRaises(ValueError): math_ops.segment_sum(data=tf_x, segment_ids=indices) + @test_util.run_deprecated_v1 def testSegmentIdsSize(self): shape = [4, 4] for use_gpu in [True, False]: @@ -141,8 +144,9 @@ class SegmentReductionOpTest(SegmentReductionHelper): indices = [0, 1] s = math_ops.segment_sum(data=tf_x, segment_ids=indices) with self.assertRaisesOpError("segment_ids should be the same size"): - s.eval() + self.evaluate(s) + @test_util.run_deprecated_v1 def testSegmentIdsValid(self): # This is a baseline for the following SegmentIdsInvalid* tests. shape = [4, 4] @@ -161,7 +165,7 @@ class SegmentReductionOpTest(SegmentReductionHelper): indices = [1, 1, 2, 2] np_ans = self._segmentReduce(indices, np_x, np.add) s = math_ops.segment_sum(data=tf_x, segment_ids=indices) - tf_ans = s.eval() + tf_ans = self.evaluate(s) self.assertAllClose(np_ans, tf_ans) def testSegmentIdsHole(self): @@ -172,9 +176,10 @@ class SegmentReductionOpTest(SegmentReductionHelper): indices = [0, 0, 3, 3] np_ans = self._segmentReduce(indices, np_x, np.add) s = math_ops.segment_sum(data=tf_x, segment_ids=indices) - tf_ans = s.eval() + tf_ans = self.evaluate(s) self.assertAllClose(np_ans, tf_ans) + @test_util.run_deprecated_v1 def testSegmentIdsInvalid1(self): shape = [4, 4] with self.cached_session(): @@ -184,8 +189,9 @@ class SegmentReductionOpTest(SegmentReductionHelper): with self.assertRaisesOpError( r"Segment id -1 out of range \[0, 1\), possibly because " "'segment_ids' input is not sorted."): - s.eval() + self.evaluate(s) + @test_util.run_deprecated_v1 def testSegmentIdsInvalid2(self): shape = [4, 4] with self.cached_session(): @@ -193,8 +199,9 @@ class SegmentReductionOpTest(SegmentReductionHelper): indices = [0, 1, 0, 1] s = math_ops.segment_sum(data=tf_x, segment_ids=indices) with self.assertRaisesOpError("segment ids are not increasing"): - s.eval() + self.evaluate(s) + @test_util.run_deprecated_v1 def testSegmentIdsInvalid3(self): shape = [4, 4] with self.cached_session(): @@ -204,8 +211,9 @@ class SegmentReductionOpTest(SegmentReductionHelper): with self.assertRaisesOpError( r"Segment id 1 out of range \[0, 1\), possibly " "because 'segment_ids' input is not sorted."): - s.eval() + self.evaluate(s) + @test_util.run_deprecated_v1 def testSegmentIdsInvalid4(self): shape = [4, 4] for use_gpu in [True, False]: @@ -214,8 +222,9 @@ class SegmentReductionOpTest(SegmentReductionHelper): indices = [0, 0, 0, -1] s = math_ops.segment_sum(data=tf_x, segment_ids=indices) with self.assertRaisesOpError("segment ids must be >= 0"): - s.eval() + self.evaluate(s) + @test_util.run_deprecated_v1 def testSegmentIdsInvalid5(self): shape = [4, 4] for use_gpu in [True, False]: @@ -224,8 +233,9 @@ class SegmentReductionOpTest(SegmentReductionHelper): indices = [0, 0, 0, -2] s = math_ops.segment_sum(data=tf_x, segment_ids=indices) with self.assertRaisesOpError("segment ids must be >= 0"): - s.eval() + self.evaluate(s) + @test_util.run_deprecated_v1 def testGradient(self): shape = [4, 4] indices = [0, 1, 2, 2] @@ -297,7 +307,7 @@ class UnsortedSegmentTest(SegmentReductionHelper): indices, np_x, np_op1, np_op2, num_segments=num_segments, initial_value=init_op(dtype)) s = tf_op(tf_x, segment_ids=indices, num_segments=num_segments) - tf_ans = s.eval() + tf_ans = self.evaluate(s) if dtype is dtypes_lib.bfloat16: tf_ans = tf_ans.astype(np.float32) self.assertAllCloseAccordingToType(np_ans, tf_ans) @@ -320,10 +330,11 @@ class UnsortedSegmentTest(SegmentReductionHelper): data=tf_x, segment_ids=indices, num_segments=num_segments_constant) - tf_ans = s.eval() + tf_ans = self.evaluate(s) self.assertAllClose(np_ans, tf_ans) self.assertShapeEqual(np_ans, s) + @test_util.run_deprecated_v1 def testGradients(self): num_cols = 2 indices_flat = np.array([0, 4, 0, -1, 3, -1, 4, 7, 7, 3]) @@ -346,6 +357,7 @@ class UnsortedSegmentTest(SegmentReductionHelper): delta=1) self.assertAllClose(jacob_t, jacob_n) + @test_util.run_deprecated_v1 def testProdGrad(self): # additional test for the prod gradient to ensure correct handling of zeros values = np.array([0, 0, 1, 0, 2, 2, 3, 3, 3], dtype=np.float32) @@ -370,6 +382,7 @@ class UnsortedSegmentTest(SegmentReductionHelper): self.assertAllClose(jacob_t, jacob_n) self.assertAllClose(jacob_t, grad_gt) + @test_util.run_deprecated_v1 def testGradientMatchesSegmentSum(self): # Strategy: compute the gradient for UnsortedSegmentSum and SegmentSum # and compare the outputs, which should be identical. @@ -403,6 +416,7 @@ class UnsortedSegmentTest(SegmentReductionHelper): self.assertAllClose(unsorted_jacob_t, sorted_jacob_t) self.assertAllClose(unsorted_jacob_n, sorted_jacob_n) + @test_util.run_deprecated_v1 def testBadIndices(self): # Note: GPU kernel does not return the out-of-range error needed for this # test, so this test is marked as cpu-only. @@ -412,8 +426,9 @@ class UnsortedSegmentTest(SegmentReductionHelper): unsorted = math_ops.unsorted_segment_sum([[17]], bad, num_segments=2) with self.assertRaisesOpError( r"segment_ids\[0,0\] = %d is out of range \[0, 2\)" % bad[0][0]): - unsorted.eval() + self.evaluate(unsorted) + @test_util.run_deprecated_v1 def testEmptySecondDimension(self): dtypes = [np.float16, np.float32, np.float64, np.int64, np.int32, np.complex64, np.complex128] @@ -443,7 +458,7 @@ class UnsortedSegmentTest(SegmentReductionHelper): np.place(indices, indices == 8, [-1]) s = math_ops.unsorted_segment_sum( data=tf_x, segment_ids=indices, num_segments=num_segments) - tf_ans = s.eval() + tf_ans = self.evaluate(s) self.assertAllClose(np_ans, tf_ans) self.assertShapeEqual(np_ans, s) @@ -499,7 +514,7 @@ class SparseSegmentReductionOpTest(SparseSegmentReductionHelper): np_ans = self._sparseSegmentReduce(np_x, np_indices, segment_indices, np_op1, np_op2) s = tf_op(data=tf_x, indices=tf_indices, segment_ids=segment_indices) - tf_ans = s.eval() + tf_ans = self.evaluate(s) self.assertAllClose(np_ans, tf_ans) # NOTE(mrry): The static shape inference that computes # `tf_ans.shape` can only infer that sizes from dimension 1 @@ -518,7 +533,7 @@ class SparseSegmentReductionOpTest(SparseSegmentReductionHelper): np_ans = self._sparseSegmentReduce(np_x, tf_indices, segment_indices, np_op1, np_op2) s = tf_op(data=tf_x, indices=tf_indices, segment_ids=segment_indices) - tf_ans = s.eval() + tf_ans = self.evaluate(s) self.assertAllClose(np_ans, tf_ans) def testWithNumSegments(self): @@ -543,7 +558,7 @@ class SparseSegmentReductionOpTest(SparseSegmentReductionHelper): indices=tf_indices, segment_ids=segment_indices, num_segments=num_segments) - tf_ans = s.eval() + tf_ans = self.evaluate(s) self.assertAllClose(np_ans, tf_ans) def testWithEmptySegments(self): @@ -562,7 +577,7 @@ class SparseSegmentReductionOpTest(SparseSegmentReductionHelper): indices=tf_indices, segment_ids=segment_indices, num_segments=num_segments) - tf_ans = s.eval() + tf_ans = self.evaluate(s) self.assertAllClose(np.zeros([5, 4]), tf_ans) def testSegmentIdsGreaterThanZero(self): @@ -576,7 +591,7 @@ class SparseSegmentReductionOpTest(SparseSegmentReductionHelper): np_ans = self._sparseSegmentReduce(np_x, tf_indices, segment_indices, np_op1, np_op2) s = tf_op(data=tf_x, indices=tf_indices, segment_ids=segment_indices) - tf_ans = s.eval() + tf_ans = self.evaluate(s) self.assertAllClose(np_ans, tf_ans) def testValid(self): @@ -588,8 +603,9 @@ class SparseSegmentReductionOpTest(SparseSegmentReductionHelper): with self.session(use_gpu=False): for tf_op in ops_list: s = tf_op(data=tf_x, indices=tf_indices, segment_ids=segment_indices) - s.eval() + self.evaluate(s) + @test_util.run_deprecated_v1 def testIndicesInvalid1(self): tf_x, _ = self._input([10, 4], dtype=dtypes_lib.float32) ops_list = [math_ops.sparse_segment_sum, math_ops.sparse_segment_mean] @@ -600,8 +616,9 @@ class SparseSegmentReductionOpTest(SparseSegmentReductionHelper): s = tf_op(data=tf_x, indices=tf_indices, segment_ids=segment_indices) with self.assertRaisesOpError( r"indices\[1\] == -1 out of range \[0, 10\)"): - s.eval() + self.evaluate(s) + @test_util.run_deprecated_v1 def testIndicesInvalid2(self): tf_x, _ = self._input([10, 4], dtype=dtypes_lib.float32) ops_list = [math_ops.sparse_segment_sum, math_ops.sparse_segment_mean] @@ -612,8 +629,9 @@ class SparseSegmentReductionOpTest(SparseSegmentReductionHelper): s = tf_op(data=tf_x, indices=tf_indices, segment_ids=segment_indices) with self.assertRaisesOpError( r"indices\[3\] == 10 out of range \[0, 10\)"): - s.eval() + self.evaluate(s) + @test_util.run_deprecated_v1 def testSegmentsInvalid2(self): tf_x, _ = self._input([10, 4], dtype=dtypes_lib.float32) ops_list = [math_ops.sparse_segment_sum, math_ops.sparse_segment_mean] @@ -623,8 +641,9 @@ class SparseSegmentReductionOpTest(SparseSegmentReductionHelper): for tf_op in ops_list: s = tf_op(data=tf_x, indices=tf_indices, segment_ids=segment_indices) with self.assertRaisesOpError("segment ids are not increasing"): - s.eval() + self.evaluate(s) + @test_util.run_deprecated_v1 def testSegmentsInvalid3(self): tf_x, _ = self._input([10, 4], dtype=dtypes_lib.float32) ops_list = [math_ops.sparse_segment_sum, math_ops.sparse_segment_mean] @@ -636,8 +655,9 @@ class SparseSegmentReductionOpTest(SparseSegmentReductionHelper): with self.assertRaisesOpError( r"Segment id 1 out of range \[0, 1\), possibly because " "'segment_ids' input is not sorted"): - s.eval() + self.evaluate(s) + @test_util.run_deprecated_v1 def testSegmentsInvalid4(self): tf_x, _ = self._input([10, 4], dtype=dtypes_lib.float32) ops_list = [math_ops.sparse_segment_sum, math_ops.sparse_segment_mean] @@ -649,8 +669,9 @@ class SparseSegmentReductionOpTest(SparseSegmentReductionHelper): with self.assertRaisesOpError( r"Segment id -1 out of range \[0, 2\), possibly because " "'segment_ids' input is not sorted"): - s.eval() + self.evaluate(s) + @test_util.run_deprecated_v1 def testSegmentsInvalid6(self): tf_x, _ = self._input([10, 4], dtype=dtypes_lib.float32) ops_list = [math_ops.sparse_segment_sum, math_ops.sparse_segment_mean] @@ -660,8 +681,9 @@ class SparseSegmentReductionOpTest(SparseSegmentReductionHelper): for tf_op in ops_list: s = tf_op(data=tf_x, indices=tf_indices, segment_ids=segment_indices) with self.assertRaisesOpError("segment ids must be >= 0"): - s.eval() + self.evaluate(s) + @test_util.run_deprecated_v1 def testSegmentsInvalid7(self): tf_x, _ = self._input([10, 4], dtype=dtypes_lib.float32) ops_list = [math_ops.sparse_segment_sum, math_ops.sparse_segment_mean] @@ -671,7 +693,7 @@ class SparseSegmentReductionOpTest(SparseSegmentReductionHelper): for tf_op in ops_list: s = tf_op(data=tf_x, indices=tf_indices, segment_ids=segment_indices) with self.assertRaisesOpError("segment ids must be >= 0"): - s.eval() + self.evaluate(s) def testSegmentWithNumSegmentsValid(self): # Baseline for the test*WithNumSegmentsInvalid* methods below. @@ -690,8 +712,9 @@ class SparseSegmentReductionOpTest(SparseSegmentReductionHelper): indices=tf_indices, segment_ids=segment_indices, num_segments=num_segments) - s.eval() + self.evaluate(s) + @test_util.run_deprecated_v1 def testSegmentWithNumSegmentsInvalid1(self): tf_x, _ = self._input([10, 4], dtype=dtypes_lib.float32) ops_list = [ @@ -709,8 +732,9 @@ class SparseSegmentReductionOpTest(SparseSegmentReductionHelper): segment_ids=segment_indices, num_segments=num_segments) with self.assertRaisesOpError("segment ids must be < num_segments"): - s.eval() + self.evaluate(s) + @test_util.run_deprecated_v1 def testSegmentWithNumSegmentsInvalid2(self): tf_x, _ = self._input([10, 4], dtype=dtypes_lib.float32) ops_list = [ @@ -730,6 +754,7 @@ class SparseSegmentReductionOpTest(SparseSegmentReductionHelper): segment_ids=segment_indices, num_segments=num_segments) + @test_util.run_deprecated_v1 def testGradient(self): shape = [10, 4] @@ -748,6 +773,7 @@ class SparseSegmentReductionOpTest(SparseSegmentReductionHelper): delta=1) self.assertAllClose(jacob_t, jacob_n) + @test_util.run_deprecated_v1 def testGradientWithEmptySegmentsAtEnd(self): shape = [10, 4] @@ -785,8 +811,9 @@ class SparseSegmentReductionOpTest(SparseSegmentReductionHelper): with self.session(use_gpu=False): for tf_op in ops_list: s = tf_op(tf_x, tf_indices, segment_indices, 10) - s.eval() + self.evaluate(s) + @test_util.run_deprecated_v1 def testGradientIndicesInvalid1(self): tf_x, _ = self._input([3, 4], dtype=dtypes_lib.float32) ops_list = [ @@ -798,8 +825,9 @@ class SparseSegmentReductionOpTest(SparseSegmentReductionHelper): for tf_op in ops_list: s = tf_op(tf_x, tf_indices, segment_indices, 10) with self.assertRaisesOpError(r"Index 10 out of range \[0, 10\)"): - s.eval() + self.evaluate(s) + @test_util.run_deprecated_v1 def testGradientIndicesInvalid2(self): tf_x, _ = self._input([3, 4], dtype=dtypes_lib.float32) ops_list = [ @@ -811,8 +839,9 @@ class SparseSegmentReductionOpTest(SparseSegmentReductionHelper): for tf_op in ops_list: s = tf_op(tf_x, tf_indices, segment_indices, 10) with self.assertRaisesOpError(r"Index -1 out of range \[0, 10\)"): - s.eval() + self.evaluate(s) + @test_util.run_deprecated_v1 def testGradientSegmentsInvalid1(self): tf_x, _ = self._input( [3, 4], dtype=dtypes_lib.float32) # expecting 3 segments @@ -825,8 +854,9 @@ class SparseSegmentReductionOpTest(SparseSegmentReductionHelper): for tf_op in ops_list: s = tf_op(tf_x, tf_indices, segment_indices, 10) with self.assertRaisesOpError("Invalid number of segments"): - s.eval() + self.evaluate(s) + @test_util.run_deprecated_v1 def testGradientSegmentsInvalid2(self): tf_x, _ = self._input([1, 4], dtype=dtypes_lib.float32) ops_list = [ @@ -838,8 +868,9 @@ class SparseSegmentReductionOpTest(SparseSegmentReductionHelper): for tf_op in ops_list: s = tf_op(tf_x, tf_indices, segment_indices, 10) with self.assertRaisesOpError(r"Segment id 1 out of range \[0, 1\)"): - s.eval() + self.evaluate(s) + @test_util.run_deprecated_v1 def testGradientSegmentsInvalid3(self): tf_x, _ = self._input([2, 4], dtype=dtypes_lib.float32) ops_list = [ @@ -851,8 +882,9 @@ class SparseSegmentReductionOpTest(SparseSegmentReductionHelper): for tf_op in ops_list: s = tf_op(tf_x, tf_indices, segment_indices, 10) with self.assertRaisesOpError(r"Segment id -1 out of range \[0, 2\)"): - s.eval() + self.evaluate(s) + @test_util.run_deprecated_v1 def testGradientSegmentsInvalid4(self): tf_x, _ = self._input([0, 4], dtype=dtypes_lib.float32) ops_list = [ @@ -864,7 +896,8 @@ class SparseSegmentReductionOpTest(SparseSegmentReductionHelper): for tf_op in ops_list: s = tf_op(tf_x, tf_indices, segment_indices, 10) with self.assertRaisesOpError(r"Segment id 0 out of range \[0, 0\)"): - s.eval() + self.evaluate(s) + class SegmentReductionOpBenchmark(test.Benchmark): outer_dim_options = [2**x for x in range(9, 14, 2)] diff --git a/tensorflow/python/kernel_tests/self_adjoint_eig_op_test.py b/tensorflow/python/kernel_tests/self_adjoint_eig_op_test.py index 1b4aff8c9ca..42577f7e423 100644 --- a/tensorflow/python/kernel_tests/self_adjoint_eig_op_test.py +++ b/tensorflow/python/kernel_tests/self_adjoint_eig_op_test.py @@ -63,7 +63,7 @@ class SelfAdjointEigTest(test.TestCase): e1 = linalg_ops.self_adjoint_eigvals(matrix1) e2 = linalg_ops.self_adjoint_eigvals(matrix2) all_ops += [e1, e2] - val = sess.run(all_ops) + val = self.evaluate(all_ops) self.assertAllEqual(val[0], val[2]) # The algorithm is slightly different for compute_v being True and False, # so require approximate equality only here. @@ -81,7 +81,7 @@ class SelfAdjointEigTest(test.TestCase): self.assertEqual(matrix.shape, (32, 32)) matrix_tensor = constant_op.constant(matrix) with self.session(use_gpu=True) as sess: - (e, v) = sess.run(linalg_ops.self_adjoint_eig(matrix_tensor)) + (e, v) = self.evaluate(linalg_ops.self_adjoint_eig(matrix_tensor)) self.assertEqual(e.size, 32) self.assertAllClose( np.matmul(v, v.transpose()), np.eye(32, dtype=np.float32), atol=2e-3) @@ -164,8 +164,8 @@ def _GetSelfAdjointEigTest(dtype_, shape_, compute_v_): self.assertAllClose(a_ev.eval(), a, atol=atol) # Compare to numpy.linalg.eigh. - CompareEigenDecompositions(self, np_e, np_v, - tf_e.eval(), tf_v.eval(), atol) + CompareEigenDecompositions(self, np_e, np_v, self.evaluate(tf_e), + self.evaluate(tf_v), atol) else: tf_e = linalg_ops.self_adjoint_eigvals(constant_op.constant(a)) self.assertAllClose( diff --git a/tensorflow/python/kernel_tests/session_ops_test.py b/tensorflow/python/kernel_tests/session_ops_test.py index 03e1ae852fc..dc663cb091c 100644 --- a/tensorflow/python/kernel_tests/session_ops_test.py +++ b/tensorflow/python/kernel_tests/session_ops_test.py @@ -37,7 +37,7 @@ class SessionOpsTest(test.TestCase): b = constant_op.constant(5) c = math_ops.multiply(a, b) h = session_ops.get_session_handle(c) - h = sess.run(h) + h = self.evaluate(h) # Feed a tensor handle. f, x = session_ops.get_session_tensor(h.handle, dtypes.int32) @@ -51,7 +51,7 @@ class SessionOpsTest(test.TestCase): b = constant_op.constant(5) c = math_ops.multiply(a, b) h = session_ops.get_session_handle(c) - h = sess.run(h) + h = self.evaluate(h) # Get the tensor from its handle. self.assertEqual(50, h.eval()) @@ -64,7 +64,7 @@ class SessionOpsTest(test.TestCase): c = math_ops.multiply(a, b) h = session_ops.get_session_handle(c) v = math_ops.multiply(a, c) - h, v = sess.run([h, v]) + h, v = self.evaluate([h, v]) self.assertEqual(50, h.eval()) self.assertEqual(500, v) @@ -77,7 +77,7 @@ class SessionOpsTest(test.TestCase): p = math_ops.less(a, b) c = math_ops.multiply(a, b) h = session_ops.get_session_handle(c) - p, h = sess.run([p, h]) + p, h = self.evaluate([p, h]) # Run by feeding a tensor handle. f, x = session_ops.get_session_tensor(h.handle, dtypes.int32) @@ -94,7 +94,7 @@ class SessionOpsTest(test.TestCase): # Initialize a handle. a = constant_op.constant(0) h = session_ops.get_session_handle(a) - h = sess.run(h) + h = self.evaluate(h) # Do some computation. f, x = session_ops.get_session_tensor(h.handle, dtypes.int32) @@ -111,7 +111,7 @@ class SessionOpsTest(test.TestCase): # Initialize a handle. a = constant_op.constant(0) h = session_ops.get_session_handle(a) - h = sess.run(h) + h = self.evaluate(h) # Do some computation. f, x = session_ops.get_session_tensor(h.handle, dtypes.int32) @@ -133,7 +133,7 @@ class SessionOpsTest(test.TestCase): b = constant_op.constant(5) c = math_ops.multiply(a, b) h = session_ops.get_session_handle(c) - h = sess.run(h) + h = self.evaluate(h) # Feed a tensor handle. f, x = session_ops.get_session_tensor(h.handle, dtypes.int32) @@ -144,7 +144,7 @@ class SessionOpsTest(test.TestCase): with ops.device(test.gpu_device_name()): a = constant_op.constant(10) h = session_ops.get_session_handle(a) - h = sess.run(h) + h = self.evaluate(h) self.assertEqual(100, sess.run(y, feed_dict={f: h.handle})) def testHandleDelete(self): @@ -154,7 +154,7 @@ class SessionOpsTest(test.TestCase): b = constant_op.constant(5) c = math_ops.multiply(a, b) h = session_ops.get_session_handle(c) - sess.run(h).delete() + self.evaluate(h).delete() def testHandleDeleteRaw(self): with self.cached_session() as sess: @@ -163,7 +163,7 @@ class SessionOpsTest(test.TestCase): b = constant_op.constant(5) c = math_ops.multiply(a, b) h = session_ops.get_session_handle(c) - h = sess.run(h) + h = self.evaluate(h) # Delete using a raw tensor handle. raw_h = h.get_raw_handle() @@ -174,10 +174,10 @@ class SessionOpsTest(test.TestCase): with self.cached_session() as sess: with ops.device(test.gpu_device_name()): a = constant_op.constant(1.0) - a_handle = sess.run(session_ops.get_session_handle(a)) + a_handle = self.evaluate(session_ops.get_session_handle(a)) with ops.device("/cpu:0"): b = constant_op.constant(2.0) - b_handle = sess.run(session_ops.get_session_handle(b)) + b_handle = self.evaluate(session_ops.get_session_handle(b)) a_p, a_t = session_ops.get_session_tensor(a_handle.handle, dtypes.float32) b_p, b_t = session_ops.get_session_tensor(b_handle.handle, dtypes.float32) @@ -193,8 +193,8 @@ class SessionOpsTest(test.TestCase): # initial values live on CPU with ops.device("/cpu:0"): one = constant_op.constant(1, dtype=dtypes.float32) - one_handle = sess.run(session_ops.get_session_handle(one)) - x_handle = sess.run(session_ops.get_session_handle(one)) + one_handle = self.evaluate(session_ops.get_session_handle(one)) + x_handle = self.evaluate(session_ops.get_session_handle(one)) # addition lives on GPU with ops.device(test.gpu_device_name()): @@ -219,8 +219,8 @@ class SessionOpsTest(test.TestCase): b = constant_op.constant(2.0) b_handle_op = session_ops.get_session_handle(b) - a_handle = sess.run(a_handle_op) - b_handle = sess.run(b_handle_op) + a_handle = self.evaluate(a_handle_op) + b_handle = self.evaluate(b_handle_op) a_p, a_t = session_ops.get_session_tensor(a_handle.handle, dtypes.float32) b_p, b_t = session_ops.get_session_tensor(b_handle.handle, dtypes.float32) @@ -239,7 +239,7 @@ class SessionOpsTest(test.TestCase): c = math_ops.multiply(a, b) d = math_ops.multiply(c, c) - h_c = sess.run(session_ops.get_session_handle(c)) + h_c = self.evaluate(session_ops.get_session_handle(c)) self.assertAllClose(2500.0, sess.run(d, feed_dict={c: h_c})) @@ -248,7 +248,7 @@ class SessionOpsTest(test.TestCase): a = constant_op.constant(10.0) b = constant_op.constant(5.0) c = math_ops.multiply(a, b) - h_c = sess.run(session_ops.get_session_handle(c)) + h_c = self.evaluate(session_ops.get_session_handle(c)) d = array_ops.identity(c) c_val = sess.run(c, feed_dict={c: h_c}) @@ -277,8 +277,8 @@ class SessionOpsTest(test.TestCase): d = math_ops.div(a, b) e = math_ops.subtract(c, d) - h_c = sess.run(session_ops.get_session_handle(c)) - h_d = sess.run(session_ops.get_session_handle(d)) + h_c = self.evaluate(session_ops.get_session_handle(c)) + h_d = self.evaluate(session_ops.get_session_handle(d)) self.assertAllClose(48.0, sess.run(e, feed_dict={c: h_c, d: h_d})) self.assertAllClose(-48.0, sess.run(e, feed_dict={c: h_d, d: h_c})) @@ -288,13 +288,13 @@ class SessionOpsTest(test.TestCase): a = variables.Variable(12.0) inc_a = state_ops.assign_add(a, 2.0) b = math_ops.add(a, 5.0) - sess.run(a.initializer) + self.evaluate(a.initializer) h_a_read = sess.run(session_ops.get_session_handle(a.read_value())) - self.assertAllClose(12.0, sess.run(a)) + self.assertAllClose(12.0, self.evaluate(a)) self.assertAllClose(17.0, sess.run(b, feed_dict={a: h_a_read})) - sess.run(inc_a) + self.evaluate(inc_a) self.assertAllClose(19.0, sess.run(b, feed_dict={a: h_a_read})) diff --git a/tensorflow/python/kernel_tests/sets_test.py b/tensorflow/python/kernel_tests/sets_test.py index 8335e9c139a..b4f23229348 100644 --- a/tensorflow/python/kernel_tests/sets_test.py +++ b/tensorflow/python/kernel_tests/sets_test.py @@ -70,6 +70,7 @@ def _dense_to_sparse(dense, dtype): class SetOpsTest(test_util.TensorFlowTestCase): + @test_util.run_deprecated_v1 def test_set_size_2d(self): for dtype in _DTYPES: self._test_set_size_2d(dtype) @@ -83,6 +84,7 @@ class SetOpsTest(test_util.TensorFlowTestCase): self.assertAllEqual( [0, 3], self._set_size(_dense_to_sparse([[], [1, 9, 2]], dtype))) + @test_util.run_deprecated_v1 def test_set_size_duplicates_2d(self): for dtype in _DTYPES: self._test_set_size_duplicates_2d(dtype) @@ -96,6 +98,7 @@ class SetOpsTest(test_util.TensorFlowTestCase): 6, 7, 8, 8, 6, 7, 5, 3, 3, 0, 6, 6, 9, 0, 0, 0 ], [999, 1, -1000], [], [-1]], dtype))) + @test_util.run_deprecated_v1 def test_set_size_3d(self): for dtype in _DTYPES: self._test_set_size_3d(dtype) @@ -159,10 +162,11 @@ class SetOpsTest(test_util.TensorFlowTestCase): self.assertEqual(None, op.get_shape().dims) self.assertEqual(dtypes.int32, op.dtype) with self.cached_session() as sess: - results = sess.run(ops) + results = self.evaluate(ops) self.assertAllEqual(results[0], results[1]) return results[0] + @test_util.run_deprecated_v1 def test_set_intersection_multirow_2d(self): for dtype in _DTYPES: self._test_set_intersection_multirow_2d(dtype) @@ -199,6 +203,7 @@ class SetOpsTest(test_util.TensorFlowTestCase): self.assertAllEqual(expected_counts, self._set_intersection_count(sp_a, sp_b)) + @test_util.run_deprecated_v1 def test_dense_set_intersection_multirow_2d(self): for dtype in _DTYPES: self._test_dense_set_intersection_multirow_2d(dtype) @@ -223,6 +228,7 @@ class SetOpsTest(test_util.TensorFlowTestCase): dtype=dtype) self.assertAllEqual(expected_counts, self._set_intersection_count(a, b)) + @test_util.run_deprecated_v1 def test_set_intersection_duplicates_2d(self): for dtype in _DTYPES: self._test_set_intersection_duplicates_2d(dtype) @@ -270,6 +276,7 @@ class SetOpsTest(test_util.TensorFlowTestCase): self.assertAllEqual(expected_counts, self._set_intersection_count(sp_a, sp_b)) + @test_util.run_deprecated_v1 def test_set_intersection_3d(self): for dtype in _DTYPES: self._test_set_intersection_3d(dtype=dtype) @@ -534,8 +541,9 @@ class SetOpsTest(test_util.TensorFlowTestCase): def _set_intersection_count(self, a, b): op = sets.set_size(sets.set_intersection(a, b)) with self.cached_session() as sess: - return sess.run(op) + return self.evaluate(op) + @test_util.run_deprecated_v1 def test_set_difference_multirow_2d(self): for dtype in _DTYPES: self._test_set_difference_multirow_2d(dtype) @@ -604,6 +612,7 @@ class SetOpsTest(test_util.TensorFlowTestCase): self.assertAllEqual(expected_counts, self._set_difference_count(sp_a, sp_b, False)) + @test_util.run_deprecated_v1 def test_dense_set_difference_multirow_2d(self): for dtype in _DTYPES: self._test_dense_set_difference_multirow_2d(dtype) @@ -647,6 +656,7 @@ class SetOpsTest(test_util.TensorFlowTestCase): self.assertAllEqual(expected_counts, self._set_difference_count(a, b, False)) + @test_util.run_deprecated_v1 def test_sparse_set_difference_multirow_2d(self): for dtype in _DTYPES: self._test_sparse_set_difference_multirow_2d(dtype) @@ -688,6 +698,7 @@ class SetOpsTest(test_util.TensorFlowTestCase): self.assertAllEqual(expected_counts, self._set_difference_count(sp_a, sp_b, False)) + @test_util.run_deprecated_v1 def test_set_difference_duplicates_2d(self): for dtype in _DTYPES: self._test_set_difference_duplicates_2d(dtype) @@ -755,6 +766,7 @@ class SetOpsTest(test_util.TensorFlowTestCase): self.assertAllEqual(expected_counts, self._set_difference_count(a, sp_b, False)) + @test_util.run_deprecated_v1 def test_sparse_set_difference_3d(self): for dtype in _DTYPES: self._test_sparse_set_difference_3d(dtype) @@ -972,8 +984,9 @@ class SetOpsTest(test_util.TensorFlowTestCase): def _set_difference_count(self, a, b, aminusb=True): op = sets.set_size(sets.set_difference(a, b, aminusb)) with self.cached_session() as sess: - return sess.run(op) + return self.evaluate(op) + @test_util.run_deprecated_v1 def test_set_union_multirow_2d(self): for dtype in _DTYPES: self._test_set_union_multirow_2d(dtype) @@ -1001,6 +1014,7 @@ class SetOpsTest(test_util.TensorFlowTestCase): expected_indices, expected_values, expected_shape, union, dtype=dtype) self.assertAllEqual(expected_counts, self._set_union_count(sp_a, sp_b)) + @test_util.run_deprecated_v1 def test_dense_set_union_multirow_2d(self): for dtype in _DTYPES: self._test_dense_set_union_multirow_2d(dtype) @@ -1021,6 +1035,7 @@ class SetOpsTest(test_util.TensorFlowTestCase): expected_indices, expected_values, expected_shape, union, dtype=dtype) self.assertAllEqual(expected_counts, self._set_union_count(a, b)) + @test_util.run_deprecated_v1 def test_set_union_duplicates_2d(self): for dtype in _DTYPES: self._test_set_union_duplicates_2d(dtype) @@ -1047,6 +1062,7 @@ class SetOpsTest(test_util.TensorFlowTestCase): expected_indices, expected_values, expected_shape, union, dtype=dtype) self.assertAllEqual([2], self._set_union_count(sp_a, sp_b)) + @test_util.run_deprecated_v1 def test_sparse_set_union_3d(self): for dtype in _DTYPES: self._test_sparse_set_union_3d(dtype) @@ -1221,7 +1237,7 @@ class SetOpsTest(test_util.TensorFlowTestCase): def _set_union_count(self, a, b): op = sets.set_size(sets.set_union(a, b)) with self.cached_session() as sess: - return sess.run(op) + return self.evaluate(op) def _assert_set_operation(self, expected_indices, expected_values, expected_shape, sparse_tensor_value, dtype): diff --git a/tensorflow/python/kernel_tests/shape_ops_test.py b/tensorflow/python/kernel_tests/shape_ops_test.py index ee813e5ffd9..c8e7c143ade 100644 --- a/tensorflow/python/kernel_tests/shape_ops_test.py +++ b/tensorflow/python/kernel_tests/shape_ops_test.py @@ -26,6 +26,7 @@ from tensorflow.python.framework import dtypes from tensorflow.python.framework import errors_impl from tensorflow.python.framework import importer from tensorflow.python.framework import sparse_tensor +from tensorflow.python.framework import test_util from tensorflow.python.ops import array_ops from tensorflow.python.ops import gradient_checker from tensorflow.python.ops import gradients_impl @@ -53,8 +54,8 @@ class ShapeOpsTest(test.TestCase): with self.cached_session(use_gpu=use_gpu): tf_ans = array_ops.shape(x) tf_ans_64 = array_ops.shape(x, out_type=dtypes.int64) - result = tf_ans.eval() - result_64 = tf_ans_64.eval() + result = self.evaluate(tf_ans) + result_64 = self.evaluate(tf_ans_64) self.assertAllEqual(np_ans, result) self.assertAllEqual(np_ans, result_64) self.assertShapeEqual(np_ans, tf_ans) @@ -64,7 +65,7 @@ class ShapeOpsTest(test.TestCase): x_tf, unused_nnz = _sparsify(x_np) with self.cached_session(use_gpu=use_gpu): tf_ans = array_ops.shape(x_tf) - result = tf_ans.eval() + result = self.evaluate(tf_ans) self.assertAllEqual(np_ans, result) self.assertShapeEqual(np_ans, tf_ans) @@ -73,8 +74,8 @@ class ShapeOpsTest(test.TestCase): with self.cached_session(use_gpu=use_gpu) as sess: tf_ans = array_ops.shape_n([x, x, x]) tf_ans_64 = array_ops.shape_n([x, x, x], out_type=dtypes.int64) - result = sess.run(tf_ans) - result_64 = sess.run(tf_ans_64) + result = self.evaluate(tf_ans) + result_64 = self.evaluate(tf_ans_64) for i in range(3): self.assertAllEqual(np_ans, result[i]) self.assertAllEqual(np_ans, result_64[i]) @@ -84,7 +85,7 @@ class ShapeOpsTest(test.TestCase): np_ans = np.asarray(np.ndim(x)) with self.cached_session(use_gpu=use_gpu): tf_ans = array_ops.rank(x) - result = tf_ans.eval() + result = self.evaluate(tf_ans) self.assertAllEqual(np_ans, result) self.assertShapeEqual(np_ans, tf_ans) @@ -93,7 +94,7 @@ class ShapeOpsTest(test.TestCase): x_tf, unused_nnz = _sparsify(x_np) with self.cached_session(use_gpu=use_gpu): tf_ans = array_ops.rank(x_tf) - result = tf_ans.eval() + result = self.evaluate(tf_ans) self.assertAllEqual(np_ans, result) self.assertShapeEqual(np_ans, tf_ans) @@ -101,9 +102,9 @@ class ShapeOpsTest(test.TestCase): np_ans = np.asarray(np.size(x)) with self.cached_session(use_gpu=use_gpu): tf_ans = array_ops.size(x) - result = tf_ans.eval() + result = self.evaluate(tf_ans) tf_ans_64 = array_ops.size(x, out_type=dtypes.int64) - result_64 = tf_ans_64.eval() + result_64 = self.evaluate(tf_ans_64) self.assertAllEqual(np_ans, result) self.assertAllEqual(np_ans, result_64) self.assertShapeEqual(np_ans, tf_ans) @@ -113,7 +114,7 @@ class ShapeOpsTest(test.TestCase): x_tf, unused_nnz = _sparsify(x_np) with self.cached_session(use_gpu=use_gpu): tf_ans = array_ops.size(x_tf) - result = tf_ans.eval() + result = self.evaluate(tf_ans) self.assertAllEqual(np_ans, result) self.assertShapeEqual(np_ans, tf_ans) @@ -162,7 +163,7 @@ class ShapeOpsTest(test.TestCase): inp = array_ops.zeros([2**31]) num_elements = array_ops.size_internal( inp, optimize=False, out_type=dtypes.int64) - self.assertEqual(2**31, num_elements.eval()) + self.assertEqual(2**31, self.evaluate(num_elements)) # Too large for tf.int32 output. with self.assertRaises(errors_impl.InvalidArgumentError): @@ -170,13 +171,13 @@ class ShapeOpsTest(test.TestCase): inp = array_ops.zeros([2**31]) num_elements = array_ops.size_internal( inp, optimize=False, out_type=dtypes.int32) - self.assertEqual(2**31, num_elements.eval()) + self.assertEqual(2**31, self.evaluate(num_elements)) def _compareExpandDims(self, x, dim, use_gpu): np_ans = np.expand_dims(x, axis=dim) with self.cached_session(use_gpu=use_gpu): tensor = array_ops.expand_dims(x, dim) - tf_ans = tensor.eval() + tf_ans = self.evaluate(tensor) self.assertShapeEqual(np_ans, tensor) self.assertAllEqual(np_ans, tf_ans) @@ -227,6 +228,7 @@ class ShapeOpsTest(test.TestCase): self._compareExpandDimsAll(choice([2, 3, 5]), -3) self._compareExpandDimsAll(choice([2, 3, 5]), -4) + @test_util.run_deprecated_v1 def testExpandDimsErrors(self): with self.cached_session(): self.assertRaises(ValueError, array_ops.expand_dims, @@ -238,6 +240,7 @@ class ShapeOpsTest(test.TestCase): self.assertRaises(ValueError, array_ops.expand_dims, [False, True, True], 4) + @test_util.run_deprecated_v1 def testExpandDimsGradient(self): with self.cached_session(): inp = constant_op.constant( @@ -248,6 +251,7 @@ class ShapeOpsTest(test.TestCase): [4, 1, 2]) self.assertLess(err, 1e-3) + @test_util.run_deprecated_v1 def testExpandDimsScalar(self): with self.cached_session(): inp = constant_op.constant(7) @@ -264,7 +268,7 @@ class ShapeOpsTest(test.TestCase): np_ans = np.expand_dims(x, axis=0) with self.cached_session(use_gpu=True): tensor = array_ops.expand_dims(x, constant_op.constant(0, dtype)) - tf_ans = tensor.eval() + tf_ans = self.evaluate(tensor) self.assertShapeEqual(np_ans, tensor) self.assertAllEqual(np_ans, tf_ans) @@ -273,11 +277,11 @@ class ShapeOpsTest(test.TestCase): if squeeze_dims: np_ans = np.squeeze(x, axis=tuple(squeeze_dims)) tensor = array_ops.squeeze(x, squeeze_dims) - tf_ans = tensor.eval() + tf_ans = self.evaluate(tensor) else: np_ans = np.squeeze(x) tensor = array_ops.squeeze(x) - tf_ans = tensor.eval() + tf_ans = self.evaluate(tensor) self.assertShapeEqual(np_ans, tensor) self.assertAllEqual(np_ans, tf_ans) @@ -340,7 +344,7 @@ class ShapeOpsTest(test.TestCase): with self.cached_session(use_gpu=use_gpu): tensor = array_ops.squeeze(np.zeros([1, 1, 1]), []) self.assertEqual(np.shape(1), tensor.get_shape()) - tf_ans = tensor.eval() + tf_ans = self.evaluate(tensor) self.assertEqual(np.shape(1), tf_ans.shape) def testSqueezeAllOnesBool(self): @@ -350,9 +354,10 @@ class ShapeOpsTest(test.TestCase): with self.cached_session(use_gpu=use_gpu): tensor = array_ops.squeeze([[[False]]], []) self.assertEqual(np.shape(1), tensor.get_shape()) - tf_ans = tensor.eval() + tf_ans = self.evaluate(tensor) self.assertEqual(np.shape(1), tf_ans.shape) + @test_util.run_deprecated_v1 def testSqueezeOnlyOnes(self): for use_gpu in [False, True]: with self.cached_session(use_gpu=use_gpu): @@ -362,6 +367,7 @@ class ShapeOpsTest(test.TestCase): self._compareSqueezeAll(input_1x1x3, [1]) self.assertRaises(ValueError, array_ops.squeeze, input_1x1x3, [2]) + @test_util.run_deprecated_v1 def testSqueezeErrors(self): for use_gpu in [False, True]: with self.cached_session(use_gpu=use_gpu): @@ -374,6 +380,7 @@ class ShapeOpsTest(test.TestCase): self.assertRaises(ValueError, array_ops.squeeze, np.zeros([1, 2, 1]), [2, 3]) + @test_util.run_deprecated_v1 def testSqueezeGradient(self): with self.cached_session(): inp = np.random.rand(4, 2).astype("f") @@ -384,6 +391,7 @@ class ShapeOpsTest(test.TestCase): [4, 2]) self.assertLess(err, 1e-3) + @test_util.run_deprecated_v1 def testSqueezeGradientWithSqueezeDims(self): with self.cached_session(): inp = np.random.rand(4, 2).astype("f") @@ -394,6 +402,7 @@ class ShapeOpsTest(test.TestCase): [4, 2, 1]) self.assertLess(err, 1e-3) + @test_util.run_deprecated_v1 def testSqueezeWithUnknownShape(self): with self.cached_session(): a = array_ops.placeholder(dtypes.float32, shape=[2, None]) @@ -415,7 +424,7 @@ class TileTest(test.TestCase): with self.cached_session(use_gpu=use_gpu): a = constant_op.constant(7, shape=[], dtype=dtypes.float32) tiled = array_ops.tile(a, []) - result = tiled.eval() + result = self.evaluate(tiled) self.assertEqual(result.shape, ()) self.assertEqual([], tiled.get_shape()) self.assertEqual(7, result) @@ -427,7 +436,7 @@ class TileTest(test.TestCase): inp = np.random.rand(4, 1).astype(np.float32) a = constant_op.constant(inp) tiled = array_ops.tile(a, constant_op.constant([1, 4], dtype=dtype)) - result = tiled.eval() + result = self.evaluate(tiled) self.assertEqual(result.shape, (4, 4)) self.assertEqual([4, 4], tiled.get_shape()) self.assertTrue((result == np.tile(inp, (1, 4))).all()) @@ -437,7 +446,7 @@ class TileTest(test.TestCase): inp = np.random.rand(4, 1).astype(np.float32) a = constant_op.constant(inp) tiled = array_ops.tile(a, [1, 1]) - result = tiled.eval() + result = self.evaluate(tiled) self.assertEqual(result.shape, (4, 1)) self.assertEqual([4, 1], tiled.get_shape()) self.assertTrue((result == np.tile(inp, (1, 1))).all()) @@ -447,10 +456,11 @@ class TileTest(test.TestCase): inp = np.random.rand(2, 3).astype(np.float32) a = constant_op.constant(inp) tiled = array_ops.tile(a, [5, 0]) - result = tiled.eval() + result = self.evaluate(tiled) self.assertEqual(result.shape, (10, 0)) self.assertEqual([10, 0], tiled.get_shape()) + @test_util.run_deprecated_v1 def testUnknownInputShape(self): """Importing can call _TileShape without shape of known.""" with self.cached_session(): @@ -497,11 +507,12 @@ class TileTest(test.TestCase): shape=[4, 1], dtype=dtype_tf) tiled = array_ops.tile(a, [1, 4]) - result = tiled.eval() + result = self.evaluate(tiled) self.assertEqual(result.shape, (4, 4)) self.assertEqual([4, 4], tiled.get_shape()) self.assertAllEqual(result, np.tile(inp, (1, 4))) + @test_util.run_deprecated_v1 def testInvalidDim(self): with self.cached_session(): inp = np.random.rand(4, 1).astype("f") @@ -527,7 +538,7 @@ class TileTest(test.TestCase): dtype=dtypes.float32) multiples = np.random.randint(1, 4, size=rank).astype(np.int32) tiled = array_ops.tile(a, multiples) - result = tiled.eval() + result = self.evaluate(tiled) self.assertTrue((np.array(multiples) * np.array(inp.shape) == np.array( result.shape)).all()) self.assertAllEqual(result, np.tile(inp, tuple(multiples))) @@ -545,6 +556,7 @@ class TileTest(test.TestCase): for _ in range(5): self._RunAndVerifyResult(10, use_gpu=True) + @test_util.run_deprecated_v1 def testGradientSimpleReduction(self): with self.cached_session(): inp = np.random.rand(4, 1).astype("f") @@ -557,9 +569,10 @@ class TileTest(test.TestCase): [float(x) for x in grad_inp.flatten()], shape=grad_shape) grad = gradients_impl.gradients([tiled], [a], [grad_tensor])[0] self.assertShapeEqual(inp, grad) - result = grad.eval() + result = self.evaluate(grad) self.assertAllClose(np.sum(grad_inp, axis=1).reshape(4, 1), result, 1e-3) + @test_util.run_deprecated_v1 def testGradientStridedReduction(self): with self.cached_session(): inp = np.random.rand(4, 2).astype("f") @@ -572,13 +585,14 @@ class TileTest(test.TestCase): [float(x) for x in grad_inp.flatten()], shape=grad_shape) grad = gradients_impl.gradients([tiled], [a], [grad_tensor])[0] self.assertShapeEqual(inp, grad) - result = grad.eval() + result = self.evaluate(grad) expected_shape = [4, 2] expected = np.zeros(expected_shape) expected[:, 0] = grad_inp[:, 0] + grad_inp[:, 2] expected[:, 1] = grad_inp[:, 1] + grad_inp[:, 3] self.assertTrue((np.abs(expected - result) < 1e-3).all()) + @test_util.run_deprecated_v1 def testGradientSimpleReductionOnGPU(self): with self.session(use_gpu=True): inp = np.random.rand(4, 1).astype("f") @@ -590,9 +604,10 @@ class TileTest(test.TestCase): grad_tensor = constant_op.constant( [float(x) for x in grad_inp.flatten()], shape=grad_shape) grad = gradients_impl.gradients([tiled], [a], [grad_tensor])[0] - result = grad.eval() + result = self.evaluate(grad) self.assertAllClose(np.sum(grad_inp, axis=1).reshape(4, 1), result, 1e-3) + @test_util.run_deprecated_v1 def testGradientStridedReductionOnGPU(self): with self.session(use_gpu=True): inp = np.random.rand(4, 2).astype("f") @@ -604,7 +619,7 @@ class TileTest(test.TestCase): grad_tensor = constant_op.constant( [float(x) for x in grad_inp.flatten()], shape=grad_shape) grad = gradients_impl.gradients([tiled], [a], [grad_tensor])[0] - result = grad.eval() + result = self.evaluate(grad) expected_shape = [4, 2] expected = np.zeros(expected_shape) expected[:, 0] = grad_inp[:, 0] + grad_inp[:, 2] @@ -624,15 +639,18 @@ class TileTest(test.TestCase): print("tile(float) error = ", err) self.assertLess(err, 1e-3) + @test_util.run_deprecated_v1 def testGradientRandomScalar(self): self._RunAndVerifyGradientResult([], []) + @test_util.run_deprecated_v1 def testGradientRandom(self): self._RunAndVerifyGradientResult([2, 2, 1, 1, 3], [1, 1, 1, 1, 1]) self._RunAndVerifyGradientResult([2, 2, 1, 1, 3], [1, 2, 1, 3, 1]) self._RunAndVerifyGradientResult([2, 3, 1, 1, 3], [3, 1, 1, 2, 2]) self._RunAndVerifyGradientResult([2, 1, 3, 3, 2], [1, 3, 3, 1, 2]) + @test_util.run_deprecated_v1 def testGradientStridedReductionGC(self): with self.cached_session(): inp = np.random.rand(4, 2).astype("f") @@ -642,6 +660,7 @@ class TileTest(test.TestCase): err = gradient_checker.compute_gradient_error(a, [4, 2], tiled, [4, 4]) self.assertLess(err, 1e-3) + @test_util.run_deprecated_v1 def testGradientWithSparseGradWithRank1(self): inputs = constant_op.constant([1.0, 2.0, 3.0, 4.0], dtype=dtypes.float32) @@ -653,6 +672,7 @@ class TileTest(test.TestCase): outputs, outputs.get_shape().as_list()) self.assertLess(error, 1e-4) + @test_util.run_deprecated_v1 def testGradientWithSparseGradWithRank3(self): inputs = constant_op.constant([1.0, 2.0, 3.0, 4.0], dtype=dtypes.float32) @@ -665,6 +685,7 @@ class TileTest(test.TestCase): outputs, outputs.get_shape().as_list()) self.assertLess(error, 1e-4) + @test_util.run_deprecated_v1 def testShapeFunctionEdgeCases(self): # Unknown multiples shape. inp = constant_op.constant(0.0, shape=[4, 4, 4, 4]) diff --git a/tensorflow/python/kernel_tests/signal/BUILD b/tensorflow/python/kernel_tests/signal/BUILD index 56d4d46d462..8f4e31abe3c 100644 --- a/tensorflow/python/kernel_tests/signal/BUILD +++ b/tensorflow/python/kernel_tests/signal/BUILD @@ -18,6 +18,35 @@ py_library( ], ) +cuda_py_tests( + name = "dct_ops_test", + srcs = ["dct_ops_test.py"], + additional_deps = [ + "@absl_py//absl/testing:parameterized", + "//third_party/py/numpy", + "//tensorflow/python:client_testlib", + "//tensorflow/python:framework_for_generated_wrappers", + "//tensorflow/python:spectral_ops_test_util", + "//tensorflow/python/ops/signal", + ], +) + +cuda_py_tests( + name = "fft_ops_test", + size = "medium", + srcs = ["fft_ops_test.py"], + additional_deps = [ + "//third_party/py/numpy", + "//tensorflow/python:client_testlib", + "//tensorflow/python:framework_for_generated_wrappers", + "//tensorflow/python:math_ops", + "//tensorflow/python:spectral_ops_test_util", + "//tensorflow/python/ops/signal", + ], + shard_count = 4, + tags = ["optonly"], +) + cuda_py_tests( name = "mel_ops_test", srcs = ["mel_ops_test.py"], @@ -91,9 +120,9 @@ cuda_py_tests( "//tensorflow/python:framework", "//tensorflow/python:framework_for_generated_wrappers", "//tensorflow/python:framework_test_lib", - "//tensorflow/python/ops/signal", "//tensorflow/python:platform_test", "//tensorflow/python:spectral_ops_test_util", + "//tensorflow/python/ops/signal", ], tags = ["nomac"], ) diff --git a/tensorflow/python/kernel_tests/dct_ops_test.py b/tensorflow/python/kernel_tests/signal/dct_ops_test.py similarity index 67% rename from tensorflow/python/kernel_tests/dct_ops_test.py rename to tensorflow/python/kernel_tests/signal/dct_ops_test.py index c9d0167608e..a3ac15bab8a 100644 --- a/tensorflow/python/kernel_tests/dct_ops_test.py +++ b/tensorflow/python/kernel_tests/signal/dct_ops_test.py @@ -20,10 +20,12 @@ from __future__ import print_function import importlib +from absl.testing import parameterized import numpy as np -from tensorflow.python.ops import spectral_ops +from tensorflow.python.framework import test_util from tensorflow.python.ops import spectral_ops_test_util +from tensorflow.python.ops.signal import dct_ops from tensorflow.python.platform import test from tensorflow.python.platform import tf_logging @@ -40,6 +42,20 @@ def try_import(name): # pylint: disable=invalid-name fftpack = try_import("scipy.fftpack") +def _np_dct1(signals, norm=None): + """Computes the DCT-I manually with NumPy.""" + # X_k = (x_0 + (-1)**k * x_{N-1} + + # 2 * sum_{n=0}^{N-2} x_n * cos(\frac{pi}{N-1} * n * k) k=0,...,N-1 + del norm + dct_size = signals.shape[-1] + dct = np.zeros_like(signals) + for k in range(dct_size): + phi = np.cos(np.pi * np.arange(1, dct_size - 1) * k / (dct_size - 1)) + dct[..., k] = 2 * np.sum(signals[..., 1:-1] * phi, axis=-1) + ( + signals[..., 0] + (-1) ** k * signals[..., -1]) + return dct + + def _np_dct2(signals, norm=None): """Computes the DCT-II manually with NumPy.""" # X_k = sum_{n=0}^{N-1} x_n * cos(\frac{pi}{N} * (n + 0.5) * k) k=0,...,N-1 @@ -81,19 +97,19 @@ def _np_dct3(signals, norm=None): return dct -NP_DCT = {2: _np_dct2, 3: _np_dct3} -NP_IDCT = {2: _np_dct3, 3: _np_dct2} +NP_DCT = {1: _np_dct1, 2: _np_dct2, 3: _np_dct3} +NP_IDCT = {1: _np_dct1, 2: _np_dct3, 3: _np_dct2} -class DCTOpsTest(test.TestCase): +class DCTOpsTest(parameterized.TestCase, test.TestCase): def _compare(self, signals, norm, dct_type, atol=5e-4, rtol=5e-4): """Compares (I)DCT to SciPy (if available) and a NumPy implementation.""" np_dct = NP_DCT[dct_type](signals, norm) - tf_dct = spectral_ops.dct(signals, type=dct_type, norm=norm).eval() + tf_dct = dct_ops.dct(signals, type=dct_type, norm=norm).eval() self.assertAllClose(np_dct, tf_dct, atol=atol, rtol=rtol) np_idct = NP_IDCT[dct_type](signals, norm) - tf_idct = spectral_ops.idct(signals, type=dct_type, norm=norm).eval() + tf_idct = dct_ops.idct(signals, type=dct_type, norm=norm).eval() self.assertAllClose(np_idct, tf_idct, atol=atol, rtol=rtol) if fftpack: scipy_dct = fftpack.dct(signals, type=dct_type, norm=norm) @@ -101,38 +117,52 @@ class DCTOpsTest(test.TestCase): scipy_idct = fftpack.idct(signals, type=dct_type, norm=norm) self.assertAllClose(scipy_idct, tf_idct, atol=atol, rtol=rtol) # Verify inverse(forward(s)) == s, up to a normalization factor. - tf_idct_dct = spectral_ops.idct( + tf_idct_dct = dct_ops.idct( tf_dct, type=dct_type, norm=norm).eval() - tf_dct_idct = spectral_ops.dct( + tf_dct_idct = dct_ops.dct( tf_idct, type=dct_type, norm=norm).eval() if norm is None: - tf_idct_dct *= 0.5 / signals.shape[-1] - tf_dct_idct *= 0.5 / signals.shape[-1] + if dct_type == 1: + tf_idct_dct *= 0.5 / (signals.shape[-1] - 1) + tf_dct_idct *= 0.5 / (signals.shape[-1] - 1) + else: + tf_idct_dct *= 0.5 / signals.shape[-1] + tf_dct_idct *= 0.5 / signals.shape[-1] self.assertAllClose(signals, tf_idct_dct, atol=atol, rtol=rtol) self.assertAllClose(signals, tf_dct_idct, atol=atol, rtol=rtol) - def test_random(self): + @parameterized.parameters([ + [[2]], [[3]], [[10]], [[2, 20]], [[2, 3, 25]]]) + @test_util.run_deprecated_v1 + def test_random(self, shape): """Test randomly generated batches of data.""" with spectral_ops_test_util.fft_kernel_label_map(): with self.session(use_gpu=True): - for shape in ([1], [2], [3], [10], [2, 20], [2, 3, 25]): - signals = np.random.rand(*shape).astype(np.float32) - for norm in (None, "ortho"): - self._compare(signals, norm, 2) - self._compare(signals, norm, 3) + signals = np.random.rand(*shape).astype(np.float32) + # Normalization not implemented for orthonormal. + self._compare(signals, norm=None, dct_type=1) + for norm in (None, "ortho"): + self._compare(signals, norm, 2) + self._compare(signals, norm, 3) def test_error(self): signals = np.random.rand(10) # Unsupported type. with self.assertRaises(ValueError): - spectral_ops.dct(signals, type=1) + dct_ops.dct(signals, type=5) + # DCT-I normalization not implemented. + with self.assertRaises(ValueError): + dct_ops.dct(signals, type=1, norm="ortho") + # DCT-I requires at least two inputs. + with self.assertRaises(ValueError): + dct_ops.dct(np.random.rand(1), type=1) # Unknown normalization. with self.assertRaises(ValueError): - spectral_ops.dct(signals, norm="bad") + dct_ops.dct(signals, norm="bad") with self.assertRaises(NotImplementedError): - spectral_ops.dct(signals, n=10) + dct_ops.dct(signals, n=10) with self.assertRaises(NotImplementedError): - spectral_ops.dct(signals, axis=0) + dct_ops.dct(signals, axis=0) if __name__ == "__main__": diff --git a/tensorflow/python/kernel_tests/fft_ops_test.py b/tensorflow/python/kernel_tests/signal/fft_ops_test.py similarity index 96% rename from tensorflow/python/kernel_tests/fft_ops_test.py rename to tensorflow/python/kernel_tests/signal/fft_ops_test.py index 8592550f99a..5b1053428c0 100644 --- a/tensorflow/python/kernel_tests/fft_ops_test.py +++ b/tensorflow/python/kernel_tests/signal/fft_ops_test.py @@ -25,12 +25,13 @@ from tensorflow.core.protobuf import config_pb2 from tensorflow.python.framework import dtypes from tensorflow.python.framework import errors from tensorflow.python.framework import ops +from tensorflow.python.framework import test_util from tensorflow.python.ops import array_ops from tensorflow.python.ops import gen_spectral_ops from tensorflow.python.ops import gradient_checker from tensorflow.python.ops import math_ops -from tensorflow.python.ops import spectral_ops from tensorflow.python.ops import spectral_ops_test_util +from tensorflow.python.ops.signal import fft_ops from tensorflow.python.platform import test VALID_FFT_RANKS = (1, 2, 3) @@ -139,24 +140,25 @@ class FFTOpsTest(BaseFFTOpsTest): def _tfFFTForRank(self, rank): if rank == 1: - return spectral_ops.fft + return fft_ops.fft elif rank == 2: - return spectral_ops.fft2d + return fft_ops.fft2d elif rank == 3: - return spectral_ops.fft3d + return fft_ops.fft3d else: raise ValueError("invalid rank") def _tfIFFTForRank(self, rank): if rank == 1: - return spectral_ops.ifft + return fft_ops.ifft elif rank == 2: - return spectral_ops.ifft2d + return fft_ops.ifft2d elif rank == 3: - return spectral_ops.ifft3d + return fft_ops.ifft3d else: raise ValueError("invalid rank") + @test_util.run_deprecated_v1 def testEmpty(self): with spectral_ops_test_util.fft_kernel_label_map(): for np_type in (np.complex64, np.complex128): @@ -166,6 +168,7 @@ class FFTOpsTest(BaseFFTOpsTest): self.assertEqual(x.shape, self._tfFFT(x, rank).shape) self.assertEqual(x.shape, self._tfIFFT(x, rank).shape) + @test_util.run_deprecated_v1 def testBasic(self): with spectral_ops_test_util.fft_kernel_label_map(): for np_type, tol in ((np.complex64, 1e-4), (np.complex128, 1e-8)): @@ -194,6 +197,7 @@ class FFTOpsTest(BaseFFTOpsTest): # np.mod(np.arange(np.power(128, dims)), 64).reshape( # (128,) * dims).astype(np.complex64), rank) + @test_util.run_deprecated_v1 def testBasicPlaceholder(self): with spectral_ops_test_util.fft_kernel_label_map(): for np_type, tol in ((np.complex64, 1e-4), (np.complex128, 1e-8)): @@ -204,6 +208,7 @@ class FFTOpsTest(BaseFFTOpsTest): (4,) * dims).astype(np_type), rank, use_placeholder=True, rtol=tol, atol=tol) + @test_util.run_deprecated_v1 def testRandom(self): with spectral_ops_test_util.fft_kernel_label_map(): for np_type, tol in ((np.complex64, 1e-4), (np.complex128, 5e-6)): @@ -218,6 +223,7 @@ class FFTOpsTest(BaseFFTOpsTest): self._compare(gen((4,) * dims).astype(np_type), rank, rtol=tol, atol=tol) + @test_util.run_deprecated_v1 def testRandom1D(self): with spectral_ops_test_util.fft_kernel_label_map(): for np_type in (np.complex64, np.complex128): @@ -240,6 +246,7 @@ class FFTOpsTest(BaseFFTOpsTest): for dim in (127, 255, 511, 1023): self._compare(gen((dim,)).astype(np_type), 1, rtol=tol, atol=tol) + @test_util.run_deprecated_v1 def testError(self): for rank in VALID_FFT_RANKS: for dims in xrange(0, rank): @@ -251,6 +258,7 @@ class FFTOpsTest(BaseFFTOpsTest): ValueError, "Shape must be .*rank {}.*".format(rank)): self._tfIFFT(x, rank) + @test_util.run_deprecated_v1 def testGrad_Simple(self): with spectral_ops_test_util.fft_kernel_label_map(): for np_type, tol in ((np.float32, 1e-4), (np.float64, 1e-10)): @@ -263,6 +271,7 @@ class FFTOpsTest(BaseFFTOpsTest): self._checkGradComplex(self._tfIFFTForRank(rank), re, im, rtol=tol, atol=tol) + @test_util.run_deprecated_v1 def testGrad_Random(self): with spectral_ops_test_util.fft_kernel_label_map(): for np_type, tol in ((np.float32, 1e-2), (np.float64, 1e-10)): @@ -312,24 +321,25 @@ class RFFTOpsTest(BaseFFTOpsTest): def _tfFFTForRank(self, rank): if rank == 1: - return spectral_ops.rfft + return fft_ops.rfft elif rank == 2: - return spectral_ops.rfft2d + return fft_ops.rfft2d elif rank == 3: - return spectral_ops.rfft3d + return fft_ops.rfft3d else: raise ValueError("invalid rank") def _tfIFFTForRank(self, rank): if rank == 1: - return spectral_ops.irfft + return fft_ops.irfft elif rank == 2: - return spectral_ops.irfft2d + return fft_ops.irfft2d elif rank == 3: - return spectral_ops.irfft3d + return fft_ops.irfft3d else: raise ValueError("invalid rank") + @test_util.run_deprecated_v1 def testEmpty(self): with spectral_ops_test_util.fft_kernel_label_map(): for rank in VALID_FFT_RANKS: @@ -339,6 +349,7 @@ class RFFTOpsTest(BaseFFTOpsTest): x = np.zeros((0,) * dims).astype(np.complex64) self.assertEqual(x.shape, self._tfIFFT(x, rank).shape) + @test_util.run_deprecated_v1 def testBasic(self): with spectral_ops_test_util.fft_kernel_label_map(): for rank in VALID_FFT_RANKS: @@ -366,6 +377,7 @@ class RFFTOpsTest(BaseFFTOpsTest): 10).reshape((size,) * (dims - 1) + (inner_dim,)) self._compareBackward(c2r.astype(np.complex64), rank, (size,) * rank) + @test_util.run_deprecated_v1 def testBasicPlaceholder(self): with spectral_ops_test_util.fft_kernel_label_map(): for rank in VALID_FFT_RANKS: @@ -427,6 +439,7 @@ class RFFTOpsTest(BaseFFTOpsTest): fft_length, use_placeholder=True) + @test_util.run_deprecated_v1 def testRandom(self): with spectral_ops_test_util.fft_kernel_label_map(): def gen_real(shape): @@ -451,6 +464,7 @@ class RFFTOpsTest(BaseFFTOpsTest): self._compareBackward( gen_complex(complex_dims), rank, (size,) * rank) + @test_util.run_deprecated_v1 def testError(self): with spectral_ops_test_util.fft_kernel_label_map(): for rank in VALID_FFT_RANKS: @@ -507,6 +521,7 @@ class RFFTOpsTest(BaseFFTOpsTest): with self.cached_session(): irfft_fn(x, fft_length).eval() + @test_util.run_deprecated_v1 def testGrad_Simple(self): with spectral_ops_test_util.fft_kernel_label_map(): for rank in VALID_FFT_RANKS: @@ -521,6 +536,7 @@ class RFFTOpsTest(BaseFFTOpsTest): self._checkGradComplex( self._tfIFFTForRank(rank), re, im, result_is_complex=False) + @test_util.run_deprecated_v1 def testGrad_Random(self): with spectral_ops_test_util.fft_kernel_label_map(): for rank in VALID_FFT_RANKS: diff --git a/tensorflow/python/kernel_tests/signal/mel_ops_test.py b/tensorflow/python/kernel_tests/signal/mel_ops_test.py index 1ed4429b42a..3134503daec 100644 --- a/tensorflow/python/kernel_tests/signal/mel_ops_test.py +++ b/tensorflow/python/kernel_tests/signal/mel_ops_test.py @@ -22,6 +22,7 @@ import numpy as np from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops +from tensorflow.python.framework import test_util as tf_test_util from tensorflow.python.kernel_tests.signal import test_util from tensorflow.python.ops import array_ops from tensorflow.python.ops.signal import mel_ops @@ -141,14 +142,16 @@ class LinearToMelTest(test.TestCase): for config in configs: mel_matrix_np = spectrogram_to_mel_matrix(*config) mel_matrix = mel_ops.linear_to_mel_weight_matrix(*config) - self.assertAllClose(mel_matrix_np, mel_matrix.eval(), atol=3e-6) + self.assertAllClose(mel_matrix_np, self.evaluate(mel_matrix), atol=3e-6) + @tf_test_util.run_deprecated_v1 def test_dtypes(self): # LinSpace is not supported for tf.float16. for dtype in (dtypes.bfloat16, dtypes.float32, dtypes.float64): self.assertEqual(dtype, mel_ops.linear_to_mel_weight_matrix(dtype=dtype).dtype) + @tf_test_util.run_deprecated_v1 def test_error(self): with self.assertRaises(ValueError): mel_ops.linear_to_mel_weight_matrix(num_mel_bins=0) @@ -177,6 +180,7 @@ class LinearToMelTest(test.TestCase): rewritten_graph = test_util.grappler_optimize(g, [mel_matrix]) self.assertEqual(1, len(rewritten_graph.node)) + @tf_test_util.run_deprecated_v1 def test_num_spectrogram_bins_dynamic(self): with self.session(use_gpu=True): num_spectrogram_bins = array_ops.placeholder(shape=(), diff --git a/tensorflow/python/kernel_tests/signal/mfcc_ops_test.py b/tensorflow/python/kernel_tests/signal/mfcc_ops_test.py index 79d23d77d1e..935922657cd 100644 --- a/tensorflow/python/kernel_tests/signal/mfcc_ops_test.py +++ b/tensorflow/python/kernel_tests/signal/mfcc_ops_test.py @@ -20,6 +20,7 @@ from __future__ import print_function from tensorflow.python.framework import dtypes from tensorflow.python.framework import tensor_shape +from tensorflow.python.framework import test_util from tensorflow.python.ops import array_ops from tensorflow.python.ops import random_ops from tensorflow.python.ops import spectral_ops_test_util @@ -32,6 +33,7 @@ from tensorflow.python.platform import test # HTK conventions. class MFCCTest(test.TestCase): + @test_util.run_deprecated_v1 def test_error(self): # num_mel_bins must be positive. with self.assertRaises(ValueError): @@ -43,6 +45,7 @@ class MFCCTest(test.TestCase): signal = array_ops.zeros((2, 3, 5), dtype=dtypes.float64) mfcc_ops.mfccs_from_log_mel_spectrograms(signal) + @test_util.run_deprecated_v1 def test_basic(self): """A basic test that the op runs on random input.""" with spectral_ops_test_util.fft_kernel_label_map(): @@ -50,6 +53,7 @@ class MFCCTest(test.TestCase): signal = random_ops.random_normal((2, 3, 5)) mfcc_ops.mfccs_from_log_mel_spectrograms(signal).eval() + @test_util.run_deprecated_v1 def test_unknown_shape(self): """A test that the op runs when shape and rank are unknown.""" with spectral_ops_test_util.fft_kernel_label_map(): diff --git a/tensorflow/python/kernel_tests/signal/reconstruction_ops_test.py b/tensorflow/python/kernel_tests/signal/reconstruction_ops_test.py index c4e5b6f6740..4cb6cedee99 100644 --- a/tensorflow/python/kernel_tests/signal/reconstruction_ops_test.py +++ b/tensorflow/python/kernel_tests/signal/reconstruction_ops_test.py @@ -20,8 +20,10 @@ from __future__ import print_function import numpy as np +from tensorflow.python.eager import context from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes +from tensorflow.python.framework import test_util from tensorflow.python.ops import array_ops from tensorflow.python.ops import gradients_impl from tensorflow.python.ops import math_ops @@ -52,16 +54,75 @@ class ReconstructionOpsTest(test.TestCase): "100000000000000"] def test_all_ones(self): - signal = constant_op.constant(np.ones((3, 5)), dtype=dtypes.int64) + signal = array_ops.ones([3, 5]) reconstruction = reconstruction_ops.overlap_and_add(signal, 2) - with self.session(use_gpu=True) as sess: - output = sess.run(reconstruction) + self.assertEqual(reconstruction.shape.as_list(), [9]) + + with self.session(use_gpu=True): + output = self.evaluate(reconstruction) expected_output = np.array([1, 1, 2, 2, 3, 2, 2, 1, 1]) self.assertAllClose(output, expected_output) + def test_unknown_shapes(self): + # This test uses placeholders and does not work in eager mode. + if context.executing_eagerly(): + return + + signal = array_ops.placeholder(dtype=dtypes.int32, shape=[None, None, None]) + frame_step = array_ops.placeholder(dtype=dtypes.int32, shape=[]) + reconstruction = reconstruction_ops.overlap_and_add(signal, frame_step) + + self.assertEqual(reconstruction.shape.as_list(), [None, None]) + + with self.session(use_gpu=True) as sess: + output = sess.run(reconstruction, + feed_dict={signal: np.ones([4, 3, 5]), frame_step: 2}) + + expected_output = np.array([[1, 1, 2, 2, 3, 2, 2, 1, 1]] * 4) + + self.assertAllClose(output, expected_output) + + def test_unknown_rank(self): + # This test uses placeholders and does not work in eager mode. + if context.executing_eagerly(): + return + + signal = array_ops.placeholder(dtype=dtypes.int32, shape=None) + frame_step = array_ops.placeholder(dtype=dtypes.int32, shape=[]) + reconstruction = reconstruction_ops.overlap_and_add(signal, frame_step) + + self.assertEqual(reconstruction.shape, None) + + with self.session(use_gpu=True) as sess: + output = sess.run(reconstruction, + feed_dict={signal: np.ones([4, 3, 5]), frame_step: 2}) + + expected_output = np.array([[1, 1, 2, 2, 3, 2, 2, 1, 1]] * 4) + + self.assertAllClose(output, expected_output) + + def test_fast_path(self): + # This test uses tensor names and does not work in eager mode. + if context.executing_eagerly(): + return + + signal = array_ops.ones([3, 5]) + frame_step = 5 + reconstruction = reconstruction_ops.overlap_and_add(signal, frame_step) + + self.assertEqual(reconstruction.name, "overlap_and_add/fast_path:0") + + with self.session(use_gpu=True) as sess: + output = self.evaluate(reconstruction) + + expected_output = np.ones([15]) + + self.assertAllClose(output, expected_output) + + @test_util.run_deprecated_v1 def test_simple(self): def make_input(frame_length, num_frames=3): """Generate a tensor of num_frames frames of frame_length.""" @@ -98,8 +159,8 @@ class ReconstructionOpsTest(test.TestCase): dtype=dtypes.int64) reconstruction = reconstruction_ops.overlap_and_add(signal, self.frame_hop) - with self.session(use_gpu=True) as sess: - output = sess.run(reconstruction) + with self.session(use_gpu=True): + output = self.evaluate(reconstruction) string_output = [np.base_repr(x, self.bases[0]) for x in output] self.assertEqual(string_output, self.expected_string) @@ -108,8 +169,8 @@ class ReconstructionOpsTest(test.TestCase): signal = constant_op.constant(self.powers, dtype=dtypes.int64) reconstruction = reconstruction_ops.overlap_and_add(signal, self.frame_hop) - with self.session(use_gpu=True) as sess: - output = sess.run(reconstruction) + with self.session(use_gpu=True): + output = self.evaluate(reconstruction) accumulator = True for i in range(self.batch_size): @@ -124,8 +185,8 @@ class ReconstructionOpsTest(test.TestCase): signal = constant_op.constant(input_matrix, dtype=dtypes.float32) reconstruction = reconstruction_ops.overlap_and_add(signal, self.frame_hop) - with self.session(use_gpu=True) as sess: - output = sess.run(reconstruction) + with self.session(use_gpu=True): + output = self.evaluate(reconstruction) string_output = [np.base_repr(int(x), self.bases[0]) for x in np.squeeze(output)] @@ -133,6 +194,7 @@ class ReconstructionOpsTest(test.TestCase): self.assertEqual(output.shape, (1, 9)) self.assertEqual(string_output, self.expected_string) + @test_util.run_deprecated_v1 def test_gradient(self): configurations = [ ((1, 128), 1), @@ -154,6 +216,7 @@ class ReconstructionOpsTest(test.TestCase): gradient = sess.run(gradients_impl.gradients([loss], [signal])[0]) self.assertTrue((gradient == 1.0).all()) + @test_util.run_deprecated_v1 def test_gradient_batch(self): with self.session(use_gpu=True) as sess: signal = array_ops.zeros((2, 10, 10)) @@ -176,6 +239,7 @@ class ReconstructionOpsTest(test.TestCase): np.reshape(np.arange(100).astype(np.float32), (10, 10))]) self.assertAllEqual(expected_gradient, gradient) + @test_util.run_deprecated_v1 def test_gradient_numerical(self): with self.session(use_gpu=True): shape = (2, 10, 10) diff --git a/tensorflow/python/kernel_tests/signal/shape_ops_test.py b/tensorflow/python/kernel_tests/signal/shape_ops_test.py index 398fba8b6df..32ac76e80d0 100644 --- a/tensorflow/python/kernel_tests/signal/shape_ops_test.py +++ b/tensorflow/python/kernel_tests/signal/shape_ops_test.py @@ -23,6 +23,7 @@ import numpy as np from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops +from tensorflow.python.framework import test_util as tf_test_util from tensorflow.python.kernel_tests.signal import test_util from tensorflow.python.ops import array_ops from tensorflow.python.ops import math_ops @@ -32,6 +33,7 @@ from tensorflow.python.platform import test class FrameTest(test.TestCase): + @tf_test_util.run_deprecated_v1 def test_mapping_of_indices_without_padding(self): with self.session(use_gpu=True): tensor = constant_op.constant(np.arange(9152), dtypes.int32) @@ -47,6 +49,7 @@ class FrameTest(test.TestCase): self.assertAllEqual(expected, result) + @tf_test_util.run_deprecated_v1 def test_mapping_of_indices_with_padding(self): with self.session(use_gpu=True): tensor = constant_op.constant(np.arange(10000), dtypes.int32) @@ -64,6 +67,7 @@ class FrameTest(test.TestCase): self.assertAllEqual(expected, result) + @tf_test_util.run_deprecated_v1 def test_invalid_inputs(self): # Rank 0 input signal. with self.assertRaises(ValueError): @@ -84,6 +88,7 @@ class FrameTest(test.TestCase): with self.assertRaises(ValueError): shape_ops.frame([1], 1, 1, pad_end=True, pad_value=[1]) + @tf_test_util.run_deprecated_v1 def test_length_zero(self): signal = constant_op.constant([], dtype=dtypes.float32) frame_length = 2 @@ -98,6 +103,7 @@ class FrameTest(test.TestCase): pad_end=False).eval() self.assertEqual((0, 2), result.shape) + @tf_test_util.run_deprecated_v1 def test_shape_inference(self): signal = array_ops.placeholder(dtypes.int32, shape=[1, 1]) frame_length = 2 @@ -150,9 +156,10 @@ class FrameTest(test.TestCase): op = shape_ops.frame(signal, frame_length, frame_step, pad_end=pad_end, pad_value=99) with self.cached_session(use_gpu=True): - result = op.eval() + result = self.evaluate(op) self.assertEqual(op.shape.as_list(), list(result.shape)) + @tf_test_util.run_deprecated_v1 def test_basic_mono(self): signal = np.arange(6) frame_length = 3 @@ -178,6 +185,7 @@ class FrameTest(test.TestCase): pad_end=False).eval() self.assertAllEqual(expected, result) + @tf_test_util.run_deprecated_v1 def test_basic_stereo(self): signal = np.vstack([np.arange(6), np.arange(6) + 10]) @@ -207,6 +215,7 @@ class FrameTest(test.TestCase): pad_end=False).eval() self.assertAllEqual(expected, result) + @tf_test_util.run_deprecated_v1 def test_complex_shape(self): signal = np.vstack([np.arange(6), np.arange(6) + 10, @@ -248,7 +257,7 @@ class FrameTest(test.TestCase): result = shape_ops.frame(signal, frame_length=2, frame_step=2, pad_end=True, axis=1) expected = np.reshape(np.arange(16), (2, 2, 2, 2)) - self.assertAllEqual(expected, result.eval()) + self.assertAllEqual(expected, self.evaluate(result)) result = shape_ops.frame(signal, frame_length=2, frame_step=1, pad_end=True, axis=1) @@ -260,7 +269,7 @@ class FrameTest(test.TestCase): [[10, 11], [12, 13]], [[12, 13], [14, 15]], [[14, 15], [0, 0]]]] - self.assertAllEqual(expected, result.eval()) + self.assertAllEqual(expected, self.evaluate(result)) result = shape_ops.frame(signal, frame_length=3, frame_step=1, pad_end=True, axis=1) @@ -272,8 +281,9 @@ class FrameTest(test.TestCase): [[10, 11], [12, 13], [14, 15]], [[12, 13], [14, 15], [0, 0]], [[14, 15], [0, 0], [0, 0]]]] - self.assertAllEqual(expected, result.eval()) + self.assertAllEqual(expected, self.evaluate(result)) + @tf_test_util.run_deprecated_v1 def test_window_larger_than_signal(self): signal = constant_op.constant([[1, 2], [11, 12]], dtype=dtypes.float32) frame_length = 4 @@ -307,6 +317,7 @@ class FrameTest(test.TestCase): result = shape_ops.frame(signal, frame_length, frame_step) self.assertEqual(result.dtype, signal.dtype) + @tf_test_util.run_deprecated_v1 def test_dynamic_tensor(self): # Show that frame works even when the dimensions of its input are # not known at graph creation time. @@ -325,6 +336,7 @@ class FrameTest(test.TestCase): [[10, 11], [12, 13]], [[20, 21], [22, 23]]], result) + @tf_test_util.run_deprecated_v1 def test_gradient_numerical(self): with self.session(use_gpu=True): signal_shape = (2, 128) diff --git a/tensorflow/python/kernel_tests/signal/spectral_ops_test.py b/tensorflow/python/kernel_tests/signal/spectral_ops_test.py index 26cb1270639..7b9748c7f26 100644 --- a/tensorflow/python/kernel_tests/signal/spectral_ops_test.py +++ b/tensorflow/python/kernel_tests/signal/spectral_ops_test.py @@ -125,22 +125,22 @@ class SpectralOpsTest(test.TestCase): stft = spectral_ops.stft(signal, frame_length=7, frame_step=8, pad_end=True) self.assertAllEqual([64, 5], stft.shape.as_list()) - self.assertAllEqual([64, 5], stft.eval().shape) + self.assertAllEqual([64, 5], self.evaluate(stft).shape) stft = spectral_ops.stft(signal, frame_length=8, frame_step=8, pad_end=True) self.assertAllEqual([64, 5], stft.shape.as_list()) - self.assertAllEqual([64, 5], stft.eval().shape) + self.assertAllEqual([64, 5], self.evaluate(stft).shape) stft = spectral_ops.stft(signal, frame_length=8, frame_step=8, fft_length=16, pad_end=True) self.assertAllEqual([64, 9], stft.shape.as_list()) - self.assertAllEqual([64, 9], stft.eval().shape) + self.assertAllEqual([64, 9], self.evaluate(stft).shape) stft = spectral_ops.stft(signal, frame_length=16, frame_step=8, fft_length=8, pad_end=True) self.assertAllEqual([64, 5], stft.shape.as_list()) - self.assertAllEqual([64, 5], stft.eval().shape) + self.assertAllEqual([64, 5], self.evaluate(stft).shape) stft = np.zeros((32, 9)).astype(np.complex64) @@ -148,7 +148,7 @@ class SpectralOpsTest(test.TestCase): fft_length=16, frame_step=8) expected_length = (stft.shape[0] - 1) * 8 + 8 self.assertAllEqual([256], inverse_stft.shape.as_list()) - self.assertAllEqual([expected_length], inverse_stft.eval().shape) + self.assertAllEqual([expected_length], self.evaluate(inverse_stft).shape) def test_stft_and_inverse_stft(self): """Test that spectral_ops.stft/inverse_stft match a NumPy implementation.""" @@ -235,7 +235,8 @@ class SpectralOpsTest(test.TestCase): inverse_window = inverse_window_fn(frame_length, dtype=dtypes.float32) with self.cached_session(use_gpu=True) as sess: - hann_window, inverse_window = sess.run([hann_window, inverse_window]) + hann_window, inverse_window = self.evaluate( + [hann_window, inverse_window]) # Expect unit gain at each phase of the window. product_window = hann_window * inverse_window @@ -263,7 +264,8 @@ class SpectralOpsTest(test.TestCase): inverse_window = inverse_window_fn(frame_length, dtype=dtypes.float32) with self.cached_session(use_gpu=True) as sess: - hann_window, inverse_window = sess.run([hann_window, inverse_window]) + hann_window, inverse_window = self.evaluate( + [hann_window, inverse_window]) self.assertAllClose(hann_window, inverse_window * 1.5) @@ -293,7 +295,7 @@ class SpectralOpsTest(test.TestCase): # the sum of the magnitude STFT. sinusoid = math_ops.sin( 2 * np.pi * math_ops.linspace(0.0, 1.0, signal_length)) - sinusoid_gradient = sess.run(self._compute_stft_gradient(sinusoid)) + sinusoid_gradient = self.evaluate(self._compute_stft_gradient(sinusoid)) self.assertFalse((sinusoid_gradient == 0.0).all()) def test_gradients_numerical(self): diff --git a/tensorflow/python/kernel_tests/signal/test_util.py b/tensorflow/python/kernel_tests/signal/test_util.py index f2c4d0dc8f8..0a8a621c3ee 100644 --- a/tensorflow/python/kernel_tests/signal/test_util.py +++ b/tensorflow/python/kernel_tests/signal/test_util.py @@ -18,12 +18,12 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function -from tensorflow.core.protobuf import rewriter_config_pb2 +from tensorflow.core.protobuf import config_pb2 from tensorflow.python.grappler import tf_optimizer from tensorflow.python.training import saver -def grappler_optimize(graph, fetches=None, rewriter_config=None): +def grappler_optimize(graph, fetches=None, config_proto=None): """Tries to optimize the provided graph using grappler. Args: @@ -31,17 +31,17 @@ def grappler_optimize(graph, fetches=None, rewriter_config=None): fetches: An optional list of `Tensor`s to fetch (i.e. not optimize away). Grappler uses the 'train_op' collection to look for fetches, so if not provided this collection should be non-empty. - rewriter_config: An optional `tf.RewriterConfig` to use when rewriting the + config_proto: An optional `tf.ConfigProto` to use when rewriting the graph. Returns: A `tf.GraphDef` containing the rewritten graph. """ - if rewriter_config is None: - rewriter_config = rewriter_config_pb2.RewriterConfig() - rewriter_config.min_graph_nodes = -1 + if config_proto is None: + config_proto = config_pb2.ConfigProto() + config_proto.graph_options.rewrite_options.min_graph_nodes = -1 if fetches is not None: for fetch in fetches: graph.add_to_collection('train_op', fetch) metagraph = saver.export_meta_graph(graph_def=graph.as_graph_def()) - return tf_optimizer.OptimizeGraph(rewriter_config, metagraph) + return tf_optimizer.OptimizeGraph(config_proto, metagraph) diff --git a/tensorflow/python/kernel_tests/signal/window_ops_test.py b/tensorflow/python/kernel_tests/signal/window_ops_test.py index 2f19134f5a8..a72cdb288bb 100644 --- a/tensorflow/python/kernel_tests/signal/window_ops_test.py +++ b/tensorflow/python/kernel_tests/signal/window_ops_test.py @@ -24,6 +24,7 @@ import numpy as np from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops +from tensorflow.python.framework import test_util as tf_test_util from tensorflow.python.kernel_tests.signal import test_util from tensorflow.python.ops.signal import window_ops from tensorflow.python.platform import test @@ -75,6 +76,7 @@ class WindowOpsTest(test.TestCase): dtype=tf_dtype).eval() self.assertAllClose(expected, actual, tol, tol) + @tf_test_util.run_deprecated_v1 def test_hann_window(self): """Check that hann_window matches scipy.signal.hann behavior.""" # The Hann window is a raised cosine window with parameters alpha=0.5 and @@ -84,6 +86,7 @@ class WindowOpsTest(test.TestCase): functools.partial(_scipy_raised_cosine, a=0.5, b=0.5), window_ops.hann_window) + @tf_test_util.run_deprecated_v1 def test_hamming_window(self): """Check that hamming_window matches scipy.signal.hamming's behavior.""" # The Hamming window is a raised cosine window with parameters alpha=0.54 diff --git a/tensorflow/python/kernel_tests/slice_op_test.py b/tensorflow/python/kernel_tests/slice_op_test.py index 41f040ab739..8f7245214a2 100644 --- a/tensorflow/python/kernel_tests/slice_op_test.py +++ b/tensorflow/python/kernel_tests/slice_op_test.py @@ -24,6 +24,7 @@ from six.moves import xrange # pylint: disable=redefined-builtin from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes from tensorflow.python.framework import errors_impl +from tensorflow.python.framework import test_util from tensorflow.python.ops import array_ops from tensorflow.python.ops import gradients_impl from tensorflow.python.ops import math_ops @@ -38,7 +39,7 @@ class SliceTest(test.TestCase): with self.cached_session(use_gpu=True): a = constant_op.constant(inp, shape=[4, 4], dtype=dtypes.float32) slice_t = a[2, k:k] - slice_val = slice_t.eval() + slice_val = self.evaluate(slice_t) self.assertAllEqual(slice_val, inp[2, k:k]) def testInt32(self): @@ -47,7 +48,7 @@ class SliceTest(test.TestCase): with self.cached_session(use_gpu=True): a = constant_op.constant(inp, shape=[4, 4], dtype=dtypes.int32) slice_t = a[2, k:k] - slice_val = slice_t.eval() + slice_val = self.evaluate(slice_t) self.assertAllEqual(slice_val, inp[2, k:k]) def testSlicingWithInt64Index(self): @@ -57,33 +58,33 @@ class SliceTest(test.TestCase): # Slice using int64 Tensor. i = constant_op.constant(1, dtype=dtypes.int64) slice_t = a[i] - slice_val = slice_t.eval() + slice_val = self.evaluate(slice_t) self.assertAllEqual(1, slice_val) slice_t = a[i:i+1] - slice_val = slice_t.eval() + slice_val = self.evaluate(slice_t) self.assertAllEqual([1], slice_val) # Slice using int64 integer. i = np.asarray(1).astype(np.int64) slice_t = a[i] - slice_val = slice_t.eval() + slice_val = self.evaluate(slice_t) self.assertAllEqual(1, slice_val) slice_t = a[i:i+1] - slice_val = slice_t.eval() + slice_val = self.evaluate(slice_t) self.assertAllEqual([1], slice_val) a_int32 = constant_op.constant([0, 1, 2], dtype=dtypes.int32) slice_t = array_ops.slice(a_int32, np.asarray([1]).astype(np.int64), np.asarray([2]).astype(np.int64)) - slice_val = slice_t.eval() + slice_val = self.evaluate(slice_t) self.assertAllEqual([1, 2], slice_val) a_float32 = constant_op.constant([0, 1, 2], dtype=dtypes.float32) slice_t = array_ops.slice(a_float32, np.asarray([1]).astype(np.int64), np.asarray([2]).astype(np.int64)) - slice_val = slice_t.eval() + slice_val = self.evaluate(slice_t) self.assertAllEqual([1, 2], slice_val) def testSlicingInt64Tensor(self): @@ -93,23 +94,23 @@ class SliceTest(test.TestCase): # Slice using int32 Tensor. i = constant_op.constant(1, dtype=dtypes.int32) slice_t = a[i] - slice_val = slice_t.eval() + slice_val = self.evaluate(slice_t) self.assertAllEqual(1, slice_val) slice_t = a[i:i + 1] - slice_val = slice_t.eval() + slice_val = self.evaluate(slice_t) self.assertAllEqual([1], slice_val) # Slice using int32 integer. i = np.asarray(1).astype(np.int32) slice_t = a[i] - slice_val = slice_t.eval() + slice_val = self.evaluate(slice_t) self.assertAllEqual(1, slice_val) slice_t = a[i:i + 1] - slice_val = slice_t.eval() + slice_val = self.evaluate(slice_t) self.assertAllEqual([1], slice_val) slice_t = array_ops.slice(a, [1], [2]) - slice_val = slice_t.eval() + slice_val = self.evaluate(slice_t) self.assertAllEqual([1, 2], slice_val) def testSelectAll(self): @@ -121,8 +122,8 @@ class SliceTest(test.TestCase): slice_explicit_t = array_ops.slice(a, [0, 0, 0, 0], [-1, -1, -1, -1]) slice_implicit_t = a[:, :, :, :] - self.assertAllEqual(inp, slice_explicit_t.eval()) - self.assertAllEqual(inp, slice_implicit_t.eval()) + self.assertAllEqual(inp, self.evaluate(slice_explicit_t)) + self.assertAllEqual(inp, self.evaluate(slice_implicit_t)) self.assertEqual(inp.shape, slice_explicit_t.get_shape()) self.assertEqual(inp.shape, slice_implicit_t.get_shape()) @@ -134,7 +135,7 @@ class SliceTest(test.TestCase): hi = np.random.randint(0, 9) scalar_t = a[hi] - scalar_val = scalar_t.eval() + scalar_val = self.evaluate(scalar_t) self.assertAllEqual(scalar_val, inp[hi]) if hi > 0: @@ -142,9 +143,10 @@ class SliceTest(test.TestCase): else: lo = 0 slice_t = a[lo:hi] - slice_val = slice_t.eval() + slice_val = self.evaluate(slice_t) self.assertAllEqual(slice_val, inp[lo:hi]) + @test_util.run_deprecated_v1 def testScalarInput(self): input_val = 0 with self.cached_session() as sess: @@ -159,6 +161,7 @@ class SliceTest(test.TestCase): "out of range"): sess.run([slice_t], feed_dict={input_t: input_val}) + @test_util.run_deprecated_v1 def testInvalidIndex(self): input_val = [1, 2] with self.cached_session() as sess: @@ -179,6 +182,7 @@ class SliceTest(test.TestCase): np_ans = x[begin:begin + size, :] self.assertAllEqual(tf_ans, np_ans) + @test_util.run_deprecated_v1 def testSliceMatrixDim0(self): x = np.random.rand(8, 4).astype("f") self._testSliceMatrixDim0(x, 1, 2) @@ -195,7 +199,7 @@ class SliceTest(test.TestCase): x, y = np.random.randint(0, 3, size=2).tolist() slice_t = a[x, 0:y] - slice_val = slice_t.eval() + slice_val = self.evaluate(slice_t) self.assertAllEqual(slice_val, inp[x, 0:y]) def testSimple(self): @@ -207,12 +211,13 @@ class SliceTest(test.TestCase): dtype=dtypes.float32) slice_t = array_ops.slice(a, [0, 0], [2, 2]) slice2_t = a[:2, :2] - slice_val, slice2_val = sess.run([slice_t, slice2_t]) + slice_val, slice2_val = self.evaluate([slice_t, slice2_t]) self.assertAllEqual(slice_val, inp[:2, :2]) self.assertAllEqual(slice2_val, inp[:2, :2]) self.assertEqual(slice_val.shape, slice_t.get_shape()) self.assertEqual(slice2_val.shape, slice2_t.get_shape()) + @test_util.run_deprecated_v1 def testComplex(self): with self.session(use_gpu=True): inp = np.random.rand(4, 10, 10, 4).astype("f") @@ -247,7 +252,7 @@ class SliceTest(test.TestCase): + sizes[3], indices[4]:indices[4] + sizes[4], indices[5]: indices[5] + sizes[5]] - slice_val, slice2_val = sess.run([slice_t, slice2_t]) + slice_val, slice2_val = self.evaluate([slice_t, slice2_t]) expected_val = inp[indices[0]:indices[0] + sizes[0], indices[1]:indices[ 1] + sizes[1], indices[2]:indices[2] + sizes[2], indices[3]:indices[ @@ -282,7 +287,7 @@ class SliceTest(test.TestCase): grads = np.random.rand(num_grads).astype("f").reshape(slice_size) grad_tensor = constant_op.constant(grads) grad = gradients_impl.gradients(slice_t, [a], grad_tensor)[0] - result = grad.eval() + result = self.evaluate(grad) # Create a zero tensor of the input shape ane place # the grads into the right location to compare against TensorFlow. @@ -313,9 +318,10 @@ class SliceTest(test.TestCase): g1 = gradients_impl.gradients(loss1, x)[0] g2 = gradients_impl.gradients(loss2, x)[0] - g1_val, g2_val = sess.run([g1, g2]) + g1_val, g2_val = self.evaluate([g1, g2]) self.assertAllEqual(g1_val, g2_val) + @test_util.run_deprecated_v1 def testGradientsAll(self): # Slice the middle square out of a 4x4 input self._testGradientSlice([4, 4], [1, 1], [2, 2]) @@ -335,6 +341,7 @@ class SliceTest(test.TestCase): # Use -1 as a slice dimension on a 2D tensor. self._testGradientVariableSize2D() + @test_util.run_deprecated_v1 def testNotIterable(self): # NOTE(mrry): If we register __getitem__ as an overloaded # operator, Python will valiantly attempt to iterate over the @@ -346,6 +353,7 @@ class SliceTest(test.TestCase): for _ in c: pass + @test_util.run_deprecated_v1 def testComputedShape(self): # NOTE(mrry): We cannot currently handle partially-known values, # because `tf.slice()` uses -1 to specify a wildcard size, and @@ -368,7 +376,7 @@ class SliceTest(test.TestCase): c = b[:-1, :] d = c[1, :] res = 2 * d - c[1, :] + a[2, :] - 2 * b[-2, :] - self.assertAllEqual([0, 0, 0], res.eval()) + self.assertAllEqual([0, 0, 0], self.evaluate(res)) if __name__ == "__main__": diff --git a/tensorflow/python/kernel_tests/softmax_op_test.py b/tensorflow/python/kernel_tests/softmax_op_test.py index ef9301d4e35..707b8a429f2 100644 --- a/tensorflow/python/kernel_tests/softmax_op_test.py +++ b/tensorflow/python/kernel_tests/softmax_op_test.py @@ -24,6 +24,7 @@ import numpy as np from tensorflow.python.framework import dtypes from tensorflow.python.framework import errors_impl +from tensorflow.python.framework import test_util from tensorflow.python.ops import array_ops from tensorflow.python.ops import nn_ops from tensorflow.python.platform import test @@ -64,7 +65,7 @@ class SoftmaxTest(test.TestCase): tf_softmax = nn_ops.log_softmax(np_features, axis=dim, name=name) else: tf_softmax = nn_ops.softmax(np_features, axis=dim, name=name) - out = tf_softmax.eval() + out = self.evaluate(tf_softmax) self.assertAllCloseAccordingToType(np_softmax, out) self.assertShapeEqual(np_softmax, tf_softmax) if not log: @@ -113,7 +114,7 @@ class SoftmaxTest(test.TestCase): features = np.array([[1., 1., 1., 1.], [max, 1., 2., 3.]]).astype(type) with self.cached_session(use_gpu=use_gpu): tf_log_softmax = nn_ops.log_softmax(features) - out = tf_log_softmax.eval() + out = self.evaluate(tf_log_softmax) self.assertAllClose( np.array([[-1.386294, -1.386294, -1.386294, -1.386294], [0, -max, -max, -max]]), @@ -206,6 +207,7 @@ class SoftmaxTest(test.TestCase): [[5., 4., 3., 2.], [1., 2., 3., 4.]]]) self.assertEqual([3, 2, 4], op.get_shape()) + @test_util.run_deprecated_v1 def testEmptyInput(self): with self.cached_session(): x = array_ops.placeholder(dtypes.float32, shape=[0, 3]) @@ -229,6 +231,7 @@ class SoftmaxTest(test.TestCase): with self.assertRaises(errors_impl.InvalidArgumentError): nn_ops.softmax(ones, axis=2).eval() + @test_util.run_deprecated_v1 def testLargeDims(self): # Make sure that we properly handle large inputs. See # https://github.com/tensorflow/tensorflow/issues/4425 for details diff --git a/tensorflow/python/kernel_tests/softplus_op_test.py b/tensorflow/python/kernel_tests/softplus_op_test.py index 50a8291ea88..5273dd7ffc7 100644 --- a/tensorflow/python/kernel_tests/softplus_op_test.py +++ b/tensorflow/python/kernel_tests/softplus_op_test.py @@ -21,6 +21,7 @@ from __future__ import print_function import numpy as np from tensorflow.python.framework import constant_op +from tensorflow.python.framework import test_util from tensorflow.python.ops import gradient_checker from tensorflow.python.ops import gradients_impl from tensorflow.python.ops import nn_ops @@ -39,7 +40,7 @@ class SoftplusTest(test.TestCase): np_softplus = self._npSoftplus(np_features) with self.cached_session(use_gpu=use_gpu): softplus = nn_ops.softplus(np_features) - tf_softplus = softplus.eval() + tf_softplus = self.evaluate(softplus) self.assertAllCloseAccordingToType(np_softplus, tf_softplus) self.assertTrue(np.all(tf_softplus > 0)) self.assertShapeEqual(np_softplus, softplus) @@ -70,6 +71,7 @@ class SoftplusTest(test.TestCase): ], use_gpu=True) + @test_util.run_deprecated_v1 def testGradient(self): with self.cached_session(): x = constant_op.constant( @@ -86,6 +88,7 @@ class SoftplusTest(test.TestCase): print("softplus (float) gradient err = ", err) self.assertLess(err, 1e-4) + @test_util.run_deprecated_v1 def testGradGrad(self): with self.cached_session(): x = constant_op.constant( @@ -103,6 +106,7 @@ class SoftplusTest(test.TestCase): print("softplus (float) gradient of gradient err = ", err) self.assertLess(err, 5e-5) + @test_util.run_deprecated_v1 def testGradGradGrad(self): with self.cached_session(): x = constant_op.constant( @@ -121,6 +125,7 @@ class SoftplusTest(test.TestCase): print("softplus (float) third-order gradient err = ", err) self.assertLess(err, 5e-5) + @test_util.run_deprecated_v1 def testNoInts(self): with self.cached_session(): with self.assertRaisesRegexp( diff --git a/tensorflow/python/kernel_tests/softsign_op_test.py b/tensorflow/python/kernel_tests/softsign_op_test.py index ee2e2e03032..5554240c826 100644 --- a/tensorflow/python/kernel_tests/softsign_op_test.py +++ b/tensorflow/python/kernel_tests/softsign_op_test.py @@ -21,6 +21,7 @@ from __future__ import print_function import numpy as np from tensorflow.python.framework import constant_op +from tensorflow.python.framework import test_util from tensorflow.python.ops import gradient_checker from tensorflow.python.ops import nn_ops import tensorflow.python.ops.nn_grad # pylint: disable=unused-import @@ -36,7 +37,7 @@ class SoftsignTest(test.TestCase): np_softsign = self._npSoftsign(np_features) with self.cached_session(use_gpu=use_gpu): softsign = nn_ops.softsign(np_features) - tf_softsign = softsign.eval() + tf_softsign = self.evaluate(softsign) self.assertAllClose(np_softsign, tf_softsign) self.assertShapeEqual(np_softsign, softsign) @@ -49,6 +50,7 @@ class SoftsignTest(test.TestCase): np.array([[-9, 7, -5, 3, -1], [1, -3, 5, -7, 9]]).astype(t), use_gpu=True) + @test_util.run_deprecated_v1 def testGradient(self): with self.cached_session(): x = constant_op.constant( @@ -65,6 +67,7 @@ class SoftsignTest(test.TestCase): print("softsign (float) gradient err = ", err) self.assertLess(err, 1e-4) + @test_util.run_deprecated_v1 def testNoInts(self): with self.cached_session(): with self.assertRaisesRegexp( diff --git a/tensorflow/python/kernel_tests/spacetobatch_op_test.py b/tensorflow/python/kernel_tests/spacetobatch_op_test.py index 21134adf2ca..8641156604c 100644 --- a/tensorflow/python/kernel_tests/spacetobatch_op_test.py +++ b/tensorflow/python/kernel_tests/spacetobatch_op_test.py @@ -23,6 +23,7 @@ import numpy as np from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops from tensorflow.python.framework import tensor_util +from tensorflow.python.framework import test_util from tensorflow.python.ops import array_ops from tensorflow.python.ops import gen_array_ops from tensorflow.python.ops import gradient_checker @@ -115,6 +116,7 @@ class SpaceToBatchTest(test.TestCase, PythonOpImpl): self._testPad(inputs, paddings, block_size, outputs) # [1, 2, 2, 1] <-> [4, 1, 1, 1] + @test_util.run_deprecated_v1 def testSmallInput2x2(self): x_np = [[[[1], [2]], [[3], [4]]]] block_size = 2 @@ -122,6 +124,7 @@ class SpaceToBatchTest(test.TestCase, PythonOpImpl): self._testOne(x_np, block_size, x_out) # [1, 2, 2, 1] <-> [1, 3, 3, 1] (padding) <-> [9, 1, 1, 1] + @test_util.run_deprecated_v1 def testSmallInput2x2Pad1x0(self): x_np = [[[[1], [2]], [[3], [4]]]] paddings = np.array([[1, 0], [1, 0]], dtype=np.int32) @@ -132,6 +135,7 @@ class SpaceToBatchTest(test.TestCase, PythonOpImpl): # Test with depth larger than 1. # [1, 2, 2, 3] <-> [4, 1, 1, 3] + @test_util.run_deprecated_v1 def testDepthInput2x2(self): x_np = [[[[1, 2, 3], [4, 5, 6]], [[7, 8, 9], [10, 11, 12]]]] block_size = 2 @@ -140,6 +144,7 @@ class SpaceToBatchTest(test.TestCase, PythonOpImpl): # Test for larger input dimensions. # [1, 4, 4, 1] <-> [4, 2, 2, 1] + @test_util.run_deprecated_v1 def testLargerInput2x2(self): x_np = [[[[1], [2], [3], [4]], [[5], [6], [7], [8]], [[9], [10], [11], [12]], [[13], [14], [15], [16]]]] @@ -150,6 +155,7 @@ class SpaceToBatchTest(test.TestCase, PythonOpImpl): # Test with batch larger than 1. # [2, 2, 4, 1] <-> [8, 1, 2, 1] + @test_util.run_deprecated_v1 def testBatchInput2x2(self): x_np = [[[[1], [2], [3], [4]], [[5], [6], [7], [8]]], [[[9], [10], [11], [12]], [[13], [14], [15], [16]]]] @@ -162,6 +168,7 @@ class SpaceToBatchTest(test.TestCase, PythonOpImpl): # that elements are correctly laid out spatially and properly interleaved # along the batch dimension. # [2, 4, 4, 1] <-> [8, 2, 2, 1] + @test_util.run_deprecated_v1 def testLargerInputBatch2x2(self): x_np = [[[[1], [2], [3], [4]], [[5], [6], [7], [8]], [[9], [10], [11], [12]], [[13], [14], [15], [16]]], @@ -206,6 +213,7 @@ class SpaceToBatchNDTest(test.TestCase): self._testPad(inputs, block_shape, paddings, space_to_batch_direct(inputs, block_shape, paddings)) + @test_util.run_deprecated_v1 def testZeroBlockDimsZeroRemainingDims(self): self._testPad( inputs=[1, 2], @@ -213,6 +221,7 @@ class SpaceToBatchNDTest(test.TestCase): paddings=[], outputs=[1, 2],) + @test_util.run_deprecated_v1 def testZeroBlockDimsOneRemainingDim(self): self._testPad( inputs=[[1, 2], [3, 4]], @@ -227,6 +236,7 @@ class SpaceToBatchNDTest(test.TestCase): paddings=[[0, 0]], outputs=[[1, 2], [3, 4]]) + @test_util.run_deprecated_v1 def testZeroBlockDimsTwoRemainingDims(self): self._testPad( inputs=[[[1, 2], [3, 4]], [[5, 6], [7, 8]]], @@ -248,6 +258,7 @@ class SpaceToBatchNDTest(test.TestCase): paddings=[[0, 0], [0, 0]], outputs=[[[1, 2], [3, 4]], [[5, 6], [7, 8]]]) + @test_util.run_deprecated_v1 def testOneBlockDimZeroRemainingDims(self): self._testPad( inputs=[[1, 2, 3], [4, 5, 6]], @@ -255,6 +266,7 @@ class SpaceToBatchNDTest(test.TestCase): paddings=[1, 0], outputs=[[0, 2], [0, 5], [1, 3], [4, 6]]) + @test_util.run_deprecated_v1 def testOneBlockDimOneRemainingDim(self): self._testPad( inputs=[[[1, 11], [2, 21], [3, 31]], [[4, 41], [5, 51], [6, 61]]], @@ -263,6 +275,7 @@ class SpaceToBatchNDTest(test.TestCase): outputs=[[[0, 0], [2, 21]], [[0, 0], [5, 51]], [[1, 11], [3, 31]], [[4, 41], [6, 61]]]) + @test_util.run_deprecated_v1 def testDirect(self): # Test with zero-size remaining dimension. self._testDirect( @@ -300,6 +313,7 @@ class SpaceToBatchNDTest(test.TestCase): class SpaceToBatchSpaceToDepth(test.TestCase, PythonOpImpl): # Verifies that: space_to_batch(x) = transpose(space_to_depth(transpose(x))) + @test_util.run_deprecated_v1 def testSpaceToDepthTranspose(self): x = np.arange(5 * 10 * 16 * 7, dtype=np.float32).reshape([5, 10, 16, 7]) block_size = 2 @@ -319,6 +333,7 @@ class SpaceToBatchSpaceToDepthCpp(SpaceToBatchSpaceToDepth, CppOpImpl): class SpaceToBatchErrorHandlingTest(test.TestCase, PythonOpImpl): + @test_util.run_deprecated_v1 def testInputWrongDimMissingBatch(self): # The input is missing the first dimension ("batch") x_np = [[[1], [2]], [[3], [4]]] @@ -327,6 +342,7 @@ class SpaceToBatchErrorHandlingTest(test.TestCase, PythonOpImpl): with self.assertRaises(ValueError): _ = self.space_to_batch(x_np, paddings, block_size) + @test_util.run_deprecated_v1 def testBlockSize0(self): # The block size is 0. x_np = [[[[1], [2]], [[3], [4]]]] @@ -336,6 +352,7 @@ class SpaceToBatchErrorHandlingTest(test.TestCase, PythonOpImpl): out_tf = self.space_to_batch(x_np, paddings, block_size) out_tf.eval() + @test_util.run_deprecated_v1 def testBlockSizeOne(self): # The block size is 1. The block size needs to be > 1. x_np = [[[[1], [2]], [[3], [4]]]] @@ -345,6 +362,7 @@ class SpaceToBatchErrorHandlingTest(test.TestCase, PythonOpImpl): out_tf = self.space_to_batch(x_np, paddings, block_size) out_tf.eval() + @test_util.run_deprecated_v1 def testBlockSizeLarger(self): # The block size is too large for this input. x_np = [[[[1], [2]], [[3], [4]]]] @@ -354,6 +372,7 @@ class SpaceToBatchErrorHandlingTest(test.TestCase, PythonOpImpl): out_tf = self.space_to_batch(x_np, paddings, block_size) out_tf.eval() + @test_util.run_deprecated_v1 def testBlockSizeNotDivisibleWidth(self): # The block size divides width but not height. x_np = [[[[1], [2], [3]], [[3], [4], [7]]]] @@ -362,6 +381,7 @@ class SpaceToBatchErrorHandlingTest(test.TestCase, PythonOpImpl): with self.assertRaises(ValueError): _ = self.space_to_batch(x_np, paddings, block_size) + @test_util.run_deprecated_v1 def testBlockSizeNotDivisibleHeight(self): # The block size divides height but not width. x_np = [[[[1], [2]], [[3], [4]], [[5], [6]]]] @@ -370,6 +390,7 @@ class SpaceToBatchErrorHandlingTest(test.TestCase, PythonOpImpl): with self.assertRaises(ValueError): _ = self.space_to_batch(x_np, paddings, block_size) + @test_util.run_deprecated_v1 def testBlockSizeNotDivisibleBoth(self): # The block size does not divide neither width or height. x_np = [[[[1], [2]], [[3], [4]]]] @@ -378,6 +399,7 @@ class SpaceToBatchErrorHandlingTest(test.TestCase, PythonOpImpl): with self.assertRaises(ValueError): _ = self.space_to_batch(x_np, paddings, block_size) + @test_util.run_deprecated_v1 def testUnknownShape(self): t = self.space_to_batch( array_ops.placeholder(dtypes.float32), @@ -424,25 +446,31 @@ class SpaceToBatchNDErrorHandlingTest(test.TestCase): self._testStaticShape(input_shape, block_shape, paddings, error) self._testDynamicShape(input_shape, block_shape, paddings) + @test_util.run_deprecated_v1 def testBlockSize0(self): # The block size is 0. self._testShape([1, 2, 2], [0, 2], [[0, 0], [0, 0]], ValueError) + @test_util.run_deprecated_v1 def testBlockSizeNegative(self): self._testShape([1, 2, 2], [-1, 2], [[0, 0], [0, 0]], ValueError) + @test_util.run_deprecated_v1 def testNegativePadding(self): # The padding is negative. self._testShape([1, 2, 2], [1, 1], [[0, -1], [0, 0]], ValueError) + @test_util.run_deprecated_v1 def testBlockSizeNotDivisible(self): # The padded size is not divisible by the block size. self._testShape([1, 2, 3, 1], [3, 3], [[0, 0], [0, 0]], ValueError) + @test_util.run_deprecated_v1 def testBlockDimsMismatch(self): # Shape of block_shape does not match shape of paddings. self._testStaticShape([1, 3, 3, 1], [3, 3], [[0, 0]], ValueError) + @test_util.run_deprecated_v1 def testUnknown(self): # Verify that input shape and paddings shape can be unknown. _ = array_ops.space_to_batch_nd( @@ -522,18 +550,21 @@ class SpaceToBatchGradientTest(test.TestCase, PythonOpImpl): # Don't use very large numbers as dimensions here as the result is tensor # with cartesian product of the dimensions. + @test_util.run_deprecated_v1 def testSmall(self): block_size = 2 pad_beg = 0 pad_end = 0 self._compare(1, 2, 3, 5, block_size, pad_beg, pad_end) + @test_util.run_deprecated_v1 def testSmall2(self): block_size = 2 pad_beg = 0 pad_end = 0 self._compare(2, 4, 3, 2, block_size, pad_beg, pad_end) + @test_util.run_deprecated_v1 def testSmallPad1x1(self): block_size = 2 pad_beg = 1 @@ -572,15 +603,19 @@ class SpaceToBatchNDGradientTest(test.TestCase): # Don't use very large numbers as dimensions here as the result is tensor # with cartesian product of the dimensions. + @test_util.run_deprecated_v1 def testSmall(self): self._compare([1, 4, 6, 5], [2, 2], [[0, 0], [0, 0]]) + @test_util.run_deprecated_v1 def testSmall2(self): self._compare([2, 8, 6, 2], [2, 2], [[0, 0], [0, 0]]) + @test_util.run_deprecated_v1 def testSmallPad1(self): self._compare([2, 4, 6, 2], [2, 2], [[1, 1], [1, 1]]) + @test_util.run_deprecated_v1 def testSmallPadThreeBlockDims(self): self._compare([2, 2, 4, 3, 2], [2, 2, 2], [[1, 1], [1, 1], [1, 0]]) @@ -644,6 +679,7 @@ class RequiredSpaceToBatchPaddingsTest(test.TestCase): self.assertAllEqual(paddings_result, paddings_const) self.assertAllEqual(crops_result, crops_const) + @test_util.run_deprecated_v1 def testSimple(self): self._test( input_shape=np.zeros((0,), np.int32), diff --git a/tensorflow/python/kernel_tests/spacetodepth_op_test.py b/tensorflow/python/kernel_tests/spacetodepth_op_test.py index b05f14f7381..e96bc09f365 100644 --- a/tensorflow/python/kernel_tests/spacetodepth_op_test.py +++ b/tensorflow/python/kernel_tests/spacetodepth_op_test.py @@ -36,21 +36,22 @@ class SpaceToDepthTest(test.TestCase): def _testOne(self, inputs, block_size, outputs, dtype=dtypes.float32): input_nhwc = math_ops.cast(inputs, dtype) - with self.session(use_gpu=False): + with test_util.force_cpu(): # test NHWC (default) on CPU x_tf = array_ops.space_to_depth(input_nhwc, block_size) - self.assertAllEqual(x_tf.eval(), outputs) - if test.is_gpu_available(): - with self.session(force_gpu=True): + self.assertAllEqual(self.evaluate(x_tf), outputs) + + if test_util.is_gpu_available(): + with test_util.force_gpu(): # test NHWC (default) on GPU x_tf = array_ops.space_to_depth(input_nhwc, block_size) - self.assertAllEqual(x_tf.eval(), outputs) + self.assertAllEqual(self.evaluate(x_tf), outputs) # test NCHW on GPU input_nchw = test_util.NHWCToNCHW(input_nhwc) output_nchw = array_ops.space_to_depth( input_nchw, block_size, data_format="NCHW") output_nhwc = test_util.NCHWToNHWC(output_nchw) - self.assertAllEqual(output_nhwc.eval(), outputs) + self.assertAllEqual(self.evaluate(output_nhwc), outputs) def testBasic(self): x_np = [[[[1], [2]], [[3], [4]]]] @@ -134,17 +135,18 @@ class SpaceToDepthTest(test.TestCase): input_nhwc = array_ops.ones([batch_size, 4, 6, 3]) x_out = array_ops.ones([batch_size, 2, 3, 12]) - with self.session(use_gpu=False): + with test_util.force_cpu(): # test NHWC (default) on CPU x_tf = array_ops.space_to_depth(input_nhwc, block_size) self.assertAllEqual(x_tf.shape, x_out.shape) - x_tf.eval() + self.evaluate(x_tf) + if test.is_gpu_available(): - with self.session(use_gpu=True): + with test_util.use_gpu(): # test NHWC (default) on GPU x_tf = array_ops.space_to_depth(input_nhwc, block_size) self.assertAllEqual(x_tf.shape, x_out.shape) - x_tf.eval() + self.evaluate(x_tf) # Tests for different width and height. def testNonSquare(self): @@ -157,14 +159,16 @@ class SpaceToDepthTest(test.TestCase): # Error handling: + @test_util.run_deprecated_v1 def testInputWrongDimMissingDepth(self): # The input is missing the last dimension ("depth") x_np = [[[1, 2], [3, 4]]] block_size = 2 with self.assertRaises(ValueError): out_tf = array_ops.space_to_depth(x_np, block_size) - out_tf.eval() + self.evaluate(out_tf) + @test_util.run_deprecated_v1 def testInputWrongDimMissingBatch(self): # The input is missing the first dimension ("batch") x_np = [[[1], [2]], [[3], [4]]] @@ -172,30 +176,34 @@ class SpaceToDepthTest(test.TestCase): with self.assertRaises(ValueError): _ = array_ops.space_to_depth(x_np, block_size) + @test_util.run_deprecated_v1 def testBlockSize0(self): # The block size is 0. x_np = [[[[1], [2]], [[3], [4]]]] block_size = 0 with self.assertRaises(ValueError): out_tf = array_ops.space_to_depth(x_np, block_size) - out_tf.eval() + self.evaluate(out_tf) + @test_util.run_deprecated_v1 def testBlockSizeOne(self): # The block size is 1. The block size needs to be > 1. x_np = [[[[1], [2]], [[3], [4]]]] block_size = 1 with self.assertRaises(ValueError): out_tf = array_ops.space_to_depth(x_np, block_size) - out_tf.eval() + self.evaluate(out_tf) + @test_util.run_deprecated_v1 def testBlockSizeLarger(self): # The block size is too large for this input. x_np = [[[[1], [2]], [[3], [4]]]] block_size = 10 with self.assertRaises(ValueError): out_tf = array_ops.space_to_depth(x_np, block_size) - out_tf.eval() + self.evaluate(out_tf) + @test_util.run_deprecated_v1 def testBlockSizeNotDivisibleWidth(self): # The block size divides width but not height. x_np = [[[[1], [2], [3]], [[3], [4], [7]]]] @@ -203,6 +211,7 @@ class SpaceToDepthTest(test.TestCase): with self.assertRaises(ValueError): _ = array_ops.space_to_depth(x_np, block_size) + @test_util.run_deprecated_v1 def testBlockSizeNotDivisibleHeight(self): # The block size divides height but not width. x_np = [[[[1], [2]], [[3], [4]], [[5], [6]]]] @@ -210,6 +219,7 @@ class SpaceToDepthTest(test.TestCase): with self.assertRaises(ValueError): _ = array_ops.space_to_depth(x_np, block_size) + @test_util.run_deprecated_v1 def testBlockSizeNotDivisibleBoth(self): # The block size does not divide neither width or height. x_np = [[[[1], [2]], [[3], [4]]]] @@ -217,6 +227,7 @@ class SpaceToDepthTest(test.TestCase): with self.assertRaises(ValueError): _ = array_ops.space_to_depth(x_np, block_size) + @test_util.run_deprecated_v1 def testUnknownShape(self): t = array_ops.space_to_depth( array_ops.placeholder(dtypes.float32), block_size=4) @@ -271,7 +282,7 @@ class SpaceToDepthTest(test.TestCase): actual = array_ops.space_to_depth(t, block_size, data_format=data_format) with self.cached_session(use_gpu=use_gpu) as sess: - actual_vals, expected_vals = sess.run([actual, expected]) + actual_vals, expected_vals = self.evaluate([actual, expected]) self.assertTrue(np.array_equal(actual_vals, expected_vals)) def testAgainstTranspose(self): @@ -332,11 +343,13 @@ class SpaceToDepthGradientTest(test.TestCase): # Don't use very large numbers as dimensions here as the result is tensor # with cartesian product of the dimensions. + @test_util.run_deprecated_v1 def testSmall(self): block_size = 2 self._compare(1, 2, 3, 5, block_size, "NHWC") self._compare(1, 2, 3, 5, block_size, "NCHW") + @test_util.run_deprecated_v1 def testSmall2(self): block_size = 2 self._compare(2, 4, 3, 2, block_size, "NHWC") diff --git a/tensorflow/python/kernel_tests/sparse_add_op_test.py b/tensorflow/python/kernel_tests/sparse_add_op_test.py index a746830afb3..00eff54077c 100644 --- a/tensorflow/python/kernel_tests/sparse_add_op_test.py +++ b/tensorflow/python/kernel_tests/sparse_add_op_test.py @@ -28,6 +28,7 @@ from tensorflow.python.framework import dtypes from tensorflow.python.framework import errors_impl from tensorflow.python.framework import ops from tensorflow.python.framework import sparse_tensor +from tensorflow.python.framework import test_util from tensorflow.python.ops import gradient_checker from tensorflow.python.ops import math_ops from tensorflow.python.ops import sparse_ops @@ -85,13 +86,13 @@ class SparseAddTest(test.TestCase): constant_op.constant(shape, dtypes.int64)) def testAddSelf(self): - with self.session(use_gpu=False) as sess: + with test_util.force_cpu(): for sp_a in (self._SparseTensorValue_3x3(), self._SparseTensor_3x3()): for sp_b in (self._SparseTensorValue_3x3(), self._SparseTensor_3x3()): sp_sum = sparse_ops.sparse_add(sp_a, sp_b) self.assertAllEqual((3, 3), sp_sum.get_shape()) - sum_out = sess.run(sp_sum) + sum_out = self.evaluate(sp_sum) self.assertEqual(sp_sum.dense_shape.get_shape(), [2]) self.assertAllEqual(sum_out.indices, [[0, 1], [1, 0], [2, 0], [2, 1]]) @@ -99,12 +100,12 @@ class SparseAddTest(test.TestCase): self.assertAllEqual(sum_out.dense_shape, [3, 3]) def testAddSelfAndNegation(self): - with self.session(use_gpu=False) as sess: + with test_util.force_cpu(): sp_a = self._SparseTensor_3x3() sp_b = self._SparseTensor_3x3(negate=True) sp_sum = sparse_ops.sparse_add(sp_a, sp_b, 0.1) - sum_out = sess.run(sp_sum) + sum_out = self.evaluate(sp_sum) self.assertEqual(sp_sum.dense_shape.get_shape(), [2]) self.assertAllEqual(sum_out.indices, np.empty([0, 2])) @@ -112,7 +113,7 @@ class SparseAddTest(test.TestCase): self.assertAllEqual(sum_out.dense_shape, [3, 3]) def testSmallValuesShouldVanish(self): - with self.session(use_gpu=False) as sess: + with test_util.force_cpu(): sp_a = self._SparseTensor_3x3() sp_b = self._SparseTensor_3x3_v2() @@ -123,7 +124,7 @@ class SparseAddTest(test.TestCase): # two values should vanish: |.1| < .21, and |-.2| < .21 sp_sum = sparse_ops.sparse_add(sp_a, sp_b, thresh=0.21) - sum_out = sess.run(sp_sum) + sum_out = self.evaluate(sp_sum) self.assertEqual(sp_sum.dense_shape.get_shape(), [2]) self.assertAllEqual(sum_out.indices, [[0, 1], [2, 0]]) @@ -132,13 +133,14 @@ class SparseAddTest(test.TestCase): # only .1 vanishes sp_sum = sparse_ops.sparse_add(sp_a, sp_b, thresh=0.11) - sum_out = sess.run(sp_sum) + sum_out = self.evaluate(sp_sum) self.assertEqual(sp_sum.dense_shape.get_shape(), [2]) self.assertAllEqual(sum_out.indices, [[0, 1], [2, 0], [2, 1]]) self.assertAllClose(sum_out.values, [2, 6, -.2]) self.assertAllEqual(sum_out.dense_shape, [3, 3]) + @test_util.run_deprecated_v1 def testGradients(self): np.random.seed(1618) # Make it reproducible. with self.session(use_gpu=False): @@ -147,7 +149,7 @@ class SparseAddTest(test.TestCase): sp_a, nnz_a = self._randomTensor([n, m], np.float32) sp_b, nnz_b = self._randomTensor([n, m], np.float32) sp_sum = sparse_ops.sparse_add(sp_a, sp_b) - nnz_sum = len(sp_sum.values.eval()) + nnz_sum = len(self.evaluate(sp_sum.values)) err = gradient_checker.compute_gradient_error( [sp_a.values, sp_b.values], [(nnz_a,), (nnz_b,)], sp_sum.values, @@ -162,19 +164,20 @@ class SparseAddTest(test.TestCase): rand_vals_np = np.random.randn(n, m).astype(dtype) dense_np = np.random.randn(n, m).astype(dtype) - with self.cached_session(use_gpu=False): + with test_util.force_cpu(): sparse, unused_nnz = _sparsify(rand_vals_np, index_dtype=index_dtype) - s = sparse_ops.sparse_add(sparse, - constant_op.constant(dense_np)).eval() + s = self.evaluate( + sparse_ops.sparse_add(sparse, constant_op.constant(dense_np))) self.assertAllEqual(dense_np + rand_vals_np, s) self.assertTrue(s.dtype == dtype) # check commutativity - s = sparse_ops.sparse_add(constant_op.constant(dense_np), - sparse).eval() + s = self.evaluate( + sparse_ops.sparse_add(constant_op.constant(dense_np), sparse)) self.assertAllEqual(dense_np + rand_vals_np, s) self.assertTrue(s.dtype == dtype) + @test_util.run_deprecated_v1 def testSparseTensorDenseAddGradients(self): np.random.seed(1618) # Make it reproducible. n, m = np.random.randint(30, size=2) @@ -190,8 +193,9 @@ class SparseAddTest(test.TestCase): [(nnz,), (n, m)], s, (n, m)) self.assertLess(err, 1e-3) + @test_util.run_deprecated_v1 def testInvalidSparseTensor(self): - with self.session(use_gpu=False) as sess: + with test_util.force_cpu(): shape = [2, 2] val = [0] dense = constant_op.constant(np.zeros(shape, dtype=np.int32)) @@ -205,7 +209,7 @@ class SparseAddTest(test.TestCase): with self.assertRaisesRegexp(errors_impl.InvalidArgumentError, "invalid index"): - sess.run(s) + self.evaluate(s) ######################## Benchmarking code diff --git a/tensorflow/python/kernel_tests/sparse_concat_op_test.py b/tensorflow/python/kernel_tests/sparse_concat_op_test.py index 402c5eb4ea3..04b6b9b8d20 100644 --- a/tensorflow/python/kernel_tests/sparse_concat_op_test.py +++ b/tensorflow/python/kernel_tests/sparse_concat_op_test.py @@ -23,6 +23,7 @@ import numpy as np from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes from tensorflow.python.framework import sparse_tensor +from tensorflow.python.framework import test_util from tensorflow.python.ops import array_ops from tensorflow.python.ops import sparse_ops from tensorflow.python.platform import test @@ -147,7 +148,7 @@ class SparseConcatTest(test.TestCase): self.assertEqual(sp_concat.values.get_shape(), [4]) self.assertEqual(sp_concat.dense_shape.get_shape(), [2]) - concat_out = sess.run(sp_concat) + concat_out = self.evaluate(sp_concat) self.assertAllEqual(concat_out.indices, [[0, 2], [1, 0], [2, 0], [2, 2]]) @@ -169,7 +170,7 @@ class SparseConcatTest(test.TestCase): self.assertEqual(sp_concat.values.get_shape(), [8]) self.assertEqual(sp_concat.dense_shape.get_shape(), [2]) - concat_out = sess.run(sp_concat) + concat_out = self.evaluate(sp_concat) self.assertAllEqual(concat_out.indices, [[0, 2], [1, 0], [1, 4], [2, 0], [2, 2], [2, 3], @@ -195,7 +196,7 @@ class SparseConcatTest(test.TestCase): self.assertEqual(sp_concat.values.get_shape(), [7]) self.assertEqual(sp_concat.dense_shape.get_shape(), [2]) - concat_out = sess.run(sp_concat) + concat_out = self.evaluate(sp_concat) self.assertAllEqual( concat_out.indices, @@ -220,7 +221,7 @@ class SparseConcatTest(test.TestCase): self.assertEqual(sp_concat.values.get_shape(), [10]) self.assertEqual(sp_concat.dense_shape.get_shape(), [2]) - concat_out = sess.run(sp_concat) + concat_out = self.evaluate(sp_concat) self.assertAllEqual(concat_out.indices, [[0, 2], [1, 0], [1, 4], [1, 8], [2, 0], [2, 2], [2, 3], [2, 6], @@ -244,7 +245,7 @@ class SparseConcatTest(test.TestCase): self.assertEqual(sp_concat.values.get_shape(), [8]) self.assertEqual(sp_concat.dense_shape.get_shape(), [2]) - concat_out = sess.run(sp_concat) + concat_out = self.evaluate(sp_concat) self.assertAllEqual( concat_out.indices, @@ -253,6 +254,7 @@ class SparseConcatTest(test.TestCase): [b"a", b"b", b"e", b"c", b"d", b"f", b"g", b"h"]) self.assertAllEqual(concat_out.dense_shape, [3, 8]) + @test_util.run_deprecated_v1 def testMismatchedRank(self): with self.session(use_gpu=False): sp_a = self._SparseTensor_3x3() @@ -263,6 +265,7 @@ class SparseConcatTest(test.TestCase): with self.assertRaises(ValueError): sparse_ops.sparse_concat(concat_dim, [sp_a, sp_e]) + @test_util.run_deprecated_v1 def testMismatchedRankExpandNonconcatDim(self): with self.session(use_gpu=False): sp_a = self._SparseTensor_3x3() @@ -275,6 +278,7 @@ class SparseConcatTest(test.TestCase): sparse_ops.sparse_concat( concat_dim, [sp_a, sp_e], expand_nonconcat_dim=True) + @test_util.run_deprecated_v1 def testMismatchedShapes(self): with self.session(use_gpu=False) as sess: sp_a = self._SparseTensor_3x3() @@ -287,7 +291,7 @@ class SparseConcatTest(test.TestCase): # Shape mismatches can only be caught when the op is run with self.assertRaisesOpError("Input shapes must match"): - sess.run(sp_concat) + self.evaluate(sp_concat) def testMismatchedShapesExpandNonconcatDim(self): with self.session(use_gpu=False) as sess: @@ -302,8 +306,8 @@ class SparseConcatTest(test.TestCase): sp_concat_dim1 = sparse_ops.sparse_concat( concat_dim1, [sp_a, sp_b, sp_c, sp_d], expand_nonconcat_dim=True) - sp_concat_dim0_out = sess.run(sp_concat_dim0) - sp_concat_dim1_out = sess.run(sp_concat_dim1) + sp_concat_dim0_out = self.evaluate(sp_concat_dim0) + sp_concat_dim1_out = self.evaluate(sp_concat_dim1) self.assertAllEqual(sp_concat_dim0_out.indices, [[0, 2], [1, 0], [2, 0], [2, 2], [4, 1], [5, 0], @@ -321,6 +325,7 @@ class SparseConcatTest(test.TestCase): [1, 1, 2, 1, 1, 1, 2, 3, 4, 2, 1, 0, 2]) self.assertAllEqual(sp_concat_dim1_out.dense_shape, [3, 13]) + @test_util.run_deprecated_v1 def testShapeInferenceUnknownShapes(self): with self.session(use_gpu=False): sp_inputs = [ diff --git a/tensorflow/python/kernel_tests/sparse_conditional_accumulator_test.py b/tensorflow/python/kernel_tests/sparse_conditional_accumulator_test.py index a824d5c8263..275c86e5349 100644 --- a/tensorflow/python/kernel_tests/sparse_conditional_accumulator_test.py +++ b/tensorflow/python/kernel_tests/sparse_conditional_accumulator_test.py @@ -26,6 +26,7 @@ from tensorflow.python.framework import dtypes as dtypes_lib from tensorflow.python.framework import errors_impl from tensorflow.python.framework import ops from tensorflow.python.framework import tensor_shape +from tensorflow.python.framework import test_util from tensorflow.python.ops import array_ops from tensorflow.python.ops import data_flow_ops from tensorflow.python.platform import test @@ -98,12 +99,14 @@ class IndexedSlicesConditionalAccumulatorTest(test.TestCase): attr { key: 'reduction_type' value {s: 'MEAN'} } """, q.accumulator_ref.op.node_def) + @test_util.run_deprecated_v1 def testAccumulatorSizeEmpty(self): with self.cached_session(): q = data_flow_ops.SparseConditionalAccumulator( dtypes_lib.float32, name="Q") self.assertEqual(q.num_accumulated().eval(), 0) + @test_util.run_deprecated_v1 def testAccumulatorSetGlobalStep(self): with self.cached_session(): q = data_flow_ops.SparseConditionalAccumulator( @@ -111,6 +114,7 @@ class IndexedSlicesConditionalAccumulatorTest(test.TestCase): set_global_step_op = q.set_global_step(1) set_global_step_op.run() + @test_util.run_deprecated_v1 def testAccumulatorApplyGradFloat32(self): with self.cached_session(): q = data_flow_ops.SparseConditionalAccumulator( @@ -122,6 +126,7 @@ class IndexedSlicesConditionalAccumulatorTest(test.TestCase): accum_op.run() self.assertEqual(q.num_accumulated().eval(), 1) + @test_util.run_deprecated_v1 def testDtypes(self): with self.cached_session() as sess: dtypes = [dtypes_lib.float16, dtypes_lib.float32, dtypes_lib.float64] @@ -140,10 +145,11 @@ class IndexedSlicesConditionalAccumulatorTest(test.TestCase): t = _indexedslice(mat_to_add) q.apply_indexed_slices_grad(t).run() - result = sess.run(q.take_indexed_slices_grad(1)) + result = self.evaluate(q.take_indexed_slices_grad(1)) self._assertEqual_nparray(sum_elems / len(elems), result, sess) + @test_util.run_deprecated_v1 def testAccumulatorMultipleAccumulators(self): with self.cached_session() as sess: q_f32_0 = data_flow_ops.SparseConditionalAccumulator( @@ -174,6 +180,7 @@ class IndexedSlicesConditionalAccumulatorTest(test.TestCase): result = sess.run(accums[i].take_indexed_slices_grad(1)) self._assertEqual_indexedslices(expected_tensors[i], result) + @test_util.run_deprecated_v1 def testAccumulatorTakeGradMean(self): with self.cached_session() as sess: q = data_flow_ops.SparseConditionalAccumulator( @@ -189,11 +196,12 @@ class IndexedSlicesConditionalAccumulatorTest(test.TestCase): accum_op.run() takeg_t = q.take_indexed_slices_grad(1) - val = sess.run(takeg_t) + val = self.evaluate(takeg_t) self.assertAllEqual([0, 1, 2], val.indices) self.assertAllEqual([[0.5, 0.5], [0, 2], [3, 0]], val.values) self.assertAllEqual([-1, 2], val.dense_shape) + @test_util.run_deprecated_v1 def testAccumulatorTakeGradSum(self): with self.cached_session() as sess: q = data_flow_ops.SparseConditionalAccumulator( @@ -209,16 +217,18 @@ class IndexedSlicesConditionalAccumulatorTest(test.TestCase): accum_op.run() takeg_t = q.take_indexed_slices_grad(1) - val = sess.run(takeg_t) + val = self.evaluate(takeg_t) self.assertAllEqual([0, 1, 2], val.indices) self.assertAllEqual([[1, 1], [0, 2], [3, 0]], val.values) self.assertAllEqual([-1, 2], val.dense_shape) + @test_util.run_deprecated_v1 def testAccumulatorTakeGradInvalidReductionType(self): with self.assertRaises(ValueError): data_flow_ops.SparseConditionalAccumulator( dtypes_lib.float32, name="Q", shape=(), reduction_type="Invalid") + @test_util.run_deprecated_v1 def testAccumulatorRepeatedTakeGrad(self): with self.cached_session() as sess: q = data_flow_ops.SparseConditionalAccumulator( @@ -235,7 +245,7 @@ class IndexedSlicesConditionalAccumulatorTest(test.TestCase): accum_op.run() takeg_t = q.take_indexed_slices_grad(1) - val = sess.run(takeg_t) + val = self.evaluate(takeg_t) self.assertAllEqual(val.indices, [0, 1, 2]) self.assertAllEqual(val.values, [[0.5, 0.5], [0, 2], [3, 0]]) self.assertAllEqual(val.dense_shape, [-1, 2]) @@ -252,11 +262,12 @@ class IndexedSlicesConditionalAccumulatorTest(test.TestCase): accum_op.run() takeg_t = q.take_indexed_slices_grad(1) - val = sess.run(takeg_t) + val = self.evaluate(takeg_t) self.assertAllEqual(val.indices, [0, 1, 2]) self.assertAllEqual(val.values, [[5, 5], [0, 20], [30, 0]]) self.assertAllEqual(val.dense_shape, [-1, 2]) + @test_util.run_deprecated_v1 def testParallelApplyGradMean(self): with self.cached_session() as sess: q = data_flow_ops.SparseConditionalAccumulator( @@ -269,7 +280,7 @@ class IndexedSlicesConditionalAccumulatorTest(test.TestCase): takeg_t = q.take_indexed_slices_grad(1) def apply_indexed_slices_grad(accum_op): - sess.run(accum_op) + self.evaluate(accum_op) threads = [ self.checkedThread( @@ -281,13 +292,14 @@ class IndexedSlicesConditionalAccumulatorTest(test.TestCase): for thread in threads: thread.join() - val = sess.run(takeg_t) + val = self.evaluate(takeg_t) expected_val = sum(elems) / len(elems) self._assertEqual_nparray( np.array([[expected_val, 0], [0, expected_val]]).astype(np.float32), val, sess) + @test_util.run_deprecated_v1 def testParallelApplyGradSum(self): with self.cached_session() as sess: q = data_flow_ops.SparseConditionalAccumulator( @@ -303,7 +315,7 @@ class IndexedSlicesConditionalAccumulatorTest(test.TestCase): takeg_t = q.take_indexed_slices_grad(1) def apply_indexed_slices_grad(accum_op): - sess.run(accum_op) + self.evaluate(accum_op) threads = [ self.checkedThread(target=apply_indexed_slices_grad, args=(o,)) @@ -315,13 +327,14 @@ class IndexedSlicesConditionalAccumulatorTest(test.TestCase): for thread in threads: thread.join() - val = sess.run(takeg_t) + val = self.evaluate(takeg_t) expected_val = 550.0 self._assertEqual_nparray( np.array([[expected_val, 0], [0, expected_val]]).astype(np.float32), val, sess) + @test_util.run_deprecated_v1 def testParallelTakeGrad(self): with self.cached_session() as sess: q = data_flow_ops.SparseConditionalAccumulator( @@ -338,13 +351,13 @@ class IndexedSlicesConditionalAccumulatorTest(test.TestCase): def apply_indexed_slices_grad(): for accum_op in accum_ops: time.sleep(1.0) - sess.run(accum_op) + self.evaluate(accum_op) apply_indexed_slices_grad_thread = self.checkedThread( target=apply_indexed_slices_grad) def take_grad(): - t = sess.run(takeg_t) + t = self.evaluate(takeg_t) results.append(t) threads = [self.checkedThread(target=take_grad) for _ in range(10)] @@ -361,6 +374,7 @@ class IndexedSlicesConditionalAccumulatorTest(test.TestCase): self._assertEqual_nparray( np.array([[0, 0], [elems[i], 0]]), results[i], sess) + @test_util.run_deprecated_v1 def testAccumulatorApplyAndBlockingTake(self): with self.cached_session() as sess: q = data_flow_ops.SparseConditionalAccumulator( @@ -378,10 +392,10 @@ class IndexedSlicesConditionalAccumulatorTest(test.TestCase): def apply_indexed_slices_grad(): for accum_op in accum_ops: - sess.run(accum_op) + self.evaluate(accum_op) def take_grad(): - results.append(sess.run(takeg_t)) + results.append(self.evaluate(takeg_t)) accum_thread = self.checkedThread(target=apply_indexed_slices_grad) takeg_thread = self.checkedThread(target=take_grad) @@ -394,8 +408,9 @@ class IndexedSlicesConditionalAccumulatorTest(test.TestCase): def _blocking_takeg(self, sess, takeg_op): with self.assertRaisesOpError("was cancelled"): - sess.run(takeg_op) + self.evaluate(takeg_op) + @test_util.run_deprecated_v1 def testAccumulatorCancel(self): with self.cached_session() as sess: q = data_flow_ops.SparseConditionalAccumulator( @@ -415,6 +430,7 @@ class IndexedSlicesConditionalAccumulatorTest(test.TestCase): takeg_thread.join() + @test_util.run_deprecated_v1 def testNonVectorIndices(self): with self.cached_session(): q = data_flow_ops.SparseConditionalAccumulator( @@ -427,6 +443,7 @@ class IndexedSlicesConditionalAccumulatorTest(test.TestCase): grad_indices=[[0, 1], [1, 0]], grad_values=np.array([1, 2]).astype(np.float32)).run() + @test_util.run_deprecated_v1 def testZeroDimensionValues(self): with self.cached_session(): q = data_flow_ops.SparseConditionalAccumulator( @@ -437,6 +454,7 @@ class IndexedSlicesConditionalAccumulatorTest(test.TestCase): q.apply_grad( grad_indices=[0], grad_values=np.array(1).astype(np.float32)).run() + @test_util.run_deprecated_v1 def testWrongNonEmptyInputValues(self): with self.cached_session(): q = data_flow_ops.SparseConditionalAccumulator( @@ -448,6 +466,7 @@ class IndexedSlicesConditionalAccumulatorTest(test.TestCase): grad_indices=[0, 1], grad_values=np.array([[0, 1, 1]]).astype(np.float32)).run() + @test_util.run_deprecated_v1 def testDynamicNonVectorIndices(self): with self.cached_session() as sess: q = data_flow_ops.SparseConditionalAccumulator( @@ -467,6 +486,7 @@ class IndexedSlicesConditionalAccumulatorTest(test.TestCase): x_values: np.array([1, 2]).astype(np.float32) }) + @test_util.run_deprecated_v1 def testDynamicWrongNonEmptyInputValues(self): with self.cached_session() as sess: q = data_flow_ops.SparseConditionalAccumulator( @@ -485,6 +505,7 @@ class IndexedSlicesConditionalAccumulatorTest(test.TestCase): x_values: np.array([[0, 1, 1]]).astype(np.float32) }) + @test_util.run_deprecated_v1 def testEmptyShapeApply(self): with self.cached_session(): q = data_flow_ops.SparseConditionalAccumulator( @@ -510,6 +531,7 @@ class IndexedSlicesConditionalAccumulatorTest(test.TestCase): q.apply_grad(grad_indices=[0], grad_values=[1.0], grad_shape=[]).run() q.apply_grad(grad_indices=[0], grad_values=[1.0]).run() + @test_util.run_deprecated_v1 def testValidateShape(self): with self.cached_session() as sess: q = data_flow_ops.SparseConditionalAccumulator( @@ -585,7 +607,7 @@ class IndexedSlicesConditionalAccumulatorTest(test.TestCase): np.float32)).run() # After take grad, constraints on accumulated gradient are removed - sess.run(q.take_grad(1)) + self.evaluate(q.take_grad(1)) # First successful gradient imposes new constraints. # Hereafter, shape will additionally constrained to [None,2,2,3] @@ -605,6 +627,7 @@ class IndexedSlicesConditionalAccumulatorTest(test.TestCase): [[[[1, 2], [3, 4]], [[5, 6], [7, 8]]]]).astype(np.float32), local_step=1).run() + @test_util.run_deprecated_v1 def testReturnShape(self): with self.cached_session() as sess: q = data_flow_ops.SparseConditionalAccumulator( @@ -615,7 +638,7 @@ class IndexedSlicesConditionalAccumulatorTest(test.TestCase): grad_values=np.array( [[[[1, 2], [3, 4]], [[5, 6], [7, 8]]]]).astype(np.float32)).run() - val = sess.run(q.take_indexed_slices_grad(1)) + val = self.evaluate(q.take_indexed_slices_grad(1)) self.assertAllEqual(val.dense_shape, [2, 2, 2, 2]) q = data_flow_ops.SparseConditionalAccumulator( @@ -627,9 +650,10 @@ class IndexedSlicesConditionalAccumulatorTest(test.TestCase): [[[[1, 2, 3], [4, 5, 6]], [[7, 8, 9], [10, 11, 12]]]]).astype( np.float32)).run() - val = sess.run(q.take_indexed_slices_grad(1)) + val = self.evaluate(q.take_indexed_slices_grad(1)) self.assertAllEqual(val.dense_shape, [-1, 2, 2, 3]) + @test_util.run_deprecated_v1 def testApplyGradtInt32IndicesAndShape(self): with self.cached_session() as sess: q = data_flow_ops.SparseConditionalAccumulator( @@ -653,7 +677,7 @@ class IndexedSlicesConditionalAccumulatorTest(test.TestCase): accum_op.run() self.assertEqual(q.num_accumulated().eval(), 2) - val = sess.run(q.take_indexed_slices_grad(1)) + val = self.evaluate(q.take_indexed_slices_grad(1)) self.assertAllEqual(val.indices, [0, 2]) self.assertAllEqual(val.values, [[0, 0, 1], [3, 0, 4]]) self.assertAllEqual(val.dense_shape, [3, 3]) diff --git a/tensorflow/python/kernel_tests/sparse_cross_op_test.py b/tensorflow/python/kernel_tests/sparse_cross_op_test.py index 6e0714da702..566bbb56f00 100644 --- a/tensorflow/python/kernel_tests/sparse_cross_op_test.py +++ b/tensorflow/python/kernel_tests/sparse_cross_op_test.py @@ -24,12 +24,14 @@ from tensorflow.python.client import session from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes from tensorflow.python.framework import sparse_tensor +from tensorflow.python.framework import test_util from tensorflow.python.ops import sparse_ops from tensorflow.python.platform import test class SparseCrossOpTest(test.TestCase): + @test_util.run_deprecated_v1 def test_simple(self): """Tests a simple scenario.""" op = sparse_ops.sparse_cross([ @@ -43,8 +45,9 @@ class SparseCrossOpTest(test.TestCase): 'batch2-FC1-F2_X_batch2-FC2-F1', 'batch2-FC1-F2_X_batch2-FC2-F2' ]]) with self.cached_session() as sess: - self._assert_sparse_tensor_equals(expected_out, sess.run(op)) + self._assert_sparse_tensor_equals(expected_out, self.evaluate(op)) + @test_util.run_deprecated_v1 def test_dense(self): """Tests only dense inputs.""" op = sparse_ops.sparse_cross([ @@ -63,8 +66,9 @@ class SparseCrossOpTest(test.TestCase): 'batch2-FC1-F2_X_batch2-FC2-F1', 'batch2-FC1-F2_X_batch2-FC2-F2' ]]) with self.cached_session() as sess: - self._assert_sparse_tensor_equals(expected_out, sess.run(op)) + self._assert_sparse_tensor_equals(expected_out, self.evaluate(op)) + @test_util.run_deprecated_v1 def test_integer_mixed_string_sparse(self): """Tests mixed type.""" op = sparse_ops.sparse_cross([ @@ -77,8 +81,9 @@ class SparseCrossOpTest(test.TestCase): '55555_X_batch2-FC2-F2' ]]) with self.cached_session() as sess: - self._assert_sparse_tensor_equals(expected_out, sess.run(op)) + self._assert_sparse_tensor_equals(expected_out, self.evaluate(op)) + @test_util.run_deprecated_v1 def test_integer_mixed_string_dense(self): """Tests mixed dense inputs.""" op = sparse_ops.sparse_cross([ @@ -95,8 +100,9 @@ class SparseCrossOpTest(test.TestCase): '999999_X_batch2-FC2-F1', '999999_X_batch2-FC2-F2' ]]) with self.cached_session() as sess: - self._assert_sparse_tensor_equals(expected_out, sess.run(op)) + self._assert_sparse_tensor_equals(expected_out, self.evaluate(op)) + @test_util.run_deprecated_v1 def test_sparse_cross_dense(self): """Tests sparse and dense inputs.""" op = sparse_ops.sparse_cross([ @@ -112,8 +118,9 @@ class SparseCrossOpTest(test.TestCase): 'batch2-FC1-F2_X_batch2-FC2-F1', 'batch2-FC1-F2_X_batch2-FC2-F2' ]]) with self.cached_session() as sess: - self._assert_sparse_tensor_equals(expected_out, sess.run(op)) + self._assert_sparse_tensor_equals(expected_out, self.evaluate(op)) + @test_util.run_deprecated_v1 def test_integer_sparse_input(self): """Tests mixed type sparse and dense inputs.""" op = sparse_ops.sparse_cross([ @@ -128,8 +135,9 @@ class SparseCrossOpTest(test.TestCase): '5555_X_batch2-FC2-F1', '5555_X_batch2-FC2-F2' ]]) with self.cached_session() as sess: - self._assert_sparse_tensor_equals(expected_out, sess.run(op)) + self._assert_sparse_tensor_equals(expected_out, self.evaluate(op)) + @test_util.run_deprecated_v1 def test_permutation_3x3x3(self): """Tests 3x3x3 permutation.""" op = sparse_ops.sparse_cross([ @@ -170,8 +178,9 @@ class SparseCrossOpTest(test.TestCase): 'batch1-FC1-F3_X_batch1-FC2-F3_X_batch1-FC3-F3' ]]) with self.cached_session() as sess: - self._assert_sparse_tensor_equals(expected_out, sess.run(op)) + self._assert_sparse_tensor_equals(expected_out, self.evaluate(op)) + @test_util.run_deprecated_v1 def test_permutation_3x1x2(self): """Tests 3x1x2 permutation.""" op = sparse_ops.sparse_cross([ @@ -189,8 +198,9 @@ class SparseCrossOpTest(test.TestCase): 'batch1-FC1-F3_X_batch1-FC2-F1_X_batch1-FC3-F2' ]]) with self.cached_session() as sess: - self._assert_sparse_tensor_equals(expected_out, sess.run(op)) + self._assert_sparse_tensor_equals(expected_out, self.evaluate(op)) + @test_util.run_deprecated_v1 def test_large_batch(self): """Tests with large batch size to force multithreading.""" batch_size = 5000 @@ -222,8 +232,9 @@ class SparseCrossOpTest(test.TestCase): expected_out = self._sparse_tensor(col_out) with self.cached_session() as sess: - self._assert_sparse_tensor_equals(expected_out, sess.run(op)) + self._assert_sparse_tensor_equals(expected_out, self.evaluate(op)) + @test_util.run_deprecated_v1 def test_one_column_empty(self): """Tests when one column is empty. @@ -235,8 +246,9 @@ class SparseCrossOpTest(test.TestCase): self._sparse_tensor([['batch1-FC3-F1', 'batch1-FC3-F2']]) ]) with self.cached_session() as sess: - self._assert_sparse_tensor_empty(sess.run(op)) + self._assert_sparse_tensor_empty(self.evaluate(op)) + @test_util.run_deprecated_v1 def test_some_columns_empty(self): """Tests when more than one columns are empty. @@ -254,8 +266,9 @@ class SparseCrossOpTest(test.TestCase): 'batch1-FC1-F2_X_batch1-FC2-F1_X_batch1-FC3-F2' ]], 2) with self.cached_session() as sess: - self._assert_sparse_tensor_equals(expected_out, sess.run(op)) + self._assert_sparse_tensor_equals(expected_out, self.evaluate(op)) + @test_util.run_deprecated_v1 def test_all_columns_empty(self): """Tests when all columns are empty. @@ -267,8 +280,9 @@ class SparseCrossOpTest(test.TestCase): self._sparse_tensor([]) ]) with self.cached_session() as sess: - self._assert_sparse_tensor_empty(sess.run(op)) + self._assert_sparse_tensor_empty(self.evaluate(op)) + @test_util.run_deprecated_v1 def test_hashed_zero_bucket_no_hash_key(self): op = sparse_ops.sparse_cross_hashed([ self._sparse_tensor([['batch1-FC1-F1']]), @@ -278,8 +292,9 @@ class SparseCrossOpTest(test.TestCase): # Check actual hashed output to prevent unintentional hashing changes. expected_out = self._sparse_tensor([[1971693436396284976]]) with self.cached_session() as sess: - self._assert_sparse_tensor_equals(expected_out, sess.run(op)) + self._assert_sparse_tensor_equals(expected_out, self.evaluate(op)) + @test_util.run_deprecated_v1 def test_hashed_zero_bucket(self): op = sparse_ops.sparse_cross_hashed( [ @@ -291,9 +306,10 @@ class SparseCrossOpTest(test.TestCase): # Check actual hashed output to prevent unintentional hashing changes. expected_out = self._sparse_tensor([[4847552627144134031]]) with self.cached_session() as sess: - self._assert_sparse_tensor_equals(expected_out, sess.run(op)) + self._assert_sparse_tensor_equals(expected_out, self.evaluate(op)) # TODO(sibyl-Aix6ihai): Add benchmark to compare Hashed vs Non-hashed. + @test_util.run_deprecated_v1 def test_hashed_no_hash_key(self): op = sparse_ops.sparse_cross_hashed( [ @@ -305,8 +321,9 @@ class SparseCrossOpTest(test.TestCase): # Check actual hashed output to prevent unintentional hashing changes. expected_out = self._sparse_tensor([[83]]) with self.cached_session() as sess: - self._assert_sparse_tensor_equals(expected_out, sess.run(op)) + self._assert_sparse_tensor_equals(expected_out, self.evaluate(op)) + @test_util.run_deprecated_v1 def test_hashed_output(self): op = sparse_ops.sparse_cross_hashed( [ @@ -319,8 +336,9 @@ class SparseCrossOpTest(test.TestCase): # Check actual hashed output to prevent unintentional hashing changes. expected_out = self._sparse_tensor([[31]]) with self.cached_session() as sess: - self._assert_sparse_tensor_equals(expected_out, sess.run(op)) + self._assert_sparse_tensor_equals(expected_out, self.evaluate(op)) + @test_util.run_deprecated_v1 def test_hashed__has_no_collision(self): """Tests that fingerprint concatenation has no collisions.""" # Although the last 10 bits of 359 and 1024+359 are identical. @@ -331,7 +349,7 @@ class SparseCrossOpTest(test.TestCase): [t2, t1], num_buckets=1024, hash_key=sparse_ops._DEFAULT_HASH_KEY + 1) cross_dense = sparse_ops.sparse_tensor_to_dense(cross) with session.Session(): - values = cross_dense.eval() + values = self.evaluate(cross_dense) self.assertTrue(numpy.not_equal(values[0], values[1]).all()) def test_hashed_3x1x2(self): @@ -345,7 +363,7 @@ class SparseCrossOpTest(test.TestCase): ], num_buckets=1000) with self.cached_session() as sess: - out = sess.run(op) + out = self.evaluate(op) self.assertEqual(6, len(out.values)) self.assertAllEqual([[0, i] for i in range(6)], out.indices) self.assertTrue(all(x < 1000 and x >= 0 for x in out.values)) diff --git a/tensorflow/python/kernel_tests/sparse_matmul_op_test.py b/tensorflow/python/kernel_tests/sparse_matmul_op_test.py index 541463e76bb..2e17a9c608f 100644 --- a/tensorflow/python/kernel_tests/sparse_matmul_op_test.py +++ b/tensorflow/python/kernel_tests/sparse_matmul_op_test.py @@ -22,6 +22,7 @@ import numpy as np from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes +from tensorflow.python.framework import test_util from tensorflow.python.ops import gradient_checker from tensorflow.python.ops import math_ops from tensorflow.python.platform import test @@ -58,7 +59,7 @@ class SparseMatMulTest(test.TestCase): transpose_b=tr_b, a_is_sparse=sp_a, b_is_sparse=sp_b) - out = tf_ans.eval() + out = self.evaluate(tf_ans) np_x = math_ops.cast(tf_x, dtypes.float32).eval() np_y = math_ops.cast(tf_y, dtypes.float32).eval() @@ -71,6 +72,7 @@ class SparseMatMulTest(test.TestCase): self.assertShapeEqual(np_ans, tf_ans) self.assertAllCloseAccordingToType(np_ans, out, rtol=1e-4, atol=1e-4) + @test_util.run_deprecated_v1 def testBasic(self): x = np.arange(0., 4.).reshape([4, 1]).astype(np.float32) y = np.arange(-1., 1.).reshape([1, 2]).astype(np.float32) @@ -78,6 +80,7 @@ class SparseMatMulTest(test.TestCase): for y_dtype in (dtypes.float32, dtypes.bfloat16): self._testCpuMatmul(x, y, x_dtype=x_dtype, y_dtype=y_dtype) + @test_util.run_deprecated_v1 def testZeroDim(self): x = np.ones((4, 0)).astype(np.float32) y = np.ones((0, 3)).astype(np.float32) @@ -85,6 +88,7 @@ class SparseMatMulTest(test.TestCase): for y_dtype in (dtypes.float32, dtypes.bfloat16): self._testCpuMatmul(x, y, x_dtype=x_dtype, y_dtype=y_dtype) + @test_util.run_deprecated_v1 def testEmpty(self): x = np.ones((0, 0)).astype(np.float32) y = np.ones((0, 0)).astype(np.float32) @@ -93,6 +97,7 @@ class SparseMatMulTest(test.TestCase): self._testCpuMatmul(x, y, x_dtype=x_dtype, y_dtype=y_dtype) # Tests setting one dimension to be a high value. + @test_util.run_deprecated_v1 def testLarge(self): r1 = np.random.randint(6000, 20000) r2 = np.random.randint(1, 10) @@ -105,6 +110,7 @@ class SparseMatMulTest(test.TestCase): self._testCpuMatmul(x, y, x_dtype=x_dtype, y_dtype=y_dtype) # Tests random sized matrices. + @test_util.run_deprecated_v1 def testRandom(self): for tr_a in [True, False]: for tr_b in [True, False]: @@ -159,6 +165,7 @@ class MatMulGradientTest(test.TestCase): delta=delta)) self.assertLessEqual(err, delta / 2.) + @test_util.run_deprecated_v1 def testGradientInput(self): for tr_a in [True, False]: for tr_b in [True, False]: diff --git a/tensorflow/python/kernel_tests/sparse_ops_test.py b/tensorflow/python/kernel_tests/sparse_ops_test.py index a45ce2e13b4..75f65e62517 100644 --- a/tensorflow/python/kernel_tests/sparse_ops_test.py +++ b/tensorflow/python/kernel_tests/sparse_ops_test.py @@ -71,6 +71,7 @@ class SparseToIndicatorTest(test_util.TensorFlowTestCase): constant_op.constant(val, dtype), constant_op.constant(shape, dtypes.int64)) + @test_util.run_deprecated_v1 def testInt32(self): with self.session(use_gpu=False): sp_input = self._SparseTensor_5x6(dtypes.int32) @@ -83,6 +84,7 @@ class SparseToIndicatorTest(test_util.TensorFlowTestCase): self.assertAllEqual(output, expected_output) + @test_util.run_deprecated_v1 def testInt64(self): with self.session(use_gpu=False): sp_input = self._SparseTensor_5x6(dtypes.int64) @@ -95,6 +97,7 @@ class SparseToIndicatorTest(test_util.TensorFlowTestCase): self.assertAllEqual(output, expected_output) + @test_util.run_deprecated_v1 def testHigherRank(self): with self.session(use_gpu=False): sp_input = self._SparseTensor_2x3x4(dtypes.int64) @@ -154,7 +157,7 @@ class SparseMergeTest(test_util.TensorFlowTestCase): sparse_tensor.SparseTensor.from_value(values_v)): sp_output = sparse_ops.sparse_merge(indices, values, vocab_size) - output = sess.run(sp_output) + output = self.evaluate(sp_output) self._AssertResultsSorted(output, vocab_size) def testInt64AndFloat32(self): @@ -163,7 +166,7 @@ class SparseMergeTest(test_util.TensorFlowTestCase): indices, values = self._SparseTensor_3x50(np.int64, np.float32) sp_output = sparse_ops.sparse_merge(indices, values, vocab_size) - output = sess.run(sp_output) + output = self.evaluate(sp_output) self._AssertResultsSorted(output, vocab_size) def testInt64AndFloat64(self): @@ -172,7 +175,7 @@ class SparseMergeTest(test_util.TensorFlowTestCase): indices, values = self._SparseTensor_3x50(np.int64, np.float64) sp_output = sparse_ops.sparse_merge(indices, values, vocab_size) - output = sess.run(sp_output) + output = self.evaluate(sp_output) self._AssertResultsSorted(output, vocab_size) def testInt32AndFloat32NonCanonicalOrder(self): @@ -182,7 +185,7 @@ class SparseMergeTest(test_util.TensorFlowTestCase): sp_output = sparse_ops.sparse_merge( indices, values, vocab_size, already_sorted=True) - output = sess.run(sp_output) + output = self.evaluate(sp_output) self._AssertResultsNotSorted(output, vocab_size) def testInt64AndFloat32NonCanonicalOrder(self): @@ -192,7 +195,7 @@ class SparseMergeTest(test_util.TensorFlowTestCase): sp_output = sparse_ops.sparse_merge( indices, values, vocab_size, already_sorted=True) - output = sess.run(sp_output) + output = self.evaluate(sp_output) self._AssertResultsNotSorted(output, vocab_size) def testInt64AndFloat64NonCanonicalOrder(self): @@ -203,7 +206,7 @@ class SparseMergeTest(test_util.TensorFlowTestCase): sp_output = sparse_ops.sparse_merge( indices, values, vocab_size_tensor, already_sorted=True) - output = sess.run(sp_output) + output = self.evaluate(sp_output) self._AssertResultsNotSorted(output, vocab_size) def testShouldSetLastDimensionInDynamicShape(self): @@ -261,7 +264,7 @@ class SparseMergeHighDimTest(test_util.TensorFlowTestCase): indices, values = self._SparseTensor_3x50(np.int64, np.float32) sp_output = sparse_ops.sparse_merge(indices, values, vocab_size) - output = sess.run(sp_output) + output = self.evaluate(sp_output) self._AssertResultsSorted(output, vocab_size) def testInt64AndFloat64(self): @@ -270,7 +273,7 @@ class SparseMergeHighDimTest(test_util.TensorFlowTestCase): indices, values = self._SparseTensor_3x50(np.int64, np.float64) sp_output = sparse_ops.sparse_merge(indices, values, vocab_size) - output = sess.run(sp_output) + output = self.evaluate(sp_output) self._AssertResultsSorted(output, vocab_size) def testInt64AndFloat64Shape(self): @@ -279,7 +282,7 @@ class SparseMergeHighDimTest(test_util.TensorFlowTestCase): indices, values = self._SparseTensor_3x50(np.int64, np.float64) sp_output = sparse_ops.sparse_merge(indices, values, vocab_size) - output = sess.run(sp_output) + output = self.evaluate(sp_output) self._AssertResultsSorted(output, vocab_size) @@ -296,13 +299,14 @@ class SparseRetainTest(test_util.TensorFlowTestCase): def _SparseTensor_5x6(self): return sparse_tensor.SparseTensor.from_value(self._SparseTensorValue_5x6()) + @test_util.run_deprecated_v1 def testBasic(self): with self.session(use_gpu=False) as sess: for sp_input in (self._SparseTensorValue_5x6(), self._SparseTensor_5x6()): to_retain = np.array([1, 0, 0, 1, 1, 0], dtype=np.bool) sp_output = sparse_ops.sparse_retain(sp_input, to_retain) - output = sess.run(sp_output) + output = self.evaluate(sp_output) self.assertAllEqual(output.indices, [[0, 0], [1, 4], [3, 2]]) self.assertAllEqual(output.values, [0, 14, 32]) @@ -314,7 +318,7 @@ class SparseRetainTest(test_util.TensorFlowTestCase): to_retain = np.zeros((6,), dtype=np.bool) sp_output = sparse_ops.sparse_retain(sp_input, to_retain) - output = sess.run(sp_output) + output = self.evaluate(sp_output) self.assertAllEqual(output.indices, np.array([]).reshape((0, 2))) self.assertAllEqual(output.values, []) @@ -353,38 +357,42 @@ class SparseResetShapeTest(test_util.TensorFlowTestCase): return sparse_tensor.SparseTensorValue(self._IND_2_5_6, self._VAL_2_5_6, self._SHP_2_5_6) + @test_util.run_deprecated_v1 def testStaticShapeInfoPreservedWhenNewShapeIsProvidedAndStatic(self): sp_input = self._SparseTensor_2x5x6() new_shape = np.array([3, 6, 7], dtype=np.int64) sp_output = sparse_ops.sparse_reset_shape(sp_input, new_shape) self.assertAllEqual([3, 6, 7], sp_output.get_shape()) + @test_util.run_deprecated_v1 def testBasic(self): with self.session(use_gpu=False) as sess: sp_input = self._SparseTensor_2x5x6() new_shape = np.array([3, 6, 7], dtype=np.int64) sp_output = sparse_ops.sparse_reset_shape(sp_input, new_shape) - output = sess.run(sp_output) + output = self.evaluate(sp_output) self.assertAllEqual(output.indices, [[0, 0, 0], [0, 1, 0], [0, 1, 3], [1, 1, 4], [1, 3, 2], [1, 3, 3]]) self.assertAllEqual(output.values, [0, 10, 13, 14, 32, 33]) self.assertAllEqual(output.dense_shape, [3, 6, 7]) + @test_util.run_deprecated_v1 def testInputUnavailableInGraphConstructionOk(self): with self.session(use_gpu=False) as sess: sp_input = self._SparseTensorValue_2x5x6() new_shape = np.array([3, 6, 7], dtype=np.int64) sp_output = sparse_ops.sparse_reset_shape(sp_input, new_shape) - output = sess.run(sp_output) + output = self.evaluate(sp_output) self.assertAllEqual(output.indices, [[0, 0, 0], [0, 1, 0], [0, 1, 3], [1, 1, 4], [1, 3, 2], [1, 3, 3]]) self.assertAllEqual(output.values, [0, 10, 13, 14, 32, 33]) self.assertAllEqual(output.dense_shape, [3, 6, 7]) + @test_util.run_deprecated_v1 def testFeedInputUnavailableInGraphConstructionOk(self): with self.session(use_gpu=False) as sess: sp_input = array_ops.sparse_placeholder(dtype=dtypes.int32) @@ -404,7 +412,7 @@ class SparseResetShapeTest(test_util.TensorFlowTestCase): sp_input = self._SparseTensor_2x5x6() sp_output = sparse_ops.sparse_reset_shape(sp_input) - output = sess.run(sp_output) + output = self.evaluate(sp_output) self.assertAllEqual(output.indices, [[0, 0, 0], [0, 1, 0], [0, 1, 3], [1, 1, 4], [1, 3, 2], [1, 3, 3]]) @@ -416,12 +424,13 @@ class SparseResetShapeTest(test_util.TensorFlowTestCase): sp_input = self._SparseTensor_2x5x6_Empty() sp_output = sparse_ops.sparse_reset_shape(sp_input) - output = sess.run(sp_output) + output = self.evaluate(sp_output) self.assertAllEqual(output.indices.shape, [0, 3]) self.assertAllEqual(output.values.shape, [0]) self.assertAllEqual(output.dense_shape, [0, 0, 0]) + @test_util.run_deprecated_v1 def testInvalidRank(self): with self.session(use_gpu=False): sp_input = self._SparseTensor_2x5x6() @@ -430,6 +439,7 @@ class SparseResetShapeTest(test_util.TensorFlowTestCase): with self.assertRaises(ValueError): sparse_ops.sparse_reset_shape(sp_input, new_shape) + @test_util.run_deprecated_v1 def testInvalidRankNewShapeUnavailableInGraphConstruction(self): with self.session(use_gpu=False) as sess: new_shape = array_ops.placeholder(dtype=dtypes.int64) @@ -439,6 +449,7 @@ class SparseResetShapeTest(test_util.TensorFlowTestCase): with self.assertRaisesOpError("x == y did not hold element-wise"): sess.run(out, feed_dict={new_shape: np.array([3, 7], dtype=np.int64)}) + @test_util.run_deprecated_v1 def testInvalidDimensionSizeStatic(self): sp_input = self._SparseTensor_2x5x6() new_shape = np.array([3, 7, 5], dtype=np.int64) @@ -446,6 +457,7 @@ class SparseResetShapeTest(test_util.TensorFlowTestCase): with self.assertRaisesRegexp(ValueError, "should have dimension sizes"): sparse_ops.sparse_reset_shape(sp_input, new_shape) + @test_util.run_deprecated_v1 def testInvalidDimensionSizeDynamic(self): with self.session(use_gpu=False) as sess: sp_input = self._SparseTensor_2x5x6() @@ -455,6 +467,7 @@ class SparseResetShapeTest(test_util.TensorFlowTestCase): with self.assertRaisesOpError("x <= y did not hold element-wise"): sess.run(out, feed_dict={new_shape: [3, 7, 5]}) + @test_util.run_deprecated_v1 def testInvalidDimensionSizeInputUnavailableInGraphConstruction(self): sp_input = array_ops.sparse_placeholder(dtype=dtypes.int32) with self.session(use_gpu=False) as sess: @@ -496,6 +509,7 @@ class SparseFillEmptyRowsTest(test_util.TensorFlowTestCase): constant_op.constant(val, dtypes.int32), constant_op.constant(shape, dtypes.int64)) + @test_util.run_deprecated_v1 def testFillNumber(self): with self.session(use_gpu=False) as sess: for sp_input in (self._SparseTensorValue_5x6(), self._SparseTensor_5x6()): @@ -513,6 +527,7 @@ class SparseFillEmptyRowsTest(test_util.TensorFlowTestCase): self.assertAllEqual(empty_row_indicator_out, np.array([0, 0, 1, 0, 1]).astype(np.bool)) + @test_util.run_deprecated_v1 def testFillFloat(self): with self.session(use_gpu=False) as sess: values = constant_op.constant( @@ -547,6 +562,7 @@ class SparseFillEmptyRowsTest(test_util.TensorFlowTestCase): self.assertGreater(default_value_grad_err, 0) self.assertLess(default_value_grad_err, 1e-8) + @test_util.run_deprecated_v1 def testFillString(self): with self.session(use_gpu=False) as sess: sp_input = self._SparseTensor_String5x6() @@ -565,6 +581,7 @@ class SparseFillEmptyRowsTest(test_util.TensorFlowTestCase): self.assertAllEqual(empty_row_indicator_out, np.array([0, 0, 1, 0, 1]).astype(np.bool)) + @test_util.run_deprecated_v1 def testNoEmptyRows(self): with self.session(use_gpu=False) as sess: sp_input = self._SparseTensor_2x6() @@ -582,6 +599,7 @@ class SparseFillEmptyRowsTest(test_util.TensorFlowTestCase): class SparseAddTest(test_util.TensorFlowTestCase): + @test_util.run_deprecated_v1 def testValuesInVariable(self): indices = constant_op.constant([[1]], dtype=dtypes.int64) values = variables.Variable([1], trainable=False, dtype=dtypes.float32) @@ -591,8 +609,8 @@ class SparseAddTest(test_util.TensorFlowTestCase): sp_output = sparse_ops.sparse_add(sp_input, sp_input) with self.session(use_gpu=False) as sess: - sess.run(variables.global_variables_initializer()) - output = sess.run(sp_output) + self.evaluate(variables.global_variables_initializer()) + output = self.evaluate(sp_output) self.assertAllEqual(output.values, [2]) @@ -635,7 +653,7 @@ class SparseReduceTest(test_util.TensorFlowTestCase): else: tf_dense_ans = sparse_ops.sparse_reduce_max(sp_t, reduction_axes, keep_dims) - out_dense = tf_dense_ans.eval() + out_dense = self.evaluate(tf_dense_ans) if do_sum: tf_sparse_ans = sparse_ops.sparse_reduce_sum_sparse(sp_t, @@ -657,6 +675,7 @@ class SparseReduceTest(test_util.TensorFlowTestCase): self._compare(sp_t, reduction_axes, ndims, True, False) self._compare(sp_t, reduction_axes, ndims, True, True) + @test_util.run_deprecated_v1 def testSimpleAndRandomInputs(self): if np.__version__ == "1.13.0": self.skipTest("numpy 1.13.0 bug") @@ -696,6 +715,7 @@ class SparseReduceTest(test_util.TensorFlowTestCase): with self.assertRaisesOpError("Invalid reduction dimension 2"): sparse_ops.sparse_reduce_max(sp_t, 2).eval() + @test_util.run_deprecated_v1 def testGradient(self): if np.__version__ == "1.13.0": self.skipTest("numpy 1.13.0 bug") @@ -710,18 +730,59 @@ class SparseReduceTest(test_util.TensorFlowTestCase): axes = np.random.choice(len(dims), size=d, replace=False).tolist() reduced = sparse_ops.sparse_reduce_sum(sp_t, axes) - err = gradient_checker.compute_gradient_error(sp_t.values, (nnz,), - reduced, - reduced.eval().shape) + err = gradient_checker.compute_gradient_error( + sp_t.values, (nnz,), reduced, + self.evaluate(reduced).shape) self.assertLess(err, 1e-3) # Tests for negative axes. reduced = sparse_ops.sparse_reduce_sum(sp_t, -1) - err = gradient_checker.compute_gradient_error(sp_t.values, (nnz,), - reduced, - reduced.eval().shape) + err = gradient_checker.compute_gradient_error( + sp_t.values, (nnz,), reduced, + self.evaluate(reduced).shape) self.assertLess(err, 1e-3) + def _testSparseReduceShape(self, sp_t, reduction_axes, ndims, keep_dims, + do_sum): + densified = sparse_ops.sparse_tensor_to_dense(sp_t).eval() + + np_op = np.sum + tf_op = sparse_ops.sparse_reduce_sum + if not do_sum: + np_op = np.max + tf_op = sparse_ops.sparse_reduce_max + + np_ans = densified + if reduction_axes is None: + np_ans = np_op(np_ans, keepdims=keep_dims) + else: + if not isinstance(reduction_axes, list): # Single scalar. + reduction_axes = [reduction_axes] + reduction_axes = np.array(reduction_axes).astype(np.int32) + # Handles negative axes. + reduction_axes = (reduction_axes + ndims) % ndims + # Loop below depends on sorted. + reduction_axes.sort() + for ra in reduction_axes.ravel()[::-1]: + np_ans = np_op(np_ans, axis=ra, keepdims=keep_dims) + + tf_ans = tf_op(sp_t, reduction_axes, keep_dims) + self.assertAllEqual(np_ans.shape, tf_ans.get_shape().as_list()) + + def testSparseReduceSumOrMaxShape(self): + sp_t = sparse_tensor.SparseTensor(self.ind, self.vals, self.dense_shape) + + with self.session(use_gpu=False): + for do_sum in [True, False]: + for keep_dims in [True, False]: + self._testSparseReduceShape(sp_t, None, 2, keep_dims, do_sum) + self._testSparseReduceShape(sp_t, 0, 2, keep_dims, do_sum) + self._testSparseReduceShape(sp_t, [1], 2, keep_dims, do_sum) + self._testSparseReduceShape(sp_t, [0, 1], 2, keep_dims, do_sum) + self._testSparseReduceShape(sp_t, [1, 0], 2, keep_dims, do_sum) + self._testSparseReduceShape(sp_t, [-1], 2, keep_dims, do_sum) + self._testSparseReduceShape(sp_t, [1, -2], 2, keep_dims, do_sum) + class SparseMathOpsTest(test_util.TensorFlowTestCase): @@ -737,6 +798,7 @@ class SparseMathOpsTest(test_util.TensorFlowTestCase): result_tensor.values).eval() self.assertAllEqual(result_np, res_densified) + @test_util.run_deprecated_v1 def testCwiseDivAndMul(self): np.random.seed(1618) sp_shapes = [(10, 10, 10), (5, 5), (1618,), (3, 3, 7)] @@ -760,6 +822,7 @@ class SparseMathOpsTest(test_util.TensorFlowTestCase): res = sp_t / dense_t # should invoke "__truediv__" self.assertEqual(res.values.eval().dtype, np.float64) + @test_util.run_deprecated_v1 def testCwiseAdd(self): with self.session(use_gpu=False): # Identity(2) + AllOnes(2,2). Should be equal to 2 * Identity(2). @@ -779,6 +842,7 @@ class SparseMathOpsTest(test_util.TensorFlowTestCase): sparse_ops.sparse_dense_cwise_add(sp_t, dense_t), np.identity(2) * 2, sp_t) + @test_util.run_deprecated_v1 def testGradients(self): np.random.seed(1618) sp_shapes = [(10, 10, 10), (5, 5), (1618,), (3, 3, 7)] @@ -812,6 +876,7 @@ class SparseMathOpsTest(test_util.TensorFlowTestCase): class SparseSoftmaxTest(test_util.TensorFlowTestCase): + @test_util.run_deprecated_v1 def testEquivalentToDensified(self): np.random.seed(1618) n, m = np.random.choice(20, size=2) @@ -831,6 +896,7 @@ class SparseSoftmaxTest(test_util.TensorFlowTestCase): self.assertAllClose(dense_result.eval(), sp_result) + @test_util.run_deprecated_v1 def testHigherRanks(self): # For the first shape: # First batch: @@ -860,6 +926,7 @@ class SparseSoftmaxTest(test_util.TensorFlowTestCase): self.assertAllEqual(sp_t.indices.eval(), result.indices) self.assertAllEqual(shape, result.dense_shape) + @test_util.run_deprecated_v1 def testGradient(self): x_shape = [2, 5, 10] with self.cached_session(use_gpu=False): @@ -879,6 +946,7 @@ class SparseMinimumMaximumTest(test_util.TensorFlowTestCase): self.assertAllEqual(a.values, b.values) self.assertAllEqual(a.dense_shape, b.dense_shape) + @test_util.run_deprecated_v1 def testBasic(self): with self.cached_session(use_gpu=False): # 1-D, values at index 0. @@ -898,6 +966,7 @@ class SparseMinimumMaximumTest(test_util.TensorFlowTestCase): self._assertSparseTensorValueEqual(expected.eval(), max_tf) self._assertSparseTensorValueEqual(expected.eval(), min_tf) + @test_util.run_deprecated_v1 def testRandom(self): np.random.seed(1618) shapes = [(13,), (6, 8), (1, 7, 1)] @@ -939,6 +1008,7 @@ class SparseMinimumMaximumTest(test_util.TensorFlowTestCase): class SparseTransposeTest(test.TestCase): + @test_util.run_deprecated_v1 def testTranspose(self): if np.__version__ == "1.13.0": self.skipTest("numpy 1.13.0 bug") @@ -961,16 +1031,19 @@ class SparseTransposeTest(test.TestCase): class SparsePlaceholderTest(test.TestCase): + @test_util.run_deprecated_v1 def testPlaceholder(self): foo = array_ops.sparse_placeholder(dtypes.float32, shape=(10, 47)) self.assertAllEqual([10, 47], foo.get_shape()) self.assertAllEqual([None, 2], foo.indices.get_shape().as_list()) + @test_util.run_deprecated_v1 def testPartialShapePlaceholder(self): foo = array_ops.sparse_placeholder(dtypes.float32, shape=(None, 47)) self.assertAllEqual([None, None], foo.get_shape().as_list()) self.assertAllEqual([None, 2], foo.indices.get_shape().as_list()) + @test_util.run_deprecated_v1 def testNoShapePlaceholder(self): foo = array_ops.sparse_placeholder(dtypes.float32, shape=None) self.assertAllEqual(None, foo.get_shape()) diff --git a/tensorflow/python/kernel_tests/sparse_reorder_op_test.py b/tensorflow/python/kernel_tests/sparse_reorder_op_test.py index 7b83ae51779..93fcc6a18e6 100644 --- a/tensorflow/python/kernel_tests/sparse_reorder_op_test.py +++ b/tensorflow/python/kernel_tests/sparse_reorder_op_test.py @@ -22,6 +22,7 @@ import numpy as np from tensorflow.python.framework import dtypes from tensorflow.python.framework import sparse_tensor +from tensorflow.python.framework import test_util from tensorflow.python.ops import array_ops from tensorflow.python.ops import gradient_checker from tensorflow.python.ops import sparse_ops @@ -60,11 +61,12 @@ class SparseReorderTest(test.TestCase): input_val = self._SparseTensorValue_5x6(np.arange(6)) sp_output = sparse_ops.sparse_reorder(input_val) - output_val = sess.run(sp_output) + output_val = self.evaluate(sp_output) self.assertAllEqual(output_val.indices, input_val.indices) self.assertAllEqual(output_val.values, input_val.values) self.assertAllEqual(output_val.dense_shape, input_val.dense_shape) + @test_util.run_deprecated_v1 def testFeedAlreadyInOrder(self): with self.session(use_gpu=False) as sess: sp_input = self._SparseTensorPlaceholder() @@ -83,12 +85,13 @@ class SparseReorderTest(test.TestCase): input_val = self._SparseTensorValue_5x6(np.random.permutation(6)) sp_output = sparse_ops.sparse_reorder(input_val) - output_val = sess.run(sp_output) + output_val = self.evaluate(sp_output) self.assertAllEqual(output_val.indices, expected_output_val.indices) self.assertAllEqual(output_val.values, expected_output_val.values) self.assertAllEqual(output_val.dense_shape, expected_output_val.dense_shape) + @test_util.run_deprecated_v1 def testFeedOutOfOrder(self): expected_output_val = self._SparseTensorValue_5x6(np.arange(6)) with self.session(use_gpu=False) as sess: @@ -103,6 +106,7 @@ class SparseReorderTest(test.TestCase): self.assertAllEqual(output_val.dense_shape, expected_output_val.dense_shape) + @test_util.run_deprecated_v1 def testGradients(self): with self.session(use_gpu=False): for _ in range(5): # To test various random permutations diff --git a/tensorflow/python/kernel_tests/sparse_reshape_op_test.py b/tensorflow/python/kernel_tests/sparse_reshape_op_test.py index f7be397c333..9341228d57e 100644 --- a/tensorflow/python/kernel_tests/sparse_reshape_op_test.py +++ b/tensorflow/python/kernel_tests/sparse_reshape_op_test.py @@ -22,6 +22,7 @@ import numpy as np from tensorflow.python.framework import dtypes from tensorflow.python.framework import sparse_tensor +from tensorflow.python.framework import test_util from tensorflow.python.ops import array_ops from tensorflow.python.ops import sparse_ops from tensorflow.python.platform import test @@ -64,12 +65,14 @@ class SparseReshapeTest(test.TestCase): sp_output = sparse_ops.sparse_reshape(sp_input, shape=(2, -1)) self.assertAllEqual((2, 3 * 4), sp_output.get_shape()) + @test_util.run_deprecated_v1 def testRaisesIfMoreThanOneInferredDim(self): sp_input = sparse_tensor.SparseTensor.from_value( self._SparseTensorValue_2x3x4()) with self.assertRaisesRegexp(ValueError, "At most one dimension can"): sparse_ops.sparse_reshape(sp_input, shape=(-1, 2, -1)) + @test_util.run_deprecated_v1 def testRaisesIfInferredShapeNotPossible(self): sp_input = sparse_tensor.SparseTensor.from_value( self._SparseTensorValue_2x3x4()) @@ -81,11 +84,12 @@ class SparseReshapeTest(test.TestCase): input_val = self._SparseTensorValue_5x6() sp_output = sparse_ops.sparse_reshape(input_val, [5, 6]) - output_val = sess.run(sp_output) + output_val = self.evaluate(sp_output) self.assertAllEqual(output_val.indices, input_val.indices) self.assertAllEqual(output_val.values, input_val.values) self.assertAllEqual(output_val.dense_shape, input_val.dense_shape) + @test_util.run_deprecated_v1 def testFeedSameShape(self): with self.session(use_gpu=False) as sess: sp_input = self._SparseTensorPlaceholder() @@ -97,6 +101,7 @@ class SparseReshapeTest(test.TestCase): self.assertAllEqual(output_val.values, input_val.values) self.assertAllEqual(output_val.dense_shape, input_val.dense_shape) + @test_util.run_deprecated_v1 def testWorksWellWithTfShape(self): with self.session(use_gpu=False) as sess: sp_input = self._SparseTensorPlaceholder() @@ -109,6 +114,7 @@ class SparseReshapeTest(test.TestCase): self.assertAllEqual(output_val.values, input_val.values) self.assertAllEqual(output_val.dense_shape, input_val.dense_shape) + @test_util.run_deprecated_v1 def testFeedSameShapeWithInferredDim(self): with self.session(use_gpu=False) as sess: sp_input = self._SparseTensorPlaceholder() @@ -120,6 +126,7 @@ class SparseReshapeTest(test.TestCase): self.assertAllEqual(output_val.values, input_val.values) self.assertAllEqual(output_val.dense_shape, input_val.dense_shape) + @test_util.run_deprecated_v1 def testFeedNewShapeSameRank(self): with self.session(use_gpu=False) as sess: sp_input = self._SparseTensorPlaceholder() @@ -133,6 +140,7 @@ class SparseReshapeTest(test.TestCase): self.assertAllEqual(output_val.values, input_val.values) self.assertAllEqual(output_val.dense_shape, [3, 10]) + @test_util.run_deprecated_v1 def testFeedNewShapeSameRankWithInferredDim(self): with self.session(use_gpu=False) as sess: sp_input = self._SparseTensorPlaceholder() @@ -151,13 +159,14 @@ class SparseReshapeTest(test.TestCase): input_val = self._SparseTensorValue_5x6() sp_output = sparse_ops.sparse_reshape(input_val, [2, 3, 5]) - output_val = sess.run(sp_output) + output_val = self.evaluate(sp_output) self.assertAllEqual(output_val.indices, np.array([[0, 0, 0], [0, 1, 1], [0, 1, 4], [0, 2, 0], [1, 1, 0], [1, 1, 1]])) self.assertAllEqual(output_val.values, input_val.values) self.assertAllEqual(output_val.dense_shape, [2, 3, 5]) + @test_util.run_deprecated_v1 def testFeedUpRank(self): with self.session(use_gpu=False) as sess: sp_input = self._SparseTensorPlaceholder() @@ -171,6 +180,7 @@ class SparseReshapeTest(test.TestCase): self.assertAllEqual(output_val.values, input_val.values) self.assertAllEqual(output_val.dense_shape, [2, 3, 5]) + @test_util.run_deprecated_v1 def testFeedUpRankWithInferredDim(self): with self.session(use_gpu=False) as sess: sp_input = self._SparseTensorPlaceholder() @@ -184,6 +194,7 @@ class SparseReshapeTest(test.TestCase): self.assertAllEqual(output_val.values, input_val.values) self.assertAllEqual(output_val.dense_shape, [2, 3, 5]) + @test_util.run_deprecated_v1 def testFeedDownRank(self): with self.session(use_gpu=False) as sess: sp_input = self._SparseTensorPlaceholder() @@ -197,6 +208,7 @@ class SparseReshapeTest(test.TestCase): self.assertAllEqual(output_val.values, input_val.values) self.assertAllEqual(output_val.dense_shape, [6, 4]) + @test_util.run_deprecated_v1 def testFeedDownRankWithInferredDim(self): with self.session(use_gpu=False) as sess: sp_input = self._SparseTensorPlaceholder() @@ -210,6 +222,7 @@ class SparseReshapeTest(test.TestCase): self.assertAllEqual(output_val.values, input_val.values) self.assertAllEqual(output_val.dense_shape, [6, 4]) + @test_util.run_deprecated_v1 def testFeedMultipleInferredDims(self): with self.session(use_gpu=False) as sess: sp_input = self._SparseTensorPlaceholder() @@ -218,12 +231,14 @@ class SparseReshapeTest(test.TestCase): with self.assertRaisesOpError("only one output dimension may be -1"): sess.run(sp_output, {sp_input: input_val}) + @test_util.run_deprecated_v1 def testProvideStaticallyMismatchedSizes(self): input_val = self._SparseTensorValue_5x6() sp_input = sparse_tensor.SparseTensor.from_value(input_val) with self.assertRaisesRegexp(ValueError, "Cannot reshape"): sparse_ops.sparse_reshape(sp_input, [4, 7]) + @test_util.run_deprecated_v1 def testFeedMismatchedSizes(self): with self.session(use_gpu=False) as sess: sp_input = self._SparseTensorPlaceholder() @@ -233,6 +248,7 @@ class SparseReshapeTest(test.TestCase): "Input to reshape is a tensor with 30 dense values"): sess.run(sp_output, {sp_input: input_val}) + @test_util.run_deprecated_v1 def testFeedMismatchedSizesWithInferredDim(self): with self.session(use_gpu=False) as sess: sp_input = self._SparseTensorPlaceholder() @@ -241,6 +257,7 @@ class SparseReshapeTest(test.TestCase): with self.assertRaisesOpError("requested shape requires a multiple"): sess.run(sp_output, {sp_input: input_val}) + @test_util.run_deprecated_v1 def testFeedPartialShapes(self): with self.session(use_gpu=False): # Incorporate new rank into shape information if known @@ -266,6 +283,7 @@ class SparseReshapeTest(test.TestCase): self.assertListEqual(sp_output.indices.get_shape().as_list(), [5, None]) self.assertListEqual(sp_output.dense_shape.get_shape().as_list(), [None]) + @test_util.run_deprecated_v1 def testFeedDenseReshapeSemantics(self): with self.session(use_gpu=False) as sess: # Compute a random rank-5 initial shape and new shape, randomly sparsify diff --git a/tensorflow/python/kernel_tests/sparse_serialization_ops_test.py b/tensorflow/python/kernel_tests/sparse_serialization_ops_test.py index b24a0869699..5a48eb825db 100644 --- a/tensorflow/python/kernel_tests/sparse_serialization_ops_test.py +++ b/tensorflow/python/kernel_tests/sparse_serialization_ops_test.py @@ -22,6 +22,7 @@ import numpy as np from tensorflow.python.framework import dtypes from tensorflow.python.framework import sparse_tensor as sparse_tensor_lib +from tensorflow.python.framework import test_util from tensorflow.python.ops import array_ops from tensorflow.python.ops import sparse_ops from tensorflow.python.platform import test @@ -73,7 +74,7 @@ class SerializeSparseTest(test.TestCase): serialized = serialize_fn(sp_input, out_type=out_type) sp_deserialized = deserialize_fn(serialized, dtype=dtypes.int32) - indices, values, shape = sess.run(sp_deserialized) + indices, values, shape = self.evaluate(sp_deserialized) self.assertAllEqual(indices, sp_input[0]) self.assertAllEqual(values, sp_input[1]) @@ -110,14 +111,17 @@ class SerializeSparseTest(test.TestCase): self.assertAllEqual(combined_values[6:], sp_input[1]) self.assertAllEqual(combined_shape, [2, 5, 6]) + @test_util.run_deprecated_v1 def testSerializeDeserializeBatch(self): self._testSerializeDeserializeBatchHelper(sparse_ops.serialize_sparse, sparse_ops.deserialize_sparse) + @test_util.run_deprecated_v1 def testSerializeDeserializeManyBatch(self): self._testSerializeDeserializeBatchHelper( sparse_ops.serialize_sparse, sparse_ops.deserialize_many_sparse) + @test_util.run_deprecated_v1 def testVariantSerializeDeserializeBatch(self): self._testSerializeDeserializeBatchHelper(sparse_ops.serialize_sparse, sparse_ops.deserialize_sparse, @@ -145,10 +149,12 @@ class SerializeSparseTest(test.TestCase): self.assertAllEqual(combined_values[6:], sp_input1[1]) self.assertAllEqual(combined_shape, [2, 5, 6]) + @test_util.run_deprecated_v1 def testSerializeDeserializeBatchInconsistentShape(self): self._testSerializeDeserializeBatchInconsistentShapeHelper( sparse_ops.serialize_sparse, sparse_ops.deserialize_sparse) + @test_util.run_deprecated_v1 def testVariantSerializeDeserializeBatchInconsistentShape(self): self._testSerializeDeserializeBatchInconsistentShapeHelper( sparse_ops.serialize_sparse, sparse_ops.deserialize_sparse, @@ -188,10 +194,12 @@ class SerializeSparseTest(test.TestCase): self.assertAllEqual(combined_shape, [2, 2, 5, 6]) + @test_util.run_deprecated_v1 def testSerializeDeserializeNestedBatch(self): self._testSerializeDeserializeNestedBatchHelper( sparse_ops.serialize_sparse, sparse_ops.deserialize_sparse) + @test_util.run_deprecated_v1 def testVariantSerializeDeserializeNestedBatch(self): self._testSerializeDeserializeNestedBatchHelper( sparse_ops.serialize_sparse, sparse_ops.deserialize_sparse, @@ -224,14 +232,17 @@ class SerializeSparseTest(test.TestCase): self.assertAllEqual(combined_values[6:], input1_val[1]) self.assertAllEqual(combined_shape, [2, 5, 6]) + @test_util.run_deprecated_v1 def testFeedSerializeDeserializeBatch(self): self._testFeedSerializeDeserializeBatchHelper(sparse_ops.serialize_sparse, sparse_ops.deserialize_sparse) + @test_util.run_deprecated_v1 def testFeedSerializeDeserializeManyBatch(self): self._testFeedSerializeDeserializeBatchHelper( sparse_ops.serialize_sparse, sparse_ops.deserialize_many_sparse) + @test_util.run_deprecated_v1 def testFeedVariantSerializeDeserializeBatch(self): self._testFeedSerializeDeserializeBatchHelper(sparse_ops.serialize_sparse, sparse_ops.deserialize_sparse, @@ -256,6 +267,7 @@ class SerializeSparseTest(test.TestCase): }) self.assertEqual(serialized_value.shape, (4, 3)) + @test_util.run_deprecated_v1 def testSerializeManyShape(self): self._testSerializeManyShapeHelper(sparse_ops.serialize_many_sparse) @@ -287,19 +299,23 @@ class SerializeSparseTest(test.TestCase): self.assertAllEqual(deserialized_value.values, values_value) self.assertAllEqual(deserialized_value.dense_shape, shape_value) + @test_util.run_deprecated_v1 def testSerializeManyDeserializeBatch(self): self._testSerializeManyDeserializeBatchHelper( sparse_ops.serialize_many_sparse, sparse_ops.deserialize_sparse) + @test_util.run_deprecated_v1 def testSerializeManyDeserializeManyBatch(self): self._testSerializeManyDeserializeBatchHelper( sparse_ops.serialize_many_sparse, sparse_ops.deserialize_many_sparse) + @test_util.run_deprecated_v1 def testVariantSerializeManyDeserializeBatch(self): self._testSerializeManyDeserializeBatchHelper( sparse_ops.serialize_many_sparse, sparse_ops.deserialize_sparse, dtypes.variant) + @test_util.run_deprecated_v1 def testVariantSerializeDeserializeScalar(self): with self.session(use_gpu=False) as sess: indices_value = np.array([[]], dtype=np.int64) @@ -321,6 +337,7 @@ class SerializeSparseTest(test.TestCase): self.assertAllEqual(deserialized_value.values, values_value) self.assertAllEqual(deserialized_value.dense_shape, shape_value) + @test_util.run_deprecated_v1 def testVariantSerializeDeserializeScalarBatch(self): with self.session(use_gpu=False) as sess: indices_value = np.array([[]], dtype=np.int64) @@ -367,14 +384,17 @@ class SerializeSparseTest(test.TestCase): {sp_input0: input0_val, sp_input1: input1_val}) + @test_util.run_deprecated_v1 def testDeserializeFailsWrongType(self): self._testDeserializeFailsWrongTypeHelper(sparse_ops.serialize_sparse, sparse_ops.deserialize_sparse) + @test_util.run_deprecated_v1 def testDeserializeManyFailsWrongType(self): self._testDeserializeFailsWrongTypeHelper( sparse_ops.serialize_sparse, sparse_ops.deserialize_many_sparse) + @test_util.run_deprecated_v1 def testVariantDeserializeFailsWrongType(self): self._testDeserializeFailsWrongTypeHelper(sparse_ops.serialize_sparse, sparse_ops.deserialize_sparse, @@ -402,14 +422,17 @@ class SerializeSparseTest(test.TestCase): {sp_input0: input0_val, sp_input1: input1_val}) + @test_util.run_deprecated_v1 def testDeserializeFailsInconsistentRank(self): self._testDeserializeFailsInconsistentRankHelper( sparse_ops.serialize_sparse, sparse_ops.deserialize_sparse) + @test_util.run_deprecated_v1 def testDeserializeManyFailsInconsistentRank(self): self._testDeserializeFailsInconsistentRankHelper( sparse_ops.serialize_sparse, sparse_ops.deserialize_many_sparse) + @test_util.run_deprecated_v1 def testVariantDeserializeFailsInconsistentRank(self): self._testDeserializeFailsInconsistentRankHelper( sparse_ops.serialize_sparse, sparse_ops.deserialize_sparse, @@ -431,10 +454,12 @@ class SerializeSparseTest(test.TestCase): with self.assertRaisesOpError(r"Could not parse serialized proto"): sess.run(sp_deserialized, {sp_input0: input0_val}) + @test_util.run_deprecated_v1 def testDeserializeFailsInvalidProto(self): self._testDeserializeFailsInvalidProtoHelper(sparse_ops.serialize_sparse, sparse_ops.deserialize_sparse) + @test_util.run_deprecated_v1 def testDeserializeManyFailsInvalidProto(self): self._testDeserializeFailsInvalidProtoHelper( sparse_ops.serialize_sparse, sparse_ops.deserialize_many_sparse) diff --git a/tensorflow/python/kernel_tests/sparse_slice_op_test.py b/tensorflow/python/kernel_tests/sparse_slice_op_test.py index 098353741f3..7f8c91bde67 100644 --- a/tensorflow/python/kernel_tests/sparse_slice_op_test.py +++ b/tensorflow/python/kernel_tests/sparse_slice_op_test.py @@ -21,6 +21,7 @@ from __future__ import print_function import numpy as np from tensorflow.python.framework import sparse_tensor +from tensorflow.python.framework import test_util from tensorflow.python.ops import gradient_checker from tensorflow.python.ops import sparse_ops import tensorflow.python.ops.sparse_grad # pylint: disable=unused-import @@ -79,6 +80,7 @@ class SparseSliceOpTest(test.TestCase): return sparse_tensor.SparseTensor.from_value( self._SparseTensorValue_3x4x2()) + @test_util.run_deprecated_v1 def testSliceMatrixRows(self): with self.session(use_gpu=False): sp_input = self._SparseTensor_4x6() @@ -96,6 +98,7 @@ class SparseSliceOpTest(test.TestCase): [20, 23, 25, 30, 32, 33, 35]) self.assertAllEqual(sp_tensor1.dense_shape.eval(), [2, 6]) + @test_util.run_deprecated_v1 def testSliceMatrixUnevenCols(self): with self.session(use_gpu=False): sp_input = self._SparseTensor_5x7() @@ -137,6 +140,7 @@ class SparseSliceOpTest(test.TestCase): self.assertAllEqual(sp_tensor3.values.eval(), [16, 46]) self.assertAllEqual(sp_tensor3.dense_shape.eval(), [5, 1]) + @test_util.run_deprecated_v1 def testSliceMatrixUnevenRows(self): with self.session(use_gpu=False): sp_input = self._SparseTensor_5x7() @@ -173,6 +177,7 @@ class SparseSliceOpTest(test.TestCase): self.assertAllEqual(sp_tensor2.dense_shape.eval(), [1, 7]) return + @test_util.run_deprecated_v1 def testSliceAllRows(self): with self.session(use_gpu=False): sp_input = self._SparseTensor_4x6() @@ -195,6 +200,7 @@ class SparseSliceOpTest(test.TestCase): self.assertAllEqual(sp_tensor3.values.eval(), [30, 32, 33, 35]) self.assertAllEqual(sp_tensor3.dense_shape.eval(), [1, 6]) + @test_util.run_deprecated_v1 def testSliceColumns(self): with self.session(use_gpu=False): sp_input = self._SparseTensor_4x6() @@ -215,6 +221,7 @@ class SparseSliceOpTest(test.TestCase): self.assertAllEqual(sparse_tensor2.values.eval(), [4, 5, 14, 25, 35]) self.assertAllEqual(sparse_tensor2.dense_shape.eval(), [4, 2]) + @test_util.run_deprecated_v1 def testSliceAllColumns(self): with self.session(use_gpu=False): sp_input = self._SparseTensor_4x6() @@ -246,6 +253,7 @@ class SparseSliceOpTest(test.TestCase): self.assertAllEqual(sparse_tensor5.values.eval(), [5, 25, 35]) self.assertAllEqual(sparse_tensor5.dense_shape.eval(), [4, 1]) + @test_util.run_deprecated_v1 def testGradients(self): sp_input = self._SparseTensor_4x6(val_dtype=np.float32) start_and_size = [([0, 0], [4, 2]), diff --git a/tensorflow/python/kernel_tests/sparse_split_op_test.py b/tensorflow/python/kernel_tests/sparse_split_op_test.py index 95661ded4be..f4bb7498b02 100644 --- a/tensorflow/python/kernel_tests/sparse_split_op_test.py +++ b/tensorflow/python/kernel_tests/sparse_split_op_test.py @@ -21,6 +21,7 @@ from __future__ import print_function import numpy as np from tensorflow.python.framework import sparse_tensor +from tensorflow.python.framework import test_util from tensorflow.python.ops import sparse_ops from tensorflow.python.platform import test @@ -75,6 +76,7 @@ class SparseSplitOpTest(test.TestCase): return sparse_tensor.SparseTensor.from_value(self._SparseTensorValue_3x4x2( )) + @test_util.run_deprecated_v1 def testSplitMatrixRows(self): with self.session(use_gpu=False): sp_tensors = sparse_ops.sparse_split( @@ -92,6 +94,7 @@ class SparseSplitOpTest(test.TestCase): [20, 23, 25, 30, 32, 33, 35]) self.assertAllEqual(sp_tensors[1].dense_shape.eval(), [2, 6]) + @test_util.run_deprecated_v1 def testSplitMatrixUnevenCols(self): with self.session(use_gpu=False): sp_tensors_3 = sparse_ops.sparse_split( @@ -131,6 +134,7 @@ class SparseSplitOpTest(test.TestCase): self.assertAllEqual(sp_tensors_4[3].values.eval(), [16, 46]) self.assertAllEqual(sp_tensors_4[3].dense_shape.eval(), [5, 1]) + @test_util.run_deprecated_v1 def testSplitMatrixUnevenRows(self): with self.session(use_gpu=False): sp_tensors_2 = sparse_ops.sparse_split( @@ -167,6 +171,7 @@ class SparseSplitOpTest(test.TestCase): self.assertAllEqual(sp_tensors_3[2].dense_shape.eval(), [1, 7]) return + @test_util.run_deprecated_v1 def testSplitAllRows(self): with self.session(use_gpu=False): sp_tensors = sparse_ops.sparse_split( @@ -189,6 +194,7 @@ class SparseSplitOpTest(test.TestCase): self.assertAllEqual(sp_tensors[3].values.eval(), [30, 32, 33, 35]) self.assertAllEqual(sp_tensors[3].dense_shape.eval(), [1, 6]) + @test_util.run_deprecated_v1 def testSplitColumns(self): with self.session(use_gpu=False): sparse_tensors = sparse_ops.sparse_split( @@ -207,6 +213,7 @@ class SparseSplitOpTest(test.TestCase): self.assertAllEqual(sparse_tensors[2].values.eval(), [4, 5, 14, 25, 35]) self.assertAllEqual(sparse_tensors[2].dense_shape.eval(), [4, 2]) + @test_util.run_deprecated_v1 def testSplitAllColumns(self): with self.session(use_gpu=False): sparse_tensors = sparse_ops.sparse_split( @@ -234,6 +241,7 @@ class SparseSplitOpTest(test.TestCase): self.assertAllEqual(sparse_tensors[5].values.eval(), [5, 25, 35]) self.assertAllEqual(sparse_tensors[5].dense_shape.eval(), [4, 1]) + @test_util.run_deprecated_v1 def testSliceConcat(self): for sp_input in (self._SparseTensorValue_3x4x2(), self._SparseTensor_3x4x2()): diff --git a/tensorflow/python/kernel_tests/sparse_tensor_dense_matmul_grad_test.py b/tensorflow/python/kernel_tests/sparse_tensor_dense_matmul_grad_test.py index b8f33d6a813..fa2bab1fca6 100644 --- a/tensorflow/python/kernel_tests/sparse_tensor_dense_matmul_grad_test.py +++ b/tensorflow/python/kernel_tests/sparse_tensor_dense_matmul_grad_test.py @@ -22,6 +22,7 @@ import numpy as np from tensorflow.python.framework import constant_op from tensorflow.python.framework import sparse_tensor +from tensorflow.python.framework import test_util from tensorflow.python.ops import gradient_checker from tensorflow.python.ops import sparse_ops import tensorflow.python.ops.sparse_grad # pylint: disable=unused-import @@ -89,6 +90,7 @@ class SparseTensorDenseMatMulGradientTest(test.TestCase): self._testGradients(adjoint_a, adjoint_b, name, values_dtype, indices_dtype) + @test_util.run_deprecated_v1 def testGradients(self): np.random.seed(5) # Fix seed to avoid flakiness self._testGradientsType(np.float32, np.int64) diff --git a/tensorflow/python/kernel_tests/sparse_tensor_dense_matmul_op_test.py b/tensorflow/python/kernel_tests/sparse_tensor_dense_matmul_op_test.py index fe334045afe..637cfaec990 100644 --- a/tensorflow/python/kernel_tests/sparse_tensor_dense_matmul_op_test.py +++ b/tensorflow/python/kernel_tests/sparse_tensor_dense_matmul_op_test.py @@ -30,6 +30,7 @@ from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops from tensorflow.python.framework import sparse_tensor from tensorflow.python.framework import tensor_shape +from tensorflow.python.framework import test_util from tensorflow.python.ops import array_ops from tensorflow.python.ops import control_flow_ops from tensorflow.python.ops import math_ops @@ -80,7 +81,7 @@ class SparseTensorDenseMatMulTest(test.TestCase): self.assertEqual(tf_value_ans.get_shape()[1], np_ans.shape[1]) self.assertEqual(tf_tensor_ans.get_shape()[1], np_ans.shape[1]) - for out in (tf_value_ans.eval(), tf_tensor_ans.eval()): + for out in (tf_value_ans.eval(), self.evaluate(tf_tensor_ans)): if x.dtype == np.float32: self.assertAllClose(np_ans, out, rtol=1e-4, atol=1e-4) elif x.dtype == np.float64: @@ -96,6 +97,7 @@ class SparseTensorDenseMatMulTest(test.TestCase): self._testMatmul(x, y, indices_dtype=indices_dtype) + @test_util.run_deprecated_v1 def testBasic(self): np.random.seed(127) # Repeatable results self._testBasic(np.int32) @@ -106,6 +108,7 @@ class SparseTensorDenseMatMulTest(test.TestCase): self._testBasic(np.int32, indices_dtype=np.int32) self._testBasic(np.float32, indices_dtype=np.int32) + @test_util.run_deprecated_v1 def testShapeInference(self): x = np.random.rand(10, 10) x[np.abs(x) < 0.5] = 0 # Make it sparse @@ -229,6 +232,7 @@ class SparseTensorDenseMatMulTest(test.TestCase): self._testLarge(np.complex128) # Tests random sized matrices. + @test_util.run_deprecated_v1 def testFloatRandom(self): np.random.seed(127) # Repeatable results for _ in range(8): diff --git a/tensorflow/python/kernel_tests/sparse_tensors_map_ops_test.py b/tensorflow/python/kernel_tests/sparse_tensors_map_ops_test.py index e08464a701c..6039ff1afa7 100644 --- a/tensorflow/python/kernel_tests/sparse_tensors_map_ops_test.py +++ b/tensorflow/python/kernel_tests/sparse_tensors_map_ops_test.py @@ -24,6 +24,7 @@ from tensorflow.python.client import session from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops from tensorflow.python.framework import sparse_tensor as sparse_tensor_lib +from tensorflow.python.framework import test_util from tensorflow.python.ops import array_ops from tensorflow.python.ops import sparse_ops from tensorflow.python.ops import variables @@ -76,6 +77,7 @@ class SparseTensorsMapTest(test.TestCase): shape = np.array([3, 4, 5]).astype(np.int64) return sparse_tensor_lib.SparseTensorValue(ind, val, shape) + @test_util.run_deprecated_v1 def testAddTakeMany(self): with self.session(graph=ops.Graph(), use_gpu=False) as sess: sp_input0 = self._SparseTensorValue_5x6(np.arange(6)) @@ -88,7 +90,7 @@ class SparseTensorsMapTest(test.TestCase): sp_out = take_many_sparse_from_tensors_map( sparse_map_op=handle0.op, sparse_handles=handles_concat) - combined_indices, combined_values, combined_shape = sess.run(sp_out) + combined_indices, combined_values, combined_shape = self.evaluate(sp_out) self.assertAllEqual(combined_indices[:6, 0], [0] * 6) # minibatch 0 self.assertAllEqual(combined_indices[:6, 1:], sp_input0[0]) @@ -98,6 +100,7 @@ class SparseTensorsMapTest(test.TestCase): self.assertAllEqual(combined_values[6:], sp_input1[1]) self.assertAllEqual(combined_shape, [2, 5, 6]) + @test_util.run_deprecated_v1 def testFeedAddTakeMany(self): with self.session(use_gpu=False) as sess: sp_input = self._SparseTensorPlaceholder() @@ -114,7 +117,8 @@ class SparseTensorsMapTest(test.TestCase): sp_roundtrip = take_many_sparse_from_tensors_map( sparse_map_op=handle.op, sparse_handles=sparse_handles) - combined_indices, combined_values, combined_shape = sess.run(sp_roundtrip) + combined_indices, combined_values, combined_shape = self.evaluate( + sp_roundtrip) self.assertAllEqual(combined_indices[:6, 0], [0] * 6) # minibatch 0 self.assertAllEqual(combined_indices[:6, 1:], input0_val[0]) @@ -124,6 +128,7 @@ class SparseTensorsMapTest(test.TestCase): self.assertAllEqual(combined_values[6:], input1_val[1]) self.assertAllEqual(combined_shape, [2, 5, 6]) + @test_util.run_deprecated_v1 def testAddManyTakeManyRoundTrip(self): with self.session(use_gpu=False) as sess: # N == 4 because shape_value == [4, 5] @@ -146,6 +151,7 @@ class SparseTensorsMapTest(test.TestCase): self.assertAllEqual(roundtrip_value.values, values_value) self.assertAllEqual(roundtrip_value.dense_shape, shape_value) + @test_util.run_deprecated_v1 def testDeserializeFailsInconsistentRank(self): with self.session(use_gpu=False) as sess: sp_input = self._SparseTensorPlaceholder() @@ -165,19 +171,20 @@ class SparseTensorsMapTest(test.TestCase): with self.assertRaisesOpError( r"Inconsistent rank across SparseTensors: rank prior to " r"SparseTensor\[1\] was: 3 but rank of SparseTensor\[1\] is: 4"): - sess.run(sp_roundtrip) + self.evaluate(sp_roundtrip) + @test_util.run_deprecated_v1 def testTakeManyFailsWrongInputOp(self): with self.session(use_gpu=False) as sess: input_val = self._SparseTensorValue_5x6(np.arange(6)) handle = add_sparse_to_tensors_map(input_val) - handle_value = sess.run(handle) + handle_value = self.evaluate(handle) bad_handle = handle_value + 10 sp_roundtrip = take_many_sparse_from_tensors_map( sparse_map_op=handle.op, sparse_handles=[handle_value, bad_handle]) with self.assertRaisesOpError(r"Unable to find SparseTensor: 10"): - sess.run(sp_roundtrip) + self.evaluate(sp_roundtrip) class BenchmarkSparseTensorsMapVsSerialization(test.Benchmark): @@ -212,8 +219,8 @@ class BenchmarkSparseTensorsMapVsSerialization(test.Benchmark): variables.global_variables_initializer().run() - st_roundtrip_values = sess.run(st_roundtrip) - st_deserialized_values = sess.run(st_deserialized) + st_roundtrip_values = self.evaluate(st_roundtrip) + st_deserialized_values = self.evaluate(st_deserialized) np.testing.assert_equal(st_roundtrip_values.values, st_deserialized_values.values) np.testing.assert_equal(st_roundtrip_values.indices, diff --git a/tensorflow/python/kernel_tests/sparse_to_dense_op_py_test.py b/tensorflow/python/kernel_tests/sparse_to_dense_op_py_test.py index 7f63532e10d..c6c45db4f9a 100644 --- a/tensorflow/python/kernel_tests/sparse_to_dense_op_py_test.py +++ b/tensorflow/python/kernel_tests/sparse_to_dense_op_py_test.py @@ -21,6 +21,7 @@ from __future__ import print_function import numpy as np from tensorflow.python.framework import dtypes +from tensorflow.python.framework import test_util from tensorflow.python.ops import array_ops from tensorflow.python.ops import sparse_ops from tensorflow.python.platform import test @@ -41,36 +42,42 @@ def _SparseToDense(sparse_indices, class SparseToDenseTest(test.TestCase): + @test_util.run_deprecated_v1 def testInt(self): with self.session(use_gpu=False): tf_ans = _SparseToDense([1, 3], [5], 1, 0).eval() np_ans = np.array([0, 1, 0, 1, 0]).astype(np.int32) self.assertAllClose(np_ans, tf_ans) + @test_util.run_deprecated_v1 def testFloat(self): with self.session(use_gpu=False): tf_ans = _SparseToDense([1, 3], [5], 1.0, 0.0).eval() np_ans = np.array([0, 1, 0, 1, 0]).astype(np.float32) self.assertAllClose(np_ans, tf_ans) + @test_util.run_deprecated_v1 def testString(self): with self.session(use_gpu=False): tf_ans = _SparseToDense([1, 3], [5], "a", "b").eval() np_ans = np.array(["b", "a", "b", "a", "b"]).astype(np.string_) self.assertAllEqual(np_ans, tf_ans) + @test_util.run_deprecated_v1 def testSetValue(self): with self.session(use_gpu=False): tf_ans = _SparseToDense([1, 3], [5], [1, 2], -1).eval() np_ans = np.array([-1, 1, -1, 2, -1]).astype(np.int32) self.assertAllClose(np_ans, tf_ans) + @test_util.run_deprecated_v1 def testSetSingleValue(self): with self.session(use_gpu=False): tf_ans = _SparseToDense([1, 3], [5], 1, -1).eval() np_ans = np.array([-1, 1, -1, 1, -1]).astype(np.int32) self.assertAllClose(np_ans, tf_ans) + @test_util.run_deprecated_v1 def test2d(self): # pylint: disable=bad-whitespace with self.session(use_gpu=False): @@ -80,11 +87,13 @@ class SparseToDenseTest(test.TestCase): [ 1, -1, -1, -1]]).astype(np.int32) self.assertAllClose(np_ans, tf_ans) + @test_util.run_deprecated_v1 def testZeroDefault(self): with self.cached_session(): x = sparse_ops.sparse_to_dense(2, [4], 7).eval() self.assertAllEqual(x, [0, 0, 7, 0]) + @test_util.run_deprecated_v1 def test3d(self): with self.session(use_gpu=False): tf_ans = _SparseToDense([[1, 3, 0], [2, 0, 1]], [3, 4, 2], 1, -1).eval() @@ -93,32 +102,37 @@ class SparseToDenseTest(test.TestCase): np_ans[2, 0, 1] = 1 self.assertAllClose(np_ans, tf_ans) + @test_util.run_deprecated_v1 def testBadShape(self): with self.cached_session(): with self.assertRaisesWithPredicateMatch(ValueError, "must be rank 1"): _SparseToDense([1, 3], [[5], [3]], 1, -1) + @test_util.run_deprecated_v1 def testBadValue(self): with self.cached_session(): dense = _SparseToDense([1, 3], [5], [[5], [3]], -1) with self.assertRaisesOpError( r"sparse_values has incorrect shape \[2,1\], " r"should be \[\] or \[2\]"): - dense.eval() + self.evaluate(dense) + @test_util.run_deprecated_v1 def testBadNumValues(self): with self.cached_session(): dense = _SparseToDense([1, 3], [5], [1, 2, 3], -1) with self.assertRaisesOpError( r"sparse_values has incorrect shape \[3\], should be \[\] or \[2\]"): - dense.eval() + self.evaluate(dense) + @test_util.run_deprecated_v1 def testBadDefault(self): with self.cached_session(): dense = _SparseToDense([1, 3], [5], [1, 2], [0]) with self.assertRaisesOpError("default_value should be a scalar"): - dense.eval() + self.evaluate(dense) + @test_util.run_deprecated_v1 def testOutOfBoundsIndicesWithWithoutValidation(self): with self.cached_session(): dense = _SparseToDense( @@ -128,7 +142,7 @@ class SparseToDenseTest(test.TestCase): default_value=0.0) with self.assertRaisesOpError( r"indices\[1\] = \[10\] is out of bounds: need 0 <= index < \[5\]"): - dense.eval() + self.evaluate(dense) # Disable checks, the allocation should still fail. with self.assertRaisesOpError("out of bounds"): dense_without_validation = _SparseToDense( @@ -137,8 +151,9 @@ class SparseToDenseTest(test.TestCase): sparse_values=[-1.0, 1.0], default_value=0.0, validate_indices=False) - dense_without_validation.eval() + self.evaluate(dense_without_validation) + @test_util.run_deprecated_v1 def testRepeatingIndicesWithWithoutValidation(self): with self.cached_session(): dense = _SparseToDense( @@ -147,7 +162,7 @@ class SparseToDenseTest(test.TestCase): sparse_values=[-1.0, 1.0], default_value=0.0) with self.assertRaisesOpError(r"indices\[1\] = \[1\] is repeated"): - dense.eval() + self.evaluate(dense) # Disable checks dense_without_validation = _SparseToDense( sparse_indices=[[1], [1]], @@ -155,8 +170,9 @@ class SparseToDenseTest(test.TestCase): sparse_values=[-1.0, 1.0], default_value=0.0, validate_indices=False) - dense_without_validation.eval() + self.evaluate(dense_without_validation) + @test_util.run_deprecated_v1 def testUnsortedIndicesWithWithoutValidation(self): with self.cached_session(): dense = _SparseToDense( @@ -165,7 +181,7 @@ class SparseToDenseTest(test.TestCase): sparse_values=[-1.0, 1.0], default_value=0.0) with self.assertRaisesOpError(r"indices\[1\] = \[1\] is out of order"): - dense.eval() + self.evaluate(dense) # Disable checks dense_without_validation = _SparseToDense( sparse_indices=[[2], [1]], @@ -173,8 +189,9 @@ class SparseToDenseTest(test.TestCase): sparse_values=[-1.0, 1.0], default_value=0.0, validate_indices=False) - dense_without_validation.eval() + self.evaluate(dense_without_validation) + @test_util.run_deprecated_v1 def testShapeInferenceKnownShape(self): with self.session(use_gpu=False): indices = array_ops.placeholder(dtypes.int64) @@ -187,6 +204,7 @@ class SparseToDenseTest(test.TestCase): output = sparse_ops.sparse_to_dense(indices, shape, 1, 0) self.assertEqual(output.get_shape().as_list(), [None, None, None]) + @test_util.run_deprecated_v1 def testShapeInferenceUnknownShape(self): with self.session(use_gpu=False): indices = array_ops.placeholder(dtypes.int64) diff --git a/tensorflow/python/kernel_tests/sparse_xent_op_test.py b/tensorflow/python/kernel_tests/sparse_xent_op_test.py index 0510bc53214..8f0842f7f50 100644 --- a/tensorflow/python/kernel_tests/sparse_xent_op_test.py +++ b/tensorflow/python/kernel_tests/sparse_xent_op_test.py @@ -29,6 +29,7 @@ from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes from tensorflow.python.framework import errors_impl from tensorflow.python.framework import ops as ops_lib +from tensorflow.python.framework import test_util from tensorflow.python.ops import array_ops from tensorflow.python.ops import gen_nn_ops from tensorflow.python.ops import gradient_checker @@ -66,7 +67,7 @@ class SparseXentTest(test.TestCase): with self.cached_session(use_gpu=True) as sess: loss, backprop = gen_nn_ops.sparse_softmax_cross_entropy_with_logits( np_features, np_labels) - tf_loss, tf_backprop = sess.run([loss, backprop]) + tf_loss, tf_backprop = self.evaluate([loss, backprop]) self.assertAllCloseAccordingToType(np_loss, tf_loss) self.assertAllCloseAccordingToType(np_backprop, tf_backprop) @@ -76,10 +77,11 @@ class SparseXentTest(test.TestCase): loss, backprop = gen_nn_ops.sparse_softmax_cross_entropy_with_logits( np.array([[1.], [-1.], [0.]]).astype(np.float32), np.array([0, 0, 0]).astype(label_dtype)) - tf_loss, tf_backprop = sess.run([loss, backprop]) + tf_loss, tf_backprop = self.evaluate([loss, backprop]) self.assertAllClose([0.0, 0.0, 0.0], tf_loss) self.assertAllClose([[0.0], [0.0], [0.0]], tf_backprop) + @test_util.run_deprecated_v1 def testInvalidLabel(self): features = [[1., 1., 1., 1.], [1., 1., 1., 1.], [1., 2., 3., 4.], [1., 2., 3., 4.]] @@ -90,7 +92,7 @@ class SparseXentTest(test.TestCase): loss, backprop = ( gen_nn_ops.sparse_softmax_cross_entropy_with_logits( features, labels)) - tf_loss, tf_backprop = sess.run([loss, backprop]) + tf_loss, tf_backprop = self.evaluate([loss, backprop]) self.assertAllClose( [[np.nan] * 4, [0.25, 0.25, 0.25, -0.75], [-0.968, 0.087, 0.237, 0.6439], [np.nan] * 4], @@ -104,7 +106,7 @@ class SparseXentTest(test.TestCase): loss, backprop = ( gen_nn_ops.sparse_softmax_cross_entropy_with_logits(features, labels)) with self.assertRaisesOpError("Received a label value of"): - sess.run([loss, backprop]) + self.evaluate([loss, backprop]) def testNpXent(self): # We create 2 batches of logits for testing. @@ -152,6 +154,7 @@ class SparseXentTest(test.TestCase): nn_ops.sparse_softmax_cross_entropy_with_logits( labels=constant_op.constant(0), logits=constant_op.constant(1.0)) + @test_util.run_deprecated_v1 def testLabelsPlaceholderScalar(self): with self.session(use_gpu=True): labels = array_ops.placeholder(np.int32) @@ -164,7 +167,7 @@ class SparseXentTest(test.TestCase): with self.session(use_gpu=True): loss = nn_ops.sparse_softmax_cross_entropy_with_logits( labels=constant_op.constant(0), logits=constant_op.constant([1.0])) - self.assertAllClose(0.0, loss.eval()) + self.assertAllClose(0.0, self.evaluate(loss)) def testFloat(self): for label_dtype in np.int32, np.int64: @@ -187,6 +190,7 @@ class SparseXentTest(test.TestCase): def testEmpty(self): self._testXent(np.zeros((0, 3)), np.zeros((0,), dtype=np.int32)) + @test_util.run_deprecated_v1 def testGradient(self): with self.session(use_gpu=True): l = constant_op.constant([3, 0, 1], name="l") @@ -201,6 +205,7 @@ class SparseXentTest(test.TestCase): print("cross entropy gradient err = ", err) self.assertLess(err, 5e-8) + @test_util.run_deprecated_v1 def testSecondGradient(self): images_placeholder = array_ops.placeholder(dtypes.float32, shape=(3, 2)) labels_placeholder = array_ops.placeholder(dtypes.int32, shape=(3)) @@ -226,21 +231,24 @@ class SparseXentTest(test.TestCase): loss = nn_ops.sparse_softmax_cross_entropy_with_logits( labels=labels, logits=features) backprop = loss.op.inputs[0].op.outputs[1] - tf_loss, tf_backprop = sess.run([loss, backprop]) + tf_loss, tf_backprop = self.evaluate([loss, backprop]) self.assertAllCloseAccordingToType(np_loss, tf_loss) self.assertAllCloseAccordingToType(np_backprop, tf_backprop) + @test_util.run_deprecated_v1 def testHighDim(self): features = [[[1., 1., 1., 1.]], [[1., 2., 3., 4.]]] labels = [[3], [0]] self._testHighDim(features, labels) + @test_util.run_deprecated_v1 def testHighDim2(self): features = [[[1., 1., 1., 1.], [2., 2., 2., 2.]], [[1., 2., 3., 4.], [5., 6., 7., 8.]]] labels = [[3, 2], [0, 3]] self._testHighDim(features, labels) + @test_util.run_deprecated_v1 def testScalarHandling(self): with self.session(use_gpu=False) as sess: with self.assertRaisesRegexp(errors_impl.InvalidArgumentError, @@ -318,7 +326,7 @@ def sparse_vs_dense_xent_benchmark(batch_size, num_entries, use_gpu): # Using sparse_softmax_cross_entropy_with_logits with session.Session(config=config) as sess: if not use_gpu: - with ops_lib.device("/cpu:0"): + with test_util.device("/cpu:0"): ops = _sparse_vs_dense_xent_benchmark_sparse(labels, logits) else: ops = _sparse_vs_dense_xent_benchmark_sparse(labels, logits) diff --git a/tensorflow/python/kernel_tests/sparsemask_op_test.py b/tensorflow/python/kernel_tests/sparsemask_op_test.py index 6f5dd45b616..b1cd0227bc0 100644 --- a/tensorflow/python/kernel_tests/sparsemask_op_test.py +++ b/tensorflow/python/kernel_tests/sparsemask_op_test.py @@ -20,12 +20,14 @@ from __future__ import print_function import numpy as np from tensorflow.python.framework import ops +from tensorflow.python.framework import test_util from tensorflow.python.ops import array_ops from tensorflow.python.platform import test class SparseMaskTest(test.TestCase): + @test_util.run_deprecated_v1 def testBasic(self): values = np.random.rand(4, 4).astype(np.single) indices = np.array([0, 2, 3, 4], dtype=np.int32) diff --git a/tensorflow/python/kernel_tests/split_op_test.py b/tensorflow/python/kernel_tests/split_op_test.py index 944b0e59b12..517db3450f3 100644 --- a/tensorflow/python/kernel_tests/split_op_test.py +++ b/tensorflow/python/kernel_tests/split_op_test.py @@ -42,6 +42,7 @@ class SplitOpTest(test.TestCase): data -= 1j * data return data + @test_util.run_deprecated_v1 def testShapeInference(self): model_input = array_ops.placeholder(dtypes.float32, shape=(1, 10)) @@ -85,6 +86,7 @@ class SplitOpTest(test.TestCase): with self.cached_session(use_gpu=True) as sess: sess.run(result, feed_dict={model_input2: np.ones([4, 2])}) + @test_util.run_deprecated_v1 def testFailWithoutExplicitNum(self): size_splits = array_ops.placeholder(dtype=dtypes.int32, shape=[None]) @@ -209,6 +211,7 @@ class SplitOpTest(test.TestCase): self.assertAllEqual(result[:, 0:1], inp_grads[0]) self.assertAllEqual(result[:, 1:4], inp_grads[1]) + @test_util.run_deprecated_v1 def testOutputShape(self): for axis in [1, -1]: with self.cached_session(use_gpu=True): @@ -318,15 +321,17 @@ class SplitOpTest(test.TestCase): inp_grads = [self._makeData((4, 1), dtype)for _ in range(4)] grad_tensors = [constant_op.constant(x) for x in inp_grads] grad = gradients_impl.gradients(s, [inp_tensor], grad_tensors)[0] - result = grad.eval() + result = self.evaluate(grad) for i in range(4): self.assertAllEqual(result[:, i:i + 1], inp_grads[i]) + @test_util.run_deprecated_v1 def testGradientsAll(self): for dtype in _TEST_DTYPES: self._testGradientsSimple(dtype) self._testGradientsSimpleVariable(dtype) + @test_util.run_deprecated_v1 def testShapeFunctionEdgeCases(self): # split_dim greater than rank of input. with self.assertRaises(ValueError): @@ -356,6 +361,7 @@ class SplitOpTest(test.TestCase): for s in splits: self.assertEqual(None, s.get_shape().ndims) + @test_util.run_deprecated_v1 def testVariableShapeFunction(self): # size_splits too big with self.assertRaises(ValueError): @@ -366,6 +372,7 @@ class SplitOpTest(test.TestCase): assert s0.shape.as_list() == [2] assert s1.shape.as_list() == [1] + @test_util.run_deprecated_v1 def testNonexistentDimTensor(self): x = array_ops.placeholder(dtypes.int32) values = np.zeros([5, 30]) diff --git a/tensorflow/python/kernel_tests/stack_op_test.py b/tensorflow/python/kernel_tests/stack_op_test.py index 4b355620bf9..ca3357a0ed8 100644 --- a/tensorflow/python/kernel_tests/stack_op_test.py +++ b/tensorflow/python/kernel_tests/stack_op_test.py @@ -24,6 +24,7 @@ from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes from tensorflow.python.framework import errors_impl from tensorflow.python.framework import ops +from tensorflow.python.framework import test_util from tensorflow.python.ops import array_ops from tensorflow.python.ops import gradient_checker from tensorflow.python.ops import variables @@ -41,6 +42,7 @@ def np_split_squeeze(array, axis): class StackOpTest(test.TestCase): + @test_util.run_deprecated_v1 def testSimple(self): np.random.seed(7) with self.session(use_gpu=True): @@ -54,6 +56,7 @@ class StackOpTest(test.TestCase): c = array_ops.stack(xs) self.assertAllEqual(c.eval(), data) + @test_util.run_deprecated_v1 def testSimpleParallelCPU(self): np.random.seed(7) with self.session(use_gpu=False): @@ -63,6 +66,7 @@ class StackOpTest(test.TestCase): c = array_ops.parallel_stack(xs) self.assertAllEqual(c.eval(), data) + @test_util.run_deprecated_v1 def testSimpleParallelGPU(self): np.random.seed(7) with self.session(use_gpu=True): @@ -72,6 +76,7 @@ class StackOpTest(test.TestCase): c = array_ops.parallel_stack(xs) self.assertAllEqual(c.eval(), data) + @test_util.run_deprecated_v1 def testConst(self): np.random.seed(7) with self.session(use_gpu=True): @@ -96,6 +101,7 @@ class StackOpTest(test.TestCase): b = array_ops.reshape(a, array_ops.stack([2, 3])) self.assertAllEqual(b.get_shape(), [2, 3]) + @test_util.run_deprecated_v1 def testConstParallelCPU(self): np.random.seed(7) with self.session(use_gpu=False): @@ -110,6 +116,7 @@ class StackOpTest(test.TestCase): c = array_ops.parallel_stack(data) self.assertAllEqual(c.eval(), data) + @test_util.run_deprecated_v1 def testConstParallelGPU(self): np.random.seed(7) with self.session(use_gpu=True): @@ -124,6 +131,7 @@ class StackOpTest(test.TestCase): c = array_ops.parallel_stack(data) self.assertAllEqual(c.eval(), data) + @test_util.run_deprecated_v1 def testGradientsAxis0(self): np.random.seed(7) for shape in (2,), (3,), (2, 3), (3, 2), (4, 3, 2): @@ -136,6 +144,7 @@ class StackOpTest(test.TestCase): err = gradient_checker.compute_gradient_error(xs, shapes, c, shape) self.assertLess(err, 1e-6) + @test_util.run_deprecated_v1 def testGradientsAxis1(self): np.random.seed(7) for shape in (2, 3), (3, 2), (4, 3, 2): @@ -150,6 +159,7 @@ class StackOpTest(test.TestCase): err = gradient_checker.compute_gradient_error(xs, shapes, c, out_shape) self.assertLess(err, 1e-6) + @test_util.run_deprecated_v1 def testZeroSizeCPU(self): # Verify that stack doesn't crash for zero size inputs with self.session(use_gpu=False): @@ -161,6 +171,7 @@ class StackOpTest(test.TestCase): p = array_ops.parallel_stack(list(x)).eval() self.assertAllEqual(p, x) + @test_util.run_deprecated_v1 def testZeroSizeGPU(self): # Verify that stack doesn't crash for zero size inputs with self.session(use_gpu=True): @@ -172,6 +183,7 @@ class StackOpTest(test.TestCase): p = array_ops.parallel_stack(list(x)).eval() self.assertAllEqual(p, x) + @test_util.run_deprecated_v1 def testAxis0DefaultCPU(self): with self.session(use_gpu=False): t = [constant_op.constant([1, 2, 3]), constant_op.constant([4, 5, 6])] @@ -182,6 +194,7 @@ class StackOpTest(test.TestCase): self.assertAllEqual(stacked, expected) self.assertAllEqual(parallel_stacked, expected) + @test_util.run_deprecated_v1 def testAxis0DefaultGPU(self): with self.session(use_gpu=True): t = [constant_op.constant([1, 2, 3]), constant_op.constant([4, 5, 6])] @@ -204,11 +217,11 @@ class StackOpTest(test.TestCase): with self.cached_session(use_gpu=True): actual_pack = array_ops.stack(test_arrays, axis=j) self.assertEqual(expected.shape, actual_pack.get_shape()) - actual_pack = actual_pack.eval() + actual_pack = self.evaluate(actual_pack) actual_stack = array_ops.stack(test_arrays, axis=j) self.assertEqual(expected.shape, actual_stack.get_shape()) - actual_stack = actual_stack.eval() + actual_stack = self.evaluate(actual_stack) self.assertNDArrayNear(expected, actual_stack, 1e-6) @@ -225,6 +238,7 @@ class StackOpTest(test.TestCase): class AutomaticStackingTest(test.TestCase): + @test_util.run_deprecated_v1 def testSimple(self): with self.session(use_gpu=True): self.assertAllEqual( @@ -253,17 +267,20 @@ class AutomaticStackingTest(test.TestCase): [[2., 2.], [3., 3.]], dtype=np.float32)]) self.assertAllEqual([[[0., 0.], [1., 1.]], [[2., 2.], [3., 3.]]], - result.eval()) + self.evaluate(result)) + @test_util.run_deprecated_v1 def testVariable(self): with self.session(use_gpu=True): v = variables.Variable(17) result = ops.convert_to_tensor([[0, 0, 0], [0, v, 0], [0, 0, 0]]) v.initializer.run() - self.assertAllEqual([[0, 0, 0], [0, 17, 0], [0, 0, 0]], result.eval()) + self.assertAllEqual([[0, 0, 0], [0, 17, 0], [0, 0, 0]], + self.evaluate(result)) v.assign(38).op.run() - self.assertAllEqual([[0, 0, 0], [0, 38, 0], [0, 0, 0]], result.eval()) + self.assertAllEqual([[0, 0, 0], [0, 38, 0], [0, 0, 0]], + self.evaluate(result)) def testDtype(self): t_0 = ops.convert_to_tensor([[0., 0., 0.], [0., 0., 0.], [0., 0., 0.]]) @@ -306,6 +323,7 @@ class AutomaticStackingTest(test.TestCase): t_2 = ops.convert_to_tensor([t_0, t_0, t_1], dtype=dtypes.float64) self.assertEqual(dtypes.float64, t_2.dtype) + @test_util.run_deprecated_v1 def testPlaceholder(self): with self.session(use_gpu=True): # Test using placeholder with a defined shape. @@ -324,6 +342,7 @@ class AutomaticStackingTest(test.TestCase): self.assertAllEqual( [[0, 0, 0], [0, 2, 0], [0, 0, 0]], result_1.eval(feed_dict={ph_1: 2})) + @test_util.run_deprecated_v1 def testShapeErrors(self): # Static shape error. ph_0 = array_ops.placeholder(dtypes.int32, shape=[1]) diff --git a/tensorflow/python/kernel_tests/stack_ops_test.py b/tensorflow/python/kernel_tests/stack_ops_test.py index 1aa12009ea5..d50f3f46806 100644 --- a/tensorflow/python/kernel_tests/stack_ops_test.py +++ b/tensorflow/python/kernel_tests/stack_ops_test.py @@ -24,6 +24,7 @@ from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops from tensorflow.python.framework import tensor_shape +from tensorflow.python.framework import test_util from tensorflow.python.ops import control_flow_ops from tensorflow.python.ops import gen_data_flow_ops from tensorflow.python.ops import math_ops @@ -39,8 +40,9 @@ class StackOpTest(test.TestCase): c = gen_data_flow_ops.stack_push_v2(h, [[4.0, 5.0]]) with ops.control_dependencies([c]): c1 = gen_data_flow_ops.stack_pop_v2(h, dtypes.float32) - self.assertAllClose([[4.0, 5.0]], c1.eval()) + self.assertAllClose([[4.0, 5.0]], self.evaluate(c1)) + @test_util.run_deprecated_v1 def testStackPushPop(self): self._testStackPushPop(use_gpu=False) self._testStackPushPop(use_gpu=True) @@ -54,8 +56,9 @@ class StackOpTest(test.TestCase): c = gen_data_flow_ops.stack_push_v2(h, x, swap_memory=True) with ops.control_dependencies([c]): c1 = gen_data_flow_ops.stack_pop_v2(h, dtypes.float32) - self.assertAllClose(a, c1.eval()) + self.assertAllClose(a, self.evaluate(c1)) + @test_util.run_deprecated_v1 def testStackPushPopSwap(self): self._testStackPushPopSwap(use_gpu=False) self._testStackPushPopSwap(use_gpu=True) @@ -91,8 +94,9 @@ class StackOpTest(test.TestCase): _, ry = control_flow_ops.while_loop( c1, b1, [r, v], [r.get_shape(), tensor_shape.unknown_shape()]) - self.assertAllClose(np.ones(2000) * 10.0, ry.eval()) + self.assertAllClose(np.ones(2000) * 10.0, self.evaluate(ry)) + @test_util.run_deprecated_v1 def testStackWhileSwap(self): self._testStackWhileSwap(use_gpu=False) self._testStackWhileSwap(use_gpu=True) @@ -110,8 +114,9 @@ class StackOpTest(test.TestCase): with ops.control_dependencies([c2]): c2 = gen_data_flow_ops.stack_pop_v2(h2, dtypes.float32) r = c1 + c2 - self.assertAllClose(9.0, r.eval()) + self.assertAllClose(9.0, self.evaluate(r)) + @test_util.run_deprecated_v1 def testMultiStack(self): self._testMultiStack(use_gpu=False) self._testMultiStack(use_gpu=True) @@ -131,10 +136,11 @@ class StackOpTest(test.TestCase): pop1 = gen_data_flow_ops.stack_pop_v2(h1, dtypes.float32) pop2 = gen_data_flow_ops.stack_pop_v2(h2, dtypes.float32) - out1, out2 = sess.run([pop1, pop2]) + out1, out2 = self.evaluate([pop1, pop2]) self.assertAllClose(out1, 4.0) self.assertAllClose(out2, 5.0) + @test_util.run_deprecated_v1 def testSameNameStacks(self): self._testSameNameStacks(use_gpu=False) self._testSameNameStacks(use_gpu=True) @@ -144,8 +150,9 @@ class StackOpTest(test.TestCase): h = gen_data_flow_ops.stack_v2( -1, elem_type=dtypes.float32, stack_name="foo") c1 = gen_data_flow_ops.stack_close_v2(h) - sess.run(c1) + self.evaluate(c1) + @test_util.run_deprecated_v1 def testCloseStack(self): self._testCloseStack(use_gpu=False) self._testCloseStack(use_gpu=True) @@ -157,8 +164,9 @@ class StackOpTest(test.TestCase): c = gen_data_flow_ops.stack_push_v2(h, [[4.0, 5.0]]) with ops.control_dependencies([c]): c1 = gen_data_flow_ops.stack_close_v2(h) - sess.run(c1) + self.evaluate(c1) + @test_util.run_deprecated_v1 def testPushCloseStack(self): self._testPushCloseStack(use_gpu=False) self._testPushCloseStack(use_gpu=True) @@ -173,8 +181,9 @@ class StackOpRefTest(test.TestCase): c = gen_data_flow_ops.stack_push(h, [[4.0, 5.0]]) with ops.control_dependencies([c]): c1 = gen_data_flow_ops.stack_pop(h, dtypes.float32) - self.assertAllClose([[4.0, 5.0]], c1.eval()) + self.assertAllClose([[4.0, 5.0]], self.evaluate(c1)) + @test_util.run_deprecated_v1 def testStackPushPop(self): self._testStackPushPop(use_gpu=False) self._testStackPushPop(use_gpu=True) @@ -187,8 +196,9 @@ class StackOpRefTest(test.TestCase): c = gen_data_flow_ops.stack_push(h, x, swap_memory=True) with ops.control_dependencies([c]): c1 = gen_data_flow_ops.stack_pop(h, dtypes.float32) - self.assertAllClose(a, c1.eval()) + self.assertAllClose(a, self.evaluate(c1)) + @test_util.run_deprecated_v1 def testStackPushPopSwap(self): self._testStackPushPopSwap(use_gpu=False) self._testStackPushPopSwap(use_gpu=True) @@ -204,7 +214,7 @@ class StackOpRefTest(test.TestCase): with ops.control_dependencies([c2]): c2 = gen_data_flow_ops.stack_pop(h2, dtypes.float32) r = c1 + c2 - self.assertAllClose(9.0, r.eval()) + self.assertAllClose(9.0, self.evaluate(r)) def _testStackWhileSwap(self, use_gpu): with self.cached_session(use_gpu=use_gpu): @@ -236,12 +246,14 @@ class StackOpRefTest(test.TestCase): _, ry = control_flow_ops.while_loop( c1, b1, [r, v], [r.get_shape(), tensor_shape.unknown_shape()]) - self.assertAllClose(np.ones(2000) * 10.0, ry.eval()) + self.assertAllClose(np.ones(2000) * 10.0, self.evaluate(ry)) + @test_util.run_deprecated_v1 def testStackWhileSwap(self): self._testStackWhileSwap(use_gpu=False) self._testStackWhileSwap(use_gpu=True) + @test_util.run_deprecated_v1 def testMultiStack(self): self._testMultiStack(use_gpu=False) self._testMultiStack(use_gpu=True) @@ -253,8 +265,9 @@ class StackOpRefTest(test.TestCase): h2 = gen_data_flow_ops._stack(dtypes.float32, stack_name="foo") c2 = gen_data_flow_ops.stack_push(h2, 5.0) _ = c1 + c2 - self.assertNotEqual(h1.eval()[1], h2.eval()[1]) + self.assertNotEqual(h1.eval()[1], self.evaluate(h2)[1]) + @test_util.run_deprecated_v1 def testSameNameStacks(self): self._testSameNameStacks(use_gpu=False) self._testSameNameStacks(use_gpu=True) @@ -263,8 +276,9 @@ class StackOpRefTest(test.TestCase): with self.cached_session(use_gpu=use_gpu) as sess: h = gen_data_flow_ops._stack(dtypes.float32, stack_name="foo") c1 = gen_data_flow_ops.stack_close(h) - sess.run(c1) + self.evaluate(c1) + @test_util.run_deprecated_v1 def testCloseStack(self): self._testCloseStack(use_gpu=False) self._testCloseStack(use_gpu=True) @@ -275,8 +289,9 @@ class StackOpRefTest(test.TestCase): c = gen_data_flow_ops.stack_push(h, [[4.0, 5.0]]) with ops.control_dependencies([c]): c1 = gen_data_flow_ops.stack_close(h) - sess.run(c1) + self.evaluate(c1) + @test_util.run_deprecated_v1 def testPushCloseStack(self): self._testPushCloseStack(use_gpu=False) self._testPushCloseStack(use_gpu=True) diff --git a/tensorflow/python/kernel_tests/stage_op_test.py b/tensorflow/python/kernel_tests/stage_op_test.py index b814843b86c..83e06ba48bd 100644 --- a/tensorflow/python/kernel_tests/stage_op_test.py +++ b/tensorflow/python/kernel_tests/stage_op_test.py @@ -18,6 +18,7 @@ from __future__ import print_function from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops +from tensorflow.python.framework import test_util from tensorflow.python.ops import array_ops from tensorflow.python.ops import data_flow_ops from tensorflow.python.ops import math_ops @@ -28,6 +29,7 @@ TIMEOUT = 1 class StageTest(test.TestCase): + @test_util.run_deprecated_v1 def testSimple(self): with ops.Graph().as_default() as G: with ops.device('/cpu:0'): @@ -47,6 +49,7 @@ class StageTest(test.TestCase): _, yval = sess.run([stage, y], feed_dict={x: i}) self.assertAllClose(4 * (i - 1) * (i - 1) * 128, yval, rtol=1e-4) + @test_util.run_deprecated_v1 def testMultiple(self): with ops.Graph().as_default() as G: with ops.device('/cpu:0'): @@ -67,6 +70,7 @@ class StageTest(test.TestCase): self.assertAllClose( 4 * (i - 1) * (i - 1) * (i - 1) * 128, yval, rtol=1e-4) + @test_util.run_deprecated_v1 def testDictionary(self): with ops.Graph().as_default() as G: with ops.device('/cpu:0'): @@ -110,6 +114,7 @@ class StageTest(test.TestCase): G.finalize() + @test_util.run_deprecated_v1 def testPeek(self): with ops.Graph().as_default() as G: with ops.device('/cpu:0'): @@ -133,6 +138,7 @@ class StageTest(test.TestCase): for i in range(10): self.assertTrue(sess.run(peek, feed_dict={p: i}) == [i]) + @test_util.run_deprecated_v1 def testSizeAndClear(self): with ops.Graph().as_default() as G: with ops.device('/cpu:0'): @@ -158,6 +164,7 @@ class StageTest(test.TestCase): sess.run(clear) self.assertEqual(sess.run(size), 0) + @test_util.run_deprecated_v1 def testCapacity(self): capacity = 3 @@ -219,6 +226,7 @@ class StageTest(test.TestCase): # It should now be empty self.assertTrue(sess.run(size) == 0) + @test_util.run_deprecated_v1 def testMemoryLimit(self): memory_limit = 512 * 1024 # 512K chunk = 200 * 1024 # 256K diff --git a/tensorflow/python/kernel_tests/string_join_op_test.py b/tensorflow/python/kernel_tests/string_join_op_test.py index e4371ab5b93..2548e8695fe 100644 --- a/tensorflow/python/kernel_tests/string_join_op_test.py +++ b/tensorflow/python/kernel_tests/string_join_op_test.py @@ -17,12 +17,14 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function +from tensorflow.python.framework import test_util from tensorflow.python.ops import string_ops from tensorflow.python.platform import test class StringJoinOpTest(test.TestCase): + @test_util.run_deprecated_v1 def testStringJoin(self): input0 = ["a", "b"] input1 = "a" diff --git a/tensorflow/python/kernel_tests/string_length_op_test.py b/tensorflow/python/kernel_tests/string_length_op_test.py index 57db7302b15..bfa6ac2454a 100644 --- a/tensorflow/python/kernel_tests/string_length_op_test.py +++ b/tensorflow/python/kernel_tests/string_length_op_test.py @@ -18,6 +18,7 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function +from tensorflow.python.framework import test_util from tensorflow.python.ops import string_ops from tensorflow.python.platform import test @@ -29,9 +30,10 @@ class StringLengthOpTest(test.TestCase): with self.cached_session() as sess: lengths = string_ops.string_length(strings) - values = sess.run(lengths) + values = self.evaluate(lengths) self.assertAllEqual(values, [[[1, 2], [3, 4], [5, 6]]]) + @test_util.run_deprecated_v1 def testUnit(self): unicode_strings = [u"H\xc3llo", u"\U0001f604"] utf8_strings = [s.encode("utf-8") for s in unicode_strings] @@ -43,14 +45,15 @@ class StringLengthOpTest(test.TestCase): utf8_char_lengths = string_ops.string_length( utf8_strings, unit="UTF8_CHAR") self.assertAllEqual( - sess.run(utf8_byte_lengths), expected_utf8_byte_lengths) + self.evaluate(utf8_byte_lengths), expected_utf8_byte_lengths) self.assertAllEqual( - sess.run(utf8_char_lengths), expected_utf8_char_lengths) + self.evaluate(utf8_char_lengths), expected_utf8_char_lengths) with self.assertRaisesRegexp( ValueError, "Attr 'unit' of 'StringLength' Op passed string 'XYZ' " 'not in: "BYTE", "UTF8_CHAR"'): string_ops.string_length(utf8_strings, unit="XYZ") + @test_util.run_deprecated_v1 def testLegacyPositionalName(self): # Code that predates the 'unit' parameter may have used a positional # argument for the 'name' parameter. Check that we don't break such code. diff --git a/tensorflow/python/kernel_tests/string_split_op_test.py b/tensorflow/python/kernel_tests/string_split_op_test.py index b968e885eda..0c91deb5220 100644 --- a/tensorflow/python/kernel_tests/string_split_op_test.py +++ b/tensorflow/python/kernel_tests/string_split_op_test.py @@ -22,6 +22,7 @@ import numpy as np from tensorflow.python.framework import dtypes from tensorflow.python.framework import errors_impl +from tensorflow.python.framework import test_util from tensorflow.python.ops import array_ops from tensorflow.python.ops import string_ops from tensorflow.python.platform import test @@ -34,17 +35,18 @@ class StringSplitOpTest(test.TestCase): with self.cached_session() as sess: tokens = string_ops.string_split(strings) - indices, values, shape = sess.run(tokens) + indices, values, shape = self.evaluate(tokens) self.assertAllEqual(indices, [[0, 0], [0, 1], [0, 2], [0, 3], [1, 0]]) self.assertAllEqual(values, [b"pigs", b"on", b"the", b"wing", b"animals"]) self.assertAllEqual(shape, [2, 4]) + @test_util.run_deprecated_v1 def testStringSplitEmptyDelimiter(self): strings = ["hello", "hola", b"\xF0\x9F\x98\x8E"] # Last string is U+1F60E with self.cached_session() as sess: tokens = string_ops.string_split(strings, delimiter="") - indices, values, shape = sess.run(tokens) + indices, values, shape = self.evaluate(tokens) self.assertAllEqual(indices, [[0, 0], [0, 1], [0, 2], [0, 3], [0, 4], [1, 0], [1, 1], [1, 2], [1, 3], [2, 0], [2, 1], [2, 2], [2, 3]]) @@ -62,7 +64,7 @@ class StringSplitOpTest(test.TestCase): with self.cached_session() as sess: tokens = string_ops.string_split(strings) - indices, values, shape = sess.run(tokens) + indices, values, shape = self.evaluate(tokens) self.assertAllEqual( indices, [[1, 0], [2, 0], [3, 0], [5, 0], [6, 0], [7, 0], [8, 0]]) @@ -74,13 +76,14 @@ class StringSplitOpTest(test.TestCase): with self.cached_session() as sess: tokens = string_ops.string_split(strings, delimiter=" .") - indices, values, shape = sess.run(tokens) + indices, values, shape = self.evaluate(tokens) self.assertAllEqual( indices, [[1, 0], [2, 0], [3, 0], [5, 0], [6, 0], [7, 0], [8, 0]]) self.assertAllEqual(values, [b"a", b"b", b"c", b"d", b"e", b"f", b"g"]) self.assertAllEqual(shape, [10, 1]) + @test_util.run_deprecated_v1 def testStringSplitWithDelimiter(self): strings = ["hello|world", "hello world"] @@ -92,17 +95,18 @@ class StringSplitOpTest(test.TestCase): ValueError, string_ops.string_split, strings, delimiter=["a"]) tokens = string_ops.string_split(strings, delimiter="|") - indices, values, shape = sess.run(tokens) + indices, values, shape = self.evaluate(tokens) self.assertAllEqual(indices, [[0, 0], [0, 1], [1, 0]]) self.assertAllEqual(values, [b"hello", b"world", b"hello world"]) self.assertAllEqual(shape, [2, 2]) tokens = string_ops.string_split(strings, delimiter="| ") - indices, values, shape = sess.run(tokens) + indices, values, shape = self.evaluate(tokens) self.assertAllEqual(indices, [[0, 0], [0, 1], [1, 0], [1, 1]]) self.assertAllEqual(values, [b"hello", b"world", b"hello", b"world"]) self.assertAllEqual(shape, [2, 2]) + @test_util.run_deprecated_v1 def testStringSplitWithDelimiterTensor(self): strings = ["hello|world", "hello world"] @@ -121,6 +125,7 @@ class StringSplitOpTest(test.TestCase): self.assertAllEqual(values, [b"hello", b"world", b"hello world"]) self.assertAllEqual(shape, [2, 2]) + @test_util.run_deprecated_v1 def testStringSplitWithDelimitersTensor(self): strings = ["hello.cruel,world", "hello cruel world"] @@ -145,7 +150,7 @@ class StringSplitOpTest(test.TestCase): with self.cached_session() as sess: tokens = string_ops.string_split(strings, "#", skip_empty=False) - indices, values, shape = sess.run(tokens) + indices, values, shape = self.evaluate(tokens) self.assertAllEqual(indices, [[0, 0], [0, 1], [1, 0], [1, 1], [2, 0], [2, 1], [2, 2]]) @@ -154,7 +159,7 @@ class StringSplitOpTest(test.TestCase): with self.cached_session() as sess: tokens = string_ops.string_split(strings, "#") - indices, values, shape = sess.run(tokens) + indices, values, shape = self.evaluate(tokens) self.assertAllEqual(values, [b"a", b"b", b"c"]) self.assertAllEqual(indices, [[0, 0], [1, 0], [2, 0]]) self.assertAllEqual(shape, [3, 1]) @@ -167,7 +172,7 @@ class StringSplitV2OpTest(test.TestCase): with self.cached_session() as sess: tokens = string_ops.string_split_v2(strings) - indices, values, shape = sess.run(tokens) + indices, values, shape = self.evaluate(tokens) self.assertAllEqual(indices, [[0, 0], [0, 1], [0, 2], [0, 3], [1, 0]]) self.assertAllEqual(values, [b"pigs", b"on", b"the", b"wing", b"animals"]) self.assertAllEqual(shape, [2, 4]) @@ -182,7 +187,7 @@ class StringSplitV2OpTest(test.TestCase): with self.cached_session() as sess: tokens = string_ops.string_split_v2(strings, sep="<>") - indices, values, shape = sess.run(tokens) + indices, values, shape = self.evaluate(tokens) self.assertAllEqual( indices, [[0, 0], [0, 1], [0, 2], [1, 0], [1, 1], [1, 2], [1, 3], [1, 4], [1, 5], [1, 6]]) @@ -200,7 +205,7 @@ class StringSplitV2OpTest(test.TestCase): with self.cached_session() as sess: tokens = string_ops.string_split_v2(strings, sep=',') - indices, values, shape = sess.run(tokens) + indices, values, shape = self.evaluate(tokens) self.assertAllEqual(indices, [[0, 0], [0, 1], [0, 2], [1, 0], [1, 1], [1, 2], [1, 3], [1, 4]]) self.assertAllEqual(values, [b"1", b"2", b"3", @@ -217,7 +222,7 @@ class StringSplitV2OpTest(test.TestCase): with self.cached_session() as sess: tokens = string_ops.string_split_v2(strings) - indices, values, shape = sess.run(tokens) + indices, values, shape = self.evaluate(tokens) self.assertAllEqual(indices, [[0, 0], [0, 1], [0, 2], [1, 0], [1, 1], [1, 2]]) self.assertAllEqual(values, [b"1", b"2", b"3", b"4", b"5", b"6"]) @@ -233,7 +238,7 @@ class StringSplitV2OpTest(test.TestCase): with self.cached_session() as sess: tokens = string_ops.string_split_v2(strings, sep=',', maxsplit=1) - indices, values, shape = sess.run(tokens) + indices, values, shape = self.evaluate(tokens) self.assertAllEqual(indices, [[0, 0], [0, 1], [1, 0], [1, 1]]) self.assertAllEqual(values, [b"1", b"2,3", b"4", b"5,,6,"]) @@ -249,7 +254,7 @@ class StringSplitV2OpTest(test.TestCase): with self.cached_session() as sess: tokens = string_ops.string_split_v2(strings, maxsplit=1) - indices, values, shape = sess.run(tokens) + indices, values, shape = self.evaluate(tokens) self.assertAllEqual(indices, [[0, 0], [0, 1], [1, 0], [1, 1]]) self.assertAllEqual(values, [b"1", b"2 3", b"4", b"5 6 "]) diff --git a/tensorflow/python/kernel_tests/string_strip_op_test.py b/tensorflow/python/kernel_tests/string_strip_op_test.py index 1e404b71462..edff3862ff6 100644 --- a/tensorflow/python/kernel_tests/string_strip_op_test.py +++ b/tensorflow/python/kernel_tests/string_strip_op_test.py @@ -30,7 +30,7 @@ class StringStripOpTest(test.TestCase): with self.cached_session() as sess: output = string_ops.string_strip(strings) - output = sess.run(output) + output = self.evaluate(output) self.assertAllEqual(output, [b"pigs on the wing", b"animals"]) def test_string_strip_2d(self): @@ -39,7 +39,7 @@ class StringStripOpTest(test.TestCase): with self.cached_session() as sess: output = string_ops.string_strip(strings) - output = sess.run(output) + output = self.evaluate(output) self.assertAllEqual(output, [[b"pigs on the wing", b"animals"], [b"hello", b"world"]]) @@ -48,7 +48,7 @@ class StringStripOpTest(test.TestCase): with self.cached_session() as sess: output = string_ops.string_strip(strings) - output = sess.run(output) + output = self.evaluate(output) self.assertAllEqual(output, [b"hello", b"", b"world", b""]) diff --git a/tensorflow/python/kernel_tests/string_to_hash_bucket_op_test.py b/tensorflow/python/kernel_tests/string_to_hash_bucket_op_test.py index 9cb0c9d18f3..25f573fc144 100644 --- a/tensorflow/python/kernel_tests/string_to_hash_bucket_op_test.py +++ b/tensorflow/python/kernel_tests/string_to_hash_bucket_op_test.py @@ -19,6 +19,7 @@ from __future__ import print_function from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes +from tensorflow.python.framework import test_util from tensorflow.python.ops import array_ops from tensorflow.python.ops import string_ops from tensorflow.python.platform import test @@ -26,6 +27,7 @@ from tensorflow.python.platform import test class StringToHashBucketOpTest(test.TestCase): + @test_util.run_deprecated_v1 def testStringToOneHashBucketFast(self): with self.cached_session(): input_string = array_ops.placeholder(dtypes.string) @@ -34,6 +36,7 @@ class StringToHashBucketOpTest(test.TestCase): self.assertAllEqual([0, 0, 0], result) + @test_util.run_deprecated_v1 def testStringToHashBucketsFast(self): with self.cached_session(): input_string = array_ops.placeholder(dtypes.string) @@ -46,6 +49,7 @@ class StringToHashBucketOpTest(test.TestCase): # Fingerprint64('d') -> 4470636696479570465 -> mod 10 -> 5 self.assertAllEqual([9, 2, 2, 5], result) + @test_util.run_deprecated_v1 def testStringToOneHashBucketLegacyHash(self): with self.cached_session(): input_string = array_ops.placeholder(dtypes.string) @@ -54,6 +58,7 @@ class StringToHashBucketOpTest(test.TestCase): self.assertAllEqual([0, 0, 0], result) + @test_util.run_deprecated_v1 def testStringToHashBucketsLegacyHash(self): with self.cached_session(): input_string = array_ops.placeholder(dtypes.string) @@ -70,7 +75,7 @@ class StringToHashBucketOpTest(test.TestCase): input_string = constant_op.constant(['a', 'b', 'c']) output = string_ops.string_to_hash_bucket_strong( input_string, 1, key=[123, 345]) - self.assertAllEqual([0, 0, 0], output.eval()) + self.assertAllEqual([0, 0, 0], self.evaluate(output)) def testStringToHashBucketsStrong(self): with self.cached_session(): @@ -81,7 +86,7 @@ class StringToHashBucketOpTest(test.TestCase): # StrongKeyedHash(key, 'a') -> 7157389809176466784 -> mod 10 -> 4 # StrongKeyedHash(key, 'b') -> 15805638358933211562 -> mod 10 -> 2 # StrongKeyedHash(key, 'c') -> 18100027895074076528 -> mod 10 -> 8 - self.assertAllEqual([4, 2, 8], output.eval()) + self.assertAllEqual([4, 2, 8], self.evaluate(output)) def testStringToHashBucketsStrongInvalidKey(self): with self.cached_session(): diff --git a/tensorflow/python/kernel_tests/string_to_number_op_test.py b/tensorflow/python/kernel_tests/string_to_number_op_test.py index 99ee25e1253..49ccfd1028f 100644 --- a/tensorflow/python/kernel_tests/string_to_number_op_test.py +++ b/tensorflow/python/kernel_tests/string_to_number_op_test.py @@ -19,6 +19,7 @@ from __future__ import division from __future__ import print_function from tensorflow.python.framework import dtypes +from tensorflow.python.framework import test_util from tensorflow.python.ops import array_ops from tensorflow.python.ops import parsing_ops from tensorflow.python.platform import test @@ -45,6 +46,7 @@ class StringToNumberOpTest(test.TestCase): with self.assertRaisesOpError(outstr): output.eval(feed_dict={input_string: [instr]}) + @test_util.run_deprecated_v1 def testToFloat(self): self._test(dtypes.float32, [("0", 0), ("3", 3), ("-1", -1), @@ -58,6 +60,7 @@ class StringToNumberOpTest(test.TestCase): ("INF", float("INF"))], [("10foobar", _ERROR_MESSAGE + "10foobar")]) + @test_util.run_deprecated_v1 def testToDouble(self): self._test(dtypes.float64, [("0", 0), ("3", 3), ("-1", -1), @@ -71,6 +74,7 @@ class StringToNumberOpTest(test.TestCase): ("INF", float("INF"))], [("10foobar", _ERROR_MESSAGE + "10foobar")]) + @test_util.run_deprecated_v1 def testToInt32(self): self._test(dtypes.int32, [("0", 0), ("3", 3), ("-1", -1), @@ -84,6 +88,7 @@ class StringToNumberOpTest(test.TestCase): ("2.9", _ERROR_MESSAGE + "2.9"), ("10foobar", _ERROR_MESSAGE + "10foobar")]) + @test_util.run_deprecated_v1 def testToInt64(self): self._test(dtypes.int64, [("0", 0), ("3", 3), ("-1", -1), diff --git a/tensorflow/python/kernel_tests/substr_op_test.py b/tensorflow/python/kernel_tests/substr_op_test.py index 37aa624b07e..9302152e82b 100644 --- a/tensorflow/python/kernel_tests/substr_op_test.py +++ b/tensorflow/python/kernel_tests/substr_op_test.py @@ -22,6 +22,7 @@ from absl.testing import parameterized import numpy as np from tensorflow.python.framework import errors_impl +from tensorflow.python.framework import test_util from tensorflow.python.ops import string_ops from tensorflow.python.platform import test @@ -51,7 +52,7 @@ class SubstrOpTest(test.TestCase, parameterized.TestCase): length = np.array(3, dtype) substr_op = string_ops.substr(test_string, position, length, unit=unit) with self.cached_session(): - substr = substr_op.eval() + substr = self.evaluate(substr_op) self.assertAllEqual(substr, expected_value) @parameterized.parameters( @@ -71,7 +72,7 @@ class SubstrOpTest(test.TestCase, parameterized.TestCase): length = np.array(3, dtype) substr_op = string_ops.substr(test_string, position, length, unit=unit) with self.cached_session(): - substr = substr_op.eval() + substr = self.evaluate(substr_op) self.assertAllEqual(substr, expected_value) # Full string @@ -83,7 +84,7 @@ class SubstrOpTest(test.TestCase, parameterized.TestCase): length = np.array(5, dtype) substr_op = string_ops.substr(test_string, position, length, unit=unit) with self.cached_session(): - substr = substr_op.eval() + substr = self.evaluate(substr_op) self.assertAllEqual(substr, test_string) # Full string (Negative) @@ -95,7 +96,7 @@ class SubstrOpTest(test.TestCase, parameterized.TestCase): length = np.array(5, dtype) substr_op = string_ops.substr(test_string, position, length, unit=unit) with self.cached_session(): - substr = substr_op.eval() + substr = self.evaluate(substr_op) self.assertAllEqual(substr, test_string) # Length is larger in magnitude than a negative position @@ -111,7 +112,7 @@ class SubstrOpTest(test.TestCase, parameterized.TestCase): length = np.array(5, dtype) substr_op = string_ops.substr(test_string, position, length, unit=unit) with self.cached_session(): - substr = substr_op.eval() + substr = self.evaluate(substr_op) self.assertAllEqual(substr, expected_string) @parameterized.parameters( @@ -138,7 +139,7 @@ class SubstrOpTest(test.TestCase, parameterized.TestCase): length = np.array(3, dtype) substr_op = string_ops.substr(test_string, position, length, unit=unit) with self.cached_session(): - substr = substr_op.eval() + substr = self.evaluate(substr_op) self.assertAllEqual(substr, expected_value) @parameterized.parameters( @@ -173,7 +174,7 @@ class SubstrOpTest(test.TestCase, parameterized.TestCase): }[unit] substr_op = string_ops.substr(test_string, position, length, unit=unit) with self.cached_session(): - substr = substr_op.eval() + substr = self.evaluate(substr_op) self.assertAllEqual(substr, expected_value) position = np.array(-3, dtype) @@ -188,7 +189,7 @@ class SubstrOpTest(test.TestCase, parameterized.TestCase): }[unit] substr_op = string_ops.substr(test_string, position, length, unit=unit) with self.cached_session(): - substr = substr_op.eval() + substr = self.evaluate(substr_op) self.assertAllEqual(substr, expected_value) @parameterized.parameters( @@ -229,7 +230,7 @@ class SubstrOpTest(test.TestCase, parameterized.TestCase): }[unit] substr_op = string_ops.substr(test_string, position, length, unit=unit) with self.cached_session(): - substr = substr_op.eval() + substr = self.evaluate(substr_op) self.assertAllEqual(substr, expected_value) @parameterized.parameters( @@ -271,7 +272,7 @@ class SubstrOpTest(test.TestCase, parameterized.TestCase): }[unit] substr_op = string_ops.substr(test_string, position, length, unit=unit) with self.cached_session(): - substr = substr_op.eval() + substr = self.evaluate(substr_op) self.assertAllEqual(substr, expected_value) # Broadcast input string onto pos/len @@ -294,7 +295,7 @@ class SubstrOpTest(test.TestCase, parameterized.TestCase): }[unit] substr_op = string_ops.substr(test_string, position, length, unit=unit) with self.cached_session(): - substr = substr_op.eval() + substr = self.evaluate(substr_op) self.assertAllEqual(substr, expected_value) # Test 1D broadcast @@ -310,7 +311,7 @@ class SubstrOpTest(test.TestCase, parameterized.TestCase): }[unit] substr_op = string_ops.substr(test_string, position, length, unit=unit) with self.cached_session(): - substr = substr_op.eval() + substr = self.evaluate(substr_op) self.assertAllEqual(substr, expected_value) @parameterized.parameters( @@ -319,6 +320,7 @@ class SubstrOpTest(test.TestCase, parameterized.TestCase): (np.int32, "UTF8_CHAR"), (np.int64, "UTF8_CHAR"), ) + @test_util.run_deprecated_v1 def testBadBroadcast(self, dtype, unit): test_string = [[b"ten", b"eleven", b"twelve"], [b"thirteen", b"fourteen", b"fifteen"], @@ -338,6 +340,7 @@ class SubstrOpTest(test.TestCase, parameterized.TestCase): (np.int32, -6, "UTF8_CHAR"), (np.int64, -6, "UTF8_CHAR"), ) + @test_util.run_deprecated_v1 def testOutOfRangeError_Scalar(self, dtype, pos, unit): # Scalar/Scalar test_string = { @@ -349,7 +352,7 @@ class SubstrOpTest(test.TestCase, parameterized.TestCase): substr_op = string_ops.substr(test_string, position, length, unit=unit) with self.cached_session(): with self.assertRaises(errors_impl.InvalidArgumentError): - substr_op.eval() + self.evaluate(substr_op) @parameterized.parameters( (np.int32, 4, "BYTE"), @@ -361,6 +364,7 @@ class SubstrOpTest(test.TestCase, parameterized.TestCase): (np.int32, -4, "UTF8_CHAR"), (np.int64, -4, "UTF8_CHAR"), ) + @test_util.run_deprecated_v1 def testOutOfRangeError_VectorScalar(self, dtype, pos, unit): # Vector/Scalar test_string = { @@ -373,7 +377,7 @@ class SubstrOpTest(test.TestCase, parameterized.TestCase): substr_op = string_ops.substr(test_string, position, length, unit=unit) with self.cached_session(): with self.assertRaises(errors_impl.InvalidArgumentError): - substr_op.eval() + self.evaluate(substr_op) @parameterized.parameters( (np.int32, "BYTE"), @@ -381,6 +385,7 @@ class SubstrOpTest(test.TestCase, parameterized.TestCase): (np.int32, "UTF8_CHAR"), (np.int64, "UTF8_CHAR"), ) + @test_util.run_deprecated_v1 def testOutOfRangeError_MatrixMatrix(self, dtype, unit): # Matrix/Matrix test_string = { @@ -398,7 +403,7 @@ class SubstrOpTest(test.TestCase, parameterized.TestCase): substr_op = string_ops.substr(test_string, position, length, unit=unit) with self.cached_session(): with self.assertRaises(errors_impl.InvalidArgumentError): - substr_op.eval() + self.evaluate(substr_op) # Matrix/Matrix (with negative) position = np.array([[1, 2, -3], [1, 2, -4], [1, 2, -3]], dtype) @@ -406,7 +411,7 @@ class SubstrOpTest(test.TestCase, parameterized.TestCase): substr_op = string_ops.substr(test_string, position, length, unit=unit) with self.cached_session(): with self.assertRaises(errors_impl.InvalidArgumentError): - substr_op.eval() + self.evaluate(substr_op) @parameterized.parameters( (np.int32, "BYTE"), @@ -414,6 +419,7 @@ class SubstrOpTest(test.TestCase, parameterized.TestCase): (np.int32, "UTF8_CHAR"), (np.int64, "UTF8_CHAR"), ) + @test_util.run_deprecated_v1 def testOutOfRangeError_Broadcast(self, dtype, unit): # Broadcast test_string = { @@ -428,7 +434,7 @@ class SubstrOpTest(test.TestCase, parameterized.TestCase): substr_op = string_ops.substr(test_string, position, length, unit=unit) with self.cached_session(): with self.assertRaises(errors_impl.InvalidArgumentError): - substr_op.eval() + self.evaluate(substr_op) # Broadcast (with negative) position = np.array([-1, -2, -4], dtype) @@ -436,7 +442,7 @@ class SubstrOpTest(test.TestCase, parameterized.TestCase): substr_op = string_ops.substr(test_string, position, length, unit=unit) with self.cached_session(): with self.assertRaises(errors_impl.InvalidArgumentError): - substr_op.eval() + self.evaluate(substr_op) @parameterized.parameters( (np.int32, "BYTE"), @@ -444,6 +450,7 @@ class SubstrOpTest(test.TestCase, parameterized.TestCase): (np.int32, "UTF8_CHAR"), (np.int64, "UTF8_CHAR"), ) + @test_util.run_deprecated_v1 def testMismatchPosLenShapes(self, dtype, unit): test_string = { "BYTE": [[b"ten", b"eleven", b"twelve"], @@ -471,6 +478,7 @@ class SubstrOpTest(test.TestCase, parameterized.TestCase): with self.assertRaises(ValueError): string_ops.substr(test_string, position, length) + @test_util.run_deprecated_v1 def testWrongDtype(self): with self.cached_session(): with self.assertRaises(TypeError): @@ -478,6 +486,7 @@ class SubstrOpTest(test.TestCase, parameterized.TestCase): with self.assertRaises(TypeError): string_ops.substr(b"test", 3, 1.0) + @test_util.run_deprecated_v1 def testInvalidUnit(self): with self.cached_session(): with self.assertRaises(ValueError): diff --git a/tensorflow/python/kernel_tests/summary_v1_audio_op_test.py b/tensorflow/python/kernel_tests/summary_v1_audio_op_test.py index 63ce77b9d55..1547c55f8b0 100644 --- a/tensorflow/python/kernel_tests/summary_v1_audio_op_test.py +++ b/tensorflow/python/kernel_tests/summary_v1_audio_op_test.py @@ -60,7 +60,7 @@ class SummaryV1AudioOpTest(test.TestCase): sample_rate = 8000 summ = summary.audio( "snd", const, max_outputs=3, sample_rate=sample_rate) - value = sess.run(summ) + value = self.evaluate(summ) self.assertEqual([], summ.get_shape()) audio_summ = self._AsSummary(value) diff --git a/tensorflow/python/kernel_tests/summary_v1_image_op_test.py b/tensorflow/python/kernel_tests/summary_v1_image_op_test.py index 094606944ff..56de2e933db 100644 --- a/tensorflow/python/kernel_tests/summary_v1_image_op_test.py +++ b/tensorflow/python/kernel_tests/summary_v1_image_op_test.py @@ -24,6 +24,7 @@ from six.moves import xrange # pylint: disable=redefined-builtin from tensorflow.core.framework import summary_pb2 from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops +from tensorflow.python.framework import test_util from tensorflow.python.ops import image_ops import tensorflow.python.ops.nn_grad # pylint: disable=unused-import from tensorflow.python.platform import test @@ -49,6 +50,7 @@ class SummaryV1ImageOpTest(test.TestCase): }""" % ((i,) + shape[1:]) for i in xrange(3)) self.assertProtoEquals(expected, image_summ) + @test_util.run_deprecated_v1 def testImageSummary(self): for depth in (1, 3, 4): for positive in False, True: @@ -70,7 +72,7 @@ class SummaryV1ImageOpTest(test.TestCase): # Summarize summ = summary.image("img", const) - value = sess.run(summ) + value = self.evaluate(summ) self.assertEqual([], summ.get_shape()) image_summ = self._AsSummary(value) @@ -84,6 +86,7 @@ class SummaryV1ImageOpTest(test.TestCase): # Check the rest of the proto self._CheckProto(image_summ, shape) + @test_util.run_deprecated_v1 def testImageSummaryUint8(self): np.random.seed(7) for depth in (1, 3, 4): @@ -97,7 +100,7 @@ class SummaryV1ImageOpTest(test.TestCase): # Summarize summ = summary.image("img", tf_images) - value = sess.run(summ) + value = self.evaluate(summ) self.assertEqual([], summ.get_shape()) image_summ = self._AsSummary(value) diff --git a/tensorflow/python/kernel_tests/summary_v1_ops_test.py b/tensorflow/python/kernel_tests/summary_v1_ops_test.py index 6c4e106b118..e070f5bf6f5 100644 --- a/tensorflow/python/kernel_tests/summary_v1_ops_test.py +++ b/tensorflow/python/kernel_tests/summary_v1_ops_test.py @@ -26,6 +26,7 @@ from __future__ import print_function from tensorflow.core.framework import summary_pb2 from tensorflow.python.framework import constant_op from tensorflow.python.framework import ops +from tensorflow.python.framework import test_util from tensorflow.python.ops import logging_ops from tensorflow.python.platform import test from tensorflow.python.summary import summary @@ -42,7 +43,7 @@ class SummaryV1OpsTest(test.TestCase): with self.cached_session() as sess: const = constant_op.constant([10.0, 20.0]) summ = logging_ops.scalar_summary(["c1", "c2"], const, name="mysumm") - value = sess.run(summ) + value = self.evaluate(summ) self.assertEqual([], summ.get_shape()) self.assertProtoEquals(""" value { tag: "c1" simple_value: 10.0 } @@ -53,20 +54,21 @@ class SummaryV1OpsTest(test.TestCase): with self.cached_session() as sess: const = constant_op.constant([10.0, 20.0]) summ = logging_ops.scalar_summary(["c1", "c2"], const) - value = sess.run(summ) + value = self.evaluate(summ) self.assertEqual([], summ.get_shape()) self.assertProtoEquals(""" value { tag: "c1" simple_value: 10.0 } value { tag: "c2" simple_value: 20.0 } """, self._AsSummary(value)) + @test_util.run_deprecated_v1 def testMergeSummary(self): with self.cached_session() as sess: const = constant_op.constant(10.0) summ1 = summary.histogram("h", const) summ2 = logging_ops.scalar_summary("c", const) merge = summary.merge([summ1, summ2]) - value = sess.run(merge) + value = self.evaluate(merge) self.assertEqual([], merge.get_shape()) self.assertProtoEquals(""" value { diff --git a/tensorflow/python/kernel_tests/summary_v1_tensor_op_test.py b/tensorflow/python/kernel_tests/summary_v1_tensor_op_test.py index 34f771679ae..b8e5b5b882a 100644 --- a/tensorflow/python/kernel_tests/summary_v1_tensor_op_test.py +++ b/tensorflow/python/kernel_tests/summary_v1_tensor_op_test.py @@ -50,7 +50,7 @@ class SummaryV1TensorOpTest(test.TestCase): with ops.name_scope("zod"): s3 = summary_lib.tensor_summary("s3", c) s4 = summary_lib.tensor_summary("TensorSummary", c) - summ1, summ2, summ3, summ4 = sess.run([s1, s2, s3, s4]) + summ1, summ2, summ3, summ4 = self.evaluate([s1, s2, s3, s4]) v1 = self._SummarySingleValue(summ1) self.assertEqual(v1.tag, "s1") @@ -68,7 +68,7 @@ class SummaryV1TensorOpTest(test.TestCase): with self.cached_session() as sess: const = constant_op.constant(10.0) summ = summary_lib.tensor_summary("foo", const) - result = sess.run(summ) + result = self.evaluate(summ) value = self._SummarySingleValue(result) n = tensor_util.MakeNdarray(value.tensor) @@ -79,7 +79,7 @@ class SummaryV1TensorOpTest(test.TestCase): with self.cached_session() as sess: const = constant_op.constant(s) summ = summary_lib.tensor_summary("foo", const) - result = sess.run(summ) + result = self.evaluate(summ) value = self._SummarySingleValue(result) n = tensor_util.MakeNdarray(value.tensor) @@ -89,7 +89,7 @@ class SummaryV1TensorOpTest(test.TestCase): with self.cached_session() as sess: const = array_ops.ones([5, 5, 5]) summ = summary_lib.tensor_summary("foo", const) - result = sess.run(summ) + result = self.evaluate(summ) value = self._SummarySingleValue(result) n = tensor_util.MakeNdarray(value.tensor) self._AssertNumpyEq(n, np.ones([5, 5, 5])) @@ -99,7 +99,7 @@ class SummaryV1TensorOpTest(test.TestCase): with self.cached_session() as sess: const = constant_op.constant(strings) summ = summary_lib.tensor_summary("foo", const) - result = sess.run(summ) + result = self.evaluate(summ) value = self._SummarySingleValue(result) n = tensor_util.MakeNdarray(value.tensor) self._AssertNumpyEq(n, strings) @@ -109,7 +109,7 @@ class SummaryV1TensorOpTest(test.TestCase): with self.cached_session() as sess: const = constant_op.constant(bools) summ = summary_lib.tensor_summary("foo", const) - result = sess.run(summ) + result = self.evaluate(summ) value = self._SummarySingleValue(result) n = tensor_util.MakeNdarray(value.tensor) @@ -119,7 +119,7 @@ class SummaryV1TensorOpTest(test.TestCase): with self.cached_session() as sess: def get_description(summary_op): - summ_str = sess.run(summary_op) + summ_str = self.evaluate(summary_op) summ = summary_pb2.Summary() summ.ParseFromString(summ_str) return summ.value[0].metadata diff --git a/tensorflow/python/kernel_tests/svd_op_test.py b/tensorflow/python/kernel_tests/svd_op_test.py index 57298c0fecc..97a280ef51c 100644 --- a/tensorflow/python/kernel_tests/svd_op_test.py +++ b/tensorflow/python/kernel_tests/svd_op_test.py @@ -68,7 +68,7 @@ class SvdOpTest(test.TestCase): s2 = linalg_ops.svd( matrix2, compute_uv=compute_uv_, full_matrices=full_matrices_) all_ops += [s1, s2] - val = sess.run(all_ops) + val = self.evaluate(all_ops) for i in range(2): s = 6 * i self.assertAllEqual(val[s], val[s + 3]) # s1 == s2 @@ -123,7 +123,7 @@ def _GetSvdOpTest(dtype_, shape_, use_static_shape_, compute_uv_, # Tests that x[...,:,:]^H * x[...,:,:] is close to the identity. xx = math_ops.matmul(x, x, adjoint_a=True) identity = array_ops.matrix_band_part(array_ops.ones_like(xx), 0, 0) - self.assertAllClose(identity.eval(), xx.eval(), atol=tol) + self.assertAllClose(identity.eval(), self.evaluate(xx), atol=tol) def Test(self): is_complex = dtype_ in (np.complex64, np.complex128) @@ -150,7 +150,7 @@ def _GetSvdOpTest(dtype_, shape_, use_static_shape_, compute_uv_, s_tf, u_tf, v_tf = linalg_ops.svd( x_tf, compute_uv=compute_uv_, full_matrices=full_matrices_) if use_static_shape_: - s_tf_val, u_tf_val, v_tf_val = sess.run([s_tf, u_tf, v_tf]) + s_tf_val, u_tf_val, v_tf_val = self.evaluate([s_tf, u_tf, v_tf]) else: s_tf_val, u_tf_val, v_tf_val = sess.run( [s_tf, u_tf, v_tf], feed_dict={x_tf: x_np}) @@ -158,7 +158,7 @@ def _GetSvdOpTest(dtype_, shape_, use_static_shape_, compute_uv_, s_tf = linalg_ops.svd( x_tf, compute_uv=compute_uv_, full_matrices=full_matrices_) if use_static_shape_: - s_tf_val = sess.run(s_tf) + s_tf_val = self.evaluate(s_tf) else: s_tf_val = sess.run(s_tf, feed_dict={x_tf: x_np}) diff --git a/tensorflow/python/kernel_tests/template_test.py b/tensorflow/python/kernel_tests/template_test.py index 9dcdaa61ed2..3b2a56bd1ff 100644 --- a/tensorflow/python/kernel_tests/template_test.py +++ b/tensorflow/python/kernel_tests/template_test.py @@ -72,6 +72,7 @@ def variable_scoped_function_with_local_variable(): class TemplateTest(test.TestCase): + @test_util.run_deprecated_v1 def test_end_to_end(self): """This test shows a very simple line model with test_loss. @@ -104,10 +105,10 @@ class TemplateTest(test.TestCase): train_op = optimizer.minimize(train_loss) with session.Session() as sess: - sess.run(variables.global_variables_initializer()) - initial_test_loss = sess.run(test_loss) - sess.run(train_op) - final_test_loss = sess.run(test_loss) + self.evaluate(variables.global_variables_initializer()) + initial_test_loss = self.evaluate(test_loss) + self.evaluate(train_op) + final_test_loss = self.evaluate(test_loss) # Parameters are tied, so the loss should have gone down when we trained it. self.assertLess(final_test_loss, initial_test_loss) @@ -172,6 +173,7 @@ class TemplateTest(test.TestCase): self.assertEqual("s1/dummy:0", v1.name) self.assertEqual("s1_1/dummy:0", v3.name) + @test_util.run_deprecated_v1 def test_same_unique_name_raise_error(self): tmpl1 = template.make_template( "_", variable_scoped_function, unique_name_="s1") @@ -190,6 +192,7 @@ class TemplateTest(test.TestCase): template.make_template( "_", variable_scoped_function, unique_name_="s1") + @test_util.run_deprecated_v1 def test_unique_name_and_reuse(self): tmpl1 = template.make_template( "_", variable_scoped_function, unique_name_="s1") @@ -260,6 +263,7 @@ class TemplateTest(test.TestCase): self.assertEqual("s1/test/dummy:0", v1.name) self.assertEqual("s1_1/test/dummy:0", v3.name) + @test_util.run_deprecated_v1 def test_enforces_no_extra_trainable_variables(self): tmpl = template.make_template("s", function_with_create, trainable=True) @@ -675,6 +679,7 @@ class TemplateTest(test.TestCase): self.assertEqual(1, len(tb.variables)) # TODO(apassos) handle local variables in Eager + @test_util.run_deprecated_v1 def test_local_variables(self): # Make sure trainable_variables are created. with variable_scope.variable_scope("foo3"): diff --git a/tensorflow/python/kernel_tests/tensor_array_ops_test.py b/tensorflow/python/kernel_tests/tensor_array_ops_test.py index 0188eb246f0..884c04eb7ac 100644 --- a/tensorflow/python/kernel_tests/tensor_array_ops_test.py +++ b/tensorflow/python/kernel_tests/tensor_array_ops_test.py @@ -63,6 +63,8 @@ def _make_ta(size, name, dtype=dtypes.float32, infer_shape=False): dtype=dtype, tensor_array_name=name, size=size, infer_shape=infer_shape) +@test_util.run_all_in_graph_and_eager_modes +@test_util.with_control_flow_v2 class TensorArrayTest(test.TestCase): @classmethod @@ -123,11 +125,9 @@ class TensorArrayTest(test.TestCase): self._testTensorArrayWritePack(dtypes.complex128) self._testTensorArrayWritePack(dtypes.string) - @test_util.run_in_graph_and_eager_modes def testTensorArrayWritePack(self): self._testTensorArrayWritePackMaybeLegacy() - @test_util.run_in_graph_and_eager_modes def testEmptyTensorArrayPack(self): with self.session(use_gpu=True): ta = tensor_array_ops.TensorArray( @@ -161,7 +161,7 @@ class TensorArrayTest(test.TestCase): convert([[4.0, 5.0], [104.0, 105.0], [204.0, 205.0], [6.0, 7.0], [106.0, 107.0], [8.0, 9.0]]), c0) - @test_util.run_in_graph_and_eager_modes + @test_util.run_deprecated_v1 def testTensorArrayWriteConcat(self): self._testTensorArrayWriteConcat(dtypes.float32) self._testTensorArrayWriteConcat(dtypes.float64) @@ -184,7 +184,8 @@ class TensorArrayTest(test.TestCase): self.assertAllEqual([[0.0, 0.0], [4.0, 5.0], [0.0, 0.0]], self.evaluate(ta.write(1, [[4.0, 5.0]]).concat())) - @test_util.run_in_graph_and_eager_modes + @test_util.disable_control_flow_v2("b/118890905") + @test_util.run_v1_only("b/118890905") def testTensorArrayReadOrPackNotAllValuesAvailableFillsZeros(self): self._testTensorArrayReadOrPackNotAllValuesAvailableFillsZeros() @@ -200,7 +201,8 @@ class TensorArrayTest(test.TestCase): self.assertAllEqual([[0.0, 0.0], [4.0, 5.0], [0.0, 0.0]], self.evaluate(ta.write(1, [[4.0, 5.0]]).concat())) - @test_util.run_in_graph_and_eager_modes + @test_util.disable_control_flow_v2("b/118890905") + @test_util.run_v1_only("b/118890905") def testTensorArrayReadOrPackNotAllValuesAvailableInferShapeFillsZeros(self): self._testTensorArrayReadOrPackNotAllValuesAvailableInferShapeFillsZeros() @@ -251,7 +253,6 @@ class TensorArrayTest(test.TestCase): self._testTensorArrayUnpackRead(dtypes.complex128) self._testTensorArrayUnpackRead(dtypes.string) - @test_util.run_in_graph_and_eager_modes def testTensorArrayUnpackRead(self): self._testTensorArrayUnpackReadMaybeLegacy() @@ -297,7 +298,7 @@ class TensorArrayTest(test.TestCase): self.assertAllEqual(convert([]).reshape(0, 2), d1) self.assertAllEqual(convert([[3.0, 301.0]]), d2) - @test_util.run_in_graph_and_eager_modes + @test_util.run_deprecated_v1 def testTensorArraySplitRead(self): self._testTensorArraySplitRead(dtypes.float32) self._testTensorArraySplitRead(dtypes.float64) @@ -307,7 +308,9 @@ class TensorArrayTest(test.TestCase): self._testTensorArraySplitRead(dtypes.complex128) self._testTensorArraySplitRead(dtypes.string) - def testTensorGradArrayWriteRead(self): + @test_util.disable_control_flow_v2("v2 does not support TensorArray.grad.") + @test_util.run_deprecated_v1 + def testSkipEagerTensorGradArrayWriteRead(self): with self.session(use_gpu=True) as session: ta = tensor_array_ops.TensorArray( dtype=dtypes.float32, @@ -340,7 +343,29 @@ class TensorArrayTest(test.TestCase): self.assertAllEqual([[2.0]], g_d1) self.assertAllEqual(-2.0, g_d2) - def testTensorGradArrayDynamicWriteRead(self): + @test_util.run_deprecated_v1 + def testSkipEagerTensorArrayGradGrad(self): + if not tensor_array_ops.ENABLE_TENSOR_ARRAY_V2: + self.skipTest("Legacy TensorArray does not support double derivatives.") + with self.test_session(use_gpu=True) as session: + x = constant_op.constant(4.0) + + ta = tensor_array_ops.TensorArray( + dtype=dtypes.float32, + tensor_array_name="foo", + size=1, + infer_shape=False) + w0 = ta.write(0, x) + r0 = w0.read(0) + y = r0 * r0 + + g1 = gradients_impl.gradients(ys=[y], xs=[x]) + g2 = gradients_impl.gradients(ys=[g1], xs=[x]) + self.assertAllEqual([2.0], session.run(g2)) + + @test_util.disable_control_flow_v2("v2 does not support TensorArray.grad.") + @test_util.run_deprecated_v1 + def testSkipEagerTensorGradArrayDynamicWriteRead(self): with self.session(use_gpu=True) as session: ta = tensor_array_ops.TensorArray( dtype=dtypes.float32, @@ -381,7 +406,9 @@ class TensorArrayTest(test.TestCase): self.assertAllEqual(3, vs) self.assertAllEqual(3, g_vs) - def testTensorGradAccessTwiceReceiveSameObject(self): + @test_util.disable_control_flow_v2("v2 does not support TensorArray.grad.") + @test_util.run_deprecated_v1 + def testSkipEagerTensorGradAccessTwiceReceiveSameObject(self): with self.session(use_gpu=True) as session: ta = tensor_array_ops.TensorArray( dtype=dtypes.float32, tensor_array_name="foo", size=3) @@ -397,26 +424,41 @@ class TensorArrayTest(test.TestCase): self.assertAllEqual(t_g_ta_0, t_g_ta_1) self.assertAllEqual([[4.0, 5.0]], d_r1_0) - @test_util.run_in_graph_and_eager_modes + @test_util.run_deprecated_v1 def testTensorArrayWriteWrongIndexOrDataTypeFails(self): with self.session(use_gpu=True): ta = _make_ta(3, "foo", dtype=dtypes.float32) # Test writing the wrong datatype - with self.assertRaisesOpError( - "TensorArray dtype is (float|float32) but Op is trying to write " - "dtype string"): + if (tensor_array_ops.ENABLE_TENSOR_ARRAY_V2 and + not context.executing_eagerly()): + error_msg = ("Invalid data types; op elements string but list elements " + "float") + else: + error_msg = ( + "TensorArray dtype is (float|float32) but Op is trying to write " + "dtype string") + with self.assertRaisesOpError(error_msg): self.evaluate(ta.write(0, "wrong_type_scalar").flow) - with self.assertRaisesOpError("index -1"): + if (tensor_array_ops.ENABLE_TENSOR_ARRAY_V2 and + not context.executing_eagerly()): + error_msg = "Trying to modify element -1 in a list with 3 elements." + else: + error_msg = "index -1" + with self.assertRaisesOpError(error_msg): self.evaluate(ta.write(-1, 3.0).flow) + if (tensor_array_ops.ENABLE_TENSOR_ARRAY_V2 and + not context.executing_eagerly()): + error_msg = "Trying to modify element 3 in a list with 3 elements" + else: + error_msg = ("Tried to write to index 3 but array is not " + "resizeable and size is: 3") # Test reading from too large an index - with self.assertRaisesOpError( - "Tried to write to index 3 but array is not " - "resizeable and size is: 3"): + with self.assertRaisesOpError(error_msg): self.evaluate(ta.write(3, 3.0).flow) - @test_util.run_in_graph_and_eager_modes + @test_util.run_deprecated_v1 def testTensorArrayReadWrongIndexOrDataTypeFails(self): with self.session(use_gpu=True): ta = _make_ta(3, "foo", dtype=dtypes.float32) @@ -424,23 +466,35 @@ class TensorArrayTest(test.TestCase): w0 = ta.write(0, [[4.0, 5.0]]) # Test reading wrong datatype (only possible when constructing graphs). - if not context.executing_eagerly(): + if (not context.executing_eagerly() and + not tensor_array_ops.ENABLE_TENSOR_ARRAY_V2): r0_bad = gen_data_flow_ops.tensor_array_read_v3( handle=w0.handle, index=0, dtype=dtypes.float64, flow_in=w0.flow) with self.assertRaisesOpError( "TensorArray dtype is float but Op requested dtype double."): - r0_bad.eval() + self.evaluate(r0_bad) + if (tensor_array_ops.ENABLE_TENSOR_ARRAY_V2 and + not context.executing_eagerly()): + error_msg = "Trying to access element -1 in a list with 3 elements." + else: + error_msg = "index -1" # Test reading from a negative index, which is not allowed - with self.assertRaisesOpError("index -1"): + with self.assertRaisesOpError(error_msg): self.evaluate(ta.read(-1)) + if (tensor_array_ops.ENABLE_TENSOR_ARRAY_V2 and + not context.executing_eagerly()): + error_msg = "Trying to access element 3 in a list with 3 elements." + else: + error_msg = "Tried to read from index 3 but array size is: 3" # Test reading from too large an index - with self.assertRaisesOpError( - "Tried to read from index 3 but array size is: 3"): + with self.assertRaisesOpError(error_msg): self.evaluate(ta.read(3)) - def testTensorArrayWriteMultipleFails(self): + @test_util.disable_control_flow_v2("v2 allows multiple writes.") + @test_util.run_deprecated_v1 + def testSkipEagerTensorArrayWriteMultipleFails(self): with self.session(use_gpu=True): ta = tensor_array_ops.TensorArray( dtype=dtypes.float32, tensor_array_name="foo", size=3) @@ -450,7 +504,7 @@ class TensorArrayTest(test.TestCase): "it has already been written to."): self.evaluate(ta.write(2, 3.0).write(2, 3.0).flow) - @test_util.run_in_graph_and_eager_modes + @test_util.run_deprecated_v1 def testTensorArrayConcatIncompatibleShapesFails(self): with self.session(use_gpu=True): ta = tensor_array_ops.TensorArray( @@ -482,7 +536,7 @@ class TensorArrayTest(test.TestCase): with self.assertRaisesOpError("shape"): self.evaluate(w3.concat()) - @test_util.run_in_graph_and_eager_modes + @test_util.run_deprecated_v1 def testTensorArraySplitIncompatibleShapesFails(self): with self.session(use_gpu=True): in_eager_mode = context.executing_eagerly() @@ -495,22 +549,32 @@ class TensorArrayTest(test.TestCase): lengths = array_ops.placeholder(dtypes.int64) ta.split([1.0, 2.0, 3.0], lengths).flow.eval(feed_dict={lengths: 1}) - with self.assertRaisesOpError( - r"Expected sum of lengths to be equal to values.shape\[0\], " - r"but sum of lengths is 1 and value's shape is: \[3\]"): + error_msg = ("Unused values in tensor. Length of tensor: 3 Values used: 1" + if tensor_array_ops.ENABLE_TENSOR_ARRAY_V2 and + not in_eager_mode else + r"Expected sum of lengths to be equal to values.shape\[0\], " + r"but sum of lengths is 1 and value's shape is: \[3\]") + with self.assertRaisesOpError(error_msg): self.evaluate(ta.split([1.0, 2.0, 3.0], [1]).flow) ta = _make_ta(1, "baz") - with self.assertRaisesOpError( - r"Expected value to be at least a vector, but received shape: \[\]"): - self.evaluate(ta.split(1.0, [1]).flow) + if tensor_array_ops.ENABLE_TENSOR_ARRAY_V2 and not in_eager_mode: + with self.assertRaisesRegexp( + ValueError, "Shape must be at least rank 1 but is rank 0"): + self.evaluate(ta.split(1.0, [1]).flow) + else: + with self.assertRaisesOpError( + r"Expected value to be at least a vector, but received shape: \[\]" + ): + self.evaluate(ta.split(1.0, [1]).flow) - ta = _make_ta(2, "buz") - with self.assertRaisesOpError( - r"TensorArray's size is not equal to the size of lengths " - r"\(2 vs. 1\), and the TensorArray is not marked as " - r"dynamically resizeable"): - self.evaluate(ta.split([1.0], [1]).flow) + if not tensor_array_ops.ENABLE_TENSOR_ARRAY_V2 or in_eager_mode: + ta = _make_ta(2, "buz") + with self.assertRaisesOpError( + r"TensorArray's size is not equal to the size of lengths " + r"\(2 vs. 1\), and the TensorArray is not marked as " + r"dynamically resizeable"): + self.evaluate(ta.split([1.0], [1]).flow) def _testTensorArrayWriteGradientAddMultipleAdds(self, dtype): with self.cached_session(use_gpu=True): @@ -546,12 +610,16 @@ class TensorArrayTest(test.TestCase): r"existing shape is \[\] but the new input shape is \[1\]"): wb1_grad.flow.eval() - def testTensorArrayWriteGradientAddMultipleAdds(self): + @test_util.disable_control_flow_v2("v2 does not support TensorArray.grad.") + @test_util.run_deprecated_v1 + def testSkipEagerTensorArrayWriteGradientAddMultipleAdds(self): for dtype in (dtypes.int32, dtypes.int64, dtypes.float32, dtypes.float64, dtypes.complex64, dtypes.complex128): self._testTensorArrayWriteGradientAddMultipleAdds(dtype) - def testTensorArrayGradWithShapeKnownElementShape(self): + @test_util.disable_control_flow_v2("Low level legacy TA op test.") + @test_util.run_deprecated_v1 + def testSkipEagerTensorArrayGradWithShapeKnownElementShape(self): with self.session(use_gpu=True) as sess: ta = tensor_array_ops.TensorArray( size=3, @@ -580,7 +648,9 @@ class TensorArrayTest(test.TestCase): self.assertAllClose(fed_value, sess.run(read_value, feed_dict={value: fed_value})) - def testTensorArrayGradWithShapeUnknownElementShape(self): + @test_util.disable_control_flow_v2("Low level legacy TA op test.") + @test_util.run_deprecated_v1 + def testSkipEagerTensorArrayGradWithShapeUnknownElementShape(self): with self.session(use_gpu=True) as sess: ta = tensor_array_ops.TensorArray( size=3, dtype=dtypes.float32, @@ -603,7 +673,6 @@ class TensorArrayTest(test.TestCase): self.assertAllClose(fed_value, sess.run(read_value, feed_dict={value: fed_value})) - @test_util.run_in_graph_and_eager_modes def testMultiTensorArray(self): with self.session(use_gpu=True): h1 = tensor_array_ops.TensorArray( @@ -667,7 +736,8 @@ class TensorArrayTest(test.TestCase): self.assertAllEqual(c([[3.0, 2.0]]), grad_vals[0]) self.assertAllEqual(c(-2.0), grad_vals[1]) - def testTensorArrayGradientWriteRead(self): + @test_util.run_deprecated_v1 + def testSkipEagerTensorArrayGradientWriteRead(self): for dtype in (np.float32, np.float64, np.complex64, np.complex128): self._testTensorArrayGradientWriteReadType(dtype) @@ -698,15 +768,17 @@ class TensorArrayTest(test.TestCase): [-0.5, 1.5], # read(0) gradient [20.0, 30.0, 40.0, 50.0] ]) # concat gradient - grad_vals = sess.run(grad_r) # 2 + 2 entries + grad_vals = self.evaluate(grad_r) # 2 + 2 entries self.assertAllClose([2.0 - 0.5 + 20.0, 3.0 + 1.5 + 30.0], grad_vals[0]) self.assertAllEqual([4.0 + 40.0, 5.0 + 50.0], grad_vals[1]) - def testTensorArrayGradientWritePackConcatAndRead(self): + @test_util.run_deprecated_v1 + def testSkipEagerTensorArrayGradientWritePackConcatAndRead(self): self._testTensorArrayGradientWritePackConcatAndRead() - @test_util.run_in_graph_and_eager_modes + @test_util.disable_control_flow_v2("v2 does not support clear_after_read.") + @test_util.run_deprecated_v1 def testTensorArrayReadTwice(self): with self.session(use_gpu=True): value = constant_op.constant([[1.0, -1.0], [10.0, -10.0]]) @@ -760,10 +832,12 @@ class TensorArrayTest(test.TestCase): self.assertEqual(len(grad_vals), 1) self.assertAllEqual([[2.0 - 1.5, 3.0 + 1.5], [4.0, 5.0]], grad_vals[0]) - def testTensorArrayGradientUnpackRead(self): + @test_util.run_deprecated_v1 + def testSkipEagerTensorArrayGradientUnpackRead(self): self._testTensorArrayGradientUnpackRead() - def testTensorArrayGradientSplitConcat(self): + @test_util.run_deprecated_v1 + def testSkipEagerTensorArrayGradientSplitConcat(self): with self.session(use_gpu=True) as session: ta = tensor_array_ops.TensorArray( dtype=dtypes.float32, tensor_array_name="foo", size=2, @@ -808,17 +882,16 @@ class TensorArrayTest(test.TestCase): self.assertEqual(len(grad_vals), 1) self.assertAllEqual([[2.0, 3.0], [4.0, 5.0]], grad_vals[0]) - def testTensorArrayGradientDynamicUnpackRead(self): + @test_util.run_deprecated_v1 + def testSkipEagerTensorArrayGradientDynamicUnpackRead(self): self._testTensorArrayGradientDynamicUnpackRead() - @test_util.run_in_graph_and_eager_modes def testCloseTensorArray(self): with self.session(use_gpu=True): ta = tensor_array_ops.TensorArray( dtype=dtypes.float32, tensor_array_name="foo", size=3) self.evaluate(ta.close()) - @test_util.run_in_graph_and_eager_modes def testSizeTensorArray(self): with self.session(use_gpu=True): ta = tensor_array_ops.TensorArray( @@ -826,7 +899,6 @@ class TensorArrayTest(test.TestCase): s = ta.size() self.assertAllEqual(3, self.evaluate(s)) - @test_util.run_in_graph_and_eager_modes def testWriteCloseTensorArray(self): with self.session(use_gpu=True): ta = tensor_array_ops.TensorArray( @@ -924,7 +996,6 @@ class TensorArrayTest(test.TestCase): self.assertAllClose(grad_val.sum(axis=0), var_grad_t) self.assertAllClose(grad_val.sum(axis=0), state0_grad_t) - @test_util.run_in_graph_and_eager_modes def testWhileLoopWritePackGradients(self): self._testWhileLoopWritePackGradients( dynamic_size=False, dtype=dtypes.float32) @@ -932,11 +1003,28 @@ class TensorArrayTest(test.TestCase): # self._testWhileLoopWritePackGradients( # dynamic_size=False, dtype=tf.int64) - def testWhileLoopDynamicWritePackGradients(self): + @test_util.disable_control_flow_v2("Testing v1 while_loop with v2 TA") + @test_util.enable_tensor_array_v2 + def testWhileLoopV1WithTensorArrayV2(self): + size = 3 + ta = tensor_array_ops.TensorArray( + dtype=dtypes.int32, size=size, element_shape=tensor_shape.scalar()) + + def Body(counter, ta): + return counter + 1, ta.write(counter, counter) + + _, ta = control_flow_ops.while_loop(lambda i, _: i < size, Body, [0, ta]) + + for i in range(size): + self.assertEqual(self.evaluate(ta.read(i)), i) + + @test_util.disable_control_flow_v2("b/117943489 (dynamic_size)") + @test_util.run_v1_only("b/117943489") + def testSkipEagerWhileLoopDynamicWritePackGradients(self): self._testWhileLoopWritePackGradients( dynamic_size=True, dtype=dtypes.float32) - @test_util.run_in_graph_and_eager_modes + @test_util.disable_control_flow_v2("b/119323158") def testGradSerialTwoLoops(self): with self.session(use_gpu=True): def loop(x): @@ -976,7 +1064,8 @@ class TensorArrayTest(test.TestCase): grad = gradients_impl.gradients(loop(x), [x])[0] self.assertAllClose(31.0, self.evaluate(grad)) - def testSumOfTwoReadVariablesWithoutRepeatGrad(self): + @test_util.run_deprecated_v1 + def testSkipEagerSumOfTwoReadVariablesWithoutRepeatGrad(self): with self.session(use_gpu=True) as session: a = array_ops.identity( np.arange( @@ -1011,7 +1100,8 @@ class TensorArrayTest(test.TestCase): def _grad_source_for_name(self, name): return tensor_array_grad._GetGradSource(constant_op.constant(0, name=name)) - def testGetGradSource_Invalid(self): + @test_util.run_deprecated_v1 + def testSkipEagerGetGradSource_Invalid(self): with self.assertRaises(ValueError): self._grad_source_for_name("") with self.assertRaises(ValueError): @@ -1019,7 +1109,8 @@ class TensorArrayTest(test.TestCase): with self.assertRaises(ValueError): self._grad_source_for_name("foo/bar") - def testGetGradSource_NoEnclosingScope(self): + @test_util.run_deprecated_v1 + def testSkipEagerGetGradSource_NoEnclosingScope(self): self.assertEqual("gradients:0", self._grad_source_for_name("gradients")) self.assertEqual("gradients_0:0", self._grad_source_for_name("gradients_0")) self.assertEqual("gradients", self._grad_source_for_name("gradients/foo")) @@ -1030,7 +1121,8 @@ class TensorArrayTest(test.TestCase): self.assertEqual("gradients_0", self._grad_source_for_name("gradients_0/foo/bar")) - def testGetGradSource_EnclosingScope(self): + @test_util.run_deprecated_v1 + def testSkipEagerGetGradSource_EnclosingScope(self): self.assertEqual("foo/gradients:0", self._grad_source_for_name("foo/gradients")) self.assertEqual("foo/gradients_0:0", @@ -1044,12 +1136,14 @@ class TensorArrayTest(test.TestCase): self.assertEqual("foo/bar/gradients_0", self._grad_source_for_name("foo/bar/gradients_0/baz")) - def testGetGradSource_NestedUsesInnermost(self): + @test_util.run_deprecated_v1 + def testSkipEagerGetGradSource_NestedUsesInnermost(self): self.assertEqual( "foo/gradients/bar/gradients_0", self._grad_source_for_name("foo/gradients/bar/gradients_0/baz")) - def testWriteShape(self): + @test_util.run_deprecated_v1 + def testSkipEagerWriteShape(self): with self.session(use_gpu=True): ta = tensor_array_ops.TensorArray( dtype=dtypes.float32, tensor_array_name="foo", size=3) @@ -1073,7 +1167,8 @@ class TensorArrayTest(test.TestCase): with self.assertRaises(ValueError): w0.write(0, c2) - def testPartlyUnknownShape(self): + @test_util.run_deprecated_v1 + def testSkipEagerPartlyUnknownShape(self): with self.session(use_gpu=True): ta = tensor_array_ops.TensorArray( dtype=dtypes.float32, tensor_array_name="foo", size=6) @@ -1113,7 +1208,6 @@ class TensorArrayTest(test.TestCase): r5 = w5.read(0) self.assertAllEqual([5, 4, 2, 3], r5.get_shape().as_list()) - @test_util.run_in_graph_and_eager_modes def _testUnpackShape(self): with self.cached_session(use_gpu=True): ta = tensor_array_ops.TensorArray( @@ -1144,10 +1238,12 @@ class TensorArrayTest(test.TestCase): with self.assertRaises(ValueError): w1.write(4, c2) + @test_util.disable_control_flow_v2("b/117943489 (dynamic_size)") + @test_util.run_v1_only("b/117943489") def testUnpackShape(self): self._testUnpackShape() - @test_util.run_in_graph_and_eager_modes + @test_util.run_deprecated_v1 def testSplitShape(self): with self.session(use_gpu=True): ta = tensor_array_ops.TensorArray( @@ -1174,11 +1270,13 @@ class TensorArrayTest(test.TestCase): self.assertEqual((2, 2), w0.read(1).get_shape()) else: self.assertEqual(r0.get_shape().ndims, None) - self.assertEqual( - tensor_shape.TensorShape( - ta1.handle.op.get_attr("element_shape")).ndims, None) + if not tensor_array_ops.ENABLE_TENSOR_ARRAY_V2: + self.assertEqual( + tensor_shape.TensorShape( + ta1.handle.op.get_attr("element_shape")).ndims, None) - def testWriteUnknownShape(self): + @test_util.run_deprecated_v1 + def testSkipEagerWriteUnknownShape(self): with self.session(use_gpu=True): ta = tensor_array_ops.TensorArray( dtype=dtypes.float32, @@ -1201,7 +1299,11 @@ class TensorArrayTest(test.TestCase): grad_r0_vals = session.run(grad_r0)[0] self.assertAllEqual(grad_r0_vals, [1.0, 0.0]) - def testGradientWhenNotAllComponentsRead(self): + # TODO(srbs): Figure out how to enable this. This is probably failing + # because we are trying to stack a TensorList with invalid tensors. + # That is because we do not receive gradients for all list indices. + # Figure out how TensorArray handles this. + def disabletestGradientWhenNotAllComponentsRead(self): self._testGradientWhenNotAllComponentsRead() def _testTensorArrayUnpackDynamic(self): @@ -1212,14 +1314,18 @@ class TensorArrayTest(test.TestCase): w0 = ta.unstack(x) w1 = w0.write(3, 4.0) r = w1.stack() - self.assertAllEqual(np.array([1.0, 2.0, 3.0, 4.0]), r.eval()) + self.assertAllEqual(np.array([1.0, 2.0, 3.0, 4.0]), self.evaluate(r)) grad = gradients_impl.gradients(ys=[r], xs=[x]) - self.assertAllEqual(np.array([1.0, 1.0, 1.0]), sess.run(grad)[0]) + self.assertAllEqual(np.array([1.0, 1.0, 1.0]), self.evaluate(grad)[0]) - def testTensorArrayUnpackDynamic(self): + @test_util.disable_control_flow_v2("b/117943489") + @test_util.run_v1_only("b/117943489") + def testSkipEagerTensorArrayUnpackDynamic(self): self._testTensorArrayUnpackDynamic() - def testTensorArraySplitDynamic(self): + @test_util.disable_control_flow_v2("b/117943489") + @test_util.run_v1_only("b/117943489") + def testSkipEagerTensorArraySplitDynamic(self): with self.session(use_gpu=True) as sess: ta = tensor_array_ops.TensorArray( dtype=dtypes.float32, size=3, dynamic_size=True) @@ -1227,21 +1333,26 @@ class TensorArrayTest(test.TestCase): w0 = ta.split(x, [1, 1, 1]) w1 = w0.write(3, [4.0]) r = w1.concat() - self.assertAllEqual(np.array([1.0, 2.0, 3.0, 4.0]), r.eval()) + self.assertAllEqual(np.array([1.0, 2.0, 3.0, 4.0]), self.evaluate(r)) grad = gradients_impl.gradients(ys=[r], xs=[x]) - self.assertAllEqual(np.array([1.0, 1.0, 1.0]), sess.run(grad)[0]) + self.assertAllEqual(np.array([1.0, 1.0, 1.0]), self.evaluate(grad)[0]) def _testTensorArrayEvalEmpty(self): with self.cached_session(use_gpu=True): ta = tensor_array_ops.TensorArray( dtype=dtypes.float32, size=0, dynamic_size=False, infer_shape=False) - with self.assertRaisesOpError( - "TensorArray has size zero, but element shape is not fully " - "defined. Currently only static shapes are supported when packing " - "zero-size TensorArrays."): + v2_msg = ("Tried to stack elements of a empty list with " + "non-fully-defined shape") + v1_msg = ( + "TensorArray has size zero, but element shape is not " + "fully defined. Currently only static shapes are supported when " + "packing zero-size TensorArrays.") + with self.assertRaisesOpError(v2_msg if tensor_array_ops + .ENABLE_TENSOR_ARRAY_V2 else v1_msg): ta.stack().eval() - def testTensorArrayEvalEmpty(self): + @test_util.run_deprecated_v1 + def testSkipEagerTensorArrayEvalEmpty(self): self._testTensorArrayEvalEmpty() # this test is ill-defined for Eager mode --- unpacking an empty tensor @@ -1255,15 +1366,19 @@ class TensorArrayTest(test.TestCase): ta.unstack(array_ops.zeros([0, 3, 5])).mark_used() packed = ta.stack() concatenated = ta.concat() - self.assertAllEqual([0, 3, 5], packed.eval().shape) + self.assertAllEqual([0, 3, 5], self.evaluate(packed).shape) # Concatenating zero tensors along their first dimension gives a # first dimension of zero - self.assertAllEqual([0, 5], concatenated.eval().shape) + self.assertAllEqual([0, 5], self.evaluate(concatenated).shape) - def testTensorArrayEvalEmptyWithDefault(self): + @test_util.disable_control_flow_v2("b/117943489") + @test_util.run_v1_only("b/117943489") + def testSkipEagerTensorArrayEvalEmptyWithDefault(self): self._testTensorArrayEvalEmptyWithDefault() - def testTensorArrayScatterReadAndGradients(self): + @test_util.disable_control_flow_v2("b/117943489") + @test_util.run_v1_only("b/117943489") + def testSkipEagerTensorArrayScatterReadAndGradients(self): with self.session(use_gpu=True) as session: ta = tensor_array_ops.TensorArray( dtype=dtypes.float32, @@ -1289,7 +1404,8 @@ class TensorArrayTest(test.TestCase): self.assertAllEqual([10.0, -10.0], read_vals[1]) self.assertAllEqual([[2.0, 3.0], [4.0, 5.0]], grad_vals[0]) - @test_util.run_in_graph_and_eager_modes + @test_util.disable_control_flow_v2("b/117943286") + @test_util.run_v1_only("b/117943286") def testTensorArrayWriteGatherAndGradients(self): with self.session(use_gpu=True) as session: ta = tensor_array_ops.TensorArray( @@ -1326,7 +1442,9 @@ class TensorArrayTest(test.TestCase): self.assertAllEqual([[1.0, -1.0], [8.0, -8.0]], g_vals[0]) self.assertAllEqual(expected_grad, grad_vals[0]) - def testTensorArrayGetsDeviceFromFirstWrite(self): + @test_util.disable_control_flow_v2("colocate_with not supported in v2.") + @test_util.run_deprecated_v1 + def testSkipEagerTensorArrayGetsDeviceFromFirstWrite(self): with ops.device("/job:worker/task:0/cpu:0"): # this initial device will be ignored. ta = tensor_array_ops.TensorArray(dtype=dtypes.float32, size=2) @@ -1374,7 +1492,9 @@ class TensorArrayTest(test.TestCase): self.assertFalse( [s for s in dev_stats[d] if "/TensorArray" in s.node_name]) - def testTensorArrayGetsDeviceFromFirstWriteInWhileLoop(self): + @test_util.disable_control_flow_v2("colocate_with not supported in v2.") + @test_util.run_deprecated_v1 + def testSkipEagerTensorArrayGetsDeviceFromFirstWriteInWhileLoop(self): with ops.device("/job:worker/task:0/cpu:0"): ta = tensor_array_ops.TensorArray(dtype=dtypes.float32, size=2) @@ -1403,7 +1523,9 @@ class TensorArrayTest(test.TestCase): self.assertFalse( [s for s in dev_stats[d] if "TensorArray" == s.node_name]) - def testTensorArrayDisabledColocateWithFirstWriteCall(self): + @test_util.disable_control_flow_v2("colocate_with not supported in v2.") + @test_util.run_deprecated_v1 + def testSkipEagerTensorArrayDisabledColocateWithFirstWriteCall(self): with ops.device("/job:worker/task:0/cpu:0"): ta = tensor_array_ops.TensorArray( dtype=dtypes.float32, size=2, colocate_with_first_write_call=False) @@ -1433,7 +1555,6 @@ class TensorArrayTest(test.TestCase): self.assertFalse( [s for s in dev_stats[d] if "TensorArray" == s.node_name]) - @test_util.run_in_graph_and_eager_modes def testTensorArrayIdentity(self): with self.session(use_gpu=True): ta0 = tensor_array_ops.TensorArray(dtype=dtypes.float32, size=2, @@ -1486,7 +1607,8 @@ class TensorArrayTest(test.TestCase): self.assertEqual(size0_v, 2) self.assertEqual(size1_v, 4) - def testTensorArrayGradYsInCorrectScope(self): + @test_util.run_deprecated_v1 + def testSkipEagerTensorArrayGradYsInCorrectScope(self): n_time = 1 n_dim = 1 x = constant_op.constant([[1.42]]) @@ -1501,10 +1623,10 @@ class TensorArrayTest(test.TestCase): # wrap it in the correct name scope. dx, = gradients_impl.gradients(ys=[y], xs=[x], grad_ys=[dy]) with self.cached_session(use_gpu=True) as sess: - vdx, vdy = sess.run([dx, dy]) + vdx, vdy = self.evaluate([dx, dy]) self.assertAllClose(vdx, vdy) - def testTensorArrayInt64GPU(self): + def testSkipEagerTensorArrayInt64GPU(self): if not test.is_gpu_available(): return with self.session(use_gpu=True, force_gpu=True) as sess: diff --git a/tensorflow/python/kernel_tests/topk_op_test.py b/tensorflow/python/kernel_tests/topk_op_test.py index d9f340de6b2..5d46176bce8 100644 --- a/tensorflow/python/kernel_tests/topk_op_test.py +++ b/tensorflow/python/kernel_tests/topk_op_test.py @@ -27,6 +27,7 @@ from tensorflow.python.client import session from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops +from tensorflow.python.framework import test_util from tensorflow.python.ops import array_ops from tensorflow.python.ops import gradients_impl from tensorflow.python.ops import nn_ops @@ -48,7 +49,7 @@ class TopKTest(test.TestCase): np_expected_indices = np.array(expected_indices) with self.cached_session(use_gpu=True) as sess: values_op, indices_op = nn_ops.top_k(inputs, k, sorted=sorted) - values, indices = sess.run([values_op, indices_op]) + values, indices = self.evaluate([values_op, indices_op]) self.assertShapeEqual(np_expected_values, values_op) self.assertShapeEqual(np_expected_indices, indices_op) @@ -181,6 +182,7 @@ class TopKTest(test.TestCase): k = constant_op.constant(3) self._validateTopK(inputs, k, [19, 18, 17], [11, 3, 7]) + @test_util.run_deprecated_v1 def testKNegative(self): inputs = [[0.1, 0.2], [0.3, 0.4]] with self.session(use_gpu=True): @@ -189,12 +191,14 @@ class TopKTest(test.TestCase): with self.assertRaisesOpError("Need k >= 0, got -7"): values.eval(feed_dict={k: -7}) + @test_util.run_deprecated_v1 def testKTooLarge(self): inputs = [[0.1, 0.2], [0.3, 0.4]] with self.assertRaisesRegexp(ValueError, r"must have last dimension >= k = 4"): nn_ops.top_k(inputs, 4) + @test_util.run_deprecated_v1 def testTopKGradients(self): with self.session(use_gpu=True) as sess: inputs = array_ops.placeholder(dtypes.float32, shape=[2, 5]) diff --git a/tensorflow/python/kernel_tests/trace_op_test.py b/tensorflow/python/kernel_tests/trace_op_test.py index f1abaefb66b..52640c02c22 100644 --- a/tensorflow/python/kernel_tests/trace_op_test.py +++ b/tensorflow/python/kernel_tests/trace_op_test.py @@ -19,6 +19,7 @@ from __future__ import print_function import numpy as np +from tensorflow.python.framework import test_util from tensorflow.python.ops import math_ops from tensorflow.python.platform import test @@ -34,6 +35,7 @@ class TraceTest(test.TestCase): tf_ans = math_ops.trace(x).eval() self.assertAllClose(tf_ans, np_ans) + @test_util.run_deprecated_v1 def testTrace(self): for dtype in [np.int32, np.float32, np.float64]: for shape in [[2, 2], [2, 3], [3, 2], [2, 3, 2], [2, 2, 2, 3]]: diff --git a/tensorflow/python/kernel_tests/transpose_op_test.py b/tensorflow/python/kernel_tests/transpose_op_test.py index 8c11c207097..76e1002ee1b 100644 --- a/tensorflow/python/kernel_tests/transpose_op_test.py +++ b/tensorflow/python/kernel_tests/transpose_op_test.py @@ -50,7 +50,7 @@ class TransposeTest(test.TestCase): with self.cached_session(use_gpu=False): inx = ops.convert_to_tensor(x) y = array_ops.transpose(inx, p, conjugate=conjugate) - tf_ans = y.eval() + tf_ans = self.evaluate(y) self.assertShapeEqual(np_ans, y) self.assertAllEqual(np_ans, tf_ans) @@ -81,7 +81,7 @@ class TransposeTest(test.TestCase): with self.cached_session(use_gpu=True): inx = ops.convert_to_tensor(x) y = array_ops.transpose(inx, p, conjugate=conjugate) - tf_ans = y.eval() + tf_ans = self.evaluate(y) self.assertAllEqual(np_ans, tf_ans) self.assertShapeEqual(np_ans, y) @@ -168,7 +168,7 @@ class TransposeTest(test.TestCase): with self.cached_session(use_gpu=True): inx = ops.convert_to_tensor(inp) y = array_ops.transpose(inx, perm) - tf_ans = y.eval() + tf_ans = self.evaluate(y) self.assertAllEqual(np_ans, tf_ans) self.assertShapeEqual(np_ans, y) @@ -189,7 +189,7 @@ class TransposeTest(test.TestCase): with self.cached_session(use_gpu=True): inx = ops.convert_to_tensor(inp) y = array_ops.transpose(inx, perm) - tf_ans = y.eval() + tf_ans = self.evaluate(y) self.assertAllEqual(np_ans, tf_ans) self.assertShapeEqual(np_ans, y) @@ -224,7 +224,7 @@ class TransposeTest(test.TestCase): with self.cached_session(use_gpu=True): inx = ops.convert_to_tensor(inp) y = array_ops.transpose(inx, perm) - tf_ans = y.eval() + tf_ans = self.evaluate(y) self.assertAllEqual(np_ans, tf_ans) self.assertShapeEqual(np_ans, y) @@ -246,7 +246,7 @@ class TransposeTest(test.TestCase): with self.cached_session(use_gpu=True): inx = ops.convert_to_tensor(inp) y = array_ops.transpose(inx, perm) - tf_ans = y.eval() + tf_ans = self.evaluate(y) self.assertAllEqual(np_ans, tf_ans) self.assertShapeEqual(np_ans, y) @@ -267,7 +267,7 @@ class TransposeTest(test.TestCase): with self.cached_session(use_gpu=True): inx = ops.convert_to_tensor(inp) y = array_ops.transpose(inx, perm) - tf_ans = y.eval() + tf_ans = self.evaluate(y) self.assertAllEqual(np_ans, tf_ans) self.assertShapeEqual(np_ans, y) @@ -319,7 +319,7 @@ class TransposeTest(test.TestCase): with self.cached_session(use_gpu=True): inx = ops.convert_to_tensor(inp) y = array_ops.transpose(inx, perm) - tf_ans = y.eval() + tf_ans = self.evaluate(y) self.assertAllEqual(np_ans, tf_ans) self.assertShapeEqual(np_ans, y) self._ClearCachedSession() @@ -341,7 +341,7 @@ class TransposeTest(test.TestCase): inx = ops.convert_to_tensor(x) inp = constant_op.constant(p) y = array_ops.transpose(inx, inp) - tf_ans = y.eval() + tf_ans = self.evaluate(y) self.assertShapeEqual(np_ans, y) self.assertAllEqual(np_ans, tf_ans) diff --git a/tensorflow/python/kernel_tests/unicode_decode_op_test.py b/tensorflow/python/kernel_tests/unicode_decode_op_test.py new file mode 100644 index 00000000000..c165021eea3 --- /dev/null +++ b/tensorflow/python/kernel_tests/unicode_decode_op_test.py @@ -0,0 +1,153 @@ +# -*- coding: utf-8 -*- +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Tests for unicode_decode and unicode_decode_with_splits.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from tensorflow.python.framework import constant_op +from tensorflow.python.framework import errors_impl as errors +from tensorflow.python.framework import test_util +from tensorflow.python.ops import gen_string_ops +from tensorflow.python.platform import test + + +# Account for python2 and python3 execution of the test. +def codepoint(s): + if isinstance(s, bytes): + return ord(s.decode("utf-8")) + elif isinstance(s, str): + return ord(s) + + +class UnicodeDecodeTest(test.TestCase): + + def testBatchDecode(self): + text = constant_op.constant( + ["仅今年前", "分享介面終於迎來更新"]) + row_splits, utf8_text, offsets = gen_string_ops.unicode_decode_with_offsets( + text, "utf-8") + + with self.test_session(): + self.assertAllEqual([ + codepoint("仅"), + codepoint("今"), + codepoint("年"), + codepoint("前"), + codepoint("分"), + codepoint("享"), + codepoint("介"), + codepoint("面"), + codepoint("終"), + codepoint("於"), + codepoint("迎"), + codepoint("來"), + codepoint("更"), + codepoint("新") + ], + self.evaluate(utf8_text).tolist()) + self.assertAllEqual([0, 4, 14], self.evaluate(row_splits).tolist()) + self.assertAllEqual([0, 3, 6, 9, 0, 3, 6, 9, 12, 15, 18, 21, 24, 27], + self.evaluate(offsets).tolist()) + + def testBasicDecodeWithOffset(self): + text = constant_op.constant(["仅今年前"]) + row_splits, utf8_text, starts = gen_string_ops.unicode_decode_with_offsets( + text, "utf-8") + + with self.test_session(): + self.assertAllEqual([ + codepoint("仅"), + codepoint("今"), + codepoint("年"), + codepoint("前"), + ], + self.evaluate(utf8_text).tolist()) + self.assertAllEqual(self.evaluate(row_splits).tolist(), [0, 4]) + self.assertAllEqual(self.evaluate(starts).tolist(), [0, 3, 6, 9]) + + @test_util.run_deprecated_v1 + def testStrictError(self): + text = constant_op.constant([b"\xFEED"]) + _, error, _ = gen_string_ops.unicode_decode_with_offsets( + text, "utf-8", errors="strict") + + with self.assertRaises(errors.InvalidArgumentError): + with self.test_session(): + self.evaluate(error) + + def testReplaceOnError(self): + text = constant_op.constant([b"\xFE"]) + + _, utf8_text, _ = gen_string_ops.unicode_decode_with_offsets( + text, "utf-8", errors="replace") + + with self.test_session(): + self.assertAllEqual(self.evaluate(utf8_text).tolist(), [65533]) + + @test_util.run_deprecated_v1 + def testBadReplacementChar(self): + text = constant_op.constant([b"\xFE"]) + _, error, _ = gen_string_ops.unicode_decode_with_offsets( + text, "utf-8", errors="replace", replacement_char=11141111) + + with self.assertRaises(errors.InvalidArgumentError): + with self.test_session(): + self.evaluate(error) + + def testIgnoreOnError(self): + text = constant_op.constant([b"\xFEhello"]) + + _, utf8_text, _ = gen_string_ops.unicode_decode_with_offsets( + text, "utf-8", errors="ignore") + + with self.test_session(): + self.assertAllEqual(self.evaluate(utf8_text).tolist(), [ + codepoint("h"), + codepoint("e"), + codepoint("l"), + codepoint("l"), + codepoint("o") + ]) + + @test_util.run_deprecated_v1 + def testBadErrorPolicy(self): + text = constant_op.constant(["hippopotamus"]) + + with self.assertRaises(ValueError): + _, _, _ = gen_string_ops.unicode_decode_with_offsets( + text, "utf-8", errors="oranguatan") + + def testReplaceControlChars(self): + text = constant_op.constant(["\x02仅今年前"]) + row_splits, utf8_text, _ = gen_string_ops.unicode_decode_with_offsets( + text, "utf-8", replace_control_characters=True) + + with self.test_session(): + self.assertAllEqual([ + 65533, + codepoint("仅"), + codepoint("今"), + codepoint("年"), + codepoint("前"), + ], + self.evaluate(utf8_text).tolist()) + self.assertAllEqual([0, 5], self.evaluate(row_splits).tolist()) + + +if __name__ == "__main__": + test.main() diff --git a/tensorflow/python/kernel_tests/unicode_encode_op_test.py b/tensorflow/python/kernel_tests/unicode_encode_op_test.py new file mode 100644 index 00000000000..a5a5c2017c6 --- /dev/null +++ b/tensorflow/python/kernel_tests/unicode_encode_op_test.py @@ -0,0 +1,301 @@ +# Copyright 2016 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Tests for UnicodeEncode op from ragged_string_ops.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from absl.testing import parameterized +import numpy as np + +from tensorflow.python.framework import constant_op +from tensorflow.python.framework import errors_impl as errors +from tensorflow.python.framework import ops +from tensorflow.python.ops.ragged import ragged_factory_ops +from tensorflow.python.ops.ragged import ragged_string_ops +from tensorflow.python.platform import test + + +class UnicodeEncodeOpTest(test.TestCase, parameterized.TestCase): + + def testScalar(self): + with self.cached_session(): + with self.assertRaises(ValueError): + ragged_string_ops.unicode_encode(72, "UTF-8") + with self.cached_session(): + with self.assertRaises(ValueError): + ragged_string_ops.unicode_encode(constant_op.constant(72), "UTF-8") + + def testRequireParams(self): + with self.cached_session(): + with self.assertRaises(TypeError): + ragged_string_ops.unicode_encode() + with self.cached_session(): + with self.assertRaises(TypeError): + ragged_string_ops.unicode_encode(72) + with self.cached_session(): + with self.assertRaises(TypeError): + ragged_string_ops.unicode_encode(encoding="UTF-8") + + @parameterized.parameters("UTF-8", "UTF-16-BE", "UTF-32-BE") + def testStrictErrors(self, encoding): + test_value = np.array([72, 101, 2147483647, -1, 111], np.int32) + with self.cached_session(): + with self.assertRaises(errors.InvalidArgumentError): + ragged_string_ops.unicode_encode(test_value, encoding, "strict").eval() + + @parameterized.parameters("UTF-8", "UTF-16-BE", "UTF-32-BE") + def testIgnoreErrors(self, encoding): + test_value = np.array([72, 101, 2147483647, -1, 111], np.int32) + expected_value = u"Heo".encode(encoding) + unicode_encode_op = ragged_string_ops.unicode_encode(test_value, encoding, + "ignore") + with self.cached_session(): + result = unicode_encode_op.eval() + self.assertIsInstance(result, bytes) + self.assertAllEqual(result, expected_value) + + @parameterized.parameters("UTF-8", "UTF-16-BE", "UTF-32-BE") + def testReplaceErrors(self, encoding): + test_value = np.array([72, 101, 2147483647, -1, 111], np.int32) + expected_value = u"He\U0000fffd\U0000fffdo".encode(encoding) + unicode_encode_op = ragged_string_ops.unicode_encode(test_value, encoding, + "replace") + with self.cached_session(): + result = unicode_encode_op.eval() + self.assertIsInstance(result, bytes) + self.assertAllEqual(result, expected_value) + + # Test custom replacement character + test_value = np.array([72, 101, 2147483647, -1, 111], np.int32) + expected_value = u"Heooo".encode(encoding) + unicode_encode_op = ragged_string_ops.unicode_encode(test_value, encoding, + "replace", 111) + with self.cached_session(): + result = unicode_encode_op.eval() + self.assertIsInstance(result, bytes) + self.assertAllEqual(result, expected_value) + + # Verify "replace" is default + test_value = np.array([72, 101, 2147483647, -1, 111], np.int32) + expected_value = u"He\U0000fffd\U0000fffdo".encode(encoding) + unicode_encode_op = ragged_string_ops.unicode_encode(test_value, encoding) + with self.cached_session(): + result = unicode_encode_op.eval() + self.assertIsInstance(result, bytes) + self.assertAllEqual(result, expected_value) + + # Replacement_char must be within range + test_value = np.array([72, 101, 2147483647, -1, 111], np.int32) + unicode_encode_op = ragged_string_ops.unicode_encode(test_value, encoding, + "replace", 1114112) + with self.cached_session(): + with self.assertRaises(errors.InvalidArgumentError): + unicode_encode_op.eval() + + # -- regular Tensor tests -- # + + @parameterized.parameters("UTF-8", "UTF-16-BE", "UTF-32-BE") + def testVector(self, encoding): + test_value = np.array([72, 101, 108, 108, 111], np.int32) + expected_value = u"Hello".encode(encoding) + unicode_encode_op = ragged_string_ops.unicode_encode(test_value, encoding) + with self.cached_session(): + result = unicode_encode_op.eval() + self.assertIsInstance(result, bytes) + self.assertAllEqual(result, expected_value) + + test_value = np.array([72, 101, 195, 195, 128516], np.int32) + expected_value = u"He\xc3\xc3\U0001f604".encode(encoding) + unicode_encode_op = ragged_string_ops.unicode_encode(test_value, encoding) + with self.cached_session(): + result = unicode_encode_op.eval() + self.assertIsInstance(result, bytes) + self.assertAllEqual(result, expected_value) + + # Single character string + test_value = np.array([72], np.int32) + expected_value = u"H".encode(encoding) + unicode_encode_op = ragged_string_ops.unicode_encode(test_value, encoding) + with self.cached_session(): + result = unicode_encode_op.eval() + self.assertIsInstance(result, bytes) + self.assertAllEqual(result, expected_value) + + test_value = np.array([128516], np.int32) + expected_value = u"\U0001f604".encode(encoding) + unicode_encode_op = ragged_string_ops.unicode_encode(test_value, encoding) + with self.cached_session(): + result = unicode_encode_op.eval() + self.assertIsInstance(result, bytes) + self.assertAllEqual(result, expected_value) + + @parameterized.parameters("UTF-8", "UTF-16-BE", "UTF-32-BE") + def testMatrix(self, encoding): + test_value = np.array( + [[72, 128516, 108, 108, 111], [87, 128516, 114, 108, 100]], np.int32) + expected_value = [ + u"H\U0001f604llo".encode(encoding), u"W\U0001f604rld".encode(encoding) + ] + unicode_encode_op = ragged_string_ops.unicode_encode(test_value, encoding) + with self.cached_session(): + result = unicode_encode_op.eval() + self.assertIsInstance(unicode_encode_op, ops.Tensor) + self.assertAllEqual(result, expected_value) + + @parameterized.parameters("UTF-8", "UTF-16-BE", "UTF-32-BE") + def test3DimMatrix(self, encoding): + test_value = constant_op.constant( + [[[72, 101, 108, 108, 111], [87, 111, 114, 108, 100]], + [[102, 105, 120, 101, 100], [119, 111, 114, 100, 115]], + [[72, 121, 112, 101, 114], [99, 117, 98, 101, 46]]], np.int32) + expected_value = [[u"Hello".encode(encoding), u"World".encode(encoding)], + [u"fixed".encode(encoding), u"words".encode(encoding)], + [u"Hyper".encode(encoding), u"cube.".encode(encoding)]] + unicode_encode_op = ragged_string_ops.unicode_encode(test_value, encoding) + with self.cached_session(): + result = unicode_encode_op.eval() + self.assertIsInstance(unicode_encode_op, ops.Tensor) + self.assertAllEqual(result, expected_value) + + @parameterized.parameters("UTF-8", "UTF-16-BE", "UTF-32-BE") + def test4DimMatrix(self, encoding): + test_value = constant_op.constant( + [[[[72, 101, 108, 108, 111]], [[87, 111, 114, 108, 100]]], + [[[102, 105, 120, 101, 100]], [[119, 111, 114, 100, 115]]], + [[[72, 121, 112, 101, 114]], [[99, 117, 98, 101, 46]]]], np.int32) + expected_value = [[[u"Hello".encode(encoding)], + [u"World".encode(encoding)]], + [[u"fixed".encode(encoding)], + [u"words".encode(encoding)]], + [[u"Hyper".encode(encoding)], + [u"cube.".encode(encoding)]]] + unicode_encode_op = ragged_string_ops.unicode_encode(test_value, encoding) + with self.cached_session(): + result = unicode_encode_op.eval() + self.assertIsInstance(unicode_encode_op, ops.Tensor) + self.assertAllEqual(result, expected_value) + + # -- Ragged Tensor tests -- # + + @parameterized.parameters("UTF-8", "UTF-16-BE", "UTF-32-BE") + def testRaggedMatrix(self, encoding): + test_value = ragged_factory_ops.constant( + [[72, 195, 108, 108, 111], [87, 128516, 114, 108, 100, 46]], np.int32) + expected_value = [ + u"H\xc3llo".encode(encoding), u"W\U0001f604rld.".encode(encoding) + ] + unicode_encode_op = ragged_string_ops.unicode_encode(test_value, encoding) + with self.cached_session(): + result = unicode_encode_op.eval() + self.assertIsInstance(unicode_encode_op, ops.Tensor) + self.assertAllEqual(result, expected_value) + + @parameterized.parameters("UTF-8", "UTF-16-BE", "UTF-32-BE") + def test3DimMatrixWithRagged2ndDim(self, encoding): + test_value = ragged_factory_ops.constant( + [[[72, 101, 108, 108, 111], [87, 111, 114, 108, 100]], + [[102, 105, 120, 101, 100]], + [[72, 121, 112, 101, 114], [119, 111, 114, 100, 115], + [99, 117, 98, 101, 46]]], np.int32) + expected_value = [[u"Hello".encode(encoding), u"World".encode(encoding)], + [u"fixed".encode(encoding)], + [ + u"Hyper".encode(encoding), u"words".encode(encoding), + u"cube.".encode(encoding) + ]] + unicode_encode_op = ragged_string_ops.unicode_encode(test_value, encoding) + with self.cached_session(): + result = unicode_encode_op.eval() + self.assertEqual(unicode_encode_op.ragged_rank, 1) + self.assertAllEqual(result.tolist(), expected_value) + + @parameterized.parameters("UTF-8", "UTF-16-BE", "UTF-32-BE") + def test3DimMatrixWithRagged3rdDim(self, encoding): + test_value = ragged_factory_ops.constant( + [[[72, 101, 108, 108, 111], [87, 111, 114, 108, 100, 46]], + [[68, 111, 110, 39, 116], [119, 195, 114, 114, 121, 44, 32, 98, 101]], + [[128516], []]], np.int32) + expected_value = [[u"Hello".encode(encoding), u"World.".encode(encoding)], + [ + u"Don't".encode(encoding), + u"w\xc3rry, be".encode(encoding) + ], [u"\U0001f604".encode(encoding), u"".encode(encoding)]] + unicode_encode_op = ragged_string_ops.unicode_encode(test_value, encoding) + with self.cached_session(): + result = unicode_encode_op.eval() + self.assertEqual(unicode_encode_op.ragged_rank, 1) + self.assertAllEqual(result.tolist(), expected_value) + + @parameterized.parameters("UTF-8", "UTF-16-BE", "UTF-32-BE") + def test3DimMatrixWithRagged2ndAnd3rdDim(self, encoding): + test_value = ragged_factory_ops.constant( + [[[72, 101, 108, 108, 111], [87, 111, 114, 108, 100, 46]], [], + [[128516]]], np.int32) + expected_value = [[u"Hello".encode(encoding), u"World.".encode(encoding)], + [], [u"\U0001f604".encode(encoding)]] + unicode_encode_op = ragged_string_ops.unicode_encode(test_value, encoding) + with self.cached_session(): + result = unicode_encode_op.eval() + self.assertEqual(unicode_encode_op.ragged_rank, 1) + self.assertAllEqual(result.tolist(), expected_value) + + @parameterized.parameters("UTF-8", "UTF-16-BE", "UTF-32-BE") + def test4DimRaggedMatrix(self, encoding): + test_value = ragged_factory_ops.constant( + [[[[72, 101, 108, 108, 111], [87, 111, 114, 108, 100]]], + [[[]], [[72, 121, 112, 101]]]], np.int32) + expected_value = [[[u"Hello".encode(encoding), u"World".encode(encoding)]], + [[u"".encode(encoding)], [u"Hype".encode(encoding)]]] + unicode_encode_op = ragged_string_ops.unicode_encode(test_value, encoding) + with self.cached_session(): + result = unicode_encode_op.eval() + self.assertEqual(unicode_encode_op.ragged_rank, 2) + self.assertAllEqual(result.tolist(), expected_value) + + @parameterized.parameters("UTF-8", "UTF-16-BE", "UTF-32-BE") + def testRaggedMatrixWithMultiDimensionInnerValues(self, encoding): + test_inner_values = constant_op.constant([[[72, 101, 108, 108, 111], + [87, 111, 114, 108, 100]], + [[102, 105, 120, 101, 100], + [119, 111, 114, 100, 115]], + [[72, 121, 112, 101, 114], + [99, 117, 98, 101, 46]]]) + test_row_splits = [ + constant_op.constant([0, 2, 3], dtype=np.int64), + constant_op.constant([0, 1, 1, 3], dtype=np.int64) + ] + test_value = ragged_factory_ops.from_nested_row_splits(test_inner_values, + test_row_splits) + expected_value = [[[[u"Hello".encode(encoding), u"World".encode(encoding)]], + []], + [[[u"fixed".encode(encoding), u"words".encode(encoding)], + [u"Hyper".encode(encoding), + u"cube.".encode(encoding)]]]] + unicode_encode_op = ragged_string_ops.unicode_encode(test_value, encoding) + with self.cached_session(): + result = unicode_encode_op.eval() + self.assertEqual(unicode_encode_op.ragged_rank, 2) + self.assertAllEqual(result.tolist(), expected_value) + # These next two assertions don't necessarily need to be here as they test + # internal representations and we already verified the value is correct. + self.assertAllEqual(len(result.nested_row_splits), len(test_row_splits)) + self.assertEqual(unicode_encode_op.inner_values.shape.ndims, + test_inner_values.shape.ndims - 1) + + +if __name__ == "__main__": + test.main() diff --git a/tensorflow/python/kernel_tests/unicode_script_op_test.py b/tensorflow/python/kernel_tests/unicode_script_op_test.py index 927e5459ed2..83cfeb20216 100644 --- a/tensorflow/python/kernel_tests/unicode_script_op_test.py +++ b/tensorflow/python/kernel_tests/unicode_script_op_test.py @@ -20,12 +20,14 @@ from __future__ import print_function from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes +from tensorflow.python.framework import test_util from tensorflow.python.ops import string_ops from tensorflow.python.platform import test class UnicodeScriptOpTest(test.TestCase): + @test_util.run_deprecated_v1 def testValidScripts(self): inputs = [ ord("a"), @@ -45,6 +47,7 @@ class UnicodeScriptOpTest(test.TestCase): 0 # USCRIPT_COMMON (ZYYY) ]) + @test_util.run_deprecated_v1 def testInvalidScript(self): inputs = [-100, 0xffffff] with self.cached_session(): diff --git a/tensorflow/python/kernel_tests/unicode_transcode_op_test.py b/tensorflow/python/kernel_tests/unicode_transcode_op_test.py index 2908e2bfc56..a3b4fd03474 100644 --- a/tensorflow/python/kernel_tests/unicode_transcode_op_test.py +++ b/tensorflow/python/kernel_tests/unicode_transcode_op_test.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================== -"""Tests for string_length_op.""" +"""Tests for unicode_transcode op.""" from __future__ import absolute_import from __future__ import division @@ -22,6 +22,7 @@ from absl.testing import parameterized from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes +from tensorflow.python.framework import test_util from tensorflow.python.ops import string_ops from tensorflow.python.platform import test @@ -42,7 +43,7 @@ class UnicodeTranscodeOpTest(test.TestCase, parameterized.TestCase): errors="replace", replacement_char=ord(" "), replace_control_characters=False) - values = sess.run(outputs) + values = self.evaluate(outputs) self.assertAllEqual(values, strings) outputs = string_ops.unicode_transcode( @@ -52,7 +53,7 @@ class UnicodeTranscodeOpTest(test.TestCase, parameterized.TestCase): errors="replace", replacement_char=ord(" "), replace_control_characters=False) - values = sess.run(outputs) + values = self.evaluate(outputs) self.assertAllEqual(values, strings) outputs = string_ops.unicode_transcode( @@ -62,7 +63,7 @@ class UnicodeTranscodeOpTest(test.TestCase, parameterized.TestCase): errors="replace", replacement_char=ord(" "), replace_control_characters=False) - values = sess.run(outputs) + values = self.evaluate(outputs) self.assertAllEqual(values, strings) def test_transcode_utf16_to_utf8(self): @@ -77,7 +78,7 @@ class UnicodeTranscodeOpTest(test.TestCase, parameterized.TestCase): errors="replace", replacement_char=ord(" "), replace_control_characters=False) - values = sess.run(outputs) + values = self.evaluate(outputs) self.assertAllEqual(values, expected) def test_transcode_bad_utf8(self): @@ -90,7 +91,7 @@ class UnicodeTranscodeOpTest(test.TestCase, parameterized.TestCase): errors="replace", replacement_char=ord(" "), replace_control_characters=True) - values = sess.run(outputs) + values = self.evaluate(outputs) self.assertAllEqual(values, b" ") outputs = string_ops.unicode_transcode( @@ -100,7 +101,7 @@ class UnicodeTranscodeOpTest(test.TestCase, parameterized.TestCase): errors="replace", replacement_char=ord(" "), replace_control_characters=False) - values = sess.run(outputs) + values = self.evaluate(outputs) self.assertAllEqual(values, b"\x00 ") def test_transcode_bad_utf8_with_some_good(self): @@ -113,7 +114,7 @@ class UnicodeTranscodeOpTest(test.TestCase, parameterized.TestCase): errors="replace", replacement_char=ord(" "), replace_control_characters=False) - values = sess.run(outputs) + values = self.evaluate(outputs) self.assertAllEqual(values, b"abc abcdefg") def test_transcode_bad_utf8_with_defaults(self): @@ -121,7 +122,7 @@ class UnicodeTranscodeOpTest(test.TestCase, parameterized.TestCase): with self.cached_session() as sess: outputs = string_ops.unicode_transcode( bad_string, input_encoding="UTF-8", output_encoding="UTF-8") - values = sess.run(outputs) + values = self.evaluate(outputs) self.assertAllEqual(values, b"\x00\xef\xbf\xbd") def test_transcode_bad_utf8_with_space_replacement(self): @@ -130,9 +131,10 @@ class UnicodeTranscodeOpTest(test.TestCase, parameterized.TestCase): outputs = string_ops.unicode_transcode( bad_string, input_encoding="UTF-8", output_encoding="UTF-8", replacement_char=ord(" ")) - values = sess.run(outputs) + values = self.evaluate(outputs) self.assertAllEqual(values, b"\x00 ") + @test_util.run_deprecated_v1 def test_transcode_bad_utf8_with_strict_errors(self): bad_string = b"\x00\xff" with self.cached_session() as sess: @@ -143,8 +145,9 @@ class UnicodeTranscodeOpTest(test.TestCase, parameterized.TestCase): errors="strict") with self.assertRaisesOpError( "Invalid formatting on input string"): - sess.run(outputs) + self.evaluate(outputs) + @test_util.run_deprecated_v1 def test_transcode_bad_utf8_start_with_strict_errors(self): bad_string = b"\xffabcd" with self.cached_session() as sess: @@ -155,7 +158,7 @@ class UnicodeTranscodeOpTest(test.TestCase, parameterized.TestCase): errors="strict") with self.assertRaisesOpError( "Invalid formatting on input string"): - sess.run(outputs) + self.evaluate(outputs) def test_transcode_bad_utf8_with_elision_of_malformatting(self): bad_string = b"\x00\xff" @@ -165,7 +168,7 @@ class UnicodeTranscodeOpTest(test.TestCase, parameterized.TestCase): input_encoding="UTF-8", output_encoding="UTF-8", errors="ignore") - values = sess.run(outputs) + values = self.evaluate(outputs) self.assertAllEqual(values, b"\x00") def test_transcode_bad_utf8_with_elision_including_control_chars(self): @@ -177,7 +180,7 @@ class UnicodeTranscodeOpTest(test.TestCase, parameterized.TestCase): output_encoding="UTF-8", errors="ignore", replace_control_characters=True) - values = sess.run(outputs) + values = self.evaluate(outputs) self.assertAllEqual(values, b"") def test_transcode_bad_utf8_termination_with_defaults(self): @@ -185,7 +188,7 @@ class UnicodeTranscodeOpTest(test.TestCase, parameterized.TestCase): with self.cached_session() as sess: outputs = string_ops.unicode_transcode( bad_string, input_encoding="UTF-8", output_encoding="UTF-8") - values = sess.run(outputs) + values = self.evaluate(outputs) self.assertAllEqual(values, b"a\xef\xbf\xbd") # 0xFFFD def test_transcode_utf8_with_replacement_char(self): @@ -194,13 +197,13 @@ class UnicodeTranscodeOpTest(test.TestCase, parameterized.TestCase): outputs = string_ops.unicode_transcode( strings, input_encoding="UTF-8", output_encoding="UTF-8", errors="strict") - values = sess.run(outputs) + values = self.evaluate(outputs) self.assertAllEqual(values, [b"a\xef\xbf\xbd"]) outputs = string_ops.unicode_transcode( strings, input_encoding="UTF-8", output_encoding="UTF-8", errors="replace", replacement_char=ord("?")) - values = sess.run(outputs) + values = self.evaluate(outputs) self.assertAllEqual(values, [b"a\xef\xbf\xbd"]) def test_transcode_utf8_to_utf16(self): @@ -214,7 +217,7 @@ class UnicodeTranscodeOpTest(test.TestCase, parameterized.TestCase): output_encoding="UTF-16-BE", replacement_char=ord(" "), replace_control_characters=False) - values = sess.run(outputs) + values = self.evaluate(outputs) print("values=", values) self.assertAllEqual(values, expected) @@ -230,7 +233,7 @@ class UnicodeTranscodeOpTest(test.TestCase, parameterized.TestCase): output_encoding="UTF-8", replacement_char=ord(" "), replace_control_characters=False) - values = sess.run(outputs) + values = self.evaluate(outputs) self.assertAllEqual(values, expected) def test_transcode_utf8_to_utf32(self): @@ -243,7 +246,7 @@ class UnicodeTranscodeOpTest(test.TestCase, parameterized.TestCase): output_encoding="UTF-32-BE", replacement_char=ord(" "), replace_control_characters=False) - values = sess.run(outputs) + values = self.evaluate(outputs) self.assertAllEqual(values, expected) # Documentation in ICU suggests that getNextUChar may produce a different @@ -258,7 +261,7 @@ class UnicodeTranscodeOpTest(test.TestCase, parameterized.TestCase): output_encoding="UTF-8", replacement_char=ord(" "), replace_control_characters=False) - values = sess.run(outputs) + values = self.evaluate(outputs) self.assertAllEqual(values, strings) def test_transcode_utf8_with_bom(self): @@ -266,12 +269,12 @@ class UnicodeTranscodeOpTest(test.TestCase, parameterized.TestCase): with self.cached_session() as sess: outputs = string_ops.unicode_transcode( bom_string, input_encoding="UTF-8", output_encoding="UTF-8") - values = sess.run(outputs) + values = self.evaluate(outputs) self.assertAllEqual(values, b"\xef\xbb\xbfabcdefg") # BOM preserved outputs = string_ops.unicode_transcode( bom_string, input_encoding="UTF-8", output_encoding="UTF-16-BE") - values = sess.run(outputs) + values = self.evaluate(outputs) utf16expected = bom_string.decode("UTF-8").encode("UTF-16-BE") self.assertAllEqual(values, utf16expected) @@ -280,20 +283,20 @@ class UnicodeTranscodeOpTest(test.TestCase, parameterized.TestCase): with self.cached_session() as sess: outputs = string_ops.unicode_transcode( bom_string, input_encoding="UTF-16-BE", output_encoding="UTF-8") - values = sess.run(outputs) + values = self.evaluate(outputs) # BOM is preserved in output self.assertAllEqual(values, b"\xef\xbb\xbfa") outputs = string_ops.unicode_transcode( bom_string, input_encoding="UTF-16-LE", output_encoding="UTF-8") - values = sess.run(outputs) + values = self.evaluate(outputs) # mangled BOM and value from (incorrect) LE encoding self.assertAllEqual(values, b"\xef\xbf\xbe\xe6\x84\x80") bom_string = b"\xff\xfe\x61\x00" # Little-endian BOM with 'a' encoded outputs = string_ops.unicode_transcode( bom_string, input_encoding="UTF-16-LE", output_encoding="UTF-8") - values = sess.run(outputs) + values = self.evaluate(outputs) self.assertAllEqual(values, b"\xef\xbb\xbfa") @parameterized.parameters( @@ -317,12 +320,14 @@ class UnicodeTranscodeOpTest(test.TestCase, parameterized.TestCase): (b"\xfe\xff\x00<\xfe\xff\x00>", "UTF-16", b"<\xef\xbb\xbf>"), (b"\xff\xfe<\x00\xff\xfe>\x00", "UTF-16", b"<\xef\xbb\xbf>"), ) + @test_util.run_deprecated_v1 def test_bom_handling(self, string, input_encoding, expected): with self.test_session(): output = string_ops.unicode_transcode( string, input_encoding=input_encoding, output_encoding="UTF-8") self.assertAllEqual(output.eval(), expected) + @test_util.run_deprecated_v1 def test_invalid_encoding_causes_errors(self): strings = [[b"a", b"abc"], [b"ABC", b"DEF"]] @@ -336,7 +341,7 @@ class UnicodeTranscodeOpTest(test.TestCase, parameterized.TestCase): replace_control_characters=False) with self.assertRaisesOpError( "Could not create converter for input encoding: invalid"): - sess.run(outputs) + self.evaluate(outputs) with self.assertRaisesRegexp(ValueError, "Op passed string 'invalid'"): with self.cached_session() as sess: @@ -347,8 +352,9 @@ class UnicodeTranscodeOpTest(test.TestCase, parameterized.TestCase): errors="replace", replacement_char=ord(" "), replace_control_characters=False) - sess.run(outputs) + self.evaluate(outputs) + @test_util.run_deprecated_v1 def test_invalid_error_policy_causes_errors(self): strings = [[b"a", b"abc"], [b"ABC", b"DEF"]] @@ -362,7 +368,7 @@ class UnicodeTranscodeOpTest(test.TestCase, parameterized.TestCase): errors="invalid", replacement_char=ord(" "), replace_control_characters=False) - sess.run(outputs) + self.evaluate(outputs) def test_forwarding(self): with self.cached_session(): @@ -378,6 +384,61 @@ class UnicodeTranscodeOpTest(test.TestCase, parameterized.TestCase): self.assertAllEqual([b"AbCdE", b"HiJkL"], transcoded) + @test_util.run_deprecated_v1 + def test_cjk_encodings(self): + strings_ja = [ + b"\x5c\x5c", # Yen sign + b"\x8f\x70", # kanji character "waza" + b"\x83\x4f" + ] # katakana character "gu" + strings_zh_cn = [b"\xca\xf5"] # simplified "shu4" + strings_zh_tw = [b"\xb3\x4e"] # traditional "shu4" + strings_ko = [b"\xc7\xd1\xb9\xce"] # hangul "hanmin" + + expected_ja = [s.decode("shift_jis").encode("UTF-8") for s in strings_ja] + expected_zh_cn = [ + s.decode("gb18030").encode("UTF-8") for s in strings_zh_cn + ] + expected_zh_tw = [s.decode("big5").encode("UTF-8") for s in strings_zh_tw] + expected_ko = [s.decode("euc_kr").encode("UTF-8") for s in strings_ko] + + with self.cached_session() as sess: + outputs_ja = string_ops.unicode_transcode( + strings_ja, + input_encoding="shift_jis", + output_encoding="UTF-8", + replacement_char=ord(" "), + replace_control_characters=False) + + outputs_zh_cn = string_ops.unicode_transcode( + strings_zh_cn, + input_encoding="gb18030", + output_encoding="UTF-8", + replacement_char=ord(" "), + replace_control_characters=False) + + outputs_zh_tw = string_ops.unicode_transcode( + strings_zh_tw, + input_encoding="big5", + output_encoding="UTF-8", + replacement_char=ord(" "), + replace_control_characters=False) + + outputs_ko = string_ops.unicode_transcode( + strings_ko, + input_encoding="euc_kr", + output_encoding="UTF-8", + replacement_char=ord(" "), + replace_control_characters=False) + + result_ja, result_zh_cn, result_zh_tw, result_ko = sess.run( + [outputs_ja, outputs_zh_cn, outputs_zh_tw, outputs_ko]) + + self.assertAllEqual(result_ja, expected_ja) + self.assertAllEqual(result_zh_cn, expected_zh_cn) + self.assertAllEqual(result_zh_tw, expected_zh_tw) + self.assertAllEqual(result_ko, expected_ko) + if __name__ == "__main__": test.main() diff --git a/tensorflow/python/kernel_tests/unique_op_test.py b/tensorflow/python/kernel_tests/unique_op_test.py index 316570e13e2..f203263e0c5 100644 --- a/tensorflow/python/kernel_tests/unique_op_test.py +++ b/tensorflow/python/kernel_tests/unique_op_test.py @@ -32,7 +32,7 @@ class UniqueTest(test.TestCase): x = np.random.randint(2, high=10, size=7000) with self.cached_session() as sess: y, idx = array_ops.unique(x) - tf_y, tf_idx = sess.run([y, idx]) + tf_y, tf_idx = self.evaluate([y, idx]) self.assertEqual(len(x), len(tf_idx)) self.assertEqual(len(tf_y), len(np.unique(x))) @@ -43,7 +43,7 @@ class UniqueTest(test.TestCase): x = np.random.randint(2, high=10, size=7000) with self.cached_session() as sess: y, idx = array_ops.unique(x, out_idx=dtypes.int64) - tf_y, tf_idx = sess.run([y, idx]) + tf_y, tf_idx = self.evaluate([y, idx]) self.assertEqual(len(x), len(tf_idx)) self.assertEqual(len(tf_y), len(np.unique(x))) @@ -55,7 +55,7 @@ class UniqueTest(test.TestCase): x = [chr(i) for i in indx] with self.cached_session() as sess: y, idx = array_ops.unique(x) - tf_y, tf_idx = sess.run([y, idx]) + tf_y, tf_idx = self.evaluate([y, idx]) self.assertEqual(len(x), len(tf_idx)) self.assertEqual(len(tf_y), len(np.unique(x))) @@ -67,9 +67,9 @@ class UniqueTest(test.TestCase): x = np.array([[1, 0, 0], [1, 0, 0], [2, 0, 0]]) with self.cached_session() as sess: y0, idx0 = gen_array_ops.unique_v2(x, axis=np.array([0], dtype)) - tf_y0, tf_idx0 = sess.run([y0, idx0]) + tf_y0, tf_idx0 = self.evaluate([y0, idx0]) y1, idx1 = gen_array_ops.unique_v2(x, axis=np.array([1], dtype)) - tf_y1, tf_idx1 = sess.run([y1, idx1]) + tf_y1, tf_idx1 = self.evaluate([y1, idx1]) self.assertAllEqual(tf_y0, np.array([[1, 0, 0], [2, 0, 0]])) self.assertAllEqual(tf_idx0, np.array([0, 0, 1])) self.assertAllEqual(tf_y1, np.array([[1, 0], [1, 0], [2, 0]])) @@ -81,7 +81,7 @@ class UniqueTest(test.TestCase): x = np.random.randint(2, high=10, size=7000) with self.cached_session() as sess: y, idx = gen_array_ops.unique_v2(x, axis=np.array([], np.int32)) - tf_y, tf_idx = sess.run([y, idx]) + tf_y, tf_idx = self.evaluate([y, idx]) self.assertEqual(len(x), len(tf_idx)) self.assertEqual(len(tf_y), len(np.unique(x))) @@ -95,7 +95,7 @@ class UniqueWithCountsTest(test.TestCase): x = np.random.randint(2, high=10, size=7000) with self.cached_session() as sess: y, idx, count = array_ops.unique_with_counts(x) - tf_y, tf_idx, tf_count = sess.run([y, idx, count]) + tf_y, tf_idx, tf_count = self.evaluate([y, idx, count]) self.assertEqual(len(x), len(tf_idx)) self.assertEqual(len(tf_y), len(np.unique(x))) @@ -108,7 +108,7 @@ class UniqueWithCountsTest(test.TestCase): x = np.random.randint(2, high=10, size=7000) with self.cached_session() as sess: y, idx, count = array_ops.unique_with_counts(x, out_idx=dtypes.int64) - tf_y, tf_idx, tf_count = sess.run([y, idx, count]) + tf_y, tf_idx, tf_count = self.evaluate([y, idx, count]) self.assertEqual(len(x), len(tf_idx)) self.assertEqual(len(tf_y), len(np.unique(x))) @@ -123,7 +123,7 @@ class UniqueWithCountsTest(test.TestCase): with self.cached_session() as sess: y, idx, count = array_ops.unique_with_counts(x) - tf_y, tf_idx, tf_count = sess.run([y, idx, count]) + tf_y, tf_idx, tf_count = self.evaluate([y, idx, count]) self.assertEqual(len(x), len(tf_idx)) self.assertEqual(len(tf_y), len(np.unique(x))) @@ -139,10 +139,10 @@ class UniqueWithCountsTest(test.TestCase): with self.cached_session() as sess: y0, idx0, count0 = gen_array_ops.unique_with_counts_v2( x, axis=np.array([0], dtype)) - tf_y0, tf_idx0, tf_count0 = sess.run([y0, idx0, count0]) + tf_y0, tf_idx0, tf_count0 = self.evaluate([y0, idx0, count0]) y1, idx1, count1 = gen_array_ops.unique_with_counts_v2( x, axis=np.array([1], dtype)) - tf_y1, tf_idx1, tf_count1 = sess.run([y1, idx1, count1]) + tf_y1, tf_idx1, tf_count1 = self.evaluate([y1, idx1, count1]) self.assertAllEqual(tf_y0, np.array([[1, 0, 0], [2, 0, 0]])) self.assertAllEqual(tf_idx0, np.array([0, 0, 1])) self.assertAllEqual(tf_count0, np.array([2, 1])) @@ -157,7 +157,7 @@ class UniqueWithCountsTest(test.TestCase): with self.cached_session() as sess: y, idx, count = gen_array_ops.unique_with_counts_v2( x, axis=np.array([], np.int32)) - tf_y, tf_idx, tf_count = sess.run([y, idx, count]) + tf_y, tf_idx, tf_count = self.evaluate([y, idx, count]) self.assertEqual(len(x), len(tf_idx)) self.assertEqual(len(tf_y), len(np.unique(x))) diff --git a/tensorflow/python/kernel_tests/unstack_op_test.py b/tensorflow/python/kernel_tests/unstack_op_test.py index 6aea42990ac..f5ba475e7ad 100644 --- a/tensorflow/python/kernel_tests/unstack_op_test.py +++ b/tensorflow/python/kernel_tests/unstack_op_test.py @@ -41,7 +41,7 @@ class UnstackOpTest(test.TestCase): def testSimple(self): np.random.seed(7) - with self.session(use_gpu=True): + with test_util.use_gpu(): for shape in (2,), (3,), (2, 3), (3, 2), (4, 3, 2): for dtype in [ np.bool, np.float16, np.float32, np.float64, np.int32, np.int64 @@ -53,14 +53,15 @@ class UnstackOpTest(test.TestCase): cs = array_ops.unstack(x, num=shape[0]) self.assertEqual(type(cs), list) self.assertEqual(len(cs), shape[0]) - cs = [c.eval() for c in cs] + cs = [self.evaluate(c) for c in cs] self.assertAllEqual(cs, data) def testSimpleGpu(self): if not test_util.is_gpu_available(): self.skipTest('No GPU available') + np.random.seed(7) - with self.session(use_gpu=True, force_gpu=True): + with test_util.force_gpu(): for shape in (2,), (3,), (2, 3), (3, 2), (4, 3, 2): for dtype in [np.float16, np.float32, np.float64, np.int32, np.int64]: data = np.random.randn(*shape).astype(dtype) @@ -70,9 +71,10 @@ class UnstackOpTest(test.TestCase): cs = array_ops.unstack(x, num=shape[0]) self.assertEqual(type(cs), list) self.assertEqual(len(cs), shape[0]) - cs = [c.eval() for c in cs] + cs = [self.evaluate(c) for c in cs] self.assertAllEqual(cs, data) + @test_util.run_deprecated_v1 def testGradientsAxis0(self): for shape in (2,), (3,), (2, 3), (3, 2), (4, 3, 2): data = np.random.randn(*shape) @@ -85,6 +87,7 @@ class UnstackOpTest(test.TestCase): shapes[i]) self.assertLess(err, 1e-6) + @test_util.run_deprecated_v1 def testGradientsAxis1(self): for shape in (2, 3), (3, 2), (4, 3, 2): data = np.random.randn(*shape) @@ -98,6 +101,7 @@ class UnstackOpTest(test.TestCase): out_shape) self.assertLess(err, 1e-6) + @test_util.run_deprecated_v1 def testInferNum(self): with self.cached_session(): for shape in (2,), (3,), (2, 3), (3, 2), (4, 3, 2): @@ -106,16 +110,19 @@ class UnstackOpTest(test.TestCase): self.assertEqual(type(cs), list) self.assertEqual(len(cs), shape[0]) + @test_util.run_deprecated_v1 def testCannotInferNumFromUnknownShape(self): x = array_ops.placeholder(np.float32) with self.assertRaisesRegexp(ValueError, r'Cannot infer num from shape '): array_ops.unstack(x) + @test_util.run_deprecated_v1 def testUnknownShapeOkWithNum(self): x = array_ops.placeholder(np.float32) array_ops.unstack(x, num=2) + @test_util.run_deprecated_v1 def testCannotInferNumFromNoneShape(self): x = array_ops.placeholder(np.float32, shape=(None,)) with self.assertRaisesRegexp(ValueError, @@ -131,15 +138,13 @@ class UnstackOpTest(test.TestCase): for j in range(-i, i): expected = np_split_squeeze(a, j) - with self.cached_session() as sess: - actual_unstack = sess.run(array_ops.unstack(a, axis=j)) + actual_unstack = self.evaluate(array_ops.unstack(a, axis=j)) self.assertAllEqual(expected, actual_unstack) def testAxis0Default(self): - with self.cached_session() as sess: - a = constant_op.constant([[1, 2, 3], [4, 5, 6]], name='a') - unstacked = sess.run(array_ops.unstack(a)) + a = constant_op.constant([[1, 2, 3], [4, 5, 6]], name='a') + unstacked = self.evaluate(array_ops.unstack(a)) self.assertEqual(len(unstacked), 2) self.assertAllEqual(unstacked[0], [1, 2, 3]) @@ -156,10 +161,9 @@ class UnstackOpTest(test.TestCase): array_ops.unstack(a, axis=-3) def testZeroLengthDim(self): - with self.cached_session(): - x = array_ops.zeros(shape=(0, 1, 2)) - y = array_ops.unstack(x, axis=1)[0].eval() - self.assertEqual(y.shape, (0, 2)) + x = array_ops.zeros(shape=(0, 1, 2)) + y = self.evaluate(array_ops.unstack(x, axis=1)[0]) + self.assertEqual(y.shape, (0, 2)) if __name__ == '__main__': diff --git a/tensorflow/python/kernel_tests/variable_ops_test.py b/tensorflow/python/kernel_tests/variable_ops_test.py index 3d2f8b61555..0f3e2619925 100644 --- a/tensorflow/python/kernel_tests/variable_ops_test.py +++ b/tensorflow/python/kernel_tests/variable_ops_test.py @@ -24,6 +24,7 @@ from tensorflow.python.framework import dtypes from tensorflow.python.framework import errors from tensorflow.python.framework import ops from tensorflow.python.framework import tensor_shape +from tensorflow.python.framework import test_util from tensorflow.python.ops import array_ops from tensorflow.python.ops import gen_state_ops from tensorflow.python.ops import math_ops @@ -46,7 +47,7 @@ class VariableOpTest(test.TestCase): p = state_ops.variable_op(x.shape, tftype) op = state_ops.assign(p, x) op.op.run() - return p.eval() + return self.evaluate(p) def _testTypes(self, vals): for dtype in [np.float32, np.float64, np.int32, np.int64]: @@ -59,15 +60,18 @@ class VariableOpTest(test.TestCase): # that Variable and Assign have GPU implementations for matching tf. self.assertAllEqual(x, self._initFetch(x, tftype, use_gpu=True)) + @test_util.run_deprecated_v1 def testBasic(self): self._testTypes(np.arange(0, 20).reshape([4, 5])) + @test_util.run_deprecated_v1 def testset_shape(self): p = state_ops.variable_op([1, 2], dtypes.float32) self.assertEqual([1, 2], p.get_shape()) p = state_ops.variable_op([1, 2], dtypes.float32, set_shape=False) self.assertEqual(tensor_shape.unknown_shape(), p.get_shape()) + @test_util.run_deprecated_v1 def testAssign(self): value = np.array([[42.0, 43.0]]) var = state_ops.variable_op(value.shape, dtypes.float32) @@ -75,6 +79,7 @@ class VariableOpTest(test.TestCase): assigned = state_ops.assign(var, value) self.assertShapeEqual(value, assigned) + @test_util.run_deprecated_v1 def testAssignNoValidateShape(self): value = np.array([[42.0, 43.0]]) var = state_ops.variable_op(value.shape, dtypes.float32) @@ -82,6 +87,7 @@ class VariableOpTest(test.TestCase): assigned = state_ops.assign(var, value, validate_shape=False) self.assertShapeEqual(value, assigned) + @test_util.run_deprecated_v1 def testAssignNoVarShape(self): value = np.array([[42.0, 43.0]]) var = state_ops.variable_op(value.shape, dtypes.float32, set_shape=False) @@ -89,6 +95,7 @@ class VariableOpTest(test.TestCase): assigned = state_ops.assign(var, value) self.assertShapeEqual(value, assigned) + @test_util.run_deprecated_v1 def testAssignNoVarShapeNoValidateShape(self): value = np.array([[42.0, 43.0]]) var = state_ops.variable_op(value.shape, dtypes.float32, set_shape=False) @@ -101,6 +108,7 @@ class VariableOpTest(test.TestCase): self.assertEqual(tensor_shape.unknown_shape(), tensor.get_shape()) return tensor + @test_util.run_deprecated_v1 def testAssignNoValueShape(self): value = self._NewShapelessTensor() shape = [1, 2] @@ -109,6 +117,7 @@ class VariableOpTest(test.TestCase): self.assertEqual(shape, var.get_shape()) self.assertEqual(shape, assigned.get_shape()) + @test_util.run_deprecated_v1 def testAssignNoValueShapeNoValidateShape(self): value = self._NewShapelessTensor() shape = [1, 2] @@ -117,6 +126,7 @@ class VariableOpTest(test.TestCase): assigned = state_ops.assign(var, value, validate_shape=False) self.assertEqual(tensor_shape.unknown_shape(), assigned.get_shape()) + @test_util.run_deprecated_v1 def testAssignNoShape(self): with self.cached_session(): value = self._NewShapelessTensor() @@ -125,6 +135,7 @@ class VariableOpTest(test.TestCase): self.assertEqual(tensor_shape.unknown_shape(), state_ops.assign(var, value).get_shape()) + @test_util.run_deprecated_v1 def testAssignNoShapeNoValidateShape(self): with self.cached_session(): value = self._NewShapelessTensor() @@ -135,6 +146,7 @@ class VariableOpTest(test.TestCase): state_ops.assign( var, value, validate_shape=False).get_shape()) + @test_util.run_deprecated_v1 def testAssignUpdate(self): var = state_ops.variable_op([1, 2], dtypes.float32) added = state_ops.assign_add(var, [[2.0, 3.0]]) @@ -142,6 +154,7 @@ class VariableOpTest(test.TestCase): subbed = state_ops.assign_sub(var, [[12.0, 13.0]]) self.assertEqual([1, 2], subbed.get_shape()) + @test_util.run_deprecated_v1 def testAssignUpdateNoVarShape(self): var = state_ops.variable_op([1, 2], dtypes.float32, set_shape=False) added = state_ops.assign_add(var, [[2.0, 3.0]]) @@ -149,6 +162,7 @@ class VariableOpTest(test.TestCase): subbed = state_ops.assign_sub(var, [[12.0, 13.0]]) self.assertEqual([1, 2], subbed.get_shape()) + @test_util.run_deprecated_v1 def testAssignUpdateNoValueShape(self): var = state_ops.variable_op([1, 2], dtypes.float32) added = state_ops.assign_add(var, self._NewShapelessTensor()) @@ -156,6 +170,7 @@ class VariableOpTest(test.TestCase): subbed = state_ops.assign_sub(var, self._NewShapelessTensor()) self.assertEqual([1, 2], subbed.get_shape()) + @test_util.run_deprecated_v1 def testAssignUpdateNoShape(self): var = state_ops.variable_op([1, 2], dtypes.float32, set_shape=False) added = state_ops.assign_add(var, self._NewShapelessTensor()) @@ -163,24 +178,27 @@ class VariableOpTest(test.TestCase): subbed = state_ops.assign_sub(var, self._NewShapelessTensor()) self.assertEqual(tensor_shape.unknown_shape(), subbed.get_shape()) + @test_util.run_deprecated_v1 def testTemporaryVariable(self): - with self.test_session(use_gpu=True): + with test_util.use_gpu(): var = gen_state_ops.temporary_variable( [1, 2], dtypes.float32, var_name="foo") var = state_ops.assign(var, [[4.0, 5.0]]) var = state_ops.assign_add(var, [[6.0, 7.0]]) final = gen_state_ops.destroy_temporary_variable(var, var_name="foo") - self.assertAllClose([[10.0, 12.0]], final.eval()) + self.assertAllClose([[10.0, 12.0]], self.evaluate(final)) + @test_util.run_deprecated_v1 def testDestroyNonexistentTemporaryVariable(self): - with self.test_session(use_gpu=True): + with test_util.use_gpu(): var = gen_state_ops.temporary_variable([1, 2], dtypes.float32) final = gen_state_ops.destroy_temporary_variable(var, var_name="bad") with self.assertRaises(errors.NotFoundError): - final.eval() + self.evaluate(final) + @test_util.run_deprecated_v1 def testDuplicateTemporaryVariable(self): - with self.test_session(use_gpu=True): + with test_util.use_gpu(): var1 = gen_state_ops.temporary_variable( [1, 2], dtypes.float32, var_name="dup") var1 = state_ops.assign(var1, [[1.0, 2.0]]) @@ -189,48 +207,53 @@ class VariableOpTest(test.TestCase): var2 = state_ops.assign(var2, [[3.0, 4.0]]) final = var1 + var2 with self.assertRaises(errors.AlreadyExistsError): - final.eval() + self.evaluate(final) + @test_util.run_deprecated_v1 def testDestroyTemporaryVariableTwice(self): - with self.test_session(use_gpu=True): + with test_util.use_gpu(): var = gen_state_ops.temporary_variable([1, 2], dtypes.float32) val1 = gen_state_ops.destroy_temporary_variable(var, var_name="dup") val2 = gen_state_ops.destroy_temporary_variable(var, var_name="dup") final = val1 + val2 with self.assertRaises(errors.NotFoundError): - final.eval() + self.evaluate(final) + @test_util.run_deprecated_v1 def testTemporaryVariableNoLeak(self): - with self.test_session(use_gpu=True): + with test_util.use_gpu(): var = gen_state_ops.temporary_variable( [1, 2], dtypes.float32, var_name="bar") final = array_ops.identity(var) - final.eval() + self.evaluate(final) + @test_util.run_deprecated_v1 def testTwoTemporaryVariablesNoLeaks(self): - with self.test_session(use_gpu=True): + with test_util.use_gpu(): var1 = gen_state_ops.temporary_variable( [1, 2], dtypes.float32, var_name="var1") var2 = gen_state_ops.temporary_variable( [1, 2], dtypes.float32, var_name="var2") final = var1 + var2 - final.eval() + self.evaluate(final) + @test_util.run_deprecated_v1 def testAssignDependencyAcrossDevices(self): - with self.test_session(use_gpu=True): + with test_util.use_gpu(): # The variable and an op to increment it are on the GPU. var = state_ops.variable_op([1], dtypes.float32) - state_ops.assign(var, [1.0]).eval() + self.evaluate(state_ops.assign(var, [1.0])) increment = state_ops.assign_add(var, [1.0]) with ops.control_dependencies([increment]): - with ops.device("/cpu:0"): + with test_util.force_cpu(): # This mul op is pinned to the CPU, but reads the variable from the # GPU. The test ensures that the dependency on 'increment' is still # honored, i.e., the Send and Recv from GPU to CPU should take place # only after the increment. result = math_ops.multiply(var, var) - self.assertAllClose([4.0], result.eval()) + self.assertAllClose([4.0], self.evaluate(result)) + @test_util.run_deprecated_v1 def testIsVariableInitialized(self): for use_gpu in [True, False]: with self.test_session(use_gpu=use_gpu): diff --git a/tensorflow/python/kernel_tests/variable_scope_test.py b/tensorflow/python/kernel_tests/variable_scope_test.py index 2ba064f8a50..44d4bd5e30f 100644 --- a/tensorflow/python/kernel_tests/variable_scope_test.py +++ b/tensorflow/python/kernel_tests/variable_scope_test.py @@ -152,6 +152,7 @@ class VariableScopeTest(test.TestCase): # TypeError: Fetch argument # has invalid type , must be a string or Tensor. # (Can not convert a ResourceVariable into a Tensor or Operation.) + @test_util.run_deprecated_v1 def testStringDefaultInitializer(self): with self.cached_session(): v = variable_scope.get_variable("string", shape=[], dtype=dtypes.string) @@ -308,9 +309,9 @@ class VariableScopeTest(test.TestCase): self.evaluate(variables_lib.global_variables_initializer()) self.assertAllEqual(self.evaluate(x.value()), self.evaluate(y.value())) - # TODO(alive): support variable partitioning/caching in eager mode. # TODO(mihaimaruseac): Not converted to use wrap_function because of # InvalidArgumentError: /job:moo/replica:0/task:0/device:CPU:0 unknown device. + @test_util.run_deprecated_v1 def testVarScopeCachingDevice(self): with self.cached_session(): caching_device = "/job:moo" @@ -425,6 +426,7 @@ class VariableScopeTest(test.TestCase): # invalid type , must # be a string or Tensor. (Can not convert a ResourceVariable into a Tensor or # Operation.) + @test_util.run_deprecated_v1 def testControlDeps(self): with self.cached_session() as sess: v0 = variable_scope.get_variable( @@ -435,22 +437,23 @@ class VariableScopeTest(test.TestCase): add = v1 + v0 # v0 should be uninitialized. with self.assertRaisesRegexp(errors.OpError, "uninitialized"): - sess.run(v0) + self.evaluate(v0) # We should be able to initialize and run v1 without initializing # v0, even if the variable was created with a control dep on v0. - sess.run(v1.initializer) - self.assertEqual(1, sess.run(v1)) + self.evaluate(v1.initializer) + self.assertEqual(1, self.evaluate(v1)) # v0 should still be uninitialized. with self.assertRaisesRegexp(errors.OpError, "uninitialized"): - sess.run(v0) + self.evaluate(v0) with self.assertRaisesRegexp(errors.OpError, "uninitialized"): - sess.run(add) + self.evaluate(add) # If we initialize v0 we should be able to run 'add'. - sess.run(v0.initializer) - sess.run(add) + self.evaluate(v0.initializer) + self.evaluate(add) # TODO(mihaimaruseac): Not converted to use wrap_function because of # AssertionError: True is not false (last assertFalse) + @test_util.run_deprecated_v1 def testEnableResourceVariables(self): old = variable_scope._DEFAULT_USE_RESOURCE try: @@ -465,6 +468,7 @@ class VariableScopeTest(test.TestCase): # TODO(mihaimaruseac): Not converted to use wrap_function because of # TypeError: Fetch argument None has invalid type + @test_util.run_deprecated_v1 def testControlFlow(self): with self.cached_session() as sess: v0 = variable_scope.get_variable( @@ -490,19 +494,19 @@ class VariableScopeTest(test.TestCase): v2 = var_dict["v2"] # We should be able to initialize and run v1 and v2 without initializing # v0, even if the variable was created with a control dep on v0. - sess.run(v1.initializer) - self.assertEqual([1], sess.run(v1)) - sess.run(v2.initializer) - self.assertEqual([2], sess.run(v2)) + self.evaluate(v1.initializer) + self.assertEqual([1], self.evaluate(v1)) + self.evaluate(v2.initializer) + self.assertEqual([2], self.evaluate(v2)) # v0 should still be uninitialized. with self.assertRaisesRegexp(errors.OpError, "uninitialized"): - sess.run(v0) + self.evaluate(v0) # We should not be able to run 'add' yet. with self.assertRaisesRegexp(errors.OpError, "uninitialized"): - sess.run(add) + self.evaluate(add) # If we initialize v0 we should be able to run 'add'. - sess.run(v0.initializer) - sess.run(add) + self.evaluate(v0.initializer) + self.evaluate(add) # TODO(mihaimaruseac): Not converted to use wrap_function because of # TypeError: Expected tf.group() expected Tensor arguments not 'None' with @@ -649,7 +653,7 @@ class VariableScopeTest(test.TestCase): "testVarScopeGetOrCreateReuse_bar", reuse=variable_scope.AUTO_REUSE): _ = variable_scope.get_variable("var", []) - self.assertEqual(value, x.eval()) + self.assertEqual(value, self.evaluate(x)) test_value(42.) # Variable is created. test_value(13.) # Variable is reused hereafter. @@ -1149,6 +1153,7 @@ class VariableScopeTest(test.TestCase): # TODO(mihaimaruseac): Not converted to use wrap_function because of # obtaining different results in the eager case compared to the graph one + @test_util.run_deprecated_v1 def testGetCollection(self): with self.cached_session(): _ = variable_scope.get_variable("testGetCollection_a", []) @@ -1205,6 +1210,7 @@ class VariableScopeTest(test.TestCase): # TODO(mihaimaruseac): Not converted to use wrap_function because of # obtaining different results in the eager case compared to the graph one + @test_util.run_deprecated_v1 def testGetTrainableVariablesWithGetVariable(self): with self.cached_session(): _ = variable_scope.get_variable("testGetTrainableVariables_a", []) @@ -1243,6 +1249,7 @@ class VariableScopeTest(test.TestCase): # TODO(mihaimaruseac): Not converted to use wrap_function because of # obtaining different results in the eager case compared to the graph one + @test_util.run_deprecated_v1 def testGetTrainableVariablesWithVariable(self): with self.cached_session(): _ = variable_scope.variable(1.0, name="testGetTrainableVariables_a") @@ -1284,6 +1291,7 @@ class VariableScopeTest(test.TestCase): # TODO(mihaimaruseac): Not converted to use wrap_function because of # obtaining different results in the eager case compared to the graph one + @test_util.run_deprecated_v1 def testGetGlobalVariables(self): with self.cached_session(): _ = variable_scope.get_variable("testGetGlobalVariables_a", []) @@ -1296,6 +1304,7 @@ class VariableScopeTest(test.TestCase): # TODO(mihaimaruseac): Not converted to use wrap_function because of # obtaining different results in the eager case compared to the graph one + @test_util.run_deprecated_v1 def testGetLocalVariables(self): with self.cached_session(): _ = variable_scope.get_variable( @@ -1313,6 +1322,28 @@ class VariableScopeTest(test.TestCase): # Ensure it is possible to do get_variable with a _ref dtype passed in. _ = variable_scope.get_variable("w", shape=[5, 6], dtype=v.dtype) + @test_util.run_in_graph_and_eager_modes + @run_inside_wrap_function_in_eager_mode + def testGetVariableWithInitializerWhichTakesNoArgs(self): + v = variable_scope.get_variable("foo", initializer=lambda: [2]) + self.assertEqual(v.name, "foo:0") + + @test_util.run_in_graph_and_eager_modes + @run_inside_wrap_function_in_eager_mode + def testGetVariableWithInitializerWhichTakesOptionalArgs(self): + v = variable_scope.get_variable("foo", initializer=lambda x=True: [2]) + self.assertEqual(v.name, "foo:0") + + @test_util.run_in_graph_and_eager_modes + @run_inside_wrap_function_in_eager_mode + def testGetVariableWithInitializerWhichTakesUnprovidedArgsAndNoShape(self): + with self.assertRaisesRegexp( + ValueError, + "The initializer passed is not valid. It should be a callable with no " + "arguments and the shape should not be provided or an instance of " + "`tf.keras.initializers.*' and `shape` should be fully defined."): + variable_scope.get_variable("foo", initializer=lambda x: [2]) + @test_util.run_in_graph_and_eager_modes @run_inside_wrap_function_in_eager_mode def testTwoGraphs(self): @@ -1349,6 +1380,7 @@ class VariableScopeWithPartitioningTest(test.TestCase): # TODO(mihaimaruseac): Not converted to use wrap_function because of # obtaining different results in the eager case compared to the graph one + @test_util.run_deprecated_v1 def testResultNameMatchesRequested(self): with variable_scope.variable_scope( "scope0", partitioner=axis0_into2_partitioner): @@ -1404,6 +1436,14 @@ class VariableScopeWithPartitioningTest(test.TestCase): v_reused = variable_scope.get_variable("name0") self.assertEqual(v, v_reused) + def testNoReuseInEagerByDefault(self): + with context.eager_mode(): + with variable_scope.variable_scope( + "scope0", partitioner=axis0_into2_partitioner): + v1 = variable_scope.get_variable("name0", shape=(3, 1, 1)) + v2 = variable_scope.get_variable("name0", shape=(3, 1, 1)) + self.assertIsNot(v1, v2) + @test_util.run_in_graph_and_eager_modes @run_inside_wrap_function_in_eager_mode def testPropagatePartitionerOnReopening(self): @@ -1415,6 +1455,7 @@ class VariableScopeWithPartitioningTest(test.TestCase): # TODO(mihaimaruseac): Not converted to use wrap_function because of # obtaining different results in the eager case compared to the graph one + @test_util.run_deprecated_v1 def testScalarIgnoresPartitioner(self): with variable_scope.variable_scope( "scope0", partitioner=axis0_into2_partitioner): @@ -1459,6 +1500,10 @@ class VariableScopeWithPartitioningTest(test.TestCase): def testPartitionConcatenatesAlongCorrectAxisResource(self): self._testPartitionConcatenatesAlongCorrectAxis(use_resource=True) + def testPartitionConcatenatesAlongCorrectAxisResourceInEager(self): + with context.eager_mode(): + self._testPartitionConcatenatesAlongCorrectAxis(use_resource=True) + class VariableScopeWithCustomGetterTest(test.TestCase): @@ -1550,6 +1595,7 @@ class VariableScopeWithCustomGetterTest(test.TestCase): # dtype=float32> cannot be interpreted as a Tensor. (Tensor # Tensor("custom_getter/add:0", shape=(1, 2, 3), dtype=float32) is not an # element of this graph.) + @test_util.run_deprecated_v1 def testGetterThatCreatesTwoVariablesAndSumsThem(self): def custom_getter(getter, name, *args, **kwargs): @@ -1569,7 +1615,7 @@ class VariableScopeWithCustomGetterTest(test.TestCase): self.assertEqual("custom_getter/add:0", v.name) with self.cached_session() as sess: variables_lib.global_variables_initializer().run() - np_vars, np_v = sess.run([true_vars, v]) + np_vars, np_v = self.evaluate([true_vars, v]) self.assertAllClose(np_v, sum(np_vars)) # TODO(mihaimaruseac): Not converted to use wrap_function because of @@ -1577,6 +1623,7 @@ class VariableScopeWithCustomGetterTest(test.TestCase): # dtype=float32> cannot be interpreted as a Tensor. (Tensor # Tensor("sum_getter_2/add:0", shape=(1, 2, 3), dtype=float32) is not an # element of this graph.) + @test_util.run_deprecated_v1 def testNestedCustomGetters(self): def sum_getter(getter, name, *args, **kwargs): @@ -1614,7 +1661,7 @@ class VariableScopeWithCustomGetterTest(test.TestCase): with self.cached_session() as sess: variables_lib.global_variables_initializer().run() - np_vars, np_v = sess.run([true_vars, v]) + np_vars, np_v = self.evaluate([true_vars, v]) # take products of sums of products self.assertAllClose( np_v, (((np_vars[0] * np_vars[1]) + (np_vars[2] * np_vars[3])) + ( diff --git a/tensorflow/python/kernel_tests/variables_test.py b/tensorflow/python/kernel_tests/variables_test.py index b3eebf83168..08d885e8a87 100644 --- a/tensorflow/python/kernel_tests/variables_test.py +++ b/tensorflow/python/kernel_tests/variables_test.py @@ -18,6 +18,7 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function +import functools import operator import numpy as np @@ -27,6 +28,7 @@ from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes from tensorflow.python.framework import errors_impl from tensorflow.python.framework import ops +from tensorflow.python.framework import test_util from tensorflow.python.ops import array_ops from tensorflow.python.ops import control_flow_ops from tensorflow.python.ops import gen_state_ops @@ -41,6 +43,7 @@ from tensorflow.python.util import compat class VariablesTestCase(test.TestCase): + @test_util.run_deprecated_v1 def testInitialization(self): with self.cached_session(): var0 = variables.VariableV1(0.0) @@ -58,16 +61,17 @@ class VariablesTestCase(test.TestCase): self.assertEqual([], var1.shape) with self.assertRaisesOpError("Attempting to use uninitialized value"): - var0.eval() + self.evaluate(var0) with self.assertRaisesOpError("Attempting to use uninitialized value"): - var1.eval() + self.evaluate(var1) variables.global_variables_initializer().run() - self.assertAllClose(0.0, var0.eval()) - self.assertAllClose(1.1, var1.eval()) + self.assertAllClose(0.0, self.evaluate(var0)) + self.assertAllClose(1.1, self.evaluate(var1)) + @test_util.run_deprecated_v1 def testInitializationOrder(self): with self.cached_session(): rnd = variables.Variable(random_ops.random_uniform([3, 6]), name="rnd") @@ -94,8 +98,9 @@ class VariablesTestCase(test.TestCase): variables.global_variables_initializer().run() - self.assertAllClose(rnd.eval(), dep.eval()) - self.assertAllClose(rnd.eval() + dep.eval() + 2.0, depdep.eval()) + self.assertAllClose(rnd.eval(), self.evaluate(dep)) + self.assertAllClose(rnd.eval() + self.evaluate(dep) + 2.0, + self.evaluate(depdep)) def testIterable(self): with self.assertRaisesRegexp(TypeError, "not iterable"): @@ -105,6 +110,7 @@ class VariablesTestCase(test.TestCase): for _ in variables.Variable([0.0, 1.0]): pass + @test_util.run_deprecated_v1 def testAssignments(self): with self.cached_session(): var = variables.Variable(0.0) @@ -112,17 +118,18 @@ class VariablesTestCase(test.TestCase): minus_one = var.assign_sub(2.0) four = var.assign(4.0) variables.global_variables_initializer().run() - self.assertAllClose(0.0, var.eval()) + self.assertAllClose(0.0, self.evaluate(var)) - self.assertAllClose(1.0, plus_one.eval()) - self.assertAllClose(1.0, var.eval()) + self.assertAllClose(1.0, self.evaluate(plus_one)) + self.assertAllClose(1.0, self.evaluate(var)) - self.assertAllClose(-1.0, minus_one.eval()) - self.assertAllClose(-1.0, var.eval()) + self.assertAllClose(-1.0, self.evaluate(minus_one)) + self.assertAllClose(-1.0, self.evaluate(var)) - self.assertAllClose(4.0, four.eval()) - self.assertAllClose(4.0, var.eval()) + self.assertAllClose(4.0, self.evaluate(four)) + self.assertAllClose(4.0, self.evaluate(var)) + @test_util.run_deprecated_v1 def testResourceAssignments(self): with self.session(use_gpu=True): var = resource_variable_ops.ResourceVariable(0.0) @@ -130,16 +137,16 @@ class VariablesTestCase(test.TestCase): minus_one = var.assign_sub(2.0) four = var.assign(4.0) variables.global_variables_initializer().run() - self.assertAllClose(0.0, var.eval()) + self.assertAllClose(0.0, self.evaluate(var)) - plus_one.eval() - self.assertAllClose(1.0, var.eval()) + self.evaluate(plus_one) + self.assertAllClose(1.0, self.evaluate(var)) - minus_one.eval() - self.assertAllClose(-1.0, var.eval()) + self.evaluate(minus_one) + self.assertAllClose(-1.0, self.evaluate(var)) - four.eval() - self.assertAllClose(4.0, var.eval()) + self.evaluate(four) + self.assertAllClose(4.0, self.evaluate(var)) def testZeroSizeStringAssign(self): with self.cached_session() as sess: @@ -148,10 +155,10 @@ class VariablesTestCase(test.TestCase): name="foo", trainable=False, collections=[ops.GraphKeys.LOCAL_VARIABLES]) - sess.run(variables.local_variables_initializer()) + self.evaluate(variables.local_variables_initializer()) old_value = array.value() copy_op = array.assign(old_value) - self.assertEqual([], list(sess.run(copy_op))) + self.assertEqual([], list(self.evaluate(copy_op))) def _countUpToTest(self, dtype): with self.cached_session(): @@ -160,31 +167,34 @@ class VariablesTestCase(test.TestCase): count_up_to = var.count_up_to(3) variables.global_variables_initializer().run() - self.assertEqual(0, var.eval()) + self.assertEqual(0, self.evaluate(var)) - self.assertEqual(0, count_up_to.eval()) - self.assertEqual(1, var.eval()) + self.assertEqual(0, self.evaluate(count_up_to)) + self.assertEqual(1, self.evaluate(var)) - self.assertEqual(1, count_up_to.eval()) - self.assertEqual(2, var.eval()) + self.assertEqual(1, self.evaluate(count_up_to)) + self.assertEqual(2, self.evaluate(var)) - self.assertEqual(2, count_up_to.eval()) - self.assertEqual(3, var.eval()) + self.assertEqual(2, self.evaluate(count_up_to)) + self.assertEqual(3, self.evaluate(var)) with self.assertRaisesOpError("Reached limit of 3"): - count_up_to.eval() - self.assertEqual(3, var.eval()) + self.evaluate(count_up_to) + self.assertEqual(3, self.evaluate(var)) with self.assertRaisesOpError("Reached limit of 3"): - count_up_to.eval() - self.assertEqual(3, var.eval()) + self.evaluate(count_up_to) + self.assertEqual(3, self.evaluate(var)) + @test_util.run_deprecated_v1 def testCountUpToInt32(self): self._countUpToTest(dtypes.int32) + @test_util.run_deprecated_v1 def testCountUpToInt64(self): self._countUpToTest(dtypes.int64) + @test_util.run_deprecated_v1 def testControlDepsNone(self): with self.cached_session(): c = constant_op.constant(1.0) @@ -198,6 +208,7 @@ class VariablesTestCase(test.TestCase): self.assertEqual([], var_x.value().op.control_inputs) self.assertEqual([], var_x._ref().op.control_inputs) # pylint: disable=protected-access + @test_util.run_deprecated_v1 def testControlFlow(self): with self.cached_session() as sess: v0 = variables.Variable(0, name="v0") @@ -220,20 +231,21 @@ class VariablesTestCase(test.TestCase): v2 = var_dict["v2"] # We should be able to initialize and run v1 and v2 without initializing # v0, even if the variable was created with a control dep on v0. - sess.run(v1.initializer) - self.assertEqual([1], sess.run(v1)) - sess.run(v2.initializer) - self.assertEqual([2], sess.run(v2)) + self.evaluate(v1.initializer) + self.assertEqual([1], self.evaluate(v1)) + self.evaluate(v2.initializer) + self.assertEqual([2], self.evaluate(v2)) # v0 should still be uninitialized. with self.assertRaisesRegexp(errors_impl.OpError, "uninitialized"): - sess.run(v0) + self.evaluate(v0) # We should not be able to run 'add' yet. with self.assertRaisesRegexp(errors_impl.OpError, "uninitialized"): - sess.run(add) + self.evaluate(add) # If we initialize v0 we should be able to run 'add'. - sess.run(v0.initializer) - sess.run(add) + self.evaluate(v0.initializer) + self.evaluate(add) + @test_util.run_deprecated_v1 def testControlFlowInitialization(self): """Expects an error if an initializer is in a control-flow scope.""" def cond(i, _): @@ -247,15 +259,17 @@ class VariablesTestCase(test.TestCase): with self.assertRaisesRegexp(ValueError, "inside a control-flow"): control_flow_ops.while_loop(cond, body, [0, 0]) + @test_util.run_deprecated_v1 def testUseVariableAsTensor(self): with self.cached_session(): var_x = variables.Variable(2.0) var_y = variables.Variable(3.0) variables.global_variables_initializer().run() - self.assertAllClose(2.0, var_x.eval()) - self.assertAllClose(3.0, var_y.eval()) + self.assertAllClose(2.0, self.evaluate(var_x)) + self.assertAllClose(3.0, self.evaluate(var_y)) self.assertAllClose(5.0, math_ops.add(var_x, var_y).eval()) + @test_util.run_deprecated_v1 def testZeroSizeVarSameAsConst(self): with self.cached_session(): zero_size_var = variables.Variable(array_ops.zeros([0, 2])) @@ -264,10 +278,11 @@ class VariablesTestCase(test.TestCase): const_mul = math_ops.matmul( zero_size_const, zero_size_const, transpose_b=True) variables.global_variables_initializer().run() - variable_output = variable_mul.eval() + variable_output = self.evaluate(variable_mul) self.assertAllClose(const_mul.eval(), variable_output) self.assertAllClose([[0., 0.], [0., 0.]], variable_output) + @test_util.run_deprecated_v1 def testCachingDevice(self): with self.cached_session(): var = variables.Variable(2.0) @@ -278,6 +293,7 @@ class VariablesTestCase(test.TestCase): self.assertFalse(var_cached.device.startswith("/job:foo")) self.assertTrue(var_cached.value().device.startswith("/job:foo")) + @test_util.run_deprecated_v1 def testCollections(self): with self.cached_session(): var_x = variables.VariableV1(2.0) @@ -293,6 +309,7 @@ class VariablesTestCase(test.TestCase): variables.global_variables()) self.assertEqual([var_x, var_z, var_t], variables.trainable_variables()) + @test_util.run_deprecated_v1 def testCollectionsWithScope(self): with self.cached_session(): with ops.name_scope("scope_1"): @@ -308,6 +325,13 @@ class VariablesTestCase(test.TestCase): self.assertEqual([var_x], variables.trainable_variables("scope_1")) self.assertEqual([var_y], variables.trainable_variables("scope_2")) + def testOperatorWrapping(self): + for attr in functools.WRAPPER_ASSIGNMENTS: + self.assertEqual( + getattr(variables.Variable.__add__, attr), + getattr(ops.Tensor.__add__, attr)) + + @test_util.run_deprecated_v1 def testOperators(self): with self.cached_session(): var_f = variables.Variable([2.0]) @@ -349,54 +373,46 @@ class VariablesTestCase(test.TestCase): rmatmul = var_m.__rmatmul__([[10.0], [20.0]]) variables.global_variables_initializer().run() - self.assertAllClose([2.0], add.eval()) - self.assertAllClose([3.0], radd.eval()) - self.assertAllClose([1.0], sub.eval()) - self.assertAllClose([-1.0], rsub.eval()) - self.assertAllClose([20.0], mul.eval()) - self.assertAllClose([20.0], rmul.eval()) - self.assertAllClose([0.2], div.eval()) - self.assertAllClose([5.0], rdiv.eval()) - self.assertAllClose([-2.0], neg.eval()) - self.assertAllClose([2.0], abs_v.eval()) - self.assertAllClose([True], lt.eval()) - self.assertAllClose([False], rlt.eval()) - self.assertAllClose([True], le.eval()) - self.assertAllClose([True], rle.eval()) - self.assertAllClose([False], gt.eval()) - self.assertAllClose([True], rgt.eval()) - self.assertAllClose([True], ge.eval()) - self.assertAllClose([True], rge.eval()) + self.assertAllClose([2.0], self.evaluate(add)) + self.assertAllClose([3.0], self.evaluate(radd)) + self.assertAllClose([1.0], self.evaluate(sub)) + self.assertAllClose([-1.0], self.evaluate(rsub)) + self.assertAllClose([20.0], self.evaluate(mul)) + self.assertAllClose([20.0], self.evaluate(rmul)) + self.assertAllClose([0.2], self.evaluate(div)) + self.assertAllClose([5.0], self.evaluate(rdiv)) + self.assertAllClose([-2.0], self.evaluate(neg)) + self.assertAllClose([2.0], self.evaluate(abs_v)) + self.assertAllClose([True], self.evaluate(lt)) + self.assertAllClose([False], self.evaluate(rlt)) + self.assertAllClose([True], self.evaluate(le)) + self.assertAllClose([True], self.evaluate(rle)) + self.assertAllClose([False], self.evaluate(gt)) + self.assertAllClose([True], self.evaluate(rgt)) + self.assertAllClose([True], self.evaluate(ge)) + self.assertAllClose([True], self.evaluate(rge)) - self.assertAllClose([6], mod.eval()) - self.assertAllClose([3], rmod.eval()) + self.assertAllClose([6], self.evaluate(mod)) + self.assertAllClose([3], self.evaluate(rmod)) - self.assertAllClose([True, False], and_v.eval()) - self.assertAllClose([True, True], or_v.eval()) - self.assertAllClose([True, False], xor_v.eval()) - self.assertAllClose([False, True], invert_v.eval()) + self.assertAllClose([True, False], self.evaluate(and_v)) + self.assertAllClose([True, True], self.evaluate(or_v)) + self.assertAllClose([True, False], self.evaluate(xor_v)) + self.assertAllClose([False, True], self.evaluate(invert_v)) - self.assertAllClose(rnd[2, 0:0], slice_v.eval()) + self.assertAllClose(rnd[2, 0:0], self.evaluate(slice_v)) - self.assertAllClose([[80.0]], matmul.eval()) - self.assertAllClose([[20.0, 30.0], [40.0, 60.0]], rmatmul.eval()) + self.assertAllClose([[80.0]], self.evaluate(matmul)) + self.assertAllClose([[20.0, 30.0], [40.0, 60.0]], self.evaluate(rmatmul)) + @test_util.run_deprecated_v1 def testSession(self): with self.cached_session() as sess: var = variables.Variable([1, 12]) variables.global_variables_initializer().run() - self.assertAllClose([1, 12], sess.run(var)) - - def testDevicePlacement(self): - with self.cached_session() as sess: - with ops.device("/cpu:0"): - var = variables.Variable([1, 12]) - init_value = var.initialized_value() - init_op = variables.global_variables_initializer() - self.assertEqual(var.op.device, init_value.device) - self.assertEqual(var.op.device, init_op.device) - sess.run(init_op) + self.assertAllClose([1, 12], self.evaluate(var)) + @test_util.run_deprecated_v1 def testColocation(self): with ops.device("/job:ps"): var = variables.VariableV1(0, name="v") @@ -405,6 +421,7 @@ class VariablesTestCase(test.TestCase): self.assertDeviceEqual("/job:ps", assign_op.device) self.assertEqual([b"loc:@v"], assign_op.op.colocation_groups()) + @test_util.run_deprecated_v1 def testInitializerFunction(self): value = [[-42], [133.7]] shape = [2, 1] @@ -416,7 +433,7 @@ class VariablesTestCase(test.TestCase): self.assertEqual(shape, v1.shape) self.assertAllClose(value, v1.initial_value.eval()) with self.assertRaises(errors_impl.FailedPreconditionError): - v1.eval() + self.evaluate(v1) v2 = variables.Variable( math_ops.negative(v1.initialized_value()), dtype=dtypes.float32) @@ -425,9 +442,9 @@ class VariablesTestCase(test.TestCase): self.assertAllClose(np.negative(value), v2.initial_value.eval()) with self.assertRaises(errors_impl.FailedPreconditionError): - v2.eval() + self.evaluate(v2) variables.global_variables_initializer().run() - self.assertAllClose(np.negative(value), v2.eval()) + self.assertAllClose(np.negative(value), self.evaluate(v2)) def testConstraintArg(self): constraint = lambda x: x @@ -442,6 +459,7 @@ class VariablesTestCase(test.TestCase): lambda: constant_op.constant(1.), constraint=constraint) + @test_util.run_deprecated_v1 def testNoRefDataRace(self): with self.cached_session(): a = variables.Variable([1, 2, 3], dtype=dtypes.float32) @@ -452,6 +470,7 @@ class VariablesTestCase(test.TestCase): self.assertAllEqual(b.eval(), [3, 4, 5]) self.assertAllEqual(c.eval(), [5, 6, 7]) + @test_util.run_deprecated_v1 def testInitializerFunctionDevicePlacement(self): with self.cached_session(): initializer = lambda: constant_op.constant(42.0) @@ -470,6 +489,7 @@ class VariablesTestCase(test.TestCase): for i in v2.initializer.inputs: self.assertEqual(expected_group_v2, i.op.colocation_groups()) + @test_util.run_deprecated_v1 def testVariableDefInitializedInstances(self): with ops.Graph().as_default(), self.cached_session() as sess: v_def = variables.Variable( @@ -478,11 +498,11 @@ class VariablesTestCase(test.TestCase): with ops.Graph().as_default(), self.cached_session() as sess: # v describes a VariableDef-based variable without an initial value. v = variables.Variable(variable_def=v_def) - self.assertEqual(3.0, sess.run(v.initialized_value())) + self.assertEqual(3.0, self.evaluate(v.initialized_value())) # initialized_value should not rerun the initializer_op if the variable # has already been initialized elsewhere. - sess.run(v.assign(1.0)) + self.evaluate(v.assign(1.0)) self.assertEqual(1.0, v.initialized_value().eval()) v_def.ClearField("initial_value_name") @@ -494,7 +514,7 @@ class VariablesTestCase(test.TestCase): self.assertProtoEquals(v_def, v.to_proto()) # But attempts to use initialized_value will result in errors. with self.assertRaises(ValueError): - sess.run(v.initialized_value()) + self.evaluate(v.initialized_value()) def testTrainableInProto(self): with ops.Graph().as_default(): @@ -513,14 +533,16 @@ class VariablesTestCase(test.TestCase): variables.Variable(variable_def=trainable_variable.to_proto()) .trainable) + @test_util.run_deprecated_v1 def testLoad(self): with self.cached_session(): var = variables.Variable(np.zeros((5, 5), np.float32)) variables.global_variables_initializer().run() var.load(np.ones((5, 5), np.float32)) - self.assertAllClose(np.ones((5, 5), np.float32), var.eval()) + self.assertAllClose(np.ones((5, 5), np.float32), self.evaluate(var)) + @test_util.run_deprecated_v1 def testRepr(self): var = variables.VariableV1(np.zeros((5, 5), np.float32), name="noop") self.assertEqual( @@ -542,7 +564,7 @@ class IsInitializedTest(test.TestCase): def testNoVars(self): with ops.Graph().as_default(), self.cached_session() as sess: uninited = variables.report_uninitialized_variables() - self.assertEqual(0, sess.run(uninited).size) + self.assertEqual(0, self.evaluate(uninited).size) def testAssertVariablesInitialized(self): with ops.Graph().as_default(), self.cached_session() as sess: @@ -550,27 +572,28 @@ class IsInitializedTest(test.TestCase): w = variables.Variable([3, 4], name="w") _ = v, w uninited = variables.report_uninitialized_variables() - self.assertAllEqual(np.array([b"v", b"w"]), sess.run(uninited)) + self.assertAllEqual(np.array([b"v", b"w"]), self.evaluate(uninited)) variables.global_variables_initializer().run() - self.assertEqual(0, sess.run(uninited).size) + self.assertEqual(0, self.evaluate(uninited).size) + @test_util.run_deprecated_v1 def testVariableList(self): with ops.Graph().as_default(), self.cached_session() as sess: v = variables.VariableV1([1, 2], name="v") w = variables.VariableV1([3, 4], name="w") uninited = variables.report_uninitialized_variables() - self.assertAllEqual(np.array([b"v", b"w"]), sess.run(uninited)) - sess.run(w.initializer) - self.assertAllEqual(np.array([b"v"]), sess.run(uninited)) + self.assertAllEqual(np.array([b"v", b"w"]), self.evaluate(uninited)) + self.evaluate(w.initializer) + self.assertAllEqual(np.array([b"v"]), self.evaluate(uninited)) v.initializer.run() - self.assertEqual(0, sess.run(uninited).size) + self.assertEqual(0, self.evaluate(uninited).size) def testZeroSizeVarInitialized(self): with ops.Graph().as_default(), self.cached_session() as sess: v = variables.Variable(array_ops.zeros([0, 2]), name="v") uninited = variables.report_uninitialized_variables() v.initializer.run() # not strictly necessary - self.assertEqual(0, sess.run(uninited).size) + self.assertEqual(0, self.evaluate(uninited).size) def testTrainingWithZeroSizeVar(self): with ops.Graph().as_default(), self.cached_session() as sess: @@ -581,8 +604,8 @@ class IsInitializedTest(test.TestCase): variables.global_variables_initializer().run() do_opt = gradient_descent.GradientDescentOptimizer(0.1).minimize( objective) - sess.run([do_opt]) - self.assertAllClose([[0.9, 0.9], [0.9, 0.9]], b.eval()) + self.evaluate([do_opt]) + self.assertAllClose([[0.9, 0.9], [0.9, 0.9]], self.evaluate(b)) class ObsoleteIsInitializedTest(test.TestCase): @@ -591,6 +614,7 @@ class ObsoleteIsInitializedTest(test.TestCase): with ops.Graph().as_default(): self.assertEqual(None, variables.assert_variables_initialized()) + @test_util.run_deprecated_v1 def testVariables(self): with ops.Graph().as_default(), self.cached_session() as sess: v = variables.VariableV1([1, 2]) @@ -598,10 +622,11 @@ class ObsoleteIsInitializedTest(test.TestCase): _ = v, w inited = variables.assert_variables_initialized() with self.assertRaisesOpError("Attempting to use uninitialized value"): - sess.run(inited) + self.evaluate(inited) variables.global_variables_initializer().run() - sess.run(inited) + self.evaluate(inited) + @test_util.run_deprecated_v1 def testVariableList(self): with ops.Graph().as_default(), self.cached_session() as sess: v = variables.VariableV1([1, 2]) @@ -609,7 +634,7 @@ class ObsoleteIsInitializedTest(test.TestCase): inited = variables.assert_variables_initialized([v]) with self.assertRaisesOpError("Attempting to use uninitialized value"): inited.op.run() - sess.run(w.initializer) + self.evaluate(w.initializer) with self.assertRaisesOpError("Attempting to use uninitialized value"): inited.op.run() v.initializer.run() @@ -744,34 +769,34 @@ class PartitionedVariableTest(test.TestCase): variables.global_variables_initializer().run() self.assertEqual([1.0], plus_delta[0].eval()) - self.assertEqual([1.0], v0.eval()) + self.assertEqual([1.0], self.evaluate(v0)) self.assertEqual([3.0], plus_delta[1].eval()) - self.assertEqual([3.0], v1.eval()) + self.assertEqual([3.0], self.evaluate(v1)) self.assertEqual([-2.0], minus_delta[0].eval()) - self.assertEqual([-2.0], v0.eval()) + self.assertEqual([-2.0], self.evaluate(v0)) self.assertEqual([-1.0], minus_delta[1].eval()) - self.assertEqual([-1.0], v1.eval()) + self.assertEqual([-1.0], self.evaluate(v1)) self.assertEqual([1.0], assign_ones[0].eval()) - self.assertEqual([1.0], v0.eval()) + self.assertEqual([1.0], self.evaluate(v0)) self.assertEqual([1.0], assign_ones[1].eval()) - self.assertEqual([1.0], v1.eval()) + self.assertEqual([1.0], self.evaluate(v1)) self.assertEqual([2.0], assign_list[0].eval()) - self.assertEqual([2.0], v2.eval()) + self.assertEqual([2.0], self.evaluate(v2)) self.assertEqual([3.0], assign_list[1].eval()) - self.assertEqual([3.0], v3.eval()) + self.assertEqual([3.0], self.evaluate(v3)) self.assertEqual([3.0], assign_part_value[0].eval()) - self.assertEqual([3.0], v2.eval()) + self.assertEqual([3.0], self.evaluate(v2)) self.assertEqual([4.0], assign_part_value[1].eval()) - self.assertEqual([4.0], v3.eval()) + self.assertEqual([4.0], self.evaluate(v3)) self.assertEqual([2.0], assign_part_var[0].eval()) - self.assertEqual([2.0], v2.eval()) + self.assertEqual([2.0], self.evaluate(v2)) self.assertEqual([3.0], assign_part_var[1].eval()) - self.assertEqual([3.0], v3.eval()) + self.assertEqual([3.0], self.evaluate(v3)) class VariableContainerTest(test.TestCase): diff --git a/tensorflow/python/kernel_tests/weights_broadcast_test.py b/tensorflow/python/kernel_tests/weights_broadcast_test.py index 85f9abc69f7..677d8f2f22f 100644 --- a/tensorflow/python/kernel_tests/weights_broadcast_test.py +++ b/tensorflow/python/kernel_tests/weights_broadcast_test.py @@ -23,6 +23,7 @@ import numpy as np from tensorflow.python.framework import dtypes as dtypes_lib from tensorflow.python.framework import errors_impl from tensorflow.python.framework import ops +from tensorflow.python.framework import test_util from tensorflow.python.ops import array_ops from tensorflow.python.ops import weights_broadcast_ops from tensorflow.python.platform import test @@ -51,40 +52,48 @@ class AssertBroadcastableTest(test.TestCase): values_placeholder: values, }) + @test_util.run_deprecated_v1 def testScalar(self): self._test_valid(weights=5, values=_test_values((3, 2, 4))) + @test_util.run_deprecated_v1 def test1x1x1(self): self._test_valid( weights=np.asarray((5,)).reshape((1, 1, 1)), values=_test_values((3, 2, 4))) + @test_util.run_deprecated_v1 def test1x1xN(self): self._test_valid( weights=np.asarray((5, 7, 11, 3)).reshape((1, 1, 4)), values=_test_values((3, 2, 4))) + @test_util.run_deprecated_v1 def test1xNx1(self): self._test_valid( weights=np.asarray((5, 11)).reshape((1, 2, 1)), values=_test_values((3, 2, 4))) + @test_util.run_deprecated_v1 def test1xNxN(self): self._test_valid( weights=np.asarray((5, 7, 11, 3, 2, 13, 7, 5)).reshape((1, 2, 4)), values=_test_values((3, 2, 4))) + @test_util.run_deprecated_v1 def testNx1x1(self): self._test_valid( weights=np.asarray((5, 7, 11)).reshape((3, 1, 1)), values=_test_values((3, 2, 4))) + @test_util.run_deprecated_v1 def testNx1xN(self): self._test_valid( weights=np.asarray(( 5, 7, 11, 3, 2, 12, 7, 5, 2, 17, 11, 3)).reshape((3, 1, 4)), values=_test_values((3, 2, 4))) + @test_util.run_deprecated_v1 def testNxNxN(self): self._test_valid( weights=np.asarray(( @@ -107,29 +116,35 @@ class AssertBroadcastableTest(test.TestCase): values_placeholder: values, }) + @test_util.run_deprecated_v1 def testInvalid1(self): self._test_invalid(weights=np.asarray((5,)), values=_test_values((3, 2, 4))) + @test_util.run_deprecated_v1 def testInvalid1x1(self): self._test_invalid( weights=np.asarray((5,)).reshape((1, 1)), values=_test_values((3, 2, 4))) + @test_util.run_deprecated_v1 def testInvalidPrefixMatch(self): self._test_invalid( weights=np.asarray((5, 7, 11, 3, 2, 12)).reshape((3, 2)), values=_test_values((3, 2, 4))) + @test_util.run_deprecated_v1 def testInvalidSuffixMatch(self): self._test_invalid( weights=np.asarray((5, 7, 11, 3, 2, 12, 7, 5)).reshape((2, 4)), values=_test_values((3, 2, 4))) + @test_util.run_deprecated_v1 def testInvalidOnesExtraDim(self): self._test_invalid( weights=np.asarray((5,)).reshape((1, 1, 1, 1)), values=_test_values((3, 2, 4))) + @test_util.run_deprecated_v1 def testInvalidPrefixMatchExtraDim(self): self._test_invalid( weights=np.asarray(( @@ -137,6 +152,7 @@ class AssertBroadcastableTest(test.TestCase): 2, 17, 11, 3, 5, 7, 11, 3, 2, 12, 7, 5)).reshape((3, 2, 4, 1)), values=_test_values((3, 2, 4))) + @test_util.run_deprecated_v1 def testInvalidSuffixMatchExtraDim(self): self._test_invalid( weights=np.asarray(( @@ -158,24 +174,27 @@ class BroadcastWeightsTest(test.TestCase): dynamic_op = weights_broadcast_ops.broadcast_weights( weights=weights_placeholder, values=values_placeholder) with self.cached_session(): - self.assertAllEqual(expected, static_op.eval()) + self.assertAllEqual(expected, self.evaluate(static_op)) self.assertAllEqual(expected, dynamic_op.eval(feed_dict={ weights_placeholder: weights, values_placeholder: values, })) + @test_util.run_deprecated_v1 def testScalar(self): self._test_valid( weights=5, values=_test_values((3, 2, 4)), expected=5 * np.ones((3, 2, 4))) + @test_util.run_deprecated_v1 def test1x1x1(self): self._test_valid( weights=np.asarray((5,)).reshape((1, 1, 1)), values=_test_values((3, 2, 4)), expected=5 * np.ones((3, 2, 4))) + @test_util.run_deprecated_v1 def test1x1xN(self): weights = np.asarray((5, 7, 11, 3)).reshape((1, 1, 4)) self._test_valid( @@ -183,6 +202,7 @@ class BroadcastWeightsTest(test.TestCase): values=_test_values((3, 2, 4)), expected=np.tile(weights, reps=(3, 2, 1))) + @test_util.run_deprecated_v1 def test1xNx1(self): weights = np.asarray((5, 11)).reshape((1, 2, 1)) self._test_valid( @@ -190,6 +210,7 @@ class BroadcastWeightsTest(test.TestCase): values=_test_values((3, 2, 4)), expected=np.tile(weights, reps=(3, 1, 4))) + @test_util.run_deprecated_v1 def test1xNxN(self): weights = np.asarray((5, 7, 11, 3, 2, 13, 7, 5)).reshape((1, 2, 4)) self._test_valid( @@ -197,6 +218,7 @@ class BroadcastWeightsTest(test.TestCase): values=_test_values((3, 2, 4)), expected=np.tile(weights, reps=(3, 1, 1))) + @test_util.run_deprecated_v1 def testNx1x1(self): weights = np.asarray((5, 7, 11)).reshape((3, 1, 1)) self._test_valid( @@ -204,6 +226,7 @@ class BroadcastWeightsTest(test.TestCase): values=_test_values((3, 2, 4)), expected=np.tile(weights, reps=(1, 2, 4))) + @test_util.run_deprecated_v1 def testNx1xN(self): weights = np.asarray(( 5, 7, 11, 3, 2, 12, 7, 5, 2, 17, 11, 3)).reshape((3, 1, 4)) @@ -212,6 +235,7 @@ class BroadcastWeightsTest(test.TestCase): values=_test_values((3, 2, 4)), expected=np.tile(weights, reps=(1, 2, 1))) + @test_util.run_deprecated_v1 def testNxNxN(self): weights = np.asarray(( 5, 7, 11, 3, 2, 12, 7, 5, 2, 17, 11, 3, @@ -234,29 +258,35 @@ class BroadcastWeightsTest(test.TestCase): values_placeholder: values, }) + @test_util.run_deprecated_v1 def testInvalid1(self): self._test_invalid(weights=np.asarray((5,)), values=_test_values((3, 2, 4))) + @test_util.run_deprecated_v1 def testInvalid1x1(self): self._test_invalid( weights=np.asarray((5,)).reshape((1, 1)), values=_test_values((3, 2, 4))) + @test_util.run_deprecated_v1 def testInvalidPrefixMatch(self): self._test_invalid( weights=np.asarray((5, 7, 11, 3, 2, 12)).reshape((3, 2)), values=_test_values((3, 2, 4))) + @test_util.run_deprecated_v1 def testInvalidSuffixMatch(self): self._test_invalid( weights=np.asarray((5, 7, 11, 3, 2, 12, 7, 5)).reshape((2, 4)), values=_test_values((3, 2, 4))) + @test_util.run_deprecated_v1 def testInvalidOnesExtraDim(self): self._test_invalid( weights=np.asarray((5,)).reshape((1, 1, 1, 1)), values=_test_values((3, 2, 4))) + @test_util.run_deprecated_v1 def testInvalidPrefixMatchExtraDim(self): self._test_invalid( weights=np.asarray(( @@ -264,6 +294,7 @@ class BroadcastWeightsTest(test.TestCase): 2, 17, 11, 3, 5, 7, 11, 3, 2, 12, 7, 5)).reshape((3, 2, 4, 1)), values=_test_values((3, 2, 4))) + @test_util.run_deprecated_v1 def testInvalidSuffixMatchExtraDim(self): self._test_invalid( weights=np.asarray(( diff --git a/tensorflow/python/kernel_tests/where_op_test.py b/tensorflow/python/kernel_tests/where_op_test.py index fca45c3ece4..56c13904113 100644 --- a/tensorflow/python/kernel_tests/where_op_test.py +++ b/tensorflow/python/kernel_tests/where_op_test.py @@ -27,6 +27,7 @@ from tensorflow.python.client import session from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops +from tensorflow.python.framework import test_util from tensorflow.python.ops import array_ops from tensorflow.python.ops import random_ops from tensorflow.python.ops import resource_variable_ops @@ -41,11 +42,11 @@ class WhereOpTest(test.TestCase): ans = array_ops.where(x) self.assertEqual([None, x.ndim], ans.get_shape().as_list()) if expected_err_re is None: - tf_ans = ans.eval() + tf_ans = self.evaluate(ans) self.assertAllClose(tf_ans, truth, atol=1e-10) else: with self.assertRaisesOpError(expected_err_re): - ans.eval() + self.evaluate(ans) def testWrongNumbers(self): with self.session(use_gpu=True): @@ -54,6 +55,7 @@ class WhereOpTest(test.TestCase): with self.assertRaises(ValueError): array_ops.where([False, True], None, [1, 2]) + @test_util.run_deprecated_v1 def testBasicVec(self): x = np.asarray([True, False]) truth = np.asarray([[0]], dtype=np.int64) @@ -67,11 +69,13 @@ class WhereOpTest(test.TestCase): truth = np.asarray([[2], [4]], dtype=np.int64) self._testWhere(x, truth) + @test_util.run_deprecated_v1 def testRandomVec(self): x = np.random.rand(1000000) > 0.5 truth = np.vstack([np.where(x)[0].astype(np.int64)]).T self._testWhere(x, truth) + @test_util.run_deprecated_v1 def testBasicMat(self): x = np.asarray([[True, False], [True, False]]) @@ -80,6 +84,7 @@ class WhereOpTest(test.TestCase): self._testWhere(x, truth) + @test_util.run_deprecated_v1 def testBasic3Tensor(self): x = np.asarray([[[True, False], [True, False]], [[False, True], [False, True]], @@ -99,36 +104,47 @@ class WhereOpTest(test.TestCase): truth = np.vstack(truth).T # Convert to [num_true, indices]. self._testWhere(x, truth, expected_err_re) + @test_util.run_deprecated_v1 def testRandomBool(self): self._testRandom(np.bool) + @test_util.run_deprecated_v1 def testRandomInt32(self): self._testRandom(np.int32) + @test_util.run_deprecated_v1 def testRandomInt64(self): self._testRandom(np.int64) + @test_util.run_deprecated_v1 def testRandomFloat(self): self._testRandom(np.float32) + @test_util.run_deprecated_v1 def testRandomDouble(self): self._testRandom(np.float64) + @test_util.run_deprecated_v1 def testRandomComplex64(self): self._testRandom(np.complex64) + @test_util.run_deprecated_v1 def testRandomComplex128(self): self._testRandom(np.complex128) + @test_util.run_deprecated_v1 def testRandomUint8(self): self._testRandom(np.uint8) + @test_util.run_deprecated_v1 def testRandomInt8(self): self._testRandom(np.int8) + @test_util.run_deprecated_v1 def testRandomInt16(self): self._testRandom(np.int16) + @test_util.run_deprecated_v1 def testThreeArgument(self): x = np.array([[-2, 3, -1], [1, -3, -3]]) np_val = np.where(x > 0, x * x, -x) @@ -136,6 +152,7 @@ class WhereOpTest(test.TestCase): tf_val = array_ops.where(constant_op.constant(x) > 0, x * x, -x).eval() self.assertAllEqual(tf_val, np_val) + @test_util.run_deprecated_v1 def testBatchSelect(self): x = np.array([[-2, 3, -1] * 64, [1, -3, -3] * 64] * 8192) # [16384, 192] c_mat = np.array([[False] * 192, [True] * 192] * 8192) # [16384, 192] diff --git a/tensorflow/python/kernel_tests/while_v2_test.py b/tensorflow/python/kernel_tests/while_v2_test.py index dc1bcb78b80..09cbeb1a0d5 100644 --- a/tensorflow/python/kernel_tests/while_v2_test.py +++ b/tensorflow/python/kernel_tests/while_v2_test.py @@ -20,14 +20,15 @@ from __future__ import print_function from absl.testing import parameterized +from tensorflow.core.protobuf import config_pb2 from tensorflow.core.protobuf import rewriter_config_pb2 from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes from tensorflow.python.framework import meta_graph from tensorflow.python.framework import ops +from tensorflow.python.framework import test_util from tensorflow.python.grappler import tf_optimizer from tensorflow.python.ops import array_ops -from tensorflow.python.ops import control_flow_ops from tensorflow.python.ops import functional_ops from tensorflow.python.ops import gradients_impl from tensorflow.python.ops import list_ops @@ -42,14 +43,29 @@ from tensorflow.python.platform import test class WhileV2Test(test.TestCase, parameterized.TestCase): + @test_util.run_deprecated_v1 def testSingleLoopVar(self): x = constant_op.constant(2.) - ret = while_loop_v2(lambda v: v < 8., lambda v: v * v, [x]) + ret = while_loop_v2( + lambda v: v < 8., lambda v: v * v, [x], return_same_structure=False) grad = gradients_impl.gradients(ret, [x]) with self.cached_session() as sess: - self.assertEqual(sess.run(ret), 16.) + self.assertEqual(self.evaluate(ret), 16.) + self.assertSequenceEqual(self.evaluate(grad), [32.]) + + def testReturnSameStructureTrue(self): + x = constant_op.constant(2.) + ret = while_loop_v2( + lambda v: v < 8., lambda v: v * v, [x], return_same_structure=True) + grad = gradients_impl.gradients(ret, [x]) + with self.cached_session() as sess: + eval_result = sess.run(ret) + self.assertIsInstance(eval_result, list) + self.assertLen(eval_result, 1) + self.assertEqual(16., eval_result[0]) self.assertSequenceEqual(sess.run(grad), [32.]) + @test_util.run_deprecated_v1 def testMultipleLoopVarsBasic(self): x = constant_op.constant(5.) y = constant_op.constant(3.) @@ -58,15 +74,19 @@ class WhileV2Test(test.TestCase, parameterized.TestCase): # y = 3. # while x < 45.: # x = x * y - ret = while_loop_v2(lambda v, _: v < 45., lambda v, w: (v * w, w), [x, y]) + ret = while_loop_v2( + lambda v, _: v < 45., + lambda v, w: (v * w, w), [x, y], + return_same_structure=False) # ret = [x*y^2, y] # Note: This is simply d_ret[0]/d_x since d_ret[1]/d_x is 0. grad = gradients_impl.gradients(ret, [x]) # [2*x*y] with self.cached_session() as sess: - self.assertSequenceEqual(sess.run(ret), [45., 3.]) - self.assertSequenceEqual(sess.run(grad), [9.]) + self.assertSequenceEqual(self.evaluate(ret), [45., 3.]) + self.assertSequenceEqual(self.evaluate(grad), [9.]) + @test_util.run_deprecated_v1 def testMultipleLoopVars(self): x = constant_op.constant(5.) y = constant_op.constant(3.) @@ -76,8 +96,10 @@ class WhileV2Test(test.TestCase, parameterized.TestCase): # while x < 45.: # x = x * y # y = x + y - ret = while_loop_v2(lambda v, _: v < 45., lambda v, w: (v * w, v + w), - [x, y]) + ret = while_loop_v2( + lambda v, _: v < 45., + lambda v, w: (v * w, v + w), [x, y], + return_same_structure=False) # ret = [y*x**2 + x*y**2, x*y + x + y] gradx_0 = gradients_impl.gradients(ret[0], [x]) # [2*x*y + y**2] @@ -87,34 +109,43 @@ class WhileV2Test(test.TestCase, parameterized.TestCase): grady_1 = gradients_impl.gradients(ret[1], [y]) # [x + 1] grady_2 = gradients_impl.gradients(ret, [y]) # [2*x*y + x**2 + x + 1] with self.cached_session() as sess: - self.assertSequenceEqual(sess.run(ret), [120., 23.]) - self.assertSequenceEqual(sess.run(gradx_0), [39.]) - self.assertSequenceEqual(sess.run(gradx_1), [4.]) - self.assertSequenceEqual(sess.run(gradx_2), [43.]) - self.assertSequenceEqual(sess.run(grady_0), [55.]) - self.assertSequenceEqual(sess.run(grady_1), [6.]) - self.assertSequenceEqual(sess.run(grady_2), [61.]) + self.assertSequenceEqual(self.evaluate(ret), [120., 23.]) + self.assertSequenceEqual(self.evaluate(gradx_0), [39.]) + self.assertSequenceEqual(self.evaluate(gradx_1), [4.]) + self.assertSequenceEqual(self.evaluate(gradx_2), [43.]) + self.assertSequenceEqual(self.evaluate(grady_0), [55.]) + self.assertSequenceEqual(self.evaluate(grady_1), [6.]) + self.assertSequenceEqual(self.evaluate(grady_2), [61.]) + @test_util.run_deprecated_v1 def testMultipleWhileLoops(self): x = constant_op.constant(2.) - ret1 = while_loop_v2(lambda v: v < 4., lambda v: v * v, [x]) # x**2 - ret2 = while_loop_v2(lambda v: v < 16., lambda v: v * v, [ret1]) # x**4 + ret1 = while_loop_v2( + lambda v: v < 4., lambda v: v * v, [x], + return_same_structure=False) # x**2 + ret2 = while_loop_v2( + lambda v: v < 16., lambda v: v * v, [ret1], + return_same_structure=False) # x**4 grad = gradients_impl.gradients(ret2, [x]) # 4x**3 grad_grad = gradients_impl.gradients(grad, [x]) # 12x**2 with self.cached_session() as sess: - self.assertSequenceEqual(sess.run(grad), [32.]) - self.assertSequenceEqual(sess.run(grad_grad), [48.]) + self.assertSequenceEqual(self.evaluate(grad), [32.]) + self.assertSequenceEqual(self.evaluate(grad_grad), [48.]) + @test_util.run_deprecated_v1 def testDoubleDerivative(self): x = constant_op.constant(2.) - ret = while_loop_v2(lambda v: v < 8., lambda v: v**2, [x]) # x**4 + ret = while_loop_v2( + lambda v: v < 8., lambda v: v**2, [x], + return_same_structure=False) # x**4 grad = gradients_impl.gradients(ret, [x]) # 4x**3 grad_grad = gradients_impl.gradients(grad, [x]) # 12x**2 with self.cached_session() as sess: - self.assertEqual(sess.run(ret), 16.) - self.assertSequenceEqual(sess.run(grad), [32.]) - self.assertSequenceEqual(sess.run(grad_grad), [48.]) + self.assertEqual(self.evaluate(ret), 16.) + self.assertSequenceEqual(self.evaluate(grad), [32.]) + self.assertSequenceEqual(self.evaluate(grad_grad), [48.]) + @test_util.run_deprecated_v1 def testPruning(self): x = constant_op.constant(1) @@ -135,10 +166,12 @@ class WhileV2Test(test.TestCase, parameterized.TestCase): def GetOptimizedGraph(): mg = meta_graph.create_meta_graph_def(graph=ops.get_default_graph()) - rewriter_config = rewriter_config_pb2.RewriterConfig( - constant_folding=rewriter_config_pb2.RewriterConfig.OFF, - memory_optimization=rewriter_config_pb2.RewriterConfig.MANUAL) - return tf_optimizer.OptimizeGraph(rewriter_config, mg) + config = config_pb2.ConfigProto() + config.graph_options.rewrite_options.CopyFrom( + rewriter_config_pb2.RewriterConfig( + constant_folding=rewriter_config_pb2.RewriterConfig.OFF, + memory_optimization=rewriter_config_pb2.RewriterConfig.MANUAL)) + return tf_optimizer.OptimizeGraph(config, mg) g = GetOptimizedGraph() self.assertEqual(len([n for n in g.node if n.op == "Enter"]), 1) @@ -148,24 +181,31 @@ class WhileV2Test(test.TestCase, parameterized.TestCase): g = GetOptimizedGraph() self.assertEqual(len([n for n in g.node if n.op == "Enter"]), 2) + @test_util.run_deprecated_v1 def testCaptureExternalTensorInCond(self): x = constant_op.constant(2.) y = constant_op.constant(1.) - ret = while_loop_v2(lambda v: v + y < 9., lambda v: v * 3., [x]) + ret = while_loop_v2( + lambda v: v + y < 9., + lambda v: v * 3., [x], + return_same_structure=False) grad = gradients_impl.gradients(ret, [x]) with self.cached_session() as sess: - self.assertEqual(sess.run(ret), 18.) - self.assertSequenceEqual(sess.run(grad), [9.]) + self.assertEqual(self.evaluate(ret), 18.) + self.assertSequenceEqual(self.evaluate(grad), [9.]) + @test_util.run_deprecated_v1 def testCaptureExternalTensorInBody(self): x = constant_op.constant(2.) y = constant_op.constant(3.) - ret = while_loop_v2(lambda v: v < 8., lambda v: v * y, [x]) + ret = while_loop_v2( + lambda v: v < 8., lambda v: v * y, [x], return_same_structure=False) grad = gradients_impl.gradients(ret, [x]) with self.cached_session() as sess: - self.assertEqual(sess.run(ret), 18.) - self.assertSequenceEqual(sess.run(grad), [9.]) + self.assertEqual(self.evaluate(ret), 18.) + self.assertSequenceEqual(self.evaluate(grad), [9.]) + @test_util.run_deprecated_v1 def testLoopWithTensorListPushBack(self): x = constant_op.constant(2.) @@ -181,12 +221,14 @@ class WhileV2Test(test.TestCase, parameterized.TestCase): tl = list_ops.tensor_list_push_back(tl, constant_op.constant(100.)) return x**2., tl - ret = while_loop_v2(Cond, Body, [x, tensor_list]) + ret = while_loop_v2( + Cond, Body, [x, tensor_list], return_same_structure=False) grad = gradients_impl.gradients(ret[0], x) with self.cached_session() as sess: self.assertEqual(sess.run(ret[0]), 16.) - self.assertSequenceEqual(sess.run(grad), [32.]) + self.assertSequenceEqual(self.evaluate(grad), [32.]) + @test_util.run_deprecated_v1 def testDuplicateAccumulator(self): x = constant_op.constant(2.) @@ -203,7 +245,8 @@ class WhileV2Test(test.TestCase, parameterized.TestCase): tl = list_ops.tensor_list_push_back(tl, x) return x**2., tl - ret = while_loop_v2(Cond, Body, [x, tensor_list]) + ret = while_loop_v2( + Cond, Body, [x, tensor_list], return_same_structure=False) for op in ops.get_default_graph().get_operations(): if op.type == "While": @@ -219,13 +262,14 @@ class WhileV2Test(test.TestCase, parameterized.TestCase): grad = gradients_impl.gradients(ret[0], x) with self.cached_session() as sess: self.assertEqual(sess.run(ret[0]), 16.) - self.assertSequenceEqual(sess.run(grad), [32.]) + self.assertSequenceEqual(self.evaluate(grad), [32.]) @parameterized.named_parameters( ("UnknownShape", None), ("PartiallyDefinedShape", [None, 2]), ("FullyDefinedShape", [1, 2]), ) + @test_util.run_deprecated_v1 def testAccumulatorElementShape(self, shape): def MatchShape(actual_tensor_shape): @@ -250,7 +294,10 @@ class WhileV2Test(test.TestCase, parameterized.TestCase): y = array_ops.placeholder(dtype=dtypes.float32, shape=shape) # Forward pass. - ret = while_loop_v2(lambda v, u: v < 8., lambda v, u: (v * v, u), [x, y]) + ret = while_loop_v2( + lambda v, u: v < 8., + lambda v, u: (v * v, u), [x, y], + return_same_structure=False) while_op = ret[0].op.inputs[0].op # Get the TensorList output of While op containing the accumulated values # of y. @@ -262,7 +309,7 @@ class WhileV2Test(test.TestCase, parameterized.TestCase): # Gradient pass. grad = gradients_impl.gradients(ret[1], y) - grad_while_op = grad[0].op + grad_while_op = grad[0].op.inputs[0].op # Get the TensorList output of gradient While op containing the accumulated # values of grad_y. # grad_while_op.inputs: @@ -274,8 +321,10 @@ class WhileV2Test(test.TestCase, parameterized.TestCase): def _createWhile(self, name): """Helper function testDefaultName.""" - output = while_v2.while_loop(lambda i: i < 3, lambda i: i + 1, - [constant_op.constant(0)]) + output = while_v2.while_loop( + lambda i: i < 3, + lambda i: i + 1, [constant_op.constant(0)], + return_same_structure=False) while_op = output.op.inputs[0].op self.assertEqual(while_op.type, "While") return while_op @@ -305,19 +354,19 @@ class WhileV2Test(test.TestCase, parameterized.TestCase): self.assertRegexpMatches( while2_op.get_attr("body").name, r"foo_while_1_body_\d*") + @test_util.enable_control_flow_v2 + @test_util.run_deprecated_v1 def testWhileAndTensorArray(self): - old_enable_while_v2 = control_flow_ops.ENABLE_WHILE_V2 - control_flow_ops.ENABLE_WHILE_V2 = True with self.cached_session() as sess: param = constant_op.constant(2.0) y0 = constant_op.constant([1.0, 2.0, 3.0, 4.0, 5.0, 6.0], name="elems") # map_fn uses TensorArray internally. r = functional_ops.map_fn(lambda x: math_ops.multiply(x, param), y0) - self.assertAllClose([2.0, 4.0, 6.0, 8.0, 10.0, 12.0], sess.run(r)) + self.assertAllClose([2.0, 4.0, 6.0, 8.0, 10.0, 12.0], self.evaluate(r)) r = gradients_impl.gradients(r, param)[0] - self.assertAllClose(21.0, sess.run(r)) - control_flow_ops.ENABLE_WHILE_V2 = old_enable_while_v2 + self.assertAllClose(21.0, self.evaluate(r)) + @test_util.run_deprecated_v1 def testNestedWhile(self): # Compute sum of geometric progression: n^0 + n^1 + ... + n^m # We compute the pow using a while loop. @@ -328,14 +377,20 @@ class WhileV2Test(test.TestCase, parameterized.TestCase): def Body(i, previous_sum): prod = constant_op.constant(1.) return i - 1., previous_sum + while_loop_v2( - lambda c, _: c > 0, lambda c, v: (c - 1., v * n), [i, prod])[1] + lambda c, _: c > 0, + lambda c, v: (c - 1., v * n), [i, prod], + return_same_structure=False)[1] - result = while_loop_v2(lambda i, _: i >= 0, Body, [m, sum_of_powers])[1] + result = while_loop_v2( + lambda i, _: i >= 0, + Body, [m, sum_of_powers], + return_same_structure=False)[1] grad = gradients_impl.gradients(result, [n]) with self.cached_session() as sess: - self.assertEqual(sess.run(result), 364.) - self.assertSequenceEqual(sess.run(grad), [547.]) + self.assertEqual(self.evaluate(result), 364.) + self.assertSequenceEqual(self.evaluate(grad), [547.]) + @test_util.run_deprecated_v1 def testIdentityNodeInBody(self): def Body(v): @@ -344,12 +399,14 @@ class WhileV2Test(test.TestCase, parameterized.TestCase): return v * v x = constant_op.constant(2.) - ret = while_loop_v2(lambda v: v < 8., Body, [x]) + ret = while_loop_v2( + lambda v: v < 8., Body, [x], return_same_structure=False) grad = gradients_impl.gradients(ret, [x]) with self.cached_session() as sess: - self.assertEqual(sess.run(ret), 16.) - self.assertSequenceEqual(sess.run(grad), [32.]) + self.assertEqual(self.evaluate(ret), 16.) + self.assertSequenceEqual(self.evaluate(grad), [32.]) + @test_util.run_deprecated_v1 def testNestedWhileAndTensorArray(self): n = constant_op.constant(3.0) @@ -362,13 +419,17 @@ class WhileV2Test(test.TestCase, parameterized.TestCase): return row, col + 1., ta, n # TODO(b/118457764): Remove n from loop_vars from both loops once fixed. - ta = while_loop_v2(lambda _, col, _1, n: col <= n, InnerBody, - [row, constant_op.constant(1.), ta, n])[2] + ta = while_loop_v2( + lambda _, col, _1, n: col <= n, + InnerBody, [row, constant_op.constant(1.), ta, n], + return_same_structure=False)[2] return row + 1., ta, n ta = tensor_array_ops.TensorArray(dtype=dtypes.float32, size=9) - ta = while_loop_v2(lambda row, _, _1: row <= n, Body, - [constant_op.constant(1.), ta, n])[1] + ta = while_loop_v2( + lambda row, _, _1: row <= n, + Body, [constant_op.constant(1.), ta, n], + return_same_structure=False)[1] output = array_ops.reshape(ta.stack(), [3, 3]) self.assertAllEqual( diff --git a/tensorflow/python/kernel_tests/xent_op_test.py b/tensorflow/python/kernel_tests/xent_op_test.py index c3c7f867a1e..f5d03c23701 100644 --- a/tensorflow/python/kernel_tests/xent_op_test.py +++ b/tensorflow/python/kernel_tests/xent_op_test.py @@ -27,6 +27,7 @@ from tensorflow.python.client import session from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops +from tensorflow.python.framework import test_util from tensorflow.python.ops import array_ops from tensorflow.python.ops import gen_nn_ops from tensorflow.python.ops import gradient_checker @@ -56,7 +57,7 @@ class XentTest(test.TestCase): with self.cached_session(use_gpu=use_gpu) as sess: loss, backprop = gen_nn_ops.softmax_cross_entropy_with_logits( np_features, np_labels) - tf_loss, tf_backprop = sess.run([loss, backprop]) + tf_loss, tf_backprop = self.evaluate([loss, backprop]) self.assertAllCloseAccordingToType(np_loss, tf_loss) self.assertAllCloseAccordingToType(np_backprop, tf_backprop) @@ -65,7 +66,7 @@ class XentTest(test.TestCase): with self.cached_session(use_gpu=use_gpu) as sess: loss = nn_ops.softmax_cross_entropy_with_logits( labels=np_labels, logits=np_features, dim=dim) - tf_loss = sess.run(loss) + tf_loss = self.evaluate(loss) print("np_loss:", np_loss) print("tf_loss:", tf_loss) self.assertAllCloseAccordingToType(np_loss, tf_loss) @@ -80,7 +81,7 @@ class XentTest(test.TestCase): loss, backprop = gen_nn_ops.softmax_cross_entropy_with_logits( np.array([[1.], [-1.], [0.]]).astype(dtype), np.array([[-1.], [0.], [1.]]).astype(dtype)) - tf_loss, tf_backprop = sess.run([loss, backprop]) + tf_loss, tf_backprop = self.evaluate([loss, backprop]) self.assertAllClose([0.0, 0.0, 0.0], tf_loss) self.assertAllClose([[2.0], [1.0], [0.0]], tf_backprop) @@ -88,6 +89,7 @@ class XentTest(test.TestCase): self._testSingleClass(True) self._testSingleClass(False) + @test_util.run_deprecated_v1 def testRankTooLarge(self): for dtype in np.float16, np.float32: np_features = np.array([[[1., 1., 1., 1.]], [[1., 2., 3., @@ -148,16 +150,18 @@ class XentTest(test.TestCase): with self.cached_session(use_gpu=use_gpu) as sess: loss, backprop = gen_nn_ops.softmax_cross_entropy_with_logits( tf_f, tf_l) - tf_loss, tf_backprop = sess.run([loss, backprop]) + tf_loss, tf_backprop = self.evaluate([loss, backprop]) self.assertAllCloseAccordingToType(np_loss, tf_loss) self.assertAllCloseAccordingToType(np_backprop, tf_backprop) + @test_util.run_deprecated_v1 def testShapeMismatch(self): with self.cached_session(): with self.assertRaises(ValueError): gen_nn_ops.softmax_cross_entropy_with_logits( [[0., 1.], [2., 3.]], [[0., 1., 0.], [1., 0., 0.]]) + @test_util.run_deprecated_v1 def testNotMatrix(self): with self.cached_session(): with self.assertRaises(ValueError): @@ -179,6 +183,7 @@ class XentTest(test.TestCase): np.array([[1., 1., 1., 1.], [1., 2., 3., 4.]]).astype(np.float64), np.array([[0., 0., 0., 1.], [0., .5, .5, 0.]]).astype(np.float64)) + @test_util.run_deprecated_v1 def testGradient(self): with self.cached_session() as sess: l = constant_op.constant( @@ -206,6 +211,7 @@ class XentTest(test.TestCase): print("cross entropy gradient err = ", err) self.assertLess(err, 5e-8) + @test_util.run_deprecated_v1 def testGradientLabelWithV2(self): with self.cached_session(): l = constant_op.constant( @@ -224,6 +230,7 @@ class XentTest(test.TestCase): self.assertLess(err, 5e-8) + @test_util.run_deprecated_v1 def testSecondGradient(self): with self.cached_session() as sess: l = constant_op.constant( @@ -280,7 +287,7 @@ class XentTest(test.TestCase): with self.session(use_gpu=True) as sess: loss = nn_ops.softmax_cross_entropy_with_logits( labels=labels, logits=features) - tf_loss = sess.run(loss) + tf_loss = self.evaluate(loss) self.assertAllEqual(np_loss, tf_loss) diff --git a/tensorflow/python/kernel_tests/zero_division_test.py b/tensorflow/python/kernel_tests/zero_division_test.py index e68b96e670f..3dd9ec4ba94 100644 --- a/tensorflow/python/kernel_tests/zero_division_test.py +++ b/tensorflow/python/kernel_tests/zero_division_test.py @@ -21,13 +21,15 @@ from __future__ import print_function from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes from tensorflow.python.framework import errors_impl +from tensorflow.python.framework import test_util from tensorflow.python.platform import test class ZeroDivisionTest(test.TestCase): + @test_util.run_deprecated_v1 def testZeros(self): - with self.session(use_gpu=True): + with test_util.use_gpu(): for dtype in dtypes.uint8, dtypes.int16, dtypes.int32, dtypes.int64: zero = constant_op.constant(0, dtype=dtype) one = constant_op.constant(1, dtype=dtype) @@ -36,7 +38,7 @@ class ZeroDivisionTest(test.TestCase): bads.append(one % zero) for bad in bads: try: - result = bad.eval() + result = self.evaluate(bad) except errors_impl.OpError as e: # Ideally, we'd get a nice exception. In theory, this should only # happen on CPU, but 32 bit integer GPU division is actually on diff --git a/tensorflow/python/layers/base.py b/tensorflow/python/layers/base.py index fccea484b0f..bfe591f8755 100644 --- a/tensorflow/python/layers/base.py +++ b/tensorflow/python/layers/base.py @@ -23,6 +23,7 @@ from tensorflow.python.eager import context from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops from tensorflow.python.keras.engine import base_layer +from tensorflow.python.keras.engine import base_layer_utils from tensorflow.python.ops import variable_scope as vs from tensorflow.python.ops import variables as tf_variables from tensorflow.python.util import function_utils @@ -30,10 +31,10 @@ from tensorflow.python.util import nest from tensorflow.python.util import tf_contextlib from tensorflow.python.util.tf_export import tf_export - +# Avoid breaking users who directly import this symbol from this file. +# TODO(fchollet): remove this. InputSpec = base_layer.InputSpec # pylint: disable=invalid-name - _KERAS_STYLE_SCOPE = False @@ -242,11 +243,11 @@ class Layer(base_layer.Layer): def _make_unique_name(self, name_uid_map=None, avoid_names=None, namespace='', zero_based=False): base_name = base_layer.to_snake_case(self.__class__.__name__) - name = base_layer.unique_layer_name(base_name, - name_uid_map=name_uid_map, - avoid_names=avoid_names, - namespace=namespace, - zero_based=zero_based) + name = base_layer_utils.unique_layer_name(base_name, + name_uid_map=name_uid_map, + avoid_names=avoid_names, + namespace=namespace, + zero_based=zero_based) return (name, base_name) @property diff --git a/tensorflow/python/layers/base_test.py b/tensorflow/python/layers/base_test.py index 90abf35e875..d0ec4f4425f 100644 --- a/tensorflow/python/layers/base_test.py +++ b/tensorflow/python/layers/base_test.py @@ -26,6 +26,7 @@ from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops from tensorflow.python.framework import test_util from tensorflow.python.keras.engine import base_layer as keras_base_layer +from tensorflow.python.keras.engine import input_spec from tensorflow.python.layers import base as base_layers from tensorflow.python.layers import core as core_layers from tensorflow.python.ops import array_ops @@ -142,6 +143,7 @@ class BaseLayerTest(test.TestCase): synchronization=variable_scope.VariableSynchronization.ON_READ, trainable=True) + @test_util.run_deprecated_v1 def testReusePartitionedVaraiblesAndRegularizers(self): regularizer = lambda x: math_ops.reduce_sum(x) * 1e-3 partitioner = partitioned_variables.fixed_size_partitioner(3) @@ -251,7 +253,7 @@ class BaseLayerTest(test.TestCase): def __init__(self): super(CustomerLayer, self).__init__() - self.input_spec = base_layers.InputSpec(ndim=2) + self.input_spec = input_spec.InputSpec(ndim=2) def call(self, inputs): return inputs @@ -278,7 +280,7 @@ class BaseLayerTest(test.TestCase): def __init__(self): super(CustomerLayer, self).__init__() - self.input_spec = base_layers.InputSpec(min_ndim=2) + self.input_spec = input_spec.InputSpec(min_ndim=2) def call(self, inputs): return inputs @@ -306,7 +308,7 @@ class BaseLayerTest(test.TestCase): def __init__(self): super(CustomerLayer, self).__init__() - self.input_spec = base_layers.InputSpec(max_ndim=2) + self.input_spec = input_spec.InputSpec(max_ndim=2) def call(self, inputs): return inputs @@ -334,7 +336,7 @@ class BaseLayerTest(test.TestCase): def __init__(self): super(CustomerLayer, self).__init__() - self.input_spec = base_layers.InputSpec(dtype='float32') + self.input_spec = input_spec.InputSpec(dtype='float32') def call(self, inputs): return inputs @@ -354,7 +356,7 @@ class BaseLayerTest(test.TestCase): def __init__(self): super(CustomerLayer, self).__init__() - self.input_spec = base_layers.InputSpec(axes={-1: 2}) + self.input_spec = input_spec.InputSpec(axes={-1: 2}) def call(self, inputs): return inputs @@ -376,7 +378,7 @@ class BaseLayerTest(test.TestCase): def __init__(self): super(CustomerLayer, self).__init__() - self.input_spec = base_layers.InputSpec(shape=(None, 3)) + self.input_spec = input_spec.InputSpec(shape=(None, 3)) def call(self, inputs): return inputs @@ -444,6 +446,7 @@ class BaseLayerTest(test.TestCase): self.assertTrue(isinstance(result, dict)) self.assertEqual(set(['label', 'logits']), set(result.keys())) + @test_util.run_deprecated_v1 def testActivityRegularizer(self): regularizer = math_ops.reduce_sum layer = base_layers.Layer(activity_regularizer=regularizer) @@ -532,6 +535,7 @@ class BaseLayerTest(test.TestCase): self.assertEqual(len(layer.trainable_variables), 1) self.assertEqual(layer.variables[0].graph, outer_graph) + @test_util.run_deprecated_v1 def testGetUpdateFor(self): class MyLayer(base_layers.Layer): @@ -576,6 +580,7 @@ class BaseLayerTest(test.TestCase): self.assertEqual(len(layer.get_updates_for([intermediate_inputs])), 1) self.assertEqual(len(layer.get_updates_for([outputs])), 0) + @test_util.run_deprecated_v1 def testGetLossesFor(self): class MyLayer(base_layers.Layer): diff --git a/tensorflow/python/layers/convolutional_test.py b/tensorflow/python/layers/convolutional_test.py index 257fa271567..a3e493edfea 100644 --- a/tensorflow/python/layers/convolutional_test.py +++ b/tensorflow/python/layers/convolutional_test.py @@ -22,6 +22,7 @@ import numpy as np from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops +from tensorflow.python.framework import test_util from tensorflow.python.layers import convolutional as conv_layers from tensorflow.python.ops import array_ops from tensorflow.python.ops import init_ops @@ -59,6 +60,7 @@ class ConvTest(test.TestCase): with self.assertRaisesRegexp(ValueError, 'kernel_size'): conv_layers.conv2d(images, 32, None) + @test_util.run_deprecated_v1 def testCreateConv2D(self): height, width = 7, 9 images = random_ops.random_uniform((5, height, width, 4)) @@ -87,6 +89,7 @@ class ConvTest(test.TestCase): self.assertListEqual(layer.kernel.get_shape().as_list(), [3, 3, 4, 32]) self.assertListEqual(layer.bias.get_shape().as_list(), [32]) + @test_util.run_deprecated_v1 def testCreateConv2DChannelsFirst(self): height, width = 7, 9 images = random_ops.random_uniform((5, 4, height, width)) @@ -97,6 +100,7 @@ class ConvTest(test.TestCase): self.assertListEqual(layer.kernel.get_shape().as_list(), [3, 3, 4, 32]) self.assertListEqual(layer.bias.get_shape().as_list(), [32]) + @test_util.run_deprecated_v1 def testUnknownInputChannels(self): images = array_ops.placeholder(dtypes.float32, (5, 7, 9, None)) layer = conv_layers.Conv2D(32, [3, 3], activation=nn_ops.relu) @@ -140,6 +144,7 @@ class ConvTest(test.TestCase): self.assertListEqual(output.get_shape().as_list(), [5, height / 2, width, 32]) + @test_util.run_deprecated_v1 def testCreateConv1D(self): width = 7 data = random_ops.random_uniform((5, width, 4)) @@ -156,6 +161,7 @@ class ConvTest(test.TestCase): output = conv_layers.conv1d(data, 32, 3, activation=nn_ops.relu) self.assertListEqual(output.get_shape().as_list(), [5, width - 2, 32]) + @test_util.run_deprecated_v1 def testCreateConv1DChannelsFirst(self): width = 7 data = random_ops.random_uniform((5, 4, width)) @@ -165,6 +171,7 @@ class ConvTest(test.TestCase): self.assertListEqual(layer.kernel.get_shape().as_list(), [3, 4, 32]) self.assertListEqual(layer.bias.get_shape().as_list(), [32]) + @test_util.run_deprecated_v1 def testUnknownInputChannelsConv1D(self): data = array_ops.placeholder(dtypes.float32, (5, 4, None)) layer = conv_layers.Conv1D(32, 3, activation=nn_ops.relu) @@ -180,6 +187,7 @@ class ConvTest(test.TestCase): 'should be defined. Found `None`.'): _ = layer.apply(data) + @test_util.run_deprecated_v1 def testCreateConv3D(self): depth, height, width = 6, 7, 9 volumes = random_ops.random_uniform((5, depth, height, width, 4)) @@ -191,6 +199,7 @@ class ConvTest(test.TestCase): self.assertListEqual(layer.kernel.get_shape().as_list(), [3, 3, 3, 4, 32]) self.assertListEqual(layer.bias.get_shape().as_list(), [32]) + @test_util.run_deprecated_v1 def testUnknownInputChannelsConv3D(self): volumes = array_ops.placeholder(dtypes.float32, (5, 6, 7, 9, None)) layer = conv_layers.Conv3D(32, [3, 3, 3], activation=nn_ops.relu) @@ -199,6 +208,7 @@ class ConvTest(test.TestCase): 'should be defined. Found `None`.'): _ = layer.apply(volumes) + @test_util.run_deprecated_v1 def testConv2DKernelRegularizer(self): height, width = 7, 9 images = random_ops.random_uniform((5, height, width, 4)) @@ -210,6 +220,7 @@ class ConvTest(test.TestCase): self.evaluate([v.initializer for v in layer.variables]) self.assertListEqual(self.evaluate(layer.losses), self.evaluate(loss_keys)) + @test_util.run_deprecated_v1 def testConv2DBiasRegularizer(self): height, width = 7, 9 images = random_ops.random_uniform((5, height, width, 4)) @@ -221,6 +232,7 @@ class ConvTest(test.TestCase): self.evaluate([v.initializer for v in layer.variables]) self.assertListEqual(self.evaluate(layer.losses), self.evaluate(loss_keys)) + @test_util.run_deprecated_v1 def testConv2DNoBias(self): height, width = 7, 9 images = random_ops.random_uniform((5, height, width, 4)) @@ -247,6 +259,7 @@ class ConvTest(test.TestCase): output = layer.apply(images) self.assertListEqual(output.get_shape().as_list(), [5, height - 2, 3, 32]) + @test_util.run_deprecated_v1 def testFunctionalConv2DReuse(self): height, width = 7, 9 images = random_ops.random_uniform((5, height, width, 3), seed=1) @@ -255,6 +268,7 @@ class ConvTest(test.TestCase): conv_layers.conv2d(images, 32, [3, 3], name='conv1', reuse=True) self.assertEqual(len(variables.trainable_variables()), 2) + @test_util.run_deprecated_v1 def testFunctionalConv2DReuseFromScope(self): with variable_scope.variable_scope('scope'): height, width = 7, 9 @@ -265,6 +279,7 @@ class ConvTest(test.TestCase): conv_layers.conv2d(images, 32, [3, 3], name='conv1') self.assertEqual(len(variables.trainable_variables()), 2) + @test_util.run_deprecated_v1 def testFunctionalConv2DInitializerFromScope(self): with self.cached_session() as sess: with variable_scope.variable_scope( @@ -276,13 +291,14 @@ class ConvTest(test.TestCase): # Check the names of weights in order. self.assertTrue('kernel' in weights[0].name) self.assertTrue('bias' in weights[1].name) - sess.run(variables.global_variables_initializer()) - weights = sess.run(weights) + self.evaluate(variables.global_variables_initializer()) + weights = self.evaluate(weights) # Check that the kernel weights got initialized to ones (from scope) self.assertAllClose(weights[0], np.ones((3, 3, 3, 32))) # Check that the bias still got initialized to zeros. self.assertAllClose(weights[1], np.zeros((32))) + @test_util.run_deprecated_v1 def testFunctionalConv2DNoReuse(self): height, width = 7, 9 images = random_ops.random_uniform((5, height, width, 3), seed=1) @@ -325,6 +341,7 @@ class ConvTest(test.TestCase): self.assertEqual(conv3d.kernel_constraint, k_constraint) self.assertEqual(conv3d.bias_constraint, b_constraint) + @test_util.run_deprecated_v1 def testConv3DChannelsFirst(self): # Test case for GitHub issue 15655 images = array_ops.placeholder( @@ -358,6 +375,7 @@ class SeparableConv1DTest(test.TestCase): with self.assertRaisesRegexp(ValueError, 'kernel_size'): conv_layers.separable_conv1d(data, 32, None) + @test_util.run_deprecated_v1 def testCreateSeparableConv1D(self): length = 9 data = random_ops.random_uniform((5, length, 4)) @@ -379,6 +397,7 @@ class SeparableConv1DTest(test.TestCase): self.assertEqual(layer.pointwise_kernel.get_shape().as_list(), [1, 8, 32]) self.assertEqual(layer.bias.get_shape().as_list(), [32]) + @test_util.run_deprecated_v1 def testCreateSeparableConv1DChannelsFirst(self): length = 9 data = random_ops.random_uniform((5, 4, length)) @@ -404,6 +423,7 @@ class SeparableConv1DTest(test.TestCase): output = layer.apply(data) self.assertEqual(output.get_shape().as_list(), [5, length // 2, 32]) + @test_util.run_deprecated_v1 def testCreateSeparableConv1DWithStridesChannelsFirst(self): data_format = 'channels_first' length = 10 @@ -413,6 +433,7 @@ class SeparableConv1DTest(test.TestCase): output = layer.apply(data) self.assertEqual(output.get_shape().as_list(), [5, 32, length // 2]) + @test_util.run_deprecated_v1 def testFunctionalConv1DReuse(self): length = 10 data = random_ops.random_uniform((5, length, 3), seed=1) @@ -421,6 +442,7 @@ class SeparableConv1DTest(test.TestCase): conv_layers.separable_conv1d(data, 32, 3, name='sepconv1', reuse=True) self.assertEqual(len(variables.trainable_variables()), 3) + @test_util.run_deprecated_v1 def testFunctionalConv1DReuseFromScope(self): with variable_scope.variable_scope('scope'): length = 10 @@ -431,6 +453,7 @@ class SeparableConv1DTest(test.TestCase): conv_layers.separable_conv1d(data, 32, 3, name='sepconv1') self.assertEqual(len(variables.trainable_variables()), 3) + @test_util.run_deprecated_v1 def testFunctionalConv1DNoReuse(self): length = 10 data = random_ops.random_uniform((5, length, 3), seed=1) @@ -439,6 +462,7 @@ class SeparableConv1DTest(test.TestCase): conv_layers.separable_conv1d(data, 32, 3) self.assertEqual(len(variables.trainable_variables()), 6) + @test_util.run_deprecated_v1 def testSeparableConv1DDepthwiseRegularizer(self): length = 9 data = random_ops.random_uniform((5, length, 4)) @@ -450,6 +474,7 @@ class SeparableConv1DTest(test.TestCase): self.evaluate([v.initializer for v in layer.variables]) self.assertListEqual(self.evaluate(layer.losses), self.evaluate(loss_keys)) + @test_util.run_deprecated_v1 def testSeparableConv1DPointwiseRegularizer(self): length = 9 data = random_ops.random_uniform((5, length, 4)) @@ -461,6 +486,7 @@ class SeparableConv1DTest(test.TestCase): self.evaluate([v.initializer for v in layer.variables]) self.assertListEqual(self.evaluate(layer.losses), self.evaluate(loss_keys)) + @test_util.run_deprecated_v1 def testSeparableConv1DBiasRegularizer(self): length = 9 data = random_ops.random_uniform((5, length, 4)) @@ -472,6 +498,7 @@ class SeparableConv1DTest(test.TestCase): self.evaluate([v.initializer for v in layer.variables]) self.assertListEqual(self.evaluate(layer.losses), self.evaluate(loss_keys)) + @test_util.run_deprecated_v1 def testSeparableConv1DNoBias(self): length = 9 data = random_ops.random_uniform((5, length, 4)) @@ -522,6 +549,7 @@ class SeparableConv2DTest(test.TestCase): with self.assertRaisesRegexp(ValueError, 'kernel_size'): conv_layers.separable_conv2d(images, 32, None) + @test_util.run_deprecated_v1 def testCreateSeparableConv2D(self): height, width = 7, 9 images = random_ops.random_uniform((5, height, width, 4)) @@ -562,6 +590,7 @@ class SeparableConv2DTest(test.TestCase): [1, 1, 4, 32]) self.assertListEqual(layer.bias.get_shape().as_list(), [32]) + @test_util.run_deprecated_v1 def testCreateSeparableConv2DChannelsFirst(self): height, width = 7, 9 images = random_ops.random_uniform((5, 4, height, width)) @@ -584,6 +613,7 @@ class SeparableConv2DTest(test.TestCase): output = layer.apply(images) self.assertListEqual(output.get_shape().as_list(), [5, height, width, 64]) + @test_util.run_deprecated_v1 def testCreateSeparableConvWithStrides(self): height, width = 6, 8 # Test strides tuple @@ -607,6 +637,7 @@ class SeparableConv2DTest(test.TestCase): self.assertListEqual(output.get_shape().as_list(), [5, height / 2, width, 32]) + @test_util.run_deprecated_v1 def testCreateSeparableConvWithStridesChannelsFirst(self): data_format = 'channels_first' height, width = 6, 8 @@ -632,6 +663,7 @@ class SeparableConv2DTest(test.TestCase): self.assertListEqual(output.get_shape().as_list(), [5, 32, height / 2, width]) + @test_util.run_deprecated_v1 def testFunctionalConv2DReuse(self): height, width = 7, 9 images = random_ops.random_uniform((5, height, width, 3), seed=1) @@ -641,6 +673,7 @@ class SeparableConv2DTest(test.TestCase): images, 32, [3, 3], name='sepconv1', reuse=True) self.assertEqual(len(variables.trainable_variables()), 3) + @test_util.run_deprecated_v1 def testFunctionalConv2DReuseFromScope(self): with variable_scope.variable_scope('scope'): height, width = 7, 9 @@ -651,6 +684,7 @@ class SeparableConv2DTest(test.TestCase): conv_layers.separable_conv2d(images, 32, [3, 3], name='sepconv1') self.assertEqual(len(variables.trainable_variables()), 3) + @test_util.run_deprecated_v1 def testFunctionalConv2DInitializerFromScope(self): with self.cached_session() as sess: with variable_scope.variable_scope( @@ -663,14 +697,15 @@ class SeparableConv2DTest(test.TestCase): self.assertTrue('depthwise_kernel' in weights[0].name) self.assertTrue('pointwise_kernel' in weights[1].name) self.assertTrue('bias' in weights[2].name) - sess.run(variables.global_variables_initializer()) - weights = sess.run(weights) + self.evaluate(variables.global_variables_initializer()) + weights = self.evaluate(weights) # Check that the kernel weights got initialized to ones (from scope) self.assertAllClose(weights[0], np.ones((3, 3, 3, 1))) self.assertAllClose(weights[1], np.ones((1, 1, 3, 32))) # Check that the bias still got initialized to zeros. self.assertAllClose(weights[2], np.zeros((32))) + @test_util.run_deprecated_v1 def testFunctionalConv2DNoReuse(self): height, width = 7, 9 images = random_ops.random_uniform((5, height, width, 3), seed=1) @@ -679,6 +714,7 @@ class SeparableConv2DTest(test.TestCase): conv_layers.separable_conv2d(images, 32, [3, 3]) self.assertEqual(len(variables.trainable_variables()), 6) + @test_util.run_deprecated_v1 def testSeparableConv2DDepthwiseRegularizer(self): height, width = 7, 9 images = random_ops.random_uniform((5, height, width, 4)) @@ -690,6 +726,7 @@ class SeparableConv2DTest(test.TestCase): self.evaluate([v.initializer for v in layer.variables]) self.assertListEqual(self.evaluate(layer.losses), self.evaluate(loss_keys)) + @test_util.run_deprecated_v1 def testSeparableConv2DPointwiseRegularizer(self): height, width = 7, 9 images = random_ops.random_uniform((5, height, width, 4)) @@ -701,6 +738,7 @@ class SeparableConv2DTest(test.TestCase): self.evaluate([v.initializer for v in layer.variables]) self.assertListEqual(self.evaluate(layer.losses), self.evaluate(loss_keys)) + @test_util.run_deprecated_v1 def testSeparableConv2DBiasRegularizer(self): height, width = 7, 9 images = random_ops.random_uniform((5, height, width, 4)) @@ -712,6 +750,7 @@ class SeparableConv2DTest(test.TestCase): self.evaluate([v.initializer for v in layer.variables]) self.assertListEqual(self.evaluate(layer.losses), self.evaluate(loss_keys)) + @test_util.run_deprecated_v1 def testSeparableConv2DNoBias(self): height, width = 7, 9 images = random_ops.random_uniform((5, height, width, 4)) @@ -768,6 +807,7 @@ class Conv2DTransposeTest(test.TestCase): with self.assertRaisesRegexp(ValueError, 'kernel_size'): conv_layers.conv2d_transpose(images, 32, None) + @test_util.run_deprecated_v1 def testCreateConv2DTranspose(self): height, width = 7, 9 images = random_ops.random_uniform((5, height, width, 4)) @@ -839,6 +879,7 @@ class Conv2DTransposeTest(test.TestCase): self.assertListEqual(output.get_shape().as_list(), [5, height * 2, width, 32]) + @test_util.run_deprecated_v1 def testConv2DTransposeKernelRegularizer(self): height, width = 7, 9 images = random_ops.random_uniform((5, height, width, 4)) @@ -850,6 +891,7 @@ class Conv2DTransposeTest(test.TestCase): self.evaluate([v.initializer for v in layer.variables]) self.assertListEqual(self.evaluate(layer.losses), self.evaluate(loss_keys)) + @test_util.run_deprecated_v1 def testConv2DTransposeBiasRegularizer(self): height, width = 7, 9 images = random_ops.random_uniform((5, height, width, 4)) @@ -861,6 +903,7 @@ class Conv2DTransposeTest(test.TestCase): self.evaluate([v.initializer for v in layer.variables]) self.assertListEqual(self.evaluate(layer.losses), self.evaluate(loss_keys)) + @test_util.run_deprecated_v1 def testConv2DTransposeNoBias(self): height, width = 7, 9 images = random_ops.random_uniform((5, height, width, 4)) @@ -873,6 +916,7 @@ class Conv2DTransposeTest(test.TestCase): self.assertListEqual(layer.kernel.get_shape().as_list(), [3, 3, 32, 4]) self.assertEqual(layer.bias, None) + @test_util.run_deprecated_v1 def testFunctionalConv2DTransposeReuse(self): height, width = 7, 9 images = random_ops.random_uniform((5, height, width, 3), seed=1) @@ -881,6 +925,7 @@ class Conv2DTransposeTest(test.TestCase): conv_layers.conv2d_transpose(images, 32, [3, 3], name='deconv1', reuse=True) self.assertEqual(len(variables.trainable_variables()), 2) + @test_util.run_deprecated_v1 def testFunctionalConv2DTransposeReuseFromScope(self): with variable_scope.variable_scope('scope'): height, width = 7, 9 @@ -891,6 +936,7 @@ class Conv2DTransposeTest(test.TestCase): conv_layers.conv2d_transpose(images, 32, [3, 3], name='deconv1') self.assertEqual(len(variables.trainable_variables()), 2) + @test_util.run_deprecated_v1 def testFunctionalConv2DTransposeInitializerFromScope(self): with self.cached_session() as sess: with variable_scope.variable_scope( @@ -902,13 +948,14 @@ class Conv2DTransposeTest(test.TestCase): # Check the names of weights in order. self.assertTrue('kernel' in weights[0].name) self.assertTrue('bias' in weights[1].name) - sess.run(variables.global_variables_initializer()) - weights = sess.run(weights) + self.evaluate(variables.global_variables_initializer()) + weights = self.evaluate(weights) # Check that the kernel weights got initialized to ones (from scope) self.assertAllClose(weights[0], np.ones((3, 3, 32, 3))) # Check that the bias still got initialized to zeros. self.assertAllClose(weights[1], np.zeros((32))) + @test_util.run_deprecated_v1 def testFunctionalConv2DTransposeNoReuse(self): height, width = 7, 9 images = random_ops.random_uniform((5, height, width, 3), seed=1) @@ -955,6 +1002,7 @@ class Conv3DTransposeTest(test.TestCase): with self.assertRaisesRegexp(ValueError, 'kernel_size'): conv_layers.conv3d_transpose(volumes, 4, None) + @test_util.run_deprecated_v1 def testCreateConv3DTranspose(self): depth, height, width = 5, 7, 9 volumes = random_ops.random_uniform((5, depth, height, width, 32)) @@ -976,6 +1024,7 @@ class Conv3DTransposeTest(test.TestCase): self.assertListEqual(layer.kernel.get_shape().as_list(), [3, 3, 3, 4, 32]) self.assertListEqual(layer.bias.get_shape().as_list(), [4]) + @test_util.run_deprecated_v1 def testCreateConv3DTransposeChannelsFirst(self): depth, height, width = 5, 7, 9 volumes = random_ops.random_uniform((5, 32, depth, height, width)) @@ -1019,6 +1068,7 @@ class Conv3DTransposeTest(test.TestCase): self.assertListEqual(output.get_shape().as_list(), [5, depth * 2, height, width, 4]) + @test_util.run_deprecated_v1 def testConv3DTransposeKernelRegularizer(self): depth, height, width = 5, 7, 9 volumes = random_ops.random_uniform((5, depth, height, width, 32)) @@ -1030,6 +1080,7 @@ class Conv3DTransposeTest(test.TestCase): self.evaluate([v.initializer for v in layer.variables]) self.assertListEqual(self.evaluate(layer.losses), self.evaluate(loss_keys)) + @test_util.run_deprecated_v1 def testConv3DTransposeBiasRegularizer(self): depth, height, width = 5, 7, 9 volumes = random_ops.random_uniform((5, depth, height, width, 32)) @@ -1041,6 +1092,7 @@ class Conv3DTransposeTest(test.TestCase): self.evaluate([v.initializer for v in layer.variables]) self.assertListEqual(self.evaluate(layer.losses), self.evaluate(loss_keys)) + @test_util.run_deprecated_v1 def testConv3DTransposeNoBias(self): depth, height, width = 5, 7, 9 volumes = random_ops.random_uniform((5, depth, height, width, 32)) @@ -1053,6 +1105,7 @@ class Conv3DTransposeTest(test.TestCase): self.assertListEqual(layer.kernel.get_shape().as_list(), [3, 3, 3, 4, 32]) self.assertEqual(layer.bias, None) + @test_util.run_deprecated_v1 def testFunctionalConv3DTransposeReuse(self): depth, height, width = 5, 7, 9 volumes = random_ops.random_uniform((5, depth, height, width, 32), seed=1) @@ -1062,6 +1115,7 @@ class Conv3DTransposeTest(test.TestCase): volumes, 4, [3, 3, 3], name='deconv1', reuse=True) self.assertEqual(len(variables.trainable_variables()), 2) + @test_util.run_deprecated_v1 def testFunctionalConv3DTransposeReuseFromScope(self): with variable_scope.variable_scope('scope'): depth, height, width = 5, 7, 9 @@ -1072,6 +1126,7 @@ class Conv3DTransposeTest(test.TestCase): conv_layers.conv3d_transpose(volumes, 4, [3, 3, 3], name='deconv1') self.assertEqual(len(variables.trainable_variables()), 2) + @test_util.run_deprecated_v1 def testFunctionalConv3DTransposeInitializerFromScope(self): with self.cached_session() as sess: with variable_scope.variable_scope( @@ -1084,13 +1139,14 @@ class Conv3DTransposeTest(test.TestCase): # Check the names of weights in order. self.assertTrue('kernel' in weights[0].name) self.assertTrue('bias' in weights[1].name) - sess.run(variables.global_variables_initializer()) - weights = sess.run(weights) + self.evaluate(variables.global_variables_initializer()) + weights = self.evaluate(weights) # Check that the kernel weights got initialized to ones (from scope) self.assertAllClose(weights[0], np.ones((3, 3, 3, 4, 32))) # Check that the bias still got initialized to zeros. self.assertAllClose(weights[1], np.zeros((4))) + @test_util.run_deprecated_v1 def testFunctionalConv3DTransposeNoReuse(self): depth, height, width = 5, 7, 9 volumes = random_ops.random_uniform((5, depth, height, width, 32), seed=1) diff --git a/tensorflow/python/layers/core_test.py b/tensorflow/python/layers/core_test.py index 0343bfa8bd2..cf6f0fbb700 100644 --- a/tensorflow/python/layers/core_test.py +++ b/tensorflow/python/layers/core_test.py @@ -59,6 +59,7 @@ class DenseTest(test.TestCase): dense.apply(random_ops.random_uniform((5, 2))) self.assertEqual(dense.name, 'dense_2') + @test_util.run_deprecated_v1 def testVariableInput(self): with self.cached_session(): v = variable_scope.get_variable( @@ -140,6 +141,7 @@ class DenseTest(test.TestCase): outputs = dense.apply(inputs) self.assertEqual(outputs.get_shape().as_list(), [1, 2, 4, 7]) + @test_util.run_deprecated_v1 def testCallOnPlaceHolder(self): inputs = array_ops.placeholder(dtype=dtypes.float32) dense = core_layers.Dense(4, name='my_dense') @@ -179,6 +181,7 @@ class DenseTest(test.TestCase): if not context.executing_eagerly(): self.assertEqual(outputs.op.name, 'dense2/BiasAdd') + @test_util.run_deprecated_v1 def testActivityRegularizer(self): regularizer = lambda x: math_ops.reduce_sum(x) * 1e-3 dense = core_layers.Dense( @@ -189,6 +192,7 @@ class DenseTest(test.TestCase): self.assertEqual(len(loss_keys), 1) self.assertListEqual(dense.losses, loss_keys) + @test_util.run_deprecated_v1 def testKernelRegularizer(self): regularizer = lambda x: math_ops.reduce_sum(x) * 1e-3 dense = core_layers.Dense( @@ -200,6 +204,7 @@ class DenseTest(test.TestCase): self.evaluate([v.initializer for v in dense.variables]) self.assertAllEqual(self.evaluate(dense.losses), self.evaluate(loss_keys)) + @test_util.run_deprecated_v1 def testKernelRegularizerWithReuse(self): regularizer = lambda x: math_ops.reduce_sum(x) * 1e-3 inputs = random_ops.random_uniform((5, 3), seed=1) @@ -212,6 +217,7 @@ class DenseTest(test.TestCase): self.assertEqual( len(ops.get_collection(ops.GraphKeys.REGULARIZATION_LOSSES)), 1) + @test_util.run_deprecated_v1 def testBiasRegularizer(self): regularizer = lambda x: math_ops.reduce_sum(x) * 1e-3 dense = core_layers.Dense(2, name='my_dense', bias_regularizer=regularizer) @@ -222,6 +228,7 @@ class DenseTest(test.TestCase): self.evaluate([v.initializer for v in dense.variables]) self.assertAllEqual(self.evaluate(dense.losses), self.evaluate(loss_keys)) + @test_util.run_deprecated_v1 def testFunctionalDense(self): with self.cached_session(): inputs = random_ops.random_uniform((5, 3), seed=1) @@ -231,6 +238,7 @@ class DenseTest(test.TestCase): len(ops.get_collection(ops.GraphKeys.TRAINABLE_VARIABLES)), 2) self.assertEqual(outputs.op.name, 'my_dense/Relu') + @test_util.run_deprecated_v1 def testFunctionalDenseTwice(self): inputs = random_ops.random_uniform((5, 3), seed=1) core_layers.dense(inputs, 2) @@ -262,6 +270,7 @@ class DenseTest(test.TestCase): vars2 = variables.trainable_variables() self.assertEqual(vars1, vars2) + @test_util.run_deprecated_v1 def testFunctionalDenseInitializerFromScope(self): with variable_scope.variable_scope( 'scope', @@ -307,6 +316,7 @@ class DenseTest(test.TestCase): core_layers.dense(inputs, 2) self.assertEqual(called[0], 2) + @test_util.run_deprecated_v1 def testFunctionalDenseInScope(self): with self.cached_session(): with variable_scope.variable_scope('test'): @@ -393,6 +403,7 @@ class DropoutTest(test.TestCase): np_output = self.evaluate(dropped) self.assertAllClose(np.ones((5, 3)), np_output) + @test_util.run_deprecated_v1 def testDynamicLearningPhase(self): with self.cached_session() as sess: dp = core_layers.Dropout(0.5, seed=1) @@ -426,6 +437,7 @@ class DropoutTest(test.TestCase): self.assertAlmostEqual(0., np_output.min()) self.assertAllClose(np_output[:, 0, :], np_output[:, 1, :]) + @test_util.run_deprecated_v1 def testFunctionalDropout(self): with self.cached_session(): inputs = array_ops.ones((5, 5)) @@ -437,13 +449,14 @@ class DropoutTest(test.TestCase): np_output = self.evaluate(dropped) self.assertAllClose(np.ones((5, 5)), np_output) + @test_util.run_deprecated_v1 def testDynamicRate(self): with self.cached_session() as sess: rate = array_ops.placeholder(dtype='float32', name='rate') dp = core_layers.Dropout(rate, name='dropout') inputs = array_ops.ones((5, 5)) dropped = dp.apply(inputs, training=True) - sess.run(variables.global_variables_initializer()) + self.evaluate(variables.global_variables_initializer()) np_output = sess.run(dropped, feed_dict={rate: 0.5}) self.assertAlmostEqual(0., np_output.min()) np_output = sess.run(dropped, feed_dict={rate: 0.0}) @@ -452,6 +465,7 @@ class DropoutTest(test.TestCase): class FlattenTest(test.TestCase): + @test_util.run_deprecated_v1 def testCreateFlatten(self): with self.cached_session() as sess: x = array_ops.placeholder(shape=(None, 2, 3), dtype='float32') @@ -476,6 +490,7 @@ class FlattenTest(test.TestCase): shape = core_layers.Flatten().compute_output_shape((None, 3, None)) self.assertEqual(shape.as_list(), [None, None]) + @test_util.run_deprecated_v1 def testDataFormat5d(self): np_input_channels_last = np.arange( 120, dtype='float32').reshape([1, 5, 4, 3, 2]) @@ -493,6 +508,7 @@ class FlattenTest(test.TestCase): self.assertAllEqual(np_output_cl, np_output_cf) + @test_util.run_deprecated_v1 def testDataFormat4d(self): np_input_channels_last = np.arange( 24, dtype='float32').reshape([1, 4, 3, 2]) @@ -510,16 +526,19 @@ class FlattenTest(test.TestCase): self.assertAllEqual(np_output_cl, np_output_cf) + @test_util.run_deprecated_v1 def testFunctionalFlatten(self): x = array_ops.placeholder(shape=(None, 2, 3), dtype='float32') y = core_layers.flatten(x, name='flatten') self.assertEqual(y.get_shape().as_list(), [None, 6]) + @test_util.run_deprecated_v1 def testFlattenValueError(self): x = array_ops.placeholder(shape=(None,), dtype='float32') with self.assertRaises(ValueError): core_layers.Flatten()(x) + @test_util.run_deprecated_v1 def testFlattenUnknownAxes(self): with self.cached_session() as sess: x = array_ops.placeholder(shape=(5, None, None), dtype='float32') diff --git a/tensorflow/python/layers/layers.py b/tensorflow/python/layers/layers.py index 11a2ebc040f..93eec38a08c 100644 --- a/tensorflow/python/layers/layers.py +++ b/tensorflow/python/layers/layers.py @@ -24,7 +24,7 @@ from __future__ import print_function # Base objects. from tensorflow.python.layers.base import Layer -from tensorflow.python.layers.base import InputSpec +from tensorflow.python.keras.engine.input_spec import InputSpec # Core layers. from tensorflow.python.layers.core import Dense diff --git a/tensorflow/python/layers/normalization_test.py b/tensorflow/python/layers/normalization_test.py index ba2bf10cf3a..07d8e40b759 100644 --- a/tensorflow/python/layers/normalization_test.py +++ b/tensorflow/python/layers/normalization_test.py @@ -24,6 +24,7 @@ import numpy as np from tensorflow.core.protobuf import saver_pb2 from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops +from tensorflow.python.framework import test_util from tensorflow.python.layers import convolutional as conv_layers from tensorflow.python.layers import normalization as normalization_layers from tensorflow.python.ops import array_ops @@ -78,7 +79,7 @@ class BNTest(test.TestCase): if restore: saver.restore(sess, checkpoint_path) else: - sess.run(variables.global_variables_initializer()) + self.evaluate(variables.global_variables_initializer()) np.random.seed(0) for _ in range(2): image_val = np.random.rand(*shape).astype(dtype.as_numpy_dtype) @@ -143,6 +144,7 @@ class BNTest(test.TestCase): return train_vars, loss_val + @test_util.run_deprecated_v1 def testHalfPrecision(self): ref_vars, ref_loss = self._trainEvalSequence( dtype=dtypes.float32, @@ -228,33 +230,43 @@ class BNTest(test.TestCase): ckpt_b_use_gpu, use_gpu_test_a, use_gpu_test_b, freeze_mode) + @test_util.run_deprecated_v1 def testCheckpointFusedCPUAndFusedGPU(self): self._testCheckpointCrossDevice(True, False, True, True) + @test_util.run_deprecated_v1 def testCheckpointFusedCPUAndFusedCPU(self): self._testCheckpointCrossDevice(True, False, True, False) + @test_util.run_deprecated_v1 def testCheckpointFusedGPUAndFusedGPU(self): self._testCheckpointCrossDevice(True, True, True, True) + @test_util.run_deprecated_v1 def testCheckpointNonFusedCPUAndNonFusedGPU(self): self._testCheckpointCrossDevice(False, False, False, True) + @test_util.run_deprecated_v1 def testCheckpointNonFusedCPUAndNonFusedCPU(self): self._testCheckpointCrossDevice(False, False, False, False) + @test_util.run_deprecated_v1 def testCheckpointNonFusedGPUAndNonFusedGPU(self): self._testCheckpointCrossDevice(False, True, False, True) + @test_util.run_deprecated_v1 def testCheckpointNonFusedGPUAndFusedGPU(self): self._testCheckpointCrossDevice(False, True, True, True) + @test_util.run_deprecated_v1 def testCheckpointNonFusedGPUAndFusedCPU(self): self._testCheckpointCrossDevice(False, True, True, False) + @test_util.run_deprecated_v1 def testCheckpointNonFusedCPUAndFusedCPU(self): self._testCheckpointCrossDevice(False, False, True, False) + @test_util.run_deprecated_v1 def testCreateBN(self): # Call layer. bn = normalization_layers.BatchNormalization(axis=1) @@ -281,6 +293,7 @@ class BNTest(test.TestCase): ops.get_collection(ops.GraphKeys.TRAINABLE_VARIABLES), bn.trainable_variables) + @test_util.run_deprecated_v1 def testCreateFusedBNFloat16(self): # Call layer. bn = normalization_layers.BatchNormalization(axis=1, fused=True) @@ -310,6 +323,7 @@ class BNTest(test.TestCase): ops.get_collection(ops.GraphKeys.TRAINABLE_VARIABLES), bn.trainable_variables) + @test_util.run_deprecated_v1 def test3DInputAxis1(self): epsilon = 1e-3 bn = normalization_layers.BatchNormalization( @@ -321,9 +335,9 @@ class BNTest(test.TestCase): with self.cached_session() as sess: # Test training with placeholder learning phase. - sess.run(variables.global_variables_initializer()) + self.evaluate(variables.global_variables_initializer()) - np_gamma, np_beta = sess.run([bn.gamma, bn.beta]) + np_gamma, np_beta = self.evaluate([bn.gamma, bn.beta]) np_gamma = np.reshape(np_gamma, (1, 4, 1)) np_beta = np.reshape(np_beta, (1, 4, 1)) @@ -336,8 +350,9 @@ class BNTest(test.TestCase): self.assertAlmostEqual(np.std(normed_np_output), 1., places=1) # Verify that the statistics are updated during training. - moving_mean, moving_var = sess.run([bn.moving_mean, bn.moving_variance]) - np_inputs = sess.run(inputs) + moving_mean, moving_var = self.evaluate( + [bn.moving_mean, bn.moving_variance]) + np_inputs = self.evaluate(inputs) mean = np.mean(np_inputs, axis=(0, 2)) std = np.std(np_inputs, axis=(0, 2)) variance = np.square(std) @@ -352,6 +367,7 @@ class BNTest(test.TestCase): self.assertAlmostEqual(np.mean(normed_np_output), 0., places=1) self.assertAlmostEqual(np.std(normed_np_output), 1., places=1) + @test_util.run_deprecated_v1 def test3DInputAxis2(self): epsilon = 1e-3 bn = normalization_layers.BatchNormalization( @@ -363,8 +379,8 @@ class BNTest(test.TestCase): with self.cached_session() as sess: # Test training with placeholder learning phase. - sess.run(variables.global_variables_initializer()) - np_gamma, np_beta = sess.run([bn.gamma, bn.beta]) + self.evaluate(variables.global_variables_initializer()) + np_gamma, np_beta = self.evaluate([bn.gamma, bn.beta]) np_gamma = np.reshape(np_gamma, (1, 1, 3)) np_beta = np.reshape(np_beta, (1, 1, 3)) for _ in range(100): @@ -376,8 +392,9 @@ class BNTest(test.TestCase): self.assertAlmostEqual(np.std(normed_np_output), 1., places=1) # Verify that the statistics are updated during training. - moving_mean, moving_var = sess.run([bn.moving_mean, bn.moving_variance]) - np_inputs = sess.run(inputs) + moving_mean, moving_var = self.evaluate( + [bn.moving_mean, bn.moving_variance]) + np_inputs = self.evaluate(inputs) mean = np.mean(np_inputs, axis=(0, 1)) std = np.std(np_inputs, axis=(0, 1)) variance = np.square(std) @@ -404,8 +421,8 @@ class BNTest(test.TestCase): with self.session(use_gpu=True) as sess: # Test training with placeholder learning phase. - sess.run(variables.global_variables_initializer()) - np_gamma, np_beta = sess.run([bn.gamma, bn.beta]) + self.evaluate(variables.global_variables_initializer()) + np_gamma, np_beta = self.evaluate([bn.gamma, bn.beta]) np_gamma = np.reshape(np_gamma, (1, 4, 1, 1)) np_beta = np.reshape(np_beta, (1, 4, 1, 1)) for _ in range(100): @@ -417,8 +434,9 @@ class BNTest(test.TestCase): self.assertAlmostEqual(np.std(normed_np_output), 1., places=1) # Verify that the statistics are updated during training. - moving_mean, moving_var = sess.run([bn.moving_mean, bn.moving_variance]) - np_inputs = sess.run(inputs) + moving_mean, moving_var = self.evaluate( + [bn.moving_mean, bn.moving_variance]) + np_inputs = self.evaluate(inputs) mean = np.mean(np_inputs, axis=(0, 2, 3)) std = np.std(np_inputs, axis=(0, 2, 3)) variance = np.square(std) @@ -433,6 +451,7 @@ class BNTest(test.TestCase): self.assertAlmostEqual(np.mean(normed_np_output), 0., places=1) self.assertAlmostEqual(np.std(normed_np_output), 1., places=1) + @test_util.run_deprecated_v1 def test4DInputAxis2(self): epsilon = 1e-3 bn = normalization_layers.BatchNormalization( @@ -444,8 +463,8 @@ class BNTest(test.TestCase): with self.cached_session() as sess: # Test training with placeholder learning phase. - sess.run(variables.global_variables_initializer()) - np_gamma, np_beta = sess.run([bn.gamma, bn.beta]) + self.evaluate(variables.global_variables_initializer()) + np_gamma, np_beta = self.evaluate([bn.gamma, bn.beta]) np_gamma = np.reshape(np_gamma, (1, 1, 3, 1)) np_beta = np.reshape(np_beta, (1, 1, 3, 1)) for _ in range(100): @@ -457,8 +476,9 @@ class BNTest(test.TestCase): self.assertAlmostEqual(np.std(normed_np_output), 1., places=1) # Verify that the statistics are updated during training. - moving_mean, moving_var = sess.run([bn.moving_mean, bn.moving_variance]) - np_inputs = sess.run(inputs) + moving_mean, moving_var = self.evaluate( + [bn.moving_mean, bn.moving_variance]) + np_inputs = self.evaluate(inputs) mean = np.mean(np_inputs, axis=(0, 1, 3)) std = np.std(np_inputs, axis=(0, 1, 3)) variance = np.square(std) @@ -473,6 +493,7 @@ class BNTest(test.TestCase): self.assertAlmostEqual(np.mean(normed_np_output), 0., places=1) self.assertAlmostEqual(np.std(normed_np_output), 1., places=1) + @test_util.run_deprecated_v1 def test4DInputAxis3(self): epsilon = 1e-3 bn = normalization_layers.BatchNormalization( @@ -484,8 +505,8 @@ class BNTest(test.TestCase): with self.cached_session() as sess: # Test training with placeholder learning phase. - sess.run(variables.global_variables_initializer()) - np_gamma, np_beta = sess.run([bn.gamma, bn.beta]) + self.evaluate(variables.global_variables_initializer()) + np_gamma, np_beta = self.evaluate([bn.gamma, bn.beta]) np_gamma = np.reshape(np_gamma, (1, 1, 1, 6)) np_beta = np.reshape(np_beta, (1, 1, 1, 6)) for _ in range(100): @@ -497,8 +518,9 @@ class BNTest(test.TestCase): self.assertAlmostEqual(np.std(normed_np_output), 1., places=1) # Verify that the statistics are updated during training. - moving_mean, moving_var = sess.run([bn.moving_mean, bn.moving_variance]) - np_inputs = sess.run(inputs) + moving_mean, moving_var = self.evaluate( + [bn.moving_mean, bn.moving_variance]) + np_inputs = self.evaluate(inputs) mean = np.mean(np_inputs, axis=(0, 1, 2)) std = np.std(np_inputs, axis=(0, 1, 2)) variance = np.square(std) @@ -513,6 +535,7 @@ class BNTest(test.TestCase): self.assertAlmostEqual(np.mean(normed_np_output), 0., places=1) self.assertAlmostEqual(np.std(normed_np_output), 1., places=1) + @test_util.run_deprecated_v1 def test4DInputAxis3Fused(self): epsilon = 1e-3 bn = normalization_layers.BatchNormalization( @@ -524,8 +547,8 @@ class BNTest(test.TestCase): with self.cached_session() as sess: # Test training with placeholder learning phase. - sess.run(variables.global_variables_initializer()) - np_gamma, np_beta = sess.run([bn.gamma, bn.beta]) + self.evaluate(variables.global_variables_initializer()) + np_gamma, np_beta = self.evaluate([bn.gamma, bn.beta]) np_gamma = np.reshape(np_gamma, (1, 1, 1, 6)) np_beta = np.reshape(np_beta, (1, 1, 1, 6)) for _ in range(100): @@ -537,8 +560,9 @@ class BNTest(test.TestCase): self.assertAlmostEqual(np.std(normed_np_output), 1., places=1) # Verify that the statistics are updated during training. - moving_mean, moving_var = sess.run([bn.moving_mean, bn.moving_variance]) - np_inputs = sess.run(inputs) + moving_mean, moving_var = self.evaluate( + [bn.moving_mean, bn.moving_variance]) + np_inputs = self.evaluate(inputs) mean = np.mean(np_inputs, axis=(0, 1, 2)) std = np.std(np_inputs, axis=(0, 1, 2)) variance = np.square(std) @@ -565,8 +589,8 @@ class BNTest(test.TestCase): with self.cached_session() as sess: # Test training with placeholder learning phase. - sess.run(variables.global_variables_initializer()) - np_gamma, np_beta = sess.run([bn.gamma, bn.beta]) + self.evaluate(variables.global_variables_initializer()) + np_gamma, np_beta = self.evaluate([bn.gamma, bn.beta]) np_gamma = np.reshape(np_gamma, (1, 4, 1, 1)) np_beta = np.reshape(np_beta, (1, 4, 1, 1)) for _ in range(100): @@ -578,8 +602,9 @@ class BNTest(test.TestCase): self.assertAlmostEqual(np.std(normed_np_output), 1., places=1) # Verify that the statistics are updated during training. - moving_mean, moving_var = sess.run([bn.moving_mean, bn.moving_variance]) - np_inputs = sess.run(inputs) + moving_mean, moving_var = self.evaluate( + [bn.moving_mean, bn.moving_variance]) + np_inputs = self.evaluate(inputs) mean = np.mean(np_inputs, axis=(0, 2, 3)) std = np.std(np_inputs, axis=(0, 2, 3)) variance = np.square(std) @@ -594,6 +619,7 @@ class BNTest(test.TestCase): self.assertAlmostEqual(np.mean(normed_np_output), 0., places=1) self.assertAlmostEqual(np.std(normed_np_output), 1., places=1) + @test_util.run_deprecated_v1 def testNegativeAxis(self): epsilon = 1e-3 bn = normalization_layers.BatchNormalization( @@ -605,8 +631,8 @@ class BNTest(test.TestCase): with self.cached_session() as sess: # Test training with placeholder learning phase. - sess.run(variables.global_variables_initializer()) - np_gamma, np_beta = sess.run([bn.gamma, bn.beta]) + self.evaluate(variables.global_variables_initializer()) + np_gamma, np_beta = self.evaluate([bn.gamma, bn.beta]) np_gamma = np.reshape(np_gamma, (1, 1, 1, 6)) np_beta = np.reshape(np_beta, (1, 1, 1, 6)) for _ in range(100): @@ -619,8 +645,9 @@ class BNTest(test.TestCase): self.assertAlmostEqual(np.std(normed_np_output), 1., places=1) # Verify that the statistics are updated during training. - moving_mean, moving_var = sess.run([bn.moving_mean, bn.moving_variance]) - np_inputs = sess.run(inputs) + moving_mean, moving_var = self.evaluate( + [bn.moving_mean, bn.moving_variance]) + np_inputs = self.evaluate(inputs) mean = np.mean(np_inputs, axis=(0, 1, 2)) std = np.std(np_inputs, axis=(0, 1, 2)) variance = np.square(std) @@ -635,6 +662,7 @@ class BNTest(test.TestCase): self.assertAlmostEqual(np.mean(normed_np_output), 0., places=1) self.assertAlmostEqual(np.std(normed_np_output), 1., places=1) + @test_util.run_deprecated_v1 def testBooleanLearningPhase(self): epsilon = 1e-3 bn = normalization_layers.BatchNormalization( @@ -646,8 +674,8 @@ class BNTest(test.TestCase): with self.cached_session() as sess: # Test training with placeholder learning phase. - sess.run(variables.global_variables_initializer()) - np_gamma, np_beta = sess.run([bn.gamma, bn.beta]) + self.evaluate(variables.global_variables_initializer()) + np_gamma, np_beta = self.evaluate([bn.gamma, bn.beta]) np_gamma = np.reshape(np_gamma, (1, 1, 1, 6)) np_beta = np.reshape(np_beta, (1, 1, 1, 6)) for _ in range(100): @@ -658,8 +686,9 @@ class BNTest(test.TestCase): self.assertAlmostEqual(np.std(normed_np_output), 1., places=1) # Verify that the statistics are updated during training. - moving_mean, moving_var = sess.run([bn.moving_mean, bn.moving_variance]) - np_inputs = sess.run(inputs) + moving_mean, moving_var = self.evaluate( + [bn.moving_mean, bn.moving_variance]) + np_inputs = self.evaluate(inputs) mean = np.mean(np_inputs, axis=(0, 1, 2)) std = np.std(np_inputs, axis=(0, 1, 2)) variance = np.square(std) @@ -667,13 +696,14 @@ class BNTest(test.TestCase): self.assertAllClose(variance, moving_var, atol=1e-2) # Test inference with placeholder learning phase. - np_output = sess.run(outputs_infer) + np_output = self.evaluate(outputs_infer) # Verify that the axis is normalized during inference. normed_np_output = ((np_output - epsilon) * np_gamma) + np_beta self.assertAlmostEqual(np.mean(normed_np_output), 0., places=1) self.assertAlmostEqual(np.std(normed_np_output), 1., places=1) + @test_util.run_deprecated_v1 def testFunctionalNoReuse(self): inputs = variables.Variable( np.random.random((5, 4, 3, 6)), dtype=dtypes.float32) @@ -696,8 +726,8 @@ class BNTest(test.TestCase): with self.cached_session() as sess: # Test training with placeholder learning phase. - sess.run(variables.global_variables_initializer()) - np_gamma, np_beta = sess.run([gamma, beta]) + self.evaluate(variables.global_variables_initializer()) + np_gamma, np_beta = self.evaluate([gamma, beta]) np_gamma = np.reshape(np_gamma, (1, 1, 1, 6)) np_beta = np.reshape(np_beta, (1, 1, 1, 6)) for _ in range(100): @@ -709,8 +739,9 @@ class BNTest(test.TestCase): self.assertAlmostEqual(np.std(normed_np_output), 1., places=1) # Verify that the statistics are updated during training. - np_moving_mean, np_moving_var = sess.run([moving_mean, moving_variance]) - np_inputs = sess.run(inputs) + np_moving_mean, np_moving_var = self.evaluate( + [moving_mean, moving_variance]) + np_inputs = self.evaluate(inputs) np_mean = np.mean(np_inputs, axis=(0, 1, 2)) np_std = np.std(np_inputs, axis=(0, 1, 2)) np_variance = np.square(np_std) @@ -725,6 +756,7 @@ class BNTest(test.TestCase): self.assertAlmostEqual(np.mean(normed_np_output), 0., places=1) self.assertAlmostEqual(np.std(normed_np_output), 1., places=1) + @test_util.run_deprecated_v1 def testFunctionalReuse(self): inputs1 = variables.Variable( np.random.random((5, 4, 3, 6)), dtype=dtypes.float32) @@ -758,14 +790,15 @@ class BNTest(test.TestCase): with self.cached_session() as sess: # Test training with placeholder learning phase. - sess.run(variables.global_variables_initializer()) + self.evaluate(variables.global_variables_initializer()) for _ in range(100): np_output, _, _ = sess.run([outputs2] + updates, feed_dict={training: True}) # Verify that the statistics are updated during training. - np_moving_mean, np_moving_var = sess.run([moving_mean, moving_variance]) - np_inputs = sess.run(inputs2) + np_moving_mean, np_moving_var = self.evaluate( + [moving_mean, moving_variance]) + np_inputs = self.evaluate(inputs2) np_mean = np.mean(np_inputs, axis=(0, 1, 2)) np_std = np.std(np_inputs, axis=(0, 1, 2)) np_variance = np.square(np_std) @@ -773,7 +806,7 @@ class BNTest(test.TestCase): self.assertAllClose(np_variance, np_moving_var, atol=1e-2) # Verify that the axis is normalized during training. - np_gamma, np_beta = sess.run([gamma, beta]) + np_gamma, np_beta = self.evaluate([gamma, beta]) np_gamma = np.reshape(np_gamma, (1, 1, 1, 6)) np_beta = np.reshape(np_beta, (1, 1, 1, 6)) normed_np_output = ((np_output - epsilon) * np_gamma) + np_beta @@ -788,6 +821,7 @@ class BNTest(test.TestCase): self.assertAlmostEqual(np.mean(normed_np_output), 0., places=2) self.assertAlmostEqual(np.std(normed_np_output), 1., places=1) + @test_util.run_deprecated_v1 def testFunctionalReuseFromScope(self): inputs = variables.Variable( np.random.random((5, 4, 3, 6)), dtype=dtypes.float32) @@ -802,6 +836,7 @@ class BNTest(test.TestCase): inputs, axis=-1, momentum=0.9, epsilon=epsilon, training=training) self.assertEqual(len(variables.global_variables()), 5) + @test_util.run_deprecated_v1 def testNoCenter(self): bn = normalization_layers.BatchNormalization(axis=1, center=False) inputs = random_ops.random_uniform((5, 4, 3), seed=1) @@ -817,6 +852,7 @@ class BNTest(test.TestCase): self.assertEqual(len(bn.trainable_variables), 1) self.assertEqual(len(bn.non_trainable_variables), 2) + @test_util.run_deprecated_v1 def testNoScale(self): bn = normalization_layers.BatchNormalization(axis=1, scale=False) inputs = random_ops.random_uniform((5, 4, 3), seed=1) @@ -832,6 +868,7 @@ class BNTest(test.TestCase): self.assertEqual(len(bn.trainable_variables), 1) self.assertEqual(len(bn.non_trainable_variables), 2) + @test_util.run_deprecated_v1 def testRegularizers(self): reg = lambda x: 0.1 * math_ops.reduce_sum(x) bn = normalization_layers.BatchNormalization(axis=1, beta_regularizer=reg) @@ -857,6 +894,7 @@ class BNTest(test.TestCase): self.assertEqual(bn.gamma_constraint, g_constraint) self.assertEqual(bn.beta_constraint, b_constraint) + @test_util.run_deprecated_v1 def testRenorm(self): shape = (4, 3) xt = array_ops.placeholder(dtypes.float32, shape) @@ -885,7 +923,7 @@ class BNTest(test.TestCase): renorm_mean = renorm_stddev = 0. renorm_weight = 0. with self.session(use_gpu=True) as sess: - sess.run(variables.global_variables_initializer()) + self.evaluate(variables.global_variables_initializer()) for _ in range(5): x = np.random.random(shape) @@ -915,6 +953,7 @@ class BNTest(test.TestCase): self.assertAllClose(y_train, yt_val_train, atol=1e-5) self.assertAllClose(y_test, yt_val_test, atol=1e-5) + @test_util.run_deprecated_v1 def testAdjustment(self): shape = (4, 3) xt = array_ops.placeholder(dtypes.float32, shape) @@ -937,7 +976,7 @@ class BNTest(test.TestCase): moving_mean = 0. moving_variance = 1. with self.session(use_gpu=True) as sess: - sess.run(variables.global_variables_initializer()) + self.evaluate(variables.global_variables_initializer()) for _ in range(5): x = np.random.random(shape) yt_val_train, adj_scale_val, adj_bias_val = sess.run( @@ -959,6 +998,7 @@ class BNTest(test.TestCase): self.assertAllClose(y_train, yt_val_train, atol=1e-5) self.assertAllClose(y_test, yt_val_test, atol=1e-5) + @test_util.run_deprecated_v1 def testRenormWithAdjustment(self): shape = (4, 3) xt = array_ops.placeholder(dtypes.float32, shape) @@ -990,7 +1030,7 @@ class BNTest(test.TestCase): renorm_mean = renorm_stddev = 0. renorm_weight = 0. with self.session(use_gpu=True) as sess: - sess.run(variables.global_variables_initializer()) + self.evaluate(variables.global_variables_initializer()) for _ in range(5): x = np.random.random(shape) yt_val_train, adj_scale_val, adj_bias_val = sess.run( @@ -1029,6 +1069,7 @@ class BNTest(test.TestCase): normalization_layers.batch_normalization( inp, virtual_batch_size=-1) + @test_util.run_deprecated_v1 def testGhostBNVirtualBatchFull(self): shape = [6, 5, 4, 3] inp = random_ops.random_uniform(shape, seed=1) @@ -1040,7 +1081,7 @@ class BNTest(test.TestCase): out1.shape.as_list(), out2.shape.as_list()) with self.session(use_gpu=True) as sess: - sess.run(variables.global_variables_initializer()) + self.evaluate(variables.global_variables_initializer()) x = np.random.random(shape) y1, y2 = sess.run([out1, out2], feed_dict={inp: x}) @@ -1054,6 +1095,7 @@ class BNTest(test.TestCase): inp, virtual_batch_size=3) self.assertListEqual(out.shape.as_list(), shape) + @test_util.run_deprecated_v1 def testGhostBNUnknownBatchSize(self): np_shape = [10, 5, 4] tf_shape = [None, 5, 4] @@ -1062,13 +1104,14 @@ class BNTest(test.TestCase): inp, virtual_batch_size=2) with self.session(use_gpu=True) as sess: - sess.run(variables.global_variables_initializer()) + self.evaluate(variables.global_variables_initializer()) x = np.random.random(np_shape) y = sess.run(out, feed_dict={inp: x}) self.assertListEqual(list(y.shape), np_shape) + @test_util.run_deprecated_v1 def testGhostBN2Dims(self): shape = [6, 2] virtual_batch_size = 3 @@ -1093,7 +1136,7 @@ class BNTest(test.TestCase): shape[1]]) with self.session(use_gpu=True) as sess: - sess.run(variables.global_variables_initializer()) + self.evaluate(variables.global_variables_initializer()) for _ in range(5): x = np.random.random(shape) @@ -1122,6 +1165,7 @@ class BNTest(test.TestCase): self.assertAllClose(y_train, y_val_train, atol=1e-5) self.assertAllClose(y_test, y_val_test, atol=1e-5) + @test_util.run_deprecated_v1 def testGhostBN4DimsAxis3(self): shape = [6, 10, 10, 3] virtual_batch_size = 2 @@ -1146,7 +1190,7 @@ class BNTest(test.TestCase): shape[1:]) with self.session(use_gpu=True) as sess: - sess.run(variables.global_variables_initializer()) + self.evaluate(variables.global_variables_initializer()) for _ in range(5): x = np.random.random(shape) @@ -1175,6 +1219,7 @@ class BNTest(test.TestCase): self.assertAllClose(y_train, y_val_train, atol=1e-2) self.assertAllClose(y_test, y_val_test, atol=1e-2) + @test_util.run_deprecated_v1 def testGhostBN4DimsAxis1(self): shape = [6, 3, 10, 10] virtual_batch_size = 2 @@ -1200,7 +1245,7 @@ class BNTest(test.TestCase): shape[1:]) with self.session(use_gpu=True) as sess: - sess.run(variables.global_variables_initializer()) + self.evaluate(variables.global_variables_initializer()) for _ in range(5): x = np.random.random(shape) @@ -1245,6 +1290,7 @@ class BNTest(test.TestCase): normalization_layers.batch_normalization( inp, axis=[1, 2, 1]) # duplicate + @test_util.run_deprecated_v1 def test3DInputMultiAxis12(self): epsilon = 1e-3 bn = normalization_layers.BatchNormalization( @@ -1256,9 +1302,9 @@ class BNTest(test.TestCase): with self.cached_session() as sess: # Test training with placeholder learning phase. - sess.run(variables.global_variables_initializer()) + self.evaluate(variables.global_variables_initializer()) - np_gamma, np_beta = sess.run([bn.gamma, bn.beta]) + np_gamma, np_beta = self.evaluate([bn.gamma, bn.beta]) for _ in range(100): np_output, _, _ = sess.run([outputs] + bn.updates, @@ -1269,8 +1315,9 @@ class BNTest(test.TestCase): self.assertAlmostEqual(np.std(normed_np_output), 1., places=1) # Verify that the statistics are updated during training. - moving_mean, moving_var = sess.run([bn.moving_mean, bn.moving_variance]) - np_inputs = sess.run(inputs) + moving_mean, moving_var = self.evaluate( + [bn.moving_mean, bn.moving_variance]) + np_inputs = self.evaluate(inputs) mean = np.mean(np_inputs, axis=0, keepdims=True) std = np.std(np_inputs, axis=0, keepdims=True) variance = np.square(std) @@ -1285,6 +1332,7 @@ class BNTest(test.TestCase): self.assertAlmostEqual(np.mean(normed_np_output), 0., places=1) self.assertAlmostEqual(np.std(normed_np_output), 1., places=1) + @test_util.run_deprecated_v1 def test5DInputMultiAxis123(self): epsilon = 1e-3 bn = normalization_layers.BatchNormalization( @@ -1296,9 +1344,9 @@ class BNTest(test.TestCase): with self.cached_session() as sess: # Test training with placeholder learning phase. - sess.run(variables.global_variables_initializer()) + self.evaluate(variables.global_variables_initializer()) - np_gamma, np_beta = sess.run([bn.gamma, bn.beta]) + np_gamma, np_beta = self.evaluate([bn.gamma, bn.beta]) for _ in range(100): np_output, _, _ = sess.run([outputs] + bn.updates, @@ -1309,8 +1357,9 @@ class BNTest(test.TestCase): self.assertAlmostEqual(np.std(normed_np_output), 1., places=1) # Verify that the statistics are updated during training. - moving_mean, moving_var = sess.run([bn.moving_mean, bn.moving_variance]) - np_inputs = sess.run(inputs) + moving_mean, moving_var = self.evaluate( + [bn.moving_mean, bn.moving_variance]) + np_inputs = self.evaluate(inputs) mean = np.mean(np_inputs, axis=(0, 4), keepdims=True) std = np.std(np_inputs, axis=(0, 4), keepdims=True) variance = np.square(std) @@ -1325,6 +1374,7 @@ class BNTest(test.TestCase): self.assertAlmostEqual(np.mean(normed_np_output), 0., places=1) self.assertAlmostEqual(np.std(normed_np_output), 1., places=1) + @test_util.run_deprecated_v1 def testGhostBN5DimsMultiAxis14(self): shape = [6, 3, 10, 10, 4] virtual_batch_size = 3 @@ -1350,7 +1400,7 @@ class BNTest(test.TestCase): shape[1:]) with self.session(use_gpu=True) as sess: - sess.run(variables.global_variables_initializer()) + self.evaluate(variables.global_variables_initializer()) for _ in range(5): x = np.random.random(shape) diff --git a/tensorflow/python/layers/pooling_test.py b/tensorflow/python/layers/pooling_test.py index 7533674e5a0..cf1fa1e6915 100644 --- a/tensorflow/python/layers/pooling_test.py +++ b/tensorflow/python/layers/pooling_test.py @@ -18,6 +18,7 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function +from tensorflow.python.framework import test_util from tensorflow.python.layers import pooling as pooling_layers from tensorflow.python.ops import array_ops from tensorflow.python.ops import random_ops @@ -64,6 +65,7 @@ class PoolingTest(test.TestCase): output = layer.apply(images) self.assertListEqual(output.get_shape().as_list(), [5, 3, 4, 4]) + @test_util.run_deprecated_v1 def testCreateMaxPooling2DChannelsFirst(self): height, width = 7, 9 images = random_ops.random_uniform((5, 2, height, width)) @@ -73,6 +75,7 @@ class PoolingTest(test.TestCase): output = layer.apply(images) self.assertListEqual(output.get_shape().as_list(), [5, 2, 6, 8]) + @test_util.run_deprecated_v1 def testCreateAveragePooling2DChannelsFirst(self): height, width = 5, 6 images = random_ops.random_uniform((3, 4, height, width)) @@ -83,6 +86,7 @@ class PoolingTest(test.TestCase): output = layer.apply(images) self.assertListEqual(output.get_shape().as_list(), [3, 4, 4, 5]) + @test_util.run_deprecated_v1 def testCreateAveragePooling2DChannelsFirstWithNoneBatch(self): height, width = 5, 6 images = array_ops.placeholder(dtype='float32', diff --git a/tensorflow/python/lib/io/file_io.py b/tensorflow/python/lib/io/file_io.py index f22fb253e4d..4caa5750bf6 100644 --- a/tensorflow/python/lib/io/file_io.py +++ b/tensorflow/python/lib/io/file_io.py @@ -241,7 +241,7 @@ class FileIO(object): self._writable_file = None -@tf_export("gfile.Exists") +@tf_export(v1=["gfile.Exists"]) def file_exists(filename): """Determines whether a path exists or not. @@ -252,18 +252,35 @@ def file_exists(filename): True if the path exists, whether its a file or a directory. False if the path does not exist and there are no filesystem errors. + Raises: + errors.OpError: Propagates any errors reported by the FileSystem API. + """ + return file_exists_v2(filename) + + +@tf_export("io.gfile.exists") +def file_exists_v2(path): + """Determines whether a path exists or not. + + Args: + path: string, a path + + Returns: + True if the path exists, whether its a file or a directory. + False if the path does not exist and there are no filesystem errors. + Raises: errors.OpError: Propagates any errors reported by the FileSystem API. """ try: with errors.raise_exception_on_not_ok_status() as status: - pywrap_tensorflow.FileExists(compat.as_bytes(filename), status) + pywrap_tensorflow.FileExists(compat.as_bytes(path), status) except errors.NotFoundError: return False return True -@tf_export("gfile.Remove") +@tf_export(v1=["gfile.Remove"]) def delete_file(filename): """Deletes the file located at 'filename'. @@ -274,8 +291,22 @@ def delete_file(filename): errors.OpError: Propagates any errors reported by the FileSystem API. E.g., NotFoundError if the file does not exist. """ + delete_file_v2(filename) + + +@tf_export("io.gfile.remove") +def delete_file_v2(path): + """Deletes the path located at 'path'. + + Args: + path: string, a path + + Raises: + errors.OpError: Propagates any errors reported by the FileSystem API. E.g., + NotFoundError if the path does not exist. + """ with errors.raise_exception_on_not_ok_status() as status: - pywrap_tensorflow.DeleteFile(compat.as_bytes(filename), status) + pywrap_tensorflow.DeleteFile(compat.as_bytes(path), status) def read_file_to_string(filename, binary_mode=False): @@ -314,7 +345,7 @@ def write_string_to_file(filename, file_content): f.write(file_content) -@tf_export("gfile.Glob") +@tf_export(v1=["gfile.Glob"]) def get_matching_files(filename): """Returns a list of files that match the given pattern(s). @@ -324,28 +355,44 @@ def get_matching_files(filename): Returns: A list of strings containing filenames that match the given pattern(s). + Raises: + errors.OpError: If there are filesystem / directory listing errors. + """ + return get_matching_files_v2(filename) + + +@tf_export("io.gfile.glob") +def get_matching_files_v2(pattern): + """Returns a list of files that match the given pattern(s). + + Args: + pattern: string or iterable of strings. The glob pattern(s). + + Returns: + A list of strings containing filenames that match the given pattern(s). + Raises: errors.OpError: If there are filesystem / directory listing errors. """ with errors.raise_exception_on_not_ok_status() as status: - if isinstance(filename, six.string_types): + if isinstance(pattern, six.string_types): return [ # Convert the filenames to string from bytes. compat.as_str_any(matching_filename) for matching_filename in pywrap_tensorflow.GetMatchingFiles( - compat.as_bytes(filename), status) + compat.as_bytes(pattern), status) ] else: return [ # Convert the filenames to string from bytes. compat.as_str_any(matching_filename) - for single_filename in filename + for single_filename in pattern for matching_filename in pywrap_tensorflow.GetMatchingFiles( compat.as_bytes(single_filename), status) ] -@tf_export("gfile.MkDir") +@tf_export(v1=["gfile.MkDir"]) def create_dir(dirname): """Creates a directory with the name 'dirname'. @@ -356,14 +403,31 @@ def create_dir(dirname): The parent directories need to exist. Use recursive_create_dir instead if there is the possibility that the parent dirs don't exist. + Raises: + errors.OpError: If the operation fails. + """ + create_dir_v2(dirname) + + +@tf_export("io.gfile.mkdir") +def create_dir_v2(path): + """Creates a directory with the name given by 'path'. + + Args: + path: string, name of the directory to be created + + Notes: + The parent directories need to exist. Use recursive_create_dir instead if + there is the possibility that the parent dirs don't exist. + Raises: errors.OpError: If the operation fails. """ with errors.raise_exception_on_not_ok_status() as status: - pywrap_tensorflow.CreateDir(compat.as_bytes(dirname), status) + pywrap_tensorflow.CreateDir(compat.as_bytes(path), status) -@tf_export("gfile.MakeDirs") +@tf_export(v1=["gfile.MakeDirs"]) def recursive_create_dir(dirname): """Creates a directory and all parent/intermediate directories. @@ -372,14 +436,29 @@ def recursive_create_dir(dirname): Args: dirname: string, name of the directory to be created + Raises: + errors.OpError: If the operation fails. + """ + recursive_create_dir_v2(dirname) + + +@tf_export("io.gfile.makedirs") +def recursive_create_dir_v2(path): + """Creates a directory and all parent/intermediate directories. + + It succeeds if path already exists and is writable. + + Args: + path: string, name of the directory to be created + Raises: errors.OpError: If the operation fails. """ with errors.raise_exception_on_not_ok_status() as status: - pywrap_tensorflow.RecursivelyCreateDir(compat.as_bytes(dirname), status) + pywrap_tensorflow.RecursivelyCreateDir(compat.as_bytes(path), status) -@tf_export("gfile.Copy") +@tf_export(v1=["gfile.Copy"]) def copy(oldpath, newpath, overwrite=False): """Copies data from oldpath to newpath. @@ -389,15 +468,31 @@ def copy(oldpath, newpath, overwrite=False): overwrite: boolean, if false its an error for newpath to be occupied by an existing file. + Raises: + errors.OpError: If the operation fails. + """ + copy_v2(oldpath, newpath, overwrite) + + +@tf_export("io.gfile.copy") +def copy_v2(src, dst, overwrite=False): + """Copies data from src to dst. + + Args: + src: string, name of the file whose contents need to be copied + dst: string, name of the file to which to copy to + overwrite: boolean, if false its an error for newpath to be occupied by an + existing file. + Raises: errors.OpError: If the operation fails. """ with errors.raise_exception_on_not_ok_status() as status: pywrap_tensorflow.CopyFile( - compat.as_bytes(oldpath), compat.as_bytes(newpath), overwrite, status) + compat.as_bytes(src), compat.as_bytes(dst), overwrite, status) -@tf_export("gfile.Rename") +@tf_export(v1=["gfile.Rename"]) def rename(oldname, newname, overwrite=False): """Rename or move a file / directory. @@ -407,12 +502,28 @@ def rename(oldname, newname, overwrite=False): overwrite: boolean, if false it's an error for `newname` to be occupied by an existing file. + Raises: + errors.OpError: If the operation fails. + """ + rename_v2(oldname, newname, overwrite) + + +@tf_export("io.gfile.rename") +def rename_v2(src, dst, overwrite=False): + """Rename or move a file / directory. + + Args: + src: string, pathname for a file + dst: string, pathname to which the file needs to be moved + overwrite: boolean, if false it's an error for `dst` to be occupied by + an existing file. + Raises: errors.OpError: If the operation fails. """ with errors.raise_exception_on_not_ok_status() as status: pywrap_tensorflow.RenameFile( - compat.as_bytes(oldname), compat.as_bytes(newname), overwrite, status) + compat.as_bytes(src), compat.as_bytes(dst), overwrite, status) def atomic_write_string_to_file(filename, contents, overwrite=True): @@ -439,35 +550,61 @@ def atomic_write_string_to_file(filename, contents, overwrite=True): raise -@tf_export("gfile.DeleteRecursively") +@tf_export(v1=["gfile.DeleteRecursively"]) def delete_recursively(dirname): """Deletes everything under dirname recursively. Args: dirname: string, a path to a directory + Raises: + errors.OpError: If the operation fails. + """ + delete_recursively_v2(dirname) + + +@tf_export("io.gfile.rmtree") +def delete_recursively_v2(path): + """Deletes everything under path recursively. + + Args: + path: string, a path + Raises: errors.OpError: If the operation fails. """ with errors.raise_exception_on_not_ok_status() as status: - pywrap_tensorflow.DeleteRecursively(compat.as_bytes(dirname), status) + pywrap_tensorflow.DeleteRecursively(compat.as_bytes(path), status) -@tf_export("gfile.IsDirectory") +@tf_export(v1=["gfile.IsDirectory"]) def is_directory(dirname): """Returns whether the path is a directory or not. Args: dirname: string, path to a potential directory + Returns: + True, if the path is a directory; False otherwise + """ + return is_directory_v2(dirname) + + +@tf_export("io.gfile.isdir") +def is_directory_v2(path): + """Returns whether the path is a directory or not. + + Args: + path: string, path to a potential directory + Returns: True, if the path is a directory; False otherwise """ status = c_api_util.ScopedTFStatus() - return pywrap_tensorflow.IsDirectory(compat.as_bytes(dirname), status) + return pywrap_tensorflow.IsDirectory(compat.as_bytes(path), status) -@tf_export("gfile.ListDirectory") +@tf_export(v1=["gfile.ListDirectory"]) def list_directory(dirname): """Returns a list of entries contained within a directory. @@ -483,7 +620,26 @@ def list_directory(dirname): Raises: errors.NotFoundError if directory doesn't exist """ - if not is_directory(dirname): + return list_directory_v2(dirname) + + +@tf_export("io.gfile.listdir") +def list_directory_v2(path): + """Returns a list of entries contained within a directory. + + The list is in arbitrary order. It does not contain the special entries "." + and "..". + + Args: + path: string, path to a directory + + Returns: + [filename1, filename2, ... filenameN] as strings + + Raises: + errors.NotFoundError if directory doesn't exist + """ + if not is_directory(path): raise errors.NotFoundError(None, None, "Could not find directory") with errors.raise_exception_on_not_ok_status() as status: # Convert each element to string, since the return values of the @@ -491,11 +647,11 @@ def list_directory(dirname): return [ compat.as_str_any(filename) for filename in pywrap_tensorflow.GetChildren( - compat.as_bytes(dirname), status) + compat.as_bytes(path), status) ] -@tf_export("gfile.Walk") +@tf_export(v1=["gfile.Walk"]) def walk(top, in_order=True): """Recursive directory tree generator for directories. @@ -505,6 +661,27 @@ def walk(top, in_order=True): Errors that happen while listing directories are ignored. + Yields: + Each yield is a 3-tuple: the pathname of a directory, followed by lists of + all its subdirectories and leaf files. + (dirname, [subdirname, subdirname, ...], [filename, filename, ...]) + as strings + """ + return walk_v2(top, in_order) + + +@tf_export("io.gfile.walk") +def walk_v2(top, topdown, onerror=None): + """Recursive directory tree generator for directories. + + Args: + top: string, a Directory name + topdown: bool, Traverse pre order if True, post order if False. + onerror: optional handler for errors. Should be a function, it will be + called with the error as argument. Rethrowing the error aborts the walk. + + Errors that happen while listing directories are ignored. + Yields: Each yield is a 3-tuple: the pathname of a directory, followed by lists of all its subdirectories and leaf files. @@ -514,8 +691,11 @@ def walk(top, in_order=True): top = compat.as_str_any(top) try: listing = list_directory(top) - except errors.NotFoundError: - return + except errors.NotFoundError as err: + if onerror: + onerror(err) + else: + return files = [] subdirs = [] @@ -528,18 +708,18 @@ def walk(top, in_order=True): here = (top, subdirs, files) - if in_order: + if topdown: yield here for subdir in subdirs: - for subitem in walk(os.path.join(top, subdir), in_order): + for subitem in walk_v2(os.path.join(top, subdir), topdown, onerror=onerror): yield subitem - if not in_order: + if not topdown: yield here -@tf_export("gfile.Stat") +@tf_export(v1=["gfile.Stat"]) def stat(filename): """Returns file statistics for a given path. @@ -549,12 +729,28 @@ def stat(filename): Returns: FileStatistics struct that contains information about the path + Raises: + errors.OpError: If the operation fails. + """ + return stat_v2(filename) + + +@tf_export("io.gfile.stat") +def stat_v2(path): + """Returns file statistics for a given path. + + Args: + path: string, path to a file + + Returns: + FileStatistics struct that contains information about the path + Raises: errors.OpError: If the operation fails. """ file_statistics = pywrap_tensorflow.FileStatistics() with errors.raise_exception_on_not_ok_status() as status: - pywrap_tensorflow.Stat(compat.as_bytes(filename), file_statistics, status) + pywrap_tensorflow.Stat(compat.as_bytes(path), file_statistics, status) return file_statistics diff --git a/tensorflow/python/lib/io/tf_record.py b/tensorflow/python/lib/io/tf_record.py index b7fae852955..43086ab18d7 100644 --- a/tensorflow/python/lib/io/tf_record.py +++ b/tensorflow/python/lib/io/tf_record.py @@ -150,10 +150,11 @@ class TFRecordOptions(object): return options -@tf_export( - "io.tf_record_iterator", - v1=["io.tf_record_iterator", "python_io.tf_record_iterator"]) -@deprecation.deprecated_endpoints("python_io.tf_record_iterator") +@tf_export(v1=["io.tf_record_iterator", "python_io.tf_record_iterator"]) +@deprecation.deprecated( + date=None, + instructions=("Use eager execution and: \n" + "`tf.data.TFRecordDataset(path)`")) def tf_record_iterator(path, options=None): """An iterator that read the records from a TFRecords file. diff --git a/tensorflow/python/ops/array_grad.py b/tensorflow/python/ops/array_grad.py index 68c392bf28d..45e741ef222 100644 --- a/tensorflow/python/ops/array_grad.py +++ b/tensorflow/python/ops/array_grad.py @@ -489,10 +489,12 @@ def _GatherNdGrad(op, grad): @ops.RegisterGradient("CheckNumerics") -def _CheckNumericsGrad(_, grad): +def _CheckNumericsGrad(op, grad): """Gradient for check_numerics op.""" return array_ops.check_numerics( - grad, "Not a number (NaN) or infinity (Inf) values detected in gradient.") + grad, + "Not a number (NaN) or infinity (Inf) values detected in gradient. %s" % + op.get_attr("message")) @ops.RegisterGradient("PlaceholderWithDefault") @@ -800,6 +802,32 @@ def _ScatterNdGrad(op, grad): return [None, updates_grad, None] +@ops.RegisterGradient("TensorScatterUpdate") +def _TensorScatterUpdateGrad(op, grad): + indices = op.inputs[1] + updates_grad = array_ops.gather_nd(grad, indices) + tensor_grad = array_ops.tensor_scatter_update( + array_ops.identity(grad), indices, + array_ops.zeros_like(op.inputs[2], dtype=grad.dtype)) + return [tensor_grad, None, updates_grad] + + +@ops.RegisterGradient("TensorScatterAdd") +def _TensorScatterAddGrad(op, grad): + indices = op.inputs[1] + updates_grad = array_ops.gather_nd(grad, indices) + tensor_grad = array_ops.identity(grad) + return [tensor_grad, None, updates_grad] + + +@ops.RegisterGradient("TensorScatterSub") +def _TensorScatterSubGrad(op, grad): + indices = op.inputs[1] + updates_grad = array_ops.gather_nd(grad, indices) + tensor_grad = array_ops.identity(grad) + return [tensor_grad, None, -updates_grad] + + @ops.RegisterGradient("ScatterNdNonAliasingAdd") def _ScatterNdNonAliasingAddGrad(op, grad): indices = op.inputs[1] diff --git a/tensorflow/python/ops/array_ops.py b/tensorflow/python/ops/array_ops.py index bbf7d166bf9..7b6242b1cd7 100644 --- a/tensorflow/python/ops/array_ops.py +++ b/tensorflow/python/ops/array_ops.py @@ -22,6 +22,7 @@ from __future__ import print_function import sys import numpy as np +import six from tensorflow.python.eager import context from tensorflow.python.framework import common_shapes @@ -40,6 +41,7 @@ from tensorflow.python.ops import gen_math_ops from tensorflow.python.ops.gen_array_ops import * from tensorflow.python.ops.gen_array_ops import reverse_v2 as reverse # pylint: disable=unused-import from tensorflow.python.util import deprecation +from tensorflow.python.util import dispatch from tensorflow.python.util import nest from tensorflow.python.util.tf_export import tf_export # pylint: enable=wildcard-import @@ -79,7 +81,7 @@ def identity(input, name=None): # pylint: disable=redefined-builtin # pylint: disable=redefined-builtin,protected-access -@tf_export("expand_dims") +@tf_export(v1=["expand_dims"]) @deprecation.deprecated_args(None, "Use the `axis` argument instead", "dim") def expand_dims(input, axis=None, name=None, dim=None): """Inserts a dimension of 1 into a tensor's shape. @@ -133,6 +135,55 @@ def expand_dims(input, axis=None, name=None, dim=None): axis = deprecation.deprecated_argument_lookup("axis", axis, "dim", dim) if axis is None: raise ValueError("Must specify an axis argument to tf.expand_dims()") + return expand_dims_v2(input, axis, name) + + +@tf_export("expand_dims", v1=[]) +def expand_dims_v2(input, axis, name=None): + """Inserts a dimension of 1 into a tensor's shape. + + Given a tensor `input`, this operation inserts a dimension of 1 at the + dimension index `axis` of `input`'s shape. The dimension index `axis` starts + at zero; if you specify a negative number for `axis` it is counted backward + from the end. + + This operation is useful if you want to add a batch dimension to a single + element. For example, if you have a single image of shape `[height, width, + channels]`, you can make it a batch of 1 image with `expand_dims(image, 0)`, + which will make the shape `[1, height, width, channels]`. + + Other examples: + + ```python + # 't' is a tensor of shape [2] + tf.shape(tf.expand_dims(t, 0)) # [1, 2] + tf.shape(tf.expand_dims(t, 1)) # [2, 1] + tf.shape(tf.expand_dims(t, -1)) # [2, 1] + + # 't2' is a tensor of shape [2, 3, 5] + tf.shape(tf.expand_dims(t2, 0)) # [1, 2, 3, 5] + tf.shape(tf.expand_dims(t2, 2)) # [2, 3, 1, 5] + tf.shape(tf.expand_dims(t2, 3)) # [2, 3, 5, 1] + ``` + + This operation requires that: + + `-1-input.dims() <= dim <= input.dims()` + + This operation is related to `squeeze()`, which removes dimensions of + size 1. + + Args: + input: A `Tensor`. + axis: 0-D (scalar). Specifies the dimension index at which to + expand the shape of `input`. Must be in the range + `[-rank(input) - 1, rank(input)]`. + name: The name of the output `Tensor` (optional). + + Returns: + A `Tensor` with the same data as `input`, but its shape has an additional + dimension of size 1 added. + """ return gen_array_ops.expand_dims(input, axis, name) @@ -219,7 +270,13 @@ def broadcast_static_shape(shape_x, shape_y): return common_shapes.broadcast_shape(shape_x, shape_y) -@tf_export("shape") +@tf_export("shape", v1=[]) +def shape_v2(input, out_type=dtypes.int32, name=None): + # pylint: disable=redefined-builtin + return shape(input, name, out_type) + + +@tf_export(v1=["shape"]) def shape(input, name=None, out_type=dtypes.int32): # pylint: disable=redefined-builtin """Returns the shape of a tensor. @@ -292,7 +349,13 @@ def shape_n(input, out_type=dtypes.int32, name=None): return gen_array_ops.shape_n(input, out_type=out_type, name=name) -@tf_export("size") +@tf_export("size", v1=[]) +def size_v2(input, out_type=dtypes.int32, name=None): + # pylint: disable=redefined-builtin + return size(input, name, out_type) + + +@tf_export(v1=["size"]) def size(input, name=None, out_type=dtypes.int32): # pylint: disable=redefined-builtin """Returns the size of a tensor. @@ -341,7 +404,7 @@ def size_internal(input, name=None, optimize=True, out_type=dtypes.int32): input, (sparse_tensor.SparseTensor, sparse_tensor.SparseTensorValue)): input = ops.convert_to_tensor(input) np_out_type = out_type.as_numpy_dtype - num_elements = np.prod(input._shape_tuple(), dtype=np_out_type) # pylint: disable=protected-acces: + num_elements = np.prod(input._shape_tuple(), dtype=np_out_type) # pylint: disable=protected-access return ops.convert_to_tensor(num_elements, dtype=out_type) with ops.name_scope(name, "Size", [input]) as name: if isinstance(input, (sparse_tensor.SparseTensor, @@ -431,7 +494,7 @@ _SUPPORTED_SLICE_DTYPES = ( def _check_index(idx): """Check if a given value is a valid index into a tensor.""" - if isinstance(idx, (int, tensor_shape.Dimension)): + if isinstance(idx, (six.integer_types, tensor_shape.Dimension)): return # Optimistic check. Assumptions: @@ -1445,7 +1508,75 @@ def split(value, num_or_size_splits, axis=0, num=None, name="split"): value=value, size_splits=size_splits, axis=axis, num_split=num, name=name) -@tf_export("transpose") +@tf_export("transpose", v1=[]) +def transpose_v2(a, perm=None, conjugate=False, name="transpose"): + """Transposes `a`. Permutes the dimensions according to `perm`. + + The returned tensor's dimension i will correspond to the input dimension + `perm[i]`. If `perm` is not given, it is set to (n-1...0), where n is + the rank of the input tensor. Hence by default, this operation performs a + regular matrix transpose on 2-D input Tensors. If conjugate is True and + `a.dtype` is either `complex64` or `complex128` then the values of `a` + are conjugated and transposed. + + @compatibility(numpy) + In `numpy` transposes are memory-efficient constant time operations as they + simply return a new view of the same data with adjusted `strides`. + + TensorFlow does not support strides, so `transpose` returns a new tensor with + the items permuted. + @end_compatibility + + For example: + + ```python + x = tf.constant([[1, 2, 3], [4, 5, 6]]) + tf.transpose(x) # [[1, 4] + # [2, 5] + # [3, 6]] + + # Equivalently + tf.transpose(x, perm=[1, 0]) # [[1, 4] + # [2, 5] + # [3, 6]] + + # If x is complex, setting conjugate=True gives the conjugate transpose + x = tf.constant([[1 + 1j, 2 + 2j, 3 + 3j], + [4 + 4j, 5 + 5j, 6 + 6j]]) + tf.transpose(x, conjugate=True) # [[1 - 1j, 4 - 4j], + # [2 - 2j, 5 - 5j], + # [3 - 3j, 6 - 6j]] + + # 'perm' is more useful for n-dimensional tensors, for n > 2 + x = tf.constant([[[ 1, 2, 3], + [ 4, 5, 6]], + [[ 7, 8, 9], + [10, 11, 12]]]) + + # Take the transpose of the matrices in dimension-0 + # (this common operation has a shorthand `linalg.transpose`) + tf.transpose(x, perm=[0, 2, 1]) # [[[1, 4], + # [2, 5], + # [3, 6]], + # [[7, 10], + # [8, 11], + # [9, 12]]] + ``` + + Args: + a: A `Tensor`. + perm: A permutation of the dimensions of `a`. + conjugate: Optional bool. Setting it to `True` is mathematically equivalent + to tf.conj(tf.transpose(input)). + name: A name for the operation (optional). + + Returns: + A transposed `Tensor`. + """ + return transpose(a=a, perm=perm, name=name, conjugate=conjugate) + + +@tf_export(v1=["transpose"]) def transpose(a, perm=None, name="transpose", conjugate=False): """Transposes `a`. Permutes the dimensions according to `perm`. @@ -1678,7 +1809,7 @@ def zeros(shape, dtype=dtypes.float32, name=None): return output -@tf_export("zeros_like") +@tf_export(v1=["zeros_like"]) def zeros_like(tensor, dtype=None, name=None, optimize=True): """Creates a tensor with all elements set to zero. @@ -1705,6 +1836,42 @@ def zeros_like(tensor, dtype=None, name=None, optimize=True): Returns: A `Tensor` with all elements set to zero. """ + return zeros_like_impl(tensor, dtype, name, optimize) + + +@tf_export("zeros_like", v1=[]) +def zeros_like_v2( + input, # pylint: disable=redefined-builtin + dtype=None, + name=None): + """Creates a tensor with all elements set to zero. + + Given a single tensor (`tensor`), this operation returns a tensor of the + same type and shape as `tensor` with all elements set to zero. Optionally, + you can use `dtype` to specify a new type for the returned tensor. + + For example: + + ```python + tensor = tf.constant([[1, 2, 3], [4, 5, 6]]) + tf.zeros_like(tensor) # [[0, 0, 0], [0, 0, 0]] + ``` + + Args: + input: A `Tensor`. + dtype: A type for the returned `Tensor`. Must be `float16`, `float32`, + `float64`, `int8`, `uint8`, `int16`, `uint16`, `int32`, `int64`, + `complex64`, `complex128`, `bool` or `string`. + name: A name for the operation (optional). + + Returns: + A `Tensor` with all elements set to zero. + """ + return zeros_like_impl(input, dtype, name, optimize=True) + + +def zeros_like_impl(tensor, dtype, name, optimize=True): + """Internal implementation for the v1/v2 zeros_like API calls.""" with ops.name_scope(name, "zeros_like", [tensor]) as name: tensor = ops.convert_to_tensor(tensor, name="tensor") @@ -1731,7 +1898,7 @@ def zeros_like(tensor, dtype=None, name=None, optimize=True): return gen_array_ops.zeros_like(tensor, name=name) -@tf_export("ones_like") +@tf_export(v1=["ones_like"]) def ones_like(tensor, dtype=None, name=None, optimize=True): """Creates a tensor with all elements set to 1. @@ -1758,6 +1925,42 @@ def ones_like(tensor, dtype=None, name=None, optimize=True): Returns: A `Tensor` with all elements set to 1. """ + return ones_like_impl(tensor, dtype, name, optimize) + + +@tf_export("ones_like", v1=[]) +def ones_like_v2( + input, # pylint: disable=redefined-builtin + dtype=None, + name=None): + """Creates a tensor with all elements set to zero. + + Given a single tensor (`tensor`), this operation returns a tensor of the + same type and shape as `tensor` with all elements set to zero. Optionally, + you can use `dtype` to specify a new type for the returned tensor. + + For example: + + ```python + tensor = tf.constant([[1, 2, 3], [4, 5, 6]]) + tf.ones_like(tensor) # [[1, 1, 1], [1, 1, 1]] + ``` + + Args: + input: A `Tensor`. + dtype: A type for the returned `Tensor`. Must be `float16`, `float32`, + `float64`, `int8`, `uint8`, `int16`, `uint16`, `int32`, `int64`, + `complex64`, `complex128`, `bool` or `string`. + name: A name for the operation (optional). + + Returns: + A `Tensor` with all elements set to zero. + """ + return ones_like_impl(input, dtype, name, optimize=True) + + +def ones_like_impl(tensor, dtype, name, optimize=True): + """Internal implementation for the v1/v2 ones_like API calls.""" with ops.name_scope(name, "ones_like", [tensor]) as name: tensor = ops.convert_to_tensor(tensor, name="tensor") ones_shape = shape_internal(tensor, optimize=optimize) @@ -1955,7 +2158,65 @@ def sparse_placeholder(dtype, shape=None, name=None): # pylint: enable=redefined-outer-name -@tf_export("pad") +@tf_export("pad", v1=[]) +def pad_v2(tensor, paddings, mode="CONSTANT", constant_values=0, name=None): + """Pads a tensor. + + This operation pads a `tensor` according to the `paddings` you specify. + `paddings` is an integer tensor with shape `[n, 2]`, where n is the rank of + `tensor`. For each dimension D of `input`, `paddings[D, 0]` indicates how + many values to add before the contents of `tensor` in that dimension, and + `paddings[D, 1]` indicates how many values to add after the contents of + `tensor` in that dimension. If `mode` is "REFLECT" then both `paddings[D, 0]` + and `paddings[D, 1]` must be no greater than `tensor.dim_size(D) - 1`. If + `mode` is "SYMMETRIC" then both `paddings[D, 0]` and `paddings[D, 1]` must be + no greater than `tensor.dim_size(D)`. + + The padded size of each dimension D of the output is: + + `paddings[D, 0] + tensor.dim_size(D) + paddings[D, 1]` + + For example: + + ```python + t = tf.constant([[1, 2, 3], [4, 5, 6]]) + paddings = tf.constant([[1, 1,], [2, 2]]) + # 'constant_values' is 0. + # rank of 't' is 2. + tf.pad(t, paddings, "CONSTANT") # [[0, 0, 0, 0, 0, 0, 0], + # [0, 0, 1, 2, 3, 0, 0], + # [0, 0, 4, 5, 6, 0, 0], + # [0, 0, 0, 0, 0, 0, 0]] + + tf.pad(t, paddings, "REFLECT") # [[6, 5, 4, 5, 6, 5, 4], + # [3, 2, 1, 2, 3, 2, 1], + # [6, 5, 4, 5, 6, 5, 4], + # [3, 2, 1, 2, 3, 2, 1]] + + tf.pad(t, paddings, "SYMMETRIC") # [[2, 1, 1, 2, 3, 3, 2], + # [2, 1, 1, 2, 3, 3, 2], + # [5, 4, 4, 5, 6, 6, 5], + # [5, 4, 4, 5, 6, 6, 5]] + ``` + + Args: + tensor: A `Tensor`. + paddings: A `Tensor` of type `int32`. + mode: One of "CONSTANT", "REFLECT", or "SYMMETRIC" (case-insensitive) + constant_values: In "CONSTANT" mode, the scalar pad value to use. Must be + same type as `tensor`. + name: A name for the operation (optional). + + Returns: + A `Tensor`. Has the same type as `tensor`. + + Raises: + ValueError: When mode is not one of "CONSTANT", "REFLECT", or "SYMMETRIC". + """ + return pad(tensor, paddings, mode, name, constant_values) + + +@tf_export(v1=["pad"]) def pad(tensor, paddings, mode="CONSTANT", name=None, constant_values=0): # pylint: disable=invalid-name """Pads a tensor. @@ -2415,7 +2676,7 @@ def depth_to_space(input, block_size, name=None, data_format="NHWC"): # pylint: depth_to_space.__doc__ = gen_array_ops.depth_to_space.__doc__ -@tf_export("batch_to_space") +@tf_export(v1=["batch_to_space"]) def batch_to_space(input, crops, block_size, name=None): # pylint: disable=redefined-builtin result = batch_to_space_nd( input, @@ -2429,6 +2690,151 @@ def batch_to_space(input, crops, block_size, name=None): # pylint: disable=rede batch_to_space.__doc__ = gen_array_ops.batch_to_space.__doc__ +@tf_export("batch_to_space", v1=[]) +def batch_to_space_v2(input, block_shape, crops, name=None): # pylint: disable=redefined-builtin + """BatchToSpace for N-D tensors of type T. + + This operation reshapes the "batch" dimension 0 into `M + 1` dimensions of + shape `block_shape + [batch]`, interleaves these blocks back into the grid + defined by the spatial dimensions `[1, ..., M]`, to obtain a result with the + same rank as the input. The spatial dimensions of this intermediate result + are then optionally cropped according to `crops` to produce the output. This + is the reverse of SpaceToBatch. See below for a precise description. + + Args: + input: A `Tensor`. + N-D with shape `input_shape = [batch] + spatial_shape + remaining_shape`, + where spatial_shape has M dimensions. + block_shape: A `Tensor`. Must be one of the following types: + `int32`, `int64`. 1-D with shape `[M]`, all values must be >= 1. + For backwards compatibility with TF 1.0, this parameter may be an int, in + which case it is converted to + `numpy.array([block_shape, block_shape], dtype=numpy.int64)`. + crops: A `Tensor`. Must be one of the following types: `int32`, `int64`. + 2-D with shape `[M, 2]`, all values must be >= 0. + `crops[i] = [crop_start, crop_end]` specifies the amount to crop from + input dimension `i + 1`, which corresponds to spatial dimension `i`. It + is required that + `crop_start[i] + crop_end[i] <= block_shape[i] * input_shape[i + 1]`. + + This operation is equivalent to the following steps: + + 1. Reshape `input` to `reshaped` of shape: + [block_shape[0], ..., block_shape[M-1], + batch / prod(block_shape), + input_shape[1], ..., input_shape[N-1]] + + 2. Permute dimensions of `reshaped` to produce `permuted` of shape + [batch / prod(block_shape), + + input_shape[1], block_shape[0], + ..., + input_shape[M], block_shape[M-1], + + input_shape[M+1], ..., input_shape[N-1]] + + 3. Reshape `permuted` to produce `reshaped_permuted` of shape + [batch / prod(block_shape), + + input_shape[1] * block_shape[0], + ..., + input_shape[M] * block_shape[M-1], + + input_shape[M+1], + ..., + input_shape[N-1]] + + 4. Crop the start and end of dimensions `[1, ..., M]` of + `reshaped_permuted` according to `crops` to produce the + output of shape: + [batch / prod(block_shape), + + input_shape[1] * block_shape[0] - crops[0,0] - crops[0,1], + ..., + input_shape[M] * block_shape[M-1] - crops[M-1,0] - crops[M-1,1], + + input_shape[M+1], ..., input_shape[N-1]] + + Some examples: + + (1) For the following input of shape `[4, 1, 1, 1]`, + `block_shape = [2, 2]`, and `crops = [[0, 0], [0, 0]]`: + + ``` + [[[[1]]], [[[2]]], [[[3]]], [[[4]]]] + ``` + + The output tensor has shape `[1, 2, 2, 1]` and value: + + ``` + x = [[[[1], [2]], [[3], [4]]]] + ``` + + (2) For the following input of shape `[4, 1, 1, 3]`, + `block_shape = [2, 2]`, and `crops = [[0, 0], [0, 0]]`: + + ``` + [[[1, 2, 3]], [[4, 5, 6]], [[7, 8, 9]], [[10, 11, 12]]] + ``` + + The output tensor has shape `[1, 2, 2, 3]` and value: + + ``` + x = [[[[1, 2, 3], [4, 5, 6]], + [[7, 8, 9], [10, 11, 12]]]] + ``` + + (3) For the following input of shape `[4, 2, 2, 1]`, + `block_shape = [2, 2]`, and `crops = [[0, 0], [0, 0]]`: + + ``` + x = [[[[1], [3]], [[9], [11]]], + [[[2], [4]], [[10], [12]]], + [[[5], [7]], [[13], [15]]], + [[[6], [8]], [[14], [16]]]] + ``` + + The output tensor has shape `[1, 4, 4, 1]` and value: + + ``` + x = [[[1], [2], [3], [4]], + [[5], [6], [7], [8]], + [[9], [10], [11], [12]], + [[13], [14], [15], [16]]] + ``` + + (4) For the following input of shape `[8, 1, 3, 1]`, + `block_shape = [2, 2]`, and `crops = [[0, 0], [2, 0]]`: + + ``` + x = [[[[0], [1], [3]]], [[[0], [9], [11]]], + [[[0], [2], [4]]], [[[0], [10], [12]]], + [[[0], [5], [7]]], [[[0], [13], [15]]], + [[[0], [6], [8]]], [[[0], [14], [16]]]] + ``` + + The output tensor has shape `[2, 2, 4, 1]` and value: + + ``` + x = [[[[1], [2], [3], [4]], + [[5], [6], [7], [8]]], + [[[9], [10], [11], [12]], + [[13], [14], [15], [16]]]] + ``` + name: A name for the operation (optional). + + Returns: + A `Tensor`. Has the same type as `input`. + """ + if isinstance(block_shape, int): + block_shape = np.array([block_shape, block_shape], dtype=np.int64) + + return batch_to_space_nd(input=input, + block_shape=block_shape, + crops=crops, + name=name) + + @tf_export("one_hot") def one_hot(indices, depth, @@ -2652,7 +3058,7 @@ def sequence_mask(lengths, maxlen=None, dtype=dtypes.bool, name=None): return gen_math_ops.cast(result, dtype) -@tf_export("squeeze") +@tf_export(v1=["squeeze"]) @deprecation.deprecated_args(None, "Use the `axis` argument instead", "squeeze_dims") def squeeze(input, axis=None, name=None, squeeze_dims=None): @@ -2702,6 +3108,12 @@ def squeeze(input, axis=None, name=None, squeeze_dims=None): return gen_array_ops.squeeze(input, axis, name) +@tf_export("squeeze", v1=[]) +def squeeze_v2(input, axis=None, name=None): + # pylint: disable=redefined-builtin + return squeeze(input, axis, name) + + @tf_export("where") def where(condition, x=None, y=None, name=None): """Return the elements, either from `x` or `y`, depending on the `condition`. @@ -2756,7 +3168,7 @@ def where(condition, x=None, y=None, name=None): # pylint: disable=redefined-builtin -@tf_export("reverse_sequence") +@tf_export(v1=["reverse_sequence"]) @deprecation.deprecated_args( None, "seq_dim is deprecated, use seq_axis instead", "seq_dim") @deprecation.deprecated_args( @@ -2780,15 +3192,32 @@ def reverse_sequence(input, name=name) -# pylint: enable=redefined-builtin - reverse_sequence.__doc__ = deprecation.rewrite_argument_docstring( deprecation.rewrite_argument_docstring( gen_array_ops.reverse_sequence.__doc__, "batch_dim", "batch_axis"), "seq_dim", "seq_axis") -@tf_export("gather") +@tf_export("reverse_sequence", v1=[]) +def reverse_sequence_v2( + input, seq_lengths, seq_axis=None, batch_axis=None, name=None): + return gen_array_ops.reverse_sequence( + input=input, + seq_lengths=seq_lengths, + seq_dim=seq_axis, + batch_dim=batch_axis, + name=name) + + +reverse_sequence_v2.__doc__ = deprecation.rewrite_argument_docstring( + deprecation.rewrite_argument_docstring( + gen_array_ops.reverse_sequence.__doc__, "batch_dim", "batch_axis"), + "seq_dim", "seq_axis") + +# pylint: enable=redefined-builtin + + +@tf_export(v1=["gather"]) def gather(params, indices, validate_indices=None, name=None, axis=0): del validate_indices if axis != 0: @@ -2804,10 +3233,18 @@ def gather(params, indices, validate_indices=None, name=None, axis=0): return gen_array_ops.gather_v2(params, indices, axis, name=name) -gather.__doc__ = gen_array_ops.gather_v2.__doc__ +@tf_export("gather", v1=[]) +def gather_v2(params, indices, validate_indices=None, axis=0, name=None): + return gather(params, indices, validate_indices=validate_indices, name=name, + axis=axis) + + +gather.__doc__ = gather_v2.__doc__ = gen_array_ops.gather_v2.__doc__ + @tf_export("batch_gather") +@dispatch.add_dispatch_support def batch_gather(params, indices, name=None): """Gather slices from `params` according to `indices` with leading batch dims. @@ -2885,7 +3322,7 @@ def batch_gather(params, indices, name=None): # Define quantize_v2 here in order to make name the second-to-last attribute, # because round_mode was added later. -@tf_export("quantize_v2") +@tf_export(v1=["quantize_v2"]) @deprecation.deprecated( "2017-10-25", "`tf.quantize_v2` is deprecated, please use `tf.quantization.quantize` " @@ -2906,7 +3343,7 @@ def quantize_v2(input, # pylint: disable=redefined-builtin round_mode=round_mode) -quantize_v2.__doc__ = """Please use `tf.quantize` instead.""" +quantize_v2.__doc__ = """Please use `tf.quantization.quantize` instead.""" # We want to expose tf.quantize instead of tf.quantize_v2; we can deprecate @@ -2992,3 +3429,48 @@ def searchsorted(sorted_sequence, quantize.__doc__ = gen_array_ops.quantize_v2.__doc__ + + +@tf_export("image.extract_image_patches", v1=[]) +def extract_image_patches_v2( + images, + sizes, + strides, + rates, + padding, + name=None): + # pylint: disable=line-too-long + r"""Extract `patches` from `images` and put them in the \"depth\" output dimension. + + Args: + images: A 4-D Tensor with shape `[batch, in_rows, in_cols, depth] + sizes: The size of the sliding window for each dimension of `images`. + strides: A 1-D Tensor of length 4. How far the centers of two consecutive + patches are in the images. Must be: `[1, stride_rows, stride_cols, 1]`. + rates: A 1-D Tensor of length 4. Must be: `[1, rate_rows, rate_cols, 1]`. + This is the input stride, specifying how far two consecutive patch samples + are in the input. Equivalent to extracting patches with `patch_sizes_eff = + patch_sizes + (patch_sizes - 1) * (rates - 1)`, followed by subsampling + them spatially by a factor of `rates`. This is equivalent to `rate` in + dilated (a.k.a. Atrous) convolutions. + padding: The type of padding algorithm to use. + We specify the size-related attributes as: ```python ksizes = [1, + ksize_rows, ksize_cols, 1] strides = [1, strides_rows, strides_cols, 1] + rates = [1, rates_rows, rates_cols, 1] + name: A name for the operation (optional). + + Returns: + A 4-D Tensor. Has the same type as `images`, and with shape `[batch, + out_rows, out_cols, ksize_rows * ksize_cols * depth]` containing image + patches with size `ksize_rows x ksize_cols x depth` vectorized in the + \"depth\" dimension. Note `out_rows` and `out_cols` are the dimensions of + the output patches. + """ + # pylint: enable=line-too-long + return gen_array_ops.extract_image_patches( + images, sizes, strides, rates, padding, name) + +extract_image_patches_deprecation = deprecation.deprecated_args( + None, "ksizes is deprecated, use sizes instead", "ksizes") +tf_export(v1=["image.extract_image_patches", "extract_image_patches"])( + extract_image_patches_deprecation(gen_array_ops.extract_image_patches)) diff --git a/tensorflow/python/ops/bitwise_ops_test.py b/tensorflow/python/ops/bitwise_ops_test.py index dfb40db2d5a..d154b6759bf 100644 --- a/tensorflow/python/ops/bitwise_ops_test.py +++ b/tensorflow/python/ops/bitwise_ops_test.py @@ -34,6 +34,7 @@ class BitwiseOpTest(test_util.TensorFlowTestCase): def __init__(self, method_name="runTest"): super(BitwiseOpTest, self).__init__(method_name) + @test_util.run_deprecated_v1 def testBinaryOps(self): dtype_list = [dtypes.int8, dtypes.int16, dtypes.int32, dtypes.int64, dtypes.uint8, dtypes.uint16, dtypes.uint32, dtypes.uint64] @@ -59,16 +60,18 @@ class BitwiseOpTest(test_util.TensorFlowTestCase): 2**31 - 1, 2**31, 2**32 - 1, 2**32, -2**32 + 1, -2**32, -2**63 + 1, 2**63 - 1] def count_bits(x): - return sum([bin(z).count("1") for z in six.iterbytes(x.tobytes())]) + return sum(bin(z).count("1") for z in six.iterbytes(x.tobytes())) for dtype in dtype_list: with self.cached_session(use_gpu=True) as sess: print("PopulationCount test: ", dtype) inputs = np.array(raw_inputs, dtype=dtype.as_numpy_dtype) truth = [count_bits(x) for x in inputs] input_tensor = constant_op.constant(inputs, dtype=dtype) - popcnt_result = sess.run(gen_bitwise_ops.population_count(input_tensor)) + popcnt_result = self.evaluate( + gen_bitwise_ops.population_count(input_tensor)) self.assertAllEqual(truth, popcnt_result) + @test_util.run_deprecated_v1 def testInvertOp(self): dtype_list = [dtypes.int8, dtypes.int16, dtypes.int32, dtypes.int64, dtypes.uint8, dtypes.uint16, dtypes.uint32, dtypes.uint64] @@ -89,10 +92,11 @@ class BitwiseOpTest(test_util.TensorFlowTestCase): self.assertAllEqual(not_a_or_a, [not_0] * 4) # For unsigned dtypes let's also check the result directly. if dtype.is_unsigned: - inverted = sess.run(bitwise_ops.invert(input_tensor)) + inverted = self.evaluate(bitwise_ops.invert(input_tensor)) expected = [dtype.max - x for x in inputs] self.assertAllEqual(inverted, expected) + @test_util.run_deprecated_v1 def testShiftsWithPositiveLHS(self): dtype_list = [np.int8, np.int16, np.int32, np.int64, np.uint8, np.uint16, np.uint32, np.uint64] @@ -107,6 +111,7 @@ class BitwiseOpTest(test_util.TensorFlowTestCase): self.assertAllEqual(left_shift_result, np.left_shift(lhs, rhs)) self.assertAllEqual(right_shift_result, np.right_shift(lhs, rhs)) + @test_util.run_deprecated_v1 def testShiftsWithNegativeLHS(self): dtype_list = [np.int8, np.int16, np.int32, np.int64] @@ -120,6 +125,7 @@ class BitwiseOpTest(test_util.TensorFlowTestCase): self.assertAllEqual(left_shift_result, np.left_shift(lhs, rhs)) self.assertAllEqual(right_shift_result, np.right_shift(lhs, rhs)) + @test_util.run_deprecated_v1 def testImplementationDefinedShiftsDoNotCrash(self): dtype_list = [np.int8, np.int16, np.int32, np.int64] @@ -135,6 +141,7 @@ class BitwiseOpTest(test_util.TensorFlowTestCase): bitwise_ops.right_shift(lhs, rhs)]) + @test_util.run_deprecated_v1 def testShapeInference(self): dtype_list = [dtypes.int8, dtypes.int16, dtypes.int32, dtypes.int64, dtypes.uint8, dtypes.uint16] diff --git a/tensorflow/python/ops/candidate_sampling_ops.py b/tensorflow/python/ops/candidate_sampling_ops.py index f0bfdb2b7a3..c64000b65d4 100644 --- a/tensorflow/python/ops/candidate_sampling_ops.py +++ b/tensorflow/python/ops/candidate_sampling_ops.py @@ -208,7 +208,9 @@ def learned_unigram_candidate_sampler(true_classes, num_true, num_sampled, seed2=seed2, name=name) -@tf_export('nn.fixed_unigram_candidate_sampler') +@tf_export('random.fixed_unigram_candidate_sampler', + 'nn.fixed_unigram_candidate_sampler', + v1=['nn.fixed_unigram_candidate_sampler']) def fixed_unigram_candidate_sampler(true_classes, num_true, num_sampled, @@ -300,7 +302,8 @@ def fixed_unigram_candidate_sampler(true_classes, unigrams=unigrams, seed=seed1, seed2=seed2, name=name) -@tf_export('nn.all_candidate_sampler') +@tf_export('random.all_candidate_sampler', 'nn.all_candidate_sampler', + v1=['nn.all_candidate_sampler']) def all_candidate_sampler(true_classes, num_true, num_sampled, unique, seed=None, name=None): """Generate the set of all classes. diff --git a/tensorflow/python/ops/check_ops.py b/tensorflow/python/ops/check_ops.py index 5589bbc8485..f1f36269cf2 100644 --- a/tensorflow/python/ops/check_ops.py +++ b/tensorflow/python/ops/check_ops.py @@ -119,9 +119,31 @@ def assert_proper_iterable(values): 'Expected argument "values" to be iterable. Found: %s' % type(values)) -@tf_export( - 'debugging.assert_negative', - v1=['debugging.assert_negative', 'assert_negative']) +@tf_export('debugging.assert_negative', v1=[]) +def assert_negative_v2(x, message=None, summarize=None, name=None): + """Assert the condition `x < 0` holds element-wise. + + This Op checks that `x[i] < 0` holds for every element of `x`. If `x` is + empty, this is trivially satisfied. + + If `x` is not negative everywhere, `message`, as well as the first `summarize` + entries of `x` are printed, and `InvalidArgumentError` is raised. + + Args: + x: Numeric `Tensor`. + message: A string to prefix to the default message. + summarize: Print this many entries of each tensor. + name: A name for this operation (optional). Defaults to "assert_negative". + + Raises: + InvalidArgumentError: if the check can be performed immediately and + `x[i] < 0` is False. The check can be performed immediately during eager + execution or if `x` is statically known. + """ + assert_negative(x=x, message=message, summarize=summarize, name=name) + + +@tf_export(v1=['debugging.assert_negative', 'assert_negative']) @deprecation.deprecated_endpoints('assert_negative') def assert_negative(x, data=None, summarize=None, message=None, name=None): """Assert the condition `x < 0` holds element-wise. @@ -163,9 +185,31 @@ def assert_negative(x, data=None, summarize=None, message=None, name=None): return assert_less(x, zero, data=data, summarize=summarize) -@tf_export( - 'debugging.assert_positive', - v1=['debugging.assert_positive', 'assert_positive']) +@tf_export('debugging.assert_positive', v1=[]) +def assert_positive_v2(x, message=None, summarize=None, name=None): + """Assert the condition `x > 0` holds element-wise. + + This Op checks that `x[i] > 0` holds for every element of `x`. If `x` is + empty, this is trivially satisfied. + + If `x` is not positive everywhere, `message`, as well as the first `summarize` + entries of `x` are printed, and `InvalidArgumentError` is raised. + + Args: + x: Numeric `Tensor`. + message: A string to prefix to the default message. + summarize: Print this many entries of each tensor. + name: A name for this operation (optional). Defaults to "assert_positive". + + Raises: + InvalidArgumentError: if the check can be performed immediately and + `x[i] > 0` is False. The check can be performed immediately during eager + execution or if `x` is statically known. + """ + assert_positive(x=x, summarize=summarize, message=message, name=name) + + +@tf_export(v1=['debugging.assert_positive', 'assert_positive']) @deprecation.deprecated_endpoints('assert_positive') def assert_positive(x, data=None, summarize=None, message=None, name=None): """Assert the condition `x > 0` holds element-wise. @@ -206,9 +250,32 @@ def assert_positive(x, data=None, summarize=None, message=None, name=None): return assert_less(zero, x, data=data, summarize=summarize) -@tf_export( - 'debugging.assert_non_negative', - v1=['debugging.assert_non_negative', 'assert_non_negative']) +@tf_export('debugging.assert_non_negative', v1=[]) +def assert_non_negative_v2(x, message=None, summarize=None, name=None): + """Assert the condition `x >= 0` holds element-wise. + + This Op checks that `x[i] >= 0` holds for every element of `x`. If `x` is + empty, this is trivially satisfied. + + If `x` is not >= 0 everywhere, `message`, as well as the first `summarize` + entries of `x` are printed, and `InvalidArgumentError` is raised. + + Args: + x: Numeric `Tensor`. + message: A string to prefix to the default message. + summarize: Print this many entries of each tensor. + name: A name for this operation (optional). Defaults to + "assert_non_negative". + + Raises: + InvalidArgumentError: if the check can be performed immediately and + `x[i] >= 0` is False. The check can be performed immediately during eager + execution or if `x` is statically known. + """ + assert_non_negative(x=x, summarize=summarize, message=message, name=name) + + +@tf_export(v1=['debugging.assert_non_negative', 'assert_non_negative']) @deprecation.deprecated_endpoints('assert_non_negative') def assert_non_negative(x, data=None, summarize=None, message=None, name=None): """Assert the condition `x >= 0` holds element-wise. @@ -251,9 +318,32 @@ def assert_non_negative(x, data=None, summarize=None, message=None, name=None): return assert_less_equal(zero, x, data=data, summarize=summarize) -@tf_export( - 'debugging.assert_non_positive', - v1=['debugging.assert_non_positive', 'assert_non_positive']) +@tf_export('debugging.assert_non_positive', v1=[]) +def assert_non_positive_v2(x, message=None, summarize=None, name=None): + """Assert the condition `x <= 0` holds element-wise. + + This Op checks that `x[i] <= 0` holds for every element of `x`. If `x` is + empty, this is trivially satisfied. + + If `x` is not <= 0 everywhere, `message`, as well as the first `summarize` + entries of `x` are printed, and `InvalidArgumentError` is raised. + + Args: + x: Numeric `Tensor`. + message: A string to prefix to the default message. + summarize: Print this many entries of each tensor. + name: A name for this operation (optional). Defaults to + "assert_non_positive". + + Raises: + InvalidArgumentError: if the check can be performed immediately and + `x[i] <= 0` is False. The check can be performed immediately during eager + execution or if `x` is statically known. + """ + assert_non_positive(x=x, summarize=summarize, message=message, name=name) + + +@tf_export(v1=['debugging.assert_non_positive', 'assert_non_positive']) @deprecation.deprecated_endpoints('assert_non_positive') def assert_non_positive(x, data=None, summarize=None, message=None, name=None): """Assert the condition `x <= 0` holds element-wise. @@ -296,7 +386,33 @@ def assert_non_positive(x, data=None, summarize=None, message=None, name=None): return assert_less_equal(x, zero, data=data, summarize=summarize) -@tf_export('debugging.assert_equal', 'assert_equal') +@tf_export('debugging.assert_equal', 'assert_equal', v1=[]) +def assert_equal_v2(x, y, message=None, summarize=None, name=None): + """Assert the condition `x == y` holds element-wise. + + This Op checks that `x[i] == y[i]` holds for every pair of (possibly + broadcast) elements of `x` and `y`. If both `x` and `y` are empty, this is + trivially satisfied. + + If `x` and `y` are not equal, `message`, as well as the first `summarize` + entries of `x` and `y` are printed, and `InvalidArgumentError` is raised. + + Args: + x: Numeric `Tensor`. + y: Numeric `Tensor`, same dtype as and broadcastable to `x`. + message: A string to prefix to the default message. + summarize: Print this many entries of each tensor. + name: A name for this operation (optional). Defaults to "assert_equal". + + Raises: + InvalidArgumentError: if the check can be performed immediately and + `x == y` is False. The check can be performed immediately during eager + execution or if `x` and `y` are statically known. + """ + assert_equal(x=x, y=y, summarize=summarize, message=message, name=name) + + +@tf_export(v1=['debugging.assert_equal', 'assert_equal']) def assert_equal(x, y, data=None, summarize=None, message=None, name=None): """Assert the condition `x == y` holds element-wise. @@ -396,9 +512,36 @@ def assert_equal(x, y, data=None, summarize=None, message=None, name=None): return control_flow_ops.Assert(condition, data, summarize=summarize) -@tf_export( - 'debugging.assert_none_equal', - v1=['debugging.assert_none_equal', 'assert_none_equal']) +@tf_export('debugging.assert_none_equal', v1=[]) +def assert_none_equal_v2(x, y, summarize=None, message=None, name=None): + """Assert the condition `x != y` holds for all elements. + + This Op checks that `x[i] != y[i]` holds for every pair of (possibly + broadcast) elements of `x` and `y`. If both `x` and `y` are empty, this is + trivially satisfied. + + If any elements of `x` and `y` are equal, `message`, as well as the first + `summarize` entries of `x` and `y` are printed, and `InvalidArgumentError` + is raised. + + Args: + x: Numeric `Tensor`. + y: Numeric `Tensor`, same dtype as and broadcastable to `x`. + summarize: Print this many entries of each tensor. + message: A string to prefix to the default message. + name: A name for this operation (optional). Defaults to + "assert_none_equal". + + Raises: + InvalidArgumentError: if the check can be performed immediately and + `x != y` is False for any pair of elements in `x` and `y`. The check can + be performed immediately during eager execution or if `x` and `y` are + statically known. + """ + assert_none_equal(x=x, y=y, summarize=summarize, message=message, name=name) + + +@tf_export(v1=['debugging.assert_none_equal', 'assert_none_equal']) @deprecation.deprecated_endpoints('assert_none_equal') def assert_none_equal( x, y, data=None, summarize=None, message=None, name=None): @@ -450,7 +593,52 @@ def assert_none_equal( return control_flow_ops.Assert(condition, data, summarize=summarize) -@tf_export('debugging.assert_near', v1=['debugging.assert_near', 'assert_near']) +@tf_export('debugging.assert_near', v1=[]) +def assert_near_v2(x, y, rtol=None, atol=None, message=None, summarize=None, + name=None): + """Assert the condition `x` and `y` are close element-wise. + + This Op checks that `x[i] - y[i] < atol + rtol * tf.abs(y[i])` holds for every + pair of (possibly broadcast) elements of `x` and `y`. If both `x` and `y` are + empty, this is trivially satisfied. + + If any elements of `x` and `y` are not close, `message`, as well as the first + `summarize` entries of `x` and `y` are printed, and `InvalidArgumentError` + is raised. + + The default `atol` and `rtol` is `10 * eps`, where `eps` is the smallest + representable positive number such that `1 + eps != 1`. This is about + `1.2e-6` in `32bit`, `2.22e-15` in `64bit`, and `0.00977` in `16bit`. + See `numpy.finfo`. + + Args: + x: Float or complex `Tensor`. + y: Float or complex `Tensor`, same dtype as and broadcastable to `x`. + rtol: `Tensor`. Same `dtype` as, and broadcastable to, `x`. + The relative tolerance. Default is `10 * eps`. + atol: `Tensor`. Same `dtype` as, and broadcastable to, `x`. + The absolute tolerance. Default is `10 * eps`. + message: A string to prefix to the default message. + summarize: Print this many entries of each tensor. + name: A name for this operation (optional). Defaults to "assert_near". + + Raises: + InvalidArgumentError: if the check can be performed immediately and + `x != y` is False for any pair of elements in `x` and `y`. The check can + be performed immediately during eager execution or if `x` and `y` are + statically known. + + @compatibility(numpy) + Similar to `numpy.assert_allclose`, except tolerance depends on data type. + This is due to the fact that `TensorFlow` is often used with `32bit`, `64bit`, + and even `16bit` data. + @end_compatibility + """ + assert_near(x=x, y=y, rtol=rtol, atol=atol, summarize=summarize, + message=message, name=name) + + +@tf_export(v1=['debugging.assert_near', 'assert_near']) @deprecation.deprecated_endpoints('assert_near') def assert_near( x, y, rtol=None, atol=None, data=None, summarize=None, message=None, @@ -529,7 +717,34 @@ def assert_near( return control_flow_ops.Assert(condition, data, summarize=summarize) -@tf_export('debugging.assert_less', 'assert_less') +@tf_export('debugging.assert_less', 'assert_less', v1=[]) +def assert_less_v2(x, y, message=None, summarize=None, name=None): + """Assert the condition `x < y` holds element-wise. + + This Op checks that `x[i] < y[i]` holds for every pair of (possibly + broadcast) elements of `x` and `y`. If both `x` and `y` are empty, this is + trivially satisfied. + + If `x` is not less than `y` element-wise, `message`, as well as the first + `summarize` entries of `x` and `y` are printed, and `InvalidArgumentError` is + raised. + + Args: + x: Numeric `Tensor`. + y: Numeric `Tensor`, same dtype as and broadcastable to `x`. + message: A string to prefix to the default message. + summarize: Print this many entries of each tensor. + name: A name for this operation (optional). Defaults to "assert_less". + + Raises: + InvalidArgumentError: if the check can be performed immediately and + `x < y` is False. The check can be performed immediately during eager + execution or if `x` and `y` are statically known. + """ + assert_less(x=x, y=y, summarize=summarize, message=message, name=name) + + +@tf_export(v1=['debugging.assert_less', 'assert_less']) def assert_less(x, y, data=None, summarize=None, message=None, name=None): """Assert the condition `x < y` holds element-wise. @@ -577,9 +792,34 @@ def assert_less(x, y, data=None, summarize=None, message=None, name=None): return control_flow_ops.Assert(condition, data, summarize=summarize) -@tf_export( - 'debugging.assert_less_equal', - v1=['debugging.assert_less_equal', 'assert_less_equal']) +@tf_export('debugging.assert_less_equal', v1=[]) +def assert_less_equal_v2(x, y, message=None, summarize=None, name=None): + """Assert the condition `x <= y` holds element-wise. + + This Op checks that `x[i] <= y[i]` holds for every pair of (possibly + broadcast) elements of `x` and `y`. If both `x` and `y` are empty, this is + trivially satisfied. + + If `x` is not less or equal than `y` element-wise, `message`, as well as the + first `summarize` entries of `x` and `y` are printed, and + `InvalidArgumentError` is raised. + + Args: + x: Numeric `Tensor`. + y: Numeric `Tensor`, same dtype as and broadcastable to `x`. + message: A string to prefix to the default message. + summarize: Print this many entries of each tensor. + name: A name for this operation (optional). Defaults to "assert_less_equal". + + Raises: + InvalidArgumentError: if the check can be performed immediately and + `x <= y` is False. The check can be performed immediately during eager + execution or if `x` and `y` are statically known. + """ + assert_less_equal(x=x, y=y, summarize=summarize, message=message, name=name) + + +@tf_export(v1=['debugging.assert_less_equal', 'assert_less_equal']) @deprecation.deprecated_endpoints('assert_less_equal') def assert_less_equal(x, y, data=None, summarize=None, message=None, name=None): """Assert the condition `x <= y` holds element-wise. @@ -628,7 +868,34 @@ def assert_less_equal(x, y, data=None, summarize=None, message=None, name=None): return control_flow_ops.Assert(condition, data, summarize=summarize) -@tf_export('debugging.assert_greater', 'assert_greater') +@tf_export('debugging.assert_greater', 'assert_greater', v1=[]) +def assert_greater_v2(x, y, message=None, summarize=None, name=None): + """Assert the condition `x > y` holds element-wise. + + This Op checks that `x[i] > y[i]` holds for every pair of (possibly + broadcast) elements of `x` and `y`. If both `x` and `y` are empty, this is + trivially satisfied. + + If `x` is not greater than `y` element-wise, `message`, as well as the first + `summarize` entries of `x` and `y` are printed, and `InvalidArgumentError` is + raised. + + Args: + x: Numeric `Tensor`. + y: Numeric `Tensor`, same dtype as and broadcastable to `x`. + message: A string to prefix to the default message. + summarize: Print this many entries of each tensor. + name: A name for this operation (optional). Defaults to "assert_greater". + + Raises: + InvalidArgumentError: if the check can be performed immediately and + `x > y` is False. The check can be performed immediately during eager + execution or if `x` and `y` are statically known. + """ + assert_greater(x=x, y=y, summarize=summarize, message=message, name=name) + + +@tf_export(v1=['debugging.assert_greater', 'assert_greater']) def assert_greater(x, y, data=None, summarize=None, message=None, name=None): """Assert the condition `x > y` holds element-wise. @@ -676,9 +943,36 @@ def assert_greater(x, y, data=None, summarize=None, message=None, name=None): return control_flow_ops.Assert(condition, data, summarize=summarize) -@tf_export( - 'debugging.assert_greater_equal', - v1=['debugging.assert_greater_equal', 'assert_greater_equal']) +@tf_export('debugging.assert_greater_equal', v1=[]) +def assert_greater_equal_v2(x, y, message=None, summarize=None, name=None): + """Assert the condition `x >= y` holds element-wise. + + This Op checks that `x[i] >= y[i]` holds for every pair of (possibly + broadcast) elements of `x` and `y`. If both `x` and `y` are empty, this is + trivially satisfied. + + If `x` is not greater or equal to `y` element-wise, `message`, as well as the + first `summarize` entries of `x` and `y` are printed, and + `InvalidArgumentError` is raised. + + Args: + x: Numeric `Tensor`. + y: Numeric `Tensor`, same dtype as and broadcastable to `x`. + message: A string to prefix to the default message. + summarize: Print this many entries of each tensor. + name: A name for this operation (optional). Defaults to + "assert_greater_equal". + + Raises: + InvalidArgumentError: if the check can be performed immediately and + `x >= y` is False. The check can be performed immediately during eager + execution or if `x` and `y` are statically known. + """ + assert_greater_equal(x=x, y=y, summarize=summarize, message=message, + name=name) + + +@tf_export(v1=['debugging.assert_greater_equal', 'assert_greater_equal']) @deprecation.deprecated_endpoints('assert_greater_equal') def assert_greater_equal(x, y, data=None, summarize=None, message=None, name=None): @@ -777,7 +1071,31 @@ def _assert_rank_condition( return control_flow_ops.Assert(condition, data, summarize=summarize) -@tf_export('debugging.assert_rank', 'assert_rank') +@tf_export('debugging.assert_rank', 'assert_rank', v1=[]) +def assert_rank_v2(x, rank, message=None, name=None): + """Assert that `x` has rank equal to `rank`. + + This Op checks that the rank of `x` is equal to `rank`. + + If `x` has a different rank, `message`, as well as the shape of `x` are + printed, and `InvalidArgumentError` is raised. + + Args: + x: `Tensor`. + rank: Scalar integer `Tensor`. + message: A string to prefix to the default message. + name: A name for this operation (optional). Defaults to + "assert_rank". + + Raises: + InvalidArgumentError: if the check can be performed immediately and + `x` does not have rank `rank`. The check can be performed immediately + during eager execution or if the shape of `x` is statically known. + """ + assert_rank(x=x, rank=rank, message=message, name=name) + + +@tf_export(v1=['debugging.assert_rank', 'assert_rank']) def assert_rank(x, rank, data=None, summarize=None, message=None, name=None): """Assert `x` has rank equal to `rank`. @@ -792,7 +1110,7 @@ def assert_rank(x, rank, data=None, summarize=None, message=None, name=None): x: Numeric `Tensor`. rank: Scalar integer `Tensor`. data: The tensors to print out if the condition is False. Defaults to - error message and first few entries of `x`. + error message and the shape of `x`. summarize: Print this many entries of each tensor. message: A string to prefix to the default message. name: A name for this operation (optional). Defaults to "assert_rank". @@ -839,9 +1157,31 @@ def assert_rank(x, rank, data=None, summarize=None, message=None, name=None): return assert_op -@tf_export( - 'debugging.assert_rank_at_least', - v1=['debugging.assert_rank_at_least', 'assert_rank_at_least']) +@tf_export('debugging.assert_rank_at_least', v1=[]) +def assert_rank_at_least_v2(x, rank, message=None, name=None): + """Assert that `x` has rank of at least `rank`. + + This Op checks that the rank of `x` is greater or equal to `rank`. + + If `x` has a rank lower than `rank`, `message`, as well as the shape of `x` + are printed, and `InvalidArgumentError` is raised. + + Args: + x: `Tensor`. + rank: Scalar integer `Tensor`. + message: A string to prefix to the default message. + name: A name for this operation (optional). Defaults to + "assert_rank_at_least". + + Raises: + InvalidArgumentError: `x` does not have rank at least `rank`, but the rank + cannot be statically determined. + ValueError: If static checks determine `x` has mismatched rank. + """ + assert_rank_at_least(x=x, rank=rank, message=message, name=name) + + +@tf_export(v1=['debugging.assert_rank_at_least', 'assert_rank_at_least']) @deprecation.deprecated_endpoints('assert_rank_at_least') def assert_rank_at_least( x, rank, data=None, summarize=None, message=None, name=None): @@ -973,9 +1313,30 @@ def _assert_ranks_condition( return control_flow_ops.Assert(condition, data, summarize=summarize) -@tf_export( - 'debugging.assert_rank_in', - v1=['debugging.assert_rank_in', 'assert_rank_in']) +@tf_export('debugging.assert_rank_in', v1=[]) +def assert_rank_in_v2(x, ranks, message=None, name=None): + """Assert that `x` has a rank in `ranks`. + + This Op checks that the rank of `x` is in `ranks`. + + If `x` has a different rank, `message`, as well as the shape of `x` are + printed, and `InvalidArgumentError` is raised. + + Args: + x: `Tensor`. + ranks: `Iterable` of scalar `Tensor` objects. + message: A string to prefix to the default message. + name: A name for this operation (optional). Defaults to "assert_rank_in". + + Raises: + InvalidArgumentError: `x` does not have rank in `ranks`, but the rank cannot + be statically determined. + ValueError: If static checks determine `x` has mismatched rank. + """ + assert_rank_in(x=x, ranks=ranks, message=message, name=name) + + +@tf_export(v1=['debugging.assert_rank_in', 'assert_rank_in']) @deprecation.deprecated_endpoints('assert_rank_in') def assert_rank_in( x, ranks, data=None, summarize=None, message=None, name=None): @@ -1038,9 +1399,25 @@ def assert_rank_in( return assert_op -@tf_export( - 'debugging.assert_integer', - v1=['debugging.assert_integer', 'assert_integer']) +@tf_export('debugging.assert_integer', v1=[]) +def assert_integer_v2(x, message=None, name=None): + """Assert that `x` is of integer dtype. + + If `x` has a non-integer type, `message`, as well as the dtype of `x` are + printed, and `InvalidArgumentError` is raised. + + Args: + x: A `Tensor`. + message: A string to prefix to the default message. + name: A name for this operation (optional). Defaults to "assert_integer". + + Raises: + TypeError: If `x.dtype` is not a non-quantized integer type. + """ + assert_integer(x=x, message=message, name=name) + + +@tf_export(v1=['debugging.assert_integer', 'assert_integer']) @deprecation.deprecated_endpoints('assert_integer') def assert_integer(x, message=None, name=None): """Assert that `x` is of integer dtype. @@ -1079,13 +1456,30 @@ def assert_integer(x, message=None, name=None): return control_flow_ops.no_op('statically_determined_was_integer') -@tf_export('debugging.assert_type', v1=['debugging.assert_type', 'assert_type']) +@tf_export('debugging.assert_type', v1=[]) +def assert_type_v2(tensor, tf_type, message=None, name=None): + """Asserts that the given `Tensor` is of the specified type. + + Args: + tensor: A `Tensor`. + tf_type: A tensorflow type (`dtypes.float32`, `tf.int64`, `dtypes.bool`, + etc). + message: A string to prefix to the default message. + name: A name for this operation. Defaults to "assert_type" + + Raises: + TypeError: If the tensor's data type doesn't match `tf_type`. + """ + assert_type(tensor=tensor, tf_type=tf_type, message=message, name=name) + + +@tf_export(v1=['debugging.assert_type', 'assert_type']) @deprecation.deprecated_endpoints('assert_type') def assert_type(tensor, tf_type, message=None, name=None): """Statically asserts that the given `Tensor` is of the specified type. Args: - tensor: A tensorflow `Tensor`. + tensor: A `Tensor`. tf_type: A tensorflow type (`dtypes.float32`, `tf.int64`, `dtypes.bool`, etc). message: A string to prefix to the default message. @@ -1136,9 +1530,13 @@ def is_numeric_tensor(tensor): @tf_export( - 'debugging.is_non_decreasing', - v1=['debugging.is_non_decreasing', 'is_non_decreasing']) -@deprecation.deprecated_endpoints('is_non_decreasing') + 'math.is_non_decreasing', + v1=[ + 'math.is_non_decreasing', 'debugging.is_non_decreasing', + 'is_non_decreasing' + ]) +@deprecation.deprecated_endpoints('debugging.is_non_decreasing', + 'is_non_decreasing') def is_non_decreasing(x, name=None): """Returns `True` if `x` is non-decreasing. @@ -1166,9 +1564,13 @@ def is_non_decreasing(x, name=None): @tf_export( - 'debugging.is_strictly_increasing', - v1=['debugging.is_strictly_increasing', 'is_strictly_increasing']) -@deprecation.deprecated_endpoints('is_strictly_increasing') + 'math.is_strictly_increasing', + v1=[ + 'math.is_strictly_increasing', 'debugging.is_strictly_increasing', + 'is_strictly_increasing' + ]) +@deprecation.deprecated_endpoints('debugging.is_strictly_increasing', + 'is_strictly_increasing') def is_strictly_increasing(x, name=None): """Returns `True` if `x` is strictly increasing. @@ -1260,8 +1662,10 @@ def assert_same_float_dtype(tensors=None, dtype=None): tensors: Tensors of input values. Can include `None` elements, which will be ignored. dtype: Expected type. + Returns: Validated type. + Raises: ValueError: if neither `tensors` nor `dtype` is supplied, or result is not float, or the common type of the inputs is not a floating point type. @@ -1275,20 +1679,57 @@ def assert_same_float_dtype(tensors=None, dtype=None): return dtype -@tf_export( - 'debugging.assert_scalar', v1=['debugging.assert_scalar', 'assert_scalar']) +@tf_export('debugging.assert_scalar', v1=[]) +def assert_scalar_v2(tensor, message=None, name=None): + """Asserts that the given `tensor` is a scalar. + + This function raises `ValueError` unless it can be certain that the given + `tensor` is a scalar. `ValueError` is also raised if the shape of `tensor` is + unknown. + + Args: + tensor: A `Tensor`. + message: A string to prefix to the default message. + name: A name for this operation. Defaults to "assert_scalar" + + Raises: + ValueError: If the tensor is not scalar (rank 0), or if its shape is + unknown. + """ + assert_scalar(tensor=tensor, message=message, name=name) + + +@tf_export(v1=['debugging.assert_scalar', 'assert_scalar']) @deprecation.deprecated_endpoints('assert_scalar') -def assert_scalar(tensor, name=None): +def assert_scalar(tensor, name=None, message=None): + """Asserts that the given `tensor` is a scalar. + + This function raises `ValueError` unless it can be certain that the given + `tensor` is a scalar. `ValueError` is also raised if the shape of `tensor` is + unknown. + + Args: + tensor: A `Tensor`. + name: A name for this operation. Defaults to "assert_scalar" + message: A string to prefix to the default message. + + Returns: + The input tensor (potentially converted to a `Tensor`). + + Raises: + ValueError: If the tensor is not scalar (rank 0), or if its shape is + unknown. + """ with ops.name_scope(name, 'assert_scalar', [tensor]) as name_scope: tensor = ops.convert_to_tensor(tensor, name=name_scope) shape = tensor.get_shape() if shape.ndims != 0: if context.executing_eagerly(): - raise ValueError('Expected scalar shape, saw shape: %s.' - % (shape,)) + raise ValueError('%sExpected scalar shape, saw shape: %s.' + % (message or '', shape,)) else: - raise ValueError('Expected scalar shape for %s, saw shape: %s.' - % (tensor.name, shape)) + raise ValueError('%sExpected scalar shape for %s, saw shape: %s.' + % (message or '', tensor.name, shape)) return tensor diff --git a/tensorflow/python/ops/clip_ops.py b/tensorflow/python/ops/clip_ops.py index 5cd626b92dc..82803ac3516 100644 --- a/tensorflow/python/ops/clip_ops.py +++ b/tensorflow/python/ops/clip_ops.py @@ -300,7 +300,12 @@ def clip_by_global_norm(t_list, clip_norm, use_norm=None, name=None): return list_clipped, use_norm -@tf_export("clip_by_average_norm") +@deprecation.deprecated( + date=None, + instructions= + "clip_by_average_norm is deprecated in TensorFlow 2.0. Please use " + "clip_by_norm(t, clip_norm * tf.to_float(tf.size(t), name)) instead.") +@tf_export(v1=["clip_by_average_norm"]) def clip_by_average_norm(t, clip_norm, name=None): """Clips tensor values to a maximum average L2-norm. diff --git a/tensorflow/python/ops/clip_ops_test.py b/tensorflow/python/ops/clip_ops_test.py index 8aa9c4ffb34..a59a0c22d40 100644 --- a/tensorflow/python/ops/clip_ops_test.py +++ b/tensorflow/python/ops/clip_ops_test.py @@ -20,6 +20,7 @@ from __future__ import print_function from tensorflow.python.framework import constant_op from tensorflow.python.framework import ops +from tensorflow.python.framework import test_util from tensorflow.python.ops import clip_ops from tensorflow.python.ops import numerics from tensorflow.python.platform import test @@ -35,7 +36,7 @@ class ClipOpsTest(test.TestCase): input_op = constant_op.constant(inputs) clipped = clip_ops.clip_by_norm(input_op, max_norm) check_op = numerics.add_check_numerics_ops() - result, _ = sess.run([clipped, check_op]) + result, _ = self.evaluate([clipped, check_op]) self.assertAllClose(result, expected) def _testClipIndexedSlicesByNorm(self, values, indices, shape, max_norm, @@ -54,9 +55,10 @@ class ClipOpsTest(test.TestCase): # Tensor mode dense_tensor = ops.convert_to_tensor(indixed_slices) dense_clipped = clip_ops.clip_by_norm(dense_tensor, max_norm, axes) - result, expected = sess.run([clipped, dense_clipped]) + result, expected = self.evaluate([clipped, dense_clipped]) self.assertAllClose(result, expected) + @test_util.run_deprecated_v1 def testClipTensorByNorm(self): # Simple example self._testClipTensorByNorm([[-3.0, 0.0, 0.0], [4.0, 0.0, 0.0]], 4.0, diff --git a/tensorflow/python/ops/collective_ops_test.py b/tensorflow/python/ops/collective_ops_test.py index 9c772a93548..0fd9368d219 100644 --- a/tensorflow/python/ops/collective_ops_test.py +++ b/tensorflow/python/ops/collective_ops_test.py @@ -21,6 +21,7 @@ from __future__ import print_function from tensorflow.core.protobuf import config_pb2 from tensorflow.python.framework import constant_op from tensorflow.python.framework import ops +from tensorflow.python.framework import test_util from tensorflow.python.ops import collective_ops from tensorflow.python.platform import test @@ -49,16 +50,19 @@ class CollectiveOpTest(test.TestCase): self.assertAllClose(results[0], expected, rtol=1e-5, atol=1e-5) self.assertAllClose(results[1], expected, rtol=1e-5, atol=1e-5) + @test_util.run_deprecated_v1 def testCollectiveReduce(self): self._testCollectiveReduce([0.1, 1.1, 2.1, 3.1, 4.1, 5.1, 6.1, 7.1], [0.3, 1.3, 2.3, 3.3, 4.3, 5.3, 6.3, 7.3], [0.2, 1.2, 2.2, 3.2, 4.2, 5.2, 6.2, 7.2], True) + @test_util.run_deprecated_v1 def testCollectiveAutoGraphKey(self): self._testCollectiveReduce([0.1, 1.1, 2.1, 3.1, 4.1, 5.1, 6.1, 7.1], [0.3, 1.3, 2.3, 3.3, 4.3, 5.3, 6.3, 7.3], [0.2, 1.2, 2.2, 3.2, 4.2, 5.2, 6.2, 7.2], False) + @test_util.run_deprecated_v1 def testCollectiveReduceScalar(self): self._testCollectiveReduce(0.1, 0.3, 0.2, True) @@ -81,6 +85,7 @@ class CollectiveOpTest(test.TestCase): self.assertAllClose(results[0], t0, rtol=1e-5, atol=1e-5) self.assertAllClose(results[1], t0, rtol=1e-5, atol=1e-5) + @test_util.run_deprecated_v1 def testCollectiveBroadcast(self): self._testCollectiveBroadcast([0.1, 1.1, 2.1, 3.1, 4.1, 5.1, 6.1, 7.1]) diff --git a/tensorflow/python/ops/cond_v2.py b/tensorflow/python/ops/cond_v2.py index 998c3e08f6f..e882a270c8c 100644 --- a/tensorflow/python/ops/cond_v2.py +++ b/tensorflow/python/ops/cond_v2.py @@ -25,15 +25,19 @@ from __future__ import print_function import collections -from tensorflow.core.framework import attr_value_pb2 +from tensorflow.python.framework import dtypes from tensorflow.python.framework import func_graph as func_graph_module from tensorflow.python.framework import function_def_to_graph from tensorflow.python.framework import ops from tensorflow.python.ops import array_ops from tensorflow.python.ops import control_flow_util from tensorflow.python.ops import control_flow_util_v2 as util +from tensorflow.python.ops import gen_dataset_ops from tensorflow.python.ops import gen_functional_ops +from tensorflow.python.ops import gen_resource_variable_ops from tensorflow.python.ops import gradients_impl +from tensorflow.python.util import nest + # NOTE(skyewm): TensorFlow uses protected class methods and fields to signify # that they aren't part of the official public API. These protected members @@ -74,73 +78,17 @@ def cond_v2(pred, true_fn, false_fn, name="cond"): false_name, read_only_collections=False), add_control_dependencies=add_control_dependencies, op_return_value=pred) - _check_same_outputs(true_graph, false_graph) - # Add inputs to true_graph and false_graph to make them match. Note that - # this modifies true_graph and false_graph. - cond_inputs = _make_inputs_match(true_graph, false_graph, - true_graph.external_captures, - false_graph.external_captures) + outputs = _build_cond(pred, true_graph, false_graph, + true_graph.external_captures, + false_graph.external_captures, + name=scope) - # Add all intermediate tensors as function outputs so they're available for - # the gradient computation. - - true_intermediates = _get_intermediates(true_graph) - false_intermediates = _get_intermediates(false_graph) - - # Save the original number of outputs to return to the caller. - num_cond_outputs = len(true_graph.outputs) - - # Make the number/type of new intermediate outputs match. - extra_true_outputs, extra_false_outputs = _pad_params( - true_graph, false_graph, true_intermediates, false_intermediates) - - true_graph.outputs.extend(extra_true_outputs) - false_graph.outputs.extend(extra_false_outputs) - - # Create the If op. - tensors = gen_functional_ops._if( # pylint: disable=protected-access - pred, - cond_inputs, [t.dtype for t in true_graph.outputs], - util.create_new_tf_function(true_graph), - util.create_new_tf_function(false_graph), - output_shapes=_get_output_shapes(true_graph.outputs, - false_graph.outputs), - name=scope) - - # Set the flag to enable lowering on the `if` op if necessary - # Lowering allows cond_v2 to avoid some of the limitations of Functions, - # allowing users to specify devices & colocation inside of cond_v2 branches, - # and enabling non-strict evaluation & partial pruning of cond_v2 branches. - # This brings cond_v2 closer to feature parity with tf.cond. - # - # However, we do not lower `If` in the XLA context because it is easier for - # XLA to apply its own optimizations when dealing with un-lowered `If` - # operators than with lowered switch/merge control flow. - # - # TODO(b/110167197) this approach requires cond_v2 to have at least 1 output - if_op = tensors[0].op - if not control_flow_util.IsInXLAContext(if_op): - # pylint: disable=protected-access - if_op._set_attr("_lower_using_switch_merge", - attr_value_pb2.AttrValue(b=True)) - # pylint: enable=protected-access - - # Return identities for each output of the If op, rather than the output of - # the If op directly. This makes pruning work if the output of cond() is - # fetched: the lowering pass converts the If outputs into IdentityN outputs, - # which if fetched will cause all ops in the taken branch to be run (since - # it takes all merge ops as input). After lowering, each output identity op - # will end up with only the appropriate merge op as input. - # TODO(b/79984175): this doesn't have to be a tuple once we covert to the - # correct output structure - tensors = tuple(array_ops.identity(t) for t in tensors) - - result = tuple(tensors[:num_cond_outputs]) - if len(result) == 1: - return result[0] - else: - return result + # Packing output tensors in the same nested structure as the true and false + # functions return + result = nest.pack_sequence_as(structure=true_graph.structured_outputs, + flat_sequence=tensors[:num_cond_outputs]) + return result @ops.RegisterGradient("If") @@ -168,39 +116,105 @@ def _IfGrad(op, *grads): # pylint: disable=invalid-name true_grad_inputs = _resolve_grad_inputs(true_graph, true_grad_graph) false_grad_inputs = _resolve_grad_inputs(false_graph, false_grad_graph) - # Make the inputs to true_grad_graph and false_grad_graph match. Note that - # this modifies true_grad_graph and false_grad_graph. - grad_inputs = _make_inputs_match(true_grad_graph, false_grad_graph, - true_grad_inputs, false_grad_inputs) - - # Add all intermediate tensors as function outputs so they're available for - # higher-order gradient computations. - - true_grad_intermediates = _get_intermediates(true_grad_graph) - false_grad_intermediates = _get_intermediates(false_grad_graph) - - # Save the original number of gradient outputs to return. - num_grad_outputs = len(true_grad_graph.outputs) - - # Make the number/type of new intermediate outputs match. - extra_true_grad_outputs, extra_false_grad_outputs = _pad_params( - true_grad_graph, false_grad_graph, - true_grad_intermediates, false_grad_intermediates) - - true_grad_graph.outputs.extend(extra_true_grad_outputs) - false_grad_graph.outputs.extend(extra_false_grad_outputs) - - # Create the gradient If op. - tensors = gen_functional_ops._if( - op.inputs[0], - grad_inputs, [t.dtype for t in true_grad_graph.outputs], - util.create_new_tf_function(true_grad_graph), - util.create_new_tf_function(false_grad_graph), - output_shapes=_get_output_shapes(true_grad_graph.outputs, - false_grad_graph.outputs)) + outputs = _build_cond(op.inputs[0], true_grad_graph, false_grad_graph, + true_grad_inputs, false_grad_inputs) # The predicate has no gradient. - return [None] + tensors[:num_grad_outputs] + return [None] + outputs + + +def _build_cond(pred, true_graph, false_graph, true_inputs, false_inputs, + name=None): + """Creates an If op from the specified predicate, branch functions and inputs. + + Note that this modifies true_graph and false_graph to make the inputs match, + and to output all intermediates values so they're available for the gradient + computation. + + true_graph and false_graph need not have the same input types, but they must + have the same outpute types. + + Args: + pred: boolean Tensor + true_graph: FuncGraph + false_graph: FuncGraph + true_inputs: a list of Tensors to be passed to true_graph as input. + false_inputs: a list of Tensors to be passed to false_graph as input. + name: the name for the If op. + + Returns: + A list of Tensors which are the outputs of the If op. Does not include added + intermediate outputs. + """ + _check_same_outputs(true_graph, false_graph) + + # Add inputs to true_graph and false_graph to make them match. Note that + # this modifies true_graph and false_graph. + cond_inputs = _make_inputs_match(true_graph, false_graph, + true_inputs, false_inputs) + + # Add all intermediate tensors as function outputs so they're available for + # the gradient computation. Since the outputs of the two functions must match, + # we wrap all the intermediates in optionals. Each intermediate output will + # have a value iff its corresponding branch is taken. + + true_intermediates = _get_intermediates(true_graph) + false_intermediates = _get_intermediates(false_graph) + + # Save the original number of outputs to return to the caller. + num_cond_outputs = len(true_graph.outputs) + + if control_flow_util.InXlaContext(ops.get_default_graph()): + # XLA does not yet support optionals, so output intermediates directly and + # make them match via FakeParams, which can be converted to zeros in XLA. + # TODO(skyewm,jpienaar): can XLA support optionals? + extra_true_outputs, extra_false_outputs = _make_intermediates_match_xla( + true_graph, false_graph, true_intermediates, false_intermediates) + else: + # Wrap intermediates in optionals. + wrapped_true_intermediates = _wrap_intermediates(true_graph, + true_intermediates) + wrapped_false_intermediates = _wrap_intermediates(false_graph, + false_intermediates) + + # Make outputs match by adding none optionals. + extra_true_outputs, extra_false_outputs = _make_intermediates_match( + true_graph, false_graph, + wrapped_true_intermediates, wrapped_false_intermediates) + + true_graph.outputs.extend(extra_true_outputs) + false_graph.outputs.extend(extra_false_outputs) + # TODO(skyewm): somehow indicate it's a bug if this fails. + _check_same_outputs(true_graph, false_graph) + + # Create the If op. + tensors = gen_functional_ops._if( # pylint: disable=protected-access + pred, + cond_inputs, [t.dtype for t in true_graph.outputs], + util.create_new_tf_function(true_graph), + util.create_new_tf_function(false_graph), + output_shapes=_get_output_shapes(true_graph.outputs, + false_graph.outputs), + name=name) + + # TODO(b/110167197) this approach requires cond_v2 to have at least 1 output + if_op = tensors[0].op + util.maybe_set_lowering_attr(if_op) + + # Return identities for each output of the If op, rather than the output of + # the If op directly. This makes pruning work if the output of cond() is + # fetched: the lowering pass converts the If outputs into IdentityN outputs, + # which if fetched will cause all ops in the taken branch to be run (since + # it takes all merge ops as input). After lowering, each output identity op + # will end up with only the appropriate merge op as input. + # TODO(b/79984175): this doesn't have to be a tuple once we covert to the + # correct output structure + tensors = [array_ops.identity(t) for t in tensors] + + # Prevent fetching since the variant outputs can't be fetched directly. + if_op.graph.prevent_fetching(if_op) + + return tensors[:num_cond_outputs] def _get_func_graphs(if_op): @@ -277,7 +291,11 @@ def _grad_fn(func_graph, grads): # both branches have zero gradient. for i in range(len(result)): if result[i] is None: - result[i] = array_ops.zeros_like(func_graph.inputs[i]) + if func_graph.inputs[i].dtype == dtypes.resource: + result[i] = array_ops.zeros( + gen_resource_variable_ops.variable_shape(func_graph.inputs[i])) + else: + result[i] = array_ops.zeros_like(func_graph.inputs[i]) return result @@ -287,7 +305,7 @@ def _create_grad_func(func_graph, grads, name): return func_graph_module.func_graph_from_py_func( name, lambda: _grad_fn(func_graph, grads), [], {}, - func_graph=util.CondBranchFuncGraph(name, read_only_collections=False)) + func_graph=_CondGradFuncGraph(name, func_graph)) def _resolve_grad_inputs(cond_graph, grad_graph): @@ -369,28 +387,39 @@ def _separate_unique_inputs(true_inputs, false_inputs): return list(shared_inputs), list(true_only_inputs), list(false_only_inputs) -def _pad_params(true_graph, false_graph, true_params, false_params): - """Returns new param lists that have matching signatures. +def _make_intermediates_match(true_graph, false_graph, + true_optionals, false_optionals): + """Returns new optionals lists that have matching signatures. - This is done by mirroring each param list in the other using dummy params. - There is no merging of params. + This is done by mirroring each list in the other using none optionals. + There is no merging of like optionals. Args: true_graph: FuncGraph false_graph: FuncGraph - true_params: a list of Tensors from true_graph - false_params: a list of Tensors from false_graph + true_optionals: a list of optional Tensors from true_graph + false_optionals: a list of optional Tensors from false_graph Returns: A new list of Tensors in true_graph and a new list of Tensors in - false_graph. The two lists have the same number of Tensors, with matching - types and shapes across the lists. + false_graph. The two lists have the same number of Tensors, all of which + will be optionals of the same shape/type. """ - new_true_params = (true_params + - _create_dummy_params(true_graph, false_params)) - new_false_inputs = (_create_dummy_params(false_graph, true_params) - + false_params) - return new_true_params, new_false_inputs + new_true_optionals = (true_optionals + + _create_none_optionals(true_graph, false_optionals)) + new_false_optionals = (_create_none_optionals(false_graph, true_optionals) + + false_optionals) + return new_true_optionals, new_false_optionals + + +def _make_intermediates_match_xla(true_graph, false_graph, true_intermediates, + false_intermediates): + """Like _make_intermediates_match but for the XLA case.""" + new_true_intermediates = (true_intermediates + + _create_fakeparams(true_graph, false_intermediates)) + new_false_intermediates = (_create_fakeparams(false_graph, true_intermediates) + + false_intermediates) + return new_true_intermediates, new_false_intermediates def _make_inputs_match(true_graph, false_graph, true_inputs, false_inputs): @@ -425,11 +454,11 @@ def _make_inputs_match(true_graph, false_graph, true_inputs, false_inputs): true_graph.inputs = ( [true_input_to_param[t] for t in shared_inputs] + [true_input_to_param[t] for t in true_only_inputs] + - _create_dummy_params(true_graph, false_only_inputs)) + _create_dummy_inputs(true_graph, false_only_inputs)) false_graph.inputs = ( [false_input_to_param[t] for t in shared_inputs] + - _create_dummy_params(false_graph, true_only_inputs) + + _create_dummy_inputs(false_graph, true_only_inputs) + [false_input_to_param[t] for t in false_only_inputs]) # Rewrite the FuncGraphs' state to reflect the new inputs. @@ -441,7 +470,12 @@ def _make_inputs_match(true_graph, false_graph, true_inputs, false_inputs): return new_inputs -def _create_dummy_params(func_graph, template_tensors): +def _wrap_intermediates(func_graph, intermediates): + with func_graph.as_default(): + return [gen_dataset_ops.optional_from_value([t]) for t in intermediates] + + +def _create_dummy_inputs(func_graph, template_tensors): """Creates tensors in func_graph to represent template_tensors. Args: @@ -451,6 +485,27 @@ def _create_dummy_params(func_graph, template_tensors): Returns: A list of tensors in func_graph. """ + with func_graph.as_default(): + return [array_ops.placeholder(t.dtype, shape=t.shape) + for t in template_tensors] + + +def _create_none_optionals(func_graph, template_tensors): + """Creates none optionals in func_graph to represent template_tensors. + + Args: + func_graph: FuncGraph. + template_tensors: a list of tensors in func_graph. + + Returns: + A list of tensors in func_graph. + """ + with func_graph.as_default(): + return [gen_dataset_ops.optional_none() for _ in template_tensors] + + +def _create_fakeparams(func_graph, template_tensors): + """Create FakeParams for the XLA case.""" with func_graph.as_default(): return [gen_functional_ops.fake_param(dtype=t.dtype, shape=t.shape) for t in template_tensors] @@ -462,12 +517,16 @@ def _check_same_outputs(true_graph, false_graph): false_output_types = [t.dtype for t in false_graph.outputs] if (len(true_graph.outputs) != len(false_graph.outputs) or true_output_types != false_output_types): - raise ValueError( + raise TypeError( "true_fn() and false_fn() must return the same number and type of " "arguments, got:\n" " true_fn: %s\n" " false_fn: %s" % (true_output_types, false_output_types)) + # Make sure both structured outputs for both graphs have the same structure + nest.assert_same_structure(true_graph.structured_outputs, + false_graph.structured_outputs) + def _get_output_shapes(true_graph_outputs, false_graph_outputs): output_shapes = [ @@ -475,3 +534,38 @@ def _get_output_shapes(true_graph_outputs, false_graph_outputs): for t_out, f_out in zip(true_graph_outputs, false_graph_outputs) ] return output_shapes + + +class _CondGradFuncGraph(util.CondBranchFuncGraph): + """FuncGraph for the gradient function of the branch of an If op. + + Handles unwrapping optional intermediate values that are captured by the + gradient computation. + """ + + def __init__(self, name, forward_graph): + super(_CondGradFuncGraph, self).__init__(name, read_only_collections=False) + self._forward_graph = forward_graph + + def _capture_helper(self, tensor, name): + if (tensor.graph is not self._forward_graph or + tensor in self._forward_graph.inputs or + tensor in self._forward_graph.outputs): + return super(_CondGradFuncGraph, self)._capture_helper(tensor, name) + + # 'tensor' is an intermediate in the forward graph. We find the corresonding + # optional tensor, which is output from the If op, and capture it as + # normal. We then unwrap the captured optional value to get the raw + # intermediate value. + for consumer in tensor.consumers(): + if (consumer.type == "OptionalFromValue" + and consumer.outputs[0] in self._forward_graph.outputs): + optional = consumer.outputs[0] + captured_optional = super(_CondGradFuncGraph, self)._capture_helper( + optional, name) + return gen_dataset_ops.optional_get_value( + captured_optional, [tensor.dtype], [tensor.shape])[0] + raise ValueError( + "Couldn't find OptionalFromValue consumer for tensor '%s'.\n" + "This is an internal bug, please report at " + "https://github.com/tensorflow/tensorflow/issues." % tensor.name) diff --git a/tensorflow/python/ops/confusion_matrix.py b/tensorflow/python/ops/confusion_matrix.py index b86b174afe5..ccfe3b65c2d 100644 --- a/tensorflow/python/ops/confusion_matrix.py +++ b/tensorflow/python/ops/confusion_matrix.py @@ -90,12 +90,13 @@ def remove_squeezable_dimensions( return labels, predictions -@tf_export( - 'math.confusion_matrix', - v1=['math.confusion_matrix', 'confusion_matrix']) -@deprecation.deprecated_endpoints('confusion_matrix', 'train.confusion_matrix') -def confusion_matrix(labels, predictions, num_classes=None, dtype=dtypes.int32, - name=None, weights=None): +@tf_export('math.confusion_matrix', v1=[]) +def confusion_matrix(labels, + predictions, + num_classes=None, + weights=None, + dtype=dtypes.int32, + name=None): """Computes the confusion matrix from predictions and labels. The matrix columns represent the prediction labels and the rows represent the @@ -132,9 +133,9 @@ def confusion_matrix(labels, predictions, num_classes=None, dtype=dtypes.int32, num_classes: The possible number of labels the classification task can have. If this value is not provided, it will be calculated using both predictions and labels array. + weights: An optional `Tensor` whose shape matches `predictions`. dtype: Data type of the confusion matrix. name: Scope name. - weights: An optional `Tensor` whose shape matches `predictions`. Returns: A `Tensor` of type `dtype` with shape `[n, n]` representing the confusion @@ -193,3 +194,65 @@ def confusion_matrix(labels, predictions, num_classes=None, dtype=dtypes.int32, zero_matrix = array_ops.zeros(math_ops.to_int32(shape), dtype) return sparse_ops.sparse_add(zero_matrix, cm_sparse) + + +@tf_export(v1=['math.confusion_matrix', 'confusion_matrix']) +@deprecation.deprecated_endpoints('confusion_matrix', 'train.confusion_matrix') +def confusion_matrix_v1(labels, + predictions, + num_classes=None, + dtype=dtypes.int32, + name=None, + weights=None): + """Computes the confusion matrix from predictions and labels. + + The matrix columns represent the prediction labels and the rows represent the + real labels. The confusion matrix is always a 2-D array of shape `[n, n]`, + where `n` is the number of valid labels for a given classification task. Both + prediction and labels must be 1-D arrays of the same shape in order for this + function to work. + + If `num_classes` is `None`, then `num_classes` will be set to one plus the + maximum value in either predictions or labels. Class labels are expected to + start at 0. For example, if `num_classes` is 3, then the possible labels + would be `[0, 1, 2]`. + + If `weights` is not `None`, then each prediction contributes its + corresponding weight to the total value of the confusion matrix cell. + + For example: + + ```python + tf.confusion_matrix([1, 2, 4], [2, 2, 4]) ==> + [[0 0 0 0 0] + [0 0 1 0 0] + [0 0 1 0 0] + [0 0 0 0 0] + [0 0 0 0 1]] + ``` + + Note that the possible labels are assumed to be `[0, 1, 2, 3, 4]`, + resulting in a 5x5 confusion matrix. + + Args: + labels: 1-D `Tensor` of real labels for the classification task. + predictions: 1-D `Tensor` of predictions for a given classification. + num_classes: The possible number of labels the classification task can have. + If this value is not provided, it will be calculated using both + predictions and labels array. + dtype: Data type of the confusion matrix. + name: Scope name. + weights: An optional `Tensor` whose shape matches `predictions`. + + Returns: + A `Tensor` of type `dtype` with shape `[n, n]` representing the confusion + matrix, where `n` is the number of possible labels in the classification + task. + + Raises: + ValueError: If both predictions and labels are not 1-D vectors and have + mismatched shapes, or if `weights` is not `None` and its shape doesn't + match `predictions`. + """ + return confusion_matrix(labels, predictions, num_classes, weights, dtype, + name) diff --git a/tensorflow/python/ops/control_flow_ops.py b/tensorflow/python/ops/control_flow_ops.py index 0d04f0697df..b7e50c1dae5 100644 --- a/tensorflow/python/ops/control_flow_ops.py +++ b/tensorflow/python/ops/control_flow_ops.py @@ -158,7 +158,7 @@ def Assert(condition, data, summarize=None, name=None): with ops.name_scope(name, "Assert", [condition, data]) as name: xs = ops.convert_n_to_tensor(data) - if all([x.dtype in {dtypes.string, dtypes.int32} for x in xs]): + if all(x.dtype in {dtypes.string, dtypes.int32} for x in xs): # As a simple heuristic, we assume that string and int32 are # on host to avoid the need to use cond. If it is not case, # we will pay the price copying the tensor to host memory. @@ -457,19 +457,19 @@ def merge(inputs, name=None): ValueError: If any of the inputs is None, or inputs are IndexedSlices and some but not all have a dense_shape property. """ - if any([inp is None for inp in inputs]): + if any(inp is None for inp in inputs): raise ValueError("At least one of the merge inputs is None: %s" % inputs) with ops.name_scope(name, "Merge", inputs) as name: inputs = [ ops.internal_convert_to_tensor_or_indexed_slices(inp, as_ref=True) for inp in inputs ] - if all([isinstance(v, ops.Tensor) for v in inputs]): - if all([v.dtype._is_ref_dtype for v in inputs]): # pylint: disable=protected-access + if all(isinstance(v, ops.Tensor) for v in inputs): + if all(v.dtype._is_ref_dtype for v in inputs): # pylint: disable=protected-access return gen_control_flow_ops.ref_merge(inputs, name) else: return gen_control_flow_ops.merge(inputs, name) - elif all([isinstance(v, sparse_tensor.SparseTensor) for v in inputs]): + elif all(isinstance(v, sparse_tensor.SparseTensor) for v in inputs): # Only handle the case when all inputs are SparseTensor. values, _ = merge([inp.values for inp in inputs], name=name) indices, chosen_index = gen_control_flow_ops.merge( @@ -557,7 +557,7 @@ def _SetShapeInvariants(input_vars, enter_vars, shapes): if shapes is None: return flat_shapes = nest.flatten(shapes) - if not all([isinstance(s, tensor_shape.TensorShape) for s in flat_shapes]): + if not all(isinstance(s, tensor_shape.TensorShape) for s in flat_shapes): raise ValueError("`shapes` must be a (possibly nested) list of shapes.") # Check that the shapes of the inputs are less than the shape invariants, # and set the shapes of `enter_vars` to the shape invariants. @@ -1976,7 +1976,7 @@ def _UnpackIfSingleton(res): # pylint: disable=redefined-outer-name # pylint: disable=g-doc-args -@tf_export("cond") +@tf_export(v1=["cond"]) @deprecation.deprecated_args( None, "fn1/fn2 are deprecated in favor of the true_fn/false_fn arguments.", "fn1", "fn2") @@ -2173,6 +2173,77 @@ def cond(pred, # pylint: enable=redefined-outer-name +@tf_export("cond", v1=[]) +def cond_for_tf_v2(pred, + true_fn=None, + false_fn=None, + name=None): + """Return `true_fn()` if the predicate `pred` is true else `false_fn()`. + + `true_fn` and `false_fn` both return lists of output tensors. `true_fn` and + `false_fn` must have the same non-zero number and type of outputs. + + **WARNING**: Any Tensors or Operations created outside of `true_fn` and + `false_fn` will be executed regardless of which branch is selected at runtime. + + Although this behavior is consistent with the dataflow model of TensorFlow, + it has frequently surprised users who expected a lazier semantics. + Consider the following simple program: + + ```python + z = tf.multiply(a, b) + result = tf.cond(x < y, lambda: tf.add(x, z), lambda: tf.square(y)) + ``` + + If `x < y`, the `tf.add` operation will be executed and `tf.square` + operation will not be executed. Since `z` is needed for at least one + branch of the `cond`, the `tf.multiply` operation is always executed, + unconditionally. + + Note that `cond` calls `true_fn` and `false_fn` *exactly once* (inside the + call to `cond`, and not at all during `Session.run()`). `cond` + stitches together the graph fragments created during the `true_fn` and + `false_fn` calls with some additional graph nodes to ensure that the right + branch gets executed depending on the value of `pred`. + + `tf.cond` supports nested structures as implemented in + `tensorflow.python.util.nest`. Both `true_fn` and `false_fn` must return the + same (possibly nested) value structure of lists, tuples, and/or named tuples. + Singleton lists and tuples form the only exceptions to this: when returned by + `true_fn` and/or `false_fn`, they are implicitly unpacked to single values. + + Args: + pred: A scalar determining whether to return the result of `true_fn` or + `false_fn`. + true_fn: The callable to be performed if pred is true. + false_fn: The callable to be performed if pred is false. + name: Optional name prefix for the returned tensors. + + Returns: + Tensors returned by the call to either `true_fn` or `false_fn`. If the + callables return a singleton list, the element is extracted from the list. + + Raises: + TypeError: if `true_fn` or `false_fn` is not callable. + ValueError: if `true_fn` and `false_fn` do not return the same number of + tensors, or return tensors of different types. + + Example: + + ```python + x = tf.constant(2) + y = tf.constant(5) + def f1(): return tf.multiply(x, 17) + def f2(): return tf.add(y, 23) + r = tf.cond(tf.less(x, y), f1, f2) + # r is set to f1(). + # Operations in f2 (e.g., tf.add) are not executed. + ``` + + """ + return cond(pred, true_fn=true_fn, false_fn=false_fn, strict=True, name=name) + + def _resource_safe_shape(t): """Returns the shape of t or the variable it points to.""" if t.dtype == dtypes.resource: @@ -3065,7 +3136,186 @@ class WhileContext(ControlFlowContext): # pylint: disable=redefined-outer-name -@tf_export("while_loop") +@tf_export("while_loop", v1=[]) +def while_loop_v2(cond, + body, + loop_vars, + shape_invariants=None, + parallel_iterations=10, + back_prop=True, + swap_memory=False, + maximum_iterations=None, + name=None): + """Repeat `body` while the condition `cond` is true. + + `cond` is a callable returning a boolean scalar tensor. `body` is a callable + returning a (possibly nested) tuple, namedtuple or list of tensors of the same + arity (length and structure) and types as `loop_vars`. `loop_vars` is a + (possibly nested) tuple, namedtuple or list of tensors that is passed to both + `cond` and `body`. `cond` and `body` both take as many arguments as there are + `loop_vars`. + + In addition to regular Tensors or IndexedSlices, the body may accept and + return TensorArray objects. The flows of the TensorArray objects will + be appropriately forwarded between loops and during gradient calculations. + + Note that `while_loop` calls `cond` and `body` *exactly once* (inside the + call to `while_loop`, and not at all during `Session.run()`). `while_loop` + stitches together the graph fragments created during the `cond` and `body` + calls with some additional graph nodes to create the graph flow that + repeats `body` until `cond` returns false. + + For correctness, `tf.while_loop()` strictly enforces shape invariants for + the loop variables. A shape invariant is a (possibly partial) shape that + is unchanged across the iterations of the loop. An error will be raised + if the shape of a loop variable after an iteration is determined to be more + general than or incompatible with its shape invariant. For example, a shape + of [11, None] is more general than a shape of [11, 17], and [11, 21] is not + compatible with [11, 17]. By default (if the argument `shape_invariants` is + not specified), it is assumed that the initial shape of each tensor in + `loop_vars` is the same in every iteration. The `shape_invariants` argument + allows the caller to specify a less specific shape invariant for each loop + variable, which is needed if the shape varies between iterations. The + `tf.Tensor.set_shape` + function may also be used in the `body` function to indicate that + the output loop variable has a particular shape. The shape invariant for + SparseTensor and IndexedSlices are treated specially as follows: + + a) If a loop variable is a SparseTensor, the shape invariant must be + TensorShape([r]) where r is the rank of the dense tensor represented + by the sparse tensor. It means the shapes of the three tensors of the + SparseTensor are ([None], [None, r], [r]). NOTE: The shape invariant here + is the shape of the SparseTensor.dense_shape property. It must be the shape of + a vector. + + b) If a loop variable is an IndexedSlices, the shape invariant must be + a shape invariant of the values tensor of the IndexedSlices. It means + the shapes of the three tensors of the IndexedSlices are (shape, [shape[0]], + [shape.ndims]). + + `while_loop` implements non-strict semantics, enabling multiple iterations + to run in parallel. The maximum number of parallel iterations can be + controlled by `parallel_iterations`, which gives users some control over + memory consumption and execution order. For correct programs, `while_loop` + should return the same result for any parallel_iterations > 0. + + For training, TensorFlow stores the tensors that are produced in the + forward inference and are needed in back propagation. These tensors are a + main source of memory consumption and often cause OOM errors when training + on GPUs. When the flag swap_memory is true, we swap out these tensors from + GPU to CPU. This for example allows us to train RNN models with very long + sequences and large batches. + + Args: + cond: A callable that represents the termination condition of the loop. + body: A callable that represents the loop body. + loop_vars: A (possibly nested) tuple, namedtuple or list of numpy array, + `Tensor`, and `TensorArray` objects. + shape_invariants: The shape invariants for the loop variables. + parallel_iterations: The number of iterations allowed to run in parallel. It + must be a positive integer. + back_prop: Whether backprop is enabled for this while loop. + swap_memory: Whether GPU-CPU memory swap is enabled for this loop. + maximum_iterations: Optional maximum number of iterations of the while loop + to run. If provided, the `cond` output is AND-ed with an additional + condition ensuring the number of iterations executed is no greater than + `maximum_iterations`. + name: Optional name prefix for the returned tensors. + + Returns: + The output tensors for the loop variables after the loop. The return value + has the same structure as `loop_vars`. + + Raises: + TypeError: if `cond` or `body` is not callable. + ValueError: if `loop_vars` is empty. + + Example: + + ```python + i = tf.constant(0) + c = lambda i: tf.less(i, 10) + b = lambda i: tf.add(i, 1) + r = tf.while_loop(c, b, [i]) + ``` + + Example with nesting and a namedtuple: + + ```python + import collections + Pair = collections.namedtuple('Pair', 'j, k') + ijk_0 = (tf.constant(0), Pair(tf.constant(1), tf.constant(2))) + c = lambda i, p: i < 10 + b = lambda i, p: (i + 1, Pair((p.j + p.k), (p.j - p.k))) + ijk_final = tf.while_loop(c, b, ijk_0) + ``` + + Example using shape_invariants: + + ```python + i0 = tf.constant(0) + m0 = tf.ones([2, 2]) + c = lambda i, m: i < 10 + b = lambda i, m: [i+1, tf.concat([m, m], axis=0)] + tf.while_loop( + c, b, loop_vars=[i0, m0], + shape_invariants=[i0.get_shape(), tf.TensorShape([None, 2])]) + ``` + + Example which demonstrates non-strict semantics: In the following + example, the final value of the counter `i` does not depend on `x`. So + the `while_loop` can increment the counter parallel to updates of `x`. + However, because the loop counter at one loop iteration depends + on the value at the previous iteration, the loop counter itself cannot + be incremented in parallel. Hence if we just want the final value of the + counter (which we print on the line `print(sess.run(i))`), then + `x` will never be incremented, but the counter will be updated on a + single thread. Conversely, if we want the value of the output (which we + print on the line `print(sess.run(out).shape)`), then the counter may be + incremented on its own thread, while `x` can be incremented in + parallel on a separate thread. In the extreme case, it is conceivable + that the thread incrementing the counter runs until completion before + `x` is incremented even a single time. The only thing that can never + happen is that the thread updating `x` can never get ahead of the + counter thread because the thread incrementing `x` depends on the value + of the counter. + + ```python + import tensorflow as tf + + n = 10000 + x = tf.constant(list(range(n))) + c = lambda i, x: i < n + b = lambda i, x: (tf.Print(i + 1, [i]), tf.Print(x + 1, [i], "x:")) + i, out = tf.while_loop(c, b, (0, x)) + with tf.Session() as sess: + print(sess.run(i)) # prints [0] ... [9999] + + # The following line may increment the counter and x in parallel. + # The counter thread may get ahead of the other thread, but not the + # other way around. So you may see things like + # [9996] x:[9987] + # meaning that the counter thread is on iteration 9996, + # while the other thread is on iteration 9987 + print(sess.run(out).shape) + ``` + + """ + return while_loop( + cond=cond, + body=body, + loop_vars=loop_vars, + shape_invariants=shape_invariants, + parallel_iterations=parallel_iterations, + back_prop=back_prop, + swap_memory=swap_memory, + name=name, + maximum_iterations=maximum_iterations, + return_same_structure=True) + + +# pylint: disable=redefined-outer-name +@tf_export(v1=["while_loop"]) def while_loop(cond, body, loop_vars, @@ -3244,7 +3494,8 @@ def while_loop(cond, loop_vars, shape_invariants=shape_invariants, maximum_iterations=maximum_iterations, - name=name) + name=name, + return_same_structure=return_same_structure) with ops.name_scope(name, "while", loop_vars): if not loop_vars: @@ -3465,7 +3716,43 @@ def group(*inputs, **kwargs): return no_op(name=name) -@tf_export("tuple") +@tf_export("tuple", v1=[]) +def tuple_v2(tensors, control_inputs=None, name=None): + """Group tensors together. + + This creates a tuple of tensors with the same values as the `tensors` + argument, except that the value of each tensor is only returned after the + values of all tensors have been computed. + + `control_inputs` contains additional ops that have to finish before this op + finishes, but whose outputs are not returned. + + This can be used as a "join" mechanism for parallel computations: all the + argument tensors can be computed in parallel, but the values of any tensor + returned by `tuple` are only available after all the parallel computations + are done. + + See also `tf.group` and + `tf.control_dependencies`. + + Args: + tensors: A list of `Tensor`s or `IndexedSlices`, some entries can be `None`. + control_inputs: List of additional ops to finish before returning. + name: (optional) A name to use as a `name_scope` for the operation. + + Returns: + Same as `tensors`. + + Raises: + ValueError: If `tensors` does not contain any `Tensor` or `IndexedSlices`. + TypeError: If `control_inputs` is not a list of `Operation` or `Tensor` + objects. + + """ + return tuple(tensors=tensors, name=name, control_inputs=control_inputs) # pylint: disable=redefined-builtin + + +@tf_export(v1=["tuple"]) def tuple(tensors, name=None, control_inputs=None): # pylint: disable=redefined-builtin """Group tensors together. diff --git a/tensorflow/python/ops/control_flow_ops_test.py b/tensorflow/python/ops/control_flow_ops_test.py index c3514c183c4..c020189ad63 100644 --- a/tensorflow/python/ops/control_flow_ops_test.py +++ b/tensorflow/python/ops/control_flow_ops_test.py @@ -155,9 +155,9 @@ class WithDependenciesTestCase(test_util.TensorFlowTestCase): constant_op.constant(7)) with self.cached_session(): variables.global_variables_initializer().run() - self.assertEquals(0, counter.eval()) - self.assertEquals(7, const_with_dep.eval()) - self.assertEquals(1, counter.eval()) + self.assertEquals(0, self.evaluate(counter)) + self.assertEquals(7, self.evaluate(const_with_dep)) + self.assertEquals(1, self.evaluate(counter)) def testListDependencies(self): with ops.Graph().as_default(): @@ -169,9 +169,9 @@ class WithDependenciesTestCase(test_util.TensorFlowTestCase): constant_op.constant(7)) with self.cached_session(): variables.global_variables_initializer().run() - self.assertEquals(0, counter.eval()) - self.assertEquals(7, const_with_dep.eval()) - self.assertEquals(1, counter.eval()) + self.assertEquals(0, self.evaluate(counter)) + self.assertEquals(7, self.evaluate(const_with_dep)) + self.assertEquals(1, self.evaluate(counter)) class SwitchTestCase(test_util.TensorFlowTestCase): @@ -209,9 +209,9 @@ class SwitchTestCase(test_util.TensorFlowTestCase): optimizer = momentum.MomentumOptimizer(0.1, 0.9) train_op = optimizer.minimize(cost) with self.cached_session() as sess: - sess.run(variables.global_variables_initializer()) + self.evaluate(variables.global_variables_initializer()) for _ in range(10): - sess.run([train_op]) + self.evaluate([train_op]) def testResourceReadInLoop(self): with ops.Graph().as_default(): @@ -232,8 +232,8 @@ class SwitchTestCase(test_util.TensorFlowTestCase): cond, body, [constant_op.constant(0), constant_op.constant(0.0)]) with self.cached_session() as sess: - sess.run(variables.global_variables_initializer()) - self.assertAllEqual(10.0, cost.eval()) + self.evaluate(variables.global_variables_initializer()) + self.assertAllEqual(10.0, self.evaluate(cost)) def doTestIndexedSlicesGradientInCondInWhileLoop(self, use_resource=False): with ops.Graph().as_default(): @@ -269,8 +269,8 @@ class SwitchTestCase(test_util.TensorFlowTestCase): static_grads.indices) with self.cached_session() as sess: - sess.run(variables.global_variables_initializer()) - self.assertAllEqual(*sess.run([static_grads, dynamic_grads])) + self.evaluate(variables.global_variables_initializer()) + self.assertAllEqual(*self.evaluate([static_grads, dynamic_grads])) def testIndexedSlicesGradientInCondInWhileLoop(self): self.doTestIndexedSlicesGradientInCondInWhileLoop(use_resource=False) @@ -398,9 +398,9 @@ class CondTest(test_util.TensorFlowTestCase): pred=bool_var, true_fn=lambda: state_ops.assign(bool_var, False), false_fn=lambda: True) - sess.run(bool_var.initializer) - self.assertEquals(sess.run(cond_on_bool_var), False) - self.assertEquals(sess.run(cond_on_bool_var), True) + self.evaluate(bool_var.initializer) + self.assertEquals(self.evaluate(cond_on_bool_var), False) + self.assertEquals(self.evaluate(cond_on_bool_var), True) def testCondMissingArg1(self): with ops.Graph().as_default(): diff --git a/tensorflow/python/ops/control_flow_util.py b/tensorflow/python/ops/control_flow_util.py index 72c074ed1af..cb628f4aa64 100644 --- a/tensorflow/python/ops/control_flow_util.py +++ b/tensorflow/python/ops/control_flow_util.py @@ -38,6 +38,11 @@ def IsInXLAContext(op): return GetContainingXLAContext(ctxt) is not None +def InXlaContext(graph): + ctxt = graph._get_control_flow_context() # pylint: disable=protected-access + return GetContainingXLAContext(ctxt) is not None + + def IsInWhileLoop(op): ctxt = op._get_control_flow_context() # pylint: disable=protected-access return GetContainingWhileContext(ctxt) is not None diff --git a/tensorflow/python/ops/control_flow_util_v2.py b/tensorflow/python/ops/control_flow_util_v2.py index cab1d7b02e1..5f56850884a 100644 --- a/tensorflow/python/ops/control_flow_util_v2.py +++ b/tensorflow/python/ops/control_flow_util_v2.py @@ -19,10 +19,12 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function +from tensorflow.core.framework import attr_value_pb2 from tensorflow.python.eager import context from tensorflow.python.eager import function from tensorflow.python.framework import ops from tensorflow.python.framework.func_graph import FuncGraph +from tensorflow.python.ops import control_flow_util class CondBranchFuncGraph(FuncGraph): @@ -90,3 +92,31 @@ def unique_fn_name(scope, name): def unique_grad_fn_name(forward_name): return "%s_grad_%s" % (forward_name, ops.uid()) + + +def maybe_set_lowering_attr(op): + """Sets the flag to enable lowering on `op` if necessary. + + Lowering allows cond_v2 and while_v2 to avoid some of the limitations of + Functions, allowing users to specify devices & colocation inside of cond_v2 + and while_v2 input functions, and enabling non-strict evaluation & partial + pruning. This brings v2 control flow closer to feature parity with v1 control + flow. + + However, we do not lower in the following cases: + - When the `If` or `While` ops are in the XLA context. Because it is easier + for XLA to apply its own optimizations when dealing with un-lowered + control flow operators than with low-level control flow primitives. + - When the eager execution context specifies the executor of functions to + be the single threaded executor (see context.function_executor_type()). + Because the single threaded executor does not support v1 control flow ops. + + Args: + op: An `If` or `While` Operation. + """ + if (not control_flow_util.IsInXLAContext(op) and + context.context().get_function_call_options().executor_type + != "SINGLE_THREADED_EXECUTOR"): + # pylint: disable=protected-access + op._set_attr("_lower_using_switch_merge", attr_value_pb2.AttrValue(b=True)) + # pylint: enable=protected-access diff --git a/tensorflow/python/ops/ctc_ops.py b/tensorflow/python/ops/ctc_ops.py index e1071afd8e0..3a7eb9355a6 100644 --- a/tensorflow/python/ops/ctc_ops.py +++ b/tensorflow/python/ops/ctc_ops.py @@ -19,17 +19,27 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function +from tensorflow.python.framework import constant_op +from tensorflow.python.framework import dtypes +from tensorflow.python.framework import function from tensorflow.python.framework import ops from tensorflow.python.framework import sparse_tensor from tensorflow.python.ops import array_ops +from tensorflow.python.ops import functional_ops from tensorflow.python.ops import gen_ctc_ops +from tensorflow.python.ops import inplace_ops +from tensorflow.python.ops import linalg_ops +from tensorflow.python.ops import math_ops +from tensorflow.python.ops import nn_ops +from tensorflow.python.ops import sparse_ops from tensorflow.python.ops.nn_grad import _BroadcastMul +from tensorflow.python.util import nest from tensorflow.python.util.tf_export import tf_export # pylint: disable=protected-access, invalid-name -@tf_export("nn.ctc_loss") +@tf_export(v1=["nn.ctc_loss"]) def ctc_loss(labels, inputs, sequence_length, preprocess_collapse_repeated=False, ctc_merge_repeated=True, @@ -336,6 +346,785 @@ def ctc_beam_search_decoder_v2(inputs, sequence_length, beam_width=100, ops.NotDifferentiable("CTCGreedyDecoder") - - ops.NotDifferentiable("CTCBeamSearchDecoder") + + +def _ctc_state_trans(label_seq): + """Compute CTC alignment model transition matrix. + + Args: + label_seq: tensor of shape [batch_size, max_seq_length] + + Returns: + tensor of shape [batch_size, states, states] with a state transition matrix + computed for each sequence of the batch. + """ + + with ops.name_scope("ctc_state_trans"): + label_seq = ops.convert_to_tensor(label_seq, name="label_seq") + batch_size = _get_dim(label_seq, 0) + num_labels = _get_dim(label_seq, 1) + + num_label_states = num_labels + 1 + num_states = 2 * num_label_states + + label_states = math_ops.range(num_label_states) + blank_states = label_states + num_label_states + + # Start state to first label. + start_to_label = [[1, 0]] + + # Blank to label transitions. + blank_to_label = array_ops.stack([label_states[1:], blank_states[:-1]], 1) + + # Label to blank transitions. + label_to_blank = array_ops.stack([blank_states, label_states], 1) + + # Scatter transitions that don't depend on sequence. + indices = array_ops.concat( + [start_to_label, blank_to_label, label_to_blank], 0) + values = array_ops.ones([_get_dim(indices, 0)]) + trans = array_ops.scatter_nd( + indices, values, shape=[num_states, num_states]) + trans += linalg_ops.eye(num_states) # Self-loops. + + # Label to label transitions. Disallow transitions between repeated labels + # with no blank state in between. + batch_idx = array_ops.zeros_like(label_states[2:]) + indices = array_ops.stack( + [batch_idx, label_states[2:], label_states[1:-1]], 1) + indices = array_ops.tile( + array_ops.expand_dims(indices, 0), [batch_size, 1, 1]) + batch_idx = array_ops.expand_dims(math_ops.range(batch_size), 1) * [1, 0, 0] + indices += array_ops.expand_dims(batch_idx, 1) + repeats = math_ops.equal(label_seq[:, :-1], label_seq[:, 1:]) + values = 1.0 - math_ops.cast(repeats, dtypes.float32) + batched_shape = [batch_size, num_states, num_states] + label_to_label = array_ops.scatter_nd(indices, values, batched_shape) + + return array_ops.expand_dims(trans, 0) + label_to_label + + +def ctc_state_log_probs(seq_lengths, max_seq_length): + """Computes CTC alignment initial and final state log probabilities. + + Create the initial/final state values directly as log values to avoid + having to take a float64 log on tpu (which does not exist). + + Args: + seq_lengths: int tensor of shape [batch_size], seq lengths in the batch. + max_seq_length: int, max sequence length possible. + + Returns: + initial_state_log_probs, final_state_log_probs + """ + + batch_size = _get_dim(seq_lengths, 0) + num_label_states = max_seq_length + 1 + num_duration_states = 2 + num_states = num_duration_states * num_label_states + log_0 = math_ops.cast( + math_ops.log(math_ops.cast(0, dtypes.float64) + 1e-307), + dtypes.float32) + + initial_state_log_probs = array_ops.one_hot( + indices=array_ops.zeros([batch_size], dtype=dtypes.int32), + depth=num_states, + on_value=0.0, + off_value=log_0, axis=1) + + label_final_state_mask = array_ops.one_hot( + seq_lengths, depth=num_label_states, axis=0) + duration_final_state_mask = array_ops.ones( + [num_duration_states, 1, batch_size]) + final_state_mask = duration_final_state_mask * label_final_state_mask + final_state_log_probs = (1.0 - final_state_mask) * log_0 + final_state_log_probs = array_ops.reshape( + final_state_log_probs, [num_states, batch_size]) + + return initial_state_log_probs, array_ops.transpose(final_state_log_probs) + + +def _ilabel_to_state(labels, num_labels, ilabel_log_probs): + """Project ilabel log probs to state log probs.""" + + num_label_states = _get_dim(labels, 1) + blank = ilabel_log_probs[:, :, :1] + blank = array_ops.tile(blank, [1, 1, num_label_states + 1]) + one_hot = array_ops.one_hot(labels, depth=num_labels) + one_hot = array_ops.expand_dims(one_hot, axis=0) + ilabel_log_probs = array_ops.expand_dims(ilabel_log_probs, axis=2) + state_log_probs = math_ops.reduce_sum(ilabel_log_probs * one_hot, axis=3) + state_log_probs = array_ops.concat([state_log_probs, blank], axis=2) + return array_ops.pad( + state_log_probs, [[0, 0], [0, 0], [1, 0]], + constant_values=math_ops.log(0.0)) + + +def _state_to_olabel(labels, num_labels, states): + """Sum state log probs to ilabel log probs.""" + + num_label_states = _get_dim(labels, 1) + 1 + label_states = states[:, :, 1:num_label_states] + blank_states = states[:, :, num_label_states:] + one_hot = array_ops.one_hot( + labels - 1, depth=(num_labels - 1), + on_value=0.0, off_value=math_ops.log(0.0)) + one_hot = array_ops.expand_dims(one_hot, axis=0) + label_states = array_ops.expand_dims(label_states, axis=3) + label_olabels = math_ops.reduce_logsumexp(label_states + one_hot, axis=2) + blank_olabels = math_ops.reduce_logsumexp( + blank_states, axis=2, keepdims=True) + return array_ops.concat([blank_olabels, label_olabels], axis=-1) + + +# pylint: disable=redefined-outer-name +def _state_to_olabel_unique(labels, num_labels, states, unique): + """Sum state log probs to ilabel log probs using unique label indices.""" + + num_label_states = _get_dim(labels, 1) + 1 + label_states = states[:, :, 1:num_label_states] + blank_states = states[:, :, num_label_states:] + + unique_y, unique_idx = unique + mul_reduce = _sum_states(unique_idx, label_states) + + num_frames = states.shape[0] + batch_size = states.shape[1] + num_states = num_label_states - 1 + batch_state_major = array_ops.transpose(mul_reduce, perm=[1, 2, 0]) + batch_state_major = array_ops.reshape( + batch_state_major, [batch_size * num_states, num_frames]) + batch_offset = math_ops.range(batch_size, dtype=unique_y.dtype) * num_labels + indices = unique_y + array_ops.expand_dims(batch_offset, axis=-1) + indices = array_ops.reshape(indices, [-1, 1]) + scatter = array_ops.scatter_nd( + indices=indices, + updates=batch_state_major, + shape=[batch_size * num_labels, num_frames]) + scatter = array_ops.reshape(scatter, [batch_size, num_labels, num_frames]) + scatter = array_ops.where( + math_ops.equal(scatter, 0.0), + array_ops.fill(array_ops.shape(scatter), math_ops.log(0.0)), + scatter) + label_olabels = array_ops.transpose(scatter, [2, 0, 1]) + label_olabels = label_olabels[:, :, 1:] + + blank_olabels = math_ops.reduce_logsumexp( + blank_states, axis=2, keepdims=True) + + return array_ops.concat([blank_olabels, label_olabels], axis=-1) + + +def ctc_loss_and_grad(logits, labels, label_length, logit_length, unique=None): + """Computes the CTC loss and gradients. + + Most users will want fwd_bwd.ctc_loss + + This function returns the computed gradient, it does not have a gradient + of its own defined. + + Args: + logits: tensor of shape [frames, batch_size, num_labels] + labels: tensor of shape [batch_size, max_label_seq_length] + label_length: tensor of shape [batch_size] + Length of reference label sequence in labels. + logit_length: tensor of shape [batch_size] + Length of input sequence in logits. + unique: (optional) unique label indices as computed by unique(labels) + If supplied, enables an implementation that is faster and more memory + efficient on TPU. + + Returns: + loss: tensor of shape [batch_size] + gradient: tensor of shape [frames, batch_size, num_labels] + """ + + num_labels = _get_dim(logits, 2) + max_label_seq_length = _get_dim(labels, 1) + + ilabel_log_probs = nn_ops.log_softmax(logits) + state_log_probs = _ilabel_to_state(labels, num_labels, ilabel_log_probs) + state_trans_probs = _ctc_state_trans(labels) + initial_state_log_probs, final_state_log_probs = ctc_state_log_probs( + label_length, max_label_seq_length) + fwd_bwd_log_probs, log_likelihood = _forward_backward_log( + state_trans_log_probs=math_ops.log(state_trans_probs), + initial_state_log_probs=initial_state_log_probs, + final_state_log_probs=final_state_log_probs, + observed_log_probs=state_log_probs, + sequence_length=logit_length) + + if unique: + olabel_log_probs = _state_to_olabel_unique( + labels, num_labels, fwd_bwd_log_probs, unique) + else: + olabel_log_probs = _state_to_olabel(labels, num_labels, fwd_bwd_log_probs) + + grad = math_ops.exp(ilabel_log_probs) - math_ops.exp(olabel_log_probs) + loss = -log_likelihood + return loss, grad + + +def _ctc_loss_grad(op, grad_loss, _): + grad = op.outputs[1] + grad = [array_ops.reshape(grad_loss, [1, -1, 1]) * grad] + grad += [None] * (len(op.inputs) - len(grad)) + return grad + + +def _ctc_loss_shape(op): + return [op.inputs[2].get_shape(), op.inputs[0].get_shape()] + + +@tf_export("nn.ctc_loss", v1=["nn.ctc_loss_v2"]) +def ctc_loss_v2(labels, logits, label_length, logit_length, + logits_time_major=True, unique=None, + blank_index=None, name=None): + """Computes CTC (Connectionist Temporal Classification) loss. + + This op implements the CTC loss as presented in the article: + + [A. Graves, S. Fernandez, F. Gomez, J. Schmidhuber. + Connectionist Temporal Classification: Labeling Unsegmented Sequence Data + with Recurrent Neural Networks. ICML 2006, Pittsburgh, USA, + pp. 369-376.](http://www.cs.toronto.edu/~graves/icml_2006.pdf) + + Notes: + - Same as the "Classic CTC" in TensorFlow 1.x's tf.nn.ctc_loss setting of + preprocess_collapse_repeated=False, ctc_merge_repeated=True + - Labels may be supplied as either a dense, zero-padded tensor with a + vector of label sequence lengths OR as a SparseTensor. + - On TPU and GPU: + - Only dense padded labels are supported. + - On CPU: + - Caller may use SparseTensor or dense padded labels but calling with + a SparseTensor will be significantly faster. + - Default blank label is 0 rather num_classes - 1, unless overridden by + blank_index. + + Args: + labels: tensor of shape [batch_size, max_label_seq_length] or SparseTensor + logits: tensor of shape [frames, batch_size, num_labels], + if logits_time_major == False, shape is [batch_size, frames, num_labels]. + label_length: tensor of shape [batch_size], None if labels is SparseTensor + Length of reference label sequence in labels. + logit_length: tensor of shape [batch_size] + Length of input sequence in logits. + logits_time_major: (optional) If True (default), logits is shaped + [time, batch, logits]. If False, shape is [batch, time, logits] + unique: (optional) Unique label indices as computed by + ctc_unique_labels(labels). If supplied, enable a faster, memory + efficient implementation on TPU. + blank_index: (optional) Set the class index to use for the blank label. + Negative values will start from num_classes, ie, -1 will reproduce the + ctc_loss behavior of using num_classes - 1 for the blank symbol. + There is some memory/performance overhead to switching from the default + of 0 as an additional shifted copy of the logits may be created. + name: A name for this `Op`. Defaults to "ctc_loss_dense". + + Returns: + loss: tensor of shape [batch_size], negative log probabilities. + """ + if isinstance(labels, sparse_tensor.SparseTensor): + if blank_index is None: + raise ValueError( + "blank_index must be given when using SparseTensor labels.") + + if blank_index < 0: + blank_index += _get_dim(logits, 2) + + if blank_index != _get_dim(logits, 2) - 1: + logits = array_ops.concat([ + logits[:, :, :blank_index], + logits[:, :, blank_index+1:], + logits[:, :, blank_index:blank_index+1], + ], axis=2) + labels = sparse_tensor.SparseTensor( + labels.indices, + array_ops.where(labels.values < blank_index, + labels.values, + labels.values - 1), + labels.dense_shape) + + return ctc_loss(labels=labels, + inputs=logits, + sequence_length=logit_length, + time_major=logits_time_major) + + if blank_index is None: + blank_index = 0 + + return ctc_loss_dense(labels=labels, + logits=logits, + label_length=label_length, + logit_length=logit_length, + logits_time_major=logits_time_major, + unique=unique, + blank_index=blank_index, + name=name) + + +def ctc_loss_dense(labels, logits, label_length, logit_length, + logits_time_major=True, unique=None, + blank_index=0, name=None): + """Computes CTC (Connectionist Temporal Classification) loss. + + This op implements the CTC loss as presented in the article: + + [A. Graves, S. Fernandez, F. Gomez, J. Schmidhuber. + Connectionist Temporal Classification: Labeling Unsegmented Sequence Data + with Recurrent Neural Networks. ICML 2006, Pittsburgh, USA, + pp. 369-376.](http://www.cs.toronto.edu/~graves/icml_2006.pdf) + + Using the batched forward backward algorithm described in: + + [Sim, K. C., Narayanan, A., Bagby, T., Sainath, T. N., & Bacchiani, M. + Improving the efficiency of forward-backward algorithm using batched + computation in TensorFlow. + Automatic Speech Recognition and Understanding Workshop (ASRU), + 2017 IEEE (pp. 258-264). + ](https://ieeexplore.ieee.org/iel7/8260578/8268903/08268944.pdf) + + Notes: + Significant differences from tf.nn.ctc_loss: + Supports GPU and TPU (tf.nn.ctc_loss supports CPU only): + For batched operations, GPU and TPU are significantly faster than using + ctc_loss on CPU. + This implementation runs on CPU, but significantly slower than ctc_loss. + Blank label is 0 rather num_classes - 1, unless overridden by blank_index. + Logits and labels are dense arrays with padding rather than SparseTensor. + The only mode supported is the same as: + preprocess_collapse_repeated=False, ctc_merge_repeated=True + To collapse labels, the caller can preprocess label sequence first. + + The dense implementation supports both CPU, GPU and TPU. A fast path is + provided that significantly improves memory use for large vocabulary if the + caller preprocesses label sequences to get unique label indices on the CPU + (eg. in the data input pipeline) using ctc_ops.unique and simplies this in + the optional "unique" kwarg. This is especially useful for TPU and GPU but + also works with if used on CPU. + + Args: + labels: tensor of shape [batch_size, max_label_seq_length] + logits: tensor of shape [frames, batch_size, num_labels], + if logits_time_major == False, shape is [batch_size, frames, num_labels]. + label_length: tensor of shape [batch_size] + Length of reference label sequence in labels. + logit_length: tensor of shape [batch_size] + Length of input sequence in logits. + logits_time_major: (optional) If True (default), logits is shaped + [time, batch, logits]. If False, shape is [batch, time, logits] + unique: (optional) Unique label indices as computed by unique(labels). + If supplied, enable a faster, memory efficient implementation on TPU. + blank_index: (optional) Set the class index to use for the blank label. + Negative values will start from num_classes, ie, -1 will reproduce the + ctc_loss behavior of using num_classes - 1 for the blank symbol. + There is some memory/performance overhead to switching from the default + of 0 as an additional shifted copy of the logits may be created. + name: A name for this `Op`. Defaults to "ctc_loss_dense". + + Returns: + loss: tensor of shape [batch_size], negative log probabilities. + """ + + with ops.name_scope(name, "ctc_loss_dense", + [logits, labels, label_length, logit_length]): + logits = ops.convert_to_tensor(logits, name="logits") + labels = ops.convert_to_tensor(labels, name="labels") + label_length = ops.convert_to_tensor(label_length, name="label_length") + logit_length = ops.convert_to_tensor(logit_length, name="logit_length") + + if not logits_time_major: + logits = array_ops.transpose(logits, perm=[1, 0, 2]) + + if blank_index != 0: + if blank_index < 0: + blank_index += _get_dim(logits, 2) + logits = array_ops.concat([ + logits[:, :, blank_index:blank_index+1], + logits[:, :, :blank_index], + logits[:, :, blank_index+1:], + ], axis=2) + labels = array_ops.where(labels < blank_index, labels + 1, labels) + + args = [logits, labels, label_length, logit_length] + + if unique: + unique_y, unique_idx = unique + args.extend([unique_y, unique_idx]) + + # TODO(tombagby): Update to tfe.defun + @function.Defun(*[x.dtype for x in args], + python_grad_func=_ctc_loss_grad, + shape_func=_ctc_loss_shape) + def compute_ctc_loss(logits_t, labels_t, label_length_t, logit_length_t, + *unique_t): + """Compute CTC loss.""" + logits_t.set_shape(logits.shape) + labels_t.set_shape(labels.shape) + label_length_t.set_shape(label_length.shape) + logit_length_t.set_shape(logit_length.shape) + kwargs = dict( + logits=logits_t, + labels=labels_t, + label_length=label_length_t, + logit_length=logit_length_t) + if unique_t: + kwargs["unique"] = unique_t + return ctc_loss_and_grad(**kwargs) + + return compute_ctc_loss(*args)[0] + + +@tf_export("nn.collapse_repeated") +def collapse_repeated(labels, seq_length, name=None): + """Merge repeated labels into single labels. + + Args: + labels: Tensor of shape (batch, max value in seq_length) + seq_length: Tensor of shape (batch), sequence length of each batch element. + name: A name for this `Op`. Defaults to "collapse_repeated_labels". + + Returns: + tuple of Tensor of shape (batch, max_seq_length) with repeated labels + collapsed and padded to max_seq_length, eg: + [[A, A, B, B, A], + [A, B, C, D, E]] => [[A, B, A, 0, 0], + [A, B, C, D, E]] + and int tensor of shape [batch] with new sequence lengths. + """ + + with ops.name_scope(name, "collapse_repeated_labels", + [labels, seq_length]): + labels = ops.convert_to_tensor(labels, name="labels") + seq_length = ops.convert_to_tensor(seq_length, name="seq_length") + + # Mask labels that don't equal previous label. + label_mask = array_ops.concat( + [array_ops.ones_like(labels[:, :1], dtypes.bool), + math_ops.not_equal(labels[:, 1:], labels[:, :-1])], + axis=1) + + # Filter labels that aren't in the original sequence. + maxlen = _get_dim(labels, 1) + seq_mask = array_ops.sequence_mask(seq_length, maxlen=maxlen) + label_mask = math_ops.logical_and(label_mask, seq_mask) + + # Count masks for new sequence lengths. + new_seq_len = math_ops.reduce_sum( + math_ops.cast(label_mask, dtypes.int32), axis=1) + + # Mask indexes based on sequence length mask. + new_maxlen = math_ops.reduce_max(new_seq_len) + idx_mask = array_ops.sequence_mask(new_seq_len, maxlen=new_maxlen) + + # Flatten everything and mask out labels to keep and sparse indices. + flat_labels = array_ops.reshape(labels, [-1]) + flat_label_mask = array_ops.reshape(label_mask, [-1]) + flat_idx_mask = array_ops.reshape(idx_mask, [-1]) + idx = math_ops.range(_get_dim(flat_idx_mask, 0)) + + # Scatter to flat shape. + flat = array_ops.scatter_nd( + indices=array_ops.expand_dims( + array_ops.boolean_mask(idx, flat_idx_mask), axis=1), + updates=array_ops.boolean_mask(flat_labels, flat_label_mask), + shape=array_ops.shape(flat_idx_mask)) + + # Reshape back to square batch. + batch_size = _get_dim(labels, 0) + new_shape = [batch_size, new_maxlen] + return (array_ops.reshape(flat, new_shape), + math_ops.cast(new_seq_len, seq_length.dtype)) + + +def dense_labels_to_sparse(dense, length): + """Convert dense labels with sequence lengths to sparse tensor. + + Args: + dense: tensor of shape [batch, max_length] + length: int tensor of shape [batch] + The length of each sequence in dense. + + Returns: + tf.SparseTensor with values only for the valid elements of sequences. + """ + + flat_values = array_ops.reshape(dense, [-1]) + flat_indices = math_ops.range( + array_ops.shape(flat_values, out_type=dtypes.int64)[0]) + mask = array_ops.sequence_mask(length, maxlen=array_ops.shape(dense)[1]) + flat_mask = array_ops.reshape(mask, [-1]) + indices = array_ops.expand_dims( + array_ops.boolean_mask(flat_indices, flat_mask), 1) + values = array_ops.boolean_mask(flat_values, flat_mask) + sparse = sparse_tensor.SparseTensor( + indices=indices, values=math_ops.cast(values, dtypes.int32), + dense_shape=array_ops.shape(flat_values, out_type=dtypes.int64)) + reshaped = sparse_ops.sparse_reshape(sparse, array_ops.shape(dense)) + max_length = math_ops.reduce_max(length) + return sparse_tensor.SparseTensor( + indices=reshaped.indices, + values=reshaped.values, + dense_shape=[ + math_ops.cast(reshaped.dense_shape[0], dtypes.int64), + math_ops.cast(max_length, dtypes.int64)]) + + +@tf_export("nn.ctc_unique_labels") +def ctc_unique_labels(labels, name=None): + """Get unique labels and indices for batched labels for tf.nn.ctc_loss. + + For use with tf.nn.ctc_loss_v2 optional argument `unique`: This op can be + used to preprocess labels in input pipeline to for better speed/memory use + computing the ctc loss on TPU. + + Example: + ctc_unique_labels([[3, 4, 4, 3]]) -> + unique labels padded with 0: [[3, 4, 0, 0]] + indices of original labels in unique: [0, 1, 1, 0] + + Args: + labels: tensor of shape [batch_size, max_label_length] padded with 0. + name: A name for this `Op`. Defaults to "ctc_unique_labels". + + Returns: + tuple of + - unique labels, tensor of shape `[batch_size, max_label_length]` + - indices into unique labels, shape `[batch_size, max_label_length]` + """ + + with ops.name_scope(name, "ctc_unique_labels", [labels]): + labels = ops.convert_to_tensor(labels, name="labels") + def _unique(x): + u = array_ops.unique(x) + y = array_ops.pad( + u.y, [[0, _get_dim(u.idx, 0) - _get_dim(u.y, 0)]]) + y = math_ops.cast(y, dtypes.int64) + return [y, u.idx] + return functional_ops.map_fn( + _unique, labels, dtype=[dtypes.int64, dtypes.int32]) + + +def _sum_states(idx, states): + """Take logsumexp for each unique state out of all label states. + + Args: + idx: tensor of shape [batch, label_length] + For each sequence, indices into a set of unique labels as computed by + calling unique. + states: tensor of shape [frames, batch, label_length] + Log probabilities for each label state. + + Returns: + tensor of shape [frames, batch_size, label_length], log probabilites summed + for each unique label of the sequence. + """ + + with ops.name_scope("sum_states"): + idx = ops.convert_to_tensor(idx, name="idx") + num_states = _get_dim(states, 2) + states = array_ops.expand_dims(states, axis=2) + one_hot = array_ops.one_hot( + idx, depth=num_states, on_value=0.0, off_value=math_ops.log(0.0), + axis=1) + return math_ops.reduce_logsumexp(states + one_hot, axis=-1) + + +def _forward_backward_log(state_trans_log_probs, initial_state_log_probs, + final_state_log_probs, observed_log_probs, + sequence_length): + """Forward-backward algorithm computed in log domain. + + Args: + state_trans_log_probs: tensor of shape [states, states] or + if different transition matrix per batch [batch_size, states, states] + initial_state_log_probs: tensor of shape [batch_size, states] + final_state_log_probs: tensor of shape [batch_size, states] + observed_log_probs: tensor of shape [frames, batch_size, states] + sequence_length: tensor of shape [batch_size] + + Returns: + forward backward log probabilites: tensor of shape [frames, batch, states] + log_likelihood: tensor of shape [batch_size] + + Raises: + ValueError: If state_trans_log_probs has unknown or incorrect rank. + """ + + if state_trans_log_probs.shape.ndims == 2: + perm = [1, 0] + elif state_trans_log_probs.shape.ndims == 3: + perm = [0, 2, 1] + else: + raise ValueError( + "state_trans_log_probs rank must be known and == 2 or 3, is: %s" % + state_trans_log_probs.shape.ndims) + + bwd_state_trans_log_probs = array_ops.transpose(state_trans_log_probs, perm) + batch_size = _get_dim(observed_log_probs, 1) + + def _forward(state_log_prob, obs_log_prob): + state_log_prob = array_ops.expand_dims(state_log_prob, axis=1) # Broadcast. + state_log_prob += state_trans_log_probs + state_log_prob = math_ops.reduce_logsumexp(state_log_prob, axis=-1) + state_log_prob += obs_log_prob + log_prob_sum = math_ops.reduce_logsumexp( + state_log_prob, axis=-1, keepdims=True) + state_log_prob -= log_prob_sum + return state_log_prob + + fwd = _scan(_forward, observed_log_probs, initial_state_log_probs, + inclusive=True) + + def _backward(accs, elems): + """Calculate log probs and cumulative sum masked for sequence length.""" + state_log_prob, cum_log_sum = accs + obs_log_prob, mask = elems + state_log_prob += obs_log_prob + state_log_prob = array_ops.expand_dims(state_log_prob, axis=1) # Broadcast. + state_log_prob += bwd_state_trans_log_probs + state_log_prob = math_ops.reduce_logsumexp(state_log_prob, axis=-1) + + log_prob_sum = math_ops.reduce_logsumexp( + state_log_prob, axis=-1, keepdims=True) + state_log_prob -= log_prob_sum + + cum_log_sum += array_ops.squeeze(log_prob_sum) * mask + batched_mask = array_ops.expand_dims(mask, axis=1) + out = state_log_prob * batched_mask + out += final_state_log_probs * (1.0 - batched_mask) + return out, cum_log_sum + + zero_log_sum = array_ops.zeros([batch_size]) + maxlen = _get_dim(observed_log_probs, 0) + mask = array_ops.sequence_mask(sequence_length, maxlen, dtypes.float32) + mask = array_ops.transpose(mask, perm=[1, 0]) + + bwd, cum_log_sum = _scan(_backward, (observed_log_probs, mask), + (final_state_log_probs, zero_log_sum), + reverse=True, inclusive=True) + + fwd_bwd_log_probs = fwd[1:] + bwd[1:] + fwd_bwd_log_probs_sum = math_ops.reduce_logsumexp( + fwd_bwd_log_probs, axis=2, keepdims=True) + fwd_bwd_log_probs -= fwd_bwd_log_probs_sum + fwd_bwd_log_probs += math_ops.log(array_ops.expand_dims(mask, axis=2)) + + log_likelihood = bwd[0, :, 0] + cum_log_sum[0] + + return fwd_bwd_log_probs, log_likelihood + + +# TODO(tombagby): This is currently faster for the ctc implementation than using +# functional_ops.scan, but could be replaced by that or something similar if +# things change. +def _scan(fn, elems, initial, reverse=False, inclusive=False, final_only=False): + """Repeatedly applies callable `fn` to a sequence of elements. + + Implemented by functional_ops.While, tpu friendly, no gradient. + + This is similar to functional_ops.scan but significantly faster on tpu/gpu + for the forward backward use case. + + Examples: + scan(lambda a, e: a + e, [1.0, 2.0, 3.0], 1.0) => [2.0, 3.0, 4.0] + + Multiple accumulators: + scan(lambda a, e: (a[0] + e, a[1] * e), [1.0, 2.0, 3.0], (0.0, 1.0)) + + Multiple inputs: + scan(lambda a, e: a + (e[0] * e[1]), (elems1, elems2), 0.0) + + Args: + fn: callable, fn(accumulators, element) return new accumulator values. + The (possibly nested) sequence of accumulators is the same as `initial` + and the return value must have the same structure. + elems: A (possibly nested) tensor which will be unpacked along the first + dimension. The resulting slices will be the second argument to fn. The + first dimension of all nested input tensors must be the same. + initial: A tensor or (possibly nested) sequence of tensors with initial + values for the accumulators. + reverse: (optional) True enables scan and output elems in reverse order. + inclusive: (optional) True includes the initial accumulator values in the + output. Length of output will be len(elem sequence) + 1. Not meaningful + if final_only is True. + final_only: (optional) When True, return only the final accumulated values, + not the concatenation of accumulated values for each input. + + Returns: + A (possibly nested) sequence of tensors with the results of applying fn + to tensors unpacked from elems and previous accumulator values. + """ + + flat_elems = [ops.convert_to_tensor(x) for x in nest.flatten(elems)] + num_elems = array_ops.shape(flat_elems[0])[0] + pack_elems = lambda x: nest.pack_sequence_as(structure=elems, flat_sequence=x) + flat_initial = [ops.convert_to_tensor(x) for x in nest.flatten(initial)] + pack = lambda x: nest.pack_sequence_as(structure=initial, flat_sequence=x) + accum_dtypes = [x.dtype for x in flat_initial] + num_accums = len(flat_initial) + + # Types for counter, [outputs], [accumulators] loop arguments. + if final_only: + loop_dtypes = [dtypes.int32, dtypes.int32] + accum_dtypes + else: + loop_dtypes = [dtypes.int32, dtypes.int32] + accum_dtypes + accum_dtypes + + # TODO(tombagby): Update to tfe.defun + @function.Defun(*loop_dtypes) + def cond(i, num_elems, *args): + del args + return i >= 0 if reverse else i < num_elems + + # The loop *args are [output tensors] + [accumulator tensors] which must + # be paired. Each output corresponds to one accumulator. + @function.Defun(*loop_dtypes) + def body(i, num_elems, *args): + """Loop body.""" + i.set_shape([]) + if final_only: + accum = args + else: + out, accum = args[:num_accums], args[num_accums:] + slices = [array_ops.gather(e, i) for e in flat_elems] + accum = fn(pack(accum), pack_elems(slices)) + flat_accum = nest.flatten(accum) + if final_only: + new_out = [] + else: + update_i = i + 1 if inclusive and not reverse else i + new_out = [inplace_ops.alias_inplace_update(x, update_i, y) + for x, y in zip(out, flat_accum)] + i = i - 1 if reverse else i + 1 + return [i, num_elems] + new_out + flat_accum + + init_i = (array_ops.shape(flat_elems[0])[0] - 1 if reverse + else constant_op.constant(0, dtype=dtypes.int32)) + outputs = [] + if not final_only: + num_outputs = array_ops.shape(flat_elems[0])[0] + (1 if inclusive else 0) + for initial_accum in flat_initial: + out_shape = array_ops.concat( + [[num_outputs], array_ops.shape(initial_accum)], 0) + out = inplace_ops.empty(out_shape, dtype=initial_accum.dtype, init=True) + if inclusive: + out = inplace_ops.alias_inplace_add( + out, init_i + (1 if reverse else 0), initial_accum) + outputs.append(out) + loop_in = [init_i, num_elems] + outputs + flat_initial + hostmem = [ + i for i, x in enumerate(loop_in) + if x.dtype.base_dtype in (dtypes.int32, dtypes.int64) + ] + + # TODO(tombagby): Update to while_v2. + loop_results = functional_ops.While(loop_in, cond, body, hostmem=hostmem) + out = loop_results[2:num_accums + 2] + return pack(out) + + +def _get_dim(tensor, i): + """Get value of tensor shape[i] preferring static value if available.""" + return tensor.shape[i].value or array_ops.shape(tensor)[i] diff --git a/tensorflow/python/ops/data_flow_ops.py b/tensorflow/python/ops/data_flow_ops.py index cca8e12b434..2030332e4ea 100644 --- a/tensorflow/python/ops/data_flow_ops.py +++ b/tensorflow/python/ops/data_flow_ops.py @@ -79,7 +79,7 @@ def _as_shape_list(shapes, shapes = [shapes] shapes = [tensor_shape.as_shape(shape) for shape in shapes] if not unknown_dim_allowed: - if any([not shape.is_fully_defined() for shape in shapes]): + if any(not shape.is_fully_defined() for shape in shapes): raise ValueError("All shapes must be fully defined: %s" % shapes) if not unknown_rank_allowed: if any([shape.dims is None for shape in shapes]): @@ -171,7 +171,10 @@ class QueueBase(object): self._names = None self._queue_ref = queue_ref if context.executing_eagerly(): - self._name = context.context().scope_name + if context.context().scope_name: + self._name = context.context().scope_name + else: + self._name = "Empty" self._resource_deleter = resource_variable_ops.EagerResourceDeleter( queue_ref, None) else: @@ -198,11 +201,11 @@ class QueueBase(object): raise TypeError("A list of queues expected") dtypes = queues[0].dtypes - if not all([dtypes == q.dtypes for q in queues[1:]]): + if not all(dtypes == q.dtypes for q in queues[1:]): raise TypeError("Queues do not have matching component dtypes.") names = queues[0].names - if not all([names == q.names for q in queues[1:]]): + if not all(names == q.names for q in queues[1:]): raise TypeError("Queues do not have matching component names.") queue_shapes = [q.shapes for q in queues] @@ -1148,7 +1151,7 @@ class Barrier(object): self._barrier_ref, name=name) -@tf_export("ConditionalAccumulatorBase") +@tf_export(v1=["ConditionalAccumulatorBase"]) class ConditionalAccumulatorBase(object): """A conditional accumulator for aggregating gradients. @@ -1227,7 +1230,7 @@ class ConditionalAccumulatorBase(object): name=name) -@tf_export("ConditionalAccumulator") +@tf_export(v1=["ConditionalAccumulator"]) class ConditionalAccumulator(ConditionalAccumulatorBase): """A conditional accumulator for aggregating gradients. diff --git a/tensorflow/python/ops/dequantize_op_test.py b/tensorflow/python/ops/dequantize_op_test.py index 13e50273d86..794985b2dbb 100644 --- a/tensorflow/python/ops/dequantize_op_test.py +++ b/tensorflow/python/ops/dequantize_op_test.py @@ -35,7 +35,7 @@ class DequantizeOpTest(test.TestCase): with self.cached_session(): input_op = constant_op.constant(inputs, shape=[len(inputs)], dtype=dtype) dequantized = array_ops.dequantize(input_op, min_range, max_range) - tf_ans = dequantized.eval() + tf_ans = self.evaluate(dequantized) # TODO(vrv): Add support for DT_QINT32 quantization if needed. type_dict = { diff --git a/tensorflow/python/ops/distributions/util.py b/tensorflow/python/ops/distributions/util.py index 760e7a8a84b..24314e8fc92 100644 --- a/tensorflow/python/ops/distributions/util.py +++ b/tensorflow/python/ops/distributions/util.py @@ -343,7 +343,7 @@ def embed_check_categorical_event_shape( x_dtype = x.dtype.base_dtype max_event_size = (_largest_integer_by_dtype(x_dtype) if x_dtype.is_floating else 0) - if max_event_size is 0: + if max_event_size == 0: raise TypeError("Unable to validate size of unrecognized dtype " "({}).".format(x_dtype.name)) try: diff --git a/tensorflow/python/ops/embedding_ops.py b/tensorflow/python/ops/embedding_ops.py index 9ce024ad965..b398601e6f0 100644 --- a/tensorflow/python/ops/embedding_ops.py +++ b/tensorflow/python/ops/embedding_ops.py @@ -554,7 +554,10 @@ def safe_embedding_lookup_sparse(embedding_weights, dtype = sparse_weights.dtype if sparse_weights is not None else None embedding_weights = [ - ops.convert_to_tensor(w, dtype=dtype) for w in embedding_weights + w if (isinstance(w, resource_variable_ops.ResourceVariable) + and dtype in (None, w.dtype)) + else ops.convert_to_tensor(w, dtype=dtype) + for w in embedding_weights ] with ops.name_scope(name, 'embedding_lookup', diff --git a/tensorflow/python/ops/functional_ops.py b/tensorflow/python/ops/functional_ops.py index fecd7ddbf9f..57542e3c7ba 100644 --- a/tensorflow/python/ops/functional_ops.py +++ b/tensorflow/python/ops/functional_ops.py @@ -19,7 +19,7 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function -from tensorflow.core.protobuf import rewriter_config_pb2 +from tensorflow.core.protobuf import config_pb2 from tensorflow.core.framework import attr_value_pb2 from tensorflow.python.eager import context from tensorflow.python.framework import constant_op @@ -1027,9 +1027,10 @@ _rewriter_config_optimizer_disabled = None def _get_disabled_rewriter_config(): global _rewriter_config_optimizer_disabled if _rewriter_config_optimizer_disabled is None: - rewriter_config = rewriter_config_pb2.RewriterConfig() + config = config_pb2.ConfigProto() + rewriter_config = config.graph_options.rewrite_options rewriter_config.disable_meta_optimizer = True - _rewriter_config_optimizer_disabled = rewriter_config.SerializeToString() + _rewriter_config_optimizer_disabled = config.SerializeToString() return _rewriter_config_optimizer_disabled @@ -1048,7 +1049,7 @@ def partitioned_call(args, f, tout=None, executing_eagerly=None, config=None, the signature of `f`. executing_eagerly: (Optional) A boolean indicating whether the context is executing eagerly. If `None`, fetched from the global context. - config: (Optional) A tensorflow::RewriterConfig proto, serialized. If + config: (Optional) A `tensorflow::ConfigProto` proto, serialized. If `None`, all optimizations are disabled. Currently only handled for eager defined functions. executor_type: (Optional) A string for the name of the executor to be used @@ -1076,10 +1077,12 @@ def partitioned_call(args, f, tout=None, executing_eagerly=None, config=None, if executing_eagerly or len(tout): if f.stateful_ops: outputs = gen_functional_ops.stateful_partitioned_call( - args=args, Tout=tout, f=f, config=config, executor_type=executor_type) + args=args, Tout=tout, f=f, config_proto=config, + executor_type=executor_type) else: outputs = gen_functional_ops.partitioned_call( - args=args, Tout=tout, f=f, config=config, executor_type=executor_type) + args=args, Tout=tout, f=f, config_proto=config, + executor_type=executor_type) return outputs if outputs else None # The generated binding returns an empty list for functions that don't @@ -1098,7 +1101,7 @@ def partitioned_call(args, f, tout=None, executing_eagerly=None, config=None, # When running in graph mode, the graph and function graphs are optimized # (i.e. run through grappler) per the session options, so we can disable any # eager-specific rewriting. - rewriter_config = attr_value_pb2.AttrValue(s=_get_disabled_rewriter_config()) + config_proto = attr_value_pb2.AttrValue(s=_get_disabled_rewriter_config()) graph = ops.get_default_graph() f.add_to_graph(graph) @@ -1113,7 +1116,7 @@ def partitioned_call(args, f, tout=None, executing_eagerly=None, config=None, "Tin": tin_attr, "Tout": tout_attr, "f": func_attr, - "config": rewriter_config, + "config_proto": config_proto, "executor_type": executor_type_attr, }) outputs = op.outputs diff --git a/tensorflow/python/ops/gradient_checker.py b/tensorflow/python/ops/gradient_checker.py index 1665219c80c..683f78ce9b2 100644 --- a/tensorflow/python/ops/gradient_checker.py +++ b/tensorflow/python/ops/gradient_checker.py @@ -31,7 +31,6 @@ from tensorflow.python.ops import array_ops from tensorflow.python.ops import gradients from tensorflow.python.ops import math_ops from tensorflow.python.platform import tf_logging as logging -from tensorflow.python.util import deprecation from tensorflow.python.util.tf_export import tf_export @@ -158,7 +157,8 @@ def _compute_numeric_jacobian(x, x_shape, x_data, y, y_shape, delta, # as delta. Convert to float32 here. Since numeric_jacobian is expected to # be the groundtruth to compare against, it shouldn't lose any information. if x.dtype == dtypes.bfloat16: - x = math_ops.cast(x, dtypes.float32) + x = math_ops.cast(x, dtypes.float32) # TODO(wangpeng): Now that the new x + # is an output of the old x, isn't feeding to the new x a mistake? if y.dtype == dtypes.bfloat16: y = math_ops.cast(y, dtypes.float32) if x_data.dtype == dtypes.bfloat16.as_numpy_dtype: @@ -266,7 +266,7 @@ def _compute_gradient_list(x, return ret -@tf_export("test.compute_gradient") +@tf_export(v1=["test.compute_gradient"]) def compute_gradient(x, x_shape, y, @@ -301,7 +301,6 @@ def compute_gradient(x, as the initial value. delta: (optional) the amount of perturbation. init_targets: list of targets to run to initialize model params. - TODO(mrry): remove this argument. extra_feed_dict: dict that allows fixing specified tensor values during the Jacobian calculation. @@ -311,6 +310,7 @@ def compute_gradient(x, where "x_size" is the number of elements in x and "y_size" is the number of elements in y. If x is a list, returns a list of two numpy arrays. """ + # TODO(mrry): remove argument `init_targets` if extra_feed_dict is None: extra_feed_dict = {} @@ -328,10 +328,17 @@ def compute_gradient(x, return ret +def _compute_error(grad): + if isinstance(grad, tuple): + grad = [grad] + error = 0 + for j_t, j_n in grad: + if j_t.size or j_n.size: # Handle zero size tensors correctly + error = np.maximum(error, np.fabs(j_t - j_n).max()) + return error + + @tf_export(v1=["test.compute_gradient_error"]) -@deprecation.deprecated_args( - None, "init_targets will be deprecated in TensorFlow 2.0", - ("init_targets", None)) # Do not trigger warning in V2 def compute_gradient_error(x, x_shape, y, @@ -373,59 +380,4 @@ def compute_gradient_error(x, """ grad = compute_gradient(x, x_shape, y, y_shape, x_init_value, delta, init_targets, extra_feed_dict=extra_feed_dict) - if isinstance(grad, tuple): - grad = [grad] - error = 0 - for j_t, j_n in grad: - if j_t.size or j_n.size: # Handle zero size tensors correctly - error = np.maximum(error, np.fabs(j_t - j_n).max()) - return error - - -@tf_export("test.compute_gradient_error", v1=[]) -def compute_gradient_error_v2(x, - x_shape, - y, - y_shape, - x_init_value=None, - delta=1e-3, - extra_feed_dict=None): - """Computes the gradient error. - - Computes the maximum error for dy/dx between the computed Jacobian and the - numerically estimated Jacobian. - - This function will modify the tensors passed in as it adds more operations - and hence changing the consumers of the operations of the input tensors. - - This function adds operations to the current session. To compute the error - using a particular device, such as a GPU, use the standard methods for - setting a device (e.g. using with sess.graph.device() or setting a device - function in the session constructor). - - Args: - x: a tensor or list of tensors - x_shape: the dimensions of x as a tuple or an array of ints. If x is a list, - then this is the list of shapes. - y: a tensor - y_shape: the dimensions of y as a tuple or an array of ints. - x_init_value: (optional) a numpy array of the same shape as "x" representing - the initial value of x. If x is a list, this should be a list of numpy - arrays. If this is none, the function will pick a random tensor as the - initial value. - delta: (optional) the amount of perturbation. - extra_feed_dict: dict that allows fixing specified tensor values during the - Jacobian calculation. - - Returns: - The maximum error in between the two Jacobians. - """ - return compute_gradient_error( - x, - x_shape, - y, - y_shape, - x_init_value=x_init_value, - delta=delta, - init_targets=None, - extra_feed_dict=extra_feed_dict) + return _compute_error(grad) diff --git a/tensorflow/python/ops/gradient_checker_test.py b/tensorflow/python/ops/gradient_checker_test.py index 66c7b9a71b5..4d2b5efac7b 100644 --- a/tensorflow/python/ops/gradient_checker_test.py +++ b/tensorflow/python/ops/gradient_checker_test.py @@ -23,6 +23,7 @@ import numpy as np from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops +from tensorflow.python.framework import test_util from tensorflow.python.ops import array_ops from tensorflow.python.ops import gradient_checker from tensorflow.python.ops import math_ops @@ -46,6 +47,7 @@ def _nan_grad(unused_op, grad): class GradientCheckerTest(test.TestCase): + @test_util.run_deprecated_v1 def testAddSimple(self): np.random.seed(1) # Fix seed to avoid flakiness with self.session(use_gpu=False): @@ -60,6 +62,7 @@ class GradientCheckerTest(test.TestCase): tf_logging.info("x1 error = %f", error) assert error < 1e-4 + @test_util.run_deprecated_v1 def testAddSimpleGPU(self): np.random.seed(2) # Fix seed to avoid flakiness with self.session(use_gpu=True): @@ -74,6 +77,7 @@ class GradientCheckerTest(test.TestCase): tf_logging.info("x1 error = %f", error) assert error < 1e-4 + @test_util.run_deprecated_v1 def testAddCustomized(self): np.random.seed(3) # Fix seed to avoid flakiness with self.cached_session(): @@ -92,6 +96,7 @@ class GradientCheckerTest(test.TestCase): tf_logging.info("x2 error = %f", error) assert error < 1e-10 + @test_util.run_deprecated_v1 def testGather(self): np.random.seed(4) # Fix seed to avoid flakiness with self.cached_session(): @@ -109,6 +114,7 @@ class GradientCheckerTest(test.TestCase): tf_logging.info("gather error = %f", error) assert error < 1e-4 + @test_util.run_deprecated_v1 def testNestedGather(self): np.random.seed(5) # Fix seed to avoid flakiness with self.cached_session(): @@ -130,6 +136,7 @@ class GradientCheckerTest(test.TestCase): tf_logging.info("nested gather error = %f", error) assert error < 1e-4 + @test_util.run_deprecated_v1 def testComplexMul(self): with self.cached_session(): size = () @@ -144,6 +151,7 @@ class GradientCheckerTest(test.TestCase): self.assertLess( gradient_checker.compute_gradient_error(x, size, y, size), 2e-4) + @test_util.run_deprecated_v1 def testComplexConj(self): with self.cached_session(): size = () @@ -157,6 +165,7 @@ class GradientCheckerTest(test.TestCase): self.assertLess( gradient_checker.compute_gradient_error(x, size, y, size), 2e-5) + @test_util.run_deprecated_v1 def testEmptySucceeds(self): with self.cached_session(): x = array_ops.placeholder(dtypes.float32) @@ -279,18 +288,23 @@ class MiniMNISTTest(test.TestCase): tf_logging.info("Mini MNIST: %s gradient error = %g", tag, err) return err + @test_util.run_deprecated_v1 def testInputGradient(self): self.assertLess(self._BuildAndTestMiniMNIST(0, "input"), 1e-8) + @test_util.run_deprecated_v1 def testHiddenWeightGradient(self): self.assertLess(self._BuildAndTestMiniMNIST(1, "hidden_weight"), 1e-8) + @test_util.run_deprecated_v1 def testHiddenBiasGradient(self): self.assertLess(self._BuildAndTestMiniMNIST(2, "hidden_bias"), 1e-8) + @test_util.run_deprecated_v1 def testSoftmaxWeightGradient(self): self.assertLess(self._BuildAndTestMiniMNIST(3, "softmax_weight"), 1e-8) + @test_util.run_deprecated_v1 def testSoftmaxBiasGradient(self): self.assertLess(self._BuildAndTestMiniMNIST(4, "softmax_bias"), 1e-8) diff --git a/tensorflow/python/ops/gradient_checker_v2.py b/tensorflow/python/ops/gradient_checker_v2.py new file mode 100644 index 00000000000..cf844841127 --- /dev/null +++ b/tensorflow/python/ops/gradient_checker_v2.py @@ -0,0 +1,318 @@ +# Copyright 2015 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +"""Gradient checker for functions. + +The gradient checker verifies numerically that an function properly +computes the gradients +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import numpy as np + +from tensorflow.python.eager import backprop +from tensorflow.python.eager import context +from tensorflow.python.framework import dtypes +from tensorflow.python.framework import ops +from tensorflow.python.ops import array_ops +from tensorflow.python.ops import math_ops +from tensorflow.python.platform import tf_logging as logging +from tensorflow.python.util.tf_export import tf_export + + +def _product(t): + if isinstance(t, int): + return t + else: + y = 1 + for x in t: + y *= x + return y + + +def _to_numpy(a): + """Converts tensors to numpy arrays. + + Converts Tensors and EagerTensors to numpy arrays. + When eager execution is enabled, converts IndexedSlices + to IndexedSlicesValue with numpy indices/values + + Args: + a: any value. + + Returns: + If a is EagerTensor or Tensor, returns the evaluation of a by calling + numpy() or run(). + If a is IndexedSlices and eager execution is enabled, calls numpy() on a's + fields. Otherwise returns a unchanged. + """ + if isinstance(a, ops.EagerTensor): + return a.numpy() + if isinstance(a, ops.Tensor): + sess = ops.get_default_session() + return sess.run(a) + if isinstance(a, ops.IndexedSlices) and context.executing_eagerly(): + return ops.IndexedSlicesValue( + indices=[x.numpy() for x in a.indices], + values=[x.numpy() for x in a.values], + dense_shape=a.dense_shape) + return a + + +def _prepare(f, xs_dtypes): + """Return a function that executes 'f'. + + In TF 2.x, this is the same as `f`. + In TF 1.x, returns a Python function that executes the graph defined by `f` + in a Session. + + Args: + f: the function. + xs_dtypes: dtypes of f's arguments. + + Returns: + a function that will be evaluated in both graph and eager mode + """ + if context.executing_eagerly(): + + def decorated_eager(*xs_data): + return f(*map(ops.convert_to_tensor, xs_data)) + + return decorated_eager + xs = [array_ops.placeholder(x_dtype) for x_dtype in xs_dtypes] + y = f(*xs) + sess = ops.get_default_session() + def decorated_graph(*xs_data): + xs_data = [_to_numpy(a) for a in xs_data] + return sess.run(y, feed_dict=dict(zip(xs, xs_data))) + return decorated_graph + + +def _compute_theoretical_jacobian(f, y_shape, y_dtype, xs, param): + """Computes the theoretical Jacobian for f regarding xs[param]. + + One can think of the relation among f, xs and y as y = f(xs). + + Args: + f: the function. + y_shape: the shape of the result. + y_dtype: the dtype of the result. + xs: a list of tensors. + param: the index of the target parameter. + + Returns: + A 2-d numpy array representing the Jacobian. It has "x_size" rows + and "y_size" columns where "x_size" is the number of elements in xs[param] + and "y_size" is the number of elements in the result. + + Raises: + ValueError: If result is empty but the gradient is nonzero. + """ + x = xs[param] + # Complex vectors are treated as vectors of twice as many reals. + x_shape = tuple(x.shape) + (2,) if x.dtype.is_complex else x.shape + y_factor = 2 if y_dtype.is_complex else 1 + + # To compute the jacobian, we treat x and y as one-dimensional vectors. + x_size = _product(x_shape) + x_val_size = _product(x_shape[1:]) # This is used for sparse gradients + y_size = _product(y_shape) * y_factor + + # Allocate 2-D Jacobian, with x dimensions smashed into the first + # dimension and y dimensions smashed into the second. + jacobian = np.zeros((x_size, y_size), dtype=x.dtype.real_dtype.as_numpy_dtype) + + # For each of the entry of dy, we set this to be 1 and + # everything else to be 0 and compute the gradients -- this will give us one + # one column of the Jacobian matrix. + dy_data = np.zeros(y_shape, dtype=y_dtype.as_numpy_dtype) + dy_data_flat = dy_data.ravel().view(y_dtype.real_dtype.as_numpy_dtype) + grad_fn_unprep = backprop.gradients_function(f, [param]) + grad_fn = _prepare(lambda dy, *xs: grad_fn_unprep(*xs, dy=dy), + [y_dtype] + [x.dtype for x in xs]) + for col in range(y_size): + dy_data_flat[col] = 1 + grad = _to_numpy(grad_fn(dy_data, *xs)[0]) + dy_data_flat[col] = 0 + if isinstance(grad, ops.IndexedSlicesValue): + for i, v in zip(grad.indices, grad.values): + r_begin = i * x_val_size + r_end = r_begin + x_val_size + jacobian[r_begin:r_end, col] += v.flat + else: + jacobian[:, col] = grad.ravel().view(jacobian.dtype) + + # If the output is empty, run the gradients at least once and make sure + # they produce zeros. + if y_size == 0: # don't use 'not y_size', because y_size may not be an int + grad = _to_numpy(grad_fn(dy_data, *xs)[0]) + if grad.shape != x.shape: + raise ValueError("Empty gradient has wrong shape: expected %s, got %s" % + (x.shape, grad.shape)) + if np.any(grad): + raise ValueError("Empty tensor with nonzero gradients") + + logging.vlog(1, "Theoretical Jacobian =\n%s", jacobian) + return jacobian + + +def _compute_numeric_jacobian(f, y_size, y_dtype, xs, param, + delta): + """Computes the numeric Jacobian for f regarding xs[param]. + + One can think of the relation among f, xs and y as y = f(xs). + + Args: + f: the function. + y_size: the number of elements of the result. + y_dtype: the dtype of the result. + xs: a list of tensors. + param: the index of the target parameter. + delta: the amount of perturbation we give to the input. + + Returns: + A 2-d numpy array representing the Jacobian. It has "x_size" rows + and "y_size" columns where "x_size" is the number of elements in xs[param] + and "y_size" is the number of elements in the result. + """ + # bfloat16 doesn't have enough bits to represent high precision numbers such + # as delta. Convert to float32 here. Since numeric_jacobian is expected to + # be the groundtruth to compare against, it shouldn't lose any information. + x_shape = xs[param].shape + x_dtype = xs[param].dtype + if y_dtype == dtypes.bfloat16: + f = lambda *xs: math_ops.cast(f(*xs), dtypes.float32) + y_dtype = dtypes.float32 + + # To compute the jacobian, we treat x and y as one-dimensional vectors + x_size = _product(x_shape) * (2 if x_dtype.is_complex else 1) + y_size = y_size * (2 if y_dtype.is_complex else 1) + x_dtype = x_dtype.real_dtype.as_numpy_dtype + y_dtype = y_dtype.real_dtype.as_numpy_dtype + + xs_dtypes = [x.dtype for x in xs] + # Converts xs to numpy arrays to do in-place perturbation. + # Calls asarray() to avoid copying in ravel() later. + xs = [np.asarray(_to_numpy(x)) for x in xs] + x = xs[param] + + # Make sure we have the right types + scale = np.asarray(2 * delta, dtype=y_dtype)[()] + + jacobian = np.zeros((x_size, y_size), dtype=x_dtype) + # For each of the entry of x, we slightly perturbs this by adding and + # subtracting a delta and then compute difference between the outputs. This + # will give us one row of the Jacobian matrix. + + f = _prepare(f, xs_dtypes) + for row in range(x_size): + original = x.ravel().view(x_dtype)[row] + x.ravel().view(x_dtype)[row] += delta + y_pos = _to_numpy(f(*xs)) + x.ravel().view(x_dtype)[row] = original + x.ravel().view(x_dtype)[row] -= delta + y_neg = _to_numpy(f(*xs)) + x.ravel().view(x_dtype)[row] = original + diff = (y_pos - y_neg) / scale + jacobian[row, :] = diff.ravel().view(y_dtype) + + logging.vlog(1, "Numeric Jacobian =\n%s", jacobian) + return jacobian + + +def _compute_gradient(f, + y_shape, + y_dtype, + xs, + param, + delta): + """Computes the theoretical and numerical jacobian.""" + x = xs[param] + t = x.dtype + allowed_types = [dtypes.float16, dtypes.bfloat16, dtypes.float32, + dtypes.float64, dtypes.complex64, dtypes.complex128] + assert t.base_dtype in allowed_types, ("Cannot compute gradient for" + "unsupported type %s of argument %s" % + (t.name, param)) + t2 = y_dtype + assert t2.base_dtype in allowed_types, ("Cannot compute gradient for" + "unsupported type %s of y" % t2.name) + y_size = _product(y_shape) + jacob_t = _compute_theoretical_jacobian(f, y_shape, y_dtype, + xs, param) + jacob_n = _compute_numeric_jacobian(f, y_size, y_dtype, xs, + param, delta) + return jacob_t, jacob_n + + +def _compute_gradient_list(f, xs, delta): + """Compute gradients for a list of x values.""" + # convert xs to tensors so that dtype and shape have uniform types + xs = list(map(ops.convert_to_tensor, xs)) + # run the function to get info of the result + xs_dtypes = [x.dtype for x in xs] + f_temp = _prepare(f, xs_dtypes) + y = f_temp(*xs) + return zip(*[_compute_gradient(f, y.shape, dtypes.as_dtype(y.dtype), + xs, i, delta) for i in range(len(xs))]) + + +@tf_export("test.compute_gradient", v1=[]) +def compute_gradient(f, x, delta=1e-3): + """Computes the theoretical and numeric Jacobian of f. + + With y = f(x), computes the theoretical and numeric Jacobian dy/dx. + + Args: + f: the function. + x: a list of tensors. + delta: (optional) perturbation used to compute numeric Jacobian. + + Returns: + A pair of lists, where the first is a list of 2-d numpy arrays representing + the theoretical Jacobians for each argument, and the second list is the + numerical ones. Each 2-d array has "x_size" rows + and "y_size" columns where "x_size" is the number of elements in the + corresponding argument and "y_size" is the number of elements in f(x). + + Raises: + ValueError: If result is empty but the gradient is nonzero. + """ + if not isinstance(x, list): + raise ValueError( + "`x` must be a list of Tensors (arguments to `f`), not a %s" % type(x)) + return _compute_gradient_list(f, x, delta) + + +def max_error(grad1, grad2): + """Computes maximum elementwise gap. + + Computes the maximum elementwise gap between two lists of tensors of the same + shape. + + Args: + grad1: a lists of tensors. + grad2: a lists of tensors with the same shape as grad1. + + Returns: + The maximum elementwise gap between the two. + """ + error = 0 + for j_t, j_n in zip(grad1, grad2): + if j_t.size or j_n.size: # Handle zero size tensors correctly + error = np.maximum(error, np.fabs(j_t - j_n).max()) + return error diff --git a/tensorflow/python/ops/gradient_checker_v2_test.py b/tensorflow/python/ops/gradient_checker_v2_test.py new file mode 100644 index 00000000000..ce9ff47d617 --- /dev/null +++ b/tensorflow/python/ops/gradient_checker_v2_test.py @@ -0,0 +1,300 @@ +# Copyright 2015 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Tests for compute_gradient. +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import numpy as np + +from tensorflow.python.eager import backprop +from tensorflow.python.eager import context +from tensorflow.python.framework import constant_op +from tensorflow.python.framework import dtypes +from tensorflow.python.framework import test_util +from tensorflow.python.ops import array_ops +from tensorflow.python.ops import custom_gradient +from tensorflow.python.ops import \ +gradient_checker_v2 as gradient_checker +from tensorflow.python.ops import math_ops +from tensorflow.python.ops import nn_ops +# needs this to register gradient for SoftmaxCrossEntropyWithLogits: +import tensorflow.python.ops.nn_grad # pylint: disable=unused-import +from tensorflow.python.platform import test +from tensorflow.python.platform import tf_logging + + +@test_util.run_all_in_graph_and_eager_modes +class GradientCheckerTest(test.TestCase): + + def testAddSimple(self): + # if context.executing_eagerly(): + # return + np.random.seed(1) # Fix seed to avoid flakiness + size = (2, 3) + x1 = constant_op.constant(2.0, shape=size, name="x1") + x2 = constant_op.constant(3.0, shape=size, name="x2") + error = gradient_checker.max_error(*gradient_checker.compute_gradient( + lambda x1: math_ops.add(x1, x2), [x1])) + tf_logging.info("x1 error = %f", error) + assert error < 1e-4 + + def testAddCustomized(self): + np.random.seed(3) # Fix seed to avoid flakiness + size = (2, 3) + x1 = constant_op.constant( + 2.0, shape=size, dtype=dtypes.float64, name="x1") + x2 = np.asarray(np.arange(6, dtype=np.float64).reshape(2, 3)) + # checkint gradients for x2 using a special delta + error = gradient_checker.max_error(*gradient_checker.compute_gradient( + lambda x2: math_ops.add(x1, x2), + [x2], delta=1e-2)) + tf_logging.info("x2 error = %f", error) + assert error < 1e-10 + + def testGather(self): + np.random.seed(4) # Fix seed to avoid flakiness + def f(params): + index_values = [1, 3] + indices = constant_op.constant(index_values, name="i") + return array_ops.gather(params, indices, name="y") + p_shape = (4, 2) + p_size = 8 + params = constant_op.constant( + np.arange(p_size).astype(np.float), shape=p_shape, name="p") + error = gradient_checker.max_error(*gradient_checker.compute_gradient( + f, [params])) + tf_logging.info("gather error = %f", error) + assert error < 1e-4 + + def testNestedGather(self): + np.random.seed(5) # Fix seed to avoid flakiness + def f(params): + index_values = [1, 3, 5, 6] + indices = constant_op.constant(index_values, name="i") + y = array_ops.gather(params, indices, name="y") + index_values2 = [0, 2] + indices2 = constant_op.constant(index_values2, name="i2") + return array_ops.gather(y, indices2, name="y2") + p_shape = (8, 2) + p_size = 16 + params = constant_op.constant( + np.arange(p_size).astype(np.float), shape=p_shape, name="p") + error = gradient_checker.max_error(*gradient_checker.compute_gradient( + f, [params])) + tf_logging.info("nested gather error = %f", error) + assert error < 1e-4 + + def testComplexMul(self): + if not context.executing_eagerly(): + return + c = constant_op.constant(5 + 7j, dtype=dtypes.complex64) + def f(x): + return c * x + x = constant_op.constant(11 - 13j, dtype=dtypes.complex64) + analytical, numerical = gradient_checker.compute_gradient( + f, [x], delta=0.1) + correct = np.array([[5, 7], [-7, 5]]) + self.assertAllEqual(correct, analytical[0]) + self.assertAllClose(correct, numerical[0], rtol=1e-4) + self.assertLess( + gradient_checker.max_error(*gradient_checker.compute_gradient( + f, [x], delta=0.1)), 2e-4) + + def testComplexConj(self): + def f(x): + return math_ops.conj(x) + x = constant_op.constant(11 - 13j, dtype=dtypes.complex64) + analytical, numerical = gradient_checker.compute_gradient( + f, [x], delta=0.1) + correct = np.array([[1, 0], [0, -1]]) + self.assertAllEqual(correct, analytical[0]) + self.assertAllClose(correct, numerical[0], rtol=2e-5) + self.assertLess( + gradient_checker.max_error(*gradient_checker.compute_gradient( + f, [x], delta=0.1)), 2e-5) + + def testEmptySucceeds(self): + def f(x): + return array_ops.identity(x) + x = constant_op.constant(np.random.random_sample((0, 3)), + dtype=dtypes.float32) + for grad in gradient_checker.compute_gradient(f, [x]): + self.assertEqual(grad[0].shape, (0, 0)) + error = gradient_checker.max_error(*gradient_checker.compute_gradient( + f, [x])) + self.assertEqual(error, 0) + + def testEmptyFails(self): + # if not context.executing_eagerly(): + # return + @custom_gradient.custom_gradient + def id_bad_grad(x): + y = array_ops.identity(x) + def grad_fn(dy): + # dx = constant_op.constant(np.zeros((1, 4)), dtype=dtypes.float32) + dx = array_ops.transpose(dy) + return dx + return y, grad_fn + def f(x): + return id_bad_grad(x) + x = constant_op.constant(np.random.random_sample((0, 3)), + dtype=dtypes.float32) + bad = r"Empty gradient has wrong shape: expected \(0, 3\), got \(3, 0\)" + with self.assertRaisesRegexp(ValueError, bad): + gradient_checker.compute_gradient(f, [x]) + + def testNaNGradFails(self): + @custom_gradient.custom_gradient + def id_nan_grad(x): + y = array_ops.identity(x) + def grad_fn(dy): + dx = np.nan * dy + # dx = dy + return dx + return y, grad_fn + def f(x): + return id_nan_grad(x) + x = constant_op.constant(np.random.random_sample((1, 1)), + dtype=dtypes.float32) + error = gradient_checker.max_error(*gradient_checker.compute_gradient( + f, [x])) + # Typical test would assert error < max_err, so assert this test would + # raise AssertionError, since NaN is not < 1.0. + with self.assertRaisesRegexp(AssertionError, "False is not true"): + self.assertTrue(error < 1.0) + + def testGradGrad(self): + + def f(x): + with backprop.GradientTape() as tape: + tape.watch(x) + y = math_ops.square(x) + z = math_ops.square(y) + return tape.gradient(z, x) + + analytical, numerical = gradient_checker.compute_gradient(f, [2.0]) + self.assertAllEqual([[[48.]]], analytical) + self.assertAllClose([[[48.]]], numerical, rtol=1e-4) + + +@test_util.run_all_in_graph_and_eager_modes +class MiniMNISTTest(test.TestCase): + + # Gradient checker for MNIST. + def _BuildAndTestMiniMNIST(self, param_index, tag): + # Fix seed to avoid occasional flakiness + np.random.seed(6) + + # Hyperparameters + batch = 3 + inputs = 16 + features = 32 + classes = 10 + + # Define the parameters + inp_data = np.random.random_sample(inputs * batch) + hidden_weight_data = np.random.randn(inputs * features) / np.sqrt(inputs) + hidden_bias_data = np.random.random_sample(features) + sm_weight_data = np.random.randn(features * classes) / np.sqrt(features) + sm_bias_data = np.random.random_sample(classes) + + # special care for labels since they need to be normalized per batch + label_data = np.random.random(batch * classes).reshape((batch, classes)) + s = label_data.sum(axis=1) + label_data /= s[:, None] + + # We treat the inputs as "parameters" here + inp = constant_op.constant( + inp_data.tolist(), + shape=[batch, inputs], + dtype=dtypes.float64, + name="inp") + hidden_weight = constant_op.constant( + hidden_weight_data.tolist(), + shape=[inputs, features], + dtype=dtypes.float64, + name="hidden_weight") + hidden_bias = constant_op.constant( + hidden_bias_data.tolist(), + shape=[features], + dtype=dtypes.float64, + name="hidden_bias") + softmax_weight = constant_op.constant( + sm_weight_data.tolist(), + shape=[features, classes], + dtype=dtypes.float64, + name="softmax_weight") + softmax_bias = constant_op.constant( + sm_bias_data.tolist(), + shape=[classes], + dtype=dtypes.float64, + name="softmax_bias") + + # List all the parameter so that we can test them one at a time + all_params = [ + inp, hidden_weight, hidden_bias, softmax_weight, softmax_bias + ] + + # Now, Building MNIST + def f(inp, hidden_weight, hidden_bias, softmax_weight, softmax_bias): + features = nn_ops.relu( + nn_ops.xw_plus_b(inp, hidden_weight, hidden_bias), name="features") + logits = nn_ops.xw_plus_b( + features, softmax_weight, softmax_bias, name="logits") + labels = constant_op.constant( + label_data.tolist(), + shape=[batch, classes], + dtype=dtypes.float64, + name="labels") + cost = nn_ops.softmax_cross_entropy_with_logits( + labels=labels, logits=logits, name="cost") + return cost + + def f_restricted(x): + xs = all_params + i = param_index + # use x for the i-th parameter + xs = xs[0:i]+[x]+xs[i+1:] + return f(*xs) + # Test the gradients. + err = gradient_checker.max_error(*gradient_checker.compute_gradient( + f_restricted, [all_params[param_index]], delta=1e-5)) + + tf_logging.info("Mini MNIST: %s gradient error = %g", tag, err) + return err + + def testInputGradient(self): + # if context.executing_eagerly(): + # return + self.assertLess(self._BuildAndTestMiniMNIST(0, "input"), 1e-8) + + def testHiddenWeightGradient(self): + self.assertLess(self._BuildAndTestMiniMNIST(1, "hidden_weight"), 1e-8) + + def testHiddenBiasGradient(self): + self.assertLess(self._BuildAndTestMiniMNIST(2, "hidden_bias"), 1e-8) + + def testSoftmaxWeightGradient(self): + self.assertLess(self._BuildAndTestMiniMNIST(3, "softmax_weight"), 1e-8) + + def testSoftmaxBiasGradient(self): + self.assertLess(self._BuildAndTestMiniMNIST(4, "softmax_bias"), 1e-8) + + +if __name__ == "__main__": + test.main() diff --git a/tensorflow/python/ops/gradients_impl.py b/tensorflow/python/ops/gradients_impl.py index 4f0fb54dcab..8cc4d926c7f 100644 --- a/tensorflow/python/ops/gradients_impl.py +++ b/tensorflow/python/ops/gradients_impl.py @@ -49,9 +49,9 @@ from tensorflow.python.ops import logging_ops # pylint: disable=unused-import from tensorflow.python.ops import manip_grad # pylint: disable=unused-import from tensorflow.python.ops import math_grad # pylint: disable=unused-import from tensorflow.python.ops import math_ops +from tensorflow.python.ops import optional_grad # pylint: disable=unused-import from tensorflow.python.ops import random_grad # pylint: disable=unused-import from tensorflow.python.ops import resource_variable_ops -from tensorflow.python.ops import spectral_grad # pylint: disable=unused-import from tensorflow.python.ops import tensor_array_ops from tensorflow.python.ops.unconnected_gradients import UnconnectedGradients from tensorflow.python.platform import tf_logging as logging @@ -540,7 +540,7 @@ def _Consumers(t, func_graphs): return consumers -@tf_export("gradients") +@tf_export(v1=["gradients"]) def gradients(ys, xs, grad_ys=None, @@ -656,6 +656,119 @@ def gradients(ys, unconnected_gradients) +@tf_export("gradients", v1=[]) +def gradients_v2(ys, # pylint: disable=invalid-name + xs, + grad_ys=None, + name="gradients", + gate_gradients=False, + aggregation_method=None, + stop_gradients=None, + unconnected_gradients=UnconnectedGradients.NONE): + """Constructs symbolic derivatives of sum of `ys` w.r.t. x in `xs`. + + `ys` and `xs` are each a `Tensor` or a list of tensors. `grad_ys` + is a list of `Tensor`, holding the gradients received by the + `ys`. The list must be the same length as `ys`. + + `gradients()` adds ops to the graph to output the derivatives of `ys` with + respect to `xs`. It returns a list of `Tensor` of length `len(xs)` where + each tensor is the `sum(dy/dx)` for y in `ys`. + + `grad_ys` is a list of tensors of the same length as `ys` that holds + the initial gradients for each y in `ys`. When `grad_ys` is None, + we fill in a tensor of '1's of the shape of y for each y in `ys`. A + user can provide their own initial `grad_ys` to compute the + derivatives using a different initial gradient for each y (e.g., if + one wanted to weight the gradient differently for each value in + each y). + + `stop_gradients` is a `Tensor` or a list of tensors to be considered constant + with respect to all `xs`. These tensors will not be backpropagated through, + as though they had been explicitly disconnected using `stop_gradient`. Among + other things, this allows computation of partial derivatives as opposed to + total derivatives. For example: + + ```python + a = tf.constant(0.) + b = 2 * a + g = tf.gradients(a + b, [a, b], stop_gradients=[a, b]) + ``` + + Here the partial derivatives `g` evaluate to `[1.0, 1.0]`, compared to the + total derivatives `tf.gradients(a + b, [a, b])`, which take into account the + influence of `a` on `b` and evaluate to `[3.0, 1.0]`. Note that the above is + equivalent to: + + ```python + a = tf.stop_gradient(tf.constant(0.)) + b = tf.stop_gradient(2 * a) + g = tf.gradients(a + b, [a, b]) + ``` + + `stop_gradients` provides a way of stopping gradient after the graph has + already been constructed, as compared to `tf.stop_gradient` which is used + during graph construction. When the two approaches are combined, + backpropagation stops at both `tf.stop_gradient` nodes and nodes in + `stop_gradients`, whichever is encountered first. + + All integer tensors are considered constant with respect to all `xs`, as if + they were included in `stop_gradients`. + + `unconnected_gradients` determines the value returned for each x in xs if it + is unconnected in the graph to ys. By default this is None to safeguard + against errors. MAthematically these gradients are zero which can be requested + using the `'zero'` option. `tf.UnconnectedGradients` provides the + following options and behaviors: + + ```python + a = tf.ones([1, 2]) + b = tf.ones([3, 1]) + g1 = tf.gradients([b], [a], unnconnected_gradients='none') + sess.run(g1) # [None] + + g2 = tf.gradients([b], [a], unconnected_gradients='zero') + sess.run(g2) # [array([[0., 0.]], dtype=float32)] + ``` + + + Args: + ys: A `Tensor` or list of tensors to be differentiated. + xs: A `Tensor` or list of tensors to be used for differentiation. + grad_ys: Optional. A `Tensor` or list of tensors the same size as + `ys` and holding the gradients computed for each y in `ys`. + name: Optional name to use for grouping all the gradient ops together. + defaults to 'gradients'. + gate_gradients: If True, add a tuple around the gradients returned + for an operations. This avoids some race conditions. + aggregation_method: Specifies the method used to combine gradient terms. + Accepted values are constants defined in the class `AggregationMethod`. + stop_gradients: Optional. A `Tensor` or list of tensors not to differentiate + through. + unconnected_gradients: Optional. Specifies the gradient value returned when + the given input tensors are unconnected. Accepted values are constants + defined in the class `tf.UnconnectedGradients` and the default value is + `none`. + + Returns: + A list of `sum(dy/dx)` for each x in `xs`. + + Raises: + LookupError: if one of the operations between `x` and `y` does not + have a registered gradient function. + ValueError: if the arguments are invalid. + RuntimeError: if called in Eager mode. + + """ + # Creating the gradient graph for control flow mutates Operations. + # _mutation_lock ensures a Session.run call cannot occur between creating and + # mutating new ops. + with ops.get_default_graph()._mutation_lock(): # pylint: disable=protected-access + return _GradientsHelper(ys, xs, grad_ys, name, True, gate_gradients, + aggregation_method, stop_gradients, + unconnected_gradients) + + def _GradientsHelper(ys, xs, grad_ys=None, @@ -896,7 +1009,7 @@ def _HasAnyNotNoneGrads(grads, op): if isinstance(out_grad, (ops.Tensor, ops.IndexedSlices)): return True if out_grad and isinstance(out_grad, collections.Sequence): - if any([g is not None for g in out_grad]): + if any(g is not None for g in out_grad): return True return False @@ -1111,11 +1224,11 @@ def _AggregatedGrads(grads, assert control_flow_util.IsLoopSwitch(op) continue # Grads have to be Tensors or IndexedSlices - if (isinstance(out_grad, collections.Sequence) and not all([ + if (isinstance(out_grad, collections.Sequence) and not all( isinstance(g, (ops.Tensor, ops.IndexedSlices)) for g in out_grad if g is not None - ])): + )): raise TypeError("gradients have to be either all Tensors " "or all IndexedSlices") # Aggregate multiple gradients, and convert [] to None. @@ -1123,7 +1236,7 @@ def _AggregatedGrads(grads, if len(out_grad) < 2: used = "nop" out_grads[i] = out_grad[0] - elif all([isinstance(g, ops.Tensor) for g in out_grad if g is not None]): + elif all(isinstance(g, ops.Tensor) for g in out_grad if g is not None): tensor_shape = _AccumulatorShape(out_grad) if (aggregation_method == AggregationMethod.EXPERIMENTAL_ACCUMULATE_N and len(out_grad) > 2 and tensor_shape.is_fully_defined()): @@ -1240,7 +1353,7 @@ def _hessian_vector_product(ys, xs, v): return gradients(elemwise_products, xs) -@tf_export("hessians") +@tf_export(v1=["hessians"]) def hessians(ys, xs, name="hessians", @@ -1305,3 +1418,16 @@ def hessians(ys, array_ops.concat((_shape, _shape), 0)) hessians.append(_reshaped_hessian) return hessians + + +@tf_export("hessians", v1=[]) +def HessiansV2(ys, + xs, + gate_gradients=False, + aggregation_method=None, + name="hessians"): + return hessians(ys, xs, name=name, gate_gradients=gate_gradients, + aggregation_method=aggregation_method) + + +HessiansV2.__doc__ = hessians.__doc__ diff --git a/tensorflow/python/ops/gradients_test.py b/tensorflow/python/ops/gradients_test.py index 103e3902b60..a9058c4a341 100644 --- a/tensorflow/python/ops/gradients_test.py +++ b/tensorflow/python/ops/gradients_test.py @@ -144,7 +144,7 @@ class GradientsTest(test_util.TensorFlowTestCase): gate_gradients=True)[0] with session.Session(): # Make sure the placer doesn't complain. - gz_x.eval() + self.evaluate(gz_x) def testBoundaryStop(self): # Test that we don't differentiate 'x'. The gradient function for 'x' is @@ -365,7 +365,7 @@ class GradientsTest(test_util.TensorFlowTestCase): grads = gradients.gradients( [y], [x], unconnected_gradients="zero") with self.cached_session() as sess: - self.assertAllEqual([[0.0, 0.0], [0.0, 0.0]], sess.run(grads)[0]) + self.assertAllEqual([[0.0, 0.0], [0.0, 0.0]], self.evaluate(grads)[0]) def testUnconnectedGradientsZeroConnectedGradients(self): with ops.Graph().as_default(): @@ -374,7 +374,7 @@ class GradientsTest(test_util.TensorFlowTestCase): grad = gradients.gradients( [y], [x], unconnected_gradients="zero") with self.cached_session() as sess: - self.assertEquals(3.0, sess.run(grad)[0]) + self.assertEquals(3.0, self.evaluate(grad)[0]) def testUnknownUnconnectedGradientsValueGiven(self): with ops.Graph().as_default(): @@ -438,8 +438,8 @@ class FunctionGradientsTest(test_util.TensorFlowTestCase): grads = gradients.gradients(y, [x, b1]) with self.cached_session() as sess: - self.assertAllEqual([40.0], sess.run(grads)[0]) - self.assertAllEqual([10.0], sess.run(grads)[1]) + self.assertAllEqual([40.0], self.evaluate(grads)[0]) + self.assertAllEqual([10.0], self.evaluate(grads)[1]) def testFunctionGradientsWithGradFunc(self): g = ops.Graph() @@ -487,7 +487,7 @@ class FunctionGradientsTest(test_util.TensorFlowTestCase): f = Foo() with self.cached_session() as sess: - self.assertEqual(sess.run(f), 2.0) + self.assertEqual(self.evaluate(f), 2.0) def testGradientOfCaptured(self): with ops.Graph().as_default(): @@ -501,7 +501,7 @@ class FunctionGradientsTest(test_util.TensorFlowTestCase): f = Foo() with self.cached_session() as sess: - self.assertEqual(sess.run(f), 2.0) + self.assertEqual(self.evaluate(f), 2.0) def testCapturedResourceVariable(self): with ops.Graph().as_default(): @@ -515,8 +515,8 @@ class FunctionGradientsTest(test_util.TensorFlowTestCase): f = Foo() with self.cached_session() as sess: - sess.run(variables.global_variables_initializer()) - self.assertEqual(sess.run(f), 2.0) + self.evaluate(variables.global_variables_initializer()) + self.assertEqual(self.evaluate(f), 2.0) def testCapturedNested(self): with ops.Graph().as_default(): @@ -541,9 +541,9 @@ class FunctionGradientsTest(test_util.TensorFlowTestCase): x1_grad, x2_grad = Outer() with self.cached_session() as sess: # 1.0 + None + 2.0 + 1.0 = 4.0 - self.assertEqual(sess.run(x1_grad), 4.0) + self.assertEqual(self.evaluate(x1_grad), 4.0) # None + 1.0 + 1.0 + None = 2.0 - self.assertEqual(sess.run(x2_grad), 2.0) + self.assertEqual(self.evaluate(x2_grad), 2.0) def testCapturedFromFunction(self): with ops.Graph().as_default(): @@ -563,7 +563,7 @@ class FunctionGradientsTest(test_util.TensorFlowTestCase): z_grad = Outer() with self.cached_session() as sess: - self.assertEqual(sess.run(z_grad), 3.0) + self.assertEqual(self.evaluate(z_grad), 3.0) def testCapturedEagerTensors(self): # Test that we can handle captured eager tensors unrelated to the gradient @@ -628,7 +628,7 @@ class HessianVectorProductTest(test_util.TensorFlowTestCase): mat_x = math_ops.matmul(mat, x, name="Ax") x_mat_x = math_ops.matmul(array_ops.transpose(x), mat_x, name="xAx") hess_v = gradients_impl._hessian_vector_product(x_mat_x, [x], [v])[0] - hess_v_actual = hess_v.eval() + hess_v_actual = self.evaluate(hess_v) self.assertAllClose(hess_v_value, hess_v_actual) @@ -648,7 +648,7 @@ class HessianTest(test_util.TensorFlowTestCase): x = constant_op.constant(x_value) x_mat_x = math_ops.reduce_sum(x[:, None] * mat * x[None, :]) hess = gradients.hessians(x_mat_x, x)[0] - hess_actual = hess.eval() + hess_actual = self.evaluate(hess) self.assertAllClose(hess_value, hess_actual) def testHessian1D_multi(self): @@ -692,7 +692,7 @@ class HessianTest(test_util.TensorFlowTestCase): math_ops.matmul(array_ops.transpose(x), x) * 0.5 ) hess = gradients.hessians(x_square, x)[0] - hess_actual = hess.eval() + hess_actual = self.evaluate(hess) hess_value = np.bmat([ [elem*np.ones((m, m)) for elem in vec] for vec in np.eye(m) @@ -711,7 +711,7 @@ class HessianTest(test_util.TensorFlowTestCase): math_ops.matmul(array_ops.transpose(x), x) * 0.5 ) hess = gradients.hessians(x_square, x)[0] - hess_actual = hess.eval() + hess_actual = self.evaluate(hess) hess_value = np.bmat([ [elem*np.ones((n, n)) for elem in vec] for vec in np.eye(m) @@ -729,7 +729,7 @@ class IndexedSlicesToTensorTest(test_util.TensorFlowTestCase): c_sparse = math_ops._as_indexed_slices(c) self.assertAllEqual(np_val.shape, c_sparse.dense_shape.eval()) c_dense = math_ops.multiply(c_sparse, 1.0) - self.assertAllClose(np_val, c_dense.eval()) + self.assertAllClose(np_val, self.evaluate(c_dense)) def testIndexedSlicesToTensorList(self): with self.cached_session(): @@ -745,7 +745,7 @@ class IndexedSlicesToTensorTest(test_util.TensorFlowTestCase): sparse_list.append(c_sparse) packed_dense = array_ops.stack(dense_list) packed_sparse = array_ops.stack(sparse_list) - self.assertAllClose(packed_dense.eval(), packed_sparse.eval()) + self.assertAllClose(packed_dense.eval(), self.evaluate(packed_sparse)) def testInt64Indices(self): with self.cached_session(): @@ -757,7 +757,7 @@ class IndexedSlicesToTensorTest(test_util.TensorFlowTestCase): math_ops.cast(c_sparse.indices, dtypes.int64), c_sparse.dense_shape) self.assertAllEqual(np_val.shape, c_sparse.dense_shape.eval()) c_dense = math_ops.multiply(c_sparse, 1.0) - self.assertAllClose(np_val, c_dense.eval()) + self.assertAllClose(np_val, self.evaluate(c_dense)) def testWarnings(self): # TODO(gunan) Reenable after this issue is fixed: @@ -853,7 +853,7 @@ class CustomGradientTest(test_util.TensorFlowTestCase): y = MyIdentity(MyIdentity(x)) dy = gradients.gradients(y, x)[0] with session.Session(): - self.assertEqual(9., dy.eval()) + self.assertEqual(9., self.evaluate(dy)) def testCustomGradient(self): @@ -873,7 +873,7 @@ class CustomGradientTest(test_util.TensorFlowTestCase): y = MyMultiply(x1, x2) dy = gradients.gradients(y, [x1, x2]) with session.Session() as sess: - self.assertAllEqual([3., 5.], sess.run(dy)) + self.assertAllEqual([3., 5.], self.evaluate(dy)) def testCustomGradientErrors(self): @@ -914,7 +914,7 @@ class CustomGradientTest(test_util.TensorFlowTestCase): for g in grads: self.assertTrue(g is not None) with session.Session() as sess: - sess.run(variables.global_variables_initializer()) + self.evaluate(variables.global_variables_initializer()) dw = sess.run(math_ops.reduce_sum(grads[1])) self.assertEqual(12., dw) @@ -1074,7 +1074,7 @@ class TensorListGradientsTest(test_util.TensorFlowTestCase): grad = gradients.gradients(tl, a, grad_ys=grad_tl)[0] with self.cached_session() as sess: - self.assertEquals(sess.run(grad), 5.) + self.assertEquals(self.evaluate(grad), 5.) if __name__ == "__main__": diff --git a/tensorflow/python/ops/histogram_ops_test.py b/tensorflow/python/ops/histogram_ops_test.py index e7fe0efba4e..b48ef67196b 100644 --- a/tensorflow/python/ops/histogram_ops_test.py +++ b/tensorflow/python/ops/histogram_ops_test.py @@ -21,6 +21,7 @@ from __future__ import print_function import numpy as np from tensorflow.python.framework import dtypes +from tensorflow.python.framework import test_util from tensorflow.python.framework import constant_op from tensorflow.python.ops import array_ops from tensorflow.python.ops import histogram_ops @@ -39,7 +40,7 @@ class BinValuesFixedWidth(test.TestCase): bins = histogram_ops.histogram_fixed_width_bins( values, value_range, nbins=5) self.assertEqual(dtypes.int32, bins.dtype) - self.assertAllClose(expected_bins, bins.eval()) + self.assertAllClose(expected_bins, self.evaluate(bins)) def test_1d_values_int32_output(self): # Bins will be: @@ -51,7 +52,7 @@ class BinValuesFixedWidth(test.TestCase): bins = histogram_ops.histogram_fixed_width_bins( values, value_range, nbins=5, dtype=dtypes.int64) self.assertEqual(dtypes.int32, bins.dtype) - self.assertAllClose(expected_bins, bins.eval()) + self.assertAllClose(expected_bins, self.evaluate(bins)) def test_1d_float64_values_int32_output(self): # Bins will be: @@ -63,7 +64,7 @@ class BinValuesFixedWidth(test.TestCase): bins = histogram_ops.histogram_fixed_width_bins( values, value_range, nbins=5) self.assertEqual(dtypes.int32, bins.dtype) - self.assertAllClose(expected_bins, bins.eval()) + self.assertAllClose(expected_bins, self.evaluate(bins)) def test_2d_values(self): # Bins will be: @@ -76,7 +77,7 @@ class BinValuesFixedWidth(test.TestCase): bins = histogram_ops.histogram_fixed_width_bins( values, value_range, nbins=5) self.assertEqual(dtypes.int32, bins.dtype) - self.assertAllClose(expected_bins, bins.eval()) + self.assertAllClose(expected_bins, self.evaluate(bins)) class HistogramFixedWidthTest(test.TestCase): @@ -84,6 +85,7 @@ class HistogramFixedWidthTest(test.TestCase): def setUp(self): self.rng = np.random.RandomState(0) + @test_util.run_deprecated_v1 def test_with_invalid_value_range(self): values = [-1.0, 0.0, 1.5, 2.0, 5.0, 15] with self.assertRaisesRegexp( @@ -92,6 +94,7 @@ class HistogramFixedWidthTest(test.TestCase): with self.assertRaisesRegexp(ValueError, "Dimension must be 2 but is 3"): histogram_ops.histogram_fixed_width(values, [1.0, 2.0, 3.0]) + @test_util.run_deprecated_v1 def test_with_invalid_nbins(self): values = [-1.0, 0.0, 1.5, 2.0, 5.0, 15] with self.assertRaisesRegexp( @@ -110,7 +113,7 @@ class HistogramFixedWidthTest(test.TestCase): with self.session(use_gpu=True): hist = histogram_ops.histogram_fixed_width(values, value_range, nbins=5) self.assertEqual(dtypes.int32, hist.dtype) - self.assertAllClose(expected_bin_counts, hist.eval()) + self.assertAllClose(expected_bin_counts, self.evaluate(hist)) def test_1d_values_int64_output(self): # Bins will be: @@ -122,7 +125,7 @@ class HistogramFixedWidthTest(test.TestCase): hist = histogram_ops.histogram_fixed_width( values, value_range, nbins=5, dtype=dtypes.int64) self.assertEqual(dtypes.int64, hist.dtype) - self.assertAllClose(expected_bin_counts, hist.eval()) + self.assertAllClose(expected_bin_counts, self.evaluate(hist)) def test_1d_float64_values(self): # Bins will be: @@ -133,7 +136,7 @@ class HistogramFixedWidthTest(test.TestCase): with self.session(use_gpu=True): hist = histogram_ops.histogram_fixed_width(values, value_range, nbins=5) self.assertEqual(dtypes.int32, hist.dtype) - self.assertAllClose(expected_bin_counts, hist.eval()) + self.assertAllClose(expected_bin_counts, self.evaluate(hist)) def test_2d_values(self): # Bins will be: @@ -144,8 +147,9 @@ class HistogramFixedWidthTest(test.TestCase): with self.session(use_gpu=True): hist = histogram_ops.histogram_fixed_width(values, value_range, nbins=5) self.assertEqual(dtypes.int32, hist.dtype) - self.assertAllClose(expected_bin_counts, hist.eval()) + self.assertAllClose(expected_bin_counts, self.evaluate(hist)) + @test_util.run_deprecated_v1 def test_shape_inference(self): value_range = [0.0, 5.0] values = [[-1.0, 0.0, 1.5], [2.0, 5.0, 15]] @@ -155,7 +159,7 @@ class HistogramFixedWidthTest(test.TestCase): hist = histogram_ops.histogram_fixed_width(values, value_range, nbins=5) self.assertAllEqual(hist.shape.as_list(), (5,)) self.assertEqual(dtypes.int32, hist.dtype) - self.assertAllClose(expected_bin_counts, hist.eval()) + self.assertAllClose(expected_bin_counts, self.evaluate(hist)) hist = histogram_ops.histogram_fixed_width( values, value_range, nbins=placeholder) diff --git a/tensorflow/python/ops/image_grad_test.py b/tensorflow/python/ops/image_grad_test.py index 32c2f37c0b7..c481266dd71 100644 --- a/tensorflow/python/ops/image_grad_test.py +++ b/tensorflow/python/ops/image_grad_test.py @@ -21,6 +21,7 @@ from __future__ import print_function import numpy as np from tensorflow.python.framework import constant_op +from tensorflow.python.framework import test_util from tensorflow.python.ops import gradient_checker from tensorflow.python.ops import gradients_impl from tensorflow.python.ops import image_ops @@ -44,9 +45,10 @@ class ResizeNearestNeighborOpTest(test.TestCase): out_shape[1:3]) self.assertEqual(out_shape, list(resize_out.get_shape())) - resize_out = sess.run(resize_out) + resize_out = self.evaluate(resize_out) self.assertEqual(out_shape, list(resize_out.shape)) + @test_util.run_deprecated_v1 def testGradFromResizeToLargerInBothDims(self): in_shape = [1, 2, 3, 1] out_shape = [1, 4, 6, 1] @@ -62,6 +64,7 @@ class ResizeNearestNeighborOpTest(test.TestCase): input_tensor, in_shape, resize_out, out_shape, x_init_value=x) self.assertLess(err, 1e-3) + @test_util.run_deprecated_v1 def testGradFromResizeToSmallerInBothDims(self): in_shape = [1, 4, 6, 1] out_shape = [1, 2, 3, 1] @@ -77,6 +80,7 @@ class ResizeNearestNeighborOpTest(test.TestCase): input_tensor, in_shape, resize_out, out_shape, x_init_value=x) self.assertLess(err, 1e-3) + @test_util.run_deprecated_v1 def testCompareGpuVsCpu(self): in_shape = [1, 4, 6, 3] out_shape = [1, 8, 16, 3] @@ -113,9 +117,10 @@ class ResizeBilinearOpTest(test.TestCase): resize_out = image_ops.resize_bilinear(input_tensor, out_shape[1:3]) self.assertEqual(out_shape, list(resize_out.get_shape())) - resize_out = sess.run(resize_out) + resize_out = self.evaluate(resize_out) self.assertEqual(out_shape, list(resize_out.shape)) + @test_util.run_deprecated_v1 def testGradFromResizeToLargerInBothDims(self): in_shape = [1, 2, 3, 1] out_shape = [1, 4, 6, 1] @@ -129,6 +134,7 @@ class ResizeBilinearOpTest(test.TestCase): input_tensor, in_shape, resize_out, out_shape, x_init_value=x) self.assertLess(err, 1e-3) + @test_util.run_deprecated_v1 def testGradFromResizeToSmallerInBothDims(self): in_shape = [1, 4, 6, 1] out_shape = [1, 2, 3, 1] @@ -142,6 +148,7 @@ class ResizeBilinearOpTest(test.TestCase): input_tensor, in_shape, resize_out, out_shape, x_init_value=x) self.assertLess(err, 1e-3) + @test_util.run_deprecated_v1 def testCompareGpuVsCpu(self): in_shape = [2, 4, 6, 3] out_shape = [2, 8, 16, 3] @@ -160,6 +167,7 @@ class ResizeBilinearOpTest(test.TestCase): self.assertAllClose(grad[False], grad[True], rtol=1e-4, atol=1e-4) + @test_util.run_deprecated_v1 def testTypes(self): in_shape = [1, 4, 6, 1] out_shape = [1, 2, 3, 1] @@ -196,9 +204,10 @@ class ResizeBicubicOpTest(test.TestCase): align_corners=align_corners) self.assertEqual(out_shape, list(resize_out.get_shape())) - resize_out = sess.run(resize_out) + resize_out = self.evaluate(resize_out) self.assertEqual(out_shape, list(resize_out.shape)) + @test_util.run_deprecated_v1 def testGradFromResizeToLargerInBothDims(self): in_shape = [1, 2, 3, 1] out_shape = [1, 4, 6, 1] @@ -214,6 +223,7 @@ class ResizeBicubicOpTest(test.TestCase): input_tensor, in_shape, resize_out, out_shape, x_init_value=x) self.assertLess(err, 1e-3) + @test_util.run_deprecated_v1 def testGradFromResizeToSmallerInBothDims(self): in_shape = [1, 4, 6, 1] out_shape = [1, 2, 3, 1] @@ -229,6 +239,7 @@ class ResizeBicubicOpTest(test.TestCase): input_tensor, in_shape, resize_out, out_shape, x_init_value=x) self.assertLess(err, 1e-3) + @test_util.run_deprecated_v1 def testGradOnUnsupportedType(self): in_shape = [1, 4, 6, 1] out_shape = [1, 2, 3, 1] @@ -273,7 +284,7 @@ class CropAndResizeOpTest(test.TestCase): constant_op.constant( crop_size, shape=[2])) self.assertEqual(crops_shape, list(crops.get_shape())) - crops = sess.run(crops) + crops = self.evaluate(crops) self.assertEqual(crops_shape, list(crops.shape)) def _randomUniformAvoidAnchors(self, low, high, anchors, radius, num_samples): @@ -306,6 +317,7 @@ class CropAndResizeOpTest(test.TestCase): samples.append(sample) return samples + @test_util.run_deprecated_v1 def testGradRandomBoxes(self): """Test that the gradient is correct for randomly generated boxes. diff --git a/tensorflow/python/ops/image_ops_impl.py b/tensorflow/python/ops/image_ops_impl.py index 3ab3695a03c..229393c9703 100644 --- a/tensorflow/python/ops/image_ops_impl.py +++ b/tensorflow/python/ops/image_ops_impl.py @@ -24,6 +24,7 @@ from tensorflow.python.compat import compat from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops +from tensorflow.python.framework import random_seed from tensorflow.python.framework import tensor_shape from tensorflow.python.framework import tensor_util from tensorflow.python.ops import array_ops @@ -37,6 +38,7 @@ from tensorflow.python.ops import nn_ops from tensorflow.python.ops import random_ops from tensorflow.python.ops import string_ops from tensorflow.python.ops import variables +from tensorflow.python.util import deprecation from tensorflow.python.util.tf_export import tf_export ops.NotDifferentiable('RandomCrop') @@ -511,15 +513,20 @@ def _rot90_4D(images, k, name_scope): result.set_shape([shape[0], None, None, shape[3]]) return result -@tf_export('image.transpose_image') -def transpose_image(image): - """Transpose image(s) by swapping the height and width dimension. - See also `transpose()`. +@tf_export(v1=['image.transpose', 'image.transpose_image']) +def transpose_image(image): + return transpose(image=image, name=None) + + +@tf_export('image.transpose', v1=[]) +def transpose(image, name=None): + """Transpose image(s) by swapping the height and width dimension. Args: image: 4-D Tensor of shape `[batch, height, width, channels]` or 3-D Tensor of shape `[height, width, channels]`. + name: A name for this operation (optional). Returns: If `image` was 4-D, a 4-D float Tensor of shape @@ -530,14 +537,14 @@ def transpose_image(image): Raises: ValueError: if the shape of `image` not supported. """ - with ops.name_scope(None, 'transpose_image', [image]): + with ops.name_scope(name, 'transpose', [image]): image = ops.convert_to_tensor(image, name='image') image = _AssertAtLeast3DImage(image) shape = image.get_shape() if shape.ndims == 3 or shape.ndims is None: - return array_ops.transpose(image, [1, 0, 2], name='transpose_image') + return array_ops.transpose(image, [1, 0, 2], name=name) elif shape.ndims == 4: - return array_ops.transpose(image, [0, 2, 1, 3], name='transpose_image') + return array_ops.transpose(image, [0, 2, 1, 3], name=name) else: raise ValueError('\'image\' must have either 3 or 4 dimensions.') @@ -938,12 +945,28 @@ class ResizeMethod(object): AREA = 3 -@tf_export('image.resize_images') +@tf_export(v1=['image.resize_images', 'image.resize']) def resize_images(images, size, method=ResizeMethod.BILINEAR, align_corners=False, preserve_aspect_ratio=False): + return resize_images_v2( + images=images, + size=size, + method=method, + align_corners=align_corners, + preserve_aspect_ratio=preserve_aspect_ratio, + name=None) + + +@tf_export('image.resize', v1=[]) +def resize_images_v2(images, + size, + method=ResizeMethod.BILINEAR, + align_corners=False, + preserve_aspect_ratio=False, + name=None): """Resize `images` to `size` using the specified `method`. Resized images will be distorted if their original aspect ratio is not @@ -979,6 +1002,7 @@ def resize_images(images, then `images` will be resized to a size that fits in `size` while preserving the aspect ratio of the original image. Scales up the image if `size` is bigger than the current size of the `image`. Defaults to False. + name: A name for this operation (optional). Raises: ValueError: if the shape of `images` is incompatible with the @@ -992,7 +1016,7 @@ def resize_images(images, If `images` was 3-D, a 3-D float Tensor of shape `[new_height, new_width, channels]`. """ - with ops.name_scope(None, 'resize_images', [images, size]): + with ops.name_scope(name, 'resize', [images, size]): images = ops.convert_to_tensor(images, name='images') if images.get_shape().ndims is None: raise ValueError('\'images\' contains no shape.') @@ -1736,7 +1760,7 @@ def adjust_saturation(image, saturation_factor, name=None): orig_dtype) -@tf_export('image.is_jpeg') +@tf_export('io.is_jpeg', 'image.is_jpeg', v1=['io.is_jpeg', 'image.is_jpeg']) def is_jpeg(contents, name=None): r"""Convenience function to check if the 'contents' encodes a JPEG image. @@ -1771,8 +1795,28 @@ def _is_png(contents, name=None): substr = string_ops.substr(contents, 0, 3) return math_ops.equal(substr, b'\211PN', name=name) +tf_export('io.decode_and_crop_jpeg', 'image.decode_and_crop_jpeg', + v1=['io.decode_and_crop_jpeg', 'image.decode_and_crop_jpeg'])( + gen_image_ops.decode_and_crop_jpeg) -@tf_export('image.decode_image') +tf_export('io.decode_bmp', 'image.decode_bmp', + v1=['io.decode_bmp', 'image.decode_bmp'])(gen_image_ops.decode_bmp) +tf_export('io.decode_gif', 'image.decode_gif', + v1=['io.decode_gif', 'image.decode_gif'])(gen_image_ops.decode_gif) +tf_export('io.decode_jpeg', 'image.decode_jpeg', + v1=['io.decode_jpeg', 'image.decode_jpeg'])(gen_image_ops.decode_jpeg) +tf_export('io.decode_png', 'image.decode_png', + v1=['io.decode_png', 'image.decode_png'])(gen_image_ops.decode_png) + +tf_export('io.encode_jpeg', 'image.encode_jpeg', + v1=['io.encode_jpeg', 'image.encode_jpeg'])(gen_image_ops.encode_jpeg) +tf_export('io.extract_jpeg_shape', 'image.extract_jpeg_shape', + v1=['io.extract_jpeg_shape', 'image.extract_jpeg_shape'])( + gen_image_ops.extract_jpeg_shape) + + +@tf_export('io.decode_image', 'image.decode_image', + v1=['io.decode_image', 'image.decode_image']) def decode_image(contents, channels=None, dtype=dtypes.uint8, name=None): """Convenience function for `decode_bmp`, `decode_gif`, `decode_jpeg`, and `decode_png`. @@ -1942,7 +1986,114 @@ def total_variation(images, name=None): return tot_var -@tf_export('image.sample_distorted_bounding_box') +@tf_export('image.sample_distorted_bounding_box', v1=[]) +def sample_distorted_bounding_box_v2(image_size, + bounding_boxes, + seed=0, + min_object_covered=0.1, + aspect_ratio_range=None, + area_range=None, + max_attempts=None, + use_image_if_no_bounding_boxes=None, + name=None): + """Generate a single randomly distorted bounding box for an image. + + Bounding box annotations are often supplied in addition to ground-truth labels + in image recognition or object localization tasks. A common technique for + training such a system is to randomly distort an image while preserving + its content, i.e. *data augmentation*. This Op outputs a randomly distorted + localization of an object, i.e. bounding box, given an `image_size`, + `bounding_boxes` and a series of constraints. + + The output of this Op is a single bounding box that may be used to crop the + original image. The output is returned as 3 tensors: `begin`, `size` and + `bboxes`. The first 2 tensors can be fed directly into `tf.slice` to crop the + image. The latter may be supplied to `tf.image.draw_bounding_boxes` to + visualize what the bounding box looks like. + + Bounding boxes are supplied and returned as `[y_min, x_min, y_max, x_max]`. + The bounding box coordinates are floats in `[0.0, 1.0]` relative to the width + and height of the underlying image. + + For example, + + ```python + # Generate a single distorted bounding box. + begin, size, bbox_for_draw = tf.image.sample_distorted_bounding_box( + tf.shape(image), + bounding_boxes=bounding_boxes, + min_object_covered=0.1) + + # Draw the bounding box in an image summary. + image_with_box = tf.image.draw_bounding_boxes(tf.expand_dims(image, 0), + bbox_for_draw) + tf.summary.image('images_with_box', image_with_box) + + # Employ the bounding box to distort the image. + distorted_image = tf.slice(image, begin, size) + ``` + + Note that if no bounding box information is available, setting + `use_image_if_no_bounding_boxes = true` will assume there is a single implicit + bounding box covering the whole image. If `use_image_if_no_bounding_boxes` is + false and no bounding boxes are supplied, an error is raised. + + Args: + image_size: A `Tensor`. Must be one of the following types: `uint8`, `int8`, + `int16`, `int32`, `int64`. + 1-D, containing `[height, width, channels]`. + bounding_boxes: A `Tensor` of type `float32`. + 3-D with shape `[batch, N, 4]` describing the N bounding boxes + associated with the image. + seed: An optional `int`. Defaults to `0`. + If either `seed` or `seed2` are set to non-zero, the random number + generator is seeded by the given `seed`. Otherwise, it is seeded by a + random seed. + min_object_covered: A Tensor of type `float32`. Defaults to `0.1`. + The cropped area of the image must contain at least this + fraction of any bounding box supplied. The value of this parameter should + be non-negative. In the case of 0, the cropped area does not need to + overlap any of the bounding boxes supplied. + aspect_ratio_range: An optional list of `floats`. Defaults to `[0.75, + 1.33]`. + The cropped area of the image must have an aspect `ratio = + width / height` within this range. + area_range: An optional list of `floats`. Defaults to `[0.05, 1]`. + The cropped area of the image must contain a fraction of the + supplied image within this range. + max_attempts: An optional `int`. Defaults to `100`. + Number of attempts at generating a cropped region of the image + of the specified constraints. After `max_attempts` failures, return the + entire image. + use_image_if_no_bounding_boxes: An optional `bool`. Defaults to `False`. + Controls behavior if no bounding boxes supplied. + If true, assume an implicit bounding box covering the whole input. If + false, raise an error. + name: A name for the operation (optional). + + Returns: + A tuple of `Tensor` objects (begin, size, bboxes). + + begin: A `Tensor`. Has the same type as `image_size`. 1-D, containing + `[offset_height, offset_width, 0]`. Provide as input to + `tf.slice`. + size: A `Tensor`. Has the same type as `image_size`. 1-D, containing + `[target_height, target_width, -1]`. Provide as input to + `tf.slice`. + bboxes: A `Tensor` of type `float32`. 3-D with shape `[1, 1, 4]` containing + the distorted bounding box. + Provide as input to `tf.image.draw_bounding_boxes`. + """ + seed1, seed2 = random_seed.get_seed(seed) if seed else (0, 0) + return sample_distorted_bounding_box( + image_size, bounding_boxes, seed1, seed2, min_object_covered, + aspect_ratio_range, area_range, max_attempts, + use_image_if_no_bounding_boxes, name) + + +@tf_export(v1=['image.sample_distorted_bounding_box']) +@deprecation.deprecated(date=None, instructions='`seed2` arg is deprecated.' + 'Use sample_distorted_bounding_box_v2 instead.') def sample_distorted_bounding_box(image_size, bounding_boxes, seed=None, @@ -2808,3 +2959,102 @@ def sobel_edges(image): output = array_ops.reshape(output, shape=shape) output.set_shape(static_image_shape.concatenate([num_kernels])) return output + + +resize_area_deprecation = deprecation.deprecated( + date=None, + instructions=( + 'Use `tf.image.resize(...method=ResizeMethod.AREA...)` instead.')) +tf_export(v1=['image.resize_area'])( + resize_area_deprecation(gen_image_ops.resize_area)) + +resize_bicubic_deprecation = deprecation.deprecated( + date=None, + instructions=( + 'Use `tf.image.resize(...method=ResizeMethod.BICUBIC...)` instead.')) +tf_export(v1=['image.resize_bicubic'])( + resize_bicubic_deprecation(gen_image_ops.resize_bicubic)) + +resize_bilinear_deprecation = deprecation.deprecated( + date=None, + instructions=( + 'Use `tf.image.resize(...method=ResizeMethod.BILINEAR...)` instead.')) +tf_export(v1=['image.resize_bilinear'])( + resize_bilinear_deprecation(gen_image_ops.resize_bilinear)) + +resize_nearest_neighbor_deprecation = deprecation.deprecated( + date=None, + instructions=( + 'Use `tf.image.resize(...method=ResizeMethod.NEAREST_NEIGHBOR...)` ' + 'instead.')) +tf_export(v1=['image.resize_nearest_neighbor'])( + resize_nearest_neighbor_deprecation(gen_image_ops.resize_nearest_neighbor)) + + +@tf_export('image.crop_and_resize', v1=[]) +def crop_and_resize_v2( + image, + boxes, + box_indices, + crop_size, + method='bilinear', + extrapolation_value=0, + name=None): + """Extracts crops from the input image tensor and resizes them. + + Extracts crops from the input image tensor and resizes them using bilinear + sampling or nearest neighbor sampling (possibly with aspect ratio change) to a + common output size specified by `crop_size`. This is more general than the + `crop_to_bounding_box` op which extracts a fixed size slice from the input + image and does not allow resizing or aspect ratio change. + + Returns a tensor with `crops` from the input `image` at positions defined at + the bounding box locations in `boxes`. The cropped boxes are all resized (with + bilinear or nearest neighbor interpolation) to a fixed + `size = [crop_height, crop_width]`. The result is a 4-D tensor + `[num_boxes, crop_height, crop_width, depth]`. The resizing is corner aligned. + In particular, if `boxes = [[0, 0, 1, 1]]`, the method will give identical + results to using `tf.image.resize_bilinear()` or + `tf.image.resize_nearest_neighbor()`(depends on the `method` argument) with + `align_corners=True`. + + Args: + image: A 4-D tensor of shape `[batch, image_height, image_width, depth]`. + Both `image_height` and `image_width` need to be positive. + boxes: A 2-D tensor of shape `[num_boxes, 4]`. The `i`-th row of the tensor + specifies the coordinates of a box in the `box_ind[i]` image and is + specified in normalized coordinates `[y1, x1, y2, x2]`. A normalized + coordinate value of `y` is mapped to the image coordinate at `y * + (image_height - 1)`, so as the `[0, 1]` interval of normalized image + height is mapped to `[0, image_height - 1]` in image height coordinates. + We do allow `y1` > `y2`, in which case the sampled crop is an up-down + flipped version of the original image. The width dimension is treated + similarly. Normalized coordinates outside the `[0, 1]` range are allowed, + in which case we use `extrapolation_value` to extrapolate the input image + values. + box_indices: A 1-D tensor of shape `[num_boxes]` with int32 values in `[0, + batch)`. The value of `box_ind[i]` specifies the image that the `i`-th box + refers to. + crop_size: A 1-D tensor of 2 elements, `size = [crop_height, crop_width]`. + All cropped image patches are resized to this size. The aspect ratio of + the image content is not preserved. Both `crop_height` and `crop_width` + need to be positive. + method: An optional string specifying the sampling method for resizing. It + can be either `"bilinear"` or `"nearest"` and default to `"bilinear"`. + Currently two sampling methods are supported: Bilinear and Nearest + Neighbor. + extrapolation_value: An optional `float`. Defaults to `0`. Value used for + extrapolation, when applicable. + name: A name for the operation (optional). + + Returns: + A 4-D tensor of shape `[num_boxes, crop_height, crop_width, depth]`. + """ + return gen_image_ops.crop_and_resize( + image, boxes, box_indices, crop_size, method, extrapolation_value, name) + + +crop_and_resize_deprecation = deprecation.deprecated_args( + None, 'box_ind is deprecated, use box_indices instead', 'box_ind') +tf_export(v1=['image.crop_and_resize'])( + crop_and_resize_deprecation(gen_image_ops.crop_and_resize)) diff --git a/tensorflow/python/ops/image_ops_test.py b/tensorflow/python/ops/image_ops_test.py index a3aeb79586b..e7249333bd3 100644 --- a/tensorflow/python/ops/image_ops_test.py +++ b/tensorflow/python/ops/image_ops_test.py @@ -70,7 +70,8 @@ class RGBToHSVTest(test_util.TensorFlowTestCase): split2 = list(map(image_ops.hsv_to_rgb, split1)) join1 = array_ops.stack(split1) join2 = array_ops.stack(split2) - batch1, batch2, join1, join2 = sess.run([batch1, batch2, join1, join2]) + batch1, batch2, join1, join2 = self.evaluate( + [batch1, batch2, join1, join2]) # Verify that processing batch elements together is the same as separate self.assertAllClose(batch1, join1) @@ -84,7 +85,7 @@ class RGBToHSVTest(test_util.TensorFlowTestCase): with self.test_session(use_gpu=True): hsv = image_ops.rgb_to_hsv(rgb_np) rgb = image_ops.hsv_to_rgb(hsv) - rgb_tf = rgb.eval() + rgb_tf = self.evaluate(rgb) self.assertAllClose(rgb_tf, rgb_np) @@ -109,7 +110,8 @@ class RGBToYIQTest(test_util.TensorFlowTestCase): split2 = list(map(image_ops.yiq_to_rgb, split1)) join1 = array_ops.stack(split1) join2 = array_ops.stack(split2) - batch1, batch2, join1, join2 = sess.run([batch1, batch2, join1, join2]) + batch1, batch2, join1, join2 = self.evaluate( + [batch1, batch2, join1, join2]) # Verify that processing batch elements together is the same as separate self.assertAllClose(batch1, join1, rtol=1e-4, atol=1e-4) @@ -138,7 +140,8 @@ class RGBToYUVTest(test_util.TensorFlowTestCase): split2 = list(map(image_ops.yuv_to_rgb, split1)) join1 = array_ops.stack(split1) join2 = array_ops.stack(split2) - batch1, batch2, join1, join2 = sess.run([batch1, batch2, join1, join2]) + batch1, batch2, join1, join2 = self.evaluate( + [batch1, batch2, join1, join2]) # Verify that processing batch elements together is the same as separate self.assertAllClose(batch1, join1, rtol=1e-4, atol=1e-4) @@ -173,7 +176,7 @@ class GrayscaleToRGBTest(test_util.TensorFlowTestCase): with self.test_session(use_gpu=True): x_tf = constant_op.constant(x_np, shape=x_np.shape) y = image_ops.rgb_to_grayscale(x_tf) - y_tf = y.eval() + y_tf = self.evaluate(y) self.assertAllEqual(y_tf, y_np) def testBasicRGBToGrayscale(self): @@ -195,7 +198,7 @@ class GrayscaleToRGBTest(test_util.TensorFlowTestCase): with self.test_session(use_gpu=True): x_tf = constant_op.constant(x_np, shape=x_np.shape) y = image_ops.grayscale_to_rgb(x_tf) - y_tf = y.eval() + y_tf = self.evaluate(y) self.assertAllEqual(y_tf, y_np) # 3-D input with no batch dimension. @@ -205,9 +208,10 @@ class GrayscaleToRGBTest(test_util.TensorFlowTestCase): with self.test_session(use_gpu=True): x_tf = constant_op.constant(x_np, shape=x_np.shape) y = image_ops.grayscale_to_rgb(x_tf) - y_tf = y.eval() + y_tf = self.evaluate(y) self.assertAllEqual(y_tf, y_np) + @test_util.run_deprecated_v1 def testShapeInference(self): # Shape inference works and produces expected output where possible rgb_shape = [7, None, 19, 3] @@ -245,7 +249,7 @@ class AdjustGamma(test_util.TensorFlowTestCase): x = constant_op.constant(x_np, shape=x_np.shape) y = image_ops.adjust_gamma(x, gamma=1) - y_tf = y.eval() + y_tf = self.evaluate(y) y_np = x_np self.assertAllClose(y_tf, y_np, 1e-6) @@ -268,6 +272,7 @@ class AdjustGamma(test_util.TensorFlowTestCase): else: raise AssertionError("Exception not raised: %s" % err_msg) + @test_util.run_deprecated_v1 def test_adjust_gamma_less_zero_tensor(self): """White image should be returned for gamma equal to zero""" with self.cached_session(): @@ -281,7 +286,7 @@ class AdjustGamma(test_util.TensorFlowTestCase): err_msg = "Gamma should be a non-negative real number." try: - image.eval() + self.evaluate(image) except Exception as e: if err_msg not in str(e): raise @@ -297,7 +302,7 @@ class AdjustGamma(test_util.TensorFlowTestCase): x = constant_op.constant(x_np, shape=x_np.shape) y = image_ops.adjust_gamma(x, gamma=0) - y_tf = y.eval() + y_tf = self.evaluate(y) dtype = x.dtype.as_numpy_dtype y_np = np.array([dtypes.dtype_range[dtype][1]] * x_np.size) @@ -305,6 +310,7 @@ class AdjustGamma(test_util.TensorFlowTestCase): self.assertAllClose(y_tf, y_np, 1e-6) + @test_util.run_deprecated_v1 def test_adjust_gamma_less_one(self): """Verifying the output with expected results for gamma correction with gamma equal to half""" @@ -326,6 +332,7 @@ class AdjustGamma(test_util.TensorFlowTestCase): self.assertAllClose(y_tf, y_np, 1e-6) + @test_util.run_deprecated_v1 def test_adjust_gamma_greater_one(self): """Verifying the output with expected results for gamma correction with gamma equal to two""" @@ -360,7 +367,7 @@ class AdjustHueTest(test_util.TensorFlowTestCase): with self.test_session(use_gpu=True): x = constant_op.constant(x_np, shape=x_shape) y = image_ops.adjust_hue(x, delta) - y_tf = y.eval() + y_tf = self.evaluate(y) self.assertAllEqual(y_tf, y_np) def testAdjustPositiveHue(self): @@ -375,7 +382,7 @@ class AdjustHueTest(test_util.TensorFlowTestCase): with self.test_session(use_gpu=True): x = constant_op.constant(x_np, shape=x_shape) y = image_ops.adjust_hue(x, delta) - y_tf = y.eval() + y_tf = self.evaluate(y) self.assertAllEqual(y_tf, y_np) def testBatchAdjustHue(self): @@ -390,7 +397,7 @@ class AdjustHueTest(test_util.TensorFlowTestCase): with self.test_session(use_gpu=True): x = constant_op.constant(x_np, shape=x_shape) y = image_ops.adjust_hue(x, delta) - y_tf = y.eval() + y_tf = self.evaluate(y) self.assertAllEqual(y_tf, y_np) def _adjustHueNp(self, x_np, delta_h): @@ -415,7 +422,7 @@ class AdjustHueTest(test_util.TensorFlowTestCase): with self.test_session(use_gpu=True): x = constant_op.constant(x_np) y = image_ops.adjust_hue(x, delta_h) - y_tf = y.eval() + y_tf = self.evaluate(y) return y_tf def testAdjustRandomHue(self): @@ -488,11 +495,11 @@ class FlipImageBenchmark(test.Benchmark): trainable=False, dtype=dtypes.float32) run_op = image_ops.flip_left_right(inputs) - sess.run(variables.global_variables_initializer()) + self.evaluate(variables.global_variables_initializer()) for i in xrange(warmup_rounds + benchmark_rounds): if i == warmup_rounds: start = time.time() - sess.run(run_op) + self.evaluate(run_op) end = time.time() step_time = (end - start) / benchmark_rounds tag = device + "_%s" % (cpu_count if cpu_count is not None else "_all") @@ -518,11 +525,11 @@ class FlipImageBenchmark(test.Benchmark): trainable=False, dtype=dtypes.float32) run_op = image_ops.random_flip_left_right(inputs) - sess.run(variables.global_variables_initializer()) + self.evaluate(variables.global_variables_initializer()) for i in xrange(warmup_rounds + benchmark_rounds): if i == warmup_rounds: start = time.time() - sess.run(run_op) + self.evaluate(run_op) end = time.time() step_time = (end - start) / benchmark_rounds tag = device + "_%s" % (cpu_count if cpu_count is not None else "_all") @@ -548,11 +555,11 @@ class FlipImageBenchmark(test.Benchmark): trainable=False, dtype=dtypes.float32) run_op = image_ops.random_flip_left_right(inputs) - sess.run(variables.global_variables_initializer()) + self.evaluate(variables.global_variables_initializer()) for i in xrange(warmup_rounds + benchmark_rounds): if i == warmup_rounds: start = time.time() - sess.run(run_op) + self.evaluate(run_op) end = time.time() step_time = (end - start) / benchmark_rounds tag = device + "_%s" % (cpu_count if cpu_count is not None else "_all") @@ -610,11 +617,11 @@ class AdjustHueBenchmark(test.Benchmark): delta = constant_op.constant(0.1, dtype=dtypes.float32) outputs = image_ops.adjust_hue(inputs, delta) run_op = control_flow_ops.group(outputs) - sess.run(variables.global_variables_initializer()) + self.evaluate(variables.global_variables_initializer()) for i in xrange(warmup_rounds + benchmark_rounds): if i == warmup_rounds: start = time.time() - sess.run(run_op) + self.evaluate(run_op) end = time.time() step_time = (end - start) / benchmark_rounds tag = device + "_%s" % (cpu_count if cpu_count is not None else "_all") @@ -653,12 +660,12 @@ class AdjustSaturationBenchmark(test.Benchmark): delta = constant_op.constant(0.1, dtype=dtypes.float32) outputs = image_ops.adjust_saturation(inputs, delta) run_op = control_flow_ops.group(outputs) - sess.run(variables.global_variables_initializer()) + self.evaluate(variables.global_variables_initializer()) for _ in xrange(warmup_rounds): - sess.run(run_op) + self.evaluate(run_op) start = time.time() for _ in xrange(benchmark_rounds): - sess.run(run_op) + self.evaluate(run_op) end = time.time() step_time = (end - start) / benchmark_rounds tag = device + "_%s" % (cpu_count if cpu_count is not None else "_all") @@ -698,7 +705,7 @@ class ResizeBilinearBenchmark(test.Benchmark): benchmark_op = control_flow_ops.group(*deps) with self.benchmark_session() as sess: - sess.run(variables.global_variables_initializer()) + self.evaluate(variables.global_variables_initializer()) results = self.run_op_benchmark( sess, benchmark_op, @@ -746,7 +753,7 @@ class ResizeBicubicBenchmark(test.Benchmark): benchmark_op = control_flow_ops.group(*deps) with self.benchmark_session() as sess: - sess.run(variables.global_variables_initializer()) + self.evaluate(variables.global_variables_initializer()) results = self.run_op_benchmark( sess, benchmark_op, @@ -803,7 +810,7 @@ class ResizeAreaBenchmark(test.Benchmark): benchmark_op = control_flow_ops.group(*deps) with self.benchmark_session() as sess: - sess.run(variables.global_variables_initializer()) + self.evaluate(variables.global_variables_initializer()) results = self.run_op_benchmark( sess, benchmark_op, @@ -846,7 +853,7 @@ class AdjustSaturationTest(test_util.TensorFlowTestCase): with self.test_session(use_gpu=True): x = constant_op.constant(x_np, shape=x_shape) y = image_ops.adjust_saturation(x, saturation_factor) - y_tf = y.eval() + y_tf = self.evaluate(y) self.assertAllEqual(y_tf, y_np) def testTwiceSaturation(self): @@ -861,7 +868,7 @@ class AdjustSaturationTest(test_util.TensorFlowTestCase): with self.test_session(use_gpu=True): x = constant_op.constant(x_np, shape=x_shape) y = image_ops.adjust_saturation(x, saturation_factor) - y_tf = y.eval() + y_tf = self.evaluate(y) self.assertAllEqual(y_tf, y_np) def testBatchSaturation(self): @@ -876,7 +883,7 @@ class AdjustSaturationTest(test_util.TensorFlowTestCase): with self.test_session(use_gpu=True): x = constant_op.constant(x_np, shape=x_shape) y = image_ops.adjust_saturation(x, saturation_factor) - y_tf = y.eval() + y_tf = self.evaluate(y) self.assertAllEqual(y_tf, y_np) def _adjust_saturation(self, image, saturation_factor): @@ -899,7 +906,7 @@ class AdjustSaturationTest(test_util.TensorFlowTestCase): with self.test_session(use_gpu=True): x = constant_op.constant(x_np, shape=x_shape) y = self._adjust_saturation(x, saturation_factor) - y_tf = y.eval() + y_tf = self.evaluate(y) self.assertAllEqual(y_tf, y_np) def testTwiceSaturationFused(self): @@ -914,7 +921,7 @@ class AdjustSaturationTest(test_util.TensorFlowTestCase): with self.test_session(use_gpu=True): x = constant_op.constant(x_np, shape=x_shape) y = self._adjust_saturation(x, saturation_factor) - y_tf = y.eval() + y_tf = self.evaluate(y) self.assertAllEqual(y_tf, y_np) def _adjustSaturationNp(self, x_np, scale): @@ -935,6 +942,7 @@ class AdjustSaturationTest(test_util.TensorFlowTestCase): y_v[i][2] = b return y_v.reshape(x_np.shape) + @test_util.run_deprecated_v1 def testAdjustRandomSaturation(self): x_shapes = [ [2, 2, 3], @@ -980,7 +988,7 @@ class FlipTransposeRotateTest(test_util.TensorFlowTestCase): with self.test_session(use_gpu=True): x_tf = constant_op.constant(x_np, shape=x_np.shape) y = image_ops.flip_left_right(image_ops.flip_left_right(x_tf)) - y_tf = y.eval() + y_tf = self.evaluate(y) self.assertAllEqual(y_tf, x_np) def testInvolutionLeftRightWithBatch(self): @@ -990,9 +998,10 @@ class FlipTransposeRotateTest(test_util.TensorFlowTestCase): with self.test_session(use_gpu=True): x_tf = constant_op.constant(x_np, shape=x_np.shape) y = image_ops.flip_left_right(image_ops.flip_left_right(x_tf)) - y_tf = y.eval() + y_tf = self.evaluate(y) self.assertAllEqual(y_tf, x_np) + @test_util.run_deprecated_v1 def testLeftRight(self): x_np = np.array([[1, 2, 3], [1, 2, 3]], dtype=np.uint8).reshape([2, 3, 1]) y_np = np.array([[3, 2, 1], [3, 2, 1]], dtype=np.uint8).reshape([2, 3, 1]) @@ -1001,7 +1010,7 @@ class FlipTransposeRotateTest(test_util.TensorFlowTestCase): x_tf = constant_op.constant(x_np, shape=x_np.shape) y = image_ops.flip_left_right(x_tf) self.assertTrue(y.op.name.startswith("flip_left_right")) - y_tf = y.eval() + y_tf = self.evaluate(y) self.assertAllEqual(y_tf, y_np) def testLeftRightWithBatch(self): @@ -1015,9 +1024,10 @@ class FlipTransposeRotateTest(test_util.TensorFlowTestCase): with self.test_session(use_gpu=True): x_tf = constant_op.constant(x_np, shape=x_np.shape) y = image_ops.flip_left_right(x_tf) - y_tf = y.eval() + y_tf = self.evaluate(y) self.assertAllEqual(y_tf, y_np) + @test_util.run_deprecated_v1 def testRandomFlipLeftRight(self): x_np = np.array([[1, 2, 3], [1, 2, 3]], dtype=np.uint8).reshape([2, 3, 1]) y_np = np.array([[3, 2, 1], [3, 2, 1]], dtype=np.uint8).reshape([2, 3, 1]) @@ -1031,7 +1041,7 @@ class FlipTransposeRotateTest(test_util.TensorFlowTestCase): count_flipped = 0 count_unflipped = 0 for _ in range(100): - y_tf = y.eval() + y_tf = self.evaluate(y) if y_tf[0][0] == 1: self.assertAllEqual(y_tf, x_np) count_unflipped += 1 @@ -1046,6 +1056,7 @@ class FlipTransposeRotateTest(test_util.TensorFlowTestCase): self.assertGreaterEqual(count_flipped, 20) self.assertGreaterEqual(count_unflipped, 20) + @test_util.run_deprecated_v1 def testRandomFlipLeftRightWithBatch(self): batch_size = 16 seed = 42 @@ -1070,7 +1081,7 @@ class FlipTransposeRotateTest(test_util.TensorFlowTestCase): count_flipped = 0 count_unflipped = 0 for _ in range(100): - y_tf = y.eval() + y_tf = self.evaluate(y) # check every element of the batch for i in range(batch_size): @@ -1096,7 +1107,7 @@ class FlipTransposeRotateTest(test_util.TensorFlowTestCase): with self.test_session(use_gpu=True): x_tf = constant_op.constant(x_np, shape=x_np.shape) y = image_ops.flip_up_down(image_ops.flip_up_down(x_tf)) - y_tf = y.eval() + y_tf = self.evaluate(y) self.assertAllEqual(y_tf, x_np) def testInvolutionUpDownWithBatch(self): @@ -1107,9 +1118,10 @@ class FlipTransposeRotateTest(test_util.TensorFlowTestCase): with self.test_session(use_gpu=True): x_tf = constant_op.constant(x_np, shape=x_np.shape) y = image_ops.flip_up_down(image_ops.flip_up_down(x_tf)) - y_tf = y.eval() + y_tf = self.evaluate(y) self.assertAllEqual(y_tf, x_np) + @test_util.run_deprecated_v1 def testUpDown(self): x_np = np.array([[1, 2, 3], [4, 5, 6]], dtype=np.uint8).reshape([2, 3, 1]) y_np = np.array([[4, 5, 6], [1, 2, 3]], dtype=np.uint8).reshape([2, 3, 1]) @@ -1118,7 +1130,7 @@ class FlipTransposeRotateTest(test_util.TensorFlowTestCase): x_tf = constant_op.constant(x_np, shape=x_np.shape) y = image_ops.flip_up_down(x_tf) self.assertTrue(y.op.name.startswith("flip_up_down")) - y_tf = y.eval() + y_tf = self.evaluate(y) self.assertAllEqual(y_tf, y_np) def testUpDownWithBatch(self): @@ -1132,9 +1144,10 @@ class FlipTransposeRotateTest(test_util.TensorFlowTestCase): with self.test_session(use_gpu=True): x_tf = constant_op.constant(x_np, shape=x_np.shape) y = image_ops.flip_up_down(x_tf) - y_tf = y.eval() + y_tf = self.evaluate(y) self.assertAllEqual(y_tf, y_np) + @test_util.run_deprecated_v1 def testRandomFlipUpDown(self): x_np = np.array([[1, 2, 3], [4, 5, 6]], dtype=np.uint8).reshape([2, 3, 1]) y_np = np.array([[4, 5, 6], [1, 2, 3]], dtype=np.uint8).reshape([2, 3, 1]) @@ -1148,7 +1161,7 @@ class FlipTransposeRotateTest(test_util.TensorFlowTestCase): count_flipped = 0 count_unflipped = 0 for _ in range(100): - y_tf = y.eval() + y_tf = self.evaluate(y) if y_tf[0][0] == 1: self.assertAllEqual(y_tf, x_np) count_unflipped += 1 @@ -1163,6 +1176,7 @@ class FlipTransposeRotateTest(test_util.TensorFlowTestCase): self.assertGreaterEqual(count_flipped, 20) self.assertGreaterEqual(count_unflipped, 20) + @test_util.run_deprecated_v1 def testRandomFlipUpDownWithBatch(self): batch_size = 16 seed = 42 @@ -1187,7 +1201,7 @@ class FlipTransposeRotateTest(test_util.TensorFlowTestCase): count_flipped = 0 count_unflipped = 0 for _ in range(100): - y_tf = y.eval() + y_tf = self.evaluate(y) # check every element of the batch for i in range(batch_size): @@ -1213,7 +1227,7 @@ class FlipTransposeRotateTest(test_util.TensorFlowTestCase): with self.test_session(use_gpu=True): x_tf = constant_op.constant(x_np, shape=x_np.shape) y = image_ops.transpose_image(image_ops.transpose_image(x_tf)) - y_tf = y.eval() + y_tf = self.evaluate(y) self.assertAllEqual(y_tf, x_np) def testInvolutionTransposeWithBatch(self): @@ -1224,9 +1238,10 @@ class FlipTransposeRotateTest(test_util.TensorFlowTestCase): with self.test_session(use_gpu=True): x_tf = constant_op.constant(x_np, shape=x_np.shape) y = image_ops.transpose_image(image_ops.transpose_image(x_tf)) - y_tf = y.eval() + y_tf = self.evaluate(y) self.assertAllEqual(y_tf, x_np) + @test_util.run_deprecated_v1 def testTranspose(self): x_np = np.array([[1, 2, 3], [4, 5, 6]], dtype=np.uint8).reshape([2, 3, 1]) y_np = np.array([[1, 4], [2, 5], [3, 6]], dtype=np.uint8).reshape([3, 2, 1]) @@ -1234,8 +1249,8 @@ class FlipTransposeRotateTest(test_util.TensorFlowTestCase): with self.test_session(use_gpu=True): x_tf = constant_op.constant(x_np, shape=x_np.shape) y = image_ops.transpose_image(x_tf) - self.assertTrue(y.op.name.startswith("transpose_image")) - y_tf = y.eval() + self.assertTrue(y.op.name.startswith("transpose")) + y_tf = self.evaluate(y) self.assertAllEqual(y_tf, y_np) def testTransposeWithBatch(self): @@ -1250,9 +1265,10 @@ class FlipTransposeRotateTest(test_util.TensorFlowTestCase): with self.test_session(use_gpu=True): x_tf = constant_op.constant(x_np, shape=x_np.shape) y = image_ops.transpose_image(x_tf) - y_tf = y.eval() + y_tf = self.evaluate(y) self.assertAllEqual(y_tf, y_np) + @test_util.run_deprecated_v1 def testPartialShapes(self): p_unknown_rank = array_ops.placeholder(dtypes.uint8) p_unknown_dims_3 = array_ops.placeholder( @@ -1301,7 +1317,7 @@ class FlipTransposeRotateTest(test_util.TensorFlowTestCase): rotated = image for _ in xrange(4): rotated = image_ops.rot90(rotated) - self.assertAllEqual(image, rotated.eval()) + self.assertAllEqual(image, self.evaluate(rotated)) def testRot90GroupOrderWithBatch(self): image = np.arange(48, dtype=np.uint8).reshape([2, 2, 4, 3]) @@ -1309,8 +1325,9 @@ class FlipTransposeRotateTest(test_util.TensorFlowTestCase): rotated = image for _ in xrange(4): rotated = image_ops.rot90(rotated) - self.assertAllEqual(image, rotated.eval()) + self.assertAllEqual(image, self.evaluate(rotated)) + @test_util.run_deprecated_v1 def testRot90NumpyEquivalence(self): image = np.arange(24, dtype=np.uint8).reshape([2, 4, 3]) with self.test_session(use_gpu=True): @@ -1320,6 +1337,7 @@ class FlipTransposeRotateTest(test_util.TensorFlowTestCase): y_np = np.rot90(image, k=k) self.assertAllEqual(y_np, y_tf.eval({k_placeholder: k})) + @test_util.run_deprecated_v1 def testRot90NumpyEquivalenceWithBatch(self): image = np.arange(48, dtype=np.uint8).reshape([2, 2, 4, 3]) with self.test_session(use_gpu=True): @@ -1335,7 +1353,7 @@ class AdjustContrastTest(test_util.TensorFlowTestCase): with self.test_session(use_gpu=True): x = constant_op.constant(x_np, shape=x_np.shape) y = image_ops.adjust_contrast(x, contrast_factor) - y_tf = y.eval() + y_tf = self.evaluate(y) self.assertAllClose(y_tf, y_np, 1e-6) def testDoubleContrastUint8(self): @@ -1390,7 +1408,7 @@ class AdjustContrastTest(test_util.TensorFlowTestCase): with self.test_session(use_gpu=True): x = constant_op.constant(x_np) y = image_ops.adjust_contrast(x, contrast_factor) - y_tf = y.eval() + y_tf = self.evaluate(y) return y_tf def testRandomContrast(self): @@ -1408,6 +1426,7 @@ class AdjustContrastTest(test_util.TensorFlowTestCase): y_tf = self._adjustContrastTf(x_np, contrast_factor) self.assertAllClose(y_tf, y_np, rtol=1e-5, atol=1e-5) + @test_util.run_deprecated_v1 def testContrastFactorShape(self): x_shape = [1, 2, 2, 3] x_data = [0, 5, 13, 54, 135, 226, 37, 8, 234, 90, 255, 1] @@ -1423,7 +1442,7 @@ class AdjustBrightnessTest(test_util.TensorFlowTestCase): with self.test_session(use_gpu=True): x = constant_op.constant(x_np, shape=x_np.shape) y = image_ops.adjust_brightness(x, delta) - y_tf = y.eval() + y_tf = self.evaluate(y) self.assertAllClose(y_tf, y_np, 1e-6) def testPositiveDeltaUint8(self): @@ -1471,6 +1490,7 @@ class PerImageWhiteningTest(test_util.TensorFlowTestCase): y /= stddev return y + @test_util.run_deprecated_v1 def testBasic(self): x_shape = [13, 9, 3] x_np = np.arange(0, np.prod(x_shape), dtype=np.int32).reshape(x_shape) @@ -1480,7 +1500,7 @@ class PerImageWhiteningTest(test_util.TensorFlowTestCase): x = constant_op.constant(x_np, shape=x_shape) y = image_ops.per_image_standardization(x) self.assertTrue(y.op.name.startswith("per_image_standardization")) - y_tf = y.eval() + y_tf = self.evaluate(y) self.assertAllClose(y_tf, y_np, atol=1e-4) def testUniformImage(self): @@ -1488,7 +1508,7 @@ class PerImageWhiteningTest(test_util.TensorFlowTestCase): im = constant_op.constant(im_np) whiten = image_ops.per_image_standardization(im) with self.test_session(use_gpu=True): - whiten_np = whiten.eval() + whiten_np = self.evaluate(whiten) self.assertFalse(np.any(np.isnan(whiten_np))) def testBatchWhitening(self): @@ -1497,7 +1517,7 @@ class PerImageWhiteningTest(test_util.TensorFlowTestCase): with self.test_session(use_gpu=True): imgs = constant_op.constant(imgs_np) whiten = image_ops.per_image_standardization(imgs) - whiten_tf = whiten.eval() + whiten_tf = self.evaluate(whiten) for w_tf, w_np in zip(whiten_tf, whiten_np): self.assertAllClose(w_tf, w_np, atol=1e-4) @@ -1571,11 +1591,13 @@ class CropToBoundingBoxTest(test_util.TensorFlowTestCase): y = image_ops.crop_to_bounding_box(image, 0, 0, height, width) self.assertEqual(y.get_shape().as_list(), post_shape) + @test_util.run_deprecated_v1 def testNoOp(self): x_shape = [10, 10, 10] x = np.random.uniform(size=x_shape) self._assertReturns(x, x_shape, 0, 0, x, x_shape) + @test_util.run_deprecated_v1 def testCrop(self): x = [1, 2, 3, 4, 5, 6, 7, 8, 9] x_shape = [3, 3, 1] @@ -1600,6 +1622,7 @@ class CropToBoundingBoxTest(test_util.TensorFlowTestCase): y = [1, 2, 4, 5, 7, 8] self._assertReturns(x, x_shape, offset_height, offset_width, y, y_shape) + @test_util.run_deprecated_v1 def testShapeInference(self): self._assertShapeInference([55, 66, 3], 55, 66, [55, 66, 3]) self._assertShapeInference([59, 69, 3], 55, 66, [55, 66, 3]) @@ -1613,6 +1636,7 @@ class CropToBoundingBoxTest(test_util.TensorFlowTestCase): self._assertShapeInference([None, None, None], 55, 66, [55, 66, None]) self._assertShapeInference(None, 55, 66, [55, 66, None]) + @test_util.run_deprecated_v1 def testNon3DInput(self): # Input image is not 3D x = [0] * 15 @@ -1624,6 +1648,7 @@ class CropToBoundingBoxTest(test_util.TensorFlowTestCase): target_width, "'image' must have either 3 or 4 dimensions.") + @test_util.run_deprecated_v1 def testZeroLengthInput(self): # Input image has 0-length dimension(s). # Each line is a test configuration: @@ -1655,6 +1680,7 @@ class CropToBoundingBoxTest(test_util.TensorFlowTestCase): "assertion failed:", use_tensor_inputs_options=[True]) + @test_util.run_deprecated_v1 def testBadParams(self): x_shape = [4, 4, 1] x = np.zeros(x_shape) @@ -1672,6 +1698,7 @@ class CropToBoundingBoxTest(test_util.TensorFlowTestCase): for params, err_msg in test_config: self._assertRaises(x, x_shape, *params, err_msg=err_msg) + @test_util.run_deprecated_v1 def testNameScope(self): image = array_ops.placeholder(dtypes.float32, shape=[55, 66, 3]) y = image_ops.crop_to_bounding_box(image, 0, 0, 55, 66) @@ -1688,6 +1715,7 @@ class CentralCropTest(test_util.TensorFlowTestCase): else: self.assertEqual(y.get_shape().as_list(), post_shape) + @test_util.run_deprecated_v1 def testNoOp(self): x_shapes = [[13, 9, 3], [5, 13, 9, 3]] for x_shape in x_shapes: @@ -1696,7 +1724,7 @@ class CentralCropTest(test_util.TensorFlowTestCase): with self.test_session(use_gpu=use_gpu): x = constant_op.constant(x_np, shape=x_shape) y = image_ops.central_crop(x, 1.0) - y_tf = y.eval() + y_tf = self.evaluate(y) self.assertAllEqual(y_tf, x_np) self.assertEqual(y.op.name, x.op.name) @@ -1711,7 +1739,7 @@ class CentralCropTest(test_util.TensorFlowTestCase): with self.test_session(use_gpu=use_gpu): x = constant_op.constant(x_np, shape=x_shape) y = image_ops.central_crop(x, 0.5) - y_tf = y.eval() + y_tf = self.evaluate(y) self.assertAllEqual(y_tf, y_np) self.assertAllEqual(y_tf.shape, y_np.shape) @@ -1727,10 +1755,11 @@ class CentralCropTest(test_util.TensorFlowTestCase): with self.test_session(use_gpu=True): x = constant_op.constant(x_np, shape=x_shape) y = image_ops.central_crop(x, 0.5) - y_tf = y.eval() + y_tf = self.evaluate(y) self.assertAllEqual(y_tf, y_np) self.assertAllEqual(y_tf.shape, y_np.shape) + @test_util.run_deprecated_v1 def testCropping2(self): # Test case for 10315 x_shapes = [[240, 320, 3], [5, 240, 320, 3]] @@ -1747,6 +1776,7 @@ class CentralCropTest(test_util.TensorFlowTestCase): self.assertAllEqual(y_tf, y_np) self.assertAllEqual(y_tf.shape, y_np.shape) + @test_util.run_deprecated_v1 def testShapeInference(self): # Test no-op fraction=1.0, with 3-D tensors. self._assertShapeInference([50, 60, 3], 1.0, [50, 60, 3]) @@ -1807,6 +1837,7 @@ class CentralCropTest(test_util.TensorFlowTestCase): with self.assertRaises(ValueError): _ = image_ops.central_crop(x, 0.5) + @test_util.run_deprecated_v1 def testNameScope(self): x_shape = [13, 9, 3] x_np = np.ones(x_shape, dtype=np.float32) @@ -1897,14 +1928,16 @@ class PadToBoundingBoxTest(test_util.TensorFlowTestCase): i = constant_op.constant([1, 0, 4, 3], dtype=dtypes.int64) y_tf = image_ops.pad_to_bounding_box(x, i[0], i[1], i[2], i[3]) with self.test_session(use_gpu=True): - self.assertAllClose(y, y_tf.eval()) + self.assertAllClose(y, self.evaluate(y_tf)) + @test_util.run_deprecated_v1 def testNoOp(self): x_shape = [10, 10, 10] x = np.random.uniform(size=x_shape) offset_height, offset_width = [0, 0] self._assertReturns(x, x_shape, offset_height, offset_width, x, x_shape) + @test_util.run_deprecated_v1 def testPadding(self): x = [1, 2, 3, 4, 5, 6, 7, 8, 9] x_shape = [3, 3, 1] @@ -1929,6 +1962,7 @@ class PadToBoundingBoxTest(test_util.TensorFlowTestCase): y_shape = [3, 4, 1] self._assertReturns(x, x_shape, offset_height, offset_width, y, y_shape) + @test_util.run_deprecated_v1 def testShapeInference(self): self._assertShapeInference([55, 66, 3], 55, 66, [55, 66, 3]) self._assertShapeInference([50, 60, 3], 55, 66, [55, 66, 3]) @@ -1942,6 +1976,7 @@ class PadToBoundingBoxTest(test_util.TensorFlowTestCase): self._assertShapeInference([None, None, None], 55, 66, [55, 66, None]) self._assertShapeInference(None, 55, 66, [55, 66, None]) + @test_util.run_deprecated_v1 def testNon3DInput(self): # Input image is not 3D x = [0] * 15 @@ -1953,6 +1988,7 @@ class PadToBoundingBoxTest(test_util.TensorFlowTestCase): target_width, "'image' must have either 3 or 4 dimensions.") + @test_util.run_deprecated_v1 def testZeroLengthInput(self): # Input image has 0-length dimension(s). # Each line is a test configuration: @@ -1985,6 +2021,7 @@ class PadToBoundingBoxTest(test_util.TensorFlowTestCase): "all dims of \\'image.shape\\' must be > 0", use_tensor_inputs_options=[True]) + @test_util.run_deprecated_v1 def testBadParams(self): x_shape = [3, 3, 1] x = np.zeros(x_shape) @@ -1999,6 +2036,7 @@ class PadToBoundingBoxTest(test_util.TensorFlowTestCase): for config_item in test_config: self._assertRaises(x, x_shape, *config_item) + @test_util.run_deprecated_v1 def testNameScope(self): image = array_ops.placeholder(dtypes.float32, shape=[55, 66, 3]) y = image_ops.pad_to_bounding_box(image, 0, 0, 55, 66) @@ -2040,7 +2078,7 @@ class SelectDistortedCropBoxTest(test_util.TensorFlowTestCase): y = array_ops.strided_slice(image_tf, begin, begin + size) for _ in xrange(num_iter): - y_tf = y.eval() + y_tf = self.evaluate(y) crop_height = y_tf.shape[0] crop_width = y_tf.shape[1] aspect_ratio = float(crop_width) / float(crop_height) @@ -2106,6 +2144,7 @@ class SelectDistortedCropBoxTest(test_util.TensorFlowTestCase): # TODO(wicke, shlens, dga): Restore this test so that it is no longer flaky. # self.assertGreaterEqual(min(fraction_object_covered), min_object_covered) + @test_util.run_deprecated_v1 def testWholeImageBoundingBox(self): height = 40 width = 50 @@ -2120,6 +2159,7 @@ class SelectDistortedCropBoxTest(test_util.TensorFlowTestCase): aspect_ratio_range=(0.75, 1.33), area_range=(0.05, 1.0)) + @test_util.run_deprecated_v1 def testWithBoundingBox(self): height = 40 width = 50 @@ -2150,6 +2190,7 @@ class SelectDistortedCropBoxTest(test_util.TensorFlowTestCase): aspect_ratio_range=(0.75, 1.33), area_range=(0.05, 1.0)) + @test_util.run_deprecated_v1 def testSampleDistortedBoundingBoxShape(self): with self.test_session(use_gpu=True): image_size = constant_op.constant( @@ -2171,9 +2212,9 @@ class SelectDistortedCropBoxTest(test_util.TensorFlowTestCase): self.assertAllEqual([3], end.get_shape().as_list()) self.assertAllEqual([1, 1, 4], bbox_for_drawing.get_shape().as_list()) # Actual run to make sure shape is correct inside Compute(). - begin = begin.eval() - end = end.eval() - bbox_for_drawing = bbox_for_drawing.eval() + begin = self.evaluate(begin) + end = self.evaluate(end) + bbox_for_drawing = self.evaluate(bbox_for_drawing) begin, end, bbox_for_drawing = image_ops.sample_distorted_bounding_box( image_size=image_size, @@ -2207,9 +2248,9 @@ class SelectDistortedCropBoxTest(test_util.TensorFlowTestCase): self.assertAllEqual([3], end.get_shape().as_list()) self.assertAllEqual([1, 1, 4], bbox_for_drawing.get_shape().as_list()) # Actual run to make sure shape is correct inside Compute(). - begin = begin.eval() - end = end.eval() - bbox_for_drawing = bbox_for_drawing.eval() + begin = self.evaluate(begin) + end = self.evaluate(end) + bbox_for_drawing = self.evaluate(bbox_for_drawing) class ResizeImagesTest(test_util.TensorFlowTestCase): @@ -2245,6 +2286,7 @@ class ResizeImagesTest(test_util.TensorFlowTestCase): else: return False + @test_util.run_deprecated_v1 def testNoOp(self): img_shape = [1, 6, 4, 1] single_shape = [6, 4, 1] @@ -2265,7 +2307,7 @@ class ResizeImagesTest(test_util.TensorFlowTestCase): image = constant_op.constant(img_np, shape=img_shape) y = image_ops.resize_images(image, [target_height, target_width], opt) yshape = array_ops.shape(y) - resized, newshape = sess.run([y, yshape]) + resized, newshape = self.evaluate([y, yshape]) self.assertAllEqual(img_shape, newshape) self.assertAllClose(resized, img_np, atol=1e-5) @@ -2276,9 +2318,10 @@ class ResizeImagesTest(test_util.TensorFlowTestCase): y = image_ops.resize_images(image, [target_height, target_width], self.OPTIONS[0]) yshape = array_ops.shape(y) - newshape = yshape.eval() + newshape = self.evaluate(yshape) self.assertAllEqual(single_shape, newshape) + @test_util.run_deprecated_v1 def testTensorArguments(self): img_shape = [1, 6, 4, 1] single_shape = [6, 4, 1] @@ -2340,6 +2383,7 @@ class ResizeImagesTest(test_util.TensorFlowTestCase): _ = image_ops.resize_images(image, [6, None], image_ops.ResizeMethod.BILINEAR) + @test_util.run_deprecated_v1 def testReturnDtype(self): target_shapes = [[6, 4], [3, 2], [ array_ops.placeholder(dtypes.int32), @@ -2379,7 +2423,7 @@ class ResizeImagesTest(test_util.TensorFlowTestCase): image = constant_op.constant(img_np, shape=img_shape) y = image_ops.resize_images(image, [height, width], opt) yshape = array_ops.shape(y) - resized, newshape = sess.run([y, yshape]) + resized, newshape = self.evaluate([y, yshape]) self.assertAllEqual(img_shape, newshape) self.assertAllClose(resized, img_np, atol=1e-5) @@ -2411,7 +2455,7 @@ class ResizeImagesTest(test_util.TensorFlowTestCase): y = image_ops.resize_images(image, [target_height, target_width], opt) expected = np.array(expected_data).reshape(target_shape) - resized = y.eval() + resized = self.evaluate(y) self.assertAllClose(resized, expected, atol=1e-5) def testResizeUpAlignCornersFalse(self): @@ -2446,7 +2490,7 @@ class ResizeImagesTest(test_util.TensorFlowTestCase): image = constant_op.constant(img_np, shape=img_shape) y = image_ops.resize_images( image, [target_height, target_width], opt, align_corners=False) - resized = y.eval() + resized = self.evaluate(y) expected = np.array(expected_data[opt]).reshape( [1, target_height, target_width, 1]) self.assertAllClose(resized, expected, atol=1e-05) @@ -2482,7 +2526,7 @@ class ResizeImagesTest(test_util.TensorFlowTestCase): image = constant_op.constant(img_np, shape=img_shape) y = image_ops.resize_images( image, [target_height, target_width], opt, align_corners=True) - resized = y.eval() + resized = self.evaluate(y) expected = np.array(expected_data[opt]).reshape( [1, target_height, target_width, 1]) self.assertAllClose(resized, expected, atol=1e-05) @@ -2509,7 +2553,7 @@ class ResizeImagesTest(test_util.TensorFlowTestCase): image = constant_op.constant(img_np, shape=img_shape) y = image_ops.resize_images(image, [target_height, target_width], image_ops.ResizeMethod.BICUBIC) - resized = y.eval() + resized = self.evaluate(y) expected = np.array(expected_data).reshape( [1, target_height, target_width, 1]) self.assertAllClose(resized, expected, atol=1) @@ -2534,7 +2578,7 @@ class ResizeImagesTest(test_util.TensorFlowTestCase): image_ops.ResizeMethod.AREA) expected = np.array(expected_data).reshape( [1, target_height, target_width, 1]) - resized = y.eval() + resized = self.evaluate(y) self.assertAllClose(resized, expected, atol=1) def testCompareNearestNeighbor(self): @@ -2554,7 +2598,7 @@ class ResizeImagesTest(test_util.TensorFlowTestCase): new_size, image_ops.ResizeMethod.NEAREST_NEIGHBOR, align_corners=align_corners) - gpu_val = out_op.eval() + gpu_val = self.evaluate(out_op) with self.test_session(use_gpu=False): image = constant_op.constant(img_np, shape=input_shape) new_size = constant_op.constant([target_height, target_width]) @@ -2563,7 +2607,7 @@ class ResizeImagesTest(test_util.TensorFlowTestCase): new_size, image_ops.ResizeMethod.NEAREST_NEIGHBOR, align_corners=align_corners) - cpu_val = out_op.eval() + cpu_val = self.evaluate(out_op) self.assertAllClose(cpu_val, gpu_val, rtol=1e-5, atol=1e-5) def testCompareBilinear(self): @@ -2585,9 +2629,10 @@ class ResizeImagesTest(test_util.TensorFlowTestCase): new_size, image_ops.ResizeMethod.BILINEAR, align_corners=align_corners) - value[use_gpu] = out_op.eval() + value[use_gpu] = self.evaluate(out_op) self.assertAllClose(value[True], value[False], rtol=1e-5, atol=1e-5) + @test_util.run_deprecated_v1 def testShapeInference(self): self._assertShapeInference([50, 60, 3], [55, 66], [55, 66, 3]) self._assertShapeInference([55, 66, 3], [55, 66], [55, 66, 3]) @@ -2608,12 +2653,13 @@ class ResizeImagesTest(test_util.TensorFlowTestCase): self._assertShapeInference([59, 60, None], [55, 66], [55, 66, None]) self._assertShapeInference([None, None, None], [55, 66], [55, 66, None]) + @test_util.run_deprecated_v1 def testNameScope(self): img_shape = [1, 3, 2, 1] with self.test_session(use_gpu=True): single_image = array_ops.placeholder(dtypes.float32, shape=[50, 60, 3]) y = image_ops.resize_images(single_image, [55, 66]) - self.assertTrue(y.op.name.startswith("resize_images")) + self.assertTrue(y.op.name.startswith("resize")) def _ResizeImageCall(self, x, max_h, max_w, preserve_aspect_ratio, use_tensor_inputs): @@ -2658,6 +2704,7 @@ class ResizeImagesTest(test_util.TensorFlowTestCase): preserve_aspect_ratio, use_tensor_inputs) self.assertShapeEqual(y, ops.convert_to_tensor(y_tf)) + @test_util.run_deprecated_v1 def testPreserveAspectRatioMultipleImages(self): x_shape = [10, 100, 100, 10] x = np.random.uniform(size=x_shape) @@ -2665,36 +2712,42 @@ class ResizeImagesTest(test_util.TensorFlowTestCase): self._assertResizeCheckShape(x, x_shape, [250, 250], [10, 250, 250, 10], preserve_aspect_ratio=False) + @test_util.run_deprecated_v1 def testPreserveAspectRatioNoOp(self): x_shape = [10, 10, 10] x = np.random.uniform(size=x_shape) self._assertResizeEqual(x, x_shape, x, x_shape) + @test_util.run_deprecated_v1 def testPreserveAspectRatioSmaller(self): x_shape = [100, 100, 10] x = np.random.uniform(size=x_shape) self._assertResizeCheckShape(x, x_shape, [75, 50], [50, 50, 10]) + @test_util.run_deprecated_v1 def testPreserveAspectRatioSmallerMultipleImages(self): x_shape = [10, 100, 100, 10] x = np.random.uniform(size=x_shape) self._assertResizeCheckShape(x, x_shape, [75, 50], [10, 50, 50, 10]) + @test_util.run_deprecated_v1 def testPreserveAspectRatioLarger(self): x_shape = [100, 100, 10] x = np.random.uniform(size=x_shape) self._assertResizeCheckShape(x, x_shape, [150, 200], [150, 150, 10]) + @test_util.run_deprecated_v1 def testPreserveAspectRatioSameRatio(self): x_shape = [1920, 1080, 3] x = np.random.uniform(size=x_shape) self._assertResizeCheckShape(x, x_shape, [3840, 2160], [3840, 2160, 3]) + @test_util.run_deprecated_v1 def testPreserveAspectRatioSquare(self): x_shape = [299, 299, 3] x = np.random.uniform(size=x_shape) @@ -2764,12 +2817,14 @@ class ResizeImageWithPadTest(test_util.TensorFlowTestCase): y = image_ops.resize_image_with_pad(image, height, width) self.assertEqual(y.get_shape().as_list(), post_shape) + @test_util.run_deprecated_v1 def testNoOp(self): x_shape = [10, 10, 10] x = np.random.uniform(size=x_shape) self._assertReturns(x, x_shape, x, x_shape) + @test_util.run_deprecated_v1 def testPad(self): # Reduce vertical dimension x = [1, 2, 3, 4, 5, 6, 7, 8] @@ -2860,12 +2915,14 @@ class ResizeImageWithCropOrPadTest(test_util.TensorFlowTestCase): y = image_ops.resize_image_with_crop_or_pad(image, height, width) self.assertEqual(y.get_shape().as_list(), post_shape) + @test_util.run_deprecated_v1 def testNoOp(self): x_shape = [10, 10, 10] x = np.random.uniform(size=x_shape) self._assertReturns(x, x_shape, x, x_shape) + @test_util.run_deprecated_v1 def testPad(self): # Pad even along col. x = [1, 2, 3, 4, 5, 6, 7, 8] @@ -2903,6 +2960,7 @@ class ResizeImageWithCropOrPadTest(test_util.TensorFlowTestCase): self._assertReturns(x, x_shape, y, y_shape) + @test_util.run_deprecated_v1 def testCrop(self): # Crop even along col. x = [1, 2, 3, 4, 5, 6, 7, 8] @@ -2940,6 +2998,7 @@ class ResizeImageWithCropOrPadTest(test_util.TensorFlowTestCase): self._assertReturns(x, x_shape, y, y_shape) + @test_util.run_deprecated_v1 def testCropAndPad(self): # Pad along row but crop along col. x = [1, 2, 3, 4, 5, 6, 7, 8] @@ -2959,6 +3018,7 @@ class ResizeImageWithCropOrPadTest(test_util.TensorFlowTestCase): self._assertReturns(x, x_shape, y, y_shape) + @test_util.run_deprecated_v1 def testShapeInference(self): self._assertShapeInference([50, 60, 3], 55, 66, [55, 66, 3]) self._assertShapeInference([55, 66, 3], 55, 66, [55, 66, 3]) @@ -2980,6 +3040,7 @@ class ResizeImageWithCropOrPadTest(test_util.TensorFlowTestCase): self._assertShapeInference([None, None, None], 55, 66, [55, 66, None]) self._assertShapeInference(None, 55, 66, [55, 66, None]) + @test_util.run_deprecated_v1 def testNon3DInput(self): # Input image is not 3D x = [0] * 15 @@ -2993,6 +3054,7 @@ class ResizeImageWithCropOrPadTest(test_util.TensorFlowTestCase): self._assertRaises(x, x_shape, target_height, target_width, "'image' must have either 3 or 4 dimensions.") + @test_util.run_deprecated_v1 def testZeroLengthInput(self): # Input image has 0-length dimension(s). target_height, target_width = [1, 1] @@ -3018,6 +3080,7 @@ class ResizeImageWithCropOrPadTest(test_util.TensorFlowTestCase): "all dims of \\'image.shape\\' must be > 0", use_tensor_inputs_options=[True]) + @test_util.run_deprecated_v1 def testBadParams(self): x_shape = [4, 4, 1] x = np.zeros(x_shape) @@ -3032,6 +3095,7 @@ class ResizeImageWithCropOrPadTest(test_util.TensorFlowTestCase): self._assertRaises(x, x_shape, target_height, target_width, "target_width must be > 0") + @test_util.run_deprecated_v1 def testNameScope(self): image = array_ops.placeholder(dtypes.float32, shape=[50, 60, 3]) y = image_ops.resize_image_with_crop_or_pad(image, 55, 66) @@ -3066,7 +3130,7 @@ class JpegTest(test_util.TensorFlowTestCase): jpeg0 = io_ops.read_file(path) image0 = image_ops.decode_jpeg(jpeg0) image1 = image_ops.decode_jpeg(image_ops.encode_jpeg(image0)) - jpeg0, image0, image1 = sess.run([jpeg0, image0, image1]) + jpeg0, image0, image1 = self.evaluate([jpeg0, image0, image1]) self.assertEqual(len(jpeg0), 3771) self.assertEqual(image0.shape, (256, 128, 3)) self.assertLess(self.averageError(image0, image1), 1.4) @@ -3083,7 +3147,7 @@ class JpegTest(test_util.TensorFlowTestCase): io_ops.read_file(rgb_path), channels=channels) cmyk = image_ops.decode_jpeg( io_ops.read_file(cmyk_path), channels=channels) - rgb, cmyk = sess.run([rgb, cmyk]) + rgb, cmyk = self.evaluate([rgb, cmyk]) self.assertEqual(rgb.shape, shape) self.assertEqual(cmyk.shape, shape) error = self.averageError(rgb, cmyk) @@ -3112,9 +3176,10 @@ class JpegTest(test_util.TensorFlowTestCase): image2.get_shape().as_list()) # CropAndDecode should be equal to DecodeJpeg+Crop. - image1_crop, image2 = sess.run([image1_crop, image2]) + image1_crop, image2 = self.evaluate([image1_crop, image2]) self.assertAllEqual(image1_crop, image2) + @test_util.run_deprecated_v1 def testCropAndDecodeJpegWithInvalidCropWindow(self): with self.cached_session() as sess: # Encode it, then decode it, then encode it @@ -3131,7 +3196,7 @@ class JpegTest(test_util.TensorFlowTestCase): with self.assertRaisesWithPredicateMatch( errors.InvalidArgumentError, lambda e: "Invalid JPEG data or crop window" in str(e)): - sess.run(result) + self.evaluate(result) def testSynthetic(self): with self.test_session(use_gpu=True) as sess: @@ -3141,7 +3206,8 @@ class JpegTest(test_util.TensorFlowTestCase): image1 = image_ops.decode_jpeg(jpeg0, dct_method="INTEGER_ACCURATE") image2 = image_ops.decode_jpeg( image_ops.encode_jpeg(image1), dct_method="INTEGER_ACCURATE") - jpeg0, image0, image1, image2 = sess.run([jpeg0, image0, image1, image2]) + jpeg0, image0, image1, image2 = self.evaluate( + [jpeg0, image0, image1, image2]) # The decoded-encoded image should be similar to the input self.assertLess(self.averageError(image0, image1), 0.6) @@ -3161,7 +3227,8 @@ class JpegTest(test_util.TensorFlowTestCase): image1 = image_ops.decode_jpeg(jpeg0, dct_method="INTEGER_FAST") image2 = image_ops.decode_jpeg( image_ops.encode_jpeg(image1), dct_method="INTEGER_FAST") - jpeg0, image0, image1, image2 = sess.run([jpeg0, image0, image1, image2]) + jpeg0, image0, image1, image2 = self.evaluate( + [jpeg0, image0, image1, image2]) # The decoded-encoded image should be similar to the input, but # note this is worse than the slower algorithm because it is @@ -3184,11 +3251,12 @@ class JpegTest(test_util.TensorFlowTestCase): jpeg0 = image_ops.encode_jpeg(image0) image1 = image_ops.decode_jpeg(jpeg0, dct_method="INTEGER_FAST") image2 = image_ops.decode_jpeg(jpeg0) - image1, image2 = sess.run([image1, image2]) + image1, image2 = self.evaluate([image1, image2]) # The images should be the same. self.assertAllClose(image1, image2) + @test_util.run_deprecated_v1 def testShape(self): with self.test_session(use_gpu=True) as sess: jpeg = constant_op.constant("nonsense") @@ -3197,6 +3265,7 @@ class JpegTest(test_util.TensorFlowTestCase): self.assertEqual(image.get_shape().as_list(), [None, None, channels or None]) + @test_util.run_deprecated_v1 def testExtractJpegShape(self): # Read a real jpeg and verify shape. path = ("tensorflow/core/lib/jpeg/testdata/" @@ -3207,6 +3276,7 @@ class JpegTest(test_util.TensorFlowTestCase): [image_shape] = sess.run([image_ops.extract_jpeg_shape(jpeg)]) self.assertEqual(image_shape.tolist(), [256, 128, 3]) + @test_util.run_deprecated_v1 def testExtractJpegShapeforCmyk(self): # Read a cmyk jpeg image, and verify its shape. path = ("tensorflow/core/lib/jpeg/testdata/" @@ -3230,11 +3300,11 @@ class PngTest(test_util.TensorFlowTestCase): with self.test_session(use_gpu=True) as sess: png0 = io_ops.read_file(prefix + filename) image0 = image_ops.decode_png(png0, channels=channels) - png0, image0 = sess.run([png0, image0]) + png0, image0 = self.evaluate([png0, image0]) self.assertEqual(image0.shape, (26, 51, channels or channels_in)) if channels == channels_in: image1 = image_ops.decode_png(image_ops.encode_png(image0)) - self.assertAllEqual(image0, image1.eval()) + self.assertAllEqual(image0, self.evaluate(image1)) def testSynthetic(self): with self.test_session(use_gpu=True) as sess: @@ -3242,7 +3312,7 @@ class PngTest(test_util.TensorFlowTestCase): image0 = constant_op.constant(_SimpleColorRamp()) png0 = image_ops.encode_png(image0, compression=7) image1 = image_ops.decode_png(png0) - png0, image0, image1 = sess.run([png0, image0, image1]) + png0, image0, image1 = self.evaluate([png0, image0, image1]) # PNG is lossless self.assertAllEqual(image0, image1) @@ -3257,7 +3327,7 @@ class PngTest(test_util.TensorFlowTestCase): image0 = constant_op.constant(_SimpleColorRamp(), dtype=dtypes.uint16) png0 = image_ops.encode_png(image0, compression=7) image1 = image_ops.decode_png(png0, dtype=dtypes.uint16) - png0, image0, image1 = sess.run([png0, image0, image1]) + png0, image0, image1 = self.evaluate([png0, image0, image1]) # PNG is lossless self.assertAllEqual(image0, image1) @@ -3273,7 +3343,7 @@ class PngTest(test_util.TensorFlowTestCase): image0 = constant_op.constant(gray_alpha) png0 = image_ops.encode_png(image0, compression=7) image1 = image_ops.decode_png(png0) - png0, image0, image1 = sess.run([png0, image0, image1]) + png0, image0, image1 = self.evaluate([png0, image0, image1]) self.assertEqual(2, image0.shape[-1]) self.assertAllEqual(image0, image1) @@ -3284,10 +3354,11 @@ class PngTest(test_util.TensorFlowTestCase): image0 = constant_op.constant(gray_alpha, dtype=dtypes.uint16) png0 = image_ops.encode_png(image0, compression=7) image1 = image_ops.decode_png(png0, dtype=dtypes.uint16) - png0, image0, image1 = sess.run([png0, image0, image1]) + png0, image0, image1 = self.evaluate([png0, image0, image1]) self.assertEqual(2, image0.shape[-1]) self.assertAllEqual(image0, image1) + @test_util.run_deprecated_v1 def testShape(self): with self.test_session(use_gpu=True): png = constant_op.constant("nonsense") @@ -3310,7 +3381,7 @@ class GifTest(test_util.TensorFlowTestCase): with self.test_session(use_gpu=True) as sess: gif0 = io_ops.read_file(prefix + filename) image0 = image_ops.decode_gif(gif0) - gif0, image0 = sess.run([gif0, image0]) + gif0, image0 = self.evaluate([gif0, image0]) self.assertEqual(image0.shape, shape) @@ -3332,6 +3403,7 @@ class GifTest(test_util.TensorFlowTestCase): self._testValid("scan.gif") self._testValid("optimized.gif") + @test_util.run_deprecated_v1 def testShape(self): with self.test_session(use_gpu=True) as sess: gif = constant_op.constant("nonsense") @@ -3358,6 +3430,7 @@ class ConvertImageTest(test_util.TensorFlowTestCase): self.assertTrue(y_saturate.dtype == output_dtype) self.assertAllClose(y_saturate.eval(), y_np, atol=1e-5) + @test_util.run_deprecated_v1 def testNoConvert(self): # Make sure converting to the same data type creates only an identity op with self.test_session(use_gpu=True): @@ -3367,6 +3440,7 @@ class ConvertImageTest(test_util.TensorFlowTestCase): self.assertEquals(y.op.type, "Identity") self.assertEquals(y.op.inputs[0], image) + @test_util.run_deprecated_v1 def testConvertBetweenInteger(self): # Make sure converting to between integer types scales appropriately with self.test_session(use_gpu=True): @@ -3375,6 +3449,7 @@ class ConvertImageTest(test_util.TensorFlowTestCase): self._convert([0, 2**32], dtypes.int64, dtypes.int32, [0, 1]) self._convert([0, 1], dtypes.int32, dtypes.int64, [0, 2**32]) + @test_util.run_deprecated_v1 def testConvertBetweenFloat(self): # Make sure converting to between float types does nothing interesting with self.test_session(use_gpu=True): @@ -3383,6 +3458,7 @@ class ConvertImageTest(test_util.TensorFlowTestCase): self._convert([-1.0, 0, 1.0, 200000], dtypes.float64, dtypes.float32, [-1.0, 0, 1.0, 200000]) + @test_util.run_deprecated_v1 def testConvertBetweenIntegerAndFloat(self): # Make sure converting from and to a float type scales appropriately with self.test_session(use_gpu=True): @@ -3391,6 +3467,7 @@ class ConvertImageTest(test_util.TensorFlowTestCase): self._convert([0, 1.1 / 255.0, 1], dtypes.float32, dtypes.uint8, [0, 1, 255]) + @test_util.run_deprecated_v1 def testConvertBetweenInt16AndInt8(self): with self.test_session(use_gpu=True): # uint8, uint16 @@ -3431,7 +3508,7 @@ class TotalVariationTest(test_util.TensorFlowTestCase): y = image_ops.total_variation(images=x_tf) # Run the TensorFlow session to calculate the result. - y_tf = y.eval() + y_tf = self.evaluate(y) # Assert that the results are as expected within # some small error-bound in case they are float-values. @@ -3582,6 +3659,7 @@ class TotalVariationTest(test_util.TensorFlowTestCase): class FormatTest(test_util.TensorFlowTestCase): + @test_util.run_deprecated_v1 def testFormats(self): prefix = "tensorflow/core/lib" paths = ("png/testdata/lena_gray.png", "jpeg/testdata/jpeg_merge_test1.jpg", @@ -3614,6 +3692,7 @@ class FormatTest(test_util.TensorFlowTestCase): class NonMaxSuppressionTest(test_util.TensorFlowTestCase): + @test_util.run_deprecated_v1 def testSelectFromThreeClusters(self): boxes_np = [[0, 0, 1, 1], [0, 0.1, 1, 1.1], [0, -0.1, 1, 0.9], [0, 10, 1, 11], [0, 10.1, 1, 11.1], [0, 100, 1, 101]] @@ -3629,6 +3708,7 @@ class NonMaxSuppressionTest(test_util.TensorFlowTestCase): boxes, scores, max_output_size, iou_threshold).eval() self.assertAllClose(selected_indices, [3, 0, 5]) + @test_util.run_deprecated_v1 def testInvalidShape(self): # The boxes should be 2D of shape [num_boxes, 4]. with self.assertRaisesRegexp(ValueError, @@ -3671,6 +3751,7 @@ class NonMaxSuppressionTest(test_util.TensorFlowTestCase): scores = constant_op.constant([0.9]) image_ops.non_max_suppression(boxes, scores, 3, [[0.5]]) + @test_util.run_deprecated_v1 def testDataTypes(self): # Test case for GitHub issue 20199. boxes_np = [[0, 0, 1, 1], [0, 0.1, 1, 1.1], [0, -0.1, 1, 0.9], @@ -3709,12 +3790,13 @@ class NonMaxSuppressionTest(test_util.TensorFlowTestCase): iou_threshold = constant_op.constant(iou_threshold_np) selected_indices, _ = gen_image_ops.non_max_suppression_v4( boxes, scores, max_output_size, iou_threshold, score_threshold) - selected_indices = selected_indices.eval() + selected_indices = self.evaluate(selected_indices) self.assertAllClose(selected_indices, [3, 0, 5]) class NonMaxSuppressionPaddedTest(test_util.TensorFlowTestCase): + @test_util.run_deprecated_v1 def testSelectFromThreeClusters(self): boxes_np = [[0, 0, 1, 1], [0, 0.1, 1, 1.1], [0, -0.1, 1, 0.9], [0, 10, 1, 11], [0, 10.1, 1, 11.1], [0, 100, 1, 101]] @@ -3747,6 +3829,7 @@ class NonMaxSuppressionPaddedTest(test_util.TensorFlowTestCase): self.assertAllClose(selected_indices.eval(), [3, 0, 5]) self.assertEqual(num_valid.eval(), 3) + @test_util.run_deprecated_v1 def testSelectFromContinuousOverLap(self): boxes_np = [[0, 0, 1, 1], [0, 0.2, 1, 1.2], [0, 0.4, 1, 1.4], [0, 0.6, 1, 1.6], [0, 0.8, 1, 1.8], [0, 2, 1, 2]] @@ -3774,6 +3857,7 @@ class NonMaxSuppressionPaddedTest(test_util.TensorFlowTestCase): class NonMaxSuppressionWithOverlapsTest(test_util.TensorFlowTestCase): + @test_util.run_deprecated_v1 def testSelectOneFromThree(self): overlaps_np = [ [1.0, 0.7, 0.2], @@ -3799,6 +3883,7 @@ class NonMaxSuppressionWithOverlapsTest(test_util.TensorFlowTestCase): class VerifyCompatibleImageShapesTest(test_util.TensorFlowTestCase): """Tests utility function used by ssim() and psnr().""" + @test_util.run_deprecated_v1 def testWrongDims(self): img = array_ops.placeholder(dtype=dtypes.float32) img_np = np.array((2, 2)) @@ -3808,6 +3893,7 @@ class VerifyCompatibleImageShapesTest(test_util.TensorFlowTestCase): with self.assertRaises(errors.InvalidArgumentError): sess.run(checks, {img: img_np}) + @test_util.run_deprecated_v1 def testShapeMismatch(self): img1 = array_ops.placeholder(dtype=dtypes.float32) img2 = array_ops.placeholder(dtype=dtypes.float32) @@ -3829,7 +3915,7 @@ class PSNRTest(test_util.TensorFlowTestCase): "tensorflow/core/lib/psnr/testdata", filename)) im = image_ops.decode_jpeg(content, dct_method="INTEGER_ACCURATE") im = image_ops.convert_image_dtype(im, dtypes.float32) - im, = sess.run([im]) + im, = self.evaluate([im]) return np.expand_dims(im, axis=0) def _LoadTestImages(self): @@ -3848,6 +3934,7 @@ class PSNRTest(test_util.TensorFlowTestCase): """Returns an image or image batch with given shape.""" return np.random.rand(*shape).astype(np.float32) * max_val + @test_util.run_deprecated_v1 def testPSNRSingleImage(self): image1 = self._RandomImage((8, 8, 1), 1) image2 = self._RandomImage((8, 8, 1), 1) @@ -3861,6 +3948,7 @@ class PSNRTest(test_util.TensorFlowTestCase): tf_psnr = image_ops.psnr(tf_image1, tf_image2, 1.0, "psnr").eval() self.assertAllClose(psnr, tf_psnr, atol=0.001) + @test_util.run_deprecated_v1 def testPSNRMultiImage(self): image1 = self._RandomImage((10, 8, 8, 1), 1) image2 = self._RandomImage((10, 8, 8, 1), 1) @@ -3874,6 +3962,7 @@ class PSNRTest(test_util.TensorFlowTestCase): tf_psnr = image_ops.psnr(tf_image1, tf_image2, 1, "psnr").eval() self.assertAllClose(psnr, tf_psnr, atol=0.001) + @test_util.run_deprecated_v1 def testGoldenPSNR(self): q20, q72, q95 = self._LoadTestImages() @@ -3898,6 +3987,7 @@ class PSNRTest(test_util.TensorFlowTestCase): self.assertAllClose(psnr2, tf_psnr2, atol=0.001) self.assertAllClose(psnr3, tf_psnr3, atol=0.001) + @test_util.run_deprecated_v1 def testInfinity(self): q20, _, _ = self._LoadTestImages() psnr = self._PSNR_NumPy(q20, q20, 1) @@ -3906,6 +3996,7 @@ class PSNRTest(test_util.TensorFlowTestCase): tf_psnr = image_ops.psnr(tf_q20, tf_q20, 1, "psnr").eval() self.assertAllClose(psnr, tf_psnr, atol=0.001) + @test_util.run_deprecated_v1 def testInt(self): img1 = self._RandomImage((10, 8, 8, 1), 255) img2 = self._RandomImage((10, 8, 8, 1), 255) @@ -3916,7 +4007,8 @@ class PSNRTest(test_util.TensorFlowTestCase): img2 = image_ops.convert_image_dtype(img2, dtypes.float32) psnr_float32 = image_ops.psnr(img1, img2, 1.0) with self.test_session(use_gpu=True): - self.assertAllClose(psnr_uint8.eval(), psnr_float32.eval(), atol=0.001) + self.assertAllClose( + psnr_uint8.eval(), self.evaluate(psnr_float32), atol=0.001) class SSIMTest(test_util.TensorFlowTestCase): @@ -3935,7 +4027,7 @@ class SSIMTest(test_util.TensorFlowTestCase): "tensorflow/core/lib/ssim/testdata", filename)) im = image_ops.decode_png(content) im = image_ops.convert_image_dtype(im, dtypes.float32) - im, = sess.run([im]) + im, = self.evaluate([im]) return np.expand_dims(im, axis=0) def _LoadTestImages(self): @@ -3946,6 +4038,7 @@ class SSIMTest(test_util.TensorFlowTestCase): """Returns an image or image batch with given shape.""" return np.random.rand(*shape).astype(np.float32) * max_val + @test_util.run_deprecated_v1 def testAgainstMatlab(self): """Tests against values produced by Matlab.""" img = self._LoadTestImages() @@ -3969,7 +4062,7 @@ class SSIMTest(test_util.TensorFlowTestCase): ssim = image_ops.ssim(constant_op.constant(img1), constant_op.constant(img2), 1.0) with self.test_session(use_gpu=True): - self.assertAllClose(expected, ssim.eval(), atol=1e-4) + self.assertAllClose(expected, self.evaluate(ssim), atol=1e-4) def testBroadcast(self): img = self._LoadTestImages()[:2] @@ -3981,8 +4074,9 @@ class SSIMTest(test_util.TensorFlowTestCase): ssim = image_ops.ssim(img1, img2, 1.0) with self.test_session(use_gpu=True): - self.assertAllClose(expected, ssim.eval(), atol=1e-4) + self.assertAllClose(expected, self.evaluate(ssim), atol=1e-4) + @test_util.run_deprecated_v1 def testNegative(self): """Tests against negative SSIM index.""" step = np.expand_dims(np.arange(0, 256, 16, dtype=np.uint8), axis=0) @@ -3997,6 +4091,7 @@ class SSIMTest(test_util.TensorFlowTestCase): with self.test_session(use_gpu=True): self.assertLess(ssim.eval(), 0) + @test_util.run_deprecated_v1 def testInt(self): img1 = self._RandomImage((1, 16, 16, 3), 255) img2 = self._RandomImage((1, 16, 16, 3), 255) @@ -4007,7 +4102,8 @@ class SSIMTest(test_util.TensorFlowTestCase): img2 = image_ops.convert_image_dtype(img2, dtypes.float32) ssim_float32 = image_ops.ssim(img1, img2, 1.0) with self.test_session(use_gpu=True): - self.assertAllClose(ssim_uint8.eval(), ssim_float32.eval(), atol=0.001) + self.assertAllClose( + ssim_uint8.eval(), self.evaluate(ssim_float32), atol=0.001) class MultiscaleSSIMTest(test_util.TensorFlowTestCase): @@ -4026,7 +4122,7 @@ class MultiscaleSSIMTest(test_util.TensorFlowTestCase): "tensorflow/core/lib/ssim/testdata", filename)) im = image_ops.decode_png(content) im = image_ops.convert_image_dtype(im, dtypes.float32) - im, = sess.run([im]) + im, = self.evaluate([im]) return np.expand_dims(im, axis=0) def _LoadTestImages(self): @@ -4037,6 +4133,7 @@ class MultiscaleSSIMTest(test_util.TensorFlowTestCase): """Returns an image or image batch with given shape.""" return np.random.rand(*shape).astype(np.float32) * max_val + @test_util.run_deprecated_v1 def testAgainstMatlab(self): """Tests against MS-SSIM computed with Matlab implementation. @@ -4053,6 +4150,7 @@ class MultiscaleSSIMTest(test_util.TensorFlowTestCase): self.assertAllClose(expected, np.squeeze(scores), atol=1e-4) + @test_util.run_deprecated_v1 def testUnweightedIsDifferentiable(self): img = self._LoadTestImages() ph = [array_ops.placeholder(dtype=dtypes.float32) for _ in range(2)] @@ -4077,7 +4175,7 @@ class MultiscaleSSIMTest(test_util.TensorFlowTestCase): msssim = image_ops.ssim_multiscale(constant_op.constant(img1), constant_op.constant(img2), 1.0) with self.test_session(use_gpu=True): - self.assertAllClose(expected, msssim.eval(), 1e-4) + self.assertAllClose(expected, self.evaluate(msssim), 1e-4) def testBroadcast(self): """Tests MS-SSIM broadcasting.""" @@ -4090,7 +4188,7 @@ class MultiscaleSSIMTest(test_util.TensorFlowTestCase): score_tensor = image_ops.ssim_multiscale(img1, img2, 1.0) with self.test_session(use_gpu=True): - self.assertAllClose(expected, score_tensor.eval(), 1e-4) + self.assertAllClose(expected, self.evaluate(score_tensor), 1e-4) def testRange(self): """Tests against low MS-SSIM score. @@ -4108,12 +4206,13 @@ class MultiscaleSSIMTest(test_util.TensorFlowTestCase): images = [ops.convert_to_tensor(x, dtype=dtypes.float32) for x in images] msssim_ops = [image_ops.ssim_multiscale(x, y, 1.0) for x, y in itertools.combinations(images, 2)] - msssim = sess.run(msssim_ops) + msssim = self.evaluate(msssim_ops) msssim = np.squeeze(msssim) self.assertTrue(np.all(msssim >= 0.0)) self.assertTrue(np.all(msssim <= 1.0)) + @test_util.run_deprecated_v1 def testInt(self): img1 = self._RandomImage((1, 180, 240, 3), 255) img2 = self._RandomImage((1, 180, 240, 3), 255) @@ -4124,7 +4223,8 @@ class MultiscaleSSIMTest(test_util.TensorFlowTestCase): img2 = image_ops.convert_image_dtype(img2, dtypes.float32) ssim_float32 = image_ops.ssim_multiscale(img1, img2, 1.0) with self.test_session(use_gpu=True): - self.assertAllClose(ssim_uint8.eval(), ssim_float32.eval(), atol=0.001) + self.assertAllClose( + ssim_uint8.eval(), self.evaluate(ssim_float32), atol=0.001) class ImageGradientsTest(test_util.TensorFlowTestCase): @@ -4139,8 +4239,8 @@ class ImageGradientsTest(test_util.TensorFlowTestCase): dy, dx = image_ops.image_gradients(img) with self.cached_session(): - actual_dy = dy.eval() - actual_dx = dx.eval() + actual_dy = self.evaluate(dy) + actual_dx = self.evaluate(dx) self.assertAllClose(expected_dy, actual_dy) self.assertAllClose(expected_dx, actual_dx) @@ -4164,8 +4264,8 @@ class ImageGradientsTest(test_util.TensorFlowTestCase): assert batch.get_shape().as_list() == [2, 2, 3, 2] dy, dx = image_ops.image_gradients(batch) with self.test_session(use_gpu=True): - actual_dy = dy.eval() - actual_dx = dx.eval() + actual_dy = self.evaluate(dy) + actual_dx = self.evaluate(dx) self.assertAllClose(expected_dy, actual_dy) self.assertAllClose(expected_dx, actual_dx) @@ -4185,7 +4285,7 @@ class SobelEdgesTest(test_util.TensorFlowTestCase): [[0, 0], [0, 12], [0, 0]]], [1, 2, 3, 1, 2]) sobel = image_ops.sobel_edges(img) with self.test_session(use_gpu=True): - actual_sobel = sobel.eval() + actual_sobel = self.evaluate(sobel) self.assertAllClose(expected, actual_sobel) def testSobelEdges5x3x4x2(self): @@ -4207,7 +4307,7 @@ class SobelEdgesTest(test_util.TensorFlowTestCase): sobel = image_ops.sobel_edges(img) with self.test_session(use_gpu=True): - actual_sobel = sobel.eval() + actual_sobel = self.evaluate(sobel) self.assertAllClose(expected_batch, actual_sobel) @@ -4220,7 +4320,7 @@ class DecodeImageTest(test_util.TensorFlowTestCase): image0 = image_ops.decode_image(jpeg0, dtype=dtypes.uint16) image1 = image_ops.convert_image_dtype(image_ops.decode_jpeg(jpeg0), dtypes.uint16) - image0, image1 = sess.run([image0, image1]) + image0, image1 = self.evaluate([image0, image1]) self.assertAllEqual(image0, image1) def testPngUint16(self): @@ -4230,7 +4330,7 @@ class DecodeImageTest(test_util.TensorFlowTestCase): image0 = image_ops.decode_image(png0, dtype=dtypes.uint16) image1 = image_ops.convert_image_dtype( image_ops.decode_png(png0, dtype=dtypes.uint16), dtypes.uint16) - image0, image1 = sess.run([image0, image1]) + image0, image1 = self.evaluate([image0, image1]) self.assertAllEqual(image0, image1) def testGifUint16(self): @@ -4240,7 +4340,7 @@ class DecodeImageTest(test_util.TensorFlowTestCase): image0 = image_ops.decode_image(gif0, dtype=dtypes.uint16) image1 = image_ops.convert_image_dtype(image_ops.decode_gif(gif0), dtypes.uint16) - image0, image1 = sess.run([image0, image1]) + image0, image1 = self.evaluate([image0, image1]) self.assertAllEqual(image0, image1) def testBmpUint16(self): @@ -4250,7 +4350,7 @@ class DecodeImageTest(test_util.TensorFlowTestCase): image0 = image_ops.decode_image(bmp0, dtype=dtypes.uint16) image1 = image_ops.convert_image_dtype(image_ops.decode_bmp(bmp0), dtypes.uint16) - image0, image1 = sess.run([image0, image1]) + image0, image1 = self.evaluate([image0, image1]) self.assertAllEqual(image0, image1) def testJpegFloat32(self): @@ -4260,7 +4360,7 @@ class DecodeImageTest(test_util.TensorFlowTestCase): image0 = image_ops.decode_image(jpeg0, dtype=dtypes.float32) image1 = image_ops.convert_image_dtype(image_ops.decode_jpeg(jpeg0), dtypes.float32) - image0, image1 = sess.run([image0, image1]) + image0, image1 = self.evaluate([image0, image1]) self.assertAllEqual(image0, image1) def testPngFloat32(self): @@ -4270,7 +4370,7 @@ class DecodeImageTest(test_util.TensorFlowTestCase): image0 = image_ops.decode_image(png0, dtype=dtypes.float32) image1 = image_ops.convert_image_dtype( image_ops.decode_png(png0, dtype=dtypes.uint16), dtypes.float32) - image0, image1 = sess.run([image0, image1]) + image0, image1 = self.evaluate([image0, image1]) self.assertAllEqual(image0, image1) def testGifFloat32(self): @@ -4280,7 +4380,7 @@ class DecodeImageTest(test_util.TensorFlowTestCase): image0 = image_ops.decode_image(gif0, dtype=dtypes.float32) image1 = image_ops.convert_image_dtype(image_ops.decode_gif(gif0), dtypes.float32) - image0, image1 = sess.run([image0, image1]) + image0, image1 = self.evaluate([image0, image1]) self.assertAllEqual(image0, image1) def testBmpFloat32(self): @@ -4290,7 +4390,7 @@ class DecodeImageTest(test_util.TensorFlowTestCase): image0 = image_ops.decode_image(bmp0, dtype=dtypes.float32) image1 = image_ops.convert_image_dtype(image_ops.decode_bmp(bmp0), dtypes.float32) - image0, image1 = sess.run([image0, image1]) + image0, image1 = self.evaluate([image0, image1]) self.assertAllEqual(image0, image1) diff --git a/tensorflow/python/ops/init_ops.py b/tensorflow/python/ops/init_ops.py index 4fe6d05620f..c0a4bcd51dd 100644 --- a/tensorflow/python/ops/init_ops.py +++ b/tensorflow/python/ops/init_ops.py @@ -55,6 +55,15 @@ class Initializer(object): """ def __call__(self, shape, dtype=None, partition_info=None): + """Returns a tensor object initialized as specified by the initializer. + + Args: + shape: Shape of the tensor. + dtype: Optional dtype of the tensor. If not provided use the initializer + dtype. + partition_info: Optional information about the possible partitioning of a + tensor. + """ raise NotImplementedError def get_config(self): @@ -143,7 +152,8 @@ class Constant(Initializer): value: A Python scalar, list or tuple of values, or a N-dimensional numpy array. All elements of the initialized variable will be set to the corresponding value in the `value` argument. - dtype: The data type. + dtype: Default data type, used if no `dtype` argument is provided when + calling the initializer. verify_shape: Boolean that enables verification of the shape of `value`. If `True`, the initializer will throw an error if the shape of `value` is not compatible with the shape of the initialized tensor. @@ -216,7 +226,7 @@ class Constant(Initializer): dtype = self.dtype if verify_shape is None: verify_shape = self._verify_shape - return constant_op.constant( + return constant_op.constant_v1( self.value, dtype=dtype, shape=shape, verify_shape=verify_shape) def get_config(self): @@ -239,7 +249,8 @@ class RandomUniform(Initializer): seed: A Python integer. Used to create random seeds. See `tf.set_random_seed` for behavior. - dtype: The data type. + dtype: Default data type, used if no `dtype` argument is provided when + calling the initializer. """ def __init__(self, minval=0, maxval=None, seed=None, dtype=dtypes.float32): @@ -275,7 +286,8 @@ class RandomNormal(Initializer): seed: A Python integer. Used to create random seeds. See `tf.set_random_seed` for behavior. - dtype: The data type. Only floating point types are supported. + dtype: Default data type, used if no `dtype` argument is provided when + calling the initializer. Only floating point types are supported. """ def __init__(self, mean=0.0, stddev=1.0, seed=None, dtype=dtypes.float32): @@ -316,7 +328,8 @@ class TruncatedNormal(Initializer): seed: A Python integer. Used to create random seeds. See `tf.set_random_seed` for behavior. - dtype: The data type. Only floating point types are supported. + dtype: Default data type, used if no `dtype` argument is provided when + calling the initializer. Only floating point types are supported. """ def __init__(self, mean=0.0, stddev=1.0, seed=None, dtype=dtypes.float32): @@ -360,8 +373,7 @@ class UniformUnitScaling(Initializer): A similar calculation for convolutional networks gives an analogous result with `dim` equal to the product of the first 3 dimensions. When nonlinearities are present, we need to multiply this by a constant `factor`. - See [Sussillo et al., 2014](https://arxiv.org/abs/1412.6558) - ([pdf](http://arxiv.org/pdf/1412.6558.pdf)) for deeper motivation, experiments + See (Sussillo et al., 2014) for deeper motivation, experiments and the calculation of constants. In section 2.3 there, the constants were numerically computed: for a linear layer it's 1.0, relu: ~1.43, tanh: ~1.15. @@ -370,7 +382,12 @@ class UniformUnitScaling(Initializer): seed: A Python integer. Used to create random seeds. See `tf.set_random_seed` for behavior. - dtype: The data type. Only floating point types are supported. + dtype: Default data type, used if no `dtype` argument is provided when + calling the initializer. Only floating point types are supported. + + References: + [Sussillo et al., 2014](https://arxiv.org/abs/1412.6558) + ([pdf](http://arxiv.org/pdf/1412.6558.pdf)) """ @deprecated(None, @@ -434,7 +451,8 @@ class VarianceScaling(Initializer): seed: A Python integer. Used to create random seeds. See `tf.set_random_seed` for behavior. - dtype: The data type. Only floating point types are supported. + dtype: Default data type, used if no `dtype` argument is provided when + calling the initializer. Only floating point types are supported. Raises: ValueError: In case of an invalid value for the "scale", mode" or @@ -480,7 +498,7 @@ class VarianceScaling(Initializer): else: scale /= max(1., (fan_in + fan_out) / 2.) if self.distribution == "normal" or self.distribution == "truncated_normal": - # constant taken from scipy.stats.truncnorm.std(a=-2, b=2, loc=0., scale=1.) + # constant taken from scipy.stats.truncnorm.std(a=-2, b=2, loc=0., scale=1.) stddev = math.sqrt(scale) / .87962566103423978 return random_ops.truncated_normal( shape, 0.0, stddev, dtype, seed=self.seed) @@ -531,7 +549,12 @@ class Orthogonal(Initializer): seed: A Python integer. Used to create random seeds. See `tf.set_random_seed` for behavior. - dtype: The data type. + dtype: Default data type, used if no `dtype` argument is provided when + calling the initializer. Only floating point types are supported. + + References: + [Saxe et al., 2014](https://openreview.net/forum?id=_wzZwKpTDF_9C) + ([pdf](https://arxiv.org/pdf/1312.6120.pdf)) """ def __init__(self, gain=1.0, seed=None, dtype=dtypes.float32): @@ -576,16 +599,21 @@ class ConvolutionDeltaOrthogonal(Initializer): The shape of the tensor must have length 3, 4 or 5. The number of input filters must not exceed the number of output filters. The center pixels of the tensor form an orthogonal matrix. Other pixels are set to be zero. See - algorithm 2 in [Xiao et al., 2018]: https://arxiv.org/abs/1806.05393 + algorithm 2 in (Xiao et al., 2018). Args: - gain: Multiplicative factor to apply to the orthogonal matrix. Default is 1. - The 2-norm of an input is multiplied by a factor of 'sqrt(gain)' after - applying this convolution. + gain: Multiplicative factor to apply to the orthogonal + matrix. Default is 1. The 2-norm of an input is multiplied by a factor of + `gain` after applying this convolution. seed: A Python integer. Used to create random seeds. See `tf.set_random_seed` for behavior. - dtype: The data type. + dtype: Default data type, used if no `dtype` argument is provided when + calling the initializer. Only floating point types are supported. + + References: + [Xiao et al., 2018](http://proceedings.mlr.press/v80/xiao18a.html) + ([pdf](http://proceedings.mlr.press/v80/xiao18a/xiao18a.pdf)) """ def __init__(self, gain=1.0, seed=None, dtype=dtypes.float32): @@ -613,7 +641,7 @@ class ConvolutionDeltaOrthogonal(Initializer): d = array_ops.diag_part(r) q *= math_ops.sign(d) q = q[:shape[-2], :] - q *= math_ops.sqrt(math_ops.cast(self.gain, dtype=dtype)) + q *= math_ops.cast(self.gain, dtype=dtype) if len(shape) == 3: weight = array_ops.scatter_nd([[(shape[0]-1)//2]], array_ops.expand_dims(q, 0), shape) @@ -636,12 +664,17 @@ class ConvolutionOrthogonal(Initializer): Base class used to construct 1D, 2D and 3D orthogonal kernels for convolution. Args: - gain: multiplicative factor to apply to the orthogonal matrix. Default is 1. - The 2-norm of an input is multiplied by a factor of 'sqrt(gain)' after - applying this convolution. + gain: multiplicative factor to apply to the orthogonal + matrix. Default is 1. The 2-norm of an input is multiplied by a factor of + `gain` after applying this convolution. seed: A Python integer. Used to create random seeds. See `tf.set_random_seed` for behavior. - dtype: The data type. + dtype: Default data type, used if no `dtype` argument is provided when + calling the initializer. Only floating point types are supported. + + References: + [Xiao et al., 2018](http://proceedings.mlr.press/v80/xiao18a.html) + ([pdf](http://proceedings.mlr.press/v80/xiao18a/xiao18a.pdf)) """ def __init__(self, gain=1.0, seed=None, dtype=dtypes.float32): @@ -698,15 +731,20 @@ class ConvolutionOrthogonal2D(ConvolutionOrthogonal): filters must not exceed the number of output filters. The orthogonality(==isometry) is exact when the inputs are circular padded. There are finite-width effects with non-circular padding (e.g. zero padding). - See algorithm 1 in [Xiao et al., 2018]: https://arxiv.org/abs/1806.05393 + See algorithm 1 in (Xiao et al., 2018). Args: - gain: Multiplicative factor to apply to the orthogonal matrix. Default is 1. - This has the effect of scaling the output 2-norm by a factor of - `sqrt(gain)`. + gain: Multiplicative factor to apply to the orthogonal + matrix. Default is 1. This has the effect of scaling the output 2-norm by + a factor of `gain`. seed: A Python integer. Used to create random seeds. See `tf.set_random_seed` for behavior. - dtype: The data type. + dtype: Default data type, used if no `dtype` argument is provided when + calling the initializer. Only floating point types are supported. + + References: + [Xiao et al., 2018](http://proceedings.mlr.press/v80/xiao18a.html) + ([pdf](http://proceedings.mlr.press/v80/xiao18a/xiao18a.pdf)) """ def __call__(self, shape, dtype=None, partition_info=None): @@ -722,7 +760,7 @@ class ConvolutionOrthogonal2D(ConvolutionOrthogonal): raise ValueError("Kernel sizes must be equal.") kernel = self._orthogonal_kernel(shape[0], shape[2], shape[3]) - kernel *= math_ops.sqrt(math_ops.cast(self.gain, dtype=dtype)) + kernel *= math_ops.cast(self.gain, dtype=dtype) return kernel def _dict_to_tensor(self, x, k1, k2): @@ -834,16 +872,21 @@ class ConvolutionOrthogonal1D(ConvolutionOrthogonal): filters must not exceed the number of output filters. The orthogonality(==isometry) is exact when the inputs are circular padded. There are finite-width effects with non-circular padding (e.g. zero padding). - See algorithm 1 in [Xiao et al., 2018]: https://arxiv.org/abs/1806.05393 + See algorithm 1 in (Xiao et al., 2018). Args: - gain: Multiplicative factor to apply to the orthogonal matrix. Default is 1. - The 2-norm of an input is multiplied by a factor of 'sqrt(gain)' after - applying this convolution. + gain: Multiplicative factor to apply to the orthogonal + matrix. Default is 1. The 2-norm of an input is multiplied by a factor of + `gain` after applying this convolution. seed: A Python integer. Used to create random seeds. See `tf.set_random_seed` for behavior. - dtype: The data type. + dtype: Default data type, used if no `dtype` argument is provided when + calling the initializer. Only floating point types are supported. + + References: + [Xiao et al., 2018](http://proceedings.mlr.press/v80/xiao18a.html) + ([pdf](http://proceedings.mlr.press/v80/xiao18a/xiao18a.pdf)) """ def __call__(self, shape, dtype=None, partition_info=None): @@ -856,7 +899,7 @@ class ConvolutionOrthogonal1D(ConvolutionOrthogonal): raise ValueError("In_filters cannot be greater than out_filters.") kernel = self._orthogonal_kernel(shape[0], shape[-2], shape[-1]) - kernel *= math_ops.sqrt(math_ops.cast(self.gain, dtype=dtype)) + kernel *= math_ops.cast(self.gain, dtype=dtype) return kernel def _dict_to_tensor(self, x, k): @@ -951,15 +994,20 @@ class ConvolutionOrthogonal3D(ConvolutionOrthogonal): filters must not exceed the number of output filters. The orthogonality(==isometry) is exact when the inputs are circular padded. There are finite-width effects with non-circular padding (e.g. zero padding). - See algorithm 1 [Xiao et al., 2018] in: https://arxiv.org/abs/1806.05393 + See algorithm 1 (Xiao et al., 2018). Args: - gain: Multiplicative factor to apply to the orthogonal matrix. Default is 1. - The 2-norm of an input is multiplied by a factor of 'sqrt(gain)' after - applying this convolution. + gain: Multiplicative factor to apply to the orthogonal + matrix. Default is 1. The 2-norm of an input is multiplied by a factor of + `gain` after applying this convolution. seed: A Python integer. Used to create random seeds. See `tf.set_random_seed` for behavior. - dtype: The data type. + dtype: Default data type, used if no `dtype` argument is provided when + calling the initializer. Only floating point types are supported. + + References: + [Xiao et al., 2018](http://proceedings.mlr.press/v80/xiao18a.html) + ([pdf](http://proceedings.mlr.press/v80/xiao18a/xiao18a.pdf)) """ def __call__(self, shape, dtype=None, partition_info=None): @@ -975,7 +1023,7 @@ class ConvolutionOrthogonal3D(ConvolutionOrthogonal): raise ValueError("Kernel sizes must be equal.") kernel = self._orthogonal_kernel(shape[0], shape[-2], shape[-1]) - kernel *= math_ops.sqrt(math_ops.cast(self.gain, dtype=dtype)) + kernel *= math_ops.cast(self.gain, dtype=dtype) return kernel def _dict_to_tensor(self, x, k1, k2, k3): @@ -1105,7 +1153,8 @@ class Identity(Initializer): Args: gain: Multiplicative factor to apply to the identity matrix. - dtype: The type of the output. + dtype: Default data type, used if no `dtype` argument is provided when + calling the initializer. Only floating point types are supported. """ def __init__(self, gain=1.0, dtype=dtypes.float32): @@ -1139,18 +1188,19 @@ class GlorotUniform(VarianceScaling): where `fan_in` is the number of input units in the weight tensor and `fan_out` is the number of output units in the weight tensor. - Reference: http://jmlr.org/proceedings/papers/v9/glorot10a/glorot10a.pdf - Args: seed: A Python integer. Used to create random seeds. See `tf.set_random_seed` for behavior. - dtype: The data type. Only floating point types are supported. + dtype: Default data type, used if no `dtype` argument is provided when + calling the initializer. Only floating point types are supported. + + References: + [Glorot et al., 2010](http://proceedings.mlr.press/v9/glorot10a.html) + ([pdf](http://jmlr.org/proceedings/papers/v9/glorot10a/glorot10a.pdf)) """ - def __init__(self, - seed=None, - dtype=dtypes.float32): + def __init__(self, seed=None, dtype=dtypes.float32): super(GlorotUniform, self).__init__( scale=1.0, mode="fan_avg", @@ -1159,10 +1209,7 @@ class GlorotUniform(VarianceScaling): dtype=dtype) def get_config(self): - return { - "seed": self.seed, - "dtype": self.dtype.name - } + return {"seed": self.seed, "dtype": self.dtype.name} @tf_export( @@ -1181,18 +1228,18 @@ class GlorotNormal(VarianceScaling): where `fan_in` is the number of input units in the weight tensor and `fan_out` is the number of output units in the weight tensor. - Reference: http://jmlr.org/proceedings/papers/v9/glorot10a/glorot10a.pdf - Args: seed: A Python integer. Used to create random seeds. See - `tf.set_random_seed` - for behavior. - dtype: The data type. Only floating point types are supported. + `tf.set_random_seed` for behavior. + dtype: Default data type, used if no `dtype` argument is provided when + calling the initializer. Only floating point types are supported. + + References: + [Glorot et al., 2010](http://proceedings.mlr.press/v9/glorot10a.html) + ([pdf](http://jmlr.org/proceedings/papers/v9/glorot10a/glorot10a.pdf)) """ - def __init__(self, - seed=None, - dtype=dtypes.float32): + def __init__(self, seed=None, dtype=dtypes.float32): super(GlorotNormal, self).__init__( scale=1.0, mode="fan_avg", @@ -1201,10 +1248,7 @@ class GlorotNormal(VarianceScaling): dtype=dtype) def get_config(self): - return { - "seed": self.seed, - "dtype": self.dtype.name - } + return {"seed": self.seed, "dtype": self.dtype.name} # Aliases. @@ -1244,9 +1288,11 @@ def lecun_normal(seed=None): An initializer. References: - - [Self-Normalizing Neural Networks](https://arxiv.org/abs/1706.02515) - - [Efficient - Backprop](http://yann.lecun.com/exdb/publis/pdf/lecun-98b.pdf) + - Self-Normalizing Neural Networks, + [Klambauer et al., 2017](https://papers.nips.cc/paper/6698-self-normalizing-neural-networks) + ([pdf](https://papers.nips.cc/paper/6698-self-normalizing-neural-networks.pdf)) + - Efficient Backprop, + [Lecun et al., 1998](http://yann.lecun.com/exdb/publis/pdf/lecun-98b.pdf) """ return VarianceScaling( scale=1., mode="fan_in", distribution="truncated_normal", seed=seed) @@ -1267,8 +1313,11 @@ def lecun_uniform(seed=None): An initializer. References: - LeCun 98, Efficient Backprop, - http://yann.lecun.com/exdb/publis/pdf/lecun-98b.pdf + - Self-Normalizing Neural Networks, + [Klambauer et al., 2017](https://papers.nips.cc/paper/6698-self-normalizing-neural-networks) + ([pdf](https://papers.nips.cc/paper/6698-self-normalizing-neural-networks.pdf)) + - Efficient Backprop, + [Lecun et al., 1998](http://yann.lecun.com/exdb/publis/pdf/lecun-98b.pdf) """ return VarianceScaling( scale=1., mode="fan_in", distribution="uniform", seed=seed) @@ -1289,7 +1338,8 @@ def he_normal(seed=None): An initializer. References: - He et al., http://arxiv.org/abs/1502.01852 + [He et al., 2015](https://www.cv-foundation.org/openaccess/content_iccv_2015/html/He_Delving_Deep_into_ICCV_2015_paper.html) + ([pdf](https://www.cv-foundation.org/openaccess/content_iccv_2015/papers/He_Delving_Deep_into_ICCV_2015_paper.pdf)) """ return VarianceScaling( scale=2., mode="fan_in", distribution="truncated_normal", seed=seed) @@ -1310,7 +1360,8 @@ def he_uniform(seed=None): An initializer. References: - He et al., http://arxiv.org/abs/1502.01852 + [He et al., 2015](https://www.cv-foundation.org/openaccess/content_iccv_2015/html/He_Delving_Deep_into_ICCV_2015_paper.html) + ([pdf](https://www.cv-foundation.org/openaccess/content_iccv_2015/papers/He_Delving_Deep_into_ICCV_2015_paper.pdf)) """ return VarianceScaling( scale=2., mode="fan_in", distribution="uniform", seed=seed) diff --git a/tensorflow/python/ops/init_ops_test.py b/tensorflow/python/ops/init_ops_test.py index 5693c3caaf5..1f222480046 100644 --- a/tensorflow/python/ops/init_ops_test.py +++ b/tensorflow/python/ops/init_ops_test.py @@ -45,8 +45,8 @@ class InitializersTest(test.TestCase): output = variable.numpy() else: sess = ops.get_default_session() - sess.run(variable.initializer) - output = sess.run(variable) + self.evaluate(variable.initializer) + output = self.evaluate(variable) lim = 3e-2 if target_std is not None: self.assertGreater(lim, abs(output.std() - target_std)) diff --git a/tensorflow/python/ops/linalg/BUILD b/tensorflow/python/ops/linalg/BUILD index c7314d77749..5df2d6b8381 100644 --- a/tensorflow/python/ops/linalg/BUILD +++ b/tensorflow/python/ops/linalg/BUILD @@ -18,6 +18,7 @@ py_library( "//tensorflow/python:random_ops", "//tensorflow/python:tensor_util", "//tensorflow/python:util", + "//tensorflow/python/ops/signal", "//third_party/py/numpy", "@six_archive//:six", ], diff --git a/tensorflow/python/ops/linalg/cholesky_registrations.py b/tensorflow/python/ops/linalg/cholesky_registrations.py new file mode 100644 index 00000000000..e5284cf22ac --- /dev/null +++ b/tensorflow/python/ops/linalg/cholesky_registrations.py @@ -0,0 +1,101 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Registrations for LinearOperator.cholesky.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from tensorflow.python.ops import linalg_ops +from tensorflow.python.ops import math_ops +from tensorflow.python.ops.linalg import linear_operator +from tensorflow.python.ops.linalg import linear_operator_algebra +from tensorflow.python.ops.linalg import linear_operator_block_diag +from tensorflow.python.ops.linalg import linear_operator_diag +from tensorflow.python.ops.linalg import linear_operator_identity +from tensorflow.python.ops.linalg import linear_operator_kronecker +from tensorflow.python.ops.linalg import linear_operator_lower_triangular + + +# By default, compute the Cholesky of the dense matrix, and return a +# LowerTriangular operator. Methods below specialize this registration. +@linear_operator_algebra.RegisterCholesky(linear_operator.LinearOperator) +def _cholesky_linear_operator(linop): + return linear_operator_lower_triangular.LinearOperatorLowerTriangular( + linalg_ops.cholesky(linop.to_dense()), + is_non_singular=True, + is_self_adjoint=False, + is_square=True) + + +@linear_operator_algebra.RegisterCholesky( + linear_operator_diag.LinearOperatorDiag) +def _cholesky_diag(diag_operator): + return linear_operator_diag.LinearOperatorDiag( + math_ops.sqrt(diag_operator.diag), + is_non_singular=True, + is_self_adjoint=True, + is_positive_definite=True, + is_square=True) + + +@linear_operator_algebra.RegisterCholesky( + linear_operator_identity.LinearOperatorIdentity) +def _cholesky_identity(identity_operator): + return linear_operator_identity.LinearOperatorIdentity( + num_rows=identity_operator._num_rows, # pylint: disable=protected-access + batch_shape=identity_operator.batch_shape, + dtype=identity_operator.dtype, + is_non_singular=True, + is_self_adjoint=True, + is_positive_definite=True, + is_square=True) + + +@linear_operator_algebra.RegisterCholesky( + linear_operator_identity.LinearOperatorScaledIdentity) +def _cholesky_scaled_identity(identity_operator): + return linear_operator_identity.LinearOperatorScaledIdentity( + num_rows=identity_operator._num_rows, # pylint: disable=protected-access + multiplier=math_ops.sqrt(identity_operator.multiplier), + is_non_singular=True, + is_self_adjoint=True, + is_positive_definite=True, + is_square=True) + + +@linear_operator_algebra.RegisterCholesky( + linear_operator_block_diag.LinearOperatorBlockDiag) +def _cholesky_block_diag(block_diag_operator): + # We take the cholesky of each block on the diagonal. + return linear_operator_block_diag.LinearOperatorBlockDiag( + operators=[ + operator.cholesky() for operator in block_diag_operator.operators], + is_non_singular=True, + is_self_adjoint=False, + is_square=True) + + +@linear_operator_algebra.RegisterCholesky( + linear_operator_kronecker.LinearOperatorKronecker) +def _cholesky_kronecker(kronecker_operator): + # Cholesky decomposition of a Kronecker product is the Kronecker product + # of cholesky decompositions. + return linear_operator_kronecker.LinearOperatorKronecker( + operators=[ + operator.cholesky() for operator in kronecker_operator.operators], + is_non_singular=True, + is_self_adjoint=False, + is_square=True) diff --git a/tensorflow/python/ops/linalg/linalg.py b/tensorflow/python/ops/linalg/linalg.py index c29b5033bb1..ac4fd4ebc60 100644 --- a/tensorflow/python/ops/linalg/linalg.py +++ b/tensorflow/python/ops/linalg/linalg.py @@ -20,6 +20,9 @@ from __future__ import print_function # go/tf-wildcard-import # pylint: disable=wildcard-import,unused-import +from tensorflow.python.ops.linalg import cholesky_registrations as _cholesky_registrations +from tensorflow.python.ops.linalg import linear_operator_algebra as _linear_operator_algebra +from tensorflow.python.ops.linalg import matmul_registrations as _matmul_registrations from tensorflow.python.ops.linalg.linalg_impl import * from tensorflow.python.ops.linalg.linear_operator import * from tensorflow.python.ops.linalg.linear_operator_block_diag import * diff --git a/tensorflow/python/ops/linalg/linalg_impl.py b/tensorflow/python/ops/linalg/linalg_impl.py index 08d50ce622f..2c9476a9bd3 100644 --- a/tensorflow/python/ops/linalg/linalg_impl.py +++ b/tensorflow/python/ops/linalg/linalg_impl.py @@ -88,7 +88,7 @@ def logdet(matrix, name=None): chol = gen_linalg_ops.cholesky(matrix) return 2.0 * math_ops.reduce_sum( math_ops.log(math_ops.real(array_ops.matrix_diag_part(chol))), - reduction_indices=[-1]) + axis=[-1]) @tf_export('linalg.adjoint') diff --git a/tensorflow/python/ops/linalg/linear_operator.py b/tensorflow/python/ops/linalg/linear_operator.py index 9ef6c42b04c..8efafda3a1e 100644 --- a/tensorflow/python/ops/linalg/linear_operator.py +++ b/tensorflow/python/ops/linalg/linear_operator.py @@ -32,6 +32,7 @@ from tensorflow.python.ops import check_ops from tensorflow.python.ops import linalg_ops from tensorflow.python.ops import math_ops from tensorflow.python.ops.linalg import linalg_impl as linalg +from tensorflow.python.ops.linalg import linear_operator_algebra from tensorflow.python.ops.linalg import linear_operator_util from tensorflow.python.platform import tf_logging as logging from tensorflow.python.util.tf_export import tf_export @@ -284,7 +285,7 @@ class LinearOperator(object): `[B1,...,Bb, M, N]`, equivalent to `tf.shape(A)`. Args: - name: A name for this `Op. + name: A name for this `Op`. Returns: `int32` `Tensor` @@ -318,7 +319,7 @@ class LinearOperator(object): `[B1,...,Bb]`. Args: - name: A name for this `Op. + name: A name for this `Op`. Returns: `int32` `Tensor` @@ -340,7 +341,7 @@ class LinearOperator(object): `A.shape = [B1,...,Bb, M, N]`, then this returns `b + 2`. Args: - name: A name for this `Op. + name: A name for this `Op`. Returns: Python integer, or None if the tensor rank is undefined. @@ -356,7 +357,7 @@ class LinearOperator(object): `A.shape = [B1,...,Bb, M, N]`, then this returns `b + 2`. Args: - name: A name for this `Op. + name: A name for this `Op`. Returns: `int32` `Tensor`, determined at runtime. @@ -581,16 +582,29 @@ class LinearOperator(object): ``` Args: - x: `Tensor` with compatible shape and same `dtype` as `self`. - See class docstring for definition of compatibility. + x: `LinearOperator` or `Tensor` with compatible shape and same `dtype` as + `self`. See class docstring for definition of compatibility. adjoint: Python `bool`. If `True`, left multiply by the adjoint: `A^H x`. adjoint_arg: Python `bool`. If `True`, compute `A x^H` where `x^H` is the hermitian transpose (transposition and complex conjugation). - name: A name for this `Op. + name: A name for this `Op`. Returns: - A `Tensor` with shape `[..., M, R]` and same `dtype` as `self`. + A `LinearOperator` or `Tensor` with shape `[..., M, R]` and same `dtype` + as `self`. """ + if isinstance(x, LinearOperator): + if adjoint or adjoint_arg: + raise ValueError(".matmul not supported with adjoints.") + if (x.range_dimension is not None and + self.domain_dimension is not None and + x.range_dimension != self.domain_dimension): + raise ValueError( + "Operators are incompatible. Expected `x` to have dimension" + " {} but got {}.".format(self.domain_dimension, x.range_dimension)) + with self._name_scope(name): + return linear_operator_algebra.matmul(self, x) + with self._name_scope(name, values=[x]): x = ops.convert_to_tensor(x, name="x") self._check_input_dtype(x) @@ -630,7 +644,7 @@ class LinearOperator(object): dimensions, the last dimension defines a vector. See class docstring for definition of compatibility. adjoint: Python `bool`. If `True`, left multiply by the adjoint: `A^H x`. - name: A name for this `Op. + name: A name for this `Op`. Returns: A `Tensor` with shape `[..., M]` and same `dtype` as `self`. @@ -655,7 +669,7 @@ class LinearOperator(object): """Determinant for every batch member. Args: - name: A name for this `Op. + name: A name for this `Op`. Returns: `Tensor` with shape `self.batch_shape` and same `dtype` as `self`. @@ -676,7 +690,7 @@ class LinearOperator(object): " Requires conversion to a dense matrix and O(N^3) operations.") if self._can_use_cholesky(): diag = array_ops.matrix_diag_part(linalg_ops.cholesky(self.to_dense())) - return 2 * math_ops.reduce_sum(math_ops.log(diag), reduction_indices=[-1]) + return 2 * math_ops.reduce_sum(math_ops.log(diag), axis=[-1]) _, log_abs_det = linalg.slogdet(self.to_dense()) return log_abs_det @@ -684,7 +698,7 @@ class LinearOperator(object): """Log absolute value of determinant for every batch member. Args: - name: A name for this `Op. + name: A name for this `Op`. Returns: `Tensor` with shape `self.batch_shape` and same `dtype` as `self`. @@ -830,6 +844,31 @@ class LinearOperator(object): return self._solvevec(rhs, adjoint=adjoint) + def cholesky(self, name="cholesky"): + """Returns a Cholesky factor as a `LinearOperator`. + + Given `A` representing this `LinearOperator`, if `A` is positive definite + self-adjoint, return `L`, where `A = L L^T`, i.e. the cholesky + decomposition. + + Args: + name: A name for this `Op`. + + Returns: + `LinearOperator` which represents the lower triangular matrix + in the Cholesky decomposition. + + Raises: + ValueError: When the `LinearOperator` is not hinted to be positive + definite and self adjoint. + """ + + if not self._can_use_cholesky(): + raise ValueError("Cannot take the Cholesky decomposition: " + "Not a positive definite self adjoint matrix.") + with self._name_scope(name): + return linear_operator_algebra.cholesky(self) + def _to_dense(self): """Generic and often inefficient implementation. Override often.""" logging.warn("Using (possibly slow) default implementation of to_dense." @@ -922,6 +961,4 @@ class LinearOperator(object): return self._add_to_tensor(x) def _can_use_cholesky(self): - # TODO(langmore) Add complex types when tf.cholesky can use them. - return (not self.dtype.is_complex and self.is_self_adjoint and - self.is_positive_definite) + return self.is_self_adjoint and self.is_positive_definite diff --git a/tensorflow/python/ops/linalg/linear_operator_adjoint.py b/tensorflow/python/ops/linalg/linear_operator_adjoint.py new file mode 100644 index 00000000000..858e224b9ad --- /dev/null +++ b/tensorflow/python/ops/linalg/linear_operator_adjoint.py @@ -0,0 +1,207 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Takes the adjoint of a `LinearOperator`.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from tensorflow.python.framework import ops +from tensorflow.python.ops import math_ops +from tensorflow.python.ops.linalg import linalg_impl as linalg +from tensorflow.python.ops.linalg import linear_operator +from tensorflow.python.util.tf_export import tf_export + +__all__ = [] + + +@tf_export("linalg.LinearOperatorAdjoint") +class LinearOperatorAdjoint(linear_operator.LinearOperator): + """`LinearOperator` representing the adjoint of another operator. + + This operator represents the adjoint of another operator. + + ```python + # Create a 2 x 2 linear operator. + operator = LinearOperatorFullMatrix([[1 - i., 3.], [0., 1. + i]]) + operator_adjoint = LinearOperatorAdjoint(operator) + + operator_adjoint.to_dense() + ==> [[1. + i, 0.] + [3., 1 - i]] + + operator_adjoint.shape + ==> [2, 2] + + operator_adjoint.log_abs_determinant() + ==> - log(2) + + x = ... Shape [2, 4] Tensor + operator_adjoint.matmul(x) + ==> Shape [2, 4] Tensor, equal to operator.matmul(x, adjoint=True) + ``` + + #### Performance + + The performance of `LinearOperatorAdjoint` depends on the underlying + operators performance. + + #### Matrix property hints + + This `LinearOperator` is initialized with boolean flags of the form `is_X`, + for `X = non_singular, self_adjoint, positive_definite, square`. + These have the following meaning: + + * If `is_X == True`, callers should expect the operator to have the + property `X`. This is a promise that should be fulfilled, but is *not* a + runtime assert. For example, finite floating point precision may result + in these promises being violated. + * If `is_X == False`, callers should expect the operator to not have `X`. + * If `is_X == None` (the default), callers should have no expectation either + way. + """ + + def __init__(self, + operator, + is_non_singular=None, + is_self_adjoint=None, + is_positive_definite=None, + is_square=None, + name=None): + r"""Initialize a `LinearOperatorAdjoint`. + + `LinearOperatorAdjoint` is initialized with an operator `A`. The `solve` + and `matmul` methods effectively flip the `adjoint` argument. E.g. + + ``` + A = MyLinearOperator(...) + B = LinearOperatorAdjoint(A) + x = [....] # a vector + + assert A.matvec(x, adjoint=True) == B.matvec(x, adjoint=False) + ``` + + Args: + operator: `LinearOperator` object. + is_non_singular: Expect that this operator is non-singular. + is_self_adjoint: Expect that this operator is equal to its hermitian + transpose. + is_positive_definite: Expect that this operator is positive definite, + meaning the quadratic form `x^H A x` has positive real part for all + nonzero `x`. Note that we do not require the operator to be + self-adjoint to be positive-definite. See: + https://en.wikipedia.org/wiki/Positive-definite_matrix#Extension_for_non-symmetric_matrices + is_square: Expect that this operator acts like square [batch] matrices. + name: A name for this `LinearOperator`. Default is `operator.name + + "_adjoint"`. + + Raises: + ValueError: If `operator.is_non_singular` is False. + """ + + self._operator = operator + + # The congruency of is_non_singular and is_self_adjoint was checked in the + # base operator. + def _combined_hint(hint_str, provided_hint_value, message): + """Get combined hint in the case where operator.hint should equal hint.""" + op_hint = getattr(operator, hint_str) + if op_hint is False and provided_hint_value: + raise ValueError(message) + if op_hint and provided_hint_value is False: + raise ValueError(message) + return (op_hint or provided_hint_value) or None + + is_square = _combined_hint( + "is_square", is_square, + "An operator is square if and only if its adjoint is square.") + + is_non_singular = _combined_hint( + "is_non_singular", is_non_singular, + "An operator is non-singular if and only if its adjoint is " + "non-singular.") + + is_self_adjoint = _combined_hint( + "is_self_adjoint", is_self_adjoint, + "An operator is self-adjoint if and only if its adjoint is " + "self-adjoint.") + + is_positive_definite = _combined_hint( + "is_positive_definite", is_positive_definite, + "An operator is positive-definite if and only if its adjoint is " + "positive-definite.") + + is_square = _combined_hint( + "is_square", is_square, + "An operator is square if and only if its adjoint is square.") + + # Initialization. + if name is None: + name = operator.name + "_adjoint" + with ops.name_scope(name, values=operator.graph_parents): + super(LinearOperatorAdjoint, self).__init__( + dtype=operator.dtype, + graph_parents=operator.graph_parents, + is_non_singular=is_non_singular, + is_self_adjoint=is_self_adjoint, + is_positive_definite=is_positive_definite, + is_square=is_square, + name=name) + + @property + def operator(self): + """The operator before taking the adjoint.""" + return self._operator + + def _assert_non_singular(self): + return self.operator.assert_non_singular() + + def _assert_positive_definite(self): + return self.operator.assert_positive_definite() + + def _assert_self_adjoint(self): + return self.operator.assert_self_adjoint() + + def _shape(self): + return self.operator.shape + + def _shape_tensor(self): + return self.operator.shape_tensor() + + def _matmul(self, x, adjoint=False, adjoint_arg=False): + return self.operator.matmul( + x, adjoint=(not adjoint), adjoint_arg=adjoint_arg) + + def _determinant(self): + if self.is_self_adjoint: + return self.operator.determinant() + return math_ops.conj(self.operator.determinant()) + + def _log_abs_determinant(self): + return self.operator.log_abs_determinant() + + def _trace(self): + if self.is_self_adjoint: + return self.operator.trace() + return math_ops.conj(self.operator.trace()) + + def _solve(self, rhs, adjoint=False, adjoint_arg=False): + return self.operator.solve( + rhs, adjoint=(not adjoint), adjoint_arg=adjoint_arg) + + def _to_dense(self): + if self.is_self_adjoint: + return self.operator.to_dense() + return linalg.adjoint(self.operator.to_dense()) diff --git a/tensorflow/python/ops/linalg/linear_operator_algebra.py b/tensorflow/python/ops/linalg/linear_operator_algebra.py new file mode 100644 index 00000000000..7b99066e4c1 --- /dev/null +++ b/tensorflow/python/ops/linalg/linear_operator_algebra.py @@ -0,0 +1,191 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +"""Registration mechanisms for various n-ary operations on LinearOperators.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import itertools + +from tensorflow.python.framework import ops +from tensorflow.python.util import tf_inspect + + +_CHOLESKY_DECOMPS = {} +_MATMUL = {} + + +def _registered_function(type_list, registry): + """Given a list of classes, finds the most specific function registered.""" + enumerated_hierarchies = [enumerate(tf_inspect.getmro(t)) for t in type_list] + # Get all possible combinations of hierarchies. + cls_combinations = list(itertools.product(*enumerated_hierarchies)) + + def hierarchy_distance(cls_combination): + candidate_distance = sum(c[0] for c in cls_combination) + if tuple(c[1] for c in cls_combination) in registry: + return candidate_distance + return 10000 + + registered_combination = min(cls_combinations, key=hierarchy_distance) + return registry.get(tuple(r[1] for r in registered_combination), None) + + +def _registered_cholesky(type_a): + """Get the Cholesky function registered for class a.""" + return _registered_function([type_a], _CHOLESKY_DECOMPS) + + +def _registered_matmul(type_a, type_b): + """Get the Matmul function registered for classes a and b.""" + return _registered_function([type_a, type_b], _MATMUL) + + +def cholesky(lin_op_a, name=None): + """Get the Cholesky factor associated to lin_op_a. + + Args: + lin_op_a: The LinearOperator to decompose. + name: Name to use for this operation. + + Returns: + A LinearOperator that represents the lower Cholesky factor of `lin_op_a`. + + Raises: + NotImplementedError: If no Cholesky method is defined for the LinearOperator + type of `lin_op_a`. + """ + cholesky_fn = _registered_cholesky(type(lin_op_a)) + if cholesky_fn is None: + raise ValueError("No cholesky decomposition registered for {}".format( + type(lin_op_a))) + + with ops.name_scope(name, "Cholesky"): + return cholesky_fn(lin_op_a) + + +def matmul(lin_op_a, lin_op_b, name=None): + """Compute lin_op_a.matmul(lin_op_b). + + Args: + lin_op_a: The LinearOperator on the left. + lin_op_b: The LinearOperator on the right. + name: Name to use for this operation. + + Returns: + A LinearOperator that represents the matmul between `lin_op_a` and + `lin_op_b`. + + Raises: + NotImplementedError: If no matmul method is defined between types of + `lin_op_a` and `lin_op_b`. + """ + matmul_fn = _registered_matmul(type(lin_op_a), type(lin_op_b)) + if matmul_fn is None: + raise ValueError("No matmul registered for {}.matmul({})".format( + type(lin_op_a), type(lin_op_b))) + + with ops.name_scope(name, "Matmul"): + return matmul_fn(lin_op_a, lin_op_b) + + +class RegisterCholesky(object): + """Decorator to register a Cholesky implementation function. + + Usage: + + @linear_operator_algebra.RegisterCholesky(lin_op.LinearOperatorIdentity) + def _cholesky_identity(lin_op_a): + # Return the identity matrix. + """ + + def __init__(self, lin_op_cls_a): + """Initialize the LinearOperator registrar. + + Args: + lin_op_cls_a: the class of the LinearOperator to decompose. + """ + self._key = (lin_op_cls_a,) + + def __call__(self, cholesky_fn): + """Perform the Cholesky registration. + + Args: + cholesky_fn: The function to use for the Cholesky. + + Returns: + cholesky_fn + + Raises: + TypeError: if cholesky_fn is not a callable. + ValueError: if a Cholesky function has already been registered for + the given argument classes. + """ + if not callable(cholesky_fn): + raise TypeError( + "cholesky_fn must be callable, received: {}".format(cholesky_fn)) + if self._key in _CHOLESKY_DECOMPS: + raise ValueError("Cholesky({}) has already been registered to: {}".format( + self._key[0].__name__, _CHOLESKY_DECOMPS[self._key])) + _CHOLESKY_DECOMPS[self._key] = cholesky_fn + return cholesky_fn + + +class RegisterMatmul(object): + """Decorator to register a Matmul implementation function. + + Usage: + + @linear_operator_algebra.RegisterMatmul( + lin_op.LinearOperatorIdentity, + lin_op.LinearOperatorIdentity) + def _matmul_identity(a, b): + # Return the identity matrix. + """ + + def __init__(self, lin_op_cls_a, lin_op_cls_b): + """Initialize the LinearOperator registrar. + + Args: + lin_op_cls_a: the class of the LinearOperator to multiply. + lin_op_cls_b: the class of the second LinearOperator to multiply. + """ + self._key = (lin_op_cls_a, lin_op_cls_b) + + def __call__(self, matmul_fn): + """Perform the Matmul registration. + + Args: + matmul_fn: The function to use for the Matmul. + + Returns: + matmul_fn + + Raises: + TypeError: if matmul_fn is not a callable. + ValueError: if a Matmul function has already been registered for + the given argument classes. + """ + if not callable(matmul_fn): + raise TypeError( + "matmul_fn must be callable, received: {}".format(matmul_fn)) + if self._key in _MATMUL: + raise ValueError("Matmul({}, {}) has already been registered.".format( + self._key[0].__name__, + self._key[1].__name__)) + _MATMUL[self._key] = matmul_fn + return matmul_fn diff --git a/tensorflow/python/ops/linalg/linear_operator_block_diag.py b/tensorflow/python/ops/linalg/linear_operator_block_diag.py index 438c3496bdf..b0b418c9970 100644 --- a/tensorflow/python/ops/linalg/linear_operator_block_diag.py +++ b/tensorflow/python/ops/linalg/linear_operator_block_diag.py @@ -29,9 +29,7 @@ from tensorflow.python.ops.linalg import linear_operator from tensorflow.python.ops.linalg import linear_operator_util from tensorflow.python.util.tf_export import tf_export -__all__ = [ - "LinearOperatorBlockDiag", -] +__all__ = ["LinearOperatorBlockDiag"] @tf_export("linalg.LinearOperatorBlockDiag") diff --git a/tensorflow/python/ops/linalg/linear_operator_circulant.py b/tensorflow/python/ops/linalg/linear_operator_circulant.py index 021ef473836..b74baa5dfdb 100644 --- a/tensorflow/python/ops/linalg/linear_operator_circulant.py +++ b/tensorflow/python/ops/linalg/linear_operator_circulant.py @@ -30,6 +30,7 @@ from tensorflow.python.ops.distributions import util as distribution_util from tensorflow.python.ops.linalg import linalg_impl as linalg from tensorflow.python.ops.linalg import linear_operator from tensorflow.python.ops.linalg import linear_operator_util +from tensorflow.python.ops.signal import fft_ops from tensorflow.python.util.tf_export import tf_export __all__ = [ @@ -39,8 +40,8 @@ __all__ = [ ] # Different FFT Ops will be used for different block depths. -_FFT_OP = {1: math_ops.fft, 2: math_ops.fft2d, 3: math_ops.fft3d} -_IFFT_OP = {1: math_ops.ifft, 2: math_ops.ifft2d, 3: math_ops.ifft3d} +_FFT_OP = {1: fft_ops.fft, 2: fft_ops.fft2d, 3: fft_ops.fft3d} +_IFFT_OP = {1: fft_ops.ifft, 2: fft_ops.ifft2d, 3: fft_ops.ifft3d} # This is the only dtype allowed with fft ops. # TODO(langmore) Add other types once available. @@ -417,15 +418,13 @@ class _BaseLinearOperatorCirculant(linear_operator.LinearOperator): return math_ops.cast(y, self.dtype) def _determinant(self): - reduction_indices = [-(i + 1) for i in range(self.block_depth)] - det = math_ops.reduce_prod( - self.spectrum, reduction_indices=reduction_indices) + axis = [-(i + 1) for i in range(self.block_depth)] + det = math_ops.reduce_prod(self.spectrum, axis=axis) return math_ops.cast(det, self.dtype) def _log_abs_determinant(self): - reduction_indices = [-(i + 1) for i in range(self.block_depth)] - lad = math_ops.reduce_sum( - math_ops.log(self._abs_spectrum), reduction_indices=reduction_indices) + axis = [-(i + 1) for i in range(self.block_depth)] + lad = math_ops.reduce_sum(math_ops.log(self._abs_spectrum), axis=axis) return math_ops.cast(lad, self.dtype) def _solve(self, rhs, adjoint=False, adjoint_arg=False): diff --git a/tensorflow/python/ops/linalg/linear_operator_composition.py b/tensorflow/python/ops/linalg/linear_operator_composition.py index 0292bc51dcf..f499b306612 100644 --- a/tensorflow/python/ops/linalg/linear_operator_composition.py +++ b/tensorflow/python/ops/linalg/linear_operator_composition.py @@ -275,6 +275,3 @@ class LinearOperatorComposition(linear_operator.LinearOperator): for operator in solve_order_list[1:]: solution = operator.solve(solution, adjoint=adjoint) return solution - - def _add_to_tensor(self, x): - return self.to_dense() + x diff --git a/tensorflow/python/ops/linalg/linear_operator_diag.py b/tensorflow/python/ops/linalg/linear_operator_diag.py index ed53decc00d..be893c705c9 100644 --- a/tensorflow/python/ops/linalg/linear_operator_diag.py +++ b/tensorflow/python/ops/linalg/linear_operator_diag.py @@ -228,11 +228,11 @@ class LinearOperatorDiag(linear_operator.LinearOperator): return diag_mat * x def _determinant(self): - return math_ops.reduce_prod(self._diag, reduction_indices=[-1]) + return math_ops.reduce_prod(self._diag, axis=[-1]) def _log_abs_determinant(self): log_det = math_ops.reduce_sum( - math_ops.log(math_ops.abs(self._diag)), reduction_indices=[-1]) + math_ops.log(math_ops.abs(self._diag)), axis=[-1]) if self.dtype.is_complex: log_det = math_ops.cast(log_det, dtype=self.dtype) return log_det diff --git a/tensorflow/python/ops/linalg/linear_operator_inversion.py b/tensorflow/python/ops/linalg/linear_operator_inversion.py new file mode 100644 index 00000000000..7aa4b40e16b --- /dev/null +++ b/tensorflow/python/ops/linalg/linear_operator_inversion.py @@ -0,0 +1,207 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Inverts a non-singular `LinearOperator`.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from tensorflow.python.framework import ops +from tensorflow.python.ops.linalg import linear_operator +from tensorflow.python.util.tf_export import tf_export + +__all__ = [] + + +@tf_export("linalg.LinearOperatorInversion") +class LinearOperatorInversion(linear_operator.LinearOperator): + """`LinearOperator` representing the inverse of another operator. + + This operator represents the inverse of another operator. + + ```python + # Create a 2 x 2 linear operator. + operator = LinearOperatorFullMatrix([[1., 0.], [0., 2.]]) + operator_inv = LinearOperatorInversion(operator) + + operator_inv.to_dense() + ==> [[1., 0.] + [0., 0.5]] + + operator_inv.shape + ==> [2, 2] + + operator_inv.log_abs_determinant() + ==> - log(2) + + x = ... Shape [2, 4] Tensor + operator_inv.matmul(x) + ==> Shape [2, 4] Tensor, equal to operator.solve(x) + ``` + + #### Performance + + The performance of `LinearOperatorInversion` depends on the underlying + operators performance: `solve` and `matmul` are swapped, and determinant is + inverted. + + #### Matrix property hints + + This `LinearOperator` is initialized with boolean flags of the form `is_X`, + for `X = non_singular, self_adjoint, positive_definite, square`. + These have the following meaning: + + * If `is_X == True`, callers should expect the operator to have the + property `X`. This is a promise that should be fulfilled, but is *not* a + runtime assert. For example, finite floating point precision may result + in these promises being violated. + * If `is_X == False`, callers should expect the operator to not have `X`. + * If `is_X == None` (the default), callers should have no expectation either + way. + """ + + def __init__(self, + operator, + is_non_singular=None, + is_self_adjoint=None, + is_positive_definite=None, + is_square=None, + name=None): + r"""Initialize a `LinearOperatorInversion`. + + `LinearOperatorInversion` is initialized with an operator `A`. The `solve` + and `matmul` methods are effectively swapped. E.g. + + ``` + A = MyLinearOperator(...) + B = LinearOperatorInversion(A) + x = [....] # a vector + + assert A.matvec(x) == B.solvevec(x) + ``` + + Args: + operator: `LinearOperator` object. If `operator.is_non_singular == False`, + an exception is raised. We do allow `operator.is_non_singular == None`, + in which case this operator will have `is_non_singular == None`. + Similarly for `is_self_adjoint` and `is_positive_definite`. + is_non_singular: Expect that this operator is non-singular. + is_self_adjoint: Expect that this operator is equal to its hermitian + transpose. + is_positive_definite: Expect that this operator is positive definite, + meaning the quadratic form `x^H A x` has positive real part for all + nonzero `x`. Note that we do not require the operator to be + self-adjoint to be positive-definite. See: + https://en.wikipedia.org/wiki/Positive-definite_matrix#Extension_for_non-symmetric_matrices + is_square: Expect that this operator acts like square [batch] matrices. + name: A name for this `LinearOperator`. Default is `operator.name + + "_inv"`. + + Raises: + ValueError: If `operator.is_non_singular` is False. + """ + + self._operator = operator + + # Auto-set and check hints. + if operator.is_non_singular is False or is_non_singular is False: + raise ValueError( + "operator and supplied hints must have `is_non_singular` equal to " + "`True` or `None`. Found %s, %s" % (operator.is_non_singular, + is_non_singular)) + if operator.is_square is False or is_square is False: + raise ValueError( + "operator and supplied hints must have `is_square` equal to " + "`True` or `None`. Found %s, %s" % (operator.is_square, is_square)) + + # The congruency of is_non_singular and is_self_adjoint was checked in the + # base operator. Other hints are, in this special case of inversion, ones + # that must be the same for base/derived operator. + def _combined_hint(hint_str, provided_hint_value, message): + """Get combined hint in the case where operator.hint should equal hint.""" + op_hint = getattr(operator, hint_str) + if op_hint is False and provided_hint_value: + raise ValueError(message) + if op_hint and provided_hint_value is False: + raise ValueError(message) + return (op_hint or provided_hint_value) or None + + is_square = _combined_hint( + "is_square", is_square, + "An operator is square if and only if its inverse is square.") + + is_non_singular = _combined_hint( + "is_non_singular", is_non_singular, + "An operator is non-singular if and only if its inverse is " + "non-singular.") + + is_self_adjoint = _combined_hint( + "is_self_adjoint", is_self_adjoint, + "An operator is self-adjoint if and only if its inverse is " + "self-adjoint.") + + is_positive_definite = _combined_hint( + "is_positive_definite", is_positive_definite, + "An operator is positive-definite if and only if its inverse is " + "positive-definite.") + + is_square = _combined_hint( + "is_square", is_square, + "An operator is square if and only if its inverse is square.") + + # Initialization. + if name is None: + name = operator.name + "_inv" + with ops.name_scope(name, values=operator.graph_parents): + super(LinearOperatorInversion, self).__init__( + dtype=operator.dtype, + graph_parents=operator.graph_parents, + is_non_singular=is_non_singular, + is_self_adjoint=is_self_adjoint, + is_positive_definite=is_positive_definite, + is_square=is_square, + name=name) + + @property + def operator(self): + """The operator before inversion.""" + return self._operator + + def _assert_non_singular(self): + return self.operator.assert_non_singular() + + def _assert_positive_definite(self): + return self.operator.assert_positive_definite() + + def _assert_self_adjoint(self): + return self.operator.assert_self_adjoint() + + def _shape(self): + return self.operator.shape + + def _shape_tensor(self): + return self.operator.shape_tensor() + + def _matmul(self, x, adjoint=False, adjoint_arg=False): + return self.operator.solve(x, adjoint=adjoint, adjoint_arg=adjoint_arg) + + def _determinant(self): + return 1. / self.operator.determinant() + + def _log_abs_determinant(self): + return -1. * self.operator.log_abs_determinant() + + def _solve(self, rhs, adjoint=False, adjoint_arg=False): + return self.operator.matmul(rhs, adjoint=adjoint, adjoint_arg=adjoint_arg) diff --git a/tensorflow/python/ops/linalg/linear_operator_kronecker.py b/tensorflow/python/ops/linalg/linear_operator_kronecker.py index 1fd5073c178..f7e785caa5d 100644 --- a/tensorflow/python/ops/linalg/linear_operator_kronecker.py +++ b/tensorflow/python/ops/linalg/linear_operator_kronecker.py @@ -30,9 +30,7 @@ from tensorflow.python.ops.linalg import linalg_impl as linalg from tensorflow.python.ops.linalg import linear_operator from tensorflow.python.util.tf_export import tf_export -__all__ = [ - "LinearOperatorKronecker", -] +__all__ = ["LinearOperatorKronecker"] def _vec(x): diff --git a/tensorflow/python/ops/linalg/linear_operator_low_rank_update.py b/tensorflow/python/ops/linalg/linear_operator_low_rank_update.py index c4288ff8f87..aa0500aff06 100644 --- a/tensorflow/python/ops/linalg/linear_operator_low_rank_update.py +++ b/tensorflow/python/ops/linalg/linear_operator_low_rank_update.py @@ -391,7 +391,7 @@ class LinearOperatorLowRankUpdate(linear_operator.LinearOperator): if self._use_cholesky: chol_cap_diag = array_ops.matrix_diag_part(self._chol_capacitance) log_abs_det_c = 2 * math_ops.reduce_sum( - math_ops.log(chol_cap_diag), reduction_indices=[-1]) + math_ops.log(chol_cap_diag), axis=[-1]) else: det_c = linalg_ops.matrix_determinant(self._capacitance) log_abs_det_c = math_ops.log(math_ops.abs(det_c)) diff --git a/tensorflow/python/ops/linalg/linear_operator_lower_triangular.py b/tensorflow/python/ops/linalg/linear_operator_lower_triangular.py index ca6d3f54051..d33fe17e042 100644 --- a/tensorflow/python/ops/linalg/linear_operator_lower_triangular.py +++ b/tensorflow/python/ops/linalg/linear_operator_lower_triangular.py @@ -195,11 +195,11 @@ class LinearOperatorLowerTriangular(linear_operator.LinearOperator): self._tril, x, adjoint_a=adjoint, adjoint_b=adjoint_arg) def _determinant(self): - return math_ops.reduce_prod(self._diag, reduction_indices=[-1]) + return math_ops.reduce_prod(self._diag, axis=[-1]) def _log_abs_determinant(self): return math_ops.reduce_sum( - math_ops.log(math_ops.abs(self._diag)), reduction_indices=[-1]) + math_ops.log(math_ops.abs(self._diag)), axis=[-1]) def _solve(self, rhs, adjoint=False, adjoint_arg=False): rhs = linalg.adjoint(rhs) if adjoint_arg else rhs diff --git a/tensorflow/python/ops/linalg/linear_operator_test_util.py b/tensorflow/python/ops/linalg/linear_operator_test_util.py index 76d659f1097..e50f572b5f4 100644 --- a/tensorflow/python/ops/linalg/linear_operator_test_util.py +++ b/tensorflow/python/ops/linalg/linear_operator_test_util.py @@ -102,7 +102,9 @@ class LinearOperatorDerivedClassTest(test.TestCase): raise NotImplementedError("operator_build_infos has not been implemented.") @abc.abstractmethod - def _operator_and_matrix(self, build_info, dtype, use_placeholder): + def _operator_and_matrix( + self, build_info, dtype, use_placeholder, + ensure_self_adjoint_and_pd=False): """Build a batch matrix and an Operator that should have similar behavior. Every operator acts like a (batch) matrix. This method returns both @@ -114,6 +116,11 @@ class LinearOperatorDerivedClassTest(test.TestCase): dtype: Numpy dtype. Data type of returned array/operator. use_placeholder: Python bool. If True, initialize the operator with a placeholder of undefined shape and correct dtype. + ensure_self_adjoint_and_pd: If `True`, + construct this operator to be Hermitian Positive Definite, as well + as ensuring the hints `is_positive_definite` and `is_self_adjoint` + are set. + This is useful for testing methods such as `cholesky`. Returns: operator: `LinearOperator` subclass instance. @@ -271,6 +278,21 @@ class LinearOperatorDerivedClassTest(test.TestCase): self._skip_if_tests_to_skip_contains("matmul_with_broadcast") self._test_matmul(with_batch=False) + def test_cholesky(self): + self._skip_if_tests_to_skip_contains("cholesky") + for use_placeholder in self._use_placeholder_options: + for build_info in self._operator_build_infos: + for dtype in self._dtypes_to_test: + with self.test_session(graph=ops.Graph()) as sess: + sess.graph.seed = random_seed.DEFAULT_GRAPH_SEED + operator, mat = self._operator_and_matrix( + build_info, dtype, use_placeholder=use_placeholder, + ensure_self_adjoint_and_pd=True) + op_chol = operator.cholesky().to_dense() + mat_chol = linalg_ops.cholesky(mat) + op_chol_v, mat_chol_v = sess.run([op_chol, mat_chol]) + self.assertAC(mat_chol_v, op_chol_v) + def _test_solve(self, with_batch): for use_placeholder in self._use_placeholder_options: for build_info in self._operator_build_infos: @@ -441,7 +463,7 @@ class NonSquareLinearOperatorDerivedClassTest(LinearOperatorDerivedClassTest): @property def _tests_to_skip(self): """List of test names to skip.""" - return ["solve", "solve_with_broadcast", "det", "log_abs_det"] + return ["cholesky", "solve", "solve_with_broadcast", "det", "log_abs_det"] @property def _operator_build_infos(self): diff --git a/tensorflow/python/ops/linalg/matmul_registrations.py b/tensorflow/python/ops/linalg/matmul_registrations.py new file mode 100644 index 00000000000..e0ac988ba27 --- /dev/null +++ b/tensorflow/python/ops/linalg/matmul_registrations.py @@ -0,0 +1,252 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Registrations for LinearOperator.matmul.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from tensorflow.python.ops.linalg import linear_operator +from tensorflow.python.ops.linalg import linear_operator_algebra +from tensorflow.python.ops.linalg import linear_operator_circulant +from tensorflow.python.ops.linalg import linear_operator_composition +from tensorflow.python.ops.linalg import linear_operator_diag +from tensorflow.python.ops.linalg import linear_operator_identity +from tensorflow.python.ops.linalg import linear_operator_lower_triangular +from tensorflow.python.ops.linalg import linear_operator_zeros + + +def _combined_self_adjoint_hint(operator_a, operator_b): + """Get combined hint for self-adjoint-ness.""" + # Note: only use this method in the commuting case. + # The property is preserved under composition when the operators commute. + if operator_a.is_self_adjoint and operator_b.is_self_adjoint: + return True + + # The property is not preserved when an operator with the property is composed + # with an operator without the property. + if ((operator_a.is_self_adjoint is True and + operator_b.is_self_adjoint is False) or + (operator_a.is_self_adjoint is False and + operator_b.is_self_adjoint is True)): + return False + + # The property is not known when operators are not known to have the property + # or both operators don't have the property (the property for the complement + # class is not closed under composition). + return None + + +def _is_square(operator_a, operator_b): + """Return a hint to whether the composition is square.""" + if operator_a.is_square and operator_b.is_square: + return True + if operator_a.is_square is False and operator_b.is_square is False: + # Let A have shape [B, M, N], B have shape [B, N, L]. + m = operator_a.range_dimension + l = operator_b.domain_dimension + if m is not None and l is not None: + return m == l + + return None + + +def _combined_positive_definite_hint(operator_a, operator_b): + """Get combined PD hint for compositions.""" + # Note: Positive definiteness is only guaranteed to be preserved + # when the operators commute and are symmetric. Only use this method in + # commuting cases. + + if (operator_a.is_positive_definite is True and + operator_a.is_self_adjoint is True and + operator_b.is_positive_definite is True and + operator_b.is_self_adjoint is True): + return True + + return None + + +def _combined_non_singular_hint(operator_a, operator_b): + """Get combined hint for when .""" + # If either operator is not-invertible the composition isn't. + if (operator_a.is_non_singular is False or + operator_b.is_non_singular is False): + return False + + return operator_a.is_non_singular and operator_b.is_non_singular + + +# By default, use a LinearOperatorComposition to delay the computation. +@linear_operator_algebra.RegisterMatmul( + linear_operator.LinearOperator, linear_operator.LinearOperator) +def _matmul_linear_operator(linop_a, linop_b): + """Generic matmul of two `LinearOperator`s.""" + is_square = _is_square(linop_a, linop_b) + is_non_singular = None + is_self_adjoint = None + is_positive_definite = None + + if is_square: + is_non_singular = _combined_non_singular_hint(linop_a, linop_b) + is_self_adjoint = _combined_self_adjoint_hint(linop_a, linop_b) + elif is_square is False: + is_non_singular = False + is_self_adjoint = False + is_positive_definite = False + + return linear_operator_composition.LinearOperatorComposition( + operators=[linop_a, linop_b], + is_non_singular=is_non_singular, + is_self_adjoint=is_self_adjoint, + is_positive_definite=is_positive_definite, + is_square=is_square, + ) + +# Identity + + +@linear_operator_algebra.RegisterMatmul( + linear_operator_identity.LinearOperatorIdentity, + linear_operator.LinearOperator) +def _matmul_linear_operator_identity_left(identity, linop): + del identity + return linop + + +@linear_operator_algebra.RegisterMatmul( + linear_operator.LinearOperator, + linear_operator_identity.LinearOperatorIdentity) +def _matmul_linear_operator_identity_right(linop, identity): + del identity + return linop + + +# Zeros + + +@linear_operator_algebra.RegisterMatmul( + linear_operator.LinearOperator, + linear_operator_zeros.LinearOperatorZeros) +def _matmul_linear_operator_zeros_right(linop, zeros): + if not zeros.is_square or not linop.is_square: + raise ValueError("Matmul with non-square `LinearOperator`s or non-square " + "`LinearOperatorZeros` not supported at this time.") + return zeros + + +@linear_operator_algebra.RegisterMatmul( + linear_operator_zeros.LinearOperatorZeros, + linear_operator.LinearOperator) +def _matmul_linear_operator_zeros_left(zeros, linop): + if not zeros.is_square or not linop.is_square: + raise ValueError("Matmul with non-square `LinearOperator`s or non-square " + "`LinearOperatorZeros` not supported at this time.") + return zeros + + +# Diag. + + +@linear_operator_algebra.RegisterMatmul( + linear_operator_diag.LinearOperatorDiag, + linear_operator_diag.LinearOperatorDiag) +def _matmul_linear_operator_diag(linop_a, linop_b): + return linear_operator_diag.LinearOperatorDiag( + diag=linop_a.diag * linop_b.diag, + is_non_singular=_combined_non_singular_hint(linop_a, linop_b), + is_self_adjoint=_combined_self_adjoint_hint( + linop_a, linop_b), + is_positive_definite=_combined_positive_definite_hint( + linop_a, linop_b), + is_square=True) + + +@linear_operator_algebra.RegisterMatmul( + linear_operator_diag.LinearOperatorDiag, + linear_operator_identity.LinearOperatorScaledIdentity) +def _matmul_linear_operator_diag_scaled_identity_right( + linop_diag, linop_scaled_identity): + return linear_operator_diag.LinearOperatorDiag( + diag=linop_diag.diag * linop_scaled_identity.multiplier, + is_non_singular=_combined_non_singular_hint( + linop_diag, linop_scaled_identity), + is_self_adjoint=_combined_self_adjoint_hint( + linop_diag, linop_scaled_identity), + is_positive_definite=_combined_positive_definite_hint( + linop_diag, linop_scaled_identity), + is_square=True) + + +@linear_operator_algebra.RegisterMatmul( + linear_operator_identity.LinearOperatorScaledIdentity, + linear_operator_diag.LinearOperatorDiag) +def _matmul_linear_operator_diag_scaled_identity_left( + linop_scaled_identity, linop_diag): + return linear_operator_diag.LinearOperatorDiag( + diag=linop_diag.diag * linop_scaled_identity.multiplier, + is_non_singular=_combined_non_singular_hint( + linop_diag, linop_scaled_identity), + is_self_adjoint=_combined_self_adjoint_hint( + linop_diag, linop_scaled_identity), + is_positive_definite=_combined_positive_definite_hint( + linop_diag, linop_scaled_identity), + is_square=True) + + +@linear_operator_algebra.RegisterMatmul( + linear_operator_diag.LinearOperatorDiag, + linear_operator_lower_triangular.LinearOperatorLowerTriangular) +def _matmul_linear_operator_diag_tril(linop_diag, linop_triangular): + return linear_operator_lower_triangular.LinearOperatorLowerTriangular( + tril=linop_diag.diag[..., None] * linop_triangular.to_dense(), + is_non_singular=_combined_non_singular_hint( + linop_diag, linop_triangular), + # This is safe to do since the Triangular matrix is only self-adjoint + # when it is a diagonal matrix, and hence commutes. + is_self_adjoint=_combined_self_adjoint_hint( + linop_diag, linop_triangular), + is_positive_definite=None, + is_square=True) + + +@linear_operator_algebra.RegisterMatmul( + linear_operator_lower_triangular.LinearOperatorLowerTriangular, + linear_operator_diag.LinearOperatorDiag) +def _matmul_linear_operator_tril_diag(linop_triangular, linop_diag): + return linear_operator_lower_triangular.LinearOperatorLowerTriangular( + tril=linop_triangular.to_dense() * linop_diag.diag, + is_non_singular=_combined_non_singular_hint( + linop_diag, linop_triangular), + # This is safe to do since the Triangular matrix is only self-adjoint + # when it is a diagonal matrix, and hence commutes. + is_self_adjoint=_combined_self_adjoint_hint( + linop_diag, linop_triangular), + is_positive_definite=None, + is_square=True) + +# Circulant. + + +@linear_operator_algebra.RegisterMatmul( + linear_operator_circulant.LinearOperatorCirculant, + linear_operator_circulant.LinearOperatorCirculant) +def _matmul_linear_operator_circulant_circulant(linop_a, linop_b): + return linear_operator_circulant.LinearOperatorCirculant( + spectrum=linop_a.spectrum * linop_b.spectrum, + is_non_singular=_combined_non_singular_hint(linop_a, linop_b), + is_self_adjoint=_combined_self_adjoint_hint(linop_a, linop_b), + is_positive_definite=_combined_positive_definite_hint( + linop_a, linop_b), + is_square=True) diff --git a/tensorflow/python/ops/linalg_ops.py b/tensorflow/python/ops/linalg_ops.py index bbccc7e0369..1a9e7112b45 100644 --- a/tensorflow/python/ops/linalg_ops.py +++ b/tensorflow/python/ops/linalg_ops.py @@ -423,7 +423,78 @@ def svd(tensor, full_matrices=False, compute_uv=True, name=None): # pylint: disable=redefined-builtin -@tf_export('norm', 'linalg.norm') +@tf_export('norm', 'linalg.norm', v1=[]) +def norm_v2(tensor, + ord='euclidean', + axis=None, + keepdims=None, + name=None): + r"""Computes the norm of vectors, matrices, and tensors. + + This function can compute several different vector norms (the 1-norm, the + Euclidean or 2-norm, the inf-norm, and in general the p-norm for p > 0) and + matrix norms (Frobenius, 1-norm, 2-norm and inf-norm). + + Args: + tensor: `Tensor` of types `float32`, `float64`, `complex64`, `complex128` + ord: Order of the norm. Supported values are 'fro', 'euclidean', + `1`, `2`, `np.inf` and any positive real number yielding the corresponding + p-norm. Default is 'euclidean' which is equivalent to Frobenius norm if + `tensor` is a matrix and equivalent to 2-norm for vectors. + Some restrictions apply: + a) The Frobenius norm `fro` is not defined for vectors, + b) If axis is a 2-tuple (matrix norm), only 'euclidean', 'fro', `1`, + `2`, `np.inf` are supported. + See the description of `axis` on how to compute norms for a batch of + vectors or matrices stored in a tensor. + axis: If `axis` is `None` (the default), the input is considered a vector + and a single vector norm is computed over the entire set of values in the + tensor, i.e. `norm(tensor, ord=ord)` is equivalent to + `norm(reshape(tensor, [-1]), ord=ord)`. + If `axis` is a Python integer, the input is considered a batch of vectors, + and `axis` determines the axis in `tensor` over which to compute vector + norms. + If `axis` is a 2-tuple of Python integers it is considered a batch of + matrices and `axis` determines the axes in `tensor` over which to compute + a matrix norm. + Negative indices are supported. Example: If you are passing a tensor that + can be either a matrix or a batch of matrices at runtime, pass + `axis=[-2,-1]` instead of `axis=None` to make sure that matrix norms are + computed. + keepdims: If True, the axis indicated in `axis` are kept with size 1. + Otherwise, the dimensions in `axis` are removed from the output shape. + name: The name of the op. + + Returns: + output: A `Tensor` of the same type as tensor, containing the vector or + matrix norms. If `keepdims` is True then the rank of output is equal to + the rank of `tensor`. Otherwise, if `axis` is none the output is a scalar, + if `axis` is an integer, the rank of `output` is one less than the rank + of `tensor`, if `axis` is a 2-tuple the rank of `output` is two less + than the rank of `tensor`. + + Raises: + ValueError: If `ord` or `axis` is invalid. + + @compatibility(numpy) + Mostly equivalent to numpy.linalg.norm. + Not supported: ord <= 0, 2-norm for matrices, nuclear norm. + Other differences: + a) If axis is `None`, treats the flattened `tensor` as a vector + regardless of rank. + b) Explicitly supports 'euclidean' norm as the default, including for + higher order tensors. + @end_compatibility + """ + return norm(tensor=tensor, + ord=ord, + axis=axis, + keepdims=keepdims, + name=name) + + +# pylint: disable=redefined-builtin +@tf_export(v1=['norm', 'linalg.norm']) @deprecation.deprecated_args( None, 'keep_dims is deprecated, use keepdims instead', 'keep_dims') def norm(tensor, diff --git a/tensorflow/python/ops/list_ops.py b/tensorflow/python/ops/list_ops.py index b4a1fc6af61..dbaae886d43 100644 --- a/tensorflow/python/ops/list_ops.py +++ b/tensorflow/python/ops/list_ops.py @@ -21,6 +21,7 @@ from __future__ import print_function from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops +from tensorflow.python.framework import tensor_shape from tensorflow.python.ops import array_ops from tensorflow.python.ops import gen_list_ops # go/tf-wildcard-import @@ -29,7 +30,9 @@ from tensorflow.python.ops.gen_list_ops import * # pylint: enable=wildcard-import -ops.NotDifferentiable("TensorListConcat") +ops.NotDifferentiable("TensorListConcatLists") +ops.NotDifferentiable("TensorListElementShape") +ops.NotDifferentiable("TensorListLength") ops.NotDifferentiable("TensorListPushBackBatch") @@ -41,12 +44,42 @@ def empty_tensor_list(element_shape, max_num_elements = -1 return gen_list_ops.empty_tensor_list( - element_shape=element_shape, + element_shape=_build_element_shape(element_shape), element_dtype=element_dtype, max_num_elements=max_num_elements, name=name) +def tensor_list_reserve(element_shape, num_elements, element_dtype, name=None): + return gen_list_ops.tensor_list_reserve( + element_shape=_build_element_shape(element_shape), + num_elements=num_elements, + element_dtype=element_dtype, + name=name) + + +def tensor_list_from_tensor(tensor, element_shape, name=None): + return gen_list_ops.tensor_list_from_tensor( + tensor=tensor, + element_shape=_build_element_shape(element_shape), + name=name) + + +def tensor_list_concat(input_handle, element_dtype, name=None): + # Ignore the lengths output of TensorListConcat. It is only used during + # gradient computation. + return gen_list_ops.tensor_list_concat( + input_handle=input_handle, element_dtype=element_dtype, name=name)[0] + + +def tensor_list_split(tensor, element_shape, lengths, name=None): + return gen_list_ops.tensor_list_split( + tensor=tensor, + element_shape=_build_element_shape(element_shape), + lengths=lengths, + name=name) + + @ops.RegisterGradient("TensorListPushBack") def _PushBackGrad(op, dresult): return gen_list_ops.tensor_list_pop_back( @@ -65,14 +98,32 @@ def _PopBackGrad(op, dlist, delement): @ops.RegisterGradient("TensorListStack") def _TensorListStackGrad(unused_op, dtensor): - return gen_list_ops.tensor_list_from_tensor(dtensor, - element_shape=dtensor.shape[1:]) + return tensor_list_from_tensor(dtensor, element_shape=dtensor.shape[1:]) + + +@ops.RegisterGradient("TensorListConcat") +def _TensorListConcatGrad(op, dtensor, unused_dlengths): + # TODO(srbs): We lose the element_shape information in tensor_list_concat. + # Consider providing that as an output of TensorListConcat? + if dtensor.shape.rank is None: + element_shape = None + else: + element_shape = [None] + dtensor.shape.as_list()[1:] + return tensor_list_split( + dtensor, + element_shape=_build_element_shape(element_shape), + lengths=op.outputs[1]) + + +@ops.RegisterGradient("TensorListSplit") +def _TensorListSplitGrad(op, dlist): + return tensor_list_concat(dlist, element_dtype=op.inputs[0].dtype), None, None @ops.RegisterGradient("TensorListFromTensor") def _TensorListFromTensorGrad(op, dlist): """Gradient for TensorListFromTensor.""" - if op.inputs[0].shape.dims[0].value is not None: + if op.inputs[0].shape.dims and op.inputs[0].shape.dims[0].value is not None: num_elements = op.inputs[0].shape.dims[0].value else: num_elements = None @@ -126,3 +177,40 @@ def _TensorListScatterGrad(op, dlist): t, indices, _ = op.inputs return gen_list_ops.tensor_list_gather( dlist, indices, element_dtype=t.dtype), None + + +def _build_element_shape(shape): + """Converts shape to a format understood by list_ops for element_shape. + + If `shape` is already a `Tensor` it is returned as-is. We do not perform a + type check here. + + If shape is None or a TensorShape with unknown rank, -1 is returned. + + If shape is a scalar, an int32 tensor with empty list is returned. Note we + do directly return an empty list since ops.convert_to_tensor would conver it + to a float32 which is not a valid type for element_shape. + + If shape is a sequence of dims, None's in the list are replaced with -1. We + do not check the dtype of the other dims. + + Args: + shape: Could be None, Tensor, TensorShape or a list of dims (each dim could + be a None, scalar or Tensor). + + Returns: + A None-free shape that can be converted to a tensor. + """ + if isinstance(shape, ops.Tensor): + return shape + if isinstance(shape, tensor_shape.TensorShape): + # `TensorShape.as_list` requires rank to be known. + shape = shape.as_list() if shape else None + # Shape is unknown. + if shape is None: + return -1 + # Shape is a scalar. + if not shape: + return ops.convert_to_tensor(shape, dtype=dtypes.int32) + # Shape is a sequence of dimensions. Convert None dims to -1. + return [d if d is not None else -1 for d in shape] diff --git a/tensorflow/python/ops/lookup_ops.py b/tensorflow/python/ops/lookup_ops.py index 397d56ef409..758cb8041da 100644 --- a/tensorflow/python/ops/lookup_ops.py +++ b/tensorflow/python/ops/lookup_ops.py @@ -39,6 +39,7 @@ from tensorflow.python.ops import string_ops # pylint: disable=wildcard-import from tensorflow.python.ops.gen_lookup_ops import * # pylint: enable=wildcard-import +from tensorflow.python.training.checkpointable import base as checkpointable_base from tensorflow.python.training.checkpointable import tracking as checkpointable from tensorflow.python.util import compat from tensorflow.python.util.deprecation import deprecated @@ -160,7 +161,9 @@ class InitializableLookupTableBase(LookupInterface): self._default_value = ops.convert_to_tensor( default_value, dtype=self._value_dtype) self._default_value.get_shape().merge_with(tensor_shape.scalar()) - self._initializer = initializer + if isinstance(initializer, checkpointable_base.CheckpointableBase): + self._initializer = self._track_checkpointable( + initializer, "_initializer") self._resource_handle = self.create_resource() self._init_op = self.initialize() @@ -309,7 +312,7 @@ class HashTable(InitializableLookupTableBase): return exported_keys, exported_values -class TableInitializerBase(object): +class TableInitializerBase(checkpointable_base.CheckpointableBase): """Base class for lookup table initializers.""" def __init__(self, key_dtype, value_dtype): @@ -522,12 +525,14 @@ class TextFileInitializer(TableInitializerBase): if (vocab_size is not None) and (vocab_size <= 0): raise ValueError("Invalid vocab_size %s." % vocab_size) - self._filename = filename self._key_index = key_index self._value_index = value_index self._vocab_size = vocab_size self._delimiter = delimiter self._name = name + self._filename = self._track_checkpointable( + checkpointable.TrackableAsset(filename), + "_filename") super(TextFileInitializer, self).__init__(key_dtype, value_dtype) diff --git a/tensorflow/python/ops/losses/losses_impl.py b/tensorflow/python/ops/losses/losses_impl.py index 53c09ee8ddf..9e9de62e6ca 100644 --- a/tensorflow/python/ops/losses/losses_impl.py +++ b/tensorflow/python/ops/losses/losses_impl.py @@ -18,7 +18,6 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function -from tensorflow.python.compat import compat from tensorflow.python.eager import context from tensorflow.python.framework import ops from tensorflow.python.ops import array_ops @@ -34,28 +33,48 @@ from tensorflow.python.util.deprecation import deprecated_argument_lookup from tensorflow.python.util.tf_export import tf_export -@tf_export("losses.Reduction") +@tf_export("losses.Reduction", "keras.losses.Reduction", v1=[]) +class ReductionV2(object): + """Types of loss reduction. + + Contains the following values: + `NONE`: Un-reduced weighted losses with the same shape as input. + `SUM`: Scalar sum of weighted losses. + `SUM_OVER_BATCH_SIZE`: Scalar `SUM` divided by number of elements in losses. + """ + + NONE = "none" + SUM = "sum" + SUM_OVER_BATCH_SIZE = "sum_over_batch_size" + + @classmethod + def all(cls): + return (cls.NONE, cls.SUM, cls.SUM_OVER_BATCH_SIZE) + + @classmethod + def validate(cls, key): + if key not in cls.all(): + raise ValueError("Invalid Reduction Key %s." % key) + + +@tf_export(v1=["losses.Reduction"]) class Reduction(object): """Types of loss reduction. Contains the following values: `NONE`: Un-reduced weighted losses with the same shape as input. `SUM`: Scalar sum of weighted losses. - `MEAN`: Scalar `SUM` divided by sum of weights. + `MEAN`: Scalar `SUM` divided by sum of weights. DEPRECATED. `SUM_OVER_BATCH_SIZE`: Scalar `SUM` divided by number of elements in losses. `SUM_OVER_NONZERO_WEIGHTS`: Scalar `SUM` divided by number of non-zero - weights. + weights. DEPRECATED. `SUM_BY_NONZERO_WEIGHTS`: Same as `SUM_OVER_NONZERO_WEIGHTS`. """ NONE = "none" - SUM = "weighted_sum" - - MEAN = "weighted_mean" - SUM_OVER_BATCH_SIZE = "weighted_sum_over_batch_size" - + MEAN = "weighted_mean" SUM_BY_NONZERO_WEIGHTS = "weighted_sum_by_nonzero_weights" SUM_OVER_NONZERO_WEIGHTS = SUM_BY_NONZERO_WEIGHTS @@ -72,35 +91,7 @@ class Reduction(object): @classmethod def validate(cls, key): if key not in cls.all(): - raise ValueError("Invalid ReductionKey %s." % key) - - -def _safe_div(numerator, denominator, name="value"): - """Computes a safe divide which returns 0 if the denominator is zero. - - Note that the function contains an additional conditional check that is - necessary for avoiding situations where the loss is zero causing NaNs to - creep into the gradient computation. - - Args: - numerator: An arbitrary `Tensor`. - denominator: A `Tensor` whose shape matches `numerator` and whose values are - assumed to be non-negative. - name: An optional name for the returned op. - - Returns: - The element-wise value of the numerator divided by the denominator. - """ - if compat.forward_compatible(2018, 11, 1): - return math_ops.div_no_nan(numerator, denominator, name=name) - return array_ops.where( - math_ops.greater(denominator, 0), - math_ops.div(numerator, - array_ops.where( - math_ops.equal(denominator, 0), - array_ops.ones_like(denominator), denominator)), - array_ops.zeros_like(numerator), - name=name) + raise ValueError("Invalid Reduction Key %s." % key) def _safe_mean(losses, num_present): @@ -115,7 +106,7 @@ def _safe_mean(losses, num_present): then zero is returned. """ total_loss = math_ops.reduce_sum(losses) - return _safe_div(total_loss, num_present) + return math_ops.div_no_nan(total_loss, num_present, name="value") def _num_present(losses, weights, per_batch=False): @@ -166,7 +157,7 @@ def _num_elements(losses): return math_ops.cast(array_ops.size(losses, name=scope), dtype=losses.dtype) -@tf_export("losses.compute_weighted_loss") +@tf_export(v1=["losses.compute_weighted_loss"]) def compute_weighted_loss( losses, weights=1.0, scope=None, loss_collection=ops.GraphKeys.LOSSES, reduction=Reduction.SUM_BY_NONZERO_WEIGHTS): @@ -236,7 +227,7 @@ def compute_weighted_loss( return loss -@tf_export("losses.absolute_difference") +@tf_export(v1=["losses.absolute_difference"]) def absolute_difference( labels, predictions, weights=1.0, scope=None, loss_collection=ops.GraphKeys.LOSSES, @@ -289,7 +280,7 @@ def absolute_difference( losses, weights, scope, loss_collection, reduction=reduction) -@tf_export("losses.cosine_distance") +@tf_export(v1=["losses.cosine_distance"]) @deprecated_args(None, "dim is deprecated, use axis instead", "dim") def cosine_distance( labels, predictions, axis=None, weights=1.0, scope=None, @@ -345,7 +336,7 @@ def cosine_distance( losses, weights, scope, loss_collection, reduction=reduction) -@tf_export("losses.hinge_loss") +@tf_export(v1=["losses.hinge_loss"]) def hinge_loss(labels, logits, weights=1.0, scope=None, loss_collection=ops.GraphKeys.LOSSES, reduction=Reduction.SUM_BY_NONZERO_WEIGHTS): @@ -395,7 +386,7 @@ def hinge_loss(labels, logits, weights=1.0, scope=None, losses, weights, scope, loss_collection, reduction=reduction) -@tf_export("losses.huber_loss") +@tf_export(v1=["losses.huber_loss"]) def huber_loss(labels, predictions, weights=1.0, delta=1.0, scope=None, loss_collection=ops.GraphKeys.LOSSES, reduction=Reduction.SUM_BY_NONZERO_WEIGHTS): @@ -473,7 +464,7 @@ def huber_loss(labels, predictions, weights=1.0, delta=1.0, scope=None, losses, weights, scope, loss_collection, reduction=reduction) -@tf_export("losses.log_loss") +@tf_export(v1=["losses.log_loss"]) def log_loss(labels, predictions, weights=1.0, epsilon=1e-7, scope=None, loss_collection=ops.GraphKeys.LOSSES, reduction=Reduction.SUM_BY_NONZERO_WEIGHTS): @@ -530,7 +521,7 @@ def log_loss(labels, predictions, weights=1.0, epsilon=1e-7, scope=None, # TODO(b/37208492): Add reduction arg. -@tf_export("losses.mean_pairwise_squared_error") +@tf_export(v1=["losses.mean_pairwise_squared_error"]) def mean_pairwise_squared_error( labels, predictions, weights=1.0, scope=None, loss_collection=ops.GraphKeys.LOSSES): @@ -595,26 +586,24 @@ def mean_pairwise_squared_error( diffs = math_ops.subtract(predictions, labels) - reduction_indices = math_ops.range(1, array_ops.rank(diffs)) + axis = math_ops.range(1, array_ops.rank(diffs)) sum_squares_diff_per_batch = math_ops.reduce_sum( - math_ops.square(diffs), - reduction_indices=reduction_indices, - keepdims=True) + math_ops.square(diffs), axis=axis, keepdims=True) num_present_per_batch = _num_present(diffs, weights, per_batch=True) - term1 = 2.0 * _safe_div( + term1 = 2.0 * math_ops.div_no_nan( sum_squares_diff_per_batch, - math_ops.maximum(num_present_per_batch - 1, 0)) + math_ops.maximum(num_present_per_batch - 1, 0), + name="value") - sum_diff = math_ops.reduce_sum( - diffs, reduction_indices=reduction_indices, keepdims=True) - term2 = 2.0 * _safe_div( + sum_diff = math_ops.reduce_sum(diffs, axis=axis, keepdims=True) + term2 = 2.0 * math_ops.div_no_nan( math_ops.square(sum_diff), math_ops.maximum( math_ops.multiply(num_present_per_batch, - num_present_per_batch - 1), - 0)) + num_present_per_batch - 1), 0), + name="value") weighted_losses = math_ops.multiply(term1 - term2, weights) loss = math_ops.reduce_sum(weighted_losses) @@ -628,7 +617,7 @@ def mean_pairwise_squared_error( return mean_loss -@tf_export("losses.mean_squared_error") +@tf_export(v1=["losses.mean_squared_error"]) def mean_squared_error( labels, predictions, weights=1.0, scope=None, loss_collection=ops.GraphKeys.LOSSES, @@ -681,7 +670,7 @@ def mean_squared_error( losses, weights, scope, loss_collection, reduction=reduction) -@tf_export("losses.sigmoid_cross_entropy") +@tf_export(v1=["losses.sigmoid_cross_entropy"]) def sigmoid_cross_entropy( multi_class_labels, logits, weights=1.0, label_smoothing=0, scope=None, loss_collection=ops.GraphKeys.LOSSES, @@ -745,7 +734,7 @@ def sigmoid_cross_entropy( losses, weights, scope, loss_collection, reduction=reduction) -@tf_export("losses.softmax_cross_entropy") +@tf_export(v1=["losses.softmax_cross_entropy"]) def softmax_cross_entropy( onehot_labels, logits, weights=1.0, label_smoothing=0, scope=None, loss_collection=ops.GraphKeys.LOSSES, @@ -867,7 +856,7 @@ def _remove_squeezable_dimensions( return labels, predictions, weights -@tf_export("losses.sparse_softmax_cross_entropy") +@tf_export(v1=["losses.sparse_softmax_cross_entropy"]) def sparse_softmax_cross_entropy( labels, logits, weights=1.0, scope=None, loss_collection=ops.GraphKeys.LOSSES, diff --git a/tensorflow/python/ops/losses/util_test.py b/tensorflow/python/ops/losses/util_test.py index df2e60e2e45..22a8eaae266 100644 --- a/tensorflow/python/ops/losses/util_test.py +++ b/tensorflow/python/ops/losses/util_test.py @@ -20,12 +20,14 @@ from __future__ import print_function from tensorflow.python.framework import constant_op from tensorflow.python.framework import ops +from tensorflow.python.framework import test_util from tensorflow.python.ops.losses import util from tensorflow.python.platform import test class LossesUtilTest(test.TestCase): + @test_util.run_deprecated_v1 def testGetRegularizationLoss(self): # Empty regularization collection should evaluate to 0.0. with self.cached_session(): diff --git a/tensorflow/python/ops/math_grad.py b/tensorflow/python/ops/math_grad.py index 35278d96804..c7ec1c57d1b 100644 --- a/tensorflow/python/ops/math_grad.py +++ b/tensorflow/python/ops/math_grad.py @@ -1041,11 +1041,12 @@ def _PowGrad(op, grad): # Avoid false singularity at x = 0 if x.dtype.is_complex: # real(x) < 0 is fine for the complex case - log_x = array_ops.where( - math_ops.not_equal(x, 0), math_ops.log(x), array_ops.zeros_like(x)) + mask = math_ops.not_equal(x, 0) else: # There's no sensible real value to return if x < 0, so return 0 - log_x = array_ops.where(x > 0, math_ops.log(x), array_ops.zeros_like(x)) + mask = x > 0 + safe_x = array_ops.where(mask, x, array_ops.ones_like(x)) + log_x = array_ops.where(mask, math_ops.log(safe_x), array_ops.zeros_like(x)) gy = array_ops.reshape(math_ops.reduce_sum(grad * z * log_x, ry), sy) return gx, gy diff --git a/tensorflow/python/ops/math_grad_test.py b/tensorflow/python/ops/math_grad_test.py index d1fe834fc78..822f89768c5 100644 --- a/tensorflow/python/ops/math_grad_test.py +++ b/tensorflow/python/ops/math_grad_test.py @@ -20,9 +20,13 @@ from __future__ import print_function import numpy as np +from tensorflow.python.eager import backprop +from tensorflow.python.eager import context +from tensorflow.python.eager import execution_callbacks from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops +from tensorflow.python.framework import test_util from tensorflow.python.ops import array_ops from tensorflow.python.ops import gradient_checker from tensorflow.python.ops import gradients @@ -52,6 +56,7 @@ class SquaredDifferenceOpTest(test.TestCase): self.assertLess(left_err, 1e-10) self.assertLess(right_err, 1e-10) + @test_util.run_deprecated_v1 def testGrad(self): self._testGrad([1, 2, 3, 2], [3, 2]) self._testGrad([2, 4], [3, 2, 4]) @@ -83,6 +88,7 @@ class AbsOpTest(test.TestCase): value, shape, output, output.get_shape().as_list()) self.assertLess(error, max_error) + @test_util.run_deprecated_v1 def testComplexAbs(self): # Bias random test values away from zero to avoid numeric instabilities. self._testGrad( @@ -99,6 +105,7 @@ class AbsOpTest(test.TestCase): class MinOrMaxGradientTest(test.TestCase): + @test_util.run_deprecated_v1 def testMinGradient(self): inputs = constant_op.constant([1.0], dtype=dtypes.float32) outputs = math_ops.reduce_min(array_ops.concat([inputs, inputs], 0)) @@ -106,6 +113,7 @@ class MinOrMaxGradientTest(test.TestCase): error = gradient_checker.compute_gradient_error(inputs, [1], outputs, []) self.assertLess(error, 1e-4) + @test_util.run_deprecated_v1 def testMaxGradient(self): inputs = constant_op.constant([1.0], dtype=dtypes.float32) outputs = math_ops.reduce_max(array_ops.concat([inputs, inputs], 0)) @@ -116,6 +124,7 @@ class MinOrMaxGradientTest(test.TestCase): class MaximumOrMinimumGradientTest(test.TestCase): + @test_util.run_deprecated_v1 def testMaximumGradient(self): inputs = constant_op.constant([1.0, 2.0, 3.0, 4.0], dtype=dtypes.float32) outputs = math_ops.maximum(inputs, 3.0) @@ -123,6 +132,7 @@ class MaximumOrMinimumGradientTest(test.TestCase): error = gradient_checker.compute_gradient_error(inputs, [4], outputs, [4]) self.assertLess(error, 1e-4) + @test_util.run_deprecated_v1 def testMinimumGradient(self): inputs = constant_op.constant([1.0, 2.0, 3.0, 4.0], dtype=dtypes.float32) outputs = math_ops.minimum(inputs, 2.0) @@ -133,6 +143,7 @@ class MaximumOrMinimumGradientTest(test.TestCase): class ProdGradientTest(test.TestCase): + @test_util.run_deprecated_v1 def testProdGradient(self): inputs = constant_op.constant([[1., 2.], [3., 4.]], dtype=dtypes.float32) @@ -143,6 +154,7 @@ class ProdGradientTest(test.TestCase): outputs, outputs.get_shape().as_list()) self.assertLess(error, 1e-4) + @test_util.run_deprecated_v1 def testProdGradientForNegativeAxis(self): inputs = constant_op.constant([[1., 2.], [3., 4.]], dtype=dtypes.float32) @@ -153,6 +165,7 @@ class ProdGradientTest(test.TestCase): outputs, outputs.get_shape().as_list()) self.assertLess(error, 1e-4) + @test_util.run_deprecated_v1 def testProdGradientComplex(self): for dtype in dtypes.complex64, dtypes.complex128: inputs = constant_op.constant([[1 + 3j, 2 - 1j], [3j, 4]], @@ -164,6 +177,7 @@ class ProdGradientTest(test.TestCase): outputs, outputs.get_shape().as_list()) self.assertLess(error, 1e-4) + @test_util.run_deprecated_v1 def testProdGradientForNegativeAxisComplex(self): for dtype in dtypes.complex64, dtypes.complex128: inputs = constant_op.constant([[1 + 3j, 2 - 1j], [3j, 4]], @@ -178,6 +192,7 @@ class ProdGradientTest(test.TestCase): class SegmentMinOrMaxGradientTest(test.TestCase): + @test_util.run_deprecated_v1 def testSegmentMinGradient(self): data = constant_op.constant([1.0, 2.0, 3.0], dtype=dtypes.float32) segment_ids = constant_op.constant([0, 0, 1], dtype=dtypes.int64) @@ -187,6 +202,7 @@ class SegmentMinOrMaxGradientTest(test.TestCase): [2]) self.assertLess(error, 1e-4) + @test_util.run_deprecated_v1 def testSegmentMaxGradient(self): data = constant_op.constant([1.0, 2.0, 3.0], dtype=dtypes.float32) segment_ids = constant_op.constant([0, 0, 1], dtype=dtypes.int64) @@ -196,6 +212,7 @@ class SegmentMinOrMaxGradientTest(test.TestCase): [2]) self.assertLess(error, 1e-4) + @test_util.run_deprecated_v1 def testSegmentMinGradientWithTies(self): inputs = constant_op.constant([1.0], dtype=dtypes.float32) data = array_ops.concat([inputs, inputs], 0) @@ -206,6 +223,7 @@ class SegmentMinOrMaxGradientTest(test.TestCase): [1]) self.assertLess(error, 1e-4) + @test_util.run_deprecated_v1 def testSegmentMaxGradientWithTies(self): inputs = constant_op.constant([1.0], dtype=dtypes.float32) data = array_ops.concat([inputs, inputs], 0) @@ -219,6 +237,7 @@ class SegmentMinOrMaxGradientTest(test.TestCase): class FloorModGradientTest(test.TestCase): + @test_util.run_deprecated_v1 def testFloorModGradient(self): # Making sure the input is not near the discontinuity point where # x/y == floor(x/y) @@ -233,6 +252,7 @@ class FloorModGradientTest(test.TestCase): class DivNoNanGradientTest(test.TestCase): + @test_util.run_deprecated_v1 def testBasicGradient(self): inputs = constant_op.constant(np.arange(-3, 3), dtype=dtypes.float32) @@ -244,6 +264,7 @@ class DivNoNanGradientTest(test.TestCase): outputs.get_shape().as_list()) self.assertLess(error, 1e-4) + @test_util.run_deprecated_v1 def testGradientWithDenominatorIsZero(self): x = constant_op.constant(np.arange(-3, 3), dtype=dtypes.float32) @@ -263,6 +284,7 @@ class XlogyTest(test.TestCase): xlogy_ygrad = self.evaluate(gradients.gradients(math_ops.xlogy(x, y), y)[0]) return xlogy_xgrad, xlogy_ygrad + @test_util.run_deprecated_v1 def testNonZeroValuesGrad(self): for dtype in [dtypes.float16, dtypes.float32, dtypes.float64]: x = constant_op.constant(0.1, dtype=dtype) @@ -273,6 +295,7 @@ class XlogyTest(test.TestCase): self.assertAllClose(xlogy_expected_xgrad, xlogy_xgrad) self.assertAllClose(xlogy_expected_ygrad, xlogy_ygrad) + @test_util.run_deprecated_v1 def testZeroXGrad(self): for dtype in [dtypes.float16, dtypes.float32, dtypes.float64]: x = constant_op.constant(0., dtype=dtype) @@ -282,6 +305,7 @@ class XlogyTest(test.TestCase): self.assertAllClose(zero, xlogy_xgrad) self.assertAllClose(zero, xlogy_ygrad) + @test_util.run_deprecated_v1 def testZeroYGrad(self): for dtype in [dtypes.float16, dtypes.float32, dtypes.float64]: x = constant_op.constant(0.1, dtype=dtype) @@ -290,6 +314,7 @@ class XlogyTest(test.TestCase): self.assertAllClose(-np.inf, xlogy_xgrad) self.assertAllClose(np.inf, xlogy_ygrad) + @test_util.run_deprecated_v1 def testZeroXYGrad(self): for dtype in [dtypes.float16, dtypes.float32, dtypes.float64]: x = constant_op.constant(0., dtype=dtype) @@ -307,6 +332,7 @@ class XdivyTest(test.TestCase): xdivy_ygrad = self.evaluate(gradients.gradients(math_ops.xdivy(x, y), y)[0]) return xdivy_xgrad, xdivy_ygrad + @test_util.run_deprecated_v1 def testNonZeroValuesGrad(self): for dtype in [dtypes.float16, dtypes.float32, dtypes.float64]: x = constant_op.constant(0.1, dtype=dtype) @@ -317,6 +343,7 @@ class XdivyTest(test.TestCase): self.assertAllClose(xdivy_expected_xgrad, xdivy_xgrad) self.assertAllClose(xdivy_expected_ygrad, xdivy_ygrad) + @test_util.run_deprecated_v1 def testZeroXGrad(self): for dtype in [dtypes.float16, dtypes.float32, dtypes.float64]: x = constant_op.constant(0., dtype=dtype) @@ -326,6 +353,7 @@ class XdivyTest(test.TestCase): self.assertAllClose(zero, xdivy_xgrad) self.assertAllClose(zero, xdivy_ygrad) + @test_util.run_deprecated_v1 def testZeroYGrad(self): for dtype in [dtypes.float16, dtypes.float32, dtypes.float64]: x = constant_op.constant(0.1, dtype=dtype) @@ -334,6 +362,7 @@ class XdivyTest(test.TestCase): self.assertAllClose(np.inf, xdivy_xgrad) self.assertAllClose(-np.inf, xdivy_ygrad) + @test_util.run_deprecated_v1 def testZeroXYGrad(self): for dtype in [dtypes.float16, dtypes.float32, dtypes.float64]: x = constant_op.constant(0., dtype=dtype) @@ -344,5 +373,25 @@ class XdivyTest(test.TestCase): self.assertAllClose(zero, xdivy_ygrad) +@test_util.run_all_in_graph_and_eager_modes +class PowGradTest(test.TestCase): + + def test_zero_grad_tf_gradients(self): + if context.executing_eagerly(): + self.skipTest("tf.gradients not supported in eager.") + + x = constant_op.constant([-1., 0., 1.]) + g = self.evaluate(gradients.gradients(math_ops.pow(x, 2), x)[0]) + self.assertAllClose([-2., 0., 2.], g) + + def test_zero_grad_tape(self): + with execution_callbacks.errstate(inf_or_nan=execution_callbacks.RAISE): + x = constant_op.constant([-1, 0., 1.]) + with backprop.GradientTape() as tape: + tape.watch(x) + g = tape.gradient(math_ops.pow(x, 2), x) + g = self.evaluate(g) + self.assertAllClose([-2., 0., 2.], g) + if __name__ == "__main__": test.main() diff --git a/tensorflow/python/ops/math_ops.py b/tensorflow/python/ops/math_ops.py index 39b1ca8993e..f0d8bed5087 100644 --- a/tensorflow/python/ops/math_ops.py +++ b/tensorflow/python/ops/math_ops.py @@ -36,7 +36,6 @@ from tensorflow.python.ops import gen_data_flow_ops from tensorflow.python.ops import gen_math_ops from tensorflow.python.ops import gen_nn_ops from tensorflow.python.ops import gen_sparse_ops -from tensorflow.python.ops import gen_spectral_ops # go/tf-wildcard-import # pylint: disable=wildcard-import from tensorflow.python.ops.gen_math_ops import * @@ -44,6 +43,7 @@ from tensorflow.python.ops.gen_math_ops import * from tensorflow.python.platform import tf_logging as logging from tensorflow.python.util import compat from tensorflow.python.util import deprecation +from tensorflow.python.util import dispatch from tensorflow.python.util import nest from tensorflow.python.util.tf_export import tf_export @@ -52,8 +52,8 @@ linspace = gen_math_ops.lin_space arg_max = deprecation.deprecated(None, "Use `tf.math.argmax` instead")(arg_max) # pylint: disable=used-before-assignment arg_min = deprecation.deprecated(None, "Use `tf.math.argmin` instead")(arg_min) # pylint: disable=used-before-assignment -tf_export("arg_max")(arg_max) -tf_export("arg_min")(arg_min) +tf_export(v1=["arg_max"])(arg_max) +tf_export(v1=["arg_min"])(arg_min) # This is set by resource_variable_ops.py. It is included in this way since # there is a circular dependency between math_ops and resource_variable_ops @@ -83,8 +83,6 @@ def argmax(input, output_type=dtypes.int64): axis = deprecation.deprecated_argument_lookup( "axis", axis, "dimension", dimension) - if axis is None: - axis = 0 return argmax_v2(input, axis, output_type, name) @@ -112,6 +110,8 @@ def argmax_v2(input, Returns: A `Tensor` of type `output_type`. """ + if axis is None: + axis = 0 return gen_math_ops.arg_max(input, axis, name=name, output_type=output_type) @@ -128,8 +128,6 @@ def argmin(input, output_type=dtypes.int64): axis = deprecation.deprecated_argument_lookup( "axis", axis, "dimension", dimension) - if axis is None: - axis = 0 return argmin_v2(input, axis, output_type, name) @@ -157,6 +155,8 @@ def argmin_v2(input, Returns: A `Tensor` of type `output_type`. """ + if axis is None: + axis = 0 return gen_math_ops.arg_min(input, axis, name=name, output_type=output_type) @@ -166,6 +166,7 @@ def argmin_v2(input, # pylint: disable=anomalous-backslash-in-string,protected-access # pylint: disable=g-docstring-has-escape @tf_export("math.abs", "abs") +@dispatch.add_dispatch_support def abs(x, name=None): # pylint: disable=redefined-builtin r"""Computes the absolute value of a tensor. @@ -190,22 +191,10 @@ def abs(x, name=None): # pylint: disable=redefined-builtin of type `float32` or `float64`, respectively. """ with ops.name_scope(name, "Abs", [x]) as name: - if isinstance(x, sparse_tensor.SparseTensor): - if x.values.dtype.is_complex: - x_abs = gen_math_ops.complex_abs( - x.values, Tout=x.values.dtype.real_dtype, name=name) - return sparse_tensor.SparseTensor( - indices=x.indices, values=x_abs, dense_shape=x.dense_shape) - x_abs = gen_math_ops._abs(x.values, name=name) - return sparse_tensor.SparseTensor( - indices=x.indices, values=x_abs, dense_shape=x.dense_shape) - else: - x = ops.convert_to_tensor(x, name="x") - if x.dtype.is_complex: - return gen_math_ops.complex_abs(x, Tout=x.dtype.real_dtype, name=name) - return gen_math_ops._abs(x, name=name) - - + x = ops.convert_to_tensor(x, name="x") + if x.dtype.is_complex: + return gen_math_ops.complex_abs(x, Tout=x.dtype.real_dtype, name=name) + return gen_math_ops._abs(x, name=name) # pylint: enable=g-docstring-has-escape @@ -292,31 +281,7 @@ _sub.__doc__ = ( gen_math_ops.sub.__doc__ + ("" if _sub.__doc__ is None else _sub.__doc__)) -# pylint: disable=g-docstring-has-escape -@tf_export("math.negative", "negative") -def negative(x, name=None): - """Computes numerical negative value element-wise. - - I.e., \\(y = -x\\). - - Args: - x: A `Tensor` or `SparseTensor`. Must be one of the following types: `half`, - `float32`, `float64`, `int32`, `int64`, `complex64`, `complex128`. - name: A name for the operation (optional). - - Returns: - A `Tensor` or `SparseTensor`, respectively. Has the same type as `x`. - """ - with ops.name_scope(name, "Neg", [x]) as name: - if isinstance(x, sparse_tensor.SparseTensor): - x_neg = gen_math_ops.neg(x.values, name=name) - return sparse_tensor.SparseTensor( - indices=x.indices, values=x_neg, dense_shape=x.dense_shape) - else: - return gen_math_ops.neg(x, name=name) - - -# pylint: enable=g-docstring-has-escape +negative = gen_math_ops.neg # pylint: disable=g-docstring-has-escape @@ -342,107 +307,8 @@ def _neg(x, name=None): # pylint: enable=g-docstring-has-escape -@tf_export("math.sign", "sign") -def sign(x, name=None): - """Returns an element-wise indication of the sign of a number. - - `y = sign(x) = -1` if `x < 0`; 0 if `x == 0` or `tf.is_nan(x)`; 1 if `x > 0`. - - Zero is returned for NaN inputs. - - For complex numbers, `y = sign(x) = x / |x|` if `x != 0`, otherwise `y = 0`. - - Args: - x: A `Tensor` or `SparseTensor`. Must be one of the following types: `half`, - `float32`, `float64`, `int32`, `int64`, `complex64`, `complex128`. - name: A name for the operation (optional). - - Returns: - A `Tensor` or `SparseTensor`, respectively. Has the same type as `x`. - - @compatibility(numpy) - Equivalent to numpy.sign except for the behavior for input values of NaN. - @end_compatibility - """ - with ops.name_scope(name, "Sign", [x]) as name: - if isinstance(x, sparse_tensor.SparseTensor): - x_sign = gen_math_ops.sign(x.values, name=name) - return sparse_tensor.SparseTensor( - indices=x.indices, values=x_sign, dense_shape=x.dense_shape) - else: - return gen_math_ops.sign(x, name=name) - - -@tf_export("math.square", "square") -def square(x, name=None): - r"""Computes square of x element-wise. - - I.e., \\(y = x * x = x^2\\). - - Args: - x: A `Tensor` or `SparseTensor`. Must be one of the following types: `half`, - `float32`, `float64`, `int32`, `int64`, `complex64`, `complex128`. - name: A name for the operation (optional). - - Returns: - A `Tensor` or `SparseTensor`. Has the same type as `x`. - """ - with ops.name_scope(name, "Square", [x]) as name: - if isinstance(x, sparse_tensor.SparseTensor): - x_square = gen_math_ops.square(x.values, name=name) - return sparse_tensor.SparseTensor( - indices=x.indices, values=x_square, dense_shape=x.dense_shape) - else: - return gen_math_ops.square(x, name=name) - - -@tf_export("math.sqrt", "sqrt") -def sqrt(x, name=None): - r"""Computes square root of x element-wise. - - I.e., \\(y = \sqrt{x} = x^{1/2}\\). - - Args: - x: A `Tensor` or `SparseTensor`. Must be one of the following types: `half`, - `float32`, `float64`, `complex64`, `complex128`. - name: A name for the operation (optional). - - Returns: - A `Tensor` or `SparseTensor`, respectively. Has the same type as `x`. - """ - with ops.name_scope(name, "Sqrt", [x]) as name: - if isinstance(x, sparse_tensor.SparseTensor): - x_sqrt = gen_math_ops.sqrt(x.values, name=name) - return sparse_tensor.SparseTensor( - indices=x.indices, values=x_sqrt, dense_shape=x.dense_shape) - else: - return gen_math_ops.sqrt(x, name=name) - - -@tf_export("math.erf", v1=["math.erf", "erf"]) -@deprecation.deprecated_endpoints("erf") -def erf(x, name=None): - """Computes the Gauss error function of `x` element-wise. - - Args: - x: A `Tensor` or `SparseTensor`. Must be one of the following types: `half`, - `float32`, `float64`. - name: A name for the operation (optional). - - Returns: - A `Tensor` or `SparseTensor`, respectively. Has the same type as `x`. - """ - with ops.name_scope(name, "Erf", [x]) as name: - if isinstance(x, sparse_tensor.SparseTensor): - x_erf = gen_math_ops.erf(x.values, name=name) - return sparse_tensor.SparseTensor( - indices=x.indices, values=x_erf, dense_shape=x.dense_shape) - else: - return gen_math_ops.erf(x, name=name) - - -@tf_export("math.scalar_mul", "scalar_mul") -def scalar_mul(scalar, x): +@tf_export(v1=["math.scalar_mul", "scalar_mul"]) +def scalar_mul(scalar, x, name=None): """Multiplies a scalar times a `Tensor` or `IndexedSlices` object. Intended for use in gradient code which might deal with `IndexedSlices` @@ -452,6 +318,7 @@ def scalar_mul(scalar, x): Args: scalar: A 0-D scalar `Tensor`. Must have known shape. x: A `Tensor` or `IndexedSlices` to be scaled. + name: A name for the operation (optional). Returns: `scalar * x` of the same type (`Tensor` or `IndexedSlices`) as `x`. @@ -464,13 +331,21 @@ def scalar_mul(scalar, x): shape = scalar.get_shape() if shape.ndims == 0: if isinstance(x, ops.IndexedSlices): - return ops.IndexedSlices(scalar * x.values, x.indices, x.dense_shape) + return ops.IndexedSlices(gen_math_ops.mul(scalar, x.values, name), + x.indices, x.dense_shape) else: - return scalar * x + return gen_math_ops.mul(scalar, x, name) else: raise ValueError("Only scalar multiply works, got shape %s" % shape) +@tf_export("math.scalar_mul", "scalar_mul", v1=[]) +@_set_doc(scalar_mul.__doc__) +def scalar_mul_v2(scalar, x, name=None): + with ops.name_scope(name, "scalar_mul", [x]) as name: + return scalar_mul(scalar, x, name) + + @tf_export("math.pow", "pow") def pow(x, y, name=None): # pylint: disable=redefined-builtin r"""Computes the power of one value to another. @@ -1091,7 +966,10 @@ def truediv(x, y, name=None): return _truediv_python3(x, y, name) -@tf_export("div") +@deprecation.deprecated( + date=None, + instructions="Deprecated in favor of operator or tf.math.divide.") +@tf_export(v1=["div"]) def div(x, y, name=None): """Divides x / y elementwise (using Python 2 division operator semantics). @@ -1312,7 +1190,7 @@ def range(start, limit=None, delta=1, dtype=None, name="range"): # pylint: disa # Reduction operations -def _ReductionDims(x, axis, reduction_indices): +def _ReductionDims(x, axis, reduction_indices=None): # pylint: disable=invalid-name """Returns range(0, rank(x)) if reduction_indices is None.""" # TODO(aselle): Remove this after deprecation if reduction_indices is not None: @@ -1335,23 +1213,23 @@ def _ReductionDims(x, axis, reduction_indices): return range(0, array_ops.rank(x)) -def _may_reduce_to_scalar(keepdims, axis, reduction_indices, output): +def _may_reduce_to_scalar(keepdims, axis, output): """Set a reduction's output shape to be a scalar if we are certain.""" if not common_shapes.has_fully_defined_shape(output) and (not keepdims) and ( - axis is None) and (reduction_indices is None): + axis is None): output.set_shape(()) return output -@tf_export("math.reduce_sum", "reduce_sum") +@tf_export(v1=["math.reduce_sum", "reduce_sum"]) @deprecation.deprecated_args( None, "keep_dims is deprecated, use keepdims instead", "keep_dims") -def reduce_sum(input_tensor, - axis=None, - keepdims=None, - name=None, - reduction_indices=None, - keep_dims=None): +def reduce_sum_v1(input_tensor, + axis=None, + keepdims=None, + name=None, + reduction_indices=None, + keep_dims=None): """Computes the sum of elements across dimensions of a tensor. Reduces `input_tensor` along the dimensions given in `axis`. @@ -1391,21 +1269,61 @@ def reduce_sum(input_tensor, int64 while tensorflow returns the same dtype as the input. @end_compatibility """ + axis = deprecation.deprecated_argument_lookup( + "axis", axis, "reduction_indices", reduction_indices) keepdims = deprecation.deprecated_argument_lookup("keepdims", keepdims, "keep_dims", keep_dims) - if keepdims is None: - keepdims = False - - return _may_reduce_to_scalar(keepdims, axis, reduction_indices, - gen_math_ops._sum( - input_tensor, - _ReductionDims(input_tensor, axis, - reduction_indices), - keepdims, - name=name)) + return reduce_sum(input_tensor, axis, keepdims, name) -@tf_export("math.count_nonzero", "count_nonzero") +@tf_export("math.reduce_sum", "reduce_sum", v1=[]) +def reduce_sum(input_tensor, axis=None, keepdims=False, name=None): + """Computes the sum of elements across dimensions of a tensor. + + Reduces `input_tensor` along the dimensions given in `axis`. + Unless `keepdims` is true, the rank of the tensor is reduced by 1 for each + entry in `axis`. If `keepdims` is true, the reduced dimensions + are retained with length 1. + + If `axis` is None, all dimensions are reduced, and a + tensor with a single element is returned. + + For example: + + ```python + x = tf.constant([[1, 1, 1], [1, 1, 1]]) + tf.reduce_sum(x) # 6 + tf.reduce_sum(x, 0) # [2, 2, 2] + tf.reduce_sum(x, 1) # [3, 3] + tf.reduce_sum(x, 1, keepdims=True) # [[3], [3]] + tf.reduce_sum(x, [0, 1]) # 6 + ``` + + Args: + input_tensor: The tensor to reduce. Should have numeric type. + axis: The dimensions to reduce. If `None` (the default), reduces all + dimensions. Must be in the range `[-rank(input_tensor), + rank(input_tensor))`. + keepdims: If true, retains reduced dimensions with length 1. + name: A name for the operation (optional). + + Returns: + The reduced tensor, of the same dtype as the input_tensor. + + @compatibility(numpy) + Equivalent to np.sum apart the fact that numpy upcast uint8 and int32 to + int64 while tensorflow returns the same dtype as the input. + @end_compatibility + """ + keepdims = False if keepdims is None else keepdims + return _may_reduce_to_scalar( + keepdims, axis, + gen_math_ops._sum( + input_tensor, _ReductionDims(input_tensor, axis), keepdims, + name=name)) + + +@tf_export(v1=["math.count_nonzero", "count_nonzero"]) @deprecation.deprecated_args( None, "keep_dims is deprecated, use keepdims instead", "keep_dims") def count_nonzero(input_tensor, @@ -1466,32 +1384,89 @@ def count_nonzero(input_tensor, """ keepdims = deprecation.deprecated_argument_lookup("keepdims", keepdims, "keep_dims", keep_dims) + axis = deprecation.deprecated_argument_lookup( + "axis", axis, + "reduction_indices", reduction_indices + ) + + return count_nonzero_v2(input_tensor, axis, keepdims, dtype, name) + + +@tf_export("math.count_nonzero", v1=[]) +def count_nonzero_v2(input, # pylint: disable=redefined-builtin + axis=None, + keepdims=None, + dtype=dtypes.int64, + name=None): + """Computes number of nonzero elements across dimensions of a tensor. + + Reduces `input` along the dimensions given in `axis`. + Unless `keepdims` is true, the rank of the tensor is reduced by 1 for each + entry in `axis`. If `keepdims` is true, the reduced dimensions + are retained with length 1. + + If `axis` has no entries, all dimensions are reduced, and a + tensor with a single element is returned. + + **NOTE** Floating point comparison to zero is done by exact floating point + equality check. Small values are **not** rounded to zero for purposes of + the nonzero check. + + For example: + + ```python + x = tf.constant([[0, 1, 0], [1, 1, 0]]) + tf.count_nonzero(x) # 3 + tf.count_nonzero(x, 0) # [1, 2, 0] + tf.count_nonzero(x, 1) # [1, 2] + tf.count_nonzero(x, 1, keepdims=True) # [[1], [2]] + tf.count_nonzero(x, [0, 1]) # 3 + ``` + + **NOTE** Strings are compared against zero-length empty string `""`. Any + string with a size greater than zero is already considered as nonzero. + + For example: + ```python + x = tf.constant(["", "a", " ", "b", ""]) + tf.count_nonzero(x) # 3, with "a", " ", and "b" as nonzero strings. + ``` + + Args: + input: The tensor to reduce. Should be of numeric type, `bool`, + or `string`. + axis: The dimensions to reduce. If `None` (the default), + reduces all dimensions. Must be in the range + `[-rank(input), rank(input))`. + keepdims: If true, retains reduced dimensions with length 1. + dtype: The output dtype; defaults to `tf.int64`. + name: A name for the operation (optional). + + Returns: + The reduced tensor (number of nonzero values). + """ if keepdims is None: keepdims = False - - with ops.name_scope(name, "count_nonzero", [input_tensor]): - input_tensor = ops.convert_to_tensor(input_tensor, name="input_tensor") + with ops.name_scope(name, "count_nonzero", [input]): + input = ops.convert_to_tensor(input, name="input") # A scalar of 'zero' is enough as `not_equal` will broadcast. - zero = array_ops.zeros([], dtype=input_tensor.dtype) + zero = array_ops.zeros([], dtype=input.dtype) return cast( reduce_sum( # int64 reduction happens on GPU - to_int64(gen_math_ops.not_equal(input_tensor, zero)), + to_int64(gen_math_ops.not_equal(input, zero)), axis=axis, - keepdims=keepdims, - reduction_indices=reduction_indices), + keepdims=keepdims), dtype=dtype) -@tf_export("math.reduce_mean", "reduce_mean") -@deprecation.deprecated_args( - None, "keep_dims is deprecated, use keepdims instead", "keep_dims") -def reduce_mean(input_tensor, - axis=None, - keepdims=None, - name=None, - reduction_indices=None, - keep_dims=None): +@tf_export(v1=["math.reduce_mean", "reduce_mean"]) +def reduce_mean_v1(input_tensor, + axis=None, + keepdims=None, + name=None, + reduction_indices=None, + keep_dims=None): """Computes the mean of elements across dimensions of a tensor. Reduces `input_tensor` along the dimensions given in `axis`. @@ -1541,22 +1516,72 @@ def reduce_mean(input_tensor, @end_compatibility """ + axis = deprecation.deprecated_argument_lookup( + "axis", axis, "reduction_indices", reduction_indices) keepdims = deprecation.deprecated_argument_lookup("keepdims", keepdims, "keep_dims", keep_dims) + return reduce_mean(input_tensor, axis, keepdims, name) - if keepdims is None: - keepdims = False - return _may_reduce_to_scalar(keepdims, axis, reduction_indices, - gen_math_ops.mean( - input_tensor, - _ReductionDims(input_tensor, axis, - reduction_indices), - keepdims, - name=name)) + +@tf_export("math.reduce_mean", "reduce_mean", v1=[]) +def reduce_mean(input_tensor, axis=None, keepdims=False, name=None): + """Computes the mean of elements across dimensions of a tensor. + + Reduces `input_tensor` along the dimensions given in `axis`. + Unless `keepdims` is true, the rank of the tensor is reduced by 1 for each + entry in `axis`. If `keepdims` is true, the reduced dimensions + are retained with length 1. + + If `axis` is None, all dimensions are reduced, and a + tensor with a single element is returned. + + For example: + + ```python + x = tf.constant([[1., 1.], [2., 2.]]) + tf.reduce_mean(x) # 1.5 + tf.reduce_mean(x, 0) # [1.5, 1.5] + tf.reduce_mean(x, 1) # [1., 2.] + ``` + + Args: + input_tensor: The tensor to reduce. Should have numeric type. + axis: The dimensions to reduce. If `None` (the default), reduces all + dimensions. Must be in the range `[-rank(input_tensor), + rank(input_tensor))`. + keepdims: If true, retains reduced dimensions with length 1. + name: A name for the operation (optional). + + Returns: + The reduced tensor. + + @compatibility(numpy) + Equivalent to np.mean + + Please note that `np.mean` has a `dtype` parameter that could be used to + specify the output type. By default this is `dtype=float64`. On the other + hand, `tf.reduce_mean` has an aggressive type inference from `input_tensor`, + for example: + + ```python + x = tf.constant([1, 0, 1, 0]) + tf.reduce_mean(x) # 0 + y = tf.constant([1., 0., 1., 0.]) + tf.reduce_mean(y) # 0.5 + ``` + + @end_compatibility + """ + keepdims = False if keepdims is None else keepdims + return _may_reduce_to_scalar( + keepdims, axis, + gen_math_ops.mean( + input_tensor, _ReductionDims(input_tensor, axis), keepdims, + name=name)) @tf_export("math.reduce_variance") -def reduce_variance(input_tensor, axis=None, keepdims=None, name=None): +def reduce_variance(input_tensor, axis=None, keepdims=False, name=None): """Computes the variance of elements across dimensions of a tensor. Reduces `input_tensor` along the dimensions given in `axis`. @@ -1599,12 +1624,12 @@ def reduce_variance(input_tensor, axis=None, keepdims=None, name=None): name = name if name else "reduce_variance" with ops.name_scope(name): means = reduce_mean(input_tensor, axis=axis, keepdims=True) - squared_deviations = square(input_tensor - means) + squared_deviations = gen_math_ops.square(input_tensor - means) return reduce_mean(squared_deviations, axis=axis, keepdims=keepdims) @tf_export("math.reduce_std") -def reduce_std(input_tensor, axis=None, keepdims=None, name=None): +def reduce_std(input_tensor, axis=None, keepdims=False, name=None): """Computes the standard deviation of elements across dimensions of a tensor. Reduces `input_tensor` along the dimensions given in `axis`. @@ -1646,18 +1671,11 @@ def reduce_std(input_tensor, axis=None, keepdims=None, name=None): name = name if name else "reduce_std" with ops.name_scope(name): variance = reduce_variance(input_tensor, axis=axis, keepdims=keepdims) - return sqrt(variance) + return gen_math_ops.sqrt(variance) -@tf_export("math.reduce_prod", "reduce_prod") -@deprecation.deprecated_args( - None, "keep_dims is deprecated, use keepdims instead", "keep_dims") -def reduce_prod(input_tensor, - axis=None, - keepdims=None, - name=None, - reduction_indices=None, - keep_dims=None): +@tf_export("math.reduce_prod", "reduce_prod", v1=[]) +def reduce_prod(input_tensor, axis=None, keepdims=False, name=None): """Computes the product of elements across dimensions of a tensor. Reduces `input_tensor` along the dimensions given in `axis`. @@ -1675,6 +1693,48 @@ def reduce_prod(input_tensor, `[-rank(input_tensor), rank(input_tensor))`. keepdims: If true, retains reduced dimensions with length 1. name: A name for the operation (optional). + + Returns: + The reduced tensor. + + @compatibility(numpy) + Equivalent to np.prod + @end_compatibility + """ + keepdims = False if keepdims is None else keepdims + return _may_reduce_to_scalar( + keepdims, axis, + gen_math_ops.prod( + input_tensor, _ReductionDims(input_tensor, axis), keepdims, + name=name)) + + +@tf_export(v1=["math.reduce_prod", "reduce_prod"]) +@deprecation.deprecated_args( + None, "keep_dims is deprecated, use keepdims instead", "keep_dims") +def reduce_prod_v1(input_tensor, + axis=None, + keepdims=None, + name=None, + reduction_indices=None, + keep_dims=None): + """Computes the product of elements across dimensions of a tensor. + + Reduces `input_tensor` along the dimensions given in `axis`. + Unless `keepdims` is true, the rank of the tensor is reduced by 1 for each + entry in `axis`. If `keepdims` is true, the reduced dimensions + are retained with length 1. + + If `axis` is None, all dimensions are reduced, and a + tensor with a single element is returned. + + Args: + input_tensor: The tensor to reduce. Should have numeric type. + axis: The dimensions to reduce. If `None` (the default), reduces all + dimensions. Must be in the range `[-rank(input_tensor), + rank(input_tensor))`. + keepdims: If true, retains reduced dimensions with length 1. + name: A name for the operation (optional). reduction_indices: The old (deprecated) name for axis. keep_dims: Deprecated alias for `keepdims`. @@ -1685,29 +1745,22 @@ def reduce_prod(input_tensor, Equivalent to np.prod @end_compatibility """ + axis = deprecation.deprecated_argument_lookup( + "axis", axis, "reduction_indices", reduction_indices) keepdims = deprecation.deprecated_argument_lookup("keepdims", keepdims, "keep_dims", keep_dims) - - if keepdims is None: - keepdims = False - return _may_reduce_to_scalar(keepdims, axis, reduction_indices, - gen_math_ops.prod( - input_tensor, - _ReductionDims(input_tensor, axis, - reduction_indices), - keepdims, - name=name)) + return reduce_prod(input_tensor, axis, keepdims, name) -@tf_export("math.reduce_min", "reduce_min") +@tf_export(v1=["math.reduce_min", "reduce_min"]) @deprecation.deprecated_args( None, "keep_dims is deprecated, use keepdims instead", "keep_dims") -def reduce_min(input_tensor, - axis=None, - keepdims=None, - name=None, - reduction_indices=None, - keep_dims=None): +def reduce_min_v1(input_tensor, + axis=None, + keepdims=None, + name=None, + reduction_indices=None, + keep_dims=None): """Computes the minimum of elements across dimensions of a tensor. Reduces `input_tensor` along the dimensions given in `axis`. @@ -1720,9 +1773,9 @@ def reduce_min(input_tensor, Args: input_tensor: The tensor to reduce. Should have real numeric type. - axis: The dimensions to reduce. If `None` (the default), - reduces all dimensions. Must be in the range - `[-rank(input_tensor), rank(input_tensor))`. + axis: The dimensions to reduce. If `None` (the default), reduces all + dimensions. Must be in the range `[-rank(input_tensor), + rank(input_tensor))`. keepdims: If true, retains reduced dimensions with length 1. name: A name for the operation (optional). reduction_indices: The old (deprecated) name for axis. @@ -1735,28 +1788,57 @@ def reduce_min(input_tensor, Equivalent to np.min @end_compatibility """ + axis = deprecation.deprecated_argument_lookup( + "axis", axis, "reduction_indices", reduction_indices) keepdims = deprecation.deprecated_argument_lookup("keepdims", keepdims, "keep_dims", keep_dims) - if keepdims is None: - keepdims = False - return _may_reduce_to_scalar(keepdims, axis, reduction_indices, - gen_math_ops._min( - input_tensor, - _ReductionDims(input_tensor, axis, - reduction_indices), - keepdims, - name=name)) + return reduce_min(input_tensor, axis, keepdims, name) -@tf_export("math.reduce_max", "reduce_max") +@tf_export("math.reduce_min", "reduce_min", v1=[]) +def reduce_min(input_tensor, axis=None, keepdims=False, name=None): + """Computes the minimum of elements across dimensions of a tensor. + + Reduces `input_tensor` along the dimensions given in `axis`. + Unless `keepdims` is true, the rank of the tensor is reduced by 1 for each + entry in `axis`. If `keepdims` is true, the reduced dimensions + are retained with length 1. + + If `axis` is None, all dimensions are reduced, and a + tensor with a single element is returned. + + Args: + input_tensor: The tensor to reduce. Should have real numeric type. + axis: The dimensions to reduce. If `None` (the default), reduces all + dimensions. Must be in the range `[-rank(input_tensor), + rank(input_tensor))`. + keepdims: If true, retains reduced dimensions with length 1. + name: A name for the operation (optional). + + Returns: + The reduced tensor. + + @compatibility(numpy) + Equivalent to np.min + @end_compatibility + """ + keepdims = False if keepdims is None else keepdims + return _may_reduce_to_scalar( + keepdims, axis, + gen_math_ops._min( + input_tensor, _ReductionDims(input_tensor, axis), keepdims, + name=name)) + + +@tf_export(v1=["math.reduce_max", "reduce_max"]) @deprecation.deprecated_args( None, "keep_dims is deprecated, use keepdims instead", "keep_dims") -def reduce_max(input_tensor, - axis=None, - keepdims=None, - name=None, - reduction_indices=None, - keep_dims=None): +def reduce_max_v1(input_tensor, + axis=None, + keepdims=None, + name=None, + reduction_indices=None, + keep_dims=None): """Computes the maximum of elements across dimensions of a tensor. Reduces `input_tensor` along the dimensions given in `axis`. @@ -1784,28 +1866,57 @@ def reduce_max(input_tensor, Equivalent to np.max @end_compatibility """ + axis = deprecation.deprecated_argument_lookup( + "axis", axis, "reduction_indices", reduction_indices) keepdims = deprecation.deprecated_argument_lookup("keepdims", keepdims, "keep_dims", keep_dims) - if keepdims is None: - keepdims = False - return _may_reduce_to_scalar(keepdims, axis, reduction_indices, - gen_math_ops._max( - input_tensor, - _ReductionDims(input_tensor, axis, - reduction_indices), - keepdims, - name=name)) + return reduce_max(input_tensor, axis, keepdims, name) -@tf_export("math.reduce_all", "reduce_all") +@tf_export("math.reduce_max", "reduce_max", v1=[]) +def reduce_max(input_tensor, axis=None, keepdims=False, name=None): + """Computes the maximum of elements across dimensions of a tensor. + + Reduces `input_tensor` along the dimensions given in `axis`. + Unless `keepdims` is true, the rank of the tensor is reduced by 1 for each + entry in `axis`. If `keepdims` is true, the reduced dimensions + are retained with length 1. + + If `axis` is None, all dimensions are reduced, and a + tensor with a single element is returned. + + Args: + input_tensor: The tensor to reduce. Should have real numeric type. + axis: The dimensions to reduce. If `None` (the default), reduces all + dimensions. Must be in the range `[-rank(input_tensor), + rank(input_tensor))`. + keepdims: If true, retains reduced dimensions with length 1. + name: A name for the operation (optional). + + Returns: + The reduced tensor. + + @compatibility(numpy) + Equivalent to np.max + @end_compatibility + """ + keepdims = False if keepdims is None else keepdims + return _may_reduce_to_scalar( + keepdims, axis, + gen_math_ops._max( + input_tensor, _ReductionDims(input_tensor, axis), keepdims, + name=name)) + + +@tf_export(v1=["math.reduce_all", "reduce_all"]) @deprecation.deprecated_args( None, "keep_dims is deprecated, use keepdims instead", "keep_dims") -def reduce_all(input_tensor, - axis=None, - keepdims=None, - name=None, - reduction_indices=None, - keep_dims=None): +def reduce_all_v1(input_tensor, + axis=None, + keepdims=None, + name=None, + reduction_indices=None, + keep_dims=None): """Computes the "logical and" of elements across dimensions of a tensor. Reduces `input_tensor` along the dimensions given in `axis`. @@ -1827,9 +1938,9 @@ def reduce_all(input_tensor, Args: input_tensor: The boolean tensor to reduce. - axis: The dimensions to reduce. If `None` (the default), - reduces all dimensions. Must be in the range - `[-rank(input_tensor), rank(input_tensor))`. + axis: The dimensions to reduce. If `None` (the default), reduces all + dimensions. Must be in the range `[-rank(input_tensor), + rank(input_tensor))`. keepdims: If true, retains reduced dimensions with length 1. name: A name for the operation (optional). reduction_indices: The old (deprecated) name for axis. @@ -1842,28 +1953,66 @@ def reduce_all(input_tensor, Equivalent to np.all @end_compatibility """ + axis = deprecation.deprecated_argument_lookup( + "axis", axis, "reduction_indices", reduction_indices) keepdims = deprecation.deprecated_argument_lookup("keepdims", keepdims, "keep_dims", keep_dims) - if keepdims is None: - keepdims = False - return _may_reduce_to_scalar(keepdims, axis, reduction_indices, - gen_math_ops._all( - input_tensor, - _ReductionDims(input_tensor, axis, - reduction_indices), - keepdims, - name=name)) + return reduce_all(input_tensor, axis, keepdims, name) -@tf_export("math.reduce_any", "reduce_any") +@tf_export("reduce_all", "math.reduce_all", v1=[]) +def reduce_all(input_tensor, axis=None, keepdims=False, name=None): + """Computes the "logical and" of elements across dimensions of a tensor. + + Reduces `input_tensor` along the dimensions given in `axis`. + Unless `keepdims` is true, the rank of the tensor is reduced by 1 for each + entry in `axis`. If `keepdims` is true, the reduced dimensions + are retained with length 1. + + If `axis` is None, all dimensions are reduced, and a + tensor with a single element is returned. + + For example: + + ```python + x = tf.constant([[True, True], [False, False]]) + tf.reduce_all(x) # False + tf.reduce_all(x, 0) # [False, False] + tf.reduce_all(x, 1) # [True, False] + ``` + + Args: + input_tensor: The boolean tensor to reduce. + axis: The dimensions to reduce. If `None` (the default), reduces all + dimensions. Must be in the range `[-rank(input_tensor), + rank(input_tensor))`. + keepdims: If true, retains reduced dimensions with length 1. + name: A name for the operation (optional). + + Returns: + The reduced tensor. + + @compatibility(numpy) + Equivalent to np.all + @end_compatibility + """ + keepdims = False if keepdims is None else keepdims + return _may_reduce_to_scalar( + keepdims, axis, + gen_math_ops._all( + input_tensor, _ReductionDims(input_tensor, axis), keepdims, + name=name)) + + +@tf_export(v1=["math.reduce_any", "reduce_any"]) @deprecation.deprecated_args( None, "keep_dims is deprecated, use keepdims instead", "keep_dims") -def reduce_any(input_tensor, - axis=None, - keepdims=None, - name=None, - reduction_indices=None, - keep_dims=None): +def reduce_any_v1(input_tensor, + axis=None, + keepdims=None, + name=None, + reduction_indices=None, + keep_dims=None): """Computes the "logical or" of elements across dimensions of a tensor. Reduces `input_tensor` along the dimensions given in `axis`. @@ -1885,9 +2034,9 @@ def reduce_any(input_tensor, Args: input_tensor: The boolean tensor to reduce. - axis: The dimensions to reduce. If `None` (the default), - reduces all dimensions. Must be in the range - `[-rank(input_tensor), rank(input_tensor))`. + axis: The dimensions to reduce. If `None` (the default), reduces all + dimensions. Must be in the range `[-rank(input_tensor), + rank(input_tensor))`. keepdims: If true, retains reduced dimensions with length 1. name: A name for the operation (optional). reduction_indices: The old (deprecated) name for axis. @@ -1900,28 +2049,66 @@ def reduce_any(input_tensor, Equivalent to np.any @end_compatibility """ + axis = deprecation.deprecated_argument_lookup( + "axis", axis, "reduction_indices", reduction_indices) keepdims = deprecation.deprecated_argument_lookup("keepdims", keepdims, "keep_dims", keep_dims) - if keepdims is None: - keepdims = False - return _may_reduce_to_scalar(keepdims, axis, reduction_indices, - gen_math_ops._any( - input_tensor, - _ReductionDims(input_tensor, axis, - reduction_indices), - keepdims, - name=name)) + return reduce_any(input_tensor, axis, keepdims, name) -@tf_export("math.reduce_logsumexp", "reduce_logsumexp") +@tf_export("math.reduce_any", "reduce_any", v1=[]) +def reduce_any(input_tensor, axis=None, keepdims=False, name=None): + """Computes the "logical or" of elements across dimensions of a tensor. + + Reduces `input_tensor` along the dimensions given in `axis`. + Unless `keepdims` is true, the rank of the tensor is reduced by 1 for each + entry in `axis`. If `keepdims` is true, the reduced dimensions + are retained with length 1. + + If `axis` is None, all dimensions are reduced, and a + tensor with a single element is returned. + + For example: + + ```python + x = tf.constant([[True, True], [False, False]]) + tf.reduce_any(x) # True + tf.reduce_any(x, 0) # [True, True] + tf.reduce_any(x, 1) # [True, False] + ``` + + Args: + input_tensor: The boolean tensor to reduce. + axis: The dimensions to reduce. If `None` (the default), reduces all + dimensions. Must be in the range `[-rank(input_tensor), + rank(input_tensor))`. + keepdims: If true, retains reduced dimensions with length 1. + name: A name for the operation (optional). + + Returns: + The reduced tensor. + + @compatibility(numpy) + Equivalent to np.any + @end_compatibility + """ + keepdims = False if keepdims is None else keepdims + return _may_reduce_to_scalar( + keepdims, axis, + gen_math_ops._any( + input_tensor, _ReductionDims(input_tensor, axis), keepdims, + name=name)) + + +@tf_export(v1=["math.reduce_logsumexp", "reduce_logsumexp"]) @deprecation.deprecated_args( None, "keep_dims is deprecated, use keepdims instead", "keep_dims") -def reduce_logsumexp(input_tensor, - axis=None, - keepdims=None, - name=None, - reduction_indices=None, - keep_dims=None): +def reduce_logsumexp_v1(input_tensor, + axis=None, + keepdims=None, + name=None, + reduction_indices=None, + keep_dims=None): """Computes log(sum(exp(elements across dimensions of a tensor))). Reduces `input_tensor` along the dimensions given in `axis`. @@ -1949,9 +2136,9 @@ def reduce_logsumexp(input_tensor, Args: input_tensor: The tensor to reduce. Should have numeric type. - axis: The dimensions to reduce. If `None` (the default), - reduces all dimensions. Must be in the range - `[-rank(input_tensor), rank(input_tensor))`. + axis: The dimensions to reduce. If `None` (the default), reduces all + dimensions. Must be in the range `[-rank(input_tensor), + rank(input_tensor))`. keepdims: If true, retains reduced dimensions with length 1. name: A name for the operation (optional). reduction_indices: The old (deprecated) name for axis. @@ -1960,16 +2147,57 @@ def reduce_logsumexp(input_tensor, Returns: The reduced tensor. """ + axis = deprecation.deprecated_argument_lookup( + "axis", axis, "reduction_indices", reduction_indices) keepdims = deprecation.deprecated_argument_lookup("keepdims", keepdims, "keep_dims", keep_dims) - if keepdims is None: - keepdims = False + return reduce_logsumexp(input_tensor, axis, keepdims, name) + + +@tf_export("math.reduce_logsumexp", "reduce_logsumexp", v1=[]) +def reduce_logsumexp(input_tensor, axis=None, keepdims=False, name=None): + """Computes log(sum(exp(elements across dimensions of a tensor))). + + Reduces `input_tensor` along the dimensions given in `axis`. + Unless `keepdims` is true, the rank of the tensor is reduced by 1 for each + entry in `axis`. If `keepdims` is true, the reduced dimensions + are retained with length 1. + + If `axis` has no entries, all dimensions are reduced, and a + tensor with a single element is returned. + + This function is more numerically stable than log(sum(exp(input))). It avoids + overflows caused by taking the exp of large inputs and underflows caused by + taking the log of small inputs. + + For example: + + ```python + x = tf.constant([[0., 0., 0.], [0., 0., 0.]]) + tf.reduce_logsumexp(x) # log(6) + tf.reduce_logsumexp(x, 0) # [log(2), log(2), log(2)] + tf.reduce_logsumexp(x, 1) # [log(3), log(3)] + tf.reduce_logsumexp(x, 1, keepdims=True) # [[log(3)], [log(3)]] + tf.reduce_logsumexp(x, [0, 1]) # log(6) + ``` + + Args: + input_tensor: The tensor to reduce. Should have numeric type. + axis: The dimensions to reduce. If `None` (the default), reduces all + dimensions. Must be in the range `[-rank(input_tensor), + rank(input_tensor))`. + keepdims: If true, retains reduced dimensions with length 1. + name: A name for the operation (optional). + + Returns: + The reduced tensor. + """ + keepdims = False if keepdims is None else keepdims input_tensor = ops.convert_to_tensor(input_tensor) with ops.name_scope(name, "ReduceLogSumExp", [input_tensor]) as name: raw_max = reduce_max( input_tensor, axis=axis, - reduction_indices=reduction_indices, keepdims=True) my_max = array_ops.stop_gradient( array_ops.where( @@ -1979,12 +2207,11 @@ def reduce_logsumexp(input_tensor, reduce_sum( gen_math_ops.exp(gen_math_ops.sub(input_tensor, my_max)), axis, - keepdims=keepdims, - reduction_indices=reduction_indices)) + keepdims=keepdims)) if not keepdims: my_max = array_ops.reshape(my_max, array_ops.shape(result)) result = gen_math_ops.add(result, my_max) - return _may_reduce_to_scalar(keepdims, axis, reduction_indices, result) + return _may_reduce_to_scalar(keepdims, axis, result) @tf_export("linalg.trace", v1=["linalg.trace", "trace"]) @@ -2311,7 +2538,8 @@ def matvec(a, _OverrideBinaryOperatorHelper(matmul, "matmul") -sparse_matmul = gen_math_ops.sparse_mat_mul +sparse_matmul = deprecation.deprecated(None, "Use `tf.linalg.matmul` instead")( + gen_math_ops.sparse_mat_mul) tf_export(v1=["sparse_matmul"])(sparse_matmul) @@ -2555,34 +2783,13 @@ def log_sigmoid(x, name=None): return gen_math_ops.neg(gen_nn_ops.softplus(-x), name=name) -@tf_export("math.tanh", "nn.tanh", "tanh") -def tanh(x, name=None): - """Computes hyperbolic tangent of `x` element-wise. - - Args: - x: A Tensor or SparseTensor with type `float16`, `float32`, `double`, - `complex64`, or `complex128`. - name: A name for the operation (optional). - - Returns: - A Tensor or SparseTensor respectively with the same type as `x`. - """ - with ops.name_scope(name, "Tanh", [x]) as name: - if isinstance(x, sparse_tensor.SparseTensor): - x_tanh = gen_math_ops.tanh(x.values, name=name) - return sparse_tensor.SparseTensor( - indices=x.indices, values=x_tanh, dense_shape=x.dense_shape) - else: - return gen_math_ops.tanh(x, name=name) - - -@tf_export("math.bincount", v1=["math.bincount", "bincount"]) -@deprecation.deprecated_endpoints("bincount") +@tf_export("math.bincount", v1=[]) def bincount(arr, weights=None, minlength=None, maxlength=None, - dtype=dtypes.int32): + dtype=dtypes.int32, + name=None): """Counts the number of occurrences of each value in an integer array. If `minlength` and `maxlength` are not given, returns a vector with length @@ -2594,34 +2801,70 @@ def bincount(arr, Args: arr: An int32 tensor of non-negative values. weights: If non-None, must be the same shape as arr. For each value in - `arr`, the bin will be incremented by the corresponding weight instead - of 1. + `arr`, the bin will be incremented by the corresponding weight instead of + 1. minlength: If given, ensures the output has length at least `minlength`, - padding with zeros at the end if necessary. + padding with zeros at the end if necessary. maxlength: If given, skips values in `arr` that are equal or greater than - `maxlength`, ensuring that the output has length at most `maxlength`. + `maxlength`, ensuring that the output has length at most `maxlength`. + dtype: If `weights` is None, determines the type of the output bins. + name: A name scope for the associated operations (optional). + + Returns: + A vector with the same dtype as `weights` or the given `dtype`. The bin + values. + """ + name = "bincount" if name is None else name + with ops.name_scope(name): + arr = ops.convert_to_tensor(arr, name="arr", dtype=dtypes.int32) + array_is_nonempty = reduce_prod(array_ops.shape(arr)) > 0 + output_size = cast(array_is_nonempty, dtypes.int32) * (reduce_max(arr) + 1) + if minlength is not None: + minlength = ops.convert_to_tensor( + minlength, name="minlength", dtype=dtypes.int32) + output_size = gen_math_ops.maximum(minlength, output_size) + if maxlength is not None: + maxlength = ops.convert_to_tensor( + maxlength, name="maxlength", dtype=dtypes.int32) + output_size = gen_math_ops.minimum(maxlength, output_size) + if weights is not None: + weights = ops.convert_to_tensor(weights, name="weights") + return gen_math_ops.unsorted_segment_sum(weights, arr, output_size) + weights = constant_op.constant([], dtype) + return gen_math_ops.bincount(arr, output_size, weights) + + +@tf_export(v1=["math.bincount", "bincount"]) +@deprecation.deprecated_endpoints("bincount") +def bincount_v1(arr, + weights=None, + minlength=None, + maxlength=None, + dtype=dtypes.int32): + """Counts the number of occurrences of each value in an integer array. + + If `minlength` and `maxlength` are not given, returns a vector with length + `tf.reduce_max(arr) + 1` if `arr` is non-empty, and length 0 otherwise. + If `weights` are non-None, then index `i` of the output stores the sum of the + value in `weights` at each index where the corresponding value in `arr` is + `i`. + + Args: + arr: An int32 tensor of non-negative values. + weights: If non-None, must be the same shape as arr. For each value in + `arr`, the bin will be incremented by the corresponding weight instead of + 1. + minlength: If given, ensures the output has length at least `minlength`, + padding with zeros at the end if necessary. + maxlength: If given, skips values in `arr` that are equal or greater than + `maxlength`, ensuring that the output has length at most `maxlength`. dtype: If `weights` is None, determines the type of the output bins. Returns: A vector with the same dtype as `weights` or the given `dtype`. The bin values. """ - arr = ops.convert_to_tensor(arr, name="arr", dtype=dtypes.int32) - array_is_nonempty = reduce_prod(array_ops.shape(arr)) > 0 - output_size = cast(array_is_nonempty, dtypes.int32) * (reduce_max(arr) + 1) - if minlength is not None: - minlength = ops.convert_to_tensor( - minlength, name="minlength", dtype=dtypes.int32) - output_size = gen_math_ops.maximum(minlength, output_size) - if maxlength is not None: - maxlength = ops.convert_to_tensor( - maxlength, name="maxlength", dtype=dtypes.int32) - output_size = gen_math_ops.minimum(maxlength, output_size) - if weights is not None: - weights = ops.convert_to_tensor(weights, name="weights") - return gen_math_ops.unsorted_segment_sum(weights, arr, output_size) - weights = constant_op.constant([], dtype) - return gen_math_ops.bincount(arr, output_size, weights) + return bincount(arr, weights, minlength, maxlength, dtype) @tf_export("math.cumsum", "cumsum") @@ -2923,8 +3166,7 @@ def unsorted_segment_sqrt_n(data, segment_ids, num_segments, name=None): return summed / gen_math_ops.sqrt(N) -@tf_export( - "sparse.segment_sum", v1=["sparse.segment_sum", "sparse_segment_sum"]) +@tf_export(v1=["sparse.segment_sum", "sparse_segment_sum"]) @deprecation.deprecated_endpoints("sparse_segment_sum") def sparse_segment_sum(data, indices, segment_ids, name=None, num_segments=None): @@ -2998,8 +3240,17 @@ def sparse_segment_sum(data, indices, segment_ids, name=None, data=data, indices=indices, segment_ids=segment_ids, name=name) -@tf_export( - "sparse.segment_mean", v1=["sparse.segment_mean", "sparse_segment_mean"]) +@tf_export("sparse.segment_sum", v1=[]) +def sparse_segment_sum_v2(data, + indices, + segment_ids, + num_segments=None, + name=None): + return sparse_segment_mean( + data, indices, segment_ids, name=name, num_segments=num_segments) + + +@tf_export(v1=["sparse.segment_mean", "sparse_segment_mean"]) @deprecation.deprecated_endpoints("sparse_segment_mean") def sparse_segment_mean(data, indices, @@ -3045,9 +3296,44 @@ def sparse_segment_mean(data, data=data, indices=indices, segment_ids=segment_ids, name=name) -@tf_export( - "sparse.segment_sqrt_n", - v1=["sparse.segment_sqrt_n", "sparse_segment_sqrt_n"]) +@tf_export("sparse.segment_mean", v1=[]) +def sparse_segment_mean_v2(data, + indices, + segment_ids, + num_segments=None, + name=None): + r"""Computes the mean along sparse segments of a tensor. + + Read [the section on + segmentation](https://tensorflow.org/api_guides/python/math_ops#Segmentation) + for an explanation of segments. + + Like `SegmentMean`, but `segment_ids` can have rank less than `data`'s first + dimension, selecting a subset of dimension 0, specified by `indices`. + `segment_ids` is allowed to have missing ids, in which case the output will + be zeros at those indices. In those cases `num_segments` is used to determine + the size of the output. + + Args: + data: A `Tensor` with data that will be assembled in the output. + indices: A 1-D `Tensor` with indices into `data`. Has same rank as + `segment_ids`. + segment_ids: A 1-D `Tensor` with indices into the output `Tensor`. Values + should be sorted and can be repeated. + num_segments: An optional int32 scalar. Indicates the size of the output + `Tensor`. + name: A name for the operation (optional). + + Returns: + A `tensor` of the shape as data, except for dimension 0 which + has size `k`, the number of segments specified via `num_segments` or + inferred for the last element in `segments_ids`. + """ + return sparse_segment_mean( + data, indices, segment_ids, name=name, num_segments=num_segments) + + +@tf_export(v1=["sparse.segment_sqrt_n", "sparse_segment_sqrt_n"]) @deprecation.deprecated_endpoints("sparse_segment_sqrt_n") def sparse_segment_sqrt_n(data, indices, @@ -3085,6 +3371,35 @@ def sparse_segment_sqrt_n(data, data=data, indices=indices, segment_ids=segment_ids, name=name) +@tf_export("sparse.segment_sqrt_n", v1=[]) +def sparse_segment_sqrt_n_v2(data, + indices, + segment_ids, + num_segments=None, + name=None): + r"""Computes the sum along sparse segments of a tensor divided by the sqrt(N). + + `N` is the size of the segment being reduced. + + Args: + data: A `Tensor` with data that will be assembled in the output. + indices: A 1-D `Tensor` with indices into `data`. Has same rank as + `segment_ids`. + segment_ids: A 1-D `Tensor` with indices into the output `Tensor`. Values + should be sorted and can be repeated. + num_segments: An optional int32 scalar. Indicates the size of the output + `Tensor`. + name: A name for the operation (optional). + + Returns: + A `tensor` of the shape as data, except for dimension 0 which + has size `k`, the number of segments specified via `num_segments` or + inferred for the last element in `segments_ids`. + """ + return sparse_segment_sqrt_n( + data, indices, segment_ids, name=name, num_segments=num_segments) + + @tf_export("tensordot", "linalg.tensordot") def tensordot(a, b, axes, name=None): r"""Tensor contraction of a and b along specified axes. @@ -3118,12 +3433,11 @@ def tensordot(a, b, axes, name=None): a: `Tensor` of type `float32` or `float64`. b: `Tensor` with the same type as `a`. axes: Either a scalar `N`, or a list or an `int32` `Tensor` of shape [2, k]. - If axes is a scalar, sum over the last N axes of a and the first N axes - of b in order. - If axes is a list or `Tensor` the first and second row contain the set of - unique integers specifying axes along which the contraction is computed, - for `a` and `b`, respectively. The number of axes for `a` and `b` must - be equal. + If axes is a scalar, sum over the last N axes of a and the first N axes of + b in order. If axes is a list or `Tensor` the first and second row contain + the set of unique integers specifying axes along which the contraction is + computed, for `a` and `b`, respectively. The number of axes for `a` and + `b` must be equal. name: A name for the operation (optional). Returns: @@ -3295,73 +3609,3 @@ def polyval(coeffs, x, name=None): for c in coeffs[1:]: p = c + p * x return p - - -@tf_export("math.bessel_i0e") -def bessel_i0e(x, name=None): - """Computes the Bessel i0e function of `x` element-wise. - - Exponentially scaled modified Bessel function of order 0 defined as - `bessel_i0e(x) = exp(-abs(x)) bessel_i0(x)`. - - This function is faster and numerically stabler than `bessel_i0(x)`. - - Args: - x: A `Tensor` or `SparseTensor`. Must be one of the following types: `half`, - `float32`, `float64`. - name: A name for the operation (optional). - - Returns: - A `Tensor` or `SparseTensor`, respectively. Has the same type as `x`. - - @compatibility(scipy) - Equivalent to scipy.special.i0e - @end_compatibility - """ - with ops.name_scope(name, "bessel_i0e", [x]) as name: - if isinstance(x, sparse_tensor.SparseTensor): - x_i0e = gen_math_ops.bessel_i0e(x.values, name=name) - return sparse_tensor.SparseTensor( - indices=x.indices, values=x_i0e, dense_shape=x.dense_shape) - else: - return gen_math_ops.bessel_i0e(x, name=name) - - -@tf_export("math.bessel_i1e") -def bessel_i1e(x, name=None): - """Computes the Bessel i1e function of `x` element-wise. - - Exponentially scaled modified Bessel function of order 1 defined as - `bessel_i1e(x) = exp(-abs(x)) bessel_i1(x)`. - - This function is faster and numerically stabler than `bessel_i1(x)`. - - Args: - x: A `Tensor` or `SparseTensor`. Must be one of the following types: `half`, - `float32`, `float64`. - name: A name for the operation (optional). - - Returns: - A `Tensor` or `SparseTensor`, respectively. Has the same type as `x`. - - @compatibility(scipy) - Equivalent to scipy.special.i1e - @end_compatibility - """ - with ops.name_scope(name, "bessel_i1e", [x]) as name: - if isinstance(x, sparse_tensor.SparseTensor): - x_i1e = gen_math_ops.bessel_i1e(x.values, name=name) - return sparse_tensor.SparseTensor( - indices=x.indices, values=x_i1e, dense_shape=x.dense_shape) - else: - return gen_math_ops.bessel_i1e(x, name=name) - - -# FFT ops were moved to tf.spectral. tf.fft symbols were part of the TensorFlow -# 1.0 API so we leave these here for backwards compatibility. -fft = gen_spectral_ops.fft -ifft = gen_spectral_ops.ifft -fft2d = gen_spectral_ops.fft2d -ifft2d = gen_spectral_ops.ifft2d -fft3d = gen_spectral_ops.fft3d -ifft3d = gen_spectral_ops.ifft3d diff --git a/tensorflow/python/ops/math_ops_test.py b/tensorflow/python/ops/math_ops_test.py index a4da0c6c339..add1621a56b 100644 --- a/tensorflow/python/ops/math_ops_test.py +++ b/tensorflow/python/ops/math_ops_test.py @@ -92,6 +92,7 @@ class ReduceTest(test_util.TensorFlowTestCase): class LogSumExpTest(test_util.TensorFlowTestCase): + @test_util.run_deprecated_v1 def testReduceLogSumExp(self): for dtype in [np.float16, np.float32, np.double]: x_np = np.random.rand(5, 5).astype(dtype) @@ -104,22 +105,23 @@ class LogSumExpTest(test_util.TensorFlowTestCase): for dtype in [np.float16, np.float32, np.double]: x_np = np.random.rand(5, 5).astype(dtype) with self.cached_session(use_gpu=True): - y_tf = math_ops.reduce_logsumexp(x_np, reduction_indices=[0]) + y_tf = math_ops.reduce_logsumexp(x_np, axis=[0]) y_np = log(np.sum(exp(x_np), axis=0)) self.assertShapeEqual(y_np, y_tf) - y_tf_np = y_tf.eval() + y_tf_np = self.evaluate(y_tf) self.assertAllClose(y_tf_np, y_np) def testReductionIndices2(self): for dtype in [np.float16, np.float32, np.double]: x_np = np.random.rand(5, 5).astype(dtype) with self.cached_session(use_gpu=True): - y_tf = math_ops.reduce_logsumexp(x_np, reduction_indices=0) + y_tf = math_ops.reduce_logsumexp(x_np, axis=0) y_np = log(np.sum(exp(x_np), axis=0)) self.assertShapeEqual(y_np, y_tf) - y_tf_np = y_tf.eval() + y_tf_np = self.evaluate(y_tf) self.assertAllClose(y_tf_np, y_np) + @test_util.run_deprecated_v1 def testKeepDims(self): for dtype in [np.float16, np.float32, np.double]: x_np = np.random.rand(5, 5).astype(dtype) @@ -129,6 +131,7 @@ class LogSumExpTest(test_util.TensorFlowTestCase): y_np = log(np.sum(exp(x_np), keepdims=True)) self.assertAllClose(y_tf_np, y_np) + @test_util.run_deprecated_v1 def testOverflow(self): x = [1000, 1001, 1002, 1003] for dtype in [np.float16, np.float32, np.double]: @@ -146,6 +149,7 @@ class LogSumExpTest(test_util.TensorFlowTestCase): y_np = log(np.sum(exp(x_np - max_np))) + max_np self.assertAllClose(y_tf_np, y_np) + @test_util.run_deprecated_v1 def testUnderflow(self): x = [-1000, -1001, -1002, -1003] for dtype in [np.float16, np.float32, np.double]: @@ -163,6 +167,7 @@ class LogSumExpTest(test_util.TensorFlowTestCase): y_np = log(np.sum(exp(x_np - max_np))) + max_np self.assertAllClose(y_tf_np, y_np) + @test_util.run_deprecated_v1 def testInfinity(self): with self.session(use_gpu=True): res = math_ops.reduce_logsumexp(-np.inf).eval() @@ -186,6 +191,7 @@ class RoundTest(test_util.TensorFlowTestCase): class ModTest(test_util.TensorFlowTestCase): + @test_util.run_deprecated_v1 def testFloat(self): x = [0.5, 0.7, 0.3] for dtype in [np.float32, np.double]: @@ -195,7 +201,7 @@ class ModTest(test_util.TensorFlowTestCase): with self.cached_session(use_gpu=True): x_tf = constant_op.constant(x_np, shape=x_np.shape) y_tf = math_ops.mod(x_tf, denom) - y_tf_np = y_tf.eval() + y_tf_np = self.evaluate(y_tf) y_np = np.fmod(x_np, denom) self.assertAllClose(y_tf_np, y_np, atol=1e-2) @@ -208,7 +214,7 @@ class ModTest(test_util.TensorFlowTestCase): with self.cached_session(use_gpu=True): x_tf = constant_op.constant(x_np, shape=x_np.shape) y_tf = math_ops.mod(x_tf, denom) - y_tf_np = y_tf.eval() + y_tf_np = self.evaluate(y_tf) y_np = np.mod(x_np, denom) self.assertAllClose(y_tf_np, y_np) @@ -256,6 +262,7 @@ class ApproximateEqualTest(test_util.TensorFlowTestCase): z_tf = self.evaluate(math_ops.approximate_equal(x, y, tolerance=0.0001)) self.assertAllEqual(z, z_tf) + @test_util.run_deprecated_v1 def testApproximateEqualShape(self): for dtype in [np.float32, np.double]: x = np.array([1, 2], dtype=dtype) @@ -309,6 +316,7 @@ class ScalarMulTest(test_util.TensorFlowTestCase): class AccumulateNTest(test_util.TensorFlowTestCase): + @test_util.run_deprecated_v1 def testFloat(self): np.random.seed(12345) x = [np.random.random((1, 2, 3, 4, 5)) - 0.5 for _ in range(5)] @@ -317,6 +325,7 @@ class AccumulateNTest(test_util.TensorFlowTestCase): self.assertAllClose(sum(x), math_ops.accumulate_n(tf_x).eval()) self.assertAllClose(x[0] * 5, math_ops.accumulate_n([tf_x[0]] * 5).eval()) + @test_util.run_deprecated_v1 def testInt(self): np.random.seed(54321) x = [np.random.randint(-128, 128, (5, 4, 3, 2, 1)) for _ in range(6)] @@ -328,6 +337,7 @@ class AccumulateNTest(test_util.TensorFlowTestCase): class AddNTest(test_util.TensorFlowTestCase): + @test_util.run_deprecated_v1 def testPartials(self): """Test that previously revealed a bug in buffer forwarding for AddN.""" partials = [] @@ -341,6 +351,7 @@ class AddNTest(test_util.TensorFlowTestCase): with self.session(use_gpu=True): self.assertAllEqual(res.eval(), 100) + @test_util.run_deprecated_v1 def testFloat(self): np.random.seed(12345) for num_inputs in range(1, 10): @@ -351,6 +362,7 @@ class AddNTest(test_util.TensorFlowTestCase): self.assertAllClose(x[0] * num_inputs, math_ops.add_n([tf_x[0]] * num_inputs).eval()) + @test_util.run_deprecated_v1 def testInt(self): np.random.seed(54321) for num_inputs in range(1, 10): @@ -364,6 +376,7 @@ class AddNTest(test_util.TensorFlowTestCase): self.assertAllEqual(x[0] * num_inputs, math_ops.add_n([tf_x[0]] * num_inputs).eval()) + @test_util.run_deprecated_v1 def testGrad(self): np.random.seed(42) for num_inputs in range(1, 10): @@ -373,7 +386,7 @@ class AddNTest(test_util.TensorFlowTestCase): for i in range(0, num_inputs) ] addn = math_ops.add_n(input_vars) - sess.run(variables.global_variables_initializer()) + self.evaluate(variables.global_variables_initializer()) add_n_grad = gradients.gradients(addn, input_vars) self.assertAllEqual(np.repeat(1.0, num_inputs), # d/dx (x + y + ...) = 1 [g.eval() for g in add_n_grad]) @@ -392,6 +405,7 @@ class DivAndModTest(test_util.TensorFlowTestCase): divs = np.arange(-3, 0, .25).reshape(1, 12) return nums, divs + @test_util.run_deprecated_v1 def testFloorModInt(self): nums, divs = self.intTestData() with self.cached_session(): @@ -401,6 +415,7 @@ class DivAndModTest(test_util.TensorFlowTestCase): np_result = nums % divs self.assertAllEqual(tf_result, np_result) + @test_util.run_deprecated_v1 def testFloorModFloat(self): nums, divs = self.floatTestData() with self.cached_session(): @@ -412,6 +427,7 @@ class DivAndModTest(test_util.TensorFlowTestCase): # % array_ops.constant(divs)).eval() # self.assertAllEqual(tf2_result, tf_result) + @test_util.run_deprecated_v1 def testTruncateModInt(self): nums, divs = self.intTestData() with self.cached_session(): @@ -419,6 +435,7 @@ class DivAndModTest(test_util.TensorFlowTestCase): np_result = np.fmod(nums, divs) self.assertAllEqual(tf_result, np_result) + @test_util.run_deprecated_v1 def testTruncateModFloat(self): nums, divs = self.floatTestData() with self.cached_session(): @@ -426,6 +443,7 @@ class DivAndModTest(test_util.TensorFlowTestCase): np_result = np.fmod(nums, divs) self.assertAllEqual(tf_result, np_result) + @test_util.run_deprecated_v1 def testDivideInt(self): nums, divs = self.intTestData() with self.cached_session(): @@ -437,12 +455,14 @@ class DivAndModTest(test_util.TensorFlowTestCase): # // array_ops.constant(divs)).eval() # self.assertAllEqual(tf2_result, tf_result) + @test_util.run_deprecated_v1 def testDivideName(self): with self.cached_session(): op = math_ops.divide( array_ops.constant(3), array_ops.constant(4), name="my_cool_divide") self.assertEqual(op.name, "my_cool_divide:0") + @test_util.run_deprecated_v1 def testRealDiv(self): nums, divs = self.floatTestData() with self.cached_session(): @@ -450,26 +470,30 @@ class DivAndModTest(test_util.TensorFlowTestCase): np_result = np.divide(nums, divs) self.assertAllEqual(tf_result, np_result) + @test_util.run_deprecated_v1 def testComplexDiv(self): foo = array_ops.constant([1. + 3.j]) with self.cached_session(): _ = math_ops.divide(foo, 1.).eval() _ = math_ops.div(foo, 2.).eval() + @test_util.run_deprecated_v1 def testFloorDivGrad(self): with self.cached_session(): a = variables.Variable(2.) b = variables.Variable(4.) with self.cached_session() as sess: - sess.run(variables.global_variables_initializer()) + self.evaluate(variables.global_variables_initializer()) c_grad = gradients.gradients(math_ops.divide(a, b), [a, b]) self.assertAllEqual([x.eval() for x in c_grad], [.25, -.125]) c_grad = gradients.gradients(math_ops.div(a, b), [a, b]) self.assertAllEqual([x.eval() for x in c_grad], [.25, -.125]) c_grad = gradients.gradients(math_ops.floordiv(a, b), [a, b]) - self.assertAllEqual([None if x is None else x.eval() - for x in c_grad], [None, None]) + self.assertAllEqual( + [None if x is None else self.evaluate(x) for x in c_grad], + [None, None]) + @test_util.run_deprecated_v1 def testConsistent(self): nums, divs = self.intTestData() with self.cached_session(): @@ -496,6 +520,7 @@ class DivAndModTest(test_util.TensorFlowTestCase): class DivNoNanTest(test_util.TensorFlowTestCase): + @test_util.run_deprecated_v1 def testBasic(self): for dtype in [np.float32, np.float64]: nums = np.arange(-10, 10, .25, dtype=dtype).reshape(80, 1) diff --git a/tensorflow/python/ops/metrics_impl.py b/tensorflow/python/ops/metrics_impl.py index e86a3b85360..cb421990112 100644 --- a/tensorflow/python/ops/metrics_impl.py +++ b/tensorflow/python/ops/metrics_impl.py @@ -18,7 +18,6 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function -from tensorflow.python.compat import compat from tensorflow.python.eager import context from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops @@ -213,26 +212,6 @@ def _maybe_expand_labels(labels, predictions): lambda: array_ops.expand_dims(labels, -1, name=scope), lambda: labels) -def _safe_div(numerator, denominator, name): - """Divides two tensors element-wise, returning 0 if the denominator is <= 0. - - Args: - numerator: A real `Tensor`. - denominator: A real `Tensor`, with dtype matching `numerator`. - name: Name for the returned op. - - Returns: - 0 if `denominator` <= 0, else `numerator` / `denominator` - """ - if compat.forward_compatible(2018, 11, 1): - return math_ops.div_no_nan(numerator, denominator, name=name) - t = math_ops.truediv(numerator, denominator) - zero = array_ops.zeros_like(t, dtype=denominator.dtype) - condition = math_ops.greater(denominator, zero) - zero = math_ops.cast(zero, t.dtype) - return array_ops.where(condition, t, zero, name=name) - - def _safe_scalar_div(numerator, denominator, name): """Divides two values, returning 0 if the denominator is 0. @@ -246,7 +225,7 @@ def _safe_scalar_div(numerator, denominator, name): """ numerator.get_shape().with_rank_at_most(1) denominator.get_shape().with_rank_at_most(1) - return _safe_div(numerator, denominator, name=name) + return math_ops.div_no_nan(numerator, denominator, name=name) def _streaming_confusion_matrix(labels, predictions, num_classes, weights=None): @@ -302,7 +281,7 @@ def _aggregate_across_replicas(metrics_collections, metric_value_fn, *args): """Aggregate metric value across replicas.""" def fn(distribution, *a): """Call `metric_value_fn` in the correct control flow context.""" - if hasattr(distribution, '_outer_control_flow_context'): + if hasattr(distribution.extended, '_outer_control_flow_context'): # If there was an outer context captured before this method was called, # then we enter that context to create the metric value op. If the # caputred context is `None`, ops.control_dependencies(None) gives the @@ -315,13 +294,13 @@ def _aggregate_across_replicas(metrics_collections, metric_value_fn, *args): # once the update ops have been evaluted. # pylint: disable=protected-access - if distribution._outer_control_flow_context is None: + if distribution.extended._outer_control_flow_context is None: with ops.control_dependencies(None): metric_value = metric_value_fn(distribution, *a) else: - distribution._outer_control_flow_context.Enter() + distribution.extended._outer_control_flow_context.Enter() metric_value = metric_value_fn(distribution, *a) - distribution._outer_control_flow_context.Exit() + distribution.extended._outer_control_flow_context.Exit() # pylint: enable=protected-access else: metric_value = metric_value_fn(distribution, *a) @@ -330,10 +309,10 @@ def _aggregate_across_replicas(metrics_collections, metric_value_fn, *args): return metric_value return distribution_strategy_context.get_replica_context().merge_call( - fn, *args) + fn, args=args) -@tf_export('metrics.mean') +@tf_export(v1=['metrics.mean']) def mean(values, weights=None, metrics_collections=None, @@ -401,13 +380,12 @@ def mean(values, update_count_op = state_ops.assign_add(count, num_values) def compute_mean(_, t, c): - return _safe_div(t, math_ops.maximum(c, 0), name='value') + return math_ops.div_no_nan(t, math_ops.maximum(c, 0), name='value') mean_t = _aggregate_across_replicas( metrics_collections, compute_mean, total, count) - update_op = _safe_div(update_total_op, - math_ops.maximum(update_count_op, 0), - name='update_op') + update_op = math_ops.div_no_nan( + update_total_op, math_ops.maximum(update_count_op, 0), name='update_op') if updates_collections: ops.add_to_collections(updates_collections, update_op) @@ -415,7 +393,7 @@ def mean(values, return mean_t, update_op -@tf_export('metrics.accuracy') +@tf_export(v1=['metrics.accuracy']) def accuracy(labels, predictions, weights=None, @@ -647,7 +625,7 @@ def _aggregate_variable(v, collections): return _aggregate_across_replicas(collections, f, v) -@tf_export('metrics.auc') +@tf_export(v1=['metrics.auc']) def auc(labels, predictions, weights=None, @@ -779,19 +757,19 @@ def auc(labels, """ dtp = tp[:num_thresholds - 1] - tp[1:] p = tp + fp - prec_slope = _safe_div( + prec_slope = math_ops.div_no_nan( dtp, math_ops.maximum(p[:num_thresholds - 1] - p[1:], 0), name='prec_slope') intercept = tp[1:] - math_ops.multiply(prec_slope, p[1:]) safe_p_ratio = array_ops.where( math_ops.logical_and(p[:num_thresholds - 1] > 0, p[1:] > 0), - _safe_div(p[:num_thresholds - 1], - math_ops.maximum(p[1:], 0), - name='recall_relative_ratio'), - array_ops.ones_like(p[1:])) + math_ops.div_no_nan( + p[:num_thresholds - 1], + math_ops.maximum(p[1:], 0), + name='recall_relative_ratio'), array_ops.ones_like(p[1:])) return math_ops.reduce_sum( - _safe_div( + math_ops.div_no_nan( prec_slope * (dtp + intercept * math_ops.log(safe_p_ratio)), math_ops.maximum(tp[1:] + fn[1:], 0), name='pr_auc_increment'), @@ -852,7 +830,7 @@ def auc(labels, return auc_value, update_op -@tf_export('metrics.mean_absolute_error') +@tf_export(v1=['metrics.mean_absolute_error']) def mean_absolute_error(labels, predictions, weights=None, @@ -913,7 +891,7 @@ def mean_absolute_error(labels, updates_collections, name or 'mean_absolute_error') -@tf_export('metrics.mean_cosine_distance') +@tf_export(v1=['metrics.mean_cosine_distance']) def mean_cosine_distance(labels, predictions, dim, @@ -970,7 +948,7 @@ def mean_cosine_distance(labels, predictions=predictions, labels=labels, weights=weights) radial_diffs = math_ops.multiply(predictions, labels) radial_diffs = math_ops.reduce_sum( - radial_diffs, reduction_indices=[ + radial_diffs, axis=[ dim, ], keepdims=True) mean_distance, update_op = mean(radial_diffs, weights, None, None, name or @@ -987,7 +965,7 @@ def mean_cosine_distance(labels, return mean_distance, update_op -@tf_export('metrics.mean_per_class_accuracy') +@tf_export(v1=['metrics.mean_per_class_accuracy']) def mean_per_class_accuracy(labels, predictions, num_classes, @@ -1074,7 +1052,7 @@ def mean_per_class_accuracy(labels, update_count_op = state_ops.scatter_add(count, labels, is_correct) def compute_mean_accuracy(_, count, total): - per_class_accuracy = _safe_div( + per_class_accuracy = math_ops.div_no_nan( count, math_ops.maximum(total, 0), name=None) mean_accuracy_v = math_ops.reduce_mean( per_class_accuracy, name='mean_accuracy') @@ -1083,16 +1061,15 @@ def mean_per_class_accuracy(labels, mean_accuracy_v = _aggregate_across_replicas( metrics_collections, compute_mean_accuracy, count, total) - update_op = _safe_div(update_count_op, - math_ops.maximum(update_total_op, 0), - name='update_op') + update_op = math_ops.div_no_nan( + update_count_op, math_ops.maximum(update_total_op, 0), name='update_op') if updates_collections: ops.add_to_collections(updates_collections, update_op) return mean_accuracy_v, update_op -@tf_export('metrics.mean_iou') +@tf_export(v1=['metrics.mean_iou']) def mean_iou(labels, predictions, num_classes, @@ -1193,7 +1170,7 @@ def mean_iou(labels, return mean_iou_v, update_op -@tf_export('metrics.mean_relative_error') +@tf_export(v1=['metrics.mean_relative_error']) def mean_relative_error(labels, predictions, normalizer, @@ -1262,7 +1239,7 @@ def mean_relative_error(labels, updates_collections, name or 'mean_relative_error') -@tf_export('metrics.mean_squared_error') +@tf_export(v1=['metrics.mean_squared_error']) def mean_squared_error(labels, predictions, weights=None, @@ -1323,7 +1300,7 @@ def mean_squared_error(labels, name or 'mean_squared_error') -@tf_export('metrics.mean_tensor') +@tf_export(v1=['metrics.mean_tensor']) def mean_tensor(values, weights=None, metrics_collections=None, @@ -1394,22 +1371,21 @@ def mean_tensor(values, with ops.control_dependencies([values]): update_count_op = state_ops.assign_add(count, num_values) - compute_mean = lambda _, t, c: _safe_div( + compute_mean = lambda _, t, c: math_ops.div_no_nan( # pylint: disable=g-long-lambda t, math_ops.maximum(c, 0), name='value') mean_t = _aggregate_across_replicas( metrics_collections, compute_mean, total, count) - update_op = _safe_div(update_total_op, - math_ops.maximum(update_count_op, 0), - name='update_op') + update_op = math_ops.div_no_nan( + update_total_op, math_ops.maximum(update_count_op, 0), name='update_op') if updates_collections: ops.add_to_collections(updates_collections, update_op) return mean_t, update_op -@tf_export('metrics.percentage_below') +@tf_export(v1=['metrics.percentage_below']) def percentage_below(values, threshold, weights=None, @@ -1509,7 +1485,7 @@ def _count_condition(values, return value_tensor, update_op -@tf_export('metrics.false_negatives') +@tf_export(v1=['metrics.false_negatives']) def false_negatives(labels, predictions, weights=None, @@ -1561,7 +1537,7 @@ def false_negatives(labels, updates_collections) -@tf_export('metrics.false_negatives_at_thresholds') +@tf_export(v1=['metrics.false_negatives_at_thresholds']) def false_negatives_at_thresholds(labels, predictions, thresholds, @@ -1617,7 +1593,7 @@ def false_negatives_at_thresholds(labels, return fn_value, update_ops['fn'] -@tf_export('metrics.false_positives') +@tf_export(v1=['metrics.false_positives']) def false_positives(labels, predictions, weights=None, @@ -1670,7 +1646,7 @@ def false_positives(labels, updates_collections) -@tf_export('metrics.false_positives_at_thresholds') +@tf_export(v1=['metrics.false_positives_at_thresholds']) def false_positives_at_thresholds(labels, predictions, thresholds, @@ -1726,7 +1702,7 @@ def false_positives_at_thresholds(labels, return fp_value, update_ops['fp'] -@tf_export('metrics.true_negatives') +@tf_export(v1=['metrics.true_negatives']) def true_negatives(labels, predictions, weights=None, @@ -1779,7 +1755,7 @@ def true_negatives(labels, updates_collections) -@tf_export('metrics.true_negatives_at_thresholds') +@tf_export(v1=['metrics.true_negatives_at_thresholds']) def true_negatives_at_thresholds(labels, predictions, thresholds, @@ -1835,7 +1811,7 @@ def true_negatives_at_thresholds(labels, return tn_value, update_ops['tn'] -@tf_export('metrics.true_positives') +@tf_export(v1=['metrics.true_positives']) def true_positives(labels, predictions, weights=None, @@ -1888,7 +1864,7 @@ def true_positives(labels, updates_collections) -@tf_export('metrics.true_positives_at_thresholds') +@tf_export(v1=['metrics.true_positives_at_thresholds']) def true_positives_at_thresholds(labels, predictions, thresholds, @@ -1944,7 +1920,7 @@ def true_positives_at_thresholds(labels, return tp_value, update_ops['tp'] -@tf_export('metrics.precision') +@tf_export(v1=['metrics.precision']) def precision(labels, predictions, weights=None, @@ -2039,7 +2015,7 @@ def precision(labels, return p, update_op -@tf_export('metrics.precision_at_thresholds') +@tf_export(v1=['metrics.precision_at_thresholds']) def precision_at_thresholds(labels, predictions, thresholds, @@ -2120,7 +2096,7 @@ def precision_at_thresholds(labels, return prec, update_op -@tf_export('metrics.recall') +@tf_export(v1=['metrics.recall']) def recall(labels, predictions, weights=None, @@ -2471,7 +2447,7 @@ def _streaming_sparse_false_negative_at_k(labels, return var, state_ops.assign_add(var, batch_total_fn, name='update') -@tf_export('metrics.recall_at_k') +@tf_export(v1=['metrics.recall_at_k']) def recall_at_k(labels, predictions, k, @@ -2564,7 +2540,7 @@ def recall_at_k(labels, name=scope) -@tf_export('metrics.recall_at_top_k') +@tf_export(v1=['metrics.recall_at_top_k']) def recall_at_top_k(labels, predictions_idx, k=None, @@ -2648,7 +2624,7 @@ def recall_at_top_k(labels, return metric, update -@tf_export('metrics.recall_at_thresholds') +@tf_export(v1=['metrics.recall_at_thresholds']) def recall_at_thresholds(labels, predictions, thresholds, @@ -2726,7 +2702,7 @@ def recall_at_thresholds(labels, return rec, update_op -@tf_export('metrics.root_mean_squared_error') +@tf_export(v1=['metrics.root_mean_squared_error']) def root_mean_squared_error(labels, predictions, weights=None, @@ -2797,7 +2773,7 @@ def root_mean_squared_error(labels, return rmse, update_rmse_op -@tf_export('metrics.sensitivity_at_specificity') +@tf_export(v1=['metrics.sensitivity_at_specificity']) def sensitivity_at_specificity(labels, predictions, specificity, @@ -3069,7 +3045,7 @@ def _sparse_average_precision_at_top_k(labels, predictions_idx): # Reduce along k dimension to get the sum, yielding a [D1, ... DN] tensor. precision_sum = math_ops.reduce_sum( - relevant_precision_per_k, reduction_indices=(-1,), name='precision_sum') + relevant_precision_per_k, axis=(-1,), name='precision_sum') # Divide by number of relevant items to get average precision. These are # the "num_relevant_items" and "AveP" terms from the formula above. @@ -3170,7 +3146,7 @@ def _streaming_sparse_average_precision_at_top_k(labels, return mean_average_precision, update -@tf_export('metrics.sparse_average_precision_at_k') +@tf_export(v1=['metrics.sparse_average_precision_at_k']) @deprecated(None, 'Use average_precision_at_k instead') def sparse_average_precision_at_k(labels, predictions, @@ -3190,7 +3166,7 @@ def sparse_average_precision_at_k(labels, name=name) -@tf_export('metrics.average_precision_at_k') +@tf_export(v1=['metrics.average_precision_at_k']) def average_precision_at_k(labels, predictions, k, @@ -3364,7 +3340,7 @@ def _streaming_sparse_false_positive_at_k(labels, return var, state_ops.assign_add(var, batch_total_fp, name='update') -@tf_export('metrics.precision_at_top_k') +@tf_export(v1=['metrics.precision_at_top_k']) def precision_at_top_k(labels, predictions_idx, k=None, @@ -3453,7 +3429,7 @@ def precision_at_top_k(labels, return metric, update -@tf_export('metrics.sparse_precision_at_k') +@tf_export(v1=['metrics.sparse_precision_at_k']) @deprecated(None, 'Use precision_at_k instead') def sparse_precision_at_k(labels, predictions, @@ -3475,7 +3451,7 @@ def sparse_precision_at_k(labels, name=name) -@tf_export('metrics.precision_at_k') +@tf_export(v1=['metrics.precision_at_k']) def precision_at_k(labels, predictions, k, @@ -3569,7 +3545,7 @@ def precision_at_k(labels, name=scope) -@tf_export('metrics.specificity_at_sensitivity') +@tf_export(v1=['metrics.specificity_at_sensitivity']) def specificity_at_sensitivity(labels, predictions, sensitivity, diff --git a/tensorflow/python/ops/nccl_ops_test.py b/tensorflow/python/ops/nccl_ops_test.py index 1b496fec473..3b2e2b0175f 100644 --- a/tensorflow/python/ops/nccl_ops_test.py +++ b/tensorflow/python/ops/nccl_ops_test.py @@ -102,7 +102,7 @@ class NcclTestCase(test.TestCase): continue # Test execution and results. - for t in sess.run(result_tensors): + for t in self.evaluate(result_tensors): self.assertAllClose(t, np_ans) def _TestGradient(self, nccl_reduce, numpy_fn): diff --git a/tensorflow/python/ops/nn_batchnorm_test.py b/tensorflow/python/ops/nn_batchnorm_test.py index c8a5b58e458..e978f1d3260 100644 --- a/tensorflow/python/ops/nn_batchnorm_test.py +++ b/tensorflow/python/ops/nn_batchnorm_test.py @@ -71,6 +71,7 @@ class BatchNormalizationTest(test.TestCase): gamma if scale_after_normalization else None, epsilon) + @test_util.run_deprecated_v1 def testBatchNorm(self): x_shape = [3, 5, 4, 2] param_shape = [2] @@ -169,16 +170,20 @@ class BatchNormalizationTest(test.TestCase): shift_after_normalization, v, err_tolerance) + @test_util.run_deprecated_v1 def testBatchNormInputGradient(self): self._testBatchNormGradientInAllNeedConfigs(0, "x") + @test_util.run_deprecated_v1 def testBatchNormMeanGradient(self): self._testBatchNormGradientInAllNeedConfigs(1, "mean") + @test_util.run_deprecated_v1 def testBatchNormVarianceGradient(self): self._testBatchNormGradientInAllNeedConfigs( 2, "variance", err_tolerance=1e-03) + @test_util.run_deprecated_v1 def testBatchNormBetaGradient(self): # Since beta does not exist when scale_after_normalization=False, we only # test for scale_after_normalization=True. @@ -187,6 +192,7 @@ class BatchNormalizationTest(test.TestCase): self._testBatchNormGradient(3, "beta", scale_after_normalization, True, v) + @test_util.run_deprecated_v1 def testBatchNormGammaGradient(self): # If scale_after_normalization is False, backprop for gamma in v1 # will be 0. In version 2 of the API, if scale_after_normalization is False, @@ -199,6 +205,7 @@ class BatchNormalizationTest(test.TestCase): self._testBatchNormGradient(4, "gamma", True, shift_after_normalization, 2) + @test_util.run_deprecated_v1 def testBatchNormGradImpl(self): x_shape = [7, 5, 4, 6] param_shape = [6] @@ -235,15 +242,17 @@ class BatchNormalizationTest(test.TestCase): odx, odm, odv, odb, odg = gradients_impl.gradients( [on], [x, m, v, beta, gamma], [backprop]) if scale_after_normalization: - all_grads = sess.run([dx, dm, dv, db, dg, odx, odm, odv, odb, odg]) + all_grads = self.evaluate( + [dx, dm, dv, db, dg, odx, odm, odv, odb, odg]) to_check = ["dx", "dm", "dv", "db", "dg"] else: - all_grads = sess.run([dx, dm, dv, db, odx, odm, odv, odb]) + all_grads = self.evaluate([dx, dm, dv, db, odx, odm, odv, odb]) to_check = ["dx", "dm", "dv", "db"] for i, _ in enumerate(to_check): self.assertAllClose( all_grads[i + len(to_check)], all_grads[i], atol=0.000001) + @test_util.run_deprecated_v1 def testBatchNormKeepDims(self): """Test for tf.nn.moments(..., keep_dims=True / False). @@ -318,7 +327,7 @@ class BatchNormalizationTest(test.TestCase): gamma_val, epsilon, scale_after_normalization, shift_after_normalization) - [tf_batch_norm] = sess.run([bn]) + [tf_batch_norm] = self.evaluate([bn]) self.assertEquals(x_shape, np_batch_norm.shape) self.assertEquals(x_shape, tf_batch_norm.shape) self.assertAllClose(np_batch_norm, tf_batch_norm, atol=atol) @@ -371,9 +380,9 @@ class SufficientStatisticsTest(test.TestCase): x.set_shape(x_shape) op_c, op_m, op_v, op_s = self._opSuffStats(x, axes, shift, keep_dims) if shift: - tf_c, tf_m, tf_v, tf_s = sess.run([op_c, op_m, op_v, op_s]) + tf_c, tf_m, tf_v, tf_s = self.evaluate([op_c, op_m, op_v, op_s]) else: - tf_c, tf_m, tf_v = sess.run([op_c, op_m, op_v]) + tf_c, tf_m, tf_v = self.evaluate([op_c, op_m, op_v]) else: x = array_ops.placeholder( dtype=dtypes.float32, shape=[None] * len(x_shape), name="x") @@ -390,6 +399,7 @@ class SufficientStatisticsTest(test.TestCase): if shift: self.assertAllClose(np_s, tf_s, atol=0.000001) + @test_util.run_deprecated_v1 def testSuffStats(self): for has_shape in [True, False]: for keep_dims in [True, False]: @@ -432,7 +442,7 @@ class NormalizeMomentsTest(test.TestCase): tf_shift_v = None opm, opv = self._opNormalizeMoments(tf_counts, tf_mean_ss, tf_variance_ss, tf_shift_v) - tfm, tfv = sess.run([opm, opv]) + tfm, tfv = self.evaluate([opm, opv]) self.assertAllClose(npm, tfm, atol=0.000001) self.assertAllClose(npv, tfv, atol=0.000001) @@ -507,9 +517,10 @@ class MomentsTest(test.TestCase): expected_variance = expected_x_squared - expected_mean_squared # Check that the moments are correct. - self.assertAllCloseAccordingToType(expected_mean, mean.eval()) - self.assertAllCloseAccordingToType(expected_variance, var.eval()) + self.assertAllCloseAccordingToType(expected_mean, self.evaluate(mean)) + self.assertAllCloseAccordingToType(expected_variance, self.evaluate(var)) + @test_util.run_deprecated_v1 def testBasic(self): for keep_dims in [False, True]: for dtype in [dtypes.float32, dtypes.float16]: @@ -518,6 +529,7 @@ class MomentsTest(test.TestCase): self.RunMomentTestWithDynamicShape( shape=[2, 3, 5, 4], axes=[0], keep_dims=keep_dims, dtype=dtype) + @test_util.run_deprecated_v1 def testGlobalNormalization(self): for keep_dims in [False, True]: for dtype in [dtypes.float32, dtypes.float16]: @@ -532,6 +544,7 @@ class MomentsTest(test.TestCase): keep_dims=keep_dims, dtype=dtype) + @test_util.run_deprecated_v1 def testAxes(self): for keep_dims in [False, True]: for dtype in [dtypes.float32, dtypes.float16]: @@ -572,9 +585,11 @@ class MomentsTest(test.TestCase): print("Moments %s gradient err vs input %d = %g" % (from_y, i, err)) self.assertLess(err, 1e-11) + @test_util.run_deprecated_v1 def testMeanGlobalGradient(self): self._testGlobalGradient(from_y="mean") + @test_util.run_deprecated_v1 def testVarGlobalGradient(self): self._testGlobalGradient(from_y="var") diff --git a/tensorflow/python/ops/nn_fused_batchnorm_test.py b/tensorflow/python/ops/nn_fused_batchnorm_test.py index 5ac8eba6f73..4bc33ff8bdb 100644 --- a/tensorflow/python/ops/nn_fused_batchnorm_test.py +++ b/tensorflow/python/ops/nn_fused_batchnorm_test.py @@ -50,7 +50,7 @@ class BatchNormalizationTest(test.TestCase): y = self._batch_norm(x, mean, var, offset, scale, epsilon) if data_format == 'NCHW': y = array_ops.transpose(y, [0, 3, 1, 2]) - return y.eval() + return self.evaluate(y) def _test_inference(self, x_shape, @@ -82,7 +82,7 @@ class BatchNormalizationTest(test.TestCase): epsilon=epsilon, data_format=data_format, is_training=False) - y_val = sess.run(y) + y_val = self.evaluate(y) y_ref = self._inference_ref(x, scale, offset, mean, var, epsilon, data_format) # An atol value of 1e-3 is too small for float16's, because some adjacent @@ -102,7 +102,7 @@ class BatchNormalizationTest(test.TestCase): y = self._batch_norm(x, mean, var, offset, scale, epsilon) if data_format == 'NCHW': y = array_ops.transpose(y, [0, 3, 1, 2]) - return y.eval(), mean.eval(), var.eval() + return self.evaluate(y), self.evaluate(mean), self.evaluate(var) def _test_training(self, x_shape, @@ -127,7 +127,7 @@ class BatchNormalizationTest(test.TestCase): epsilon=epsilon, data_format=data_format, is_training=True) - y_val, mean_val, var_val = sess.run([y, mean, var]) + y_val, mean_val, var_val = self.evaluate([y, mean, var]) y_ref, mean_ref, var_ref = self._training_ref(x, scale, offset, epsilon, data_format) y_atol = 2e-3 if x_dtype == np.float16 else 1e-3 @@ -277,10 +277,10 @@ class BatchNormalizationTest(test.TestCase): if is_training: epsilon = y.op.get_attr('epsilon') data_format = y.op.get_attr('data_format') - grad_vals = sess.run([grad_x, grad_scale, grad_offset]) + grad_vals = self.evaluate([grad_x, grad_scale, grad_offset]) grad_internal = nn_grad._BatchNormGrad(grad_y, x, scale, pop_mean, pop_var, epsilon, data_format) - grad_internal_vals = sess.run(list(grad_internal)) + grad_internal_vals = self.evaluate(list(grad_internal)) for grad_val, grad_internal_val in zip(grad_vals, grad_internal_vals): self.assertAllClose(grad_val, grad_internal_val, atol=err_tolerance) diff --git a/tensorflow/python/ops/nn_grad.py b/tensorflow/python/ops/nn_grad.py index 902653befc4..34404edc9a1 100644 --- a/tensorflow/python/ops/nn_grad.py +++ b/tensorflow/python/ops/nn_grad.py @@ -18,13 +18,13 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function +from tensorflow.python.eager import backprop from tensorflow.python.eager import context from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops from tensorflow.python.framework import tensor_util from tensorflow.python.ops import array_ops from tensorflow.python.ops import gen_nn_ops -from tensorflow.python.ops import gradients_impl from tensorflow.python.ops import math_ops from tensorflow.python.ops import nn_ops @@ -948,10 +948,14 @@ def _FusedBatchNormGradGrad(op, *grad): grad_grad_x = grad[0] grad_grad_scale = grad[1] grad_grad_offset = grad[2] - grad_x, grad_scale, grad_offset = _BatchNormGrad( - grad_y, x, scale, pop_mean, pop_var, epsilon, data_format, is_training) - grad_initial = [grad_grad_x, grad_grad_scale, grad_grad_offset] - grad_grad_y, grad_x, grad_scale = gradients_impl.gradients( + with backprop.GradientTape() as tape: + tape.watch(grad_y) + tape.watch(x) + tape.watch(scale) + grad_x, grad_scale, grad_offset = _BatchNormGrad( + grad_y, x, scale, pop_mean, pop_var, epsilon, data_format, is_training) + grad_initial = [grad_grad_x, grad_grad_scale, grad_grad_offset] + grad_grad_y, grad_x, grad_scale = tape.gradient( [grad_x, grad_scale, grad_offset], [grad_y, x, scale], grad_initial) return grad_grad_y, grad_x, grad_scale, None, None diff --git a/tensorflow/python/ops/nn_grad_test.py b/tensorflow/python/ops/nn_grad_test.py index 8065df4b165..95e05a977b8 100644 --- a/tensorflow/python/ops/nn_grad_test.py +++ b/tensorflow/python/ops/nn_grad_test.py @@ -22,6 +22,7 @@ import numpy as np from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes +from tensorflow.python.framework import test_util from tensorflow.python.ops import gradient_checker from tensorflow.python.ops import gradients_impl from tensorflow.python.ops import nn_grad # pylint: disable=unused-import @@ -31,6 +32,7 @@ from tensorflow.python.platform import test class Relu6OpTest(test.TestCase): + @test_util.run_deprecated_v1 def testRelu6GradGrad(self): inputs = constant_op.constant( [[-2, -1, 1, 3], [5, 7, 8, 9]], dtype=dtypes.float32) diff --git a/tensorflow/python/ops/nn_impl.py b/tensorflow/python/ops/nn_impl.py index ef763a4b614..8f74f831c1d 100644 --- a/tensorflow/python/ops/nn_impl.py +++ b/tensorflow/python/ops/nn_impl.py @@ -329,7 +329,7 @@ def swish(features): return features * math_ops.sigmoid(features) -@tf_export("math.l2_normalize", "linalg.l2_normalize", "nn.l2_normalize") +@tf_export(v1=["math.l2_normalize", "linalg.l2_normalize", "nn.l2_normalize"]) @deprecated_args(None, "dim is deprecated, use axis instead", "dim") def l2_normalize(x, axis=None, epsilon=1e-12, name=None, dim=None): """Normalizes along dimension `axis` using an L2 norm. @@ -350,11 +350,36 @@ def l2_normalize(x, axis=None, epsilon=1e-12, name=None, dim=None): name: A name for this operation (optional). dim: Deprecated alias for axis. + Returns: + A `Tensor` with the same shape as `x`. + """ + axis = deprecated_argument_lookup("axis", axis, "dim", dim) + return l2_normalize_v2(x, axis, epsilon, name) + + +@tf_export("math.l2_normalize", "linalg.l2_normalize", "nn.l2_normalize", v1=[]) +def l2_normalize_v2(x, axis=None, epsilon=1e-12, name=None): + """Normalizes along dimension `axis` using an L2 norm. + + For a 1-D tensor with `axis = 0`, computes + + output = x / sqrt(max(sum(x**2), epsilon)) + + For `x` with more dimensions, independently normalizes each 1-D slice along + dimension `axis`. + + Args: + x: A `Tensor`. + axis: Dimension along which to normalize. A scalar or a vector of + integers. + epsilon: A lower bound value for the norm. Will use `sqrt(epsilon)` as the + divisor if `norm < sqrt(epsilon)`. + name: A name for this operation (optional). + Returns: A `Tensor` with the same shape as `x`. """ with ops.name_scope(name, "l2_normalize", [x]) as name: - axis = deprecated_argument_lookup("axis", axis, "dim", dim) x = ops.convert_to_tensor(x, name="x") square_sum = math_ops.reduce_sum(math_ops.square(x), axis, keepdims=True) x_inv_norm = math_ops.rsqrt(math_ops.maximum(square_sum, epsilon)) @@ -424,7 +449,7 @@ def zero_fraction(value, name=None): # pylint: disable=redefined-builtin -@tf_export("nn.depthwise_conv2d") +@tf_export(v1=["nn.depthwise_conv2d"]) def depthwise_conv2d(input, filter, strides, @@ -497,11 +522,68 @@ def depthwise_conv2d(input, op=op) +@tf_export("nn.depthwise_conv2d", v1=[]) +def depthwise_conv2d_v2(input, + filter, + strides, + padding, + data_format=None, + dilations=None, + name=None): + """Depthwise 2-D convolution. + + Given a 4D input tensor ('NHWC' or 'NCHW' data formats) + and a filter tensor of shape + `[filter_height, filter_width, in_channels, channel_multiplier]` + containing `in_channels` convolutional filters of depth 1, `depthwise_conv2d` + applies a different filter to each input channel (expanding from 1 channel + to `channel_multiplier` channels for each), then concatenates the results + together. The output has `in_channels * channel_multiplier` channels. + + In detail, + + output[b, i, j, k * channel_multiplier + q] = sum_{di, dj} + filter[di, dj, k, q] * input[b, strides[1] * i + rate[0] * di, + strides[2] * j + rate[1] * dj, k] + + Must have `strides[0] = strides[3] = 1`. For the most common case of the + same horizontal and vertical strides, `strides = [1, stride, stride, 1]`. + If any value in `rate` is greater than 1, we perform atrous depthwise + convolution, in which case all values in the `strides` tensor must be equal + to 1. + + Args: + input: 4-D with shape according to `data_format`. + filter: 4-D with shape + `[filter_height, filter_width, in_channels, channel_multiplier]`. + strides: 1-D of size 4. The stride of the sliding window for each + dimension of `input`. + padding: A string, either `'VALID'` or `'SAME'`. The padding algorithm. + See the "returns" section of `tf.nn.convolution` for details. + data_format: The data format for input. Either "NHWC" (default) or "NCHW". + dilations: 1-D of size 2. The dilation rate in which we sample input values + across the `height` and `width` dimensions in atrous convolution. If it is + greater than 1, then all values of strides must be 1. + name: A name for this operation (optional). + + Returns: + A 4-D `Tensor` with shape according to `data_format`. E.g., for + "NHWC" format, shape is + `[batch, out_height, out_width, in_channels * channel_multiplier].` + """ + return depthwise_conv2d(input=input, + filter=filter, + strides=strides, + padding=padding, + rate=dilations, + name=name, + data_format=data_format) + # pylint: enable=redefined-builtin # pylint: disable=redefined-builtin,line-too-long -@tf_export("nn.separable_conv2d") +@tf_export(v1=["nn.separable_conv2d"]) def separable_conv2d(input, depthwise_filter, pointwise_filter, @@ -599,10 +681,76 @@ def separable_conv2d(input, name=name) +@tf_export("nn.separable_conv2d", v1=[]) +def separable_conv2d_v2( + input, + depthwise_filter, + pointwise_filter, + strides, + padding, + data_format=None, + dilations=None, + name=None, +): + """2-D convolution with separable filters. + + Performs a depthwise convolution that acts separately on channels followed by + a pointwise convolution that mixes channels. Note that this is separability + between dimensions `[1, 2]` and `3`, not spatial separability between + dimensions `1` and `2`. + + In detail, + + output[b, i, j, k] = sum_{di, dj, q, r} + input[b, strides[1] * i + di, strides[2] * j + dj, q] * + depthwise_filter[di, dj, q, r] * + pointwise_filter[0, 0, q * channel_multiplier + r, k] + + `strides` controls the strides for the depthwise convolution only, since + the pointwise convolution has implicit strides of `[1, 1, 1, 1]`. Must have + `strides[0] = strides[3] = 1`. For the most common case of the same + horizontal and vertical strides, `strides = [1, stride, stride, 1]`. + If any value in `rate` is greater than 1, we perform atrous depthwise + convolution, in which case all values in the `strides` tensor must be equal + to 1. + + Args: + input: 4-D `Tensor` with shape according to `data_format`. + depthwise_filter: 4-D `Tensor` with shape `[filter_height, filter_width, + in_channels, channel_multiplier]`. Contains `in_channels` convolutional + filters of depth 1. + pointwise_filter: 4-D `Tensor` with shape `[1, 1, channel_multiplier * + in_channels, out_channels]`. Pointwise filter to mix channels after + `depthwise_filter` has convolved spatially. + strides: 1-D of size 4. The strides for the depthwise convolution for each + dimension of `input`. + padding: A string, either `'VALID'` or `'SAME'`. The padding algorithm. See + the "returns" section of `tf.nn.convolution` for details. + data_format: The data format for input. Either "NHWC" (default) or "NCHW". + dilations: 1-D of size 2. The dilation rate in which we sample input values + across the `height` and `width` dimensions in atrous convolution. If it is + greater than 1, then all values of strides must be 1. + name: A name for this operation (optional). + + Returns: + A 4-D `Tensor` with shape according to 'data_format'. For + example, with data_format="NHWC", shape is [batch, out_height, + out_width, out_channels]. + """ + return separable_conv2d( + input, + depthwise_filter, + pointwise_filter, + strides, + padding, + rate=dilations, + name=name, + data_format=data_format) + # pylint: enable=redefined-builtin,line-too-long -@tf_export("nn.sufficient_statistics") +@tf_export(v1=["nn.sufficient_statistics"]) def sufficient_statistics(x, axes, shift=None, keep_dims=False, name=None): """Calculate the sufficient statistics for the mean and variance of `x`. @@ -652,6 +800,35 @@ def sufficient_statistics(x, axes, shift=None, keep_dims=False, name=None): return counts, m_ss, v_ss, shift +@tf_export("nn.sufficient_statistics", v1=[]) +def sufficient_statistics_v2(x, axes, shift=None, keepdims=False, name=None): + """Calculate the sufficient statistics for the mean and variance of `x`. + + These sufficient statistics are computed using the one pass algorithm on + an input that's optionally shifted. See: + https://en.wikipedia.org/wiki/Algorithms_for_calculating_variance#Computing_shifted_data + + Args: + x: A `Tensor`. + axes: Array of ints. Axes along which to compute mean and variance. + shift: A `Tensor` containing the value by which to shift the data for + numerical stability, or `None` if no shift is to be performed. A shift + close to the true mean provides the most numerically stable results. + keepdims: produce statistics with the same dimensionality as the input. + name: Name used to scope the operations that compute the sufficient stats. + + Returns: + Four `Tensor` objects of the same type as `x`: + + * the count (number of elements to average over). + * the (possibly shifted) sum of the elements in the array. + * the (possibly shifted) sum of squares of the elements in the array. + * the shift by which the mean must be corrected or None if `shift` is None. + """ + return sufficient_statistics( + x=x, axes=axes, shift=shift, keep_dims=keepdims, name=name) + + @tf_export("nn.normalize_moments") def normalize_moments(counts, mean_ss, variance_ss, shift, name=None): """Calculate the mean and variance of based on the sufficient statistics. @@ -684,7 +861,7 @@ def normalize_moments(counts, mean_ss, variance_ss, shift, name=None): return (mean, variance) -@tf_export("nn.moments") +@tf_export(v1=["nn.moments"]) def moments( x, axes, @@ -743,7 +920,43 @@ def moments( return (mean, variance) -@tf_export("nn.weighted_moments") +@tf_export("nn.moments", v1=[]) +def moments_v2( + x, + axes, + shift=None, + keepdims=False, + name=None): + """Calculates the mean and variance of `x`. + + The mean and variance are calculated by aggregating the contents of `x` + across `axes`. If `x` is 1-D and `axes = [0]` this is just the mean + and variance of a vector. + + Note: shift is currently not used; the true mean is computed and used. + + When using these moments for batch normalization (see + `tf.nn.batch_normalization`): + + * for so-called "global normalization", used with convolutional filters with + shape `[batch, height, width, depth]`, pass `axes=[0, 1, 2]`. + * for simple batch normalization pass `axes=[0]` (batch only). + + Args: + x: A `Tensor`. + axes: Array of ints. Axes along which to compute mean and + variance. + shift: Not used in the current implementation. + keepdims: produce moments with the same dimensionality as the input. + name: Name used to scope the operations that compute the moments. + + Returns: + Two `Tensor` objects: `mean` and `variance`. + """ + return moments(x=x, axes=axes, shift=shift, name=name, keep_dims=keepdims) + + +@tf_export(v1=["nn.weighted_moments"]) def weighted_moments(x, axes, frequency_weights, name=None, keep_dims=False): """Returns the frequency-weighted mean and variance of `x`. @@ -815,6 +1028,30 @@ def weighted_moments(x, axes, frequency_weights, name=None, keep_dims=False): return weighted_mean, weighted_variance +@tf_export("nn.weighted_moments", v1=[]) +def weighted_moments_v2(x, axes, frequency_weights, keepdims=False, name=None): + """Returns the frequency-weighted mean and variance of `x`. + + Args: + x: A tensor. + axes: 1-d tensor of int32 values; these are the axes along which + to compute mean and variance. + frequency_weights: A tensor of positive weights which can be + broadcast with x. + keepdims: Produce moments with the same dimensionality as the input. + name: Name used to scope the operation. + + Returns: + Two tensors: `weighted_mean` and `weighted_variance`. + """ + return weighted_moments( + x=x, + axes=axes, + frequency_weights=frequency_weights, + name=name, + keep_dims=keepdims) + + @tf_export("nn.batch_normalization") def batch_normalization(x, mean, @@ -875,7 +1112,7 @@ def batch_normalization(x, offset - mean * inv if offset is not None else -mean * inv, x.dtype) -@tf_export("nn.fused_batch_norm") +@tf_export(v1=["nn.fused_batch_norm"]) def fused_batch_norm( x, scale, @@ -946,7 +1183,7 @@ def fused_batch_norm( return y, batch_mean, batch_var -@tf_export("nn.batch_norm_with_global_normalization") +@tf_export(v1=["nn.batch_norm_with_global_normalization"]) def batch_norm_with_global_normalization(t, m, v, @@ -984,6 +1221,53 @@ def batch_norm_with_global_normalization(t, else None, variance_epsilon, name) +# pylint: disable=redefined-builtin,line-too-long +@tf_export("nn.batch_norm_with_global_normalization", v1=[]) +def batch_norm_with_global_normalization_v2(input, + mean, + variance, + beta, + gamma, + variance_epsilon, + scale_after_normalization, + name=None): + """Batch normalization. + + This op is deprecated. See `tf.nn.batch_normalization`. + + Args: + input: A 4D input Tensor. + mean: A 1D mean Tensor with size matching the last dimension of t. + This is the first output from tf.nn.moments, + or a saved moving average thereof. + variance: A 1D variance Tensor with size matching the last dimension of t. + This is the second output from tf.nn.moments, + or a saved moving average thereof. + beta: A 1D beta Tensor with size matching the last dimension of t. + An offset to be added to the normalized tensor. + gamma: A 1D gamma Tensor with size matching the last dimension of t. + If "scale_after_normalization" is true, this tensor will be multiplied + with the normalized tensor. + variance_epsilon: A small float number to avoid dividing by 0. + scale_after_normalization: A bool indicating whether the resulted tensor + needs to be multiplied with gamma. + name: A name for this operation (optional). + + Returns: + A batch-normalized `t`. + """ + return batch_norm_with_global_normalization(t=input, + m=mean, + v=variance, + beta=beta, + gamma=gamma, + variance_epsilon=variance_epsilon, + scale_after_normalization=scale_after_normalization, + name=name) + +# pylint: enable=redefined-builtin,line-too-long + + def _sum_rows(x): """Returns a vector summing up each row of the matrix x.""" # _sum_rows(x) is equivalent to math_ops.reduce_sum(x, 1) when x is @@ -1178,7 +1462,111 @@ def _compute_sampled_logits(weights, return out_logits, out_labels -@tf_export("nn.nce_loss") +@tf_export("nn.nce_loss", v1=[]) +def nce_loss_v2(weights, + biases, + labels, + inputs, + num_sampled, + num_classes, + num_true=1, + sampled_values=None, + remove_accidental_hits=False, + name="nce_loss"): + """Computes and returns the noise-contrastive estimation training loss. + + See [Noise-contrastive estimation: A new estimation principle for + unnormalized statistical + models](http://www.jmlr.org/proceedings/papers/v9/gutmann10a/gutmann10a.pdf). + Also see our [Candidate Sampling Algorithms + Reference](https://www.tensorflow.org/extras/candidate_sampling.pdf) + + A common use case is to use this method for training, and calculate the full + sigmoid loss for evaluation or inference as in the following example: + + ```python + if mode == "train": + loss = tf.nn.nce_loss( + weights=weights, + biases=biases, + labels=labels, + inputs=inputs, + ...) + elif mode == "eval": + logits = tf.matmul(inputs, tf.transpose(weights)) + logits = tf.nn.bias_add(logits, biases) + labels_one_hot = tf.one_hot(labels, n_classes) + loss = tf.nn.sigmoid_cross_entropy_with_logits( + labels=labels_one_hot, + logits=logits) + loss = tf.reduce_sum(loss, axis=1) + ``` + + Note: when doing embedding lookup on `weights` and `bias`, "div" partition + strategy will be used. Support for other partition strategy will be added + later. + + Note: By default this uses a log-uniform (Zipfian) distribution for sampling, + so your labels must be sorted in order of decreasing frequency to achieve + good results. For more details, see + `tf.nn.log_uniform_candidate_sampler`. + + Note: In the case where `num_true` > 1, we assign to each target class + the target probability 1 / `num_true` so that the target probabilities + sum to 1 per-example. + + Note: It would be useful to allow a variable number of target classes per + example. We hope to provide this functionality in a future release. + For now, if you have a variable number of target classes, you can pad them + out to a constant number by either repeating them or by padding + with an otherwise unused class. + + Args: + weights: A `Tensor` of shape `[num_classes, dim]`, or a list of `Tensor` + objects whose concatenation along dimension 0 has shape [num_classes, + dim]. The (possibly-partitioned) class embeddings. + biases: A `Tensor` of shape `[num_classes]`. The class biases. + labels: A `Tensor` of type `int64` and shape `[batch_size, num_true]`. The + target classes. + inputs: A `Tensor` of shape `[batch_size, dim]`. The forward activations of + the input network. + num_sampled: An `int`. The number of negative classes to randomly sample + per batch. This single sample of negative classes is evaluated for each + element in the batch. + num_classes: An `int`. The number of possible classes. + num_true: An `int`. The number of target classes per training example. + sampled_values: a tuple of (`sampled_candidates`, `true_expected_count`, + `sampled_expected_count`) returned by a `*_candidate_sampler` function. + (if None, we default to `log_uniform_candidate_sampler`) + remove_accidental_hits: A `bool`. Whether to remove "accidental hits" + where a sampled class equals one of the target classes. If set to `True`, + this is a "Sampled Logistic" loss instead of NCE, and we are learning to + generate log-odds instead of log probabilities. See our [Candidate + Sampling Algorithms Reference] + (https://www.tensorflow.org/extras/candidate_sampling.pdf). Default is + False. + name: A name for the operation (optional). + + Returns: + A `batch_size` 1-D tensor of per-example NCE losses. + """ + # TODO(yuefengz): get partition_strategy from either variables or distribution + # strategies. + return nce_loss( + weights, + biases, + labels, + inputs, + num_sampled, + num_classes, + num_true=num_true, + sampled_values=sampled_values, + remove_accidental_hits=remove_accidental_hits, + partition_strategy="div", + name=name) + + +@tf_export(v1=["nn.nce_loss"]) def nce_loss(weights, biases, labels, diff --git a/tensorflow/python/ops/nn_ops.py b/tensorflow/python/ops/nn_ops.py index bc195993c2e..225904854fa 100644 --- a/tensorflow/python/ops/nn_ops.py +++ b/tensorflow/python/ops/nn_ops.py @@ -28,6 +28,7 @@ from tensorflow.python.framework import dtypes from tensorflow.python.framework import errors_impl from tensorflow.python.framework import graph_util from tensorflow.python.framework import ops +from tensorflow.python.framework import random_seed from tensorflow.python.framework import tensor_shape from tensorflow.python.framework import tensor_util from tensorflow.python.ops import array_ops @@ -35,13 +36,14 @@ from tensorflow.python.ops import check_ops from tensorflow.python.ops import gen_nn_ops from tensorflow.python.ops import math_ops from tensorflow.python.ops import random_ops - # go/tf-wildcard-import # pylint: disable=wildcard-import from tensorflow.python.ops.gen_nn_ops import * # pylint: enable=wildcard-import - from tensorflow.python.util import deprecation +from tensorflow.python.util.deprecation import deprecated_args +from tensorflow.python.util.deprecation import deprecated_argument_lookup + from tensorflow.python.util.tf_export import tf_export # Aliases for some automatically-generated names. @@ -206,6 +208,73 @@ class _NonAtrousConvolution(object): name=self.name) +@tf_export("nn.dilation2d", v1=[]) +def dilation2d_v2( + input, # pylint: disable=redefined-builtin + filters, # pylint: disable=redefined-builtin + strides, + padding, + data_format, + dilations, + name=None): + """Computes the grayscale dilation of 4-D `input` and 3-D `filters` tensors. + + The `input` tensor has shape `[batch, in_height, in_width, depth]` and the + `filters` tensor has shape `[filter_height, filter_width, depth]`, i.e., each + input channel is processed independently of the others with its own + structuring function. The `output` tensor has shape + `[batch, out_height, out_width, depth]`. The spatial dimensions of the output + tensor depend on the `padding` algorithm. We currently only support the + default "NHWC" `data_format`. + + In detail, the grayscale morphological 2-D dilation is the max-sum correlation + (for consistency with `conv2d`, we use unmirrored filters): + + output[b, y, x, c] = + max_{dy, dx} input[b, + strides[1] * y + rates[1] * dy, + strides[2] * x + rates[2] * dx, + c] + + filters[dy, dx, c] + + Max-pooling is a special case when the filter has size equal to the pooling + kernel size and contains all zeros. + + Note on duality: The dilation of `input` by the `filters` is equal to the + negation of the erosion of `-input` by the reflected `filters`. + + Args: + input: A `Tensor`. Must be one of the following types: `float32`, `float64`, + `int32`, `uint8`, `int16`, `int8`, `int64`, `bfloat16`, `uint16`, `half`, + `uint32`, `uint64`. + 4-D with shape `[batch, in_height, in_width, depth]`. + filters: A `Tensor`. Must have the same type as `input`. + 3-D with shape `[filter_height, filter_width, depth]`. + strides: A list of `ints` that has length `>= 4`. + The stride of the sliding window for each dimension of the input + tensor. Must be: `[1, stride_height, stride_width, 1]`. + padding: A `string` from: `"SAME", "VALID"`. + The type of padding algorithm to use. + data_format: A `string`, only `"NCHW"` is currently supported. + dilations: A list of `ints` that has length `>= 4`. + The input stride for atrous morphological dilation. Must be: + `[1, rate_height, rate_width, 1]`. + name: A name for the operation (optional). + + Returns: + A `Tensor`. Has the same type as `input`. + """ + if data_format != "NCHW": + raise ValueError("Data formats other than NCHW are not yet supported") + + return gen_nn_ops.dilation2d(input=input, + filter=filters, + strides=strides, + rates=dilations, + padding=padding, + name=name) + + @tf_export("nn.with_space_to_batch") def with_space_to_batch( input, # pylint: disable=redefined-builtin @@ -644,7 +713,7 @@ def _get_strides_and_dilation_rate(num_spatial_dims, strides, dilation_rate): return strides, dilation_rate -@tf_export("nn.convolution") +@tf_export(v1=["nn.convolution"]) def convolution( input, # pylint: disable=redefined-builtin filter, # pylint: disable=redefined-builtin @@ -782,6 +851,30 @@ def convolution( return op(input, filter) +@tf_export("nn.convolution", v1=[]) +def convolution_v2( + input, # pylint: disable=redefined-builtin + filters, + strides=None, + padding="VALID", + data_format=None, + dilations=None, + name=None): + return convolution( + input, # pylint: disable=redefined-builtin + filters, + padding=padding, + strides=strides, + dilation_rate=dilations, + name=name, + data_format=data_format) + +convolution_v2.__doc__ = deprecation.rewrite_argument_docstring( + deprecation.rewrite_argument_docstring( + convolution.__doc__, "dilation_rate", "dilations"), + "filter", "filters") + + class Convolution(object): """Helper class for convolution. @@ -873,7 +966,7 @@ class Convolution(object): return self.conv_op(inp, filter) -@tf_export("nn.pool") +@tf_export(v1=["nn.pool"]) def pool( input, # pylint: disable=redefined-builtin window_shape, @@ -1044,6 +1137,105 @@ def pool( filter_shape=window_shape) +@tf_export("nn.pool", v1=[]) +def pool_v2( + input, # pylint: disable=redefined-builtin + window_shape, + pooling_type, + strides=None, + padding="VALID", + data_format=None, + dilations=None, + name=None): + # pylint: disable=line-too-long + """Performs an N-D pooling operation. + + In the case that `data_format` does not start with "NC", computes for + 0 <= b < batch_size, + 0 <= x[i] < output_spatial_shape[i], + 0 <= c < num_channels: + + ``` + output[b, x[0], ..., x[N-1], c] = + REDUCE_{z[0], ..., z[N-1]} + input[b, + x[0] * strides[0] - pad_before[0] + dilation_rate[0]*z[0], + ... + x[N-1]*strides[N-1] - pad_before[N-1] + dilation_rate[N-1]*z[N-1], + c], + ``` + + where the reduction function REDUCE depends on the value of `pooling_type`, + and pad_before is defined based on the value of `padding` as described in + the "returns" section of `tf.nn.convolution` for details. + The reduction never includes out-of-bounds positions. + + In the case that `data_format` starts with `"NC"`, the `input` and output are + simply transposed as follows: + + ``` + pool(input, data_format, **kwargs) = + tf.transpose(pool(tf.transpose(input, [0] + range(2,N+2) + [1]), + **kwargs), + [0, N+1] + range(1, N+1)) + ``` + + Args: + input: Tensor of rank N+2, of shape `[batch_size] + input_spatial_shape + + [num_channels]` if data_format does not start with "NC" (default), or + `[batch_size, num_channels] + input_spatial_shape` if data_format starts + with "NC". Pooling happens over the spatial dimensions only. + window_shape: Sequence of N ints >= 1. + pooling_type: Specifies pooling operation, must be "AVG" or "MAX". + strides: Optional. Sequence of N ints >= 1. Defaults to [1]*N. If any value of + strides is > 1, then all values of dilation_rate must be 1. + padding: The padding algorithm, must be "SAME" or "VALID". Defaults to "SAME". + See the "returns" section of `tf.nn.convolution` for details. + data_format: A string or None. Specifies whether the channel dimension of + the `input` and output is the last dimension (default, or if `data_format` + does not start with "NC"), or the second dimension (if `data_format` + starts with "NC"). For N=1, the valid values are "NWC" (default) and + "NCW". For N=2, the valid values are "NHWC" (default) and "NCHW". For + N=3, the valid values are "NDHWC" (default) and "NCDHW". + dilations: Optional. Dilation rate. List of N ints >= 1. Defaults to + [1]*N. If any value of dilation_rate is > 1, then all values of strides + must be 1. + name: Optional. Name of the op. + + Returns: + Tensor of rank N+2, of shape + [batch_size] + output_spatial_shape + [num_channels] + + if data_format is None or does not start with "NC", or + + [batch_size, num_channels] + output_spatial_shape + + if data_format starts with "NC", + where `output_spatial_shape` depends on the value of padding: + + If padding = "SAME": + output_spatial_shape[i] = ceil(input_spatial_shape[i] / strides[i]) + + If padding = "VALID": + output_spatial_shape[i] = + ceil((input_spatial_shape[i] - (window_shape[i] - 1) * dilation_rate[i]) + / strides[i]). + + Raises: + ValueError: if arguments are invalid. + + """ + return pool( + input=input, + window_shape=window_shape, + pooling_type=pooling_type, + padding=padding, + dilation_rate=dilations, + strides=strides, + name=name, + data_format=data_format) + + @tf_export("nn.atrous_conv2d") def atrous_conv2d(value, filters, rate, padding, name=None): """Atrous convolution (a.k.a. convolution with holes or dilated convolution). @@ -1181,7 +1373,208 @@ def atrous_conv2d(value, filters, rate, padding, name=None): name=name) -@tf_export("nn.conv2d_transpose") +@tf_export("nn.conv2d", v1=[]) +def conv2d_v2(input, # pylint: disable=redefined-builtin + filters, + strides, + padding, + data_format="NHWC", + dilations=None, + name=None): + # pylint: disable=line-too-long + r"""Computes a 2-D convolution given 4-D `input` and `filters` tensors. + + Given an input tensor of shape `[batch, in_height, in_width, in_channels]` + and a filter / kernel tensor of shape + `[filter_height, filter_width, in_channels, out_channels]`, this op + performs the following: + + 1. Flattens the filter to a 2-D matrix with shape + `[filter_height * filter_width * in_channels, output_channels]`. + 2. Extracts image patches from the input tensor to form a *virtual* + tensor of shape `[batch, out_height, out_width, + filter_height * filter_width * in_channels]`. + 3. For each patch, right-multiplies the filter matrix and the image patch + vector. + + In detail, with the default NHWC format, + + output[b, i, j, k] = + sum_{di, dj, q} input[b, strides[1] * i + di, strides[2] * j + dj, q] * + filter[di, dj, q, k] + + Must have `strides[0] = strides[3] = 1`. For the most common case of the same + horizontal and vertices strides, `strides = [1, stride, stride, 1]`. + + Args: + input: A `Tensor`. Must be one of the following types: + `half`, `bfloat16`, `float32`, `float64`. + A 4-D tensor. The dimension order is interpreted according to the value + of `data_format`, see below for details. + filters: A `Tensor`. Must have the same type as `input`. + A 4-D tensor of shape + `[filter_height, filter_width, in_channels, out_channels]` + strides: A list of `ints`. + 1-D tensor of length 4. The stride of the sliding window for each + dimension of `input`. The dimension order is determined by the value of + `data_format`, see below for details. + padding: A `string` from: `"SAME", "VALID"`. + The type of padding algorithm to use. + data_format: An optional `string` from: `"NHWC", "NCHW"`. + Defaults to `"NHWC"`. + Specify the data format of the input and output data. With the + default format "NHWC", the data is stored in the order of: + [batch, height, width, channels]. + Alternatively, the format could be "NCHW", the data storage order of: + [batch, channels, height, width]. + dilations: An optional list of `ints`. Defaults to `[1, 1, 1, 1]`. + 1-D tensor of length 4. The dilation factor for each dimension of + `input`. If set to k > 1, there will be k-1 skipped cells between each + filter element on that dimension. The dimension order is determined by the + value of `data_format`, see above for details. Dilations in the batch and + depth dimensions must be 1. + name: A name for the operation (optional). + + Returns: + A `Tensor`. Has the same type as `input`. + """ + # pylint: enable=line-too-long + if dilations is None: + dilations = [1, 1, 1, 1] + return gen_nn_ops.conv2d(input, # pylint: disable=redefined-builtin + filters, + strides, + padding, + use_cudnn_on_gpu=True, + data_format=data_format, + dilations=dilations, + name=name) +tf_export(v1=["nn.conv2d"])(gen_nn_ops.conv2d) + + +@tf_export("nn.conv2d_backprop_filter", v1=[]) +def conv2d_backprop_filter_v2(input, # pylint: disable=redefined-builtin + filter_sizes, + out_backprop, + strides, + padding, + data_format="NHWC", + dilations=None, + name=None): + r"""Computes the gradients of convolution with respect to the filter. + + Args: + input: A `Tensor`. Must be one of the following types: + `half`, `bfloat16`, `float32`, `float64`. + 4-D with shape `[batch, in_height, in_width, in_channels]`. + filter_sizes: A `Tensor` of type `int32`. + An integer vector representing the tensor shape of `filter`, + where `filter` is a 4-D + `[filter_height, filter_width, in_channels, out_channels]` tensor. + out_backprop: A `Tensor`. Must have the same type as `input`. + 4-D with shape `[batch, out_height, out_width, out_channels]`. + Gradients w.r.t. the output of the convolution. + strides: A list of `ints`. + The stride of the sliding window for each dimension of the input + of the convolution. Must be in the same order as the dimension specified + with format. + padding: A `string` from: `"SAME", "VALID"`. + The type of padding algorithm to use. + data_format: An optional `string` from: `"NHWC", "NCHW"`. + Defaults to `"NHWC"`. + Specify the data format of the input and output data. With the + default format "NHWC", the data is stored in the order of: + [batch, in_height, in_width, in_channels]. + Alternatively, the format could be "NCHW", the data storage order of: + [batch, in_channels, in_height, in_width]. + dilations: An optional list of `ints`. Defaults to `[1, 1, 1, 1]`. + 1-D tensor of length 4. The dilation factor for each dimension of + `input`. If set to k > 1, there will be k-1 skipped cells between each + filter element on that dimension. The dimension order is determined by + the value of `data_format`, see above for details. Dilations in the batch + and depth dimensions must be 1. + name: A name for the operation (optional). + + Returns: + A `Tensor`. Has the same type as `input`. + """ + if dilations is None: + dilations = [1, 1, 1, 1] + return gen_nn_ops.conv2d_backprop_filter(input, # pylint: disable=redefined-builtin + filter_sizes, + out_backprop, + strides, + padding, + use_cudnn_on_gpu=True, + data_format=data_format, + dilations=dilations, + name=name) +tf_export(v1=["nn.conv2d_backprop_filter"])( + gen_nn_ops.conv2d_backprop_filter) + + +@tf_export("nn.conv2d_backprop_input", v1=[]) +def conv2d_backprop_input_v2(input_sizes, + filters, + out_backprop, + strides, + padding, + data_format="NHWC", + dilations=None, + name=None): + r"""Computes the gradients of convolution with respect to the input. + + Args: + input_sizes: A `Tensor` of type `int32`. + An integer vector representing the shape of `input`, + where `input` is a 4-D `[batch, height, width, channels]` tensor. + filters: A `Tensor`. Must be one of the following types: + `half`, `bfloat16`, `float32`, `float64`. + 4-D with shape + `[filter_height, filter_width, in_channels, out_channels]`. + out_backprop: A `Tensor`. Must have the same type as `filters`. + 4-D with shape `[batch, out_height, out_width, out_channels]`. + Gradients w.r.t. the output of the convolution. + strides: A list of `ints`. + The stride of the sliding window for each dimension of the input + of the convolution. Must be in the same order as the dimension specified + with format. + padding: A `string` from: `"SAME", "VALID"`. + The type of padding algorithm to use. + data_format: An optional `string` from: `"NHWC", "NCHW"`. + Defaults to `"NHWC"`. + Specify the data format of the input and output data. With the + default format "NHWC", the data is stored in the order of: + [batch, in_height, in_width, in_channels]. + Alternatively, the format could be "NCHW", the data storage order of: + [batch, in_channels, in_height, in_width]. + dilations: An optional list of `ints`. Defaults to `[1, 1, 1, 1]`. + 1-D tensor of length 4. The dilation factor for each dimension of + `input`. If set to k > 1, there will be k-1 skipped cells between each + filter element on that dimension. The dimension order is determined by + the value of `data_format`, see above for details. Dilations in the batch + and depth dimensions must be 1. + name: A name for the operation (optional). + + Returns: + A `Tensor`. Has the same type as `filters`. + """ + if dilations is None: + dilations = [1, 1, 1, 1] + return gen_nn_ops.conv2d_backprop_input(input_sizes, + filters, + out_backprop, + strides, + padding, + use_cudnn_on_gpu=True, + data_format=data_format, + dilations=dilations, + name=name) +tf_export(v1=["nn.conv2d_backprop_input"])( + gen_nn_ops.conv2d_backprop_input) + + +@tf_export(v1=["nn.conv2d_transpose"]) def conv2d_transpose( value, filter, # pylint: disable=redefined-builtin @@ -1261,6 +1654,31 @@ def conv2d_transpose( name=name) +# pylint: disable=redefined-builtin +@tf_export("nn.conv2d_transpose", v1=[]) +def conv2d_transpose_v2( + input, + filters, # pylint: disable=redefined-builtin + output_shape, + strides, + padding="SAME", + data_format="NHWC", + name=None): + return conv2d_transpose( + input, + filters, + output_shape, + strides, + padding=padding, + data_format=data_format, + name=name) +# pylint: enable=redefined-builtin +conv2d_transpose_v2.__doc__ = deprecation.rewrite_argument_docstring( + deprecation.rewrite_argument_docstring( + conv2d_transpose.__doc__, "filter", "filters"), + "value", "input") + + @tf_export("nn.atrous_conv2d_transpose") def atrous_conv2d_transpose(value, filters, @@ -1409,7 +1827,29 @@ def atrous_conv2d_transpose(value, input=value, crops=batch_to_space_crop, block_size=rate) -@tf_export("nn.conv3d_transpose") +@tf_export("nn.conv3d", v1=[]) +def conv3d_v2(input, # pylint: disable=redefined-builtin,missing-docstring + filters, + strides, + padding, + data_format="NDHWC", + dilations=None, + name=None): + if dilations is None: + dilations = [1, 1, 1, 1, 1] + return gen_nn_ops.conv3d(input, # pylint: disable=redefined-builtin + filters, + strides, + padding, + data_format=data_format, + dilations=dilations, + name=name) +tf_export(v1=["nn.conv3d"])(gen_nn_ops.conv3d) +conv3d_v2.__doc__ = deprecation.rewrite_argument_docstring( + gen_nn_ops.conv3d.__doc__, "filter", "filters") + + +@tf_export(v1=["nn.conv3d_transpose"]) def conv3d_transpose( value, filter, # pylint: disable=redefined-builtin @@ -1487,6 +1927,31 @@ def conv3d_transpose( name=name) +# pylint: disable=redefined-builtin +@tf_export("nn.conv3d_transpose", v1=[]) +def conv3d_transpose_v2( + input, + filters, + output_shape, + strides, + padding="SAME", + data_format="NDHWC", + name=None): + return conv3d_transpose( + input, + filters, + output_shape, + strides, + padding=padding, + data_format=data_format, + name=name) +# pylint: enable=redefined-builtin +conv3d_transpose_v2.__doc__ = deprecation.rewrite_argument_docstring( + deprecation.rewrite_argument_docstring( + conv3d_transpose.__doc__, "filter", "filters"), + "value", "input") + + @tf_export("nn.bias_add") def bias_add(value, bias, data_format=None, name=None): """Adds `bias` to `value`. @@ -1542,7 +2007,7 @@ def bias_add_v1(value, bias, name=None): return gen_nn_ops.bias_add_v1(value, bias, name=name) -@tf_export("nn.crelu") +@tf_export(v1=["nn.crelu"]) def crelu(features, name=None, axis=-1): """Computes Concatenated ReLU. @@ -1568,6 +2033,12 @@ def crelu(features, name=None, axis=-1): return gen_nn_ops.relu(c) +@tf_export("nn.crelu", v1=[]) +def crelu_v2(features, axis=-1, name=None): + return crelu(features, name=name, axis=axis) +crelu_v2.__doc__ = crelu.__doc__ + + @tf_export("nn.relu6") def relu6(features, name=None): """Computes Rectified Linear 6: `min(max(features, 0), 6)`. @@ -1715,7 +2186,7 @@ def _softmax(logits, compute_op, dim=-1, name=None): return output -@tf_export("nn.softmax", "math.softmax") +@tf_export(v1=["nn.softmax", "math.softmax"]) @deprecation.deprecated_args(None, "dim is deprecated, use axis instead", "dim") def softmax(logits, axis=None, name=None, dim=None): """Computes softmax activations. @@ -1745,7 +2216,34 @@ def softmax(logits, axis=None, name=None, dim=None): return _softmax(logits, gen_nn_ops.softmax, axis, name) -@tf_export("nn.log_softmax", "math.log_softmax") +@tf_export("nn.softmax", "math.softmax", v1=[]) +def softmax_v2(logits, axis=None, name=None): + """Computes softmax activations. + + This function performs the equivalent of + + softmax = tf.exp(logits) / tf.reduce_sum(tf.exp(logits), axis) + + Args: + logits: A non-empty `Tensor`. Must be one of the following types: `half`, + `float32`, `float64`. + axis: The dimension softmax would be performed on. The default is -1 which + indicates the last dimension. + name: A name for the operation (optional). + + Returns: + A `Tensor`. Has the same type and shape as `logits`. + + Raises: + InvalidArgumentError: if `logits` is empty or `axis` is beyond the last + dimension of `logits`. + """ + if axis is None: + axis = -1 + return _softmax(logits, gen_nn_ops.softmax, axis, name) + + +@tf_export(v1=["nn.log_softmax", "math.log_softmax"]) @deprecation.deprecated_args(None, "dim is deprecated, use axis instead", "dim") def log_softmax(logits, axis=None, name=None, dim=None): """Computes log softmax activations. @@ -1775,6 +2273,33 @@ def log_softmax(logits, axis=None, name=None, dim=None): return _softmax(logits, gen_nn_ops.log_softmax, axis, name) +@tf_export("nn.log_softmax", "math.log_softmax", v1=[]) +def log_softmax_v2(logits, axis=None, name=None): + """Computes log softmax activations. + + For each batch `i` and class `j` we have + + logsoftmax = logits - log(reduce_sum(exp(logits), axis)) + + Args: + logits: A non-empty `Tensor`. Must be one of the following types: `half`, + `float32`, `float64`. + axis: The dimension softmax would be performed on. The default is -1 which + indicates the last dimension. + name: A name for the operation (optional). + + Returns: + A `Tensor`. Has the same type as `logits`. Same shape as `logits`. + + Raises: + InvalidArgumentError: if `logits` is empty or `axis` is beyond the last + dimension of `logits`. + """ + if axis is None: + axis = -1 + return _softmax(logits, gen_nn_ops.log_softmax, axis, name) + + def _ensure_xent_args(name, sentinel, labels, logits): # Make sure that all arguments were passed as named arguments. if sentinel is not None: @@ -1784,9 +2309,8 @@ def _ensure_xent_args(name, sentinel, labels, logits): raise ValueError("Both labels and logits must be provided.") -@tf_export("nn.softmax_cross_entropy_with_logits", - v1=["nn.softmax_cross_entropy_with_logits_v2"]) -def softmax_cross_entropy_with_logits_v2(labels, logits, dim=-1, name=None): +@tf_export("nn.softmax_cross_entropy_with_logits", v1=[]) +def softmax_cross_entropy_with_logits_v2(labels, logits, axis=-1, name=None): """Computes softmax cross entropy between `logits` and `labels`. Measures the probability error in discrete classification tasks in which the @@ -1808,7 +2332,7 @@ def softmax_cross_entropy_with_logits_v2(labels, logits, dim=-1, name=None): A common use case is to have logits and labels of shape `[batch_size, num_classes]`, but higher dimensions are supported, with - the `dim` argument specifying the class dimension. + the `axis` argument specifying the class dimension. `logits` and `labels` must have the same dtype (either `float16`, `float32`, or `float64`). @@ -1826,9 +2350,65 @@ def softmax_cross_entropy_with_logits_v2(labels, logits, dim=-1, name=None): `[batch_size, num_classes]`, each row of `labels[i]` must be a valid probability distribution. logits: Unscaled log probabilities. - dim: The class dimension. Defaulted to -1 which is the last dimension. + axis: The class dimension. Defaulted to -1 which is the last dimension. name: A name for the operation (optional). + Returns: + A `Tensor` that contains the softmax cross entropy loss. Its type is the + same as `logits` and its shape is the same as `labels` except that it does + not have the last dimension of `labels`. + """ + return softmax_cross_entropy_with_logits_v2_helper( + labels=labels, logits=logits, axis=axis, name=name) + + +@tf_export(v1=["nn.softmax_cross_entropy_with_logits_v2"]) +@deprecated_args(None, "dim is deprecated, use axis instead", "dim") +def softmax_cross_entropy_with_logits_v2_helper( + labels, logits, axis=None, name=None, dim=None): + """Computes softmax cross entropy between `logits` and `labels`. + + Measures the probability error in discrete classification tasks in which the + classes are mutually exclusive (each entry is in exactly one class). For + example, each CIFAR-10 image is labeled with one and only one label: an image + can be a dog or a truck, but not both. + + **NOTE:** While the classes are mutually exclusive, their probabilities + need not be. All that is required is that each row of `labels` is + a valid probability distribution. If they are not, the computation of the + gradient will be incorrect. + + If using exclusive `labels` (wherein one and only + one class is true at a time), see `sparse_softmax_cross_entropy_with_logits`. + + **WARNING:** This op expects unscaled logits, since it performs a `softmax` + on `logits` internally for efficiency. Do not call this op with the + output of `softmax`, as it will produce incorrect results. + + A common use case is to have logits and labels of shape + `[batch_size, num_classes]`, but higher dimensions are supported, with + the `axis` argument specifying the class dimension. + + `logits` and `labels` must have the same dtype (either `float16`, `float32`, + or `float64`). + + Backpropagation will happen into both `logits` and `labels`. To disallow + backpropagation into `labels`, pass label tensors through `tf.stop_gradient` + before feeding it to this function. + + **Note that to avoid confusion, it is required to pass only named arguments to + this function.** + + Args: + labels: Each vector along the class dimension should hold a valid + probability distribution e.g. for the case in which labels are of shape + `[batch_size, num_classes]`, each row of `labels[i]` must be a valid + probability distribution. + logits: Unscaled log probabilities. + axis: The class dimension. Defaulted to -1 which is the last dimension. + name: A name for the operation (optional). + dim: Deprecated alias for axis. + Returns: A `Tensor` that contains the softmax cross entropy loss. Its type is the same as `logits` and its shape is the same as `labels` except that it does @@ -1837,6 +2417,10 @@ def softmax_cross_entropy_with_logits_v2(labels, logits, dim=-1, name=None): # TODO(pcmurray) Raise an error when the labels do not sum to 1. Note: This # could break users who call this with bad labels, but disregard the bad # results. + axis = deprecated_argument_lookup("axis", axis, "dim", dim) + del dim + if axis is None: + axis = -1 with ops.name_scope(name, "softmax_cross_entropy_with_logits", [logits, labels]) as name: @@ -1853,7 +2437,7 @@ def softmax_cross_entropy_with_logits_v2(labels, logits, dim=-1, name=None): shape = logits.get_shape() # Move the dim to the end if dim is not the last dimension. - if dim is not -1: + if axis != -1: def _move_dim_to_end(tensor, dim_index, rank): return array_ops.transpose( @@ -1863,8 +2447,8 @@ def softmax_cross_entropy_with_logits_v2(labels, logits, dim=-1, name=None): math_ops.range(dim_index + 1, rank), [dim_index] ], 0)) - precise_logits = _move_dim_to_end(precise_logits, dim, input_rank) - labels = _move_dim_to_end(labels, dim, input_rank) + precise_logits = _move_dim_to_end(precise_logits, axis, input_rank) + labels = _move_dim_to_end(labels, axis, input_rank) input_shape = array_ops.shape(precise_logits) @@ -1878,7 +2462,7 @@ def softmax_cross_entropy_with_logits_v2(labels, logits, dim=-1, name=None): cost, unused_backprop = gen_nn_ops.softmax_cross_entropy_with_logits( precise_logits, labels, name=name) - # The output cost shape should be the input minus dim. + # The output cost shape should be the input minus axis. output_shape = array_ops.slice(input_shape, [0], [math_ops.subtract(input_rank, 1)]) cost = array_ops.reshape(cost, output_shape) @@ -1888,7 +2472,7 @@ def softmax_cross_entropy_with_logits_v2(labels, logits, dim=-1, name=None): if not context.executing_eagerly( ) and shape is not None and shape.dims is not None: shape = shape.as_list() - del shape[dim] + del shape[axis] cost.set_shape(shape) if convert_to_float32: @@ -1966,7 +2550,7 @@ def softmax_cross_entropy_with_logits( labels = array_ops.stop_gradient(labels, name="labels_stop_gradient") return softmax_cross_entropy_with_logits_v2( - labels=labels, logits=logits, dim=dim, name=name) + labels=labels, logits=logits, axis=dim, name=name) @tf_export("nn.sparse_softmax_cross_entropy_with_logits") @@ -2155,6 +2739,67 @@ def max_pool(value, ksize, strides, padding, data_format="NHWC", name=None): name=name) +# pylint: disable=redefined-builtin +@tf_export("nn.max_pool_with_argmax", v1=[]) +def max_pool_with_argmax_v2(input, + ksize, + strides, + padding, + data_format="NHWC", + output_dtype=dtypes.int64, + name=None): + """Performs max pooling on the input and outputs both max values and indices. + + The indices in `argmax` are flattened, so that a maximum value at position + `[b, y, x, c]` becomes flattened index + `((b * height + y) * width + x) * channels + c`. + + The indices returned are always in `[0, height) x [0, width)` before + flattening, even if padding is involved and the mathematically correct answer + is outside (either negative or too large). This is a bug, but fixing it is + difficult to do in a safe backwards compatible way, especially due to + flattening. + + Args: + input: A `Tensor`. Must be one of the following types: `float32`, `float64`, + `int32`, `uint8`, `int16`, `int8`, `int64`, `bfloat16`, `uint16`, `half`, + `uint32`, `uint64`. + 4-D with shape `[batch, height, width, channels]`. Input to pool over. + ksize: A list of `ints` that has length `>= 4`. + The size of the window for each dimension of the input tensor. + strides: A list of `ints` that has length `>= 4`. + The stride of the sliding window for each dimension of the + input tensor. + padding: A `string` from: `"SAME", "VALID"`. + The type of padding algorithm to use. + data_format: An optional `string`, must be set to `"NHWC"`. Defaults to + `"NHWC"`. + Specify the data format of the input and output data. + output_dtype: An optional `tf.DType` from: `tf.int32, tf.int64`. + Defaults to `tf.int64`. + The dtype of the returned argmax tensor. + name: A name for the operation (optional). + + Returns: + A tuple of `Tensor` objects (output, argmax). + + output: A `Tensor`. Has the same type as `input`. + argmax: A `Tensor` of type `output_dtype`. + """ + + if data_format != "NHWC": + raise ValueError("Data formats other than 'NHWC' are not yet supported") + + return gen_nn_ops.max_pool_with_argmax(input=input, + ksize=ksize, + strides=strides, + padding=padding, + Targmax=output_dtype, + name=name) + +# pylint: enable=redefined-builtin + + @ops.RegisterStatistics("Conv2D", "flops") def _calc_conv_flops(graph, node): """Calculates the compute resources needed for Conv2D.""" @@ -2199,7 +2844,7 @@ def _calc_bias_add_flops(graph, node): return ops.OpStats("flops", input_count) -@tf_export("nn.xw_plus_b") +@tf_export(v1=["nn.xw_plus_b"]) def xw_plus_b(x, weights, biases, name=None): # pylint: disable=invalid-name """Computes matmul(x, weights) + biases. @@ -2271,12 +2916,16 @@ def _get_noise_shape(x, noise_shape): return noise_shape -@tf_export("nn.dropout") -def dropout(x, keep_prob, noise_shape=None, seed=None, name=None): # pylint: disable=invalid-name +@tf_export(v1=["nn.dropout"]) +@deprecation.deprecated_args(None, "Please use `rate` instead of `keep_prob`. " + "Rate should be set to `rate = 1 - keep_prob`.", + "keep_prob") +def dropout(x, keep_prob=None, noise_shape=None, seed=None, name=None, + rate=None): # pylint: disable=invalid-name """Computes dropout. - With probability `keep_prob`, outputs the input element scaled up by - `1 / keep_prob`, otherwise outputs `0`. The scaling is so that the expected + For each element of `x`, with probability `rate`, outputs `0`, and otherwise + scales up the input by `1 / (1-rate)`. The scaling is such that the expected sum is unchanged. By default, each element is kept or dropped independently. If `noise_shape` @@ -2289,8 +2938,59 @@ def dropout(x, keep_prob, noise_shape=None, seed=None, name=None): # pylint: di Args: x: A floating point tensor. - keep_prob: A scalar `Tensor` with the same type as x. The probability - that each element is kept. + keep_prob: (deprecated) A deprecated alias for `(1-rate)`. + noise_shape: A 1-D `Tensor` of type `int32`, representing the + shape for randomly generated keep/drop flags. + seed: A Python integer. Used to create random seeds. See + `tf.set_random_seed` for behavior. + name: A name for this operation (optional). + rate: A scalar `Tensor` with the same type as `x`. The probability that each + element of `x` is discarded. + + Returns: + A Tensor of the same shape of `x`. + + Raises: + ValueError: If `rate` is not in `[0, 1)` or if `x` is not a floating + point tensor. + """ + try: + keep = 1. - keep_prob if keep_prob is not None else None + except TypeError: + raise ValueError("keep_prob must be a floating point number or Tensor " + "(got %r)" % keep_prob) + + rate = deprecation.deprecated_argument_lookup( + "rate", rate, + "keep_prob", keep) + + if rate is None: + raise ValueError("You must provide a rate to dropout.") + + return dropout_v2(x, rate, noise_shape=noise_shape, seed=seed, name=name) + + +@tf_export("nn.dropout", v1=[]) +def dropout_v2(x, rate, noise_shape=None, seed=None, name=None): # pylint: disable=invalid-name + """Computes dropout. + + With probability `rate`, drops elements of `x`. Input that are kept are + scaled up by `1 / (1 - rate)`, otherwise outputs `0`. The scaling is so that + the expected sum is unchanged. + + By default, each element is kept or dropped independently. If `noise_shape` + is specified, it must be + [broadcastable](http://docs.scipy.org/doc/numpy/user/basics.broadcasting.html) + to the shape of `x`, and only dimensions with `noise_shape[i] == shape(x)[i]` + will make independent decisions. For example, if `shape(x) = [k, l, m, n]` + and `noise_shape = [k, 1, 1, n]`, each batch and channel component will be + kept independently and each row and column will be kept or not kept together. + + Args: + x: A floating point tensor. + rate: A scalar `Tensor` with the same type as x. The probability + that each element is dropped. For example, setting rate=0.1 would drop + 10% of input elements. noise_shape: A 1-D `Tensor` of type `int32`, representing the shape for randomly generated keep/drop flags. seed: A Python integer. Used to create random seeds. See @@ -2310,28 +3010,29 @@ def dropout(x, keep_prob, noise_shape=None, seed=None, name=None): # pylint: di if not x.dtype.is_floating: raise ValueError("x has to be a floating point tensor since it's going to" " be scaled. Got a %s tensor instead." % x.dtype) - if isinstance(keep_prob, numbers.Real) and not 0 < keep_prob <= 1: - raise ValueError("keep_prob must be a scalar tensor or a float in the " - "range (0, 1], got %g" % keep_prob) + if isinstance(rate, numbers.Real) and not (rate >= 0 and rate < 1): + raise ValueError("rate must be a scalar tensor or a float in the " + "range [0, 1), got %g" % rate) # Early return if nothing needs to be dropped. - if isinstance(keep_prob, float) and keep_prob == 1: + if isinstance(rate, numbers.Real) and rate == 0: return x if context.executing_eagerly(): - if isinstance(keep_prob, ops.EagerTensor): - if keep_prob.numpy() == 1: + if isinstance(rate, ops.EagerTensor): + if rate.numpy() == 0: return x else: - keep_prob = ops.convert_to_tensor( - keep_prob, dtype=x.dtype, name="keep_prob") - keep_prob.get_shape().assert_is_compatible_with(tensor_shape.scalar()) + rate = ops.convert_to_tensor( + rate, dtype=x.dtype, name="rate") + rate.get_shape().assert_is_compatible_with(tensor_shape.scalar()) - # Do nothing if we know keep_prob == 1 - if tensor_util.constant_value(keep_prob) == 1: + # Do nothing if we know rate == 0 + if tensor_util.constant_value(rate) == 0: return x noise_shape = _get_noise_shape(x, noise_shape) + keep_prob = 1 - rate # uniform [keep_prob, 1.0 + keep_prob) random_tensor = keep_prob random_tensor += random_ops.random_uniform( @@ -2402,7 +3103,293 @@ def nth_element(input, n, reverse=False, name=None): # pylint: disable=redefine return gen_nn_ops.nth_element(input, n, reverse=reverse, name=name) -@tf_export("nn.conv1d") +@tf_export(v1=["nn.fractional_max_pool"]) +@deprecation.deprecated(date=None, instructions="`seed2` and `deterministic` " + "args are deprecated. Use fractional_max_pool_v2.") +def fractional_max_pool(value, + pooling_ratio, + pseudo_random=False, + overlapping=False, + deterministic=False, + seed=0, + seed2=0, + name=None): # pylint: disable=redefined-builtin + r"""Performs fractional max pooling on the input. + + This is a deprecated version of `fractional_max_pool`. + + Fractional max pooling is slightly different than regular max pooling. In + regular max pooling, you downsize an input set by taking the maximum value of + smaller N x N subsections of the set (often 2x2), and try to reduce the set by + a factor of N, where N is an integer. Fractional max pooling, as you might + expect from the word "fractional", means that the overall reduction ratio N + does not have to be an integer. + + The sizes of the pooling regions are generated randomly but are fairly + uniform. For example, let's look at the height dimension, and the constraints + on the list of rows that will be pool boundaries. + + First we define the following: + + 1. input_row_length : the number of rows from the input set + 2. output_row_length : which will be smaller than the input + 3. alpha = input_row_length / output_row_length : our reduction ratio + 4. K = floor(alpha) + 5. row_pooling_sequence : this is the result list of pool boundary rows + + Then, row_pooling_sequence should satisfy: + + 1. a[0] = 0 : the first value of the sequence is 0 + 2. a[end] = input_row_length : the last value of the sequence is the size + 3. K <= (a[i+1] - a[i]) <= K+1 : all intervals are K or K+1 size + 4. length(row_pooling_sequence) = output_row_length+1 + + For more details on fractional max pooling, see this paper: [Benjamin Graham, + Fractional Max-Pooling](http://arxiv.org/abs/1412.6071) + + Args: + value: A `Tensor`. 4-D with shape `[batch, height, width, channels]`. + pooling_ratio: A list of `floats` that has length >= 4. Pooling ratio for + each dimension of `value`, currently only supports row and col dimension + and should be >= 1.0. For example, a valid pooling ratio looks like [1.0, + 1.44, 1.73, 1.0]. The first and last elements must be 1.0 because we don't + allow pooling on batch and channels dimensions. 1.44 and 1.73 are pooling + ratio on height and width dimensions respectively. + pseudo_random: An optional `bool`. Defaults to `False`. When set to `True`, + generates the pooling sequence in a pseudorandom fashion, otherwise, in a + random fashion. Check paper [Benjamin Graham, Fractional + Max-Pooling](http://arxiv.org/abs/1412.6071) for difference between + pseudorandom and random. + overlapping: An optional `bool`. Defaults to `False`. When set to `True`, + it means when pooling, the values at the boundary of adjacent pooling + cells are used by both cells. For example: + `index 0 1 2 3 4` + `value 20 5 16 3 7` + If the pooling sequence is [0, 2, 4], then 16, at index 2 will be used + twice. The result would be [20, 16] for fractional max pooling. + deterministic: An optional `bool`. Deprecated; use `fractional_max_pool_v2` + instead. + seed: An optional `int`. Defaults to `0`. If set to be non-zero, the + random number generator is seeded by the given seed. Otherwise it is + seeded by a random seed. + seed2: An optional `int`. Deprecated; use `fractional_max_pool_v2` instead. + name: A name for the operation (optional). + + Returns: + A tuple of `Tensor` objects (`output`, `row_pooling_sequence`, + `col_pooling_sequence`). + output: Output `Tensor` after fractional max pooling. Has the same type as + `value`. + row_pooling_sequence: A `Tensor` of type `int64`. + col_pooling_sequence: A `Tensor` of type `int64`. + """ + return gen_nn_ops.fractional_max_pool(value, pooling_ratio, pseudo_random, + overlapping, deterministic, seed, seed2, + name) + + +@tf_export("nn.fractional_max_pool", v1=[]) +def fractional_max_pool_v2(value, + pooling_ratio, + pseudo_random=False, + overlapping=False, + seed=0, + name=None): # pylint: disable=redefined-builtin + r"""Performs fractional max pooling on the input. + + Fractional max pooling is slightly different than regular max pooling. In + regular max pooling, you downsize an input set by taking the maximum value of + smaller N x N subsections of the set (often 2x2), and try to reduce the set by + a factor of N, where N is an integer. Fractional max pooling, as you might + expect from the word "fractional", means that the overall reduction ratio N + does not have to be an integer. + + The sizes of the pooling regions are generated randomly but are fairly + uniform. For example, let's look at the height dimension, and the constraints + on the list of rows that will be pool boundaries. + + First we define the following: + + 1. input_row_length : the number of rows from the input set + 2. output_row_length : which will be smaller than the input + 3. alpha = input_row_length / output_row_length : our reduction ratio + 4. K = floor(alpha) + 5. row_pooling_sequence : this is the result list of pool boundary rows + + Then, row_pooling_sequence should satisfy: + + 1. a[0] = 0 : the first value of the sequence is 0 + 2. a[end] = input_row_length : the last value of the sequence is the size + 3. K <= (a[i+1] - a[i]) <= K+1 : all intervals are K or K+1 size + 4. length(row_pooling_sequence) = output_row_length+1 + + For more details on fractional max pooling, see this paper: [Benjamin Graham, + Fractional Max-Pooling](http://arxiv.org/abs/1412.6071) + + Args: + value: A `Tensor`. 4-D with shape `[batch, height, width, channels]`. + pooling_ratio: A list of `floats` that has length >= 4. Pooling ratio for + each dimension of `value`, currently only supports row and col dimension + and should be >= 1.0. For example, a valid pooling ratio looks like [1.0, + 1.44, 1.73, 1.0]. The first and last elements must be 1.0 because we don't + allow pooling on batch and channels dimensions. 1.44 and 1.73 are pooling + ratio on height and width dimensions respectively. + pseudo_random: An optional `bool`. Defaults to `False`. When set to `True`, + generates the pooling sequence in a pseudorandom fashion, otherwise, in a + random fashion. Check paper [Benjamin Graham, Fractional + Max-Pooling](http://arxiv.org/abs/1412.6071) for difference between + pseudorandom and random. + overlapping: An optional `bool`. Defaults to `False`. When set to `True`, + it means when pooling, the values at the boundary of adjacent pooling + cells are used by both cells. For example: + `index 0 1 2 3 4` + `value 20 5 16 3 7` + If the pooling sequence is [0, 2, 4], then 16, at index 2 will be used + twice. The result would be [20, 16] for fractional max pooling. + seed: An optional `int`. Defaults to `0`. If set to be non-zero, the + random number generator is seeded by the given seed. Otherwise it is + seeded by a random seed. + name: A name for the operation (optional). + + Returns: + A tuple of `Tensor` objects (`output`, `row_pooling_sequence`, + `col_pooling_sequence`). + output: Output `Tensor` after fractional max pooling. Has the same type as + `value`. + row_pooling_sequence: A `Tensor` of type `int64`. + col_pooling_sequence: A `Tensor` of type `int64`. + """ + if seed == 0: + return gen_nn_ops.fractional_max_pool(value, pooling_ratio, pseudo_random, + overlapping, deterministic=False, + seed=0, seed2=0, name=name) + else: + seed1, seed2 = random_seed.get_seed(seed) + return gen_nn_ops.fractional_max_pool(value, pooling_ratio, pseudo_random, + overlapping, deterministic=True, + seed=seed1, seed2=seed2, name=name) + + +@tf_export(v1=["nn.fractional_avg_pool"]) +@deprecation.deprecated(date=None, instructions="`seed2` and `deterministic` " + "args are deprecated. Use fractional_avg_pool_v2.") +def fractional_avg_pool(value, + pooling_ratio, + pseudo_random=False, + overlapping=False, + deterministic=False, + seed=0, + seed2=0, + name=None): # pylint: disable=redefined-builtin + r"""Performs fractional average pooling on the input. + + This is a deprecated version of `fractional_avg_pool`. + + Fractional average pooling is similar to Fractional max pooling in the pooling + region generation step. The only difference is that after pooling regions are + generated, a mean operation is performed instead of a max operation in each + pooling region. + + Args: + value: A `Tensor`. 4-D with shape `[batch, height, width, channels]`. + pooling_ratio: A list of `floats` that has length >= 4. Pooling ratio for + each dimension of `value`, currently only supports row and col dimension + and should be >= 1.0. For example, a valid pooling ratio looks like [1.0, + 1.44, 1.73, 1.0]. The first and last elements must be 1.0 because we don't + allow pooling on batch and channels dimensions. 1.44 and 1.73 are pooling + ratio on height and width dimensions respectively. + pseudo_random: An optional `bool`. Defaults to `False`. When set to `True`, + generates the pooling sequence in a pseudorandom fashion, otherwise, in a + random fashion. Check paper [Benjamin Graham, Fractional + Max-Pooling](http://arxiv.org/abs/1412.6071) for difference between + pseudorandom and random. + overlapping: An optional `bool`. Defaults to `False`. When set to `True`, + it means when pooling, the values at the boundary of adjacent pooling + cells are used by both cells. For example: + `index 0 1 2 3 4` + `value 20 5 16 3 7` + If the pooling sequence is [0, 2, 4], then 16, at index 2 will be used + twice. The result would be [20, 16] for fractional avg pooling. + deterministic: An optional `bool`. Deprecated; use `fractional_avg_pool_v2` + instead. + seed: An optional `int`. Defaults to `0`. If set to be non-zero, the + random number generator is seeded by the given seed. Otherwise it is + seeded by a random seed. + seed2: An optional `int`. Deprecated; use `fractional_avg_pool_v2` instead. + name: A name for the operation (optional). + + Returns: + A tuple of `Tensor` objects (`output`, `row_pooling_sequence`, + `col_pooling_sequence`). + output: Output `Tensor` after fractional avg pooling. Has the same type as + `value`. + row_pooling_sequence: A `Tensor` of type `int64`. + col_pooling_sequence: A `Tensor` of type `int64`. + """ + return gen_nn_ops.fractional_avg_pool(value, pooling_ratio, pseudo_random, + overlapping, deterministic, seed, seed2, + name=name) + + +@tf_export("nn.fractional_avg_pool", v1=[]) +def fractional_avg_pool_v2(value, + pooling_ratio, + pseudo_random=False, + overlapping=False, + seed=0, + name=None): # pylint: disable=redefined-builtin + r"""Performs fractional average pooling on the input. + + Fractional average pooling is similar to Fractional max pooling in the pooling + region generation step. The only difference is that after pooling regions are + generated, a mean operation is performed instead of a max operation in each + pooling region. + + Args: + value: A `Tensor`. 4-D with shape `[batch, height, width, channels]`. + pooling_ratio: A list of `floats` that has length >= 4. Pooling ratio for + each dimension of `value`, currently only supports row and col dimension + and should be >= 1.0. For example, a valid pooling ratio looks like [1.0, + 1.44, 1.73, 1.0]. The first and last elements must be 1.0 because we don't + allow pooling on batch and channels dimensions. 1.44 and 1.73 are pooling + ratio on height and width dimensions respectively. + pseudo_random: An optional `bool`. Defaults to `False`. When set to `True`, + generates the pooling sequence in a pseudorandom fashion, otherwise, in a + random fashion. Check paper [Benjamin Graham, Fractional + Max-Pooling](http://arxiv.org/abs/1412.6071) for difference between + pseudorandom and random. + overlapping: An optional `bool`. Defaults to `False`. When set to `True`, + it means when pooling, the values at the boundary of adjacent pooling + cells are used by both cells. For example: + `index 0 1 2 3 4` + `value 20 5 16 3 7` + If the pooling sequence is [0, 2, 4], then 16, at index 2 will be used + twice. The result would be [20, 16] for fractional avg pooling. + seed: An optional `int`. Defaults to `0`. If set to be non-zero, the + random number generator is seeded by the given seed. Otherwise it is + seeded by a random seed. + name: A name for the operation (optional). + + Returns: + A tuple of `Tensor` objects (`output`, `row_pooling_sequence`, + `col_pooling_sequence`). + output: Output `Tensor` after fractional avg pooling. Has the same type as + `value`. + row_pooling_sequence: A `Tensor` of type `int64`. + col_pooling_sequence: A `Tensor` of type `int64`. + """ + if seed == 0: + return gen_nn_ops.fractional_avg_pool(value, pooling_ratio, pseudo_random, + overlapping, deterministic=False, + seed=0, seed2=0, name=name) + else: + seed1, seed2 = random_seed.get_seed(seed) + return gen_nn_ops.fractional_avg_pool(value, pooling_ratio, pseudo_random, + overlapping, deterministic=True, + seed=seed1, seed2=seed2, name=name) + + +@tf_export(v1=["nn.conv1d"]) @deprecation.deprecated_arg_values( None, "`NCHW` for data_format is deprecated, use `NCW` instead", @@ -2487,6 +3474,64 @@ def conv1d(value, return array_ops.squeeze(result, [spatial_start_dim]) +@tf_export("nn.conv1d", v1=[]) +def conv1d_v2(input, # pylint: disable=redefined-builtin + filters, + stride, + padding, + data_format=None, + name=None): + r"""Computes a 1-D convolution given 3-D input and filter tensors. + + Given an input tensor of shape + [batch, in_width, in_channels] + if data_format is "NWC", or + [batch, in_channels, in_width] + if data_format is "NCW", + and a filter / kernel tensor of shape + [filter_width, in_channels, out_channels], this op reshapes + the arguments to pass them to conv2d to perform the equivalent + convolution operation. + + Internally, this op reshapes the input tensors and invokes `tf.nn.conv2d`. + For example, if `data_format` does not start with "NC", a tensor of shape + [batch, in_width, in_channels] + is reshaped to + [batch, 1, in_width, in_channels], + and the filter is reshaped to + [1, filter_width, in_channels, out_channels]. + The result is then reshaped back to + [batch, out_width, out_channels] + \(where out_width is a function of the stride and padding as in conv2d\) and + returned to the caller. + + Args: + input: A 3D `Tensor`. Must be of type `float16`, `float32`, or `float64`. + filters: A 3D `Tensor`. Must have the same type as `input`. + stride: An `integer`. The number of entries by which + the filter is moved right at each step. + padding: 'SAME' or 'VALID' + data_format: An optional `string` from `"NWC", "NCW"`. Defaults + to `"NWC"`, the data is stored in the order of + [batch, in_width, in_channels]. The `"NCW"` format stores + data as [batch, in_channels, in_width]. + name: A name for the operation (optional). + + Returns: + A `Tensor`. Has the same type as input. + + Raises: + ValueError: if `data_format` is invalid. + """ + return conv1d(input, # pylint: disable=redefined-builtin + filters, + stride, + padding, + use_cudnn_on_gpu=True, + data_format=data_format, + name=name) + + def conv1d_transpose( value, filter, # pylint: disable=redefined-builtin @@ -2602,7 +3647,7 @@ def _calc_dilation2d_flops(graph, node): return ops.OpStats("flops", (output_count * filter_height * filter_width * 2)) -@tf_export("nn.erosion2d") +@tf_export(v1=["nn.erosion2d"]) def erosion2d(value, kernel, strides, rates, padding, name=None): """Computes the grayscale erosion of 4-D `value` and 3-D `kernel` tensors. @@ -2661,6 +3706,75 @@ def erosion2d(value, kernel, strides, rates, padding, name=None): name=name)) +@tf_export("nn.erosion2d", v1=[]) +def erosion2d_v2(value, + filters, + strides, + padding, + data_format, + dilations, + name=None): + """Computes the grayscale erosion of 4-D `value` and 3-D `filters` tensors. + + The `value` tensor has shape `[batch, in_height, in_width, depth]` and the + `filters` tensor has shape `[filters_height, filters_width, depth]`, i.e., + each input channel is processed independently of the others with its own + structuring function. The `output` tensor has shape + `[batch, out_height, out_width, depth]`. The spatial dimensions of the + output tensor depend on the `padding` algorithm. We currently only support the + default "NHWC" `data_format`. + + In detail, the grayscale morphological 2-D erosion is given by: + + output[b, y, x, c] = + min_{dy, dx} value[b, + strides[1] * y - dilations[1] * dy, + strides[2] * x - dilations[2] * dx, + c] - + filters[dy, dx, c] + + Duality: The erosion of `value` by the `filters` is equal to the negation of + the dilation of `-value` by the reflected `filters`. + + Args: + value: A `Tensor`. 4-D with shape `[batch, in_height, in_width, depth]`. + filters: A `Tensor`. Must have the same type as `value`. + 3-D with shape `[filters_height, filters_width, depth]`. + strides: A list of `ints` that has length `>= 4`. + 1-D of length 4. The stride of the sliding window for each dimension of + the input tensor. Must be: `[1, stride_height, stride_width, 1]`. + padding: A `string` from: `"SAME", "VALID"`. + The type of padding algorithm to use. + data_format: A `string`, only `"NHWC"` is currently supported. + dilations: A list of `ints` that has length `>= 4`. + 1-D of length 4. The input stride for atrous morphological dilation. + Must be: `[1, rate_height, rate_width, 1]`. + name: A name for the operation (optional). If not specified "erosion2d" + is used. + + Returns: + A `Tensor`. Has the same type as `value`. + 4-D with shape `[batch, out_height, out_width, depth]`. + + Raises: + ValueError: If the `value` depth does not match `filters`' shape, or if + padding is other than `'VALID'` or `'SAME'`. + """ + if data_format != "NHWC": + raise ValueError("Data formats other than NHWC are not yet supported") + + with ops.name_scope(name, "erosion2d", [value, filters]) as name: + # Reduce erosion to dilation by duality. + return math_ops.negative( + gen_nn_ops.dilation2d( + input=math_ops.negative(value), + filter=array_ops.reverse_v2(filters, [0, 1]), + strides=strides, + rates=dilations, + padding=padding, + name=name)) + + @tf_export("math.in_top_k", "nn.in_top_k") def in_top_k(predictions, targets, k, name=None): r"""Says whether the targets are in the top `K` predictions. @@ -2693,3 +3807,10 @@ def in_top_k(predictions, targets, k, name=None): """ with ops.name_scope(name, "in_top_k"): return gen_nn_ops.in_top_kv2(predictions, targets, k, name=name) + + +tf_export(v1=["nn.quantized_avg_pool"])(gen_nn_ops.quantized_avg_pool) +tf_export(v1=["nn.quantized_conv2d"])(gen_nn_ops.quantized_conv2d) +tf_export(v1=["nn.quantized_relu_x"])(gen_nn_ops.quantized_relu_x) +tf_export(v1=["nn.quantized_max_pool"])(gen_nn_ops.quantized_max_pool) + diff --git a/tensorflow/python/ops/nn_test.py b/tensorflow/python/ops/nn_test.py index 152b2020ebb..0336d0d27cb 100644 --- a/tensorflow/python/ops/nn_test.py +++ b/tensorflow/python/ops/nn_test.py @@ -49,36 +49,39 @@ class ZeroFractionTest(test_lib.TestCase): nonzeros = np.count_nonzero(x.flatten()) return 1.0 - nonzeros / total_elements + @test_util.run_deprecated_v1 def testZeroFraction(self): x_shape = [5, 17] x_np = np.random.randint(0, 2, size=x_shape).astype(np.float32) y_np = self._ZeroFraction(x_np) - with self.cached_session(): - x_tf = constant_op.constant(x_np) - x_tf.set_shape(x_shape) - y_tf = nn_impl.zero_fraction(x_tf) - y_tf_np = y_tf.eval() + + x_tf = constant_op.constant(x_np) + x_tf.set_shape(x_shape) + y_tf = nn_impl.zero_fraction(x_tf) + y_tf_np = self.evaluate(y_tf) + eps = 1e-8 self.assertAllClose(y_tf_np, y_np, eps) + @test_util.run_deprecated_v1 def testZeroFractionEmpty(self): - with self.cached_session(): - x = np.zeros(0) - y = nn_impl.zero_fraction(x).eval() - self.assertTrue(np.isnan(y)) + x = np.zeros(0) + y = self.evaluate(nn_impl.zero_fraction(x)) + self.assertTrue(np.isnan(y)) + @test_util.run_deprecated_v1 def testZeroFraction2_27Zeros(self): sparsity = nn_impl.zero_fraction( array_ops.zeros([int(2**27 * 1.01)], dtype=dtypes.int8)) - with self.cached_session(): - self.assertAllClose(1.0, sparsity.eval()) + self.assertAllClose(1.0, self.evaluate(sparsity)) + @test_util.run_deprecated_v1 def testZeroFraction2_27Ones(self): sparsity = nn_impl.zero_fraction( array_ops.ones([int(2**27 * 1.01)], dtype=dtypes.int8)) - with self.cached_session(): - self.assertAllClose(0.0, sparsity.eval()) + self.assertAllClose(0.0, self.evaluate(sparsity)) + @test_util.run_deprecated_v1 def testUnknownSize(self): value = array_ops.placeholder(dtype=dtypes.float32) sparsity = nn_impl.zero_fraction(value) @@ -103,8 +106,8 @@ class SoftmaxTest(test_lib.TestCase, parameterized.TestCase): x_np = np.random.randn(*x_shape).astype(np.float32) y_np = self._softmax(x_np) x_tf = constant_op.constant(x_np) - y_tf = nn_ops.softmax(x_tf) - y_tf_last_dim = nn_ops.softmax(x_tf, 1) + y_tf = nn_ops.softmax_v2(x_tf) + y_tf_last_dim = nn_ops.softmax_v2(x_tf, 1) y_tf_np = self.evaluate(y_tf) y_tf_last_dim_np = self.evaluate(y_tf_last_dim) eps = 1e-3 @@ -113,9 +116,9 @@ class SoftmaxTest(test_lib.TestCase, parameterized.TestCase): def testSoftmaxAxes(self): arr = np.linspace(0., 1, 12).reshape(3, 4) - x_neg_axis = nn_ops.softmax(arr, axis=-2) - y_pos_axis = nn_ops.softmax(arr, axis=0) - z_gt_axis = nn_ops.softmax(arr, axis=0) + x_neg_axis = nn_ops.softmax_v2(arr, axis=-2) + y_pos_axis = nn_ops.softmax_v2(arr, axis=0) + z_gt_axis = nn_ops.softmax_v2(arr, axis=0) x_neg_axis_tf = self.evaluate(x_neg_axis) y_pos_axis_tf = self.evaluate(y_pos_axis) z_gt_axis_tf = self.evaluate(z_gt_axis) @@ -124,11 +127,12 @@ class SoftmaxTest(test_lib.TestCase, parameterized.TestCase): self.assertAllClose(y_pos_axis_tf, z_gt_axis_tf, eps) @parameterized.parameters(((5, 10),), ((2, 3, 4),)) + @test_util.run_deprecated_v1 def testGradient(self, x_shape): x_np = np.random.randn(*x_shape).astype(np.float64) with self.cached_session(): x_tf = constant_op.constant(x_np) - y_tf = nn_ops.softmax(x_tf) + y_tf = nn_ops.softmax_v2(x_tf) err = gradient_checker.compute_gradient_error(x_tf, x_shape, y_tf, x_shape) eps = 2e-8 @@ -159,6 +163,7 @@ class LogPoissonLossTest(test_lib.TestCase): self.assertAllClose(y_tf_np, y_np, eps) self.assertAllClose(y_tf_np_stirling, y_np_stirling, eps) + @test_util.run_deprecated_v1 def testGradient(self): x_shape = [5, 10] x_np = np.random.randn(*x_shape).astype(np.float64) @@ -191,16 +196,16 @@ class LogSoftmaxTest(test_lib.TestCase, parameterized.TestCase): x_np = np.random.randn(*x_shape).astype(np.float32) y_np = self._log_softmax(x_np) x_tf = constant_op.constant(x_np) - y_tf = nn_ops.log_softmax(x_tf) + y_tf = nn_ops.log_softmax_v2(x_tf) y_tf_np = self.evaluate(y_tf) eps = 1e-3 self.assertAllClose(y_tf_np, y_np, eps) def testLogSoftmaxAxes(self): arr = np.linspace(0., 1, 12).reshape(3, 4) - x_neg_axis = nn_ops.log_softmax(arr, axis=-2) - y_pos_axis = nn_ops.log_softmax(arr, axis=0) - z_gt_axis = nn_ops.log_softmax(arr, axis=0) + x_neg_axis = nn_ops.log_softmax_v2(arr, axis=-2) + y_pos_axis = nn_ops.log_softmax_v2(arr, axis=0) + z_gt_axis = nn_ops.log_softmax_v2(arr, axis=0) x_neg_axis_tf = self.evaluate(x_neg_axis) y_pos_axis_tf = self.evaluate(y_pos_axis) z_gt_axis_tf = self.evaluate(z_gt_axis) @@ -209,11 +214,12 @@ class LogSoftmaxTest(test_lib.TestCase, parameterized.TestCase): self.assertAllClose(y_pos_axis_tf, z_gt_axis_tf, eps) @parameterized.parameters(((5, 10),), ((2, 3, 4),)) + @test_util.run_deprecated_v1 def testGradient(self, x_shape): x_np = np.random.randn(*x_shape).astype(np.float64) with self.cached_session(): x_tf = constant_op.constant(x_np) - y_tf = nn_ops.log_softmax(x_tf) + y_tf = nn_ops.log_softmax_v2(x_tf) err = gradient_checker.compute_gradient_error(x_tf, x_shape, y_tf, x_shape) eps = 1e-7 @@ -231,6 +237,7 @@ class L2LossTest(test_lib.TestCase): value = self.evaluate(l2loss) self.assertAllClose(7.0, value) + @test_util.run_deprecated_v1 def testGradient(self): x_shape = [20, 7, 3] np.random.seed(1) # Make it reproducible. @@ -264,7 +271,7 @@ class L2NormalizeTest(test_lib.TestCase): for dim in range(len(x_shape)): y_np = self._l2Normalize(x_np, dim) x_tf = constant_op.constant(x_np, name="x") - y_tf = nn_impl.l2_normalize(x_tf, dim) + y_tf = nn_impl.l2_normalize_v2(x_tf, dim) self.assertAllClose(y_np, self.evaluate(y_tf)) @test_util.run_in_graph_and_eager_modes @@ -275,9 +282,10 @@ class L2NormalizeTest(test_lib.TestCase): dim = [1, 2] y_np = self._l2Normalize(x_np, dim) x_tf = constant_op.constant(x_np, name="x") - y_tf = nn_impl.l2_normalize(x_tf, dim) + y_tf = nn_impl.l2_normalize_v2(x_tf, dim) self.assertAllClose(y_np, self.evaluate(y_tf)) + @test_util.run_deprecated_v1 def testL2NormalizeGradient(self): x_shape = [20, 7, 3] np.random.seed(1) @@ -285,7 +293,7 @@ class L2NormalizeTest(test_lib.TestCase): for dim in range(len(x_shape)): with self.cached_session(): x_tf = constant_op.constant(x_np, name="x") - y_tf = nn_impl.l2_normalize(x_tf, dim) + y_tf = nn_impl.l2_normalize_v2(x_tf, dim) err = gradient_checker.compute_gradient_error(x_tf, x_shape, y_tf, x_shape) print("L2Normalize gradient err = %g " % err) @@ -302,19 +310,18 @@ class DropoutTest(test_lib.TestCase): y_dim = 30 num_iter = 10 for keep_prob in [0.1, 0.5, 0.8]: - with self.cached_session(): - t = constant_op.constant( - 1.0, shape=[x_dim, y_dim], dtype=dtypes.float32) - dropout = nn_ops.dropout(t, keep_prob) - final_count = 0 - self.assertEqual([x_dim, y_dim], dropout.get_shape()) - for _ in xrange(0, num_iter): - value = dropout.eval() - final_count += np.count_nonzero(value) - # Verifies that there are only two values: 0 and 1/keep_prob. - sorted_value = np.unique(np.sort(value)) - self.assertEqual(0, sorted_value[0]) - self.assertAllClose(1 / keep_prob, sorted_value[1]) + t = constant_op.constant(1.0, shape=[x_dim, y_dim], dtype=dtypes.float32) + dropout = nn_ops.dropout(t, keep_prob) + final_count = 0 + self.assertEqual([x_dim, y_dim], dropout.get_shape()) + for _ in xrange(0, num_iter): + value = self.evaluate(dropout) + final_count += np.count_nonzero(value) + # Verifies that there are only two values: 0 and 1/keep_prob. + sorted_value = np.unique(np.sort(value)) + self.assertEqual(0, sorted_value[0]) + self.assertAllClose(1 / keep_prob, sorted_value[1]) + # Check that we are in the 15% error range expected_count = x_dim * y_dim * keep_prob * num_iter rel_error = math.fabs(final_count - expected_count) / expected_count @@ -330,19 +337,18 @@ class DropoutTest(test_lib.TestCase): y_dim = 3 num_iter = 10 for keep_prob in [0.1, 0.5, 0.8]: - with self.cached_session(): - t = constant_op.constant( - 1.0, shape=[x_dim, y_dim], dtype=dtypes.float32) - dropout = nn_ops.dropout(t, keep_prob, noise_shape=[x_dim, 1]) - self.assertEqual([x_dim, y_dim], dropout.get_shape()) - final_count = 0 - for _ in xrange(0, num_iter): - value = dropout.eval() - final_count += np.count_nonzero(value) - # Verifies that there are only two values: 0 and 1/keep_prob. - sorted_value = np.unique(np.sort(value)) - self.assertEqual(0, sorted_value[0]) - self.assertAllClose(1 / keep_prob, sorted_value[1]) + t = constant_op.constant(1.0, shape=[x_dim, y_dim], dtype=dtypes.float32) + dropout = nn_ops.dropout(t, keep_prob, noise_shape=[x_dim, 1]) + self.assertEqual([x_dim, y_dim], dropout.get_shape()) + final_count = 0 + for _ in xrange(0, num_iter): + value = self.evaluate(dropout) + final_count += np.count_nonzero(value) + # Verifies that there are only two values: 0 and 1/keep_prob. + sorted_value = np.unique(np.sort(value)) + self.assertEqual(0, sorted_value[0]) + self.assertAllClose(1 / keep_prob, sorted_value[1]) + # Check that we are in the 15% error range expected_count = x_dim * y_dim * keep_prob * num_iter rel_error = math.fabs(final_count - expected_count) / expected_count @@ -355,18 +361,17 @@ class DropoutTest(test_lib.TestCase): y_dim = 30 num_iter = 10 for keep_prob in [0.1, 0.5, 0.8]: - with self.cached_session(): - t = constant_op.constant( - 1.0, shape=[x_dim, y_dim], dtype=dtypes.float32) - dropout = nn_ops.dropout(t, keep_prob, noise_shape=[x_dim, 1]) - self.assertEqual([x_dim, y_dim], dropout.get_shape()) - for _ in xrange(0, num_iter): - value = dropout.eval() - # Verifies that each y column as only one type of activation. - for i in xrange(x_dim): - sorted_value = np.unique(np.sort(value[i, :])) - self.assertEqual(sorted_value.size, 1) + t = constant_op.constant(1.0, shape=[x_dim, y_dim], dtype=dtypes.float32) + dropout = nn_ops.dropout(t, keep_prob, noise_shape=[x_dim, 1]) + self.assertEqual([x_dim, y_dim], dropout.get_shape()) + for _ in xrange(0, num_iter): + value = self.evaluate(dropout) + # Verifies that each y column as only one type of activation. + for i in xrange(x_dim): + sorted_value = np.unique(np.sort(value[i, :])) + self.assertEqual(sorted_value.size, 1) + @test_util.run_deprecated_v1 def testDropoutPlaceholderKeepProb(self): # Runs dropout with 0-1 tensor 10 times, sum the number of ones and validate # that it is producing approximately the right number of ones over a large @@ -395,6 +400,7 @@ class DropoutTest(test_lib.TestCase): print(rel_error) self.assertTrue(rel_error < 0.15) + @test_util.run_deprecated_v1 def testShapedDropoutUnknownShape(self): x_dim = 40 y_dim = 30 @@ -409,26 +415,26 @@ class DropoutTest(test_lib.TestCase): y_dim = 3 num_iter = 10 for keep_prob in [0.1, 0.5, 0.8]: - with self.cached_session(): - t = constant_op.constant( - 1.0, shape=[x_dim, y_dim], dtype=dtypes.float32) - # Set noise_shape=[None, 1] which means [x_dim, 1]. - dropout = nn_ops.dropout(t, keep_prob, noise_shape=[None, 1]) - self.assertEqual([x_dim, y_dim], dropout.get_shape()) - final_count = 0 - for _ in xrange(0, num_iter): - value = dropout.eval() - final_count += np.count_nonzero(value) - # Verifies that there are only two values: 0 and 1/keep_prob. - sorted_value = np.unique(np.sort(value)) - self.assertEqual(0, sorted_value[0]) - self.assertAllClose(1 / keep_prob, sorted_value[1]) + t = constant_op.constant(1.0, shape=[x_dim, y_dim], dtype=dtypes.float32) + # Set noise_shape=[None, 1] which means [x_dim, 1]. + dropout = nn_ops.dropout(t, keep_prob, noise_shape=[None, 1]) + self.assertEqual([x_dim, y_dim], dropout.get_shape()) + final_count = 0 + for _ in xrange(0, num_iter): + value = self.evaluate(dropout) + final_count += np.count_nonzero(value) + # Verifies that there are only two values: 0 and 1/keep_prob. + sorted_value = np.unique(np.sort(value)) + self.assertEqual(0, sorted_value[0]) + self.assertAllClose(1 / keep_prob, sorted_value[1]) + # Check that we are in the 15% error range expected_count = x_dim * y_dim * keep_prob * num_iter rel_error = math.fabs(final_count - expected_count) / expected_count print(rel_error) self.assertTrue(rel_error < 0.15) + @test_util.run_deprecated_v1 def testInvalidKeepProb(self): x_dim = 40 y_dim = 30 @@ -444,6 +450,18 @@ class DropoutTest(test_lib.TestCase): with self.assertRaises(ValueError): nn_ops.dropout(t, array_ops.placeholder(dtypes.float32, shape=[2])) + def testInvalidRate(self): + x_dim = 40 + y_dim = 30 + t = constant_op.constant(1.0, shape=[x_dim, y_dim], dtype=dtypes.float32) + with self.assertRaises(ValueError): + nn_ops.dropout_v2(t, -1.0) + with self.assertRaises(ValueError): + nn_ops.dropout_v2(t, 1.1) + with self.assertRaises(ValueError): + nn_ops.dropout_v2(t, [0.0, 1.0]) + + @test_util.run_deprecated_v1 def testShapedDropoutShapeError(self): # Runs shaped dropout and verifies an error is thrown on misshapen noise. x_dim = 40 @@ -466,9 +484,11 @@ class DropoutTest(test_lib.TestCase): def testNoDropoutFast(self): x = array_ops.zeros((5,)) - for p in 1, constant_op.constant(1.0): - y = nn_ops.dropout(x, keep_prob=p) - self.assertTrue(x is y) + y = nn_ops.dropout(x, keep_prob=1) + self.assertTrue(x is y) + + y = nn_ops.dropout_v2(x, rate=0) + self.assertTrue(x is y) def testDropoutWithIntegerInputs(self): x = constant_op.constant([1, 1, 1, 1, 1]) @@ -563,78 +583,78 @@ class ComputeSampledLogitsTest(test_lib.TestCase): initializer=constant_op.constant(biases)) with self.session(graph=g) as sess: variables.global_variables_initializer().run() - return sess.run([list(sharded_weights), list(sharded_biases)]) + return self.evaluate([list(sharded_weights), list(sharded_biases)]) def testShapes(self): np.random.seed(0) num_classes = 5 batch_size = 3 - with self.cached_session() as sess: - for num_true in range(1, 5): - labels = np.random.randint( - low=0, high=num_classes, size=batch_size * num_true) - (weights, biases, hidden_acts, sampled_vals, exp_logits, - exp_labels) = self._GenerateTestData( - num_classes=num_classes, - dim=10, - batch_size=batch_size, - num_true=num_true, - labels=labels, - sampled=[1, 0, 2, 3], - subtract_log_q=False) - logits_tensor, labels_tensor = _compute_sampled_logits( - weights=constant_op.constant(weights), - biases=constant_op.constant(biases), - labels=constant_op.constant( - labels, dtype=dtypes.int64, shape=(batch_size, num_true)), - inputs=constant_op.constant(hidden_acts), - num_sampled=4, - num_classes=num_classes, - num_true=num_true, - sampled_values=sampled_vals, - subtract_log_q=False, - remove_accidental_hits=False, - partition_strategy="div", - name="sampled_logits_basic_num_true_%d" % num_true) - got_logits, got_labels = sess.run([logits_tensor, labels_tensor]) - self.assertEqual(exp_logits.shape, got_logits.shape, self._eps) - self.assertEqual(exp_labels.shape, got_labels.shape, self._eps) + + for num_true in range(1, 5): + labels = np.random.randint( + low=0, high=num_classes, size=batch_size * num_true) + (weights, biases, hidden_acts, sampled_vals, exp_logits, + exp_labels) = self._GenerateTestData( + num_classes=num_classes, + dim=10, + batch_size=batch_size, + num_true=num_true, + labels=labels, + sampled=[1, 0, 2, 3], + subtract_log_q=False) + logits_tensor, labels_tensor = _compute_sampled_logits( + weights=constant_op.constant(weights), + biases=constant_op.constant(biases), + labels=constant_op.constant( + labels, dtype=dtypes.int64, shape=(batch_size, num_true)), + inputs=constant_op.constant(hidden_acts), + num_sampled=4, + num_classes=num_classes, + num_true=num_true, + sampled_values=sampled_vals, + subtract_log_q=False, + remove_accidental_hits=False, + partition_strategy="div", + name="sampled_logits_basic_num_true_%d" % num_true) + got_logits, got_labels = self.evaluate([logits_tensor, labels_tensor]) + self.assertEqual(exp_logits.shape, got_logits.shape, self._eps) + self.assertEqual(exp_labels.shape, got_labels.shape, self._eps) def testBasic(self): """Without accidental hit removal or subtract_log_q.""" np.random.seed(0) num_classes = 5 batch_size = 3 - with self.cached_session() as sess: - for num_true in range(1, 5): - labels = np.random.randint( - low=0, high=num_classes, size=batch_size * num_true) - (weights, biases, hidden_acts, sampled_vals, exp_logits, - exp_labels) = self._GenerateTestData( - num_classes=num_classes, - dim=10, - batch_size=batch_size, - num_true=num_true, - labels=labels, - sampled=[1, 0, 2, 3], - subtract_log_q=False) - logits_tensor, labels_tensor = _compute_sampled_logits( - weights=constant_op.constant(weights), - biases=constant_op.constant(biases), - labels=constant_op.constant( - labels, dtype=dtypes.int64, shape=(batch_size, num_true)), - inputs=constant_op.constant(hidden_acts), - num_sampled=4, - num_classes=num_classes, - num_true=num_true, - sampled_values=sampled_vals, - subtract_log_q=False, - remove_accidental_hits=False, - partition_strategy="div", - name="sampled_logits_basic_num_true_%d" % num_true) - got_logits, got_labels = sess.run([logits_tensor, labels_tensor]) - self.assertAllClose(exp_logits, got_logits, self._eps) - self.assertAllClose(exp_labels, got_labels, self._eps) + + for num_true in range(1, 5): + labels = np.random.randint( + low=0, high=num_classes, size=batch_size * num_true) + (weights, biases, hidden_acts, sampled_vals, exp_logits, + exp_labels) = self._GenerateTestData( + num_classes=num_classes, + dim=10, + batch_size=batch_size, + num_true=num_true, + labels=labels, + sampled=[1, 0, 2, 3], + subtract_log_q=False) + logits_tensor, labels_tensor = _compute_sampled_logits( + weights=constant_op.constant(weights), + biases=constant_op.constant(biases), + labels=constant_op.constant( + labels, dtype=dtypes.int64, shape=(batch_size, num_true)), + inputs=constant_op.constant(hidden_acts), + num_sampled=4, + num_classes=num_classes, + num_true=num_true, + sampled_values=sampled_vals, + subtract_log_q=False, + remove_accidental_hits=False, + partition_strategy="div", + name="sampled_logits_basic_num_true_%d" % num_true) + got_logits, got_labels = self.evaluate([logits_tensor, labels_tensor]) + self.assertAllClose(exp_logits, got_logits, self._eps) + self.assertAllClose(exp_labels, got_labels, self._eps) def testAccidentalHitRemoval(self): """With accidental hit removal, no subtract_log_q.""" @@ -642,118 +662,118 @@ class ComputeSampledLogitsTest(test_lib.TestCase): num_classes = 5 batch_size = 3 sampled = [1, 0, 2, 3] - with self.cached_session(): - for num_true in range(1, 5): - labels = np.random.randint( - low=0, high=num_classes, size=batch_size * num_true) - (weights, biases, hidden_acts, sampled_vals, _, - _) = self._GenerateTestData( - num_classes=num_classes, - dim=10, - batch_size=batch_size, - num_true=num_true, - labels=labels, - sampled=sampled, - subtract_log_q=False) - logits_tensor, _ = _compute_sampled_logits( - weights=constant_op.constant(weights), - biases=constant_op.constant(biases), - labels=constant_op.constant( - labels, dtype=dtypes.int64, shape=(batch_size, num_true)), - inputs=constant_op.constant(hidden_acts), - num_sampled=len(sampled), - num_classes=num_classes, - num_true=num_true, - sampled_values=sampled_vals, - subtract_log_q=False, - remove_accidental_hits=True, - partition_strategy="div", - name="sampled_logits_accidental_hit_removal_num_true_%d" % num_true) - # Test that the exponentiated logits of accidental hits are near 0. - # First we need to find the hits in this random test run: - labels_reshape = labels.reshape((batch_size, num_true)) - got_logits = logits_tensor.eval() - for row in xrange(batch_size): - row_labels = labels_reshape[row, :] - for col in xrange(len(sampled)): - if sampled[col] in row_labels: - # We need to add the num_true_test offset into logits_* - self.assertNear( - np.exp(got_logits[row, col + num_true]), 0., self._eps) + + for num_true in range(1, 5): + labels = np.random.randint( + low=0, high=num_classes, size=batch_size * num_true) + (weights, biases, hidden_acts, sampled_vals, _, + _) = self._GenerateTestData( + num_classes=num_classes, + dim=10, + batch_size=batch_size, + num_true=num_true, + labels=labels, + sampled=sampled, + subtract_log_q=False) + logits_tensor, _ = _compute_sampled_logits( + weights=constant_op.constant(weights), + biases=constant_op.constant(biases), + labels=constant_op.constant( + labels, dtype=dtypes.int64, shape=(batch_size, num_true)), + inputs=constant_op.constant(hidden_acts), + num_sampled=len(sampled), + num_classes=num_classes, + num_true=num_true, + sampled_values=sampled_vals, + subtract_log_q=False, + remove_accidental_hits=True, + partition_strategy="div", + name="sampled_logits_accidental_hit_removal_num_true_%d" % num_true) + # Test that the exponentiated logits of accidental hits are near 0. + # First we need to find the hits in this random test run: + labels_reshape = labels.reshape((batch_size, num_true)) + got_logits = self.evaluate(logits_tensor) + for row in xrange(batch_size): + row_labels = labels_reshape[row, :] + for col in xrange(len(sampled)): + if sampled[col] in row_labels: + # We need to add the num_true_test offset into logits_* + self.assertNear( + np.exp(got_logits[row, col + num_true]), 0., self._eps) def testSubtractLogQ(self): """With subtract_log_q, no accidental hit removal.""" np.random.seed(0) num_classes = 5 batch_size = 3 - with self.cached_session() as sess: - for num_true in range(1, 5): - labels = np.random.randint( - low=0, high=num_classes, size=batch_size * num_true) - (weights, biases, hidden_acts, sampled_vals, exp_logits, - exp_labels) = self._GenerateTestData( - num_classes=num_classes, - dim=10, - batch_size=batch_size, - num_true=num_true, - labels=labels, - sampled=[1, 0, 2, 3], - subtract_log_q=True) - logits_tensor, labels_tensor = _compute_sampled_logits( - weights=constant_op.constant(weights), - biases=constant_op.constant(biases), - labels=constant_op.constant( - labels, dtype=dtypes.int64, shape=(batch_size, num_true)), - inputs=constant_op.constant(hidden_acts), - num_sampled=4, - num_classes=num_classes, - num_true=num_true, - sampled_values=sampled_vals, - subtract_log_q=True, - remove_accidental_hits=False, - partition_strategy="div", - name="sampled_logits_subtract_log_q_num_true_%d" % num_true) - got_logits, got_labels = sess.run([logits_tensor, labels_tensor]) - self.assertAllClose(exp_logits, got_logits, self._eps) - self.assertAllClose(exp_labels, got_labels, self._eps) + + for num_true in range(1, 5): + labels = np.random.randint( + low=0, high=num_classes, size=batch_size * num_true) + (weights, biases, hidden_acts, sampled_vals, exp_logits, + exp_labels) = self._GenerateTestData( + num_classes=num_classes, + dim=10, + batch_size=batch_size, + num_true=num_true, + labels=labels, + sampled=[1, 0, 2, 3], + subtract_log_q=True) + logits_tensor, labels_tensor = _compute_sampled_logits( + weights=constant_op.constant(weights), + biases=constant_op.constant(biases), + labels=constant_op.constant( + labels, dtype=dtypes.int64, shape=(batch_size, num_true)), + inputs=constant_op.constant(hidden_acts), + num_sampled=4, + num_classes=num_classes, + num_true=num_true, + sampled_values=sampled_vals, + subtract_log_q=True, + remove_accidental_hits=False, + partition_strategy="div", + name="sampled_logits_subtract_log_q_num_true_%d" % num_true) + got_logits, got_labels = self.evaluate([logits_tensor, labels_tensor]) + self.assertAllClose(exp_logits, got_logits, self._eps) + self.assertAllClose(exp_labels, got_labels, self._eps) def testSharded(self): """With sharded weights and sharded biases.""" np.random.seed(0) num_classes = 5 batch_size = 3 - with self.cached_session() as sess: - for num_true in range(1, 5): - labels = np.random.randint( - low=0, high=num_classes, size=batch_size * num_true) - (weights, biases, hidden_acts, sampled_vals, exp_logits, - exp_labels) = self._GenerateTestData( - num_classes=num_classes, - dim=10, - batch_size=batch_size, - num_true=num_true, - labels=labels, - sampled=[1, 0, 2, 3], - subtract_log_q=False) - weight_shards, bias_shards = self._ShardTestEmbeddings( - weights, biases, num_shards=3) - logits_tensor, labels_tensor = _compute_sampled_logits( - weights=[constant_op.constant(shard) for shard in weight_shards], - biases=[constant_op.constant(shard) for shard in bias_shards], - labels=constant_op.constant( - labels, dtype=dtypes.int64, shape=(batch_size, num_true)), - inputs=constant_op.constant(hidden_acts), - num_sampled=4, - num_classes=num_classes, - num_true=num_true, - sampled_values=sampled_vals, - subtract_log_q=False, - remove_accidental_hits=False, - partition_strategy="div", - name="sampled_logits_sharded_num_true_%d" % num_true) - got_logits, got_labels = sess.run([logits_tensor, labels_tensor]) - self.assertAllClose(exp_logits, got_logits, self._eps) - self.assertAllClose(exp_labels, got_labels, self._eps) + + for num_true in range(1, 5): + labels = np.random.randint( + low=0, high=num_classes, size=batch_size * num_true) + (weights, biases, hidden_acts, sampled_vals, exp_logits, + exp_labels) = self._GenerateTestData( + num_classes=num_classes, + dim=10, + batch_size=batch_size, + num_true=num_true, + labels=labels, + sampled=[1, 0, 2, 3], + subtract_log_q=False) + weight_shards, bias_shards = self._ShardTestEmbeddings( + weights, biases, num_shards=3) + logits_tensor, labels_tensor = _compute_sampled_logits( + weights=[constant_op.constant(shard) for shard in weight_shards], + biases=[constant_op.constant(shard) for shard in bias_shards], + labels=constant_op.constant( + labels, dtype=dtypes.int64, shape=(batch_size, num_true)), + inputs=constant_op.constant(hidden_acts), + num_sampled=4, + num_classes=num_classes, + num_true=num_true, + sampled_values=sampled_vals, + subtract_log_q=False, + remove_accidental_hits=False, + partition_strategy="div", + name="sampled_logits_sharded_num_true_%d" % num_true) + got_logits, got_labels = self.evaluate([logits_tensor, labels_tensor]) + self.assertAllClose(exp_logits, got_logits, self._eps) + self.assertAllClose(exp_labels, got_labels, self._eps) def testNCELoss(self): # A simple test to verify the numerics. @@ -782,35 +802,32 @@ class ComputeSampledLogitsTest(test_lib.TestCase): exp_nce_loss = np.sum( _SigmoidCrossEntropyWithLogits(exp_logits, exp_labels), 1) - with self.cached_session(): - got_nce_loss = nn_impl.nce_loss( - weights=constant_op.constant(weights), - biases=constant_op.constant(biases), - labels=constant_op.constant(labels, shape=(batch_size, 1)), - inputs=constant_op.constant(hidden_acts), - num_sampled=4, - num_classes=num_classes, - num_true=1, - sampled_values=sampled_vals, - partition_strategy="div") + got_nce_loss = nn_impl.nce_loss_v2( + weights=constant_op.constant(weights), + biases=constant_op.constant(biases), + labels=constant_op.constant(labels, shape=(batch_size, 1)), + inputs=constant_op.constant(hidden_acts), + num_sampled=4, + num_classes=num_classes, + num_true=1, + sampled_values=sampled_vals) - self.assertAllClose(exp_nce_loss, got_nce_loss.eval(), 1e-4) + self.assertAllClose(exp_nce_loss, self.evaluate(got_nce_loss), 1e-4) - # Test with sharded weights and sharded biases. - weight_shards, bias_shards = self._ShardTestEmbeddings( - weights, biases, num_shards=3) - got_nce_loss = nn_impl.nce_loss( - weights=[constant_op.constant(shard) for shard in weight_shards], - biases=[constant_op.constant(shard) for shard in bias_shards], - labels=constant_op.constant(labels, shape=(batch_size, 1)), - inputs=constant_op.constant(hidden_acts), - num_sampled=4, - num_classes=num_classes, - num_true=1, - sampled_values=sampled_vals, - partition_strategy="div") + # Test with sharded weights and sharded biases. + weight_shards, bias_shards = self._ShardTestEmbeddings( + weights, biases, num_shards=3) + got_nce_loss = nn_impl.nce_loss_v2( + weights=[constant_op.constant(shard) for shard in weight_shards], + biases=[constant_op.constant(shard) for shard in bias_shards], + labels=constant_op.constant(labels, shape=(batch_size, 1)), + inputs=constant_op.constant(hidden_acts), + num_sampled=4, + num_classes=num_classes, + num_true=1, + sampled_values=sampled_vals) - self.assertAllClose(exp_nce_loss, got_nce_loss.eval(), 1e-4) + self.assertAllClose(exp_nce_loss, self.evaluate(got_nce_loss), 1e-4) def testSampledSoftmaxLoss(self): # A simple test to verify the numerics. @@ -839,39 +856,38 @@ class ComputeSampledLogitsTest(test_lib.TestCase): exp_sampled_softmax_loss = _SoftmaxCrossEntropyWithLogits( exp_logits, exp_labels) - with self.cached_session(): - got_sampled_softmax_loss = nn_impl.sampled_softmax_loss( - weights=constant_op.constant(weights), - biases=constant_op.constant(biases), - labels=constant_op.constant(labels, shape=(batch_size, 1)), - inputs=constant_op.constant(hidden_acts), - num_sampled=4, - num_classes=num_classes, - num_true=1, - sampled_values=sampled_vals, - remove_accidental_hits=False, - partition_strategy="div") + got_sampled_softmax_loss = nn_impl.sampled_softmax_loss( + weights=constant_op.constant(weights), + biases=constant_op.constant(biases), + labels=constant_op.constant(labels, shape=(batch_size, 1)), + inputs=constant_op.constant(hidden_acts), + num_sampled=4, + num_classes=num_classes, + num_true=1, + sampled_values=sampled_vals, + remove_accidental_hits=False, + partition_strategy="div") - self.assertAllClose(exp_sampled_softmax_loss, - got_sampled_softmax_loss.eval(), 1e-4) + self.assertAllClose(exp_sampled_softmax_loss, + self.evaluate(got_sampled_softmax_loss), 1e-4) - # Test with sharded weights and sharded biases. - weight_shards, bias_shards = self._ShardTestEmbeddings( - weights, biases, num_shards=3) - got_sampled_softmax_loss = nn_impl.sampled_softmax_loss( - weights=[constant_op.constant(shard) for shard in weight_shards], - biases=[constant_op.constant(shard) for shard in bias_shards], - labels=constant_op.constant(labels, shape=(batch_size, 1)), - inputs=constant_op.constant(hidden_acts), - num_sampled=4, - num_classes=num_classes, - num_true=1, - sampled_values=sampled_vals, - remove_accidental_hits=False, - partition_strategy="div") + # Test with sharded weights and sharded biases. + weight_shards, bias_shards = self._ShardTestEmbeddings( + weights, biases, num_shards=3) + got_sampled_softmax_loss = nn_impl.sampled_softmax_loss( + weights=[constant_op.constant(shard) for shard in weight_shards], + biases=[constant_op.constant(shard) for shard in bias_shards], + labels=constant_op.constant(labels, shape=(batch_size, 1)), + inputs=constant_op.constant(hidden_acts), + num_sampled=4, + num_classes=num_classes, + num_true=1, + sampled_values=sampled_vals, + remove_accidental_hits=False, + partition_strategy="div") - self.assertAllClose(exp_sampled_softmax_loss, - got_sampled_softmax_loss.eval(), 1e-4) + self.assertAllClose(exp_sampled_softmax_loss, + self.evaluate(got_sampled_softmax_loss), 1e-4) def testSampledSoftmaxLossBf16(self): # A simple test to verify the numerics for bfloat16. @@ -900,29 +916,30 @@ class ComputeSampledLogitsTest(test_lib.TestCase): exp_sampled_softmax_loss = _SoftmaxCrossEntropyWithLogits( exp_logits, exp_labels) - with self.cached_session(): - true_exp_bf16 = np.full( - [batch_size, 1], fill_value=0.5, dtype=dtypes.bfloat16.as_numpy_dtype) - sampled_exp_bf16 = np.full( - [len(sampled)], fill_value=0.5, dtype=dtypes.bfloat16.as_numpy_dtype) - sampled_vals_bf16 = (sampled, true_exp_bf16, sampled_exp_bf16) + true_exp_bf16 = np.full([batch_size, 1], + fill_value=0.5, + dtype=dtypes.bfloat16.as_numpy_dtype) + sampled_exp_bf16 = np.full([len(sampled)], + fill_value=0.5, + dtype=dtypes.bfloat16.as_numpy_dtype) + sampled_vals_bf16 = (sampled, true_exp_bf16, sampled_exp_bf16) - got_sampled_softmax_loss = math_ops.cast( - nn_impl.sampled_softmax_loss( - weights=constant_op.constant(weights, dtype=dtypes.bfloat16), - biases=constant_op.constant(biases, dtype=dtypes.bfloat16), - labels=constant_op.constant( - labels, shape=(batch_size, 1), dtype=dtypes.bfloat16), - inputs=constant_op.constant(hidden_acts, dtype=dtypes.bfloat16), - num_sampled=4, - num_classes=num_classes, - num_true=1, - sampled_values=sampled_vals_bf16, - remove_accidental_hits=False, - partition_strategy="div"), dtypes.float32) + got_sampled_softmax_loss = math_ops.cast( + nn_impl.sampled_softmax_loss( + weights=constant_op.constant(weights, dtype=dtypes.bfloat16), + biases=constant_op.constant(biases, dtype=dtypes.bfloat16), + labels=constant_op.constant( + labels, shape=(batch_size, 1), dtype=dtypes.bfloat16), + inputs=constant_op.constant(hidden_acts, dtype=dtypes.bfloat16), + num_sampled=4, + num_classes=num_classes, + num_true=1, + sampled_values=sampled_vals_bf16, + remove_accidental_hits=False, + partition_strategy="div"), dtypes.float32) - self.assertAllClose(exp_sampled_softmax_loss, - got_sampled_softmax_loss.eval(), 1e-1) + self.assertAllClose(exp_sampled_softmax_loss, + self.evaluate(got_sampled_softmax_loss), 1e-1) class CReluTest(test_lib.TestCase): @@ -931,9 +948,9 @@ class CReluTest(test_lib.TestCase): np.random.seed(1) # Make it reproducible. x = np.random.randn(3, 4).astype(np.float32) y = np.concatenate([x * (x > 0), -x * (x < 0)], axis=1) - with self.cached_session(): - z = nn_ops.crelu(constant_op.constant(x)).eval() - self.assertAllClose(y, z, 1e-4) + + z = self.evaluate(nn_ops.crelu(constant_op.constant(x))) + self.assertAllClose(y, z, 1e-4) class ReluTest(test_lib.TestCase): @@ -942,10 +959,11 @@ class ReluTest(test_lib.TestCase): np.random.seed(1) # Make it reproducible. x = np.random.randn(3, 4).astype(np.float32) y = np.maximum(x, 0.0) - with self.cached_session(): - z = nn_ops.relu(constant_op.constant(x)).eval() - self.assertAllEqual(y, z) + z = self.evaluate(nn_ops.relu(constant_op.constant(x))) + self.assertAllEqual(y, z) + + @test_util.run_deprecated_v1 def testNaNs(self): # Test that relu(nan) = nan for various sizes. for i in range(18): @@ -967,22 +985,26 @@ class LeakyReluTest(test_lib.TestCase): outputs = nn_ops.leaky_relu(inputs) self.assertEquals(inputs.shape, outputs.shape) - with self.cached_session() as sess: - inputs, outputs = sess.run([inputs, outputs]) + + inputs, outputs = self.evaluate([inputs, outputs]) + self.assertGreaterEqual(outputs.min(), 0.0) self.assertLessEqual(outputs.max(), 1.0) self.assertAllClose(inputs, outputs) + @test_util.run_deprecated_v1 def testValues(self): for dtype in [np.int32, np.int64, np.float16, np.float32, np.float64]: np_values = np.array([-2, -1, 0, 1, 2], dtype=dtype) outputs = nn_ops.leaky_relu(constant_op.constant(np_values)) - with self.cached_session() as sess: - outputs = sess.run(outputs) + + outputs = self.evaluate(outputs) + tol = 2e-3 if dtype == np.float16 else 1e-6 self.assertAllClose( outputs, [-0.4, -0.2, 0.0, 1.0, 2.0], rtol=tol, atol=tol) + @test_util.run_deprecated_v1 def testName(self): np_values = np.array([-2, -1, 0, 1, 2], dtype=np.float64) outputs_with_name_set = nn_ops.leaky_relu( @@ -996,6 +1018,7 @@ class LeakyReluTest(test_lib.TestCase): class SwishTest(test_lib.TestCase): + @test_util.run_deprecated_v1 def testValues(self): np_values = np.array( [np.linspace(-10.0, 0.0, 100), @@ -1004,11 +1027,13 @@ class SwishTest(test_lib.TestCase): tf_values = constant_op.constant(np_values) actual_tf_outputs = nn_impl.swish(tf_values) expected_tf_outputs = tf_values * math_ops.sigmoid(tf_values) - with self.cached_session() as sess: - actual_outputs, expected_outputs = sess.run( - [actual_tf_outputs, expected_tf_outputs]) + + actual_outputs, expected_outputs = self.evaluate( + [actual_tf_outputs, expected_tf_outputs]) + self.assertAllClose(actual_outputs, expected_outputs) + @test_util.run_deprecated_v1 def testGradients(self): shape = [5, 3, 4] sigma = 5 @@ -1039,8 +1064,8 @@ class MomentsTest(test_lib.TestCase): with self.session(graph=g) as sess: inputs = constant_op.constant( input_values, shape=input_shape, dtype=dtypes.float32) - mean, variance = nn_impl.moments( - inputs, moments_axes, keep_dims=keep_dims) + mean, variance = nn_impl.moments_v2( + inputs, moments_axes, keepdims=keep_dims) if check_gradients: err = gradient_checker.compute_gradient_error( @@ -1051,7 +1076,7 @@ class MomentsTest(test_lib.TestCase): self.assertLess(err, 1e-3) # Evaluate. - [mean, variance] = sess.run([mean, variance]) + [mean, variance] = self.evaluate([mean, variance]) # Make sure that there are no NaNs self.assertFalse(np.isnan(mean).any()) self.assertFalse(np.isnan(variance).any()) @@ -1094,9 +1119,9 @@ class DataFormatDimMapTest(test_lib.TestCase): def _test(self, x_val, y_val_expected): x = constant_op.constant(x_val) y = nn_ops.data_format_dim_map(x) - with self.cached_session(use_gpu=test_lib.is_gpu_available()) as sess: - y_val = sess.run(y) - self.assertAllEqual(y_val, y_val_expected) + + y_val = self.evaluate(y) + self.assertAllEqual(y_val, y_val_expected) def test(self): self._test(0, 0) @@ -1117,8 +1142,8 @@ class DataFormatDimMapTest(test_lib.TestCase): y_val_expected = [2, 2, 3] x = constant_op.constant(x_val) y = nn_ops.data_format_dim_map(x, src_format="NHWC", dst_format="NCHW") - with self.session(use_gpu=test_lib.is_gpu_available()) as sess: - y_val = sess.run(y) + with test_util.use_gpu(): + y_val = self.evaluate(y) self.assertAllEqual(y_val, y_val_expected) def testNHWCtoHWNC(self): @@ -1126,8 +1151,8 @@ class DataFormatDimMapTest(test_lib.TestCase): y_val_expected = [2, 0, 1, 3, 2, 0, 1, 3] x = constant_op.constant(x_val) y = nn_ops.data_format_dim_map(x, src_format="NHWC", dst_format="HWNC") - with self.session(use_gpu=test_lib.is_gpu_available()) as sess: - y_val = sess.run(y) + with test_util.use_gpu(): + y_val = self.evaluate(y) self.assertAllEqual(y_val, y_val_expected) def testNHWCtoWHCN(self): @@ -1135,8 +1160,8 @@ class DataFormatDimMapTest(test_lib.TestCase): y_val_expected = [3, 1, 0, 2, 3, 1, 0, 2] x = constant_op.constant(x_val) y = nn_ops.data_format_dim_map(x, src_format="NHWC", dst_format="WHCN") - with self.session(use_gpu=test_lib.is_gpu_available()) as sess: - y_val = sess.run(y) + with test_util.use_gpu(): + y_val = self.evaluate(y) self.assertAllEqual(y_val, y_val_expected) def testArbitraryASCII(self): @@ -1144,8 +1169,8 @@ class DataFormatDimMapTest(test_lib.TestCase): y_val_expected = [3, 2, 1, 0, 3, 2, 1, 0] x = constant_op.constant(x_val) y = nn_ops.data_format_dim_map(x, src_format="qwer", dst_format="rewq") - with self.session(use_gpu=test_lib.is_gpu_available()) as sess: - y_val = sess.run(y) + with test_util.use_gpu(): + y_val = self.evaluate(y) self.assertAllEqual(y_val, y_val_expected) @@ -1155,64 +1180,64 @@ class DataFormatVectorPermuteTest(test_lib.TestCase): x_val = [7, 4, 9, 3] x = constant_op.constant(x_val) y = nn_ops.data_format_vec_permute(x) - with self.session(use_gpu=test_lib.is_gpu_available()) as sess: - y_val = sess.run(y) + with test_util.use_gpu(): + y_val = self.evaluate(y) self.assertAllEqual(y_val, [7, 3, 4, 9]) def testNCHWToNHWC(self): x_val = [7, 4, 9, 3] x = constant_op.constant(x_val) y = nn_ops.data_format_vec_permute(x, src_format="NCHW", dst_format="NHWC") - with self.session(use_gpu=test_lib.is_gpu_available()) as sess: - y_val = sess.run(y) + with test_util.use_gpu(): + y_val = self.evaluate(y) self.assertAllEqual(y_val, [7, 9, 3, 4]) def testNHWCToHWNC(self): x_val = [7, 4, 9, 3] x = constant_op.constant(x_val) y = nn_ops.data_format_vec_permute(x, src_format="NHWC", dst_format="HWNC") - with self.session(use_gpu=test_lib.is_gpu_available()) as sess: - y_val = sess.run(y) + with test_util.use_gpu(): + y_val = self.evaluate(y) self.assertAllEqual(y_val, [4, 9, 7, 3]) def testHWNCToNHWC(self): x_val = [7, 4, 9, 3] x = constant_op.constant(x_val) y = nn_ops.data_format_vec_permute(x, src_format="HWNC", dst_format="NHWC") - with self.session(use_gpu=test_lib.is_gpu_available()) as sess: - y_val = sess.run(y) + with test_util.use_gpu(): + y_val = self.evaluate(y) self.assertAllEqual(y_val, [9, 7, 4, 3]) def testNHWCToNCHW2D(self): x_val = [[7, 4], [9, 3], [4, 5], [5, 1]] x = constant_op.constant(x_val) y = nn_ops.data_format_vec_permute(x) - with self.session(use_gpu=test_lib.is_gpu_available()) as sess: - y_val = sess.run(y) + with test_util.use_gpu(): + y_val = self.evaluate(y) self.assertAllEqual(y_val, [[7, 4], [5, 1], [9, 3], [4, 5]]) def testNHWCToHWNC2D(self): x_val = [[7, 4], [9, 3], [4, 5], [5, 1]] x = constant_op.constant(x_val) y = nn_ops.data_format_vec_permute(x, src_format="NHWC", dst_format="HWNC") - with self.session(use_gpu=test_lib.is_gpu_available()) as sess: - y_val = sess.run(y) + with test_util.use_gpu(): + y_val = self.evaluate(y) self.assertAllEqual(y_val, [[9, 3], [4, 5], [7, 4], [5, 1]]) def testHWNCToNHWC2D(self): x_val = [[7, 4], [9, 3], [4, 5], [5, 1]] x = constant_op.constant(x_val) y = nn_ops.data_format_vec_permute(x, src_format="HWNC", dst_format="NHWC") - with self.session(use_gpu=test_lib.is_gpu_available()) as sess: - y_val = sess.run(y) + with test_util.use_gpu(): + y_val = self.evaluate(y) self.assertAllEqual(y_val, [[4, 5], [7, 4], [9, 3], [5, 1]]) def testNCHWToNHWC2D(self): x_val = [[7, 4], [9, 3], [4, 5], [5, 1]] x = constant_op.constant(x_val) y = nn_ops.data_format_vec_permute(x, src_format="NCHW", dst_format="NHWC") - with self.session(use_gpu=test_lib.is_gpu_available()) as sess: - y_val = sess.run(y) + with test_util.use_gpu(): + y_val = self.evaluate(y) self.assertAllEqual(y_val, [[7, 4], [4, 5], [5, 1], [9, 3]]) diff --git a/tensorflow/python/ops/nn_xent_test.py b/tensorflow/python/ops/nn_xent_test.py index 57ce4fd0a99..3e5c198fc6a 100644 --- a/tensorflow/python/ops/nn_xent_test.py +++ b/tensorflow/python/ops/nn_xent_test.py @@ -24,6 +24,7 @@ import numpy as np from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes +from tensorflow.python.framework import test_util from tensorflow.python.ops import gradient_checker from tensorflow.python.ops import gradients_impl from tensorflow.python.ops import nn_impl @@ -53,6 +54,7 @@ class SigmoidCrossEntropyWithLogitsTest(test.TestCase): losses = np.array(self._SigmoidCrossEntropyWithLogits(x, y)).reshape(*sizes) return logits, targets, losses + @test_util.run_deprecated_v1 def testConstructionNamed(self): with self.cached_session(): logits, targets, _ = self._Inputs() @@ -68,7 +70,7 @@ class SigmoidCrossEntropyWithLogitsTest(test.TestCase): loss = nn_impl.sigmoid_cross_entropy_with_logits( labels=targets, logits=logits) np_loss = np.array(losses).astype(np.float32) - tf_loss = loss.eval() + tf_loss = self.evaluate(loss) self.assertAllClose(np_loss, tf_loss, atol=0.001) def testLogisticOutputMultiDim(self): @@ -79,9 +81,10 @@ class SigmoidCrossEntropyWithLogitsTest(test.TestCase): loss = nn_impl.sigmoid_cross_entropy_with_logits( labels=targets, logits=logits) np_loss = np.array(losses).astype(np.float32) - tf_loss = loss.eval() + tf_loss = self.evaluate(loss) self.assertAllClose(np_loss, tf_loss, atol=0.001) + @test_util.run_deprecated_v1 def testGradient(self): sizes = [4, 2] with self.cached_session(): @@ -92,6 +95,7 @@ class SigmoidCrossEntropyWithLogitsTest(test.TestCase): print("logistic loss gradient err = ", err) self.assertLess(err, 1e-7) + @test_util.run_deprecated_v1 def testGradientAtZero(self): with self.cached_session(): logits = constant_op.constant([0.0, 0.0], dtype=dtypes.float64) @@ -129,6 +133,7 @@ class WeightedCrossEntropyTest(test.TestCase): losses = np.array(self._WeightedCrossEntropy(x, y, q)).reshape(*sizes) return logits, targets, q, losses + @test_util.run_deprecated_v1 def testConstructionNamed(self): with self.cached_session(): logits, targets, pos_weight, _ = self._Inputs() @@ -143,7 +148,7 @@ class WeightedCrossEntropyTest(test.TestCase): loss = nn_impl.weighted_cross_entropy_with_logits( targets=targets, logits=logits, pos_weight=pos_weight) np_loss = np.array(losses).astype(np.float32) - tf_loss = loss.eval() + tf_loss = self.evaluate(loss) self.assertAllClose(np_loss, tf_loss, atol=0.001) def testOutputMultiDim(self): @@ -154,9 +159,10 @@ class WeightedCrossEntropyTest(test.TestCase): loss = nn_impl.weighted_cross_entropy_with_logits( targets=targets, logits=logits, pos_weight=pos_weight) np_loss = np.array(losses).astype(np.float32) - tf_loss = loss.eval() + tf_loss = self.evaluate(loss) self.assertAllClose(np_loss, tf_loss, atol=0.001) + @test_util.run_deprecated_v1 def testGradient(self): sizes = [4, 2] with self.cached_session(): diff --git a/tensorflow/python/ops/numerics.py b/tensorflow/python/ops/numerics.py index 1a235de90cf..0ab39ad0a8e 100644 --- a/tensorflow/python/ops/numerics.py +++ b/tensorflow/python/ops/numerics.py @@ -28,9 +28,7 @@ from tensorflow.python.util import deprecation from tensorflow.python.util.tf_export import tf_export -@tf_export( - "debugging.assert_all_finite", - v1=["debugging.assert_all_finite", "verify_tensor_all_finite"]) +@tf_export(v1=["debugging.assert_all_finite", "verify_tensor_all_finite"]) @deprecation.deprecated_endpoints("verify_tensor_all_finite") def verify_tensor_all_finite(t, msg, name=None): """Assert that the tensor does not contain any NaN's or Inf's. @@ -43,11 +41,26 @@ def verify_tensor_all_finite(t, msg, name=None): Returns: Same tensor as `t`. """ - with ops.name_scope(name, "VerifyFinite", [t]) as name: - t = ops.convert_to_tensor(t, name="t") - with ops.colocate_with(t): - verify_input = array_ops.check_numerics(t, message=msg) - out = control_flow_ops.with_dependencies([verify_input], t) + return verify_tensor_all_finite_v2(t, msg, name) + + +@tf_export("debugging.assert_all_finite", v1=[]) +def verify_tensor_all_finite_v2(x, message, name=None): + """Assert that the tensor does not contain any NaN's or Inf's. + + Args: + x: Tensor to check. + message: Message to log on failure. + name: A name for this operation (optional). + + Returns: + Same tensor as `x`. + """ + with ops.name_scope(name, "VerifyFinite", [x]) as name: + x = ops.convert_to_tensor(x, name="x") + with ops.colocate_with(x): + verify_input = array_ops.check_numerics(x, message=message) + out = control_flow_ops.with_dependencies([verify_input], x) return out diff --git a/tensorflow/tools/compatibility/testdata/test_file_v1_10.py b/tensorflow/python/ops/optional_grad.py similarity index 62% rename from tensorflow/tools/compatibility/testdata/test_file_v1_10.py rename to tensorflow/python/ops/optional_grad.py index e5ca8d3e2e2..0d1eae3cda4 100644 --- a/tensorflow/tools/compatibility/testdata/test_file_v1_10.py +++ b/tensorflow/python/ops/optional_grad.py @@ -12,23 +12,22 @@ # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================== -"""Tests for tf upgrader.""" +"""Gradient functions for optional ops.""" from __future__ import absolute_import from __future__ import division from __future__ import print_function -import tensorflow as tf -from tensorflow.python.framework import test_util -from tensorflow.python.platform import test as test_lib + +from tensorflow.python.framework import ops +from tensorflow.python.ops import gen_dataset_ops -class TestUpgrade(test_util.TensorFlowTestCase): - """Test various APIs that have been changed in 2.0.""" +@ops.RegisterGradient("OptionalFromValue") +def _OptionalFromValueGrad(op, grad): + return gen_dataset_ops.optional_get_value( + grad, [t.dtype for t in op.inputs], [t.shape for t in op.inputs]) - def testRenames(self): - with self.cached_session(): - self.assertAllClose(1.04719755, tf.acos(0.5).eval()) - self.assertAllClose(0.5, tf.rsqrt(4.0).eval()) -if __name__ == "__main__": - test_lib.main() +@ops.RegisterGradient("OptionalGetValue") +def _OptionalGetValueGrad(unused_op, *grads): + return gen_dataset_ops.optional_from_value(grads) diff --git a/tensorflow/python/ops/parallel_for/control_flow_ops.py b/tensorflow/python/ops/parallel_for/control_flow_ops.py index ead7ae5478c..8f652e9c509 100644 --- a/tensorflow/python/ops/parallel_for/control_flow_ops.py +++ b/tensorflow/python/ops/parallel_for/control_flow_ops.py @@ -17,16 +17,20 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function +from tensorflow.python.eager import context +from tensorflow.python.eager import function from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops +from tensorflow.python.framework import tensor_util from tensorflow.python.ops import array_ops from tensorflow.python.ops import control_flow_ops +from tensorflow.python.ops import math_ops from tensorflow.python.ops import tensor_array_ops from tensorflow.python.ops.parallel_for.pfor import PFor from tensorflow.python.util import nest -def for_loop(loop_fn, loop_fn_dtypes, iters): +def for_loop(loop_fn, loop_fn_dtypes, iters, parallel_iterations=None): """Runs `loop_fn` `iters` times and stacks the outputs. @@ -39,6 +43,8 @@ def for_loop(loop_fn, loop_fn_dtypes, iters): objects. The shape of these outputs should not depend on the input. loop_fn_dtypes: dtypes for the outputs of loop_fn. iters: Number of iterations for which to run loop_fn. + parallel_iterations: The number of iterations that can be dispatched in + parallel. This knob can be used to control the total memory usage. Returns: Returns a nested structure of stacked output tensor objects with the same @@ -66,11 +72,16 @@ def for_loop(loop_fn, loop_fn_dtypes, iters): outputs.append(ta) return tuple([i + 1] + outputs) + if parallel_iterations is not None: + extra_args = {"parallel_iterations": parallel_iterations} + else: + extra_args = {} ta_list = control_flow_ops.while_loop( - lambda i, *ta: i < iters, while_body, [0] + [ - tensor_array_ops.TensorArray(dtype, iters) - for dtype in flat_loop_fn_dtypes - ])[1:] + lambda i, *ta: i < iters, + while_body, + [0] + [tensor_array_ops.TensorArray(dtype, iters) + for dtype in flat_loop_fn_dtypes], + **extra_args)[1:] # TODO(rachelim): enable this for sparse tensors @@ -79,7 +90,15 @@ def for_loop(loop_fn, loop_fn_dtypes, iters): return nest.pack_sequence_as(loop_fn_dtypes, output) -def pfor(loop_fn, iters): +def _flatten_first_two_dims(x): + """Flattens the first two dimensions of x into a single dimension.""" + old_shape = array_ops.shape(x) + new_shape = array_ops.concat([[old_shape[0] * old_shape[1]], old_shape[2:]], + axis=0) + return array_ops.reshape(x, new_shape) + + +def pfor(loop_fn, iters, parallel_iterations=None): """Equivalent to running `loop_fn` `iters` times and stacking the outputs. `pfor` has functionality similar to `for_loop`, i.e. running `loop_fn` `iters` @@ -99,8 +118,8 @@ def pfor(loop_fn, iters): reads, etc). - Conversion works only on a limited set of kernels for which a converter has been registered. - - loop_fn cannot currently contain control flow operations like - tf.while_loop or tf.cond. + - loop_fn has limited support for control flow operations. tf.cond in + particular is not supported. - `loop_fn` should return nested structure of Tensors or Operations. However if an Operation is returned, it should have zero outputs. - The shape and dtype of `loop_fn` outputs should not depend on the input @@ -109,22 +128,92 @@ def pfor(loop_fn, iters): Args: loop_fn: A function that takes an int32 scalar tf.Tensor object representing the iteration number, and returns a possibly nested structure of Tensor or - Operation objects. + Operation objects. Note that if setting `parallel_iterations` argument to + something other than None, `loop_fn` may be called more than once during + graph construction. So it may need to avoid mutating global state. iters: Number of iterations for which to run loop_fn. + parallel_iterations: A knob to control how many iterations are vectorized + and dispatched in parallel. The default value of None corresponds to + vectorizing all the iterations. If `parallel_iterations` is smaller than + `iters`, then chunks of at most that many iterations are dispatched in + sequence. This knob can be used to control the total memory usage. Returns: Returns a nested structure of stacked tensor objects with the same nested structure as the output of `loop_fn`. + Raises: + ValueError: If parallel_iterations is not None and not an integer > 1. """ + def f(): + return _pfor_impl(loop_fn, iters, parallel_iterations=parallel_iterations) + if context.executing_eagerly(): + f = function.defun(f) + return f() + + +def _pfor_impl(loop_fn, iters, parallel_iterations=None): + """Implementation of pfor.""" existing_ops = set(ops.get_default_graph().get_operations()) with ops.name_scope("loop_body"): loop_var = array_ops.placeholder(dtypes.int32, shape=[]) loop_fn_outputs = loop_fn(loop_var) new_ops = set(ops.get_default_graph().get_operations()) - existing_ops iters = ops.convert_to_tensor(iters) - with ops.name_scope("pfor"): - converter = PFor(loop_var, iters, new_ops) - outputs = [] - for loop_fn_output in nest.flatten(loop_fn_outputs): - outputs.append(converter.convert(loop_fn_output)) - return nest.pack_sequence_as(loop_fn_outputs, outputs) + if parallel_iterations is not None: + if parallel_iterations < 1: + raise ValueError("parallel_iterations must be None or a positive integer") + if parallel_iterations == 1: + raise ValueError("Found parallel_iterations == 1. Use for_loop instead.") + iters_value = tensor_util.constant_value(iters) + if iters_value is not None and iters_value < parallel_iterations: + parallel_iterations = None + if parallel_iterations is None: + with ops.name_scope("pfor"): + converter = PFor(loop_var, iters, new_ops) + outputs = [] + for loop_fn_output in nest.flatten(loop_fn_outputs): + outputs.append(converter.convert(loop_fn_output)) + return nest.pack_sequence_as(loop_fn_outputs, outputs) + else: + num_tiled_iterations = iters // parallel_iterations + num_remaining_iterations = iters % parallel_iterations + # TODO(agarwal): Avoid calling loop_fn twice. Generate the loop body inside + # a tf.function and extract the graph from there to vectorize it. + with ops.name_scope("pfor_untiled"): + converter = PFor(loop_var, num_remaining_iterations, new_ops) + remaining_outputs = [] + flattened_loop_fn_outputs = nest.flatten(loop_fn_outputs) + for loop_fn_output in flattened_loop_fn_outputs: + remaining_outputs.append(converter.convert(loop_fn_output)) + + with ops.name_scope("pfor_tiled"): + loop_fn_dtypes = [ops.convert_to_tensor(x).dtype + for x in flattened_loop_fn_outputs] + + def tiled_loop_body(j): + offset = j * parallel_iterations + num_remaining_iterations + + def tiled_loop_fn(i): + return nest.flatten(loop_fn(i + offset)) + + return pfor(tiled_loop_fn, parallel_iterations) + + tiled_outputs = for_loop(tiled_loop_body, loop_fn_dtypes, + num_tiled_iterations, parallel_iterations=1) + tiled_outputs = [_flatten_first_two_dims(y) for y in tiled_outputs] + + with ops.name_scope("pfor"): + iters_value = tensor_util.constant_value(iters) + if iters_value is None or iters_value % parallel_iterations: + outputs = control_flow_ops.cond( + math_ops.equal(num_remaining_iterations, 0), + lambda: tiled_outputs, + lambda: [array_ops.concat([x, y], axis=0) + for x, y in zip(remaining_outputs, tiled_outputs)]) + else: + outputs = tiled_outputs + return nest.pack_sequence_as(loop_fn_outputs, nest.flatten(outputs)) + + + + diff --git a/tensorflow/python/ops/parallel_for/control_flow_ops_test.py b/tensorflow/python/ops/parallel_for/control_flow_ops_test.py index 171369b724a..cc20d7ca6aa 100644 --- a/tensorflow/python/ops/parallel_for/control_flow_ops_test.py +++ b/tensorflow/python/ops/parallel_for/control_flow_ops_test.py @@ -26,10 +26,12 @@ import numpy as np from tensorflow.core.example import example_pb2 from tensorflow.core.example import feature_pb2 from tensorflow.python.client import session +from tensorflow.python.eager import backprop from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops from tensorflow.python.framework import sparse_tensor +from tensorflow.python.framework import test_util from tensorflow.python.ops import array_ops from tensorflow.python.ops import bitwise_ops from tensorflow.python.ops import clip_ops @@ -52,6 +54,7 @@ from tensorflow.python.platform import test from tensorflow.python.util import nest +@test_util.run_all_in_graph_and_eager_modes class PForTest(test.TestCase): def _run_targets(self, targets1, targets2=None, run_init=True): @@ -73,9 +76,13 @@ class PForTest(test.TestCase): else: self.assertAllEqual(outputs[i + n], outputs[i]) - def _test_loop_fn(self, loop_fn, iters, loop_fn_dtypes=dtypes.float32): - t1 = pfor_control_flow_ops.pfor(loop_fn, iters=iters) - t2 = pfor_control_flow_ops.for_loop(loop_fn, loop_fn_dtypes, iters=iters) + def _test_loop_fn(self, loop_fn, iters, + loop_fn_dtypes=dtypes.float32, + parallel_iterations=None): + t1 = pfor_control_flow_ops.pfor(loop_fn, iters=iters, + parallel_iterations=parallel_iterations) + t2 = pfor_control_flow_ops.for_loop(loop_fn, loop_fn_dtypes, iters=iters, + parallel_iterations=parallel_iterations) self.run_and_assert_equal(t1, t2) def test_op_conversion_fallback_to_while_loop(self): @@ -96,7 +103,32 @@ class PForTest(test.TestCase): loop_fn, 3, loop_fn_dtypes=[dtypes.float32, dtypes.int32]) flags.FLAGS.op_conversion_fallback_to_while_loop = False + def test_parallel_iterations(self): + for parallel_iterations in [2, 3, 8, 10]: + x = random_ops.random_uniform([8, 3]) + # pylint: disable=cell-var-from-loop + def loop_fn(i): + return array_ops.gather(x, i) + # pylint: enable=cell-var-from-loop + + self._test_loop_fn(loop_fn, 8, parallel_iterations=parallel_iterations) + self._test_loop_fn(loop_fn, 4 * constant_op.constant(2), + parallel_iterations=parallel_iterations) + + def test_parallel_iterations_zero(self): + with self.assertRaisesRegexp(ValueError, "positive integer"): + pfor_control_flow_ops.pfor(lambda i: 1, 8, parallel_iterations=0) + with self.assertRaisesRegexp(TypeError, "positive integer"): + pfor_control_flow_ops.for_loop(lambda i: 1, dtypes.int32, 8, + parallel_iterations=0) + + def test_parallel_iterations_one(self): + with self.assertRaisesRegexp(ValueError, "Use for_loop instead"): + pfor_control_flow_ops.pfor(lambda i: 1, 8, parallel_iterations=1) + + +@test_util.run_all_in_graph_and_eager_modes class ArrayTest(PForTest): def test_gather(self): @@ -288,14 +320,17 @@ class ArrayTest(PForTest): def test_unary_cwise_ops(self): for op in [array_ops.identity, array_ops.stop_gradient]: - x = random_ops.random_uniform([3, 5]) + with backprop.GradientTape(persistent=True) as g: + x = random_ops.random_uniform([3, 5]) + g.watch(x) # pylint: disable=cell-var-from-loop def loop_fn(i): - x1 = array_ops.gather(x, i) - y = op(x1) + x1 - loss = nn.l2_loss(y) - return op(x), y, gradient_ops.gradients(loss, x1) + with g: + x1 = array_ops.gather(x, i) + y = op(x1) + x1 + loss = nn.l2_loss(y) + return op(x), y, g.gradient(loss, x1) # pylint: enable=cell-var-from-loop @@ -318,17 +353,21 @@ class ArrayTest(PForTest): self._test_loop_fn(loop_fn, 3, loop_fn_dtypes=[dtypes.float32]) def test_strided_slice(self): - x = random_ops.random_uniform([3, 3, 4, 4, 2, 2, 2]) + with backprop.GradientTape(persistent=True) as g: + x = random_ops.random_uniform([3, 3, 4, 4, 2, 2, 2]) + g.watch(x) def loop_fn(i): - x_i = array_ops.gather(x, i) - y = x_i[:2, ::2, 1::3, ..., array_ops.newaxis, 1] - loss = nn.l2_loss(y) - return y, gradient_ops.gradients(loss, x_i) + with g: + x_i = array_ops.gather(x, i) + y = x_i[:2, ::2, 1::3, ..., array_ops.newaxis, 1] + loss = nn.l2_loss(y) + return y, g.gradient(loss, x_i) self._test_loop_fn(loop_fn, 3, loop_fn_dtypes=[dtypes.float32] * 2) +@test_util.run_all_in_graph_and_eager_modes class BitwiseTest(PForTest): def test_unary_cwise(self): @@ -368,6 +407,7 @@ class BitwiseTest(PForTest): self._test_loop_fn(loop_fn, 3, loop_fn_dtypes=output_dtypes) +@test_util.run_all_in_graph_and_eager_modes class MathTest(PForTest): def test_unary_cwise_ops(self): @@ -424,22 +464,29 @@ class MathTest(PForTest): nn.softsign, ] for op in complex_ops + real_ops: - x = random_ops.random_uniform([3, 5]) - if op in complex_ops: - y = random_ops.random_uniform([3, 5]) - x = math_ops.complex(x, y) + with backprop.GradientTape(persistent=True) as g: + x = random_ops.random_uniform([3, 5]) + g.watch(x) + if op in complex_ops: + y = random_ops.random_uniform([3, 5]) + g.watch(y) + x = math_ops.complex(x, y) # pylint: disable=cell-var-from-loop output_dtypes = [] def loop_fn(i): - x1 = array_ops.gather(x, i) - y1 = op(x1) - outputs = [op(x), y1] - if y1.dtype == dtypes.float32: - loss = math_ops.reduce_sum(y1 * y1) - grad = gradient_ops.gradients(loss, x1) - if grad and grad[0] is not None: - outputs.extend(grad) + with g: + x1 = array_ops.gather(x, i) + y1 = op(x1) + outputs = [op(x), y1] + if y1.dtype == dtypes.float32: + loss = math_ops.reduce_sum(y1 * y1) + else: + loss = None + if loss is not None: + grad = g.gradient(loss, x1) + if grad is not None: + outputs.append(grad) del output_dtypes[:] output_dtypes.extend([t.dtype for t in outputs]) return outputs @@ -656,17 +703,19 @@ class MathTest(PForTest): x_shape = [2, 3, 4, 5, 6] x = random_ops.random_uniform(x_shape) for data_format in ("NCHW", "NHWC"): - bias_dim = 2 if data_format == "NCHW" else -1 - bias_shape = x_shape[bias_dim] - bias = random_ops.random_uniform([bias_shape]) + with backprop.GradientTape(persistent=True) as g: + bias_dim = 2 if data_format == "NCHW" else -1 + bias_shape = x_shape[bias_dim] + bias = random_ops.random_uniform([bias_shape]) + g.watch(bias) # pylint: disable=cell-var-from-loop def loop_fn(i): - a = array_ops.gather(x, i) - y = nn.bias_add(a, bias, data_format=data_format) - loss = math_ops.reduce_sum(y * y) - return y, gradient_ops.gradients(loss, bias) - + with g: + a = array_ops.gather(x, i) + y = nn.bias_add(a, bias, data_format=data_format) + loss = math_ops.reduce_sum(y * y) + return y, g.gradient(loss, bias) # pylint: enable=cell-var-from-loop self._test_loop_fn( @@ -727,6 +776,7 @@ class MathTest(PForTest): self._test_loop_fn(loop_fn, 2) +@test_util.run_all_in_graph_and_eager_modes class NNTest(PForTest): def test_conv2d(self): @@ -779,30 +829,60 @@ class NNTest(PForTest): self._test_loop_fn(loop_fn, 3, loop_fn_dtypes=[dtypes.float32] * 2) def test_avg_pool(self): - x = random_ops.random_uniform([3, 2, 12, 12, 3]) - ksize = [1, 3, 3, 1] + with backprop.GradientTape(persistent=True) as g: + x = random_ops.random_uniform([3, 2, 12, 12, 3]) + g.watch(x) + ksize = [1, 3, 3, 1] def loop_fn(i): - x1 = array_ops.gather(x, i) - output = nn.avg_pool( - x1, ksize, strides=[1, 2, 2, 1], padding="VALID", data_format="NHWC") - loss = nn.l2_loss(output) - return output, gradient_ops.gradients(loss, x1) + with g: + x1 = array_ops.gather(x, i) + output = nn.avg_pool( + x1, ksize, strides=[1, 2, 2, 1], padding="VALID", + data_format="NHWC") + loss = nn.l2_loss(output) + return output, g.gradient(loss, x1) self._test_loop_fn(loop_fn, 3, loop_fn_dtypes=[dtypes.float32] * 2) def test_max_pool(self): - x = random_ops.random_uniform([3, 2, 12, 12, 3]) - ksize = [1, 3, 3, 1] + with backprop.GradientTape(persistent=True) as g: + x = random_ops.random_uniform([3, 2, 12, 12, 3]) + g.watch(x) + ksize = [1, 3, 3, 1] + strides = [1, 2, 2, 1] def loop_fn(i): - x1 = array_ops.gather(x, i) - output = nn.max_pool( - x1, ksize, strides=[1, 2, 2, 1], padding="VALID", data_format="NHWC") - loss = nn.l2_loss(output) - ones = array_ops.ones_like(output) - grad = gradient_ops.gradients(loss, x1, grad_ys=ones) - grad_grad = gradient_ops.gradients(grad, ones) + with g: + x1 = array_ops.gather(x, i) + output = nn.max_pool( + x1, ksize, strides=strides, padding="VALID", data_format="NHWC") + loss = nn.l2_loss(output) + ones = array_ops.ones_like(output) + g.watch(ones) + grad = g.gradient(loss, x1, output_gradients=ones) + grad_grad = g.gradient(grad, ones) + return output, grad, grad_grad + + self._test_loop_fn(loop_fn, 3, loop_fn_dtypes=[dtypes.float32] * 3) + + def test_max_pool3d(self): + with backprop.GradientTape(persistent=True) as g: + x = random_ops.random_uniform([3, 3, 2, 12, 12, 3]) + g.watch(x) + ksize = [1, 1, 3, 3, 1] + strides = [1, 1, 2, 2, 1] + + def loop_fn(i): + with g: + x1 = array_ops.gather(x, i) + output = nn.max_pool3d( + x1, ksize, strides=strides, padding="VALID", data_format="NDHWC") + loss = nn.l2_loss(output) + ones = array_ops.ones_like(output) + g.watch(ones) + grad = g.gradient(loss, x1, output_gradients=ones) + grad_grad = g.gradient(grad, ones) return output, grad, grad_grad self._test_loop_fn(loop_fn, 3, loop_fn_dtypes=[dtypes.float32] * 3) @@ -813,36 +893,41 @@ class NNTest(PForTest): data_formats.append("NCHW") for is_training in (True, False): for data_format in data_formats: - if data_format == "NCHW": - x = random_ops.random_uniform([3, 1, 2, 5, 5]) - else: - x = random_ops.random_uniform([3, 1, 5, 5, 2]) - scale = random_ops.random_uniform([2]) - offset = random_ops.random_uniform([2]) - mean = None if is_training else random_ops.random_uniform([2]) - variance = None if is_training else random_ops.random_uniform([2]) + with backprop.GradientTape(persistent=True) as g: + if data_format == "NCHW": + x = random_ops.random_uniform([3, 1, 2, 5, 5]) + else: + x = random_ops.random_uniform([3, 1, 5, 5, 2]) + g.watch(x) + scale = random_ops.random_uniform([2]) + g.watch(scale) + offset = random_ops.random_uniform([2]) + g.watch(offset) + mean = None if is_training else random_ops.random_uniform([2]) + variance = None if is_training else random_ops.random_uniform([2]) # pylint: disable=cell-var-from-loop def loop_fn(i): - x1 = array_ops.gather(x, i) - outputs = nn.fused_batch_norm( - x1, - scale, - offset, - mean=mean, - variance=variance, - epsilon=0.01, - data_format=data_format, - is_training=is_training) - outputs = list(outputs) - # We only test the first value of outputs when is_training is False. - # It looks like CPU and GPU have different outputs for batch_mean and - # batch_variance for this case. - if not is_training: - outputs[1] = constant_op.constant(0.) - outputs[2] = constant_op.constant(0.) - loss = nn.l2_loss(outputs[0]) - gradients = gradient_ops.gradients(loss, [x1, scale, offset]) + with g: + x1 = array_ops.gather(x, i) + outputs = nn.fused_batch_norm( + x1, + scale, + offset, + mean=mean, + variance=variance, + epsilon=0.01, + data_format=data_format, + is_training=is_training) + outputs = list(outputs) + # We only test the first value of outputs when is_training is False. + # It looks like CPU and GPU have different outputs for batch_mean + # and batch_variance for this case. + if not is_training: + outputs[1] = constant_op.constant(0.) + outputs[2] = constant_op.constant(0.) + loss = nn.l2_loss(outputs[0]) + gradients = g.gradient(loss, [x1, scale, offset]) return outputs + gradients # pylint: enable=cell-var-from-loop @@ -850,16 +935,20 @@ class NNTest(PForTest): self._test_loop_fn(loop_fn, 3, loop_fn_dtypes=[dtypes.float32] * 6) def test_softmax_cross_entropy_with_logits(self): - logits = random_ops.random_uniform([3, 2, 4]) - labels = random_ops.random_uniform([3, 2, 4]) - labels /= math_ops.reduce_sum(labels, axis=[2], keepdims=True) + with backprop.GradientTape(persistent=True) as g: + logits = random_ops.random_uniform([3, 2, 4]) + g.watch(logits) + labels = random_ops.random_uniform([3, 2, 4]) + labels /= math_ops.reduce_sum(labels, axis=[2], keepdims=True) def loop_fn(i): - logits_i = array_ops.gather(logits, i) - labels_i = array_ops.gather(labels, i) - loss = nn.softmax_cross_entropy_with_logits( - labels=labels_i, logits=logits_i) - return loss, gradient_ops.gradients(math_ops.reduce_sum(loss), logits_i) + with g: + logits_i = array_ops.gather(logits, i) + labels_i = array_ops.gather(labels, i) + loss = nn.softmax_cross_entropy_with_logits( + labels=labels_i, logits=logits_i) + total_loss = math_ops.reduce_sum(loss) + return loss, g.gradient(total_loss, logits_i) self._test_loop_fn(loop_fn, 3, loop_fn_dtypes=[dtypes.float32] * 2) @@ -1278,13 +1367,12 @@ class ControlFlowTest(PForTest): pfor_out, pfor_out_grad = pfor_control_flow_ops.pfor(loop_fn, 4) # Note that tf.while_loop does not work in the setup above. So we manually # construct the equivalent computation of the above loops here. - real_out = math_ops.reduce_sum(inp, reduction_indices=[0]) - real_out = math_ops.reduce_prod(real_out, reduction_indices=[1]) + real_out = math_ops.reduce_sum(inp, axis=[0]) + real_out = math_ops.reduce_prod(real_out, axis=[1]) # Note that gradients of real_out will accumulate the gradients across the # output value. Hence we do the same aggregation on pfor_out_grad. real_out_grad = gradient_ops.gradients(real_out, inp)[0] - sum_pfor_out_grad = math_ops.reduce_sum( - pfor_out_grad, reduction_indices=[0]) + sum_pfor_out_grad = math_ops.reduce_sum(pfor_out_grad, axis=[0]) with session.Session() as sess: v1, v2, v1_grad, v2_grad = sess.run( diff --git a/tensorflow/python/ops/parallel_for/gradients.py b/tensorflow/python/ops/parallel_for/gradients.py index 1f026b3660c..3ba1bde3476 100644 --- a/tensorflow/python/ops/parallel_for/gradients.py +++ b/tensorflow/python/ops/parallel_for/gradients.py @@ -25,7 +25,7 @@ from tensorflow.python.ops.parallel_for import control_flow_ops from tensorflow.python.util import nest -def jacobian(output, inputs, use_pfor=True): +def jacobian(output, inputs, use_pfor=True, parallel_iterations=None): """Computes jacobian of `output` w.r.t. `inputs`. Args: @@ -33,6 +33,8 @@ def jacobian(output, inputs, use_pfor=True): inputs: A tensor or a nested structure of tensor objects. use_pfor: If true, uses pfor for computing the jacobian. Else uses tf.while_loop. + parallel_iterations: A knob to control how many iterations and dispatched in + parallel. This knob can be used to control the total memory usage. Returns: A tensor or a nested strucutre of tensors with the same structure as @@ -56,10 +58,14 @@ def jacobian(output, inputs, use_pfor=True): output_size = array_ops.shape(output)[0] if use_pfor: - pfor_outputs = control_flow_ops.pfor(loop_fn, output_size) + pfor_outputs = control_flow_ops.pfor( + loop_fn, output_size, parallel_iterations=parallel_iterations) else: pfor_outputs = control_flow_ops.for_loop( - loop_fn, [output.dtype] * len(flat_inputs), output_size) + loop_fn, + [output.dtype] * len(flat_inputs), + output_size, + parallel_iterations=parallel_iterations) for i, out in enumerate(pfor_outputs): if out is not None: @@ -72,7 +78,7 @@ def jacobian(output, inputs, use_pfor=True): return nest.pack_sequence_as(inputs, pfor_outputs) -def batch_jacobian(output, inp, use_pfor=True): +def batch_jacobian(output, inp, use_pfor=True, parallel_iterations=None): """Computes and stacks jacobians of `output[i,...]` w.r.t. `input[i,...]`. e.g. @@ -87,6 +93,8 @@ def batch_jacobian(output, inp, use_pfor=True): inp: A tensor with shape [b, x1, ..., x_m] use_pfor: If true, uses pfor for computing the Jacobian. Else uses a tf.while_loop. + parallel_iterations: A knob to control how many iterations and dispatched in + parallel. This knob can be used to control the total memory usage. Returns: A tensor `t` with shape [b, y_1, ..., y_n, x1, ..., x_m] where `t[i, ...]` @@ -118,10 +126,13 @@ def batch_jacobian(output, inp, use_pfor=True): return gradient_ops.gradients(y, inp)[0] if use_pfor: - pfor_output = control_flow_ops.pfor(loop_fn, output_row_size) + pfor_output = control_flow_ops.pfor(loop_fn, output_row_size, + parallel_iterations=parallel_iterations) else: - pfor_output = control_flow_ops.for_loop(loop_fn, output.dtype, - output_row_size) + pfor_output = control_flow_ops.for_loop( + loop_fn, output.dtype, + output_row_size, + parallel_iterations=parallel_iterations) if pfor_output is None: return None pfor_output = array_ops.reshape(pfor_output, diff --git a/tensorflow/python/ops/parallel_for/gradients_test.py b/tensorflow/python/ops/parallel_for/gradients_test.py index 5a058bae825..4342833e3eb 100644 --- a/tensorflow/python/ops/parallel_for/gradients_test.py +++ b/tensorflow/python/ops/parallel_for/gradients_test.py @@ -416,6 +416,12 @@ class GradientsTest(test.TestCase): self.assertAllClose(ans, pfor_value) self.assertAllClose(ans, while_value) + def test_jacobian_parallel_iterations(self): + x = constant_op.constant([[1., 2], [3, 4]]) + y = math_ops.matmul(x, x) + self.assertAllClose(gradients.jacobian(y, x, parallel_iterations=2), + gradients.jacobian(y, x, parallel_iterations=3)) + def test_batch_jacobian_bad_shapes(self): x = random_ops.random_uniform([2, 2]) y = random_ops.random_uniform([3, 2]) @@ -459,6 +465,13 @@ class GradientsTest(test.TestCase): self.assertAllClose(ans, pfor_value) self.assertAllClose(ans, while_value) + def test_batch_jacobian_parallel_iterations(self): + x = constant_op.constant([[1., 2], [3, 4]]) + w = constant_op.constant([[1., 2, 3, 4], [5, 6, 7, 8]]) + y = math_ops.matmul(x, w) + self.assertAllClose(gradients.batch_jacobian(y, x, parallel_iterations=2), + gradients.batch_jacobian(y, x, parallel_iterations=3)) + def test_fc_batch_jacobian(self): pfor_jacobian, while_jacobian = create_fc_batch_jacobian(8, 4, 2) self.run_and_assert_equal(pfor_jacobian, while_jacobian) @@ -471,8 +484,8 @@ class GradientsTest(test.TestCase): pfor_jacobian, while_gradients = create_dynamic_lstm_batch_jacobian(8, 4, 3) with session.Session() as sess: init = variables.global_variables_initializer() - sess.run(init) - pfor = sess.run(pfor_jacobian) + self.evaluate(init) + pfor = self.evaluate(pfor_jacobian) for i in range(4): while_i = sess.run(while_gradients[i]) self.assertAllClose(while_i, pfor[:, i, ...]) @@ -547,11 +560,11 @@ class GradientsBenchmarks(test.Benchmark): sess = session.Session() with sess: init = variables.global_variables_initializer() - sess.run(init) - sess.run(targets) + self.evaluate(init) + self.evaluate(targets) begin = time.time() for _ in range(iters): - sess.run(targets) + self.evaluate(targets) end = time.time() avg_time_ms = 1000 * (end - begin) / iters self.report_benchmark(iters=iters, wall_time=avg_time_ms, name=name) diff --git a/tensorflow/python/ops/parallel_for/pfor.py b/tensorflow/python/ops/parallel_for/pfor.py index e6f140a9410..a22c1126c93 100644 --- a/tensorflow/python/ops/parallel_for/pfor.py +++ b/tensorflow/python/ops/parallel_for/pfor.py @@ -1152,9 +1152,8 @@ class PFor(object): continue converted_inputs = [self._conversion_map[inp] for inp in y_op.inputs] - some_input_converted = any( - [self._was_converted(x) for x in y_op.inputs]) - some_input_stacked = any([x.is_stacked for x in converted_inputs]) + some_input_converted = any(self._was_converted(x) for x in y_op.inputs) + some_input_stacked = any(x.is_stacked for x in converted_inputs) converted_control_ops = set() some_control_input_converted = False @@ -1198,7 +1197,7 @@ class PFor(object): # All inputs are unstacked or uncoverted but some control inputs are # converted. # TODO(rachelim): Handle the case where some inputs are sparsely - # stacked (i.e. any([x.is_sparse_stacked for x in converted_inputs])) + # stacked (i.e. any(x.is_sparse_stacked for x in converted_inputs)) new_op = _create_op(y_op.type, [x.t for x in converted_inputs], [x.dtype for x in y_op.outputs], y_op.node_def.attr) @@ -1303,7 +1302,10 @@ def _inputs_with_flattening(pfor_input, input_indices): @RegisterPForWithArgs("Conv2D", dims=[0]) @RegisterPForWithArgs("AvgPool", dims=[0]) @RegisterPForWithArgs("MaxPool", dims=[0]) +@RegisterPForWithArgs("MaxPool3D", dims=[0]) +@RegisterPForWithArgs("MaxPool3DGrad", dims=[0, 1, 2]) @RegisterPForWithArgs("MaxPoolGrad", dims=[0, 1, 2]) +@RegisterPForWithArgs("MaxPool3DGradGrad", dims=[0, 1, 2]) @RegisterPForWithArgs("MaxPoolGradGrad", dims=[0, 1, 2]) @RegisterPForWithArgs("SoftmaxCrossEntropyWithLogits", dims=[0, 1]) def _convert_flatten_batch(pfor_input, op_type, dims): diff --git a/tensorflow/python/ops/parsing_ops.py b/tensorflow/python/ops/parsing_ops.py index 484caf01796..a84af6c5cf2 100644 --- a/tensorflow/python/ops/parsing_ops.py +++ b/tensorflow/python/ops/parsing_ops.py @@ -363,7 +363,7 @@ def _prepend_none_dimension(features): return features -@tf_export("io.parse_example", v1=["io.parse_example", "parse_example"]) +@tf_export(v1=["io.parse_example", "parse_example"]) def parse_example(serialized, features, name=None, example_names=None): # pylint: disable=line-too-long """Parses `Example` protos into a `dict` of tensors. @@ -574,6 +574,223 @@ def parse_example(serialized, features, name=None, example_names=None): Returns: A `dict` mapping feature keys to `Tensor` and `SparseTensor` values. + Raises: + ValueError: if any feature is invalid. + """ + return parse_example_v2(serialized, features, example_names, name) + + +@tf_export("io.parse_example", v1=[]) +def parse_example_v2(serialized, features, example_names=None, name=None): + # pylint: disable=line-too-long + """Parses `Example` protos into a `dict` of tensors. + + Parses a number of serialized [`Example`](https://www.tensorflow.org/code/tensorflow/core/example/example.proto) + protos given in `serialized`. We refer to `serialized` as a batch with + `batch_size` many entries of individual `Example` protos. + + `example_names` may contain descriptive names for the corresponding serialized + protos. These may be useful for debugging purposes, but they have no effect on + the output. If not `None`, `example_names` must be the same length as + `serialized`. + + This op parses serialized examples into a dictionary mapping keys to `Tensor` + and `SparseTensor` objects. `features` is a dict from keys to `VarLenFeature`, + `SparseFeature`, and `FixedLenFeature` objects. Each `VarLenFeature` + and `SparseFeature` is mapped to a `SparseTensor`, and each + `FixedLenFeature` is mapped to a `Tensor`. + + Each `VarLenFeature` maps to a `SparseTensor` of the specified type + representing a ragged matrix. Its indices are `[batch, index]` where `batch` + identifies the example in `serialized`, and `index` is the value's index in + the list of values associated with that feature and example. + + Each `SparseFeature` maps to a `SparseTensor` of the specified type + representing a Tensor of `dense_shape` `[batch_size] + SparseFeature.size`. + Its `values` come from the feature in the examples with key `value_key`. + A `values[i]` comes from a position `k` in the feature of an example at batch + entry `batch`. This positional information is recorded in `indices[i]` as + `[batch, index_0, index_1, ...]` where `index_j` is the `k-th` value of + the feature in the example at with key `SparseFeature.index_key[j]`. + In other words, we split the indices (except the first index indicating the + batch entry) of a `SparseTensor` by dimension into different features of the + `Example`. Due to its complexity a `VarLenFeature` should be preferred over a + `SparseFeature` whenever possible. + + Each `FixedLenFeature` `df` maps to a `Tensor` of the specified type (or + `tf.float32` if not specified) and shape `(serialized.size(),) + df.shape`. + + `FixedLenFeature` entries with a `default_value` are optional. With no default + value, we will fail if that `Feature` is missing from any example in + `serialized`. + + Each `FixedLenSequenceFeature` `df` maps to a `Tensor` of the specified type + (or `tf.float32` if not specified) and shape + `(serialized.size(), None) + df.shape`. + All examples in `serialized` will be padded with `default_value` along the + second dimension. + + Examples: + + For example, if one expects a `tf.float32` `VarLenFeature` `ft` and three + serialized `Example`s are provided: + + ``` + serialized = [ + features + { feature { key: "ft" value { float_list { value: [1.0, 2.0] } } } }, + features + { feature []}, + features + { feature { key: "ft" value { float_list { value: [3.0] } } } + ] + ``` + + then the output will look like: + + ```python + {"ft": SparseTensor(indices=[[0, 0], [0, 1], [2, 0]], + values=[1.0, 2.0, 3.0], + dense_shape=(3, 2)) } + ``` + + If instead a `FixedLenSequenceFeature` with `default_value = -1.0` and + `shape=[]` is used then the output will look like: + + ```python + {"ft": [[1.0, 2.0], [3.0, -1.0]]} + ``` + + Given two `Example` input protos in `serialized`: + + ``` + [ + features { + feature { key: "kw" value { bytes_list { value: [ "knit", "big" ] } } } + feature { key: "gps" value { float_list { value: [] } } } + }, + features { + feature { key: "kw" value { bytes_list { value: [ "emmy" ] } } } + feature { key: "dank" value { int64_list { value: [ 42 ] } } } + feature { key: "gps" value { } } + } + ] + ``` + + And arguments + + ``` + example_names: ["input0", "input1"], + features: { + "kw": VarLenFeature(tf.string), + "dank": VarLenFeature(tf.int64), + "gps": VarLenFeature(tf.float32), + } + ``` + + Then the output is a dictionary: + + ```python + { + "kw": SparseTensor( + indices=[[0, 0], [0, 1], [1, 0]], + values=["knit", "big", "emmy"] + dense_shape=[2, 2]), + "dank": SparseTensor( + indices=[[1, 0]], + values=[42], + dense_shape=[2, 1]), + "gps": SparseTensor( + indices=[], + values=[], + dense_shape=[2, 0]), + } + ``` + + For dense results in two serialized `Example`s: + + ``` + [ + features { + feature { key: "age" value { int64_list { value: [ 0 ] } } } + feature { key: "gender" value { bytes_list { value: [ "f" ] } } } + }, + features { + feature { key: "age" value { int64_list { value: [] } } } + feature { key: "gender" value { bytes_list { value: [ "f" ] } } } + } + ] + ``` + + We can use arguments: + + ``` + example_names: ["input0", "input1"], + features: { + "age": FixedLenFeature([], dtype=tf.int64, default_value=-1), + "gender": FixedLenFeature([], dtype=tf.string), + } + ``` + + And the expected output is: + + ```python + { + "age": [[0], [-1]], + "gender": [["f"], ["f"]], + } + ``` + + An alternative to `VarLenFeature` to obtain a `SparseTensor` is + `SparseFeature`. For example, given two `Example` input protos in + `serialized`: + + ``` + [ + features { + feature { key: "val" value { float_list { value: [ 0.5, -1.0 ] } } } + feature { key: "ix" value { int64_list { value: [ 3, 20 ] } } } + }, + features { + feature { key: "val" value { float_list { value: [ 0.0 ] } } } + feature { key: "ix" value { int64_list { value: [ 42 ] } } } + } + ] + ``` + + And arguments + + ``` + example_names: ["input0", "input1"], + features: { + "sparse": SparseFeature( + index_key="ix", value_key="val", dtype=tf.float32, size=100), + } + ``` + + Then the output is a dictionary: + + ```python + { + "sparse": SparseTensor( + indices=[[0, 3], [0, 20], [1, 42]], + values=[0.5, -1.0, 0.0] + dense_shape=[2, 100]), + } + ``` + + Args: + serialized: A vector (1-D Tensor) of strings, a batch of binary + serialized `Example` protos. + features: A `dict` mapping feature keys to `FixedLenFeature`, + `VarLenFeature`, and `SparseFeature` values. + example_names: A vector (1-D Tensor) of strings (optional), the names of + the serialized protos in the batch. + name: A name for this operation (optional). + + Returns: + A `dict` mapping feature keys to `Tensor` and `SparseTensor` values. + Raises: ValueError: if any feature is invalid. """ @@ -764,8 +981,7 @@ def _process_raw_parameters(names, dense_defaults, sparse_keys, sparse_types, dense_shapes_as_proto, dense_shapes) -@tf_export("io.parse_single_example", - v1=["io.parse_single_example", "parse_single_example"]) +@tf_export(v1=["io.parse_single_example", "parse_single_example"]) def parse_single_example(serialized, features, name=None, example_names=None): """Parses a single `Example` proto. @@ -795,6 +1011,48 @@ def parse_single_example(serialized, features, name=None, example_names=None): Returns: A `dict` mapping feature keys to `Tensor` and `SparseTensor` values. + Raises: + ValueError: if any feature is invalid. + """ + return parse_single_example_v2_unoptimized( + serialized, features, example_names, name + ) + + +# TODO(b/70890287): Combine the implementation of this op and +# `parse_single_example_v2()` after 1/10/2018. +@tf_export("io.parse_single_example", v1=[]) +def parse_single_example_v2_unoptimized( + serialized, features, example_names=None, name=None + ): + """Parses a single `Example` proto. + + Similar to `parse_example`, except: + + For dense tensors, the returned `Tensor` is identical to the output of + `parse_example`, except there is no batch dimension, the output shape is the + same as the shape given in `dense_shape`. + + For `SparseTensor`s, the first (batch) column of the indices matrix is removed + (the indices matrix is a column vector), the values vector is unchanged, and + the first (`batch_size`) entry of the shape vector is removed (it is now a + single element vector). + + One might see performance advantages by batching `Example` protos with + `parse_example` instead of using this function directly. + + Args: + serialized: A scalar string Tensor, a single serialized Example. + See `_parse_single_example_raw` documentation for more details. + features: A `dict` mapping feature keys to `FixedLenFeature` or + `VarLenFeature` values. + example_names: (Optional) A scalar string Tensor, the associated name. + See `_parse_single_example_raw` documentation for more details. + name: A name for this operation (optional). + + Returns: + A `dict` mapping feature keys to `Tensor` and `SparseTensor` values. + Raises: ValueError: if any feature is invalid. """ @@ -1570,7 +1828,7 @@ def _parse_single_sequence_example_raw(serialized, # Swap `name` and `na_value` for backward compatibility. -@tf_export("io.decode_csv", v1=["io.decode_csv", "decode_csv"]) +@tf_export(v1=["io.decode_csv", "decode_csv"]) @deprecation.deprecated_endpoints("decode_csv") def decode_csv(records, record_defaults, @@ -1609,6 +1867,54 @@ def decode_csv(records, A list of `Tensor` objects. Has the same type as `record_defaults`. Each tensor will have the same shape as records. + Raises: + ValueError: If any of the arguments is malformed. + """ + return decode_csv_v2( + records, record_defaults, + field_delim, use_quote_delim, + na_value, select_cols, name + ) + + +@tf_export("io.decode_csv", v1=[]) +def decode_csv_v2(records, + record_defaults, + field_delim=",", + use_quote_delim=True, + na_value="", + select_cols=None, + name=None): + """Convert CSV records to tensors. Each column maps to one tensor. + + RFC 4180 format is expected for the CSV records. + (https://tools.ietf.org/html/rfc4180) + Note that we allow leading and trailing spaces with int or float field. + + Args: + records: A `Tensor` of type `string`. + Each string is a record/row in the csv and all records should have + the same format. + record_defaults: A list of `Tensor` objects with specific types. + Acceptable types are `float32`, `float64`, `int32`, `int64`, `string`. + One tensor per column of the input record, with either a + scalar default value for that column or an empty vector if the column is + required. + field_delim: An optional `string`. Defaults to `","`. + char delimiter to separate fields in a record. + use_quote_delim: An optional `bool`. Defaults to `True`. + If false, treats double quotation marks as regular + characters inside of the string fields (ignoring RFC 4180, Section 2, + Bullet 5). + na_value: Additional string to recognize as NA/NaN. + select_cols: Optional sorted list of column indices to select. If specified, + only this subset of columns will be parsed and returned. + name: A name for the operation (optional). + + Returns: + A list of `Tensor` objects. Has the same type as `record_defaults`. + Each tensor will have the same shape as records. + Raises: ValueError: If any of the arguments is malformed. """ diff --git a/tensorflow/python/ops/partitioned_variables.py b/tensorflow/python/ops/partitioned_variables.py index 7743b634e8f..c1084c25592 100644 --- a/tensorflow/python/ops/partitioned_variables.py +++ b/tensorflow/python/ops/partitioned_variables.py @@ -57,7 +57,7 @@ import math from tensorflow.python.framework import dtypes from tensorflow.python.framework import tensor_shape from tensorflow.python.ops import variable_scope -from tensorflow.python.platform import tf_logging as logging +from tensorflow.python.util import deprecation from tensorflow.python.util.tf_export import tf_export __all__ = [ @@ -68,7 +68,7 @@ __all__ = [ ] -@tf_export("variable_axis_size_partitioner") +@tf_export(v1=["variable_axis_size_partitioner"]) def variable_axis_size_partitioner( max_shard_bytes, axis=0, bytes_per_string_element=16, max_shards=None): """Get a partitioner for VariableScope to keep shards below `max_shard_bytes`. @@ -96,7 +96,7 @@ def variable_axis_size_partitioner( Returns: A partition function usable as the `partitioner` argument to - `variable_scope`, `get_variable`, and `get_partitioned_variable_list`. + `variable_scope` and `get_variable`. Raises: ValueError: If any of the byte counts are non-positive. @@ -154,7 +154,7 @@ def variable_axis_size_partitioner( return _partitioner -@tf_export("min_max_variable_partitioner") +@tf_export(v1=["min_max_variable_partitioner"]) def min_max_variable_partitioner(max_partitions=1, axis=0, min_slice_size=256 << 10, bytes_per_string_element=16): @@ -175,7 +175,7 @@ def min_max_variable_partitioner(max_partitions=1, axis=0, Returns: A partition function usable as the `partitioner` argument to - `variable_scope`, `get_variable`, and `get_partitioned_variable_list`. + `variable_scope` and `get_variable`. """ def _partitioner(shape, dtype): @@ -218,7 +218,7 @@ def min_max_variable_partitioner(max_partitions=1, axis=0, return _partitioner -@tf_export("fixed_size_partitioner") +@tf_export(v1=["fixed_size_partitioner"]) def fixed_size_partitioner(num_shards, axis=0): """Partitioner to specify a fixed number of shards along given axis. @@ -228,7 +228,7 @@ def fixed_size_partitioner(num_shards, axis=0): Returns: A partition function usable as the `partitioner` argument to - `variable_scope`, `get_variable`, and `get_partitioned_variable_list`. + `variable_scope` and `get_variable`. """ def _partitioner(shape, **unused_args): partitions_list = [1] * len(shape) @@ -237,7 +237,10 @@ def fixed_size_partitioner(num_shards, axis=0): return _partitioner -@tf_export("create_partitioned_variables") +@tf_export(v1=["create_partitioned_variables"]) +@deprecation.deprecated( + date=None, + instructions="Use tf.get_variable with a partitioner set.") def create_partitioned_variables( shape, slicing, initializer, dtype=dtypes.float32, trainable=True, collections=None, name=None, reuse=None): @@ -282,11 +285,6 @@ def create_partitioned_variables( Raises: ValueError: If any of the arguments is malformed. """ - logging.warn( - "create_partitioned_variables is deprecated. Use " - "tf.get_variable with a partitioner set, or " - "tf.get_partitioned_variable_list, instead.") - if len(shape) != len(slicing): raise ValueError("The 'shape' and 'slicing' of a partitioned Variable " "must have the length: shape: %s, slicing: %s" % diff --git a/tensorflow/python/ops/quantized_conv_ops_test.py b/tensorflow/python/ops/quantized_conv_ops_test.py index f7fa264461e..6b469a954f6 100644 --- a/tensorflow/python/ops/quantized_conv_ops_test.py +++ b/tensorflow/python/ops/quantized_conv_ops_test.py @@ -73,7 +73,7 @@ class Conv2DTest(test.TestCase): max_input=x1_max, min_filter=x2_min, max_filter=x2_max) - value = sess.run(conv) + value = self.evaluate(conv) quantized_output = value[0] output_min = value[1] output_max = value[2] diff --git a/tensorflow/python/ops/quantized_ops_test.py b/tensorflow/python/ops/quantized_ops_test.py index 0f3b04e4ad0..b81843d1748 100644 --- a/tensorflow/python/ops/quantized_ops_test.py +++ b/tensorflow/python/ops/quantized_ops_test.py @@ -41,7 +41,7 @@ class QuantizedOpsTest(test.TestCase): x_min = 0.0 x_max = 255.0 op = array_ops.quantize(x, x_min, x_max, dtypes.quint8, mode="MIN_FIRST") - value = sess.run(op) + value = self.evaluate(op) self.assertArrayNear(expected_output, value.output, 0.1) def testDequantizeOp(self): @@ -52,7 +52,7 @@ class QuantizedOpsTest(test.TestCase): x_min = 0.0 x_max = 255.0 op = array_ops.dequantize(x, x_min, x_max, mode="MIN_FIRST") - value = sess.run(op) + value = self.evaluate(op) self.assertArrayNear(expected_output, value, 0.1) diff --git a/tensorflow/python/ops/ragged/BUILD b/tensorflow/python/ops/ragged/BUILD index 152c6dc8416..e335c5cb6f3 100644 --- a/tensorflow/python/ops/ragged/BUILD +++ b/tensorflow/python/ops/ragged/BUILD @@ -32,7 +32,9 @@ py_library( ":ragged_map_ops", ":ragged_math_ops", ":ragged_operators", + ":ragged_string_ops", ":ragged_tensor", + ":ragged_tensor_shape", ":ragged_tensor_value", ":ragged_util", ":segment_id_ops", @@ -155,6 +157,7 @@ py_library( deps = [ ":ragged_factory_ops", ":ragged_tensor", + ":ragged_tensor_shape", ":ragged_util", "//tensorflow/python:array_ops", "//tensorflow/python:clip_ops", @@ -178,6 +181,21 @@ py_library( ], ) +py_library( + name = "ragged_string_ops", + srcs = ["ragged_string_ops.py"], + srcs_version = "PY2AND3", + deps = [ + ":ragged_array_ops", + ":ragged_conversion_ops", + ":ragged_factory_ops", + ":ragged_tensor", + "//tensorflow/python:array_ops", + "//tensorflow/python:math_ops", + "//tensorflow/python:util", + ], +) + py_library( name = "ragged_tensor", srcs = ["ragged_tensor.py"], @@ -190,6 +208,25 @@ py_library( ], ) +py_library( + name = "ragged_tensor_shape", + srcs = ["ragged_tensor_shape.py"], + srcs_version = "PY2AND3", + deps = [ + ":ragged_array_ops", + ":ragged_conversion_ops", + ":ragged_factory_ops", + ":ragged_tensor", + ":ragged_util", + "//tensorflow/python:array_ops", + "//tensorflow/python:dtypes", + "//tensorflow/python:framework_ops", + "//tensorflow/python:math_ops", + "//tensorflow/python:tensor_shape", + "//tensorflow/python:tensor_util", + ], +) + py_library( name = "ragged_tensor_value", srcs = ["ragged_tensor_value.py"], @@ -207,6 +244,7 @@ py_library( "//tensorflow/python:dtypes", "//tensorflow/python:framework_ops", "//tensorflow/python:math_ops", + "//tensorflow/python:ragged_math_ops_gen", ], ) @@ -256,6 +294,9 @@ py_test( size = "medium", srcs = ["ragged_tensor_test.py"], srcs_version = "PY2AND3", + tags = [ + "no_windows", + ], deps = [ ":ragged", "//tensorflow/python:array_ops", @@ -407,6 +448,9 @@ py_test( name = "ragged_to_sparse_op_test", srcs = ["ragged_to_sparse_op_test.py"], srcs_version = "PY2AND3", + tags = [ + "no_windows", + ], deps = [ ":ragged", "//tensorflow/python:array_ops", @@ -513,6 +557,9 @@ py_test( name = "ragged_constant_value_op_test", srcs = ["ragged_constant_value_op_test.py"], srcs_version = "PY2AND3", + tags = [ + "no_windows", + ], deps = [ ":ragged", "//tensorflow/python:framework_test_lib", @@ -681,3 +728,15 @@ py_test( "@absl_py//absl/testing:parameterized", ], ) + +py_test( + name = "ragged_tensor_shape_test", + srcs = ["ragged_tensor_shape_test.py"], + srcs_version = "PY2AND3", + deps = [ + ":ragged", + "//tensorflow/python:framework_test_lib", + "//tensorflow/python:platform_test", + "@absl_py//absl/testing:parameterized", + ], +) diff --git a/tensorflow/python/ops/ragged/__init__.py b/tensorflow/python/ops/ragged/__init__.py index 3a288485454..1b2a7be95fc 100644 --- a/tensorflow/python/ops/ragged/__init__.py +++ b/tensorflow/python/ops/ragged/__init__.py @@ -143,6 +143,11 @@ The following operations are specific to ragged tensors: @@make_elementwise_op + +@@RaggedTensorDynamicShape +@@broadcast_to +@@broadcast_dynamic_shape + """ @@ -151,6 +156,7 @@ from __future__ import division from __future__ import print_function from tensorflow.python.ops.ragged import ragged_operators +from tensorflow.python.ops.ragged import ragged_string_ops from tensorflow.python.ops.ragged.ragged_array_ops import batch_gather from tensorflow.python.ops.ragged.ragged_array_ops import boolean_mask @@ -214,6 +220,10 @@ from tensorflow.python.ops.ragged.ragged_tensor import is_ragged from tensorflow.python.ops.ragged.ragged_tensor import RaggedTensor from tensorflow.python.ops.ragged.ragged_tensor import RaggedTensorType +from tensorflow.python.ops.ragged.ragged_tensor_shape import broadcast_dynamic_shape +from tensorflow.python.ops.ragged.ragged_tensor_shape import broadcast_to +from tensorflow.python.ops.ragged.ragged_tensor_shape import RaggedTensorDynamicShape + from tensorflow.python.ops.ragged.ragged_tensor_value import RaggedTensorValue from tensorflow.python.ops.ragged.segment_id_ops import row_splits_to_segment_ids diff --git a/tensorflow/python/ops/ragged/convert_to_tensor_or_ragged_tensor_op_test.py b/tensorflow/python/ops/ragged/convert_to_tensor_or_ragged_tensor_op_test.py index b43470dfa11..ef3464f2437 100644 --- a/tensorflow/python/ops/ragged/convert_to_tensor_or_ragged_tensor_op_test.py +++ b/tensorflow/python/ops/ragged/convert_to_tensor_or_ragged_tensor_op_test.py @@ -90,6 +90,7 @@ class RaggedConvertToTensorOrRaggedTensorTest(test_util.TensorFlowTestCase, preferred_dtype=dtypes.string, expected_dtype=dtypes.int32), ]) + @test_util.run_deprecated_v1 def testConvertRaggedTensorValue(self, value, dtype=None, @@ -102,7 +103,7 @@ class RaggedConvertToTensorOrRaggedTensorTest(test_util.TensorFlowTestCase, self.assertEqual(value.ragged_rank, converted.ragged_rank) self.assertEqual(dtypes.as_dtype(expected_dtype), converted.dtype) with self.test_session(): - self.assertEqual(value.tolist(), converted.eval().tolist()) + self.assertEqual(value.tolist(), self.evaluate(converted).tolist()) @parameterized.parameters([ dict( @@ -145,6 +146,7 @@ class RaggedConvertToTensorOrRaggedTensorTest(test_util.TensorFlowTestCase, message=('Tensor conversion requested dtype string for ' 'Tensor with dtype int32')), ]) + @test_util.run_deprecated_v1 def testConvertTensorError(self, pylist, message, diff --git a/tensorflow/python/ops/ragged/ragged_array_ops.py b/tensorflow/python/ops/ragged/ragged_array_ops.py index 425f3957c38..603e39d1dcf 100644 --- a/tensorflow/python/ops/ragged/ragged_array_ops.py +++ b/tensorflow/python/ops/ragged/ragged_array_ops.py @@ -225,6 +225,28 @@ def row_lengths(rt_input, axis=1, name=None): return array_ops.ones(shape[:axis], dtypes.int64) * shape[axis] +def nested_row_lengths(rt_input, name=None): + """Returns a tuple containing the row_lengths for all ragged dimensions. + + `nested_row_lengths(rt)` is a tuple containing the `row_lengths` tensors for + all ragged dimensions in `rt`, ordered from outermost to innermost. + + Args: + rt_input: A potentially ragged tensor. + name: A name prefix for the returned tensors (optional). + + Returns: + A `tuple` of 1-D `int64` `Tensors`. The length of the tuple is equal to + `rt_input.ragged_rank`. + """ + with ops.name_scope(name, 'RaggedNestedRowLengths', [rt_input]): + rt_nested_row_lengths = [] + while isinstance(rt_input, ragged_tensor.RaggedTensor): + rt_nested_row_lengths.append(row_lengths(rt_input)) + rt_input = rt_input.values + return tuple(rt_nested_row_lengths) + + #=============================================================================== # Bounding Shape #=============================================================================== @@ -451,8 +473,7 @@ def batch_gather(params, indices, name=None): adjusted_indices = math_ops.to_int64(indices) + adjustments return gather(params.values, adjusted_indices) else: - raise ValueError( - 'batch shape from indices does not match params shape') + raise ValueError('batch shape from indices does not match params shape') #=============================================================================== @@ -719,7 +740,7 @@ def boolean_mask(data, mask, keepdims=False, name=None): int_mask = ragged_functional_ops.map_inner_values( math_ops.cast, mask, dtype=dtypes.int64) masked_row_lengths = ragged_math_ops.reduce_sum(int_mask, axis=1) - splits.append(_lengths_to_splits(masked_row_lengths)) + splits.append(ragged_util.lengths_to_splits(masked_row_lengths)) mask = mask.values data = data.values @@ -741,7 +762,7 @@ def boolean_mask(data, mask, keepdims=False, name=None): # masks back to a splits tensor. lengths = row_lengths(data) masked_lengths = array_ops.boolean_mask(lengths, mask) - masked_splits = _lengths_to_splits(masked_lengths) + masked_splits = ragged_util.lengths_to_splits(masked_lengths) # Get the masked values: first get row ids corresponding to each # value, then use tf.gather to build a boolean mask that's false for @@ -977,7 +998,7 @@ def _ragged_stack_concat_axis_0(rt_inputs, stack_values): # If we are performing a stack operation, then add another splits. if stack_values: stack_lengths = array_ops.stack([nrows(rt) for rt in rt_inputs]) - stack_splits = _lengths_to_splits(stack_lengths) + stack_splits = ragged_util.lengths_to_splits(stack_lengths) concatenated_nested_splits.insert(0, stack_splits) return ragged_factory_ops.from_nested_row_splits(concatenated_inner_values, @@ -1131,7 +1152,8 @@ def _tile_ragged_values(rt_input, multiples, const_multiples=None): # Repeat each element in this ragged dimension `multiples[axis]` times. if const_multiples is None or const_multiples[axis] != 1: - inner_value_ids = _repeat_ranges(inner_value_ids, splits, multiples[axis]) + inner_value_ids = ragged_util.repeat_ranges(inner_value_ids, splits, + multiples[axis]) prev_splits = splits @@ -1172,6 +1194,17 @@ def _tile_ragged_splits(rt_input, multiples, const_multiples=None): ragged_rank = rt_input.ragged_rank nested_splits = rt_input.nested_row_splits + # projected_splits[src_axis, dst_axis] contains the split points that divide + # the rows from src_axis in the list of dst_axis values. E.g., + # projected_splits[i, i] = nested_splits[i], and + # projected_splits[i, i+1] = gather(nested_splits[i+1], nested_splits[i]). + projected_splits = [{i: nested_splits[i]} for i in range(ragged_rank)] + for src_axis in range(ragged_rank): + for dst_axis in range(src_axis + 1, ragged_rank - 1): + projected_splits[src_axis][dst_axis] = array_ops.gather( + nested_splits[dst_axis], + projected_splits[src_axis][dst_axis - 1]) + # For each ragged dimension: nested_splits[axis] -> result_splits[axis]. result_splits = [] for axis in range(ragged_rank): @@ -1188,16 +1221,16 @@ def _tile_ragged_splits(rt_input, multiples, const_multiples=None): repeats = 1 for d in range(axis - 1, -1, -1): if const_multiples is None or const_multiples[d + 1] != 1: - splits = nested_splits[d] * repeats - output_lengths = _repeat_ranges(output_lengths, splits, - multiples[d + 1]) + splits = projected_splits[d][axis - 1] * repeats + output_lengths = ragged_util.repeat_ranges(output_lengths, splits, + multiples[d + 1]) repeats *= multiples[d + 1] # Tile splits for the outermost (uniform) dimension. output_lengths = array_ops.tile(output_lengths, multiples[:1]) # Convert to splits. - result_splits.append(_lengths_to_splits(output_lengths)) + result_splits.append(ragged_util.lengths_to_splits(output_lengths)) return result_splits @@ -1425,11 +1458,6 @@ def _coordinate_where(condition): #=============================================================================== -def _lengths_to_splits(lengths): - """Returns splits corresponding to the given lengths.""" - return array_ops.concat([[0], math_ops.cumsum(lengths)], axis=0) - - def _increase_ragged_rank_to(rt_input, ragged_rank): """Adds ragged dimensions to `rt_input` so it has the desired ragged rank.""" if ragged_rank > 0: @@ -1449,45 +1477,3 @@ def _concat_ragged_splits(splits_list): pieces.append(splits[1:] + splits_offset) splits_offset += splits[-1] return array_ops.concat(pieces, axis=0) - - -def _repeat_ranges(params, splits, multiple): - """Repeats each range of `params` (as specified by `splits`) `multiple` times. - - Let the `i`th range of `params` be defined as - `params[splits[i]:splits[i + 1]]`. Then this function returns a tensor - containing range 0 repeated `multiple` times, followed by range 1 repeated - `multiple`, ..., followed by the last range repeated `multiple` times. - - Args: - params: The `Tensor` whose values should be repeated. - splits: A splits tensor indicating the ranges of `params` that should be - repeated. - multiple: The number of times each range should be repeated. - - Returns: - A `Tensor` with the same rank and type as `params`. - - #### Example: - ```python - >>> _repeat_ranges(['a', 'b', 'c'], [0, 2, 3], 3) - ['a', 'b', 'a', 'b', 'a', 'b', 'c', 'c', 'c'] - ``` - """ - # Repeat each split value `multiple` times. E.g., if `splits=[0 3 4]` and - # `multiples=3`, then `repeated_splits=[0 0 0 3 3 3 4 4 4]`. - repeated_splits = array_ops.tile( - array_ops.expand_dims(splits, axis=1), array_ops.stack([1, multiple])) - repeated_splits = array_ops.reshape(repeated_splits, [-1]) - - # Divide the splits into repeated starts & repeated limits. E.g., if - # `repeated_splits=[0 0 0 3 3 3 4 4 4]` then `repeated_starts=[0 0 0 3 3 3]` - # and `repeated_limits=[3 3 3 4 4 4]`. - n_splits = array_ops.shape(repeated_splits, out_type=dtypes.int64)[0] - repeated_starts = repeated_splits[:n_splits - multiple] - repeated_limits = repeated_splits[multiple:] - - # Get indices for each range from starts to limits, and use those to gather - # the values in the desired repetition pattern. - offsets = ragged_math_ops.range(repeated_starts, repeated_limits).values - return array_ops.gather(params, offsets) diff --git a/tensorflow/python/ops/ragged/ragged_batch_gather_op_test.py b/tensorflow/python/ops/ragged/ragged_batch_gather_op_test.py index 79a2ecd87ae..d9d840500cb 100644 --- a/tensorflow/python/ops/ragged/ragged_batch_gather_op_test.py +++ b/tensorflow/python/ops/ragged/ragged_batch_gather_op_test.py @@ -135,6 +135,7 @@ class RaggedBatchGatherOpTest(test_util.TensorFlowTestCase, expected=ragged.constant_value( [[[[b'c', b'a'], [b'd', b'd']], [[b'f', b'e']]]], ragged_rank=2)), ]) + @test_util.run_deprecated_v1 def testRaggedBatchGather(self, descr, params, indices, expected): result = ragged.batch_gather(params, indices) self.assertEqual( @@ -144,6 +145,7 @@ class RaggedBatchGatherOpTest(test_util.TensorFlowTestCase, expected = expected.tolist() self.assertEqual(result.eval().tolist(), expected) + @test_util.run_deprecated_v1 def testRaggedBatchGatherUnknownRankError(self): params = [['a', 'b'], ['c', 'd']] indices = array_ops.placeholder(dtypes.int32, shape=None) @@ -186,6 +188,7 @@ class RaggedBatchGatherOpTest(test_util.TensorFlowTestCase, indices=[[[0]]], message='batch shape from indices does not match params shape'), ]) + @test_util.run_deprecated_v1 def testRaggedBatchGatherStaticError(self, params, indices, diff --git a/tensorflow/python/ops/ragged/ragged_boolean_mask_op_test.py b/tensorflow/python/ops/ragged/ragged_boolean_mask_op_test.py index b3279c1e840..d939d9d6341 100644 --- a/tensorflow/python/ops/ragged/ragged_boolean_mask_op_test.py +++ b/tensorflow/python/ops/ragged/ragged_boolean_mask_op_test.py @@ -298,6 +298,7 @@ class RaggedBooleanMaskOpTest(test_util.TensorFlowTestCase, keepdims=True, expected=ragged.constant_value([[[1], [4, 6]], [[7, 9], []]])), ]) # pyformat: disable + @test_util.run_deprecated_v1 def testBooleanMask(self, descr, data, mask, keepdims, expected): actual = ragged.boolean_mask(data, mask, keepdims=keepdims) self.assertEqual( @@ -307,6 +308,7 @@ class RaggedBooleanMaskOpTest(test_util.TensorFlowTestCase, expected = expected.tolist() self.assertEqual(actual.eval().tolist(), expected) + @test_util.run_deprecated_v1 def testErrors(self): self.assertRaisesRegexp(ValueError, r'mask\.shape\.ndims must be kown statically', diff --git a/tensorflow/python/ops/ragged/ragged_concat_op_test.py b/tensorflow/python/ops/ragged/ragged_concat_op_test.py index 6b1a602d049..3699f90f46b 100644 --- a/tensorflow/python/ops/ragged/ragged_concat_op_test.py +++ b/tensorflow/python/ops/ragged/ragged_concat_op_test.py @@ -41,6 +41,11 @@ class RaggedConcatOpTest(test_util.TensorFlowTestCase, parameterized.TestCase): ] @parameterized.parameters( + dict( + descr='Two rank-2 inputs with empty value axis=1', + rt_inputs=([[]], [[]]), + axis=1, + expected=[[]]), dict( descr='Two rank-2 inputs (ragged_rank=1), axis=0', rt_inputs=( @@ -216,6 +221,7 @@ class RaggedConcatOpTest(test_util.TensorFlowTestCase, parameterized.TestCase): axis=0, expected=[[b'a00', b'a01'], [], [b'a20', b'a21']]), ) # pyformat: disable + @test_util.run_deprecated_v1 def testRaggedConcat(self, descr, rt_inputs, @@ -261,6 +267,7 @@ class RaggedConcatOpTest(test_util.TensorFlowTestCase, parameterized.TestCase): error=ValueError, message='Dimension 0 in both shapes must be equal'), ) + @test_util.run_deprecated_v1 def testStaticError(self, rt_inputs, axis, error, message, ragged_ranks=None): rt_inputs = self._rt_inputs_to_tensors(rt_inputs, ragged_ranks) self.assertRaisesRegexp(error, message, ragged.concat, rt_inputs, axis) @@ -273,6 +280,7 @@ class RaggedConcatOpTest(test_util.TensorFlowTestCase, parameterized.TestCase): error=errors.InvalidArgumentError, message='Input tensors have incompatible shapes'), ]) + @test_util.run_deprecated_v1 def testRuntimeError(self, rt_inputs, axis, error, message, ragged_ranks=None): rt_inputs = [ @@ -282,6 +290,7 @@ class RaggedConcatOpTest(test_util.TensorFlowTestCase, parameterized.TestCase): with self.test_session(): self.assertRaisesRegexp(error, message, concatenated.eval) + @test_util.run_deprecated_v1 def testNegativeAxisWithUnknownRankError(self): rt_inputs = [ array_ops.placeholder(dtypes.int64), @@ -291,6 +300,7 @@ class RaggedConcatOpTest(test_util.TensorFlowTestCase, parameterized.TestCase): ValueError, r'axis may only be negative if ndims is statically known.', ragged.concat, rt_inputs, -1) + @test_util.run_deprecated_v1 def testSingleTensorInput(self): """Tests ragged_concat with a single tensor input. diff --git a/tensorflow/python/ops/ragged/ragged_const_op_test.py b/tensorflow/python/ops/ragged/ragged_const_op_test.py index 13f79c57292..2505b23912a 100644 --- a/tensorflow/python/ops/ragged/ragged_const_op_test.py +++ b/tensorflow/python/ops/ragged/ragged_const_op_test.py @@ -133,6 +133,7 @@ class RaggedConstOpTest(test_util.TensorFlowTestCase, parameterized.TestCase): dict(pylist=[[b'a', b'b'], [b'c'], [b'd', b'e', b'f']], dtype=dtypes.string), ) + @test_util.run_deprecated_v1 def testRaggedConst(self, pylist, dtype=None, @@ -183,7 +184,7 @@ class RaggedConstOpTest(test_util.TensorFlowTestCase, parameterized.TestCase): self.assertEqual(tuple(rt.shape.as_list()), expected_shape) with self.test_session(): - result = rt.eval() + result = self.evaluate(rt) if rt.shape.ndims > 0: self.assertEqual(result.tolist(), pylist) if expected_shape is not None: @@ -238,8 +239,8 @@ class RaggedConstOpTest(test_util.TensorFlowTestCase, parameterized.TestCase): dict( pylist=[1, 2, 3], inner_shape=(1, 1), - exception=ValueError, - message='Too many elements provided.'), + exception=TypeError, + message='Expected Tensor\'s shape'), dict( pylist=[[[1, 2], [3, 4]], [[5, 6], [7, 8]]], inner_shape=(2, 2), @@ -258,6 +259,7 @@ class RaggedConstOpTest(test_util.TensorFlowTestCase, parameterized.TestCase): exception=ValueError, message='inner values have inconsistent shape'), ) + @test_util.run_deprecated_v1 def testRaggedConstError(self, pylist, dtype=None, diff --git a/tensorflow/python/ops/ragged/ragged_conversion_ops.py b/tensorflow/python/ops/ragged/ragged_conversion_ops.py index 3ec246ccaf1..83212e49cf7 100644 --- a/tensorflow/python/ops/ragged/ragged_conversion_ops.py +++ b/tensorflow/python/ops/ragged/ragged_conversion_ops.py @@ -196,7 +196,7 @@ def to_tensor(rt_input, default_value=None, name=None): Args: rt_input: The input `RaggedTensor`. default_value: Value to set for indices not specified in `rt_input`. - Defaults to zero. `default_value.shape` must be equal to + Defaults to zero. `default_value` must be broadcastable to `rt_input.shape[rt_input.ragged_rank + 1:]`. name: A name prefix for the returned tensors (optional). @@ -210,6 +210,9 @@ def to_tensor(rt_input, default_value=None, name=None): rt_input, name='rt_input') if not ragged_tensor.is_ragged(rt_input): return rt_input # already dense + if default_value is not None: + default_value = ops.convert_to_tensor( + default_value, name='default_value', dtype=rt_input.dtype) # If ragged_rank > 1, then recursively convert the ragged values into a # `Tensor` before we proceed. @@ -217,6 +220,16 @@ def to_tensor(rt_input, default_value=None, name=None): if ragged_tensor.is_ragged(values): values = to_tensor(values, default_value) + # Tile the default value, if necessary. + if default_value is not None: + if values.shape.ndims is not None: + default_value.shape.with_rank_at_most(values.shape.ndims - 1) + if (values.shape.ndims is None or default_value.shape.ndims is None or + values.shape.ndims != default_value.shape.ndims + 1): + value_shape = array_ops.shape(values)[1:] + default_value = array_ops.broadcast_to(default_value, value_shape) + default_value.shape.assert_is_compatible_with(values.shape[1:]) + # Get the expected dense shape ([nrows, ncols] + value_shape). rt_row_lengths = [rt_input.row_splits[1:] - rt_input.row_splits[:-1]] nrows = array_ops.shape(rt_input.row_splits, out_type=dtypes.int64)[0] - 1 @@ -228,9 +241,6 @@ def to_tensor(rt_input, default_value=None, name=None): # Build a default value if none was supplied. if default_value is None: default_value = array_ops.zeros(value_shape, dtype=values.dtype) - else: - default_value = ops.convert_to_tensor( - default_value, name='default_value', dtype=values.dtype) default_value.shape.assert_is_compatible_with(values.shape[1:]) default_value.set_shape(values.shape[1:]) @@ -351,9 +361,14 @@ def from_sparse(st_input, name=None): st_input = sparse_tensor.convert_to_tensor_or_sparse_tensor( st_input, name='rt_input') - if (st_input.dense_shape.shape.ndims != 2 and - st_input.indices.shape.ndims is None or - st_input.indices.shape.dims[1].value != 2): + static_rank_from_dense_shape = ( + None if st_input.dense_shape.shape.ndims is None + else st_input.dense_shape.shape.dims[0].value) + static_rank_from_indices = ( + None if st_input.indices.shape.ndims is None + else st_input.indices.shape.dims[1].value) + + if static_rank_from_dense_shape != 2 and static_rank_from_indices != 2: raise ValueError('rank(st_input) must be 2') with ops.control_dependencies( diff --git a/tensorflow/python/ops/ragged/ragged_elementwise_ops.py b/tensorflow/python/ops/ragged/ragged_elementwise_ops.py index 23d0e8b5fc4..59b7dd16617 100644 --- a/tensorflow/python/ops/ragged/ragged_elementwise_ops.py +++ b/tensorflow/python/ops/ragged/ragged_elementwise_ops.py @@ -28,7 +28,7 @@ from tensorflow.python.ops import parsing_ops from tensorflow.python.ops import string_ops from tensorflow.python.ops.ragged import ragged_factory_ops from tensorflow.python.ops.ragged import ragged_tensor -from tensorflow.python.ops.ragged import ragged_util +from tensorflow.python.ops.ragged import ragged_tensor_shape from tensorflow.python.util import tf_decorator from tensorflow.python.util import tf_export from tensorflow.python.util import tf_inspect @@ -209,28 +209,45 @@ def _broadcast_elementwise_args(elementwise_args): if not any(is_ragged): return elementwise_args, (), () - # Support limited broadcasting (namely, scalar + ragged). Full - # broadcasting support will be added later. - if all((ragged_tensor.is_ragged(t) or t.shape.ndims == 0) - for t in elementwise_args.values()): + # If we have a single ragged tensor plus a set of scalars, then we can + # rely on the underlying elementwise op to do broadcasting. + if (sum(is_ragged) == 1 and + all((ragged_tensor.is_ragged(t) or t.shape.ndims == 0) + for t in elementwise_args.values())): nested_splits_lists = [ t.nested_row_splits for t in elementwise_args.values() - if ragged_tensor.is_ragged(t) - ] - if len(nested_splits_lists) == 1: - checks = () - else: - if any(t.shape.ndims is None for t in elementwise_args.values()): - raise ValueError('Ragged elementwise ops require that rank (number ' - 'of dimensions) be statically known.') - if len(set(t.shape.ndims for t in elementwise_args.values())) != 1: - raise ValueError('Ragged elementwise ops do not support ' - 'broadcasting yet') - checks = ragged_util.assert_splits_match(nested_splits_lists) - return (elementwise_args, nested_splits_lists[0], checks) + if ragged_tensor.is_ragged(t)][0] + return elementwise_args, nested_splits_lists, () + else: - raise ValueError('Ragged elementwise ops do not support broadcasting yet') + # Get the shapes of all the elementwise arguments. + shapes = [ragged_tensor_shape.RaggedTensorDynamicShape.from_tensor(t) + for t in elementwise_args.values()] + + # Broadcast the shapes to all have the same rank (the max rank). + ranks = [t.shape.ndims for t in elementwise_args.values()] + if any(rank is None for rank in ranks): + raise ValueError('Unable to broadcast: unknown rank') + broadcast_rank = max(ranks) + shapes = [shape.broadcast_to_rank(broadcast_rank) for shape in shapes] + + # For each dimension, broadcast the shapes to be compatible. + for axis in range(broadcast_rank): + # For each i, broadcast shape[i+1] to be compatible with shape[i]; and + # then finally broadcast shape[0] to be compatible with shape[-1]. + for i in range(len(shapes)): + j = (i + 1) % len(shapes) + dim_size = shapes[i].dimension_size(axis) + shapes[j] = shapes[j].broadcast_dimension(axis, dim_size) + broadcast_shape = shapes[0] + + # Broadcast every elementwise arg to the shape that we calculated. + elementwise_args = dict([ + (key, ragged_tensor_shape.broadcast_to(t, broadcast_shape, False)) + for (key, t) in elementwise_args.items()]) + nested_splits_lists = list(elementwise_args.values())[0].nested_row_splits + return elementwise_args, nested_splits_lists, () # A list of symbols that should be exported in the "ragged" package. @@ -252,6 +269,10 @@ def _add_elementwise_ops_to_this_module(specs, verbose=False): op_name = canonical_name else: op_name = original_op.__name__ + + # Temporary hack (will be removed once dispatch is added for RaggedTensors): + if op_name == 'neg': op_name = 'negative' + if verbose: print('Adding ragged_elementwise_op: tf.ragged.%s (based on tf.%s)' % (op_name, canonical_name)) @@ -348,7 +369,7 @@ _TF_ELEMENTWISE_OPS = [ (string_ops.regex_replace, 'input'), (string_ops.string_join, '[inputs]'), (string_ops.string_strip, 'input'), - (string_ops.string_to_hash_bucket, 'string_tensor'), + (string_ops.string_to_hash_bucket, 'input'), (string_ops.string_to_hash_bucket_fast, 'input'), (string_ops.string_to_hash_bucket_strong, 'input'), (string_ops.substr, 'input'), @@ -365,3 +386,4 @@ _TF_ELEMENTWISE_OPS = [ (parsing_ops.string_to_number, 'string_tensor'), ] _add_elementwise_ops_to_this_module(_TF_ELEMENTWISE_OPS) + diff --git a/tensorflow/python/ops/ragged/ragged_elementwise_ops_test.py b/tensorflow/python/ops/ragged/ragged_elementwise_ops_test.py index 5dfa5cff45d..305a96df9cc 100644 --- a/tensorflow/python/ops/ragged/ragged_elementwise_ops_test.py +++ b/tensorflow/python/ops/ragged/ragged_elementwise_ops_test.py @@ -394,49 +394,43 @@ class RaggedElementwiseOpsTest(test_util.TensorFlowTestCase, result_flat_values = array_ops.reshape(result, [-1]) self.assertAllEqual(expected_flat_values, result_flat_values) + @test_util.run_deprecated_v1 def testUnknownRankError(self): x = ragged.constant([[1, 2], [3]]) y = ragged.from_row_splits( array_ops.placeholder_with_default([1, 2, 3], shape=None), x.row_splits) with self.assertRaisesRegexp( - ValueError, r'Ragged elementwise ops require that rank \(number ' - r'of dimensions\) be statically known.'): + ValueError, r'Unable to broadcast: unknown rank'): ragged.add(x, y) - def testBroadcastError1(self): - x = ragged.constant([[1, 2], [3]]) - y = [[12]] - with self.assertRaisesRegexp( - ValueError, 'Ragged elementwise ops do not support broadcasting yet'): - ragged.add(x, y) - - def testBroadcastError2(self): - x = ragged.constant([[[1, 2], [3, 4]], [[5]]], ragged_rank=2) - y = ragged.constant([[[8], [3]], [[2]]], ragged_rank=1) - with self.assertRaisesRegexp(ValueError, - 'Inputs must have identical ragged splits'): - ragged.add(x, y) - - def testBroadcastError3(self): - x = ragged.constant([[[1, 2], [3]], [[4, 5], [6]]], ragged_rank=2) - y = ragged.constant([[7, 8], [9]], ragged_rank=1) - with self.assertRaisesRegexp( - ValueError, 'Ragged elementwise ops do not support broadcasting yet'): - ragged.add(x, y) - - def testBroadcastError4(self): - x = ragged.constant([[[1]]]) - y = ragged.constant([[1]]) - with self.assertRaisesRegexp( - ValueError, 'Ragged elementwise ops do not support broadcasting yet'): - ragged.add(x, y) + @parameterized.parameters([ + dict( + x=ragged.constant_value([[1, 2], [3]]), + y=[[10]], + expected=[[11, 12], [13]]), + dict( + x=ragged.constant_value([[[1, 2], [3, 4]], [[5]]], ragged_rank=2), + y=ragged.constant_value([[[10], [20]], [[30]]], ragged_rank=1), + expected=[[[11, 12], [23, 24]], [[35]]]), + dict( + x=ragged.constant_value([[[1]]]), + y=ragged.constant_value([[1]]), + expected=[[[2]]]), + ]) + def testBroadcastAdd(self, x, y, expected): + x = ragged.convert_to_tensor_or_ragged_tensor(x, dtype=dtypes.int32) + y = ragged.convert_to_tensor_or_ragged_tensor(y, dtype=dtypes.int32) + result = x + y + with self.cached_session(): + self.assertEqual(result.eval().tolist(), expected) def testShapeMismatch(self): x = ragged.constant([[1, 2, 3], [4, 5]]) y = ragged.constant([[1, 2, 3], [4, 5, 6]]) with self.assertRaisesRegexp(errors.InvalidArgumentError, - 'Inputs must have identical ragged splits'): - ragged.add(x, y) + 'Incompatible shapes'): + with self.cached_session(): + ragged.add(x, y).eval() def testDocstring(self): self.assertRegexpMatches( diff --git a/tensorflow/python/ops/ragged/ragged_expand_dims_op_test.py b/tensorflow/python/ops/ragged/ragged_expand_dims_op_test.py index 0c4fd458c23..3ff66973b6f 100644 --- a/tensorflow/python/ops/ragged/ragged_expand_dims_op_test.py +++ b/tensorflow/python/ops/ragged/ragged_expand_dims_op_test.py @@ -105,6 +105,7 @@ class RaggedExpandDimsOpTest(test_util.TensorFlowTestCase, expected=EXAMPLE4D_EXPAND_AXIS[4], expected_shape=[3, None, None, 2, 1]), ]) # pyformat: disable + @test_util.run_deprecated_v1 def testRaggedExpandDims(self, rt_input, axis, diff --git a/tensorflow/python/ops/ragged/ragged_factory_ops.py b/tensorflow/python/ops/ragged/ragged_factory_ops.py index de3a2d5b10b..d1f301bc58f 100644 --- a/tensorflow/python/ops/ragged/ragged_factory_ops.py +++ b/tensorflow/python/ops/ragged/ragged_factory_ops.py @@ -676,3 +676,33 @@ def from_nested_row_splits(inner_values, nested_row_splits, name=None): for splits in reversed(nested_row_splits): result = from_row_splits(result, splits) return result + + +def from_nested_row_lengths(inner_values, nested_row_lengths, name=None): + """Creates a `RaggedTensor` from a nested list of `row_lengths` tensors. + + Equivalent to: + + ```python + result = inner_values + for row_lengths in reversed(nested_row_lengths): + result = from_row_lengths(result, row_lengths) + ``` + + Args: + inner_values: A potentially ragged tensor. + nested_row_lengths: A list of 1-D int64 tensors. The `i`th tensor is used + as the `row_lengths` for the `i`th ragged dimension. + name: A name prefix for the RaggedTensor (optional). + + Returns: + A `RaggedTensor` (or `inner_values` if `nested_row_lengths` is empty). + """ + if isinstance(nested_row_lengths, ops.Tensor): + raise TypeError('nested_row_lengths must be a list of Tensors') + with ops.name_scope(name, 'RaggedFromNestedRowlengths', + [inner_values] + list(nested_row_lengths)): + result = inner_values + for lengths in reversed(nested_row_lengths): + result = from_row_lengths(result, lengths) + return result diff --git a/tensorflow/python/ops/ragged/ragged_from_sparse_op_test.py b/tensorflow/python/ops/ragged/ragged_from_sparse_op_test.py index ff19ddedebf..3c0db9e8fb6 100644 --- a/tensorflow/python/ops/ragged/ragged_from_sparse_op_test.py +++ b/tensorflow/python/ops/ragged/ragged_from_sparse_op_test.py @@ -29,6 +29,7 @@ from tensorflow.python.platform import googletest class RaggedTensorToSparseOpTest(test_util.TensorFlowTestCase): + @test_util.run_deprecated_v1 def testDocStringExample(self): st = sparse_tensor.SparseTensor( indices=[[0, 0], [0, 1], [0, 2], [1, 0], [3, 0]], @@ -39,6 +40,7 @@ class RaggedTensorToSparseOpTest(test_util.TensorFlowTestCase): with self.test_session(): self.assertEqual(rt.eval().tolist(), [[1, 2, 3], [4], [], [5]]) + @test_util.run_deprecated_v1 def testEmpty(self): st = sparse_tensor.SparseTensor( indices=array_ops.zeros([0, 2], dtype=dtypes.int64), @@ -49,6 +51,7 @@ class RaggedTensorToSparseOpTest(test_util.TensorFlowTestCase): with self.test_session(): self.assertEqual(rt.eval().tolist(), [[], [], [], []]) + @test_util.run_deprecated_v1 def testBadSparseTensorRank(self): st1 = sparse_tensor.SparseTensor(indices=[[0]], values=[0], dense_shape=[3]) st2 = sparse_tensor.SparseTensor( @@ -64,6 +67,22 @@ class RaggedTensorToSparseOpTest(test_util.TensorFlowTestCase): self.assertRaisesRegexp(ValueError, r'rank\(st_input\) must be 2', ragged.from_sparse, st3) + @test_util.run_deprecated_v1 + def testGoodPartialSparseTensorRank(self): + st1 = sparse_tensor.SparseTensor( + indices=[[0, 0]], + values=[0], + dense_shape=array_ops.placeholder(dtypes.int64)) + st2 = sparse_tensor.SparseTensor( + indices=array_ops.placeholder(dtypes.int64), + values=[0], + dense_shape=[4, 3]) + + # Shouldn't throw ValueError + ragged.from_sparse(st1) + ragged.from_sparse(st2) + + @test_util.run_deprecated_v1 def testNonRaggedSparseTensor(self): # "index_suffix" means the value of the innermost dimension of the index # (i.e., indices[i][-1]). diff --git a/tensorflow/python/ops/ragged/ragged_from_tensor_op_test.py b/tensorflow/python/ops/ragged/ragged_from_tensor_op_test.py index eb237f4c956..1d8a00cc18d 100644 --- a/tensorflow/python/ops/ragged/ragged_from_tensor_op_test.py +++ b/tensorflow/python/ops/ragged/ragged_from_tensor_op_test.py @@ -31,6 +31,7 @@ from tensorflow.python.platform import googletest class RaggedFromTensorOpTest(test_util.TensorFlowTestCase, parameterized.TestCase): + @test_util.run_deprecated_v1 def testDocStringExamples(self): # The examples from ragged.from_tensor.__doc__. dt = constant_op.constant([[5, 7, 0], [0, 3, 0], [6, 0, 0]]) @@ -262,6 +263,7 @@ class RaggedFromTensorOpTest(test_util.TensorFlowTestCase, [[[5, 6], [7]], [[0, 8], []]]] }, ) # pyformat: disable + @test_util.run_deprecated_v1 def testRaggedFromTensor(self, tensor, expected, @@ -278,6 +280,7 @@ class RaggedFromTensorOpTest(test_util.TensorFlowTestCase, with self.test_session(): self.assertEqual(rt.eval().tolist(), expected) + @test_util.run_deprecated_v1 def testHighDimensions(self): # Use distinct prime numbers for all dimension shapes in this test, so # we can see any errors that are caused by mixing up dimension sizes. @@ -291,7 +294,7 @@ class RaggedFromTensorOpTest(test_util.TensorFlowTestCase, dt.shape.is_compatible_with(rt.shape), '%s is incompatible with %s' % (dt.shape, rt.shape)) with self.test_session(): - self.assertEqual(rt.eval().tolist(), dt.eval().tolist()) + self.assertEqual(rt.eval().tolist(), self.evaluate(dt).tolist()) @parameterized.parameters( # With no padding or lengths @@ -395,6 +398,7 @@ class RaggedFromTensorOpTest(test_util.TensorFlowTestCase, 'expected': [[], []] }, ) + @test_util.run_deprecated_v1 def testEmpty(self, dt_shape, expected, lengths=None, padding=None): dt = array_ops.zeros(dt_shape) rt = ragged.from_tensor(dt, lengths, padding) @@ -447,6 +451,7 @@ class RaggedFromTensorOpTest(test_util.TensorFlowTestCase, 'error': (ValueError, r'ragged_rank must be greater than 0; got -1') }, ) + @test_util.run_deprecated_v1 def testErrors(self, tensor, lengths=None, diff --git a/tensorflow/python/ops/ragged/ragged_gather_nd_op_test.py b/tensorflow/python/ops/ragged/ragged_gather_nd_op_test.py index dcf1feaa696..62c6819374a 100644 --- a/tensorflow/python/ops/ragged/ragged_gather_nd_op_test.py +++ b/tensorflow/python/ops/ragged/ragged_gather_nd_op_test.py @@ -183,6 +183,7 @@ class RaggedGatherNdOpTest(test_util.TensorFlowTestCase, indices=[[0, 0, 1], [0, 0, 0], [0, 1, 0]], expected=[[b'c', b'd'], [b'a', b'b'], [b'e', b'f']]), ]) # pyformat: disable + @test_util.run_deprecated_v1 def testRaggedGatherNd(self, descr, params, indices, expected): result = ragged.gather_nd(params, indices) self.assertEqual( @@ -190,8 +191,9 @@ class RaggedGatherNdOpTest(test_util.TensorFlowTestCase, with self.test_session() as sess: if hasattr(expected, 'tolist'): expected = expected.tolist() - self.assertEqual(sess.run(result).tolist(), expected) + self.assertEqual(self.evaluate(result).tolist(), expected) + @test_util.run_deprecated_v1 def testRaggedGatherNdUnknownRankError(self): params = ragged.constant([['a', 'b'], ['c', 'd']]) indices1 = array_ops.placeholder(dtypes.int32, shape=None) @@ -219,6 +221,7 @@ class RaggedGatherNdOpTest(test_util.TensorFlowTestCase, indices=ragged.constant([[0]]), message='The innermost dimension of indices may not be ragged'), ]) + @test_util.run_deprecated_v1 def testRaggedGatherNdStaticError(self, params, indices, diff --git a/tensorflow/python/ops/ragged/ragged_gather_op_test.py b/tensorflow/python/ops/ragged/ragged_gather_op_test.py index bb52d05c32e..76c90cdfeeb 100644 --- a/tensorflow/python/ops/ragged/ragged_gather_op_test.py +++ b/tensorflow/python/ops/ragged/ragged_gather_op_test.py @@ -30,6 +30,7 @@ from tensorflow.python.platform import googletest class RaggedTensorOpsTest(test_util.TensorFlowTestCase): + @test_util.run_deprecated_v1 def testDocStringExamples(self): params = constant_op.constant(['a', 'b', 'c', 'd', 'e']) indices = constant_op.constant([3, 1, 2, 1, 0]) @@ -46,6 +47,7 @@ class RaggedTensorOpsTest(test_util.TensorFlowTestCase): ragged.gather(ragged_params, ragged_indices).eval().tolist(), [[[b'e'], [b'd'], []], [[b'd']], [], [[b'a', b'b', b'c']]]) + @test_util.run_deprecated_v1 def testTensorParamsAndTensorIndices(self): params = ['a', 'b', 'c', 'd', 'e'] indices = [2, 0, 2, 1] @@ -55,6 +57,7 @@ class RaggedTensorOpsTest(test_util.TensorFlowTestCase): [b'c', b'a', b'c', b'b']) self.assertEqual(type(ragged.gather(params, indices)), ops.Tensor) + @test_util.run_deprecated_v1 def testRaggedParamsAndTensorIndices(self): params = ragged.constant([['a', 'b'], ['c', 'd', 'e'], ['f'], [], ['g']]) indices = [2, 0, 2, 1] @@ -63,6 +66,7 @@ class RaggedTensorOpsTest(test_util.TensorFlowTestCase): ragged.gather(params, indices).eval().tolist(), [[b'f'], [b'a', b'b'], [b'f'], [b'c', b'd', b'e']]) + @test_util.run_deprecated_v1 def testTensorParamsAndRaggedIndices(self): params = ['a', 'b', 'c', 'd', 'e'] indices = ragged.constant([[2, 1], [1, 2, 0], [3]]) @@ -71,6 +75,7 @@ class RaggedTensorOpsTest(test_util.TensorFlowTestCase): ragged.gather(params, indices).eval().tolist(), [[b'c', b'b'], [b'b', b'c', b'a'], [b'd']]) + @test_util.run_deprecated_v1 def testRaggedParamsAndRaggedIndices(self): params = ragged.constant([['a', 'b'], ['c', 'd', 'e'], ['f'], [], ['g']]) indices = ragged.constant([[2, 1], [1, 2, 0], [3]]) @@ -82,6 +87,7 @@ class RaggedTensorOpsTest(test_util.TensorFlowTestCase): [[]]] # [p[3] ]] ) # pyformat: disable + @test_util.run_deprecated_v1 def testRaggedParamsAndScalarIndices(self): params = ragged.constant([['a', 'b'], ['c', 'd', 'e'], ['f'], [], ['g']]) indices = 1 @@ -89,6 +95,7 @@ class RaggedTensorOpsTest(test_util.TensorFlowTestCase): self.assertEqual( ragged.gather(params, indices).eval().tolist(), [b'c', b'd', b'e']) + @test_util.run_deprecated_v1 def test3DRaggedParamsAnd2DTensorIndices(self): params = ragged.constant([[['a', 'b'], []], [['c', 'd'], ['e'], ['f']], [['g']]]) @@ -101,6 +108,7 @@ class RaggedTensorOpsTest(test_util.TensorFlowTestCase): [[[b'g']], [[b'g']]]] # [p2, p2]] ) # pyformat: disable + @test_util.run_deprecated_v1 def testTensorParamsAnd4DRaggedIndices(self): indices = ragged.constant( [[[[3, 4], [0, 6]], []], [[[2, 1], [1, 0]], [[2, 5]], [[2, 3]]], @@ -115,6 +123,7 @@ class RaggedTensorOpsTest(test_util.TensorFlowTestCase): [[[b'c', b'b'], [b'b', b'a']], [[b'c', b'f']], [[b'c', b'd']]], [[[b'b', b'a']]]]) # pyformat: disable + @test_util.run_deprecated_v1 def testOutOfBoundsError(self): tensor_params = ['a', 'b', 'c'] tensor_indices = [0, 1, 2] @@ -131,6 +140,7 @@ class RaggedTensorOpsTest(test_util.TensorFlowTestCase): r'indices\[1\] = 3 is not in \[0, 2\)', ragged.gather(ragged_params, ragged_indices).eval) + @test_util.run_deprecated_v1 def testUnknownIndicesRankError(self): params = ragged.constant([], ragged_rank=1) indices = constant_op.constant([0], dtype=dtypes.int64) diff --git a/tensorflow/python/ops/ragged/ragged_map_fn_op_test.py b/tensorflow/python/ops/ragged/ragged_map_fn_op_test.py index 6f3f33b4441..7a8603c949a 100644 --- a/tensorflow/python/ops/ragged/ragged_map_fn_op_test.py +++ b/tensorflow/python/ops/ragged/ragged_map_fn_op_test.py @@ -140,6 +140,7 @@ class RaggedMapOpTest(test_util.TensorFlowTestCase, parameterized.TestCase): ), ]) + @test_util.run_deprecated_v1 def testRaggedMap( self, fn, @@ -161,9 +162,10 @@ class RaggedMapOpTest(test_util.TensorFlowTestCase, parameterized.TestCase): with self.test_session(): if ragged.is_ragged(expected_output): self.assertEqual(output.ragged_rank, expected_rt.ragged_rank) - output_values = output.eval() + output_values = self.evaluate(output) self.assertAllEqual(expected_output, output_values.tolist()) + @test_util.run_deprecated_v1 def testRaggedMapOnStructure(self): batman = ragged.constant([[1, 2, 3], [4], [5, 6, 7]]) # [[10, 20, 30], [40], [50, 60, 70]] @@ -184,6 +186,7 @@ class RaggedMapOpTest(test_util.TensorFlowTestCase, parameterized.TestCase): self.assertAllEqual(output.eval().tolist(), [66, 44, 198]) # Test mapping over a dict of RTs can produce a dict of RTs. + @test_util.run_deprecated_v1 def testRaggedMapOnStructure_RaggedOutputs(self): batman = ragged.constant([[1, 2, 3], [4], [5, 6, 7]]) # [[10, 20, 30], [40], [50, 60, 70]] @@ -215,6 +218,7 @@ class RaggedMapOpTest(test_util.TensorFlowTestCase, parameterized.TestCase): self.assertAllEqual(output['robin'].eval().tolist(), [[11, 21, 31], [41], [51, 61, 71]]) + @test_util.run_deprecated_v1 def testZip(self): x = ragged.constant([[10, 20], [30, 40], [50, 60], [70], [80, 90, 100]], dtypes.int64) @@ -232,11 +236,12 @@ class RaggedMapOpTest(test_util.TensorFlowTestCase, parameterized.TestCase): infer_shape=False) with self.test_session(): - result = output.eval().tolist() + result = self.evaluate(output).tolist() self.assertAllEqual( result, [[[0, 10], [0, 20]], [[1, 30], [1, 40]], [[2, 50], [2, 60]], [[3, 70]], [[4, 80], [4, 90], [4, 100]]]) + @test_util.run_deprecated_v1 def testBatchGather(self): tokens = ragged.constant([['hello', '.', 'there'], ['merhaba'], ['bonjour', '.', 'ca va', '?']]) @@ -255,7 +260,7 @@ class RaggedMapOpTest(test_util.TensorFlowTestCase, parameterized.TestCase): with self.test_session(): self.assertAllEqual( - out.eval().tolist(), + self.evaluate(out).tolist(), [[b'hello', b'there'], [b'merhaba'], [b'bonjour', b'ca va']]) def testMismatchRaggedRank(self): diff --git a/tensorflow/python/ops/ragged/ragged_map_inner_values_op_test.py b/tensorflow/python/ops/ragged/ragged_map_inner_values_op_test.py index 798d7c3ce81..b5802cb82d9 100644 --- a/tensorflow/python/ops/ragged/ragged_map_inner_values_op_test.py +++ b/tensorflow/python/ops/ragged/ragged_map_inner_values_op_test.py @@ -43,6 +43,7 @@ class RaggedMapInnerValuesOpTest(test_util.TensorFlowTestCase, with self.test_session(): self.assertEqual(result.eval().tolist(), expected) + @test_util.run_deprecated_v1 def testDocStringExamples(self): """Test the examples in apply_op_to_ragged_values.__doc__.""" rt = ragged.constant([[1, 2, 3], [], [4, 5], [6]]) @@ -54,6 +55,7 @@ class RaggedMapInnerValuesOpTest(test_util.TensorFlowTestCase, self.assertEqual(v2.eval().tolist(), [[1, 4, 9], [], [16, 25], [36]]) self.assertEqual(v3.eval().tolist(), [[6, 7, 8], [], [9, 10], [11]]) + @test_util.run_deprecated_v1 def testOpWithSingleRaggedTensorArg(self): tensor = ragged.constant([[1, 2, 3], [], [4, 5]]) self.assertRaggedMapInnerValuesReturns( @@ -61,17 +63,20 @@ class RaggedMapInnerValuesOpTest(test_util.TensorFlowTestCase, args=(tensor,), expected=[[0, 0, 0], [], [0, 0]]) + @test_util.run_deprecated_v1 def testOpWithTwoRaggedTensorArgs(self): x = ragged.constant([[3, 1, 4], [], [1, 5]]) y = ragged.constant([[1, 2, 3], [], [4, 5]]) self.assertRaggedMapInnerValuesReturns( op=math_ops.multiply, args=(x, y), expected=[[3, 2, 12], [], [4, 25]]) + @test_util.run_deprecated_v1 def testOpWithRaggedTensorAndScalarArgs(self): y = ragged.constant([[1, 2, 3], [], [4, 5]]) self.assertRaggedMapInnerValuesReturns( op=math_ops.multiply, args=(5, y), expected=[[5, 10, 15], [], [20, 25]]) + @test_util.run_deprecated_v1 def testOpWithThreeRaggedTensorArgs(self): condition = ragged.constant( [[True, True, False], [], [True, False]]) # pyformat: disable @@ -82,6 +87,7 @@ class RaggedMapInnerValuesOpTest(test_util.TensorFlowTestCase, args=(condition, x, y), expected=[[b'a', b'b', b'C'], [], [b'd', b'E']]) + @test_util.run_deprecated_v1 def testOpWithRaggedTensorListArg(self): x = ragged.constant([[1, 2, 3], [], [4, 5]]) y = ragged.constant([[10, 20, 30], [], [40, 50]]) @@ -90,6 +96,7 @@ class RaggedMapInnerValuesOpTest(test_util.TensorFlowTestCase, args=([x, y, x],), expected=[[12, 24, 36], [], [48, 60]]) + @test_util.run_deprecated_v1 def testOpWithKeywordArgs(self): x = ragged.constant([[3, 1, 4], [], [1, 5]]) y = ragged.constant([[1, 2, 3], [], [4, 5]]) @@ -98,6 +105,7 @@ class RaggedMapInnerValuesOpTest(test_util.TensorFlowTestCase, kwargs=dict(x=x, y=y), expected=[[3, 2, 12], [], [4, 25]]) + @test_util.run_deprecated_v1 def testOpWithMixedPositionalAndKeywordArgs(self): x = ragged.constant([[3, 1, 4], [], [1, 5]]) y = ragged.constant([[1, 2, 3], [], [4, 5]]) @@ -107,6 +115,7 @@ class RaggedMapInnerValuesOpTest(test_util.TensorFlowTestCase, kwargs=dict(y=y), expected=[[3, 2, 12], [], [4, 25]]) + @test_util.run_deprecated_v1 def testNonElementWiseOp(self): x = ragged.constant( [[[3, 1, 4], [1, 5, 9], [2, 6, 5]], [], [[3, 5, 8], [9, 7, 9]]], @@ -119,6 +128,7 @@ class RaggedMapInnerValuesOpTest(test_util.TensorFlowTestCase, }, expected=[[8, 15, 13], [], [16, 25]]) + @test_util.run_deprecated_v1 def testOpWithRaggedRankGreaterThanOne(self): # ragged_rank=0 x0 = [3, 1, 4, 1, 5, 9, 2, 6, 5] @@ -163,6 +173,7 @@ class RaggedMapInnerValuesOpTest(test_util.TensorFlowTestCase, [[[54, 14], [48, 45]]] # row 3 ]) # pyformat: disable + @test_util.run_deprecated_v1 def testOpWithRaggedRankThree(self): x = ragged.constant([[[3, 1, 4]], [], [[], [1, 5]]]) y = ragged.constant([[[1, 2, 3]], [], [[], [4, 5]]]) @@ -171,6 +182,7 @@ class RaggedMapInnerValuesOpTest(test_util.TensorFlowTestCase, args=(x, y), expected=[[[3, 2, 12]], [], [[], [4, 25]]]) + @test_util.run_deprecated_v1 def testOpWithInnerValuesOnly(self): x = constant_op.constant([[1, 2], [3, 4], [5, 6]]) y = constant_op.constant(2) @@ -191,6 +203,7 @@ class RaggedMapInnerValuesOpTest(test_util.TensorFlowTestCase, r'Inputs must have identical ragged splits.*', ragged.map_inner_values, math_ops.add, x, y) + @test_util.run_deprecated_v1 def testRaggedTensorSplitsMismatchErrorAtRuntime(self): splits1 = array_ops.placeholder_with_default( constant_op.constant([0, 3, 3, 5], dtypes.int64), None) diff --git a/tensorflow/python/ops/ragged/ragged_operators_test.py b/tensorflow/python/ops/ragged/ragged_operators_test.py index a99d788ef79..7fe8159d822 100644 --- a/tensorflow/python/ops/ragged/ragged_operators_test.py +++ b/tensorflow/python/ops/ragged/ragged_operators_test.py @@ -27,6 +27,7 @@ class RaggedElementwiseOpsTest(test_util.TensorFlowTestCase): # @TODO(edloper): Test right-handed versions of operators once we add # broadcasting support for elementwise ops. + @test_util.run_deprecated_v1 def testOrderingOperators(self): x = ragged.constant([[1, 5], [3]]) y = ragged.constant([[4, 5], [1]]) @@ -40,6 +41,7 @@ class RaggedElementwiseOpsTest(test_util.TensorFlowTestCase): if a != b: print('%30s %s' % (b, a)) + @test_util.run_deprecated_v1 def testArithmeticOperators(self): x = ragged.constant([[1.0, -2.0], [8.0]]) y = ragged.constant([[4.0, 4.0], [2.0]]) @@ -75,6 +77,7 @@ class RaggedElementwiseOpsTest(test_util.TensorFlowTestCase): self.assertEqual((2.0 % y).eval().tolist(), [[2.0, 2.0], [0.0]]) self.assertEqual((x % 2.0).eval().tolist(), [[1.0, 0.0], [0.0]]) + @test_util.run_deprecated_v1 def testLogicalOperators(self): a = ragged.constant([[True, True], [False]]) b = ragged.constant([[True, False], [False]]) diff --git a/tensorflow/python/ops/ragged/ragged_range_op_test.py b/tensorflow/python/ops/ragged/ragged_range_op_test.py index 3c6a6fb75c8..644423ecb7f 100644 --- a/tensorflow/python/ops/ragged/ragged_range_op_test.py +++ b/tensorflow/python/ops/ragged/ragged_range_op_test.py @@ -26,6 +26,7 @@ from tensorflow.python.platform import googletest class RaggedRangeOpTest(test_util.TensorFlowTestCase): + @test_util.run_deprecated_v1 def testDocStringExamples(self): """Examples from ragged_range.__doc__.""" with self.test_session(): @@ -38,6 +39,7 @@ class RaggedRangeOpTest(test_util.TensorFlowTestCase): rt3 = ragged.range([0, 5, 8], [3, 3, 12], 2).eval().tolist() self.assertEqual(rt3, [[0, 2], [], [8, 10]]) + @test_util.run_deprecated_v1 def testBasicRanges(self): with self.test_session(): # Specify limits only. @@ -56,6 +58,7 @@ class RaggedRangeOpTest(test_util.TensorFlowTestCase): [list(range(0, 4, 2)), list(range(3, 4, 3)), list(range(5, 15, 4))]) + @test_util.run_deprecated_v1 def testFloatRanges(self): with self.test_session(): expected = [[0.0, 0.4, 0.8, 1.2, 1.6, 2.0, 2.4, 2.8, 3.2, 3.6], [3.0], @@ -64,6 +67,7 @@ class RaggedRangeOpTest(test_util.TensorFlowTestCase): [0.4, 1.5, 2.2]).eval().tolist() self.assertEqual(expected, [[round(v, 5) for v in row] for row in actual]) + @test_util.run_deprecated_v1 def testNegativeDeltas(self): with self.test_session(): self.assertEqual( @@ -77,6 +81,7 @@ class RaggedRangeOpTest(test_util.TensorFlowTestCase): [list(range(0, 0, -1)), list(range(-3, 0, 1)), list(range(5, 0, -2))]) + @test_util.run_deprecated_v1 def testBroadcast(self): with self.test_session(): # Specify starts and limits, broadcast deltas. @@ -89,6 +94,7 @@ class RaggedRangeOpTest(test_util.TensorFlowTestCase): self.assertEqual( ragged.range(0, 5, 1).eval().tolist(), [list(range(0, 5, 1))]) + @test_util.run_deprecated_v1 def testEmptyRanges(self): rt1 = ragged.range([0, 5, 3], [0, 3, 5]) rt2 = ragged.range([0, 5, 5], [0, 3, 5], -1) @@ -96,6 +102,7 @@ class RaggedRangeOpTest(test_util.TensorFlowTestCase): self.assertEqual(rt1.eval().tolist(), [[], [], [3, 4]]) self.assertEqual(rt2.eval().tolist(), [[], [5, 4], []]) + @test_util.run_deprecated_v1 def testShapeFnErrors(self): with self.test_session(): self.assertRaisesRegexp(ValueError, r'Shape must be at most rank 1.*', @@ -107,12 +114,14 @@ class RaggedRangeOpTest(test_util.TensorFlowTestCase): self.assertRaisesRegexp(ValueError, r'Dimensions must be equal.*', ragged.range, [0], [1, 2]) + @test_util.run_deprecated_v1 def testKernelErrors(self): with self.test_session(): self.assertRaisesRegexp(errors.InvalidArgumentError, r'Requires delta != 0', ragged.range(0, 0, 0).eval) + @test_util.run_deprecated_v1 def testShape(self): self.assertEqual(ragged.range(0, 0, 0).shape.as_list(), [1, None]) self.assertEqual(ragged.range([1, 2, 3]).shape.as_list(), [3, None]) diff --git a/tensorflow/python/ops/ragged/ragged_reduce_op_test.py b/tensorflow/python/ops/ragged/ragged_reduce_op_test.py index 93176c738df..9f51d59ba3c 100644 --- a/tensorflow/python/ops/ragged/ragged_reduce_op_test.py +++ b/tensorflow/python/ops/ragged/ragged_reduce_op_test.py @@ -300,6 +300,7 @@ class RaggedReduceOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase): axis=2, expected=[[mean(1, 2), mean(3, 4, 5)], [mean(6, 7), 8], [9]]), ) + @test_util.run_deprecated_v1 def testReduce(self, ragged_reduce_op, rt_input, axis, expected): rt_input = ragged.constant(rt_input) reduced = ragged_reduce_op(rt_input, axis) @@ -311,6 +312,7 @@ class RaggedReduceOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase): self.assertTrue( ((actual == expected) | (np.isnan(actual) & np.isnan(expected))).all()) + @test_util.run_deprecated_v1 def testMeanNan(self): rt_as_list = [[0, 1, 2, 3], [4], [], [5, 6], [7], [8, 9]] expected = ( @@ -321,6 +323,7 @@ class RaggedReduceOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase): with self.test_session(): self.assertEqualWithNan(reduced.eval(), expected) + @test_util.run_deprecated_v1 def testMeanWithTensorInputs(self): tensor = [[1.0, 2.0, 3.0], [10.0, 20.0, 30.0]] expected = [2.0, 20.0] @@ -328,6 +331,7 @@ class RaggedReduceOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase): with self.test_session(): self.assertAllEqual(reduced.eval(), expected) + @test_util.run_deprecated_v1 def testErrors(self): rt_input = ragged.constant([[1, 2, 3], [4, 5]]) axis = array_ops.placeholder_with_default(constant_op.constant([0]), None) diff --git a/tensorflow/python/ops/ragged/ragged_row_lengths_op_test.py b/tensorflow/python/ops/ragged/ragged_row_lengths_op_test.py index 4d5a0a5d11c..4a705be4848 100644 --- a/tensorflow/python/ops/ragged/ragged_row_lengths_op_test.py +++ b/tensorflow/python/ops/ragged/ragged_row_lengths_op_test.py @@ -143,6 +143,7 @@ class RaggedRowLengthsOp(test_util.TensorFlowTestCase, parameterized.TestCase): expected=[[2, 3, 0], [4, 1]], expected_ragged_rank=1), ]) # pyformat: disable + @test_util.run_deprecated_v1 def testRowLengths(self, rt_input, expected, diff --git a/tensorflow/python/ops/ragged/ragged_row_splits_to_segment_ids_op_test.py b/tensorflow/python/ops/ragged/ragged_row_splits_to_segment_ids_op_test.py index f246bf35524..7f5f4e91bde 100644 --- a/tensorflow/python/ops/ragged/ragged_row_splits_to_segment_ids_op_test.py +++ b/tensorflow/python/ops/ragged/ragged_row_splits_to_segment_ids_op_test.py @@ -26,6 +26,7 @@ from tensorflow.python.platform import googletest class RaggedSplitsToSegmentIdsOpTest(test_util.TensorFlowTestCase): + @test_util.run_deprecated_v1 def testDocStringExample(self): splits = [0, 3, 3, 5, 6, 9] expected = [0, 0, 0, 2, 2, 3, 4, 4, 4] @@ -33,12 +34,14 @@ class RaggedSplitsToSegmentIdsOpTest(test_util.TensorFlowTestCase): with self.test_session(): self.assertEqual(segment_ids.eval().tolist(), expected) + @test_util.run_deprecated_v1 def testEmptySplits(self): # Note: the splits for an empty ragged tensor contains a single zero. segment_ids = ragged.row_splits_to_segment_ids([0]) with self.test_session(): self.assertEqual(segment_ids.eval().tolist(), []) + @test_util.run_deprecated_v1 def testErrors(self): self.assertRaisesRegexp(ValueError, r'Invalid row_splits: \[\]', ragged.row_splits_to_segment_ids, []) diff --git a/tensorflow/python/ops/ragged/ragged_segment_ids_to_row_splits_op_test.py b/tensorflow/python/ops/ragged/ragged_segment_ids_to_row_splits_op_test.py index fa7adf66b0b..7e52f2d844b 100644 --- a/tensorflow/python/ops/ragged/ragged_segment_ids_to_row_splits_op_test.py +++ b/tensorflow/python/ops/ragged/ragged_segment_ids_to_row_splits_op_test.py @@ -26,6 +26,7 @@ from tensorflow.python.platform import googletest class RaggedSplitsToSegmentIdsOpTest(test_util.TensorFlowTestCase): + @test_util.run_deprecated_v1 def testDocStringExample(self): segment_ids = [0, 0, 0, 2, 2, 3, 4, 4, 4] expected = [0, 3, 3, 5, 6, 9] @@ -33,6 +34,7 @@ class RaggedSplitsToSegmentIdsOpTest(test_util.TensorFlowTestCase): with self.test_session(): self.assertEqual(splits.eval().tolist(), expected) + @test_util.run_deprecated_v1 def testEmptySegmentIds(self): # Note: the splits for an empty ragged tensor contains a single zero. segment_ids = ragged.segment_ids_to_row_splits([]) @@ -49,6 +51,7 @@ class RaggedSplitsToSegmentIdsOpTest(test_util.TensorFlowTestCase): self.assertRaisesRegexp(ValueError, r'Shape \(1, 1\) must have rank 1', ragged.segment_ids_to_row_splits, [[0]]) + @test_util.run_deprecated_v1 def testNumSegments(self): segment_ids = [0, 0, 0, 2, 2, 3, 4, 4, 4] num_segments = 7 @@ -57,6 +60,7 @@ class RaggedSplitsToSegmentIdsOpTest(test_util.TensorFlowTestCase): with self.test_session(): self.assertEqual(splits.eval().tolist(), expected) + @test_util.run_deprecated_v1 def testUnsortedSegmentIds(self): # Segment ids are not required to be sorted. segment_ids = [0, 4, 3, 2, 4, 4, 2, 0, 0] diff --git a/tensorflow/python/ops/ragged/ragged_segment_op_test.py b/tensorflow/python/ops/ragged/ragged_segment_op_test.py index 7d41eb7f753..9e4877ae3e6 100644 --- a/tensorflow/python/ops/ragged/ragged_segment_op_test.py +++ b/tensorflow/python/ops/ragged/ragged_segment_op_test.py @@ -110,6 +110,7 @@ class RaggedSegmentOpsTest(test_util.TensorFlowTestCase, (ragged.segment_mean, mean, [5, 4, 3, 2, 1, 0]), (ragged.segment_mean, mean, [0, 0, 0, 10, 10, 10]), ) + @test_util.run_deprecated_v1 def testRaggedSegment_Int(self, segment_op, combiner, segment_ids): rt_as_list = [[0, 1, 2, 3], [4], [], [5, 6], [7], [8, 9]] rt = ragged.constant(rt_as_list) @@ -118,8 +119,7 @@ class RaggedSegmentOpsTest(test_util.TensorFlowTestCase, combiner) segmented = segment_op(rt, segment_ids, num_segments) - with self.test_session(): - self.assertListEqual(segmented.eval().tolist(), expected) + self.assertListEqual(self.evaluate(segmented).tolist(), expected) @parameterized.parameters( (ragged.segment_sum, sum, [0, 0, 1, 1, 2, 2]), @@ -147,6 +147,7 @@ class RaggedSegmentOpsTest(test_util.TensorFlowTestCase, (ragged.segment_sqrt_n, sqrt_n, [5, 4, 3, 2, 1, 0]), (ragged.segment_sqrt_n, sqrt_n, [0, 0, 0, 10, 10, 10]), ) + @test_util.run_deprecated_v1 def testRaggedSegment_Float(self, segment_op, combiner, segment_ids): rt_as_list = [[0., 1., 2., 3.], [4.], [], [5., 6.], [7.], [8., 9.]] rt = ragged.constant(rt_as_list) @@ -155,10 +156,10 @@ class RaggedSegmentOpsTest(test_util.TensorFlowTestCase, combiner) segmented = segment_op(rt, segment_ids, num_segments) - with self.test_session(): - self.assertNestedListAmostEqual( - segmented.eval().tolist(), expected, places=5) + self.assertNestedListAmostEqual( + self.evaluate(segmented).tolist(), expected, places=5) + @test_util.run_deprecated_v1 def testRaggedRankTwo(self): rt = ragged.constant([ [[111, 112, 113, 114], [121],], # row 0 @@ -172,17 +173,16 @@ class RaggedSegmentOpsTest(test_util.TensorFlowTestCase, [], # row 1 [[411, 412], [321, 322], [331]] # row 2 ] # pyformat: disable - with self.test_session(): - self.assertEqual(segmented1.eval().tolist(), expected1) + self.assertEqual(self.evaluate(segmented1).tolist(), expected1) segment_ids2 = [1, 2, 1, 1] segmented2 = ragged.segment_sum(rt, segment_ids2, 3) expected2 = [[], [[111+411, 112+412, 113, 114], [121+321, 322], [331]], []] # pyformat: disable - with self.test_session(): - self.assertEqual(segmented2.eval().tolist(), expected2) + self.assertEqual(self.evaluate(segmented2).tolist(), expected2) + @test_util.run_deprecated_v1 def testRaggedSegmentIds(self): rt = ragged.constant([ [[111, 112, 113, 114], [121],], # row 0 @@ -195,8 +195,7 @@ class RaggedSegmentOpsTest(test_util.TensorFlowTestCase, expected = [[], [111+321, 112+322, 113, 114], [121+331+411, 412]] # pyformat: disable - with self.test_session(): - self.assertEqual(segmented.eval().tolist(), expected) + self.assertEqual(self.evaluate(segmented).tolist(), expected) def testShapeMismatchError1(self): dt = constant_op.constant([1, 2, 3, 4, 5, 6]) @@ -206,6 +205,7 @@ class RaggedSegmentOpsTest(test_util.TensorFlowTestCase, 'but segment_ids is ragged and data is not.', ragged.segment_sum, dt, segment_ids, 3) + @test_util.run_deprecated_v1 def testShapeMismatchError2(self): rt = ragged.constant([ [[111, 112, 113, 114], [121]], # row 0 @@ -226,7 +226,7 @@ class RaggedSegmentOpsTest(test_util.TensorFlowTestCase, array_ops.placeholder_with_default(segment_ids.values, None), array_ops.placeholder_with_default(segment_ids.row_splits, None)) segmented2 = ragged.segment_sum(rt, segment_ids2, 3) - with self.test_session(): + with self.cached_session(): self.assertRaisesRegexp( errors.InvalidArgumentError, 'segment_ids.shape must be a prefix of data.shape.*', segmented2.eval) diff --git a/tensorflow/python/ops/ragged/ragged_stack_op_test.py b/tensorflow/python/ops/ragged/ragged_stack_op_test.py index d474a749f04..43434716942 100644 --- a/tensorflow/python/ops/ragged/ragged_stack_op_test.py +++ b/tensorflow/python/ops/ragged/ragged_stack_op_test.py @@ -265,6 +265,7 @@ class RaggedStackOpTest(test_util.TensorFlowTestCase, parameterized.TestCase): axis=0, expected=[[[b'a00', b'a01'], [], [b'a20', b'a21']]]), ) # pyformat: disable + @test_util.run_deprecated_v1 def testRaggedStack(self, descr, rt_inputs, @@ -313,6 +314,7 @@ class RaggedStackOpTest(test_util.TensorFlowTestCase, parameterized.TestCase): def testError(self, rt_inputs, axis, error, message): self.assertRaisesRegexp(error, message, ragged.stack, rt_inputs, axis) + @test_util.run_deprecated_v1 def testSingleTensorInput(self): """Tests ragged_stack with a single tensor input. diff --git a/tensorflow/python/ops/ragged/ragged_string_ops.py b/tensorflow/python/ops/ragged/ragged_string_ops.py new file mode 100644 index 00000000000..cdcdbdff07b --- /dev/null +++ b/tensorflow/python/ops/ragged/ragged_string_ops.py @@ -0,0 +1,119 @@ +# Copyright 2015 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Ragged operations for working with string Tensors.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from tensorflow.python.framework import dtypes +from tensorflow.python.framework import ops +from tensorflow.python.ops import array_ops +from tensorflow.python.ops import gen_string_ops +from tensorflow.python.ops.ragged import ragged_conversion_ops +from tensorflow.python.ops.ragged import ragged_factory_ops +from tensorflow.python.ops.ragged import ragged_tensor +from tensorflow.python.util.tf_export import tf_export + + +# pylint: disable=redefined-builtin +@tf_export("strings.unicode_encode") +def unicode_encode(input, output_encoding, errors="replace", + replacement_char=65533, name=None): + r"""Encodes each sequence of Unicode code points in `input` into a string. + + `result[i1...iN]` is the string formed by concatenating the Unicode + codepoints `input[1...iN, :]`, encoded using `output_encoding`. + + Args: + input: An `N+1` dimensional potentially ragged integer tensor with + shape `[D1...DN, num_chars]`. + output_encoding: Unicode encoding that should be used to encode each + codepoint sequence. Can be `"UTF-8"`, `"UTF-16-BE"`, or `"UTF-32-BE"`. + errors: Specifies the response when an invalid codepoint is encountered + (optional). One of: + * `'replace'`: Replace invalid codepoint with the + `replacement_char`. (default) + * `'ignore'`: Skip invalid codepoints. + * `'strict'`: Raise an exception for any invalid codepoint. + replacement_char: The replacement character codepoint to be used in place of + any invalid input when `errors='replace'`. Any valid unicode codepoint may + be used. The default value is the default unicode replacement character + which is 0xFFFD (U+65533). + name: A name for the operation (optional). + + Returns: + A `N` dimensional `string` tensor with shape `[D1...DN]`. + + #### Example: + ```python + >>> input = [[71, 246, 246, 100, 110, 105, 103, 104, 116], [128522]] + >>> unicode_encode(input, 'UTF8') + ['G\xc3\xb6\xc3\xb6dnight', '\xf0\x9f\x98\x8a'] + ``` + """ + with ops.name_scope(name, "UnicodeEncode", [input]): + input_tensor = ragged_factory_ops.convert_to_tensor_or_ragged_tensor(input) + if input_tensor.shape.ndims is None: + raise ValueError("Rank of input_tensor must be statically known.") + if ragged_tensor.is_ragged(input_tensor): + if input_tensor.inner_values.shape.ndims > 1: + # If the inner_values of our ragged tensor is multi-dimensional, we can + # process it separately and our output will have the same nested splits + # as our input. + return input_tensor.with_inner_values( + unicode_encode(input_tensor.inner_values, output_encoding, errors, + replacement_char)) + elif input_tensor.ragged_rank > 1: + # Recursively process the values of the ragged tensor. + return input_tensor.with_values( + unicode_encode(input_tensor.values, output_encoding, errors, + replacement_char)) + else: + # Our ragged tensor is of the correct shape (rank 1 inner_values tensor + # with ragged_rank of 1) so we can process it as normal. + return gen_string_ops.unicode_encode( + input_values=input_tensor.values, + input_splits=input_tensor.row_splits, + output_encoding=output_encoding, + errors=errors, + replacement_char=replacement_char) + else: + if input_tensor.shape.ndims == 2: + # The input tensor is of the correct 2-D shape, it's just not ragged. + return unicode_encode(ragged_conversion_ops.from_tensor(input_tensor), + output_encoding, errors, replacement_char) + elif input_tensor.shape.ndims > 2: + # We need to initially flatten the input tensor to 2-D, and then can + # reshape the output of our processed flattened tensor. + flat_input_tensor = array_ops.reshape( + input_tensor, + array_ops.stack([-1, array_ops.shape(input_tensor)[-1]])) + flat_output_tensor = unicode_encode(flat_input_tensor, output_encoding, + errors, replacement_char) + return array_ops.reshape(flat_output_tensor, input_tensor.shape[:-1]) + elif input_tensor.shape.ndims == 0: + raise ValueError("input_tensor's rank must be at least 1.") + else: + # Our input tensor is rank 1, so we create a ragged tensor with an added + # dimension to create the correct input shape & type, and then remove + # the additional dimension from the output and return the string scalar. + ragged_input_tensor = ragged_factory_ops.from_row_splits( + input_tensor, + array_ops.stack([0, array_ops.shape(input_tensor, + out_type=dtypes.int64)[0]])) + output_tensor = unicode_encode(ragged_input_tensor, output_encoding, + errors, replacement_char) + return array_ops.reshape(output_tensor, []) diff --git a/tensorflow/python/ops/ragged/ragged_tensor.py b/tensorflow/python/ops/ragged/ragged_tensor.py index abb27fc3c08..ddeabfb4649 100644 --- a/tensorflow/python/ops/ragged/ragged_tensor.py +++ b/tensorflow/python/ops/ragged/ragged_tensor.py @@ -64,7 +64,7 @@ class RaggedTensor(object): a 3-D `RaggedTensor` that stores the fixed-size word embedding for each word in a sentence, for each sentence in a batch, could be written as `[num_sentences, (num_words), embedding_size]`. The parentheses around - `(num_words)` indicate that that dimension is ragged, and that the length + `(num_words)` indicate that dimension is ragged, and that the length of each element list in that dimension may vary for each item. ### Component Tensors @@ -257,6 +257,7 @@ class RaggedTensor(object): raise TypeError("Row-partitioning argument must be a Tensor.") values.shape.with_rank_at_least(1) row_splits.shape.assert_has_rank(1) + row_splits.set_shape([None]) self._values = values self._row_splits = row_splits diff --git a/tensorflow/python/ops/ragged/ragged_tensor_bounding_shape_op_test.py b/tensorflow/python/ops/ragged/ragged_tensor_bounding_shape_op_test.py index a1c10aff9de..befe30f0e10 100644 --- a/tensorflow/python/ops/ragged/ragged_tensor_bounding_shape_op_test.py +++ b/tensorflow/python/ops/ragged/ragged_tensor_bounding_shape_op_test.py @@ -28,41 +28,39 @@ class RaggedTensorBoundingShapeOp(test_util.TensorFlowTestCase): def testDocStringExample(self): # This is the example from ragged.bounding_shape.__doc__. rt = ragged.constant([[1, 2, 3, 4], [5], [], [6, 7, 8, 9], [10]]) - with self.test_session(): - self.assertEqual(ragged.bounding_shape(rt).eval().tolist(), [5, 4]) + self.assertEqual(self.evaluate(ragged.bounding_shape(rt)).tolist(), [5, 4]) def test2DRaggedTensorWithOneRaggedDimension(self): values = ['a', 'b', 'c', 'd', 'e', 'f', 'g'] rt1 = ragged.from_row_splits(values, [0, 2, 5, 6, 6, 7]) rt2 = ragged.from_row_splits(values, [0, 7]) rt3 = ragged.from_row_splits(values, [0, 0, 7, 7]) - with self.test_session(): - self.assertEqual(ragged.bounding_shape(rt1).eval().tolist(), [5, 3]) - self.assertEqual(ragged.bounding_shape(rt2).eval().tolist(), [1, 7]) - self.assertEqual(ragged.bounding_shape(rt3).eval().tolist(), [3, 7]) + self.assertEqual(self.evaluate(ragged.bounding_shape(rt1)).tolist(), [5, 3]) + self.assertEqual(self.evaluate(ragged.bounding_shape(rt2)).tolist(), [1, 7]) + self.assertEqual(self.evaluate(ragged.bounding_shape(rt3)).tolist(), [3, 7]) def test3DRaggedTensorWithOneRaggedDimension(self): values = [[0, 1], [2, 3], [4, 5], [6, 7], [8, 9], [10, 11], [12, 13]] rt1 = ragged.from_row_splits(values, [0, 2, 5, 6, 6, 7]) rt2 = ragged.from_row_splits(values, [0, 7]) rt3 = ragged.from_row_splits(values, [0, 0, 7, 7]) - with self.test_session(): - self.assertEqual(ragged.bounding_shape(rt1).eval().tolist(), [5, 3, 2]) - self.assertEqual(ragged.bounding_shape(rt2).eval().tolist(), [1, 7, 2]) - self.assertEqual(ragged.bounding_shape(rt3).eval().tolist(), [3, 7, 2]) + self.assertEqual( + self.evaluate(ragged.bounding_shape(rt1)).tolist(), [5, 3, 2]) + self.assertEqual( + self.evaluate(ragged.bounding_shape(rt2)).tolist(), [1, 7, 2]) + self.assertEqual( + self.evaluate(ragged.bounding_shape(rt3)).tolist(), [3, 7, 2]) def testNonRaggedTensor(self): dt = [[0, 1, 2], [3, 4, 5], [6, 7, 8], [9, 10, 11]] - with self.test_session(): - self.assertEqual(ragged.bounding_shape(dt).eval().tolist(), [4, 3]) + self.assertEqual(self.evaluate(ragged.bounding_shape(dt)).tolist(), [4, 3]) def testExplicitAxisOptimizations(self): rt = ragged.from_row_splits(b'a b c d e f g'.split(), [0, 2, 5, 6, 6, 7]) - with self.test_session(): - self.assertEqual(ragged.bounding_shape(rt, 0).eval().tolist(), 5) - self.assertEqual(ragged.bounding_shape(rt, 1).eval().tolist(), 3) - self.assertEqual( - ragged.bounding_shape(rt, [1, 0]).eval().tolist(), [3, 5]) + self.assertEqual(self.evaluate(ragged.bounding_shape(rt, 0)).tolist(), 5) + self.assertEqual(self.evaluate(ragged.bounding_shape(rt, 1)).tolist(), 3) + self.assertEqual( + self.evaluate(ragged.bounding_shape(rt, [1, 0])).tolist(), [3, 5]) if __name__ == '__main__': diff --git a/tensorflow/python/ops/ragged/ragged_tensor_shape.py b/tensorflow/python/ops/ragged/ragged_tensor_shape.py new file mode 100644 index 00000000000..9129b4b10b4 --- /dev/null +++ b/tensorflow/python/ops/ragged/ragged_tensor_shape.py @@ -0,0 +1,570 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Shapes & broadcasting for RaggedTensors.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from tensorflow.python.framework import constant_op +from tensorflow.python.framework import dtypes +from tensorflow.python.framework import ops +from tensorflow.python.framework import tensor_util +from tensorflow.python.ops import array_ops +from tensorflow.python.ops import control_flow_ops +from tensorflow.python.ops import math_ops +from tensorflow.python.ops.ragged import ragged_array_ops +from tensorflow.python.ops.ragged import ragged_conversion_ops +from tensorflow.python.ops.ragged import ragged_factory_ops +from tensorflow.python.ops.ragged import ragged_tensor +from tensorflow.python.ops.ragged import ragged_util + + +class RaggedTensorDynamicShape(object): + """A collection of tensors encoding the shape of a potentially ragged tensor. + + Each `RaggedTensorDynamicShape` consists of an ordered list of dimension + sizes. There are two dimension types: + + * "Uniform dimensions" are dimenisons where all slices have the same + length. `RaggedTensorDynamicShape` records the size of each uniform + dimension using a single scalar integer. + + * "Ragged dimensions" are dimensions whose slices may have different + lengths. `RaggedTensorDynamicShape` records the size of each ragged + dimension using an integer vector containing the slice lengths for all + the slices across that dimension. + + Furthermore, there are two ways a dimension might be encoded: + + * "Partitioned dimensions" are dimensions that are encoded using a + `RaggedTensor`'s `nested_row_splits`. The outermostmost partitioned + dimension must be uniform, and the innermost partitioned dimension must + be ragged. + + * "Inner dimensions" are dimensions that are encoded using a + `RaggedTensor`'s `inner_values`. Inner dimensions are always uniform. + + The sizes of partitioned dimensions are recorded using `partitioned_dim_sizes` + and `inner_dim_sizes`: + + * `paritioned_dim_sizes` is a list of tensors (one for each partitioned + dimension). + + * For uniform dimensions, the tensor is an integer scalar specifying the + size of all slices across that dimension. + * For ragged dimensions, the tensor is an integer vector specifying the + size of each slice across that dimension. + + * `inner_dim_sizes` is a single integer vector, where each element + specifies the size of a single inner dimension. + + Examples: + + Tensor | Ragged | Partitioned Dim Sizes | Inner Dim + : Rank : : Sizes + ------------------------------ | ------ | ---------------------- | ---------- + `[[1, 2, 3], [4, 5, 6]]` | 0 | | `2, 3` + `[[1, 2], [], [3, 4, 5]]` | 1 | `3, (2, 0, 3)` | + `[[[1, 2], [3, 4]], [[5, 6]]]` | 1 | `2, (2, 1)` | 2 + `[[[1, 2], [3]], [[4, 5]]]` | 2 | `2, (2, 1), (2, 1, 2)` | + """ + + def __init__(self, partitioned_dim_sizes, inner_dim_sizes): + """Creates a RaggedTensorDynamicShape. + + Args: + partitioned_dim_sizes: A `list` of 0-D or 1-D integer `Tensor`, one for + each partitioned dimension. If dimension `d` is uniform, then + `partitioned_dim_sizes[d]` must be an integer scalar, specifying the + size of all slices across dimension `d`. If dimension `d` is ragged, + then `partitioned_dim_sizes[d]` must be an integer vector, specifying + the size of each slice across dimension `d`. + inner_dim_sizes: A 1-D integer `Tensor`, whose length is equal to the + number of inner dimensions. `inner_dim_sizes[n]` is the size of all + slices across the `n`th inner dimension (which is the + `(len(partitioned_dim_sizes)+n)`th dimension in the overall tensor. + """ + assert isinstance(partitioned_dim_sizes, (list, tuple)) + with ops.name_scope(None, 'RaggedTensorDynamicShape', + (partitioned_dim_sizes, inner_dim_sizes)): + partitioned_dim_sizes = tuple( + ragged_util.convert_to_int_tensor( + size, dtype=dtypes.int64, name='partitioned_dimension_size') + for size in partitioned_dim_sizes) + inner_dim_sizes = ragged_util.convert_to_int_tensor( + inner_dim_sizes, dtype=dtypes.int64, name='inner_dim_sizes') + + # Validate shapes. + if partitioned_dim_sizes: + for axis, dimension_size in enumerate(partitioned_dim_sizes): + if dimension_size.shape.ndims is None: + raise ValueError( + 'rank of partitioned_dim_sizes[%d] is unknown' % axis) + dimension_size.shape.with_rank_at_most(1) + if partitioned_dim_sizes[0].shape.ndims == 1: + raise ValueError('outermost partitioned dimension must be uniform') + if partitioned_dim_sizes[-1].shape.ndims == 0: + raise ValueError('innermost partitioned dimension must be ragged') + inner_dim_sizes.shape.assert_has_rank(1) + + self._partitioned_dim_sizes = partitioned_dim_sizes + self._inner_dim_sizes = inner_dim_sizes + + def __repr__(self): + return ('RaggedTensorDynamicShape' + '(partitioned_dim_sizes=%r, inner_dim_sizes=%r)' % + (self._partitioned_dim_sizes, self._inner_dim_sizes)) + + @staticmethod + def from_dim_sizes(dim_sizes): + """Constructs a ragged shape from a list of dimension sizes. + + This list contains a single tensor for each dimension, where the tensor + is a scalar if the dimension is uniform, or a vector if the dimension is + ragged. + + Args: + dim_sizes: List of int64 scalars or vectors. + + Returns: + A RaggedTensorDynamicShape. + """ + with ops.name_scope(None, 'RaggedTensorDynamicShapeFromDimensionSizes', + [dim_sizes]): + dim_sizes = tuple( + ragged_util.convert_to_int_tensor( + size, dtype=dtypes.int64, name='dim_sizes') for size in dim_sizes) + # Split the dimensions into partitioned & inner dimensions. + inner_split = 0 + for dim, dim_size in enumerate(dim_sizes): + if dim_size.shape.ndims == 1: + inner_split = dim + 1 + elif dim_size.shape.ndims != 0: + raise ValueError('Each dim_size must be a scalar or a vector') + return RaggedTensorDynamicShape(dim_sizes[:inner_split], + dim_sizes[inner_split:]) + + @classmethod + def from_tensor(cls, rt_input): + """Constructs a ragged shape for a potentially ragged tensor.""" + with ops.name_scope(None, 'RaggedTensorDynamicShapeFromTensor', [rt_input]): + rt_input = ragged_factory_ops.convert_to_tensor_or_ragged_tensor(rt_input) + if not ragged_tensor.is_ragged(rt_input): + return cls([], array_ops.shape(rt_input)) + else: + partitioned_dim_sizes = ((ragged_array_ops.nrows(rt_input),) + + ragged_array_ops.nested_row_lengths(rt_input)) + return RaggedTensorDynamicShape( + partitioned_dim_sizes, + array_ops.shape(rt_input.inner_values)[1:]) + + def dimension_size(self, axis): + """Returns the size of slices across the specified dimension.""" + if not isinstance(axis, int): + raise TypeError('axis must be an integer') + partitioned_ndims = len(self._partitioned_dim_sizes) + if axis < partitioned_ndims: + return self._partitioned_dim_sizes[axis] + else: + return self._inner_dim_sizes[axis - partitioned_ndims] + + def is_ragged(self, axis): + """Returns true if the indicated dimension is ragged.""" + if not isinstance(axis, int): + raise TypeError('axis must be an integer') + rank = self.rank + if axis < 0: + raise ValueError('Negative axis values are not supported') + elif rank is not None and axis >= rank: + raise ValueError('Expected axis=%s < rank=%s' % (axis, rank)) + else: + return (axis > 0 and axis < len(self._partitioned_dim_sizes) and + self._partitioned_dim_sizes[axis].shape.ndims == 1) + + @property + def rank(self): + """The number of dimensions in this shape, or None if unknown.""" + inner_ndims = self._inner_dim_sizes.shape[0].value + if inner_ndims is None: + return None + else: + return len(self._partitioned_dim_sizes) + inner_ndims + + @property + def partitioned_dim_sizes(self): + """The partitioned dimension sizes for this shape. + + Returns: + A `list` of 0-D or 1-D integer `Tensor`. + """ + return self._partitioned_dim_sizes + + @property + def inner_dim_sizes(self): + """The inner dimension sizes for this shape. + + Returns: + A 1-D integer `Tensor`. + """ + return self._inner_dim_sizes + + @property + def num_partitioned_dimensions(self): + """The number of partitioned dimensions in this shape.""" + return len(self._partitioned_dim_sizes) + + @property + def num_inner_dimensions(self): + """The number of inner dimensions, or `None` if not statically known.""" + return self._inner_dim_sizes.shape[0].value + + def broadcast_to_rank(self, rank): + """Adds leading size-1 dimensions to broadcast `self` to the given rank. + + E.g., if `shape1` is `[3, (D2), 4]`, then `shape1.broadcast_to_rank(5)` + is `[1, 1, 3, (D2), 4]`. + + Args: + rank: The rank for the returned shape. + + Returns: + A RaggedTensorDynamicShape with `rank` dimensions, whose inner dimensions + have the same size as `self` and whose outer dimensions have size `1`. + + Raises: + ValueError: If `self.rank` is unknown or greater than `rank`. + """ + if self.rank is None: + raise ValueError('Unable to broadcast: self.rank is unknown') + dims_to_add = rank - self.rank + if dims_to_add < 0: + raise ValueError('Unable to broadcast: rank=%d must be greater than ' + 'self.rank=%d.' % (rank, self.rank)) + elif dims_to_add == 0: + return self + elif self._partitioned_dim_sizes: + partitioned_dims = (1,) * dims_to_add + self._partitioned_dim_sizes + return RaggedTensorDynamicShape(partitioned_dims, self._inner_dim_sizes) + else: + inner_dims = array_ops.concat( + [array_ops.ones([dims_to_add], dtypes.int64), self.inner_dim_sizes], + axis=0) + return RaggedTensorDynamicShape([], inner_dims) + + def broadcast_dimension(self, axis, lengths): + """Returns a shape that is broadcast-compatible with self & lengths. + + * If dimension[axis] is uniform and lengths is a scalar, the check + that either lengths==1 or axis==1 or lengths==axis, and tile + dimension[axis] with tf.where(lengths==axis, 1, axis) repeats. + + * If dimension[axis] is uniform and lengths is a vector, then check + that dimension[axis]==1, and raggedly tile dimension[axis] with + lengths repeats. (we can skip tiling if we statically know that + slice_lengths == 1??) + + * If dimension[axis] is ragged and lengths is a scalar, then check + that lengths==1. + + * If dimension[axis] is ragged and lengths is a vector, then check + that self.dimension_size(axis) == lengths. + + Args: + axis: `int`. The dimension to broadcast. + lengths: 0-D or 1-D integer `Tensor`. + + Returns: + A `RaggedTensorDynamicShape`. + """ + lengths = ragged_util.convert_to_int_tensor( + lengths, name='lengths', dtype=dtypes.int64) + # Check whether lengths is a scalar (for uniform dimensions) or + # vector (for ragged dimensions). + if lengths.shape.ndims is None: + raise ValueError('lengths must have a known rank.') + elif lengths.shape.ndims > 1: + raise ValueError('lengths must be a scalar or vector') + else: + lengths_is_scalar = (lengths.shape.ndims == 0) + + # Verify that the shapes are compatible. + if self.is_ragged(axis): + if lengths_is_scalar: + condition = math_ops.equal(lengths, 1) + else: + condition = math_ops.reduce_all( + math_ops.equal(lengths, self.dimension_size(axis))) + else: + axis_dim_size = self.dimension_size(axis) + if lengths_is_scalar: + condition = ( + math_ops.equal(lengths, 1) | math_ops.equal(axis_dim_size, 1) + | math_ops.equal(axis_dim_size, lengths)) + else: + condition = math_ops.equal(axis_dim_size, 1) + broadcast_err = [ + 'Unable to broadcast: dimension size mismatch in dimension', axis, + 'lengths=', lengths, 'dim_size=', + self.dimension_size(axis) + ] + broadcast_check = control_flow_ops.Assert( + condition, data=broadcast_err, summarize=10) + + with ops.control_dependencies([broadcast_check]): + # Partitioned dimensions: + if axis < self.num_partitioned_dimensions: + if self.is_ragged(axis): + # Use an identity op to make sure the check actually gets run. + return RaggedTensorDynamicShape( + self._partitioned_dim_sizes, + array_ops.identity(self.inner_dim_sizes)) + else: + return self._broadcast_uniform_partitioned_dimension(axis, lengths) + + # Inner dimensions: + else: + if lengths_is_scalar: + return self._broadcast_inner_dimension_to_uniform(axis, lengths) + else: + if axis == 0: + raise ValueError('Unable to broadcast: ' + 'outermost dimension must be uniform.') + return self._broadcast_inner_dimension_to_ragged(axis, lengths) + + def num_slices_in_dimension(self, axis): + """Returns the total number of slices across the indicated dimension.""" + if axis < 0: + return constant_op.constant(1, dtype=dtypes.int64) + elif self.is_ragged(axis): + return math_ops.reduce_sum(self._partitioned_dim_sizes[axis]) + else: + return self.dimension_size(axis) * self.num_slices_in_dimension(axis - 1) + + def _broadcast_uniform_partitioned_dimension(self, axis, lengths): + """Broadcasts the partitioned dimension `axis` to match `lengths`.""" + axis_dim_size = self.dimension_size(axis) + partitioned_sizes = list(self._partitioned_dim_sizes[:axis]) + + if lengths.shape.ndims == 0: + lengths = array_ops.where( + math_ops.equal(axis_dim_size, 1), lengths, axis_dim_size) + repeats = array_ops.where(math_ops.equal(axis_dim_size, 1), lengths, 1) + splits = array_ops.stack([0, self.num_slices_in_dimension(axis)]) + else: + splits = math_ops.range( + array_ops.size(lengths, out_type=dtypes.int64) + 1) + repeats = lengths + + partitioned_sizes.append(lengths) + + for dim_size in self._partitioned_dim_sizes[axis + 1:]: + if dim_size.shape.ndims == 0: + partitioned_sizes.append(dim_size) + splits *= dim_size + else: + partitioned_sizes.append( + ragged_util.repeat_ranges(dim_size, splits, repeats)) + splits = array_ops.gather( + ragged_util.lengths_to_splits(dim_size), splits) + inner_sizes = self._inner_dim_sizes + return RaggedTensorDynamicShape(partitioned_sizes, inner_sizes) + + def _broadcast_inner_dimension_to_uniform(self, axis, length): + """Broadcasts the inner dimension `axis` to match `lengths`.""" + dim_size = self.dimension_size(axis) + axis_in_inner_dims = axis - self.num_partitioned_dimensions + partitioned_sizes = self._partitioned_dim_sizes + inner_sizes = array_ops.concat([ + self._inner_dim_sizes[:axis_in_inner_dims], + [array_ops.where(math_ops.equal(dim_size, 1), length, dim_size)], + self._inner_dim_sizes[axis_in_inner_dims + 1:] + ], + axis=0) + return RaggedTensorDynamicShape(partitioned_sizes, inner_sizes) + + def _broadcast_inner_dimension_to_ragged(self, axis, lengths): + axis_in_inner_dims = axis - self.num_partitioned_dimensions + partitioned_sizes = ( + self._partitioned_dim_sizes + tuple([ + self._inner_dim_sizes[i] for i in range(axis_in_inner_dims) + ]) + (lengths,)) + inner_sizes = self._inner_dim_sizes[axis_in_inner_dims + 1:] + return RaggedTensorDynamicShape(partitioned_sizes, inner_sizes) + + +def broadcast_dynamic_shape(shape_x, shape_y): + """Returns the shape formed by broadcasting two shapes to be compatible. + + Args: + shape_x: A `RaggedTensorDynamicShape` + shape_y: A `RaggedTensorDynamicShape` + + Returns: + A `RaggedTensorDynamicShape`. + Raises: + ValueError: If `shape_x` and `shape_y` are not broadcast-compatible. + """ + if not isinstance(shape_x, RaggedTensorDynamicShape): + raise TypeError('shape_x must be a RaggedTensorDynamicShape') + if not isinstance(shape_y, RaggedTensorDynamicShape): + raise TypeError('shape_y must be a RaggedTensorDynamicShape') + + # Broadcast both shapes to have the same rank. + if shape_x.rank is None or shape_y.rank is None: + raise ValueError('Unable to broadcast: unknown rank') + broadcast_rank = max(shape_x.rank, shape_y.rank) + shape_x = shape_x.broadcast_to_rank(broadcast_rank) + shape_y = shape_y.broadcast_to_rank(broadcast_rank) + + # Broadcast dimensions one at a time, starting from the outermost dimension. + for axis in range(broadcast_rank): + shape_x = shape_x.broadcast_dimension(axis, shape_y.dimension_size(axis)) + shape_y = shape_y.broadcast_dimension(axis, shape_x.dimension_size(axis)) + + return shape_x + + +def broadcast_to(rt_input, shape, broadcast_inner_dimensions=True): + """Broadcasts a potentially ragged tensor to a ragged shape. + + Tiles `rt_input` as necessary to match the given shape. + + Behavior is undefined if `rt_input` is not broadcast-compatible with `shape`. + + Args: + rt_input: The potentially ragged tensor to broadcast. + shape: A `RaggedTensorDynamicShape` + broadcast_inner_dimensions: If false, then inner dimensions will not be + tiled. + + Returns: + A potentially ragged tensor whose values are taken from + `rt_input`, and whose shape matches `shape`. + """ + if not isinstance(shape, RaggedTensorDynamicShape): + raise TypeError('shape must be a RaggedTensorDynamicShape') + rt_input = ragged_factory_ops.convert_to_tensor_or_ragged_tensor(rt_input) + + # Broadcasting to a uniform shape. + if shape.num_partitioned_dimensions == 0: + return _broadcast_to_uniform_shape(rt_input, shape, + broadcast_inner_dimensions) + else: + return _broadcast_to_ragged_shape(rt_input, shape, + broadcast_inner_dimensions) + + +def _broadcast_to_uniform_shape(rt_input, shape, broadcast_inner_dimensions): + """Broadcasts rt_input to the uniform shape `shape`.""" + if isinstance(rt_input, ragged_tensor.RaggedTensor): + raise ValueError('Incompatible with shape: ragged rank mismatch') + if broadcast_inner_dimensions: + return array_ops.broadcast_to(rt_input, shape.inner_dim_sizes) + else: + return rt_input + + +def _broadcast_to_ragged_shape(rt_input, dst_shape, broadcast_inner_dimensions): + """Broadcasts rt_input to the ragged shape `dst_shape`.""" + # dst_shape's rank and ragged_rank must be greater than or equal to rt_input's + if rt_input.shape.ndims is None or dst_shape.rank is None: + raise ValueError('Unable to broadcast: unknown rank') + if rt_input.shape.ndims > dst_shape.rank: + raise ValueError('Incompatible with shape: rank mismatch') + if (isinstance(rt_input, ragged_tensor.RaggedTensor) and + rt_input.ragged_rank >= dst_shape.num_partitioned_dimensions): + raise ValueError('Incompatible with shape: ragged rank mismatch') + + src_shape = RaggedTensorDynamicShape.from_tensor(rt_input) + src_shape = src_shape.broadcast_to_rank(dst_shape.rank) + + # Add dimensions to rt_input so its rank and ragged_rank matches dst_shape. + if dst_shape.rank > rt_input.shape.ndims: + if rt_input.shape.ndims < dst_shape.num_inner_dimensions + 1: + rt_input = array_ops.reshape( + rt_input, array_ops.concat([[-1], dst_shape.inner_dim_sizes], axis=0)) + for _ in range(dst_shape.rank - rt_input.shape.ndims): + rt_input = ragged_factory_ops.from_row_lengths( + rt_input, [ragged_array_ops.nrows(rt_input)]) + + # Add ragged dimensions to match dst_shape. + if ragged_tensor.is_ragged(rt_input): + inner_rank_diff = ( + rt_input.inner_values.shape.ndims - 1 - dst_shape.num_inner_dimensions) + if inner_rank_diff > 0: + rt_input = rt_input.with_inner_values( + ragged_conversion_ops.from_tensor( + rt_input.inner_values, ragged_rank=inner_rank_diff)) + else: + rt_input = ragged_conversion_ops.from_tensor( + rt_input, ragged_rank=dst_shape.num_partitioned_dimensions - 1) + + # Do broadcasting for any dimensions that will remain uniform. We can do + # these all at once, since they're independent of one another. + multiples = [1] * dst_shape.rank + for axis in range(dst_shape.num_partitioned_dimensions): + if not src_shape.is_ragged(axis) and not dst_shape.is_ragged(axis): + src_size = src_shape.dimension_size(axis) + dst_size = dst_shape.dimension_size(axis) + if ((tensor_util.constant_value(src_size) in (1, None)) and + (tensor_util.constant_value(dst_size) != 1)): + multiples[axis] = array_ops.where( + math_ops.equal(src_size, 1), dst_size, 1) + if not all(isinstance(v, int) and v == 1 for v in multiples): + multiples = array_ops.stack(multiples, axis=0) + rt_input = ragged_array_ops.tile(rt_input, multiples) + + if broadcast_inner_dimensions: + rt_input = rt_input.with_inner_values( + array_ops.reshape( + rt_input.inner_values, + array_ops.concat([[-1], dst_shape.inner_dim_sizes], axis=0))) + + # Do broadcasting for dimensions that become ragged. We must do these from + # outermost to innermost. + for axis in range(dst_shape.num_partitioned_dimensions): + if not src_shape.is_ragged(axis) and dst_shape.is_ragged(axis): + dst_size = dst_shape.dimension_size(axis) + rt_input = _ragged_tile_axis(rt_input, axis, dst_size) + + return rt_input + + +def _ragged_tile_axis(rt_input, axis, repeats): + """Tile a dimension of a RaggedTensor to match a ragged shape.""" + assert axis > 0 # Outermost dimension may not be ragged. + + if not ragged_tensor.is_ragged(rt_input): + rt_input = ragged_conversion_ops.from_tensor(rt_input, ragged_rank=1) + + if axis > 1: + return rt_input.with_values( + _ragged_tile_axis(rt_input.values, axis - 1, repeats)) + else: + src_row_splits = rt_input.nested_row_splits + src_row_lengths = ragged_array_ops.nested_row_lengths(rt_input) + splits = src_row_splits[0] + + dst_row_lengths = [repeats] + for i in range(1, len(src_row_lengths)): + dst_row_lengths.append( + ragged_util.repeat_ranges(src_row_lengths[i], splits, repeats)) + splits = array_ops.gather(src_row_splits[i], splits) + dst_values = ragged_util.repeat_ranges(rt_input.inner_values, splits, + repeats) + return ragged_factory_ops.from_nested_row_lengths(dst_values, + dst_row_lengths) + diff --git a/tensorflow/python/ops/ragged/ragged_tensor_shape_test.py b/tensorflow/python/ops/ragged/ragged_tensor_shape_test.py new file mode 100644 index 00000000000..9c2dd260503 --- /dev/null +++ b/tensorflow/python/ops/ragged/ragged_tensor_shape_test.py @@ -0,0 +1,487 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Tests for tf.ragged.ragged_tensor_shape.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from absl.testing import parameterized +import numpy as np + +from tensorflow.python.framework import dtypes +from tensorflow.python.framework import test_util +from tensorflow.python.ops import ragged +from tensorflow.python.platform import googletest + + +class RaggedTensorShapeTest(test_util.TensorFlowTestCase, + parameterized.TestCase): + + def assertShapeEq(self, x, y): + assert isinstance(x, ragged.RaggedTensorDynamicShape) + assert isinstance(y, ragged.RaggedTensorDynamicShape) + x_partitioned_dim_sizes = [ + splits.eval().tolist() # + for splits in x.partitioned_dim_sizes + ] + y_partitioned_dim_sizes = [ + splits.eval().tolist() # + for splits in y.partitioned_dim_sizes + ] + self.assertEqual(x_partitioned_dim_sizes, y_partitioned_dim_sizes) + self.assertEqual(x.inner_dim_sizes.eval().tolist(), + y.inner_dim_sizes.eval().tolist()) + + @parameterized.parameters([ + dict(value='x', expected_dim_sizes=[]), + dict(value=['a', 'b', 'c'], expected_dim_sizes=[3]), + dict(value=[['a', 'b', 'c'], ['d', 'e', 'f']], expected_dim_sizes=[2, 3]), + dict( + value=[[['a', 'b', 'c'], ['d', 'e', 'f']]], + expected_dim_sizes=[1, 2, 3]), + dict( + value=ragged.constant_value([['a', 'b', 'c'], ['d', 'e']]), + expected_dim_sizes=[2, [3, 2]]), + dict( + value=ragged.constant_value([[['a', 'b', 'c'], ['d', 'e']]]), + expected_dim_sizes=[1, [2], [3, 2]]), + dict( + value=ragged.constant_value([[['a', 'b', 'c'], ['d', 'e', 'f']]], + ragged_rank=1), + expected_dim_sizes=[1, [2], 3]), + dict( + value=ragged.constant_value([[[[1], [2]], [[3], [4]]], + [[[5], [6]]]], ragged_rank=1), + expected_dim_sizes=[2, [2, 1], 2, 1]), + dict( + value=ragged.constant_value([[10, 20], [30]]), + expected_dim_sizes=[2, [2, 1]]), + # Docstring examples: + dict(value=[[1, 2, 3], [4, 5, 6]], expected_dim_sizes=[2, 3]), + dict( + value=ragged.constant_value([[1, 2], [], [3, 4, 5]]), + expected_dim_sizes=[3, [2, 0, 3]]), + dict( + value=ragged.constant_value([[[1, 2], [3, 4]], [[5, 6]]], + ragged_rank=1), + expected_dim_sizes=[2, [2, 1], 2]), + dict( + value=ragged.constant_value([[[1, 2], [3]], [[4, 5]]]), + expected_dim_sizes=[2, [2, 1], [2, 1, 2]]), + ]) + def testFromTensor(self, value, expected_dim_sizes): + shape = ragged.RaggedTensorDynamicShape.from_tensor(value) + expected = ragged.RaggedTensorDynamicShape.from_dim_sizes( + expected_dim_sizes) + with self.cached_session(): + self.assertShapeEq(shape, expected) + + @parameterized.parameters([ + dict(dim_sizes=[], rank=0, expected_dim_sizes=[]), + dict(dim_sizes=[], rank=3, expected_dim_sizes=[1, 1, 1]), + dict(dim_sizes=[3], rank=1, expected_dim_sizes=[3]), + dict(dim_sizes=[3], rank=3, expected_dim_sizes=[1, 1, 3]), + dict(dim_sizes=[2, 3], rank=3, expected_dim_sizes=[1, 2, 3]), + dict(dim_sizes=[3, [3, 2, 4]], rank=2, expected_dim_sizes=[3, [3, 2, 4]]), + dict( + dim_sizes=[3, [3, 2, 4]], + rank=4, + expected_dim_sizes=[1, 1, 3, [3, 2, 4]]), + dict( + dim_sizes=[3, [3, 2, 4], 2, 3], + rank=5, + expected_dim_sizes=[1, 3, [3, 2, 4], 2, 3]), + ]) + def testBroadcastToRank(self, dim_sizes, rank, expected_dim_sizes): + shape = ragged.RaggedTensorDynamicShape.from_dim_sizes(dim_sizes) + expected = ragged.RaggedTensorDynamicShape.from_dim_sizes( + expected_dim_sizes) + broadcasted_shape = shape.broadcast_to_rank(rank) + with self.cached_session(): + self.assertShapeEq(broadcasted_shape, expected) + self.assertEqual(broadcasted_shape.rank, rank) + + @parameterized.parameters([ + #========================================================================= + # dimension[axis] is uniform inner; and row_lengths is a scalar + #========================================================================= + # shape: [BROADCAST(UNIFORM), UNIFORM, UNIFORM] + dict(axis=0, + row_length=3, + original_dim_sizes=[1, 4, 5], + broadcast_dim_sizes=[3, 4, 5]), + + # shape: [UNIFORM, UNIFORM, BROADCAST(UNIFORM)] + dict(axis=2, + row_length=5, + original_dim_sizes=[3, 4, 1], + broadcast_dim_sizes=[3, 4, 5]), + + # shape: [UNIFORM, RAGGED, BROADCAST(UNIFORM)] + dict(axis=2, + row_length=5, + original_dim_sizes=[3, [3, 2, 8], 1], + broadcast_dim_sizes=[3, [3, 2, 8], 5]), + + # shape: [UNIFORM, RAGGED, RAGGED, UNIFORM, UNIFORM, BROADCAST(UNIFORM)] + dict(axis=5, + row_length=5, + original_dim_sizes=[2, [2, 1], [3, 2, 8], 3, 4, 1], + broadcast_dim_sizes=[2, [2, 1], [3, 2, 8], 3, 4, 5]), + + #========================================================================= + # dimension[axis] is uniform inner; and row_lengths is a vector + #========================================================================= + # shape: [UNIFORM, BROADCAST(UNIFORM)] + dict(axis=1, + row_length=[2, 0, 1], + original_dim_sizes=[3, 1], + broadcast_dim_sizes=[3, [2, 0, 1]]), + # shape: [UNIFORM, BROADCAST(UNIFORM), UNIFORM] + dict(axis=1, + row_length=[2, 0, 1], + original_dim_sizes=[3, 1, 5], + broadcast_dim_sizes=[3, [2, 0, 1], 5]), + + # shape: [UNIFORM, UNIFORM, BROADCAST(UNIFORM)] + dict(axis=2, + row_length=[2, 0, 1, 3, 8, 2, 3, 4, 1, 8, 7, 0], + original_dim_sizes=[4, 3, 1], + broadcast_dim_sizes=[4, 3, [2, 0, 1, 3, 8, 2, 3, 4, 1, 8, 7, 0]]), + + # shape: [UNIFORM, RAGGED, BROADCAST(UNIFORM)] + dict(axis=2, + row_length=[2, 5, 3], + original_dim_sizes=[2, [2, 1], 1], + broadcast_dim_sizes=[2, [2, 1], [2, 5, 3]]), + + # shape: [UNIFORM, RAGGED, UNIFORM, UNIFORM, BROADCAST(UNIFORM), UNIFORM] + dict(axis=4, + row_length=list(range(18)), + original_dim_sizes=[2, [2, 1], 3, 2, 1, 8], + broadcast_dim_sizes=[2, [2, 1], 3, 2, list(range(18)), 8]), + + #========================================================================= + # dimension[axis] is uniform partitioned; and row_lengths is a scalar + #========================================================================= + # shape: [BROADCAST(UNIFORM), RAGGED] + dict(axis=0, + row_length=3, + original_dim_sizes=[1, [5]], + broadcast_dim_sizes=[3, [5, 5, 5]]), + + # shape: [BROADCAST(UNIFORM), UNIFORM, RAGGED] + dict(axis=0, + row_length=2, + original_dim_sizes=[1, 3, [3, 0, 2]], + broadcast_dim_sizes=[2, 3, [3, 0, 2, 3, 0, 2]]), + + # shape: [BROADCAST(UNIFORM), RAGGED, RAGGED, UNIFORM, UNIFORM] + dict(axis=0, + row_length=3, + original_dim_sizes=[1, [3], [3, 5, 2], 9, 4, 5], + broadcast_dim_sizes=[3, [3, 3, 3], [3, 5, 2, 3, 5, 2, 3, 5, 2], + 9, 4, 5]), + + # shape: [BROADCAST(UNIFORM), UNIFORM, RAGGED, UNIFORM] + dict(axis=0, + row_length=2, + original_dim_sizes=[1, 2, [2, 1], [3, 5, 2], 2], + broadcast_dim_sizes=[2, 2, [2, 1, 2, 1], [3, 5, 2, 3, 5, 2], 2]), + + # shape: [UNIFORM, BROADCAST(UNIFORM), RAGGED, UNIFORM] + dict(axis=1, + row_length=2, + original_dim_sizes=[3, 1, [4, 0, 2], 5], + broadcast_dim_sizes=[3, 2, [4, 0, 2, 4, 0, 2], 5]), + + # shape: [UNIFORM, BROADCAST(UNIFORM), RAGGED] + dict(axis=1, + row_length=1, + original_dim_sizes=[2, 3, (1, 2, 3, 4, 5, 6)], + broadcast_dim_sizes=[2, 3, (1, 2, 3, 4, 5, 6)]), + + #========================================================================= + # dimension[axis] is uniform partitioned; and row_lengths is a vector + #========================================================================= + # shape: [UNIFORM, BROADCAST(UNIFORM), RAGGED, UNIFORM] + dict(axis=1, + row_length=[4, 1, 2], + original_dim_sizes=[ + 3, # axis=0 + 1, # axis=1 (broadcast) + [3, 1, 2], # axis=2 + 5], # axis=3 + broadcast_dim_sizes=[ + 3, # axis=0 + [4, 1, 2], # axis=1 (broadcast) + [3, 3, 3, 3, 1, 2, 2], # axis=2 + 5]), # axis=3 + + # shape: [UNIFORM, BROADCAST(UNIFORM), RAGGED, RAGGED] + dict(axis=1, + row_length=[2, 0, 3], + original_dim_sizes=[ + 3, # axis=0 + 1, # axis=1 (broadcast) + [3, 1, 2], # axis=2 + [3, 1, 4, 1, 5, 9]], # axis=3 + broadcast_dim_sizes=[ + 3, # axis=0 + [2, 0, 3], # axis=1 (broadcast) + [3, 3, 2, 2, 2], # axis=2 + [3, 1, 4, 3, 1, 4, 5, 9, 5, 9, 5, 9]]), # axis=3 + + # shape: [UNIFORM, RAGGED, BROADCAST(UNIFORM), RAGGED, RAGGED, UNIFORM] + dict(axis=2, + row_length=[4, 1, 2], + original_dim_sizes=[ + 3, # axis=0 + [2, 0, 1], # axis=1 + 1, # axis=2 (broadcast) + [3, 2, 1], # axis=3 + [1, 0, 1, 0, 2, 3], # axis=4 + 5], # axis=5 + broadcast_dim_sizes=[ + 3, # axis=0 + [2, 0, 1], # axis=2 + [4, 1, 2], # axis=2 (broadcast) + [3, 3, 3, 3, 2, 1, 1], # axis=3 + [1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 0, # axis=4 + 2, 3, 3], + 5]), # axis=5 + + dict(axis=0, + row_length=2, + original_dim_sizes=[1, 1, 2, (2, 1)], + broadcast_dim_sizes=[2, 1, 2, (2, 1, 2, 1)]), + dict(axis=1, + row_length=(2, 1), + original_dim_sizes=[2, 1, 2, (2, 1, 2, 1)], + broadcast_dim_sizes=[2, (2, 1), 2, (2, 1, 2, 1, 2, 1)]), + dict(axis=2, + row_length=2, + original_dim_sizes=[2, (2, 1), 2, (2, 1, 2, 1, 2, 1)], + broadcast_dim_sizes=[2, (2, 1), 2, (2, 1, 2, 1, 2, 1)]), + dict(axis=3, + row_length=(2, 1, 2, 1, 2, 1), + original_dim_sizes=[2, (2, 1), 2, 1], + broadcast_dim_sizes=[2, (2, 1), 2, (2, 1, 2, 1, 2, 1)]), + ]) # pyformat: disable + def testBroadcastDimension(self, axis, row_length, original_dim_sizes, + broadcast_dim_sizes): + """Tests for the broadcast_dimension method. + + Verifies that: + + * `original.broadcast_dimension(axis, row_length) == broadcast` + * `broadcast.broadcast_dimension(axis, row_length) == broadcast` + * `broadcast.broadcast_dimension(axis, 1) == broadcast` + + Args: + axis: The axis to broadcast + row_length: The slice lengths to broadcast to. + original_dim_sizes: The dimension sizes before broadcasting. + original_dim_sizes[axis] should be equal to `1` or `row_length`. + broadcast_dim_sizes: THe dimension sizes after broadcasting. + """ + original_shape = ragged.RaggedTensorDynamicShape.from_dim_sizes( + original_dim_sizes) + broadcast_shape = ragged.RaggedTensorDynamicShape.from_dim_sizes( + broadcast_dim_sizes) + self.assertEqual(original_shape.rank, broadcast_shape.rank) + with self.cached_session(): + # shape[axis].value == 1 and row_length > 1: + bcast1 = original_shape.broadcast_dimension(axis, row_length) + # shape[axis].value > 1 and row_length == shape[axis].value: + bcast2 = broadcast_shape.broadcast_dimension(axis, row_length) + # shape[axis].value > 1 and row_length == 1: + bcast3 = broadcast_shape.broadcast_dimension(axis, 1) + + self.assertShapeEq(bcast1, broadcast_shape) + self.assertShapeEq(bcast2, broadcast_shape) + self.assertShapeEq(bcast3, broadcast_shape) + + @parameterized.parameters( + [ + # Broadcast scalar + dict(x_dims=[], y_dims=[], expected_dims=[]), + dict(x_dims=[], y_dims=[2], expected_dims=[2]), + dict(x_dims=[], y_dims=[2, 3], expected_dims=[2, 3]), + dict( + x_dims=[], + y_dims=[2, (2, 3), (5, 7, 2, 0, 9)], + expected_dims=[2, (2, 3), (5, 7, 2, 0, 9)]), + # Broadcast vector + dict(x_dims=[3], y_dims=[4, 2, 3], expected_dims=[4, 2, 3]), + dict(x_dims=[1], y_dims=[4, 2, 3], expected_dims=[4, 2, 3]), + dict(x_dims=[3], y_dims=[4, 2, 1], expected_dims=[4, 2, 3]), + dict( + x_dims=[3], + y_dims=[3, (2, 3, 1), 1], + expected_dims=[3, (2, 3, 1), 3]), + dict(x_dims=[1], y_dims=[3, (2, 1, 3)], expected_dims=[3, (2, 1, 3)]), + dict( + x_dims=[1], + y_dims=[3, (2, 1, 3), 8], + expected_dims=[3, (2, 1, 3), 8]), + dict( + x_dims=[1], + y_dims=[2, (2, 3), (5, 7, 2, 0, 9)], + expected_dims=[2, (2, 3), (5, 7, 2, 0, 9)]), + # Mixed broadcasting + dict( + x_dims=[ + 1, # axis=0 + 3, # axis=1 + (3, 0, 2), # axis=2 + 1, # axis=3 + 2, # axis=4 + ], + y_dims=[ + 2, # axis=0 + 1, # axis=1 + 1, # axis=2 + (7, 2), # axis=3 + 1, # axis=4 + ], + expected_dims=[ + 2, # axis=0 + 3, # axis=1 + (3, 0, 2, 3, 0, 2), # axis=2 + (7, 7, 7, 7, 7, 2, 2, 2, 2, 2), # axis=3 + 2, # axis=4 + ]), + dict( + x_dims=[2, (2, 1), 2, 1], + y_dims=[1, 1, 2, (2, 1)], + expected_dims=[2, (2, 1), 2, (2, 1, 2, 1, 2, 1)]), + ]) + def testBroadcastDynamicShape(self, x_dims, y_dims, expected_dims): + x_shape = ragged.RaggedTensorDynamicShape.from_dim_sizes(x_dims) + y_shape = ragged.RaggedTensorDynamicShape.from_dim_sizes(y_dims) + expected = ragged.RaggedTensorDynamicShape.from_dim_sizes(expected_dims) + result1 = ragged.broadcast_dynamic_shape(x_shape, y_shape) + result2 = ragged.broadcast_dynamic_shape(y_shape, x_shape) + with self.cached_session(): + self.assertShapeEq(expected, result1) + self.assertShapeEq(expected, result2) + + def testRepr(self): + shape = ragged.RaggedTensorDynamicShape.from_dim_sizes([2, (2, 1), 2, 1]) + self.assertRegexpMatches( + repr(shape), + r'RaggedTensorDynamicShape\(' + r'partitioned_dim_sizes=\(<[^>]+>, <[^>]+>\), ' + r'inner_dim_sizes=<[^>]+>\)') + + @parameterized.parameters([ + dict( + x=[[10], [20], [30]], # shape=[3, 1] + dim_sizes=[3, 2], + expected=[[10, 10], [20, 20], [30, 30]]), + dict( + x=[[10], [20], [30]], # shape=[3, 1] + dim_sizes=[3, [3, 0, 2]], + expected=ragged.constant_value([[10, 10, 10], [], [30, 30]], + dtype=np.int32)), + dict( + x=[[[1, 2, 3]], [[4, 5, 6]]], # shape = [2, 1, 3] + dim_sizes=[2, [2, 3], 3], + expected=ragged.constant_value( + [[[1, 2, 3], [1, 2, 3]], [[4, 5, 6], [4, 5, 6], [4, 5, 6]]], + dtype=np.int32, + ragged_rank=1)), + dict( + x=[[[1]], [[2]]], # shape = [2, 1, 1] + dim_sizes=[2, [2, 3], [0, 2, 1, 2, 0]], + expected=ragged.constant_value([[[], [1, 1]], [[2], [2, 2], []]], + dtype=np.int32, + ragged_rank=2)), + dict( + x=10, + dim_sizes=[3, [3, 0, 2]], + expected=ragged.constant_value([[10, 10, 10], [], [10, 10]])), + ]) + def testRaggedBroadcastTo(self, x, dim_sizes, expected): + shape = ragged.RaggedTensorDynamicShape.from_dim_sizes(dim_sizes) + result = ragged.broadcast_to(x, shape) + with self.cached_session(): + self.assertEqual( + getattr(result, 'ragged_rank', 0), getattr(expected, 'ragged_rank', + 0)) + if hasattr(expected, 'tolist'): + expected = expected.tolist() + self.assertEqual(result.eval().tolist(), expected) + + @parameterized.parameters([ + dict( + doc='x.shape=[3, (D1)]; y.shape=[3, 1]; bcast.shape=[3, (D1)]', + x=ragged.constant_value([[1, 2, 3], [], [4, 5]], dtype=np.int32), + y=[[10], [20], [30]], + expected=ragged.constant_value([[11, 12, 13], [], [34, 35]])), + dict( + doc='x.shape=[3, (D1)]; y.shape=[]; bcast.shape=[3, (D1)]', + x=ragged.constant_value([[1, 2, 3], [], [4, 5]], dtype=np.int32), + y=10, + expected=ragged.constant_value([[11, 12, 13], [], [14, 15]])), + dict( + doc='x.shape=[1, (D1)]; y.shape=[3, 1]; bcast.shape=[3, (D1)]', + x=ragged.constant_value([[1, 2, 3]], dtype=np.int32), + y=[[10], [20], [30]], + expected=ragged.constant_value( + [[11, 12, 13], [21, 22, 23], [31, 32, 33]], dtype=np.int32)), + dict( + doc=('x.shape=[2, (D1), 1]; y.shape=[1, (D2)]; ' + 'bcast.shape=[2, (D1), (D2)]'), + x=ragged.constant_value([[[1], [2], [3]], [[4]]], ragged_rank=1), + y=ragged.constant_value([[10, 20, 30]]), + expected=ragged.constant_value([[[11, 21, 31], [12, 22, 32], + [13, 23, 33]], [[14, 24, 34]]])), + dict( + doc=('x.shape=[2, (D1), 1]; y.shape=[1, 1, 4]; ' + 'bcast.shape=[2, (D1), 4]'), + x=ragged.constant_value([[[10], [20]], [[30]]], ragged_rank=1), + y=[[[1, 2, 3, 4]]], + expected=ragged.constant_value( + [[[11, 12, 13, 14], [21, 22, 23, 24]], [[31, 32, 33, 34]]], + ragged_rank=1)), + dict( + doc=('x.shape=[2, (D1), 2, 1]; y.shape=[2, (D2)]; ' + 'bcast.shape=[2, (D1), (2), (D2)'), + x=ragged.constant_value([[[[1], [2]], [[3], [4]]], + [[[5], [6]]]], + ragged_rank=1), + y=ragged.constant_value([[10, 20], [30]]), + expected=ragged.constant_value( + [[[[11, 21], [32]], [[13, 23], [34]]], + [[[15, 25], [36]]]])), + ]) + def testRaggedAddWithBroadcasting(self, x, y, expected, doc): + expected_rrank = getattr(expected, 'ragged_rank', 0) + x = ragged.convert_to_tensor_or_ragged_tensor(x, dtype=dtypes.int32) + y = ragged.convert_to_tensor_or_ragged_tensor(y, dtype=dtypes.int32) + result = x + y + result_rrank = getattr(result, 'ragged_rank', 0) + self.assertEqual(expected_rrank, result_rrank) + if hasattr(expected, 'tolist'): + expected = expected.tolist() + with self.cached_session(): + self.assertEqual(result.eval().tolist(), expected) + + +if __name__ == '__main__': + googletest.main() diff --git a/tensorflow/python/ops/ragged/ragged_tensor_test.py b/tensorflow/python/ops/ragged/ragged_tensor_test.py index 61bfcb68090..608fbd6e5b7 100644 --- a/tensorflow/python/ops/ragged/ragged_tensor_test.py +++ b/tensorflow/python/ops/ragged/ragged_tensor_test.py @@ -114,13 +114,13 @@ class RaggedTensorTest(test_util.TensorFlowTestCase, parameterized.TestCase): # RaggedTensor class docstring examples #============================================================================= + @test_util.run_deprecated_v1 def testClassDocStringExamples(self): # From section: "Component Tensors" rt = ragged.from_row_splits( values=[3, 1, 4, 1, 5, 9, 2, 6], row_splits=[0, 4, 4, 7, 8, 8]) - with self.test_session(): - self.assertEqual(rt.tolist(), - [[3, 1, 4, 1], [], [5, 9, 2], [6], []]) + self.assertEqual( + self.evaluate(rt).tolist(), [[3, 1, 4, 1], [], [5, 9, 2], [6], []]) del rt # From section: "Alternative Row-Partitioning Schemes" @@ -132,9 +132,8 @@ class RaggedTensorTest(test_util.TensorFlowTestCase, parameterized.TestCase): rt4 = ragged.from_row_starts(values, row_starts=[0, 4, 4, 7, 8]) rt5 = ragged.from_row_limits(values, row_limits=[4, 4, 7, 8, 8]) for rt in (rt1, rt2, rt3, rt4, rt5): - with self.test_session(): - self.assertEqual(rt.tolist(), - [[3, 1, 4, 1], [], [5, 9, 2], [6], []]) + self.assertEqual( + self.evaluate(rt).tolist(), [[3, 1, 4, 1], [], [5, 9, 2], [6], []]) del rt1, rt2, rt3, rt4, rt5 # From section: "Multiple Ragged Dimensions" @@ -142,28 +141,27 @@ class RaggedTensorTest(test_util.TensorFlowTestCase, parameterized.TestCase): values=[3, 1, 4, 1, 5, 9, 2, 6], row_splits=[0, 4, 4, 7, 8, 8]) outer_rt = ragged.from_row_splits(values=inner_rt, row_splits=[0, 3, 3, 5]) self.assertEqual(outer_rt.ragged_rank, 2) - with self.test_session(): - self.assertEqual(outer_rt.tolist(), - [[[3, 1, 4, 1], [], [5, 9, 2]], [], [[6], []]]) + self.assertEqual( + self.evaluate(outer_rt).tolist(), + [[[3, 1, 4, 1], [], [5, 9, 2]], [], [[6], []]]) del inner_rt, outer_rt # From section: "Multiple Ragged Dimensions" rt = ragged.from_nested_row_splits( inner_values=[3, 1, 4, 1, 5, 9, 2, 6], nested_row_splits=([0, 3, 3, 5], [0, 4, 4, 7, 8, 8])) - with self.test_session(): - self.assertEqual(rt.tolist(), - [[[3, 1, 4, 1], [], [5, 9, 2]], [], [[6], []]]) + self.assertEqual( + self.evaluate(rt).tolist(), + [[[3, 1, 4, 1], [], [5, 9, 2]], [], [[6], []]]) del rt # From section: "Uniform Inner Dimensions" rt = ragged.from_row_splits( values=array_ops.ones([5, 3]), row_splits=[0, 2, 5]) - with self.test_session(): - self.assertEqual( - rt.tolist(), - [[[1, 1, 1], [1, 1, 1]], [[1, 1, 1], [1, 1, 1], [1, 1, 1]]]) - self.assertEqual(rt.shape.as_list(), [2, None, 3]) + self.assertEqual( + self.evaluate(rt).tolist(), + [[[1, 1, 1], [1, 1, 1]], [[1, 1, 1], [1, 1, 1], [1, 1, 1]]]) + self.assertEqual(rt.shape.as_list(), [2, None, 3]) del rt #============================================================================= @@ -202,15 +200,16 @@ class RaggedTensorTest(test_util.TensorFlowTestCase, parameterized.TestCase): # RaggedTensor Constructor (private) #============================================================================= + @test_util.run_deprecated_v1 def testRaggedTensorConstruction(self): values = constant_op.constant(['a', 'b', 'c', 'd', 'e', 'f', 'g']) row_splits = constant_op.constant([0, 2, 2, 5, 6, 7], dtypes.int64) rt = ragged.RaggedTensor( values=values, row_splits=row_splits, internal=True) - with self.test_session(): - self.assertEqual(rt.tolist(), - [[b'a', b'b'], [], [b'c', b'd', b'e'], [b'f'], [b'g']]) + self.assertEqual( + self.evaluate(rt).tolist(), + [[b'a', b'b'], [], [b'c', b'd', b'e'], [b'f'], [b'g']]) def testRaggedTensorConstructionErrors(self): values = constant_op.constant(['a', 'b', 'c', 'd', 'e', 'f', 'g']) @@ -246,6 +245,7 @@ class RaggedTensorTest(test_util.TensorFlowTestCase, parameterized.TestCase): # RaggedTensor Factory Ops #============================================================================= + @test_util.run_deprecated_v1 def testFromValueRowIdsWithDerivedNRows(self): # nrows is known at graph creation time. values = constant_op.constant(['a', 'b', 'c', 'd', 'e', 'f', 'g']) @@ -262,12 +262,13 @@ class RaggedTensorTest(test_util.TensorFlowTestCase, parameterized.TestCase): self.assertIs(rt_values, values) self.assertIs(rt_value_rowids, value_rowids) # cached_value_rowids - with self.test_session(): - self.assertAllEqual(rt_value_rowids, value_rowids) - self.assertEqual(rt_nrows.eval(), 5) - self.assertEqual(rt.tolist(), - [[b'a', b'b'], [], [b'c', b'd', b'e'], [b'f'], [b'g']]) + self.assertAllEqual(rt_value_rowids, value_rowids) + self.assertEqual(self.evaluate(rt_nrows), 5) + self.assertEqual( + self.evaluate(rt).tolist(), + [[b'a', b'b'], [], [b'c', b'd', b'e'], [b'f'], [b'g']]) + @test_util.run_deprecated_v1 def testFromValueRowIdsWithDerivedNRowsDynamic(self): # nrows is not known at graph creation time. values = constant_op.constant(['a', 'b', 'c', 'd', 'e', 'f', 'g']) @@ -285,12 +286,13 @@ class RaggedTensorTest(test_util.TensorFlowTestCase, parameterized.TestCase): self.assertIs(rt_values, values) self.assertIs(rt_value_rowids, value_rowids) # cached_value_rowids - with self.test_session(): - self.assertAllEqual(rt_value_rowids, value_rowids) - self.assertEqual(rt_nrows.eval(), 5) - self.assertEqual(rt.tolist(), - [[b'a', b'b'], [], [b'c', b'd', b'e'], [b'f'], [b'g']]) + self.assertAllEqual(rt_value_rowids, value_rowids) + self.assertEqual(self.evaluate(rt_nrows), 5) + self.assertEqual( + self.evaluate(rt).tolist(), + [[b'a', b'b'], [], [b'c', b'd', b'e'], [b'f'], [b'g']]) + @test_util.run_deprecated_v1 def testFromValueRowIdsWithExplicitNRows(self): values = constant_op.constant(['a', 'b', 'c', 'd', 'e', 'f', 'g']) value_rowids = constant_op.constant([0, 0, 2, 2, 2, 3, 4], dtypes.int64) @@ -308,11 +310,11 @@ class RaggedTensorTest(test_util.TensorFlowTestCase, parameterized.TestCase): self.assertIs(rt_values, values) self.assertIs(rt_value_rowids, value_rowids) # cached_value_rowids self.assertIs(rt_nrows, nrows) # cached_nrows - with self.test_session(): - self.assertEqual( - rt.tolist(), - [[b'a', b'b'], [], [b'c', b'd', b'e'], [b'f'], [b'g'], [], []]) + self.assertEqual( + self.evaluate(rt).tolist(), + [[b'a', b'b'], [], [b'c', b'd', b'e'], [b'f'], [b'g'], [], []]) + @test_util.run_deprecated_v1 def testFromValueRowIdsWithExplicitNRowsEqualToDefault(self): values = constant_op.constant(['a', 'b', 'c', 'd', 'e', 'f', 'g']) value_rowids = constant_op.constant([0, 0, 2, 2, 2, 3, 4], dtypes.int64) @@ -330,12 +332,13 @@ class RaggedTensorTest(test_util.TensorFlowTestCase, parameterized.TestCase): self.assertIs(rt_values, values) self.assertIs(rt_value_rowids, value_rowids) # cached_value_rowids self.assertIs(rt_nrows, nrows) # cached_nrows - with self.test_session(): - self.assertAllEqual(rt_value_rowids, value_rowids) - self.assertAllEqual(rt_nrows, nrows) - self.assertEqual(rt.tolist(), - [[b'a', b'b'], [], [b'c', b'd', b'e'], [b'f'], [b'g']]) + self.assertAllEqual(rt_value_rowids, value_rowids) + self.assertAllEqual(rt_nrows, nrows) + self.assertEqual( + self.evaluate(rt).tolist(), + [[b'a', b'b'], [], [b'c', b'd', b'e'], [b'f'], [b'g']]) + @test_util.run_deprecated_v1 def testFromValueRowIdsWithEmptyValues(self): rt = ragged.from_value_rowids([], []) rt_nrows = ragged.nrows(rt) @@ -344,10 +347,10 @@ class RaggedTensorTest(test_util.TensorFlowTestCase, parameterized.TestCase): self.assertEqual(rt.ragged_rank, 1) self.assertEqual(rt.values.shape.as_list(), [0]) self.assertEqual(ragged.value_rowids(rt).shape.as_list(), [0]) - with self.test_session(): - self.assertEqual(rt_nrows.eval().tolist(), 0) - self.assertEqual(rt.tolist(), []) + self.assertEqual(self.evaluate(rt_nrows).tolist(), 0) + self.assertEqual(self.evaluate(rt).tolist(), []) + @test_util.run_deprecated_v1 def testFromRowSplits(self): values = constant_op.constant(['a', 'b', 'c', 'd', 'e', 'f', 'g']) row_splits = constant_op.constant([0, 2, 2, 5, 6, 7], dtypes.int64) @@ -363,16 +366,17 @@ class RaggedTensorTest(test_util.TensorFlowTestCase, parameterized.TestCase): self.assertIs(rt_values, values) self.assertIs(rt_row_splits, row_splits) - with self.test_session(): - self.assertEqual(rt_nrows.eval(), 5) - self.assertEqual(rt.tolist(), - [[b'a', b'b'], [], [b'c', b'd', b'e'], [b'f'], [b'g']]) + self.assertEqual(self.evaluate(rt_nrows), 5) + self.assertEqual( + self.evaluate(rt).tolist(), + [[b'a', b'b'], [], [b'c', b'd', b'e'], [b'f'], [b'g']]) def testFromRowSplitsWithEmptySplits(self): err_msg = 'row_splits tensor may not be empty' with self.assertRaisesRegexp(ValueError, err_msg): ragged.from_row_splits([], []) + @test_util.run_deprecated_v1 def testFromRowStarts(self): values = constant_op.constant(['a', 'b', 'c', 'd', 'e', 'f', 'g']) row_starts = constant_op.constant([0, 2, 2, 5, 6], dtypes.int64) @@ -387,12 +391,13 @@ class RaggedTensorTest(test_util.TensorFlowTestCase, parameterized.TestCase): rt_nrows = ragged.nrows(rt) self.assertIs(rt_values, values) - with self.test_session(): - self.assertEqual(rt_nrows.eval(), 5) - self.assertAllEqual(rt_row_starts, row_starts) - self.assertEqual(rt.tolist(), - [[b'a', b'b'], [], [b'c', b'd', b'e'], [b'f'], [b'g']]) + self.assertEqual(self.evaluate(rt_nrows), 5) + self.assertAllEqual(rt_row_starts, row_starts) + self.assertEqual( + self.evaluate(rt).tolist(), + [[b'a', b'b'], [], [b'c', b'd', b'e'], [b'f'], [b'g']]) + @test_util.run_deprecated_v1 def testFromRowLimits(self): values = constant_op.constant(['a', 'b', 'c', 'd', 'e', 'f', 'g']) row_limits = constant_op.constant([2, 2, 5, 6, 7], dtypes.int64) @@ -407,12 +412,13 @@ class RaggedTensorTest(test_util.TensorFlowTestCase, parameterized.TestCase): rt_nrows = ragged.nrows(rt) self.assertIs(rt_values, values) - with self.test_session(): - self.assertEqual(rt_nrows.eval(), 5) - self.assertAllEqual(rt_row_limits, row_limits) - self.assertEqual(rt.tolist(), - [[b'a', b'b'], [], [b'c', b'd', b'e'], [b'f'], [b'g']]) + self.assertEqual(self.evaluate(rt_nrows), 5) + self.assertAllEqual(rt_row_limits, row_limits) + self.assertEqual( + self.evaluate(rt).tolist(), + [[b'a', b'b'], [], [b'c', b'd', b'e'], [b'f'], [b'g']]) + @test_util.run_deprecated_v1 def testFromRowLengths(self): values = constant_op.constant(['a', 'b', 'c', 'd', 'e', 'f', 'g']) row_lengths = constant_op.constant([2, 0, 3, 1, 1], dtypes.int64) @@ -428,12 +434,13 @@ class RaggedTensorTest(test_util.TensorFlowTestCase, parameterized.TestCase): self.assertIs(rt_values, values) self.assertIs(rt_row_lengths, row_lengths) # cached_nrows - with self.test_session(): - self.assertEqual(rt_nrows.eval(), 5) - self.assertAllEqual(rt_row_lengths, row_lengths) - self.assertEqual(rt.tolist(), - [[b'a', b'b'], [], [b'c', b'd', b'e'], [b'f'], [b'g']]) + self.assertEqual(self.evaluate(rt_nrows), 5) + self.assertAllEqual(rt_row_lengths, row_lengths) + self.assertEqual( + self.evaluate(rt).tolist(), + [[b'a', b'b'], [], [b'c', b'd', b'e'], [b'f'], [b'g']]) + @test_util.run_deprecated_v1 def testFromNestedValueRowIdsWithDerivedNRows(self): values = constant_op.constant(['a', 'b', 'c', 'd', 'e', 'f', 'g']) nested_value_rowids = [ @@ -452,13 +459,13 @@ class RaggedTensorTest(test_util.TensorFlowTestCase, parameterized.TestCase): rt_values_value_rowids = ragged.value_rowids(rt_values) self.assertIs(rt_values_values, values) - with self.test_session(): - self.assertAllEqual(rt_value_rowids, nested_value_rowids[0]) - self.assertAllEqual(rt_values_value_rowids, nested_value_rowids[1]) - self.assertEqual( - rt.tolist(), - [[[b'a', b'b'], []], [[b'c', b'd', b'e']], [], [[b'f'], [b'g']]]) + self.assertAllEqual(rt_value_rowids, nested_value_rowids[0]) + self.assertAllEqual(rt_values_value_rowids, nested_value_rowids[1]) + self.assertEqual( + self.evaluate(rt).tolist(), + [[[b'a', b'b'], []], [[b'c', b'd', b'e']], [], [[b'f'], [b'g']]]) + @test_util.run_deprecated_v1 def testFromNestedValueRowIdsWithExplicitNRows(self): values = constant_op.constant(['a', 'b', 'c', 'd', 'e', 'f', 'g']) nested_value_rowids = [ @@ -483,14 +490,14 @@ class RaggedTensorTest(test_util.TensorFlowTestCase, parameterized.TestCase): rt_values_nrows = ragged.nrows(rt_values) self.assertIs(rt_values_values, values) - with self.test_session(): - self.assertAllEqual(rt_value_rowids, nested_value_rowids[0]) - self.assertAllEqual(rt_values_value_rowids, nested_value_rowids[1]) - self.assertAllEqual(rt_nrows, nrows[0]) - self.assertAllEqual(rt_values_nrows, nrows[1]) - self.assertEqual(rt.tolist(), - [[[b'a', b'b'], []], [[b'c', b'd', b'e']], [], - [[b'f'], [b'g'], []], [], []]) + self.assertAllEqual(rt_value_rowids, nested_value_rowids[0]) + self.assertAllEqual(rt_values_value_rowids, nested_value_rowids[1]) + self.assertAllEqual(rt_nrows, nrows[0]) + self.assertAllEqual(rt_values_nrows, nrows[1]) + self.assertEqual( + self.evaluate(rt).tolist(), + [[[b'a', b'b'], []], [[b'c', b'd', b'e']], [], [[b'f'], [b'g'], []], [], + []]) def testFromNestedValueRowIdsWithExplicitNRowsMismatch(self): values = constant_op.constant(['a', 'b', 'c', 'd', 'e', 'f', 'g']) @@ -515,6 +522,7 @@ class RaggedTensorTest(test_util.TensorFlowTestCase, parameterized.TestCase): ragged.from_nested_value_rowids([1, 2, 3], [[0, 1, 2], [0, 1, 2]], constant_op.constant([3, 3])) + @test_util.run_deprecated_v1 def testFromNestedRowSplits(self): inner_values = constant_op.constant(['a', 'b', 'c', 'd', 'e', 'f', 'g']) nested_row_splits = [ @@ -535,10 +543,9 @@ class RaggedTensorTest(test_util.TensorFlowTestCase, parameterized.TestCase): self.assertIs(rt_values_values, inner_values) self.assertIs(rt_row_splits, nested_row_splits[0]) self.assertIs(rt_values_row_splits, nested_row_splits[1]) - with self.test_session(): - self.assertEqual( - rt.tolist(), - [[[b'a', b'b'], []], [[b'c', b'd', b'e']], [], [[b'f'], [b'g']]]) + self.assertEqual( + self.evaluate(rt).tolist(), + [[[b'a', b'b'], []], [[b'c', b'd', b'e']], [], [[b'f'], [b'g']]]) def testFromNestedRowSplitsWithNonListInput(self): with self.assertRaisesRegexp(TypeError, @@ -583,6 +590,7 @@ class RaggedTensorTest(test_util.TensorFlowTestCase, parameterized.TestCase): value_rowids=value_rowids, nrows=array_ops.expand_dims(nrows, 0)) + @test_util.run_deprecated_v1 def testGraphMismatch(self): with ops.Graph().as_default(): values = constant_op.constant([1, 2, 3]) @@ -595,6 +603,7 @@ class RaggedTensorTest(test_util.TensorFlowTestCase, parameterized.TestCase): # Ragged Value & Row-Partitioning Tensor Accessors #============================================================================= + @test_util.run_deprecated_v1 def testRaggedTensorAccessors_2d(self): values = constant_op.constant(['a', 'b', 'c', 'd', 'e', 'f', 'g']) row_splits = constant_op.constant([0, 2, 2, 5, 6, 7], dtypes.int64) @@ -603,25 +612,33 @@ class RaggedTensorTest(test_util.TensorFlowTestCase, parameterized.TestCase): rt2 = ragged.from_value_rowids(values, value_rowids) for rt in [rt1, rt2]: - with self.test_session(): - self.assertEqual(rt.tolist(), - [[b'a', b'b'], [], [b'c', b'd', b'e'], [b'f'], [b'g']]) - self.assertEqual(rt.values.eval().tolist(), - [b'a', b'b', b'c', b'd', b'e', b'f', b'g']) - self.assertEqual(rt.values.shape.dims[0].value, 7) - self.assertEqual( - ragged.value_rowids(rt).eval().tolist(), [0, 0, 2, 2, 2, 3, 4]) - self.assertEqual(ragged.nrows(rt).eval().tolist(), 5) - self.assertEqual(rt.row_splits.eval().tolist(), [0, 2, 2, 5, 6, 7]) - self.assertEqual(ragged.row_starts(rt).eval().tolist(), [0, 2, 2, 5, 6]) - self.assertEqual(ragged.row_limits(rt).eval().tolist(), [2, 2, 5, 6, 7]) - self.assertEqual( - ragged.row_lengths(rt).eval().tolist(), [2, 0, 3, 1, 1]) - self.assertEqual(rt.inner_values.eval().tolist(), - [b'a', b'b', b'c', b'd', b'e', b'f', b'g']) - self.assertEqual([s.eval().tolist() for s in rt.nested_row_splits], - [[0, 2, 2, 5, 6, 7]]) + self.assertEqual( + self.evaluate(rt).tolist(), + [[b'a', b'b'], [], [b'c', b'd', b'e'], [b'f'], [b'g']]) + self.assertEqual( + self.evaluate(rt.values).tolist(), + [b'a', b'b', b'c', b'd', b'e', b'f', b'g']) + self.assertEqual(rt.values.shape.dims[0].value, 7) + self.assertEqual( + self.evaluate(ragged.value_rowids(rt)).tolist(), + [0, 0, 2, 2, 2, 3, 4]) + self.assertEqual(self.evaluate(ragged.nrows(rt)).tolist(), 5) + self.assertEqual( + self.evaluate(rt.row_splits).tolist(), [0, 2, 2, 5, 6, 7]) + self.assertEqual( + self.evaluate(ragged.row_starts(rt)).tolist(), [0, 2, 2, 5, 6]) + self.assertEqual( + self.evaluate(ragged.row_limits(rt)).tolist(), [2, 2, 5, 6, 7]) + self.assertEqual( + self.evaluate(ragged.row_lengths(rt)).tolist(), [2, 0, 3, 1, 1]) + self.assertEqual( + self.evaluate(rt.inner_values).tolist(), + [b'a', b'b', b'c', b'd', b'e', b'f', b'g']) + self.assertEqual( + [self.evaluate(s).tolist() for s in rt.nested_row_splits], + [[0, 2, 2, 5, 6, 7]]) + @test_util.run_deprecated_v1 def testRaggedTensorAccessors_3d_with_ragged_rank_1(self): values = [[0, 1], [2, 3], [4, 5], [6, 7], [8, 9], [10, 11], [12, 13]] row_splits = constant_op.constant([0, 2, 2, 5, 6, 7], dtypes.int64) @@ -630,28 +647,34 @@ class RaggedTensorTest(test_util.TensorFlowTestCase, parameterized.TestCase): rt2 = ragged.from_value_rowids(values, value_rowids) for rt in [rt1, rt2]: - with self.test_session(): - self.assertEqual(rt.tolist(), - [[[0, 1], [2, 3]], [], [[4, 5], [6, 7], [8, 9]], - [[10, 11]], [[12, 13]]]) - self.assertEqual( - rt.values.eval().tolist(), - [[0, 1], [2, 3], [4, 5], [6, 7], [8, 9], [10, 11], [12, 13]]) - self.assertEqual(rt.values.shape.dims[0].value, 7) - self.assertEqual( - ragged.value_rowids(rt).eval().tolist(), [0, 0, 2, 2, 2, 3, 4]) - self.assertEqual(ragged.nrows(rt).eval().tolist(), 5) - self.assertEqual(rt.row_splits.eval().tolist(), [0, 2, 2, 5, 6, 7]) - self.assertEqual(ragged.row_starts(rt).eval().tolist(), [0, 2, 2, 5, 6]) - self.assertEqual(ragged.row_limits(rt).eval().tolist(), [2, 2, 5, 6, 7]) - self.assertEqual( - ragged.row_lengths(rt).eval().tolist(), [2, 0, 3, 1, 1]) - self.assertEqual( - rt.inner_values.eval().tolist(), - [[0, 1], [2, 3], [4, 5], [6, 7], [8, 9], [10, 11], [12, 13]]) - self.assertEqual([s.eval().tolist() for s in rt.nested_row_splits], - [[0, 2, 2, 5, 6, 7]]) + self.assertEqual( + self.evaluate(rt).tolist(), + [[[0, 1], [2, 3]], [], [[4, 5], [6, 7], [8, 9]], [[10, 11]], + [[12, 13]]]) + self.assertEqual( + self.evaluate(rt.values).tolist(), + [[0, 1], [2, 3], [4, 5], [6, 7], [8, 9], [10, 11], [12, 13]]) + self.assertEqual(rt.values.shape.dims[0].value, 7) + self.assertEqual( + self.evaluate(ragged.value_rowids(rt)).tolist(), + [0, 0, 2, 2, 2, 3, 4]) + self.assertEqual(self.evaluate(ragged.nrows(rt)).tolist(), 5) + self.assertEqual( + self.evaluate(rt.row_splits).tolist(), [0, 2, 2, 5, 6, 7]) + self.assertEqual( + self.evaluate(ragged.row_starts(rt)).tolist(), [0, 2, 2, 5, 6]) + self.assertEqual( + self.evaluate(ragged.row_limits(rt)).tolist(), [2, 2, 5, 6, 7]) + self.assertEqual( + self.evaluate(ragged.row_lengths(rt)).tolist(), [2, 0, 3, 1, 1]) + self.assertEqual( + self.evaluate(rt.inner_values).tolist(), + [[0, 1], [2, 3], [4, 5], [6, 7], [8, 9], [10, 11], [12, 13]]) + self.assertEqual( + [self.evaluate(s).tolist() for s in rt.nested_row_splits], + [[0, 2, 2, 5, 6, 7]]) + @test_util.run_deprecated_v1 def testRaggedTensorAccessors_3d_with_ragged_rank_2(self): values = constant_op.constant(['a', 'b', 'c', 'd', 'e', 'f', 'g']) nested_row_splits = [ @@ -666,41 +689,45 @@ class RaggedTensorTest(test_util.TensorFlowTestCase, parameterized.TestCase): rt2 = ragged.from_nested_value_rowids(values, nested_value_rowids) for rt in [rt1, rt2]: - with self.test_session(): - self.assertEqual( - rt.tolist(), - [[[b'a', b'b'], []], [[b'c', b'd', b'e']], [], [[b'f'], [b'g']]]) - self.assertEqual(rt.values.eval().tolist(), - [[b'a', b'b'], [], [b'c', b'd', b'e'], [b'f'], [b'g']]) - self.assertEqual(rt.values.shape.dims[0].value, 5) - self.assertEqual( - ragged.value_rowids(rt).eval().tolist(), [0, 0, 1, 3, 3]) - self.assertEqual(ragged.nrows(rt).eval().tolist(), 4) - self.assertEqual(rt.row_splits.eval().tolist(), [0, 2, 3, 3, 5]) - self.assertEqual(ragged.row_starts(rt).eval().tolist(), [0, 2, 3, 3]) - self.assertEqual(ragged.row_limits(rt).eval().tolist(), [2, 3, 3, 5]) - self.assertEqual(ragged.row_lengths(rt).eval().tolist(), [2, 1, 0, 2]) - self.assertEqual(rt.inner_values.eval().tolist(), - [b'a', b'b', b'c', b'd', b'e', b'f', b'g']) - self.assertEqual([s.eval().tolist() for s in rt.nested_row_splits], - [[0, 2, 3, 3, 5], [0, 2, 2, 5, 6, 7]]) + self.assertEqual( + self.evaluate(rt).tolist(), + [[[b'a', b'b'], []], [[b'c', b'd', b'e']], [], [[b'f'], [b'g']]]) + self.assertEqual( + self.evaluate(rt.values).tolist(), + [[b'a', b'b'], [], [b'c', b'd', b'e'], [b'f'], [b'g']]) + self.assertEqual(rt.values.shape.dims[0].value, 5) + self.assertEqual( + self.evaluate(ragged.value_rowids(rt)).tolist(), [0, 0, 1, 3, 3]) + self.assertEqual(self.evaluate(ragged.nrows(rt)).tolist(), 4) + self.assertEqual(self.evaluate(rt.row_splits).tolist(), [0, 2, 3, 3, 5]) + self.assertEqual( + self.evaluate(ragged.row_starts(rt)).tolist(), [0, 2, 3, 3]) + self.assertEqual( + self.evaluate(ragged.row_limits(rt)).tolist(), [2, 3, 3, 5]) + self.assertEqual( + self.evaluate(ragged.row_lengths(rt)).tolist(), [2, 1, 0, 2]) + self.assertEqual( + self.evaluate(rt.inner_values).tolist(), + [b'a', b'b', b'c', b'd', b'e', b'f', b'g']) + self.assertEqual( + [self.evaluate(s).tolist() for s in rt.nested_row_splits], + [[0, 2, 3, 3, 5], [0, 2, 2, 5, 6, 7]]) def testNRowsWithTensorInput(self): dt = constant_op.constant([[1, 2, 3], [4, 5, 6]]) nrows = ragged.nrows(dt) - with self.test_session(): - self.assertEqual(nrows.eval(), 2) + self.assertEqual(self.evaluate(nrows), 2) def testRowLengthsWithTensorInput(self): dt = constant_op.constant([[1, 2, 3], [4, 5, 6]]) row_lengths = ragged.row_lengths(dt) - with self.test_session(): - self.assertEqual(row_lengths.eval().tolist(), [3, 3]) + self.assertEqual(self.evaluate(row_lengths).tolist(), [3, 3]) #============================================================================= # RaggedTensor.shape #============================================================================= + @test_util.run_deprecated_v1 def testShape(self): """Tests for RaggedTensor.shape.""" rt1 = ragged.from_row_splits(b'a b c d e f g'.split(), [0, 2, 5, 6, 6, 7]) @@ -748,29 +775,27 @@ class RaggedTensorTest(test_util.TensorFlowTestCase, parameterized.TestCase): expected: The expected value of rt.__getitem__(slice_spec), as a python list; or an exception class. """ - with self.test_session(): - tensor_slice_spec1 = _make_tensor_slice_spec(slice_spec, True) - tensor_slice_spec2 = _make_tensor_slice_spec(slice_spec, False) - value1 = rt.__getitem__(slice_spec).eval() - value2 = rt.__getitem__(tensor_slice_spec1).eval() - value3 = rt.__getitem__(tensor_slice_spec2).eval() - if hasattr(value1, 'tolist'): - value1 = value1.tolist() - if hasattr(value2, 'tolist'): - value2 = value2.tolist() - if hasattr(value3, 'tolist'): - value3 = value3.tolist() - self.assertEqual(value1, expected, 'slice_spec=%s' % (slice_spec,)) - self.assertEqual(value2, expected, 'slice_spec=%s' % (slice_spec,)) - self.assertEqual(value3, expected, 'slice_spec=%s' % (slice_spec,)) + tensor_slice_spec1 = _make_tensor_slice_spec(slice_spec, True) + tensor_slice_spec2 = _make_tensor_slice_spec(slice_spec, False) + value1 = self.evaluate(rt.__getitem__(slice_spec)) + value2 = self.evaluate(rt.__getitem__(tensor_slice_spec1)) + value3 = self.evaluate(rt.__getitem__(tensor_slice_spec2)) + if hasattr(value1, 'tolist'): + value1 = value1.tolist() + if hasattr(value2, 'tolist'): + value2 = value2.tolist() + if hasattr(value3, 'tolist'): + value3 = value3.tolist() + self.assertEqual(value1, expected, 'slice_spec=%s' % (slice_spec,)) + self.assertEqual(value2, expected, 'slice_spec=%s' % (slice_spec,)) + self.assertEqual(value3, expected, 'slice_spec=%s' % (slice_spec,)) def _TestGetItemException(self, rt, slice_spec, expected, message): """Helper function for testing RaggedTensor.__getitem__ exceptions.""" - with self.test_session(): - tensor_slice_spec1 = _make_tensor_slice_spec(slice_spec, True) - self.assertRaisesRegexp(expected, message, rt.__getitem__, slice_spec) - self.assertRaisesRegexp(expected, message, rt.__getitem__, - tensor_slice_spec1) + tensor_slice_spec1 = _make_tensor_slice_spec(slice_spec, True) + self.assertRaisesRegexp(expected, message, rt.__getitem__, slice_spec) + self.assertRaisesRegexp(expected, message, rt.__getitem__, + tensor_slice_spec1) @parameterized.parameters( # Tests for rt[i] @@ -836,14 +861,14 @@ class RaggedTensorTest(test_util.TensorFlowTestCase, parameterized.TestCase): (SLICE_BUILDER[:, -2:], [row[-2:] for row in EXAMPLE_RAGGED_TENSOR_2D]), # TODO(edloper): Add tests for strided slices, once support is added. ) + @test_util.run_deprecated_v1 def testRaggedTensorGetItemWithRaggedRank1(self, slice_spec, expected): """Test that rt.__getitem__(slice_spec) == expected.""" # Ragged tensor rt = ragged.from_row_splits(EXAMPLE_RAGGED_TENSOR_2D_VALUES, EXAMPLE_RAGGED_TENSOR_2D_SPLITS) - with self.test_session(): - self.assertEqual(rt.tolist(), EXAMPLE_RAGGED_TENSOR_2D) + self.assertEqual(self.evaluate(rt).tolist(), EXAMPLE_RAGGED_TENSOR_2D) self._TestGetItem(rt, slice_spec, expected) # pylint: disable=invalid-slice-index @@ -878,6 +903,7 @@ class RaggedTensorTest(test_util.TensorFlowTestCase, parameterized.TestCase): (SLICE_BUILDER[..., 0, 0, 0], IndexError, 'Too many indices for RaggedTensor'), ) + @test_util.run_deprecated_v1 def testRaggedTensorGetItemErrorsWithRaggedRank1(self, slice_spec, expected, message): """Test that rt.__getitem__(slice_spec) == expected.""" @@ -887,8 +913,7 @@ class RaggedTensorTest(test_util.TensorFlowTestCase, parameterized.TestCase): # if sys.version_info[0] == 3: # message = 'must be str, not int' - with self.test_session(): - self.assertEqual(rt.tolist(), EXAMPLE_RAGGED_TENSOR_2D) + self.assertEqual(self.evaluate(rt).tolist(), EXAMPLE_RAGGED_TENSOR_2D) self._TestGetItemException(rt, slice_spec, expected, message) @parameterized.parameters( @@ -957,13 +982,13 @@ class RaggedTensorTest(test_util.TensorFlowTestCase, parameterized.TestCase): # TODO(edloper): Add tests slicing inner ragged dimensions, one support # is added. ) + @test_util.run_deprecated_v1 def testRaggedTensorGetItemWithRaggedRank2(self, slice_spec, expected): """Test that rt.__getitem__(slice_spec) == expected.""" rt = ragged.from_nested_row_splits( EXAMPLE_RAGGED_TENSOR_4D_VALUES, [EXAMPLE_RAGGED_TENSOR_4D_SPLITS1, EXAMPLE_RAGGED_TENSOR_4D_SPLITS2]) - with self.test_session(): - self.assertEqual(rt.tolist(), EXAMPLE_RAGGED_TENSOR_4D) + self.assertEqual(self.evaluate(rt).tolist(), EXAMPLE_RAGGED_TENSOR_4D) self._TestGetItem(rt, slice_spec, expected) @parameterized.parameters( @@ -979,14 +1004,14 @@ class RaggedTensorTest(test_util.TensorFlowTestCase, parameterized.TestCase): (SLICE_BUILDER[5], ValueError, '.*out of bounds.*'), (SLICE_BUILDER[0, 5], ValueError, '.*out of bounds.*'), ) + @test_util.run_deprecated_v1 def testRaggedTensorGetItemErrorsWithRaggedRank2(self, slice_spec, expected, message): """Test that rt.__getitem__(slice_spec) == expected.""" rt = ragged.from_nested_row_splits( EXAMPLE_RAGGED_TENSOR_4D_VALUES, [EXAMPLE_RAGGED_TENSOR_4D_SPLITS1, EXAMPLE_RAGGED_TENSOR_4D_SPLITS2]) - with self.test_session(): - self.assertEqual(rt.tolist(), EXAMPLE_RAGGED_TENSOR_4D) + self.assertEqual(self.evaluate(rt).tolist(), EXAMPLE_RAGGED_TENSOR_4D) self._TestGetItemException(rt, slice_spec, expected, message) @parameterized.parameters( @@ -994,6 +1019,7 @@ class RaggedTensorTest(test_util.TensorFlowTestCase, parameterized.TestCase): (SLICE_BUILDER[2:], []), (SLICE_BUILDER[:-3], []), ) + @test_util.run_deprecated_v1 def testRaggedTensorGetItemWithEmptyTensor(self, slice_spec, expected): """Test that rt.__getitem__(slice_spec) == expected.""" rt = ragged.from_row_splits([], [0]) @@ -1003,6 +1029,7 @@ class RaggedTensorTest(test_util.TensorFlowTestCase, parameterized.TestCase): (SLICE_BUILDER[0], ValueError, '.*out of bounds.*'), (SLICE_BUILDER[-1], ValueError, '.*out of bounds.*'), ) + @test_util.run_deprecated_v1 def testRaggedTensorGetItemErrorsWithEmptyTensor(self, slice_spec, expected, message): """Test that rt.__getitem__(slice_spec) == expected.""" @@ -1018,6 +1045,7 @@ class RaggedTensorTest(test_util.TensorFlowTestCase, parameterized.TestCase): (SLICE_BUILDER[0, 1], EXAMPLE_RAGGED_TENSOR_2D[0][1]), (SLICE_BUILDER[-3, 0], EXAMPLE_RAGGED_TENSOR_2D[-3][0]), ) + @test_util.run_deprecated_v1 def testRaggedTensorGetItemWithPlaceholderShapes(self, slice_spec, expected): """Test that rt.__getitem__(slice_spec) == expected.""" # Intentionally use an unknown shape for `splits`, to force the code path @@ -1026,13 +1054,13 @@ class RaggedTensorTest(test_util.TensorFlowTestCase, parameterized.TestCase): EXAMPLE_RAGGED_TENSOR_2D_SPLITS, dtype=dtypes.int64) splits = array_ops.placeholder_with_default(splits, None) rt = ragged.from_row_splits(EXAMPLE_RAGGED_TENSOR_2D_VALUES, splits) - with self.test_session(): - self.assertEqual(rt.tolist(), EXAMPLE_RAGGED_TENSOR_2D) + self.assertEqual(self.evaluate(rt).tolist(), EXAMPLE_RAGGED_TENSOR_2D) self._TestGetItem(rt, slice_spec, expected) @parameterized.parameters( (SLICE_BUILDER[..., 2], ValueError, 'Ellipsis not supported for unknown shape RaggedTensors'),) + @test_util.run_deprecated_v1 def testRaggedTensorGetItemErrorsWithPlaceholderShapes( self, slice_spec, expected, message): """Test that rt.__getitem__(slice_spec) == expected.""" @@ -1041,55 +1069,55 @@ class RaggedTensorTest(test_util.TensorFlowTestCase, parameterized.TestCase): rt = ragged.from_row_splits(values, [0, 1]) self._TestGetItemException(rt, slice_spec, expected, message) - # TODO(edloper): Remove this decorator once c shapes become the default. - @test_util.enable_c_shapes + @test_util.run_deprecated_v1 def testGetItemNewAxis(self): # rt: [[[['a', 'b'], ['c', 'd']], [], [['e', 'f']]], []] splits1 = [0, 3, 3] splits2 = [0, 2, 2, 3] values = constant_op.constant([['a', 'b'], ['c', 'd'], ['e', 'f']]) rt = ragged.from_nested_row_splits(values, [splits1, splits2]) - with self.test_session(): - rt_newaxis0 = rt[array_ops.newaxis] - rt_newaxis1 = rt[:, array_ops.newaxis] - rt_newaxis2 = rt[:, :, array_ops.newaxis] - rt_newaxis3 = rt[:, :, :, array_ops.newaxis] - rt_newaxis4 = rt[:, :, :, :, array_ops.newaxis] + rt_newaxis0 = rt[array_ops.newaxis] + rt_newaxis1 = rt[:, array_ops.newaxis] + rt_newaxis2 = rt[:, :, array_ops.newaxis] + rt_newaxis3 = rt[:, :, :, array_ops.newaxis] + rt_newaxis4 = rt[:, :, :, :, array_ops.newaxis] - self.assertEqual(rt.tolist(), - [[[[b'a', b'b'], [b'c', b'd']], [], [[b'e', b'f']]], []]) - self.assertEqual( - rt_newaxis0.tolist(), - [[[[[b'a', b'b'], [b'c', b'd']], [], [[b'e', b'f']]], []]]) - self.assertEqual( - rt_newaxis1.tolist(), - [[[[[b'a', b'b'], [b'c', b'd']], [], [[b'e', b'f']]]], [[]]]) - self.assertEqual( - rt_newaxis2.tolist(), - [[[[[b'a', b'b'], [b'c', b'd']]], [[]], [[[b'e', b'f']]]], []]) - self.assertEqual( - rt_newaxis3.tolist(), - [[[[[b'a', b'b']], [[b'c', b'd']]], [], [[[b'e', b'f']]]], []]) - self.assertEqual( - rt_newaxis4.tolist(), - [[[[[b'a'], [b'b']], [[b'c'], [b'd']]], [], [[[b'e'], [b'f']]]], []]) + self.assertEqual( + self.evaluate(rt).tolist(), + [[[[b'a', b'b'], [b'c', b'd']], [], [[b'e', b'f']]], []]) + self.assertEqual( + self.evaluate(rt_newaxis0).tolist(), + [[[[[b'a', b'b'], [b'c', b'd']], [], [[b'e', b'f']]], []]]) + self.assertEqual( + self.evaluate(rt_newaxis1).tolist(), + [[[[[b'a', b'b'], [b'c', b'd']], [], [[b'e', b'f']]]], [[]]]) + self.assertEqual( + self.evaluate(rt_newaxis2).tolist(), + [[[[[b'a', b'b'], [b'c', b'd']]], [[]], [[[b'e', b'f']]]], []]) + self.assertEqual( + self.evaluate(rt_newaxis3).tolist(), + [[[[[b'a', b'b']], [[b'c', b'd']]], [], [[[b'e', b'f']]]], []]) + self.assertEqual( + self.evaluate(rt_newaxis4).tolist(), + [[[[[b'a'], [b'b']], [[b'c'], [b'd']]], [], [[[b'e'], [b'f']]]], []]) - self.assertEqual(rt.ragged_rank, 2) - self.assertEqual(rt_newaxis0.ragged_rank, 3) - self.assertEqual(rt_newaxis1.ragged_rank, 3) - self.assertEqual(rt_newaxis2.ragged_rank, 3) - self.assertEqual(rt_newaxis3.ragged_rank, 2) - self.assertEqual(rt_newaxis4.ragged_rank, 2) + self.assertEqual(rt.ragged_rank, 2) + self.assertEqual(rt_newaxis0.ragged_rank, 3) + self.assertEqual(rt_newaxis1.ragged_rank, 3) + self.assertEqual(rt_newaxis2.ragged_rank, 3) + self.assertEqual(rt_newaxis3.ragged_rank, 2) + self.assertEqual(rt_newaxis4.ragged_rank, 2) - self.assertEqual(rt_newaxis0.shape.as_list(), [1, None, None, None, 2]) - self.assertEqual(rt_newaxis1.shape.as_list(), [2, None, None, None, 2]) - self.assertEqual(rt_newaxis2.shape.as_list(), [2, None, None, None, 2]) - self.assertEqual(rt_newaxis3.shape.as_list(), [2, None, None, 1, 2]) - self.assertEqual(rt_newaxis4.shape.as_list(), [2, None, None, 2, 1]) + self.assertEqual(rt_newaxis0.shape.as_list(), [1, None, None, None, 2]) + self.assertEqual(rt_newaxis1.shape.as_list(), [2, None, None, None, 2]) + self.assertEqual(rt_newaxis2.shape.as_list(), [2, None, None, None, 2]) + self.assertEqual(rt_newaxis3.shape.as_list(), [2, None, None, 1, 2]) + self.assertEqual(rt_newaxis4.shape.as_list(), [2, None, None, 2, 1]) #============================================================================= # RaggedTensor.__str__ #============================================================================= + @test_util.run_deprecated_v1 def testRaggedTensorStr(self): rt1 = ragged.from_row_splits(b'a b c d e f g'.split(), [0, 2, 5, 6, 6, 7]) expected1 = ('RaggedTensor(values=Tensor("RaggedFromRowSplits/values:0", ' @@ -1127,6 +1155,7 @@ class RaggedTensorTest(test_util.TensorFlowTestCase, parameterized.TestCase): # RaggedTensor.with_values() and RaggedTensor.with_inner_values(). #============================================================================= + @test_util.run_deprecated_v1 def testWithValues(self): rt1 = ragged.constant([[1, 2], [3, 4, 5], [6], [], [7]]) rt2 = ragged.constant([[[1, 2], [3, 4, 5]], [[6]], [], [[], [7]]]) @@ -1135,17 +1164,20 @@ class RaggedTensorTest(test_util.TensorFlowTestCase, parameterized.TestCase): rt2_times_10 = rt2.with_inner_values(rt2.inner_values * 10) rt1_expanded = rt1.with_values(array_ops.expand_dims(rt1.values, axis=1)) - with self.test_session(): - self.assertEqual(rt1_plus_10.tolist(), - [[11, 12], [13, 14, 15], [16], [], [17]]) - self.assertEqual(rt2_times_10.tolist(), - [[[10, 20], [30, 40, 50]], [[60]], [], [[], [70]]]) - self.assertEqual(rt1_expanded.tolist(), - [[[1], [2]], [[3], [4], [5]], [[6]], [], [[7]]]) + self.assertEqual( + self.evaluate(rt1_plus_10).tolist(), + [[11, 12], [13, 14, 15], [16], [], [17]]) + self.assertEqual( + self.evaluate(rt2_times_10).tolist(), + [[[10, 20], [30, 40, 50]], [[60]], [], [[], [70]]]) + self.assertEqual( + self.evaluate(rt1_expanded).tolist(), + [[[1], [2]], [[3], [4], [5]], [[6]], [], [[7]]]) #============================================================================= # Session.run #============================================================================= + @test_util.run_deprecated_v1 def testSessionRun(self): rt1 = ragged.constant([[1, 2, 3], [4]]) rt2 = ragged.constant([[[], [1, 2]], [[3]]]) @@ -1155,6 +1187,7 @@ class RaggedTensorTest(test_util.TensorFlowTestCase, parameterized.TestCase): self.assertEqual(result['rt1'].tolist(), [[1, 2, 3], [4]]) self.assertEqual(result['rt2'].tolist(), [[[], [1, 2]], [[3]]]) + @test_util.run_deprecated_v1 def testSessionRunFeed(self): rt1 = ragged.from_row_splits( array_ops.placeholder(dtypes.int32), @@ -1175,6 +1208,7 @@ class RaggedTensorTest(test_util.TensorFlowTestCase, parameterized.TestCase): self.assertEqual(result['rt1'].tolist(), [[1, 2, 3], [4]]) self.assertEqual(result['rt2'].tolist(), [[[], [1, 2]], [[3]]]) + @test_util.run_deprecated_v1 def testSessionPartialRunFeed(self): # Placeholder inputs. a = ragged.from_row_splits( diff --git a/tensorflow/python/ops/ragged/ragged_tile_op_test.py b/tensorflow/python/ops/ragged/ragged_tile_op_test.py index bf62d96e7a9..f335b15dd15 100644 --- a/tensorflow/python/ops/ragged/ragged_tile_op_test.py +++ b/tensorflow/python/ops/ragged/ragged_tile_op_test.py @@ -170,8 +170,18 @@ class RaggedTileOpTest(test_util.TensorFlowTestCase, parameterized.TestCase): rt_input=[[[[1], [2]], [[3]]], [[]], [[[4, 5]]]], multiples=[1, 1, 1, 0], expected=[[[[], []], [[]]], [[]], [[[]]]]), + #========================================================================= + # multiple=1 + #========================================================================= + dict( + descr='rank=4, multiples=1 (no repeats)', + rt_input=[[[[1], [2]], [[3], [4]]], [[[5], [6]]]], + multiples=[1, 1, 1, 1], + expected=[[[[1], [2]], [[3], [4]]], + [[[5], [6]]]]), ]) # pyformat: disable + @test_util.run_deprecated_v1 def testRaggedTile(self, descr, rt_input, @@ -200,6 +210,7 @@ class RaggedTileOpTest(test_util.TensorFlowTestCase, parameterized.TestCase): with self.test_session(): self.assertEqual(tiled.eval().tolist(), expected) + @test_util.run_deprecated_v1 def testRaggedTileWithTensorInput(self): # When the input is a `Tensor`, ragged_tile just delegates to tf.tile. dt = constant_op.constant([[1, 2], [3, 4]]) diff --git a/tensorflow/python/ops/ragged/ragged_to_sparse_op_test.py b/tensorflow/python/ops/ragged/ragged_to_sparse_op_test.py index 2fd31837c62..69b31ad0e97 100644 --- a/tensorflow/python/ops/ragged/ragged_to_sparse_op_test.py +++ b/tensorflow/python/ops/ragged/ragged_to_sparse_op_test.py @@ -23,12 +23,14 @@ from tensorflow.python.framework import errors from tensorflow.python.framework import test_util from tensorflow.python.ops import array_ops from tensorflow.python.ops import gradients_impl +from tensorflow.python.ops import math_ops from tensorflow.python.ops import ragged from tensorflow.python.platform import googletest class RaggedTensorToSparseOpTest(test_util.TensorFlowTestCase): + @test_util.run_deprecated_v1 def testDocStringExample(self): rt = ragged.constant([[1, 2, 3], [4], [], [5, 6]]) st = ragged.to_sparse(rt) @@ -39,6 +41,7 @@ class RaggedTensorToSparseOpTest(test_util.TensorFlowTestCase): with self.test_session(): self.assertEqual(' '.join(repr(st.eval()).split()), expected) + @test_util.run_deprecated_v1 def test2DRaggedTensorWithOneRaggedDimension(self): rt = ragged.constant([['a', 'b'], ['c', 'd', 'e'], ['f'], [], ['g']]) with self.test_session(): @@ -48,6 +51,7 @@ class RaggedTensorToSparseOpTest(test_util.TensorFlowTestCase): self.assertAllEqual(st.values, b'a b c d e f g'.split()) self.assertAllEqual(st.dense_shape, [5, 3]) + @test_util.run_deprecated_v1 def test3DRaggedTensorWithOneRaggedDimension(self): rt = ragged.constant([[[1, 2], [3, 4]], [[5, 6], [7, 8], [9, 10]], [[11, 12]], [], [[13, 14]]], @@ -62,6 +66,7 @@ class RaggedTensorToSparseOpTest(test_util.TensorFlowTestCase): [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14]) self.assertAllEqual(st.dense_shape, [5, 3, 2]) + @test_util.run_deprecated_v1 def test4DRaggedTensorWithOneRaggedDimension(self): rt = ragged.constant( [[[[1, 2], [3, 4]], [[5, 6], [7, 8]]], [], [[[9, 10], [11, 12]]]], @@ -87,6 +92,7 @@ class RaggedTensorToSparseOpTest(test_util.TensorFlowTestCase): ]) self.assertAllEqual(st.dense_shape, [3, 2, 2, 2]) + @test_util.run_deprecated_v1 def test4DRaggedTensorWithTwoRaggedDimensions(self): rt = ragged.constant([[[[1, 2], [3, 4]], [[5, 6], [7, 8], [9, 10]]], [[[11, 12]], [], [[13, 14]]], []], @@ -134,6 +140,7 @@ class RaggedTensorToSparseOpTest(test_util.TensorFlowTestCase): self.assertEqual(st.values.shape.as_list(), [7]) self.assertEqual(st.dense_shape.shape.as_list(), [3]) + @test_util.run_deprecated_v1 def testKernelErrors(self): # An empty vector, defined using a placeholder to ensure that we can't # determine that it's invalid at graph-construction time. @@ -172,13 +179,14 @@ class RaggedTensorToSparseOpTest(test_util.TensorFlowTestCase): self.assertRaisesRegexp(errors.InvalidArgumentError, empty_splits_error, ragged.to_sparse(bad_rt5).eval) + @test_util.run_deprecated_v1 def testGradient(self): # rt1.shape == rt2.shape == [2, (D2), (D3), 2]. rt1 = ragged.constant([[[[1.0, 2.0], [3.0, 4.0]], [[5.0, 6.0]]]], ragged_rank=2) rt2 = ragged.constant([[[[9.0, 8.0], [7.0, 6.0]], [[5.0, 4.0]]]], ragged_rank=2) - rt = rt1 + rt2 * 2.0 + rt = ragged.map_inner_values(math_ops.add, rt1, rt2 * 2.0) st = ragged.to_sparse(rt) g1, g2 = gradients_impl.gradients(st.values, [rt1.inner_values, diff --git a/tensorflow/python/ops/ragged/ragged_to_tensor_op_test.py b/tensorflow/python/ops/ragged/ragged_to_tensor_op_test.py index 0ccc214a9c7..77499b9cb3c 100644 --- a/tensorflow/python/ops/ragged/ragged_to_tensor_op_test.py +++ b/tensorflow/python/ops/ragged/ragged_to_tensor_op_test.py @@ -30,6 +30,7 @@ from tensorflow.python.platform import googletest class RaggedTensorToTensorOpTest(test_util.TensorFlowTestCase, parameterized.TestCase): + @test_util.run_deprecated_v1 def testDocStringExamples(self): """Example from ragged_to_tensor.__doc__.""" rt = ragged.constant([[9, 8, 7], [], [6, 5], [4]]) @@ -71,10 +72,33 @@ class RaggedTensorToTensorOpTest(test_util.TensorFlowTestCase, [[1, 2], [0, 0], [3, 4]], # [[0, 0], [0, 0], [0, 0]], # [[5, 0], [0, 0], [0, 0]], # - [[6, 7], [8, 0], [0, 0]] - ] # + [[6, 7], [8, 0], [0, 0]], # + ] + }, + { + 'rt_input': [[[1, 2], [], [3, 4]], [], [[5]], [[6, 7], [8]]], + 'default': + 9, + 'expected': [ + [[1, 2], [9, 9], [3, 4]], # + [[9, 9], [9, 9], [9, 9]], # + [[5, 9], [9, 9], [9, 9]], # + [[6, 7], [8, 9], [9, 9]], # + ] + }, + { + 'rt_input': [[[1], [2], [3]]], + 'ragged_rank': 1, + 'default': 0, + 'expected': [[[1], [2], [3]]], + }, + { + 'rt_input': [[[[1], [2]], [], [[3]]]], + 'default': 9, + 'expected': [[[[1], [2]], [[9], [9]], [[3], [9]]]], }, ) + @test_util.run_deprecated_v1 def testRaggedTensorToTensor(self, rt_input, expected, @@ -96,17 +120,13 @@ class RaggedTensorToTensorOpTest(test_util.TensorFlowTestCase, { 'rt_input': [[1, 2, 3]], 'default': [0], - 'error': (ValueError, r'Shapes \(1,\) and \(\) are incompatible'), + 'error': (ValueError, r'Shape \(1,\) must have rank at most 0'), }, { - 'rt_input': [[[1], [2], [3]]], - 'default': 0, - 'error': (ValueError, r'Shapes \(\) and \(1,\) are incompatible'), - }, - { - 'rt_input': [[[[1], [2]], [], [[3]]]], - 'default': 0, - 'error': (ValueError, r'Shapes \(\) and \(1,\) are incompatible'), + 'rt_input': [[[1, 2], [3, 4]], [[5, 6]]], + 'ragged_rank': 1, + 'default': [7, 8, 9], + 'error': (ValueError, r'Shapes \(3,\) and \(2,\) are incompatible'), }, { 'rt_input': [[1, 2, 3]], @@ -114,9 +134,11 @@ class RaggedTensorToTensorOpTest(test_util.TensorFlowTestCase, 'error': (TypeError, "Expected int32, got 'a' of type 'str' instead"), }, ) - def testError(self, rt_input, default, error): - rt = ragged.constant(rt_input) - self.assertRaisesRegexp(error[0], error[1], ragged.to_tensor, rt, default) + @test_util.run_deprecated_v1 + def testError(self, rt_input, default, error, ragged_rank=None): + rt = ragged.constant(rt_input, ragged_rank=ragged_rank) + with self.assertRaisesRegexp(error[0], error[1]): + ragged.to_tensor(rt, default) if __name__ == '__main__': diff --git a/tensorflow/python/ops/ragged/ragged_util.py b/tensorflow/python/ops/ragged/ragged_util.py index 03f050de514..a832f937d16 100644 --- a/tensorflow/python/ops/ragged/ragged_util.py +++ b/tensorflow/python/ops/ragged/ragged_util.py @@ -25,6 +25,7 @@ from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops from tensorflow.python.ops import array_ops from tensorflow.python.ops import check_ops +from tensorflow.python.ops import gen_ragged_math_ops from tensorflow.python.ops import math_ops @@ -229,3 +230,51 @@ def _with_nonzero_rank(data): return array_ops.reshape( data, array_ops.concat([[1], data_shape], axis=0)[-data_ndims:]) + + +def lengths_to_splits(lengths): + """Returns splits corresponding to the given lengths.""" + return array_ops.concat([[0], math_ops.cumsum(lengths)], axis=-1) + + +def repeat_ranges(params, splits, repeats): + """Repeats each range of `params` (as specified by `splits`) `repeats` times. + + Let the `i`th range of `params` be defined as + `params[splits[i]:splits[i + 1]]`. Then this function returns a tensor + containing range 0 repeated `repeats[0]` times, followed by range 1 repeated + `repeats[1]`, ..., followed by the last range repeated `repeats[-1]` times. + + Args: + params: The `Tensor` whose values should be repeated. + splits: A splits tensor indicating the ranges of `params` that should be + repeated. + repeats: The number of times each range should be repeated. Supports + broadcasting from a scalar value. + + Returns: + A `Tensor` with the same rank and type as `params`. + + #### Example: + ```python + >>> repeat_ranges(['a', 'b', 'c'], [0, 2, 3], 3) + ['a', 'b', 'a', 'b', 'a', 'b', 'c', 'c', 'c'] + ``` + """ + # Divide `splits` into starts and limits, and repeat them `repeats` times. + if repeats.shape.ndims != 0: + repeated_starts = repeat(splits[:-1], repeats, axis=0) + repeated_limits = repeat(splits[1:], repeats, axis=0) + else: + # Optimization: we can just call repeat once, and then slice the result. + repeated_splits = repeat(splits, repeats, axis=0) + n_splits = array_ops.shape(repeated_splits, out_type=dtypes.int64)[0] + repeated_starts = repeated_splits[:n_splits - repeats] + repeated_limits = repeated_splits[repeats:] + + # Get indices for each range from starts to limits, and use those to gather + # the values in the desired repetition pattern. + one = array_ops.ones((), repeated_starts.dtype) + offsets = gen_ragged_math_ops.ragged_range( + repeated_starts, repeated_limits, one) + return array_ops.gather(params, offsets.rt_dense_values) diff --git a/tensorflow/python/ops/ragged/ragged_where_op_test.py b/tensorflow/python/ops/ragged/ragged_where_op_test.py index 755333de392..de83a549771 100644 --- a/tensorflow/python/ops/ragged/ragged_where_op_test.py +++ b/tensorflow/python/ops/ragged/ragged_where_op_test.py @@ -165,12 +165,13 @@ class RaggedWhereOpTest(test_util.TensorFlowTestCase, parameterized.TestCase): y=ragged.constant_value([[[['a']]], [[['b']]]]), expected=ragged.constant_value([[[[], [b'A']]], [[[b'b']]]])), ]) # pyformat: disable + @test_util.run_deprecated_v1 def testRaggedWhere(self, condition, expected, x=None, y=None): result = ragged.where(condition, x, y) self.assertEqual( getattr(result, 'ragged_rank', 0), getattr(expected, 'ragged_rank', 0)) with self.test_session(): - result_value = result.eval() + result_value = self.evaluate(result) if hasattr(result_value, 'tolist'): result_value = result_value.tolist() if hasattr(expected, 'tolist'): diff --git a/tensorflow/python/ops/random_ops.py b/tensorflow/python/ops/random_ops.py index 1f7db0af61f..62e2f6d1025 100644 --- a/tensorflow/python/ops/random_ops.py +++ b/tensorflow/python/ops/random_ops.py @@ -138,7 +138,9 @@ def parameterized_truncated_normal(shape, return rnd -@tf_export("random.truncated_normal", "truncated_normal") +@tf_export("random.truncated_normal", + v1=["random.truncated_normal", "truncated_normal"]) +@deprecation.deprecated_endpoints("truncated_normal") def truncated_normal(shape, mean=0.0, stddev=1.0, @@ -325,7 +327,9 @@ def random_crop(value, size, seed=None, name=None): return array_ops.slice(value, offset, size, name=name) -@tf_export("random.multinomial", "multinomial") +@tf_export(v1=["random.multinomial", "multinomial"]) +@deprecation.deprecated( + date=None, instructions="Use tf.random.categorical instead.") def multinomial(logits, num_samples, seed=None, name=None, output_dtype=None): """Draws samples from a multinomial distribution. @@ -342,9 +346,7 @@ def multinomial(logits, num_samples, seed=None, name=None, output_dtype=None): `[i, :]` represents the unnormalized log-probabilities for all classes. num_samples: 0-D. Number of independent samples to draw for each row slice. seed: A Python integer. Used to create a random seed for the distribution. - See - `tf.set_random_seed` - for behavior. + See `tf.set_random_seed` for behavior. name: Optional name for the operation. output_dtype: integer type to use for the output. Defaults to int64. @@ -352,10 +354,43 @@ def multinomial(logits, num_samples, seed=None, name=None, output_dtype=None): The drawn samples of shape `[batch_size, num_samples]`. """ with ops.name_scope(name, "multinomial", [logits]): - logits = ops.convert_to_tensor(logits, name="logits") - seed1, seed2 = random_seed.get_seed(seed) - return gen_random_ops.multinomial( - logits, num_samples, seed=seed1, seed2=seed2, output_dtype=output_dtype) + return multinomial_categorical_impl(logits, num_samples, output_dtype, seed) + + +@tf_export("random.categorical") +def categorical(logits, num_samples, dtype=None, seed=None, name=None): + """Draws samples from a categorical distribution. + + Example: + + ```python + # samples has shape [1, 5], where each value is either 0 or 1 with equal + # probability. + samples = tf.random.categorical(tf.log([[10., 10.]]), 5) + ``` + + Args: + logits: 2-D Tensor with shape `[batch_size, num_classes]`. Each slice + `[i, :]` represents the unnormalized log-probabilities for all classes. + num_samples: 0-D. Number of independent samples to draw for each row slice. + dtype: integer type to use for the output. Defaults to int64. + seed: A Python integer. Used to create a random seed for the distribution. + See `tf.set_random_seed` for behavior. + name: Optional name for the operation. + + Returns: + The drawn samples of shape `[batch_size, num_samples]`. + """ + with ops.name_scope(name, "categorical", [logits]): + return multinomial_categorical_impl(logits, num_samples, dtype, seed) + + +def multinomial_categorical_impl(logits, num_samples, dtype, seed): + """Implementation for random.multinomial (v1) and random.categorical (v2).""" + logits = ops.convert_to_tensor(logits, name="logits") + seed1, seed2 = random_seed.get_seed(seed) + return gen_random_ops.multinomial( + logits, num_samples, seed=seed1, seed2=seed2, output_dtype=dtype) ops.NotDifferentiable("Multinomial") @@ -445,7 +480,7 @@ def random_gamma(shape, shape, alpha_broadcast, seed=seed1, seed2=seed2) / beta) -@tf_export("random.poisson", v1=["random.poisson", "random_poisson"]) +@tf_export(v1=["random.poisson", "random_poisson"]) @deprecation.deprecated_endpoints("random_poisson") def random_poisson(lam, shape, dtype=dtypes.float32, seed=None, name=None): """Draws `shape` samples from each of the given Poisson distribution(s). @@ -478,6 +513,45 @@ def random_poisson(lam, shape, dtype=dtypes.float32, seed=None, name=None): for behavior. name: Optional name for the operation. + Returns: + samples: a `Tensor` of shape `tf.concat([shape, tf.shape(lam)], axis=0)` + with values of type `dtype`. + """ + return random_poisson_v2(shape, lam, dtype, seed, name) + + +@tf_export("random.poisson", v1=[]) +def random_poisson_v2(shape, lam, dtype=dtypes.float32, seed=None, name=None): + """Draws `shape` samples from each of the given Poisson distribution(s). + + `lam` is the rate parameter describing the distribution(s). + + Example: + + ```python + samples = tf.random_poisson([10], [0.5, 1.5]) + # samples has shape [10, 2], where each slice [:, 0] and [:, 1] represents + # the samples drawn from each distribution + + samples = tf.random_poisson([7, 5], [12.2, 3.3]) + # samples has shape [7, 5, 2], where each slice [:, :, 0] and [:, :, 1] + # represents the 7x5 samples drawn from each of the two distributions + ``` + + Args: + shape: A 1-D integer Tensor or Python array. The shape of the output samples + to be drawn per "rate"-parameterized distribution. + lam: A Tensor or Python value or N-D array of type `dtype`. + `lam` provides the rate parameter(s) describing the poisson + distribution(s) to sample. + dtype: The type of the output: `float16`, `float32`, `float64`, `int32` or + `int64`. + seed: A Python integer. Used to create a random seed for the distributions. + See + `tf.set_random_seed` + for behavior. + name: Optional name for the operation. + Returns: samples: a `Tensor` of shape `tf.concat([shape, tf.shape(lam)], axis=0)` with values of type `dtype`. diff --git a/tensorflow/python/ops/resource_variable_ops.py b/tensorflow/python/ops/resource_variable_ops.py index 488b6fcbcdb..1066b357b43 100644 --- a/tensorflow/python/ops/resource_variable_ops.py +++ b/tensorflow/python/ops/resource_variable_ops.py @@ -26,6 +26,7 @@ from tensorflow.core.framework import variable_pb2 from tensorflow.python import pywrap_tensorflow from tensorflow.python.eager import context from tensorflow.python.eager import tape +from tensorflow.python.framework import constant_op from tensorflow.python.framework import cpp_shape_inference_pb2 from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops @@ -64,6 +65,7 @@ def eager_safe_variable_handle(shape, dtype, shared_name, name, graph_mode): name=name, container=container) if graph_mode: + handle._handle_data = get_resource_handle_data(handle) # pylint: disable=protected-access return handle # We do not want two distinct ResourceVariable objects for the same @@ -519,7 +521,10 @@ class ResourceVariable(variables.RefVariable): snapshot = g.as_graph_element( ops.prepend_name_scope( variable_def.snapshot_name, import_scope=import_scope)) - self._cached_value = snapshot + if snapshot.op.type != "ReadVariableOp": + self._cached_value = snapshot + else: + self._cached_value = None while snapshot.op.type != "ReadVariableOp": snapshot = snapshot.op.inputs[0] self._graph_element = snapshot @@ -802,16 +807,6 @@ class ResourceVariable(variables.RefVariable): return ResourceVariable( variable_def=variable_def, import_scope=import_scope) - @staticmethod - def _OverloadAllOperators(): # pylint: disable=invalid-name - """Register overloads for all operators.""" - for operator in ops.Tensor.OVERLOADABLE_OPERATORS: - ResourceVariable._OverloadOperator(operator) - # For slicing, bind getitem differently than a tensor (use SliceHelperVar - # instead) - # pylint: disable=protected-access - setattr(ResourceVariable, "__getitem__", array_ops._SliceHelperVar) - def _AsTensor(self): return self.value() @@ -823,30 +818,6 @@ class ResourceVariable(variables.RefVariable): """Unsupported.""" raise NotImplementedError("ResourceVariable does not implement set_shape()") - @staticmethod - def _OverloadOperator(operator): # pylint: disable=invalid-name - """Defer an operator overload to `ops.Tensor`. - - We pull the operator out of ops.Tensor dynamically to avoid ordering issues. - - Args: - operator: string. The operator name. - """ - - tensor_oper = getattr(ops.Tensor, operator) - def _run_op(a, *args): - # pylint: disable=protected-access - value = a._AsTensor() - return tensor_oper(value, *args) - - # Propagate __doc__ to wrapper - try: - _run_op.__doc__ = tensor_oper.__doc__ - except AttributeError: - pass - - setattr(ResourceVariable, operator, _run_op) - __array_priority__ = 100 def is_initialized(self, name=None): @@ -1432,7 +1403,6 @@ ops.register_tensor_conversion_function( variables.Variable, variables.Variable._TensorConversionFunction) # pylint: disable=protected-access # pylint: disable=protected-access -ResourceVariable._OverloadAllOperators() ops.register_dense_tensor_like_type(ResourceVariable) @@ -1442,13 +1412,23 @@ def _ReadGrad(_, grad): return grad +def variable_shape(handle, out_type=dtypes.int32): + if getattr( + handle, "_handle_data", None) is None or not handle._handle_data.is_set: + return gen_resource_variable_ops.variable_shape(handle, out_type=out_type) + shape_proto = handle._handle_data.shape_and_type[0].shape + if shape_proto.unknown_rank or any(x.size == -1 for x in shape_proto.dim): + return gen_resource_variable_ops.variable_shape(handle, out_type=out_type) + return constant_op.constant([x.size for x in shape_proto.dim], dtype=out_type) + + @ops.RegisterGradient("ResourceGather") def _GatherGrad(op, grad): """Gradient for gather op.""" # Build appropriately shaped IndexedSlices handle = op.inputs[0] indices = op.inputs[1] - params_shape = gen_resource_variable_ops.variable_shape(handle) + params_shape = variable_shape(handle) size = array_ops.expand_dims(array_ops.size(indices), 0) values_shape = array_ops.concat([size, params_shape[1:]], 0) values = array_ops.reshape(grad, values_shape) @@ -1522,3 +1502,6 @@ def copy_to_graph_uninitialized(var): new_variable._maybe_initialize_checkpointable() # pylint: enable=protected-access return new_variable + +ops.NotDifferentiable("VarIsInitializedOp") +ops.NotDifferentiable("VariableShape") diff --git a/tensorflow/python/ops/resources.py b/tensorflow/python/ops/resources.py index db6740643cf..6cd12b7bbbe 100644 --- a/tensorflow/python/ops/resources.py +++ b/tensorflow/python/ops/resources.py @@ -21,6 +21,7 @@ from __future__ import division from __future__ import print_function import collections +import os from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops @@ -86,7 +87,8 @@ def report_uninitialized_resources(resource_list=None, resource_list = shared_resources() + local_resources() with ops.name_scope(name): # Run all operations on CPU - with ops.device("/cpu:0"): + local_device = os.environ.get("TF_DEVICE_FOR_UNINITIALIZED_VARIABLE_REPORTING", "/cpu:0") + with ops.device(local_device): if not resource_list: # Return an empty tensor so we only need to check for returned tensor # size being 0 as an indication of model ready. diff --git a/tensorflow/python/ops/rnn.py b/tensorflow/python/ops/rnn.py index 57ecb505573..ec48cab91d1 100644 --- a/tensorflow/python/ops/rnn.py +++ b/tensorflow/python/ops/rnn.py @@ -117,7 +117,7 @@ def _infer_state_dtype(explicit_dtype, state): inferred_dtypes = [element.dtype for element in nest.flatten(state)] if not inferred_dtypes: raise ValueError("Unable to infer dtype from empty state.") - all_same = all([x == inferred_dtypes[0] for x in inferred_dtypes]) + all_same = all(x == inferred_dtypes[0] for x in inferred_dtypes) if not all_same: raise ValueError( "State has tensors of different inferred_dtypes. Unable to infer a " @@ -348,7 +348,10 @@ def _reverse_seq(input_seq, lengths): return results -@tf_export("nn.bidirectional_dynamic_rnn") +@deprecation.deprecated(None, "Please use `keras.layers.Bidirectional(" + "keras.layers.RNN(cell))`, which is equivalent to " + "this API") +@tf_export(v1=["nn.bidirectional_dynamic_rnn"]) def bidirectional_dynamic_rnn(cell_fw, cell_bw, inputs, sequence_length=None, initial_state_fw=None, initial_state_bw=None, dtype=None, parallel_iterations=None, @@ -1490,7 +1493,10 @@ def static_state_saving_rnn(cell, return (outputs, state) -@tf_export("nn.static_bidirectional_rnn") +@deprecation.deprecated(None, "Please use `keras.layers.Bidirectional(" + "keras.layers.RNN(cell, unroll=True))`, which is " + "equivalent to this API") +@tf_export(v1=["nn.static_bidirectional_rnn"]) def static_bidirectional_rnn(cell_fw, cell_bw, inputs, diff --git a/tensorflow/python/ops/rnn_cell_impl.py b/tensorflow/python/ops/rnn_cell_impl.py index 050b4868939..ffc45619a74 100644 --- a/tensorflow/python/ops/rnn_cell_impl.py +++ b/tensorflow/python/ops/rnn_cell_impl.py @@ -36,6 +36,7 @@ from tensorflow.python.framework import tensor_shape from tensorflow.python.framework import tensor_util from tensorflow.python.keras import activations from tensorflow.python.keras import initializers +from tensorflow.python.keras.engine import input_spec from tensorflow.python.keras.utils import tf_utils from tensorflow.python.layers import base as base_layer from tensorflow.python.ops import array_ops @@ -410,7 +411,7 @@ class BasicRNNCell(LayerRNNCell): "performance on GPU.", self) # Inputs must be 2-dimensional. - self.input_spec = base_layer.InputSpec(ndim=2) + self.input_spec = input_spec.InputSpec(ndim=2) self._num_units = num_units if activation: @@ -507,7 +508,7 @@ class GRUCell(LayerRNNCell): "Please use tf.contrib.cudnn_rnn.CudnnGRU for better " "performance on GPU.", self) # Inputs must be 2-dimensional. - self.input_spec = base_layer.InputSpec(ndim=2) + self.input_spec = input_spec.InputSpec(ndim=2) self._num_units = num_units if activation: @@ -683,7 +684,7 @@ class BasicLSTMCell(LayerRNNCell): "performance on GPU.", self) # Inputs must be 2-dimensional. - self.input_spec = base_layer.InputSpec(ndim=2) + self.input_spec = input_spec.InputSpec(ndim=2) self._num_units = num_units self._forget_bias = forget_bias @@ -871,7 +872,7 @@ class LSTMCell(LayerRNNCell): "performance on GPU.", self) # Inputs must be 2-dimensional. - self.input_spec = base_layer.InputSpec(ndim=2) + self.input_spec = input_spec.InputSpec(ndim=2) self._num_units = num_units self._use_peepholes = use_peepholes @@ -1394,7 +1395,7 @@ class DeviceWrapper(RNNCell): return self._cell(inputs, state, scope=scope) -@tf_export("nn.rnn_cell.MultiRNNCell") +@tf_export(v1=["nn.rnn_cell.MultiRNNCell"]) class MultiRNNCell(RNNCell): """RNN cell composed sequentially of multiple simple cells. @@ -1407,6 +1408,9 @@ class MultiRNNCell(RNNCell): ``` """ + @deprecated(None, "This class is equivalent as " + "tf.keras.layers.StackedRNNCells, and will be replaced by " + "that in Tensorflow 2.0.") def __init__(self, cells, state_is_tuple=True): """Create a RNN cell composed sequentially of a number of RNNCells. @@ -1452,7 +1456,7 @@ class MultiRNNCell(RNNCell): if self._state_is_tuple: return tuple(cell.state_size for cell in self._cells) else: - return sum([cell.state_size for cell in self._cells]) + return sum(cell.state_size for cell in self._cells) @property def output_size(self): diff --git a/tensorflow/python/ops/sets_impl.py b/tensorflow/python/ops/sets_impl.py index 21e08d03d21..ee9c9b6bc0b 100644 --- a/tensorflow/python/ops/sets_impl.py +++ b/tensorflow/python/ops/sets_impl.py @@ -31,7 +31,7 @@ _VALID_DTYPES = set([ dtypes.uint8, dtypes.uint16, dtypes.string]) -@tf_export("sets.set_size") +@tf_export("sets.size", v1=["sets.size", "sets.set_size"]) def set_size(a, validate_indices=True): """Compute number of unique elements along last dimension of `a`. @@ -133,7 +133,8 @@ def _set_operation(a, b, set_operation, validate_indices=True): return sparse_tensor.SparseTensor(indices, values, shape) -@tf_export("sets.set_intersection") +@tf_export( + "sets.intersection", v1=["sets.intersection", "sets.set_intersection"]) def set_intersection(a, b, validate_indices=True): """Compute set intersection of elements in last dimension of `a` and `b`. @@ -200,7 +201,8 @@ def set_intersection(a, b, validate_indices=True): return _set_operation(a, b, "intersection", validate_indices) -@tf_export("sets.set_difference") +@tf_export( + "sets.difference", v1=["sets.difference", "sets.set_difference"]) def set_difference(a, b, aminusb=True, validate_indices=True): """Compute set difference of elements in last dimension of `a` and `b`. @@ -271,7 +273,8 @@ def set_difference(a, b, aminusb=True, validate_indices=True): return _set_operation(a, b, "a-b" if aminusb else "b-a", validate_indices) -@tf_export("sets.set_union") +@tf_export( + "sets.union", v1=["sets.union", "sets.set_union"]) def set_union(a, b, validate_indices=True): """Compute set union of elements in last dimension of `a` and `b`. diff --git a/tensorflow/python/ops/signal/BUILD b/tensorflow/python/ops/signal/BUILD index 0d04dc0c1bf..da2bf9c1d2d 100644 --- a/tensorflow/python/ops/signal/BUILD +++ b/tensorflow/python/ops/signal/BUILD @@ -6,16 +6,29 @@ exports_files(["LICENSE"]) py_library( name = "signal", - srcs = glob(["*.py"]), + srcs = [ + "dct_ops.py", + "fft_ops.py", + "mel_ops.py", + "mfcc_ops.py", + "reconstruction_ops.py", + "shape_ops.py", + "signal.py", + "spectral_ops.py", + "util_ops.py", + "window_ops.py", + ], srcs_version = "PY2AND3", deps = [ "//tensorflow/python:array_ops", "//tensorflow/python:constant_op", "//tensorflow/python:control_flow_ops", "//tensorflow/python:dtypes", + "//tensorflow/python:framework", + "//tensorflow/python:framework_for_generated_wrappers", "//tensorflow/python:framework_ops", "//tensorflow/python:math_ops", - "//tensorflow/python:spectral_ops", + "//tensorflow/python:spectral_ops_gen", "//tensorflow/python:tensor_util", "//tensorflow/python:util", "//third_party/py/numpy", diff --git a/tensorflow/python/ops/signal/__init__.py b/tensorflow/python/ops/signal/__init__.py deleted file mode 100644 index 3fa4e94e588..00000000000 --- a/tensorflow/python/ops/signal/__init__.py +++ /dev/null @@ -1,37 +0,0 @@ -"""Signal processing operations. - -See the [tf.signal](https://tensorflow.org/api_guides/python/contrib.signal) -guide. - -@@frame -@@hamming_window -@@hann_window -@@inverse_stft -@@inverse_stft_window_fn -@@mfccs_from_log_mel_spectrograms -@@linear_to_mel_weight_matrix -@@overlap_and_add -@@stft - -[hamming]: https://en.wikipedia.org/wiki/Window_function#Hamming_window -[hann]: https://en.wikipedia.org/wiki/Window_function#Hann_window -[mel]: https://en.wikipedia.org/wiki/Mel_scale -[mfcc]: https://en.wikipedia.org/wiki/Mel-frequency_cepstrum -[stft]: https://en.wikipedia.org/wiki/Short-time_Fourier_transform -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -# pylint: disable=unused-import -from tensorflow.python.ops.signal.mel_ops import linear_to_mel_weight_matrix -from tensorflow.python.ops.signal.mfcc_ops import mfccs_from_log_mel_spectrograms -from tensorflow.python.ops.signal.reconstruction_ops import overlap_and_add -from tensorflow.python.ops.signal.shape_ops import frame -from tensorflow.python.ops.signal.spectral_ops import inverse_stft -from tensorflow.python.ops.signal.spectral_ops import inverse_stft_window_fn -from tensorflow.python.ops.signal.spectral_ops import stft -from tensorflow.python.ops.signal.window_ops import hamming_window -from tensorflow.python.ops.signal.window_ops import hann_window -# pylint: enable=unused-import diff --git a/tensorflow/python/ops/signal/dct_ops.py b/tensorflow/python/ops/signal/dct_ops.py new file mode 100644 index 00000000000..d042c95c049 --- /dev/null +++ b/tensorflow/python/ops/signal/dct_ops.py @@ -0,0 +1,192 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Discrete Cosine Transform ops.""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import math as _math + +from tensorflow.python.framework import dtypes as _dtypes +from tensorflow.python.framework import ops as _ops +from tensorflow.python.framework import tensor_shape +from tensorflow.python.ops import array_ops as _array_ops +from tensorflow.python.ops import math_ops as _math_ops +from tensorflow.python.ops.signal import fft_ops +from tensorflow.python.util.tf_export import tf_export + + +def _validate_dct_arguments(input_tensor, dct_type, n, axis, norm): + """Checks that DCT/IDCT arguments are compatible and well formed.""" + if n is not None: + raise NotImplementedError("The DCT length argument is not implemented.") + if axis != -1: + raise NotImplementedError("axis must be -1. Got: %s" % axis) + if dct_type not in (1, 2, 3): + raise ValueError("Only Types I, II and III (I)DCT are supported.") + if dct_type == 1: + if norm == "ortho": + raise ValueError("Normalization is not supported for the Type-I DCT.") + if input_tensor.shape[-1] is not None and input_tensor.shape[-1] < 2: + raise ValueError( + "Type-I DCT requires the dimension to be greater than one.") + + if norm not in (None, "ortho"): + raise ValueError( + "Unknown normalization. Expected None or 'ortho', got: %s" % norm) + + +# TODO(rjryan): Implement `n` and `axis` parameters. +@tf_export("signal.dct", v1=["signal.dct", "spectral.dct"]) +def dct(input, type=2, n=None, axis=-1, norm=None, name=None): # pylint: disable=redefined-builtin + """Computes the 1D [Discrete Cosine Transform (DCT)][dct] of `input`. + + Currently only Types I, II and III are supported. + Type I is implemented using a length `2N` padded `tf.spectral.rfft`. + Type II is implemented using a length `2N` padded `tf.spectral.rfft`, as + described here: + https://dsp.stackexchange.com/a/10606. + Type III is a fairly straightforward inverse of Type II + (i.e. using a length `2N` padded `tf.spectral.irfft`). + + @compatibility(scipy) + Equivalent to scipy.fftpack.dct for Type-I, Type-II and Type-III DCT. + https://docs.scipy.org/doc/scipy-0.14.0/reference/generated/scipy.fftpack.dct.html + @end_compatibility + + Args: + input: A `[..., samples]` `float32` `Tensor` containing the signals to + take the DCT of. + type: The DCT type to perform. Must be 1, 2 or 3. + n: For future expansion. The length of the transform. Must be `None`. + axis: For future expansion. The axis to compute the DCT along. Must be `-1`. + norm: The normalization to apply. `None` for no normalization or `'ortho'` + for orthonormal normalization. + name: An optional name for the operation. + + Returns: + A `[..., samples]` `float32` `Tensor` containing the DCT of `input`. + + Raises: + ValueError: If `type` is not `1`, `2` or `3`, `n` is not `None, `axis` is + not `-1`, or `norm` is not `None` or `'ortho'`. + ValueError: If `type` is `1` and `norm` is `ortho`. + + [dct]: https://en.wikipedia.org/wiki/Discrete_cosine_transform + """ + _validate_dct_arguments(input, type, n, axis, norm) + with _ops.name_scope(name, "dct", [input]): + # We use the RFFT to compute the DCT and TensorFlow only supports float32 + # for FFTs at the moment. + input = _ops.convert_to_tensor(input, dtype=_dtypes.float32) + + axis_dim = (tensor_shape.dimension_value(input.shape[-1]) + or _array_ops.shape(input)[-1]) + axis_dim_float = _math_ops.to_float(axis_dim) + + if type == 1: + dct1_input = _array_ops.concat([input, input[..., -2:0:-1]], axis=-1) + dct1 = _math_ops.real(fft_ops.rfft(dct1_input)) + return dct1 + + if type == 2: + scale = 2.0 * _math_ops.exp( + _math_ops.complex( + 0.0, -_math_ops.range(axis_dim_float) * _math.pi * 0.5 / + axis_dim_float)) + + # TODO(rjryan): Benchmark performance and memory usage of the various + # approaches to computing a DCT via the RFFT. + dct2 = _math_ops.real( + fft_ops.rfft( + input, fft_length=[2 * axis_dim])[..., :axis_dim] * scale) + + if norm == "ortho": + n1 = 0.5 * _math_ops.rsqrt(axis_dim_float) + n2 = n1 * _math_ops.sqrt(2.0) + # Use tf.pad to make a vector of [n1, n2, n2, n2, ...]. + weights = _array_ops.pad( + _array_ops.expand_dims(n1, 0), [[0, axis_dim - 1]], + constant_values=n2) + dct2 *= weights + + return dct2 + + elif type == 3: + if norm == "ortho": + n1 = _math_ops.sqrt(axis_dim_float) + n2 = n1 * _math_ops.sqrt(0.5) + # Use tf.pad to make a vector of [n1, n2, n2, n2, ...]. + weights = _array_ops.pad( + _array_ops.expand_dims(n1, 0), [[0, axis_dim - 1]], + constant_values=n2) + input *= weights + else: + input *= axis_dim_float + scale = 2.0 * _math_ops.exp( + _math_ops.complex( + 0.0, + _math_ops.range(axis_dim_float) * _math.pi * 0.5 / + axis_dim_float)) + dct3 = _math_ops.real( + fft_ops.irfft( + scale * _math_ops.complex(input, 0.0), + fft_length=[2 * axis_dim]))[..., :axis_dim] + + return dct3 + + +# TODO(rjryan): Implement `n` and `axis` parameters. +@tf_export("signal.idct", v1=["signal.idct", "spectral.idct"]) +def idct(input, type=2, n=None, axis=-1, norm=None, name=None): # pylint: disable=redefined-builtin + """Computes the 1D [Inverse Discrete Cosine Transform (DCT)][idct] of `input`. + + Currently only Types I, II and III are supported. Type III is the inverse of + Type II, and vice versa. + + Note that you must re-normalize by 1/(2n) to obtain an inverse if `norm` is + not `'ortho'`. That is: + `signal == idct(dct(signal)) * 0.5 / signal.shape[-1]`. + When `norm='ortho'`, we have: + `signal == idct(dct(signal, norm='ortho'), norm='ortho')`. + + @compatibility(scipy) + Equivalent to scipy.fftpack.idct for Type-I, Type-II and Type-III DCT. + https://docs.scipy.org/doc/scipy-0.14.0/reference/generated/scipy.fftpack.idct.html + @end_compatibility + + Args: + input: A `[..., samples]` `float32` `Tensor` containing the signals to take + the DCT of. + type: The IDCT type to perform. Must be 1, 2 or 3. + n: For future expansion. The length of the transform. Must be `None`. + axis: For future expansion. The axis to compute the DCT along. Must be `-1`. + norm: The normalization to apply. `None` for no normalization or `'ortho'` + for orthonormal normalization. + name: An optional name for the operation. + + Returns: + A `[..., samples]` `float32` `Tensor` containing the IDCT of `input`. + + Raises: + ValueError: If `type` is not `1`, `2` or `3`, `n` is not `None, `axis` is + not `-1`, or `norm` is not `None` or `'ortho'`. + + [idct]: + https://en.wikipedia.org/wiki/Discrete_cosine_transform#Inverse_transforms + """ + _validate_dct_arguments(input, type, n, axis, norm) + inverse_type = {1: 1, 2: 3, 3: 2}[type] + return dct(input, type=inverse_type, n=n, axis=axis, norm=norm, name=name) diff --git a/tensorflow/python/ops/spectral_ops.py b/tensorflow/python/ops/signal/fft_ops.py similarity index 51% rename from tensorflow/python/ops/spectral_ops.py rename to tensorflow/python/ops/signal/fft_ops.py index 4dcc90aefa9..2d14b2bbd75 100644 --- a/tensorflow/python/ops/spectral_ops.py +++ b/tensorflow/python/ops/signal/fft_ops.py @@ -12,16 +12,15 @@ # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================== -"""Spectral operators (e.g. DCT, FFT, RFFT).""" +"""Fast-Fourier Transform ops.""" from __future__ import absolute_import from __future__ import division from __future__ import print_function -import math as _math +import numpy as np from tensorflow.python.framework import dtypes as _dtypes from tensorflow.python.framework import ops as _ops -from tensorflow.python.framework import tensor_shape from tensorflow.python.framework import tensor_util as _tensor_util from tensorflow.python.ops import array_ops as _array_ops from tensorflow.python.ops import gen_spectral_ops @@ -112,6 +111,7 @@ def _rfft_wrapper(fft_fn, fft_rank, default_name): """Wrapper around gen_spectral_ops.rfft* that infers fft_length argument.""" def _rfft(input_tensor, fft_length=None, name=None): + """Wrapper around gen_spectral_ops.rfft* that infers fft_length argument.""" with _ops.name_scope(name, default_name, [input_tensor, fft_length]) as name: input_tensor = _ops.convert_to_tensor(input_tensor, _dtypes.float32) @@ -130,6 +130,7 @@ def _irfft_wrapper(ifft_fn, fft_rank, default_name): """Wrapper around gen_spectral_ops.irfft* that infers fft_length argument.""" def _irfft(input_tensor, fft_length=None, name=None): + """Wrapper irfft* that infers fft_length argument.""" with _ops.name_scope(name, default_name, [input_tensor, fft_length]) as name: input_tensor = _ops.convert_to_tensor(input_tensor, _dtypes.complex64) @@ -145,6 +146,8 @@ def _irfft_wrapper(ifft_fn, fft_rank, default_name): return _irfft +# FFT/IFFT 1/2/3D are exported via +# third_party/tensorflow/core/api_def/python_api/ fft = gen_spectral_ops.fft ifft = gen_spectral_ops.ifft fft2d = gen_spectral_ops.fft2d @@ -152,159 +155,176 @@ ifft2d = gen_spectral_ops.ifft2d fft3d = gen_spectral_ops.fft3d ifft3d = gen_spectral_ops.ifft3d rfft = _rfft_wrapper(gen_spectral_ops.rfft, 1, "rfft") -tf_export("spectral.rfft")(rfft) +tf_export("signal.rfft", v1=["signal.rfft", "spectral.rfft"])(rfft) irfft = _irfft_wrapper(gen_spectral_ops.irfft, 1, "irfft") -tf_export("spectral.irfft")(irfft) +tf_export("signal.irfft", v1=["signal.irfft", "spectral.irfft"])(irfft) rfft2d = _rfft_wrapper(gen_spectral_ops.rfft2d, 2, "rfft2d") -tf_export("spectral.rfft2d")(rfft2d) +tf_export("signal.rfft2d", v1=["signal.rfft2d", "spectral.rfft2d"])(rfft2d) irfft2d = _irfft_wrapper(gen_spectral_ops.irfft2d, 2, "irfft2d") -tf_export("spectral.irfft2d")(irfft2d) +tf_export("signal.irfft2d", v1=["signal.irfft2d", "spectral.irfft2d"])(irfft2d) rfft3d = _rfft_wrapper(gen_spectral_ops.rfft3d, 3, "rfft3d") -tf_export("spectral.rfft3d")(rfft3d) +tf_export("signal.rfft3d", v1=["signal.rfft3d", "spectral.rfft3d"])(rfft3d) irfft3d = _irfft_wrapper(gen_spectral_ops.irfft3d, 3, "irfft3d") -tf_export("spectral.irfft3d")(irfft3d) +tf_export("signal.irfft3d", v1=["signal.irfft3d", "spectral.irfft3d"])(irfft3d) -def _validate_dct_arguments(dct_type, n, axis, norm): - if n is not None: - raise NotImplementedError("The DCT length argument is not implemented.") - if axis != -1: - raise NotImplementedError("axis must be -1. Got: %s" % axis) - if dct_type not in (2, 3): - raise ValueError("Only Types II and III (I)DCT are supported.") - if norm not in (None, "ortho"): - raise ValueError( - "Unknown normalization. Expected None or 'ortho', got: %s" % norm) +def _fft_size_for_grad(grad, rank): + return _math_ops.reduce_prod(_array_ops.shape(grad)[-rank:]) -# TODO(rjryan): Implement `type`, `n` and `axis` parameters. -@tf_export("spectral.dct") -def dct(input, type=2, n=None, axis=-1, norm=None, name=None): # pylint: disable=redefined-builtin - """Computes the 1D [Discrete Cosine Transform (DCT)][dct] of `input`. - - Currently only Types II and III are supported. Type II is implemented using a - length `2N` padded `tf.spectral.rfft`, as described here: - https://dsp.stackexchange.com/a/10606. Type III is a fairly straightforward - inverse of Type II (i.e. using a length `2N` padded `tf.spectral.irfft`). - - @compatibility(scipy) - Equivalent to scipy.fftpack.dct for Type-II and Type-III DCT. - https://docs.scipy.org/doc/scipy-0.14.0/reference/generated/scipy.fftpack.dct.html - @end_compatibility - - Args: - input: A `[..., samples]` `float32` `Tensor` containing the signals to - take the DCT of. - type: The DCT type to perform. Must be 2 or 3. - n: For future expansion. The length of the transform. Must be `None`. - axis: For future expansion. The axis to compute the DCT along. Must be `-1`. - norm: The normalization to apply. `None` for no normalization or `'ortho'` - for orthonormal normalization. - name: An optional name for the operation. - - Returns: - A `[..., samples]` `float32` `Tensor` containing the DCT of `input`. - - Raises: - ValueError: If `type` is not `2` or `3`, `n` is not `None, `axis` is not - `-1`, or `norm` is not `None` or `'ortho'`. - - [dct]: https://en.wikipedia.org/wiki/Discrete_cosine_transform - """ - _validate_dct_arguments(type, n, axis, norm) - with _ops.name_scope(name, "dct", [input]): - # We use the RFFT to compute the DCT and TensorFlow only supports float32 - # for FFTs at the moment. - input = _ops.convert_to_tensor(input, dtype=_dtypes.float32) - - axis_dim = (tensor_shape.dimension_value(input.shape[-1]) - or _array_ops.shape(input)[-1]) - axis_dim_float = _math_ops.to_float(axis_dim) - if type == 2: - scale = 2.0 * _math_ops.exp( - _math_ops.complex( - 0.0, -_math_ops.range(axis_dim_float) * _math.pi * 0.5 / - axis_dim_float)) - - # TODO(rjryan): Benchmark performance and memory usage of the various - # approaches to computing a DCT via the RFFT. - dct2 = _math_ops.real( - rfft(input, fft_length=[2 * axis_dim])[..., :axis_dim] * scale) - - if norm == "ortho": - n1 = 0.5 * _math_ops.rsqrt(axis_dim_float) - n2 = n1 * _math_ops.sqrt(2.0) - # Use tf.pad to make a vector of [n1, n2, n2, n2, ...]. - weights = _array_ops.pad( - _array_ops.expand_dims(n1, 0), [[0, axis_dim - 1]], - constant_values=n2) - dct2 *= weights - - return dct2 - - elif type == 3: - if norm == "ortho": - n1 = _math_ops.sqrt(axis_dim_float) - n2 = n1 * _math_ops.sqrt(0.5) - # Use tf.pad to make a vector of [n1, n2, n2, n2, ...]. - weights = _array_ops.pad( - _array_ops.expand_dims(n1, 0), [[0, axis_dim - 1]], - constant_values=n2) - input *= weights - else: - input *= axis_dim_float - scale = 2.0 * _math_ops.exp( - _math_ops.complex( - 0.0, - _math_ops.range(axis_dim_float) * _math.pi * 0.5 / - axis_dim_float)) - dct3 = _math_ops.real( - irfft( - scale * _math_ops.complex(input, 0.0), - fft_length=[2 * axis_dim]))[..., :axis_dim] - - return dct3 +@_ops.RegisterGradient("FFT") +def _fft_grad(_, grad): + size = _math_ops.cast(_fft_size_for_grad(grad, 1), grad.dtype) + return ifft(grad) * size -# TODO(rjryan): Implement `type`, `n` and `axis` parameters. -@tf_export("spectral.idct") -def idct(input, type=2, n=None, axis=-1, norm=None, name=None): # pylint: disable=redefined-builtin - """Computes the 1D [Inverse Discrete Cosine Transform (DCT)][idct] of `input`. +@_ops.RegisterGradient("IFFT") +def _ifft_grad(_, grad): + rsize = _math_ops.cast( + 1. / _math_ops.cast(_fft_size_for_grad(grad, 1), grad.dtype.real_dtype), + grad.dtype) + return fft(grad) * rsize - Currently only Types II and III are supported. Type III is the inverse of - Type II, and vice versa. - Note that you must re-normalize by 1/(2n) to obtain an inverse if `norm` is - not `'ortho'`. That is: - `signal == idct(dct(signal)) * 0.5 / signal.shape[-1]`. - When `norm='ortho'`, we have: - `signal == idct(dct(signal, norm='ortho'), norm='ortho')`. +@_ops.RegisterGradient("FFT2D") +def _fft2d_grad(_, grad): + size = _math_ops.cast(_fft_size_for_grad(grad, 2), grad.dtype) + return ifft2d(grad) * size - @compatibility(scipy) - Equivalent to scipy.fftpack.idct for Type-II and Type-III DCT. - https://docs.scipy.org/doc/scipy-0.14.0/reference/generated/scipy.fftpack.idct.html - @end_compatibility - Args: - input: A `[..., samples]` `float32` `Tensor` containing the signals to take - the DCT of. - type: The IDCT type to perform. Must be 2 or 3. - n: For future expansion. The length of the transform. Must be `None`. - axis: For future expansion. The axis to compute the DCT along. Must be `-1`. - norm: The normalization to apply. `None` for no normalization or `'ortho'` - for orthonormal normalization. - name: An optional name for the operation. +@_ops.RegisterGradient("IFFT2D") +def _ifft2d_grad(_, grad): + rsize = _math_ops.cast( + 1. / _math_ops.cast(_fft_size_for_grad(grad, 2), grad.dtype.real_dtype), + grad.dtype) + return fft2d(grad) * rsize - Returns: - A `[..., samples]` `float32` `Tensor` containing the IDCT of `input`. - Raises: - ValueError: If `type` is not `2` or `3`, `n` is not `None, `axis` is not - `-1`, or `norm` is not `None` or `'ortho'`. +@_ops.RegisterGradient("FFT3D") +def _fft3d_grad(_, grad): + size = _math_ops.cast(_fft_size_for_grad(grad, 3), grad.dtype) + return ifft3d(grad) * size - [idct]: - https://en.wikipedia.org/wiki/Discrete_cosine_transform#Inverse_transforms - """ - _validate_dct_arguments(type, n, axis, norm) - inverse_type = {2: 3, 3: 2}[type] - return dct(input, type=inverse_type, n=n, axis=axis, norm=norm, name=name) + +@_ops.RegisterGradient("IFFT3D") +def _ifft3d_grad(_, grad): + rsize = _math_ops.cast( + 1. / _math_ops.cast(_fft_size_for_grad(grad, 3), grad.dtype.real_dtype), + grad.dtype) + return fft3d(grad) * rsize + + +def _rfft_grad_helper(rank, irfft_fn): + """Returns a gradient function for an RFFT of the provided rank.""" + # Can't happen because we don't register a gradient for RFFT3D. + assert rank in (1, 2), "Gradient for RFFT3D is not implemented." + + def _grad(op, grad): + """A gradient function for RFFT with the provided `rank` and `irfft_fn`.""" + fft_length = op.inputs[1] + input_shape = _array_ops.shape(op.inputs[0]) + is_even = _math_ops.cast(1 - (fft_length[-1] % 2), _dtypes.complex64) + + def _tile_for_broadcasting(matrix, t): + expanded = _array_ops.reshape( + matrix, + _array_ops.concat([ + _array_ops.ones([_array_ops.rank(t) - 2], _dtypes.int32), + _array_ops.shape(matrix) + ], 0)) + return _array_ops.tile( + expanded, _array_ops.concat([_array_ops.shape(t)[:-2], [1, 1]], 0)) + + def _mask_matrix(length): + """Computes t_n = exp(sqrt(-1) * pi * n^2 / line_len).""" + # TODO(rjryan): Speed up computation of twiddle factors using the + # following recurrence relation and cache them across invocations of RFFT. + # + # t_n = exp(sqrt(-1) * pi * n^2 / line_len) + # for n = 0, 1,..., line_len-1. + # For n > 2, use t_n = t_{n-1}^2 / t_{n-2} * t_1^2 + a = _array_ops.tile( + _array_ops.expand_dims(_math_ops.range(length), 0), (length, 1)) + b = _array_ops.transpose(a, [1, 0]) + return _math_ops.exp( + -2j * np.pi * _math_ops.cast(a * b, _dtypes.complex64) / + _math_ops.cast(length, _dtypes.complex64)) + + def _ymask(length): + """A sequence of [1+0j, -1+0j, 1+0j, -1+0j, ...] with length `length`.""" + return _math_ops.cast(1 - 2 * (_math_ops.range(length) % 2), + _dtypes.complex64) + + y0 = grad[..., 0:1] + if rank == 1: + ym = grad[..., -1:] + extra_terms = y0 + is_even * ym * _ymask(input_shape[-1]) + elif rank == 2: + # Create a mask matrix for y0 and ym. + base_mask = _mask_matrix(input_shape[-2]) + + # Tile base_mask to match y0 in shape so that we can batch-matmul the + # inner 2 dimensions. + tiled_mask = _tile_for_broadcasting(base_mask, y0) + + y0_term = _math_ops.matmul(tiled_mask, _math_ops.conj(y0)) + extra_terms = y0_term + + ym = grad[..., -1:] + ym_term = _math_ops.matmul(tiled_mask, _math_ops.conj(ym)) + + inner_dim = input_shape[-1] + ym_term = _array_ops.tile( + ym_term, + _array_ops.concat([ + _array_ops.ones([_array_ops.rank(grad) - 1], _dtypes.int32), + [inner_dim] + ], 0)) * _ymask(inner_dim) + + extra_terms += is_even * ym_term + + # The gradient of RFFT is the IRFFT of the incoming gradient times a scaling + # factor, plus some additional terms to make up for the components dropped + # due to Hermitian symmetry. + input_size = _math_ops.to_float(_fft_size_for_grad(op.inputs[0], rank)) + the_irfft = irfft_fn(grad, fft_length) + return 0.5 * (the_irfft * input_size + _math_ops.real(extra_terms)), None + + return _grad + + +def _irfft_grad_helper(rank, rfft_fn): + """Returns a gradient function for an IRFFT of the provided rank.""" + # Can't happen because we don't register a gradient for IRFFT3D. + assert rank in (1, 2), "Gradient for IRFFT3D is not implemented." + + def _grad(op, grad): + """A gradient function for IRFFT with the provided `rank` and `rfft_fn`.""" + # Generate a simple mask like [1.0, 2.0, ..., 2.0, 1.0] for even-length FFTs + # and [1.0, 2.0, ..., 2.0] for odd-length FFTs. To reduce extra ops in the + # graph we special-case the situation where the FFT length and last + # dimension of the input are known at graph construction time. + fft_length = op.inputs[1] + is_odd = _math_ops.mod(fft_length[-1], 2) + input_last_dimension = _array_ops.shape(op.inputs[0])[-1] + mask = _array_ops.concat( + [[1.0], 2.0 * _array_ops.ones([input_last_dimension - 2 + is_odd]), + _array_ops.ones([1 - is_odd])], 0) + + rsize = _math_ops.reciprocal(_math_ops.to_float( + _fft_size_for_grad(grad, rank))) + + # The gradient of IRFFT is the RFFT of the incoming gradient times a scaling + # factor and a mask. The mask scales the gradient for the Hermitian + # symmetric components of the RFFT by a factor of two, since these + # components are de-duplicated in the RFFT. + the_rfft = rfft_fn(grad, fft_length) + return the_rfft * _math_ops.cast(rsize * mask, _dtypes.complex64), None + + return _grad + + +_ops.RegisterGradient("RFFT")(_rfft_grad_helper(1, irfft)) +_ops.RegisterGradient("IRFFT")(_irfft_grad_helper(1, rfft)) +_ops.RegisterGradient("RFFT2D")(_rfft_grad_helper(2, irfft2d)) +_ops.RegisterGradient("IRFFT2D")(_irfft_grad_helper(2, rfft2d)) diff --git a/tensorflow/python/ops/signal/mfcc_ops.py b/tensorflow/python/ops/signal/mfcc_ops.py index 6ae3b222ba5..601409dea90 100644 --- a/tensorflow/python/ops/signal/mfcc_ops.py +++ b/tensorflow/python/ops/signal/mfcc_ops.py @@ -22,7 +22,7 @@ from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops from tensorflow.python.ops import array_ops from tensorflow.python.ops import math_ops -from tensorflow.python.ops import spectral_ops +from tensorflow.python.ops.signal import dct_ops from tensorflow.python.util.tf_export import tf_export @@ -106,5 +106,5 @@ def mfccs_from_log_mel_spectrograms(log_mel_spectrograms, name=None): else: num_mel_bins = array_ops.shape(log_mel_spectrograms)[-1] - dct2 = spectral_ops.dct(log_mel_spectrograms) + dct2 = dct_ops.dct(log_mel_spectrograms, type=2) return dct2 * math_ops.rsqrt(math_ops.to_float(num_mel_bins) * 2.0) diff --git a/tensorflow/python/ops/signal/reconstruction_ops.py b/tensorflow/python/ops/signal/reconstruction_ops.py index 0fc7fec2393..4eaab4e0a0c 100644 --- a/tensorflow/python/ops/signal/reconstruction_ops.py +++ b/tensorflow/python/ops/signal/reconstruction_ops.py @@ -18,46 +18,14 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function +from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops from tensorflow.python.framework import tensor_util from tensorflow.python.ops import array_ops from tensorflow.python.ops import math_ops -from tensorflow.python.ops.signal import shape_ops -from tensorflow.python.ops.signal import util_ops from tensorflow.python.util.tf_export import tf_export -def _shuffle_to_front(input_tensor, k): - """Shuffles the last `k` indices of `input_tensor` to the front. - - Transposes `input_tensor` to have the last `k` indices at the front. The input - may have arbitrary rank and unknown shape. - - Args: - input_tensor: A `Tensor` of arbitrary rank and unknown shape. - k: A scalar `Tensor` specifying how many indices to shuffle. - - Returns: - A transposed version of `input_tensor` with `k` indices shuffled to the - front. - - Raises: - ValueError: If `input_tensor` is not at least rank `k` or `k` is not scalar. - """ - k = ops.convert_to_tensor(k, name="k") - k.shape.with_rank(0) - k_static = tensor_util.constant_value(k) - if k_static is not None: - input_tensor.shape.with_rank_at_least(k_static) - - rank = array_ops.rank(input_tensor) - outer_indices, inner_indices = array_ops.split(math_ops.range(rank), - [rank - k, k]) - permutation = array_ops.concat([inner_indices, outer_indices], 0) - - return array_ops.transpose(input_tensor, perm=permutation) - - @tf_export("signal.overlap_and_add") def overlap_and_add(signal, frame_step, name=None): """Reconstructs a signal from a framed representation. @@ -80,8 +48,8 @@ def overlap_and_add(signal, frame_step, name=None): frames of `signal`'s inner-most two dimensions. Raises: - ValueError: If `signal`'s rank is less than 2, `frame_step` is not a scalar - integer or `frame_step` is greater than `frame_length`. + ValueError: If `signal`'s rank is less than 2, or `frame_step` is not a + scalar integer. """ with ops.name_scope(name, "overlap_and_add", [signal, frame_step]): signal = ops.convert_to_tensor(signal, name="signal") @@ -97,56 +65,91 @@ def overlap_and_add(signal, frame_step, name=None): # All dimensions that are not part of the overlap-and-add. Can be empty for # rank 2 inputs. outer_dimensions = signal_shape[:-2] + outer_rank = array_ops.size(outer_dimensions) - # If frame_length and frame_step are known at graph construction time, check - # frame_step is less than or equal to frame_length. - frame_step_static = tensor_util.constant_value(frame_step) - if (frame_step_static is not None and signal.shape.ndims is not None and - signal.shape.dims[-1].value is not None): - if frame_step_static > signal.shape.dims[-1].value: - raise ValueError( - "frame_step (%d) must be less than or equal to " - "frame_length (%d)" % ( - frame_step_static, signal.shape.dims[-1].value)) - # If frame_length is equal to frame_step, there's no overlap so just - # reshape the tensor. - if frame_step_static == signal.shape.dims[-1].value: - return array_ops.reshape(signal, array_ops.concat( - [outer_dimensions, [-1]], 0)) + def full_shape(inner_shape): + return array_ops.concat([outer_dimensions, inner_shape], 0) - signal_rank = array_ops.rank(signal) - frames = signal_shape[-2] frame_length = signal_shape[-1] + frames = signal_shape[-2] - subframe_length = util_ops.gcd(frame_length, frame_step) - subframe_step = frame_step // subframe_length - subframes_per_frame = frame_length // subframe_length - output_size = frame_step * (frames - 1) + frame_length - output_subframes = output_size // subframe_length + # Compute output length. + output_length = frame_length + frame_step * (frames - 1) - # To avoid overlap-adding sample-by-sample, we overlap-add at the "subframe" - # level, where a subframe is gcd(frame_length, frame_step). Reshape signal - # from [..., frames, frame_length] into [..., subframes, subframe_length]. - subframe_shape = array_ops.concat( - [outer_dimensions, [-1, subframe_length]], 0) - subframe_signal = array_ops.reshape(signal, subframe_shape) + # If frame_length is equal to frame_step, there's no overlap so just + # reshape the tensor. + frame_step_static = tensor_util.constant_value(frame_step) + if (frame_step_static is not None and signal.shape.dims is not None and + frame_step_static == signal.shape.dims[-1].value): + output_shape = full_shape([output_length]) + return array_ops.reshape(signal, output_shape, name="fast_path") - # Now we shuffle the last [subframes, subframe_length] dimensions to the - # front. - # TODO(rjryan): Add an axis argument to unsorted_segment_sum so we can - # avoid this pair of transposes. - subframe_signal = _shuffle_to_front(subframe_signal, 2) + # The following code is documented using this example: + # + # frame_step = 2 + # signal.shape = (3, 5) + # a b c d e + # f g h i j + # k l m n o - # Use unsorted_segment_sum to add overlapping subframes together. - segment_ids = array_ops.reshape(shape_ops.frame( - math_ops.range(output_subframes), subframes_per_frame, subframe_step, - pad_end=False), [-1]) - result = math_ops.unsorted_segment_sum(subframe_signal, segment_ids, - num_segments=output_subframes) + # Compute the number of segments, per frame. + segments = -(-frame_length // frame_step) # Divide and round up. - # result is a [subframes, subframe_length, ...outer_dimensions] tensor. We - # return a [...outer_dimensions, output_size] tensor with a transpose and - # reshape. - result_shape = array_ops.concat([outer_dimensions, [output_size]], 0) - return array_ops.reshape(_shuffle_to_front(result, signal_rank - 2), - result_shape) + # Pad the frame_length dimension to a multiple of the frame step. + # Pad the frames dimension by `segments` so that signal.shape = (6, 6) + # a b c d e 0 + # f g h i j 0 + # k l m n o 0 + # 0 0 0 0 0 0 + # 0 0 0 0 0 0 + # 0 0 0 0 0 0 + paddings = [[0, segments], [0, segments * frame_step - frame_length]] + outer_paddings = array_ops.zeros([outer_rank, 2], dtypes.int32) + paddings = array_ops.concat([outer_paddings, paddings], 0) + signal = array_ops.pad(signal, paddings) + + # Reshape so that signal.shape = (3, 6, 2) + # ab cd e0 + # fg hi j0 + # kl mn o0 + # 00 00 00 + # 00 00 00 + # 00 00 00 + shape = full_shape([frames + segments, segments, frame_step]) + signal = array_ops.reshape(signal, shape) + + # Transpose dimensions so that signal.shape = (3, 6, 2) + # ab fg kl 00 00 00 + # cd hi mn 00 00 00 + # e0 j0 o0 00 00 00 + perm = array_ops.concat( + [math_ops.range(outer_rank), outer_rank + [1, 0, 2]], 0) + signal = array_ops.transpose(signal, perm) + + # Reshape so that signal.shape = (18, 2) + # ab fg kl 00 00 00 cd hi mn 00 00 00 e0 j0 o0 00 00 00 + shape = full_shape([(frames + segments) * segments, frame_step]) + signal = array_ops.reshape(signal, shape) + + # Truncate so that signal.shape = (15, 2) + # ab fg kl 00 00 00 cd hi mn 00 00 00 e0 j0 o0 + signal = signal[..., :(frames + segments - 1) * segments, :] + + # Reshape so that signal.shape = (3, 5, 2) + # ab fg kl 00 00 + # 00 cd hi mn 00 + # 00 00 e0 j0 o0 + shape = full_shape([segments, (frames + segments - 1), frame_step]) + signal = array_ops.reshape(signal, shape) + + # Now, reduce over the columns, to achieve the desired sum. + signal = math_ops.reduce_sum(signal, -3) + + # Flatten the array. + shape = full_shape([(frames + segments - 1) * frame_step]) + signal = array_ops.reshape(signal, shape) + + # Truncate to final length. + signal = signal[..., :output_length] + + return signal diff --git a/tensorflow/python/ops/signal/shape_ops.py b/tensorflow/python/ops/signal/shape_ops.py index 02dd7c97e8f..ae9c2ef28e4 100644 --- a/tensorflow/python/ops/signal/shape_ops.py +++ b/tensorflow/python/ops/signal/shape_ops.py @@ -71,7 +71,7 @@ def frame(signal, frame_length, frame_step, pad_end=False, pad_value=0, axis=-1, ```python pcm = tf.placeholder(tf.float32, [None, 9152]) frames = tf.signal.frame(pcm, 512, 180) - magspec = tf.abs(tf.spectral.rfft(frames, [512])) + magspec = tf.abs(tf.signal.rfft(frames, [512])) image = tf.expand_dims(magspec, 3) ``` diff --git a/tensorflow/python/ops/signal/signal.py b/tensorflow/python/ops/signal/signal.py new file mode 100644 index 00000000000..cdc4d1c1911 --- /dev/null +++ b/tensorflow/python/ops/signal/signal.py @@ -0,0 +1,65 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Signal processing operations. + +See the [tf.signal](https://tensorflow.org/api_guides/python/contrib.signal) +guide. + +@@frame +@@hamming_window +@@hann_window +@@inverse_stft +@@inverse_stft_window_fn +@@mfccs_from_log_mel_spectrograms +@@linear_to_mel_weight_matrix +@@overlap_and_add +@@stft + +[hamming]: https://en.wikipedia.org/wiki/Window_function#Hamming_window +[hann]: https://en.wikipedia.org/wiki/Window_function#Hann_window +[mel]: https://en.wikipedia.org/wiki/Mel_scale +[mfcc]: https://en.wikipedia.org/wiki/Mel-frequency_cepstrum +[stft]: https://en.wikipedia.org/wiki/Short-time_Fourier_transform +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +# pylint: disable=unused-import +from tensorflow.python.ops.signal.dct_ops import dct +from tensorflow.python.ops.signal.dct_ops import idct +from tensorflow.python.ops.signal.fft_ops import fft +from tensorflow.python.ops.signal.fft_ops import fft2d +from tensorflow.python.ops.signal.fft_ops import fft3d +from tensorflow.python.ops.signal.fft_ops import ifft +from tensorflow.python.ops.signal.fft_ops import ifft2d +from tensorflow.python.ops.signal.fft_ops import ifft3d +from tensorflow.python.ops.signal.fft_ops import irfft +from tensorflow.python.ops.signal.fft_ops import irfft2d +from tensorflow.python.ops.signal.fft_ops import irfft3d +from tensorflow.python.ops.signal.fft_ops import rfft +from tensorflow.python.ops.signal.fft_ops import rfft2d +from tensorflow.python.ops.signal.fft_ops import rfft3d +from tensorflow.python.ops.signal.mel_ops import linear_to_mel_weight_matrix +from tensorflow.python.ops.signal.mfcc_ops import mfccs_from_log_mel_spectrograms +from tensorflow.python.ops.signal.reconstruction_ops import overlap_and_add +from tensorflow.python.ops.signal.shape_ops import frame +from tensorflow.python.ops.signal.spectral_ops import inverse_stft +from tensorflow.python.ops.signal.spectral_ops import inverse_stft_window_fn +from tensorflow.python.ops.signal.spectral_ops import stft +from tensorflow.python.ops.signal.window_ops import hamming_window +from tensorflow.python.ops.signal.window_ops import hann_window +# pylint: enable=unused-import diff --git a/tensorflow/python/ops/signal/spectral_ops.py b/tensorflow/python/ops/signal/spectral_ops.py index b0b7d964b93..f029e0a8b59 100644 --- a/tensorflow/python/ops/signal/spectral_ops.py +++ b/tensorflow/python/ops/signal/spectral_ops.py @@ -25,7 +25,7 @@ from tensorflow.python.framework import ops from tensorflow.python.framework import tensor_util from tensorflow.python.ops import array_ops from tensorflow.python.ops import math_ops -from tensorflow.python.ops import spectral_ops +from tensorflow.python.ops.signal import fft_ops from tensorflow.python.ops.signal import reconstruction_ops from tensorflow.python.ops.signal import shape_ops from tensorflow.python.ops.signal import window_ops @@ -86,9 +86,9 @@ def stft(signals, frame_length, frame_step, fft_length=None, window = window_fn(frame_length, dtype=framed_signals.dtype) framed_signals *= window - # spectral_ops.rfft produces the (fft_length/2 + 1) unique components of the + # fft_ops.rfft produces the (fft_length/2 + 1) unique components of the # FFT of the real windowed signals in framed_signals. - return spectral_ops.rfft(framed_signals, [fft_length]) + return fft_ops.rfft(framed_signals, [fft_length]) @tf_export('signal.inverse_stft_window_fn') @@ -232,7 +232,7 @@ def inverse_stft(stfts, fft_length = ops.convert_to_tensor(fft_length, name='fft_length') fft_length.shape.assert_has_rank(0) - real_frames = spectral_ops.irfft(stfts, [fft_length]) + real_frames = fft_ops.irfft(stfts, [fft_length]) # frame_length may be larger or smaller than fft_length, so we pad or # truncate real_frames to frame_length. diff --git a/tensorflow/python/ops/sort_ops.py b/tensorflow/python/ops/sort_ops.py new file mode 100644 index 00000000000..c3e23d701ed --- /dev/null +++ b/tensorflow/python/ops/sort_ops.py @@ -0,0 +1,197 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Support for sorting tensors. + +@@argsort +@@sort +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import numpy as np + +from tensorflow.python.framework import constant_op +from tensorflow.python.framework import ops as framework_ops +from tensorflow.python.framework import tensor_util +from tensorflow.python.ops import array_ops +from tensorflow.python.ops import math_ops +from tensorflow.python.ops import nn_ops +from tensorflow.python.util.tf_export import tf_export + + +@tf_export('sort') +def sort(values, axis=-1, direction='ASCENDING', name=None): + """Sorts a tensor. + + Args: + values: 1-D or higher numeric `Tensor`. + axis: The axis along which to sort. The default is -1, which sorts the last + axis. + direction: The direction in which to sort the values (`'ASCENDING'` or + `'DESCENDING'`). + name: Optional name for the operation. + + Returns: + A `Tensor` with the same dtype and shape as `values`, with the elements + sorted along the given `axis`. + + Raises: + ValueError: If axis is not a constant scalar, or the direction is invalid. + """ + with framework_ops.name_scope(name, 'sort'): + return _sort_or_argsort(values, axis, direction, return_argsort=False) + + +@tf_export('argsort') +def argsort(values, axis=-1, direction='ASCENDING', stable=False, name=None): + """Returns the indices of a tensor that give its sorted order along an axis. + + For a 1D tensor, `tf.gather(values, tf.argsort(values))` is equivalent to + `tf.sort(values)`. For higher dimensions, the output has the same shape as + `values`, but along the given axis, values represent the index of the sorted + element in that slice of the tensor at the given position. + + Args: + values: 1-D or higher numeric `Tensor`. + axis: The axis along which to sort. The default is -1, which sorts the last + axis. + direction: The direction in which to sort the values (`'ASCENDING'` or + `'DESCENDING'`). + stable: If True, equal elements in the original tensor will not be + re-ordered in the returned order. Unstable sort is not yet implemented, + but will eventually be the default for performance reasons. If you require + a stable order, pass `stable=True` for forwards compatibility. + name: Optional name for the operation. + + Returns: + An int32 `Tensor` with the same shape as `values`. The indices that would + sort each slice of the given `values` along the given `axis`. + + Raises: + ValueError: If axis is not a constant scalar, or the direction is invalid. + """ + del stable # Unused. + with framework_ops.name_scope(name, 'argsort'): + return _sort_or_argsort(values, axis, direction, return_argsort=True) + + +def _sort_or_argsort(values, axis, direction, return_argsort): + """Internal sort/argsort implementation. + + Args: + values: The input values. + axis: The axis along which to sort. + direction: 'ASCENDING' or 'DESCENDING'. + return_argsort: Whether to return the argsort result. + + Returns: + Either the sorted values, or the indices of the sorted values in the + original tensor. See the `sort` and `argsort` docstrings. + + Raises: + ValueError: If axis is not a constant scalar, or the direction is invalid. + """ + if direction not in _SORT_IMPL: + raise ValueError('%s should be one of %s' % (direction, ', '.join( + sorted(_SORT_IMPL.keys())))) + # Axis must be an integer, not a Tensor. + axis = framework_ops.convert_to_tensor(axis, name='axis') + axis_static = tensor_util.constant_value(axis) + if axis.shape.ndims != 0 or axis_static is None: + raise ValueError('axis must be a constant scalar') + axis_static = int(axis_static) # Avoids NumPy casting error + + values = framework_ops.convert_to_tensor(values, name='values') + + return _SORT_IMPL[direction](values, axis_static, return_argsort) + + +def _descending_sort(values, axis, return_argsort=False): + """Sorts values in reverse using `top_k`. + + Args: + values: Tensor of numeric values. + axis: Index of the axis which values should be sorted along. + return_argsort: If False, return the sorted values. If True, return the + indices that would sort the values. + + Returns: + The sorted values. + """ + k = array_ops.shape(values)[axis] + rank = array_ops.rank(values) + static_rank = values.shape.ndims + # Fast path: sorting the last axis. + if axis == -1 or axis + 1 == values.get_shape().ndims: + top_k_input = values + transposition = None + else: + # Otherwise, transpose the array. Swap axes `axis` and `rank - 1`. + if axis < 0: + # Calculate the actual axis index if counting from the end. Use the static + # rank if available, or else make the axis back into a tensor. + axis += static_rank or rank + if static_rank is not None: + # Prefer to calculate the transposition array in NumPy and make it a + # constant. + transposition = constant_op.constant( + np.r_[ + # Axes up to axis are unchanged. + np.arange(axis), + # Swap axis and rank - 1. + [static_rank - 1], + # Axes in [axis + 1, rank - 1) are unchanged. + np.arange(axis + 1, static_rank - 1), + # Swap axis and rank - 1. + [axis]], + name='transposition') + else: + # Generate the transposition array from the tensors. + transposition = array_ops.concat( + [ + # Axes up to axis are unchanged. + math_ops.range(axis), + # Swap axis and rank - 1. + [rank - 1], + # Axes in [axis + 1, rank - 1) are unchanged. + math_ops.range(axis + 1, rank - 1), + # Swap axis and rank - 1. + [axis] + ], + axis=0) + top_k_input = array_ops.transpose(values, transposition) + + values, indices = nn_ops.top_k(top_k_input, k) + return_value = indices if return_argsort else values + if transposition is not None: + # transposition contains a single cycle of length 2 (swapping 2 elements), + # so it is an involution (it is its own inverse). + return_value = array_ops.transpose(return_value, transposition) + return return_value + + +def _ascending_sort(values, axis, return_argsort=False): + # Negate the values to get the ascending order from descending sort. + values_or_indices = _descending_sort(-values, axis, return_argsort) + # If not argsort, negate the values again. + return values_or_indices if return_argsort else -values_or_indices + + +_SORT_IMPL = { + 'ASCENDING': _ascending_sort, + 'DESCENDING': _descending_sort, +} diff --git a/tensorflow/contrib/framework/python/ops/sort_ops_test.py b/tensorflow/python/ops/sort_ops_test.py similarity index 90% rename from tensorflow/contrib/framework/python/ops/sort_ops_test.py rename to tensorflow/python/ops/sort_ops_test.py index 791b32cd1e2..17ce604cbf1 100644 --- a/tensorflow/contrib/framework/python/ops/sort_ops_test.py +++ b/tensorflow/python/ops/sort_ops_test.py @@ -20,22 +20,25 @@ from __future__ import print_function import numpy as np -from tensorflow.contrib.framework.python.ops import sort_ops from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes from tensorflow.python.framework import errors from tensorflow.python.framework import ops from tensorflow.python.framework import tensor_util +from tensorflow.python.framework import test_util from tensorflow.python.ops import array_ops from tensorflow.python.ops import random_ops +from tensorflow.python.ops import sort_ops from tensorflow.python.platform import test class SortTest(test.TestCase): + @test_util.run_deprecated_v1 def testRandom_lowDimensionality(self): self._testRandom_lowDimensionality(negative_axis=False) + @test_util.run_deprecated_v1 def testRandom_lowDimensionality_negative(self): self._testRandom_lowDimensionality(negative_axis=True) @@ -53,6 +56,7 @@ class SortTest(test.TestCase): np.sort(arr, axis=sort_axis), sort_ops.sort(constant_op.constant(arr), axis=sort_axis).eval()) + @test_util.run_deprecated_v1 def testRandom_highDimensionality(self): np.random.seed(100) for _ in range(20): @@ -65,6 +69,7 @@ class SortTest(test.TestCase): np.sort(arr, axis=sort_axis), sort_ops.sort(constant_op.constant(arr), axis=sort_axis).eval()) + @test_util.run_deprecated_v1 def testScalar(self): # Create an empty scalar where the static shape is unknown. zeros_length_1 = array_ops.zeros( @@ -77,21 +82,22 @@ class SortTest(test.TestCase): with self.assertRaises(errors.InvalidArgumentError): sort.eval() + @test_util.run_deprecated_v1 def testNegativeOutOfBounds_staticShape(self): arr = constant_op.constant([3, 4, 5]) with self.assertRaises(ValueError): sort_ops.sort(arr, axis=-4) + @test_util.run_deprecated_v1 def testDescending(self): arr = np.random.random((10, 5, 5)) with self.cached_session(): self.assertAllEqual( np.sort(arr, axis=0)[::-1], sort_ops.sort( - constant_op.constant(arr), - axis=0, - direction='DESCENDING').eval()) + constant_op.constant(arr), axis=0, direction='DESCENDING').eval()) + @test_util.run_deprecated_v1 def testSort_staticallyKnownRank_constantTransposition(self): # The transposition array should be a constant if the rank of "values" is # statically known. @@ -109,6 +115,7 @@ class SortTest(test.TestCase): tensor_util.constant_value(transposition), [0, 4, 2, 3, 1]) + @test_util.run_deprecated_v1 def testArgsort_1d(self): arr = np.random.random(42) with self.cached_session(): @@ -116,6 +123,7 @@ class SortTest(test.TestCase): np.sort(arr), array_ops.gather(arr, sort_ops.argsort(arr)).eval()) + @test_util.run_deprecated_v1 def testArgsort(self): arr = np.random.random((5, 6, 7, 8)) for axis in range(4): diff --git a/tensorflow/python/ops/sparse_grad.py b/tensorflow/python/ops/sparse_grad.py index 1223b290ff6..2ca9c0c647d 100644 --- a/tensorflow/python/ops/sparse_grad.py +++ b/tensorflow/python/ops/sparse_grad.py @@ -195,7 +195,7 @@ def _SparseTensorDenseMatMulGrad(op, grad): parts_a = array_ops.gather(grad, rows if not adj_a else cols) parts_b = array_ops.gather(b if not adj_b else array_ops.transpose(b), cols if not adj_a else rows) - a_values_grad = math_ops.reduce_sum(parts_a * parts_b, reduction_indices=1) + a_values_grad = math_ops.reduce_sum(parts_a * parts_b, axis=1) # gradients w.r.t. (a_indices, a_values, a_shape, b) return (None, a_values_grad, None, b_grad) diff --git a/tensorflow/python/ops/sparse_ops.py b/tensorflow/python/ops/sparse_ops.py index b98c7f5f65b..346ab9c0cb4 100644 --- a/tensorflow/python/ops/sparse_ops.py +++ b/tensorflow/python/ops/sparse_ops.py @@ -44,6 +44,9 @@ from tensorflow.python.ops.gen_sparse_ops import * # pylint: enable=wildcard-import from tensorflow.python.util import compat from tensorflow.python.util import deprecation +from tensorflow.python.util import dispatch +from tensorflow.python.util import tf_inspect +from tensorflow.python.util.tf_export import get_canonical_name_for_symbol from tensorflow.python.util.tf_export import tf_export @@ -186,7 +189,7 @@ def sparse_eye(num_rows, # pylint: disable=protected-access -@tf_export("sparse.concat", "sparse_concat") +@tf_export(v1=["sparse.concat", "sparse_concat"]) @deprecation.deprecated_endpoints("sparse_concat") @deprecation.deprecated_args( None, "concat_dim is deprecated, use axis instead", "concat_dim") @@ -292,6 +295,11 @@ def sparse_concat(axis, """ axis = deprecation.deprecated_argument_lookup("axis", axis, "concat_dim", concat_dim) + return sparse_concat_v2(axis, sp_inputs, expand_nonconcat_dim, name) + + +@tf_export("sparse.concat", v1=[]) +def sparse_concat_v2(axis, sp_inputs, expand_nonconcat_dim=False, name=None): # pylint: disable=missing-docstring sp_inputs = _convert_to_sparse_tensors(sp_inputs) if len(sp_inputs) == 1: # Degenerate case of one tensor. @@ -319,9 +327,15 @@ def sparse_concat(axis, return sparse_tensor.SparseTensor(output_ind, output_val, output_shape) -@tf_export("sparse.add", v1=["sparse.add", "sparse_add"]) +sparse_concat_v2.__doc__ = sparse_concat.__doc__.replace( + " concat_dim: The old (deprecated) name for axis.\n", "") + + +@tf_export(v1=["sparse.add", "sparse_add"]) @deprecation.deprecated_endpoints("sparse_add") -def sparse_add(a, b, thresh=0): +@deprecation.deprecated_args( + None, "thresh is deprecated, use threshold instead", "thresh") +def sparse_add(a, b, threshold=None, thresh=None): """Adds two tensors, at least one of each is a `SparseTensor`. If one `SparseTensor` and one `Tensor` are passed in, returns a `Tensor`. If @@ -359,12 +373,74 @@ def sparse_add(a, b, thresh=0): Args: a: The first operand; `SparseTensor` or `Tensor`. - b: The second operand; `SparseTensor` or `Tensor`. At least one operand + b: The second operand; `SparseTensor` or `Tensor`. At least one operand must be sparse. - thresh: A 0-D `Tensor`. The magnitude threshold that determines if an - output value/index pair takes space. Its dtype should match that of the - values if they are real; if the latter are complex64/complex128, then the - dtype should be float32/float64, correspondingly. + threshold: An optional 0-D `Tensor` (defaults to `0`). The magnitude + threshold that determines if an output value/index pair takes space. Its + dtype should match that of the values if they are real; if the latter are + complex64/complex128, then the dtype should be float32/float64, + correspondingly. + thresh: Deprecated alias for `threshold`. + + Returns: + A `SparseTensor` or a `Tensor`, representing the sum. + + Raises: + TypeError: If both `a` and `b` are `Tensor`s. Use `tf.add()` instead. + """ + threshold = deprecation.deprecated_argument_lookup("threshold", threshold, + "thresh", thresh) + if threshold is None: + threshold = 0 + return sparse_add_v2(a, b, threshold) + + +@tf_export("sparse.add", v1=[]) +def sparse_add_v2(a, b, threshold=0): + """Adds two tensors, at least one of each is a `SparseTensor`. + + If one `SparseTensor` and one `Tensor` are passed in, returns a `Tensor`. If + both arguments are `SparseTensor`s, this returns a `SparseTensor`. The order + of arguments does not matter. Use vanilla `tf.add()` for adding two dense + `Tensor`s. + + The shapes of the two operands must match: broadcasting is not supported. + + The indices of any input `SparseTensor` are assumed ordered in standard + lexicographic order. If this is not the case, before this step run + `SparseReorder` to restore index ordering. + + If both arguments are sparse, we perform "clipping" as follows. By default, + if two values sum to zero at some index, the output `SparseTensor` would still + include that particular location in its index, storing a zero in the + corresponding value slot. To override this, callers can specify `threshold`, + indicating that if the sum has a magnitude strictly smaller than `threshold`, + its corresponding value and index would then not be included. In particular, + `threshold == 0.0` (default) means everything is kept and actual thresholding + happens only for a positive value. + + For example, suppose the logical sum of two sparse operands is (densified): + + [ 2] + [.1 0] + [ 6 -.2] + + Then, + + * `threshold == 0` (the default): all 5 index/value pairs will be + returned. + * `threshold == 0.11`: only .1 and 0 will vanish, and the remaining three + index/value pairs will be returned. + * `threshold == 0.21`: .1, 0, and -.2 will vanish. + + Args: + a: The first operand; `SparseTensor` or `Tensor`. + b: The second operand; `SparseTensor` or `Tensor`. At least one operand + must be sparse. + threshold: A 0-D `Tensor`. The magnitude threshold that determines if an + output value/index pair takes space. Its dtype should match that of the + values if they are real; if the latter are complex64/complex128, then the + dtype should be float32/float64, correspondingly. Returns: A `SparseTensor` or a `Tensor`, representing the sum. @@ -380,11 +456,12 @@ def sparse_add(a, b, thresh=0): if all(isinstance(inp, sparse_classes) for inp in [a, b]): a = _convert_to_sparse_tensor(a) b = _convert_to_sparse_tensor(b) - thresh = ops.convert_to_tensor( - thresh, dtype=a.values.dtype.real_dtype.base_dtype, name="thresh") + threshold = ops.convert_to_tensor( + threshold, dtype=a.values.dtype.real_dtype.base_dtype, name="threshold") output_ind, output_val, output_shape = ( gen_sparse_ops.sparse_add(a.indices, a.values, a.dense_shape, - b.indices, b.values, b.dense_shape, thresh)) + b.indices, b.values, b.dense_shape, + threshold)) # Attempt to get output_shape statically. a.get_shape().assert_is_compatible_with(b.get_shape()) @@ -705,7 +782,7 @@ class KeywordRequired(object): return "KeywordRequired()" -@tf_export("sparse.split", "sparse_split") +@tf_export(v1=["sparse.split", "sparse_split"]) @deprecation.deprecated_endpoints("sparse_split") @deprecation.deprecated_args( None, "split_dim is deprecated, use axis instead", "split_dim") @@ -779,6 +856,51 @@ def sparse_split(keyword_required=KeywordRequired(), return sparse_tensors +@tf_export("sparse.split", v1=[]) +def sparse_split_v2(sp_input=None, + num_split=None, + axis=None, + name=None): + """Split a `SparseTensor` into `num_split` tensors along `axis`. + + If the `sp_input.dense_shape[axis]` is not an integer multiple of `num_split` + each slice starting from 0:`shape[axis] % num_split` gets extra one + dimension. For example, if `axis = 1` and `num_split = 2` and the + input is: + + input_tensor = shape = [2, 7] + [ a d e ] + [b c ] + + Graphically the output tensors are: + + output_tensor[0] = + [ a ] + [b c ] + + output_tensor[1] = + [ d e ] + [ ] + + Args: + sp_input: The `SparseTensor` to split. + num_split: A Python integer. The number of ways to split. + axis: A 0-D `int32` `Tensor`. The dimension along which to split. + name: A name for the operation (optional). + + Returns: + `num_split` `SparseTensor` objects resulting from splitting `value`. + + Raises: + TypeError: If `sp_input` is not a `SparseTensor`. + """ + return sparse_split(sp_input=sp_input, + num_split=num_split, + axis=axis, + name=name, + split_dim=None) + + @tf_export("sparse.slice", v1=["sparse.slice", "sparse_slice"]) @deprecation.deprecated_endpoints("sparse_slice") def sparse_slice(sp_input, start, size, name=None): @@ -829,7 +951,7 @@ def sparse_slice(sp_input, start, size, name=None): output_shape) -@tf_export("sparse_to_dense") +@tf_export(v1=["sparse_to_dense"]) @deprecation.deprecated( None, "Create a `tf.sparse.SparseTensor` and use `tf.sparse.to_dense` instead.") @@ -888,7 +1010,86 @@ def sparse_to_dense(sparse_indices, name=name) -@tf_export("sparse.reduce_max", "sparse_reduce_max") +@tf_export("sparse.reduce_max", v1=[]) +def sparse_reduce_max_v2( + sp_input, axis=None, keepdims=None, output_is_sparse=False, name=None): + """Computes the max of elements across dimensions of a SparseTensor. + + This Op takes a SparseTensor and is the sparse counterpart to + `tf.reduce_max()`. In particular, this Op also returns a dense `Tensor` + if `output_is_sparse` is `False`, or a `SparseTensor` if `output_is_sparse` + is `True`. + + Note: A gradient is not defined for this function, so it can't be used + in training models that need gradient descent. + + Reduces `sp_input` along the dimensions given in `axis`. Unless + `keepdims` is true, the rank of the tensor is reduced by 1 for each entry in + `axis`. If `keepdims` is true, the reduced dimensions are retained + with length 1. + + If `axis` has no entries, all dimensions are reduced, and a tensor + with a single element is returned. Additionally, the axes can be negative, + similar to the indexing rules in Python. + + The values not defined in `sp_input` don't participate in the reduce max, + as opposed to be implicitly assumed 0 -- hence it can return negative values + for sparse `axis`. But, in case there are no values in + `axis`, it will reduce to 0. See second example below. + + For example: + + ```python + # 'x' represents [[1, ?, 2] + # [?, 3, ?]] + # where ? is implicitly-zero. + tf.sparse.reduce_max(x) ==> 3 + tf.sparse.reduce_max(x, 0) ==> [1, 3, 2] + tf.sparse.reduce_max(x, 1) ==> [2, 3] # Can also use -1 as the axis. + tf.sparse.reduce_max(x, 1, keepdims=True) ==> [[2], [3]] + tf.sparse.reduce_max(x, [0, 1]) ==> 3 + + # 'y' represents [[-7, ?] + # [ 4, 3] + # [ ?, ?] + tf.sparse.reduce_max(x, 1) ==> [-7, 4, 0] + ``` + + Args: + sp_input: The SparseTensor to reduce. Should have numeric type. + axis: The dimensions to reduce; list or scalar. If `None` (the + default), reduces all dimensions. + keepdims: If true, retain reduced dimensions with length 1. + output_is_sparse: If true, returns a `SparseTensor` instead of a dense + `Tensor` (the default). + name: A name for the operation (optional). + + Returns: + The reduced Tensor or the reduced SparseTensor if `output_is_sparse` is + True. + """ + if keepdims is None: + keepdims = False + + # reduction_axes is the deprecated name for axis. + reduction_axes = None + + if output_is_sparse: + output_ind, output_val, output_shape = ( + gen_sparse_ops.sparse_reduce_max_sparse( + sp_input.indices, sp_input.values, sp_input.dense_shape, + math_ops._ReductionDims(sp_input, axis, reduction_axes), keepdims, + name=name)) + + return sparse_tensor.SparseTensor(output_ind, output_val, output_shape) + + return gen_sparse_ops.sparse_reduce_max( + sp_input.indices, sp_input.values, sp_input.dense_shape, + math_ops._ReductionDims(sp_input, axis, reduction_axes), keepdims, + name=name) + + +@tf_export(v1=["sparse.reduce_max", "sparse_reduce_max"]) @deprecation.deprecated_endpoints("sparse_reduce_max") @deprecation.deprecated_args( None, "keep_dims is deprecated, use keepdims instead", "keep_dims") @@ -956,7 +1157,7 @@ def sparse_reduce_max(sp_input, axis=None, keepdims=None, math_ops._ReductionDims(sp_input, axis, reduction_axes), keepdims) -@tf_export("sparse.reduce_max_sparse", "sparse_reduce_max_sparse") +@tf_export(v1=["sparse.reduce_max_sparse", "sparse_reduce_max_sparse"]) @deprecation.deprecated_endpoints("sparse_reduce_max_sparse") @deprecation.deprecated_args( None, "keep_dims is deprecated, use keepdims instead", "keep_dims") @@ -1007,7 +1208,74 @@ def sparse_reduce_max_sparse(sp_input, return sparse_tensor.SparseTensor(output_ind, output_val, output_shape) -@tf_export("sparse.reduce_sum", "sparse_reduce_sum") +@tf_export("sparse.reduce_sum", v1=[]) +def sparse_reduce_sum_v2( + sp_input, axis=None, keepdims=None, output_is_sparse=False, name=None): + """Computes the sum of elements across dimensions of a SparseTensor. + + This Op takes a SparseTensor and is the sparse counterpart to + `tf.reduce_sum()`. In particular, this Op also returns a dense `Tensor` + if `output_is_sparse` is `False`, or a `SparseTensor` if `output_is_sparse` + is `True`. + + Note: if `output_is_sparse` is True, a gradient is not defined for this + function, so it can't be used in training models that need gradient descent. + + Reduces `sp_input` along the dimensions given in `axis`. Unless `keepdims` is + true, the rank of the tensor is reduced by 1 for each entry in `axis`. If + `keepdims` is true, the reduced dimensions are retained with length 1. + + If `axis` has no entries, all dimensions are reduced, and a tensor + with a single element is returned. Additionally, the axes can be negative, + similar to the indexing rules in Python. + + For example: + + ```python + # 'x' represents [[1, ?, 1] + # [?, 1, ?]] + # where ? is implicitly-zero. + tf.sparse.reduce_sum(x) ==> 3 + tf.sparse.reduce_sum(x, 0) ==> [1, 1, 1] + tf.sparse.reduce_sum(x, 1) ==> [2, 1] # Can also use -1 as the axis. + tf.sparse.reduce_sum(x, 1, keepdims=True) ==> [[2], [1]] + tf.sparse.reduce_sum(x, [0, 1]) ==> 3 + ``` + + Args: + sp_input: The SparseTensor to reduce. Should have numeric type. + axis: The dimensions to reduce; list or scalar. If `None` (the + default), reduces all dimensions. + keepdims: If true, retain reduced dimensions with length 1. + output_is_sparse: If true, returns a `SparseTensor` instead of a dense + `Tensor` (the default). + name: A name for the operation (optional). + + Returns: + The reduced Tensor or the reduced SparseTensor if `output_is_sparse` is + True. + """ + if keepdims is None: + keepdims = False + + # reduction_axes is the deprecated name for axis. + reduction_axes = None + + if output_is_sparse: + output_ind, output_val, output_shape = ( + gen_sparse_ops.sparse_reduce_sum_sparse( + sp_input.indices, sp_input.values, sp_input.dense_shape, + math_ops._ReductionDims(sp_input, axis, reduction_axes), keepdims, + name=name)) + return sparse_tensor.SparseTensor(output_ind, output_val, output_shape) + + return gen_sparse_ops.sparse_reduce_sum( + sp_input.indices, sp_input.values, sp_input.dense_shape, + math_ops._ReductionDims(sp_input, axis, reduction_axes), keepdims, + name=name) + + +@tf_export(v1=["sparse.reduce_sum", "sparse_reduce_sum"]) @deprecation.deprecated_endpoints("sparse_reduce_sum") @deprecation.deprecated_args( None, "keep_dims is deprecated, use keepdims instead", "keep_dims") @@ -1062,7 +1330,7 @@ def sparse_reduce_sum(sp_input, axis=None, keepdims=None, math_ops._ReductionDims(sp_input, axis, reduction_axes), keepdims) -@tf_export("sparse.reduce_sum_sparse", "sparse_reduce_sum_sparse") +@tf_export(v1=["sparse.reduce_sum_sparse", "sparse_reduce_sum_sparse"]) @deprecation.deprecated_endpoints("sparse_reduce_sum_sparse") @deprecation.deprecated_args( None, "keep_dims is deprecated, use keepdims instead", "keep_dims") @@ -1157,7 +1425,7 @@ def sparse_tensor_to_dense(sp_input, """ sp_input = _convert_to_sparse_tensor(sp_input) - return sparse_to_dense( + return gen_sparse_ops.sparse_to_dense( sp_input.indices, sp_input.dense_shape, sp_input.values, @@ -1231,8 +1499,8 @@ def sparse_to_indicator(sp_input, vocab_size, name=None): sp_new, default_value=False, validate_indices=False, name=name) -@tf_export("sparse.merge", v1=["sparse.merge", "sparse_merge"]) -@deprecation.deprecated_endpoints("sparse_merge") +@tf_export(v1=["sparse.merge", "sparse_merge"]) +@deprecation.deprecated(None, "No similar op available at this time.") def sparse_merge(sp_ids, sp_values, vocab_size, name=None, already_sorted=False): """Combines a batch of feature ids and values into a single `SparseTensor`. @@ -1593,8 +1861,7 @@ def sparse_fill_empty_rows(sp_input, default_value, name=None): dense_shape=sp_input.dense_shape), empty_row_indicator) -@tf_export( - "io.serialize_sparse", v1=["io.serialize_sparse", "serialize_sparse"]) +@tf_export(v1=["io.serialize_sparse", "serialize_sparse"]) @deprecation.deprecated_endpoints("serialize_sparse") def serialize_sparse(sp_input, name=None, out_type=dtypes.string): """Serialize a `SparseTensor` into a 3-vector (1-D `Tensor`) object. @@ -1608,6 +1875,25 @@ def serialize_sparse(sp_input, name=None, out_type=dtypes.string): A 3-vector (1-D `Tensor`), with each column representing the serialized `SparseTensor`'s indices, values, and shape (respectively). + Raises: + TypeError: If `sp_input` is not a `SparseTensor`. + """ + return serialize_sparse_v2(sp_input, out_type, name) + + +@tf_export("io.serialize_sparse", v1=[]) +def serialize_sparse_v2(sp_input, out_type=dtypes.string, name=None): + """Serialize a `SparseTensor` into a 3-vector (1-D `Tensor`) object. + + Args: + sp_input: The input `SparseTensor`. + out_type: The `dtype` to use for serialization. + name: A name prefix for the returned tensors (optional). + + Returns: + A 3-vector (1-D `Tensor`), with each column representing the serialized + `SparseTensor`'s indices, values, and shape (respectively). + Raises: TypeError: If `sp_input` is not a `SparseTensor`. """ @@ -1621,9 +1907,7 @@ def serialize_sparse(sp_input, name=None, out_type=dtypes.string): out_type=out_type) -@tf_export( - "io.serialize_many_sparse", - v1=["io.serialize_many_sparse", "serialize_many_sparse"]) +@tf_export(v1=["io.serialize_many_sparse", "serialize_many_sparse"]) @deprecation.deprecated_endpoints("serialize_many_sparse") def serialize_many_sparse(sp_input, name=None, out_type=dtypes.string): """Serialize `N`-minibatch `SparseTensor` into an `[N, 3]` `Tensor`. @@ -1646,6 +1930,34 @@ def serialize_many_sparse(sp_input, name=None, out_type=dtypes.string): represents serialized `SparseTensor`'s indices, values, and shape (respectively). + Raises: + TypeError: If `sp_input` is not a `SparseTensor`. + """ + return serialize_many_sparse_v2(sp_input, out_type, name) + + +@tf_export("io.serialize_many_sparse", v1=[]) +def serialize_many_sparse_v2(sp_input, out_type=dtypes.string, name=None): + """Serialize `N`-minibatch `SparseTensor` into an `[N, 3]` `Tensor`. + + The `SparseTensor` must have rank `R` greater than 1, and the first dimension + is treated as the minibatch dimension. Elements of the `SparseTensor` + must be sorted in increasing order of this first dimension. The serialized + `SparseTensor` objects going into each row of the output `Tensor` will have + rank `R-1`. + + The minibatch size `N` is extracted from `sparse_shape[0]`. + + Args: + sp_input: The input rank `R` `SparseTensor`. + out_type: The `dtype` to use for serialization. + name: A name prefix for the returned tensors (optional). + + Returns: + A matrix (2-D `Tensor`) with `N` rows and `3` columns. Each column + represents serialized `SparseTensor`'s indices, values, and shape + (respectively). + Raises: TypeError: If `sp_input` is not a `SparseTensor`. """ @@ -1798,7 +2110,9 @@ def deserialize_many_sparse(serialized_sparse, dtype, rank=None, name=None): return sparse_tensor.SparseTensor(output_indices, output_values, output_shape) -@tf_export("sparse.matmul", v1=["sparse.matmul", "sparse_tensor_dense_matmul"]) +@tf_export("sparse.sparse_dense_matmul", + v1=["sparse.sparse_dense_matmul", "sparse.matmul", + "sparse_tensor_dense_matmul"]) @deprecation.deprecated_endpoints("sparse_tensor_dense_matmul") def sparse_tensor_dense_matmul(sp_a, b, @@ -2362,3 +2676,47 @@ def _take_many_sparse_from_tensors_map(sparse_map_op, output_shape.set_shape([rank]) return sparse_tensor.SparseTensor(output_indices, output_values, output_shape) + + +class _UnaryMapValueDispatcher(dispatch.OpDispatcher): + """OpDispatcher for unary ops that maps base function across sparse values.""" + + def __init__(self, original_func): + self._original_func = original_func + func_name = get_canonical_name_for_symbol(original_func) + arg_names = tf_inspect.getfullargspec(original_func)[0] + self._x = arg_names[0] + original_func.__doc__ = ( + original_func.__doc__.rstrip() + "\n\n" + + (" If `{x}` is a `SparseTensor`, returns\n" + " `SparseTensor({x}.indices, tf.{func}({x}.values, ...), " + "{x}.dense_shape)`").format(x=self._x, func=func_name)) + + def handle(self, args, kwargs): + if args: + x, args = args[0], args[1:] + else: + x = kwargs.pop(self._x, None) + if isinstance(x, sparse_tensor.SparseTensor): + return sparse_tensor.SparseTensor( + indices=x.indices, + values=self._original_func(x.values, *args, **kwargs), + dense_shape=x.dense_shape) + else: + return self.NOT_SUPPORTED + + +_UNARY_OPS = [ + # TODO(b/120307967) Add dispatchers for additional TensorFlow ops. + math_ops.abs, + math_ops.negative, + math_ops.sign, + math_ops.square, + math_ops.sqrt, + math_ops.erf, + math_ops.tanh, + math_ops.bessel_i0e, + math_ops.bessel_i1e, +] +for unary_op in _UNARY_OPS: + _UnaryMapValueDispatcher(unary_op).register(unary_op) diff --git a/tensorflow/python/ops/sparse_ops_test.py b/tensorflow/python/ops/sparse_ops_test.py index 4ee1569249b..031069a0f01 100644 --- a/tensorflow/python/ops/sparse_ops_test.py +++ b/tensorflow/python/ops/sparse_ops_test.py @@ -18,18 +18,20 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function +from absl.testing import parameterized import numpy as np from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes from tensorflow.python.framework import sparse_tensor from tensorflow.python.framework import test_util +from tensorflow.python.ops import math_ops from tensorflow.python.ops import sparse_ops from tensorflow.python.platform import googletest @test_util.run_all_in_graph_and_eager_modes -class SparseOpsTest(test_util.TensorFlowTestCase): +class SparseOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase): def testSparseEye(self): def test_one(n, m, as_tensors): @@ -77,5 +79,23 @@ class SparseOpsTest(test_util.TensorFlowTestCase): d = sparse_ops.sparse_to_dense(s.indices, s.dense_shape, s.values) self.assertAllEqual(self.evaluate(d), expected_after) + @parameterized.parameters([ + (math_ops.abs, [1.0, -1.0, 3.0, -4.0], [1.0, 1.0, 3.0, 4.0]), + (math_ops.negative, [1.0, -1.0, 3.0, -4.0], [-1.0, 1.0, -3.0, 4.0]), + (math_ops.sign, [3.0, -2.0, 0.0, -4.0], [1.0, -1.0, 0.0, -1.0]), + (math_ops.square, [1.0, -1.0, 3.0, -4.0], [1.0, 1.0, 9.0, 16.0]), + ]) + def testUnarySparseDispatch(self, op, values, expected): + st = sparse_tensor.SparseTensor( + indices=[[0, 0], [0, 1], [2, 0], [2, 4]], + values=values, + dense_shape=[3, 6]) + result = op(st) + result_value = self.evaluate(result) + self.assertAllEqual(result_value.indices, st.indices) + self.assertAllEqual(result_value.values, expected) + self.assertAllEqual(result_value.dense_shape, st.dense_shape) + + if __name__ == '__main__': googletest.main() diff --git a/tensorflow/python/ops/special_math_ops.py b/tensorflow/python/ops/special_math_ops.py index f44f694109e..21f4996798e 100644 --- a/tensorflow/python/ops/special_math_ops.py +++ b/tensorflow/python/ops/special_math_ops.py @@ -70,8 +70,7 @@ def lbeta(x, name=None): x = ops.convert_to_tensor(x, name='x') # Note reduce_sum([]) = 0. - log_prod_gamma_x = math_ops.reduce_sum( - math_ops.lgamma(x), reduction_indices=[-1]) + log_prod_gamma_x = math_ops.reduce_sum(math_ops.lgamma(x), axis=[-1]) # Note lgamma(0) = infinity, so if x = [] # log_gamma_sum_x = lgamma(0) = infinity, and @@ -264,11 +263,11 @@ def einsum(equation, *inputs, **kwargs): missing_indices = set(temp_axis_labels) - set(output_axis_labels) if missing_indices: - reduction_indices = [ + axis = [ i for i, a in enumerate(temp_axis_labels) if a not in output_axis_labels ] - temp = math_ops.reduce_sum(temp, reduction_indices=reduction_indices) + temp = math_ops.reduce_sum(temp, axis=axis) temp_axis_labels = ''.join( a for a in temp_axis_labels if a in output_axis_labels) diff --git a/tensorflow/python/ops/special_math_ops_test.py b/tensorflow/python/ops/special_math_ops_test.py index 7438cdb3f11..94aaebed951 100644 --- a/tensorflow/python/ops/special_math_ops_test.py +++ b/tensorflow/python/ops/special_math_ops_test.py @@ -46,6 +46,7 @@ class LBetaTest(test.TestCase): 0.5, self.evaluate(math_ops.exp(special_math_ops.lbeta(x_one_half)))) self.assertEqual([], special_math_ops.lbeta(x_one).get_shape()) + @test_util.run_deprecated_v1 def test_one_dimensional_arg_dynamic(self): # Should evaluate to 1 and 1/2. x_one = [1, 1.] @@ -57,6 +58,7 @@ class LBetaTest(test.TestCase): self.assertAllClose(0.5, beta_ph.eval(feed_dict={ph: x_one_half})) + @test_util.run_deprecated_v1 def test_four_dimensional_arg_with_partial_shape_dynamic(self): x_ = np.ones((3, 2, 3, 4)) # Gamma(1) = 0! = 1 @@ -81,6 +83,7 @@ class LBetaTest(test.TestCase): self.evaluate(math_ops.exp(special_math_ops.lbeta(x_one_half)))) self.assertEqual((2,), special_math_ops.lbeta(x_one_half).get_shape()) + @test_util.run_deprecated_v1 def test_two_dimensional_arg_dynamic(self): # Should evaluate to 1/2. x_one_half = [[2, 1.], [2, 1.]] @@ -288,6 +291,7 @@ class EinsumTest(test.TestCase): for case in self.long_cases: self.run_test(case) + @test_util.run_deprecated_v1 def test_invalid(self): for axes in self.invalid_cases: inputs = [ @@ -297,6 +301,7 @@ class EinsumTest(test.TestCase): with self.assertRaises(ValueError): _ = special_math_ops.einsum(axes, *inputs) + @test_util.run_deprecated_v1 def test_invalid_keyword_arguments(self): m0 = array_ops.placeholder(dtypes.int32, shape=(1, None)) m1 = array_ops.placeholder(dtypes.int32, shape=(None, 1)) @@ -311,11 +316,13 @@ class EinsumTest(test.TestCase): invalid1='value1', invalid2='value2') + @test_util.run_deprecated_v1 def test_repeated_axis_single_input(self): x = array_ops.placeholder(dtypes.float32, shape=[2, 2]) with self.assertRaises(ValueError): _ = special_math_ops.einsum('ii->', x) + @test_util.run_deprecated_v1 def test_dim_mismatch(self): for axes, input_shapes in self.dim_mismatch_cases: inputs = [ diff --git a/tensorflow/python/ops/spectral_grad.py b/tensorflow/python/ops/spectral_grad.py deleted file mode 100644 index 0af24114acb..00000000000 --- a/tensorflow/python/ops/spectral_grad.py +++ /dev/null @@ -1,185 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Gradients for operators defined in spectral_ops.py.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np - -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import spectral_ops - - -def _FFTSizeForGrad(grad, rank): - return math_ops.reduce_prod(array_ops.shape(grad)[-rank:]) - - -@ops.RegisterGradient("FFT") -def _FFTGrad(_, grad): - size = math_ops.cast(_FFTSizeForGrad(grad, 1), grad.dtype) - return spectral_ops.ifft(grad) * size - - -@ops.RegisterGradient("IFFT") -def _IFFTGrad(_, grad): - rsize = math_ops.cast( - 1. / math_ops.cast(_FFTSizeForGrad(grad, 1), grad.dtype.real_dtype), - grad.dtype) - return spectral_ops.fft(grad) * rsize - - -@ops.RegisterGradient("FFT2D") -def _FFT2DGrad(_, grad): - size = math_ops.cast(_FFTSizeForGrad(grad, 2), grad.dtype) - return spectral_ops.ifft2d(grad) * size - - -@ops.RegisterGradient("IFFT2D") -def _IFFT2DGrad(_, grad): - rsize = math_ops.cast( - 1. / math_ops.cast(_FFTSizeForGrad(grad, 2), grad.dtype.real_dtype), - grad.dtype) - return spectral_ops.fft2d(grad) * rsize - - -@ops.RegisterGradient("FFT3D") -def _FFT3DGrad(_, grad): - size = math_ops.cast(_FFTSizeForGrad(grad, 3), grad.dtype) - return spectral_ops.ifft3d(grad) * size - - -@ops.RegisterGradient("IFFT3D") -def _IFFT3DGrad(_, grad): - rsize = math_ops.cast( - 1. / math_ops.cast(_FFTSizeForGrad(grad, 3), grad.dtype.real_dtype), - grad.dtype) - return spectral_ops.fft3d(grad) * rsize - - -def _RFFTGradHelper(rank, irfft_fn): - """Returns a gradient function for an RFFT of the provided rank.""" - # Can't happen because we don't register a gradient for RFFT3D. - assert rank in (1, 2), "Gradient for RFFT3D is not implemented." - - def _Grad(op, grad): - """A gradient function for RFFT with the provided `rank` and `irfft_fn`.""" - fft_length = op.inputs[1] - input_shape = array_ops.shape(op.inputs[0]) - is_even = math_ops.cast(1 - (fft_length[-1] % 2), dtypes.complex64) - - def _TileForBroadcasting(matrix, t): - expanded = array_ops.reshape( - matrix, - array_ops.concat([ - array_ops.ones([array_ops.rank(t) - 2], dtypes.int32), - array_ops.shape(matrix) - ], 0)) - return array_ops.tile( - expanded, array_ops.concat([array_ops.shape(t)[:-2], [1, 1]], 0)) - - def _MaskMatrix(length): - # TODO(rjryan): Speed up computation of twiddle factors using the - # following recurrence relation and cache them across invocations of RFFT. - # - # t_n = exp(sqrt(-1) * pi * n^2 / line_len) - # for n = 0, 1,..., line_len-1. - # For n > 2, use t_n = t_{n-1}^2 / t_{n-2} * t_1^2 - a = array_ops.tile( - array_ops.expand_dims(math_ops.range(length), 0), (length, 1)) - b = array_ops.transpose(a, [1, 0]) - return math_ops.exp(-2j * np.pi * math_ops.cast(a * b, dtypes.complex64) / - math_ops.cast(length, dtypes.complex64)) - - def _YMMask(length): - """A sequence of [1+0j, -1+0j, 1+0j, -1+0j, ...] with length `length`.""" - return math_ops.cast(1 - 2 * (math_ops.range(length) % 2), - dtypes.complex64) - - y0 = grad[..., 0:1] - if rank == 1: - ym = grad[..., -1:] - extra_terms = y0 + is_even * ym * _YMMask(input_shape[-1]) - elif rank == 2: - # Create a mask matrix for y0 and ym. - base_mask = _MaskMatrix(input_shape[-2]) - - # Tile base_mask to match y0 in shape so that we can batch-matmul the - # inner 2 dimensions. - tiled_mask = _TileForBroadcasting(base_mask, y0) - - y0_term = math_ops.matmul(tiled_mask, math_ops.conj(y0)) - extra_terms = y0_term - - ym = grad[..., -1:] - ym_term = math_ops.matmul(tiled_mask, math_ops.conj(ym)) - - inner_dim = input_shape[-1] - ym_term = array_ops.tile( - ym_term, - array_ops.concat([ - array_ops.ones([array_ops.rank(grad) - 1], dtypes.int32), - [inner_dim] - ], 0)) * _YMMask(inner_dim) - - extra_terms += is_even * ym_term - - # The gradient of RFFT is the IRFFT of the incoming gradient times a scaling - # factor, plus some additional terms to make up for the components dropped - # due to Hermitian symmetry. - input_size = math_ops.to_float(_FFTSizeForGrad(op.inputs[0], rank)) - irfft = irfft_fn(grad, fft_length) - return 0.5 * (irfft * input_size + math_ops.real(extra_terms)), None - - return _Grad - - -def _IRFFTGradHelper(rank, rfft_fn): - """Returns a gradient function for an IRFFT of the provided rank.""" - # Can't happen because we don't register a gradient for IRFFT3D. - assert rank in (1, 2), "Gradient for IRFFT3D is not implemented." - - def _Grad(op, grad): - """A gradient function for IRFFT with the provided `rank` and `rfft_fn`.""" - # Generate a simple mask like [1.0, 2.0, ..., 2.0, 1.0] for even-length FFTs - # and [1.0, 2.0, ..., 2.0] for odd-length FFTs. To reduce extra ops in the - # graph we special-case the situation where the FFT length and last - # dimension of the input are known at graph construction time. - fft_length = op.inputs[1] - is_odd = math_ops.mod(fft_length[-1], 2) - input_last_dimension = array_ops.shape(op.inputs[0])[-1] - mask = array_ops.concat( - [[1.0], 2.0 * array_ops.ones([input_last_dimension - 2 + is_odd]), - array_ops.ones([1 - is_odd])], 0) - - rsize = math_ops.reciprocal(math_ops.to_float(_FFTSizeForGrad(grad, rank))) - - # The gradient of IRFFT is the RFFT of the incoming gradient times a scaling - # factor and a mask. The mask scales the gradient for the Hermitian - # symmetric components of the RFFT by a factor of two, since these - # components are de-duplicated in the RFFT. - rfft = rfft_fn(grad, fft_length) - return rfft * math_ops.cast(rsize * mask, dtypes.complex64), None - - return _Grad - - -ops.RegisterGradient("RFFT")(_RFFTGradHelper(1, spectral_ops.irfft)) -ops.RegisterGradient("IRFFT")(_IRFFTGradHelper(1, spectral_ops.rfft)) -ops.RegisterGradient("RFFT2D")(_RFFTGradHelper(2, spectral_ops.irfft2d)) -ops.RegisterGradient("IRFFT2D")(_IRFFTGradHelper(2, spectral_ops.rfft2d)) diff --git a/tensorflow/python/ops/standard_ops.py b/tensorflow/python/ops/standard_ops.py index 4f1662ab086..c614d072bad 100644 --- a/tensorflow/python/ops/standard_ops.py +++ b/tensorflow/python/ops/standard_ops.py @@ -31,7 +31,6 @@ from tensorflow.python.ops import manip_grad from tensorflow.python.ops import math_grad from tensorflow.python.ops import random_grad from tensorflow.python.ops import sparse_grad -from tensorflow.python.ops import spectral_grad from tensorflow.python.ops import state_grad from tensorflow.python.ops import tensor_array_grad @@ -51,6 +50,7 @@ from tensorflow.python.ops.control_flow_ops import group from tensorflow.python.ops.control_flow_ops import no_op from tensorflow.python.ops.control_flow_ops import tuple # pylint: disable=redefined-builtin # pylint: enable=redefined-builtin +from tensorflow.python.eager import wrap_function from tensorflow.python.ops.control_flow_ops import while_loop from tensorflow.python.ops.data_flow_ops import * from tensorflow.python.ops.functional_ops import * @@ -72,6 +72,7 @@ from tensorflow.python.ops.partitioned_variables import * from tensorflow.python.ops.random_ops import * from tensorflow.python.ops.script_ops import py_func from tensorflow.python.ops.session_ops import * +from tensorflow.python.ops.sort_ops import * from tensorflow.python.ops.sparse_ops import * from tensorflow.python.ops.state_ops import assign from tensorflow.python.ops.state_ops import assign_add diff --git a/tensorflow/python/ops/stateless_random_ops.py b/tensorflow/python/ops/stateless_random_ops.py index c6defabacdb..b119049b163 100644 --- a/tensorflow/python/ops/stateless_random_ops.py +++ b/tensorflow/python/ops/stateless_random_ops.py @@ -24,6 +24,7 @@ from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops from tensorflow.python.ops import random_ops from tensorflow.python.ops import math_ops +from tensorflow.python.util import deprecation from tensorflow.python.util.tf_export import tf_export ops.NotDifferentiable("StatelessMultinomial") @@ -179,7 +180,9 @@ def stateless_truncated_normal(shape, return math_ops.add(rnd * stddev, mean, name=name) -@tf_export("random.stateless_multinomial") +@tf_export(v1=["random.stateless_multinomial"]) +@deprecation.deprecated( + date=None, instructions="Use tf.random.stateless_categorical instead.") def stateless_multinomial(logits, num_samples, seed, @@ -207,13 +210,58 @@ def stateless_multinomial(logits, `[i, :]` represents the unnormalized log-probabilities for all classes. num_samples: 0-D. Number of independent samples to draw for each row slice. seed: A shape [2] integer Tensor of seeds to the random number generator. - name: Optional name for the operation. output_dtype: integer type to use for the output. Defaults to int64. + name: Optional name for the operation. Returns: The drawn samples of shape `[batch_size, num_samples]`. """ with ops.name_scope(name, "stateless_multinomial", [logits, seed]): - logits = ops.convert_to_tensor(logits, name="logits") - return gen_stateless_random_ops.stateless_multinomial( - logits, num_samples, seed, output_dtype=output_dtype) + return stateless_multinomial_categorical_impl(logits, num_samples, + output_dtype, seed) + + +@tf_export("random.stateless_categorical") +def stateless_categorical(logits, + num_samples, + seed, + dtype=dtypes.int64, + name=None): + """Draws deterministic pseudorandom samples from a categorical distribution. + + This is a stateless version of `tf.categorical`: if run twice with the + same seeds, it will produce the same pseudorandom numbers. The output is + consistent across multiple runs on the same hardware (and between CPU + and GPU), but may change between versions of TensorFlow or on non-CPU/GPU + hardware. + + Example: + + ```python + # samples has shape [1, 5], where each value is either 0 or 1 with equal + # probability. + samples = tf.random.stateless_categorical( + tf.log([[10., 10.]]), 5, seed=[7, 17]) + ``` + + Args: + logits: 2-D Tensor with shape `[batch_size, num_classes]`. Each slice + `[i, :]` represents the unnormalized log-probabilities for all classes. + num_samples: 0-D. Number of independent samples to draw for each row slice. + seed: A shape [2] integer Tensor of seeds to the random number generator. + dtype: integer type to use for the output. Defaults to int64. + name: Optional name for the operation. + + Returns: + The drawn samples of shape `[batch_size, num_samples]`. + """ + with ops.name_scope(name, "stateless_categorical", [logits, seed]): + return stateless_multinomial_categorical_impl(logits, num_samples, dtype, + seed) + + +def stateless_multinomial_categorical_impl(logits, num_samples, dtype, seed): + """Implementation for stateless multinomial/categorical ops (v1/v2).""" + logits = ops.convert_to_tensor(logits, name="logits") + return gen_stateless_random_ops.stateless_multinomial( + logits, num_samples, seed, output_dtype=dtype) diff --git a/tensorflow/python/ops/string_ops.py b/tensorflow/python/ops/string_ops.py index 25e86cadeb6..b6b329c4865 100644 --- a/tensorflow/python/ops/string_ops.py +++ b/tensorflow/python/ops/string_ops.py @@ -28,6 +28,7 @@ from tensorflow.python.framework import ops from tensorflow.python.framework import sparse_tensor from tensorflow.python.framework import tensor_util from tensorflow.python.ops import array_ops +from tensorflow.python.ops import gen_parsing_ops from tensorflow.python.ops import gen_string_ops from tensorflow.python.ops import math_ops @@ -311,7 +312,7 @@ def _reduce_join_reduction_dims(x, axis, reduction_indices): return math_ops.range(array_ops.rank(x) - 1, -1, -1) -@tf_export("strings.reduce_join", v1=["strings.reduce_join", "reduce_join"]) +@tf_export(v1=["strings.reduce_join", "reduce_join"]) @deprecation.deprecated_endpoints("reduce_join") def reduce_join(inputs, axis=None, # pylint: disable=missing-docstring keep_dims=False, @@ -329,6 +330,17 @@ def reduce_join(inputs, axis=None, # pylint: disable=missing-docstring name=name) +@tf_export("strings.reduce_join", v1=[]) +def reduce_join_v2( # pylint: disable=missing-docstring + inputs, + axis=None, + keepdims=False, + separator="", + name=None): + return reduce_join( + inputs, axis, keep_dims=keepdims, separator=separator, name=name) + + reduce_join.__doc__ = deprecation.rewrite_argument_docstring( gen_string_ops.reduce_join.__doc__, "reduction_indices", "axis") reduce_join.__doc__ = reduce_join.__doc__.replace("tf.reduce_join(", @@ -337,10 +349,14 @@ reduce_join.__doc__ = reduce_join.__doc__.replace("tf.reduce_join(", # This wrapper provides backwards compatibility for code that predates the # unit argument and that passed 'name' as a positional argument. -@tf_export("strings.length") +@tf_export(v1=["strings.length"]) def string_length(input, name=None, unit="BYTE"): return gen_string_ops.string_length(input, unit=unit, name=name) +@tf_export("strings.length", v1=[]) +def string_length_v2(input, unit="BYTE", name=None): + return string_length(input, name, unit) + string_length.__doc__ = gen_string_ops.string_length.__doc__ @@ -353,11 +369,16 @@ def substr_deprecated(input, pos, len, name=None, unit="BYTE"): substr_deprecated.__doc__ = gen_string_ops.substr.__doc__ -@tf_export("strings.substr") +@tf_export(v1=["strings.substr"]) def substr(input, pos, len, name=None, unit="BYTE"): return gen_string_ops.substr(input, pos, len, unit=unit, name=name) +@tf_export("strings.substr", v1=[]) +def substr_v2(input, pos, len, unit="BYTE", name=None): + return substr(input, pos, len, name=name, unit=unit) + + substr.__doc__ = gen_string_ops.substr.__doc__ @@ -371,3 +392,53 @@ ops.NotDifferentiable("StringSplit") ops.NotDifferentiable("AsString") ops.NotDifferentiable("EncodeBase64") ops.NotDifferentiable("DecodeBase64") + + +@tf_export("strings.to_number", v1=[]) +def string_to_number(input, out_type=dtypes.float32, name=None): + r"""Converts each string in the input Tensor to the specified numeric type. + + (Note that int32 overflow results in an error while float overflow + results in a rounded value.) + + Args: + input: A `Tensor` of type `string`. + out_type: An optional `tf.DType` from: `tf.float32, tf.float64, tf.int32, + tf.int64`. Defaults to `tf.float32`. + The numeric type to interpret each string in `string_tensor` as. + name: A name for the operation (optional). + + Returns: + A `Tensor` of type `out_type`. + """ + return gen_parsing_ops.string_to_number(input, out_type, name) +tf_export(v1=["strings.to_number", "string_to_number"])( + gen_parsing_ops.string_to_number + ) + + +@tf_export("strings.to_hash_bucket", v1=[]) +def string_to_hash_bucket(input, num_buckets, name=None): + # pylint: disable=line-too-long + r"""Converts each string in the input Tensor to its hash mod by a number of buckets. + + The hash function is deterministic on the content of the string within the + process. + + Note that the hash function may change from time to time. + This functionality will be deprecated and it's recommended to use + `tf.string_to_hash_bucket_fast()` or `tf.string_to_hash_bucket_strong()`. + + Args: + input: A `Tensor` of type `string`. + num_buckets: An `int` that is `>= 1`. The number of buckets. + name: A name for the operation (optional). + + Returns: + A `Tensor` of type `int64`. + """ + # pylint: enable=line-too-long + return gen_string_ops.string_to_hash_bucket(input, num_buckets, name) +tf_export(v1=["strings.to_hash_bucket", "string_to_hash_bucket"])( + gen_string_ops.string_to_hash_bucket + ) diff --git a/tensorflow/python/ops/summary_op_util.py b/tensorflow/python/ops/summary_op_util.py index 14aa44a9208..c72a9aefc3f 100644 --- a/tensorflow/python/ops/summary_op_util.py +++ b/tensorflow/python/ops/summary_op_util.py @@ -22,6 +22,7 @@ import contextlib import re from tensorflow.python.framework import ops +from tensorflow.python.framework import tensor_util from tensorflow.python.platform import tf_logging from tensorflow.python.training import distribution_strategy_context @@ -44,13 +45,27 @@ _INVALID_TAG_CHARACTERS = re.compile(r'[^-/\w\.]') def skip_summary(): - # If using multiple replicas in distributed strategy, skip summaries on all - # replicas except the first one (replica_id=0). + """Determines if summary should be skipped. + + If using multiple replicas in distributed strategy, skip summaries on all + replicas except the first one (replica_id=0). + + Returns: + True if the summary is skipped; False otherwise. + """ + # TODO(priyag): Add a new optional argument that will provide multiple # alternatives to override default behavior. (e.g. run on last replica, # compute sum or mean across replicas). replica_context = distribution_strategy_context.get_replica_context() - return replica_context and replica_context.replica_id > 0 + if not replica_context: + return False + # TODO(b/118385803): when replica_id of _TPUReplicaContext is properly + # initialized, remember to change here as well. + replica_id = replica_context.replica_id_in_sync_group + if isinstance(replica_id, ops.Tensor): + replica_id = tensor_util.constant_value(replica_id) + return replica_id and replica_id > 0 def clean_tag(name): diff --git a/tensorflow/python/ops/summary_ops_v2.py b/tensorflow/python/ops/summary_ops_v2.py index 18cefb8e1c4..3f99b9f8773 100644 --- a/tensorflow/python/ops/summary_ops_v2.py +++ b/tensorflow/python/ops/summary_ops_v2.py @@ -40,11 +40,14 @@ from tensorflow.python.ops import resource_variable_ops from tensorflow.python.ops import summary_op_util from tensorflow.python.platform import tf_logging as logging from tensorflow.python.training import training_util +from tensorflow.python.util import deprecation from tensorflow.python.util import tf_contextlib +from tensorflow.python.util.tf_export import tf_export -# A global dictionary mapping graph keys to boolean values indicating whether -# we should record summaries for this particular graph or not. +# Dictionary mapping graph keys to a boolean Tensor (or callable returning +# a boolean Tensor) indicating whether we should record summaries for the +# graph identified by the key of the dictionary. _SHOULD_RECORD_SUMMARIES = {} # A global dictionary mapping graph keys to a list of summary writer init ops. @@ -59,58 +62,67 @@ def should_record_summaries(): """Returns boolean Tensor which is true if summaries should be recorded.""" global _SHOULD_RECORD_SUMMARIES key = ops.get_default_graph()._graph_key # pylint: disable=protected-access - return _SHOULD_RECORD_SUMMARIES.setdefault(key, False) + should = _SHOULD_RECORD_SUMMARIES.setdefault(key, False) + return should() if callable(should) else should + + +@tf_contextlib.contextmanager +def _record_summaries(boolean=True): + """Sets summary recording on or off per the provided boolean value. + + The provided value can be a python boolean, a scalar boolean Tensor, or + or a callable providing such a value; if a callable is passed it will be + invoked each time should_record_summaries() is called to determine whether + summary writing should be enabled. + + Args: + boolean: can be True, False, a bool Tensor, or a callable providing such. + Defaults to True. + + Yields: + Returns a context manager that sets this value on enter and restores the + previous value on exit. + """ + # TODO(nickfelt): make this threadlocal + global _SHOULD_RECORD_SUMMARIES + key = ops.get_default_graph()._graph_key # pylint: disable=protected-access + old = _SHOULD_RECORD_SUMMARIES.setdefault(key, False) + try: + _SHOULD_RECORD_SUMMARIES[key] = boolean + yield + finally: + _SHOULD_RECORD_SUMMARIES[key] = old # TODO(apassos) consider how to handle local step here. -@tf_contextlib.contextmanager def record_summaries_every_n_global_steps(n, global_step=None): """Sets the should_record_summaries Tensor to true if global_step % n == 0.""" if global_step is None: global_step = training_util.get_or_create_global_step() - global _SHOULD_RECORD_SUMMARIES - key = ops.get_default_graph()._graph_key # pylint: disable=protected-access - old = _SHOULD_RECORD_SUMMARIES.setdefault(key, False) - try: - with ops.device("cpu:0"): - _SHOULD_RECORD_SUMMARIES[key] = math_ops.equal(global_step % n, 0) - yield - finally: - _SHOULD_RECORD_SUMMARIES[key] = old + with ops.device("cpu:0"): + should = lambda: math_ops.equal(global_step % n, 0) + if not context.executing_eagerly(): + should = should() + return _record_summaries(should) -@tf_contextlib.contextmanager def always_record_summaries(): """Sets the should_record_summaries Tensor to always true.""" - global _SHOULD_RECORD_SUMMARIES - key = ops.get_default_graph()._graph_key # pylint: disable=protected-access - old = _SHOULD_RECORD_SUMMARIES.setdefault(key, False) - try: - _SHOULD_RECORD_SUMMARIES[key] = True - yield - finally: - _SHOULD_RECORD_SUMMARIES[key] = old + return _record_summaries(True) -@tf_contextlib.contextmanager def never_record_summaries(): """Sets the should_record_summaries Tensor to always false.""" - global _SHOULD_RECORD_SUMMARIES - key = ops.get_default_graph()._graph_key # pylint: disable=protected-access - old = _SHOULD_RECORD_SUMMARIES.setdefault(key, False) - try: - _SHOULD_RECORD_SUMMARIES[key] = False - yield - finally: - _SHOULD_RECORD_SUMMARIES[key] = old + return _record_summaries(False) +@tf_export("summary.SummaryWriter", v1=[]) class SummaryWriter(object): """Encapsulates a stateful summary writer resource. See also: - - `tf.contrib.summary.create_file_writer` - - `tf.contrib.summary.create_db_writer` + - `tf.summary.create_file_writer` + - `tf.summary.create_db_writer` """ def __init__(self, resource, init_op_fn): @@ -205,6 +217,7 @@ def initialize( session.run(_graph(x, 0), feed_dict={x: data}) +@tf_export("summary.create_file_writer", v1=[]) def create_file_writer(logdir, max_queue=None, flush_millis=None, @@ -280,7 +293,7 @@ def create_db_writer(db_uri, `tf.Graph`. Returns: - A `tf.contrib.summary.SummaryWriter` instance. + A `tf.summary.SummaryWriter` instance. """ with ops.device("cpu:0"): if experiment_name is None: @@ -329,7 +342,7 @@ def _nothing(): def all_summary_ops(): """Graph-mode only. Returns all summary ops. - Please note this excludes `tf.contrib.summary.graph` ops. + Please note this excludes `tf.summary.graph` ops. Returns: The summary ops. @@ -497,7 +510,7 @@ def graph(param, step=None, name=None): """Writes a TensorFlow graph to the summary interface. The graph summary is, strictly speaking, not a summary. Conditions - like `tf.contrib.summary.never_record_summaries` do not apply. Only + like `tf.summary.should_record_summaries` do not apply. Only a single graph can be associated with a particular run. If multiple graphs are written, then only the last one will be considered by TensorBoard. @@ -541,14 +554,13 @@ def graph(param, step=None, name=None): _graph = graph # for functions with a graph parameter +@tf_export("summary.import_event", v1=[]) def import_event(tensor, name=None): """Writes a `tf.Event` binary proto. - When using create_db_writer(), this can be used alongside - `tf.TFRecordReader` to load event logs into the database. Please - note that this is lower level than the other summary functions and - will ignore any conditions set by methods like - `tf.contrib.summary.should_record_summaries`. + This can be used to import existing event logs into a new summary writer sink. + Please note that this is lower level than the other summary functions and + will ignore the `tf.summary.should_record_summaries` setting. Args: tensor: A `tf.Tensor` of type `string` containing a serialized @@ -562,13 +574,14 @@ def import_event(tensor, name=None): context.context().summary_writer_resource, tensor, name=name) +@tf_export("summary.flush", v1=[]) def flush(writer=None, name=None): """Forces summary writer to send any buffered data to storage. This operation blocks until that finishes. Args: - writer: The `tf.contrib.summary.SummaryWriter` resource to flush. + writer: The `tf.summary.SummaryWriter` resource to flush. The thread default will be used if this parameter is None. Otherwise a `tf.no_op` is returned. name: A name for the operation (optional). @@ -595,6 +608,8 @@ def eval_dir(model_dir, name=None): return os.path.join(model_dir, "eval" if not name else "eval_" + name) +@deprecation.deprecated(date=None, + instructions="Renamed to create_file_writer().") def create_summary_file_writer(*args, **kwargs): """Please use `tf.contrib.summary.create_file_writer`.""" logging.warning("Deprecation Warning: create_summary_file_writer was renamed " diff --git a/tensorflow/python/ops/tensor_array_ops.py b/tensorflow/python/ops/tensor_array_ops.py index f86dfb35276..d1516949517 100644 --- a/tensorflow/python/ops/tensor_array_ops.py +++ b/tensorflow/python/ops/tensor_array_ops.py @@ -20,8 +20,10 @@ from __future__ import division from __future__ import print_function import contextlib +import os import weakref +from tensorflow.python import tf2 from tensorflow.python.eager import context from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes @@ -30,12 +32,18 @@ from tensorflow.python.framework import ops from tensorflow.python.framework import tensor_shape from tensorflow.python.framework import tensor_util from tensorflow.python.ops import array_ops +from tensorflow.python.ops import gen_control_flow_ops from tensorflow.python.ops import gen_data_flow_ops +from tensorflow.python.ops import list_ops from tensorflow.python.ops import math_ops from tensorflow.python.util import tf_should_use from tensorflow.python.util.tf_export import tf_export +ENABLE_TENSOR_ARRAY_V2 = ( + tf2.enabled() or os.getenv("TF_ENABLE_TENSOR_ARRAY_V2") is not None) + + # _GraphTensorArray accesses many of the hidden generated ops, but is in # fact built to wrap these methods. # pylint: disable=protected-access @@ -393,6 +401,273 @@ class _GraphTensorArray(object): return gen_data_flow_ops.tensor_array_close_v3( handle=self._handle, name=name) + +class _GraphTensorArrayV2(object): + """Graph-mode implementation of TensorArray backed by TensorLists. + + The backing tensor of this TensorArray is a TensorList variant tensor which is + stored in the `flow`. The `handle` is always none here. The reason we use the + `flow` field and not the `handle` field is to ensure backwards compatibility + with legacy control flow. + """ + + def __init__(self, + dtype, + size=None, + dynamic_size=None, + clear_after_read=None, + tensor_array_name=None, + handle=None, + flow=None, + infer_shape=True, + element_shape=None, + colocate_with_first_write_call=True, + name=None): + """Constructs a graph mode TensorArray. + + Args: + dtype: (required) data type of the TensorArray. + size: (optional) int32 scalar `Tensor`: the size of the TensorArray. + Required if flow is not provided. + dynamic_size: (optional) Python bool: If true, writes to the TensorArray + can grow the TensorArray past its initial size. Default: False. + clear_after_read: (optional) unused. Not supported in TensorLists. + tensor_array_name: (optional) unused. + handle: (optional) Must always be None. + flow: (optional) A variant `Tensor` scalar for a TensorList. + infer_shape: (optional, default: True) If True, shape inference is + enabled. In this case, all elements must have the same shape. + element_shape: (optional, default: None) A `TensorShape` object specifying + the shape constraints of each of the elements of the TensorArray. Need + not be fully defined. + colocate_with_first_write_call: (optional). unused. + name: (optional) A name for the operation. + + Raises: + ValueError: if both handle and tensor_array_name are provided. + TypeError: if handle is provided but is not a Tensor. + """ + assert handle is None + del handle + del clear_after_read + del tensor_array_name + del colocate_with_first_write_call + + del dynamic_size # TODO(b/117943489): Unused for now. + + if (flow is not None and + (not isinstance(flow, ops.Tensor) or flow.dtype != dtypes.variant)): + raise TypeError("flow must be a variant tensor") + if flow is None and size is None: + raise ValueError("Size must be provided if flow is not provided") + if flow is not None and size is not None: + raise ValueError("Cannot provide both a flow and size " + "at the same time") + if flow is not None and element_shape is not None: + raise ValueError("Cannot provide both a flow and element_shape " + "at the same time") + + self._dtype = dtype + + # Record the current static shape for the array elements. The element + # shape is defined either by `element_shape` or the shape of the tensor + # of the first write. If `infer_shape` is true, all writes checks for + # shape equality. + if element_shape is None: + self._infer_shape = infer_shape + self._element_shape = [] + else: + self._infer_shape = True + self._element_shape = [tensor_shape.TensorShape(element_shape)] + with ops.name_scope(name, "TensorArrayV2", [size, flow]) as scope: + if flow is None: + self._flow = list_ops.tensor_list_reserve( + element_shape=element_shape, + num_elements=size, + element_dtype=dtype, + name=scope) + else: + self._flow = flow + + # For backwards compatibility. + self._colocate_with_first_write_call = None + self._colocate_with = None + + @property + def flow(self): + return self._flow + + @property + def dtype(self): + return self._dtype + + @property + def handle(self): + # We intentionally do not raise an error so that legacy while_loop does not + # complain. + return None + + def _merge_element_shape(self, shape): + """Changes the element shape of the array given a shape to merge with. + + Args: + shape: A `TensorShape` object to merge with. + + Raises: + ValueError: if the provided shape is incompatible with the current + element shape of the `TensorArray`. + """ + + if self._element_shape: + if not shape.is_compatible_with(self._element_shape[0]): + raise ValueError( + "Inconsistent shapes: saw %s but expected %s " + "(and infer_shape=True)" % (shape, self._element_shape[0])) + self._element_shape[0] = self._element_shape[0].merge_with(shape) + else: + self._element_shape.append(shape) + + def identity(self): + """See TensorArray.""" + flow = array_ops.identity(self._flow) + ta = TensorArray( + dtype=self._dtype, flow=flow, infer_shape=self._infer_shape) + ta._element_shape = self._element_shape + return ta + + def grad(self, source, flow=None, name=None): + """Not supported.""" + raise NotImplementedError() + + def read(self, index, name=None): + """See TensorArray.""" + value = list_ops.tensor_list_get_item( + input_handle=self._flow, + index=index, + element_dtype=self._dtype, + name=name) + if self._element_shape: + value.set_shape(self._element_shape[0].dims) + return value + + @tf_should_use.should_use_result + def write(self, index, value, name=None): + """See TensorArray.""" + with ops.name_scope(name, "TensorArrayV2Write", [self._flow, index, value]): + value = ops.convert_to_tensor(value, name="value") + if self._infer_shape: + self._merge_element_shape(value.shape) + flow_out = list_ops.tensor_list_set_item( + input_handle=self._flow, index=index, item=value, name=name) + ta = TensorArray(dtype=self._dtype, handle=None, flow=flow_out) + ta._infer_shape = self._infer_shape + ta._element_shape = self._element_shape + return ta + + def stack(self, name=None): + """See TensorArray.""" + with ops.name_scope(name, "TensorArrayV2Stack", [self._flow]): + value = list_ops.tensor_list_stack( + input_handle=self._flow, element_dtype=self._dtype) + if self._element_shape and self._element_shape[0].dims is not None: + value.set_shape([None] + self._element_shape[0].dims) + return value + + def gather(self, indices, name=None): + """See TensorArray.""" + value = list_ops.tensor_list_gather( + input_handle=self._flow, + indices=indices, + element_dtype=self._dtype, + name=name) + if self._element_shape and self._element_shape[0].dims is not None: + value.set_shape([None] + self._element_shape[0].dims) + return value + + def concat(self, name=None): + """See TensorArray.""" + value = list_ops.tensor_list_concat( + input_handle=self._flow, element_dtype=self._dtype, name=name) + if self._element_shape and self._element_shape[0].dims is not None: + value.set_shape([None] + self._element_shape[0].dims[1:]) + return value + + @tf_should_use.should_use_result + def unstack(self, value, name=None): + """See TensorArray.""" + with ops.name_scope(name, "TensorArrayUnstack", [self._flow, value]): + value = ops.convert_to_tensor(value, name="value") + if self._infer_shape and not context.executing_eagerly(): + self._merge_element_shape(value.shape[1:]) + flow_out = list_ops.tensor_list_from_tensor( + tensor=value, element_shape=value.shape[1:]) + ta = TensorArray( + dtype=self._dtype, + handle=self.handle, + flow=flow_out, + colocate_with_first_write_call=self._colocate_with_first_write_call) + ta._infer_shape = self._infer_shape + ta._element_shape = self._element_shape + ta._colocate_with = self._colocate_with + return ta + + @tf_should_use.should_use_result + def scatter(self, indices, value, name=None): + """See TensorArray.""" + with ops.name_scope(name, "TensorArrayScatter", + [self._flow, value, indices]): + value = ops.convert_to_tensor(value, name="value") + if self._infer_shape and not context.executing_eagerly(): + self._merge_element_shape(value.shape[1:]) + flow_out = list_ops.tensor_list_scatter( + tensor=value, indices=indices, element_shape=-1) + ta = TensorArray( + dtype=self._dtype, + handle=self.handle, + flow=flow_out, + colocate_with_first_write_call=self._colocate_with_first_write_call) + ta._infer_shape = self._infer_shape + ta._element_shape = self._element_shape + ta._colocate_with = self._colocate_with + return ta + + @tf_should_use.should_use_result + def split(self, value, lengths, name=None): + """See TensorArray.""" + with ops.name_scope(name, "TensorArraySplit", [self._flow, value, lengths]): + value = ops.convert_to_tensor(value, name="value") + lengths_64 = math_ops.to_int64(lengths) + if self._infer_shape and not context.executing_eagerly(): + clengths = tensor_util.constant_value(lengths_64) + if value.shape.dims is not None: + if clengths is not None and clengths.max() == clengths.min(): + self._merge_element_shape( + tensor_shape.TensorShape([clengths[0]]).concatenate( + value.shape[1:])) + flow_out = list_ops.tensor_list_split( + tensor=value, + lengths=lengths_64, + element_shape=self._element_shape[0] if self._element_shape else None, + name=name) + ta = TensorArray( + dtype=self._dtype, + handle=self.handle, + flow=flow_out, + colocate_with_first_write_call=self._colocate_with_first_write_call) + ta._infer_shape = self._infer_shape + ta._element_shape = self._element_shape + ta._colocate_with = self._colocate_with + return ta + + def size(self, name=None): + """See TensorArray.""" + return list_ops.tensor_list_length(input_handle=self._flow, name=name) + + @tf_should_use.should_use_result + def close(self, name=None): + """See TensorArray.""" + return gen_control_flow_ops.no_op(name=name) + # pylint: enable=protected-access @@ -738,8 +1013,10 @@ class TensorArray(object): if context.executing_eagerly(): implementation = _EagerTensorArray else: - implementation = _GraphTensorArray - + if ENABLE_TENSOR_ARRAY_V2: + implementation = _GraphTensorArrayV2 + else: + implementation = _GraphTensorArray self._implementation = implementation( dtype, size=size, @@ -768,7 +1045,7 @@ class TensorArray(object): @property def handle(self): """The reference to the TensorArray.""" - return self._implementation._handle + return self._implementation.handle @property def _infer_shape(self): @@ -953,4 +1230,16 @@ class TensorArray(object): """Close the current TensorArray.""" return self._implementation.close(name=name) + +def build_ta_with_new_flow(old_ta, flow): + ta = TensorArray( + dtype=old_ta.dtype, + handle=old_ta.handle, + flow=flow, + infer_shape=old_ta._infer_shape, + colocate_with_first_write_call=old_ta._colocate_with_first_write_call) + ta._colocate_with = old_ta._colocate_with + ta._element_shape = old_ta._element_shape + return ta + # pylint: enable=protected-access diff --git a/tensorflow/python/ops/tensor_forest_ops.py b/tensorflow/python/ops/tensor_forest_ops.py new file mode 100644 index 00000000000..42f3cdf324a --- /dev/null +++ b/tensorflow/python/ops/tensor_forest_ops.py @@ -0,0 +1,110 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Ops for tensor_forest.""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from tensorflow.python.ops import resources + +from tensorflow.python import ops +from tensorflow.python.ops import gen_tensor_forest_ops +from tensorflow.python.training import saver + + +class TreeVariableSaveable(saver.BaseSaverBuilder.SaveableObject): + + def __init__(self, type_name, name, container, config, resource_handle_func, + create_op_func, is_initialized_op_func, serialize_op_func, + deserialize_op_func): + + with ops.name_scope(name, type_name) as name: + self._resource_handle = resource_handle_func( + container, shared_name=name, name=name) + + self._is_initialized_op = is_initialized_op_func( + self._resource_handle) + tensor = serialize_op_func(self._resource_handle) + self._create_op = create_op_func( + self._resource_handle, + config) + # slice_spec is useful for saving a slice from a variable. + # It's not meaningful the tree variable. So we just pass an empty + # value. + slice_spec = "" + specs = [saver.BaseSaverBuilder.SaveSpec(tensor, slice_spec, name)] + super(TreeVariableSaveable, + self).__init__(self._resource_handle, specs, name) + + ops.add_to_collection( + ops.GraphKeys.SAVEABLE_OBJECTS, self) + + resources.register_resource( + self._resource_handle, self._create_op, self._is_initialized_op) + self._deserialize_op_func = deserialize_op_func + + def restore(self, restored_tensors, unused_restored_shapes): + """Restores the associated tree from 'restored_tensors'. + + Args: + restored_tensors: the tensors that were loaded from a checkpoint. + unused_restored_shapes: the shapes this object should conform to after + restore. Not meaningful for trees. + + Returns: + The operation that restores the state of the tree variable. + """ + with ops.control_dependencies([self._create_op]): + return self._deserialize_op_func( + self._resource_handle, + restored_tensors[0], + ) + + @property + def resource(self): + return self._resource_handle + + +def tree_variable(tree_config, name, container=None): + return TreeVariableSaveable( + "TreeVariable", + name, + container, + tree_config, + gen_tensor_forest_ops.tensor_forest_tree_resource_handle_op, + gen_tensor_forest_ops.tensor_forest_create_tree_variable, + gen_tensor_forest_ops.tensor_forest_tree_is_initialized_op, + gen_tensor_forest_ops.tensor_forest_tree_serialize, + gen_tensor_forest_ops.tensor_forest_tree_deserialize).resource + + +class ForestVariables(object): + + def __init__(self, params, + tree_configs=None): + + self._variables = [] + + for i in range(params.n_trees): + tree_config = '' + if tree_configs is not None: + tree_config = tree_configs[i] + self._variables.append(tree_variable( + tree_config, + 'tree-%s' % i, + )) + + def __getitem__(self, t): + return self._variables[t] diff --git a/tensorflow/python/ops/variable_scope.py b/tensorflow/python/ops/variable_scope.py index fe93bfb61f7..ccce9e2f93b 100644 --- a/tensorflow/python/ops/variable_scope.py +++ b/tensorflow/python/ops/variable_scope.py @@ -646,14 +646,8 @@ class _VariableStore(object): when violating reuse during variable creation, or if an existing sharded variable exists for the given name but with different sharding. """ - if context.executing_eagerly(): - raise NotImplementedError("Partitioned variables are not yet supported " - "when eager execution is enabled.") - initializing_from_value = initializer is not None and isinstance( initializer, ops.Tensor) - reuse_without_partition = reuse and not partitioner - if name in self._vars: raise ValueError( "A partitioner was provided, but an unpartitioned version of the " @@ -664,30 +658,9 @@ class _VariableStore(object): if initializing_from_value: shape = shape.merge_with(initializer.get_shape()) - if not reuse_without_partition: - if not shape.is_fully_defined(): - raise ValueError("Shape of a new partitioned variable (%s) must be " - "fully defined, but instead was %s." % (name, shape)) - - if shape.ndims < 1: - raise ValueError("A partitioned Variable must have rank at least 1, " - "shape: %s" % shape) - - partitions = partitioner(shape=shape, dtype=dtype) - - if not isinstance(partitions, collections_lib.Sequence): - raise ValueError("Partitioner must return a sequence, but saw: %s" - % partitions) - - if len(partitions) != shape.ndims: - raise ValueError( - "Partitioner returned a partition list that does not match the " - "Variable's rank: %s vs. %s" % (partitions, shape)) - - if any([p < 1 for p in partitions]): - raise ValueError( - "Partitioner returned zero partitions for some axes: %s" % - partitions) + partitions = None + if not reuse or partitioner: + partitions = _call_partitioner(partitioner, shape, dtype) if name in self._partitioned_vars: if reuse is False: @@ -709,7 +682,7 @@ class _VariableStore(object): % (name, dtype.name, existing_var.dtype.name)) # pylint: disable=protected-access - if (not reuse_without_partition and + if (partitions is not None and existing_var._get_partitions() != partitions): raise ValueError( "Trying to reuse partitioned variable %s, but specified partitions " @@ -724,14 +697,7 @@ class _VariableStore(object): "created with tf.get_variable(). Did you mean to set " "reuse=False or reuse=tf.AUTO_REUSE in VarScope?" % name) - slice_dim, slice_shape = _compute_slice_dim_and_shape( - shape.as_list(), partitions) - - vs = [] - num_slices = partitions[slice_dim] - num_slices_with_excess = shape.dims[slice_dim].value % num_slices - - slice_offset = [0] * shape.ndims + slice_dim, num_slices = _get_slice_dim_and_num_slices(partitions) if "%s/part_0" % name in self._vars: if "%s/part_%d" % (name, num_slices - 1) not in self._vars: @@ -747,15 +713,14 @@ class _VariableStore(object): "%s/part_0 was found, but so was the extra shard %s/part_%d." % (num_slices, name, name, num_slices)) - for i in xrange(num_slices): - var_shape = slice_shape[:] - var_offset = slice_offset[:] + vs = [] + for i, (var_offset, var_shape) in enumerate(_iter_slices( + shape.as_list(), + num_slices, + slice_dim + )): partition_info = _PartitionInfo( full_shape=shape.as_list(), var_offset=var_offset) - if i < num_slices_with_excess: - var_shape[slice_dim] += 1 - slice_offset[slice_dim] += var_shape[slice_dim] - var_full_name = "%s/part_%d" % (name, i) with ops.name_scope(var_full_name + "/PartitionedInitializer"): # Create the tensor to initialize the variable with default value. @@ -803,15 +768,13 @@ class _VariableStore(object): vs.append(var) # pylint: enable=protected-access - # pylint: disable=protected-access partitioned_var = variables.PartitionedVariable(name=name, shape=shape, dtype=dtype, variable_list=vs, partitions=partitions) - # pylint: enable=protected-access - - self._partitioned_vars[name] = partitioned_var + if not context.executing_eagerly() or self._store_eager_variables: + self._partitioned_vars[name] = partitioned_var return partitioned_var def _get_single_variable(self, @@ -913,20 +876,22 @@ class _VariableStore(object): variable_dtype = None else: # Instantiate initializer if provided initializer is a type object. - if isinstance(initializer, type(init_ops.Initializer)): + if tf_inspect.isclass(initializer): initializer = initializer(dtype=dtype) - if shape and shape.is_fully_defined(): + if shape is not None and shape.is_fully_defined(): init_val = lambda: initializer( # pylint: disable=g-long-lambda shape.as_list(), dtype=dtype, partition_info=partition_info) - elif not tf_inspect.getargspec(initializer).args: + variable_dtype = dtype.base_dtype + elif len(tf_inspect.getargspec(initializer).args) == len( + tf_inspect.getargspec(initializer).defaults or []): init_val = initializer + variable_dtype = None else: - raise ValueError("You can only pass an initializer function that " - "expects no arguments to its callable when the " - "shape is not fully defined. The given initializer " - "function expects the following args %s" % - tf_inspect.getargspec(initializer).args) - variable_dtype = dtype.base_dtype + raise ValueError("The initializer passed is not valid. It should " + "be a callable with no arguments and the " + "shape should not be provided or an instance of " + "`tf.keras.initializers.*' and `shape` should be " + "fully defined.") # Create the variable. if use_resource is None: @@ -1080,9 +1045,6 @@ class VariableScope(object): if self._caching_device is not None: raise NotImplementedError("Caching devices is not yet supported " "when eager execution is enabled.") - if self._partitioner is not None: - raise NotImplementedError("Partitioned variables are not yet supported " - "when eager execution is enabled.") self._reuse = AUTO_REUSE self._use_resource = True @@ -1162,9 +1124,6 @@ class VariableScope(object): def set_partitioner(self, partitioner): """Set partitioner for this scope.""" - if partitioner and context.executing_eagerly(): - raise NotImplementedError("Partitioned variables are not yet supported " - "when eager execution is enabled.") self._partitioner = partitioner def set_custom_getter(self, custom_getter): @@ -1277,9 +1236,6 @@ class VariableScope(object): synchronization=VariableSynchronization.AUTO, aggregation=VariableAggregation.NONE): """Gets an existing variable with this name or create a new one.""" - if context.executing_eagerly(): - raise NotImplementedError("Partitioned variables are not yet supported " - "when eager execution is enabled.") if initializer is None: initializer = self._initializer if regularizer is None: @@ -2246,8 +2202,8 @@ class variable_scope(object): try: return self._enter_scope_uncached() - except: - if not self._building_function: + except Exception: + if self._in_graph_mode and not self._building_function: if self._graph_context_manager is not None: self._graph_context_manager.__exit__(*sys.exc_info()) raise @@ -2413,34 +2369,71 @@ def variable_op_scope(values, yield scope -def _compute_slice_dim_and_shape(full_shape, slicing): - """Computes which dimension is being sliced and the typical slice shape.""" +def _call_partitioner(partitioner, shape, dtype): + """Call partitioner validating its inputs/output. - slice_shape = [0] * len(full_shape) - slice_dim = None - for dim, num_slices in enumerate(slicing): - dim_size = full_shape[dim] - if num_slices <= 0 or dim_size < num_slices: - raise ValueError("Cannot create %d slices for size %d. shape: %s, " - "slicing: %s" % - (num_slices, full_shape[dim], full_shape, slicing)) - if num_slices == 1: - # Not slicing in this dimension. - slice_shape[dim] = dim_size - elif slice_dim is not None: - # We only support slicing along one of the dimensions. - raise ValueError("Can only slice a variable along one dimension: " - "shape: %s, slicing: %s" % (full_shape, slicing)) - else: - # Note: We will add any extras onto the last slice, later. - slice_dim = dim - slice_shape[dim] = dim_size // num_slices + Args: + partitioner: a function mapping `Tensor` shape and dtype to a + list of partitions. + shape: shape of the `Tensor` to partition, must have at least two + dimensions. + dtype: dtype of the elements in the `Tensor`. - # Degenerate case: If "slicing" was all ones, pretend we are slicing along - # the first dimension. - if slice_dim is None: + Returns: + A list with elements >=1 and exactly one >1. The index of that + element corresponds to the partitioning axis. + """ + if not shape.is_fully_defined(): + raise ValueError("Shape of a new partitioned variable must be " + "fully defined, but instead was %s." % (shape,)) + if shape.ndims < 1: + raise ValueError("A partitioned Variable must have rank at least 1, " + "shape: %s" % shape) + + slicing = partitioner(shape=shape, dtype=dtype) + if not isinstance(slicing, collections_lib.Sequence): + raise ValueError("Partitioner must return a sequence, but saw: %s" + % slicing) + if len(slicing) != shape.ndims: + raise ValueError( + "Partitioner returned a partition list that does not match the " + "Variable's rank: %s vs. %s" % (slicing, shape)) + if any(p < 1 for p in slicing): + raise ValueError( + "Partitioner returned zero partitions for some axes: %s" % + slicing) + if sum(p > 1 for p in slicing) > 1: + raise ValueError( + "Can only slice a variable along one dimension: " + "shape: %s, partitioning: %s" % (shape, slicing)) + return slicing + + +# TODO(slebedev): could be inlined, but +# `_VariableStore._get_partitioned_variable` is too complex even +# without this logic. +def _get_slice_dim_and_num_slices(slicing): + """Get slicing dimension and number of slices from the partitioner output.""" + for slice_dim, num_slices in enumerate(slicing): + if num_slices > 1: + break + else: + # Degenerate case: no partitioning applied. slice_dim = 0 - return slice_dim, slice_shape + num_slices = 1 + return slice_dim, num_slices + + +def _iter_slices(full_shape, num_slices, slice_dim): + """Slices a given a shape along the specified dimension.""" + num_slices_with_excess = full_shape[slice_dim] % num_slices + offset = [0] * len(full_shape) + min_slice_len = full_shape[slice_dim] // num_slices + for i in xrange(num_slices): + shape = full_shape[:] + shape[slice_dim] = min_slice_len + bool(i < num_slices_with_excess) + yield offset[:], shape + offset[slice_dim] += shape[slice_dim] def _get_trainable_value(synchronization, trainable): diff --git a/tensorflow/python/ops/variables.py b/tensorflow/python/ops/variables.py index e43736069e3..d809c81b982 100644 --- a/tensorflow/python/ops/variables.py +++ b/tensorflow/python/ops/variables.py @@ -18,7 +18,8 @@ from __future__ import division from __future__ import print_function import enum # pylint: disable=g-bad-import-order - +import functools +import os import six from tensorflow.core.framework import attr_value_pb2 @@ -860,18 +861,18 @@ class Variable(six.with_metaclass(VariableMetaclass, else: return v.value() - @staticmethod - def _OverloadAllOperators(): # pylint: disable=invalid-name + @classmethod + def _OverloadAllOperators(cls): # pylint: disable=invalid-name """Register overloads for all operators.""" for operator in ops.Tensor.OVERLOADABLE_OPERATORS: - Variable._OverloadOperator(operator) + cls._OverloadOperator(operator) # For slicing, bind getitem differently than a tensor (use SliceHelperVar # instead) # pylint: disable=protected-access - setattr(Variable, "__getitem__", array_ops._SliceHelperVar) + setattr(cls, "__getitem__", array_ops._SliceHelperVar) - @staticmethod - def _OverloadOperator(operator): # pylint: disable=invalid-name + @classmethod + def _OverloadOperator(cls, operator): # pylint: disable=invalid-name """Defer an operator overload to `ops.Tensor`. We pull the operator out of ops.Tensor dynamically to avoid ordering issues. @@ -879,17 +880,26 @@ class Variable(six.with_metaclass(VariableMetaclass, Args: operator: string. The operator name. """ + tensor_oper = getattr(ops.Tensor, operator) - def _run_op(a, *args): + def _run_op(a, *args, **kwargs): # pylint: disable=protected-access - return getattr(ops.Tensor, operator)(a._AsTensor(), *args) - # Propagate __doc__ to wrapper - try: - _run_op.__doc__ = getattr(ops.Tensor, operator).__doc__ - except AttributeError: - pass + return tensor_oper(a._AsTensor(), *args, **kwargs) - setattr(Variable, operator, _run_op) + functools.update_wrapper(_run_op, tensor_oper) + setattr(cls, operator, _run_op) + + def __iter__(self): + """Dummy method to prevent iteration. Do not call. + + NOTE(mrry): If we register __getitem__ as an overloaded operator, + Python will valiantly attempt to iterate over the variable's Tensor from 0 + to infinity. Declaring this method prevents this unintended behavior. + + Raises: + TypeError: when invoked. + """ + raise TypeError("'Variable' object is not iterable.") # NOTE(mrry): This enables the Variable's overloaded "right" binary # operators to run when the left operand is an ndarray, because it @@ -1045,27 +1055,6 @@ class Variable(six.with_metaclass(VariableMetaclass, else: return None - def __iadd__(self, other): - raise NotImplementedError - - def __isub__(self, other): - raise NotImplementedError - - def __imul__(self, other): - raise NotImplementedError - - def __idiv__(self, other): - raise NotImplementedError - - def __itruediv__(self, other): - raise NotImplementedError - - def __irealdiv__(self, other): - raise NotImplementedError - - def __ipow__(self, other): - raise NotImplementedError - @tf_export(v1=["Variable"]) class VariableV1(Variable): @@ -1576,18 +1565,6 @@ class RefVariable(VariableV1): """ return self._snapshot - def __iter__(self): - """Dummy method to prevent iteration. Do not call. - - NOTE(mrry): If we register __getitem__ as an overloaded operator, - Python will valiantly attempt to iterate over the variable's Tensor from 0 - to infinity. Declaring this method prevents this unintended behavior. - - Raises: - TypeError: when invoked. - """ - raise TypeError("'Variable' object is not iterable.") - def value(self): """Returns the last snapshot of this variable. @@ -2123,37 +2100,6 @@ class RefVariable(VariableV1): else: return v.value() - @staticmethod - def _OverloadAllOperators(): # pylint: disable=invalid-name - """Register overloads for all operators.""" - for operator in ops.Tensor.OVERLOADABLE_OPERATORS: - Variable._OverloadOperator(operator) # pylint: disable=protected-access - # For slicing, bind getitem differently than a tensor (use SliceHelperVar - # instead) - # pylint: disable=protected-access - setattr(Variable, "__getitem__", array_ops._SliceHelperVar) - - @staticmethod - def _OverloadOperator(operator): # pylint: disable=invalid-name - """Defer an operator overload to `ops.Tensor`. - - We pull the operator out of ops.Tensor dynamically to avoid ordering issues. - - Args: - operator: string. The operator name. - """ - - def _run_op(a, *args): - # pylint: disable=protected-access - return getattr(ops.Tensor, operator)(a._AsTensor(), *args) - # Propagate __doc__ to wrapper - try: - _run_op.__doc__ = getattr(ops.Tensor, operator).__doc__ - except AttributeError: - pass - - setattr(Variable, operator, _run_op) - def _gather_saveables_for_checkpoint(self): """For implementing `Checkpointable`. This object is saveable on its own.""" return {checkpointable.VARIABLE_VALUE_KEY: self} @@ -2457,34 +2403,6 @@ class PartitionedVariable(object): @end_compatibility """ - class PartitionedVariableIterator(object): - """An iterator that allows accessing the underlying `Variable` objects. - - This iterator is necessary to control order of access when Variables - are not partitioned in a standard way along a single axis. - - Allows e.g. `list(partitioned_variable)` to return a proper list. - """ - - def __init__(self, partitioned_variable): - self._ix = 0 - self._partitioned_variable = partitioned_variable - - def __iter__(self): - return self - - def __next__(self): # For python3 compatibility. - return self.next() - - def next(self): - # pylint: disable=protected-access - if self._ix >= len(self._partitioned_variable._variable_list): - raise StopIteration() - variable = self._partitioned_variable._variable_list[self._ix] - # pylint: enable=protected-access - self._ix += 1 - return variable - def __init__(self, name, shape, dtype, variable_list, partitions): """Creates a new partitioned variable wrapper. @@ -2504,31 +2422,27 @@ class PartitionedVariable(object): `partitions` is not a list. ValueError: If `variable_list` is empty, or the `Variable` shape information does not match `shape`, or `partitions` has invalid values. - RuntimeError: If eager execution is enabled """ - if context.executing_eagerly(): - raise RuntimeError( - "tf.PartitionedVariable not supported with eager execution enabled.") if not isinstance(variable_list, (list, tuple)): raise TypeError( "variable_list is not a list or tuple: %s" % variable_list) if not isinstance(partitions, (list, tuple)): raise TypeError("partitions is not a list or tuple: %s" % partitions) - if not all([p >= 1 for p in partitions]): + if not all(p >= 1 for p in partitions): raise ValueError("partition values must be positive: %s" % partitions) if not variable_list: raise ValueError("variable_list may not be empty") # pylint: disable=protected-access for v in variable_list: # Sort the variable_list lexicographically according to var offset value. - if not all([v._get_save_slice_info() is not None for v in variable_list]): + if not all(v._get_save_slice_info() is not None for v in variable_list): raise ValueError( "All variables must have a save_slice_info available: %s" % [v.name for v in variable_list]) if len(shape) != len(partitions): raise ValueError("len(shape) != len(partitions): %s vs. %s" % (shape, partitions)) - if not all([v._get_save_slice_info().full_shape == shape]): + if v._get_save_slice_info().full_shape != shape: raise ValueError( "All variables' full shapes must match shape: %s; " "but full shapes were: %s" @@ -2545,7 +2459,7 @@ class PartitionedVariable(object): def __iter__(self): """Return an iterable for accessing the underlying partition Variables.""" - return self.PartitionedVariableIterator(self) + return iter(self._variable_list) def __len__(self): num_partition_axes = len(self._partition_axes()) @@ -2555,7 +2469,7 @@ class PartitionedVariable(object): return len(self._variable_list) def _partition_axes(self): - if all([p == 1 for p in self._partitions]): + if all(p == 1 for p in self._partitions): return [0] else: return [i for i, p in enumerate(self._partitions) if p > 1] @@ -2995,7 +2909,8 @@ def report_uninitialized_variables(var_list=None, # Run all operations on CPU if var_list: init_vars = [state_ops.is_variable_initialized(v) for v in var_list] - with ops.device("/cpu:0"): + local_device = os.environ.get("TF_DEVICE_FOR_UNINITIALIZED_VARIABLE_REPORTING", "/cpu:0") + with ops.device(local_device): if not var_list: # Return an empty tensor so we only need to check for returned tensor # size being 0 as an indication of model ready. diff --git a/tensorflow/python/ops/while_v2.py b/tensorflow/python/ops/while_v2.py index 7b0f0ed4fc8..59ca29e3bad 100644 --- a/tensorflow/python/ops/while_v2.py +++ b/tensorflow/python/ops/while_v2.py @@ -23,7 +23,6 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function -from tensorflow.core.framework import attr_value_pb2 from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes from tensorflow.python.framework import func_graph as func_graph_module @@ -65,7 +64,8 @@ def while_loop(cond, loop_vars, shape_invariants=None, maximum_iterations=None, - name=None): + name=None, + return_same_structure=True): """Like tf.while_loop, except emits a single While op.""" maximum_iterations = _validate_and_convert_to_tensor(maximum_iterations) # Keep the original loop_vars around to know which args were TensorArrays. @@ -100,7 +100,8 @@ def while_loop(cond, # Add loop counter needed for computing gradients. loop_vars = [loop_counter] + loop_vars - shape_invariants = [tensor_shape.scalar()] + shape_invariants + shape_invariants = type(shape_invariants)([tensor_shape.scalar() + ]) + shape_invariants # Automatic control dependencies are added in defuns, but not in v1 # graphs. Propagate that behavior here. @@ -133,9 +134,8 @@ def while_loop(cond, # the value of that tensor in each iteration is the same as it was at the # beginning of the loop execution. loop_vars = loop_vars + cond_graph.external_captures - shape_invariants = shape_invariants + [ - t.shape for t in cond_graph.external_captures - ] + shape_invariants = shape_invariants + type(shape_invariants)( + [t.shape for t in cond_graph.external_captures]) def wrapped_body(loop_counter, *args): """Loop body augmented with counter update. @@ -208,8 +208,7 @@ def while_loop(cond, for intermediate_tensor in intermediate_tensors: tensor_list = list_ops.empty_tensor_list( element_dtype=intermediate_tensor.dtype, - element_shape=_get_tensor_convertible_shape( - intermediate_tensor.shape), + element_shape=intermediate_tensor.shape, max_num_elements=maximum_iterations) loop_vars.append(tensor_list) with cond_graph.as_default(): @@ -245,7 +244,7 @@ def while_loop(cond, name=scope) _copy_handle_data(body_graph.outputs, outputs) - _maybe_set_lowering_attr(outputs[0].op) + util.maybe_set_lowering_attr(outputs[0].op) _maybe_set_maximum_iterations_attr(outputs[0].op, maximum_iterations) # Return identities for each output of the While op, rather than the output @@ -257,11 +256,17 @@ def while_loop(cond, outputs = tuple(array_ops.identity(t) for t in outputs) # First var is loop counter. - if num_flattened_outputs == 1: - return outputs[1] + outputs = _pack_sequence_as(orig_loop_vars, + outputs[1:1 + num_flattened_outputs]) + + if return_same_structure: + return outputs + + flattened_outputs = nest.flatten(outputs) + if len(flattened_outputs) == 1: + return flattened_outputs[0] else: - return _pack_sequence_as(orig_loop_vars, - outputs[1:1 + num_flattened_outputs]) + return outputs @ops.RegisterGradient("While") @@ -313,7 +318,7 @@ def _WhileGrad(op, *grads): # pylint: disable=invalid-name for intermediate_tensor in intermediate_tensors: tensor_list = list_ops.empty_tensor_list( element_dtype=intermediate_tensor.dtype, - element_shape=_get_tensor_convertible_shape(intermediate_tensor.shape), + element_shape=intermediate_tensor.shape, max_num_elements=maximum_iterations) with body_grad_graph.as_default(): @@ -343,9 +348,12 @@ def _WhileGrad(op, *grads): # pylint: disable=invalid-name name="%s_grad" % op.name) _copy_handle_data(body_grad_graph.outputs, outputs) - _maybe_set_lowering_attr(outputs[0].op) + util.maybe_set_lowering_attr(outputs[0].op) _maybe_set_maximum_iterations_attr(outputs[0].op, maximum_iterations) + # See comment in while_loop. + outputs = [array_ops.identity(t) for t in outputs] + # Set None as the output gradient for tensors with None input gradient # e.g. TensorArray handles. # outputs[0] is the loop counter. @@ -505,7 +513,7 @@ def _grad_fn(ys, xs, args, func_graph): # TODO(b/118712257): Handle the case when grad_outs has None's e.g. when there # is a tf.StopGradient in the loop body. - assert all([g is not None for g in grad_outs]) + assert all(g is not None for g in grad_outs) counter = args[0] total_iters = args[1] return [counter + 1, total_iters] + grad_outs @@ -792,29 +800,6 @@ def _copy_handle_data(src_tensors, tgt_tensors): custom_gradient.copy_handle_data(src_t, tgt_t) -# TODO(srbs): Move to common utils for cond_v2 and while_v2. -def _maybe_set_lowering_attr(op): - """Sets the flag to enable lowering on the `While` op if necessary. - - Lowering allows while_v2 to avoid some of the limitations of Functions, - allowing users to specify devices & colocation inside of while_v2 - branches, and enabling non-strict evaluation & partial pruning of while_v2 - branches. This brings while_v2 closer to feature parity with - tf.while_loop. - - However, we do not lower `While` in the XLA context because it is easier - for XLA to apply its own optimizations when dealing with un-lowered - `While` operators than with low-level control flow primitives. - - Args: - op: The While op. - """ - if not control_flow_util.IsInXLAContext(op): - # pylint: disable=protected-access - op._set_attr("_lower_using_switch_merge", attr_value_pb2.AttrValue(b=True)) - # pylint: enable=protected-access - - def _maybe_set_maximum_iterations_attr(op, maximum_iterations): if control_flow_util.IsInXLAContext(op): # Store the maximum_iterations to use in the gradient pass. @@ -837,18 +822,6 @@ def _is_in_xla_context(): return control_flow_util.GetContainingXLAContext(cur_ctxt) is not None -def _get_tensor_convertible_shape(shape): - assert isinstance(shape, tensor_shape.TensorShape) - if shape.is_fully_defined(): - return shape - if not shape: # Unknown shape. - return -1 - # Partially defined shape. - shape_list = shape.as_list() - shape_list = [s if s is not None else -1 for s in shape_list] - return ops.convert_to_tensor(shape_list) - - def _graph_name(graph): if isinstance(graph, func_graph_module.FuncGraph): return graph.name @@ -870,6 +843,10 @@ def _is_tensor_array_handle(tensor): # TODO(b/118452219): add test coverage for this. tensor = func_graph_module.maybe_captured(tensor) + if isinstance(tensor, ops.EagerTensor): + # Eager execution doesn't quite support legacy tensorarray + return False + return tensor.op.type in TENSOR_ARRAY_HANDLE_OPS diff --git a/tensorflow/python/platform/__init__.py b/tensorflow/python/platform/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tensorflow/python/platform/app.py b/tensorflow/python/platform/app.py index 4c91bc3652d..7b917235c0a 100644 --- a/tensorflow/python/platform/app.py +++ b/tensorflow/python/platform/app.py @@ -108,7 +108,7 @@ def _define_help_flags(): _define_help_flags_called = True -@tf_export('app.run') +@tf_export(v1=['app.run']) def run(main=None, argv=None): """Runs the program with an optional 'main' function and 'argv' list.""" diff --git a/tensorflow/python/platform/benchmark.py b/tensorflow/python/platform/benchmark.py index 4f7abb311a7..d6773d7b813 100644 --- a/tensorflow/python/platform/benchmark.py +++ b/tensorflow/python/platform/benchmark.py @@ -30,6 +30,7 @@ from tensorflow.core.protobuf import config_pb2 from tensorflow.core.protobuf import rewriter_config_pb2 from tensorflow.core.util import test_log_pb2 from tensorflow.python.client import timeline +from tensorflow.python.framework import ops from tensorflow.python.platform import app from tensorflow.python.platform import gfile from tensorflow.python.platform import tf_logging as logging @@ -299,6 +300,18 @@ class TensorFlowBenchmark(Benchmark): benchmark_values["extras"].update(unreported_extras) return benchmark_values + def evaluate(self, tensors): + """Evaluates tensors and returns numpy values. + + Args: + tensors: A Tensor or a nested list/tuple of Tensors. + + Returns: + tensors numpy values. + """ + sess = ops.get_default_session() or self.cached_session() + return sess.run(tensors) + def _run_benchmarks(regex): """Run benchmarks that match regex `regex`. diff --git a/tensorflow/python/platform/gfile.py b/tensorflow/python/platform/gfile.py index 5927bc2409b..d0159e9e981 100644 --- a/tensorflow/python/platform/gfile.py +++ b/tensorflow/python/platform/gfile.py @@ -37,7 +37,7 @@ from tensorflow.python.util.deprecation import deprecated from tensorflow.python.util.tf_export import tf_export -@tf_export('gfile.GFile', 'gfile.Open') +@tf_export(v1=['gfile.GFile', 'gfile.Open'], v2=['io.gfile.GFile']) class GFile(_FileIO): """File I/O wrappers without thread locking. @@ -52,7 +52,7 @@ class GFile(_FileIO): super(GFile, self).__init__(name=name, mode=mode) -@tf_export('gfile.FastGFile') +@tf_export(v1=['gfile.FastGFile']) class FastGFile(_FileIO): """File I/O wrappers without thread locking. diff --git a/tensorflow/python/platform/googletest.py b/tensorflow/python/platform/googletest.py index 8141cf92c56..4d34c508da8 100644 --- a/tensorflow/python/platform/googletest.py +++ b/tensorflow/python/platform/googletest.py @@ -104,10 +104,13 @@ def GetTempDir(): """Return a temporary directory for tests to use.""" global _googletest_temp_dir if not _googletest_temp_dir: - first_frame = tf_inspect.stack()[-1][0] - temp_dir = os.path.join(tempfile.gettempdir(), - os.path.basename(tf_inspect.getfile(first_frame))) - temp_dir = tempfile.mkdtemp(prefix=temp_dir.rstrip('.py')) + if os.environ.get('TEST_TMPDIR'): + temp_dir = tempfile.mkdtemp(prefix=os.environ['TEST_TMPDIR']) + else: + first_frame = tf_inspect.stack()[-1][0] + temp_dir = os.path.join(tempfile.gettempdir(), + os.path.basename(tf_inspect.getfile(first_frame))) + temp_dir = tempfile.mkdtemp(prefix=temp_dir.rstrip('.py')) def delete_temp_dir(dirname=temp_dir): try: diff --git a/tensorflow/python/platform/test.py b/tensorflow/python/platform/test.py index d084870b255..943832af7a2 100644 --- a/tensorflow/python/platform/test.py +++ b/tensorflow/python/platform/test.py @@ -46,9 +46,9 @@ from tensorflow.python.util.tf_export import tf_export if sys.version_info.major == 2: import mock # pylint: disable=g-import-not-at-top,unused-import else: - from unittest import mock # pylint: disable=g-import-not-at-top + from unittest import mock # pylint: disable=g-import-not-at-top,g-importing-member -tf_export('test.mock')(mock) +tf_export(v1=['test.mock'])(mock) # Import Benchmark class Benchmark = _googletest.Benchmark # pylint: disable=invalid-name diff --git a/tensorflow/python/platform/tf_logging.py b/tensorflow/python/platform/tf_logging.py index 59e60856ae8..813bcb89bea 100644 --- a/tensorflow/python/platform/tf_logging.py +++ b/tensorflow/python/platform/tf_logging.py @@ -37,7 +37,7 @@ import six from tensorflow.python.util.tf_export import tf_export -# Don't use this directly. Use _get_logger() instead. +# Don't use this directly. Use get_logger() instead. _logger = None _logger_lock = threading.Lock() @@ -78,7 +78,8 @@ else: return '(unknown file)', 0, '(unknown function)' -def _get_logger(): +@tf_export('get_logger') +def get_logger(): """Return TF logger instance.""" global _logger @@ -130,39 +131,39 @@ def _get_logger(): _logger_lock.release() -@tf_export('logging.log') +@tf_export(v1=['logging.log']) def log(level, msg, *args, **kwargs): - _get_logger().log(level, msg, *args, **kwargs) + get_logger().log(level, msg, *args, **kwargs) -@tf_export('logging.debug') +@tf_export(v1=['logging.debug']) def debug(msg, *args, **kwargs): - _get_logger().debug(msg, *args, **kwargs) + get_logger().debug(msg, *args, **kwargs) -@tf_export('logging.error') +@tf_export(v1=['logging.error']) def error(msg, *args, **kwargs): - _get_logger().error(msg, *args, **kwargs) + get_logger().error(msg, *args, **kwargs) -@tf_export('logging.fatal') +@tf_export(v1=['logging.fatal']) def fatal(msg, *args, **kwargs): - _get_logger().fatal(msg, *args, **kwargs) + get_logger().fatal(msg, *args, **kwargs) -@tf_export('logging.info') +@tf_export(v1=['logging.info']) def info(msg, *args, **kwargs): - _get_logger().info(msg, *args, **kwargs) + get_logger().info(msg, *args, **kwargs) -@tf_export('logging.warn') +@tf_export(v1=['logging.warn']) def warn(msg, *args, **kwargs): - _get_logger().warn(msg, *args, **kwargs) + get_logger().warn(msg, *args, **kwargs) -@tf_export('logging.warning') +@tf_export(v1=['logging.warning']) def warning(msg, *args, **kwargs): - _get_logger().warning(msg, *args, **kwargs) + get_logger().warning(msg, *args, **kwargs) _level_names = { @@ -183,20 +184,20 @@ _log_prefix = None # later set to google2_log_prefix _log_counter_per_token = {} -@tf_export('logging.TaskLevelStatusMessage') +@tf_export(v1=['logging.TaskLevelStatusMessage']) def TaskLevelStatusMessage(msg): error(msg) -@tf_export('logging.flush') +@tf_export(v1=['logging.flush']) def flush(): raise NotImplementedError() # Code below is taken from pyglib/logging -@tf_export('logging.vlog') +@tf_export(v1=['logging.vlog']) def vlog(level, msg, *args, **kwargs): - _get_logger().log(level, msg, *args, **kwargs) + get_logger().log(level, msg, *args, **kwargs) def _GetNextLogCountPerToken(token): @@ -214,7 +215,7 @@ def _GetNextLogCountPerToken(token): return _log_counter_per_token[token] -@tf_export('logging.log_every_n') +@tf_export(v1=['logging.log_every_n']) def log_every_n(level, msg, n, *args): """Log 'msg % args' at level 'level' once per 'n' times. @@ -231,7 +232,7 @@ def log_every_n(level, msg, n, *args): log_if(level, msg, not (count % n), *args) -@tf_export('logging.log_first_n') +@tf_export(v1=['logging.log_first_n']) def log_first_n(level, msg, n, *args): # pylint: disable=g-bad-name """Log 'msg % args' at level 'level' only first 'n' times. @@ -247,7 +248,7 @@ def log_first_n(level, msg, n, *args): # pylint: disable=g-bad-name log_if(level, msg, count < n, *args) -@tf_export('logging.log_if') +@tf_export(v1=['logging.log_if']) def log_if(level, msg, condition, *args): """Log 'msg % args' at level 'level' only if condition is fulfilled.""" if condition: @@ -296,16 +297,16 @@ def google2_log_prefix(level, timestamp=None, file_and_line=None): return s -@tf_export('logging.get_verbosity') +@tf_export(v1=['logging.get_verbosity']) def get_verbosity(): """Return how much logging output will be produced.""" - return _get_logger().getEffectiveLevel() + return get_logger().getEffectiveLevel() -@tf_export('logging.set_verbosity') +@tf_export(v1=['logging.set_verbosity']) def set_verbosity(v): """Sets the threshold for what messages will be logged.""" - _get_logger().setLevel(v) + get_logger().setLevel(v) def _get_thread_id(): @@ -318,8 +319,8 @@ def _get_thread_id(): _log_prefix = google2_log_prefix -tf_export('logging.DEBUG').export_constant(__name__, 'DEBUG') -tf_export('logging.ERROR').export_constant(__name__, 'ERROR') -tf_export('logging.FATAL').export_constant(__name__, 'FATAL') -tf_export('logging.INFO').export_constant(__name__, 'INFO') -tf_export('logging.WARN').export_constant(__name__, 'WARN') +tf_export(v1=['logging.DEBUG']).export_constant(__name__, 'DEBUG') +tf_export(v1=['logging.ERROR']).export_constant(__name__, 'ERROR') +tf_export(v1=['logging.FATAL']).export_constant(__name__, 'FATAL') +tf_export(v1=['logging.INFO']).export_constant(__name__, 'INFO') +tf_export(v1=['logging.WARN']).export_constant(__name__, 'WARN') diff --git a/tensorflow/python/profiler/internal/run_metadata_test.py b/tensorflow/python/profiler/internal/run_metadata_test.py index 216cc3dd54b..a8859f845b3 100644 --- a/tensorflow/python/profiler/internal/run_metadata_test.py +++ b/tensorflow/python/profiler/internal/run_metadata_test.py @@ -26,6 +26,7 @@ from tensorflow.core.protobuf import config_pb2 from tensorflow.core.protobuf import rewriter_config_pb2 from tensorflow.python.client import session from tensorflow.python.framework import ops +from tensorflow.python.framework import test_util from tensorflow.python.ops import math_ops from tensorflow.python.ops import random_ops from tensorflow.python.ops import variables @@ -154,6 +155,7 @@ class RunMetadataTest(test.TestCase): # deallocates the memory after matmul started. self.assertGreater(random_allocs[1].alloc_micros, mm.all_start_micros) + @test_util.run_deprecated_v1 def testCPU(self): ops.reset_default_graph() with ops.device('/cpu:0'): @@ -167,6 +169,7 @@ class RunMetadataTest(test.TestCase): ret = _extract_node(run_meta, 'MatMul:MatMul') self.assertEqual(len(ret), 0) + @test_util.run_deprecated_v1 def testLoopCPU(self): ops.reset_default_graph() with ops.device('/cpu:0'): diff --git a/tensorflow/python/profiler/model_analyzer.py b/tensorflow/python/profiler/model_analyzer.py index 5f19eac0436..4b2d9052b78 100644 --- a/tensorflow/python/profiler/model_analyzer.py +++ b/tensorflow/python/profiler/model_analyzer.py @@ -122,7 +122,7 @@ def _build_advisor_options(options): return opts -@tf_export('profiler.Profiler') +@tf_export(v1=['profiler.Profiler']) class Profiler(object): """TensorFlow multi-step profiler. @@ -306,7 +306,7 @@ class Profiler(object): print_mdl.WriteProfile(filename) -@tf_export('profiler.profile') +@tf_export(v1=['profiler.profile']) def profile(graph=None, run_meta=None, op_log=None, @@ -381,7 +381,7 @@ def profile(graph=None, return tfprof_node -@tf_export('profiler.advise') +@tf_export(v1=['profiler.advise']) def advise(graph=None, run_meta=None, options=_DEFAULT_ADVISE_OPTIONS): """Auto profile and advise. diff --git a/tensorflow/python/profiler/model_analyzer_test.py b/tensorflow/python/profiler/model_analyzer_test.py index 94c685274a7..8648f0b5148 100644 --- a/tensorflow/python/profiler/model_analyzer_test.py +++ b/tensorflow/python/profiler/model_analyzer_test.py @@ -93,10 +93,10 @@ class PrintModelAnalysisTest(test.TestCase): config=self._no_rewrite_session_config()) as sess, ops.device(dev): x = lib.BuildSmallModel() - sess.run(variables.global_variables_initializer()) + self.evaluate(variables.global_variables_initializer()) pctx.trace_next_step() pctx.dump_next_step() - _ = sess.run(x) + _ = self.evaluate(x) pctx.profiler.profile_name_scope(options=opts) @@ -160,7 +160,7 @@ class PrintModelAnalysisTest(test.TestCase): ) as sess, ops.device('/device:CPU:0'): x = lib.BuildSmallModel() - sess.run(variables.global_variables_initializer()) + self.evaluate(variables.global_variables_initializer()) run_meta = config_pb2.RunMetadata() _ = sess.run(x, options=config_pb2.RunOptions( @@ -186,7 +186,7 @@ class PrintModelAnalysisTest(test.TestCase): with session.Session(config=self._no_rewrite_session_config()) as sess: x = lib.BuildSmallModel() - sess.run(variables.global_variables_initializer()) + self.evaluate(variables.global_variables_initializer()) run_meta = config_pb2.RunMetadata() _ = sess.run(x, options=config_pb2.RunOptions( @@ -220,9 +220,9 @@ class PrintModelAnalysisTest(test.TestCase): with session.Session(config=self._no_rewrite_session_config()) as sess: x = lib.BuildFullModel() - sess.run(variables.global_variables_initializer()) + self.evaluate(variables.global_variables_initializer()) pctx.trace_next_step() - _ = sess.run(x) + _ = self.evaluate(x) tfprof_node = pctx.profiler.profile_python(options=opts) # pylint: disable=line-too-long @@ -281,7 +281,7 @@ class PrintModelAnalysisTest(test.TestCase): with session.Session(config=self._no_rewrite_session_config()) as sess: x = lib.BuildSmallModel() - sess.run(variables.global_variables_initializer()) + self.evaluate(variables.global_variables_initializer()) run_meta = config_pb2.RunMetadata() _ = sess.run(x, options=config_pb2.RunOptions( @@ -309,7 +309,7 @@ class PrintModelAnalysisTest(test.TestCase): with session.Session(config=self._no_rewrite_session_config()) as sess: x = lib.BuildFullModel() - sess.run(variables.global_variables_initializer()) + self.evaluate(variables.global_variables_initializer()) run_meta = config_pb2.RunMetadata() _ = sess.run( x, @@ -345,7 +345,7 @@ class PrintModelAnalysisTest(test.TestCase): with session.Session(config=self._no_rewrite_session_config()) as sess: x = lib.BuildFullModel() - sess.run(variables.global_variables_initializer()) + self.evaluate(variables.global_variables_initializer()) run_meta = config_pb2.RunMetadata() _ = sess.run(x, options=config_pb2.RunOptions( @@ -391,7 +391,7 @@ class PrintModelAnalysisTest(test.TestCase): with session.Session(config=self._no_rewrite_session_config()) as sess: x = lib.BuildFullModel() - sess.run(variables.global_variables_initializer()) + self.evaluate(variables.global_variables_initializer()) run_meta = config_pb2.RunMetadata() _ = sess.run( x, @@ -424,7 +424,7 @@ class PrintModelAnalysisTest(test.TestCase): with session.Session(config=self._no_rewrite_session_config()) as sess: x = lib.BuildFullModel() - sess.run(variables.global_variables_initializer()) + self.evaluate(variables.global_variables_initializer()) run_meta = config_pb2.RunMetadata() _ = sess.run( x, @@ -490,7 +490,7 @@ class PrintModelAnalysisTest(test.TestCase): with session.Session(config=self._no_rewrite_session_config()) as sess: x = lib.BuildSmallModel() - sess.run(variables.global_variables_initializer()) + self.evaluate(variables.global_variables_initializer()) run_meta = config_pb2.RunMetadata() _ = sess.run(x, options=config_pb2.RunOptions( @@ -555,7 +555,7 @@ class PrintModelAnalysisTest(test.TestCase): with session.Session(config=self._no_rewrite_session_config()) as sess: x = lib.BuildSmallModel() - sess.run(variables.global_variables_initializer()) + self.evaluate(variables.global_variables_initializer()) run_meta = config_pb2.RunMetadata() _ = sess.run(x, options=config_pb2.RunOptions( @@ -587,10 +587,10 @@ class PrintModelAnalysisTest(test.TestCase): def _trainLoop(self, train_op, train_steps, time_dir, time_step, memory_dir, memory_step, profile_dir, dump_step): with session.Session(config=self._no_rewrite_session_config()) as sess: - sess.run(variables.global_variables_initializer()) + self.evaluate(variables.global_variables_initializer()) # start from 1 because variable_initializer took one step. for i in range(1, train_steps + 1): - _ = sess.run(train_op) + _ = self.evaluate(train_op) if i in time_step: ret = gfile.ListDirectory(time_dir) self.assertEqual(len(ret), 1) diff --git a/tensorflow/python/profiler/option_builder.py b/tensorflow/python/profiler/option_builder.py index 2ad7adf7693..9d8f7683a65 100644 --- a/tensorflow/python/profiler/option_builder.py +++ b/tensorflow/python/profiler/option_builder.py @@ -23,7 +23,7 @@ from tensorflow.python.profiler import tfprof_logger from tensorflow.python.util.tf_export import tf_export -@tf_export('profiler.ProfileOptionBuilder') +@tf_export(v1=['profiler.ProfileOptionBuilder']) class ProfileOptionBuilder(object): # pylint: disable=line-too-long """Option Builder for Profiling API. diff --git a/tensorflow/python/profiler/pprof_profiler_test.py b/tensorflow/python/profiler/pprof_profiler_test.py index 11a3487360c..120a0d0eaa6 100644 --- a/tensorflow/python/profiler/pprof_profiler_test.py +++ b/tensorflow/python/profiler/pprof_profiler_test.py @@ -24,6 +24,7 @@ from proto import profile_pb2 from tensorflow.core.framework import step_stats_pb2 from tensorflow.core.protobuf import config_pb2 from tensorflow.python.framework import constant_op +from tensorflow.python.framework import test_util from tensorflow.python.ops import control_flow_ops from tensorflow.python.ops import math_ops from tensorflow.python.platform import test @@ -135,6 +136,7 @@ comment: 9 profile.ParseFromString(profile_contents) self.assertEquals(expected_proto, str(profile)) + @test_util.run_deprecated_v1 def testProfileWithWhileLoop(self): options = config_pb2.RunOptions() options.trace_level = config_pb2.RunOptions.FULL_TRACE diff --git a/tensorflow/python/profiler/profile_context_test.py b/tensorflow/python/profiler/profile_context_test.py index 107ad443c32..885f08ca4b9 100644 --- a/tensorflow/python/profiler/profile_context_test.py +++ b/tensorflow/python/profiler/profile_context_test.py @@ -21,6 +21,7 @@ import os from tensorflow.python.client import session from tensorflow.python.framework import ops +from tensorflow.python.framework import test_util from tensorflow.python.ops import variables from tensorflow.python.platform import gfile from tensorflow.python.platform import test @@ -35,6 +36,7 @@ builder = option_builder.ProfileOptionBuilder class ProfilerContextTest(test.TestCase): + @test_util.run_deprecated_v1 def testBasics(self): ops.reset_default_graph() outfile = os.path.join(test.get_temp_dir(), "dump") @@ -48,10 +50,10 @@ class ProfilerContextTest(test.TestCase): with profile_context.ProfileContext(test.get_temp_dir()) as pctx: pctx.add_auto_profiling("op", options=opts, profile_steps=[15, 50, 100]) with session.Session() as sess: - sess.run(variables.global_variables_initializer()) + self.evaluate(variables.global_variables_initializer()) total_steps = 101 for i in range(total_steps): - sess.run(x) + self.evaluate(x) if i == 14 or i == 49: self.assertTrue(gfile.Exists(outfile)) gfile.Remove(outfile) @@ -69,45 +71,47 @@ class ProfilerContextTest(test.TestCase): with gfile.Open(outfile, "r") as f: self.assertEqual(profile_str, f.read()) + @test_util.run_deprecated_v1 def testAutoTracingInDeubMode(self): ops.reset_default_graph() x = lib.BuildFullModel() with profile_context.ProfileContext(test.get_temp_dir(), debug=True): with session.Session() as sess: - sess.run(variables.global_variables_initializer()) + self.evaluate(variables.global_variables_initializer()) for _ in range(10): - sess.run(x) + self.evaluate(x) for f in gfile.ListDirectory(test.get_temp_dir()): # Warm up, no tracing. self.assertFalse("run_meta" in f) - sess.run(x) + self.evaluate(x) self.assertTrue( gfile.Exists(os.path.join(test.get_temp_dir(), "run_meta_11"))) gfile.Remove(os.path.join(test.get_temp_dir(), "run_meta_11")) # fetched already. - sess.run(x) + self.evaluate(x) for f in gfile.ListDirectory(test.get_temp_dir()): self.assertFalse("run_meta" in f) + @test_util.run_deprecated_v1 def testDisabled(self): ops.reset_default_graph() x = lib.BuildFullModel() with profile_context.ProfileContext(test.get_temp_dir(), enabled=False) as pctx: with session.Session() as sess: - sess.run(variables.global_variables_initializer()) + self.evaluate(variables.global_variables_initializer()) for _ in range(10): - sess.run(x) + self.evaluate(x) self.assertTrue(pctx.profiler is None) self.assertTrue( getattr(session.BaseSession, "profile_context", None) is None) with profile_context.ProfileContext(test.get_temp_dir()) as pctx: with session.Session() as sess: - sess.run(variables.global_variables_initializer()) + self.evaluate(variables.global_variables_initializer()) for _ in range(10): - sess.run(x) + self.evaluate(x) self.assertFalse(pctx.profiler is None) self.assertFalse( getattr(session.BaseSession, "profile_context", None) is None) diff --git a/tensorflow/python/profiler/profiler.py b/tensorflow/python/profiler/profiler.py index efbdd1ba684..5f62690b54e 100644 --- a/tensorflow/python/profiler/profiler.py +++ b/tensorflow/python/profiler/profiler.py @@ -49,7 +49,7 @@ _allowed_symbols.extend([ ]) # Export protos -tf_export('profiler.GraphNodeProto')(GraphNodeProto) -tf_export('profiler.MultiGraphNodeProto')(MultiGraphNodeProto) -tf_export('profiler.AdviceProto')(AdviceProto) -tf_export('profiler.OpLogProto')(OpLogProto) +tf_export(v1=['profiler.GraphNodeProto'])(GraphNodeProto) +tf_export(v1=['profiler.MultiGraphNodeProto'])(MultiGraphNodeProto) +tf_export(v1=['profiler.AdviceProto'])(AdviceProto) +tf_export(v1=['profiler.OpLogProto'])(OpLogProto) diff --git a/tensorflow/python/profiler/profiler_test.py b/tensorflow/python/profiler/profiler_test.py index eacb7d21e6a..e4f7361e5d7 100644 --- a/tensorflow/python/profiler/profiler_test.py +++ b/tensorflow/python/profiler/profiler_test.py @@ -21,6 +21,7 @@ import os from tensorflow.core.protobuf import config_pb2 from tensorflow.python.client import session from tensorflow.python.framework import ops +from tensorflow.python.framework import test_util from tensorflow.python.ops import variables from tensorflow.python.platform import gfile from tensorflow.python.platform import test @@ -35,6 +36,7 @@ builder = option_builder.ProfileOptionBuilder class ProfilerTest(test.TestCase): + @test_util.run_deprecated_v1 def testProfileBasic(self): ops.reset_default_graph() outfile = os.path.join(test.get_temp_dir(), 'dump') @@ -171,6 +173,7 @@ class ProfilerTest(test.TestCase): checker = advice_pb.checkers['ExpensiveOperationChecker'] self.assertGreater(len(checker.reports), 0) + @test_util.run_deprecated_v1 def testMultipleProfilePerStep(self): ops.reset_default_graph() opts = (builder(builder.trainable_variables_parameter()) diff --git a/tensorflow/python/profiler/tfprof_logger.py b/tensorflow/python/profiler/tfprof_logger.py index e651de32ea3..6ccd0e0ff3b 100644 --- a/tensorflow/python/profiler/tfprof_logger.py +++ b/tensorflow/python/profiler/tfprof_logger.py @@ -188,7 +188,7 @@ def merge_default_with_oplog(graph, op_log=None, run_meta=None, return tmp_op_log -@tf_export('profiler.write_op_log') +@tf_export(v1=['profiler.write_op_log']) def write_op_log(graph, log_dir, op_log=None, run_meta=None, add_trace=True): """Log provided 'op_log', and add additional model information below. diff --git a/tensorflow/python/saved_model/BUILD b/tensorflow/python/saved_model/BUILD index e7a3b8afd5d..53d0640542f 100644 --- a/tensorflow/python/saved_model/BUILD +++ b/tensorflow/python/saved_model/BUILD @@ -12,6 +12,8 @@ licenses(["notice"]) # Apache 2.0 exports_files(["LICENSE"]) load("//tensorflow:tensorflow.bzl", "py_test") +load("//tensorflow/core:platform/default/build_config.bzl", "tf_proto_library") +load("//tensorflow/core:platform/default/build_config.bzl", "tf_additional_all_protos") py_library( name = "saved_model", @@ -21,6 +23,7 @@ py_library( deps = [ ":builder", ":constants", + ":load", ":loader", ":main_op", ":save", @@ -83,12 +86,13 @@ py_library( srcs_version = "PY2AND3", deps = [ ":constants", + ":signature_def_utils", ":utils", "//tensorflow/core:protos_all_py", "//tensorflow/python:framework_for_generated_wrappers", "//tensorflow/python:lib", "//tensorflow/python:platform", - "//tensorflow/python:training", + "//tensorflow/python:saver", "//tensorflow/python:util", "//tensorflow/python:variables", ], @@ -114,6 +118,7 @@ py_test( "//tensorflow/python:state_ops", "//tensorflow/python:training", "//tensorflow/python:variables", + "@absl_py//absl/testing:parameterized", ], ) @@ -166,14 +171,15 @@ py_test( ":signature_def_utils", ":tag_constants", "//tensorflow/core:protos_all_py", - "//tensorflow/python:client", "//tensorflow/python:client_testlib", "//tensorflow/python:control_flow_ops", "//tensorflow/python:errors", "//tensorflow/python:framework_for_generated_wrappers", + "//tensorflow/python:framework_test_lib", "//tensorflow/python:lib", "//tensorflow/python:math_ops", "//tensorflow/python:saver_test_utils", + "//tensorflow/python:session", "//tensorflow/python:state_ops", "//tensorflow/python:test_ops", "//tensorflow/python:training", @@ -264,6 +270,14 @@ py_test( ], ) +tf_proto_library( + name = "saved_object_graph", + srcs = ["saved_object_graph.proto"], + cc_api_version = 2, + protodeps = tf_additional_all_protos(), + visibility = ["//tensorflow:internal"], +) + py_library( name = "save", srcs = [ @@ -271,16 +285,26 @@ py_library( ], srcs_version = "PY2AND3", deps = [ + ":builder", + ":constants", ":loader", + ":saved_object_graph_py", ":signature_constants", ":signature_def_utils", + ":tag_constants", ":utils", "//tensorflow/core:protos_all_py", "//tensorflow/python:array_ops", + "//tensorflow/python:framework", "//tensorflow/python:framework_ops", + "//tensorflow/python:lib", + "//tensorflow/python:resource_variable_ops", "//tensorflow/python:util", + "//tensorflow/python/eager:context", "//tensorflow/python/eager:def_function", - "//tensorflow/python/eager:test", + "//tensorflow/python/eager:function", + "//tensorflow/python/training/checkpointable:base", + "//tensorflow/python/training/checkpointable:util", ], ) @@ -289,13 +313,42 @@ py_test( srcs = ["save_test.py"], srcs_version = "PY2AND3", deps = [ + ":loader", ":save", ":signature_constants", ":tag_constants", "//tensorflow/python/eager:def_function", + "//tensorflow/python/eager:test", "@absl_py//absl/testing:parameterized", ], ) -# ----------------------------------------------------------------------------- -# Google-internal targets. These must be at the end for syncrepo. +py_library( + name = "load", + srcs = [ + "load.py", + ], + srcs_version = "PY2AND3", + deps = [ + ":loader", + ":saved_object_graph_py", + "//tensorflow/python:lib", + "//tensorflow/python:util", + "//tensorflow/python/training/checkpointable:tracking", + ], +) + +py_test( + name = "load_test", + srcs = ["load_test.py"], + srcs_version = "PY2AND3", + deps = [ + ":load", + ":save", + "//tensorflow/python:dtypes", + "//tensorflow/python:tensor_spec", + "//tensorflow/python/eager:def_function", + "//tensorflow/python/eager:test", + "//tensorflow/python/training/checkpointable:tracking", + ], +) diff --git a/tensorflow/python/saved_model/builder.py b/tensorflow/python/saved_model/builder.py index be49c70c604..b929934eebb 100644 --- a/tensorflow/python/saved_model/builder.py +++ b/tensorflow/python/saved_model/builder.py @@ -24,5 +24,6 @@ from __future__ import division from __future__ import print_function # pylint: disable=unused-import +from tensorflow.python.saved_model.builder_impl import _SavedModelBuilder from tensorflow.python.saved_model.builder_impl import SavedModelBuilder # pylint: enable=unused-import diff --git a/tensorflow/python/saved_model/builder_impl.py b/tensorflow/python/saved_model/builder_impl.py index 4f68f7c5aea..f37d283a2a2 100644 --- a/tensorflow/python/saved_model/builder_impl.py +++ b/tensorflow/python/saved_model/builder_impl.py @@ -18,6 +18,7 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function +import functools import os from google.protobuf.any_pb2 import Any @@ -32,6 +33,7 @@ from tensorflow.python.lib.io import file_io from tensorflow.python.ops import variables from tensorflow.python.platform import tf_logging from tensorflow.python.saved_model import constants +from tensorflow.python.saved_model import signature_def_utils from tensorflow.python.saved_model import utils_impl as saved_model_utils from tensorflow.python.training import saver as tf_saver from tensorflow.python.util import compat @@ -39,8 +41,9 @@ from tensorflow.python.util.deprecation import deprecated_args from tensorflow.python.util.tf_export import tf_export -@tf_export(v1=["saved_model.Builder", "saved_model.builder.SavedModelBuilder"]) -class SavedModelBuilder(object): +# Base class for the SavedModelBuilder that is only used by Tensorflow +# internally. Please use tf.compat.v1.saved_model.SavedModelBuilder instead. +class _SavedModelBuilder(object): """Builds the `SavedModel` protocol buffer and saves variables and assets. The `SavedModelBuilder` class provides functionality to build a `SavedModel` @@ -68,7 +71,7 @@ class SavedModelBuilder(object): builder.add_meta_graph_and_variables(sess, ["foo-tag"], signature_def_map=foo_signatures, - assets_collection=foo_assets) + assets_list=foo_assets) ... with tf.Session(graph=tf.Graph()) as sess: @@ -105,82 +108,24 @@ class SavedModelBuilder(object): # weights. self._has_saved_variables = False - def _save_and_write_assets(self, assets_collection_to_add=None): + def _save_and_write_assets(self, meta_graph_def, assets_list=None): """Saves asset to the meta graph and writes asset files to disk. Args: - assets_collection_to_add: The collection where the asset paths are setup. + meta_graph_def: The meta graph def to which the assets will be added. + assets_list: The list where the asset paths are setup. """ - asset_filename_map = _maybe_save_assets(assets_collection_to_add) + # Creates a function that adds assets into the meta graph def. + write_fn = functools.partial(_add_asset_to_metagraph, meta_graph_def) + asset_filename_map = _maybe_save_assets(write_fn, assets_list) # Return if there are no assets to write. if not asset_filename_map: tf_logging.info("No assets to write.") return - assets_destination_dir = saved_model_utils.get_or_create_assets_dir( - self._export_dir) - - # Copy each asset from source path to destination path. - for asset_basename, asset_source_filepath in asset_filename_map.items(): - asset_destination_filepath = os.path.join( - compat.as_bytes(assets_destination_dir), - compat.as_bytes(asset_basename)) - - # Only copy the asset file to the destination if it does not already - # exist. This is to ensure that an asset with the same name defined as - # part of multiple graphs is only copied the first time. - if not file_io.file_exists(asset_destination_filepath): - file_io.copy(asset_source_filepath, asset_destination_filepath) - - tf_logging.info("Assets written to: %s", - compat.as_text(assets_destination_dir)) - - def _maybe_add_main_op(self, main_op): - """Adds main op to the SavedModel. - - Args: - main_op: Main op to run as part of graph initialization. If None, no - main op will be added to the graph. - - Raises: - TypeError: if main op is provided but is not of type `Operation`. - ValueError: if the Graph already contains an init op. - """ - if main_op is None: - return - - if not isinstance(main_op, ops.Operation): - raise TypeError("main_op needs to be an Operation: %r" % main_op) - - # Validate that no other init ops have been added to this graph already. - # We check main_op and legacy_init_op for thoroughness and explicitness. - for init_op_key in (constants.MAIN_OP_KEY, constants.LEGACY_INIT_OP_KEY): - if ops.get_collection(init_op_key): - raise ValueError( - "Graph already contains one or more main ops under the " - "collection {}.".format(init_op_key)) - - ops.add_to_collection(constants.MAIN_OP_KEY, main_op) - - def _add_train_op(self, train_op): - """Add train op to the SavedModel. - - Note that this functionality is in development, and liable to be - moved elsewhere. - - Args: - train_op: Op or group of ops that are used for training. These are - stored as a collection with key TRAIN_OP_KEY, but not executed. - - Raises: - TypeError if Train op is not of type `Operation`. - """ - if train_op is not None: - if (not isinstance(train_op, ops.Tensor) and - not isinstance(train_op, ops.Operation)): - raise TypeError("train_op needs to be a Tensor or Op: %r" % train_op) - ops.add_to_collection(constants.TRAIN_OP_KEY, train_op) + # Copy assets from source path to destination path. + copy_assets_to_destination_dir(asset_filename_map, self._export_dir) def _tag_and_add_meta_graph(self, meta_graph_def, tags, signature_def_map): """Tags the meta graph def and adds it to the SavedModel. @@ -237,30 +182,32 @@ class SavedModelBuilder(object): Validation of entries in the signature def map includes ensuring that the `name` and `dtype` fields of the TensorInfo protos of the `inputs` and - `outputs` of each `SignatureDef` are populated. + `outputs` of each `SignatureDef` are populated. Also ensures that reserved + SigantureDef keys for the initialization and train ops are not used. Args: signature_def_map: The map of signature defs to be validated. + + Raises: + AssertionError: If a TensorInfo is not valid. + KeyError: If a reserved signature key is used in the map. """ - if signature_def_map is not None: - for signature_def_key in signature_def_map: - signature_def = signature_def_map[signature_def_key] - inputs = signature_def.inputs - outputs = signature_def.outputs - for inputs_key in inputs: - self._validate_tensor_info(inputs[inputs_key]) - for outputs_key in outputs: - self._validate_tensor_info(outputs[outputs_key]) - - def _add_collections( - self, assets_collection, main_op, train_op): - """Add asset and op collections to be saved.""" - # Save asset files and write them to disk, if any. - self._save_and_write_assets(assets_collection) - - self._maybe_add_main_op(main_op) - - self._add_train_op(train_op) + for signature_def_key in signature_def_map: + signature_def = signature_def_map[signature_def_key] + inputs = signature_def.inputs + outputs = signature_def.outputs + for inputs_key in inputs: + self._validate_tensor_info(inputs[inputs_key]) + for outputs_key in outputs: + self._validate_tensor_info(outputs[outputs_key]) + if constants.INIT_OP_SIGNATURE_KEY in signature_def_map: + raise KeyError( + "SignatureDef map key \"{}\" is reserved for initialization. Please " + "use a different key.".format(constants.INIT_OP_SIGNATURE_KEY)) + if constants.TRAIN_OP_SIGNATURE_KEY in signature_def_map: + raise KeyError( + "SignatureDef map key \"{}\" is reserved for the train op. Please " + "use a different key.".format(constants.TRAIN_OP_SIGNATURE_KEY)) def _maybe_create_saver(self, saver=None): """Creates a sharded saver if one does not already exist.""" @@ -274,6 +221,283 @@ class SavedModelBuilder(object): allow_empty=True) return saver + def add_meta_graph(self, + tags, + signature_def_map=None, + assets_list=None, + clear_devices=False, + init_op=None, + train_op=None, + saver=None): + """Adds the current meta graph to the SavedModel. + + Creates a Saver in the current scope and uses the Saver to export the meta + graph def. Invoking this API requires the `add_meta_graph_and_variables()` + API to have been invoked before. + + Args: + tags: The set of tags to annotate the meta graph def with. + signature_def_map: The map of signature defs to be added to the meta graph + def. + assets_list: Assets to be saved with SavedModel. Note + that this list should be a subset of the assets saved as part of + the first meta graph in the SavedModel. + clear_devices: Set to true if the device info on the default graph should + be cleared. + init_op: Op or group of ops to execute when the graph is loaded. Note + that when the init_op is specified it is run after the restore op at + load-time. + train_op: Op or group of opts that trains the model when run. This will + not be run automatically when the graph is loaded, instead saved in + a SignatureDef accessible through the exported MetaGraph. + saver: An instance of tf.train.Saver that will be used to export the + metagraph. If None, a sharded Saver that restores all variables will + be used. + + Raises: + AssertionError: If the variables for the SavedModel have not been saved + yet, or if the graph already contains one or more legacy init ops. + """ + if not self._has_saved_variables: + raise AssertionError( + "Graph state including variables and assets has not been saved yet. " + "Please invoke `add_meta_graph_and_variables()` first.") + + # Validate the signature def map to ensure all included TensorInfos are + # properly populated. + signature_def_map = signature_def_map or {} + self._validate_signature_def_map(signature_def_map) + + # Create a SignatureDef pointing to the graph initialization op, which will + # be added to the MetaGraphDef. + _add_op_to_signature_def_map(signature_def_map, init_op, + constants.INIT_OP_SIGNATURE_KEY) + _add_op_to_signature_def_map(signature_def_map, train_op, + constants.TRAIN_OP_SIGNATURE_KEY) + + saver = self._maybe_create_saver(saver) + + # The graph almost certainly previously contained at least one Saver, and + # possibly several (e.g. one for loading a pretrained embedding, and another + # for the model weights). Removing the preexisting ones was the + # motivation for the clear_extraneous_savers option, but it turns out that + # there are edge cases where that option breaks the graph. Until that is + # resolved, we just leave the option set to False for now. + # TODO(soergel): Reinstate clear_extraneous_savers=True when possible. + meta_graph_def = saver.export_meta_graph( + clear_devices=clear_devices, strip_default_attrs=True) + + # Save asset files and write them to disk, if any. + self._save_and_write_assets(meta_graph_def, assets_list) + + # Tag the meta graph def and add it to the SavedModel. + self._tag_and_add_meta_graph(meta_graph_def, tags, signature_def_map) + + def add_meta_graph_and_variables(self, + sess, + tags, + signature_def_map=None, + assets_list=None, + clear_devices=False, + init_op=None, + train_op=None, + strip_default_attrs=False, + saver=None): + # pylint: disable=line-too-long + """Adds the current meta graph to the SavedModel and saves variables. + + Creates a Saver to save the variables from the provided session. Exports the + corresponding meta graph def. This function assumes that the variables to be + saved have been initialized. For a given `SavedModelBuilder`, this API must + be called exactly once and for the first meta graph to save. For subsequent + meta graph defs to be added, the `add_meta_graph()` API must be used. + + Args: + sess: The TensorFlow session from which to save the meta graph and + variables. + tags: The set of tags with which to save the meta graph. + signature_def_map: The map of signature def map to add to the meta graph + def. + assets_list: Assets to be saved with SavedModel. + clear_devices: Set to true if the device info on the default graph should + be cleared. + init_op: Op or group of ops to execute when the graph is loaded. Note + that when the init_op is specified it is run after the restore op at + load-time. + train_op: Op or group of ops that trains the model when run. This will + not be run automatically when the graph is loaded, instead saved in + a SignatureDef accessible through the exported MetaGraph. + strip_default_attrs: Boolean. If `True`, default-valued attributes will be + removed from the NodeDefs. For a detailed guide, see + [Stripping Default-Valued Attributes](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/python/saved_model/README.md#stripping-default-valued-attributes). + saver: An instance of tf.train.Saver that will be used to export the + metagraph and save variables. If None, a sharded Saver that restores + all variables will be used. + + """ + # pylint: enable=line-too-long + if self._has_saved_variables: + raise AssertionError("Graph state including variables and assets has " + "already been saved. Please invoke " + "`add_meta_graph()` instead.") + + # Validate the signature def map to ensure all included TensorInfos are + # properly populated. + signature_def_map = signature_def_map or {} + self._validate_signature_def_map(signature_def_map) + + # Create a SignatureDef pointing to the graph initialization op, which will + # be added to the MetaGraphDef. + _add_op_to_signature_def_map(signature_def_map, init_op, + constants.INIT_OP_SIGNATURE_KEY) + _add_op_to_signature_def_map(signature_def_map, train_op, + constants.TRAIN_OP_SIGNATURE_KEY) + + saved_model_utils.get_or_create_variables_dir(self._export_dir) + variables_path = saved_model_utils.get_variables_path(self._export_dir) + + saver = self._maybe_create_saver(saver) + + # Save the variables. Also, disable writing the checkpoint state proto. The + # file is not used during SavedModel loading. In addition, since a + # SavedModel can be copied or moved, this avoids the checkpoint state to + # become outdated. + saver.save(sess, variables_path, write_meta_graph=False, write_state=False) + + # Export the meta graph def. + + # The graph almost certainly previously contained at least one Saver, and + # possibly several (e.g. one for loading a pretrained embedding, and another + # for the model weights). Removing the preexisting ones was the + # motivation for the clear_extraneous_savers option, but it turns out that + # there are edge cases where that option breaks the graph. Until that is + # resolved, we just leave the option set to False for now. + # TODO(soergel): Reinstate clear_extraneous_savers=True when possible. + meta_graph_def = saver.export_meta_graph( + clear_devices=clear_devices, strip_default_attrs=strip_default_attrs) + + # Save asset files and write them to disk, if any. + self._save_and_write_assets(meta_graph_def, assets_list) + + # Tag the meta graph def and add it to the SavedModel. + self._tag_and_add_meta_graph(meta_graph_def, tags, signature_def_map) + + # Mark this instance of SavedModel as having saved variables, such that + # subsequent attempts to save variables will fail. + self._has_saved_variables = True + + def save(self, as_text=False): + """Writes a `SavedModel` protocol buffer to disk. + + The function writes the SavedModel protocol buffer to the export directory + in serialized format. + + Args: + as_text: Writes the SavedModel protocol buffer in text format to disk. + + Returns: + The path to which the SavedModel protocol buffer was written. + """ + if not file_io.file_exists(self._export_dir): + file_io.recursive_create_dir(self._export_dir) + + if as_text: + path = os.path.join( + compat.as_bytes(self._export_dir), + compat.as_bytes(constants.SAVED_MODEL_FILENAME_PBTXT)) + file_io.write_string_to_file(path, str(self._saved_model)) + else: + path = os.path.join( + compat.as_bytes(self._export_dir), + compat.as_bytes(constants.SAVED_MODEL_FILENAME_PB)) + file_io.write_string_to_file(path, self._saved_model.SerializeToString()) + tf_logging.info("SavedModel written to: %s", compat.as_text(path)) + + return path + + +@tf_export(v1=["saved_model.Builder", "saved_model.builder.SavedModelBuilder"]) # pylint: disable=missing-docstring +class SavedModelBuilder(_SavedModelBuilder): + __doc__ = _SavedModelBuilder.__doc__.replace("assets_list", + "assets_collection") + + def __init__(self, export_dir): + super(SavedModelBuilder, self).__init__(export_dir=export_dir) + + def _add_collections(self, assets_collection, main_op, train_op): + """Add asset and op collections to be saved.""" + # Save asset files and write them to disk, if any. + self._save_and_write_assets(assets_collection) + + self._maybe_add_main_op(main_op) + + self._add_train_op(train_op) + + def _save_and_write_assets(self, assets_collection_to_add=None): + """Saves asset to the meta graph and writes asset files to disk. + + Args: + assets_collection_to_add: The collection where the asset paths are setup. + """ + # Add assets to the collection with key `constants.ASSETS_KEY`, in the + # graph. + asset_filename_map = _maybe_save_assets(_add_asset_to_collection, + assets_collection_to_add) + + # Return if there are no assets to write. + if not asset_filename_map: + tf_logging.info("No assets to write.") + return + + # Copy assets from source path to destination path. + copy_assets_to_destination_dir(asset_filename_map, self._export_dir) + + def _maybe_add_main_op(self, main_op): + """Adds main op to the SavedModel. + + Args: + main_op: Main op to run as part of graph initialization. If None, no main + op will be added to the graph. + + Raises: + TypeError: if main op is provided but is not of type `Operation`. + ValueError: if the Graph already contains an init op. + """ + if main_op is None: + return + + if not isinstance(main_op, ops.Operation): + raise TypeError("main_op needs to be an Operation: %r" % main_op) + + # Validate that no other init ops have been added to this graph already. + # We check main_op and legacy_init_op for thoroughness and explicitness. + for init_op_key in (constants.MAIN_OP_KEY, constants.LEGACY_INIT_OP_KEY): + if ops.get_collection(init_op_key): + raise ValueError( + "Graph already contains one or more main ops under the " + "collection {}.".format(init_op_key)) + + ops.add_to_collection(constants.MAIN_OP_KEY, main_op) + + def _add_train_op(self, train_op): + """Add train op to the SavedModel. + + Note that this functionality is in development, and liable to be + moved elsewhere. + + Args: + train_op: Op or group of ops that are used for training. These are stored + as a collection with key TRAIN_OP_KEY, but not executed. + + Raises: + TypeError if Train op is not of type `Operation`. + """ + if train_op is not None: + if (not isinstance(train_op, ops.Tensor) and + not isinstance(train_op, ops.Operation)): + raise TypeError("train_op needs to be a Tensor or Op: %r" % train_op) + ops.add_to_collection(constants.TRAIN_OP_KEY, train_op) + @deprecated_args(None, "Pass your op to the equivalent parameter main_op instead.", "legacy_init_op") @@ -286,39 +510,6 @@ class SavedModelBuilder(object): main_op=None, strip_default_attrs=False, saver=None): - # pylint: disable=line-too-long - """Adds the current meta graph to the SavedModel. - - Creates a Saver in the current scope and uses the Saver to export the meta - graph def. Invoking this API requires the `add_meta_graph_and_variables()` - API to have been invoked before. - - Args: - tags: The set of tags to annotate the meta graph def with. - signature_def_map: The map of signature defs to be added to the meta graph - def. - assets_collection: Assets collection to be saved with SavedModel. Note - that this collection should be a subset of the assets saved as part of - the first meta graph in the SavedModel. - legacy_init_op: Legacy support for op or group of ops to execute after the - restore op upon a load. Deprecated; please use main_op instead. - clear_devices: Set to true if the device info on the default graph should - be cleared. - main_op: Op or group of ops to execute when the graph is loaded. Note - that when the main_op is specified it is run after the restore op at - load-time. - strip_default_attrs: Boolean. If `True`, default-valued attributes will be - removed from the NodeDefs. For a detailed guide, see - [Stripping Default-Valued Attributes](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/python/saved_model/README.md#stripping-default-valued-attributes). - saver: An instance of tf.train.Saver that will be used to export the - metagraph. If None, a sharded Saver that restores all variables will - be used. - - Raises: - AssertionError: If the variables for the SavedModel have not been saved - yet, or if the graph already contains one or more legacy init ops. - """ - # pylint: enable=line-too-long if not self._has_saved_variables: raise AssertionError( "Graph state including variables and assets has not been saved yet. " @@ -326,6 +517,7 @@ class SavedModelBuilder(object): # Validate the signature def map to ensure all included TensorInfos are # properly populated. + signature_def_map = signature_def_map or {} self._validate_signature_def_map(signature_def_map) # legacy_init_op is deprecated, and going away in TF 2.0. @@ -363,38 +555,6 @@ class SavedModelBuilder(object): main_op=None, strip_default_attrs=False, saver=None): - # pylint: disable=line-too-long - """Adds the current meta graph to the SavedModel and saves variables. - - Creates a Saver to save the variables from the provided session. Exports the - corresponding meta graph def. This function assumes that the variables to be - saved have been initialized. For a given `SavedModelBuilder`, this API must - be called exactly once and for the first meta graph to save. For subsequent - meta graph defs to be added, the `add_meta_graph()` API must be used. - - Args: - sess: The TensorFlow session from which to save the meta graph and - variables. - tags: The set of tags with which to save the meta graph. - signature_def_map: The map of signature def map to add to the meta graph - def. - assets_collection: Assets collection to be saved with SavedModel. - legacy_init_op: Legacy support for op or group of ops to execute after the - restore op upon a load. Deprecated; please use main_op instead. - clear_devices: Set to true if the device info on the default graph should - be cleared. - main_op: Op or group of ops to execute when the graph is loaded. Note - that when the main_op is specified it is run after the restore op at - load-time. - strip_default_attrs: Boolean. If `True`, default-valued attributes will be - removed from the NodeDefs. For a detailed guide, see - [Stripping Default-Valued Attributes](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/python/saved_model/README.md#stripping-default-valued-attributes). - saver: An instance of tf.train.Saver that will be used to export the - metagraph and save variables. If None, a sharded Saver that restores - all variables will be used. - - """ - # pylint: enable=line-too-long if self._has_saved_variables: raise AssertionError("Graph state including variables and assets has " "already been saved. Please invoke " @@ -402,6 +562,7 @@ class SavedModelBuilder(object): # Validate the signature def map to ensure all included TensorInfos are # properly populated. + signature_def_map = signature_def_map or {} self._validate_signature_def_map(signature_def_map) # legacy_init_op is deprecated, and going away in TF 2.0. @@ -441,41 +602,19 @@ class SavedModelBuilder(object): # subsequent attempts to save variables will fail. self._has_saved_variables = True - def save(self, as_text=False): - """Writes a `SavedModel` protocol buffer to disk. - - The function writes the SavedModel protocol buffer to the export directory - in serialized format. - - Args: - as_text: Writes the SavedModel protocol buffer in text format to disk. - - Returns: - The path to which the SavedModel protocol buffer was written. - """ - if not file_io.file_exists(self._export_dir): - file_io.recursive_create_dir(self._export_dir) - - if as_text: - path = os.path.join( - compat.as_bytes(self._export_dir), - compat.as_bytes(constants.SAVED_MODEL_FILENAME_PBTXT)) - file_io.write_string_to_file(path, str(self._saved_model)) - else: - path = os.path.join( - compat.as_bytes(self._export_dir), - compat.as_bytes(constants.SAVED_MODEL_FILENAME_PB)) - file_io.write_string_to_file(path, self._saved_model.SerializeToString()) - tf_logging.info("SavedModel written to: %s", compat.as_text(path)) - - return path + add_meta_graph.__doc__ = _SavedModelBuilder.add_meta_graph.__doc__.replace( + "assets_list", "assets_collection") + add_meta_graph_and_variables.__doc__ = \ + _SavedModelBuilder.add_meta_graph_and_variables.__doc__.replace( + "assets_list", "assets_collection") -def _maybe_save_assets(assets_collection_to_add=None): +def _maybe_save_assets(write_fn, assets_to_add=None): """Saves assets to the meta graph. Args: - assets_collection_to_add: The collection where the asset paths are setup. + write_fn: A function callback that writes asset into meta graph. + assets_to_add: The list where the asset paths are setup. Returns: A dict of asset basenames for saving to the original full path to the asset. @@ -486,25 +625,25 @@ def _maybe_save_assets(assets_collection_to_add=None): # Map of target file names to original filenames asset_filename_map = {} - if assets_collection_to_add is None: + if assets_to_add is None: tf_logging.info("No assets to save.") return asset_filename_map - # Iterate over the supplied asset collection, build the `AssetFile` proto - # and add them to the collection with key `constants.ASSETS_KEY`, in the - # graph. - for asset_tensor in assets_collection_to_add: + # Iterate over the supplied assets, build the `AssetFile` proto and add them + # to the meta graph. + for asset_tensor in assets_to_add: asset_source_filepath = _asset_path_from_tensor(asset_tensor) if not asset_source_filepath: raise ValueError("Invalid asset filepath tensor %s" % asset_tensor) - asset_filename = _get_asset_filename_to_add( + asset_filename = get_asset_filename_to_add( asset_source_filepath, asset_filename_map) - # Build `AssetFile` proto and add it to the asset collection in the graph. + # Call the passed-in function that builds AssetFileDef proto and adds it + # to either the collection or asset_file_def field of the meta graph. # Note that this should be done even when the file is a duplicate of an # already-added file, as the tensor reference should still exist. - _add_asset_to_collection(asset_filename, asset_tensor) + write_fn(asset_filename, asset_tensor) # In the cases where we are adding a duplicate, this will result in the # last of the filepaths being the one used for copying the file to the @@ -516,7 +655,7 @@ def _maybe_save_assets(assets_collection_to_add=None): return asset_filename_map -def _get_asset_filename_to_add(asset_filepath, asset_filename_map): +def get_asset_filename_to_add(asset_filepath, asset_filename_map): """Get a unique basename to add to the SavedModel if this file is unseen. Assets come from users as full paths, and we save them out to the @@ -542,7 +681,7 @@ def _get_asset_filename_to_add(asset_filepath, asset_filename_map): other_asset_filepath = asset_filename_map[asset_filename] if other_asset_filepath == asset_filepath: - # This is the same file, stored twice in the collection list. No need + # This is the same file, stored twice in the list. No need # to make unique. return asset_filename @@ -589,6 +728,41 @@ def _asset_path_from_tensor(path_tensor): return str_values[0] +def _add_asset_to_metagraph(meta_graph_def, asset_filename, asset_tensor): + """Builds an asset proto and adds it to the meta graph def. + + Args: + meta_graph_def: The meta graph def to which the asset will be added. + asset_filename: The filename of the asset to be added. + asset_tensor: The asset tensor used to populate the tensor info of the asset + proto. + """ + asset_proto = meta_graph_def.asset_file_def.add() + asset_proto.filename = asset_filename + asset_proto.tensor_info.name = asset_tensor.name + + +def copy_assets_to_destination_dir(asset_filename_map, destination_dir): + """Copy all assets from source path to destination path.""" + assets_destination_dir = saved_model_utils.get_or_create_assets_dir( + destination_dir) + + # Copy each asset from source path to destination path. + for asset_basename, asset_source_filepath in asset_filename_map.items(): + asset_destination_filepath = os.path.join( + compat.as_bytes(assets_destination_dir), + compat.as_bytes(asset_basename)) + + # Only copy the asset file to the destination if it does not already + # exist. This is to ensure that an asset with the same name defined as + # part of multiple graphs is only copied the first time. + if not file_io.file_exists(asset_destination_filepath): + file_io.copy(asset_source_filepath, asset_destination_filepath) + + tf_logging.info("Assets written to: %s", + compat.as_text(assets_destination_dir)) + + def _add_asset_to_collection(asset_filename, asset_tensor): """Builds an asset proto and adds it to the asset collection of the graph. @@ -604,3 +778,8 @@ def _add_asset_to_collection(asset_filename, asset_tensor): asset_any_proto = Any() asset_any_proto.Pack(asset_proto) ops.add_to_collection(constants.ASSETS_KEY, asset_any_proto) + + +def _add_op_to_signature_def_map(signature_def_map, op, key): + if op is not None: + signature_def_map[key] = signature_def_utils.op_signature_def(op, key) diff --git a/tensorflow/python/saved_model/constants.py b/tensorflow/python/saved_model/constants.py index 0addbdc9686..90511a409ed 100644 --- a/tensorflow/python/saved_model/constants.py +++ b/tensorflow/python/saved_model/constants.py @@ -29,6 +29,9 @@ tf_export( "saved_model.ASSETS_DIRECTORY", "saved_model.constants.ASSETS_DIRECTORY" ]).export_constant(__name__, "ASSETS_DIRECTORY") +# Subdirectory name containing unmanaged files from higher-level APIs. +EXTRA_ASSETS_DIRECTORY = "assets.extra" + # CollectionDef key containing SavedModel assets. ASSETS_KEY = "saved_model_assets" tf_export( @@ -40,7 +43,6 @@ tf_export( # CollectionDef key for the legacy init op. LEGACY_INIT_OP_KEY = "legacy_init_op" tf_export( - "saved_model.LEGACY_INIT_OP_KEY", v1=[ "saved_model.LEGACY_INIT_OP_KEY", "saved_model.constants.LEGACY_INIT_OP_KEY" @@ -49,13 +51,12 @@ tf_export( # CollectionDef key for the SavedModel main op. MAIN_OP_KEY = "saved_model_main_op" tf_export( - "saved_model.MAIN_OP_KEY", v1=["saved_model.MAIN_OP_KEY", "saved_model.constants.MAIN_OP_KEY"]).export_constant( __name__, "MAIN_OP_KEY") # CollectionDef key for the SavedModel train op. -# Not exported while export_all_saved_models is in contrib. +# Not exported while export_all_saved_models is experimental. TRAIN_OP_KEY = "saved_model_train_op" # Schema version for SavedModel. @@ -106,3 +107,8 @@ tf_export( "saved_model.VARIABLES_FILENAME", "saved_model.constants.VARIABLES_FILENAME" ]).export_constant(__name__, "VARIABLES_FILENAME") + +# The initialization and train ops for a MetaGraph are stored in the +# signature def map. The ops are added to the map with the following keys. +INIT_OP_SIGNATURE_KEY = "__saved_model_init_op" +TRAIN_OP_SIGNATURE_KEY = "__saved_model_train_op" diff --git a/tensorflow/python/saved_model/load.py b/tensorflow/python/saved_model/load.py new file mode 100644 index 00000000000..397086e1069 --- /dev/null +++ b/tensorflow/python/saved_model/load.py @@ -0,0 +1,61 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Import a checkpointable object from a SavedModel.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import os + +from tensorflow.python.lib.io import file_io +from tensorflow.python.saved_model import constants +from tensorflow.python.saved_model import saved_object_graph_pb2 +from tensorflow.python.training.checkpointable import tracking +from tensorflow.python.util import compat + + +def _recreate_object_graph(object_graph_proto): + """Recreates Python objects from an ObjectGraph proto.""" + objects = [] + for _ in object_graph_proto.nodes: + # TODO(allenl): re-create variables and other types + objects.append(tracking.Checkpointable()) + for obj, object_proto in zip(objects, object_graph_proto.nodes): + for reference in object_proto.children: + setattr(obj, reference.local_name, objects[reference.node_id]) + return objects[0] + + +def load(export_dir): + """Load a SavedModel from `export_dir`.""" + object_graph_filename = os.path.join( + compat.as_bytes(export_dir), + compat.as_bytes(constants.EXTRA_ASSETS_DIRECTORY), + compat.as_bytes("object_graph.pb")) + if file_io.file_exists(object_graph_filename): + # If there is an object graph associated with the SavedModel, we'll create a + # root object from that. + object_graph_string = file_io.FileIO(object_graph_filename, "rb").read() + object_graph_proto = ( + saved_object_graph_pb2.SavedObjectGraph()) + object_graph_proto.ParseFromString(object_graph_string) + root = _recreate_object_graph(object_graph_proto) + else: + raise NotImplementedError( + "Currently only SavedModels exported with `tf.saved_model.save` may be " + "imported. Other SavedModels may eventually be supported via load().") + # TODO(allenl): load functions from the SavedModel into the eager context + return root diff --git a/tensorflow/python/saved_model/load_test.py b/tensorflow/python/saved_model/load_test.py new file mode 100644 index 00000000000..bfa201ad630 --- /dev/null +++ b/tensorflow/python/saved_model/load_test.py @@ -0,0 +1,51 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Tests for checkpointable object SavedModel loading.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import os + +from tensorflow.python.eager import def_function +from tensorflow.python.eager import test +from tensorflow.python.framework import dtypes +from tensorflow.python.framework import tensor_spec +from tensorflow.python.saved_model import load +from tensorflow.python.saved_model import save +from tensorflow.python.training.checkpointable import tracking + + +class LoadTest(test.TestCase): + + def test_structure_import(self): + root = tracking.Checkpointable() + root.f = def_function.function( + lambda x: 2. * x, + input_signature=[tensor_spec.TensorSpec(None, dtypes.float32)]) + root.dep_one = tracking.Checkpointable() + root.dep_two = tracking.Checkpointable() + root.dep_two.dep = tracking.Checkpointable() + root.dep_three = root.dep_two.dep + save_dir = os.path.join(self.get_temp_dir(), "saved_model") + save.save(root, save_dir) + imported = load.load(save_dir) + self.assertIs(imported.dep_three, imported.dep_two.dep) + self.assertIsNot(imported.dep_one, imported.dep_two) + + +if __name__ == "__main__": + test.main() diff --git a/tensorflow/python/saved_model/loader_impl.py b/tensorflow/python/saved_model/loader_impl.py index 8c8eaf038a1..6bf39a2c676 100644 --- a/tensorflow/python/saved_model/loader_impl.py +++ b/tensorflow/python/saved_model/loader_impl.py @@ -31,6 +31,7 @@ from tensorflow.python.lib.io import file_io from tensorflow.python.ops import variables from tensorflow.python.platform import tf_logging from tensorflow.python.saved_model import constants +from tensorflow.python.saved_model import signature_def_utils from tensorflow.python.saved_model import utils_impl as saved_model_utils from tensorflow.python.training import saver as tf_saver from tensorflow.python.util import compat @@ -99,22 +100,29 @@ def _get_asset_tensors(export_dir, meta_graph_def_to_load, import_scope=None): collection_def = meta_graph_def_to_load.collection_def asset_tensor_dict = {} - if constants.ASSETS_KEY in collection_def: - # Location of the assets for SavedModel. - assets_directory = os.path.join( - compat.as_bytes(export_dir), - compat.as_bytes(constants.ASSETS_DIRECTORY)) + asset_protos = [] + + if meta_graph_def_to_load.asset_file_def: + asset_protos = meta_graph_def_to_load.asset_file_def + elif constants.ASSETS_KEY in collection_def: assets_any_proto = collection_def[constants.ASSETS_KEY].any_list.value - # Process each asset and add it to the asset tensor dictionary. for asset_any_proto in assets_any_proto: asset_proto = meta_graph_pb2.AssetFileDef() asset_any_proto.Unpack(asset_proto) - tensor_name = asset_proto.tensor_info.name - if import_scope: - tensor_name = "%s/%s" % (import_scope, tensor_name) - asset_tensor_dict[tensor_name] = os.path.join( - compat.as_bytes(assets_directory), - compat.as_bytes(asset_proto.filename)) + asset_protos.append(asset_proto) + + # Location of the assets for SavedModel. + assets_directory = os.path.join( + compat.as_bytes(export_dir), compat.as_bytes(constants.ASSETS_DIRECTORY)) + # Process each asset and add it to the asset tensor dictionary. + for asset_proto in asset_protos: + tensor_name = asset_proto.tensor_info.name + if import_scope: + tensor_name = "%s/%s" % (import_scope, tensor_name) + asset_tensor_dict[tensor_name] = os.path.join( + compat.as_bytes(assets_directory), + compat.as_bytes(asset_proto.filename)) + return asset_tensor_dict @@ -134,23 +142,53 @@ def _get_main_op_tensor( RuntimeError: If the collection def corresponding to the main op key has other than exactly one tensor. """ + # TODO(kathywu): Rename this method to _get_op_from_collection when + # dependency from SavedModelEstimator is removed. collection_def = meta_graph_def_to_load.collection_def - main_op_tensor = None + init_op = None if init_op_key in collection_def: - main_ops = collection_def[init_op_key].node_list.value - if len(main_ops) != 1: - raise RuntimeError("Expected exactly one SavedModel main op. " - "Found: {}".format(main_ops)) - main_op_tensor = ops.get_collection(init_op_key)[0] - return main_op_tensor + init_op_list = collection_def[init_op_key].node_list.value + if len(init_op_list) != 1: + raise RuntimeError("Expected exactly one SavedModel init op. " + "Found: {}".format(init_op_list)) + init_op = ops.get_collection(init_op_key)[0] + return init_op -@tf_export( +def _get_op_from_collection(meta_graph_def, op_key): + return _get_main_op_tensor(meta_graph_def, op_key) + + +def _get_op_from_signature_def(meta_graph_def, op_signature_key, import_scope): + """Retrieve op stored in the imported meta graph's signature def.""" + if op_signature_key in meta_graph_def.signature_def: + return signature_def_utils.load_op_from_signature_def( + meta_graph_def.signature_def[op_signature_key], op_signature_key, + import_scope) + else: + return None + + +def get_init_op(meta_graph_def, import_scope=None): + return (_get_op_from_signature_def( + meta_graph_def, constants.INIT_OP_SIGNATURE_KEY, import_scope) or + _get_op_from_collection(meta_graph_def, constants.MAIN_OP_KEY) or + _get_op_from_collection(meta_graph_def, constants.LEGACY_INIT_OP_KEY)) + + +def get_train_op(meta_graph_def, import_scope=None): + train_op = _get_op_from_signature_def( + meta_graph_def, constants.TRAIN_OP_SIGNATURE_KEY, import_scope) + if train_op is None: + train_op = _get_op_from_collection(meta_graph_def, constants.TRAIN_OP_KEY) + return train_op + + +@tf_export(v1=[ + "saved_model.contains_saved_model", "saved_model.maybe_saved_model_directory", - v1=[ - "saved_model.maybe_saved_model_directory", - "saved_model.loader.maybe_saved_model_directory" - ]) + "saved_model.loader.maybe_saved_model_directory" +]) @deprecation.deprecated_endpoints( "saved_model.loader.maybe_saved_model_directory") def maybe_saved_model_directory(export_dir): @@ -173,6 +211,25 @@ def maybe_saved_model_directory(export_dir): return file_io.file_exists(txt_path) or file_io.file_exists(pb_path) +@tf_export("saved_model.contains_saved_model", v1=[]) +def contains_saved_model(export_dir): + """Checks whether the provided export directory could contain a SavedModel. + + Note that the method does not load any data by itself. If the method returns + `false`, the export directory definitely does not contain a SavedModel. If the + method returns `true`, the export directory may contain a SavedModel but + provides no guarantee that it can be loaded. + + Args: + export_dir: Absolute string path to possible export location. For example, + '/my/foo/model'. + + Returns: + True if the export directory contains SavedModel files, False otherwise. + """ + return maybe_saved_model_directory(export_dir) + + @tf_export(v1=["saved_model.load", "saved_model.loader.load"]) @deprecation.deprecated( None, @@ -334,11 +391,9 @@ class SavedModelLoader(object): asset_tensors_dictionary = _get_asset_tensors( self._export_dir, meta_graph_def, import_scope=import_scope) - main_op_tensor = ( - _get_main_op_tensor(meta_graph_def, constants.MAIN_OP_KEY) or - _get_main_op_tensor(meta_graph_def, constants.LEGACY_INIT_OP_KEY)) - if main_op_tensor is not None: - sess.run(fetches=[main_op_tensor], feed_dict=asset_tensors_dictionary) + init_op = get_init_op(meta_graph_def, import_scope) + if init_op is not None: + sess.run(fetches=[init_op], feed_dict=asset_tensors_dictionary) def load(self, sess, tags, import_scope=None, **saver_kwargs): """Load the MetaGraphDef graph and restore variable values into the session. diff --git a/tensorflow/python/saved_model/loader_test.py b/tensorflow/python/saved_model/loader_test.py index 924b2e7c065..3b7f0b250e7 100644 --- a/tensorflow/python/saved_model/loader_test.py +++ b/tensorflow/python/saved_model/loader_test.py @@ -19,11 +19,14 @@ from __future__ import division from __future__ import print_function import os +import shutil + +from absl.testing import parameterized from tensorflow.python.client import session from tensorflow.python.framework import errors from tensorflow.python.framework import ops -from tensorflow.python.lib.io import file_io +from tensorflow.python.framework import test_util from tensorflow.python.ops import control_flow_ops from tensorflow.python.ops import state_ops from tensorflow.python.ops import variables @@ -42,55 +45,74 @@ SIMPLE_ADD_SAVED_MODEL = _get_export_dir("simple_add_saved_model") SAVED_MODEL_WITH_MAIN_OP = _get_export_dir("saved_model_with_main_op") -class SavedModelLoaderTest(test.TestCase): +def build_graph_helper(): + g = ops.Graph() + with g.as_default(): + x = variables.VariableV1(5, name="x") + y = variables.VariableV1(11, name="y") + z = x + y - def setUp(self): - """Write test SavedModels to a temp directory.""" - with session.Session(graph=ops.Graph()) as sess: - x = variables.VariableV1(5, name="x") - y = variables.VariableV1(11, name="y") - z = x + y - sess.run(variables.global_variables_initializer()) + foo_sig_def = signature_def_utils.build_signature_def({ + "foo_input": utils.build_tensor_info(x) + }, {"foo_output": utils.build_tensor_info(z)}) + bar_sig_def = signature_def_utils.build_signature_def({ + "bar_x": utils.build_tensor_info(x), + "bar_y": utils.build_tensor_info(y) + }, {"bar_z": utils.build_tensor_info(z)}) + return g, {"foo": foo_sig_def, "bar": bar_sig_def}, y - foo_sig_def = signature_def_utils.build_signature_def( - {"foo_input": utils.build_tensor_info(x)}, - {"foo_output": utils.build_tensor_info(z)}) - bar_sig_def = signature_def_utils.build_signature_def( - {"bar_x": utils.build_tensor_info(x), - "bar_y": utils.build_tensor_info(y)}, - {"bar_z": utils.build_tensor_info(z)}) - builder = saved_model_builder.SavedModelBuilder(SIMPLE_ADD_SAVED_MODEL) - builder.add_meta_graph_and_variables( - sess, ["foo_graph"], {"foo": foo_sig_def, "bar": bar_sig_def}) +@parameterized.parameters((saved_model_builder.SavedModelBuilder,), + (saved_model_builder._SavedModelBuilder,)) +class SavedModelLoaderTest(test.TestCase, parameterized.TestCase): + + def export_simple_graph(self, builder_cls): + g, sig_def_map, _ = build_graph_helper() + with session.Session(graph=g) as sess: + self.evaluate(variables.global_variables_initializer()) + builder = builder_cls(SIMPLE_ADD_SAVED_MODEL) + builder.add_meta_graph_and_variables(sess, ["foo_graph"], sig_def_map) builder.save() - # Write SavedModel with a main_op + def export_graph_with_main_op(self, builder_cls): + g, sig_def_map, y = build_graph_helper() + with session.Session(graph=g) as sess: + self.evaluate(variables.global_variables_initializer()) assign_op = control_flow_ops.group(state_ops.assign(y, 7)) - builder = saved_model_builder.SavedModelBuilder(SAVED_MODEL_WITH_MAIN_OP) - builder.add_meta_graph_and_variables( - sess, ["foo_graph"], {"foo": foo_sig_def, "bar": bar_sig_def}, - main_op=assign_op) + builder = builder_cls(SAVED_MODEL_WITH_MAIN_OP) + + if builder_cls == saved_model_builder._SavedModelBuilder: + builder.add_meta_graph_and_variables( + sess, ["foo_graph"], sig_def_map, init_op=assign_op) + else: + builder.add_meta_graph_and_variables( + sess, ["foo_graph"], sig_def_map, main_op=assign_op) builder.save() def tearDown(self): - file_io.delete_recursively(test.get_temp_dir()) + super(SavedModelLoaderTest, self).tearDown() + shutil.rmtree(test.get_temp_dir(), ignore_errors=True) - def test_load_function(self): + @test_util.run_deprecated_v1 + def test_load_function(self, builder_cls): + self.export_simple_graph(builder_cls) loader = loader_impl.SavedModelLoader(SIMPLE_ADD_SAVED_MODEL) with self.session(graph=ops.Graph()) as sess: loader.load(sess, ["foo_graph"]) self.assertEqual(5, sess.graph.get_tensor_by_name("x:0").eval()) self.assertEqual(11, sess.graph.get_tensor_by_name("y:0").eval()) + self.export_graph_with_main_op(builder_cls) loader2 = loader_impl.SavedModelLoader(SAVED_MODEL_WITH_MAIN_OP) with self.session(graph=ops.Graph()) as sess: loader2.load(sess, ["foo_graph"]) self.assertEqual(5, sess.graph.get_tensor_by_name("x:0").eval()) self.assertEqual(7, sess.graph.get_tensor_by_name("y:0").eval()) - def test_load_graph(self): + @test_util.run_deprecated_v1 + def test_load_graph(self, builder_cls): + self.export_simple_graph(builder_cls) loader = loader_impl.SavedModelLoader(SIMPLE_ADD_SAVED_MODEL) graph = ops.Graph() loader.load_graph(graph, ["foo_graph"]) @@ -101,14 +123,16 @@ class SavedModelLoaderTest(test.TestCase): with self.assertRaises(KeyError): graph.get_tensor_by_name("z:0") - with self.session(graph=graph) as sess: + with self.session(graph=graph): # Check that x and y are not initialized with self.assertRaises(errors.FailedPreconditionError): - sess.run(x) + self.evaluate(x) with self.assertRaises(errors.FailedPreconditionError): - sess.run(y) + self.evaluate(y) - def test_load_with_import_scope(self): + @test_util.run_deprecated_v1 + def test_load_with_import_scope(self, builder_cls): + self.export_graph_with_main_op(builder_cls) loader = loader_impl.SavedModelLoader(SAVED_MODEL_WITH_MAIN_OP) with self.session(graph=ops.Graph()) as sess: saver, _ = loader.load_graph( @@ -119,7 +143,13 @@ class SavedModelLoaderTest(test.TestCase): loader.restore_variables(sess, tf_saver.Saver()) loader.restore_variables(sess, saver) - loader.run_init_ops(sess, ["foo_graph"]) + + if builder_cls == saved_model_builder._SavedModelBuilder: + with self.assertRaises(errors.NotFoundError): + loader.run_init_ops(sess, ["foo_graph"]) + loader.run_init_ops(sess, ["foo_graph"], import_scope="baz") + else: + loader.run_init_ops(sess, ["foo_graph"]) self.assertEqual(5, sess.graph.get_tensor_by_name("baz/x:0").eval()) self.assertEqual(7, sess.graph.get_tensor_by_name("baz/y:0").eval()) @@ -131,23 +161,27 @@ class SavedModelLoaderTest(test.TestCase): self.assertEqual(5, sess.graph.get_tensor_by_name("baa/x:0").eval()) self.assertEqual(7, sess.graph.get_tensor_by_name("baa/y:0").eval()) - def test_restore_variables(self): + @test_util.run_deprecated_v1 + def test_restore_variables(self, builder_cls): + self.export_graph_with_main_op(builder_cls) loader = loader_impl.SavedModelLoader(SAVED_MODEL_WITH_MAIN_OP) with self.session(graph=ops.Graph()) as sess: x = variables.VariableV1(0, name="x") y = variables.VariableV1(0, name="y") z = x * y - sess.run(variables.global_variables_initializer()) + self.evaluate(variables.global_variables_initializer()) # There are variables to restore, so a saver must be created. with self.assertRaises(ValueError): loader.restore_variables(sess, None) loader.restore_variables(sess, tf_saver.Saver()) - self.assertEqual(55, z.eval()) + self.assertEqual(55, self.evaluate(z)) - def test_run_init_op(self): + @test_util.run_deprecated_v1 + def test_run_init_op(self, builder_cls): + self.export_graph_with_main_op(builder_cls) loader = loader_impl.SavedModelLoader(SAVED_MODEL_WITH_MAIN_OP) graph = ops.Graph() saver, _ = loader.load_graph(graph, ["foo_graph"]) @@ -160,14 +194,16 @@ class SavedModelLoaderTest(test.TestCase): self.assertEqual(5, sess.graph.get_tensor_by_name("x:0").eval()) self.assertEqual(7, sess.graph.get_tensor_by_name("y:0").eval()) - def test_parse_saved_model(self): + def test_parse_saved_model(self, builder_cls): + self.export_simple_graph(builder_cls) loader = loader_impl.SavedModelLoader(SIMPLE_ADD_SAVED_MODEL) meta_graph = loader.get_meta_graph_def_from_tags(["foo_graph"]) self.assertIsNotNone(meta_graph) self.assertIn("foo", meta_graph.signature_def) self.assertIn("bar", meta_graph.signature_def) - def test_load_invalid_meta_graph(self): + def test_load_invalid_meta_graph(self, builder_cls): + self.export_simple_graph(builder_cls) loader = loader_impl.SavedModelLoader(SIMPLE_ADD_SAVED_MODEL) with self.assertRaises(RuntimeError): loader.get_meta_graph_def_from_tags([]) @@ -176,13 +212,17 @@ class SavedModelLoaderTest(test.TestCase): with self.assertRaises(RuntimeError): loader.get_meta_graph_def_from_tags(["not_a_graph"]) - def test_load_saved_model_with_no_variables(self): + @test_util.run_deprecated_v1 + def test_load_saved_model_with_no_variables(self, builder_cls): """Test that SavedModel runs saver when there appear to be no variables. When no variables are detected, this may mean that the variables were saved to different collections, or the collections weren't saved to the SavedModel. If the SavedModel MetaGraphDef contains a saver, it should still run in either of these cases. + + Args: + builder_cls: SavedModelBuilder or _SavedModelBuilder class """ path = _get_export_dir("no_variable_saved_model") with session.Session(graph=ops.Graph()) as sess: @@ -192,7 +232,7 @@ class SavedModelLoaderTest(test.TestCase): 11, name="y", collections=["not_global_variable"]) self.assertFalse(variables._all_saveable_objects()) z = x + y - sess.run(variables.variables_initializer([x, y])) + self.evaluate(variables.variables_initializer([x, y])) foo_sig_def = signature_def_utils.build_signature_def( {"foo_input": utils.build_tensor_info(x)}, @@ -215,8 +255,9 @@ class SavedModelLoaderTest(test.TestCase): self.assertEqual(5, sess.graph.get_tensor_by_name("x:0").eval()) self.assertEqual(11, sess.graph.get_tensor_by_name("y:0").eval()) - def test_load_saved_model_graph_with_return_elements(self): + def test_load_saved_model_graph_with_return_elements(self, builder_cls): """Ensure that the correct elements are returned.""" + self.export_simple_graph(builder_cls) loader = loader_impl.SavedModelLoader(SIMPLE_ADD_SAVED_MODEL) graph = ops.Graph() _, ret = loader.load_graph(graph, ["foo_graph"], @@ -228,5 +269,6 @@ class SavedModelLoaderTest(test.TestCase): with self.assertRaisesRegexp(ValueError, "not found in graph"): loader.load_graph(graph, ["foo_graph"], return_elements=["z:0"]) + if __name__ == "__main__": test.main() diff --git a/tensorflow/python/saved_model/save.py b/tensorflow/python/saved_model/save.py index 63575f631eb..a66e19b1995 100644 --- a/tensorflow/python/saved_model/save.py +++ b/tensorflow/python/saved_model/save.py @@ -19,39 +19,82 @@ from __future__ import division from __future__ import print_function import collections +import functools import os +from tensorflow.core.protobuf import meta_graph_pb2 from tensorflow.core.protobuf import saved_model_pb2 from tensorflow.python.eager import context from tensorflow.python.eager import def_function -from tensorflow.python.eager import function +from tensorflow.python.eager import function as defun +from tensorflow.python.framework import constant_op +from tensorflow.python.framework import dtypes from tensorflow.python.framework import meta_graph from tensorflow.python.framework import ops +from tensorflow.python.framework import tensor_spec from tensorflow.python.lib.io import file_io from tensorflow.python.ops import array_ops +from tensorflow.python.ops import control_flow_ops from tensorflow.python.ops import resource_variable_ops +from tensorflow.python.saved_model import builder_impl from tensorflow.python.saved_model import constants +from tensorflow.python.saved_model import saved_object_graph_pb2 from tensorflow.python.saved_model import signature_constants from tensorflow.python.saved_model import signature_def_utils +from tensorflow.python.saved_model import tag_constants from tensorflow.python.saved_model import utils_impl from tensorflow.python.training.checkpointable import base +from tensorflow.python.training.checkpointable import tracking from tensorflow.python.training.checkpointable import util from tensorflow.python.util import compat from tensorflow.python.util import nest from tensorflow.python.util.tf_export import tf_export +def _check_for_functional_keras_model(root): + """Makes an export signature for `root` if it's a functional Keras Model.""" + # If nothing is decorated yet but this is a functional Keras Model (duck + # typed), we'll try to make a signature ourselves. + try: + inputs = root.inputs + input_names = root.input_names + except AttributeError: + return None + input_signature = [] + for input_tensor, input_name in zip(inputs, input_names): + input_signature.append(tensor_spec.TensorSpec( + shape=input_tensor.shape, dtype=input_tensor.dtype, + name=input_name)) + + @def_function.function(input_signature=input_signature) + def _wrapped_model(*args): + outputs_list = nest.flatten(root(inputs=list(args))) + return {name: output for name, output + in zip(root.output_names, outputs_list)} + return _wrapped_model + + def _find_function_to_export(root): """Iterate over `root`'s attributes, finding traced functions.""" - functions = [] - function_attribute_names = [] + exported_function = None + previous_attribute_name = None for attribute_name in dir(root): attribute_value = getattr(root, attribute_name, None) if isinstance(attribute_value, def_function.PolymorphicFunction): - functions.append(attribute_value) - function_attribute_names.append(attribute_name) - # TODO(allenl): Automatically infer signatures for Keras functional models? - if not functions: + if exported_function is not None: + raise ValueError( + ("Exporting an object with no " + "tf.saved_model.save(..., signatures=...) " + "argument specified, and with more than one " + "@tf.function-decorated method attached to it: {}. The signature " + "keys for these functions are ambiguous. Specify signature " + "functions explicitly.").format( + [previous_attribute_name, attribute_name])) + exported_function = attribute_value + previous_attribute_name = attribute_name + if exported_function is None: + exported_function = _check_for_functional_keras_model(root) + if exported_function is None: raise ValueError( ("Exporting an object with no tf.saved_model.save(..., signatures=...) " "argument specified, and with no @tf.function-decorated methods " @@ -60,14 +103,7 @@ def _find_function_to_export(root): "signatures does not make sense, as the only consumers will expect " "signatures. Either decorate a method or specify a signature function " "explicitly.")) - elif len(functions) > 1: - raise ValueError( - ("Exporting an object with no tf.saved_model.save(..., signatures=...) " - "argument specified, and with more than one @tf.function-decorated " - "method attached to it: {}. The signature keys for these functions " - "are ambiguous. Specify signature functions explicitly.").format( - function_attribute_names)) - return functions[0] + return exported_function def _canonicalize_signatures(signatures): @@ -77,7 +113,7 @@ def _canonicalize_signatures(signatures): signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY: signatures} concrete_signatures = {} for serving_key, signature_function in signatures.items(): - if isinstance(signature_function, (function.PolymorphicFunction, + if isinstance(signature_function, (defun.PolymorphicFunction, def_function.PolymorphicFunction)): input_signature = signature_function._input_signature # pylint: disable=protected-access if input_signature is None: @@ -88,7 +124,7 @@ def _canonicalize_signatures(signatures): "converted to concrete functions using " "`f.get_concrete_function(...)`.").format(signature_function)) signature_function = signature_function.get_concrete_function() - elif not isinstance(signature_function, function.Function): + elif not isinstance(signature_function, defun.Function): raise ValueError( ("Expected a TensorFlow function to generate a signature for, but " "got {}. Python functions may be decorated with " @@ -145,64 +181,65 @@ def _tensor_dict_to_tensorinfo(tensor_dict): for key, value in tensor_dict.items()} -def _map_captured_resources_to_created_resources( +def _map_captures_to_created_tensors( original_captures, resource_map): - """Maps eager resources captured by a function to Graph resources for export. + """Maps eager tensors captured by a function to Graph resources for export. Args: - original_captures: A dictionary mapping from resource tensors captured by - the function to interior placeholders for those resources (inside the - function body). + original_captures: A dictionary mapping from tensors captured by the + function to interior placeholders for those tensors (inside the function + body). resource_map: A dictionary mapping from resource tensors owned by the eager context to resource tensors in the exported graph. Returns: - A dictionary mapping from interior placeholders in the function body to - exterior stand-in resource tensors which belong to the exported graph. + A list of stand-in tensors which belong to the exported graph, corresponding + to the function's captures. Raises: AssertionError: If the function references a resource which is not part of `resource_map`. """ - export_captures = {} + export_captures = [] for exterior, interior in original_captures.items(): mapped_resource = resource_map.get(exterior, None) if mapped_resource is None: - raise AssertionError( - ("Tried to export a function which references untracked stateful " - "object {}. Stateful TensorFlow objects (e.g. tf.Variable) must " - "be tracked by the main object. Objects may be tracked by " - "assigning them to an attribute of another tracked object, or to " - "an attribute of the main object directly.") - .format(interior)) - export_captures[interior] = mapped_resource + if exterior.dtype == dtypes.resource: + raise AssertionError( + ("Tried to export a function which references untracked stateful " + "object {}. Stateful TensorFlow objects (e.g. tf.Variable) must " + "be tracked by the main object. Objects may be tracked by " + "assigning them to an attribute of another tracked object, or to " + "an attribute of the main object directly.") + .format(interior)) + else: + # This is a captured Tensor, but it's not a resource. We'll just add it + # to the graph as a constant. + mapped_resource = constant_op.constant(exterior.numpy()) + export_captures.append(mapped_resource) return export_captures -def _map_function_inputs_to_created_inputs( - function_inputs, export_captures, signature_key, function_name): - """Creates exterior placeholders in the exported graph for function inputs. +def _map_function_arguments_to_created_inputs( + function_arguments, signature_key, function_name): + """Creates exterior placeholders in the exported graph for function arguments. Functions have two types of inputs: tensors captured from the outside (eager) context, and arguments to the function which we expect to receive from the - user at each call. `_map_captured_resources_to_created_resources` replaces + user at each call. `_map_captures_to_created_tensors` replaces captured tensors with stand-ins (typically these are resource dtype tensors associated with variables). `_map_function_inputs_to_created_inputs` runs over - every input, either captured or argument. For captures, it uses the mapped - resource from `export_captures`. For arguments, it creates a new placeholder - which will belong to the exported graph rather than the function body. + every argument, creating a new placeholder for each which will belong to the + exported graph rather than the function body. Args: - function_inputs: A list of all placeholders in the function body. - export_captures: A dictionary mapping from interior placeholders in the - function body to exterior stand-in resource tensors which belong to the - exported graph (see `_map_captured_resources_to_created_resources`). + function_arguments: A list of argument placeholders in the function body. signature_key: The name of the signature being exported, for error messages. function_name: The name of the function, for error messages. Returns: A tuple of (mapped_inputs, exterior_placeholders) - mapped_inputs: A list with entries corresponding to `function_inputs` + mapped_inputs: A list with entries corresponding to `function_arguments` containing all of the inputs of the function gathered from the exported graph (both captured resources and arguments). exterior_argument_placeholders: A dictionary mapping from argument names @@ -220,12 +257,7 @@ def _map_function_inputs_to_created_inputs( # MetaGraph. exterior_argument_placeholders = {} mapped_inputs = [] - for placeholder in function_inputs: - mapped_resource_tensor = export_captures.get(placeholder, None) - if mapped_resource_tensor is not None: - # This is a captured resource. - mapped_inputs.append(mapped_resource_tensor) - continue + for placeholder in function_arguments: # `export_captures` contains an exhaustive set of captures, so if we don't # find the input there then we now know we have an argument. user_input_name = compat.as_str_any( @@ -258,6 +290,20 @@ def _map_function_inputs_to_created_inputs( return mapped_inputs, exterior_argument_placeholders +def _call_function_with_mapped_captures(function, args, resource_map): + """Calls `function` in the exported graph, using mapped resource captures.""" + export_captures = _map_captures_to_created_tensors( + function.graph.captures, resource_map) + mapped_inputs = args + export_captures + # Calls the function quite directly, since we have new captured resource + # tensors we need to feed in which weren't part of the original function + # definition. + # pylint: disable=protected-access + outputs = function._build_call_outputs( + function._inference_function.call(context.context(), mapped_inputs)) + return outputs + + def _generate_signatures(signature_functions, resource_map): """Validates and calls `signature_functions` in the default graph. @@ -287,35 +333,77 @@ def _generate_signatures(signature_functions, resource_map): SignatureDefs as part of that MetaGraph. """ signatures = {} - for signature_key, func in sorted(signature_functions.items()): - # Register the inference function for this signature in the exported - # graph. There is no direct use for the gradient of this function, so we - # don't generate/register a gradient function here (but may end up with one - # if another function relies on it). Users can still take symbolic gradients - # of the function on import, the gradient just won't be in the saved - # graph. When exporting a signature which already computes gradients, this - # stops us from taking needless second-order gradients. - func.add_to_graph(register_gradient_functions=False) - export_captures = _map_captured_resources_to_created_resources( - func.graph.captures, resource_map) + for signature_key, function in sorted(signature_functions.items()): + if function.graph.captures: + argument_inputs = function.graph.inputs[:-len(function.graph.captures)] + else: + argument_inputs = function.graph.inputs mapped_inputs, exterior_argument_placeholders = ( - _map_function_inputs_to_created_inputs( - func.inputs, export_captures, signature_key, func.name)) - # Calls the function quite directly, since we have new captured resource - # tensors we need to feed in which weren't part of the original function - # definition. - # pylint: disable=protected-access + _map_function_arguments_to_created_inputs( + argument_inputs, signature_key, function.name)) outputs = _normalize_outputs( - func._build_call_outputs( - func._inference_function.call(context.context(), mapped_inputs)), - func.name, signature_key) - # pylint: enable=protected-access + _call_function_with_mapped_captures( + function, mapped_inputs, resource_map), + function.name, signature_key) signatures[signature_key] = signature_def_utils.build_signature_def( _tensor_dict_to_tensorinfo(exterior_argument_placeholders), _tensor_dict_to_tensorinfo(outputs)) return signatures +def _trace_resource_initializers(accessible_objects): + """Create concrete functions from `TrackableResource` objects.""" + resource_initializers = [] + + def _wrap_initializer(obj): + obj.initialize() + return constant_op.constant(1.) # Dummy control output + + for obj in accessible_objects: + if isinstance(obj, tracking.TrackableResource): + resource_initializers.append(def_function.function( + functools.partial(_wrap_initializer, obj), + # All inputs are captures. + input_signature=[]).get_concrete_function()) + return resource_initializers + + +_AssetInfo = collections.namedtuple( + "_AssetInfo", [ + # List of AssetFileDef protocol buffers + "asset_defs", + # Map from asset variable resource Tensors to their init ops + "asset_initializers_by_resource", + # Map from base asset filenames to full paths + "asset_filename_map"]) + + +def _process_asset(trackable_asset, asset_info, resource_map): + """Add `trackable_asset` to `asset_info` and `resource_map`.""" + original_variable = trackable_asset.asset_path + with context.eager_mode(): + original_path = original_variable.numpy() + path = builder_impl.get_asset_filename_to_add( + asset_filepath=original_path, + asset_filename_map=asset_info.asset_filename_map) + asset_variable = asset_info.asset_filename_map.get(path, None) + if asset_variable is None: + asset_path_initializer = array_ops.placeholder( + shape=original_variable.shape, + dtype=dtypes.string, + name="asset_path_initializer") + asset_variable = resource_variable_ops.ResourceVariable( + asset_path_initializer) + asset_info.asset_filename_map[path] = original_path + asset_def = meta_graph_pb2.AssetFileDef() + asset_def.filename = path + asset_def.tensor_info.name = asset_path_initializer.name + asset_info.asset_defs.append(asset_def) + asset_info.asset_initializers_by_resource[original_variable.handle] = ( + asset_variable.initializer) + resource_map[original_variable.handle] = asset_variable.handle + + def _map_resources(accessible_objects): """Makes new resource handle ops corresponding to existing resource tensors. @@ -329,34 +417,83 @@ def _map_resources(accessible_objects): to create replacements for. Returns: - A tuple of (object_map, resource_map): + A tuple of (object_map, resource_map, asset_info): object_map: A dictionary mapping from object in `accessible_objects` to replacement objects created to hold the new resource tensors. resource_map: A dictionary mapping from resource tensors extracted from `accessible_objects` to newly created resource tensors. + asset_info: An _AssetInfo tuple describing external assets referenced from + accessible_objects. """ - # TODO(allenl, rohanj): Map generic resources rather than just variables. # TODO(allenl): Handle MirroredVariables and other types of variables which # may need special casing. object_map = {} resource_map = {} + asset_info = _AssetInfo( + asset_defs=[], + asset_initializers_by_resource={}, + asset_filename_map={}) for obj in accessible_objects: - if resource_variable_ops.is_resource_variable(obj): + if isinstance(obj, tracking.TrackableResource): + new_resource = obj.create_resource() + resource_map[obj.resource_handle] = new_resource + elif resource_variable_ops.is_resource_variable(obj): new_variable = resource_variable_ops.copy_to_graph_uninitialized(obj) object_map[obj] = new_variable resource_map[obj.handle] = new_variable.handle - return object_map, resource_map + if isinstance(obj, tracking.TrackableAsset): + _process_asset(obj, asset_info, resource_map) + return object_map, resource_map, asset_info -def _make_graph_def(root, signature_functions, object_saver): - """Generates and exports call ops for `signature_functions`.""" +def _fill_meta_graph_def(meta_graph_def, obj, signature_functions, + object_saver): + """Generates a MetaGraph which calls `signature_functions`. + + Args: + meta_graph_def: The MetaGraphDef proto to fill. + obj: The checkpointable object being exported. + signature_functions: A dictionary mapping signature keys to concrete + functions containing signatures to add to the MetaGraph. + object_saver: A CheckpointableSaver to add to the MetaGraph. + + Returns: + asset_filename_map, a dictionary mapping from asset base names to + user-specified full asset paths, which should be copied to the SavedModel's + assets/ directory. + """ signatures = {} # List objects from the eager context to make sure Optimizers give us the # right Graph-dependent variables. - accessible_objects = util.list_objects(root) + accessible_objects = util.list_objects(obj) + resource_initializer_functions = _trace_resource_initializers( + accessible_objects) exported_graph = ops.Graph() + resource_initializer_ops = [] with exported_graph.as_default(): - object_map, resource_map = _map_resources(accessible_objects) + object_map, resource_map, asset_info = _map_resources(accessible_objects) + for resource_initializer_function in resource_initializer_functions: + asset_dependencies = [] + for capture in resource_initializer_function.graph.external_captures: + asset_initializer = asset_info.asset_initializers_by_resource.get( + capture, None) + if asset_initializer is not None: + asset_dependencies.append(asset_initializer) + with ops.control_dependencies(asset_dependencies): + resource_initializer_ops.append( + _call_function_with_mapped_captures( + resource_initializer_function, [], resource_map)) + with ops.control_dependencies(resource_initializer_ops): + init_op = control_flow_ops.no_op() + # Add the same op to the main_op collection and to the init_op + # signature. The collection is for compatibility with older loader APIs; + # only one will be executed. + meta_graph_def.collection_def[constants.MAIN_OP_KEY].node_list.value.append( + init_op.name) + meta_graph_def.signature_def[constants.INIT_OP_SIGNATURE_KEY].CopyFrom( + signature_def_utils.op_signature_def( + init_op, constants.INIT_OP_SIGNATURE_KEY)) + # Saving an object-based checkpoint again gathers variables. We need to do the # gathering from the eager context so Optimizers save the right set of # variables, but want any operations associated with the save/restore to be in @@ -365,11 +502,35 @@ def _make_graph_def(root, signature_functions, object_saver): with exported_graph.as_default(): signatures = _generate_signatures(signature_functions, resource_map) saver_def = saver.to_proto() + meta_graph_def.saver_def.CopyFrom(saver_def) graph_def = exported_graph.as_graph_def(add_shapes=True) # Clean reference cycles so repeated export()s don't make work for the garbage # collector. ops.dismantle_graph(exported_graph) - return graph_def, signatures, saver_def + + meta_graph_def.graph_def.CopyFrom(graph_def) + meta_graph_def.meta_info_def.tags.append(tag_constants.SERVING) + meta_graph_def.asset_file_def.extend(asset_info.asset_defs) + for signature_key, signature in signatures.items(): + meta_graph_def.signature_def[signature_key].CopyFrom(signature) + meta_graph.strip_graph_default_valued_attrs(meta_graph_def) + return asset_info.asset_filename_map + + +def _write_object_graph(obj, export_dir): + """Save a SavedObjectGraph proto for `obj`.""" + # SavedObjectGraph is similar to the CheckpointableObjectGraph proto in the + # checkpoint. It will eventually go into the SavedModel. + object_proto = util.make_object_graph_without_attributes( + obj, proto=saved_object_graph_pb2.SavedObjectGraph()) + extra_asset_dir = os.path.join( + compat.as_bytes(export_dir), + compat.as_bytes(constants.EXTRA_ASSETS_DIRECTORY)) + file_io.recursive_create_dir(extra_asset_dir) + object_graph_filename = os.path.join( + extra_asset_dir, compat.as_bytes("object_graph.pb")) + file_io.write_string_to_file(object_graph_filename, + object_proto.SerializeToString()) @tf_export("saved_model.save", v1=["saved_model.experimental.save"]) @@ -450,6 +611,19 @@ def save(obj, export_dir, signatures=None): tf.TensorSpec(shape=[None, 3], dtype=tf.float32, name="inp"))) ``` + `tf.keras.Model` instances constructed from inputs and outputs already have a + signature and so do not require a `@tf.function` decorator or a `signatures` + argument. If neither are specified, the model's forward pass is exported. + + ```python + x = input_layer.Input((4,), name="x") + y = core.Dense(5, name="out")(x) + model = training.Model(x, y) + tf.saved_model.save(model, '/tmp/saved_model/') + # The exported SavedModel takes "x" with shape [None, 4] and returns "out" + # with shape [None, 5] + ``` + Variables must be tracked by assigning them to an attribute of a tracked object or to an attribute of `obj` directly. TensorFlow objects (e.g. layers from `tf.keras.layers`, optimizers from `tf.train`) track their variables @@ -515,26 +689,26 @@ def save(obj, export_dir, signatures=None): # Note that we run this before saving the checkpoint, since looping over # attributes may have the side effect of creating variables in some cases. signatures = _find_function_to_export(obj) - object_saver = util.CheckpointableSaver(obj) - utils_impl.get_or_create_variables_dir(export_dir) - object_saver.save(utils_impl.get_variables_path(export_dir)) signatures = _canonicalize_signatures(signatures) - graph_def, signatures, saver_def = _make_graph_def( - obj, signatures, object_saver) - saved_model = saved_model_pb2.SavedModel() - saved_model.saved_model_schema_version = ( - constants.SAVED_MODEL_SCHEMA_VERSION) - meta_graph_def = saved_model.meta_graphs.add() - meta_graph_def.saver_def.CopyFrom(saver_def) # TODO(allenl): Factor out some subset of SavedModelBuilder which is 2.x # compatible (no sessions) and share it with this export API rather than # making a SavedModel proto and writing it directly. - meta_graph_def.graph_def.MergeFrom(graph_def) - for signature_key, signature in signatures.items(): - meta_graph_def.signature_def[signature_key].MergeFrom(signature) - meta_graph.strip_graph_default_valued_attrs(meta_graph_def) + saved_model = saved_model_pb2.SavedModel() + meta_graph_def = saved_model.meta_graphs.add() + object_saver = util.CheckpointableSaver(obj) + asset_filename_map = _fill_meta_graph_def( + meta_graph_def, obj, signatures, object_saver) + saved_model.saved_model_schema_version = ( + constants.SAVED_MODEL_SCHEMA_VERSION) + # So far we've just been generating protocol buffers with no I/O. Now we write + # the checkpoint, copy assets into the assets directory, and write out the + # SavedModel proto itself. + utils_impl.get_or_create_variables_dir(export_dir) + object_saver.save(utils_impl.get_variables_path(export_dir)) + builder_impl.copy_assets_to_destination_dir(asset_filename_map, export_dir) path = os.path.join( compat.as_bytes(export_dir), compat.as_bytes(constants.SAVED_MODEL_FILENAME_PB)) file_io.write_string_to_file(path, saved_model.SerializeToString()) + _write_object_graph(obj, export_dir) diff --git a/tensorflow/python/saved_model/save_test.py b/tensorflow/python/saved_model/save_test.py index 42ff508b38a..9e5b9b97176 100644 --- a/tensorflow/python/saved_model/save_test.py +++ b/tensorflow/python/saved_model/save_test.py @@ -21,6 +21,9 @@ from __future__ import print_function import os import sys +import numpy + +from tensorflow.python.client import session as session_lib from tensorflow.python.eager import backprop from tensorflow.python.eager import def_function from tensorflow.python.eager import test @@ -29,13 +32,19 @@ from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops from tensorflow.python.framework import tensor_spec from tensorflow.python.framework import test_util +from tensorflow.python.keras.engine import input_layer from tensorflow.python.keras.engine import training from tensorflow.python.keras.layers import core +from tensorflow.python.keras.layers import merge +from tensorflow.python.lib.io import file_io +from tensorflow.python.ops import array_ops +from tensorflow.python.ops import lookup_ops from tensorflow.python.ops import math_ops from tensorflow.python.ops import variables from tensorflow.python.saved_model import loader from tensorflow.python.saved_model import save from tensorflow.python.saved_model import signature_constants +from tensorflow.python.saved_model import tag_constants from tensorflow.python.training import adam from tensorflow.python.training.checkpointable import tracking from tensorflow.python.training.checkpointable import util @@ -60,26 +69,27 @@ class _ModelWithOptimizer(training.Model): return {"loss": loss} -class SaveTest(test.TestCase): +def _import_and_infer( + save_dir, inputs, + signature_key=signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY): + """Import a SavedModel into a TF 1.x-style graph and run `signature_key`.""" + graph = ops.Graph() + with graph.as_default(), session_lib.Session() as session: + model = loader.load(session, [tag_constants.SERVING], save_dir) + signature = model.signature_def[signature_key] + assert set(inputs.keys()) == set(signature.inputs.keys()) + feed_dict = {} + for arg_name in inputs.keys(): + feed_dict[graph.get_tensor_by_name(signature.inputs[arg_name].name)] = ( + inputs[arg_name]) + output_dict = {} + for output_name, output_tensor_info in signature.outputs.items(): + output_dict[output_name] = graph.get_tensor_by_name( + output_tensor_info.name) + return session.run(output_dict, feed_dict=feed_dict) - def _import_and_infer( - self, save_dir, inputs, - signature_key=signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY): - """Import a SavedModel into a TF 1.x-style graph and run `signature_key`.""" - graph = ops.Graph() - with graph.as_default(), self.session(graph) as session: - model = loader.load(session, [], save_dir) - signature = model.signature_def[signature_key] - self.assertEqual(set(inputs.keys()), set(signature.inputs.keys())) - feed_dict = {} - for arg_name in inputs.keys(): - feed_dict[graph.get_tensor_by_name(signature.inputs[arg_name].name)] = ( - inputs[arg_name]) - output_dict = {} - for output_name, output_tensor_info in signature.outputs.items(): - output_dict[output_name] = graph.get_tensor_by_name( - output_tensor_info.name) - return session.run(output_dict, feed_dict=feed_dict) + +class SaveTest(test.TestCase): def test_method_save_signature(self): root = tracking.Checkpointable() @@ -91,7 +101,7 @@ class SaveTest(test.TestCase): save.save(root, save_dir, root.f) self.assertEqual( {"output_0": 2.}, - self._import_and_infer(save_dir, {"x": 1.})) + _import_and_infer(save_dir, {"x": 1.})) def test_method_save_concrete(self): root = tracking.Checkpointable() @@ -106,7 +116,7 @@ class SaveTest(test.TestCase): tensor_spec.TensorSpec(None, dtypes.float32))}) self.assertEqual( {"out": 2.}, - self._import_and_infer( + _import_and_infer( save_dir, {"z": 1.}, signature_key="non_default_key")) def test_non_concrete_error(self): @@ -163,7 +173,7 @@ class SaveTest(test.TestCase): save_dir = os.path.join(self.get_temp_dir(), "saved_model") save.save(root, save_dir, to_save) self.assertAllEqual({"output_0": 12.}, - self._import_and_infer(save_dir, {"x": 2.})) + _import_and_infer(save_dir, {"x": 2.})) def test_optimizer(self): x = constant_op.constant([[3., 4.]]) @@ -176,7 +186,7 @@ class SaveTest(test.TestCase): self.assertNotEqual(first_loss, second_loss) self.assertAllClose( second_loss, - self._import_and_infer(save_dir, {"x": [[3., 4.]], "y": [2.]})) + _import_and_infer(save_dir, {"x": [[3., 4.]], "y": [2.]})) def test_trivial_save_exception(self): save_dir = os.path.join(self.get_temp_dir(), "saved_model") @@ -191,8 +201,8 @@ class SaveTest(test.TestCase): save_dir = os.path.join(self.get_temp_dir(), "saved_model") save.save(model, save_dir) self.assertIn("loss", - self._import_and_infer(save_dir, - {"x": [[3., 4.]], "y": [2.]})) + _import_and_infer(save_dir, + {"x": [[3., 4.]], "y": [2.]})) def test_single_function_default_signature(self): model = tracking.Checkpointable() @@ -201,7 +211,7 @@ class SaveTest(test.TestCase): save_dir = os.path.join(self.get_temp_dir(), "saved_model") save.save(model, save_dir) self.assertAllClose({"output_0": 3.}, - self._import_and_infer(save_dir, {})) + _import_and_infer(save_dir, {})) def test_ambiguous_signatures(self): model = _ModelWithOptimizer() @@ -213,6 +223,19 @@ class SaveTest(test.TestCase): with self.assertRaisesRegexp(ValueError, "call.*second_function"): save.save(model, save_dir) + def test_subclassed_no_signature(self): + + class Subclassed(training.Model): + + def call(self, inputs): + return inputs * 2. + + save_dir = os.path.join(self.get_temp_dir(), "saved_model") + model = Subclassed() + with self.assertRaisesRegexp( + ValueError, "no @tf.function-decorated methods"): + save.save(model, save_dir) + def test_docstring(self): class Adder(util.Checkpoint): @@ -227,7 +250,7 @@ class SaveTest(test.TestCase): save_dir = os.path.join(self.get_temp_dir(), "saved_model") save.save(to_save, save_dir) self.assertAllClose({"output_0": 7.}, - self._import_and_infer(save_dir, {"x": 3.})) + _import_and_infer(save_dir, {"x": 3.})) def test_default_attr_stripping(self): @@ -246,13 +269,90 @@ class SaveTest(test.TestCase): save.save(to_save, save_dir) graph = ops.Graph() with graph.as_default(), self.session(graph) as session: - loader.load(session, [], save_dir) + loader.load(session, [tag_constants.SERVING], save_dir) func, = graph._functions.values() complex_node, = [ node for node in func.definition.node_def if node.op == "Complex"] self.assertNotIn("T", complex_node.attr) self.assertNotIn("Tout", complex_node.attr) + def test_export_functional_keras_model(self): + x = input_layer.Input((4,), name="x") + y = core.Dense(4, name="out")(x) + model = training.Model(x, y) + save_dir = os.path.join(self.get_temp_dir(), "saved_model") + save.save(model, save_dir) + self.assertAllClose( + {"out": model(array_ops.ones([1, 4]))}, + _import_and_infer(save_dir, {"x": [[1., 1., 1., 1.]]})) + + @test_util.run_deprecated_v1 + def test_export_functional_keras_model_after_fit(self): + x = input_layer.Input((1,)) + y = core.Dense(1, name="y")(x) + model = training.Model(x, y) + model.compile(optimizer="sgd", loss="mse") + model.fit(x=numpy.array([[1.]]), + y=numpy.array([2.]), epochs=2) + save_dir = os.path.join(self.get_temp_dir(), "saved_model") + save.save(model, save_dir) + self.assertAllClose( + {"y": model(constant_op.constant([[1.], [2.]]))}, + _import_and_infer(save_dir, {"input_1": [[1.], [2.]]})) + + def test_export_multi_input_functional_keras_model(self): + x1 = input_layer.Input((2,), name="x1") + x2 = input_layer.Input((2,), name="x2") + y1 = core.Dense(4)(merge.Add()([x1, x2])) + y2 = core.Dense(4)(merge.Multiply()([x1, x2])) + model = training.Model([x1, x2], [y1, y2]) + save_dir = os.path.join(self.get_temp_dir(), "saved_model") + save.save(model, save_dir) + outputs = model([array_ops.ones([1, 2]), 2. * array_ops.ones([1, 2])]) + self.assertAllClose( + {"dense": outputs[0], "dense_1": outputs[1]}, + _import_and_infer( + save_dir, + {"x1": [[1., 1.]], + "x2": [[2., 2.]]})) + + +class AssetTests(test.TestCase): + + def setUp(self): + super(AssetTests, self).setUp() + self._vocab_path = os.path.join(self.get_temp_dir(), "vocab.txt") + with open(self._vocab_path, "w") as f: + f.write("alpha\nbeta\ngamma\n") + + def test_table(self): + initializer = lookup_ops.TextFileInitializer( + self._vocab_path, + key_dtype=dtypes.string, + key_index=lookup_ops.TextFileIndex.WHOLE_LINE, + value_dtype=dtypes.int64, + value_index=lookup_ops.TextFileIndex.LINE_NUMBER) + root = util.Checkpoint(table=lookup_ops.HashTable( + initializer, default_value=-1)) + root.table_user = def_function.function( + root.table.lookup, + input_signature=[tensor_spec.TensorSpec(None, dtypes.string)]) + self.assertEqual( + 2, + self.evaluate(root.table_user(constant_op.constant("gamma")))) + save_dir = os.path.join(self.get_temp_dir(), "saved_model") + save.save(root, save_dir) + file_io.delete_file(self._vocab_path) + self.assertAllClose( + {"output_0": [2, 0]}, + _import_and_infer(save_dir, {"keys": ["gamma", "alpha"]})) + second_dir = os.path.join(self.get_temp_dir(), "second_dir") + # Asset paths should track the location the SavedModel is loaded from. + file_io.rename(save_dir, second_dir) + self.assertAllClose( + {"output_0": [2, 1]}, + _import_and_infer(second_dir, {"keys": ["gamma", "beta"]})) + class MemoryTests(test.TestCase): diff --git a/tensorflow/python/saved_model/saved_model_test.py b/tensorflow/python/saved_model/saved_model_test.py index 5d6167ab38f..0f18fb1a016 100644 --- a/tensorflow/python/saved_model/saved_model_test.py +++ b/tensorflow/python/saved_model/saved_model_test.py @@ -54,15 +54,15 @@ def tearDownModule(): file_io.delete_recursively(test.get_temp_dir()) -class SavedModelTest(test.TestCase): +class SavedModelTestBase(test.TestCase): def _get_export_dir(self, label): return os.path.join(test.get_temp_dir(), label) def _init_and_validate_variable(self, sess, variable_name, variable_value): v = variables.VariableV1(variable_value, name=variable_name) - sess.run(variables.global_variables_initializer()) - self.assertEqual(variable_value, v.eval()) + self.evaluate(variables.global_variables_initializer()) + self.assertEqual(variable_value, self.evaluate(v)) def _build_asset_collection(self, asset_file_name, asset_file_contents, asset_file_tensor_name, asset_subdir=""): @@ -78,14 +78,16 @@ class SavedModelTest(test.TestCase): asset_collection = ops.get_collection(ops.GraphKeys.ASSET_FILEPATHS) return asset_collection - def _validate_asset_collection(self, export_dir, graph_collection_def, - expected_asset_file_name, - expected_asset_file_contents, - expected_asset_tensor_name, - asset_id=0): - assets_any = graph_collection_def[constants.ASSETS_KEY].any_list.value - asset = meta_graph_pb2.AssetFileDef() - assets_any[asset_id].Unpack(asset) + +class SavedModelTest(SavedModelTestBase): + + def _validate_assets(self, + export_dir, + asset_file_def, + expected_asset_file_name, + expected_asset_file_contents, + expected_asset_tensor_name, + asset_id=0): assets_path = os.path.join( compat.as_bytes(export_dir), compat.as_bytes(constants.ASSETS_DIRECTORY), @@ -93,8 +95,10 @@ class SavedModelTest(test.TestCase): actual_asset_contents = file_io.read_file_to_string(assets_path) self.assertEqual(expected_asset_file_contents, compat.as_text(actual_asset_contents)) - self.assertEqual(expected_asset_file_name, asset.filename) - self.assertEqual(expected_asset_tensor_name, asset.tensor_info.name) + self.assertEqual(expected_asset_file_name, + asset_file_def[asset_id].filename) + self.assertEqual(expected_asset_tensor_name, + asset_file_def[asset_id].tensor_info.name) def _validate_inputs_tensor_info_fail(self, builder, tensor_info): with self.session(graph=ops.Graph()) as sess: @@ -142,6 +146,18 @@ class SavedModelTest(test.TestCase): sess, ["foo"], signature_def_map={"foo_key": foo_signature}) + def _validate_sig_def_keys(self, builder, valid_tensor_info, invalid_key): + with self.session(graph=ops.Graph()) as sess: + self._init_and_validate_variable(sess, "v", 42) + + foo_signature = signature_def_utils.build_signature_def( + dict(), {"foo_key": valid_tensor_info}, "foo") + self.assertRaises( + KeyError, + builder.add_meta_graph_and_variables, + sess, ["foo"], + signature_def_map={invalid_key: foo_signature}) + def testMaybeSavedModelDir(self): base_path = test.test_src_dir_path("/python/saved_model") self.assertFalse(loader.maybe_saved_model_directory(base_path)) @@ -183,9 +199,10 @@ class SavedModelTest(test.TestCase): constants.SAVED_MODEL_FILENAME_PBTXT): loader.load(sess, ["foo"], export_dir) + @test_util.run_deprecated_v1 def testVerifySessionGraphUsage(self): export_dir = self._get_export_dir("test_verify_session_graph_usage") - builder = saved_model_builder.SavedModelBuilder(export_dir) + builder = saved_model_builder._SavedModelBuilder(export_dir) with self.session(graph=ops.Graph()) as sess: self._init_and_validate_variable(sess, "v", 42) @@ -203,9 +220,10 @@ class SavedModelTest(test.TestCase): self.assertEqual( 42, ops.get_collection(ops.GraphKeys.GLOBAL_VARIABLES)[0].eval()) + @test_util.run_deprecated_v1 def testSequence(self): export_dir = self._get_export_dir("test_sequence") - builder = saved_model_builder.SavedModelBuilder(export_dir) + builder = saved_model_builder._SavedModelBuilder(export_dir) # Expect an assertion error since add_meta_graph_and_variables() should be # invoked before any add_meta_graph() calls. @@ -220,9 +238,10 @@ class SavedModelTest(test.TestCase): self.assertRaises(AssertionError, builder.add_meta_graph_and_variables, sess, ["baz"]) + @test_util.run_deprecated_v1 def testTags(self): export_dir = self._get_export_dir("test_tags") - builder = saved_model_builder.SavedModelBuilder(export_dir) + builder = saved_model_builder._SavedModelBuilder(export_dir) # Graph with a single variable. SavedModel invoked to: # - add with weights. @@ -309,9 +328,10 @@ class SavedModelTest(test.TestCase): self.assertRaises(RuntimeError, loader.load, sess, ["foo", "baz"], export_dir) + @test_util.run_deprecated_v1 def testVariables(self): export_dir = self._get_export_dir("test_variables") - builder = saved_model_builder.SavedModelBuilder(export_dir) + builder = saved_model_builder._SavedModelBuilder(export_dir) # Graph with two variables. SavedModel invoked to: # - add with weights. @@ -361,9 +381,10 @@ class SavedModelTest(test.TestCase): self.assertRaises(errors.NotFoundError, loader.load, sess, ["baz"], export_dir) + @test_util.run_deprecated_v1 def testGraphWithoutVariables(self): export_dir = self._get_export_dir("test_graph_has_variables") - builder = saved_model_builder.SavedModelBuilder(export_dir) + builder = saved_model_builder._SavedModelBuilder(export_dir) # Graph with no variables. with self.session(graph=ops.Graph()) as sess: @@ -385,7 +406,7 @@ class SavedModelTest(test.TestCase): a = ops.get_default_graph().get_tensor_by_name(constant_5_name) b = constant_op.constant(6.0) c = a * b - self.assertEqual(30.0, sess.run(c)) + self.assertEqual(30.0, self.evaluate(c)) # Restore the graph with tag "bar". with self.session(graph=ops.Graph()) as sess: @@ -394,11 +415,12 @@ class SavedModelTest(test.TestCase): a = ops.get_default_graph().get_tensor_by_name(constant_6_name) b = constant_op.constant(5.0) c = a * b - self.assertEqual(30.0, sess.run(c)) + self.assertEqual(30.0, self.evaluate(c)) + @test_util.run_deprecated_v1 def testNoOverwrite(self): export_dir = self._get_export_dir("test_no_overwrite") - builder = saved_model_builder.SavedModelBuilder(export_dir) + builder = saved_model_builder._SavedModelBuilder(export_dir) # Graph with a single variable. SavedModel invoked to: # - add with weights. @@ -417,12 +439,13 @@ class SavedModelTest(test.TestCase): # An attempt to create another builder with the same export directory should # result in an assertion error. - self.assertRaises(AssertionError, saved_model_builder.SavedModelBuilder, + self.assertRaises(AssertionError, saved_model_builder._SavedModelBuilder, export_dir) + @test_util.run_deprecated_v1 def testSaveAsText(self): export_dir = self._get_export_dir("test_astext") - builder = saved_model_builder.SavedModelBuilder(export_dir) + builder = saved_model_builder._SavedModelBuilder(export_dir) # Graph with a single variable. SavedModel invoked to: # - add with weights. @@ -451,17 +474,18 @@ class SavedModelTest(test.TestCase): self.assertEqual( 42, ops.get_collection(ops.GraphKeys.GLOBAL_VARIABLES)[0].eval()) + @test_util.run_deprecated_v1 def testCollections(self): export_dir = self._get_export_dir("test_collections") - builder = saved_model_builder.SavedModelBuilder(export_dir) + builder = saved_model_builder._SavedModelBuilder(export_dir) # Graph with a single variable added to a collection. SavedModel invoked to: # - add with weights. with self.session(graph=ops.Graph()) as sess: v = variables.VariableV1(42, name="v") ops.add_to_collection("foo_vars", v) - sess.run(variables.global_variables_initializer()) - self.assertEqual(42, v.eval()) + self.evaluate(variables.global_variables_initializer()) + self.assertEqual(42, self.evaluate(v)) builder.add_meta_graph_and_variables(sess, ["foo"]) # Graph with the same single variable added to a different collection. @@ -470,8 +494,8 @@ class SavedModelTest(test.TestCase): with self.session(graph=ops.Graph()) as sess: v = variables.VariableV1(43, name="v") ops.add_to_collection("bar_vars", v) - sess.run(variables.global_variables_initializer()) - self.assertEqual(43, v.eval()) + self.evaluate(variables.global_variables_initializer()) + self.assertEqual(43, self.evaluate(v)) builder.add_meta_graph(["bar"]) # Save the SavedModel to disk. @@ -501,9 +525,10 @@ class SavedModelTest(test.TestCase): self.assertEqual(len(ops.get_collection("foo_vars")), 0) + @test_util.run_deprecated_v1 def testSignatureDefs(self): export_dir = self._get_export_dir("test_signature_defs") - builder = saved_model_builder.SavedModelBuilder(export_dir) + builder = saved_model_builder._SavedModelBuilder(export_dir) # Graph with a single variable and a single entry in the signature def map. # SavedModel is invoked to add with weights. @@ -563,7 +588,7 @@ class SavedModelTest(test.TestCase): def testSignatureDefValidationFails(self): export_dir = self._get_export_dir("test_signature_def_validation_fail") - builder = saved_model_builder.SavedModelBuilder(export_dir) + builder = saved_model_builder._SavedModelBuilder(export_dir) tensor_without_encoding = meta_graph_pb2.TensorInfo() tensor_without_encoding.dtype = types_pb2.DT_FLOAT @@ -579,19 +604,30 @@ class SavedModelTest(test.TestCase): self._validate_inputs_tensor_info_fail(builder, tensor_empty) self._validate_outputs_tensor_info_fail(builder, tensor_empty) + valid_tensor_info = meta_graph_pb2.TensorInfo() + valid_tensor_info.name = "foo" + valid_tensor_info.dtype = types_pb2.DT_FLOAT + + self._validate_sig_def_keys(builder, valid_tensor_info, + constants.INIT_OP_SIGNATURE_KEY) + self._validate_sig_def_keys(builder, valid_tensor_info, + constants.TRAIN_OP_SIGNATURE_KEY) + + @test_util.run_deprecated_v1 def testSignatureDefValidationSucceedsWithName(self): tensor_with_name = meta_graph_pb2.TensorInfo() tensor_with_name.name = "foo" tensor_with_name.dtype = types_pb2.DT_FLOAT export_dir = self._get_export_dir("test_signature_def_validation_name_1") - builder = saved_model_builder.SavedModelBuilder(export_dir) + builder = saved_model_builder._SavedModelBuilder(export_dir) self._validate_inputs_tensor_info_accept(builder, tensor_with_name) export_dir = self._get_export_dir("test_signature_def_validation_name_2") - builder = saved_model_builder.SavedModelBuilder(export_dir) + builder = saved_model_builder._SavedModelBuilder(export_dir) self._validate_outputs_tensor_info_accept(builder, tensor_with_name) + @test_util.run_deprecated_v1 def testSignatureDefValidationSucceedsWithCoo(self): tensor_with_coo = meta_graph_pb2.TensorInfo() # TODO(soergel) test validation of each of the fields of coo_sparse @@ -599,16 +635,17 @@ class SavedModelTest(test.TestCase): tensor_with_coo.dtype = types_pb2.DT_FLOAT export_dir = self._get_export_dir("test_signature_def_validation_coo_1") - builder = saved_model_builder.SavedModelBuilder(export_dir) + builder = saved_model_builder._SavedModelBuilder(export_dir) self._validate_inputs_tensor_info_accept(builder, tensor_with_coo) export_dir = self._get_export_dir("test_signature_def_validation_coo_2") - builder = saved_model_builder.SavedModelBuilder(export_dir) + builder = saved_model_builder._SavedModelBuilder(export_dir) self._validate_outputs_tensor_info_accept(builder, tensor_with_coo) + @test_util.run_deprecated_v1 def testAssets(self): export_dir = self._get_export_dir("test_assets") - builder = saved_model_builder.SavedModelBuilder(export_dir) + builder = saved_model_builder._SavedModelBuilder(export_dir) with self.session(graph=ops.Graph()) as sess: self._init_and_validate_variable(sess, "v", 42) @@ -618,145 +655,151 @@ class SavedModelTest(test.TestCase): compat.as_bytes(test.get_temp_dir()), compat.as_bytes("ignored.txt")) file_io.write_string_to_file(ignored_filepath, "will be ignored") - asset_collection = self._build_asset_collection("hello42.txt", - "foo bar baz", - "asset_file_tensor") + asset_list = self._build_asset_collection("hello42.txt", "foo bar baz", + "asset_file_tensor") builder.add_meta_graph_and_variables( - sess, ["foo"], assets_collection=asset_collection) + sess, ["foo"], assets_list=asset_list) # Save the SavedModel to disk. builder.save() with self.session(graph=ops.Graph()) as sess: foo_graph = loader.load(sess, ["foo"], export_dir) - self._validate_asset_collection(export_dir, foo_graph.collection_def, - "hello42.txt", "foo bar baz", - "asset_file_tensor:0") + self._validate_assets(export_dir, foo_graph.asset_file_def, "hello42.txt", + "foo bar baz", "asset_file_tensor:0") ignored_asset_path = os.path.join( compat.as_bytes(export_dir), compat.as_bytes(constants.ASSETS_DIRECTORY), compat.as_bytes("ignored.txt")) self.assertFalse(file_io.file_exists(ignored_asset_path)) + @test_util.run_deprecated_v1 def testAssetsNameCollisionDiffFile(self): export_dir = self._get_export_dir("test_assets_name_collision_diff_file") - builder = saved_model_builder.SavedModelBuilder(export_dir) + builder = saved_model_builder._SavedModelBuilder(export_dir) with self.session(graph=ops.Graph()) as sess: self._init_and_validate_variable(sess, "v", 42) - asset_collection = self._build_asset_collection( - "hello42.txt", "foo bar bak", "asset_file_tensor", - asset_subdir="1") + asset_list = self._build_asset_collection( + "hello42.txt", "foo bar bak", "asset_file_tensor", asset_subdir="1") - asset_collection = self._build_asset_collection( - "hello42.txt", "foo bar baz", "asset_file_tensor_1", - asset_subdir="2") + asset_list = self._build_asset_collection( + "hello42.txt", "foo bar baz", "asset_file_tensor_1", asset_subdir="2") builder.add_meta_graph_and_variables( - sess, ["foo"], assets_collection=asset_collection) + sess, ["foo"], assets_list=asset_list) # Save the SavedModel to disk. builder.save() with self.session(graph=ops.Graph()) as sess: foo_graph = loader.load(sess, ["foo"], export_dir) - self._validate_asset_collection(export_dir, foo_graph.collection_def, - "hello42.txt", "foo bar bak", - "asset_file_tensor:0") - self._validate_asset_collection(export_dir, foo_graph.collection_def, - "hello42.txt_1", "foo bar baz", - "asset_file_tensor_1:0", - asset_id=1) + self._validate_assets(export_dir, foo_graph.asset_file_def, "hello42.txt", + "foo bar bak", "asset_file_tensor:0") + self._validate_assets( + export_dir, + foo_graph.asset_file_def, + "hello42.txt_1", + "foo bar baz", + "asset_file_tensor_1:0", + asset_id=1) + @test_util.run_deprecated_v1 def testAssetsNameCollisionSameFilepath(self): export_dir = self._get_export_dir("test_assets_name_collision_same_path") - builder = saved_model_builder.SavedModelBuilder(export_dir) + builder = saved_model_builder._SavedModelBuilder(export_dir) with self.session(graph=ops.Graph()) as sess: self._init_and_validate_variable(sess, "v", 42) - asset_collection = self._build_asset_collection( - "hello42.txt", "foo bar baz", "asset_file_tensor") + asset_list = self._build_asset_collection("hello42.txt", "foo bar baz", + "asset_file_tensor") - asset_collection = self._build_asset_collection( - "hello42.txt", "foo bar baz", "asset_file_tensor_1") + asset_list = self._build_asset_collection("hello42.txt", "foo bar baz", + "asset_file_tensor_1") builder.add_meta_graph_and_variables( - sess, ["foo"], assets_collection=asset_collection) + sess, ["foo"], assets_list=asset_list) # Save the SavedModel to disk. builder.save() with self.session(graph=ops.Graph()) as sess: foo_graph = loader.load(sess, ["foo"], export_dir) - self._validate_asset_collection(export_dir, foo_graph.collection_def, - "hello42.txt", "foo bar baz", - "asset_file_tensor:0") + self._validate_assets(export_dir, foo_graph.asset_file_def, "hello42.txt", + "foo bar baz", "asset_file_tensor:0") # The second tensor should be recorded, but the same. - self._validate_asset_collection(export_dir, foo_graph.collection_def, - "hello42.txt", "foo bar baz", - "asset_file_tensor_1:0", - asset_id=1) + self._validate_assets( + export_dir, + foo_graph.asset_file_def, + "hello42.txt", + "foo bar baz", + "asset_file_tensor_1:0", + asset_id=1) ignored_asset_path = os.path.join( compat.as_bytes(export_dir), compat.as_bytes(constants.ASSETS_DIRECTORY), compat.as_bytes("hello42.txt_1")) self.assertFalse(file_io.file_exists(ignored_asset_path)) + @test_util.run_deprecated_v1 def testAssetsNameCollisionSameFile(self): export_dir = self._get_export_dir("test_assets_name_collision_same_file") - builder = saved_model_builder.SavedModelBuilder(export_dir) + builder = saved_model_builder._SavedModelBuilder(export_dir) with self.session(graph=ops.Graph()) as sess: self._init_and_validate_variable(sess, "v", 42) - asset_collection = self._build_asset_collection( - "hello42.txt", "foo bar baz", "asset_file_tensor", - asset_subdir="1") + asset_list = self._build_asset_collection( + "hello42.txt", "foo bar baz", "asset_file_tensor", asset_subdir="1") - asset_collection = self._build_asset_collection( - "hello42.txt", "foo bar baz", "asset_file_tensor_1", - asset_subdir="2") + asset_list = self._build_asset_collection( + "hello42.txt", "foo bar baz", "asset_file_tensor_1", asset_subdir="2") builder.add_meta_graph_and_variables( - sess, ["foo"], assets_collection=asset_collection) + sess, ["foo"], assets_list=asset_list) # Save the SavedModel to disk. builder.save() with self.session(graph=ops.Graph()) as sess: foo_graph = loader.load(sess, ["foo"], export_dir) - self._validate_asset_collection(export_dir, foo_graph.collection_def, - "hello42.txt", "foo bar baz", - "asset_file_tensor:0") + self._validate_assets(export_dir, foo_graph.asset_file_def, "hello42.txt", + "foo bar baz", "asset_file_tensor:0") # The second tensor should be recorded, but the same. - self._validate_asset_collection(export_dir, foo_graph.collection_def, - "hello42.txt", "foo bar baz", - "asset_file_tensor_1:0", - asset_id=1) + self._validate_assets( + export_dir, + foo_graph.asset_file_def, + "hello42.txt", + "foo bar baz", + "asset_file_tensor_1:0", + asset_id=1) ignored_asset_path = os.path.join( compat.as_bytes(export_dir), compat.as_bytes(constants.ASSETS_DIRECTORY), compat.as_bytes("hello42.txt_1")) self.assertFalse(file_io.file_exists(ignored_asset_path)) + @test_util.run_deprecated_v1 def testAssetsNameCollisionManyFiles(self): export_dir = self._get_export_dir("test_assets_name_collision_many_files") - builder = saved_model_builder.SavedModelBuilder(export_dir) + builder = saved_model_builder._SavedModelBuilder(export_dir) with self.session(graph=ops.Graph()) as sess: self._init_and_validate_variable(sess, "v", 42) for i in range(5): idx = str(i) - asset_collection = self._build_asset_collection( - "hello42.txt", "foo bar baz " + idx, "asset_file_tensor_" + idx, + asset_list = self._build_asset_collection( + "hello42.txt", + "foo bar baz " + idx, + "asset_file_tensor_" + idx, asset_subdir=idx) builder.add_meta_graph_and_variables( - sess, ["foo"], assets_collection=asset_collection) + sess, ["foo"], assets_list=asset_list) # Save the SavedModel to disk. builder.save() @@ -765,18 +808,20 @@ class SavedModelTest(test.TestCase): foo_graph = loader.load(sess, ["foo"], export_dir) for i in range(1, 5): idx = str(i) - self._validate_asset_collection( - export_dir, foo_graph.collection_def, "hello42.txt_" + idx, - "foo bar baz " + idx, "asset_file_tensor_{}:0".format(idx), + self._validate_assets( + export_dir, + foo_graph.asset_file_def, + "hello42.txt_" + idx, + "foo bar baz " + idx, + "asset_file_tensor_{}:0".format(idx), asset_id=i) - self._validate_asset_collection(export_dir, foo_graph.collection_def, - "hello42.txt", "foo bar baz 0", - "asset_file_tensor_0:0") + self._validate_assets(export_dir, foo_graph.asset_file_def, "hello42.txt", + "foo bar baz 0", "asset_file_tensor_0:0") - def testCustomMainOp(self): + def testCustomInitOp(self): export_dir = self._get_export_dir("test_main_op") - builder = saved_model_builder.SavedModelBuilder(export_dir) + builder = saved_model_builder._SavedModelBuilder(export_dir) with self.session(graph=ops.Graph()) as sess: # Add `v1` and `v2` variables to the graph. @@ -792,11 +837,11 @@ class SavedModelTest(test.TestCase): # Set up an assignment op to be run as part of the main_op. with ops.control_dependencies([main_op.main_op()]): add_v1_v2 = math_ops.add(v1._ref(), v2._ref()) - custom_main_op = control_flow_ops.group(state_ops.assign(v3, add_v1_v2)) + custom_init_op = control_flow_ops.group(state_ops.assign(v3, add_v1_v2)) - sess.run(custom_main_op) + self.evaluate(custom_init_op) builder.add_meta_graph_and_variables( - sess, ["foo"], main_op=custom_main_op) + sess, ["foo"], init_op=custom_init_op) # Save the SavedModel to disk. builder.save() @@ -809,83 +854,10 @@ class SavedModelTest(test.TestCase): # the main_op, following a restore. self.assertEqual(3, ops.get_collection("v")[2].eval()) - def testLegacyInitOp(self): - export_dir = self._get_export_dir("test_legacy_init_op") - builder = saved_model_builder.SavedModelBuilder(export_dir) - - with self.session(graph=ops.Graph()) as sess: - # Add `v1` and `v2` variables to the graph. - v1 = variables.VariableV1(1, name="v1") - ops.add_to_collection("v", v1) - v2 = variables.VariableV1(2, name="v2") - ops.add_to_collection("v", v2) - - # Initialize another variable `v3` to 42. - v3 = variables.VariableV1(42, name="v3", trainable=False, collections=[]) - ops.add_to_collection("v", v3) - - # Set up an assignment op to be run as part of the legacy_init_op. - assign_v3 = state_ops.assign(v3, math_ops.add(v1, v2)) - legacy_init_op = control_flow_ops.group(assign_v3, name="legacy_init_op") - - sess.run(variables.global_variables_initializer()) - builder.add_meta_graph_and_variables( - sess, ["foo"], legacy_init_op=legacy_init_op) - - # Save the SavedModel to disk. - builder.save() - - with self.session(graph=ops.Graph()) as sess: - loader.load(sess, ["foo"], export_dir) - self.assertEqual(1, ops.get_collection("v")[0].eval()) - self.assertEqual(2, ops.get_collection("v")[1].eval()) - # Evaluates to the sum of the first two variables and assigned as part of - # the legacy_init_op, following a restore. - self.assertEqual(3, ops.get_collection("v")[2].eval()) - - def testLegacyInitOpWithNonEmptyCollection(self): - export_dir = self._get_export_dir( - "test_legacy_init_op_with_non_empty_collection") - self._testInitOpsWithNonEmptyCollection( - export_dir, constants.LEGACY_INIT_OP_KEY) - - def testMainOpWithNonEmptyCollection(self): - export_dir = self._get_export_dir( - "test_main_op_with_non_empty_collection") - self._testInitOpsWithNonEmptyCollection(export_dir, constants.MAIN_OP_KEY) - - def _testInitOpsWithNonEmptyCollection(self, export_dir, key): - builder = saved_model_builder.SavedModelBuilder(export_dir) - - g = ops.Graph() - with self.session(graph=g) as sess: - # Initialize variable `v1` to 1. - v1 = variables.VariableV1(1, name="v1") - ops.add_to_collection("v", v1) - - # Initialize another variable `v2` to 42. - v2 = variables.VariableV1(42, name="v2", trainable=False, collections=[]) - ops.add_to_collection("v", v2) - - # Set up an assignment op to be run as part of the init op. - assign_v2 = state_ops.assign(v2, v1) - init_op = control_flow_ops.group(assign_v2, name="init_op") - - sess.run(variables.global_variables_initializer()) - - ops.add_to_collection(key, control_flow_ops.no_op()) - # ValueError should be raised since the LEGACY_INIT_OP_KEY collection - # is not empty and we don't support multiple init ops. - with self.assertRaisesRegexp(ValueError, "Graph already contains"): - builder.add_meta_graph_and_variables( - sess, ["foo"], legacy_init_op=init_op) - # We shouldn't be able to add as MAIN_OP, either. - with self.assertRaisesRegexp(ValueError, "Graph already contains"): - builder.add_meta_graph_and_variables(sess, ["foo"], main_op=init_op) - + @test_util.run_deprecated_v1 def testTrainOp(self): export_dir = self._get_export_dir("test_train_op") - builder = saved_model_builder.SavedModelBuilder(export_dir) + builder = saved_model_builder._SavedModelBuilder(export_dir) with self.session(graph=ops.Graph()) as sess: # Add `v1` and `v2` variables to the graph. @@ -894,27 +866,26 @@ class SavedModelTest(test.TestCase): v2 = variables.VariableV1(2, name="v2") ops.add_to_collection("v", v2) - sess.run(variables.global_variables_initializer()) + self.evaluate(variables.global_variables_initializer()) train_op = state_ops.assign_add(v1, v2) - sess.run(train_op) - # TODO(karmel): remove explicit call when in the public method. - builder._add_train_op(train_op) - builder.add_meta_graph_and_variables(sess, ["foo"]) + self.evaluate(train_op) + builder.add_meta_graph_and_variables(sess, ["foo"], train_op=train_op) # Save the SavedModel to disk. builder.save() with self.session(graph=ops.Graph()) as sess: - loader.load(sess, ["foo"], export_dir) + meta_graph_def = loader.load(sess, ["foo"], export_dir) self.assertEqual(3, ops.get_collection("v")[0].eval()) self.assertEqual(2, ops.get_collection("v")[1].eval()) self.assertIsInstance( - ops.get_collection(constants.TRAIN_OP_KEY)[0], ops.Tensor) + loader_impl.get_train_op(meta_graph_def), ops.Tensor) + @test_util.run_deprecated_v1 def testTrainOpGroup(self): export_dir = self._get_export_dir("test_train_op_group") - builder = saved_model_builder.SavedModelBuilder(export_dir) + builder = saved_model_builder._SavedModelBuilder(export_dir) with self.session(graph=ops.Graph()) as sess: # Add `v1` and `v2` variables to the graph. @@ -923,27 +894,26 @@ class SavedModelTest(test.TestCase): v2 = variables.VariableV1(2, name="v2") ops.add_to_collection("v", v2) - sess.run(variables.global_variables_initializer()) + self.evaluate(variables.global_variables_initializer()) train_op = control_flow_ops.group() - sess.run(train_op) - # TODO(karmel): remove explicit call when in the public method. - builder._add_train_op(train_op) - builder.add_meta_graph_and_variables(sess, ["foo"]) + self.evaluate(train_op) + builder.add_meta_graph_and_variables(sess, ["foo"], train_op=train_op) # Save the SavedModel to disk. builder.save() with self.session(graph=ops.Graph()) as sess: - loader.load(sess, ["foo"], export_dir) + meta_graph_def = loader.load(sess, ["foo"], export_dir) self.assertEqual(1, ops.get_collection("v")[0].eval()) self.assertEqual(2, ops.get_collection("v")[1].eval()) self.assertIsInstance( - ops.get_collection(constants.TRAIN_OP_KEY)[0], ops.Operation) + loader_impl.get_train_op(meta_graph_def), ops.Operation) + @test_util.run_deprecated_v1 def testTrainOpAfterVariables(self): export_dir = self._get_export_dir("test_train_op_after_variables") - builder = saved_model_builder.SavedModelBuilder(export_dir) + builder = saved_model_builder._SavedModelBuilder(export_dir) with self.session(graph=ops.Graph()) as sess: # Add `v1` and `v2` variables to the graph. @@ -952,51 +922,50 @@ class SavedModelTest(test.TestCase): v2 = variables.VariableV1(2, name="v2") ops.add_to_collection("v", v2) - sess.run(variables.global_variables_initializer()) + self.evaluate(variables.global_variables_initializer()) builder.add_meta_graph_and_variables(sess, ["pre_foo"]) train_op = state_ops.assign_add(v1, v2) - sess.run(train_op) - # TODO(karmel): remove explicit call when in the public method. - builder._add_train_op(train_op) - builder.add_meta_graph(["foo"]) + self.evaluate(train_op) + builder.add_meta_graph(["foo"], train_op=train_op) # Save the SavedModel to disk. builder.save() with self.session(graph=ops.Graph()) as sess: - loader.load(sess, ["foo"], export_dir) + meta_graph_def = loader.load(sess, ["foo"], export_dir) self.assertIsInstance( - ops.get_collection(constants.TRAIN_OP_KEY)[0], ops.Tensor) + loader_impl.get_train_op(meta_graph_def), ops.Tensor) with self.session(graph=ops.Graph()) as sess: loader.load(sess, ["pre_foo"], export_dir) self.assertFalse(ops.get_collection(constants.TRAIN_OP_KEY)) + @test_util.run_deprecated_v1 def testMultipleAssets(self): export_dir = self._get_export_dir("test_multiple_assets") - builder = saved_model_builder.SavedModelBuilder(export_dir) + builder = saved_model_builder._SavedModelBuilder(export_dir) with self.session(graph=ops.Graph()) as sess: self._init_and_validate_variable(sess, "v", 42) # Build an asset collection specific to `foo` graph. - asset_collection = self._build_asset_collection("foo.txt", "content_foo", - "asset_file_tensor") + asset_list = self._build_asset_collection("foo.txt", "content_foo", + "asset_file_tensor") # Add the asset collection as part of the graph with tag "foo". builder.add_meta_graph_and_variables( - sess, ["foo"], assets_collection=asset_collection) + sess, ["foo"], assets_list=asset_list) with self.session(graph=ops.Graph()) as sess: self._init_and_validate_variable(sess, "v", 42) # Build an asset collection specific to `bar` graph. - asset_collection = self._build_asset_collection("bar.txt", "content_bar", - "asset_file_tensor") + asset_list = self._build_asset_collection("bar.txt", "content_bar", + "asset_file_tensor") # Add the asset collection as part of the graph with tag "bar". - builder.add_meta_graph(["bar"], assets_collection=asset_collection) + builder.add_meta_graph(["bar"], assets_list=asset_list) # Save the SavedModel to disk. builder.save() @@ -1004,43 +973,42 @@ class SavedModelTest(test.TestCase): # Check assets restored for graph with tag "foo". with self.session(graph=ops.Graph()) as sess: foo_graph = loader.load(sess, ["foo"], export_dir) - self._validate_asset_collection(export_dir, foo_graph.collection_def, - "foo.txt", "content_foo", - "asset_file_tensor:0") + self._validate_assets(export_dir, foo_graph.asset_file_def, "foo.txt", + "content_foo", "asset_file_tensor:0") # Check assets restored for graph with tag "bar". with self.session(graph=ops.Graph()) as sess: bar_graph = loader.load(sess, ["bar"], export_dir) - self._validate_asset_collection(export_dir, bar_graph.collection_def, - "bar.txt", "content_bar", - "asset_file_tensor:0") + self._validate_assets(export_dir, bar_graph.asset_file_def, "bar.txt", + "content_bar", "asset_file_tensor:0") + @test_util.run_deprecated_v1 def testDuplicateAssets(self): export_dir = self._get_export_dir("test_duplicate_assets") - builder = saved_model_builder.SavedModelBuilder(export_dir) + builder = saved_model_builder._SavedModelBuilder(export_dir) with self.session(graph=ops.Graph()) as sess: self._init_and_validate_variable(sess, "v", 42) # Build an asset collection with `foo.txt` that has `foo` specific # content. - asset_collection = self._build_asset_collection("foo.txt", "content_foo", - "asset_file_tensor") + asset_list = self._build_asset_collection("foo.txt", "content_foo", + "asset_file_tensor") # Add the asset collection as part of the graph with tag "foo". builder.add_meta_graph_and_variables( - sess, ["foo"], assets_collection=asset_collection) + sess, ["foo"], assets_list=asset_list) with self.session(graph=ops.Graph()) as sess: self._init_and_validate_variable(sess, "v", 42) # Build an asset collection with `foo.txt` that has `bar` specific # content. - asset_collection = self._build_asset_collection("foo.txt", "content_bar", - "asset_file_tensor") + asset_list = self._build_asset_collection("foo.txt", "content_bar", + "asset_file_tensor") # Add the asset collection as part of the graph with tag "bar". - builder.add_meta_graph(["bar"], assets_collection=asset_collection) + builder.add_meta_graph(["bar"], assets_list=asset_list) # Save the SavedModel to disk. builder.save() @@ -1048,9 +1016,8 @@ class SavedModelTest(test.TestCase): # Check assets restored for graph with tag "foo". with self.session(graph=ops.Graph()) as sess: foo_graph = loader.load(sess, ["foo"], export_dir) - self._validate_asset_collection(export_dir, foo_graph.collection_def, - "foo.txt", "content_foo", - "asset_file_tensor:0") + self._validate_assets(export_dir, foo_graph.asset_file_def, "foo.txt", + "content_foo", "asset_file_tensor:0") # Check assets restored for graph with tag "bar". with self.session(graph=ops.Graph()) as sess: @@ -1059,13 +1026,13 @@ class SavedModelTest(test.TestCase): # Validate the assets for `bar` graph. `foo.txt` should contain the # original contents corresponding to `foo` graph since an asset with the # same name across multiple graphs is only stored the first time - self._validate_asset_collection(export_dir, bar_graph.collection_def, - "foo.txt", "content_foo", - "asset_file_tensor:0") + self._validate_assets(export_dir, bar_graph.asset_file_def, "foo.txt", + "content_foo", "asset_file_tensor:0") + @test_util.run_deprecated_v1 def testOp(self): export_dir = self._get_export_dir("test_op") - builder = saved_model_builder.SavedModelBuilder(export_dir) + builder = saved_model_builder._SavedModelBuilder(export_dir) with session.Session( graph=ops.Graph(), @@ -1086,7 +1053,7 @@ class SavedModelTest(test.TestCase): ops.add_to_collection("v", v3) ops.add_to_collection("init_op", init_op) - sess.run(variables.global_variables_initializer()) + self.evaluate(variables.global_variables_initializer()) self.assertEqual(1, ops.get_collection("v")[0].eval()) self.assertEqual(2, ops.get_collection("v")[1].eval()) @@ -1108,7 +1075,7 @@ class SavedModelTest(test.TestCase): def testCustomSaveable(self): export_dir = self._get_export_dir("custom_saveable") - builder = saved_model_builder.SavedModelBuilder(export_dir) + builder = saved_model_builder._SavedModelBuilder(export_dir) with session.Session( graph=ops.Graph(), @@ -1135,13 +1102,14 @@ class SavedModelTest(test.TestCase): self.assertEqual(b"k1", v1.keys().eval()) self.assertEqual(3.0, v1.values().eval()) + @test_util.run_deprecated_v1 def testCustomSaver(self): export_dir = self._get_export_dir("test_custom_saver") - builder = saved_model_builder.SavedModelBuilder(export_dir) + builder = saved_model_builder._SavedModelBuilder(export_dir) with self.session(graph=ops.Graph()) as sess: variables.VariableV1(1, name="v1") - sess.run(variables.global_variables_initializer()) + self.evaluate(variables.global_variables_initializer()) custom_saver = training.Saver(name="my_saver") builder.add_meta_graph_and_variables(sess, ["tag"], saver=custom_saver) @@ -1157,13 +1125,14 @@ class SavedModelTest(test.TestCase): self.assertEqual( saved_graph.saver_def.restore_op_name, "my_saver/restore_all") + @test_util.run_deprecated_v1 def testNoCustomSaver(self): export_dir = self._get_export_dir("test_no_custom_saver") - builder = saved_model_builder.SavedModelBuilder(export_dir) + builder = saved_model_builder._SavedModelBuilder(export_dir) with self.session(graph=ops.Graph()) as sess: variables.VariableV1(1, name="v1") - sess.run(variables.global_variables_initializer()) + self.evaluate(variables.global_variables_initializer()) training.Saver(name="my_saver") builder.add_meta_graph_and_variables(sess, ["tag"]) @@ -1179,13 +1148,14 @@ class SavedModelTest(test.TestCase): self.assertEqual( saved_graph.saver_def.restore_op_name, "save/restore_all") + @test_util.run_deprecated_v1 def testMultipleCustomSavers(self): export_dir = self._get_export_dir("test_multiple_custom_savers") - builder = saved_model_builder.SavedModelBuilder(export_dir) + builder = saved_model_builder._SavedModelBuilder(export_dir) with self.session(graph=ops.Graph()) as sess: variables.VariableV1(1, name="v1") - sess.run(variables.global_variables_initializer()) + self.evaluate(variables.global_variables_initializer()) builder.add_meta_graph_and_variables(sess, ["tag_0"]) saver_1 = training.Saver() @@ -1209,21 +1179,22 @@ class SavedModelTest(test.TestCase): _validate_custom_saver("tag_1", "save_1/restore_all") _validate_custom_saver("tag_2", "save_2/restore_all") + @test_util.run_deprecated_v1 def testImportScope(self): export_dir = self._get_export_dir("test_scoped_assets") - builder = saved_model_builder.SavedModelBuilder(export_dir) + builder = saved_model_builder._SavedModelBuilder(export_dir) # Build a SavedModel with a variable, an asset, and a constant tensor. with self.session(graph=ops.Graph()) as sess: self._init_and_validate_variable(sess, "v", 42) - asset_collection = self._build_asset_collection("foo.txt", "content_foo", - "asset_file_tensor") + asset_list = self._build_asset_collection("foo.txt", "content_foo", + "asset_file_tensor") constant_op.constant("constant value", name="constant_tensor_name") builder.add_meta_graph_and_variables( - sess, ["tag_name"], assets_collection=asset_collection) + sess, ["tag_name"], assets_list=asset_list) # Save the asset file path for later comparison. - asset_file_path = asset_collection[0].eval() + asset_file_path = asset_list[0].eval() # Save the SavedModel to disk. builder.save() @@ -1244,16 +1215,14 @@ class SavedModelTest(test.TestCase): # The loaded asset tensor should be scoped, but the asset file path and # contents should be unchanged. - asset_collection = ops.get_collection(ops.GraphKeys.ASSET_FILEPATHS) - self.assertEqual(1, len(asset_collection)) - self.assertEqual(asset_file_path, asset_collection[0].eval()) - self.assertEqual("scope_name/asset_file_tensor:0", - asset_collection[0].name) + asset_list = ops.get_collection(ops.GraphKeys.ASSET_FILEPATHS) + self.assertEqual(1, len(asset_list)) + self.assertEqual(asset_file_path, asset_list[0].eval()) + self.assertEqual("scope_name/asset_file_tensor:0", asset_list[0].name) # The static asset data inside graph_proto.collection_def should not be # scoped. - self._validate_asset_collection(export_dir, graph_proto.collection_def, - "foo.txt", "content_foo", - "asset_file_tensor:0") + self._validate_assets(export_dir, graph_proto.asset_file_def, "foo.txt", + "content_foo", "asset_file_tensor:0") # The constant tensor should be scoped, but its contents should be # unchanged. @@ -1262,9 +1231,10 @@ class SavedModelTest(test.TestCase): ops.get_default_graph().get_tensor_by_name( "scope_name/constant_tensor_name:0").eval()) + @test_util.run_deprecated_v1 def testClearDevices(self): export_dir = self._get_export_dir("test_clear_devices") - builder = saved_model_builder.SavedModelBuilder(export_dir) + builder = saved_model_builder._SavedModelBuilder(export_dir) # Specify a device and save a variable. ops.reset_default_graph() @@ -1286,89 +1256,19 @@ class SavedModelTest(test.TestCase): self.assertEqual( 42, ops.get_collection(ops.GraphKeys.GLOBAL_VARIABLES)[0].eval()) - def testStripDefaultAttrs(self): - export_dir = self._get_export_dir("test_strip_default_attrs") - builder = saved_model_builder.SavedModelBuilder(export_dir) - - # Add a graph with two float32 variables and a Complex Op composing them - # with strip_default_attrs enabled. - with session.Session(graph=ops.Graph()) as sess: - real_num = variables.VariableV1(1.0, dtype=dtypes.float32, name="real") - imag_num = variables.VariableV1(2.0, dtype=dtypes.float32, name="imag") - math_ops.complex(real_num, imag_num, name="complex") - sess.run(variables.global_variables_initializer()) - builder.add_meta_graph_and_variables( - sess, ["foo"], strip_default_attrs=True) - - # Add a graph with the same float32 variables and a Complex Op composing - # them with strip_default_attrs disabled. - with session.Session(graph=ops.Graph()) as sess: - real_num = variables.VariableV1(1.0, dtype=dtypes.float32, name="real") - imag_num = variables.VariableV1(2.0, dtype=dtypes.float32, name="imag") - math_ops.complex(real_num, imag_num, name="complex") - sess.run(variables.global_variables_initializer()) - builder.add_meta_graph(["bar"], strip_default_attrs=False) - - # Save the SavedModel to disk in text format. - builder.save(as_text=True) - - # Loading graph "foo" via the loader must restore the defaults for the - # "Complex" node based on the "Complex" OpDef in the Op registry. - sess = session.Session(graph=ops.Graph()) - meta_graph_def = loader.load(sess, ["foo"], export_dir) - complex_node = test_util.get_node_def_from_graph("complex", - meta_graph_def.graph_def) - self.assertIn("T", complex_node.attr) - self.assertIn("Tout", complex_node.attr) - - # Load graph "foo" from disk as-is to verify default attrs are stripped. - # pylint: disable=protected-access - saved_model_pb = loader_impl._parse_saved_model(export_dir) - self.assertIsNotNone(saved_model_pb) - # pylint: enable=protected-access - - meta_graph_foo_def = None - meta_graph_bar_def = None - for meta_graph_def in saved_model_pb.meta_graphs: - if set(meta_graph_def.meta_info_def.tags) == set(["foo"]): - meta_graph_foo_def = meta_graph_def - elif set(meta_graph_def.meta_info_def.tags) == set(["bar"]): - meta_graph_bar_def = meta_graph_def - - self.assertIsNotNone(meta_graph_foo_def) - self.assertIsNotNone(meta_graph_bar_def) - - # "Complex" Op has 2 attributes with defaults: - # o "T" : float32. (input type) - # o "Tout" : complex64. (output type) - - # "Complex" Op in graph "foo" shouldn't have attributes "T" and "Tout". - # Graph "foo" was saved with strip_default_attrs set to True. - node_def = test_util.get_node_def_from_graph("complex", - meta_graph_foo_def.graph_def) - self.assertNotIn("T", node_def.attr) - self.assertNotIn("Tout", node_def.attr) - - # "Complex" Op in graph "bar" must have attributes "T" and "Tout". - # Graph "bar" was saved with strip_default_attrs set to False. - node_def = test_util.get_node_def_from_graph("complex", - meta_graph_bar_def.graph_def) - self.assertIn("T", node_def.attr) - self.assertIn("Tout", node_def.attr) - # Tests the behavior of loading SavedModels that having missing attrs or attrs # with incorrect types. def testInconsistentConsumerDefaultAttrs(self): export_dir = self._get_export_dir( "test_strip_default_attrs_no_consumer_defaults") - builder = saved_model_builder.SavedModelBuilder(export_dir) + builder = saved_model_builder._SavedModelBuilder(export_dir) # Add a graph with a single variable and a test op with a defaultless # float32 attr, "test_attr". with session.Session(graph=ops.Graph()) as sess: variables.VariableV1(1.0, dtype=dtypes.float64, name="var") test_ops.test_attr(T=dtypes.float32, name="test_attr") - sess.run(variables.global_variables_initializer()) + self.evaluate(variables.global_variables_initializer()) builder.add_meta_graph_and_variables(sess, ["foo"]) # Save the SavedModel to disk in text format. @@ -1428,5 +1328,207 @@ class SavedModelTest(test.TestCase): loader.load(sess, ["foo"], export_dir) +class SavedModelV1Test(SavedModelTestBase): + + def _validate_asset_collection(self, + export_dir, + graph_collection_def, + expected_asset_file_name, + expected_asset_file_contents, + expected_asset_tensor_name, + asset_id=0): + assets_any = graph_collection_def[constants.ASSETS_KEY].any_list.value + asset = meta_graph_pb2.AssetFileDef() + assets_any[asset_id].Unpack(asset) + assets_path = os.path.join( + compat.as_bytes(export_dir), + compat.as_bytes(constants.ASSETS_DIRECTORY), + compat.as_bytes(expected_asset_file_name)) + actual_asset_contents = file_io.read_file_to_string(assets_path) + self.assertEqual(expected_asset_file_contents, + compat.as_text(actual_asset_contents)) + self.assertEqual(expected_asset_file_name, asset.filename) + self.assertEqual(expected_asset_tensor_name, asset.tensor_info.name) + + @test_util.run_deprecated_v1 + def testWritingAssetsToCollection(self): + export_dir = self._get_export_dir("test_writing_assets_to_collection") + builder = saved_model_builder.SavedModelBuilder(export_dir) + + with self.session(graph=ops.Graph()) as sess: + self._init_and_validate_variable(sess, "v", 42) + + # Build an asset list. + ignored_filepath = os.path.join( + compat.as_bytes(test.get_temp_dir()), compat.as_bytes("ignored.txt")) + file_io.write_string_to_file(ignored_filepath, "will be ignored") + + asset_collection = self._build_asset_collection( + "hello42.txt", "foo bar baz", "asset_file_tensor") + + builder.add_meta_graph_and_variables( + sess, ["foo"], assets_collection=asset_collection) + + # Save the SavedModel to disk. + builder.save() + + with self.session(graph=ops.Graph()) as sess: + foo_graph = loader.load(sess, ["foo"], export_dir) + self._validate_asset_collection(export_dir, foo_graph.collection_def, + "hello42.txt", "foo bar baz", + "asset_file_tensor:0") + ignored_asset_path = os.path.join( + compat.as_bytes(export_dir), + compat.as_bytes(constants.ASSETS_DIRECTORY), + compat.as_bytes("ignored.txt")) + self.assertFalse(file_io.file_exists(ignored_asset_path)) + + @test_util.run_deprecated_v1 + def testLegacyInitOpWithNonEmptyCollection(self): + export_dir = self._get_export_dir( + "test_legacy_init_op_with_non_empty_collection") + self._testInitOpsWithNonEmptyCollection(export_dir, + constants.LEGACY_INIT_OP_KEY) + + @test_util.run_deprecated_v1 + def testMainOpWithNonEmptyCollection(self): + export_dir = self._get_export_dir("test_main_op_with_non_empty_collection") + self._testInitOpsWithNonEmptyCollection(export_dir, constants.MAIN_OP_KEY) + + def _testInitOpsWithNonEmptyCollection(self, export_dir, key): + builder = saved_model_builder.SavedModelBuilder(export_dir) + + g = ops.Graph() + with self.session(graph=g) as sess: + # Initialize variable `v1` to 1. + v1 = variables.VariableV1(1, name="v1") + ops.add_to_collection("v", v1) + + # Initialize another variable `v2` to 42. + v2 = variables.VariableV1(42, name="v2", trainable=False, collections=[]) + ops.add_to_collection("v", v2) + + # Set up an assignment op to be run as part of the init op. + assign_v2 = state_ops.assign(v2, v1) + init_op = control_flow_ops.group(assign_v2, name="init_op") + + self.evaluate(variables.global_variables_initializer()) + + ops.add_to_collection(key, control_flow_ops.no_op()) + # ValueError should be raised since the LEGACY_INIT_OP_KEY collection + # is not empty and we don't support multiple init ops. + with self.assertRaisesRegexp(ValueError, "Graph already contains"): + builder.add_meta_graph_and_variables( + sess, ["foo"], legacy_init_op=init_op) + # We shouldn't be able to add as MAIN_OP, either. + with self.assertRaisesRegexp(ValueError, "Graph already contains"): + builder.add_meta_graph_and_variables(sess, ["foo"], main_op=init_op) + + def testStripDefaultAttrs(self): + export_dir = self._get_export_dir("test_strip_default_attrs") + builder = saved_model_builder.SavedModelBuilder(export_dir) + + # Add a graph with two float32 variables and a Complex Op composing them + # with strip_default_attrs enabled. + with session.Session(graph=ops.Graph()) as sess: + real_num = variables.VariableV1(1.0, dtype=dtypes.float32, name="real") + imag_num = variables.VariableV1(2.0, dtype=dtypes.float32, name="imag") + math_ops.complex(real_num, imag_num, name="complex") + self.evaluate(variables.global_variables_initializer()) + builder.add_meta_graph_and_variables( + sess, ["foo"], strip_default_attrs=True) + + # Add a graph with the same float32 variables and a Complex Op composing + # them with strip_default_attrs disabled. + with session.Session(graph=ops.Graph()) as sess: + real_num = variables.VariableV1(1.0, dtype=dtypes.float32, name="real") + imag_num = variables.VariableV1(2.0, dtype=dtypes.float32, name="imag") + math_ops.complex(real_num, imag_num, name="complex") + self.evaluate(variables.global_variables_initializer()) + builder.add_meta_graph(["bar"], strip_default_attrs=False) + + # Save the SavedModel to disk in text format. + builder.save(as_text=True) + + # Loading graph "foo" via the loader must restore the defaults for the + # "Complex" node based on the "Complex" OpDef in the Op registry. + sess = session.Session(graph=ops.Graph()) + meta_graph_def = loader.load(sess, ["foo"], export_dir) + complex_node = test_util.get_node_def_from_graph("complex", + meta_graph_def.graph_def) + self.assertIn("T", complex_node.attr) + self.assertIn("Tout", complex_node.attr) + + # Load graph "foo" from disk as-is to verify default attrs are stripped. + # pylint: disable=protected-access + saved_model_pb = loader_impl._parse_saved_model(export_dir) + self.assertIsNotNone(saved_model_pb) + # pylint: enable=protected-access + + meta_graph_foo_def = None + meta_graph_bar_def = None + for meta_graph_def in saved_model_pb.meta_graphs: + if set(meta_graph_def.meta_info_def.tags) == set(["foo"]): + meta_graph_foo_def = meta_graph_def + elif set(meta_graph_def.meta_info_def.tags) == set(["bar"]): + meta_graph_bar_def = meta_graph_def + + self.assertIsNotNone(meta_graph_foo_def) + self.assertIsNotNone(meta_graph_bar_def) + + # "Complex" Op has 2 attributes with defaults: + # o "T" : float32. (input type) + # o "Tout" : complex64. (output type) + + # "Complex" Op in graph "foo" shouldn't have attributes "T" and "Tout". + # Graph "foo" was saved with strip_default_attrs set to True. + node_def = test_util.get_node_def_from_graph("complex", + meta_graph_foo_def.graph_def) + self.assertNotIn("T", node_def.attr) + self.assertNotIn("Tout", node_def.attr) + + # "Complex" Op in graph "bar" must have attributes "T" and "Tout". + # Graph "bar" was saved with strip_default_attrs set to False. + node_def = test_util.get_node_def_from_graph("complex", + meta_graph_bar_def.graph_def) + self.assertIn("T", node_def.attr) + self.assertIn("Tout", node_def.attr) + + @test_util.run_deprecated_v1 + def testLegacyInitOp(self): + export_dir = self._get_export_dir("test_legacy_init_op") + builder = saved_model_builder.SavedModelBuilder(export_dir) + + with self.session(graph=ops.Graph()) as sess: + # Add `v1` and `v2` variables to the graph. + v1 = variables.VariableV1(1, name="v1") + ops.add_to_collection("v", v1) + v2 = variables.VariableV1(2, name="v2") + ops.add_to_collection("v", v2) + + # Initialize another variable `v3` to 42. + v3 = variables.VariableV1(42, name="v3", trainable=False, collections=[]) + ops.add_to_collection("v", v3) + + # Set up an assignment op to be run as part of the init_op. + assign_v3 = state_ops.assign(v3, math_ops.add(v1, v2)) + legacy_init_op = control_flow_ops.group(assign_v3, name="legacy_init_op") + + self.evaluate(variables.global_variables_initializer()) + builder.add_meta_graph_and_variables( + sess, ["foo"], legacy_init_op=legacy_init_op) + + # Save the SavedModel to disk. + builder.save() + + with self.session(graph=ops.Graph()) as sess: + loader.load(sess, ["foo"], export_dir) + self.assertEqual(1, ops.get_collection("v")[0].eval()) + self.assertEqual(2, ops.get_collection("v")[1].eval()) + # Evaluates to the sum of the first two variables and assigned as part of + # the legacy_init_op, following a restore. + self.assertEqual(3, ops.get_collection("v")[2].eval()) + + if __name__ == "__main__": test.main() diff --git a/tensorflow/python/saved_model/saved_object_graph.proto b/tensorflow/python/saved_model/saved_object_graph.proto new file mode 100644 index 00000000000..89f82b90d61 --- /dev/null +++ b/tensorflow/python/saved_model/saved_object_graph.proto @@ -0,0 +1,38 @@ +syntax = "proto3"; + +import "tensorflow/core/protobuf/checkpointable_object_graph.proto"; + +option cc_enable_arenas = true; + +package tensorflow; + +// A SavedObjectGraph is part of object-based SavedModels in TF 2.0. It +// describes the directed graph of Python objects (or equivalent in other +// languages) that make up a model, with nodes[0] at the root. + +// SavedObjectGraph shares some structure with CheckpointableObjectGraph, but +// ObjectGraph belongs to the SavedModel and contains pointers to functions and +// type information, while CheckpointableObjectGraph lives in the checkpoint and +// contains pointers only to variable values. + +// NOTE: This protocol buffer format is experimental and subject to change. + +message SavedObjectGraph { + message SavedObject { + // Objects which this object depends on: named edges in the dependency + // graph. + repeated CheckpointableObjectGraph.CheckpointableObject.ObjectReference + children = 1; + // Removed when forking from CheckpointableObjectGraph. + reserved "attributes"; + reserved 2; + // Slot variables owned by this object. This describes the three-way + // (optimizer, variable, slot variable) relationship; none of the three + // depend on the others directly. + repeated + CheckpointableObjectGraph.CheckpointableObject.SlotVariableReference + slot_variables = 3; + } + + repeated SavedObject nodes = 1; +} diff --git a/tensorflow/python/saved_model/signature_constants.py b/tensorflow/python/saved_model/signature_constants.py index 96460717ec5..0efe1763430 100644 --- a/tensorflow/python/saved_model/signature_constants.py +++ b/tensorflow/python/saved_model/signature_constants.py @@ -135,7 +135,7 @@ tf_export( ################################################################################ # Train/Eval API constants. -# Not exported while export_all_saved_models is in contrib. +# Not exported while export_all_saved_models is experimental. SUPERVISED_TRAIN_METHOD_NAME = "tensorflow/supervised/training" diff --git a/tensorflow/python/saved_model/signature_def_utils.py b/tensorflow/python/saved_model/signature_def_utils.py index 27d6b70e9dc..6a3c0aaf385 100644 --- a/tensorflow/python/saved_model/signature_def_utils.py +++ b/tensorflow/python/saved_model/signature_def_utils.py @@ -24,6 +24,8 @@ from __future__ import print_function from tensorflow.python.saved_model.signature_def_utils_impl import build_signature_def from tensorflow.python.saved_model.signature_def_utils_impl import classification_signature_def from tensorflow.python.saved_model.signature_def_utils_impl import is_valid_signature +from tensorflow.python.saved_model.signature_def_utils_impl import load_op_from_signature_def +from tensorflow.python.saved_model.signature_def_utils_impl import op_signature_def from tensorflow.python.saved_model.signature_def_utils_impl import predict_signature_def from tensorflow.python.saved_model.signature_def_utils_impl import regression_signature_def from tensorflow.python.saved_model.signature_def_utils_impl import supervised_eval_signature_def diff --git a/tensorflow/python/saved_model/signature_def_utils_impl.py b/tensorflow/python/saved_model/signature_def_utils_impl.py index 6e5e3bc6822..f6e6e1d13ec 100644 --- a/tensorflow/python/saved_model/signature_def_utils_impl.py +++ b/tensorflow/python/saved_model/signature_def_utils_impl.py @@ -21,9 +21,10 @@ from __future__ import print_function from tensorflow.core.framework import types_pb2 from tensorflow.core.protobuf import meta_graph_pb2 +from tensorflow.python.framework import errors from tensorflow.python.framework import ops from tensorflow.python.saved_model import signature_constants -from tensorflow.python.saved_model import utils +from tensorflow.python.saved_model import utils_impl as utils from tensorflow.python.util import deprecation from tensorflow.python.util.tf_export import tf_export @@ -349,3 +350,51 @@ def _is_valid_classification_signature(signature_def): return False return True + + +def op_signature_def(op, key): + """Creates a signature def with the output pointing to an op. + + Note that op isn't strictly enforced to be an Op object, and may be a Tensor. + It is recommended to use the build_signature_def() function for Tensors. + + Args: + op: An Op (or possibly Tensor). + key: Key to graph element in the SignatureDef outputs. + + Returns: + A SignatureDef with a single output pointing to the op. + """ + # Use build_tensor_info_from_op, which creates a TensorInfo from the element's + # name. + return build_signature_def(outputs={key: utils.build_tensor_info_from_op(op)}) + + +def load_op_from_signature_def(signature_def, key, import_scope=None): + """Load an Op from a SignatureDef created by op_signature_def(). + + Args: + signature_def: a SignatureDef proto + key: string key to op in the SignatureDef outputs. + import_scope: Scope used to import the op + + Returns: + Op (or possibly Tensor) in the graph with the same name as saved in the + SignatureDef. + + Raises: + NotFoundError: If the op could not be found in the graph. + """ + tensor_info = signature_def.outputs[key] + try: + # The init and train ops are not strictly enforced to be operations, so + # retrieve any graph element (can be either op or tensor). + return utils.get_element_from_tensor_info( + tensor_info, import_scope=import_scope) + except KeyError: + raise errors.NotFoundError( + None, None, + 'The {0} could not be found in the graph. Please make sure the ' + 'SavedModel was created by the internal _SavedModelBuilder. If you ' + 'are using the public API, please make sure the SignatureDef in the ' + 'SavedModel does not contain the key "{0}".'.format(key)) diff --git a/tensorflow/python/saved_model/signature_def_utils_test.py b/tensorflow/python/saved_model/signature_def_utils_test.py index 18c55d8d332..53c452359f1 100644 --- a/tensorflow/python/saved_model/signature_def_utils_test.py +++ b/tensorflow/python/saved_model/signature_def_utils_test.py @@ -22,7 +22,9 @@ from tensorflow.core.framework import types_pb2 from tensorflow.core.protobuf import meta_graph_pb2 from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes +from tensorflow.python.framework import test_util from tensorflow.python.ops import array_ops +from tensorflow.python.ops import math_ops from tensorflow.python.platform import test from tensorflow.python.saved_model import signature_constants from tensorflow.python.saved_model import signature_def_utils_impl @@ -58,6 +60,7 @@ def _make_signature(inputs, outputs, name=None): class SignatureDefUtilsTest(test.TestCase): + @test_util.run_deprecated_v1 def testBuildSignatureDef(self): x = array_ops.placeholder(dtypes.float32, 1, name="x") x_tensor_info = utils.build_tensor_info(x) @@ -88,6 +91,7 @@ class SignatureDefUtilsTest(test.TestCase): self.assertEqual(types_pb2.DT_FLOAT, y_tensor_info_actual.dtype) self.assertEqual(0, len(y_tensor_info_actual.tensor_shape.dim)) + @test_util.run_deprecated_v1 def testRegressionSignatureDef(self): input1 = constant_op.constant("a", name="input-1") output1 = constant_op.constant(2.2, name="output-1") @@ -113,6 +117,7 @@ class SignatureDefUtilsTest(test.TestCase): self.assertEqual(types_pb2.DT_FLOAT, y_tensor_info_actual.dtype) self.assertEqual(0, len(y_tensor_info_actual.tensor_shape.dim)) + @test_util.run_deprecated_v1 def testClassificationSignatureDef(self): input1 = constant_op.constant("a", name="input-1") output1 = constant_op.constant("b", name="output-1") @@ -144,6 +149,7 @@ class SignatureDefUtilsTest(test.TestCase): self.assertEqual(types_pb2.DT_FLOAT, scores_tensor_info_actual.dtype) self.assertEqual(0, len(scores_tensor_info_actual.tensor_shape.dim)) + @test_util.run_deprecated_v1 def testPredictionSignatureDef(self): input1 = constant_op.constant("a", name="input-1") input2 = constant_op.constant("b", name="input-2") @@ -180,11 +186,13 @@ class SignatureDefUtilsTest(test.TestCase): self.assertEqual(types_pb2.DT_STRING, output2_tensor_info_actual.dtype) self.assertEqual(0, len(output2_tensor_info_actual.tensor_shape.dim)) + @test_util.run_deprecated_v1 def testTrainSignatureDef(self): self._testSupervisedSignatureDef( signature_def_utils_impl.supervised_train_signature_def, signature_constants.SUPERVISED_TRAIN_METHOD_NAME) + @test_util.run_deprecated_v1 def testEvalSignatureDef(self): self._testSupervisedSignatureDef( signature_def_utils_impl.supervised_eval_signature_def, @@ -238,11 +246,13 @@ class SignatureDefUtilsTest(test.TestCase): self.assertEqual( types_pb2.DT_FLOAT, signature_def.outputs["metrics/value"].dtype) + @test_util.run_deprecated_v1 def testTrainSignatureDefMissingInputs(self): self._testSupervisedSignatureDefMissingInputs( signature_def_utils_impl.supervised_train_signature_def, signature_constants.SUPERVISED_TRAIN_METHOD_NAME) + @test_util.run_deprecated_v1 def testEvalSignatureDefMissingInputs(self): self._testSupervisedSignatureDefMissingInputs( signature_def_utils_impl.supervised_eval_signature_def, @@ -413,5 +423,22 @@ class SignatureDefUtilsTest(test.TestCase): {}, signature_constants.PREDICT_METHOD_NAME) + def testOpSignatureDef(self): + key = "adding_1_and_2_key" + add_op = math_ops.add(1, 2, name="adding_1_and_2") + signature_def = signature_def_utils_impl.op_signature_def(add_op, key) + self.assertIn(key, signature_def.outputs) + self.assertEqual(add_op.name, signature_def.outputs[key].name) + + def testLoadOpFromSignatureDef(self): + key = "adding_1_and_2_key" + add_op = math_ops.add(1, 2, name="adding_1_and_2") + signature_def = signature_def_utils_impl.op_signature_def(add_op, key) + + self.assertEqual( + add_op, + signature_def_utils_impl.load_op_from_signature_def(signature_def, key)) + + if __name__ == "__main__": test.main() diff --git a/tensorflow/python/saved_model/simple_save_test.py b/tensorflow/python/saved_model/simple_save_test.py index 18f82daadad..21c2e9df2fa 100644 --- a/tensorflow/python/saved_model/simple_save_test.py +++ b/tensorflow/python/saved_model/simple_save_test.py @@ -21,6 +21,7 @@ from __future__ import print_function import os from tensorflow.python.framework import ops +from tensorflow.python.framework import test_util from tensorflow.python.ops import variables from tensorflow.python.platform import test from tensorflow.python.saved_model import loader @@ -33,8 +34,8 @@ class SimpleSaveTest(test.TestCase): def _init_and_validate_variable(self, sess, variable_name, variable_value): v = variables.Variable(variable_value, name=variable_name) - sess.run(variables.global_variables_initializer()) - self.assertEqual(variable_value, v.eval()) + self.evaluate(variables.global_variables_initializer()) + self.assertEqual(variable_value, self.evaluate(v)) return v def _check_variable_info(self, actual_variable, expected_variable): @@ -53,6 +54,7 @@ class SimpleSaveTest(test.TestCase): self.assertEqual(actual_tensor_info.tensor_shape.dim[i].size, expected_tensor.shape[i]) + @test_util.run_deprecated_v1 def testSimpleSave(self): """Test simple_save that uses the default parameters.""" export_dir = os.path.join(test.get_temp_dir(), diff --git a/tensorflow/python/saved_model/utils_impl.py b/tensorflow/python/saved_model/utils_impl.py index 10667419761..5caabe59fec 100644 --- a/tensorflow/python/saved_model/utils_impl.py +++ b/tensorflow/python/saved_model/utils_impl.py @@ -141,6 +141,27 @@ def get_tensor_from_tensor_info(tensor_info, graph=None, import_scope=None): raise ValueError("Invalid TensorInfo.encoding: %s" % encoding) +def get_element_from_tensor_info(tensor_info, graph=None, import_scope=None): + """Returns the element in the graph described by a TensorInfo proto. + + Args: + tensor_info: A TensorInfo proto describing an Op or Tensor by name. + graph: The tf.Graph in which tensors are looked up. If None, the current + default graph is used. + import_scope: If not None, names in `tensor_info` are prefixed with this + string before lookup. + + Returns: + Op or tensor in `graph` described by `tensor_info`. + + Raises: + KeyError: If `tensor_info` does not correspond to an op or tensor in `graph` + """ + graph = graph or ops.get_default_graph() + return graph.as_graph_element( + ops.prepend_name_scope(tensor_info.name, import_scope=import_scope)) + + # Path helpers. diff --git a/tensorflow/python/summary/summary.py b/tensorflow/python/summary/summary.py index 9e9e6ed9035..0c13016712f 100644 --- a/tensorflow/python/summary/summary.py +++ b/tensorflow/python/summary/summary.py @@ -52,7 +52,7 @@ from tensorflow.python.util import compat as _compat from tensorflow.python.util.tf_export import tf_export -@tf_export('summary.scalar') +@tf_export(v1=['summary.scalar']) def scalar(name, tensor, collections=None, family=None): """Outputs a `Summary` protocol buffer containing a single scalar value. @@ -82,7 +82,7 @@ def scalar(name, tensor, collections=None, family=None): return val -@tf_export('summary.image') +@tf_export(v1=['summary.image']) def image(name, tensor, max_outputs=3, collections=None, family=None): """Outputs a `Summary` protocol buffer with images. @@ -138,7 +138,7 @@ def image(name, tensor, max_outputs=3, collections=None, family=None): return val -@tf_export('summary.histogram') +@tf_export(v1=['summary.histogram']) def histogram(name, values, collections=None, family=None): # pylint: disable=line-too-long """Outputs a `Summary` protocol buffer with a histogram. @@ -179,7 +179,7 @@ def histogram(name, values, collections=None, family=None): return val -@tf_export('summary.audio') +@tf_export(v1=['summary.audio']) def audio(name, tensor, sample_rate, max_outputs=3, collections=None, family=None): # pylint: disable=line-too-long @@ -228,7 +228,7 @@ def audio(name, tensor, sample_rate, max_outputs=3, collections=None, return val -@tf_export('summary.text') +@tf_export(v1=['summary.text']) def text(name, tensor, collections=None): """Summarizes textual data. @@ -269,7 +269,7 @@ def text(name, tensor, collections=None): return t_summary -@tf_export('summary.tensor_summary') +@tf_export(v1=['summary.tensor_summary']) def tensor_summary(name, tensor, summary_description=None, @@ -325,7 +325,7 @@ def tensor_summary(name, return val -@tf_export('summary.merge') +@tf_export(v1=['summary.merge']) def merge(inputs, collections=None, name=None): # pylint: disable=line-too-long """Merges summaries. @@ -371,7 +371,7 @@ def merge(inputs, collections=None, name=None): return val -@tf_export('summary.merge_all') +@tf_export(v1=['summary.merge_all']) def merge_all(key=_ops.GraphKeys.SUMMARIES, scope=None, name=None): """Merges all summaries collected in the default graph. @@ -404,7 +404,7 @@ def merge_all(key=_ops.GraphKeys.SUMMARIES, scope=None, name=None): return merge(summary_ops, name=name) -@tf_export('summary.get_summary_description') +@tf_export(v1=['summary.get_summary_description']) def get_summary_description(node_def): """Given a TensorSummary node_def, retrieve its SummaryDescription. diff --git a/tensorflow/python/summary/summary_test.py b/tensorflow/python/summary/summary_test.py index cacc28cc596..64f0f315c58 100644 --- a/tensorflow/python/summary/summary_test.py +++ b/tensorflow/python/summary/summary_test.py @@ -30,6 +30,7 @@ from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes from tensorflow.python.framework import meta_graph from tensorflow.python.framework import ops +from tensorflow.python.framework import test_util from tensorflow.python.ops import array_ops from tensorflow.python.ops import variables from tensorflow.python.platform import test @@ -38,6 +39,7 @@ from tensorflow.python.summary import summary as summary_lib class SummaryTest(test.TestCase): + @test_util.run_deprecated_v1 def testScalarSummary(self): with self.cached_session() as s: i = constant_op.constant(3) @@ -51,6 +53,7 @@ class SummaryTest(test.TestCase): self.assertEqual(values[0].tag, 'outer/inner') self.assertEqual(values[0].simple_value, 3.0) + @test_util.run_deprecated_v1 def testScalarSummaryWithFamily(self): with self.cached_session() as s: i = constant_op.constant(7) @@ -74,6 +77,7 @@ class SummaryTest(test.TestCase): self.assertEqual(values[0].tag, 'family/outer/family/inner_1') self.assertEqual(values[0].simple_value, 7.0) + @test_util.run_deprecated_v1 def testSummarizingVariable(self): with self.cached_session() as s: c = constant_op.constant(42.0) @@ -89,6 +93,7 @@ class SummaryTest(test.TestCase): self.assertEqual(value.tag, 'summary') self.assertEqual(value.simple_value, 42.0) + @test_util.run_deprecated_v1 def testImageSummary(self): with self.cached_session() as s: i = array_ops.ones((5, 4, 4, 3)) @@ -103,6 +108,7 @@ class SummaryTest(test.TestCase): expected = sorted('outer/inner/image/{}'.format(i) for i in xrange(3)) self.assertEqual(tags, expected) + @test_util.run_deprecated_v1 def testImageSummaryWithFamily(self): with self.cached_session() as s: i = array_ops.ones((5, 2, 3, 1)) @@ -119,6 +125,7 @@ class SummaryTest(test.TestCase): for i in xrange(3)) self.assertEqual(tags, expected) + @test_util.run_deprecated_v1 def testHistogramSummary(self): with self.cached_session() as s: i = array_ops.ones((5, 4, 4, 3)) @@ -130,6 +137,7 @@ class SummaryTest(test.TestCase): self.assertEqual(len(summary.value), 1) self.assertEqual(summary.value[0].tag, 'outer/inner') + @test_util.run_deprecated_v1 def testHistogramSummaryWithFamily(self): with self.cached_session() as s: i = array_ops.ones((5, 4, 4, 3)) @@ -148,6 +156,7 @@ class SummaryTest(test.TestCase): const = constant_op.constant(10, dtype=dtype) summary_lib.histogram('h', const) + @test_util.run_deprecated_v1 def testAudioSummary(self): with self.cached_session() as s: i = array_ops.ones((5, 3, 4)) @@ -162,6 +171,7 @@ class SummaryTest(test.TestCase): expected = sorted('outer/inner/audio/{}'.format(i) for i in xrange(3)) self.assertEqual(tags, expected) + @test_util.run_deprecated_v1 def testAudioSummaryWithFamily(self): with self.cached_session() as s: i = array_ops.ones((5, 3, 4)) @@ -178,6 +188,7 @@ class SummaryTest(test.TestCase): for i in xrange(3)) self.assertEqual(tags, expected) + @test_util.run_deprecated_v1 def testTextSummary(self): with self.cached_session(): with self.assertRaises(ValueError): @@ -193,6 +204,7 @@ class SummaryTest(test.TestCase): summ = summary_lib.text('foo', array_ops.constant('one')) self.assertEqual(summ.op.type, 'TensorSummaryV2') + @test_util.run_deprecated_v1 def testSummaryNameConversion(self): c = constant_op.constant(3) s = summary_lib.scalar('name with spaces', c) @@ -204,6 +216,7 @@ class SummaryTest(test.TestCase): s3 = summary_lib.scalar('/name/with/leading/slash', c) self.assertEqual(s3.op.name, 'name/with/leading/slash') + @test_util.run_deprecated_v1 def testSummaryWithFamilyMetaGraphExport(self): with ops.name_scope('outer'): i = constant_op.constant(11) diff --git a/tensorflow/python/summary/writer/writer_test.py b/tensorflow/python/summary/writer/writer_test.py index 09d4b63fbb6..d702ddc0a27 100644 --- a/tensorflow/python/summary/writer/writer_test.py +++ b/tensorflow/python/summary/writer/writer_test.py @@ -35,6 +35,7 @@ from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes from tensorflow.python.framework import meta_graph from tensorflow.python.framework import ops +from tensorflow.python.framework import test_util from tensorflow.python.ops import summary_ops_v2 from tensorflow.python.platform import gfile from tensorflow.python.platform import test @@ -100,6 +101,7 @@ class FileWriterTestCase(test.TestCase): # We should be done. self.assertRaises(StopIteration, lambda: next(rr)) + @test_util.run_deprecated_v1 def testAddingSummaryGraphAndRunMetadata(self): test_dir = self._CleanTestDir("basics") sw = self._FileWriter(test_dir) @@ -173,6 +175,7 @@ class FileWriterTestCase(test.TestCase): # We should be done. self.assertRaises(StopIteration, lambda: next(rr)) + @test_util.run_deprecated_v1 def testGraphAsNamed(self): test_dir = self._CleanTestDir("basics_named_graph") with ops.Graph().as_default() as g: @@ -181,6 +184,7 @@ class FileWriterTestCase(test.TestCase): sw.close() self._assertEventsWithGraph(test_dir, g, True) + @test_util.run_deprecated_v1 def testGraphAsPositional(self): test_dir = self._CleanTestDir("basics_positional_graph") with ops.Graph().as_default() as g: @@ -189,6 +193,7 @@ class FileWriterTestCase(test.TestCase): sw.close() self._assertEventsWithGraph(test_dir, g, True) + @test_util.run_deprecated_v1 def testGraphDefAsNamed(self): test_dir = self._CleanTestDir("basics_named_graph_def") with ops.Graph().as_default() as g: @@ -198,6 +203,7 @@ class FileWriterTestCase(test.TestCase): sw.close() self._assertEventsWithGraph(test_dir, g, False) + @test_util.run_deprecated_v1 def testGraphDefAsPositional(self): test_dir = self._CleanTestDir("basics_positional_graph_def") with ops.Graph().as_default() as g: @@ -207,6 +213,7 @@ class FileWriterTestCase(test.TestCase): sw.close() self._assertEventsWithGraph(test_dir, g, False) + @test_util.run_deprecated_v1 def testGraphAndGraphDef(self): with self.assertRaises(ValueError): test_dir = self._CleanTestDir("basics_graph_and_graph_def") @@ -216,12 +223,14 @@ class FileWriterTestCase(test.TestCase): sw = self._FileWriter(test_dir, graph=g, graph_def=gd) sw.close() + @test_util.run_deprecated_v1 def testNeitherGraphNorGraphDef(self): with self.assertRaises(TypeError): test_dir = self._CleanTestDir("basics_string_instead_of_graph") sw = self._FileWriter(test_dir, "string instead of graph object") sw.close() + @test_util.run_deprecated_v1 def testCloseAndReopen(self): test_dir = self._CleanTestDir("close_and_reopen") sw = self._FileWriter(test_dir) @@ -265,6 +274,7 @@ class FileWriterTestCase(test.TestCase): # We should be done. self.assertRaises(StopIteration, lambda: next(rr)) + @test_util.run_deprecated_v1 def testNonBlockingClose(self): test_dir = self._CleanTestDir("non_blocking_close") sw = self._FileWriter(test_dir) @@ -274,6 +284,7 @@ class FileWriterTestCase(test.TestCase): sw.close() self._assertRecent(time_before_close) + @test_util.run_deprecated_v1 def testUseAfterClose(self): test_dir = self._CleanTestDir("use_after_close") sw = self._FileWriter(test_dir) @@ -289,6 +300,7 @@ class FileWriterTestCase(test.TestCase): for w in triggered: self.assertEqual(w.category, UserWarning) + @test_util.run_deprecated_v1 def testWithStatement(self): test_dir = self._CleanTestDir("with_statement") with self._FileWriter(test_dir) as sw: @@ -299,6 +311,7 @@ class FileWriterTestCase(test.TestCase): # Checks that values returned from session Run() calls are added correctly to # summaries. These are numpy types so we need to check they fit in the # protocol buffers correctly. + @test_util.run_deprecated_v1 def testAddingSummariesFromSessionRunCalls(self): test_dir = self._CleanTestDir("global_step") sw = self._FileWriter(test_dir) @@ -309,12 +322,11 @@ class FileWriterTestCase(test.TestCase): summ = summary_pb2.Summary( value=[summary_pb2.Summary.Value( tag="i", simple_value=1.0)]) - sw.add_summary(summ.SerializeToString(), i.eval()) + sw.add_summary(summ.SerializeToString(), self.evaluate(i)) sw.add_summary( summary_pb2.Summary( - value=[summary_pb2.Summary.Value( - tag="l", simple_value=2.0)]), - l.eval()) + value=[summary_pb2.Summary.Value(tag="l", simple_value=2.0)]), + self.evaluate(l)) sw.close() rr = self._EventsReader(test_dir) @@ -346,6 +358,7 @@ class FileWriterTestCase(test.TestCase): # We should be done. self.assertRaises(StopIteration, lambda: next(rr)) + @test_util.run_deprecated_v1 def testPluginMetadataStrippedFromSubsequentEvents(self): test_dir = self._CleanTestDir("basics") sw = self._FileWriter(test_dir) @@ -405,6 +418,7 @@ class FileWriterTestCase(test.TestCase): # We should be done. self.assertRaises(StopIteration, lambda: next(rr)) + @test_util.run_deprecated_v1 def testFileWriterWithSuffix(self): test_dir = self._CleanTestDir("test_suffix") sw = self._FileWriter(test_dir, filename_suffix="_test_suffix") diff --git a/tensorflow/python/tf2.py b/tensorflow/python/tf2.py index c9782a71199..75748f8f2c5 100644 --- a/tensorflow/python/tf2.py +++ b/tensorflow/python/tf2.py @@ -25,6 +25,21 @@ from __future__ import print_function import os +_force_enable = False + + +def enable(): + """Enables v2 behaviors.""" + global _force_enable + _force_enable = True + + +def disable(): + """Disables v2 behaviors (TF2_BEHAVIOR env variable is still respected).""" + global _force_enable + _force_enable = False + + def enabled(): """Returns True iff TensorFlow 2.0 behavior should be enabled.""" - return os.getenv("TF2_BEHAVIOR") is not None + return _force_enable or os.getenv("TF2_BEHAVIOR", "0") != "0" diff --git a/tensorflow/python/tools/BUILD b/tensorflow/python/tools/BUILD index 384c7a82d27..901d6bc335f 100644 --- a/tensorflow/python/tools/BUILD +++ b/tensorflow/python/tools/BUILD @@ -29,6 +29,8 @@ py_library( ":optimize_for_inference_lib", ":selective_registration_header_lib", ":strip_unused_lib", + # Include the TF upgrade script to users can run it directly after install TF + "//tensorflow/tools/compatibility:tf_upgrade_v2", ], ) diff --git a/tensorflow/python/tools/api/generator/api_gen.bzl b/tensorflow/python/tools/api/generator/api_gen.bzl index 2e5d875a58a..5e64cc64d24 100644 --- a/tensorflow/python/tools/api/generator/api_gen.bzl +++ b/tensorflow/python/tools/api/generator/api_gen.bzl @@ -20,7 +20,8 @@ def gen_api_init_files( packages = ["tensorflow.python", "tensorflow.lite.python.lite"], package_deps = ["//tensorflow/python:no_contrib"], output_package = "tensorflow", - output_dir = ""): + output_dir = "", + root_file_name = "__init__.py"): """Creates API directory structure and __init__.py files. Creates a genrule that generates a directory structure with __init__.py @@ -54,13 +55,14 @@ def gen_api_init_files( output_package: Package where generated API will be added to. output_dir: Subdirectory to output API to. If non-empty, must end with '/'. + root_file_name: Name of the root file with all the root imports. """ root_init_template_flag = "" if root_init_template: root_init_template_flag = "--root_init_template=$(location " + root_init_template + ")" primary_package = packages[0] - api_gen_binary_target = ("create_" + primary_package + "_api_%d") % api_version + api_gen_binary_target = ("create_" + primary_package + "_api_%d_%s") % (api_version, name) native.py_binary( name = api_gen_binary_target, srcs = ["//tensorflow/python/tools/api/generator:create_python_api.py"], @@ -73,6 +75,11 @@ def gen_api_init_files( ], ) + # Replace name of root file with root_file_name. + output_files = [ + root_file_name if f == "__init__.py" else f + for f in output_files + ] all_output_files = ["%s%s" % (output_dir, f) for f in output_files] compat_api_version_flags = "" for compat_api_version in compat_api_versions: diff --git a/tensorflow/python/tools/api/generator/api_init_files.bzl b/tensorflow/python/tools/api/generator/api_init_files.bzl index 5699d86e6d5..3517c11cc93 100644 --- a/tensorflow/python/tools/api/generator/api_init_files.bzl +++ b/tensorflow/python/tools/api/generator/api_init_files.bzl @@ -4,17 +4,17 @@ TENSORFLOW_API_INIT_FILES = [ # BEGIN GENERATED FILES "__init__.py", - "app/__init__.py", "bitwise/__init__.py", "compat/__init__.py", "data/__init__.py", "data/experimental/__init__.py", "debugging/__init__.py", + "distribute/__init__.py", "dtypes/__init__.py", "errors/__init__.py", "experimental/__init__.py", "feature_column/__init__.py", - "gfile/__init__.py", + "io/gfile/__init__.py", "graph_util/__init__.py", "image/__init__.py", "io/__init__.py", @@ -62,20 +62,17 @@ TENSORFLOW_API_INIT_FILES = [ "linalg/__init__.py", "lite/__init__.py", "lite/constants/__init__.py", - "logging/__init__.py", "losses/__init__.py", "math/__init__.py", "metrics/__init__.py", "nn/__init__.py", "nn/rnn_cell/__init__.py", - "profiler/__init__.py", "quantization/__init__.py", "random/__init__.py", "saved_model/__init__.py", "sets/__init__.py", "signal/__init__.py", "sparse/__init__.py", - "spectral/__init__.py", "strings/__init__.py", "summary/__init__.py", "sysconfig/__init__.py", diff --git a/tensorflow/python/tools/api/generator/api_init_files_v1.bzl b/tensorflow/python/tools/api/generator/api_init_files_v1.bzl index 89c817f6090..e35b9c43740 100644 --- a/tensorflow/python/tools/api/generator/api_init_files_v1.bzl +++ b/tensorflow/python/tools/api/generator/api_init_files_v1.bzl @@ -10,12 +10,14 @@ TENSORFLOW_API_INIT_FILES_V1 = [ "data/__init__.py", "data/experimental/__init__.py", "debugging/__init__.py", + "distribute/__init__.py", "distributions/__init__.py", "dtypes/__init__.py", "errors/__init__.py", "experimental/__init__.py", "feature_column/__init__.py", "gfile/__init__.py", + "io/gfile/__init__.py", "graph_util/__init__.py", "image/__init__.py", "io/__init__.py", diff --git a/tensorflow/python/tools/api/generator/create_python_api.py b/tensorflow/python/tools/api/generator/create_python_api.py index f6258034213..51c2bfba7c1 100644 --- a/tensorflow/python/tools/api/generator/create_python_api.py +++ b/tensorflow/python/tools/api/generator/create_python_api.py @@ -45,10 +45,10 @@ _GENERATED_FILE_HEADER = """# This file is MACHINE GENERATED! Do not edit. \"\"\"%s \"\"\" -from __future__ import print_function +from __future__ import print_function as _print_function """ -_GENERATED_FILE_FOOTER = '\n\ndel print_function\n' +_GENERATED_FILE_FOOTER = '\n\ndel _print_function\n' class SymbolExposedTwiceError(Exception): @@ -463,8 +463,9 @@ def create_api_files(output_files, packages, root_init_template, output_dir, raise ValueError( """Missing outputs for genrule:\n%s. Be sure to add these targets to tensorflow/python/tools/api/generator/api_init_files_v1.bzl and -tensorflow/python/tools/api/generator/api_init_files.bzl""" % ',\n'.join( - sorted(missing_output_files))) +tensorflow/python/tools/api/generator/api_init_files.bzl (tensorflow repo), or +tensorflow_estimator/python/estimator/api/api_gen.bzl (estimator repo)""" + % ',\n'.join(sorted(missing_output_files))) def main(): diff --git a/tensorflow/python/tools/api/generator/doc_srcs.py b/tensorflow/python/tools/api/generator/doc_srcs.py index 479d5006d1e..abb5886deb3 100644 --- a/tensorflow/python/tools/api/generator/doc_srcs.py +++ b/tensorflow/python/tools/api/generator/doc_srcs.py @@ -35,10 +35,11 @@ DocSource.__new__.__defaults__ = (None,) * len(DocSource._fields) _TENSORFLOW_DOC_SOURCES = { 'app': DocSource(docstring_module_name='platform.app'), + 'bitwise': DocSource(docstring_module_name='ops.bitwise_ops'), 'compat': DocSource(docstring_module_name='util.compat'), + 'distribute': DocSource(docstring_module_name='distribute.distribute_lib'), 'distributions': DocSource( docstring_module_name='ops.distributions.distributions'), - 'bitwise': DocSource(docstring_module_name='ops.bitwise_ops'), 'errors': DocSource(docstring_module_name='framework.errors'), 'gfile': DocSource(docstring_module_name='platform.gfile'), 'graph_util': DocSource(docstring_module_name='framework.graph_util'), @@ -56,9 +57,8 @@ _TENSORFLOW_DOC_SOURCES = { 'resource_loader': DocSource( docstring_module_name='platform.resource_loader'), 'sets': DocSource(docstring_module_name='ops.sets'), - 'signal': DocSource(docstring_module_name='ops.signal'), + 'signal': DocSource(docstring_module_name='ops.signal.signal'), 'sparse': DocSource(docstring_module_name='ops.sparse_ops'), - 'spectral': DocSource(docstring_module_name='ops.spectral_ops'), 'strings': DocSource(docstring_module_name='ops.string_ops'), 'sysconfig': DocSource(docstring_module_name='platform.sysconfig'), 'test': DocSource(docstring_module_name='platform.test'), diff --git a/tensorflow/python/tools/freeze_graph_test.py b/tensorflow/python/tools/freeze_graph_test.py index 5dc14a6961e..efdf7dd2cf1 100644 --- a/tensorflow/python/tools/freeze_graph_test.py +++ b/tensorflow/python/tools/freeze_graph_test.py @@ -161,9 +161,11 @@ class FreezeGraphTest(test_util.TensorFlowTestCase): },) builder.save(as_text=True) + @test_util.run_deprecated_v1 def testFreezeGraphV1(self): self._testFreezeGraph(saver_pb2.SaverDef.V1) + @test_util.run_deprecated_v1 def testFreezeGraphV2(self): self._testFreezeGraph(saver_pb2.SaverDef.V2) diff --git a/tensorflow/python/tools/inspect_checkpoint.py b/tensorflow/python/tools/inspect_checkpoint.py index 6504fbc1075..ea1f6aa5555 100644 --- a/tensorflow/python/tools/inspect_checkpoint.py +++ b/tensorflow/python/tools/inspect_checkpoint.py @@ -63,7 +63,7 @@ def print_tensors_in_checkpoint_file(file_name, tensor_name, all_tensors, print("It's likely that your checkpoint file has been compressed " "with SNAPPY.") if ("Data loss" in str(e) and - (any([e in file_name for e in [".index", ".meta", ".data"]]))): + any(e in file_name for e in [".index", ".meta", ".data"])): proposed_file = ".".join(file_name.split(".")[0:-1]) v2_file_error_template = """ It's likely that this is a V2 checkpoint and you need to provide the filename diff --git a/tensorflow/python/tools/optimize_for_inference_test.py b/tensorflow/python/tools/optimize_for_inference_test.py index 10bfb0dc706..310776ff1b0 100644 --- a/tensorflow/python/tools/optimize_for_inference_test.py +++ b/tensorflow/python/tools/optimize_for_inference_test.py @@ -128,6 +128,7 @@ class OptimizeForInferenceTest(test.TestCase): graph_def, [], [add_name], dtypes.float32.as_datatype_enum) self.assertProtoEquals(expected_output, output) + @test_util.run_deprecated_v1 def testFoldBatchNorms(self): with self.cached_session() as sess: inputs = [1, 4, 2, 5, 3, 6, -1, -4, -2, -5, -3, -6] @@ -171,6 +172,7 @@ class OptimizeForInferenceTest(test.TestCase): for node in optimized_graph_def.node: self.assertNotEqual("BatchNormWithGlobalNormalization", node.op) + @test_util.run_deprecated_v1 def testFoldFusedBatchNorms(self): for data_format, use_gpu in [("NHWC", False), ("NCHW", True)]: with self.cached_session(use_gpu=use_gpu) as sess: @@ -222,6 +224,7 @@ class OptimizeForInferenceTest(test.TestCase): for node in optimized_graph_def.node: self.assertNotEqual("FusedBatchNorm", node.op) + @test_util.run_deprecated_v1 def testFuseResizePadAndConv(self): with self.cached_session() as sess: inputs = [1, 4, 2, 5, 3, 6, -1, -4, -2, -5, -3, -6] @@ -253,6 +256,7 @@ class OptimizeForInferenceTest(test.TestCase): self.assertNotEqual("MirrorPad", node.op) self.assertNotEqual("ResizeBilinear", node.op) + @test_util.run_deprecated_v1 def testFuseResizeAndConv(self): with self.cached_session() as sess: inputs = [1, 4, 2, 5, 3, 6, -1, -4, -2, -5, -3, -6] @@ -282,6 +286,7 @@ class OptimizeForInferenceTest(test.TestCase): self.assertNotEqual("MirrorPad", node.op) + @test_util.run_deprecated_v1 def testFusePadAndConv(self): with self.cached_session() as sess: inputs = [1, 4, 2, 5, 3, 6, -1, -4, -2, -5, -3, -6] diff --git a/tensorflow/python/tools/strip_unused_test.py b/tensorflow/python/tools/strip_unused_test.py index 7cf0c3e3ed9..e906ff94ba8 100644 --- a/tensorflow/python/tools/strip_unused_test.py +++ b/tensorflow/python/tools/strip_unused_test.py @@ -50,7 +50,7 @@ class StripUnusedTest(test_util.TensorFlowTestCase): wanted_input_node, 2.0, name="output_node") math_ops.add(output_node, 2.0, name="later_node") sess = session.Session() - output = sess.run(output_node) + output = self.evaluate(output_node) self.assertNear(-4.0, output, 0.00001) graph_io.write_graph(sess.graph, self.get_temp_dir(), input_graph_name) @@ -113,7 +113,7 @@ class StripUnusedTest(test_util.TensorFlowTestCase): input_node1, input_node2, name="output_node") math_ops.add(output_node, 2.0, name="later_node") sess = session.Session() - output = sess.run(output_node) + output = self.evaluate(output_node) self.assertNear(6.0, output, 0.00001) graph_io.write_graph(sess.graph, self.get_temp_dir(), input_graph_name) diff --git a/tensorflow/python/training/adadelta.py b/tensorflow/python/training/adadelta.py index 95eca764969..dd210160004 100644 --- a/tensorflow/python/training/adadelta.py +++ b/tensorflow/python/training/adadelta.py @@ -25,7 +25,7 @@ from tensorflow.python.training import training_ops from tensorflow.python.util.tf_export import tf_export -@tf_export("train.AdadeltaOptimizer") +@tf_export(v1=["train.AdadeltaOptimizer"]) class AdadeltaOptimizer(optimizer.Optimizer): """Optimizer that implements the Adadelta algorithm. diff --git a/tensorflow/python/training/adadelta_test.py b/tensorflow/python/training/adadelta_test.py index a14ac895ac0..0e5af5a9222 100644 --- a/tensorflow/python/training/adadelta_test.py +++ b/tensorflow/python/training/adadelta_test.py @@ -166,6 +166,7 @@ class AdadeltaOptimizerTest(test.TestCase): with context.eager_mode(): self.doTestBasic(use_resource=True, use_callable_params=True) + @test_util.run_deprecated_v1 def testMinimizeSparseResourceVariable(self): for dtype in [dtypes.half, dtypes.float32, dtypes.float64]: with self.cached_session(): @@ -177,12 +178,11 @@ class AdadeltaOptimizerTest(test.TestCase): 1.0, 1.0, 1.0).minimize(loss) variables.global_variables_initializer().run() # Fetch params to validate initial values - self.assertAllCloseAccordingToType([[1.0, 2.0]], var0.eval()) + self.assertAllCloseAccordingToType([[1.0, 2.0]], self.evaluate(var0)) # Run 1 step of sgd sgd_op.run() # Validate updated params - self.assertAllCloseAccordingToType( - [[-111, -138]], var0.eval()) + self.assertAllCloseAccordingToType([[-111, -138]], self.evaluate(var0)) if __name__ == "__main__": diff --git a/tensorflow/python/training/adagrad.py b/tensorflow/python/training/adagrad.py index cc0da26b279..10c043bae17 100644 --- a/tensorflow/python/training/adagrad.py +++ b/tensorflow/python/training/adagrad.py @@ -28,7 +28,7 @@ from tensorflow.python.training import training_ops from tensorflow.python.util.tf_export import tf_export -@tf_export("train.AdagradOptimizer") +@tf_export(v1=["train.AdagradOptimizer"]) class AdagradOptimizer(optimizer.Optimizer): """Optimizer that implements the Adagrad algorithm. diff --git a/tensorflow/python/training/adagrad_da.py b/tensorflow/python/training/adagrad_da.py index 5ba403554f5..e23b7134b3b 100644 --- a/tensorflow/python/training/adagrad_da.py +++ b/tensorflow/python/training/adagrad_da.py @@ -26,7 +26,7 @@ from tensorflow.python.training import training_ops from tensorflow.python.util.tf_export import tf_export -@tf_export("train.AdagradDAOptimizer") +@tf_export(v1=["train.AdagradDAOptimizer"]) class AdagradDAOptimizer(optimizer.Optimizer): """Adagrad Dual Averaging algorithm for sparse linear models. diff --git a/tensorflow/python/training/adagrad_da_test.py b/tensorflow/python/training/adagrad_da_test.py index 00801be3b4d..aacfe6faf4e 100644 --- a/tensorflow/python/training/adagrad_da_test.py +++ b/tensorflow/python/training/adagrad_da_test.py @@ -22,6 +22,7 @@ import numpy as np from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes +from tensorflow.python.framework import test_util from tensorflow.python.ops import embedding_ops from tensorflow.python.ops import math_ops from tensorflow.python.ops import resource_variable_ops @@ -54,14 +55,14 @@ class AdagradDAOptimizerTest(test.TestCase): zip([grads0, grads1], [var0, var1]), global_step=global_step) variables.global_variables_initializer().run() - v0_val, v1_val = sess.run([var0, var1]) + v0_val, v1_val = self.evaluate([var0, var1]) self.assertAllClose([0.0, 0.0], v0_val) self.assertAllClose([0.0, 0.0], v1_val) # Run a step of AdagradDA update.run() - v0_val, v1_val = sess.run([var0, var1]) + v0_val, v1_val = self.evaluate([var0, var1]) # Let g to be gradient accumulator, gg to be gradient squared # accumulator, T be the global step, lr is the learning rate, and k the # initial gradient squared accumulator value. @@ -73,12 +74,15 @@ class AdagradDAOptimizerTest(test.TestCase): self.assertAllCloseAccordingToType( np.array([-0.094821, -0.189358]), v1_val) + @test_util.run_deprecated_v1 def testAdagradDAWithoutRegularizationBasic1(self): self.doTestAdagradDAwithoutRegularizationBasic1() + @test_util.run_deprecated_v1 def testResourceAdagradDAWithoutRegularizationBasic1(self): self.doTestAdagradDAwithoutRegularizationBasic1(use_resource=True) + @test_util.run_deprecated_v1 def testMinimizeSparseResourceVariable(self): for dtype in [dtypes.float32, dtypes.float64]: with self.cached_session(): @@ -92,13 +96,15 @@ class AdagradDAOptimizerTest(test.TestCase): 1.0, global_step).minimize(loss) variables.global_variables_initializer().run() # Fetch params to validate initial values - self.assertAllCloseAccordingToType([[1.0, 2.0]], var0.eval()) + self.assertAllCloseAccordingToType([[1.0, 2.0]], self.evaluate(var0)) # Run 1 step of sgd sgd_op.run() # Validate updated params - self.assertAllCloseAccordingToType( - [[-1, -1]], var0.eval(), rtol=0.01) + self.assertAllCloseAccordingToType([[-1, -1]], + self.evaluate(var0), + rtol=0.01) + @test_util.run_deprecated_v1 def testAdagradDAwithoutRegularizationBasic2(self): for dtype in [dtypes.float64, dtypes.float32]: with self.cached_session() as sess: @@ -118,19 +124,20 @@ class AdagradDAOptimizerTest(test.TestCase): zip([grads0, grads1], [var0, var1]), global_step=global_step) variables.global_variables_initializer().run() - v0_val, v1_val = sess.run([var0, var1]) + v0_val, v1_val = self.evaluate([var0, var1]) self.assertAllCloseAccordingToType([1.0, 2.0], v0_val) self.assertAllCloseAccordingToType([4.0, 3.0], v1_val) # Run a step of AdagradDA update.run() - v0_val, v1_val = sess.run([var0, var1]) + v0_val, v1_val = self.evaluate([var0, var1]) self.assertAllCloseAccordingToType( np.array([-0.904534, -1.603567]), v0_val) self.assertAllCloseAccordingToType( np.array([-0.094821, -0.189358]), v1_val) + @test_util.run_deprecated_v1 def testAdagradDAWithL1(self): for dtype in [dtypes.float64, dtypes.float32]: with self.cached_session() as sess: @@ -150,19 +157,20 @@ class AdagradDAOptimizerTest(test.TestCase): zip([grads0, grads1], [var0, var1]), global_step=global_step) variables.global_variables_initializer().run() - v0_val, v1_val = sess.run([var0, var1]) + v0_val, v1_val = self.evaluate([var0, var1]) self.assertAllCloseAccordingToType([1.0, 2.0], v0_val) self.assertAllCloseAccordingToType([4.0, 3.0], v1_val) # Run a step of AdagradDA update.run() - v0_val, v1_val = sess.run([var0, var1]) + v0_val, v1_val = self.evaluate([var0, var1]) self.assertAllCloseAccordingToType( np.array([-0.895489, -1.59555]), v0_val) self.assertAllCloseAccordingToType( np.array([-0.085339, -0.17989]), v1_val) + @test_util.run_deprecated_v1 def testAdagradDAWithL1_L2(self): for dtype in [dtypes.float64, dtypes.float32]: with self.cached_session() as sess: @@ -182,14 +190,14 @@ class AdagradDAOptimizerTest(test.TestCase): zip([grads0, grads1], [var0, var1]), global_step=global_step) variables.global_variables_initializer().run() - v0_val, v1_val = sess.run([var0, var1]) + v0_val, v1_val = self.evaluate([var0, var1]) self.assertAllCloseAccordingToType([1.0, 2.0], v0_val) self.assertAllCloseAccordingToType([4.0, 3.0], v1_val) # Run a step of AdagradDA update.run() - v0_val, v1_val = sess.run([var0, var1]) + v0_val, v1_val = self.evaluate([var0, var1]) self.assertAllCloseAccordingToType( np.array([-0.046907, -0.093659]), v0_val) self.assertAllCloseAccordingToType( diff --git a/tensorflow/python/training/adagrad_test.py b/tensorflow/python/training/adagrad_test.py index 7caf01f64d5..da26fcdb7f6 100644 --- a/tensorflow/python/training/adagrad_test.py +++ b/tensorflow/python/training/adagrad_test.py @@ -96,6 +96,7 @@ class AdagradOptimizerTest(test.TestCase): def testBasicLocked(self): self.doTestBasic(use_locking=True) + @test_util.run_deprecated_v1 def testMinimizeSparseResourceVariable(self): for dtype in [dtypes.half, dtypes.float32, dtypes.float64]: with self.cached_session(): @@ -107,14 +108,16 @@ class AdagradOptimizerTest(test.TestCase): sgd_op = adagrad.AdagradOptimizer(1.0).minimize(loss) variables.global_variables_initializer().run() # Fetch params to validate initial values - self.assertAllCloseAccordingToType( - [[1.0, 2.0], [3.0, 4.0]], var0.eval()) + self.assertAllCloseAccordingToType([[1.0, 2.0], [3.0, 4.0]], + self.evaluate(var0)) # Run 1 step of sgd sgd_op.run() # Validate updated params - self.assertAllCloseAccordingToType( - [[0, 1], [3, 4]], var0.eval(), atol=0.01) + self.assertAllCloseAccordingToType([[0, 1], [3, 4]], + self.evaluate(var0), + atol=0.01) + @test_util.run_deprecated_v1 def testTensorLearningRate(self): for dtype in [dtypes.half, dtypes.float32, dtypes.float64]: with self.cached_session(): @@ -128,17 +131,20 @@ class AdagradOptimizerTest(test.TestCase): zip([grads0, grads1], [var0, var1])) variables.global_variables_initializer().run() # Fetch params to validate initial values - self.assertAllClose([1.0, 2.0], var0.eval()) - self.assertAllClose([3.0, 4.0], var1.eval()) + self.assertAllClose([1.0, 2.0], self.evaluate(var0)) + self.assertAllClose([3.0, 4.0], self.evaluate(var1)) # Run 3 steps of adagrad for _ in range(3): ada_update.run() # Validate updated params self.assertAllCloseAccordingToType( - np.array([-1.6026098728179932, -0.6026098728179932]), var0.eval()) + np.array([-1.6026098728179932, -0.6026098728179932]), + self.evaluate(var0)) self.assertAllCloseAccordingToType( - np.array([2.715679168701172, 3.715679168701172]), var1.eval()) + np.array([2.715679168701172, 3.715679168701172]), + self.evaluate(var1)) + @test_util.run_deprecated_v1 def testSparseBasic(self): for dtype in [dtypes.half, dtypes.float32, dtypes.float64]: with self.cached_session(): @@ -159,17 +165,18 @@ class AdagradOptimizerTest(test.TestCase): zip([grads0, grads1], [var0, var1])) variables.global_variables_initializer().run() # Fetch params to validate initial values - self.assertAllClose([[1.0], [2.0]], var0.eval()) - self.assertAllClose([[3.0], [4.0]], var1.eval()) + self.assertAllClose([[1.0], [2.0]], self.evaluate(var0)) + self.assertAllClose([[3.0], [4.0]], self.evaluate(var1)) # Run 3 step of sgd for _ in range(3): ada_update.run() # Validate updated params self.assertAllCloseAccordingToType( - np.array([[-1.6026098728179932], [2.0]]), var0.eval()) + np.array([[-1.6026098728179932], [2.0]]), self.evaluate(var0)) self.assertAllCloseAccordingToType( - np.array([[3.0], [3.715679168701172]]), var1.eval()) + np.array([[3.0], [3.715679168701172]]), self.evaluate(var1)) + @test_util.run_deprecated_v1 def testSparseRepeatedIndices(self): for dtype in [dtypes.half, dtypes.float32, dtypes.float64]: with self.cached_session(): @@ -193,13 +200,14 @@ class AdagradOptimizerTest(test.TestCase): [(grad_aggregated, aggregated_update_var)]) variables.global_variables_initializer().run() self.assertAllClose(aggregated_update_var.eval(), - repeated_index_update_var.eval()) + self.evaluate(repeated_index_update_var)) for _ in range(3): repeated_update.run() aggregated_update.run() self.assertAllClose(aggregated_update_var.eval(), - repeated_index_update_var.eval()) + self.evaluate(repeated_index_update_var)) + @test_util.run_deprecated_v1 def testSparseRepeatedIndicesResourceVariable(self): for dtype in [dtypes.half, dtypes.float32, dtypes.float64]: with self.cached_session(): @@ -217,13 +225,14 @@ class AdagradOptimizerTest(test.TestCase): 2.0).minimize(loss_aggregated) variables.global_variables_initializer().run() self.assertAllCloseAccordingToType( - var_repeated.eval(), var_aggregated.eval()) + self.evaluate(var_repeated), self.evaluate(var_aggregated)) for _ in range(3): update_op_repeated.run() update_op_aggregated.run() self.assertAllCloseAccordingToType( - var_repeated.eval(), var_aggregated.eval()) + self.evaluate(var_repeated), self.evaluate(var_aggregated)) + @test_util.run_deprecated_v1 def testSparseStability(self): for dtype in [dtypes.half, dtypes.float32, dtypes.float64]: with self.cached_session(): @@ -253,13 +262,14 @@ class AdagradOptimizerTest(test.TestCase): init.run() ada_update.run() self.assertAllCloseAccordingToType( - np.array([[0.1, 0.1, 0.1, 0.1, 0.1, 0.1]]), slot0.eval()) + np.array([[0.1, 0.1, 0.1, 0.1, 0.1, 0.1]]), self.evaluate(slot0)) self.assertAllCloseAccordingToType( np.array([[ 0.00891194, -0.10712013, 0.11047515, 0.22636929, -0.0144573, -0.01029443 - ]]), var0.eval()) + ]]), self.evaluate(var0)) + @test_util.run_deprecated_v1 def testSharing(self): for dtype in [dtypes.half, dtypes.float32, dtypes.float64]: with self.cached_session(): @@ -282,18 +292,21 @@ class AdagradOptimizerTest(test.TestCase): variables.global_variables_initializer().run() # Fetch params to validate initial values. - self.assertAllClose([1.0, 2.0], var0.eval()) - self.assertAllClose([3.0, 4.0], var1.eval()) + self.assertAllClose([1.0, 2.0], self.evaluate(var0)) + self.assertAllClose([3.0, 4.0], self.evaluate(var1)) # Mix the first and the second adagrad for 3 steps. ada_update1.run() ada_update2.run() ada_update1.run() # Validate updated params (the same as with only 1 Adagrad). self.assertAllCloseAccordingToType( - np.array([-1.6026098728179932, -0.6026098728179932]), var0.eval()) + np.array([-1.6026098728179932, -0.6026098728179932]), + self.evaluate(var0)) self.assertAllCloseAccordingToType( - np.array([2.715679168701172, 3.715679168701172]), var1.eval()) + np.array([2.715679168701172, 3.715679168701172]), + self.evaluate(var1)) + @test_util.run_deprecated_v1 def testDynamicShapeVariable_Ok(self): with self.cached_session(): v = variable_scope.get_variable("v", initializer=constant_op.constant(1.), @@ -302,6 +315,7 @@ class AdagradOptimizerTest(test.TestCase): # Creating optimizer should cause no exception. adagrad.AdagradOptimizer(3.0, initial_accumulator_value=0.1) + @test_util.run_deprecated_v1 def testDynamicShapeVariableWithCallableInit(self): var0 = variable_scope.get_variable("var0", initializer=constant_op.constant(1.), diff --git a/tensorflow/python/training/adam.py b/tensorflow/python/training/adam.py index 704ad6d3fe8..0c701f47122 100644 --- a/tensorflow/python/training/adam.py +++ b/tensorflow/python/training/adam.py @@ -29,7 +29,7 @@ from tensorflow.python.training import training_ops from tensorflow.python.util.tf_export import tf_export -@tf_export("train.AdamOptimizer") +@tf_export(v1=["train.AdamOptimizer"]) class AdamOptimizer(optimizer.Optimizer): """Optimizer that implements the Adam algorithm. diff --git a/tensorflow/python/training/adam_test.py b/tensorflow/python/training/adam_test.py index 0d42cc7b9c6..b0bae275773 100644 --- a/tensorflow/python/training/adam_test.py +++ b/tensorflow/python/training/adam_test.py @@ -83,30 +83,34 @@ class AdamOptimizerTest(test.TestCase): variables.global_variables_initializer().run() # Fetch params to validate initial values - self.assertAllClose([1.0, 2.0], var0.eval()) - self.assertAllClose([3.0, 4.0], var1.eval()) + self.assertAllClose([1.0, 2.0], self.evaluate(var0)) + self.assertAllClose([3.0, 4.0], self.evaluate(var1)) beta1_power, beta2_power = opt._get_beta_accumulators() # Run 3 steps of Adam for t in range(1, 4): - self.assertAllCloseAccordingToType(0.9**t, beta1_power.eval()) - self.assertAllCloseAccordingToType(0.999**t, beta2_power.eval()) + self.assertAllCloseAccordingToType(0.9**t, self.evaluate(beta1_power)) + self.assertAllCloseAccordingToType(0.999**t, + self.evaluate(beta2_power)) update.run() var0_np, m0, v0 = adam_update_numpy(var0_np, grads0_np, t, m0, v0) var1_np, m1, v1 = adam_update_numpy(var1_np, grads1_np, t, m1, v1) # Validate updated params - self.assertAllCloseAccordingToType(var0_np, var0.eval()) - self.assertAllCloseAccordingToType(var1_np, var1.eval()) + self.assertAllCloseAccordingToType(var0_np, self.evaluate(var0)) + self.assertAllCloseAccordingToType(var1_np, self.evaluate(var1)) + @test_util.run_deprecated_v1 def testSparse(self): self.doTestSparse(use_resource=False) + @test_util.run_deprecated_v1 def testResourceSparse(self): self.doTestSparse(use_resource=True) + @test_util.run_deprecated_v1 def testSparseDevicePlacement(self): for index_dtype in [dtypes.int32, dtypes.int64]: with self.cached_session(force_gpu=test.is_gpu_available()): @@ -120,6 +124,7 @@ class AdamOptimizerTest(test.TestCase): variables.global_variables_initializer().run() minimize_op.run() + @test_util.run_deprecated_v1 def testSparseRepeatedIndices(self): for dtype in [dtypes.half, dtypes.float32, dtypes.float64]: with self.cached_session(): @@ -143,12 +148,12 @@ class AdamOptimizerTest(test.TestCase): [(grad_aggregated, aggregated_update_var)]) variables.global_variables_initializer().run() self.assertAllClose(aggregated_update_var.eval(), - repeated_index_update_var.eval()) + self.evaluate(repeated_index_update_var)) for _ in range(3): repeated_update.run() aggregated_update.run() self.assertAllClose(aggregated_update_var.eval(), - repeated_index_update_var.eval()) + self.evaluate(repeated_index_update_var)) def doTestBasic(self, use_resource=False, use_callable_params=False): for i, dtype in enumerate([dtypes.half, dtypes.float32, dtypes.float64]): @@ -235,6 +240,7 @@ class AdamOptimizerTest(test.TestCase): with context.eager_mode(): self.doTestBasic(use_resource=True, use_callable_params=True) + @test_util.run_deprecated_v1 def testTensorLearningRate(self): for dtype in [dtypes.half, dtypes.float32, dtypes.float64]: with self.cached_session(): @@ -254,24 +260,26 @@ class AdamOptimizerTest(test.TestCase): variables.global_variables_initializer().run() # Fetch params to validate initial values - self.assertAllClose([1.0, 2.0], var0.eval()) - self.assertAllClose([3.0, 4.0], var1.eval()) + self.assertAllClose([1.0, 2.0], self.evaluate(var0)) + self.assertAllClose([3.0, 4.0], self.evaluate(var1)) beta1_power, beta2_power = opt._get_beta_accumulators() # Run 3 steps of Adam for t in range(1, 4): - self.assertAllCloseAccordingToType(0.9**t, beta1_power.eval()) - self.assertAllCloseAccordingToType(0.999**t, beta2_power.eval()) + self.assertAllCloseAccordingToType(0.9**t, self.evaluate(beta1_power)) + self.assertAllCloseAccordingToType(0.999**t, + self.evaluate(beta2_power)) update.run() var0_np, m0, v0 = adam_update_numpy(var0_np, grads0_np, t, m0, v0) var1_np, m1, v1 = adam_update_numpy(var1_np, grads1_np, t, m1, v1) # Validate updated params - self.assertAllCloseAccordingToType(var0_np, var0.eval()) - self.assertAllCloseAccordingToType(var1_np, var1.eval()) + self.assertAllCloseAccordingToType(var0_np, self.evaluate(var0)) + self.assertAllCloseAccordingToType(var1_np, self.evaluate(var1)) + @test_util.run_deprecated_v1 def testSharing(self): for dtype in [dtypes.half, dtypes.float32, dtypes.float64]: with self.cached_session(): @@ -294,13 +302,14 @@ class AdamOptimizerTest(test.TestCase): beta1_power, beta2_power = opt._get_beta_accumulators() # Fetch params to validate initial values - self.assertAllClose([1.0, 2.0], var0.eval()) - self.assertAllClose([3.0, 4.0], var1.eval()) + self.assertAllClose([1.0, 2.0], self.evaluate(var0)) + self.assertAllClose([3.0, 4.0], self.evaluate(var1)) # Run 3 steps of intertwined Adam1 and Adam2. for t in range(1, 4): - self.assertAllCloseAccordingToType(0.9**t, beta1_power.eval()) - self.assertAllCloseAccordingToType(0.999**t, beta2_power.eval()) + self.assertAllCloseAccordingToType(0.9**t, self.evaluate(beta1_power)) + self.assertAllCloseAccordingToType(0.999**t, + self.evaluate(beta2_power)) if t % 2 == 0: update1.run() else: @@ -310,8 +319,8 @@ class AdamOptimizerTest(test.TestCase): var1_np, m1, v1 = adam_update_numpy(var1_np, grads1_np, t, m1, v1) # Validate updated params - self.assertAllCloseAccordingToType(var0_np, var0.eval()) - self.assertAllCloseAccordingToType(var1_np, var1.eval()) + self.assertAllCloseAccordingToType(var0_np, self.evaluate(var0)) + self.assertAllCloseAccordingToType(var1_np, self.evaluate(var1)) def testTwoSessions(self): optimizer = adam.AdamOptimizer() diff --git a/tensorflow/python/training/basic_loops_test.py b/tensorflow/python/training/basic_loops_test.py index 5f5718e64a6..511a8334d56 100644 --- a/tensorflow/python/training/basic_loops_test.py +++ b/tensorflow/python/training/basic_loops_test.py @@ -23,6 +23,7 @@ import shutil from tensorflow.python.framework import errors_impl from tensorflow.python.framework import ops +from tensorflow.python.framework import test_util from tensorflow.python.platform import test from tensorflow.python.training import basic_loops from tensorflow.python.training import supervisor @@ -37,6 +38,7 @@ def _test_dir(test_name): class BasicTrainLoopTest(test.TestCase): + @test_util.run_deprecated_v1 def testBasicTrainLoop(self): logdir = _test_dir("basic_train_loop") sv = supervisor.Supervisor(logdir=logdir) @@ -55,6 +57,7 @@ class BasicTrainLoopTest(test.TestCase): sv, train_fn, args=(sv, "y"), kwargs={"a": "A"}) self.assertEqual(3, num_calls[0]) + @test_util.run_deprecated_v1 def testBasicTrainLoopExceptionAborts(self): logdir = _test_dir("basic_train_loop_exception_aborts") sv = supervisor.Supervisor(logdir=logdir) @@ -71,6 +74,7 @@ class BasicTrainLoopTest(test.TestCase): with self.assertRaisesRegexp(RuntimeError, "Failed"): basic_loops.basic_train_loop(sv, train_fn) + @test_util.run_deprecated_v1 def testBasicTrainLoopRetryOnAborted(self): logdir = _test_dir("basic_train_loop_exception_aborts") sv = supervisor.Supervisor(logdir=logdir) diff --git a/tensorflow/python/training/basic_session_run_hooks.py b/tensorflow/python/training/basic_session_run_hooks.py index 1efabcd854d..b64c7ada62a 100644 --- a/tensorflow/python/training/basic_session_run_hooks.py +++ b/tensorflow/python/training/basic_session_run_hooks.py @@ -83,7 +83,7 @@ class _HookTimer(object): raise NotImplementedError -@tf_export("train.SecondOrStepTimer") +@tf_export(v1=["train.SecondOrStepTimer"]) class SecondOrStepTimer(_HookTimer): """Timer that triggers at most once every N seconds or once every N steps. """ @@ -429,7 +429,7 @@ class StopAtStepHook(session_run_hook.SessionRunHook): run_context.request_stop() -@tf_export("train.CheckpointSaverListener") +@tf_export(v1=["train.CheckpointSaverListener"]) class CheckpointSaverListener(object): """Interface for listeners that take action before or after checkpoint save. @@ -718,7 +718,7 @@ class StepCounterHook(session_run_hook.SessionRunHook): self._last_global_step = stale_global_step -@tf_export("train.NanLossDuringTrainingError") +@tf_export(v1=["train.NanLossDuringTrainingError"]) class NanLossDuringTrainingError(RuntimeError): def __str__(self): @@ -976,7 +976,7 @@ class FeedFnHook(session_run_hook.SessionRunHook): fetches=None, feed_dict=self.feed_fn()) -@tf_export("train.ProfilerHook") +@tf_export(v1=["train.ProfilerHook"]) class ProfilerHook(session_run_hook.SessionRunHook): """Captures CPU/GPU profiling information every N steps or seconds. diff --git a/tensorflow/python/training/basic_session_run_hooks_test.py b/tensorflow/python/training/basic_session_run_hooks_test.py index 2d469634e0e..08942c5bb6e 100644 --- a/tensorflow/python/training/basic_session_run_hooks_test.py +++ b/tensorflow/python/training/basic_session_run_hooks_test.py @@ -22,7 +22,6 @@ from __future__ import print_function import os.path import shutil import tempfile -import threading import time from tensorflow.contrib.framework.python.framework import checkpoint_utils @@ -35,6 +34,7 @@ from tensorflow.python.framework import dtypes from tensorflow.python.framework import errors from tensorflow.python.framework import meta_graph from tensorflow.python.framework import ops +from tensorflow.python.framework import test_util from tensorflow.python.ops import array_ops from tensorflow.python.ops import control_flow_ops from tensorflow.python.ops import state_ops @@ -52,6 +52,11 @@ from tensorflow.python.training import session_run_hook from tensorflow.python.training import training_util +# Provide a realistic start time for unit tests where we need to mock out +# calls to time.time(). +MOCK_START_TIME = 1484695987.209386 + + class MockCheckpointSaverListener( basic_session_run_hooks.CheckpointSaverListener): @@ -87,15 +92,19 @@ class MockCheckpointSaverListener( class SecondOrStepTimerTest(test.TestCase): + @test_util.run_deprecated_v1 def test_raise_in_both_secs_and_steps(self): with self.assertRaises(ValueError): basic_session_run_hooks.SecondOrStepTimer(every_secs=2.0, every_steps=10) + @test_util.run_deprecated_v1 def test_raise_in_none_secs_and_steps(self): with self.assertRaises(ValueError): basic_session_run_hooks.SecondOrStepTimer() - def test_every_secs(self): + @test.mock.patch.object(time, 'time') + def test_every_secs(self, mock_time): + mock_time.return_value = MOCK_START_TIME timer = basic_session_run_hooks.SecondOrStepTimer(every_secs=1.0) self.assertTrue(timer.should_trigger_for_step(1)) @@ -103,7 +112,7 @@ class SecondOrStepTimerTest(test.TestCase): self.assertFalse(timer.should_trigger_for_step(1)) self.assertFalse(timer.should_trigger_for_step(2)) - time.sleep(1.0) + mock_time.return_value += 1.0 self.assertFalse(timer.should_trigger_for_step(1)) self.assertTrue(timer.should_trigger_for_step(2)) @@ -243,7 +252,7 @@ class LoggingTensorHookTest(test.TestCase): tensors=[t.name], at_end=True) hook.begin() mon_sess = monitored_session._HookedSession(sess, [hook]) - sess.run(variables_lib.global_variables_initializer()) + self.evaluate(variables_lib.global_variables_initializer()) self.logged_message = '' for _ in range(3): mon_sess.run(train_op) @@ -261,7 +270,7 @@ class LoggingTensorHookTest(test.TestCase): tensors=[t.name], every_n_iter=10, at_end=at_end) hook.begin() mon_sess = monitored_session._HookedSession(sess, [hook]) - sess.run(variables_lib.global_variables_initializer()) + self.evaluate(variables_lib.global_variables_initializer()) mon_sess.run(train_op) self.assertRegexpMatches(str(self.logged_message), t.name) for _ in range(3): @@ -308,13 +317,13 @@ class LoggingTensorHookTest(test.TestCase): tensors={'foo': t}, every_n_iter=1) hook.begin() mon_sess = monitored_session._HookedSession(sess, [hook]) - sess.run(variables_lib.global_variables_initializer()) + self.evaluate(variables_lib.global_variables_initializer()) mon_sess.run(train_op) self.assertRegexpMatches(str(self.logged_message), 'foo') # in first run, elapsed time is None. self.assertEqual(str(self.logged_message).find('sec'), -1) - def _validate_print_every_n_secs(self, sess, at_end): + def _validate_print_every_n_secs(self, sess, at_end, mock_time): t = constant_op.constant(42.0, name='foo') train_op = constant_op.constant(3) @@ -322,7 +331,7 @@ class LoggingTensorHookTest(test.TestCase): tensors=[t.name], every_n_secs=1.0, at_end=at_end) hook.begin() mon_sess = monitored_session._HookedSession(sess, [hook]) - sess.run(variables_lib.global_variables_initializer()) + self.evaluate(variables_lib.global_variables_initializer()) mon_sess.run(train_op) self.assertRegexpMatches(str(self.logged_message), t.name) @@ -331,7 +340,7 @@ class LoggingTensorHookTest(test.TestCase): self.logged_message = '' mon_sess.run(train_op) self.assertEqual(str(self.logged_message).find(t.name), -1) - time.sleep(1.0) + mock_time.return_value += 1.0 self.logged_message = '' mon_sess.run(train_op) @@ -345,17 +354,21 @@ class LoggingTensorHookTest(test.TestCase): # assertNotRegexpMatches is not supported by python 3.1 and later self.assertEqual(str(self.logged_message).find(t.name), -1) - def test_print_every_n_secs(self): + @test.mock.patch.object(time, 'time') + def test_print_every_n_secs(self, mock_time): with ops.Graph().as_default(), session_lib.Session() as sess: - self._validate_print_every_n_secs(sess, at_end=False) + mock_time.return_value = MOCK_START_TIME + self._validate_print_every_n_secs(sess, at_end=False, mock_time=mock_time) # Verify proper reset. - self._validate_print_every_n_secs(sess, at_end=False) + self._validate_print_every_n_secs(sess, at_end=False, mock_time=mock_time) - def test_print_every_n_secs_and_end(self): + @test.mock.patch.object(time, 'time') + def test_print_every_n_secs_and_end(self, mock_time): with ops.Graph().as_default(), session_lib.Session() as sess: - self._validate_print_every_n_secs(sess, at_end=True) + mock_time.return_value = MOCK_START_TIME + self._validate_print_every_n_secs(sess, at_end=True, mock_time=mock_time) # Verify proper reset. - self._validate_print_every_n_secs(sess, at_end=True) + self._validate_print_every_n_secs(sess, at_end=True, mock_time=mock_time) def test_print_formatter(self): with ops.Graph().as_default(), session_lib.Session() as sess: @@ -366,7 +379,7 @@ class LoggingTensorHookTest(test.TestCase): formatter=lambda items: 'qqq=%s' % items[t.name]) hook.begin() mon_sess = monitored_session._HookedSession(sess, [hook]) - sess.run(variables_lib.global_variables_initializer()) + self.evaluate(variables_lib.global_variables_initializer()) mon_sess.run(train_op) self.assertEqual(self.logged_message[0], 'qqq=42.0') @@ -403,11 +416,13 @@ class CheckpointSaverHookTest(test.TestCase): basic_session_run_hooks.CheckpointSaverHook( self.model_dir, saver=self.scaffold.saver, scaffold=self.scaffold) + @test_util.run_deprecated_v1 def test_raise_in_both_secs_and_steps(self): with self.assertRaises(ValueError): basic_session_run_hooks.CheckpointSaverHook( self.model_dir, save_secs=10, save_steps=20) + @test_util.run_deprecated_v1 def test_raise_in_none_secs_and_steps(self): with self.assertRaises(ValueError): basic_session_run_hooks.CheckpointSaverHook(self.model_dir) @@ -562,11 +577,8 @@ class CheckpointSaverHookTest(test.TestCase): @test.mock.patch.object(time, 'time') def test_save_secs_saves_periodically(self, mock_time): - # Let's have a realistic start time - current_time = 1484695987.209386 - with self.graph.as_default(): - mock_time.return_value = current_time + mock_time.return_value = MOCK_START_TIME hook = basic_session_run_hooks.CheckpointSaverHook( self.model_dir, save_secs=2, scaffold=self.scaffold) hook.begin() @@ -576,10 +588,10 @@ class CheckpointSaverHookTest(test.TestCase): sess.run(self.scaffold.init_op) mon_sess = monitored_session._HookedSession(sess, [hook]) - mock_time.return_value = current_time + mock_time.return_value = MOCK_START_TIME mon_sess.run(self.train_op) # Saved. - mock_time.return_value = current_time + 0.5 + mock_time.return_value = MOCK_START_TIME + 0.5 mon_sess.run(self.train_op) # Not saved. self.assertEqual(1, @@ -587,13 +599,13 @@ class CheckpointSaverHookTest(test.TestCase): self.global_step.name)) # Simulate 2.5 seconds of sleep. - mock_time.return_value = current_time + 2.5 + mock_time.return_value = MOCK_START_TIME + 2.5 mon_sess.run(self.train_op) # Saved. - mock_time.return_value = current_time + 2.6 + mock_time.return_value = MOCK_START_TIME + 2.6 mon_sess.run(self.train_op) # Not saved. - mock_time.return_value = current_time + 2.7 + mock_time.return_value = MOCK_START_TIME + 2.7 mon_sess.run(self.train_op) # Not saved. self.assertEqual(3, @@ -601,7 +613,7 @@ class CheckpointSaverHookTest(test.TestCase): self.global_step.name)) # Simulate 7.5 more seconds of sleep (10 seconds from start. - mock_time.return_value = current_time + 10 + mock_time.return_value = MOCK_START_TIME + 10 mon_sess.run(self.train_op) # Saved. self.assertEqual(6, checkpoint_utils.load_variable(self.model_dir, @@ -609,11 +621,8 @@ class CheckpointSaverHookTest(test.TestCase): @test.mock.patch.object(time, 'time') def test_save_secs_calls_listeners_periodically(self, mock_time): - # Let's have a realistic start time - current_time = 1484695987.209386 - with self.graph.as_default(): - mock_time.return_value = current_time + mock_time.return_value = MOCK_START_TIME listener = MockCheckpointSaverListener() hook = basic_session_run_hooks.CheckpointSaverHook( self.model_dir, @@ -626,28 +635,28 @@ class CheckpointSaverHookTest(test.TestCase): sess.run(self.scaffold.init_op) mon_sess = monitored_session._HookedSession(sess, [hook]) - mock_time.return_value = current_time + 0.5 + mock_time.return_value = MOCK_START_TIME + 0.5 mon_sess.run(self.train_op) # hook runs here - mock_time.return_value = current_time + 0.5 + mock_time.return_value = MOCK_START_TIME + 0.5 mon_sess.run(self.train_op) - mock_time.return_value = current_time + 3.0 + mock_time.return_value = MOCK_START_TIME + 3.0 mon_sess.run(self.train_op) # hook runs here - mock_time.return_value = current_time + 3.5 + mock_time.return_value = MOCK_START_TIME + 3.5 mon_sess.run(self.train_op) - mock_time.return_value = current_time + 4.0 + mock_time.return_value = MOCK_START_TIME + 4.0 mon_sess.run(self.train_op) - mock_time.return_value = current_time + 6.5 + mock_time.return_value = MOCK_START_TIME + 6.5 mon_sess.run(self.train_op) # hook runs here - mock_time.return_value = current_time + 7.0 + mock_time.return_value = MOCK_START_TIME + 7.0 mon_sess.run(self.train_op) # hook won't run here, so it does at end - mock_time.return_value = current_time + 7.5 + mock_time.return_value = MOCK_START_TIME + 7.5 hook.end(sess) # hook runs here self.assertEqual({ 'begin': 1, @@ -913,7 +922,9 @@ class StepCounterHookTest(test.TestCase): def tearDown(self): shutil.rmtree(self.log_dir, ignore_errors=True) - def test_step_counter_every_n_steps(self): + @test.mock.patch.object(time, 'time') + def test_step_counter_every_n_steps(self, mock_time): + mock_time.return_value = MOCK_START_TIME with ops.Graph().as_default() as g, session_lib.Session() as sess: variables.get_or_create_global_step() train_op = training_util._increment_global_step(1) @@ -921,11 +932,11 @@ class StepCounterHookTest(test.TestCase): hook = basic_session_run_hooks.StepCounterHook( summary_writer=summary_writer, every_n_steps=10) hook.begin() - sess.run(variables_lib.global_variables_initializer()) + self.evaluate(variables_lib.global_variables_initializer()) mon_sess = monitored_session._HookedSession(sess, [hook]) with test.mock.patch.object(tf_logging, 'warning') as mock_log: for _ in range(30): - time.sleep(0.01) + mock_time.return_value += 0.01 mon_sess.run(train_op) # logging.warning should not be called. self.assertIsNone(mock_log.call_args) @@ -941,7 +952,9 @@ class StepCounterHookTest(test.TestCase): self.assertEqual('global_step/sec', summary_value.tag) self.assertGreater(summary_value.simple_value, 0) - def test_step_counter_every_n_secs(self): + @test.mock.patch.object(time, 'time') + def test_step_counter_every_n_secs(self, mock_time): + mock_time.return_value = MOCK_START_TIME with ops.Graph().as_default() as g, session_lib.Session() as sess: variables.get_or_create_global_step() train_op = training_util._increment_global_step(1) @@ -950,12 +963,12 @@ class StepCounterHookTest(test.TestCase): summary_writer=summary_writer, every_n_steps=None, every_n_secs=0.1) hook.begin() - sess.run(variables_lib.global_variables_initializer()) + self.evaluate(variables_lib.global_variables_initializer()) mon_sess = monitored_session._HookedSession(sess, [hook]) mon_sess.run(train_op) - time.sleep(0.2) + mock_time.return_value += 0.2 mon_sess.run(train_op) - time.sleep(0.2) + mock_time.return_value += 0.2 mon_sess.run(train_op) hook.end(sess) @@ -987,7 +1000,7 @@ class StepCounterHookTest(test.TestCase): summary_writer=summary_writer, every_n_steps=1, every_n_secs=None) hook.begin() - sess.run(variables_lib.global_variables_initializer()) + self.evaluate(variables_lib.global_variables_initializer()) mon_sess = monitored_session._HookedSession(sess, [hook]) mon_sess.run(train_op) mon_sess.run(train_op) @@ -1007,7 +1020,7 @@ class StepCounterHookTest(test.TestCase): with ops.Graph().as_default(), session_lib.Session() as sess: variables.get_or_create_global_step() train_op = training_util._increment_global_step(0) # keep same. - sess.run(variables_lib.global_variables_initializer()) + self.evaluate(variables_lib.global_variables_initializer()) hook = basic_session_run_hooks.StepCounterHook( every_n_steps=1, every_n_secs=None) hook.begin() @@ -1034,16 +1047,18 @@ class StepCounterHookTest(test.TestCase): summary_writer=self.summary_writer, every_n_steps=every_n_steps) self.hook._set_steps_per_run(steps_per_run) self.hook.begin() - sess.run(variables_lib.global_variables_initializer()) + self.evaluate(variables_lib.global_variables_initializer()) self.mon_sess = monitored_session._HookedSession(sess, [self.hook]) - def test_steps_per_run_less_than_every_n_steps(self): + @test.mock.patch.object(time, 'time') + def test_steps_per_run_less_than_every_n_steps(self, mock_time): + mock_time.return_value = MOCK_START_TIME with ops.Graph().as_default() as g, session_lib.Session() as sess: self._setup_steps_per_run_test(10, 5, g, sess) # Logs at 15, 25 for _ in range(5): - time.sleep(0.01) + mock_time.return_value += 0.01 self.mon_sess.run(self.train_op) self.hook.end(sess) @@ -1058,13 +1073,15 @@ class StepCounterHookTest(test.TestCase): self.assertEqual('global_step/sec', summary_value.tag) self.assertGreater(summary_value.simple_value, 0) - def test_steps_per_run_equal_every_n_steps(self): + @test.mock.patch.object(time, 'time') + def test_steps_per_run_equal_every_n_steps(self, mock_time): + mock_time.return_value = MOCK_START_TIME with ops.Graph().as_default() as g, session_lib.Session() as sess: self._setup_steps_per_run_test(5, 5, g, sess) # Logs at 10, 15, 20, 25 for _ in range(5): - time.sleep(0.01) + mock_time.return_value += 0.01 self.mon_sess.run(self.train_op) self.hook.end(sess) @@ -1080,13 +1097,15 @@ class StepCounterHookTest(test.TestCase): self.assertEqual('global_step/sec', summary_value.tag) self.assertGreater(summary_value.simple_value, 0) - def test_steps_per_run_greater_than_every_n_steps(self): + @test.mock.patch.object(time, 'time') + def test_steps_per_run_greater_than_every_n_steps(self, mock_time): + mock_time.return_value = MOCK_START_TIME with ops.Graph().as_default() as g, session_lib.Session() as sess: self._setup_steps_per_run_test(5, 10, g, sess) # Logs at 20, 30, 40, 50 for _ in range(5): - time.sleep(0.01) + mock_time.return_value += 0.01 self.mon_sess.run(self.train_op) self.hook.end(sess) @@ -1129,11 +1148,13 @@ class SummarySaverHookTest(test.TestCase): basic_session_run_hooks.SummarySaverHook( scaffold=monitored_session.Scaffold(), summary_op=self.summary_op) + @test_util.run_deprecated_v1 def test_raise_in_both_secs_and_steps(self): with self.assertRaises(ValueError): basic_session_run_hooks.SummarySaverHook( save_secs=10, save_steps=20, summary_writer=self.summary_writer) + @test_util.run_deprecated_v1 def test_raise_in_none_secs_and_steps(self): with self.assertRaises(ValueError): basic_session_run_hooks.SummarySaverHook( @@ -1147,7 +1168,7 @@ class SummarySaverHookTest(test.TestCase): with self.cached_session() as sess: hook.begin() - sess.run(variables_lib.global_variables_initializer()) + self.evaluate(variables_lib.global_variables_initializer()) mon_sess = monitored_session._HookedSession(sess, [hook]) for _ in range(30): mon_sess.run(self.train_op) @@ -1179,7 +1200,7 @@ class SummarySaverHookTest(test.TestCase): with self.cached_session() as sess: hook.begin() - sess.run(variables_lib.global_variables_initializer()) + self.evaluate(variables_lib.global_variables_initializer()) mon_sess = monitored_session._HookedSession(sess, [hook]) for _ in range(10): mon_sess.run(self.train_op) @@ -1199,7 +1220,9 @@ class SummarySaverHookTest(test.TestCase): }, }) - def test_save_secs_saving_once_every_step(self): + @test.mock.patch.object(time, 'time') + def test_save_secs_saving_once_every_step(self, mock_time): + mock_time.return_value = MOCK_START_TIME hook = basic_session_run_hooks.SummarySaverHook( save_secs=0.5, summary_writer=self.summary_writer, @@ -1207,11 +1230,11 @@ class SummarySaverHookTest(test.TestCase): with self.cached_session() as sess: hook.begin() - sess.run(variables_lib.global_variables_initializer()) + self.evaluate(variables_lib.global_variables_initializer()) mon_sess = monitored_session._HookedSession(sess, [hook]) for _ in range(4): mon_sess.run(self.train_op) - time.sleep(0.5) + mock_time.return_value += 0.5 hook.end(sess) self.summary_writer.assert_summaries( @@ -1242,7 +1265,7 @@ class SummarySaverHookTest(test.TestCase): with self.cached_session() as sess: hook.begin() - sess.run(variables_lib.global_variables_initializer()) + self.evaluate(variables_lib.global_variables_initializer()) mon_sess = monitored_session._HookedSession(sess, [hook]) for _ in range(8): mon_sess.run(self.train_op) @@ -1279,27 +1302,43 @@ class GlobalStepWaiterHookTest(test.TestCase): session_run_hook.SessionRunContext( original_args=None, session=sess)) - def test_wait_for_step(self): + @test.mock.patch.object(time, 'sleep') + def test_wait_for_step(self, mock_sleep): with ops.Graph().as_default(): gstep = variables.get_or_create_global_step() hook = basic_session_run_hooks.GlobalStepWaiterHook(wait_until_step=1000) hook.begin() + with session_lib.Session() as sess: - sess.run(variables_lib.global_variables_initializer()) - waiter = threading.Thread( - target=hook.before_run, - args=(session_run_hook.SessionRunContext( - original_args=None, session=sess),)) - waiter.daemon = True - waiter.start() - time.sleep(1.0) - self.assertTrue(waiter.is_alive()) - sess.run(state_ops.assign(gstep, 500)) - time.sleep(1.0) - self.assertTrue(waiter.is_alive()) - sess.run(state_ops.assign(gstep, 1100)) - time.sleep(1.2) - self.assertFalse(waiter.is_alive()) + # Mock out calls to time.sleep() to update the global step. + + class Context(object): + counter = 0 + + def mock_sleep_side_effect(seconds): + del seconds # argument is ignored + Context.counter += 1 + if Context.counter == 1: + # The first time sleep() is called, we update the global_step from + # 0 to 500. + sess.run(state_ops.assign(gstep, 500)) + elif Context.counter == 2: + # The second time sleep() is called, we update the global_step from + # 500 to 1100. + sess.run(state_ops.assign(gstep, 1100)) + else: + raise AssertionError( + 'Expected before_run() to terminate after the second call to ' + 'time.sleep()') + + mock_sleep.side_effect = mock_sleep_side_effect + + # Run the mocked-out interaction with the hook. + self.evaluate(variables_lib.global_variables_initializer()) + run_context = session_run_hook.SessionRunContext( + original_args=None, session=sess) + hook.before_run(run_context) + self.assertEqual(Context.counter, 2) class FinalOpsHookTest(test.TestCase): @@ -1333,7 +1372,7 @@ class FinalOpsHookTest(test.TestCase): def test_final_ops_triggers_out_of_range_error(self): with ops.Graph().as_default(): dataset = dataset_ops.Dataset.range(1) - iterator = dataset.make_one_shot_iterator() + iterator = dataset_ops.make_one_shot_iterator(dataset) read_ops = iterator.get_next() final_ops = read_ops @@ -1390,7 +1429,7 @@ class ResourceSummarySaverHookTest(test.TestCase): with self.cached_session() as sess: hook.begin() - sess.run(variables_lib.global_variables_initializer()) + self.evaluate(variables_lib.global_variables_initializer()) mon_sess = monitored_session._HookedSession(sess, [hook]) for _ in range(30): mon_sess.run(self.train_op) @@ -1446,10 +1485,12 @@ class ProfilerHookTest(test.TestCase): def _count_timeline_files(self): return len(gfile.Glob(self.filepattern)) + @test_util.run_deprecated_v1 def test_raise_in_both_secs_and_steps(self): with self.assertRaises(ValueError): basic_session_run_hooks.ProfilerHook(save_secs=10, save_steps=20) + @test_util.run_deprecated_v1 def test_raise_in_none_secs_and_steps(self): with self.assertRaises(ValueError): basic_session_run_hooks.ProfilerHook(save_secs=None, save_steps=None) @@ -1465,29 +1506,27 @@ class ProfilerHookTest(test.TestCase): @test.mock.patch.object(time, 'time') def test_save_secs_saves_periodically(self, mock_time): # Pick a fixed start time. - current_time = 1484863632. - with self.graph.as_default(): - mock_time.return_value = current_time + mock_time.return_value = MOCK_START_TIME hook = basic_session_run_hooks.ProfilerHook( save_secs=2, output_dir=self.output_dir) with monitored_session.SingularMonitoredSession(hooks=[hook]) as sess: sess.run(self.train_op) # Not saved. self.assertEqual(0, self._count_timeline_files()) # Simulate 2.5 seconds of sleep. - mock_time.return_value = current_time + 2.5 + mock_time.return_value = MOCK_START_TIME + 2.5 sess.run(self.train_op) # Saved. self.assertEqual(1, self._count_timeline_files()) # Pretend some small amount of time has passed. - mock_time.return_value = current_time + 2.6 + mock_time.return_value = MOCK_START_TIME + 2.6 sess.run(self.train_op) # Not saved. # Edge test just before we should save the timeline. - mock_time.return_value = current_time + 4.4 + mock_time.return_value = MOCK_START_TIME + 4.4 sess.run(self.train_op) # Not saved. self.assertEqual(1, self._count_timeline_files()) - mock_time.return_value = current_time + 4.5 + mock_time.return_value = MOCK_START_TIME + 4.5 sess.run(self.train_op) # Saved. self.assertEqual(2, self._count_timeline_files()) diff --git a/tensorflow/python/training/checkpoint_management_test.py b/tensorflow/python/training/checkpoint_management_test.py index 3a061bcb35c..8606ec4a206 100644 --- a/tensorflow/python/training/checkpoint_management_test.py +++ b/tensorflow/python/training/checkpoint_management_test.py @@ -62,6 +62,7 @@ class LatestCheckpointWithRelativePaths(test.TestCase): finally: shutil.rmtree(tempdir) + @test_util.run_deprecated_v1 def testNameCollision(self): # Make sure we have a clean directory to work in. with self.tempDir() as tempdir: @@ -99,6 +100,7 @@ class LatestCheckpointWithRelativePaths(test.TestCase): self.assertIsNotNone( checkpoint_management.latest_checkpoint(traindir)) + @test_util.run_deprecated_v1 def testRelativePath(self): # Make sure we have a clean directory to work in. with self.tempDir() as tempdir: @@ -123,9 +125,9 @@ class LatestCheckpointWithRelativePaths(test.TestCase): # Record a short training history. variables.global_variables_initializer().run() save.save(sess, filepath, global_step=0) - inc.eval() + self.evaluate(inc) save.save(sess, filepath, global_step=1) - inc.eval() + self.evaluate(inc) save.save(sess, filepath, global_step=2) with self.cached_session() as sess: @@ -270,6 +272,7 @@ class SaverUtilsTest(test.TestCase): def tearDown(self): gfile.DeleteRecursively(self._base_dir) + @test_util.run_deprecated_v1 def testCheckpointExists(self): for sharded in (False, True): for version in (saver_pb2.SaverDef.V2, saver_pb2.SaverDef.V1): @@ -288,6 +291,7 @@ class SaverUtilsTest(test.TestCase): ckpt_prefix = checkpoint_management.latest_checkpoint(self._base_dir) self.assertTrue(checkpoint_management.checkpoint_exists(ckpt_prefix)) + @test_util.run_deprecated_v1 def testGetCheckpointMtimes(self): prefixes = [] for version in (saver_pb2.SaverDef.V2, saver_pb2.SaverDef.V1): @@ -302,6 +306,7 @@ class SaverUtilsTest(test.TestCase): self.assertEqual(2, len(mtimes)) self.assertTrue(mtimes[1] >= mtimes[0]) + @test_util.run_deprecated_v1 def testRemoveCheckpoint(self): for sharded in (False, True): for version in (saver_pb2.SaverDef.V2, saver_pb2.SaverDef.V1): diff --git a/tensorflow/python/training/checkpoint_ops_test.py b/tensorflow/python/training/checkpoint_ops_test.py index dde84314977..21ad3df1c8f 100644 --- a/tensorflow/python/training/checkpoint_ops_test.py +++ b/tensorflow/python/training/checkpoint_ops_test.py @@ -47,7 +47,7 @@ class LoadAndRemapWrappersTest(test.TestCase): with variable_scope.variable_scope('some_scope'): variable_scope.get_variable(name='embeddings', shape=[5, 16], initializer=initializer) - sess.run(variables.global_variables_initializer()) + self.evaluate(variables.global_variables_initializer()) saver = saver_lib.Saver() saver.save(sess, checkpoint_prefix, global_step=5) self.checkpoint_file = '{}-5'.format(checkpoint_prefix) @@ -115,7 +115,8 @@ class LoadAndRemapWrappersTest(test.TestCase): axis=1) with self.cached_session(): - self.assertAllClose(expected_remapped_matrix, remapped_matrix.eval()) + self.assertAllClose(expected_remapped_matrix, + self.evaluate(remapped_matrix)) def test_load_and_remap_output_layer_weight_initializer_linear(self): """Tests for the output layer initializer in the linear multi-class case.""" diff --git a/tensorflow/python/training/checkpoint_utils.py b/tensorflow/python/training/checkpoint_utils.py index 857da431db2..58166dbb681 100644 --- a/tensorflow/python/training/checkpoint_utils.py +++ b/tensorflow/python/training/checkpoint_utils.py @@ -101,7 +101,7 @@ def list_variables(ckpt_dir_or_file): return result -@tf_export("train.init_from_checkpoint") +@tf_export(v1=["train.init_from_checkpoint"]) def init_from_checkpoint(ckpt_dir_or_file, assignment_map): """Replaces `tf.Variable` initializers so they load from a checkpoint file. @@ -187,7 +187,7 @@ def init_from_checkpoint(ckpt_dir_or_file, assignment_map): _init_from_checkpoint(None, ckpt_dir_or_file, assignment_map) else: distribution_strategy_context.get_replica_context().merge_call( - _init_from_checkpoint, ckpt_dir_or_file, assignment_map) + _init_from_checkpoint, args=(ckpt_dir_or_file, assignment_map)) def _init_from_checkpoint(_, ckpt_dir_or_file, assignment_map): diff --git a/tensorflow/python/training/checkpointable/BUILD b/tensorflow/python/training/checkpointable/BUILD index d26932c1aae..f97f42a6593 100644 --- a/tensorflow/python/training/checkpointable/BUILD +++ b/tensorflow/python/training/checkpointable/BUILD @@ -152,7 +152,7 @@ py_test( "//tensorflow/python:variable_scope", "//tensorflow/python/eager:backprop", "//tensorflow/python/eager:context", - "//tensorflow/python/eager:function", + "//tensorflow/python/eager:def_function", "//tensorflow/python/eager:test", "//tensorflow/python/keras:engine", "//tensorflow/python/keras:layers", diff --git a/tensorflow/python/training/checkpointable/data_structures.py b/tensorflow/python/training/checkpointable/data_structures.py index c29e5db0753..817552f3269 100644 --- a/tensorflow/python/training/checkpointable/data_structures.py +++ b/tensorflow/python/training/checkpointable/data_structures.py @@ -111,9 +111,6 @@ class CheckpointableDataStructure(base.CheckpointableBase): """Base class for data structures which contain checkpointable objects.""" def __init__(self): - # An append-only ordered set - self._layers = [] - self.trainable = True self._extra_variables = [] @@ -128,21 +125,30 @@ class CheckpointableDataStructure(base.CheckpointableBase): ("Only checkpointable objects (such as Layers or Optimizers) may be " "stored in a List object. Got %s, which does not inherit from " "CheckpointableBase.") % (value,)) - if (isinstance(value, CheckpointableDataStructure) - or layer_utils.is_layer(value) - or layer_utils.has_weights(value)): - # Check for object-identity rather than with __eq__ to avoid - # de-duplicating empty container types. Automatically generated list - # wrappers keep things like "[] == []" true, which means "[] in [[]]" is - # also true. This becomes not true once one of the lists is mutated. - if not any((layer is value for layer in self._layers)): - self._layers.append(value) - if hasattr(value, "_use_resource_variables"): - # In subclassed models, legacy layers (tf.layers) must always use - # resource variables. - value._use_resource_variables = True # pylint: disable=protected-access + if hasattr(value, "_use_resource_variables"): + # In subclassed models, legacy layers (tf.layers) must always use + # resource variables. + value._use_resource_variables = True # pylint: disable=protected-access return value + @property + def _values(self): + """An iterable/sequence which may contain checkpointable objects.""" + raise NotImplementedError("Abstract method") + + @property + def _layers(self): + """All Layers and Layer containers, including empty containers.""" + # Filter objects on demand so that wrapper objects use values from the thing + # they're wrapping if out of sync. + collected = [] + for obj in self._values: + if (isinstance(obj, CheckpointableDataStructure) + or layer_utils.is_layer(obj) + or layer_utils.has_weights(obj)): + collected.append(obj) + return collected + @property def layers(self): return layer_utils.filter_empty_layer_containers(self._layers) @@ -265,6 +271,10 @@ class List(CheckpointableDataStructure, collections.Sequence): def _name_element(self, index): return "%d" % (index,) + @property + def _values(self): + return self + def append(self, value): """Add a new checkpointable value.""" value = self._track_value(value, self._name_element(len(self._storage))) @@ -479,6 +489,14 @@ class Mapping(CheckpointableDataStructure, collections.Mapping): def _make_storage(self, *args, **kwargs): return dict(*args, **kwargs) + @property + def _values(self): + # Sort items deterministically by key + ordered = list(zip(*sorted(self.items(), key=lambda it: it[0]))) + if ordered: + return ordered[1] + return [] + def _name_element(self, key): if not isinstance(key, six.string_types): raise TypeError( diff --git a/tensorflow/python/training/checkpointable/data_structures_test.py b/tensorflow/python/training/checkpointable/data_structures_test.py index ff7d1f1d2d7..9cefd942ac9 100644 --- a/tensorflow/python/training/checkpointable/data_structures_test.py +++ b/tensorflow/python/training/checkpointable/data_structures_test.py @@ -253,6 +253,13 @@ class ListTests(test.TestCase): l.append(1) self.assertEqual([1], l_wrapper) + def testLayerCollectionWithExternalMutation(self): + l = [] + l_wrapper = data_structures._ListWrapper(l) + layer = core.Dense(1) + l.append(layer) + self.assertEqual([layer], l_wrapper.layers) + def testHashing(self): has_sequences = set([data_structures.List(), data_structures.List()]) @@ -324,6 +331,20 @@ class MappingTests(test.TestCase): with self.assertRaises(TypeError): mapping[1] = data_structures.List() + def testLayerCollectionWithExternalMutation(self): + d = {} + root = tracking.Checkpointable() + root.wrapper = d + self.assertEqual([], root.wrapper.layers) + self.assertEqual([], root.wrapper.trainable_weights) + layer1 = core.Dense(1) + layer2 = core.Dense(1) + d["a"] = layer1 + d["b"] = layer2 + self.assertEqual([layer1, layer2], root.wrapper.layers) + # The layers have still not created variables + self.assertEqual([], root.wrapper.trainable_weights) + def testHashing(self): has_mappings = set([data_structures.Mapping(), data_structures.Mapping()]) diff --git a/tensorflow/python/training/checkpointable/tracking.py b/tensorflow/python/training/checkpointable/tracking.py index c85b208d479..4e96aee0c51 100644 --- a/tensorflow/python/training/checkpointable/tracking.py +++ b/tensorflow/python/training/checkpointable/tracking.py @@ -17,6 +17,10 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function +from tensorflow.python.eager import context +from tensorflow.python.framework import dtypes +from tensorflow.python.framework import ops +from tensorflow.python.ops import resource_variable_ops from tensorflow.python.training.checkpointable import base from tensorflow.python.training.checkpointable import data_structures from tensorflow.python.util import tf_contextlib @@ -145,3 +149,36 @@ class TrackableResource(base.CheckpointableBase): if self._resource_handle is None: self._resource_handle = self.create_resource() return self._resource_handle + + +class TrackableAsset(base.CheckpointableBase): + """Base class for asset files which need to be tracked.""" + + def __init__(self, path): + """Record the full path to the asset.""" + # We use a variable here so that @tf.functions do not capture a literal + # value. The init_scope prevents functions from capturing `path` in an + # initialization graph, since it is transient and should not end up in a + # serialized function body. When serialized in a SavedModel, the variable + # will be set during the loading process to its location in the assets/ + # directory. + with ops.init_scope(): + if context.executing_eagerly(): + self._path = self._no_dependency( + resource_variable_ops.ResourceVariable( + path, dtype=dtypes.string, + name="asset_path")) + else: + # Adding a variable is too disruptive when v1-style graph building, + # since things may get fed and local variable initializers would then + # need to be run. + self._path = path + + @property + def asset_path(self): + """Fetch the current asset path.""" + return self._path + +ops.register_tensor_conversion_function( + TrackableAsset, + lambda asset, **kw: ops.internal_convert_to_tensor(asset.asset_path, **kw)) diff --git a/tensorflow/python/training/checkpointable/util.py b/tensorflow/python/training/checkpointable/util.py index f45f7445f13..36d3a7bebd4 100644 --- a/tensorflow/python/training/checkpointable/util.py +++ b/tensorflow/python/training/checkpointable/util.py @@ -31,6 +31,7 @@ from tensorflow.python.framework import dtypes from tensorflow.python.framework import errors_impl from tensorflow.python.framework import ops from tensorflow.python.framework import tensor_shape +from tensorflow.python.lib.io import file_io from tensorflow.python.ops import array_ops from tensorflow.python.ops import gen_io_ops as io_ops from tensorflow.python.ops import init_ops @@ -549,13 +550,11 @@ def _serialize_slot_variables(checkpointable_objects, node_ids, object_names): return slot_variables -def _serialize_checkpointables( - checkpointable_objects, node_ids, object_names, slot_variables, +def _add_attributes_to_object_graph( + checkpointable_objects, object_graph_proto, node_ids, object_names, saveables_cache, object_map): - """Name non-slot `Checkpointable`s and add them to `object_graph_proto`.""" - object_graph_proto = ( - checkpointable_object_graph_pb2.CheckpointableObjectGraph()) - named_saveables = [] + """Create SaveableObjects and corresponding SerializedTensor protos.""" + named_saveable_objects = [] if saveables_cache is None: # No SaveableObject caching. Either we're executing eagerly, or building a # static save which is specialized to the current Python state. @@ -564,10 +563,9 @@ def _serialize_checkpointables( # If we are caching SaveableObjects, we need to build up a feed_dict with # functions computing volatile Python state to be saved with the checkpoint. feed_additions = {} - for checkpoint_id, checkpointable in enumerate(checkpointable_objects): + for checkpoint_id, (checkpointable, object_proto) in enumerate( + zip(checkpointable_objects, object_graph_proto.nodes)): assert node_ids[checkpointable] == checkpoint_id - object_proto = object_graph_proto.nodes.add() - object_proto.slot_variables.extend(slot_variables.get(checkpointable, ())) object_name = object_names[checkpointable] if object_map: object_to_save = object_map.get(checkpointable, checkpointable) @@ -645,14 +643,26 @@ def _serialize_checkpointables( "value.") % (checkpointable, new_feed_key)) feed_additions.update(saveable_feed_dict) - named_saveables.append(saveable) + named_saveable_objects.append(saveable) + return named_saveable_objects, feed_additions + + +def _fill_object_graph_proto(checkpointable_objects, node_ids, slot_variables, + object_graph_proto=None): + """Name non-slot `Checkpointable`s and add them to `object_graph_proto`.""" + if object_graph_proto is None: + object_graph_proto = ( + checkpointable_object_graph_pb2.CheckpointableObjectGraph()) + for checkpoint_id, checkpointable in enumerate(checkpointable_objects): + assert node_ids[checkpointable] == checkpoint_id + object_proto = object_graph_proto.nodes.add() + object_proto.slot_variables.extend(slot_variables.get(checkpointable, ())) for child in checkpointable._checkpoint_dependencies: # pylint: disable=protected-access child_proto = object_proto.children.add() child_proto.node_id = node_ids[child.ref] child_proto.local_name = child.name - - return named_saveables, object_graph_proto, feed_additions + return object_graph_proto def _serialize_gathered_objects( @@ -668,13 +678,18 @@ def _serialize_gathered_objects( checkpointable_objects=checkpointable_objects, node_ids=node_ids, object_names=object_names) - return _serialize_checkpointables( + object_graph_proto = _fill_object_graph_proto( checkpointable_objects=checkpointable_objects, node_ids=node_ids, + slot_variables=slot_variables) + named_saveable_objects, feed_additions = _add_attributes_to_object_graph( + checkpointable_objects=checkpointable_objects, + object_graph_proto=object_graph_proto, + node_ids=node_ids, object_names=object_names, - slot_variables=slot_variables, saveables_cache=saveables_cache, object_map=object_map) + return named_saveable_objects, object_graph_proto, feed_additions def _serialize_object_graph(root_checkpointable, saveables_cache): @@ -716,6 +731,23 @@ def named_saveables(root_checkpointable): return _serialize_object_graph(root_checkpointable, None)[0] +def _find_objects(root_checkpointable): + """Find and number objects which are dependencies of `root_checkpointable`.""" + checkpointable_objects, path_to_root = ( + _breadth_first_checkpointable_traversal(root_checkpointable)) + object_names = _ObjectIdentityDictionary() + for obj, path in path_to_root.items(): + object_names[obj] = _object_prefix_from_path(path) + node_ids = _ObjectIdentityDictionary() + for node_id, node in enumerate(checkpointable_objects): + node_ids[node] = node_id + slot_variables = _serialize_slot_variables( + checkpointable_objects=checkpointable_objects, + node_ids=node_ids, + object_names=object_names) + return checkpointable_objects, node_ids, slot_variables + + def list_objects(root_checkpointable): """Traverse the object graph and list all accessible objects. @@ -730,23 +762,18 @@ def list_objects(root_checkpointable): Returns: A flat list of objects. """ - # TODO(allenl): Extract out gathering logic so the naming logic doesn't have - # to run. - checkpointable_objects, path_to_root = ( - _breadth_first_checkpointable_traversal(root_checkpointable)) - object_names = _ObjectIdentityDictionary() - for obj, path in path_to_root.items(): - object_names[obj] = _object_prefix_from_path(path) - node_ids = _ObjectIdentityDictionary() - for node_id, node in enumerate(checkpointable_objects): - node_ids[node] = node_id - _serialize_slot_variables( - checkpointable_objects=checkpointable_objects, - node_ids=node_ids, - object_names=object_names) + checkpointable_objects, _, _ = _find_objects(root_checkpointable) return checkpointable_objects +def make_object_graph_without_attributes(root_checkpointable, proto=None): + """Fill an object graph proto, ignoring variable values.""" + checkpointable_objects, node_ids, slot_variables = _find_objects( + root_checkpointable) + return _fill_object_graph_proto( + checkpointable_objects, node_ids, slot_variables, proto) + + def gather_initializers(root_checkpointable): """Traverse the object graph and find initialization ops. @@ -1434,6 +1461,7 @@ class CheckpointableSaver(object): elif session is None: session = ops.get_default_session() + file_io.recursive_create_dir(os.path.dirname(file_prefix)) with ops.device("/cpu:0"): save_path = saver.save( sess=_SessionWithFeedDictAdditions( @@ -1898,3 +1926,4 @@ class Checkpoint(tracking.Checkpointable): # initialization when executing eagerly. self._maybe_create_save_counter() return status + diff --git a/tensorflow/python/training/checkpointable/util_test.py b/tensorflow/python/training/checkpointable/util_test.py index 19955140123..de9cac08632 100644 --- a/tensorflow/python/training/checkpointable/util_test.py +++ b/tensorflow/python/training/checkpointable/util_test.py @@ -26,7 +26,7 @@ from tensorflow.python import pywrap_tensorflow from tensorflow.python.client import session as session_lib from tensorflow.python.eager import backprop from tensorflow.python.eager import context -from tensorflow.python.eager import function +from tensorflow.python.eager import def_function from tensorflow.python.eager import test from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes @@ -44,6 +44,7 @@ from tensorflow.python.ops import variable_scope from tensorflow.python.ops import variables from tensorflow.python.training import adam from tensorflow.python.training import checkpoint_management +from tensorflow.python.training import momentum from tensorflow.python.training import saver as saver_lib from tensorflow.python.training import training_util from tensorflow.python.training.checkpointable import base @@ -198,6 +199,17 @@ class InterfaceTests(test.TestCase): with self.assertRaises(NotImplementedError): checkpoint_reversed.save(prefix) + @test_util.run_in_graph_and_eager_modes(assert_no_eager_garbage=True) + def test_object_graph_no_attributes(self): + root = tracking.Checkpointable() + root.v = resource_variable_ops.ResourceVariable(1.) + root.opt = momentum.MomentumOptimizer(0.01, 0.5) + root.opt.minimize(root.v.read_value) + object_graph = checkpointable_utils.make_object_graph_without_attributes( + root) + # Four objects: Root, v, opt, and a slot variable for v + self.assertEqual(4, len(object_graph.nodes)) + class _MirroringSaveable(saver_lib.BaseSaverBuilder.SaveableObject): @@ -632,7 +644,7 @@ class CheckpointingTests(test.TestCase): checkpoint_directory) status = root.restore(save_path=checkpoint_path) def train_fn(): - @function.defun + @def_function.function def _call_model(x): return model(x) with backprop.GradientTape() as tape: diff --git a/tensorflow/python/training/coordinator.py b/tensorflow/python/training/coordinator.py index 0ff97d85e37..b7e5c98c78e 100644 --- a/tensorflow/python/training/coordinator.py +++ b/tensorflow/python/training/coordinator.py @@ -408,7 +408,7 @@ class Coordinator(object): # Threads for the standard services. -@tf_export("train.LooperThread") +@tf_export(v1=["train.LooperThread"]) class LooperThread(threading.Thread): """A thread that runs code repeatedly, optionally on a timer. diff --git a/tensorflow/python/training/device_setter.py b/tensorflow/python/training/device_setter.py index be80c365715..5874a1ff415 100644 --- a/tensorflow/python/training/device_setter.py +++ b/tensorflow/python/training/device_setter.py @@ -130,7 +130,7 @@ class _ReplicaDeviceChooser(object): return worker_device.to_string() -@tf_export("train.replica_device_setter") +@tf_export(v1=["train.replica_device_setter"]) def replica_device_setter(ps_tasks=0, ps_device="/job:ps", worker_device="/job:worker", merge_devices=True, cluster=None, ps_ops=None, ps_strategy=None): diff --git a/tensorflow/python/training/device_setter_test.py b/tensorflow/python/training/device_setter_test.py index 85b75502ab0..3cff87b326f 100644 --- a/tensorflow/python/training/device_setter_test.py +++ b/tensorflow/python/training/device_setter_test.py @@ -19,6 +19,7 @@ from __future__ import division from __future__ import print_function from tensorflow.python.framework import ops +from tensorflow.python.framework import test_util from tensorflow.python.ops import resource_variable_ops from tensorflow.python.ops import variables from tensorflow.python.platform import test @@ -33,6 +34,7 @@ class DeviceSetterTest(test.TestCase): "worker": ["worker0:2222", "worker1:2222", "worker2:2222"] }) + @test_util.run_deprecated_v1 def testCPUOverride(self): with ops.device( device_setter.replica_device_setter(cluster=self._cluster_spec)): @@ -47,12 +49,14 @@ class DeviceSetterTest(test.TestCase): self.assertDeviceEqual("/job:ps/task:1", w.initializer.device) self.assertDeviceEqual("/job:worker/cpu:0", a.device) + @test_util.run_deprecated_v1 def testResource(self): with ops.device( device_setter.replica_device_setter(cluster=self._cluster_spec)): v = resource_variable_ops.ResourceVariable([1, 2]) self.assertDeviceEqual("/job:ps/task:0", v.device) + @test_util.run_deprecated_v1 def testPS2TasksWithClusterSpecClass(self): with ops.device( device_setter.replica_device_setter(cluster=self._cluster_spec)): @@ -65,6 +69,7 @@ class DeviceSetterTest(test.TestCase): self.assertDeviceEqual("/job:ps/task:1", w.initializer.device) self.assertDeviceEqual("/job:worker", a.device) + @test_util.run_deprecated_v1 def testPS2TasksPinVariableToJob(self): with ops.device( device_setter.replica_device_setter(cluster=self._cluster_spec)): @@ -82,6 +87,7 @@ class DeviceSetterTest(test.TestCase): self.assertDeviceEqual("/job:ps/task:1", x.initializer.device) self.assertDeviceEqual("/job:worker", a.device) + @test_util.run_deprecated_v1 def testPS2TasksUseCpuForPS(self): with ops.device( device_setter.replica_device_setter(ps_tasks=1, ps_device="/cpu:0")): @@ -95,6 +101,7 @@ class DeviceSetterTest(test.TestCase): self.assertDeviceEqual("/job:moon/cpu:0", w.initializer.device) self.assertDeviceEqual("/job:worker", a.device) + @test_util.run_deprecated_v1 def testPS2TasksNoMerging(self): with ops.device( device_setter.replica_device_setter( @@ -109,6 +116,7 @@ class DeviceSetterTest(test.TestCase): self.assertDeviceEqual("/job:ps", w.initializer.device) self.assertDeviceEqual("/job:worker", a.device) + @test_util.run_deprecated_v1 def testPS2TasksWithClusterSpecDict(self): with ops.device( device_setter.replica_device_setter(cluster=self._cluster_spec.as_dict( @@ -122,6 +130,7 @@ class DeviceSetterTest(test.TestCase): self.assertDeviceEqual("/job:ps/task:1", w.initializer.device) self.assertDeviceEqual("/job:worker", a.device) + @test_util.run_deprecated_v1 def testPS2TasksWithClusterDef(self): with ops.device( device_setter.replica_device_setter( @@ -135,6 +144,7 @@ class DeviceSetterTest(test.TestCase): self.assertDeviceEqual("/job:ps/task:1", w.initializer.device) self.assertDeviceEqual("/job:worker", a.device) + @test_util.run_deprecated_v1 def testPS2TasksWithDevice(self): cluster_spec = server_lib.ClusterSpec({ "sun": ["sun0:2222", "sun1:2222", "sun2:2222"], @@ -155,6 +165,7 @@ class DeviceSetterTest(test.TestCase): self.assertDeviceEqual("/job:moon/task:1", w.initializer.device) self.assertDeviceEqual("/job:sun", a.device) + @test_util.run_deprecated_v1 def testPS2TasksWithCPUConstraint(self): cluster_spec = server_lib.ClusterSpec({ "sun": ["sun0:2222", "sun1:2222", "sun2:2222"], diff --git a/tensorflow/python/training/distribute.py b/tensorflow/python/training/distribute.py index 95104ad5779..ad27bc8a702 100644 --- a/tensorflow/python/training/distribute.py +++ b/tensorflow/python/training/distribute.py @@ -12,1234 +12,11 @@ # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================== -"""Class DistributionStrategy, ReplicaContext, and supporting APIs.""" +"""Deprecated, please use ../distribute/distribute_lib.py.""" from __future__ import absolute_import from __future__ import division from __future__ import print_function -import threading - -from tensorflow.python.data.ops import dataset_ops -from tensorflow.python.framework import ops -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import control_flow_ops -from tensorflow.python.ops import resource_variable_ops -from tensorflow.python.ops import variable_scope -from tensorflow.python.ops.losses import losses_impl -from tensorflow.python.platform import tf_logging -from tensorflow.python.training import device_util -from tensorflow.python.training import distribution_strategy_context -from tensorflow.python.util import nest - - -# ------------------------------------------------------------------------------ -# Context tracking whether in a distribution.update() or .update_non_slot() -# call. - - -_update_device = threading.local() - - -def get_update_device(): - """Get the current device if in a `DistributionStrategy.update()` call.""" - try: - return _update_device.current - except AttributeError: - return None - - -class UpdateContext(object): - """Context manager when you are in `update()` or `update_non_slot()`.""" - - def __init__(self, device): - self._device = device - self._old_device = None - - def __enter__(self): - self._old_device = get_update_device() - _update_device.current = self._device - - def __exit__(self, exception_type, exception_value, traceback): - del exception_type, exception_value, traceback - _update_device.current = self._old_device - - -# ------------------------------------------------------------------------------ -# Public utility functions. - - -def get_loss_reduction(): - """Reduce `aggregation` corresponding to the last loss reduction.""" - loss_reduction = ops.get_default_graph()._last_loss_reduction # pylint: disable=protected-access - if loss_reduction == losses_impl.Reduction.SUM: - return variable_scope.VariableAggregation.SUM - return variable_scope.VariableAggregation.MEAN - - -# ------------------------------------------------------------------------------ -# Internal API for validating the current thread mode - - -def _require_cross_replica_context(distribution_strategy): - """Verify in cross-replica context for `distribution_strategy`.""" - context = _get_per_thread_mode() - if context.cross_replica_context is distribution_strategy: return - # We have an error to report, figure out the right message. - if context.distribution_strategy is not distribution_strategy: - if not distribution_strategy_context.has_distribution_strategy(): - raise RuntimeError( - 'Need to be inside "with distribution_strategy.scope()" for %s' % - (distribution_strategy,)) - else: - raise RuntimeError( - "Mixing different DistributionStrategy objects: %s is not %s" % - (context.distribution_strategy, distribution_strategy)) - assert context.cross_replica_context is None - raise RuntimeError("Method requires being in cross-replica context, use " - "get_replica_context().merge_call()") - - -def require_replica_context(replica_ctx): - """Verify in `replica_ctx` replica context.""" - context = _get_per_thread_mode() - if context.replica_context is replica_ctx: return - # We have an error to report, figure out the right message. - if context.replica_context is None: - raise RuntimeError("Need to be inside `call_for_each_replica()`") - if context.distribution_strategy is replica_ctx.distribution_strategy: - # Two different ReplicaContexts with the same DistributionStrategy. - raise RuntimeError("Mismatching replica context.") - raise RuntimeError( - "Mismatching DistributionStrategy objects: %s is not %s." % - (context.distribution_strategy, replica_ctx.distribution_strategy)) - - -def _require_distribution_strategy_scope(distribution_strategy): - """Verify in a `distribution_strategy.scope()` in this thread.""" - context = _get_per_thread_mode() - if context.distribution_strategy is distribution_strategy: return - # We have an error to report, figure out the right message. - if not distribution_strategy_context.has_distribution_strategy(): - raise RuntimeError( - 'Need to be inside "with distribution_strategy.scope()" for %s' % - (distribution_strategy,)) - else: - raise RuntimeError( - "Mixing different DistributionStrategy objects: %s is not %s" % - (context.distribution_strategy, distribution_strategy)) - - -# ------------------------------------------------------------------------------ -# Internal context managers used to implement the DistributionStrategy -# base class - - -class _CurrentDistributionContext(object): - """Context manager for setting the `DistributionStrategy` and var creator.""" - - def __init__(self, - distribution_strategy, - var_creator_scope, - var_scope=None, - default_device=None): - self._context = distribution_strategy_context._CrossReplicaThreadMode( # pylint: disable=protected-access - distribution_strategy) - self._var_creator_scope = var_creator_scope - self._var_scope = var_scope - if default_device: - self._device_scope = ops.device(default_device) - else: - self._device_scope = None - - def __enter__(self): - _push_per_thread_mode(self._context) - if self._var_scope: - self._var_scope.__enter__() - self._var_creator_scope.__enter__() - if self._device_scope: - self._device_scope.__enter__() - return self._context.distribution_strategy - - def __exit__(self, exception_type, exception_value, traceback): - if self._device_scope: - self._device_scope.__exit__(exception_type, exception_value, traceback) - self._var_creator_scope.__exit__(exception_type, exception_value, traceback) - if self._var_scope: - self._var_scope.__exit__(exception_type, exception_value, traceback) - _pop_per_thread_mode() - - -class _SameScopeAgainContext(object): - """Trivial context manager when you are already in `scope()`.""" - - def __init__(self, distribution_strategy): - self._distribution_strategy = distribution_strategy - - def __enter__(self): - return self._distribution_strategy - - def __exit__(self, exception_type, exception_value, traceback): - del exception_type, exception_value, traceback - - -# ------------------------------------------------------------------------------ -# Base classes for all distribution strategies. - - -class DistributionStrategy(object): - """A list of devices with a state & compute distribution policy. - - See [tensorflow/contrib/distribute/README.md]( - https://www.tensorflow.org/code/tensorflow/contrib/distribute/README.md) - for overview and examples. - - The intent is that you can write an algorithm in a stylized way and - it will be usable with a variety of different `DistributionStrategy` - implementations. Each descendant will implement a different strategy - for distributing the algorithm across multiple devices/machines. - Furthermore, these changes can be hidden inside the specific layers - and other library classes that need special treatment to run in a - distributed setting, so that most users' model definition code can - run unchanged. The `DistributionStrategy` API works the same way - with eager and graph execution. - - First let's introduce a few high-level concepts: - - * _Data parallelism_ is where we run multiple copies of the model - on different slices of the input data. This is in contrast to - _model parallelism_ where we divide up a single copy of a model - across multiple devices. - Note: we only support data parallelism for now, but - hope to add support for model parallelism in the future. - * A _replica_ is one copy of the model, running on one slice of the - input data. - * _Synchronous_, or more commonly _sync_, training is where the - updates from each replica are aggregated together before updating - the model variables. This is in contrast to _asynchronous_, or - _async_ training, where each replica updates the model variables - independently. - * Furthermore you might run your computation on multiple devices - on one machine (or "host"), or on multiple machines/hosts. - If you are running on multiple machines, you might have a - single master host that drives computation across all of them, - or you might have multiple clients driving the computation - asynchronously. - - To distribute an algorithm, we might use some of these ingredients: - - * Parameter servers: These are hosts that hold a single copy of - parameters/variables. All replicas that want to operate on a variable - retrieve it at the beginning of a step and send an update to be - applied at the end of the step. Can support either sync or async - training. - * Mirrored variables: These are variables that are copied to multiple - devices, where we keep the copies in sync by applying the same - updates to every copy. Normally would only be used with sync training. - * Reductions and Allreduce: A _reduction_ is some method of - aggregating multiple values into one value, like "sum" or - "mean". If doing sync training, we will perform a reduction on the - gradients to a parameter from all replicas before applying the - update. Allreduce is an algorithm for performing a reduction on - values from multiple devices and making the result available on - all of those devices. - * In the future we will have support for TensorFlow's partitioned - variables, where a single variable is split across multiple - devices. - - We have then a few approaches we want to support: - - * Code written (as if) with no knowledge of class `DistributionStrategy`. - This code should work as before, even if some of the layers, etc. - used by that code are written to be distribution-aware. This is done - by having a default `DistributionStrategy` that gives ordinary behavior, - and by default being in a single replica context. - * Ordinary model code that you want to run using a specific - `DistributionStrategy`. This can be as simple as: - - ``` - with my_distribution.scope(): - iterator = my_distribution.distribute_dataset( - dataset).make_one_shot_iterator() - replica_train_ops = my_distribution.call_for_each_replica( - replica_fn, args=(iterator.get_next(),)) - train_op = tf.group(my_distribution.unwrap(replica_train_ops)) - ``` - - This takes an ordinary `dataset` and `replica_fn` and runs it - distributed using a particular `DistributionStrategy` in - `my_distribution`. Any variables created in `replica_fn` are created - using `my_distribution`'s policy, and library functions called by - `replica_fn` can use the `get_replica_context()` API to get enhanced - behavior in this case. - - You can also create an initializable iterator instead of a one-shot - iterator. In that case, you will need to ensure that you initialize the - iterator before calling get_next. - ``` - iterator = my_distribution.distribute_dataset( - dataset).make_initializable_iterator()) - session.run(iterator.initializer) - ``` - - * If you want to write a distributed algorithm, you may use any of - the `DistributionStrategy` APIs inside a - `with my_distribution.scope():` block of code. - - Lower-level concepts: - - * Wrapped values: In order to represent values parallel across devices - (either replicas or the devices associated with a particular value), we - wrap them in a "PerReplica" or "Mirrored" object that contains a map - from device to values. "PerReplica" is used when the value may be - different across replicas, and "Mirrored" when the value are the same. - * Unwrapping and merging: Consider calling a function `fn` on - multiple replicas, like `call_for_each_replica(fn, args=[w])` with an - argument `w` that is a wrapped value. This means `w` will have a - map taking replica device `d0` to `w0`, replica device `d1` to `w1`, - etc. `call_for_each_replica()` unwraps `w` before calling `fn`, so - it calls `fn(w0)` on `d0`, `fn(w1)` on `d1`, etc. It then merges - the return values from `fn()`, which can possibly result in - wrapped values. For example, let's say `fn()` returns a tuple with - three components: `(x, a, v0)` from replica 0, `(x, b, v1)` on replica 1, - etc. If the first component is the same object `x` from every - replica, then the first component of the merged result will also be - `x`. If the second component is different (`a`, `b`, ...) from - each replica, then the merged value will have a wrapped map from - replica device to the different values. If the third component is - the members of a mirrored variable (`v` maps `d0` to `v0`, `d1` to - `v1`, etc.), then the merged result will be that mirrored variable - (`v`). - * Replica context vs. Cross-replica context: _replica context_ is when we - are in some function that is being called once for each replica. - Otherwise we are in cross-replica context, which is useful for - calling `DistributionStrategy` methods which operate across the - replicas (like `reduce()`). By default you start in a replica context - (the default "single replica context") and then some methods can - switch you back and forth, as described below. - * Worker devices vs. parameter devices: Most replica computations will - happen on worker devices. Since we don't yet support model - parallelism, there will be one worker device per replica. When using - parameter servers (see above), the set of devices holding - variables may be different, otherwise the parameter devices might - match the worker devices. - * Non-slot devices are some subset of the parameter devices where we - put all the non-slot variables. We need to ensure that all - non-slot variables are allocated on the same device, or mirrored - across the same set of devices. If you have some variable you want - to colocate all the non-slot variables with, you can use - `colocate_vars_with()` to get the remaining non-slot variables on - the same device. Otherwise you can use `non_slot_devices()` to - pick a consistent set of devices to pass to both - `colocate_vars_with()` and `update_non_slot()`. - - When using a `DistributionStrategy`, we have a new type dimension - called _locality_ that says what values are compatible with which - APIs: - - * T: different value for each replica (e.g. a PerReplica-wrapped value). - * M: value is "mirrored" across replicas, i.e. there are copies with the - same value on each replica (e.g. a Mirrored-wrapped value). - * V(`v`): value is "mirrored" across all the devices which have a - copy of variable `v` (also a Mirrored-wrapped value, but over - parameter devices instead of worker devices). - * N: value is "mirrored" across all the "non-slot" devices - - Rules for methods with respect to locality and single-replica vs. - cross-replica context: - - * `with d.scope()`: default single-replica context -> cross-replica context - for `d` - * `with d.colocate_vars_with(v)`: in replica/cross-replica context, variables - will be created with locality V(`v`). That is, if we write - `with d.colocate_vars_with(v1): v2 = tf.get_variable(...)`, then - `v2` will have locality V(`v1`), i.e. locality V(`v2`) will equal - V(`v1`). - * `with d.colocate_vars_with(d.non_slot_devices(...))`: in - replica/cross-replica context, variables will be created with locality N - * `v = tf.get_variable(...)`: in replica/cross-replica context, creates - a variable (which by definition will have locality V(`v`), though - will match another locality if inside a `colocate_vars_with` - scope). - * `d.distribute_dataset(dataset).make_one_shot_iterator()`: in cross-replica - context, produces an iterator with locality T - * `d.broadcast(t)`: in cross-replica context, produces a value with locality M - * `d.broadcast(t, v)`: in cross-replica context, produces a value with - locality V(`v`) - * `d.call_for_each_replica(fn, ...)`: in cross-replica context, runs - `fn()` in a replica context (and so may call `get_replica_context()` and - use its API, including `merge_call()` to get back to cross-replica - context), once for each replica. May use values with locality T or - M, and any variable. - * `d.reduce(m, t, t)`: in cross-replica context, accepts t with locality T - and produces a value with locality M. - * `d.reduce(m, t, v)`: in cross-replica context, accepts t with - locality T and produces a value with locality V(`v`). - * `d.batch_reduce(m, [(t, v)]): see `d.reduce()` - * `d.update(v, fn, ...)`: in cross-replica context, runs `fn()` once - for each device `v` is copied to, all inputs should have locality - V(`v`), output will have locality V(`v`) as well. - * `d.update_non_slot(d.non_slot_devices(), fn)`: in cross-replica - context, like `d.update()` except with locality N. - * `d.read_var(v)`: Gets the (read-only) value of the variable `v` (on - the device determined by the current device scope), aggregating - across replicas for replica-local variables. Frequently, this will be - done automatically when using `v` in an expression or fetching it in - a cross-replica context, but this function can be used to force that - conversion happens at a particular point in time (for example, to - add the result of the conversion to a graph collection). - - The standard pattern for updating variables is to: - - 1. Wrap your input dataset in `d.distribute_dataset()` and create an iterator. - 2. Define each replica `d.call_for_each_replica()` up to the point of - getting a list of gradient, variable pairs. - 3. Call `d.reduce(VariableAggregation.SUM, t, v)` or `d.batch_reduce()` to sum - the gradients (with locality T) into values with locality V(`v`). - 4. Call `d.update(v)` for each variable to update its value. - - Steps 3 and 4 are done automatically by class `Optimizer` if you call - its `apply_gradients` method in a replica context. Otherwise you can - manually call its `_distributed_apply` method in a cross-replica context. - - Another thing you might want to do in the middle of your replica function - is an all-reduce of some intermediate value, using `d.reduce()` or - `d.batch_reduce()`. You simply provide the same tensor as the input and - destination. - - Layers should expect to be called in a replica context, and can use - the `get_replica_context()` function to get a `ReplicaContext` object. The - `ReplicaContext` object has a `merge_call()` method for entering - cross-replica context where you can use `reduce()` (or - `batch_reduce()`) and then optionally `update()` to update state. - - You may use this API whether or not a `DistributionStrategy` is - being used, since there is a default implementation of - `ReplicaContext` and `DistributionStrategy`. - """ - - # TODO(josh11b): Raise an exception if variable partitioning requested before - # we add support. - # TODO(josh11b): Also `parameter_device_index` property? - # TODO(josh11b): `map()` - # TODO(josh11b): ClusterSpec/ClusterResolver - # TODO(josh11b): Partitioned computations, state; sharding - # TODO(josh11b): Model parallelism: "replicas" with multiple devices; shuffling - # TODO(josh11b): List of replicas with their worker and parameter devices - # (where the parameter devices may overlap in the ps case). - - def __init__(self): - self._default_device = None - # This property is used to determine if we should set drop_remainder=True - # when creating Datasets from numpy array inputs. - self._require_static_shapes = False - - def scope(self): - """Returns a context manager selecting this DistributionStrategy as current. - - Inside a `with distribution_strategy.scope():` code block, this thread - will use a variable creator set by `distribution_strategy`, and will - enter its "cross-replica context". - - Returns: - A context manager. - """ - if distribution_strategy_context.has_distribution_strategy(): - _require_cross_replica_context(self) - return _SameScopeAgainContext(self) - - def creator_with_resource_vars(*args, **kwargs): - _require_distribution_strategy_scope(self) - kwargs["use_resource"] = True - return self._create_variable(*args, **kwargs) - - def distributed_getter(getter, *args, **kwargs): - if not self._allow_variable_partition(): - if kwargs.pop("partitioner", None) is not None: - tf_logging.log_first_n( - tf_logging.WARN, "Partitioned variables are disabled when using " - "current DistributionStrategy.", 1) - return getter(*args, **kwargs) - - return _CurrentDistributionContext( - self, variable_scope.variable_creator_scope(creator_with_resource_vars), - variable_scope.variable_scope( - variable_scope.get_variable_scope(), - custom_getter=distributed_getter), self._default_device) - - def _allow_variable_partition(self): - return False - - def _create_variable(self, next_creator, *args, **kwargs): - # Note: should support "colocate_with" argument. - raise NotImplementedError("must be implemented in descendants") - - def read_var(self, v): - """Reads the value of a variable. - - Returns the aggregate value of a replica-local variable, or the - (read-only) value of any other variable. - - Args: - v: A variable allocated within the scope of this `DistributionStrategy`. - - Returns: - A tensor representing the value of `v`, aggregated across replicas if - necessary. - """ - raise NotImplementedError("must be implemented in descendants") - - def colocate_vars_with(self, colocate_with_variable): - """Scope that controls which devices variables will be created on. - - No operations should be added to the graph inside this scope, it - should only be used when creating variables (some implementations - work by changing variable creation, others work by using a - tf.colocate_with() scope). - - This may only be used inside `self.scope()`. - - Example usage: - - ``` - with distribution_strategy.scope(): - var1 = tf.get_variable(...) - with distribution_strategy.colocate_vars_with(v1): - # var2 and var3 will be created on the same device(s) as var1 - var2 = tf.get_variable(...) - var3 = tf.get_variable(...) - - def fn(v1, v2, v3): - # operates on v1 from var1, v2 from var2, and v3 from var3 - - # `fn` runs on every device `v1` is on, `v2` and `v3` will be there too. - distribution_strategy.update(v1, fn, v2, v3) - ``` - - Args: - colocate_with_variable: A created in `self.scope()`. Variables created - while in the returned context manager will be on the same set of - devices as `colocate_with_variable`. - - Returns: - A context manager. - """ - def create_colocated_variable(next_creator, *args, **kwargs): - _require_distribution_strategy_scope(self) - kwargs["use_resource"] = True - kwargs["colocate_with"] = colocate_with_variable - return next_creator(*args, **kwargs) - - _require_distribution_strategy_scope(self) - return variable_scope.variable_creator_scope(create_colocated_variable) - - def _call_dataset_fn(self, dataset_fn): - result = dataset_fn() - if not isinstance(result, dataset_ops.Dataset): - raise ValueError( - "dataset_fn() must return a tf.data.Dataset when using a " - "DistributionStrategy.") - return result - - # TODO(josh11b): `PerReplicaDataset` currently only implements a few methods of - # Dataset API such as make_one_shot_iterator and make_initializable_iterator. - # Extend to implement more functionality of datasets. - def distribute_dataset(self, dataset_fn): - """Return a `dataset` split across all replicas. - - Suitable for providing input to for `call_for_each_replica()` by creating an - iterator: - - ``` - def dataset_fn(): - return tf.data.Dataset.from_tensors([[1.]]).repeat() - with distribution_strategy.scope(): - distributed_dataset = distribution_strategy.distribute_dataset(dataset_fn) - iterator = distributed_dataset.make_one_shot_iterator() - replica_results = distribution_strategy.call_for_each_replica( - replica_fn, args=(iterator.get_next(),)) - ``` - - Args: - dataset_fn: A function that returns a `tf.data.Dataset`. - - Returns: - A `PerReplicaDataset` that will produce data for each replica. - """ - raise NotImplementedError("must be implemented in descendants") - - def broadcast(self, tensor, destinations=None): - """Mirror a tensor on one device to all worker devices. - - Args: - tensor: A Tensor value to broadcast. - destinations: An optional mirrored variable, device string, or - list of device strings, specifying the destination devices - to copy `tensor` to. Defaults to `self.worker_devices`. - - Returns: - A value mirrored to `destinations` devices. - """ - # TODO(josh11b): More docstring - _require_cross_replica_context(self) - return self._broadcast(tensor, destinations) - - def _broadcast(self, tensor, destinations): - raise NotImplementedError("must be implemented in descendants") - - def initialize(self): - """Any initialization to be done before running any computations. - - In eager mode, it executes any initialization as a side effect. - In graph mode, it creates the initialization ops and returns them. - - For example, TPU initialize_system ops. - - Returns: - A list of ops to execute. - """ - return [] - - def finalize(self): - """Any final actions to be done at the end of all computations. - - In eager mode, it executes any finalize actions as a side effect. - In graph mode, it creates the finalize ops and returns them. - - For example, TPU shutdown ops. - - Returns: - A list of ops to execute. - """ - return [] - - def run_steps_on_dataset(self, fn, iterator, iterations=1, - initial_loop_values=None): - """Run `fn` with input from `iterator` for `iterations` times. - - This method can be used to run a step function for training a number of - times using input from a dataset. - - Args: - fn: function to run using this distribution strategy. The function must - have the following signature: `def fn(context, *inputs)`. - `context` is an instance of `MultiStepContext` that will be passed when - `fn` is run. `context` can be used to specify the outputs to be returned - from `fn` by calling `context.set_last_step_output`. It can also be used - to capture non tensor outputs by `context.set_non_tensor_output`. - See `MultiStepContext` documentation for more information. - `inputs` will have same type/structure as `iterator.get_next()`. If the - `iterator.get_next()` returns a tuple say `return x, y` then whose will - be unpacked and passed to the `step_fn`; and step_fn signature would - look like `def step_fn(context, x, y)`. If the iterator returns a single - value say `return x` then the value is passed as is; the step_fn - signature would look like `def step_fn(context, x)`. - Typically, `fn` will use `call_for_each_replica` method of the strategy - to distribute the computation over multiple replicas. - iterator: Iterator of a dataset that represents the input for `fn`. The - caller is responsible for initializing the iterator as needed. - iterations: (Optional) Number of iterations that `fn` should be run. - Defaults to 1. - initial_loop_values: (Optional) Initial values to be passed into the - loop that runs `fn`. Defaults to `None`. # TODO(priyag): Remove - initial_loop_values argument when we have a mechanism to infer the - outputs of `fn`. - - Returns: - Returns the `MultiStepContext` object which has the following properties, - among other things: - - run_op: An op that runs `fn` `iterations` times. - - last_step_outputs: A dictionary containing tensors set using - `context.set_last_step_output`. Evaluating this returns the value of - the tensors after the last iteration. - - non_tensor_outputs: A dictionatry containing anything that was set by - `fn` by calling `context.set_non_tensor_output`. - """ - _require_cross_replica_context(self) - return self._run_steps_on_dataset(fn, iterator, iterations, - initial_loop_values) - - def _run_steps_on_dataset(self, fn, iterator, iterations, - initial_loop_values): - raise NotImplementedError("must be implemented in descendants") - - def call_for_each_replica(self, fn, *args, **kwargs): - """Run `fn` once per replica. - - `fn` may call `tf.get_replica_context()` to access methods such as - `replica_id()` and `merge_call()`. - - `merge_call()` is used to communicate between the replicas and - re-enter the cross-replica context. All replicas pause their execution - having encountered a `merge_call()` call. After that the - `merge_fn`-function is executed. Its results are then unwrapped and - given back to each replica call. After that execution resumes until - `fn` is complete or encounters another `merge_call()`. Example: - - ```python - # Called once in "cross-replica" context. - def merge_fn(distribution, three_plus_replica_id): - # sum the values across replicas - return sum(distribution.unwrap(three_plus_replica_id)) - - # Called once per replica in `distribution`, in a "replica" context. - def fn(three): - replica_ctx = tf.get_replica_context() - v = three + replica_ctx.replica_id - # Computes the sum of the `v` values across all replicas. - s = replica_ctx.merge_call(merge_fn, args=(v,)) - return s + v - - with distribution.scope(): - # in "cross-replica" context - ... - merged_results = distribution.call_for_each_replica(fn, args=[3]) - # merged_results has the values from every replica execution of `fn`. - print(distribution.unwrap(merged_results)) # Prints a list - ``` - - Args: - fn: function to run (will be run once per replica). - args: Tuple or list with positional arguments for `fn`. - kwargs: Dict with keyword arguments for `fn`. - - Returns: - Merged return value of `fn` across all replicas. - """ - _require_cross_replica_context(self) - # Handle old *args, **kwargs, and new args=(...), kwargs={...}, to - # allow transition. - a = kwargs.pop("args", None) - if a is not None: - if args: - raise ValueError( - "Can't pass *args and args=... to call_for_each_replica") - args = a - k = kwargs.pop("kwargs", None) - if k is not None: - if kwargs: - raise ValueError( - "Can't pass **kwargs and kwargs=... to call_for_each_replica") - kwargs = k - kwargs.pop("run_concurrently", None) # Ignore old option. - return self._call_for_each_replica(fn, args, kwargs) - - def _call_for_each_replica(self, fn, args, kwargs): - raise NotImplementedError("must be implemented in descendants") - - def reduce(self, aggregation, value, destinations): - """Combine (via e.g. sum or mean) values across replicas. - - Args: - aggregation: Indicates how a variable will be aggregated. Accepted values - are `tf.VariableAggregation.SUM`, `tf.VariableAggregation.MEAN`, - `tf.VariableAggregation.ONLY_FIRST_REPLICA`. - value: A per-replica value with one value per replica. - destinations: A mirrored variable, a per-replica tensor, a device string, - or list of device strings. The return value will be copied to all - destination devices (or all the devices where the `destinations` value - resides). To perform an all-reduction, pass `value` to `destinations`. - - Returns: - A value mirrored to `destinations`. - """ - # TODO(josh11b): More docstring - # TODO(josh11b): Return an unwrapped value if colocate_with is a - # single device. - _require_cross_replica_context(self) - assert aggregation in [ - variable_scope.VariableAggregation.SUM, - variable_scope.VariableAggregation.MEAN, - variable_scope.VariableAggregation.ONLY_FIRST_REPLICA - ] - return self._reduce(aggregation, value, destinations) - - def _reduce(self, aggregation, value, destinations): - raise NotImplementedError("must be implemented in descendants") - - def batch_reduce(self, aggregation, value_destination_pairs): - """Combine multiple `reduce` calls into one for faster execution. - - Args: - aggregation: Indicates how a variable will be aggregated. Accepted values - are `tf.VariableAggregation.SUM`, `tf.VariableAggregation.MEAN`, - `tf.VariableAggregation.ONLY_FIRST_REPLICA`. - value_destination_pairs: A sequence of (value, destinations) - pairs. See `reduce()` for a description. - - Returns: - A list of mirrored values, one per pair in `value_destination_pairs`. - """ - # TODO(josh11b): More docstring - _require_cross_replica_context(self) - assert aggregation in [ - variable_scope.VariableAggregation.SUM, - variable_scope.VariableAggregation.MEAN, - variable_scope.VariableAggregation.ONLY_FIRST_REPLICA - ] - return self._batch_reduce(aggregation, value_destination_pairs) - - def _batch_reduce(self, aggregation, value_destination_pairs): - return [ - self.reduce(aggregation, t, destinations=v) - for t, v in value_destination_pairs - ] - - def update(self, var, fn, *args, **kwargs): - """Run `fn` to update `var` using inputs mirrored to the same devices. - - If `var` is mirrored across multiple devices, then this implements - logic like: - - ``` - results = {} - for device, v in var: - with tf.device(device): - # *args and **kwargs will be unwrapped if they are mirrored. - results[device] = fn(v, *args, **kwargs) - return merged(results) - ``` - - Otherwise this returns `fn(var, *args, **kwargs)` colocated with `var`. - - Neither `*args` nor `**kwargs` may contain per-replica values. - If they contain mirrored values, they will be unwrapped before - calling `fn`. - - Args: - var: Variable, possibly mirrored to multiple devices, to operate on. - fn: Function to call. Should take the variable as the first argument. - *args: Additional positional arguments to pass to `fn()`. - **kwargs: Keyword arguments to pass to `fn()`. If "grouped=False" is - specified, the return value will be unwrapped. - - Returns: - By default, the merged return value of `fn` across all replicas. The - merged result has dependencies to make sure that if it is evaluated at - all, the side effects (updates) will happen on every replica. If instead - "grouped=False" is specified, this function will return a nest of lists - where each list has an element per replica, and the caller is responsible - for ensuring all elements are executed. - """ - _require_cross_replica_context(self) - options = {"grouped": kwargs.pop("grouped", True)} - return self._update(var, options, fn, *args, **kwargs) - - def _update(self, var, options, fn, *args, **kwargs): - raise NotImplementedError("must be implemented in descendants") - - def update_non_slot(self, colocate_with, fn, *args, **kwargs): - """Runs `fn(*args, **kwargs)` on `colocate_with` devices. - - Args: - colocate_with: The return value of `non_slot_devices()`. - fn: Function to execute. - *args: Positional arguments to pass to `fn()`. - **kwargs: Keyword arguments to pass to `fn()`. If "grouped=False" is - specified, the return value will be unwrapped and the caller is - responsible for ensuring all elements are executed. - - Returns: - Return value of `fn`, possibly merged across devices. - """ - _require_cross_replica_context(self) - options = {"grouped": kwargs.pop("grouped", True)} - return self._update_non_slot(colocate_with, options, fn, *args, **kwargs) - - def _update_non_slot(self, colocate_with, options, fn, *args, **kwargs): - raise NotImplementedError("must be implemented in descendants") - - def unwrap(self, value): - """Returns the list of all per-replica values contained in `value`. - - Args: - value: A value returned by `call_for_each_replica()` or a variable - created in `scope()`. - - Returns: - A list of values contained in `value`. If `value` represents a single - value, this returns `[value].` - """ - return self._unwrap(value) - - def value_container(self, value): - """Returns the container that this per-replica `value` belongs to. - - Args: - value: A value returned by `call_for_each_replica()` or a variable - created in `scope()`. - - Returns: - A container that `value` belongs to. - If value does not belong to any container (including the case of - container having been destroyed), returns the value itself. - `value in unwrap(value_container(value))` will always be true. - """ - raise NotImplementedError("must be implemented in descendants") - - def _unwrap(self, distributed_value): - raise NotImplementedError("must be implemented in descendants") - - def group(self, value, name=None): - """Shortcut for `tf.group(distribution.unwrap(value))`.""" - value = nest.flatten(self.unwrap(value)) - - if len(value) != 1 or name is not None: - return control_flow_ops.group(value, name=name) - # Special handling for the common case of one op. - v, = value - if hasattr(v, "op"): - v = v.op - return v - - @property - def require_static_shapes(self): - return self._require_static_shapes - - @property - def num_replicas(self): - """Returns number of replicas, for purposes of averaging across replicas. - - DEPRECATED: use `num_replicas_in_sync` instead. - """ - raise NotImplementedError("must be implemented in descendants") - - @property - def num_replicas_in_sync(self): - """Returns number of replicas over which gradients are aggregated.""" - raise NotImplementedError("must be implemented in descendants") - - @property - def worker_devices(self): - """Returns the list of devices used to run `call_for_each_replica()` calls. - """ - # TODO(josh11b): More docstring - raise NotImplementedError("must be implemented in descendants") - - @property - def parameter_devices(self): - """Returns the list of devices used for variable and `update` placement.""" - # TODO(josh11b): More docstring - raise NotImplementedError("must be implemented in descendants") - - def non_slot_devices(self, var_list): - """Device(s) for non-slot variables. - - Create variables on these devices in a - `with colocate_vars_with(non_slot_devices(...)):` block. - Update those using `update_non_slot()`. - - Args: - var_list: The list of variables being optimized, needed with the - default `DistributionStrategy`. - """ - raise NotImplementedError("must be implemented in descendants") - - @property - def worker_device_index(self): - """An object mapping worker device to an id. - - This might be passed as an argument to `call_for_each_replica()`, as in: - - ``` - with distribution_strategy.scope(): - - def fn(device_id): - # device_id is an integer. `fn` is being executed on device: - # distribution_strategy.worker_devices[device_id]. - - distribution_strategy.call_for_each_replica( - fn, distribution_strategy.worker_device_index) - ``` - - Returns: - An index object, or the integer 0 if there is only a single replica. - """ - _require_cross_replica_context(self) - return self._worker_device_index() - - def _worker_device_index(self): - raise NotImplementedError("must be implemented in descendants") - - @property - def between_graph(self): - """Whether the strategy uses between-graph replication or not. - - This is expected to return a constant value that will not be changed - throughout its life cycle. - """ - raise NotImplementedError("must be implemented in descendants") - - def configure(self, - session_config=None, - cluster_spec=None, - task_type=None, - task_id=None): - """Configures the strategy class.""" - del session_config, cluster_spec, task_type, task_id - - @property - def should_init(self): - """Whether initialization is needed.""" - raise NotImplementedError("must be implemented in descendants") - - @property - def should_checkpoint(self): - """Whether checkpointing is needed.""" - raise NotImplementedError("must be implemented in descendants") - - @property - def should_save_summary(self): - """Whether saving summaries is needed.""" - raise NotImplementedError("must be implemented in descendants") - - -# A note about the difference between the context managers -# `ReplicaContext` (defined here) and `_CurrentDistributionContext` -# (defined above) used by `DistributionStrategy.scope()`: -# -# * a ReplicaContext is only present during a `call_for_each_replica()` -# call (except during a `merge_run` call) and in such a scope it -# will be returned by calls to `get_replica_context()`. Implementers of new -# DistributionStrategy descendants will frequently also need to -# define a descendant of ReplicaContext, and are responsible for -# entering and exiting this context. -# -# * DistributionStrategy.scope() sets up a variable_creator scope that -# changes variable creation calls (e.g. to make mirrored -# variables). This is intended as an outer scope that users enter once -# around their model creation and graph definition. There is no -# anticipated need to define descendants of _CurrentDistributionContext. -# It sets the current DistributionStrategy for purposes of -# `get_distribution_strategy()` and `has_distribution_strategy()` -# and switches the thread mode to a "cross-replica context". -class ReplicaContext(object): - """DistributionStrategy API inside a `call_for_each_replica()` call.""" - - def __init__(self, distribution_strategy, replica_id): - self._distribution_strategy = distribution_strategy - self._thread_context = distribution_strategy_context._InReplicaThreadMode( # pylint: disable=protected-access - self) - self._replica_id = replica_id - - def __enter__(self): - _push_per_thread_mode(self._thread_context) - - def __exit__(self, exception_type, exception_value, traceback): - _pop_per_thread_mode() - - def merge_call(self, merge_fn, *args, **kwargs): - """Merge args across replicas and run `merge_fn` in a cross-replica context. - - This allows communication and coordination when there are multiple calls - to a model function triggered by a call to - `distribution.call_for_each_replica(model_fn, ...)`. - - See `MirroredDistribution.call_for_each_replica()` for an explanation. - - Otherwise, this is equivalent to: - - ``` - distribution = get_distribution_strategy() - with cross-replica-context(distribution): - return merge_fn(distribution, *args, **kwargs) - ``` - - Args: - merge_fn: function that joins arguments from threads that are given as - PerReplica. It accepts `DistributionStrategy` object as the first - argument. - args: List or tuple with positional per-thread arguments for `merge_fn` - kwargs: Dict with keyword per-thread arguments for `merge_fn`. - - Returns: - The return value of `merge_fn`, except for `PerReplica` values which are - unpacked. - """ - require_replica_context(self) - # Handle old *args, **kwargs, and new args=(...), kwargs={...}, to - # allow transition. - a = kwargs.pop("args", None) - if a is not None: - if args: - raise ValueError( - "Can't pass *args and args=... to merge_call") - args = a - k = kwargs.pop("kwargs", None) - if k is not None: - if kwargs: - raise ValueError( - "Can't pass **kwargs and kwargs=... to merge_call") - kwargs = k - return self._merge_call(merge_fn, args, kwargs) - - def _merge_call(self, merge_fn, args, kwargs): - """Default implementation for single replica.""" - _push_per_thread_mode( # thread-local, so not needed with multiple threads - distribution_strategy_context._CrossReplicaThreadMode( # pylint: disable=protected-access - self._distribution_strategy)) - try: - return merge_fn(self._distribution_strategy, *args, **kwargs) - finally: - _pop_per_thread_mode() - - @property - def num_replicas(self): - """Returns number of replicas, for purposes of averaging across replicas.""" - return self._distribution_strategy.num_replicas - - @property - def num_replicas_in_sync(self): - """Returns number of replicas over which gradients are aggregated.""" - return self._distribution_strategy.num_replicas_in_sync - - @property - def replica_id(self): - """Which replica is being defined, a number from 0 to `num_replicas - 1`.""" - require_replica_context(self) - return self._replica_id - - @property - def distribution_strategy(self): - """The current `DistributionStrategy` object.""" - return self._distribution_strategy - - @property - def device(self): - """BEING DELETED: use .devices instead.""" - raise RuntimeError("Use .devices instead") - - @property - def devices(self): - """The devices this replica is to be executed on, as a list of strings.""" - require_replica_context(self) - return [device_util.current()] - - # TODO(josh11b): Implement `start_all_reduce(method, t)` for efficient - # all-reduce. It would return a function returning the result of reducing `t` - # across all replicas. The caller would wait to call this function until they - # needed the reduce result, allowing an efficient implementation: - # * With eager execution, the reduction could be performed asynchronously - # in the background, not blocking until the result was needed. - # * When constructing a graph, it could batch up all reduction requests up - # to that point that the first result is needed. Most likely this can be - # implemented in terms of `merge_call()` and `batch_reduce()`. - -# ------------------------------------------------------------------------------ - - -class _DefaultDistributionStrategy(DistributionStrategy): - """Default `DistributionStrategy` if none is explicitly selected.""" - - def scope(self): - """Context manager setting a variable creator and `self` as current.""" - if distribution_strategy_context.has_distribution_strategy(): - raise RuntimeError("Must not nest DistributionStrategy scopes.") - - def creator(next_creator, *args, **kwargs): - _require_distribution_strategy_scope(self) - return next_creator(*args, **kwargs) - - return _CurrentDistributionContext( - self, variable_scope.variable_creator_scope(creator)) - - def colocate_vars_with(self, colocate_with_variable): - """Does not require `self.scope`.""" - _require_distribution_strategy_scope(self) - return ops.colocate_with(colocate_with_variable) - - def distribute_dataset(self, dataset_fn): - return self._call_dataset_fn(dataset_fn) - - def _broadcast(self, tensor, destinations): - if destinations is None: - return tensor - else: - raise NotImplementedError("TODO") - - def _call_for_each_replica(self, fn, args, kwargs): - with ReplicaContext(self, replica_id=0): - return fn(*args, **kwargs) - - def _reduce(self, aggregation, value, destinations): - # TODO(josh11b): Use destinations? - del aggregation, destinations - return value - - def _update(self, var, options, fn, *args, **kwargs): - # The implementations of _update() and _update_non_slot() are identical - # except _update() passes `var` as the first argument to `fn()`. - return self._update_non_slot(var, options, fn, var, *args, **kwargs) - - def _update_non_slot(self, colocate_with, options, fn, *args, **kwargs): - should_group = options.pop("grouped") - assert not options # Validate that we are processing all of the options. - # TODO(josh11b): Figure out what we should be passing to UpdateContext() - # once that value is used for something. - with ops.colocate_with(colocate_with), UpdateContext(colocate_with): - result = fn(*args, **kwargs) - if should_group: - return result - else: - return nest.map_structure(self._unwrap, result) - - def read_var(self, replica_local_var): - return array_ops.identity(replica_local_var) - - def _unwrap(self, distributed_value): - return [distributed_value] - - def value_container(self, value): - return value - - @property - def num_replicas(self): - return 1 - - @property - def num_replicas_in_sync(self): - return 1 - - @property - def worker_devices(self): - raise RuntimeError( - "worker_devices() method unsupported by _DefaultDistributionStrategy.") - - @property - def parameter_devices(self): - raise RuntimeError("parameter_devices() method unsupported by " - "_DefaultDistributionStrategy.") - - def non_slot_devices(self, var_list): - return min(var_list, key=lambda x: x.name) - - def _worker_device_index(self): - raise RuntimeError("worker_device_index() method unsupported by " - "_DefaultDistributionStrategy.") - - -# ------------------------------------------------------------------------------ -# We haven't yet implemented deserialization for DistributedVariables. -# So here we catch any attempts to deserialize variables -# when using distribution strategies. -# pylint: disable=protected-access -_original_from_proto = resource_variable_ops._from_proto_fn - - -def _from_proto_fn(v, import_scope=None): - if distribution_strategy_context.has_distribution_strategy(): - raise NotImplementedError( - "Deserialization of variables is not yet supported when using" - "distributed strategies.") - else: - return _original_from_proto(v, import_scope=import_scope) - -resource_variable_ops._from_proto_fn = _from_proto_fn -# pylint: enable=protected-access - - -#------------------------------------------------------------------------------- -# Shorthand for some methods from distribution_strategy_context. -_push_per_thread_mode = distribution_strategy_context._push_per_thread_mode # pylint: disable=protected-access -_get_per_thread_mode = distribution_strategy_context._get_per_thread_mode # pylint: disable=protected-access -_pop_per_thread_mode = distribution_strategy_context._pop_per_thread_mode # pylint: disable=protected-access +# pylint: disable=wildcard-import +from tensorflow.python.distribute.distribute_lib import * diff --git a/tensorflow/python/training/distribution_strategy_context.py b/tensorflow/python/training/distribution_strategy_context.py index 278f35b97e4..7391bf3b22d 100644 --- a/tensorflow/python/training/distribution_strategy_context.py +++ b/tensorflow/python/training/distribution_strategy_context.py @@ -12,195 +12,11 @@ # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================== -"""Utility to get distribution strategy related contexts.""" +"""Deprecated, please use ../distribute/distribution_strategy_context.py.""" from __future__ import absolute_import from __future__ import division from __future__ import print_function -from tensorflow.python.framework import ops -from tensorflow.python.util.lazy_loader import LazyLoader - - -# There is a circular dependency between this and `distribute` module. So we -# load it lazily to workaround this. -distribute_lib = LazyLoader( - "distribute_lib", globals(), - "tensorflow.python.training.distribute") - -# ------------------------------------------------------------------------------ -# Internal API for setting the current thread mode as being either in a -# replica or cross-replica context for a particular distribution strategy. - - -class _ThreadMode(object): - - def __init__(self, dist, cross, replica): - self.distribution_strategy = dist - self.cross_replica_context = cross - self.replica_context = replica - - -class _CrossReplicaThreadMode(_ThreadMode): - - def __init__(self, distribution_strategy): - _ThreadMode.__init__( - self, distribution_strategy, distribution_strategy, None) - - -class _InReplicaThreadMode(_ThreadMode): - - def __init__(self, replica_ctx): - _ThreadMode.__init__( - self, replica_ctx.distribution_strategy, None, replica_ctx) - - -def _push_per_thread_mode(context): - ops.get_default_graph()._distribution_strategy_stack.append(context) # pylint: disable=protected-access - - -def _pop_per_thread_mode(): - ops.get_default_graph()._distribution_strategy_stack.pop(-1) # pylint: disable=protected-access - - -class _DefaultReplicaThreadMode(_ThreadMode): - """Type of default value returned by `_get_per_thread_mode()`. - - Used when the thread-local stack is empty. - """ - - def __init__(self): - _ThreadMode.__init__(self, _get_default_distribution_strategy(), None, - _get_default_replica_context()) - - -def _get_per_thread_mode(): - try: - return ops.get_default_graph()._distribution_strategy_stack[-1] # pylint: disable=protected-access - except (AttributeError, IndexError): - return _get_default_replica_mode() - - -# ------------------------------------------------------------------------------ -# Public API for accessing the current thread mode - - -def get_replica_context(): - """Returns the current ReplicaContext or None if in a cross-replica context. - - Note that execution: - - 1. starts in the default (single-replica) replica context (this function - will return the default ReplicaContext object); - 2. switches to cross-replica context (in which case this will return - None) when entering a `with DistributionStrategy.scope():` block; - 3. switches to a (non-default) replica context inside - `call_for_each_replica(fn, ...)`; - 4. if `fn` calls `get_replica_context()->merge_call(merge_fn, ...)`, then - inside `merge_fn` you are back in the cross-replica context (and again - this function will return None). - - Note that you can also go directly from step 1 to 4 to switch to a - cross-replica context for the default `DistributionStrategy`. You may - also switch from the cross-replica context of 4 to a replica context by - calling `call_for_each_replica()`, jumping back to step 3. - - Most `DistributionStrategy` methods may only be executed in - a cross-replica context, in a replica context you should use the - `ReplicaContext` API instead. - - Returns: - The current `ReplicaContext` object when in a replica context scope, - else None. - - Exactly one of `get_replica_context()` and `get_cross_replica_context()` - will return None in a particular block. - """ - return _get_per_thread_mode().replica_context - - -def get_cross_replica_context(): - """Returns the current DistributionStrategy if in a cross-replica context. - - Note that execution: - - 1. starts in the default (single-replica) replica context; - 2. switches to cross-replica context when entering a - `with DistributionStrategy.scope():` block; - 3. switches to a (non-default) replica context inside - `call_for_each_replica(fn, ...)`; - 4. if `fn` calls `get_replica_context()->merge_call(merge_fn, ...)`, then - inside `merge_fn` you are back in the cross-replica context. - - Note that you can also go directly from step 1 to 4 to switch to a - cross-replica context for the default `DistributionStrategy`. You may - also switch from the cross-replica context of 4 to a replica context by - calling `call_for_each_replica()`, jumping back to step 3. - - Most `DistributionStrategy` methods may only be executed in - a cross-replica context. - - Returns: - Returns the current `DistributionStrategy` object in a cross-replica - context, or None. - - Exactly one of `get_replica_context()` and `get_cross_replica_context()` - will return None in a particular block. - """ - return _get_per_thread_mode().cross_replica_context - - -def get_distribution_strategy(): - """Returns the current `DistributionStrategy` object. - - Prefer to use `get_replica_context()` or `get_cross_replica_context()` - instead when possible. - - Returns: - A `DistributionStrategy` object. Inside a - `with distribution_strategy.scope()` block, it returns - `distribution_strategy`, otherwise it returns the default - (single-replica) `DistributionStrategy` object. - """ - return _get_per_thread_mode().distribution_strategy - - -def has_distribution_strategy(): - """Return if there is a current non-default `DistributionStrategy`. - - Returns: - True if inside a `with distribution_strategy.scope():`. - """ - return get_distribution_strategy() is not _get_default_distribution_strategy() - - -# ------------------------------------------------------------------------------ -# Defaults that are used when no distribution strategy is explicitly created. -# We create them lazily in a function so that we can workaround the circular -# dependency on distribute_lib. See lazy loader at the top of this file. - -_defaults = { - "distribution_strategy": None, - "replica_context": None, - "replica_mode": None -} - - -def _get_default_distribution_strategy(): - if _defaults["distribution_strategy"] is None: - _defaults["distribution_strategy"] = ( - distribute_lib._DefaultDistributionStrategy()) # pylint: disable=protected-access - return _defaults["distribution_strategy"] - - -def _get_default_replica_context(): - if _defaults["replica_context"] is None: - _defaults["replica_context"] = distribute_lib.ReplicaContext( - _get_default_distribution_strategy(), replica_id=0) - return _defaults["replica_context"] - - -def _get_default_replica_mode(): - if _defaults["replica_mode"] is None: - _defaults["replica_mode"] = _DefaultReplicaThreadMode() - return _defaults["replica_mode"] +# pylint: disable=wildcard-import +from tensorflow.python.distribute.distribution_strategy_context import * diff --git a/tensorflow/python/training/evaluation.py b/tensorflow/python/training/evaluation.py index 2c4eb02d533..a10178f8cfe 100644 --- a/tensorflow/python/training/evaluation.py +++ b/tensorflow/python/training/evaluation.py @@ -230,7 +230,7 @@ def _evaluate_once(checkpoint_path, hooks = list(hooks or []) if eval_ops is not None: - if any([isinstance(h, _MultiStepStopAfterNEvalsHook) for h in hooks]): + if any(isinstance(h, _MultiStepStopAfterNEvalsHook) for h in hooks): steps_per_run_variable = \ basic_session_run_hooks.get_or_create_steps_per_run_variable() update_eval_step = state_ops.assign_add( diff --git a/tensorflow/python/training/ftrl.py b/tensorflow/python/training/ftrl.py index 2fafc9a2d80..a2ef3c76b4e 100644 --- a/tensorflow/python/training/ftrl.py +++ b/tensorflow/python/training/ftrl.py @@ -25,7 +25,7 @@ from tensorflow.python.training import training_ops from tensorflow.python.util.tf_export import tf_export -@tf_export("train.FtrlOptimizer") +@tf_export(v1=["train.FtrlOptimizer"]) class FtrlOptimizer(optimizer.Optimizer): """Optimizer that implements the FTRL algorithm. diff --git a/tensorflow/python/training/ftrl_test.py b/tensorflow/python/training/ftrl_test.py index 15c50bc8788..39b299c64a3 100644 --- a/tensorflow/python/training/ftrl_test.py +++ b/tensorflow/python/training/ftrl_test.py @@ -23,6 +23,7 @@ import numpy as np from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops +from tensorflow.python.framework import test_util from tensorflow.python.ops import embedding_ops from tensorflow.python.ops import math_ops from tensorflow.python.ops import resource_variable_ops @@ -54,7 +55,7 @@ class FtrlOptimizerTest(test.TestCase): update = opt.apply_gradients(zip([grads0, grads1], [var0, var1])) variables.global_variables_initializer().run() - v0_val, v1_val = sess.run([var0, var1]) + v0_val, v1_val = self.evaluate([var0, var1]) self.assertAllClose([0.0, 0.0], v0_val) self.assertAllClose([0.0, 0.0], v1_val) @@ -62,18 +63,21 @@ class FtrlOptimizerTest(test.TestCase): for _ in range(3): update.run() - v0_val, v1_val = sess.run([var0, var1]) + v0_val, v1_val = self.evaluate([var0, var1]) self.assertAllCloseAccordingToType( np.array([-2.60260963, -4.29698515]), v0_val) self.assertAllCloseAccordingToType( np.array([-0.28432083, -0.56694895]), v1_val) + @test_util.run_deprecated_v1 def testFtrlWithoutRegularization(self): self.doTestFtrlwithoutRegularization(use_resource=False) + @test_util.run_deprecated_v1 def testResourceFtrlWithoutRegularization(self): self.doTestFtrlwithoutRegularization(use_resource=True) + @test_util.run_deprecated_v1 def testFtrlwithoutRegularization2(self): for dtype in [dtypes.half, dtypes.float32]: with self.cached_session() as sess: @@ -90,19 +94,20 @@ class FtrlOptimizerTest(test.TestCase): update = opt.apply_gradients(zip([grads0, grads1], [var0, var1])) variables.global_variables_initializer().run() - v0_val, v1_val = sess.run([var0, var1]) + v0_val, v1_val = self.evaluate([var0, var1]) self.assertAllCloseAccordingToType([1.0, 2.0], v0_val) self.assertAllCloseAccordingToType([4.0, 3.0], v1_val) # Run 3 steps FTRL for _ in range(3): update.run() - v0_val, v1_val = sess.run([var0, var1]) + v0_val, v1_val = self.evaluate([var0, var1]) self.assertAllCloseAccordingToType( np.array([-2.55607247, -3.98729396]), v0_val) self.assertAllCloseAccordingToType( np.array([-0.28232238, -0.56096673]), v1_val) + @test_util.run_deprecated_v1 def testMinimizeSparseResourceVariable(self): for dtype in [dtypes.half, dtypes.float32, dtypes.float64]: with self.cached_session(): @@ -113,12 +118,15 @@ class FtrlOptimizerTest(test.TestCase): sgd_op = ftrl.FtrlOptimizer(1.0).minimize(loss) variables.global_variables_initializer().run() # Fetch params to validate initial values - self.assertAllCloseAccordingToType([[1.0, 2.0]], var0.eval()) + self.assertAllCloseAccordingToType([[1.0, 2.0]], self.evaluate(var0)) # Run 1 step of sgd sgd_op.run() # Validate updated params - self.assertAllCloseAccordingToType([[0, 1]], var0.eval(), atol=0.01) + self.assertAllCloseAccordingToType([[0, 1]], + self.evaluate(var0), + atol=0.01) + @test_util.run_deprecated_v1 def testFtrlWithL1(self): for dtype in [dtypes.half, dtypes.float32]: with self.cached_session() as sess: @@ -135,19 +143,20 @@ class FtrlOptimizerTest(test.TestCase): update = opt.apply_gradients(zip([grads0, grads1], [var0, var1])) variables.global_variables_initializer().run() - v0_val, v1_val = sess.run([var0, var1]) + v0_val, v1_val = self.evaluate([var0, var1]) self.assertAllCloseAccordingToType([1.0, 2.0], v0_val) self.assertAllCloseAccordingToType([4.0, 3.0], v1_val) # Run 10 steps FTRL for _ in range(10): update.run() - v0_val, v1_val = sess.run([var0, var1]) + v0_val, v1_val = self.evaluate([var0, var1]) self.assertAllCloseAccordingToType( np.array([-7.66718769, -10.91273689]), v0_val) self.assertAllCloseAccordingToType( np.array([-0.93460727, -1.86147261]), v1_val) + @test_util.run_deprecated_v1 def testFtrlWithL1_L2(self): for dtype in [dtypes.half, dtypes.float32]: with self.cached_session() as sess: @@ -164,7 +173,7 @@ class FtrlOptimizerTest(test.TestCase): update = opt.apply_gradients(zip([grads0, grads1], [var0, var1])) variables.global_variables_initializer().run() - v0_val, v1_val = sess.run([var0, var1]) + v0_val, v1_val = self.evaluate([var0, var1]) self.assertAllCloseAccordingToType([1.0, 2.0], v0_val) self.assertAllCloseAccordingToType([4.0, 3.0], v1_val) @@ -172,12 +181,13 @@ class FtrlOptimizerTest(test.TestCase): for _ in range(10): update.run() - v0_val, v1_val = sess.run([var0, var1]) + v0_val, v1_val = self.evaluate([var0, var1]) self.assertAllCloseAccordingToType( np.array([-0.24059935, -0.46829352]), v0_val) self.assertAllCloseAccordingToType( np.array([-0.02406147, -0.04830509]), v1_val) + @test_util.run_deprecated_v1 def testFtrlWithL1_L2_L2Shrinkage(self): """Test the new FTRL op with support for l2 shrinkage. @@ -201,7 +211,7 @@ class FtrlOptimizerTest(test.TestCase): update = opt.apply_gradients(zip([grads0, grads1], [var0, var1])) variables.global_variables_initializer().run() - v0_val, v1_val = sess.run([var0, var1]) + v0_val, v1_val = self.evaluate([var0, var1]) self.assertAllCloseAccordingToType([1.0, 2.0], v0_val) self.assertAllCloseAccordingToType([4.0, 3.0], v1_val) @@ -209,12 +219,13 @@ class FtrlOptimizerTest(test.TestCase): for _ in range(10): update.run() - v0_val, v1_val = sess.run([var0, var1]) + v0_val, v1_val = self.evaluate([var0, var1]) self.assertAllCloseAccordingToType( np.array([-0.22578995, -0.44345796]), v0_val) self.assertAllCloseAccordingToType( np.array([-0.14378493, -0.13229476]), v1_val) + @test_util.run_deprecated_v1 def testFtrlWithL1_L2_L2ShrinkageSparse(self): """Tests the new FTRL op with support for l2 shrinkage on sparse grads.""" for dtype in [dtypes.half, dtypes.float32]: @@ -237,7 +248,7 @@ class FtrlOptimizerTest(test.TestCase): update = opt.apply_gradients(zip([grads0, grads1], [var0, var1])) variables.global_variables_initializer().run() - v0_val, v1_val = sess.run([var0, var1]) + v0_val, v1_val = self.evaluate([var0, var1]) self.assertAllCloseAccordingToType([[1.0], [2.0]], v0_val) self.assertAllCloseAccordingToType([[4.0], [3.0]], v1_val) @@ -245,10 +256,11 @@ class FtrlOptimizerTest(test.TestCase): for _ in range(10): update.run() - v0_val, v1_val = sess.run([var0, var1]) + v0_val, v1_val = self.evaluate([var0, var1]) self.assertAllCloseAccordingToType([[-0.22578995], [2.]], v0_val) self.assertAllCloseAccordingToType([[4.], [-0.13229476]], v1_val) + @test_util.run_deprecated_v1 def testFtrlWithL2ShrinkageDoesNotChangeLrSchedule(self): """Verifies that l2 shrinkage in FTRL does not change lr schedule.""" for dtype in [dtypes.half, dtypes.float32]: @@ -273,7 +285,7 @@ class FtrlOptimizerTest(test.TestCase): update1 = opt1.apply_gradients([(grads1, var1)]) variables.global_variables_initializer().run() - v0_val, v1_val = sess.run([var0, var1]) + v0_val, v1_val = self.evaluate([var0, var1]) self.assertAllCloseAccordingToType([1.0, 2.0], v0_val) self.assertAllCloseAccordingToType([1.0, 2.0], v1_val) @@ -282,12 +294,12 @@ class FtrlOptimizerTest(test.TestCase): update0.run() update1.run() - v0_val, v1_val = sess.run([var0, var1]) + v0_val, v1_val = self.evaluate([var0, var1]) # var0 is experiencing L2 shrinkage so it should be smaller than var1 # in magnitude. self.assertTrue((v0_val**2 < v1_val**2).all()) - accum0 = list(sess.run(opt0._slots)["accum"].values())[0] - accum1 = list(sess.run(opt1._slots)["accum"].values())[0] + accum0 = list(self.evaluate(opt0._slots)["accum"].values())[0] + accum1 = list(self.evaluate(opt1._slots)["accum"].values())[0] # L2 shrinkage should not change how we update grad accumulator. self.assertAllCloseAccordingToType(accum0, accum1) @@ -311,7 +323,7 @@ class FtrlOptimizerTest(test.TestCase): variables.global_variables_initializer().run() sess = ops.get_default_session() - v0_val, v1_val = sess.run([var0, var1]) + v0_val, v1_val = self.evaluate([var0, var1]) if is_sparse: self.assertAllCloseAccordingToType([[0.0], [0.0]], v0_val) self.assertAllCloseAccordingToType([[0.0], [0.0]], v1_val) @@ -323,7 +335,7 @@ class FtrlOptimizerTest(test.TestCase): for _ in range(steps): update.run() - v0_val, v1_val = sess.run([var0, var1]) + v0_val, v1_val = self.evaluate([var0, var1]) return v0_val, v1_val # When variables are initialized with Zero, FTRL-Proximal has two properties: @@ -333,6 +345,7 @@ class FtrlOptimizerTest(test.TestCase): # with Adagrad. # So, basing on these two properties, we test if our implementation of # FTRL-Proximal performs same updates as Adagrad or GradientDescent. + @test_util.run_deprecated_v1 def testEquivAdagradwithoutRegularization(self): for dtype in [dtypes.half, dtypes.float32]: with self.cached_session(): @@ -353,6 +366,7 @@ class FtrlOptimizerTest(test.TestCase): self.assertAllCloseAccordingToType(val0, val2) self.assertAllCloseAccordingToType(val1, val3) + @test_util.run_deprecated_v1 def testEquivSparseAdagradwithoutRegularization(self): for dtype in [dtypes.half, dtypes.float32]: with self.cached_session(): @@ -376,6 +390,7 @@ class FtrlOptimizerTest(test.TestCase): self.assertAllCloseAccordingToType(val0, val2) self.assertAllCloseAccordingToType(val1, val3) + @test_util.run_deprecated_v1 def testEquivSparseGradientDescentwithoutRegularization(self): for dtype in [dtypes.half, dtypes.float32]: with self.cached_session(): @@ -399,6 +414,7 @@ class FtrlOptimizerTest(test.TestCase): self.assertAllCloseAccordingToType(val0, val2) self.assertAllCloseAccordingToType(val1, val3) + @test_util.run_deprecated_v1 def testEquivGradientDescentwithoutRegularization(self): for dtype in [dtypes.half, dtypes.float32]: with self.cached_session(): diff --git a/tensorflow/python/training/gradient_descent.py b/tensorflow/python/training/gradient_descent.py index ef50f6315dd..1a527345ef6 100644 --- a/tensorflow/python/training/gradient_descent.py +++ b/tensorflow/python/training/gradient_descent.py @@ -26,7 +26,7 @@ from tensorflow.python.training import training_ops from tensorflow.python.util.tf_export import tf_export -@tf_export("train.GradientDescentOptimizer") +@tf_export(v1=["train.GradientDescentOptimizer"]) class GradientDescentOptimizer(optimizer.Optimizer): """Optimizer that implements the gradient descent algorithm. """ diff --git a/tensorflow/python/training/gradient_descent_test.py b/tensorflow/python/training/gradient_descent_test.py index 1ddea598e52..5a6c5cfa747 100644 --- a/tensorflow/python/training/gradient_descent_test.py +++ b/tensorflow/python/training/gradient_descent_test.py @@ -24,6 +24,7 @@ from tensorflow.python.eager import function from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops +from tensorflow.python.framework import test_util from tensorflow.python.ops import embedding_ops from tensorflow.python.ops import math_ops from tensorflow.python.ops import resource_variable_ops @@ -35,6 +36,7 @@ from tensorflow.python.training import gradient_descent class GradientDescentOptimizerTest(test.TestCase): + @test_util.run_deprecated_v1 def testBasic(self): for dtype in [dtypes.half, dtypes.float32, dtypes.float64]: with self.cached_session(): @@ -47,17 +49,18 @@ class GradientDescentOptimizerTest(test.TestCase): zip([grads0, grads1], [var0, var1])) variables.global_variables_initializer().run() # Fetch params to validate initial values - self.assertAllCloseAccordingToType([1.0, 2.0], var0.eval()) - self.assertAllCloseAccordingToType([3.0, 4.0], var1.eval()) + self.assertAllCloseAccordingToType([1.0, 2.0], self.evaluate(var0)) + self.assertAllCloseAccordingToType([3.0, 4.0], self.evaluate(var1)) # Run 1 step of sgd sgd_op.run() # Validate updated params self.assertAllCloseAccordingToType([1.0 - 3.0 * 0.1, 2.0 - 3.0 * 0.1], - var0.eval()) + self.evaluate(var0)) self.assertAllCloseAccordingToType([3.0 - 3.0 * 0.01, 4.0 - 3.0 * 0.01], - var1.eval()) + self.evaluate(var1)) self.assertEqual(0, len(optimizer.variables())) + @test_util.run_deprecated_v1 def testBasicResourceVariable(self): for dtype in [dtypes.half, dtypes.float32, dtypes.float64]: with self.cached_session(): @@ -73,16 +76,17 @@ class GradientDescentOptimizerTest(test.TestCase): # a long-term solution for this. resources.initialize_resources([var0, var1]).run() # Fetch params to validate initial values - self.assertAllCloseAccordingToType([1.0, 2.0], var0.eval()) - self.assertAllCloseAccordingToType([3.0, 4.0], var1.eval()) + self.assertAllCloseAccordingToType([1.0, 2.0], self.evaluate(var0)) + self.assertAllCloseAccordingToType([3.0, 4.0], self.evaluate(var1)) # Run 1 step of sgd sgd_op.run() # Validate updated params self.assertAllCloseAccordingToType([1.0 - 3.0 * 0.1, 2.0 - 3.0 * 0.1], - var0.eval()) + self.evaluate(var0)) self.assertAllCloseAccordingToType([3.0 - 3.0 * 0.01, 4.0 - 3.0 * 0.01], - var1.eval()) + self.evaluate(var1)) + @test_util.run_deprecated_v1 def testBasicCallableParams(self): for dtype in [dtypes.half, dtypes.float32, dtypes.float64]: with self.cached_session(): @@ -99,16 +103,17 @@ class GradientDescentOptimizerTest(test.TestCase): # a long-term solution for this. resources.initialize_resources([var0, var1]).run() # Fetch params to validate initial values - self.assertAllCloseAccordingToType([1.0, 2.0], var0.eval()) - self.assertAllCloseAccordingToType([3.0, 4.0], var1.eval()) + self.assertAllCloseAccordingToType([1.0, 2.0], self.evaluate(var0)) + self.assertAllCloseAccordingToType([3.0, 4.0], self.evaluate(var1)) # Run 1 step of sgd sgd_op.run() # Validate updated params self.assertAllCloseAccordingToType([1.0 - 3.0 * 0.1, 2.0 - 3.0 * 0.1], - var0.eval()) + self.evaluate(var0)) self.assertAllCloseAccordingToType([3.0 - 3.0 * 0.01, 4.0 - 3.0 * 0.01], - var1.eval()) + self.evaluate(var1)) + @test_util.run_deprecated_v1 def testMinimizeResourceVariable(self): for dtype in [dtypes.half, dtypes.float32, dtypes.float64]: with self.cached_session(): @@ -124,17 +129,18 @@ class GradientDescentOptimizerTest(test.TestCase): # a long-term solution for this. resources.initialize_resources([var0, var1]).run() # Fetch params to validate initial values - self.assertAllCloseAccordingToType([[1.0, 2.0]], var0.eval()) - self.assertAllCloseAccordingToType([3.0], var1.eval()) + self.assertAllCloseAccordingToType([[1.0, 2.0]], self.evaluate(var0)) + self.assertAllCloseAccordingToType([3.0], self.evaluate(var1)) # Run 1 step of sgd sgd_op.run() # Validate updated params np_pred = 1.0 * 4.0 + 2.0 * 5.0 + 3.0 np_grad = 2 * np_pred self.assertAllCloseAccordingToType( - [[1.0 - np_grad * 4.0, 2.0 - np_grad * 5.0]], var0.eval()) - self.assertAllCloseAccordingToType([3.0 - np_grad], var1.eval()) + [[1.0 - np_grad * 4.0, 2.0 - np_grad * 5.0]], self.evaluate(var0)) + self.assertAllCloseAccordingToType([3.0 - np_grad], self.evaluate(var1)) + @test_util.run_deprecated_v1 def testMinimizeSparseResourceVariable(self): for dtype in [dtypes.half, dtypes.float32, dtypes.float64]: with self.cached_session(): @@ -151,17 +157,18 @@ class GradientDescentOptimizerTest(test.TestCase): # a long-term solution for this. variables.global_variables_initializer().run() # Fetch params to validate initial values - self.assertAllCloseAccordingToType([[1.0, 2.0]], var0.eval()) - self.assertAllCloseAccordingToType([3.0], var1.eval()) + self.assertAllCloseAccordingToType([[1.0, 2.0]], self.evaluate(var0)) + self.assertAllCloseAccordingToType([3.0], self.evaluate(var1)) # Run 1 step of sgd sgd_op.run() # Validate updated params np_pred = 1.0 * 4.0 + 2.0 * 5.0 + 3.0 np_grad = 2 * np_pred self.assertAllCloseAccordingToType( - [[1.0 - np_grad * 4.0, 2.0 - np_grad * 5.0]], var0.eval()) - self.assertAllCloseAccordingToType([3.0 - np_grad], var1.eval()) + [[1.0 - np_grad * 4.0, 2.0 - np_grad * 5.0]], self.evaluate(var0)) + self.assertAllCloseAccordingToType([3.0 - np_grad], self.evaluate(var1)) + @test_util.run_deprecated_v1 def testTensorLearningRate(self): for dtype in [dtypes.half, dtypes.float32, dtypes.float64]: with self.cached_session(): @@ -174,16 +181,17 @@ class GradientDescentOptimizerTest(test.TestCase): lrate).apply_gradients(zip([grads0, grads1], [var0, var1])) variables.global_variables_initializer().run() # Fetch params to validate initial values - self.assertAllCloseAccordingToType([1.0, 2.0], var0.eval()) - self.assertAllCloseAccordingToType([3.0, 4.0], var1.eval()) + self.assertAllCloseAccordingToType([1.0, 2.0], self.evaluate(var0)) + self.assertAllCloseAccordingToType([3.0, 4.0], self.evaluate(var1)) # Run 1 step of sgd sgd_op.run() # Validate updated params self.assertAllCloseAccordingToType([1.0 - 3.0 * 0.1, 2.0 - 3.0 * 0.1], - var0.eval()) + self.evaluate(var0)) self.assertAllCloseAccordingToType([3.0 - 3.0 * 0.01, 4.0 - 3.0 * 0.01], - var1.eval()) + self.evaluate(var1)) + @test_util.run_deprecated_v1 def testGradWrtRef(self): for dtype in [dtypes.half, dtypes.float32, dtypes.float64]: with self.cached_session(): @@ -193,8 +201,9 @@ class GradientDescentOptimizerTest(test.TestCase): grads_and_vars = opt.compute_gradients(vars_[0] + vars_[1], vars_) variables.global_variables_initializer().run() for grad, _ in grads_and_vars: - self.assertAllCloseAccordingToType([1.0], grad.eval()) + self.assertAllCloseAccordingToType([1.0], self.evaluate(grad)) + @test_util.run_deprecated_v1 def testWithGlobalStep(self): for dtype in [dtypes.half, dtypes.float32, dtypes.float64]: with self.cached_session(): @@ -207,17 +216,18 @@ class GradientDescentOptimizerTest(test.TestCase): zip([grads0, grads1], [var0, var1]), global_step=global_step) variables.global_variables_initializer().run() # Fetch params to validate initial values - self.assertAllCloseAccordingToType([1.0, 2.0], var0.eval()) - self.assertAllCloseAccordingToType([3.0, 4.0], var1.eval()) + self.assertAllCloseAccordingToType([1.0, 2.0], self.evaluate(var0)) + self.assertAllCloseAccordingToType([3.0, 4.0], self.evaluate(var1)) # Run 1 step of sgd sgd_op.run() # Validate updated params and global_step self.assertAllCloseAccordingToType([1.0 - 3.0 * 0.1, 2.0 - 3.0 * 0.1], - var0.eval()) + self.evaluate(var0)) self.assertAllCloseAccordingToType([3.0 - 3.0 * 0.01, 4.0 - 3.0 * 0.01], - var1.eval()) - self.assertAllCloseAccordingToType(1, global_step.eval()) + self.evaluate(var1)) + self.assertAllCloseAccordingToType(1, self.evaluate(global_step)) + @test_util.run_deprecated_v1 def testSparseBasic(self): for dtype in [dtypes.half, dtypes.float32, dtypes.float64]: with self.cached_session(): @@ -237,15 +247,15 @@ class GradientDescentOptimizerTest(test.TestCase): zip([grads0, grads1], [var0, var1])) variables.global_variables_initializer().run() # Fetch params to validate initial values - self.assertAllCloseAccordingToType([[1.0], [2.0]], var0.eval()) - self.assertAllCloseAccordingToType([[3.0], [4.0]], var1.eval()) + self.assertAllCloseAccordingToType([[1.0], [2.0]], self.evaluate(var0)) + self.assertAllCloseAccordingToType([[3.0], [4.0]], self.evaluate(var1)) # Run 1 step of sgd sgd_op.run() # Validate updated params self.assertAllCloseAccordingToType([[1.0 - 3.0 * 0.1], [2.0]], - var0.eval()) + self.evaluate(var0)) self.assertAllCloseAccordingToType([[3.0], [4.0 - 3.0 * 0.01]], - var1.eval()) + self.evaluate(var1)) def testCapturingInDefunWhileExecutingEagerly(self): with context.eager_mode(): diff --git a/tensorflow/python/training/input_test.py b/tensorflow/python/training/input_test.py index 085b77d1d6a..a3d268a0174 100644 --- a/tensorflow/python/training/input_test.py +++ b/tensorflow/python/training/input_test.py @@ -28,6 +28,7 @@ from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes from tensorflow.python.framework import errors_impl from tensorflow.python.framework import sparse_tensor +from tensorflow.python.framework import test_util from tensorflow.python.ops import array_ops from tensorflow.python.ops import math_ops from tensorflow.python.ops import variables @@ -41,6 +42,7 @@ from tensorflow.python.util import compat class MatchFilenamesOnceTest(test_lib.TestCase): + @test_util.run_deprecated_v1 def test(self): temp_dir = self.get_temp_dir() filenames = [os.path.join(temp_dir, n) for n in os.listdir(temp_dir)] @@ -58,35 +60,41 @@ class MatchFilenamesOnceTest(test_lib.TestCase): one = inp.match_filenames_once(additional[1]) variables.global_variables_initializer().run() variables.local_variables_initializer().run() - self.assertItemsEqual(map(compat.as_bytes, filenames), star.eval()) - self.assertItemsEqual(map(compat.as_bytes, additional), question.eval()) - self.assertItemsEqual([compat.as_bytes(additional[1])], one.eval()) + self.assertItemsEqual( + map(compat.as_bytes, filenames), self.evaluate(star)) + self.assertItemsEqual( + map(compat.as_bytes, additional), self.evaluate(question)) + self.assertItemsEqual([compat.as_bytes(additional[1])], + self.evaluate(one)) class LimitEpochsTest(test_lib.TestCase): + @test_util.run_deprecated_v1 def testNoLimit(self): with self.cached_session(): seven = constant_op.constant(7) seven_forever = inp.limit_epochs(seven) variables.local_variables_initializer().run() for _ in range(100): - self.assertEqual(7, seven_forever.eval()) + self.assertEqual(7, self.evaluate(seven_forever)) + @test_util.run_deprecated_v1 def testLimit(self): with self.cached_session(): love_me = constant_op.constant("Love Me") love_me_two_times = inp.limit_epochs(love_me, num_epochs=2) variables.global_variables_initializer().run() variables.local_variables_initializer().run() - self.assertEqual(b"Love Me", love_me_two_times.eval()) - self.assertEqual(b"Love Me", love_me_two_times.eval()) + self.assertEqual(b"Love Me", self.evaluate(love_me_two_times)) + self.assertEqual(b"Love Me", self.evaluate(love_me_two_times)) with self.assertRaises(errors_impl.OutOfRangeError): - love_me_two_times.eval() + self.evaluate(love_me_two_times) class InputProducerTest(test_lib.TestCase): + @test_util.run_deprecated_v1 def testNoShuffle(self): with self.cached_session(): input_tensor = [[1, 2, 3, 4], @@ -102,14 +110,16 @@ class InputProducerTest(test_lib.TestCase): threads = queue_runner_impl.start_queue_runners() # No randomness, so just see repeated copies of the input. - self.assertAllEqual(input_tensor * num_epochs, dequeue_many.eval()) + self.assertAllEqual(input_tensor * num_epochs, + self.evaluate(dequeue_many)) # Reached the limit. with self.assertRaises(errors_impl.OutOfRangeError): - dequeue.eval() + self.evaluate(dequeue) for thread in threads: thread.join() + @test_util.run_deprecated_v1 def testNoShapeInference(self): with self.cached_session(): # Disable shape inference for the input. @@ -127,14 +137,15 @@ class InputProducerTest(test_lib.TestCase): threads = queue_runner_impl.start_queue_runners() # No randomness, so just see repeated copies of the input. - self.assertAllEqual(input_value * num_epochs, dequeue_many.eval()) + self.assertAllEqual(input_value * num_epochs, self.evaluate(dequeue_many)) # Reached the limit. with self.assertRaises(errors_impl.OutOfRangeError): - dequeue.eval() + self.evaluate(dequeue) for thread in threads: thread.join() + @test_util.run_deprecated_v1 def testShapeError(self): input_tensor = array_ops.placeholder(dtypes.float32, None) with self.assertRaisesRegexp(ValueError, "fully defined shape"): @@ -143,6 +154,7 @@ class InputProducerTest(test_lib.TestCase): class StringInputProducerTest(test_lib.TestCase): + @test_util.run_deprecated_v1 def testNoShuffle(self): with self.cached_session(): strings = [b"to", b"be", b"or", b"not", b"to", b"be"] @@ -156,15 +168,16 @@ class StringInputProducerTest(test_lib.TestCase): threads = queue_runner_impl.start_queue_runners() # No randomness, so just see repeated copies of the input. - output = dequeue_many.eval() + output = self.evaluate(dequeue_many) self.assertAllEqual(strings * num_epochs, output) # Reached the limit. with self.assertRaises(errors_impl.OutOfRangeError): - dequeue.eval() + self.evaluate(dequeue) for thread in threads: thread.join() + @test_util.run_deprecated_v1 def testShuffle(self): with self.cached_session(): strings = [b"a", b"b", b"c"] @@ -184,7 +197,7 @@ class StringInputProducerTest(test_lib.TestCase): for e in expected: frequency[e] = 0 for _ in range(num_epochs): - output = dequeue_many.eval() + output = self.evaluate(dequeue_many) key = b"".join(output) self.assertIn(key, expected) frequency[key] += 1 @@ -200,7 +213,7 @@ class StringInputProducerTest(test_lib.TestCase): # Reached the limit. with self.assertRaises(errors_impl.OutOfRangeError): - dequeue.eval() + self.evaluate(dequeue) for thread in threads: thread.join() @@ -210,6 +223,7 @@ class StringInputProducerTest(test_lib.TestCase): with self.assertRaises(ValueError): _ = inp.string_input_producer([]) + @test_util.run_deprecated_v1 def testNullString(self): # Runtime check for empty string list. This is slightly oblique: # The queue runner should die with an assertion error on the null @@ -224,11 +238,12 @@ class StringInputProducerTest(test_lib.TestCase): variables.local_variables_initializer().run() threads = queue_runner_impl.start_queue_runners(coord=coord) with self.assertRaises(errors_impl.OutOfRangeError): - dequeue.eval() + self.evaluate(dequeue) coord.request_stop() for thread in threads: thread.join() + @test_util.run_deprecated_v1 def testSharedName(self): with self.cached_session(): strings = [b"to", b"be", b"or", b"not", b"to", b"be"] @@ -237,6 +252,7 @@ class StringInputProducerTest(test_lib.TestCase): self.assertProtoEquals("s: 'SHARED_NAME_XYZ'", queue.queue_ref.op.node_def.attr["shared_name"]) + @test_util.run_deprecated_v1 def testConstructionRace(self): with self.cached_session() as sess: strings = [b"to", b"be", b"or", b"not", b"to", b"be"] @@ -252,13 +268,14 @@ class StringInputProducerTest(test_lib.TestCase): # writing of the `tf.Graph` object. However, many users # write code this way, so we include this test to ensure # that we can support it. - self.assertEquals(string, sess.run(queue.dequeue())) + self.assertEquals(string, self.evaluate(queue.dequeue())) coord.request_stop() coord.join(threads) class RangeInputProducerTest(test_lib.TestCase): + @test_util.run_deprecated_v1 def testNoShuffle(self): with self.cached_session(): num_epochs = 3 @@ -272,15 +289,16 @@ class RangeInputProducerTest(test_lib.TestCase): threads = queue_runner_impl.start_queue_runners() # No randomness, so just see repeated copies of the input. - output = dequeue_many.eval() + output = self.evaluate(dequeue_many) self.assertAllEqual(list(xrange(range_size)) * num_epochs, output) # Reached the limit. with self.assertRaises(errors_impl.OutOfRangeError): - dequeue.eval() + self.evaluate(dequeue) for thread in threads: thread.join() + @test_util.run_deprecated_v1 def testShuffle(self): with self.cached_session(): num_epochs = 200 @@ -300,7 +318,7 @@ class RangeInputProducerTest(test_lib.TestCase): for e in expected: frequency[e] = 0 for _ in range(num_epochs): - output = dequeue_many.eval() + output = self.evaluate(dequeue_many) key = 10 * (output[0] + 1) + (output[1] + 1) self.assertIn(key, expected) frequency[key] += 1 @@ -316,10 +334,11 @@ class RangeInputProducerTest(test_lib.TestCase): # Reached the limit. with self.assertRaises(errors_impl.OutOfRangeError): - dequeue.eval() + self.evaluate(dequeue) for thread in threads: thread.join() + @test_util.run_deprecated_v1 def testSharedName(self): with self.cached_session(): range_size = 5 @@ -331,6 +350,7 @@ class RangeInputProducerTest(test_lib.TestCase): class SliceInputProducerTest(test_lib.TestCase): + @test_util.run_deprecated_v1 def testNoShuffle(self): with self.cached_session() as sess: num_epochs = 3 @@ -344,17 +364,18 @@ class SliceInputProducerTest(test_lib.TestCase): # No randomness, so just see repeated copies of the input. num_items = len(source_strings) * num_epochs - output = [sess.run(slices) for _ in range(num_items)] + output = [self.evaluate(slices) for _ in range(num_items)] out_strings, out_ints = zip(*output) self.assertAllEqual(source_strings * num_epochs, out_strings) self.assertAllEqual(source_ints * num_epochs, out_ints) # Reached the limit. with self.assertRaises(errors_impl.OutOfRangeError): - sess.run(slices) + self.evaluate(slices) for thread in threads: thread.join() + @test_util.run_deprecated_v1 def testShuffle(self): with self.cached_session() as sess: num_epochs = 1200 @@ -379,7 +400,7 @@ class SliceInputProducerTest(test_lib.TestCase): for e in expected: frequency[e] = 0 for _ in range(num_epochs): - output = [sess.run(slices) for _ in range(len(source_strings))] + output = [self.evaluate(slices) for _ in range(len(source_strings))] key = b",".join([s + compat.as_bytes(str(i)) for s, i in output]) self.assertIn(key, expected) frequency[key] += 1 @@ -395,10 +416,11 @@ class SliceInputProducerTest(test_lib.TestCase): # Reached the limit. with self.assertRaises(errors_impl.OutOfRangeError): - sess.run(slices) + self.evaluate(slices) for thread in threads: thread.join() + @test_util.run_deprecated_v1 def testSharedName(self): with self.cached_session(): source_strings = ["A", "B", "D", "G"] @@ -470,7 +492,7 @@ class BatchTest(test_lib.TestCase): threads = queue_runner_impl.start_queue_runners() for i in range(num_batches): - results = sess.run(batched_fetch) + results = self.evaluate(batched_fetch) self.assertAllEqual(results[0], np.arange(i * batch_size, (i + 1) * batch_size)) self.assertAllEqual( @@ -487,38 +509,43 @@ class BatchTest(test_lib.TestCase): # Reached the limit. with self.assertRaises(errors_impl.OutOfRangeError): - sess.run(batched_fetch) + self.evaluate(batched_fetch) for thread in threads: thread.join() + @test_util.run_deprecated_v1 def testOneThread(self): self._testOneThreadHelper(use_dict=False) + @test_util.run_deprecated_v1 def testOneThreadDict(self): self._testOneThreadHelper(use_dict=True) + @test_util.run_deprecated_v1 def testUint32DataTypes(self): values = constant_op.constant([0, 1, 2, 3, 4, 5], dtype=dtypes.uint32) batched = inp.batch([values], batch_size=2) with self.cached_session() as sess: coord = coordinator.Coordinator() threads = queue_runner_impl.start_queue_runners(sess=sess, coord=coord) - sess.run(batched) + self.evaluate(batched) coord.request_stop() for thread in threads: thread.join() + @test_util.run_deprecated_v1 def testUint64DataTypes(self): values = constant_op.constant([0, 1, 2, 3, 4, 5], dtype=dtypes.uint64) batched = inp.batch([values], batch_size=2) with self.cached_session() as sess: coord = coordinator.Coordinator() threads = queue_runner_impl.start_queue_runners(sess=sess, coord=coord) - sess.run(batched) + self.evaluate(batched) coord.request_stop() for thread in threads: thread.join() + @test_util.run_deprecated_v1 def testOneThreadDynamicPad(self): with self.cached_session() as sess: batch_size = 10 @@ -535,7 +562,7 @@ class BatchTest(test_lib.TestCase): threads = queue_runner_impl.start_queue_runners() for i in range(num_batches): - results = sess.run(batched) + results = self.evaluate(batched) expected_results = np.arange(i * batch_size, (i + 1) * batch_size) max_len = expected_results[-1] self.assertAllEqual(results[0], expected_results) @@ -545,10 +572,11 @@ class BatchTest(test_lib.TestCase): # Reached the limit. with self.assertRaises(errors_impl.OutOfRangeError): - sess.run(batched) + self.evaluate(batched) for thread in threads: thread.join() + @test_util.run_deprecated_v1 def testOneThreadEnqueueMany(self): with self.cached_session() as sess: batch_size = 10 @@ -567,7 +595,7 @@ class BatchTest(test_lib.TestCase): threads = queue_runner_impl.start_queue_runners() for i in range(num_batches): - results = sess.run(batched) + results = self.evaluate(batched) self.assertAllEqual(results[0], np.arange(i * batch_size, (i + 1) * batch_size)) self.assertAllEqual( @@ -580,10 +608,11 @@ class BatchTest(test_lib.TestCase): # Reached the limit. with self.assertRaises(errors_impl.OutOfRangeError): - sess.run(batched) + self.evaluate(batched) for thread in threads: thread.join() + @test_util.run_deprecated_v1 def testManyThreads(self): with self.cached_session() as sess: batch_size = 10 @@ -606,7 +635,7 @@ class BatchTest(test_lib.TestCase): all_counts = [] for i in range(num_batches): - results = sess.run(batched) + results = self.evaluate(batched) tf_logging.info("Batch %d: %s", i, results[0]) self.assertEqual(len(results[0]), batch_size) self.assertAllEqual(results[0], results[1].values) @@ -620,10 +649,11 @@ class BatchTest(test_lib.TestCase): # Reached the limit. with self.assertRaises(errors_impl.OutOfRangeError): - sess.run(batched) + self.evaluate(batched) for thread in threads: thread.join() + @test_util.run_deprecated_v1 def testOneThreadSmallerBatch(self): with self.cached_session() as sess: batch_size = 10 @@ -647,7 +677,7 @@ class BatchTest(test_lib.TestCase): threads = queue_runner_impl.start_queue_runners() for i in range(num_batches): - results = sess.run(batched) + results = self.evaluate(batched) self.assertAllEqual(results[0], np.arange(i * batch_size, (i + 1) * batch_size)) self.assertAllEqual( @@ -663,7 +693,7 @@ class BatchTest(test_lib.TestCase): self.assertAllEqual(results[2], [b"string"] * batch_size) # Reached the final batch with extra_elements. - results = sess.run(batched) + results = self.evaluate(batched) self.assertAllEqual(results[0], np.arange(num_batches * batch_size, num_batches * batch_size + extra_elements)) @@ -677,10 +707,11 @@ class BatchTest(test_lib.TestCase): # Reached the limit. with self.assertRaises(errors_impl.OutOfRangeError): - sess.run(batched) + self.evaluate(batched) for thread in threads: thread.join() + @test_util.run_deprecated_v1 def testManyThreadsSmallerBatch(self): with self.cached_session() as sess: batch_size = 10 @@ -705,7 +736,7 @@ class BatchTest(test_lib.TestCase): all_counts = [] for i in range(num_batches): - results = sess.run(batched) + results = self.evaluate(batched) tf_logging.info("Batch %d: %s", i, results[0]) self.assertEqual(len(results[0]), batch_size) self.assertAllEqual(results[0], results[1].values) @@ -717,7 +748,7 @@ class BatchTest(test_lib.TestCase): self.assertAllEqual(results[2], [b"string"] * batch_size) # Reached the final batch with extra_elements. - results = sess.run(batched) + results = self.evaluate(batched) tf_logging.info("Last Batch: %s", results[0]) self.assertEqual(len(results[0]), extra_elements) self.assertAllEqual(results[0], results[1].values) @@ -732,10 +763,11 @@ class BatchTest(test_lib.TestCase): # Reached the limit. with self.assertRaises(errors_impl.OutOfRangeError): - sess.run(batched) + self.evaluate(batched) for thread in threads: thread.join() + @test_util.run_deprecated_v1 def testSharedName(self): with self.cached_session(): batch_size = 10 @@ -753,12 +785,14 @@ class BatchTest(test_lib.TestCase): "s: 'SHARED_NAME_XYZ'", batched[0].op.inputs[0].op.node_def.attr["shared_name"]) + @test_util.run_deprecated_v1 def testCannotInferRankError(self): with self.cached_session(): x = array_ops.placeholder(dtype=dtypes.int64) with self.assertRaisesRegexp(ValueError, "Cannot infer Tensor's rank"): inp.batch([x], batch_size=2) + @test_util.run_deprecated_v1 def testBatchedSparseTensorInferredShape(self): sparse = sparse_tensor.SparseTensor( indices=[[0]], values=[1.0], dense_shape=[1]) @@ -766,6 +800,7 @@ class BatchTest(test_lib.TestCase): batched = inp.batch([sparse], batch_size=2) self.assertAllEqual((2,), batched.dense_shape.get_shape().as_list()) + @test_util.run_deprecated_v1 def testBatchedSparseTensorInferredShapeEnqueueMany(self): sparse = sparse_tensor.SparseTensor( indices=[[0]], values=[1.0], dense_shape=[1]) @@ -773,6 +808,7 @@ class BatchTest(test_lib.TestCase): batched = inp.batch([sparse], batch_size=2, enqueue_many=True) self.assertAllEqual((1,), batched.dense_shape.get_shape().as_list()) + @test_util.run_deprecated_v1 def testBatchedSparseTensorInferredShapeUnknownRank(self): sparse = sparse_tensor.SparseTensor( indices=array_ops.placeholder(dtypes.int64), @@ -782,6 +818,7 @@ class BatchTest(test_lib.TestCase): batched = inp.batch([sparse], batch_size=2) self.assertIs(None, batched.dense_shape.get_shape().num_elements()) + @test_util.run_deprecated_v1 def testBatchedSparseTensorInferredShapeUnknownRankEnqueueMany(self): sparse = sparse_tensor.SparseTensor( indices=array_ops.placeholder(dtypes.int64), @@ -791,6 +828,7 @@ class BatchTest(test_lib.TestCase): batched = inp.batch([sparse], batch_size=2, enqueue_many=True) self.assertIs(None, batched.dense_shape.get_shape().num_elements()) + @test_util.run_deprecated_v1 def testSingleElementDict(self): x = inp.batch({"c": [12, 12]}, batch_size=8) self.assertAllEqual((8, 2), x["c"].get_shape().as_list()) @@ -823,35 +861,42 @@ class BatchTest(test_lib.TestCase): threads = queue_runner_impl.start_queue_runners() for _ in range(num_batches): - results = sess.run(batched) + results = self.evaluate(batched) self.assertAllEqual([0] * batch_size, np.mod(results[0], 2)) self.assertAllEqual([0] * batch_size, np.mod(results[1].values, 2)) self.assertAllEqual([b"string"] * batch_size, results[2]) # Reached the limit. with self.assertRaises(errors_impl.OutOfRangeError): - sess.run(batched) + self.evaluate(batched) for thread in threads: thread.join() + @test_util.run_deprecated_v1 def testSingleThreadKeepInput(self): self._testKeepInputHelper(1, False) + @test_util.run_deprecated_v1 def testSingleThreadKeepInputEnqueueMany(self): self._testKeepInputHelper(1, True) + @test_util.run_deprecated_v1 def testMultipleThreadKeepInput(self): self._testKeepInputHelper(5, False) + @test_util.run_deprecated_v1 def testMultipleThreadKeepInputEnqueueMany(self): self._testKeepInputHelper(5, True) + @test_util.run_deprecated_v1 def testMaybeEnqueuePerExample(self): self._testKeepInputHelper(1, True, keep_input_vector=True) + @test_util.run_deprecated_v1 def testMultipleThreadMaybeEnqueuePerExample(self): self._testKeepInputHelper(5, True, keep_input_vector=True) + @test_util.run_deprecated_v1 def testInvalidKeepInputVector(self): # Can't have vector `keep_input` with `enqueue_many=False`. with self.assertRaisesRegexp(ValueError, "`keep_input` cannot be a vector"): @@ -873,6 +918,7 @@ class BatchTest(test_lib.TestCase): batch_size=1, enqueue_many=True) + @test_util.run_deprecated_v1 def testMaybeBatchedSparseTensorInferredShape(self): sparse = sparse_tensor.SparseTensor( indices=[[0]], values=[1.0], dense_shape=[1]) @@ -880,6 +926,7 @@ class BatchTest(test_lib.TestCase): batched = inp.maybe_batch([sparse], keep_input=True, batch_size=2) self.assertAllEqual((2,), batched.dense_shape.get_shape().as_list()) + @test_util.run_deprecated_v1 def testMaybeBatchedSparseTensorInferredShapeEnqueueMany(self): sparse = sparse_tensor.SparseTensor( indices=[[0]], values=[1.0], dense_shape=[1]) @@ -888,6 +935,7 @@ class BatchTest(test_lib.TestCase): [sparse], keep_input=True, batch_size=2, enqueue_many=True) self.assertAllEqual((1,), batched.dense_shape.get_shape().as_list()) + @test_util.run_deprecated_v1 def testMaybeBatchedSparseTensorInferredShapeEnqueueManyPerExample(self): sparse = sparse_tensor.SparseTensor( indices=[[0], [0]], values=[1.0, 2.0], dense_shape=[2]) @@ -896,6 +944,7 @@ class BatchTest(test_lib.TestCase): [sparse], keep_input=[True, False], batch_size=2, enqueue_many=True) self.assertAllEqual((1,), batched.dense_shape.get_shape().as_list()) + @test_util.run_deprecated_v1 def testMaybeBatchedSparseTensorInferredShapeUnknownRank(self): sparse = sparse_tensor.SparseTensor( indices=array_ops.placeholder(dtypes.int64), @@ -905,6 +954,7 @@ class BatchTest(test_lib.TestCase): batched = inp.maybe_batch([sparse], keep_input=True, batch_size=2) self.assertIs(None, batched.dense_shape.get_shape().num_elements()) + @test_util.run_deprecated_v1 def testMaybeBatchedSparseTensorInferredShapeUnknownRankEnqueueMany(self): sparse = sparse_tensor.SparseTensor( indices=array_ops.placeholder(dtypes.int64), @@ -915,6 +965,7 @@ class BatchTest(test_lib.TestCase): [sparse], keep_input=True, batch_size=2, enqueue_many=True) self.assertIs(None, batched.dense_shape.get_shape().num_elements()) + @test_util.run_deprecated_v1 def testMaybeBatchedSparseTensorInferredShapeUnknownRankPerExample(self): sparse = sparse_tensor.SparseTensor( indices=array_ops.placeholder(dtypes.int64), @@ -925,6 +976,7 @@ class BatchTest(test_lib.TestCase): [sparse], keep_input=[True, False], batch_size=2, enqueue_many=True) self.assertIs(None, batched.dense_shape.get_shape().num_elements()) + @test_util.run_deprecated_v1 def testMaybeBatchCorrectValues(self): sparse_t = sparse_tensor.SparseTensor( indices=[[0, 1], [0, 2], [1, 0], [1, 3]], @@ -938,7 +990,7 @@ class BatchTest(test_lib.TestCase): coord = coordinator.Coordinator() threads = queue_runner_impl.start_queue_runners(coord=coord) - batched_np = batched.eval() + batched_np = self.evaluate(batched) coord.request_stop() for thread in threads: @@ -1016,7 +1068,7 @@ class BatchJoinTest(test_lib.TestCase): saw_both = 0 num_batches = (num_a + num_b) // batch_size for i in range(num_batches): - results = sess.run(batched_fetch) + results = self.evaluate(batched_fetch) self.assertEqual(3, len(results)) self.assertEqual(batch_size, len(results[0])) self.assertEqual(batch_size, len(results[2])) @@ -1047,16 +1099,19 @@ class BatchJoinTest(test_lib.TestCase): # Reached the limit. with self.assertRaises(errors_impl.OutOfRangeError): - sess.run(batched_fetch) + self.evaluate(batched_fetch) for thread in threads: thread.join() + @test_util.run_deprecated_v1 def testTwoThreads(self): self._testTwoThreadsHelper(use_dict=False) + @test_util.run_deprecated_v1 def testTwoThreadsDict(self): self._testTwoThreadsHelper(use_dict=True) + @test_util.run_deprecated_v1 def testMismatchedDictKeys(self): with self.assertRaisesRegexp(ValueError, "must have the same keys"): inp.batch_join( @@ -1071,6 +1126,7 @@ class BatchJoinTest(test_lib.TestCase): }], batch_size=8) + @test_util.run_deprecated_v1 def testTwoThreadsDynamicPad(self): with self.cached_session() as sess: # Two threads, the first generates (0..69, ["a"] * 1..70). @@ -1112,7 +1168,7 @@ class BatchJoinTest(test_lib.TestCase): saw_both = 0 num_batches = (num_a + num_b) // batch_size for i in range(num_batches): - results = sess.run(batched) + results = self.evaluate(batched) self.assertEqual(2, len(results)) self.assertEqual(len(results[0]), batch_size) self.assertEqual(len(results[1]), batch_size) @@ -1144,10 +1200,11 @@ class BatchJoinTest(test_lib.TestCase): # Reached the limit. with self.assertRaises(errors_impl.OutOfRangeError): - sess.run(batched) + self.evaluate(batched) for thread in threads: thread.join() + @test_util.run_deprecated_v1 def testTwoThreadsSmallerBatch(self): with self.cached_session() as sess: extra_elements = 2 @@ -1197,7 +1254,7 @@ class BatchJoinTest(test_lib.TestCase): saw_both = 0 num_batches = (num_a + num_b) // batch_size for i in range(num_batches): - results = sess.run(batched) + results = self.evaluate(batched) tf_logging.info("Batch %d: %s", i, results[0]) self.assertEqual(len(results[0]), batch_size) self.assertEqual(len(results[2]), batch_size) @@ -1217,7 +1274,7 @@ class BatchJoinTest(test_lib.TestCase): [results[0][i] for i in which_b]) # Reached the final batch with 2 * extra_elements. - results = sess.run(batched) + results = self.evaluate(batched) tf_logging.info("Last Batch: %s", results[0]) self.assertEqual(len(results[0]), 2 * extra_elements) self.assertEqual(len(results[2]), 2 * extra_elements) @@ -1245,10 +1302,11 @@ class BatchJoinTest(test_lib.TestCase): # Reached the limit. with self.assertRaises(errors_impl.OutOfRangeError): - sess.run(batched) + self.evaluate(batched) for thread in threads: thread.join() + @test_util.run_deprecated_v1 def testTwoThreadsDynamicPadSmallerBatch(self): with self.cached_session() as sess: extra_elements = 2 @@ -1292,7 +1350,7 @@ class BatchJoinTest(test_lib.TestCase): saw_both = 0 num_batches = (num_a + num_b) // batch_size for i in range(num_batches): - results = sess.run(batched) + results = self.evaluate(batched) tf_logging.info("Batch %d: %s", i, results[0]) self.assertEqual(len(results[0]), batch_size) self.assertEqual(len(results[1]), batch_size) @@ -1312,7 +1370,7 @@ class BatchJoinTest(test_lib.TestCase): [results[0][i] for i in which_b]) # Reached the final batch with 2 * extra_elements. - results = sess.run(batched) + results = self.evaluate(batched) tf_logging.info("Last Batch: %s", results[0]) self.assertEqual(len(results[0]), 2 * extra_elements) self.assertEqual(len(results[1]), 2 * extra_elements) @@ -1343,10 +1401,11 @@ class BatchJoinTest(test_lib.TestCase): # Reached the limit. with self.assertRaises(errors_impl.OutOfRangeError): - sess.run(batched) + self.evaluate(batched) for thread in threads: thread.join() + @test_util.run_deprecated_v1 def testSharedName(self): with self.cached_session(): batch_size = 10 @@ -1369,12 +1428,14 @@ class BatchJoinTest(test_lib.TestCase): "s: 'SHARED_NAME_XYZ'", batched[0].op.inputs[0].op.node_def.attr["shared_name"]) + @test_util.run_deprecated_v1 def testCannotInferRankError(self): with self.cached_session(): x = array_ops.placeholder(dtype=dtypes.int64) with self.assertRaisesRegexp(ValueError, "Cannot infer Tensor's rank"): inp.batch_join([[x]], batch_size=2) + @test_util.run_deprecated_v1 def testSingleElementDict(self): x = inp.batch_join([{"c": [12, 12]}], batch_size=8) self.assertAllEqual((8, 2), x["c"].get_shape().as_list()) @@ -1406,7 +1467,7 @@ class BatchJoinTest(test_lib.TestCase): threads = queue_runner_impl.start_queue_runners() for _ in range(num_batches): - results = sess.run(batched) + results = self.evaluate(batched) self.assertAllEqual( [0] * batch_size, np.mod(results[0], 2),) @@ -1417,28 +1478,35 @@ class BatchJoinTest(test_lib.TestCase): # Reached the limit. with self.assertRaises(errors_impl.OutOfRangeError): - sess.run(batched) + self.evaluate(batched) for thread in threads: thread.join() + @test_util.run_deprecated_v1 def testSingleThreadKeepInput(self): self._testKeepInputHelper(1, False) + @test_util.run_deprecated_v1 def testSingleThreadKeepInputEnqueueMany(self): self._testKeepInputHelper(1, True) + @test_util.run_deprecated_v1 def testMultipleThreadKeepInput(self): self._testKeepInputHelper(5, False) + @test_util.run_deprecated_v1 def testMultipleThreadKeepInputEnqueueMany(self): self._testKeepInputHelper(5, True) + @test_util.run_deprecated_v1 def testSingleThreadKeepInputPerExample(self): self._testKeepInputHelper(1, True, keep_input_vector=True) + @test_util.run_deprecated_v1 def testMultipleThreadKeepInputPerExample(self): self._testKeepInputHelper(5, True, keep_input_vector=True) + @test_util.run_deprecated_v1 def testInvalidKeepInputVector(self): # Can't have vector `keep_input` with `enqueue_many=False`. with self.assertRaisesRegexp(ValueError, "`keep_input` cannot be a vector"): @@ -1460,6 +1528,7 @@ class BatchJoinTest(test_lib.TestCase): batch_size=1, enqueue_many=True) + @test_util.run_deprecated_v1 def testMaybeBatchedSparseTensorInferredShape(self): sparse = sparse_tensor.SparseTensor( indices=[[0]], values=[1.0], dense_shape=[1]) @@ -1467,6 +1536,7 @@ class BatchJoinTest(test_lib.TestCase): batched = inp.maybe_batch_join([[sparse]], keep_input=True, batch_size=2) self.assertAllEqual((2,), batched.dense_shape.get_shape().as_list()) + @test_util.run_deprecated_v1 def testMaybeBatchedSparseTensorInferredShapeEnqueueMany(self): sparse = sparse_tensor.SparseTensor( indices=[[0]], values=[1.0], dense_shape=[1]) @@ -1475,6 +1545,7 @@ class BatchJoinTest(test_lib.TestCase): [[sparse]], keep_input=True, batch_size=2, enqueue_many=True) self.assertAllEqual((1,), batched.dense_shape.get_shape().as_list()) + @test_util.run_deprecated_v1 def testMaybeBatchedSparseTensorInferredShapeEnqueueManyPerExample(self): sparse = sparse_tensor.SparseTensor( indices=[[0], [0]], values=[1.0, 2.0], dense_shape=[2]) @@ -1483,6 +1554,7 @@ class BatchJoinTest(test_lib.TestCase): [[sparse]], keep_input=[True, False], batch_size=2, enqueue_many=True) self.assertAllEqual((1,), batched.dense_shape.get_shape().as_list()) + @test_util.run_deprecated_v1 def testMaybeBatchedSparseTensorInferredShapeUnknownRank(self): sparse = sparse_tensor.SparseTensor( indices=array_ops.placeholder(dtypes.int64), @@ -1492,6 +1564,7 @@ class BatchJoinTest(test_lib.TestCase): batched = inp.maybe_batch_join([[sparse]], keep_input=True, batch_size=2) self.assertIs(None, batched.dense_shape.get_shape().num_elements()) + @test_util.run_deprecated_v1 def testMaybeBatchedSparseTensorInferredShapeUnknownRankEnqueueMany(self): sparse = sparse_tensor.SparseTensor( indices=array_ops.placeholder(dtypes.int64), @@ -1502,6 +1575,7 @@ class BatchJoinTest(test_lib.TestCase): [[sparse]], keep_input=True, batch_size=2, enqueue_many=True) self.assertIs(None, batched.dense_shape.get_shape().num_elements()) + @test_util.run_deprecated_v1 def testMaybeBatchedSparseTensorInferredShapeUnknownRankPerExample(self): sparse = sparse_tensor.SparseTensor( indices=array_ops.placeholder(dtypes.int64), @@ -1512,6 +1586,7 @@ class BatchJoinTest(test_lib.TestCase): [[sparse]], keep_input=[True, False], batch_size=2, enqueue_many=True) self.assertIs(None, batched.dense_shape.get_shape().num_elements()) + @test_util.run_deprecated_v1 def testMaybeBatchCorrectValues(self): sparse = sparse_tensor.SparseTensor( indices=[[0, 1], [0, 2], [1, 0], [1, 3]], @@ -1525,7 +1600,7 @@ class BatchJoinTest(test_lib.TestCase): coord = coordinator.Coordinator() threads = queue_runner_impl.start_queue_runners(coord=coord) - batched_np = batched.eval() + batched_np = self.evaluate(batched) coord.request_stop() for thread in threads: @@ -1575,7 +1650,7 @@ class ShuffleBatchTest(test_lib.TestCase): all_counts = [] for i in range(num_batches): - results = sess.run(batched_fetch) + results = self.evaluate(batched_fetch) self.assertEqual(len(results[0]), batch_size) all_counts.extend(results[0]) self.assertAllEqual( @@ -1593,16 +1668,19 @@ class ShuffleBatchTest(test_lib.TestCase): # Reached the limit. with self.assertRaises(errors_impl.OutOfRangeError): - sess.run(batched_fetch) + self.evaluate(batched_fetch) for thread in threads: thread.join() + @test_util.run_deprecated_v1 def testOneThread(self): self._testOneThreadHelper(use_dict=False) + @test_util.run_deprecated_v1 def testOneThreadDict(self): self._testOneThreadHelper(use_dict=True) + @test_util.run_deprecated_v1 def testOneThreadSmallerBatch(self): with self.cached_session() as sess: batch_size = 10 @@ -1630,7 +1708,7 @@ class ShuffleBatchTest(test_lib.TestCase): all_counts = [] for _ in range(num_batches): - results = sess.run(batched_fetch) + results = self.evaluate(batched_fetch) self.assertEqual(len(results[0]), batch_size) all_counts.extend(results[0]) self.assertAllEqual( @@ -1641,7 +1719,7 @@ class ShuffleBatchTest(test_lib.TestCase): self.assertAllEqual(results[2], [b"string"] * batch_size) # Reached the final batch with extra elements. - results = sess.run(batched) + results = self.evaluate(batched) self.assertAllEqual(results[1].dense_shape, [extra_elements, 1]) self.assertAllEqual(results[2], [b"string"] * extra_elements) all_counts.extend(results[0]) @@ -1655,10 +1733,11 @@ class ShuffleBatchTest(test_lib.TestCase): # Reached the limit. with self.assertRaises(errors_impl.OutOfRangeError): - sess.run(batched_fetch) + self.evaluate(batched_fetch) for thread in threads: thread.join() + @test_util.run_deprecated_v1 def testManyThreads(self): with self.cached_session() as sess: batch_size = 10 @@ -1683,7 +1762,7 @@ class ShuffleBatchTest(test_lib.TestCase): all_counts = [] for i in range(num_batches): - results = sess.run(batched) + results = self.evaluate(batched) tf_logging.info("Batch %d: %s", i, results[0]) self.assertEqual(len(results[0]), batch_size) all_counts.extend(results[0]) @@ -1702,10 +1781,11 @@ class ShuffleBatchTest(test_lib.TestCase): # Reached the limit. with self.assertRaises(errors_impl.OutOfRangeError): - sess.run(batched) + self.evaluate(batched) for thread in threads: thread.join() + @test_util.run_deprecated_v1 def testManyThreadsSmallerBatch(self): with self.cached_session() as sess: batch_size = 10 @@ -1733,7 +1813,7 @@ class ShuffleBatchTest(test_lib.TestCase): all_counts = [] for i in range(num_batches): - results = sess.run(batched) + results = self.evaluate(batched) tf_logging.info("Batch %d: %s", i, results[0]) self.assertEqual(len(results[0]), batch_size) all_counts.extend(results[0]) @@ -1745,7 +1825,7 @@ class ShuffleBatchTest(test_lib.TestCase): self.assertAllEqual(results[2], [b"string"] * batch_size) # Reached the final batch with extra elements. - results = sess.run(batched) + results = self.evaluate(batched) self.assertAllEqual(results[0].shape, [extra_elements]) self.assertAllEqual(results[1].dense_shape, [extra_elements, 1]) self.assertAllEqual(results[2], [b"string"] * extra_elements) @@ -1760,10 +1840,11 @@ class ShuffleBatchTest(test_lib.TestCase): # Reached the limit. with self.assertRaises(errors_impl.OutOfRangeError): - sess.run(batched) + self.evaluate(batched) for thread in threads: thread.join() + @test_util.run_deprecated_v1 def testSharedName(self): with self.cached_session(): batch_size = 10 @@ -1813,35 +1894,42 @@ class ShuffleBatchTest(test_lib.TestCase): threads = queue_runner_impl.start_queue_runners() for _ in range(num_batches): - results = sess.run(batched) + results = self.evaluate(batched) self.assertAllEqual([0] * batch_size, np.mod(results[0], 2)) self.assertAllEqual([0] * batch_size, np.mod(results[1].values, 2)) self.assertAllEqual([b"string"] * batch_size, results[2]) # Reached the limit. with self.assertRaises(errors_impl.OutOfRangeError): - sess.run(batched) + self.evaluate(batched) for thread in threads: thread.join() + @test_util.run_deprecated_v1 def testSingleThreadKeepInput(self): self._testKeepInputHelper(1, False) + @test_util.run_deprecated_v1 def testSingleThreadKeepInputEnqueueMany(self): self._testKeepInputHelper(1, True) + @test_util.run_deprecated_v1 def testMultipleThreadKeepInput(self): self._testKeepInputHelper(5, False) + @test_util.run_deprecated_v1 def testMultipleThreadKeepInputEnqueueMany(self): self._testKeepInputHelper(5, True) + @test_util.run_deprecated_v1 def testSingleThreadKeepInputPerExample(self): self._testKeepInputHelper(1, True, keep_input_vector=True) + @test_util.run_deprecated_v1 def testMultipleThreadKeepInputPerExample(self): self._testKeepInputHelper(5, True, keep_input_vector=True) + @test_util.run_deprecated_v1 def testInvalidKeepInputVector(self): # Can't have vector `keep_input` with `enqueue_many=False`. with self.assertRaisesRegexp(ValueError, "`keep_input` cannot be a vector"): @@ -1860,6 +1948,7 @@ class ShuffleBatchTest(test_lib.TestCase): keep_input=array_ops.placeholder(dtypes.bool), enqueue_many=True) + @test_util.run_deprecated_v1 def testMaybeBatchedSparseTensorInferredShape(self): sparse = sparse_tensor.SparseTensor( indices=[[0]], values=[1.0], dense_shape=[1]) @@ -1867,6 +1956,7 @@ class ShuffleBatchTest(test_lib.TestCase): batched = inp.maybe_shuffle_batch([sparse], 2, 10, 1, True) self.assertAllEqual((2,), batched.dense_shape.get_shape().as_list()) + @test_util.run_deprecated_v1 def testMaybeBatchedSparseTensorInferredShapeEnqueueMany(self): sparse = sparse_tensor.SparseTensor( indices=[[0]], values=[1.0], dense_shape=[1]) @@ -1875,6 +1965,7 @@ class ShuffleBatchTest(test_lib.TestCase): [sparse], 2, 10, 1, True, enqueue_many=True) self.assertAllEqual((1,), batched.dense_shape.get_shape().as_list()) + @test_util.run_deprecated_v1 def testMaybeBatchedSparseTensorInferredShapeEnqueueManyPerExample(self): sparse = sparse_tensor.SparseTensor( indices=[[0], [0]], values=[1.0, 2.0], dense_shape=[2]) @@ -1883,6 +1974,7 @@ class ShuffleBatchTest(test_lib.TestCase): [sparse], 2, 10, 1, [True, False], enqueue_many=True) self.assertAllEqual((1,), batched.dense_shape.get_shape().as_list()) + @test_util.run_deprecated_v1 def testMaybeBatchedSparseTensorInferredShapeUnknownRank(self): sparse = sparse_tensor.SparseTensor( indices=array_ops.placeholder(dtypes.int64), @@ -1892,6 +1984,7 @@ class ShuffleBatchTest(test_lib.TestCase): batched = inp.maybe_shuffle_batch([sparse], 2, 10, 1, True) self.assertIs(None, batched.dense_shape.get_shape().num_elements()) + @test_util.run_deprecated_v1 def testMaybeBatchedSparseTensorInferredShapeUnknownRankEnqueueMany(self): sparse = sparse_tensor.SparseTensor( indices=array_ops.placeholder(dtypes.int64), @@ -1902,6 +1995,7 @@ class ShuffleBatchTest(test_lib.TestCase): [sparse], 2, 10, 1, True, enqueue_many=True) self.assertIs(None, batched.dense_shape.get_shape().num_elements()) + @test_util.run_deprecated_v1 def testMaybeBatchedSparseTensorInferredShapeUnknownRankPerExample(self): sparse = sparse_tensor.SparseTensor( indices=array_ops.placeholder(dtypes.int64), @@ -1986,7 +2080,7 @@ class ShuffleBatchJoinTest(test_lib.TestCase): saw_both = 0 num_batches = (num_a + num_b) // batch_size for i in range(num_batches): - results = sess.run(batched_fetch) + results = self.evaluate(batched_fetch) self.assertEqual(3, len(results)) self.assertEqual(len(results[0]), batch_size) self.assertEqual(len(results[2]), batch_size) @@ -2016,16 +2110,19 @@ class ShuffleBatchJoinTest(test_lib.TestCase): # Reached the limit. with self.assertRaises(errors_impl.OutOfRangeError): - sess.run(batched_fetch) + self.evaluate(batched_fetch) for thread in threads: thread.join() + @test_util.run_deprecated_v1 def testTwoThreads(self): self._testTwoThreadsHelper(use_dict=False) + @test_util.run_deprecated_v1 def testTwoThreadsDict(self): self._testTwoThreadsHelper(use_dict=True) + @test_util.run_deprecated_v1 def testTwoThreadsSmallerBatch(self): with self.cached_session() as sess: # Two threads, the first generates (0..26, "a"). @@ -2078,7 +2175,7 @@ class ShuffleBatchJoinTest(test_lib.TestCase): saw_both = 0 num_batches = (num_a + num_b) // batch_size for i in range(num_batches): - results = sess.run(batched) + results = self.evaluate(batched) tf_logging.info("Batch %d: %s", i, results[0]) self.assertEqual(len(results[0]), batch_size) self.assertEqual(len(results[2]), batch_size) @@ -2098,7 +2195,7 @@ class ShuffleBatchJoinTest(test_lib.TestCase): [results[0][i] for i in which_b]) # Reached end with 2 * extra_elements left - results = sess.run(batched) + results = self.evaluate(batched) self.assertEqual(len(results[0]), 2 * extra_elements) self.assertAllEqual(results[1].dense_shape, [2 * extra_elements, 1]) self.assertEqual(len(results[2]), 2 * extra_elements) @@ -2125,10 +2222,11 @@ class ShuffleBatchJoinTest(test_lib.TestCase): # Reached the limit. with self.assertRaises(errors_impl.OutOfRangeError): - sess.run(batched) + self.evaluate(batched) for thread in threads: thread.join() + @test_util.run_deprecated_v1 def testMismatchedDictKeys(self): with self.assertRaisesRegexp(ValueError, "must have the same keys"): inp.shuffle_batch_join( @@ -2146,6 +2244,7 @@ class ShuffleBatchJoinTest(test_lib.TestCase): min_after_dequeue=16, seed=223607) + @test_util.run_deprecated_v1 def testSharedName(self): with self.cached_session(): batch_size = 10 @@ -2199,35 +2298,42 @@ class ShuffleBatchJoinTest(test_lib.TestCase): threads = queue_runner_impl.start_queue_runners() for _ in range(num_batches): - results = sess.run(batched) + results = self.evaluate(batched) self.assertAllEqual([0] * batch_size, np.mod(results[0], 2)) self.assertAllEqual([0] * batch_size, np.mod(results[1].values, 2)) self.assertAllEqual([b"string"] * batch_size, results[2]) # Reached the limit. with self.assertRaises(errors_impl.OutOfRangeError): - sess.run(batched) + self.evaluate(batched) for thread in threads: thread.join() + @test_util.run_deprecated_v1 def testSingleThreadKeepInput(self): self._testKeepInputHelper(1, False) + @test_util.run_deprecated_v1 def testSingleThreadKeepInputEnqueueMany(self): self._testKeepInputHelper(1, True) + @test_util.run_deprecated_v1 def testMultipleThreadKeepInput(self): self._testKeepInputHelper(5, False) + @test_util.run_deprecated_v1 def testMultipleThreadKeepInputEnqueueMany(self): self._testKeepInputHelper(5, True) + @test_util.run_deprecated_v1 def testSingleThreadKeepInputPerExample(self): self._testKeepInputHelper(1, True, keep_input_vector=True) + @test_util.run_deprecated_v1 def testMultipleThreadKeepInputPerExample(self): self._testKeepInputHelper(5, True, keep_input_vector=True) + @test_util.run_deprecated_v1 def testInvalidKeepInputVector(self): # Can't have vector `keep_input` with `enqueue_many=False`. with self.assertRaisesRegexp(ValueError, "`keep_input` cannot be a vector"): @@ -2249,6 +2355,7 @@ class ShuffleBatchJoinTest(test_lib.TestCase): keep_input=array_ops.placeholder(dtypes.bool), enqueue_many=True) + @test_util.run_deprecated_v1 def testMaybeBatchedSparseTensorInferredShape(self): sparse = sparse_tensor.SparseTensor( indices=[[0]], values=[1.0], dense_shape=[1]) @@ -2256,6 +2363,7 @@ class ShuffleBatchJoinTest(test_lib.TestCase): batched = inp.maybe_shuffle_batch_join([[sparse]], 2, 10, 1, True) self.assertAllEqual((2,), batched.dense_shape.get_shape().as_list()) + @test_util.run_deprecated_v1 def testMaybeBatchedSparseTensorInferredShapeEnqueueMany(self): sparse = sparse_tensor.SparseTensor( indices=[[0]], values=[1.0], dense_shape=[1]) @@ -2264,6 +2372,7 @@ class ShuffleBatchJoinTest(test_lib.TestCase): [[sparse]], 2, 10, 1, True, enqueue_many=True) self.assertAllEqual((1,), batched.dense_shape.get_shape().as_list()) + @test_util.run_deprecated_v1 def testMaybeBatchedSparseTensorInferredShapeEnqueueManyPerExample(self): sparse = sparse_tensor.SparseTensor( indices=[[0], [0]], values=[1.0, 2.0], dense_shape=[2]) @@ -2272,6 +2381,7 @@ class ShuffleBatchJoinTest(test_lib.TestCase): [[sparse]], 2, 10, 1, [True, False], enqueue_many=True) self.assertAllEqual((1,), batched.dense_shape.get_shape().as_list()) + @test_util.run_deprecated_v1 def testMaybeBatchedSparseTensorInferredShapeUnknownRank(self): sparse = sparse_tensor.SparseTensor( indices=array_ops.placeholder(dtypes.int64), @@ -2281,6 +2391,7 @@ class ShuffleBatchJoinTest(test_lib.TestCase): batched = inp.maybe_shuffle_batch_join([[sparse]], 2, 10, 1, True) self.assertIs(None, batched.dense_shape.get_shape().num_elements()) + @test_util.run_deprecated_v1 def testMaybeBatchedSparseTensorInferredShapeUnknownRankEnqueueMany(self): sparse = sparse_tensor.SparseTensor( indices=array_ops.placeholder(dtypes.int64), @@ -2291,6 +2402,7 @@ class ShuffleBatchJoinTest(test_lib.TestCase): [[sparse]], 2, 10, 1, True, enqueue_many=True) self.assertIs(None, batched.dense_shape.get_shape().num_elements()) + @test_util.run_deprecated_v1 def testMaybeBatchedSparseTensorInferredShapeUnknownRankPerExample(self): sparse = sparse_tensor.SparseTensor( indices=array_ops.placeholder(dtypes.int64), diff --git a/tensorflow/python/training/learning_rate_decay.py b/tensorflow/python/training/learning_rate_decay.py index 29b54653219..c52e89db1f4 100644 --- a/tensorflow/python/training/learning_rate_decay.py +++ b/tensorflow/python/training/learning_rate_decay.py @@ -100,7 +100,7 @@ def exponential_decay(learning_rate, return decayed_lr -@tf_export(v1=["train.piecewise_constant"]) +@tf_export(v1=["train.piecewise_constant_decay", "train.piecewise_constant"]) def piecewise_constant(x, boundaries, values, name=None): """Piecewise constant from boundaries and interval values. diff --git a/tensorflow/python/training/learning_rate_decay_test.py b/tensorflow/python/training/learning_rate_decay_test.py index 03a32f6ca09..9de5bc8168f 100644 --- a/tensorflow/python/training/learning_rate_decay_test.py +++ b/tensorflow/python/training/learning_rate_decay_test.py @@ -61,24 +61,24 @@ class LRDecayTest(test_util.TensorFlowTestCase): self.evaluate(step.assign(100)) self.assertAllClose(self.evaluate(decayed_lr), expected, 1e-6) + @test_util.run_deprecated_v1 def testVariables(self): - with self.cached_session(): - step = variables.VariableV1(1) - assign_1 = step.assign(1) - assign_2 = step.assign(2) - assign_100 = step.assign(100) - decayed_lr = learning_rate_decay.exponential_decay(.1, step, 3, 0.96, - staircase=True) - variables.global_variables_initializer().run() - # No change to learning rate - assign_1.op.run() - self.assertAllClose(decayed_lr.eval(), .1, 1e-6) - assign_2.op.run() - self.assertAllClose(decayed_lr.eval(), .1, 1e-6) - # Decayed learning rate - assign_100.op.run() - expected = .1 * 0.96 ** (100 // 3) - self.assertAllClose(decayed_lr.eval(), expected, 1e-6) + step = variables.VariableV1(1) + assign_1 = step.assign(1) + assign_2 = step.assign(2) + assign_100 = step.assign(100) + decayed_lr = learning_rate_decay.exponential_decay( + .1, step, 3, 0.96, staircase=True) + self.evaluate(variables.global_variables_initializer()) + # No change to learning rate + self.evaluate(assign_1.op) + self.assertAllClose(self.evaluate(decayed_lr), .1, 1e-6) + self.evaluate(assign_2.op) + self.assertAllClose(self.evaluate(decayed_lr), .1, 1e-6) + # Decayed learning rate + self.evaluate(assign_100.op) + expected = .1 * 0.96**(100 // 3) + self.assertAllClose(self.evaluate(decayed_lr), expected, 1e-6) @test_util.run_in_graph_and_eager_modes def testPiecewiseConstant(self): @@ -101,6 +101,7 @@ class LRDecayTest(test_util.TensorFlowTestCase): self.assertAllClose(self.evaluate(decayed_lr), 0.001, 1e-6) @test_util.run_in_graph_and_eager_modes + @test_util.run_deprecated_v1 def testPiecewiseConstantEdgeCases(self): x_int = resource_variable_ops.ResourceVariable( 0, dtype=variables.dtypes.int32) diff --git a/tensorflow/python/training/learning_rate_decay_v2.py b/tensorflow/python/training/learning_rate_decay_v2.py index 9c5e144be6b..eb69feb17d3 100644 --- a/tensorflow/python/training/learning_rate_decay_v2.py +++ b/tensorflow/python/training/learning_rate_decay_v2.py @@ -117,7 +117,7 @@ def exponential_decay(learning_rate, decay_rate, staircase, name) -@tf_export("train.piecewise_constant", v1=[]) +@tf_export("train.piecewise_constant_decay", v1=[]) def piecewise_constant(x, boundaries, values, name=None): """Piecewise constant from boundaries and interval values. diff --git a/tensorflow/python/training/learning_rate_decay_v2_test.py b/tensorflow/python/training/learning_rate_decay_v2_test.py index b2ac93f06fe..cb96773e299 100644 --- a/tensorflow/python/training/learning_rate_decay_v2_test.py +++ b/tensorflow/python/training/learning_rate_decay_v2_test.py @@ -61,24 +61,24 @@ class LRDecayTestV2(test_util.TensorFlowTestCase): self.evaluate(step.assign(100)) self.assertAllClose(self.evaluate(decayed_lr()), expected, 1e-6) + @test_util.run_deprecated_v1 def testVariables(self): - with self.cached_session(): - step = variables.Variable(1) - assign_1 = step.assign(1) - assign_2 = step.assign(2) - assign_100 = step.assign(100) - decayed_lr = learning_rate_decay_v2.exponential_decay(.1, step, 3, 0.96, - staircase=True) - variables.global_variables_initializer().run() - # No change to learning rate - assign_1.op.run() - self.assertAllClose(decayed_lr().eval(), .1, 1e-6) - assign_2.op.run() - self.assertAllClose(decayed_lr().eval(), .1, 1e-6) - # Decayed learning rate - assign_100.op.run() - expected = .1 * 0.96 ** (100 // 3) - self.assertAllClose(decayed_lr().eval(), expected, 1e-6) + step = variables.Variable(1) + assign_1 = step.assign(1) + assign_2 = step.assign(2) + assign_100 = step.assign(100) + decayed_lr = learning_rate_decay_v2.exponential_decay( + .1, step, 3, 0.96, staircase=True) + self.evaluate(variables.global_variables_initializer()) + # No change to learning rate + self.evaluate(assign_1.op) + self.assertAllClose(self.evaluate(decayed_lr()), .1, 1e-6) + self.evaluate(assign_2.op) + self.assertAllClose(self.evaluate(decayed_lr()), .1, 1e-6) + # Decayed learning rate + self.evaluate(assign_100.op) + expected = .1 * 0.96**(100 // 3) + self.assertAllClose(self.evaluate(decayed_lr()), expected, 1e-6) @test_util.run_in_graph_and_eager_modes def testPiecewiseConstant(self): diff --git a/tensorflow/python/training/momentum.py b/tensorflow/python/training/momentum.py index 4a280e7c514..f3bc83bbfa1 100644 --- a/tensorflow/python/training/momentum.py +++ b/tensorflow/python/training/momentum.py @@ -25,7 +25,7 @@ from tensorflow.python.training import training_ops from tensorflow.python.util.tf_export import tf_export -@tf_export("train.MomentumOptimizer") +@tf_export(v1=["train.MomentumOptimizer"]) class MomentumOptimizer(optimizer.Optimizer): """Optimizer that implements the Momentum algorithm. diff --git a/tensorflow/python/training/momentum_test.py b/tensorflow/python/training/momentum_test.py index 8a21c39d323..ba155fa6c64 100644 --- a/tensorflow/python/training/momentum_test.py +++ b/tensorflow/python/training/momentum_test.py @@ -160,6 +160,7 @@ class MomentumOptimizerTest(test.TestCase): self.assertStartsWith(optimizer_variables[1].name, "var3") self.assertEquals(2, len(optimizer_variables)) + @test_util.run_deprecated_v1 def testNesterovMomentum(self): for dtype in [dtypes.float32, dtypes.float64]: with self.cached_session(): @@ -183,9 +184,10 @@ class MomentumOptimizerTest(test.TestCase): var1_np, accum1_np = self._update_nesterov_momentum_numpy(var1_np, accum1_np, 3, 2.0, 0.9) - self.assertAllClose(var0_np, var0.eval()) - self.assertAllClose(var1_np, var1.eval()) + self.assertAllClose(var0_np, self.evaluate(var0)) + self.assertAllClose(var1_np, self.evaluate(var1)) + @test_util.run_deprecated_v1 def testSparseNesterovMomentum(self): for dtype in [dtypes.float32, dtypes.float64]: with self.cached_session(): @@ -224,8 +226,8 @@ class MomentumOptimizerTest(test.TestCase): var1_np, accum1_np = self._update_nesterov_momentum_numpy(var1_np, accum1_np, 3, 2.0, 0.9) - self.assertAllClose(var0_np, var0.eval()) - self.assertAllClose(var1_np, var1.eval()) + self.assertAllClose(var0_np, self.evaluate(var0)) + self.assertAllClose(var1_np, self.evaluate(var1)) @test_util.run_in_graph_and_eager_modes(reset_test=True) def testMinimizeSparseResourceVariable(self): @@ -280,6 +282,7 @@ class MomentumOptimizerTest(test.TestCase): self.evaluate(sgd_op) self.assertAllCloseAccordingToType([[1, 1], [0, 0]], self.evaluate(var0)) + @test_util.run_deprecated_v1 def testTensorLearningRateAndMomentum(self): for dtype in [dtypes.half, dtypes.float32, dtypes.float64]: with self.cached_session(): @@ -303,37 +306,43 @@ class MomentumOptimizerTest(test.TestCase): self.assertFalse(slot1 in variables.trainable_variables()) # Fetch params to validate initial values - self.assertAllClose([1.0, 2.0], var0.eval()) - self.assertAllClose([3.0, 4.0], var1.eval()) + self.assertAllClose([1.0, 2.0], self.evaluate(var0)) + self.assertAllClose([3.0, 4.0], self.evaluate(var1)) # Step 1: the momentum accumulators where 0. So we should see a normal # update: v -= grad * learning_rate mom_update.run() # Check that the momentum accumulators have been updated. - self.assertAllCloseAccordingToType(np.array([0.1, 0.1]), slot0.eval()) - self.assertAllCloseAccordingToType(np.array([0.01, 0.01]), slot1.eval()) + self.assertAllCloseAccordingToType( + np.array([0.1, 0.1]), self.evaluate(slot0)) + self.assertAllCloseAccordingToType( + np.array([0.01, 0.01]), self.evaluate(slot1)) # Check that the parameters have been updated. self.assertAllCloseAccordingToType( - np.array([1.0 - (0.1 * 2.0), 2.0 - (0.1 * 2.0)]), var0.eval()) + np.array([1.0 - (0.1 * 2.0), 2.0 - (0.1 * 2.0)]), + self.evaluate(var0)) self.assertAllCloseAccordingToType( - np.array([3.0 - (0.01 * 2.0), 4.0 - (0.01 * 2.0)]), var1.eval()) + np.array([3.0 - (0.01 * 2.0), 4.0 - (0.01 * 2.0)]), + self.evaluate(var1)) # Step 2: the momentum accumulators contain the previous update. mom_update.run() # Check that the momentum accumulators have been updated. self.assertAllCloseAccordingToType( - np.array([(0.9 * 0.1 + 0.1), (0.9 * 0.1 + 0.1)]), slot0.eval()) + np.array([(0.9 * 0.1 + 0.1), (0.9 * 0.1 + 0.1)]), + self.evaluate(slot0)) self.assertAllCloseAccordingToType( - np.array([(0.9 * 0.01 + 0.01), (0.9 * 0.01 + 0.01)]), slot1.eval()) + np.array([(0.9 * 0.01 + 0.01), (0.9 * 0.01 + 0.01)]), + self.evaluate(slot1)) # Check that the parameters have been updated. self.assertAllCloseAccordingToType( np.array([ 1.0 - (0.1 * 2.0) - ((0.9 * 0.1 + 0.1) * 2.0), 2.0 - (0.1 * 2.0) - ((0.9 * 0.1 + 0.1) * 2.0) - ]), var0.eval()) + ]), self.evaluate(var0)) self.assertAllCloseAccordingToType( np.array([ - 2.98 - ((0.9 * 0.01 + 0.01) * 2.0), 3.98 - ( - (0.9 * 0.01 + 0.01) * 2.0) - ]), var1.eval()) + 2.98 - ((0.9 * 0.01 + 0.01) * 2.0), + 3.98 - ((0.9 * 0.01 + 0.01) * 2.0) + ]), self.evaluate(var1)) def _dbParamsMom01(self): """Return dist-belief momentum values. @@ -434,6 +443,7 @@ class MomentumOptimizerTest(test.TestCase): # pylint: enable=line-too-long return db_grad, db_out + @test_util.run_deprecated_v1 def testLikeDistBeliefMom01(self): with self.cached_session(): db_grad, db_out = self._dbParamsMom01() @@ -445,8 +455,9 @@ class MomentumOptimizerTest(test.TestCase): variables.global_variables_initializer().run() for i in xrange(num_samples): mom_update.run(feed_dict={grads0: db_grad[i]}) - self.assertAllClose(np.array(db_out[i]), var0.eval()) + self.assertAllClose(np.array(db_out[i]), self.evaluate(var0)) + @test_util.run_deprecated_v1 def testSparse(self): for dtype in [dtypes.half, dtypes.float32, dtypes.float64]: with self.cached_session(): @@ -476,46 +487,59 @@ class MomentumOptimizerTest(test.TestCase): self.assertEquals(slot1.get_shape(), var1.get_shape()) # Fetch params to validate initial values - self.assertAllClose([0, 0], var0.eval()[0]) - self.assertAllClose([0, 0], var0.eval()[1]) - self.assertAllClose([1, 1], var1.eval()[2]) + self.assertAllClose([0, 0], self.evaluate(var0)[0]) + self.assertAllClose([0, 0], self.evaluate(var0)[1]) + self.assertAllClose([1, 1], self.evaluate(var1)[2]) # Step 1: the momentum accumulators are 0. So we should see a normal # update: v -= grad * learning_rate mom_update.run() # Check that the momentum accumulators have been updated. - self.assertAllCloseAccordingToType(np.array([0, 0]), slot0.eval()[0]) - self.assertAllCloseAccordingToType(np.array([.1, .1]), slot0.eval()[1]) self.assertAllCloseAccordingToType( - np.array([.01, .01]), slot1.eval()[2]) + np.array([0, 0]), + self.evaluate(slot0)[0]) + self.assertAllCloseAccordingToType( + np.array([.1, .1]), + self.evaluate(slot0)[1]) + self.assertAllCloseAccordingToType( + np.array([.01, .01]), + self.evaluate(slot1)[2]) # Check that the parameters have been updated. - self.assertAllCloseAccordingToType(np.array([0, 0]), var0.eval()[0]) self.assertAllCloseAccordingToType( - np.array([-(0.1 * 2.0), -(0.1 * 2.0)]), var0.eval()[1]) + np.array([0, 0]), + self.evaluate(var0)[0]) self.assertAllCloseAccordingToType( - np.array([1.0 - (0.01 * 2.0), 1.0 - (0.01 * 2.0)]), var1.eval()[2]) + np.array([-(0.1 * 2.0), -(0.1 * 2.0)]), + self.evaluate(var0)[1]) + self.assertAllCloseAccordingToType( + np.array([1.0 - (0.01 * 2.0), 1.0 - (0.01 * 2.0)]), + self.evaluate(var1)[2]) # Step 2: the momentum accumulators contain the previous update. mom_update.run() # Check that the momentum accumulators have been updated. - self.assertAllClose(np.array([0, 0]), slot0.eval()[0]) + self.assertAllClose(np.array([0, 0]), self.evaluate(slot0)[0]) self.assertAllCloseAccordingToType( - np.array([(0.9 * 0.1 + 0.1), (0.9 * 0.1 + 0.1)]), slot0.eval()[1]) + np.array([(0.9 * 0.1 + 0.1), (0.9 * 0.1 + 0.1)]), + self.evaluate(slot0)[1]) self.assertAllCloseAccordingToType( np.array([(0.9 * 0.01 + 0.01), (0.9 * 0.01 + 0.01)]), - slot1.eval()[2]) + self.evaluate(slot1)[2]) # Check that the parameters have been updated. - self.assertAllClose(np.array([0, 0]), var0.eval()[0]) + self.assertAllClose(np.array([0, 0]), self.evaluate(var0)[0]) self.assertAllCloseAccordingToType( np.array([ - -(0.1 * 2.0) - ((0.9 * 0.1 + 0.1) * 2.0), -(0.1 * 2.0) - ( - (0.9 * 0.1 + 0.1) * 2.0) - ]), var0.eval()[1]) + -(0.1 * 2.0) - ((0.9 * 0.1 + 0.1) * 2.0), + -(0.1 * 2.0) - ((0.9 * 0.1 + 0.1) * 2.0) + ]), + self.evaluate(var0)[1]) self.assertAllCloseAccordingToType( np.array([ - 0.98 - ((0.9 * 0.01 + 0.01) * 2.0), 0.98 - ( - (0.9 * 0.01 + 0.01) * 2.0) - ]), var1.eval()[2]) + 0.98 - ((0.9 * 0.01 + 0.01) * 2.0), + 0.98 - ((0.9 * 0.01 + 0.01) * 2.0) + ]), + self.evaluate(var1)[2]) + @test_util.run_deprecated_v1 def testSharing(self): for dtype in [dtypes.half, dtypes.float32, dtypes.float64]: with self.cached_session(): @@ -538,37 +562,43 @@ class MomentumOptimizerTest(test.TestCase): self.assertEquals(slot1.get_shape(), var1.get_shape()) # Fetch params to validate initial values - self.assertAllClose([1.0, 2.0], var0.eval()) - self.assertAllClose([3.0, 4.0], var1.eval()) + self.assertAllClose([1.0, 2.0], self.evaluate(var0)) + self.assertAllClose([3.0, 4.0], self.evaluate(var1)) # Step 1: the momentum accumulators where 0. So we should see a normal # update: v -= grad * learning_rate mom_update1.run() # Check that the momentum accumulators have been updated. - self.assertAllCloseAccordingToType(np.array([0.1, 0.1]), slot0.eval()) - self.assertAllCloseAccordingToType(np.array([0.01, 0.01]), slot1.eval()) + self.assertAllCloseAccordingToType( + np.array([0.1, 0.1]), self.evaluate(slot0)) + self.assertAllCloseAccordingToType( + np.array([0.01, 0.01]), self.evaluate(slot1)) # Check that the parameters have been updated. self.assertAllCloseAccordingToType( - np.array([1.0 - (0.1 * 2.0), 2.0 - (0.1 * 2.0)]), var0.eval()) + np.array([1.0 - (0.1 * 2.0), 2.0 - (0.1 * 2.0)]), + self.evaluate(var0)) self.assertAllCloseAccordingToType( - np.array([3.0 - (0.01 * 2.0), 4.0 - (0.01 * 2.0)]), var1.eval()) + np.array([3.0 - (0.01 * 2.0), 4.0 - (0.01 * 2.0)]), + self.evaluate(var1)) # Step 2: the second momentum accumulators contain the previous update. mom_update2.run() # Check that the momentum accumulators have been updated. self.assertAllCloseAccordingToType( - np.array([(0.9 * 0.1 + 0.1), (0.9 * 0.1 + 0.1)]), slot0.eval()) + np.array([(0.9 * 0.1 + 0.1), (0.9 * 0.1 + 0.1)]), + self.evaluate(slot0)) self.assertAllCloseAccordingToType( - np.array([(0.9 * 0.01 + 0.01), (0.9 * 0.01 + 0.01)]), slot1.eval()) + np.array([(0.9 * 0.01 + 0.01), (0.9 * 0.01 + 0.01)]), + self.evaluate(slot1)) # Check that the parameters have been updated. self.assertAllCloseAccordingToType( np.array([ 1.0 - (0.1 * 2.0) - ((0.9 * 0.1 + 0.1) * 2.0), 2.0 - (0.1 * 2.0) - ((0.9 * 0.1 + 0.1) * 2.0) - ]), var0.eval()) + ]), self.evaluate(var0)) self.assertAllCloseAccordingToType( np.array([ - 2.98 - ((0.9 * 0.01 + 0.01) * 2.0), 3.98 - ( - (0.9 * 0.01 + 0.01) * 2.0) - ]), var1.eval()) + 2.98 - ((0.9 * 0.01 + 0.01) * 2.0), + 3.98 - ((0.9 * 0.01 + 0.01) * 2.0) + ]), self.evaluate(var1)) if __name__ == "__main__": diff --git a/tensorflow/python/training/monitored_session.py b/tensorflow/python/training/monitored_session.py index 162fef971db..6a7d27df5c3 100644 --- a/tensorflow/python/training/monitored_session.py +++ b/tensorflow/python/training/monitored_session.py @@ -54,7 +54,7 @@ _PREEMPTION_ERRORS = (errors.AbortedError, errors.UnavailableError) USE_DEFAULT = object() -@tf_export('train.Scaffold') +@tf_export(v1=['train.Scaffold']) class Scaffold(object): """Structure to create or gather pieces commonly needed to train a model. @@ -508,7 +508,7 @@ def MonitoredTrainingSession(master='', # pylint: disable=invalid-name stop_grace_period_secs=stop_grace_period_secs) -@tf_export('train.SessionCreator') +@tf_export(v1=['train.SessionCreator']) @six.add_metaclass(abc.ABCMeta) class SessionCreator(object): """A factory for tf.Session.""" @@ -519,7 +519,7 @@ class SessionCreator(object): 'create_session is not implemented for {}.'.format(self)) -@tf_export('train.ChiefSessionCreator') +@tf_export(v1=['train.ChiefSessionCreator']) class ChiefSessionCreator(SessionCreator): """Creates a tf.Session for a chief.""" @@ -571,7 +571,7 @@ class ChiefSessionCreator(SessionCreator): init_fn=self._scaffold.init_fn) -@tf_export('train.WorkerSessionCreator') +@tf_export(v1=['train.WorkerSessionCreator']) class WorkerSessionCreator(SessionCreator): """Creates a tf.Session for a worker.""" @@ -840,10 +840,18 @@ class _MonitoredSession(object): return self._coordinated_creator.tf_sess is None def _tf_sess(self): + """Return underlying tf.Session object. + + Warning: accessing the returned object in user code is likely to cause races + or "flaky tests". + + Returns: + A tf.Session object. + """ return self._coordinated_creator.tf_sess -@tf_export('train.MonitoredSession') +@tf_export(v1=['train.MonitoredSession']) class MonitoredSession(_MonitoredSession): """Session-like object that handles initialization, recovery and hooks. @@ -926,7 +934,7 @@ class MonitoredSession(_MonitoredSession): stop_grace_period_secs=stop_grace_period_secs) -@tf_export('train.SingularMonitoredSession') +@tf_export(v1=['train.SingularMonitoredSession']) class SingularMonitoredSession(_MonitoredSession): """Session-like object that handles initialization, restoring, and hooks. diff --git a/tensorflow/python/training/monitored_session_test.py b/tensorflow/python/training/monitored_session_test.py index c870d99de9e..9dbcfa52b7c 100644 --- a/tensorflow/python/training/monitored_session_test.py +++ b/tensorflow/python/training/monitored_session_test.py @@ -37,6 +37,7 @@ from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes from tensorflow.python.framework import errors_impl from tensorflow.python.framework import ops +from tensorflow.python.framework import test_util from tensorflow.python.ops import array_ops from tensorflow.python.ops import control_flow_ops from tensorflow.python.ops import resource_variable_ops @@ -382,6 +383,16 @@ class MonitoredTrainingSessionTest(test.TestCase): self.assertEqual(0, session.run(gstep)) +class MockExtended(object): + + def __init__(self, between_graph, should_init, should_checkpoint, + should_save_summary): + self.experimental_between_graph = between_graph + self.experimental_should_init = should_init + self.should_checkpoint = should_checkpoint + self.should_save_summary = should_save_summary + + class MockStrategy(object): def __init__(self, @@ -389,26 +400,8 @@ class MockStrategy(object): should_init=True, should_checkpoint=None, should_save_summary=None): - self._between_graph = between_graph - self._should_init = should_init - self._should_checkpoint = should_checkpoint - self._should_save_summary = should_save_summary - - @property - def between_graph(self): - return self._between_graph - - @property - def should_init(self): - return self._should_init - - @property - def should_checkpoint(self): - return self._should_checkpoint - - @property - def should_save_summary(self): - return self._should_save_summary + self.extended = MockExtended(between_graph, should_init, should_checkpoint, + should_save_summary) class MonitoredTrainingSessionWithDistributeCoordinatorTest(test.TestCase): @@ -512,6 +505,7 @@ class StopAtNSession(monitored_session._WrappedSession): class WrappedSessionTest(test.TestCase): """_WrappedSession tests.""" + @test_util.run_deprecated_v1 def test_properties(self): with self.cached_session() as sess: constant_op.constant(0.0) @@ -519,6 +513,7 @@ class WrappedSessionTest(test.TestCase): self.assertEquals(sess.graph, wrapped_sess.graph) self.assertEquals(sess.sess_str, wrapped_sess.sess_str) + @test_util.run_deprecated_v1 def test_should_stop_on_close(self): with self.cached_session() as sess: wrapped_sess = monitored_session._WrappedSession(sess) @@ -526,6 +521,7 @@ class WrappedSessionTest(test.TestCase): wrapped_sess.close() self.assertTrue(wrapped_sess.should_stop()) + @test_util.run_deprecated_v1 def test_should_stop_uses_check_stop(self): with self.cached_session() as sess: wrapped_sess = StopAtNSession(sess, 3) @@ -534,6 +530,7 @@ class WrappedSessionTest(test.TestCase): self.assertFalse(wrapped_sess.should_stop()) self.assertTrue(wrapped_sess.should_stop()) + @test_util.run_deprecated_v1 def test_should_stop_delegates_to_wrapped_session(self): with self.cached_session() as sess: wrapped_sess0 = StopAtNSession(sess, 4) @@ -552,6 +549,7 @@ class WrappedSessionTest(test.TestCase): wrapped_sess.close() self.assertTrue(wrapped_sess.should_stop()) + @test_util.run_deprecated_v1 def test_run(self): with self.cached_session() as sess: c = constant_op.constant(0) @@ -569,6 +567,7 @@ def busy_wait_for_coord_stop(coord): class CoordinatedSessionTest(test.TestCase): """_CoordinatedSession tests.""" + @test_util.run_deprecated_v1 def test_properties(self): with self.cached_session() as sess: constant_op.constant(0.0) @@ -577,6 +576,7 @@ class CoordinatedSessionTest(test.TestCase): self.assertEquals(sess.graph, coord_sess.graph) self.assertEquals(sess.sess_str, coord_sess.sess_str) + @test_util.run_deprecated_v1 def test_run(self): with self.cached_session() as sess: c = constant_op.constant(0) @@ -585,6 +585,7 @@ class CoordinatedSessionTest(test.TestCase): coord_sess = monitored_session._CoordinatedSession(sess, coord) self.assertEqual(42, coord_sess.run(v, feed_dict={c: 42})) + @test_util.run_deprecated_v1 def test_should_stop_on_close(self): with self.cached_session() as sess: coord = coordinator.Coordinator() @@ -593,6 +594,7 @@ class CoordinatedSessionTest(test.TestCase): coord_sess.close() self.assertTrue(coord_sess.should_stop()) + @test_util.run_deprecated_v1 def test_should_stop_on_coord_stop(self): with self.cached_session() as sess: coord = coordinator.Coordinator() @@ -601,6 +603,7 @@ class CoordinatedSessionTest(test.TestCase): coord.request_stop() self.assertTrue(coord_sess.should_stop()) + @test_util.run_deprecated_v1 def test_dont_request_stop_on_exception_in_main_thread(self): with self.cached_session() as sess: c = constant_op.constant(0) @@ -615,6 +618,7 @@ class CoordinatedSessionTest(test.TestCase): self.assertFalse(coord.should_stop()) self.assertFalse(coord_sess.should_stop()) + @test_util.run_deprecated_v1 def test_stop_threads_on_close_after_exception(self): with self.cached_session() as sess: c = constant_op.constant(0) @@ -662,6 +666,7 @@ class CoordinatedSessionTest(test.TestCase): self.assertTrue(coord.should_stop()) self.assertTrue(coord_sess.should_stop()) + @test_util.run_deprecated_v1 def test_propagates_exception_trace(self): assertion = control_flow_ops.Assert(False, ['This should fail.']) with self.cached_session() as sess: @@ -809,6 +814,7 @@ class RecoverableSessionTest(test.TestCase): def create_session(self): return self._sess + @test_util.run_deprecated_v1 def test_properties(self): with self.cached_session() as sess: constant_op.constant(0.0) @@ -817,6 +823,7 @@ class RecoverableSessionTest(test.TestCase): self.assertEquals(sess.graph, recoverable_sess.graph) self.assertEquals(sess.sess_str, recoverable_sess.sess_str) + @test_util.run_deprecated_v1 def test_run(self): with self.cached_session() as sess: c = constant_op.constant(0) @@ -825,6 +832,7 @@ class RecoverableSessionTest(test.TestCase): self._SessionReturner(sess)) self.assertEqual(51, recoverable_sess.run(v, feed_dict={c: 51})) + @test_util.run_deprecated_v1 def test_recovery(self): with self.cached_session() as sess: @@ -871,6 +879,7 @@ class RecoverableSessionTest(test.TestCase): with self.assertRaisesRegexp(IndexError, 'pop from empty list'): recoverable_sess.run(v, feed_dict={c: -12}) + @test_util.run_deprecated_v1 def test_recovery_from_coordinator_exception(self): with self.cached_session() as test_session: session_creator = CountingSessionCreator(test_session) @@ -896,6 +905,7 @@ class RecoverableSessionTest(test.TestCase): self.assertFalse(session.should_stop()) self.assertEqual(2, session_creator.number_of_sessions_created) + @test_util.run_deprecated_v1 def test_recovery_from_non_preemption_in_coordinator(self): with self.cached_session() as test_session: session_creator = CountingSessionCreator(test_session) @@ -925,6 +935,7 @@ class RecoverableSessionTest(test.TestCase): with self.assertRaises(errors_impl.UnknownError): session.close() + @test_util.run_deprecated_v1 def test_recovery_from_session_getting_stuck(self): with self.cached_session() as test_session: session_creator = CountingSessionCreator(test_session) @@ -949,6 +960,7 @@ class RecoverableSessionTest(test.TestCase): self.assertFalse(session.should_stop()) self.assertEqual(2, session_creator.number_of_sessions_created) + @test_util.run_deprecated_v1 def test_step_fn_recovery_from_coordinator_exception_when_run_hooks(self): with self.cached_session() as test_session: session_creator = CountingSessionCreator(test_session) @@ -979,6 +991,7 @@ class RecoverableSessionTest(test.TestCase): self.assertFalse(session.should_stop()) self.assertEqual(2, session_creator.number_of_sessions_created) + @test_util.run_deprecated_v1 def test_recovery_from_non_preemption_in_coordinator_when_run_hooks(self): with self.cached_session() as test_session: session_creator = CountingSessionCreator(test_session) @@ -1013,6 +1026,7 @@ class RecoverableSessionTest(test.TestCase): with self.assertRaises(errors_impl.UnknownError): session.close() + @test_util.run_deprecated_v1 def test_recovery_from_session_getting_stuck_when_run_hooks(self): with self.cached_session() as test_session: session_creator = CountingSessionCreator(test_session) @@ -1057,6 +1071,7 @@ class RecoverableSessionTest(test.TestCase): # exception. return session + @test_util.run_deprecated_v1 def test_step_fn_recovery_from_coordinator_exception_with_raw_session(self): with self.cached_session() as test_session: session_creator = CountingSessionCreator(test_session) @@ -1089,6 +1104,7 @@ class RecoverableSessionTest(test.TestCase): self.assertFalse(session.should_stop()) self.assertEqual(2, session_creator.number_of_sessions_created) + @test_util.run_deprecated_v1 def test_recovery_from_non_preemption_in_coordinator_with_raw_session(self): with self.cached_session() as test_session: session_creator = CountingSessionCreator(test_session) @@ -1126,6 +1142,7 @@ class RecoverableSessionTest(test.TestCase): with self.assertRaises(errors_impl.UnknownError): session.close() + @test_util.run_deprecated_v1 def test_recovery_from_session_getting_stuck_with_raw_session(self): with self.cached_session() as test_session: session_creator = CountingSessionCreator(test_session) @@ -1178,7 +1195,7 @@ class HookedSessionTest(test.TestCase): mock_run = FakeSession(sess) mon_sess = monitored_session._HookedSession(sess=mock_run, hooks=[]) a_tensor = constant_op.constant([0], name='a_tensor') - sess.run(variables.global_variables_initializer()) + self.evaluate(variables.global_variables_initializer()) output = mon_sess.run(fetches=a_tensor, feed_dict='a_feed', options='an_option', @@ -1197,7 +1214,7 @@ class HookedSessionTest(test.TestCase): mon_sess = monitored_session._HookedSession( sess=sess, hooks=[mock_hook, mock_hook2]) a_tensor = constant_op.constant([0], name='a_tensor') - sess.run(variables.global_variables_initializer()) + self.evaluate(variables.global_variables_initializer()) mon_sess.run(a_tensor) for hook in [mock_hook, mock_hook2]: @@ -1222,7 +1239,7 @@ class HookedSessionTest(test.TestCase): mon_sess = monitored_session._HookedSession( sess=sess, hooks=[mock_hook, mock_hook2]) constant_op.constant([0], name='a_tensor') - sess.run(variables.global_variables_initializer()) + self.evaluate(variables.global_variables_initializer()) mon_sess.run(fetches='a_tensor') self.assertFalse(mon_sess.should_stop()) @@ -1242,7 +1259,7 @@ class HookedSessionTest(test.TestCase): third_tensor = constant_op.constant([10], name='third_tensor') mock_hook.request = session_run_hook.SessionRunArgs([another_tensor]) mock_hook2.request = session_run_hook.SessionRunArgs([third_tensor]) - sess.run(variables.global_variables_initializer()) + self.evaluate(variables.global_variables_initializer()) output = mon_sess.run(fetches=a_tensor) self.assertEqual(output, [0]) @@ -1262,7 +1279,7 @@ class HookedSessionTest(test.TestCase): None, feed_dict={a_tensor: [5]}) mock_hook2.request = session_run_hook.SessionRunArgs( None, feed_dict={b_tensor: [10]}) - sess.run(variables.global_variables_initializer()) + self.evaluate(variables.global_variables_initializer()) self.assertEqual(mon_sess.run(fetches=add_tensor), [15]) @@ -1280,7 +1297,7 @@ class HookedSessionTest(test.TestCase): None, feed_dict={a_tensor: [5]}) mock_hook2.request = session_run_hook.SessionRunArgs( None, feed_dict={b_tensor: [10]}) - sess.run(variables.global_variables_initializer()) + self.evaluate(variables.global_variables_initializer()) feed_dict = {c_tensor: [20]} self.assertEqual( @@ -1301,7 +1318,7 @@ class HookedSessionTest(test.TestCase): None, feed_dict={a_tensor: [5]}) mock_hook2.request = session_run_hook.SessionRunArgs( None, feed_dict={a_tensor: [10]}) - sess.run(variables.global_variables_initializer()) + self.evaluate(variables.global_variables_initializer()) with self.assertRaisesRegexp(RuntimeError, 'Same tensor is fed'): mon_sess.run(fetches=add_tensor) @@ -1319,7 +1336,7 @@ class HookedSessionTest(test.TestCase): None, feed_dict={a_tensor: [5]}) mock_hook2.request = session_run_hook.SessionRunArgs( None, feed_dict={b_tensor: [10]}) - sess.run(variables.global_variables_initializer()) + self.evaluate(variables.global_variables_initializer()) with self.assertRaisesRegexp(RuntimeError, 'Same tensor is fed'): mon_sess.run(fetches=add_tensor, feed_dict={b_tensor: [10]}) @@ -1451,6 +1468,7 @@ class MonitoredSessionTest(test.TestCase): # This set of tests, verifies the supervised session behavior when exceptions # are raised next to the innermost session run() call. + @test_util.run_deprecated_v1 def test_recovery(self): logdir = _test_dir(self.get_temp_dir(), 'test_recovery') with ops.Graph().as_default(): @@ -1803,6 +1821,7 @@ class MonitoredSessionTest(test.TestCase): isinstance(hook.run_metadata_list[0], config_pb2.RunMetadata)) self.assertGreater(len(hook.run_metadata_list[0].partition_graphs), 0) + @test_util.run_deprecated_v1 def test_with_statement_and_close(self): # Test case for https://github.com/tensorflow/tensorflow/issues/12224 # where close() inside the with should have a better error message. diff --git a/tensorflow/python/training/moving_averages.py b/tensorflow/python/training/moving_averages.py index fc9eb479cc3..8785f9a8e71 100644 --- a/tensorflow/python/training/moving_averages.py +++ b/tensorflow/python/training/moving_averages.py @@ -17,6 +17,7 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function +from tensorflow.python.distribute import reduce_util as ds_reduce_util from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops from tensorflow.python.ops import control_flow_ops @@ -95,11 +96,11 @@ def assign_moving_average(variable, value, decay, zero_debias=True, name=None): # In a replica context, we update variable using the mean of value across # replicas. def merge_fn(strategy, v, value): - value = strategy.reduce( - variable_scope.VariableAggregation.MEAN, value, v) + value = strategy.extended.reduce_to( + ds_reduce_util.ReduceOp.MEAN, value, v) return strategy.update(v, update_fn, value) - return replica_context.merge_call(merge_fn, variable, value) + return replica_context.merge_call(merge_fn, args=(variable, value)) else: strategy = distribution_strategy_context.get_cross_replica_context() return strategy.update(variable, update_fn, value) diff --git a/tensorflow/python/training/moving_averages_test.py b/tensorflow/python/training/moving_averages_test.py index bb2fca66e3c..b15f7377f07 100644 --- a/tensorflow/python/training/moving_averages_test.py +++ b/tensorflow/python/training/moving_averages_test.py @@ -35,6 +35,7 @@ from tensorflow.python.training import saver as saver_lib class MovingAveragesTest(test.TestCase): + @test_util.run_deprecated_v1 def testAssignMovingAverageWithoutZeroDebias(self): with self.cached_session(): var = variables.Variable([10.0, 11.0]) @@ -43,12 +44,13 @@ class MovingAveragesTest(test.TestCase): assign = moving_averages.assign_moving_average( var, val, decay, zero_debias=False) variables.global_variables_initializer().run() - self.assertAllClose([10.0, 11.0], var.eval()) + self.assertAllClose([10.0, 11.0], self.evaluate(var)) assign.op.run() self.assertAllClose( [10.0 * 0.25 + 1.0 * (1.0 - 0.25), 11.0 * 0.25 + 2.0 * (1.0 - 0.25)], - var.eval()) + self.evaluate(var)) + @test_util.run_deprecated_v1 def testAssignMovingAverage(self): with self.cached_session(): var = variables.Variable([0.0, 0.0]) @@ -56,12 +58,13 @@ class MovingAveragesTest(test.TestCase): decay = 0.25 assign = moving_averages.assign_moving_average(var, val, decay) variables.global_variables_initializer().run() - self.assertAllClose([0.0, 0.0], var.eval()) + self.assertAllClose([0.0, 0.0], self.evaluate(var)) assign.op.run() - self.assertAllClose([ - 1.0 * (1.0 - 0.25) / (1 - 0.25), 2.0 * (1.0 - 0.25) / (1 - 0.25) - ], var.eval()) + self.assertAllClose( + [1.0 * (1.0 - 0.25) / (1 - 0.25), 2.0 * (1.0 - 0.25) / (1 - 0.25)], + self.evaluate(var)) + @test_util.run_deprecated_v1 def testAssignMovingAverageNewNamingMultipleCalls(self): with variable_scope.variable_scope("scope1") as vs1: with variable_scope.variable_scope("scope2"): @@ -76,6 +79,7 @@ class MovingAveragesTest(test.TestCase): actual_names = [v.name for v in vs1.global_variables()] self.assertSetEqual(set(expected_names), set(actual_names)) + @test_util.run_deprecated_v1 def testAssignMovingAverageNewNamingMultipleCallsWithReuse(self): with variable_scope.variable_scope("scope1") as vs1: var = variable_scope.get_variable("Var", shape=[]) @@ -86,6 +90,7 @@ class MovingAveragesTest(test.TestCase): moving_averages.assign_moving_average(var, 0.0, 0.99) moving_averages.assign_moving_average(var, 0.0, 0.99) + @test_util.run_deprecated_v1 def testWeightedMovingAverage(self): with self.cached_session() as sess: decay = 0.5 @@ -111,6 +116,7 @@ class MovingAveragesTest(test.TestCase): denominator_2 = denominator_1 * decay + weight_2 * (1.0 - decay) self.assertAllClose(numerator_2 / denominator_2, wma_array) + @test_util.run_deprecated_v1 def testWeightedMovingAverageBfloat16(self): bfloat16 = pywrap_tensorflow.TF_bfloat16_type() with self.cached_session() as sess: @@ -179,66 +185,72 @@ class ExponentialMovingAverageTest(test.TestCase): self.assertEqual("add/ExponentialMovingAverage:0", avg2.name) # Check initial values. - self.assertAllClose(tens, var0.eval()) - self.assertAllClose(thirties, var1.eval()) - self.assertAllClose(_Repeat(10.0 + 30.0, dim), tensor2.eval()) + self.assertAllClose(tens, self.evaluate(var0)) + self.assertAllClose(thirties, self.evaluate(var1)) + self.assertAllClose(_Repeat(10.0 + 30.0, dim), self.evaluate(tensor2)) # Check that averages are initialized correctly. - self.assertAllClose(tens, avg0.eval()) - self.assertAllClose(thirties, avg1.eval()) + self.assertAllClose(tens, self.evaluate(avg0)) + self.assertAllClose(thirties, self.evaluate(avg1)) # Note that averages of Tensor's initialize to zeros_like since no value # of the Tensor is known because the Op has not been run (yet). - self.assertAllClose(_Repeat(0.0, dim), avg2.eval()) + self.assertAllClose(_Repeat(0.0, dim), self.evaluate(avg2)) # Update the averages and check. update.run() dk = actual_decay expected = _Repeat(10.0 * dk + 10.0 * (1 - dk), dim) - self.assertAllClose(expected, avg0.eval()) + self.assertAllClose(expected, self.evaluate(avg0)) expected = _Repeat(30.0 * dk + 30.0 * (1 - dk), dim) - self.assertAllClose(expected, avg1.eval()) + self.assertAllClose(expected, self.evaluate(avg1)) expected = _Repeat(0.0 * dk + (10.0 + 30.0) * (1 - dk) / _Scale(dk, 1), dim) - self.assertAllClose(expected, avg2.eval()) + self.assertAllClose(expected, self.evaluate(avg2)) # Again, update the averages and check. update.run() expected = _Repeat((10.0 * dk + 10.0 * (1 - dk)) * dk + 10.0 * (1 - dk), dim) - self.assertAllClose(expected, avg0.eval()) + self.assertAllClose(expected, self.evaluate(avg0)) expected = _Repeat((30.0 * dk + 30.0 * (1 - dk)) * dk + 30.0 * (1 - dk), dim) - self.assertAllClose(expected, avg1.eval()) + self.assertAllClose(expected, self.evaluate(avg1)) expected = _Repeat(((0.0 * dk + (10.0 + 30.0) * (1 - dk)) * dk + (10.0 + 30.0) * (1 - dk)) / _Scale(dk, 2), dim) - self.assertAllClose(expected, avg2.eval()) + self.assertAllClose(expected, self.evaluate(avg2)) + @test_util.run_deprecated_v1 def testAverageVariablesNoNumUpdates_Scalar(self): with self.cached_session(): ema = moving_averages.ExponentialMovingAverage(0.25) self._CheckDecay(ema, actual_decay=0.25, dim=1) + @test_util.run_deprecated_v1 def testAverageVariablesNoNumUpdates_Scalar_Debias(self): with self.cached_session(): ema = moving_averages.ExponentialMovingAverage(0.25, zero_debias=True) self._CheckDecay(ema, actual_decay=0.25, dim=1) + @test_util.run_deprecated_v1 def testAverageVariablesNoNumUpdates_Vector(self): with self.cached_session(): ema = moving_averages.ExponentialMovingAverage(0.25) self._CheckDecay(ema, actual_decay=0.25, dim=5) + @test_util.run_deprecated_v1 def testAverageVariablesNoNumUpdates_Vector_Debias(self): with self.cached_session(): ema = moving_averages.ExponentialMovingAverage(0.25, zero_debias=True) self._CheckDecay(ema, actual_decay=0.25, dim=5) + @test_util.run_deprecated_v1 def testAverageVariablesNumUpdates_Scalar(self): with self.cached_session(): # With num_updates 1, the decay applied is 0.1818 ema = moving_averages.ExponentialMovingAverage(0.25, num_updates=1) self._CheckDecay(ema, actual_decay=0.181818, dim=1) + @test_util.run_deprecated_v1 def testAverageVariablesNumUpdates_Scalar_Debias(self): with self.cached_session(): # With num_updates 1, the decay applied is 0.1818 @@ -246,12 +258,14 @@ class ExponentialMovingAverageTest(test.TestCase): 0.25, num_updates=1, zero_debias=True) self._CheckDecay(ema, actual_decay=0.181818, dim=1) + @test_util.run_deprecated_v1 def testAverageVariablesNumUpdates_Vector(self): with self.cached_session(): # With num_updates 1, the decay applied is 0.1818 ema = moving_averages.ExponentialMovingAverage(0.25, num_updates=1) self._CheckDecay(ema, actual_decay=0.181818, dim=5) + @test_util.run_deprecated_v1 def testAverageVariablesNumUpdates_Vector_Debias(self): with self.cached_session(): # With num_updates 1, the decay applied is 0.1818 @@ -259,6 +273,7 @@ class ExponentialMovingAverageTest(test.TestCase): 0.25, num_updates=1, zero_debias=True) self._CheckDecay(ema, actual_decay=0.181818, dim=5) + @test_util.run_deprecated_v1 def testAverageVariablesWithControlDeps(self): with self.cached_session() as sess: v0 = variables.Variable(0, name="v0") @@ -274,16 +289,17 @@ class ExponentialMovingAverageTest(test.TestCase): self.assertEqual([], v1_avg.value().op.control_inputs) self.assertEqual([], v1_avg.value().op.control_inputs) # We should be able to initialize v1_avg before v0. - sess.run(v1_avg.initializer) - sess.run(v0.initializer) - self.assertEqual([10.0], sess.run(v1_avg)) + self.evaluate(v1_avg.initializer) + self.evaluate(v0.initializer) + self.assertEqual([10.0], self.evaluate(v1_avg)) # running ema_op should add to v0 (in addition to updating v1_avg) - sess.run(assign_to_v1) - sess.run(ema_op) - self.assertEqual(1, sess.run(v0)) - self.assertEqual([17.5], sess.run(v1_avg)) + self.evaluate(assign_to_v1) + self.evaluate(ema_op) + self.assertEqual(1, self.evaluate(v0)) + self.assertEqual([17.5], self.evaluate(v1_avg)) @test_util.run_in_graph_and_eager_modes + @test_util.run_deprecated_v1 def testBasicEager(self): v0 = variables.Variable(1.0) v1 = variables.Variable(2.0) @@ -339,9 +355,11 @@ class ExponentialMovingAverageTest(test.TestCase): self.assertEqual(ema.average(v1).op.name, ema.average_name(v1)) self.assertEqual(ema.average(tensor2).op.name, ema.average_name(tensor2)) + @test_util.run_deprecated_v1 def testAverageVariablesNames(self): self.averageVariablesNamesHelper(zero_debias=True) + @test_util.run_deprecated_v1 def testAverageVariablesNamesNoDebias(self): self.averageVariablesNamesHelper(zero_debias=False) @@ -387,12 +405,15 @@ class ExponentialMovingAverageTest(test.TestCase): self.assertEqual( ema.average(tensor2).op.name, ema.average_name(tensor2)) + @test_util.run_deprecated_v1 def testAverageVariablesNamesRespectScope(self): self.averageVariablesNamesRespectScopeHelper(zero_debias=True) + @test_util.run_deprecated_v1 def testAverageVariablesNamesRespectScopeNoDebias(self): self.averageVariablesNamesRespectScopeHelper(zero_debias=False) + @test_util.run_deprecated_v1 def testSubsetAverageVariablesNames(self): with self.cached_session(): v0 = variables.Variable(10.0, name="v0") @@ -421,6 +442,7 @@ class ExponentialMovingAverageTest(test.TestCase): self.assertEqual(ema.average(v1).op.name, ema.average_name(v1)) self.assertEqual(ema.average(tensor2).op.name, ema.average_name(tensor2)) + @test_util.run_deprecated_v1 def testAverageVariablesDeviceAssignment(self): with ops.device("/job:dev_v0"): v0 = variables.Variable(10.0, name="v0") @@ -451,6 +473,7 @@ class ExponentialMovingAverageTest(test.TestCase): _ = saver_lib.import_meta_graph(meta_graph) return graph_copy + @test_util.run_deprecated_v1 def testImportedGraphVariablesToRestore(self): g = ops.Graph() with g.as_default(): diff --git a/tensorflow/python/training/optimizer.py b/tensorflow/python/training/optimizer.py index 9dfa9d2afb2..a9508b862ae 100644 --- a/tensorflow/python/training/optimizer.py +++ b/tensorflow/python/training/optimizer.py @@ -24,6 +24,8 @@ import abc import six +from tensorflow.python.distribute import distribute_lib +from tensorflow.python.distribute import reduce_util as ds_reduce_util from tensorflow.python.eager import backprop from tensorflow.python.eager import context from tensorflow.python.framework import dtypes @@ -36,7 +38,6 @@ from tensorflow.python.ops import resource_variable_ops from tensorflow.python.ops import state_ops from tensorflow.python.ops import variable_scope from tensorflow.python.ops import variables -from tensorflow.python.training import distribute as distribute_lib from tensorflow.python.training import distribution_strategy_context as distribute_ctx from tensorflow.python.training import slot_creator from tensorflow.python.training.checkpointable import base as checkpointable @@ -200,8 +201,7 @@ def _get_processor(v): return _TensorProcessor(v) else: return _DenseResourceVariableProcessor(v) - if isinstance( - v, resource_variable_ops.ResourceVariable) and not v._in_graph_mode: # pylint: disable=protected-access + if resource_variable_ops.is_resource_variable(v) and not v._in_graph_mode: # pylint: disable=protected-access # True if and only if `v` was initialized eagerly. return _DenseResourceVariableProcessor(v) if v.op.type == "VarHandleOp": @@ -213,7 +213,7 @@ def _get_processor(v): raise NotImplementedError("Trying to optimize unsupported type ", v) -@tf_export("train.Optimizer") +@tf_export(v1=["train.Optimizer"]) class Optimizer( # Optimizers inherit from CheckpointableBase rather than Checkpointable # since they do most of their dependency management themselves (slot @@ -520,8 +520,7 @@ class Optimizer( @staticmethod def _scale_loss(loss_value): - if (distribute_lib.get_loss_reduction() == - variable_scope.VariableAggregation.MEAN): + if distribute_lib.get_loss_reduction() == ds_reduce_util.ReduceOp.MEAN: num_replicas = \ distribute_ctx.get_distribution_strategy().num_replicas_in_sync if num_replicas > 1: @@ -565,7 +564,7 @@ class Optimizer( if distribute_ctx.has_distribution_strategy(): grads_and_vars = get_filtered_grad_fn(lambda: grads_and_vars)() return distribute_ctx.get_replica_context().merge_call( - self._distributed_apply, grads_and_vars, global_step, name) + self._distributed_apply, args=(grads_and_vars, global_step, name)) # No DistributionStrategy case. grads_and_vars = tuple(grads_and_vars) # Make sure repeat iteration works. @@ -658,14 +657,16 @@ class Optimizer( Returns: An `Operation` that applies the specified gradients across all replicas. If `global_step` was not None, that operation also - increments `global_step`. + increments `global_step` """ - reduced_grads = distribution.batch_reduce( - variable_scope.VariableAggregation.SUM, grads_and_vars) + reduced_grads = distribution.extended.batch_reduce_to( + ds_reduce_util.ReduceOp.SUM, grads_and_vars) var_list = [v for _, v in grads_and_vars] grads_and_vars = zip(reduced_grads, var_list) + # Note that this is called in a cross-replica context. - self._create_slots(var_list) + with ops.init_scope(): + self._create_slots(var_list) def update(v, g): """Apply gradients to a replica variable.""" @@ -682,7 +683,13 @@ class Optimizer( "Gradient must be a Tensor, IndexedSlices, or None: %s" % g) p = _get_processor(v) - scope_name = "" if context.executing_eagerly() else v.op.name + if context.executing_eagerly() or ( + resource_variable_ops.is_resource_variable(v) and + not v._in_graph_mode): # pylint: disable=protected-access + scope_name = v.name.split(":")[0] + else: + scope_name = v.op.name + # device_policy is set because non-mirrored tensors will be read in # `update_op`. `_resource_apply_dense`, `lr_t`, `beta1_t` and `beta2_t` # is an example. @@ -695,21 +702,23 @@ class Optimizer( update_ops = [ op for grad, var in grads_and_vars - for op in distribution.update(var, update, grad, grouped=False) + for op in distribution.extended.update( + var, update, args=(grad,), group=False) ] def finish(self, update_ops): return self._finish(update_ops, "update") - non_slot_devices = distribution.non_slot_devices(var_list) - finish_updates = distribution.update_non_slot( - non_slot_devices, finish, self, update_ops, grouped=False) + non_slot_devices = distribution.extended.non_slot_devices(var_list) + finish_updates = distribution.extended.update_non_slot( + non_slot_devices, finish, args=(self, update_ops), group=False) if global_step is None: apply_updates = distribution.group(finish_updates, name=name) else: with ops.control_dependencies(finish_updates): - apply_updates = distribution.update( - global_step, state_ops.assign_add, 1, name=name) + apply_updates = distribution.extended.update( + global_step, state_ops.assign_add, args=(1,), + kwargs={"name": name}) if not context.executing_eagerly(): if isinstance(apply_updates, ops.Tensor): @@ -747,7 +756,7 @@ class Optimizer( # `_resource_apply_dense`. distributed_container = var._distributed_container() assert distributed_container is not None - if context.executing_eagerly(): + if ops.executing_eagerly_outside_functions(): key = distributed_container._unique_id else: key = (distributed_container.graph, distributed_container._shared_name) diff --git a/tensorflow/python/training/optimizer_test.py b/tensorflow/python/training/optimizer_test.py index 7a7d01d50e0..e175b5a7998 100644 --- a/tensorflow/python/training/optimizer_test.py +++ b/tensorflow/python/training/optimizer_test.py @@ -62,6 +62,7 @@ class OptimizerTest(test.TestCase): self.assertAllClose([-14., -13.], self.evaluate(var0)) self.assertAllClose([-6., -5.], self.evaluate(var1)) + @test_util.run_deprecated_v1 def testAggregationMethod(self): for dtype in [dtypes.half, dtypes.float32, dtypes.float64]: with self.cached_session(): @@ -79,14 +80,15 @@ class OptimizerTest(test.TestCase): variables.global_variables_initializer().run() # Fetch params to validate initial values - self.assertAllClose([1.0, 2.0], var0.eval()) - self.assertAllClose([3.0, 4.0], var1.eval()) + self.assertAllClose([1.0, 2.0], self.evaluate(var0)) + self.assertAllClose([3.0, 4.0], self.evaluate(var1)) # Run 1 step of sgd through optimizer opt_op.run() # Validate updated params - self.assertAllClose([-14., -13.], var0.eval()) - self.assertAllClose([-6., -5.], var1.eval()) + self.assertAllClose([-14., -13.], self.evaluate(var0)) + self.assertAllClose([-6., -5.], self.evaluate(var1)) + @test_util.run_deprecated_v1 def testPrecomputedGradient(self): for dtype in [dtypes.half, dtypes.float32, dtypes.float64]: with self.cached_session(): @@ -102,15 +104,15 @@ class OptimizerTest(test.TestCase): variables.global_variables_initializer().run() # Fetch params to validate initial values - self.assertAllClose([1.0, 2.0], var0.eval()) - self.assertAllClose([3.0, 4.0], var1.eval()) + self.assertAllClose([1.0, 2.0], self.evaluate(var0)) + self.assertAllClose([3.0, 4.0], self.evaluate(var1)) # Run 1 step of sgd through optimizer opt_op.run() # Validate updated params self.assertAllClose([1.0 - 3 * 5 * 42.0, 2.0 - 3 * 5 * (-42.0)], - var0.eval()) + self.evaluate(var0)) self.assertAllClose([3.0 - 3 * 3 * 42.0, 4.0 - 3 * 3 * (-42.0)], - var1.eval()) + self.evaluate(var1)) @test_util.run_in_graph_and_eager_modes def testNoVariables(self): @@ -230,6 +232,7 @@ class OptimizerTest(test.TestCase): with self.assertRaises(NotImplementedError): sgd_op.apply_gradients(grads_and_vars) + @test_util.run_deprecated_v1 def testTrainOp(self): with self.cached_session(): var0 = variables.Variable([1.0, 2.0]) @@ -241,6 +244,7 @@ class OptimizerTest(test.TestCase): opt_op = sgd_op.minimize(cost, global_step, [var0, var1]) self.assertTrue(opt_op in ops.get_collection(ops.GraphKeys.TRAIN_OP)) + @test_util.run_deprecated_v1 def testConstraint(self): constraint_01 = lambda x: clip_ops.clip_by_value(x, -0.1, 0.) constraint_0 = lambda x: clip_ops.clip_by_value(x, 0., 1.) @@ -257,13 +261,13 @@ class OptimizerTest(test.TestCase): variables.global_variables_initializer().run() # Fetch params to validate initial values - self.assertAllClose([1.0, 2.0], var0.eval()) - self.assertAllClose([3.0, 4.0], var1.eval()) + self.assertAllClose([1.0, 2.0], self.evaluate(var0)) + self.assertAllClose([3.0, 4.0], self.evaluate(var1)) # Run 1 step of sgd through optimizer opt_op.run() # Validate updated params - self.assertAllClose([-0.1, -0.1], var0.eval()) - self.assertAllClose([0., 0.], var1.eval()) + self.assertAllClose([-0.1, -0.1], self.evaluate(var0)) + self.assertAllClose([0., 0.], self.evaluate(var1)) if __name__ == '__main__': diff --git a/tensorflow/python/training/proximal_adagrad.py b/tensorflow/python/training/proximal_adagrad.py index 9bd677b8efc..2ea628a56b4 100644 --- a/tensorflow/python/training/proximal_adagrad.py +++ b/tensorflow/python/training/proximal_adagrad.py @@ -26,7 +26,7 @@ from tensorflow.python.training import training_ops from tensorflow.python.util.tf_export import tf_export -@tf_export("train.ProximalAdagradOptimizer") +@tf_export(v1=["train.ProximalAdagradOptimizer"]) class ProximalAdagradOptimizer(optimizer.Optimizer): # pylint: disable=line-too-long """Optimizer that implements the Proximal Adagrad algorithm. diff --git a/tensorflow/python/training/proximal_adagrad_test.py b/tensorflow/python/training/proximal_adagrad_test.py index 74e06a5e2e6..ce214ac418a 100644 --- a/tensorflow/python/training/proximal_adagrad_test.py +++ b/tensorflow/python/training/proximal_adagrad_test.py @@ -23,6 +23,7 @@ import numpy as np from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops +from tensorflow.python.framework import test_util from tensorflow.python.ops import embedding_ops from tensorflow.python.ops import math_ops from tensorflow.python.ops import resource_variable_ops @@ -48,7 +49,7 @@ class ProximalAdagradOptimizerTest(test.TestCase): update = opt.apply_gradients(zip([grads0, grads1], [var0, var1])) variables.global_variables_initializer().run() - v0_val, v1_val = sess.run([var0, var1]) + v0_val, v1_val = self.evaluate([var0, var1]) self.assertAllClose([0.0, 0.0], v0_val) self.assertAllClose([0.0, 0.0], v1_val) @@ -56,7 +57,7 @@ class ProximalAdagradOptimizerTest(test.TestCase): for _ in range(3): update.run() - v0_val, v1_val = sess.run([var0, var1]) + v0_val, v1_val = self.evaluate([var0, var1]) self.assertAllClose(np.array([-2.60260963, -4.29698515]), v0_val) self.assertAllClose(np.array([-0.28432083, -0.56694895]), v1_val) opt_vars = opt.variables() @@ -64,12 +65,15 @@ class ProximalAdagradOptimizerTest(test.TestCase): self.assertStartsWith(opt_vars[1].name, var1._shared_name) self.assertEqual(2, len(opt_vars)) + @test_util.run_deprecated_v1 def testProximalAdagradwithoutRegularization(self): self.doTestProximalAdagradwithoutRegularization(use_resource=False) + @test_util.run_deprecated_v1 def testResourceProximalAdagradwithoutRegularization(self): self.doTestProximalAdagradwithoutRegularization(use_resource=True) + @test_util.run_deprecated_v1 def testProximalAdagradwithoutRegularization2(self): with self.cached_session() as sess: var0 = variables.Variable([1.0, 2.0]) @@ -85,17 +89,18 @@ class ProximalAdagradOptimizerTest(test.TestCase): update = opt.apply_gradients(zip([grads0, grads1], [var0, var1])) variables.global_variables_initializer().run() - v0_val, v1_val = sess.run([var0, var1]) + v0_val, v1_val = self.evaluate([var0, var1]) self.assertAllClose([1.0, 2.0], v0_val) self.assertAllClose([4.0, 3.0], v1_val) # Run 3 steps Proximal Adagrad. for _ in range(3): update.run() - v0_val, v1_val = sess.run([var0, var1]) + v0_val, v1_val = self.evaluate([var0, var1]) self.assertAllClose(np.array([-1.60261, -2.296985]), v0_val) self.assertAllClose(np.array([3.715679, 2.433051]), v1_val) + @test_util.run_deprecated_v1 def testMinimizeSparseResourceVariable(self): for dtype in [dtypes.float32, dtypes.float64]: with self.cached_session(): @@ -106,13 +111,15 @@ class ProximalAdagradOptimizerTest(test.TestCase): sgd_op = proximal_adagrad.ProximalAdagradOptimizer(1.0).minimize(loss) variables.global_variables_initializer().run() # Fetch params to validate initial values - self.assertAllCloseAccordingToType([[1.0, 2.0]], var0.eval()) + self.assertAllCloseAccordingToType([[1.0, 2.0]], self.evaluate(var0)) # Run 1 step of sgd sgd_op.run() # Validate updated params - self.assertAllCloseAccordingToType( - [[0, 1]], var0.eval(), atol=0.01) + self.assertAllCloseAccordingToType([[0, 1]], + self.evaluate(var0), + atol=0.01) + @test_util.run_deprecated_v1 def testProximalAdagradWithL1(self): with self.cached_session() as sess: var0 = variables.Variable([1.0, 2.0]) @@ -128,17 +135,18 @@ class ProximalAdagradOptimizerTest(test.TestCase): update = opt.apply_gradients(zip([grads0, grads1], [var0, var1])) variables.global_variables_initializer().run() - v0_val, v1_val = sess.run([var0, var1]) + v0_val, v1_val = self.evaluate([var0, var1]) self.assertAllClose([1.0, 2.0], v0_val) self.assertAllClose([4.0, 3.0], v1_val) # Run 10 steps Proximal Adagrad for _ in range(10): update.run() - v0_val, v1_val = sess.run([var0, var1]) + v0_val, v1_val = self.evaluate([var0, var1]) self.assertAllClose(np.array([-6.663634, -9.190331]), v0_val) self.assertAllClose(np.array([2.959304, 1.029232]), v1_val) + @test_util.run_deprecated_v1 def testProximalAdagradWithL1_L2(self): with self.cached_session() as sess: var0 = variables.Variable([1.0, 2.0]) @@ -154,7 +162,7 @@ class ProximalAdagradOptimizerTest(test.TestCase): update = opt.apply_gradients(zip([grads0, grads1], [var0, var1])) variables.global_variables_initializer().run() - v0_val, v1_val = sess.run([var0, var1]) + v0_val, v1_val = self.evaluate([var0, var1]) self.assertAllClose([1.0, 2.0], v0_val) self.assertAllClose([4.0, 3.0], v1_val) @@ -162,7 +170,7 @@ class ProximalAdagradOptimizerTest(test.TestCase): for _ in range(10): update.run() - v0_val, v1_val = sess.run([var0, var1]) + v0_val, v1_val = self.evaluate([var0, var1]) self.assertAllClose(np.array([-0.0495, -0.0995]), v0_val) self.assertAllClose(np.array([-0.0045, -0.0095]), v1_val) @@ -190,7 +198,7 @@ class ProximalAdagradOptimizerTest(test.TestCase): variables.global_variables_initializer().run() sess = ops.get_default_session() - v0_val, v1_val = sess.run([var0, var1]) + v0_val, v1_val = self.evaluate([var0, var1]) if is_sparse: self.assertAllClose([[1.0], [2.0]], v0_val) self.assertAllClose([[3.0], [4.0]], v1_val) @@ -202,9 +210,10 @@ class ProximalAdagradOptimizerTest(test.TestCase): for _ in range(steps): update.run() - v0_val, v1_val = sess.run([var0, var1]) + v0_val, v1_val = self.evaluate([var0, var1]) return v0_val, v1_val + @test_util.run_deprecated_v1 def testEquivAdagradwithoutRegularization(self): with self.cached_session(): val0, val1 = self.applyOptimizer( @@ -222,6 +231,7 @@ class ProximalAdagradOptimizerTest(test.TestCase): self.assertAllClose(val0, val2) self.assertAllClose(val1, val3) + @test_util.run_deprecated_v1 def testEquivSparseAdagradwithoutRegularization(self): with self.cached_session(): val0, val1 = self.applyOptimizer( diff --git a/tensorflow/python/training/proximal_gradient_descent_test.py b/tensorflow/python/training/proximal_gradient_descent_test.py index f77f68b2343..25b206605dc 100644 --- a/tensorflow/python/training/proximal_gradient_descent_test.py +++ b/tensorflow/python/training/proximal_gradient_descent_test.py @@ -23,6 +23,7 @@ import numpy as np from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops +from tensorflow.python.framework import test_util from tensorflow.python.ops import embedding_ops from tensorflow.python.ops import math_ops from tensorflow.python.ops import resource_variable_ops @@ -50,7 +51,7 @@ class ProximalGradientDescentOptimizerTest(test.TestCase): update = opt.apply_gradients(zip([grads0, grads1], [var0, var1])) variables.global_variables_initializer().run() - v0_val, v1_val = sess.run([var0, var1]) + v0_val, v1_val = self.evaluate([var0, var1]) self.assertAllClose([0.0, 0.0], v0_val) self.assertAllClose([0.0, 0.0], v1_val) @@ -58,16 +59,19 @@ class ProximalGradientDescentOptimizerTest(test.TestCase): for _ in range(3): update.run() - v0_val, v1_val = sess.run([var0, var1]) + v0_val, v1_val = self.evaluate([var0, var1]) self.assertAllClose(np.array([-0.9, -1.8]), v0_val) self.assertAllClose(np.array([-0.09, -0.18]), v1_val) + @test_util.run_deprecated_v1 def testProximalGradientDescentwithoutRegularization(self): self.doTestProximalGradientDescentwithoutRegularization(use_resource=False) + @test_util.run_deprecated_v1 def testResourceProximalGradientDescentwithoutRegularization(self): self.doTestProximalGradientDescentwithoutRegularization(use_resource=True) + @test_util.run_deprecated_v1 def testProximalGradientDescentwithoutRegularization2(self): with self.cached_session() as sess: var0 = variables.Variable([1.0, 2.0]) @@ -80,7 +84,7 @@ class ProximalGradientDescentOptimizerTest(test.TestCase): update = opt.apply_gradients(zip([grads0, grads1], [var0, var1])) variables.global_variables_initializer().run() - v0_val, v1_val = sess.run([var0, var1]) + v0_val, v1_val = self.evaluate([var0, var1]) self.assertAllClose([1.0, 2.0], v0_val) self.assertAllClose([4.0, 3.0], v1_val) @@ -88,10 +92,11 @@ class ProximalGradientDescentOptimizerTest(test.TestCase): for _ in range(3): update.run() - v0_val, v1_val = sess.run([var0, var1]) + v0_val, v1_val = self.evaluate([var0, var1]) self.assertAllClose(np.array([0.1, 0.2]), v0_val) self.assertAllClose(np.array([3.91, 2.82]), v1_val) + @test_util.run_deprecated_v1 def testMinimizeSparseResourceVariable(self): for dtype in [dtypes.float32, dtypes.float64]: with self.cached_session(): @@ -103,13 +108,15 @@ class ProximalGradientDescentOptimizerTest(test.TestCase): 1.0).minimize(loss) variables.global_variables_initializer().run() # Fetch params to validate initial values - self.assertAllCloseAccordingToType([[1.0, 2.0]], var0.eval()) + self.assertAllCloseAccordingToType([[1.0, 2.0]], self.evaluate(var0)) # Run 1 step of sgd sgd_op.run() # Validate updated params - self.assertAllCloseAccordingToType( - [[-111, -138]], var0.eval(), atol=0.01) + self.assertAllCloseAccordingToType([[-111, -138]], + self.evaluate(var0), + atol=0.01) + @test_util.run_deprecated_v1 def testProximalGradientDescentWithL1_L2(self): with self.cached_session() as sess: var0 = variables.Variable([1.0, 2.0]) @@ -122,7 +129,7 @@ class ProximalGradientDescentOptimizerTest(test.TestCase): update = opt.apply_gradients(zip([grads0, grads1], [var0, var1])) variables.global_variables_initializer().run() - v0_val, v1_val = sess.run([var0, var1]) + v0_val, v1_val = self.evaluate([var0, var1]) self.assertAllClose([1.0, 2.0], v0_val) self.assertAllClose([4.0, 3.0], v1_val) @@ -130,7 +137,7 @@ class ProximalGradientDescentOptimizerTest(test.TestCase): for _ in range(10): update.run() - v0_val, v1_val = sess.run([var0, var1]) + v0_val, v1_val = self.evaluate([var0, var1]) self.assertAllClose(np.array([-0.0495, -0.0995]), v0_val) self.assertAllClose(np.array([-0.0045, -0.0095]), v1_val) @@ -158,7 +165,7 @@ class ProximalGradientDescentOptimizerTest(test.TestCase): variables.global_variables_initializer().run() sess = ops.get_default_session() - v0_val, v1_val = sess.run([var0, var1]) + v0_val, v1_val = self.evaluate([var0, var1]) if is_sparse: self.assertAllClose([[1.0], [2.0]], v0_val) self.assertAllClose([[3.0], [4.0]], v1_val) @@ -170,9 +177,10 @@ class ProximalGradientDescentOptimizerTest(test.TestCase): for _ in range(steps): update.run() - v0_val, v1_val = sess.run([var0, var1]) + v0_val, v1_val = self.evaluate([var0, var1]) return v0_val, v1_val + @test_util.run_deprecated_v1 def testEquivSparseGradientDescentwithoutRegularization(self): with self.cached_session(): val0, val1 = self.applyOptimizer( @@ -189,6 +197,7 @@ class ProximalGradientDescentOptimizerTest(test.TestCase): self.assertAllClose(val0, val2) self.assertAllClose(val1, val3) + @test_util.run_deprecated_v1 def testEquivGradientDescentwithoutRegularization(self): with self.cached_session(): val0, val1 = self.applyOptimizer( diff --git a/tensorflow/python/training/quantize_training_test.py b/tensorflow/python/training/quantize_training_test.py index 6edbf7665fb..62e783f2000 100644 --- a/tensorflow/python/training/quantize_training_test.py +++ b/tensorflow/python/training/quantize_training_test.py @@ -25,6 +25,7 @@ from tensorflow.python.client import session from tensorflow.python.framework import constant_op from tensorflow.python.framework import importer from tensorflow.python.framework import ops +from tensorflow.python.framework import test_util from tensorflow.python.ops import math_ops from tensorflow.python.ops import variables from tensorflow.python.platform import test @@ -52,6 +53,7 @@ class PywrapQuantizeTrainingTest(test.TestCase): # Test that save/restoring works for EMA variables generated in the # quantized training rewrite. + @test_util.run_deprecated_v1 def testQuantizedSaveRestore(self): save_path = os.path.join(self.get_temp_dir(), 'quantized_save_restore') @@ -73,11 +75,11 @@ class PywrapQuantizeTrainingTest(test.TestCase): _ = importer.import_graph_def(result, name='') # Initialize the variable. - sess.run(g.get_operation_by_name(init_op.name)) + self.evaluate(g.get_operation_by_name(init_op.name)) # Run the graph for one step to assign values to the quantization min/max # variables. - sess.run(g.get_tensor_by_name(c.name)) + self.evaluate(g.get_tensor_by_name(c.name)) saver.save(sess, save_path) diff --git a/tensorflow/python/training/queue_runner_test.py b/tensorflow/python/training/queue_runner_test.py index 15fe42bbd85..4113cecf55d 100644 --- a/tensorflow/python/training/queue_runner_test.py +++ b/tensorflow/python/training/queue_runner_test.py @@ -26,6 +26,7 @@ from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes from tensorflow.python.framework import errors_impl from tensorflow.python.framework import ops +from tensorflow.python.framework import test_util from tensorflow.python.ops import control_flow_ops from tensorflow.python.ops import data_flow_ops from tensorflow.python.ops import variables @@ -40,6 +41,7 @@ _MockOp = collections.namedtuple("MockOp", ["name"]) class QueueRunnerTest(test.TestCase): + @test_util.run_deprecated_v1 def testBasic(self): with self.cached_session() as sess: # CountUpTo will raise OUT_OF_RANGE when it reaches the count. @@ -58,8 +60,9 @@ class QueueRunnerTest(test.TestCase): t.join() self.assertEqual(0, len(qr.exceptions_raised)) # The variable should be 3. - self.assertEqual(3, var.eval()) + self.assertEqual(3, self.evaluate(var)) + @test_util.run_deprecated_v1 def testTwoOps(self): with self.cached_session() as sess: # CountUpTo will raise OUT_OF_RANGE when it reaches the count. @@ -80,9 +83,10 @@ class QueueRunnerTest(test.TestCase): for t in threads: t.join() self.assertEqual(0, len(qr.exceptions_raised)) - self.assertEqual(3, var0.eval()) - self.assertEqual(30, var1.eval()) + self.assertEqual(3, self.evaluate(var0)) + self.assertEqual(30, self.evaluate(var1)) + @test_util.run_deprecated_v1 def testExceptionsCaptured(self): with self.cached_session() as sess: queue = data_flow_ops.FIFOQueue(10, dtypes.float32) @@ -99,6 +103,7 @@ class QueueRunnerTest(test.TestCase): self.assertTrue("Operation not in the graph" in str(exceptions[0])) self.assertTrue("Operation not in the graph" in str(exceptions[1])) + @test_util.run_deprecated_v1 def testRealDequeueEnqueue(self): with self.cached_session() as sess: q0 = data_flow_ops.FIFOQueue(3, dtypes.float32) @@ -121,12 +126,13 @@ class QueueRunnerTest(test.TestCase): # It should have terminated cleanly. self.assertEqual(0, len(qr.exceptions_raised)) # The 2 values should be in queue1. - self.assertEqual(10.0, dequeue1.eval()) - self.assertEqual(10.0, dequeue1.eval()) + self.assertEqual(10.0, self.evaluate(dequeue1)) + self.assertEqual(10.0, self.evaluate(dequeue1)) # And queue1 should now be closed. with self.assertRaisesRegexp(errors_impl.OutOfRangeError, "is closed"): - dequeue1.eval() + self.evaluate(dequeue1) + @test_util.run_deprecated_v1 def testRespectCoordShouldStop(self): with self.cached_session() as sess: # CountUpTo will raise OUT_OF_RANGE when it reaches the count. @@ -149,8 +155,9 @@ class QueueRunnerTest(test.TestCase): coord.join() self.assertEqual(0, len(qr.exceptions_raised)) # The variable should be 0. - self.assertEqual(0, var.eval()) + self.assertEqual(0, self.evaluate(var)) + @test_util.run_deprecated_v1 def testRequestStopOnException(self): with self.cached_session() as sess: queue = data_flow_ops.FIFOQueue(10, dtypes.float32) @@ -163,6 +170,7 @@ class QueueRunnerTest(test.TestCase): with self.assertRaisesRegexp(ValueError, "Operation not in the graph"): coord.join() + @test_util.run_deprecated_v1 def testGracePeriod(self): with self.cached_session() as sess: # The enqueue will quickly block. @@ -180,6 +188,7 @@ class QueueRunnerTest(test.TestCase): # the queue to be closed and the enqueue to terminate. coord.join(stop_grace_period_secs=1.0) + @test_util.run_deprecated_v1 def testMultipleSessions(self): with self.cached_session() as sess: with session.Session() as other_sess: @@ -195,6 +204,7 @@ class QueueRunnerTest(test.TestCase): other_threads = qr.create_threads(other_sess, coord=coord) self.assertEqual(len(threads), len(other_threads)) + @test_util.run_deprecated_v1 def testIgnoreMultiStarts(self): with self.cached_session() as sess: # CountUpTo will raise OUT_OF_RANGE when it reaches the count. @@ -211,6 +221,7 @@ class QueueRunnerTest(test.TestCase): new_threads = qr.create_threads(sess, coord=coord) self.assertEqual([], new_threads) + @test_util.run_deprecated_v1 def testThreads(self): with self.cached_session() as sess: # CountUpTo will raise OUT_OF_RANGE when it reaches the count. @@ -238,6 +249,7 @@ class QueueRunnerTest(test.TestCase): self.assertEqual(1, len(exceptions)) self.assertTrue("Operation not in the graph" in str(exceptions[0])) + @test_util.run_deprecated_v1 def testName(self): with ops.name_scope("scope"): queue = data_flow_ops.FIFOQueue(10, dtypes.float32, name="queue") @@ -247,6 +259,7 @@ class QueueRunnerTest(test.TestCase): self.assertEqual( 1, len(ops.get_collection(ops.GraphKeys.QUEUE_RUNNERS, "scope"))) + @test_util.run_deprecated_v1 def testStartQueueRunners(self): # CountUpTo will raise OUT_OF_RANGE when it reaches the count. zero64 = constant_op.constant(0, dtype=dtypes.int64) @@ -263,8 +276,9 @@ class QueueRunnerTest(test.TestCase): t.join() self.assertEqual(0, len(qr.exceptions_raised)) # The variable should be 3. - self.assertEqual(3, var.eval()) + self.assertEqual(3, self.evaluate(var)) + @test_util.run_deprecated_v1 def testStartQueueRunnersRaisesIfNotASession(self): zero64 = constant_op.constant(0, dtype=dtypes.int64) var = variables.VariableV1(zero64) @@ -278,6 +292,7 @@ class QueueRunnerTest(test.TestCase): with self.assertRaisesRegexp(TypeError, "tf.Session"): queue_runner_impl.start_queue_runners("NotASession") + @test_util.run_deprecated_v1 def testStartQueueRunnersIgnoresMonitoredSession(self): zero64 = constant_op.constant(0, dtype=dtypes.int64) var = variables.VariableV1(zero64) @@ -292,6 +307,7 @@ class QueueRunnerTest(test.TestCase): monitored_session.MonitoredSession()) self.assertFalse(threads) + @test_util.run_deprecated_v1 def testStartQueueRunnersNonDefaultGraph(self): # CountUpTo will raise OUT_OF_RANGE when it reaches the count. graph = ops.Graph() @@ -310,7 +326,7 @@ class QueueRunnerTest(test.TestCase): t.join() self.assertEqual(0, len(qr.exceptions_raised)) # The variable should be 3. - self.assertEqual(3, var.eval()) + self.assertEqual(3, self.evaluate(var)) def testQueueRunnerSerializationRoundTrip(self): graph = ops.Graph() diff --git a/tensorflow/python/training/rmsprop.py b/tensorflow/python/training/rmsprop.py index f38c9861d64..fb53b5883f5 100644 --- a/tensorflow/python/training/rmsprop.py +++ b/tensorflow/python/training/rmsprop.py @@ -50,7 +50,7 @@ from tensorflow.python.training import training_ops from tensorflow.python.util.tf_export import tf_export -@tf_export("train.RMSPropOptimizer") +@tf_export(v1=["train.RMSPropOptimizer"]) class RMSPropOptimizer(optimizer.Optimizer): """Optimizer that implements the RMSProp algorithm. diff --git a/tensorflow/python/training/rmsprop_test.py b/tensorflow/python/training/rmsprop_test.py index b63abe05295..8f029d5310e 100644 --- a/tensorflow/python/training/rmsprop_test.py +++ b/tensorflow/python/training/rmsprop_test.py @@ -28,6 +28,7 @@ from tensorflow.python.eager import context from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops +from tensorflow.python.framework import test_util from tensorflow.python.ops import embedding_ops from tensorflow.python.ops import math_ops from tensorflow.python.ops import resource_variable_ops @@ -88,11 +89,12 @@ class RMSPropOptimizerTest(test.TestCase): var_t[gindex] = var[gindex] - mom_t[gindex] return var_t, mg_t, rms_t, mom_t + @test_util.run_deprecated_v1 def testDense(self): # TODO(yori): Use ParameterizedTest when available for (dtype, learning_rate, decay, momentum, epsilon, centered, use_resource) in _TESTPARAMS: - with self.cached_session(use_gpu=True): + with test_util.use_gpu(): # Initialize variables for numpy implementation. var0_np = np.array([1.0, 2.0], dtype=dtype.as_numpy_dtype) grads0_np = np.array([0.1, 0.2], dtype=dtype.as_numpy_dtype) @@ -115,7 +117,7 @@ class RMSPropOptimizerTest(test.TestCase): centered=centered) update = opt.apply_gradients(zip([grads0, grads1], [var0, var1])) - variables.global_variables_initializer().run() + self.evaluate(variables.global_variables_initializer()) mg0 = opt.get_slot(var0, "mg") self.assertEqual(mg0 is not None, centered) @@ -138,12 +140,12 @@ class RMSPropOptimizerTest(test.TestCase): mom1_np = np.array([0.0, 0.0], dtype=dtype.as_numpy_dtype) # Fetch params to validate initial values - self.assertAllClose([1.0, 2.0], var0.eval()) - self.assertAllClose([3.0, 4.0], var1.eval()) + self.assertAllClose([1.0, 2.0], self.evaluate(var0)) + self.assertAllClose([3.0, 4.0], self.evaluate(var1)) # Run 4 steps of RMSProp for _ in range(1, 5): - update.run() + self.evaluate(update) var0_np, mg0_np, rms0_np, mom0_np = self._rmsprop_update_numpy( var0_np, grads0_np, mg0_np, rms0_np, mom0_np, learning_rate, @@ -154,15 +156,16 @@ class RMSPropOptimizerTest(test.TestCase): # Validate updated params if centered: - self.assertAllCloseAccordingToType(mg0_np, mg0.eval()) - self.assertAllCloseAccordingToType(mg1_np, mg1.eval()) - self.assertAllCloseAccordingToType(rms0_np, rms0.eval()) - self.assertAllCloseAccordingToType(rms1_np, rms1.eval()) - self.assertAllCloseAccordingToType(mom0_np, mom0.eval()) - self.assertAllCloseAccordingToType(mom1_np, mom1.eval()) - self.assertAllCloseAccordingToType(var0_np, var0.eval()) - self.assertAllCloseAccordingToType(var1_np, var1.eval()) + self.assertAllCloseAccordingToType(mg0_np, self.evaluate(mg0)) + self.assertAllCloseAccordingToType(mg1_np, self.evaluate(mg1)) + self.assertAllCloseAccordingToType(rms0_np, self.evaluate(rms0)) + self.assertAllCloseAccordingToType(rms1_np, self.evaluate(rms1)) + self.assertAllCloseAccordingToType(mom0_np, self.evaluate(mom0)) + self.assertAllCloseAccordingToType(mom1_np, self.evaluate(mom1)) + self.assertAllCloseAccordingToType(var0_np, self.evaluate(var0)) + self.assertAllCloseAccordingToType(var1_np, self.evaluate(var1)) + @test_util.run_deprecated_v1 def testMinimizeSparseResourceVariable(self): for dtype in [dtypes.float32, dtypes.float64]: with self.cached_session(): @@ -176,15 +179,17 @@ class RMSPropOptimizerTest(test.TestCase): momentum=0.0, epsilon=0.0, centered=False).minimize(loss) - variables.global_variables_initializer().run() + self.evaluate(variables.global_variables_initializer()) # Fetch params to validate initial values - self.assertAllCloseAccordingToType([[1.0, 2.0]], var0.eval()) + self.assertAllCloseAccordingToType([[1.0, 2.0]], self.evaluate(var0)) # Run 1 step of sgd - sgd_op.run() + self.evaluate(sgd_op) # Validate updated params - self.assertAllCloseAccordingToType( - [[0., 1.]], var0.eval(), atol=0.01) + self.assertAllCloseAccordingToType([[0., 1.]], + self.evaluate(var0), + atol=0.01) + @test_util.run_deprecated_v1 def testMinimizeSparseResourceVariableCentered(self): for dtype in [dtypes.float32, dtypes.float64]: with self.cached_session(): @@ -198,20 +203,22 @@ class RMSPropOptimizerTest(test.TestCase): momentum=0.0, epsilon=1.0, centered=True).minimize(loss) - variables.global_variables_initializer().run() + self.evaluate(variables.global_variables_initializer()) # Fetch params to validate initial values - self.assertAllCloseAccordingToType([[1.0, 2.0]], var0.eval()) + self.assertAllCloseAccordingToType([[1.0, 2.0]], self.evaluate(var0)) # Run 1 step of sgd - sgd_op.run() + self.evaluate(sgd_op) # Validate updated params - self.assertAllCloseAccordingToType( - [[-111, -138]], var0.eval(), atol=0.01) + self.assertAllCloseAccordingToType([[-111, -138]], + self.evaluate(var0), + atol=0.01) + @test_util.run_deprecated_v1 def testSparse(self): # TODO(yori): Use ParameterizedTest when available for (dtype, learning_rate, decay, momentum, epsilon, centered, _) in _TESTPARAMS: - with self.cached_session(use_gpu=True): + with test_util.use_gpu(): # Initialize variables for numpy implementation. var0_np = np.array([1.0, 2.0], dtype=dtype.as_numpy_dtype) grads0_np = np.array([0.1], dtype=dtype.as_numpy_dtype) @@ -235,7 +242,7 @@ class RMSPropOptimizerTest(test.TestCase): epsilon=epsilon, centered=centered) update = opt.apply_gradients(zip([grads0, grads1], [var0, var1])) - variables.global_variables_initializer().run() + self.evaluate(variables.global_variables_initializer()) mg0 = opt.get_slot(var0, "mg") self.assertEqual(mg0 is not None, centered) @@ -258,12 +265,12 @@ class RMSPropOptimizerTest(test.TestCase): mom1_np = np.array([0.0, 0.0], dtype=dtype.as_numpy_dtype) # Fetch params to validate initial values - self.assertAllClose([1.0, 2.0], var0.eval()) - self.assertAllClose([3.0, 4.0], var1.eval()) + self.assertAllClose([1.0, 2.0], self.evaluate(var0)) + self.assertAllClose([3.0, 4.0], self.evaluate(var1)) # Run 4 steps of RMSProp for _ in range(1, 5): - update.run() + self.evaluate(update) var0_np, mg0_np, rms0_np, mom0_np = self._sparse_rmsprop_update_numpy( var0_np, grads0_np_indices, grads0_np, mg0_np, rms0_np, mom0_np, @@ -274,18 +281,19 @@ class RMSPropOptimizerTest(test.TestCase): # Validate updated params if centered: - self.assertAllCloseAccordingToType(mg0_np, mg0.eval()) - self.assertAllCloseAccordingToType(mg1_np, mg1.eval()) - self.assertAllCloseAccordingToType(rms0_np, rms0.eval()) - self.assertAllCloseAccordingToType(rms1_np, rms1.eval()) - self.assertAllCloseAccordingToType(mom0_np, mom0.eval()) - self.assertAllCloseAccordingToType(mom1_np, mom1.eval()) - self.assertAllCloseAccordingToType(var0_np, var0.eval()) - self.assertAllCloseAccordingToType(var1_np, var1.eval()) + self.assertAllCloseAccordingToType(mg0_np, self.evaluate(mg0)) + self.assertAllCloseAccordingToType(mg1_np, self.evaluate(mg1)) + self.assertAllCloseAccordingToType(rms0_np, self.evaluate(rms0)) + self.assertAllCloseAccordingToType(rms1_np, self.evaluate(rms1)) + self.assertAllCloseAccordingToType(mom0_np, self.evaluate(mom0)) + self.assertAllCloseAccordingToType(mom1_np, self.evaluate(mom1)) + self.assertAllCloseAccordingToType(var0_np, self.evaluate(var0)) + self.assertAllCloseAccordingToType(var1_np, self.evaluate(var1)) + @test_util.run_deprecated_v1 def testWithoutMomentum(self): for dtype in [dtypes.half, dtypes.float32]: - with self.cached_session(use_gpu=True): + with test_util.use_gpu(): var0 = variables.Variable([1.0, 2.0], dtype=dtype) var1 = variables.Variable([3.0, 4.0], dtype=dtype) grads0 = constant_op.constant([0.1, 0.1], dtype=dtype) @@ -293,7 +301,7 @@ class RMSPropOptimizerTest(test.TestCase): opt = rmsprop.RMSPropOptimizer( learning_rate=2.0, decay=0.9, momentum=0.0, epsilon=1.0) update = opt.apply_gradients(zip([grads0, grads1], [var0, var1])) - variables.global_variables_initializer().run() + self.evaluate(variables.global_variables_initializer()) rms0 = opt.get_slot(var0, "rms") self.assertTrue(rms0 is not None) @@ -305,34 +313,36 @@ class RMSPropOptimizerTest(test.TestCase): self.assertTrue(mom1 is not None) # Fetch params to validate initial values - self.assertAllClose([1.0, 2.0], var0.eval()) - self.assertAllClose([3.0, 4.0], var1.eval()) + self.assertAllClose([1.0, 2.0], self.evaluate(var0)) + self.assertAllClose([3.0, 4.0], self.evaluate(var1)) # Step 1: the rms accumulators where 1. So we should see a normal # update: v -= grad * learning_rate - update.run() + self.evaluate(update) # Check the root mean square accumulators. self.assertAllCloseAccordingToType( - np.array([0.901, 0.901]), rms0.eval()) + np.array([0.901, 0.901]), self.evaluate(rms0)) self.assertAllCloseAccordingToType( - np.array([0.90001, 0.90001]), rms1.eval()) + np.array([0.90001, 0.90001]), self.evaluate(rms1)) # Check the parameters. self.assertAllCloseAccordingToType( np.array([ 1.0 - (0.1 * 2.0 / math.sqrt(0.901 + 1.0)), 2.0 - (0.1 * 2.0 / math.sqrt(0.901 + 1.0)) - ]), var0.eval()) + ]), self.evaluate(var0)) self.assertAllCloseAccordingToType( np.array([ 3.0 - (0.01 * 2.0 / math.sqrt(0.90001 + 1.0)), 4.0 - (0.01 * 2.0 / math.sqrt(0.90001 + 1.0)) - ]), var1.eval()) + ]), self.evaluate(var1)) # Step 2: the root mean square accumulators contain the previous update. - update.run() + self.evaluate(update) # Check the rms accumulators. self.assertAllCloseAccordingToType( - np.array([0.901 * 0.9 + 0.001, 0.901 * 0.9 + 0.001]), rms0.eval()) + np.array([0.901 * 0.9 + 0.001, 0.901 * 0.9 + 0.001]), + self.evaluate(rms0)) self.assertAllCloseAccordingToType( - np.array([0.90001 * 0.9 + 1e-5, 0.90001 * 0.9 + 1e-5]), rms1.eval()) + np.array([0.90001 * 0.9 + 1e-5, 0.90001 * 0.9 + 1e-5]), + self.evaluate(rms1)) # Check the parameters. self.assertAllCloseAccordingToType( np.array([ @@ -340,18 +350,19 @@ class RMSPropOptimizerTest(test.TestCase): (0.1 * 2.0 / math.sqrt(0.901 * 0.9 + 0.001 + 1.0)), 2.0 - (0.1 * 2.0 / math.sqrt(0.901 + 1.0)) - (0.1 * 2.0 / math.sqrt(0.901 * 0.9 + 0.001 + 1.0)) - ]), var0.eval()) + ]), self.evaluate(var0)) self.assertAllCloseAccordingToType( np.array([ 3.0 - (0.01 * 2.0 / math.sqrt(0.90001 + 1.0)) - (0.01 * 2.0 / math.sqrt(0.90001 * 0.9 + 1e-5 + 1.0)), 4.0 - (0.01 * 2.0 / math.sqrt(0.90001 + 1.0)) - (0.01 * 2.0 / math.sqrt(0.90001 * 0.9 + 1e-5 + 1.0)) - ]), var1.eval()) + ]), self.evaluate(var1)) + @test_util.run_deprecated_v1 def testWithMomentum(self): for dtype in [dtypes.half, dtypes.float32]: - with self.cached_session(use_gpu=True): + with test_util.use_gpu(): var0 = variables.Variable([1.0, 2.0], dtype=dtype) var1 = variables.Variable([3.0, 4.0], dtype=dtype) grads0 = constant_op.constant([0.1, 0.1], dtype=dtype) @@ -360,7 +371,7 @@ class RMSPropOptimizerTest(test.TestCase): opt = rmsprop.RMSPropOptimizer( learning_rate=2.0, decay=0.9, momentum=0.5, epsilon=1e-5) update = opt.apply_gradients(zip([grads0, grads1], [var0, var1])) - variables.global_variables_initializer().run() + self.evaluate(variables.global_variables_initializer()) rms0 = opt.get_slot(var0, "rms") self.assertTrue(rms0 is not None) @@ -372,57 +383,61 @@ class RMSPropOptimizerTest(test.TestCase): self.assertTrue(mom1 is not None) # Fetch params to validate initial values - self.assertAllClose([1.0, 2.0], var0.eval()) - self.assertAllClose([3.0, 4.0], var1.eval()) + self.assertAllClose([1.0, 2.0], self.evaluate(var0)) + self.assertAllClose([3.0, 4.0], self.evaluate(var1)) # Step 1: rms = 1, mom = 0. So we should see a normal # update: v -= grad * learning_rate - update.run() + self.evaluate(update) # Check the root mean square accumulators. self.assertAllCloseAccordingToType( - np.array([0.901, 0.901]), rms0.eval()) + np.array([0.901, 0.901]), self.evaluate(rms0)) self.assertAllCloseAccordingToType( - np.array([0.90001, 0.90001]), rms1.eval()) + np.array([0.90001, 0.90001]), self.evaluate(rms1)) # Check the momentum accumulators self.assertAllCloseAccordingToType( np.array([(0.1 * 2.0 / math.sqrt(0.901 + 1e-5)), - (0.1 * 2.0 / math.sqrt(0.901 + 1e-5))]), mom0.eval()) + (0.1 * 2.0 / math.sqrt(0.901 + 1e-5))]), + self.evaluate(mom0)) self.assertAllCloseAccordingToType( np.array([(0.01 * 2.0 / math.sqrt(0.90001 + 1e-5)), - (0.01 * 2.0 / math.sqrt(0.90001 + 1e-5))]), mom1.eval()) + (0.01 * 2.0 / math.sqrt(0.90001 + 1e-5))]), + self.evaluate(mom1)) # Check that the parameters. self.assertAllCloseAccordingToType( np.array([ 1.0 - (0.1 * 2.0 / math.sqrt(0.901 + 1e-5)), 2.0 - (0.1 * 2.0 / math.sqrt(0.901 + 1e-5)) - ]), var0.eval()) + ]), self.evaluate(var0)) self.assertAllCloseAccordingToType( np.array([ 3.0 - (0.01 * 2.0 / math.sqrt(0.90001 + 1e-5)), 4.0 - (0.01 * 2.0 / math.sqrt(0.90001 + 1e-5)) - ]), var1.eval()) + ]), self.evaluate(var1)) # Step 2: the root mean square accumulators contain the previous update. - update.run() + self.evaluate(update) # Check the rms accumulators. self.assertAllCloseAccordingToType( - np.array([0.901 * 0.9 + 0.001, 0.901 * 0.9 + 0.001]), rms0.eval()) + np.array([0.901 * 0.9 + 0.001, 0.901 * 0.9 + 0.001]), + self.evaluate(rms0)) self.assertAllCloseAccordingToType( - np.array([0.90001 * 0.9 + 1e-5, 0.90001 * 0.9 + 1e-5]), rms1.eval()) + np.array([0.90001 * 0.9 + 1e-5, 0.90001 * 0.9 + 1e-5]), + self.evaluate(rms1)) self.assertAllCloseAccordingToType( np.array([ 0.5 * (0.1 * 2.0 / math.sqrt(0.901 + 1e-5)) + (0.1 * 2.0 / math.sqrt(0.901 * 0.9 + 0.001 + 1e-5)), 0.5 * (0.1 * 2.0 / math.sqrt(0.901 + 1e-5)) + (0.1 * 2.0 / math.sqrt(0.901 * 0.9 + 0.001 + 1e-5)) - ]), mom0.eval()) + ]), self.evaluate(mom0)) self.assertAllCloseAccordingToType( np.array([ 0.5 * (0.01 * 2.0 / math.sqrt(0.90001 + 1e-5)) + (0.01 * 2.0 / math.sqrt(0.90001 * 0.9 + 2e-5)), 0.5 * (0.01 * 2.0 / math.sqrt(0.90001 + 1e-5)) + (0.01 * 2.0 / math.sqrt(0.90001 * 0.9 + 2e-5)) - ]), mom1.eval()) + ]), self.evaluate(mom1)) # Check the parameters. self.assertAllCloseAccordingToType( @@ -433,7 +448,7 @@ class RMSPropOptimizerTest(test.TestCase): 2.0 - (0.1 * 2.0 / math.sqrt(0.901 + 1e-5)) - (0.5 * (0.1 * 2.0 / math.sqrt(0.901 + 1e-5)) + (0.1 * 2.0 / math.sqrt(0.901 * 0.9 + 0.001 + 1e-5))) - ]), var0.eval()) + ]), self.evaluate(var0)) self.assertAllCloseAccordingToType( np.array([ @@ -443,7 +458,7 @@ class RMSPropOptimizerTest(test.TestCase): 4.0 - (0.01 * 2.0 / math.sqrt(0.90001 + 1e-5)) - (0.5 * (0.01 * 2.0 / math.sqrt(0.90001 + 1e-5)) + (0.01 * 2.0 / math.sqrt(0.90001 * 0.9 + 2e-5))) - ]), var1.eval()) + ]), self.evaluate(var1)) def testCallableParams(self): with context.eager_mode(): diff --git a/tensorflow/python/training/saver.py b/tensorflow/python/training/saver.py index a29926a57df..4cd09f8a1d5 100644 --- a/tensorflow/python/training/saver.py +++ b/tensorflow/python/training/saver.py @@ -1077,16 +1077,28 @@ class Saver(object): @compatibility(eager) When eager execution is enabled, `var_list` must specify a `list` or `dict` of variables to save. Otherwise, a `RuntimeError` will be raised. + + Although Saver works in some cases when executing eagerly, it is + fragile. Please switch to `tf.train.Checkpoint` or + `tf.keras.Model.save_weights`, which perform a more robust object-based + saving. These APIs will load checkpoints written by `Saver`. @end_compatibility """ if defer_build and var_list: raise ValueError( "If `var_list` is provided then build cannot be deferred. " "Either set defer_build=False or var_list=None.") - if context.executing_eagerly() and var_list is None: - raise RuntimeError( - "When eager execution is enabled, `var_list` must specify a list or " - "dict of variables to save") + if context.executing_eagerly(): + logging.warning( + "Saver is deprecated, please switch to tf.train.Checkpoint or " + "tf.keras.Model.save_weights for training checkpoints. When " + "executing eagerly variables do not necessarily have unique names, " + "and so the variable.name-based lookups Saver performs are " + "error-prone.") + if var_list is None: + raise RuntimeError( + "When eager execution is enabled, `var_list` must specify a list " + "or dict of variables to save") self._var_list = var_list self._reshape = reshape self._sharded = sharded @@ -1899,16 +1911,40 @@ def saver_from_object_based_checkpoint( builder = BulkSaverBuilder() saveables = builder._ValidateAndSliceInputs(var_list) # pylint: disable=protected-access + current_names = set() + for saveable in saveables: + for spec in saveable.specs: + current_names.add(spec.name) + previous_names = set(names_to_keys.keys()) + missing_names = current_names - previous_names + if missing_names: + extra_names = previous_names - current_names + intersecting_names = previous_names.intersection(current_names) + raise errors.NotFoundError( + None, None, + message=( + "\n\nExisting variables not in the checkpoint: %s\n\n" + "Variables names when this checkpoint was written which don't " + "exist now: %s\n\n" + "(%d variable name(s) did match)\n\n" + "Could not find some variables in the checkpoint (see names " + "above). Saver was attempting to load an object-based checkpoint " + "(saved using tf.train.Checkpoint or tf.keras.Model.save_weights) " + "using variable names. If the checkpoint was written with eager " + "execution enabled, it's possible that variable names have " + "changed (for example missing a '_1' suffix). It's also " + "possible that there are new variables which did not exist " + "when the checkpoint was written. You can construct a " + "Saver(var_list=...) with only the variables which previously " + "existed, and if variable names have changed you may need to " + "make this a dictionary with the old names as keys. If you're " + "using an Estimator, you'll need to return a tf.train.Saver " + "inside a tf.train.Scaffold from your model_fn.") + % (", ".join(sorted(missing_names)), ", ".join(sorted(extra_names)), + len(intersecting_names))) for saveable in saveables: for spec in saveable.specs: - if spec.name not in names_to_keys: - raise errors.NotFoundError( - None, None, - message=("Attempting to load an object-based checkpoint using " - "variable names, but could not find %s in the " - "checkpoint.") % spec.name) spec.name = names_to_keys[spec.name] - if cached_saver is None: return Saver(saveables) return cached_saver diff --git a/tensorflow/python/training/saver_large_partitioned_variable_test.py b/tensorflow/python/training/saver_large_partitioned_variable_test.py index 1a44511cfeb..84458836d06 100644 --- a/tensorflow/python/training/saver_large_partitioned_variable_test.py +++ b/tensorflow/python/training/saver_large_partitioned_variable_test.py @@ -25,6 +25,7 @@ from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops from tensorflow.python.ops import partitioned_variables +from tensorflow.python.ops import variable_scope from tensorflow.python.ops import variables from tensorflow.python.platform import test from tensorflow.python.training import saver @@ -44,8 +45,12 @@ class SaverLargePartitionedVariableTest(test.TestCase): # split into smaller sized variables. init = lambda shape, dtype, partition_info: constant_op.constant( True, dtype, shape) - partitioned_var = partitioned_variables.create_partitioned_variables( - [1 << 31], [4], init, dtype=dtypes.bool, name=var_name) + partitioned_var = list(variable_scope.get_variable( + var_name, + shape=[1 << 31], + partitioner=partitioned_variables.fixed_size_partitioner(4), + initializer=init, + dtype=dtypes.bool)) variables.global_variables_initializer().run() save = saver.Saver(partitioned_var) val = save.save(sess, save_path) diff --git a/tensorflow/python/training/saver_test.py b/tensorflow/python/training/saver_test.py index eb2690985d5..5d621ba4ffa 100644 --- a/tensorflow/python/training/saver_test.py +++ b/tensorflow/python/training/saver_test.py @@ -170,6 +170,7 @@ class SaverTest(test.TestCase): def testResourceBasic(self): self.basicSaveRestore(resource_variable_ops.ResourceVariable) + @test_util.run_deprecated_v1 def testResourceColocation(self): partitioner = partitioned_variables.fixed_size_partitioner(num_shards=2) with ops_lib.device("/job:ps/device:GPU:0"): @@ -227,7 +228,7 @@ class SaverTest(test.TestCase): w1 = resource_variable_ops.ResourceVariable(1.0, name="w1") w2 = resource_variable_ops.ResourceVariable(2.0, name="w2") graph_saver = saver_module.Saver([w1, w2]) - sess.run(variables.global_variables_initializer()) + self.evaluate(variables.global_variables_initializer()) graph_saver.save(sess, graph_ckpt_prefix) with context.eager_mode(): @@ -260,7 +261,7 @@ class SaverTest(test.TestCase): w3 = resource_variable_ops.ResourceVariable(0.0, name="w3") w4 = resource_variable_ops.ResourceVariable(0.0, name="w4") graph_saver = saver_module.Saver([w3, w4]) - sess.run(variables.global_variables_initializer()) + self.evaluate(variables.global_variables_initializer()) graph_saver.restore(sess, eager_ckpt_prefix) self.assertAllEqual(w3.eval(), 3.0) self.assertAllEqual(w4.eval(), 4.0) @@ -300,6 +301,7 @@ class SaverTest(test.TestCase): not op.name.startswith("saver2/save/"))] self.assertEqual(ops_in_saver2_scope_but_not_save_scope, []) + @test_util.run_deprecated_v1 def testSaveCopyRestoreWithSaveRelativePaths(self): """Save, copy checkpoint dir and restore from copied dir. @@ -326,7 +328,7 @@ class SaverTest(test.TestCase): with self.cached_session() as sess: # Initialize all variables - sess.run(init_all_op) + self.evaluate(init_all_op) # Check that the parameter nodes have been initialized. self.assertEqual(10.0, v0.eval()) @@ -369,6 +371,7 @@ class SaverTest(test.TestCase): self.assertEqual(b"k1", v2.keys().eval()) self.assertEqual(30.0, v2.values().eval()) + @test_util.run_deprecated_v1 def testFilenameTensor(self): v0 = variables.VariableV1(0, name="v0") filename = b"somerandomfilename" @@ -376,7 +379,7 @@ class SaverTest(test.TestCase): with self.cached_session() as sess: tensor = sess.graph.get_tensor_by_name( save.saver_def.filename_tensor_name) - self.assertEqual(sess.run(tensor), filename) + self.assertEqual(self.evaluate(tensor), filename) def testInvalidPath(self): v0 = variables.VariableV1(0, name="v0") @@ -387,6 +390,7 @@ class SaverTest(test.TestCase): ValueError, "The passed save_path is not a valid checkpoint:"): save.restore(sess, "invalid path") + @test_util.run_deprecated_v1 def testInt64(self): save_path = os.path.join(self.get_temp_dir(), "int64") @@ -407,7 +411,7 @@ class SaverTest(test.TestCase): with self.assertRaisesWithPredicateMatch( errors_impl.OpError, lambda e: "uninitialized value v" in e.message): - sess.run(v) + self.evaluate(v) # Restore the saved values in the parameter nodes. save.restore(sess, save_path) @@ -462,6 +466,7 @@ class SaverTest(test.TestCase): # Verify non-duplicate names work. saver_module.Saver({"v0": v0, "v2": v2.saveable}) + @test_util.run_deprecated_v1 def testBasicsWithListOfVariables(self): save_path = os.path.join(self.get_temp_dir(), "basics_with_list") @@ -497,10 +502,10 @@ class SaverTest(test.TestCase): with self.assertRaisesWithPredicateMatch( errors_impl.OpError, lambda e: "uninitialized value v0" in e.message): - sess.run(v0) + self.evaluate(v0) with self.assertRaisesWithPredicateMatch( errors_impl.OpError, lambda e: "uninitialized value v1" in e.message): - sess.run(v1) + self.evaluate(v1) self.assertEqual(0, len(v2.keys().eval())) self.assertEqual(0, len(v2.values().eval())) @@ -557,6 +562,7 @@ class SaverTest(test.TestCase): # The cached readers should know to re-read the file. self._SaveAndLoad("var1", 1.1, 2.2, save_path) + @test_util.run_deprecated_v1 def testAllowEmpty(self): save_path = os.path.join(self.get_temp_dir(), "allow_empty") with self.cached_session() as sess: @@ -661,6 +667,7 @@ class SaverTest(test.TestCase): self.assertAllClose(1.0, one.eval()) self.assertAllClose([2.0, 2.0, 2.0], twos.eval()) + @test_util.run_deprecated_v1 def testReshape(self): save_path = os.path.join(self.get_temp_dir(), "variables_reshape") with session.Session("", graph=ops_lib.Graph()) as sess: @@ -719,6 +726,7 @@ class SaverTest(test.TestCase): def testSaveWithGlobalStepWithPadding(self): self.testSaveWithGlobalStep(pad_step_number=True) + @test_util.run_deprecated_v1 def testSaveToNonexistingPath(self): file_io.write_string_to_file( os.path.join(self.get_temp_dir(), "actually_a_file"), "") @@ -742,7 +750,7 @@ class SaverTest(test.TestCase): try: with self.cached_session() as sess: # Initialize all variables - sess.run(init_all_op) + self.evaluate(init_all_op) # Check that the parameter nodes have been initialized. self.assertEqual(10.0, v0.eval()) @@ -761,6 +769,7 @@ class SaverTest(test.TestCase): error_msg_template = "Parent directory of {} doesn't exist, can't save." self.assertEqual(error_msg_template.format(save_path), str(exc)) + @test_util.run_deprecated_v1 def testSaveToURI(self): # ParseURI functions don't work on Windows yet. # TODO(jhseu): Remove this check when it works. @@ -777,7 +786,7 @@ class SaverTest(test.TestCase): with self.cached_session() as sess: # Initialize all variables - sess.run(init_all_op) + self.evaluate(init_all_op) # Check that the parameter nodes have been initialized. self.assertEqual(10.0, v0.eval()) @@ -824,11 +833,11 @@ class SaverTest(test.TestCase): save_graph = ops_lib.Graph() with save_graph.as_default(), self.session(graph=save_graph) as sess: orig_vars = _model() - sess.run(variables.global_variables_initializer()) + self.evaluate(variables.global_variables_initializer()) save = saver_module.Saver(max_to_keep=1) variables.global_variables_initializer().run() save.save(sess, save_dir) - orig_vals = sess.run(orig_vars) + orig_vals = self.evaluate(orig_vars) restore_graph = ops_lib.Graph() with restore_graph.as_default(), self.session( @@ -836,7 +845,7 @@ class SaverTest(test.TestCase): restored_vars = _model() save = saver_module.Saver(max_to_keep=1) save.restore(sess, save_dir) - restored_vals = sess.run(restored_vars) + restored_vals = self.evaluate(restored_vars) for orig, restored in zip(orig_vals, restored_vals): self.assertAllEqual(orig, restored) @@ -982,6 +991,7 @@ class SaveRestoreShardedTest(test.TestCase): checkpoint_management.latest_checkpoint(self.get_temp_dir()), os.path.join(self.get_temp_dir(), "sharded_basics")) + @test_util.run_deprecated_v1 def testSaverDef(self): with self.cached_session(): v0 = variables.VariableV1(123, name="v0") @@ -998,19 +1008,12 @@ class SaveRestoreShardedTest(test.TestCase): call_saver_with_dict = False # updated by test loop below - def _save(slices=None, partitioner=None): + def _save(partitioner=None): with self.session(graph=ops_lib.Graph()) as sess: # Calls .eval() to return the ndarray that makes up the full variable. rnd = random_ops.random_uniform(var_full_shape).eval() - if slices: - assert not partitioner - # TODO(apassos): make create_partitioned_variables take use_resource - # option to make this test passable without creating a named - # variable_scope. - vs = partitioned_variables.create_partitioned_variables( - var_full_shape, slices, rnd, name=var_name) - elif partitioner: + if partitioner: vs = [ variable_scope.get_variable( var_name, @@ -1027,7 +1030,7 @@ class SaveRestoreShardedTest(test.TestCase): variables.global_variables_initializer().run() if call_saver_with_dict: - saver = saver_module.Saver({var_name: (vs if slices else vs[0])}) + saver = saver_module.Saver({var_name: vs[0]}) else: saver = saver_module.Saver(vs) actual_path = saver.save(sess, saved_path) @@ -1035,16 +1038,9 @@ class SaveRestoreShardedTest(test.TestCase): return rnd - def _restore(slices=None, partitioner=None): + def _restore(partitioner=None): with self.session(graph=ops_lib.Graph()) as sess: - if slices: - assert not partitioner - new_vs = partitioned_variables.create_partitioned_variables( - var_full_shape, - slices, - array_ops.zeros(var_full_shape), # != original contents. - name=var_name) - elif partitioner: + if partitioner: new_vs = [ variable_scope.get_variable( var_name, @@ -1063,7 +1059,7 @@ class SaveRestoreShardedTest(test.TestCase): variables.global_variables_initializer().run() if call_saver_with_dict: saver = saver_module.Saver({ - var_name: (new_vs if slices else new_vs[0]) + var_name: new_vs[0] }) else: saver = saver_module.Saver(new_vs) @@ -1071,11 +1067,7 @@ class SaveRestoreShardedTest(test.TestCase): if partitioner: return new_vs[0].as_tensor().eval() - elif slices and slices[0] != 1: - return array_ops.concat(new_vs, 0).eval() - elif slices and slices[1] != 1: - return array_ops.concat(new_vs, 1).eval() - else: # Non-sliced. + else: return new_vs[0].eval() for call_saver_with_dict in {False, True}: @@ -1086,32 +1078,30 @@ class SaveRestoreShardedTest(test.TestCase): restored_full = _restore() self.assertAllEqual(saved_full, restored_full) - # Saves 10 horizontal parts of a partitioned variable. - # Restores into a full variable, non-sliced. - saved_full = _save(slices=[10, 1]) - restored_full = _restore() - self.assertAllEqual(saved_full, restored_full) - - # Restores into a different number/orientation of slices. - restored_full = _restore(slices=[2, 1]) # 2 horizon parts. - self.assertAllEqual(saved_full, restored_full) - restored_full = _restore(slices=[1, 3]) # 3 vertical parts. - self.assertAllEqual(saved_full, restored_full) - - # Restores into a PartitionedVariable + # Restores into the same number of partitions. restored_full = _restore( partitioner=partitioned_variables.fixed_size_partitioner( num_shards=2)) self.assertAllEqual(saved_full, restored_full) - # Now, saves a full variable and restores in slices. - saved_full = _save() - restored_full = _restore(slices=[1, 3]) + # Restores into a different number of partitions. + restored_full = _restore( + partitioner=partitioned_variables.fixed_size_partitioner( + num_shards=3)) self.assertAllEqual(saved_full, restored_full) + # Now, saves a full variable and restores PartitionedVariable. + saved_full = _save() + restored_full = _restore( + partitioner=partitioned_variables.fixed_size_partitioner( + num_shards=3)) + self.assertAllEqual(saved_full, restored_full) + + @test_util.run_deprecated_v1 def testPartitionedVariable(self): self._testPartitionedVariables(use_resource=False) + @test_util.run_deprecated_v1 def testPartitionedResourceVariable(self): self._testPartitionedVariables(use_resource=True) @@ -1206,6 +1196,7 @@ class MaxToKeepTest(test.TestCase): # Deleted by the first helper. self.assertFalse(checkpoint_management.checkpoint_exists(s3)) + @test_util.run_deprecated_v1 def testNonSharded(self): save_dir = self._get_test_dir("max_to_keep_non_sharded") @@ -1443,6 +1434,7 @@ class MaxToKeepTest(test.TestCase): self.assertTrue( gfile.Exists(checkpoint_management.meta_graph_filename(s3))) + @test_util.run_deprecated_v1 def testNoMaxToKeep(self): save_dir = self._get_test_dir("no_max_to_keep") save_dir2 = self._get_test_dir("max_to_keep_0") @@ -1471,6 +1463,7 @@ class MaxToKeepTest(test.TestCase): self.assertEqual([], save2.last_checkpoints) self.assertTrue(checkpoint_management.checkpoint_exists(s2)) + @test_util.run_deprecated_v1 def testNoMetaGraph(self): save_dir = self._get_test_dir("no_meta_graph") @@ -1494,6 +1487,7 @@ class KeepCheckpointEveryNHoursTest(test.TestCase): @test_util.run_in_graph_and_eager_modes @test.mock.patch.object(saver_module, "time") + @test_util.run_deprecated_v1 def testNonSharded(self, mock_time): save_dir = self._get_test_dir("keep_checkpoint_every_n_hours") @@ -1613,6 +1607,7 @@ class SaveRestoreWithVariableNameMap(test.TestCase): self.assertEqual(20.0, self.evaluate(v1)) @test_util.run_in_graph_and_eager_modes + @test_util.run_deprecated_v1 def testNonReshapeResourceVariable(self): self._testNonReshape(resource_variable_ops.ResourceVariable) @@ -1627,6 +1622,7 @@ class MetaGraphTest(test.TestCase): gfile.MakeDirs(test_dir) return test_dir + @test_util.run_deprecated_v1 def testAddCollectionDef(self): test_dir = self._get_test_dir("good_collection") filename = os.path.join(test_dir, "metafile") @@ -1769,18 +1765,20 @@ class MetaGraphTest(test.TestCase): self.assertEqual([], v1.get_shape()) with self.assertRaisesWithPredicateMatch( errors_impl.OpError, lambda e: "uninitialized value v1" in e.message): - sess.run(v1) + self.evaluate(v1) # Retrieves saver1. Verifies that new_saver1 can restore v1. new_saver1 = savers[1] new_saver1.restore(sess, saver1_ckpt) v1 = sess.graph.get_tensor_by_name("v1:0") self.assertEqual(11.0, v1.eval()) + @test_util.run_deprecated_v1 def testMultiSaverCollection(self): test_dir = self._get_test_dir("saver_collection") self._testMultiSaverCollectionSave(test_dir) self._testMultiSaverCollectionRestore(test_dir) + @test_util.run_deprecated_v1 def testClearExtraneousSavers(self): test_dir = self._get_test_dir("clear_extraneous_savers") filename = os.path.join(test_dir, "metafile") @@ -1835,6 +1833,7 @@ class MetaGraphTest(test.TestCase): self.assertEqual(33, len(meta_graph_def0.graph_def.node)) self.assertEqual(21, len(meta_graph_def1.graph_def.node)) + @test_util.run_deprecated_v1 def testBinaryAndTextFormat(self): test_dir = self._get_test_dir("binary_and_text") filename = os.path.join(test_dir, "metafile") @@ -1867,6 +1866,7 @@ class MetaGraphTest(test.TestCase): lambda e: "does not exist"): saver_module.import_meta_graph(filename) + @test_util.run_deprecated_v1 def testSliceVariable(self): test_dir = self._get_test_dir("slice_saver") filename = os.path.join(test_dir, "metafile") @@ -1949,9 +1949,9 @@ class MetaGraphTest(test.TestCase): with self.cached_session() as sess: # Initializes all the variables. - sess.run(init_all_op) + self.evaluate(init_all_op) # Runs to logit. - sess.run(logits) + self.evaluate(logits) # Creates a saver. saver0 = saver_module.Saver() saver0.save(sess, saver0_ckpt) @@ -1991,7 +1991,7 @@ class MetaGraphTest(test.TestCase): ops_lib.add_to_collection("train_op", train_op) # Runs train_op. - sess.run(train_op) + self.evaluate(train_op) # Generates MetaGraphDef. saver_module.export_meta_graph(train_filename) @@ -2005,8 +2005,9 @@ class MetaGraphTest(test.TestCase): # Restores from checkpoint. new_saver.restore(sess, saver0_ckpt) train_op = ops_lib.get_collection("train_op")[0] - sess.run(train_op) + self.evaluate(train_op) + @test_util.run_deprecated_v1 def testGraphExtension(self): test_dir = self._get_test_dir("graph_extension") self._testGraphExtensionSave(test_dir) @@ -2037,8 +2038,8 @@ class MetaGraphTest(test.TestCase): # Generate a MetaGraphDef containing the while loop. with session.Session() as sess: - sess.run(init_op) - sess.run(output) + self.evaluate(init_op) + self.evaluate(output) saver = saver_module.Saver() saver.save(sess, saver_ckpt) saver.export_meta_graph(filename) @@ -2053,8 +2054,8 @@ class MetaGraphTest(test.TestCase): no_constfold_config.graph_options.rewrite_options.constant_folding = ( rewriter_config_pb2.RewriterConfig.OFF) with session.Session(config=no_constfold_config) as sess: - sess.run(init_op) - expected_grad_value = sess.run(grad) + self.evaluate(init_op) + expected_grad_value = self.evaluate(grad) # Restore the MetaGraphDef into a new Graph. with ops_lib.Graph().as_default(): @@ -2070,8 +2071,8 @@ class MetaGraphTest(test.TestCase): init_op = variables.global_variables_initializer() with session.Session(config=no_constfold_config) as sess: - sess.run(init_op) - actual_grad_value = sess.run(grad) + self.evaluate(init_op) + actual_grad_value = self.evaluate(grad) self.assertEqual(expected_grad_value, actual_grad_value) def _testWhileLoopAndGradientSerDes(self, outer_body_fn): @@ -2092,6 +2093,7 @@ class MetaGraphTest(test.TestCase): return i + 1, x + r self._testWhileLoopAndGradientSerDes(body) + @test_util.run_deprecated_v1 def testNestedControlFlowSerDes(self): # Test while loop in a cond in a while loop. # pylint: disable=g-long-lambda @@ -2120,6 +2122,7 @@ class MetaGraphTest(test.TestCase): lambda: math_ops.multiply(x, -1.0)))) # pylint: enable=g-long-lambda + @test_util.run_deprecated_v1 def testStrippedOpListDef(self): with self.cached_session(): # Creates a graph. @@ -2157,6 +2160,7 @@ class MetaGraphTest(test.TestCase): self.assertEqual(o.summary, "") self.assertEqual(o.description, "") + @test_util.run_deprecated_v1 def testStripDefaultValuedAttrs(self): """Verifies that default valued attrs are stripped, unless disabled.""" @@ -2193,6 +2197,7 @@ class MetaGraphTest(test.TestCase): self.assertIn("T", node_def.attr) self.assertIn("Tout", node_def.attr) + @test_util.run_deprecated_v1 def testImportIntoNamescope(self): # Test that we can import a meta graph into a namescope. test_dir = self._get_test_dir("import_into_namescope") @@ -2209,7 +2214,7 @@ class MetaGraphTest(test.TestCase): logits=logit, name="cost") adam.AdamOptimizer().minimize(cost, name="optimize") saver = saver_module.Saver() - sess.run(variables.global_variables_initializer()) + self.evaluate(variables.global_variables_initializer()) saver.save(sess, filename) graph = ops_lib.Graph() @@ -2246,7 +2251,7 @@ class MetaGraphTest(test.TestCase): # Create a variable in graph_2 under scope "my_scope". variables.VariableV1(array_ops.zeros([10]), name="my_scope/my_var") - sess.run(variables.global_variables_initializer()) + self.evaluate(variables.global_variables_initializer()) # Restore the checkpoint into a different scope "subgraph_2". new_saver_2 = saver_module.import_meta_graph( filename + ".meta", graph=graph_2, import_scope="subgraph_2") @@ -2263,6 +2268,7 @@ class MetaGraphTest(test.TestCase): filename + ".meta", graph=graph_2, import_scope="my_scope") self.assertIsInstance(new_saver_3, saver_module.Saver) + @test_util.run_deprecated_v1 def testImportIntoImplicitNamescope(self): # Test that we can import a meta graph into an implicit namescope. test_dir = self._get_test_dir("import_into_namescope") @@ -2279,7 +2285,7 @@ class MetaGraphTest(test.TestCase): logits=logit, name="cost") adam.AdamOptimizer().minimize(cost, name="optimize") saver = saver_module.Saver() - sess.run(variables.global_variables_initializer()) + self.evaluate(variables.global_variables_initializer()) saver.save(sess, filename) graph = ops_lib.Graph() @@ -2316,12 +2322,12 @@ class MetaGraphTest(test.TestCase): meta_graph_def, clear_devices=False, import_scope="new_model") # Device refers to GPU, which is not available here. with self.assertRaises(errors_impl.InvalidArgumentError): - sess.run(variables.global_variables_initializer()) + self.evaluate(variables.global_variables_initializer()) with session.Session(graph=ops_lib.Graph()) as sess: saver_module.import_meta_graph( meta_graph_def, clear_devices=True, import_scope="new_model") - sess.run(variables.global_variables_initializer()) + self.evaluate(variables.global_variables_initializer()) sess.run(["new_model/optimize"], { "new_model/image:0": np.random.random([1, 784]), "new_model/label:0": np.random.randint( @@ -2348,7 +2354,7 @@ class MetaGraphTest(test.TestCase): with session.Session(graph=ops_lib.Graph()) as sess: saver_module.import_meta_graph(meta_graph_def, import_scope="new_model") - sess.run(variables.global_variables_initializer()) + self.evaluate(variables.global_variables_initializer()) sess.run(["new_model/optimize"], { "new_model/image:0": np.random.random([1, 784]), "new_model/label:0": np.random.randint( @@ -2358,7 +2364,7 @@ class MetaGraphTest(test.TestCase): def testPreserveDatasetAndFunctions(self): with ops_lib.Graph().as_default() as g: dataset = dataset_ops.Dataset.range(10).map(lambda x: x * x) - iterator = dataset.make_one_shot_iterator() + iterator = dataset_ops.make_one_shot_iterator(dataset) next_element = iterator.get_next() _ = array_ops.identity(next_element, name="output") @@ -2374,7 +2380,7 @@ class MetaGraphTest(test.TestCase): meta_graph_def_from_graph_def]: with session.Session(graph=ops_lib.Graph()) as sess: saver_module.import_meta_graph(meta_graph_def, import_scope="new_model") - sess.run(variables.global_variables_initializer()) + self.evaluate(variables.global_variables_initializer()) for i in range(10): self.assertEqual(i * i, sess.run("new_model/output:0")) with self.assertRaises(errors.OutOfRangeError): @@ -2385,6 +2391,7 @@ class CheckpointReaderTest(test.TestCase): _WRITE_VERSION = saver_pb2.SaverDef.V1 + @test_util.run_deprecated_v1 def testDebugString(self): # Builds a graph. v0 = variables.VariableV1( @@ -2400,7 +2407,7 @@ class CheckpointReaderTest(test.TestCase): save_path = os.path.join(self.get_temp_dir(), "ckpt_for_debug_string" + str(self._WRITE_VERSION)) with self.cached_session() as sess: - sess.run(init_all_op) + self.evaluate(init_all_op) # Saves a checkpoint. save.save(sess, save_path) @@ -2546,7 +2553,7 @@ class ScopedGraphTest(test.TestCase): self.assertEqual(["biases:0", "weights:0"], sorted(var_list.keys())) with self.session(graph=graph) as sess: - sess.run(variables.global_variables_initializer()) + self.evaluate(variables.global_variables_initializer()) saver = saver_module.Saver(var_list=var_list, max_to_keep=1) saver.save(sess, os.path.join(test_dir, ckpt_filename), write_state=False) @@ -2609,13 +2616,14 @@ class ScopedGraphTest(test.TestCase): saver = saver_module.Saver(var_list=var_list, max_to_keep=1) saver.restore(sess, os.path.join(test_dir, ckpt_filename)) # Verify that we have restored weights1 and biases1. - sess.run([weights1, biases1]) + self.evaluate([weights1, biases1]) # Initialize the rest of the variables and run logits. - sess.run(init_rest_op) - sess.run(logits) + self.evaluate(init_rest_op) + self.evaluate(logits) # Verifies that we can save the subgraph under "hidden1" and restore it # into "new_hidden1" in the new graph. + @test_util.run_deprecated_v1 def testScopedSaveAndRestore(self): test_dir = self._get_test_dir("scoped_export_import") ckpt_filename = "ckpt" @@ -2625,6 +2633,7 @@ class ScopedGraphTest(test.TestCase): # Verifies that we can copy the subgraph under "hidden1" and copy it # to different name scope in the same graph or different graph. + @test_util.run_deprecated_v1 def testCopyScopedGraph(self): test_dir = self._get_test_dir("scoped_copy") saver0_ckpt = os.path.join(test_dir, "saver0.ckpt") @@ -2640,7 +2649,7 @@ class ScopedGraphTest(test.TestCase): # Run the graph and save scoped checkpoint. with self.session(graph=graph1) as sess: - sess.run(variables.global_variables_initializer()) + self.evaluate(variables.global_variables_initializer()) _, var_list_1 = meta_graph.export_scoped_meta_graph( export_scope="hidden1") saver = saver_module.Saver(var_list=var_list_1, max_to_keep=1) @@ -2681,6 +2690,7 @@ class ScopedGraphTest(test.TestCase): saver3.restore(sess, saver0_ckpt) self.assertAllClose(expected, sess.run("new_hidden1/relu:0")) + @test_util.run_deprecated_v1 def testExportGraphDefWithScope(self): test_dir = self._get_test_dir("export_graph_def") saver0_ckpt = os.path.join(test_dir, "saver0.ckpt") @@ -2696,7 +2706,7 @@ class ScopedGraphTest(test.TestCase): # Run the graph and save scoped checkpoint. with self.session(graph=graph1) as sess: - sess.run(variables.global_variables_initializer()) + self.evaluate(variables.global_variables_initializer()) _, var_list_1 = meta_graph.export_scoped_meta_graph( graph_def=graph1.as_graph_def(), export_scope="hidden1") saver = saver_module.Saver(var_list=var_list_1, max_to_keep=1) @@ -2717,6 +2727,7 @@ class ScopedGraphTest(test.TestCase): saver3.restore(sess, saver0_ckpt) self.assertAllClose(expected, sess.run("new_hidden1/relu:0")) + @test_util.run_deprecated_v1 def testSerializeSaverWithScope(self): test_dir = self._get_test_dir("export_graph_def") saver1_ckpt = os.path.join(test_dir, "saver1.ckpt") @@ -2964,7 +2975,7 @@ class CheckpointableCompatibilityTests(test.TestCase): a_saver = saver_module.Saver([a]) b_saver = saver_module.Saver([b]) with self.cached_session() as sess: - sess.run(a.initializer) + self.evaluate(a.initializer) save_path = a_saver.save(sess=sess, save_path=checkpoint_prefix) with self.assertRaisesRegexp( errors.NotFoundError, "Key b not found in checkpoint"): @@ -2977,6 +2988,7 @@ class CheckpointableCompatibilityTests(test.TestCase): # exception" block in Python 3. self.assertNotIn("NewCheckpointReader", cs.exception.message) + @test_util.run_deprecated_v1 def testGraphChangedForRestoreErrorRaised(self): checkpoint_directory = self.get_temp_dir() checkpoint_prefix = os.path.join(checkpoint_directory, "ckpt") @@ -2986,7 +2998,7 @@ class CheckpointableCompatibilityTests(test.TestCase): a_saver = saver_module.Saver([a]) with self.session(graph=g) as sess: - sess.run(a.initializer) + self.evaluate(a.initializer) save_path = a_saver.save(sess=sess, save_path=checkpoint_prefix) with ops_lib.Graph().as_default() as g: @@ -2998,6 +3010,7 @@ class CheckpointableCompatibilityTests(test.TestCase): "a mismatch between the current graph and the graph"): a_saver.restore(sess=sess, save_path=save_path) + @test_util.run_deprecated_v1 def testLoadFromObjectBasedGraph(self): checkpoint_directory = self.get_temp_dir() checkpoint_prefix = os.path.join(checkpoint_directory, "ckpt") @@ -3029,7 +3042,7 @@ class CheckpointableCompatibilityTests(test.TestCase): self.assertEqual(before_second_restore_ops, restore_graph.get_operations()) with self.assertRaisesRegexp(errors.NotFoundError, - "could not find a_variable"): + "Could not find some variables"): saver.restore(sess=sess, save_path=second_path) def testLoadFromObjectBasedEager(self): diff --git a/tensorflow/python/training/server_lib_multiple_containers_test.py b/tensorflow/python/training/server_lib_multiple_containers_test.py index f599e9b55b9..fb6118942bd 100644 --- a/tensorflow/python/training/server_lib_multiple_containers_test.py +++ b/tensorflow/python/training/server_lib_multiple_containers_test.py @@ -21,6 +21,7 @@ from __future__ import print_function from tensorflow.python.client import session from tensorflow.python.framework import errors_impl from tensorflow.python.framework import ops +from tensorflow.python.framework import test_util from tensorflow.python.ops import variables from tensorflow.python.platform import test from tensorflow.python.training import server_lib @@ -33,6 +34,7 @@ class MultipleContainersTest(test.TestCase): # TODO(b/34465411): Starting multiple servers with different configurations # in the same test is flaky. Move this test case back into # "server_lib_test.py" when this is no longer the case. + @test_util.run_deprecated_v1 def testMultipleContainers(self): with ops.container("test0"): v0 = variables.Variable(1.0, name="v0") diff --git a/tensorflow/python/training/server_lib_same_variables_clear_container_test.py b/tensorflow/python/training/server_lib_same_variables_clear_container_test.py index 11e6f28ab05..e0ab21bbd97 100644 --- a/tensorflow/python/training/server_lib_same_variables_clear_container_test.py +++ b/tensorflow/python/training/server_lib_same_variables_clear_container_test.py @@ -20,6 +20,7 @@ from __future__ import print_function from tensorflow.python.client import session from tensorflow.python.framework import errors_impl +from tensorflow.python.framework import test_util from tensorflow.python.ops import variables from tensorflow.python.platform import test from tensorflow.python.training import server_lib @@ -32,6 +33,7 @@ class SameVariablesClearContainerTest(test.TestCase): # TODO(b/34465411): Starting multiple servers with different configurations # in the same test is flaky. Move this test case back into # "server_lib_test.py" when this is no longer the case. + @test_util.run_deprecated_v1 def testSameVariablesClearContainer(self): # Starts two servers with different names so they map to different # resource "containers". @@ -60,9 +62,9 @@ class SameVariablesClearContainerTest(test.TestCase): session.Session.reset(server0.target, ["local0"]) sess = session.Session(server0.target) with self.assertRaises(errors_impl.FailedPreconditionError): - sess.run(v0) + self.evaluate(v0) # Reinitializes v0 for the following test. - sess.run(v0.initializer) + self.evaluate(v0.initializer) # Verifies that v1 is still valid. self.assertAllEqual(2.0, sess_1.run(v1)) @@ -71,10 +73,10 @@ class SameVariablesClearContainerTest(test.TestCase): session.Session.reset(server1.target, ["local1"]) sess = session.Session(server1.target) with self.assertRaises(errors_impl.FailedPreconditionError): - sess.run(v1) + self.evaluate(v1) # Verifies that v0 is still valid. sess = session.Session(server0.target) - self.assertAllEqual(1.0, sess.run(v0)) + self.assertAllEqual(1.0, self.evaluate(v0)) if __name__ == "__main__": diff --git a/tensorflow/python/training/server_lib_same_variables_clear_test.py b/tensorflow/python/training/server_lib_same_variables_clear_test.py index 4682f1ab84d..7b147af6c55 100644 --- a/tensorflow/python/training/server_lib_same_variables_clear_test.py +++ b/tensorflow/python/training/server_lib_same_variables_clear_test.py @@ -20,6 +20,7 @@ from __future__ import print_function from tensorflow.python.client import session from tensorflow.python.framework import errors_impl +from tensorflow.python.framework import test_util from tensorflow.python.ops import math_ops from tensorflow.python.ops import variables from tensorflow.python.platform import test @@ -32,6 +33,7 @@ class SameVariablesClearTest(test.TestCase): # TODO(b/34465411): Starting multiple servers with different configurations # in the same test is flaky. Move this test case back into # "server_lib_test.py" when this is no longer the case. + @test_util.run_deprecated_v1 def testSameVariablesClear(self): server = server_lib.Server.create_local_server() diff --git a/tensorflow/python/training/server_lib_same_variables_no_clear_test.py b/tensorflow/python/training/server_lib_same_variables_no_clear_test.py index 5aa7f45c2b3..1b2d588f444 100644 --- a/tensorflow/python/training/server_lib_same_variables_no_clear_test.py +++ b/tensorflow/python/training/server_lib_same_variables_no_clear_test.py @@ -20,6 +20,7 @@ from __future__ import print_function from tensorflow.python.client import session from tensorflow.python.framework import ops +from tensorflow.python.framework import test_util from tensorflow.python.ops import math_ops from tensorflow.python.ops import variables from tensorflow.python.platform import test @@ -33,6 +34,7 @@ class SameVariablesNoClearTest(test.TestCase): # TODO(b/34465411): Starting multiple servers with different configurations # in the same test is flaky. Move this test case back into # "server_lib_test.py" when this is no longer the case. + @test_util.run_deprecated_v1 def testSameVariablesNoClear(self): server = server_lib.Server.create_local_server() diff --git a/tensorflow/python/training/server_lib_sparse_job_test.py b/tensorflow/python/training/server_lib_sparse_job_test.py index 1a6b44b90e8..93b06e62160 100644 --- a/tensorflow/python/training/server_lib_sparse_job_test.py +++ b/tensorflow/python/training/server_lib_sparse_job_test.py @@ -21,6 +21,7 @@ from __future__ import print_function from tensorflow.python.client import session from tensorflow.python.framework import constant_op from tensorflow.python.framework import ops +from tensorflow.python.framework import test_util from tensorflow.python.platform import test from tensorflow.python.training import server_lib @@ -30,13 +31,14 @@ class SparseJobTest(test.TestCase): # TODO(b/34465411): Starting multiple servers with different configurations # in the same test is flaky. Move this test case back into # "server_lib_test.py" when this is no longer the case. + @test_util.run_deprecated_v1 def testSparseJob(self): server = server_lib.Server({"local": {37: "localhost:0"}}) with ops.device("/job:local/task:37"): a = constant_op.constant(1.0) with session.Session(server.target) as sess: - self.assertEqual(1.0, sess.run(a)) + self.assertEqual(1.0, self.evaluate(a)) if __name__ == "__main__": diff --git a/tensorflow/python/training/server_lib_test.py b/tensorflow/python/training/server_lib_test.py index cf995707fc5..323e94c257c 100644 --- a/tensorflow/python/training/server_lib_test.py +++ b/tensorflow/python/training/server_lib_test.py @@ -174,7 +174,7 @@ class GrpcServerTest(test.TestCase): # is not supported, but it should successfully ignore it. sess = session.InteractiveSession(server.target) c = constant_op.constant(42.0) - self.assertEqual(42.0, c.eval()) + self.assertEqual(42.0, self.evaluate(c)) sess.close() def testSetConfiguration(self): diff --git a/tensorflow/python/training/session_manager.py b/tensorflow/python/training/session_manager.py index cd313c2ce05..14658630c55 100644 --- a/tensorflow/python/training/session_manager.py +++ b/tensorflow/python/training/session_manager.py @@ -46,7 +46,7 @@ def _maybe_name(obj): return "" % type(obj) -@tf_export("train.SessionManager") +@tf_export(v1=["train.SessionManager"]) class SessionManager(object): """Training helper that restores from checkpoint and creates session. diff --git a/tensorflow/python/training/session_manager_test.py b/tensorflow/python/training/session_manager_test.py index 2b5c3b01def..4294ffa8512 100644 --- a/tensorflow/python/training/session_manager_test.py +++ b/tensorflow/python/training/session_manager_test.py @@ -25,6 +25,7 @@ from tensorflow.python.framework import dtypes from tensorflow.python.framework import errors from tensorflow.python.framework import errors_impl from tensorflow.python.framework import ops +from tensorflow.python.framework import test_util from tensorflow.python.ops import array_ops from tensorflow.python.ops import control_flow_ops from tensorflow.python.ops import variables @@ -68,6 +69,7 @@ class SessionManagerTest(test.TestCase): "", init_fn=lambda sess: sess.run(v.initializer)) self.assertAllClose([125], sess.run(v)) + @test_util.run_deprecated_v1 def testPrepareSessionFails(self): checkpoint_dir = os.path.join(self.get_temp_dir(), "prepare_session") checkpoint_dir2 = os.path.join(self.get_temp_dir(), "prepare_session2") @@ -152,6 +154,7 @@ class SessionManagerTest(test.TestCase): sess.graph.get_tensor_by_name("v:0")).eval(session=sess)) self.assertEquals(1, sess.run(v)) + @test_util.run_deprecated_v1 def testRecoverSession(self): # Create a checkpoint. checkpoint_dir = os.path.join(self.get_temp_dir(), "recover_session") @@ -206,6 +209,7 @@ class SessionManagerTest(test.TestCase): variables.global_variables()), local_init_op=None) + @test_util.run_deprecated_v1 def testRecoverSessionWithReadyForLocalInitOp(self): # Create a checkpoint. checkpoint_dir = os.path.join(self.get_temp_dir(), @@ -259,6 +263,7 @@ class SessionManagerTest(test.TestCase): self.assertEquals(1, sess.run(v)) self.assertEquals(1, sess.run(w)) + @test_util.run_deprecated_v1 def testRecoverSessionWithReadyForLocalInitOpFailsToReadyLocal(self): # We use ready_for_local_init_op=tf.report_uninitialized_variables(), # which causes recover_session to not run local_init_op, and to return @@ -315,6 +320,7 @@ class SessionManagerTest(test.TestCase): sess.graph.get_tensor_by_name("w:0")).eval(session=sess)) self.assertEquals(1, sess.run(v)) + @test_util.run_deprecated_v1 def testRecoverSessionNoChkptStillRunsLocalInitOp(self): # This test checks for backwards compatibility. # In particular, we continue to ensure that recover_session will execute @@ -343,6 +349,7 @@ class SessionManagerTest(test.TestCase): sess.graph.get_tensor_by_name("w:0")).eval(session=sess)) self.assertEquals(1, sess.run(w)) + @test_util.run_deprecated_v1 def testRecoverSessionFailsStillRunsLocalInitOp(self): # Create a checkpoint. checkpoint_dir = os.path.join( @@ -386,6 +393,7 @@ class SessionManagerTest(test.TestCase): sess.graph.get_tensor_by_name("w:0")).eval(session=sess)) self.assertEquals(1, sess.run(w)) + @test_util.run_deprecated_v1 def testWaitForSessionLocalInit(self): server = server_lib.Server.create_local_server() with ops.Graph().as_default() as graph: @@ -437,6 +445,7 @@ class SessionManagerTest(test.TestCase): # because of overly restrictive ready_for_local_init_op sm.wait_for_session("", max_wait_secs=3) + @test_util.run_deprecated_v1 def testWaitForSessionInsufficientReadyForLocalInitCheck(self): with ops.Graph().as_default() as graph: v = variables.VariableV1(1, name="v") @@ -454,6 +463,7 @@ class SessionManagerTest(test.TestCase): "Session was not ready after waiting.*"): sm.wait_for_session("", max_wait_secs=3) + @test_util.run_deprecated_v1 def testPrepareSessionWithReadyForLocalInitOp(self): with ops.Graph().as_default(): v = variables.VariableV1(1, name="v") @@ -493,6 +503,7 @@ class SessionManagerTest(test.TestCase): self.assertEquals(1, sess.run(w)) self.assertEquals(3, sess.run(x)) + @test_util.run_deprecated_v1 def testPrepareSessionWithPartialInitOp(self): with ops.Graph().as_default(): v = variables.VariableV1(1, name="v") @@ -559,6 +570,7 @@ class SessionManagerTest(test.TestCase): self.assertEquals(1, sess.run(w_res)) self.assertEquals(3, sess.run(x_res)) + @test_util.run_deprecated_v1 def testPrepareSessionWithCyclicInitializer(self): # Regression test. Previously Variable._build_initializer_expr would enter # into an infinite recursion when the variable's initial_value involved @@ -632,6 +644,7 @@ class SessionManagerTest(test.TestCase): "Init operations did not make model ready for local_init"): sm2.prepare_session("", init_op=None) + @test_util.run_deprecated_v1 def testPrepareSessionWithInsufficientReadyForLocalInitCheck(self): with ops.Graph().as_default(): v = variables.VariableV1(1, name="v") @@ -684,6 +697,7 @@ class ObsoleteSessionManagerTest(test.TestCase): "", init_fn=lambda sess: sess.run(v.initializer)) self.assertAllClose([125], sess.run(v)) + @test_util.run_deprecated_v1 def testPrepareSessionFails(self): checkpoint_dir = os.path.join(self.get_temp_dir(), "prepare_session") checkpoint_dir2 = os.path.join(self.get_temp_dir(), "prepare_session2") @@ -745,6 +759,7 @@ class ObsoleteSessionManagerTest(test.TestCase): variables.is_variable_initialized( sess.graph.get_tensor_by_name("v:0")).eval(session=sess)) + @test_util.run_deprecated_v1 def testRecoverSession(self): # Create a checkpoint. checkpoint_dir = os.path.join(self.get_temp_dir(), "recover_session") diff --git a/tensorflow/python/training/session_run_hook.py b/tensorflow/python/training/session_run_hook.py index 5daea931288..e9a61def743 100644 --- a/tensorflow/python/training/session_run_hook.py +++ b/tensorflow/python/training/session_run_hook.py @@ -186,7 +186,7 @@ class SessionRunHook(object): pass -@tf_export("train.SessionRunArgs") +@tf_export(v1=["train.SessionRunArgs"]) class SessionRunArgs( collections.namedtuple("SessionRunArgs", ["fetches", "feed_dict", "options"])): @@ -211,7 +211,7 @@ class SessionRunArgs( return super(SessionRunArgs, cls).__new__(cls, fetches, feed_dict, options) -@tf_export("train.SessionRunContext") +@tf_export(v1=["train.SessionRunContext"]) class SessionRunContext(object): """Provides information about the `session.run()` call being made. @@ -263,7 +263,7 @@ class SessionRunContext(object): self._stop_requested = True -@tf_export("train.SessionRunValues") +@tf_export(v1=["train.SessionRunValues"]) class SessionRunValues( collections.namedtuple("SessionRunValues", ["results", "options", "run_metadata"])): diff --git a/tensorflow/python/training/slot_creator_test.py b/tensorflow/python/training/slot_creator_test.py index 6d6364169fd..1f26aaa434e 100644 --- a/tensorflow/python/training/slot_creator_test.py +++ b/tensorflow/python/training/slot_creator_test.py @@ -21,6 +21,7 @@ from __future__ import print_function from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops +from tensorflow.python.framework import test_util from tensorflow.python.ops import array_ops from tensorflow.python.ops import random_ops from tensorflow.python.ops import variable_scope @@ -31,6 +32,7 @@ from tensorflow.python.training import slot_creator class SlotCreatorTest(test.TestCase): + @test_util.run_deprecated_v1 def testCreateSlotFromVariable(self): with self.cached_session(): v = variables.Variable([1.0, 2.5], name="var") @@ -41,8 +43,9 @@ class SlotCreatorTest(test.TestCase): self.assertEqual("var/slot", slot.op.name) self.assertEqual([2], slot.get_shape().as_list()) self.assertEqual(dtypes.float32, slot.dtype.base_dtype) - self.assertAllEqual([1.0, 2.5], slot.eval()) + self.assertAllEqual([1.0, 2.5], self.evaluate(slot)) + @test_util.run_deprecated_v1 def testCreateSlotFromTensor(self): with self.cached_session(): v = constant_op.constant([1.0, 2.5], name="const") @@ -53,8 +56,9 @@ class SlotCreatorTest(test.TestCase): self.assertEqual("const/slot", slot.op.name) self.assertEqual([2], slot.get_shape().as_list()) self.assertEqual(dtypes.float32, slot.dtype.base_dtype) - self.assertAllEqual([2.0, 5.0], slot.eval()) + self.assertAllEqual([2.0, 5.0], self.evaluate(slot)) + @test_util.run_deprecated_v1 def testCreateZerosSlotFromVariable(self): with self.cached_session(): v = variables.Variable([1.0, 2.5], name="var") @@ -67,8 +71,9 @@ class SlotCreatorTest(test.TestCase): self.assertEqual("var/slot", slot.op.name) self.assertEqual([2], slot.get_shape().as_list()) self.assertEqual(dtypes.float64, slot.dtype.base_dtype) - self.assertAllEqual([0.0, 0.0], slot.eval()) + self.assertAllEqual([0.0, 0.0], self.evaluate(slot)) + @test_util.run_deprecated_v1 def testCreateZerosSlotFromDynamicShapedVariable(self): with self.cached_session(): dyn_shape = constant_op.constant([2], dtype=dtypes.int32) @@ -88,8 +93,9 @@ class SlotCreatorTest(test.TestCase): self.assertEqual("var/slot", slot.op.name) self.assertEqual([2], array_ops.shape(slot).eval()) self.assertEqual(dtypes.float64, slot.dtype.base_dtype) - self.assertAllEqual([0.0, 0.0], slot.eval()) + self.assertAllEqual([0.0, 0.0], self.evaluate(slot)) + @test_util.run_deprecated_v1 def testCreateZerosSlotFromTensor(self): with self.cached_session(): v = constant_op.constant([1.0, 2.5], name="const") @@ -101,8 +107,9 @@ class SlotCreatorTest(test.TestCase): self.assertEqual("const/slot", slot.op.name) self.assertEqual([2], slot.get_shape().as_list()) self.assertEqual(dtypes.float32, slot.dtype.base_dtype) - self.assertAllEqual([0.0, 0.0], slot.eval()) + self.assertAllEqual([0.0, 0.0], self.evaluate(slot)) + @test_util.run_deprecated_v1 def testCreateZerosSlotFromDynamicShapedTensor(self): with self.cached_session(): v = random_ops.random_uniform([2], dtype=dtypes.float64) @@ -116,8 +123,9 @@ class SlotCreatorTest(test.TestCase): self.assertEqual("const/slot", slot.op.name) self.assertEqual([2], array_ops.shape(slot).eval()) self.assertEqual(dtypes.float64, slot.dtype.base_dtype) - self.assertAllEqual([0.0, 0.0], slot.eval()) + self.assertAllEqual([0.0, 0.0], self.evaluate(slot)) + @test_util.run_deprecated_v1 def testCreateSlotFromVariableRespectsScope(self): # See discussion on #2740. with self.cached_session(): diff --git a/tensorflow/python/training/supervisor.py b/tensorflow/python/training/supervisor.py index a5e626d3204..de60dd456ff 100644 --- a/tensorflow/python/training/supervisor.py +++ b/tensorflow/python/training/supervisor.py @@ -40,7 +40,7 @@ from tensorflow.python.util import deprecation from tensorflow.python.util.tf_export import tf_export -@tf_export("train.Supervisor") +@tf_export(v1=["train.Supervisor"]) class Supervisor(object): """A training helper that checkpoints models and computes summaries. diff --git a/tensorflow/python/training/supervisor_test.py b/tensorflow/python/training/supervisor_test.py index 7cd99d86801..f6505acc9ac 100644 --- a/tensorflow/python/training/supervisor_test.py +++ b/tensorflow/python/training/supervisor_test.py @@ -35,6 +35,7 @@ from tensorflow.python.framework import dtypes from tensorflow.python.framework import errors_impl from tensorflow.python.framework import meta_graph from tensorflow.python.framework import ops +from tensorflow.python.framework import test_util from tensorflow.python.ops import array_ops from tensorflow.python.ops import io_ops from tensorflow.python.ops import parsing_ops @@ -100,7 +101,7 @@ class SupervisorTest(test.TestCase): sv = supervisor.Supervisor(logdir=logdir) sess = sv.prepare_or_wait_for_session("") for _ in xrange(10): - sess.run(my_op) + self.evaluate(my_op) sess.close() sv.stop() @@ -111,7 +112,7 @@ class SupervisorTest(test.TestCase): sv = supervisor.Supervisor(logdir=logdir) with sv.managed_session("") as sess: for _ in xrange(10): - sess.run(my_op) + self.evaluate(my_op) # Supervisor has been stopped. self.assertTrue(sv.should_stop()) @@ -128,7 +129,7 @@ class SupervisorTest(test.TestCase): if step == 1: raise RuntimeError("failing here") else: - sess.run(my_op) + self.evaluate(my_op) # Supervisor has been stopped. self.assertTrue(sv.should_stop()) self.assertEqual(1, last_step) @@ -146,7 +147,7 @@ class SupervisorTest(test.TestCase): raise errors_impl.OutOfRangeError(my_op.op.node_def, my_op.op, "all done") else: - sess.run(my_op) + self.evaluate(my_op) # Supervisor has been stopped. OutOfRangeError was not thrown. self.assertTrue(sv.should_stop()) self.assertEqual(3, last_step) @@ -335,7 +336,7 @@ class SupervisorTest(test.TestCase): sess = sv.prepare_or_wait_for_session( "", config=config_pb2.ConfigProto(device_count={"CPU": 2})) for _ in xrange(10): - sess.run(my_op) + self.evaluate(my_op) sess.close() sv.stop() @@ -420,6 +421,7 @@ class SupervisorTest(test.TestCase): with self.assertRaisesRegexp(RuntimeError, "requires a summary writer"): sv.summary_computed(sess, sess.run(summ)) + @test_util.run_deprecated_v1 def testLogdirButExplicitlyNoSummaryWriter(self): logdir = self._test_dir("explicit_no_summary_writer") with ops.Graph().as_default(): @@ -505,6 +507,7 @@ class SupervisorTest(test.TestCase): sv = supervisor.Supervisor(logdir="", session_manager=sm) sv.prepare_or_wait_for_session("") + @test_util.run_deprecated_v1 def testInitOp(self): logdir = self._test_dir("default_init_op") with ops.Graph().as_default(): @@ -514,6 +517,7 @@ class SupervisorTest(test.TestCase): self.assertAllClose([1.0, 2.0, 3.0], sess.run(v)) sv.stop() + @test_util.run_deprecated_v1 def testInitFn(self): logdir = self._test_dir("default_init_op") with ops.Graph().as_default(): @@ -527,6 +531,7 @@ class SupervisorTest(test.TestCase): self.assertAllClose([1.0, 2.0, 3.0], sess.run(v)) sv.stop() + @test_util.run_deprecated_v1 def testInitOpWithFeedDict(self): logdir = self._test_dir("feed_dict_init_op") with ops.Graph().as_default(): @@ -540,6 +545,7 @@ class SupervisorTest(test.TestCase): self.assertAllClose([1.0, 2.0, 3.0], sess.run(v)) sv.stop() + @test_util.run_deprecated_v1 def testReadyForLocalInitOp(self): server = server_lib.Server.create_local_server() logdir = self._test_dir("default_ready_for_local_init_op") @@ -582,6 +588,7 @@ class SupervisorTest(test.TestCase): sv0.stop() sv1.stop() + @test_util.run_deprecated_v1 def testReadyForLocalInitOpRestoreFromCheckpoint(self): server = server_lib.Server.create_local_server() logdir = self._test_dir("ready_for_local_init_op_restore") @@ -713,6 +720,7 @@ class SupervisorTest(test.TestCase): "Variables not initialized: w"): sv.prepare_or_wait_for_session(server.target) + @test_util.run_deprecated_v1 def testSetupFail(self): logdir = self._test_dir("setup_fail") with ops.Graph().as_default(): @@ -723,6 +731,7 @@ class SupervisorTest(test.TestCase): variables.VariableV1([1.0, 2.0, 3.0], name="v") supervisor.Supervisor(logdir=logdir, is_chief=False) + @test_util.run_deprecated_v1 def testDefaultGlobalStep(self): logdir = self._test_dir("default_global_step") with ops.Graph().as_default(): @@ -732,6 +741,7 @@ class SupervisorTest(test.TestCase): self.assertEquals(287, sess.run(sv.global_step)) sv.stop() + @test_util.run_deprecated_v1 def testRestoreFromMetaGraph(self): logdir = self._test_dir("restore_from_meta_graph") with ops.Graph().as_default(): @@ -753,6 +763,7 @@ class SupervisorTest(test.TestCase): # This test is based on the fact that the standard services start # right away and get to run once before sv.stop() returns. # We still sleep a bit to make the test robust. + @test_util.run_deprecated_v1 def testStandardServicesWithoutGlobalStep(self): logdir = self._test_dir("standard_services_without_global_step") # Create a checkpoint. @@ -799,10 +810,11 @@ class SupervisorTest(test.TestCase): v = variables.VariableV1([10.10], name="foo") sav = saver_lib.Saver([v]) sav.restore(sess, save_path) - self.assertEqual(1.0, v.eval()[0]) + self.assertEqual(1.0, self.evaluate(v)[0]) # Same as testStandardServicesNoGlobalStep but with a global step. # We should get a summary about the step time. + @test_util.run_deprecated_v1 def testStandardServicesWithGlobalStep(self): logdir = self._test_dir("standard_services_with_global_step") # Create a checkpoint. @@ -863,7 +875,7 @@ class SupervisorTest(test.TestCase): v = variables.VariableV1([-12], name="global_step") sav = saver_lib.Saver([v]) sav.restore(sess, save_path) - self.assertEqual(123, v.eval()[0]) + self.assertEqual(123, self.evaluate(v)[0]) def testNoQueueRunners(self): with ops.Graph().as_default(), self.cached_session() as sess: diff --git a/tensorflow/python/training/sync_replicas_optimizer.py b/tensorflow/python/training/sync_replicas_optimizer.py index fbde8fe3c2a..172c1411505 100644 --- a/tensorflow/python/training/sync_replicas_optimizer.py +++ b/tensorflow/python/training/sync_replicas_optimizer.py @@ -44,6 +44,9 @@ from tensorflow.python.util.tf_export import tf_export class SyncReplicasOptimizer(optimizer.Optimizer): """Class to synchronize, aggregate gradients and pass them to the optimizer. + This class is deprecated. For synchrononous training, please use [Distribution + Strategies](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/distribute). + In a typical asynchronous training environment, it's common to have some stale gradients. For example, with a N-replica asynchronous training, gradients will be applied to the variables N times independently. Depending @@ -142,9 +145,9 @@ class SyncReplicasOptimizer(optimizer.Optimizer): @deprecation.deprecated( None, - "The `SyncReplicaOptimizer` is deprecated. For synchrononous training, " - "please use [Distribution Strategies](https://github.com/tensorflow/" - "tensorflow/tree/master/tensorflow/contrib/distribute).", + "The `SyncReplicaOptimizer` class is deprecated. For synchrononous " + "training, please use [Distribution Strategies](https://github.com/" + "tensorflow/tensorflow/tree/master/tensorflow/contrib/distribute).", warn_once=True) def __init__(self, opt, diff --git a/tensorflow/python/training/training_ops_test.py b/tensorflow/python/training/training_ops_test.py index 02164828250..51f49ca0818 100644 --- a/tensorflow/python/training/training_ops_test.py +++ b/tensorflow/python/training/training_ops_test.py @@ -24,6 +24,7 @@ import numpy as np from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes +from tensorflow.python.framework import test_util from tensorflow.python.framework.test_util import TensorFlowTestCase # Import resource_variable_ops for the variables-to-tensor implicit conversion. from tensorflow.python.ops import resource_variable_ops # pylint: disable=unused-import @@ -53,12 +54,13 @@ class TrainingOpsTest(TensorFlowTestCase): with self.session(use_gpu=use_gpu): var = variables.VariableV1(x) variables.global_variables_initializer().run() - self.assertAllCloseAccordingToType(x, var.eval()) + self.assertAllCloseAccordingToType(x, self.evaluate(var)) apply_sgd = training_ops.apply_gradient_descent(var, alpha, delta) - out = apply_sgd.eval() + out = self.evaluate(apply_sgd) self.assertShapeEqual(out, apply_sgd) self.assertAllCloseAccordingToType(x - alpha * delta, out) + @test_util.run_deprecated_v1 def testApplyGradientDescent(self): for (dtype, use_gpu) in itertools.product( [np.float16, np.float32, np.float64], [False, True]): @@ -74,13 +76,13 @@ class TrainingOpsTest(TensorFlowTestCase): accum = variables.VariableV1(y) variables.global_variables_initializer().run() - self.assertAllCloseAccordingToType(x, var.eval()) + self.assertAllCloseAccordingToType(x, self.evaluate(var)) apply_adagrad = training_ops.apply_adagrad(var, accum, lr, grad) - out = apply_adagrad.eval() + out = self.evaluate(apply_adagrad) self.assertShapeEqual(out, apply_adagrad) self.assertAllCloseAccordingToType(x - lr * grad * (y + grad * grad)** (-0.5), out) - self.assertAllCloseAccordingToType(y + grad * grad, accum.eval()) + self.assertAllCloseAccordingToType(y + grad * grad, self.evaluate(accum)) def _testTypesForFtrl(self, x, @@ -99,10 +101,10 @@ class TrainingOpsTest(TensorFlowTestCase): linear = variables.VariableV1(z) variables.global_variables_initializer().run() - self.assertAllCloseAccordingToType(x, var.eval()) + self.assertAllCloseAccordingToType(x, self.evaluate(var)) apply_ftrl = training_ops.apply_ftrl(var, accum, linear, grad, lr, l1, l2, lr_power) - out = apply_ftrl.eval() + out = self.evaluate(apply_ftrl) self.assertShapeEqual(out, apply_ftrl) accum_update = y + grad * grad linear_update = z + grad - (accum_update**(-lr_power) - y** @@ -112,19 +114,22 @@ class TrainingOpsTest(TensorFlowTestCase): np.sign(linear_update[i]) * l1 - linear_update[i]) / (quadratic[i]) if np.abs(linear_update[i]) > l1 else 0.0 for i in range(linear_update.size)]) - self.assertAllCloseAccordingToType(accum_update, accum.eval()) + self.assertAllCloseAccordingToType(accum_update, self.evaluate(accum)) if x.dtype == np.float16: # The calculations here really are not very precise in float16. - self.assertAllClose(linear_update, linear.eval(), rtol=2e-2, atol=2e-2) + self.assertAllClose( + linear_update, self.evaluate(linear), rtol=2e-2, atol=2e-2) self.assertAllClose(expected_out, out, rtol=2e-2, atol=2e-2) elif x.dtype == np.float32: # The calculations here not sufficiently precise in float32. - self.assertAllClose(linear_update, linear.eval(), rtol=1e-5, atol=1e-5) + self.assertAllClose( + linear_update, self.evaluate(linear), rtol=1e-5, atol=1e-5) self.assertAllClose(expected_out, out, rtol=1e-5, atol=1e-5) else: - self.assertAllClose(linear_update, linear.eval()) + self.assertAllClose(linear_update, self.evaluate(linear)) self.assertAllClose(expected_out, out) + @test_util.run_deprecated_v1 def testApplyAdagrad(self): for (dtype, use_gpu) in itertools.product( [np.float16, np.float32, np.float64], [False, True]): @@ -134,6 +139,7 @@ class TrainingOpsTest(TensorFlowTestCase): grad = np.arange(100).astype(dtype) self._testTypesForAdagrad(x, y, lr, grad, use_gpu) + @test_util.run_deprecated_v1 def testApplyFtrl(self): for dtype in [np.float16, np.float32, np.float64]: x = np.arange(100).astype(dtype) @@ -152,19 +158,19 @@ class TrainingOpsTest(TensorFlowTestCase): accum = variables.VariableV1(y) variables.global_variables_initializer().run() - self.assertAllCloseAccordingToType(x, var.eval()) + self.assertAllCloseAccordingToType(x, self.evaluate(var)) sparse_apply_adagrad = training_ops.sparse_apply_adagrad( var, accum, lr, grad, constant_op.constant(indices, self._toType(indices.dtype))) - out = sparse_apply_adagrad.eval() + out = self.evaluate(sparse_apply_adagrad) self.assertShapeEqual(out, sparse_apply_adagrad) for (i, index) in enumerate(indices): self.assertAllCloseAccordingToType( x[index] - lr * grad[i] * (y[index] + grad[i] * grad[i])**(-0.5), - var.eval()[index]) + self.evaluate(var)[index]) self.assertAllCloseAccordingToType(y[index] + grad[i] * grad[i], - accum.eval()[index]) + self.evaluate(accum)[index]) def _testTypesForSparseFtrl(self, x, @@ -183,7 +189,7 @@ class TrainingOpsTest(TensorFlowTestCase): linear = variables.VariableV1(z) variables.global_variables_initializer().run() - self.assertAllCloseAccordingToType(x, var.eval()) + self.assertAllCloseAccordingToType(x, self.evaluate(var)) sparse_apply_ftrl = training_ops.sparse_apply_ftrl( var, accum, @@ -194,16 +200,18 @@ class TrainingOpsTest(TensorFlowTestCase): l1, l2, lr_power=lr_power) - out = sparse_apply_ftrl.eval() + out = self.evaluate(sparse_apply_ftrl) self.assertShapeEqual(out, sparse_apply_ftrl) for (i, index) in enumerate(indices): - self.assertAllCloseAccordingToType(x[index] - lr * grad[i] * - (y[index] + grad[i] * grad[i])** - (lr_power), var.eval()[index]) + self.assertAllCloseAccordingToType( + x[index] - lr * grad[i] * (y[index] + grad[i] * grad[i])** + (lr_power), + self.evaluate(var)[index]) self.assertAllCloseAccordingToType(y[index] + grad[i] * grad[i], - accum.eval()[index]) + self.evaluate(accum)[index]) + @test_util.run_deprecated_v1 def testSparseApplyAdagrad(self): for (dtype, index_type) in itertools.product( [np.float16, np.float32, np.float64], [np.int32, np.int64]): @@ -217,6 +225,7 @@ class TrainingOpsTest(TensorFlowTestCase): indices = np.array([0, 2]).astype(index_type) self._testTypesForSparseAdagrad(x, y, lr, grad, indices) + @test_util.run_deprecated_v1 def testSparseApplyAdagradDim1(self): for (dtype, index_type) in itertools.product( [np.float16, np.float32, np.float64], [np.int32, np.int64]): @@ -230,6 +239,7 @@ class TrainingOpsTest(TensorFlowTestCase): indices = np.array([0, 2]).astype(index_type) self._testTypesForSparseAdagrad(x, y, lr, grad, indices) + @test_util.run_deprecated_v1 def testSparseApplyFtrlDim1(self): for (dtype, index_type) in itertools.product( [np.float16, np.float32, np.float64], [np.int32, np.int64]): @@ -245,6 +255,7 @@ class TrainingOpsTest(TensorFlowTestCase): indices = np.array([0, 2]).astype(index_type) self._testTypesForSparseFtrl(x, y, z, lr, grad, indices) + @test_util.run_deprecated_v1 def testApplyAdam(self): for dtype, use_gpu in itertools.product( [np.float16, np.float32, np.float64], [False, True]): @@ -276,13 +287,13 @@ class TrainingOpsTest(TensorFlowTestCase): epsilon_t = constant_op.constant(epsilon, self._toType(var.dtype), []) variables.global_variables_initializer().run() - self.assertAllCloseAccordingToType(var, var_t.eval()) + self.assertAllCloseAccordingToType(var, self.evaluate(var_t)) new_var, _, _ = self._adamUpdateNumpy(var, grad, t, m, v, lr, beta1, beta2, epsilon) apply_adam = training_ops.apply_adam(var_t, m_t, v_t, beta1_power_t, beta2_power_t, lr_t, beta1_t, beta2_t, epsilon_t, grad) - out = apply_adam.eval() + out = self.evaluate(apply_adam) self.assertShapeEqual(out, apply_adam) self.assertAllCloseAccordingToType(new_var, out) diff --git a/tensorflow/python/training/training_util_test.py b/tensorflow/python/training/training_util_test.py index ba64e785ac6..3317008fce0 100644 --- a/tensorflow/python/training/training_util_test.py +++ b/tensorflow/python/training/training_util_test.py @@ -20,6 +20,7 @@ from __future__ import print_function from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops +from tensorflow.python.framework import test_util from tensorflow.python.ops import variables from tensorflow.python.platform import test from tensorflow.python.training import monitored_session @@ -46,6 +47,7 @@ class GlobalStepTest(test.TestCase): self.assertRaisesRegexp(TypeError, 'does not have integer type', training_util.get_global_step, g) + @test_util.run_deprecated_v1 def test_invalid_shape(self): with ops.Graph().as_default() as g: self.assertIsNone(training_util.get_global_step()) @@ -70,6 +72,7 @@ class GlobalStepTest(test.TestCase): training_util.create_global_step, g) self._assert_global_step(training_util.create_global_step(ops.Graph())) + @test_util.run_deprecated_v1 def test_get_global_step(self): with ops.Graph().as_default() as g: self.assertIsNone(training_util.get_global_step()) diff --git a/tensorflow/python/training/warm_starting_util.py b/tensorflow/python/training/warm_starting_util.py index 78dbb465b55..8c97f101da8 100644 --- a/tensorflow/python/training/warm_starting_util.py +++ b/tensorflow/python/training/warm_starting_util.py @@ -32,7 +32,7 @@ from tensorflow.python.training import saver from tensorflow.python.util.tf_export import tf_export -@tf_export("train.VocabInfo") +@tf_export(v1=["train.VocabInfo"]) class VocabInfo( collections.namedtuple("VocabInfo", [ "new_vocab", @@ -248,7 +248,7 @@ def _warm_start_var_with_vocab(var, prev_tensor_name = _infer_var_name(var) # TODO(eddz): Fix functionality for rank-1 Variables (like FC biases). - total_v_first_axis = sum([v.get_shape().as_list()[0] for v in var]) + total_v_first_axis = sum(v.get_shape().as_list()[0] for v in var) for v in var: v_shape = v.get_shape().as_list() slice_info = v._get_save_slice_info() @@ -333,12 +333,12 @@ def _get_grouped_variables(vars_to_warm_start): ops.GraphKeys.TRAINABLE_VARIABLES, scope=vars_to_warm_start) elif isinstance(vars_to_warm_start, list): - if all([isinstance(v, str) for v in vars_to_warm_start]): + if all(isinstance(v, str) for v in vars_to_warm_start): list_of_vars = [] for v in vars_to_warm_start: list_of_vars += ops.get_collection(ops.GraphKeys.GLOBAL_VARIABLES, scope=v) - elif all([checkpoint_utils._is_variable(v) for v in vars_to_warm_start]): # pylint: disable=protected-access + elif all(checkpoint_utils._is_variable(v) for v in vars_to_warm_start): # pylint: disable=protected-access list_of_vars = vars_to_warm_start else: raise ValueError("If `vars_to_warm_start` is a list, it must be all " @@ -360,7 +360,7 @@ def _get_grouped_variables(vars_to_warm_start): return grouped_variables -@tf_export("train.warm_start") +@tf_export(v1=["train.warm_start"]) def warm_start(ckpt_to_initialize_from, vars_to_warm_start=".*", var_name_to_vocab_info=None, diff --git a/tensorflow/python/training/warm_starting_util_test.py b/tensorflow/python/training/warm_starting_util_test.py index 91a0b53b3a8..fa1f370f41e 100644 --- a/tensorflow/python/training/warm_starting_util_test.py +++ b/tensorflow/python/training/warm_starting_util_test.py @@ -22,7 +22,7 @@ import os import numpy as np import six -from tensorflow.python.feature_column import feature_column as fc +from tensorflow.python.feature_column import feature_column_lib as fc from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops from tensorflow.python.ops import array_ops @@ -49,7 +49,7 @@ class WarmStartingUtilTest(test.TestCase): return vocab_file def _write_checkpoint(self, sess): - sess.run(variables.global_variables_initializer()) + self.evaluate(variables.global_variables_initializer()) saver = saver_lib.Saver() ckpt_prefix = os.path.join(self.get_temp_dir(), "model") saver.save(sess, ckpt_prefix, global_step=0) @@ -70,7 +70,7 @@ class WarmStartingUtilTest(test.TestCase): if partitioner: self.assertTrue(isinstance(var, variables.PartitionedVariable)) var = var._get_variable_list() - return var, sess.run(var) + return var, self.evaluate(var) def _create_prev_run_vars(self, var_names, @@ -86,7 +86,7 @@ class WarmStartingUtilTest(test.TestCase): shape=shape, initializer=initializer)) self._write_checkpoint(sess) - return [sess.run(var) for var in all_vars] + return [self.evaluate(var) for var in all_vars] def _create_dummy_inputs(self): return { @@ -125,7 +125,7 @@ class WarmStartingUtilTest(test.TestCase): prev_tensor_name, var = ws_util._get_var_info(fruit_weights) checkpoint_utils.init_from_checkpoint(self.get_temp_dir(), {prev_tensor_name: var}) - sess.run(variables.global_variables_initializer()) + self.evaluate(variables.global_variables_initializer()) self.assertAllClose(prev_val, fruit_weights.eval(sess)) def testWarmStartVarPrevVarPartitioned(self): @@ -143,7 +143,7 @@ class WarmStartingUtilTest(test.TestCase): prev_tensor_name, var = ws_util._get_var_info(fruit_weights) checkpoint_utils.init_from_checkpoint(self.get_temp_dir(), {prev_tensor_name: var}) - sess.run(variables.global_variables_initializer()) + self.evaluate(variables.global_variables_initializer()) self.assertAllClose(prev_val, fruit_weights.eval(sess)) def testWarmStartVarCurrentVarPartitioned(self): @@ -162,7 +162,7 @@ class WarmStartingUtilTest(test.TestCase): prev_tensor_name, var = ws_util._get_var_info(fruit_weights) checkpoint_utils.init_from_checkpoint(self.get_temp_dir(), {prev_tensor_name: var}) - sess.run(variables.global_variables_initializer()) + self.evaluate(variables.global_variables_initializer()) fruit_weights = fruit_weights._get_variable_list() new_val = np.concatenate( [fruit_weights[0].eval(sess), fruit_weights[1].eval(sess)], axis=0) @@ -189,7 +189,7 @@ class WarmStartingUtilTest(test.TestCase): fruit_weights, prev_tensor_name="old_scope/fruit_weights") checkpoint_utils.init_from_checkpoint(self.get_temp_dir(), {prev_tensor_name: var}) - sess.run(variables.global_variables_initializer()) + self.evaluate(variables.global_variables_initializer()) fruit_weights = fruit_weights._get_variable_list() new_val = np.concatenate( [fruit_weights[0].eval(sess), fruit_weights[1].eval(sess)], axis=0) @@ -211,7 +211,7 @@ class WarmStartingUtilTest(test.TestCase): "fruit_weights", initializer=[[0.], [0.], [0.], [0.], [0.]]) ws_util._warm_start_var_with_vocab(fruit_weights, new_vocab_path, 5, self.get_temp_dir(), prev_vocab_path) - sess.run(variables.global_variables_initializer()) + self.evaluate(variables.global_variables_initializer()) self.assertAllClose([[2.], [1.5], [1.], [0.5], [0.]], fruit_weights.eval(sess)) @@ -236,7 +236,7 @@ class WarmStartingUtilTest(test.TestCase): prev_ckpt=self.get_temp_dir(), prev_vocab_path=prev_vocab_path, axis=1) - sess.run(variables.global_variables_initializer()) + self.evaluate(variables.global_variables_initializer()) self.assertAllClose([[0.3, 0.5, 0.], [0.8, 1.0, 0.], [1.2, 1.5, 0.], [2.3, 2., 0.]], fruit_output_layer.eval(sess)) @@ -261,7 +261,7 @@ class WarmStartingUtilTest(test.TestCase): self.get_temp_dir(), prev_vocab_path, previous_vocab_size=2) - sess.run(variables.global_variables_initializer()) + self.evaluate(variables.global_variables_initializer()) # Old vocabulary limited to ['apple', 'banana']. self.assertAllClose([[0.], [0.], [1.], [0.5], [0.]], fruit_weights.eval(sess)) @@ -285,7 +285,7 @@ class WarmStartingUtilTest(test.TestCase): "fruit_weights", initializer=[[0.], [0.], [0.], [0.], [0.]]) ws_util._warm_start_var_with_vocab(fruit_weights, new_vocab_path, 5, self.get_temp_dir(), prev_vocab_path) - sess.run(variables.global_variables_initializer()) + self.evaluate(variables.global_variables_initializer()) self.assertAllClose([[2.], [1.5], [1.], [0.5], [0.]], fruit_weights.eval(sess)) @@ -312,7 +312,7 @@ class WarmStartingUtilTest(test.TestCase): prev_ckpt=self.get_temp_dir(), prev_vocab_path=prev_vocab_path, axis=1) - sess.run(variables.global_variables_initializer()) + self.evaluate(variables.global_variables_initializer()) self.assertAllClose([[0.3, 0.5, 0.], [0.8, 1.0, 0.], [1.2, 1.5, 0.], [2.3, 2., 0.]], fruit_output_layer.eval(sess)) @@ -340,7 +340,7 @@ class WarmStartingUtilTest(test.TestCase): self.get_temp_dir(), prev_vocab_path, current_oov_buckets=1) - sess.run(variables.global_variables_initializer()) + self.evaluate(variables.global_variables_initializer()) self.assertTrue( isinstance(fruit_weights, variables.PartitionedVariable)) fruit_weights_vars = fruit_weights._get_variable_list() @@ -372,7 +372,7 @@ class WarmStartingUtilTest(test.TestCase): prev_ckpt=self.get_temp_dir(), prev_vocab_path=prev_vocab_path, axis=1) - sess.run(variables.global_variables_initializer()) + self.evaluate(variables.global_variables_initializer()) self.assertTrue( isinstance(fruit_output_layer, variables.PartitionedVariable)) fruit_output_layer_vars = fruit_output_layer._get_variable_list() @@ -404,7 +404,7 @@ class WarmStartingUtilTest(test.TestCase): partitioner=lambda shape, dtype: [2, 1]) ws_util._warm_start_var_with_vocab(fruit_weights, new_vocab_path, 6, self.get_temp_dir(), prev_vocab_path) - sess.run(variables.global_variables_initializer()) + self.evaluate(variables.global_variables_initializer()) self.assertTrue( isinstance(fruit_weights, variables.PartitionedVariable)) fruit_weights_vars = fruit_weights._get_variable_list() @@ -438,7 +438,7 @@ class WarmStartingUtilTest(test.TestCase): prev_ckpt=self.get_temp_dir(), prev_vocab_path=prev_vocab_path, axis=1) - sess.run(variables.global_variables_initializer()) + self.evaluate(variables.global_variables_initializer()) self.assertTrue( isinstance(fruit_output_layer, variables.PartitionedVariable)) fruit_output_layer_vars = fruit_output_layer._get_variable_list() @@ -463,7 +463,7 @@ class WarmStartingUtilTest(test.TestCase): shape=[10, 1], initializer=zeros()) ws_util.warm_start(self.get_temp_dir(), vars_to_warm_start=[var]) - sess.run(variables.global_variables_initializer()) + self.evaluate(variables.global_variables_initializer()) # Verify weights were correctly warm-started (init overridden to ones). self.assertAllEqual(var.eval(), prev_int_val) @@ -483,7 +483,7 @@ class WarmStartingUtilTest(test.TestCase): shape=[10, 1], initializer=zeros()) ws_util.warm_start(self.get_temp_dir(), vars_to_warm_start=["v1"]) - sess.run(variables.global_variables_initializer()) + self.evaluate(variables.global_variables_initializer()) # Verify weights were correctly warm-started (init overridden to ones). self.assertAllEqual(var.eval(), prev_int_val) @@ -519,7 +519,7 @@ class WarmStartingUtilTest(test.TestCase): # This warm-starts both v1 and v1/Momentum, but only # v2 (and not v2/Momentum). vars_to_warm_start=["v1", "v2[^/]"]) - sess.run(variables.global_variables_initializer()) + self.evaluate(variables.global_variables_initializer()) # Verify the selection of weights were correctly warm-started (init # overridden to ones). self.assertAllEqual(v1.eval(), prev_v1_val) @@ -542,7 +542,7 @@ class WarmStartingUtilTest(test.TestCase): with ops.Graph().as_default() as g: with self.session(graph=g) as sess: cols_to_vars = self._create_linear_model([sc_int], partitioner) - sess.run(variables.global_variables_initializer()) + self.evaluate(variables.global_variables_initializer()) # Without warm-starting, the weights should be initialized using default # initializer (which is init_ops.zeros_initializer). self._assert_cols_to_vars(cols_to_vars, {sc_int: [np.zeros([10, 1])]}, @@ -553,7 +553,7 @@ class WarmStartingUtilTest(test.TestCase): with self.session(graph=g) as sess: cols_to_vars = self._create_linear_model([sc_int], partitioner) ws_util.warm_start(self.get_temp_dir(), vars_to_warm_start=".*sc_int.*") - sess.run(variables.global_variables_initializer()) + self.evaluate(variables.global_variables_initializer()) # Verify weights were correctly warm-started. self._assert_cols_to_vars(cols_to_vars, {sc_int: [prev_int_val]}, sess) @@ -571,7 +571,7 @@ class WarmStartingUtilTest(test.TestCase): with ops.Graph().as_default() as g: with self.session(graph=g) as sess: cols_to_vars = self._create_linear_model([sc_hash], partitioner) - sess.run(variables.global_variables_initializer()) + self.evaluate(variables.global_variables_initializer()) # Without warm-starting, the weights should be initialized using default # initializer (which is init_ops.zeros_initializer). self._assert_cols_to_vars(cols_to_vars, {sc_hash: [np.zeros([15, 1])]}, @@ -583,7 +583,7 @@ class WarmStartingUtilTest(test.TestCase): cols_to_vars = self._create_linear_model([sc_hash], partitioner) ws_util.warm_start( self.get_temp_dir(), vars_to_warm_start=".*sc_hash.*") - sess.run(variables.global_variables_initializer()) + self.evaluate(variables.global_variables_initializer()) # Verify weights were correctly warm-started. self._assert_cols_to_vars(cols_to_vars, {sc_hash: [prev_hash_val]}, sess) @@ -605,7 +605,7 @@ class WarmStartingUtilTest(test.TestCase): with ops.Graph().as_default() as g: with self.session(graph=g) as sess: cols_to_vars = self._create_linear_model([sc_vocab], partitioner) - sess.run(variables.global_variables_initializer()) + self.evaluate(variables.global_variables_initializer()) # Without warm-starting, the weights should be initialized using default # initializer (which is init_ops.zeros_initializer). self._assert_cols_to_vars(cols_to_vars, {sc_vocab: [np.zeros([4, 1])]}, @@ -619,7 +619,7 @@ class WarmStartingUtilTest(test.TestCase): # vocab is assumed to be same as new vocab. ws_util.warm_start( self.get_temp_dir(), vars_to_warm_start=".*sc_vocab.*") - sess.run(variables.global_variables_initializer()) + self.evaluate(variables.global_variables_initializer()) # Verify weights were correctly warm-started. self._assert_cols_to_vars(cols_to_vars, {sc_vocab: [prev_vocab_val]}, sess) @@ -641,7 +641,7 @@ class WarmStartingUtilTest(test.TestCase): with ops.Graph().as_default() as g: with self.session(graph=g) as sess: cols_to_vars = self._create_linear_model([sc_vocab], partitioner) - sess.run(variables.global_variables_initializer()) + self.evaluate(variables.global_variables_initializer()) # Without warm-starting, the weights should be initialized using default # initializer (which is init_ops.zeros_initializer). self._assert_cols_to_vars(cols_to_vars, {sc_vocab: [np.zeros([4, 1])]}, @@ -657,7 +657,7 @@ class WarmStartingUtilTest(test.TestCase): # Explicitly provide the file prefix instead of just the dir. os.path.join(self.get_temp_dir(), "model-0"), vars_to_warm_start=".*sc_vocab.*") - sess.run(variables.global_variables_initializer()) + self.evaluate(variables.global_variables_initializer()) # Verify weights were correctly warm-started. self._assert_cols_to_vars(cols_to_vars, {sc_vocab: [prev_vocab_val]}, sess) @@ -686,7 +686,7 @@ class WarmStartingUtilTest(test.TestCase): with ops.Graph().as_default() as g: with self.session(graph=g) as sess: cols_to_vars = self._create_linear_model([sc_vocab], partitioner) - sess.run(variables.global_variables_initializer()) + self.evaluate(variables.global_variables_initializer()) # Without warm-starting, the weights should be initialized using default # initializer (which is init_ops.zeros_initializer). self._assert_cols_to_vars(cols_to_vars, {sc_vocab: [np.zeros([2, 1])]}, @@ -708,7 +708,7 @@ class WarmStartingUtilTest(test.TestCase): var_name_to_vocab_info={ "linear_model/sc_vocab/weights": vocab_info }) - sess.run(variables.global_variables_initializer()) + self.evaluate(variables.global_variables_initializer()) # Verify weights were correctly warm-started. 'banana' isn't in the # first two entries of the old vocabulary, so it's newly initialized. self._assert_cols_to_vars(cols_to_vars, {sc_vocab: [[[1], [0]]]}, sess) @@ -729,7 +729,7 @@ class WarmStartingUtilTest(test.TestCase): with ops.Graph().as_default() as g: with self.session(graph=g) as sess: cols_to_vars = self._create_linear_model([real_bucket], partitioner) - sess.run(variables.global_variables_initializer()) + self.evaluate(variables.global_variables_initializer()) # Without warm-starting, the weights should be initialized using default # initializer (which is init_ops.zeros_initializer). self._assert_cols_to_vars(cols_to_vars, @@ -741,7 +741,7 @@ class WarmStartingUtilTest(test.TestCase): cols_to_vars = self._create_linear_model([real_bucket], partitioner) ws_util.warm_start( self.get_temp_dir(), vars_to_warm_start=".*real_bucketized.*") - sess.run(variables.global_variables_initializer()) + self.evaluate(variables.global_variables_initializer()) # Verify weights were correctly warm-started. self._assert_cols_to_vars(cols_to_vars, {real_bucket: [prev_bucket_val]}, sess) @@ -800,7 +800,7 @@ class WarmStartingUtilTest(test.TestCase): with ops.Graph().as_default() as g: with self.session(graph=g) as sess: cols_to_vars = self._create_linear_model(all_linear_cols, partitioner) - sess.run(variables.global_variables_initializer()) + self.evaluate(variables.global_variables_initializer()) # Without warm-starting, all weights should be initialized using default # initializer (which is init_ops.zeros_initializer). self._assert_cols_to_vars(cols_to_vars, { @@ -826,7 +826,7 @@ class WarmStartingUtilTest(test.TestCase): var_name_to_vocab_info={ "linear_model/sc_vocab/weights": vocab_info }) - sess.run(variables.global_variables_initializer()) + self.evaluate(variables.global_variables_initializer()) # Verify weights were correctly warm-started. self._assert_cols_to_vars(cols_to_vars, { sc_int: [prev_int_val], @@ -865,7 +865,7 @@ class WarmStartingUtilTest(test.TestCase): "linear_model/sc_vocab/weights", initializer=[[0.5], [1.], [2.], [3.]]) self._write_checkpoint(sess) - prev_keys_val = sess.run(sc_keys_weights) + prev_keys_val = self.evaluate(sc_keys_weights) def _partitioner(shape, dtype): # pylint:disable=unused-argument # Partition each var into 2 equal slices. @@ -892,7 +892,7 @@ class WarmStartingUtilTest(test.TestCase): ws_util._infer_var_name(cols_to_vars[sc_keys]): "some_other_name" }) - sess.run(variables.global_variables_initializer()) + self.evaluate(variables.global_variables_initializer()) # Verify weights were correctly warm-started. Var corresponding to # sc_hash should not be warm-started. Var corresponding to sc_vocab # should be correctly warm-started after vocab remapping. @@ -933,7 +933,7 @@ class WarmStartingUtilTest(test.TestCase): "linear_model/sc_vocab/weights", initializer=[[0.5], [1.], [2.], [3.]]) self._write_checkpoint(sess) - prev_keys_val = sess.run(sc_keys_weights) + prev_keys_val = self.evaluate(sc_keys_weights) # New graph, new session with warm-starting. with ops.Graph().as_default() as g: @@ -955,7 +955,7 @@ class WarmStartingUtilTest(test.TestCase): ws_util._infer_var_name(cols_to_vars[sc_keys]): "some_other_name" }) - sess.run(variables.global_variables_initializer()) + self.evaluate(variables.global_variables_initializer()) # Verify weights were correctly warm-started. Var corresponding to # sc_hash should not be warm-started. Var corresponding to sc_vocab # should be correctly warm-started after vocab remapping. @@ -1024,7 +1024,7 @@ class WarmStartingUtilTest(test.TestCase): ws_util._infer_var_name(cols_to_vars[sc_keys]): "some_other_name" }) - sess.run(variables.global_variables_initializer()) + self.evaluate(variables.global_variables_initializer()) # Verify weights were correctly warm-started. Var corresponding to # sc_vocab should be correctly warm-started after vocab remapping, # and neither of the other two should be warm-started.. @@ -1091,7 +1091,7 @@ class WarmStartingUtilTest(test.TestCase): ws_util._infer_var_name(cols_to_vars[emb_vocab_column]): vocab_info }) - sess.run(variables.global_variables_initializer()) + self.evaluate(variables.global_variables_initializer()) # Verify weights were correctly warm-started. Var corresponding to # emb_vocab_column should be correctly warm-started after vocab # remapping. Missing values are filled in with the EmbeddingColumn's @@ -1163,7 +1163,7 @@ class WarmStartingUtilTest(test.TestCase): var_name_to_vocab_info={ "linear_model/sc_vocab_embedding/embedding_weights": vocab_info }) - sess.run(variables.global_variables_initializer()) + self.evaluate(variables.global_variables_initializer()) # Verify weights were correctly warm-started. Var corresponding to # emb_vocab should be correctly warm-started after vocab remapping. # Missing values are filled in with the EmbeddingColumn's initializer. diff --git a/tensorflow/python/util/deprecation.py b/tensorflow/python/util/deprecation.py index 4c68d1aaae3..9aaf0c2de97 100644 --- a/tensorflow/python/util/deprecation.py +++ b/tensorflow/python/util/deprecation.py @@ -28,6 +28,7 @@ from tensorflow.python.util import is_in_graph_mode from tensorflow.python.util import tf_contextlib from tensorflow.python.util import tf_decorator from tensorflow.python.util import tf_inspect +from tensorflow.python.util import tf_stack # Allow deprecation warnings to be silenced temporarily with a context manager. @@ -98,21 +99,9 @@ def _validate_deprecation_args(date, instructions): def _call_location(outer=False): """Returns call location given level up from current call.""" - frame = tf_inspect.currentframe() - if frame: - # CPython internals are available, use them for performance. - # walk back two frames to get to deprecated function caller. - frame = frame.f_back - if frame.f_back: - frame = frame.f_back - if outer and frame.f_back: - frame = frame.f_back - return '%s:%d' % (frame.f_code.co_filename, frame.f_lineno) - else: - # Slow fallback path - stack = tf_inspect.stack(0) # 0 avoids generating unused context - entry = stack[3 if outer else 2] - return '%s:%d' % (entry[1], entry[2]) + stack = tf_stack.extract_stack() + frame = stack[-4 if outer else -3] + return '{filename}:{lineno}'.format(filename=frame[0], lineno=frame[1]) def _wrap_decorator(wrapped_function): diff --git a/tensorflow/python/util/deprecation_test.py b/tensorflow/python/util/deprecation_test.py index 34cbca52a1b..035c416d793 100644 --- a/tensorflow/python/util/deprecation_test.py +++ b/tensorflow/python/util/deprecation_test.py @@ -19,6 +19,7 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function +from tensorflow.python.framework import test_util from tensorflow.python.platform import test from tensorflow.python.platform import tf_logging as logging from tensorflow.python.util import deprecation @@ -174,6 +175,7 @@ class DeprecationTest(test.TestCase): set(args[1:])) @test.mock.patch.object(logging, "warning", autospec=True) + @test_util.run_deprecated_v1 def test_static_fn_with_doc(self, mock_warning): date = "2016-07-04" instructions = "This is how you update..." @@ -214,6 +216,7 @@ class DeprecationTest(test.TestCase): self._assert_subset(set(["after " + date, instructions]), set(args[1:])) @test.mock.patch.object(logging, "warning", autospec=True) + @test_util.run_deprecated_v1 def test_static_fn_with_one_line_doc(self, mock_warning): date = "2016-07-04" instructions = "This is how you update..." @@ -239,6 +242,7 @@ class DeprecationTest(test.TestCase): self._assert_subset(set(["after " + date, instructions]), set(args[1:])) @test.mock.patch.object(logging, "warning", autospec=True) + @test_util.run_deprecated_v1 def test_static_fn_no_doc(self, mock_warning): date = "2016-07-04" instructions = "This is how you update..." @@ -488,6 +492,7 @@ class DeprecatedArgsTest(test.TestCase): deprecation.deprecated_args(date, instructions, "missing")(_fn) @test.mock.patch.object(logging, "warning", autospec=True) + @test_util.run_deprecated_v1 def test_static_fn_with_doc(self, mock_warning): date = "2016-07-04" instructions = "This is how you update..." @@ -535,6 +540,7 @@ class DeprecatedArgsTest(test.TestCase): self._assert_subset(set(["after " + date, instructions]), set(args[1:])) @test.mock.patch.object(logging, "warning", autospec=True) + @test_util.run_deprecated_v1 def test_static_fn_with_one_line_doc(self, mock_warning): date = "2016-07-04" instructions = "This is how you update..." @@ -565,6 +571,7 @@ class DeprecatedArgsTest(test.TestCase): self._assert_subset(set(["after " + date, instructions]), set(args[1:])) @test.mock.patch.object(logging, "warning", autospec=True) + @test_util.run_deprecated_v1 def test_static_fn_no_doc(self, mock_warning): date = "2016-07-04" instructions = "This is how you update..." @@ -595,6 +602,7 @@ class DeprecatedArgsTest(test.TestCase): self._assert_subset(set(["after " + date, instructions]), set(args[1:])) @test.mock.patch.object(logging, "warning", autospec=True) + @test_util.run_deprecated_v1 def test_varargs(self, mock_warning): date = "2016-07-04" instructions = "This is how you update..." @@ -615,6 +623,7 @@ class DeprecatedArgsTest(test.TestCase): self._assert_subset(set(["after " + date, instructions]), set(args[1:])) @test.mock.patch.object(logging, "warning", autospec=True) + @test_util.run_deprecated_v1 def test_kwargs(self, mock_warning): date = "2016-07-04" instructions = "This is how you update..." @@ -635,6 +644,7 @@ class DeprecatedArgsTest(test.TestCase): self._assert_subset(set(["after " + date, instructions]), set(args[1:])) @test.mock.patch.object(logging, "warning", autospec=True) + @test_util.run_deprecated_v1 def test_positional_and_named(self, mock_warning): date = "2016-07-04" instructions = "This is how you update..." @@ -660,6 +670,7 @@ class DeprecatedArgsTest(test.TestCase): set(args2[1:])) @test.mock.patch.object(logging, "warning", autospec=True) + @test_util.run_deprecated_v1 def test_positional_and_named_with_ok_vals(self, mock_warning): date = "2016-07-04" instructions = "This is how you update..." @@ -692,6 +703,7 @@ class DeprecatedArgsTest(test.TestCase): self.assertEqual(0, mock_warning.call_count) @test.mock.patch.object(logging, "warning", autospec=True) + @test_util.run_deprecated_v1 def test_deprecated_args_once(self, mock_warning): date = "2016-07-04" instructions = "This is how you update..." @@ -708,6 +720,7 @@ class DeprecatedArgsTest(test.TestCase): self.assertEqual(1, mock_warning.call_count) @test.mock.patch.object(logging, "warning", autospec=True) + @test_util.run_deprecated_v1 def test_deprecated_multiple_args_once_each(self, mock_warning): date = "2016-07-04" instructions = "This is how you update..." @@ -752,6 +765,7 @@ class DeprecatedArgValuesTest(test.TestCase): deprecation.deprecated_arg_values(date, instructions) @test.mock.patch.object(logging, "warning", autospec=True) + @test_util.run_deprecated_v1 def test_static_fn_with_doc(self, mock_warning): date = "2016-07-04" instructions = "This is how you update..." @@ -804,6 +818,7 @@ class DeprecatedArgValuesTest(test.TestCase): self.assertEqual(2, mock_warning.call_count) @test.mock.patch.object(logging, "warning", autospec=True) + @test_util.run_deprecated_v1 def test_static_fn_with_one_line_doc(self, mock_warning): date = "2016-07-04" instructions = "This is how you update..." @@ -839,6 +854,7 @@ class DeprecatedArgValuesTest(test.TestCase): self.assertEqual(2, mock_warning.call_count) @test.mock.patch.object(logging, "warning", autospec=True) + @test_util.run_deprecated_v1 def test_static_fn_no_doc(self, mock_warning): date = "2016-07-04" instructions = "This is how you update..." diff --git a/tensorflow/python/util/dispatch.py b/tensorflow/python/util/dispatch.py new file mode 100644 index 00000000000..e7a56b5922c --- /dev/null +++ b/tensorflow/python/util/dispatch.py @@ -0,0 +1,192 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Type-based dispatch for TensorFlow ops. + +"Operation dispatchers" can be used to override the behavior for TensorFlow ops +when they are called with otherwise unsupported argument types. In particular, +when an operation is called with arguments that would cause it to raise a +TypeError, it falls back on its registered operation dispatchers. If any +registered dispatchers can handle the arguments, then its result is returned. +Otherwise, the original TypeError is raised. + +By default, dispatch support is added to the generated op wrappers for any +visible ops by default. Ops that are implemented in Python can opt in to +dispatch support using the `add_dispatch_support` decorator. +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import itertools + +from tensorflow.python.util import tf_decorator +from tensorflow.python.util import tf_inspect + +# Private function attribute used to store a list of dispatchers. +DISPATCH_ATTR = "_tf_dispatchers" + + +class OpDispatcher(object): + """Abstract base class for TensorFlow operator dispatchers. + + Each operation dispatcher acts as an override handler for a single + TensorFlow operation, and its results are used when the handler indicates + that it can handle the operation's arguments (by returning any value other + than `OpDispatcher.NOT_SUPPORTED`). + """ + + # Sentinel value that can be returned to indicate that an operation + # dispatcher does not support a given set of arguments. + NOT_SUPPORTED = object() + + def handle(self, args, kwargs): # pylint: disable=unused-argument + """Handle this dispatcher's operation with the specified arguments. + + If this operation dispatcher can handle the given arguments, then + return an appropriate value (or raise an appropriate exception). + + Args: + args: The arguments to the operation. + kwargs: They keyword arguments to the operation. + + Returns: + The result of the operation, or `OpDispatcher.NOT_SUPPORTED` if this + dispatcher can not handle the given arguments. + """ + return self.NOT_SUPPORTED + + def register(self, op): + """Register this dispatcher as a handler for `op`. + + Args: + op: Python function: the TensorFlow operation that should be handled. Must + have a dispatch list (which is added automatically for generated ops, + and can be added to Python ops using the `add_dispatch_support` + decorator). + """ + if not hasattr(op, DISPATCH_ATTR): + raise AssertionError("Dispatching not enabled for %s" % op) + getattr(op, DISPATCH_ATTR).append(self) + + +def dispatch(op, *args, **kwargs): + """Returns the result from the first successful dispatcher for a given op. + + Calls the `handle` method of each `OpDispatcher` that has been registered + to handle `op`, and returns the value from the first successful handler. + + Args: + op: Python function: the operation to dispatch for. + *args: The arguments to the operation. + **kwargs: They keyword arguments to the operation. + + Returns: + The result of the operation, or `NOT_SUPPORTED` if no registered + dispatcher can handle the given arguments. + """ + for dispatcher in getattr(op, DISPATCH_ATTR): + result = dispatcher.handle(args, kwargs) + if result is not OpDispatcher.NOT_SUPPORTED: + return result + return OpDispatcher.NOT_SUPPORTED + + +class _TypeBasedDispatcher(OpDispatcher): + """Dispatcher that handles op if any arguments have a specified type. + + Checks the types of the arguments and keyword arguments (including elements + of lists or tuples), and if any argument values have the indicated type(s), + then delegates to an override function. + """ + + def __init__(self, override_func, types): + self._types = types + self._override_func = override_func + + def _handles(self, args, kwargs): + for arg in itertools.chain(args, kwargs.values()): + if (isinstance(arg, self._types) or + (isinstance(arg, (list, tuple)) and + any(isinstance(elt, self._types) for elt in arg))): + return True + return False + + def handle(self, args, kwargs): + if self._handles(args, kwargs): + return self._override_func(*args, **kwargs) + else: + return self.NOT_SUPPORTED + + +# pylint: disable=g-doc-return-or-yield +def dispatch_for_types(op, *types): + """Decorator to declare that a Python function overrides an op for a type. + + The decorated function is used to override `op` if any of the arguments or + keyword arguments (including elements of lists or tuples) have one of the + specified types. + + Example: + + ```python + @dispatch_for_types(math_ops.add, RaggedTensor, RaggedTensorValue) + def ragged_add(x, y, name=None): ... + ``` + + Args: + op: Python function: the operation that should be overridden. + *types: The argument types for which this function should be used. + """ + + def decorator(func): + if tf_inspect.getargspec(func) != tf_inspect.getargspec(op): + raise AssertionError("The decorated function's signature must exactly " + "match the signature of the overridden op.") + _TypeBasedDispatcher(func, types).register(op) + return func + + return decorator + + +# pylint: enable=g-doc-return-or-yield + + +def add_dispatch_list(target): + """Decorator that adds a dispatch_list attribute to an op.""" + assert not hasattr(target, DISPATCH_ATTR) + setattr(target, DISPATCH_ATTR, []) + return target + + +def add_dispatch_support(target): + """Decorator that adds a dispatch handling wrapper to an op.""" + add_dispatch_list(target) + + def wrapper(*args, **kwargs): + """Call target, and fall back on dispatchers if there is a TypeError.""" + try: + return target(*args, **kwargs) + except (TypeError, ValueError): + # Note: convert_to_eager_tensor currently raises a ValueError, not a + # TypeError, when given unexpected types. So we need to catch both. + result = dispatch(wrapper, *args, **kwargs) + if result is not OpDispatcher.NOT_SUPPORTED: + return result + else: + raise + + setattr(wrapper, DISPATCH_ATTR, []) + return tf_decorator.make_decorator(target, wrapper) diff --git a/tensorflow/python/util/dispatch_test.py b/tensorflow/python/util/dispatch_test.py new file mode 100644 index 00000000000..b7c5c8eca8d --- /dev/null +++ b/tensorflow/python/util/dispatch_test.py @@ -0,0 +1,120 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Tests for operator dispatch.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from tensorflow.python.framework import ops +from tensorflow.python.framework import test_util +from tensorflow.python.ops import gen_math_ops +from tensorflow.python.ops import math_ops +from tensorflow.python.platform import googletest +from tensorflow.python.util import dispatch +from tensorflow.python.util.tf_export import tf_export + + +class CustomTensor(object): + """A fake composite tensor class, for testing type-based dispatching.""" + + def __init__(self, tensor, score): + self.tensor = ops.convert_to_tensor(tensor) + self.score = score + + +@tf_export("test_op") +@dispatch.add_dispatch_support +def test_op(x, y, z): + """A fake op for testing dispatch of Python ops.""" + return x + (2 * y) + (3 * z) + + +@test_util.run_all_in_graph_and_eager_modes +class DispatchTest(test_util.TensorFlowTestCase): + + def testAddDispatchForTypes_With_CppOp(self): + original_handlers = gen_math_ops.add._tf_dispatchers[:] + + # Override the behavior of gen_math_ops.add. + @dispatch.dispatch_for_types(gen_math_ops.add, CustomTensor) + def custom_add(x, y, name=None): # pylint: disable=unused-variable + return CustomTensor(gen_math_ops.add(x.tensor, y.tensor, name), + (x.score+y.score) / 2.0) + self.assertEqual(len(math_ops.add._tf_dispatchers), + len(original_handlers) + 1) + + # Test that we see the overridden behavior when using CustomTensors. + x = CustomTensor([1, 2, 3], 2.0) + y = CustomTensor([7, 8, 2], 0.0) + x_plus_y = gen_math_ops.add(x, y) + self.assertAllEqual(self.evaluate(x_plus_y.tensor), [8, 10, 5]) + self.assertNear(x_plus_y.score, 1.0, 0.001) + + # Test that we still get the right behavior when using normal Tensors. + a = [1, 2, 3] + b = [4, 5, 6] + a_plus_b = gen_math_ops.add(a, b) + self.assertAllEqual(a_plus_b, [5, 7, 9]) + + # Test that we still get a TypeError or ValueError if we pass some + # type that's not supported by any dispatcher. + with self.assertRaises((TypeError, ValueError)): + gen_math_ops.add(a, None) + + # Clean up + gen_math_ops.add._tf_dispatchers = original_handlers + + def testAddDispatchForTypes_With_PythonOp(self): + original_handlers = test_op._tf_dispatchers[:] + + @dispatch.dispatch_for_types(test_op, CustomTensor) + def override_for_test_op(x, y, z): # pylint: disable=unused-variable + return CustomTensor(test_op(x.tensor, y.tensor, z.tensor), + (x.score + y.score + z.score) / 3.0) + + x = CustomTensor([1, 2, 3], 0.2) + y = CustomTensor([7, 8, 2], 0.4) + z = CustomTensor([0, 1, 2], 0.6) + + result = test_op(x, y, z) + self.assertAllEqual(self.evaluate(result.tensor), [15, 21, 13]) + self.assertNear(result.score, 0.4, 0.001) + + # Clean up + test_op._tf_dispatchers = original_handlers + + def testDispatchForTypes_SignatureMismatch(self): + with self.assertRaisesRegexp(AssertionError, "The decorated function's " + "signature must exactly match.*"): + @dispatch.dispatch_for_types(test_op, CustomTensor) + def override_for_test_op(a, b, c): # pylint: disable=unused-variable + return CustomTensor(test_op(a.tensor, b.tensor, c.tensor), + (a.score + b.score + c.score) / 3.0) + + def testDispatchForTypes_OpDoesNotSupportDispatch(self): + def some_op(x, y): + return x + y + + with self.assertRaisesRegexp(AssertionError, "Dispatching not enabled for"): + @dispatch.dispatch_for_types(some_op, CustomTensor) + def override_for_some_op(x, y): # pylint: disable=unused-variable + return x if x.score > 0 else y + + +if __name__ == "__main__": + googletest.main() + + diff --git a/tensorflow/python/util/nest_test.py b/tensorflow/python/util/nest_test.py index 997a3c5c36f..d0d0c5f7935 100644 --- a/tensorflow/python/util/nest_test.py +++ b/tensorflow/python/util/nest_test.py @@ -482,6 +482,7 @@ class NestTest(parameterized.TestCase, test.TestCase): self.assertEqual(nt.a[1][::-1], rev_nt.a[1]) self.assertEqual(nt.b[::-1], rev_nt.b) + @test_util.run_deprecated_v1 def testMapStructureOverPlaceholders(self): inp_a = (array_ops.placeholder(dtypes.float32, shape=[3, 4]), array_ops.placeholder(dtypes.float32, shape=[3, 7])) diff --git a/tensorflow/python/util/py_checkpoint_reader.i b/tensorflow/python/util/py_checkpoint_reader.i index 1c73f7f06f1..a1b98a2a759 100644 --- a/tensorflow/python/util/py_checkpoint_reader.i +++ b/tensorflow/python/util/py_checkpoint_reader.i @@ -165,7 +165,6 @@ def NewCheckpointReader(filepattern): from tensorflow.python.util import compat return CheckpointReader(compat.as_bytes(filepattern), status) -NewCheckpointReader._tf_api_names = ['train.NewCheckpointReader'] NewCheckpointReader._tf_api_names_v1 = ['train.NewCheckpointReader'] %} diff --git a/tensorflow/python/util/tf_export.py b/tensorflow/python/util/tf_export.py index 0924b36ade8..ec70cae7d2f 100644 --- a/tensorflow/python/util/tf_export.py +++ b/tensorflow/python/util/tf_export.py @@ -50,6 +50,10 @@ from tensorflow.python.util import tf_decorator ESTIMATOR_API_NAME = 'estimator' TENSORFLOW_API_NAME = 'tensorflow' +# List of subpackage names used by TensorFlow components. Have to check that +# TensorFlow core repo does not export any symbols under these names. +SUBPACKAGE_NAMESPACES = [ESTIMATOR_API_NAME] + _Attributes = collections.namedtuple( 'ExportedApiAttributes', ['names', 'constants']) @@ -78,6 +82,11 @@ class SymbolAlreadyExposedError(Exception): pass +class InvalidSymbolNameError(Exception): + """Raised when trying to export symbol as an invalid or unallowed name.""" + pass + + def get_canonical_name_for_symbol( symbol, api_name=TENSORFLOW_API_NAME, add_prefix_to_v1_names=False): @@ -163,6 +172,37 @@ class api_export(object): # pylint: disable=invalid-name self._overrides = kwargs.get('overrides', []) self._allow_multiple_exports = kwargs.get('allow_multiple_exports', False) + self._validate_symbol_names() + + def _validate_symbol_names(self): + """Validate you are exporting symbols under an allowed package. + + We need to ensure things exported by tf_export, estimator_export, etc. + export symbols under disjoint top-level package names. + + For TensorFlow, we check that it does not export anything under subpackage + names used by components (estimator, keras, etc.). + + For each component, we check that it exports everything under its own + subpackage. + + Raises: + InvalidSymbolNameError: If you try to export symbol under disallowed name. + """ + all_symbol_names = set(self._names) | set(self._names_v1) + if self._api_name == TENSORFLOW_API_NAME: + for subpackage in SUBPACKAGE_NAMESPACES: + if any(n.startswith(subpackage) for n in all_symbol_names): + raise InvalidSymbolNameError( + '@tf_export is not allowed to export symbols under %s.*' % ( + subpackage)) + else: + if not all(n.startswith(self._api_name) for n in all_symbol_names): + raise InvalidSymbolNameError( + 'Can only export symbols under package name of component. ' + 'e.g. tensorflow_estimator must export all symbols under ' + 'tf.estimator') + def __call__(self, func): """Calls this decorator. diff --git a/tensorflow/python/util/tf_export_test.py b/tensorflow/python/util/tf_export_test.py index 4ae1dc55e06..a0fac8bf362 100644 --- a/tensorflow/python/util/tf_export_test.py +++ b/tensorflow/python/util/tf_export_test.py @@ -130,6 +130,26 @@ class ValidateExportTest(test.TestCase): with self.assertRaises(tf_export.SymbolAlreadyExposedError): export_decorator(_test_function) + def testRaisesExceptionIfInvalidSymbolName(self): + # TensorFlow code is not allowed to export symbols under package + # tf.estimator + with self.assertRaises(tf_export.InvalidSymbolNameError): + tf_export.tf_export('estimator.invalid') + + # All symbols exported by Estimator must be under tf.estimator package. + with self.assertRaises(tf_export.InvalidSymbolNameError): + tf_export.estimator_export('invalid') + with self.assertRaises(tf_export.InvalidSymbolNameError): + tf_export.estimator_export('Estimator.invalid') + with self.assertRaises(tf_export.InvalidSymbolNameError): + tf_export.estimator_export('invalid.estimator') + + def testRaisesExceptionIfInvalidV1SymbolName(self): + with self.assertRaises(tf_export.InvalidSymbolNameError): + tf_export.tf_export('valid', v1=['estimator.invalid']) + with self.assertRaises(tf_export.InvalidSymbolNameError): + tf_export.estimator_export('estimator.valid', v1=['invalid']) + def testOverridesFunction(self): _test_function2._tf_api_names = ['abc'] diff --git a/tensorflow/python/util/tf_should_use_test.py b/tensorflow/python/util/tf_should_use_test.py index fedbe1dff6a..65d848cf2a5 100644 --- a/tensorflow/python/util/tf_should_use_test.py +++ b/tensorflow/python/util/tf_should_use_test.py @@ -24,6 +24,7 @@ import gc import sys from tensorflow.python.framework import constant_op +from tensorflow.python.framework import test_util from tensorflow.python.platform import test from tensorflow.python.platform import tf_logging from tensorflow.python.util import tf_should_use @@ -39,6 +40,7 @@ def reroute_error(): class TfShouldUseTest(test.TestCase): + @test_util.run_deprecated_v1 def testAddShouldUseWarningWhenNotUsed(self): c = constant_op.constant(0, name='blah0') def in_this_function(): @@ -52,6 +54,7 @@ class TfShouldUseTest(test.TestCase): self.assertIn('in_this_function', msg) self.assertFalse(gc.garbage) + @test_util.run_deprecated_v1 def testAddShouldUseFatalWhenNotUsed(self): c = constant_op.constant(0, name='blah0') def in_this_function(): @@ -74,6 +77,7 @@ class TfShouldUseTest(test.TestCase): error.assert_not_called() fatal.assert_not_called() + @test_util.run_deprecated_v1 def testAddShouldUseWarningWhenUsedWithAdd(self): def add(h): _ = h + 1 @@ -81,6 +85,7 @@ class TfShouldUseTest(test.TestCase): gc.collect() self.assertFalse(gc.garbage) + @test_util.run_deprecated_v1 def testAddShouldUseWarningWhenUsedWithGetName(self): def get_name(h): _ = h.name @@ -88,6 +93,7 @@ class TfShouldUseTest(test.TestCase): gc.collect() self.assertFalse(gc.garbage) + @test_util.run_deprecated_v1 def testShouldUseResult(self): @tf_should_use.should_use_result def return_const(value): @@ -101,6 +107,7 @@ class TfShouldUseTest(test.TestCase): gc.collect() self.assertFalse(gc.garbage) + @test_util.run_deprecated_v1 def testShouldUseResultWhenNotReallyUsed(self): @tf_should_use.should_use_result def return_const(value): @@ -111,7 +118,7 @@ class TfShouldUseTest(test.TestCase): # Creating another op and executing it does not mark the # unused op as being "used". v = constant_op.constant(1.0, name='meh') - v.eval() + self.evaluate(v) msg = '\n'.join(error.call_args[0]) self.assertIn('Object was never used', msg) self.assertIn('blah3:0', msg) diff --git a/tensorflow/stream_executor/BUILD b/tensorflow/stream_executor/BUILD index 5c9d85acf4e..4c764a7b099 100644 --- a/tensorflow/stream_executor/BUILD +++ b/tensorflow/stream_executor/BUILD @@ -1,6 +1,8 @@ licenses(["restricted"]) load("@local_config_cuda//cuda:build_defs.bzl", "if_cuda_is_configured") +load("//tensorflow/core:platform/default/build_config.bzl", "tf_proto_library") +load("//tensorflow/core:platform/default/build_config.bzl", "tf_additional_all_protos") load("//tensorflow/core:platform/default/build_config_root.bzl", "if_static") load("//tensorflow:tensorflow.bzl", "cc_header_only_library") @@ -13,6 +15,14 @@ STREAM_EXECUTOR_HEADERS = glob([ "platform/**/*.h", ]) +tf_proto_library( + name = "dnn_proto", + srcs = ["dnn.proto"], + cc_api_version = 2, + default_header = True, + protodeps = tf_additional_all_protos(), +) + cc_library( name = "stream_executor_impl", srcs = glob( @@ -35,6 +45,7 @@ cc_library( }), visibility = ["//visibility:public"], deps = [ + ":dnn_proto_cc_impl", "//tensorflow/core:lib", "//tensorflow/core:ptr_util", "@com_google_absl//absl/container:flat_hash_map", @@ -51,6 +62,7 @@ cc_library( hdrs = STREAM_EXECUTOR_HEADERS, visibility = ["//visibility:public"], deps = [ + ":dnn_proto_cc", "//tensorflow/core:lib", "//tensorflow/core:ptr_util", "@com_google_absl//absl/strings", @@ -96,11 +108,8 @@ cc_library( "@local_config_cuda//cuda:cuda_headers", ] + if_cuda_is_configured([ "//tensorflow/core:cuda", - "@local_config_cuda//cuda:cublas", "@local_config_cuda//cuda:cuda_driver", "@local_config_cuda//cuda:cudnn", - "@local_config_cuda//cuda:cufft", - "@local_config_cuda//cuda:curand", ]), alwayslink = 1, ) diff --git a/tensorflow/stream_executor/cuda/cuda_blas.cc b/tensorflow/stream_executor/cuda/cuda_blas.cc index 7fabb35e28c..957f6c98da5 100644 --- a/tensorflow/stream_executor/cuda/cuda_blas.cc +++ b/tensorflow/stream_executor/cuda/cuda_blas.cc @@ -58,6 +58,11 @@ limitations under the License. #include "tensorflow/stream_executor/cuda/cuda_stream.h" #include "tensorflow/stream_executor/cuda/cuda_timer.h" #include "tensorflow/stream_executor/device_memory.h" + +#ifndef PLATFORM_GOOGLE +#include "tensorflow/stream_executor/dso_loader.h" +#endif + #include "tensorflow/stream_executor/lib/env.h" #include "tensorflow/stream_executor/lib/initialize.h" #include "tensorflow/stream_executor/lib/status.h" @@ -76,21 +81,8 @@ PLUGIN_REGISTRY_DEFINE_PLUGIN_ID(kCuBlasPlugin); namespace wrap { -#define STREAM_EXECUTOR_CUBLAS_WRAP(__name) \ - struct WrapperShim__##__name { \ - static const char *kName; \ - template \ - cublasStatus_t operator()(CUDAExecutor *parent, Args... args) { \ - cuda::ScopedActivateExecutorContext sac{parent}; \ - return ::__name(args...); \ - } \ - } __name; \ - const char *WrapperShim__##__name::kName = #__name; - -#define STREAM_EXECUTOR_CUBLAS_V2_WRAP(__name) \ - STREAM_EXECUTOR_CUBLAS_WRAP(__name) - -#define CUBLAS_BLAS_ROUTINE_EACH(__macro) \ +// clang-format off +#define CUBLAS_ROUTINE_EACH(__macro) \ __macro(cublasSnrm2) \ __macro(cublasDnrm2) \ __macro(cublasScnrm2) \ @@ -262,6 +254,58 @@ namespace wrap { __macro(cublasCdgmm) \ __macro(cublasZdgmm) +// clang-format off + +#ifdef PLATFORM_GOOGLE +#define STREAM_EXECUTOR_CUBLAS_WRAP(__name) \ + struct WrapperShim__##__name { \ + static const char *kName; \ + template \ + cublasStatus_t operator()(CUDAExecutor *parent, Args... args) { \ + cuda::ScopedActivateExecutorContext sac{parent}; \ + return ::__name(args...); \ + } \ + } __name; \ + const char *WrapperShim__##__name::kName = #__name; + +#define STREAM_EXECUTOR_CUBLAS_V2_WRAP(__name) \ + STREAM_EXECUTOR_CUBLAS_WRAP(__name) + +#else + +#define STREAM_EXECUTOR_CUBLAS_WRAP(__name) \ + struct DynLoadShim__##__name { \ + static const char* kName; \ + using FuncPtrT = std::add_pointer::type; \ + static void* GetDsoHandle() { \ + auto s = internal::CachedDsoLoader::GetCublasDsoHandle(); \ + return s.ValueOrDie(); \ + } \ + static FuncPtrT LoadOrDie() { \ + void* f; \ + auto s = port::Env::Default()->GetSymbolFromLibrary(GetDsoHandle(), \ + kName, &f); \ + CHECK(s.ok()) << "could not find " << kName \ + << " in cublas DSO; dlerror: " << s.error_message(); \ + return reinterpret_cast(f); \ + } \ + static FuncPtrT DynLoad() { \ + static FuncPtrT f = LoadOrDie(); \ + return f; \ + } \ + template \ + cublasStatus_t operator()(CUDAExecutor* parent, Args... args) { \ + cuda::ScopedActivateExecutorContext sac{parent}; \ + return DynLoad()(args...); \ + } \ + } __name; \ + const char* DynLoadShim__##__name::kName = #__name; + +#define STREAM_EXECUTOR_CUBLAS_V2_WRAP(__name) \ + STREAM_EXECUTOR_CUBLAS_WRAP(__name) + +#endif + STREAM_EXECUTOR_CUBLAS_V2_WRAP(cublasCreate) STREAM_EXECUTOR_CUBLAS_V2_WRAP(cublasDestroy) STREAM_EXECUTOR_CUBLAS_V2_WRAP(cublasSetStream) @@ -271,7 +315,7 @@ STREAM_EXECUTOR_CUBLAS_WRAP(cublasSgemmBatched) STREAM_EXECUTOR_CUBLAS_WRAP(cublasDgemmBatched) STREAM_EXECUTOR_CUBLAS_WRAP(cublasCgemmBatched) STREAM_EXECUTOR_CUBLAS_WRAP(cublasZgemmBatched) -CUBLAS_BLAS_ROUTINE_EACH(STREAM_EXECUTOR_CUBLAS_V2_WRAP) +CUBLAS_ROUTINE_EACH(STREAM_EXECUTOR_CUBLAS_V2_WRAP) #if CUDA_VERSION >= 7050 STREAM_EXECUTOR_CUBLAS_WRAP(cublasSgemmEx) @@ -424,7 +468,8 @@ class ScopedCublasMathMode { // Note that when false is returned, an appropriate error has already been // logged. bool Init(cublasMath_t new_mode) { - cublasStatus_t ret = wrap::cublasGetMathMode(parent_, handle_, &old_mode_); + cublasStatus_t ret = + wrap::cublasGetMathMode(parent_, handle_, &old_mode_); if (ret != CUBLAS_STATUS_SUCCESS) { LOG(ERROR) << "failed to get old cublas math mode: " << ToString(ret); return ok_ = false; @@ -442,7 +487,8 @@ class ScopedCublasMathMode { // successful in the first place. ~ScopedCublasMathMode() { if (ok_) { - cublasStatus_t ret = wrap::cublasSetMathMode(parent_, handle_, old_mode_); + cublasStatus_t ret = + wrap::cublasSetMathMode(parent_, handle_, old_mode_); if (ret != CUBLAS_STATUS_SUCCESS) { LOG(ERROR) << "failed to set former cublas math mode: " << ToString(ret); @@ -675,16 +721,16 @@ bool CUDABlas::DoBlasAsum(Stream *stream, uint64 elem_count, const DeviceMemory> &x, int incx, DeviceMemory *result) { return DoBlasInternal( - wrap::cublasScasum, stream, false /* = pointer_mode_host */, elem_count, - CUDAComplex(CUDAMemory(x)), incx, CUDAMemoryMutable(result)); + wrap::cublasScasum, stream, false /* = pointer_mode_host */, + elem_count, CUDAComplex(CUDAMemory(x)), incx, CUDAMemoryMutable(result)); } bool CUDABlas::DoBlasAsum(Stream *stream, uint64 elem_count, const DeviceMemory> &x, int incx, DeviceMemory *result) { return DoBlasInternal( - wrap::cublasDzasum, stream, false /* = pointer_mode_host */, elem_count, - CUDAComplex(CUDAMemory(x)), incx, CUDAMemoryMutable(result)); + wrap::cublasDzasum, stream, false /* = pointer_mode_host */, + elem_count, CUDAComplex(CUDAMemory(x)), incx, CUDAMemoryMutable(result)); } bool CUDABlas::DoBlasAxpy(Stream *stream, uint64 elem_count, float alpha, @@ -835,16 +881,16 @@ bool CUDABlas::DoBlasNrm2(Stream *stream, uint64 elem_count, const DeviceMemory> &x, int incx, DeviceMemory *result) { return DoBlasInternal( - wrap::cublasScnrm2, stream, false /* = pointer_mode_host */, elem_count, - CUDAComplex(CUDAMemory(x)), incx, CUDAMemoryMutable(result)); + wrap::cublasScnrm2, stream, false /* = pointer_mode_host */, + elem_count, CUDAComplex(CUDAMemory(x)), incx, CUDAMemoryMutable(result)); } bool CUDABlas::DoBlasNrm2(Stream *stream, uint64 elem_count, const DeviceMemory> &x, int incx, DeviceMemory *result) { return DoBlasInternal( - wrap::cublasDznrm2, stream, false /* = pointer_mode_host */, elem_count, - CUDAComplex(CUDAMemory(x)), incx, CUDAMemoryMutable(result)); + wrap::cublasDznrm2, stream, false /* = pointer_mode_host */, + elem_count, CUDAComplex(CUDAMemory(x)), incx, CUDAMemoryMutable(result)); } bool CUDABlas::DoBlasRot(Stream *stream, uint64 elem_count, @@ -1060,48 +1106,48 @@ bool CUDABlas::DoBlasIamax(Stream *stream, uint64 elem_count, const DeviceMemory> &x, int incx, DeviceMemory *result) { return DoBlasInternal( - wrap::cublasIcamax, stream, false /* = pointer_mode_host */, elem_count, - CUDAComplex(CUDAMemory(x)), incx, CUDAMemoryMutable(result)); + wrap::cublasIcamax, stream, false /* = pointer_mode_host */, + elem_count, CUDAComplex(CUDAMemory(x)), incx, CUDAMemoryMutable(result)); } bool CUDABlas::DoBlasIamax(Stream *stream, uint64 elem_count, const DeviceMemory> &x, int incx, DeviceMemory *result) { return DoBlasInternal( - wrap::cublasIzamax, stream, false /* = pointer_mode_host */, elem_count, - CUDAComplex(CUDAMemory(x)), incx, CUDAMemoryMutable(result)); + wrap::cublasIzamax, stream, false /* = pointer_mode_host */, + elem_count, CUDAComplex(CUDAMemory(x)), incx, CUDAMemoryMutable(result)); } bool CUDABlas::DoBlasIamin(Stream *stream, uint64 elem_count, const DeviceMemory &x, int incx, DeviceMemory *result) { return DoBlasInternal( - wrap::cublasIsamin, stream, false /* = pointer_mode_host */, elem_count, - CUDAComplex(CUDAMemory(x)), incx, CUDAMemoryMutable(result)); + wrap::cublasIsamin, stream, false /* = pointer_mode_host */, + elem_count, CUDAComplex(CUDAMemory(x)), incx, CUDAMemoryMutable(result)); } bool CUDABlas::DoBlasIamin(Stream *stream, uint64 elem_count, const DeviceMemory &x, int incx, DeviceMemory *result) { return DoBlasInternal( - wrap::cublasIdamin, stream, false /* = pointer_mode_host */, elem_count, - CUDAComplex(CUDAMemory(x)), incx, CUDAMemoryMutable(result)); + wrap::cublasIdamin, stream, false /* = pointer_mode_host */, + elem_count, CUDAComplex(CUDAMemory(x)), incx, CUDAMemoryMutable(result)); } bool CUDABlas::DoBlasIamin(Stream *stream, uint64 elem_count, const DeviceMemory> &x, int incx, DeviceMemory *result) { return DoBlasInternal( - wrap::cublasIcamin, stream, false /* = pointer_mode_host */, elem_count, - CUDAComplex(CUDAMemory(x)), incx, CUDAMemoryMutable(result)); + wrap::cublasIcamin, stream, false /* = pointer_mode_host */, + elem_count, CUDAComplex(CUDAMemory(x)), incx, CUDAMemoryMutable(result)); } bool CUDABlas::DoBlasIamin(Stream *stream, uint64 elem_count, const DeviceMemory> &x, int incx, DeviceMemory *result) { return DoBlasInternal( - wrap::cublasIzamin, stream, false /* = pointer_mode_host */, elem_count, - CUDAComplex(CUDAMemory(x)), incx, CUDAMemoryMutable(result)); + wrap::cublasIzamin, stream, false /* = pointer_mode_host */, + elem_count, CUDAComplex(CUDAMemory(x)), incx, CUDAMemoryMutable(result)); } bool CUDABlas::DoBlasGbmv(Stream *stream, blas::Transpose trans, uint64 m, diff --git a/tensorflow/stream_executor/cuda/cuda_dnn.cc b/tensorflow/stream_executor/cuda/cuda_dnn.cc index 19397c7dbf2..1f2e2f48bbd 100644 --- a/tensorflow/stream_executor/cuda/cuda_dnn.cc +++ b/tensorflow/stream_executor/cuda/cuda_dnn.cc @@ -132,43 +132,6 @@ string ToString(cudnnStatus_t status) { } } -template -cudnnDataType_t GetCudnnDataType( - dnn::DataLayout = dnn::DataLayout::kBatchDepthYX); - -template <> -cudnnDataType_t GetCudnnDataType(dnn::DataLayout) { - return CUDNN_DATA_DOUBLE; -} - -template <> -cudnnDataType_t GetCudnnDataType(dnn::DataLayout) { - return CUDNN_DATA_FLOAT; -} - -template <> -cudnnDataType_t GetCudnnDataType(dnn::DataLayout) { - return CUDNN_DATA_HALF; -} - -template <> -cudnnDataType_t GetCudnnDataType(dnn::DataLayout layout) { - switch (layout) { - case dnn::DataLayout::kYXDepthBatch: - case dnn::DataLayout::kYXBatchDepth: - case dnn::DataLayout::kBatchYXDepth: - case dnn::DataLayout::kBatchDepthYX: - return CUDNN_DATA_INT8; - case dnn::DataLayout::kBatchDepthYX4: - return CUDNN_DATA_INT8x4; - } -} - -template <> -cudnnDataType_t GetCudnnDataType(dnn::DataLayout) { - return CUDNN_DATA_INT32; -} - // RAII wrapper for all calls to cuDNN with a cuDNN handle argument. // // See CudnnAccess::GetHandle() for details. @@ -685,10 +648,10 @@ class CudnnConvolutionDescriptor { CHECK_CUDNN_OK(cudnnSetConvolutionNdDescriptor( handle_.get(), convolution_descriptor.ndims(), padding.data(), strides.data(), dilations.data(), - // NOTE(keveman): cuDNN supports convolution and cross correlation. - // However, almost all the use cases do cross correlation, so just - // hard coding it here. - CUDNN_CROSS_CORRELATION, data_type)); + convolution_descriptor.convolution_not_crosscorr() + ? CUDNN_CONVOLUTION + : CUDNN_CROSS_CORRELATION, + data_type)); // NOTE(benbarsdell): This only applies if tensor op math is enabled // and algo selection is set to Default. @@ -861,11 +824,19 @@ cudnnDataType_t ToCudnnDataType( case dnn::DataType::kInt8: return data_layout == dnn::DataLayout::kBatchDepthYX4 ? CUDNN_DATA_INT8x4 : CUDNN_DATA_INT8; + case dnn::DataType::kInt32: + return CUDNN_DATA_INT32; default: LOG(FATAL) << "Invalid DNN data type: " << static_cast(data_type); } } +template +cudnnDataType_t GetCudnnDataType( + dnn::DataLayout data_layout = dnn::DataLayout::kBatchDepthYX) { + return ToCudnnDataType(dnn::ToDataType::value, data_layout); +} + cudnnRNNInputMode_t ToCudnnRnnInputMode(dnn::RnnInputMode input_mode) { switch (input_mode) { case dnn::RnnInputMode::kRnnLinearSkip: @@ -2345,27 +2316,6 @@ struct ConvDoFP32ComputationFP16Input { static constexpr bool kDefaultFlag = true; }; -// A group of helper functions to return the internal compute type for -// convolutions in cudnn. -template -cudnnDataType_t GetConvComputeType() { - return CUDNN_DATA_FLOAT; -} - -template <> -cudnnDataType_t GetConvComputeType() { - if (CudnnEnvVar::IsEnabled()) { - return CUDNN_DATA_FLOAT; - } else { - return CUDNN_DATA_HALF; - } -} - -template <> -cudnnDataType_t GetConvComputeType() { - return CUDNN_DATA_DOUBLE; -} - // A helper struct to decide whether to use FP32 as the internal compute type // for rnn when the input data type is FP16. At present it is turned off, // users can explicitly control them through an env-var @@ -2437,7 +2387,7 @@ port::Status CudnnSupport::DoConvolveImpl( const DeviceMemory& filter_data, const dnn::ConvolutionDescriptor& convolution_descriptor, const dnn::BatchDescriptor& output_descriptor, DeviceMemory* output_data, - ScratchAllocator* scratch_allocator, + dnn::DataType accumulator_type, ScratchAllocator* scratch_allocator, const dnn::AlgorithmConfig& algorithm_config, dnn::ProfileResult* output_profile_result) { cudnnDataType_t cudnn_type = GetCudnnDataType(); @@ -2445,7 +2395,7 @@ port::Status CudnnSupport::DoConvolveImpl( CudnnTensorDescriptor output_nd(output_descriptor, cudnn_type); CudnnFilterDescriptor filter(filter_descriptor, cudnn_type); CudnnConvolutionDescriptor conv(convolution_descriptor, - GetConvComputeType()); + ToCudnnDataType(accumulator_type)); auto cudnn = cudnn_->GetHandle(parent_, stream); // Alpha is the scaling factor for input. @@ -2536,8 +2486,7 @@ port::Status CudnnSupport::DoConvolveImpl( return port::Status::OK(); } -template +template port::Status CudnnSupport::DoFusedConvolveImpl( Stream* stream, const dnn::BatchDescriptor& conv_input_descriptor, const DeviceMemory& conv_input_data, @@ -2548,7 +2497,8 @@ port::Status CudnnSupport::DoFusedConvolveImpl( ScaleType side_input_scale, const dnn::BatchDescriptor& bias_descriptor, const DeviceMemory& biases, dnn::ActivationMode activation_mode, const dnn::BatchDescriptor& output_descriptor, - DeviceMemory* output_data, ScratchAllocator* scratch_allocator, + DeviceMemory* output_data, dnn::DataType accumulator_type, + ScratchAllocator* scratch_allocator, const dnn::AlgorithmConfig& algorithm_config, dnn::ProfileResult* output_profile_result) { if (activation_mode != dnn::ActivationMode::kRelu && @@ -2569,7 +2519,7 @@ port::Status CudnnSupport::DoFusedConvolveImpl( GetCudnnDataType(conv_input_descriptor.layout())); CudnnTensorDescriptor bias_nd(bias_descriptor, GetCudnnDataType()); CudnnConvolutionDescriptor conv(convolution_descriptor, - GetCudnnDataType()); + ToCudnnDataType(accumulator_type)); auto cudnn = cudnn_->GetHandle(parent_, stream); @@ -2938,10 +2888,10 @@ bool CudnnSupport::DoConvolve( const dnn::AlgorithmConfig& algorithm_config, dnn::ProfileResult* output_profile_result) { return IsStatusOk( - DoConvolveImpl( - stream, batch_descriptor, input_data, filter_descriptor, filter_data, - convolution_descriptor, output_descriptor, output_data, - scratch_allocator, algorithm_config, output_profile_result), + DoConvolveImpl(stream, batch_descriptor, input_data, filter_descriptor, + filter_data, convolution_descriptor, output_descriptor, + output_data, dnn::DataType::kFloat, scratch_allocator, + algorithm_config, output_profile_result), /*report_error=*/!output_profile_result); } @@ -2956,10 +2906,10 @@ bool CudnnSupport::DoConvolve( const dnn::AlgorithmConfig& algorithm_config, dnn::ProfileResult* output_profile_result) { return IsStatusOk( - DoConvolveImpl( - stream, batch_descriptor, input_data, filter_descriptor, filter_data, - convolution_descriptor, output_descriptor, output_data, - scratch_allocator, algorithm_config, output_profile_result), + DoConvolveImpl(stream, batch_descriptor, input_data, filter_descriptor, + filter_data, convolution_descriptor, output_descriptor, + output_data, dnn::DataType::kDouble, scratch_allocator, + algorithm_config, output_profile_result), /*report_error=*/!output_profile_result); } @@ -2973,11 +2923,15 @@ bool CudnnSupport::DoConvolve( DeviceMemory* output_data, ScratchAllocator* scratch_allocator, const dnn::AlgorithmConfig& algorithm_config, dnn::ProfileResult* output_profile_result) { + dnn::DataType acc_type = + CudnnEnvVar::IsEnabled() + ? dnn::DataType::kFloat + : dnn::DataType::kHalf; return IsStatusOk( - DoConvolveImpl( - stream, batch_descriptor, input_data, filter_descriptor, filter_data, - convolution_descriptor, output_descriptor, output_data, - scratch_allocator, algorithm_config, output_profile_result), + DoConvolveImpl(stream, batch_descriptor, input_data, filter_descriptor, + filter_data, convolution_descriptor, output_descriptor, + output_data, acc_type, scratch_allocator, algorithm_config, + output_profile_result), /*report_error=*/!output_profile_result); } @@ -2995,12 +2949,13 @@ bool CudnnSupport::DoFusedConvolve( const dnn::AlgorithmConfig& algorithm_config, dnn::ProfileResult* output_profile_result) { return IsStatusOk( - DoFusedConvolveImpl( - stream, conv_input_descriptor, conv_input_data, conv_input_scale, - filter_descriptor, filter_data, convolution_descriptor, - side_input_data, side_input_scale, bias_descriptor, biases, - activation_mode, output_descriptor, output_data, scratch_allocator, - algorithm_config, output_profile_result), + DoFusedConvolveImpl(stream, conv_input_descriptor, conv_input_data, + conv_input_scale, filter_descriptor, filter_data, + convolution_descriptor, side_input_data, + side_input_scale, bias_descriptor, biases, + activation_mode, output_descriptor, output_data, + dnn::DataType::kDouble, scratch_allocator, + algorithm_config, output_profile_result), /*report_error=*/!output_profile_result); } @@ -3018,12 +2973,13 @@ bool CudnnSupport::DoFusedConvolve( const dnn::AlgorithmConfig& algorithm_config, dnn::ProfileResult* output_profile_result) { return IsStatusOk( - DoFusedConvolveImpl( - stream, conv_input_descriptor, conv_input_data, conv_input_scale, - filter_descriptor, filter_data, convolution_descriptor, - side_input_data, side_input_scale, bias_descriptor, biases, - activation_mode, output_descriptor, output_data, scratch_allocator, - algorithm_config, output_profile_result), + DoFusedConvolveImpl(stream, conv_input_descriptor, conv_input_data, + conv_input_scale, filter_descriptor, filter_data, + convolution_descriptor, side_input_data, + side_input_scale, bias_descriptor, biases, + activation_mode, output_descriptor, output_data, + dnn::DataType::kFloat, scratch_allocator, + algorithm_config, output_profile_result), /*report_error=*/!output_profile_result); } @@ -3041,13 +2997,17 @@ bool CudnnSupport::DoFusedConvolve( DeviceMemory* output_data, ScratchAllocator* scratch_allocator, const dnn::AlgorithmConfig& algorithm_config, dnn::ProfileResult* output_profile_result) { + dnn::DataType acc_type = + CudnnEnvVar::IsEnabled() + ? dnn::DataType::kFloat + : dnn::DataType::kHalf; return IsStatusOk( - DoFusedConvolveImpl( + DoFusedConvolveImpl( stream, conv_input_descriptor, conv_input_data, conv_input_scale, filter_descriptor, filter_data, convolution_descriptor, side_input_data, side_input_scale, bias_descriptor, biases, - activation_mode, output_descriptor, output_data, scratch_allocator, - algorithm_config, output_profile_result), + activation_mode, output_descriptor, output_data, acc_type, + scratch_allocator, algorithm_config, output_profile_result), /*report_error=*/!output_profile_result); } @@ -3073,12 +3033,13 @@ bool CudnnSupport::DoFusedConvolve( return false; } return IsStatusOk( - DoFusedConvolveImpl( - stream, conv_input_descriptor, conv_input_data, conv_input_scale, - filter_descriptor, filter_data, convolution_descriptor, - side_input_data, side_input_scale, bias_descriptor, biases, - activation_mode, output_descriptor, output_data, scratch_allocator, - algorithm_config, output_profile_result), + DoFusedConvolveImpl(stream, conv_input_descriptor, conv_input_data, + conv_input_scale, filter_descriptor, filter_data, + convolution_descriptor, side_input_data, + side_input_scale, bias_descriptor, biases, + activation_mode, output_descriptor, output_data, + dnn::DataType::kInt32, scratch_allocator, + algorithm_config, output_profile_result), /*report_error=*/!output_profile_result); } @@ -3112,7 +3073,8 @@ port::Status CudnnSupport::DoConvolveBackwardDataImpl( DeviceMemory backward_output_data, const dnn::ConvolutionDescriptor& convolution_descriptor, const dnn::BatchDescriptor& input_descriptor, - DeviceMemory* backward_input_data, ScratchAllocator* scratch_allocator, + DeviceMemory* backward_input_data, dnn::DataType accumulator_type, + ScratchAllocator* scratch_allocator, const dnn::AlgorithmConfig& algorithm_config, dnn::ProfileResult* output_profile_result) { cudnnDataType_t cudnn_type = GetCudnnDataType(); @@ -3133,7 +3095,7 @@ port::Status CudnnSupport::DoConvolveBackwardDataImpl( CudnnTensorDescriptor in_back_nd(input_descriptor, cudnn_type); CudnnFilterDescriptor filter(filter_descriptor, cudnn_type); CudnnConvolutionDescriptor conv(convolution_descriptor, - GetConvComputeType()); + ToCudnnDataType(accumulator_type)); const bool is_profiling = output_profile_result != nullptr; @@ -3213,11 +3175,11 @@ bool CudnnSupport::DoConvolveBackwardData( const dnn::AlgorithmConfig& algorithm_config, dnn::ProfileResult* output_profile_result) { return IsStatusOk( - DoConvolveBackwardDataImpl(stream, filter_descriptor, filter_data, - output_descriptor, backward_output_data, - convolution_descriptor, input_descriptor, - backward_input_data, scratch_allocator, - algorithm_config, output_profile_result), + DoConvolveBackwardDataImpl( + stream, filter_descriptor, filter_data, output_descriptor, + backward_output_data, convolution_descriptor, input_descriptor, + backward_input_data, dnn::DataType::kDouble, scratch_allocator, + algorithm_config, output_profile_result), /*report_error=*/!output_profile_result); } @@ -3233,11 +3195,11 @@ bool CudnnSupport::DoConvolveBackwardData( const dnn::AlgorithmConfig& algorithm_config, dnn::ProfileResult* output_profile_result) { return IsStatusOk( - DoConvolveBackwardDataImpl(stream, filter_descriptor, filter_data, - output_descriptor, backward_output_data, - convolution_descriptor, input_descriptor, - backward_input_data, scratch_allocator, - algorithm_config, output_profile_result), + DoConvolveBackwardDataImpl( + stream, filter_descriptor, filter_data, output_descriptor, + backward_output_data, convolution_descriptor, input_descriptor, + backward_input_data, dnn::DataType::kFloat, scratch_allocator, + algorithm_config, output_profile_result), /*report_error=*/!output_profile_result); } @@ -3252,12 +3214,16 @@ bool CudnnSupport::DoConvolveBackwardData( ScratchAllocator* scratch_allocator, const dnn::AlgorithmConfig& algorithm_config, dnn::ProfileResult* output_profile_result) { + dnn::DataType acc_type = + CudnnEnvVar::IsEnabled() + ? dnn::DataType::kFloat + : dnn::DataType::kHalf; return IsStatusOk( - DoConvolveBackwardDataImpl(stream, filter_descriptor, filter_data, - output_descriptor, backward_output_data, - convolution_descriptor, input_descriptor, - backward_input_data, scratch_allocator, - algorithm_config, output_profile_result), + DoConvolveBackwardDataImpl( + stream, filter_descriptor, filter_data, output_descriptor, + backward_output_data, convolution_descriptor, input_descriptor, + backward_input_data, acc_type, scratch_allocator, algorithm_config, + output_profile_result), /*report_error=*/!output_profile_result); } @@ -3269,7 +3235,8 @@ port::Status CudnnSupport::DoConvolveBackwardFilterImpl( DeviceMemory backward_output_data, const dnn::ConvolutionDescriptor& convolution_descriptor, const dnn::FilterDescriptor& filter_descriptor, - DeviceMemory* backward_filter_data, ScratchAllocator* scratch_allocator, + DeviceMemory* backward_filter_data, dnn::DataType accumulator_type, + ScratchAllocator* scratch_allocator, const dnn::AlgorithmConfig& algorithm_config, dnn::ProfileResult* output_profile_result) { cudnnDataType_t cudnn_type = GetCudnnDataType(); @@ -3290,7 +3257,7 @@ port::Status CudnnSupport::DoConvolveBackwardFilterImpl( CudnnTensorDescriptor input_nd(input_descriptor, cudnn_type); CudnnFilterDescriptor filter(filter_descriptor, cudnn_type); CudnnConvolutionDescriptor conv(convolution_descriptor, - GetConvComputeType()); + ToCudnnDataType(accumulator_type)); const bool is_profiling = output_profile_result != nullptr; @@ -3406,11 +3373,12 @@ bool CudnnSupport::DoConvolveBackwardFilter( const dnn::AlgorithmConfig& algorithm_config, dnn::ProfileResult* output_profile_result) { return IsStatusOk( - DoConvolveBackwardFilterImpl(stream, input_descriptor, input_data, - output_descriptor, backward_output_data, - convolution_descriptor, filter_descriptor, - backward_filter_data, scratch_allocator, - algorithm_config, output_profile_result), + DoConvolveBackwardFilterImpl( + stream, input_descriptor, input_data, output_descriptor, + backward_output_data, convolution_descriptor, filter_descriptor, + backward_filter_data, dnn::DataType::kDouble, + + scratch_allocator, algorithm_config, output_profile_result), /*report_error=*/!output_profile_result); } @@ -3425,13 +3393,14 @@ bool CudnnSupport::DoConvolveBackwardFilter( ScratchAllocator* scratch_allocator, const dnn::AlgorithmConfig& algorithm_config, dnn::ProfileResult* output_profile_result) { - return IsStatusOk( - DoConvolveBackwardFilterImpl(stream, input_descriptor, input_data, - output_descriptor, backward_output_data, - convolution_descriptor, filter_descriptor, - backward_filter_data, scratch_allocator, - algorithm_config, output_profile_result), - /*report_error=*/!output_profile_result); + return IsStatusOk(DoConvolveBackwardFilterImpl( + stream, input_descriptor, input_data, output_descriptor, + backward_output_data, convolution_descriptor, + filter_descriptor, backward_filter_data, + + dnn::DataType::kFloat, scratch_allocator, + algorithm_config, output_profile_result), + /*report_error=*/!output_profile_result); } bool CudnnSupport::DoConvolveBackwardFilter( @@ -3445,12 +3414,16 @@ bool CudnnSupport::DoConvolveBackwardFilter( ScratchAllocator* scratch_allocator, const dnn::AlgorithmConfig& algorithm_config, dnn::ProfileResult* output_profile_result) { + dnn::DataType acc_type = + CudnnEnvVar::IsEnabled() + ? dnn::DataType::kFloat + : dnn::DataType::kHalf; return IsStatusOk( - DoConvolveBackwardFilterImpl(stream, input_descriptor, input_data, - output_descriptor, backward_output_data, - convolution_descriptor, filter_descriptor, - backward_filter_data, scratch_allocator, - algorithm_config, output_profile_result), + DoConvolveBackwardFilterImpl( + stream, input_descriptor, input_data, output_descriptor, + backward_output_data, convolution_descriptor, filter_descriptor, + backward_filter_data, acc_type, scratch_allocator, algorithm_config, + output_profile_result), /*report_error=*/!output_profile_result); } diff --git a/tensorflow/stream_executor/cuda/cuda_dnn.h b/tensorflow/stream_executor/cuda/cuda_dnn.h index 74f6f935b84..0641be140d2 100644 --- a/tensorflow/stream_executor/cuda/cuda_dnn.h +++ b/tensorflow/stream_executor/cuda/cuda_dnn.h @@ -670,12 +670,12 @@ class CudnnSupport : public dnn::DnnSupport { const DeviceMemory& filter_data, const dnn::ConvolutionDescriptor& convolution_descriptor, const dnn::BatchDescriptor& output_descriptor, - DeviceMemory* output_data, ScratchAllocator* scratch_allocator, + DeviceMemory* output_data, dnn::DataType accumulator_type, + ScratchAllocator* scratch_allocator, const dnn::AlgorithmConfig& algorithm_config, dnn::ProfileResult* output_profile_result); - template + template port::Status DoFusedConvolveImpl( Stream* stream, const dnn::BatchDescriptor& conv_input_descriptor, const DeviceMemory& conv_input_data, @@ -687,7 +687,7 @@ class CudnnSupport : public dnn::DnnSupport { ScaleType side_input_scale, const dnn::BatchDescriptor& bias_descriptor, const DeviceMemory& biases, dnn::ActivationMode activation_mode, const dnn::BatchDescriptor& output_descriptor, - DeviceMemory* output_data, + DeviceMemory* output_data, dnn::DataType accumulator_type, ScratchAllocator* scratch_allocator, const dnn::AlgorithmConfig& algorithm_config, dnn::ProfileResult* output_profile_result); @@ -700,7 +700,8 @@ class CudnnSupport : public dnn::DnnSupport { DeviceMemory backward_output_data, const dnn::ConvolutionDescriptor& convolution_descriptor, const dnn::BatchDescriptor& input_descriptor, - DeviceMemory* backward_input_data, ScratchAllocator* scratch_allocator, + DeviceMemory* backward_input_data, dnn::DataType accumulator_type, + ScratchAllocator* scratch_allocator, const dnn::AlgorithmConfig& algorithm_config, dnn::ProfileResult* output_profile_result); @@ -712,7 +713,7 @@ class CudnnSupport : public dnn::DnnSupport { DeviceMemory backward_output_data, const dnn::ConvolutionDescriptor& convolution_descriptor, const dnn::FilterDescriptor& filter_descriptor, - DeviceMemory* backward_filter_data, + DeviceMemory* backward_filter_data, dnn::DataType accumulator_type, ScratchAllocator* scratch_allocator, const dnn::AlgorithmConfig& algorithm_config, dnn::ProfileResult* output_profile_result); diff --git a/tensorflow/stream_executor/cuda/cuda_fft.cc b/tensorflow/stream_executor/cuda/cuda_fft.cc index cbf388a0f89..acac7d63688 100644 --- a/tensorflow/stream_executor/cuda/cuda_fft.cc +++ b/tensorflow/stream_executor/cuda/cuda_fft.cc @@ -23,6 +23,11 @@ limitations under the License. #include "tensorflow/stream_executor/cuda/cuda_platform_id.h" #include "tensorflow/stream_executor/cuda/cuda_stream.h" #include "tensorflow/stream_executor/device_memory.h" + +#ifndef PLATFORM_GOOGLE +#include "tensorflow/stream_executor/dso_loader.h" +#endif + #include "tensorflow/stream_executor/lib/env.h" #include "tensorflow/stream_executor/lib/initialize.h" #include "tensorflow/stream_executor/lib/status.h" @@ -38,6 +43,7 @@ PLUGIN_REGISTRY_DEFINE_PLUGIN_ID(kCuFftPlugin); namespace wrap { +#ifdef PLATFORM_GOOGLE // This macro wraps a global identifier, given by __name, in a callable // structure that loads the DLL symbol out of the DSO handle in a thread-safe // manner on first use. This dynamic loading technique is used to avoid DSO @@ -52,22 +58,69 @@ namespace wrap { } \ } __name; -#define CUFFT_ROUTINE_EACH(__macro) \ - __macro(cufftDestroy) __macro(cufftSetStream) __macro(cufftPlan1d) \ - __macro(cufftPlan2d) __macro(cufftPlan3d) __macro(cufftPlanMany) \ - __macro(cufftExecD2Z) __macro(cufftExecZ2D) __macro(cufftExecC2C) \ - __macro(cufftExecC2R) __macro(cufftExecZ2Z) \ - __macro(cufftExecR2C) __macro(cufftCreate) \ - __macro(cufftSetAutoAllocation) \ - __macro(cufftSetWorkArea) __macro(cufftGetSize1d) \ - __macro(cufftMakePlan1d) __macro(cufftGetSize2d) \ - __macro(cufftMakePlan2d) \ - __macro(cufftGetSize3d) \ - __macro(cufftMakePlan3d) \ - __macro(cufftGetSizeMany) \ - __macro(cufftMakePlanMany) +#else + +#define STREAM_EXECUTOR_CUFFT_WRAP(__name) \ + struct DynLoadShim__##__name { \ + static const char *kName; \ + using FuncPtrT = std::add_pointer::type; \ + static void *GetDsoHandle() { \ + auto s = internal::CachedDsoLoader::GetCufftDsoHandle(); \ + return s.ValueOrDie(); \ + } \ + static FuncPtrT LoadOrDie() { \ + void *f; \ + auto s = port::Env::Default()->GetSymbolFromLibrary(GetDsoHandle(), \ + kName, &f); \ + CHECK(s.ok()) << "could not find " << kName \ + << " in cufft DSO; dlerror: " << s.error_message(); \ + return reinterpret_cast(f); \ + } \ + static FuncPtrT DynLoad() { \ + static FuncPtrT f = LoadOrDie(); \ + return f; \ + } \ + template \ + cufftResult operator()(CUDAExecutor *parent, Args... args) { \ + cuda::ScopedActivateExecutorContext sac{parent}; \ + return DynLoad()(args...); \ + } \ + } __name; \ + const char *DynLoadShim__##__name::kName = #__name; + +#endif + +// clang-format off + +#define CUFFT_ROUTINE_EACH(__macro) \ + __macro(cufftDestroy) \ + __macro(cufftSetStream) \ + __macro(cufftPlan1d) \ + __macro(cufftPlan2d) \ + __macro(cufftPlan3d) \ + __macro(cufftPlanMany) \ + __macro(cufftExecD2Z) \ + __macro(cufftExecZ2D) \ + __macro(cufftExecC2C) \ + __macro(cufftExecC2R) \ + __macro(cufftExecZ2Z) \ + __macro(cufftExecR2C) \ + __macro(cufftCreate) \ + __macro(cufftSetAutoAllocation) \ + __macro(cufftSetWorkArea) \ + __macro(cufftGetSize1d) \ + __macro(cufftMakePlan1d) \ + __macro(cufftGetSize2d) \ + __macro(cufftMakePlan2d) \ + __macro(cufftGetSize3d) \ + __macro(cufftMakePlan3d) \ + __macro(cufftGetSizeMany) \ + __macro(cufftMakePlanMany) + +// clang-format on CUFFT_ROUTINE_EACH(STREAM_EXECUTOR_CUFFT_WRAP) +#undef CUFFT_ROUTINE_EACH } // namespace wrap diff --git a/tensorflow/stream_executor/cuda/cuda_gpu_executor.cc b/tensorflow/stream_executor/cuda/cuda_gpu_executor.cc index ad9154226c4..4874d096ad5 100644 --- a/tensorflow/stream_executor/cuda/cuda_gpu_executor.cc +++ b/tensorflow/stream_executor/cuda/cuda_gpu_executor.cc @@ -662,8 +662,13 @@ bool CUDAExecutor::MemcpyDeviceToDevice(Stream *stream, } bool CUDAExecutor::HostCallback(Stream *stream, - std::function callback) { - auto callback_ptr = new std::function(callback); + std::function callback) { + auto callback_ptr = new std::function([callback]() { + port::Status s = callback(); + if (!s.ok()) { + LOG(WARNING) << "Host callback failed: " << s; + } + }); return CUDADriver::AddStreamCallback(context_, AsCUDAStreamValue(stream), InternalHostCallback, callback_ptr); } diff --git a/tensorflow/stream_executor/cuda/cuda_gpu_executor.h b/tensorflow/stream_executor/cuda/cuda_gpu_executor.h index 90bf1c0242f..ae8e4abf920 100644 --- a/tensorflow/stream_executor/cuda/cuda_gpu_executor.h +++ b/tensorflow/stream_executor/cuda/cuda_gpu_executor.h @@ -148,7 +148,8 @@ class CUDAExecutor : public internal::StreamExecutorInterface { const DeviceMemoryBase &gpu_src, uint64 size) override; - bool HostCallback(Stream *stream, std::function callback) override; + bool HostCallback(Stream *stream, + std::function callback) override; bool AllocateStream(Stream *stream) override; diff --git a/tensorflow/stream_executor/cuda/cuda_rng.cc b/tensorflow/stream_executor/cuda/cuda_rng.cc index 88c4f157927..7f920719321 100644 --- a/tensorflow/stream_executor/cuda/cuda_rng.cc +++ b/tensorflow/stream_executor/cuda/cuda_rng.cc @@ -21,6 +21,11 @@ limitations under the License. #include "tensorflow/stream_executor/cuda/cuda_platform_id.h" #include "tensorflow/stream_executor/cuda/cuda_stream.h" #include "tensorflow/stream_executor/device_memory.h" + +#ifndef PLATFORM_GOOGLE +#include "tensorflow/stream_executor/dso_loader.h" +#endif + #include "tensorflow/stream_executor/lib/env.h" #include "tensorflow/stream_executor/lib/initialize.h" #include "tensorflow/stream_executor/lib/status.h" @@ -61,6 +66,7 @@ PLUGIN_REGISTRY_DEFINE_PLUGIN_ID(kCuRandPlugin); namespace wrap { +#ifdef PLATFORM_GOOGLE #define STREAM_EXECUTOR_CURAND_WRAP(__name) \ struct WrapperShim__##__name { \ template \ @@ -70,6 +76,36 @@ namespace wrap { } \ } __name; +#else +#define STREAM_EXECUTOR_CURAND_WRAP(__name) \ + struct DynLoadShim__##__name { \ + static const char *kName; \ + using FuncPtrT = std::add_pointer::type; \ + static void *GetDsoHandle() { \ + auto s = internal::CachedDsoLoader::GetCurandDsoHandle(); \ + return s.ValueOrDie(); \ + } \ + static FuncPtrT LoadOrDie() { \ + void *f; \ + auto s = port::Env::Default()->GetSymbolFromLibrary(GetDsoHandle(), \ + kName, &f); \ + CHECK(s.ok()) << "could not find " << kName \ + << " in curand DSO; dlerror: " << s.error_message(); \ + return reinterpret_cast(f); \ + } \ + static FuncPtrT DynLoad() { \ + static FuncPtrT f = LoadOrDie(); \ + return f; \ + } \ + template \ + curandStatus_t operator()(CUDAExecutor *parent, Args... args) { \ + cuda::ScopedActivateExecutorContext sac{parent}; \ + return DynLoad()(args...); \ + } \ + } __name; \ + const char *DynLoadShim__##__name::kName = #__name; +#endif + STREAM_EXECUTOR_CURAND_WRAP(curandCreateGenerator); STREAM_EXECUTOR_CURAND_WRAP(curandDestroyGenerator); STREAM_EXECUTOR_CURAND_WRAP(curandSetStream); diff --git a/tensorflow/stream_executor/device_description.cc b/tensorflow/stream_executor/device_description.cc index 4120e230dbf..0b991b7ba8c 100644 --- a/tensorflow/stream_executor/device_description.cc +++ b/tensorflow/stream_executor/device_description.cc @@ -140,21 +140,11 @@ void CalculateDimensionality(const DeviceDescription &device_description, uint64 element_count, uint64 *threads_per_block, uint64 *block_count) { *threads_per_block = device_description.threads_per_block_limit(); - *block_count = DivideCeil(element_count, *threads_per_block); + *block_count = port::MathUtil::CeilOfRatio(element_count, *threads_per_block); if (*block_count == 1) { CHECK_LE(element_count, *threads_per_block); *threads_per_block = element_count; } } -// Round value up to a multiple of n. -static uint64 RoundUp(uint64 value, uint64 n) { - return port::MathUtil::CeilOfRatio(value, n) * n; -} - -// Round value down to a multiple of n. -static uint64 RoundDown(uint64 value, uint64 n) { - return port::MathUtil::FloorOfRatio(value, n) * n; -} - } // namespace stream_executor diff --git a/tensorflow/stream_executor/dnn.cc b/tensorflow/stream_executor/dnn.cc index 3d8e691ab28..faa662211eb 100644 --- a/tensorflow/stream_executor/dnn.cc +++ b/tensorflow/stream_executor/dnn.cc @@ -23,7 +23,7 @@ namespace stream_executor { namespace dnn { uint64 AlgorithmDesc::hash() const { - return ::tensorflow::Hash64Combine(algo_, tensor_ops_enabled_); + return ::tensorflow::Hash64Combine(algo_id(), tensor_ops_enabled()); } bool DnnSupport::GetConvolveAlgorithms( @@ -187,6 +187,9 @@ std::tuple GetDimIndices(const DataLayout& layout, batch_idx = 0; spatial_idx = 2; break; + + default: + LOG(FATAL) << "Unknown layout " << layout; } return std::make_tuple(depth_idx, batch_idx, spatial_idx); @@ -233,28 +236,27 @@ string AlgorithmConfig::ToString() const { // -- BatchDescriptor BatchDescriptor::BatchDescriptor(int ndims) - : count_(0), - feature_map_count_(0), - spatial_size_(ndims, 0), - value_max_(0.0), + : value_max_(0.0), value_min_(0.0), - layout_(DataLayout::kYXDepthBatch), - ndims_(ndims), - quantized_activation_mode_(QuantizedActivationMode::k8Bit) {} + quantized_activation_mode_(QuantizedActivationMode::k8Bit) { + tensor_.mutable_dimensions()->Resize(ndims + 2, 0); + set_layout(DataLayout::kYXDepthBatch); +} BatchDescriptor::BatchDescriptor() : BatchDescriptor(/*ndims=*/2) {} std::vector BatchDescriptor::full_dims(const DataLayout& layout) const { - std::vector bdyx_dims(ndims_ + 2); + std::vector bdyx_dims(ndims() + 2); bdyx_dims[0] = count(); bdyx_dims[1] = feature_map_count(); - std::copy(spatial_size_.begin(), spatial_size_.end(), bdyx_dims.begin() + 2); + std::copy(spatial_size().begin(), spatial_size().end(), + bdyx_dims.begin() + 2); return ReorderDims(bdyx_dims, DataLayout::kBatchDepthYX, layout); } std::vector BatchDescriptor::full_strides( const DataLayout& layout) const { - if (layout_ == DataLayout::kBatchDepthYX4) { + if (this->layout() == DataLayout::kBatchDepthYX4) { LOG(FATAL) << "Cannot compute full strides for batch descriptor " << ToString() << ", because its layout is kBatchDepthYX4. In fact, " @@ -262,36 +264,32 @@ std::vector BatchDescriptor::full_strides( "Use cudnnSetTensor4DDescriptor to set cudnnTensorDescriptor_t " "instead."; } - std::vector phys_dims = full_dims(layout_); + std::vector phys_dims = full_dims(this->layout()); std::vector phys_strides(phys_dims.size()); - phys_strides[ndims_ + 1] = 1; - for (int i = ndims_; i >= 0; i--) { + phys_strides[ndims() + 1] = 1; + for (int i = ndims(); i >= 0; i--) { phys_strides[i] = phys_strides[i + 1] * phys_dims[i + 1]; } - return ReorderDims(phys_strides, layout_, layout); + return ReorderDims(phys_strides, this->layout(), layout); } void BatchDescriptor::CloneFrom(const BatchDescriptor& other) { - count_ = other.count_; - feature_map_count_ = other.feature_map_count_; - spatial_size_ = other.spatial_size_; + tensor_ = other.tensor_; value_max_ = other.value_max_; value_min_ = other.value_min_; - layout_ = other.layout_; - ndims_ = other.ndims_; quantized_activation_mode_ = other.quantized_activation_mode_; } string BatchDescriptor::ToString() const { string spatial; - for (int i = 0; i < ndims_; i++) { - port::Appendf(&spatial, "%lld ", spatial_size_[i]); + for (int i = 0; i < ndims(); i++) { + port::Appendf(&spatial, "%lld ", spatial_size()[i]); } return port::Printf( "{count: %lld feature_map_count: %lld spatial: %s " "value_min: %f value_max: %f layout: %s}", - count_, feature_map_count_, spatial.c_str(), value_min_, value_max_, - DataLayoutString(layout_).c_str()); + count(), feature_map_count(), spatial.c_str(), value_min_, value_max_, + DataLayoutString(layout()).c_str()); } string BatchDescriptor::ToShortString() const { @@ -302,8 +300,8 @@ string BatchDescriptor::ToShortString() const { string batch = absl::StrCat("b", count()); string spatial = "s"; - for (int i = 0; i < ndims_; i++) { - port::Appendf(&spatial, "%lld ", spatial_size_[i]); + for (int i = 0; i < ndims(); i++) { + port::Appendf(&spatial, "%lld ", spatial_size()[i]); } string suffix; @@ -333,18 +331,18 @@ string BatchDescriptor::ToShortString() const { int64 BatchDescriptor::NodesPerFeatureMap() const { int64 ret = 1; - for (int i = 0; i < ndims_; i++) { - ret *= spatial_size_[i]; + for (int i = 0; i < ndims(); i++) { + ret *= spatial_size()[i]; } return ret; } int64 BatchDescriptor::NodesAcrossFeatureMaps() const { - return NodesPerFeatureMap() * feature_map_count_; + return NodesPerFeatureMap() * feature_map_count(); } int64 BatchDescriptor::ElementCount() const { - return count_ * feature_map_count_ * NodesPerFeatureMap(); + return count() * feature_map_count() * NodesPerFeatureMap(); } int64 BatchDescriptor::FullyConnectedWeightCount( @@ -372,33 +370,27 @@ BatchDescriptor BatchDescriptor::DepthConcatenateOutputDescriptor( // -- FilterDescriptor -FilterDescriptor::FilterDescriptor(int ndims) - : output_feature_map_count_(0), - input_feature_map_count_(0), - input_filter_dims_(ndims, 0), - ndims_(ndims), - layout_(FilterLayout::kOutputInputYX) {} +FilterDescriptor::FilterDescriptor(int ndims) { + tensor_.mutable_dimensions()->Resize(ndims + 2, 0); + set_layout(FilterLayout::kOutputInputYX); +} FilterDescriptor::FilterDescriptor() : FilterDescriptor(/*ndims=*/2) {} FilterDescriptor::~FilterDescriptor() {} void FilterDescriptor::CloneFrom(const FilterDescriptor& other) { - set_output_feature_map_count(other.output_feature_map_count()) - .set_input_feature_map_count(other.input_feature_map_count()) - .set_layout(other.layout()); - input_filter_dims_ = other.input_filter_dims_; - ndims_ = other.ndims_; + tensor_ = other.tensor_; } string FilterDescriptor::ToString() const { string desc = port::Printf( "{output_feature_map_count: %lld input_feature_map_count: %lld " "layout: %s shape: ", - output_feature_map_count_, input_feature_map_count_, - FilterLayoutString(layout_).c_str()); - for (int i = 0; i < ndims_; i++) { - port::Appendf(&desc, "%lld ", input_filter_dims_[i]); + output_feature_map_count(), input_feature_map_count(), + FilterLayoutString(layout()).c_str()); + for (int i = 0; i < ndims(); i++) { + port::Appendf(&desc, "%lld ", input_filter_dims()[i]); } absl::StrAppend(&desc, "}"); @@ -409,15 +401,15 @@ string FilterDescriptor::ToShortString() const { // All the constituent strings are less than 15 characters, so the // small string optimization ensures that there will be at most one // heap memory allocation. - string od = absl::StrCat("od", output_feature_map_count_); - string id = absl::StrCat("id", input_feature_map_count_); + string od = absl::StrCat("od", output_feature_map_count()); + string id = absl::StrCat("id", input_feature_map_count()); string spatial = "s"; - for (int i = 0; i < ndims_; i++) { - port::Appendf(&spatial, "%lld ", input_filter_dims_[i]); + for (int i = 0; i < ndims(); i++) { + port::Appendf(&spatial, "%lld ", input_filter_dims()[i]); } - switch (layout_) { + switch (layout()) { case FilterLayout::kOutputInputYX: return absl::StrCat(od, id, spatial); case FilterLayout::kOutputYXInput: @@ -429,27 +421,28 @@ string FilterDescriptor::ToShortString() const { case FilterLayout::kYXInputOutput: return absl::StrCat(spatial, id, od); default: - LOG(FATAL) << "Unknown layout " << static_cast(layout_); + LOG(FATAL) << "Unknown layout " << static_cast(layout()); return ""; // Avoid return warning (unreachable) } } int64 FilterDescriptor::ComputeWeightCount() const { - int64 ret = output_feature_map_count_ * input_feature_map_count_; - for (int i = 0; i < ndims_; i++) { - ret *= input_filter_dims_[i]; + int64 ret = output_feature_map_count() * input_feature_map_count(); + for (int i = 0; i < ndims(); i++) { + ret *= input_filter_dims()[i]; } return ret; } // -- ConvolutionDescriptor -ConvolutionDescriptor::ConvolutionDescriptor(int ndims) - : zero_padding_(ndims, 0), - filter_strides_(ndims, 1), - dilation_rates_(ndims, 1), - group_count_(1), - ndims_(ndims) {} +ConvolutionDescriptor::ConvolutionDescriptor(int ndims) { + proto_.mutable_paddings()->Resize(ndims, 0); + proto_.mutable_strides()->Resize(ndims, 1); + proto_.mutable_dilations()->Resize(ndims, 1); + proto_.set_group_count(1); + proto_.set_convolution_mode(ConvolutionMode::CROSS_CORRELATION); +} ConvolutionDescriptor::ConvolutionDescriptor() : ConvolutionDescriptor(/*ndims=*/2) {} @@ -460,10 +453,10 @@ string ConvolutionDescriptor::ToString() const { string padding; string strides; string dilations; - for (int i = 0; i < ndims_; i++) { - port::Appendf(&padding, "%lld ", zero_padding_[i]); - port::Appendf(&strides, "%lld ", filter_strides_[i]); - port::Appendf(&dilations, "%lld ", dilation_rates_[i]); + for (int i = 0; i < ndims(); i++) { + port::Appendf(&padding, "%lld ", this->padding()[i]); + port::Appendf(&strides, "%lld ", this->strides()[i]); + port::Appendf(&dilations, "%lld ", this->dilations()[i]); } return port::Printf( @@ -475,15 +468,15 @@ string ConvolutionDescriptor::ToString() const { string ConvolutionDescriptor::ToShortString() const { string desc; - for (int i = 0; i < ndims_; i++) { + for (int i = 0; i < ndims(); i++) { if (i > 0) port::Appendf(&desc, "_"); - port::Appendf(&desc, "p%d:%lld", i, zero_padding_[i]); + port::Appendf(&desc, "p%d:%lld", i, padding()[i]); } - for (int i = 0; i < ndims_; i++) { - port::Appendf(&desc, "_s%d:%lld", i, filter_strides_[i]); + for (int i = 0; i < ndims(); i++) { + port::Appendf(&desc, "_s%d:%lld", i, strides()[i]); } - for (int i = 0; i < ndims_; i++) { - port::Appendf(&desc, "_d%d:%lld", i, dilation_rates_[i]); + for (int i = 0; i < ndims(); i++) { + port::Appendf(&desc, "_d%d:%lld", i, dilations()[i]); } return desc; } diff --git a/tensorflow/stream_executor/dnn.h b/tensorflow/stream_executor/dnn.h index c934301829d..c044a356efb 100644 --- a/tensorflow/stream_executor/dnn.h +++ b/tensorflow/stream_executor/dnn.h @@ -29,7 +29,9 @@ limitations under the License. #include "absl/types/optional.h" #include "absl/types/span.h" +#include "tensorflow/core/platform/protobuf.h" #include "tensorflow/stream_executor/device_memory.h" +#include "tensorflow/stream_executor/dnn.pb.h" #include "tensorflow/stream_executor/lib/array_slice.h" #include "tensorflow/stream_executor/lib/status.h" #include "tensorflow/stream_executor/lib/statusor.h" @@ -48,19 +50,6 @@ class ScratchAllocator; namespace dnn { -// Describes how an input or output layer's data is formatted. -// Specify int64 so there's no padding in BatchDescriptor. -enum class DataLayout : int64 { - kYXDepthBatch = 0, // Same as dist_belief::DF_DEPTH_MAJOR. - kYXBatchDepth, // Same as dist_belief::DF_BATCH_MAJOR. - kBatchYXDepth, // Same as run_brain output, and tensorflow's layout. - kBatchDepthYX, // cuDNN's NCHW layout, data laid out as image, feature - // maps, rows, columns. - kBatchDepthYX4, // cuDNN's NCHW_VECT_C layout, data laid out the same as - // kBatchDepthYX but each element is a vector of 4 feature - // maps. -}; - // Specifies an index to use when accessing specific spatial dimensions. enum class DimIndex : int { X = 0, @@ -73,8 +62,27 @@ inline int64 GetDim(absl::Span data, DimIndex dim) { return data.rbegin()[static_cast(dim)]; } +inline void SetDim(absl::Span data, DimIndex dim, int64 value) { + data.rbegin()[static_cast(dim)] = value; +} + inline void SetDim(std::vector* data, DimIndex dim, int64 value) { - data->rbegin()[static_cast(dim)] = value; + return SetDim(absl::MakeSpan(*data), dim, value); +} + +// tensorflow::int64 is not the same type as tensorflow::protobuf_int64 in +// open-source. Wrapper function that gives an int64 array slice view of a +// repeated int64 protobuf field. +inline absl::Span AsInt64Slice( + const tensorflow::protobuf::RepeatedField& v) { + return absl::Span(reinterpret_cast(v.data()), + v.size()); +} + +inline absl::Span AsInt64Slice( + tensorflow::protobuf::RepeatedField* v) { + return absl::Span(reinterpret_cast(v->mutable_data()), + v->size()); } // Returns a string representation of the given data layout. @@ -87,14 +95,6 @@ enum class QuantizedActivationMode { k32Bit = 4, }; -// Specifies the data type used by an operation. -enum class DataType { - kFloat = 0, - kDouble = 1, - kHalf = 2, - kInt8 = 3, -}; - // A helper class to convert C/C++ types to the proper enums. template struct ToDataType; @@ -114,6 +114,10 @@ template <> struct ToDataType { static constexpr DataType value = DataType::kInt8; }; +template <> +struct ToDataType { + static constexpr DataType value = DataType::kInt32; +}; // Specifies the types of a RNN model. enum class RnnMode { @@ -245,15 +249,15 @@ class BatchDescriptor { string ToShortString() const; // Accessors. - int64 count() const { return count_; } - int64 feature_map_count() const { return feature_map_count_; } - int64 height() const { return GetDim(spatial_size_, DimIndex::Y); } - int64 width() const { return GetDim(spatial_size_, DimIndex::X); } - int64 spatial_dim(DimIndex dim) const { return GetDim(spatial_size_, dim); } - int ndims() const { return ndims_; } + int64 count() const { return tensor_.dimensions(0); } + int64 feature_map_count() const { return tensor_.dimensions(1); } + int64 height() const { return GetDim(spatial_size(), DimIndex::Y); } + int64 width() const { return GetDim(spatial_size(), DimIndex::X); } + int64 spatial_dim(DimIndex dim) const { return GetDim(spatial_size(), dim); } + int ndims() const { return spatial_size().size(); } float value_max() const { return value_max_; } float value_min() const { return value_min_; } - DataLayout layout() const { return layout_; } + DataLayout layout() const { return tensor_.data_layout(); } QuantizedActivationMode quantized_activation_mode() const { return quantized_activation_mode_; } @@ -267,23 +271,23 @@ class BatchDescriptor { // Named-argument helpers for avoiding user error during construction. BatchDescriptor& set_count(int64 value) { - count_ = value; + tensor_.set_dimensions(0, value); return *this; } BatchDescriptor& set_feature_map_count(int64 value) { - feature_map_count_ = value; + tensor_.set_dimensions(1, value); return *this; } BatchDescriptor& set_height(int64 value) { - SetDim(&spatial_size_, DimIndex::Y, value); + SetDim(spatial_size(), DimIndex::Y, value); return *this; } BatchDescriptor& set_width(int64 value) { - SetDim(&spatial_size_, DimIndex::X, value); + SetDim(spatial_size(), DimIndex::X, value); return *this; } BatchDescriptor& set_spatial_dim(DimIndex dim, int64 value) { - SetDim(&spatial_size_, dim, value); + SetDim(spatial_size(), dim, value); return *this; } BatchDescriptor& set_value_max(float value) { @@ -295,7 +299,7 @@ class BatchDescriptor { return *this; } BatchDescriptor& set_layout(DataLayout layout) { - layout_ = layout; + tensor_.set_data_layout(layout); return *this; } BatchDescriptor& set_quantized_activation_mode( @@ -334,31 +338,20 @@ class BatchDescriptor { port::ArraySlice inputs); private: - int64 count_; - int64 feature_map_count_; - // Stored as: ..., y, x. - std::vector spatial_size_; + absl::Span spatial_size() const { + return AsInt64Slice(tensor_.dimensions()).subspan(2); + } + + absl::Span spatial_size() { + return AsInt64Slice(tensor_.mutable_dimensions()).subspan(2); + } + + TensorDescriptorProto tensor_; float value_max_; float value_min_; - DataLayout layout_; - int ndims_; QuantizedActivationMode quantized_activation_mode_; }; -// Describes how a filter is laid out in the memory. -// Specify int64 so there's no padding in FilterDescriptor. -enum class FilterLayout : int64 { - kOutputInputYX = 0, // cuDNN's default filter layout, laid out as: - // (major) output feature maps >> input feature maps >> - // rows >> columns (minor). - kOutputYXInput, // major to minor: - // (output features, row, columns, input features) - kOutputInputYX4, // laid out the same as kOutputInputYX but each element is a - // vector of 4 feature maps. - kInputYXOutput, // Same as dist_belief's default filter layout. - kYXInputOutput, // Same as tensorflow's default filter layout. -}; - // Returns a string representation of the given filter layout. string FilterLayoutString(FilterLayout layout); @@ -398,30 +391,30 @@ class FilterDescriptor { // Named-argument helpers for avoiding user error during construction. FilterDescriptor& set_output_feature_map_count(int64 value) { - output_feature_map_count_ = value; + tensor_.set_dimensions(0, value); return *this; } FilterDescriptor& set_input_feature_map_count(int64 value) { - input_feature_map_count_ = value; + tensor_.set_dimensions(1, value); return *this; } FilterDescriptor& set_input_filter_height(int64 value) { - SetDim(&input_filter_dims_, DimIndex::Y, value); + SetDim(input_filter_dims(), DimIndex::Y, value); return *this; } FilterDescriptor& set_input_filter_width(int64 value) { - SetDim(&input_filter_dims_, DimIndex::X, value); + SetDim(input_filter_dims(), DimIndex::X, value); return *this; } FilterDescriptor& set_layout(FilterLayout layout) { - layout_ = layout; + tensor_.set_filter_layout(layout); return *this; } FilterDescriptor& set_spatial_dim(DimIndex dim, int64 value) { - SetDim(&input_filter_dims_, dim, value); + SetDim(input_filter_dims(), dim, value); return *this; } - int ndims() const { return ndims_; } + int ndims() const { return input_filter_dims().size(); } void CloneFrom(const FilterDescriptor& other); @@ -434,32 +427,32 @@ class FilterDescriptor { // Returns the number of biases required as parameters for a convolution // using this filter descriptor. - int64 bias_count() const { return output_feature_map_count_; } + int64 bias_count() const { return output_feature_map_count(); } - int64 output_feature_map_count() const { return output_feature_map_count_; } - int64 input_feature_map_count() const { return input_feature_map_count_; } + int64 output_feature_map_count() const { return tensor_.dimensions(0); } + int64 input_feature_map_count() const { return tensor_.dimensions(1); } int64 input_filter_height() const { - return GetDim(input_filter_dims_, DimIndex::Y); + return GetDim(input_filter_dims(), DimIndex::Y); } int64 input_filter_width() const { - return GetDim(input_filter_dims_, DimIndex::X); + return GetDim(input_filter_dims(), DimIndex::X); } int64 input_filter_dim(DimIndex dim) const { - return GetDim(input_filter_dims_, dim); + return GetDim(input_filter_dims(), dim); } - FilterLayout layout() const { return layout_; } + FilterLayout layout() const { return tensor_.filter_layout(); } + absl::Span input_filter_dims() const { - return input_filter_dims_; + return AsInt64Slice(tensor_.dimensions()).subspan(2); } private: - int64 output_feature_map_count_; - int64 input_feature_map_count_; - // Stored as: ..., y, x. - std::vector input_filter_dims_; - int ndims_; - FilterLayout layout_; + absl::Span input_filter_dims() { + return AsInt64Slice(tensor_.mutable_dimensions()).subspan(2); + } + + TensorDescriptorProto tensor_; }; // Describes how padding should be aligned when the total number of pad @@ -500,6 +493,11 @@ std::ostream& operator<<(std::ostream& str, dnn::PadAlignment alignment); // cells between each filter element in the "y dimension". // - horizontal_dilation_rate: there will be (horizontal_dilation_rate - 1) // skipped cells between each filter element in the "x dimension". +// - convolution_not_crosscor: By default (convolution_not_crosscor == false), +// we perform cross correlation rather than convolution. With the flag set, +// we perform convolution. Convolution and cross correlation are related by +// rotating the filter by 180 degrees (or equivalently flipping all spatial +// dimensions). class ConvolutionDescriptor { public: // By default construction, there is no zero-padding and the filter stride is @@ -513,84 +511,102 @@ class ConvolutionDescriptor { string ToShortString() const; ConvolutionDescriptor& set_zero_padding_height(int64 value) { - SetDim(&zero_padding_, DimIndex::Y, value); + SetDim(padding(), DimIndex::Y, value); return *this; } ConvolutionDescriptor& set_zero_padding_width(int64 value) { - SetDim(&zero_padding_, DimIndex::X, value); + SetDim(padding(), DimIndex::X, value); return *this; } ConvolutionDescriptor& set_zero_padding(DimIndex dim, int64 value) { - SetDim(&zero_padding_, dim, value); + SetDim(padding(), dim, value); return *this; } ConvolutionDescriptor& set_vertical_filter_stride(int64 value) { - SetDim(&filter_strides_, DimIndex::Y, value); + SetDim(strides(), DimIndex::Y, value); return *this; } ConvolutionDescriptor& set_horizontal_filter_stride(int64 value) { - SetDim(&filter_strides_, DimIndex::X, value); + SetDim(strides(), DimIndex::X, value); return *this; } ConvolutionDescriptor& set_filter_stride(DimIndex dim, int64 value) { - SetDim(&filter_strides_, dim, value); + SetDim(strides(), dim, value); return *this; } ConvolutionDescriptor& set_vertical_dilation_rate(int64 value) { - SetDim(&dilation_rates_, DimIndex::Y, value); + SetDim(dilations(), DimIndex::Y, value); return *this; } ConvolutionDescriptor& set_horizontal_dilation_rate(int64 value) { - SetDim(&dilation_rates_, DimIndex::X, value); + SetDim(dilations(), DimIndex::X, value); return *this; } ConvolutionDescriptor& set_dilation_rate(DimIndex dim, int64 value) { - SetDim(&dilation_rates_, dim, value); + SetDim(dilations(), dim, value); return *this; } ConvolutionDescriptor& set_group_count(int group_count) { - group_count_ = group_count; + proto_.set_group_count(group_count); return *this; } - int64 zero_padding_height() const { - return GetDim(zero_padding_, DimIndex::Y); - } - int64 zero_padding_width() const { - return GetDim(zero_padding_, DimIndex::X); + ConvolutionDescriptor& set_convolution_not_crosscorr(bool conv) { + proto_.set_convolution_mode(conv ? ConvolutionMode::CONVOLUTION + : ConvolutionMode::CROSS_CORRELATION); + return *this; } + int64 zero_padding_height() const { return GetDim(padding(), DimIndex::Y); } + int64 zero_padding_width() const { return GetDim(padding(), DimIndex::X); } int64 vertical_filter_stride() const { - return GetDim(filter_strides_, DimIndex::Y); + return GetDim(strides(), DimIndex::Y); } int64 horizontal_filter_stride() const { - return GetDim(filter_strides_, DimIndex::X); + return GetDim(strides(), DimIndex::X); } int64 vertical_dilation_rate() const { - return GetDim(dilation_rates_, DimIndex::Y); + return GetDim(dilations(), DimIndex::Y); } int64 horizontal_dilation_rate() const { - return GetDim(dilation_rates_, DimIndex::X); + return GetDim(dilations(), DimIndex::X); } - int zero_padding(DimIndex dim) const { return GetDim(zero_padding_, dim); } - int filter_stride(DimIndex dim) const { return GetDim(filter_strides_, dim); } - int dilation_rate(DimIndex dim) const { return GetDim(dilation_rates_, dim); } + int zero_padding(DimIndex dim) const { return GetDim(padding(), dim); } + int filter_stride(DimIndex dim) const { return GetDim(strides(), dim); } + int dilation_rate(DimIndex dim) const { return GetDim(dilations(), dim); } // TODO(timshen): remove this function. No users of this class is setting a // non-default pad alignment. PadAlignment pad_alignment() const { return PadAlignment::kDefault; } - int group_count() const { return group_count_; } - int ndims() const { return ndims_; } + int group_count() const { return proto_.group_count(); } + int ndims() const { return padding().size(); } + bool convolution_not_crosscorr() const { + return proto_.convolution_mode() == ConvolutionMode::CONVOLUTION; + } - absl::Span strides() const { return filter_strides_; } - absl::Span dilations() const { return dilation_rates_; } - absl::Span padding() const { return zero_padding_; } + absl::Span strides() const { + return AsInt64Slice(proto_.strides()); + } + + absl::Span dilations() const { + return AsInt64Slice(proto_.dilations()); + } + + absl::Span padding() const { + return AsInt64Slice(proto_.paddings()); + } private: - // Stored as: .. y, x. - std::vector zero_padding_; - std::vector filter_strides_; - std::vector dilation_rates_; - int group_count_; - int ndims_; + absl::Span strides() { return AsInt64Slice(proto_.mutable_strides()); } + + absl::Span dilations() { + return AsInt64Slice(proto_.mutable_dilations()); + } + + absl::Span padding() { + return AsInt64Slice(proto_.mutable_paddings()); + } + + ConvolutionDescriptorProto proto_; + // TODO(leary) cudnn provides these fields, but need to characterize what // their effect is -- they may be boolean rather than integral. // int64 upscale_input_x; @@ -714,21 +730,23 @@ class PoolingDescriptor { class AlgorithmDesc { public: typedef int64 Index; - AlgorithmDesc(Index a, bool use_tensor_ops) - : algo_(a), tensor_ops_enabled_(use_tensor_ops) { - DCHECK_NE(a, -1); + AlgorithmDesc(Index a, bool use_tensor_ops) { + proto_.set_algo_id(a); + proto_.set_math_type(use_tensor_ops ? AlgorithmProto::TENSOR_OP_MATH + : AlgorithmProto::DEFAULT_MATH); } - bool tensor_ops_enabled() const { return tensor_ops_enabled_; } - Index algo_id() const { return algo_; } + bool tensor_ops_enabled() const { + return proto_.math_type() == AlgorithmProto::TENSOR_OP_MATH; + } + Index algo_id() const { return proto_.algo_id(); } bool operator==(const AlgorithmDesc& other) const { - return this->algo_ == other.algo_ && - this->tensor_ops_enabled_ == other.tensor_ops_enabled_; + return algo_id() == other.algo_id() && + tensor_ops_enabled() == other.tensor_ops_enabled(); } uint64 hash() const; private: - Index algo_; - bool tensor_ops_enabled_; + AlgorithmProto proto_; }; // Describes the result from a perf experiment. @@ -872,24 +890,6 @@ class NormalizeDescriptor { int32 segment_size_; }; -// Describes a kind of non-linearity (threshold-like mathematical function). -enum class ActivationMode { - kNone = 0, - kSigmoid, - // Rectified linear activation: f(x) = x < 0 ? 0 : x - kRelu, - // Rectified linear activation, where upper maximum is 6.0. - kRelu6, - // Rectified linear activation, where upper maximum specified by - // BatchDescriptor::value_max(). - kReluX, - kTanh, - // Like ReluX, but passes all values in the range [-X,X]. - kBandPass, - - kNumActivationModes, // Always in the end. -}; - // Returns a string representation of the given activation mode. string ActivationModeString(ActivationMode mode); diff --git a/tensorflow/stream_executor/dnn.proto b/tensorflow/stream_executor/dnn.proto new file mode 100644 index 00000000000..56b079c3f5b --- /dev/null +++ b/tensorflow/stream_executor/dnn.proto @@ -0,0 +1,103 @@ +// LINT: LEGACY_NAMES +syntax = "proto3"; + +package stream_executor.dnn; + +// Specifies the data type used by an operation. +enum DataType { + kFloat = 0; + kDouble = 1; + kHalf = 2; + kInt8 = 3; + kInt32 = 4; +} + +// Describes how a convolution input or output layer's data is formatted. +enum DataLayout { + // Naming convention: + // Y <-> row or height + // X <-> column or width + // Batch <-> batch, or N + // Depth <-> feature, or channel + // TODO(timshen): turn them into cuDNN names, e.g. kNCHW. + kYXDepthBatch = 0; + kYXBatchDepth = 1; + kBatchYXDepth = 2; // cuDNN's NHWC layout + kBatchDepthYX = 3; // cuDNN's NCHW layout + kBatchDepthYX4 = 4; // cuDNN's NCHW_VECT_C layout +} + +// Describes how a convolution filter is laid out in the memory. +enum FilterLayout { + // Naming convention: + // Y <-> row or height + // X <-> column or width + // Output <-> output feature, or N + // Input <-> input feature, or N + // TODO(timshen): turn them into cuDNN names, e.g. kNCHW. + kOutputInputYX = 0; // cuDNN's NCHW layout + kOutputYXInput = 1; // cuDNN's NHWC layout + kOutputInputYX4 = 2; // cuDNN's NCHW_VECT_C layout + kInputYXOutput = 3; + kYXInputOutput = 4; +} + +// Describes a kind of non-linearity (threshold-like mathematical function). +enum ActivationMode { + kNone = 0; + kSigmoid = 1; + // Rectified linear activation: f(x) = x < 0 ? 0 : x + kRelu = 2; + // Rectified linear activation; where upper maximum is 6.0. + kRelu6 = 3; + // Rectified linear activation; where upper maximum specified by + // BatchDescriptor::value_max(). + kReluX = 4; + kTanh = 5; + // Like ReluX; but passes all values in the range [-X,X]. + kBandPass = 6; +} + +// Describe the math definition for the conv op. The popular behavior is +// actually called cross-correlation in math, despite the operation is often +// referred as convolution. See cuDNN cudnnConvolutionMode_t. +enum ConvolutionMode { + CROSS_CORRELATION = 0; + CONVOLUTION = 1; +} + +// Generic tensor representation. +message TensorDescriptorProto { + repeated int64 dimensions = 1; + DataType data_type = 2; + oneof layout_oneof { + DataLayout data_layout = 3; + FilterLayout filter_layout = 4; + } +} + +// Generic algorithm representation. +message AlgorithmProto { + enum MathType { + DEFAULT_MATH = 0; + // The GPU may operate 4x4 matrix FMA. + // See cuDNN's documentation for CUDNN_TENSOR_OP_MATH. + TENSOR_OP_MATH = 1; + } + int64 algo_id = 1; + MathType math_type = 2; +} + +// Convolution-specific parameters. +message ConvolutionDescriptorProto { + repeated int64 paddings = 1; + repeated int64 strides = 2; + repeated int64 dilations = 3; + // The "accumulator" type. For example, use F32 as an accumulator for F16 + // convolutions. + // See cuDNN's cudnnConvolutionMode_t. + DataType compute_mode = 4; + // See cuDNN's group count. + int32 group_count = 5; + ConvolutionMode convolution_mode = 6; +} diff --git a/tensorflow/stream_executor/host/host_gpu_executor.cc b/tensorflow/stream_executor/host/host_gpu_executor.cc index 8adf739b170..1396a83dfb1 100644 --- a/tensorflow/stream_executor/host/host_gpu_executor.cc +++ b/tensorflow/stream_executor/host/host_gpu_executor.cc @@ -148,8 +148,13 @@ port::Status HostExecutor::SynchronousMemcpyDeviceToDevice( } bool HostExecutor::HostCallback(Stream *stream, - std::function callback) { - AsHostStream(stream)->EnqueueTask(callback); + std::function callback) { + AsHostStream(stream)->EnqueueTask([callback]() { + port::Status s = callback(); + if (!s.ok()) { + LOG(WARNING) << "Host callback failed: " << s; + } + }); return true; } diff --git a/tensorflow/stream_executor/host/host_gpu_executor.h b/tensorflow/stream_executor/host/host_gpu_executor.h index 7ba1f181015..56e3c2aa6a9 100644 --- a/tensorflow/stream_executor/host/host_gpu_executor.h +++ b/tensorflow/stream_executor/host/host_gpu_executor.h @@ -103,7 +103,8 @@ class HostExecutor : public internal::StreamExecutorInterface { const DeviceMemoryBase &gpu_src, uint64 size) override; - bool HostCallback(Stream *stream, std::function callback) override; + bool HostCallback(Stream *stream, + std::function callback) override; port::Status AllocateEvent(Event *event) override { return port::Status(port::error::UNIMPLEMENTED, ""); diff --git a/tensorflow/stream_executor/stream.cc b/tensorflow/stream_executor/stream.cc index 5421e4f4a5e..3edc66cde80 100644 --- a/tensorflow/stream_executor/stream.cc +++ b/tensorflow/stream_executor/stream.cc @@ -191,8 +191,11 @@ string ToVlogString(dnn::DataType data_type) { return "dnn::DataType::kHalf"; case dnn::DataType::kInt8: return "dnn::DataType::kInt8"; + case dnn::DataType::kInt32: + return "dnn::DataType::kInt32"; + default: + return "unknown DataType"; } - return "unknown DataType"; } // Used together with PARAM to VLOG calls made to the stream. Intended diff --git a/tensorflow/stream_executor/stream.h b/tensorflow/stream_executor/stream.h index e1629b5b308..0fc90cf83d6 100644 --- a/tensorflow/stream_executor/stream.h +++ b/tensorflow/stream_executor/stream.h @@ -2033,9 +2033,20 @@ class Stream { // transferred to the caller. internal::StreamInterface *implementation() { return implementation_.get(); } + // Entrains onto the stream a callback to the host (from the device). + // Behaves as ThenDoHostCallbackWithStatus below, but the callback should + // never fail or its failure is inconsequential. + // + // This is kept for backward compatibility. Future code should use + // ThenDoHostCallbackWithStatus and explicitly return a success status. + // TODO(b/112125301): Eventually remove this method. + Stream &ThenDoHostCallback(std::function callback); + // Entrains onto the stream a callback to the host (from the device). // Host callbacks block/occupy the stream just as device functions // (execute one at a time, block later stream operations). + // Whether the callback return status affects the result of BlockHostUntilDone + // is platform-dependent. // // Behavior is undefined when synchronizing using OpenCL user events. // Behavior is undefined if host callbacks call device routines or insert @@ -2043,11 +2054,6 @@ class Stream { // // On certain platforms, ThenDoHostCallback is expected to have significant // negative effects on performance. - Stream &ThenDoHostCallback(std::function callback); - - // Entrains onto the stream a callback to the host (from the device). - // Behaves as ThenDoHostCallback above, but returns a Status instead of void. - // This overload should be preferred if the callback could fail. Stream &ThenDoHostCallbackWithStatus(std::function callback); // Returns the StreamExecutor (parent object) associated with this stream. diff --git a/tensorflow/stream_executor/stream_executor_internal.cc b/tensorflow/stream_executor/stream_executor_internal.cc index 7df6a361c68..341c6edccd3 100644 --- a/tensorflow/stream_executor/stream_executor_internal.cc +++ b/tensorflow/stream_executor/stream_executor_internal.cc @@ -36,16 +36,15 @@ StreamExecutorFactory* MakeOpenCLExecutorImplementation() { StreamExecutorFactory MakeHostExecutorImplementation; -// TODO(b/112125301): Consolodate this down to one implementation of -// HostCallback, taking a callback that returns a Status. -bool StreamExecutorInterface::HostCallback( - Stream* stream, std::function callback) { - return HostCallback(stream, [callback]() { - port::Status s = callback(); - if (!s.ok()) { - LOG(WARNING) << "HostCallback failed: " << s; - } - }); +// The default implementation just calls the other HostCallback method. +// It should make all existing code that uses a void() callback still work. +bool StreamExecutorInterface::HostCallback(Stream* stream, + std::function callback) { + return HostCallback( + stream, std::function([callback]() -> port::Status { + callback(); + return port::Status::OK(); + })); } } // namespace internal diff --git a/tensorflow/stream_executor/stream_executor_internal.h b/tensorflow/stream_executor/stream_executor_internal.h index 32f75fd1bc1..0c2c33cfca2 100644 --- a/tensorflow/stream_executor/stream_executor_internal.h +++ b/tensorflow/stream_executor/stream_executor_internal.h @@ -237,9 +237,9 @@ class StreamExecutorInterface { virtual bool MemcpyDeviceToDevice(Stream *stream, DeviceMemoryBase *gpu_dst, const DeviceMemoryBase &gpu_src, uint64 size) = 0; - virtual bool HostCallback(Stream *stream, std::function callback) = 0; + virtual bool HostCallback(Stream *stream, std::function callback); virtual bool HostCallback(Stream *stream, - std::function callback); + std::function callback) = 0; virtual port::Status AllocateEvent(Event *event) = 0; virtual port::Status DeallocateEvent(Event *event) = 0; virtual port::Status RecordEvent(Stream *stream, Event *event) = 0; diff --git a/tensorflow/tensorflow.bzl b/tensorflow/tensorflow.bzl index 8e5ab94b536..ed1de5a31ca 100644 --- a/tensorflow/tensorflow.bzl +++ b/tensorflow/tensorflow.bzl @@ -203,8 +203,12 @@ def if_override_eigen_strong_inline(a): "//conditions:default": [], }) -def if_not_tx2_llvm_or_windows_cuda(a): - return if_not_windows_cuda(a) +def if_nccl(a): + return select({ + "//tensorflow:no_nccl_support": [], + "//tensorflow:windows": [], + "//conditions:default": a, + }) def get_win_copts(is_external = False): WINDOWS_COPTS = [ @@ -1307,13 +1311,13 @@ def _py_wrap_cc_impl(ctx): ctx.outputs.py_out.dirname, ] args += ["-l" + f.path for f in ctx.files.swig_includes] - args += ["-I" + i for i in swig_include_dirs] + args += ["-I" + i for i in swig_include_dirs.to_list()] args += [src.path] outputs = [ctx.outputs.cc_out, ctx.outputs.py_out] ctx.action( executable = ctx.executable._swig, arguments = args, - inputs = list(inputs), + inputs = inputs.to_list(), outputs = outputs, mnemonic = "PythonSwig", progress_message = "SWIGing " + src.path, @@ -1493,7 +1497,7 @@ check_deps = rule( }, ) -def tf_custom_op_library(name, srcs = [], gpu_srcs = [], deps = [], linkopts = [], **kwargs): +def tf_custom_op_library(name, srcs = [], gpu_srcs = [], deps = [], linkopts = [], copts = [], **kwargs): """Helper to build a dynamic library (.so) from the sources containing implementations of custom ops and kernels. """ cuda_deps = [ @@ -1505,12 +1509,18 @@ def tf_custom_op_library(name, srcs = [], gpu_srcs = [], deps = [], linkopts = [ clean_dep("//tensorflow/core:stream_executor_headers_lib"), ] deps = deps + tf_custom_op_library_additional_deps() + + # Override EIGEN_STRONG_INLINE to inline when + # --define=override_eigen_strong_inline=true to avoid long compiling time. + # See https://github.com/tensorflow/tensorflow/issues/10521 + copts = copts + if_override_eigen_strong_inline(["/DEIGEN_STRONG_INLINE=inline"]) + if gpu_srcs: basename = name.split(".")[0] native.cc_library( name = basename + "_gpu", srcs = gpu_srcs, - copts = _cuda_copts() + if_tensorrt(["-DGOOGLE_TENSORRT=1"]), + copts = copts + _cuda_copts() + if_tensorrt(["-DGOOGLE_TENSORRT=1"]), features = if_cuda(["-use_header_modules"]), deps = deps + if_cuda_is_configured_compat(cuda_deps) + if_rocm_is_configured(rocm_deps), **kwargs @@ -1531,7 +1541,7 @@ def tf_custom_op_library(name, srcs = [], gpu_srcs = [], deps = [], linkopts = [ srcs = srcs, deps = deps + if_cuda_is_configured_compat(cuda_deps) + if_rocm_is_configured(rocm_deps), data = if_static([name + "_check_deps"]), - copts = tf_copts(is_external = True), + copts = copts + tf_copts(is_external = True), features = ["windows_export_all_symbols"], linkopts = linkopts + select({ "//conditions:default": [ @@ -2022,3 +2032,6 @@ register_extension_info( extension_name = "cc_library_with_android_deps", label_regex_for_dep = "{extension_name}", ) + +def tensorflow_opensource_extra_deps(): + return [] diff --git a/tensorflow/tools/api/golden/v1/tensorflow.-config-proto.-experimental.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.-config-proto.-experimental.pbtxt index f7491649c22..a1083d732a1 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.-config-proto.-experimental.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.-config-proto.-experimental.pbtxt @@ -20,7 +20,13 @@ tf_proto { label: LABEL_OPTIONAL type: TYPE_INT32 } - reserved_range { + field { + name: "use_numa_affinity" + number: 5 + label: LABEL_OPTIONAL + type: TYPE_BOOL + } + reserved_range { start: 2 end: 3 } diff --git a/tensorflow/tools/api/golden/v1/tensorflow.-config-proto.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.-config-proto.pbtxt index 53b532beab3..b505d813509 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.-config-proto.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.-config-proto.pbtxt @@ -143,6 +143,12 @@ tf_proto { label: LABEL_OPTIONAL type: TYPE_INT32 } + field { + name: "use_numa_affinity" + number: 5 + label: LABEL_OPTIONAL + type: TYPE_BOOL + } reserved_range { start: 2 end: 3 diff --git a/tensorflow/tools/api/golden/v1/tensorflow.-gradient-tape.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.-gradient-tape.pbtxt index 0a16d6ab92f..2299a009d3d 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.-gradient-tape.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.-gradient-tape.pbtxt @@ -6,10 +6,18 @@ tf_class { name: "__init__" argspec: "args=[\'self\', \'persistent\', \'watch_accessed_variables\'], varargs=None, keywords=None, defaults=[\'False\', \'True\'], " } + member_method { + name: "batch_jacobian" + argspec: "args=[\'self\', \'target\', \'source\', \'unconnected_gradients\', \'parallel_iterations\', \'experimental_use_pfor\'], varargs=None, keywords=None, defaults=[\'UnconnectedGradients.NONE\', \'None\', \'True\'], " + } member_method { name: "gradient" argspec: "args=[\'self\', \'target\', \'sources\', \'output_gradients\', \'unconnected_gradients\'], varargs=None, keywords=None, defaults=[\'None\', \'UnconnectedGradients.NONE\'], " } + member_method { + name: "jacobian" + argspec: "args=[\'self\', \'target\', \'sources\', \'unconnected_gradients\', \'parallel_iterations\', \'experimental_use_pfor\'], varargs=None, keywords=None, defaults=[\'UnconnectedGradients.NONE\', \'None\', \'True\'], " + } member_method { name: "reset" argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" diff --git a/tensorflow/tools/api/golden/v1/tensorflow.-tensor-spec.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.-tensor-spec.pbtxt new file mode 100644 index 00000000000..493dcba8922 --- /dev/null +++ b/tensorflow/tools/api/golden/v1/tensorflow.-tensor-spec.pbtxt @@ -0,0 +1,33 @@ +path: "tensorflow.TensorSpec" +tf_class { + is_instance: "" + is_instance: "" + member { + name: "dtype" + mtype: "" + } + member { + name: "name" + mtype: "" + } + member { + name: "shape" + mtype: "" + } + member_method { + name: "__init__" + argspec: "args=[\'self\', \'shape\', \'dtype\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "from_spec" + argspec: "args=[\'cls\', \'spec\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "from_tensor" + argspec: "args=[\'cls\', \'tensor\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "is_compatible_with" + argspec: "args=[\'self\', \'spec_or_tensor\'], varargs=None, keywords=None, defaults=None" + } +} diff --git a/tensorflow/tools/api/golden/v1/tensorflow.data.-dataset.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.data.-dataset.pbtxt index 8b7f63e43e2..f59082baeb2 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.data.-dataset.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.data.-dataset.pbtxt @@ -1,6 +1,7 @@ path: "tensorflow.data.Dataset" tf_class { - is_instance: "" + is_instance: "" + is_instance: "" is_instance: "" member { name: "output_classes" diff --git a/tensorflow/tools/api/golden/v1/tensorflow.data.-fixed-length-record-dataset.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.data.-fixed-length-record-dataset.pbtxt index 81358cecbc0..d73168b070e 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.data.-fixed-length-record-dataset.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.data.-fixed-length-record-dataset.pbtxt @@ -1,8 +1,9 @@ path: "tensorflow.data.FixedLengthRecordDataset" tf_class { - is_instance: "" - is_instance: "" - is_instance: "" + is_instance: "" + is_instance: "" + is_instance: "" + is_instance: "" is_instance: "" member { name: "output_classes" diff --git a/tensorflow/tools/api/golden/v1/tensorflow.data.-options.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.data.-options.pbtxt index 9d032d43de1..72fc2c3a9ee 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.data.-options.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.data.-options.pbtxt @@ -1,6 +1,7 @@ path: "tensorflow.data.Options" tf_class { is_instance: "" + is_instance: "" is_instance: "" member { name: "experimental_autotune" @@ -10,50 +11,22 @@ tf_class { name: "experimental_deterministic" mtype: "" } - member { - name: "experimental_filter_fusion" - mtype: "" - } - member { - name: "experimental_hoist_random_uniform" - mtype: "" - } - member { - name: "experimental_map_and_batch_fusion" - mtype: "" - } - member { - name: "experimental_map_and_filter_fusion" - mtype: "" - } - member { - name: "experimental_map_fusion" - mtype: "" - } - member { - name: "experimental_map_parallelization" - mtype: "" - } - member { - name: "experimental_map_vectorization" - mtype: "" - } - member { - name: "experimental_noop_elimination" - mtype: "" - } member { name: "experimental_numa_aware" mtype: "" } member { - name: "experimental_shuffle_and_repeat_fusion" + name: "experimental_optimization" mtype: "" } member { name: "experimental_stats" mtype: "" } + member { + name: "experimental_threading" + mtype: "" + } member_method { name: "__init__" argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" diff --git a/tensorflow/tools/api/golden/v1/tensorflow.data.-t-f-record-dataset.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.data.-t-f-record-dataset.pbtxt index 7b7a9ebaf08..51224cd6b45 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.data.-t-f-record-dataset.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.data.-t-f-record-dataset.pbtxt @@ -1,7 +1,9 @@ path: "tensorflow.data.TFRecordDataset" tf_class { - is_instance: "" - is_instance: "" + is_instance: "" + is_instance: "" + is_instance: "" + is_instance: "" is_instance: "" member { name: "output_classes" diff --git a/tensorflow/tools/api/golden/v1/tensorflow.data.-text-line-dataset.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.data.-text-line-dataset.pbtxt index 1c305abf68c..a10add1b7e3 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.data.-text-line-dataset.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.data.-text-line-dataset.pbtxt @@ -1,8 +1,9 @@ path: "tensorflow.data.TextLineDataset" tf_class { - is_instance: "" - is_instance: "" - is_instance: "" + is_instance: "" + is_instance: "" + is_instance: "" + is_instance: "" is_instance: "" member { name: "output_classes" diff --git a/tensorflow/tools/api/golden/v1/tensorflow.data.experimental.-csv-dataset.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.data.experimental.-csv-dataset.pbtxt index 2520e28a3c7..71b597c19c5 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.data.experimental.-csv-dataset.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.data.experimental.-csv-dataset.pbtxt @@ -1,8 +1,9 @@ path: "tensorflow.data.experimental.CsvDataset" tf_class { - is_instance: "" - is_instance: "" - is_instance: "" + is_instance: "" + is_instance: "" + is_instance: "" + is_instance: "" is_instance: "" member { name: "output_classes" diff --git a/tensorflow/tools/api/golden/v1/tensorflow.data.experimental.-optimization-options.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.data.experimental.-optimization-options.pbtxt new file mode 100644 index 00000000000..9ca75828e55 --- /dev/null +++ b/tensorflow/tools/api/golden/v1/tensorflow.data.experimental.-optimization-options.pbtxt @@ -0,0 +1,46 @@ +path: "tensorflow.data.experimental.OptimizationOptions" +tf_class { + is_instance: "" + is_instance: "" + is_instance: "" + member { + name: "filter_fusion" + mtype: "" + } + member { + name: "hoist_random_uniform" + mtype: "" + } + member { + name: "map_and_batch_fusion" + mtype: "" + } + member { + name: "map_and_filter_fusion" + mtype: "" + } + member { + name: "map_fusion" + mtype: "" + } + member { + name: "map_parallelization" + mtype: "" + } + member { + name: "map_vectorization" + mtype: "" + } + member { + name: "noop_elimination" + mtype: "" + } + member { + name: "shuffle_and_repeat_fusion" + mtype: "" + } + member_method { + name: "__init__" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } +} diff --git a/tensorflow/tools/api/golden/v1/tensorflow.data.experimental.-random-dataset.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.data.experimental.-random-dataset.pbtxt index 1dd53b1eabd..20646e87b5f 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.data.experimental.-random-dataset.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.data.experimental.-random-dataset.pbtxt @@ -1,8 +1,9 @@ path: "tensorflow.data.experimental.RandomDataset" tf_class { - is_instance: "" - is_instance: "" - is_instance: "" + is_instance: "" + is_instance: "" + is_instance: "" + is_instance: "" is_instance: "" member { name: "output_classes" diff --git a/tensorflow/tools/api/golden/v1/tensorflow.data.experimental.-sql-dataset.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.data.experimental.-sql-dataset.pbtxt index 8fdd9dc52e3..86c5ff5b0bd 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.data.experimental.-sql-dataset.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.data.experimental.-sql-dataset.pbtxt @@ -1,8 +1,9 @@ path: "tensorflow.data.experimental.SqlDataset" tf_class { - is_instance: "" - is_instance: "" - is_instance: "" + is_instance: "" + is_instance: "" + is_instance: "" + is_instance: "" is_instance: "" member { name: "output_classes" diff --git a/tensorflow/tools/api/golden/v1/tensorflow.data.experimental.-stats-options.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.data.experimental.-stats-options.pbtxt index f423eed42cc..892f8c1fb89 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.data.experimental.-stats-options.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.data.experimental.-stats-options.pbtxt @@ -1,6 +1,7 @@ path: "tensorflow.data.experimental.StatsOptions" tf_class { is_instance: "" + is_instance: "" is_instance: "" member { name: "aggregator" @@ -20,6 +21,6 @@ tf_class { } member_method { name: "__init__" - argspec: "args=[\'self\', \'aggregator\'], varargs=None, keywords=None, defaults=[\'None\'], " + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" } } diff --git a/tensorflow/tools/api/golden/v1/tensorflow.data.experimental.-threading-options.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.data.experimental.-threading-options.pbtxt new file mode 100644 index 00000000000..5b5ebf10801 --- /dev/null +++ b/tensorflow/tools/api/golden/v1/tensorflow.data.experimental.-threading-options.pbtxt @@ -0,0 +1,18 @@ +path: "tensorflow.data.experimental.ThreadingOptions" +tf_class { + is_instance: "" + is_instance: "" + is_instance: "" + member { + name: "max_intra_op_parallelism" + mtype: "" + } + member { + name: "private_threadpool_size" + mtype: "" + } + member_method { + name: "__init__" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } +} diff --git a/tensorflow/tools/api/golden/v1/tensorflow.data.experimental.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.data.experimental.pbtxt index 4c253bb8adf..f981b1af177 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.data.experimental.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.data.experimental.pbtxt @@ -12,6 +12,14 @@ tf_module { name: "CsvDataset" mtype: "" } + member { + name: "INFINITE_CARDINALITY" + mtype: "" + } + member { + name: "OptimizationOptions" + mtype: "" + } member { name: "Optional" mtype: "" @@ -40,6 +48,14 @@ tf_module { name: "TFRecordWriter" mtype: "" } + member { + name: "ThreadingOptions" + mtype: "" + } + member { + name: "UNKNOWN_CARDINALITY" + mtype: "" + } member_method { name: "Counter" argspec: "args=[\'start\', \'step\', \'dtype\'], varargs=None, keywords=None, defaults=[\'0\', \'1\', \"\"], " @@ -48,6 +64,10 @@ tf_module { name: "bucket_by_sequence_length" argspec: "args=[\'element_length_func\', \'bucket_boundaries\', \'bucket_batch_sizes\', \'padded_shapes\', \'padding_values\', \'pad_to_bucket_boundary\', \'no_padding\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'False\', \'False\'], " } + member_method { + name: "cardinality" + argspec: "args=[\'dataset\'], varargs=None, keywords=None, defaults=None" + } member_method { name: "choose_from_datasets" argspec: "args=[\'datasets\', \'choice_dataset\'], varargs=None, keywords=None, defaults=None" @@ -64,6 +84,10 @@ tf_module { name: "enumerate_dataset" argspec: "args=[\'start\'], varargs=None, keywords=None, defaults=[\'0\'], " } + member_method { + name: "filter_for_shard" + argspec: "args=[\'num_shards\', \'shard_index\'], varargs=None, keywords=None, defaults=None" + } member_method { name: "get_next_as_optional" argspec: "args=[\'iterator\'], varargs=None, keywords=None, defaults=None" @@ -90,7 +114,7 @@ tf_module { } member_method { name: "make_batched_features_dataset" - argspec: "args=[\'file_pattern\', \'batch_size\', \'features\', \'reader\', \'label_key\', \'reader_args\', \'num_epochs\', \'shuffle\', \'shuffle_buffer_size\', \'shuffle_seed\', \'prefetch_buffer_size\', \'reader_num_threads\', \'parser_num_threads\', \'sloppy_ordering\', \'drop_final_batch\'], varargs=None, keywords=None, defaults=[\"\", \'None\', \'None\', \'None\', \'True\', \'10000\', \'None\', \'-1\', \'1\', \'2\', \'False\', \'False\'], " + argspec: "args=[\'file_pattern\', \'batch_size\', \'features\', \'reader\', \'label_key\', \'reader_args\', \'num_epochs\', \'shuffle\', \'shuffle_buffer_size\', \'shuffle_seed\', \'prefetch_buffer_size\', \'reader_num_threads\', \'parser_num_threads\', \'sloppy_ordering\', \'drop_final_batch\'], varargs=None, keywords=None, defaults=[\"\", \'None\', \'None\', \'None\', \'True\', \'10000\', \'None\', \'-1\', \'1\', \'2\', \'False\', \'False\'], " } member_method { name: "make_csv_dataset" diff --git a/tensorflow/tools/api/golden/v1/tensorflow.data.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.data.pbtxt index 509bbae8332..aa474680592 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.data.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.data.pbtxt @@ -28,4 +28,12 @@ tf_module { name: "experimental" mtype: "" } + member_method { + name: "make_initializable_iterator" + argspec: "args=[\'dataset\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "make_one_shot_iterator" + argspec: "args=[\'dataset\'], varargs=None, keywords=None, defaults=None" + } } diff --git a/tensorflow/tools/api/golden/v1/tensorflow.debugging.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.debugging.pbtxt index ab6287f8cd0..8a7f1e9363b 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.debugging.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.debugging.pbtxt @@ -78,7 +78,7 @@ tf_module { } member_method { name: "assert_scalar" - argspec: "args=[\'tensor\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " + argspec: "args=[\'tensor\', \'name\', \'message\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " } member_method { name: "assert_type" diff --git a/tensorflow/tools/api/golden/v1/tensorflow.distribute.-input-context.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.distribute.-input-context.pbtxt new file mode 100644 index 00000000000..583cbc66549 --- /dev/null +++ b/tensorflow/tools/api/golden/v1/tensorflow.distribute.-input-context.pbtxt @@ -0,0 +1,25 @@ +path: "tensorflow.distribute.InputContext" +tf_class { + is_instance: "" + is_instance: "" + member { + name: "input_pipeline_id" + mtype: "" + } + member { + name: "num_input_pipelines" + mtype: "" + } + member { + name: "num_replicas_in_sync" + mtype: "" + } + member_method { + name: "__init__" + argspec: "args=[\'self\', \'num_input_pipelines\', \'input_pipeline_id\', \'num_replicas_in_sync\'], varargs=None, keywords=None, defaults=[\'1\', \'0\', \'1\'], " + } + member_method { + name: "get_per_replica_batch_size" + argspec: "args=[\'self\', \'global_batch_size\'], varargs=None, keywords=None, defaults=None" + } +} diff --git a/tensorflow/tools/api/golden/v1/tensorflow.distribute.-input-replication-mode.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.distribute.-input-replication-mode.pbtxt new file mode 100644 index 00000000000..6a7a3a97aa0 --- /dev/null +++ b/tensorflow/tools/api/golden/v1/tensorflow.distribute.-input-replication-mode.pbtxt @@ -0,0 +1,8 @@ +path: "tensorflow.distribute.InputReplicationMode" +tf_class { + is_instance: "" + member { + name: "PER_WORKER" + mtype: "" + } +} diff --git a/tensorflow/tools/api/golden/v1/tensorflow.distribute.-reduce-op.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.distribute.-reduce-op.pbtxt new file mode 100644 index 00000000000..4899f38cad2 --- /dev/null +++ b/tensorflow/tools/api/golden/v1/tensorflow.distribute.-reduce-op.pbtxt @@ -0,0 +1,12 @@ +path: "tensorflow.distribute.ReduceOp" +tf_class { + is_instance: "" + member { + name: "MEAN" + mtype: "" + } + member { + name: "SUM" + mtype: "" + } +} diff --git a/tensorflow/tools/api/golden/v1/tensorflow.distribute.-replica-context.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.distribute.-replica-context.pbtxt new file mode 100644 index 00000000000..df707e8920e --- /dev/null +++ b/tensorflow/tools/api/golden/v1/tensorflow.distribute.-replica-context.pbtxt @@ -0,0 +1,33 @@ +path: "tensorflow.distribute.ReplicaContext" +tf_class { + is_instance: "" + is_instance: "" + member { + name: "devices" + mtype: "" + } + member { + name: "distribution_strategy" + mtype: "" + } + member { + name: "num_replicas_in_sync" + mtype: "" + } + member { + name: "replica_id_in_sync_group" + mtype: "" + } + member { + name: "strategy" + mtype: "" + } + member_method { + name: "__init__" + argspec: "args=[\'self\', \'strategy\', \'replica_id_in_sync_group\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "merge_call" + argspec: "args=[\'self\', \'merge_fn\', \'args\', \'kwargs\'], varargs=None, keywords=None, defaults=[\'()\', \'None\'], " + } +} diff --git a/tensorflow/tools/api/golden/v1/tensorflow.distribute.-strategy-extended.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.distribute.-strategy-extended.pbtxt new file mode 100644 index 00000000000..77706e57133 --- /dev/null +++ b/tensorflow/tools/api/golden/v1/tensorflow.distribute.-strategy-extended.pbtxt @@ -0,0 +1,81 @@ +path: "tensorflow.distribute.StrategyExtended" +tf_class { + is_instance: "" + is_instance: "" + member { + name: "experimental_between_graph" + mtype: "" + } + member { + name: "experimental_require_static_shapes" + mtype: "" + } + member { + name: "experimental_should_init" + mtype: "" + } + member { + name: "parameter_devices" + mtype: "" + } + member { + name: "should_checkpoint" + mtype: "" + } + member { + name: "should_save_summary" + mtype: "" + } + member { + name: "worker_devices" + mtype: "" + } + member_method { + name: "__init__" + argspec: "args=[\'self\', \'container_strategy\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "batch_reduce_to" + argspec: "args=[\'self\', \'reduce_op\', \'value_destination_pairs\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "broadcast_to" + argspec: "args=[\'self\', \'tensor\', \'destinations\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "call_for_each_replica" + argspec: "args=[\'self\', \'fn\', \'args\', \'kwargs\'], varargs=None, keywords=None, defaults=[\'()\', \'None\'], " + } + member_method { + name: "colocate_vars_with" + argspec: "args=[\'self\', \'colocate_with_variable\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "experimental_run_steps_on_iterator" + argspec: "args=[\'self\', \'fn\', \'iterator\', \'iterations\', \'initial_loop_values\'], varargs=None, keywords=None, defaults=[\'1\', \'None\'], " + } + member_method { + name: "non_slot_devices" + argspec: "args=[\'self\', \'var_list\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "read_var" + argspec: "args=[\'self\', \'v\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "reduce_to" + argspec: "args=[\'self\', \'reduce_op\', \'value\', \'destinations\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "update" + argspec: "args=[\'self\', \'var\', \'fn\', \'args\', \'kwargs\', \'group\'], varargs=None, keywords=None, defaults=[\'()\', \'None\', \'True\'], " + } + member_method { + name: "update_non_slot" + argspec: "args=[\'self\', \'colocate_with\', \'fn\', \'args\', \'kwargs\', \'group\'], varargs=None, keywords=None, defaults=[\'()\', \'None\', \'True\'], " + } + member_method { + name: "value_container" + argspec: "args=[\'self\', \'value\'], varargs=None, keywords=None, defaults=None" + } +} diff --git a/tensorflow/tools/api/golden/v1/tensorflow.distribute.-strategy.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.distribute.-strategy.pbtxt new file mode 100644 index 00000000000..9eb73d2c0d9 --- /dev/null +++ b/tensorflow/tools/api/golden/v1/tensorflow.distribute.-strategy.pbtxt @@ -0,0 +1,137 @@ +path: "tensorflow.distribute.Strategy" +tf_class { + is_instance: "" + is_instance: "" + member { + name: "between_graph" + mtype: "" + } + member { + name: "extended" + mtype: "" + } + member { + name: "num_replicas_in_sync" + mtype: "" + } + member { + name: "parameter_devices" + mtype: "" + } + member { + name: "require_static_shapes" + mtype: "" + } + member { + name: "should_checkpoint" + mtype: "" + } + member { + name: "should_init" + mtype: "" + } + member { + name: "should_save_summary" + mtype: "" + } + member { + name: "worker_devices" + mtype: "" + } + member_method { + name: "__init__" + argspec: "args=[\'self\', \'extended\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "batch_reduce" + argspec: "args=[\'self\', \'aggregation\', \'value_destination_pairs\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "broadcast" + argspec: "args=[\'self\', \'tensor\', \'destinations\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "call_for_each_replica" + argspec: "args=[\'self\', \'fn\'], varargs=args, keywords=kwargs, defaults=None" + } + member_method { + name: "colocate_vars_with" + argspec: "args=[\'self\', \'colocate_with_variable\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "configure" + argspec: "args=[\'self\', \'session_config\', \'cluster_spec\', \'task_type\', \'task_id\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], " + } + member_method { + name: "distribute_dataset" + argspec: "args=[\'self\', \'dataset_fn\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "experimental_finalize" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "experimental_initialize" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "finalize" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "group" + argspec: "args=[\'self\', \'value\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "initialize" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "make_dataset_iterator" + argspec: "args=[\'self\', \'dataset\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "make_input_fn_iterator" + argspec: "args=[\'self\', \'input_fn\', \'replication_mode\'], varargs=None, keywords=None, defaults=[\'InputReplicationMode.PER_WORKER\'], " + } + member_method { + name: "non_slot_devices" + argspec: "args=[\'self\', \'var_list\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "read_var" + argspec: "args=[\'self\', \'v\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "reduce" + argspec: "args=[\'self\', \'reduce_op\', \'value\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "run_steps_on_dataset" + argspec: "args=[\'self\', \'fn\', \'iterator\', \'iterations\', \'initial_loop_values\'], varargs=None, keywords=None, defaults=[\'1\', \'None\'], " + } + member_method { + name: "scope" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "unwrap" + argspec: "args=[\'self\', \'value\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "update" + argspec: "args=[\'self\', \'var\', \'fn\'], varargs=args, keywords=kwargs, defaults=None" + } + member_method { + name: "update_config_proto" + argspec: "args=[\'self\', \'config_proto\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "update_non_slot" + argspec: "args=[\'self\', \'colocate_with\', \'fn\'], varargs=args, keywords=kwargs, defaults=None" + } + member_method { + name: "value_container" + argspec: "args=[\'self\', \'value\'], varargs=None, keywords=None, defaults=None" + } +} diff --git a/tensorflow/tools/api/golden/v1/tensorflow.distribute.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.distribute.pbtxt new file mode 100644 index 00000000000..4d833b54ba0 --- /dev/null +++ b/tensorflow/tools/api/golden/v1/tensorflow.distribute.pbtxt @@ -0,0 +1,47 @@ +path: "tensorflow.distribute" +tf_module { + member { + name: "InputContext" + mtype: "" + } + member { + name: "InputReplicationMode" + mtype: "" + } + member { + name: "ReduceOp" + mtype: "" + } + member { + name: "ReplicaContext" + mtype: "" + } + member { + name: "Strategy" + mtype: "" + } + member { + name: "StrategyExtended" + mtype: "" + } + member_method { + name: "get_loss_reduction" + argspec: "args=[], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_replica_context" + argspec: "args=[], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_strategy" + argspec: "args=[], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "has_strategy" + argspec: "args=[], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "in_cross_replica_context" + argspec: "args=[], varargs=None, keywords=None, defaults=None" + } +} diff --git a/tensorflow/tools/api/golden/v1/tensorflow.estimator.-baseline-classifier.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.estimator.-baseline-classifier.pbtxt index 32b84e90ce6..ee3a72bfce7 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.estimator.-baseline-classifier.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.estimator.-baseline-classifier.pbtxt @@ -2,6 +2,7 @@ path: "tensorflow.estimator.BaselineClassifier" tf_class { is_instance: "" is_instance: "" + is_instance: "" is_instance: "" member { name: "config" @@ -31,9 +32,13 @@ tf_class { name: "evaluate" argspec: "args=[\'self\', \'input_fn\', \'steps\', \'hooks\', \'checkpoint_path\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], " } + member_method { + name: "experimental_export_all_saved_models" + argspec: "args=[\'self\', \'export_dir_base\', \'input_receiver_fn_map\', \'assets_extra\', \'as_text\', \'checkpoint_path\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\'], " + } member_method { name: "export_saved_model" - argspec: "args=[\'self\', \'export_dir_base\', \'serving_input_receiver_fn\', \'assets_extra\', \'as_text\', \'checkpoint_path\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\'], " + argspec: "args=[\'self\', \'export_dir_base\', \'serving_input_receiver_fn\', \'assets_extra\', \'as_text\', \'checkpoint_path\', \'experimental_mode\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\', \'infer\'], " } member_method { name: "export_savedmodel" diff --git a/tensorflow/tools/api/golden/v1/tensorflow.estimator.-baseline-estimator.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.estimator.-baseline-estimator.pbtxt index 94933e7ffd6..38b27f735ff 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.estimator.-baseline-estimator.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.estimator.-baseline-estimator.pbtxt @@ -2,6 +2,7 @@ path: "tensorflow.estimator.BaselineEstimator" tf_class { is_instance: "" is_instance: "" + is_instance: "" is_instance: "" member { name: "config" @@ -31,9 +32,13 @@ tf_class { name: "evaluate" argspec: "args=[\'self\', \'input_fn\', \'steps\', \'hooks\', \'checkpoint_path\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], " } + member_method { + name: "experimental_export_all_saved_models" + argspec: "args=[\'self\', \'export_dir_base\', \'input_receiver_fn_map\', \'assets_extra\', \'as_text\', \'checkpoint_path\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\'], " + } member_method { name: "export_saved_model" - argspec: "args=[\'self\', \'export_dir_base\', \'serving_input_receiver_fn\', \'assets_extra\', \'as_text\', \'checkpoint_path\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\'], " + argspec: "args=[\'self\', \'export_dir_base\', \'serving_input_receiver_fn\', \'assets_extra\', \'as_text\', \'checkpoint_path\', \'experimental_mode\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\', \'infer\'], " } member_method { name: "export_savedmodel" diff --git a/tensorflow/tools/api/golden/v1/tensorflow.estimator.-baseline-regressor.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.estimator.-baseline-regressor.pbtxt index db7776b5bf6..3874b84d5a6 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.estimator.-baseline-regressor.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.estimator.-baseline-regressor.pbtxt @@ -2,6 +2,7 @@ path: "tensorflow.estimator.BaselineRegressor" tf_class { is_instance: "" is_instance: "" + is_instance: "" is_instance: "" member { name: "config" @@ -31,9 +32,13 @@ tf_class { name: "evaluate" argspec: "args=[\'self\', \'input_fn\', \'steps\', \'hooks\', \'checkpoint_path\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], " } + member_method { + name: "experimental_export_all_saved_models" + argspec: "args=[\'self\', \'export_dir_base\', \'input_receiver_fn_map\', \'assets_extra\', \'as_text\', \'checkpoint_path\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\'], " + } member_method { name: "export_saved_model" - argspec: "args=[\'self\', \'export_dir_base\', \'serving_input_receiver_fn\', \'assets_extra\', \'as_text\', \'checkpoint_path\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\'], " + argspec: "args=[\'self\', \'export_dir_base\', \'serving_input_receiver_fn\', \'assets_extra\', \'as_text\', \'checkpoint_path\', \'experimental_mode\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\', \'infer\'], " } member_method { name: "export_savedmodel" diff --git a/tensorflow/tools/api/golden/v1/tensorflow.estimator.-boosted-trees-classifier.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.estimator.-boosted-trees-classifier.pbtxt index d530c71482a..e138ce936ec 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.estimator.-boosted-trees-classifier.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.estimator.-boosted-trees-classifier.pbtxt @@ -3,6 +3,7 @@ tf_class { is_instance: "" is_instance: "" is_instance: "" + is_instance: "" is_instance: "" member { name: "config" @@ -32,6 +33,10 @@ tf_class { name: "evaluate" argspec: "args=[\'self\', \'input_fn\', \'steps\', \'hooks\', \'checkpoint_path\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], " } + member_method { + name: "experimental_export_all_saved_models" + argspec: "args=[\'self\', \'export_dir_base\', \'input_receiver_fn_map\', \'assets_extra\', \'as_text\', \'checkpoint_path\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\'], " + } member_method { name: "experimental_feature_importances" argspec: "args=[\'self\', \'normalize\'], varargs=None, keywords=None, defaults=[\'False\'], " @@ -42,7 +47,7 @@ tf_class { } member_method { name: "export_saved_model" - argspec: "args=[\'self\', \'export_dir_base\', \'serving_input_receiver_fn\', \'assets_extra\', \'as_text\', \'checkpoint_path\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\'], " + argspec: "args=[\'self\', \'export_dir_base\', \'serving_input_receiver_fn\', \'assets_extra\', \'as_text\', \'checkpoint_path\', \'experimental_mode\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\', \'infer\'], " } member_method { name: "export_savedmodel" diff --git a/tensorflow/tools/api/golden/v1/tensorflow.estimator.-boosted-trees-regressor.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.estimator.-boosted-trees-regressor.pbtxt index 4703c0f561a..eae0a292a96 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.estimator.-boosted-trees-regressor.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.estimator.-boosted-trees-regressor.pbtxt @@ -3,6 +3,7 @@ tf_class { is_instance: "" is_instance: "" is_instance: "" + is_instance: "" is_instance: "" member { name: "config" @@ -32,6 +33,10 @@ tf_class { name: "evaluate" argspec: "args=[\'self\', \'input_fn\', \'steps\', \'hooks\', \'checkpoint_path\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], " } + member_method { + name: "experimental_export_all_saved_models" + argspec: "args=[\'self\', \'export_dir_base\', \'input_receiver_fn_map\', \'assets_extra\', \'as_text\', \'checkpoint_path\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\'], " + } member_method { name: "experimental_feature_importances" argspec: "args=[\'self\', \'normalize\'], varargs=None, keywords=None, defaults=[\'False\'], " @@ -42,7 +47,7 @@ tf_class { } member_method { name: "export_saved_model" - argspec: "args=[\'self\', \'export_dir_base\', \'serving_input_receiver_fn\', \'assets_extra\', \'as_text\', \'checkpoint_path\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\'], " + argspec: "args=[\'self\', \'export_dir_base\', \'serving_input_receiver_fn\', \'assets_extra\', \'as_text\', \'checkpoint_path\', \'experimental_mode\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\', \'infer\'], " } member_method { name: "export_savedmodel" diff --git a/tensorflow/tools/api/golden/v1/tensorflow.estimator.-d-n-n-classifier.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.estimator.-d-n-n-classifier.pbtxt index ce6040d0f27..b54133b294e 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.estimator.-d-n-n-classifier.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.estimator.-d-n-n-classifier.pbtxt @@ -2,6 +2,7 @@ path: "tensorflow.estimator.DNNClassifier" tf_class { is_instance: "" is_instance: "" + is_instance: "" is_instance: "" member { name: "config" @@ -31,9 +32,13 @@ tf_class { name: "evaluate" argspec: "args=[\'self\', \'input_fn\', \'steps\', \'hooks\', \'checkpoint_path\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], " } + member_method { + name: "experimental_export_all_saved_models" + argspec: "args=[\'self\', \'export_dir_base\', \'input_receiver_fn_map\', \'assets_extra\', \'as_text\', \'checkpoint_path\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\'], " + } member_method { name: "export_saved_model" - argspec: "args=[\'self\', \'export_dir_base\', \'serving_input_receiver_fn\', \'assets_extra\', \'as_text\', \'checkpoint_path\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\'], " + argspec: "args=[\'self\', \'export_dir_base\', \'serving_input_receiver_fn\', \'assets_extra\', \'as_text\', \'checkpoint_path\', \'experimental_mode\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\', \'infer\'], " } member_method { name: "export_savedmodel" diff --git a/tensorflow/tools/api/golden/v1/tensorflow.estimator.-d-n-n-estimator.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.estimator.-d-n-n-estimator.pbtxt index 4635a1544c3..09e0d381924 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.estimator.-d-n-n-estimator.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.estimator.-d-n-n-estimator.pbtxt @@ -2,6 +2,7 @@ path: "tensorflow.estimator.DNNEstimator" tf_class { is_instance: "" is_instance: "" + is_instance: "" is_instance: "" member { name: "config" @@ -31,9 +32,13 @@ tf_class { name: "evaluate" argspec: "args=[\'self\', \'input_fn\', \'steps\', \'hooks\', \'checkpoint_path\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], " } + member_method { + name: "experimental_export_all_saved_models" + argspec: "args=[\'self\', \'export_dir_base\', \'input_receiver_fn_map\', \'assets_extra\', \'as_text\', \'checkpoint_path\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\'], " + } member_method { name: "export_saved_model" - argspec: "args=[\'self\', \'export_dir_base\', \'serving_input_receiver_fn\', \'assets_extra\', \'as_text\', \'checkpoint_path\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\'], " + argspec: "args=[\'self\', \'export_dir_base\', \'serving_input_receiver_fn\', \'assets_extra\', \'as_text\', \'checkpoint_path\', \'experimental_mode\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\', \'infer\'], " } member_method { name: "export_savedmodel" diff --git a/tensorflow/tools/api/golden/v1/tensorflow.estimator.-d-n-n-linear-combined-classifier.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.estimator.-d-n-n-linear-combined-classifier.pbtxt index e85007e16ed..5a1d85a9b10 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.estimator.-d-n-n-linear-combined-classifier.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.estimator.-d-n-n-linear-combined-classifier.pbtxt @@ -2,6 +2,7 @@ path: "tensorflow.estimator.DNNLinearCombinedClassifier" tf_class { is_instance: "" is_instance: "" + is_instance: "" is_instance: "" member { name: "config" @@ -31,9 +32,13 @@ tf_class { name: "evaluate" argspec: "args=[\'self\', \'input_fn\', \'steps\', \'hooks\', \'checkpoint_path\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], " } + member_method { + name: "experimental_export_all_saved_models" + argspec: "args=[\'self\', \'export_dir_base\', \'input_receiver_fn_map\', \'assets_extra\', \'as_text\', \'checkpoint_path\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\'], " + } member_method { name: "export_saved_model" - argspec: "args=[\'self\', \'export_dir_base\', \'serving_input_receiver_fn\', \'assets_extra\', \'as_text\', \'checkpoint_path\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\'], " + argspec: "args=[\'self\', \'export_dir_base\', \'serving_input_receiver_fn\', \'assets_extra\', \'as_text\', \'checkpoint_path\', \'experimental_mode\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\', \'infer\'], " } member_method { name: "export_savedmodel" diff --git a/tensorflow/tools/api/golden/v1/tensorflow.estimator.-d-n-n-linear-combined-estimator.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.estimator.-d-n-n-linear-combined-estimator.pbtxt index a23f5daeac4..e311f96d3dc 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.estimator.-d-n-n-linear-combined-estimator.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.estimator.-d-n-n-linear-combined-estimator.pbtxt @@ -2,6 +2,7 @@ path: "tensorflow.estimator.DNNLinearCombinedEstimator" tf_class { is_instance: "" is_instance: "" + is_instance: "" is_instance: "" member { name: "config" @@ -31,9 +32,13 @@ tf_class { name: "evaluate" argspec: "args=[\'self\', \'input_fn\', \'steps\', \'hooks\', \'checkpoint_path\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], " } + member_method { + name: "experimental_export_all_saved_models" + argspec: "args=[\'self\', \'export_dir_base\', \'input_receiver_fn_map\', \'assets_extra\', \'as_text\', \'checkpoint_path\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\'], " + } member_method { name: "export_saved_model" - argspec: "args=[\'self\', \'export_dir_base\', \'serving_input_receiver_fn\', \'assets_extra\', \'as_text\', \'checkpoint_path\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\'], " + argspec: "args=[\'self\', \'export_dir_base\', \'serving_input_receiver_fn\', \'assets_extra\', \'as_text\', \'checkpoint_path\', \'experimental_mode\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\', \'infer\'], " } member_method { name: "export_savedmodel" diff --git a/tensorflow/tools/api/golden/v1/tensorflow.estimator.-d-n-n-linear-combined-regressor.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.estimator.-d-n-n-linear-combined-regressor.pbtxt index 8a55bb835ff..db4780e4c01 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.estimator.-d-n-n-linear-combined-regressor.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.estimator.-d-n-n-linear-combined-regressor.pbtxt @@ -2,6 +2,7 @@ path: "tensorflow.estimator.DNNLinearCombinedRegressor" tf_class { is_instance: "" is_instance: "" + is_instance: "" is_instance: "" member { name: "config" @@ -31,9 +32,13 @@ tf_class { name: "evaluate" argspec: "args=[\'self\', \'input_fn\', \'steps\', \'hooks\', \'checkpoint_path\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], " } + member_method { + name: "experimental_export_all_saved_models" + argspec: "args=[\'self\', \'export_dir_base\', \'input_receiver_fn_map\', \'assets_extra\', \'as_text\', \'checkpoint_path\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\'], " + } member_method { name: "export_saved_model" - argspec: "args=[\'self\', \'export_dir_base\', \'serving_input_receiver_fn\', \'assets_extra\', \'as_text\', \'checkpoint_path\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\'], " + argspec: "args=[\'self\', \'export_dir_base\', \'serving_input_receiver_fn\', \'assets_extra\', \'as_text\', \'checkpoint_path\', \'experimental_mode\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\', \'infer\'], " } member_method { name: "export_savedmodel" diff --git a/tensorflow/tools/api/golden/v1/tensorflow.estimator.-d-n-n-regressor.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.estimator.-d-n-n-regressor.pbtxt index 2c4128ec480..a44e719099e 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.estimator.-d-n-n-regressor.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.estimator.-d-n-n-regressor.pbtxt @@ -2,6 +2,7 @@ path: "tensorflow.estimator.DNNRegressor" tf_class { is_instance: "" is_instance: "" + is_instance: "" is_instance: "" member { name: "config" @@ -31,9 +32,13 @@ tf_class { name: "evaluate" argspec: "args=[\'self\', \'input_fn\', \'steps\', \'hooks\', \'checkpoint_path\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], " } + member_method { + name: "experimental_export_all_saved_models" + argspec: "args=[\'self\', \'export_dir_base\', \'input_receiver_fn_map\', \'assets_extra\', \'as_text\', \'checkpoint_path\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\'], " + } member_method { name: "export_saved_model" - argspec: "args=[\'self\', \'export_dir_base\', \'serving_input_receiver_fn\', \'assets_extra\', \'as_text\', \'checkpoint_path\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\'], " + argspec: "args=[\'self\', \'export_dir_base\', \'serving_input_receiver_fn\', \'assets_extra\', \'as_text\', \'checkpoint_path\', \'experimental_mode\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\', \'infer\'], " } member_method { name: "export_savedmodel" diff --git a/tensorflow/tools/api/golden/v1/tensorflow.estimator.-estimator.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.estimator.-estimator.pbtxt index 9d270a87ab8..bff6c86cd75 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.estimator.-estimator.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.estimator.-estimator.pbtxt @@ -1,6 +1,7 @@ path: "tensorflow.estimator.Estimator" tf_class { is_instance: "" + is_instance: "" is_instance: "" member { name: "config" @@ -30,9 +31,13 @@ tf_class { name: "evaluate" argspec: "args=[\'self\', \'input_fn\', \'steps\', \'hooks\', \'checkpoint_path\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], " } + member_method { + name: "experimental_export_all_saved_models" + argspec: "args=[\'self\', \'export_dir_base\', \'input_receiver_fn_map\', \'assets_extra\', \'as_text\', \'checkpoint_path\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\'], " + } member_method { name: "export_saved_model" - argspec: "args=[\'self\', \'export_dir_base\', \'serving_input_receiver_fn\', \'assets_extra\', \'as_text\', \'checkpoint_path\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\'], " + argspec: "args=[\'self\', \'export_dir_base\', \'serving_input_receiver_fn\', \'assets_extra\', \'as_text\', \'checkpoint_path\', \'experimental_mode\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\', \'infer\'], " } member_method { name: "export_savedmodel" diff --git a/tensorflow/tools/api/golden/v1/tensorflow.estimator.-linear-classifier.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.estimator.-linear-classifier.pbtxt index 4b5de2e2450..2c8e82517be 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.estimator.-linear-classifier.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.estimator.-linear-classifier.pbtxt @@ -2,6 +2,7 @@ path: "tensorflow.estimator.LinearClassifier" tf_class { is_instance: "" is_instance: "" + is_instance: "" is_instance: "" member { name: "config" @@ -31,9 +32,13 @@ tf_class { name: "evaluate" argspec: "args=[\'self\', \'input_fn\', \'steps\', \'hooks\', \'checkpoint_path\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], " } + member_method { + name: "experimental_export_all_saved_models" + argspec: "args=[\'self\', \'export_dir_base\', \'input_receiver_fn_map\', \'assets_extra\', \'as_text\', \'checkpoint_path\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\'], " + } member_method { name: "export_saved_model" - argspec: "args=[\'self\', \'export_dir_base\', \'serving_input_receiver_fn\', \'assets_extra\', \'as_text\', \'checkpoint_path\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\'], " + argspec: "args=[\'self\', \'export_dir_base\', \'serving_input_receiver_fn\', \'assets_extra\', \'as_text\', \'checkpoint_path\', \'experimental_mode\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\', \'infer\'], " } member_method { name: "export_savedmodel" diff --git a/tensorflow/tools/api/golden/v1/tensorflow.estimator.-linear-estimator.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.estimator.-linear-estimator.pbtxt index 3d6b03098aa..2148374fdee 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.estimator.-linear-estimator.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.estimator.-linear-estimator.pbtxt @@ -2,6 +2,7 @@ path: "tensorflow.estimator.LinearEstimator" tf_class { is_instance: "" is_instance: "" + is_instance: "" is_instance: "" member { name: "config" @@ -31,9 +32,13 @@ tf_class { name: "evaluate" argspec: "args=[\'self\', \'input_fn\', \'steps\', \'hooks\', \'checkpoint_path\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], " } + member_method { + name: "experimental_export_all_saved_models" + argspec: "args=[\'self\', \'export_dir_base\', \'input_receiver_fn_map\', \'assets_extra\', \'as_text\', \'checkpoint_path\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\'], " + } member_method { name: "export_saved_model" - argspec: "args=[\'self\', \'export_dir_base\', \'serving_input_receiver_fn\', \'assets_extra\', \'as_text\', \'checkpoint_path\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\'], " + argspec: "args=[\'self\', \'export_dir_base\', \'serving_input_receiver_fn\', \'assets_extra\', \'as_text\', \'checkpoint_path\', \'experimental_mode\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\', \'infer\'], " } member_method { name: "export_savedmodel" diff --git a/tensorflow/tools/api/golden/v1/tensorflow.estimator.-linear-regressor.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.estimator.-linear-regressor.pbtxt index 0d1510e9ab1..1bdc6124fe9 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.estimator.-linear-regressor.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.estimator.-linear-regressor.pbtxt @@ -2,6 +2,7 @@ path: "tensorflow.estimator.LinearRegressor" tf_class { is_instance: "" is_instance: "" + is_instance: "" is_instance: "" member { name: "config" @@ -31,9 +32,13 @@ tf_class { name: "evaluate" argspec: "args=[\'self\', \'input_fn\', \'steps\', \'hooks\', \'checkpoint_path\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], " } + member_method { + name: "experimental_export_all_saved_models" + argspec: "args=[\'self\', \'export_dir_base\', \'input_receiver_fn_map\', \'assets_extra\', \'as_text\', \'checkpoint_path\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\'], " + } member_method { name: "export_saved_model" - argspec: "args=[\'self\', \'export_dir_base\', \'serving_input_receiver_fn\', \'assets_extra\', \'as_text\', \'checkpoint_path\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\'], " + argspec: "args=[\'self\', \'export_dir_base\', \'serving_input_receiver_fn\', \'assets_extra\', \'as_text\', \'checkpoint_path\', \'experimental_mode\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\', \'infer\'], " } member_method { name: "export_savedmodel" diff --git a/tensorflow/tools/api/golden/v2/tensorflow.train.-profiler-hook.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.estimator.experimental.-in-memory-evaluator-hook.pbtxt similarity index 71% rename from tensorflow/tools/api/golden/v2/tensorflow.train.-profiler-hook.pbtxt rename to tensorflow/tools/api/golden/v1/tensorflow.estimator.experimental.-in-memory-evaluator-hook.pbtxt index 4df6c4156a8..aba120218cc 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.train.-profiler-hook.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.estimator.experimental.-in-memory-evaluator-hook.pbtxt @@ -1,11 +1,11 @@ -path: "tensorflow.train.ProfilerHook" +path: "tensorflow.estimator.experimental.InMemoryEvaluatorHook" tf_class { - is_instance: "" + is_instance: "" is_instance: "" is_instance: "" member_method { name: "__init__" - argspec: "args=[\'self\', \'save_steps\', \'save_secs\', \'output_dir\', \'show_dataflow\', \'show_memory\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'\', \'True\', \'False\'], " + argspec: "args=[\'self\', \'estimator\', \'input_fn\', \'steps\', \'hooks\', \'name\', \'every_n_iter\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'100\'], " } member_method { name: "after_create_session" diff --git a/tensorflow/tools/api/golden/v1/tensorflow.estimator.experimental.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.estimator.experimental.pbtxt index cabca3e883f..f0fd7ce782d 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.estimator.experimental.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.estimator.experimental.pbtxt @@ -1,9 +1,17 @@ path: "tensorflow.estimator.experimental" tf_module { + member { + name: "InMemoryEvaluatorHook" + mtype: "" + } member { name: "LinearSDCA" mtype: "" } + member_method { + name: "build_raw_supervised_input_receiver_fn" + argspec: "args=[\'features\', \'labels\', \'default_batch_size\'], varargs=None, keywords=None, defaults=[\'None\'], " + } member_method { name: "call_logit_fn" argspec: "args=[\'logit_fn\', \'features\', \'mode\', \'params\', \'config\'], varargs=None, keywords=None, defaults=None" @@ -20,6 +28,10 @@ tf_module { name: "make_early_stopping_hook" argspec: "args=[\'estimator\', \'should_stop_fn\', \'run_every_secs\', \'run_every_steps\'], varargs=None, keywords=None, defaults=[\'60\', \'None\'], " } + member_method { + name: "make_stop_at_checkpoint_step_hook" + argspec: "args=[\'estimator\', \'last_step\', \'wait_after_file_check_secs\'], varargs=None, keywords=None, defaults=[\'30\'], " + } member_method { name: "stop_if_higher_hook" argspec: "args=[\'estimator\', \'metric_name\', \'threshold\', \'eval_dir\', \'min_steps\', \'run_every_secs\', \'run_every_steps\'], varargs=None, keywords=None, defaults=[\'None\', \'0\', \'60\', \'None\'], " diff --git a/tensorflow/tools/api/golden/v1/tensorflow.image.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.image.pbtxt index 0a231f1b651..15d0e099bab 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.image.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.image.pbtxt @@ -172,6 +172,10 @@ tf_module { name: "random_saturation" argspec: "args=[\'image\', \'lower\', \'upper\', \'seed\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "resize" + argspec: "args=[\'images\', \'size\', \'method\', \'align_corners\', \'preserve_aspect_ratio\'], varargs=None, keywords=None, defaults=[\'0\', \'False\', \'False\'], " + } member_method { name: "resize_area" argspec: "args=[\'images\', \'size\', \'align_corners\', \'name\'], varargs=None, keywords=None, defaults=[\'False\', \'None\'], " @@ -240,6 +244,10 @@ tf_module { name: "total_variation" argspec: "args=[\'images\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "transpose" + argspec: "args=[\'image\'], varargs=None, keywords=None, defaults=None" + } member_method { name: "transpose_image" argspec: "args=[\'image\'], varargs=None, keywords=None, defaults=None" diff --git a/tensorflow/tools/api/golden/v1/tensorflow.io.gfile.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.io.gfile.pbtxt new file mode 100644 index 00000000000..93d9b0fd75b --- /dev/null +++ b/tensorflow/tools/api/golden/v1/tensorflow.io.gfile.pbtxt @@ -0,0 +1,51 @@ +path: "tensorflow.io.gfile" +tf_module { + member_method { + name: "copy" + argspec: "args=[\'src\', \'dst\', \'overwrite\'], varargs=None, keywords=None, defaults=[\'False\'], " + } + member_method { + name: "exists" + argspec: "args=[\'path\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "glob" + argspec: "args=[\'pattern\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "isdir" + argspec: "args=[\'path\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "listdir" + argspec: "args=[\'path\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "makedirs" + argspec: "args=[\'path\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "mkdir" + argspec: "args=[\'path\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "remove" + argspec: "args=[\'path\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "rename" + argspec: "args=[\'src\', \'dst\', \'overwrite\'], varargs=None, keywords=None, defaults=[\'False\'], " + } + member_method { + name: "rmtree" + argspec: "args=[\'path\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "stat" + argspec: "args=[\'path\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "walk" + argspec: "args=[\'top\', \'topdown\', \'onerror\'], varargs=None, keywords=None, defaults=[\'None\'], " + } +} diff --git a/tensorflow/tools/api/golden/v1/tensorflow.io.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.io.pbtxt index 64b63ed1a4a..b760ec38906 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.io.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.io.pbtxt @@ -44,10 +44,22 @@ tf_module { name: "VarLenFeature" mtype: "" } + member { + name: "gfile" + mtype: "" + } + member_method { + name: "decode_and_crop_jpeg" + argspec: "args=[\'contents\', \'crop_window\', \'channels\', \'ratio\', \'fancy_upscaling\', \'try_recover_truncated\', \'acceptable_fraction\', \'dct_method\', \'name\'], varargs=None, keywords=None, defaults=[\'0\', \'1\', \'True\', \'False\', \'1\', \'\', \'None\'], " + } member_method { name: "decode_base64" argspec: "args=[\'input\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "decode_bmp" + argspec: "args=[\'contents\', \'channels\', \'name\'], varargs=None, keywords=None, defaults=[\'0\', \'None\'], " + } member_method { name: "decode_compressed" argspec: "args=[\'bytes\', \'compression_type\', \'name\'], varargs=None, keywords=None, defaults=[\'\', \'None\'], " @@ -56,10 +68,26 @@ tf_module { name: "decode_csv" argspec: "args=[\'records\', \'record_defaults\', \'field_delim\', \'use_quote_delim\', \'name\', \'na_value\', \'select_cols\'], varargs=None, keywords=None, defaults=[\',\', \'True\', \'None\', \'\', \'None\'], " } + member_method { + name: "decode_gif" + argspec: "args=[\'contents\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "decode_image" + argspec: "args=[\'contents\', \'channels\', \'dtype\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \"\", \'None\'], " + } + member_method { + name: "decode_jpeg" + argspec: "args=[\'contents\', \'channels\', \'ratio\', \'fancy_upscaling\', \'try_recover_truncated\', \'acceptable_fraction\', \'dct_method\', \'name\'], varargs=None, keywords=None, defaults=[\'0\', \'1\', \'True\', \'False\', \'1\', \'\', \'None\'], " + } member_method { name: "decode_json_example" argspec: "args=[\'json_examples\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "decode_png" + argspec: "args=[\'contents\', \'channels\', \'dtype\', \'name\'], varargs=None, keywords=None, defaults=[\'0\', \"\", \'None\'], " + } member_method { name: "decode_raw" argspec: "args=[\'bytes\', \'out_type\', \'little_endian\', \'name\'], varargs=None, keywords=None, defaults=[\'True\', \'None\'], " @@ -72,6 +100,18 @@ tf_module { name: "encode_base64" argspec: "args=[\'input\', \'pad\', \'name\'], varargs=None, keywords=None, defaults=[\'False\', \'None\'], " } + member_method { + name: "encode_jpeg" + argspec: "args=[\'image\', \'format\', \'quality\', \'progressive\', \'optimize_size\', \'chroma_downsampling\', \'density_unit\', \'x_density\', \'y_density\', \'xmp_metadata\', \'name\'], varargs=None, keywords=None, defaults=[\'\', \'95\', \'False\', \'False\', \'True\', \'in\', \'300\', \'300\', \'\', \'None\'], " + } + member_method { + name: "extract_jpeg_shape" + argspec: "args=[\'contents\', \'output_type\', \'name\'], varargs=None, keywords=None, defaults=[\"\", \'None\'], " + } + member_method { + name: "is_jpeg" + argspec: "args=[\'contents\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " + } member_method { name: "match_filenames_once" argspec: "args=[\'pattern\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.-model.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.-model.pbtxt index 8ccba990bdd..a3254cbd947 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.keras.-model.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.-model.pbtxt @@ -41,6 +41,14 @@ tf_class { name: "losses" mtype: "" } + member { + name: "metrics" + mtype: "" + } + member { + name: "metrics_names" + mtype: "" + } member { name: "name" mtype: "" @@ -69,6 +77,10 @@ tf_class { name: "output_shape" mtype: "" } + member { + name: "run_eagerly" + mtype: "" + } member { name: "state_updates" mtype: "" @@ -105,13 +117,17 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None" } member_method { name: "add_weight" @@ -225,6 +241,10 @@ tf_class { name: "predict_on_batch" argspec: "args=[\'self\', \'x\'], varargs=None, keywords=None, defaults=None" } + member_method { + name: "reset_metrics" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } member_method { name: "reset_states" argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" @@ -247,7 +267,7 @@ tf_class { } member_method { name: "test_on_batch" - argspec: "args=[\'self\', \'x\', \'y\', \'sample_weight\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + argspec: "args=[\'self\', \'x\', \'y\', \'sample_weight\', \'reset_metrics\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'True\'], " } member_method { name: "to_json" @@ -259,6 +279,6 @@ tf_class { } member_method { name: "train_on_batch" - argspec: "args=[\'self\', \'x\', \'y\', \'sample_weight\', \'class_weight\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], " + argspec: "args=[\'self\', \'x\', \'y\', \'sample_weight\', \'class_weight\', \'reset_metrics\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\'], " } } diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.-sequential.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.-sequential.pbtxt index 27aa91a6452..b70e9ee98d5 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.keras.-sequential.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.-sequential.pbtxt @@ -42,6 +42,14 @@ tf_class { name: "losses" mtype: "" } + member { + name: "metrics" + mtype: "" + } + member { + name: "metrics_names" + mtype: "" + } member { name: "name" mtype: "" @@ -70,6 +78,10 @@ tf_class { name: "output_shape" mtype: "" } + member { + name: "run_eagerly" + mtype: "" + } member { name: "state_updates" mtype: "" @@ -110,13 +122,17 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None" } member_method { name: "add_weight" @@ -242,6 +258,10 @@ tf_class { name: "predict_proba" argspec: "args=[\'self\', \'x\', \'batch_size\', \'verbose\'], varargs=None, keywords=None, defaults=[\'32\', \'0\'], " } + member_method { + name: "reset_metrics" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } member_method { name: "reset_states" argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" @@ -264,7 +284,7 @@ tf_class { } member_method { name: "test_on_batch" - argspec: "args=[\'self\', \'x\', \'y\', \'sample_weight\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + argspec: "args=[\'self\', \'x\', \'y\', \'sample_weight\', \'reset_metrics\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'True\'], " } member_method { name: "to_json" @@ -276,6 +296,6 @@ tf_class { } member_method { name: "train_on_batch" - argspec: "args=[\'self\', \'x\', \'y\', \'sample_weight\', \'class_weight\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], " + argspec: "args=[\'self\', \'x\', \'y\', \'sample_weight\', \'class_weight\', \'reset_metrics\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\'], " } } diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.backend.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.backend.pbtxt index b0e5d2bde7d..8cd0c6ea5f0 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.keras.backend.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.backend.pbtxt @@ -398,7 +398,7 @@ tf_module { } member_method { name: "rnn" - argspec: "args=[\'step_function\', \'inputs\', \'initial_states\', \'go_backwards\', \'mask\', \'constants\', \'unroll\', \'input_length\', \'time_major\'], varargs=None, keywords=None, defaults=[\'False\', \'None\', \'None\', \'False\', \'None\', \'False\'], " + argspec: "args=[\'step_function\', \'inputs\', \'initial_states\', \'go_backwards\', \'mask\', \'constants\', \'unroll\', \'input_length\', \'time_major\', \'zero_output_for_mask\'], varargs=None, keywords=None, defaults=[\'False\', \'None\', \'None\', \'False\', \'None\', \'False\', \'False\'], " } member_method { name: "round" @@ -512,6 +512,10 @@ tf_module { name: "temporal_padding" argspec: "args=[\'x\', \'padding\'], varargs=None, keywords=None, defaults=[\'(1, 1)\'], " } + member_method { + name: "tile" + argspec: "args=[\'x\', \'n\'], varargs=None, keywords=None, defaults=None" + } member_method { name: "to_dense" argspec: "args=[\'tensor\'], varargs=None, keywords=None, defaults=None" diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.experimental.-peephole-l-s-t-m-cell.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.experimental.-peephole-l-s-t-m-cell.pbtxt index db69e25c5b7..1d814b2c8b5 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.keras.experimental.-peephole-l-s-t-m-cell.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.experimental.-peephole-l-s-t-m-cell.pbtxt @@ -89,6 +89,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-activation.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-activation.pbtxt index 5510465d7b0..b84629540e7 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-activation.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-activation.pbtxt @@ -88,6 +88,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-activity-regularization.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-activity-regularization.pbtxt index 38ec8a0aff0..5918a13ad86 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-activity-regularization.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-activity-regularization.pbtxt @@ -88,6 +88,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-add.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-add.pbtxt index 41cb8e30bfb..599da06427d 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-add.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-add.pbtxt @@ -89,6 +89,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-alpha-dropout.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-alpha-dropout.pbtxt index 9a7aaa8e961..f9ff1538c81 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-alpha-dropout.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-alpha-dropout.pbtxt @@ -88,6 +88,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-average-pooling1-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-average-pooling1-d.pbtxt index 014f5828fad..723fc9cdb0d 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-average-pooling1-d.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-average-pooling1-d.pbtxt @@ -89,6 +89,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-average-pooling2-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-average-pooling2-d.pbtxt index cc303bf7b98..957ce2f0ce8 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-average-pooling2-d.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-average-pooling2-d.pbtxt @@ -89,6 +89,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-average-pooling3-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-average-pooling3-d.pbtxt index 628447ce355..a52c0af6817 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-average-pooling3-d.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-average-pooling3-d.pbtxt @@ -89,6 +89,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-average.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-average.pbtxt index f03c986c222..a004db62ddc 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-average.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-average.pbtxt @@ -89,6 +89,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-avg-pool1-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-avg-pool1-d.pbtxt index a6e4856de9b..44f83d1387c 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-avg-pool1-d.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-avg-pool1-d.pbtxt @@ -89,6 +89,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-avg-pool2-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-avg-pool2-d.pbtxt index a01eaf8a126..8378faf7188 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-avg-pool2-d.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-avg-pool2-d.pbtxt @@ -89,6 +89,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-avg-pool3-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-avg-pool3-d.pbtxt index 0d6698f2ef4..9d5655c9644 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-avg-pool3-d.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-avg-pool3-d.pbtxt @@ -89,6 +89,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-batch-normalization.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-batch-normalization.pbtxt index f1b23be48f7..b3d3c84f92e 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-batch-normalization.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-batch-normalization.pbtxt @@ -1,6 +1,7 @@ path: "tensorflow.keras.layers.BatchNormalization" tf_class { - is_instance: "" + is_instance: "" + is_instance: "" is_instance: "" is_instance: "" is_instance: "" @@ -88,6 +89,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-bidirectional.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-bidirectional.pbtxt index 0672cd5b7b8..d37a6b47105 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-bidirectional.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-bidirectional.pbtxt @@ -97,6 +97,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-concatenate.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-concatenate.pbtxt index b25ae1e82e8..1ad7a91be0b 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-concatenate.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-concatenate.pbtxt @@ -89,6 +89,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-conv-l-s-t-m2-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-conv-l-s-t-m2-d.pbtxt index bb1918eba65..cb9abc25396 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-conv-l-s-t-m2-d.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-conv-l-s-t-m2-d.pbtxt @@ -178,6 +178,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-conv1-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-conv1-d.pbtxt index 16e0fd5a313..47dba1d81f8 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-conv1-d.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-conv1-d.pbtxt @@ -89,6 +89,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-conv2-d-transpose.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-conv2-d-transpose.pbtxt index 381839d6deb..fd649418961 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-conv2-d-transpose.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-conv2-d-transpose.pbtxt @@ -90,6 +90,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-conv2-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-conv2-d.pbtxt index 543bae6fa96..1b1425d5319 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-conv2-d.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-conv2-d.pbtxt @@ -89,6 +89,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-conv3-d-transpose.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-conv3-d-transpose.pbtxt index 2933f9f4b3a..1741063fe8b 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-conv3-d-transpose.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-conv3-d-transpose.pbtxt @@ -90,6 +90,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-conv3-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-conv3-d.pbtxt index 072943dc2c7..50feb4f458a 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-conv3-d.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-conv3-d.pbtxt @@ -89,6 +89,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-convolution1-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-convolution1-d.pbtxt index 222a1ef4fc5..faaa535df9f 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-convolution1-d.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-convolution1-d.pbtxt @@ -89,6 +89,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-convolution2-d-transpose.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-convolution2-d-transpose.pbtxt index 9c9c7461c8b..4079329d1ee 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-convolution2-d-transpose.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-convolution2-d-transpose.pbtxt @@ -90,6 +90,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-convolution2-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-convolution2-d.pbtxt index f9390671781..32e56696e16 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-convolution2-d.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-convolution2-d.pbtxt @@ -89,6 +89,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-convolution3-d-transpose.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-convolution3-d-transpose.pbtxt index 44ca598724a..381abe73401 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-convolution3-d-transpose.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-convolution3-d-transpose.pbtxt @@ -90,6 +90,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-convolution3-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-convolution3-d.pbtxt index 471b18ef850..b3e4bf9689d 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-convolution3-d.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-convolution3-d.pbtxt @@ -89,6 +89,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-cropping1-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-cropping1-d.pbtxt index 0f250a09b7e..7aeff8003c3 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-cropping1-d.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-cropping1-d.pbtxt @@ -88,6 +88,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-cropping2-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-cropping2-d.pbtxt index f52128483c6..a1728d9d4f9 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-cropping2-d.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-cropping2-d.pbtxt @@ -88,6 +88,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-cropping3-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-cropping3-d.pbtxt index 98daf3bab12..8d8fd142cc6 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-cropping3-d.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-cropping3-d.pbtxt @@ -88,6 +88,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-cu-d-n-n-g-r-u.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-cu-d-n-n-g-r-u.pbtxt index b207c680005..7758209adf8 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-cu-d-n-n-g-r-u.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-cu-d-n-n-g-r-u.pbtxt @@ -98,6 +98,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-cu-d-n-n-l-s-t-m.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-cu-d-n-n-l-s-t-m.pbtxt index 2d7a09ceda9..7c463ff1257 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-cu-d-n-n-l-s-t-m.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-cu-d-n-n-l-s-t-m.pbtxt @@ -98,6 +98,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-dense.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-dense.pbtxt index 3ac38257593..4960d0264e9 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-dense.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-dense.pbtxt @@ -88,6 +88,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-depthwise-conv2-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-depthwise-conv2-d.pbtxt index 280ec8c25fa..8fad7535f88 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-depthwise-conv2-d.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-depthwise-conv2-d.pbtxt @@ -90,6 +90,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-dot.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-dot.pbtxt index 560f66f9c7a..5b425f2d4d7 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-dot.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-dot.pbtxt @@ -89,6 +89,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-dropout.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-dropout.pbtxt index c0543529c38..f6c4d0a438e 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-dropout.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-dropout.pbtxt @@ -88,6 +88,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-e-l-u.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-e-l-u.pbtxt index 04eb2824b9b..82b761fc176 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-e-l-u.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-e-l-u.pbtxt @@ -88,6 +88,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-embedding.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-embedding.pbtxt index f400432915f..c9ff323877e 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-embedding.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-embedding.pbtxt @@ -88,6 +88,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-flatten.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-flatten.pbtxt index ab176b441a2..9b4165d4cbf 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-flatten.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-flatten.pbtxt @@ -88,6 +88,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-g-r-u-cell.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-g-r-u-cell.pbtxt index c3895a0ac12..f225f7c4309 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-g-r-u-cell.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-g-r-u-cell.pbtxt @@ -88,6 +88,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-g-r-u.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-g-r-u.pbtxt index 9e24bb8ae6a..855d0017001 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-g-r-u.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-g-r-u.pbtxt @@ -161,6 +161,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-gaussian-dropout.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-gaussian-dropout.pbtxt index 55e0d7ef023..2c404c99cd2 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-gaussian-dropout.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-gaussian-dropout.pbtxt @@ -88,6 +88,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-gaussian-noise.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-gaussian-noise.pbtxt index 38fbff5e4a3..6f109d59d0f 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-gaussian-noise.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-gaussian-noise.pbtxt @@ -88,6 +88,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-average-pooling1-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-average-pooling1-d.pbtxt index a8094c0bde3..69f8a9031d3 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-average-pooling1-d.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-average-pooling1-d.pbtxt @@ -89,6 +89,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-average-pooling2-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-average-pooling2-d.pbtxt index 929f48df231..4299f765e52 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-average-pooling2-d.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-average-pooling2-d.pbtxt @@ -89,6 +89,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-average-pooling3-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-average-pooling3-d.pbtxt index 2e6d59337f1..9153a1a2406 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-average-pooling3-d.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-average-pooling3-d.pbtxt @@ -89,6 +89,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-avg-pool1-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-avg-pool1-d.pbtxt index 3ebe162f573..625e81fd232 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-avg-pool1-d.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-avg-pool1-d.pbtxt @@ -89,6 +89,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-avg-pool2-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-avg-pool2-d.pbtxt index 4e3e258430c..2fc769742c7 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-avg-pool2-d.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-avg-pool2-d.pbtxt @@ -89,6 +89,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-avg-pool3-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-avg-pool3-d.pbtxt index fb9166316f6..e307a65c7c5 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-avg-pool3-d.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-avg-pool3-d.pbtxt @@ -89,6 +89,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-max-pool1-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-max-pool1-d.pbtxt index c0a53b847b4..4394ad0364e 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-max-pool1-d.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-max-pool1-d.pbtxt @@ -89,6 +89,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-max-pool2-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-max-pool2-d.pbtxt index 87b7f6797a0..050ed39fe98 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-max-pool2-d.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-max-pool2-d.pbtxt @@ -89,6 +89,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-max-pool3-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-max-pool3-d.pbtxt index 98bf96fa0c2..436191821ef 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-max-pool3-d.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-max-pool3-d.pbtxt @@ -89,6 +89,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-max-pooling1-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-max-pooling1-d.pbtxt index ff6c6f3ec4d..4ba540aa6ad 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-max-pooling1-d.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-max-pooling1-d.pbtxt @@ -89,6 +89,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-max-pooling2-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-max-pooling2-d.pbtxt index c9d4158d1c4..a2e9322cb3f 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-max-pooling2-d.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-max-pooling2-d.pbtxt @@ -89,6 +89,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-max-pooling3-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-max-pooling3-d.pbtxt index 9953102ff99..5d16a57fc1a 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-max-pooling3-d.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-max-pooling3-d.pbtxt @@ -89,6 +89,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-input-layer.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-input-layer.pbtxt index 2617f5a95fa..9dd29c1251e 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-input-layer.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-input-layer.pbtxt @@ -88,6 +88,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-input-spec.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-input-spec.pbtxt index 5fd0a47a68c..bc3ceb67a4e 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-input-spec.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-input-spec.pbtxt @@ -1,6 +1,6 @@ path: "tensorflow.keras.layers.InputSpec" tf_class { - is_instance: "" + is_instance: "" is_instance: "" member_method { name: "__init__" diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-l-s-t-m-cell.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-l-s-t-m-cell.pbtxt index e9f6ef45aaf..0045d5775e2 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-l-s-t-m-cell.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-l-s-t-m-cell.pbtxt @@ -88,6 +88,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-l-s-t-m.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-l-s-t-m.pbtxt index 1b1ccbe1180..529c750f987 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-l-s-t-m.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-l-s-t-m.pbtxt @@ -161,6 +161,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-lambda.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-lambda.pbtxt index 2e0b6bac24f..d4d1bc6b6bb 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-lambda.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-lambda.pbtxt @@ -88,6 +88,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-layer.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-layer.pbtxt index 1e93d1118a4..e1f54911809 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-layer.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-layer.pbtxt @@ -87,6 +87,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-leaky-re-l-u.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-leaky-re-l-u.pbtxt index bfd36012a7e..9b69d9a9447 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-leaky-re-l-u.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-leaky-re-l-u.pbtxt @@ -88,6 +88,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-locally-connected1-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-locally-connected1-d.pbtxt index 5ad5990d7e6..fd522594325 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-locally-connected1-d.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-locally-connected1-d.pbtxt @@ -88,6 +88,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-locally-connected2-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-locally-connected2-d.pbtxt index 40d03369a52..5fc8af0d035 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-locally-connected2-d.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-locally-connected2-d.pbtxt @@ -88,6 +88,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-masking.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-masking.pbtxt index 86666b51bb8..7f8932270e6 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-masking.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-masking.pbtxt @@ -88,6 +88,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-max-pool1-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-max-pool1-d.pbtxt index d26da270e74..4723b99cb07 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-max-pool1-d.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-max-pool1-d.pbtxt @@ -89,6 +89,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-max-pool2-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-max-pool2-d.pbtxt index 85f23df671d..173c5d4a8b1 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-max-pool2-d.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-max-pool2-d.pbtxt @@ -89,6 +89,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-max-pool3-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-max-pool3-d.pbtxt index 235806b9650..14e1899e145 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-max-pool3-d.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-max-pool3-d.pbtxt @@ -89,6 +89,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-max-pooling1-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-max-pooling1-d.pbtxt index 524c5fd69e5..a708e652bf0 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-max-pooling1-d.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-max-pooling1-d.pbtxt @@ -89,6 +89,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-max-pooling2-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-max-pooling2-d.pbtxt index fda2562fc8c..e6706b5cf9f 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-max-pooling2-d.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-max-pooling2-d.pbtxt @@ -89,6 +89,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-max-pooling3-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-max-pooling3-d.pbtxt index 71d2d09a8d1..a73c082d1bb 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-max-pooling3-d.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-max-pooling3-d.pbtxt @@ -89,6 +89,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-maximum.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-maximum.pbtxt index 12949b39a6f..f3f195554bb 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-maximum.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-maximum.pbtxt @@ -89,6 +89,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-minimum.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-minimum.pbtxt index ab16d0021e6..f345d1d67b2 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-minimum.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-minimum.pbtxt @@ -89,6 +89,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-multiply.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-multiply.pbtxt index 61ccbf59627..31cb8bc177c 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-multiply.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-multiply.pbtxt @@ -89,6 +89,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-p-re-l-u.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-p-re-l-u.pbtxt index ce2320d7030..44cccc92bd2 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-p-re-l-u.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-p-re-l-u.pbtxt @@ -88,6 +88,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-permute.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-permute.pbtxt index 69848af8cf8..b55e191ff1a 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-permute.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-permute.pbtxt @@ -88,6 +88,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-r-n-n.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-r-n-n.pbtxt index 3358f26aebf..e9575436e5b 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-r-n-n.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-r-n-n.pbtxt @@ -92,6 +92,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-re-l-u.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-re-l-u.pbtxt index 413f45f018a..98223b207f2 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-re-l-u.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-re-l-u.pbtxt @@ -88,6 +88,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-repeat-vector.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-repeat-vector.pbtxt index 9c61ff60274..2df918b16b2 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-repeat-vector.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-repeat-vector.pbtxt @@ -88,6 +88,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-reshape.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-reshape.pbtxt index baa91804c49..ce5f9e21290 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-reshape.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-reshape.pbtxt @@ -88,6 +88,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-separable-conv1-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-separable-conv1-d.pbtxt index 15a5d6ac9ea..a0bb917775f 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-separable-conv1-d.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-separable-conv1-d.pbtxt @@ -90,6 +90,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-separable-conv2-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-separable-conv2-d.pbtxt index be43bd5b3c1..d7942f201bd 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-separable-conv2-d.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-separable-conv2-d.pbtxt @@ -90,6 +90,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-separable-convolution1-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-separable-convolution1-d.pbtxt index 6105992c7a3..f7ac9042d46 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-separable-convolution1-d.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-separable-convolution1-d.pbtxt @@ -90,6 +90,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-separable-convolution2-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-separable-convolution2-d.pbtxt index 1b6cf1e9ecb..e5a92688220 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-separable-convolution2-d.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-separable-convolution2-d.pbtxt @@ -90,6 +90,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-simple-r-n-n-cell.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-simple-r-n-n-cell.pbtxt index 29488a37f8f..0fe2c974a76 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-simple-r-n-n-cell.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-simple-r-n-n-cell.pbtxt @@ -88,6 +88,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-simple-r-n-n.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-simple-r-n-n.pbtxt index 3d70cf8b659..2ee5873f0f1 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-simple-r-n-n.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-simple-r-n-n.pbtxt @@ -149,6 +149,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-softmax.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-softmax.pbtxt index d29731ecf9d..5b8f64aa357 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-softmax.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-softmax.pbtxt @@ -88,6 +88,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-spatial-dropout1-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-spatial-dropout1-d.pbtxt index a6d7494ca7d..240cb6e562f 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-spatial-dropout1-d.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-spatial-dropout1-d.pbtxt @@ -89,6 +89,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-spatial-dropout2-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-spatial-dropout2-d.pbtxt index c36e802693d..6226c469f8a 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-spatial-dropout2-d.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-spatial-dropout2-d.pbtxt @@ -89,6 +89,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-spatial-dropout3-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-spatial-dropout3-d.pbtxt index 9c46cfe40fd..34dabce6d8d 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-spatial-dropout3-d.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-spatial-dropout3-d.pbtxt @@ -89,6 +89,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-stacked-r-n-n-cells.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-stacked-r-n-n-cells.pbtxt index 8982f787940..0ddf628ace5 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-stacked-r-n-n-cells.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-stacked-r-n-n-cells.pbtxt @@ -96,6 +96,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-subtract.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-subtract.pbtxt index ec2cc502984..12eb35ad154 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-subtract.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-subtract.pbtxt @@ -89,6 +89,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-thresholded-re-l-u.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-thresholded-re-l-u.pbtxt index d7bc1980f32..c41020c2b45 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-thresholded-re-l-u.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-thresholded-re-l-u.pbtxt @@ -88,6 +88,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-time-distributed.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-time-distributed.pbtxt index fec2de6b49e..479f89cf6ae 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-time-distributed.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-time-distributed.pbtxt @@ -93,6 +93,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-up-sampling1-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-up-sampling1-d.pbtxt index 3d285e7f17d..233363ce026 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-up-sampling1-d.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-up-sampling1-d.pbtxt @@ -88,6 +88,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-up-sampling2-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-up-sampling2-d.pbtxt index b05e5ec84de..cb6228ac446 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-up-sampling2-d.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-up-sampling2-d.pbtxt @@ -88,6 +88,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-up-sampling3-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-up-sampling3-d.pbtxt index 728eca415a8..03bad3ccb61 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-up-sampling3-d.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-up-sampling3-d.pbtxt @@ -88,6 +88,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-wrapper.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-wrapper.pbtxt index da64e77c39c..158996792a4 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-wrapper.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-wrapper.pbtxt @@ -92,6 +92,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-zero-padding1-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-zero-padding1-d.pbtxt index 2f505f9293f..63a56cd3eeb 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-zero-padding1-d.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-zero-padding1-d.pbtxt @@ -88,6 +88,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-zero-padding2-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-zero-padding2-d.pbtxt index f82c77072e6..965a4cca046 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-zero-padding2-d.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-zero-padding2-d.pbtxt @@ -88,6 +88,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-zero-padding3-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-zero-padding3-d.pbtxt index 54e01a99177..1a624308878 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-zero-padding3-d.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-zero-padding3-d.pbtxt @@ -88,6 +88,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.losses.-binary-crossentropy.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.losses.-binary-crossentropy.pbtxt new file mode 100644 index 00000000000..2f7da93f6f4 --- /dev/null +++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.losses.-binary-crossentropy.pbtxt @@ -0,0 +1,22 @@ +path: "tensorflow.keras.losses.BinaryCrossentropy" +tf_class { + is_instance: "" + is_instance: "" + is_instance: "" + member_method { + name: "__init__" + argspec: "args=[\'self\', \'from_logits\', \'label_smoothing\', \'reduction\', \'name\'], varargs=None, keywords=None, defaults=[\'False\', \'0\', \'sum_over_batch_size\', \'None\'], " + } + member_method { + name: "call" + argspec: "args=[\'self\', \'y_true\', \'y_pred\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "from_config" + argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_config" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } +} diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.losses.-categorical-crossentropy.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.losses.-categorical-crossentropy.pbtxt new file mode 100644 index 00000000000..b3a7cd80973 --- /dev/null +++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.losses.-categorical-crossentropy.pbtxt @@ -0,0 +1,22 @@ +path: "tensorflow.keras.losses.CategoricalCrossentropy" +tf_class { + is_instance: "" + is_instance: "" + is_instance: "" + member_method { + name: "__init__" + argspec: "args=[\'self\', \'from_logits\', \'label_smoothing\', \'reduction\', \'name\'], varargs=None, keywords=None, defaults=[\'False\', \'0\', \'sum_over_batch_size\', \'None\'], " + } + member_method { + name: "call" + argspec: "args=[\'self\', \'y_true\', \'y_pred\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "from_config" + argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_config" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } +} diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.losses.-mean-absolute-error.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.losses.-mean-absolute-error.pbtxt new file mode 100644 index 00000000000..712bb2ecd35 --- /dev/null +++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.losses.-mean-absolute-error.pbtxt @@ -0,0 +1,22 @@ +path: "tensorflow.keras.losses.MeanAbsoluteError" +tf_class { + is_instance: "" + is_instance: "" + is_instance: "" + member_method { + name: "__init__" + argspec: "args=[\'self\', \'reduction\', \'name\'], varargs=None, keywords=None, defaults=[\'sum_over_batch_size\', \'None\'], " + } + member_method { + name: "call" + argspec: "args=[\'self\', \'y_true\', \'y_pred\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "from_config" + argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_config" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } +} diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.losses.-mean-absolute-percentage-error.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.losses.-mean-absolute-percentage-error.pbtxt new file mode 100644 index 00000000000..7fe362da89b --- /dev/null +++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.losses.-mean-absolute-percentage-error.pbtxt @@ -0,0 +1,22 @@ +path: "tensorflow.keras.losses.MeanAbsolutePercentageError" +tf_class { + is_instance: "" + is_instance: "" + is_instance: "" + member_method { + name: "__init__" + argspec: "args=[\'self\', \'reduction\', \'name\'], varargs=None, keywords=None, defaults=[\'sum_over_batch_size\', \'None\'], " + } + member_method { + name: "call" + argspec: "args=[\'self\', \'y_true\', \'y_pred\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "from_config" + argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_config" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } +} diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.losses.-mean-squared-error.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.losses.-mean-squared-error.pbtxt new file mode 100644 index 00000000000..a5718533500 --- /dev/null +++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.losses.-mean-squared-error.pbtxt @@ -0,0 +1,22 @@ +path: "tensorflow.keras.losses.MeanSquaredError" +tf_class { + is_instance: "" + is_instance: "" + is_instance: "" + member_method { + name: "__init__" + argspec: "args=[\'self\', \'reduction\', \'name\'], varargs=None, keywords=None, defaults=[\'sum_over_batch_size\', \'None\'], " + } + member_method { + name: "call" + argspec: "args=[\'self\', \'y_true\', \'y_pred\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "from_config" + argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_config" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } +} diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.losses.-mean-squared-logarithmic-error.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.losses.-mean-squared-logarithmic-error.pbtxt new file mode 100644 index 00000000000..200006db355 --- /dev/null +++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.losses.-mean-squared-logarithmic-error.pbtxt @@ -0,0 +1,22 @@ +path: "tensorflow.keras.losses.MeanSquaredLogarithmicError" +tf_class { + is_instance: "" + is_instance: "" + is_instance: "" + member_method { + name: "__init__" + argspec: "args=[\'self\', \'reduction\', \'name\'], varargs=None, keywords=None, defaults=[\'sum_over_batch_size\', \'None\'], " + } + member_method { + name: "call" + argspec: "args=[\'self\', \'y_true\', \'y_pred\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "from_config" + argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_config" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } +} diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.losses.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.losses.pbtxt index eca6b915388..9e26ddbdca0 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.keras.losses.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.losses.pbtxt @@ -1,5 +1,29 @@ path: "tensorflow.keras.losses" tf_module { + member { + name: "BinaryCrossentropy" + mtype: "" + } + member { + name: "CategoricalCrossentropy" + mtype: "" + } + member { + name: "MeanAbsoluteError" + mtype: "" + } + member { + name: "MeanAbsolutePercentageError" + mtype: "" + } + member { + name: "MeanSquaredError" + mtype: "" + } + member { + name: "MeanSquaredLogarithmicError" + mtype: "" + } member_method { name: "KLD" argspec: "args=[\'y_true\', \'y_pred\'], varargs=None, keywords=None, defaults=None" @@ -22,11 +46,11 @@ tf_module { } member_method { name: "binary_crossentropy" - argspec: "args=[\'y_true\', \'y_pred\'], varargs=None, keywords=None, defaults=None" + argspec: "args=[\'y_true\', \'y_pred\', \'from_logits\'], varargs=None, keywords=None, defaults=[\'False\'], " } member_method { name: "categorical_crossentropy" - argspec: "args=[\'y_true\', \'y_pred\'], varargs=None, keywords=None, defaults=None" + argspec: "args=[\'y_true\', \'y_pred\', \'from_logits\'], varargs=None, keywords=None, defaults=[\'False\'], " } member_method { name: "categorical_hinge" @@ -106,7 +130,7 @@ tf_module { } member_method { name: "sparse_categorical_crossentropy" - argspec: "args=[\'y_true\', \'y_pred\'], varargs=None, keywords=None, defaults=None" + argspec: "args=[\'y_true\', \'y_pred\', \'from_logits\'], varargs=None, keywords=None, defaults=[\'False\'], " } member_method { name: "squared_hinge" diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-accuracy.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-accuracy.pbtxt new file mode 100644 index 00000000000..2db07df5235 --- /dev/null +++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-accuracy.pbtxt @@ -0,0 +1,194 @@ +path: "tensorflow.keras.metrics.Accuracy" +tf_class { + is_instance: "" + is_instance: "" + is_instance: "" + is_instance: "" + is_instance: "" + is_instance: "" + is_instance: "" + member { + name: "activity_regularizer" + mtype: "" + } + member { + name: "dtype" + mtype: "" + } + member { + name: "inbound_nodes" + mtype: "" + } + member { + name: "input" + mtype: "" + } + member { + name: "input_mask" + mtype: "" + } + member { + name: "input_shape" + mtype: "" + } + member { + name: "losses" + mtype: "" + } + member { + name: "name" + mtype: "" + } + member { + name: "non_trainable_variables" + mtype: "" + } + member { + name: "non_trainable_weights" + mtype: "" + } + member { + name: "outbound_nodes" + mtype: "" + } + member { + name: "output" + mtype: "" + } + member { + name: "output_mask" + mtype: "" + } + member { + name: "output_shape" + mtype: "" + } + member { + name: "trainable_variables" + mtype: "" + } + member { + name: "trainable_weights" + mtype: "" + } + member { + name: "updates" + mtype: "" + } + member { + name: "variables" + mtype: "" + } + member { + name: "weights" + mtype: "" + } + member_method { + name: "__init__" + argspec: "args=[\'self\', \'name\', \'dtype\'], varargs=None, keywords=None, defaults=[\'accuracy\', \'None\'], " + } + member_method { + name: "add_loss" + argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } + member_method { + name: "add_update" + argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "add_variable" + argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None" + } + member_method { + name: "add_weight" + argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\'], " + } + member_method { + name: "apply" + argspec: "args=[\'self\', \'inputs\'], varargs=args, keywords=kwargs, defaults=None" + } + member_method { + name: "build" + argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "call" + argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=kwargs, defaults=None" + } + member_method { + name: "compute_mask" + argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "compute_output_shape" + argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "count_params" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "from_config" + argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_config" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_input_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_input_mask_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_input_shape_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_losses_for" + argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_output_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_output_mask_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_output_shape_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_updates_for" + argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_weights" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "reset_states" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "result" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "set_weights" + argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "update_state" + argspec: "args=[\'self\', \'y_true\', \'y_pred\', \'sample_weight\'], varargs=None, keywords=None, defaults=[\'None\'], " + } +} diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-binary-accuracy.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-binary-accuracy.pbtxt new file mode 100644 index 00000000000..904ad3a21a0 --- /dev/null +++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-binary-accuracy.pbtxt @@ -0,0 +1,194 @@ +path: "tensorflow.keras.metrics.BinaryAccuracy" +tf_class { + is_instance: "" + is_instance: "" + is_instance: "" + is_instance: "" + is_instance: "" + is_instance: "" + is_instance: "" + member { + name: "activity_regularizer" + mtype: "" + } + member { + name: "dtype" + mtype: "" + } + member { + name: "inbound_nodes" + mtype: "" + } + member { + name: "input" + mtype: "" + } + member { + name: "input_mask" + mtype: "" + } + member { + name: "input_shape" + mtype: "" + } + member { + name: "losses" + mtype: "" + } + member { + name: "name" + mtype: "" + } + member { + name: "non_trainable_variables" + mtype: "" + } + member { + name: "non_trainable_weights" + mtype: "" + } + member { + name: "outbound_nodes" + mtype: "" + } + member { + name: "output" + mtype: "" + } + member { + name: "output_mask" + mtype: "" + } + member { + name: "output_shape" + mtype: "" + } + member { + name: "trainable_variables" + mtype: "" + } + member { + name: "trainable_weights" + mtype: "" + } + member { + name: "updates" + mtype: "" + } + member { + name: "variables" + mtype: "" + } + member { + name: "weights" + mtype: "" + } + member_method { + name: "__init__" + argspec: "args=[\'self\', \'name\', \'dtype\', \'threshold\'], varargs=None, keywords=None, defaults=[\'binary_accuracy\', \'None\', \'0.5\'], " + } + member_method { + name: "add_loss" + argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } + member_method { + name: "add_update" + argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "add_variable" + argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None" + } + member_method { + name: "add_weight" + argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\'], " + } + member_method { + name: "apply" + argspec: "args=[\'self\', \'inputs\'], varargs=args, keywords=kwargs, defaults=None" + } + member_method { + name: "build" + argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "call" + argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=kwargs, defaults=None" + } + member_method { + name: "compute_mask" + argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "compute_output_shape" + argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "count_params" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "from_config" + argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_config" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_input_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_input_mask_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_input_shape_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_losses_for" + argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_output_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_output_mask_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_output_shape_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_updates_for" + argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_weights" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "reset_states" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "result" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "set_weights" + argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "update_state" + argspec: "args=[\'self\', \'y_true\', \'y_pred\', \'sample_weight\'], varargs=None, keywords=None, defaults=[\'None\'], " + } +} diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-categorical-accuracy.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-categorical-accuracy.pbtxt new file mode 100644 index 00000000000..17b74924fab --- /dev/null +++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-categorical-accuracy.pbtxt @@ -0,0 +1,194 @@ +path: "tensorflow.keras.metrics.CategoricalAccuracy" +tf_class { + is_instance: "" + is_instance: "" + is_instance: "" + is_instance: "" + is_instance: "" + is_instance: "" + is_instance: "" + member { + name: "activity_regularizer" + mtype: "" + } + member { + name: "dtype" + mtype: "" + } + member { + name: "inbound_nodes" + mtype: "" + } + member { + name: "input" + mtype: "" + } + member { + name: "input_mask" + mtype: "" + } + member { + name: "input_shape" + mtype: "" + } + member { + name: "losses" + mtype: "" + } + member { + name: "name" + mtype: "" + } + member { + name: "non_trainable_variables" + mtype: "" + } + member { + name: "non_trainable_weights" + mtype: "" + } + member { + name: "outbound_nodes" + mtype: "" + } + member { + name: "output" + mtype: "" + } + member { + name: "output_mask" + mtype: "" + } + member { + name: "output_shape" + mtype: "" + } + member { + name: "trainable_variables" + mtype: "" + } + member { + name: "trainable_weights" + mtype: "" + } + member { + name: "updates" + mtype: "" + } + member { + name: "variables" + mtype: "" + } + member { + name: "weights" + mtype: "" + } + member_method { + name: "__init__" + argspec: "args=[\'self\', \'name\', \'dtype\'], varargs=None, keywords=None, defaults=[\'categorical_accuracy\', \'None\'], " + } + member_method { + name: "add_loss" + argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } + member_method { + name: "add_update" + argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "add_variable" + argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None" + } + member_method { + name: "add_weight" + argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\'], " + } + member_method { + name: "apply" + argspec: "args=[\'self\', \'inputs\'], varargs=args, keywords=kwargs, defaults=None" + } + member_method { + name: "build" + argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "call" + argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=kwargs, defaults=None" + } + member_method { + name: "compute_mask" + argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "compute_output_shape" + argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "count_params" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "from_config" + argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_config" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_input_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_input_mask_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_input_shape_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_losses_for" + argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_output_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_output_mask_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_output_shape_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_updates_for" + argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_weights" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "reset_states" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "result" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "set_weights" + argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "update_state" + argspec: "args=[\'self\', \'y_true\', \'y_pred\', \'sample_weight\'], varargs=None, keywords=None, defaults=[\'None\'], " + } +} diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-false-negatives.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-false-negatives.pbtxt new file mode 100644 index 00000000000..49f577e1367 --- /dev/null +++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-false-negatives.pbtxt @@ -0,0 +1,193 @@ +path: "tensorflow.keras.metrics.FalseNegatives" +tf_class { + is_instance: "" + is_instance: "" + is_instance: "" + is_instance: "" + is_instance: "" + is_instance: "" + member { + name: "activity_regularizer" + mtype: "" + } + member { + name: "dtype" + mtype: "" + } + member { + name: "inbound_nodes" + mtype: "" + } + member { + name: "input" + mtype: "" + } + member { + name: "input_mask" + mtype: "" + } + member { + name: "input_shape" + mtype: "" + } + member { + name: "losses" + mtype: "" + } + member { + name: "name" + mtype: "" + } + member { + name: "non_trainable_variables" + mtype: "" + } + member { + name: "non_trainable_weights" + mtype: "" + } + member { + name: "outbound_nodes" + mtype: "" + } + member { + name: "output" + mtype: "" + } + member { + name: "output_mask" + mtype: "" + } + member { + name: "output_shape" + mtype: "" + } + member { + name: "trainable_variables" + mtype: "" + } + member { + name: "trainable_weights" + mtype: "" + } + member { + name: "updates" + mtype: "" + } + member { + name: "variables" + mtype: "" + } + member { + name: "weights" + mtype: "" + } + member_method { + name: "__init__" + argspec: "args=[\'self\', \'thresholds\', \'name\', \'dtype\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], " + } + member_method { + name: "add_loss" + argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } + member_method { + name: "add_update" + argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "add_variable" + argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None" + } + member_method { + name: "add_weight" + argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\'], " + } + member_method { + name: "apply" + argspec: "args=[\'self\', \'inputs\'], varargs=args, keywords=kwargs, defaults=None" + } + member_method { + name: "build" + argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "call" + argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=kwargs, defaults=None" + } + member_method { + name: "compute_mask" + argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "compute_output_shape" + argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "count_params" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "from_config" + argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_config" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_input_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_input_mask_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_input_shape_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_losses_for" + argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_output_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_output_mask_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_output_shape_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_updates_for" + argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_weights" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "reset_states" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "result" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "set_weights" + argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "update_state" + argspec: "args=[\'self\', \'y_true\', \'y_pred\', \'sample_weight\'], varargs=None, keywords=None, defaults=[\'None\'], " + } +} diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-false-positives.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-false-positives.pbtxt new file mode 100644 index 00000000000..e8baf858669 --- /dev/null +++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-false-positives.pbtxt @@ -0,0 +1,193 @@ +path: "tensorflow.keras.metrics.FalsePositives" +tf_class { + is_instance: "" + is_instance: "" + is_instance: "" + is_instance: "" + is_instance: "" + is_instance: "" + member { + name: "activity_regularizer" + mtype: "" + } + member { + name: "dtype" + mtype: "" + } + member { + name: "inbound_nodes" + mtype: "" + } + member { + name: "input" + mtype: "" + } + member { + name: "input_mask" + mtype: "" + } + member { + name: "input_shape" + mtype: "" + } + member { + name: "losses" + mtype: "" + } + member { + name: "name" + mtype: "" + } + member { + name: "non_trainable_variables" + mtype: "" + } + member { + name: "non_trainable_weights" + mtype: "" + } + member { + name: "outbound_nodes" + mtype: "" + } + member { + name: "output" + mtype: "" + } + member { + name: "output_mask" + mtype: "" + } + member { + name: "output_shape" + mtype: "" + } + member { + name: "trainable_variables" + mtype: "" + } + member { + name: "trainable_weights" + mtype: "" + } + member { + name: "updates" + mtype: "" + } + member { + name: "variables" + mtype: "" + } + member { + name: "weights" + mtype: "" + } + member_method { + name: "__init__" + argspec: "args=[\'self\', \'thresholds\', \'name\', \'dtype\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], " + } + member_method { + name: "add_loss" + argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } + member_method { + name: "add_update" + argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "add_variable" + argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None" + } + member_method { + name: "add_weight" + argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\'], " + } + member_method { + name: "apply" + argspec: "args=[\'self\', \'inputs\'], varargs=args, keywords=kwargs, defaults=None" + } + member_method { + name: "build" + argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "call" + argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=kwargs, defaults=None" + } + member_method { + name: "compute_mask" + argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "compute_output_shape" + argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "count_params" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "from_config" + argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_config" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_input_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_input_mask_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_input_shape_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_losses_for" + argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_output_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_output_mask_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_output_shape_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_updates_for" + argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_weights" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "reset_states" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "result" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "set_weights" + argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "update_state" + argspec: "args=[\'self\', \'y_true\', \'y_pred\', \'sample_weight\'], varargs=None, keywords=None, defaults=[\'None\'], " + } +} diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-mean.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-mean.pbtxt new file mode 100644 index 00000000000..40fe64bbd2c --- /dev/null +++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-mean.pbtxt @@ -0,0 +1,192 @@ +path: "tensorflow.keras.metrics.Mean" +tf_class { + is_instance: "" + is_instance: "" + is_instance: "" + is_instance: "" + is_instance: "" + member { + name: "activity_regularizer" + mtype: "" + } + member { + name: "dtype" + mtype: "" + } + member { + name: "inbound_nodes" + mtype: "" + } + member { + name: "input" + mtype: "" + } + member { + name: "input_mask" + mtype: "" + } + member { + name: "input_shape" + mtype: "" + } + member { + name: "losses" + mtype: "" + } + member { + name: "name" + mtype: "" + } + member { + name: "non_trainable_variables" + mtype: "" + } + member { + name: "non_trainable_weights" + mtype: "" + } + member { + name: "outbound_nodes" + mtype: "" + } + member { + name: "output" + mtype: "" + } + member { + name: "output_mask" + mtype: "" + } + member { + name: "output_shape" + mtype: "" + } + member { + name: "trainable_variables" + mtype: "" + } + member { + name: "trainable_weights" + mtype: "" + } + member { + name: "updates" + mtype: "" + } + member { + name: "variables" + mtype: "" + } + member { + name: "weights" + mtype: "" + } + member_method { + name: "__init__" + argspec: "args=[\'self\', \'name\', \'dtype\'], varargs=None, keywords=None, defaults=[\'mean\', \'None\'], " + } + member_method { + name: "add_loss" + argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } + member_method { + name: "add_update" + argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "add_variable" + argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None" + } + member_method { + name: "add_weight" + argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\'], " + } + member_method { + name: "apply" + argspec: "args=[\'self\', \'inputs\'], varargs=args, keywords=kwargs, defaults=None" + } + member_method { + name: "build" + argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "call" + argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=kwargs, defaults=None" + } + member_method { + name: "compute_mask" + argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "compute_output_shape" + argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "count_params" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "from_config" + argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_config" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_input_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_input_mask_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_input_shape_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_losses_for" + argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_output_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_output_mask_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_output_shape_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_updates_for" + argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_weights" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "reset_states" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "result" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "set_weights" + argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "update_state" + argspec: "args=[\'self\', \'values\', \'sample_weight\'], varargs=None, keywords=None, defaults=[\'None\'], " + } +} diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-precision.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-precision.pbtxt new file mode 100644 index 00000000000..ae6a85026da --- /dev/null +++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-precision.pbtxt @@ -0,0 +1,192 @@ +path: "tensorflow.keras.metrics.Precision" +tf_class { + is_instance: "" + is_instance: "" + is_instance: "" + is_instance: "" + is_instance: "" + member { + name: "activity_regularizer" + mtype: "" + } + member { + name: "dtype" + mtype: "" + } + member { + name: "inbound_nodes" + mtype: "" + } + member { + name: "input" + mtype: "" + } + member { + name: "input_mask" + mtype: "" + } + member { + name: "input_shape" + mtype: "" + } + member { + name: "losses" + mtype: "" + } + member { + name: "name" + mtype: "" + } + member { + name: "non_trainable_variables" + mtype: "" + } + member { + name: "non_trainable_weights" + mtype: "" + } + member { + name: "outbound_nodes" + mtype: "" + } + member { + name: "output" + mtype: "" + } + member { + name: "output_mask" + mtype: "" + } + member { + name: "output_shape" + mtype: "" + } + member { + name: "trainable_variables" + mtype: "" + } + member { + name: "trainable_weights" + mtype: "" + } + member { + name: "updates" + mtype: "" + } + member { + name: "variables" + mtype: "" + } + member { + name: "weights" + mtype: "" + } + member_method { + name: "__init__" + argspec: "args=[\'self\', \'thresholds\', \'name\', \'dtype\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], " + } + member_method { + name: "add_loss" + argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } + member_method { + name: "add_update" + argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "add_variable" + argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None" + } + member_method { + name: "add_weight" + argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\'], " + } + member_method { + name: "apply" + argspec: "args=[\'self\', \'inputs\'], varargs=args, keywords=kwargs, defaults=None" + } + member_method { + name: "build" + argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "call" + argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=kwargs, defaults=None" + } + member_method { + name: "compute_mask" + argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "compute_output_shape" + argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "count_params" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "from_config" + argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_config" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_input_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_input_mask_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_input_shape_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_losses_for" + argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_output_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_output_mask_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_output_shape_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_updates_for" + argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_weights" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "reset_states" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "result" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "set_weights" + argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "update_state" + argspec: "args=[\'self\', \'y_true\', \'y_pred\', \'sample_weight\'], varargs=None, keywords=None, defaults=[\'None\'], " + } +} diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-recall.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-recall.pbtxt new file mode 100644 index 00000000000..31068a51d51 --- /dev/null +++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-recall.pbtxt @@ -0,0 +1,192 @@ +path: "tensorflow.keras.metrics.Recall" +tf_class { + is_instance: "" + is_instance: "" + is_instance: "" + is_instance: "" + is_instance: "" + member { + name: "activity_regularizer" + mtype: "" + } + member { + name: "dtype" + mtype: "" + } + member { + name: "inbound_nodes" + mtype: "" + } + member { + name: "input" + mtype: "" + } + member { + name: "input_mask" + mtype: "" + } + member { + name: "input_shape" + mtype: "" + } + member { + name: "losses" + mtype: "" + } + member { + name: "name" + mtype: "" + } + member { + name: "non_trainable_variables" + mtype: "" + } + member { + name: "non_trainable_weights" + mtype: "" + } + member { + name: "outbound_nodes" + mtype: "" + } + member { + name: "output" + mtype: "" + } + member { + name: "output_mask" + mtype: "" + } + member { + name: "output_shape" + mtype: "" + } + member { + name: "trainable_variables" + mtype: "" + } + member { + name: "trainable_weights" + mtype: "" + } + member { + name: "updates" + mtype: "" + } + member { + name: "variables" + mtype: "" + } + member { + name: "weights" + mtype: "" + } + member_method { + name: "__init__" + argspec: "args=[\'self\', \'thresholds\', \'name\', \'dtype\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], " + } + member_method { + name: "add_loss" + argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } + member_method { + name: "add_update" + argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "add_variable" + argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None" + } + member_method { + name: "add_weight" + argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\'], " + } + member_method { + name: "apply" + argspec: "args=[\'self\', \'inputs\'], varargs=args, keywords=kwargs, defaults=None" + } + member_method { + name: "build" + argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "call" + argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=kwargs, defaults=None" + } + member_method { + name: "compute_mask" + argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "compute_output_shape" + argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "count_params" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "from_config" + argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_config" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_input_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_input_mask_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_input_shape_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_losses_for" + argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_output_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_output_mask_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_output_shape_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_updates_for" + argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_weights" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "reset_states" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "result" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "set_weights" + argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "update_state" + argspec: "args=[\'self\', \'y_true\', \'y_pred\', \'sample_weight\'], varargs=None, keywords=None, defaults=[\'None\'], " + } +} diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-sparse-categorical-accuracy.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-sparse-categorical-accuracy.pbtxt new file mode 100644 index 00000000000..0c17452292a --- /dev/null +++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-sparse-categorical-accuracy.pbtxt @@ -0,0 +1,194 @@ +path: "tensorflow.keras.metrics.SparseCategoricalAccuracy" +tf_class { + is_instance: "" + is_instance: "" + is_instance: "" + is_instance: "" + is_instance: "" + is_instance: "" + is_instance: "" + member { + name: "activity_regularizer" + mtype: "" + } + member { + name: "dtype" + mtype: "" + } + member { + name: "inbound_nodes" + mtype: "" + } + member { + name: "input" + mtype: "" + } + member { + name: "input_mask" + mtype: "" + } + member { + name: "input_shape" + mtype: "" + } + member { + name: "losses" + mtype: "" + } + member { + name: "name" + mtype: "" + } + member { + name: "non_trainable_variables" + mtype: "" + } + member { + name: "non_trainable_weights" + mtype: "" + } + member { + name: "outbound_nodes" + mtype: "" + } + member { + name: "output" + mtype: "" + } + member { + name: "output_mask" + mtype: "" + } + member { + name: "output_shape" + mtype: "" + } + member { + name: "trainable_variables" + mtype: "" + } + member { + name: "trainable_weights" + mtype: "" + } + member { + name: "updates" + mtype: "" + } + member { + name: "variables" + mtype: "" + } + member { + name: "weights" + mtype: "" + } + member_method { + name: "__init__" + argspec: "args=[\'self\', \'name\', \'dtype\'], varargs=None, keywords=None, defaults=[\'sparse_categorical_accuracy\', \'None\'], " + } + member_method { + name: "add_loss" + argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } + member_method { + name: "add_update" + argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "add_variable" + argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None" + } + member_method { + name: "add_weight" + argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\'], " + } + member_method { + name: "apply" + argspec: "args=[\'self\', \'inputs\'], varargs=args, keywords=kwargs, defaults=None" + } + member_method { + name: "build" + argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "call" + argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=kwargs, defaults=None" + } + member_method { + name: "compute_mask" + argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "compute_output_shape" + argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "count_params" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "from_config" + argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_config" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_input_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_input_mask_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_input_shape_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_losses_for" + argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_output_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_output_mask_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_output_shape_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_updates_for" + argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_weights" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "reset_states" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "result" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "set_weights" + argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "update_state" + argspec: "args=[\'self\', \'y_true\', \'y_pred\', \'sample_weight\'], varargs=None, keywords=None, defaults=[\'None\'], " + } +} diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-true-negatives.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-true-negatives.pbtxt new file mode 100644 index 00000000000..1b5eb8d0de5 --- /dev/null +++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-true-negatives.pbtxt @@ -0,0 +1,193 @@ +path: "tensorflow.keras.metrics.TrueNegatives" +tf_class { + is_instance: "" + is_instance: "" + is_instance: "" + is_instance: "" + is_instance: "" + is_instance: "" + member { + name: "activity_regularizer" + mtype: "" + } + member { + name: "dtype" + mtype: "" + } + member { + name: "inbound_nodes" + mtype: "" + } + member { + name: "input" + mtype: "" + } + member { + name: "input_mask" + mtype: "" + } + member { + name: "input_shape" + mtype: "" + } + member { + name: "losses" + mtype: "" + } + member { + name: "name" + mtype: "" + } + member { + name: "non_trainable_variables" + mtype: "" + } + member { + name: "non_trainable_weights" + mtype: "" + } + member { + name: "outbound_nodes" + mtype: "" + } + member { + name: "output" + mtype: "" + } + member { + name: "output_mask" + mtype: "" + } + member { + name: "output_shape" + mtype: "" + } + member { + name: "trainable_variables" + mtype: "" + } + member { + name: "trainable_weights" + mtype: "" + } + member { + name: "updates" + mtype: "" + } + member { + name: "variables" + mtype: "" + } + member { + name: "weights" + mtype: "" + } + member_method { + name: "__init__" + argspec: "args=[\'self\', \'thresholds\', \'name\', \'dtype\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], " + } + member_method { + name: "add_loss" + argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } + member_method { + name: "add_update" + argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "add_variable" + argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None" + } + member_method { + name: "add_weight" + argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\'], " + } + member_method { + name: "apply" + argspec: "args=[\'self\', \'inputs\'], varargs=args, keywords=kwargs, defaults=None" + } + member_method { + name: "build" + argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "call" + argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=kwargs, defaults=None" + } + member_method { + name: "compute_mask" + argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "compute_output_shape" + argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "count_params" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "from_config" + argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_config" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_input_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_input_mask_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_input_shape_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_losses_for" + argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_output_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_output_mask_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_output_shape_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_updates_for" + argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_weights" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "reset_states" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "result" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "set_weights" + argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "update_state" + argspec: "args=[\'self\', \'y_true\', \'y_pred\', \'sample_weight\'], varargs=None, keywords=None, defaults=[\'None\'], " + } +} diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-true-positives.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-true-positives.pbtxt new file mode 100644 index 00000000000..5b9c470e32d --- /dev/null +++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-true-positives.pbtxt @@ -0,0 +1,193 @@ +path: "tensorflow.keras.metrics.TruePositives" +tf_class { + is_instance: "" + is_instance: "" + is_instance: "" + is_instance: "" + is_instance: "" + is_instance: "" + member { + name: "activity_regularizer" + mtype: "" + } + member { + name: "dtype" + mtype: "" + } + member { + name: "inbound_nodes" + mtype: "" + } + member { + name: "input" + mtype: "" + } + member { + name: "input_mask" + mtype: "" + } + member { + name: "input_shape" + mtype: "" + } + member { + name: "losses" + mtype: "" + } + member { + name: "name" + mtype: "" + } + member { + name: "non_trainable_variables" + mtype: "" + } + member { + name: "non_trainable_weights" + mtype: "" + } + member { + name: "outbound_nodes" + mtype: "" + } + member { + name: "output" + mtype: "" + } + member { + name: "output_mask" + mtype: "" + } + member { + name: "output_shape" + mtype: "" + } + member { + name: "trainable_variables" + mtype: "" + } + member { + name: "trainable_weights" + mtype: "" + } + member { + name: "updates" + mtype: "" + } + member { + name: "variables" + mtype: "" + } + member { + name: "weights" + mtype: "" + } + member_method { + name: "__init__" + argspec: "args=[\'self\', \'thresholds\', \'name\', \'dtype\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], " + } + member_method { + name: "add_loss" + argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } + member_method { + name: "add_update" + argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "add_variable" + argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None" + } + member_method { + name: "add_weight" + argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\'], " + } + member_method { + name: "apply" + argspec: "args=[\'self\', \'inputs\'], varargs=args, keywords=kwargs, defaults=None" + } + member_method { + name: "build" + argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "call" + argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=kwargs, defaults=None" + } + member_method { + name: "compute_mask" + argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "compute_output_shape" + argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "count_params" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "from_config" + argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_config" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_input_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_input_mask_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_input_shape_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_losses_for" + argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_output_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_output_mask_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_output_shape_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_updates_for" + argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_weights" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "reset_states" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "result" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "set_weights" + argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "update_state" + argspec: "args=[\'self\', \'y_true\', \'y_pred\', \'sample_weight\'], varargs=None, keywords=None, defaults=[\'None\'], " + } +} diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.pbtxt index a296e131586..8cab17edc59 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.pbtxt @@ -1,5 +1,49 @@ path: "tensorflow.keras.metrics" tf_module { + member { + name: "Accuracy" + mtype: "" + } + member { + name: "BinaryAccuracy" + mtype: "" + } + member { + name: "CategoricalAccuracy" + mtype: "" + } + member { + name: "FalseNegatives" + mtype: "" + } + member { + name: "FalsePositives" + mtype: "" + } + member { + name: "Mean" + mtype: "" + } + member { + name: "Precision" + mtype: "" + } + member { + name: "Recall" + mtype: "" + } + member { + name: "SparseCategoricalAccuracy" + mtype: "" + } + member { + name: "TrueNegatives" + mtype: "" + } + member { + name: "TruePositives" + mtype: "" + } member_method { name: "KLD" argspec: "args=[\'y_true\', \'y_pred\'], varargs=None, keywords=None, defaults=None" @@ -26,7 +70,7 @@ tf_module { } member_method { name: "binary_crossentropy" - argspec: "args=[\'y_true\', \'y_pred\'], varargs=None, keywords=None, defaults=None" + argspec: "args=[\'y_true\', \'y_pred\', \'from_logits\'], varargs=None, keywords=None, defaults=[\'False\'], " } member_method { name: "categorical_accuracy" @@ -34,7 +78,7 @@ tf_module { } member_method { name: "categorical_crossentropy" - argspec: "args=[\'y_true\', \'y_pred\'], varargs=None, keywords=None, defaults=None" + argspec: "args=[\'y_true\', \'y_pred\', \'from_logits\'], varargs=None, keywords=None, defaults=[\'False\'], " } member_method { name: "cosine" @@ -110,7 +154,7 @@ tf_module { } member_method { name: "sparse_categorical_crossentropy" - argspec: "args=[\'y_true\', \'y_pred\'], varargs=None, keywords=None, defaults=None" + argspec: "args=[\'y_true\', \'y_pred\', \'from_logits\'], varargs=None, keywords=None, defaults=[\'False\'], " } member_method { name: "sparse_top_k_categorical_accuracy" diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.models.-model.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.models.-model.pbtxt index ccff809f2b2..c58c7bef22d 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.keras.models.-model.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.models.-model.pbtxt @@ -41,6 +41,14 @@ tf_class { name: "losses" mtype: "" } + member { + name: "metrics" + mtype: "" + } + member { + name: "metrics_names" + mtype: "" + } member { name: "name" mtype: "" @@ -69,6 +77,10 @@ tf_class { name: "output_shape" mtype: "" } + member { + name: "run_eagerly" + mtype: "" + } member { name: "state_updates" mtype: "" @@ -105,13 +117,17 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None" } member_method { name: "add_weight" @@ -225,6 +241,10 @@ tf_class { name: "predict_on_batch" argspec: "args=[\'self\', \'x\'], varargs=None, keywords=None, defaults=None" } + member_method { + name: "reset_metrics" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } member_method { name: "reset_states" argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" @@ -247,7 +267,7 @@ tf_class { } member_method { name: "test_on_batch" - argspec: "args=[\'self\', \'x\', \'y\', \'sample_weight\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + argspec: "args=[\'self\', \'x\', \'y\', \'sample_weight\', \'reset_metrics\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'True\'], " } member_method { name: "to_json" @@ -259,6 +279,6 @@ tf_class { } member_method { name: "train_on_batch" - argspec: "args=[\'self\', \'x\', \'y\', \'sample_weight\', \'class_weight\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], " + argspec: "args=[\'self\', \'x\', \'y\', \'sample_weight\', \'class_weight\', \'reset_metrics\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\'], " } } diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.models.-sequential.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.models.-sequential.pbtxt index b0fc7f97f1d..473a1c16fb1 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.keras.models.-sequential.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.models.-sequential.pbtxt @@ -42,6 +42,14 @@ tf_class { name: "losses" mtype: "" } + member { + name: "metrics" + mtype: "" + } + member { + name: "metrics_names" + mtype: "" + } member { name: "name" mtype: "" @@ -70,6 +78,10 @@ tf_class { name: "output_shape" mtype: "" } + member { + name: "run_eagerly" + mtype: "" + } member { name: "state_updates" mtype: "" @@ -110,13 +122,17 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None" } member_method { name: "add_weight" @@ -242,6 +258,10 @@ tf_class { name: "predict_proba" argspec: "args=[\'self\', \'x\', \'batch_size\', \'verbose\'], varargs=None, keywords=None, defaults=[\'32\', \'0\'], " } + member_method { + name: "reset_metrics" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } member_method { name: "reset_states" argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" @@ -264,7 +284,7 @@ tf_class { } member_method { name: "test_on_batch" - argspec: "args=[\'self\', \'x\', \'y\', \'sample_weight\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + argspec: "args=[\'self\', \'x\', \'y\', \'sample_weight\', \'reset_metrics\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'True\'], " } member_method { name: "to_json" @@ -276,6 +296,6 @@ tf_class { } member_method { name: "train_on_batch" - argspec: "args=[\'self\', \'x\', \'y\', \'sample_weight\', \'class_weight\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], " + argspec: "args=[\'self\', \'x\', \'y\', \'sample_weight\', \'class_weight\', \'reset_metrics\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\'], " } } diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.utils.-progbar.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.utils.-progbar.pbtxt index be4496e753f..8177cc71ed3 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.keras.utils.-progbar.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.utils.-progbar.pbtxt @@ -4,7 +4,7 @@ tf_class { is_instance: "" member_method { name: "__init__" - argspec: "args=[\'self\', \'target\', \'width\', \'verbose\', \'interval\', \'stateful_metrics\'], varargs=None, keywords=None, defaults=[\'30\', \'1\', \'0.05\', \'None\'], " + argspec: "args=[\'self\', \'target\', \'width\', \'verbose\', \'interval\', \'stateful_metrics\', \'unit_name\'], varargs=None, keywords=None, defaults=[\'30\', \'1\', \'0.05\', \'None\', \'step\'], " } member_method { name: "add" diff --git a/tensorflow/tools/api/golden/v1/tensorflow.layers.-average-pooling1-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.layers.-average-pooling1-d.pbtxt index c82e67526b2..059c91f724a 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.layers.-average-pooling1-d.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.layers.-average-pooling1-d.pbtxt @@ -99,6 +99,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v1/tensorflow.layers.-average-pooling2-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.layers.-average-pooling2-d.pbtxt index 1d031cb5f84..d06c8e81ee5 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.layers.-average-pooling2-d.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.layers.-average-pooling2-d.pbtxt @@ -99,6 +99,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v1/tensorflow.layers.-average-pooling3-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.layers.-average-pooling3-d.pbtxt index a8dda6655df..6be8e7c210f 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.layers.-average-pooling3-d.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.layers.-average-pooling3-d.pbtxt @@ -99,6 +99,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v1/tensorflow.layers.-batch-normalization.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.layers.-batch-normalization.pbtxt index 97f65ed8943..16d9ecce10c 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.layers.-batch-normalization.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.layers.-batch-normalization.pbtxt @@ -1,7 +1,8 @@ path: "tensorflow.layers.BatchNormalization" tf_class { is_instance: "" - is_instance: "" + is_instance: "" + is_instance: "" is_instance: "" is_instance: "" is_instance: "" @@ -98,6 +99,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v1/tensorflow.layers.-conv1-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.layers.-conv1-d.pbtxt index ccd9578f0d6..21c695935ce 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.layers.-conv1-d.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.layers.-conv1-d.pbtxt @@ -99,6 +99,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v1/tensorflow.layers.-conv2-d-transpose.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.layers.-conv2-d-transpose.pbtxt index 9cbb58d721b..f24d0307207 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.layers.-conv2-d-transpose.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.layers.-conv2-d-transpose.pbtxt @@ -100,6 +100,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v1/tensorflow.layers.-conv2-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.layers.-conv2-d.pbtxt index c75ea3911e1..0a510ece355 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.layers.-conv2-d.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.layers.-conv2-d.pbtxt @@ -99,6 +99,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v1/tensorflow.layers.-conv3-d-transpose.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.layers.-conv3-d-transpose.pbtxt index 5dc834e5141..d0ee44bed3c 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.layers.-conv3-d-transpose.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.layers.-conv3-d-transpose.pbtxt @@ -100,6 +100,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v1/tensorflow.layers.-conv3-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.layers.-conv3-d.pbtxt index 96ab209874a..546de3cdab3 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.layers.-conv3-d.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.layers.-conv3-d.pbtxt @@ -99,6 +99,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v1/tensorflow.layers.-dense.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.layers.-dense.pbtxt index 7e9656b3525..3ad311581eb 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.layers.-dense.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.layers.-dense.pbtxt @@ -98,6 +98,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v1/tensorflow.layers.-dropout.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.layers.-dropout.pbtxt index e9a2269a6e8..9b83271350c 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.layers.-dropout.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.layers.-dropout.pbtxt @@ -98,6 +98,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v1/tensorflow.layers.-flatten.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.layers.-flatten.pbtxt index 7d2eaaab2a8..87a7fb3d843 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.layers.-flatten.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.layers.-flatten.pbtxt @@ -98,6 +98,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v1/tensorflow.layers.-input-spec.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.layers.-input-spec.pbtxt index fd02c919aeb..80834e08f7a 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.layers.-input-spec.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.layers.-input-spec.pbtxt @@ -1,6 +1,6 @@ path: "tensorflow.layers.InputSpec" tf_class { - is_instance: "" + is_instance: "" is_instance: "" member_method { name: "__init__" diff --git a/tensorflow/tools/api/golden/v1/tensorflow.layers.-layer.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.layers.-layer.pbtxt index 8bc3eb26e9c..32b17e90ade 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.layers.-layer.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.layers.-layer.pbtxt @@ -96,6 +96,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v1/tensorflow.layers.-max-pooling1-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.layers.-max-pooling1-d.pbtxt index 6a0dcce56ac..643c469717c 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.layers.-max-pooling1-d.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.layers.-max-pooling1-d.pbtxt @@ -99,6 +99,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v1/tensorflow.layers.-max-pooling2-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.layers.-max-pooling2-d.pbtxt index b6c84edf2a2..434e25adc12 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.layers.-max-pooling2-d.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.layers.-max-pooling2-d.pbtxt @@ -99,6 +99,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v1/tensorflow.layers.-max-pooling3-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.layers.-max-pooling3-d.pbtxt index 062a02fa590..089fc6f9243 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.layers.-max-pooling3-d.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.layers.-max-pooling3-d.pbtxt @@ -99,6 +99,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v1/tensorflow.layers.-separable-conv1-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.layers.-separable-conv1-d.pbtxt index eaad0fb23ef..bc3d58b9ca9 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.layers.-separable-conv1-d.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.layers.-separable-conv1-d.pbtxt @@ -100,6 +100,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v1/tensorflow.layers.-separable-conv2-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.layers.-separable-conv2-d.pbtxt index ece28a8ce96..fe7d71af3a4 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.layers.-separable-conv2-d.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.layers.-separable-conv2-d.pbtxt @@ -100,6 +100,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-block-diag.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-block-diag.pbtxt index 973705dae2f..773c74e64d1 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-block-diag.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-block-diag.pbtxt @@ -79,6 +79,10 @@ tf_class { name: "batch_shape_tensor" argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'batch_shape_tensor\'], " } + member_method { + name: "cholesky" + argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'cholesky\'], " + } member_method { name: "determinant" argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'det\'], " diff --git a/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-circulant.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-circulant.pbtxt index de917706d55..533544d21f2 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-circulant.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-circulant.pbtxt @@ -96,6 +96,10 @@ tf_class { name: "block_shape_tensor" argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" } + member_method { + name: "cholesky" + argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'cholesky\'], " + } member_method { name: "convolution_kernel" argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'convolution_kernel\'], " diff --git a/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-circulant2-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-circulant2-d.pbtxt index c4e6a21c3ac..e3926eb6d47 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-circulant2-d.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-circulant2-d.pbtxt @@ -96,6 +96,10 @@ tf_class { name: "block_shape_tensor" argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" } + member_method { + name: "cholesky" + argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'cholesky\'], " + } member_method { name: "convolution_kernel" argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'convolution_kernel\'], " diff --git a/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-circulant3-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-circulant3-d.pbtxt index 2e085a8e289..ba209df7824 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-circulant3-d.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-circulant3-d.pbtxt @@ -96,6 +96,10 @@ tf_class { name: "block_shape_tensor" argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" } + member_method { + name: "cholesky" + argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'cholesky\'], " + } member_method { name: "convolution_kernel" argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'convolution_kernel\'], " diff --git a/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-composition.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-composition.pbtxt index 42d22bce42d..081fb0e08bc 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-composition.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-composition.pbtxt @@ -79,6 +79,10 @@ tf_class { name: "batch_shape_tensor" argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'batch_shape_tensor\'], " } + member_method { + name: "cholesky" + argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'cholesky\'], " + } member_method { name: "determinant" argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'det\'], " diff --git a/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-diag.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-diag.pbtxt index d6749fdcec6..2014a043016 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-diag.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-diag.pbtxt @@ -79,6 +79,10 @@ tf_class { name: "batch_shape_tensor" argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'batch_shape_tensor\'], " } + member_method { + name: "cholesky" + argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'cholesky\'], " + } member_method { name: "determinant" argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'det\'], " diff --git a/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-full-matrix.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-full-matrix.pbtxt index d9f363d1336..9a87ae96877 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-full-matrix.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-full-matrix.pbtxt @@ -75,6 +75,10 @@ tf_class { name: "batch_shape_tensor" argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'batch_shape_tensor\'], " } + member_method { + name: "cholesky" + argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'cholesky\'], " + } member_method { name: "determinant" argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'det\'], " diff --git a/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-identity.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-identity.pbtxt index aac7ee31ed6..33afb835ce1 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-identity.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-identity.pbtxt @@ -76,6 +76,10 @@ tf_class { name: "batch_shape_tensor" argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'batch_shape_tensor\'], " } + member_method { + name: "cholesky" + argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'cholesky\'], " + } member_method { name: "determinant" argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'det\'], " diff --git a/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-kronecker.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-kronecker.pbtxt index c11d3908293..a9078c8ab5c 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-kronecker.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-kronecker.pbtxt @@ -79,6 +79,10 @@ tf_class { name: "batch_shape_tensor" argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'batch_shape_tensor\'], " } + member_method { + name: "cholesky" + argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'cholesky\'], " + } member_method { name: "determinant" argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'det\'], " diff --git a/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-low-rank-update.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-low-rank-update.pbtxt index 3ee800269e6..4cfa3bb30d7 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-low-rank-update.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-low-rank-update.pbtxt @@ -99,6 +99,10 @@ tf_class { name: "batch_shape_tensor" argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'batch_shape_tensor\'], " } + member_method { + name: "cholesky" + argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'cholesky\'], " + } member_method { name: "determinant" argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'det\'], " diff --git a/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-lower-triangular.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-lower-triangular.pbtxt index 63a1bc2321e..a87649133fd 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-lower-triangular.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-lower-triangular.pbtxt @@ -75,6 +75,10 @@ tf_class { name: "batch_shape_tensor" argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'batch_shape_tensor\'], " } + member_method { + name: "cholesky" + argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'cholesky\'], " + } member_method { name: "determinant" argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'det\'], " diff --git a/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-scaled-identity.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-scaled-identity.pbtxt index e2c5a505a7d..32656467840 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-scaled-identity.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-scaled-identity.pbtxt @@ -80,6 +80,10 @@ tf_class { name: "batch_shape_tensor" argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'batch_shape_tensor\'], " } + member_method { + name: "cholesky" + argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'cholesky\'], " + } member_method { name: "determinant" argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'det\'], " diff --git a/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-zeros.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-zeros.pbtxt index a1b0e06b475..49d8890c894 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-zeros.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-zeros.pbtxt @@ -75,6 +75,10 @@ tf_class { name: "batch_shape_tensor" argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'batch_shape_tensor\'], " } + member_method { + name: "cholesky" + argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'cholesky\'], " + } member_method { name: "determinant" argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'det\'], " diff --git a/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator.pbtxt index 6d849dc040f..c89dc067b33 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator.pbtxt @@ -74,6 +74,10 @@ tf_class { name: "batch_shape_tensor" argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'batch_shape_tensor\'], " } + member_method { + name: "cholesky" + argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'cholesky\'], " + } member_method { name: "determinant" argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'det\'], " diff --git a/tensorflow/tools/api/golden/v1/tensorflow.lite.constants.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.lite.constants.pbtxt index 08845553e55..ef6c777665c 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.lite.constants.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.lite.constants.pbtxt @@ -2,7 +2,7 @@ path: "tensorflow.lite.constants" tf_module { member { name: "FLOAT" - mtype: "" + mtype: "" } member { name: "GRAPHVIZ_DOT" @@ -10,19 +10,19 @@ tf_module { } member { name: "INT32" - mtype: "" + mtype: "" } member { name: "INT64" - mtype: "" + mtype: "" } member { name: "QUANTIZED_UINT8" - mtype: "" + mtype: "" } member { name: "STRING" - mtype: "" + mtype: "" } member { name: "TFLITE" diff --git a/tensorflow/tools/api/golden/v1/tensorflow.math.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.math.pbtxt index a8334fdd1d1..f34e2c2aa5a 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.math.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.math.pbtxt @@ -176,6 +176,26 @@ tf_module { name: "invert_permutation" argspec: "args=[\'x\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "is_finite" + argspec: "args=[\'x\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "is_inf" + argspec: "args=[\'x\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "is_nan" + argspec: "args=[\'x\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "is_non_decreasing" + argspec: "args=[\'x\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "is_strictly_increasing" + argspec: "args=[\'x\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " + } member_method { name: "l2_normalize" argspec: "args=[\'x\', \'axis\', \'epsilon\', \'name\', \'dim\'], varargs=None, keywords=None, defaults=[\'None\', \'1e-12\', \'None\', \'None\'], " @@ -298,7 +318,7 @@ tf_module { } member_method { name: "reduce_std" - argspec: "args=[\'input_tensor\', \'axis\', \'keepdims\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], " + argspec: "args=[\'input_tensor\', \'axis\', \'keepdims\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\'], " } member_method { name: "reduce_sum" @@ -306,7 +326,7 @@ tf_module { } member_method { name: "reduce_variance" - argspec: "args=[\'input_tensor\', \'axis\', \'keepdims\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], " + argspec: "args=[\'input_tensor\', \'axis\', \'keepdims\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\'], " } member_method { name: "rint" @@ -322,7 +342,7 @@ tf_module { } member_method { name: "scalar_mul" - argspec: "args=[\'scalar\', \'x\'], varargs=None, keywords=None, defaults=None" + argspec: "args=[\'scalar\', \'x\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " } member_method { name: "segment_max" diff --git a/tensorflow/tools/api/golden/v1/tensorflow.metrics.-accuracy.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.metrics.-accuracy.pbtxt new file mode 100644 index 00000000000..f8e12f88173 --- /dev/null +++ b/tensorflow/tools/api/golden/v1/tensorflow.metrics.-accuracy.pbtxt @@ -0,0 +1,194 @@ +path: "tensorflow.metrics.Accuracy" +tf_class { + is_instance: "" + is_instance: "" + is_instance: "" + is_instance: "" + is_instance: "" + is_instance: "" + is_instance: "" + member { + name: "activity_regularizer" + mtype: "" + } + member { + name: "dtype" + mtype: "" + } + member { + name: "inbound_nodes" + mtype: "" + } + member { + name: "input" + mtype: "" + } + member { + name: "input_mask" + mtype: "" + } + member { + name: "input_shape" + mtype: "" + } + member { + name: "losses" + mtype: "" + } + member { + name: "name" + mtype: "" + } + member { + name: "non_trainable_variables" + mtype: "" + } + member { + name: "non_trainable_weights" + mtype: "" + } + member { + name: "outbound_nodes" + mtype: "" + } + member { + name: "output" + mtype: "" + } + member { + name: "output_mask" + mtype: "" + } + member { + name: "output_shape" + mtype: "" + } + member { + name: "trainable_variables" + mtype: "" + } + member { + name: "trainable_weights" + mtype: "" + } + member { + name: "updates" + mtype: "" + } + member { + name: "variables" + mtype: "" + } + member { + name: "weights" + mtype: "" + } + member_method { + name: "__init__" + argspec: "args=[\'self\', \'name\', \'dtype\'], varargs=None, keywords=None, defaults=[\'accuracy\', \'None\'], " + } + member_method { + name: "add_loss" + argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } + member_method { + name: "add_update" + argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "add_variable" + argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None" + } + member_method { + name: "add_weight" + argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\'], " + } + member_method { + name: "apply" + argspec: "args=[\'self\', \'inputs\'], varargs=args, keywords=kwargs, defaults=None" + } + member_method { + name: "build" + argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "call" + argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=kwargs, defaults=None" + } + member_method { + name: "compute_mask" + argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "compute_output_shape" + argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "count_params" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "from_config" + argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_config" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_input_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_input_mask_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_input_shape_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_losses_for" + argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_output_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_output_mask_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_output_shape_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_updates_for" + argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_weights" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "reset_states" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "result" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "set_weights" + argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "update_state" + argspec: "args=[\'self\', \'y_true\', \'y_pred\', \'sample_weight\'], varargs=None, keywords=None, defaults=[\'None\'], " + } +} diff --git a/tensorflow/tools/api/golden/v1/tensorflow.metrics.-binary-accuracy.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.metrics.-binary-accuracy.pbtxt new file mode 100644 index 00000000000..b9bc6a716a1 --- /dev/null +++ b/tensorflow/tools/api/golden/v1/tensorflow.metrics.-binary-accuracy.pbtxt @@ -0,0 +1,194 @@ +path: "tensorflow.metrics.BinaryAccuracy" +tf_class { + is_instance: "" + is_instance: "" + is_instance: "" + is_instance: "" + is_instance: "" + is_instance: "" + is_instance: "" + member { + name: "activity_regularizer" + mtype: "" + } + member { + name: "dtype" + mtype: "" + } + member { + name: "inbound_nodes" + mtype: "" + } + member { + name: "input" + mtype: "" + } + member { + name: "input_mask" + mtype: "" + } + member { + name: "input_shape" + mtype: "" + } + member { + name: "losses" + mtype: "" + } + member { + name: "name" + mtype: "" + } + member { + name: "non_trainable_variables" + mtype: "" + } + member { + name: "non_trainable_weights" + mtype: "" + } + member { + name: "outbound_nodes" + mtype: "" + } + member { + name: "output" + mtype: "" + } + member { + name: "output_mask" + mtype: "" + } + member { + name: "output_shape" + mtype: "" + } + member { + name: "trainable_variables" + mtype: "" + } + member { + name: "trainable_weights" + mtype: "" + } + member { + name: "updates" + mtype: "" + } + member { + name: "variables" + mtype: "" + } + member { + name: "weights" + mtype: "" + } + member_method { + name: "__init__" + argspec: "args=[\'self\', \'name\', \'dtype\', \'threshold\'], varargs=None, keywords=None, defaults=[\'binary_accuracy\', \'None\', \'0.5\'], " + } + member_method { + name: "add_loss" + argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } + member_method { + name: "add_update" + argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "add_variable" + argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None" + } + member_method { + name: "add_weight" + argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\'], " + } + member_method { + name: "apply" + argspec: "args=[\'self\', \'inputs\'], varargs=args, keywords=kwargs, defaults=None" + } + member_method { + name: "build" + argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "call" + argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=kwargs, defaults=None" + } + member_method { + name: "compute_mask" + argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "compute_output_shape" + argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "count_params" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "from_config" + argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_config" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_input_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_input_mask_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_input_shape_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_losses_for" + argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_output_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_output_mask_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_output_shape_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_updates_for" + argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_weights" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "reset_states" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "result" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "set_weights" + argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "update_state" + argspec: "args=[\'self\', \'y_true\', \'y_pred\', \'sample_weight\'], varargs=None, keywords=None, defaults=[\'None\'], " + } +} diff --git a/tensorflow/tools/api/golden/v1/tensorflow.metrics.-categorical-accuracy.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.metrics.-categorical-accuracy.pbtxt new file mode 100644 index 00000000000..0ef75d8756f --- /dev/null +++ b/tensorflow/tools/api/golden/v1/tensorflow.metrics.-categorical-accuracy.pbtxt @@ -0,0 +1,194 @@ +path: "tensorflow.metrics.CategoricalAccuracy" +tf_class { + is_instance: "" + is_instance: "" + is_instance: "" + is_instance: "" + is_instance: "" + is_instance: "" + is_instance: "" + member { + name: "activity_regularizer" + mtype: "" + } + member { + name: "dtype" + mtype: "" + } + member { + name: "inbound_nodes" + mtype: "" + } + member { + name: "input" + mtype: "" + } + member { + name: "input_mask" + mtype: "" + } + member { + name: "input_shape" + mtype: "" + } + member { + name: "losses" + mtype: "" + } + member { + name: "name" + mtype: "" + } + member { + name: "non_trainable_variables" + mtype: "" + } + member { + name: "non_trainable_weights" + mtype: "" + } + member { + name: "outbound_nodes" + mtype: "" + } + member { + name: "output" + mtype: "" + } + member { + name: "output_mask" + mtype: "" + } + member { + name: "output_shape" + mtype: "" + } + member { + name: "trainable_variables" + mtype: "" + } + member { + name: "trainable_weights" + mtype: "" + } + member { + name: "updates" + mtype: "" + } + member { + name: "variables" + mtype: "" + } + member { + name: "weights" + mtype: "" + } + member_method { + name: "__init__" + argspec: "args=[\'self\', \'name\', \'dtype\'], varargs=None, keywords=None, defaults=[\'categorical_accuracy\', \'None\'], " + } + member_method { + name: "add_loss" + argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } + member_method { + name: "add_update" + argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "add_variable" + argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None" + } + member_method { + name: "add_weight" + argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\'], " + } + member_method { + name: "apply" + argspec: "args=[\'self\', \'inputs\'], varargs=args, keywords=kwargs, defaults=None" + } + member_method { + name: "build" + argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "call" + argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=kwargs, defaults=None" + } + member_method { + name: "compute_mask" + argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "compute_output_shape" + argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "count_params" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "from_config" + argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_config" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_input_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_input_mask_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_input_shape_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_losses_for" + argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_output_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_output_mask_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_output_shape_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_updates_for" + argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_weights" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "reset_states" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "result" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "set_weights" + argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "update_state" + argspec: "args=[\'self\', \'y_true\', \'y_pred\', \'sample_weight\'], varargs=None, keywords=None, defaults=[\'None\'], " + } +} diff --git a/tensorflow/tools/api/golden/v1/tensorflow.metrics.-false-negatives.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.metrics.-false-negatives.pbtxt new file mode 100644 index 00000000000..33226a2df62 --- /dev/null +++ b/tensorflow/tools/api/golden/v1/tensorflow.metrics.-false-negatives.pbtxt @@ -0,0 +1,193 @@ +path: "tensorflow.metrics.FalseNegatives" +tf_class { + is_instance: "" + is_instance: "" + is_instance: "" + is_instance: "" + is_instance: "" + is_instance: "" + member { + name: "activity_regularizer" + mtype: "" + } + member { + name: "dtype" + mtype: "" + } + member { + name: "inbound_nodes" + mtype: "" + } + member { + name: "input" + mtype: "" + } + member { + name: "input_mask" + mtype: "" + } + member { + name: "input_shape" + mtype: "" + } + member { + name: "losses" + mtype: "" + } + member { + name: "name" + mtype: "" + } + member { + name: "non_trainable_variables" + mtype: "" + } + member { + name: "non_trainable_weights" + mtype: "" + } + member { + name: "outbound_nodes" + mtype: "" + } + member { + name: "output" + mtype: "" + } + member { + name: "output_mask" + mtype: "" + } + member { + name: "output_shape" + mtype: "" + } + member { + name: "trainable_variables" + mtype: "" + } + member { + name: "trainable_weights" + mtype: "" + } + member { + name: "updates" + mtype: "" + } + member { + name: "variables" + mtype: "" + } + member { + name: "weights" + mtype: "" + } + member_method { + name: "__init__" + argspec: "args=[\'self\', \'thresholds\', \'name\', \'dtype\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], " + } + member_method { + name: "add_loss" + argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } + member_method { + name: "add_update" + argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "add_variable" + argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None" + } + member_method { + name: "add_weight" + argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\'], " + } + member_method { + name: "apply" + argspec: "args=[\'self\', \'inputs\'], varargs=args, keywords=kwargs, defaults=None" + } + member_method { + name: "build" + argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "call" + argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=kwargs, defaults=None" + } + member_method { + name: "compute_mask" + argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "compute_output_shape" + argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "count_params" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "from_config" + argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_config" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_input_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_input_mask_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_input_shape_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_losses_for" + argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_output_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_output_mask_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_output_shape_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_updates_for" + argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_weights" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "reset_states" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "result" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "set_weights" + argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "update_state" + argspec: "args=[\'self\', \'y_true\', \'y_pred\', \'sample_weight\'], varargs=None, keywords=None, defaults=[\'None\'], " + } +} diff --git a/tensorflow/tools/api/golden/v1/tensorflow.metrics.-false-positives.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.metrics.-false-positives.pbtxt new file mode 100644 index 00000000000..9953162ea3e --- /dev/null +++ b/tensorflow/tools/api/golden/v1/tensorflow.metrics.-false-positives.pbtxt @@ -0,0 +1,193 @@ +path: "tensorflow.metrics.FalsePositives" +tf_class { + is_instance: "" + is_instance: "" + is_instance: "" + is_instance: "" + is_instance: "" + is_instance: "" + member { + name: "activity_regularizer" + mtype: "" + } + member { + name: "dtype" + mtype: "" + } + member { + name: "inbound_nodes" + mtype: "" + } + member { + name: "input" + mtype: "" + } + member { + name: "input_mask" + mtype: "" + } + member { + name: "input_shape" + mtype: "" + } + member { + name: "losses" + mtype: "" + } + member { + name: "name" + mtype: "" + } + member { + name: "non_trainable_variables" + mtype: "" + } + member { + name: "non_trainable_weights" + mtype: "" + } + member { + name: "outbound_nodes" + mtype: "" + } + member { + name: "output" + mtype: "" + } + member { + name: "output_mask" + mtype: "" + } + member { + name: "output_shape" + mtype: "" + } + member { + name: "trainable_variables" + mtype: "" + } + member { + name: "trainable_weights" + mtype: "" + } + member { + name: "updates" + mtype: "" + } + member { + name: "variables" + mtype: "" + } + member { + name: "weights" + mtype: "" + } + member_method { + name: "__init__" + argspec: "args=[\'self\', \'thresholds\', \'name\', \'dtype\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], " + } + member_method { + name: "add_loss" + argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } + member_method { + name: "add_update" + argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "add_variable" + argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None" + } + member_method { + name: "add_weight" + argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\'], " + } + member_method { + name: "apply" + argspec: "args=[\'self\', \'inputs\'], varargs=args, keywords=kwargs, defaults=None" + } + member_method { + name: "build" + argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "call" + argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=kwargs, defaults=None" + } + member_method { + name: "compute_mask" + argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "compute_output_shape" + argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "count_params" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "from_config" + argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_config" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_input_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_input_mask_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_input_shape_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_losses_for" + argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_output_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_output_mask_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_output_shape_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_updates_for" + argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_weights" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "reset_states" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "result" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "set_weights" + argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "update_state" + argspec: "args=[\'self\', \'y_true\', \'y_pred\', \'sample_weight\'], varargs=None, keywords=None, defaults=[\'None\'], " + } +} diff --git a/tensorflow/tools/api/golden/v1/tensorflow.metrics.-mean.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.metrics.-mean.pbtxt new file mode 100644 index 00000000000..7fe6d6fda96 --- /dev/null +++ b/tensorflow/tools/api/golden/v1/tensorflow.metrics.-mean.pbtxt @@ -0,0 +1,192 @@ +path: "tensorflow.metrics.Mean" +tf_class { + is_instance: "" + is_instance: "" + is_instance: "" + is_instance: "" + is_instance: "" + member { + name: "activity_regularizer" + mtype: "" + } + member { + name: "dtype" + mtype: "" + } + member { + name: "inbound_nodes" + mtype: "" + } + member { + name: "input" + mtype: "" + } + member { + name: "input_mask" + mtype: "" + } + member { + name: "input_shape" + mtype: "" + } + member { + name: "losses" + mtype: "" + } + member { + name: "name" + mtype: "" + } + member { + name: "non_trainable_variables" + mtype: "" + } + member { + name: "non_trainable_weights" + mtype: "" + } + member { + name: "outbound_nodes" + mtype: "" + } + member { + name: "output" + mtype: "" + } + member { + name: "output_mask" + mtype: "" + } + member { + name: "output_shape" + mtype: "" + } + member { + name: "trainable_variables" + mtype: "" + } + member { + name: "trainable_weights" + mtype: "" + } + member { + name: "updates" + mtype: "" + } + member { + name: "variables" + mtype: "" + } + member { + name: "weights" + mtype: "" + } + member_method { + name: "__init__" + argspec: "args=[\'self\', \'name\', \'dtype\'], varargs=None, keywords=None, defaults=[\'mean\', \'None\'], " + } + member_method { + name: "add_loss" + argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } + member_method { + name: "add_update" + argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "add_variable" + argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None" + } + member_method { + name: "add_weight" + argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\'], " + } + member_method { + name: "apply" + argspec: "args=[\'self\', \'inputs\'], varargs=args, keywords=kwargs, defaults=None" + } + member_method { + name: "build" + argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "call" + argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=kwargs, defaults=None" + } + member_method { + name: "compute_mask" + argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "compute_output_shape" + argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "count_params" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "from_config" + argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_config" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_input_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_input_mask_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_input_shape_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_losses_for" + argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_output_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_output_mask_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_output_shape_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_updates_for" + argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_weights" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "reset_states" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "result" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "set_weights" + argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "update_state" + argspec: "args=[\'self\', \'values\', \'sample_weight\'], varargs=None, keywords=None, defaults=[\'None\'], " + } +} diff --git a/tensorflow/tools/api/golden/v1/tensorflow.metrics.-precision.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.metrics.-precision.pbtxt new file mode 100644 index 00000000000..8c3271a109c --- /dev/null +++ b/tensorflow/tools/api/golden/v1/tensorflow.metrics.-precision.pbtxt @@ -0,0 +1,192 @@ +path: "tensorflow.metrics.Precision" +tf_class { + is_instance: "" + is_instance: "" + is_instance: "" + is_instance: "" + is_instance: "" + member { + name: "activity_regularizer" + mtype: "" + } + member { + name: "dtype" + mtype: "" + } + member { + name: "inbound_nodes" + mtype: "" + } + member { + name: "input" + mtype: "" + } + member { + name: "input_mask" + mtype: "" + } + member { + name: "input_shape" + mtype: "" + } + member { + name: "losses" + mtype: "" + } + member { + name: "name" + mtype: "" + } + member { + name: "non_trainable_variables" + mtype: "" + } + member { + name: "non_trainable_weights" + mtype: "" + } + member { + name: "outbound_nodes" + mtype: "" + } + member { + name: "output" + mtype: "" + } + member { + name: "output_mask" + mtype: "" + } + member { + name: "output_shape" + mtype: "" + } + member { + name: "trainable_variables" + mtype: "" + } + member { + name: "trainable_weights" + mtype: "" + } + member { + name: "updates" + mtype: "" + } + member { + name: "variables" + mtype: "" + } + member { + name: "weights" + mtype: "" + } + member_method { + name: "__init__" + argspec: "args=[\'self\', \'thresholds\', \'name\', \'dtype\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], " + } + member_method { + name: "add_loss" + argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } + member_method { + name: "add_update" + argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "add_variable" + argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None" + } + member_method { + name: "add_weight" + argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\'], " + } + member_method { + name: "apply" + argspec: "args=[\'self\', \'inputs\'], varargs=args, keywords=kwargs, defaults=None" + } + member_method { + name: "build" + argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "call" + argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=kwargs, defaults=None" + } + member_method { + name: "compute_mask" + argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "compute_output_shape" + argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "count_params" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "from_config" + argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_config" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_input_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_input_mask_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_input_shape_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_losses_for" + argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_output_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_output_mask_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_output_shape_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_updates_for" + argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_weights" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "reset_states" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "result" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "set_weights" + argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "update_state" + argspec: "args=[\'self\', \'y_true\', \'y_pred\', \'sample_weight\'], varargs=None, keywords=None, defaults=[\'None\'], " + } +} diff --git a/tensorflow/tools/api/golden/v1/tensorflow.metrics.-recall.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.metrics.-recall.pbtxt new file mode 100644 index 00000000000..840a68bbc78 --- /dev/null +++ b/tensorflow/tools/api/golden/v1/tensorflow.metrics.-recall.pbtxt @@ -0,0 +1,192 @@ +path: "tensorflow.metrics.Recall" +tf_class { + is_instance: "" + is_instance: "" + is_instance: "" + is_instance: "" + is_instance: "" + member { + name: "activity_regularizer" + mtype: "" + } + member { + name: "dtype" + mtype: "" + } + member { + name: "inbound_nodes" + mtype: "" + } + member { + name: "input" + mtype: "" + } + member { + name: "input_mask" + mtype: "" + } + member { + name: "input_shape" + mtype: "" + } + member { + name: "losses" + mtype: "" + } + member { + name: "name" + mtype: "" + } + member { + name: "non_trainable_variables" + mtype: "" + } + member { + name: "non_trainable_weights" + mtype: "" + } + member { + name: "outbound_nodes" + mtype: "" + } + member { + name: "output" + mtype: "" + } + member { + name: "output_mask" + mtype: "" + } + member { + name: "output_shape" + mtype: "" + } + member { + name: "trainable_variables" + mtype: "" + } + member { + name: "trainable_weights" + mtype: "" + } + member { + name: "updates" + mtype: "" + } + member { + name: "variables" + mtype: "" + } + member { + name: "weights" + mtype: "" + } + member_method { + name: "__init__" + argspec: "args=[\'self\', \'thresholds\', \'name\', \'dtype\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], " + } + member_method { + name: "add_loss" + argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } + member_method { + name: "add_update" + argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "add_variable" + argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None" + } + member_method { + name: "add_weight" + argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\'], " + } + member_method { + name: "apply" + argspec: "args=[\'self\', \'inputs\'], varargs=args, keywords=kwargs, defaults=None" + } + member_method { + name: "build" + argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "call" + argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=kwargs, defaults=None" + } + member_method { + name: "compute_mask" + argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "compute_output_shape" + argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "count_params" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "from_config" + argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_config" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_input_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_input_mask_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_input_shape_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_losses_for" + argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_output_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_output_mask_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_output_shape_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_updates_for" + argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_weights" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "reset_states" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "result" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "set_weights" + argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "update_state" + argspec: "args=[\'self\', \'y_true\', \'y_pred\', \'sample_weight\'], varargs=None, keywords=None, defaults=[\'None\'], " + } +} diff --git a/tensorflow/tools/api/golden/v1/tensorflow.metrics.-sparse-categorical-accuracy.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.metrics.-sparse-categorical-accuracy.pbtxt new file mode 100644 index 00000000000..7bce43fbdeb --- /dev/null +++ b/tensorflow/tools/api/golden/v1/tensorflow.metrics.-sparse-categorical-accuracy.pbtxt @@ -0,0 +1,194 @@ +path: "tensorflow.metrics.SparseCategoricalAccuracy" +tf_class { + is_instance: "" + is_instance: "" + is_instance: "" + is_instance: "" + is_instance: "" + is_instance: "" + is_instance: "" + member { + name: "activity_regularizer" + mtype: "" + } + member { + name: "dtype" + mtype: "" + } + member { + name: "inbound_nodes" + mtype: "" + } + member { + name: "input" + mtype: "" + } + member { + name: "input_mask" + mtype: "" + } + member { + name: "input_shape" + mtype: "" + } + member { + name: "losses" + mtype: "" + } + member { + name: "name" + mtype: "" + } + member { + name: "non_trainable_variables" + mtype: "" + } + member { + name: "non_trainable_weights" + mtype: "" + } + member { + name: "outbound_nodes" + mtype: "" + } + member { + name: "output" + mtype: "" + } + member { + name: "output_mask" + mtype: "" + } + member { + name: "output_shape" + mtype: "" + } + member { + name: "trainable_variables" + mtype: "" + } + member { + name: "trainable_weights" + mtype: "" + } + member { + name: "updates" + mtype: "" + } + member { + name: "variables" + mtype: "" + } + member { + name: "weights" + mtype: "" + } + member_method { + name: "__init__" + argspec: "args=[\'self\', \'name\', \'dtype\'], varargs=None, keywords=None, defaults=[\'sparse_categorical_accuracy\', \'None\'], " + } + member_method { + name: "add_loss" + argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } + member_method { + name: "add_update" + argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "add_variable" + argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None" + } + member_method { + name: "add_weight" + argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\'], " + } + member_method { + name: "apply" + argspec: "args=[\'self\', \'inputs\'], varargs=args, keywords=kwargs, defaults=None" + } + member_method { + name: "build" + argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "call" + argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=kwargs, defaults=None" + } + member_method { + name: "compute_mask" + argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "compute_output_shape" + argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "count_params" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "from_config" + argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_config" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_input_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_input_mask_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_input_shape_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_losses_for" + argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_output_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_output_mask_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_output_shape_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_updates_for" + argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_weights" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "reset_states" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "result" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "set_weights" + argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "update_state" + argspec: "args=[\'self\', \'y_true\', \'y_pred\', \'sample_weight\'], varargs=None, keywords=None, defaults=[\'None\'], " + } +} diff --git a/tensorflow/tools/api/golden/v1/tensorflow.metrics.-true-negatives.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.metrics.-true-negatives.pbtxt new file mode 100644 index 00000000000..83cd5b736bc --- /dev/null +++ b/tensorflow/tools/api/golden/v1/tensorflow.metrics.-true-negatives.pbtxt @@ -0,0 +1,193 @@ +path: "tensorflow.metrics.TrueNegatives" +tf_class { + is_instance: "" + is_instance: "" + is_instance: "" + is_instance: "" + is_instance: "" + is_instance: "" + member { + name: "activity_regularizer" + mtype: "" + } + member { + name: "dtype" + mtype: "" + } + member { + name: "inbound_nodes" + mtype: "" + } + member { + name: "input" + mtype: "" + } + member { + name: "input_mask" + mtype: "" + } + member { + name: "input_shape" + mtype: "" + } + member { + name: "losses" + mtype: "" + } + member { + name: "name" + mtype: "" + } + member { + name: "non_trainable_variables" + mtype: "" + } + member { + name: "non_trainable_weights" + mtype: "" + } + member { + name: "outbound_nodes" + mtype: "" + } + member { + name: "output" + mtype: "" + } + member { + name: "output_mask" + mtype: "" + } + member { + name: "output_shape" + mtype: "" + } + member { + name: "trainable_variables" + mtype: "" + } + member { + name: "trainable_weights" + mtype: "" + } + member { + name: "updates" + mtype: "" + } + member { + name: "variables" + mtype: "" + } + member { + name: "weights" + mtype: "" + } + member_method { + name: "__init__" + argspec: "args=[\'self\', \'thresholds\', \'name\', \'dtype\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], " + } + member_method { + name: "add_loss" + argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } + member_method { + name: "add_update" + argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "add_variable" + argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None" + } + member_method { + name: "add_weight" + argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\'], " + } + member_method { + name: "apply" + argspec: "args=[\'self\', \'inputs\'], varargs=args, keywords=kwargs, defaults=None" + } + member_method { + name: "build" + argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "call" + argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=kwargs, defaults=None" + } + member_method { + name: "compute_mask" + argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "compute_output_shape" + argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "count_params" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "from_config" + argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_config" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_input_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_input_mask_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_input_shape_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_losses_for" + argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_output_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_output_mask_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_output_shape_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_updates_for" + argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_weights" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "reset_states" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "result" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "set_weights" + argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "update_state" + argspec: "args=[\'self\', \'y_true\', \'y_pred\', \'sample_weight\'], varargs=None, keywords=None, defaults=[\'None\'], " + } +} diff --git a/tensorflow/tools/api/golden/v1/tensorflow.metrics.-true-positives.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.metrics.-true-positives.pbtxt new file mode 100644 index 00000000000..5b2502eafee --- /dev/null +++ b/tensorflow/tools/api/golden/v1/tensorflow.metrics.-true-positives.pbtxt @@ -0,0 +1,193 @@ +path: "tensorflow.metrics.TruePositives" +tf_class { + is_instance: "" + is_instance: "" + is_instance: "" + is_instance: "" + is_instance: "" + is_instance: "" + member { + name: "activity_regularizer" + mtype: "" + } + member { + name: "dtype" + mtype: "" + } + member { + name: "inbound_nodes" + mtype: "" + } + member { + name: "input" + mtype: "" + } + member { + name: "input_mask" + mtype: "" + } + member { + name: "input_shape" + mtype: "" + } + member { + name: "losses" + mtype: "" + } + member { + name: "name" + mtype: "" + } + member { + name: "non_trainable_variables" + mtype: "" + } + member { + name: "non_trainable_weights" + mtype: "" + } + member { + name: "outbound_nodes" + mtype: "" + } + member { + name: "output" + mtype: "" + } + member { + name: "output_mask" + mtype: "" + } + member { + name: "output_shape" + mtype: "" + } + member { + name: "trainable_variables" + mtype: "" + } + member { + name: "trainable_weights" + mtype: "" + } + member { + name: "updates" + mtype: "" + } + member { + name: "variables" + mtype: "" + } + member { + name: "weights" + mtype: "" + } + member_method { + name: "__init__" + argspec: "args=[\'self\', \'thresholds\', \'name\', \'dtype\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], " + } + member_method { + name: "add_loss" + argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } + member_method { + name: "add_update" + argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "add_variable" + argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None" + } + member_method { + name: "add_weight" + argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\'], " + } + member_method { + name: "apply" + argspec: "args=[\'self\', \'inputs\'], varargs=args, keywords=kwargs, defaults=None" + } + member_method { + name: "build" + argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "call" + argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=kwargs, defaults=None" + } + member_method { + name: "compute_mask" + argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "compute_output_shape" + argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "count_params" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "from_config" + argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_config" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_input_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_input_mask_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_input_shape_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_losses_for" + argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_output_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_output_mask_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_output_shape_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_updates_for" + argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_weights" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "reset_states" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "result" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "set_weights" + argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "update_state" + argspec: "args=[\'self\', \'y_true\', \'y_pred\', \'sample_weight\'], varargs=None, keywords=None, defaults=[\'None\'], " + } +} diff --git a/tensorflow/tools/api/golden/v1/tensorflow.metrics.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.metrics.pbtxt index e9b996c9f53..f5c267a1664 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.metrics.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.metrics.pbtxt @@ -1,5 +1,49 @@ path: "tensorflow.metrics" tf_module { + member { + name: "Accuracy" + mtype: "" + } + member { + name: "BinaryAccuracy" + mtype: "" + } + member { + name: "CategoricalAccuracy" + mtype: "" + } + member { + name: "FalseNegatives" + mtype: "" + } + member { + name: "FalsePositives" + mtype: "" + } + member { + name: "Mean" + mtype: "" + } + member { + name: "Precision" + mtype: "" + } + member { + name: "Recall" + mtype: "" + } + member { + name: "SparseCategoricalAccuracy" + mtype: "" + } + member { + name: "TrueNegatives" + mtype: "" + } + member { + name: "TruePositives" + mtype: "" + } member_method { name: "accuracy" argspec: "args=[\'labels\', \'predictions\', \'weights\', \'metrics_collections\', \'updates_collections\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], " diff --git a/tensorflow/tools/api/golden/v1/tensorflow.nn.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.nn.pbtxt index 93f2fda2acf..40e20f8c919 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.nn.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.nn.pbtxt @@ -44,6 +44,10 @@ tf_module { name: "bidirectional_dynamic_rnn" argspec: "args=[\'cell_fw\', \'cell_bw\', \'inputs\', \'sequence_length\', \'initial_state_fw\', \'initial_state_bw\', \'dtype\', \'parallel_iterations\', \'swap_memory\', \'time_major\', \'scope\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'False\', \'False\', \'None\'], " } + member_method { + name: "collapse_repeated" + argspec: "args=[\'labels\', \'seq_length\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " + } member_method { name: "compute_accidental_hits" argspec: "args=[\'true_classes\', \'sampled_candidates\', \'num_true\', \'seed\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " @@ -72,6 +76,10 @@ tf_module { name: "conv3d" argspec: "args=[\'input\', \'filter\', \'strides\', \'padding\', \'data_format\', \'dilations\', \'name\'], varargs=None, keywords=None, defaults=[\'NDHWC\', \'[1, 1, 1, 1, 1]\', \'None\'], " } + member_method { + name: "conv3d_backprop_filter" + argspec: "args=[\'input\', \'filter_sizes\', \'out_backprop\', \'strides\', \'padding\', \'data_format\', \'dilations\', \'name\'], varargs=None, keywords=None, defaults=[\'NDHWC\', \'[1, 1, 1, 1, 1]\', \'None\'], " + } member_method { name: "conv3d_backprop_filter_v2" argspec: "args=[\'input\', \'filter_sizes\', \'out_backprop\', \'strides\', \'padding\', \'data_format\', \'dilations\', \'name\'], varargs=None, keywords=None, defaults=[\'NDHWC\', \'[1, 1, 1, 1, 1]\', \'None\'], " @@ -104,6 +112,14 @@ tf_module { name: "ctc_loss" argspec: "args=[\'labels\', \'inputs\', \'sequence_length\', \'preprocess_collapse_repeated\', \'ctc_merge_repeated\', \'ignore_longer_outputs_than_inputs\', \'time_major\'], varargs=None, keywords=None, defaults=[\'False\', \'True\', \'False\', \'True\'], " } + member_method { + name: "ctc_loss_v2" + argspec: "args=[\'labels\', \'logits\', \'label_length\', \'logit_length\', \'logits_time_major\', \'unique\', \'blank_index\', \'name\'], varargs=None, keywords=None, defaults=[\'True\', \'None\', \'None\', \'None\'], " + } + member_method { + name: "ctc_unique_labels" + argspec: "args=[\'labels\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " + } member_method { name: "depth_to_space" argspec: "args=[\'input\', \'block_size\', \'name\', \'data_format\'], varargs=None, keywords=None, defaults=[\'None\', \'NHWC\'], " @@ -112,6 +128,14 @@ tf_module { name: "depthwise_conv2d" argspec: "args=[\'input\', \'filter\', \'strides\', \'padding\', \'rate\', \'name\', \'data_format\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], " } + member_method { + name: "depthwise_conv2d_backprop_filter" + argspec: "args=[\'input\', \'filter_sizes\', \'out_backprop\', \'strides\', \'padding\', \'data_format\', \'dilations\', \'name\'], varargs=None, keywords=None, defaults=[\'NHWC\', \'[1, 1, 1, 1]\', \'None\'], " + } + member_method { + name: "depthwise_conv2d_backprop_input" + argspec: "args=[\'input_sizes\', \'filter\', \'out_backprop\', \'strides\', \'padding\', \'data_format\', \'dilations\', \'name\'], varargs=None, keywords=None, defaults=[\'NHWC\', \'[1, 1, 1, 1]\', \'None\'], " + } member_method { name: "depthwise_conv2d_native" argspec: "args=[\'input\', \'filter\', \'strides\', \'padding\', \'data_format\', \'dilations\', \'name\'], varargs=None, keywords=None, defaults=[\'NHWC\', \'[1, 1, 1, 1]\', \'None\'], " @@ -130,7 +154,7 @@ tf_module { } member_method { name: "dropout" - argspec: "args=[\'x\', \'keep_prob\', \'noise_shape\', \'seed\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], " + argspec: "args=[\'x\', \'keep_prob\', \'noise_shape\', \'seed\', \'name\', \'rate\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\'], " } member_method { name: "dynamic_rnn" @@ -302,7 +326,7 @@ tf_module { } member_method { name: "softmax_cross_entropy_with_logits_v2" - argspec: "args=[\'labels\', \'logits\', \'dim\', \'name\'], varargs=None, keywords=None, defaults=[\'-1\', \'None\'], " + argspec: "args=[\'labels\', \'logits\', \'axis\', \'name\', \'dim\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], " } member_method { name: "softplus" diff --git a/tensorflow/tools/api/golden/v1/tensorflow.nn.rnn_cell.-basic-l-s-t-m-cell.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.nn.rnn_cell.-basic-l-s-t-m-cell.pbtxt index 88b8f37c4ff..f7f9978c063 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.nn.rnn_cell.-basic-l-s-t-m-cell.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.nn.rnn_cell.-basic-l-s-t-m-cell.pbtxt @@ -107,6 +107,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v1/tensorflow.nn.rnn_cell.-basic-r-n-n-cell.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.nn.rnn_cell.-basic-r-n-n-cell.pbtxt index a4483fefa27..f9e898484b9 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.nn.rnn_cell.-basic-r-n-n-cell.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.nn.rnn_cell.-basic-r-n-n-cell.pbtxt @@ -107,6 +107,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v1/tensorflow.nn.rnn_cell.-device-wrapper.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.nn.rnn_cell.-device-wrapper.pbtxt index 381c4975d7d..9e52a425261 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.nn.rnn_cell.-device-wrapper.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.nn.rnn_cell.-device-wrapper.pbtxt @@ -106,6 +106,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v1/tensorflow.nn.rnn_cell.-dropout-wrapper.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.nn.rnn_cell.-dropout-wrapper.pbtxt index 912365a28b1..9836433d08c 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.nn.rnn_cell.-dropout-wrapper.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.nn.rnn_cell.-dropout-wrapper.pbtxt @@ -110,6 +110,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v1/tensorflow.nn.rnn_cell.-g-r-u-cell.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.nn.rnn_cell.-g-r-u-cell.pbtxt index a4bb3219c79..5fd9b329bde 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.nn.rnn_cell.-g-r-u-cell.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.nn.rnn_cell.-g-r-u-cell.pbtxt @@ -107,6 +107,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v1/tensorflow.nn.rnn_cell.-l-s-t-m-cell.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.nn.rnn_cell.-l-s-t-m-cell.pbtxt index 715bfd5fc7c..76c8cff22b1 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.nn.rnn_cell.-l-s-t-m-cell.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.nn.rnn_cell.-l-s-t-m-cell.pbtxt @@ -107,6 +107,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v1/tensorflow.nn.rnn_cell.-multi-r-n-n-cell.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.nn.rnn_cell.-multi-r-n-n-cell.pbtxt index b66c0f89cc9..f53567af52f 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.nn.rnn_cell.-multi-r-n-n-cell.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.nn.rnn_cell.-multi-r-n-n-cell.pbtxt @@ -106,6 +106,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v1/tensorflow.nn.rnn_cell.-r-n-n-cell.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.nn.rnn_cell.-r-n-n-cell.pbtxt index faeb4f35133..d3b68e4f297 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.nn.rnn_cell.-r-n-n-cell.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.nn.rnn_cell.-r-n-n-cell.pbtxt @@ -105,6 +105,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v1/tensorflow.nn.rnn_cell.-residual-wrapper.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.nn.rnn_cell.-residual-wrapper.pbtxt index caa2e600800..1f7840ab919 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.nn.rnn_cell.-residual-wrapper.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.nn.rnn_cell.-residual-wrapper.pbtxt @@ -106,6 +106,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v1/tensorflow.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.pbtxt index f4dce81659d..584c74f99d8 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.pbtxt @@ -244,6 +244,10 @@ tf_module { name: "TensorShape" mtype: "" } + member { + name: "TensorSpec" + mtype: "" + } member { name: "TextLineReader" mtype: "" @@ -320,6 +324,10 @@ tf_module { name: "debugging" mtype: "" } + member { + name: "distribute" + mtype: "" + } member { name: "distributions" mtype: "" @@ -692,6 +700,10 @@ tf_module { name: "argmin" argspec: "args=[\'input\', \'axis\', \'name\', \'dimension\', \'output_type\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \"\"], " } + member_method { + name: "argsort" + argspec: "args=[\'values\', \'axis\', \'direction\', \'stable\', \'name\'], varargs=None, keywords=None, defaults=[\'-1\', \'ASCENDING\', \'False\', \'None\'], " + } member_method { name: "as_dtype" argspec: "args=[\'type_value\'], varargs=None, keywords=None, defaults=None" @@ -778,7 +790,7 @@ tf_module { } member_method { name: "assert_scalar" - argspec: "args=[\'tensor\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " + argspec: "args=[\'tensor\', \'name\', \'message\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " } member_method { name: "assert_type" @@ -1040,10 +1052,18 @@ tf_module { name: "dimension_value" argspec: "args=[\'dimension\'], varargs=None, keywords=None, defaults=None" } + member_method { + name: "disable_eager_execution" + argspec: "args=[], varargs=None, keywords=None, defaults=None" + } member_method { name: "disable_resource_variables" argspec: "args=[], varargs=None, keywords=None, defaults=None" } + member_method { + name: "disable_v2_behavior" + argspec: "args=[], varargs=None, keywords=None, defaults=None" + } member_method { name: "disable_v2_tensorshape" argspec: "args=[], varargs=None, keywords=None, defaults=None" @@ -1084,6 +1104,10 @@ tf_module { name: "enable_resource_variables" argspec: "args=[], varargs=None, keywords=None, defaults=None" } + member_method { + name: "enable_v2_behavior" + argspec: "args=[], varargs=None, keywords=None, defaults=None" + } member_method { name: "enable_v2_tensorshape" argspec: "args=[], varargs=None, keywords=None, defaults=None" @@ -1232,6 +1256,10 @@ tf_module { name: "get_local_variable" argspec: "args=[\'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'collections\', \'caching_device\', \'partitioner\', \'validate_shape\', \'use_resource\', \'custom_getter\', \'constraint\', \'synchronization\', \'aggregation\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\', \'False\', \'None\', \'None\', \'None\', \'True\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], " } + member_method { + name: "get_logger" + argspec: "args=[], varargs=None, keywords=None, defaults=None" + } member_method { name: "get_seed" argspec: "args=[\'op_seed\'], varargs=None, keywords=None, defaults=None" @@ -1466,7 +1494,7 @@ tf_module { } member_method { name: "make_tensor_proto" - argspec: "args=[\'values\', \'dtype\', \'shape\', \'verify_shape\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'False\'], " + argspec: "args=[\'values\', \'dtype\', \'shape\', \'verify_shape\', \'allow_broadcast\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'False\', \'False\'], " } member_method { name: "map_fn" @@ -1810,7 +1838,7 @@ tf_module { } member_method { name: "scalar_mul" - argspec: "args=[\'scalar\', \'x\'], varargs=None, keywords=None, defaults=None" + argspec: "args=[\'scalar\', \'x\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " } member_method { name: "scan" @@ -1948,6 +1976,10 @@ tf_module { name: "slice" argspec: "args=[\'input_\', \'begin\', \'size\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "sort" + argspec: "args=[\'values\', \'axis\', \'direction\', \'name\'], varargs=None, keywords=None, defaults=[\'-1\', \'ASCENDING\', \'None\'], " + } member_method { name: "space_to_batch" argspec: "args=[\'input\', \'paddings\', \'block_size\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " @@ -1962,7 +1994,7 @@ tf_module { } member_method { name: "sparse_add" - argspec: "args=[\'a\', \'b\', \'thresh\'], varargs=None, keywords=None, defaults=[\'0\'], " + argspec: "args=[\'a\', \'b\', \'threshold\', \'thresh\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " } member_method { name: "sparse_concat" @@ -2156,6 +2188,18 @@ tf_module { name: "tanh" argspec: "args=[\'x\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "tensor_scatter_add" + argspec: "args=[\'tensor\', \'indices\', \'updates\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "tensor_scatter_sub" + argspec: "args=[\'tensor\', \'indices\', \'updates\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "tensor_scatter_update" + argspec: "args=[\'tensor\', \'indices\', \'updates\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " + } member_method { name: "tensordot" argspec: "args=[\'a\', \'b\', \'axes\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " @@ -2296,6 +2340,10 @@ tf_module { name: "while_loop" argspec: "args=[\'cond\', \'body\', \'loop_vars\', \'shape_invariants\', \'parallel_iterations\', \'back_prop\', \'swap_memory\', \'name\', \'maximum_iterations\', \'return_same_structure\'], varargs=None, keywords=None, defaults=[\'None\', \'10\', \'True\', \'False\', \'None\', \'None\', \'False\'], " } + member_method { + name: "wrap_function" + argspec: "args=[\'fn\', \'signature\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " + } member_method { name: "write_file" argspec: "args=[\'filename\', \'contents\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v1/tensorflow.quantization.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.quantization.pbtxt index 2948b7318ea..632c2f8f83c 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.quantization.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.quantization.pbtxt @@ -34,7 +34,7 @@ tf_module { } member_method { name: "quantize_and_dequantize" - argspec: "args=[\'input\', \'input_min\', \'input_max\', \'signed_input\', \'num_bits\', \'range_given\', \'name\'], varargs=None, keywords=None, defaults=[\'True\', \'8\', \'False\', \'None\'], " + argspec: "args=[\'input\', \'input_min\', \'input_max\', \'signed_input\', \'num_bits\', \'range_given\', \'round_mode\', \'name\'], varargs=None, keywords=None, defaults=[\'True\', \'8\', \'False\', \'HALF_TO_EVEN\', \'None\'], " } member_method { name: "quantized_concat" diff --git a/tensorflow/tools/api/golden/v1/tensorflow.random.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.random.pbtxt index 160c09798d0..107534e0869 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.random.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.random.pbtxt @@ -1,5 +1,9 @@ path: "tensorflow.random" tf_module { + member_method { + name: "categorical" + argspec: "args=[\'logits\', \'num_samples\', \'dtype\', \'seed\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], " + } member_method { name: "gamma" argspec: "args=[\'shape\', \'alpha\', \'beta\', \'dtype\', \'seed\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \"\", \'None\', \'None\'], " @@ -32,6 +36,10 @@ tf_module { name: "shuffle" argspec: "args=[\'value\', \'seed\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " } + member_method { + name: "stateless_categorical" + argspec: "args=[\'logits\', \'num_samples\', \'seed\', \'dtype\', \'name\'], varargs=None, keywords=None, defaults=[\"\", \'None\'], " + } member_method { name: "stateless_multinomial" argspec: "args=[\'logits\', \'num_samples\', \'seed\', \'output_dtype\', \'name\'], varargs=None, keywords=None, defaults=[\"\", \'None\'], " diff --git a/tensorflow/tools/api/golden/v1/tensorflow.saved_model.-builder.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.saved_model.-builder.pbtxt index 67457de0708..e4cc0061a95 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.saved_model.-builder.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.saved_model.-builder.pbtxt @@ -1,6 +1,7 @@ path: "tensorflow.saved_model.Builder" tf_class { is_instance: "" + is_instance: "" is_instance: "" member_method { name: "__init__" diff --git a/tensorflow/tools/api/golden/v1/tensorflow.saved_model.builder.-saved-model-builder.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.saved_model.builder.-saved-model-builder.pbtxt index 83bd7035409..44860b11720 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.saved_model.builder.-saved-model-builder.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.saved_model.builder.-saved-model-builder.pbtxt @@ -1,6 +1,7 @@ path: "tensorflow.saved_model.builder.SavedModelBuilder" tf_class { is_instance: "" + is_instance: "" is_instance: "" member_method { name: "__init__" diff --git a/tensorflow/tools/api/golden/v1/tensorflow.saved_model.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.saved_model.pbtxt index 2055bfbf066..3929003fa1f 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.saved_model.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.saved_model.pbtxt @@ -148,6 +148,10 @@ tf_module { name: "classification_signature_def" argspec: "args=[\'examples\', \'classes\', \'scores\'], varargs=None, keywords=None, defaults=None" } + member_method { + name: "contains_saved_model" + argspec: "args=[\'export_dir\'], varargs=None, keywords=None, defaults=None" + } member_method { name: "get_tensor_from_tensor_info" argspec: "args=[\'tensor_info\', \'graph\', \'import_scope\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " diff --git a/tensorflow/tools/api/golden/v1/tensorflow.sets.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.sets.pbtxt index 8a196b1a556..09d6f1424b7 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.sets.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.sets.pbtxt @@ -1,5 +1,13 @@ path: "tensorflow.sets" tf_module { + member_method { + name: "difference" + argspec: "args=[\'a\', \'b\', \'aminusb\', \'validate_indices\'], varargs=None, keywords=None, defaults=[\'True\', \'True\'], " + } + member_method { + name: "intersection" + argspec: "args=[\'a\', \'b\', \'validate_indices\'], varargs=None, keywords=None, defaults=[\'True\'], " + } member_method { name: "set_difference" argspec: "args=[\'a\', \'b\', \'aminusb\', \'validate_indices\'], varargs=None, keywords=None, defaults=[\'True\', \'True\'], " @@ -16,4 +24,12 @@ tf_module { name: "set_union" argspec: "args=[\'a\', \'b\', \'validate_indices\'], varargs=None, keywords=None, defaults=[\'True\'], " } + member_method { + name: "size" + argspec: "args=[\'a\', \'validate_indices\'], varargs=None, keywords=None, defaults=[\'True\'], " + } + member_method { + name: "union" + argspec: "args=[\'a\', \'b\', \'validate_indices\'], varargs=None, keywords=None, defaults=[\'True\'], " + } } diff --git a/tensorflow/tools/api/golden/v1/tensorflow.signal.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.signal.pbtxt index 2c50c41f186..ea717b4d719 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.signal.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.signal.pbtxt @@ -1,5 +1,9 @@ path: "tensorflow.signal" tf_module { + member_method { + name: "dct" + argspec: "args=[\'input\', \'type\', \'n\', \'axis\', \'norm\', \'name\'], varargs=None, keywords=None, defaults=[\'2\', \'None\', \'-1\', \'None\', \'None\'], " + } member_method { name: "fft" argspec: "args=[\'input\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " @@ -24,6 +28,10 @@ tf_module { name: "hann_window" argspec: "args=[\'window_length\', \'periodic\', \'dtype\', \'name\'], varargs=None, keywords=None, defaults=[\'True\', \"\", \'None\'], " } + member_method { + name: "idct" + argspec: "args=[\'input\', \'type\', \'n\', \'axis\', \'norm\', \'name\'], varargs=None, keywords=None, defaults=[\'2\', \'None\', \'-1\', \'None\', \'None\'], " + } member_method { name: "ifft" argspec: "args=[\'input\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " @@ -44,6 +52,18 @@ tf_module { name: "inverse_stft_window_fn" argspec: "args=[\'frame_step\', \'forward_window_fn\', \'name\'], varargs=None, keywords=None, defaults=[\'\', \'None\'], " } + member_method { + name: "irfft" + argspec: "args=[\'input_tensor\', \'fft_length\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } + member_method { + name: "irfft2d" + argspec: "args=[\'input_tensor\', \'fft_length\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } + member_method { + name: "irfft3d" + argspec: "args=[\'input_tensor\', \'fft_length\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "linear_to_mel_weight_matrix" argspec: "args=[\'num_mel_bins\', \'num_spectrogram_bins\', \'sample_rate\', \'lower_edge_hertz\', \'upper_edge_hertz\', \'dtype\', \'name\'], varargs=None, keywords=None, defaults=[\'20\', \'129\', \'8000\', \'125.0\', \'3800.0\', \"\", \'None\'], " @@ -56,6 +76,18 @@ tf_module { name: "overlap_and_add" argspec: "args=[\'signal\', \'frame_step\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "rfft" + argspec: "args=[\'input_tensor\', \'fft_length\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } + member_method { + name: "rfft2d" + argspec: "args=[\'input_tensor\', \'fft_length\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } + member_method { + name: "rfft3d" + argspec: "args=[\'input_tensor\', \'fft_length\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "stft" argspec: "args=[\'signals\', \'frame_length\', \'frame_step\', \'fft_length\', \'window_fn\', \'pad_end\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'\', \'False\', \'None\'], " diff --git a/tensorflow/tools/api/golden/v1/tensorflow.sparse.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.sparse.pbtxt index 32bd8d5f8ed..33e342bc754 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.sparse.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.sparse.pbtxt @@ -10,7 +10,7 @@ tf_module { } member_method { name: "add" - argspec: "args=[\'a\', \'b\', \'thresh\'], varargs=None, keywords=None, defaults=[\'0\'], " + argspec: "args=[\'a\', \'b\', \'threshold\', \'thresh\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " } member_method { name: "concat" @@ -112,6 +112,10 @@ tf_module { name: "softmax" argspec: "args=[\'sp_input\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "sparse_dense_matmul" + argspec: "args=[\'sp_a\', \'b\', \'adjoint_a\', \'adjoint_b\', \'name\'], varargs=None, keywords=None, defaults=[\'False\', \'False\', \'None\'], " + } member_method { name: "split" argspec: "args=[\'keyword_required\', \'sp_input\', \'num_split\', \'axis\', \'name\', \'split_dim\'], varargs=None, keywords=None, defaults=[\'KeywordRequired()\', \'None\', \'None\', \'None\', \'None\', \'None\'], " diff --git a/tensorflow/tools/api/golden/v1/tensorflow.strings.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.strings.pbtxt index 03144cbe709..a1cd581a86b 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.strings.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.strings.pbtxt @@ -52,6 +52,10 @@ tf_module { name: "to_number" argspec: "args=[\'string_tensor\', \'out_type\', \'name\'], varargs=None, keywords=None, defaults=[\"\", \'None\'], " } + member_method { + name: "unicode_encode" + argspec: "args=[\'input\', \'output_encoding\', \'errors\', \'replacement_char\', \'name\'], varargs=None, keywords=None, defaults=[\'replace\', \'65533\', \'None\'], " + } member_method { name: "unicode_script" argspec: "args=[\'input\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v1/tensorflow.test.-benchmark.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.test.-benchmark.pbtxt index df528e26b60..6fc489c8604 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.test.-benchmark.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.test.-benchmark.pbtxt @@ -6,6 +6,10 @@ tf_class { member_method { name: "__init__" } + member_method { + name: "evaluate" + argspec: "args=[\'self\', \'tensors\'], varargs=None, keywords=None, defaults=None" + } member_method { name: "is_abstract" argspec: "args=[\'cls\'], varargs=None, keywords=None, defaults=None" diff --git a/tensorflow/tools/api/golden/v1/tensorflow.train.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.train.pbtxt index 877c55c6b38..bdb3ea2197c 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.train.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.train.pbtxt @@ -396,6 +396,10 @@ tf_module { name: "piecewise_constant" argspec: "args=[\'x\', \'boundaries\', \'values\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "piecewise_constant_decay" + argspec: "args=[\'x\', \'boundaries\', \'values\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " + } member_method { name: "polynomial_decay" argspec: "args=[\'learning_rate\', \'global_step\', \'decay_steps\', \'end_learning_rate\', \'power\', \'cycle\', \'name\'], varargs=None, keywords=None, defaults=[\'0.0001\', \'1.0\', \'False\', \'None\'], " diff --git a/tensorflow/tools/api/golden/v2/tensorflow.-conditional-accumulator-base.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.-conditional-accumulator-base.pbtxt deleted file mode 100644 index c9a32c16b34..00000000000 --- a/tensorflow/tools/api/golden/v2/tensorflow.-conditional-accumulator-base.pbtxt +++ /dev/null @@ -1,29 +0,0 @@ -path: "tensorflow.ConditionalAccumulatorBase" -tf_class { - is_instance: "" - is_instance: "" - member { - name: "accumulator_ref" - mtype: "" - } - member { - name: "dtype" - mtype: "" - } - member { - name: "name" - mtype: "" - } - member_method { - name: "__init__" - argspec: "args=[\'self\', \'dtype\', \'shape\', \'accumulator_ref\'], varargs=None, keywords=None, defaults=None" - } - member_method { - name: "num_accumulated" - argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " - } - member_method { - name: "set_global_step" - argspec: "args=[\'self\', \'new_global_step\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " - } -} diff --git a/tensorflow/tools/api/golden/v2/tensorflow.-conditional-accumulator.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.-conditional-accumulator.pbtxt deleted file mode 100644 index 15e0ab76b6f..00000000000 --- a/tensorflow/tools/api/golden/v2/tensorflow.-conditional-accumulator.pbtxt +++ /dev/null @@ -1,38 +0,0 @@ -path: "tensorflow.ConditionalAccumulator" -tf_class { - is_instance: "" - is_instance: "" - is_instance: "" - member { - name: "accumulator_ref" - mtype: "" - } - member { - name: "dtype" - mtype: "" - } - member { - name: "name" - mtype: "" - } - member_method { - name: "__init__" - argspec: "args=[\'self\', \'dtype\', \'shape\', \'shared_name\', \'name\', \'reduction_type\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'conditional_accumulator\', \'MEAN\'], " - } - member_method { - name: "apply_grad" - argspec: "args=[\'self\', \'grad\', \'local_step\', \'name\'], varargs=None, keywords=None, defaults=[\'0\', \'None\'], " - } - member_method { - name: "num_accumulated" - argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " - } - member_method { - name: "set_global_step" - argspec: "args=[\'self\', \'new_global_step\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " - } - member_method { - name: "take_grad" - argspec: "args=[\'self\', \'num_required\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " - } -} diff --git a/tensorflow/tools/api/golden/v2/tensorflow.-config-proto.-experimental.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.-config-proto.-experimental.pbtxt index f7491649c22..caa72fe5a61 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.-config-proto.-experimental.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.-config-proto.-experimental.pbtxt @@ -20,6 +20,12 @@ tf_proto { label: LABEL_OPTIONAL type: TYPE_INT32 } + field { + name: "use_numa_affinity" + number: 5 + label: LABEL_OPTIONAL + type: TYPE_BOOL + } reserved_range { start: 2 end: 3 diff --git a/tensorflow/tools/api/golden/v2/tensorflow.-config-proto.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.-config-proto.pbtxt index 53b532beab3..b505d813509 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.-config-proto.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.-config-proto.pbtxt @@ -143,6 +143,12 @@ tf_proto { label: LABEL_OPTIONAL type: TYPE_INT32 } + field { + name: "use_numa_affinity" + number: 5 + label: LABEL_OPTIONAL + type: TYPE_BOOL + } reserved_range { start: 2 end: 3 diff --git a/tensorflow/tools/api/golden/v2/tensorflow.-device-spec.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.-device-spec.pbtxt deleted file mode 100644 index 92e535c3414..00000000000 --- a/tensorflow/tools/api/golden/v2/tensorflow.-device-spec.pbtxt +++ /dev/null @@ -1,37 +0,0 @@ -path: "tensorflow.DeviceSpec" -tf_class { - is_instance: "" - is_instance: "" - member { - name: "job" - mtype: "" - } - member { - name: "replica" - mtype: "" - } - member { - name: "task" - mtype: "" - } - member_method { - name: "__init__" - argspec: "args=[\'self\', \'job\', \'replica\', \'task\', \'device_type\', \'device_index\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\'], " - } - member_method { - name: "from_string" - argspec: "args=[\'spec\'], varargs=None, keywords=None, defaults=None" - } - member_method { - name: "merge_from" - argspec: "args=[\'self\', \'dev\'], varargs=None, keywords=None, defaults=None" - } - member_method { - name: "parse_from_string" - argspec: "args=[\'self\', \'spec\'], varargs=None, keywords=None, defaults=None" - } - member_method { - name: "to_string" - argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" - } -} diff --git a/tensorflow/tools/api/golden/v2/tensorflow.-dimension.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.-dimension.pbtxt deleted file mode 100644 index a9ab27719b4..00000000000 --- a/tensorflow/tools/api/golden/v2/tensorflow.-dimension.pbtxt +++ /dev/null @@ -1,25 +0,0 @@ -path: "tensorflow.Dimension" -tf_class { - is_instance: "" - is_instance: "" - member { - name: "value" - mtype: "" - } - member_method { - name: "__init__" - argspec: "args=[\'self\', \'value\'], varargs=None, keywords=None, defaults=None" - } - member_method { - name: "assert_is_compatible_with" - argspec: "args=[\'self\', \'other\'], varargs=None, keywords=None, defaults=None" - } - member_method { - name: "is_compatible_with" - argspec: "args=[\'self\', \'other\'], varargs=None, keywords=None, defaults=None" - } - member_method { - name: "merge_with" - argspec: "args=[\'self\', \'other\'], varargs=None, keywords=None, defaults=None" - } -} diff --git a/tensorflow/tools/api/golden/v2/tensorflow.-gradient-tape.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.-gradient-tape.pbtxt index 0a16d6ab92f..2299a009d3d 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.-gradient-tape.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.-gradient-tape.pbtxt @@ -6,10 +6,18 @@ tf_class { name: "__init__" argspec: "args=[\'self\', \'persistent\', \'watch_accessed_variables\'], varargs=None, keywords=None, defaults=[\'False\', \'True\'], " } + member_method { + name: "batch_jacobian" + argspec: "args=[\'self\', \'target\', \'source\', \'unconnected_gradients\', \'parallel_iterations\', \'experimental_use_pfor\'], varargs=None, keywords=None, defaults=[\'UnconnectedGradients.NONE\', \'None\', \'True\'], " + } member_method { name: "gradient" argspec: "args=[\'self\', \'target\', \'sources\', \'output_gradients\', \'unconnected_gradients\'], varargs=None, keywords=None, defaults=[\'None\', \'UnconnectedGradients.NONE\'], " } + member_method { + name: "jacobian" + argspec: "args=[\'self\', \'target\', \'sources\', \'unconnected_gradients\', \'parallel_iterations\', \'experimental_use_pfor\'], varargs=None, keywords=None, defaults=[\'UnconnectedGradients.NONE\', \'None\', \'True\'], " + } member_method { name: "reset" argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" diff --git a/tensorflow/tools/api/golden/v2/tensorflow.-graph-keys.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.-graph-keys.pbtxt deleted file mode 100644 index ffe47909339..00000000000 --- a/tensorflow/tools/api/golden/v2/tensorflow.-graph-keys.pbtxt +++ /dev/null @@ -1,140 +0,0 @@ -path: "tensorflow.GraphKeys" -tf_class { - is_instance: "" - is_instance: "" - member { - name: "ACTIVATIONS" - mtype: "" - } - member { - name: "ASSET_FILEPATHS" - mtype: "" - } - member { - name: "BIASES" - mtype: "" - } - member { - name: "CONCATENATED_VARIABLES" - mtype: "" - } - member { - name: "COND_CONTEXT" - mtype: "" - } - member { - name: "EVAL_STEP" - mtype: "" - } - member { - name: "GLOBAL_STEP" - mtype: "" - } - member { - name: "GLOBAL_VARIABLES" - mtype: "" - } - member { - name: "INIT_OP" - mtype: "" - } - member { - name: "LOCAL_INIT_OP" - mtype: "" - } - member { - name: "LOCAL_RESOURCES" - mtype: "" - } - member { - name: "LOCAL_VARIABLES" - mtype: "" - } - member { - name: "LOSSES" - mtype: "" - } - member { - name: "METRIC_VARIABLES" - mtype: "" - } - member { - name: "MODEL_VARIABLES" - mtype: "" - } - member { - name: "MOVING_AVERAGE_VARIABLES" - mtype: "" - } - member { - name: "QUEUE_RUNNERS" - mtype: "" - } - member { - name: "READY_FOR_LOCAL_INIT_OP" - mtype: "" - } - member { - name: "READY_OP" - mtype: "" - } - member { - name: "REGULARIZATION_LOSSES" - mtype: "" - } - member { - name: "RESOURCES" - mtype: "" - } - member { - name: "SAVEABLE_OBJECTS" - mtype: "" - } - member { - name: "SAVERS" - mtype: "" - } - member { - name: "SUMMARIES" - mtype: "" - } - member { - name: "SUMMARY_OP" - mtype: "" - } - member { - name: "TABLE_INITIALIZERS" - mtype: "" - } - member { - name: "TRAINABLE_RESOURCE_VARIABLES" - mtype: "" - } - member { - name: "TRAINABLE_VARIABLES" - mtype: "" - } - member { - name: "TRAIN_OP" - mtype: "" - } - member { - name: "UPDATE_OPS" - mtype: "" - } - member { - name: "VARIABLES" - mtype: "" - } - member { - name: "WEIGHTS" - mtype: "" - } - member { - name: "WHILE_CONTEXT" - mtype: "" - } - member_method { - name: "__init__" - } -} diff --git a/tensorflow/tools/api/golden/v2/tensorflow.-tensor-info.-coo-sparse.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.-tensor-info.-coo-sparse.pbtxt deleted file mode 100644 index 0064c8460cb..00000000000 --- a/tensorflow/tools/api/golden/v2/tensorflow.-tensor-info.-coo-sparse.pbtxt +++ /dev/null @@ -1,24 +0,0 @@ -path: "tensorflow.TensorInfo.CooSparse" -tf_proto { - descriptor { - name: "CooSparse" - field { - name: "values_tensor_name" - number: 1 - label: LABEL_OPTIONAL - type: TYPE_STRING - } - field { - name: "indices_tensor_name" - number: 2 - label: LABEL_OPTIONAL - type: TYPE_STRING - } - field { - name: "dense_shape_tensor_name" - number: 3 - label: LABEL_OPTIONAL - type: TYPE_STRING - } - } -} diff --git a/tensorflow/tools/api/golden/v2/tensorflow.-tensor-info.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.-tensor-info.pbtxt deleted file mode 100644 index 63566c808e5..00000000000 --- a/tensorflow/tools/api/golden/v2/tensorflow.-tensor-info.pbtxt +++ /dev/null @@ -1,59 +0,0 @@ -path: "tensorflow.TensorInfo" -tf_proto { - descriptor { - name: "TensorInfo" - field { - name: "name" - number: 1 - label: LABEL_OPTIONAL - type: TYPE_STRING - oneof_index: 0 - } - field { - name: "coo_sparse" - number: 4 - label: LABEL_OPTIONAL - type: TYPE_MESSAGE - type_name: ".tensorflow.TensorInfo.CooSparse" - oneof_index: 0 - } - field { - name: "dtype" - number: 2 - label: LABEL_OPTIONAL - type: TYPE_ENUM - type_name: ".tensorflow.DataType" - } - field { - name: "tensor_shape" - number: 3 - label: LABEL_OPTIONAL - type: TYPE_MESSAGE - type_name: ".tensorflow.TensorShapeProto" - } - nested_type { - name: "CooSparse" - field { - name: "values_tensor_name" - number: 1 - label: LABEL_OPTIONAL - type: TYPE_STRING - } - field { - name: "indices_tensor_name" - number: 2 - label: LABEL_OPTIONAL - type: TYPE_STRING - } - field { - name: "dense_shape_tensor_name" - number: 3 - label: LABEL_OPTIONAL - type: TYPE_STRING - } - } - oneof_decl { - name: "encoding" - } - } -} diff --git a/tensorflow/tools/api/golden/v2/tensorflow.-tensor-spec.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.-tensor-spec.pbtxt new file mode 100644 index 00000000000..493dcba8922 --- /dev/null +++ b/tensorflow/tools/api/golden/v2/tensorflow.-tensor-spec.pbtxt @@ -0,0 +1,33 @@ +path: "tensorflow.TensorSpec" +tf_class { + is_instance: "" + is_instance: "" + member { + name: "dtype" + mtype: "" + } + member { + name: "name" + mtype: "" + } + member { + name: "shape" + mtype: "" + } + member_method { + name: "__init__" + argspec: "args=[\'self\', \'shape\', \'dtype\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "from_spec" + argspec: "args=[\'cls\', \'spec\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "from_tensor" + argspec: "args=[\'cls\', \'tensor\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "is_compatible_with" + argspec: "args=[\'self\', \'spec_or_tensor\'], varargs=None, keywords=None, defaults=None" + } +} diff --git a/tensorflow/tools/api/golden/v2/tensorflow.app.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.app.pbtxt deleted file mode 100644 index 67e1b76caba..00000000000 --- a/tensorflow/tools/api/golden/v2/tensorflow.app.pbtxt +++ /dev/null @@ -1,7 +0,0 @@ -path: "tensorflow.app" -tf_module { - member_method { - name: "run" - argspec: "args=[\'main\', \'argv\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " - } -} diff --git a/tensorflow/tools/api/golden/v2/tensorflow.data.-dataset.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.data.-dataset.pbtxt index 8b7f63e43e2..ac8dd2de7fe 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.data.-dataset.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.data.-dataset.pbtxt @@ -1,6 +1,6 @@ path: "tensorflow.data.Dataset" tf_class { - is_instance: "" + is_instance: "" is_instance: "" member { name: "output_classes" @@ -16,7 +16,6 @@ tf_class { } member_method { name: "__init__" - argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" } member_method { name: "apply" @@ -46,10 +45,6 @@ tf_class { name: "from_generator" argspec: "args=[\'generator\', \'output_types\', \'output_shapes\', \'args\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " } - member_method { - name: "from_sparse_tensor_slices" - argspec: "args=[\'sparse_tensor\'], varargs=None, keywords=None, defaults=None" - } member_method { name: "from_tensor_slices" argspec: "args=[\'tensors\'], varargs=None, keywords=None, defaults=None" @@ -66,14 +61,6 @@ tf_class { name: "list_files" argspec: "args=[\'file_pattern\', \'shuffle\', \'seed\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " } - member_method { - name: "make_initializable_iterator" - argspec: "args=[\'self\', \'shared_name\'], varargs=None, keywords=None, defaults=[\'None\'], " - } - member_method { - name: "make_one_shot_iterator" - argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" - } member_method { name: "map" argspec: "args=[\'self\', \'map_func\', \'num_parallel_calls\'], varargs=None, keywords=None, defaults=[\'None\'], " @@ -102,10 +89,6 @@ tf_class { name: "repeat" argspec: "args=[\'self\', \'count\'], varargs=None, keywords=None, defaults=[\'None\'], " } - member_method { - name: "shard" - argspec: "args=[\'self\', \'num_shards\', \'index\'], varargs=None, keywords=None, defaults=None" - } member_method { name: "shuffle" argspec: "args=[\'self\', \'buffer_size\', \'seed\', \'reshuffle_each_iteration\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " diff --git a/tensorflow/tools/api/golden/v2/tensorflow.data.-fixed-length-record-dataset.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.data.-fixed-length-record-dataset.pbtxt index 81358cecbc0..f1573512438 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.data.-fixed-length-record-dataset.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.data.-fixed-length-record-dataset.pbtxt @@ -1,8 +1,8 @@ path: "tensorflow.data.FixedLengthRecordDataset" tf_class { - is_instance: "" + is_instance: "" is_instance: "" - is_instance: "" + is_instance: "" is_instance: "" member { name: "output_classes" @@ -48,10 +48,6 @@ tf_class { name: "from_generator" argspec: "args=[\'generator\', \'output_types\', \'output_shapes\', \'args\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " } - member_method { - name: "from_sparse_tensor_slices" - argspec: "args=[\'sparse_tensor\'], varargs=None, keywords=None, defaults=None" - } member_method { name: "from_tensor_slices" argspec: "args=[\'tensors\'], varargs=None, keywords=None, defaults=None" @@ -68,14 +64,6 @@ tf_class { name: "list_files" argspec: "args=[\'file_pattern\', \'shuffle\', \'seed\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " } - member_method { - name: "make_initializable_iterator" - argspec: "args=[\'self\', \'shared_name\'], varargs=None, keywords=None, defaults=[\'None\'], " - } - member_method { - name: "make_one_shot_iterator" - argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" - } member_method { name: "map" argspec: "args=[\'self\', \'map_func\', \'num_parallel_calls\'], varargs=None, keywords=None, defaults=[\'None\'], " @@ -104,10 +92,6 @@ tf_class { name: "repeat" argspec: "args=[\'self\', \'count\'], varargs=None, keywords=None, defaults=[\'None\'], " } - member_method { - name: "shard" - argspec: "args=[\'self\', \'num_shards\', \'index\'], varargs=None, keywords=None, defaults=None" - } member_method { name: "shuffle" argspec: "args=[\'self\', \'buffer_size\', \'seed\', \'reshuffle_each_iteration\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " diff --git a/tensorflow/tools/api/golden/v2/tensorflow.data.-iterator.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.data.-iterator.pbtxt deleted file mode 100644 index 4f0147a5238..00000000000 --- a/tensorflow/tools/api/golden/v2/tensorflow.data.-iterator.pbtxt +++ /dev/null @@ -1,46 +0,0 @@ -path: "tensorflow.data.Iterator" -tf_class { - is_instance: "" - is_instance: "" - is_instance: "" - member { - name: "initializer" - mtype: "" - } - member { - name: "output_classes" - mtype: "" - } - member { - name: "output_shapes" - mtype: "" - } - member { - name: "output_types" - mtype: "" - } - member_method { - name: "__init__" - argspec: "args=[\'self\', \'iterator_resource\', \'initializer\', \'output_types\', \'output_shapes\', \'output_classes\'], varargs=None, keywords=None, defaults=None" - } - member_method { - name: "from_string_handle" - argspec: "args=[\'string_handle\', \'output_types\', \'output_shapes\', \'output_classes\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " - } - member_method { - name: "from_structure" - argspec: "args=[\'output_types\', \'output_shapes\', \'shared_name\', \'output_classes\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], " - } - member_method { - name: "get_next" - argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " - } - member_method { - name: "make_initializer" - argspec: "args=[\'self\', \'dataset\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " - } - member_method { - name: "string_handle" - argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " - } -} diff --git a/tensorflow/tools/api/golden/v2/tensorflow.data.-options.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.data.-options.pbtxt index 9d032d43de1..72fc2c3a9ee 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.data.-options.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.data.-options.pbtxt @@ -1,6 +1,7 @@ path: "tensorflow.data.Options" tf_class { is_instance: "" + is_instance: "" is_instance: "" member { name: "experimental_autotune" @@ -10,50 +11,22 @@ tf_class { name: "experimental_deterministic" mtype: "" } - member { - name: "experimental_filter_fusion" - mtype: "" - } - member { - name: "experimental_hoist_random_uniform" - mtype: "" - } - member { - name: "experimental_map_and_batch_fusion" - mtype: "" - } - member { - name: "experimental_map_and_filter_fusion" - mtype: "" - } - member { - name: "experimental_map_fusion" - mtype: "" - } - member { - name: "experimental_map_parallelization" - mtype: "" - } - member { - name: "experimental_map_vectorization" - mtype: "" - } - member { - name: "experimental_noop_elimination" - mtype: "" - } member { name: "experimental_numa_aware" mtype: "" } member { - name: "experimental_shuffle_and_repeat_fusion" + name: "experimental_optimization" mtype: "" } member { name: "experimental_stats" mtype: "" } + member { + name: "experimental_threading" + mtype: "" + } member_method { name: "__init__" argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" diff --git a/tensorflow/tools/api/golden/v2/tensorflow.data.-t-f-record-dataset.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.data.-t-f-record-dataset.pbtxt index 7b7a9ebaf08..690da98b1ac 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.data.-t-f-record-dataset.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.data.-t-f-record-dataset.pbtxt @@ -1,7 +1,7 @@ path: "tensorflow.data.TFRecordDataset" tf_class { - is_instance: "" - is_instance: "" + is_instance: "" + is_instance: "" is_instance: "" member { name: "output_classes" @@ -47,10 +47,6 @@ tf_class { name: "from_generator" argspec: "args=[\'generator\', \'output_types\', \'output_shapes\', \'args\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " } - member_method { - name: "from_sparse_tensor_slices" - argspec: "args=[\'sparse_tensor\'], varargs=None, keywords=None, defaults=None" - } member_method { name: "from_tensor_slices" argspec: "args=[\'tensors\'], varargs=None, keywords=None, defaults=None" @@ -67,14 +63,6 @@ tf_class { name: "list_files" argspec: "args=[\'file_pattern\', \'shuffle\', \'seed\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " } - member_method { - name: "make_initializable_iterator" - argspec: "args=[\'self\', \'shared_name\'], varargs=None, keywords=None, defaults=[\'None\'], " - } - member_method { - name: "make_one_shot_iterator" - argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" - } member_method { name: "map" argspec: "args=[\'self\', \'map_func\', \'num_parallel_calls\'], varargs=None, keywords=None, defaults=[\'None\'], " @@ -103,10 +91,6 @@ tf_class { name: "repeat" argspec: "args=[\'self\', \'count\'], varargs=None, keywords=None, defaults=[\'None\'], " } - member_method { - name: "shard" - argspec: "args=[\'self\', \'num_shards\', \'index\'], varargs=None, keywords=None, defaults=None" - } member_method { name: "shuffle" argspec: "args=[\'self\', \'buffer_size\', \'seed\', \'reshuffle_each_iteration\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " diff --git a/tensorflow/tools/api/golden/v2/tensorflow.data.-text-line-dataset.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.data.-text-line-dataset.pbtxt index 1c305abf68c..fe0bc1a4db5 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.data.-text-line-dataset.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.data.-text-line-dataset.pbtxt @@ -1,8 +1,8 @@ path: "tensorflow.data.TextLineDataset" tf_class { - is_instance: "" + is_instance: "" is_instance: "" - is_instance: "" + is_instance: "" is_instance: "" member { name: "output_classes" @@ -48,10 +48,6 @@ tf_class { name: "from_generator" argspec: "args=[\'generator\', \'output_types\', \'output_shapes\', \'args\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " } - member_method { - name: "from_sparse_tensor_slices" - argspec: "args=[\'sparse_tensor\'], varargs=None, keywords=None, defaults=None" - } member_method { name: "from_tensor_slices" argspec: "args=[\'tensors\'], varargs=None, keywords=None, defaults=None" @@ -68,14 +64,6 @@ tf_class { name: "list_files" argspec: "args=[\'file_pattern\', \'shuffle\', \'seed\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " } - member_method { - name: "make_initializable_iterator" - argspec: "args=[\'self\', \'shared_name\'], varargs=None, keywords=None, defaults=[\'None\'], " - } - member_method { - name: "make_one_shot_iterator" - argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" - } member_method { name: "map" argspec: "args=[\'self\', \'map_func\', \'num_parallel_calls\'], varargs=None, keywords=None, defaults=[\'None\'], " @@ -104,10 +92,6 @@ tf_class { name: "repeat" argspec: "args=[\'self\', \'count\'], varargs=None, keywords=None, defaults=[\'None\'], " } - member_method { - name: "shard" - argspec: "args=[\'self\', \'num_shards\', \'index\'], varargs=None, keywords=None, defaults=None" - } member_method { name: "shuffle" argspec: "args=[\'self\', \'buffer_size\', \'seed\', \'reshuffle_each_iteration\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " diff --git a/tensorflow/tools/api/golden/v2/tensorflow.data.experimental.-csv-dataset.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.data.experimental.-csv-dataset.pbtxt index 2520e28a3c7..261129b1321 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.data.experimental.-csv-dataset.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.data.experimental.-csv-dataset.pbtxt @@ -1,8 +1,8 @@ path: "tensorflow.data.experimental.CsvDataset" tf_class { - is_instance: "" + is_instance: "" is_instance: "" - is_instance: "" + is_instance: "" is_instance: "" member { name: "output_classes" @@ -48,10 +48,6 @@ tf_class { name: "from_generator" argspec: "args=[\'generator\', \'output_types\', \'output_shapes\', \'args\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " } - member_method { - name: "from_sparse_tensor_slices" - argspec: "args=[\'sparse_tensor\'], varargs=None, keywords=None, defaults=None" - } member_method { name: "from_tensor_slices" argspec: "args=[\'tensors\'], varargs=None, keywords=None, defaults=None" @@ -68,14 +64,6 @@ tf_class { name: "list_files" argspec: "args=[\'file_pattern\', \'shuffle\', \'seed\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " } - member_method { - name: "make_initializable_iterator" - argspec: "args=[\'self\', \'shared_name\'], varargs=None, keywords=None, defaults=[\'None\'], " - } - member_method { - name: "make_one_shot_iterator" - argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" - } member_method { name: "map" argspec: "args=[\'self\', \'map_func\', \'num_parallel_calls\'], varargs=None, keywords=None, defaults=[\'None\'], " @@ -104,10 +92,6 @@ tf_class { name: "repeat" argspec: "args=[\'self\', \'count\'], varargs=None, keywords=None, defaults=[\'None\'], " } - member_method { - name: "shard" - argspec: "args=[\'self\', \'num_shards\', \'index\'], varargs=None, keywords=None, defaults=None" - } member_method { name: "shuffle" argspec: "args=[\'self\', \'buffer_size\', \'seed\', \'reshuffle_each_iteration\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " diff --git a/tensorflow/tools/api/golden/v2/tensorflow.data.experimental.-optimization-options.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.data.experimental.-optimization-options.pbtxt new file mode 100644 index 00000000000..9ca75828e55 --- /dev/null +++ b/tensorflow/tools/api/golden/v2/tensorflow.data.experimental.-optimization-options.pbtxt @@ -0,0 +1,46 @@ +path: "tensorflow.data.experimental.OptimizationOptions" +tf_class { + is_instance: "" + is_instance: "" + is_instance: "" + member { + name: "filter_fusion" + mtype: "" + } + member { + name: "hoist_random_uniform" + mtype: "" + } + member { + name: "map_and_batch_fusion" + mtype: "" + } + member { + name: "map_and_filter_fusion" + mtype: "" + } + member { + name: "map_fusion" + mtype: "" + } + member { + name: "map_parallelization" + mtype: "" + } + member { + name: "map_vectorization" + mtype: "" + } + member { + name: "noop_elimination" + mtype: "" + } + member { + name: "shuffle_and_repeat_fusion" + mtype: "" + } + member_method { + name: "__init__" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } +} diff --git a/tensorflow/tools/api/golden/v2/tensorflow.data.experimental.-random-dataset.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.data.experimental.-random-dataset.pbtxt index 1dd53b1eabd..0b34bbc9426 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.data.experimental.-random-dataset.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.data.experimental.-random-dataset.pbtxt @@ -1,8 +1,8 @@ path: "tensorflow.data.experimental.RandomDataset" tf_class { - is_instance: "" + is_instance: "" is_instance: "" - is_instance: "" + is_instance: "" is_instance: "" member { name: "output_classes" @@ -48,10 +48,6 @@ tf_class { name: "from_generator" argspec: "args=[\'generator\', \'output_types\', \'output_shapes\', \'args\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " } - member_method { - name: "from_sparse_tensor_slices" - argspec: "args=[\'sparse_tensor\'], varargs=None, keywords=None, defaults=None" - } member_method { name: "from_tensor_slices" argspec: "args=[\'tensors\'], varargs=None, keywords=None, defaults=None" @@ -68,14 +64,6 @@ tf_class { name: "list_files" argspec: "args=[\'file_pattern\', \'shuffle\', \'seed\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " } - member_method { - name: "make_initializable_iterator" - argspec: "args=[\'self\', \'shared_name\'], varargs=None, keywords=None, defaults=[\'None\'], " - } - member_method { - name: "make_one_shot_iterator" - argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" - } member_method { name: "map" argspec: "args=[\'self\', \'map_func\', \'num_parallel_calls\'], varargs=None, keywords=None, defaults=[\'None\'], " @@ -104,10 +92,6 @@ tf_class { name: "repeat" argspec: "args=[\'self\', \'count\'], varargs=None, keywords=None, defaults=[\'None\'], " } - member_method { - name: "shard" - argspec: "args=[\'self\', \'num_shards\', \'index\'], varargs=None, keywords=None, defaults=None" - } member_method { name: "shuffle" argspec: "args=[\'self\', \'buffer_size\', \'seed\', \'reshuffle_each_iteration\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " diff --git a/tensorflow/tools/api/golden/v2/tensorflow.data.experimental.-sql-dataset.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.data.experimental.-sql-dataset.pbtxt index 8fdd9dc52e3..0e61890eee4 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.data.experimental.-sql-dataset.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.data.experimental.-sql-dataset.pbtxt @@ -1,8 +1,8 @@ path: "tensorflow.data.experimental.SqlDataset" tf_class { - is_instance: "" + is_instance: "" is_instance: "" - is_instance: "" + is_instance: "" is_instance: "" member { name: "output_classes" @@ -48,10 +48,6 @@ tf_class { name: "from_generator" argspec: "args=[\'generator\', \'output_types\', \'output_shapes\', \'args\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " } - member_method { - name: "from_sparse_tensor_slices" - argspec: "args=[\'sparse_tensor\'], varargs=None, keywords=None, defaults=None" - } member_method { name: "from_tensor_slices" argspec: "args=[\'tensors\'], varargs=None, keywords=None, defaults=None" @@ -68,14 +64,6 @@ tf_class { name: "list_files" argspec: "args=[\'file_pattern\', \'shuffle\', \'seed\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " } - member_method { - name: "make_initializable_iterator" - argspec: "args=[\'self\', \'shared_name\'], varargs=None, keywords=None, defaults=[\'None\'], " - } - member_method { - name: "make_one_shot_iterator" - argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" - } member_method { name: "map" argspec: "args=[\'self\', \'map_func\', \'num_parallel_calls\'], varargs=None, keywords=None, defaults=[\'None\'], " @@ -104,10 +92,6 @@ tf_class { name: "repeat" argspec: "args=[\'self\', \'count\'], varargs=None, keywords=None, defaults=[\'None\'], " } - member_method { - name: "shard" - argspec: "args=[\'self\', \'num_shards\', \'index\'], varargs=None, keywords=None, defaults=None" - } member_method { name: "shuffle" argspec: "args=[\'self\', \'buffer_size\', \'seed\', \'reshuffle_each_iteration\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " diff --git a/tensorflow/tools/api/golden/v2/tensorflow.data.experimental.-stats-options.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.data.experimental.-stats-options.pbtxt index f423eed42cc..892f8c1fb89 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.data.experimental.-stats-options.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.data.experimental.-stats-options.pbtxt @@ -1,6 +1,7 @@ path: "tensorflow.data.experimental.StatsOptions" tf_class { is_instance: "" + is_instance: "" is_instance: "" member { name: "aggregator" @@ -20,6 +21,6 @@ tf_class { } member_method { name: "__init__" - argspec: "args=[\'self\', \'aggregator\'], varargs=None, keywords=None, defaults=[\'None\'], " + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" } } diff --git a/tensorflow/tools/api/golden/v2/tensorflow.data.experimental.-threading-options.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.data.experimental.-threading-options.pbtxt new file mode 100644 index 00000000000..5b5ebf10801 --- /dev/null +++ b/tensorflow/tools/api/golden/v2/tensorflow.data.experimental.-threading-options.pbtxt @@ -0,0 +1,18 @@ +path: "tensorflow.data.experimental.ThreadingOptions" +tf_class { + is_instance: "" + is_instance: "" + is_instance: "" + member { + name: "max_intra_op_parallelism" + mtype: "" + } + member { + name: "private_threadpool_size" + mtype: "" + } + member_method { + name: "__init__" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } +} diff --git a/tensorflow/tools/api/golden/v2/tensorflow.data.experimental.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.data.experimental.pbtxt index 4c253bb8adf..f981b1af177 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.data.experimental.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.data.experimental.pbtxt @@ -12,6 +12,14 @@ tf_module { name: "CsvDataset" mtype: "" } + member { + name: "INFINITE_CARDINALITY" + mtype: "" + } + member { + name: "OptimizationOptions" + mtype: "" + } member { name: "Optional" mtype: "" @@ -40,6 +48,14 @@ tf_module { name: "TFRecordWriter" mtype: "" } + member { + name: "ThreadingOptions" + mtype: "" + } + member { + name: "UNKNOWN_CARDINALITY" + mtype: "" + } member_method { name: "Counter" argspec: "args=[\'start\', \'step\', \'dtype\'], varargs=None, keywords=None, defaults=[\'0\', \'1\', \"\"], " @@ -48,6 +64,10 @@ tf_module { name: "bucket_by_sequence_length" argspec: "args=[\'element_length_func\', \'bucket_boundaries\', \'bucket_batch_sizes\', \'padded_shapes\', \'padding_values\', \'pad_to_bucket_boundary\', \'no_padding\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'False\', \'False\'], " } + member_method { + name: "cardinality" + argspec: "args=[\'dataset\'], varargs=None, keywords=None, defaults=None" + } member_method { name: "choose_from_datasets" argspec: "args=[\'datasets\', \'choice_dataset\'], varargs=None, keywords=None, defaults=None" @@ -64,6 +84,10 @@ tf_module { name: "enumerate_dataset" argspec: "args=[\'start\'], varargs=None, keywords=None, defaults=[\'0\'], " } + member_method { + name: "filter_for_shard" + argspec: "args=[\'num_shards\', \'shard_index\'], varargs=None, keywords=None, defaults=None" + } member_method { name: "get_next_as_optional" argspec: "args=[\'iterator\'], varargs=None, keywords=None, defaults=None" @@ -90,7 +114,7 @@ tf_module { } member_method { name: "make_batched_features_dataset" - argspec: "args=[\'file_pattern\', \'batch_size\', \'features\', \'reader\', \'label_key\', \'reader_args\', \'num_epochs\', \'shuffle\', \'shuffle_buffer_size\', \'shuffle_seed\', \'prefetch_buffer_size\', \'reader_num_threads\', \'parser_num_threads\', \'sloppy_ordering\', \'drop_final_batch\'], varargs=None, keywords=None, defaults=[\"\", \'None\', \'None\', \'None\', \'True\', \'10000\', \'None\', \'-1\', \'1\', \'2\', \'False\', \'False\'], " + argspec: "args=[\'file_pattern\', \'batch_size\', \'features\', \'reader\', \'label_key\', \'reader_args\', \'num_epochs\', \'shuffle\', \'shuffle_buffer_size\', \'shuffle_seed\', \'prefetch_buffer_size\', \'reader_num_threads\', \'parser_num_threads\', \'sloppy_ordering\', \'drop_final_batch\'], varargs=None, keywords=None, defaults=[\"\", \'None\', \'None\', \'None\', \'True\', \'10000\', \'None\', \'-1\', \'1\', \'2\', \'False\', \'False\'], " } member_method { name: "make_csv_dataset" diff --git a/tensorflow/tools/api/golden/v2/tensorflow.data.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.data.pbtxt index 509bbae8332..4c3d6ddd852 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.data.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.data.pbtxt @@ -8,10 +8,6 @@ tf_module { name: "FixedLengthRecordDataset" mtype: "" } - member { - name: "Iterator" - mtype: "" - } member { name: "Options" mtype: "" diff --git a/tensorflow/tools/api/golden/v2/tensorflow.debugging.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.debugging.pbtxt index ab6287f8cd0..314aedda909 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.debugging.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.debugging.pbtxt @@ -6,19 +6,19 @@ tf_module { } member_method { name: "assert_all_finite" - argspec: "args=[\'t\', \'msg\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " + argspec: "args=[\'x\', \'message\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " } member_method { name: "assert_equal" - argspec: "args=[\'x\', \'y\', \'data\', \'summarize\', \'message\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], " + argspec: "args=[\'x\', \'y\', \'message\', \'summarize\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], " } member_method { name: "assert_greater" - argspec: "args=[\'x\', \'y\', \'data\', \'summarize\', \'message\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], " + argspec: "args=[\'x\', \'y\', \'message\', \'summarize\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], " } member_method { name: "assert_greater_equal" - argspec: "args=[\'x\', \'y\', \'data\', \'summarize\', \'message\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], " + argspec: "args=[\'x\', \'y\', \'message\', \'summarize\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], " } member_method { name: "assert_integer" @@ -26,35 +26,35 @@ tf_module { } member_method { name: "assert_less" - argspec: "args=[\'x\', \'y\', \'data\', \'summarize\', \'message\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], " + argspec: "args=[\'x\', \'y\', \'message\', \'summarize\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], " } member_method { name: "assert_less_equal" - argspec: "args=[\'x\', \'y\', \'data\', \'summarize\', \'message\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], " + argspec: "args=[\'x\', \'y\', \'message\', \'summarize\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], " } member_method { name: "assert_near" - argspec: "args=[\'x\', \'y\', \'rtol\', \'atol\', \'data\', \'summarize\', \'message\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\'], " + argspec: "args=[\'x\', \'y\', \'rtol\', \'atol\', \'message\', \'summarize\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\'], " } member_method { name: "assert_negative" - argspec: "args=[\'x\', \'data\', \'summarize\', \'message\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], " + argspec: "args=[\'x\', \'message\', \'summarize\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], " } member_method { name: "assert_non_negative" - argspec: "args=[\'x\', \'data\', \'summarize\', \'message\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], " + argspec: "args=[\'x\', \'message\', \'summarize\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], " } member_method { name: "assert_non_positive" - argspec: "args=[\'x\', \'data\', \'summarize\', \'message\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], " + argspec: "args=[\'x\', \'message\', \'summarize\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], " } member_method { name: "assert_none_equal" - argspec: "args=[\'x\', \'y\', \'data\', \'summarize\', \'message\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], " + argspec: "args=[\'x\', \'y\', \'summarize\', \'message\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], " } member_method { name: "assert_positive" - argspec: "args=[\'x\', \'data\', \'summarize\', \'message\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], " + argspec: "args=[\'x\', \'message\', \'summarize\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], " } member_method { name: "assert_proper_iterable" @@ -62,15 +62,15 @@ tf_module { } member_method { name: "assert_rank" - argspec: "args=[\'x\', \'rank\', \'data\', \'summarize\', \'message\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], " + argspec: "args=[\'x\', \'rank\', \'message\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " } member_method { name: "assert_rank_at_least" - argspec: "args=[\'x\', \'rank\', \'data\', \'summarize\', \'message\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], " + argspec: "args=[\'x\', \'rank\', \'message\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " } member_method { name: "assert_rank_in" - argspec: "args=[\'x\', \'ranks\', \'data\', \'summarize\', \'message\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], " + argspec: "args=[\'x\', \'ranks\', \'message\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " } member_method { name: "assert_same_float_dtype" @@ -78,7 +78,7 @@ tf_module { } member_method { name: "assert_scalar" - argspec: "args=[\'tensor\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " + argspec: "args=[\'tensor\', \'message\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " } member_method { name: "assert_type" @@ -88,28 +88,8 @@ tf_module { name: "check_numerics" argspec: "args=[\'tensor\', \'message\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " } - member_method { - name: "is_finite" - argspec: "args=[\'x\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " - } - member_method { - name: "is_inf" - argspec: "args=[\'x\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " - } - member_method { - name: "is_nan" - argspec: "args=[\'x\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " - } - member_method { - name: "is_non_decreasing" - argspec: "args=[\'x\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " - } member_method { name: "is_numeric_tensor" argspec: "args=[\'tensor\'], varargs=None, keywords=None, defaults=None" } - member_method { - name: "is_strictly_increasing" - argspec: "args=[\'x\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " - } } diff --git a/tensorflow/tools/api/golden/v2/tensorflow.distribute.-input-context.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.distribute.-input-context.pbtxt new file mode 100644 index 00000000000..583cbc66549 --- /dev/null +++ b/tensorflow/tools/api/golden/v2/tensorflow.distribute.-input-context.pbtxt @@ -0,0 +1,25 @@ +path: "tensorflow.distribute.InputContext" +tf_class { + is_instance: "" + is_instance: "" + member { + name: "input_pipeline_id" + mtype: "" + } + member { + name: "num_input_pipelines" + mtype: "" + } + member { + name: "num_replicas_in_sync" + mtype: "" + } + member_method { + name: "__init__" + argspec: "args=[\'self\', \'num_input_pipelines\', \'input_pipeline_id\', \'num_replicas_in_sync\'], varargs=None, keywords=None, defaults=[\'1\', \'0\', \'1\'], " + } + member_method { + name: "get_per_replica_batch_size" + argspec: "args=[\'self\', \'global_batch_size\'], varargs=None, keywords=None, defaults=None" + } +} diff --git a/tensorflow/tools/api/golden/v2/tensorflow.distribute.-input-replication-mode.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.distribute.-input-replication-mode.pbtxt new file mode 100644 index 00000000000..6a7a3a97aa0 --- /dev/null +++ b/tensorflow/tools/api/golden/v2/tensorflow.distribute.-input-replication-mode.pbtxt @@ -0,0 +1,8 @@ +path: "tensorflow.distribute.InputReplicationMode" +tf_class { + is_instance: "" + member { + name: "PER_WORKER" + mtype: "" + } +} diff --git a/tensorflow/tools/api/golden/v2/tensorflow.distribute.-reduce-op.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.distribute.-reduce-op.pbtxt new file mode 100644 index 00000000000..4899f38cad2 --- /dev/null +++ b/tensorflow/tools/api/golden/v2/tensorflow.distribute.-reduce-op.pbtxt @@ -0,0 +1,12 @@ +path: "tensorflow.distribute.ReduceOp" +tf_class { + is_instance: "" + member { + name: "MEAN" + mtype: "" + } + member { + name: "SUM" + mtype: "" + } +} diff --git a/tensorflow/tools/api/golden/v2/tensorflow.distribute.-replica-context.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.distribute.-replica-context.pbtxt new file mode 100644 index 00000000000..df707e8920e --- /dev/null +++ b/tensorflow/tools/api/golden/v2/tensorflow.distribute.-replica-context.pbtxt @@ -0,0 +1,33 @@ +path: "tensorflow.distribute.ReplicaContext" +tf_class { + is_instance: "" + is_instance: "" + member { + name: "devices" + mtype: "" + } + member { + name: "distribution_strategy" + mtype: "" + } + member { + name: "num_replicas_in_sync" + mtype: "" + } + member { + name: "replica_id_in_sync_group" + mtype: "" + } + member { + name: "strategy" + mtype: "" + } + member_method { + name: "__init__" + argspec: "args=[\'self\', \'strategy\', \'replica_id_in_sync_group\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "merge_call" + argspec: "args=[\'self\', \'merge_fn\', \'args\', \'kwargs\'], varargs=None, keywords=None, defaults=[\'()\', \'None\'], " + } +} diff --git a/tensorflow/tools/api/golden/v2/tensorflow.distribute.-strategy-extended.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.distribute.-strategy-extended.pbtxt new file mode 100644 index 00000000000..77706e57133 --- /dev/null +++ b/tensorflow/tools/api/golden/v2/tensorflow.distribute.-strategy-extended.pbtxt @@ -0,0 +1,81 @@ +path: "tensorflow.distribute.StrategyExtended" +tf_class { + is_instance: "" + is_instance: "" + member { + name: "experimental_between_graph" + mtype: "" + } + member { + name: "experimental_require_static_shapes" + mtype: "" + } + member { + name: "experimental_should_init" + mtype: "" + } + member { + name: "parameter_devices" + mtype: "" + } + member { + name: "should_checkpoint" + mtype: "" + } + member { + name: "should_save_summary" + mtype: "" + } + member { + name: "worker_devices" + mtype: "" + } + member_method { + name: "__init__" + argspec: "args=[\'self\', \'container_strategy\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "batch_reduce_to" + argspec: "args=[\'self\', \'reduce_op\', \'value_destination_pairs\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "broadcast_to" + argspec: "args=[\'self\', \'tensor\', \'destinations\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "call_for_each_replica" + argspec: "args=[\'self\', \'fn\', \'args\', \'kwargs\'], varargs=None, keywords=None, defaults=[\'()\', \'None\'], " + } + member_method { + name: "colocate_vars_with" + argspec: "args=[\'self\', \'colocate_with_variable\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "experimental_run_steps_on_iterator" + argspec: "args=[\'self\', \'fn\', \'iterator\', \'iterations\', \'initial_loop_values\'], varargs=None, keywords=None, defaults=[\'1\', \'None\'], " + } + member_method { + name: "non_slot_devices" + argspec: "args=[\'self\', \'var_list\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "read_var" + argspec: "args=[\'self\', \'v\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "reduce_to" + argspec: "args=[\'self\', \'reduce_op\', \'value\', \'destinations\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "update" + argspec: "args=[\'self\', \'var\', \'fn\', \'args\', \'kwargs\', \'group\'], varargs=None, keywords=None, defaults=[\'()\', \'None\', \'True\'], " + } + member_method { + name: "update_non_slot" + argspec: "args=[\'self\', \'colocate_with\', \'fn\', \'args\', \'kwargs\', \'group\'], varargs=None, keywords=None, defaults=[\'()\', \'None\', \'True\'], " + } + member_method { + name: "value_container" + argspec: "args=[\'self\', \'value\'], varargs=None, keywords=None, defaults=None" + } +} diff --git a/tensorflow/tools/api/golden/v2/tensorflow.distribute.-strategy.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.distribute.-strategy.pbtxt new file mode 100644 index 00000000000..9eb73d2c0d9 --- /dev/null +++ b/tensorflow/tools/api/golden/v2/tensorflow.distribute.-strategy.pbtxt @@ -0,0 +1,137 @@ +path: "tensorflow.distribute.Strategy" +tf_class { + is_instance: "" + is_instance: "" + member { + name: "between_graph" + mtype: "" + } + member { + name: "extended" + mtype: "" + } + member { + name: "num_replicas_in_sync" + mtype: "" + } + member { + name: "parameter_devices" + mtype: "" + } + member { + name: "require_static_shapes" + mtype: "" + } + member { + name: "should_checkpoint" + mtype: "" + } + member { + name: "should_init" + mtype: "" + } + member { + name: "should_save_summary" + mtype: "" + } + member { + name: "worker_devices" + mtype: "" + } + member_method { + name: "__init__" + argspec: "args=[\'self\', \'extended\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "batch_reduce" + argspec: "args=[\'self\', \'aggregation\', \'value_destination_pairs\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "broadcast" + argspec: "args=[\'self\', \'tensor\', \'destinations\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "call_for_each_replica" + argspec: "args=[\'self\', \'fn\'], varargs=args, keywords=kwargs, defaults=None" + } + member_method { + name: "colocate_vars_with" + argspec: "args=[\'self\', \'colocate_with_variable\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "configure" + argspec: "args=[\'self\', \'session_config\', \'cluster_spec\', \'task_type\', \'task_id\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], " + } + member_method { + name: "distribute_dataset" + argspec: "args=[\'self\', \'dataset_fn\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "experimental_finalize" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "experimental_initialize" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "finalize" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "group" + argspec: "args=[\'self\', \'value\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "initialize" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "make_dataset_iterator" + argspec: "args=[\'self\', \'dataset\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "make_input_fn_iterator" + argspec: "args=[\'self\', \'input_fn\', \'replication_mode\'], varargs=None, keywords=None, defaults=[\'InputReplicationMode.PER_WORKER\'], " + } + member_method { + name: "non_slot_devices" + argspec: "args=[\'self\', \'var_list\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "read_var" + argspec: "args=[\'self\', \'v\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "reduce" + argspec: "args=[\'self\', \'reduce_op\', \'value\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "run_steps_on_dataset" + argspec: "args=[\'self\', \'fn\', \'iterator\', \'iterations\', \'initial_loop_values\'], varargs=None, keywords=None, defaults=[\'1\', \'None\'], " + } + member_method { + name: "scope" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "unwrap" + argspec: "args=[\'self\', \'value\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "update" + argspec: "args=[\'self\', \'var\', \'fn\'], varargs=args, keywords=kwargs, defaults=None" + } + member_method { + name: "update_config_proto" + argspec: "args=[\'self\', \'config_proto\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "update_non_slot" + argspec: "args=[\'self\', \'colocate_with\', \'fn\'], varargs=args, keywords=kwargs, defaults=None" + } + member_method { + name: "value_container" + argspec: "args=[\'self\', \'value\'], varargs=None, keywords=None, defaults=None" + } +} diff --git a/tensorflow/tools/api/golden/v2/tensorflow.distribute.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.distribute.pbtxt new file mode 100644 index 00000000000..4d833b54ba0 --- /dev/null +++ b/tensorflow/tools/api/golden/v2/tensorflow.distribute.pbtxt @@ -0,0 +1,47 @@ +path: "tensorflow.distribute" +tf_module { + member { + name: "InputContext" + mtype: "" + } + member { + name: "InputReplicationMode" + mtype: "" + } + member { + name: "ReduceOp" + mtype: "" + } + member { + name: "ReplicaContext" + mtype: "" + } + member { + name: "Strategy" + mtype: "" + } + member { + name: "StrategyExtended" + mtype: "" + } + member_method { + name: "get_loss_reduction" + argspec: "args=[], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_replica_context" + argspec: "args=[], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_strategy" + argspec: "args=[], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "has_strategy" + argspec: "args=[], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "in_cross_replica_context" + argspec: "args=[], varargs=None, keywords=None, defaults=None" + } +} diff --git a/tensorflow/tools/api/golden/v2/tensorflow.estimator.-baseline-classifier.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.estimator.-baseline-classifier.pbtxt index 32b84e90ce6..efe9e746970 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.estimator.-baseline-classifier.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.estimator.-baseline-classifier.pbtxt @@ -1,7 +1,7 @@ path: "tensorflow.estimator.BaselineClassifier" tf_class { - is_instance: "" - is_instance: "" + is_instance: "" + is_instance: "" is_instance: "" member { name: "config" @@ -21,7 +21,7 @@ tf_class { } member_method { name: "__init__" - argspec: "args=[\'self\', \'model_dir\', \'n_classes\', \'weight_column\', \'label_vocabulary\', \'optimizer\', \'config\', \'loss_reduction\'], varargs=None, keywords=None, defaults=[\'None\', \'2\', \'None\', \'None\', \'Ftrl\', \'None\', \'weighted_sum\'], " + argspec: "args=[\'self\', \'model_dir\', \'n_classes\', \'weight_column\', \'label_vocabulary\', \'optimizer\', \'config\', \'loss_reduction\'], varargs=None, keywords=None, defaults=[\'None\', \'2\', \'None\', \'None\', \'Ftrl\', \'None\', \'weighted_sum_over_batch_size\'], " } member_method { name: "eval_dir" @@ -32,12 +32,12 @@ tf_class { argspec: "args=[\'self\', \'input_fn\', \'steps\', \'hooks\', \'checkpoint_path\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], " } member_method { - name: "export_saved_model" - argspec: "args=[\'self\', \'export_dir_base\', \'serving_input_receiver_fn\', \'assets_extra\', \'as_text\', \'checkpoint_path\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\'], " + name: "experimental_export_all_saved_models" + argspec: "args=[\'self\', \'export_dir_base\', \'input_receiver_fn_map\', \'assets_extra\', \'as_text\', \'checkpoint_path\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\'], " } member_method { - name: "export_savedmodel" - argspec: "args=[\'self\', \'export_dir_base\', \'serving_input_receiver_fn\', \'assets_extra\', \'as_text\', \'checkpoint_path\', \'strip_default_attrs\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\', \'False\'], " + name: "export_saved_model" + argspec: "args=[\'self\', \'export_dir_base\', \'serving_input_receiver_fn\', \'assets_extra\', \'as_text\', \'checkpoint_path\', \'experimental_mode\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\', \'infer\'], " } member_method { name: "get_variable_names" diff --git a/tensorflow/tools/api/golden/v2/tensorflow.estimator.-baseline-estimator.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.estimator.-baseline-estimator.pbtxt index 94933e7ffd6..382d392f39e 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.estimator.-baseline-estimator.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.estimator.-baseline-estimator.pbtxt @@ -1,7 +1,7 @@ path: "tensorflow.estimator.BaselineEstimator" tf_class { - is_instance: "" - is_instance: "" + is_instance: "" + is_instance: "" is_instance: "" member { name: "config" @@ -32,12 +32,12 @@ tf_class { argspec: "args=[\'self\', \'input_fn\', \'steps\', \'hooks\', \'checkpoint_path\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], " } member_method { - name: "export_saved_model" - argspec: "args=[\'self\', \'export_dir_base\', \'serving_input_receiver_fn\', \'assets_extra\', \'as_text\', \'checkpoint_path\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\'], " + name: "experimental_export_all_saved_models" + argspec: "args=[\'self\', \'export_dir_base\', \'input_receiver_fn_map\', \'assets_extra\', \'as_text\', \'checkpoint_path\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\'], " } member_method { - name: "export_savedmodel" - argspec: "args=[\'self\', \'export_dir_base\', \'serving_input_receiver_fn\', \'assets_extra\', \'as_text\', \'checkpoint_path\', \'strip_default_attrs\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\', \'False\'], " + name: "export_saved_model" + argspec: "args=[\'self\', \'export_dir_base\', \'serving_input_receiver_fn\', \'assets_extra\', \'as_text\', \'checkpoint_path\', \'experimental_mode\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\', \'infer\'], " } member_method { name: "get_variable_names" diff --git a/tensorflow/tools/api/golden/v2/tensorflow.estimator.-baseline-regressor.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.estimator.-baseline-regressor.pbtxt index db7776b5bf6..a7300bf06bb 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.estimator.-baseline-regressor.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.estimator.-baseline-regressor.pbtxt @@ -1,7 +1,7 @@ path: "tensorflow.estimator.BaselineRegressor" tf_class { - is_instance: "" - is_instance: "" + is_instance: "" + is_instance: "" is_instance: "" member { name: "config" @@ -21,7 +21,7 @@ tf_class { } member_method { name: "__init__" - argspec: "args=[\'self\', \'model_dir\', \'label_dimension\', \'weight_column\', \'optimizer\', \'config\', \'loss_reduction\'], varargs=None, keywords=None, defaults=[\'None\', \'1\', \'None\', \'Ftrl\', \'None\', \'weighted_sum\'], " + argspec: "args=[\'self\', \'model_dir\', \'label_dimension\', \'weight_column\', \'optimizer\', \'config\', \'loss_reduction\'], varargs=None, keywords=None, defaults=[\'None\', \'1\', \'None\', \'Ftrl\', \'None\', \'weighted_sum_over_batch_size\'], " } member_method { name: "eval_dir" @@ -32,12 +32,12 @@ tf_class { argspec: "args=[\'self\', \'input_fn\', \'steps\', \'hooks\', \'checkpoint_path\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], " } member_method { - name: "export_saved_model" - argspec: "args=[\'self\', \'export_dir_base\', \'serving_input_receiver_fn\', \'assets_extra\', \'as_text\', \'checkpoint_path\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\'], " + name: "experimental_export_all_saved_models" + argspec: "args=[\'self\', \'export_dir_base\', \'input_receiver_fn_map\', \'assets_extra\', \'as_text\', \'checkpoint_path\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\'], " } member_method { - name: "export_savedmodel" - argspec: "args=[\'self\', \'export_dir_base\', \'serving_input_receiver_fn\', \'assets_extra\', \'as_text\', \'checkpoint_path\', \'strip_default_attrs\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\', \'False\'], " + name: "export_saved_model" + argspec: "args=[\'self\', \'export_dir_base\', \'serving_input_receiver_fn\', \'assets_extra\', \'as_text\', \'checkpoint_path\', \'experimental_mode\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\', \'infer\'], " } member_method { name: "get_variable_names" diff --git a/tensorflow/tools/api/golden/v2/tensorflow.estimator.-boosted-trees-classifier.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.estimator.-boosted-trees-classifier.pbtxt index d530c71482a..e138ce936ec 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.estimator.-boosted-trees-classifier.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.estimator.-boosted-trees-classifier.pbtxt @@ -3,6 +3,7 @@ tf_class { is_instance: "" is_instance: "" is_instance: "" + is_instance: "" is_instance: "" member { name: "config" @@ -32,6 +33,10 @@ tf_class { name: "evaluate" argspec: "args=[\'self\', \'input_fn\', \'steps\', \'hooks\', \'checkpoint_path\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], " } + member_method { + name: "experimental_export_all_saved_models" + argspec: "args=[\'self\', \'export_dir_base\', \'input_receiver_fn_map\', \'assets_extra\', \'as_text\', \'checkpoint_path\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\'], " + } member_method { name: "experimental_feature_importances" argspec: "args=[\'self\', \'normalize\'], varargs=None, keywords=None, defaults=[\'False\'], " @@ -42,7 +47,7 @@ tf_class { } member_method { name: "export_saved_model" - argspec: "args=[\'self\', \'export_dir_base\', \'serving_input_receiver_fn\', \'assets_extra\', \'as_text\', \'checkpoint_path\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\'], " + argspec: "args=[\'self\', \'export_dir_base\', \'serving_input_receiver_fn\', \'assets_extra\', \'as_text\', \'checkpoint_path\', \'experimental_mode\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\', \'infer\'], " } member_method { name: "export_savedmodel" diff --git a/tensorflow/tools/api/golden/v2/tensorflow.estimator.-boosted-trees-regressor.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.estimator.-boosted-trees-regressor.pbtxt index 4703c0f561a..eae0a292a96 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.estimator.-boosted-trees-regressor.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.estimator.-boosted-trees-regressor.pbtxt @@ -3,6 +3,7 @@ tf_class { is_instance: "" is_instance: "" is_instance: "" + is_instance: "" is_instance: "" member { name: "config" @@ -32,6 +33,10 @@ tf_class { name: "evaluate" argspec: "args=[\'self\', \'input_fn\', \'steps\', \'hooks\', \'checkpoint_path\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], " } + member_method { + name: "experimental_export_all_saved_models" + argspec: "args=[\'self\', \'export_dir_base\', \'input_receiver_fn_map\', \'assets_extra\', \'as_text\', \'checkpoint_path\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\'], " + } member_method { name: "experimental_feature_importances" argspec: "args=[\'self\', \'normalize\'], varargs=None, keywords=None, defaults=[\'False\'], " @@ -42,7 +47,7 @@ tf_class { } member_method { name: "export_saved_model" - argspec: "args=[\'self\', \'export_dir_base\', \'serving_input_receiver_fn\', \'assets_extra\', \'as_text\', \'checkpoint_path\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\'], " + argspec: "args=[\'self\', \'export_dir_base\', \'serving_input_receiver_fn\', \'assets_extra\', \'as_text\', \'checkpoint_path\', \'experimental_mode\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\', \'infer\'], " } member_method { name: "export_savedmodel" diff --git a/tensorflow/tools/api/golden/v2/tensorflow.estimator.-d-n-n-classifier.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.estimator.-d-n-n-classifier.pbtxt index ce6040d0f27..a540085aba4 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.estimator.-d-n-n-classifier.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.estimator.-d-n-n-classifier.pbtxt @@ -1,7 +1,7 @@ path: "tensorflow.estimator.DNNClassifier" tf_class { - is_instance: "" - is_instance: "" + is_instance: "" + is_instance: "" is_instance: "" member { name: "config" @@ -21,7 +21,7 @@ tf_class { } member_method { name: "__init__" - argspec: "args=[\'self\', \'hidden_units\', \'feature_columns\', \'model_dir\', \'n_classes\', \'weight_column\', \'label_vocabulary\', \'optimizer\', \'activation_fn\', \'dropout\', \'input_layer_partitioner\', \'config\', \'warm_start_from\', \'loss_reduction\', \'batch_norm\'], varargs=None, keywords=None, defaults=[\'None\', \'2\', \'None\', \'None\', \'Adagrad\', \'\', \'None\', \'None\', \'None\', \'None\', \'weighted_sum\', \'False\'], " + argspec: "args=[\'self\', \'hidden_units\', \'feature_columns\', \'model_dir\', \'n_classes\', \'weight_column\', \'label_vocabulary\', \'optimizer\', \'activation_fn\', \'dropout\', \'input_layer_partitioner\', \'config\', \'warm_start_from\', \'loss_reduction\', \'batch_norm\'], varargs=None, keywords=None, defaults=[\'None\', \'2\', \'None\', \'None\', \'Adagrad\', \'\', \'None\', \'None\', \'None\', \'None\', \'weighted_sum_over_batch_size\', \'False\'], " } member_method { name: "eval_dir" @@ -32,12 +32,12 @@ tf_class { argspec: "args=[\'self\', \'input_fn\', \'steps\', \'hooks\', \'checkpoint_path\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], " } member_method { - name: "export_saved_model" - argspec: "args=[\'self\', \'export_dir_base\', \'serving_input_receiver_fn\', \'assets_extra\', \'as_text\', \'checkpoint_path\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\'], " + name: "experimental_export_all_saved_models" + argspec: "args=[\'self\', \'export_dir_base\', \'input_receiver_fn_map\', \'assets_extra\', \'as_text\', \'checkpoint_path\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\'], " } member_method { - name: "export_savedmodel" - argspec: "args=[\'self\', \'export_dir_base\', \'serving_input_receiver_fn\', \'assets_extra\', \'as_text\', \'checkpoint_path\', \'strip_default_attrs\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\', \'False\'], " + name: "export_saved_model" + argspec: "args=[\'self\', \'export_dir_base\', \'serving_input_receiver_fn\', \'assets_extra\', \'as_text\', \'checkpoint_path\', \'experimental_mode\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\', \'infer\'], " } member_method { name: "get_variable_names" diff --git a/tensorflow/tools/api/golden/v2/tensorflow.estimator.-d-n-n-estimator.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.estimator.-d-n-n-estimator.pbtxt index 4635a1544c3..d1b29d670a0 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.estimator.-d-n-n-estimator.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.estimator.-d-n-n-estimator.pbtxt @@ -1,7 +1,7 @@ path: "tensorflow.estimator.DNNEstimator" tf_class { - is_instance: "" - is_instance: "" + is_instance: "" + is_instance: "" is_instance: "" member { name: "config" @@ -32,12 +32,12 @@ tf_class { argspec: "args=[\'self\', \'input_fn\', \'steps\', \'hooks\', \'checkpoint_path\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], " } member_method { - name: "export_saved_model" - argspec: "args=[\'self\', \'export_dir_base\', \'serving_input_receiver_fn\', \'assets_extra\', \'as_text\', \'checkpoint_path\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\'], " + name: "experimental_export_all_saved_models" + argspec: "args=[\'self\', \'export_dir_base\', \'input_receiver_fn_map\', \'assets_extra\', \'as_text\', \'checkpoint_path\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\'], " } member_method { - name: "export_savedmodel" - argspec: "args=[\'self\', \'export_dir_base\', \'serving_input_receiver_fn\', \'assets_extra\', \'as_text\', \'checkpoint_path\', \'strip_default_attrs\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\', \'False\'], " + name: "export_saved_model" + argspec: "args=[\'self\', \'export_dir_base\', \'serving_input_receiver_fn\', \'assets_extra\', \'as_text\', \'checkpoint_path\', \'experimental_mode\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\', \'infer\'], " } member_method { name: "get_variable_names" diff --git a/tensorflow/tools/api/golden/v2/tensorflow.estimator.-d-n-n-linear-combined-classifier.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.estimator.-d-n-n-linear-combined-classifier.pbtxt index e85007e16ed..f6c3910a9fe 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.estimator.-d-n-n-linear-combined-classifier.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.estimator.-d-n-n-linear-combined-classifier.pbtxt @@ -1,7 +1,7 @@ path: "tensorflow.estimator.DNNLinearCombinedClassifier" tf_class { - is_instance: "" - is_instance: "" + is_instance: "" + is_instance: "" is_instance: "" member { name: "config" @@ -21,7 +21,7 @@ tf_class { } member_method { name: "__init__" - argspec: "args=[\'self\', \'model_dir\', \'linear_feature_columns\', \'linear_optimizer\', \'dnn_feature_columns\', \'dnn_optimizer\', \'dnn_hidden_units\', \'dnn_activation_fn\', \'dnn_dropout\', \'n_classes\', \'weight_column\', \'label_vocabulary\', \'input_layer_partitioner\', \'config\', \'warm_start_from\', \'loss_reduction\', \'batch_norm\', \'linear_sparse_combiner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'Ftrl\', \'None\', \'Adagrad\', \'None\', \'\', \'None\', \'2\', \'None\', \'None\', \'None\', \'None\', \'None\', \'weighted_sum\', \'False\', \'sum\'], " + argspec: "args=[\'self\', \'model_dir\', \'linear_feature_columns\', \'linear_optimizer\', \'dnn_feature_columns\', \'dnn_optimizer\', \'dnn_hidden_units\', \'dnn_activation_fn\', \'dnn_dropout\', \'n_classes\', \'weight_column\', \'label_vocabulary\', \'input_layer_partitioner\', \'config\', \'warm_start_from\', \'loss_reduction\', \'batch_norm\', \'linear_sparse_combiner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'Ftrl\', \'None\', \'Adagrad\', \'None\', \'\', \'None\', \'2\', \'None\', \'None\', \'None\', \'None\', \'None\', \'weighted_sum_over_batch_size\', \'False\', \'sum\'], " } member_method { name: "eval_dir" @@ -32,12 +32,12 @@ tf_class { argspec: "args=[\'self\', \'input_fn\', \'steps\', \'hooks\', \'checkpoint_path\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], " } member_method { - name: "export_saved_model" - argspec: "args=[\'self\', \'export_dir_base\', \'serving_input_receiver_fn\', \'assets_extra\', \'as_text\', \'checkpoint_path\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\'], " + name: "experimental_export_all_saved_models" + argspec: "args=[\'self\', \'export_dir_base\', \'input_receiver_fn_map\', \'assets_extra\', \'as_text\', \'checkpoint_path\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\'], " } member_method { - name: "export_savedmodel" - argspec: "args=[\'self\', \'export_dir_base\', \'serving_input_receiver_fn\', \'assets_extra\', \'as_text\', \'checkpoint_path\', \'strip_default_attrs\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\', \'False\'], " + name: "export_saved_model" + argspec: "args=[\'self\', \'export_dir_base\', \'serving_input_receiver_fn\', \'assets_extra\', \'as_text\', \'checkpoint_path\', \'experimental_mode\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\', \'infer\'], " } member_method { name: "get_variable_names" diff --git a/tensorflow/tools/api/golden/v2/tensorflow.estimator.-d-n-n-linear-combined-estimator.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.estimator.-d-n-n-linear-combined-estimator.pbtxt index a23f5daeac4..b78527279ca 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.estimator.-d-n-n-linear-combined-estimator.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.estimator.-d-n-n-linear-combined-estimator.pbtxt @@ -1,7 +1,7 @@ path: "tensorflow.estimator.DNNLinearCombinedEstimator" tf_class { - is_instance: "" - is_instance: "" + is_instance: "" + is_instance: "" is_instance: "" member { name: "config" @@ -32,12 +32,12 @@ tf_class { argspec: "args=[\'self\', \'input_fn\', \'steps\', \'hooks\', \'checkpoint_path\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], " } member_method { - name: "export_saved_model" - argspec: "args=[\'self\', \'export_dir_base\', \'serving_input_receiver_fn\', \'assets_extra\', \'as_text\', \'checkpoint_path\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\'], " + name: "experimental_export_all_saved_models" + argspec: "args=[\'self\', \'export_dir_base\', \'input_receiver_fn_map\', \'assets_extra\', \'as_text\', \'checkpoint_path\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\'], " } member_method { - name: "export_savedmodel" - argspec: "args=[\'self\', \'export_dir_base\', \'serving_input_receiver_fn\', \'assets_extra\', \'as_text\', \'checkpoint_path\', \'strip_default_attrs\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\', \'False\'], " + name: "export_saved_model" + argspec: "args=[\'self\', \'export_dir_base\', \'serving_input_receiver_fn\', \'assets_extra\', \'as_text\', \'checkpoint_path\', \'experimental_mode\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\', \'infer\'], " } member_method { name: "get_variable_names" diff --git a/tensorflow/tools/api/golden/v2/tensorflow.estimator.-d-n-n-linear-combined-regressor.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.estimator.-d-n-n-linear-combined-regressor.pbtxt index 8a55bb835ff..9133f0d3b28 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.estimator.-d-n-n-linear-combined-regressor.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.estimator.-d-n-n-linear-combined-regressor.pbtxt @@ -1,7 +1,7 @@ path: "tensorflow.estimator.DNNLinearCombinedRegressor" tf_class { - is_instance: "" - is_instance: "" + is_instance: "" + is_instance: "" is_instance: "" member { name: "config" @@ -21,7 +21,7 @@ tf_class { } member_method { name: "__init__" - argspec: "args=[\'self\', \'model_dir\', \'linear_feature_columns\', \'linear_optimizer\', \'dnn_feature_columns\', \'dnn_optimizer\', \'dnn_hidden_units\', \'dnn_activation_fn\', \'dnn_dropout\', \'label_dimension\', \'weight_column\', \'input_layer_partitioner\', \'config\', \'warm_start_from\', \'loss_reduction\', \'batch_norm\', \'linear_sparse_combiner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'Ftrl\', \'None\', \'Adagrad\', \'None\', \'\', \'None\', \'1\', \'None\', \'None\', \'None\', \'None\', \'weighted_sum\', \'False\', \'sum\'], " + argspec: "args=[\'self\', \'model_dir\', \'linear_feature_columns\', \'linear_optimizer\', \'dnn_feature_columns\', \'dnn_optimizer\', \'dnn_hidden_units\', \'dnn_activation_fn\', \'dnn_dropout\', \'label_dimension\', \'weight_column\', \'input_layer_partitioner\', \'config\', \'warm_start_from\', \'loss_reduction\', \'batch_norm\', \'linear_sparse_combiner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'Ftrl\', \'None\', \'Adagrad\', \'None\', \'\', \'None\', \'1\', \'None\', \'None\', \'None\', \'None\', \'weighted_sum_over_batch_size\', \'False\', \'sum\'], " } member_method { name: "eval_dir" @@ -32,12 +32,12 @@ tf_class { argspec: "args=[\'self\', \'input_fn\', \'steps\', \'hooks\', \'checkpoint_path\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], " } member_method { - name: "export_saved_model" - argspec: "args=[\'self\', \'export_dir_base\', \'serving_input_receiver_fn\', \'assets_extra\', \'as_text\', \'checkpoint_path\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\'], " + name: "experimental_export_all_saved_models" + argspec: "args=[\'self\', \'export_dir_base\', \'input_receiver_fn_map\', \'assets_extra\', \'as_text\', \'checkpoint_path\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\'], " } member_method { - name: "export_savedmodel" - argspec: "args=[\'self\', \'export_dir_base\', \'serving_input_receiver_fn\', \'assets_extra\', \'as_text\', \'checkpoint_path\', \'strip_default_attrs\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\', \'False\'], " + name: "export_saved_model" + argspec: "args=[\'self\', \'export_dir_base\', \'serving_input_receiver_fn\', \'assets_extra\', \'as_text\', \'checkpoint_path\', \'experimental_mode\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\', \'infer\'], " } member_method { name: "get_variable_names" diff --git a/tensorflow/tools/api/golden/v2/tensorflow.estimator.-d-n-n-regressor.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.estimator.-d-n-n-regressor.pbtxt index 2c4128ec480..a58d733302d 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.estimator.-d-n-n-regressor.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.estimator.-d-n-n-regressor.pbtxt @@ -1,7 +1,7 @@ path: "tensorflow.estimator.DNNRegressor" tf_class { - is_instance: "" - is_instance: "" + is_instance: "" + is_instance: "" is_instance: "" member { name: "config" @@ -21,7 +21,7 @@ tf_class { } member_method { name: "__init__" - argspec: "args=[\'self\', \'hidden_units\', \'feature_columns\', \'model_dir\', \'label_dimension\', \'weight_column\', \'optimizer\', \'activation_fn\', \'dropout\', \'input_layer_partitioner\', \'config\', \'warm_start_from\', \'loss_reduction\', \'batch_norm\'], varargs=None, keywords=None, defaults=[\'None\', \'1\', \'None\', \'Adagrad\', \'\', \'None\', \'None\', \'None\', \'None\', \'weighted_sum\', \'False\'], " + argspec: "args=[\'self\', \'hidden_units\', \'feature_columns\', \'model_dir\', \'label_dimension\', \'weight_column\', \'optimizer\', \'activation_fn\', \'dropout\', \'input_layer_partitioner\', \'config\', \'warm_start_from\', \'loss_reduction\', \'batch_norm\'], varargs=None, keywords=None, defaults=[\'None\', \'1\', \'None\', \'Adagrad\', \'\', \'None\', \'None\', \'None\', \'None\', \'weighted_sum_over_batch_size\', \'False\'], " } member_method { name: "eval_dir" @@ -32,12 +32,12 @@ tf_class { argspec: "args=[\'self\', \'input_fn\', \'steps\', \'hooks\', \'checkpoint_path\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], " } member_method { - name: "export_saved_model" - argspec: "args=[\'self\', \'export_dir_base\', \'serving_input_receiver_fn\', \'assets_extra\', \'as_text\', \'checkpoint_path\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\'], " + name: "experimental_export_all_saved_models" + argspec: "args=[\'self\', \'export_dir_base\', \'input_receiver_fn_map\', \'assets_extra\', \'as_text\', \'checkpoint_path\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\'], " } member_method { - name: "export_savedmodel" - argspec: "args=[\'self\', \'export_dir_base\', \'serving_input_receiver_fn\', \'assets_extra\', \'as_text\', \'checkpoint_path\', \'strip_default_attrs\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\', \'False\'], " + name: "export_saved_model" + argspec: "args=[\'self\', \'export_dir_base\', \'serving_input_receiver_fn\', \'assets_extra\', \'as_text\', \'checkpoint_path\', \'experimental_mode\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\', \'infer\'], " } member_method { name: "get_variable_names" diff --git a/tensorflow/tools/api/golden/v2/tensorflow.estimator.-estimator.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.estimator.-estimator.pbtxt index 9d270a87ab8..a1f0e76c8b8 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.estimator.-estimator.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.estimator.-estimator.pbtxt @@ -1,6 +1,6 @@ path: "tensorflow.estimator.Estimator" tf_class { - is_instance: "" + is_instance: "" is_instance: "" member { name: "config" @@ -31,12 +31,12 @@ tf_class { argspec: "args=[\'self\', \'input_fn\', \'steps\', \'hooks\', \'checkpoint_path\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], " } member_method { - name: "export_saved_model" - argspec: "args=[\'self\', \'export_dir_base\', \'serving_input_receiver_fn\', \'assets_extra\', \'as_text\', \'checkpoint_path\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\'], " + name: "experimental_export_all_saved_models" + argspec: "args=[\'self\', \'export_dir_base\', \'input_receiver_fn_map\', \'assets_extra\', \'as_text\', \'checkpoint_path\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\'], " } member_method { - name: "export_savedmodel" - argspec: "args=[\'self\', \'export_dir_base\', \'serving_input_receiver_fn\', \'assets_extra\', \'as_text\', \'checkpoint_path\', \'strip_default_attrs\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\', \'False\'], " + name: "export_saved_model" + argspec: "args=[\'self\', \'export_dir_base\', \'serving_input_receiver_fn\', \'assets_extra\', \'as_text\', \'checkpoint_path\', \'experimental_mode\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\', \'infer\'], " } member_method { name: "get_variable_names" diff --git a/tensorflow/tools/api/golden/v2/tensorflow.estimator.-linear-classifier.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.estimator.-linear-classifier.pbtxt index 4acbff2cfff..47de660a386 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.estimator.-linear-classifier.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.estimator.-linear-classifier.pbtxt @@ -1,7 +1,7 @@ path: "tensorflow.estimator.LinearClassifier" tf_class { is_instance: "" - is_instance: "" + is_instance: "" is_instance: "" member { name: "config" @@ -32,12 +32,12 @@ tf_class { argspec: "args=[\'self\', \'input_fn\', \'steps\', \'hooks\', \'checkpoint_path\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], " } member_method { - name: "export_saved_model" - argspec: "args=[\'self\', \'export_dir_base\', \'serving_input_receiver_fn\', \'assets_extra\', \'as_text\', \'checkpoint_path\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\'], " + name: "experimental_export_all_saved_models" + argspec: "args=[\'self\', \'export_dir_base\', \'input_receiver_fn_map\', \'assets_extra\', \'as_text\', \'checkpoint_path\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\'], " } member_method { - name: "export_savedmodel" - argspec: "args=[\'self\', \'export_dir_base\', \'serving_input_receiver_fn\', \'assets_extra\', \'as_text\', \'checkpoint_path\', \'strip_default_attrs\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\', \'False\'], " + name: "export_saved_model" + argspec: "args=[\'self\', \'export_dir_base\', \'serving_input_receiver_fn\', \'assets_extra\', \'as_text\', \'checkpoint_path\', \'experimental_mode\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\', \'infer\'], " } member_method { name: "get_variable_names" diff --git a/tensorflow/tools/api/golden/v2/tensorflow.estimator.-linear-estimator.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.estimator.-linear-estimator.pbtxt index 3d6b03098aa..66a127606a5 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.estimator.-linear-estimator.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.estimator.-linear-estimator.pbtxt @@ -1,7 +1,7 @@ path: "tensorflow.estimator.LinearEstimator" tf_class { - is_instance: "" - is_instance: "" + is_instance: "" + is_instance: "" is_instance: "" member { name: "config" @@ -32,12 +32,12 @@ tf_class { argspec: "args=[\'self\', \'input_fn\', \'steps\', \'hooks\', \'checkpoint_path\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], " } member_method { - name: "export_saved_model" - argspec: "args=[\'self\', \'export_dir_base\', \'serving_input_receiver_fn\', \'assets_extra\', \'as_text\', \'checkpoint_path\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\'], " + name: "experimental_export_all_saved_models" + argspec: "args=[\'self\', \'export_dir_base\', \'input_receiver_fn_map\', \'assets_extra\', \'as_text\', \'checkpoint_path\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\'], " } member_method { - name: "export_savedmodel" - argspec: "args=[\'self\', \'export_dir_base\', \'serving_input_receiver_fn\', \'assets_extra\', \'as_text\', \'checkpoint_path\', \'strip_default_attrs\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\', \'False\'], " + name: "export_saved_model" + argspec: "args=[\'self\', \'export_dir_base\', \'serving_input_receiver_fn\', \'assets_extra\', \'as_text\', \'checkpoint_path\', \'experimental_mode\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\', \'infer\'], " } member_method { name: "get_variable_names" diff --git a/tensorflow/tools/api/golden/v2/tensorflow.estimator.-linear-regressor.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.estimator.-linear-regressor.pbtxt index 0d1510e9ab1..5c094fe1318 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.estimator.-linear-regressor.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.estimator.-linear-regressor.pbtxt @@ -1,7 +1,7 @@ path: "tensorflow.estimator.LinearRegressor" tf_class { - is_instance: "" - is_instance: "" + is_instance: "" + is_instance: "" is_instance: "" member { name: "config" @@ -21,7 +21,7 @@ tf_class { } member_method { name: "__init__" - argspec: "args=[\'self\', \'feature_columns\', \'model_dir\', \'label_dimension\', \'weight_column\', \'optimizer\', \'config\', \'partitioner\', \'warm_start_from\', \'loss_reduction\', \'sparse_combiner\'], varargs=None, keywords=None, defaults=[\'None\', \'1\', \'None\', \'Ftrl\', \'None\', \'None\', \'None\', \'weighted_sum\', \'sum\'], " + argspec: "args=[\'self\', \'feature_columns\', \'model_dir\', \'label_dimension\', \'weight_column\', \'optimizer\', \'config\', \'partitioner\', \'warm_start_from\', \'loss_reduction\', \'sparse_combiner\'], varargs=None, keywords=None, defaults=[\'None\', \'1\', \'None\', \'Ftrl\', \'None\', \'None\', \'None\', \'weighted_sum_over_batch_size\', \'sum\'], " } member_method { name: "eval_dir" @@ -32,12 +32,12 @@ tf_class { argspec: "args=[\'self\', \'input_fn\', \'steps\', \'hooks\', \'checkpoint_path\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], " } member_method { - name: "export_saved_model" - argspec: "args=[\'self\', \'export_dir_base\', \'serving_input_receiver_fn\', \'assets_extra\', \'as_text\', \'checkpoint_path\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\'], " + name: "experimental_export_all_saved_models" + argspec: "args=[\'self\', \'export_dir_base\', \'input_receiver_fn_map\', \'assets_extra\', \'as_text\', \'checkpoint_path\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\'], " } member_method { - name: "export_savedmodel" - argspec: "args=[\'self\', \'export_dir_base\', \'serving_input_receiver_fn\', \'assets_extra\', \'as_text\', \'checkpoint_path\', \'strip_default_attrs\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\', \'False\'], " + name: "export_saved_model" + argspec: "args=[\'self\', \'export_dir_base\', \'serving_input_receiver_fn\', \'assets_extra\', \'as_text\', \'checkpoint_path\', \'experimental_mode\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\', \'infer\'], " } member_method { name: "get_variable_names" diff --git a/tensorflow/tools/api/golden/v2/tensorflow.estimator.experimental.-in-memory-evaluator-hook.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.estimator.experimental.-in-memory-evaluator-hook.pbtxt new file mode 100644 index 00000000000..aba120218cc --- /dev/null +++ b/tensorflow/tools/api/golden/v2/tensorflow.estimator.experimental.-in-memory-evaluator-hook.pbtxt @@ -0,0 +1,30 @@ +path: "tensorflow.estimator.experimental.InMemoryEvaluatorHook" +tf_class { + is_instance: "" + is_instance: "" + is_instance: "" + member_method { + name: "__init__" + argspec: "args=[\'self\', \'estimator\', \'input_fn\', \'steps\', \'hooks\', \'name\', \'every_n_iter\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'100\'], " + } + member_method { + name: "after_create_session" + argspec: "args=[\'self\', \'session\', \'coord\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "after_run" + argspec: "args=[\'self\', \'run_context\', \'run_values\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "before_run" + argspec: "args=[\'self\', \'run_context\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "begin" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "end" + argspec: "args=[\'self\', \'session\'], varargs=None, keywords=None, defaults=None" + } +} diff --git a/tensorflow/tools/api/golden/v2/tensorflow.estimator.experimental.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.estimator.experimental.pbtxt index cabca3e883f..f0fd7ce782d 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.estimator.experimental.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.estimator.experimental.pbtxt @@ -1,9 +1,17 @@ path: "tensorflow.estimator.experimental" tf_module { + member { + name: "InMemoryEvaluatorHook" + mtype: "" + } member { name: "LinearSDCA" mtype: "" } + member_method { + name: "build_raw_supervised_input_receiver_fn" + argspec: "args=[\'features\', \'labels\', \'default_batch_size\'], varargs=None, keywords=None, defaults=[\'None\'], " + } member_method { name: "call_logit_fn" argspec: "args=[\'logit_fn\', \'features\', \'mode\', \'params\', \'config\'], varargs=None, keywords=None, defaults=None" @@ -20,6 +28,10 @@ tf_module { name: "make_early_stopping_hook" argspec: "args=[\'estimator\', \'should_stop_fn\', \'run_every_secs\', \'run_every_steps\'], varargs=None, keywords=None, defaults=[\'60\', \'None\'], " } + member_method { + name: "make_stop_at_checkpoint_step_hook" + argspec: "args=[\'estimator\', \'last_step\', \'wait_after_file_check_secs\'], varargs=None, keywords=None, defaults=[\'30\'], " + } member_method { name: "stop_if_higher_hook" argspec: "args=[\'estimator\', \'metric_name\', \'threshold\', \'eval_dir\', \'min_steps\', \'run_every_secs\', \'run_every_steps\'], varargs=None, keywords=None, defaults=[\'None\', \'0\', \'60\', \'None\'], " diff --git a/tensorflow/tools/api/golden/v2/tensorflow.feature_column.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.feature_column.pbtxt index f06e7989537..3aadd7dc341 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.feature_column.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.feature_column.pbtxt @@ -14,7 +14,7 @@ tf_module { } member_method { name: "categorical_column_with_vocabulary_file" - argspec: "args=[\'key\', \'vocabulary_file\', \'vocabulary_size\', \'num_oov_buckets\', \'default_value\', \'dtype\'], varargs=None, keywords=None, defaults=[\'None\', \'0\', \'None\', \"\"], " + argspec: "args=[\'key\', \'vocabulary_file\', \'vocabulary_size\', \'dtype\', \'default_value\', \'num_oov_buckets\'], varargs=None, keywords=None, defaults=[\'None\', \"\", \'None\', \'0\'], " } member_method { name: "categorical_column_with_vocabulary_list" @@ -32,14 +32,6 @@ tf_module { name: "indicator_column" argspec: "args=[\'categorical_column\'], varargs=None, keywords=None, defaults=None" } - member_method { - name: "input_layer" - argspec: "args=[\'features\', \'feature_columns\', \'weight_collections\', \'trainable\', \'cols_to_vars\', \'cols_to_output_tensors\'], varargs=None, keywords=None, defaults=[\'None\', \'True\', \'None\', \'None\'], " - } - member_method { - name: "linear_model" - argspec: "args=[\'features\', \'feature_columns\', \'units\', \'sparse_combiner\', \'weight_collections\', \'trainable\', \'cols_to_vars\'], varargs=None, keywords=None, defaults=[\'1\', \'sum\', \'None\', \'True\', \'None\'], " - } member_method { name: "make_parse_example_spec" argspec: "args=[\'feature_columns\'], varargs=None, keywords=None, defaults=None" diff --git a/tensorflow/tools/api/golden/v2/tensorflow.gfile.-fast-g-file.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.gfile.-fast-g-file.pbtxt deleted file mode 100644 index eecfaffd0a6..00000000000 --- a/tensorflow/tools/api/golden/v2/tensorflow.gfile.-fast-g-file.pbtxt +++ /dev/null @@ -1,58 +0,0 @@ -path: "tensorflow.gfile.FastGFile" -tf_class { - is_instance: "" - is_instance: "" - is_instance: "" - member { - name: "mode" - mtype: "" - } - member { - name: "name" - mtype: "" - } - member_method { - name: "__init__" - argspec: "args=[\'self\', \'name\', \'mode\'], varargs=None, keywords=None, defaults=[\'r\'], " - } - member_method { - name: "close" - argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" - } - member_method { - name: "flush" - argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" - } - member_method { - name: "next" - argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" - } - member_method { - name: "read" - argspec: "args=[\'self\', \'n\'], varargs=None, keywords=None, defaults=[\'-1\'], " - } - member_method { - name: "readline" - argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" - } - member_method { - name: "readlines" - argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" - } - member_method { - name: "seek" - argspec: "args=[\'self\', \'offset\', \'whence\', \'position\'], varargs=None, keywords=None, defaults=[\'None\', \'0\', \'None\'], " - } - member_method { - name: "size" - argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" - } - member_method { - name: "tell" - argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" - } - member_method { - name: "write" - argspec: "args=[\'self\', \'file_content\'], varargs=None, keywords=None, defaults=None" - } -} diff --git a/tensorflow/tools/api/golden/v2/tensorflow.gfile.-g-file.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.gfile.-g-file.pbtxt deleted file mode 100644 index 305251059d9..00000000000 --- a/tensorflow/tools/api/golden/v2/tensorflow.gfile.-g-file.pbtxt +++ /dev/null @@ -1,58 +0,0 @@ -path: "tensorflow.gfile.GFile" -tf_class { - is_instance: "" - is_instance: "" - is_instance: "" - member { - name: "mode" - mtype: "" - } - member { - name: "name" - mtype: "" - } - member_method { - name: "__init__" - argspec: "args=[\'self\', \'name\', \'mode\'], varargs=None, keywords=None, defaults=[\'r\'], " - } - member_method { - name: "close" - argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" - } - member_method { - name: "flush" - argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" - } - member_method { - name: "next" - argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" - } - member_method { - name: "read" - argspec: "args=[\'self\', \'n\'], varargs=None, keywords=None, defaults=[\'-1\'], " - } - member_method { - name: "readline" - argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" - } - member_method { - name: "readlines" - argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" - } - member_method { - name: "seek" - argspec: "args=[\'self\', \'offset\', \'whence\', \'position\'], varargs=None, keywords=None, defaults=[\'None\', \'0\', \'None\'], " - } - member_method { - name: "size" - argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" - } - member_method { - name: "tell" - argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" - } - member_method { - name: "write" - argspec: "args=[\'self\', \'file_content\'], varargs=None, keywords=None, defaults=None" - } -} diff --git a/tensorflow/tools/api/golden/v2/tensorflow.gfile.-open.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.gfile.-open.pbtxt deleted file mode 100644 index 6e8894180a4..00000000000 --- a/tensorflow/tools/api/golden/v2/tensorflow.gfile.-open.pbtxt +++ /dev/null @@ -1,58 +0,0 @@ -path: "tensorflow.gfile.Open" -tf_class { - is_instance: "" - is_instance: "" - is_instance: "" - member { - name: "mode" - mtype: "" - } - member { - name: "name" - mtype: "" - } - member_method { - name: "__init__" - argspec: "args=[\'self\', \'name\', \'mode\'], varargs=None, keywords=None, defaults=[\'r\'], " - } - member_method { - name: "close" - argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" - } - member_method { - name: "flush" - argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" - } - member_method { - name: "next" - argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" - } - member_method { - name: "read" - argspec: "args=[\'self\', \'n\'], varargs=None, keywords=None, defaults=[\'-1\'], " - } - member_method { - name: "readline" - argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" - } - member_method { - name: "readlines" - argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" - } - member_method { - name: "seek" - argspec: "args=[\'self\', \'offset\', \'whence\', \'position\'], varargs=None, keywords=None, defaults=[\'None\', \'0\', \'None\'], " - } - member_method { - name: "size" - argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" - } - member_method { - name: "tell" - argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" - } - member_method { - name: "write" - argspec: "args=[\'self\', \'file_content\'], varargs=None, keywords=None, defaults=None" - } -} diff --git a/tensorflow/tools/api/golden/v2/tensorflow.gfile.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.gfile.pbtxt deleted file mode 100644 index 65b55a8b7c4..00000000000 --- a/tensorflow/tools/api/golden/v2/tensorflow.gfile.pbtxt +++ /dev/null @@ -1,63 +0,0 @@ -path: "tensorflow.gfile" -tf_module { - member { - name: "FastGFile" - mtype: "" - } - member { - name: "GFile" - mtype: "" - } - member { - name: "Open" - mtype: "" - } - member_method { - name: "Copy" - argspec: "args=[\'oldpath\', \'newpath\', \'overwrite\'], varargs=None, keywords=None, defaults=[\'False\'], " - } - member_method { - name: "DeleteRecursively" - argspec: "args=[\'dirname\'], varargs=None, keywords=None, defaults=None" - } - member_method { - name: "Exists" - argspec: "args=[\'filename\'], varargs=None, keywords=None, defaults=None" - } - member_method { - name: "Glob" - argspec: "args=[\'filename\'], varargs=None, keywords=None, defaults=None" - } - member_method { - name: "IsDirectory" - argspec: "args=[\'dirname\'], varargs=None, keywords=None, defaults=None" - } - member_method { - name: "ListDirectory" - argspec: "args=[\'dirname\'], varargs=None, keywords=None, defaults=None" - } - member_method { - name: "MakeDirs" - argspec: "args=[\'dirname\'], varargs=None, keywords=None, defaults=None" - } - member_method { - name: "MkDir" - argspec: "args=[\'dirname\'], varargs=None, keywords=None, defaults=None" - } - member_method { - name: "Remove" - argspec: "args=[\'filename\'], varargs=None, keywords=None, defaults=None" - } - member_method { - name: "Rename" - argspec: "args=[\'oldname\', \'newname\', \'overwrite\'], varargs=None, keywords=None, defaults=[\'False\'], " - } - member_method { - name: "Stat" - argspec: "args=[\'filename\'], varargs=None, keywords=None, defaults=None" - } - member_method { - name: "Walk" - argspec: "args=[\'top\', \'in_order\'], varargs=None, keywords=None, defaults=[\'True\'], " - } -} diff --git a/tensorflow/tools/api/golden/v2/tensorflow.image.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.image.pbtxt index 0a231f1b651..3c6ed1cfb83 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.image.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.image.pbtxt @@ -38,7 +38,7 @@ tf_module { } member_method { name: "crop_and_resize" - argspec: "args=[\'image\', \'boxes\', \'box_ind\', \'crop_size\', \'method\', \'extrapolation_value\', \'name\'], varargs=None, keywords=None, defaults=[\'bilinear\', \'0\', \'None\'], " + argspec: "args=[\'image\', \'boxes\', \'box_indices\', \'crop_size\', \'method\', \'extrapolation_value\', \'name\'], varargs=None, keywords=None, defaults=[\'bilinear\', \'0\', \'None\'], " } member_method { name: "crop_to_bounding_box" @@ -86,7 +86,7 @@ tf_module { } member_method { name: "extract_image_patches" - argspec: "args=[\'images\', \'ksizes\', \'strides\', \'rates\', \'padding\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " + argspec: "args=[\'images\', \'sizes\', \'strides\', \'rates\', \'padding\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " } member_method { name: "extract_jpeg_shape" @@ -173,16 +173,8 @@ tf_module { argspec: "args=[\'image\', \'lower\', \'upper\', \'seed\'], varargs=None, keywords=None, defaults=[\'None\'], " } member_method { - name: "resize_area" - argspec: "args=[\'images\', \'size\', \'align_corners\', \'name\'], varargs=None, keywords=None, defaults=[\'False\', \'None\'], " - } - member_method { - name: "resize_bicubic" - argspec: "args=[\'images\', \'size\', \'align_corners\', \'name\'], varargs=None, keywords=None, defaults=[\'False\', \'None\'], " - } - member_method { - name: "resize_bilinear" - argspec: "args=[\'images\', \'size\', \'align_corners\', \'name\'], varargs=None, keywords=None, defaults=[\'False\', \'None\'], " + name: "resize" + argspec: "args=[\'images\', \'size\', \'method\', \'align_corners\', \'preserve_aspect_ratio\', \'name\'], varargs=None, keywords=None, defaults=[\'0\', \'False\', \'False\', \'None\'], " } member_method { name: "resize_image_with_crop_or_pad" @@ -192,14 +184,6 @@ tf_module { name: "resize_image_with_pad" argspec: "args=[\'image\', \'target_height\', \'target_width\', \'method\'], varargs=None, keywords=None, defaults=[\'0\'], " } - member_method { - name: "resize_images" - argspec: "args=[\'images\', \'size\', \'method\', \'align_corners\', \'preserve_aspect_ratio\'], varargs=None, keywords=None, defaults=[\'0\', \'False\', \'False\'], " - } - member_method { - name: "resize_nearest_neighbor" - argspec: "args=[\'images\', \'size\', \'align_corners\', \'name\'], varargs=None, keywords=None, defaults=[\'False\', \'None\'], " - } member_method { name: "rgb_to_grayscale" argspec: "args=[\'images\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " @@ -222,7 +206,7 @@ tf_module { } member_method { name: "sample_distorted_bounding_box" - argspec: "args=[\'image_size\', \'bounding_boxes\', \'seed\', \'seed2\', \'min_object_covered\', \'aspect_ratio_range\', \'area_range\', \'max_attempts\', \'use_image_if_no_bounding_boxes\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'0.1\', \'None\', \'None\', \'None\', \'None\', \'None\'], " + argspec: "args=[\'image_size\', \'bounding_boxes\', \'seed\', \'min_object_covered\', \'aspect_ratio_range\', \'area_range\', \'max_attempts\', \'use_image_if_no_bounding_boxes\', \'name\'], varargs=None, keywords=None, defaults=[\'0\', \'0.1\', \'None\', \'None\', \'None\', \'None\', \'None\'], " } member_method { name: "sobel_edges" @@ -241,8 +225,8 @@ tf_module { argspec: "args=[\'images\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " } member_method { - name: "transpose_image" - argspec: "args=[\'image\'], varargs=None, keywords=None, defaults=None" + name: "transpose" + argspec: "args=[\'image\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " } member_method { name: "yiq_to_rgb" diff --git a/tensorflow/tools/api/golden/v2/tensorflow.io.gfile.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.io.gfile.pbtxt new file mode 100644 index 00000000000..93d9b0fd75b --- /dev/null +++ b/tensorflow/tools/api/golden/v2/tensorflow.io.gfile.pbtxt @@ -0,0 +1,51 @@ +path: "tensorflow.io.gfile" +tf_module { + member_method { + name: "copy" + argspec: "args=[\'src\', \'dst\', \'overwrite\'], varargs=None, keywords=None, defaults=[\'False\'], " + } + member_method { + name: "exists" + argspec: "args=[\'path\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "glob" + argspec: "args=[\'pattern\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "isdir" + argspec: "args=[\'path\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "listdir" + argspec: "args=[\'path\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "makedirs" + argspec: "args=[\'path\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "mkdir" + argspec: "args=[\'path\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "remove" + argspec: "args=[\'path\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "rename" + argspec: "args=[\'src\', \'dst\', \'overwrite\'], varargs=None, keywords=None, defaults=[\'False\'], " + } + member_method { + name: "rmtree" + argspec: "args=[\'path\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "stat" + argspec: "args=[\'path\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "walk" + argspec: "args=[\'top\', \'topdown\', \'onerror\'], varargs=None, keywords=None, defaults=[\'None\'], " + } +} diff --git a/tensorflow/tools/api/golden/v2/tensorflow.io.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.io.pbtxt index 64b63ed1a4a..8906329742c 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.io.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.io.pbtxt @@ -44,22 +44,50 @@ tf_module { name: "VarLenFeature" mtype: "" } + member { + name: "gfile" + mtype: "" + } + member_method { + name: "decode_and_crop_jpeg" + argspec: "args=[\'contents\', \'crop_window\', \'channels\', \'ratio\', \'fancy_upscaling\', \'try_recover_truncated\', \'acceptable_fraction\', \'dct_method\', \'name\'], varargs=None, keywords=None, defaults=[\'0\', \'1\', \'True\', \'False\', \'1\', \'\', \'None\'], " + } member_method { name: "decode_base64" argspec: "args=[\'input\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "decode_bmp" + argspec: "args=[\'contents\', \'channels\', \'name\'], varargs=None, keywords=None, defaults=[\'0\', \'None\'], " + } member_method { name: "decode_compressed" argspec: "args=[\'bytes\', \'compression_type\', \'name\'], varargs=None, keywords=None, defaults=[\'\', \'None\'], " } member_method { name: "decode_csv" - argspec: "args=[\'records\', \'record_defaults\', \'field_delim\', \'use_quote_delim\', \'name\', \'na_value\', \'select_cols\'], varargs=None, keywords=None, defaults=[\',\', \'True\', \'None\', \'\', \'None\'], " + argspec: "args=[\'records\', \'record_defaults\', \'field_delim\', \'use_quote_delim\', \'na_value\', \'select_cols\', \'name\'], varargs=None, keywords=None, defaults=[\',\', \'True\', \'\', \'None\', \'None\'], " + } + member_method { + name: "decode_gif" + argspec: "args=[\'contents\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "decode_image" + argspec: "args=[\'contents\', \'channels\', \'dtype\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \"\", \'None\'], " + } + member_method { + name: "decode_jpeg" + argspec: "args=[\'contents\', \'channels\', \'ratio\', \'fancy_upscaling\', \'try_recover_truncated\', \'acceptable_fraction\', \'dct_method\', \'name\'], varargs=None, keywords=None, defaults=[\'0\', \'1\', \'True\', \'False\', \'1\', \'\', \'None\'], " } member_method { name: "decode_json_example" argspec: "args=[\'json_examples\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "decode_png" + argspec: "args=[\'contents\', \'channels\', \'dtype\', \'name\'], varargs=None, keywords=None, defaults=[\'0\', \"\", \'None\'], " + } member_method { name: "decode_raw" argspec: "args=[\'bytes\', \'out_type\', \'little_endian\', \'name\'], varargs=None, keywords=None, defaults=[\'True\', \'None\'], " @@ -72,6 +100,18 @@ tf_module { name: "encode_base64" argspec: "args=[\'input\', \'pad\', \'name\'], varargs=None, keywords=None, defaults=[\'False\', \'None\'], " } + member_method { + name: "encode_jpeg" + argspec: "args=[\'image\', \'format\', \'quality\', \'progressive\', \'optimize_size\', \'chroma_downsampling\', \'density_unit\', \'x_density\', \'y_density\', \'xmp_metadata\', \'name\'], varargs=None, keywords=None, defaults=[\'\', \'95\', \'False\', \'False\', \'True\', \'in\', \'300\', \'300\', \'\', \'None\'], " + } + member_method { + name: "extract_jpeg_shape" + argspec: "args=[\'contents\', \'output_type\', \'name\'], varargs=None, keywords=None, defaults=[\"\", \'None\'], " + } + member_method { + name: "is_jpeg" + argspec: "args=[\'contents\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " + } member_method { name: "match_filenames_once" argspec: "args=[\'pattern\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " @@ -82,7 +122,7 @@ tf_module { } member_method { name: "parse_example" - argspec: "args=[\'serialized\', \'features\', \'name\', \'example_names\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + argspec: "args=[\'serialized\', \'features\', \'example_names\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " } member_method { name: "parse_sequence_example" @@ -90,7 +130,7 @@ tf_module { } member_method { name: "parse_single_example" - argspec: "args=[\'serialized\', \'features\', \'name\', \'example_names\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + argspec: "args=[\'serialized\', \'features\', \'example_names\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " } member_method { name: "parse_single_sequence_example" @@ -106,20 +146,16 @@ tf_module { } member_method { name: "serialize_many_sparse" - argspec: "args=[\'sp_input\', \'name\', \'out_type\'], varargs=None, keywords=None, defaults=[\'None\', \"\"], " + argspec: "args=[\'sp_input\', \'out_type\', \'name\'], varargs=None, keywords=None, defaults=[\"\", \'None\'], " } member_method { name: "serialize_sparse" - argspec: "args=[\'sp_input\', \'name\', \'out_type\'], varargs=None, keywords=None, defaults=[\'None\', \"\"], " + argspec: "args=[\'sp_input\', \'out_type\', \'name\'], varargs=None, keywords=None, defaults=[\"\", \'None\'], " } member_method { name: "serialize_tensor" argspec: "args=[\'tensor\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " } - member_method { - name: "tf_record_iterator" - argspec: "args=[\'path\', \'options\'], varargs=None, keywords=None, defaults=[\'None\'], " - } member_method { name: "write_file" argspec: "args=[\'filename\', \'contents\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.-model.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.-model.pbtxt index 8ccba990bdd..a3254cbd947 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.keras.-model.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.-model.pbtxt @@ -41,6 +41,14 @@ tf_class { name: "losses" mtype: "" } + member { + name: "metrics" + mtype: "" + } + member { + name: "metrics_names" + mtype: "" + } member { name: "name" mtype: "" @@ -69,6 +77,10 @@ tf_class { name: "output_shape" mtype: "" } + member { + name: "run_eagerly" + mtype: "" + } member { name: "state_updates" mtype: "" @@ -105,13 +117,17 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None" } member_method { name: "add_weight" @@ -225,6 +241,10 @@ tf_class { name: "predict_on_batch" argspec: "args=[\'self\', \'x\'], varargs=None, keywords=None, defaults=None" } + member_method { + name: "reset_metrics" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } member_method { name: "reset_states" argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" @@ -247,7 +267,7 @@ tf_class { } member_method { name: "test_on_batch" - argspec: "args=[\'self\', \'x\', \'y\', \'sample_weight\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + argspec: "args=[\'self\', \'x\', \'y\', \'sample_weight\', \'reset_metrics\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'True\'], " } member_method { name: "to_json" @@ -259,6 +279,6 @@ tf_class { } member_method { name: "train_on_batch" - argspec: "args=[\'self\', \'x\', \'y\', \'sample_weight\', \'class_weight\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], " + argspec: "args=[\'self\', \'x\', \'y\', \'sample_weight\', \'class_weight\', \'reset_metrics\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\'], " } } diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.-sequential.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.-sequential.pbtxt index 27aa91a6452..b70e9ee98d5 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.keras.-sequential.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.-sequential.pbtxt @@ -42,6 +42,14 @@ tf_class { name: "losses" mtype: "" } + member { + name: "metrics" + mtype: "" + } + member { + name: "metrics_names" + mtype: "" + } member { name: "name" mtype: "" @@ -70,6 +78,10 @@ tf_class { name: "output_shape" mtype: "" } + member { + name: "run_eagerly" + mtype: "" + } member { name: "state_updates" mtype: "" @@ -110,13 +122,17 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None" } member_method { name: "add_weight" @@ -242,6 +258,10 @@ tf_class { name: "predict_proba" argspec: "args=[\'self\', \'x\', \'batch_size\', \'verbose\'], varargs=None, keywords=None, defaults=[\'32\', \'0\'], " } + member_method { + name: "reset_metrics" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } member_method { name: "reset_states" argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" @@ -264,7 +284,7 @@ tf_class { } member_method { name: "test_on_batch" - argspec: "args=[\'self\', \'x\', \'y\', \'sample_weight\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + argspec: "args=[\'self\', \'x\', \'y\', \'sample_weight\', \'reset_metrics\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'True\'], " } member_method { name: "to_json" @@ -276,6 +296,6 @@ tf_class { } member_method { name: "train_on_batch" - argspec: "args=[\'self\', \'x\', \'y\', \'sample_weight\', \'class_weight\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], " + argspec: "args=[\'self\', \'x\', \'y\', \'sample_weight\', \'class_weight\', \'reset_metrics\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\'], " } } diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.backend.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.backend.pbtxt index b30778b2a08..d200d3d26d7 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.keras.backend.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.backend.pbtxt @@ -394,7 +394,7 @@ tf_module { } member_method { name: "rnn" - argspec: "args=[\'step_function\', \'inputs\', \'initial_states\', \'go_backwards\', \'mask\', \'constants\', \'unroll\', \'input_length\', \'time_major\'], varargs=None, keywords=None, defaults=[\'False\', \'None\', \'None\', \'False\', \'None\', \'False\'], " + argspec: "args=[\'step_function\', \'inputs\', \'initial_states\', \'go_backwards\', \'mask\', \'constants\', \'unroll\', \'input_length\', \'time_major\', \'zero_output_for_mask\'], varargs=None, keywords=None, defaults=[\'False\', \'None\', \'None\', \'False\', \'None\', \'False\', \'False\'], " } member_method { name: "round" @@ -508,6 +508,10 @@ tf_module { name: "temporal_padding" argspec: "args=[\'x\', \'padding\'], varargs=None, keywords=None, defaults=[\'(1, 1)\'], " } + member_method { + name: "tile" + argspec: "args=[\'x\', \'n\'], varargs=None, keywords=None, defaults=None" + } member_method { name: "to_dense" argspec: "args=[\'tensor\'], varargs=None, keywords=None, defaults=None" diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.experimental.-peephole-l-s-t-m-cell.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.experimental.-peephole-l-s-t-m-cell.pbtxt index db69e25c5b7..1d814b2c8b5 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.keras.experimental.-peephole-l-s-t-m-cell.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.experimental.-peephole-l-s-t-m-cell.pbtxt @@ -89,6 +89,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-activation.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-activation.pbtxt index 5510465d7b0..b84629540e7 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-activation.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-activation.pbtxt @@ -88,6 +88,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-activity-regularization.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-activity-regularization.pbtxt index 38ec8a0aff0..5918a13ad86 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-activity-regularization.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-activity-regularization.pbtxt @@ -88,6 +88,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-add.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-add.pbtxt index 41cb8e30bfb..599da06427d 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-add.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-add.pbtxt @@ -89,6 +89,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-alpha-dropout.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-alpha-dropout.pbtxt index 9a7aaa8e961..f9ff1538c81 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-alpha-dropout.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-alpha-dropout.pbtxt @@ -88,6 +88,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-average-pooling1-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-average-pooling1-d.pbtxt index 014f5828fad..723fc9cdb0d 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-average-pooling1-d.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-average-pooling1-d.pbtxt @@ -89,6 +89,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-average-pooling2-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-average-pooling2-d.pbtxt index cc303bf7b98..957ce2f0ce8 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-average-pooling2-d.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-average-pooling2-d.pbtxt @@ -89,6 +89,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-average-pooling3-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-average-pooling3-d.pbtxt index 628447ce355..a52c0af6817 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-average-pooling3-d.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-average-pooling3-d.pbtxt @@ -89,6 +89,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-average.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-average.pbtxt index f03c986c222..a004db62ddc 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-average.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-average.pbtxt @@ -89,6 +89,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-avg-pool1-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-avg-pool1-d.pbtxt index a6e4856de9b..44f83d1387c 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-avg-pool1-d.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-avg-pool1-d.pbtxt @@ -89,6 +89,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-avg-pool2-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-avg-pool2-d.pbtxt index a01eaf8a126..8378faf7188 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-avg-pool2-d.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-avg-pool2-d.pbtxt @@ -89,6 +89,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-avg-pool3-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-avg-pool3-d.pbtxt index 0d6698f2ef4..9d5655c9644 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-avg-pool3-d.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-avg-pool3-d.pbtxt @@ -89,6 +89,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-batch-normalization.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-batch-normalization.pbtxt index f1b23be48f7..5da79268129 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-batch-normalization.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-batch-normalization.pbtxt @@ -1,6 +1,6 @@ path: "tensorflow.keras.layers.BatchNormalization" tf_class { - is_instance: "" + is_instance: "" is_instance: "" is_instance: "" is_instance: "" @@ -88,6 +88,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-bidirectional.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-bidirectional.pbtxt index 0672cd5b7b8..d37a6b47105 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-bidirectional.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-bidirectional.pbtxt @@ -97,6 +97,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-concatenate.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-concatenate.pbtxt index b25ae1e82e8..1ad7a91be0b 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-concatenate.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-concatenate.pbtxt @@ -89,6 +89,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-conv-l-s-t-m2-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-conv-l-s-t-m2-d.pbtxt index bb1918eba65..cb9abc25396 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-conv-l-s-t-m2-d.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-conv-l-s-t-m2-d.pbtxt @@ -178,6 +178,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-conv1-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-conv1-d.pbtxt index 16e0fd5a313..47dba1d81f8 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-conv1-d.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-conv1-d.pbtxt @@ -89,6 +89,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-conv2-d-transpose.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-conv2-d-transpose.pbtxt index 381839d6deb..fd649418961 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-conv2-d-transpose.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-conv2-d-transpose.pbtxt @@ -90,6 +90,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-conv2-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-conv2-d.pbtxt index 543bae6fa96..1b1425d5319 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-conv2-d.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-conv2-d.pbtxt @@ -89,6 +89,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-conv3-d-transpose.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-conv3-d-transpose.pbtxt index 2933f9f4b3a..1741063fe8b 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-conv3-d-transpose.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-conv3-d-transpose.pbtxt @@ -90,6 +90,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-conv3-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-conv3-d.pbtxt index 072943dc2c7..50feb4f458a 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-conv3-d.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-conv3-d.pbtxt @@ -89,6 +89,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-convolution1-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-convolution1-d.pbtxt index 222a1ef4fc5..faaa535df9f 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-convolution1-d.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-convolution1-d.pbtxt @@ -89,6 +89,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-convolution2-d-transpose.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-convolution2-d-transpose.pbtxt index 9c9c7461c8b..4079329d1ee 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-convolution2-d-transpose.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-convolution2-d-transpose.pbtxt @@ -90,6 +90,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-convolution2-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-convolution2-d.pbtxt index f9390671781..32e56696e16 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-convolution2-d.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-convolution2-d.pbtxt @@ -89,6 +89,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-convolution3-d-transpose.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-convolution3-d-transpose.pbtxt index 44ca598724a..381abe73401 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-convolution3-d-transpose.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-convolution3-d-transpose.pbtxt @@ -90,6 +90,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-convolution3-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-convolution3-d.pbtxt index 471b18ef850..b3e4bf9689d 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-convolution3-d.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-convolution3-d.pbtxt @@ -89,6 +89,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-cropping1-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-cropping1-d.pbtxt index 0f250a09b7e..7aeff8003c3 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-cropping1-d.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-cropping1-d.pbtxt @@ -88,6 +88,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-cropping2-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-cropping2-d.pbtxt index f52128483c6..a1728d9d4f9 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-cropping2-d.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-cropping2-d.pbtxt @@ -88,6 +88,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-cropping3-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-cropping3-d.pbtxt index 98daf3bab12..8d8fd142cc6 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-cropping3-d.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-cropping3-d.pbtxt @@ -88,6 +88,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-cu-d-n-n-g-r-u.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-cu-d-n-n-g-r-u.pbtxt index b207c680005..7758209adf8 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-cu-d-n-n-g-r-u.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-cu-d-n-n-g-r-u.pbtxt @@ -98,6 +98,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-cu-d-n-n-l-s-t-m.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-cu-d-n-n-l-s-t-m.pbtxt index 2d7a09ceda9..7c463ff1257 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-cu-d-n-n-l-s-t-m.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-cu-d-n-n-l-s-t-m.pbtxt @@ -98,6 +98,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v2/tensorflow.nn.rnn_cell.-multi-r-n-n-cell.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-dense-features.pbtxt similarity index 77% rename from tensorflow/tools/api/golden/v2/tensorflow.nn.rnn_cell.-multi-r-n-n-cell.pbtxt rename to tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-dense-features.pbtxt index b66c0f89cc9..0781a93bd56 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.nn.rnn_cell.-multi-r-n-n-cell.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-dense-features.pbtxt @@ -1,8 +1,6 @@ -path: "tensorflow.nn.rnn_cell.MultiRNNCell" +path: "tensorflow.keras.layers.DenseFeatures" tf_class { - is_instance: "" - is_instance: "" - is_instance: "" + is_instance: "" is_instance: "" is_instance: "" is_instance: "" @@ -14,10 +12,6 @@ tf_class { name: "dtype" mtype: "" } - member { - name: "graph" - mtype: "" - } member { name: "inbound_nodes" mtype: "" @@ -66,18 +60,6 @@ tf_class { name: "output_shape" mtype: "" } - member { - name: "output_size" - mtype: "" - } - member { - name: "scope_name" - mtype: "" - } - member { - name: "state_size" - mtype: "" - } member { name: "trainable_variables" mtype: "" @@ -100,12 +82,16 @@ tf_class { } member_method { name: "__init__" - argspec: "args=[\'self\', \'cells\', \'state_is_tuple\'], varargs=None, keywords=None, defaults=[\'True\'], " + argspec: "args=[\'self\', \'feature_columns\', \'trainable\', \'name\'], varargs=None, keywords=kwargs, defaults=[\'True\', \'None\'], " } member_method { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " @@ -116,7 +102,7 @@ tf_class { } member_method { name: "add_weight" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'use_resource\', \'synchronization\', \'aggregation\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], " } member_method { name: "apply" @@ -128,7 +114,7 @@ tf_class { } member_method { name: "call" - argspec: "args=[\'self\', \'inputs\', \'state\'], varargs=None, keywords=None, defaults=None" + argspec: "args=[\'self\', \'features\', \'cols_to_output_tensors\'], varargs=None, keywords=None, defaults=[\'None\'], " } member_method { name: "compute_mask" @@ -150,10 +136,6 @@ tf_class { name: "get_config" argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" } - member_method { - name: "get_initial_state" - argspec: "args=[\'self\', \'inputs\', \'batch_size\', \'dtype\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], " - } member_method { name: "get_input_at" argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" @@ -194,8 +176,4 @@ tf_class { name: "set_weights" argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None" } - member_method { - name: "zero_state" - argspec: "args=[\'self\', \'batch_size\', \'dtype\'], varargs=None, keywords=None, defaults=None" - } } diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-dense.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-dense.pbtxt index 3ac38257593..4960d0264e9 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-dense.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-dense.pbtxt @@ -88,6 +88,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-depthwise-conv2-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-depthwise-conv2-d.pbtxt index 280ec8c25fa..8fad7535f88 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-depthwise-conv2-d.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-depthwise-conv2-d.pbtxt @@ -90,6 +90,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-dot.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-dot.pbtxt index 560f66f9c7a..5b425f2d4d7 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-dot.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-dot.pbtxt @@ -89,6 +89,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-dropout.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-dropout.pbtxt index c0543529c38..f6c4d0a438e 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-dropout.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-dropout.pbtxt @@ -88,6 +88,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-e-l-u.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-e-l-u.pbtxt index 04eb2824b9b..82b761fc176 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-e-l-u.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-e-l-u.pbtxt @@ -88,6 +88,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-embedding.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-embedding.pbtxt index f400432915f..c9ff323877e 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-embedding.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-embedding.pbtxt @@ -88,6 +88,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-flatten.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-flatten.pbtxt index ab176b441a2..9b4165d4cbf 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-flatten.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-flatten.pbtxt @@ -88,6 +88,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-g-r-u-cell.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-g-r-u-cell.pbtxt index c3895a0ac12..f225f7c4309 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-g-r-u-cell.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-g-r-u-cell.pbtxt @@ -88,6 +88,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-g-r-u.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-g-r-u.pbtxt index 9e24bb8ae6a..855d0017001 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-g-r-u.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-g-r-u.pbtxt @@ -161,6 +161,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-gaussian-dropout.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-gaussian-dropout.pbtxt index 55e0d7ef023..2c404c99cd2 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-gaussian-dropout.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-gaussian-dropout.pbtxt @@ -88,6 +88,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-gaussian-noise.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-gaussian-noise.pbtxt index 38fbff5e4a3..6f109d59d0f 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-gaussian-noise.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-gaussian-noise.pbtxt @@ -88,6 +88,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-average-pooling1-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-average-pooling1-d.pbtxt index a8094c0bde3..69f8a9031d3 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-average-pooling1-d.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-average-pooling1-d.pbtxt @@ -89,6 +89,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-average-pooling2-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-average-pooling2-d.pbtxt index 929f48df231..4299f765e52 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-average-pooling2-d.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-average-pooling2-d.pbtxt @@ -89,6 +89,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-average-pooling3-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-average-pooling3-d.pbtxt index 2e6d59337f1..9153a1a2406 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-average-pooling3-d.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-average-pooling3-d.pbtxt @@ -89,6 +89,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-avg-pool1-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-avg-pool1-d.pbtxt index 3ebe162f573..625e81fd232 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-avg-pool1-d.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-avg-pool1-d.pbtxt @@ -89,6 +89,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-avg-pool2-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-avg-pool2-d.pbtxt index 4e3e258430c..2fc769742c7 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-avg-pool2-d.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-avg-pool2-d.pbtxt @@ -89,6 +89,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-avg-pool3-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-avg-pool3-d.pbtxt index fb9166316f6..e307a65c7c5 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-avg-pool3-d.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-avg-pool3-d.pbtxt @@ -89,6 +89,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-max-pool1-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-max-pool1-d.pbtxt index c0a53b847b4..4394ad0364e 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-max-pool1-d.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-max-pool1-d.pbtxt @@ -89,6 +89,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-max-pool2-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-max-pool2-d.pbtxt index 87b7f6797a0..050ed39fe98 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-max-pool2-d.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-max-pool2-d.pbtxt @@ -89,6 +89,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-max-pool3-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-max-pool3-d.pbtxt index 98bf96fa0c2..436191821ef 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-max-pool3-d.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-max-pool3-d.pbtxt @@ -89,6 +89,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-max-pooling1-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-max-pooling1-d.pbtxt index ff6c6f3ec4d..4ba540aa6ad 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-max-pooling1-d.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-max-pooling1-d.pbtxt @@ -89,6 +89,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-max-pooling2-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-max-pooling2-d.pbtxt index c9d4158d1c4..a2e9322cb3f 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-max-pooling2-d.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-max-pooling2-d.pbtxt @@ -89,6 +89,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-max-pooling3-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-max-pooling3-d.pbtxt index 9953102ff99..5d16a57fc1a 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-max-pooling3-d.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-max-pooling3-d.pbtxt @@ -89,6 +89,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-input-layer.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-input-layer.pbtxt index 2617f5a95fa..9dd29c1251e 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-input-layer.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-input-layer.pbtxt @@ -88,6 +88,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-input-spec.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-input-spec.pbtxt index 5fd0a47a68c..bc3ceb67a4e 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-input-spec.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-input-spec.pbtxt @@ -1,6 +1,6 @@ path: "tensorflow.keras.layers.InputSpec" tf_class { - is_instance: "" + is_instance: "" is_instance: "" member_method { name: "__init__" diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-l-s-t-m-cell.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-l-s-t-m-cell.pbtxt index e9f6ef45aaf..0045d5775e2 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-l-s-t-m-cell.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-l-s-t-m-cell.pbtxt @@ -88,6 +88,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-l-s-t-m.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-l-s-t-m.pbtxt index 1b1ccbe1180..529c750f987 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-l-s-t-m.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-l-s-t-m.pbtxt @@ -161,6 +161,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-lambda.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-lambda.pbtxt index 2e0b6bac24f..d4d1bc6b6bb 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-lambda.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-lambda.pbtxt @@ -88,6 +88,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-layer.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-layer.pbtxt index 1e93d1118a4..e1f54911809 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-layer.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-layer.pbtxt @@ -87,6 +87,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-leaky-re-l-u.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-leaky-re-l-u.pbtxt index bfd36012a7e..9b69d9a9447 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-leaky-re-l-u.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-leaky-re-l-u.pbtxt @@ -88,6 +88,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-linear-model.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-linear-model.pbtxt new file mode 100644 index 00000000000..2b66576c96b --- /dev/null +++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-linear-model.pbtxt @@ -0,0 +1,289 @@ +path: "tensorflow.keras.layers.LinearModel" +tf_class { + is_instance: "" + is_instance: "" + is_instance: "" + is_instance: "" + is_instance: "" + is_instance: "" + member { + name: "activity_regularizer" + mtype: "" + } + member { + name: "bias" + mtype: "" + } + member { + name: "dtype" + mtype: "" + } + member { + name: "inbound_nodes" + mtype: "" + } + member { + name: "input" + mtype: "" + } + member { + name: "input_mask" + mtype: "" + } + member { + name: "input_shape" + mtype: "" + } + member { + name: "input_spec" + mtype: "" + } + member { + name: "layers" + mtype: "" + } + member { + name: "losses" + mtype: "" + } + member { + name: "metrics" + mtype: "" + } + member { + name: "metrics_names" + mtype: "" + } + member { + name: "name" + mtype: "" + } + member { + name: "non_trainable_variables" + mtype: "" + } + member { + name: "non_trainable_weights" + mtype: "" + } + member { + name: "outbound_nodes" + mtype: "" + } + member { + name: "output" + mtype: "" + } + member { + name: "output_mask" + mtype: "" + } + member { + name: "output_shape" + mtype: "" + } + member { + name: "run_eagerly" + mtype: "" + } + member { + name: "state_updates" + mtype: "" + } + member { + name: "stateful" + mtype: "" + } + member { + name: "trainable_variables" + mtype: "" + } + member { + name: "trainable_weights" + mtype: "" + } + member { + name: "updates" + mtype: "" + } + member { + name: "variables" + mtype: "" + } + member { + name: "weights" + mtype: "" + } + member_method { + name: "__init__" + argspec: "args=[\'self\', \'feature_columns\', \'units\', \'sparse_combiner\', \'trainable\', \'name\'], varargs=None, keywords=kwargs, defaults=[\'1\', \'sum\', \'True\', \'None\'], " + } + member_method { + name: "add_loss" + argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } + member_method { + name: "add_update" + argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "add_variable" + argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None" + } + member_method { + name: "add_weight" + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], " + } + member_method { + name: "apply" + argspec: "args=[\'self\', \'inputs\'], varargs=args, keywords=kwargs, defaults=None" + } + member_method { + name: "build" + argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "call" + argspec: "args=[\'self\', \'features\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "compile" + argspec: "args=[\'self\', \'optimizer\', \'loss\', \'metrics\', \'loss_weights\', \'sample_weight_mode\', \'weighted_metrics\', \'target_tensors\', \'distribute\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\'], " + } + member_method { + name: "compute_mask" + argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "compute_output_shape" + argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "count_params" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "evaluate" + argspec: "args=[\'self\', \'x\', \'y\', \'batch_size\', \'verbose\', \'sample_weight\', \'steps\', \'max_queue_size\', \'workers\', \'use_multiprocessing\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'1\', \'None\', \'None\', \'10\', \'1\', \'False\'], " + } + member_method { + name: "evaluate_generator" + argspec: "args=[\'self\', \'generator\', \'steps\', \'max_queue_size\', \'workers\', \'use_multiprocessing\', \'verbose\'], varargs=None, keywords=None, defaults=[\'None\', \'10\', \'1\', \'False\', \'0\'], " + } + member_method { + name: "fit" + argspec: "args=[\'self\', \'x\', \'y\', \'batch_size\', \'epochs\', \'verbose\', \'callbacks\', \'validation_split\', \'validation_data\', \'shuffle\', \'class_weight\', \'sample_weight\', \'initial_epoch\', \'steps_per_epoch\', \'validation_steps\', \'max_queue_size\', \'workers\', \'use_multiprocessing\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'1\', \'1\', \'None\', \'0.0\', \'None\', \'True\', \'None\', \'None\', \'0\', \'None\', \'None\', \'10\', \'1\', \'False\'], " + } + member_method { + name: "fit_generator" + argspec: "args=[\'self\', \'generator\', \'steps_per_epoch\', \'epochs\', \'verbose\', \'callbacks\', \'validation_data\', \'validation_steps\', \'class_weight\', \'max_queue_size\', \'workers\', \'use_multiprocessing\', \'shuffle\', \'initial_epoch\'], varargs=None, keywords=None, defaults=[\'None\', \'1\', \'1\', \'None\', \'None\', \'None\', \'None\', \'10\', \'1\', \'False\', \'True\', \'0\'], " + } + member_method { + name: "from_config" + argspec: "args=[\'cls\', \'config\', \'custom_objects\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "get_config" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_input_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_input_mask_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_input_shape_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_layer" + argspec: "args=[\'self\', \'name\', \'index\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } + member_method { + name: "get_losses_for" + argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_output_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_output_mask_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_output_shape_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_updates_for" + argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_weights" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "load_weights" + argspec: "args=[\'self\', \'filepath\', \'by_name\'], varargs=None, keywords=None, defaults=[\'False\'], " + } + member_method { + name: "predict" + argspec: "args=[\'self\', \'x\', \'batch_size\', \'verbose\', \'steps\', \'max_queue_size\', \'workers\', \'use_multiprocessing\'], varargs=None, keywords=None, defaults=[\'None\', \'0\', \'None\', \'10\', \'1\', \'False\'], " + } + member_method { + name: "predict_generator" + argspec: "args=[\'self\', \'generator\', \'steps\', \'max_queue_size\', \'workers\', \'use_multiprocessing\', \'verbose\'], varargs=None, keywords=None, defaults=[\'None\', \'10\', \'1\', \'False\', \'0\'], " + } + member_method { + name: "predict_on_batch" + argspec: "args=[\'self\', \'x\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "reset_metrics" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "reset_states" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "save" + argspec: "args=[\'self\', \'filepath\', \'overwrite\', \'include_optimizer\'], varargs=None, keywords=None, defaults=[\'True\', \'True\'], " + } + member_method { + name: "save_weights" + argspec: "args=[\'self\', \'filepath\', \'overwrite\', \'save_format\'], varargs=None, keywords=None, defaults=[\'True\', \'None\'], " + } + member_method { + name: "set_weights" + argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "summary" + argspec: "args=[\'self\', \'line_length\', \'positions\', \'print_fn\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], " + } + member_method { + name: "test_on_batch" + argspec: "args=[\'self\', \'x\', \'y\', \'sample_weight\', \'reset_metrics\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'True\'], " + } + member_method { + name: "to_json" + argspec: "args=[\'self\'], varargs=None, keywords=kwargs, defaults=None" + } + member_method { + name: "to_yaml" + argspec: "args=[\'self\'], varargs=None, keywords=kwargs, defaults=None" + } + member_method { + name: "train_on_batch" + argspec: "args=[\'self\', \'x\', \'y\', \'sample_weight\', \'class_weight\', \'reset_metrics\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\'], " + } +} diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-locally-connected1-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-locally-connected1-d.pbtxt index 5ad5990d7e6..fd522594325 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-locally-connected1-d.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-locally-connected1-d.pbtxt @@ -88,6 +88,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-locally-connected2-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-locally-connected2-d.pbtxt index 40d03369a52..5fc8af0d035 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-locally-connected2-d.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-locally-connected2-d.pbtxt @@ -88,6 +88,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-masking.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-masking.pbtxt index 86666b51bb8..7f8932270e6 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-masking.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-masking.pbtxt @@ -88,6 +88,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-max-pool1-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-max-pool1-d.pbtxt index d26da270e74..4723b99cb07 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-max-pool1-d.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-max-pool1-d.pbtxt @@ -89,6 +89,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-max-pool2-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-max-pool2-d.pbtxt index 85f23df671d..173c5d4a8b1 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-max-pool2-d.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-max-pool2-d.pbtxt @@ -89,6 +89,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-max-pool3-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-max-pool3-d.pbtxt index 235806b9650..14e1899e145 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-max-pool3-d.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-max-pool3-d.pbtxt @@ -89,6 +89,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-max-pooling1-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-max-pooling1-d.pbtxt index 524c5fd69e5..a708e652bf0 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-max-pooling1-d.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-max-pooling1-d.pbtxt @@ -89,6 +89,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-max-pooling2-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-max-pooling2-d.pbtxt index fda2562fc8c..e6706b5cf9f 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-max-pooling2-d.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-max-pooling2-d.pbtxt @@ -89,6 +89,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-max-pooling3-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-max-pooling3-d.pbtxt index 71d2d09a8d1..a73c082d1bb 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-max-pooling3-d.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-max-pooling3-d.pbtxt @@ -89,6 +89,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-maximum.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-maximum.pbtxt index 12949b39a6f..f3f195554bb 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-maximum.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-maximum.pbtxt @@ -89,6 +89,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-minimum.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-minimum.pbtxt index ab16d0021e6..f345d1d67b2 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-minimum.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-minimum.pbtxt @@ -89,6 +89,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-multiply.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-multiply.pbtxt index 61ccbf59627..31cb8bc177c 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-multiply.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-multiply.pbtxt @@ -89,6 +89,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-p-re-l-u.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-p-re-l-u.pbtxt index ce2320d7030..44cccc92bd2 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-p-re-l-u.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-p-re-l-u.pbtxt @@ -88,6 +88,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-permute.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-permute.pbtxt index 69848af8cf8..b55e191ff1a 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-permute.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-permute.pbtxt @@ -88,6 +88,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-r-n-n.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-r-n-n.pbtxt index 3358f26aebf..e9575436e5b 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-r-n-n.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-r-n-n.pbtxt @@ -92,6 +92,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-re-l-u.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-re-l-u.pbtxt index 413f45f018a..98223b207f2 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-re-l-u.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-re-l-u.pbtxt @@ -88,6 +88,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-repeat-vector.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-repeat-vector.pbtxt index 9c61ff60274..2df918b16b2 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-repeat-vector.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-repeat-vector.pbtxt @@ -88,6 +88,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-reshape.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-reshape.pbtxt index baa91804c49..ce5f9e21290 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-reshape.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-reshape.pbtxt @@ -88,6 +88,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-separable-conv1-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-separable-conv1-d.pbtxt index 15a5d6ac9ea..a0bb917775f 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-separable-conv1-d.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-separable-conv1-d.pbtxt @@ -90,6 +90,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-separable-conv2-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-separable-conv2-d.pbtxt index be43bd5b3c1..d7942f201bd 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-separable-conv2-d.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-separable-conv2-d.pbtxt @@ -90,6 +90,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-separable-convolution1-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-separable-convolution1-d.pbtxt index 6105992c7a3..f7ac9042d46 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-separable-convolution1-d.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-separable-convolution1-d.pbtxt @@ -90,6 +90,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-separable-convolution2-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-separable-convolution2-d.pbtxt index 1b6cf1e9ecb..e5a92688220 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-separable-convolution2-d.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-separable-convolution2-d.pbtxt @@ -90,6 +90,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-simple-r-n-n-cell.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-simple-r-n-n-cell.pbtxt index 29488a37f8f..0fe2c974a76 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-simple-r-n-n-cell.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-simple-r-n-n-cell.pbtxt @@ -88,6 +88,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-simple-r-n-n.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-simple-r-n-n.pbtxt index 3d70cf8b659..2ee5873f0f1 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-simple-r-n-n.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-simple-r-n-n.pbtxt @@ -149,6 +149,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-softmax.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-softmax.pbtxt index d29731ecf9d..5b8f64aa357 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-softmax.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-softmax.pbtxt @@ -88,6 +88,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-spatial-dropout1-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-spatial-dropout1-d.pbtxt index a6d7494ca7d..240cb6e562f 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-spatial-dropout1-d.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-spatial-dropout1-d.pbtxt @@ -89,6 +89,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-spatial-dropout2-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-spatial-dropout2-d.pbtxt index c36e802693d..6226c469f8a 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-spatial-dropout2-d.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-spatial-dropout2-d.pbtxt @@ -89,6 +89,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-spatial-dropout3-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-spatial-dropout3-d.pbtxt index 9c46cfe40fd..34dabce6d8d 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-spatial-dropout3-d.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-spatial-dropout3-d.pbtxt @@ -89,6 +89,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-stacked-r-n-n-cells.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-stacked-r-n-n-cells.pbtxt index 8982f787940..0ddf628ace5 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-stacked-r-n-n-cells.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-stacked-r-n-n-cells.pbtxt @@ -96,6 +96,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-subtract.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-subtract.pbtxt index ec2cc502984..12eb35ad154 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-subtract.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-subtract.pbtxt @@ -89,6 +89,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-thresholded-re-l-u.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-thresholded-re-l-u.pbtxt index d7bc1980f32..c41020c2b45 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-thresholded-re-l-u.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-thresholded-re-l-u.pbtxt @@ -88,6 +88,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-time-distributed.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-time-distributed.pbtxt index fec2de6b49e..479f89cf6ae 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-time-distributed.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-time-distributed.pbtxt @@ -93,6 +93,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-up-sampling1-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-up-sampling1-d.pbtxt index 3d285e7f17d..233363ce026 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-up-sampling1-d.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-up-sampling1-d.pbtxt @@ -88,6 +88,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-up-sampling2-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-up-sampling2-d.pbtxt index b05e5ec84de..cb6228ac446 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-up-sampling2-d.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-up-sampling2-d.pbtxt @@ -88,6 +88,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-up-sampling3-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-up-sampling3-d.pbtxt index 728eca415a8..03bad3ccb61 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-up-sampling3-d.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-up-sampling3-d.pbtxt @@ -88,6 +88,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-wrapper.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-wrapper.pbtxt index da64e77c39c..158996792a4 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-wrapper.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-wrapper.pbtxt @@ -92,6 +92,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-zero-padding1-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-zero-padding1-d.pbtxt index 2f505f9293f..63a56cd3eeb 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-zero-padding1-d.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-zero-padding1-d.pbtxt @@ -88,6 +88,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-zero-padding2-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-zero-padding2-d.pbtxt index f82c77072e6..965a4cca046 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-zero-padding2-d.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-zero-padding2-d.pbtxt @@ -88,6 +88,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-zero-padding3-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-zero-padding3-d.pbtxt index 54e01a99177..1a624308878 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-zero-padding3-d.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-zero-padding3-d.pbtxt @@ -88,6 +88,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.pbtxt index 9d7e5bb8c78..3b4724ef104 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.pbtxt @@ -124,6 +124,10 @@ tf_module { name: "Dense" mtype: "" } + member { + name: "DenseFeatures" + mtype: "" + } member { name: "DepthwiseConv2D" mtype: "" @@ -240,6 +244,10 @@ tf_module { name: "LeakyReLU" mtype: "" } + member { + name: "LinearModel" + mtype: "" + } member { name: "LocallyConnected1D" mtype: "" diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.losses.-binary-crossentropy.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.losses.-binary-crossentropy.pbtxt new file mode 100644 index 00000000000..2f7da93f6f4 --- /dev/null +++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.losses.-binary-crossentropy.pbtxt @@ -0,0 +1,22 @@ +path: "tensorflow.keras.losses.BinaryCrossentropy" +tf_class { + is_instance: "" + is_instance: "" + is_instance: "" + member_method { + name: "__init__" + argspec: "args=[\'self\', \'from_logits\', \'label_smoothing\', \'reduction\', \'name\'], varargs=None, keywords=None, defaults=[\'False\', \'0\', \'sum_over_batch_size\', \'None\'], " + } + member_method { + name: "call" + argspec: "args=[\'self\', \'y_true\', \'y_pred\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "from_config" + argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_config" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } +} diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.losses.-categorical-crossentropy.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.losses.-categorical-crossentropy.pbtxt new file mode 100644 index 00000000000..b3a7cd80973 --- /dev/null +++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.losses.-categorical-crossentropy.pbtxt @@ -0,0 +1,22 @@ +path: "tensorflow.keras.losses.CategoricalCrossentropy" +tf_class { + is_instance: "" + is_instance: "" + is_instance: "" + member_method { + name: "__init__" + argspec: "args=[\'self\', \'from_logits\', \'label_smoothing\', \'reduction\', \'name\'], varargs=None, keywords=None, defaults=[\'False\', \'0\', \'sum_over_batch_size\', \'None\'], " + } + member_method { + name: "call" + argspec: "args=[\'self\', \'y_true\', \'y_pred\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "from_config" + argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_config" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } +} diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.losses.-mean-absolute-error.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.losses.-mean-absolute-error.pbtxt new file mode 100644 index 00000000000..712bb2ecd35 --- /dev/null +++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.losses.-mean-absolute-error.pbtxt @@ -0,0 +1,22 @@ +path: "tensorflow.keras.losses.MeanAbsoluteError" +tf_class { + is_instance: "" + is_instance: "" + is_instance: "" + member_method { + name: "__init__" + argspec: "args=[\'self\', \'reduction\', \'name\'], varargs=None, keywords=None, defaults=[\'sum_over_batch_size\', \'None\'], " + } + member_method { + name: "call" + argspec: "args=[\'self\', \'y_true\', \'y_pred\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "from_config" + argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_config" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } +} diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.losses.-mean-absolute-percentage-error.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.losses.-mean-absolute-percentage-error.pbtxt new file mode 100644 index 00000000000..7fe362da89b --- /dev/null +++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.losses.-mean-absolute-percentage-error.pbtxt @@ -0,0 +1,22 @@ +path: "tensorflow.keras.losses.MeanAbsolutePercentageError" +tf_class { + is_instance: "" + is_instance: "" + is_instance: "" + member_method { + name: "__init__" + argspec: "args=[\'self\', \'reduction\', \'name\'], varargs=None, keywords=None, defaults=[\'sum_over_batch_size\', \'None\'], " + } + member_method { + name: "call" + argspec: "args=[\'self\', \'y_true\', \'y_pred\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "from_config" + argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_config" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } +} diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.losses.-mean-squared-error.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.losses.-mean-squared-error.pbtxt new file mode 100644 index 00000000000..a5718533500 --- /dev/null +++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.losses.-mean-squared-error.pbtxt @@ -0,0 +1,22 @@ +path: "tensorflow.keras.losses.MeanSquaredError" +tf_class { + is_instance: "" + is_instance: "" + is_instance: "" + member_method { + name: "__init__" + argspec: "args=[\'self\', \'reduction\', \'name\'], varargs=None, keywords=None, defaults=[\'sum_over_batch_size\', \'None\'], " + } + member_method { + name: "call" + argspec: "args=[\'self\', \'y_true\', \'y_pred\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "from_config" + argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_config" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } +} diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.losses.-mean-squared-logarithmic-error.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.losses.-mean-squared-logarithmic-error.pbtxt new file mode 100644 index 00000000000..200006db355 --- /dev/null +++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.losses.-mean-squared-logarithmic-error.pbtxt @@ -0,0 +1,22 @@ +path: "tensorflow.keras.losses.MeanSquaredLogarithmicError" +tf_class { + is_instance: "" + is_instance: "" + is_instance: "" + member_method { + name: "__init__" + argspec: "args=[\'self\', \'reduction\', \'name\'], varargs=None, keywords=None, defaults=[\'sum_over_batch_size\', \'None\'], " + } + member_method { + name: "call" + argspec: "args=[\'self\', \'y_true\', \'y_pred\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "from_config" + argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_config" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } +} diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.losses.-reduction.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.losses.-reduction.pbtxt new file mode 100644 index 00000000000..f20ed26e2ea --- /dev/null +++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.losses.-reduction.pbtxt @@ -0,0 +1,28 @@ +path: "tensorflow.keras.losses.Reduction" +tf_class { + is_instance: "" + is_instance: "" + member { + name: "NONE" + mtype: "" + } + member { + name: "SUM" + mtype: "" + } + member { + name: "SUM_OVER_BATCH_SIZE" + mtype: "" + } + member_method { + name: "__init__" + } + member_method { + name: "all" + argspec: "args=[\'cls\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "validate" + argspec: "args=[\'cls\', \'key\'], varargs=None, keywords=None, defaults=None" + } +} diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.losses.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.losses.pbtxt index eca6b915388..c198096d252 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.keras.losses.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.losses.pbtxt @@ -1,5 +1,33 @@ path: "tensorflow.keras.losses" tf_module { + member { + name: "BinaryCrossentropy" + mtype: "" + } + member { + name: "CategoricalCrossentropy" + mtype: "" + } + member { + name: "MeanAbsoluteError" + mtype: "" + } + member { + name: "MeanAbsolutePercentageError" + mtype: "" + } + member { + name: "MeanSquaredError" + mtype: "" + } + member { + name: "MeanSquaredLogarithmicError" + mtype: "" + } + member { + name: "Reduction" + mtype: "" + } member_method { name: "KLD" argspec: "args=[\'y_true\', \'y_pred\'], varargs=None, keywords=None, defaults=None" @@ -22,11 +50,11 @@ tf_module { } member_method { name: "binary_crossentropy" - argspec: "args=[\'y_true\', \'y_pred\'], varargs=None, keywords=None, defaults=None" + argspec: "args=[\'y_true\', \'y_pred\', \'from_logits\'], varargs=None, keywords=None, defaults=[\'False\'], " } member_method { name: "categorical_crossentropy" - argspec: "args=[\'y_true\', \'y_pred\'], varargs=None, keywords=None, defaults=None" + argspec: "args=[\'y_true\', \'y_pred\', \'from_logits\'], varargs=None, keywords=None, defaults=[\'False\'], " } member_method { name: "categorical_hinge" @@ -106,7 +134,7 @@ tf_module { } member_method { name: "sparse_categorical_crossentropy" - argspec: "args=[\'y_true\', \'y_pred\'], varargs=None, keywords=None, defaults=None" + argspec: "args=[\'y_true\', \'y_pred\', \'from_logits\'], varargs=None, keywords=None, defaults=[\'False\'], " } member_method { name: "squared_hinge" diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-accuracy.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-accuracy.pbtxt new file mode 100644 index 00000000000..2db07df5235 --- /dev/null +++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-accuracy.pbtxt @@ -0,0 +1,194 @@ +path: "tensorflow.keras.metrics.Accuracy" +tf_class { + is_instance: "" + is_instance: "" + is_instance: "" + is_instance: "" + is_instance: "" + is_instance: "" + is_instance: "" + member { + name: "activity_regularizer" + mtype: "" + } + member { + name: "dtype" + mtype: "" + } + member { + name: "inbound_nodes" + mtype: "" + } + member { + name: "input" + mtype: "" + } + member { + name: "input_mask" + mtype: "" + } + member { + name: "input_shape" + mtype: "" + } + member { + name: "losses" + mtype: "" + } + member { + name: "name" + mtype: "" + } + member { + name: "non_trainable_variables" + mtype: "" + } + member { + name: "non_trainable_weights" + mtype: "" + } + member { + name: "outbound_nodes" + mtype: "" + } + member { + name: "output" + mtype: "" + } + member { + name: "output_mask" + mtype: "" + } + member { + name: "output_shape" + mtype: "" + } + member { + name: "trainable_variables" + mtype: "" + } + member { + name: "trainable_weights" + mtype: "" + } + member { + name: "updates" + mtype: "" + } + member { + name: "variables" + mtype: "" + } + member { + name: "weights" + mtype: "" + } + member_method { + name: "__init__" + argspec: "args=[\'self\', \'name\', \'dtype\'], varargs=None, keywords=None, defaults=[\'accuracy\', \'None\'], " + } + member_method { + name: "add_loss" + argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } + member_method { + name: "add_update" + argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "add_variable" + argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None" + } + member_method { + name: "add_weight" + argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\'], " + } + member_method { + name: "apply" + argspec: "args=[\'self\', \'inputs\'], varargs=args, keywords=kwargs, defaults=None" + } + member_method { + name: "build" + argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "call" + argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=kwargs, defaults=None" + } + member_method { + name: "compute_mask" + argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "compute_output_shape" + argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "count_params" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "from_config" + argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_config" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_input_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_input_mask_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_input_shape_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_losses_for" + argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_output_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_output_mask_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_output_shape_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_updates_for" + argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_weights" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "reset_states" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "result" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "set_weights" + argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "update_state" + argspec: "args=[\'self\', \'y_true\', \'y_pred\', \'sample_weight\'], varargs=None, keywords=None, defaults=[\'None\'], " + } +} diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-binary-accuracy.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-binary-accuracy.pbtxt new file mode 100644 index 00000000000..904ad3a21a0 --- /dev/null +++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-binary-accuracy.pbtxt @@ -0,0 +1,194 @@ +path: "tensorflow.keras.metrics.BinaryAccuracy" +tf_class { + is_instance: "" + is_instance: "" + is_instance: "" + is_instance: "" + is_instance: "" + is_instance: "" + is_instance: "" + member { + name: "activity_regularizer" + mtype: "" + } + member { + name: "dtype" + mtype: "" + } + member { + name: "inbound_nodes" + mtype: "" + } + member { + name: "input" + mtype: "" + } + member { + name: "input_mask" + mtype: "" + } + member { + name: "input_shape" + mtype: "" + } + member { + name: "losses" + mtype: "" + } + member { + name: "name" + mtype: "" + } + member { + name: "non_trainable_variables" + mtype: "" + } + member { + name: "non_trainable_weights" + mtype: "" + } + member { + name: "outbound_nodes" + mtype: "" + } + member { + name: "output" + mtype: "" + } + member { + name: "output_mask" + mtype: "" + } + member { + name: "output_shape" + mtype: "" + } + member { + name: "trainable_variables" + mtype: "" + } + member { + name: "trainable_weights" + mtype: "" + } + member { + name: "updates" + mtype: "" + } + member { + name: "variables" + mtype: "" + } + member { + name: "weights" + mtype: "" + } + member_method { + name: "__init__" + argspec: "args=[\'self\', \'name\', \'dtype\', \'threshold\'], varargs=None, keywords=None, defaults=[\'binary_accuracy\', \'None\', \'0.5\'], " + } + member_method { + name: "add_loss" + argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } + member_method { + name: "add_update" + argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "add_variable" + argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None" + } + member_method { + name: "add_weight" + argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\'], " + } + member_method { + name: "apply" + argspec: "args=[\'self\', \'inputs\'], varargs=args, keywords=kwargs, defaults=None" + } + member_method { + name: "build" + argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "call" + argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=kwargs, defaults=None" + } + member_method { + name: "compute_mask" + argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "compute_output_shape" + argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "count_params" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "from_config" + argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_config" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_input_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_input_mask_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_input_shape_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_losses_for" + argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_output_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_output_mask_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_output_shape_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_updates_for" + argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_weights" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "reset_states" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "result" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "set_weights" + argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "update_state" + argspec: "args=[\'self\', \'y_true\', \'y_pred\', \'sample_weight\'], varargs=None, keywords=None, defaults=[\'None\'], " + } +} diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-categorical-accuracy.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-categorical-accuracy.pbtxt new file mode 100644 index 00000000000..17b74924fab --- /dev/null +++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-categorical-accuracy.pbtxt @@ -0,0 +1,194 @@ +path: "tensorflow.keras.metrics.CategoricalAccuracy" +tf_class { + is_instance: "" + is_instance: "" + is_instance: "" + is_instance: "" + is_instance: "" + is_instance: "" + is_instance: "" + member { + name: "activity_regularizer" + mtype: "" + } + member { + name: "dtype" + mtype: "" + } + member { + name: "inbound_nodes" + mtype: "" + } + member { + name: "input" + mtype: "" + } + member { + name: "input_mask" + mtype: "" + } + member { + name: "input_shape" + mtype: "" + } + member { + name: "losses" + mtype: "" + } + member { + name: "name" + mtype: "" + } + member { + name: "non_trainable_variables" + mtype: "" + } + member { + name: "non_trainable_weights" + mtype: "" + } + member { + name: "outbound_nodes" + mtype: "" + } + member { + name: "output" + mtype: "" + } + member { + name: "output_mask" + mtype: "" + } + member { + name: "output_shape" + mtype: "" + } + member { + name: "trainable_variables" + mtype: "" + } + member { + name: "trainable_weights" + mtype: "" + } + member { + name: "updates" + mtype: "" + } + member { + name: "variables" + mtype: "" + } + member { + name: "weights" + mtype: "" + } + member_method { + name: "__init__" + argspec: "args=[\'self\', \'name\', \'dtype\'], varargs=None, keywords=None, defaults=[\'categorical_accuracy\', \'None\'], " + } + member_method { + name: "add_loss" + argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } + member_method { + name: "add_update" + argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "add_variable" + argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None" + } + member_method { + name: "add_weight" + argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\'], " + } + member_method { + name: "apply" + argspec: "args=[\'self\', \'inputs\'], varargs=args, keywords=kwargs, defaults=None" + } + member_method { + name: "build" + argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "call" + argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=kwargs, defaults=None" + } + member_method { + name: "compute_mask" + argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "compute_output_shape" + argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "count_params" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "from_config" + argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_config" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_input_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_input_mask_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_input_shape_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_losses_for" + argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_output_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_output_mask_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_output_shape_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_updates_for" + argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_weights" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "reset_states" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "result" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "set_weights" + argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "update_state" + argspec: "args=[\'self\', \'y_true\', \'y_pred\', \'sample_weight\'], varargs=None, keywords=None, defaults=[\'None\'], " + } +} diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-false-negatives.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-false-negatives.pbtxt new file mode 100644 index 00000000000..49f577e1367 --- /dev/null +++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-false-negatives.pbtxt @@ -0,0 +1,193 @@ +path: "tensorflow.keras.metrics.FalseNegatives" +tf_class { + is_instance: "" + is_instance: "" + is_instance: "" + is_instance: "" + is_instance: "" + is_instance: "" + member { + name: "activity_regularizer" + mtype: "" + } + member { + name: "dtype" + mtype: "" + } + member { + name: "inbound_nodes" + mtype: "" + } + member { + name: "input" + mtype: "" + } + member { + name: "input_mask" + mtype: "" + } + member { + name: "input_shape" + mtype: "" + } + member { + name: "losses" + mtype: "" + } + member { + name: "name" + mtype: "" + } + member { + name: "non_trainable_variables" + mtype: "" + } + member { + name: "non_trainable_weights" + mtype: "" + } + member { + name: "outbound_nodes" + mtype: "" + } + member { + name: "output" + mtype: "" + } + member { + name: "output_mask" + mtype: "" + } + member { + name: "output_shape" + mtype: "" + } + member { + name: "trainable_variables" + mtype: "" + } + member { + name: "trainable_weights" + mtype: "" + } + member { + name: "updates" + mtype: "" + } + member { + name: "variables" + mtype: "" + } + member { + name: "weights" + mtype: "" + } + member_method { + name: "__init__" + argspec: "args=[\'self\', \'thresholds\', \'name\', \'dtype\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], " + } + member_method { + name: "add_loss" + argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } + member_method { + name: "add_update" + argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "add_variable" + argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None" + } + member_method { + name: "add_weight" + argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\'], " + } + member_method { + name: "apply" + argspec: "args=[\'self\', \'inputs\'], varargs=args, keywords=kwargs, defaults=None" + } + member_method { + name: "build" + argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "call" + argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=kwargs, defaults=None" + } + member_method { + name: "compute_mask" + argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "compute_output_shape" + argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "count_params" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "from_config" + argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_config" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_input_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_input_mask_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_input_shape_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_losses_for" + argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_output_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_output_mask_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_output_shape_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_updates_for" + argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_weights" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "reset_states" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "result" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "set_weights" + argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "update_state" + argspec: "args=[\'self\', \'y_true\', \'y_pred\', \'sample_weight\'], varargs=None, keywords=None, defaults=[\'None\'], " + } +} diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-false-positives.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-false-positives.pbtxt new file mode 100644 index 00000000000..e8baf858669 --- /dev/null +++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-false-positives.pbtxt @@ -0,0 +1,193 @@ +path: "tensorflow.keras.metrics.FalsePositives" +tf_class { + is_instance: "" + is_instance: "" + is_instance: "" + is_instance: "" + is_instance: "" + is_instance: "" + member { + name: "activity_regularizer" + mtype: "" + } + member { + name: "dtype" + mtype: "" + } + member { + name: "inbound_nodes" + mtype: "" + } + member { + name: "input" + mtype: "" + } + member { + name: "input_mask" + mtype: "" + } + member { + name: "input_shape" + mtype: "" + } + member { + name: "losses" + mtype: "" + } + member { + name: "name" + mtype: "" + } + member { + name: "non_trainable_variables" + mtype: "" + } + member { + name: "non_trainable_weights" + mtype: "" + } + member { + name: "outbound_nodes" + mtype: "" + } + member { + name: "output" + mtype: "" + } + member { + name: "output_mask" + mtype: "" + } + member { + name: "output_shape" + mtype: "" + } + member { + name: "trainable_variables" + mtype: "" + } + member { + name: "trainable_weights" + mtype: "" + } + member { + name: "updates" + mtype: "" + } + member { + name: "variables" + mtype: "" + } + member { + name: "weights" + mtype: "" + } + member_method { + name: "__init__" + argspec: "args=[\'self\', \'thresholds\', \'name\', \'dtype\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], " + } + member_method { + name: "add_loss" + argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } + member_method { + name: "add_update" + argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "add_variable" + argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None" + } + member_method { + name: "add_weight" + argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\'], " + } + member_method { + name: "apply" + argspec: "args=[\'self\', \'inputs\'], varargs=args, keywords=kwargs, defaults=None" + } + member_method { + name: "build" + argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "call" + argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=kwargs, defaults=None" + } + member_method { + name: "compute_mask" + argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "compute_output_shape" + argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "count_params" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "from_config" + argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_config" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_input_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_input_mask_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_input_shape_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_losses_for" + argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_output_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_output_mask_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_output_shape_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_updates_for" + argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_weights" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "reset_states" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "result" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "set_weights" + argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "update_state" + argspec: "args=[\'self\', \'y_true\', \'y_pred\', \'sample_weight\'], varargs=None, keywords=None, defaults=[\'None\'], " + } +} diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-mean.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-mean.pbtxt new file mode 100644 index 00000000000..40fe64bbd2c --- /dev/null +++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-mean.pbtxt @@ -0,0 +1,192 @@ +path: "tensorflow.keras.metrics.Mean" +tf_class { + is_instance: "" + is_instance: "" + is_instance: "" + is_instance: "" + is_instance: "" + member { + name: "activity_regularizer" + mtype: "" + } + member { + name: "dtype" + mtype: "" + } + member { + name: "inbound_nodes" + mtype: "" + } + member { + name: "input" + mtype: "" + } + member { + name: "input_mask" + mtype: "" + } + member { + name: "input_shape" + mtype: "" + } + member { + name: "losses" + mtype: "" + } + member { + name: "name" + mtype: "" + } + member { + name: "non_trainable_variables" + mtype: "" + } + member { + name: "non_trainable_weights" + mtype: "" + } + member { + name: "outbound_nodes" + mtype: "" + } + member { + name: "output" + mtype: "" + } + member { + name: "output_mask" + mtype: "" + } + member { + name: "output_shape" + mtype: "" + } + member { + name: "trainable_variables" + mtype: "" + } + member { + name: "trainable_weights" + mtype: "" + } + member { + name: "updates" + mtype: "" + } + member { + name: "variables" + mtype: "" + } + member { + name: "weights" + mtype: "" + } + member_method { + name: "__init__" + argspec: "args=[\'self\', \'name\', \'dtype\'], varargs=None, keywords=None, defaults=[\'mean\', \'None\'], " + } + member_method { + name: "add_loss" + argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } + member_method { + name: "add_update" + argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "add_variable" + argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None" + } + member_method { + name: "add_weight" + argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\'], " + } + member_method { + name: "apply" + argspec: "args=[\'self\', \'inputs\'], varargs=args, keywords=kwargs, defaults=None" + } + member_method { + name: "build" + argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "call" + argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=kwargs, defaults=None" + } + member_method { + name: "compute_mask" + argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "compute_output_shape" + argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "count_params" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "from_config" + argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_config" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_input_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_input_mask_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_input_shape_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_losses_for" + argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_output_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_output_mask_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_output_shape_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_updates_for" + argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_weights" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "reset_states" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "result" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "set_weights" + argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "update_state" + argspec: "args=[\'self\', \'values\', \'sample_weight\'], varargs=None, keywords=None, defaults=[\'None\'], " + } +} diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-precision.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-precision.pbtxt new file mode 100644 index 00000000000..ae6a85026da --- /dev/null +++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-precision.pbtxt @@ -0,0 +1,192 @@ +path: "tensorflow.keras.metrics.Precision" +tf_class { + is_instance: "" + is_instance: "" + is_instance: "" + is_instance: "" + is_instance: "" + member { + name: "activity_regularizer" + mtype: "" + } + member { + name: "dtype" + mtype: "" + } + member { + name: "inbound_nodes" + mtype: "" + } + member { + name: "input" + mtype: "" + } + member { + name: "input_mask" + mtype: "" + } + member { + name: "input_shape" + mtype: "" + } + member { + name: "losses" + mtype: "" + } + member { + name: "name" + mtype: "" + } + member { + name: "non_trainable_variables" + mtype: "" + } + member { + name: "non_trainable_weights" + mtype: "" + } + member { + name: "outbound_nodes" + mtype: "" + } + member { + name: "output" + mtype: "" + } + member { + name: "output_mask" + mtype: "" + } + member { + name: "output_shape" + mtype: "" + } + member { + name: "trainable_variables" + mtype: "" + } + member { + name: "trainable_weights" + mtype: "" + } + member { + name: "updates" + mtype: "" + } + member { + name: "variables" + mtype: "" + } + member { + name: "weights" + mtype: "" + } + member_method { + name: "__init__" + argspec: "args=[\'self\', \'thresholds\', \'name\', \'dtype\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], " + } + member_method { + name: "add_loss" + argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } + member_method { + name: "add_update" + argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "add_variable" + argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None" + } + member_method { + name: "add_weight" + argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\'], " + } + member_method { + name: "apply" + argspec: "args=[\'self\', \'inputs\'], varargs=args, keywords=kwargs, defaults=None" + } + member_method { + name: "build" + argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "call" + argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=kwargs, defaults=None" + } + member_method { + name: "compute_mask" + argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "compute_output_shape" + argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "count_params" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "from_config" + argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_config" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_input_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_input_mask_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_input_shape_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_losses_for" + argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_output_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_output_mask_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_output_shape_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_updates_for" + argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_weights" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "reset_states" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "result" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "set_weights" + argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "update_state" + argspec: "args=[\'self\', \'y_true\', \'y_pred\', \'sample_weight\'], varargs=None, keywords=None, defaults=[\'None\'], " + } +} diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-recall.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-recall.pbtxt new file mode 100644 index 00000000000..31068a51d51 --- /dev/null +++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-recall.pbtxt @@ -0,0 +1,192 @@ +path: "tensorflow.keras.metrics.Recall" +tf_class { + is_instance: "" + is_instance: "" + is_instance: "" + is_instance: "" + is_instance: "" + member { + name: "activity_regularizer" + mtype: "" + } + member { + name: "dtype" + mtype: "" + } + member { + name: "inbound_nodes" + mtype: "" + } + member { + name: "input" + mtype: "" + } + member { + name: "input_mask" + mtype: "" + } + member { + name: "input_shape" + mtype: "" + } + member { + name: "losses" + mtype: "" + } + member { + name: "name" + mtype: "" + } + member { + name: "non_trainable_variables" + mtype: "" + } + member { + name: "non_trainable_weights" + mtype: "" + } + member { + name: "outbound_nodes" + mtype: "" + } + member { + name: "output" + mtype: "" + } + member { + name: "output_mask" + mtype: "" + } + member { + name: "output_shape" + mtype: "" + } + member { + name: "trainable_variables" + mtype: "" + } + member { + name: "trainable_weights" + mtype: "" + } + member { + name: "updates" + mtype: "" + } + member { + name: "variables" + mtype: "" + } + member { + name: "weights" + mtype: "" + } + member_method { + name: "__init__" + argspec: "args=[\'self\', \'thresholds\', \'name\', \'dtype\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], " + } + member_method { + name: "add_loss" + argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } + member_method { + name: "add_update" + argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "add_variable" + argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None" + } + member_method { + name: "add_weight" + argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\'], " + } + member_method { + name: "apply" + argspec: "args=[\'self\', \'inputs\'], varargs=args, keywords=kwargs, defaults=None" + } + member_method { + name: "build" + argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "call" + argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=kwargs, defaults=None" + } + member_method { + name: "compute_mask" + argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "compute_output_shape" + argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "count_params" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "from_config" + argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_config" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_input_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_input_mask_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_input_shape_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_losses_for" + argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_output_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_output_mask_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_output_shape_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_updates_for" + argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_weights" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "reset_states" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "result" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "set_weights" + argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "update_state" + argspec: "args=[\'self\', \'y_true\', \'y_pred\', \'sample_weight\'], varargs=None, keywords=None, defaults=[\'None\'], " + } +} diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-sparse-categorical-accuracy.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-sparse-categorical-accuracy.pbtxt new file mode 100644 index 00000000000..0c17452292a --- /dev/null +++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-sparse-categorical-accuracy.pbtxt @@ -0,0 +1,194 @@ +path: "tensorflow.keras.metrics.SparseCategoricalAccuracy" +tf_class { + is_instance: "" + is_instance: "" + is_instance: "" + is_instance: "" + is_instance: "" + is_instance: "" + is_instance: "" + member { + name: "activity_regularizer" + mtype: "" + } + member { + name: "dtype" + mtype: "" + } + member { + name: "inbound_nodes" + mtype: "" + } + member { + name: "input" + mtype: "" + } + member { + name: "input_mask" + mtype: "" + } + member { + name: "input_shape" + mtype: "" + } + member { + name: "losses" + mtype: "" + } + member { + name: "name" + mtype: "" + } + member { + name: "non_trainable_variables" + mtype: "" + } + member { + name: "non_trainable_weights" + mtype: "" + } + member { + name: "outbound_nodes" + mtype: "" + } + member { + name: "output" + mtype: "" + } + member { + name: "output_mask" + mtype: "" + } + member { + name: "output_shape" + mtype: "" + } + member { + name: "trainable_variables" + mtype: "" + } + member { + name: "trainable_weights" + mtype: "" + } + member { + name: "updates" + mtype: "" + } + member { + name: "variables" + mtype: "" + } + member { + name: "weights" + mtype: "" + } + member_method { + name: "__init__" + argspec: "args=[\'self\', \'name\', \'dtype\'], varargs=None, keywords=None, defaults=[\'sparse_categorical_accuracy\', \'None\'], " + } + member_method { + name: "add_loss" + argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } + member_method { + name: "add_update" + argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "add_variable" + argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None" + } + member_method { + name: "add_weight" + argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\'], " + } + member_method { + name: "apply" + argspec: "args=[\'self\', \'inputs\'], varargs=args, keywords=kwargs, defaults=None" + } + member_method { + name: "build" + argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "call" + argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=kwargs, defaults=None" + } + member_method { + name: "compute_mask" + argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "compute_output_shape" + argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "count_params" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "from_config" + argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_config" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_input_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_input_mask_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_input_shape_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_losses_for" + argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_output_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_output_mask_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_output_shape_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_updates_for" + argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_weights" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "reset_states" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "result" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "set_weights" + argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "update_state" + argspec: "args=[\'self\', \'y_true\', \'y_pred\', \'sample_weight\'], varargs=None, keywords=None, defaults=[\'None\'], " + } +} diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-true-negatives.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-true-negatives.pbtxt new file mode 100644 index 00000000000..1b5eb8d0de5 --- /dev/null +++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-true-negatives.pbtxt @@ -0,0 +1,193 @@ +path: "tensorflow.keras.metrics.TrueNegatives" +tf_class { + is_instance: "" + is_instance: "" + is_instance: "" + is_instance: "" + is_instance: "" + is_instance: "" + member { + name: "activity_regularizer" + mtype: "" + } + member { + name: "dtype" + mtype: "" + } + member { + name: "inbound_nodes" + mtype: "" + } + member { + name: "input" + mtype: "" + } + member { + name: "input_mask" + mtype: "" + } + member { + name: "input_shape" + mtype: "" + } + member { + name: "losses" + mtype: "" + } + member { + name: "name" + mtype: "" + } + member { + name: "non_trainable_variables" + mtype: "" + } + member { + name: "non_trainable_weights" + mtype: "" + } + member { + name: "outbound_nodes" + mtype: "" + } + member { + name: "output" + mtype: "" + } + member { + name: "output_mask" + mtype: "" + } + member { + name: "output_shape" + mtype: "" + } + member { + name: "trainable_variables" + mtype: "" + } + member { + name: "trainable_weights" + mtype: "" + } + member { + name: "updates" + mtype: "" + } + member { + name: "variables" + mtype: "" + } + member { + name: "weights" + mtype: "" + } + member_method { + name: "__init__" + argspec: "args=[\'self\', \'thresholds\', \'name\', \'dtype\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], " + } + member_method { + name: "add_loss" + argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } + member_method { + name: "add_update" + argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "add_variable" + argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None" + } + member_method { + name: "add_weight" + argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\'], " + } + member_method { + name: "apply" + argspec: "args=[\'self\', \'inputs\'], varargs=args, keywords=kwargs, defaults=None" + } + member_method { + name: "build" + argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "call" + argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=kwargs, defaults=None" + } + member_method { + name: "compute_mask" + argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "compute_output_shape" + argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "count_params" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "from_config" + argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_config" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_input_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_input_mask_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_input_shape_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_losses_for" + argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_output_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_output_mask_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_output_shape_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_updates_for" + argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_weights" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "reset_states" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "result" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "set_weights" + argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "update_state" + argspec: "args=[\'self\', \'y_true\', \'y_pred\', \'sample_weight\'], varargs=None, keywords=None, defaults=[\'None\'], " + } +} diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-true-positives.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-true-positives.pbtxt new file mode 100644 index 00000000000..5b9c470e32d --- /dev/null +++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-true-positives.pbtxt @@ -0,0 +1,193 @@ +path: "tensorflow.keras.metrics.TruePositives" +tf_class { + is_instance: "" + is_instance: "" + is_instance: "" + is_instance: "" + is_instance: "" + is_instance: "" + member { + name: "activity_regularizer" + mtype: "" + } + member { + name: "dtype" + mtype: "" + } + member { + name: "inbound_nodes" + mtype: "" + } + member { + name: "input" + mtype: "" + } + member { + name: "input_mask" + mtype: "" + } + member { + name: "input_shape" + mtype: "" + } + member { + name: "losses" + mtype: "" + } + member { + name: "name" + mtype: "" + } + member { + name: "non_trainable_variables" + mtype: "" + } + member { + name: "non_trainable_weights" + mtype: "" + } + member { + name: "outbound_nodes" + mtype: "" + } + member { + name: "output" + mtype: "" + } + member { + name: "output_mask" + mtype: "" + } + member { + name: "output_shape" + mtype: "" + } + member { + name: "trainable_variables" + mtype: "" + } + member { + name: "trainable_weights" + mtype: "" + } + member { + name: "updates" + mtype: "" + } + member { + name: "variables" + mtype: "" + } + member { + name: "weights" + mtype: "" + } + member_method { + name: "__init__" + argspec: "args=[\'self\', \'thresholds\', \'name\', \'dtype\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], " + } + member_method { + name: "add_loss" + argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } + member_method { + name: "add_update" + argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "add_variable" + argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None" + } + member_method { + name: "add_weight" + argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\'], " + } + member_method { + name: "apply" + argspec: "args=[\'self\', \'inputs\'], varargs=args, keywords=kwargs, defaults=None" + } + member_method { + name: "build" + argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "call" + argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=kwargs, defaults=None" + } + member_method { + name: "compute_mask" + argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "compute_output_shape" + argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "count_params" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "from_config" + argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_config" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_input_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_input_mask_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_input_shape_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_losses_for" + argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_output_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_output_mask_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_output_shape_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_updates_for" + argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_weights" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "reset_states" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "result" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "set_weights" + argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "update_state" + argspec: "args=[\'self\', \'y_true\', \'y_pred\', \'sample_weight\'], varargs=None, keywords=None, defaults=[\'None\'], " + } +} diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.pbtxt index a296e131586..8cab17edc59 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.pbtxt @@ -1,5 +1,49 @@ path: "tensorflow.keras.metrics" tf_module { + member { + name: "Accuracy" + mtype: "" + } + member { + name: "BinaryAccuracy" + mtype: "" + } + member { + name: "CategoricalAccuracy" + mtype: "" + } + member { + name: "FalseNegatives" + mtype: "" + } + member { + name: "FalsePositives" + mtype: "" + } + member { + name: "Mean" + mtype: "" + } + member { + name: "Precision" + mtype: "" + } + member { + name: "Recall" + mtype: "" + } + member { + name: "SparseCategoricalAccuracy" + mtype: "" + } + member { + name: "TrueNegatives" + mtype: "" + } + member { + name: "TruePositives" + mtype: "" + } member_method { name: "KLD" argspec: "args=[\'y_true\', \'y_pred\'], varargs=None, keywords=None, defaults=None" @@ -26,7 +70,7 @@ tf_module { } member_method { name: "binary_crossentropy" - argspec: "args=[\'y_true\', \'y_pred\'], varargs=None, keywords=None, defaults=None" + argspec: "args=[\'y_true\', \'y_pred\', \'from_logits\'], varargs=None, keywords=None, defaults=[\'False\'], " } member_method { name: "categorical_accuracy" @@ -34,7 +78,7 @@ tf_module { } member_method { name: "categorical_crossentropy" - argspec: "args=[\'y_true\', \'y_pred\'], varargs=None, keywords=None, defaults=None" + argspec: "args=[\'y_true\', \'y_pred\', \'from_logits\'], varargs=None, keywords=None, defaults=[\'False\'], " } member_method { name: "cosine" @@ -110,7 +154,7 @@ tf_module { } member_method { name: "sparse_categorical_crossentropy" - argspec: "args=[\'y_true\', \'y_pred\'], varargs=None, keywords=None, defaults=None" + argspec: "args=[\'y_true\', \'y_pred\', \'from_logits\'], varargs=None, keywords=None, defaults=[\'False\'], " } member_method { name: "sparse_top_k_categorical_accuracy" diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.models.-model.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.models.-model.pbtxt index ccff809f2b2..c58c7bef22d 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.keras.models.-model.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.models.-model.pbtxt @@ -41,6 +41,14 @@ tf_class { name: "losses" mtype: "" } + member { + name: "metrics" + mtype: "" + } + member { + name: "metrics_names" + mtype: "" + } member { name: "name" mtype: "" @@ -69,6 +77,10 @@ tf_class { name: "output_shape" mtype: "" } + member { + name: "run_eagerly" + mtype: "" + } member { name: "state_updates" mtype: "" @@ -105,13 +117,17 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None" } member_method { name: "add_weight" @@ -225,6 +241,10 @@ tf_class { name: "predict_on_batch" argspec: "args=[\'self\', \'x\'], varargs=None, keywords=None, defaults=None" } + member_method { + name: "reset_metrics" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } member_method { name: "reset_states" argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" @@ -247,7 +267,7 @@ tf_class { } member_method { name: "test_on_batch" - argspec: "args=[\'self\', \'x\', \'y\', \'sample_weight\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + argspec: "args=[\'self\', \'x\', \'y\', \'sample_weight\', \'reset_metrics\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'True\'], " } member_method { name: "to_json" @@ -259,6 +279,6 @@ tf_class { } member_method { name: "train_on_batch" - argspec: "args=[\'self\', \'x\', \'y\', \'sample_weight\', \'class_weight\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], " + argspec: "args=[\'self\', \'x\', \'y\', \'sample_weight\', \'class_weight\', \'reset_metrics\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\'], " } } diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.models.-sequential.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.models.-sequential.pbtxt index b0fc7f97f1d..473a1c16fb1 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.keras.models.-sequential.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.models.-sequential.pbtxt @@ -42,6 +42,14 @@ tf_class { name: "losses" mtype: "" } + member { + name: "metrics" + mtype: "" + } + member { + name: "metrics_names" + mtype: "" + } member { name: "name" mtype: "" @@ -70,6 +78,10 @@ tf_class { name: "output_shape" mtype: "" } + member { + name: "run_eagerly" + mtype: "" + } member { name: "state_updates" mtype: "" @@ -110,13 +122,17 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None" } member_method { name: "add_weight" @@ -242,6 +258,10 @@ tf_class { name: "predict_proba" argspec: "args=[\'self\', \'x\', \'batch_size\', \'verbose\'], varargs=None, keywords=None, defaults=[\'32\', \'0\'], " } + member_method { + name: "reset_metrics" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } member_method { name: "reset_states" argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" @@ -264,7 +284,7 @@ tf_class { } member_method { name: "test_on_batch" - argspec: "args=[\'self\', \'x\', \'y\', \'sample_weight\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + argspec: "args=[\'self\', \'x\', \'y\', \'sample_weight\', \'reset_metrics\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'True\'], " } member_method { name: "to_json" @@ -276,6 +296,6 @@ tf_class { } member_method { name: "train_on_batch" - argspec: "args=[\'self\', \'x\', \'y\', \'sample_weight\', \'class_weight\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], " + argspec: "args=[\'self\', \'x\', \'y\', \'sample_weight\', \'class_weight\', \'reset_metrics\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\'], " } } diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.utils.-progbar.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.utils.-progbar.pbtxt index be4496e753f..8177cc71ed3 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.keras.utils.-progbar.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.utils.-progbar.pbtxt @@ -4,7 +4,7 @@ tf_class { is_instance: "" member_method { name: "__init__" - argspec: "args=[\'self\', \'target\', \'width\', \'verbose\', \'interval\', \'stateful_metrics\'], varargs=None, keywords=None, defaults=[\'30\', \'1\', \'0.05\', \'None\'], " + argspec: "args=[\'self\', \'target\', \'width\', \'verbose\', \'interval\', \'stateful_metrics\', \'unit_name\'], varargs=None, keywords=None, defaults=[\'30\', \'1\', \'0.05\', \'None\', \'step\'], " } member_method { name: "add" diff --git a/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-block-diag.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-block-diag.pbtxt index 973705dae2f..773c74e64d1 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-block-diag.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-block-diag.pbtxt @@ -79,6 +79,10 @@ tf_class { name: "batch_shape_tensor" argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'batch_shape_tensor\'], " } + member_method { + name: "cholesky" + argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'cholesky\'], " + } member_method { name: "determinant" argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'det\'], " diff --git a/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-circulant.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-circulant.pbtxt index de917706d55..533544d21f2 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-circulant.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-circulant.pbtxt @@ -96,6 +96,10 @@ tf_class { name: "block_shape_tensor" argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" } + member_method { + name: "cholesky" + argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'cholesky\'], " + } member_method { name: "convolution_kernel" argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'convolution_kernel\'], " diff --git a/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-circulant2-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-circulant2-d.pbtxt index c4e6a21c3ac..e3926eb6d47 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-circulant2-d.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-circulant2-d.pbtxt @@ -96,6 +96,10 @@ tf_class { name: "block_shape_tensor" argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" } + member_method { + name: "cholesky" + argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'cholesky\'], " + } member_method { name: "convolution_kernel" argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'convolution_kernel\'], " diff --git a/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-circulant3-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-circulant3-d.pbtxt index 2e085a8e289..ba209df7824 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-circulant3-d.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-circulant3-d.pbtxt @@ -96,6 +96,10 @@ tf_class { name: "block_shape_tensor" argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" } + member_method { + name: "cholesky" + argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'cholesky\'], " + } member_method { name: "convolution_kernel" argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'convolution_kernel\'], " diff --git a/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-composition.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-composition.pbtxt index 42d22bce42d..081fb0e08bc 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-composition.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-composition.pbtxt @@ -79,6 +79,10 @@ tf_class { name: "batch_shape_tensor" argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'batch_shape_tensor\'], " } + member_method { + name: "cholesky" + argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'cholesky\'], " + } member_method { name: "determinant" argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'det\'], " diff --git a/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-diag.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-diag.pbtxt index d6749fdcec6..2014a043016 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-diag.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-diag.pbtxt @@ -79,6 +79,10 @@ tf_class { name: "batch_shape_tensor" argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'batch_shape_tensor\'], " } + member_method { + name: "cholesky" + argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'cholesky\'], " + } member_method { name: "determinant" argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'det\'], " diff --git a/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-full-matrix.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-full-matrix.pbtxt index d9f363d1336..9a87ae96877 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-full-matrix.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-full-matrix.pbtxt @@ -75,6 +75,10 @@ tf_class { name: "batch_shape_tensor" argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'batch_shape_tensor\'], " } + member_method { + name: "cholesky" + argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'cholesky\'], " + } member_method { name: "determinant" argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'det\'], " diff --git a/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-identity.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-identity.pbtxt index aac7ee31ed6..33afb835ce1 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-identity.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-identity.pbtxt @@ -76,6 +76,10 @@ tf_class { name: "batch_shape_tensor" argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'batch_shape_tensor\'], " } + member_method { + name: "cholesky" + argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'cholesky\'], " + } member_method { name: "determinant" argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'det\'], " diff --git a/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-kronecker.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-kronecker.pbtxt index c11d3908293..a9078c8ab5c 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-kronecker.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-kronecker.pbtxt @@ -79,6 +79,10 @@ tf_class { name: "batch_shape_tensor" argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'batch_shape_tensor\'], " } + member_method { + name: "cholesky" + argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'cholesky\'], " + } member_method { name: "determinant" argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'det\'], " diff --git a/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-low-rank-update.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-low-rank-update.pbtxt index 3ee800269e6..4cfa3bb30d7 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-low-rank-update.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-low-rank-update.pbtxt @@ -99,6 +99,10 @@ tf_class { name: "batch_shape_tensor" argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'batch_shape_tensor\'], " } + member_method { + name: "cholesky" + argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'cholesky\'], " + } member_method { name: "determinant" argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'det\'], " diff --git a/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-lower-triangular.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-lower-triangular.pbtxt index 63a1bc2321e..a87649133fd 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-lower-triangular.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-lower-triangular.pbtxt @@ -75,6 +75,10 @@ tf_class { name: "batch_shape_tensor" argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'batch_shape_tensor\'], " } + member_method { + name: "cholesky" + argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'cholesky\'], " + } member_method { name: "determinant" argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'det\'], " diff --git a/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-scaled-identity.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-scaled-identity.pbtxt index e2c5a505a7d..32656467840 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-scaled-identity.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-scaled-identity.pbtxt @@ -80,6 +80,10 @@ tf_class { name: "batch_shape_tensor" argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'batch_shape_tensor\'], " } + member_method { + name: "cholesky" + argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'cholesky\'], " + } member_method { name: "determinant" argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'det\'], " diff --git a/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-zeros.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-zeros.pbtxt index a1b0e06b475..49d8890c894 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-zeros.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-zeros.pbtxt @@ -75,6 +75,10 @@ tf_class { name: "batch_shape_tensor" argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'batch_shape_tensor\'], " } + member_method { + name: "cholesky" + argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'cholesky\'], " + } member_method { name: "determinant" argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'det\'], " diff --git a/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator.pbtxt index 6d849dc040f..c89dc067b33 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator.pbtxt @@ -74,6 +74,10 @@ tf_class { name: "batch_shape_tensor" argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'batch_shape_tensor\'], " } + member_method { + name: "cholesky" + argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'cholesky\'], " + } member_method { name: "determinant" argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'det\'], " diff --git a/tensorflow/tools/api/golden/v2/tensorflow.linalg.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.linalg.pbtxt index 1a4098d121b..a3599bfa801 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.linalg.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.linalg.pbtxt @@ -118,7 +118,7 @@ tf_module { } member_method { name: "l2_normalize" - argspec: "args=[\'x\', \'axis\', \'epsilon\', \'name\', \'dim\'], varargs=None, keywords=None, defaults=[\'None\', \'1e-12\', \'None\', \'None\'], " + argspec: "args=[\'x\', \'axis\', \'epsilon\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'1e-12\', \'None\'], " } member_method { name: "logdet" @@ -142,7 +142,7 @@ tf_module { } member_method { name: "norm" - argspec: "args=[\'tensor\', \'ord\', \'axis\', \'keepdims\', \'name\', \'keep_dims\'], varargs=None, keywords=None, defaults=[\'euclidean\', \'None\', \'None\', \'None\', \'None\'], " + argspec: "args=[\'tensor\', \'ord\', \'axis\', \'keepdims\', \'name\'], varargs=None, keywords=None, defaults=[\'euclidean\', \'None\', \'None\', \'None\'], " } member_method { name: "qr" diff --git a/tensorflow/tools/api/golden/v2/tensorflow.lite.constants.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.lite.constants.pbtxt index 08845553e55..4d5c4893b41 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.lite.constants.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.lite.constants.pbtxt @@ -1,29 +1,9 @@ path: "tensorflow.lite.constants" tf_module { - member { - name: "FLOAT" - mtype: "" - } member { name: "GRAPHVIZ_DOT" mtype: "" } - member { - name: "INT32" - mtype: "" - } - member { - name: "INT64" - mtype: "" - } - member { - name: "QUANTIZED_UINT8" - mtype: "" - } - member { - name: "STRING" - mtype: "" - } member { name: "TFLITE" mtype: "" diff --git a/tensorflow/tools/api/golden/v2/tensorflow.logging.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.logging.pbtxt deleted file mode 100644 index 85bb15455da..00000000000 --- a/tensorflow/tools/api/golden/v2/tensorflow.logging.pbtxt +++ /dev/null @@ -1,83 +0,0 @@ -path: "tensorflow.logging" -tf_module { - member { - name: "DEBUG" - mtype: "" - } - member { - name: "ERROR" - mtype: "" - } - member { - name: "FATAL" - mtype: "" - } - member { - name: "INFO" - mtype: "" - } - member { - name: "WARN" - mtype: "" - } - member_method { - name: "TaskLevelStatusMessage" - argspec: "args=[\'msg\'], varargs=None, keywords=None, defaults=None" - } - member_method { - name: "debug" - argspec: "args=[\'msg\'], varargs=args, keywords=kwargs, defaults=None" - } - member_method { - name: "error" - argspec: "args=[\'msg\'], varargs=args, keywords=kwargs, defaults=None" - } - member_method { - name: "fatal" - argspec: "args=[\'msg\'], varargs=args, keywords=kwargs, defaults=None" - } - member_method { - name: "flush" - argspec: "args=[], varargs=None, keywords=None, defaults=None" - } - member_method { - name: "get_verbosity" - argspec: "args=[], varargs=None, keywords=None, defaults=None" - } - member_method { - name: "info" - argspec: "args=[\'msg\'], varargs=args, keywords=kwargs, defaults=None" - } - member_method { - name: "log" - argspec: "args=[\'level\', \'msg\'], varargs=args, keywords=kwargs, defaults=None" - } - member_method { - name: "log_every_n" - argspec: "args=[\'level\', \'msg\', \'n\'], varargs=args, keywords=None, defaults=None" - } - member_method { - name: "log_first_n" - argspec: "args=[\'level\', \'msg\', \'n\'], varargs=args, keywords=None, defaults=None" - } - member_method { - name: "log_if" - argspec: "args=[\'level\', \'msg\', \'condition\'], varargs=args, keywords=None, defaults=None" - } - member_method { - name: "set_verbosity" - argspec: "args=[\'v\'], varargs=None, keywords=None, defaults=None" - } - member_method { - name: "vlog" - argspec: "args=[\'level\', \'msg\'], varargs=args, keywords=kwargs, defaults=None" - } - member_method { - name: "warn" - argspec: "args=[\'msg\'], varargs=args, keywords=kwargs, defaults=None" - } - member_method { - name: "warning" - argspec: "args=[\'msg\'], varargs=args, keywords=kwargs, defaults=None" - } -} diff --git a/tensorflow/tools/api/golden/v2/tensorflow.losses.-reduction.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.losses.-reduction.pbtxt index 258ad5047eb..6a44e4ce66c 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.losses.-reduction.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.losses.-reduction.pbtxt @@ -1,11 +1,7 @@ path: "tensorflow.losses.Reduction" tf_class { - is_instance: "" + is_instance: "" is_instance: "" - member { - name: "MEAN" - mtype: "" - } member { name: "NONE" mtype: "" @@ -14,18 +10,10 @@ tf_class { name: "SUM" mtype: "" } - member { - name: "SUM_BY_NONZERO_WEIGHTS" - mtype: "" - } member { name: "SUM_OVER_BATCH_SIZE" mtype: "" } - member { - name: "SUM_OVER_NONZERO_WEIGHTS" - mtype: "" - } member_method { name: "__init__" } diff --git a/tensorflow/tools/api/golden/v2/tensorflow.losses.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.losses.pbtxt index c1d190ae116..233b1a0131a 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.losses.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.losses.pbtxt @@ -4,22 +4,10 @@ tf_module { name: "Reduction" mtype: "" } - member_method { - name: "absolute_difference" - argspec: "args=[\'labels\', \'predictions\', \'weights\', \'scope\', \'loss_collection\', \'reduction\'], varargs=None, keywords=None, defaults=[\'1.0\', \'None\', \'losses\', \'weighted_sum_by_nonzero_weights\'], " - } member_method { name: "add_loss" argspec: "args=[\'loss\', \'loss_collection\'], varargs=None, keywords=None, defaults=[\'losses\'], " } - member_method { - name: "compute_weighted_loss" - argspec: "args=[\'losses\', \'weights\', \'scope\', \'loss_collection\', \'reduction\'], varargs=None, keywords=None, defaults=[\'1.0\', \'None\', \'losses\', \'weighted_sum_by_nonzero_weights\'], " - } - member_method { - name: "cosine_distance" - argspec: "args=[\'labels\', \'predictions\', \'axis\', \'weights\', \'scope\', \'loss_collection\', \'reduction\', \'dim\'], varargs=None, keywords=None, defaults=[\'None\', \'1.0\', \'None\', \'losses\', \'weighted_sum_by_nonzero_weights\', \'None\'], " - } member_method { name: "get_losses" argspec: "args=[\'scope\', \'loss_collection\'], varargs=None, keywords=None, defaults=[\'None\', \'losses\'], " @@ -36,36 +24,4 @@ tf_module { name: "get_total_loss" argspec: "args=[\'add_regularization_losses\', \'name\'], varargs=None, keywords=None, defaults=[\'True\', \'total_loss\'], " } - member_method { - name: "hinge_loss" - argspec: "args=[\'labels\', \'logits\', \'weights\', \'scope\', \'loss_collection\', \'reduction\'], varargs=None, keywords=None, defaults=[\'1.0\', \'None\', \'losses\', \'weighted_sum_by_nonzero_weights\'], " - } - member_method { - name: "huber_loss" - argspec: "args=[\'labels\', \'predictions\', \'weights\', \'delta\', \'scope\', \'loss_collection\', \'reduction\'], varargs=None, keywords=None, defaults=[\'1.0\', \'1.0\', \'None\', \'losses\', \'weighted_sum_by_nonzero_weights\'], " - } - member_method { - name: "log_loss" - argspec: "args=[\'labels\', \'predictions\', \'weights\', \'epsilon\', \'scope\', \'loss_collection\', \'reduction\'], varargs=None, keywords=None, defaults=[\'1.0\', \'1e-07\', \'None\', \'losses\', \'weighted_sum_by_nonzero_weights\'], " - } - member_method { - name: "mean_pairwise_squared_error" - argspec: "args=[\'labels\', \'predictions\', \'weights\', \'scope\', \'loss_collection\'], varargs=None, keywords=None, defaults=[\'1.0\', \'None\', \'losses\'], " - } - member_method { - name: "mean_squared_error" - argspec: "args=[\'labels\', \'predictions\', \'weights\', \'scope\', \'loss_collection\', \'reduction\'], varargs=None, keywords=None, defaults=[\'1.0\', \'None\', \'losses\', \'weighted_sum_by_nonzero_weights\'], " - } - member_method { - name: "sigmoid_cross_entropy" - argspec: "args=[\'multi_class_labels\', \'logits\', \'weights\', \'label_smoothing\', \'scope\', \'loss_collection\', \'reduction\'], varargs=None, keywords=None, defaults=[\'1.0\', \'0\', \'None\', \'losses\', \'weighted_sum_by_nonzero_weights\'], " - } - member_method { - name: "softmax_cross_entropy" - argspec: "args=[\'onehot_labels\', \'logits\', \'weights\', \'label_smoothing\', \'scope\', \'loss_collection\', \'reduction\'], varargs=None, keywords=None, defaults=[\'1.0\', \'0\', \'None\', \'losses\', \'weighted_sum_by_nonzero_weights\'], " - } - member_method { - name: "sparse_softmax_cross_entropy" - argspec: "args=[\'labels\', \'logits\', \'weights\', \'scope\', \'loss_collection\', \'reduction\'], varargs=None, keywords=None, defaults=[\'1.0\', \'None\', \'losses\', \'weighted_sum_by_nonzero_weights\'], " - } } diff --git a/tensorflow/tools/api/golden/v2/tensorflow.math.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.math.pbtxt index a441e42b0a9..979d77ea6b3 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.math.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.math.pbtxt @@ -78,7 +78,7 @@ tf_module { } member_method { name: "bincount" - argspec: "args=[\'arr\', \'weights\', \'minlength\', \'maxlength\', \'dtype\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \"\"], " + argspec: "args=[\'arr\', \'weights\', \'minlength\', \'maxlength\', \'dtype\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \"\", \'None\'], " } member_method { name: "ceil" @@ -86,7 +86,7 @@ tf_module { } member_method { name: "confusion_matrix" - argspec: "args=[\'labels\', \'predictions\', \'num_classes\', \'dtype\', \'name\', \'weights\'], varargs=None, keywords=None, defaults=[\'None\', \"\", \'None\', \'None\'], " + argspec: "args=[\'labels\', \'predictions\', \'num_classes\', \'weights\', \'dtype\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \"\", \'None\'], " } member_method { name: "conj" @@ -102,7 +102,7 @@ tf_module { } member_method { name: "count_nonzero" - argspec: "args=[\'input_tensor\', \'axis\', \'keepdims\', \'dtype\', \'name\', \'reduction_indices\', \'keep_dims\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \"\", \'None\', \'None\', \'None\'], " + argspec: "args=[\'input\', \'axis\', \'keepdims\', \'dtype\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \"\", \'None\'], " } member_method { name: "cumprod" @@ -176,9 +176,29 @@ tf_module { name: "invert_permutation" argspec: "args=[\'x\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "is_finite" + argspec: "args=[\'x\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "is_inf" + argspec: "args=[\'x\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "is_nan" + argspec: "args=[\'x\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "is_non_decreasing" + argspec: "args=[\'x\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "is_strictly_increasing" + argspec: "args=[\'x\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " + } member_method { name: "l2_normalize" - argspec: "args=[\'x\', \'axis\', \'epsilon\', \'name\', \'dim\'], varargs=None, keywords=None, defaults=[\'None\', \'1e-12\', \'None\', \'None\'], " + argspec: "args=[\'x\', \'axis\', \'epsilon\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'1e-12\', \'None\'], " } member_method { name: "lbeta" @@ -210,7 +230,7 @@ tf_module { } member_method { name: "log_softmax" - argspec: "args=[\'logits\', \'axis\', \'name\', \'dim\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], " + argspec: "args=[\'logits\', \'axis\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " } member_method { name: "logical_and" @@ -270,43 +290,43 @@ tf_module { } member_method { name: "reduce_all" - argspec: "args=[\'input_tensor\', \'axis\', \'keepdims\', \'name\', \'reduction_indices\', \'keep_dims\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\'], " + argspec: "args=[\'input_tensor\', \'axis\', \'keepdims\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\'], " } member_method { name: "reduce_any" - argspec: "args=[\'input_tensor\', \'axis\', \'keepdims\', \'name\', \'reduction_indices\', \'keep_dims\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\'], " + argspec: "args=[\'input_tensor\', \'axis\', \'keepdims\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\'], " } member_method { name: "reduce_logsumexp" - argspec: "args=[\'input_tensor\', \'axis\', \'keepdims\', \'name\', \'reduction_indices\', \'keep_dims\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\'], " + argspec: "args=[\'input_tensor\', \'axis\', \'keepdims\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\'], " } member_method { name: "reduce_max" - argspec: "args=[\'input_tensor\', \'axis\', \'keepdims\', \'name\', \'reduction_indices\', \'keep_dims\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\'], " + argspec: "args=[\'input_tensor\', \'axis\', \'keepdims\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\'], " } member_method { name: "reduce_mean" - argspec: "args=[\'input_tensor\', \'axis\', \'keepdims\', \'name\', \'reduction_indices\', \'keep_dims\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\'], " + argspec: "args=[\'input_tensor\', \'axis\', \'keepdims\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\'], " } member_method { name: "reduce_min" - argspec: "args=[\'input_tensor\', \'axis\', \'keepdims\', \'name\', \'reduction_indices\', \'keep_dims\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\'], " + argspec: "args=[\'input_tensor\', \'axis\', \'keepdims\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\'], " } member_method { name: "reduce_prod" - argspec: "args=[\'input_tensor\', \'axis\', \'keepdims\', \'name\', \'reduction_indices\', \'keep_dims\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\'], " + argspec: "args=[\'input_tensor\', \'axis\', \'keepdims\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\'], " } member_method { name: "reduce_std" - argspec: "args=[\'input_tensor\', \'axis\', \'keepdims\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], " + argspec: "args=[\'input_tensor\', \'axis\', \'keepdims\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\'], " } member_method { name: "reduce_sum" - argspec: "args=[\'input_tensor\', \'axis\', \'keepdims\', \'name\', \'reduction_indices\', \'keep_dims\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\'], " + argspec: "args=[\'input_tensor\', \'axis\', \'keepdims\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\'], " } member_method { name: "reduce_variance" - argspec: "args=[\'input_tensor\', \'axis\', \'keepdims\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], " + argspec: "args=[\'input_tensor\', \'axis\', \'keepdims\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\'], " } member_method { name: "rint" @@ -322,7 +342,7 @@ tf_module { } member_method { name: "scalar_mul" - argspec: "args=[\'scalar\', \'x\'], varargs=None, keywords=None, defaults=None" + argspec: "args=[\'scalar\', \'x\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " } member_method { name: "segment_max" @@ -362,7 +382,7 @@ tf_module { } member_method { name: "softmax" - argspec: "args=[\'logits\', \'axis\', \'name\', \'dim\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], " + argspec: "args=[\'logits\', \'axis\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " } member_method { name: "softplus" diff --git a/tensorflow/tools/api/golden/v2/tensorflow.metrics.-accuracy.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.metrics.-accuracy.pbtxt new file mode 100644 index 00000000000..f8e12f88173 --- /dev/null +++ b/tensorflow/tools/api/golden/v2/tensorflow.metrics.-accuracy.pbtxt @@ -0,0 +1,194 @@ +path: "tensorflow.metrics.Accuracy" +tf_class { + is_instance: "" + is_instance: "" + is_instance: "" + is_instance: "" + is_instance: "" + is_instance: "" + is_instance: "" + member { + name: "activity_regularizer" + mtype: "" + } + member { + name: "dtype" + mtype: "" + } + member { + name: "inbound_nodes" + mtype: "" + } + member { + name: "input" + mtype: "" + } + member { + name: "input_mask" + mtype: "" + } + member { + name: "input_shape" + mtype: "" + } + member { + name: "losses" + mtype: "" + } + member { + name: "name" + mtype: "" + } + member { + name: "non_trainable_variables" + mtype: "" + } + member { + name: "non_trainable_weights" + mtype: "" + } + member { + name: "outbound_nodes" + mtype: "" + } + member { + name: "output" + mtype: "" + } + member { + name: "output_mask" + mtype: "" + } + member { + name: "output_shape" + mtype: "" + } + member { + name: "trainable_variables" + mtype: "" + } + member { + name: "trainable_weights" + mtype: "" + } + member { + name: "updates" + mtype: "" + } + member { + name: "variables" + mtype: "" + } + member { + name: "weights" + mtype: "" + } + member_method { + name: "__init__" + argspec: "args=[\'self\', \'name\', \'dtype\'], varargs=None, keywords=None, defaults=[\'accuracy\', \'None\'], " + } + member_method { + name: "add_loss" + argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } + member_method { + name: "add_update" + argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "add_variable" + argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None" + } + member_method { + name: "add_weight" + argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\'], " + } + member_method { + name: "apply" + argspec: "args=[\'self\', \'inputs\'], varargs=args, keywords=kwargs, defaults=None" + } + member_method { + name: "build" + argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "call" + argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=kwargs, defaults=None" + } + member_method { + name: "compute_mask" + argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "compute_output_shape" + argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "count_params" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "from_config" + argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_config" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_input_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_input_mask_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_input_shape_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_losses_for" + argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_output_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_output_mask_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_output_shape_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_updates_for" + argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_weights" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "reset_states" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "result" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "set_weights" + argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "update_state" + argspec: "args=[\'self\', \'y_true\', \'y_pred\', \'sample_weight\'], varargs=None, keywords=None, defaults=[\'None\'], " + } +} diff --git a/tensorflow/tools/api/golden/v2/tensorflow.metrics.-binary-accuracy.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.metrics.-binary-accuracy.pbtxt new file mode 100644 index 00000000000..b9bc6a716a1 --- /dev/null +++ b/tensorflow/tools/api/golden/v2/tensorflow.metrics.-binary-accuracy.pbtxt @@ -0,0 +1,194 @@ +path: "tensorflow.metrics.BinaryAccuracy" +tf_class { + is_instance: "" + is_instance: "" + is_instance: "" + is_instance: "" + is_instance: "" + is_instance: "" + is_instance: "" + member { + name: "activity_regularizer" + mtype: "" + } + member { + name: "dtype" + mtype: "" + } + member { + name: "inbound_nodes" + mtype: "" + } + member { + name: "input" + mtype: "" + } + member { + name: "input_mask" + mtype: "" + } + member { + name: "input_shape" + mtype: "" + } + member { + name: "losses" + mtype: "" + } + member { + name: "name" + mtype: "" + } + member { + name: "non_trainable_variables" + mtype: "" + } + member { + name: "non_trainable_weights" + mtype: "" + } + member { + name: "outbound_nodes" + mtype: "" + } + member { + name: "output" + mtype: "" + } + member { + name: "output_mask" + mtype: "" + } + member { + name: "output_shape" + mtype: "" + } + member { + name: "trainable_variables" + mtype: "" + } + member { + name: "trainable_weights" + mtype: "" + } + member { + name: "updates" + mtype: "" + } + member { + name: "variables" + mtype: "" + } + member { + name: "weights" + mtype: "" + } + member_method { + name: "__init__" + argspec: "args=[\'self\', \'name\', \'dtype\', \'threshold\'], varargs=None, keywords=None, defaults=[\'binary_accuracy\', \'None\', \'0.5\'], " + } + member_method { + name: "add_loss" + argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } + member_method { + name: "add_update" + argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "add_variable" + argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None" + } + member_method { + name: "add_weight" + argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\'], " + } + member_method { + name: "apply" + argspec: "args=[\'self\', \'inputs\'], varargs=args, keywords=kwargs, defaults=None" + } + member_method { + name: "build" + argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "call" + argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=kwargs, defaults=None" + } + member_method { + name: "compute_mask" + argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "compute_output_shape" + argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "count_params" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "from_config" + argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_config" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_input_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_input_mask_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_input_shape_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_losses_for" + argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_output_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_output_mask_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_output_shape_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_updates_for" + argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_weights" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "reset_states" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "result" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "set_weights" + argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "update_state" + argspec: "args=[\'self\', \'y_true\', \'y_pred\', \'sample_weight\'], varargs=None, keywords=None, defaults=[\'None\'], " + } +} diff --git a/tensorflow/tools/api/golden/v2/tensorflow.metrics.-categorical-accuracy.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.metrics.-categorical-accuracy.pbtxt new file mode 100644 index 00000000000..0ef75d8756f --- /dev/null +++ b/tensorflow/tools/api/golden/v2/tensorflow.metrics.-categorical-accuracy.pbtxt @@ -0,0 +1,194 @@ +path: "tensorflow.metrics.CategoricalAccuracy" +tf_class { + is_instance: "" + is_instance: "" + is_instance: "" + is_instance: "" + is_instance: "" + is_instance: "" + is_instance: "" + member { + name: "activity_regularizer" + mtype: "" + } + member { + name: "dtype" + mtype: "" + } + member { + name: "inbound_nodes" + mtype: "" + } + member { + name: "input" + mtype: "" + } + member { + name: "input_mask" + mtype: "" + } + member { + name: "input_shape" + mtype: "" + } + member { + name: "losses" + mtype: "" + } + member { + name: "name" + mtype: "" + } + member { + name: "non_trainable_variables" + mtype: "" + } + member { + name: "non_trainable_weights" + mtype: "" + } + member { + name: "outbound_nodes" + mtype: "" + } + member { + name: "output" + mtype: "" + } + member { + name: "output_mask" + mtype: "" + } + member { + name: "output_shape" + mtype: "" + } + member { + name: "trainable_variables" + mtype: "" + } + member { + name: "trainable_weights" + mtype: "" + } + member { + name: "updates" + mtype: "" + } + member { + name: "variables" + mtype: "" + } + member { + name: "weights" + mtype: "" + } + member_method { + name: "__init__" + argspec: "args=[\'self\', \'name\', \'dtype\'], varargs=None, keywords=None, defaults=[\'categorical_accuracy\', \'None\'], " + } + member_method { + name: "add_loss" + argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } + member_method { + name: "add_update" + argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "add_variable" + argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None" + } + member_method { + name: "add_weight" + argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\'], " + } + member_method { + name: "apply" + argspec: "args=[\'self\', \'inputs\'], varargs=args, keywords=kwargs, defaults=None" + } + member_method { + name: "build" + argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "call" + argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=kwargs, defaults=None" + } + member_method { + name: "compute_mask" + argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "compute_output_shape" + argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "count_params" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "from_config" + argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_config" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_input_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_input_mask_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_input_shape_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_losses_for" + argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_output_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_output_mask_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_output_shape_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_updates_for" + argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_weights" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "reset_states" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "result" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "set_weights" + argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "update_state" + argspec: "args=[\'self\', \'y_true\', \'y_pred\', \'sample_weight\'], varargs=None, keywords=None, defaults=[\'None\'], " + } +} diff --git a/tensorflow/tools/api/golden/v2/tensorflow.metrics.-false-negatives.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.metrics.-false-negatives.pbtxt new file mode 100644 index 00000000000..33226a2df62 --- /dev/null +++ b/tensorflow/tools/api/golden/v2/tensorflow.metrics.-false-negatives.pbtxt @@ -0,0 +1,193 @@ +path: "tensorflow.metrics.FalseNegatives" +tf_class { + is_instance: "" + is_instance: "" + is_instance: "" + is_instance: "" + is_instance: "" + is_instance: "" + member { + name: "activity_regularizer" + mtype: "" + } + member { + name: "dtype" + mtype: "" + } + member { + name: "inbound_nodes" + mtype: "" + } + member { + name: "input" + mtype: "" + } + member { + name: "input_mask" + mtype: "" + } + member { + name: "input_shape" + mtype: "" + } + member { + name: "losses" + mtype: "" + } + member { + name: "name" + mtype: "" + } + member { + name: "non_trainable_variables" + mtype: "" + } + member { + name: "non_trainable_weights" + mtype: "" + } + member { + name: "outbound_nodes" + mtype: "" + } + member { + name: "output" + mtype: "" + } + member { + name: "output_mask" + mtype: "" + } + member { + name: "output_shape" + mtype: "" + } + member { + name: "trainable_variables" + mtype: "" + } + member { + name: "trainable_weights" + mtype: "" + } + member { + name: "updates" + mtype: "" + } + member { + name: "variables" + mtype: "" + } + member { + name: "weights" + mtype: "" + } + member_method { + name: "__init__" + argspec: "args=[\'self\', \'thresholds\', \'name\', \'dtype\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], " + } + member_method { + name: "add_loss" + argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } + member_method { + name: "add_update" + argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "add_variable" + argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None" + } + member_method { + name: "add_weight" + argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\'], " + } + member_method { + name: "apply" + argspec: "args=[\'self\', \'inputs\'], varargs=args, keywords=kwargs, defaults=None" + } + member_method { + name: "build" + argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "call" + argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=kwargs, defaults=None" + } + member_method { + name: "compute_mask" + argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "compute_output_shape" + argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "count_params" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "from_config" + argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_config" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_input_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_input_mask_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_input_shape_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_losses_for" + argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_output_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_output_mask_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_output_shape_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_updates_for" + argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_weights" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "reset_states" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "result" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "set_weights" + argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "update_state" + argspec: "args=[\'self\', \'y_true\', \'y_pred\', \'sample_weight\'], varargs=None, keywords=None, defaults=[\'None\'], " + } +} diff --git a/tensorflow/tools/api/golden/v2/tensorflow.metrics.-false-positives.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.metrics.-false-positives.pbtxt new file mode 100644 index 00000000000..9953162ea3e --- /dev/null +++ b/tensorflow/tools/api/golden/v2/tensorflow.metrics.-false-positives.pbtxt @@ -0,0 +1,193 @@ +path: "tensorflow.metrics.FalsePositives" +tf_class { + is_instance: "" + is_instance: "" + is_instance: "" + is_instance: "" + is_instance: "" + is_instance: "" + member { + name: "activity_regularizer" + mtype: "" + } + member { + name: "dtype" + mtype: "" + } + member { + name: "inbound_nodes" + mtype: "" + } + member { + name: "input" + mtype: "" + } + member { + name: "input_mask" + mtype: "" + } + member { + name: "input_shape" + mtype: "" + } + member { + name: "losses" + mtype: "" + } + member { + name: "name" + mtype: "" + } + member { + name: "non_trainable_variables" + mtype: "" + } + member { + name: "non_trainable_weights" + mtype: "" + } + member { + name: "outbound_nodes" + mtype: "" + } + member { + name: "output" + mtype: "" + } + member { + name: "output_mask" + mtype: "" + } + member { + name: "output_shape" + mtype: "" + } + member { + name: "trainable_variables" + mtype: "" + } + member { + name: "trainable_weights" + mtype: "" + } + member { + name: "updates" + mtype: "" + } + member { + name: "variables" + mtype: "" + } + member { + name: "weights" + mtype: "" + } + member_method { + name: "__init__" + argspec: "args=[\'self\', \'thresholds\', \'name\', \'dtype\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], " + } + member_method { + name: "add_loss" + argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } + member_method { + name: "add_update" + argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "add_variable" + argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None" + } + member_method { + name: "add_weight" + argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\'], " + } + member_method { + name: "apply" + argspec: "args=[\'self\', \'inputs\'], varargs=args, keywords=kwargs, defaults=None" + } + member_method { + name: "build" + argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "call" + argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=kwargs, defaults=None" + } + member_method { + name: "compute_mask" + argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "compute_output_shape" + argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "count_params" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "from_config" + argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_config" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_input_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_input_mask_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_input_shape_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_losses_for" + argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_output_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_output_mask_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_output_shape_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_updates_for" + argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_weights" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "reset_states" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "result" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "set_weights" + argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "update_state" + argspec: "args=[\'self\', \'y_true\', \'y_pred\', \'sample_weight\'], varargs=None, keywords=None, defaults=[\'None\'], " + } +} diff --git a/tensorflow/tools/api/golden/v2/tensorflow.metrics.-mean.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.metrics.-mean.pbtxt new file mode 100644 index 00000000000..7fe6d6fda96 --- /dev/null +++ b/tensorflow/tools/api/golden/v2/tensorflow.metrics.-mean.pbtxt @@ -0,0 +1,192 @@ +path: "tensorflow.metrics.Mean" +tf_class { + is_instance: "" + is_instance: "" + is_instance: "" + is_instance: "" + is_instance: "" + member { + name: "activity_regularizer" + mtype: "" + } + member { + name: "dtype" + mtype: "" + } + member { + name: "inbound_nodes" + mtype: "" + } + member { + name: "input" + mtype: "" + } + member { + name: "input_mask" + mtype: "" + } + member { + name: "input_shape" + mtype: "" + } + member { + name: "losses" + mtype: "" + } + member { + name: "name" + mtype: "" + } + member { + name: "non_trainable_variables" + mtype: "" + } + member { + name: "non_trainable_weights" + mtype: "" + } + member { + name: "outbound_nodes" + mtype: "" + } + member { + name: "output" + mtype: "" + } + member { + name: "output_mask" + mtype: "" + } + member { + name: "output_shape" + mtype: "" + } + member { + name: "trainable_variables" + mtype: "" + } + member { + name: "trainable_weights" + mtype: "" + } + member { + name: "updates" + mtype: "" + } + member { + name: "variables" + mtype: "" + } + member { + name: "weights" + mtype: "" + } + member_method { + name: "__init__" + argspec: "args=[\'self\', \'name\', \'dtype\'], varargs=None, keywords=None, defaults=[\'mean\', \'None\'], " + } + member_method { + name: "add_loss" + argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } + member_method { + name: "add_update" + argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "add_variable" + argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None" + } + member_method { + name: "add_weight" + argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\'], " + } + member_method { + name: "apply" + argspec: "args=[\'self\', \'inputs\'], varargs=args, keywords=kwargs, defaults=None" + } + member_method { + name: "build" + argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "call" + argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=kwargs, defaults=None" + } + member_method { + name: "compute_mask" + argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "compute_output_shape" + argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "count_params" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "from_config" + argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_config" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_input_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_input_mask_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_input_shape_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_losses_for" + argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_output_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_output_mask_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_output_shape_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_updates_for" + argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_weights" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "reset_states" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "result" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "set_weights" + argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "update_state" + argspec: "args=[\'self\', \'values\', \'sample_weight\'], varargs=None, keywords=None, defaults=[\'None\'], " + } +} diff --git a/tensorflow/tools/api/golden/v2/tensorflow.metrics.-precision.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.metrics.-precision.pbtxt new file mode 100644 index 00000000000..8c3271a109c --- /dev/null +++ b/tensorflow/tools/api/golden/v2/tensorflow.metrics.-precision.pbtxt @@ -0,0 +1,192 @@ +path: "tensorflow.metrics.Precision" +tf_class { + is_instance: "" + is_instance: "" + is_instance: "" + is_instance: "" + is_instance: "" + member { + name: "activity_regularizer" + mtype: "" + } + member { + name: "dtype" + mtype: "" + } + member { + name: "inbound_nodes" + mtype: "" + } + member { + name: "input" + mtype: "" + } + member { + name: "input_mask" + mtype: "" + } + member { + name: "input_shape" + mtype: "" + } + member { + name: "losses" + mtype: "" + } + member { + name: "name" + mtype: "" + } + member { + name: "non_trainable_variables" + mtype: "" + } + member { + name: "non_trainable_weights" + mtype: "" + } + member { + name: "outbound_nodes" + mtype: "" + } + member { + name: "output" + mtype: "" + } + member { + name: "output_mask" + mtype: "" + } + member { + name: "output_shape" + mtype: "" + } + member { + name: "trainable_variables" + mtype: "" + } + member { + name: "trainable_weights" + mtype: "" + } + member { + name: "updates" + mtype: "" + } + member { + name: "variables" + mtype: "" + } + member { + name: "weights" + mtype: "" + } + member_method { + name: "__init__" + argspec: "args=[\'self\', \'thresholds\', \'name\', \'dtype\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], " + } + member_method { + name: "add_loss" + argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } + member_method { + name: "add_update" + argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "add_variable" + argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None" + } + member_method { + name: "add_weight" + argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\'], " + } + member_method { + name: "apply" + argspec: "args=[\'self\', \'inputs\'], varargs=args, keywords=kwargs, defaults=None" + } + member_method { + name: "build" + argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "call" + argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=kwargs, defaults=None" + } + member_method { + name: "compute_mask" + argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "compute_output_shape" + argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "count_params" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "from_config" + argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_config" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_input_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_input_mask_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_input_shape_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_losses_for" + argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_output_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_output_mask_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_output_shape_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_updates_for" + argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_weights" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "reset_states" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "result" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "set_weights" + argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "update_state" + argspec: "args=[\'self\', \'y_true\', \'y_pred\', \'sample_weight\'], varargs=None, keywords=None, defaults=[\'None\'], " + } +} diff --git a/tensorflow/tools/api/golden/v2/tensorflow.metrics.-recall.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.metrics.-recall.pbtxt new file mode 100644 index 00000000000..840a68bbc78 --- /dev/null +++ b/tensorflow/tools/api/golden/v2/tensorflow.metrics.-recall.pbtxt @@ -0,0 +1,192 @@ +path: "tensorflow.metrics.Recall" +tf_class { + is_instance: "" + is_instance: "" + is_instance: "" + is_instance: "" + is_instance: "" + member { + name: "activity_regularizer" + mtype: "" + } + member { + name: "dtype" + mtype: "" + } + member { + name: "inbound_nodes" + mtype: "" + } + member { + name: "input" + mtype: "" + } + member { + name: "input_mask" + mtype: "" + } + member { + name: "input_shape" + mtype: "" + } + member { + name: "losses" + mtype: "" + } + member { + name: "name" + mtype: "" + } + member { + name: "non_trainable_variables" + mtype: "" + } + member { + name: "non_trainable_weights" + mtype: "" + } + member { + name: "outbound_nodes" + mtype: "" + } + member { + name: "output" + mtype: "" + } + member { + name: "output_mask" + mtype: "" + } + member { + name: "output_shape" + mtype: "" + } + member { + name: "trainable_variables" + mtype: "" + } + member { + name: "trainable_weights" + mtype: "" + } + member { + name: "updates" + mtype: "" + } + member { + name: "variables" + mtype: "" + } + member { + name: "weights" + mtype: "" + } + member_method { + name: "__init__" + argspec: "args=[\'self\', \'thresholds\', \'name\', \'dtype\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], " + } + member_method { + name: "add_loss" + argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } + member_method { + name: "add_update" + argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "add_variable" + argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None" + } + member_method { + name: "add_weight" + argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\'], " + } + member_method { + name: "apply" + argspec: "args=[\'self\', \'inputs\'], varargs=args, keywords=kwargs, defaults=None" + } + member_method { + name: "build" + argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "call" + argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=kwargs, defaults=None" + } + member_method { + name: "compute_mask" + argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "compute_output_shape" + argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "count_params" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "from_config" + argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_config" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_input_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_input_mask_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_input_shape_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_losses_for" + argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_output_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_output_mask_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_output_shape_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_updates_for" + argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_weights" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "reset_states" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "result" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "set_weights" + argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "update_state" + argspec: "args=[\'self\', \'y_true\', \'y_pred\', \'sample_weight\'], varargs=None, keywords=None, defaults=[\'None\'], " + } +} diff --git a/tensorflow/tools/api/golden/v2/tensorflow.metrics.-sparse-categorical-accuracy.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.metrics.-sparse-categorical-accuracy.pbtxt new file mode 100644 index 00000000000..7bce43fbdeb --- /dev/null +++ b/tensorflow/tools/api/golden/v2/tensorflow.metrics.-sparse-categorical-accuracy.pbtxt @@ -0,0 +1,194 @@ +path: "tensorflow.metrics.SparseCategoricalAccuracy" +tf_class { + is_instance: "" + is_instance: "" + is_instance: "" + is_instance: "" + is_instance: "" + is_instance: "" + is_instance: "" + member { + name: "activity_regularizer" + mtype: "" + } + member { + name: "dtype" + mtype: "" + } + member { + name: "inbound_nodes" + mtype: "" + } + member { + name: "input" + mtype: "" + } + member { + name: "input_mask" + mtype: "" + } + member { + name: "input_shape" + mtype: "" + } + member { + name: "losses" + mtype: "" + } + member { + name: "name" + mtype: "" + } + member { + name: "non_trainable_variables" + mtype: "" + } + member { + name: "non_trainable_weights" + mtype: "" + } + member { + name: "outbound_nodes" + mtype: "" + } + member { + name: "output" + mtype: "" + } + member { + name: "output_mask" + mtype: "" + } + member { + name: "output_shape" + mtype: "" + } + member { + name: "trainable_variables" + mtype: "" + } + member { + name: "trainable_weights" + mtype: "" + } + member { + name: "updates" + mtype: "" + } + member { + name: "variables" + mtype: "" + } + member { + name: "weights" + mtype: "" + } + member_method { + name: "__init__" + argspec: "args=[\'self\', \'name\', \'dtype\'], varargs=None, keywords=None, defaults=[\'sparse_categorical_accuracy\', \'None\'], " + } + member_method { + name: "add_loss" + argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } + member_method { + name: "add_update" + argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "add_variable" + argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None" + } + member_method { + name: "add_weight" + argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\'], " + } + member_method { + name: "apply" + argspec: "args=[\'self\', \'inputs\'], varargs=args, keywords=kwargs, defaults=None" + } + member_method { + name: "build" + argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "call" + argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=kwargs, defaults=None" + } + member_method { + name: "compute_mask" + argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "compute_output_shape" + argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "count_params" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "from_config" + argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_config" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_input_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_input_mask_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_input_shape_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_losses_for" + argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_output_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_output_mask_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_output_shape_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_updates_for" + argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_weights" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "reset_states" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "result" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "set_weights" + argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "update_state" + argspec: "args=[\'self\', \'y_true\', \'y_pred\', \'sample_weight\'], varargs=None, keywords=None, defaults=[\'None\'], " + } +} diff --git a/tensorflow/tools/api/golden/v2/tensorflow.metrics.-true-negatives.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.metrics.-true-negatives.pbtxt new file mode 100644 index 00000000000..83cd5b736bc --- /dev/null +++ b/tensorflow/tools/api/golden/v2/tensorflow.metrics.-true-negatives.pbtxt @@ -0,0 +1,193 @@ +path: "tensorflow.metrics.TrueNegatives" +tf_class { + is_instance: "" + is_instance: "" + is_instance: "" + is_instance: "" + is_instance: "" + is_instance: "" + member { + name: "activity_regularizer" + mtype: "" + } + member { + name: "dtype" + mtype: "" + } + member { + name: "inbound_nodes" + mtype: "" + } + member { + name: "input" + mtype: "" + } + member { + name: "input_mask" + mtype: "" + } + member { + name: "input_shape" + mtype: "" + } + member { + name: "losses" + mtype: "" + } + member { + name: "name" + mtype: "" + } + member { + name: "non_trainable_variables" + mtype: "" + } + member { + name: "non_trainable_weights" + mtype: "" + } + member { + name: "outbound_nodes" + mtype: "" + } + member { + name: "output" + mtype: "" + } + member { + name: "output_mask" + mtype: "" + } + member { + name: "output_shape" + mtype: "" + } + member { + name: "trainable_variables" + mtype: "" + } + member { + name: "trainable_weights" + mtype: "" + } + member { + name: "updates" + mtype: "" + } + member { + name: "variables" + mtype: "" + } + member { + name: "weights" + mtype: "" + } + member_method { + name: "__init__" + argspec: "args=[\'self\', \'thresholds\', \'name\', \'dtype\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], " + } + member_method { + name: "add_loss" + argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } + member_method { + name: "add_update" + argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "add_variable" + argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None" + } + member_method { + name: "add_weight" + argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\'], " + } + member_method { + name: "apply" + argspec: "args=[\'self\', \'inputs\'], varargs=args, keywords=kwargs, defaults=None" + } + member_method { + name: "build" + argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "call" + argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=kwargs, defaults=None" + } + member_method { + name: "compute_mask" + argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "compute_output_shape" + argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "count_params" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "from_config" + argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_config" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_input_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_input_mask_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_input_shape_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_losses_for" + argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_output_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_output_mask_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_output_shape_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_updates_for" + argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_weights" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "reset_states" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "result" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "set_weights" + argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "update_state" + argspec: "args=[\'self\', \'y_true\', \'y_pred\', \'sample_weight\'], varargs=None, keywords=None, defaults=[\'None\'], " + } +} diff --git a/tensorflow/tools/api/golden/v2/tensorflow.metrics.-true-positives.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.metrics.-true-positives.pbtxt new file mode 100644 index 00000000000..5b2502eafee --- /dev/null +++ b/tensorflow/tools/api/golden/v2/tensorflow.metrics.-true-positives.pbtxt @@ -0,0 +1,193 @@ +path: "tensorflow.metrics.TruePositives" +tf_class { + is_instance: "" + is_instance: "" + is_instance: "" + is_instance: "" + is_instance: "" + is_instance: "" + member { + name: "activity_regularizer" + mtype: "" + } + member { + name: "dtype" + mtype: "" + } + member { + name: "inbound_nodes" + mtype: "" + } + member { + name: "input" + mtype: "" + } + member { + name: "input_mask" + mtype: "" + } + member { + name: "input_shape" + mtype: "" + } + member { + name: "losses" + mtype: "" + } + member { + name: "name" + mtype: "" + } + member { + name: "non_trainable_variables" + mtype: "" + } + member { + name: "non_trainable_weights" + mtype: "" + } + member { + name: "outbound_nodes" + mtype: "" + } + member { + name: "output" + mtype: "" + } + member { + name: "output_mask" + mtype: "" + } + member { + name: "output_shape" + mtype: "" + } + member { + name: "trainable_variables" + mtype: "" + } + member { + name: "trainable_weights" + mtype: "" + } + member { + name: "updates" + mtype: "" + } + member { + name: "variables" + mtype: "" + } + member { + name: "weights" + mtype: "" + } + member_method { + name: "__init__" + argspec: "args=[\'self\', \'thresholds\', \'name\', \'dtype\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], " + } + member_method { + name: "add_loss" + argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } + member_method { + name: "add_update" + argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "add_variable" + argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None" + } + member_method { + name: "add_weight" + argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\'], " + } + member_method { + name: "apply" + argspec: "args=[\'self\', \'inputs\'], varargs=args, keywords=kwargs, defaults=None" + } + member_method { + name: "build" + argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "call" + argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=kwargs, defaults=None" + } + member_method { + name: "compute_mask" + argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "compute_output_shape" + argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "count_params" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "from_config" + argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_config" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_input_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_input_mask_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_input_shape_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_losses_for" + argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_output_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_output_mask_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_output_shape_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_updates_for" + argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_weights" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "reset_states" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "result" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "set_weights" + argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "update_state" + argspec: "args=[\'self\', \'y_true\', \'y_pred\', \'sample_weight\'], varargs=None, keywords=None, defaults=[\'None\'], " + } +} diff --git a/tensorflow/tools/api/golden/v2/tensorflow.metrics.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.metrics.pbtxt index e9b996c9f53..773efd03fc8 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.metrics.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.metrics.pbtxt @@ -1,135 +1,47 @@ path: "tensorflow.metrics" tf_module { - member_method { - name: "accuracy" - argspec: "args=[\'labels\', \'predictions\', \'weights\', \'metrics_collections\', \'updates_collections\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], " + member { + name: "Accuracy" + mtype: "" } - member_method { - name: "auc" - argspec: "args=[\'labels\', \'predictions\', \'weights\', \'num_thresholds\', \'metrics_collections\', \'updates_collections\', \'curve\', \'name\', \'summation_method\'], varargs=None, keywords=None, defaults=[\'None\', \'200\', \'None\', \'None\', \'ROC\', \'None\', \'trapezoidal\'], " + member { + name: "BinaryAccuracy" + mtype: "" } - member_method { - name: "average_precision_at_k" - argspec: "args=[\'labels\', \'predictions\', \'k\', \'weights\', \'metrics_collections\', \'updates_collections\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], " + member { + name: "CategoricalAccuracy" + mtype: "" } - member_method { - name: "false_negatives" - argspec: "args=[\'labels\', \'predictions\', \'weights\', \'metrics_collections\', \'updates_collections\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], " + member { + name: "FalseNegatives" + mtype: "" } - member_method { - name: "false_negatives_at_thresholds" - argspec: "args=[\'labels\', \'predictions\', \'thresholds\', \'weights\', \'metrics_collections\', \'updates_collections\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], " + member { + name: "FalsePositives" + mtype: "" } - member_method { - name: "false_positives" - argspec: "args=[\'labels\', \'predictions\', \'weights\', \'metrics_collections\', \'updates_collections\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], " + member { + name: "Mean" + mtype: "" } - member_method { - name: "false_positives_at_thresholds" - argspec: "args=[\'labels\', \'predictions\', \'thresholds\', \'weights\', \'metrics_collections\', \'updates_collections\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], " + member { + name: "Precision" + mtype: "" } - member_method { - name: "mean" - argspec: "args=[\'values\', \'weights\', \'metrics_collections\', \'updates_collections\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], " + member { + name: "Recall" + mtype: "" } - member_method { - name: "mean_absolute_error" - argspec: "args=[\'labels\', \'predictions\', \'weights\', \'metrics_collections\', \'updates_collections\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], " + member { + name: "SparseCategoricalAccuracy" + mtype: "" } - member_method { - name: "mean_cosine_distance" - argspec: "args=[\'labels\', \'predictions\', \'dim\', \'weights\', \'metrics_collections\', \'updates_collections\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], " + member { + name: "TrueNegatives" + mtype: "" } - member_method { - name: "mean_iou" - argspec: "args=[\'labels\', \'predictions\', \'num_classes\', \'weights\', \'metrics_collections\', \'updates_collections\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], " - } - member_method { - name: "mean_per_class_accuracy" - argspec: "args=[\'labels\', \'predictions\', \'num_classes\', \'weights\', \'metrics_collections\', \'updates_collections\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], " - } - member_method { - name: "mean_relative_error" - argspec: "args=[\'labels\', \'predictions\', \'normalizer\', \'weights\', \'metrics_collections\', \'updates_collections\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], " - } - member_method { - name: "mean_squared_error" - argspec: "args=[\'labels\', \'predictions\', \'weights\', \'metrics_collections\', \'updates_collections\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], " - } - member_method { - name: "mean_tensor" - argspec: "args=[\'values\', \'weights\', \'metrics_collections\', \'updates_collections\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], " - } - member_method { - name: "percentage_below" - argspec: "args=[\'values\', \'threshold\', \'weights\', \'metrics_collections\', \'updates_collections\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], " - } - member_method { - name: "precision" - argspec: "args=[\'labels\', \'predictions\', \'weights\', \'metrics_collections\', \'updates_collections\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], " - } - member_method { - name: "precision_at_k" - argspec: "args=[\'labels\', \'predictions\', \'k\', \'class_id\', \'weights\', \'metrics_collections\', \'updates_collections\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\'], " - } - member_method { - name: "precision_at_thresholds" - argspec: "args=[\'labels\', \'predictions\', \'thresholds\', \'weights\', \'metrics_collections\', \'updates_collections\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], " - } - member_method { - name: "precision_at_top_k" - argspec: "args=[\'labels\', \'predictions_idx\', \'k\', \'class_id\', \'weights\', \'metrics_collections\', \'updates_collections\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\'], " - } - member_method { - name: "recall" - argspec: "args=[\'labels\', \'predictions\', \'weights\', \'metrics_collections\', \'updates_collections\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], " - } - member_method { - name: "recall_at_k" - argspec: "args=[\'labels\', \'predictions\', \'k\', \'class_id\', \'weights\', \'metrics_collections\', \'updates_collections\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\'], " - } - member_method { - name: "recall_at_thresholds" - argspec: "args=[\'labels\', \'predictions\', \'thresholds\', \'weights\', \'metrics_collections\', \'updates_collections\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], " - } - member_method { - name: "recall_at_top_k" - argspec: "args=[\'labels\', \'predictions_idx\', \'k\', \'class_id\', \'weights\', \'metrics_collections\', \'updates_collections\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\'], " - } - member_method { - name: "root_mean_squared_error" - argspec: "args=[\'labels\', \'predictions\', \'weights\', \'metrics_collections\', \'updates_collections\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], " - } - member_method { - name: "sensitivity_at_specificity" - argspec: "args=[\'labels\', \'predictions\', \'specificity\', \'weights\', \'num_thresholds\', \'metrics_collections\', \'updates_collections\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'200\', \'None\', \'None\', \'None\'], " - } - member_method { - name: "sparse_average_precision_at_k" - argspec: "args=[\'labels\', \'predictions\', \'k\', \'weights\', \'metrics_collections\', \'updates_collections\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], " - } - member_method { - name: "sparse_precision_at_k" - argspec: "args=[\'labels\', \'predictions\', \'k\', \'class_id\', \'weights\', \'metrics_collections\', \'updates_collections\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\'], " - } - member_method { - name: "specificity_at_sensitivity" - argspec: "args=[\'labels\', \'predictions\', \'sensitivity\', \'weights\', \'num_thresholds\', \'metrics_collections\', \'updates_collections\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'200\', \'None\', \'None\', \'None\'], " - } - member_method { - name: "true_negatives" - argspec: "args=[\'labels\', \'predictions\', \'weights\', \'metrics_collections\', \'updates_collections\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], " - } - member_method { - name: "true_negatives_at_thresholds" - argspec: "args=[\'labels\', \'predictions\', \'thresholds\', \'weights\', \'metrics_collections\', \'updates_collections\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], " - } - member_method { - name: "true_positives" - argspec: "args=[\'labels\', \'predictions\', \'weights\', \'metrics_collections\', \'updates_collections\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], " - } - member_method { - name: "true_positives_at_thresholds" - argspec: "args=[\'labels\', \'predictions\', \'thresholds\', \'weights\', \'metrics_collections\', \'updates_collections\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], " + member { + name: "TruePositives" + mtype: "" } } diff --git a/tensorflow/tools/api/golden/v2/tensorflow.nn.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.nn.pbtxt index 2dc5c48aa6e..63bf24b5d5e 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.nn.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.nn.pbtxt @@ -30,7 +30,7 @@ tf_module { } member_method { name: "batch_norm_with_global_normalization" - argspec: "args=[\'t\', \'m\', \'v\', \'beta\', \'gamma\', \'variance_epsilon\', \'scale_after_normalization\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " + argspec: "args=[\'input\', \'mean\', \'variance\', \'beta\', \'gamma\', \'variance_epsilon\', \'scale_after_normalization\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " } member_method { name: "batch_normalization" @@ -41,8 +41,8 @@ tf_module { argspec: "args=[\'value\', \'bias\', \'data_format\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " } member_method { - name: "bidirectional_dynamic_rnn" - argspec: "args=[\'cell_fw\', \'cell_bw\', \'inputs\', \'sequence_length\', \'initial_state_fw\', \'initial_state_bw\', \'dtype\', \'parallel_iterations\', \'swap_memory\', \'time_major\', \'scope\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'False\', \'False\', \'None\'], " + name: "collapse_repeated" + argspec: "args=[\'labels\', \'seq_length\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " } member_method { name: "compute_accidental_hits" @@ -50,43 +50,43 @@ tf_module { } member_method { name: "conv1d" - argspec: "args=[\'value\', \'filters\', \'stride\', \'padding\', \'use_cudnn_on_gpu\', \'data_format\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], " + argspec: "args=[\'input\', \'filters\', \'stride\', \'padding\', \'data_format\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " } member_method { name: "conv2d" - argspec: "args=[\'input\', \'filter\', \'strides\', \'padding\', \'use_cudnn_on_gpu\', \'data_format\', \'dilations\', \'name\'], varargs=None, keywords=None, defaults=[\'True\', \'NHWC\', \'[1, 1, 1, 1]\', \'None\'], " + argspec: "args=[\'input\', \'filters\', \'strides\', \'padding\', \'data_format\', \'dilations\', \'name\'], varargs=None, keywords=None, defaults=[\'NHWC\', \'None\', \'None\'], " } member_method { name: "conv2d_backprop_filter" - argspec: "args=[\'input\', \'filter_sizes\', \'out_backprop\', \'strides\', \'padding\', \'use_cudnn_on_gpu\', \'data_format\', \'dilations\', \'name\'], varargs=None, keywords=None, defaults=[\'True\', \'NHWC\', \'[1, 1, 1, 1]\', \'None\'], " + argspec: "args=[\'input\', \'filter_sizes\', \'out_backprop\', \'strides\', \'padding\', \'data_format\', \'dilations\', \'name\'], varargs=None, keywords=None, defaults=[\'NHWC\', \'None\', \'None\'], " } member_method { name: "conv2d_backprop_input" - argspec: "args=[\'input_sizes\', \'filter\', \'out_backprop\', \'strides\', \'padding\', \'use_cudnn_on_gpu\', \'data_format\', \'dilations\', \'name\'], varargs=None, keywords=None, defaults=[\'True\', \'NHWC\', \'[1, 1, 1, 1]\', \'None\'], " + argspec: "args=[\'input_sizes\', \'filters\', \'out_backprop\', \'strides\', \'padding\', \'data_format\', \'dilations\', \'name\'], varargs=None, keywords=None, defaults=[\'NHWC\', \'None\', \'None\'], " } member_method { name: "conv2d_transpose" - argspec: "args=[\'value\', \'filter\', \'output_shape\', \'strides\', \'padding\', \'data_format\', \'name\'], varargs=None, keywords=None, defaults=[\'SAME\', \'NHWC\', \'None\'], " + argspec: "args=[\'input\', \'filters\', \'output_shape\', \'strides\', \'padding\', \'data_format\', \'name\'], varargs=None, keywords=None, defaults=[\'SAME\', \'NHWC\', \'None\'], " } member_method { name: "conv3d" - argspec: "args=[\'input\', \'filter\', \'strides\', \'padding\', \'data_format\', \'dilations\', \'name\'], varargs=None, keywords=None, defaults=[\'NDHWC\', \'[1, 1, 1, 1, 1]\', \'None\'], " + argspec: "args=[\'input\', \'filters\', \'strides\', \'padding\', \'data_format\', \'dilations\', \'name\'], varargs=None, keywords=None, defaults=[\'NDHWC\', \'None\', \'None\'], " } member_method { - name: "conv3d_backprop_filter_v2" + name: "conv3d_backprop_filter" argspec: "args=[\'input\', \'filter_sizes\', \'out_backprop\', \'strides\', \'padding\', \'data_format\', \'dilations\', \'name\'], varargs=None, keywords=None, defaults=[\'NDHWC\', \'[1, 1, 1, 1, 1]\', \'None\'], " } member_method { name: "conv3d_transpose" - argspec: "args=[\'value\', \'filter\', \'output_shape\', \'strides\', \'padding\', \'data_format\', \'name\'], varargs=None, keywords=None, defaults=[\'SAME\', \'NDHWC\', \'None\'], " + argspec: "args=[\'input\', \'filters\', \'output_shape\', \'strides\', \'padding\', \'data_format\', \'name\'], varargs=None, keywords=None, defaults=[\'SAME\', \'NDHWC\', \'None\'], " } member_method { name: "convolution" - argspec: "args=[\'input\', \'filter\', \'padding\', \'strides\', \'dilation_rate\', \'name\', \'data_format\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], " + argspec: "args=[\'input\', \'filters\', \'strides\', \'padding\', \'data_format\', \'dilations\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'VALID\', \'None\', \'None\', \'None\'], " } member_method { name: "crelu" - argspec: "args=[\'features\', \'name\', \'axis\'], varargs=None, keywords=None, defaults=[\'None\', \'-1\'], " + argspec: "args=[\'features\', \'axis\', \'name\'], varargs=None, keywords=None, defaults=[\'-1\', \'None\'], " } member_method { name: "ctc_beam_search_decoder" @@ -98,7 +98,11 @@ tf_module { } member_method { name: "ctc_loss" - argspec: "args=[\'labels\', \'inputs\', \'sequence_length\', \'preprocess_collapse_repeated\', \'ctc_merge_repeated\', \'ignore_longer_outputs_than_inputs\', \'time_major\'], varargs=None, keywords=None, defaults=[\'False\', \'True\', \'False\', \'True\'], " + argspec: "args=[\'labels\', \'logits\', \'label_length\', \'logit_length\', \'logits_time_major\', \'unique\', \'blank_index\', \'name\'], varargs=None, keywords=None, defaults=[\'True\', \'None\', \'None\', \'None\'], " + } + member_method { + name: "ctc_unique_labels" + argspec: "args=[\'labels\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " } member_method { name: "depth_to_space" @@ -106,27 +110,23 @@ tf_module { } member_method { name: "depthwise_conv2d" - argspec: "args=[\'input\', \'filter\', \'strides\', \'padding\', \'rate\', \'name\', \'data_format\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], " + argspec: "args=[\'input\', \'filter\', \'strides\', \'padding\', \'data_format\', \'dilations\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], " } member_method { - name: "depthwise_conv2d_native" - argspec: "args=[\'input\', \'filter\', \'strides\', \'padding\', \'data_format\', \'dilations\', \'name\'], varargs=None, keywords=None, defaults=[\'NHWC\', \'[1, 1, 1, 1]\', \'None\'], " - } - member_method { - name: "depthwise_conv2d_native_backprop_filter" + name: "depthwise_conv2d_backprop_filter" argspec: "args=[\'input\', \'filter_sizes\', \'out_backprop\', \'strides\', \'padding\', \'data_format\', \'dilations\', \'name\'], varargs=None, keywords=None, defaults=[\'NHWC\', \'[1, 1, 1, 1]\', \'None\'], " } member_method { - name: "depthwise_conv2d_native_backprop_input" + name: "depthwise_conv2d_backprop_input" argspec: "args=[\'input_sizes\', \'filter\', \'out_backprop\', \'strides\', \'padding\', \'data_format\', \'dilations\', \'name\'], varargs=None, keywords=None, defaults=[\'NHWC\', \'[1, 1, 1, 1]\', \'None\'], " } member_method { name: "dilation2d" - argspec: "args=[\'input\', \'filter\', \'strides\', \'rates\', \'padding\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " + argspec: "args=[\'input\', \'filters\', \'strides\', \'padding\', \'data_format\', \'dilations\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " } member_method { name: "dropout" - argspec: "args=[\'x\', \'keep_prob\', \'noise_shape\', \'seed\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], " + argspec: "args=[\'x\', \'rate\', \'noise_shape\', \'seed\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], " } member_method { name: "elu" @@ -142,7 +142,7 @@ tf_module { } member_method { name: "erosion2d" - argspec: "args=[\'value\', \'kernel\', \'strides\', \'rates\', \'padding\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " + argspec: "args=[\'value\', \'filters\', \'strides\', \'padding\', \'data_format\', \'dilations\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " } member_method { name: "fixed_unigram_candidate_sampler" @@ -150,15 +150,11 @@ tf_module { } member_method { name: "fractional_avg_pool" - argspec: "args=[\'value\', \'pooling_ratio\', \'pseudo_random\', \'overlapping\', \'deterministic\', \'seed\', \'seed2\', \'name\'], varargs=None, keywords=None, defaults=[\'False\', \'False\', \'False\', \'0\', \'0\', \'None\'], " + argspec: "args=[\'value\', \'pooling_ratio\', \'pseudo_random\', \'overlapping\', \'seed\', \'name\'], varargs=None, keywords=None, defaults=[\'False\', \'False\', \'0\', \'None\'], " } member_method { name: "fractional_max_pool" - argspec: "args=[\'value\', \'pooling_ratio\', \'pseudo_random\', \'overlapping\', \'deterministic\', \'seed\', \'seed2\', \'name\'], varargs=None, keywords=None, defaults=[\'False\', \'False\', \'False\', \'0\', \'0\', \'None\'], " - } - member_method { - name: "fused_batch_norm" - argspec: "args=[\'x\', \'scale\', \'offset\', \'mean\', \'variance\', \'epsilon\', \'data_format\', \'is_training\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'0.001\', \'NHWC\', \'True\', \'None\'], " + argspec: "args=[\'value\', \'pooling_ratio\', \'pseudo_random\', \'overlapping\', \'seed\', \'name\'], varargs=None, keywords=None, defaults=[\'False\', \'False\', \'0\', \'None\'], " } member_method { name: "in_top_k" @@ -170,7 +166,7 @@ tf_module { } member_method { name: "l2_normalize" - argspec: "args=[\'x\', \'axis\', \'epsilon\', \'name\', \'dim\'], varargs=None, keywords=None, defaults=[\'None\', \'1e-12\', \'None\', \'None\'], " + argspec: "args=[\'x\', \'axis\', \'epsilon\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'1e-12\', \'None\'], " } member_method { name: "leaky_relu" @@ -190,7 +186,7 @@ tf_module { } member_method { name: "log_softmax" - argspec: "args=[\'logits\', \'axis\', \'name\', \'dim\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], " + argspec: "args=[\'logits\', \'axis\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " } member_method { name: "lrn" @@ -206,15 +202,15 @@ tf_module { } member_method { name: "max_pool_with_argmax" - argspec: "args=[\'input\', \'ksize\', \'strides\', \'padding\', \'Targmax\', \'name\'], varargs=None, keywords=None, defaults=[\"\", \'None\'], " + argspec: "args=[\'input\', \'ksize\', \'strides\', \'padding\', \'data_format\', \'output_dtype\', \'name\'], varargs=None, keywords=None, defaults=[\'NHWC\', \"\", \'None\'], " } member_method { name: "moments" - argspec: "args=[\'x\', \'axes\', \'shift\', \'name\', \'keep_dims\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'False\'], " + argspec: "args=[\'x\', \'axes\', \'shift\', \'keepdims\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\'], " } member_method { name: "nce_loss" - argspec: "args=[\'weights\', \'biases\', \'labels\', \'inputs\', \'num_sampled\', \'num_classes\', \'num_true\', \'sampled_values\', \'remove_accidental_hits\', \'partition_strategy\', \'name\'], varargs=None, keywords=None, defaults=[\'1\', \'None\', \'False\', \'mod\', \'nce_loss\'], " + argspec: "args=[\'weights\', \'biases\', \'labels\', \'inputs\', \'num_sampled\', \'num_classes\', \'num_true\', \'sampled_values\', \'remove_accidental_hits\', \'name\'], varargs=None, keywords=None, defaults=[\'1\', \'None\', \'False\', \'nce_loss\'], " } member_method { name: "normalize_moments" @@ -222,23 +218,7 @@ tf_module { } member_method { name: "pool" - argspec: "args=[\'input\', \'window_shape\', \'pooling_type\', \'padding\', \'dilation_rate\', \'strides\', \'name\', \'data_format\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], " - } - member_method { - name: "quantized_avg_pool" - argspec: "args=[\'input\', \'min_input\', \'max_input\', \'ksize\', \'strides\', \'padding\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " - } - member_method { - name: "quantized_conv2d" - argspec: "args=[\'input\', \'filter\', \'min_input\', \'max_input\', \'min_filter\', \'max_filter\', \'strides\', \'padding\', \'out_type\', \'dilations\', \'name\'], varargs=None, keywords=None, defaults=[\"\", \'[1, 1, 1, 1]\', \'None\'], " - } - member_method { - name: "quantized_max_pool" - argspec: "args=[\'input\', \'min_input\', \'max_input\', \'ksize\', \'strides\', \'padding\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " - } - member_method { - name: "quantized_relu_x" - argspec: "args=[\'features\', \'max_value\', \'min_features\', \'max_features\', \'out_type\', \'name\'], varargs=None, keywords=None, defaults=[\"\", \'None\'], " + argspec: "args=[\'input\', \'window_shape\', \'pooling_type\', \'strides\', \'padding\', \'data_format\', \'dilations\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'VALID\', \'None\', \'None\', \'None\'], " } member_method { name: "relu" @@ -266,7 +246,7 @@ tf_module { } member_method { name: "separable_conv2d" - argspec: "args=[\'input\', \'depthwise_filter\', \'pointwise_filter\', \'strides\', \'padding\', \'rate\', \'name\', \'data_format\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], " + argspec: "args=[\'input\', \'depthwise_filter\', \'pointwise_filter\', \'strides\', \'padding\', \'data_format\', \'dilations\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], " } member_method { name: "sigmoid" @@ -278,11 +258,11 @@ tf_module { } member_method { name: "softmax" - argspec: "args=[\'logits\', \'axis\', \'name\', \'dim\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], " + argspec: "args=[\'logits\', \'axis\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " } member_method { name: "softmax_cross_entropy_with_logits" - argspec: "args=[\'labels\', \'logits\', \'dim\', \'name\'], varargs=None, keywords=None, defaults=[\'-1\', \'None\'], " + argspec: "args=[\'labels\', \'logits\', \'axis\', \'name\'], varargs=None, keywords=None, defaults=[\'-1\', \'None\'], " } member_method { name: "softplus" @@ -304,17 +284,13 @@ tf_module { name: "sparse_softmax_cross_entropy_with_logits" argspec: "args=[\'_sentinel\', \'labels\', \'logits\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], " } - member_method { - name: "static_bidirectional_rnn" - argspec: "args=[\'cell_fw\', \'cell_bw\', \'inputs\', \'initial_state_fw\', \'initial_state_bw\', \'dtype\', \'sequence_length\', \'scope\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\'], " - } member_method { name: "static_state_saving_rnn" argspec: "args=[\'cell\', \'inputs\', \'state_saver\', \'state_name\', \'sequence_length\', \'scope\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " } member_method { name: "sufficient_statistics" - argspec: "args=[\'x\', \'axes\', \'shift\', \'keep_dims\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\'], " + argspec: "args=[\'x\', \'axes\', \'shift\', \'keepdims\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\'], " } member_method { name: "tanh" @@ -330,16 +306,12 @@ tf_module { } member_method { name: "weighted_moments" - argspec: "args=[\'x\', \'axes\', \'frequency_weights\', \'name\', \'keep_dims\'], varargs=None, keywords=None, defaults=[\'None\', \'False\'], " + argspec: "args=[\'x\', \'axes\', \'frequency_weights\', \'keepdims\', \'name\'], varargs=None, keywords=None, defaults=[\'False\', \'None\'], " } member_method { name: "with_space_to_batch" argspec: "args=[\'input\', \'dilation_rate\', \'padding\', \'op\', \'filter_shape\', \'spatial_dims\', \'data_format\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], " } - member_method { - name: "xw_plus_b" - argspec: "args=[\'x\', \'weights\', \'biases\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " - } member_method { name: "zero_fraction" argspec: "args=[\'value\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v2/tensorflow.nn.rnn_cell.-device-wrapper.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.nn.rnn_cell.-device-wrapper.pbtxt index 381c4975d7d..9e52a425261 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.nn.rnn_cell.-device-wrapper.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.nn.rnn_cell.-device-wrapper.pbtxt @@ -106,6 +106,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v2/tensorflow.nn.rnn_cell.-dropout-wrapper.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.nn.rnn_cell.-dropout-wrapper.pbtxt index 912365a28b1..9836433d08c 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.nn.rnn_cell.-dropout-wrapper.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.nn.rnn_cell.-dropout-wrapper.pbtxt @@ -110,6 +110,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v2/tensorflow.nn.rnn_cell.-r-n-n-cell.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.nn.rnn_cell.-r-n-n-cell.pbtxt index faeb4f35133..d3b68e4f297 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.nn.rnn_cell.-r-n-n-cell.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.nn.rnn_cell.-r-n-n-cell.pbtxt @@ -105,6 +105,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v2/tensorflow.nn.rnn_cell.-residual-wrapper.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.nn.rnn_cell.-residual-wrapper.pbtxt index caa2e600800..1f7840ab919 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.nn.rnn_cell.-residual-wrapper.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.nn.rnn_cell.-residual-wrapper.pbtxt @@ -106,6 +106,10 @@ tf_class { name: "add_loss" argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "add_update" argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v2/tensorflow.nn.rnn_cell.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.nn.rnn_cell.pbtxt index 3c78b07b394..b1f687f5296 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.nn.rnn_cell.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.nn.rnn_cell.pbtxt @@ -12,10 +12,6 @@ tf_module { name: "LSTMStateTuple" mtype: "" } - member { - name: "MultiRNNCell" - mtype: "" - } member { name: "RNNCell" mtype: "" diff --git a/tensorflow/tools/api/golden/v2/tensorflow.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.pbtxt index 078b471a4c6..cb38ae0b498 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.pbtxt @@ -8,14 +8,6 @@ tf_module { name: "AttrValue" mtype: "" } - member { - name: "ConditionalAccumulator" - mtype: "" - } - member { - name: "ConditionalAccumulatorBase" - mtype: "" - } member { name: "ConfigProto" mtype: "" @@ -24,14 +16,6 @@ tf_module { name: "DType" mtype: "" } - member { - name: "DeviceSpec" - mtype: "" - } - member { - name: "Dimension" - mtype: "" - } member { name: "Event" mtype: "" @@ -56,10 +40,6 @@ tf_module { name: "GraphDef" mtype: "" } - member { - name: "GraphKeys" - mtype: "" - } member { name: "GraphOptions" mtype: "" @@ -137,11 +117,11 @@ tf_module { mtype: "" } member { - name: "TensorInfo" - mtype: "" + name: "TensorShape" + mtype: "" } member { - name: "TensorShape" + name: "TensorSpec" mtype: "" } member { @@ -160,10 +140,6 @@ tf_module { name: "VariableSynchronization" mtype: "" } - member { - name: "app" - mtype: "" - } member { name: "bfloat16" mtype: "" @@ -200,6 +176,10 @@ tf_module { name: "debugging" mtype: "" } + member { + name: "distribute" + mtype: "" + } member { name: "double" mtype: "" @@ -224,10 +204,6 @@ tf_module { name: "feature_column" mtype: "" } - member { - name: "flags" - mtype: "" - } member { name: "float16" mtype: "" @@ -240,10 +216,6 @@ tf_module { name: "float64" mtype: "" } - member { - name: "gfile" - mtype: "" - } member { name: "glorot_uniform_initializer" mtype: "" @@ -296,10 +268,6 @@ tf_module { name: "lite" mtype: "" } - member { - name: "logging" - mtype: "" - } member { name: "losses" mtype: "" @@ -328,14 +296,6 @@ tf_module { name: "ones_initializer" mtype: "" } - member { - name: "profiler" - mtype: "" - } - member { - name: "pywrap_tensorflow" - mtype: "" - } member { name: "qint16" mtype: "" @@ -392,10 +352,6 @@ tf_module { name: "sparse" mtype: "" } - member { - name: "spectral" - mtype: "" - } member { name: "string" mtype: "" @@ -476,14 +432,6 @@ tf_module { name: "add_n" argspec: "args=[\'inputs\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " } - member_method { - name: "arg_max" - argspec: "args=[\'input\', \'dimension\', \'output_type\', \'name\'], varargs=None, keywords=None, defaults=[\"\", \'None\'], " - } - member_method { - name: "arg_min" - argspec: "args=[\'input\', \'dimension\', \'output_type\', \'name\'], varargs=None, keywords=None, defaults=[\"\", \'None\'], " - } member_method { name: "argmax" argspec: "args=[\'input\', \'axis\', \'output_type\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \"\", \'None\'], " @@ -492,6 +440,10 @@ tf_module { name: "argmin" argspec: "args=[\'input\', \'axis\', \'output_type\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \"\", \'None\'], " } + member_method { + name: "argsort" + argspec: "args=[\'values\', \'axis\', \'direction\', \'stable\', \'name\'], varargs=None, keywords=None, defaults=[\'-1\', \'ASCENDING\', \'False\', \'None\'], " + } member_method { name: "as_dtype" argspec: "args=[\'type_value\'], varargs=None, keywords=None, defaults=None" @@ -510,19 +462,19 @@ tf_module { } member_method { name: "assert_equal" - argspec: "args=[\'x\', \'y\', \'data\', \'summarize\', \'message\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], " + argspec: "args=[\'x\', \'y\', \'message\', \'summarize\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], " } member_method { name: "assert_greater" - argspec: "args=[\'x\', \'y\', \'data\', \'summarize\', \'message\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], " + argspec: "args=[\'x\', \'y\', \'message\', \'summarize\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], " } member_method { name: "assert_less" - argspec: "args=[\'x\', \'y\', \'data\', \'summarize\', \'message\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], " + argspec: "args=[\'x\', \'y\', \'message\', \'summarize\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], " } member_method { name: "assert_rank" - argspec: "args=[\'x\', \'rank\', \'data\', \'summarize\', \'message\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], " + argspec: "args=[\'x\', \'rank\', \'message\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " } member_method { name: "atan" @@ -546,10 +498,6 @@ tf_module { } member_method { name: "batch_to_space" - argspec: "args=[\'input\', \'crops\', \'block_size\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " - } - member_method { - name: "batch_to_space_nd" argspec: "args=[\'input\', \'block_shape\', \'crops\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " } member_method { @@ -580,10 +528,6 @@ tf_module { name: "cast" argspec: "args=[\'x\', \'dtype\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " } - member_method { - name: "clip_by_average_norm" - argspec: "args=[\'t\', \'clip_norm\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " - } member_method { name: "clip_by_global_norm" argspec: "args=[\'t_list\', \'clip_norm\', \'use_norm\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " @@ -606,11 +550,11 @@ tf_module { } member_method { name: "cond" - argspec: "args=[\'pred\', \'true_fn\', \'false_fn\', \'strict\', \'name\', \'fn1\', \'fn2\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'False\', \'None\', \'None\', \'None\'], " + argspec: "args=[\'pred\', \'true_fn\', \'false_fn\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], " } member_method { name: "constant" - argspec: "args=[\'value\', \'dtype\', \'shape\', \'name\', \'verify_shape\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'Const\', \'False\'], " + argspec: "args=[\'value\', \'dtype\', \'shape\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'Const\'], " } member_method { name: "control_dependencies" @@ -628,14 +572,6 @@ tf_module { name: "cosh" argspec: "args=[\'x\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " } - member_method { - name: "count_nonzero" - argspec: "args=[\'input_tensor\', \'axis\', \'keepdims\', \'dtype\', \'name\', \'reduction_indices\', \'keep_dims\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \"\", \'None\', \'None\', \'None\'], " - } - member_method { - name: "create_partitioned_variables" - argspec: "args=[\'shape\', \'slicing\', \'initializer\', \'dtype\', \'trainable\', \'collections\', \'name\', \'reuse\'], varargs=None, keywords=None, defaults=[\"\", \'True\', \'None\', \'None\', \'None\'], " - } member_method { name: "cumsum" argspec: "args=[\'x\', \'axis\', \'exclusive\', \'reverse\', \'name\'], varargs=None, keywords=None, defaults=[\'0\', \'False\', \'False\', \'None\'], " @@ -648,10 +584,6 @@ tf_module { name: "device" argspec: "args=[\'device_name\'], varargs=None, keywords=None, defaults=None" } - member_method { - name: "div" - argspec: "args=[\'x\', \'y\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " - } member_method { name: "div_no_nan" argspec: "args=[\'x\', \'y\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " @@ -676,10 +608,6 @@ tf_module { name: "einsum" argspec: "args=[\'equation\'], varargs=inputs, keywords=kwargs, defaults=None" } - member_method { - name: "enable_eager_execution" - argspec: "args=[\'config\', \'device_policy\', \'execution_mode\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], " - } member_method { name: "ensure_shape" argspec: "args=[\'x\', \'shape\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " @@ -698,7 +626,7 @@ tf_module { } member_method { name: "expand_dims" - argspec: "args=[\'input\', \'axis\', \'name\', \'dim\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], " + argspec: "args=[\'input\', \'axis\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " } member_method { name: "extract_volume_patches" @@ -712,10 +640,6 @@ tf_module { name: "fill" argspec: "args=[\'dims\', \'value\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " } - member_method { - name: "fixed_size_partitioner" - argspec: "args=[\'num_shards\', \'axis\'], varargs=None, keywords=None, defaults=[\'0\'], " - } member_method { name: "floor" argspec: "args=[\'x\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " @@ -738,31 +662,23 @@ tf_module { } member_method { name: "function" - argspec: "args=[\'func\', \'input_signature\', \'autograph\', \'experimental_autograph_options\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'False\', \'None\'], " + argspec: "args=[\'func\', \'input_signature\', \'autograph\', \'experimental_autograph_options\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'True\', \'None\'], " } member_method { name: "gather" - argspec: "args=[\'params\', \'indices\', \'validate_indices\', \'name\', \'axis\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'0\'], " + argspec: "args=[\'params\', \'indices\', \'validate_indices\', \'axis\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'0\', \'None\'], " } member_method { name: "gather_nd" argspec: "args=[\'params\', \'indices\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " } member_method { - name: "get_collection" - argspec: "args=[\'key\', \'scope\'], varargs=None, keywords=None, defaults=[\'None\'], " - } - member_method { - name: "get_collection_ref" - argspec: "args=[\'key\'], varargs=None, keywords=None, defaults=None" - } - member_method { - name: "get_default_graph" + name: "get_logger" argspec: "args=[], varargs=None, keywords=None, defaults=None" } member_method { name: "gradients" - argspec: "args=[\'ys\', \'xs\', \'grad_ys\', \'name\', \'colocate_gradients_with_ops\', \'gate_gradients\', \'aggregation_method\', \'stop_gradients\', \'unconnected_gradients\'], varargs=None, keywords=None, defaults=[\'None\', \'gradients\', \'False\', \'False\', \'None\', \'None\', \'UnconnectedGradients.NONE\'], " + argspec: "args=[\'ys\', \'xs\', \'grad_ys\', \'name\', \'gate_gradients\', \'aggregation_method\', \'stop_gradients\', \'unconnected_gradients\'], varargs=None, keywords=None, defaults=[\'None\', \'gradients\', \'False\', \'None\', \'None\', \'UnconnectedGradients.NONE\'], " } member_method { name: "greater" @@ -782,7 +698,7 @@ tf_module { } member_method { name: "hessians" - argspec: "args=[\'ys\', \'xs\', \'name\', \'colocate_gradients_with_ops\', \'gate_gradients\', \'aggregation_method\'], varargs=None, keywords=None, defaults=[\'hessians\', \'False\', \'False\', \'None\'], " + argspec: "args=[\'ys\', \'xs\', \'gate_gradients\', \'aggregation_method\', \'name\'], varargs=None, keywords=None, defaults=[\'False\', \'None\', \'hessians\'], " } member_method { name: "histogram_fixed_width" @@ -816,18 +732,10 @@ tf_module { name: "less_equal" argspec: "args=[\'x\', \'y\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " } - member_method { - name: "lin_space" - argspec: "args=[\'start\', \'stop\', \'num\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " - } member_method { name: "linspace" argspec: "args=[\'start\', \'stop\', \'num\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " } - member_method { - name: "load_file_system_library" - argspec: "args=[\'library_filename\'], varargs=None, keywords=None, defaults=None" - } member_method { name: "load_library" argspec: "args=[\'library_location\'], varargs=None, keywords=None, defaults=None" @@ -836,14 +744,6 @@ tf_module { name: "load_op_library" argspec: "args=[\'library_filename\'], varargs=None, keywords=None, defaults=None" } - member_method { - name: "log" - argspec: "args=[\'x\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " - } - member_method { - name: "log1p" - argspec: "args=[\'x\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " - } member_method { name: "logical_and" argspec: "args=[\'x\', \'y\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " @@ -880,10 +780,6 @@ tf_module { name: "meshgrid" argspec: "args=[], varargs=args, keywords=kwargs, defaults=None" } - member_method { - name: "min_max_variable_partitioner" - argspec: "args=[\'max_partitions\', \'axis\', \'min_slice_size\', \'bytes_per_string_element\'], varargs=None, keywords=None, defaults=[\'1\', \'0\', \'262144\', \'16\'], " - } member_method { name: "minimum" argspec: "args=[\'x\', \'y\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " @@ -892,10 +788,6 @@ tf_module { name: "mod" argspec: "args=[\'x\', \'y\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " } - member_method { - name: "multinomial" - argspec: "args=[\'logits\', \'num_samples\', \'seed\', \'name\', \'output_dtype\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], " - } member_method { name: "multiply" argspec: "args=[\'x\', \'y\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " @@ -918,7 +810,7 @@ tf_module { } member_method { name: "norm" - argspec: "args=[\'tensor\', \'ord\', \'axis\', \'keepdims\', \'name\', \'keep_dims\'], varargs=None, keywords=None, defaults=[\'euclidean\', \'None\', \'None\', \'None\', \'None\'], " + argspec: "args=[\'tensor\', \'ord\', \'axis\', \'keepdims\', \'name\'], varargs=None, keywords=None, defaults=[\'euclidean\', \'None\', \'None\', \'None\'], " } member_method { name: "not_equal" @@ -934,11 +826,11 @@ tf_module { } member_method { name: "ones_like" - argspec: "args=[\'tensor\', \'dtype\', \'name\', \'optimize\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'True\'], " + argspec: "args=[\'input\', \'dtype\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " } member_method { name: "pad" - argspec: "args=[\'tensor\', \'paddings\', \'mode\', \'name\', \'constant_values\'], varargs=None, keywords=None, defaults=[\'CONSTANT\', \'None\', \'0\'], " + argspec: "args=[\'tensor\', \'paddings\', \'mode\', \'constant_values\', \'name\'], varargs=None, keywords=None, defaults=[\'CONSTANT\', \'0\', \'None\'], " } member_method { name: "parallel_stack" @@ -956,10 +848,6 @@ tf_module { name: "py_function" argspec: "args=[\'func\', \'inp\', \'Tout\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " } - member_method { - name: "quantize_v2" - argspec: "args=[\'input\', \'min_range\', \'max_range\', \'T\', \'mode\', \'name\', \'round_mode\'], varargs=None, keywords=None, defaults=[\'MIN_COMBINED\', \'None\', \'HALF_AWAY_FROM_ZERO\'], " - } member_method { name: "range" argspec: "args=[\'start\', \'limit\', \'delta\', \'dtype\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'1\', \'None\', \'range\'], " @@ -974,35 +862,35 @@ tf_module { } member_method { name: "reduce_all" - argspec: "args=[\'input_tensor\', \'axis\', \'keepdims\', \'name\', \'reduction_indices\', \'keep_dims\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\'], " + argspec: "args=[\'input_tensor\', \'axis\', \'keepdims\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\'], " } member_method { name: "reduce_any" - argspec: "args=[\'input_tensor\', \'axis\', \'keepdims\', \'name\', \'reduction_indices\', \'keep_dims\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\'], " + argspec: "args=[\'input_tensor\', \'axis\', \'keepdims\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\'], " } member_method { name: "reduce_logsumexp" - argspec: "args=[\'input_tensor\', \'axis\', \'keepdims\', \'name\', \'reduction_indices\', \'keep_dims\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\'], " + argspec: "args=[\'input_tensor\', \'axis\', \'keepdims\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\'], " } member_method { name: "reduce_max" - argspec: "args=[\'input_tensor\', \'axis\', \'keepdims\', \'name\', \'reduction_indices\', \'keep_dims\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\'], " + argspec: "args=[\'input_tensor\', \'axis\', \'keepdims\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\'], " } member_method { name: "reduce_mean" - argspec: "args=[\'input_tensor\', \'axis\', \'keepdims\', \'name\', \'reduction_indices\', \'keep_dims\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\'], " + argspec: "args=[\'input_tensor\', \'axis\', \'keepdims\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\'], " } member_method { name: "reduce_min" - argspec: "args=[\'input_tensor\', \'axis\', \'keepdims\', \'name\', \'reduction_indices\', \'keep_dims\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\'], " + argspec: "args=[\'input_tensor\', \'axis\', \'keepdims\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\'], " } member_method { name: "reduce_prod" - argspec: "args=[\'input_tensor\', \'axis\', \'keepdims\', \'name\', \'reduction_indices\', \'keep_dims\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\'], " + argspec: "args=[\'input_tensor\', \'axis\', \'keepdims\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\'], " } member_method { name: "reduce_sum" - argspec: "args=[\'input_tensor\', \'axis\', \'keepdims\', \'name\', \'reduction_indices\', \'keep_dims\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\'], " + argspec: "args=[\'input_tensor\', \'axis\', \'keepdims\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\'], " } member_method { name: "register_tensor_conversion_function" @@ -1012,10 +900,6 @@ tf_module { name: "required_space_to_batch_paddings" argspec: "args=[\'input_shape\', \'block_shape\', \'base_paddings\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " } - member_method { - name: "reset_default_graph" - argspec: "args=[], varargs=None, keywords=None, defaults=None" - } member_method { name: "reshape" argspec: "args=[\'tensor\', \'shape\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " @@ -1026,7 +910,7 @@ tf_module { } member_method { name: "reverse_sequence" - argspec: "args=[\'input\', \'seq_lengths\', \'seq_axis\', \'batch_axis\', \'name\', \'seq_dim\', \'batch_dim\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\'], " + argspec: "args=[\'input\', \'seq_lengths\', \'seq_axis\', \'batch_axis\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], " } member_method { name: "roll" @@ -1042,7 +926,7 @@ tf_module { } member_method { name: "scalar_mul" - argspec: "args=[\'scalar\', \'x\'], varargs=None, keywords=None, defaults=None" + argspec: "args=[\'scalar\', \'x\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " } member_method { name: "scan" @@ -1076,13 +960,9 @@ tf_module { name: "sequence_mask" argspec: "args=[\'lengths\', \'maxlen\', \'dtype\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \"\", \'None\'], " } - member_method { - name: "set_random_seed" - argspec: "args=[\'seed\'], varargs=None, keywords=None, defaults=None" - } member_method { name: "shape" - argspec: "args=[\'input\', \'name\', \'out_type\'], varargs=None, keywords=None, defaults=[\'None\', \"\"], " + argspec: "args=[\'input\', \'out_type\', \'name\'], varargs=None, keywords=None, defaults=[\"\", \'None\'], " } member_method { name: "shape_n" @@ -1106,44 +986,20 @@ tf_module { } member_method { name: "size" - argspec: "args=[\'input\', \'name\', \'out_type\'], varargs=None, keywords=None, defaults=[\'None\', \"\"], " + argspec: "args=[\'input\', \'out_type\', \'name\'], varargs=None, keywords=None, defaults=[\"\", \'None\'], " } member_method { name: "slice" argspec: "args=[\'input_\', \'begin\', \'size\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "sort" + argspec: "args=[\'values\', \'axis\', \'direction\', \'name\'], varargs=None, keywords=None, defaults=[\'-1\', \'ASCENDING\', \'None\'], " + } member_method { name: "space_to_batch_nd" argspec: "args=[\'input\', \'block_shape\', \'paddings\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " } - member_method { - name: "sparse_concat" - argspec: "args=[\'axis\', \'sp_inputs\', \'name\', \'expand_nonconcat_dim\', \'concat_dim\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\'], " - } - member_method { - name: "sparse_reduce_max" - argspec: "args=[\'sp_input\', \'axis\', \'keepdims\', \'reduction_axes\', \'keep_dims\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], " - } - member_method { - name: "sparse_reduce_max_sparse" - argspec: "args=[\'sp_input\', \'axis\', \'keepdims\', \'reduction_axes\', \'keep_dims\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], " - } - member_method { - name: "sparse_reduce_sum" - argspec: "args=[\'sp_input\', \'axis\', \'keepdims\', \'reduction_axes\', \'keep_dims\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], " - } - member_method { - name: "sparse_reduce_sum_sparse" - argspec: "args=[\'sp_input\', \'axis\', \'keepdims\', \'reduction_axes\', \'keep_dims\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], " - } - member_method { - name: "sparse_split" - argspec: "args=[\'keyword_required\', \'sp_input\', \'num_split\', \'axis\', \'name\', \'split_dim\'], varargs=None, keywords=None, defaults=[\'KeywordRequired()\', \'None\', \'None\', \'None\', \'None\', \'None\'], " - } - member_method { - name: "sparse_to_dense" - argspec: "args=[\'sparse_indices\', \'output_shape\', \'sparse_values\', \'default_value\', \'validate_indices\', \'name\'], varargs=None, keywords=None, defaults=[\'0\', \'True\', \'None\'], " - } member_method { name: "split" argspec: "args=[\'value\', \'num_or_size_splits\', \'axis\', \'num\', \'name\'], varargs=None, keywords=None, defaults=[\'0\', \'None\', \'split\'], " @@ -1158,7 +1014,7 @@ tf_module { } member_method { name: "squeeze" - argspec: "args=[\'input\', \'axis\', \'name\', \'squeeze_dims\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], " + argspec: "args=[\'input\', \'axis\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " } member_method { name: "stack" @@ -1192,6 +1048,18 @@ tf_module { name: "tanh" argspec: "args=[\'x\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "tensor_scatter_add" + argspec: "args=[\'tensor\', \'indices\', \'updates\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "tensor_scatter_sub" + argspec: "args=[\'tensor\', \'indices\', \'updates\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "tensor_scatter_update" + argspec: "args=[\'tensor\', \'indices\', \'updates\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " + } member_method { name: "tensordot" argspec: "args=[\'a\', \'b\', \'axes\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " @@ -1206,16 +1074,12 @@ tf_module { } member_method { name: "transpose" - argspec: "args=[\'a\', \'perm\', \'name\', \'conjugate\'], varargs=None, keywords=None, defaults=[\'None\', \'transpose\', \'False\'], " + argspec: "args=[\'a\', \'perm\', \'conjugate\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'transpose\'], " } member_method { name: "truediv" argspec: "args=[\'x\', \'y\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " } - member_method { - name: "truncated_normal" - argspec: "args=[\'shape\', \'mean\', \'stddev\', \'dtype\', \'seed\', \'name\'], varargs=None, keywords=None, defaults=[\'0.0\', \'1.0\', \"\", \'None\', \'None\'], " - } member_method { name: "truncatediv" argspec: "args=[\'x\', \'y\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " @@ -1226,7 +1090,7 @@ tf_module { } member_method { name: "tuple" - argspec: "args=[\'tensors\', \'name\', \'control_inputs\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + argspec: "args=[\'tensors\', \'control_inputs\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " } member_method { name: "unique" @@ -1244,10 +1108,6 @@ tf_module { name: "unstack" argspec: "args=[\'value\', \'num\', \'axis\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'0\', \'unstack\'], " } - member_method { - name: "variable_axis_size_partitioner" - argspec: "args=[\'max_shard_bytes\', \'axis\', \'bytes_per_string_element\', \'max_shards\'], varargs=None, keywords=None, defaults=[\'0\', \'16\', \'None\'], " - } member_method { name: "variable_creator_scope" argspec: "args=[\'variable_creator\'], varargs=None, keywords=None, defaults=None" @@ -1258,7 +1118,7 @@ tf_module { } member_method { name: "while_loop" - argspec: "args=[\'cond\', \'body\', \'loop_vars\', \'shape_invariants\', \'parallel_iterations\', \'back_prop\', \'swap_memory\', \'name\', \'maximum_iterations\', \'return_same_structure\'], varargs=None, keywords=None, defaults=[\'None\', \'10\', \'True\', \'False\', \'None\', \'None\', \'False\'], " + argspec: "args=[\'cond\', \'body\', \'loop_vars\', \'shape_invariants\', \'parallel_iterations\', \'back_prop\', \'swap_memory\', \'maximum_iterations\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'10\', \'True\', \'False\', \'None\', \'None\'], " } member_method { name: "zeros" @@ -1266,6 +1126,6 @@ tf_module { } member_method { name: "zeros_like" - argspec: "args=[\'tensor\', \'dtype\', \'name\', \'optimize\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'True\'], " + argspec: "args=[\'input\', \'dtype\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " } } diff --git a/tensorflow/tools/api/golden/v2/tensorflow.profiler.-advice-proto.-checker.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.profiler.-advice-proto.-checker.pbtxt deleted file mode 100644 index e09c44cc9ce..00000000000 --- a/tensorflow/tools/api/golden/v2/tensorflow.profiler.-advice-proto.-checker.pbtxt +++ /dev/null @@ -1,12 +0,0 @@ -path: "tensorflow.profiler.AdviceProto.Checker" -tf_proto { - descriptor { - name: "Checker" - field { - name: "reports" - number: 2 - label: LABEL_REPEATED - type: TYPE_STRING - } - } -} diff --git a/tensorflow/tools/api/golden/v2/tensorflow.profiler.-advice-proto.-checkers-entry.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.profiler.-advice-proto.-checkers-entry.pbtxt deleted file mode 100644 index 87462435496..00000000000 --- a/tensorflow/tools/api/golden/v2/tensorflow.profiler.-advice-proto.-checkers-entry.pbtxt +++ /dev/null @@ -1,22 +0,0 @@ -path: "tensorflow.profiler.AdviceProto.CheckersEntry" -tf_proto { - descriptor { - name: "CheckersEntry" - field { - name: "key" - number: 1 - label: LABEL_OPTIONAL - type: TYPE_STRING - } - field { - name: "value" - number: 2 - label: LABEL_OPTIONAL - type: TYPE_MESSAGE - type_name: ".tensorflow.tfprof.AdviceProto.Checker" - } - options { - map_entry: true - } - } -} diff --git a/tensorflow/tools/api/golden/v2/tensorflow.profiler.-advice-proto.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.profiler.-advice-proto.pbtxt deleted file mode 100644 index a8a8858ccd5..00000000000 --- a/tensorflow/tools/api/golden/v2/tensorflow.profiler.-advice-proto.pbtxt +++ /dev/null @@ -1,41 +0,0 @@ -path: "tensorflow.profiler.AdviceProto" -tf_proto { - descriptor { - name: "AdviceProto" - field { - name: "checkers" - number: 1 - label: LABEL_REPEATED - type: TYPE_MESSAGE - type_name: ".tensorflow.tfprof.AdviceProto.CheckersEntry" - } - nested_type { - name: "CheckersEntry" - field { - name: "key" - number: 1 - label: LABEL_OPTIONAL - type: TYPE_STRING - } - field { - name: "value" - number: 2 - label: LABEL_OPTIONAL - type: TYPE_MESSAGE - type_name: ".tensorflow.tfprof.AdviceProto.Checker" - } - options { - map_entry: true - } - } - nested_type { - name: "Checker" - field { - name: "reports" - number: 2 - label: LABEL_REPEATED - type: TYPE_STRING - } - } - } -} diff --git a/tensorflow/tools/api/golden/v2/tensorflow.profiler.-graph-node-proto.-input-shapes-entry.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.profiler.-graph-node-proto.-input-shapes-entry.pbtxt deleted file mode 100644 index afec73f537a..00000000000 --- a/tensorflow/tools/api/golden/v2/tensorflow.profiler.-graph-node-proto.-input-shapes-entry.pbtxt +++ /dev/null @@ -1,22 +0,0 @@ -path: "tensorflow.profiler.GraphNodeProto.InputShapesEntry" -tf_proto { - descriptor { - name: "InputShapesEntry" - field { - name: "key" - number: 1 - label: LABEL_OPTIONAL - type: TYPE_INT32 - } - field { - name: "value" - number: 2 - label: LABEL_OPTIONAL - type: TYPE_MESSAGE - type_name: ".tensorflow.TensorShapeProto" - } - options { - map_entry: true - } - } -} diff --git a/tensorflow/tools/api/golden/v2/tensorflow.profiler.-graph-node-proto.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.profiler.-graph-node-proto.pbtxt deleted file mode 100644 index 3c831770053..00000000000 --- a/tensorflow/tools/api/golden/v2/tensorflow.profiler.-graph-node-proto.pbtxt +++ /dev/null @@ -1,191 +0,0 @@ -path: "tensorflow.profiler.GraphNodeProto" -tf_proto { - descriptor { - name: "GraphNodeProto" - field { - name: "name" - number: 1 - label: LABEL_OPTIONAL - type: TYPE_STRING - } - field { - name: "tensor_value" - number: 15 - label: LABEL_OPTIONAL - type: TYPE_MESSAGE - type_name: ".tensorflow.tfprof.TFProfTensorProto" - } - field { - name: "run_count" - number: 21 - label: LABEL_OPTIONAL - type: TYPE_INT64 - } - field { - name: "exec_micros" - number: 2 - label: LABEL_OPTIONAL - type: TYPE_INT64 - } - field { - name: "accelerator_exec_micros" - number: 17 - label: LABEL_OPTIONAL - type: TYPE_INT64 - } - field { - name: "cpu_exec_micros" - number: 18 - label: LABEL_OPTIONAL - type: TYPE_INT64 - } - field { - name: "requested_bytes" - number: 3 - label: LABEL_OPTIONAL - type: TYPE_INT64 - } - field { - name: "peak_bytes" - number: 24 - label: LABEL_OPTIONAL - type: TYPE_INT64 - } - field { - name: "residual_bytes" - number: 25 - label: LABEL_OPTIONAL - type: TYPE_INT64 - } - field { - name: "output_bytes" - number: 26 - label: LABEL_OPTIONAL - type: TYPE_INT64 - } - field { - name: "parameters" - number: 4 - label: LABEL_OPTIONAL - type: TYPE_INT64 - } - field { - name: "float_ops" - number: 13 - label: LABEL_OPTIONAL - type: TYPE_INT64 - } - field { - name: "devices" - number: 10 - label: LABEL_REPEATED - type: TYPE_STRING - } - field { - name: "total_definition_count" - number: 23 - label: LABEL_OPTIONAL - type: TYPE_INT64 - } - field { - name: "total_run_count" - number: 22 - label: LABEL_OPTIONAL - type: TYPE_INT64 - } - field { - name: "total_exec_micros" - number: 6 - label: LABEL_OPTIONAL - type: TYPE_INT64 - } - field { - name: "total_accelerator_exec_micros" - number: 19 - label: LABEL_OPTIONAL - type: TYPE_INT64 - } - field { - name: "total_cpu_exec_micros" - number: 20 - label: LABEL_OPTIONAL - type: TYPE_INT64 - } - field { - name: "total_requested_bytes" - number: 7 - label: LABEL_OPTIONAL - type: TYPE_INT64 - } - field { - name: "total_peak_bytes" - number: 27 - label: LABEL_OPTIONAL - type: TYPE_INT64 - } - field { - name: "total_residual_bytes" - number: 28 - label: LABEL_OPTIONAL - type: TYPE_INT64 - } - field { - name: "total_output_bytes" - number: 29 - label: LABEL_OPTIONAL - type: TYPE_INT64 - } - field { - name: "total_parameters" - number: 8 - label: LABEL_OPTIONAL - type: TYPE_INT64 - } - field { - name: "total_float_ops" - number: 14 - label: LABEL_OPTIONAL - type: TYPE_INT64 - } - field { - name: "shapes" - number: 11 - label: LABEL_REPEATED - type: TYPE_MESSAGE - type_name: ".tensorflow.TensorShapeProto" - } - field { - name: "input_shapes" - number: 16 - label: LABEL_REPEATED - type: TYPE_MESSAGE - type_name: ".tensorflow.tfprof.GraphNodeProto.InputShapesEntry" - } - field { - name: "children" - number: 12 - label: LABEL_REPEATED - type: TYPE_MESSAGE - type_name: ".tensorflow.tfprof.GraphNodeProto" - } - nested_type { - name: "InputShapesEntry" - field { - name: "key" - number: 1 - label: LABEL_OPTIONAL - type: TYPE_INT32 - } - field { - name: "value" - number: 2 - label: LABEL_OPTIONAL - type: TYPE_MESSAGE - type_name: ".tensorflow.TensorShapeProto" - } - options { - map_entry: true - } - } - } -} diff --git a/tensorflow/tools/api/golden/v2/tensorflow.profiler.-multi-graph-node-proto.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.profiler.-multi-graph-node-proto.pbtxt deleted file mode 100644 index 2b08a05437f..00000000000 --- a/tensorflow/tools/api/golden/v2/tensorflow.profiler.-multi-graph-node-proto.pbtxt +++ /dev/null @@ -1,134 +0,0 @@ -path: "tensorflow.profiler.MultiGraphNodeProto" -tf_proto { - descriptor { - name: "MultiGraphNodeProto" - field { - name: "name" - number: 1 - label: LABEL_OPTIONAL - type: TYPE_STRING - } - field { - name: "exec_micros" - number: 2 - label: LABEL_OPTIONAL - type: TYPE_INT64 - } - field { - name: "accelerator_exec_micros" - number: 12 - label: LABEL_OPTIONAL - type: TYPE_INT64 - } - field { - name: "cpu_exec_micros" - number: 13 - label: LABEL_OPTIONAL - type: TYPE_INT64 - } - field { - name: "requested_bytes" - number: 3 - label: LABEL_OPTIONAL - type: TYPE_INT64 - } - field { - name: "peak_bytes" - number: 16 - label: LABEL_OPTIONAL - type: TYPE_INT64 - } - field { - name: "residual_bytes" - number: 17 - label: LABEL_OPTIONAL - type: TYPE_INT64 - } - field { - name: "output_bytes" - number: 18 - label: LABEL_OPTIONAL - type: TYPE_INT64 - } - field { - name: "parameters" - number: 4 - label: LABEL_OPTIONAL - type: TYPE_INT64 - } - field { - name: "float_ops" - number: 5 - label: LABEL_OPTIONAL - type: TYPE_INT64 - } - field { - name: "total_exec_micros" - number: 6 - label: LABEL_OPTIONAL - type: TYPE_INT64 - } - field { - name: "total_accelerator_exec_micros" - number: 14 - label: LABEL_OPTIONAL - type: TYPE_INT64 - } - field { - name: "total_cpu_exec_micros" - number: 15 - label: LABEL_OPTIONAL - type: TYPE_INT64 - } - field { - name: "total_requested_bytes" - number: 7 - label: LABEL_OPTIONAL - type: TYPE_INT64 - } - field { - name: "total_peak_bytes" - number: 19 - label: LABEL_OPTIONAL - type: TYPE_INT64 - } - field { - name: "total_residual_bytes" - number: 20 - label: LABEL_OPTIONAL - type: TYPE_INT64 - } - field { - name: "total_output_bytes" - number: 21 - label: LABEL_OPTIONAL - type: TYPE_INT64 - } - field { - name: "total_parameters" - number: 8 - label: LABEL_OPTIONAL - type: TYPE_INT64 - } - field { - name: "total_float_ops" - number: 9 - label: LABEL_OPTIONAL - type: TYPE_INT64 - } - field { - name: "graph_nodes" - number: 10 - label: LABEL_REPEATED - type: TYPE_MESSAGE - type_name: ".tensorflow.tfprof.GraphNodeProto" - } - field { - name: "children" - number: 11 - label: LABEL_REPEATED - type: TYPE_MESSAGE - type_name: ".tensorflow.tfprof.MultiGraphNodeProto" - } - } -} diff --git a/tensorflow/tools/api/golden/v2/tensorflow.profiler.-op-log-proto.-id-to-string-entry.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.profiler.-op-log-proto.-id-to-string-entry.pbtxt deleted file mode 100644 index b3adc50c7e1..00000000000 --- a/tensorflow/tools/api/golden/v2/tensorflow.profiler.-op-log-proto.-id-to-string-entry.pbtxt +++ /dev/null @@ -1,21 +0,0 @@ -path: "tensorflow.profiler.OpLogProto.IdToStringEntry" -tf_proto { - descriptor { - name: "IdToStringEntry" - field { - name: "key" - number: 1 - label: LABEL_OPTIONAL - type: TYPE_INT64 - } - field { - name: "value" - number: 2 - label: LABEL_OPTIONAL - type: TYPE_STRING - } - options { - map_entry: true - } - } -} diff --git a/tensorflow/tools/api/golden/v2/tensorflow.profiler.-op-log-proto.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.profiler.-op-log-proto.pbtxt deleted file mode 100644 index 7510c566ba5..00000000000 --- a/tensorflow/tools/api/golden/v2/tensorflow.profiler.-op-log-proto.pbtxt +++ /dev/null @@ -1,38 +0,0 @@ -path: "tensorflow.profiler.OpLogProto" -tf_proto { - descriptor { - name: "OpLogProto" - field { - name: "log_entries" - number: 1 - label: LABEL_REPEATED - type: TYPE_MESSAGE - type_name: ".tensorflow.tfprof.OpLogEntry" - } - field { - name: "id_to_string" - number: 2 - label: LABEL_REPEATED - type: TYPE_MESSAGE - type_name: ".tensorflow.tfprof.OpLogProto.IdToStringEntry" - } - nested_type { - name: "IdToStringEntry" - field { - name: "key" - number: 1 - label: LABEL_OPTIONAL - type: TYPE_INT64 - } - field { - name: "value" - number: 2 - label: LABEL_OPTIONAL - type: TYPE_STRING - } - options { - map_entry: true - } - } - } -} diff --git a/tensorflow/tools/api/golden/v2/tensorflow.profiler.-profile-option-builder.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.profiler.-profile-option-builder.pbtxt deleted file mode 100644 index 19ff38a3900..00000000000 --- a/tensorflow/tools/api/golden/v2/tensorflow.profiler.-profile-option-builder.pbtxt +++ /dev/null @@ -1,93 +0,0 @@ -path: "tensorflow.profiler.ProfileOptionBuilder" -tf_class { - is_instance: "" - is_instance: "" - member_method { - name: "__init__" - argspec: "args=[\'self\', \'options\'], varargs=None, keywords=None, defaults=[\'None\'], " - } - member_method { - name: "account_displayed_op_only" - argspec: "args=[\'self\', \'is_true\'], varargs=None, keywords=None, defaults=None" - } - member_method { - name: "build" - argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" - } - member_method { - name: "float_operation" - argspec: "args=[], varargs=None, keywords=None, defaults=None" - } - member_method { - name: "order_by" - argspec: "args=[\'self\', \'attribute\'], varargs=None, keywords=None, defaults=None" - } - member_method { - name: "select" - argspec: "args=[\'self\', \'attributes\'], varargs=None, keywords=None, defaults=None" - } - member_method { - name: "time_and_memory" - argspec: "args=[\'min_micros\', \'min_bytes\', \'min_accelerator_micros\', \'min_cpu_micros\', \'min_peak_bytes\', \'min_residual_bytes\', \'min_output_bytes\'], varargs=None, keywords=None, defaults=[\'1\', \'1\', \'0\', \'0\', \'0\', \'0\', \'0\'], " - } - member_method { - name: "trainable_variables_parameter" - argspec: "args=[], varargs=None, keywords=None, defaults=None" - } - member_method { - name: "with_accounted_types" - argspec: "args=[\'self\', \'account_type_regexes\'], varargs=None, keywords=None, defaults=None" - } - member_method { - name: "with_empty_output" - argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" - } - member_method { - name: "with_file_output" - argspec: "args=[\'self\', \'outfile\'], varargs=None, keywords=None, defaults=None" - } - member_method { - name: "with_max_depth" - argspec: "args=[\'self\', \'max_depth\'], varargs=None, keywords=None, defaults=None" - } - member_method { - name: "with_min_execution_time" - argspec: "args=[\'self\', \'min_micros\', \'min_accelerator_micros\', \'min_cpu_micros\'], varargs=None, keywords=None, defaults=[\'0\', \'0\', \'0\'], " - } - member_method { - name: "with_min_float_operations" - argspec: "args=[\'self\', \'min_float_ops\'], varargs=None, keywords=None, defaults=None" - } - member_method { - name: "with_min_memory" - argspec: "args=[\'self\', \'min_bytes\', \'min_peak_bytes\', \'min_residual_bytes\', \'min_output_bytes\'], varargs=None, keywords=None, defaults=[\'0\', \'0\', \'0\', \'0\'], " - } - member_method { - name: "with_min_occurrence" - argspec: "args=[\'self\', \'min_occurrence\'], varargs=None, keywords=None, defaults=None" - } - member_method { - name: "with_min_parameters" - argspec: "args=[\'self\', \'min_params\'], varargs=None, keywords=None, defaults=None" - } - member_method { - name: "with_node_names" - argspec: "args=[\'self\', \'start_name_regexes\', \'show_name_regexes\', \'hide_name_regexes\', \'trim_name_regexes\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], " - } - member_method { - name: "with_pprof_output" - argspec: "args=[\'self\', \'pprof_file\'], varargs=None, keywords=None, defaults=None" - } - member_method { - name: "with_stdout_output" - argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" - } - member_method { - name: "with_step" - argspec: "args=[\'self\', \'step\'], varargs=None, keywords=None, defaults=None" - } - member_method { - name: "with_timeline_output" - argspec: "args=[\'self\', \'timeline_file\'], varargs=None, keywords=None, defaults=None" - } -} diff --git a/tensorflow/tools/api/golden/v2/tensorflow.profiler.-profiler.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.profiler.-profiler.pbtxt deleted file mode 100644 index acb61dae9f0..00000000000 --- a/tensorflow/tools/api/golden/v2/tensorflow.profiler.-profiler.pbtxt +++ /dev/null @@ -1,37 +0,0 @@ -path: "tensorflow.profiler.Profiler" -tf_class { - is_instance: "" - is_instance: "" - member_method { - name: "__init__" - argspec: "args=[\'self\', \'graph\', \'op_log\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " - } - member_method { - name: "add_step" - argspec: "args=[\'self\', \'step\', \'run_meta\'], varargs=None, keywords=None, defaults=None" - } - member_method { - name: "advise" - argspec: "args=[\'self\', \'options\'], varargs=None, keywords=None, defaults=None" - } - member_method { - name: "profile_graph" - argspec: "args=[\'self\', \'options\'], varargs=None, keywords=None, defaults=None" - } - member_method { - name: "profile_name_scope" - argspec: "args=[\'self\', \'options\'], varargs=None, keywords=None, defaults=None" - } - member_method { - name: "profile_operations" - argspec: "args=[\'self\', \'options\'], varargs=None, keywords=None, defaults=None" - } - member_method { - name: "profile_python" - argspec: "args=[\'self\', \'options\'], varargs=None, keywords=None, defaults=None" - } - member_method { - name: "serialize_to_string" - argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" - } -} diff --git a/tensorflow/tools/api/golden/v2/tensorflow.profiler.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.profiler.pbtxt deleted file mode 100644 index 7b4d3ac522a..00000000000 --- a/tensorflow/tools/api/golden/v2/tensorflow.profiler.pbtxt +++ /dev/null @@ -1,39 +0,0 @@ -path: "tensorflow.profiler" -tf_module { - member { - name: "AdviceProto" - mtype: "" - } - member { - name: "GraphNodeProto" - mtype: "" - } - member { - name: "MultiGraphNodeProto" - mtype: "" - } - member { - name: "OpLogProto" - mtype: "" - } - member { - name: "ProfileOptionBuilder" - mtype: "" - } - member { - name: "Profiler" - mtype: "" - } - member_method { - name: "advise" - argspec: "args=[\'graph\', \'run_meta\', \'options\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'0\'], " - } - member_method { - name: "profile" - argspec: "args=[\'graph\', \'run_meta\', \'op_log\', \'cmd\', \'options\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'scope\', \'0\'], " - } - member_method { - name: "write_op_log" - argspec: "args=[\'graph\', \'log_dir\', \'op_log\', \'run_meta\', \'add_trace\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'True\'], " - } -} diff --git a/tensorflow/tools/api/golden/v2/tensorflow.quantization.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.quantization.pbtxt index 2948b7318ea..632c2f8f83c 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.quantization.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.quantization.pbtxt @@ -34,7 +34,7 @@ tf_module { } member_method { name: "quantize_and_dequantize" - argspec: "args=[\'input\', \'input_min\', \'input_max\', \'signed_input\', \'num_bits\', \'range_given\', \'name\'], varargs=None, keywords=None, defaults=[\'True\', \'8\', \'False\', \'None\'], " + argspec: "args=[\'input\', \'input_min\', \'input_max\', \'signed_input\', \'num_bits\', \'range_given\', \'round_mode\', \'name\'], varargs=None, keywords=None, defaults=[\'True\', \'8\', \'False\', \'HALF_TO_EVEN\', \'None\'], " } member_method { name: "quantized_concat" diff --git a/tensorflow/tools/api/golden/v2/tensorflow.random.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.random.pbtxt index 160c09798d0..de5cb6b7172 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.random.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.random.pbtxt @@ -1,31 +1,35 @@ path: "tensorflow.random" tf_module { + member_method { + name: "all_candidate_sampler" + argspec: "args=[\'true_classes\', \'num_true\', \'num_sampled\', \'unique\', \'seed\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } + member_method { + name: "categorical" + argspec: "args=[\'logits\', \'num_samples\', \'dtype\', \'seed\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], " + } + member_method { + name: "fixed_unigram_candidate_sampler" + argspec: "args=[\'true_classes\', \'num_true\', \'num_sampled\', \'unique\', \'range_max\', \'vocab_file\', \'distortion\', \'num_reserved_ids\', \'num_shards\', \'shard\', \'unigrams\', \'seed\', \'name\'], varargs=None, keywords=None, defaults=[\'\', \'1.0\', \'0\', \'1\', \'0\', \'()\', \'None\', \'None\'], " + } member_method { name: "gamma" argspec: "args=[\'shape\', \'alpha\', \'beta\', \'dtype\', \'seed\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \"\", \'None\', \'None\'], " } - member_method { - name: "get_seed" - argspec: "args=[\'op_seed\'], varargs=None, keywords=None, defaults=None" - } member_method { name: "log_uniform_candidate_sampler" argspec: "args=[\'true_classes\', \'num_true\', \'num_sampled\', \'unique\', \'range_max\', \'seed\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " } - member_method { - name: "multinomial" - argspec: "args=[\'logits\', \'num_samples\', \'seed\', \'name\', \'output_dtype\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], " - } member_method { name: "normal" argspec: "args=[\'shape\', \'mean\', \'stddev\', \'dtype\', \'seed\', \'name\'], varargs=None, keywords=None, defaults=[\'0.0\', \'1.0\', \"\", \'None\', \'None\'], " } member_method { name: "poisson" - argspec: "args=[\'lam\', \'shape\', \'dtype\', \'seed\', \'name\'], varargs=None, keywords=None, defaults=[\"\", \'None\', \'None\'], " + argspec: "args=[\'shape\', \'lam\', \'dtype\', \'seed\', \'name\'], varargs=None, keywords=None, defaults=[\"\", \'None\', \'None\'], " } member_method { - name: "set_random_seed" + name: "set_seed" argspec: "args=[\'seed\'], varargs=None, keywords=None, defaults=None" } member_method { @@ -33,8 +37,8 @@ tf_module { argspec: "args=[\'value\', \'seed\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " } member_method { - name: "stateless_multinomial" - argspec: "args=[\'logits\', \'num_samples\', \'seed\', \'output_dtype\', \'name\'], varargs=None, keywords=None, defaults=[\"\", \'None\'], " + name: "stateless_categorical" + argspec: "args=[\'logits\', \'num_samples\', \'seed\', \'dtype\', \'name\'], varargs=None, keywords=None, defaults=[\"\", \'None\'], " } member_method { name: "stateless_normal" diff --git a/tensorflow/tools/api/golden/v2/tensorflow.saved_model.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.saved_model.pbtxt index d57936a2f1c..63bebb20bca 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.saved_model.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.saved_model.pbtxt @@ -32,14 +32,6 @@ tf_module { name: "GPU" mtype: "" } - member { - name: "LEGACY_INIT_OP_KEY" - mtype: "" - } - member { - name: "MAIN_OP_KEY" - mtype: "" - } member { name: "PREDICT_INPUTS" mtype: "" @@ -105,12 +97,12 @@ tf_module { argspec: "args=[\'examples\', \'classes\', \'scores\'], varargs=None, keywords=None, defaults=None" } member_method { - name: "is_valid_signature" - argspec: "args=[\'signature_def\'], varargs=None, keywords=None, defaults=None" + name: "contains_saved_model" + argspec: "args=[\'export_dir\'], varargs=None, keywords=None, defaults=None" } member_method { - name: "maybe_saved_model_directory" - argspec: "args=[\'export_dir\'], varargs=None, keywords=None, defaults=None" + name: "is_valid_signature" + argspec: "args=[\'signature_def\'], varargs=None, keywords=None, defaults=None" } member_method { name: "predict_signature_def" diff --git a/tensorflow/tools/api/golden/v2/tensorflow.sets.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.sets.pbtxt index 8a196b1a556..900d08ff47c 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.sets.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.sets.pbtxt @@ -1,19 +1,19 @@ path: "tensorflow.sets" tf_module { member_method { - name: "set_difference" + name: "difference" argspec: "args=[\'a\', \'b\', \'aminusb\', \'validate_indices\'], varargs=None, keywords=None, defaults=[\'True\', \'True\'], " } member_method { - name: "set_intersection" + name: "intersection" argspec: "args=[\'a\', \'b\', \'validate_indices\'], varargs=None, keywords=None, defaults=[\'True\'], " } member_method { - name: "set_size" + name: "size" argspec: "args=[\'a\', \'validate_indices\'], varargs=None, keywords=None, defaults=[\'True\'], " } member_method { - name: "set_union" + name: "union" argspec: "args=[\'a\', \'b\', \'validate_indices\'], varargs=None, keywords=None, defaults=[\'True\'], " } } diff --git a/tensorflow/tools/api/golden/v2/tensorflow.signal.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.signal.pbtxt index 2c50c41f186..ea717b4d719 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.signal.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.signal.pbtxt @@ -1,5 +1,9 @@ path: "tensorflow.signal" tf_module { + member_method { + name: "dct" + argspec: "args=[\'input\', \'type\', \'n\', \'axis\', \'norm\', \'name\'], varargs=None, keywords=None, defaults=[\'2\', \'None\', \'-1\', \'None\', \'None\'], " + } member_method { name: "fft" argspec: "args=[\'input\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " @@ -24,6 +28,10 @@ tf_module { name: "hann_window" argspec: "args=[\'window_length\', \'periodic\', \'dtype\', \'name\'], varargs=None, keywords=None, defaults=[\'True\', \"\", \'None\'], " } + member_method { + name: "idct" + argspec: "args=[\'input\', \'type\', \'n\', \'axis\', \'norm\', \'name\'], varargs=None, keywords=None, defaults=[\'2\', \'None\', \'-1\', \'None\', \'None\'], " + } member_method { name: "ifft" argspec: "args=[\'input\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " @@ -44,6 +52,18 @@ tf_module { name: "inverse_stft_window_fn" argspec: "args=[\'frame_step\', \'forward_window_fn\', \'name\'], varargs=None, keywords=None, defaults=[\'\', \'None\'], " } + member_method { + name: "irfft" + argspec: "args=[\'input_tensor\', \'fft_length\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } + member_method { + name: "irfft2d" + argspec: "args=[\'input_tensor\', \'fft_length\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } + member_method { + name: "irfft3d" + argspec: "args=[\'input_tensor\', \'fft_length\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "linear_to_mel_weight_matrix" argspec: "args=[\'num_mel_bins\', \'num_spectrogram_bins\', \'sample_rate\', \'lower_edge_hertz\', \'upper_edge_hertz\', \'dtype\', \'name\'], varargs=None, keywords=None, defaults=[\'20\', \'129\', \'8000\', \'125.0\', \'3800.0\', \"\", \'None\'], " @@ -56,6 +76,18 @@ tf_module { name: "overlap_and_add" argspec: "args=[\'signal\', \'frame_step\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "rfft" + argspec: "args=[\'input_tensor\', \'fft_length\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } + member_method { + name: "rfft2d" + argspec: "args=[\'input_tensor\', \'fft_length\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } + member_method { + name: "rfft3d" + argspec: "args=[\'input_tensor\', \'fft_length\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + } member_method { name: "stft" argspec: "args=[\'signals\', \'frame_length\', \'frame_step\', \'fft_length\', \'window_fn\', \'pad_end\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'\', \'False\', \'None\'], " diff --git a/tensorflow/tools/api/golden/v2/tensorflow.sparse.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.sparse.pbtxt index 9c9c4d838e9..9808200d72c 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.sparse.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.sparse.pbtxt @@ -10,11 +10,11 @@ tf_module { } member_method { name: "add" - argspec: "args=[\'a\', \'b\', \'thresh\'], varargs=None, keywords=None, defaults=[\'0\'], " + argspec: "args=[\'a\', \'b\', \'threshold\'], varargs=None, keywords=None, defaults=[\'0\'], " } member_method { name: "concat" - argspec: "args=[\'axis\', \'sp_inputs\', \'name\', \'expand_nonconcat_dim\', \'concat_dim\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\'], " + argspec: "args=[\'axis\', \'sp_inputs\', \'expand_nonconcat_dim\', \'name\'], varargs=None, keywords=None, defaults=[\'False\', \'None\'], " } member_method { name: "cross" @@ -40,37 +40,21 @@ tf_module { name: "mask" argspec: "args=[\'a\', \'mask_indices\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " } - member_method { - name: "matmul" - argspec: "args=[\'sp_a\', \'b\', \'adjoint_a\', \'adjoint_b\', \'name\'], varargs=None, keywords=None, defaults=[\'False\', \'False\', \'None\'], " - } member_method { name: "maximum" argspec: "args=[\'sp_a\', \'sp_b\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " } - member_method { - name: "merge" - argspec: "args=[\'sp_ids\', \'sp_values\', \'vocab_size\', \'name\', \'already_sorted\'], varargs=None, keywords=None, defaults=[\'None\', \'False\'], " - } member_method { name: "minimum" argspec: "args=[\'sp_a\', \'sp_b\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " } member_method { name: "reduce_max" - argspec: "args=[\'sp_input\', \'axis\', \'keepdims\', \'reduction_axes\', \'keep_dims\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], " - } - member_method { - name: "reduce_max_sparse" - argspec: "args=[\'sp_input\', \'axis\', \'keepdims\', \'reduction_axes\', \'keep_dims\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], " + argspec: "args=[\'sp_input\', \'axis\', \'keepdims\', \'output_is_sparse\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'False\', \'None\'], " } member_method { name: "reduce_sum" - argspec: "args=[\'sp_input\', \'axis\', \'keepdims\', \'reduction_axes\', \'keep_dims\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], " - } - member_method { - name: "reduce_sum_sparse" - argspec: "args=[\'sp_input\', \'axis\', \'keepdims\', \'reduction_axes\', \'keep_dims\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], " + argspec: "args=[\'sp_input\', \'axis\', \'keepdims\', \'output_is_sparse\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'False\', \'None\'], " } member_method { name: "reorder" @@ -90,15 +74,15 @@ tf_module { } member_method { name: "segment_mean" - argspec: "args=[\'data\', \'indices\', \'segment_ids\', \'name\', \'num_segments\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + argspec: "args=[\'data\', \'indices\', \'segment_ids\', \'num_segments\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " } member_method { name: "segment_sqrt_n" - argspec: "args=[\'data\', \'indices\', \'segment_ids\', \'name\', \'num_segments\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + argspec: "args=[\'data\', \'indices\', \'segment_ids\', \'num_segments\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " } member_method { name: "segment_sum" - argspec: "args=[\'data\', \'indices\', \'segment_ids\', \'name\', \'num_segments\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + argspec: "args=[\'data\', \'indices\', \'segment_ids\', \'num_segments\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " } member_method { name: "slice" @@ -108,9 +92,13 @@ tf_module { name: "softmax" argspec: "args=[\'sp_input\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "sparse_dense_matmul" + argspec: "args=[\'sp_a\', \'b\', \'adjoint_a\', \'adjoint_b\', \'name\'], varargs=None, keywords=None, defaults=[\'False\', \'False\', \'None\'], " + } member_method { name: "split" - argspec: "args=[\'keyword_required\', \'sp_input\', \'num_split\', \'axis\', \'name\', \'split_dim\'], varargs=None, keywords=None, defaults=[\'KeywordRequired()\', \'None\', \'None\', \'None\', \'None\', \'None\'], " + argspec: "args=[\'sp_input\', \'num_split\', \'axis\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], " } member_method { name: "to_dense" diff --git a/tensorflow/tools/api/golden/v2/tensorflow.spectral.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.spectral.pbtxt deleted file mode 100644 index b0f0783e300..00000000000 --- a/tensorflow/tools/api/golden/v2/tensorflow.spectral.pbtxt +++ /dev/null @@ -1,35 +0,0 @@ -path: "tensorflow.spectral" -tf_module { - member_method { - name: "dct" - argspec: "args=[\'input\', \'type\', \'n\', \'axis\', \'norm\', \'name\'], varargs=None, keywords=None, defaults=[\'2\', \'None\', \'-1\', \'None\', \'None\'], " - } - member_method { - name: "idct" - argspec: "args=[\'input\', \'type\', \'n\', \'axis\', \'norm\', \'name\'], varargs=None, keywords=None, defaults=[\'2\', \'None\', \'-1\', \'None\', \'None\'], " - } - member_method { - name: "irfft" - argspec: "args=[\'input_tensor\', \'fft_length\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " - } - member_method { - name: "irfft2d" - argspec: "args=[\'input_tensor\', \'fft_length\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " - } - member_method { - name: "irfft3d" - argspec: "args=[\'input_tensor\', \'fft_length\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " - } - member_method { - name: "rfft" - argspec: "args=[\'input_tensor\', \'fft_length\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " - } - member_method { - name: "rfft2d" - argspec: "args=[\'input_tensor\', \'fft_length\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " - } - member_method { - name: "rfft3d" - argspec: "args=[\'input_tensor\', \'fft_length\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " - } -} diff --git a/tensorflow/tools/api/golden/v2/tensorflow.strings.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.strings.pbtxt index 03144cbe709..f6e32ed08c8 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.strings.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.strings.pbtxt @@ -10,11 +10,11 @@ tf_module { } member_method { name: "length" - argspec: "args=[\'input\', \'name\', \'unit\'], varargs=None, keywords=None, defaults=[\'None\', \'BYTE\'], " + argspec: "args=[\'input\', \'unit\', \'name\'], varargs=None, keywords=None, defaults=[\'BYTE\', \'None\'], " } member_method { name: "reduce_join" - argspec: "args=[\'inputs\', \'axis\', \'keep_dims\', \'separator\', \'name\', \'reduction_indices\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'\', \'None\', \'None\'], " + argspec: "args=[\'inputs\', \'axis\', \'keepdims\', \'separator\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'\', \'None\'], " } member_method { name: "regex_full_match" @@ -34,11 +34,11 @@ tf_module { } member_method { name: "substr" - argspec: "args=[\'input\', \'pos\', \'len\', \'name\', \'unit\'], varargs=None, keywords=None, defaults=[\'None\', \'BYTE\'], " + argspec: "args=[\'input\', \'pos\', \'len\', \'unit\', \'name\'], varargs=None, keywords=None, defaults=[\'BYTE\', \'None\'], " } member_method { name: "to_hash_bucket" - argspec: "args=[\'string_tensor\', \'num_buckets\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " + argspec: "args=[\'input\', \'num_buckets\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " } member_method { name: "to_hash_bucket_fast" @@ -50,7 +50,11 @@ tf_module { } member_method { name: "to_number" - argspec: "args=[\'string_tensor\', \'out_type\', \'name\'], varargs=None, keywords=None, defaults=[\"\", \'None\'], " + argspec: "args=[\'input\', \'out_type\', \'name\'], varargs=None, keywords=None, defaults=[\"\", \'None\'], " + } + member_method { + name: "unicode_encode" + argspec: "args=[\'input\', \'output_encoding\', \'errors\', \'replacement_char\', \'name\'], varargs=None, keywords=None, defaults=[\'replace\', \'65533\', \'None\'], " } member_method { name: "unicode_script" diff --git a/tensorflow/tools/api/golden/v2/tensorflow.summary.-summary-writer.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.summary.-summary-writer.pbtxt new file mode 100644 index 00000000000..6715c14e168 --- /dev/null +++ b/tensorflow/tools/api/golden/v2/tensorflow.summary.-summary-writer.pbtxt @@ -0,0 +1,29 @@ +path: "tensorflow.summary.SummaryWriter" +tf_class { + is_instance: "" + is_instance: "" + member_method { + name: "__init__" + argspec: "args=[\'self\', \'resource\', \'init_op_fn\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "as_default" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "close" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "flush" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "init" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "set_as_default" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } +} diff --git a/tensorflow/tools/api/golden/v2/tensorflow.summary.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.summary.pbtxt index 7ed9cd77a01..42a74a65fbb 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.summary.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.summary.pbtxt @@ -24,44 +24,24 @@ tf_module { name: "SummaryDescription" mtype: "" } + member { + name: "SummaryWriter" + mtype: "" + } member { name: "TaggedRunMetadata" mtype: "" } member_method { - name: "audio" - argspec: "args=[\'name\', \'tensor\', \'sample_rate\', \'max_outputs\', \'collections\', \'family\'], varargs=None, keywords=None, defaults=[\'3\', \'None\', \'None\'], " + name: "create_file_writer" + argspec: "args=[\'logdir\', \'max_queue\', \'flush_millis\', \'filename_suffix\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], " } member_method { - name: "get_summary_description" - argspec: "args=[\'node_def\'], varargs=None, keywords=None, defaults=None" + name: "flush" + argspec: "args=[\'writer\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " } member_method { - name: "histogram" - argspec: "args=[\'name\', \'values\', \'collections\', \'family\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " - } - member_method { - name: "image" - argspec: "args=[\'name\', \'tensor\', \'max_outputs\', \'collections\', \'family\'], varargs=None, keywords=None, defaults=[\'3\', \'None\', \'None\'], " - } - member_method { - name: "merge" - argspec: "args=[\'inputs\', \'collections\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " - } - member_method { - name: "merge_all" - argspec: "args=[\'key\', \'scope\', \'name\'], varargs=None, keywords=None, defaults=[\'summaries\', \'None\', \'None\'], " - } - member_method { - name: "scalar" - argspec: "args=[\'name\', \'tensor\', \'collections\', \'family\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " - } - member_method { - name: "tensor_summary" - argspec: "args=[\'name\', \'tensor\', \'summary_description\', \'collections\', \'summary_metadata\', \'family\', \'display_name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\'], " - } - member_method { - name: "text" - argspec: "args=[\'name\', \'tensor\', \'collections\'], varargs=None, keywords=None, defaults=[\'None\'], " + name: "import_event" + argspec: "args=[\'tensor\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " } } diff --git a/tensorflow/tools/api/golden/v2/tensorflow.test.-benchmark.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.test.-benchmark.pbtxt index df528e26b60..6fc489c8604 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.test.-benchmark.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.test.-benchmark.pbtxt @@ -6,6 +6,10 @@ tf_class { member_method { name: "__init__" } + member_method { + name: "evaluate" + argspec: "args=[\'self\', \'tensors\'], varargs=None, keywords=None, defaults=None" + } member_method { name: "is_abstract" argspec: "args=[\'cls\'], varargs=None, keywords=None, defaults=None" diff --git a/tensorflow/tools/api/golden/v2/tensorflow.test.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.test.pbtxt index af3f06d8de3..72ce7330445 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.test.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.test.pbtxt @@ -12,26 +12,14 @@ tf_module { name: "TestCase" mtype: "" } - member { - name: "mock" - mtype: "" - } member_method { name: "assert_equal_graph_def" - argspec: "args=[\'actual\', \'expected\', \'checkpoint_v2\'], varargs=None, keywords=None, defaults=[\'False\'], " + argspec: "args=[\'actual\', \'expected\'], varargs=None, keywords=None, defaults=None" } member_method { name: "benchmark_config" argspec: "args=[], varargs=None, keywords=None, defaults=None" } - member_method { - name: "compute_gradient" - argspec: "args=[\'x\', \'x_shape\', \'y\', \'y_shape\', \'x_init_value\', \'delta\', \'init_targets\', \'extra_feed_dict\'], varargs=None, keywords=None, defaults=[\'None\', \'0.001\', \'None\', \'None\'], " - } - member_method { - name: "compute_gradient_error" - argspec: "args=[\'x\', \'x_shape\', \'y\', \'y_shape\', \'x_init_value\', \'delta\', \'extra_feed_dict\'], varargs=None, keywords=None, defaults=[\'None\', \'0.001\', \'None\'], " - } member_method { name: "create_local_cluster" argspec: "args=[\'num_workers\', \'num_ps\', \'protocol\', \'worker_config\', \'ps_config\'], varargs=None, keywords=None, defaults=[\'grpc\', \'None\', \'None\'], " diff --git a/tensorflow/tools/api/golden/v2/tensorflow.train.-adadelta-optimizer.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.train.-adadelta-optimizer.pbtxt deleted file mode 100644 index 1f1d8b6f9e2..00000000000 --- a/tensorflow/tools/api/golden/v2/tensorflow.train.-adadelta-optimizer.pbtxt +++ /dev/null @@ -1,51 +0,0 @@ -path: "tensorflow.train.AdadeltaOptimizer" -tf_class { - is_instance: "" - is_instance: "" - is_instance: "" - is_instance: "" - member { - name: "GATE_GRAPH" - mtype: "" - } - member { - name: "GATE_NONE" - mtype: "" - } - member { - name: "GATE_OP" - mtype: "" - } - member_method { - name: "__init__" - argspec: "args=[\'self\', \'learning_rate\', \'rho\', \'epsilon\', \'use_locking\', \'name\'], varargs=None, keywords=None, defaults=[\'0.001\', \'0.95\', \'1e-08\', \'False\', \'Adadelta\'], " - } - member_method { - name: "apply_gradients" - argspec: "args=[\'self\', \'grads_and_vars\', \'global_step\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " - } - member_method { - name: "compute_gradients" - argspec: "args=[\'self\', \'loss\', \'var_list\', \'gate_gradients\', \'aggregation_method\', \'colocate_gradients_with_ops\', \'grad_loss\'], varargs=None, keywords=None, defaults=[\'None\', \'1\', \'None\', \'False\', \'None\'], " - } - member_method { - name: "get_name" - argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" - } - member_method { - name: "get_slot" - argspec: "args=[\'self\', \'var\', \'name\'], varargs=None, keywords=None, defaults=None" - } - member_method { - name: "get_slot_names" - argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" - } - member_method { - name: "minimize" - argspec: "args=[\'self\', \'loss\', \'global_step\', \'var_list\', \'gate_gradients\', \'aggregation_method\', \'colocate_gradients_with_ops\', \'name\', \'grad_loss\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'1\', \'None\', \'False\', \'None\', \'None\'], " - } - member_method { - name: "variables" - argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" - } -} diff --git a/tensorflow/tools/api/golden/v2/tensorflow.train.-adagrad-d-a-optimizer.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.train.-adagrad-d-a-optimizer.pbtxt deleted file mode 100644 index a7c05d48490..00000000000 --- a/tensorflow/tools/api/golden/v2/tensorflow.train.-adagrad-d-a-optimizer.pbtxt +++ /dev/null @@ -1,51 +0,0 @@ -path: "tensorflow.train.AdagradDAOptimizer" -tf_class { - is_instance: "" - is_instance: "" - is_instance: "" - is_instance: "" - member { - name: "GATE_GRAPH" - mtype: "" - } - member { - name: "GATE_NONE" - mtype: "" - } - member { - name: "GATE_OP" - mtype: "" - } - member_method { - name: "__init__" - argspec: "args=[\'self\', \'learning_rate\', \'global_step\', \'initial_gradient_squared_accumulator_value\', \'l1_regularization_strength\', \'l2_regularization_strength\', \'use_locking\', \'name\'], varargs=None, keywords=None, defaults=[\'0.1\', \'0.0\', \'0.0\', \'False\', \'AdagradDA\'], " - } - member_method { - name: "apply_gradients" - argspec: "args=[\'self\', \'grads_and_vars\', \'global_step\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " - } - member_method { - name: "compute_gradients" - argspec: "args=[\'self\', \'loss\', \'var_list\', \'gate_gradients\', \'aggregation_method\', \'colocate_gradients_with_ops\', \'grad_loss\'], varargs=None, keywords=None, defaults=[\'None\', \'1\', \'None\', \'False\', \'None\'], " - } - member_method { - name: "get_name" - argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" - } - member_method { - name: "get_slot" - argspec: "args=[\'self\', \'var\', \'name\'], varargs=None, keywords=None, defaults=None" - } - member_method { - name: "get_slot_names" - argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" - } - member_method { - name: "minimize" - argspec: "args=[\'self\', \'loss\', \'global_step\', \'var_list\', \'gate_gradients\', \'aggregation_method\', \'colocate_gradients_with_ops\', \'name\', \'grad_loss\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'1\', \'None\', \'False\', \'None\', \'None\'], " - } - member_method { - name: "variables" - argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" - } -} diff --git a/tensorflow/tools/api/golden/v2/tensorflow.train.-adagrad-optimizer.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.train.-adagrad-optimizer.pbtxt deleted file mode 100644 index bc8b92389c6..00000000000 --- a/tensorflow/tools/api/golden/v2/tensorflow.train.-adagrad-optimizer.pbtxt +++ /dev/null @@ -1,51 +0,0 @@ -path: "tensorflow.train.AdagradOptimizer" -tf_class { - is_instance: "" - is_instance: "" - is_instance: "" - is_instance: "" - member { - name: "GATE_GRAPH" - mtype: "" - } - member { - name: "GATE_NONE" - mtype: "" - } - member { - name: "GATE_OP" - mtype: "" - } - member_method { - name: "__init__" - argspec: "args=[\'self\', \'learning_rate\', \'initial_accumulator_value\', \'use_locking\', \'name\'], varargs=None, keywords=None, defaults=[\'0.1\', \'False\', \'Adagrad\'], " - } - member_method { - name: "apply_gradients" - argspec: "args=[\'self\', \'grads_and_vars\', \'global_step\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " - } - member_method { - name: "compute_gradients" - argspec: "args=[\'self\', \'loss\', \'var_list\', \'gate_gradients\', \'aggregation_method\', \'colocate_gradients_with_ops\', \'grad_loss\'], varargs=None, keywords=None, defaults=[\'None\', \'1\', \'None\', \'False\', \'None\'], " - } - member_method { - name: "get_name" - argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" - } - member_method { - name: "get_slot" - argspec: "args=[\'self\', \'var\', \'name\'], varargs=None, keywords=None, defaults=None" - } - member_method { - name: "get_slot_names" - argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" - } - member_method { - name: "minimize" - argspec: "args=[\'self\', \'loss\', \'global_step\', \'var_list\', \'gate_gradients\', \'aggregation_method\', \'colocate_gradients_with_ops\', \'name\', \'grad_loss\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'1\', \'None\', \'False\', \'None\', \'None\'], " - } - member_method { - name: "variables" - argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" - } -} diff --git a/tensorflow/tools/api/golden/v2/tensorflow.train.-adam-optimizer.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.train.-adam-optimizer.pbtxt deleted file mode 100644 index 5d17be9378f..00000000000 --- a/tensorflow/tools/api/golden/v2/tensorflow.train.-adam-optimizer.pbtxt +++ /dev/null @@ -1,51 +0,0 @@ -path: "tensorflow.train.AdamOptimizer" -tf_class { - is_instance: "" - is_instance: "" - is_instance: "" - is_instance: "" - member { - name: "GATE_GRAPH" - mtype: "" - } - member { - name: "GATE_NONE" - mtype: "" - } - member { - name: "GATE_OP" - mtype: "" - } - member_method { - name: "__init__" - argspec: "args=[\'self\', \'learning_rate\', \'beta1\', \'beta2\', \'epsilon\', \'use_locking\', \'name\'], varargs=None, keywords=None, defaults=[\'0.001\', \'0.9\', \'0.999\', \'1e-08\', \'False\', \'Adam\'], " - } - member_method { - name: "apply_gradients" - argspec: "args=[\'self\', \'grads_and_vars\', \'global_step\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " - } - member_method { - name: "compute_gradients" - argspec: "args=[\'self\', \'loss\', \'var_list\', \'gate_gradients\', \'aggregation_method\', \'colocate_gradients_with_ops\', \'grad_loss\'], varargs=None, keywords=None, defaults=[\'None\', \'1\', \'None\', \'False\', \'None\'], " - } - member_method { - name: "get_name" - argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" - } - member_method { - name: "get_slot" - argspec: "args=[\'self\', \'var\', \'name\'], varargs=None, keywords=None, defaults=None" - } - member_method { - name: "get_slot_names" - argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" - } - member_method { - name: "minimize" - argspec: "args=[\'self\', \'loss\', \'global_step\', \'var_list\', \'gate_gradients\', \'aggregation_method\', \'colocate_gradients_with_ops\', \'name\', \'grad_loss\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'1\', \'None\', \'False\', \'None\', \'None\'], " - } - member_method { - name: "variables" - argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" - } -} diff --git a/tensorflow/tools/api/golden/v2/tensorflow.train.-checkpoint-saver-listener.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.train.-checkpoint-saver-listener.pbtxt deleted file mode 100644 index 9d3688e5657..00000000000 --- a/tensorflow/tools/api/golden/v2/tensorflow.train.-checkpoint-saver-listener.pbtxt +++ /dev/null @@ -1,24 +0,0 @@ -path: "tensorflow.train.CheckpointSaverListener" -tf_class { - is_instance: "" - is_instance: "" - member_method { - name: "__init__" - } - member_method { - name: "after_save" - argspec: "args=[\'self\', \'session\', \'global_step_value\'], varargs=None, keywords=None, defaults=None" - } - member_method { - name: "before_save" - argspec: "args=[\'self\', \'session\', \'global_step_value\'], varargs=None, keywords=None, defaults=None" - } - member_method { - name: "begin" - argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" - } - member_method { - name: "end" - argspec: "args=[\'self\', \'session\', \'global_step_value\'], varargs=None, keywords=None, defaults=None" - } -} diff --git a/tensorflow/tools/api/golden/v2/tensorflow.train.-chief-session-creator.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.train.-chief-session-creator.pbtxt deleted file mode 100644 index abbe273be32..00000000000 --- a/tensorflow/tools/api/golden/v2/tensorflow.train.-chief-session-creator.pbtxt +++ /dev/null @@ -1,14 +0,0 @@ -path: "tensorflow.train.ChiefSessionCreator" -tf_class { - is_instance: "" - is_instance: "" - is_instance: "" - member_method { - name: "__init__" - argspec: "args=[\'self\', \'scaffold\', \'master\', \'config\', \'checkpoint_dir\', \'checkpoint_filename_with_path\'], varargs=None, keywords=None, defaults=[\'None\', \'\', \'None\', \'None\', \'None\'], " - } - member_method { - name: "create_session" - argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" - } -} diff --git a/tensorflow/tools/api/golden/v2/tensorflow.train.-ftrl-optimizer.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.train.-ftrl-optimizer.pbtxt deleted file mode 100644 index d265fdeb01c..00000000000 --- a/tensorflow/tools/api/golden/v2/tensorflow.train.-ftrl-optimizer.pbtxt +++ /dev/null @@ -1,51 +0,0 @@ -path: "tensorflow.train.FtrlOptimizer" -tf_class { - is_instance: "" - is_instance: "" - is_instance: "" - is_instance: "" - member { - name: "GATE_GRAPH" - mtype: "" - } - member { - name: "GATE_NONE" - mtype: "" - } - member { - name: "GATE_OP" - mtype: "" - } - member_method { - name: "__init__" - argspec: "args=[\'self\', \'learning_rate\', \'learning_rate_power\', \'initial_accumulator_value\', \'l1_regularization_strength\', \'l2_regularization_strength\', \'use_locking\', \'name\', \'accum_name\', \'linear_name\', \'l2_shrinkage_regularization_strength\'], varargs=None, keywords=None, defaults=[\'-0.5\', \'0.1\', \'0.0\', \'0.0\', \'False\', \'Ftrl\', \'None\', \'None\', \'0.0\'], " - } - member_method { - name: "apply_gradients" - argspec: "args=[\'self\', \'grads_and_vars\', \'global_step\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " - } - member_method { - name: "compute_gradients" - argspec: "args=[\'self\', \'loss\', \'var_list\', \'gate_gradients\', \'aggregation_method\', \'colocate_gradients_with_ops\', \'grad_loss\'], varargs=None, keywords=None, defaults=[\'None\', \'1\', \'None\', \'False\', \'None\'], " - } - member_method { - name: "get_name" - argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" - } - member_method { - name: "get_slot" - argspec: "args=[\'self\', \'var\', \'name\'], varargs=None, keywords=None, defaults=None" - } - member_method { - name: "get_slot_names" - argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" - } - member_method { - name: "minimize" - argspec: "args=[\'self\', \'loss\', \'global_step\', \'var_list\', \'gate_gradients\', \'aggregation_method\', \'colocate_gradients_with_ops\', \'name\', \'grad_loss\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'1\', \'None\', \'False\', \'None\', \'None\'], " - } - member_method { - name: "variables" - argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" - } -} diff --git a/tensorflow/tools/api/golden/v2/tensorflow.train.-gradient-descent-optimizer.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.train.-gradient-descent-optimizer.pbtxt deleted file mode 100644 index c673e29cd4d..00000000000 --- a/tensorflow/tools/api/golden/v2/tensorflow.train.-gradient-descent-optimizer.pbtxt +++ /dev/null @@ -1,51 +0,0 @@ -path: "tensorflow.train.GradientDescentOptimizer" -tf_class { - is_instance: "" - is_instance: "" - is_instance: "" - is_instance: "" - member { - name: "GATE_GRAPH" - mtype: "" - } - member { - name: "GATE_NONE" - mtype: "" - } - member { - name: "GATE_OP" - mtype: "" - } - member_method { - name: "__init__" - argspec: "args=[\'self\', \'learning_rate\', \'use_locking\', \'name\'], varargs=None, keywords=None, defaults=[\'False\', \'GradientDescent\'], " - } - member_method { - name: "apply_gradients" - argspec: "args=[\'self\', \'grads_and_vars\', \'global_step\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " - } - member_method { - name: "compute_gradients" - argspec: "args=[\'self\', \'loss\', \'var_list\', \'gate_gradients\', \'aggregation_method\', \'colocate_gradients_with_ops\', \'grad_loss\'], varargs=None, keywords=None, defaults=[\'None\', \'1\', \'None\', \'False\', \'None\'], " - } - member_method { - name: "get_name" - argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" - } - member_method { - name: "get_slot" - argspec: "args=[\'self\', \'var\', \'name\'], varargs=None, keywords=None, defaults=None" - } - member_method { - name: "get_slot_names" - argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" - } - member_method { - name: "minimize" - argspec: "args=[\'self\', \'loss\', \'global_step\', \'var_list\', \'gate_gradients\', \'aggregation_method\', \'colocate_gradients_with_ops\', \'name\', \'grad_loss\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'1\', \'None\', \'False\', \'None\', \'None\'], " - } - member_method { - name: "variables" - argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" - } -} diff --git a/tensorflow/tools/api/golden/v2/tensorflow.train.-looper-thread.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.train.-looper-thread.pbtxt deleted file mode 100644 index c61859004e8..00000000000 --- a/tensorflow/tools/api/golden/v2/tensorflow.train.-looper-thread.pbtxt +++ /dev/null @@ -1,73 +0,0 @@ -path: "tensorflow.train.LooperThread" -tf_class { - is_instance: "" - is_instance: "" - member { - name: "daemon" - mtype: "" - } - member { - name: "ident" - mtype: "" - } - member { - name: "name" - mtype: "" - } - member_method { - name: "__init__" - argspec: "args=[\'self\', \'coord\', \'timer_interval_secs\', \'target\', \'args\', \'kwargs\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], " - } - member_method { - name: "getName" - argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" - } - member_method { - name: "isAlive" - argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" - } - member_method { - name: "isDaemon" - argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" - } - member_method { - name: "is_alive" - argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" - } - member_method { - name: "join" - argspec: "args=[\'self\', \'timeout\'], varargs=None, keywords=None, defaults=[\'None\'], " - } - member_method { - name: "loop" - argspec: "args=[\'coord\', \'timer_interval_secs\', \'target\', \'args\', \'kwargs\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " - } - member_method { - name: "run" - argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" - } - member_method { - name: "run_loop" - argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" - } - member_method { - name: "setDaemon" - argspec: "args=[\'self\', \'daemonic\'], varargs=None, keywords=None, defaults=None" - } - member_method { - name: "setName" - argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=None" - } - member_method { - name: "start" - argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" - } - member_method { - name: "start_loop" - argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" - } - member_method { - name: "stop_loop" - argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" - } -} diff --git a/tensorflow/tools/api/golden/v2/tensorflow.train.-momentum-optimizer.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.train.-momentum-optimizer.pbtxt deleted file mode 100644 index 8199f63b9b8..00000000000 --- a/tensorflow/tools/api/golden/v2/tensorflow.train.-momentum-optimizer.pbtxt +++ /dev/null @@ -1,51 +0,0 @@ -path: "tensorflow.train.MomentumOptimizer" -tf_class { - is_instance: "" - is_instance: "" - is_instance: "" - is_instance: "" - member { - name: "GATE_GRAPH" - mtype: "" - } - member { - name: "GATE_NONE" - mtype: "" - } - member { - name: "GATE_OP" - mtype: "" - } - member_method { - name: "__init__" - argspec: "args=[\'self\', \'learning_rate\', \'momentum\', \'use_locking\', \'name\', \'use_nesterov\'], varargs=None, keywords=None, defaults=[\'False\', \'Momentum\', \'False\'], " - } - member_method { - name: "apply_gradients" - argspec: "args=[\'self\', \'grads_and_vars\', \'global_step\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " - } - member_method { - name: "compute_gradients" - argspec: "args=[\'self\', \'loss\', \'var_list\', \'gate_gradients\', \'aggregation_method\', \'colocate_gradients_with_ops\', \'grad_loss\'], varargs=None, keywords=None, defaults=[\'None\', \'1\', \'None\', \'False\', \'None\'], " - } - member_method { - name: "get_name" - argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" - } - member_method { - name: "get_slot" - argspec: "args=[\'self\', \'var\', \'name\'], varargs=None, keywords=None, defaults=None" - } - member_method { - name: "get_slot_names" - argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" - } - member_method { - name: "minimize" - argspec: "args=[\'self\', \'loss\', \'global_step\', \'var_list\', \'gate_gradients\', \'aggregation_method\', \'colocate_gradients_with_ops\', \'name\', \'grad_loss\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'1\', \'None\', \'False\', \'None\', \'None\'], " - } - member_method { - name: "variables" - argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" - } -} diff --git a/tensorflow/tools/api/golden/v2/tensorflow.train.-monitored-session.-step-context.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.train.-monitored-session.-step-context.pbtxt deleted file mode 100644 index 03efe6639e0..00000000000 --- a/tensorflow/tools/api/golden/v2/tensorflow.train.-monitored-session.-step-context.pbtxt +++ /dev/null @@ -1,21 +0,0 @@ -path: "tensorflow.train.MonitoredSession.StepContext" -tf_class { - is_instance: "" - is_instance: "" - member { - name: "session" - mtype: "" - } - member_method { - name: "__init__" - argspec: "args=[\'self\', \'session\', \'run_with_hooks_fn\'], varargs=None, keywords=None, defaults=None" - } - member_method { - name: "request_stop" - argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" - } - member_method { - name: "run_with_hooks" - argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None" - } -} diff --git a/tensorflow/tools/api/golden/v2/tensorflow.train.-monitored-session.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.train.-monitored-session.pbtxt deleted file mode 100644 index 09b7b3fb538..00000000000 --- a/tensorflow/tools/api/golden/v2/tensorflow.train.-monitored-session.pbtxt +++ /dev/null @@ -1,34 +0,0 @@ -path: "tensorflow.train.MonitoredSession" -tf_class { - is_instance: "" - is_instance: "" - is_instance: "" - member { - name: "StepContext" - mtype: "" - } - member { - name: "graph" - mtype: "" - } - member_method { - name: "__init__" - argspec: "args=[\'self\', \'session_creator\', \'hooks\', \'stop_grace_period_secs\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'120\'], " - } - member_method { - name: "close" - argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" - } - member_method { - name: "run" - argspec: "args=[\'self\', \'fetches\', \'feed_dict\', \'options\', \'run_metadata\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], " - } - member_method { - name: "run_step_fn" - argspec: "args=[\'self\', \'step_fn\'], varargs=None, keywords=None, defaults=None" - } - member_method { - name: "should_stop" - argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" - } -} diff --git a/tensorflow/tools/api/golden/v2/tensorflow.train.-nan-loss-during-training-error.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.train.-nan-loss-during-training-error.pbtxt deleted file mode 100644 index e415819b3d7..00000000000 --- a/tensorflow/tools/api/golden/v2/tensorflow.train.-nan-loss-during-training-error.pbtxt +++ /dev/null @@ -1,12 +0,0 @@ -path: "tensorflow.train.NanLossDuringTrainingError" -tf_class { - is_instance: "" - is_instance: "" - member { - name: "args" - mtype: "" - } - member_method { - name: "__init__" - } -} diff --git a/tensorflow/tools/api/golden/v2/tensorflow.train.-optimizer.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.train.-optimizer.pbtxt deleted file mode 100644 index 876bb35e391..00000000000 --- a/tensorflow/tools/api/golden/v2/tensorflow.train.-optimizer.pbtxt +++ /dev/null @@ -1,50 +0,0 @@ -path: "tensorflow.train.Optimizer" -tf_class { - is_instance: "" - is_instance: "" - is_instance: "" - member { - name: "GATE_GRAPH" - mtype: "" - } - member { - name: "GATE_NONE" - mtype: "" - } - member { - name: "GATE_OP" - mtype: "" - } - member_method { - name: "__init__" - argspec: "args=[\'self\', \'use_locking\', \'name\'], varargs=None, keywords=None, defaults=None" - } - member_method { - name: "apply_gradients" - argspec: "args=[\'self\', \'grads_and_vars\', \'global_step\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " - } - member_method { - name: "compute_gradients" - argspec: "args=[\'self\', \'loss\', \'var_list\', \'gate_gradients\', \'aggregation_method\', \'colocate_gradients_with_ops\', \'grad_loss\'], varargs=None, keywords=None, defaults=[\'None\', \'1\', \'None\', \'False\', \'None\'], " - } - member_method { - name: "get_name" - argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" - } - member_method { - name: "get_slot" - argspec: "args=[\'self\', \'var\', \'name\'], varargs=None, keywords=None, defaults=None" - } - member_method { - name: "get_slot_names" - argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" - } - member_method { - name: "minimize" - argspec: "args=[\'self\', \'loss\', \'global_step\', \'var_list\', \'gate_gradients\', \'aggregation_method\', \'colocate_gradients_with_ops\', \'name\', \'grad_loss\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'1\', \'None\', \'False\', \'None\', \'None\'], " - } - member_method { - name: "variables" - argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" - } -} diff --git a/tensorflow/tools/api/golden/v2/tensorflow.train.-proximal-adagrad-optimizer.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.train.-proximal-adagrad-optimizer.pbtxt deleted file mode 100644 index 14349a74efb..00000000000 --- a/tensorflow/tools/api/golden/v2/tensorflow.train.-proximal-adagrad-optimizer.pbtxt +++ /dev/null @@ -1,51 +0,0 @@ -path: "tensorflow.train.ProximalAdagradOptimizer" -tf_class { - is_instance: "" - is_instance: "" - is_instance: "" - is_instance: "" - member { - name: "GATE_GRAPH" - mtype: "" - } - member { - name: "GATE_NONE" - mtype: "" - } - member { - name: "GATE_OP" - mtype: "" - } - member_method { - name: "__init__" - argspec: "args=[\'self\', \'learning_rate\', \'initial_accumulator_value\', \'l1_regularization_strength\', \'l2_regularization_strength\', \'use_locking\', \'name\'], varargs=None, keywords=None, defaults=[\'0.1\', \'0.0\', \'0.0\', \'False\', \'ProximalAdagrad\'], " - } - member_method { - name: "apply_gradients" - argspec: "args=[\'self\', \'grads_and_vars\', \'global_step\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " - } - member_method { - name: "compute_gradients" - argspec: "args=[\'self\', \'loss\', \'var_list\', \'gate_gradients\', \'aggregation_method\', \'colocate_gradients_with_ops\', \'grad_loss\'], varargs=None, keywords=None, defaults=[\'None\', \'1\', \'None\', \'False\', \'None\'], " - } - member_method { - name: "get_name" - argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" - } - member_method { - name: "get_slot" - argspec: "args=[\'self\', \'var\', \'name\'], varargs=None, keywords=None, defaults=None" - } - member_method { - name: "get_slot_names" - argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" - } - member_method { - name: "minimize" - argspec: "args=[\'self\', \'loss\', \'global_step\', \'var_list\', \'gate_gradients\', \'aggregation_method\', \'colocate_gradients_with_ops\', \'name\', \'grad_loss\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'1\', \'None\', \'False\', \'None\', \'None\'], " - } - member_method { - name: "variables" - argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" - } -} diff --git a/tensorflow/tools/api/golden/v2/tensorflow.train.-r-m-s-prop-optimizer.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.train.-r-m-s-prop-optimizer.pbtxt deleted file mode 100644 index 906384a2875..00000000000 --- a/tensorflow/tools/api/golden/v2/tensorflow.train.-r-m-s-prop-optimizer.pbtxt +++ /dev/null @@ -1,51 +0,0 @@ -path: "tensorflow.train.RMSPropOptimizer" -tf_class { - is_instance: "" - is_instance: "" - is_instance: "" - is_instance: "" - member { - name: "GATE_GRAPH" - mtype: "" - } - member { - name: "GATE_NONE" - mtype: "" - } - member { - name: "GATE_OP" - mtype: "" - } - member_method { - name: "__init__" - argspec: "args=[\'self\', \'learning_rate\', \'decay\', \'momentum\', \'epsilon\', \'use_locking\', \'centered\', \'name\'], varargs=None, keywords=None, defaults=[\'0.9\', \'0.0\', \'1e-10\', \'False\', \'False\', \'RMSProp\'], " - } - member_method { - name: "apply_gradients" - argspec: "args=[\'self\', \'grads_and_vars\', \'global_step\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " - } - member_method { - name: "compute_gradients" - argspec: "args=[\'self\', \'loss\', \'var_list\', \'gate_gradients\', \'aggregation_method\', \'colocate_gradients_with_ops\', \'grad_loss\'], varargs=None, keywords=None, defaults=[\'None\', \'1\', \'None\', \'False\', \'None\'], " - } - member_method { - name: "get_name" - argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" - } - member_method { - name: "get_slot" - argspec: "args=[\'self\', \'var\', \'name\'], varargs=None, keywords=None, defaults=None" - } - member_method { - name: "get_slot_names" - argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" - } - member_method { - name: "minimize" - argspec: "args=[\'self\', \'loss\', \'global_step\', \'var_list\', \'gate_gradients\', \'aggregation_method\', \'colocate_gradients_with_ops\', \'name\', \'grad_loss\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'1\', \'None\', \'False\', \'None\', \'None\'], " - } - member_method { - name: "variables" - argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" - } -} diff --git a/tensorflow/tools/api/golden/v2/tensorflow.train.-scaffold.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.train.-scaffold.pbtxt deleted file mode 100644 index 38cc98b48e7..00000000000 --- a/tensorflow/tools/api/golden/v2/tensorflow.train.-scaffold.pbtxt +++ /dev/null @@ -1,53 +0,0 @@ -path: "tensorflow.train.Scaffold" -tf_class { - is_instance: "" - is_instance: "" - member { - name: "init_feed_dict" - mtype: "" - } - member { - name: "init_fn" - mtype: "" - } - member { - name: "init_op" - mtype: "" - } - member { - name: "local_init_op" - mtype: "" - } - member { - name: "ready_for_local_init_op" - mtype: "" - } - member { - name: "ready_op" - mtype: "" - } - member { - name: "saver" - mtype: "" - } - member { - name: "summary_op" - mtype: "" - } - member_method { - name: "__init__" - argspec: "args=[\'self\', \'init_op\', \'init_feed_dict\', \'init_fn\', \'ready_op\', \'ready_for_local_init_op\', \'local_init_op\', \'summary_op\', \'saver\', \'copy_from_scaffold\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\'], " - } - member_method { - name: "default_local_init_op" - argspec: "args=[], varargs=None, keywords=None, defaults=None" - } - member_method { - name: "finalize" - argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" - } - member_method { - name: "get_or_default" - argspec: "args=[\'arg_name\', \'collection_key\', \'default_constructor\'], varargs=None, keywords=None, defaults=None" - } -} diff --git a/tensorflow/tools/api/golden/v2/tensorflow.train.-second-or-step-timer.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.train.-second-or-step-timer.pbtxt deleted file mode 100644 index 3c5a6ac13cc..00000000000 --- a/tensorflow/tools/api/golden/v2/tensorflow.train.-second-or-step-timer.pbtxt +++ /dev/null @@ -1,26 +0,0 @@ -path: "tensorflow.train.SecondOrStepTimer" -tf_class { - is_instance: "" - is_instance: "" - is_instance: "" - member_method { - name: "__init__" - argspec: "args=[\'self\', \'every_secs\', \'every_steps\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " - } - member_method { - name: "last_triggered_step" - argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" - } - member_method { - name: "reset" - argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" - } - member_method { - name: "should_trigger_for_step" - argspec: "args=[\'self\', \'step\'], varargs=None, keywords=None, defaults=None" - } - member_method { - name: "update_last_triggered_step" - argspec: "args=[\'self\', \'step\'], varargs=None, keywords=None, defaults=None" - } -} diff --git a/tensorflow/tools/api/golden/v2/tensorflow.train.-session-creator.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.train.-session-creator.pbtxt deleted file mode 100644 index beb232715f7..00000000000 --- a/tensorflow/tools/api/golden/v2/tensorflow.train.-session-creator.pbtxt +++ /dev/null @@ -1,12 +0,0 @@ -path: "tensorflow.train.SessionCreator" -tf_class { - is_instance: "" - is_instance: "" - member_method { - name: "__init__" - } - member_method { - name: "create_session" - argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" - } -} diff --git a/tensorflow/tools/api/golden/v2/tensorflow.train.-session-manager.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.train.-session-manager.pbtxt deleted file mode 100644 index 448764fe081..00000000000 --- a/tensorflow/tools/api/golden/v2/tensorflow.train.-session-manager.pbtxt +++ /dev/null @@ -1,21 +0,0 @@ -path: "tensorflow.train.SessionManager" -tf_class { - is_instance: "" - is_instance: "" - member_method { - name: "__init__" - argspec: "args=[\'self\', \'local_init_op\', \'ready_op\', \'ready_for_local_init_op\', \'graph\', \'recovery_wait_secs\', \'local_init_run_options\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\', \'30\', \'None\'], " - } - member_method { - name: "prepare_session" - argspec: "args=[\'self\', \'master\', \'init_op\', \'saver\', \'checkpoint_dir\', \'checkpoint_filename_with_path\', \'wait_for_checkpoint\', \'max_wait_secs\', \'config\', \'init_feed_dict\', \'init_fn\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\', \'False\', \'7200\', \'None\', \'None\', \'None\'], " - } - member_method { - name: "recover_session" - argspec: "args=[\'self\', \'master\', \'saver\', \'checkpoint_dir\', \'checkpoint_filename_with_path\', \'wait_for_checkpoint\', \'max_wait_secs\', \'config\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'False\', \'7200\', \'None\'], " - } - member_method { - name: "wait_for_session" - argspec: "args=[\'self\', \'master\', \'config\', \'max_wait_secs\'], varargs=None, keywords=None, defaults=[\'None\', \'inf\'], " - } -} diff --git a/tensorflow/tools/api/golden/v2/tensorflow.train.-session-run-args.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.train.-session-run-args.pbtxt deleted file mode 100644 index 442990893e3..00000000000 --- a/tensorflow/tools/api/golden/v2/tensorflow.train.-session-run-args.pbtxt +++ /dev/null @@ -1,27 +0,0 @@ -path: "tensorflow.train.SessionRunArgs" -tf_class { - is_instance: "" - is_instance: "" - is_instance: "" - member { - name: "feed_dict" - mtype: "" - } - member { - name: "fetches" - mtype: "" - } - member { - name: "options" - mtype: "" - } - member_method { - name: "__init__" - } - member_method { - name: "count" - } - member_method { - name: "index" - } -} diff --git a/tensorflow/tools/api/golden/v2/tensorflow.train.-session-run-context.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.train.-session-run-context.pbtxt deleted file mode 100644 index d5adb15c95f..00000000000 --- a/tensorflow/tools/api/golden/v2/tensorflow.train.-session-run-context.pbtxt +++ /dev/null @@ -1,25 +0,0 @@ -path: "tensorflow.train.SessionRunContext" -tf_class { - is_instance: "" - is_instance: "" - member { - name: "original_args" - mtype: "" - } - member { - name: "session" - mtype: "" - } - member { - name: "stop_requested" - mtype: "" - } - member_method { - name: "__init__" - argspec: "args=[\'self\', \'original_args\', \'session\'], varargs=None, keywords=None, defaults=None" - } - member_method { - name: "request_stop" - argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" - } -} diff --git a/tensorflow/tools/api/golden/v2/tensorflow.train.-session-run-values.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.train.-session-run-values.pbtxt deleted file mode 100644 index 0b401d59c40..00000000000 --- a/tensorflow/tools/api/golden/v2/tensorflow.train.-session-run-values.pbtxt +++ /dev/null @@ -1,27 +0,0 @@ -path: "tensorflow.train.SessionRunValues" -tf_class { - is_instance: "" - is_instance: "" - is_instance: "" - member { - name: "options" - mtype: "" - } - member { - name: "results" - mtype: "" - } - member { - name: "run_metadata" - mtype: "" - } - member_method { - name: "__init__" - } - member_method { - name: "count" - } - member_method { - name: "index" - } -} diff --git a/tensorflow/tools/api/golden/v2/tensorflow.train.-singular-monitored-session.-step-context.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.train.-singular-monitored-session.-step-context.pbtxt deleted file mode 100644 index 36d8ce7ff82..00000000000 --- a/tensorflow/tools/api/golden/v2/tensorflow.train.-singular-monitored-session.-step-context.pbtxt +++ /dev/null @@ -1,21 +0,0 @@ -path: "tensorflow.train.SingularMonitoredSession.StepContext" -tf_class { - is_instance: "" - is_instance: "" - member { - name: "session" - mtype: "" - } - member_method { - name: "__init__" - argspec: "args=[\'self\', \'session\', \'run_with_hooks_fn\'], varargs=None, keywords=None, defaults=None" - } - member_method { - name: "request_stop" - argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" - } - member_method { - name: "run_with_hooks" - argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None" - } -} diff --git a/tensorflow/tools/api/golden/v2/tensorflow.train.-singular-monitored-session.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.train.-singular-monitored-session.pbtxt deleted file mode 100644 index de0f2c1c1a2..00000000000 --- a/tensorflow/tools/api/golden/v2/tensorflow.train.-singular-monitored-session.pbtxt +++ /dev/null @@ -1,38 +0,0 @@ -path: "tensorflow.train.SingularMonitoredSession" -tf_class { - is_instance: "" - is_instance: "" - is_instance: "" - member { - name: "StepContext" - mtype: "" - } - member { - name: "graph" - mtype: "" - } - member_method { - name: "__init__" - argspec: "args=[\'self\', \'hooks\', \'scaffold\', \'master\', \'config\', \'checkpoint_dir\', \'stop_grace_period_secs\', \'checkpoint_filename_with_path\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'\', \'None\', \'None\', \'120\', \'None\'], " - } - member_method { - name: "close" - argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" - } - member_method { - name: "raw_session" - argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" - } - member_method { - name: "run" - argspec: "args=[\'self\', \'fetches\', \'feed_dict\', \'options\', \'run_metadata\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], " - } - member_method { - name: "run_step_fn" - argspec: "args=[\'self\', \'step_fn\'], varargs=None, keywords=None, defaults=None" - } - member_method { - name: "should_stop" - argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" - } -} diff --git a/tensorflow/tools/api/golden/v2/tensorflow.train.-supervisor.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.train.-supervisor.pbtxt deleted file mode 100644 index 9677e5a98e4..00000000000 --- a/tensorflow/tools/api/golden/v2/tensorflow.train.-supervisor.pbtxt +++ /dev/null @@ -1,153 +0,0 @@ -path: "tensorflow.train.Supervisor" -tf_class { - is_instance: "" - is_instance: "" - member { - name: "USE_DEFAULT" - mtype: "" - } - member { - name: "coord" - mtype: "" - } - member { - name: "global_step" - mtype: "" - } - member { - name: "init_feed_dict" - mtype: "" - } - member { - name: "init_op" - mtype: "" - } - member { - name: "is_chief" - mtype: "" - } - member { - name: "ready_for_local_init_op" - mtype: "" - } - member { - name: "ready_op" - mtype: "" - } - member { - name: "save_model_secs" - mtype: "" - } - member { - name: "save_path" - mtype: "" - } - member { - name: "save_summaries_secs" - mtype: "" - } - member { - name: "saver" - mtype: "" - } - member { - name: "session_manager" - mtype: "" - } - member { - name: "summary_op" - mtype: "" - } - member { - name: "summary_writer" - mtype: "" - } - member_method { - name: "Loop" - argspec: "args=[\'self\', \'timer_interval_secs\', \'target\', \'args\', \'kwargs\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " - } - member_method { - name: "PrepareSession" - argspec: "args=[\'self\', \'master\', \'config\', \'wait_for_checkpoint\', \'max_wait_secs\', \'start_standard_services\'], varargs=None, keywords=None, defaults=[\'\', \'None\', \'False\', \'7200\', \'True\'], " - } - member_method { - name: "RequestStop" - argspec: "args=[\'self\', \'ex\'], varargs=None, keywords=None, defaults=[\'None\'], " - } - member_method { - name: "ShouldStop" - argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" - } - member_method { - name: "StartQueueRunners" - argspec: "args=[\'self\', \'sess\', \'queue_runners\'], varargs=None, keywords=None, defaults=[\'None\'], " - } - member_method { - name: "StartStandardServices" - argspec: "args=[\'self\', \'sess\'], varargs=None, keywords=None, defaults=None" - } - member_method { - name: "Stop" - argspec: "args=[\'self\', \'threads\', \'close_summary_writer\', \'ignore_live_threads\'], varargs=None, keywords=None, defaults=[\'None\', \'True\', \'False\'], " - } - member_method { - name: "StopOnException" - argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" - } - member_method { - name: "SummaryComputed" - argspec: "args=[\'self\', \'sess\', \'summary\', \'global_step\'], varargs=None, keywords=None, defaults=[\'None\'], " - } - member_method { - name: "WaitForStop" - argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" - } - member_method { - name: "__init__" - argspec: "args=[\'self\', \'graph\', \'ready_op\', \'ready_for_local_init_op\', \'is_chief\', \'init_op\', \'init_feed_dict\', \'local_init_op\', \'logdir\', \'summary_op\', \'saver\', \'global_step\', \'save_summaries_secs\', \'save_model_secs\', \'recovery_wait_secs\', \'stop_grace_secs\', \'checkpoint_basename\', \'session_manager\', \'summary_writer\', \'init_fn\', \'local_init_run_options\'], varargs=None, keywords=None, defaults=[\'None\', \'0\', \'0\', \'True\', \'0\', \'None\', \'0\', \'None\', \'0\', \'0\', \'0\', \'120\', \'600\', \'30\', \'120\', \'model.ckpt\', \'None\', \'0\', \'None\', \'None\'], " - } - member_method { - name: "loop" - argspec: "args=[\'self\', \'timer_interval_secs\', \'target\', \'args\', \'kwargs\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " - } - member_method { - name: "managed_session" - argspec: "args=[], varargs=args, keywords=kwds, defaults=None" - } - member_method { - name: "prepare_or_wait_for_session" - argspec: "args=[\'self\', \'master\', \'config\', \'wait_for_checkpoint\', \'max_wait_secs\', \'start_standard_services\'], varargs=None, keywords=None, defaults=[\'\', \'None\', \'False\', \'7200\', \'True\'], " - } - member_method { - name: "request_stop" - argspec: "args=[\'self\', \'ex\'], varargs=None, keywords=None, defaults=[\'None\'], " - } - member_method { - name: "should_stop" - argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" - } - member_method { - name: "start_queue_runners" - argspec: "args=[\'self\', \'sess\', \'queue_runners\'], varargs=None, keywords=None, defaults=[\'None\'], " - } - member_method { - name: "start_standard_services" - argspec: "args=[\'self\', \'sess\'], varargs=None, keywords=None, defaults=None" - } - member_method { - name: "stop" - argspec: "args=[\'self\', \'threads\', \'close_summary_writer\', \'ignore_live_threads\'], varargs=None, keywords=None, defaults=[\'None\', \'True\', \'False\'], " - } - member_method { - name: "stop_on_exception" - argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" - } - member_method { - name: "summary_computed" - argspec: "args=[\'self\', \'sess\', \'summary\', \'global_step\'], varargs=None, keywords=None, defaults=[\'None\'], " - } - member_method { - name: "wait_for_stop" - argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" - } -} diff --git a/tensorflow/tools/api/golden/v2/tensorflow.train.-vocab-info.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.train.-vocab-info.pbtxt deleted file mode 100644 index 39b946b82f3..00000000000 --- a/tensorflow/tools/api/golden/v2/tensorflow.train.-vocab-info.pbtxt +++ /dev/null @@ -1,43 +0,0 @@ -path: "tensorflow.train.VocabInfo" -tf_class { - is_instance: "" - is_instance: "" - is_instance: "" - member { - name: "axis" - mtype: "" - } - member { - name: "backup_initializer" - mtype: "" - } - member { - name: "new_vocab" - mtype: "" - } - member { - name: "new_vocab_size" - mtype: "" - } - member { - name: "num_oov_buckets" - mtype: "" - } - member { - name: "old_vocab" - mtype: "" - } - member { - name: "old_vocab_size" - mtype: "" - } - member_method { - name: "__init__" - } - member_method { - name: "count" - } - member_method { - name: "index" - } -} diff --git a/tensorflow/tools/api/golden/v2/tensorflow.train.-worker-session-creator.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.train.-worker-session-creator.pbtxt deleted file mode 100644 index ac263580687..00000000000 --- a/tensorflow/tools/api/golden/v2/tensorflow.train.-worker-session-creator.pbtxt +++ /dev/null @@ -1,14 +0,0 @@ -path: "tensorflow.train.WorkerSessionCreator" -tf_class { - is_instance: "" - is_instance: "" - is_instance: "" - member_method { - name: "__init__" - argspec: "args=[\'self\', \'scaffold\', \'master\', \'config\', \'max_wait_secs\'], varargs=None, keywords=None, defaults=[\'None\', \'\', \'None\', \'1800\'], " - } - member_method { - name: "create_session" - argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" - } -} diff --git a/tensorflow/tools/api/golden/v2/tensorflow.train.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.train.pbtxt index a091daa2985..3ff4b69d39d 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.train.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.train.pbtxt @@ -1,21 +1,5 @@ path: "tensorflow.train" tf_module { - member { - name: "AdadeltaOptimizer" - mtype: "" - } - member { - name: "AdagradDAOptimizer" - mtype: "" - } - member { - name: "AdagradOptimizer" - mtype: "" - } - member { - name: "AdamOptimizer" - mtype: "" - } member { name: "BytesList" mtype: "" @@ -32,14 +16,6 @@ tf_module { name: "CheckpointSaverHook" mtype: "" } - member { - name: "CheckpointSaverListener" - mtype: "" - } - member { - name: "ChiefSessionCreator" - mtype: "" - } member { name: "ClusterDef" mtype: "" @@ -88,18 +64,10 @@ tf_module { name: "FloatList" mtype: "" } - member { - name: "FtrlOptimizer" - mtype: "" - } member { name: "GlobalStepWaiterHook" mtype: "" } - member { - name: "GradientDescentOptimizer" - mtype: "" - } member { name: "Int64List" mtype: "" @@ -112,54 +80,14 @@ tf_module { name: "LoggingTensorHook" mtype: "" } - member { - name: "LooperThread" - mtype: "" - } - member { - name: "MomentumOptimizer" - mtype: "" - } - member { - name: "MonitoredSession" - mtype: "" - } - member { - name: "NanLossDuringTrainingError" - mtype: "" - } member { name: "NanTensorHook" mtype: "" } - member { - name: "Optimizer" - mtype: "" - } - member { - name: "ProfilerHook" - mtype: "" - } - member { - name: "ProximalAdagradOptimizer" - mtype: "" - } member { name: "ProximalGradientDescentOptimizer" mtype: "" } - member { - name: "RMSPropOptimizer" - mtype: "" - } - member { - name: "Scaffold" - mtype: "" - } - member { - name: "SecondOrStepTimer" - mtype: "" - } member { name: "SequenceExample" mtype: "" @@ -172,34 +100,10 @@ tf_module { name: "ServerDef" mtype: "" } - member { - name: "SessionCreator" - mtype: "" - } - member { - name: "SessionManager" - mtype: "" - } - member { - name: "SessionRunArgs" - mtype: "" - } - member { - name: "SessionRunContext" - mtype: "" - } member { name: "SessionRunHook" mtype: "" } - member { - name: "SessionRunValues" - mtype: "" - } - member { - name: "SingularMonitoredSession" - mtype: "" - } member { name: "StepCounterHook" mtype: "" @@ -212,22 +116,6 @@ tf_module { name: "SummarySaverHook" mtype: "" } - member { - name: "Supervisor" - mtype: "" - } - member { - name: "VocabInfo" - mtype: "" - } - member { - name: "WorkerSessionCreator" - mtype: "" - } - member_method { - name: "NewCheckpointReader" - argspec: "args=[\'filepattern\'], varargs=None, keywords=None, defaults=None" - } member_method { name: "cosine_decay" argspec: "args=[\'learning_rate\', \'global_step\', \'decay_steps\', \'alpha\', \'name\'], varargs=None, keywords=None, defaults=[\'0.0\', \'None\'], " @@ -244,10 +132,6 @@ tf_module { name: "get_checkpoint_state" argspec: "args=[\'checkpoint_dir\', \'latest_filename\'], varargs=None, keywords=None, defaults=[\'None\'], " } - member_method { - name: "init_from_checkpoint" - argspec: "args=[\'ckpt_dir_or_file\', \'assignment_map\'], varargs=None, keywords=None, defaults=None" - } member_method { name: "inverse_time_decay" argspec: "args=[\'learning_rate\', \'global_step\', \'decay_steps\', \'decay_rate\', \'staircase\', \'name\'], varargs=None, keywords=None, defaults=[\'False\', \'None\'], " @@ -281,17 +165,13 @@ tf_module { argspec: "args=[\'learning_rate\', \'global_step\', \'decay_steps\', \'initial_variance\', \'variance_decay\', \'num_periods\', \'alpha\', \'beta\', \'name\'], varargs=None, keywords=None, defaults=[\'1.0\', \'0.55\', \'0.5\', \'0.0\', \'0.001\', \'None\'], " } member_method { - name: "piecewise_constant" + name: "piecewise_constant_decay" argspec: "args=[\'x\', \'boundaries\', \'values\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " } member_method { name: "polynomial_decay" argspec: "args=[\'learning_rate\', \'global_step\', \'decay_steps\', \'end_learning_rate\', \'power\', \'cycle\', \'name\'], varargs=None, keywords=None, defaults=[\'0.0001\', \'1.0\', \'False\', \'None\'], " } - member_method { - name: "replica_device_setter" - argspec: "args=[\'ps_tasks\', \'ps_device\', \'worker_device\', \'merge_devices\', \'cluster\', \'ps_ops\', \'ps_strategy\'], varargs=None, keywords=None, defaults=[\'0\', \'/job:ps\', \'/job:worker\', \'True\', \'None\', \'None\', \'None\'], " - } member_method { name: "sdca_fprint" argspec: "args=[\'input\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " @@ -308,8 +188,4 @@ tf_module { name: "summary_iterator" argspec: "args=[\'path\'], varargs=None, keywords=None, defaults=None" } - member_method { - name: "warm_start" - argspec: "args=[\'ckpt_to_initialize_from\', \'vars_to_warm_start\', \'var_name_to_vocab_info\', \'var_name_to_prev_var_name\'], varargs=None, keywords=None, defaults=[\'.*\', \'None\', \'None\'], " - } } diff --git a/tensorflow/tools/api/tests/api_compatibility_test.py b/tensorflow/tools/api/tests/api_compatibility_test.py index fb489ea80fb..e7f23a11740 100644 --- a/tensorflow/tools/api/tests/api_compatibility_test.py +++ b/tensorflow/tools/api/tests/api_compatibility_test.py @@ -33,12 +33,13 @@ import re import sys import tensorflow as tf -from tensorflow._api import v2 as tf_v2 +from tensorflow._api.v2 import v2 as tf_v2 from google.protobuf import message from google.protobuf import text_format from tensorflow.python.lib.io import file_io +from tensorflow.python.framework import test_util from tensorflow.python.platform import resource_loader from tensorflow.python.platform import test from tensorflow.python.platform import tf_logging as logging @@ -126,9 +127,9 @@ def _FilterNonCoreGoldenFiles(golden_file_list): filtered_file_list = [] filtered_package_prefixes = ['tensorflow.%s.' % p for p in _NON_CORE_PACKAGES] for f in golden_file_list: - if any([ + if any( f.rsplit('/')[-1].startswith(pre) for pre in filtered_package_prefixes - ]): + ): continue filtered_file_list.append(f) return filtered_file_list @@ -310,6 +311,7 @@ class ApiCompatibilityTest(test.TestCase): update_goldens=FLAGS.update_goldens, api_version=api_version) + @test_util.run_deprecated_v1 def testAPIBackwardsCompatibility(self): api_version = 1 golden_file_pattern = os.path.join( @@ -328,6 +330,7 @@ class ApiCompatibilityTest(test.TestCase): 'tensorflow.python.util.lazy_loader.LazyLoader' in str(type(tf.contrib))) + @test_util.run_deprecated_v1 def testAPIBackwardsCompatibilityV1(self): api_version = 1 golden_file_pattern = os.path.join( diff --git a/tensorflow/tools/ci_build/Dockerfile.rbe.cuda10.0-cudnn7-ubuntu14.04 b/tensorflow/tools/ci_build/Dockerfile.rbe.cuda10.0-cudnn7-ubuntu14.04 new file mode 100644 index 00000000000..03de89b7176 --- /dev/null +++ b/tensorflow/tools/ci_build/Dockerfile.rbe.cuda10.0-cudnn7-ubuntu14.04 @@ -0,0 +1,75 @@ +# To push a new version, run: +# $ docker build -f Dockerfile.rbe.cuda10.0-cudnn7-ubuntu14.04 \ +# --tag "gcr.io/asci-toolchain/nosla-cuda10.0-cudnn7-ubuntu14.04" . +# $ docker push gcr.io/asci-toolchain/nosla-cuda10.0-cudnn7-ubuntu14.04 + +FROM ubuntu:14.04 +LABEL maintainer="Manuel Klimek " + +RUN apt-get update && apt-get install -y --no-install-recommends ca-certificates apt-transport-https gnupg-curl && \ + rm -rf /var/lib/apt/lists/* && \ + NVIDIA_GPGKEY_SUM=d1be581509378368edeec8c1eb2958702feedf3bc3d17011adbf24efacce4ab5 && \ + NVIDIA_GPGKEY_FPR=ae09fe4bbd223a84b2ccfce3f60f4b3d7fa2af80 && \ + apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1604/x86_64/7fa2af80.pub && \ + apt-key adv --export --no-emit-version -a $NVIDIA_GPGKEY_FPR | tail -n +2 > cudasign.pub && \ + echo "$NVIDIA_GPGKEY_SUM cudasign.pub" | sha256sum -c --strict - && rm cudasign.pub && \ + echo "deb https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1604/x86_64 /" > /etc/apt/sources.list.d/cuda.list && \ + echo "deb https://developer.download.nvidia.com/compute/machine-learning/repos/ubuntu1604/x86_64 /" > /etc/apt/sources.list.d/nvidia-ml.list + +ENV CUDA_VERSION 10.0.130 +ENV CUDA_PKG_VERSION 10-0=$CUDA_VERSION-1 +ENV CUDNN_VERSION 7.3.1.20 +ENV NCCL_VERSION 2.3.5 +ENV NVIDIA_DRIVER_CAPABILITIES compute,utility +ENV NVIDIA_REQUIRE_CUDA "cuda>=10.0,driver>=410" +ENV NVIDIA_VISIBLE_DEVICES all +ENV PATH /usr/local/cuda/bin:${PATH} + +# TODO(b/110903506): /usr/loca/cuda/lib64/stubs should not be needed in +# LD_LIBRARY_PATH. The stubs/libcuda.so is not meant to used at runtime. The +# correct way to pass the path to bfd-ld is to pass +# -Wl,-rpath-link=/usr/local/cuda/lib64/stubs to all binaries transitively +# depending on libcuda. Optimally, builds targeting cuda would do that +# internally. +ENV LD_LIBRARY_PATH /usr/local/cuda/lib64/stubs + +LABEL com.nvidia.cudnn.version="${CUDNN_VERSION}" + +RUN apt-get update && apt-get install -y --no-install-recommends \ + cuda-command-line-tools-$CUDA_PKG_VERSION \ + cuda-compat-10-0=410.48-1 \ + cuda-cudart-$CUDA_PKG_VERSION \ + cuda-libraries-$CUDA_PKG_VERSION \ + cuda-libraries-dev-$CUDA_PKG_VERSION \ + cuda-minimal-build-$CUDA_PKG_VERSION \ + cuda-nvml-dev-$CUDA_PKG_VERSION \ + cuda-nvtx-$CUDA_PKG_VERSION \ + libcudnn7=$CUDNN_VERSION-1+cuda10.0 \ + libcudnn7=$CUDNN_VERSION-1+cuda10.0 \ + libcudnn7-dev=$CUDNN_VERSION-1+cuda10.0 \ + libnccl2=$NCCL_VERSION-2+cuda10.0 \ + libnccl-dev=$NCCL_VERSION-2+cuda10.0 && \ + ln -s cuda-10.0 /usr/local/cuda && \ + apt-mark hold libcudnn7 && \ + apt-mark hold libnccl2 && \ + rm -rf /var/lib/apt/lists/* + +# TODO(b/110903506): Provide a link to the SONAME of libcuda.so. +# https://github.com/NVIDIA/nvidia-docker/issues/775 +RUN ln -s libcuda.so /usr/local/cuda/lib64/stubs/libcuda.so.1 + +# TODO(klimek): Once the TODO in tensorflow's configure.py to correctly find +# libnccl is resolved, delete this block. +RUN ln -s /usr/lib/x86_64-linux-gnu/libnccl.so /usr/lib/libnccl.so \ + && ln -s /usr/lib/x86_64-linux-gnu/libnccl.so /usr/lib/libnccl.so.2 + +# Copy and run the install scripts. +COPY install/*.sh /install/ +ARG DEBIAN_FRONTEND=noninteractive +RUN /install/install_bootstrap_deb_packages.sh +RUN add-apt-repository -y ppa:openjdk-r/ppa && \ + add-apt-repository -y ppa:george-edison55/cmake-3.x +RUN /install/install_deb_packages.sh +RUN /install/install_pip_packages.sh +RUN /install/install_golang.sh + diff --git a/tensorflow/tools/ci_build/builds/libtensorflow.sh b/tensorflow/tools/ci_build/builds/libtensorflow.sh index 9b3ff0cba7d..44abcc309b9 100755 --- a/tensorflow/tools/ci_build/builds/libtensorflow.sh +++ b/tensorflow/tools/ci_build/builds/libtensorflow.sh @@ -55,6 +55,7 @@ function build_libtensorflow_tarball() { export CC_OPT_FLAGS='-mavx' if [ "${TF_NEED_CUDA}" == "1" ]; then BAZEL_OPTS="${BAZEL_OPTS} --config=cuda" + export TF_NEED_ROCM=0 fi bazel clean --expunge yes "" | ./configure diff --git a/tensorflow/tools/ci_build/windows/cpu/pip/build_tf_windows.sh b/tensorflow/tools/ci_build/windows/cpu/pip/build_tf_windows.sh index 177ef390dbd..46f5bdef09d 100644 --- a/tensorflow/tools/ci_build/windows/cpu/pip/build_tf_windows.sh +++ b/tensorflow/tools/ci_build/windows/cpu/pip/build_tf_windows.sh @@ -58,6 +58,8 @@ PY_TEST_DIR="py_test_dir" SKIP_TEST=0 RELEASE_BUILD=0 TEST_TARGET="//${PY_TEST_DIR}/tensorflow/python/..." +PROJECT_NAME="" +EXTRA_BUILD_FLAGS="" # --skip_test Skip running tests # --enable_remote_cache Add options to enable remote cache for build and test @@ -73,6 +75,20 @@ for ARG in "$@"; do --release_build) RELEASE_BUILD=1 ;; --test_core_only) TEST_TARGET="//${PY_TEST_DIR}/tensorflow/python/..." ;; --test_contrib_only) TEST_TARGET="//${PY_TEST_DIR}/tensorflow/contrib/..." ;; + --extra_build_flags) + shift + if [[ -z "$1" ]]; then + break + fi + EXTRA_BUILD_FLAGS="$1" + ;; + --project_name) + shift + if [[ -z "$1" ]]; then + break + fi + PROJECT_NAME="$1" + ;; *) esac done @@ -88,7 +104,11 @@ fi if [[ "$TF_NIGHTLY" == 1 ]]; then python tensorflow/tools/ci_build/update_version.py --nightly - EXTRA_PIP_FLAG="--nightly_flag" + if [ -z ${PROJECT_NAME} ]; then + EXTRA_PIP_FLAGS="--nightly_flag" + else + EXTRA_PIP_FLAGS="--project_name=${PROJECT_NAME} --nightly_flag" + fi fi # Enable short object file path to avoid long path issue on Windows. @@ -100,7 +120,8 @@ fi run_configure_for_cpu_build -bazel build --announce_rc --config=opt tensorflow/tools/pip_package:build_pip_package || exit $? +bazel build --announce_rc --config=opt ${EXTRA_BUILD_FLAGS} \ + tensorflow/tools/pip_package:build_pip_package || exit $? if [[ "$SKIP_TEST" == 1 ]]; then exit 0 @@ -109,7 +130,7 @@ fi # Create a python test directory to avoid package name conflict create_python_test_dir "${PY_TEST_DIR}" -./bazel-bin/tensorflow/tools/pip_package/build_pip_package "$PWD/${PY_TEST_DIR}" "${EXTRA_PIP_FLAG}" +./bazel-bin/tensorflow/tools/pip_package/build_pip_package "$PWD/${PY_TEST_DIR}" "${EXTRA_PIP_FLAGS}" if [[ "$TF_NIGHTLY" == 1 ]]; then exit 0 @@ -126,8 +147,8 @@ N_JOBS="${NUMBER_OF_PROCESSORS}" # which will result testing system installed tensorflow bazel test --announce_rc --config=opt -k --test_output=errors \ --define=no_tensorflow_py_deps=true --test_lang_filters=py \ - --test_tag_filters=-no_pip,-no_windows,-no_oss \ - --build_tag_filters=-no_pip,-no_windows,-no_oss --build_tests_only \ + --test_tag_filters=-no_pip,-no_windows,-no_oss,-gpu \ + --build_tag_filters=-no_pip,-no_windows,-no_oss,-gpu --build_tests_only \ --test_size_filters=small,medium \ --jobs="${N_JOBS}" --test_timeout="300,450,1200,3600" \ --flaky_test_attempts=3 \ diff --git a/tensorflow/tools/ci_build/windows/gpu/pip/build_tf_windows.sh b/tensorflow/tools/ci_build/windows/gpu/pip/build_tf_windows.sh index 6178d7794df..3aec8d65843 100644 --- a/tensorflow/tools/ci_build/windows/gpu/pip/build_tf_windows.sh +++ b/tensorflow/tools/ci_build/windows/gpu/pip/build_tf_windows.sh @@ -58,6 +58,8 @@ PY_TEST_DIR="py_test_dir" SKIP_TEST=0 RELEASE_BUILD=0 TEST_TARGET="//${PY_TEST_DIR}/tensorflow/python/..." +PROJECT_NAME="" +EXTRA_BUILD_FLAGS="" # --skip_test Skip running tests # --enable_remote_cache Add options to enable remote cache for build and test @@ -73,6 +75,20 @@ for ARG in "$@"; do --release_build) RELEASE_BUILD=1 ;; --test_core_only) TEST_TARGET="//${PY_TEST_DIR}/tensorflow/python/..." ;; --test_contrib_only) TEST_TARGET="//${PY_TEST_DIR}/tensorflow/contrib/..." ;; + --extra_build_flags) + shift + if [[ -z "$1" ]]; then + break + fi + EXTRA_BUILD_FLAGS="$1" + ;; + --project_name) + shift + if [[ -z "$1" ]]; then + break + fi + PROJECT_NAME="$1" + ;; *) esac done @@ -88,7 +104,11 @@ fi if [[ "$TF_NIGHTLY" == 1 ]]; then python tensorflow/tools/ci_build/update_version.py --nightly - EXTRA_PIP_FLAG="--nightly_flag" + if [ -z ${PROJECT_NAME} ]; then + EXTRA_PIP_FLAGS="--nightly_flag" + else + EXTRA_PIP_FLAGS="--project_name=${PROJECT_NAME} --nightly_flag" + fi fi # Enable short object file path to avoid long path issue on Windows. @@ -104,6 +124,7 @@ fi run_configure_for_gpu_build bazel build --announce_rc --config=opt --define=no_tensorflow_py_deps=true \ + ${EXTRA_BUILD_FLAGS} \ tensorflow/tools/pip_package:build_pip_package || exit $? if [[ "$SKIP_TEST" == 1 ]]; then @@ -113,7 +134,8 @@ fi # Create a python test directory to avoid package name conflict create_python_test_dir "${PY_TEST_DIR}" -./bazel-bin/tensorflow/tools/pip_package/build_pip_package "$PWD/${PY_TEST_DIR}" --gpu "${EXTRA_PIP_FLAG}" +./bazel-bin/tensorflow/tools/pip_package/build_pip_package "$PWD/${PY_TEST_DIR}" \ + --gpu "${EXTRA_PIP_FLAGS}" if [[ "$TF_NIGHTLY" == 1 ]]; then exit 0 diff --git a/tensorflow/tools/compatibility/BUILD b/tensorflow/tools/compatibility/BUILD index 5f619c4e62a..05d924c092c 100644 --- a/tensorflow/tools/compatibility/BUILD +++ b/tensorflow/tools/compatibility/BUILD @@ -14,6 +14,18 @@ py_library( srcs_version = "PY2AND3", ) +py_test( + name = "ast_edits_test", + srcs = ["ast_edits_test.py"], + srcs_version = "PY2AND3", + deps = [ + ":ast_edits", + "//tensorflow/python:client_testlib", + "//tensorflow/python:framework_test_lib", + "@six_archive//:six", + ], +) + py_binary( name = "tf_upgrade", srcs = ["tf_upgrade.py"], @@ -39,8 +51,8 @@ py_library( srcs_version = "PY2AND3", ) -py_binary( - name = "tf_upgrade_v2", +py_library( + name = "tf_upgrade_v2_lib", srcs = [ "renames_v2.py", "tf_upgrade_v2.py", @@ -49,14 +61,28 @@ py_binary( deps = [":ast_edits"], ) +py_binary( + name = "tf_upgrade_v2", + srcs = ["tf_upgrade_v2_main.py"], + main = "tf_upgrade_v2_main.py", + srcs_version = "PY2AND3", + deps = [ + ":ast_edits", + ":tf_upgrade_v2_lib", + ], +) + py_test( name = "tf_upgrade_v2_test", srcs = ["tf_upgrade_v2_test.py"], srcs_version = "PY2AND3", deps = [ ":tf_upgrade_v2", + "//tensorflow:tensorflow_py", "//tensorflow/python:client_testlib", "//tensorflow/python:framework_test_lib", + "//tensorflow/tools/common:public_api", + "//tensorflow/tools/common:traverse", "@six_archive//:six", ], ) @@ -100,18 +126,28 @@ py_test( genrule( name = "generate_upgraded_file_v2", testonly = 1, - srcs = ["testdata/test_file_v1_10.py"], + srcs = ["testdata/test_file_v1_12.py"], outs = [ "test_file_v2_0.py", "report_v2.txt", ], cmd = ("$(location :tf_upgrade_v2)" + - " --infile $(location testdata/test_file_v1_10.py)" + + " --infile $(location testdata/test_file_v1_12.py)" + " --outfile $(location test_file_v2_0.py)" + " --reportfile $(location report_v2.txt)"), tools = [":tf_upgrade_v2"], ) +py_test( + name = "test_file_v1_12", + size = "small", + srcs = ["testdata/test_file_v1_12.py"], + srcs_version = "PY2AND3", + deps = [ + "//tensorflow:tensorflow_py", + ], +) + py_test( name = "test_file_v2_0", size = "small", @@ -128,6 +164,6 @@ exports_files( "tf_upgrade.py", "renames_v2.py", "testdata/test_file_v0_11.py", - "testdata/test_file_v1_10.py", + "testdata/test_file_v1_12.py", ], ) diff --git a/tensorflow/tools/compatibility/README.md b/tensorflow/tools/compatibility/README.md index aabc7b253d6..6ff42b1fefe 100644 --- a/tensorflow/tools/compatibility/README.md +++ b/tensorflow/tools/compatibility/README.md @@ -1,60 +1,77 @@ # TensorFlow Python API Upgrade Utility This tool allows you to upgrade your existing TensorFlow Python scripts. -This script can be run on a single Python file: +Specifically: \ +`tf_upgrade_v2.py`: upgrades code from TensorFlow 1.12 to TensorFlow 2.0 preview. \ +`tf_upgrade.py`: upgrades code to TensorFlow 1.0 from TensorFlow 0.11. + +## Running the script from pip package + +First, install TensorFlow pip package. See +https://www.tensorflow.org/install/pip. + +Upgrade script can be run on a single Python file: ``` -tf_upgrade.py --infile foo.py --outfile foo-upgraded.py +tf_upgrade_v2 --infile foo.py --outfile foo-upgraded.py ``` It will print a list of errors it finds that it can't fix. You can also run it on a directory tree: ``` +# upgrade the .py files and copy all the other files to the outtree +tf_upgrade_v2 --intree coolcode --outtree coolcode-upgraded + # just upgrade the .py files -tf_upgrade.py --intree coolcode --outtree coolcode-upgraded -# after upgrade the .py files, then copy all the other files to the outtree -tf_upgrade.py --intree coolcode --outtree coolcode-upgraded --copyotherfiles True +tf_upgrade_v2 --intree coolcode --outtree coolcode-upgraded --copyotherfiles False ``` -In either case, it will also dump out a report e.g. which will detail changes + +## Report + +The script will also dump out a report e.g. which will detail changes e.g.: ``` -third_party/tensorflow/tools/compatibility/test_file_v0.11.py Line 125 +'tensorflow/tools/compatibility/testdata/test_file_v1_12.py' Line 65 +-------------------------------------------------------------------------------- -Renamed keyword argument from `dim` to `axis` -Renamed keyword argument from `squeeze_dims` to `axis` +Added keyword 'input' to reordered function 'tf.argmax' +Renamed keyword argument from 'dimension' to 'axis' + + Old: tf.argmax([[1, 3, 2]], dimension=0)) + ~~~~~~~~~~ + New: tf.argmax(input=[[1, 3, 2]], axis=0)) - Old: [[1, 2, 3]], dim=1), squeeze_dims=[1]).eval(), - ~~~~ ~~~~~~~~~~~~~ - New: [[1, 2, 3]], axis=1), axis=[1]).eval(), - ~~~~~ ~~~~~ ``` ## Caveats - Don't update parts of your code manually before running this script. In -particular, functions that have had reordered arguments like `tf.concat` -or `tf.split` will cause the script to incorrectly add keyword arguments that -mismap arguments. +particular, functions that have had reordered arguments like `tf.argmax` +or `tf.batch_to_space` will cause the script to incorrectly add keyword +arguments that mismap arguments. - This script wouldn't actually reorder arguments. Instead, the script will add keyword arguments to functions that had their arguments reordered. - This script is not able to upgrade all functions. One notable example is -`tf.reverse()` which has been changed to take a list of indices rather than -a tensor of bools. If the script detects this, it will report this to stdout +`tf.nn.conv2d` that no longer takes `use_cudnn_on_gpu` argument. +If the script detects this, it will report this to stdout (and in the report), and you can fix it manually. For example if you have -`tf.reverse(a, [False, True, True])` you will need to manually change it to -`tf.reverse(a, [1, 2])`. +`tf.nn.conv2d(inputs, filters, strides, padding, use_cudnn_on_gpu=True)` +you will need to manually change it to +`tf.nn.conv2d(input, filters, strides, padding)`. - There are some syntaxes that are not handleable with this script as this -script was designed to use only standard python packages. If the script fails -with "A necessary keyword argument failed to be inserted." or +script was designed to use only standard python packages. +There is an alternative available for TensorFlow 0.* to 1.0 upgrade script. +If the script fails with "A necessary keyword argument failed to be inserted." or "Failed to find keyword lexicographically. Fix manually.", you can try [@machrisaa's fork of this script](https://github.com/machrisaa/tf0to1). [@machrisaa](https://github.com/machrisaa) has used the [RedBaron Python refactoring engine](https://redbaron.readthedocs.io/en/latest/) which is able to localize syntactic elements more reliably than the built-in -`ast` module this script is based upon. +`ast` module this script is based upon. Note that the alternative script is not +available for TensorFlow 2.0 upgrade. diff --git a/tensorflow/tools/compatibility/ast_edits.py b/tensorflow/tools/compatibility/ast_edits.py index 56c67b83565..eac2150502d 100644 --- a/tensorflow/tools/compatibility/ast_edits.py +++ b/tensorflow/tools/compatibility/ast_edits.py @@ -21,11 +21,16 @@ from __future__ import print_function import ast import collections import os +import re import shutil import sys import tempfile import traceback +# Some regular expressions we will need for parsing +FIND_OPEN = re.compile(r"^\s*(\[).*$") +FIND_STRING_CHARS = re.compile(r"['\"]") + class APIChangeSpec(object): """This class defines the transformations that need to happen. @@ -40,6 +45,10 @@ class APIChangeSpec(object): * `function_reorders`: maps functions whose argument order has changed to the list of arguments in the new order * `function_handle`: maps function names to custom handlers for the function + * `function_warnings`: maps full names of functions to warnings that will be + printed out if the function is used. (e.g. tf.nn.convolution()) + * `unrestricted_function_warnings`: maps names of functions to warnings that + will be printed out when the function is used (e.g. foo.convolution()). For an example, see `TFAPIChangeSpec`. """ @@ -53,7 +62,7 @@ class _FileEditTuple( Fields: comment: A description of the edit and why it was made. line: The line number in the file where the edit occurs (1-indexed). - start: The line number in the file where the edit occurs (0-indexed). + start: The column number in the file where the edit occurs (0-indexed). old: text string to remove (this must match what was in file). new: text string to add in place of `old`. """ @@ -195,6 +204,29 @@ class _ASTCallVisitor(ast.NodeVisitor): except KeyError: pass + def _print_warning_for_function_unrestricted(self, node): + """Print a warning when specific functions are called. + + The function _print_warning_for_function matches the full name of the called + function, e.g., tf.foo.bar(). This function matches the function name that + is called, as long as the function is an attribute. For example, + `tf.foo.bar()` and `foo.bar()` are matched, but not `bar()`. + + Args: + node: ast.Call object + """ + function_warnings = getattr( + self._api_change_spec, "unrestricted_function_warnings", {}) + if isinstance(node.func, ast.Attribute): + function_name = node.func.attr + try: + warning_message = function_warnings[function_name] + self._file_edit.add(warning_message, + node.lineno, node.col_offset, "", "", + error="%s requires manual check." % function_name) + except KeyError: + pass + def _get_attribute_full_path(self, node): """Traverse an attribute to generate a full name e.g. tf.foo.bar. @@ -209,11 +241,11 @@ class _ASTCallVisitor(ast.NodeVisitor): items = [] while not isinstance(curr, ast.Name): if not isinstance(curr, ast.Attribute): - return None + return None, None items.append(curr.attr) curr = curr.value items.append(curr.id) - return ".".join(reversed(items)) + return ".".join(reversed(items)), items[0] def _find_true_position(self, node): """Return correct line number and column offset for a given node. @@ -221,13 +253,12 @@ class _ASTCallVisitor(ast.NodeVisitor): This is necessary mainly because ListComp's location reporting reports the next token after the list comprehension list opening. + Returns: + lineno, offset for the given node + Args: node: Node for which we wish to know the lineno and col_offset """ - import re - find_open = re.compile("^\s*(\\[).*$") - find_string_chars = re.compile("['\"]") - if isinstance(node, ast.ListComp): # Strangely, ast.ListComp returns the col_offset of the first token # after the '[' token which appears to be a bug. Workaround by @@ -241,7 +272,7 @@ class _ASTCallVisitor(ast.NodeVisitor): reversed_preceding_text = text[:col][::-1] # First find if a [ can be found with only whitespace between it and # col. - m = find_open.match(reversed_preceding_text) + m = FIND_OPEN.match(reversed_preceding_text) if m: new_col_offset = col - m.start(1) - 1 return line, new_col_offset @@ -260,7 +291,7 @@ class _ASTCallVisitor(ast.NodeVisitor): comment_start = prev_line.find("#") if comment_start == -1: col = len(prev_line) - 1 - elif find_string_chars.search(prev_line[comment_start:]) is None: + elif FIND_STRING_CHARS.search(prev_line[comment_start:]) is None: col = comment_start else: return None, None @@ -276,9 +307,10 @@ class _ASTCallVisitor(ast.NodeVisitor): Args: node: Current Node """ + self._print_warning_for_function_unrestricted(node) # Find a simple attribute name path e.g. "tf.foo.bar" - full_name = self._get_attribute_full_path(node.func) + full_name, name = self._get_attribute_full_path(node.func) # Make sure the func is marked as being part of a call node.func.is_function_for_call = True @@ -286,6 +318,9 @@ class _ASTCallVisitor(ast.NodeVisitor): if full_name: # Call special handlers function_handles = self._api_change_spec.function_handle + glob_name = "*.{}".format(name) + if glob_name in function_handles: + function_handles[glob_name](self._file_edit, node) if full_name in function_handles: function_handles[full_name](self._file_edit, node) @@ -358,10 +393,11 @@ class _ASTCallVisitor(ast.NodeVisitor): Args: node: Node that is of type ast.Attribute """ - full_name = self._get_attribute_full_path(node) + full_name, _ = self._get_attribute_full_path(node) if full_name: - self._rename_functions(node, full_name) + # Make sure the warning comes first, otherwise the name may have changed self._print_warning_for_function(node, full_name) + self._rename_functions(node, full_name) if full_name in self._api_change_spec.change_to_function: if not hasattr(node, "is_function_for_call"): new_text = full_name + "()" diff --git a/tensorflow/tools/compatibility/ast_edits_test.py b/tensorflow/tools/compatibility/ast_edits_test.py new file mode 100644 index 00000000000..99f20a026fc --- /dev/null +++ b/tensorflow/tools/compatibility/ast_edits_test.py @@ -0,0 +1,420 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Tests for ast_edits which is used in tf upgraders. + +All of the tests assume that we want to change from an API containing + + def f(a, b, kw1, kw2): ... + def g(a, b, kw1, c, kw1_alias): ... + def g2(a, b, kw1, c, d, kw1_alias): ... + def h(a, kw1, kw2, kw1_alias, kw2_alias): ... + +and the changes to the API consist of renaming, reordering, and/or removing +arguments. Thus, we want to be able to generate changes to produce each of the +following new APIs: + + def f(a, b, kw1, kw3): ... + def f(a, b, kw2, kw1): ... + def f(a, b, kw3, kw1): ... + def g(a, b, kw1, c): ... + def g(a, b, c, kw1): ... + def g2(a, b, kw1, c, d): ... + def g2(a, b, c, d, kw1): ... + def h(a, kw1, kw2): ... + +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +import six +from tensorflow.python.framework import test_util +from tensorflow.python.platform import test as test_lib +from tensorflow.tools.compatibility import ast_edits + + +class NoUpdateSpec(ast_edits.APIChangeSpec): + """A specification of an API change which doesn't change anything.""" + + def __init__(self): + self.function_handle = {} + self.function_reorders = {} + self.function_keyword_renames = {} + self.symbol_renames = {} + self.function_warnings = {} + self.unrestricted_function_warnings = {} + self.change_to_function = {} + + +class RenameKeywordSpec(NoUpdateSpec): + """A specification where kw2 gets renamed to kw3. + + The new API is + + def f(a, b, kw1, kw3): ... + + """ + + def __init__(self): + NoUpdateSpec.__init__(self) + self.update_renames() + + def update_renames(self): + self.function_keyword_renames["f"] = {"kw2": "kw3"} + + +class ReorderKeywordSpec(NoUpdateSpec): + """A specification where kw2 gets moved in front of kw1. + + The new API is + + def f(a, b, kw2, kw1): ... + + """ + + def __init__(self): + NoUpdateSpec.__init__(self) + self.update_reorders() + + def update_reorders(self): + # Note that these should be in the old order. + self.function_reorders["f"] = ["a", "b", "kw1", "kw2"] + + +class ReorderAndRenameKeywordSpec(ReorderKeywordSpec, RenameKeywordSpec): + """A specification where kw2 gets moved in front of kw1 and is changed to kw3. + + The new API is + + def f(a, b, kw3, kw1): ... + + """ + + def __init__(self): + ReorderKeywordSpec.__init__(self) + RenameKeywordSpec.__init__(self) + self.update_renames() + self.update_reorders() + + +class RemoveDeprecatedAliasKeyword(NoUpdateSpec): + """A specification where kw1_alias is removed in g. + + The new API is + + def g(a, b, kw1, c): ... + def g2(a, b, kw1, c, d): ... + + """ + + def __init__(self): + NoUpdateSpec.__init__(self) + self.function_keyword_renames["g"] = {"kw1_alias": "kw1"} + self.function_keyword_renames["g2"] = {"kw1_alias": "kw1"} + + +class RemoveDeprecatedAliasAndReorderRest(RemoveDeprecatedAliasKeyword): + """A specification where kw1_alias is removed in g. + + The new API is + + def g(a, b, c, kw1): ... + def g2(a, b, c, d, kw1): ... + + """ + + def __init__(self): + RemoveDeprecatedAliasKeyword.__init__(self) + # Note that these should be in the old order. + self.function_reorders["g"] = ["a", "b", "kw1", "c"] + self.function_reorders["g2"] = ["a", "b", "kw1", "c", "d"] + + +class RemoveMultipleKeywordArguments(NoUpdateSpec): + """A specification where both keyword aliases are removed from h. + + The new API is + + def h(a, kw1, kw2): ... + + """ + + def __init__(self): + NoUpdateSpec.__init__(self) + self.function_keyword_renames["h"] = { + "kw1_alias": "kw1", + "kw2_alias": "kw2", + } + + +class TestAstEdits(test_util.TensorFlowTestCase): + + def _upgrade(self, spec, old_file_text): + in_file = six.StringIO(old_file_text) + out_file = six.StringIO() + upgrader = ast_edits.ASTCodeUpgrader(spec) + count, report, errors = ( + upgrader.process_opened_file("test.py", in_file, + "test_out.py", out_file)) + return (count, report, errors), out_file.getvalue() + + def testNoTransformIfNothingIsSupplied(self): + text = "f(a, b, kw1=c, kw2=d)\n" + _, new_text = self._upgrade(NoUpdateSpec(), text) + self.assertEqual(new_text, text) + + text = "f(a, b, c, d)\n" + _, new_text = self._upgrade(NoUpdateSpec(), text) + self.assertEqual(new_text, text) + + def testKeywordRename(self): + """Test that we get the expected result if renaming kw2 to kw3.""" + text = "f(a, b, kw1=c, kw2=d)\n" + expected = "f(a, b, kw1=c, kw3=d)\n" + _, new_text = self._upgrade(RenameKeywordSpec(), text) + self.assertEqual(new_text, expected) + + # No keywords specified, no reordering, so we should get input as output + text = "f(a, b, c, d)\n" + _, new_text = self._upgrade(RenameKeywordSpec(), text) + self.assertEqual(new_text, text) + + def testKeywordReorder(self): + """Test that we get the expected result if kw2 is now before kw1.""" + text = "f(a, b, kw1=c, kw2=d)\n" + acceptable_outputs = [ + # No change is a valid output + text, + # Just reordering the kw.. args is also ok + "f(a, b, kw2=d, kw1=c)\n", + # Also cases where all arguments are fully specified are allowed + "f(a=a, b=b, kw1=c, kw2=d)\n", + "f(a=a, b=b, kw2=d, kw1=c)\n", + ] + _, new_text = self._upgrade(ReorderKeywordSpec(), text) + self.assertIn(new_text, acceptable_outputs) + + # Keywords are reordered, so we should reorder arguments too + text = "f(a, b, c, d)\n" + acceptable_outputs = [ + "f(a, b, d, c)\n", + "f(a=a, b=b, kw1=c, kw2=d)\n", + "f(a=a, b=b, kw2=d, kw1=c)\n", + ] + _, new_text = self._upgrade(ReorderKeywordSpec(), text) + self.assertIn(new_text, acceptable_outputs) + + def testKeywordReorderAndRename(self): + """Test that we get the expected result if kw2 is renamed and moved.""" + text = "f(a, b, kw1=c, kw2=d)\n" + acceptable_outputs = [ + "f(a, b, kw3=d, kw1=c)\n", + "f(a=a, b=b, kw1=c, kw3=d)\n", + "f(a=a, b=b, kw3=d, kw1=c)\n", + ] + _, new_text = self._upgrade(ReorderAndRenameKeywordSpec(), text) + self.assertIn(new_text, acceptable_outputs) + + # Keywords are reordered, so we should reorder arguments too + text = "f(a, b, c, d)\n" + acceptable_outputs = [ + "f(a, b, d, c)\n", + "f(a=a, b=b, kw1=c, kw3=d)\n", + "f(a=a, b=b, kw3=d, kw1=c)\n", + ] + _, new_text = self._upgrade(ReorderAndRenameKeywordSpec(), text) + self.assertIn(new_text, acceptable_outputs) + + def testRemoveDeprecatedKeywordAlias(self): + """Test that we get the expected result if a keyword alias is removed.""" + text = "g(a, b, kw1=x, c=c)\n" + acceptable_outputs = [ + # Not using deprecated alias, so original is ok + text, + "g(a=a, b=b, kw1=x, c=c)\n", + ] + _, new_text = self._upgrade(RemoveDeprecatedAliasKeyword(), text) + self.assertIn(new_text, acceptable_outputs) + + # No keyword used, should be no change + text = "g(a, b, x, c)\n" + _, new_text = self._upgrade(RemoveDeprecatedAliasKeyword(), text) + self.assertEqual(new_text, text) + + # If we used the alias, it should get renamed + text = "g(a, b, kw1_alias=x, c=c)\n" + acceptable_outputs = [ + "g(a, b, kw1=x, c=c)\n", + "g(a, b, c=c, kw1=x)\n", + "g(a=a, b=b, kw1=x, c=c)\n", + "g(a=a, b=b, c=c, kw1=x)\n", + ] + _, new_text = self._upgrade(RemoveDeprecatedAliasKeyword(), text) + self.assertIn(new_text, acceptable_outputs) + + # It should get renamed even if it's last + text = "g(a, b, c=c, kw1_alias=x)\n" + acceptable_outputs = [ + "g(a, b, kw1=x, c=c)\n", + "g(a, b, c=c, kw1=x)\n", + "g(a=a, b=b, kw1=x, c=c)\n", + "g(a=a, b=b, c=c, kw1=x)\n", + ] + _, new_text = self._upgrade(RemoveDeprecatedAliasKeyword(), text) + self.assertIn(new_text, acceptable_outputs) + + def testRemoveDeprecatedKeywordAndReorder(self): + """Test for when a keyword alias is removed and args are reordered.""" + text = "g(a, b, kw1=x, c=c)\n" + acceptable_outputs = [ + "g(a, b, c=c, kw1=x)\n", + "g(a=a, b=b, kw1=x, c=c)\n", + ] + _, new_text = self._upgrade(RemoveDeprecatedAliasAndReorderRest(), text) + self.assertIn(new_text, acceptable_outputs) + + # Keywords are reordered, so we should reorder arguments too + text = "g(a, b, x, c)\n" + # Don't accept an output which doesn't reorder c and d + acceptable_outputs = [ + "g(a, b, c, x)\n", + "g(a=a, b=b, kw1=x, c=c)\n", + ] + _, new_text = self._upgrade(RemoveDeprecatedAliasAndReorderRest(), text) + self.assertIn(new_text, acceptable_outputs) + + # If we used the alias, it should get renamed + text = "g(a, b, kw1_alias=x, c=c)\n" + acceptable_outputs = [ + "g(a, b, kw1=x, c=c)\n", + "g(a, b, c=c, kw1=x)\n", + "g(a=a, b=b, kw1=x, c=c)\n", + "g(a=a, b=b, c=c, kw1=x)\n", + ] + _, new_text = self._upgrade(RemoveDeprecatedAliasKeyword(), text) + self.assertIn(new_text, acceptable_outputs) + + # It should get renamed and reordered even if it's last + text = "g(a, b, c=c, kw1_alias=x)\n" + acceptable_outputs = [ + "g(a, b, kw1=x, c=c)\n", + "g(a, b, c=c, kw1=x)\n", + "g(a=a, b=b, kw1=x, c=c)\n", + "g(a=a, b=b, c=c, kw1=x)\n", + ] + _, new_text = self._upgrade(RemoveDeprecatedAliasKeyword(), text) + self.assertIn(new_text, acceptable_outputs) + + def testRemoveDeprecatedKeywordAndReorder2(self): + """Same as testRemoveDeprecatedKeywordAndReorder but on g2 (more args).""" + text = "g2(a, b, kw1=x, c=c, d=d)\n" + acceptable_outputs = [ + "g2(a, b, c=c, d=d, kw1=x)\n", + "g2(a=a, b=b, kw1=x, c=c, d=d)\n", + ] + _, new_text = self._upgrade(RemoveDeprecatedAliasAndReorderRest(), text) + self.assertIn(new_text, acceptable_outputs) + + # Keywords are reordered, so we should reorder arguments too + text = "g2(a, b, x, c, d)\n" + # Don't accept an output which doesn't reorder c and d + acceptable_outputs = [ + "g2(a, b, c, d, x)\n", + "g2(a=a, b=b, kw1=x, c=c, d=d)\n", + ] + _, new_text = self._upgrade(RemoveDeprecatedAliasAndReorderRest(), text) + self.assertIn(new_text, acceptable_outputs) + + # If we used the alias, it should get renamed + text = "g2(a, b, kw1_alias=x, c=c, d=d)\n" + acceptable_outputs = [ + "g2(a, b, kw1=x, c=c, d=d)\n", + "g2(a, b, c=c, d=d, kw1=x)\n", + "g2(a=a, b=b, kw1=x, c=c, d=d)\n", + "g2(a=a, b=b, c=c, d=d, kw1=x)\n", + ] + _, new_text = self._upgrade(RemoveDeprecatedAliasKeyword(), text) + self.assertIn(new_text, acceptable_outputs) + + # It should get renamed and reordered even if it's not in order + text = "g2(a, b, d=d, c=c, kw1_alias=x)\n" + acceptable_outputs = [ + "g2(a, b, kw1=x, c=c, d=d)\n", + "g2(a, b, c=c, d=d, kw1=x)\n", + "g2(a, b, d=d, c=c, kw1=x)\n", + "g2(a=a, b=b, kw1=x, c=c, d=d)\n", + "g2(a=a, b=b, c=c, d=d, kw1=x)\n", + "g2(a=a, b=b, d=d, c=c, kw1=x)\n", + ] + _, new_text = self._upgrade(RemoveDeprecatedAliasKeyword(), text) + self.assertIn(new_text, acceptable_outputs) + + def testRemoveMultipleKeywords(self): + """Remove multiple keywords at once.""" + # Not using deprecated keywords -> no rename + text = "h(a, kw1=x, kw2=y)\n" + _, new_text = self._upgrade(RemoveMultipleKeywordArguments(), text) + self.assertEqual(new_text, text) + + # Using positional arguments (in proper order) -> no change + text = "h(a, x, y)\n" + _, new_text = self._upgrade(RemoveMultipleKeywordArguments(), text) + self.assertEqual(new_text, text) + + # Use only the old names, in order + text = "h(a, kw1_alias=x, kw2_alias=y)\n" + acceptable_outputs = [ + "h(a, x, y)\n", + "h(a, kw1=x, kw2=y)\n", + "h(a=a, kw1=x, kw2=y)\n", + "h(a, kw2=y, kw1=x)\n", + "h(a=a, kw2=y, kw1=x)\n", + ] + _, new_text = self._upgrade(RemoveMultipleKeywordArguments(), text) + self.assertIn(new_text, acceptable_outputs) + + # Use only the old names, in reverse order, should give one of same outputs + text = "h(a, kw2_alias=y, kw1_alias=x)\n" + _, new_text = self._upgrade(RemoveMultipleKeywordArguments(), text) + self.assertIn(new_text, acceptable_outputs) + + # Mix old and new names + text = "h(a, kw1=x, kw2_alias=y)\n" + _, new_text = self._upgrade(RemoveMultipleKeywordArguments(), text) + self.assertIn(new_text, acceptable_outputs) + + def testUnrestrictedFunctionWarnings(self): + class FooWarningSpec(NoUpdateSpec): + """Usages of function attribute foo() prints out a warning.""" + + def __init__(self): + NoUpdateSpec.__init__(self) + self.unrestricted_function_warnings = {"foo": "not good"} + texts = ["object.foo()", "get_object().foo()", + "get_object().foo()", "object.foo().bar()"] + for text in texts: + (_, report, _), _ = self._upgrade(FooWarningSpec(), text) + self.assertIn("not good", report) + + # Note that foo() won't result in a warning, because in this case foo is + # not an attribute, but a name. + false_alarms = ["foo", "foo()", "foo.bar()", "obj.run_foo()", "obj.foo"] + for text in false_alarms: + (_, report, _), _ = self._upgrade(FooWarningSpec(), text) + self.assertNotIn("not good", report) + + +if __name__ == "__main__": + test_lib.main() diff --git a/tensorflow/tools/compatibility/renames_v2.py b/tensorflow/tools/compatibility/renames_v2.py index 5ea2fbcc4cd..78839e36bae 100644 --- a/tensorflow/tools/compatibility/renames_v2.py +++ b/tensorflow/tools/compatibility/renames_v2.py @@ -28,6 +28,10 @@ renames = { 'tf.AUTO_REUSE': 'tf.compat.v1.AUTO_REUSE', 'tf.COMPILER_VERSION': 'tf.version.COMPILER_VERSION', 'tf.CXX11_ABI_FLAG': 'tf.sysconfig.CXX11_ABI_FLAG', + 'tf.ConditionalAccumulator': 'tf.compat.v1.ConditionalAccumulator', + 'tf.ConditionalAccumulatorBase': 'tf.compat.v1.ConditionalAccumulatorBase', + 'tf.DeviceSpec': 'tf.compat.v1.DeviceSpec', + 'tf.Dimension': 'tf.compat.v1.Dimension', 'tf.FixedLenFeature': 'tf.io.FixedLenFeature', 'tf.FixedLenSequenceFeature': 'tf.io.FixedLenSequenceFeature', 'tf.FixedLengthRecordReader': 'tf.compat.v1.FixedLengthRecordReader', @@ -35,6 +39,7 @@ renames = { 'tf.GRAPH_DEF_VERSION': 'tf.version.GRAPH_DEF_VERSION', 'tf.GRAPH_DEF_VERSION_MIN_CONSUMER': 'tf.version.GRAPH_DEF_VERSION_MIN_CONSUMER', 'tf.GRAPH_DEF_VERSION_MIN_PRODUCER': 'tf.version.GRAPH_DEF_VERSION_MIN_PRODUCER', + 'tf.GraphKeys': 'tf.compat.v1.GraphKeys', 'tf.IdentityReader': 'tf.compat.v1.IdentityReader', 'tf.InteractiveSession': 'tf.compat.v1.InteractiveSession', 'tf.LMDBReader': 'tf.compat.v1.LMDBReader', @@ -53,12 +58,10 @@ renames = { 'tf.SparseConditionalAccumulator': 'tf.sparse.SparseConditionalAccumulator', 'tf.SparseFeature': 'tf.io.SparseFeature', 'tf.TFRecordReader': 'tf.compat.v1.TFRecordReader', - 'tf.TensorShape': 'tf.compat.v1.TensorShape', + 'tf.TensorInfo': 'tf.compat.v1.TensorInfo', 'tf.TextLineReader': 'tf.compat.v1.TextLineReader', 'tf.VERSION': 'tf.version.VERSION', 'tf.VarLenFeature': 'tf.io.VarLenFeature', - 'tf.Variable': 'tf.compat.v1.Variable', - 'tf.VariableAggregation': 'tf.compat.v1.VariableAggregation', 'tf.VariableScope': 'tf.compat.v1.VariableScope', 'tf.WholeFileReader': 'tf.compat.v1.WholeFileReader', 'tf.accumulate_n': 'tf.math.accumulate_n', @@ -67,59 +70,67 @@ renames = { 'tf.add_to_collections': 'tf.compat.v1.add_to_collections', 'tf.all_variables': 'tf.compat.v1.all_variables', 'tf.angle': 'tf.math.angle', - 'tf.argmax': 'tf.compat.v1.argmax', - 'tf.argmin': 'tf.compat.v1.argmin', - 'tf.assert_greater_equal': 'tf.debugging.assert_greater_equal', - 'tf.assert_integer': 'tf.debugging.assert_integer', - 'tf.assert_less_equal': 'tf.debugging.assert_less_equal', - 'tf.assert_near': 'tf.debugging.assert_near', - 'tf.assert_negative': 'tf.debugging.assert_negative', - 'tf.assert_non_negative': 'tf.debugging.assert_non_negative', - 'tf.assert_non_positive': 'tf.debugging.assert_non_positive', - 'tf.assert_none_equal': 'tf.debugging.assert_none_equal', - 'tf.assert_positive': 'tf.debugging.assert_positive', + 'tf.app.run': 'tf.compat.v1.app.run', + 'tf.arg_max': 'tf.compat.v1.arg_max', + 'tf.arg_min': 'tf.compat.v1.arg_min', + 'tf.assert_greater_equal': 'tf.compat.v1.assert_greater_equal', + 'tf.assert_integer': 'tf.compat.v1.assert_integer', + 'tf.assert_less_equal': 'tf.compat.v1.assert_less_equal', + 'tf.assert_near': 'tf.compat.v1.assert_near', + 'tf.assert_negative': 'tf.compat.v1.assert_negative', + 'tf.assert_non_negative': 'tf.compat.v1.assert_non_negative', + 'tf.assert_non_positive': 'tf.compat.v1.assert_non_positive', + 'tf.assert_none_equal': 'tf.compat.v1.assert_none_equal', + 'tf.assert_positive': 'tf.compat.v1.assert_positive', 'tf.assert_proper_iterable': 'tf.debugging.assert_proper_iterable', - 'tf.assert_rank_at_least': 'tf.debugging.assert_rank_at_least', - 'tf.assert_rank_in': 'tf.debugging.assert_rank_in', + 'tf.assert_rank_at_least': 'tf.compat.v1.assert_rank_at_least', + 'tf.assert_rank_in': 'tf.compat.v1.assert_rank_in', 'tf.assert_same_float_dtype': 'tf.debugging.assert_same_float_dtype', - 'tf.assert_scalar': 'tf.debugging.assert_scalar', - 'tf.assert_type': 'tf.debugging.assert_type', + 'tf.assert_scalar': 'tf.compat.v1.assert_scalar', + 'tf.assert_type': 'tf.compat.v1.assert_type', 'tf.assert_variables_initialized': 'tf.compat.v1.assert_variables_initialized', 'tf.assign': 'tf.compat.v1.assign', 'tf.assign_add': 'tf.compat.v1.assign_add', 'tf.assign_sub': 'tf.compat.v1.assign_sub', 'tf.betainc': 'tf.math.betainc', - 'tf.bincount': 'tf.math.bincount', 'tf.ceil': 'tf.math.ceil', 'tf.check_numerics': 'tf.debugging.check_numerics', 'tf.cholesky': 'tf.linalg.cholesky', 'tf.cholesky_solve': 'tf.linalg.cholesky_solve', + 'tf.clip_by_average_norm': 'tf.compat.v1.clip_by_average_norm', 'tf.colocate_with': 'tf.compat.v1.colocate_with', - 'tf.confusion_matrix': 'tf.math.confusion_matrix', 'tf.conj': 'tf.math.conj', 'tf.container': 'tf.compat.v1.container', - 'tf.convert_to_tensor': 'tf.compat.v1.convert_to_tensor', 'tf.convert_to_tensor_or_indexed_slices': 'tf.compat.v1.convert_to_tensor_or_indexed_slices', 'tf.convert_to_tensor_or_sparse_tensor': 'tf.compat.v1.convert_to_tensor_or_sparse_tensor', + 'tf.count_nonzero': 'tf.compat.v1.count_nonzero', 'tf.count_up_to': 'tf.compat.v1.count_up_to', + 'tf.create_partitioned_variables': 'tf.compat.v1.create_partitioned_variables', 'tf.cross': 'tf.linalg.cross', 'tf.cumprod': 'tf.math.cumprod', + 'tf.data.make_initializable_iterator': 'tf.compat.v1.data.make_initializable_iterator', + 'tf.data.make_one_shot_iterator': 'tf.compat.v1.data.make_one_shot_iterator', + 'tf.debugging.is_finite': 'tf.math.is_finite', + 'tf.debugging.is_inf': 'tf.math.is_inf', + 'tf.debugging.is_nan': 'tf.math.is_nan', + 'tf.debugging.is_non_decreasing': 'tf.math.is_non_decreasing', + 'tf.debugging.is_strictly_increasing': 'tf.math.is_strictly_increasing', 'tf.decode_base64': 'tf.io.decode_base64', 'tf.decode_compressed': 'tf.io.decode_compressed', - 'tf.decode_csv': 'tf.io.decode_csv', 'tf.decode_json_example': 'tf.io.decode_json_example', 'tf.decode_raw': 'tf.io.decode_raw', 'tf.delete_session_tensor': 'tf.compat.v1.delete_session_tensor', 'tf.depth_to_space': 'tf.nn.depth_to_space', 'tf.dequantize': 'tf.quantization.dequantize', 'tf.deserialize_many_sparse': 'tf.io.deserialize_many_sparse', - 'tf.device': 'tf.compat.v1.device', 'tf.diag': 'tf.linalg.tensor_diag', 'tf.diag_part': 'tf.linalg.tensor_diag_part', 'tf.digamma': 'tf.math.digamma', 'tf.dimension_at_index': 'tf.compat.v1.dimension_at_index', 'tf.dimension_value': 'tf.compat.v1.dimension_value', + 'tf.disable_eager_execution': 'tf.compat.v1.disable_eager_execution', 'tf.disable_resource_variables': 'tf.compat.v1.disable_resource_variables', + 'tf.disable_v2_behavior': 'tf.compat.v1.disable_v2_behavior', 'tf.disable_v2_tensorshape': 'tf.compat.v1.disable_v2_tensorshape', 'tf.distributions.Bernoulli': 'tf.compat.v1.distributions.Bernoulli', 'tf.distributions.Beta': 'tf.compat.v1.distributions.Beta', @@ -139,30 +150,41 @@ renames = { 'tf.distributions.StudentT': 'tf.compat.v1.distributions.StudentT', 'tf.distributions.Uniform': 'tf.compat.v1.distributions.Uniform', 'tf.distributions.kl_divergence': 'tf.compat.v1.distributions.kl_divergence', + 'tf.div': 'tf.compat.v1.div', + 'tf.enable_eager_execution': 'tf.compat.v1.enable_eager_execution', 'tf.enable_resource_variables': 'tf.compat.v1.enable_resource_variables', + 'tf.enable_v2_behavior': 'tf.compat.v1.enable_v2_behavior', 'tf.enable_v2_tensorshape': 'tf.compat.v1.enable_v2_tensorshape', 'tf.encode_base64': 'tf.io.encode_base64', 'tf.erf': 'tf.math.erf', 'tf.erfc': 'tf.math.erfc', 'tf.expm1': 'tf.math.expm1', - 'tf.extract_image_patches': 'tf.image.extract_image_patches', 'tf.fake_quant_with_min_max_args': 'tf.quantization.fake_quant_with_min_max_args', 'tf.fake_quant_with_min_max_args_gradient': 'tf.quantization.fake_quant_with_min_max_args_gradient', 'tf.fake_quant_with_min_max_vars': 'tf.quantization.fake_quant_with_min_max_vars', 'tf.fake_quant_with_min_max_vars_gradient': 'tf.quantization.fake_quant_with_min_max_vars_gradient', 'tf.fake_quant_with_min_max_vars_per_channel': 'tf.quantization.fake_quant_with_min_max_vars_per_channel', 'tf.fake_quant_with_min_max_vars_per_channel_gradient': 'tf.quantization.fake_quant_with_min_max_vars_per_channel_gradient', + 'tf.feature_column.input_layer': 'tf.compat.v1.feature_column.input_layer', + 'tf.feature_column.linear_model': 'tf.compat.v1.feature_column.linear_model', 'tf.fft': 'tf.signal.fft', 'tf.fft2d': 'tf.signal.fft2d', 'tf.fft3d': 'tf.signal.fft3d', + 'tf.fixed_size_partitioner': 'tf.compat.v1.fixed_size_partitioner', 'tf.floordiv': 'tf.math.floordiv', + 'tf.get_collection': 'tf.compat.v1.get_collection', + 'tf.get_collection_ref': 'tf.compat.v1.get_collection_ref', + 'tf.get_default_graph': 'tf.compat.v1.get_default_graph', 'tf.get_default_session': 'tf.compat.v1.get_default_session', 'tf.get_local_variable': 'tf.compat.v1.get_local_variable', - 'tf.get_seed': 'tf.random.get_seed', + 'tf.get_seed': 'tf.compat.v1.get_seed', 'tf.get_session_handle': 'tf.compat.v1.get_session_handle', 'tf.get_session_tensor': 'tf.compat.v1.get_session_tensor', 'tf.get_variable': 'tf.compat.v1.get_variable', 'tf.get_variable_scope': 'tf.compat.v1.get_variable_scope', + 'tf.gfile.FastGFile': 'tf.compat.v1.gfile.FastGFile', + 'tf.gfile.GFile': 'tf.compat.v1.gfile.GFile', + 'tf.gfile.Open': 'tf.compat.v1.gfile.Open', 'tf.global_norm': 'tf.linalg.global_norm', 'tf.global_variables': 'tf.compat.v1.global_variables', 'tf.global_variables_initializer': 'tf.compat.v1.global_variables_initializer', @@ -178,6 +200,12 @@ renames = { 'tf.igamma': 'tf.math.igamma', 'tf.igammac': 'tf.math.igammac', 'tf.imag': 'tf.math.imag', + 'tf.image.resize_area': 'tf.compat.v1.image.resize_area', + 'tf.image.resize_bicubic': 'tf.compat.v1.image.resize_bicubic', + 'tf.image.resize_bilinear': 'tf.compat.v1.image.resize_bilinear', + 'tf.image.resize_images': 'tf.compat.v1.image.resize_images', + 'tf.image.resize_nearest_neighbor': 'tf.compat.v1.image.resize_nearest_neighbor', + 'tf.image.transpose_image': 'tf.compat.v1.image.transpose_image', 'tf.initialize_all_tables': 'tf.compat.v1.initialize_all_tables', 'tf.initialize_all_variables': 'tf.compat.v1.initialize_all_variables', 'tf.initialize_local_variables': 'tf.compat.v1.initialize_local_variables', @@ -187,12 +215,13 @@ renames = { 'tf.initializers.tables_initializer': 'tf.compat.v1.initializers.tables_initializer', 'tf.initializers.variables': 'tf.compat.v1.initializers.variables', 'tf.invert_permutation': 'tf.math.invert_permutation', - 'tf.is_finite': 'tf.debugging.is_finite', - 'tf.is_inf': 'tf.debugging.is_inf', - 'tf.is_nan': 'tf.debugging.is_nan', - 'tf.is_non_decreasing': 'tf.debugging.is_non_decreasing', + 'tf.io.tf_record_iterator': 'tf.compat.v1.io.tf_record_iterator', + 'tf.is_finite': 'tf.math.is_finite', + 'tf.is_inf': 'tf.math.is_inf', + 'tf.is_nan': 'tf.math.is_nan', + 'tf.is_non_decreasing': 'tf.math.is_non_decreasing', 'tf.is_numeric_tensor': 'tf.debugging.is_numeric_tensor', - 'tf.is_strictly_increasing': 'tf.debugging.is_strictly_increasing', + 'tf.is_strictly_increasing': 'tf.math.is_strictly_increasing', 'tf.is_variable_initialized': 'tf.compat.v1.is_variable_initialized', 'tf.keras.backend.get_session': 'tf.compat.v1.keras.backend.get_session', 'tf.layers.AveragePooling1D': 'tf.compat.v1.layers.AveragePooling1D', @@ -235,13 +264,46 @@ renames = { 'tf.layers.separable_conv2d': 'tf.compat.v1.layers.separable_conv2d', 'tf.lbeta': 'tf.math.lbeta', 'tf.lgamma': 'tf.math.lgamma', + 'tf.lin_space': 'tf.linspace', 'tf.local_variables': 'tf.compat.v1.local_variables', 'tf.local_variables_initializer': 'tf.compat.v1.local_variables_initializer', + 'tf.log': 'tf.math.log', + 'tf.log1p': 'tf.math.log1p', 'tf.log_sigmoid': 'tf.math.log_sigmoid', + 'tf.logging.DEBUG': 'tf.compat.v1.logging.DEBUG', + 'tf.logging.ERROR': 'tf.compat.v1.logging.ERROR', + 'tf.logging.FATAL': 'tf.compat.v1.logging.FATAL', + 'tf.logging.INFO': 'tf.compat.v1.logging.INFO', + 'tf.logging.TaskLevelStatusMessage': 'tf.compat.v1.logging.TaskLevelStatusMessage', + 'tf.logging.WARN': 'tf.compat.v1.logging.WARN', + 'tf.logging.debug': 'tf.compat.v1.logging.debug', + 'tf.logging.error': 'tf.compat.v1.logging.error', + 'tf.logging.fatal': 'tf.compat.v1.logging.fatal', + 'tf.logging.flush': 'tf.compat.v1.logging.flush', + 'tf.logging.get_verbosity': 'tf.compat.v1.logging.get_verbosity', + 'tf.logging.info': 'tf.compat.v1.logging.info', + 'tf.logging.log': 'tf.compat.v1.logging.log', + 'tf.logging.log_every_n': 'tf.compat.v1.logging.log_every_n', + 'tf.logging.log_first_n': 'tf.compat.v1.logging.log_first_n', + 'tf.logging.log_if': 'tf.compat.v1.logging.log_if', + 'tf.logging.set_verbosity': 'tf.compat.v1.logging.set_verbosity', + 'tf.logging.vlog': 'tf.compat.v1.logging.vlog', + 'tf.logging.warn': 'tf.compat.v1.logging.warn', + 'tf.logging.warning': 'tf.compat.v1.logging.warning', 'tf.logical_xor': 'tf.math.logical_xor', + 'tf.losses.absolute_difference': 'tf.compat.v1.losses.absolute_difference', + 'tf.losses.compute_weighted_loss': 'tf.compat.v1.losses.compute_weighted_loss', + 'tf.losses.cosine_distance': 'tf.compat.v1.losses.cosine_distance', + 'tf.losses.hinge_loss': 'tf.compat.v1.losses.hinge_loss', + 'tf.losses.huber_loss': 'tf.compat.v1.losses.huber_loss', + 'tf.losses.log_loss': 'tf.compat.v1.losses.log_loss', + 'tf.losses.mean_pairwise_squared_error': 'tf.compat.v1.losses.mean_pairwise_squared_error', + 'tf.losses.mean_squared_error': 'tf.compat.v1.losses.mean_squared_error', + 'tf.losses.sigmoid_cross_entropy': 'tf.compat.v1.losses.sigmoid_cross_entropy', + 'tf.losses.softmax_cross_entropy': 'tf.compat.v1.losses.softmax_cross_entropy', + 'tf.losses.sparse_softmax_cross_entropy': 'tf.compat.v1.losses.sparse_softmax_cross_entropy', 'tf.make_template': 'tf.compat.v1.make_template', 'tf.make_tensor_proto': 'tf.compat.v1.make_tensor_proto', - 'tf.manip.batch_to_space_nd': 'tf.batch_to_space_nd', 'tf.manip.gather_nd': 'tf.gather_nd', 'tf.manip.reshape': 'tf.reshape', 'tf.manip.reverse': 'tf.reverse', @@ -250,8 +312,6 @@ renames = { 'tf.manip.space_to_batch_nd': 'tf.space_to_batch_nd', 'tf.manip.tile': 'tf.tile', 'tf.matching_files': 'tf.io.matching_files', - 'tf.math.argmax': 'tf.compat.v1.math.argmax', - 'tf.math.argmin': 'tf.compat.v1.math.argmin', 'tf.matrix_band_part': 'tf.linalg.band_part', 'tf.matrix_determinant': 'tf.linalg.det', 'tf.matrix_diag': 'tf.linalg.diag', @@ -262,49 +322,105 @@ renames = { 'tf.matrix_solve_ls': 'tf.linalg.lstsq', 'tf.matrix_transpose': 'tf.linalg.transpose', 'tf.matrix_triangular_solve': 'tf.linalg.triangular_solve', + 'tf.metrics.accuracy': 'tf.compat.v1.metrics.accuracy', + 'tf.metrics.auc': 'tf.compat.v1.metrics.auc', + 'tf.metrics.average_precision_at_k': 'tf.compat.v1.metrics.average_precision_at_k', + 'tf.metrics.false_negatives': 'tf.compat.v1.metrics.false_negatives', + 'tf.metrics.false_negatives_at_thresholds': 'tf.compat.v1.metrics.false_negatives_at_thresholds', + 'tf.metrics.false_positives': 'tf.compat.v1.metrics.false_positives', + 'tf.metrics.false_positives_at_thresholds': 'tf.compat.v1.metrics.false_positives_at_thresholds', + 'tf.metrics.mean': 'tf.compat.v1.metrics.mean', + 'tf.metrics.mean_absolute_error': 'tf.compat.v1.metrics.mean_absolute_error', + 'tf.metrics.mean_cosine_distance': 'tf.compat.v1.metrics.mean_cosine_distance', + 'tf.metrics.mean_iou': 'tf.compat.v1.metrics.mean_iou', + 'tf.metrics.mean_per_class_accuracy': 'tf.compat.v1.metrics.mean_per_class_accuracy', + 'tf.metrics.mean_relative_error': 'tf.compat.v1.metrics.mean_relative_error', + 'tf.metrics.mean_squared_error': 'tf.compat.v1.metrics.mean_squared_error', + 'tf.metrics.mean_tensor': 'tf.compat.v1.metrics.mean_tensor', + 'tf.metrics.percentage_below': 'tf.compat.v1.metrics.percentage_below', + 'tf.metrics.precision': 'tf.compat.v1.metrics.precision', + 'tf.metrics.precision_at_k': 'tf.compat.v1.metrics.precision_at_k', + 'tf.metrics.precision_at_thresholds': 'tf.compat.v1.metrics.precision_at_thresholds', + 'tf.metrics.precision_at_top_k': 'tf.compat.v1.metrics.precision_at_top_k', + 'tf.metrics.recall': 'tf.compat.v1.metrics.recall', + 'tf.metrics.recall_at_k': 'tf.compat.v1.metrics.recall_at_k', + 'tf.metrics.recall_at_thresholds': 'tf.compat.v1.metrics.recall_at_thresholds', + 'tf.metrics.recall_at_top_k': 'tf.compat.v1.metrics.recall_at_top_k', + 'tf.metrics.root_mean_squared_error': 'tf.compat.v1.metrics.root_mean_squared_error', + 'tf.metrics.sensitivity_at_specificity': 'tf.compat.v1.metrics.sensitivity_at_specificity', + 'tf.metrics.sparse_average_precision_at_k': 'tf.compat.v1.metrics.sparse_average_precision_at_k', + 'tf.metrics.sparse_precision_at_k': 'tf.compat.v1.metrics.sparse_precision_at_k', + 'tf.metrics.specificity_at_sensitivity': 'tf.compat.v1.metrics.specificity_at_sensitivity', + 'tf.metrics.true_negatives': 'tf.compat.v1.metrics.true_negatives', + 'tf.metrics.true_negatives_at_thresholds': 'tf.compat.v1.metrics.true_negatives_at_thresholds', + 'tf.metrics.true_positives': 'tf.compat.v1.metrics.true_positives', + 'tf.metrics.true_positives_at_thresholds': 'tf.compat.v1.metrics.true_positives_at_thresholds', + 'tf.min_max_variable_partitioner': 'tf.compat.v1.min_max_variable_partitioner', 'tf.model_variables': 'tf.compat.v1.model_variables', 'tf.moving_average_variables': 'tf.compat.v1.moving_average_variables', - 'tf.nn.ctc_beam_search_decoder': 'tf.compat.v1.nn.ctc_beam_search_decoder', + 'tf.nn.bidirectional_dynamic_rnn': 'tf.compat.v1.nn.bidirectional_dynamic_rnn', + 'tf.nn.conv3d_backprop_filter_v2': 'tf.nn.conv3d_backprop_filter', 'tf.nn.ctc_beam_search_decoder_v2': 'tf.nn.ctc_beam_search_decoder', + 'tf.nn.ctc_loss_v2': 'tf.nn.ctc_loss', + 'tf.nn.depthwise_conv2d_native': 'tf.compat.v1.nn.depthwise_conv2d_native', + 'tf.nn.depthwise_conv2d_native_backprop_filter': 'tf.nn.depthwise_conv2d_backprop_filter', + 'tf.nn.depthwise_conv2d_native_backprop_input': 'tf.nn.depthwise_conv2d_backprop_input', 'tf.nn.dynamic_rnn': 'tf.compat.v1.nn.dynamic_rnn', 'tf.nn.log_uniform_candidate_sampler': 'tf.random.log_uniform_candidate_sampler', + 'tf.nn.quantized_avg_pool': 'tf.compat.v1.nn.quantized_avg_pool', + 'tf.nn.quantized_conv2d': 'tf.compat.v1.nn.quantized_conv2d', + 'tf.nn.quantized_max_pool': 'tf.compat.v1.nn.quantized_max_pool', + 'tf.nn.quantized_relu_x': 'tf.compat.v1.nn.quantized_relu_x', 'tf.nn.raw_rnn': 'tf.compat.v1.nn.raw_rnn', 'tf.nn.rnn_cell.BasicLSTMCell': 'tf.compat.v1.nn.rnn_cell.BasicLSTMCell', 'tf.nn.rnn_cell.BasicRNNCell': 'tf.compat.v1.nn.rnn_cell.BasicRNNCell', 'tf.nn.rnn_cell.GRUCell': 'tf.compat.v1.nn.rnn_cell.GRUCell', 'tf.nn.rnn_cell.LSTMCell': 'tf.compat.v1.nn.rnn_cell.LSTMCell', - 'tf.nn.softmax_cross_entropy_with_logits': 'tf.compat.v1.nn.softmax_cross_entropy_with_logits', - 'tf.nn.softmax_cross_entropy_with_logits_v2': 'tf.nn.softmax_cross_entropy_with_logits', + 'tf.nn.rnn_cell.MultiRNNCell': 'tf.compat.v1.nn.rnn_cell.MultiRNNCell', + 'tf.nn.static_bidirectional_rnn': 'tf.compat.v1.nn.static_bidirectional_rnn', 'tf.nn.static_rnn': 'tf.compat.v1.nn.static_rnn', 'tf.nn.uniform_candidate_sampler': 'tf.random.uniform_candidate_sampler', + 'tf.nn.xw_plus_b': 'tf.compat.v1.nn.xw_plus_b', 'tf.op_scope': 'tf.compat.v1.op_scope', 'tf.orthogonal_initializer': 'tf.keras.initializers.Orthogonal', - 'tf.parse_example': 'tf.io.parse_example', - 'tf.parse_single_example': 'tf.io.parse_single_example', + 'tf.parse_example': 'tf.compat.v1.parse_example', + 'tf.parse_single_example': 'tf.compat.v1.parse_single_example', 'tf.parse_single_sequence_example': 'tf.io.parse_single_sequence_example', 'tf.parse_tensor': 'tf.io.parse_tensor', 'tf.placeholder': 'tf.compat.v1.placeholder', 'tf.placeholder_with_default': 'tf.compat.v1.placeholder_with_default', 'tf.polygamma': 'tf.math.polygamma', + 'tf.profiler.AdviceProto': 'tf.compat.v1.profiler.AdviceProto', + 'tf.profiler.GraphNodeProto': 'tf.compat.v1.profiler.GraphNodeProto', + 'tf.profiler.MultiGraphNodeProto': 'tf.compat.v1.profiler.MultiGraphNodeProto', + 'tf.profiler.OpLogProto': 'tf.compat.v1.profiler.OpLogProto', + 'tf.profiler.ProfileOptionBuilder': 'tf.compat.v1.profiler.ProfileOptionBuilder', + 'tf.profiler.Profiler': 'tf.compat.v1.profiler.Profiler', + 'tf.profiler.advise': 'tf.compat.v1.profiler.advise', + 'tf.profiler.profile': 'tf.compat.v1.profiler.profile', + 'tf.profiler.write_op_log': 'tf.compat.v1.profiler.write_op_log', + 'tf.py_func': 'tf.compat.v1.py_func', 'tf.python_io.TFRecordCompressionType': 'tf.io.TFRecordCompressionType', 'tf.python_io.TFRecordOptions': 'tf.io.TFRecordOptions', 'tf.python_io.TFRecordWriter': 'tf.io.TFRecordWriter', - 'tf.python_io.tf_record_iterator': 'tf.io.tf_record_iterator', + 'tf.python_io.tf_record_iterator': 'tf.compat.v1.python_io.tf_record_iterator', 'tf.qr': 'tf.linalg.qr', 'tf.quantize': 'tf.quantization.quantize', 'tf.quantized_concat': 'tf.quantization.quantized_concat', + 'tf.random.get_seed': 'tf.compat.v1.random.get_seed', + 'tf.random.set_random_seed': 'tf.compat.v1.random.set_random_seed', 'tf.random_crop': 'tf.image.random_crop', 'tf.random_gamma': 'tf.random.gamma', 'tf.random_normal': 'tf.random.normal', - 'tf.random_poisson': 'tf.random.poisson', + 'tf.random_poisson': 'tf.compat.v1.random_poisson', 'tf.random_shuffle': 'tf.random.shuffle', 'tf.random_uniform': 'tf.random.uniform', 'tf.read_file': 'tf.io.read_file', 'tf.real': 'tf.math.real', 'tf.reciprocal': 'tf.math.reciprocal', - 'tf.reduce_join': 'tf.strings.reduce_join', 'tf.regex_replace': 'tf.strings.regex_replace', 'tf.report_uninitialized_variables': 'tf.compat.v1.report_uninitialized_variables', + 'tf.reset_default_graph': 'tf.compat.v1.reset_default_graph', 'tf.resource_loader.get_data_files_path': 'tf.compat.v1.resource_loader.get_data_files_path', 'tf.resource_loader.get_path_to_datafile': 'tf.compat.v1.resource_loader.get_path_to_datafile', 'tf.resource_loader.get_root_dir_with_all_resources': 'tf.compat.v1.resource_loader.get_root_dir_with_all_resources', @@ -314,13 +430,15 @@ renames = { 'tf.rint': 'tf.math.rint', 'tf.rsqrt': 'tf.math.rsqrt', 'tf.saved_model.Builder': 'tf.compat.v1.saved_model.Builder', + 'tf.saved_model.LEGACY_INIT_OP_KEY': 'tf.compat.v1.saved_model.LEGACY_INIT_OP_KEY', + 'tf.saved_model.MAIN_OP_KEY': 'tf.compat.v1.saved_model.MAIN_OP_KEY', 'tf.saved_model.TRAINING': 'tf.saved_model.TRANING', 'tf.saved_model.build_tensor_info': 'tf.compat.v1.saved_model.build_tensor_info', 'tf.saved_model.builder.SavedModelBuilder': 'tf.compat.v1.saved_model.builder.SavedModelBuilder', 'tf.saved_model.constants.ASSETS_DIRECTORY': 'tf.saved_model.ASSETS_DIRECTORY', 'tf.saved_model.constants.ASSETS_KEY': 'tf.saved_model.ASSETS_KEY', - 'tf.saved_model.constants.LEGACY_INIT_OP_KEY': 'tf.saved_model.LEGACY_INIT_OP_KEY', - 'tf.saved_model.constants.MAIN_OP_KEY': 'tf.saved_model.MAIN_OP_KEY', + 'tf.saved_model.constants.LEGACY_INIT_OP_KEY': 'tf.compat.v1.saved_model.constants.LEGACY_INIT_OP_KEY', + 'tf.saved_model.constants.MAIN_OP_KEY': 'tf.compat.v1.saved_model.constants.MAIN_OP_KEY', 'tf.saved_model.constants.SAVED_MODEL_FILENAME_PB': 'tf.saved_model.SAVED_MODEL_FILENAME_PB', 'tf.saved_model.constants.SAVED_MODEL_FILENAME_PBTXT': 'tf.saved_model.SAVED_MODEL_FILENAME_PBTXT', 'tf.saved_model.constants.SAVED_MODEL_SCHEMA_VERSION': 'tf.saved_model.SAVED_MODEL_SCHEMA_VERSION', @@ -330,10 +448,11 @@ renames = { 'tf.saved_model.get_tensor_from_tensor_info': 'tf.compat.v1.saved_model.get_tensor_from_tensor_info', 'tf.saved_model.load': 'tf.compat.v1.saved_model.load', 'tf.saved_model.loader.load': 'tf.compat.v1.saved_model.loader.load', - 'tf.saved_model.loader.maybe_saved_model_directory': 'tf.saved_model.maybe_saved_model_directory', + 'tf.saved_model.loader.maybe_saved_model_directory': 'tf.compat.v1.saved_model.loader.maybe_saved_model_directory', 'tf.saved_model.main_op.main_op': 'tf.compat.v1.saved_model.main_op.main_op', 'tf.saved_model.main_op.main_op_with_restore': 'tf.compat.v1.saved_model.main_op.main_op_with_restore', 'tf.saved_model.main_op_with_restore': 'tf.compat.v1.saved_model.main_op_with_restore', + 'tf.saved_model.maybe_saved_model_directory': 'tf.compat.v1.saved_model.maybe_saved_model_directory', 'tf.saved_model.signature_constants.CLASSIFY_INPUTS': 'tf.saved_model.CLASSIFY_INPUTS', 'tf.saved_model.signature_constants.CLASSIFY_METHOD_NAME': 'tf.saved_model.CLASSIFY_METHOD_NAME', 'tf.saved_model.signature_constants.CLASSIFY_OUTPUT_CLASSES': 'tf.saved_model.CLASSIFY_OUTPUT_CLASSES', @@ -370,51 +489,80 @@ renames = { 'tf.segment_sum': 'tf.math.segment_sum', 'tf.self_adjoint_eig': 'tf.linalg.eigh', 'tf.self_adjoint_eigvals': 'tf.linalg.eigvalsh', - 'tf.serialize_many_sparse': 'tf.io.serialize_many_sparse', - 'tf.serialize_sparse': 'tf.io.serialize_sparse', + 'tf.serialize_many_sparse': 'tf.compat.v1.serialize_many_sparse', + 'tf.serialize_sparse': 'tf.compat.v1.serialize_sparse', 'tf.serialize_tensor': 'tf.io.serialize_tensor', + 'tf.set_random_seed': 'tf.compat.v1.set_random_seed', 'tf.setdiff1d': 'tf.compat.v1.setdiff1d', + 'tf.sets.set_difference': 'tf.sets.difference', + 'tf.sets.set_intersection': 'tf.sets.intersection', + 'tf.sets.set_size': 'tf.sets.size', + 'tf.sets.set_union': 'tf.sets.union', 'tf.space_to_batch': 'tf.nn.space_to_batch', 'tf.space_to_depth': 'tf.nn.space_to_depth', + 'tf.sparse.matmul': 'tf.sparse.sparse_dense_matmul', + 'tf.sparse.merge': 'tf.compat.v1.sparse.merge', 'tf.sparse.placeholder': 'tf.compat.v1.sparse.placeholder', - 'tf.sparse_add': 'tf.sparse.add', + 'tf.sparse.reduce_max_sparse': 'tf.compat.v1.sparse.reduce_max_sparse', + 'tf.sparse.reduce_sum_sparse': 'tf.compat.v1.sparse.reduce_sum_sparse', 'tf.sparse_fill_empty_rows': 'tf.sparse.fill_empty_rows', 'tf.sparse_mask': 'tf.sparse.mask', - 'tf.sparse_matmul': 'tf.compat.v1.sparse_matmul', 'tf.sparse_maximum': 'tf.sparse.maximum', - 'tf.sparse_merge': 'tf.sparse.merge', + 'tf.sparse_merge': 'tf.compat.v1.sparse_merge', 'tf.sparse_minimum': 'tf.sparse.minimum', 'tf.sparse_placeholder': 'tf.compat.v1.sparse_placeholder', + 'tf.sparse_reduce_max': 'tf.compat.v1.sparse_reduce_max', + 'tf.sparse_reduce_max_sparse': 'tf.compat.v1.sparse_reduce_max_sparse', + 'tf.sparse_reduce_sum': 'tf.compat.v1.sparse_reduce_sum', + 'tf.sparse_reduce_sum_sparse': 'tf.compat.v1.sparse_reduce_sum_sparse', 'tf.sparse_reorder': 'tf.sparse.reorder', 'tf.sparse_reset_shape': 'tf.sparse.reset_shape', 'tf.sparse_reshape': 'tf.sparse.reshape', 'tf.sparse_retain': 'tf.sparse.retain', - 'tf.sparse_segment_mean': 'tf.sparse.segment_mean', - 'tf.sparse_segment_sqrt_n': 'tf.sparse.segment_sqrt_n', - 'tf.sparse_segment_sum': 'tf.sparse.segment_sum', + 'tf.sparse_segment_mean': 'tf.compat.v1.sparse_segment_mean', + 'tf.sparse_segment_sqrt_n': 'tf.compat.v1.sparse_segment_sqrt_n', + 'tf.sparse_segment_sum': 'tf.compat.v1.sparse_segment_sum', 'tf.sparse_slice': 'tf.sparse.slice', 'tf.sparse_softmax': 'tf.sparse.softmax', - 'tf.sparse_tensor_dense_matmul': 'tf.sparse.matmul', + 'tf.sparse_tensor_dense_matmul': 'tf.sparse.sparse_dense_matmul', 'tf.sparse_tensor_to_dense': 'tf.sparse.to_dense', + 'tf.sparse_to_dense': 'tf.compat.v1.sparse_to_dense', 'tf.sparse_to_indicator': 'tf.sparse.to_indicator', 'tf.sparse_transpose': 'tf.sparse.transpose', + 'tf.spectral.dct': 'tf.signal.dct', 'tf.spectral.fft': 'tf.signal.fft', 'tf.spectral.fft2d': 'tf.signal.fft2d', 'tf.spectral.fft3d': 'tf.signal.fft3d', + 'tf.spectral.idct': 'tf.signal.idct', 'tf.spectral.ifft': 'tf.signal.ifft', 'tf.spectral.ifft2d': 'tf.signal.ifft2d', 'tf.spectral.ifft3d': 'tf.signal.ifft3d', + 'tf.spectral.irfft': 'tf.signal.irfft', + 'tf.spectral.irfft2d': 'tf.signal.irfft2d', + 'tf.spectral.irfft3d': 'tf.signal.irfft3d', + 'tf.spectral.rfft': 'tf.signal.rfft', + 'tf.spectral.rfft2d': 'tf.signal.rfft2d', + 'tf.spectral.rfft3d': 'tf.signal.rfft3d', 'tf.squared_difference': 'tf.math.squared_difference', 'tf.string_join': 'tf.strings.join', 'tf.string_strip': 'tf.strings.strip', - 'tf.string_to_hash_bucket': 'tf.strings.to_hash_bucket', 'tf.string_to_hash_bucket_fast': 'tf.strings.to_hash_bucket_fast', 'tf.string_to_hash_bucket_strong': 'tf.strings.to_hash_bucket_strong', - 'tf.string_to_number': 'tf.strings.to_number', + 'tf.summary.audio': 'tf.compat.v1.summary.audio', + 'tf.summary.get_summary_description': 'tf.compat.v1.summary.get_summary_description', + 'tf.summary.histogram': 'tf.compat.v1.summary.histogram', + 'tf.summary.image': 'tf.compat.v1.summary.image', + 'tf.summary.merge': 'tf.compat.v1.summary.merge', + 'tf.summary.merge_all': 'tf.compat.v1.summary.merge_all', + 'tf.summary.scalar': 'tf.compat.v1.summary.scalar', + 'tf.summary.tensor_summary': 'tf.compat.v1.summary.tensor_summary', + 'tf.summary.text': 'tf.compat.v1.summary.text', 'tf.svd': 'tf.linalg.svd', 'tf.tables_initializer': 'tf.compat.v1.tables_initializer', + 'tf.test.compute_gradient': 'tf.compat.v1.test.compute_gradient', 'tf.test.compute_gradient_error': 'tf.compat.v1.test.compute_gradient_error', 'tf.test.get_temp_dir': 'tf.compat.v1.test.get_temp_dir', + 'tf.test.mock': 'tf.compat.v1.test.mock', 'tf.test.test_src_dir_path': 'tf.compat.v1.test.test_src_dir_path', 'tf.to_bfloat16': 'tf.compat.v1.to_bfloat16', 'tf.to_complex128': 'tf.compat.v1.to_complex128', @@ -424,22 +572,47 @@ renames = { 'tf.to_int32': 'tf.compat.v1.to_int32', 'tf.to_int64': 'tf.compat.v1.to_int64', 'tf.trace': 'tf.linalg.trace', + 'tf.train.AdadeltaOptimizer': 'tf.compat.v1.train.AdadeltaOptimizer', + 'tf.train.AdagradDAOptimizer': 'tf.compat.v1.train.AdagradDAOptimizer', + 'tf.train.AdagradOptimizer': 'tf.compat.v1.train.AdagradOptimizer', + 'tf.train.AdamOptimizer': 'tf.compat.v1.train.AdamOptimizer', + 'tf.train.CheckpointSaverListener': 'tf.compat.v1.train.CheckpointSaverListener', + 'tf.train.ChiefSessionCreator': 'tf.compat.v1.train.ChiefSessionCreator', + 'tf.train.FtrlOptimizer': 'tf.compat.v1.train.FtrlOptimizer', + 'tf.train.GradientDescentOptimizer': 'tf.compat.v1.train.GradientDescentOptimizer', + 'tf.train.LooperThread': 'tf.compat.v1.train.LooperThread', + 'tf.train.MomentumOptimizer': 'tf.compat.v1.train.MomentumOptimizer', + 'tf.train.MonitoredSession': 'tf.compat.v1.train.MonitoredSession', 'tf.train.MonitoredTrainingSession': 'tf.compat.v1.train.MonitoredTrainingSession', + 'tf.train.NanLossDuringTrainingError': 'tf.compat.v1.train.NanLossDuringTrainingError', + 'tf.train.NewCheckpointReader': 'tf.compat.v1.train.NewCheckpointReader', + 'tf.train.Optimizer': 'tf.compat.v1.train.Optimizer', + 'tf.train.ProfilerHook': 'tf.compat.v1.train.ProfilerHook', + 'tf.train.ProximalAdagradOptimizer': 'tf.compat.v1.train.ProximalAdagradOptimizer', 'tf.train.QueueRunner': 'tf.compat.v1.train.QueueRunner', + 'tf.train.RMSPropOptimizer': 'tf.compat.v1.train.RMSPropOptimizer', 'tf.train.Saver': 'tf.compat.v1.train.Saver', 'tf.train.SaverDef': 'tf.compat.v1.train.SaverDef', + 'tf.train.Scaffold': 'tf.compat.v1.train.Scaffold', + 'tf.train.SecondOrStepTimer': 'tf.compat.v1.train.SecondOrStepTimer', + 'tf.train.SessionCreator': 'tf.compat.v1.train.SessionCreator', + 'tf.train.SessionManager': 'tf.compat.v1.train.SessionManager', + 'tf.train.SessionRunArgs': 'tf.compat.v1.train.SessionRunArgs', + 'tf.train.SessionRunContext': 'tf.compat.v1.train.SessionRunContext', + 'tf.train.SessionRunValues': 'tf.compat.v1.train.SessionRunValues', + 'tf.train.SingularMonitoredSession': 'tf.compat.v1.train.SingularMonitoredSession', + 'tf.train.Supervisor': 'tf.compat.v1.train.Supervisor', 'tf.train.SyncReplicasOptimizer': 'tf.compat.v1.train.SyncReplicasOptimizer', + 'tf.train.VocabInfo': 'tf.estimator.VocabInfo', + 'tf.train.WorkerSessionCreator': 'tf.compat.v1.train.WorkerSessionCreator', 'tf.train.add_queue_runner': 'tf.compat.v1.train.add_queue_runner', 'tf.train.assert_global_step': 'tf.compat.v1.train.assert_global_step', 'tf.train.basic_train_loop': 'tf.compat.v1.train.basic_train_loop', 'tf.train.batch': 'tf.compat.v1.train.batch', 'tf.train.batch_join': 'tf.compat.v1.train.batch_join', 'tf.train.checkpoint_exists': 'tf.compat.v1.train.checkpoint_exists', - 'tf.train.cosine_decay': 'tf.compat.v1.train.cosine_decay', - 'tf.train.cosine_decay_restarts': 'tf.compat.v1.train.cosine_decay_restarts', 'tf.train.create_global_step': 'tf.compat.v1.train.create_global_step', 'tf.train.do_quantize_training_on_graphdef': 'tf.compat.v1.train.do_quantize_training_on_graphdef', - 'tf.train.exponential_decay': 'tf.compat.v1.train.exponential_decay', 'tf.train.export_meta_graph': 'tf.compat.v1.train.export_meta_graph', 'tf.train.generate_checkpoint_state_proto': 'tf.compat.v1.train.generate_checkpoint_state_proto', 'tf.train.get_checkpoint_mtimes': 'tf.compat.v1.train.get_checkpoint_mtimes', @@ -447,32 +620,31 @@ renames = { 'tf.train.get_or_create_global_step': 'tf.compat.v1.train.get_or_create_global_step', 'tf.train.global_step': 'tf.compat.v1.train.global_step', 'tf.train.import_meta_graph': 'tf.compat.v1.train.import_meta_graph', + 'tf.train.init_from_checkpoint': 'tf.compat.v1.train.init_from_checkpoint', 'tf.train.input_producer': 'tf.compat.v1.train.input_producer', - 'tf.train.inverse_time_decay': 'tf.compat.v1.train.inverse_time_decay', 'tf.train.limit_epochs': 'tf.compat.v1.train.limit_epochs', - 'tf.train.linear_cosine_decay': 'tf.compat.v1.train.linear_cosine_decay', 'tf.train.match_filenames_once': 'tf.io.match_filenames_once', 'tf.train.maybe_batch': 'tf.compat.v1.train.maybe_batch', 'tf.train.maybe_batch_join': 'tf.compat.v1.train.maybe_batch_join', 'tf.train.maybe_shuffle_batch': 'tf.compat.v1.train.maybe_shuffle_batch', 'tf.train.maybe_shuffle_batch_join': 'tf.compat.v1.train.maybe_shuffle_batch_join', - 'tf.train.natural_exp_decay': 'tf.compat.v1.train.natural_exp_decay', - 'tf.train.noisy_linear_cosine_decay': 'tf.compat.v1.train.noisy_linear_cosine_decay', 'tf.train.piecewise_constant': 'tf.compat.v1.train.piecewise_constant', - 'tf.train.polynomial_decay': 'tf.compat.v1.train.polynomial_decay', 'tf.train.queue_runner.QueueRunner': 'tf.compat.v1.train.queue_runner.QueueRunner', 'tf.train.queue_runner.add_queue_runner': 'tf.compat.v1.train.queue_runner.add_queue_runner', 'tf.train.queue_runner.start_queue_runners': 'tf.compat.v1.train.queue_runner.start_queue_runners', 'tf.train.range_input_producer': 'tf.compat.v1.train.range_input_producer', 'tf.train.remove_checkpoint': 'tf.compat.v1.train.remove_checkpoint', + 'tf.train.replica_device_setter': 'tf.compat.v1.train.replica_device_setter', 'tf.train.shuffle_batch': 'tf.compat.v1.train.shuffle_batch', 'tf.train.shuffle_batch_join': 'tf.compat.v1.train.shuffle_batch_join', 'tf.train.slice_input_producer': 'tf.compat.v1.train.slice_input_producer', 'tf.train.start_queue_runners': 'tf.compat.v1.train.start_queue_runners', 'tf.train.string_input_producer': 'tf.compat.v1.train.string_input_producer', 'tf.train.update_checkpoint_state': 'tf.compat.v1.train.update_checkpoint_state', + 'tf.train.warm_start': 'tf.compat.v1.train.warm_start', 'tf.train.write_graph': 'tf.io.write_graph', 'tf.trainable_variables': 'tf.compat.v1.trainable_variables', + 'tf.truncated_normal': 'tf.random.truncated_normal', 'tf.uniform_unit_scaling_initializer': 'tf.initializers.uniform_unit_scaling', 'tf.unsorted_segment_max': 'tf.math.unsorted_segment_max', 'tf.unsorted_segment_mean': 'tf.math.unsorted_segment_mean', @@ -480,12 +652,13 @@ renames = { 'tf.unsorted_segment_prod': 'tf.math.unsorted_segment_prod', 'tf.unsorted_segment_sqrt_n': 'tf.math.unsorted_segment_sqrt_n', 'tf.unsorted_segment_sum': 'tf.math.unsorted_segment_sum', - 'tf.variable_creator_scope': 'tf.compat.v1.variable_creator_scope', + 'tf.variable_axis_size_partitioner': 'tf.compat.v1.variable_axis_size_partitioner', 'tf.variable_op_scope': 'tf.compat.v1.variable_op_scope', 'tf.variable_scope': 'tf.compat.v1.variable_scope', 'tf.variables_initializer': 'tf.compat.v1.variables_initializer', 'tf.variance_scaling_initializer': 'tf.keras.initializers.VarianceScaling', - 'tf.verify_tensor_all_finite': 'tf.debugging.assert_all_finite', + 'tf.verify_tensor_all_finite': 'tf.compat.v1.verify_tensor_all_finite', + 'tf.wrap_function': 'tf.compat.v1.wrap_function', 'tf.write_file': 'tf.io.write_file', 'tf.zeta': 'tf.math.zeta' } diff --git a/tensorflow/tools/compatibility/testdata/test_file_v1_12.py b/tensorflow/tools/compatibility/testdata/test_file_v1_12.py new file mode 100644 index 00000000000..fd688781b0d --- /dev/null +++ b/tensorflow/tools/compatibility/testdata/test_file_v1_12.py @@ -0,0 +1,71 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Tests for tf upgrader.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +import tensorflow as tf +from tensorflow.python.framework import test_util +from tensorflow.python.platform import test as test_lib + + +class TestUpgrade(test_util.TensorFlowTestCase): + """Test various APIs that have been changed in 2.0.""" + + def setUp(self): + tf.enable_eager_execution() + + def testRenames(self): + with self.cached_session(): + self.assertAllClose(1.04719755, tf.acos(0.5)) + self.assertAllClose(0.5, tf.rsqrt(4.0)) + + def testSerializeSparseTensor(self): + sp_input = tf.SparseTensor( + indices=tf.constant([[1]], dtype=tf.int64), + values=tf.constant([2], dtype=tf.int64), + dense_shape=[2]) + + with self.cached_session(): + serialized_sp = tf.serialize_sparse(sp_input, 'serialize_name', tf.string) + self.assertEqual((3,), serialized_sp.shape) + self.assertTrue(serialized_sp[0].numpy()) # check non-empty + + def testSerializeManySparse(self): + sp_input = tf.SparseTensor( + indices=tf.constant([[0, 1]], dtype=tf.int64), + values=tf.constant([2], dtype=tf.int64), + dense_shape=[1, 2]) + + with self.cached_session(): + serialized_sp = tf.serialize_many_sparse( + sp_input, 'serialize_name', tf.string) + self.assertEqual((1, 3), serialized_sp.shape) + + def testArgMaxMin(self): + self.assertAllClose( + [1], + tf.argmax([[1, 3, 2]], name='abc', dimension=1)) + self.assertAllClose( + [0, 0, 0], + tf.argmax([[1, 3, 2]], dimension=0)) + self.assertAllClose( + [0], + tf.argmin([[1, 3, 2]], name='abc', dimension=1)) + + +if __name__ == "__main__": + test_lib.main() diff --git a/tensorflow/tools/compatibility/tf_upgrade_v2.py b/tensorflow/tools/compatibility/tf_upgrade_v2.py index 0df8b0f3769..655e680d5bf 100644 --- a/tensorflow/tools/compatibility/tf_upgrade_v2.py +++ b/tensorflow/tools/compatibility/tf_upgrade_v2.py @@ -18,8 +18,6 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function -import argparse - from tensorflow.tools.compatibility import ast_edits from tensorflow.tools.compatibility import renames_v2 @@ -31,23 +29,452 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec): # Maps from a function name to a dictionary that describes how to # map from an old argument keyword to the new argument keyword. self.function_keyword_renames = { + "tf.argmin": { + "dimension": "axis", + }, + "tf.argmax": { + "dimension": "axis", + }, + "tf.image.crop_and_resize": { + "box_ind": "box_indices", + }, + "tf.image.extract_image_patches": { + "ksizes": "sizes", + }, + "tf.extract_image_patches": { + "ksizes": "sizes", + }, + "tf.expand_dims": { + "dim": "axis", + }, + "tf.batch_to_space": { + "block_size": "block_shape", + }, + "tf.constant": { + "verify_shape": "verify_shape_is_now_always_true", + }, "tf.convert_to_tensor": { "preferred_dtype": "dtype_hint" }, + "tf.nn.softmax_cross_entropy_with_logits_v2": { + "dim": "axis" + }, + "tf.linalg.l2_normalize": { + "dim": "axis", + }, + "tf.math.count_nonzero": { + "input_tensor": "input", + "keep_dims": "keepdims", + "reduction_indices": "axis", + }, + "tf.nn.erosion2d": { + "kernel": "filters", + "rates": "dilations", + }, + "tf.math.l2_normalize": { + "dim": "axis", + }, + "tf.math.log_softmax": { + "dim": "axis", + }, + "tf.math.softmax": { + "dim": "axis" + }, + "tf.nn.l2_normalize": { + "dim": "axis", + }, + "tf.nn.log_softmax": { + "dim": "axis", + }, + "tf.nn.moments": { + "keep_dims": "keepdims", + }, + "tf.nn.pool": { + "dilation_rate": "dilations" + }, + "tf.nn.separable_conv2d": { + "rate": "dilations" + }, + "tf.nn.softmax": { + "dim": "axis" + }, + "tf.nn.sufficient_statistics": { + "keep_dims": "keepdims" + }, + "tf.debugging.assert_all_finite": { + "t": "x", + "msg": "message", + }, + "tf.sparse.add": { + "thresh": "threshold", + }, + "tf.sparse_add": { + "thresh": "threshold", + }, + "tf.sparse.concat": { + "concat_dim": "axis", + }, + "tf.sparse_concat": { + "concat_dim": "axis", + }, + "tf.sparse.split": { + "split_dim": "axis", + }, + "tf.max_pool_with_argmax": { + "Targmax": "output_dtype", + }, + "tf.multinomial": { + "output_dtype": "dtype", + }, + "tf.random.multinomial": { + "output_dtype": "dtype", + }, + "tf.nn.batch_norm_with_global_normalization": { + "t": "input", + "m": "mean", + "v": "variance", + }, + "tf.nn.dilation2d": { + "filter": "filters", + "rates": "dilations", + }, + "tf.nn.conv3d": { + "filter": "filters" + }, + "tf.zeros_like": { + "tensor": "input", + }, + "tf.ones_like": { + "tensor": "input", + }, + "tf.nn.conv3d_transpose": { + "value": "input", + "filter": "filters", + }, + "tf.nn.convolution": { + "filter": "filters", + "dilation_rate": "dilations", + }, + "tf.gfile.Exists": { + "filename": "path", + }, + "tf.gfile.Remove": { + "filename": "path", + }, + "tf.gfile.Stat": { + "filename": "path", + }, + "tf.gfile.Glob": { + "filename": "pattern", + }, + "tf.gfile.MkDir": { + "dirname": "path", + }, + "tf.gfile.MakeDirs": { + "dirname": "path", + }, + "tf.gfile.DeleteRecursively": { + "dirname": "path", + }, + "tf.gfile.IsDirectory": { + "dirname": "path", + }, + "tf.gfile.ListDirectory": { + "dirname": "path", + }, + "tf.gfile.Copy": { + "oldpath": "src", + "newpath": "dst", + }, + "tf.gfile.Rename": { + "oldname": "src", + "newname": "dst", + }, + "tf.gfile.Walk": { + "in_order": "topdown", + }, + "tf.random.stateless_multinomial": { + "output_dtype": "dtype", + }, + "tf.string_to_number": { + "string_tensor": "input", + }, + "tf.strings.to_number": { + "string_tensor": "input", + }, + "tf.string_to_hash_bucket": { + "string_tensor": "input", + }, + "tf.strings.to_hash_bucket": { + "string_tensor": "input", + }, + "tf.reduce_all": { + "reduction_indices": "axis", + "keep_dims": "keepdims", + }, + "tf.math.reduce_all": { + "reduction_indices": "axis", + "keep_dims": "keepdims", + }, + "tf.reduce_any": { + "reduction_indices": "axis", + "keep_dims": "keepdims", + }, + "tf.math.reduce_any": { + "reduction_indices": "axis", + "keep_dims": "keepdims", + }, + "tf.reduce_min": { + "reduction_indices": "axis", + "keep_dims": "keepdims", + }, + "tf.math.reduce_min": { + "reduction_indices": "axis", + "keep_dims": "keepdims", + }, + "tf.reduce_max": { + "reduction_indices": "axis", + "keep_dims": "keepdims", + }, + "tf.math.reduce_max": { + "reduction_indices": "axis", + "keep_dims": "keepdims", + }, + "tf.reduce_sum": { + "reduction_indices": "axis", + "keep_dims": "keepdims", + }, + "tf.math.reduce_sum": { + "reduction_indices": "axis", + "keep_dims": "keepdims", + }, + "tf.reduce_mean": { + "reduction_indices": "axis", + "keep_dims": "keepdims", + }, + "tf.math.reduce_mean": { + "reduction_indices": "axis", + "keep_dims": "keepdims", + }, + "tf.reduce_prod": { + "reduction_indices": "axis", + "keep_dims": "keepdims", + }, + "tf.math.reduce_prod": { + "reduction_indices": "axis", + "keep_dims": "keepdims", + }, + "tf.reduce_logsumexp": { + "reduction_indices": "axis", + "keep_dims": "keepdims", + }, + "tf.math.reduce_logsumexp": { + "reduction_indices": "axis", + "keep_dims": "keepdims", + }, + "tf.reduce_join": { + "keep_dims": "keepdims", + "reduction_indices": "axis" + }, + "tf.strings.reduce_join": { + "keep_dims": "keepdims", + "reduction_indices": "axis" + }, + "tf.squeeze": { + "squeeze_dims": "axis", + }, } + # pylint: disable=line-too-long + # Add additional renames not in renames_v2.py here. + # IMPORTANT: For the renames in here, if you also need to add to + # function_reorders or function_keyword_renames, use the OLD function name. + # These renames happen after the arguments have been processed. + self.manual_symbol_renames = { + "tf.batch_to_space_nd": + "tf.batch_to_space", + "tf.extract_image_patches": + "tf.image.extract_image_patches", + "tf.gfile.Copy": + "tf.io.gfile.copy", + "tf.gfile.DeleteRecursively": + "tf.io.gfile.rmtree", + "tf.gfile.Exists": + "tf.io.gfile.exists", + "tf.gfile.Glob": + "tf.io.gfile.glob", + "tf.gfile.IsDirectory": + "tf.io.gfile.isdir", + "tf.gfile.ListDirectory": + "tf.io.gfile.listdir", + "tf.gfile.MakeDirs": + "tf.io.gfile.makedirs", + "tf.gfile.MkDir": + "tf.io.gfile.mkdir", + "tf.gfile.Remove": + "tf.io.gfile.remove", + "tf.gfile.Rename": + "tf.io.gfile.rename", + "tf.gfile.Stat": + "tf.io.gfile.stat", + "tf.gfile.Walk": + "tf.io.gfile.walk", + "tf.contrib.data.AUTOTUNE": + "tf.data.experimental.AUTOTUNE", + "tf.contrib.data.Counter": + "tf.data.experimental.Counter", + "tf.contrib.data.CheckpointInputPipelineHook": + "tf.data.experimental.CheckpointInputPipelineHook", + "tf.contrib.data.CsvDataset": + "tf.data.experimental.CsvDataset", + "tf.contrib.data.Optional": + "tf.data.experimental.Optional", + "tf.contrib.data.RandomDataset": + "tf.data.experimental.RandomDataset", + "tf.contrib.data.Reducer": + "tf.data.experimental.Reducer", + "tf.contrib.data.SqlDataset": + "tf.data.experimental.SqlDataset", + "tf.contrib.data.StatsAggregator": + "tf.data.experimental.StatsAggregator", + "tf.contrib.data.TFRecordWriter": + "tf.data.experimental.TFRecordWriter", + "tf.contrib.data.assert_element_shape": + "tf.data.experimental.assert_element_shape", + "tf.contrib.data.batch_and_drop_remainder": + "tf.compat.v1.contrib.data.batch_and_drop_remainder", + "tf.contrib.data.bucket_by_sequence_length": + "tf.data.experimental.bucket_by_sequence_length", + "tf.contrib.data.choose_from_datasets": + "tf.data.experimental.choose_from_datasets", + "tf.contrib.data.copy_to_device": + "tf.data.experimental.copy_to_device", + "tf.contrib.data.dense_to_sparse_batch": + "tf.data.experimental.dense_to_sparse_batch", + "tf.contrib.data.enumerate_dataset": + "tf.data.experimental.enumerate_dataset", + "tf.contrib.data.get_next_as_optional": + "tf.data.experimental.get_next_as_optional", + "tf.contrib.data.get_single_element": + "tf.data.experimental.get_single_element", + "tf.contrib.data.group_by_reducer": + "tf.data.experimental.group_by_reducer", + "tf.contrib.data.group_by_window": + "tf.data.experimental.group_by_window", + "tf.contrib.data.ignore_errors": + "tf.data.experimental.ignore_errors", + "tf.contrib.data.latency_stats": + "tf.data.experimental.latency_stats", + "tf.contrib.data.make_batched_features_dataset": + "tf.data.experimental.make_batched_features_dataset", + "tf.contrib.data.make_csv_dataset": + "tf.data.experimental.make_csv_dataset", + "tf.contrib.data.make_saveable_from_iterator": + "tf.data.experimental.make_saveable_from_iterator", + "tf.contrib.data.map_and_batch": + "tf.data.experimental.map_and_batch", + "tf.contrib.data.padded_batch_and_drop_remainder": + "tf.compat.v1.contrib.data.padded_batch_and_drop_remainder", + "tf.contrib.data.parallel_interleave": + "tf.data.experimental.parallel_interleave", + "tf.contrib.data.parse_example_dataset": + "tf.data.experimental.parse_example_dataset", + "tf.contrib.data.prefetch_to_device": + "tf.data.experimental.prefetch_to_device", + "tf.contrib.data.read_batch_features": + "tf.compat.v1.contrib.data.read_batch_features", + "tf.contrib.data.reduce_dataset": + "tf.compat.v1.contrib.data.reduce_dataset", + "tf.contrib.data.rejection_resample": + "tf.data.experimental.rejection_resample", + "tf.contrib.data.sample_from_datasets": + "tf.data.experimental.sample_from_datasets", + "tf.contrib.data.scan": + "tf.data.experimental.scan", + "tf.contrib.data.set_stats_aggregator": + "tf.data.experimental.set_stats_aggregator", + "tf.contrib.data.shuffle_and_repeat": + "tf.data.experimental.shuffle_and_repeat", + "tf.contrib.data.sliding_window_batch": + "tf.compat.v1.contrib.data.sliding_window_batch", + "tf.contrib.data.sloppy_interleave": + "tf.compat.v1.contrib.data.sloppy_interleave", + "tf.contrib.data.unbatch": + "tf.data.experimental.unbatch", + "tf.contrib.data.unique": + "tf.data.experimental.unique", + "tf.contrib.framework.sort": + "tf.sort", + "tf.contrib.framework.argsort": + "tf.argsort", + "tf.manip.batch_to_space_nd": + "tf.batch_to_space", + "tf.quantize_v2": + "tf.quantization.quantize", + "tf.sparse_add": + "tf.sparse.add", + "tf.sparse_concat": + "tf.sparse.concat", + "tf.sparse_split": + "tf.sparse.split", + "tf.sparse_matmul": + "tf.linalg.matmul", + "tf.random.stateless_multinomial": + "tf.random.stateless_categorical", + "tf.string_to_hash_bucket": + "tf.strings.to_hash_bucket", + "tf.string_to_number": + "tf.strings.to_number", + "tf.multinomial": + "tf.random.categorical", + "tf.random.multinomial": + "tf.random.categorical", + "tf.reduce_join": + "tf.strings.reduce_join", + "tf.load_file_system_library": + "tf.load_library", + "tf.pywrap_tensorflow": + "tf.compat.v1.pywrap_tensorflow", + "tf.bincount": + "tf.math.bincount", + "tf.confusion_matrix": + "tf.math.confusion_matrix", + "tf.train.confusion_matrix": + "tf.math.confusion_matrix", + "tf.decode_csv": + "tf.io.decode_csv", + "tf.data.Iterator": + "tf.compat.v1.data.Iterator", + "tf.parse_example": + "tf.io.parse_example", + "tf.parse_single_example": + "tf.io.parse_single_example", + "tf.nn.fused_batch_norm": + "tf.compat.v1.nn.fused_batch_norm", + "tf.nn.softmax_cross_entropy_with_logits_v2": + "tf.nn.softmax_cross_entropy_with_logits", + "tf.losses.Reduction.MEAN": + "tf.compat.v1.losses.Reduction.MEAN", + "tf.losses.Reduction.SUM_BY_NONZERO_WEIGHTS": + "tf.compat.v1.losses.Reduction.SUM_BY_NONZERO_WEIGHTS", + "tf.losses.Reduction.SUM_OVER_NONZERO_WEIGHTS": + "tf.compat.v1.losses.Reduction.SUM_OVER_NONZERO_WEIGHTS", + "tf.lite.constants.FLOAT": + "tf.float32", + "tf.lite.constants.INT32": + "tf.int32", + "tf.lite.constants.INT64": + "tf.int64", + "tf.lite.constants.STRING": + "tf.string", + "tf.lite.constants.QUANTIZED_UINT8": + "tf.uint8", + } + # pylint: enable=line-too-long + # Mapping from function to the new name of the function self.symbol_renames = renames_v2.renames - # pylint: disable=line-too-long - # Add additional renames not in renames_v2.py here. - self.symbol_renames.update({ - }) - # pylint: enable=line-too-long - - # For custom behavior and if auto-generate rename in renames_v2.py - # is incorrect, add the op name here to exclude it from renames_v2.py. - excluded_renames = [ - ] + self.symbol_renames.update(self.manual_symbol_renames) # Variables that should be changed to functions. self.change_to_function = {} @@ -55,22 +482,194 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec): # Functions that were reordered should be changed to the new keyword args # for safety, if positional arguments are used. If you have reversed the # positional arguments yourself, this could do the wrong thing. + # IMPORTANT: order here should correspond to OLD argument order. + # We just prepend "arg_name=" to all arguments in function calls. self.function_reorders = { - "tf.convert_to_tensor": ["value", "dtype", "preferred_dtype", "name"], - "tf.argmin": ["input", "axis", "output_type", "name"], - "tf.argmax": ["input", "axis", "output_type", "name"], + "tf.io.serialize_sparse": ["sp_input", "name", "out_type"], + "tf.io.serialize_many_sparse": ["sp_input", "name", "out_type"], + "tf.argmax": ["input", "axis", "name", "axis", "output_type"], + "tf.argmin": ["input", "axis", "name", "axis", "output_type"], + "tf.batch_to_space": ["input", "crops", "block_size", "name"], "tf.boolean_mask": ["tensor", "mask", "name", "axis"], + "tf.convert_to_tensor": ["value", "dtype", "name", "preferred_dtype"], + "tf.nn.moments": ["x", "axes", "shift", "keepdims", "name"], + "tf.nn.convolution": [ + "input", "filter", "padding", "strides", "dilation_rate", "name", + "data_format" + ], + "tf.nn.crelu": ["features", "name", "axis"], + "tf.nn.pool": [ + "input", "window_shape", "pooling_type", "padding", "dilation_rate", + "strides", "name", "data_format" + ], + "tf.nn.depthwise_conv2d": [ + "input", "filter", "strides", "padding", "rate", "name", + "data_format" + ], + "tf.multinomial": [ + "logits", "num_samples", "seed", "name", "output_dtype" + ], + "tf.random.multinomial": [ + "logits", "num_samples", "seed", "name", "output_dtype" + ], + "tf.pad": ["tensor", "paddings", "mode", "name", "constant_values"], + "tf.quantize_v2": [ + "input", "min_range", "max_range", "T", "mode", "name", "round_mode" + ], + "tf.feature_column.categorical_column_with_vocabulary_file": [ + "key", "vocabulary_file", "vocabulary_size", "num_oov_buckets", + "default_value", "dtype" + ], + "tf.shape": ["input", "name", "out_type"], + "tf.size": ["input", "name", "out_type"], + "tf.random.poisson": ["lam", "shape", "dtype", "seed", "name"], + "tf.sparse.add": ["a", "b", "thresh"], + "tf.sparse_add": ["a", "b", "thresh"], + "tf.sparse.concat": [ + "axis", "sp_inputs", "name", "expand_nonconcat_dim", "concat_dim" + ], + "tf.sparse_concat": [ + "axis", "sp_inputs", "name", "expand_nonconcat_dim", "concat_dim" + ], + "tf.sparse.segment_mean": [ + "data", "indices", "segment_ids", "name", "num_segments" + ], + "tf.sparse.segment_sqrt_n": [ + "data", "indices", "segment_ids", "name", "num_segments" + ], + "tf.sparse.segment_sum": [ + "data", "indices", "segment_ids", "name", "num_segments" + ], + "tf.sparse_matmul": [ + "a", "b", "transpose_a", "transpose_b", "a_is_sparse", + "b_is_sparse", "name" + ], + "tf.io.decode_csv": [ + "records", + "record_defaults", + "field_delim", + "use_quote_delim", + "name", + "na_value", + "select_cols", + ], + "tf.strings.substr": ["input", "pos", "len", "name", "unit"], + "tf.strings.reduce_join": [ + "input", "axis", "keep_dims", "separator", "name", + "reduction_indices" + ], + "tf.strings.length": ["input", "name", "unit"], + "tf.transpose": ["a", "perm", "name", "conjugate"], + "tf.tuple": ["tensors", "name", "control_inputs"], + "tf.parse_example": [ + "serialized", "features", "name", "example_names" + ], + "tf.parse_single_example": [ + "serialized", "features", "name", "example_names" + ], + "tf.io.parse_example": [ + "serialized", "features", "name", "example_names" + ], + "tf.io.parse_single_example": [ + "serialized", "features", "name", "example_names" + ], + "tf.while_loop": [ + "cond", "body", "loop_vars", "shape_invariants", + "parallel_iterations", "back_prop", "swap_memory", "name", + "maximum_iterations", "return_same_structure" + ], + "tf.reduce_all": [ + "input_tensor", "axis", "keepdims", "name", "reduction_indices", + "keep_dims" + ], + "tf.math.reduce_all": [ + "input_tensor", "axis", "keepdims", "name", "reduction_indices", + "keep_dims" + ], + "tf.reduce_any": [ + "input_tensor", "axis", "keepdims", "name", "reduction_indices", + "keep_dims" + ], + "tf.math.reduce_any": [ + "input_tensor", "axis", "keepdims", "name", "reduction_indices", + "keep_dims" + ], + "tf.reduce_min": [ + "input_tensor", "axis", "keepdims", "name", "reduction_indices", + "keep_dims" + ], + "tf.math.reduce_min": [ + "input_tensor", "axis", "keepdims", "name", "reduction_indices", + "keep_dims" + ], + "tf.reduce_max": [ + "input_tensor", "axis", "keepdims", "name", "reduction_indices", + "keep_dims" + ], + "tf.math.reduce_max": [ + "input_tensor", "axis", "keepdims", "name", "reduction_indices", + "keep_dims" + ], + "tf.reduce_sum": [ + "input_tensor", "axis", "keepdims", "name", "reduction_indices", + "keep_dims" + ], + "tf.math.reduce_sum": [ + "input_tensor", "axis", "keepdims", "name", "reduction_indices", + "keep_dims" + ], + "tf.reduce_mean": [ + "input_tensor", "axis", "keepdims", "name", "reduction_indices", + "keep_dims" + ], + "tf.math.reduce_mean": [ + "input_tensor", "axis", "keepdims", "name", "reduction_indices", + "keep_dims" + ], + "tf.reduce_prod": [ + "input_tensor", "axis", "keepdims", "name", "reduction_indices", + "keep_dims" + ], + "tf.math.reduce_prod": [ + "input_tensor", "axis", "keepdims", "name", "reduction_indices", + "keep_dims" + ], + "tf.reduce_logsumexp": [ + "input_tensor", "axis", "keepdims", "name", "reduction_indices", + "keep_dims" + ], + "tf.math.reduce_logsumexp": [ + "input_tensor", "axis", "keepdims", "name", "reduction_indices", + "keep_dims" + ], + "tf.reduce_join": [ + "input", "axis", "keep_dims", "separator", "name", + "reduction_indices" + ], + "tf.confusion_matrix": [ + "labels", "predictions", "num_classes", "dtype", "name", "weights" + ], + "tf.math.confusion_matrix": [ + "labels", "predictions", "num_classes", "dtype", "name", "weights" + ] } # Specially handled functions. - self.function_handle = {} + self.function_handle = { + "tf.nn.dropout": self._dropout_handler, + "tf.gradients": self._colocate_handler("tf.gradients"), + "*.minimize": self._colocate_handler("Optimizer.minimize"), + "*.compute_gradients": + self._colocate_handler("Optimizer.compute_gradients"), + } decay_function_comment = ( - "ERROR: has been changed to return a callable instead " - "of a tensor when graph building, but its functionality remains " + "WARNING: has been changed to return a callable instead" + " of a tensor when graph building, but its functionality remains " "unchanged during eager execution (returns a callable like " "before). The converter cannot detect and fix this reliably, so " - "you need to inspect this usage manually.\n" + "this usage has been converted to compat.v1 (even though it may already" + " be correct).\n" ) # TODO(b/118888586): add default value change to update script. @@ -79,99 +678,187 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec): "SUM_OVER_BATCH_SIZE.\n" ) + assert_return_type_comment = ( + "WARNING: assert_* functions have been changed to return None, the " + "data argument has been removed, and arguments have been reordered." + "\nThe calls have been converted to compat.v1 for safety (even though " + " they may already have been correct)." + ) + + assert_rank_comment = ( + "WARNING: assert_rank_* functions have been changed to return None, and" + " the data and summarize arguments have been removed." + "\nThe calls have been converted to compat.v1 for safety (even though " + " they may already have been correct)." + ) + + tf_01s_like_no_optimize_comment = ( + "WARNING: tf.zeros_like and tf.ones_like no longer have the optimize " + "argument in TF 2.0 or after (also, `tensor' argument is renamed to " + "`input')." + "\nThe calls have been converted to compat.v1 for safety (even though " + " they may already have been correct)." + ) + # Function warnings. placeholder inside warnings will be # replaced by function name. self.function_warnings = { - "tf.train.exponential_decay": decay_function_comment, - "tf.train.piecewise_constant": decay_function_comment, - "tf.train.polynomial_decay": decay_function_comment, - "tf.train.natural_exp_decay": decay_function_comment, - "tf.train.inverse_time_decay": decay_function_comment, - "tf.train.cosine_decay": decay_function_comment, - "tf.train.cosine_decay_restarts": decay_function_comment, - "tf.train.linear_cosine_decay": decay_function_comment, - "tf.train.noisy_linear_cosine_decay": decay_function_comment, - "tf.estimator.LinearClassifier": default_loss_reduction_changed, + "tf.assert_greater": assert_return_type_comment, + "tf.assert_equal": assert_return_type_comment, + "tf.assert_less": assert_return_type_comment, + "tf.assert_rank": assert_rank_comment, + "tf.debugging.assert_equal": assert_return_type_comment, + "tf.debugging.assert_greater": assert_return_type_comment, + "tf.debugging.assert_greater_equal": assert_return_type_comment, + "tf.debugging.assert_integer": assert_return_type_comment, + "tf.debugging.assert_less": assert_return_type_comment, + "tf.debugging.assert_less_equal": assert_return_type_comment, + "tf.debugging.assert_near": assert_return_type_comment, + "tf.debugging.assert_negative": assert_return_type_comment, + "tf.debugging.assert_non_negative": assert_return_type_comment, + "tf.debugging.assert_non_positive": assert_return_type_comment, + "tf.debugging.assert_none_equal": assert_return_type_comment, + "tf.debugging.assert_positive": assert_return_type_comment, + "tf.debugging.assert_rank": assert_rank_comment, + "tf.debugging.assert_rank_at_least": assert_rank_comment, + "tf.debugging.assert_rank_in": assert_rank_comment, + "tf.flags": "tf.flags has been removed, please use the argparse or absl" + " module if you need command line parsing.", + "tf.train.exponential_decay": + decay_function_comment, + "tf.train.piecewise_constant_decay": + decay_function_comment, + "tf.train.polynomial_decay": + decay_function_comment, + "tf.train.natural_exp_decay": + decay_function_comment, + "tf.train.inverse_time_decay": + decay_function_comment, + "tf.train.cosine_decay": + decay_function_comment, + "tf.train.cosine_decay_restarts": + decay_function_comment, + "tf.train.linear_cosine_decay": + decay_function_comment, + "tf.train.noisy_linear_cosine_decay": + decay_function_comment, + "tf.estimator.LinearClassifier": + default_loss_reduction_changed, + "tf.estimator.LinearRegressor": + default_loss_reduction_changed, + "tf.estimator.DNNLinearCombinedClassifier": + default_loss_reduction_changed, + "tf.estimator.DNNLinearCombinedRegressor": + default_loss_reduction_changed, + "tf.estimator.DNNRegressor": + default_loss_reduction_changed, + "tf.estimator.DNNClassifier": + default_loss_reduction_changed, + "tf.estimator.BaselineClassifier": + default_loss_reduction_changed, + "tf.estimator.BaselineRegressor": + default_loss_reduction_changed, + "tf.nn.conv1d": + "WARNING: use_cudnn_on_gpu argument has been removed and \"value\" was " + "renamed to \"input\"", + "tf.nn.conv2d": + "WARNING: use_cudnn_on_gpu argument has been removed and \"filter\" " + "was renamed to \"filters\"", + "tf.nn.conv2d_backprop_filter": + "WARNING: use_cudnn_on_gpu argument has been removed", + "tf.nn.conv2d_backprop_input": + "WARNING: use_cudnn_on_gpu argument has been removed and \"filter\" " + "was renamed to \"filters\"", + "tf.nn.erosion2d": + "WARNING: now requires a data_format argument", + "tf.nn.nce_loss": + "WARNING: `partition_strategy` has been removed from `tf.nn.nce_loss` " + " The 'div' strategy is used by default.", + "tf.zeros_like": tf_01s_like_no_optimize_comment, + "tf.ones_like": tf_01s_like_no_optimize_comment, } - # Right now we can't have both a rename and a warning. + self.symbol_renames = { name: new_name for name, new_name in self.symbol_renames.items() - if name not in self.function_warnings and name not in excluded_renames } + export_saved_model_renamed = ( + "(Manual edit required) Please rename the method export_savedmodel() " + "to export_saved_model(). Two things to note:\n\t(1) The argument " + "strip_default_attributes has been removed. The function will always " + "strip the default attributes from ops. If this breaks your code, " + "please switch to tf.compat.v1.estimator.Estimator.\n\t(2) This change " + "only effects core estimator. If you are using " + "tf.contrib.learn.Estimator, please switch to using core estimator.") -if __name__ == "__main__": - parser = argparse.ArgumentParser( - formatter_class=argparse.RawDescriptionHelpFormatter, - description="""Convert a TensorFlow Python file to 2.0 + make_initializable_iterator_deprecation = ( + "(Manual edit required) The " + "`tf.data.Dataset.make_initializable_iterator()` method has been " + "removed. If you are using the Estimator API, you can return a dataset " + "directly from your input functions without creating an iterator. " + "As a last resort, please replace calls to that method on `dataset` " + "with a call to " + "`tf.compat.v1.data.make_initializable_iterator(dataset)`.") -Simple usage: - tf_convert_v2.py --infile foo.py --outfile bar.py - tf_convert_v2.py --intree ~/code/old --outtree ~/code/new -""") - parser.add_argument( - "--infile", - dest="input_file", - help="If converting a single file, the name of the file " - "to convert") - parser.add_argument( - "--outfile", - dest="output_file", - help="If converting a single file, the output filename.") - parser.add_argument( - "--intree", - dest="input_tree", - help="If converting a whole tree of files, the directory " - "to read from (relative or absolute).") - parser.add_argument( - "--outtree", - dest="output_tree", - help="If converting a whole tree of files, the output " - "directory (relative or absolute).") - parser.add_argument( - "--copyotherfiles", - dest="copy_other_files", - help=("If converting a whole tree of files, whether to " - "copy the other files."), - type=bool, - default=False) - parser.add_argument( - "--reportfile", - dest="report_filename", - help=("The name of the file where the report log is " - "stored." - "(default: %(default)s)"), - default="report.txt") - args = parser.parse_args() + make_one_shot_iterator_deprecation = ( + "(Manual edit required) The " + "`tf.data.Dataset.make_one_shot_iterator()` method has been " + "removed. If you are using eager execution, you can iterate over " + "`dataset` using a Python `for` loop. If you are using the Estimator " + "API, you can return a dataset directly from your input functions " + "without creating an iterator. As a last resort, please replace calls " + "to that method on `dataset` with a call to " + "`tf.compat.v1.data.make_one_shot_iterator(dataset)`.") - upgrade = ast_edits.ASTCodeUpgrader(TFAPIChangeSpec()) - report_text = None - report_filename = args.report_filename - files_processed = 0 - if args.input_file: - if not args.output_file: - raise ValueError( - "--outfile= argument is required when converting a " - "single file.") - files_processed, report_text, errors = upgrade.process_file( - args.input_file, args.output_file) - files_processed = 1 - elif args.input_tree: - if not args.output_tree: - raise ValueError( - "--outtree= argument is required when converting a " - "file tree.") - files_processed, report_text, errors = upgrade.process_tree( - args.input_tree, args.output_tree, args.copy_other_files) - else: - parser.print_help() - if report_text: - open(report_filename, "w").write(report_text) - print("TensorFlow 2.0 Upgrade Script") - print("-----------------------------") - print("Converted %d files\n" % files_processed) - print("Detected %d errors that require attention" % len(errors)) - print("-" * 80) - print("\n".join(errors)) - print("\nMake sure to read the detailed log %r\n" % report_filename) + # Specify warnings for functions that aren't restricted to the tf.x.y.z + # format. This should only be used for methods with unique names, e.g. + # export_savedmodel, which is only defined in Estimator objects. + self.unrestricted_function_warnings = { + "export_savedmodel": export_saved_model_renamed, + "make_initializable_iterator": make_initializable_iterator_deprecation, + "make_one_shot_iterator": make_one_shot_iterator_deprecation, + } + + @staticmethod + def _dropout_handler(file_edit_recorder, node): + if len(node.args) < 2: + comment = ("ERROR: tf.nn.dropout did not take arguments, so automatic " + "transformation was disabled. tf.nn.dropout has changed " + "the semantics of the second argument.") + file_edit_recorder.add( + comment, + node.lineno, + node.col_offset, + "tf.nn.dropout", + "tf.nn.dropout", + error="tf.nn.dropout requires manual check.") + else: + comment = ("WARNING: tf.nn.dropout has changed the semantics of the " + "second argument. Please check the transformation.\n") + file_edit_recorder.add( + comment, + node.args[1].lineno, + node.args[1].col_offset, + "", + "1 - ") + + @staticmethod + def _colocate_handler(name): + def _helper(file_edit_recorder, node): + for keyword in node.keywords: + if keyword.arg == "colocate_gradients_with_ops": + # TODO(jhseu): Since ast_edit.py does string replacement, there's no + # straightforward way to remove the argument. Try to fix before 2.0 is + # final. + comment = ("For tf.gradients and tf.Optimizer.minimize, " + "colocate_gradients_with_op has been removed and now " + "defaults to True.") + file_edit_recorder.add( + comment, + node.lineno, + node.col_offset, + "", + "", + error="{} requires manual check.".format(name)) + return _helper diff --git a/tensorflow/tools/compatibility/tf_upgrade_v2_main.py b/tensorflow/tools/compatibility/tf_upgrade_v2_main.py new file mode 100644 index 00000000000..543d0786423 --- /dev/null +++ b/tensorflow/tools/compatibility/tf_upgrade_v2_main.py @@ -0,0 +1,104 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Upgrader for Python scripts from 1.* TensorFlow to 2.0 TensorFlow.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import argparse + +from tensorflow.tools.compatibility import ast_edits +from tensorflow.tools.compatibility import tf_upgrade_v2 + + +def main(): + parser = argparse.ArgumentParser( + formatter_class=argparse.RawDescriptionHelpFormatter, + description="""Convert a TensorFlow Python file to 2.0 + +Simple usage: + tf_upgrade_v2.py --infile foo.py --outfile bar.py + tf_upgrade_v2.py --intree ~/code/old --outtree ~/code/new +""") + parser.add_argument( + "--infile", + dest="input_file", + help="If converting a single file, the name of the file " + "to convert") + parser.add_argument( + "--outfile", + dest="output_file", + help="If converting a single file, the output filename.") + parser.add_argument( + "--intree", + dest="input_tree", + help="If converting a whole tree of files, the directory " + "to read from (relative or absolute).") + parser.add_argument( + "--outtree", + dest="output_tree", + help="If converting a whole tree of files, the output " + "directory (relative or absolute).") + parser.add_argument( + "--copyotherfiles", + dest="copy_other_files", + help=("If converting a whole tree of files, whether to " + "copy the other files."), + type=bool, + default=True) + parser.add_argument( + "--reportfile", + dest="report_filename", + help=("The name of the file where the report log is " + "stored." + "(default: %(default)s)"), + default="report.txt") + args = parser.parse_args() + + upgrade = ast_edits.ASTCodeUpgrader(tf_upgrade_v2.TFAPIChangeSpec()) + report_text = None + report_filename = args.report_filename + files_processed = 0 + if args.input_file: + if not args.output_file: + raise ValueError( + "--outfile= argument is required when converting a " + "single file.") + files_processed, report_text, errors = upgrade.process_file( + args.input_file, args.output_file) + files_processed = 1 + elif args.input_tree: + if not args.output_tree: + raise ValueError( + "--outtree= argument is required when converting a " + "file tree.") + files_processed, report_text, errors = upgrade.process_tree( + args.input_tree, args.output_tree, args.copy_other_files) + else: + parser.print_help() + if report_text: + open(report_filename, "w").write(report_text) + print("TensorFlow 2.0 Upgrade Script") + print("-----------------------------") + print("Converted %d files\n" % files_processed) + print("Detected %d errors that require attention" % len(errors)) + print("-" * 80) + print("\n".join(errors)) + print("\nMake sure to read the detailed log %r\n" % report_filename) + + +if __name__ == "__main__": + main() diff --git a/tensorflow/tools/compatibility/tf_upgrade_v2_test.py b/tensorflow/tools/compatibility/tf_upgrade_v2_test.py index 9060b1c71f1..b8b02c9c7fc 100644 --- a/tensorflow/tools/compatibility/tf_upgrade_v2_test.py +++ b/tensorflow/tools/compatibility/tf_upgrade_v2_test.py @@ -17,15 +17,71 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function + import os import tempfile + import six +import tensorflow as tf +# OSS TF V2 import placeholder. + + from tensorflow.python.framework import test_util from tensorflow.python.platform import test as test_lib +from tensorflow.python.util import tf_decorator +from tensorflow.python.util import tf_export +from tensorflow.python.util import tf_inspect +from tensorflow.tools.common import public_api +from tensorflow.tools.common import traverse from tensorflow.tools.compatibility import ast_edits from tensorflow.tools.compatibility import tf_upgrade_v2 +_TENSORFLOW_API_ATTR_V1 = ( + tf_export.API_ATTRS_V1[tf_export.TENSORFLOW_API_NAME].names) +_TENSORFLOW_API_ATTR = tf_export.API_ATTRS[tf_export.TENSORFLOW_API_NAME].names +_ESTIMATOR_API_ATTR_V1 = ( + tf_export.API_ATTRS_V1[tf_export.ESTIMATOR_API_NAME].names) +_ESTIMATOR_API_ATTR = tf_export.API_ATTRS[tf_export.ESTIMATOR_API_NAME].names + + +def get_v1_names(symbol): + names_v1 = [] + if hasattr(symbol, _TENSORFLOW_API_ATTR_V1): + names_v1.extend(getattr(symbol, _TENSORFLOW_API_ATTR_V1)) + if hasattr(symbol, _ESTIMATOR_API_ATTR_V1): + names_v1.extend(getattr(symbol, _ESTIMATOR_API_ATTR_V1)) + return names_v1 + + +def get_v2_names(symbol): + names_v2 = set() + if hasattr(symbol, _TENSORFLOW_API_ATTR): + names_v2.update(getattr(symbol, _TENSORFLOW_API_ATTR)) + if hasattr(symbol, _ESTIMATOR_API_ATTR): + names_v2.update(getattr(symbol, _ESTIMATOR_API_ATTR)) + return list(names_v2) + + +def get_func_and_args_from_str(call_str): + """Parse call string to get function and argument names. + + Args: + call_str: Call string must be in the form: + `tf.foo(arg1=val1, arg2=val2, ...)`. + + Returns: + (function_name, list of arg names) tuple. + """ + open_paren_index = call_str.find("(") + close_paren_index = call_str.rfind(")") + + function_name = call_str[:call_str.find("(")] + args = call_str[open_paren_index+1:close_paren_index].split(",") + args = [arg.split("=")[0].strip() for arg in args] + return function_name, args + + class TestUpgrade(test_util.TensorFlowTestCase): """Test various APIs that have been changed in 2.0. @@ -34,6 +90,22 @@ class TestUpgrade(test_util.TensorFlowTestCase): work when run with current TensorFlow. """ + @classmethod + def setUpClass(cls): + cls.v2_symbols = {} + if not hasattr(tf.compat, "v2"): + return + + def symbol_collector(unused_path, unused_parent, children): + for child in children: + _, attr = tf_decorator.unwrap(child[1]) + api_names_v2 = get_v2_names(attr) + for name in api_names_v2: + cls.v2_symbols["tf." + name] = attr + + visitor = public_api.PublicAPIVisitor(symbol_collector) + traverse.traverse(tf.compat.v2, visitor) + def _upgrade(self, old_file_text): in_file = six.StringIO(old_file_text) out_file = six.StringIO() @@ -64,6 +136,85 @@ class TestUpgrade(test_util.TensorFlowTestCase): _, unused_report, unused_errors, new_text = self._upgrade(text) self.assertEqual(new_text, "tf.math.rsqrt(tf.math.log_sigmoid(3.8))\n") + def testAllAPI(self): + if not hasattr(tf.compat, "v2"): + return + + # Converts all symbols in the v1 namespace to the v2 namespace, raising + # an error if the target of the conversion is not in the v2 namespace. + def conversion_visitor(unused_path, unused_parent, children): + for child in children: + _, attr = tf_decorator.unwrap(child[1]) + api_names = get_v1_names(attr) + for name in api_names: + _, _, _, text = self._upgrade("tf." + name) + if (text and + not text.startswith("tf.compat.v1") and + text not in self.v2_symbols): + self.assertFalse( + True, "Symbol %s generated from %s not in v2 API" % ( + text, name)) + + visitor = public_api.PublicAPIVisitor(conversion_visitor) + visitor.do_not_descend_map["tf"].append("contrib") + visitor.private_map["tf.compat"] = ["v1", "v2"] + traverse.traverse(tf.compat.v1, visitor) + + def testKeywordArgNames(self): + if not hasattr(tf.compat, "v2"): + return + + all_keyword_renames = ( + tf_upgrade_v2.TFAPIChangeSpec().function_keyword_renames) + v2_name_exceptions = {"verify_shape_is_now_always_true"} + + # Visitor that verifies V1 argument names, converts to V2 and checks + # V2 argument names. + def conversion_visitor(unused_path, unused_parent, children): + for child in children: + _, attr = tf_decorator.unwrap(child[1]) + names_v1 = get_v1_names(attr) + + for name in names_v1: + name = "tf.%s" % name + if name not in all_keyword_renames: + continue + arg_names_v1 = tf_inspect.getargspec(attr)[0] + keyword_renames = all_keyword_renames[name] + self.assertEqual(type(keyword_renames), dict) + + # Assert that v1 function has valid v1 argument names. + for from_name, _ in keyword_renames.items(): + self.assertIn( + from_name, arg_names_v1, + "%s not found in %s arguments: %s" % + (from_name, name, str(arg_names_v1))) + + # Assert that arg names after converting to v2 are present in + # v2 function. + # 1. First, create an input of the form: + # tf.foo(arg1=val1, arg2=val2, ...) + args = ",".join( + ["%s=%d" % (from_name, from_index) + for from_index, from_name in enumerate(keyword_renames.keys())]) + text_input = "%s(%s)" % (name, args) + # 2. Convert the input to V2. + _, _, _, text = self._upgrade(text_input) + new_function_name, new_args = get_func_and_args_from_str(text) + # 3. Verify V2 function and arguments. + # Note: If we rename arguments, new function must be available in 2.0. + # We should not be using compat.v1 in this case. + self.assertIn(new_function_name, self.v2_symbols) + args_v2 = tf_inspect.getargspec(self.v2_symbols[new_function_name])[0] + args_v2.extend(v2_name_exceptions) + for new_arg in new_args: + self.assertIn(new_arg, args_v2) + + visitor = public_api.PublicAPIVisitor(conversion_visitor) + visitor.do_not_descend_map["tf"].append("contrib") + visitor.private_map["tf.compat"] = ["v1", "v2"] + traverse.traverse(tf.compat.v1, visitor) + def testRenameConstant(self): text = "tf.MONOLITHIC_BUILD\n" _, unused_report, unused_errors, new_text = self._upgrade(text) @@ -72,6 +223,16 @@ class TestUpgrade(test_util.TensorFlowTestCase): _, unused_report, unused_errors, new_text = self._upgrade(text) self.assertEqual(new_text, "some_call(tf.sysconfig.MONOLITHIC_BUILD)\n") + def testRenameArgs(self): + text = ("tf.nn.pool(input_a, window_shape_a, pooling_type_a, padding_a, " + "dilation_rate_a, strides_a, name_a, data_format_a)\n") + _, unused_report, unused_errors, new_text = self._upgrade(text) + self.assertEqual(new_text, + ("tf.nn.pool(input=input_a, window_shape=window_shape_a," + " pooling_type=pooling_type_a, padding=padding_a, " + "dilations=dilation_rate_a, strides=strides_a, " + "name=name_a, data_format=data_format_a)\n")) + def testReorder(self): text = "tf.boolean_mask(a, b, c, d)\n" _, unused_report, unused_errors, new_text = self._upgrade(text) @@ -79,7 +240,7 @@ class TestUpgrade(test_util.TensorFlowTestCase): "tf.boolean_mask(tensor=a, mask=b, name=c, axis=d)\n") def testLearningRateDecay(self): - for decay in ["tf.train.exponential_decay", "tf.train.piecewise_constant", + for decay in ["tf.train.exponential_decay", "tf.train.polynomial_decay", "tf.train.natural_exp_decay", "tf.train.inverse_time_decay", "tf.train.cosine_decay", "tf.train.cosine_decay_restarts", @@ -87,18 +248,208 @@ class TestUpgrade(test_util.TensorFlowTestCase): "tf.train.noisy_linear_cosine_decay"]: text = "%s(a, b)\n" % decay - _, report, errors, new_text = self._upgrade(text) - self.assertEqual(text, new_text) + _, report, errors, _ = self._upgrade(text) self.assertEqual(errors, ["test.py:1: %s requires manual check." % decay]) self.assertIn("%s has been changed" % decay, report) - def testEstimatorLossReductionChangege(self): - text = "tf.estimator.LinearClassifier(a, b)\n" - _, report, errors, new_text = self._upgrade(text) + def testPiecewiseDecay(self): + text = "tf.train.piecewise_constant_decay(a, b)\n" + _, report, errors, _ = self._upgrade(text) + self.assertEqual( + errors, + ["test.py:1: tf.train.piecewise_constant_decay requires manual check."]) + self.assertIn("tf.train.piecewise_constant_decay has been changed", report) + + def testEstimatorLossReductionChange(self): + classes = [ + "LinearClassifier", "LinearRegressor", "DNNLinearCombinedClassifier", + "DNNLinearCombinedRegressor", "DNNRegressor", "DNNClassifier", + "BaselineClassifier", "BaselineRegressor" + ] + for c in classes: + ns = "tf.estimator." + c + text = ns + "(a, b)" + _, report, errors, new_text = self._upgrade(text) + self.assertEqual(text, new_text) + self.assertEqual(errors, ["test.py:1: %s requires manual check." % ns]) + self.assertIn("loss_reduction has been changed", report) + + def testDropout(self): + text = "tf.nn.dropout(x, keep_prob, name=\"foo\")\n" + _, unused_report, unused_errors, new_text = self._upgrade(text) + self.assertEqual( + new_text, + "tf.nn.dropout(x, 1 - keep_prob, name=\"foo\")\n", + ) + + text = "tf.nn.dropout(x)\n" + _, unused_report, errors, new_text = self._upgrade(text) + self.assertEqual(new_text, text) + self.assertEqual( + errors, + ["test.py:1: tf.nn.dropout requires manual check."] + ) + + def testCountNonZeroChanges(self): + text = ( + "tf.math.count_nonzero(input_tensor=input, dtype=dtype, name=name, " + "reduction_indices=axis, keep_dims=keepdims)\n" + ) + _, unused_report, unused_errors, new_text = self._upgrade(text) + expected_text = ( + "tf.math.count_nonzero(input=input, dtype=dtype, name=name, " + "axis=axis, keepdims=keepdims)\n" + ) + self.assertEqual(new_text, expected_text) + + def testRandomMultinomialToRandomCategorical(self): + text = ( + "tf.random.multinomial(logits, samples, seed, name, output_dtype)\n" + ) + _, unused_report, unused_errors, new_text = self._upgrade(text) + expected_text = ( + "tf.random.categorical(logits=logits, num_samples=samples, seed=seed, " + "name=name, dtype=output_dtype)\n" + ) + self.assertEqual(new_text, expected_text) + + text = ( + "tf.multinomial(logits, samples, seed, name, output_dtype)\n" + ) + _, unused_report, unused_errors, new_text = self._upgrade(text) + expected_text = ( + "tf.random.categorical(logits=logits, num_samples=samples, seed=seed, " + "name=name, dtype=output_dtype)\n" + ) + self.assertEqual(new_text, expected_text) + + def testConvolutionOpUpdate(self): + text = ( + "tf.nn.convolution(input, filter, padding, strides, dilation_rate, " + "name, data_format)" + ) + _, unused_report, unused_errors, new_text = self._upgrade(text) + expected_text = ( + "tf.nn.convolution(input=input, filters=filter, padding=padding, " + "strides=strides, dilations=dilation_rate, name=name, " + "data_format=data_format)" + ) + self.assertEqual(new_text, expected_text) + + def testColocateGradientsWithOps(self): + text = "tf.gradients(a, foo=False)\n" + _, unused_report, errors, new_text = self._upgrade(text) self.assertEqual(text, new_text) - self.assertEqual(errors, ["test.py:1: %s requires manual check." - % "tf.estimator.LinearClassifier"]) - self.assertIn("loss_reduction has been changed", report) + self.assertEqual(errors, []) + + text = "tf.gradients(a, colocate_gradients_with_ops=False)\n" + _, unused_report, errors, new_text = self._upgrade(text) + self.assertEqual(text, new_text) + self.assertEqual(errors, ["test.py:1: tf.gradients requires manual check."]) + + text = "optimizer.minimize(a, foo=False)\n" + _, unused_report, errors, new_text = self._upgrade(text) + self.assertEqual(text, new_text) + self.assertEqual(errors, []) + + text = "optimizer.minimize(a, colocate_gradients_with_ops=False)\n" + _, unused_report, errors, new_text = self._upgrade(text) + self.assertEqual(text, new_text) + self.assertEqual(errors, + ["test.py:1: Optimizer.minimize requires manual check."]) + + text = "optimizer.compute_gradients(a, foo=False)\n" + _, unused_report, errors, new_text = self._upgrade(text) + self.assertEqual(text, new_text) + self.assertEqual(errors, []) + + text = "optimizer.compute_gradients(a, colocate_gradients_with_ops=False)\n" + _, unused_report, errors, new_text = self._upgrade(text) + self.assertEqual(text, new_text) + self.assertEqual(errors, + ["test.py:1: Optimizer.compute_gradients " + "requires manual check."]) + + def testExportSavedModelRename(self): + text = "self.est.export_savedmodel(path)" + _, report, unused_errors, unused_new_text = self._upgrade(text) + self.assertIn( + "rename the method export_savedmodel() to export_saved_model()", + report) + + def testArgmin(self): + text = "tf.argmin(input, name=n, dimension=1, output_type=type)" + expected_text = "tf.argmin(input=input, name=n, axis=1, output_type=type)" + _, unused_report, unused_errors, new_text = self._upgrade(text) + self.assertEqual(new_text, expected_text) + + text = "tf.argmin(input, 0)" + expected_text = "tf.argmin(input=input, axis=0)" + _, unused_report, unused_errors, new_text = self._upgrade(text) + self.assertEqual(new_text, expected_text) + + def testArgmax(self): + text = "tf.argmax(input, name=n, dimension=1, output_type=type)" + expected_text = "tf.argmax(input=input, name=n, axis=1, output_type=type)" + _, unused_report, unused_errors, new_text = self._upgrade(text) + self.assertEqual(new_text, expected_text) + + text = "tf.argmax(input, 0)" + expected_text = "tf.argmax(input=input, axis=0)" + _, unused_report, unused_errors, new_text = self._upgrade(text) + self.assertEqual(new_text, expected_text) + + def testBatchToSpace(self): + text = "tf.batch_to_space_nd(input, block_shape, crops, name)" + expected_text = "tf.batch_to_space(input, block_shape, crops, name)" + _, unused_report, unused_errors, new_text = self._upgrade(text) + self.assertEqual(new_text, expected_text) + + text = "tf.batch_to_space(input, crops, block_size, name)" + expected_text = ( + "tf.batch_to_space(input=input, crops=crops, block_shape=block_size, " + "name=name)") + _, unused_report, unused_errors, new_text = self._upgrade(text) + self.assertEqual(new_text, expected_text) + + text = "tf.manip.batch_to_space_nd(input, block_shape, crops, name)" + expected_text = "tf.batch_to_space(input, block_shape, crops, name)" + _, unused_report, unused_errors, new_text = self._upgrade(text) + self.assertEqual(new_text, expected_text) + + def testExtractImagePatches(self): + text = ( + "tf.extract_image_patches(images, ksizes=ksizes, strides=strides," + "rates=rates, padding=padding, name=name)") + expected_text = ( + "tf.image.extract_image_patches(images, sizes=ksizes, strides=strides," + "rates=rates, padding=padding, name=name)") + _, unused_report, unused_errors, new_text = self._upgrade(text) + self.assertEqual(new_text, expected_text) + + def testStatelessMultinomial(self): + text = ( + "tf.random.stateless_multinomial(logits, num_samples, seed, " + "output_dtype=dtype, name=name)") + expected_text = ( + "tf.random.stateless_categorical(logits, num_samples, seed, " + "dtype=dtype, name=name)") + _, unused_report, unused_errors, new_text = self._upgrade(text) + self.assertEqual(new_text, expected_text) + + def testSoftMaxCrossEntropyWithLogitsV2(self): + text = "tf.nn.softmax_cross_entropy_with_logits_v2(labels, logits, dim=2)" + expected_text = ( + "tf.nn.softmax_cross_entropy_with_logits(labels, logits, axis=2)") + _, unused_report, unused_errors, new_text = self._upgrade(text) + self.assertEqual(new_text, expected_text) + + def testSparseMatmul(self): + text = ("tf.sparse_matmul(a, b, c, d, e, f, g)\n") + expected_text = ("tf.linalg.matmul(a=a, b=b, transpose_a=c, transpose_b=d, " + "a_is_sparse=e, b_is_sparse=f, name=g)\n") + _, unused_report, unused_errors, new_text = self._upgrade(text) + self.assertEqual(new_text, expected_text) class TestUpgradeFiles(test_util.TensorFlowTestCase): diff --git a/tensorflow/tools/compatibility/update/BUILD b/tensorflow/tools/compatibility/update/BUILD index 0ee45508155..b9725a74ee5 100644 --- a/tensorflow/tools/compatibility/update/BUILD +++ b/tensorflow/tools/compatibility/update/BUILD @@ -12,5 +12,6 @@ py_binary( "//tensorflow/python:no_contrib", "//tensorflow/tools/common:public_api", "//tensorflow/tools/common:traverse", + "//tensorflow/tools/compatibility:tf_upgrade_v2_lib", ], ) diff --git a/tensorflow/tools/compatibility/update/generate_v2_renames_map.py b/tensorflow/tools/compatibility/update/generate_v2_renames_map.py index 43aa8e057e1..19ad6c3a2a5 100644 --- a/tensorflow/tools/compatibility/update/generate_v2_renames_map.py +++ b/tensorflow/tools/compatibility/update/generate_v2_renames_map.py @@ -32,6 +32,7 @@ from tensorflow.python.util import tf_decorator from tensorflow.python.util import tf_export from tensorflow.tools.common import public_api from tensorflow.tools.common import traverse +from tensorflow.tools.compatibility import tf_upgrade_v2 _OUTPUT_FILE_PATH = 'third_party/tensorflow/tools/compatibility/renames_v2.py' @@ -71,6 +72,50 @@ _TENSORFLOW_CONSTANTS_ATTR_V1 = ( _TENSORFLOW_CONSTANTS_ATTR = ( tf_export.API_ATTRS[tf_export.TENSORFLOW_API_NAME].constants) +_ESTIMATOR_API_ATTR_V1 = ( + tf_export.API_ATTRS_V1[tf_export.ESTIMATOR_API_NAME].names) +_ESTIMATOR_API_ATTR = tf_export.API_ATTRS[tf_export.ESTIMATOR_API_NAME].names +_ESTIMATOR_CONSTANTS_ATTR_V1 = ( + tf_export.API_ATTRS_V1[tf_export.ESTIMATOR_API_NAME].constants) +_ESTIMATOR_CONSTANTS_ATTR = ( + tf_export.API_ATTRS[tf_export.ESTIMATOR_API_NAME].constants) + + +def get_v1_names(symbol): + names_v1 = [] + if hasattr(symbol, _TENSORFLOW_API_ATTR_V1): + names_v1.extend(getattr(symbol, _TENSORFLOW_API_ATTR_V1)) + if hasattr(symbol, _ESTIMATOR_API_ATTR_V1): + names_v1.extend(getattr(symbol, _ESTIMATOR_API_ATTR_V1)) + return names_v1 + + +def get_v2_names(symbol): + names_v2 = [] + if hasattr(symbol, _TENSORFLOW_API_ATTR): + names_v2.extend(getattr(symbol, _TENSORFLOW_API_ATTR)) + if hasattr(symbol, _ESTIMATOR_API_ATTR): + names_v2.extend(getattr(symbol, _ESTIMATOR_API_ATTR)) + return list(names_v2) + + +def get_v1_constants(module): + constants_v1 = [] + if hasattr(module, _TENSORFLOW_CONSTANTS_ATTR_V1): + constants_v1.extend(getattr(module, _TENSORFLOW_CONSTANTS_ATTR_V1)) + if hasattr(module, _ESTIMATOR_CONSTANTS_ATTR_V1): + constants_v1.extend(getattr(module, _ESTIMATOR_CONSTANTS_ATTR_V1)) + return constants_v1 + + +def get_v2_constants(module): + constants_v2 = [] + if hasattr(module, _TENSORFLOW_CONSTANTS_ATTR): + constants_v2.extend(getattr(module, _TENSORFLOW_CONSTANTS_ATTR)) + if hasattr(module, _ESTIMATOR_CONSTANTS_ATTR): + constants_v2.extend(getattr(module, _ESTIMATOR_CONSTANTS_ATTR)) + return constants_v2 + def get_canonical_name(v2_names, v1_name): if v2_names: @@ -78,18 +123,34 @@ def get_canonical_name(v2_names, v1_name): return 'compat.v1.%s' % v1_name +def get_all_v2_names(): + """Get a set of function/class names available in TensorFlow 2.0.""" + v2_names = set() # All op names in TensorFlow 2.0 + + def visit(unused_path, unused_parent, children): + """Visitor that collects TF 2.0 names.""" + for child in children: + _, attr = tf_decorator.unwrap(child[1]) + api_names_v2 = get_v2_names(attr) + for name in api_names_v2: + v2_names.add(name) + + visitor = public_api.PublicAPIVisitor(visit) + visitor.do_not_descend_map['tf'].append('contrib') + traverse.traverse(tf.compat.v2, visitor) + return v2_names + + def collect_constant_renames(): """Looks for constants that need to be renamed in TF 2.0. Returns: - List of tuples of the form (current name, new name). + Set of tuples of the form (current name, new name). """ renames = set() for module in sys.modules.values(): - if not hasattr(module, _TENSORFLOW_CONSTANTS_ATTR_V1): - continue - constants_v1_list = getattr(module, _TENSORFLOW_CONSTANTS_ATTR_V1) - constants_v2_list = getattr(module, _TENSORFLOW_CONSTANTS_ATTR) + constants_v1_list = get_v1_constants(module) + constants_v2_list = get_v2_constants(module) # _tf_api_constants attribute contains a list of tuples: # (api_names_list, constant_name) @@ -115,26 +176,21 @@ def collect_function_renames(): """Looks for functions/classes that need to be renamed in TF 2.0. Returns: - List of tuples of the form (current name, new name). + Set of tuples of the form (current name, new name). """ # Set of rename lines to write to output file in the form: # 'tf.deprecated_name': 'tf.canonical_name' renames = set() - v2_names = set() # All op names in TensorFlow 2.0 def visit(unused_path, unused_parent, children): """Visitor that collects rename strings to add to rename_line_set.""" for child in children: _, attr = tf_decorator.unwrap(child[1]) - if not hasattr(attr, '__dict__'): - continue - api_names_v1 = attr.__dict__.get(_TENSORFLOW_API_ATTR_V1, []) - api_names_v2 = attr.__dict__.get(_TENSORFLOW_API_ATTR, []) + api_names_v1 = get_v1_names(attr) + api_names_v2 = get_v2_names(attr) deprecated_api_names = set(api_names_v1) - set(api_names_v2) for name in deprecated_api_names: renames.add((name, get_canonical_name(api_names_v2, name))) - for name in api_names_v2: - v2_names.add(name) visitor = public_api.PublicAPIVisitor(visit) visitor.do_not_descend_map['tf'].append('contrib') @@ -144,8 +200,9 @@ def collect_function_renames(): # It is possible that a different function is exported with the # same name. For e.g. when creating a different function to # rename arguments. Exclude it from renames in this case. - renames = {name: new_name for name, new_name in renames.items() - if name not in v2_names} + v2_names = get_all_v2_names() + renames = set((name, new_name) for name, new_name in renames + if name not in v2_names) return renames @@ -163,12 +220,15 @@ def update_renames_v2(output_file_path): function_renames = collect_function_renames() constant_renames = collect_constant_renames() all_renames = function_renames.union(constant_renames) + manual_renames = set( + tf_upgrade_v2.TFAPIChangeSpec().manual_symbol_renames.keys()) # List of rename lines to write to output file in the form: # 'tf.deprecated_name': 'tf.canonical_name' rename_lines = [ get_rename_line(name, canonical_name) - for name, canonical_name in all_renames] + for name, canonical_name in all_renames + if 'tf.' + name not in manual_renames] renames_file_text = '%srenames = {\n%s\n}\n' % ( _FILE_HEADER, ',\n'.join(sorted(rename_lines))) file_io.write_string_to_file(output_file_path, renames_file_text) diff --git a/tensorflow/tools/docker/Dockerfile b/tensorflow/tools/docker/Dockerfile index 205128ad58a..6676de02a41 100644 --- a/tensorflow/tools/docker/Dockerfile +++ b/tensorflow/tools/docker/Dockerfile @@ -1,4 +1,4 @@ -FROM ubuntu:16.04 +FROM ubuntu:18.04 LABEL maintainer="Craig Citro " @@ -8,7 +8,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ curl \ libfreetype6-dev \ libhdf5-serial-dev \ - libpng12-dev \ + libpng-dev \ libzmq3-dev \ pkg-config \ python \ diff --git a/tensorflow/tools/docker/Dockerfile.devel b/tensorflow/tools/docker/Dockerfile.devel index a3893a2713d..c256dd364ef 100644 --- a/tensorflow/tools/docker/Dockerfile.devel +++ b/tensorflow/tools/docker/Dockerfile.devel @@ -1,4 +1,4 @@ -FROM ubuntu:16.04 +FROM ubuntu:18.04 LABEL maintainer="Craig Citro " @@ -9,7 +9,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ libcurl3-dev \ libfreetype6-dev \ libhdf5-serial-dev \ - libpng12-dev \ + libpng-dev \ libzmq3-dev \ pkg-config \ python-dev \ diff --git a/tensorflow/tools/docker/Dockerfile.devel-mkl b/tensorflow/tools/docker/Dockerfile.devel-mkl index bd2883ddba0..2341c0e8ccf 100755 --- a/tensorflow/tools/docker/Dockerfile.devel-mkl +++ b/tensorflow/tools/docker/Dockerfile.devel-mkl @@ -1,4 +1,4 @@ -FROM ubuntu:16.04 +FROM ubuntu:18.04 LABEL maintainer="Clayne Robison " @@ -16,7 +16,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ libcurl3-dev \ libfreetype6-dev \ libhdf5-serial-dev \ - libpng12-dev \ + libpng-dev \ libzmq3-dev \ libssl-dev \ pkg-config \ diff --git a/tensorflow/tools/docker/Dockerfile.devel-mkl-horovod b/tensorflow/tools/docker/Dockerfile.devel-mkl-horovod index df084e029c8..5e24617b219 100755 --- a/tensorflow/tools/docker/Dockerfile.devel-mkl-horovod +++ b/tensorflow/tools/docker/Dockerfile.devel-mkl-horovod @@ -1,4 +1,4 @@ -FROM ubuntu:16.04 +FROM ubuntu:18.04 LABEL maintainer="Cong Xu " @@ -16,7 +16,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ libcurl3-dev \ libfreetype6-dev \ libhdf5-serial-dev \ - libpng12-dev \ + libpng-dev \ libzmq3-dev \ pkg-config \ python-dev \ diff --git a/tensorflow/tools/docker/Dockerfile.mkl b/tensorflow/tools/docker/Dockerfile.mkl index ac41cffe4bc..dad27697fa1 100755 --- a/tensorflow/tools/docker/Dockerfile.mkl +++ b/tensorflow/tools/docker/Dockerfile.mkl @@ -1,4 +1,4 @@ -FROM ubuntu:16.04 +FROM ubuntu:18.04 LABEL maintainer="Clayne Robison " @@ -17,7 +17,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ curl \ libfreetype6-dev \ libhdf5-serial-dev \ - libpng12-dev \ + libpng-dev \ libzmq3-dev \ pkg-config \ ${PYTHON} \ diff --git a/tensorflow/tools/docker/Dockerfile.mkl-horovod b/tensorflow/tools/docker/Dockerfile.mkl-horovod index 0432cd5e80c..19dc45c62cb 100755 --- a/tensorflow/tools/docker/Dockerfile.mkl-horovod +++ b/tensorflow/tools/docker/Dockerfile.mkl-horovod @@ -1,4 +1,4 @@ -FROM ubuntu:16.04 +FROM ubuntu:18.04 LABEL maintainer="Cong Xu " @@ -17,7 +17,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ curl \ libfreetype6-dev \ libhdf5-serial-dev \ - libpng12-dev \ + libpng-dev \ libzmq3-dev \ pkg-config \ python \ diff --git a/tensorflow/tools/dockerfiles/.gitignore b/tensorflow/tools/dockerfiles/.gitignore new file mode 100644 index 00000000000..d7efa472a92 --- /dev/null +++ b/tensorflow/tools/dockerfiles/.gitignore @@ -0,0 +1 @@ +dockerfiles/*.temp.Dockerfile diff --git a/tensorflow/tools/dockerfiles/README.md b/tensorflow/tools/dockerfiles/README.md index 7c8ca1d1c7a..2ac68666d08 100644 --- a/tensorflow/tools/dockerfiles/README.md +++ b/tensorflow/tools/dockerfiles/README.md @@ -1,8 +1,12 @@ # TensorFlow Dockerfiles -This directory houses TensorFlow's Dockerfiles. **DO NOT EDIT THE DOCKERFILES -MANUALLY!** They are maintained by `assembler.py`, which builds Dockerfiles from -the files in `partials/` and the rules in `spec.yml`. See [the Contributing +This directory houses TensorFlow's Dockerfiles and the infrastructure used to +create and deploy them to [Docker +Hub](https://hub.docker.com/r/tensorflow/tensorflow). + +**DO NOT EDIT THE DOCKERFILES/ DIRECTORY MANUALLY!** The files within are +maintained by `assembler.py`, which builds Dockerfiles from the files in +`partials/` and the rules in `spec.yml`. See [the Contributing section](#contributing) for more information. These Dockerfiles are planned to replace the Dockerfiles used to generate @@ -20,10 +24,10 @@ $ docker build -f ./dockerfiles/cpu.Dockerfile -t tf . Each Dockerfile has its own set of available `--build-arg`s which are documented in the Dockerfile itself. -## Running +## Running Locally Built Images After building the image with the tag `tf` (for example), use `docker run` to -run the images. Examples are below. +run the images. Note for new Docker users: the `-v` and `-u` flags share directories between the Docker container and your machine, and very important. Without @@ -42,8 +46,10 @@ $ docker run -u $(id -u):$(id -g) -v $(pwd):/my-devel -it tf # GPU-based images (set up nvidia-docker2 first) $ docker run --runtime=nvidia -u $(id -u):$(id -g) -v $(pwd):/my-devel -it tf -# Images with Jupyter run on port 8888, and needs a volume for notebooks -$ docker run --user $(id -u):$(id -g) -p 8888:8888 -v $(pwd):/notebooks -it tf +# Images with Jupyter run on port 8888 and need a volume for your notebooks +# You can change $(PWD) to the full path to a directory if your notebooks +# live outside the current directory. +$ docker run --user $(id -u):$(id -g) -p 8888:8888 -v $(PWD):/tf/notebooks -it tf ``` These images do not come with the TensorFlow source code -- but the development @@ -60,11 +66,32 @@ You can use the `Dockerfile` in this directory to build an editing environment that has all of the Python dependencies you'll need: ```bash -$ docker build -t tf-assembler -f assembler.Dockerfile . +# Build the tools-helper image so you can run the assembler +$ docker build -t tf-tools -f tools.Dockerfile . # Set --user to set correct permissions on generated files -$ docker run --user $(id -u):$(id -g) -it -v $(pwd):/tf tf-assembler bash +$ docker run --user $(id -u):$(id -g) -it -v $(pwd):/tf tf-tools bash -# In the container... -/tf $ python3 ./assembler.py -o dockerfiles -s spec.yml +# Next you can make a handy alias depending on what you're doing. When building +# Docker images, you need to run as root with docker.sock mounted so that the +# container can run Docker commands. When assembling Dockerfiles, though, you'll +# want to run as your user so that new files have the right permissions. + +# If you're BUILDING OR DEPLOYING DOCKER IMAGES, run as root with docker.sock: +$ alias asm_images="docker run --rm -v $(pwd):/tf -v /var/run/docker.sock:/var/run/docker.sock tf-tools python3 assembler.py " + +# If you're REBUILDING OR ADDING DOCKERFILES, remove docker.sock and add -u: +$ alias asm_dockerfiles="docker run --rm -u $(id -u):$(id -g) -v $(pwd):/tf tf-tools python3 assembler.py " + +# Check flags +$ asm_dockerfiles --help + +# Assemble all of the Dockerfiles +$ asm_dockerfiles --release ubuntu-dockerfiles --construct_dockerfiles + +# Build all of the "nightly" images on your local machine: +$ asm_images --release nightly --build_images + +# Build version release for version 99.0, except "gpu" tags: +$ asm_images --release versioned --arg _TAG_PREFIX=99.0 --build_images --exclude_tags_matching '*.gpu.*' ``` diff --git a/tensorflow/tools/dockerfiles/assembler.py b/tensorflow/tools/dockerfiles/assembler.py index 9cdd9bb0cb0..9d8a59aebcf 100644 --- a/tensorflow/tools/dockerfiles/assembler.py +++ b/tensorflow/tools/dockerfiles/assembler.py @@ -11,63 +11,144 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -# ============================================================================== -"""Assemble common TF Dockerfiles from many parts. +# ============================================================================ +"""Multipurpose TensorFlow Docker Helper. -This script constructs TF's Dockerfiles by aggregating partial -Dockerfiles. See README.md for usage examples. +- Assembles Dockerfiles +- Builds images (and optionally runs image tests) +- Pushes images to Docker Hub (provided with credentials) + +Read README.md (in this directory) for instructions! """ from __future__ import absolute_import from __future__ import division from __future__ import print_function +import collections import copy import errno +import itertools +import multiprocessing import os -import os.path import re import shutil -import textwrap +import sys from absl import app from absl import flags import cerberus +import docker import yaml FLAGS = flags.FLAGS +flags.DEFINE_string('hub_username', None, + 'Dockerhub username, only used with --upload_to_hub') + +flags.DEFINE_string( + 'hub_password', None, + ('Dockerhub password, only used with --upload_to_hub. Use from an env param' + 'so your password isn\'t in your history.')) + +flags.DEFINE_integer('hub_timeout', 3600, + 'Abort Hub upload if it takes longer than this.') + +flags.DEFINE_string( + 'repository', 'tensorflow', + 'Tag local images as {repository}:tag (in addition to the ' + 'hub_repository, if uploading to hub)') + +flags.DEFINE_string( + 'hub_repository', None, + 'Push tags to this Docker Hub repository, e.g. tensorflow/tensorflow') + flags.DEFINE_boolean( - 'dry_run', False, 'Do not actually generate Dockerfiles', short_name='n') + 'upload_to_hub', + False, + ('Push built images to Docker Hub (you must also provide --hub_username, ' + '--hub_password, and --hub_repository)'), + short_name='u', +) + +flags.DEFINE_boolean( + 'construct_dockerfiles', False, 'Do not build images', short_name='d') + +flags.DEFINE_boolean( + 'keep_temp_dockerfiles', + False, + 'Retain .temp.Dockerfiles created while building images.', + short_name='k') + +flags.DEFINE_boolean( + 'build_images', False, 'Do not build images', short_name='b') flags.DEFINE_string( - 'spec_file', - './spec.yml', - 'Path to a YAML specification file', - short_name='s') + 'run_tests_path', None, + ('Execute test scripts on generated Dockerfiles before pushing them. ' + 'Flag value must be a full path to the "tests" directory, which is usually' + ' $(realpath ./tests). A failed tests counts the same as a failed build.')) + +flags.DEFINE_boolean( + 'stop_on_failure', False, + ('Stop processing tags if any one build fails. If False or not specified, ' + 'failures are reported but do not affect the other images.')) + +flags.DEFINE_boolean( + 'dry_run', + False, + 'Do not build or deploy anything at all.', + short_name='n', +) flags.DEFINE_string( - 'output_dir', - './dockerfiles', ('Path to an output directory for Dockerfiles. ' - 'Will be created if it doesn\'t exist.'), + 'exclude_tags_matching', + None, + ('Regular expression that skips processing on any tag it matches. Must ' + 'match entire string, e.g. ".*gpu.*" ignores all GPU tags.'), + short_name='x') + +flags.DEFINE_string( + 'only_tags_matching', + None, + ('Regular expression that skips processing on any tag it does not match. ' + 'Must match entire string, e.g. ".*gpu.*" includes only GPU tags.'), + short_name='i') + +flags.DEFINE_string( + 'dockerfile_dir', + './dockerfiles', 'Path to an output directory for Dockerfiles.' + ' Will be created if it doesn\'t exist.' + ' Existing files in this directory will be deleted when new Dockerfiles' + ' are made.', short_name='o') flags.DEFINE_string( 'partial_dir', './partials', - 'Path to a directory containing foo.partial.Dockerfile partial files.', + 'Path to a directory containing foo.partial.Dockerfile partial files.' + ' can have subdirectories, e.g. "bar/baz.partial.Dockerfile".', short_name='p') -flags.DEFINE_boolean( - 'quiet_dry_run', - True, - 'Do not print contents of dry run Dockerfiles.', - short_name='q') +flags.DEFINE_multi_string( + 'release', [], + 'Set of releases to build and tag. Defaults to every release type.', + short_name='r') -flags.DEFINE_boolean( - 'validate', True, 'Validate generated Dockerfiles', short_name='c') +flags.DEFINE_multi_string( + 'arg', [], + ('Extra build arguments. These are used for expanding tag names if needed ' + '(e.g. --arg _TAG_PREFIX=foo) and for using as build arguments (unused ' + 'args will print a warning).'), + short_name='a') -# Schema to verify the contents of spec.yml with Cerberus. +flags.DEFINE_string( + 'spec_file', + './spec.yml', + 'Path to the YAML specification file', + short_name='s') + +# Schema to verify the contents of tag-spec.yml with Cerberus. # Must be converted to a dict from yaml to work. # Note: can add python references with e.g. # !!python/name:builtins.str @@ -76,79 +157,76 @@ SCHEMA_TEXT = """ header: type: string -partials: +slice_sets: + type: dict + keyschema: + type: string + valueschema: + type: list + schema: + type: dict + schema: + add_to_name: + type: string + dockerfile_exclusive_name: + type: string + partials: + type: list + schema: + type: string + ispartial: true + test_runtime: + type: string + required: false + tests: + type: list + default: [] + schema: + type: string + args: + type: list + default: [] + schema: + type: string + isfullarg: true + +releases: type: dict keyschema: type: string valueschema: type: dict schema: - desc: - type: string - args: - type: dict - keyschema: - type: string - valueschema: - anyof: - - type: [ boolean, number, string ] - - type: dict - schema: - default: - type: [ boolean, number, string ] - desc: - type: string - options: - type: list - schema: - type: string - -images: - keyschema: - type: string - valueschema: - type: dict - schema: - desc: - type: string - arg-defaults: - type: list - schema: - anyof: - - type: dict - keyschema: - type: string - arg_in_use: true - valueschema: - type: string - - type: string - isimage: true - create-dockerfile: + is_dockerfiles: type: boolean - partials: + required: false + default: false + upload_images: + type: boolean + required: false + default: true + tag_specs: type: list + required: true schema: - anyof: - - type: dict - keyschema: - type: string - regex: image - valueschema: - type: string - isimage: true - - type: string - ispartial: true + type: string """ -class TfDockerValidator(cerberus.Validator): - """Custom Cerberus validator for TF dockerfile spec. +class TfDockerTagValidator(cerberus.Validator): + """Custom Cerberus validator for TF tag spec. Note: Each _validate_foo function's docstring must end with a segment describing its own validation schema, e.g. "The rule's arguments are...". If you add a new validator, you can copy/paste that section. """ + def __init__(self, *args, **kwargs): + # See http://docs.python-cerberus.org/en/stable/customize.html + if 'partials' in kwargs: + self.partials = kwargs['partials'] + super(cerberus.Validator, self).__init__(*args, **kwargs) + def _validate_ispartial(self, ispartial, field, value): """Validate that a partial references an existing partial spec. @@ -156,146 +234,190 @@ class TfDockerValidator(cerberus.Validator): ispartial: Value of the rule, a bool field: The field being validated value: The field's value - The rule's arguments are validated against this schema: {'type': 'boolean'} """ - if ispartial and value not in self.root_document.get('partials', dict()): - self._error(field, '{} is not an existing partial.'.format(value)) + if ispartial and value not in self.partials: + self._error(field, + '{} is not present in the partials directory.'.format(value)) - def _validate_isimage(self, isimage, field, value): - """Validate that an image references an existing partial spec. + def _validate_isfullarg(self, isfullarg, field, value): + """Validate that a string is either a FULL=arg or NOT. Args: - isimage: Value of the rule, a bool + isfullarg: Value of the rule, a bool field: The field being validated value: The field's value - The rule's arguments are validated against this schema: {'type': 'boolean'} """ - if isimage and value not in self.root_document.get('images', dict()): - self._error(field, '{} is not an existing image.'.format(value)) - - def _validate_arg_in_use(self, arg_in_use, field, value): - """Validate that an arg references an existing partial spec's args. - - Args: - arg_in_use: Value of the rule, a bool - field: The field being validated - value: The field's value - - The rule's arguments are validated against this schema: - {'type': 'boolean'} - """ - if arg_in_use: - for partial in self.root_document.get('partials', dict()).values(): - if value in partial.get('args', tuple()): - return - - self._error(field, '{} is not an arg used in any partial.'.format(value)) + if isfullarg and '=' not in value: + self._error(field, '{} should be of the form ARG=VALUE.'.format(value)) + if not isfullarg and '=' in value: + self._error(field, '{} should be of the form ARG (no =).'.format(value)) -def build_partial_description(partial_spec): - """Create the documentation lines for a specific partial. +def eprint(*args, **kwargs): + print(*args, file=sys.stderr, flush=True, **kwargs) - Generates something like this: - # This is the partial's description, from spec.yml. - # --build-arg ARG_NAME=argdefault - # this is one of the args. - # --build-arg ANOTHER_ARG=(some|choices) - # another arg. +def aggregate_all_slice_combinations(spec, slice_set_names): + """Figure out all of the possible slice groupings for a tag spec.""" + slice_sets = copy.deepcopy(spec['slice_sets']) + + for name in slice_set_names: + for slice_set in slice_sets[name]: + slice_set['set_name'] = name + + slices_grouped_but_not_keyed = [slice_sets[name] for name in slice_set_names] + all_slice_combos = list(itertools.product(*slices_grouped_but_not_keyed)) + return all_slice_combos + + +def build_name_from_slices(format_string, slices, args, is_dockerfile=False): + """Build the tag name (cpu-devel...) from a list of slices.""" + name_formatter = copy.deepcopy(args) + name_formatter.update({s['set_name']: s['add_to_name'] for s in slices}) + name_formatter.update({ + s['set_name']: s['dockerfile_exclusive_name'] + for s in slices + if is_dockerfile and 'dockerfile_exclusive_name' in s + }) + name = format_string.format(**name_formatter) + return name + + +def update_args_dict(args_dict, updater): + """Update a dict of arg values with more values from a list or dict.""" + if isinstance(updater, list): + for arg in updater: + key, sep, value = arg.partition('=') + if sep == '=': + args_dict[key] = value + if isinstance(updater, dict): + for key, value in updater.items(): + args_dict[key] = value + return args_dict + + +def get_slice_sets_and_required_args(slice_sets, tag_spec): + """Extract used-slice-sets and required CLI arguments from a spec string. + + For example, {FOO}{bar}{bat} finds FOO, bar, and bat. Assuming bar and bat + are both named slice sets, FOO must be specified on the command line. Args: - partial_spec: A dict representing one of the partials from spec.yml. Doesn't - include the name of the partial; is a dict like { desc: ..., args: ... }. + slice_sets: Dict of named slice sets + tag_spec: The tag spec string, e.g. {_FOO}{blep} Returns: - A commented string describing this partial. + (used_slice_sets, required_args), a tuple of lists """ + required_args = [] + used_slice_sets = [] - # Start from linewrapped desc field - lines = [] - wrapper = textwrap.TextWrapper( - initial_indent='# ', subsequent_indent='# ', width=80) - description = wrapper.fill(partial_spec.get('desc', '( no comments )')) - lines.extend(['#', description]) - - # Document each arg - for arg, arg_data in partial_spec.get('args', dict()).items(): - # Wrap arg description with comment lines - desc = arg_data.get('desc', '( no description )') - desc = textwrap.fill( - desc, - initial_indent='# ', - subsequent_indent='# ', - width=80, - drop_whitespace=False) - - # Document (each|option|like|this) - if 'options' in arg_data: - arg_options = ' ({})'.format('|'.join(arg_data['options'])) + extract_bracketed_words = re.compile(r'\{([^}]+)\}') + possible_args_or_slice_set_names = extract_bracketed_words.findall(tag_spec) + for name in possible_args_or_slice_set_names: + if name in slice_sets: + used_slice_sets.append(name) else: - arg_options = '' + required_args.append(name) - # Add usage sample - arg_use = '# --build-arg {}={}{}'.format(arg, - arg_data.get('default', '(unset)'), - arg_options) - lines.extend([arg_use, desc]) - - return '\n'.join(lines) + return (used_slice_sets, required_args) -def construct_contents(partial_specs, image_spec): - """Assemble the dockerfile contents for an image spec. +def gather_tag_args(slices, cli_input_args, required_args): + """Build a dictionary of all the CLI and slice-specified args for a tag.""" + args = dict() - It assembles a concrete list of partial references into a single, large - string. - Also expands argument defaults, so that the resulting Dockerfile doesn't have - to be configured with --build-arg=... every time. That is, any ARG directive - will be updated with a new default value. + for s in slices: + args = update_args_dict(args, s['args']) + + args = update_args_dict(args, cli_input_args) + for arg in required_args: + if arg not in args: + eprint(('> Error: {} is not a valid slice_set, and also isn\'t an arg ' + 'provided on the command line. If it is an arg, please specify ' + 'it with --arg. If not, check the slice_sets list.'.format(arg))) + exit(1) + + return args + + +def gather_slice_list_items(slices, key): + """For a list of slices, get the flattened list of all of a certain key.""" + return list(itertools.chain(*[s[key] for s in slices if key in s])) + + +def find_first_slice_value(slices, key): + """For a list of slices, get the first value for a certain key.""" + for s in slices: + if key in s: + return s[key] + + +def assemble_tags(spec, cli_args, enabled_releases, all_partials): + """Gather all the tags based on our spec. Args: - partial_specs: The dict from spec.yml["partials"]. - image_spec: One of the dict values from spec.yml["images"]. + spec: Nested dict containing full Tag spec + cli_args: List of ARG=foo arguments to pass along to Docker build + enabled_releases: List of releases to parse. Empty list = all + all_partials: Dict of every partial, for reference Returns: - A string containing a valid Dockerfile based on the partials listed in - image_spec. + Dict of tags and how to build them """ - processed_partial_strings = [] - for partial_name in image_spec['partials']: - # Apply image arg-defaults to existing arg defaults - partial_spec = copy.deepcopy(partial_specs[partial_name]) - args = partial_spec.get('args', dict()) - for k_v in image_spec.get('arg-defaults', []): - arg, value = list(k_v.items())[0] - if arg in args: - args[arg]['default'] = value + tag_data = collections.defaultdict(list) - # Read partial file contents - filename = partial_spec.get('file', partial_name) - partial_path = os.path.join(FLAGS.partial_dir, - '{}.partial.Dockerfile'.format(filename)) - with open(partial_path, 'r') as f_partial: - partial_contents = f_partial.read() + for name, release in spec['releases'].items(): + for tag_spec in release['tag_specs']: + if enabled_releases and name not in enabled_releases: + eprint('> Skipping release {}'.format(name)) + continue - # Replace ARG FOO=BAR with ARG FOO=[new-default] - for arg, arg_data in args.items(): - if 'default' in arg_data and arg_data['default']: - default = '={}'.format(arg_data['default']) - else: - default = '' - partial_contents = re.sub(r'ARG {}.*'.format(arg), 'ARG {}{}'.format( - arg, default), partial_contents) + used_slice_sets, required_cli_args = get_slice_sets_and_required_args( + spec['slice_sets'], tag_spec) - # Store updated partial contents - processed_partial_strings.append(partial_contents) + slice_combos = aggregate_all_slice_combinations(spec, used_slice_sets) + for slices in slice_combos: - # Join everything together - return '\n'.join(processed_partial_strings) + tag_args = gather_tag_args(slices, cli_args, required_cli_args) + tag_name = build_name_from_slices(tag_spec, slices, tag_args, + release['is_dockerfiles']) + used_partials = gather_slice_list_items(slices, 'partials') + used_tests = gather_slice_list_items(slices, 'tests') + test_runtime = find_first_slice_value(slices, 'test_runtime') + dockerfile_contents = merge_partials(spec['header'], used_partials, + all_partials) + + tag_data[tag_name].append({ + 'release': name, + 'tag_spec': tag_spec, + 'is_dockerfiles': release['is_dockerfiles'], + 'upload_images': release['upload_images'], + 'cli_args': tag_args, + 'partials': used_partials, + 'tests': used_tests, + 'test_runtime': test_runtime, + 'dockerfile_contents': dockerfile_contents, + }) + + return tag_data + + +def merge_partials(header, used_partials, all_partials): + """Merge all partial contents with their header.""" + used_partials = list(used_partials) + return '\n'.join([header] + [all_partials[u] for u in used_partials]) + + +def upload_in_background(hub_repository, dock, image, tag): + """Upload a docker image (to be used by multiprocessing).""" + image.tag(hub_repository, tag=tag) + for line in list(dock.images.push(hub_repository, tag=tag, stream=True)): + print(line) def mkdir_p(path): @@ -307,247 +429,228 @@ def mkdir_p(path): raise -def construct_documentation(header, partial_specs, image_spec): - """Assemble all of the documentation for a single dockerfile. - - Builds explanations of included partials and available build args. +def gather_existing_partials(partial_path): + """Find and read all available partials. Args: - header: The string from spec.yml["header"]; will be commented and wrapped. - partial_specs: The dict from spec.yml["partials"]. - image_spec: The spec for the dockerfile being built. + partial_path (string): read partials from this directory. Returns: - A string containing a commented header that documents the contents of the - dockerfile. - + Dict[string, string] of partial short names (like "ubuntu/python" or + "bazel") to the full contents of that partial. """ - # Comment and wrap header and image description - commented_header = '\n'.join( - [('# ' + l).rstrip() for l in header.splitlines()]) - commented_desc = '\n'.join( - ['# ' + l for l in image_spec.get('desc', '').splitlines()]) - partial_descriptions = [] - - # Build documentation for each partial in the image - for partial in image_spec['partials']: - # Copy partial data for default args unique to this image - partial_spec = copy.deepcopy(partial_specs[partial]) - args = partial_spec.get('args', dict()) - - # Overwrite any existing arg defaults - for k_v in image_spec.get('arg-defaults', []): - arg, value = list(k_v.items())[0] - if arg in args: - args[arg]['default'] = value - - # Build the description from new args - partial_description = build_partial_description(partial_spec) - partial_descriptions.append(partial_description) - - contents = [commented_header, '#', commented_desc] + partial_descriptions - return '\n'.join(contents) + '\n' - - -def normalize_partial_args(partial_specs): - """Normalize the shorthand form of a partial's args specification. - - Turns this: - - partial: - args: - SOME_ARG: arg_value - - Into this: - - partial: - args: - SOME_ARG: - default: arg_value - - Args: - partial_specs: The dict from spec.yml["partials"]. This dict is modified in - place. - - Returns: - The modified contents of partial_specs. - - """ - for _, partial in partial_specs.items(): - args = partial.get('args', dict()) - for arg, value in args.items(): - if not isinstance(value, dict): - new_value = {'default': value} - args[arg] = new_value - - return partial_specs - - -def flatten_args_references(image_specs): - """Resolve all default-args in each image spec to a concrete dict. - - Turns this: - - example-image: - arg-defaults: - - MY_ARG: ARG_VALUE - - another-example: - arg-defaults: - - ANOTHER_ARG: ANOTHER_VALUE - - example_image - - Into this: - - example-image: - arg-defaults: - - MY_ARG: ARG_VALUE - - another-example: - arg-defaults: - - ANOTHER_ARG: ANOTHER_VALUE - - MY_ARG: ARG_VALUE - - Args: - image_specs: A dict of image_spec dicts; should be the contents of the - "images" key in the global spec.yaml. This dict is modified in place and - then returned. - - Returns: - The modified contents of image_specs. - """ - for _, image_spec in image_specs.items(): - too_deep = 0 - while str in map(type, image_spec.get('arg-defaults', [])) and too_deep < 5: - new_args = [] - for arg in image_spec['arg-defaults']: - if isinstance(arg, str): - new_args.extend(image_specs[arg]['arg-defaults']) - else: - new_args.append(arg) - - image_spec['arg-defaults'] = new_args - too_deep += 1 - - return image_specs - - -def flatten_partial_references(image_specs): - """Resolve all partial references in each image spec to a concrete list. - - Turns this: - - example-image: - partials: - - foo - - another-example: - partials: - - bar - - image: example-image - - bat - - Into this: - - example-image: - partials: - - foo - - another-example: - partials: - - bar - - foo - - bat - Args: - image_specs: A dict of image_spec dicts; should be the contents of the - "images" key in the global spec.yaml. This dict is modified in place and - then returned. - - Returns: - The modified contents of image_specs. - """ - for _, image_spec in image_specs.items(): - too_deep = 0 - while dict in map(type, image_spec['partials']) and too_deep < 5: - new_partials = [] - for partial in image_spec['partials']: - if isinstance(partial, str): - new_partials.append(partial) - else: - new_partials.extend(image_specs[partial['image']]['partials']) - - image_spec['partials'] = new_partials - too_deep += 1 - - return image_specs - - -def construct_dockerfiles(tf_spec): - """Generate a mapping of {"cpu": , ...}. - - Args: - tf_spec: The full spec.yml loaded as a python object. - - Returns: - A string:string dict of short names ("cpu-devel") to Dockerfile contents. - """ - names_to_contents = dict() - image_specs = tf_spec['images'] - image_specs = flatten_partial_references(image_specs) - image_specs = flatten_args_references(image_specs) - partial_specs = tf_spec['partials'] - partial_specs = normalize_partial_args(partial_specs) - - for name, image_spec in image_specs.items(): - if not image_spec.get('create-dockerfile', True): - continue - documentation = construct_documentation(tf_spec['header'], partial_specs, - image_spec) - contents = construct_contents(partial_specs, image_spec) - names_to_contents[name] = '\n'.join([documentation, contents]) - - return names_to_contents + partials = dict() + for path, _, files in os.walk(partial_path): + for name in files: + fullpath = os.path.join(path, name) + if '.partial.Dockerfile' not in fullpath: + eprint(('> Probably not a problem: skipping {}, which is not a ' + 'partial.').format(fullpath)) + continue + # partial_dir/foo/bar.partial.Dockerfile -> foo/bar + simple_name = fullpath[len(partial_path) + 1:-len('.partial.dockerfile')] + with open(fullpath, 'r') as f: + partial_contents = f.read() + partials[simple_name] = partial_contents + return partials def main(argv): if len(argv) > 1: - raise app.UsageError('Unexpected command line args found: {}'.format(argv)) + raise app.UsageError('Too many command-line arguments.') + # Read the full spec file, used for everything with open(FLAGS.spec_file, 'r') as spec_file: - tf_spec = yaml.load(spec_file) + tag_spec = yaml.load(spec_file) + + # Get existing partial contents + partials = gather_existing_partials(FLAGS.partial_dir) # Abort if spec.yaml is invalid - if FLAGS.validate: - schema = yaml.load(SCHEMA_TEXT) - v = TfDockerValidator(schema) - if not v.validate(tf_spec): - print('>> ERROR: {} is an invalid spec! The errors are:'.format( - FLAGS.spec_file)) - print(yaml.dump(v.errors, indent=2)) + schema = yaml.load(SCHEMA_TEXT) + v = TfDockerTagValidator(schema, partials=partials) + if not v.validate(tag_spec): + eprint('> Error: {} is an invalid spec! The errors are:'.format( + FLAGS.spec_file)) + eprint(yaml.dump(v.errors, indent=2)) + exit(1) + tag_spec = v.normalized(tag_spec) + + # Assemble tags and images used to build them + all_tags = assemble_tags(tag_spec, FLAGS.arg, FLAGS.release, partials) + + # Empty Dockerfile directory if building new Dockerfiles + if FLAGS.construct_dockerfiles: + eprint('> Emptying Dockerfile dir "{}"'.format(FLAGS.dockerfile_dir)) + shutil.rmtree(FLAGS.dockerfile_dir, ignore_errors=True) + mkdir_p(FLAGS.dockerfile_dir) + + # Set up Docker helper + dock = docker.from_env() + + # Login to Docker if uploading images + if FLAGS.upload_to_hub: + if not FLAGS.hub_username: + eprint('> Error: please set --hub_username when uploading to Dockerhub.') exit(1) - else: - print('>> WARNING: Not validating {}'.format(FLAGS.spec_file)) + if not FLAGS.hub_repository: + eprint( + '> Error: please set --hub_repository when uploading to Dockerhub.') + exit(1) + if not FLAGS.hub_password: + eprint('> Error: please set --hub_password when uploading to Dockerhub.') + exit(1) + dock.login( + username=FLAGS.hub_username, + password=FLAGS.hub_password, + ) - # Generate mapping of { "cpu-devel": "", ... } - names_to_contents = construct_dockerfiles(tf_spec) + # Each tag has a name ('tag') and a definition consisting of the contents + # of its Dockerfile, its build arg list, etc. + failed_tags = [] + for tag, tag_defs in all_tags.items(): + for tag_def in tag_defs: + eprint('> Working on {}'.format(tag)) - # Write each completed Dockerfile - if not FLAGS.dry_run: - print('>> Emptying destination dir "{}"'.format(FLAGS.output_dir)) - shutil.rmtree(FLAGS.output_dir, ignore_errors=True) - mkdir_p(FLAGS.output_dir) - else: - print('>> Skipping creation of {} (dry run)'.format(FLAGS.output_dir)) - for name, contents in names_to_contents.items(): - path = os.path.join(FLAGS.output_dir, name + '.Dockerfile') - if FLAGS.dry_run: - print('>> Skipping writing contents of {} (dry run)'.format(path)) - print(contents) - else: - mkdir_p(FLAGS.output_dir) - print('>> Writing {}'.format(path)) - with open(path, 'w') as f: - f.write(contents) + if FLAGS.exclude_tags_matching and re.match(FLAGS.exclude_tags_matching, + tag): + eprint('>> Excluded due to match against "{}".'.format( + FLAGS.exclude_tags_matching)) + continue + + if FLAGS.only_tags_matching and not re.match(FLAGS.only_tags_matching, + tag): + eprint('>> Excluded due to failure to match against "{}".'.format( + FLAGS.only_tags_matching)) + continue + + # Write releases marked "is_dockerfiles" into the Dockerfile directory + if FLAGS.construct_dockerfiles: + path = os.path.join(FLAGS.dockerfile_dir, tag + '.Dockerfile') + if tag_def['is_dockerfiles']: + eprint('>> Writing {}...'.format(path)) + if not FLAGS.dry_run: + with open(path, 'w') as f: + f.write(tag_def['dockerfile_contents']) + + # Don't build any images for dockerfile-only releases + if not FLAGS.build_images: + continue + + # Generate a temporary Dockerfile to use to build, since docker-py + # needs a filepath relative to the build context (i.e. the current + # directory) + dockerfile = os.path.join(FLAGS.dockerfile_dir, tag + '.temp.Dockerfile') + if not FLAGS.dry_run: + with open(dockerfile, 'w') as f: + f.write(tag_def['dockerfile_contents']) + eprint('>> (Temporary) writing {}...'.format(dockerfile)) + + repo_tag = '{}:{}'.format(FLAGS.repository, tag) + eprint('>> Building {} using build args:'.format(repo_tag)) + for arg, value in tag_def['cli_args'].items(): + eprint('>>> {}={}'.format(arg, value)) + + # Note that we are NOT using cache_from, which appears to limit + # available cache layers to those from explicitly specified layers. Many + # of our layers are similar between local builds, so we want to use the + # implied local build cache. + tag_failed = False + image, logs = None, [] + if not FLAGS.dry_run: + try: + image, logs = dock.images.build( + timeout=FLAGS.hub_timeout, + path='.', + dockerfile=dockerfile, + buildargs=tag_def['cli_args'], + tag=repo_tag) + + # Print logs after finishing + log_lines = [l.get('stream', '') for l in logs] + eprint(''.join(log_lines)) + + # Run tests if requested, and dump output + # Could be improved by backgrounding, but would need better + # multiprocessing support to track failures properly. + if FLAGS.run_tests_path: + if not tag_def['tests']: + eprint('>>> No tests to run.') + for test in tag_def['tests']: + eprint('>> Testing {}...'.format(test)) + container, = dock.containers.run( + image, + '/tests/' + test, + working_dir='/', + log_config={'type': 'journald'}, + detach=True, + stderr=True, + stdout=True, + volumes={FLAGS.run_tests_path: + {'bind': '/tests', 'mode': 'ro'}}, + runtime=tag_def['test_runtime']), + ret = container.wait() + code = ret['StatusCode'] + out = container.logs(stdout=True, stderr=False) + err = container.logs(stdout=False, stderr=True) + container.remove() + if out: + eprint('>>> Output stdout:') + eprint(out.decode('utf-8')) + else: + eprint('>>> No test standard out.') + if err: + eprint('>>> Output stderr:') + eprint(out.decode('utf-8')) + else: + eprint('>>> No test standard err.') + if code != 0: + eprint('>> {} failed tests with status: "{}"'.format( + repo_tag, code)) + failed_tags.append(tag) + tag_failed = True + if FLAGS.stop_on_failure: + eprint('>> ABORTING due to --stop_on_failure!') + exit(1) + else: + eprint('>> Tests look good!') + + except docker.errors.BuildError as e: + eprint('>> {} failed to build with message: "{}"'.format( + repo_tag, e.msg)) + eprint('>> Build logs follow:') + log_lines = [l.get('stream', '') for l in e.build_log] + eprint(''.join(log_lines)) + failed_tags.append(tag) + tag_failed = True + if FLAGS.stop_on_failure: + eprint('>> ABORTING due to --stop_on_failure!') + exit(1) + + # Clean temporary dockerfiles if they were created earlier + if not FLAGS.keep_temp_dockerfiles: + os.remove(dockerfile) + + # Upload new images to DockerHub as long as they built + passed tests + if FLAGS.upload_to_hub: + if not tag_def['upload_images']: + continue + if tag_failed: + continue + + eprint('>> Uploading to {}:{}'.format(FLAGS.hub_repository, tag)) + if not FLAGS.dry_run: + p = multiprocessing.Process( + target=upload_in_background, + args=(FLAGS.hub_repository, dock, image, tag)) + p.start() + + if failed_tags: + eprint( + '> Some tags failed to build or failed testing, check scrollback for ' + 'errors: {}'.format( + ','.join(failed_tags))) + exit(1) if __name__ == '__main__': diff --git a/tensorflow/tools/dockerfiles/dockerfiles/cpu-devel-jupyter.Dockerfile b/tensorflow/tools/dockerfiles/dockerfiles/cpu-devel-jupyter.Dockerfile index dab7178db3a..14ddf081992 100644 --- a/tensorflow/tools/dockerfiles/dockerfiles/cpu-devel-jupyter.Dockerfile +++ b/tensorflow/tools/dockerfiles/dockerfiles/cpu-devel-jupyter.Dockerfile @@ -16,27 +16,12 @@ # THIS IS A GENERATED DOCKERFILE. # # This file was assembled from multiple pieces, whose use is documented -# below. Please refer to the the TensorFlow dockerfiles documentation for -# more information. Build args are documented as their default value. -# -# Ubuntu-based, CPU-only environment for developing changes for TensorFlow, with Jupyter included. -# -# Start from Ubuntu, with TF development packages (no GPU support) -# --build-arg UBUNTU_VERSION=16.04 -# ( no description ) -# -# Python is required for TensorFlow and other libraries. -# --build-arg USE_PYTHON_3_NOT_2=True -# Install python 3 over Python 2 -# -# Install the latest version of Bazel and Python development tools. -# -# Configure TensorFlow's shell prompt and login tools. -# -# Launch Jupyter on execution instead of a bash prompt. +# throughout. Please refer to the TensorFlow dockerfiles documentation +# for more information. ARG UBUNTU_VERSION=16.04 -FROM ubuntu:${UBUNTU_VERSION} + +FROM ubuntu:${UBUNTU_VERSION} AS base RUN apt-get update && apt-get install -y --no-install-recommends \ build-essential \ @@ -48,7 +33,6 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ libpng12-dev \ libzmq3-dev \ pkg-config \ - python-dev \ rsync \ software-properties-common \ unzip \ @@ -59,8 +43,11 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* + +ENV CI_BUILD_PYTHON python -ARG USE_PYTHON_3_NOT_2=True + +ARG USE_PYTHON_3_NOT_2 ARG _PY_SUFFIX=${USE_PYTHON_3_NOT_2:+3} ARG PYTHON=python${_PY_SUFFIX} ARG PIP=pip${_PY_SUFFIX} @@ -72,10 +59,13 @@ RUN apt-get update && apt-get install -y \ ${PYTHON} \ ${PYTHON}-pip -RUN ${PIP} install --upgrade \ +RUN ${PIP} --no-cache-dir install --upgrade \ pip \ setuptools +# Some TF tools expect a "python" binary +RUN ln -s $(which ${PYTHON}) /usr/local/bin/python + RUN apt-get update && apt-get install -y \ build-essential \ curl \ @@ -84,6 +74,20 @@ RUN apt-get update && apt-get install -y \ ${PYTHON}-dev \ swig +RUN ${PIP} --no-cache-dir install \ + Pillow \ + h5py \ + keras_applications \ + keras_preprocessing \ + matplotlib \ + mock \ + numpy \ + scipy \ + sklearn \ + pandas \ + && test "${USE_PYTHON_3_NOT_2}" -eq 1 && true || ${PIP} --no-cache-dir install \ + enum34 + # Install bazel RUN echo "deb [arch=amd64] http://storage.googleapis.com/bazel-apt stable jdk1.8" | tee /etc/apt/sources.list.d/bazel.list && \ curl https://bazel.build/bazel-release.pub.gpg | apt-key add - && \ @@ -93,11 +97,19 @@ RUN echo "deb [arch=amd64] http://storage.googleapis.com/bazel-apt stable jdk1.8 COPY bashrc /etc/bash.bashrc RUN chmod a+rwx /etc/bash.bashrc -RUN ${PIP} install jupyter +RUN ${PIP} install jupyter matplotlib -RUN mkdir /notebooks && chmod a+rwx /notebooks +RUN mkdir -p /tf/tensorflow-tutorials && chmod -R a+rwx /tf/ RUN mkdir /.local && chmod a+rwx /.local -WORKDIR /notebooks +RUN apt-get install -y --no-install-recommends wget +WORKDIR /tf/tensorflow-tutorials +RUN wget https://raw.githubusercontent.com/tensorflow/docs/master/site/en/tutorials/keras/basic_classification.ipynb +RUN wget https://raw.githubusercontent.com/tensorflow/docs/master/site/en/tutorials/keras/basic_text_classification.ipynb +COPY readme-for-jupyter.md README.md +RUN apt-get autoremove -y && apt-get remove -y wget +WORKDIR /tf EXPOSE 8888 -CMD ["bash", "-c", "source /etc/bash.bashrc && jupyter notebook --notebook-dir=/notebooks --ip 0.0.0.0 --no-browser --allow-root"] +RUN ${PYTHON} -m ipykernel.kernelspec + +CMD ["bash", "-c", "source /etc/bash.bashrc && jupyter notebook --notebook-dir=/tf --ip 0.0.0.0 --no-browser --allow-root"] diff --git a/tensorflow/tools/dockerfiles/dockerfiles/cpu-devel.Dockerfile b/tensorflow/tools/dockerfiles/dockerfiles/cpu-devel.Dockerfile index 68566ccc8aa..16973b47af5 100644 --- a/tensorflow/tools/dockerfiles/dockerfiles/cpu-devel.Dockerfile +++ b/tensorflow/tools/dockerfiles/dockerfiles/cpu-devel.Dockerfile @@ -16,25 +16,12 @@ # THIS IS A GENERATED DOCKERFILE. # # This file was assembled from multiple pieces, whose use is documented -# below. Please refer to the the TensorFlow dockerfiles documentation for -# more information. Build args are documented as their default value. -# -# Ubuntu-based, CPU-only environment for developing changes for TensorFlow. -# -# Start from Ubuntu, with TF development packages (no GPU support) -# --build-arg UBUNTU_VERSION=16.04 -# ( no description ) -# -# Python is required for TensorFlow and other libraries. -# --build-arg USE_PYTHON_3_NOT_2=True -# Install python 3 over Python 2 -# -# Install the latest version of Bazel and Python development tools. -# -# Configure TensorFlow's shell prompt and login tools. +# throughout. Please refer to the TensorFlow dockerfiles documentation +# for more information. ARG UBUNTU_VERSION=16.04 -FROM ubuntu:${UBUNTU_VERSION} + +FROM ubuntu:${UBUNTU_VERSION} AS base RUN apt-get update && apt-get install -y --no-install-recommends \ build-essential \ @@ -46,7 +33,6 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ libpng12-dev \ libzmq3-dev \ pkg-config \ - python-dev \ rsync \ software-properties-common \ unzip \ @@ -57,8 +43,11 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* + +ENV CI_BUILD_PYTHON python -ARG USE_PYTHON_3_NOT_2=True + +ARG USE_PYTHON_3_NOT_2 ARG _PY_SUFFIX=${USE_PYTHON_3_NOT_2:+3} ARG PYTHON=python${_PY_SUFFIX} ARG PIP=pip${_PY_SUFFIX} @@ -70,10 +59,13 @@ RUN apt-get update && apt-get install -y \ ${PYTHON} \ ${PYTHON}-pip -RUN ${PIP} install --upgrade \ +RUN ${PIP} --no-cache-dir install --upgrade \ pip \ setuptools +# Some TF tools expect a "python" binary +RUN ln -s $(which ${PYTHON}) /usr/local/bin/python + RUN apt-get update && apt-get install -y \ build-essential \ curl \ @@ -82,6 +74,20 @@ RUN apt-get update && apt-get install -y \ ${PYTHON}-dev \ swig +RUN ${PIP} --no-cache-dir install \ + Pillow \ + h5py \ + keras_applications \ + keras_preprocessing \ + matplotlib \ + mock \ + numpy \ + scipy \ + sklearn \ + pandas \ + && test "${USE_PYTHON_3_NOT_2}" -eq 1 && true || ${PIP} --no-cache-dir install \ + enum34 + # Install bazel RUN echo "deb [arch=amd64] http://storage.googleapis.com/bazel-apt stable jdk1.8" | tee /etc/apt/sources.list.d/bazel.list && \ curl https://bazel.build/bazel-release.pub.gpg | apt-key add - && \ diff --git a/tensorflow/tools/dockerfiles/dockerfiles/cpu-jupyter.Dockerfile b/tensorflow/tools/dockerfiles/dockerfiles/cpu-jupyter.Dockerfile index f889ed6f91d..d8fabadec28 100644 --- a/tensorflow/tools/dockerfiles/dockerfiles/cpu-jupyter.Dockerfile +++ b/tensorflow/tools/dockerfiles/dockerfiles/cpu-jupyter.Dockerfile @@ -16,31 +16,14 @@ # THIS IS A GENERATED DOCKERFILE. # # This file was assembled from multiple pieces, whose use is documented -# below. Please refer to the the TensorFlow dockerfiles documentation for -# more information. Build args are documented as their default value. -# -# Ubuntu-based, CPU-only environment for using TensorFlow, with Jupyter included. -# -# Start from Ubuntu (no GPU support) -# --build-arg UBUNTU_VERSION=16.04 -# ( no description ) -# -# Python is required for TensorFlow and other libraries. -# --build-arg USE_PYTHON_3_NOT_2=True -# Install python 3 over Python 2 -# -# Install the TensorFlow Python package. -# --build-arg TF_PACKAGE=tensorflow (tensorflow|tensorflow-gpu|tf-nightly|tf-nightly-gpu) -# The specific TensorFlow Python package to install -# -# Configure TensorFlow's shell prompt and login tools. -# -# Launch Jupyter on execution instead of a bash prompt. +# throughout. Please refer to the TensorFlow dockerfiles documentation +# for more information. ARG UBUNTU_VERSION=16.04 -FROM ubuntu:${UBUNTU_VERSION} -ARG USE_PYTHON_3_NOT_2=True +FROM ubuntu:${UBUNTU_VERSION} as base + +ARG USE_PYTHON_3_NOT_2 ARG _PY_SUFFIX=${USE_PYTHON_3_NOT_2:+3} ARG PYTHON=python${_PY_SUFFIX} ARG PIP=pip${_PY_SUFFIX} @@ -52,21 +35,37 @@ RUN apt-get update && apt-get install -y \ ${PYTHON} \ ${PYTHON}-pip -RUN ${PIP} install --upgrade \ +RUN ${PIP} --no-cache-dir install --upgrade \ pip \ setuptools +# Some TF tools expect a "python" binary +RUN ln -s $(which ${PYTHON}) /usr/local/bin/python + +# Options: +# tensorflow +# tensorflow-gpu +# tf-nightly +# tf-nightly-gpu ARG TF_PACKAGE=tensorflow RUN ${PIP} install ${TF_PACKAGE} COPY bashrc /etc/bash.bashrc RUN chmod a+rwx /etc/bash.bashrc -RUN ${PIP} install jupyter +RUN ${PIP} install jupyter matplotlib -RUN mkdir /notebooks && chmod a+rwx /notebooks +RUN mkdir -p /tf/tensorflow-tutorials && chmod -R a+rwx /tf/ RUN mkdir /.local && chmod a+rwx /.local -WORKDIR /notebooks +RUN apt-get install -y --no-install-recommends wget +WORKDIR /tf/tensorflow-tutorials +RUN wget https://raw.githubusercontent.com/tensorflow/docs/master/site/en/tutorials/keras/basic_classification.ipynb +RUN wget https://raw.githubusercontent.com/tensorflow/docs/master/site/en/tutorials/keras/basic_text_classification.ipynb +COPY readme-for-jupyter.md README.md +RUN apt-get autoremove -y && apt-get remove -y wget +WORKDIR /tf EXPOSE 8888 -CMD ["bash", "-c", "source /etc/bash.bashrc && jupyter notebook --notebook-dir=/notebooks --ip 0.0.0.0 --no-browser --allow-root"] +RUN ${PYTHON} -m ipykernel.kernelspec + +CMD ["bash", "-c", "source /etc/bash.bashrc && jupyter notebook --notebook-dir=/tf --ip 0.0.0.0 --no-browser --allow-root"] diff --git a/tensorflow/tools/dockerfiles/dockerfiles/cpu.Dockerfile b/tensorflow/tools/dockerfiles/dockerfiles/cpu.Dockerfile index 182a534bed9..857b5e20471 100644 --- a/tensorflow/tools/dockerfiles/dockerfiles/cpu.Dockerfile +++ b/tensorflow/tools/dockerfiles/dockerfiles/cpu.Dockerfile @@ -16,29 +16,14 @@ # THIS IS A GENERATED DOCKERFILE. # # This file was assembled from multiple pieces, whose use is documented -# below. Please refer to the the TensorFlow dockerfiles documentation for -# more information. Build args are documented as their default value. -# -# Ubuntu-based, CPU-only environment for using TensorFlow -# -# Start from Ubuntu (no GPU support) -# --build-arg UBUNTU_VERSION=16.04 -# ( no description ) -# -# Python is required for TensorFlow and other libraries. -# --build-arg USE_PYTHON_3_NOT_2=True -# Install python 3 over Python 2 -# -# Install the TensorFlow Python package. -# --build-arg TF_PACKAGE=tensorflow (tensorflow|tensorflow-gpu|tf-nightly|tf-nightly-gpu) -# The specific TensorFlow Python package to install -# -# Configure TensorFlow's shell prompt and login tools. +# throughout. Please refer to the TensorFlow dockerfiles documentation +# for more information. ARG UBUNTU_VERSION=16.04 -FROM ubuntu:${UBUNTU_VERSION} -ARG USE_PYTHON_3_NOT_2=True +FROM ubuntu:${UBUNTU_VERSION} as base + +ARG USE_PYTHON_3_NOT_2 ARG _PY_SUFFIX=${USE_PYTHON_3_NOT_2:+3} ARG PYTHON=python${_PY_SUFFIX} ARG PIP=pip${_PY_SUFFIX} @@ -50,10 +35,18 @@ RUN apt-get update && apt-get install -y \ ${PYTHON} \ ${PYTHON}-pip -RUN ${PIP} install --upgrade \ +RUN ${PIP} --no-cache-dir install --upgrade \ pip \ setuptools +# Some TF tools expect a "python" binary +RUN ln -s $(which ${PYTHON}) /usr/local/bin/python + +# Options: +# tensorflow +# tensorflow-gpu +# tf-nightly +# tf-nightly-gpu ARG TF_PACKAGE=tensorflow RUN ${PIP} install ${TF_PACKAGE} diff --git a/tensorflow/tools/dockerfiles/dockerfiles/nvidia-devel-jupyter.Dockerfile b/tensorflow/tools/dockerfiles/dockerfiles/gpu-devel-jupyter.Dockerfile similarity index 66% rename from tensorflow/tools/dockerfiles/dockerfiles/nvidia-devel-jupyter.Dockerfile rename to tensorflow/tools/dockerfiles/dockerfiles/gpu-devel-jupyter.Dockerfile index 17faa84a682..9ecaec38c2e 100644 --- a/tensorflow/tools/dockerfiles/dockerfiles/nvidia-devel-jupyter.Dockerfile +++ b/tensorflow/tools/dockerfiles/dockerfiles/gpu-devel-jupyter.Dockerfile @@ -16,28 +16,12 @@ # THIS IS A GENERATED DOCKERFILE. # # This file was assembled from multiple pieces, whose use is documented -# below. Please refer to the the TensorFlow dockerfiles documentation for -# more information. Build args are documented as their default value. -# -# Ubuntu-based, Nvidia-GPU-enabled environment for developing changes for TensorFlow, with Jupyter included. -# -# Start from Nvidia's Ubuntu base image with CUDA and CuDNN, with TF development -# packages. -# --build-arg UBUNTU_VERSION=16.04 -# ( no description ) -# -# Python is required for TensorFlow and other libraries. -# --build-arg USE_PYTHON_3_NOT_2=True -# Install python 3 over Python 2 -# -# Install the latest version of Bazel and Python development tools. -# -# Configure TensorFlow's shell prompt and login tools. -# -# Launch Jupyter on execution instead of a bash prompt. +# throughout. Please refer to the TensorFlow dockerfiles documentation +# for more information. ARG UBUNTU_VERSION=16.04 -FROM nvidia/cuda:9.0-base-ubuntu${UBUNTU_VERSION} + +FROM nvidia/cuda:9.0-base-ubuntu${UBUNTU_VERSION} as base RUN apt-get update && apt-get install -y --no-install-recommends \ build-essential \ @@ -60,6 +44,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ libpng12-dev \ libzmq3-dev \ pkg-config \ + python-dev \ rsync \ software-properties-common \ unzip \ @@ -82,11 +67,19 @@ RUN mkdir /usr/local/cuda-9.0/lib && \ ln -s /usr/lib/x86_64-linux-gnu/libnccl.so.2 /usr/local/cuda/lib/libnccl.so.2 && \ ln -s /usr/include/nccl.h /usr/local/cuda/include/nccl.h -# TODO(tobyboyd): Remove after license is excluded from BUILD file. -RUN gunzip /usr/share/doc/libnccl2/NCCL-SLA.txt.gz && \ - cp /usr/share/doc/libnccl2/NCCL-SLA.txt /usr/local/cuda/ +# Configure the build for our CUDA configuration. +ENV CI_BUILD_PYTHON python +ENV LD_LIBRARY_PATH /usr/local/cuda/extras/CUPTI/lib64:$LD_LIBRARY_PATH +ENV TF_NEED_CUDA 1 +ENV TF_NEED_TENSORRT 1 +ENV TF_CUDA_COMPUTE_CAPABILITIES=3.5,5.2,6.0,6.1,7.0 +ENV TF_CUDA_VERSION=9.0 +ENV TF_CUDNN_VERSION=7 -ARG USE_PYTHON_3_NOT_2=True +# NCCL 2.x +ENV TF_NCCL_VERSION=2 + +ARG USE_PYTHON_3_NOT_2 ARG _PY_SUFFIX=${USE_PYTHON_3_NOT_2:+3} ARG PYTHON=python${_PY_SUFFIX} ARG PIP=pip${_PY_SUFFIX} @@ -98,10 +91,13 @@ RUN apt-get update && apt-get install -y \ ${PYTHON} \ ${PYTHON}-pip -RUN ${PIP} install --upgrade \ +RUN ${PIP} --no-cache-dir install --upgrade \ pip \ setuptools +# Some TF tools expect a "python" binary +RUN ln -s $(which ${PYTHON}) /usr/local/bin/python + RUN apt-get update && apt-get install -y \ build-essential \ curl \ @@ -110,6 +106,20 @@ RUN apt-get update && apt-get install -y \ ${PYTHON}-dev \ swig +RUN ${PIP} --no-cache-dir install \ + Pillow \ + h5py \ + keras_applications \ + keras_preprocessing \ + matplotlib \ + mock \ + numpy \ + scipy \ + sklearn \ + pandas \ + && test "${USE_PYTHON_3_NOT_2}" -eq 1 && true || ${PIP} --no-cache-dir install \ + enum34 + # Install bazel RUN echo "deb [arch=amd64] http://storage.googleapis.com/bazel-apt stable jdk1.8" | tee /etc/apt/sources.list.d/bazel.list && \ curl https://bazel.build/bazel-release.pub.gpg | apt-key add - && \ @@ -119,11 +129,19 @@ RUN echo "deb [arch=amd64] http://storage.googleapis.com/bazel-apt stable jdk1.8 COPY bashrc /etc/bash.bashrc RUN chmod a+rwx /etc/bash.bashrc -RUN ${PIP} install jupyter +RUN ${PIP} install jupyter matplotlib -RUN mkdir /notebooks && chmod a+rwx /notebooks +RUN mkdir -p /tf/tensorflow-tutorials && chmod -R a+rwx /tf/ RUN mkdir /.local && chmod a+rwx /.local -WORKDIR /notebooks +RUN apt-get install -y --no-install-recommends wget +WORKDIR /tf/tensorflow-tutorials +RUN wget https://raw.githubusercontent.com/tensorflow/docs/master/site/en/tutorials/keras/basic_classification.ipynb +RUN wget https://raw.githubusercontent.com/tensorflow/docs/master/site/en/tutorials/keras/basic_text_classification.ipynb +COPY readme-for-jupyter.md README.md +RUN apt-get autoremove -y && apt-get remove -y wget +WORKDIR /tf EXPOSE 8888 -CMD ["bash", "-c", "source /etc/bash.bashrc && jupyter notebook --notebook-dir=/notebooks --ip 0.0.0.0 --no-browser --allow-root"] +RUN ${PYTHON} -m ipykernel.kernelspec + +CMD ["bash", "-c", "source /etc/bash.bashrc && jupyter notebook --notebook-dir=/tf --ip 0.0.0.0 --no-browser --allow-root"] diff --git a/tensorflow/tools/dockerfiles/dockerfiles/nvidia-devel.Dockerfile b/tensorflow/tools/dockerfiles/dockerfiles/gpu-devel.Dockerfile similarity index 76% rename from tensorflow/tools/dockerfiles/dockerfiles/nvidia-devel.Dockerfile rename to tensorflow/tools/dockerfiles/dockerfiles/gpu-devel.Dockerfile index a3ba02a684c..c79bc3cf4c0 100644 --- a/tensorflow/tools/dockerfiles/dockerfiles/nvidia-devel.Dockerfile +++ b/tensorflow/tools/dockerfiles/dockerfiles/gpu-devel.Dockerfile @@ -16,26 +16,12 @@ # THIS IS A GENERATED DOCKERFILE. # # This file was assembled from multiple pieces, whose use is documented -# below. Please refer to the the TensorFlow dockerfiles documentation for -# more information. Build args are documented as their default value. -# -# Ubuntu-based, Nvidia-GPU-enabled environment for developing changes for TensorFlow. -# -# Start from Nvidia's Ubuntu base image with CUDA and CuDNN, with TF development -# packages. -# --build-arg UBUNTU_VERSION=16.04 -# ( no description ) -# -# Python is required for TensorFlow and other libraries. -# --build-arg USE_PYTHON_3_NOT_2=True -# Install python 3 over Python 2 -# -# Install the latest version of Bazel and Python development tools. -# -# Configure TensorFlow's shell prompt and login tools. +# throughout. Please refer to the TensorFlow dockerfiles documentation +# for more information. ARG UBUNTU_VERSION=16.04 -FROM nvidia/cuda:9.0-base-ubuntu${UBUNTU_VERSION} + +FROM nvidia/cuda:9.0-base-ubuntu${UBUNTU_VERSION} as base RUN apt-get update && apt-get install -y --no-install-recommends \ build-essential \ @@ -58,6 +44,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ libpng12-dev \ libzmq3-dev \ pkg-config \ + python-dev \ rsync \ software-properties-common \ unzip \ @@ -80,11 +67,19 @@ RUN mkdir /usr/local/cuda-9.0/lib && \ ln -s /usr/lib/x86_64-linux-gnu/libnccl.so.2 /usr/local/cuda/lib/libnccl.so.2 && \ ln -s /usr/include/nccl.h /usr/local/cuda/include/nccl.h -# TODO(tobyboyd): Remove after license is excluded from BUILD file. -RUN gunzip /usr/share/doc/libnccl2/NCCL-SLA.txt.gz && \ - cp /usr/share/doc/libnccl2/NCCL-SLA.txt /usr/local/cuda/ +# Configure the build for our CUDA configuration. +ENV CI_BUILD_PYTHON python +ENV LD_LIBRARY_PATH /usr/local/cuda/extras/CUPTI/lib64:$LD_LIBRARY_PATH +ENV TF_NEED_CUDA 1 +ENV TF_NEED_TENSORRT 1 +ENV TF_CUDA_COMPUTE_CAPABILITIES=3.5,5.2,6.0,6.1,7.0 +ENV TF_CUDA_VERSION=9.0 +ENV TF_CUDNN_VERSION=7 -ARG USE_PYTHON_3_NOT_2=True +# NCCL 2.x +ENV TF_NCCL_VERSION=2 + +ARG USE_PYTHON_3_NOT_2 ARG _PY_SUFFIX=${USE_PYTHON_3_NOT_2:+3} ARG PYTHON=python${_PY_SUFFIX} ARG PIP=pip${_PY_SUFFIX} @@ -96,10 +91,13 @@ RUN apt-get update && apt-get install -y \ ${PYTHON} \ ${PYTHON}-pip -RUN ${PIP} install --upgrade \ +RUN ${PIP} --no-cache-dir install --upgrade \ pip \ setuptools +# Some TF tools expect a "python" binary +RUN ln -s $(which ${PYTHON}) /usr/local/bin/python + RUN apt-get update && apt-get install -y \ build-essential \ curl \ @@ -108,6 +106,20 @@ RUN apt-get update && apt-get install -y \ ${PYTHON}-dev \ swig +RUN ${PIP} --no-cache-dir install \ + Pillow \ + h5py \ + keras_applications \ + keras_preprocessing \ + matplotlib \ + mock \ + numpy \ + scipy \ + sklearn \ + pandas \ + && test "${USE_PYTHON_3_NOT_2}" -eq 1 && true || ${PIP} --no-cache-dir install \ + enum34 + # Install bazel RUN echo "deb [arch=amd64] http://storage.googleapis.com/bazel-apt stable jdk1.8" | tee /etc/apt/sources.list.d/bazel.list && \ curl https://bazel.build/bazel-release.pub.gpg | apt-key add - && \ diff --git a/tensorflow/tools/dockerfiles/dockerfiles/nvidia-jupyter.Dockerfile b/tensorflow/tools/dockerfiles/dockerfiles/gpu-jupyter.Dockerfile similarity index 62% rename from tensorflow/tools/dockerfiles/dockerfiles/nvidia-jupyter.Dockerfile rename to tensorflow/tools/dockerfiles/dockerfiles/gpu-jupyter.Dockerfile index fbdea4628ad..acfe4d8607d 100644 --- a/tensorflow/tools/dockerfiles/dockerfiles/nvidia-jupyter.Dockerfile +++ b/tensorflow/tools/dockerfiles/dockerfiles/gpu-jupyter.Dockerfile @@ -16,30 +16,13 @@ # THIS IS A GENERATED DOCKERFILE. # # This file was assembled from multiple pieces, whose use is documented -# below. Please refer to the the TensorFlow dockerfiles documentation for -# more information. Build args are documented as their default value. -# -# Ubuntu-based, Nvidia-GPU-enabled environment for using TensorFlow, with Jupyter included. -# -# NVIDIA with CUDA and CuDNN, no dev stuff -# --build-arg UBUNTU_VERSION=16.04 -# ( no description ) -# -# Python is required for TensorFlow and other libraries. -# --build-arg USE_PYTHON_3_NOT_2=True -# Install python 3 over Python 2 -# -# Install the TensorFlow Python package. -# --build-arg TF_PACKAGE=tensorflow-gpu (tensorflow|tensorflow-gpu|tf-nightly|tf-nightly-gpu) -# The specific TensorFlow Python package to install -# -# Configure TensorFlow's shell prompt and login tools. -# -# Launch Jupyter on execution instead of a bash prompt. +# throughout. Please refer to the TensorFlow dockerfiles documentation +# for more information. -FROM nvidia/cuda:9.0-base-ubuntu16.04 +ARG UBUNTU_VERSION=16.04 + +FROM nvidia/cuda:9.0-base-ubuntu${UBUNTU_VERSION} as base -# Pick up some TF dependencies RUN apt-get update && apt-get install -y --no-install-recommends \ build-essential \ cuda-command-line-tools-9-0 \ @@ -48,6 +31,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ cuda-curand-9-0 \ cuda-cusolver-9-0 \ cuda-cusparse-9-0 \ + curl \ libcudnn7=7.2.1.38-1+cuda9.0 \ libnccl2=2.2.13-1+cuda9.0 \ libfreetype6-dev \ @@ -55,6 +39,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ libpng12-dev \ libzmq3-dev \ pkg-config \ + rsync \ software-properties-common \ unzip \ && \ @@ -66,7 +51,10 @@ RUN apt-get update && \ apt-get update && \ apt-get install libnvinfer4=4.1.2-1+cuda9.0 -ARG USE_PYTHON_3_NOT_2=True +# For CUDA profiling, TensorFlow requires CUPTI. +ENV LD_LIBRARY_PATH /usr/local/cuda/extras/CUPTI/lib64:$LD_LIBRARY_PATH + +ARG USE_PYTHON_3_NOT_2 ARG _PY_SUFFIX=${USE_PYTHON_3_NOT_2:+3} ARG PYTHON=python${_PY_SUFFIX} ARG PIP=pip${_PY_SUFFIX} @@ -78,21 +66,37 @@ RUN apt-get update && apt-get install -y \ ${PYTHON} \ ${PYTHON}-pip -RUN ${PIP} install --upgrade \ +RUN ${PIP} --no-cache-dir install --upgrade \ pip \ setuptools -ARG TF_PACKAGE=tensorflow-gpu +# Some TF tools expect a "python" binary +RUN ln -s $(which ${PYTHON}) /usr/local/bin/python + +# Options: +# tensorflow +# tensorflow-gpu +# tf-nightly +# tf-nightly-gpu +ARG TF_PACKAGE=tensorflow RUN ${PIP} install ${TF_PACKAGE} COPY bashrc /etc/bash.bashrc RUN chmod a+rwx /etc/bash.bashrc -RUN ${PIP} install jupyter +RUN ${PIP} install jupyter matplotlib -RUN mkdir /notebooks && chmod a+rwx /notebooks +RUN mkdir -p /tf/tensorflow-tutorials && chmod -R a+rwx /tf/ RUN mkdir /.local && chmod a+rwx /.local -WORKDIR /notebooks +RUN apt-get install -y --no-install-recommends wget +WORKDIR /tf/tensorflow-tutorials +RUN wget https://raw.githubusercontent.com/tensorflow/docs/master/site/en/tutorials/keras/basic_classification.ipynb +RUN wget https://raw.githubusercontent.com/tensorflow/docs/master/site/en/tutorials/keras/basic_text_classification.ipynb +COPY readme-for-jupyter.md README.md +RUN apt-get autoremove -y && apt-get remove -y wget +WORKDIR /tf EXPOSE 8888 -CMD ["bash", "-c", "source /etc/bash.bashrc && jupyter notebook --notebook-dir=/notebooks --ip 0.0.0.0 --no-browser --allow-root"] +RUN ${PYTHON} -m ipykernel.kernelspec + +CMD ["bash", "-c", "source /etc/bash.bashrc && jupyter notebook --notebook-dir=/tf --ip 0.0.0.0 --no-browser --allow-root"] diff --git a/tensorflow/tools/dockerfiles/dockerfiles/nvidia.Dockerfile b/tensorflow/tools/dockerfiles/dockerfiles/gpu.Dockerfile similarity index 69% rename from tensorflow/tools/dockerfiles/dockerfiles/nvidia.Dockerfile rename to tensorflow/tools/dockerfiles/dockerfiles/gpu.Dockerfile index e0312dbc294..f36a21eaf0c 100644 --- a/tensorflow/tools/dockerfiles/dockerfiles/nvidia.Dockerfile +++ b/tensorflow/tools/dockerfiles/dockerfiles/gpu.Dockerfile @@ -16,28 +16,13 @@ # THIS IS A GENERATED DOCKERFILE. # # This file was assembled from multiple pieces, whose use is documented -# below. Please refer to the the TensorFlow dockerfiles documentation for -# more information. Build args are documented as their default value. -# -# Ubuntu-based, Nvidia-GPU-enabled environment for using TensorFlow. -# -# NVIDIA with CUDA and CuDNN, no dev stuff -# --build-arg UBUNTU_VERSION=16.04 -# ( no description ) -# -# Python is required for TensorFlow and other libraries. -# --build-arg USE_PYTHON_3_NOT_2=True -# Install python 3 over Python 2 -# -# Install the TensorFlow Python package. -# --build-arg TF_PACKAGE=tensorflow-gpu (tensorflow|tensorflow-gpu|tf-nightly|tf-nightly-gpu) -# The specific TensorFlow Python package to install -# -# Configure TensorFlow's shell prompt and login tools. +# throughout. Please refer to the TensorFlow dockerfiles documentation +# for more information. -FROM nvidia/cuda:9.0-base-ubuntu16.04 +ARG UBUNTU_VERSION=16.04 + +FROM nvidia/cuda:9.0-base-ubuntu${UBUNTU_VERSION} as base -# Pick up some TF dependencies RUN apt-get update && apt-get install -y --no-install-recommends \ build-essential \ cuda-command-line-tools-9-0 \ @@ -46,6 +31,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ cuda-curand-9-0 \ cuda-cusolver-9-0 \ cuda-cusparse-9-0 \ + curl \ libcudnn7=7.2.1.38-1+cuda9.0 \ libnccl2=2.2.13-1+cuda9.0 \ libfreetype6-dev \ @@ -53,6 +39,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ libpng12-dev \ libzmq3-dev \ pkg-config \ + rsync \ software-properties-common \ unzip \ && \ @@ -64,7 +51,10 @@ RUN apt-get update && \ apt-get update && \ apt-get install libnvinfer4=4.1.2-1+cuda9.0 -ARG USE_PYTHON_3_NOT_2=True +# For CUDA profiling, TensorFlow requires CUPTI. +ENV LD_LIBRARY_PATH /usr/local/cuda/extras/CUPTI/lib64:$LD_LIBRARY_PATH + +ARG USE_PYTHON_3_NOT_2 ARG _PY_SUFFIX=${USE_PYTHON_3_NOT_2:+3} ARG PYTHON=python${_PY_SUFFIX} ARG PIP=pip${_PY_SUFFIX} @@ -76,11 +66,19 @@ RUN apt-get update && apt-get install -y \ ${PYTHON} \ ${PYTHON}-pip -RUN ${PIP} install --upgrade \ +RUN ${PIP} --no-cache-dir install --upgrade \ pip \ setuptools -ARG TF_PACKAGE=tensorflow-gpu +# Some TF tools expect a "python" binary +RUN ln -s $(which ${PYTHON}) /usr/local/bin/python + +# Options: +# tensorflow +# tensorflow-gpu +# tf-nightly +# tf-nightly-gpu +ARG TF_PACKAGE=tensorflow RUN ${PIP} install ${TF_PACKAGE} COPY bashrc /etc/bash.bashrc diff --git a/tensorflow/tools/dockerfiles/partials/jupyter.partial.Dockerfile b/tensorflow/tools/dockerfiles/partials/jupyter.partial.Dockerfile index 2c9b9f3f9a0..c4ec6095c0c 100644 --- a/tensorflow/tools/dockerfiles/partials/jupyter.partial.Dockerfile +++ b/tensorflow/tools/dockerfiles/partials/jupyter.partial.Dockerfile @@ -1,8 +1,16 @@ -RUN ${PIP} install jupyter +RUN ${PIP} install jupyter matplotlib -RUN mkdir /notebooks && chmod a+rwx /notebooks +RUN mkdir -p /tf/tensorflow-tutorials && chmod -R a+rwx /tf/ RUN mkdir /.local && chmod a+rwx /.local -WORKDIR /notebooks +RUN apt-get install -y --no-install-recommends wget +WORKDIR /tf/tensorflow-tutorials +RUN wget https://raw.githubusercontent.com/tensorflow/docs/master/site/en/tutorials/keras/basic_classification.ipynb +RUN wget https://raw.githubusercontent.com/tensorflow/docs/master/site/en/tutorials/keras/basic_text_classification.ipynb +COPY readme-for-jupyter.md README.md +RUN apt-get autoremove -y && apt-get remove -y wget +WORKDIR /tf EXPOSE 8888 -CMD ["bash", "-c", "source /etc/bash.bashrc && jupyter notebook --notebook-dir=/notebooks --ip 0.0.0.0 --no-browser --allow-root"] +RUN ${PYTHON} -m ipykernel.kernelspec + +CMD ["bash", "-c", "source /etc/bash.bashrc && jupyter notebook --notebook-dir=/tf --ip 0.0.0.0 --no-browser --allow-root"] diff --git a/tensorflow/tools/dockerfiles/partials/tensorflow.partial.Dockerfile b/tensorflow/tools/dockerfiles/partials/tensorflow.partial.Dockerfile index 96e79547f0c..76758bd147e 100644 --- a/tensorflow/tools/dockerfiles/partials/tensorflow.partial.Dockerfile +++ b/tensorflow/tools/dockerfiles/partials/tensorflow.partial.Dockerfile @@ -1,2 +1,7 @@ -ARG TF_PACKAGE +# Options: +# tensorflow +# tensorflow-gpu +# tf-nightly +# tf-nightly-gpu +ARG TF_PACKAGE=tensorflow RUN ${PIP} install ${TF_PACKAGE} diff --git a/tensorflow/tools/dockerfiles/partials/test-import.partial.Dockerfile b/tensorflow/tools/dockerfiles/partials/test-import.partial.Dockerfile new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tensorflow/tools/dockerfiles/partials/ubuntu.partial.Dockerfile b/tensorflow/tools/dockerfiles/partials/ubuntu.partial.Dockerfile deleted file mode 100644 index 0a50735bf83..00000000000 --- a/tensorflow/tools/dockerfiles/partials/ubuntu.partial.Dockerfile +++ /dev/null @@ -1,2 +0,0 @@ -ARG UBUNTU_VERSION=16.04 -FROM ubuntu:${UBUNTU_VERSION} diff --git a/tensorflow/tools/dockerfiles/partials/bazel.partial.Dockerfile b/tensorflow/tools/dockerfiles/partials/ubuntu/bazel.partial.Dockerfile similarity index 58% rename from tensorflow/tools/dockerfiles/partials/bazel.partial.Dockerfile rename to tensorflow/tools/dockerfiles/partials/ubuntu/bazel.partial.Dockerfile index b08d8bdd14b..156bb019914 100644 --- a/tensorflow/tools/dockerfiles/partials/bazel.partial.Dockerfile +++ b/tensorflow/tools/dockerfiles/partials/ubuntu/bazel.partial.Dockerfile @@ -6,6 +6,20 @@ RUN apt-get update && apt-get install -y \ ${PYTHON}-dev \ swig +RUN ${PIP} --no-cache-dir install \ + Pillow \ + h5py \ + keras_applications \ + keras_preprocessing \ + matplotlib \ + mock \ + numpy \ + scipy \ + sklearn \ + pandas \ + && test "${USE_PYTHON_3_NOT_2}" -eq 1 && true || ${PIP} --no-cache-dir install \ + enum34 + # Install bazel RUN echo "deb [arch=amd64] http://storage.googleapis.com/bazel-apt stable jdk1.8" | tee /etc/apt/sources.list.d/bazel.list && \ curl https://bazel.build/bazel-release.pub.gpg | apt-key add - && \ diff --git a/tensorflow/tools/dockerfiles/partials/ubuntu-devel.partial.Dockerfile b/tensorflow/tools/dockerfiles/partials/ubuntu/cpu-devel.partial.Dockerfile similarity index 86% rename from tensorflow/tools/dockerfiles/partials/ubuntu-devel.partial.Dockerfile rename to tensorflow/tools/dockerfiles/partials/ubuntu/cpu-devel.partial.Dockerfile index bc792722766..901652cc281 100644 --- a/tensorflow/tools/dockerfiles/partials/ubuntu-devel.partial.Dockerfile +++ b/tensorflow/tools/dockerfiles/partials/ubuntu/cpu-devel.partial.Dockerfile @@ -1,5 +1,4 @@ -ARG UBUNTU_VERSION=16.04 -FROM ubuntu:${UBUNTU_VERSION} +FROM ubuntu:${UBUNTU_VERSION} AS base RUN apt-get update && apt-get install -y --no-install-recommends \ build-essential \ @@ -11,7 +10,6 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ libpng12-dev \ libzmq3-dev \ pkg-config \ - python-dev \ rsync \ software-properties-common \ unzip \ @@ -22,3 +20,6 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* + +ENV CI_BUILD_PYTHON python + diff --git a/tensorflow/tools/dockerfiles/partials/ubuntu/cpu.partial.Dockerfile b/tensorflow/tools/dockerfiles/partials/ubuntu/cpu.partial.Dockerfile new file mode 100644 index 00000000000..d01b26e27f6 --- /dev/null +++ b/tensorflow/tools/dockerfiles/partials/ubuntu/cpu.partial.Dockerfile @@ -0,0 +1 @@ +FROM ubuntu:${UBUNTU_VERSION} as base diff --git a/tensorflow/tools/dockerfiles/partials/nvidia-devel.partial.Dockerfile b/tensorflow/tools/dockerfiles/partials/ubuntu/nvidia-devel.partial.Dockerfile similarity index 78% rename from tensorflow/tools/dockerfiles/partials/nvidia-devel.partial.Dockerfile rename to tensorflow/tools/dockerfiles/partials/ubuntu/nvidia-devel.partial.Dockerfile index 45159f711fc..48d457e40cf 100644 --- a/tensorflow/tools/dockerfiles/partials/nvidia-devel.partial.Dockerfile +++ b/tensorflow/tools/dockerfiles/partials/ubuntu/nvidia-devel.partial.Dockerfile @@ -1,5 +1,4 @@ -ARG UBUNTU_VERSION=16.04 -FROM nvidia/cuda:9.0-base-ubuntu${UBUNTU_VERSION} +FROM nvidia/cuda:9.0-base-ubuntu${UBUNTU_VERSION} as base RUN apt-get update && apt-get install -y --no-install-recommends \ build-essential \ @@ -22,6 +21,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ libpng12-dev \ libzmq3-dev \ pkg-config \ + python-dev \ rsync \ software-properties-common \ unzip \ @@ -44,6 +44,14 @@ RUN mkdir /usr/local/cuda-9.0/lib && \ ln -s /usr/lib/x86_64-linux-gnu/libnccl.so.2 /usr/local/cuda/lib/libnccl.so.2 && \ ln -s /usr/include/nccl.h /usr/local/cuda/include/nccl.h -# TODO(tobyboyd): Remove after license is excluded from BUILD file. -RUN gunzip /usr/share/doc/libnccl2/NCCL-SLA.txt.gz && \ - cp /usr/share/doc/libnccl2/NCCL-SLA.txt /usr/local/cuda/ +# Configure the build for our CUDA configuration. +ENV CI_BUILD_PYTHON python +ENV LD_LIBRARY_PATH /usr/local/cuda/extras/CUPTI/lib64:$LD_LIBRARY_PATH +ENV TF_NEED_CUDA 1 +ENV TF_NEED_TENSORRT 1 +ENV TF_CUDA_COMPUTE_CAPABILITIES=3.5,5.2,6.0,6.1,7.0 +ENV TF_CUDA_VERSION=9.0 +ENV TF_CUDNN_VERSION=7 + +# NCCL 2.x +ENV TF_NCCL_VERSION=2 diff --git a/tensorflow/tools/dockerfiles/partials/nvidia.partial.Dockerfile b/tensorflow/tools/dockerfiles/partials/ubuntu/nvidia.partial.Dockerfile similarity index 78% rename from tensorflow/tools/dockerfiles/partials/nvidia.partial.Dockerfile rename to tensorflow/tools/dockerfiles/partials/ubuntu/nvidia.partial.Dockerfile index 1064390af3b..1dc8e43aadd 100644 --- a/tensorflow/tools/dockerfiles/partials/nvidia.partial.Dockerfile +++ b/tensorflow/tools/dockerfiles/partials/ubuntu/nvidia.partial.Dockerfile @@ -1,6 +1,5 @@ -FROM nvidia/cuda:9.0-base-ubuntu16.04 +FROM nvidia/cuda:9.0-base-ubuntu${UBUNTU_VERSION} as base -# Pick up some TF dependencies RUN apt-get update && apt-get install -y --no-install-recommends \ build-essential \ cuda-command-line-tools-9-0 \ @@ -9,6 +8,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ cuda-curand-9-0 \ cuda-cusolver-9-0 \ cuda-cusparse-9-0 \ + curl \ libcudnn7=7.2.1.38-1+cuda9.0 \ libnccl2=2.2.13-1+cuda9.0 \ libfreetype6-dev \ @@ -16,6 +16,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ libpng12-dev \ libzmq3-dev \ pkg-config \ + rsync \ software-properties-common \ unzip \ && \ @@ -26,3 +27,6 @@ RUN apt-get update && \ apt-get install nvinfer-runtime-trt-repo-ubuntu1604-4.0.1-ga-cuda9.0 && \ apt-get update && \ apt-get install libnvinfer4=4.1.2-1+cuda9.0 + +# For CUDA profiling, TensorFlow requires CUPTI. +ENV LD_LIBRARY_PATH /usr/local/cuda/extras/CUPTI/lib64:$LD_LIBRARY_PATH diff --git a/tensorflow/tools/dockerfiles/partials/python.partial.Dockerfile b/tensorflow/tools/dockerfiles/partials/ubuntu/python.partial.Dockerfile similarity index 66% rename from tensorflow/tools/dockerfiles/partials/python.partial.Dockerfile rename to tensorflow/tools/dockerfiles/partials/ubuntu/python.partial.Dockerfile index ee08af73a8e..6af47319538 100644 --- a/tensorflow/tools/dockerfiles/partials/python.partial.Dockerfile +++ b/tensorflow/tools/dockerfiles/partials/ubuntu/python.partial.Dockerfile @@ -10,6 +10,9 @@ RUN apt-get update && apt-get install -y \ ${PYTHON} \ ${PYTHON}-pip -RUN ${PIP} install --upgrade \ +RUN ${PIP} --no-cache-dir install --upgrade \ pip \ setuptools + +# Some TF tools expect a "python" binary +RUN ln -s $(which ${PYTHON}) /usr/local/bin/python diff --git a/tensorflow/tools/dockerfiles/partials/ubuntu/test-devel.partial.Dockerfile b/tensorflow/tools/dockerfiles/partials/ubuntu/test-devel.partial.Dockerfile new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tensorflow/tools/dockerfiles/partials/ubuntu/version.partial.Dockerfile b/tensorflow/tools/dockerfiles/partials/ubuntu/version.partial.Dockerfile new file mode 100644 index 00000000000..6ecd2b8b1ac --- /dev/null +++ b/tensorflow/tools/dockerfiles/partials/ubuntu/version.partial.Dockerfile @@ -0,0 +1 @@ +ARG UBUNTU_VERSION=16.04 diff --git a/tensorflow/tools/dockerfiles/readme-for-jupyter.md b/tensorflow/tools/dockerfiles/readme-for-jupyter.md new file mode 100644 index 00000000000..f104a7533b8 --- /dev/null +++ b/tensorflow/tools/dockerfiles/readme-for-jupyter.md @@ -0,0 +1,3 @@ +Want more tutorials like these? + +Check out tensorflow.org/tutorials! diff --git a/tensorflow/tools/dockerfiles/spec.yml b/tensorflow/tools/dockerfiles/spec.yml index 28bf9a55da1..4826ddd8e22 100644 --- a/tensorflow/tools/dockerfiles/spec.yml +++ b/tensorflow/tools/dockerfiles/spec.yml @@ -1,195 +1,135 @@ -# ====== -# HEADER -# ====== -# -# This is commented-out and prepended to each generated Dockerfile. header: | - Copyright 2018 The TensorFlow Authors. All Rights Reserved. + # Copyright 2018 The TensorFlow Authors. All Rights Reserved. + # + # Licensed under the Apache License, Version 2.0 (the "License"); + # you may not use this file except in compliance with the License. + # You may obtain a copy of the License at + # + # http://www.apache.org/licenses/LICENSE-2.0 + # + # Unless required by applicable law or agreed to in writing, software + # distributed under the License is distributed on an "AS IS" BASIS, + # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + # See the License for the specific language governing permissions and + # limitations under the License. + # ============================================================================ + # + # THIS IS A GENERATED DOCKERFILE. + # + # This file was assembled from multiple pieces, whose use is documented + # throughout. Please refer to the TensorFlow dockerfiles documentation + # for more information. - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - ============================================================================ - - THIS IS A GENERATED DOCKERFILE. - - This file was assembled from multiple pieces, whose use is documented - below. Please refer to the the TensorFlow dockerfiles documentation for - more information. Build args are documented as their default value. - -# ======== -# PARTIALS -# ======== +# A combinatorial explosion of Docker images and Dockerfiles. +# Each "release" defines all of the ways to combine related but separate chunks +# of functionality ("slices") by listing all of the "slice sets" to use when +# building. # -# Represent and document pieces of a Dockerfile. Spec: -# -# name: the name of the partial, is referenced from the images section -# desc: A description, inserted later into the Dockerfile -# file: Alternative file prefix, e.g. file.partial.Dockerfile. The default is -# the name of the partial. -# args: A dict of ARGs in the Dockerfile; each entry has the format -# ARG_NAME: VALUE where VALUE is one of: -# - a dict: -# desc: Documentation for the arg -# default: Default value for the arg; is written to the Dockerfile -# options: List of strings, part of documentation -# - a concrete value: the same as a dictionary with default: [value]. +# For example, a release that uses {nightly}{py} would create 4 Dockerfiles +# (which could become images or concrete Dockerfiles), because the "nightly" +# and "py" slice sets both have two entries: +# +# - nightly (no -py2 because the Python 2 slice set has add_to_name: "" +# - nightly-py3 +# - nightly-gpu (similar) +# - nightly-gpu-py3 -partials: - ubuntu: - desc: Start from Ubuntu (no GPU support) - args: - UBUNTU_VERSION: 16.04 +releases: + nightly: + tag_specs: + - "{nightly}{py}{jupyter}" - ubuntu-devel: - desc: Start from Ubuntu, with TF development packages (no GPU support) - args: - UBUNTU_VERSION: 16.04 + versioned: + tag_specs: + - "{_TAG_PREFIX}{ubuntu}{py}{jupyter}" - bazel: - desc: Install the latest version of Bazel and Python development tools. + ubuntu-dockerfiles: + is_dockerfiles: true + upload_images: false + tag_specs: + - "{ubuntu}{jupyter}" - nvidia: - desc: NVIDIA with CUDA and CuDNN, no dev stuff - args: - UBUNTU_VERSION: 16.04 +slice_sets: - nvidia-devel: - desc: > - Start from Nvidia's Ubuntu base image with CUDA and CuDNN, with TF - development packages. - args: - UBUNTU_VERSION: 16.04 + py: + - add_to_name: "" + args: + - USE_PYTHON_3_NOT_2= + - add_to_name: "-py3" + args: + - USE_PYTHON_3_NOT_2=1 - python: - desc: Python is required for TensorFlow and other libraries. - args: - USE_PYTHON_3_NOT_2: - default: true - desc: Install python 3 over Python 2 - - tensorflow: - desc: Install the TensorFlow Python package. - args: - TF_PACKAGE: - default: tensorflow - options: - - tensorflow - - tensorflow-gpu - - tf-nightly - - tf-nightly-gpu - desc: The specific TensorFlow Python package to install - shell: - desc: Configure TensorFlow's shell prompt and login tools. jupyter: - desc: Launch Jupyter on execution instead of a bash prompt. + - add_to_name: "" + - add_to_name: "-jupyter" + partials: + - jupyter -# ====== -# IMAGES -# ====== -# -# Represent Dockerfiles. Spec: -# -# name: the name of the image, possibly referenced by other images -# desc: A description, inserted later into the Dockerfile -# create-dockerfile: Create a dockerfile based on this. Useful for creating -# extensible base images that don't need a file. Default is true. -# partials: List of VALUEs, where a VALUE is either: -# - the name of a partial, which inserts that partial into this image -# - image: [name of another image], which inserts the partials from that -# image into this image -# arg-defaults: List of VALUEs, where a VALUE is either: -# - ARG_NAME: VALUE, which sets the ARG_NAME to VALUE wherever it appears -# in this image's partials -# - [name of another image], which loads the default args from that image -images: + ubuntu: + - add_to_name: "" + dockerfile_exclusive_name: "cpu" + partials: + - ubuntu/version + - ubuntu/cpu + - ubuntu/python + - tensorflow + - shell + - add_to_name: "-gpu" + dockerfile_exclusive_name: "gpu" + args: + - TF_PACKAGE=tensorflow-gpu + partials: + - ubuntu/version + - ubuntu/nvidia + - ubuntu/python + - tensorflow + - shell + tests: + - import-gpu.sh + test_runtime: nvidia + - add_to_name: "-devel" + dockerfile_exclusive_name: "cpu-devel" + partials: + - ubuntu/version + - ubuntu/cpu-devel + - ubuntu/python + - ubuntu/bazel + - shell + tests: + - build-cpu.sh + - add_to_name: "-gpu-devel" + dockerfile_exclusive_name: "gpu-devel" + partials: + - ubuntu/version + - ubuntu/nvidia-devel + - ubuntu/python + - ubuntu/bazel + - shell + tests: + - build-gpu.sh + test_runtime: nvidia - nodev: - create-dockerfile: false - partials: - - python - - tensorflow - - shell - - dev: - create-dockerfile: false - partials: - - python - - bazel - - shell - - cpu: - desc: Ubuntu-based, CPU-only environment for using TensorFlow - partials: - - ubuntu - - image: nodev - - cpu-devel: - desc: > - Ubuntu-based, CPU-only environment for developing changes for - TensorFlow. - partials: - - ubuntu-devel - - image: dev - - nvidia: - desc: Ubuntu-based, Nvidia-GPU-enabled environment for using TensorFlow. - arg-defaults: - - TF_PACKAGE: tensorflow-gpu - partials: - - nvidia - - image: nodev - - nvidia-devel: - desc: > - Ubuntu-based, Nvidia-GPU-enabled environment for developing changes - for TensorFlow. - arg-defaults: - - TF_PACKAGE: tensorflow-gpu - partials: - - nvidia-devel - - image: dev - - cpu-jupyter: - desc: > - Ubuntu-based, CPU-only environment for using TensorFlow, with Jupyter - included. - partials: - - image: cpu - - jupyter - - cpu-devel-jupyter: - desc: > - Ubuntu-based, CPU-only environment for developing changes for - TensorFlow, with Jupyter included. - partials: - - image: cpu-devel - - jupyter - - nvidia-jupyter: - desc: > - Ubuntu-based, Nvidia-GPU-enabled environment for using TensorFlow, with - Jupyter included. - arg-defaults: - - nvidia - partials: - - image: nvidia - - jupyter - - nvidia-devel-jupyter: - desc: > - Ubuntu-based, Nvidia-GPU-enabled environment for developing changes for - TensorFlow, with Jupyter included. - arg-defaults: - - nvidia-devel - partials: - - image: nvidia-devel - - jupyter + nightly: + - add_to_name: "nightly" + partials: + - ubuntu/version + - ubuntu/cpu + - ubuntu/python + - tensorflow + - shell + args: + - TF_PACKAGE=tf-nightly + tests: + - import.sh + - add_to_name: "nightly-gpu" + partials: + - ubuntu/version + - ubuntu/nvidia + - ubuntu/python + - tensorflow + - shell + test_runtime: nvidia + tests: + - import-gpu.sh + args: + - TF_PACKAGE=tf-nightly-gpu diff --git a/tensorflow/tools/dockerfiles/tests/build-cpu.sh b/tensorflow/tools/dockerfiles/tests/build-cpu.sh new file mode 100755 index 00000000000..bcdc4c2139c --- /dev/null +++ b/tensorflow/tools/dockerfiles/tests/build-cpu.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash + +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +# Download and build TensorFlow. +set -euxo pipefail +git clone --branch=master --depth=1 https://github.com/tensorflow/tensorflow.git /tensorflow +cd /tensorflow + +ln -s $(which ${PYTHON}) /usr/local/bin/python + +# For optimized builds appropriate for the hardware platform of your choosing, uncomment below... +# For ivy-bridge or sandy-bridge +# --copt=-march="ivybridge" \ +# for haswell, broadwell, or skylake +# --copt=-march="haswell" \ +tensorflow/tools/ci_build/builds/configured CPU \ + bazel build -c opt --copt=-mavx --cxxopt="-D_GLIBCXX_USE_CXX11_ABI=0" \ + tensorflow/tools/pip_package:build_pip_package && \ + bazel-bin/tensorflow/tools/pip_package/build_pip_package /tmp/pip && \ + pip --no-cache-dir install --upgrade /tmp/pip/tensorflow-*.whl && \ + rm -rf /tmp/pip && \ + rm -rf /root/.cache + diff --git a/tensorflow/tools/dockerfiles/tests/build-gpu.sh b/tensorflow/tools/dockerfiles/tests/build-gpu.sh new file mode 100755 index 00000000000..76b25d5a741 --- /dev/null +++ b/tensorflow/tools/dockerfiles/tests/build-gpu.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash + +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +# Download and build TensorFlow. +set -euxo pipefail +git clone --branch=master --depth=1 https://github.com/tensorflow/tensorflow.git /tensorflow +cd /tensorflow + +ln -s $(which ${PYTHON}) /usr/local/bin/python + +ln -s /usr/local/cuda/lib64/stubs/libcuda.so /usr/local/cuda/lib64/stubs/libcuda.so.1 + +LD_LIBRARY_PATH=/usr/local/cuda/lib64/stubs:${LD_LIBRARY_PATH} \ +tensorflow/tools/ci_build/builds/configured GPU \ +bazel build -c opt --copt=-mavx --config=cuda \ + --cxxopt="-D_GLIBCXX_USE_CXX11_ABI=0" \ + tensorflow/tools/pip_package:build_pip_package && \ +rm /usr/local/cuda/lib64/stubs/libcuda.so.1 && \ +bazel-bin/tensorflow/tools/pip_package/build_pip_package /tmp/pip && \ +pip --no-cache-dir install --upgrade /tmp/pip/tensorflow-*.whl && \ +rm -rf /tmp/pip && \ +rm -rf /root/.cache diff --git a/tensorflow/tools/dockerfiles/tests/import-gpu.sh b/tensorflow/tools/dockerfiles/tests/import-gpu.sh new file mode 100755 index 00000000000..6559210dcbf --- /dev/null +++ b/tensorflow/tools/dockerfiles/tests/import-gpu.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash + +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +python -c 'import tensorflow as tf; tf.test.is_gpu_available() or exit(1)' diff --git a/tensorflow/tools/dockerfiles/tests/import.sh b/tensorflow/tools/dockerfiles/tests/import.sh new file mode 100755 index 00000000000..b73bd86a852 --- /dev/null +++ b/tensorflow/tools/dockerfiles/tests/import.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash + +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +set -euxo pipefail +python -c 'import tensorflow as tf' diff --git a/tensorflow/tools/dockerfiles/assembler.Dockerfile b/tensorflow/tools/dockerfiles/tools.Dockerfile similarity index 95% rename from tensorflow/tools/dockerfiles/assembler.Dockerfile rename to tensorflow/tools/dockerfiles/tools.Dockerfile index 7a8e07fced3..e8929295a5e 100644 --- a/tensorflow/tools/dockerfiles/assembler.Dockerfile +++ b/tensorflow/tools/dockerfiles/tools.Dockerfile @@ -20,8 +20,9 @@ FROM debian:stretch LABEL maintainer="Austin Anderson " -RUN apt-get update && apt-get install -y python3 python3-pip bash -RUN pip3 install --upgrade pip setuptools pyyaml absl-py cerberus +RUN apt-get update && apt-get install -y python3 python3-pip bash curl +RUN curl -sSL https://get.docker.com/ | sh +RUN pip3 install --upgrade pip setuptools pyyaml absl-py cerberus docker WORKDIR /tf VOLUME ["/tf"] diff --git a/tensorflow/tools/docs/BUILD b/tensorflow/tools/docs/BUILD index 1a53f241773..b072853a4ec 100644 --- a/tensorflow/tools/docs/BUILD +++ b/tensorflow/tools/docs/BUILD @@ -142,15 +142,28 @@ py_test( ], ) -py_binary( - name = "generate_1_0", - srcs = ["generate_1_0.py"], +py_test( + name = "generate2_test", + srcs = ["generate2_test.py"], srcs_version = "PY2AND3", - deps = [ - ":generate_lib", - "//tensorflow:tensorflow_py", - "//tensorflow/python/debug:debug_py", + tags = [ + "manual", + # No reason to run sanitizers or fastbuild for this test. + "noasan", + "nomsan", + "notsan", + "optonly", ], + deps = [ + ":generate2", + ], +) + +py_binary( + name = "generate2", + srcs = ["generate2.py"], + srcs_version = "PY2AND3", + deps = ["//tensorflow:tensorflow_py"], ) py_library( diff --git a/tensorflow/tools/docs/generate2.py b/tensorflow/tools/docs/generate2.py new file mode 100644 index 00000000000..fba909d26de --- /dev/null +++ b/tensorflow/tools/docs/generate2.py @@ -0,0 +1,82 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +r"""A tool to generate api_docs for TensorFlow2. + +``` +python generate2.py --output_dir=/tmp/out +``` + +Requires a local installation of: + https://github.com/tensorflow/docs/tree/master/tools + tf-nightly-2.0-preview +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from os import path + +from absl import app +from absl import flags + +import tensorflow as tf + +from tensorflow_docs.api_generator import generate_lib + +FLAGS = flags.FLAGS + +flags.DEFINE_string( + "code_url_prefix", + "/code/stable/tensorflow/", + "A url to prepend to code paths when creating links to defining code") + +flags.DEFINE_string( + "output_dir", "/tmp/out", + "A directory, where the docs will be output to.") + +flags.DEFINE_bool("search_hints", True, + "Include meta-data search hints at the top of each file.") + + +def build_docs(output_dir, code_url_prefix, search_hints=True): + """Build api docs for tensorflow v2. + + Args: + output_dir: A string path, where to put the files. + code_url_prefix: prefix for "Defined in" links. + search_hints: Bool. Include meta-data search hints at the top of each file. + """ + base_dir = path.dirname(tf.__file__) + doc_generator = generate_lib.DocGenerator( + root_title="TensorFlow 2.0 Preview", + py_modules=[("tf", tf)], + base_dir=base_dir, + search_hints=search_hints, + code_url_prefix=code_url_prefix, + site_path="api_docs/") + + doc_generator.build(output_dir) + + +def main(argv): + del argv + build_docs(output_dir=FLAGS.output_dir, + code_url_prefix=FLAGS.code_url_prefix, + search_hints=FLAGS.search_hints) + + +if __name__ == "__main__": + app.run(main) diff --git a/tensorflow/contrib/estimator/python/estimator/linear.py b/tensorflow/tools/docs/generate2_test.py similarity index 60% rename from tensorflow/contrib/estimator/python/estimator/linear.py rename to tensorflow/tools/docs/generate2_test.py index b6a4444f66c..774d45c536b 100644 --- a/tensorflow/contrib/estimator/python/estimator/linear.py +++ b/tensorflow/tools/docs/generate2_test.py @@ -12,21 +12,28 @@ # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================== -"""linear python module. - -Importing from tensorflow.python.estimator is unsupported -and will soon break! -""" -# pylint: disable=unused-import,g-bad-import-order,g-import-not-at-top,wildcard-import +"""Tests for tensorflow.tools.docs.generate2.""" from __future__ import absolute_import from __future__ import division from __future__ import print_function -from tensorflow_estimator.contrib.estimator.python.estimator import linear +import os +import shutil -# Include attrs that start with single underscore. -_HAS_DYNAMIC_ATTRIBUTES = True -linear.__all__ = [s for s in dir(linear) if not s.startswith('__')] +from tensorflow.python.platform import googletest +from tensorflow.tools.docs import generate2 -from tensorflow_estimator.contrib.estimator.python.estimator.linear import * + +class Generate2Test(googletest.TestCase): + + def test_end_to_end(self): + output_dir = os.path.join(googletest.GetTempDir(), 'output') + if os.path.exists(output_dir): + shutil.rmtree(output_dir) + os.makedirs(output_dir) + generate2.build_docs(output_dir=output_dir, code_url_prefix='') + + +if __name__ == '__main__': + googletest.main() diff --git a/tensorflow/tools/docs/generate_1_0.py b/tensorflow/tools/docs/generate_1_0.py deleted file mode 100644 index f4384e0ced7..00000000000 --- a/tensorflow/tools/docs/generate_1_0.py +++ /dev/null @@ -1,92 +0,0 @@ -# Copyright 2015 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Generate docs for the TensorFlow Python API.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import os -import sys - -import tensorflow as tf - -from tensorflow.python import debug as tf_debug -from tensorflow.python.util import tf_inspect -from tensorflow.tools.docs import generate_lib - -if __name__ == '__main__': - doc_generator = generate_lib.DocGenerator() - doc_generator.add_output_dir_argument() - doc_generator.add_src_dir_argument() - - # This doc generator works on the TensorFlow codebase. Since this script lives - # at tensorflow/tools/docs, and all code is defined somewhere inside - # tensorflow/, we can compute the base directory (two levels up), which is - # valid unless we're trying to apply this to a different code base, or are - # moving the script around. - script_dir = os.path.dirname(tf_inspect.getfile(tf_inspect.currentframe())) - default_base_dir = os.path.join(script_dir, '..', '..') - doc_generator.add_base_dir_argument(default_base_dir) - - flags = doc_generator.parse_known_args() - - # tf_debug is not imported with tf, it's a separate module altogether - doc_generator.set_py_modules([('tf', tf), ('tfdbg', tf_debug)]) - - doc_generator.set_do_not_descend_map({ - 'tf': ['cli', 'lib', 'wrappers'], - 'tf.contrib': [ - 'compiler', - 'factorization', - 'grid_rnn', - 'labeled_tensor', - 'quantization', - 'session_bundle', - 'slim', - 'solvers', - 'specs', - 'tensor_forest', - 'tensorboard', - 'testing', - 'training', - 'tfprof', - ], - 'tf.contrib.bayesflow': [ - 'entropy', 'monte_carlo', 'special_math', - 'stochastic_gradient_estimators', 'stochastic_graph', - 'stochastic_tensor', 'stochastic_variables', 'variational_inference' - ], - 'tf.contrib.distributions': ['bijector'], - 'tf.contrib.ffmpeg': ['ffmpeg_ops'], - 'tf.contrib.graph_editor': [ - 'edit', 'match', 'reroute', 'subgraph', 'transform', 'select', 'util' - ], - 'tf.contrib.layers': ['feature_column', 'summaries'], - 'tf.contrib.learn': [ - 'datasets', - 'head', - 'graph_actions', - 'io', - 'models', - 'monitors', - 'ops', - 'preprocessing', - 'utils', - ], - 'tf.contrib.util': ['loader'], - }) - - sys.exit(doc_generator.build(flags)) diff --git a/tensorflow/tools/docs/parser.py b/tensorflow/tools/docs/parser.py index 83b4bf81288..6dc18ee8dc9 100644 --- a/tensorflow/tools/docs/parser.py +++ b/tensorflow/tools/docs/parser.py @@ -39,7 +39,7 @@ def is_free_function(py_object, full_name, index): """Check if input is a free function (and not a class- or static method). Args: - py_object: The the object in question. + py_object: The object in question. full_name: The full name of the object, like `tf.module.symbol`. index: The {full_name:py_object} dictionary for the public API. diff --git a/tensorflow/tools/lib_package/BUILD b/tensorflow/tools/lib_package/BUILD index 7aaa845ae92..1cac5ee1383 100644 --- a/tensorflow/tools/lib_package/BUILD +++ b/tensorflow/tools/lib_package/BUILD @@ -112,6 +112,7 @@ pkg_tar( genrule( name = "clicenses_generate", srcs = [ + "//third_party/icu/data:LICENSE", "//third_party/hadoop:LICENSE.txt", "//third_party/eigen3:LICENSE", "//third_party/fft2d:LICENSE", @@ -180,6 +181,7 @@ genrule( genrule( name = "jnilicenses_generate", srcs = [ + "//third_party/icu/data:LICENSE", "//third_party/hadoop:LICENSE.txt", "//third_party/eigen3:LICENSE", "//third_party/fft2d:LICENSE", diff --git a/tensorflow/tools/pip_package/BUILD b/tensorflow/tools/pip_package/BUILD index 3a863d3c523..82c6bf383fb 100644 --- a/tensorflow/tools/pip_package/BUILD +++ b/tensorflow/tools/pip_package/BUILD @@ -88,6 +88,9 @@ COMMON_PIP_DEPS = [ "//tensorflow/contrib/timeseries:timeseries_pip", "//tensorflow/contrib/tpu", "//tensorflow/examples/tutorials/mnist:package", + "//tensorflow/lite/python:interpreter_test_data", + "//tensorflow/lite/python:tflite_convert", + "//tensorflow/lite/toco/python:toco_from_protos", # "//tensorflow/python/autograph/converters:converters", # "//tensorflow/python/autograph/core:core", "//tensorflow/python/autograph/core:test_lib", @@ -124,7 +127,7 @@ COMMON_PIP_DEPS = [ py_binary( name = "simple_console_for_windows", srcs = ["simple_console_for_windows.py"], - data = COMMON_PIP_DEPS, + data = COMMON_PIP_DEPS + ["//tensorflow/python:pywrap_tensorflow_import_lib_file"], srcs_version = "PY2AND3", deps = ["//tensorflow:tensorflow_py"], ) @@ -132,6 +135,7 @@ py_binary( filegroup( name = "licenses", data = [ + "//third_party/icu/data:LICENSE", "//third_party/eigen3:LICENSE", "//third_party/fft2d:LICENSE", "//third_party/hadoop:LICENSE.txt", @@ -227,15 +231,9 @@ sh_binary( data = select({ "//tensorflow:windows": [ ":simple_console_for_windows", - "//tensorflow/lite/python:interpreter_test_data", - "//tensorflow/lite/python:tflite_convert", - "//tensorflow/lite/toco/python:toco_from_protos", ], "//conditions:default": COMMON_PIP_DEPS + [ ":simple_console", - "//tensorflow/lite/python:interpreter_test_data", - "//tensorflow/lite/python:tflite_convert", - "//tensorflow/lite/toco/python:toco_from_protos", ], }) + if_mkl_ml(["//third_party/mkl:intel_binary_blob"]), ) diff --git a/tensorflow/tools/pip_package/setup.py b/tensorflow/tools/pip_package/setup.py index 07475cc0c4d..85c913f1588 100644 --- a/tensorflow/tools/pip_package/setup.py +++ b/tensorflow/tools/pip_package/setup.py @@ -87,7 +87,8 @@ if 'tf_nightly' in project_name: for i, pkg in enumerate(REQUIRED_PACKAGES): if 'tensorboard' in pkg: REQUIRED_PACKAGES[i] = 'tb-nightly >= 1.13.0a0, < 1.14.0a0' - break + if 'tensorflow_estimator' in pkg: + REQUIRED_PACKAGES[i] = 'tf-estimator-nightly' # weakref.finalize and enum were introduced in Python 3.4 if sys.version_info < (3, 4): @@ -106,6 +107,7 @@ CONSOLE_SCRIPTS = [ # TensorBoard command, pip will inappropriately remove it during install, # even though the command is not removed, just moved to a different wheel. 'tensorboard = tensorboard.main:run_main', + 'tf_upgrade_v2 = tensorflow.tools.compatibility.tf_upgrade_v2_main:main', ] # pylint: enable=line-too-long diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl index 6d3562caef6..7e5f84be165 100755 --- a/tensorflow/workspace.bzl +++ b/tensorflow/workspace.bzl @@ -77,31 +77,31 @@ def tf_workspace(path_prefix = "", tf_repo_name = ""): mkl_repository( name = "mkl_linux", build_file = clean_dep("//third_party/mkl:mkl.BUILD"), - sha256 = "e2233534a9d15c387e22260997af4312a39e9f86f791768409be273b5453c4e6", - strip_prefix = "mklml_lnx_2019.0.20180710", + sha256 = "f00dc3b142a5be399bdeebd7e7ea369545a35d4fb84c86f98b6b048d72685295", + strip_prefix = "mklml_lnx_2019.0.1.20180928", urls = [ - "https://mirror.bazel.build/github.com/intel/mkl-dnn/releases/download/v0.16/mklml_lnx_2019.0.20180710.tgz", - "https://github.com/intel/mkl-dnn/releases/download/v0.16/mklml_lnx_2019.0.20180710.tgz", + "https://mirror.bazel.build/github.com/intel/mkl-dnn/releases/download/v0.17-rc/mklml_lnx_2019.0.1.20180928.tgz", + "https://github.com/intel/mkl-dnn/releases/download/v0.17-rc/mklml_lnx_2019.0.1.20180928.tgz", ], ) mkl_repository( name = "mkl_windows", build_file = clean_dep("//third_party/mkl:mkl.BUILD"), - sha256 = "3fdcff17b018a0082491adf3ba143358265336a801646e46e0191ec8d58d24a2", - strip_prefix = "mklml_win_2019.0.20180710", + sha256 = "efef90b7b9613fab10f44c8ac4ff28db613a112c64ed94826d7e44df09c44b0b", + strip_prefix = "mklml_win_2019.0.1.20180928", urls = [ - "https://mirror.bazel.build/github.com/intel/mkl-dnn/releases/download/v0.16/mklml_win_2019.0.20180710.zip", - "https://github.com/intel/mkl-dnn/releases/download/v0.16/mklml_win_2019.0.20180710.zip", + "https://mirror.bazel.build/github.com/intel/mkl-dnn/releases/download/v0.17-rc/mklml_win_2019.0.1.20180928.zip", + "https://github.com/intel/mkl-dnn/releases/download/v0.17-rc/mklml_win_2019.0.1.20180928.zip", ], ) mkl_repository( name = "mkl_darwin", build_file = clean_dep("//third_party/mkl:mkl.BUILD"), - sha256 = "411a30014a938eb83fb9f37b3dbe8e371b106fc1dd621fc23123cadc72737ce6", - strip_prefix = "mklml_mac_2019.0.20180710", + sha256 = "83f02938a0c095274db7b8b7b694157abafa3837c5cbaef740440d466c86a477", + strip_prefix = "mklml_mac_2019.0.1.20180928", urls = [ - "https://mirror.bazel.build/github.com/intel/mkl-dnn/releases/download/v0.16/mklml_mac_2019.0.20180710.tgz", - "https://github.com/intel/mkl-dnn/releases/download/v0.16/mklml_mac_2019.0.20180710.tgz", + "https://mirror.bazel.build/github.com/intel/mkl-dnn/releases/download/v0.17-rc/mklml_mac_2019.0.1.20180928.tgz", + "https://github.com/intel/mkl-dnn/releases/download/v0.17-rc/mklml_mac_2019.0.1.20180928.tgz", ], ) @@ -112,11 +112,11 @@ def tf_workspace(path_prefix = "", tf_repo_name = ""): tf_http_archive( name = "mkl_dnn", build_file = clean_dep("//third_party/mkl_dnn:mkldnn.BUILD"), - sha256 = "363cc9239eacf8e7917753c6d8c94f767e4cd049160d0654a61ef32d5e1b3049", - strip_prefix = "mkl-dnn-4e333787e0d66a1dca1218e99a891d493dbc8ef1", + sha256 = "b100f57af4a2b59a3a37a1ba38f77b644d2107d758a1a7f4e51310063cd21e73", + strip_prefix = "mkl-dnn-733fc908874c71a5285043931a1cf80aa923165c", urls = [ - "https://mirror.bazel.build/github.com/intel/mkl-dnn/archive/4e333787e0d66a1dca1218e99a891d493dbc8ef1.tar.gz", - "https://github.com/intel/mkl-dnn/archive/4e333787e0d66a1dca1218e99a891d493dbc8ef1.tar.gz", + "https://mirror.bazel.build/github.com/intel/mkl-dnn/archive/733fc908874c71a5285043931a1cf80aa923165c.tar.gz", + "https://github.com/intel/mkl-dnn/archive/733fc908874c71a5285043931a1cf80aa923165c.tar.gz", ], ) @@ -134,12 +134,11 @@ def tf_workspace(path_prefix = "", tf_repo_name = ""): tf_http_archive( name = "eigen_archive", build_file = clean_dep("//third_party:eigen.BUILD"), - patch_file = clean_dep("//third_party:eigen_reshaped.patch"), - sha256 = "d66cec3b54b3dfaa4666c1d49481a7197f93fc078cd53c54e2b4a8893a529c9f", - strip_prefix = "eigen-eigen-b4890dc6bc34", + sha256 = "37a483ec219c43219b6e0fc07e799277a4a36abb2b9f4162cfcd256aa211eae8", + strip_prefix = "eigen-eigen-2e50f4a5542a", urls = [ - "https://mirror.bazel.build/bitbucket.org/eigen/eigen/get/b4890dc6bc34.tar.gz", - "https://bitbucket.org/eigen/eigen/get/b4890dc6bc34.tar.gz", + "https://mirror.bazel.build/bitbucket.org/eigen/eigen/get/2e50f4a5542a.tar.gz", + "https://bitbucket.org/eigen/eigen/get/2e50f4a5542a.tar.gz", ], ) @@ -180,15 +179,15 @@ def tf_workspace(path_prefix = "", tf_repo_name = ""): tf_http_archive( name = "com_github_googlecloudplatform_google_cloud_cpp", - sha256 = "fdd3b3aecce60987e5525e55bf3a21d68a8695320bd5b980775af6507eec3944", - strip_prefix = "google-cloud-cpp-14760a86c4ffab9943b476305c4fe927ad95db1c", + sha256 = "3ade2072e6588ff56c0434abe6c63aa5f3f2d56be15a299bafc7e9cdf0a12c17", + strip_prefix = "google-cloud-cpp-0.3.0", system_build_file = clean_dep("//third_party/systemlibs:google_cloud_cpp.BUILD"), system_link_files = { "//third_party/systemlibs:google_cloud_cpp.google.cloud.bigtable.BUILD": "google/cloud/bigtable/BUILD", }, urls = [ - "https://mirror.bazel.build/github.com/GoogleCloudPlatform/google-cloud-cpp/archive/14760a86c4ffab9943b476305c4fe927ad95db1c.tar.gz", - "https://github.com/GoogleCloudPlatform/google-cloud-cpp/archive/14760a86c4ffab9943b476305c4fe927ad95db1c.tar.gz", + "https://mirror.bazel.build/github.com/GoogleCloudPlatform/google-cloud-cpp/archive/v0.3.0.tar.gz", + "https://github.com/GoogleCloudPlatform/google-cloud-cpp/archive/v0.3.0.tar.gz", ], ) @@ -348,11 +347,11 @@ def tf_workspace(path_prefix = "", tf_repo_name = ""): ) PROTOBUF_URLS = [ - "https://mirror.bazel.build/github.com/google/protobuf/archive/v3.6.1.tar.gz", - "https://github.com/google/protobuf/archive/v3.6.1.tar.gz", + "https://mirror.bazel.build/github.com/google/protobuf/archive/v3.6.1.1.tar.gz", + "https://github.com/google/protobuf/archive/v3.6.1.1.tar.gz", ] - PROTOBUF_SHA256 = "3d4e589d81b2006ca603c1ab712c9715a76227293032d05b26fca603f90b3f5b" - PROTOBUF_STRIP_PREFIX = "protobuf-3.6.1" + PROTOBUF_SHA256 = "1ade182f91f0fa6c6116195def5d22270e01b9d03fe91319e4c6215022d0d24b" + PROTOBUF_STRIP_PREFIX = "protobuf-3.6.1.1" tf_http_archive( name = "protobuf_archive", @@ -473,11 +472,11 @@ def tf_workspace(path_prefix = "", tf_repo_name = ""): tf_http_archive( name = "llvm", build_file = clean_dep("//third_party/llvm:llvm.autogenerated.BUILD"), - sha256 = "a22a9b4c3af50a52ba0015b6987bba7202c3ec8e1d40ae76ee7d7643638936ae", - strip_prefix = "llvm-b4ace5f3454131a3070ef7c11e19e42fc9a80b4e", + sha256 = "34170a4aa07e434dd537d98a705dcf1b3901f73820fe1d6b9370e8c1c94e9157", + strip_prefix = "llvm-0487bd8f42c8b38166ff825d56014d0ff49db604", urls = [ - "https://mirror.bazel.build/github.com/llvm-mirror/llvm/archive/b4ace5f3454131a3070ef7c11e19e42fc9a80b4e.tar.gz", - "https://github.com/llvm-mirror/llvm/archive/b4ace5f3454131a3070ef7c11e19e42fc9a80b4e.tar.gz", + "https://mirror.bazel.build/github.com/llvm-mirror/llvm/archive/0487bd8f42c8b38166ff825d56014d0ff49db604.tar.gz", + "https://github.com/llvm-mirror/llvm/archive/0487bd8f42c8b38166ff825d56014d0ff49db604.tar.gz", ], ) @@ -690,11 +689,11 @@ def tf_workspace(path_prefix = "", tf_repo_name = ""): tf_http_archive( name = "arm_neon_2_x86_sse", build_file = clean_dep("//third_party:arm_neon_2_x86_sse.BUILD"), - sha256 = "c8d90aa4357f8079d427e87a6f4c493da1fa4140aee926c05902d7ec1533d9a5", - strip_prefix = "ARM_NEON_2_x86_SSE-0f77d9d182265259b135dad949230ecbf1a2633d", + sha256 = "213733991310b904b11b053ac224fee2d4e0179e46b52fe7f8735b8831e04dcc", + strip_prefix = "ARM_NEON_2_x86_SSE-1200fe90bb174a6224a525ee60148671a786a71f", urls = [ - "https://mirror.bazel.build/github.com/intel/ARM_NEON_2_x86_SSE/archive/0f77d9d182265259b135dad949230ecbf1a2633d.tar.gz", - "https://github.com/intel/ARM_NEON_2_x86_SSE/archive/0f77d9d182265259b135dad949230ecbf1a2633d.tar.gz", + "https://mirror.bazel.build/github.com/intel/ARM_NEON_2_x86_SSE/archive/1200fe90bb174a6224a525ee60148671a786a71f.tar.gz", + "https://github.com/intel/ARM_NEON_2_x86_SSE/archive/1200fe90bb174a6224a525ee60148671a786a71f.tar.gz", ], ) diff --git a/third_party/clang_toolchain/download_clang.bzl b/third_party/clang_toolchain/download_clang.bzl index 9023e250b2f..7ced9027473 100644 --- a/third_party/clang_toolchain/download_clang.bzl +++ b/third_party/clang_toolchain/download_clang.bzl @@ -39,15 +39,15 @@ def download_clang(repo_ctx, out_folder): # Latest CLANG_REVISION and CLANG_SUB_REVISION of the Chromiums's release # can be found in https://chromium.googlesource.com/chromium/src/tools/clang/+/master/scripts/update.py - CLANG_REVISION = "346388" + CLANG_REVISION = "347933" CLANG_SUB_REVISION = 1 package_version = "%s-%s" % (CLANG_REVISION, CLANG_SUB_REVISION) checksums = { - "Linux_x64": "5e5564e4e743414c7eaec9fd9e739732ddd2a343e49bde4c88fc2530b1c598b9", - "Mac": "19271a7cc5c2bcaf9643d3dd622b5458569dc662bbc58f63b129cf6e3a4e3243", - "Win": "60b0bd1f11e53892109f4159e2aba0f803604823e07875ca98b82bd5628d7f4d", + "Linux_x64": "cae3643fdf5d46fc9bc8731212bb37573547148d90b64b083165e090133d11b0", + "Mac": "083a0e91a38c06e568652313ac7372b17a101268f7d65533d721ca30413442b4", + "Win": "43160487cfc7e88076a369a2b6e8e4a0f42e104c28d8903f3aaa62d630aba949", } platform_folder = _get_platform_folder(repo_ctx.os.name) diff --git a/third_party/eigen_reshaped.patch b/third_party/eigen_reshaped.patch deleted file mode 100644 index 7acfdcf9fef..00000000000 --- a/third_party/eigen_reshaped.patch +++ /dev/null @@ -1,48 +0,0 @@ ---- a/Eigen/src/Core/util/ReshapedHelper.h (date 1541195478000) -+++ b/Eigen/src/Core/util/ReshapedHelper.h (date 1541195478000) -@@ -39,6 +39,11 @@ - return total/other; - } - -+template -+struct get_compiletime_reshape_order { -+ enum { value = Order == AutoOrder ? Flags & RowMajorBit : Order }; -+}; -+ - } - - } // end namespace Eigen ---- a/Eigen/src/plugins/ReshapedMethods.h (date 1541195254000) -+++ b/Eigen/src/plugins/ReshapedMethods.h (date 1541195254000) -@@ -105,13 +105,13 @@ - inline Reshaped::value, - internal::get_compiletime_reshape_size::value, -- (Order==AutoOrder?Flags&RowMajorBit:Order)> -+ internal::get_compiletime_reshape_order::value> - reshaped(NRowsType nRows, NColsType nCols) EIGEN_RESHAPED_METHOD_CONST - { - return Reshaped::value, - internal::get_compiletime_reshape_size::value, -- (Order==AutoOrder?Flags&RowMajorBit:Order)> -+ internal::get_compiletime_reshape_order::value> - (derived(), - internal::get_runtime_reshape_size(nRows,internal::get_runtime_value(nCols),size()), - internal::get_runtime_reshape_size(nCols,internal::get_runtime_value(nRows),size())); -@@ -128,11 +128,13 @@ - - template - EIGEN_DEVICE_FUNC --inline Reshaped -+inline Reshaped::value> - reshaped() EIGEN_RESHAPED_METHOD_CONST - { - EIGEN_STATIC_ASSERT(Order==RowMajor || Order==ColMajor || Order==AutoOrder, INVALID_TEMPLATE_PARAMETER); -- return Reshaped -+ return Reshaped::value> - (derived(), size(), 1); - } - \ No newline at end of file diff --git a/third_party/googleapis.BUILD b/third_party/googleapis.BUILD index 95e999af188..b8871eda728 100644 --- a/third_party/googleapis.BUILD +++ b/third_party/googleapis.BUILD @@ -13,7 +13,9 @@ # limitations under the License. package(default_visibility = ["//visibility:public"]) + licenses(["notice"]) # Apache 2.0 + exports_files(["LICENSE"]) load("@protobuf_archive//:protobuf.bzl", "cc_proto_library") @@ -21,6 +23,9 @@ load("@protobuf_archive//:protobuf.bzl", "cc_proto_library") cc_proto_library( name = "bigtable_protos", srcs = [ + "google/api/annotations.proto", + "google/api/auth.proto", + "google/api/http.proto", "google/bigtable/admin/v2/bigtable_instance_admin.proto", "google/bigtable/admin/v2/bigtable_table_admin.proto", "google/bigtable/admin/v2/common.proto", @@ -31,15 +36,12 @@ cc_proto_library( "google/iam/v1/iam_policy.proto", "google/iam/v1/policy.proto", "google/longrunning/operations.proto", - "google/rpc/status.proto", "google/rpc/error_details.proto", - "google/api/annotations.proto", - "google/api/auth.proto", - "google/api/http.proto", + "google/rpc/status.proto", ], include = ".", - protoc = "@protobuf_archive//:protoc", default_runtime = "@protobuf_archive//:protobuf", - deps = ["@protobuf_archive//:cc_wkt_protos"], + protoc = "@protobuf_archive//:protoc", use_grpc_plugin = True, + deps = ["@protobuf_archive//:cc_wkt_protos"], ) diff --git a/third_party/gpus/crosstool/CROSSTOOL.tpl b/third_party/gpus/crosstool/CROSSTOOL.tpl index 3189cf8e316..921188cbb43 100644 --- a/third_party/gpus/crosstool/CROSSTOOL.tpl +++ b/third_party/gpus/crosstool/CROSSTOOL.tpl @@ -184,7 +184,8 @@ toolchain { action: "c++-link-dynamic-library" action: "c++-link-nodeps-dynamic-library" flag_group { - flag:"-no-canonical-prefixes" + flag: "-no-canonical-prefixes" + %{extra_no_canonical_prefixes_flags} } } } diff --git a/third_party/gpus/cuda_configure.bzl b/third_party/gpus/cuda_configure.bzl index 831a3067b24..03c67bcb3d7 100644 --- a/third_party/gpus/cuda_configure.bzl +++ b/third_party/gpus/cuda_configure.bzl @@ -1418,6 +1418,7 @@ def _create_local_cuda_repository(repository_ctx): flag: "-Wno-invalid-partial-specialization" """ cuda_defines["%{host_compiler_includes}"] = host_compiler_includes + cuda_defines["%{extra_no_canonical_prefixes_flags}"] = "" _tpl(repository_ctx, "crosstool:BUILD", { "%{linker_files}": ":empty", "%{win_linker_files}": ":empty" @@ -1439,6 +1440,14 @@ def _create_local_cuda_repository(repository_ctx): repository_ctx, cuda_config) + "\n cxx_builtin_include_directory: \"%s\"" % cupti_header_dir + "\n cxx_builtin_include_directory: \"%s\"" % cudnn_header_dir) + + # For gcc, do not canonicalize system header paths; some versions of gcc + # pick the shortest possible path for system includes when creating the + # .d file - given that includes that are prefixed with "../" multiple + # time quickly grow longer than the root of the tree, this can lead to + # bazel's header check failing. + cuda_defines["%{extra_no_canonical_prefixes_flags}"] = ( + "flag: \"-fno-canonical-system-headers\"") nvcc_path = str( repository_ctx.path("%s/bin/nvcc%s" % ( cuda_config.cuda_toolkit_path, diff --git a/third_party/gpus/rocm_configure.bzl b/third_party/gpus/rocm_configure.bzl index 9108639b0bf..6df6799bd76 100644 --- a/third_party/gpus/rocm_configure.bzl +++ b/third_party/gpus/rocm_configure.bzl @@ -105,7 +105,7 @@ def get_cxx_inc_directories(repository_ctx, cc): return includes_cpp + [ inc for inc in includes_c - if inc not in includes_cpp_set + if inc not in includes_cpp_set.to_list() ] def auto_configure_fail(msg): diff --git a/third_party/icu/data/BUILD.bazel b/third_party/icu/data/BUILD.bazel new file mode 100644 index 00000000000..7db21566e4e --- /dev/null +++ b/third_party/icu/data/BUILD.bazel @@ -0,0 +1,46 @@ +package( + default_visibility = ["//visibility:public"], +) + +licenses(["notice"]) # Apache 2.0 + +exports_files(["LICENSE"]) + +# Data for core MIME/Unix/Windows encodings: +# ISO 8859-2..9, 15; Windows-125x; EUC-CN; GBK (Windows cp936); GB 18030; +# Big5 (Windows cp950); SJIS (Windows cp932); EUC-JP; EUC-KR, KS C 5601; +# Windows cp949. Data is pre-processed for little-endian platforms. To replicate +# this pre-processing (if you want additional encodings, for example), do the +# following: +# +# First, download, build, and install ICU. This installs tools such as makeconv. +# Then, run the following from your icu4c/source directory: +# $ cd data/mappings +# $ rm *.cnv # there shouldn't be any .cnv files here to begin with +# $ grep \.ucm ucmcore.mk | \ +# sed 's/\(UCM_SOURCE_CORE=\)\?\([^ ]\+\.ucm\)\\\?/\2/g' | \ +# tr '\n' ' ' | xargs makeconv +# $ ls *.cnv > filelist.lst +# $ pkgdata -m common -p ucmcore filelist.lst +# $ genccode -f custom_conversion_data ucmcore.dat +# This creates custom_conversion_data.c. You will need to change the target +# :conversion_data to depend on your custom source instead of :conversion_data.c +filegroup( + name = "conversion_files", + srcs = glob(["icu_conversion_data.c.gz.*"]), +) + +# Data files are compressed and split to work around git performance degradation +# around large files. +genrule( + name = "merge_conversion_data", + srcs = [":conversion_files"], + outs = ["conversion_data.c"], + cmd = "cat $(locations :conversion_files) | gunzip > $@", +) + +cc_library( + name = "conversion_data", + srcs = [":conversion_data.c"], + deps = ["@icu//:headers"], +) diff --git a/third_party/icu/data/LICENSE b/third_party/icu/data/LICENSE new file mode 100644 index 00000000000..25b6eb9d341 --- /dev/null +++ b/third_party/icu/data/LICENSE @@ -0,0 +1,414 @@ +COPYRIGHT AND PERMISSION NOTICE (ICU 58 and later) + +Copyright © 1991-2018 Unicode, Inc. All rights reserved. +Distributed under the Terms of Use in http://www.unicode.org/copyright.html. + +Permission is hereby granted, free of charge, to any person obtaining +a copy of the Unicode data files and any associated documentation +(the "Data Files") or Unicode software and any associated documentation +(the "Software") to deal in the Data Files or Software +without restriction, including without limitation the rights to use, +copy, modify, merge, publish, distribute, and/or sell copies of +the Data Files or Software, and to permit persons to whom the Data Files +or Software are furnished to do so, provided that either +(a) this copyright and permission notice appear with all copies +of the Data Files or Software, or +(b) this copyright and permission notice appear in associated +Documentation. + +THE DATA FILES AND SOFTWARE ARE PROVIDED "AS IS", WITHOUT WARRANTY OF +ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT OF THIRD PARTY RIGHTS. +IN NO EVENT SHALL THE COPYRIGHT HOLDER OR HOLDERS INCLUDED IN THIS +NOTICE BE LIABLE FOR ANY CLAIM, OR ANY SPECIAL INDIRECT OR CONSEQUENTIAL +DAMAGES, OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, +DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER +TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +PERFORMANCE OF THE DATA FILES OR SOFTWARE. + +Except as contained in this notice, the name of a copyright holder +shall not be used in advertising or otherwise to promote the sale, +use or other dealings in these Data Files or Software without prior +written authorization of the copyright holder. + +--------------------- + +Third-Party Software Licenses + +This section contains third-party software notices and/or additional +terms for licensed third-party software components included within ICU +libraries. + +1. ICU License - ICU 1.8.1 to ICU 57.1 + +COPYRIGHT AND PERMISSION NOTICE + +Copyright (c) 1995-2016 International Business Machines Corporation and others +All rights reserved. + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, and/or sell copies of the Software, and to permit persons +to whom the Software is furnished to do so, provided that the above +copyright notice(s) and this permission notice appear in all copies of +the Software and that both the above copyright notice(s) and this +permission notice appear in supporting documentation. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT +OF THIRD PARTY RIGHTS. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR +HOLDERS INCLUDED IN THIS NOTICE BE LIABLE FOR ANY CLAIM, OR ANY +SPECIAL INDIRECT OR CONSEQUENTIAL DAMAGES, OR ANY DAMAGES WHATSOEVER +RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF +CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN +CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +Except as contained in this notice, the name of a copyright holder +shall not be used in advertising or otherwise to promote the sale, use +or other dealings in this Software without prior written authorization +of the copyright holder. + +All trademarks and registered trademarks mentioned herein are the +property of their respective owners. + +2. Chinese/Japanese Word Break Dictionary Data (cjdict.txt) + + # The Google Chrome software developed by Google is licensed under + # the BSD license. Other software included in this distribution is + # provided under other licenses, as set forth below. + # + # The BSD License + # http://opensource.org/licenses/bsd-license.php + # Copyright (C) 2006-2008, Google Inc. + # + # All rights reserved. + # + # Redistribution and use in source and binary forms, with or without + # modification, are permitted provided that the following conditions are met: + # + # Redistributions of source code must retain the above copyright notice, + # this list of conditions and the following disclaimer. + # Redistributions in binary form must reproduce the above + # copyright notice, this list of conditions and the following + # disclaimer in the documentation and/or other materials provided with + # the distribution. + # Neither the name of Google Inc. nor the names of its + # contributors may be used to endorse or promote products derived from + # this software without specific prior written permission. + # + # + # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND + # CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, + # INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF + # MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + # DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + # LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + # CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + # SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR + # BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + # + # + # The word list in cjdict.txt are generated by combining three word lists + # listed below with further processing for compound word breaking. The + # frequency is generated with an iterative training against Google web + # corpora. + # + # * Libtabe (Chinese) + # - https://sourceforge.net/project/?group_id=1519 + # - Its license terms and conditions are shown below. + # + # * IPADIC (Japanese) + # - http://chasen.aist-nara.ac.jp/chasen/distribution.html + # - Its license terms and conditions are shown below. + # + # ---------COPYING.libtabe ---- BEGIN-------------------- + # + # /* + # * Copyright (c) 1999 TaBE Project. + # * Copyright (c) 1999 Pai-Hsiang Hsiao. + # * All rights reserved. + # * + # * Redistribution and use in source and binary forms, with or without + # * modification, are permitted provided that the following conditions + # * are met: + # * + # * . Redistributions of source code must retain the above copyright + # * notice, this list of conditions and the following disclaimer. + # * . Redistributions in binary form must reproduce the above copyright + # * notice, this list of conditions and the following disclaimer in + # * the documentation and/or other materials provided with the + # * distribution. + # * . Neither the name of the TaBE Project nor the names of its + # * contributors may be used to endorse or promote products derived + # * from this software without specific prior written permission. + # * + # * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + # * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + # * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS + # * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + # * REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + # * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + # * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + # * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + # * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + # * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + # * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED + # * OF THE POSSIBILITY OF SUCH DAMAGE. + # */ + # + # /* + # * Copyright (c) 1999 Computer Systems and Communication Lab, + # * Institute of Information Science, Academia + # * Sinica. All rights reserved. + # * + # * Redistribution and use in source and binary forms, with or without + # * modification, are permitted provided that the following conditions + # * are met: + # * + # * . Redistributions of source code must retain the above copyright + # * notice, this list of conditions and the following disclaimer. + # * . Redistributions in binary form must reproduce the above copyright + # * notice, this list of conditions and the following disclaimer in + # * the documentation and/or other materials provided with the + # * distribution. + # * . Neither the name of the Computer Systems and Communication Lab + # * nor the names of its contributors may be used to endorse or + # * promote products derived from this software without specific + # * prior written permission. + # * + # * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + # * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + # * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS + # * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + # * REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + # * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + # * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + # * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + # * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + # * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + # * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED + # * OF THE POSSIBILITY OF SUCH DAMAGE. + # */ + # + # Copyright 1996 Chih-Hao Tsai @ Beckman Institute, + # University of Illinois + # c-tsai4@uiuc.edu http://casper.beckman.uiuc.edu/~c-tsai4 + # + # ---------------COPYING.libtabe-----END-------------------------------- + # + # + # ---------------COPYING.ipadic-----BEGIN------------------------------- + # + # Copyright 2000, 2001, 2002, 2003 Nara Institute of Science + # and Technology. All Rights Reserved. + # + # Use, reproduction, and distribution of this software is permitted. + # Any copy of this software, whether in its original form or modified, + # must include both the above copyright notice and the following + # paragraphs. + # + # Nara Institute of Science and Technology (NAIST), + # the copyright holders, disclaims all warranties with regard to this + # software, including all implied warranties of merchantability and + # fitness, in no event shall NAIST be liable for + # any special, indirect or consequential damages or any damages + # whatsoever resulting from loss of use, data or profits, whether in an + # action of contract, negligence or other tortuous action, arising out + # of or in connection with the use or performance of this software. + # + # A large portion of the dictionary entries + # originate from ICOT Free Software. The following conditions for ICOT + # Free Software applies to the current dictionary as well. + # + # Each User may also freely distribute the Program, whether in its + # original form or modified, to any third party or parties, PROVIDED + # that the provisions of Section 3 ("NO WARRANTY") will ALWAYS appear + # on, or be attached to, the Program, which is distributed substantially + # in the same form as set out herein and that such intended + # distribution, if actually made, will neither violate or otherwise + # contravene any of the laws and regulations of the countries having + # jurisdiction over the User or the intended distribution itself. + # + # NO WARRANTY + # + # The program was produced on an experimental basis in the course of the + # research and development conducted during the project and is provided + # to users as so produced on an experimental basis. Accordingly, the + # program is provided without any warranty whatsoever, whether express, + # implied, statutory or otherwise. The term "warranty" used herein + # includes, but is not limited to, any warranty of the quality, + # performance, merchantability and fitness for a particular purpose of + # the program and the nonexistence of any infringement or violation of + # any right of any third party. + # + # Each user of the program will agree and understand, and be deemed to + # have agreed and understood, that there is no warranty whatsoever for + # the program and, accordingly, the entire risk arising from or + # otherwise connected with the program is assumed by the user. + # + # Therefore, neither ICOT, the copyright holder, or any other + # organization that participated in or was otherwise related to the + # development of the program and their respective officials, directors, + # officers and other employees shall be held liable for any and all + # damages, including, without limitation, general, special, incidental + # and consequential damages, arising out of or otherwise in connection + # with the use or inability to use the program or any product, material + # or result produced or otherwise obtained by using the program, + # regardless of whether they have been advised of, or otherwise had + # knowledge of, the possibility of such damages at any time during the + # project or thereafter. Each user will be deemed to have agreed to the + # foregoing by his or her commencement of use of the program. The term + # "use" as used herein includes, but is not limited to, the use, + # modification, copying and distribution of the program and the + # production of secondary products from the program. + # + # In the case where the program, whether in its original form or + # modified, was distributed or delivered to or received by a user from + # any person, organization or entity other than ICOT, unless it makes or + # grants independently of ICOT any specific warranty to the user in + # writing, such person, organization or entity, will also be exempted + # from and not be held liable to the user for any such damages as noted + # above as far as the program is concerned. + # + # ---------------COPYING.ipadic-----END---------------------------------- + +3. Lao Word Break Dictionary Data (laodict.txt) + + # Copyright (c) 2013 International Business Machines Corporation + # and others. All Rights Reserved. + # + # Project: http://code.google.com/p/lao-dictionary/ + # Dictionary: http://lao-dictionary.googlecode.com/git/Lao-Dictionary.txt + # License: http://lao-dictionary.googlecode.com/git/Lao-Dictionary-LICENSE.txt + # (copied below) + # + # This file is derived from the above dictionary, with slight + # modifications. + # ---------------------------------------------------------------------- + # Copyright (C) 2013 Brian Eugene Wilson, Robert Martin Campbell. + # All rights reserved. + # + # Redistribution and use in source and binary forms, with or without + # modification, + # are permitted provided that the following conditions are met: + # + # + # Redistributions of source code must retain the above copyright notice, this + # list of conditions and the following disclaimer. Redistributions in + # binary form must reproduce the above copyright notice, this list of + # conditions and the following disclaimer in the documentation and/or + # other materials provided with the distribution. + # + # + # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS + # FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + # COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, + # INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + # (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + # SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + # HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + # STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED + # OF THE POSSIBILITY OF SUCH DAMAGE. + # -------------------------------------------------------------------------- + +4. Burmese Word Break Dictionary Data (burmesedict.txt) + + # Copyright (c) 2014 International Business Machines Corporation + # and others. All Rights Reserved. + # + # This list is part of a project hosted at: + # github.com/kanyawtech/myanmar-karen-word-lists + # + # -------------------------------------------------------------------------- + # Copyright (c) 2013, LeRoy Benjamin Sharon + # All rights reserved. + # + # Redistribution and use in source and binary forms, with or without + # modification, are permitted provided that the following conditions + # are met: Redistributions of source code must retain the above + # copyright notice, this list of conditions and the following + # disclaimer. Redistributions in binary form must reproduce the + # above copyright notice, this list of conditions and the following + # disclaimer in the documentation and/or other materials provided + # with the distribution. + # + # Neither the name Myanmar Karen Word Lists, nor the names of its + # contributors may be used to endorse or promote products derived + # from this software without specific prior written permission. + # + # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND + # CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, + # INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF + # MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + # DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS + # BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + # TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + # ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR + # TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF + # THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + # SUCH DAMAGE. + # -------------------------------------------------------------------------- + +5. Time Zone Database + + ICU uses the public domain data and code derived from Time Zone +Database for its time zone support. The ownership of the TZ database +is explained in BCP 175: Procedure for Maintaining the Time Zone +Database section 7. + + # 7. Database Ownership + # + # The TZ database itself is not an IETF Contribution or an IETF + # document. Rather it is a pre-existing and regularly updated work + # that is in the public domain, and is intended to remain in the + # public domain. Therefore, BCPs 78 [RFC5378] and 79 [RFC3979] do + # not apply to the TZ Database or contributions that individuals make + # to it. Should any claims be made and substantiated against the TZ + # Database, the organization that is providing the IANA + # Considerations defined in this RFC, under the memorandum of + # understanding with the IETF, currently ICANN, may act in accordance + # with all competent court orders. No ownership claims will be made + # by ICANN or the IETF Trust on the database or the code. Any person + # making a contribution to the database or code waives all rights to + # future claims in that contribution or in the TZ Database. + +6. Google double-conversion + +Copyright 2006-2011, the V8 project authors. All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following + disclaimer in the documentation and/or other materials provided + with the distribution. + * Neither the name of Google Inc. nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/third_party/icu/data/icu_conversion_data.c.gz.aa b/third_party/icu/data/icu_conversion_data.c.gz.aa new file mode 100644 index 0000000000000000000000000000000000000000..b68a2c6516f8183e805c509a9139cf63d1ee3fa5 GIT binary patch literal 177696 zcmd41g>M{A%qZ9}Cv7-sm^o>8Pt zVSC0I5SIyq4|YDA0B^#OXvhcvVhQ=20ha-ug&}Ks;A2Jba2|LK z@Q@`8!UMjYK=uLPDab1;_;Che2+12Eh1@xUAKL&gE07IG$T<(>{RaFD08Ifuvw+|U zAgF2wd^`g=Gz1^lftQ3ot49(2LR*x?^S_J0pGKL z?^D1JO~~yA@WZh%^*=eEj^O+F$6ZIr-VXTr1`LiTfjk*PE`V?U>2$I4`9IfPxC6c> zZ#|3vK%I~kz{3Wq&m${jUlH;N0nLEBAbT_5eZYGY06YU85r&*VK#ucOIwKA8TPylkgiUcpv!D1Od-L-VRtH z&yHX)ssF3AU7X=0t+p<^gaI5CzNl>UIsfL zNRedR;(PMWTjorEkG9{<+{K0ef_g87WzI}<+WHiizd?0PbYev>CRpge`^P}mMLt{Kjx}dq_;>tJ|31<3Ila% zA5LfPJGWl0aHp=84Q?8LlVM6vrS&&AdwQCl54-BNx4nKp zZ4ub_I4G=iew*;{TNM_P9V4k>G)XL?74sMt(mN2-10;F=X0Fco?`&W z`B(}*W0k{8;#Lx#*_r*;06U++)M=iuctt|+%#Cy%bH0$=9B6#)O>1q$ts zSNA_}$P3}6|4H7?GPvP>#OKQ8McBHP6SR5t^n$ElAAh5%d0KY8W|KfN=!vK1~-)pw&( zf6lQX5#H|FS5|t_Q{DlPcJ03y#N{najFg>kT$?ZUswwWM_xyU}+G{WJ(~1`~A15uO zrZIyU$iDD{s>j~2n0x-&SV}oE_mrC6knU7K0(ERib>HJ(73={Kr1obRPYR%{xeozK z@K=`^<|R_UGQXD>TkDzaRc3$JG|2YNqm4CC6&&dC4l~`;Rr0L3Jnx}-y){3x&H*rx z6>j(rD%h5O*cbMfeU}pE@&>-3cWJ7;2@EgIvpO6<^gnlGynFdSu*Vy2%WV`-VsJf7 z<>&lBktqc`%N(n6P`WE{FP~jqUy8;XN`3vblHIu!xk=u=Lm8PGe13J{IOCtnCVsku zaUF|Krp|JB+mr+Mx9ype0A)FBUA9Rx>4oPet5Qd9<|iGFL^)8W6}e%wKlnMD(!;Z~ zg{v~Q#t%HKf2p3mgRx)tVy`Ucrt&hq{JB+&IC%W&OT+A!pe>t9e6$d0&&5KRDsxM3wi=X9m!7J->RPS&M^v#(JCgIX;cV zj~*3)8FkP>;_>55jLzG%=bNU&zwq4n zn&gRvb&zdUaZ$G-e)Z;xlqgM$)b5l3>iO<8Rs<)JW5@6B|&dRG3 zD!XK86RRIFH68oh9RDND>HlMzS zL^6*vAb65&v{y6?{n)&C5IRXk&Czv1Bj%?+kU*H!YlgtTZNlbIh4y(y&*Ia!dz;)R zfd!6R(jGpCZwyvOH=^YNxur5JR zihg9qo^$U%0mvl9|1;6D?p=WWt}VLho53N|9%(AwzisvJ9`44~1tfls%yh%Ses10e zU^hAayU}y`%*OlrOfoX3UGUMqM155&+E zj=Nw}cmec~c(Zc^*1pNRVs-%n;y+Hh0kv(DXFbDhGz+@P72SBAwp=}2S=*WN2R(hd z6K6GNpUckR+eSzEleW?a-HA6)=!5RU+y5f_1J3T3UA=5OyW<}|WYgP9Cv+E%YL4!~ z^G7xRv*Ksl(F^)G0DspV|DA`2Qd8;Qxv6>ilzWem~Ij{oO%4 z5E+02X82Tpt-1xI3;@AR2mgol|Cc`$bptt37`F*q(NcS=^`kf&#pmccqp&-lTT(C> z%@BkJ#uKU05@wv2k_hh&3~d^D~O$ z&P-y{IN}gooU363_hXYNn5cd_ZqGRywDGZ~(WR)BKP2U(r&tu}(Yt%k79n6Kl6EQJ z@##z9uVekq5ZSLeca__XVo|`WO9~mbLu3o@J52V`xoU{*emNJ~zs+hnhQG)?e z5Eq0h?f2FV!>mV~r3v3ukl+KCd(uCOjLl1aFFHO&)rj2Thmw;sE(RGE>SRmR8IlUT z{gIUWU_;ldYqVSpgS#L2+235Rd8j!_P|qpqT-8YDrG{~I(lBk39L%ma*w9-P&xo!g z9}*b(1*UM+U*10mdkekv)E>aHv}wTEXZn#zDTOyFg5ROmcomBu@2`@i7f&zBk5tUm zKhI1rc?CBh+3R&Sz%!2`xfPQO*k6^hI0@wHvDV%_MM(-O9`~ynr5}ydNn#WS{V^{> zYUx})#UXD-^(^qZIz!S?wbFH|oHmg)Mkl^flo~j!Oa5H(VV3VlZn=kq!r!3w6P>g> zK(l90ZXPrFIg4^)k;5Dn9YO|8=PiTX{~RDS@4Am3r**m4TKqnBYdvd1L^uSp&+T$! z1F+0;n(p<)7r&*cttZzD>483lA=RltOH1u4%OV8T-vn*P4b`c+%$?OQ%x?HO)ha^a zN5@UyUG8}2%$y5~ zvZ4l2ck>YdDLez2+ZyLW{qio^89(ba)+?xuZ#TWS&SnVht5CIa1HYV>v>Lj!=8%+J z*>Ro{BeJ5t^~0A_CcsqAosxSNeGmE$*$#c3t7;ef zWfi0J<4(2Zk}4KrH)Wc68Gsb2zW1q$NcwSy+H{FcFDI{!H$7+iXTbGbJ12hnzaS90 zaB=!_!Za+mSod0GJ(G$`6ryE1?pz6%>s}(hV%tX@@8tTPuRHVxJ3(2@8mY zsZnrx_!q}5yEUJMl=#^mR(^x6ZG~u)9Gt6?|4w*#4bCT4Z+S!18){AThdZO(a4uxw z!W_o_1Sj37{trJfBf&+Z_OQlV~aB{d#7z6t`%rgblM)4fVcFWV;TBp^^C5C@BQg_((irgf$7e<^eY({UeiO4H0HPc zku!377W+S%_T;gES?yax7p2GtE?DVVyqiNi$1s4b_KnH^aYvPt#$Mq793ZDeX;*lo z0P?@nU=dl@BN^EQ|dfXg#c(ZsJ`cubyNy5V8Z2XXMhfN`#HU^3+ zcPD8g7y{Zpj$2@o%_MDBMG@UqI=r70q<<#L-#%V`p{-)r9lr=QKv?syX~u3D& zPWY|o&l0|1CJDxtH)z|B?vQ%pxwdwQhrZ=N=Ta81c*H2f^b1g7)JquID9~EwAb2=< zq5AUA*KUChGOac@ghN|D?xv~N54vQrIwuU4J2AdxoJczjwv{7wUxU*Vgl z_bMbm)*?LlTvS_EoWSG)C@J-%ubb)H?ExxtkN_ zO2W|u@tcj=3SQwO@NX&f;us$?6w~wT3pf(qrKO%uGmaEl$Tw+{rZQ1e(r4%~*Upn& zpg%N}L2O5iw$2<)Xin>vbr-M+q)9Ip^%gq6J0#2s0JLG15vmRkj83N{Q-i zqt{s=TuplJ3>8OmZ8=U&o-*EH+)3w{x%mn#N#jj&{o^F+UZ>l*>y83Geow^u=aQca ziJd-8>x^>#FHgq&k=#!!z8i)eR-`_$Q@IC{{PU*a3EGclv{AK%-q-4mS@a#42MQrUf&n{@07Z#z@mZ zocu6jmb$=R?<^^(=Kbem+a)lwAav4O3TX>>Y`*1wGvvVj2;(eE%MBL&@p}?HyKBirUNZKRj`#cg+k+iwxGMd zkG3WA#WZFvmEJcAxhi+Z79tCsd1(|L>%G!}fh~Zc?4OG7KB5<9KAgXyT#9q5RT-=i zsECE;*p&wu3Ov)&o{Pj8IqQ6`Tie(K#g%z^Z+xd4T1h_ydLOA%n5d)ejT-3Xf>z~r zh*%Km+1nR&GDBcDStsL+O_}7*%;E+>pcm=-g>ZO>omJCRoM&^z9DaNyFz#oY``Pv)M zpX87ng65Q4_z*1XsP1BU*mEZ<+gpInnW_hOV$XB<2C$=U>p z*jf3v(gXo@!|mcmg+E#+wFO-jtBVbeCZidE)~FKtoI;N5Dndr}8c*C~X{cZ?Cix-% z?MPeFLF0-ChtT+R*)$w(zT}1hrL6fclvw>=&+>Hy_YnRoLH!SVW$pE_qt&gpb3o2PDV9XKvSpkU%lkqEas zntNpCCG&fhn&JBSt79F(RZukJQPutsA7D63u=BN{_Ugx-FUK#L z{O2nA0{9jTJPmRH@}c4!Zs!3*tzY4#kpkP7^wq!;!PCUubt~N%u{%;`L$NDVFT)ri3RQy+0g2JU3J+im^edU1Kft@DiIAc&SR( z_G&t7FKa8`QDq+-43kw~Lsp-4Hp&t@(xzCKC2!jZR=!6N+bBRe@~8P&IcBjWiQ zCT?xHV3-h4@khw2uA2r5&8_IKKhyMV=1(f&7pP>L9z}R{C05HATwnd7d`5;bxExjw z&4fY(<))e3bLgd&{-)H6>2%j(&5_zz{8kB5UC!1B>-OKaJbo%#f;G*cHO3(abC?ZB zV6ubvGy4{DRlX@5vq`#WxkcF+EF)!_TP^);OWSU&>?I(q&!Wrh!ehlUD>6Z;hdVaZDPBrHT{5M zTO8t01D`{+E3Yg}sYN!-dO`l05SH@VLTHzN>L1e#X;fJscWT|h(xQ(fF8G0)qG$|% zcOxKK-k2&LOVe4Hhsy4iD`fBaM9>noIA|c#r04sVXefAl+BB%`vNLQ?_Vcbc$34R) zRk%mSR!U5c6Y8DPt?aL+XS6RG_3~wf#~l`6!3J=tG8$#x_HqxDsqW4;z@e0j_qHz)TNM*xI( zFN)>`7c7~MVPe2GBT?D4X^e=flq@iIa zQN)Bv>sHzb%+myPVh(z!gN`z6xB>+uc}X?-;Y&69(!5hj0lTmcKQp>~LGjcPG!G#8 zZS|jxpX9dd=8wpp8dA2njL$NN-^OlWK?ip7W8wi<{*rj12i^Z=gh$`vw zu-|k%z{i}(ubmm(Hnp=j!ucl1;Z4ZAQ?;{**%1NJQdOIzwPYXva0H!ZZ!K5^`MFkd z$;Xo_>QLuWDVY;^`=ER&^s`#WOV{TS&!6Gna7iv7~;% z%txhdnM8eDOMf*FLm{#L$HtC|}@#ZY_LS~ul=azvvrV#l`|JAq*Eu>1U zP5o=+@8E0{C9p5Fcy{y$@@vlmXLhG2dQ`Iu_(rm%`AvsC0AB1vrVDCXuz>qhx(*(T z63w)7;qVA{SNZ(HuCM8bEGhB8AIh)V&V@Obe!+{cCp5h-Lco@N?%SjdKc!>izeBGV zj9foDFnaA$w5^R;2Nx}Q-r2L=i&z#8&m)=%LbZPx60%L^>SrMmE0quum;Bi|i#ZYP zxz;X2Wnl|&TvXNP&w+RFR>S`?&ECl2Wpr2d9IBbw&%m-vTrg5-ORh?qgl6yk6vQSQ zWWqQVX++%14b_!~q^5G$l-ZW?mSqg!qt)hdg56sOq1vQr0zk7#j<|!xyaZS#MS@Pj z!&Y;(bTYR^g1r1t7V8sMB1Uusn^Y%JK?M0-5A_L4JQeosgb{{)rN5z<*Ac==It7Mq zn8E}&y>o`t>=5p8sn;bP(Ul8>ccgwwrowkONBDV@V;GidK@AlsAcT~&NPNLPMEt;9 zwW@p1aV5Jz!9Ih2$gRfBFZmtxsnNt1IzEWgo}v@}M~y3N4q@mze;I0uUz+bowD}n! zGvfEj@)*O{G&sy=m!-X({=~*JmHaYTe{euTwo={%AEN5t{Zdq@X^#79J3IaRj}qz9 z+l{)Tq`(bgGean_mk_95R*V199Ldm%PQusxCBwz4PYkGn|6G9O zVx@1e?^H-J_(qSwt0akyeoZClf0+Z5X)ml2CItOUbH7TVkSHEqG@bmk5%B>;3qxsM zq}-`4SIedBF8;!xM)*1}#FHvgLN1ZM>qn8X)8B6h;2Tv$@Z8@FSCI}vd zG5zH*+Uh|~E)>Hm-vqN1|Ab*Zu6&hq&qc3cLEk@IjEaT2$R}k8{QET1iZM@Gun=~ZY1CK^83(qBt z%vDX`5hd`h6SD_lmmmEQndn-N(H&~h5o=pvnl3}K82EF8ExQk#GOw-5BSz;0bmC~75>%S8s2dNI&$5yj(ciKi3fSF(+7H(sZr1wng z=9pQ=YVJP2e)GK)4J-3%a4Ba1Jv$^J<~DcS+=2w2Dr@oGoH((yISvD`jD%VuP z;&h_k%3$VhCkERC<%q@F^pdp`qBmZ&;WN0dPHtI-|Fxdo98~iO?~#T%0=76jLIM=b zqSTyAPjLp#wpCIMhO=9rEA+*8i5#J3G7;JANZ!#=+GiFHq{DWoF1}&jq0NCr)))#9 z8RUQ5V%vtvmj;aLOXEM2?#-g5)*=sUD{mzImptN9Yj=C;q|8b>)#r?AW`>Kr4x4Zo za0>3W^DKg$8gpfRqQb%~4EkEf9yu4EFsDBfByfk-^VjJKS5L3LCh9$TM3JyiL^qL8 z9VJt2V1BDOyMNbBt)N?=JPT}$l8}(6j;7H@)NvnBqD`@Mn?J|=veIdFYJ?n=Q`l4% zaqd+hp_^Z$OD@_5=ca32+c1!t?-$Lk>xO>Dq&c^hc7kqo*RTFNw<|x75!-|75A$EQ zsB8liXf56|YyyESX)9ryugl|!WWX0Yl^|F26m_waI6Y~1IpOXkf-RMI0SZcYiN(Cg zL76CyGqOqC}Jo z&&K->jcvIyHM1b>vlrWjXb)iwaVD_Zu%&RE)pVwPG(IE(B!6i7DAx?H(P>j*Y3d=M zUtNs5oUSqb(}it9|CfTsWr3AUAas<<{Z*=XXz9?H>xcH*T>Fol&VY!ee--HxCDQ5( zhc2}=>6E(DH_#n$M_ptgcA-gjP18|m0eU0sOwb9qQsk~~oj>w3OdF~chrS+g8rx}6Ohu%X)p zkm_;GgrAj83(hz&lF)v&f7ZFZzn)}tAyEh>bA61z_i8JcZ-wy5FYC%R z@IH|PNlT}LD`|gJpzk?;?Z7)heb;-mZG|Rn?6P|jvU+*DoHm{BRT(aIX)3q}QLH>H zKQ@VwE3UMN5Z-%dHY<`1;dpn8g=vy#6pguTV)TEP%pSx+O|mGs8je-^{GBthR0dq_>Y|PS7(`+ z-0#P(iAODYQTX}QkRGXRm=5;5{(pa)P@uc5hZS>VaQb1b*%dJ>ZcsQCPfa2a*6it? zp3G@nv}z=p;q5}*2dBdGWhg{PkA)a6W$v~LtBxs_Rn~$HVUkucoSkL1u!ebB)?Fnu zQWjGzjO>x5I{y3|WTQ>aEg)Q$5TyHok}Lw!Z1BgvoX#e{U*3|ocq${vTC1B^M)WRc zts~v{sU}@;6|#khBf+;)_5T*37Yd_s!sB2kn;i;3C4HWB)JiJ=ie>r9^eHIxW^-w= zew$54>>P{ikQFVV(al!-&KudUq#*Qs=$f6K*=Qz1_NkR(^bq(niZ_OZqHHj-8vbxc zPOhQM(1|o1@sQ=}n7XHm`d7gq3+h%S1-fOM!?hi_4*gsyRzAK&b6w73#ir0%7`JGF zzS6lINK1^r&oo40Po5ppr~;D;%2t?fh+A9LfBxuR6Z1 z&nopX$JPAwy;pV<1!6DvU#Wx$bW5y(uN@4#Y}Ox=FUIi24zlFZNf zQ}jgbOPXRhgA#SnY03da;g+y5ksL<02v`M~Z;5fix_y%{#gLoCT(;kjO_-+N+cnTp zqyywXlFLFdcJc6jOW1_V&4;BIqm-Rs6UYfC{vmep`>dD}_iPlGPn(Z{fcUF};Q-Q@ zMR`&A9KlWPa4l7Ve9)Y7~Y zcT4SSSi^=s77x)TZ7dcJJL`fG+UgY>gW1k-hw(w05R>pE9Lb~>-Q*Umq)^1Lw3H+W zxy(}#ZcJ+08jw{stI?j3QPpBrlw441Qh=~d z+q$2$N|A9!=%0|4K@-i>c++;6QdAr%pUn36l6)v)|M>2gTJl!yjUtM0V}{5O?S$NW zqttJAseJ_dl;{}iH?DMy&52{0mBthliMT~phbw2HFSg;xLALtJx1BzjW;c{mMAU19 zELH@TlL0DR2w%pT<7Sz5*orJWkTj)~Ng_C5FE5Bwut8UBacDw6{$PO{ioCt^UI`_T-l}vf8)$ZFMW^&UEZVdN}U-~;%p#sMs&u# zPP7Ki5BKhZ`NsbKgJtuY^{Ft=i*PZo1{8MzS!3@N@W|`WKp5v`L1c^rRL%! z)5GUci7aaET%;R&(*wttaKMf~`t=Y0o~`C;D?IB;N98mD9KCYlva77`(GjDH z8Batc_YzbU*DkPnM|DcNzY)ii>$_HD&R`t<9>Oo}7Im$uGN==t=lEHuN{gz+7L%B5 zfsXlW+a|@aV?fd|4uDKuNl&z;x|R2e<*(i>Q+lsxLx?2K-<_m+Xeo^y!n}H zJAz^A(l;NIodMhY3h%pcB>RwzF_oVwImncRv zwJMi_(sBm$OH3RlV!o$`h7V`s5otT)I^b&Y7ZA0stGgv;JspnKrlFuGApdwJ(=i|RxzLr_wS+@V$w2)f!q@IJ@ zsl=Fm*Hm2}}Q`w8YX7Q=xvokHc35c^eb75@!69v9@LTbGyebqXmIO z8W<9Jz!k6fbFVsPG4xI8j?cBirKtyW8Sv8j%O!yZi*}_NzjG1Iv;rr8lN|g5ubTAV zABL$_LrWow;(rYH^P=F>w94s!RYjtGuQ#aatCEJQincu8FY0W0oae_8i)30L^ z